diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..0f93c2e33be859f4619c090dd8d7246f5a008aec 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,10 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +data/datasets/visual_question_answering/generate_c_image/imagenet_c/frost/frost1.png filter=lfs diff=lfs merge=lfs -text +data/datasets/visual_question_answering/generate_c_image/robustness-master/assets/spatter.gif filter=lfs diff=lfs merge=lfs -text +data/datasets/visual_question_answering/generate_c_image/robustness-master/assets/tilt.gif filter=lfs diff=lfs merge=lfs -text +data/datasets/visual_question_answering/generate_c_image/robustness-master/assets/translate.gif filter=lfs diff=lfs merge=lfs -text +data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/create_c/frost1.png filter=lfs diff=lfs merge=lfs -text +data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/imagenet_c/imagenet_c/frost/frost1.png filter=lfs diff=lfs merge=lfs -text +new_results.png filter=lfs diff=lfs merge=lfs -text diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..63f14a5b3fc99541be6451b183a1ee0889436ace --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +results +logs +entry_model +__pycache__ +backup_codes diff --git a/1.png b/1.png new file mode 100644 index 0000000000000000000000000000000000000000..8970b2867c01ef2ce29f7d6e236a1ca949b9a254 Binary files /dev/null and b/1.png differ diff --git a/2.png b/2.png new file mode 100644 index 0000000000000000000000000000000000000000..9146b0500ce3ca805e051d98a08aac1ab12e2b76 Binary files /dev/null and b/2.png differ diff --git a/3.png b/3.png new file mode 100644 index 0000000000000000000000000000000000000000..bb95a3573abdcfd30da01e844efc2aff9f962521 Binary files /dev/null and b/3.png differ diff --git a/4.png b/4.png new file mode 100644 index 0000000000000000000000000000000000000000..0004309230a999507207c9821959167e32057956 Binary files /dev/null and b/4.png differ diff --git a/5.png b/5.png new file mode 100644 index 0000000000000000000000000000000000000000..a95253b07275e19cbcc67593cbd1df42a0c9f9cf Binary files /dev/null and b/5.png differ diff --git a/Implementation.png b/Implementation.png new file mode 100644 index 0000000000000000000000000000000000000000..d1d8d030011cdf76ec5cafafac86a8a096089755 Binary files /dev/null and b/Implementation.png differ diff --git a/README.md b/README.md index 4645d22f5a8cabd9119d9ce7eb4aca1353780599..4b261fad940fae9d12a188f5da9e6258ac439972 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,5 @@ --- -title: EdgeTA -emoji: 📊 -colorFrom: green -colorTo: red +title: EdgeFM sdk: static -pinned: false ---- - -Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference +app_file: index.html +--- \ No newline at end of file diff --git a/clip-baseline.png b/clip-baseline.png new file mode 100644 index 0000000000000000000000000000000000000000..337930c33765b92cd373e36b1cdea6ac67129fe6 Binary files /dev/null and b/clip-baseline.png differ diff --git a/clip-index.png b/clip-index.png new file mode 100644 index 0000000000000000000000000000000000000000..9b56b012d42ee8d3a44e46f204efc916c736285d Binary files /dev/null and b/clip-index.png differ diff --git a/clip-online.png b/clip-online.png new file mode 100644 index 0000000000000000000000000000000000000000..bcb4c4ca3b1b3de27b1773f25efe45bc15ed5040 Binary files /dev/null and b/clip-online.png differ diff --git a/cls_lora.png b/cls_lora.png new file mode 100644 index 0000000000000000000000000000000000000000..133fc79421e21334d79bc69c9fc81f8d96857097 Binary files /dev/null and b/cls_lora.png differ diff --git a/cls_md_w_fbs_index.png b/cls_md_w_fbs_index.png new file mode 100644 index 0000000000000000000000000000000000000000..0952bcea4cf966c1c09a73ad9a79763c0c527ff4 Binary files /dev/null and b/cls_md_w_fbs_index.png differ diff --git a/cls_md_wo_fbs.png b/cls_md_wo_fbs.png new file mode 100644 index 0000000000000000000000000000000000000000..a8c1a2051f7fc540df5f9e0d1a3e0de4786eae48 Binary files /dev/null and b/cls_md_wo_fbs.png differ diff --git a/cls_online.png b/cls_online.png new file mode 100644 index 0000000000000000000000000000000000000000..e90848627c31745e57033625c0924009cc6863f7 Binary files /dev/null and b/cls_online.png differ diff --git a/data/README.md b/data/README.md new file mode 100644 index 0000000000000000000000000000000000000000..5f0bf77adae7b78bfee102f7080bec042b32be04 --- /dev/null +++ b/data/README.md @@ -0,0 +1,94 @@ +## How to implement a dataset? + +For example, we want to implement a image classification dataset. + +1. create a file in corresponding directory, i.e. `benchmark/data/datasets/image_classification` + +2. create a class (inherited from `benchmark.data.datasets.ab_dataset.ABDataset`), e.g. `class YourDataset(ABDataset)` + +3. register your dataset with `benchmark.data.datasets.registry.dataset_register(name, classes, classes_aliases)`, which represents the name of your dataset, the classes of your dataset, and the possible aliases of the classes. Examples refer to `benchmark/data/datasets/image_classification/cifar10.py` or other files. + + Note that the order of `classes` must match the indexes. For example, `classes` of MNIST must be `['0', '1', '2', ..., '9']`, which means 0-th class is '0', 1-st class is '1', 2-nd class is '2', ...; `['1', '2', '0', ...]` is not correct because 0-th class is not '1' and 1-st class is not '2'. + + How to get `classes` of a dataset? For PyTorch built-in dataset (CIFAR10, MNIST, ...) and general dataset build by `ImageFolder`, you can initialize it (e.g. `dataset = CIFAR10(...)`) and get its classes by `dataset.classes`. + + ```python + # How to get classes in CIFAR10? + from torchvision.datasets import CIFAR10 + dataset = CIFAR10(...) + print(dataset.classes) + # copy this output to @dataset_register(classes=) + + # it's not recommended to dynamically get classes, e.g.: + # this works but runs slowly! + from torchvision.datasets import CIFAR10 as RawCIFAR10 + dataset = RawCIFAR10(...) + + @dataset_register( + name='CIFAR10', + classes=dataset.classes + ) + class CIFAR10(ABDataset): + # ... + ``` + + For object detection dataset, you can read the annotation JSON file and find `categories` information in it. + +4. implement abstract function `create_dataset(self, root_dir: str, split: str, transform: Optional[Compose], classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]])`. + + Arguments: + + - `root_dir`: the location of data + - `split`: `train / val / test` + - `transform`: preprocess function in `torchvision.transforms` + - `classes`: the same value with `dataset_register.classes` + - `ignore_classes`: **classes should be discarded. You should remove images which belong to these ignore classes.** + - `idx_map`: **map the original class index to new class index. For example, `{0: 2}` means the index of 0-th class will be 2 instead of 0. You should implement this by modifying the stored labels in the original dataset. ** + + You should do five things in this function: + + 1. if no user-defined transform is passed, you should implemented the default transform + 2. create the original dataset + 3. remove ignored classes in the original dataset if there are ignored classes + 4. map the original class index to new class index if there is index map + 5. split the original dataset to train / val / test dataset. If there's no val dataset in original dataset (e.g. DomainNetReal), you should split the original dataset to train / val / test dataset. If there's already val dataset in original dataset (e.g. CIFAR10 and ImageNet), regard the original val dataset as test dataset, and split the original train dataset into train / val dataset. Details just refer to existed files. + +Example (`benchmark/data/datasets/image_classification/cifar10.py`): + +```python +@dataset_register( + name='CIFAR10', + # means in the original CIFAR10, 0-th class is airplane, 1-st class is automobile, ... + classes=['airplane', 'automobile', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck'], + # means 'automobile' and 'car' are the same thing actually + class_aliases=[['automobile', 'car']] +) +class CIFAR10(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform: Optional[Compose], + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + # 1. if no user-defined transform is passed, you should implemented the default transform + if transform is None: + transform = cifar_like_image_train_aug() if split == 'train' else cifar_like_image_test_aug() + # 2. create the original dataset + dataset = RawCIFAR10(root_dir, split != 'test', transform=transform, download=True) + + # 3. remove ignored classes in the original dataset if there are ignored classes + dataset.targets = np.asarray(dataset.targets) + if len(ignore_classes) > 0: + for ignore_class in ignore_classes: + dataset.data = dataset.data[dataset.targets != classes.index(ignore_class)] + dataset.targets = dataset.targets[dataset.targets != classes.index(ignore_class)] + + # 4. map the original class index to new class index if there is index map + if idx_map is not None: + for ti, t in enumerate(dataset.targets): + dataset.targets[ti] = idx_map[t] + + # 5. split the original dataset to train / val / test dataset. + # there is not val dataset in CIFAR10 dataset, so we split the val dataset from the train dataset. + if split != 'test': + dataset = train_val_split(dataset, split) + return dataset +``` + +After implementing a new dataset, you can create a test file in `example` and load the dataset by `benchmark.data.dataset.get_dataset()`. Try using this dataset to ensure it works. (Example: `example/1.py`) diff --git a/data/__init__.py b/data/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..1868be9a3fd681e17c89f9246a9aa2f7b17135ad --- /dev/null +++ b/data/__init__.py @@ -0,0 +1,14 @@ +from .dataset import get_dataset +from .build.build import build_scenario_manually_v2 as build_scenario +from .dataloader import build_dataloader +from .build.scenario import IndexReturnedDataset, MergedDataset +from .datasets.ab_dataset import ABDataset +from .build.scenario import Scenario + +from .build_cl.build import build_cl_scenario +from .build_cl.scenario import Scenario as CLScenario + +from .build_gen.build import build_scenario_manually_v2 as build_gen_scenario +from .build_gen.scenario import Scenario as GenScenario + +from .datasets.dataset_split import split_dataset diff --git a/data/__pycache__/__init__.cpython-38.pyc b/data/__pycache__/__init__.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e18b57956ec21ab42be9e091e1aebdca92a05aec Binary files /dev/null and b/data/__pycache__/__init__.cpython-38.pyc differ diff --git a/data/__pycache__/dataloader.cpython-38.pyc b/data/__pycache__/dataloader.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..77366e376d17d8c94ad024f6cd13a991a5d12077 Binary files /dev/null and b/data/__pycache__/dataloader.cpython-38.pyc differ diff --git a/data/__pycache__/dataset.cpython-38.pyc b/data/__pycache__/dataset.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9696c21eb9b490c3c7a7bae1b309e9f533f5188f Binary files /dev/null and b/data/__pycache__/dataset.cpython-38.pyc differ diff --git a/data/build/__init__.py b/data/build/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/data/build/__pycache__/__init__.cpython-38.pyc b/data/build/__pycache__/__init__.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..431a26e92f7da7b806574d9435837325a3b163b0 Binary files /dev/null and b/data/build/__pycache__/__init__.cpython-38.pyc differ diff --git a/data/build/__pycache__/build.cpython-38.pyc b/data/build/__pycache__/build.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..57ead8fab58e77ded7ed02d6355e061efc03c61e Binary files /dev/null and b/data/build/__pycache__/build.cpython-38.pyc differ diff --git a/data/build/__pycache__/merge_alias.cpython-38.pyc b/data/build/__pycache__/merge_alias.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c062c77752b8196baf26419b6269ee104695eadc Binary files /dev/null and b/data/build/__pycache__/merge_alias.cpython-38.pyc differ diff --git a/data/build/__pycache__/scenario.cpython-38.pyc b/data/build/__pycache__/scenario.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..805b54d799c1051156a7a5aed68e5477188314a5 Binary files /dev/null and b/data/build/__pycache__/scenario.cpython-38.pyc differ diff --git a/data/build/build.py b/data/build/build.py new file mode 100644 index 0000000000000000000000000000000000000000..155d581b6cb99247121e8d872bbb4a3b79a3d7cf --- /dev/null +++ b/data/build/build.py @@ -0,0 +1,495 @@ +from typing import Dict, List, Optional, Type, Union +from ..datasets.ab_dataset import ABDataset +# from benchmark.data.visualize import visualize_classes_in_object_detection +# from benchmark.scenario.val_domain_shift import get_val_domain_shift_transform +from ..dataset import get_dataset +import copy +from torchvision.transforms import Compose + +from .merge_alias import merge_the_same_meaning_classes +from ..datasets.registery import static_dataset_registery + + +# some legacy aliases of variables: +# ignore_classes == discarded classes +# private_classes == unknown classes in partial / open-set / universal DA + + +def _merge_the_same_meaning_classes(classes_info_of_all_datasets): + final_classes_of_all_datasets, rename_map = merge_the_same_meaning_classes(classes_info_of_all_datasets) + return final_classes_of_all_datasets, rename_map + + +def _find_ignore_classes_when_sources_as_to_target_b(as_classes: List[List[str]], b_classes: List[str], da_mode): + thres = {'da': 3, 'partial_da': 2, 'open_set_da': 1, 'universal_da': 0}[da_mode] + + from functools import reduce + a_classes = reduce(lambda res, cur: res | set(cur), as_classes, set()) + + if set(a_classes) == set(b_classes): + # a is equal to b, normal + # 1. no ignore classes; 2. match class idx + a_ignore_classes, b_ignore_classes = [], [] + + elif set(a_classes) > set(b_classes): + # a contains b, partial + a_ignore_classes, b_ignore_classes = [], [] + if thres == 3 or thres == 1: # ignore extra classes in a + a_ignore_classes = set(a_classes) - set(b_classes) + + elif set(a_classes) < set(b_classes): + # a is contained by b, open set + a_ignore_classes, b_ignore_classes = [], [] + if thres == 3 or thres == 2: # ignore extra classes in b + b_ignore_classes = set(b_classes) - set(a_classes) + + elif len(set(a_classes) & set(b_classes)) > 0: + a_ignore_classes, b_ignore_classes = [], [] + if thres == 3: + a_ignore_classes = set(a_classes) - (set(a_classes) & set(b_classes)) + b_ignore_classes = set(b_classes) - (set(a_classes) & set(b_classes)) + elif thres == 2: + b_ignore_classes = set(b_classes) - (set(a_classes) & set(b_classes)) + elif thres == 1: + a_ignore_classes = set(a_classes) - (set(a_classes) & set(b_classes)) + + else: + return None # a has no intersection with b, none + + as_ignore_classes = [list(set(a_classes) & set(a_ignore_classes)) for a_classes in as_classes] + + return as_ignore_classes, list(b_ignore_classes) + + +def _find_private_classes_when_sources_as_to_target_b(as_classes: List[List[str]], b_classes: List[str], da_mode): + thres = {'da': 3, 'partial_da': 2, 'open_set_da': 1, 'universal_da': 0}[da_mode] + + from functools import reduce + a_classes = reduce(lambda res, cur: res | set(cur), as_classes, set()) + + if set(a_classes) == set(b_classes): + # a is equal to b, normal + # 1. no ignore classes; 2. match class idx + a_private_classes, b_private_classes = [], [] + + elif set(a_classes) > set(b_classes): + # a contains b, partial + a_private_classes, b_private_classes = [], [] + # if thres == 2 or thres == 0: # ignore extra classes in a + # a_private_classes = set(a_classes) - set(b_classes) + # if thres == 0: # ignore extra classes in a + # a_private_classes = set(a_classes) - set(b_classes) + + elif set(a_classes) < set(b_classes): + # a is contained by b, open set + a_private_classes, b_private_classes = [], [] + if thres == 1 or thres == 0: # ignore extra classes in b + b_private_classes = set(b_classes) - set(a_classes) + + elif len(set(a_classes) & set(b_classes)) > 0: + a_private_classes, b_private_classes = [], [] + if thres == 0: + # a_private_classes = set(a_classes) - (set(a_classes) & set(b_classes)) + + b_private_classes = set(b_classes) - (set(a_classes) & set(b_classes)) + elif thres == 1: + b_private_classes = set(b_classes) - (set(a_classes) & set(b_classes)) + elif thres == 2: + # a_private_classes = set(a_classes) - (set(a_classes) & set(b_classes)) + pass + + else: + return None # a has no intersection with b, none + + return list(b_private_classes) + + +class _ABDatasetMetaInfo: + def __init__(self, name, classes, task_type, object_type, class_aliases, shift_type): + self.name = name + self.classes = classes + self.class_aliases = class_aliases + self.shift_type = shift_type + self.task_type = task_type + self.object_type = object_type + + +def _get_dist_shift_type_when_source_a_to_target_b(a: _ABDatasetMetaInfo, b: _ABDatasetMetaInfo): + if b.shift_type is None: + return 'Dataset Shifts' + + if a.name in b.shift_type.keys(): + return b.shift_type[a.name] + + mid_dataset_name = list(b.shift_type.keys())[0] + mid_dataset_meta_info = _ABDatasetMetaInfo(mid_dataset_name, *static_dataset_registery[mid_dataset_name][1:]) + + return _get_dist_shift_type_when_source_a_to_target_b(a, mid_dataset_meta_info) + ' + ' + list(b.shift_type.values())[0] + + +def _handle_all_datasets_v2(source_datasets: List[_ABDatasetMetaInfo], target_datasets: List[_ABDatasetMetaInfo], da_mode): + + # 1. merge the same meaning classes + classes_info_of_all_datasets = { + d.name: (d.classes, d.class_aliases) + for d in source_datasets + target_datasets + } + final_classes_of_all_datasets, rename_map = _merge_the_same_meaning_classes(classes_info_of_all_datasets) + all_datasets_classes = copy.deepcopy(final_classes_of_all_datasets) + + # print(all_datasets_known_classes) + + # 2. find ignored classes according to DA mode + # source_datasets_ignore_classes, target_datasets_ignore_classes = {d.name: [] for d in source_datasets}, \ + # {d.name: [] for d in target_datasets} + # source_datasets_private_classes, target_datasets_private_classes = {d.name: [] for d in source_datasets}, \ + # {d.name: [] for d in target_datasets} + target_source_relationship_map = {td.name: {} for td in target_datasets} + # source_target_relationship_map = {sd.name: [] for sd in source_datasets} + + # 1. construct target_source_relationship_map + for sd in source_datasets:#sd和td使列表中每一个元素（类）的实例 + for td in target_datasets: + sc = all_datasets_classes[sd.name] + tc = all_datasets_classes[td.name] + + if len(set(sc) & set(tc)) == 0:#只保留有相似类别的源域和目标域 + continue + + target_source_relationship_map[td.name][sd.name] = _get_dist_shift_type_when_source_a_to_target_b(sd, td) + + # print(target_source_relationship_map) + # exit() + + source_datasets_ignore_classes = {} + for td_name, v1 in target_source_relationship_map.items(): + for sd_name, v2 in v1.items(): + source_datasets_ignore_classes[sd_name + '|' + td_name] = [] + target_datasets_ignore_classes = {d.name: [] for d in target_datasets} + target_datasets_private_classes = {d.name: [] for d in target_datasets} + # 保证对于每个目标域上的DA都符合给定的label shift + # 所以不同目标域就算对应同一个源域，该源域也可能不相同 + + for td_name, v1 in target_source_relationship_map.items(): + sd_names = list(v1.keys()) + + sds_classes = [all_datasets_classes[sd_name] for sd_name in sd_names] + td_classes = all_datasets_classes[td_name] + ss_ignore_classes, t_ignore_classes = _find_ignore_classes_when_sources_as_to_target_b(sds_classes, td_classes, da_mode)#根据DA方式不同产生ignore_classes + t_private_classes = _find_private_classes_when_sources_as_to_target_b(sds_classes, td_classes, da_mode) + + for sd_name, s_ignore_classes in zip(sd_names, ss_ignore_classes): + source_datasets_ignore_classes[sd_name + '|' + td_name] = s_ignore_classes + target_datasets_ignore_classes[td_name] = t_ignore_classes + target_datasets_private_classes[td_name] = t_private_classes + + source_datasets_ignore_classes = {k: sorted(set(v), key=v.index) for k, v in source_datasets_ignore_classes.items()} + target_datasets_ignore_classes = {k: sorted(set(v), key=v.index) for k, v in target_datasets_ignore_classes.items()} + target_datasets_private_classes = {k: sorted(set(v), key=v.index) for k, v in target_datasets_private_classes.items()} + + # for k, v in source_datasets_ignore_classes.items(): + # print(k, len(v)) + # print() + # for k, v in target_datasets_ignore_classes.items(): + # print(k, len(v)) + # print() + # for k, v in target_datasets_private_classes.items(): + # print(k, len(v)) + # print() + + # print(source_datasets_private_classes, target_datasets_private_classes) + # 3. reparse classes idx + # 3.1. agg all used classes + # all_used_classes = [] + # all_datasets_private_class_idx_map = {} + + # source_datasets_classes_idx_map = {} + # for td_name, v1 in target_source_relationship_map.items(): + # for sd_name, v2 in v1.items(): + # source_datasets_classes_idx_map[sd_name + '|' + td_name] = [] + # target_datasets_classes_idx_map = {} + + global_idx = 0 + all_used_classes_idx_map = {} + # all_datasets_known_classes = {d: [] for d in final_classes_of_all_datasets.keys()} + for dataset_name, classes in all_datasets_classes.items(): + if dataset_name not in target_datasets_ignore_classes.keys(): + ignore_classes = [0] * 100000 + for sn, sic in source_datasets_ignore_classes.items(): + if sn.startswith(dataset_name): + if len(sic) < len(ignore_classes): + ignore_classes = sic + else: + ignore_classes = target_datasets_ignore_classes[dataset_name] + private_classes = [] \ + if dataset_name not in target_datasets_ignore_classes.keys() else target_datasets_private_classes[dataset_name] + + for c in classes: + if c not in ignore_classes and c not in all_used_classes_idx_map.keys() and c not in private_classes: + all_used_classes_idx_map[c] = global_idx + global_idx += 1 + + # print(all_used_classes_idx_map) + + # dataset_private_class_idx_offset = 0 + target_private_class_idx = global_idx + target_datasets_private_class_idx = {d: None for d in target_datasets_private_classes.keys()} + + for dataset_name, classes in final_classes_of_all_datasets.items(): + if dataset_name not in target_datasets_private_classes.keys(): + continue + + # ignore_classes = target_datasets_ignore_classes[dataset_name] + private_classes = target_datasets_private_classes[dataset_name] + # private_classes = [] \ + # if dataset_name in source_datasets_private_classes.keys() else target_datasets_private_classes[dataset_name] + # for c in classes: + # if c not in ignore_classes and c not in all_used_classes_idx_map.keys() and c in private_classes: + # all_used_classes_idx_map[c] = global_idx + dataset_private_class_idx_offset + + if len(private_classes) > 0: + # all_datasets_private_class_idx[dataset_name] = global_idx + dataset_private_class_idx_offset + # dataset_private_class_idx_offset += 1 + # if dataset_name in source_datasets_private_classes.keys(): + # if source_private_class_idx is None: + # source_private_class_idx = global_idx if target_private_class_idx is None else target_private_class_idx + 1 + # all_datasets_private_class_idx[dataset_name] = source_private_class_idx + # else: + # if target_private_class_idx is None: + # target_private_class_idx = global_idx if source_private_class_idx is None else source_private_class_idx + 1 + # all_datasets_private_class_idx[dataset_name] = target_private_class_idx + target_datasets_private_class_idx[dataset_name] = target_private_class_idx + target_private_class_idx += 1 + + + # all_used_classes = sorted(set(all_used_classes), key=all_used_classes.index) + # all_used_classes_idx_map = {c: i for i, c in enumerate(all_used_classes)} + + # print('rename_map', rename_map) + + # 3.2 raw_class -> rename_map[raw_classes] -> all_used_classes_idx_map + all_datasets_e2e_idx_map = {} + all_datasets_e2e_class_to_idx_map = {} + + for td_name, v1 in target_source_relationship_map.items(): + sd_names = list(v1.keys()) + sds_classes = [all_datasets_classes[sd_name] for sd_name in sd_names] + td_classes = all_datasets_classes[td_name] + + for sd_name, sd_classes in zip(sd_names, sds_classes): + cur_e2e_idx_map = {} + cur_e2e_class_to_idx_map = {} + + for raw_ci, raw_c in enumerate(sd_classes): + renamed_c = raw_c if raw_c not in rename_map[dataset_name] else rename_map[dataset_name][raw_c] + + ignore_classes = source_datasets_ignore_classes[sd_name + '|' + td_name] + if renamed_c in ignore_classes: + continue + + idx = all_used_classes_idx_map[renamed_c] + + cur_e2e_idx_map[raw_ci] = idx + cur_e2e_class_to_idx_map[raw_c] = idx + + all_datasets_e2e_idx_map[sd_name + '|' + td_name] = cur_e2e_idx_map + all_datasets_e2e_class_to_idx_map[sd_name + '|' + td_name] = cur_e2e_class_to_idx_map + cur_e2e_idx_map = {} + cur_e2e_class_to_idx_map = {} + for raw_ci, raw_c in enumerate(td_classes): + renamed_c = raw_c if raw_c not in rename_map[dataset_name] else rename_map[dataset_name][raw_c] + + ignore_classes = target_datasets_ignore_classes[td_name] + if renamed_c in ignore_classes: + continue + + if renamed_c in target_datasets_private_classes[td_name]: + idx = target_datasets_private_class_idx[td_name] + else: + idx = all_used_classes_idx_map[renamed_c] + + cur_e2e_idx_map[raw_ci] = idx + cur_e2e_class_to_idx_map[raw_c] = idx + + all_datasets_e2e_idx_map[td_name] = cur_e2e_idx_map + all_datasets_e2e_class_to_idx_map[td_name] = cur_e2e_class_to_idx_map + + all_datasets_ignore_classes = {**source_datasets_ignore_classes, **target_datasets_ignore_classes} + # all_datasets_private_classes = {**source_datasets_private_classes, **target_datasets_private_classes} + + classes_idx_set = [] + for d, m in all_datasets_e2e_class_to_idx_map.items(): + classes_idx_set += list(m.values()) + classes_idx_set = set(classes_idx_set) + num_classes = len(classes_idx_set) + + return all_datasets_ignore_classes, target_datasets_private_classes, \ + all_datasets_e2e_idx_map, all_datasets_e2e_class_to_idx_map, target_datasets_private_class_idx, \ + target_source_relationship_map, rename_map, num_classes + + +def _build_scenario_info_v2( + source_datasets_name: List[str], + target_datasets_order: List[str], + da_mode: str +): + assert da_mode in ['close_set', 'partial', 'open_set', 'universal'] + da_mode = {'close_set': 'da', 'partial': 'partial_da', 'open_set': 'open_set_da', 'universal': 'universal_da'}[da_mode] + + source_datasets_meta_info = [_ABDatasetMetaInfo(d, *static_dataset_registery[d][1:]) for d in source_datasets_name]#获知对应的名字和对应属性，要添加数据集时，直接register就行 + target_datasets_meta_info = [_ABDatasetMetaInfo(d, *static_dataset_registery[d][1:]) for d in list(set(target_datasets_order))] + + all_datasets_ignore_classes, target_datasets_private_classes, \ + all_datasets_e2e_idx_map, all_datasets_e2e_class_to_idx_map, target_datasets_private_class_idx, \ + target_source_relationship_map, rename_map, num_classes \ + = _handle_all_datasets_v2(source_datasets_meta_info, target_datasets_meta_info, da_mode) + + return all_datasets_ignore_classes, target_datasets_private_classes, \ + all_datasets_e2e_idx_map, all_datasets_e2e_class_to_idx_map, target_datasets_private_class_idx, \ + target_source_relationship_map, rename_map, num_classes + + +def build_scenario_manually_v2( + source_datasets_name: List[str], + target_datasets_order: List[str], + da_mode: str, + data_dirs: Dict[str, str], + # transforms: Optional[Dict[str, Compose]] = None +): + configs = copy.deepcopy(locals())#返回当前局部变量 + + source_datasets_meta_info = [_ABDatasetMetaInfo(d, *static_dataset_registery[d][1:]) for d in source_datasets_name] + target_datasets_meta_info = [_ABDatasetMetaInfo(d, *static_dataset_registery[d][1:]) for d in list(set(target_datasets_order))] + + all_datasets_ignore_classes, target_datasets_private_classes, \ + all_datasets_e2e_idx_map, all_datasets_e2e_class_to_idx_map, target_datasets_private_class_idx, \ + target_source_relationship_map, rename_map, num_classes \ + = _build_scenario_info_v2(source_datasets_name, target_datasets_order, da_mode) + # from rich.console import Console + # console = Console(width=10000) + + # def print_obj(_o): + # # import pprint + # # s = pprint.pformat(_o, width=140, compact=True) + # console.print(_o) + + # console.print('configs:', style='bold red') + # print_obj(configs) + # console.print('renamed classes:', style='bold red') + # print_obj(rename_map) + # console.print('discarded classes:', style='bold red') + # print_obj(all_datasets_ignore_classes) + # console.print('unknown classes:', style='bold red') + # print_obj(target_datasets_private_classes) + # console.print('class to index map:', style='bold red') + # print_obj(all_datasets_e2e_class_to_idx_map) + # console.print('index map:', style='bold red') + # print_obj(all_datasets_e2e_idx_map) + # console = Console() + # # console.print('class distribution:', style='bold red') + # # class_dist = { + # # k: { + # # '#known classes': len(all_datasets_known_classes[k]), + # # '#unknown classes': len(all_datasets_private_classes[k]), + # # '#discarded classes': len(all_datasets_ignore_classes[k]) + # # } for k in all_datasets_ignore_classes.keys() + # # } + # # print_obj(class_dist) + # console.print('corresponding sources of each target:', style='bold red') + # print_obj(target_source_relationship_map) + + # return + + # res_source_datasets_map = {d: {split: get_dataset(d, data_dirs[d], split, getattr(transforms, d, None), + # all_datasets_ignore_classes[d], all_datasets_e2e_idx_map[d]) + # for split in ['train', 'val', 'test']} + # for d in source_datasets_name} + # res_target_datasets_map = {d: {'train': get_num_limited_dataset(get_dataset(d, data_dirs[d], 'test', getattr(transforms, d, None), + # all_datasets_ignore_classes[d], all_datasets_e2e_idx_map[d]), + # num_samples_in_each_target_domain), + # 'test': get_dataset(d, data_dirs[d], 'test', getattr(transforms, d, None), + # all_datasets_ignore_classes[d], all_datasets_e2e_idx_map[d]) + # } + # for d in list(set(target_datasets_order))} + + # res_source_datasets_map = {d: {split: get_dataset(d.split('|')[0], data_dirs[d.split('|')[0]], split, + # getattr(transforms, d.split('|')[0], None), + # all_datasets_ignore_classes[d], all_datasets_e2e_idx_map[d]) + # for split in ['train', 'val', 'test']} + # for d in all_datasets_ignore_classes.keys() if d.split('|')[0] in source_datasets_name} + + # from functools import reduce + # res_offline_train_source_datasets_map = {} + # res_offline_train_source_datasets_map_names = {} + + # for d in source_datasets_name: + # source_dataset_with_max_num_classes = None + + # for ed_name, ed in res_source_datasets_map.items(): + # if not ed_name.startswith(d): + # continue + + # if source_dataset_with_max_num_classes is None: + # source_dataset_with_max_num_classes = ed + # res_offline_train_source_datasets_map_names[d] = ed_name + + # if len(ed['train'].ignore_classes) < len(source_dataset_with_max_num_classes['train'].ignore_classes): + # source_dataset_with_max_num_classes = ed + # res_offline_train_source_datasets_map_names[d] = ed_name + + # res_offline_train_source_datasets_map[d] = source_dataset_with_max_num_classes + + # res_target_datasets_map = {d: {split: get_dataset(d, data_dirs[d], split, getattr(transforms, d, None), + # all_datasets_ignore_classes[d], all_datasets_e2e_idx_map[d]) + # for split in ['train', 'val', 'test']} + # for d in list(set(target_datasets_order))} + + from .scenario import Scenario, DatasetMetaInfo + + # test_scenario = Scenario( + # config=configs, + # offline_source_datasets_meta_info={ + # d: DatasetMetaInfo(d, + # {k: v for k, v in all_datasets_e2e_class_to_idx_map[res_offline_train_source_datasets_map_names[d]].items()}, + # None) + # for d in source_datasets_name + # }, + # offline_source_datasets={d: res_offline_train_source_datasets_map[d] for d in source_datasets_name}, + + # online_datasets_meta_info=[ + # ( + # {sd + '|' + d: DatasetMetaInfo(d, + # {k: v for k, v in all_datasets_e2e_class_to_idx_map[sd + '|' + d].items()}, + # None) + # for sd in target_source_relationship_map[d].keys()}, + # DatasetMetaInfo(d, + # {k: v for k, v in all_datasets_e2e_class_to_idx_map[d].items() if k not in target_datasets_private_classes[d]}, + # target_datasets_private_class_idx[d]) + # ) + # for d in target_datasets_order + # ], + # online_datasets={**res_source_datasets_map, **res_target_datasets_map}, + # target_domains_order=target_datasets_order, + # target_source_map=target_source_relationship_map, + # num_classes=num_classes + # ) + import os + os.environ['_ZQL_NUMC'] = str(num_classes) + + test_scenario = Scenario(config=configs, all_datasets_ignore_classes_map=all_datasets_ignore_classes, + all_datasets_idx_map=all_datasets_e2e_idx_map, + target_domains_order=target_datasets_order, + target_source_map=target_source_relationship_map, + all_datasets_e2e_class_to_idx_map=all_datasets_e2e_class_to_idx_map, + num_classes=num_classes) + + + return test_scenario + + +if __name__ == '__main__': + test_scenario = build_scenario_manually_v2(['CIFAR10', 'SVHN'], + ['STL10', 'MNIST', 'STL10', 'USPS', 'MNIST', 'STL10'], + 'close_set') + print(test_scenario.num_classes) + \ No newline at end of file diff --git a/data/build/merge_alias.py b/data/build/merge_alias.py new file mode 100644 index 0000000000000000000000000000000000000000..17dc56f5203a1dd311cc9c27ce310429ec1260ac --- /dev/null +++ b/data/build/merge_alias.py @@ -0,0 +1,106 @@ +from re import L +from typing import Dict, List +from collections import Counter + + +def grouping(bondlist): + # reference: https://blog.csdn.net/YnagShanwen/article/details/111344386 + groups = [] + break1 = False + while bondlist: + pair1 = bondlist.pop(0) + a = 11111 + b = 10000 + while b != a: + a = b + for atomid in pair1: + for i,pair2 in enumerate(bondlist): + if atomid in pair2: + pair1 = pair1 + pair2 + bondlist.pop(i) + if not bondlist: + break1 = True + break + if break1: + break + b = len(pair1) + groups.append(pair1) + return groups + + +def build_semantic_class_info(classes: List[str], aliases: List[List[str]]): + res = [] + for c in classes: + # print(res) + if len(aliases) == 0: + res += [[c]] + else: + find_alias = False + for alias in aliases: + if c in alias: + res += [alias] + find_alias = True + break + if not find_alias: + res += [[c]] + # print(classes, res) + return res + + +def merge_the_same_meaning_classes(classes_info_of_all_datasets): + # print(classes_info_of_all_datasets) + + semantic_classes_of_all_datasets = [] + all_aliases = [] + for classes, aliases in classes_info_of_all_datasets.values(): + all_aliases += aliases + for classes, aliases in classes_info_of_all_datasets.values(): + semantic_classes_of_all_datasets += build_semantic_class_info(classes, all_aliases) + + # print(semantic_classes_of_all_datasets) + + grouped_classes_of_all_datasets = grouping(semantic_classes_of_all_datasets)#匹配过后的数据 + + # print(grouped_classes_of_all_datasets) + + # final_grouped_classes_of_all_datasets = [Counter(c).most_common()[0][0] for c in grouped_classes_of_all_datasets] + # use most common class name; if the same common, use shortest class name! + final_grouped_classes_of_all_datasets = [] + for c in grouped_classes_of_all_datasets: + counter = Counter(c).most_common() + max_times = counter[0][1] + candidate_class_names = [] + for item, times in counter: + if times < max_times: + break + candidate_class_names += [item] + candidate_class_names.sort(key=lambda x: len(x)) + + final_grouped_classes_of_all_datasets += [candidate_class_names[0]] + res = {} + res_map = {d: {} for d in classes_info_of_all_datasets.keys()} + + for dataset_name, (classes, _) in classes_info_of_all_datasets.items(): + final_classes = [] + for c in classes: + for grouped_names, final_name in zip(grouped_classes_of_all_datasets, final_grouped_classes_of_all_datasets): + if c in grouped_names: + final_classes += [final_name] + if final_name != c: + res_map[dataset_name][c] = final_name + break + res[dataset_name] = sorted(set(final_classes), key=final_classes.index) + return res, res_map + + +if __name__ == '__main__': + cifar10_classes = ['airplane', 'automobile', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck'] + cifar10_aliases = [['automobile', 'car']] + stl10_classes = ['airplane', 'bird', 'car', 'cat', 'deer', 'dog', 'horse', 'monkey', 'ship', 'truck'] + + final_classes_of_all_datasets, rename_map = merge_the_same_meaning_classes({ + 'CIFAR10': (cifar10_classes, cifar10_aliases), + 'STL10': (stl10_classes, []) + }) + + print(final_classes_of_all_datasets, rename_map) diff --git a/data/build/scenario.py b/data/build/scenario.py new file mode 100644 index 0000000000000000000000000000000000000000..23897f0d3d4ca339dc6908595b465ae6723dbf23 --- /dev/null +++ b/data/build/scenario.py @@ -0,0 +1,466 @@ +import enum +from functools import reduce +from typing import Dict, List, Tuple +import numpy as np +import copy +from utils.common.log import logger +from ..datasets.ab_dataset import ABDataset +from ..dataloader import FastDataLoader, InfiniteDataLoader, build_dataloader +from data import get_dataset + + +class DatasetMetaInfo: + def __init__(self, name, + known_classes_name_idx_map, unknown_class_idx): + + assert unknown_class_idx not in known_classes_name_idx_map.keys() + + self.name = name + self.unknown_class_idx = unknown_class_idx + self.known_classes_name_idx_map = known_classes_name_idx_map + + @property + def num_classes(self): + return len(self.known_classes_idx) + 1 + + +class MergedDataset: + def __init__(self, datasets: List[ABDataset]): + self.datasets = datasets + self.datasets_len = [len(i) for i in self.datasets] + logger.info(f'create MergedDataset: len of datasets {self.datasets_len}') + self.datasets_cum_len = np.cumsum(self.datasets_len) + + def __getitem__(self, idx): + for i, cum_len in enumerate(self.datasets_cum_len): + if idx < cum_len: + return self.datasets[i][idx - sum(self.datasets_len[0: i])] + + def __len__(self): + return sum(self.datasets_len) + + +class IndexReturnedDataset: + def __init__(self, dataset: ABDataset): + self.dataset = dataset + + def __getitem__(self, idx): + res = self.dataset[idx] + + if isinstance(res, (tuple, list)): + return (*res, idx) + else: + return res, idx + + def __len__(self): + return len(self.dataset) + + +# class Scenario: +# def __init__(self, config, +# source_datasets_meta_info: Dict[str, DatasetMetaInfo], target_datasets_meta_info: Dict[str, DatasetMetaInfo], +# target_source_map: Dict[str, Dict[str, str]], +# target_domains_order: List[str], +# source_datasets: Dict[str, Dict[str, ABDataset]], target_datasets: Dict[str, Dict[str, ABDataset]]): + +# self.__config = config +# self.__source_datasets_meta_info = source_datasets_meta_info +# self.__target_datasets_meta_info = target_datasets_meta_info +# self.__target_source_map = target_source_map +# self.__target_domains_order = target_domains_order +# self.__source_datasets = source_datasets +# self.__target_datasets = target_datasets + +# # 1. basic +# def get_config(self): +# return copy.deepcopy(self.__config) + +# def get_task_type(self): +# return list(self.__source_datasets.values())[0]['train'].task_type + +# def get_num_classes(self): +# known_classes_idx = [] +# unknown_classes_idx = [] +# for v in self.__source_datasets_meta_info.values(): +# known_classes_idx += list(v.known_classes_name_idx_map.values()) +# unknown_classes_idx += [v.unknown_class_idx] +# for v in self.__target_datasets_meta_info.values(): +# known_classes_idx += list(v.known_classes_name_idx_map.values()) +# unknown_classes_idx += [v.unknown_class_idx] +# unknown_classes_idx = [i for i in unknown_classes_idx if i is not None] +# # print(known_classes_idx, unknown_classes_idx) +# res = len(set(known_classes_idx)), len(set(unknown_classes_idx)), len(set(known_classes_idx + unknown_classes_idx)) +# # print(res) +# assert res[0] + res[1] == res[2] +# return res + +# def build_dataloader(self, dataset: ABDataset, batch_size: int, num_workers: int, infinite: bool, shuffle_when_finite: bool): +# if infinite: +# dataloader = InfiniteDataLoader( +# dataset, None, batch_size, num_workers=num_workers) +# else: +# dataloader = FastDataLoader( +# dataset, batch_size, num_workers, shuffle=shuffle_when_finite) + +# return dataloader + +# def build_sub_dataset(self, dataset: ABDataset, indexes: List[int]): +# from ..data.datasets.dataset_split import _SplitDataset +# dataset.dataset = _SplitDataset(dataset.dataset, indexes) +# return dataset + +# def build_index_returned_dataset(self, dataset: ABDataset): +# return IndexReturnedDataset(dataset) + +# # 2. source +# def get_source_datasets_meta_info(self): +# return self.__source_datasets_meta_info + +# def get_source_datasets_name(self): +# return list(self.__source_datasets.keys()) + +# def get_merged_source_dataset(self, split): +# source_train_datasets = {n: d[split] for n, d in self.__source_datasets.items()} +# return MergedDataset(list(source_train_datasets.values())) + +# def get_source_datasets(self, split): +# source_train_datasets = {n: d[split] for n, d in self.__source_datasets.items()} +# return source_train_datasets + +# # 3. target **domain** +# # (do we need such API `get_ith_target_domain()`?) +# def get_target_domains_meta_info(self): +# return self.__source_datasets_meta_info + +# def get_target_domains_order(self): +# return self.__target_domains_order + +# def get_corr_source_datasets_name_of_target_domain(self, target_domain_name): +# return self.__target_source_map[target_domain_name] + +# def get_limited_target_train_dataset(self): +# if len(self.__target_domains_order) > 1: +# raise RuntimeError('this API is only for pass-in scenario in user-defined online DA algorithm') +# return list(self.__target_datasets.values())[0]['train'] + +# def get_target_domains_iterator(self, split): +# for target_domain_index, target_domain_name in enumerate(self.__target_domains_order): +# target_dataset = self.__target_datasets[target_domain_name] +# target_domain_meta_info = self.__target_datasets_meta_info[target_domain_name] + +# yield target_domain_index, target_domain_name, target_dataset[split], target_domain_meta_info + +# # 4. permission management +# def get_sub_scenario(self, source_datasets_name, source_splits, target_domains_order, target_splits): +# def get_split(dataset, splits): +# res = {} +# for s, d in dataset.items(): +# if s in splits: +# res[s] = d +# return res + +# return Scenario( +# config=self.__config, +# source_datasets_meta_info={k: v for k, v in self.__source_datasets_meta_info.items() if k in source_datasets_name}, +# target_datasets_meta_info={k: v for k, v in self.__target_datasets_meta_info.items() if k in target_domains_order}, +# target_source_map={k: v for k, v in self.__target_source_map.items() if k in target_domains_order}, +# target_domains_order=target_domains_order, +# source_datasets={k: get_split(v, source_splits) for k, v in self.__source_datasets.items() if k in source_datasets_name}, +# target_datasets={k: get_split(v, target_splits) for k, v in self.__target_datasets.items() if k in target_domains_order} +# ) + +# def get_only_source_sub_scenario_for_exp_tracker(self): +# return self.get_sub_scenario(self.get_source_datasets_name(), ['train', 'val', 'test'], [], []) + +# def get_only_source_sub_scenario_for_alg(self): +# return self.get_sub_scenario(self.get_source_datasets_name(), ['train'], [], []) + +# def get_one_da_sub_scenario_for_alg(self, target_domain_name): +# return self.get_sub_scenario(self.get_corr_source_datasets_name_of_target_domain(target_domain_name), +# ['train', 'val'], [target_domain_name], ['train']) + + +# class Scenario: +# def __init__(self, config, + +# offline_source_datasets_meta_info: Dict[str, DatasetMetaInfo], +# offline_source_datasets: Dict[str, ABDataset], + +# online_datasets_meta_info: List[Tuple[Dict[str, DatasetMetaInfo], DatasetMetaInfo]], +# online_datasets: Dict[str, ABDataset], +# target_domains_order: List[str], +# target_source_map: Dict[str, Dict[str, str]], + +# num_classes: int): + +# self.config = config + +# self.offline_source_datasets_meta_info = offline_source_datasets_meta_info +# self.offline_source_datasets = offline_source_datasets + +# self.online_datasets_meta_info = online_datasets_meta_info +# self.online_datasets = online_datasets + +# self.target_domains_order = target_domains_order +# self.target_source_map = target_source_map + +# self.num_classes = num_classes + +# def get_offline_source_datasets(self, split): +# return {n: d[split] for n, d in self.offline_source_datasets.items()} + +# def get_offline_source_merged_dataset(self, split): +# return MergedDataset([d[split] for d in self.offline_source_datasets.values()]) + +# def get_online_current_corresponding_source_datasets(self, domain_index, split): +# cur_target_domain_name = self.target_domains_order[domain_index] +# cur_source_datasets_name = list(self.target_source_map[cur_target_domain_name].keys()) +# cur_source_datasets = {n: self.online_datasets[n + '|' + cur_target_domain_name][split] for n in cur_source_datasets_name} +# return cur_source_datasets + +# def get_online_current_corresponding_merged_source_dataset(self, domain_index, split): +# cur_target_domain_name = self.target_domains_order[domain_index] +# cur_source_datasets_name = list(self.target_source_map[cur_target_domain_name].keys()) +# cur_source_datasets = {n: self.online_datasets[n + '|' + cur_target_domain_name][split] for n in cur_source_datasets_name} +# return MergedDataset([d for d in cur_source_datasets.values()]) + +# def get_online_current_target_dataset(self, domain_index, split): +# cur_target_domain_name = self.target_domains_order[domain_index] +# return self.online_datasets[cur_target_domain_name][split] + +# def build_dataloader(self, dataset: ABDataset, batch_size: int, num_workers: int, +# infinite: bool, shuffle_when_finite: bool, to_iterator: bool): +# if infinite: +# dataloader = InfiniteDataLoader( +# dataset, None, batch_size, num_workers=num_workers) +# else: +# dataloader = FastDataLoader( +# dataset, batch_size, num_workers, shuffle=shuffle_when_finite) + +# if to_iterator: +# dataloader = iter(dataloader) + +# return dataloader + +# def build_sub_dataset(self, dataset: ABDataset, indexes: List[int]): +# from data.datasets.dataset_split import _SplitDataset +# dataset.dataset = _SplitDataset(dataset.dataset, indexes) +# return dataset + +# def build_index_returned_dataset(self, dataset: ABDataset): +# return IndexReturnedDataset(dataset) + +# def get_config(self): +# return copy.deepcopy(self.config) + +# def get_task_type(self): +# return list(self.online_datasets.values())[0]['train'].task_type + +# def get_num_classes(self): +# return self.num_classes + + +class Scenario: + def __init__(self, config, all_datasets_ignore_classes_map, all_datasets_idx_map, target_domains_order, target_source_map, + all_datasets_e2e_class_to_idx_map, + num_classes): + self.config = config + self.all_datasets_ignore_classes_map = all_datasets_ignore_classes_map + self.all_datasets_idx_map = all_datasets_idx_map + self.target_domains_order = target_domains_order + self.target_source_map = target_source_map + self.all_datasets_e2e_class_to_idx_map = all_datasets_e2e_class_to_idx_map + self.num_classes = num_classes + self.cur_domain_index = 0 + + logger.info(f'[scenario build] # classes: {num_classes}') + logger.debug(f'[scenario build] idx map: {all_datasets_idx_map}') + + def to_json(self): + return dict( + config=self.config, all_datasets_ignore_classes_map=self.all_datasets_ignore_classes_map, + all_datasets_idx_map=self.all_datasets_idx_map, target_domains_order=self.target_domains_order, + target_source_map=self.target_source_map, + all_datasets_e2e_class_to_idx_map=self.all_datasets_e2e_class_to_idx_map, + num_classes=self.num_classes + ) + + def __str__(self): + return f'Scenario({self.to_json()})' + + def get_offline_datasets(self, transform=None): + # make source datasets which contains all unioned classes + res_offline_train_source_datasets_map = {} + + from .. import get_dataset + data_dirs = self.config['data_dirs'] + + source_datasets_name = self.config['source_datasets_name'] + res_source_datasets_map = {d: {split: get_dataset(d.split('|')[0], data_dirs[d.split('|')[0]], split, + transform, + self.all_datasets_ignore_classes_map[d], self.all_datasets_idx_map[d]) + for split in ['train', 'val', 'test']} + for d in self.all_datasets_ignore_classes_map.keys() if d.split('|')[0] in source_datasets_name} + + for source_dataset_name in self.config['source_datasets_name']: + source_datasets = [v for k, v in res_source_datasets_map.items() if source_dataset_name in k] + + # how to merge idx map? + # 35 79 97 + idx_maps = [d['train'].idx_map for d in source_datasets] + ignore_classes_list = [d['train'].ignore_classes for d in source_datasets] + + union_idx_map = {} + for idx_map in idx_maps: + for k, v in idx_map.items(): + if k not in union_idx_map: + union_idx_map[k] = v + else: + assert union_idx_map[k] == v + + union_ignore_classes = reduce(lambda res, cur: res & set(cur), ignore_classes_list, set(ignore_classes_list[0])) + assert len(union_ignore_classes) + len(union_idx_map) == len(source_datasets[0]['train'].raw_classes) + + logger.info(f'[scenario build] {source_dataset_name} has {len(union_idx_map)} classes in offline training') + + d = source_dataset_name + res_offline_train_source_datasets_map[d] = {split: get_dataset(d, data_dirs[d], split, + transform, + union_ignore_classes, union_idx_map) + for split in ['train', 'val', 'test']} + + return res_offline_train_source_datasets_map + + def get_offline_datasets_args(self): + # make source datasets which contains all unioned classes + res_offline_train_source_datasets_map = {} + + from .. import get_dataset + data_dirs = self.config['data_dirs'] + + source_datasets_name = self.config['source_datasets_name'] + res_source_datasets_map = {d: {split: get_dataset(d.split('|')[0], data_dirs[d.split('|')[0]], split, + None, + self.all_datasets_ignore_classes_map[d], self.all_datasets_idx_map[d]) + for split in ['train', 'val', 'test']} + for d in self.all_datasets_ignore_classes_map.keys() if d.split('|')[0] in source_datasets_name} + + for source_dataset_name in self.config['source_datasets_name']: + source_datasets = [v for k, v in res_source_datasets_map.items() if source_dataset_name in k] + + # how to merge idx map? + # 35 79 97 + idx_maps = [d['train'].idx_map for d in source_datasets] + ignore_classes_list = [d['train'].ignore_classes for d in source_datasets] + + union_idx_map = {} + for idx_map in idx_maps: + for k, v in idx_map.items(): + if k not in union_idx_map: + union_idx_map[k] = v + else: + assert union_idx_map[k] == v + + union_ignore_classes = reduce(lambda res, cur: res & set(cur), ignore_classes_list, set(ignore_classes_list[0])) + assert len(union_ignore_classes) + len(union_idx_map) == len(source_datasets[0]['train'].raw_classes) + + logger.info(f'[scenario build] {source_dataset_name} has {len(union_idx_map)} classes in offline training') + + d = source_dataset_name + res_offline_train_source_datasets_map[d] = {split: dict(d, data_dirs[d], split, + None, + union_ignore_classes, union_idx_map) + for split in ['train', 'val', 'test']} + + return res_offline_train_source_datasets_map + + # for d in source_datasets_name: + # source_dataset_with_max_num_classes = None + + # for ed_name, ed in res_source_datasets_map.items(): + # if not ed_name.startswith(d): + # continue + + # if source_dataset_with_max_num_classes is None: + # source_dataset_with_max_num_classes = ed + # res_offline_train_source_datasets_map_names[d] = ed_name + + # if len(ed['train'].ignore_classes) < len(source_dataset_with_max_num_classes['train'].ignore_classes): + # source_dataset_with_max_num_classes = ed + # res_offline_train_source_datasets_map_names[d] = ed_name + + # res_offline_train_source_datasets_map[d] = source_dataset_with_max_num_classes + + # return res_offline_train_source_datasets_map + + def get_online_ith_domain_datasets_args_for_inference(self, domain_index): + target_dataset_name = self.target_domains_order[domain_index] + # dataset_name: Any, root_dir: Any, split: Any, transform: Any | None = None, ignore_classes: Any = [], idx_map: Any | None = None + + if 'MM-CityscapesDet' in self.target_domains_order or 'CityscapesDet' in self.target_domains_order or 'BaiduPersonDet' in self.target_domains_order: + logger.info(f'use val split for inference test (only Det workload)') + split = 'test' + else: + split = 'train' + + return dict(dataset_name=target_dataset_name, + root_dir=self.config['data_dirs'][target_dataset_name], + split=split, + transform=None, + ignore_classes=self.all_datasets_ignore_classes_map[target_dataset_name], + idx_map=self.all_datasets_idx_map[target_dataset_name]) + + def get_online_ith_domain_datasets_args_for_training(self, domain_index): + target_dataset_name = self.target_domains_order[domain_index] + source_datasets_name = list(self.target_source_map[target_dataset_name].keys()) + + res = {} + # dataset_name: Any, root_dir: Any, split: Any, transform: Any | None = None, ignore_classes: Any = [], idx_map: Any | None = None + res[target_dataset_name] = {split: dict(dataset_name=target_dataset_name, + root_dir=self.config['data_dirs'][target_dataset_name], + split=split, + transform=None, + ignore_classes=self.all_datasets_ignore_classes_map[target_dataset_name], + idx_map=self.all_datasets_idx_map[target_dataset_name]) for split in ['train', 'val']} + for d in source_datasets_name: + res[d] = {split: dict(dataset_name=d, + root_dir=self.config['data_dirs'][d], + split=split, + transform=None, + ignore_classes=self.all_datasets_ignore_classes_map[d + '|' + target_dataset_name], + idx_map=self.all_datasets_idx_map[d + '|' + target_dataset_name]) for split in ['train', 'val']} + + return res + + def get_online_cur_domain_datasets_args_for_inference(self): + return self.get_online_ith_domain_datasets_args_for_inference(self.cur_domain_index) + + def get_online_cur_domain_datasets_args_for_training(self): + return self.get_online_ith_domain_datasets_args_for_training(self.cur_domain_index) + + def get_online_cur_domain_datasets_for_training(self, transform=None): + res = {} + datasets_args = self.get_online_ith_domain_datasets_args_for_training(self.cur_domain_index) + for dataset_name, dataset_args in datasets_args.items(): + res[dataset_name] = {} + for split, args in dataset_args.items(): + if transform is not None: + args['transform'] = transform + dataset = get_dataset(**args) + res[dataset_name][split] = dataset + return res + + def get_online_cur_domain_datasets_for_inference(self, transform=None): + datasets_args = self.get_online_ith_domain_datasets_args_for_inference(self.cur_domain_index) + if transform is not None: + datasets_args['transform'] = transform + return get_dataset(**datasets_args) + + def get_online_cur_domain_samples_for_training(self, num_samples, transform=None, collate_fn=None): + dataset = self.get_online_cur_domain_datasets_for_training(transform=transform) + dataset = dataset[self.target_domains_order[self.cur_domain_index]]['train'] + return next(iter(build_dataloader(dataset, num_samples, 0, True, None, collate_fn=collate_fn)))[0] + + def next_domain(self): + self.cur_domain_index += 1 + \ No newline at end of file diff --git a/data/build_cl/__pycache__/build.cpython-38.pyc b/data/build_cl/__pycache__/build.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f67c8ac268f4a247789d5c41c5893404e9201de3 Binary files /dev/null and b/data/build_cl/__pycache__/build.cpython-38.pyc differ diff --git a/data/build_cl/__pycache__/scenario.cpython-38.pyc b/data/build_cl/__pycache__/scenario.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d6660567ed71884e11bc5b4ad3fa819a0021c0eb Binary files /dev/null and b/data/build_cl/__pycache__/scenario.cpython-38.pyc differ diff --git a/data/build_cl/build.py b/data/build_cl/build.py new file mode 100644 index 0000000000000000000000000000000000000000..4b01ccb38728e16996a09111e42c96fee95b6c08 --- /dev/null +++ b/data/build_cl/build.py @@ -0,0 +1,161 @@ +from typing import Dict, List, Optional, Type, Union +from ..datasets.ab_dataset import ABDataset +# from benchmark.data.visualize import visualize_classes_in_object_detection +# from benchmark.scenario.val_domain_shift import get_val_domain_shift_transform +from ..dataset import get_dataset +import copy +from torchvision.transforms import Compose +from ..datasets.registery import static_dataset_registery +from ..build.scenario import Scenario as DAScenario +from copy import deepcopy +from utils.common.log import logger +import random +from .scenario import _ABDatasetMetaInfo, Scenario + + +def _check(source_datasets_meta_info: List[_ABDatasetMetaInfo], target_datasets_meta_info: List[_ABDatasetMetaInfo]): + # requirements for simplity + # 1. no same class in source datasets + + source_datasets_class = [i.classes for i in source_datasets_meta_info] + for ci1, c1 in enumerate(source_datasets_class): + for ci2, c2 in enumerate(source_datasets_class): + if ci1 == ci2: + continue + + c1_name = source_datasets_meta_info[ci1].name + c2_name = source_datasets_meta_info[ci2].name + intersection = set(c1).intersection(set(c2)) + assert len(intersection) == 0, f'{c1_name} has intersection with {c2_name}: {intersection}' + + +def build_cl_scenario( + da_scenario: DAScenario, + target_datasets_name: List[str], + num_classes_per_task: int, + max_num_tasks: int, + data_dirs, + sanity_check=False +): + config = deepcopy(locals()) + + source_datasets_idx_map = {} + source_class_idx_max = 0 + + for sd in da_scenario.config['source_datasets_name']: + da_scenario_idx_map = None + for k, v in da_scenario.all_datasets_idx_map.items(): + if k.startswith(sd): + da_scenario_idx_map = v + break + + source_datasets_idx_map[sd] = da_scenario_idx_map + source_class_idx_max = max(source_class_idx_max, max(list(da_scenario_idx_map.values()))) + + + target_class_idx_start = source_class_idx_max + 1 + + target_datasets_meta_info = [_ABDatasetMetaInfo(d, *static_dataset_registery[d][1:], None, None) for d in target_datasets_name] + + task_datasets_seq = [] + + num_tasks_per_dataset = {} + + for td_info_i, td_info in enumerate(target_datasets_meta_info): + + if td_info_i >= 1: + for _td_info_i, _td_info in enumerate(target_datasets_meta_info[0: td_info_i]): + if _td_info.name == td_info.name: + # print(111) + # class_idx_offset = sum([len(t.classes) for t in target_datasets_meta_info[0: td_info_i]]) + print(len(task_datasets_seq)) + + task_index_offset = sum([v if __i < _td_info_i else 0 for __i, v in enumerate(num_tasks_per_dataset.values())]) + + task_datasets_seq += task_datasets_seq[task_index_offset: task_index_offset + num_tasks_per_dataset[_td_info_i]] + print(len(task_datasets_seq)) + break + continue + + td_classes = td_info.classes + num_tasks_per_dataset[td_info_i] = 0 + + for ci in range(0, len(td_classes), num_classes_per_task): + task_i = ci // num_classes_per_task + task_datasets_seq += [_ABDatasetMetaInfo( + f'{td_info.name}|task-{task_i}|ci-{ci}-{ci + num_classes_per_task - 1}', + td_classes[ci: ci + num_classes_per_task], + td_info.task_type, + td_info.object_type, + td_info.class_aliases, + td_info.shift_type, + + td_classes[:ci] + td_classes[ci + num_classes_per_task: ], + {cii: cii + target_class_idx_start for cii in range(ci, ci + num_classes_per_task)} + )] + num_tasks_per_dataset[td_info_i] += 1 + + if ci + num_classes_per_task < len(td_classes) - 1: + task_datasets_seq += [_ABDatasetMetaInfo( + f'{td_info.name}-task-{task_i + 1}|ci-{ci}-{ci + num_classes_per_task - 1}', + td_classes[ci: len(td_classes)], + td_info.task_type, + td_info.object_type, + td_info.class_aliases, + td_info.shift_type, + + td_classes[:ci], + {cii: cii + target_class_idx_start for cii in range(ci, len(td_classes))} + )] + num_tasks_per_dataset[td_info_i] += 1 + + target_class_idx_start += len(td_classes) + + if len(task_datasets_seq) < max_num_tasks: + print(len(task_datasets_seq), max_num_tasks) + raise RuntimeError() + + task_datasets_seq = task_datasets_seq[0: max_num_tasks] + target_class_idx_start = max([max(list(td.idx_map.values())) + 1 for td in task_datasets_seq]) + + scenario = Scenario(config, task_datasets_seq, target_class_idx_start, source_class_idx_max + 1, data_dirs) + + if sanity_check: + selected_tasks_index = [] + for task_index, _ in enumerate(scenario.target_tasks_order): + cur_datasets = scenario.get_cur_task_train_datasets() + + if len(cur_datasets) < 300: + # empty_tasks_index += [task_index] + # while True: + # replaced_task_index = random.randint(0, task_index - 1) # ensure no random + replaced_task_index = task_index // 2 + assert replaced_task_index != task_index + while replaced_task_index in selected_tasks_index: + replaced_task_index += 1 + + task_datasets_seq[task_index] = deepcopy(task_datasets_seq[replaced_task_index]) + selected_tasks_index += [replaced_task_index] + + logger.warning(f'replace {task_index}-th task with {replaced_task_index}-th task') + + # print(task_index, [t.name for t in task_datasets_seq]) + + scenario.next_task() + + # print([t.name for t in task_datasets_seq]) + + if len(selected_tasks_index) > 0: + target_class_idx_start = max([max(list(td.idx_map.values())) + 1 for td in task_datasets_seq]) + scenario = Scenario(config, task_datasets_seq, target_class_idx_start, source_class_idx_max + 1, data_dirs) + + for task_index, _ in enumerate(scenario.target_tasks_order): + cur_datasets = scenario.get_cur_task_train_datasets() + logger.info(f'task {task_index}, len {len(cur_datasets)}') + assert len(cur_datasets) > 0 + + scenario.next_task() + + scenario = Scenario(config, task_datasets_seq, target_class_idx_start, source_class_idx_max + 1, data_dirs) + + return scenario \ No newline at end of file diff --git a/data/build_cl/scenario.py b/data/build_cl/scenario.py new file mode 100644 index 0000000000000000000000000000000000000000..84e3b5d843637d98d74f90ada6dd2644799cb1b0 --- /dev/null +++ b/data/build_cl/scenario.py @@ -0,0 +1,146 @@ +import enum +from functools import reduce +from typing import Dict, List, Tuple +import numpy as np +import copy +from utils.common.log import logger +from ..datasets.ab_dataset import ABDataset +from ..dataloader import FastDataLoader, InfiniteDataLoader, build_dataloader +from data import get_dataset, MergedDataset, Scenario as DAScenario + + +class _ABDatasetMetaInfo: + def __init__(self, name, classes, task_type, object_type, class_aliases, shift_type, ignore_classes, idx_map): + self.name = name + self.classes = classes + self.class_aliases = class_aliases + self.shift_type = shift_type + self.task_type = task_type + self.object_type = object_type + + self.ignore_classes = ignore_classes + self.idx_map = idx_map + + def __repr__(self) -> str: + return f'({self.name}, {self.classes}, {self.idx_map})' + + +class Scenario: + def __init__(self, config, target_datasets_info: List[_ABDatasetMetaInfo], num_classes: int, num_source_classes: int, data_dirs): + self.config = config + self.target_datasets_info = target_datasets_info + self.num_classes = num_classes + self.cur_task_index = 0 + self.num_source_classes = num_source_classes + self.cur_class_offset = num_source_classes + self.data_dirs = data_dirs + + self.target_tasks_order = [i.name for i in self.target_datasets_info] + self.num_tasks_to_be_learn = sum([len(i.classes) for i in target_datasets_info]) + + logger.info(f'[scenario build] # classes: {num_classes}, # tasks to be learnt: {len(target_datasets_info)}, ' + f'# classes per task: {config["num_classes_per_task"]}') + + def to_json(self): + config = copy.deepcopy(self.config) + config['da_scenario'] = config['da_scenario'].to_json() + target_datasets_info = [str(i) for i in self.target_datasets_info] + return dict( + config=config, target_datasets_info=target_datasets_info, + num_classes=self.num_classes + ) + + def __str__(self): + return f'Scenario({self.to_json()})' + + def get_cur_class_offset(self): + return self.cur_class_offset + + def get_cur_num_class(self): + return len(self.target_datasets_info[self.cur_task_index].classes) + + def get_nc_per_task(self): + return len(self.target_datasets_info[0].classes) + + def next_task(self): + self.cur_class_offset += len(self.target_datasets_info[self.cur_task_index].classes) + self.cur_task_index += 1 + + print(f'now, cur task: {self.cur_task_index}, cur_class_offset: {self.cur_class_offset}') + + def get_cur_task_datasets(self): + dataset_info = self.target_datasets_info[self.cur_task_index] + dataset_name = dataset_info.name.split('|')[0] + # print() + + # source_datasets_info = [] + + res ={ **{split: get_dataset(dataset_name=dataset_name, + root_dir=self.data_dirs[dataset_name], + split=split, + transform=None, + ignore_classes=dataset_info.ignore_classes, + idx_map=dataset_info.idx_map) for split in ['train']}, + + **{split: MergedDataset([get_dataset(dataset_name=dataset_name, + root_dir=self.data_dirs[dataset_name], + split=split, + transform=None, + ignore_classes=di.ignore_classes, + idx_map=di.idx_map) for di in self.target_datasets_info[0: self.cur_task_index + 1]]) + for split in ['val', 'test']} + } + + # if len(res['train']) < 200 or len(res['val']) < 200 or len(res['test']) < 200: + # return None + + + if len(res['train']) < 1000: + res['train'] = MergedDataset([res['train']] * 5) + logger.info('aug train dataset') + if len(res['val']) < 1000: + res['val'] = MergedDataset(res['val'].datasets * 5) + logger.info('aug val dataset') + if len(res['test']) < 1000: + res['test'] = MergedDataset(res['test'].datasets * 5) + logger.info('aug test dataset') + # da_scenario: DAScenario = self.config['da_scenario'] + # offline_datasets = da_scenario.get_offline_datasets() + + for k, v in res.items(): + logger.info(f'{k} dataset: {len(v)}') + + # new_val_datasets = [ + # *[d['val'] for d in offline_datasets.values()], + # res['val'] + # ] + # res['val'] = MergedDataset(new_val_datasets) + + # new_test_datasets = [ + # *[d['test'] for d in offline_datasets.values()], + # res['test'] + # ] + # res['test'] = MergedDataset(new_test_datasets) + + return res + + def get_cur_task_train_datasets(self): + dataset_info = self.target_datasets_info[self.cur_task_index] + dataset_name = dataset_info.name.split('|')[0] + # print() + + # source_datasets_info = [] + + res = get_dataset(dataset_name=dataset_name, + root_dir=self.data_dirs[dataset_name], + split='train', + transform=None, + ignore_classes=dataset_info.ignore_classes, + idx_map=dataset_info.idx_map) + + return res + + def get_online_cur_task_samples_for_training(self, num_samples): + dataset = self.get_cur_task_datasets() + dataset = dataset['train'] + return next(iter(build_dataloader(dataset, num_samples, 0, True, None)))[0] \ No newline at end of file diff --git a/data/build_gen/__pycache__/build.cpython-38.pyc b/data/build_gen/__pycache__/build.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2f4ac65f7e453fe62bbaf60c826a334807a37353 Binary files /dev/null and b/data/build_gen/__pycache__/build.cpython-38.pyc differ diff --git a/data/build_gen/__pycache__/merge_alias.cpython-38.pyc b/data/build_gen/__pycache__/merge_alias.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..464f320fa00b76abdad85cc88006cd1c21e8af36 Binary files /dev/null and b/data/build_gen/__pycache__/merge_alias.cpython-38.pyc differ diff --git a/data/build_gen/__pycache__/scenario.cpython-38.pyc b/data/build_gen/__pycache__/scenario.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..956ee60a15f949a41823cc2599c8398b2b46a2c2 Binary files /dev/null and b/data/build_gen/__pycache__/scenario.cpython-38.pyc differ diff --git a/data/build_gen/build.py b/data/build_gen/build.py new file mode 100644 index 0000000000000000000000000000000000000000..155d581b6cb99247121e8d872bbb4a3b79a3d7cf --- /dev/null +++ b/data/build_gen/build.py @@ -0,0 +1,495 @@ +from typing import Dict, List, Optional, Type, Union +from ..datasets.ab_dataset import ABDataset +# from benchmark.data.visualize import visualize_classes_in_object_detection +# from benchmark.scenario.val_domain_shift import get_val_domain_shift_transform +from ..dataset import get_dataset +import copy +from torchvision.transforms import Compose + +from .merge_alias import merge_the_same_meaning_classes +from ..datasets.registery import static_dataset_registery + + +# some legacy aliases of variables: +# ignore_classes == discarded classes +# private_classes == unknown classes in partial / open-set / universal DA + + +def _merge_the_same_meaning_classes(classes_info_of_all_datasets): + final_classes_of_all_datasets, rename_map = merge_the_same_meaning_classes(classes_info_of_all_datasets) + return final_classes_of_all_datasets, rename_map + + +def _find_ignore_classes_when_sources_as_to_target_b(as_classes: List[List[str]], b_classes: List[str], da_mode): + thres = {'da': 3, 'partial_da': 2, 'open_set_da': 1, 'universal_da': 0}[da_mode] + + from functools import reduce + a_classes = reduce(lambda res, cur: res | set(cur), as_classes, set()) + + if set(a_classes) == set(b_classes): + # a is equal to b, normal + # 1. no ignore classes; 2. match class idx + a_ignore_classes, b_ignore_classes = [], [] + + elif set(a_classes) > set(b_classes): + # a contains b, partial + a_ignore_classes, b_ignore_classes = [], [] + if thres == 3 or thres == 1: # ignore extra classes in a + a_ignore_classes = set(a_classes) - set(b_classes) + + elif set(a_classes) < set(b_classes): + # a is contained by b, open set + a_ignore_classes, b_ignore_classes = [], [] + if thres == 3 or thres == 2: # ignore extra classes in b + b_ignore_classes = set(b_classes) - set(a_classes) + + elif len(set(a_classes) & set(b_classes)) > 0: + a_ignore_classes, b_ignore_classes = [], [] + if thres == 3: + a_ignore_classes = set(a_classes) - (set(a_classes) & set(b_classes)) + b_ignore_classes = set(b_classes) - (set(a_classes) & set(b_classes)) + elif thres == 2: + b_ignore_classes = set(b_classes) - (set(a_classes) & set(b_classes)) + elif thres == 1: + a_ignore_classes = set(a_classes) - (set(a_classes) & set(b_classes)) + + else: + return None # a has no intersection with b, none + + as_ignore_classes = [list(set(a_classes) & set(a_ignore_classes)) for a_classes in as_classes] + + return as_ignore_classes, list(b_ignore_classes) + + +def _find_private_classes_when_sources_as_to_target_b(as_classes: List[List[str]], b_classes: List[str], da_mode): + thres = {'da': 3, 'partial_da': 2, 'open_set_da': 1, 'universal_da': 0}[da_mode] + + from functools import reduce + a_classes = reduce(lambda res, cur: res | set(cur), as_classes, set()) + + if set(a_classes) == set(b_classes): + # a is equal to b, normal + # 1. no ignore classes; 2. match class idx + a_private_classes, b_private_classes = [], [] + + elif set(a_classes) > set(b_classes): + # a contains b, partial + a_private_classes, b_private_classes = [], [] + # if thres == 2 or thres == 0: # ignore extra classes in a + # a_private_classes = set(a_classes) - set(b_classes) + # if thres == 0: # ignore extra classes in a + # a_private_classes = set(a_classes) - set(b_classes) + + elif set(a_classes) < set(b_classes): + # a is contained by b, open set + a_private_classes, b_private_classes = [], [] + if thres == 1 or thres == 0: # ignore extra classes in b + b_private_classes = set(b_classes) - set(a_classes) + + elif len(set(a_classes) & set(b_classes)) > 0: + a_private_classes, b_private_classes = [], [] + if thres == 0: + # a_private_classes = set(a_classes) - (set(a_classes) & set(b_classes)) + + b_private_classes = set(b_classes) - (set(a_classes) & set(b_classes)) + elif thres == 1: + b_private_classes = set(b_classes) - (set(a_classes) & set(b_classes)) + elif thres == 2: + # a_private_classes = set(a_classes) - (set(a_classes) & set(b_classes)) + pass + + else: + return None # a has no intersection with b, none + + return list(b_private_classes) + + +class _ABDatasetMetaInfo: + def __init__(self, name, classes, task_type, object_type, class_aliases, shift_type): + self.name = name + self.classes = classes + self.class_aliases = class_aliases + self.shift_type = shift_type + self.task_type = task_type + self.object_type = object_type + + +def _get_dist_shift_type_when_source_a_to_target_b(a: _ABDatasetMetaInfo, b: _ABDatasetMetaInfo): + if b.shift_type is None: + return 'Dataset Shifts' + + if a.name in b.shift_type.keys(): + return b.shift_type[a.name] + + mid_dataset_name = list(b.shift_type.keys())[0] + mid_dataset_meta_info = _ABDatasetMetaInfo(mid_dataset_name, *static_dataset_registery[mid_dataset_name][1:]) + + return _get_dist_shift_type_when_source_a_to_target_b(a, mid_dataset_meta_info) + ' + ' + list(b.shift_type.values())[0] + + +def _handle_all_datasets_v2(source_datasets: List[_ABDatasetMetaInfo], target_datasets: List[_ABDatasetMetaInfo], da_mode): + + # 1. merge the same meaning classes + classes_info_of_all_datasets = { + d.name: (d.classes, d.class_aliases) + for d in source_datasets + target_datasets + } + final_classes_of_all_datasets, rename_map = _merge_the_same_meaning_classes(classes_info_of_all_datasets) + all_datasets_classes = copy.deepcopy(final_classes_of_all_datasets) + + # print(all_datasets_known_classes) + + # 2. find ignored classes according to DA mode + # source_datasets_ignore_classes, target_datasets_ignore_classes = {d.name: [] for d in source_datasets}, \ + # {d.name: [] for d in target_datasets} + # source_datasets_private_classes, target_datasets_private_classes = {d.name: [] for d in source_datasets}, \ + # {d.name: [] for d in target_datasets} + target_source_relationship_map = {td.name: {} for td in target_datasets} + # source_target_relationship_map = {sd.name: [] for sd in source_datasets} + + # 1. construct target_source_relationship_map + for sd in source_datasets:#sd和td使列表中每一个元素（类）的实例 + for td in target_datasets: + sc = all_datasets_classes[sd.name] + tc = all_datasets_classes[td.name] + + if len(set(sc) & set(tc)) == 0:#只保留有相似类别的源域和目标域 + continue + + target_source_relationship_map[td.name][sd.name] = _get_dist_shift_type_when_source_a_to_target_b(sd, td) + + # print(target_source_relationship_map) + # exit() + + source_datasets_ignore_classes = {} + for td_name, v1 in target_source_relationship_map.items(): + for sd_name, v2 in v1.items(): + source_datasets_ignore_classes[sd_name + '|' + td_name] = [] + target_datasets_ignore_classes = {d.name: [] for d in target_datasets} + target_datasets_private_classes = {d.name: [] for d in target_datasets} + # 保证对于每个目标域上的DA都符合给定的label shift + # 所以不同目标域就算对应同一个源域，该源域也可能不相同 + + for td_name, v1 in target_source_relationship_map.items(): + sd_names = list(v1.keys()) + + sds_classes = [all_datasets_classes[sd_name] for sd_name in sd_names] + td_classes = all_datasets_classes[td_name] + ss_ignore_classes, t_ignore_classes = _find_ignore_classes_when_sources_as_to_target_b(sds_classes, td_classes, da_mode)#根据DA方式不同产生ignore_classes + t_private_classes = _find_private_classes_when_sources_as_to_target_b(sds_classes, td_classes, da_mode) + + for sd_name, s_ignore_classes in zip(sd_names, ss_ignore_classes): + source_datasets_ignore_classes[sd_name + '|' + td_name] = s_ignore_classes + target_datasets_ignore_classes[td_name] = t_ignore_classes + target_datasets_private_classes[td_name] = t_private_classes + + source_datasets_ignore_classes = {k: sorted(set(v), key=v.index) for k, v in source_datasets_ignore_classes.items()} + target_datasets_ignore_classes = {k: sorted(set(v), key=v.index) for k, v in target_datasets_ignore_classes.items()} + target_datasets_private_classes = {k: sorted(set(v), key=v.index) for k, v in target_datasets_private_classes.items()} + + # for k, v in source_datasets_ignore_classes.items(): + # print(k, len(v)) + # print() + # for k, v in target_datasets_ignore_classes.items(): + # print(k, len(v)) + # print() + # for k, v in target_datasets_private_classes.items(): + # print(k, len(v)) + # print() + + # print(source_datasets_private_classes, target_datasets_private_classes) + # 3. reparse classes idx + # 3.1. agg all used classes + # all_used_classes = [] + # all_datasets_private_class_idx_map = {} + + # source_datasets_classes_idx_map = {} + # for td_name, v1 in target_source_relationship_map.items(): + # for sd_name, v2 in v1.items(): + # source_datasets_classes_idx_map[sd_name + '|' + td_name] = [] + # target_datasets_classes_idx_map = {} + + global_idx = 0 + all_used_classes_idx_map = {} + # all_datasets_known_classes = {d: [] for d in final_classes_of_all_datasets.keys()} + for dataset_name, classes in all_datasets_classes.items(): + if dataset_name not in target_datasets_ignore_classes.keys(): + ignore_classes = [0] * 100000 + for sn, sic in source_datasets_ignore_classes.items(): + if sn.startswith(dataset_name): + if len(sic) < len(ignore_classes): + ignore_classes = sic + else: + ignore_classes = target_datasets_ignore_classes[dataset_name] + private_classes = [] \ + if dataset_name not in target_datasets_ignore_classes.keys() else target_datasets_private_classes[dataset_name] + + for c in classes: + if c not in ignore_classes and c not in all_used_classes_idx_map.keys() and c not in private_classes: + all_used_classes_idx_map[c] = global_idx + global_idx += 1 + + # print(all_used_classes_idx_map) + + # dataset_private_class_idx_offset = 0 + target_private_class_idx = global_idx + target_datasets_private_class_idx = {d: None for d in target_datasets_private_classes.keys()} + + for dataset_name, classes in final_classes_of_all_datasets.items(): + if dataset_name not in target_datasets_private_classes.keys(): + continue + + # ignore_classes = target_datasets_ignore_classes[dataset_name] + private_classes = target_datasets_private_classes[dataset_name] + # private_classes = [] \ + # if dataset_name in source_datasets_private_classes.keys() else target_datasets_private_classes[dataset_name] + # for c in classes: + # if c not in ignore_classes and c not in all_used_classes_idx_map.keys() and c in private_classes: + # all_used_classes_idx_map[c] = global_idx + dataset_private_class_idx_offset + + if len(private_classes) > 0: + # all_datasets_private_class_idx[dataset_name] = global_idx + dataset_private_class_idx_offset + # dataset_private_class_idx_offset += 1 + # if dataset_name in source_datasets_private_classes.keys(): + # if source_private_class_idx is None: + # source_private_class_idx = global_idx if target_private_class_idx is None else target_private_class_idx + 1 + # all_datasets_private_class_idx[dataset_name] = source_private_class_idx + # else: + # if target_private_class_idx is None: + # target_private_class_idx = global_idx if source_private_class_idx is None else source_private_class_idx + 1 + # all_datasets_private_class_idx[dataset_name] = target_private_class_idx + target_datasets_private_class_idx[dataset_name] = target_private_class_idx + target_private_class_idx += 1 + + + # all_used_classes = sorted(set(all_used_classes), key=all_used_classes.index) + # all_used_classes_idx_map = {c: i for i, c in enumerate(all_used_classes)} + + # print('rename_map', rename_map) + + # 3.2 raw_class -> rename_map[raw_classes] -> all_used_classes_idx_map + all_datasets_e2e_idx_map = {} + all_datasets_e2e_class_to_idx_map = {} + + for td_name, v1 in target_source_relationship_map.items(): + sd_names = list(v1.keys()) + sds_classes = [all_datasets_classes[sd_name] for sd_name in sd_names] + td_classes = all_datasets_classes[td_name] + + for sd_name, sd_classes in zip(sd_names, sds_classes): + cur_e2e_idx_map = {} + cur_e2e_class_to_idx_map = {} + + for raw_ci, raw_c in enumerate(sd_classes): + renamed_c = raw_c if raw_c not in rename_map[dataset_name] else rename_map[dataset_name][raw_c] + + ignore_classes = source_datasets_ignore_classes[sd_name + '|' + td_name] + if renamed_c in ignore_classes: + continue + + idx = all_used_classes_idx_map[renamed_c] + + cur_e2e_idx_map[raw_ci] = idx + cur_e2e_class_to_idx_map[raw_c] = idx + + all_datasets_e2e_idx_map[sd_name + '|' + td_name] = cur_e2e_idx_map + all_datasets_e2e_class_to_idx_map[sd_name + '|' + td_name] = cur_e2e_class_to_idx_map + cur_e2e_idx_map = {} + cur_e2e_class_to_idx_map = {} + for raw_ci, raw_c in enumerate(td_classes): + renamed_c = raw_c if raw_c not in rename_map[dataset_name] else rename_map[dataset_name][raw_c] + + ignore_classes = target_datasets_ignore_classes[td_name] + if renamed_c in ignore_classes: + continue + + if renamed_c in target_datasets_private_classes[td_name]: + idx = target_datasets_private_class_idx[td_name] + else: + idx = all_used_classes_idx_map[renamed_c] + + cur_e2e_idx_map[raw_ci] = idx + cur_e2e_class_to_idx_map[raw_c] = idx + + all_datasets_e2e_idx_map[td_name] = cur_e2e_idx_map + all_datasets_e2e_class_to_idx_map[td_name] = cur_e2e_class_to_idx_map + + all_datasets_ignore_classes = {**source_datasets_ignore_classes, **target_datasets_ignore_classes} + # all_datasets_private_classes = {**source_datasets_private_classes, **target_datasets_private_classes} + + classes_idx_set = [] + for d, m in all_datasets_e2e_class_to_idx_map.items(): + classes_idx_set += list(m.values()) + classes_idx_set = set(classes_idx_set) + num_classes = len(classes_idx_set) + + return all_datasets_ignore_classes, target_datasets_private_classes, \ + all_datasets_e2e_idx_map, all_datasets_e2e_class_to_idx_map, target_datasets_private_class_idx, \ + target_source_relationship_map, rename_map, num_classes + + +def _build_scenario_info_v2( + source_datasets_name: List[str], + target_datasets_order: List[str], + da_mode: str +): + assert da_mode in ['close_set', 'partial', 'open_set', 'universal'] + da_mode = {'close_set': 'da', 'partial': 'partial_da', 'open_set': 'open_set_da', 'universal': 'universal_da'}[da_mode] + + source_datasets_meta_info = [_ABDatasetMetaInfo(d, *static_dataset_registery[d][1:]) for d in source_datasets_name]#获知对应的名字和对应属性，要添加数据集时，直接register就行 + target_datasets_meta_info = [_ABDatasetMetaInfo(d, *static_dataset_registery[d][1:]) for d in list(set(target_datasets_order))] + + all_datasets_ignore_classes, target_datasets_private_classes, \ + all_datasets_e2e_idx_map, all_datasets_e2e_class_to_idx_map, target_datasets_private_class_idx, \ + target_source_relationship_map, rename_map, num_classes \ + = _handle_all_datasets_v2(source_datasets_meta_info, target_datasets_meta_info, da_mode) + + return all_datasets_ignore_classes, target_datasets_private_classes, \ + all_datasets_e2e_idx_map, all_datasets_e2e_class_to_idx_map, target_datasets_private_class_idx, \ + target_source_relationship_map, rename_map, num_classes + + +def build_scenario_manually_v2( + source_datasets_name: List[str], + target_datasets_order: List[str], + da_mode: str, + data_dirs: Dict[str, str], + # transforms: Optional[Dict[str, Compose]] = None +): + configs = copy.deepcopy(locals())#返回当前局部变量 + + source_datasets_meta_info = [_ABDatasetMetaInfo(d, *static_dataset_registery[d][1:]) for d in source_datasets_name] + target_datasets_meta_info = [_ABDatasetMetaInfo(d, *static_dataset_registery[d][1:]) for d in list(set(target_datasets_order))] + + all_datasets_ignore_classes, target_datasets_private_classes, \ + all_datasets_e2e_idx_map, all_datasets_e2e_class_to_idx_map, target_datasets_private_class_idx, \ + target_source_relationship_map, rename_map, num_classes \ + = _build_scenario_info_v2(source_datasets_name, target_datasets_order, da_mode) + # from rich.console import Console + # console = Console(width=10000) + + # def print_obj(_o): + # # import pprint + # # s = pprint.pformat(_o, width=140, compact=True) + # console.print(_o) + + # console.print('configs:', style='bold red') + # print_obj(configs) + # console.print('renamed classes:', style='bold red') + # print_obj(rename_map) + # console.print('discarded classes:', style='bold red') + # print_obj(all_datasets_ignore_classes) + # console.print('unknown classes:', style='bold red') + # print_obj(target_datasets_private_classes) + # console.print('class to index map:', style='bold red') + # print_obj(all_datasets_e2e_class_to_idx_map) + # console.print('index map:', style='bold red') + # print_obj(all_datasets_e2e_idx_map) + # console = Console() + # # console.print('class distribution:', style='bold red') + # # class_dist = { + # # k: { + # # '#known classes': len(all_datasets_known_classes[k]), + # # '#unknown classes': len(all_datasets_private_classes[k]), + # # '#discarded classes': len(all_datasets_ignore_classes[k]) + # # } for k in all_datasets_ignore_classes.keys() + # # } + # # print_obj(class_dist) + # console.print('corresponding sources of each target:', style='bold red') + # print_obj(target_source_relationship_map) + + # return + + # res_source_datasets_map = {d: {split: get_dataset(d, data_dirs[d], split, getattr(transforms, d, None), + # all_datasets_ignore_classes[d], all_datasets_e2e_idx_map[d]) + # for split in ['train', 'val', 'test']} + # for d in source_datasets_name} + # res_target_datasets_map = {d: {'train': get_num_limited_dataset(get_dataset(d, data_dirs[d], 'test', getattr(transforms, d, None), + # all_datasets_ignore_classes[d], all_datasets_e2e_idx_map[d]), + # num_samples_in_each_target_domain), + # 'test': get_dataset(d, data_dirs[d], 'test', getattr(transforms, d, None), + # all_datasets_ignore_classes[d], all_datasets_e2e_idx_map[d]) + # } + # for d in list(set(target_datasets_order))} + + # res_source_datasets_map = {d: {split: get_dataset(d.split('|')[0], data_dirs[d.split('|')[0]], split, + # getattr(transforms, d.split('|')[0], None), + # all_datasets_ignore_classes[d], all_datasets_e2e_idx_map[d]) + # for split in ['train', 'val', 'test']} + # for d in all_datasets_ignore_classes.keys() if d.split('|')[0] in source_datasets_name} + + # from functools import reduce + # res_offline_train_source_datasets_map = {} + # res_offline_train_source_datasets_map_names = {} + + # for d in source_datasets_name: + # source_dataset_with_max_num_classes = None + + # for ed_name, ed in res_source_datasets_map.items(): + # if not ed_name.startswith(d): + # continue + + # if source_dataset_with_max_num_classes is None: + # source_dataset_with_max_num_classes = ed + # res_offline_train_source_datasets_map_names[d] = ed_name + + # if len(ed['train'].ignore_classes) < len(source_dataset_with_max_num_classes['train'].ignore_classes): + # source_dataset_with_max_num_classes = ed + # res_offline_train_source_datasets_map_names[d] = ed_name + + # res_offline_train_source_datasets_map[d] = source_dataset_with_max_num_classes + + # res_target_datasets_map = {d: {split: get_dataset(d, data_dirs[d], split, getattr(transforms, d, None), + # all_datasets_ignore_classes[d], all_datasets_e2e_idx_map[d]) + # for split in ['train', 'val', 'test']} + # for d in list(set(target_datasets_order))} + + from .scenario import Scenario, DatasetMetaInfo + + # test_scenario = Scenario( + # config=configs, + # offline_source_datasets_meta_info={ + # d: DatasetMetaInfo(d, + # {k: v for k, v in all_datasets_e2e_class_to_idx_map[res_offline_train_source_datasets_map_names[d]].items()}, + # None) + # for d in source_datasets_name + # }, + # offline_source_datasets={d: res_offline_train_source_datasets_map[d] for d in source_datasets_name}, + + # online_datasets_meta_info=[ + # ( + # {sd + '|' + d: DatasetMetaInfo(d, + # {k: v for k, v in all_datasets_e2e_class_to_idx_map[sd + '|' + d].items()}, + # None) + # for sd in target_source_relationship_map[d].keys()}, + # DatasetMetaInfo(d, + # {k: v for k, v in all_datasets_e2e_class_to_idx_map[d].items() if k not in target_datasets_private_classes[d]}, + # target_datasets_private_class_idx[d]) + # ) + # for d in target_datasets_order + # ], + # online_datasets={**res_source_datasets_map, **res_target_datasets_map}, + # target_domains_order=target_datasets_order, + # target_source_map=target_source_relationship_map, + # num_classes=num_classes + # ) + import os + os.environ['_ZQL_NUMC'] = str(num_classes) + + test_scenario = Scenario(config=configs, all_datasets_ignore_classes_map=all_datasets_ignore_classes, + all_datasets_idx_map=all_datasets_e2e_idx_map, + target_domains_order=target_datasets_order, + target_source_map=target_source_relationship_map, + all_datasets_e2e_class_to_idx_map=all_datasets_e2e_class_to_idx_map, + num_classes=num_classes) + + + return test_scenario + + +if __name__ == '__main__': + test_scenario = build_scenario_manually_v2(['CIFAR10', 'SVHN'], + ['STL10', 'MNIST', 'STL10', 'USPS', 'MNIST', 'STL10'], + 'close_set') + print(test_scenario.num_classes) + \ No newline at end of file diff --git a/data/build_gen/merge_alias.py b/data/build_gen/merge_alias.py new file mode 100644 index 0000000000000000000000000000000000000000..17dc56f5203a1dd311cc9c27ce310429ec1260ac --- /dev/null +++ b/data/build_gen/merge_alias.py @@ -0,0 +1,106 @@ +from re import L +from typing import Dict, List +from collections import Counter + + +def grouping(bondlist): + # reference: https://blog.csdn.net/YnagShanwen/article/details/111344386 + groups = [] + break1 = False + while bondlist: + pair1 = bondlist.pop(0) + a = 11111 + b = 10000 + while b != a: + a = b + for atomid in pair1: + for i,pair2 in enumerate(bondlist): + if atomid in pair2: + pair1 = pair1 + pair2 + bondlist.pop(i) + if not bondlist: + break1 = True + break + if break1: + break + b = len(pair1) + groups.append(pair1) + return groups + + +def build_semantic_class_info(classes: List[str], aliases: List[List[str]]): + res = [] + for c in classes: + # print(res) + if len(aliases) == 0: + res += [[c]] + else: + find_alias = False + for alias in aliases: + if c in alias: + res += [alias] + find_alias = True + break + if not find_alias: + res += [[c]] + # print(classes, res) + return res + + +def merge_the_same_meaning_classes(classes_info_of_all_datasets): + # print(classes_info_of_all_datasets) + + semantic_classes_of_all_datasets = [] + all_aliases = [] + for classes, aliases in classes_info_of_all_datasets.values(): + all_aliases += aliases + for classes, aliases in classes_info_of_all_datasets.values(): + semantic_classes_of_all_datasets += build_semantic_class_info(classes, all_aliases) + + # print(semantic_classes_of_all_datasets) + + grouped_classes_of_all_datasets = grouping(semantic_classes_of_all_datasets)#匹配过后的数据 + + # print(grouped_classes_of_all_datasets) + + # final_grouped_classes_of_all_datasets = [Counter(c).most_common()[0][0] for c in grouped_classes_of_all_datasets] + # use most common class name; if the same common, use shortest class name! + final_grouped_classes_of_all_datasets = [] + for c in grouped_classes_of_all_datasets: + counter = Counter(c).most_common() + max_times = counter[0][1] + candidate_class_names = [] + for item, times in counter: + if times < max_times: + break + candidate_class_names += [item] + candidate_class_names.sort(key=lambda x: len(x)) + + final_grouped_classes_of_all_datasets += [candidate_class_names[0]] + res = {} + res_map = {d: {} for d in classes_info_of_all_datasets.keys()} + + for dataset_name, (classes, _) in classes_info_of_all_datasets.items(): + final_classes = [] + for c in classes: + for grouped_names, final_name in zip(grouped_classes_of_all_datasets, final_grouped_classes_of_all_datasets): + if c in grouped_names: + final_classes += [final_name] + if final_name != c: + res_map[dataset_name][c] = final_name + break + res[dataset_name] = sorted(set(final_classes), key=final_classes.index) + return res, res_map + + +if __name__ == '__main__': + cifar10_classes = ['airplane', 'automobile', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck'] + cifar10_aliases = [['automobile', 'car']] + stl10_classes = ['airplane', 'bird', 'car', 'cat', 'deer', 'dog', 'horse', 'monkey', 'ship', 'truck'] + + final_classes_of_all_datasets, rename_map = merge_the_same_meaning_classes({ + 'CIFAR10': (cifar10_classes, cifar10_aliases), + 'STL10': (stl10_classes, []) + }) + + print(final_classes_of_all_datasets, rename_map) diff --git a/data/build_gen/scenario.py b/data/build_gen/scenario.py new file mode 100644 index 0000000000000000000000000000000000000000..258f8f15dad8143519fdcb05184871aa95cfd0bd --- /dev/null +++ b/data/build_gen/scenario.py @@ -0,0 +1,473 @@ +import enum +from functools import reduce +from typing import Dict, List, Tuple +import numpy as np +import copy +from utils.common.log import logger +from ..datasets.ab_dataset import ABDataset +from ..datasets.dataset_split import train_val_split +from ..dataloader import FastDataLoader, InfiniteDataLoader, build_dataloader +from data import get_dataset + + +class DatasetMetaInfo: + def __init__(self, name, + known_classes_name_idx_map, unknown_class_idx): + + assert unknown_class_idx not in known_classes_name_idx_map.keys() + + self.name = name + self.unknown_class_idx = unknown_class_idx + self.known_classes_name_idx_map = known_classes_name_idx_map + + @property + def num_classes(self): + return len(self.known_classes_idx) + 1 + + +class MergedDataset: + def __init__(self, datasets: List[ABDataset]): + self.datasets = datasets + self.datasets_len = [len(i) for i in self.datasets] + logger.info(f'create MergedDataset: len of datasets {self.datasets_len}') + self.datasets_cum_len = np.cumsum(self.datasets_len) + + def __getitem__(self, idx): + for i, cum_len in enumerate(self.datasets_cum_len): + if idx < cum_len: + return self.datasets[i][idx - sum(self.datasets_len[0: i])] + + def __len__(self): + return sum(self.datasets_len) + + +class IndexReturnedDataset: + def __init__(self, dataset: ABDataset): + self.dataset = dataset + + def __getitem__(self, idx): + res = self.dataset[idx] + + if isinstance(res, (tuple, list)): + return (*res, idx) + else: + return res, idx + + def __len__(self): + return len(self.dataset) + + +# class Scenario: +# def __init__(self, config, +# source_datasets_meta_info: Dict[str, DatasetMetaInfo], target_datasets_meta_info: Dict[str, DatasetMetaInfo], +# target_source_map: Dict[str, Dict[str, str]], +# target_domains_order: List[str], +# source_datasets: Dict[str, Dict[str, ABDataset]], target_datasets: Dict[str, Dict[str, ABDataset]]): + +# self.__config = config +# self.__source_datasets_meta_info = source_datasets_meta_info +# self.__target_datasets_meta_info = target_datasets_meta_info +# self.__target_source_map = target_source_map +# self.__target_domains_order = target_domains_order +# self.__source_datasets = source_datasets +# self.__target_datasets = target_datasets + +# # 1. basic +# def get_config(self): +# return copy.deepcopy(self.__config) + +# def get_task_type(self): +# return list(self.__source_datasets.values())[0]['train'].task_type + +# def get_num_classes(self): +# known_classes_idx = [] +# unknown_classes_idx = [] +# for v in self.__source_datasets_meta_info.values(): +# known_classes_idx += list(v.known_classes_name_idx_map.values()) +# unknown_classes_idx += [v.unknown_class_idx] +# for v in self.__target_datasets_meta_info.values(): +# known_classes_idx += list(v.known_classes_name_idx_map.values()) +# unknown_classes_idx += [v.unknown_class_idx] +# unknown_classes_idx = [i for i in unknown_classes_idx if i is not None] +# # print(known_classes_idx, unknown_classes_idx) +# res = len(set(known_classes_idx)), len(set(unknown_classes_idx)), len(set(known_classes_idx + unknown_classes_idx)) +# # print(res) +# assert res[0] + res[1] == res[2] +# return res + +# def build_dataloader(self, dataset: ABDataset, batch_size: int, num_workers: int, infinite: bool, shuffle_when_finite: bool): +# if infinite: +# dataloader = InfiniteDataLoader( +# dataset, None, batch_size, num_workers=num_workers) +# else: +# dataloader = FastDataLoader( +# dataset, batch_size, num_workers, shuffle=shuffle_when_finite) + +# return dataloader + +# def build_sub_dataset(self, dataset: ABDataset, indexes: List[int]): +# from ..data.datasets.dataset_split import _SplitDataset +# dataset.dataset = _SplitDataset(dataset.dataset, indexes) +# return dataset + +# def build_index_returned_dataset(self, dataset: ABDataset): +# return IndexReturnedDataset(dataset) + +# # 2. source +# def get_source_datasets_meta_info(self): +# return self.__source_datasets_meta_info + +# def get_source_datasets_name(self): +# return list(self.__source_datasets.keys()) + +# def get_merged_source_dataset(self, split): +# source_train_datasets = {n: d[split] for n, d in self.__source_datasets.items()} +# return MergedDataset(list(source_train_datasets.values())) + +# def get_source_datasets(self, split): +# source_train_datasets = {n: d[split] for n, d in self.__source_datasets.items()} +# return source_train_datasets + +# # 3. target **domain** +# # (do we need such API `get_ith_target_domain()`?) +# def get_target_domains_meta_info(self): +# return self.__source_datasets_meta_info + +# def get_target_domains_order(self): +# return self.__target_domains_order + +# def get_corr_source_datasets_name_of_target_domain(self, target_domain_name): +# return self.__target_source_map[target_domain_name] + +# def get_limited_target_train_dataset(self): +# if len(self.__target_domains_order) > 1: +# raise RuntimeError('this API is only for pass-in scenario in user-defined online DA algorithm') +# return list(self.__target_datasets.values())[0]['train'] + +# def get_target_domains_iterator(self, split): +# for target_domain_index, target_domain_name in enumerate(self.__target_domains_order): +# target_dataset = self.__target_datasets[target_domain_name] +# target_domain_meta_info = self.__target_datasets_meta_info[target_domain_name] + +# yield target_domain_index, target_domain_name, target_dataset[split], target_domain_meta_info + +# # 4. permission management +# def get_sub_scenario(self, source_datasets_name, source_splits, target_domains_order, target_splits): +# def get_split(dataset, splits): +# res = {} +# for s, d in dataset.items(): +# if s in splits: +# res[s] = d +# return res + +# return Scenario( +# config=self.__config, +# source_datasets_meta_info={k: v for k, v in self.__source_datasets_meta_info.items() if k in source_datasets_name}, +# target_datasets_meta_info={k: v for k, v in self.__target_datasets_meta_info.items() if k in target_domains_order}, +# target_source_map={k: v for k, v in self.__target_source_map.items() if k in target_domains_order}, +# target_domains_order=target_domains_order, +# source_datasets={k: get_split(v, source_splits) for k, v in self.__source_datasets.items() if k in source_datasets_name}, +# target_datasets={k: get_split(v, target_splits) for k, v in self.__target_datasets.items() if k in target_domains_order} +# ) + +# def get_only_source_sub_scenario_for_exp_tracker(self): +# return self.get_sub_scenario(self.get_source_datasets_name(), ['train', 'val', 'test'], [], []) + +# def get_only_source_sub_scenario_for_alg(self): +# return self.get_sub_scenario(self.get_source_datasets_name(), ['train'], [], []) + +# def get_one_da_sub_scenario_for_alg(self, target_domain_name): +# return self.get_sub_scenario(self.get_corr_source_datasets_name_of_target_domain(target_domain_name), +# ['train', 'val'], [target_domain_name], ['train']) + + +# class Scenario: +# def __init__(self, config, + +# offline_source_datasets_meta_info: Dict[str, DatasetMetaInfo], +# offline_source_datasets: Dict[str, ABDataset], + +# online_datasets_meta_info: List[Tuple[Dict[str, DatasetMetaInfo], DatasetMetaInfo]], +# online_datasets: Dict[str, ABDataset], +# target_domains_order: List[str], +# target_source_map: Dict[str, Dict[str, str]], + +# num_classes: int): + +# self.config = config + +# self.offline_source_datasets_meta_info = offline_source_datasets_meta_info +# self.offline_source_datasets = offline_source_datasets + +# self.online_datasets_meta_info = online_datasets_meta_info +# self.online_datasets = online_datasets + +# self.target_domains_order = target_domains_order +# self.target_source_map = target_source_map + +# self.num_classes = num_classes + +# def get_offline_source_datasets(self, split): +# return {n: d[split] for n, d in self.offline_source_datasets.items()} + +# def get_offline_source_merged_dataset(self, split): +# return MergedDataset([d[split] for d in self.offline_source_datasets.values()]) + +# def get_online_current_corresponding_source_datasets(self, domain_index, split): +# cur_target_domain_name = self.target_domains_order[domain_index] +# cur_source_datasets_name = list(self.target_source_map[cur_target_domain_name].keys()) +# cur_source_datasets = {n: self.online_datasets[n + '|' + cur_target_domain_name][split] for n in cur_source_datasets_name} +# return cur_source_datasets + +# def get_online_current_corresponding_merged_source_dataset(self, domain_index, split): +# cur_target_domain_name = self.target_domains_order[domain_index] +# cur_source_datasets_name = list(self.target_source_map[cur_target_domain_name].keys()) +# cur_source_datasets = {n: self.online_datasets[n + '|' + cur_target_domain_name][split] for n in cur_source_datasets_name} +# return MergedDataset([d for d in cur_source_datasets.values()]) + +# def get_online_current_target_dataset(self, domain_index, split): +# cur_target_domain_name = self.target_domains_order[domain_index] +# return self.online_datasets[cur_target_domain_name][split] + +# def build_dataloader(self, dataset: ABDataset, batch_size: int, num_workers: int, +# infinite: bool, shuffle_when_finite: bool, to_iterator: bool): +# if infinite: +# dataloader = InfiniteDataLoader( +# dataset, None, batch_size, num_workers=num_workers) +# else: +# dataloader = FastDataLoader( +# dataset, batch_size, num_workers, shuffle=shuffle_when_finite) + +# if to_iterator: +# dataloader = iter(dataloader) + +# return dataloader + +# def build_sub_dataset(self, dataset: ABDataset, indexes: List[int]): +# from data.datasets.dataset_split import _SplitDataset +# dataset.dataset = _SplitDataset(dataset.dataset, indexes) +# return dataset + +# def build_index_returned_dataset(self, dataset: ABDataset): +# return IndexReturnedDataset(dataset) + +# def get_config(self): +# return copy.deepcopy(self.config) + +# def get_task_type(self): +# return list(self.online_datasets.values())[0]['train'].task_type + +# def get_num_classes(self): +# return self.num_classes + + +class Scenario: + def __init__(self, config, all_datasets_ignore_classes_map, all_datasets_idx_map, target_domains_order, target_source_map, + all_datasets_e2e_class_to_idx_map, + num_classes): + self.config = config + self.all_datasets_ignore_classes_map = all_datasets_ignore_classes_map + self.all_datasets_idx_map = all_datasets_idx_map + self.target_domains_order = target_domains_order + self.target_source_map = target_source_map + self.all_datasets_e2e_class_to_idx_map = all_datasets_e2e_class_to_idx_map + self.num_classes = num_classes + self.cur_domain_index = 0 + + logger.info(f'[scenario build] # classes: {num_classes}') + logger.debug(f'[scenario build] idx map: {all_datasets_idx_map}') + + def to_json(self): + return dict( + config=self.config, all_datasets_ignore_classes_map=self.all_datasets_ignore_classes_map, + all_datasets_idx_map=self.all_datasets_idx_map, target_domains_order=self.target_domains_order, + target_source_map=self.target_source_map, + all_datasets_e2e_class_to_idx_map=self.all_datasets_e2e_class_to_idx_map, + num_classes=self.num_classes + ) + + def __str__(self): + return f'Scenario({self.to_json()})' + + def get_offline_datasets(self, transform=None): + # make source datasets which contains all unioned classes + res_offline_train_source_datasets_map = {} + + from .. import get_dataset + data_dirs = self.config['data_dirs'] + + source_datasets_name = self.config['source_datasets_name'] + + # ori_datasets_map = {d: get_dataset(d, data_dirs[d], None, None, None, None) for d in source_datasets_name} + # res_source_datasets_map = {k: {split: train_val_split(copy.deepcopy(v), split, rate=0.97) for split in ['train', 'val']} for k, v in ori_datasets_map.items()} + # for ds in res_source_datasets_map.values(): + # for k, v in ds.items(): + # v.underlying_dataset.dataset.setSplit(k) + res_source_datasets_map = {d: {split: get_dataset(d, data_dirs[d], split, + transform, + self.all_datasets_ignore_classes_map[d], self.all_datasets_idx_map[d]) + for split in ['train', 'val', 'test']} + for d in self.all_datasets_ignore_classes_map.keys() if d in source_datasets_name} + + # for source_dataset_name in self.config['source_datasets_name']: + # source_datasets = [v for k, v in res_source_datasets_map.items() if source_dataset_name in k] + + # # how to merge idx map? + # # 35 79 97 + # idx_maps = [d['train'].idx_map for d in source_datasets] + # ignore_classes_list = [d['train'].ignore_classes for d in source_datasets] + + # union_idx_map = {} + # for idx_map in idx_maps: + # for k, v in idx_map.items(): + # if k not in union_idx_map: + # union_idx_map[k] = v + # else: + # assert union_idx_map[k] == v + + # union_ignore_classes = reduce(lambda res, cur: res & set(cur), ignore_classes_list, set(ignore_classes_list[0])) + # assert len(union_ignore_classes) + len(union_idx_map) == len(source_datasets[0]['train'].raw_classes) + + # logger.info(f'[scenario build] {source_dataset_name} has {len(union_idx_map)} classes in offline training') + + # d = source_dataset_name + # res_offline_train_source_datasets_map[d] = {split: get_dataset(d, data_dirs[d], split, + # transform, + # union_ignore_classes, union_idx_map) + # for split in ['train', 'val', 'test']} + + return res_source_datasets_map + + def get_offline_datasets_args(self): + # make source datasets which contains all unioned classes + res_offline_train_source_datasets_map = {} + + from .. import get_dataset + data_dirs = self.config['data_dirs'] + + source_datasets_name = self.config['source_datasets_name'] + res_source_datasets_map = {d: {split: get_dataset(d.split('|')[0], data_dirs[d.split('|')[0]], split, + None, + self.all_datasets_ignore_classes_map[d], self.all_datasets_idx_map[d]) + for split in ['train', 'val', 'test']} + for d in self.all_datasets_ignore_classes_map.keys() if d.split('|')[0] in source_datasets_name} + + for source_dataset_name in self.config['source_datasets_name']: + source_datasets = [v for k, v in res_source_datasets_map.items() if source_dataset_name in k] + + # how to merge idx map? + # 35 79 97 + idx_maps = [d['train'].idx_map for d in source_datasets] + ignore_classes_list = [d['train'].ignore_classes for d in source_datasets] + + union_idx_map = {} + for idx_map in idx_maps: + for k, v in idx_map.items(): + if k not in union_idx_map: + union_idx_map[k] = v + else: + assert union_idx_map[k] == v + + union_ignore_classes = reduce(lambda res, cur: res & set(cur), ignore_classes_list, set(ignore_classes_list[0])) + assert len(union_ignore_classes) + len(union_idx_map) == len(source_datasets[0]['train'].raw_classes) + + logger.info(f'[scenario build] {source_dataset_name} has {len(union_idx_map)} classes in offline training') + + d = source_dataset_name + res_offline_train_source_datasets_map[d] = {split: dict(d, data_dirs[d], split, + None, + union_ignore_classes, union_idx_map) + for split in ['train', 'val', 'test']} + + return res_offline_train_source_datasets_map + + # for d in source_datasets_name: + # source_dataset_with_max_num_classes = None + + # for ed_name, ed in res_source_datasets_map.items(): + # if not ed_name.startswith(d): + # continue + + # if source_dataset_with_max_num_classes is None: + # source_dataset_with_max_num_classes = ed + # res_offline_train_source_datasets_map_names[d] = ed_name + + # if len(ed['train'].ignore_classes) < len(source_dataset_with_max_num_classes['train'].ignore_classes): + # source_dataset_with_max_num_classes = ed + # res_offline_train_source_datasets_map_names[d] = ed_name + + # res_offline_train_source_datasets_map[d] = source_dataset_with_max_num_classes + + # return res_offline_train_source_datasets_map + + def get_online_ith_domain_datasets_args_for_inference(self, domain_index): + target_dataset_name = self.target_domains_order[domain_index] + # dataset_name: Any, root_dir: Any, split: Any, transform: Any | None = None, ignore_classes: Any = [], idx_map: Any | None = None + + if 'MM-CityscapesDet' in self.target_domains_order or 'CityscapesDet' in self.target_domains_order or 'BaiduPersonDet' in self.target_domains_order: + logger.info(f'use val split for inference test (only Det workload)') + split = 'test' + else: + split = 'train' + + return dict(dataset_name=target_dataset_name, + root_dir=self.config['data_dirs'][target_dataset_name], + split=split, + transform=None, + ignore_classes=self.all_datasets_ignore_classes_map[target_dataset_name], + idx_map=self.all_datasets_idx_map[target_dataset_name]) + + def get_online_ith_domain_datasets_args_for_training(self, domain_index): + target_dataset_name = self.target_domains_order[domain_index] + source_datasets_name = list(self.target_source_map[target_dataset_name].keys()) + + res = {} + # dataset_name: Any, root_dir: Any, split: Any, transform: Any | None = None, ignore_classes: Any = [], idx_map: Any | None = None + res[target_dataset_name] = {split: dict(dataset_name=target_dataset_name, + root_dir=self.config['data_dirs'][target_dataset_name], + split=split, + transform=None, + ignore_classes=self.all_datasets_ignore_classes_map[target_dataset_name], + idx_map=self.all_datasets_idx_map[target_dataset_name]) for split in ['train', 'val']} + for d in source_datasets_name: + res[d] = {split: dict(dataset_name=d, + root_dir=self.config['data_dirs'][d], + split=split, + transform=None, + ignore_classes=self.all_datasets_ignore_classes_map[d + '|' + target_dataset_name], + idx_map=self.all_datasets_idx_map[d + '|' + target_dataset_name]) for split in ['train', 'val']} + + return res + + def get_online_cur_domain_datasets_args_for_inference(self): + return self.get_online_ith_domain_datasets_args_for_inference(self.cur_domain_index) + + def get_online_cur_domain_datasets_args_for_training(self): + return self.get_online_ith_domain_datasets_args_for_training(self.cur_domain_index) + + def get_online_cur_domain_datasets_for_training(self, transform=None): + res = {} + datasets_args = self.get_online_ith_domain_datasets_args_for_training(self.cur_domain_index) + for dataset_name, dataset_args in datasets_args.items(): + res[dataset_name] = {} + for split, args in dataset_args.items(): + if transform is not None: + args['transform'] = transform + dataset = get_dataset(**args) + res[dataset_name][split] = dataset + return res + + def get_online_cur_domain_datasets_for_inference(self, transform=None): + datasets_args = self.get_online_ith_domain_datasets_args_for_inference(self.cur_domain_index) + if transform is not None: + datasets_args['transform'] = transform + return get_dataset(**datasets_args) + + def get_online_cur_domain_samples_for_training(self, num_samples, transform=None, collate_fn=None): + dataset = self.get_online_cur_domain_datasets_for_training(transform=transform) + dataset = dataset[self.target_domains_order[self.cur_domain_index]]['train'] + return next(iter(build_dataloader(dataset, num_samples, 0, True, None, collate_fn=collate_fn)))[0] + + def next_domain(self): + self.cur_domain_index += 1 + \ No newline at end of file diff --git a/data/convert_all_load_to_single_load.py b/data/convert_all_load_to_single_load.py new file mode 100644 index 0000000000000000000000000000000000000000..b38eb947f42ba22b16b987909fab5bef0472e602 --- /dev/null +++ b/data/convert_all_load_to_single_load.py @@ -0,0 +1,56 @@ +""" +convert load-all-images-into-memory-before-training dataset +to load-when-training-dataset + + +""" + + +from torchvision.datasets import CIFAR10, STL10, MNIST, USPS, SVHN +import os +import tqdm + + +def convert(datasets_of_split, new_dir): + img_idx = {} + + for d in datasets_of_split: + for x, y in tqdm.tqdm(d, total=len(d), dynamic_ncols=True): + # print(type(x), type(y)) + # break + # y = str(y) + if y not in img_idx: + img_idx[y] = -1 + img_idx[y] += 1 + + p = os.path.join(new_dir, f'{y:06d}', f'{img_idx[y]:06d}' + '.png') + os.makedirs(os.path.dirname(p), exist_ok=True) + + x.save(p) + + +if __name__ == '__main__': + # convert( + # [CIFAR10('/data/zql/datasets/CIFAR10', True, download=True), CIFAR10('/data/zql/datasets/CIFAR10', False, download=True)], + # '/data/zql/datasets/CIFAR10-single' + # ) + + # convert( + # [STL10('/data/zql/datasets/STL10', 'train', download=False), STL10('/data/zql/datasets/STL10', 'test', download=False)], + # '/data/zql/datasets/STL10-single' + # ) + + # convert( + # [MNIST('/data/zql/datasets/MNIST', True, download=True), MNIST('/data/zql/datasets/MNIST', False, download=True)], + # '/data/zql/datasets/MNIST-single' + # ) + + convert( + [SVHN('/data/zql/datasets/SVHN', 'train', download=True), SVHN('/data/zql/datasets/SVHN', 'test', download=True)], + '/data/zql/datasets/SVHN-single' + ) + + # convert( + # [USPS('/data/zql/datasets/USPS', True, download=False), USPS('/data/zql/datasets/USPS', False, download=False)], + # '/data/zql/datasets/USPS-single' + # ) \ No newline at end of file diff --git a/data/convert_det_dataset_to_cls.py b/data/convert_det_dataset_to_cls.py new file mode 100644 index 0000000000000000000000000000000000000000..fac1246ea2277f3a052df69741e12da5ad1c24ba --- /dev/null +++ b/data/convert_det_dataset_to_cls.py @@ -0,0 +1,55 @@ +from data import ABDataset +from utils.common.data_record import read_json, write_json +from PIL import Image +import os +from utils.common.file import ensure_dir +import numpy as np +from itertools import groupby +from skimage import morphology, measure +from PIL import Image +from scipy import misc +import tqdm +from PIL import ImageFile +ImageFile.LOAD_TRUNCATED_IMAGES = True +import shutil + + +def convert_det_dataset_to_det(coco_ann_json_path, data_dir, target_data_dir, min_img_size=224): + + coco_ann = read_json(coco_ann_json_path) + + img_id_to_path = {} + for img in coco_ann['images']: + img_id_to_path[img['id']] = os.path.join(data_dir, img['file_name']) + + classes_imgs_id_map = {} + for ann in tqdm.tqdm(coco_ann['annotations'], total=len(coco_ann['annotations']), dynamic_ncols=True): + img_id = ann['image_id'] + img_path = img_id_to_path[img_id] + img = Image.open(img_path) + + bbox = ann['bbox'] + if bbox[2] < min_img_size or bbox[3] < min_img_size: + continue + + bbox = [bbox[0], bbox[1], bbox[0] + bbox[2], bbox[1] + bbox[3]] + + class_idx = str(ann['category_id']) + if class_idx not in classes_imgs_id_map.keys(): + classes_imgs_id_map[class_idx] = 0 + target_cropped_img_path = os.path.join(target_data_dir, class_idx, + f'{classes_imgs_id_map[class_idx]}.{img_path.split(".")[-1]}') + classes_imgs_id_map[class_idx] += 1 + + ensure_dir(target_cropped_img_path) + img.crop(bbox).save(target_cropped_img_path) + + + +if __name__ == '__main__': + convert_det_dataset_to_det( + coco_ann_json_path='/data/zql/datasets/coco2017/train2017/coco_ann.json', + data_dir='/data/zql/datasets/coco2017/train2017', + target_data_dir='/data/zql/datasets/coco2017_for_cls_task', + min_img_size=224 + ) \ No newline at end of file diff --git a/data/convert_seg_dataset_to_cls.py b/data/convert_seg_dataset_to_cls.py new file mode 100644 index 0000000000000000000000000000000000000000..5a9f1ff6cf395d727fcbcf529a7acc1ce8bb006c --- /dev/null +++ b/data/convert_seg_dataset_to_cls.py @@ -0,0 +1,324 @@ +from data import ABDataset +from utils.common.data_record import read_json +from PIL import Image +import os +from utils.common.file import ensure_dir +import numpy as np +from itertools import groupby +from skimage import morphology, measure +from PIL import Image +from scipy import misc +import tqdm +from PIL import ImageFile +ImageFile.LOAD_TRUNCATED_IMAGES = True +import shutil + + +def convert_seg_dataset_to_cls(seg_imgs_path, seg_labels_path, target_cls_data_dir, ignore_classes_idx, thread_i, min_img_size=224, label_after_hook=lambda x: x): + """ + Reference: https://blog.csdn.net/lizaijinsheng/article/details/119889946 + + NOTE: + Background class should not be considered. + However, if a seg dataset has only one valid class, so that the generated cls dataset also has only one class and + the cls accuracy will be 100% forever. But we do not use the generated cls dataset alone, so it is ok. + """ + assert len(seg_imgs_path) == len(seg_labels_path) + + classes_imgs_id_map = {} + + for seg_img_path, seg_label_path in tqdm.tqdm(zip(seg_imgs_path, seg_labels_path), total=len(seg_imgs_path), + dynamic_ncols=True, leave=False, desc=f'thread {thread_i}'): + + try: + seg_img = Image.open(seg_img_path) + seg_label = Image.open(seg_label_path).convert('L') + seg_label = np.array(seg_label) + seg_label = label_after_hook(seg_label) + except Exception as e: + print(e) + print(f'file {seg_img_path} error, skip') + exit() + # seg_img = Image.open(seg_img_path) + # seg_label = Image.open(seg_label_path).convert('L') + # seg_label = np.array(seg_label) + + this_img_classes = set(seg_label.reshape(-1).tolist()) + # print(this_img_classes) + + for class_idx in this_img_classes: + if class_idx in ignore_classes_idx: + continue + + if class_idx not in classes_imgs_id_map.keys(): + classes_imgs_id_map[class_idx] = 0 + + mask = np.zeros((seg_label.shape[0], seg_label.shape[1]), dtype=np.uint8) + mask[seg_label == class_idx] = 1 + mask_without_small = morphology.remove_small_objects(mask, min_size=10, connectivity=2) + label_image = measure.label(mask_without_small) + + for region in measure.regionprops(label_image): + bbox = region.bbox # (top, left, bottom, right) + bbox = [bbox[1], bbox[0], bbox[3], bbox[2]] # (left, top, right, bottom) + + width, height = bbox[2] - bbox[0], bbox[3] - bbox[1] + if width < min_img_size or height < min_img_size: + continue + + target_cropped_img_path = os.path.join(target_cls_data_dir, str(class_idx), + f'{classes_imgs_id_map[class_idx]}.{seg_img_path.split(".")[-1]}') + ensure_dir(target_cropped_img_path) + seg_img.crop(bbox).save(target_cropped_img_path) + # print(target_cropped_img_path) + # exit() + + classes_imgs_id_map[class_idx] += 1 + + num_cls_imgs = 0 + for k, v in classes_imgs_id_map.items(): + # print(f'# class {k}: {v + 1}') + num_cls_imgs += v + # print(f'total: {num_cls_imgs}') + + return classes_imgs_id_map + + +from concurrent.futures import ThreadPoolExecutor + + + +# def convert_seg_dataset_to_cls_multi_thread(seg_imgs_path, seg_labels_path, target_cls_data_dir, ignore_classes_idx, num_threads): +# if os.path.exists(target_cls_data_dir): +# shutil.rmtree(target_cls_data_dir) + +# assert len(seg_imgs_path) == len(seg_labels_path) +# n = len(seg_imgs_path) // num_threads + +# pool = ThreadPoolExecutor(max_workers=num_threads) +# # threads = [] +# futures = [] +# for thread_i in range(num_threads): +# # thread = threading.Thread(target=convert_seg_dataset_to_cls, +# # args=(seg_imgs_path[thread_i * n: (thread_i + 1) * n], +# # seg_labels_path[thread_i * n: (thread_i + 1) * n], +# # target_cls_data_dir, ignore_classes_idx)) +# # threads += [thread] +# future = pool.submit(convert_seg_dataset_to_cls, *(seg_imgs_path[thread_i * n: (thread_i + 1) * n], +# seg_labels_path[thread_i * n: (thread_i + 1) * n], +# target_cls_data_dir, ignore_classes_idx, thread_i)) +# futures += [future] + +# futures += [ +# pool.submit(convert_seg_dataset_to_cls, *(seg_imgs_path[(thread_i + 1) * n: ], +# seg_labels_path[(thread_i + 1) * n: ], +# target_cls_data_dir, ignore_classes_idx, thread_i)) +# ] + +# for f in futures: +# f.done() + +# res = [] +# for f in futures: +# res += [f.result()] +# print(res[-1]) + +# res_dist = {} +# for r in res: +# for k, v in r.items(): +# if k in res_dist.keys(): +# res_dist[k] += v +# else: +# res_dist[k] = v + +# print('results:') +# print(res_dist) + +# pool.shutdown() + + + +import random +def random_crop_aug(target_dir): + for class_dir in os.listdir(target_dir): + class_dir = os.path.join(target_dir, class_dir) + + for img_path in os.listdir(class_dir): + img_path = os.path.join(class_dir, img_path) + + img = Image.open(img_path) + + w, h = img.width, img.height + + for ri in range(5): + img.crop( + [ + random.randint(0, w // 5), + random.randint(0, h // 5), + random.randint(w // 5 * 4, w), + random.randint(h // 5 * 4, h) + ] + ).save( + os.path.join(os.path.dirname(img_path), f'randaug_{ri}_' + os.path.basename(img_path)) + ) + # print(img_path) + # exit() + + +if __name__ == '__main__': + # SuperviselyPerson + # root_dir = '/data/zql/datasets/supervisely_person/Supervisely Person Dataset' + + # images_path, labels_path = [], [] + # for p in os.listdir(root_dir): + # if p.startswith('ds'): + # p1 = os.path.join(root_dir, p, 'img') + # images_path += [(p, os.path.join(p1, n)) for n in os.listdir(p1)] + # for dsi, img_p in images_path: + # target_p = os.path.join(root_dir, p, dsi, img_p.split('/')[-1]) + # labels_path += [target_p] + # images_path = [i[1] for i in images_path] + + # target_dir = '/data/zql/datasets/supervisely_person_for_cls_task' + # if os.path.exists(target_dir): + # shutil.rmtree(target_dir) + # convert_seg_dataset_to_cls( + # seg_imgs_path=images_path, + # seg_labels_path=labels_path, + # target_cls_data_dir=target_dir, + # ignore_classes_idx=[0, 2], + # # num_threads=8 + # thread_i=0 + # ) + + # random_crop_aug('/data/zql/datasets/supervisely_person_for_cls_task') + + + # GTA5 + # root_dir = '/data/zql/datasets/GTA-ls-copy/GTA5' + # images_path, labels_path = [], [] + # for p in os.listdir(os.path.join(root_dir, 'images')): + # p = os.path.join(root_dir, 'images', p) + # if not p.endswith('png'): + # continue + # images_path += [p] + # labels_path += [p.replace('images', 'labels_gt')] + + # target_dir = '/data/zql/datasets/gta5_for_cls_task' + # if os.path.exists(target_dir): + # shutil.rmtree(target_dir) + + # convert_seg_dataset_to_cls( + # seg_imgs_path=images_path, + # seg_labels_path=labels_path, + # target_cls_data_dir=target_dir, + # ignore_classes_idx=[], + # thread_i=0 + # ) + + # cityscapes + # root_dir = '/data/zql/datasets/cityscape/' + + # def _get_target_suffix(mode: str, target_type: str) -> str: + # if target_type == 'instance': + # return '{}_instanceIds.png'.format(mode) + # elif target_type == 'semantic': + # return '{}_labelIds.png'.format(mode) + # elif target_type == 'color': + # return '{}_color.png'.format(mode) + # else: + # return '{}_polygons.json'.format(mode) + + + # images_path, labels_path = [], [] + # split = 'train' + # images_dir = os.path.join(root_dir, 'leftImg8bit', split) + # targets_dir = os.path.join(root_dir, 'gtFine', split) + # for city in os.listdir(images_dir): + # img_dir = os.path.join(images_dir, city) + # target_dir = os.path.join(targets_dir, city) + # for file_name in os.listdir(img_dir): + # target_types = [] + # for t in ['semantic']: + # target_name = '{}_{}'.format(file_name.split('_leftImg8bit')[0], + # _get_target_suffix('gtFine', t)) + # target_types.append(os.path.join(target_dir, target_name)) + + # images_path.append(os.path.join(img_dir, file_name)) + # labels_path.append(target_types[0]) + + # print(images_path[0: 5], '\n', labels_path[0: 5]) + + # target_dir = '/data/zql/datasets/cityscapes_for_cls_task' + # if os.path.exists(target_dir): + # shutil.rmtree(target_dir) + # convert_seg_dataset_to_cls( + # seg_imgs_path=images_path, + # seg_labels_path=labels_path, + # target_cls_data_dir=target_dir, + # ignore_classes_idx=[], + # # num_threads=8 + # thread_i=0 + # ) + + # import shutil + + # ignore_target_dir = '/data/zql/datasets/cityscapes_for_cls_task_ignored' + + # ignore_label = 255 + # raw_idx_map_in_y_transform = {-1: ignore_label, 0: ignore_label, 1: ignore_label, 2: ignore_label, + # 3: ignore_label, 4: ignore_label, 5: ignore_label, 6: ignore_label, + # 7: 0, 8: 1, 9: ignore_label, 10: ignore_label, 11: 2, 12: 3, 13: 4, + # 14: ignore_label, 15: ignore_label, 16: ignore_label, 17: 5, + # 18: ignore_label, 19: 6, 20: 7, 21: 8, 22: 9, 23: 10, 24: 11, 25: 12, 26: 13, 27: 14, + # 28: 15, 29: ignore_label, 30: ignore_label, 31: 16, 32: 17, 33: 18} + # ignore_classes_idx = [k for k, v in raw_idx_map_in_y_transform.items() if v == ignore_label] + # ignore_classes_idx = sorted(ignore_classes_idx) + + # for class_dir in os.listdir(target_dir): + # if int(class_dir) in ignore_classes_idx: + # continue + # shutil.move( + # os.path.join(target_dir, class_dir), + # os.path.join(ignore_target_dir, class_dir) + # ) + # else: + # shutil.move( + # os.path.join(target_dir, class_dir), + # os.path.join(target_dir, str(raw_idx_map_in_y_transform[int(class_dir)])) + # ) + # continue + # print(class_dir) + # exit() + + + + # baidu person + # root_dir = '/data/zql/datasets/baidu_person/clean_images/' + + # images_path, labels_path = [], [] + # for p in os.listdir(os.path.join(root_dir, 'images')): + # images_path += [os.path.join(root_dir, 'images', p)] + # labels_path += [os.path.join(root_dir, 'profiles', p.split('.')[0] + '-profile.jpg')] + + # target_dir = '/data/zql/datasets/baiduperson_for_cls_task' + # # if os.path.exists(target_dir): + # # shutil.rmtree(target_dir) + + # def label_after_hook(x): + # x[x > 1] = 1 + # return x + + # convert_seg_dataset_to_cls( + # seg_imgs_path=images_path, + # seg_labels_path=labels_path, + # target_cls_data_dir=target_dir, + # ignore_classes_idx=[1], + # # num_threads=8 + # thread_i=1, + # min_img_size=224, + # label_after_hook=label_after_hook + # ) + + + \ No newline at end of file diff --git a/data/convert_seg_dataset_to_det.py b/data/convert_seg_dataset_to_det.py new file mode 100644 index 0000000000000000000000000000000000000000..962211cec2338a91c6062856134f7941a5d81d54 --- /dev/null +++ b/data/convert_seg_dataset_to_det.py @@ -0,0 +1,399 @@ +from data import ABDataset +from utils.common.data_record import read_json, write_json +from PIL import Image +import os +from utils.common.file import ensure_dir +import numpy as np +from itertools import groupby +from skimage import morphology, measure +from PIL import Image +from scipy import misc +import tqdm +from PIL import ImageFile +ImageFile.LOAD_TRUNCATED_IMAGES = True +import shutil + + +def convert_seg_dataset_to_det(seg_imgs_path, seg_labels_path, root_dir, target_coco_ann_path, ignore_classes_idx, thread_i, min_img_size=224, label_after_hook=lambda x: x): + """ + Reference: https://blog.csdn.net/lizaijinsheng/article/details/119889946 + + NOTE: + Background class should not be considered. + However, if a seg dataset has only one valid class, so that the generated cls dataset also has only one class and + the cls accuracy will be 100% forever. But we do not use the generated cls dataset alone, so it is ok. + """ + assert len(seg_imgs_path) == len(seg_labels_path) + + classes_imgs_id_map = {} + + coco_ann = { + 'categories': [], + "type": "instances", + 'images': [], + 'annotations': [] + } + + image_id = 0 + ann_id = 0 + + pbar = tqdm.tqdm(zip(seg_imgs_path, seg_labels_path), total=len(seg_imgs_path), + dynamic_ncols=True, leave=False, desc=f'thread {thread_i}') + for seg_img_path, seg_label_path in pbar: + + try: + seg_img = Image.open(seg_img_path) + seg_label = Image.open(seg_label_path).convert('L') + seg_label = np.array(seg_label) + seg_label = label_after_hook(seg_label) + except Exception as e: + print(e) + print(f'file {seg_img_path} error, skip') + exit() + # seg_img = Image.open(seg_img_path) + # seg_label = Image.open(seg_label_path).convert('L') + # seg_label = np.array(seg_label) + + image_coco_info = {'file_name': os.path.relpath(seg_img_path, root_dir), 'height': seg_img.height, 'width': seg_img.width, + 'id':image_id} + image_id += 1 + coco_ann['images'] += [image_coco_info] + + this_img_classes = set(seg_label.reshape(-1).tolist()) + # print(this_img_classes) + + for class_idx in this_img_classes: + if class_idx in ignore_classes_idx: + continue + + if class_idx not in classes_imgs_id_map.keys(): + classes_imgs_id_map[class_idx] = 0 + + mask = np.zeros((seg_label.shape[0], seg_label.shape[1]), dtype=np.uint8) + mask[seg_label == class_idx] = 1 + mask_without_small = morphology.remove_small_objects(mask, min_size=10, connectivity=2) + label_image = measure.label(mask_without_small) + + for region in measure.regionprops(label_image): + bbox = region.bbox # (top, left, bottom, right) + bbox = [bbox[1], bbox[0], bbox[3], bbox[2]] # (left, top, right, bottom) + + width, height = bbox[2] - bbox[0], bbox[3] - bbox[1] + if width < min_img_size or height < min_img_size: + continue + + # target_cropped_img_path = os.path.join(target_cls_data_dir, str(class_idx), + # f'{classes_imgs_id_map[class_idx]}.{seg_img_path.split(".")[-1]}') + # ensure_dir(target_cropped_img_path) + # seg_img.crop(bbox).save(target_cropped_img_path) + # print(target_cropped_img_path) + # exit() + + ann_coco_info = {'area': width*height, 'iscrowd': 0, 'image_id': + image_id - 1, 'bbox': [bbox[0], bbox[1], width, height], + 'category_id': class_idx, + 'id': ann_id, 'ignore': 0, + 'segmentation': []} + ann_id += 1 + + coco_ann['annotations'] += [ann_coco_info] + + classes_imgs_id_map[class_idx] += 1 + + pbar.set_description(f'# ann: {ann_id}') + + coco_ann['categories'] = [ + {'id': ci, 'name': f'class_{c}_in_seg'} for ci, c in enumerate(classes_imgs_id_map.keys()) + ] + c_to_ci_map = {c: ci for ci, c in enumerate(classes_imgs_id_map.keys())} + for ann in coco_ann['annotations']: + ann['category_id'] = c_to_ci_map[ + ann['category_id'] + ] + + write_json(target_coco_ann_path, coco_ann, indent=0, backup=True) + write_json(os.path.join(root_dir, 'coco_ann.json'), coco_ann, indent=0, backup=True) + + num_cls_imgs = 0 + for k, v in classes_imgs_id_map.items(): + # print(f'# class {k}: {v + 1}') + num_cls_imgs += v + # print(f'total: {num_cls_imgs}') + + return classes_imgs_id_map + + +from concurrent.futures import ThreadPoolExecutor + + + +# def convert_seg_dataset_to_cls_multi_thread(seg_imgs_path, seg_labels_path, target_cls_data_dir, ignore_classes_idx, num_threads): +# if os.path.exists(target_cls_data_dir): +# shutil.rmtree(target_cls_data_dir) + +# assert len(seg_imgs_path) == len(seg_labels_path) +# n = len(seg_imgs_path) // num_threads + +# pool = ThreadPoolExecutor(max_workers=num_threads) +# # threads = [] +# futures = [] +# for thread_i in range(num_threads): +# # thread = threading.Thread(target=convert_seg_dataset_to_cls, +# # args=(seg_imgs_path[thread_i * n: (thread_i + 1) * n], +# # seg_labels_path[thread_i * n: (thread_i + 1) * n], +# # target_cls_data_dir, ignore_classes_idx)) +# # threads += [thread] +# future = pool.submit(convert_seg_dataset_to_cls, *(seg_imgs_path[thread_i * n: (thread_i + 1) * n], +# seg_labels_path[thread_i * n: (thread_i + 1) * n], +# target_cls_data_dir, ignore_classes_idx, thread_i)) +# futures += [future] + +# futures += [ +# pool.submit(convert_seg_dataset_to_cls, *(seg_imgs_path[(thread_i + 1) * n: ], +# seg_labels_path[(thread_i + 1) * n: ], +# target_cls_data_dir, ignore_classes_idx, thread_i)) +# ] + +# for f in futures: +# f.done() + +# res = [] +# for f in futures: +# res += [f.result()] +# print(res[-1]) + +# res_dist = {} +# for r in res: +# for k, v in r.items(): +# if k in res_dist.keys(): +# res_dist[k] += v +# else: +# res_dist[k] = v + +# print('results:') +# print(res_dist) + +# pool.shutdown() + + + +# import random +# def random_crop_aug(target_dir): +# for class_dir in os.listdir(target_dir): +# class_dir = os.path.join(target_dir, class_dir) + +# for img_path in os.listdir(class_dir): +# img_path = os.path.join(class_dir, img_path) + +# img = Image.open(img_path) + +# w, h = img.width, img.height + +# for ri in range(5): +# img.crop( +# [ +# random.randint(0, w // 5), +# random.randint(0, h // 5), +# random.randint(w // 5 * 4, w), +# random.randint(h // 5 * 4, h) +# ] +# ).save( +# os.path.join(os.path.dirname(img_path), f'randaug_{ri}_' + os.path.basename(img_path)) +# ) +# # print(img_path) +# # exit() + + +def post_ignore_classes(coco_ann_json_path): + # from data.datasets.object_detection.yolox_data_util.api import remap_dataset + # remap_dataset(coco_ann_json_path, [], {}) + pass + + + +if __name__ == '__main__': + # SuperviselyPerson + # root_dir = '/data/zql/datasets/supervisely_person_full_20230635/Supervisely Person Dataset' + + # images_path, labels_path = [], [] + # for p in os.listdir(root_dir): + # if p.startswith('ds'): + # p1 = os.path.join(root_dir, p, 'img') + # images_path += [(p, os.path.join(p1, n)) for n in os.listdir(p1)] + # for dsi, img_p in images_path: + # target_p = os.path.join(root_dir, p, dsi, img_p.split('/')[-1]) + # labels_path += [target_p] + # images_path = [i[1] for i in images_path] + + # target_coco_ann_path = '/data/zql/datasets/supervisely_person_for_det_task/coco_ann.json' + # if os.path.exists(target_coco_ann_path): + # os.remove(target_coco_ann_path) + # convert_seg_dataset_to_det( + # seg_imgs_path=images_path, + # seg_labels_path=labels_path, + # root_dir=root_dir, + # target_coco_ann_path=target_coco_ann_path, + # ignore_classes_idx=[0, 2], + # # num_threads=8 + # thread_i=0 + # ) + + # random_crop_aug('/data/zql/datasets/supervisely_person_for_cls_task') + + + # GTA5 + # root_dir = '/data/zql/datasets/GTA-ls-copy/GTA5' + # images_path, labels_path = [], [] + # for p in os.listdir(os.path.join(root_dir, 'images')): + # p = os.path.join(root_dir, 'images', p) + # if not p.endswith('png'): + # continue + # images_path += [p] + # labels_path += [p.replace('images', 'labels_gt')] + + # target_coco_ann_path = '/data/zql/datasets/gta5_for_det_task/coco_ann.json' + # if os.path.exists(target_coco_ann_path): + # os.remove(target_coco_ann_path) + + # """ + # [ + # 'road', 'sidewalk', 'building', 'wall', + # 'fence', 'pole', 'light', 'sign', + # 'vegetation', 'terrain', 'sky', 'people', # person + # 'rider', 'car', 'truck', 'bus', 'train', + # 'motocycle', 'bicycle' + # ] + # """ + # need_classes_idx = [13, 15] + # convert_seg_dataset_to_det( + # seg_imgs_path=images_path, + # seg_labels_path=labels_path, + # root_dir=root_dir, + # target_coco_ann_path=target_coco_ann_path, + # ignore_classes_idx=[i for i in range(20) if i not in need_classes_idx], + # thread_i=0 + # ) + + # from data.datasets.object_detection.yolox_data_util.api import remap_dataset + # new_coco_ann_json_path = remap_dataset('/data/zql/datasets/GTA-ls-copy/GTA5/coco_ann.json', [-1], {0: 0, 1:-1, 2:-1, 3: 1, 4:-1, 5:-1}) + # print(new_coco_ann_json_path) + + # cityscapes + # root_dir = '/data/zql/datasets/cityscape/' + + # def _get_target_suffix(mode: str, target_type: str) -> str: + # if target_type == 'instance': + # return '{}_instanceIds.png'.format(mode) + # elif target_type == 'semantic': + # return '{}_labelIds.png'.format(mode) + # elif target_type == 'color': + # return '{}_color.png'.format(mode) + # else: + # return '{}_polygons.json'.format(mode) + + + # images_path, labels_path = [], [] + # split = 'train' + # images_dir = os.path.join(root_dir, 'leftImg8bit', split) + # targets_dir = os.path.join(root_dir, 'gtFine', split) + # for city in os.listdir(images_dir): + # img_dir = os.path.join(images_dir, city) + # target_dir = os.path.join(targets_dir, city) + # for file_name in os.listdir(img_dir): + # target_types = [] + # for t in ['semantic']: + # target_name = '{}_{}'.format(file_name.split('_leftImg8bit')[0], + # _get_target_suffix('gtFine', t)) + # target_types.append(os.path.join(target_dir, target_name)) + + # images_path.append(os.path.join(img_dir, file_name)) + # labels_path.append(target_types[0]) + + # # print(images_path[0: 5], '\n', labels_path[0: 5]) + + # target_coco_ann_path = '/data/zql/datasets/cityscape/coco_ann.json' + # # if os.path.exists(target_dir): + # # shutil.rmtree(target_dir) + + # need_classes_idx = [26, 28] + # convert_seg_dataset_to_det( + # seg_imgs_path=images_path, + # seg_labels_path=labels_path, + # root_dir=root_dir, + # target_coco_ann_path=target_coco_ann_path, + # ignore_classes_idx=[i for i in range(80) if i not in need_classes_idx], + # # num_threads=8 + # thread_i=0 + # ) + + # import shutil + + # ignore_target_dir = '/data/zql/datasets/cityscapes_for_cls_task_ignored' + + # ignore_label = 255 + # raw_idx_map_in_y_transform = {-1: ignore_label, 0: ignore_label, 1: ignore_label, 2: ignore_label, + # 3: ignore_label, 4: ignore_label, 5: ignore_label, 6: ignore_label, + # 7: 0, 8: 1, 9: ignore_label, 10: ignore_label, 11: 2, 12: 3, 13: 4, + # 14: ignore_label, 15: ignore_label, 16: ignore_label, 17: 5, + # 18: ignore_label, 19: 6, 20: 7, 21: 8, 22: 9, 23: 10, 24: 11, 25: 12, 26: 13, 27: 14, + # 28: 15, 29: ignore_label, 30: ignore_label, 31: 16, 32: 17, 33: 18} + # ignore_classes_idx = [k for k, v in raw_idx_map_in_y_transform.items() if v == ignore_label] + # ignore_classes_idx = sorted(ignore_classes_idx) + + # for class_dir in os.listdir(target_dir): + # if int(class_dir) in ignore_classes_idx: + # continue + # shutil.move( + # os.path.join(target_dir, class_dir), + # os.path.join(ignore_target_dir, class_dir) + # ) + # else: + # shutil.move( + # os.path.join(target_dir, class_dir), + # os.path.join(target_dir, str(raw_idx_map_in_y_transform[int(class_dir)])) + # ) + # continue + # print(class_dir) + # exit() + + + + # baidu person + # root_dir = '/data/zql/datasets/baidu_person/clean_images/' + + # images_path, labels_path = [], [] + # for p in os.listdir(os.path.join(root_dir, 'images')): + # images_path += [os.path.join(root_dir, 'images', p)] + # labels_path += [os.path.join(root_dir, 'profiles', p.split('.')[0] + '-profile.jpg')] + + # target_dir = '/data/zql/datasets/baiduperson_for_cls_task' + # # if os.path.exists(target_dir): + # # shutil.rmtree(target_dir) + + # def label_after_hook(x): + # x[x > 1] = 1 + # return x + + # convert_seg_dataset_to_det( + # seg_imgs_path=images_path, + # seg_labels_path=labels_path, + # root_dir=root_dir, + # target_coco_ann_path='/data/zql/datasets/baidu_person/clean_images/coco_ann_zql.json', + # ignore_classes_idx=[1], + # # num_threads=8 + # thread_i=1, + # min_img_size=224, + # label_after_hook=label_after_hook + # ) + + + # from data.visualize import visualize_classes_in_object_detection + # from data import get_dataset + # d = get_dataset('CityscapesDet', '/data/zql/datasets/cityscape/', 'val', None, [], None) + # visualize_classes_in_object_detection(d, {'car': 0, 'bus': 1}, {}, 'debug.png') + + # d = get_dataset('GTA5Det', '/data/zql/datasets/GTA-ls-copy/GTA5', 'val', None, [], None) + # visualize_classes_in_object_detection(d, {'car': 0, 'bus': 1}, {}, 'debug.png') + + # d = get_dataset('BaiduPersonDet', '/data/zql/datasets/baidu_person/clean_images/', 'val', None, [], None) + # visualize_classes_in_object_detection(d, {'person': 0}, {}, 'debug.png') \ No newline at end of file diff --git a/data/dataloader.py b/data/dataloader.py new file mode 100644 index 0000000000000000000000000000000000000000..fc08303e98e91eb564415956f6bd9e0c78648916 --- /dev/null +++ b/data/dataloader.py @@ -0,0 +1,131 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +# domainbed/lib/fast_data_loader.py + +import torch +from .datasets.ab_dataset import ABDataset + + +class _InfiniteSampler(torch.utils.data.Sampler): + """Wraps another Sampler to yield an infinite stream.""" + + def __init__(self, sampler): + self.sampler = sampler + + def __iter__(self): + while True: + for batch in self.sampler: + yield batch + + +class InfiniteDataLoader: + def __init__(self, dataset, weights, batch_size, num_workers, collate_fn=None): + super().__init__() + + if weights: + sampler = torch.utils.data.WeightedRandomSampler( + weights, replacement=True, num_samples=batch_size + ) + else: + sampler = torch.utils.data.RandomSampler(dataset, replacement=True) + + batch_sampler = torch.utils.data.BatchSampler( + sampler, batch_size=batch_size, drop_last=True + ) + + if collate_fn is not None: + self._infinite_iterator = iter( + torch.utils.data.DataLoader( + dataset, + num_workers=num_workers, + batch_sampler=_InfiniteSampler(batch_sampler), + pin_memory=False, + collate_fn=collate_fn + ) + ) + else: + self._infinite_iterator = iter( + torch.utils.data.DataLoader( + dataset, + num_workers=num_workers, + batch_sampler=_InfiniteSampler(batch_sampler), + pin_memory=False + ) + ) + self.dataset = dataset + + def __iter__(self): + while True: + yield next(self._infinite_iterator) + + def __len__(self): + raise ValueError + + +class FastDataLoader: + """ + DataLoader wrapper with slightly improved speed by not respawning worker + processes at every epoch. + """ + + def __init__(self, dataset, batch_size, num_workers, shuffle=False, collate_fn=None): + super().__init__() + + self.num_workers = num_workers + + if shuffle: + sampler = torch.utils.data.RandomSampler(dataset, replacement=False) + else: + sampler = torch.utils.data.SequentialSampler(dataset) + + batch_sampler = torch.utils.data.BatchSampler( + sampler, + batch_size=batch_size, + drop_last=False, + ) + if collate_fn is not None: + self._infinite_iterator = iter( + torch.utils.data.DataLoader( + dataset, + num_workers=num_workers, + batch_sampler=_InfiniteSampler(batch_sampler), + pin_memory=False, + collate_fn=collate_fn + ) + ) + else: + self._infinite_iterator = iter( + torch.utils.data.DataLoader( + dataset, + num_workers=num_workers, + batch_sampler=_InfiniteSampler(batch_sampler), + pin_memory=False, + ) + ) + + self.dataset = dataset + self.batch_size = batch_size + self._length = len(batch_sampler) + + def __iter__(self): + for _ in range(len(self)): + yield next(self._infinite_iterator) + + def __len__(self): + return self._length + + +def build_dataloader(dataset: ABDataset, batch_size: int, num_workers: int, infinite: bool, shuffle_when_finite: bool, collate_fn=None): + assert batch_size <= len(dataset), len(dataset) + if infinite: + dataloader = InfiniteDataLoader( + dataset, None, batch_size, num_workers=num_workers, collate_fn=collate_fn) + else: + dataloader = FastDataLoader( + dataset, batch_size, num_workers, shuffle=shuffle_when_finite, collate_fn=collate_fn) + + return dataloader + + +def get_a_batch_dataloader(dataset: ABDataset, batch_size: int, num_workers: int, infinite: bool, shuffle_when_finite: bool): + pass + \ No newline at end of file diff --git a/data/dataset.py b/data/dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..f13f051205b3230324f141a583f0697554f34bb6 --- /dev/null +++ b/data/dataset.py @@ -0,0 +1,43 @@ +import importlib +from typing import Type +import torch +from torch.utils.data import TensorDataset +from torch.utils.data.dataloader import DataLoader + +from .datasets.ab_dataset import ABDataset + +from .datasets import * # import all datasets +from .datasets.registery import static_dataset_registery + + +def get_dataset(dataset_name, root_dir, split, transform=None, ignore_classes=[], idx_map=None) -> ABDataset: + dataset_cls = static_dataset_registery[dataset_name][0] + dataset = dataset_cls(root_dir, split, transform, ignore_classes, idx_map) + + return dataset + + +def get_num_limited_dataset(dataset: ABDataset, num_samples: int, discard_label=True): + dataloader = iter(DataLoader(dataset, num_samples // 2, shuffle=True)) + x, y = [], [] + cur_num_samples = 0 + while True: + batch = next(dataloader) + cur_x, cur_y = batch[0], batch[1] + + x += [cur_x] + y += [cur_y] + cur_num_samples += cur_x.size(0) + + if cur_num_samples >= num_samples: + break + + x, y = torch.cat(x)[0: num_samples], torch.cat(y)[0: num_samples] + if discard_label: + new_dataset = TensorDataset(x) + else: + new_dataset = TensorDataset(x, y) + + dataset.dataset = new_dataset + + return dataset diff --git a/data/datasets/__init__.py b/data/datasets/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..0314979cf51236cb579164b7fa0216dc1dba659a --- /dev/null +++ b/data/datasets/__init__.py @@ -0,0 +1,12 @@ +from .image_classification import * +from .object_detection import * +from .semantic_segmentation import * +from .action_recognition import * + +from .sentiment_classification import * +from .text_generation import * +from .machine_translation import * +from .pos_tagging import * + +from .mm_image_classification import * +from .visual_question_answering import * diff --git a/data/datasets/__pycache__/__init__.cpython-38.pyc b/data/datasets/__pycache__/__init__.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..71e020be620851fdbaa3a3167d656464e5a2a7a5 Binary files /dev/null and b/data/datasets/__pycache__/__init__.cpython-38.pyc differ diff --git a/data/datasets/__pycache__/ab_dataset.cpython-38.pyc b/data/datasets/__pycache__/ab_dataset.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4b8d30ccfb146c3effb723bedcf050a5dba7a2a8 Binary files /dev/null and b/data/datasets/__pycache__/ab_dataset.cpython-38.pyc differ diff --git a/data/datasets/__pycache__/data_aug.cpython-38.pyc b/data/datasets/__pycache__/data_aug.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..997d42c48ff190111549fcddaf380b8cb9d9da28 Binary files /dev/null and b/data/datasets/__pycache__/data_aug.cpython-38.pyc differ diff --git a/data/datasets/__pycache__/dataset_cache.cpython-38.pyc b/data/datasets/__pycache__/dataset_cache.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e0efa4e9c7ef1ce3860e15cd00a8c1b1f48a9745 Binary files /dev/null and b/data/datasets/__pycache__/dataset_cache.cpython-38.pyc differ diff --git a/data/datasets/__pycache__/dataset_split.cpython-38.pyc b/data/datasets/__pycache__/dataset_split.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a3fdf216735915172c85425880ec9686154c030d Binary files /dev/null and b/data/datasets/__pycache__/dataset_split.cpython-38.pyc differ diff --git a/data/datasets/__pycache__/registery.cpython-38.pyc b/data/datasets/__pycache__/registery.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..997457291508533c2aed60125bafe29c79ce80f4 Binary files /dev/null and b/data/datasets/__pycache__/registery.cpython-38.pyc differ diff --git a/data/datasets/ab_dataset.py b/data/datasets/ab_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..a011f9c833077ea311ff780eb05598139ac057a0 --- /dev/null +++ b/data/datasets/ab_dataset.py @@ -0,0 +1,48 @@ +from abc import ABC, abstractmethod +from typing import Dict, List, Optional +from torchvision.transforms import Compose + + +class ABDataset(ABC): + def __init__(self, root_dir, split, transform=None, ignore_classes=[], idx_map=None): + + self.root_dir = root_dir + self.split = split + self.transform = transform + self.ignore_classes = ignore_classes + self.idx_map = idx_map + + self.dataset = None + + # injected by @dataset_register + self.name = None + self.classes = None + self.raw_classes = None + self.class_aliases = None + self.shift_type = None + self.task_type = None # ['Image Classification', 'Object Detection', ...] + self.object_type = None # ['generic object', 'digit and letter', ...] + + @abstractmethod + def create_dataset(self, root_dir: str, split: str, transform: Optional[Compose], + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + raise NotImplementedError + + def build(self): + if not hasattr(self, 'classes'): + raise AttributeError('attr `classes` is injected by `@dataset_register()`. ' + 'Your dataset class should be wrapped with @dataset_register().') + self.dataset = self.create_dataset(self.root_dir, self.split, self.transform, + self.classes, self.ignore_classes, self.idx_map) + self.raw_classes = self.classes + self.classes = [i for i in self.classes if i not in self.ignore_classes] + + def __getitem__(self, idx): + if self.dataset is None: + raise AttributeError('Real dataset is build in `@dataset_register()`. ' + 'Your dataset class should be wrapped with @dataset_register().') + return self.dataset[idx] + + def __len__(self): + return len(self.dataset) + \ No newline at end of file diff --git a/data/datasets/action_recognition/__init__.py b/data/datasets/action_recognition/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e7dff1e8341c679053d0bdcf317736e655ca4867 --- /dev/null +++ b/data/datasets/action_recognition/__init__.py @@ -0,0 +1,4 @@ +from .ucf101 import UCF101 +from .hmdb51 import HMDB51 +# from .kinetics400 import Kinetics400 +from .ixmas import IXMAS diff --git a/data/datasets/action_recognition/__pycache__/__init__.cpython-38.pyc b/data/datasets/action_recognition/__pycache__/__init__.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d1dcfd25d785f95d51b9303a6fead34d7d0b61b1 Binary files /dev/null and b/data/datasets/action_recognition/__pycache__/__init__.cpython-38.pyc differ diff --git a/data/datasets/action_recognition/__pycache__/common_dataset.cpython-38.pyc b/data/datasets/action_recognition/__pycache__/common_dataset.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..30459ba521c704be1cb0ac66c325083f8e96683e Binary files /dev/null and b/data/datasets/action_recognition/__pycache__/common_dataset.cpython-38.pyc differ diff --git a/data/datasets/action_recognition/__pycache__/hmdb51.cpython-38.pyc b/data/datasets/action_recognition/__pycache__/hmdb51.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e6f0423c9320b2f70da961bdbcf185d06d342a85 Binary files /dev/null and b/data/datasets/action_recognition/__pycache__/hmdb51.cpython-38.pyc differ diff --git a/data/datasets/action_recognition/__pycache__/ixmas.cpython-38.pyc b/data/datasets/action_recognition/__pycache__/ixmas.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..dee46ea8461e20180095d85d240cd938407aceef Binary files /dev/null and b/data/datasets/action_recognition/__pycache__/ixmas.cpython-38.pyc differ diff --git a/data/datasets/action_recognition/__pycache__/ucf101.cpython-38.pyc b/data/datasets/action_recognition/__pycache__/ucf101.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4b7311163135610f42cd6ec980b19e15ad88aeb9 Binary files /dev/null and b/data/datasets/action_recognition/__pycache__/ucf101.cpython-38.pyc differ diff --git a/data/datasets/action_recognition/common_dataset.py b/data/datasets/action_recognition/common_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..cf2fc187188be7253233e397327b3cdf8e865571 --- /dev/null +++ b/data/datasets/action_recognition/common_dataset.py @@ -0,0 +1,152 @@ +import os +from pathlib import Path + +import random + +import numpy as np +import pickle as pk +import cv2 +from tqdm import tqdm +from PIL import Image + +import torchvision.transforms as transforms +import torch + +# from prefetch_generator import BackgroundGenerator +from torch.utils.data import DataLoader, Dataset + + +class VideoDataset(Dataset): + + def __init__(self, directory_list, local_rank=0, enable_GPUs_num=0, distributed_load=False, resize_shape=[224, 224] , mode='train', clip_len=32, crop_size = 168): + + self.clip_len, self.crop_size, self.resize_shape = clip_len, crop_size, resize_shape + self.mode = mode + + self.fnames, labels = [],[] + # get the directory of the specified split + for directory in directory_list: + folder = Path(directory) + print("Load dataset from folder : ", folder) + for label in sorted(os.listdir(folder)): + for fname in os.listdir(os.path.join(folder, label)) if mode=="train" else os.listdir(os.path.join(folder, label))[:10]: + self.fnames.append(os.path.join(folder, label, fname)) + labels.append(label) + # print(labels) + random_list = list(zip(self.fnames, labels)) + random.shuffle(random_list) + self.fnames[:], labels[:] = zip(*random_list) + self.labels = labels + + # self.fnames = self.fnames[:240] + + if mode == 'train' and distributed_load: + single_num_ = len(self.fnames)//enable_GPUs_num + self.fnames = self.fnames[local_rank*single_num_:((local_rank+1)*single_num_)] + labels = labels[local_rank*single_num_:((local_rank+1)*single_num_)] + + # prepare a mapping between the label names (strings) and indices (ints) + self.label2index = {label:index for index, label in enumerate(sorted(set(labels)))} + # convert the list of label names into an array of label indices + self.label_array = np.array([self.label2index[label] for label in labels], dtype=int) + + def __getitem__(self, index): + # loading and preprocessing. TODO move them to transform classess + buffer = self.loadvideo(self.fnames[index]) + + height_index = np.random.randint(buffer.shape[2] - self.crop_size) + width_index = np.random.randint(buffer.shape[3] - self.crop_size) + + return buffer[:,:,height_index:height_index + self.crop_size, width_index:width_index + self.crop_size], self.label_array[index] + + + def __len__(self): + return len(self.fnames) + + + def loadvideo(self, fname): + # initialize a VideoCapture object to read video data into a numpy array + self.transform = transforms.Compose([ + transforms.Resize([self.resize_shape[0], self.resize_shape[1]]), + transforms.ToTensor(), + transforms.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5]) + ]) + + flip, flipCode = 1, random.choice([-1,0,1]) if np.random.random() < 0.5 and self.mode=="train" else 0 + + try: + video_stream = cv2.VideoCapture(fname) + frame_count = int(video_stream.get(cv2.CAP_PROP_FRAME_COUNT)) + except RuntimeError: + index = np.random.randint(self.__len__()) + video_stream = cv2.VideoCapture(self.fnames[index]) + frame_count = int(video_stream.get(cv2.CAP_PROP_FRAME_COUNT)) + + while frame_count self.clip_len*2+2 else 1 + time_index = np.random.randint(frame_count - self.clip_len * speed_rate) + + start_idx, end_idx, final_idx = time_index, time_index+(self.clip_len*speed_rate), frame_count-1 + count, sample_count, retaining = 0, 0, True + + # create a buffer. Must have dtype float, so it gets converted to a FloatTensor by Pytorch later + buffer = np.empty((self.clip_len, 3, self.resize_shape[0], self.resize_shape[1]), np.dtype('float32')) + + while (count <= end_idx and retaining): + retaining, frame = video_stream.read() + if count < start_idx: + count += 1 + continue + if count % speed_rate == speed_rate-1 and count >= start_idx and sample_count < self.clip_len: + if flip: + frame = cv2.flip(frame, flipCode=flipCode) + try: + buffer[sample_count] = self.transform(Image.fromarray(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))) + except cv2.error as err: + continue + sample_count += 1 + count += 1 + video_stream.release() + + return buffer.transpose((1, 0, 2, 3)) + + +if __name__ == '__main__': + + datapath = ['/data/datasets/ucf101/videos'] + + dataset = VideoDataset(datapath, + resize_shape=[224, 224], + mode='validation') + x, y = dataset[0] + # x: (3, num_frames, w, h) + print(x.shape, y.shape, y) + + # dataloader = DataLoader(dataset, batch_size=8, shuffle=True, num_workers=24, pin_memory=True) + + # bar = tqdm(total=len(dataloader), ncols=80) + + # prefetcher = DataPrefetcher(BackgroundGenerator(dataloader), 0) + # batch = prefetcher.next() + # iter_id = 0 + # while batch is not None: + # iter_id += 1 + # bar.update(1) + # if iter_id >= len(dataloader): + # break + + # batch = prefetcher.next() + # print(batch[0].shape) + # print("label: ", batch[1]) + + # ''' + # for step, (buffer, labels) in enumerate(BackgroundGenerator(dataloader)): + # print(buffer.shape) + # print("label: ", labels) + # bar.update(1) + # ''' + diff --git a/data/datasets/action_recognition/hmdb51.py b/data/datasets/action_recognition/hmdb51.py new file mode 100644 index 0000000000000000000000000000000000000000..bb006bc24049eb7d16b3485586cdc481fae77df8 --- /dev/null +++ b/data/datasets/action_recognition/hmdb51.py @@ -0,0 +1,45 @@ +from ..data_aug import cityscapes_like_image_train_aug, cityscapes_like_image_test_aug, cityscapes_like_label_aug +# from torchvision.datasets import Cityscapes as RawCityscapes +from ..ab_dataset import ABDataset +from ..dataset_split import train_val_test_split +import numpy as np +from typing import Dict, List, Optional +from torchvision.transforms import Compose, Lambda +import os + +from .common_dataset import VideoDataset +from ..registery import dataset_register + + +@dataset_register( + name='HMDB51', + classes=['brush_hair', 'cartwheel', 'catch', 'chew', 'clap', 'climb', 'climb_stairs', 'dive', 'draw_sword', 'dribble', 'drink', 'eat', 'fall_floor', 'fencing', 'flic_flac', 'golf', 'handstand', 'hit', 'hug', 'jump', 'kick', 'kick_ball', 'kiss', 'laugh', 'pick', 'pour', 'pullup', 'punch', 'push', 'pushup', 'ride_bike', 'ride_horse', 'run', 'shake_hands', 'shoot_ball', 'shoot_bow', 'shoot_gun', 'sit', 'situp', 'smile', 'smoke', 'somersault', 'stand', 'swing_baseball', 'sword', 'sword_exercise', 'talk', 'throw', 'turn', 'walk', 'wave'], + task_type='Action Recognition', + object_type='Web Video', + # class_aliases=[['automobile', 'car']], + class_aliases=[], + shift_type=None +) +class HMDB51(ABDataset): # just for demo now + def create_dataset(self, root_dir: str, split: str, transform: Optional[Compose], + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + # if transform is None: + # x_transform = cityscapes_like_image_train_aug() if split == 'train' else cityscapes_like_image_test_aug() + # y_transform = cityscapes_like_label_aug() + # self.transform = x_transform + # else: + # x_transform, y_transform = transform + + dataset = VideoDataset([root_dir], mode='train') + + if len(ignore_classes) > 0: + for ignore_class in ignore_classes: + ci = classes.index(ignore_class) + dataset.fnames = [img for img, label in zip(dataset.fnames, dataset.label_array) if label != ci] + dataset.label_array = [label for label in dataset.label_array if label != ci] + + if idx_map is not None: + dataset.label_array = [idx_map[label] for label in dataset.label_array] + + dataset = train_val_test_split(dataset, split) + return dataset diff --git a/data/datasets/action_recognition/ixmas.py b/data/datasets/action_recognition/ixmas.py new file mode 100644 index 0000000000000000000000000000000000000000..065f8a142a6b2229f2ac6a86ef923035f8de35c0 --- /dev/null +++ b/data/datasets/action_recognition/ixmas.py @@ -0,0 +1,45 @@ +from ..data_aug import cityscapes_like_image_train_aug, cityscapes_like_image_test_aug, cityscapes_like_label_aug +# from torchvision.datasets import Cityscapes as RawCityscapes +from ..ab_dataset import ABDataset +from ..dataset_split import train_val_test_split +import numpy as np +from typing import Dict, List, Optional +from torchvision.transforms import Compose, Lambda +import os + +from .common_dataset import VideoDataset +from ..registery import dataset_register + + +@dataset_register( + name='IXMAS', + classes=['check_watch', 'cross_arms', 'get_up', 'kick', 'pick_up', 'point', 'punch', 'scratch_head', 'sit_down', 'turn_around', 'walk', 'wave'], + task_type='Action Recognition', + object_type='Web Video', + # class_aliases=[['automobile', 'car']], + class_aliases=[], + shift_type=None +) +class IXMAS(ABDataset): # just for demo now + def create_dataset(self, root_dir: str, split: str, transform: Optional[Compose], + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + # if transform is None: + # x_transform = cityscapes_like_image_train_aug() if split == 'train' else cityscapes_like_image_test_aug() + # y_transform = cityscapes_like_label_aug() + # self.transform = x_transform + # else: + # x_transform, y_transform = transform + + dataset = VideoDataset([root_dir], mode='train') + + if len(ignore_classes) > 0: + for ignore_class in ignore_classes: + ci = classes.index(ignore_class) + dataset.fnames = [img for img, label in zip(dataset.fnames, dataset.label_array) if label != ci] + dataset.label_array = [label for label in dataset.label_array if label != ci] + + if idx_map is not None: + dataset.label_array = [idx_map[label] for label in dataset.label_array] + + dataset = train_val_test_split(dataset, split) + return dataset diff --git a/data/datasets/action_recognition/kinetics400.py b/data/datasets/action_recognition/kinetics400.py new file mode 100644 index 0000000000000000000000000000000000000000..3bbd51e55d6180ab648bdddca1ae9ebe1e28796f --- /dev/null +++ b/data/datasets/action_recognition/kinetics400.py @@ -0,0 +1,51 @@ +from ..data_aug import cityscapes_like_image_train_aug, cityscapes_like_image_test_aug, cityscapes_like_label_aug +# from torchvision.datasets import Cityscapes as RawCityscapes +from ..ab_dataset import ABDataset +from ..dataset_split import train_val_split, train_val_test_split +import numpy as np +from typing import Dict, List, Optional +from torchvision.transforms import Compose, Lambda +import os + +from .common_dataset import VideoDataset +from ..registery import dataset_register + + +@dataset_register( + name='Kinetics400', + classes=['abseiling', 'air drumming', 'answering questions', 'applauding', 'applying cream', 'archery', 'arm wrestling', 'arranging flowers', 'assembling computer', 'auctioning', 'baby waking up', 'baking cookies', 'balloon blowing', 'bandaging', 'barbequing', 'bartending', 'beatboxing', 'bee keeping', 'belly dancing', 'bench pressing', 'bending back', 'bending metal', 'biking through snow', 'blasting sand', 'blowing glass', 'blowing leaves', 'blowing nose', 'blowing out candles', 'bobsledding', 'bookbinding', 'bouncing on trampoline', 'bowling', 'braiding hair', 'breading or breadcrumbing', 'breakdancing', 'brush painting', 'brushing hair', 'brushing teeth', 'building cabinet', 'building shed', 'bungee jumping', 'busking', 'canoeing or kayaking', 'capoeira', 'carrying baby', 'cartwheeling', 'carving pumpkin', 'catching fish', 'catching or throwing baseball', 'catching or throwing frisbee', 'catching or throwing softball', 'celebrating', 'changing oil', 'changing wheel', 'checking tires', 'cheerleading', 'chopping wood', 'clapping', 'clay pottery making', 'clean and jerk', 'cleaning floor', 'cleaning gutters', 'cleaning pool', 'cleaning shoes', 'cleaning toilet', 'cleaning windows', 'climbing a rope', 'climbing ladder', 'climbing tree', 'contact juggling', 'cooking chicken', 'cooking egg', 'cooking on campfire', 'cooking sausages', 'counting money', 'country line dancing', 'cracking neck', 'crawling baby', 'crossing river', 'crying', 'curling hair', 'cutting nails', 'cutting pineapple', 'cutting watermelon', 'dancing ballet', 'dancing charleston', 'dancing gangnam style', 'dancing macarena', 'deadlifting', 'decorating the christmas tree', 'digging', 'dining', 'disc golfing', 'diving cliff', 'dodgeball', 'doing aerobics', 'doing laundry', 'doing nails', 'drawing', 'dribbling basketball', 'drinking', 'drinking beer', 'drinking shots', 'driving car', 'driving tractor', 'drop kicking', 'drumming fingers', 'dunking basketball', 'dying hair', 'eating burger', 'eating cake', 'eating carrots', 'eating chips', 'eating doughnuts', 'eating hotdog', 'eating ice cream', 'eating spaghetti', 'eating watermelon', 'egg hunting', 'exercising arm', 'exercising with an exercise ball', 'extinguishing fire', 'faceplanting', 'feeding birds', 'feeding fish', 'feeding goats', 'filling eyebrows', 'finger snapping', 'fixing hair', 'flipping pancake', 'flying kite', 'folding clothes', 'folding napkins', 'folding paper', 'front raises', 'frying vegetables', 'garbage collecting', 'gargling', 'getting a haircut', 'getting a tattoo', 'giving or receiving award', 'golf chipping', 'golf driving', 'golf putting', 'grinding meat', 'grooming dog', 'grooming horse', 'gymnastics tumbling', 'hammer throw', 'headbanging', 'headbutting', 'high jump', 'high kick', 'hitting baseball', 'hockey stop', 'holding snake', 'hopscotch', 'hoverboarding', 'hugging', 'hula hooping', 'hurdling', 'hurling (sport)', 'ice climbing', 'ice fishing', 'ice skating', 'ironing', 'javelin throw', 'jetskiing', 'jogging', 'juggling balls', 'juggling fire', 'juggling soccer ball', 'jumping into pool', 'jumpstyle dancing', 'kicking field goal', 'kicking soccer ball', 'kissing', 'kitesurfing', 'knitting', 'krumping', 'laughing', 'laying bricks', 'long jump', 'lunge', 'making a cake', 'making a sandwich', 'making bed', 'making jewelry', 'making pizza', 'making snowman', 'making sushi', 'making tea', 'marching', 'massaging back', 'massaging feet', 'massaging legs', "massaging person's head", 'milking cow', 'mopping floor', 'motorcycling', 'moving furniture', 'mowing lawn', 'news anchoring', 'opening bottle', 'opening present', 'paragliding', 'parasailing', 'parkour', 'passing American football (in game)', 'passing American football (not in game)', 'peeling apples', 'peeling potatoes', 'petting animal (not cat)', 'petting cat', 'picking fruit', 'planting trees', 'plastering', 'playing accordion', 'playing badminton', 'playing bagpipes', 'playing basketball', 'playing bass guitar', 'playing cards', 'playing cello', 'playing chess', 'playing clarinet', 'playing controller', 'playing cricket', 'playing cymbals', 'playing didgeridoo', 'playing drums', 'playing flute', 'playing guitar', 'playing harmonica', 'playing harp', 'playing ice hockey', 'playing keyboard', 'playing kickball', 'playing monopoly', 'playing organ', 'playing paintball', 'playing piano', 'playing poker', 'playing recorder', 'playing saxophone', 'playing squash or racquetball', 'playing tennis', 'playing trombone', 'playing trumpet', 'playing ukulele', 'playing violin', 'playing volleyball', 'playing xylophone', 'pole vault', 'presenting weather forecast', 'pull ups', 'pumping fist', 'pumping gas', 'punching bag', 'punching person (boxing)', 'push up', 'pushing car', 'pushing cart', 'pushing wheelchair', 'reading book', 'reading newspaper', 'recording music', 'riding a bike', 'riding camel', 'riding elephant', 'riding mechanical bull', 'riding mountain bike', 'riding mule', 'riding or walking with horse', 'riding scooter', 'riding unicycle', 'ripping paper', 'robot dancing', 'rock climbing', 'rock scissors paper', 'roller skating', 'running on treadmill', 'sailing', 'salsa dancing', 'sanding floor', 'scrambling eggs', 'scuba diving', 'setting table', 'shaking hands', 'shaking head', 'sharpening knives', 'sharpening pencil', 'shaving head', 'shaving legs', 'shearing sheep', 'shining shoes', 'shooting basketball', 'shooting goal (soccer)', 'shot put', 'shoveling snow', 'shredding paper', 'shuffling cards', 'side kick', 'sign language interpreting', 'singing', 'situp', 'skateboarding', 'ski jumping', 'skiing (not slalom or crosscountry)', 'skiing crosscountry', 'skiing slalom', 'skipping rope', 'skydiving', 'slacklining', 'slapping', 'sled dog racing', 'smoking', 'smoking hookah', 'snatch weight lifting', 'sneezing', 'sniffing', 'snorkeling', 'snowboarding', 'snowkiting', 'snowmobiling', 'somersaulting', 'spinning poi', 'spray painting', 'spraying', 'springboard diving', 'squat', 'sticking tongue out', 'stomping grapes', 'stretching arm', 'stretching leg', 'strumming guitar', 'surfing crowd', 'surfing water', 'sweeping floor', 'swimming backstroke', 'swimming breast stroke', 'swimming butterfly stroke', 'swing dancing', 'swinging legs', 'swinging on something', 'sword fighting', 'tai chi', 'taking a shower', 'tango dancing', 'tap dancing', 'tapping guitar', 'tapping pen', 'tasting beer', 'tasting food', 'testifying', 'texting', 'throwing axe', 'throwing ball', 'throwing discus', 'tickling', 'tobogganing', 'tossing coin', 'tossing salad', 'training dog', 'trapezing', 'trimming or shaving beard', 'trimming trees', 'triple jump', 'tying bow tie', 'tying knot (not on a tie)', 'tying tie', 'unboxing', 'unloading truck', 'using computer', 'using remote controller (not gaming)', 'using segway', 'vault', 'waiting in line', 'walking the dog', 'washing dishes', 'washing feet', 'washing hair', 'washing hands', 'water skiing', 'water sliding', 'watering plants', 'waxing back', 'waxing chest', 'waxing eyebrows', 'waxing legs', 'weaving basket', 'welding', 'whistling', 'windsurfing', 'wrapping present', 'wrestling', 'writing', 'yawning', 'yoga', 'zumba'], + task_type='Action Recognition', + object_type='Web Video', + # class_aliases=[['automobile', 'car']], + class_aliases=[], + shift_type=None +) +class Kinetics400(ABDataset): # just for demo now + def create_dataset(self, root_dir: str, split: str, transform: Optional[Compose], + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + # if transform is None: + # x_transform = cityscapes_like_image_train_aug() if split == 'train' else cityscapes_like_image_test_aug() + # y_transform = cityscapes_like_label_aug() + # self.transform = x_transform + # else: + # x_transform, y_transform = transform + + if split == 'test': + root_dir = os.path.join(root_dir, 'videos_val') + else: + root_dir = os.path.join(root_dir, 'videos_train') + # print(root_dir) + dataset = VideoDataset([root_dir], mode='train') + + if len(ignore_classes) > 0: + for ignore_class in ignore_classes: + ci = classes.index(ignore_class) + dataset.fnames = [img for img, label in zip(dataset.fnames, dataset.label_array) if label != ci] + dataset.label_array = [label for label in dataset.label_array if label != ci] + + if idx_map is not None: + dataset.label_array = [idx_map[label] for label in dataset.label_array] + + if split != 'test': + dataset = train_val_split(dataset, split) + return dataset diff --git a/data/datasets/action_recognition/ucf101.py b/data/datasets/action_recognition/ucf101.py new file mode 100644 index 0000000000000000000000000000000000000000..48d8167343bd965e006a4df0e903e833b6c05ee0 --- /dev/null +++ b/data/datasets/action_recognition/ucf101.py @@ -0,0 +1,45 @@ +from ..data_aug import cityscapes_like_image_train_aug, cityscapes_like_image_test_aug, cityscapes_like_label_aug +# from torchvision.datasets import Cityscapes as RawCityscapes +from ..ab_dataset import ABDataset +from ..dataset_split import train_val_test_split +import numpy as np +from typing import Dict, List, Optional +from torchvision.transforms import Compose, Lambda +import os + +from .common_dataset import VideoDataset +from ..registery import dataset_register + + +@dataset_register( + name='UCF101', + classes=['apply_eye_makeup', 'apply_lipstick', 'archery', 'baby_crawling', 'balance_beam', 'band_marching', 'baseball_pitch', 'basketball', 'basketball_dunk', 'bench_press', 'biking', 'billiards', 'blow_dry_hair', 'blowing_candles', 'body_weight_squats', 'bowling', 'boxing_punching_bag', 'boxing_speed_bag', 'breast_stroke', 'brushing_teeth', 'clean_and_jerk', 'cliff_diving', 'cricket_bowling', 'cricket_shot', 'cutting_in_kitchen', 'diving', 'drumming', 'fencing', 'field_hockey_penalty', 'floor_gymnastics', 'frisbee_catch', 'front_crawl', 'golf_swing', 'haircut', 'hammer_throw', 'hammering', 'handstand_pushups', 'handstand_walking', 'head_massage', 'high_jump', 'horse_race', 'horse_riding', 'hula_hoop', 'ice_dancing', 'javelin_throw', 'juggling_balls', 'jump_rope', 'jumping_jack', 'kayaking', 'knitting', 'long_jump', 'lunges', 'military_parade', 'mixing', 'mopping_floor', 'nunchucks', 'parallel_bars', 'pizza_tossing', 'playing_cello', 'playing_daf', 'playing_dhol', 'playing_flute', 'playing_guitar', 'playing_piano', 'playing_sitar', 'playing_tabla', 'playing_violin', 'pole_vault', 'pommel_horse', 'pull_ups', 'punch', 'push_ups', 'rafting', 'rock_climbing_indoor', 'rope_climbing', 'rowing', 'salsa_spin', 'shaving_beard', 'shotput', 'skate_boarding', 'skiing', 'skijet', 'sky_diving', 'soccer_juggling', 'soccer_penalty', 'still_rings', 'sumo_wrestling', 'surfing', 'swing', 'table_tennis_shot', 'tai_chi', 'tennis_swing', 'throw_discus', 'trampoline_jumping', 'typing', 'uneven_bars', 'volleyball_spiking', 'walking_with_dog', 'wall_pushups', 'writing_on_board', 'yo_yo'], + task_type='Action Recognition', + object_type='Web Video', + # class_aliases=[['automobile', 'car']], + class_aliases=[], + shift_type=None +) +class UCF101(ABDataset): # just for demo now + def create_dataset(self, root_dir: str, split: str, transform: Optional[Compose], + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + # if transform is None: + # x_transform = cityscapes_like_image_train_aug() if split == 'train' else cityscapes_like_image_test_aug() + # y_transform = cityscapes_like_label_aug() + # self.transform = x_transform + # else: + # x_transform, y_transform = transform + + dataset = VideoDataset([root_dir], mode='train') + + if len(ignore_classes) > 0: + for ignore_class in ignore_classes: + ci = classes.index(ignore_class) + dataset.fnames = [img for img, label in zip(dataset.fnames, dataset.label_array) if label != ci] + dataset.label_array = [label for label in dataset.label_array if label != ci] + + if idx_map is not None: + dataset.label_array = [idx_map[label] for label in dataset.label_array] + + dataset = train_val_test_split(dataset, split) + return dataset diff --git a/data/datasets/data_aug.py b/data/datasets/data_aug.py new file mode 100644 index 0000000000000000000000000000000000000000..9b7a13a772ad80e5f948cca27fccdf7bc76c9b52 --- /dev/null +++ b/data/datasets/data_aug.py @@ -0,0 +1,93 @@ +from torchvision import transforms +import torch + + +def one_d_image_train_aug(to_3_channels=False): + mean, std = (0.1307, 0.1307, 0.1307), (0.3081, 0.3081, 0.3081) + return transforms.Compose([ + transforms.Resize(32), + # transforms.RandomCrop(32, padding=4), + transforms.ToTensor(), + transforms.Lambda((lambda x: torch.cat([x] * 3)) if to_3_channels else (lambda x: x)), + transforms.Normalize(mean, std) + ]) + + +def one_d_image_test_aug(to_3_channels=False): + mean, std = (0.1307, 0.1307, 0.1307), (0.3081, 0.3081, 0.3081) + return transforms.Compose([ + transforms.Resize(32), + transforms.ToTensor(), + transforms.Lambda((lambda x: torch.cat([x] * 3)) if to_3_channels else (lambda x: x)), + transforms.Normalize(mean, std) + ]) + + +def cifar_like_image_train_aug(mean=None, std=None): + if mean is None: + mean, std = (0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010) + return transforms.Compose([ + transforms.Resize(40), # NOTE: this is critical!!! or you may crop a small part of an image + transforms.RandomCrop(32, padding=4), + transforms.RandomHorizontalFlip(), + transforms.ToTensor(), + transforms.Normalize(mean, std) + ]) + + +def cifar_like_image_test_aug(mean=None, std=None): + if mean is None: + mean, std = (0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010) + return transforms.Compose([ + transforms.Resize(32), + transforms.ToTensor(), + transforms.Normalize(mean, std) + ]) + +def imagenet_like_image_train_aug(): + mean, std = [0.485, 0.456, 0.406], [0.229, 0.224, 0.225] + return transforms.Compose([ + transforms.Resize((256, 256)), + transforms.RandomCrop((224, 224), padding=4), + transforms.RandomHorizontalFlip(), + transforms.ToTensor(), + transforms.Normalize(mean, std) + ]) + + +def imagenet_like_image_test_aug(): + mean, std = [0.485, 0.456, 0.406], [0.229, 0.224, 0.225] + return transforms.Compose([ + transforms.Resize((224, 224)), + transforms.ToTensor(), + transforms.Normalize(mean, std) + ]) + + +def cityscapes_like_image_train_aug(): + return transforms.Compose([ + transforms.Resize((224, 224)), + transforms.ToTensor(), + transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), + ]) + +def cityscapes_like_image_test_aug(): + return transforms.Compose([ + transforms.Resize((224, 224)), + transforms.ToTensor(), + transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), + ]) + +def cityscapes_like_label_aug(): + import numpy as np + return transforms.Compose([ + transforms.Resize((224, 224)), + transforms.Lambda(lambda x: torch.from_numpy(np.array(x)).long()) + ]) + + +def pil_image_to_tensor(img_size=224): + return transforms.Compose([ + transforms.Resize((img_size, img_size)), + transforms.ToTensor() + ]) \ No newline at end of file diff --git a/data/datasets/dataset_cache.py b/data/datasets/dataset_cache.py new file mode 100644 index 0000000000000000000000000000000000000000..58194c9758e10d946ae9dfe6bf9eaf00bfcddfa0 --- /dev/null +++ b/data/datasets/dataset_cache.py @@ -0,0 +1,40 @@ +from typing import List, Optional, Dict +import os +import torch +from utils.common.log import logger +import hashlib + + +def get_dataset_cache_path(root_dir: str, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + + def _hash(o): + if isinstance(o, list): + o = sorted(o) + elif isinstance(o, dict): + o = {k: o[k] for k in sorted(o)} + elif isinstance(o, set): + o = sorted(list(o)) + # else: + # print(type(o)) + + obj = hashlib.md5() + obj.update(str(o).encode('utf-8')) + return obj.hexdigest() + + cache_key = _hash(f'zql_data_{_hash(root_dir)}_{_hash(classes)}_{_hash(ignore_classes)}_{_hash(idx_map)}.cache') + + # print(root_dir, classes, ignore_classes, idx_map) + # print('cache key', cache_key) + + cache_file_path = os.path.join('/tmp', f'./zql_data_cache_{cache_key}.cache') + return cache_file_path + + +def cache_dataset_status(status, cache_file_path, dataset_name): + logger.info(f'cache dataset status: {dataset_name}') + torch.save(status, cache_file_path) + +def read_cached_dataset_status(cache_file_path, dataset_name): + logger.info(f'read dataset cache: {dataset_name}') + return torch.load(cache_file_path) diff --git a/data/datasets/dataset_split.py b/data/datasets/dataset_split.py new file mode 100644 index 0000000000000000000000000000000000000000..6450605327246f79d8f1c537ab6e0ef5dbc204c7 --- /dev/null +++ b/data/datasets/dataset_split.py @@ -0,0 +1,81 @@ +import torch +import os +import numpy as np +from .ab_dataset import ABDataset + + +class _SplitDataset(torch.utils.data.Dataset): + """Used by split_dataset""" + + def __init__(self, underlying_dataset, keys): + super(_SplitDataset, self).__init__() + self.underlying_dataset = underlying_dataset + self.keys = keys + + def __getitem__(self, key): + return self.underlying_dataset[self.keys[key]] + + def __len__(self): + return len(self.keys) + + +def split_dataset(dataset, n, seed=0, transform=None): + + if isinstance(dataset, ABDataset): + if dataset.task_type == 'Object Detection': + return split_dataset_det(dataset, n, seed) + if dataset.task_type == 'MM Object Detection': + return split_dataset_det_mm(dataset, n, seed, transform=transform) + + """ + Return a pair of datasets corresponding to a random split of the given + dataset, with n datapoints in the first dataset and the rest in the last, + using the given random seed + """ + assert n <= len(dataset), f'{n}_{len(dataset)}' + + cache_p = f'{n}_{seed}_{len(dataset)}' + cache_p = os.path.join(os.path.expanduser( + '~'), '.domain_benchmark_split_dataset_cache_' + str(cache_p)) + if os.path.exists(cache_p): + keys_1, keys_2 = torch.load(cache_p) + else: + keys = list(range(len(dataset))) + np.random.RandomState(seed).shuffle(keys) + keys_1 = keys[:n] + keys_2 = keys[n:] + torch.save((keys_1, keys_2), cache_p) + + return _SplitDataset(dataset, keys_1), _SplitDataset(dataset, keys_2) + + +def train_val_split(dataset, split): + assert split in ['train', 'val'] + if split == 'train': + return split_dataset(dataset, int(len(dataset) * 0.8))[0] + else: + return split_dataset(dataset, int(len(dataset) * 0.8))[1] + + +def train_val_test_split(dataset, split): + assert split in ['train', 'val', 'test'] + + train_set, test_set = split_dataset(dataset, int(len(dataset) * 0.8)) + train_set, val_set = split_dataset(train_set, int(len(train_set) * 0.8)) + + return {'train': train_set, 'val': val_set, 'test': test_set}[split] + + +def split_dataset_det(dataset: ABDataset, n, seed=0): + coco_ann_json_path = dataset.ann_json_file_path_for_split + from .object_detection.yolox_data_util.api import coco_split, get_default_yolox_coco_dataset + split_coco_ann_json_path = coco_split(coco_ann_json_path, ratio=n / len(dataset))[0] + # print(n, len(dataset)) + return get_default_yolox_coco_dataset(dataset.root_dir, split_coco_ann_json_path, train=dataset.split == 'train'), None + +def split_dataset_det_mm(dataset: ABDataset, n, seed=0, transform=None): + coco_ann_json_path = dataset.ann_json_file_path_for_split + from .object_detection.yolox_data_util.api import coco_split, get_yolox_coco_dataset_with_caption + split_coco_ann_json_path = coco_split(coco_ann_json_path, ratio=n / len(dataset))[0] + # print(n, len(dataset)) + return get_yolox_coco_dataset_with_caption(dataset.root_dir, split_coco_ann_json_path, transform=transform, train=dataset.split == 'train', classes=dataset.classes), None \ No newline at end of file diff --git a/data/datasets/image_classification/__init__.py b/data/datasets/image_classification/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..4b00e398e823cb2ff320e8c48314fb974de89ca9 --- /dev/null +++ b/data/datasets/image_classification/__init__.py @@ -0,0 +1,24 @@ +from .mnist import MNIST +from .usps import USPS +from .svhn import SVHN +from .emnist import EMNIST +from .cifar10 import CIFAR10 +from .stl10 import STL10 +from .imagenet import ImageNet +from .imagenet_a import ImageNetA +from .caltech256 import Caltech256 +from .domainnet_real import DomainNetReal +from .synsigns import SYNSIGNS +from .gtsrb import GTSRB + +from .cifar10_single import CIFAR10Single +from .stl10_single import STL10Single +from .mnist_single import MNISTSingle +from .usps_single import USPSSingle +from .svhn_single import SVHNSingle + +from .baidu_person_cls import BaiduPersonCls +from .cityscapes_cls import CityscapesCls +from .gta5_cls import GTA5Cls +from .supervisely_person_cls import SuperviselyPersonCls +from .coco_cls import COCOCls \ No newline at end of file diff --git a/data/datasets/image_classification/__pycache__/__init__.cpython-38.pyc b/data/datasets/image_classification/__pycache__/__init__.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..aa2fcafed253f2fbcb47bc82d96878dc17f08d68 Binary files /dev/null and b/data/datasets/image_classification/__pycache__/__init__.cpython-38.pyc differ diff --git a/data/datasets/image_classification/__pycache__/baidu_person_cls.cpython-38.pyc b/data/datasets/image_classification/__pycache__/baidu_person_cls.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5f18dda12b4ed95e6e7271294d07be669a11ba9d Binary files /dev/null and b/data/datasets/image_classification/__pycache__/baidu_person_cls.cpython-38.pyc differ diff --git a/data/datasets/image_classification/__pycache__/caltech256.cpython-38.pyc b/data/datasets/image_classification/__pycache__/caltech256.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a5abf40f77a31347e5f90cbdb907eca5ee948fd9 Binary files /dev/null and b/data/datasets/image_classification/__pycache__/caltech256.cpython-38.pyc differ diff --git a/data/datasets/image_classification/__pycache__/cifar10.cpython-38.pyc b/data/datasets/image_classification/__pycache__/cifar10.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..08ca73bbc3d37a6e39e62f11de598b1456781384 Binary files /dev/null and b/data/datasets/image_classification/__pycache__/cifar10.cpython-38.pyc differ diff --git a/data/datasets/image_classification/__pycache__/cifar10_single.cpython-38.pyc b/data/datasets/image_classification/__pycache__/cifar10_single.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9818d31df8e5509fffd5985e2bec32dde473e5bb Binary files /dev/null and b/data/datasets/image_classification/__pycache__/cifar10_single.cpython-38.pyc differ diff --git a/data/datasets/image_classification/__pycache__/cityscapes_cls.cpython-38.pyc b/data/datasets/image_classification/__pycache__/cityscapes_cls.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f8fc7d571fe98f50e8dc5e1aff60176243d8dc56 Binary files /dev/null and b/data/datasets/image_classification/__pycache__/cityscapes_cls.cpython-38.pyc differ diff --git a/data/datasets/image_classification/__pycache__/coco_cls.cpython-38.pyc b/data/datasets/image_classification/__pycache__/coco_cls.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..dcd70600221be5412cec757dfa9c17669ec83ff4 Binary files /dev/null and b/data/datasets/image_classification/__pycache__/coco_cls.cpython-38.pyc differ diff --git a/data/datasets/image_classification/__pycache__/domainnet_real.cpython-38.pyc b/data/datasets/image_classification/__pycache__/domainnet_real.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9a988a3066d2427fa53730a931da9f34388a1a47 Binary files /dev/null and b/data/datasets/image_classification/__pycache__/domainnet_real.cpython-38.pyc differ diff --git a/data/datasets/image_classification/__pycache__/emnist.cpython-38.pyc b/data/datasets/image_classification/__pycache__/emnist.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a78a2c7602438aa6ebc1e8a325641e8de8c50ab2 Binary files /dev/null and b/data/datasets/image_classification/__pycache__/emnist.cpython-38.pyc differ diff --git a/data/datasets/image_classification/__pycache__/gta5_cls.cpython-38.pyc b/data/datasets/image_classification/__pycache__/gta5_cls.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..00e69c12427a6b0eeb01dcf6e9bcb602155f0127 Binary files /dev/null and b/data/datasets/image_classification/__pycache__/gta5_cls.cpython-38.pyc differ diff --git a/data/datasets/image_classification/__pycache__/gtsrb.cpython-38.pyc b/data/datasets/image_classification/__pycache__/gtsrb.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c9c09b99603533f96c5b9d62a125646130453ff8 Binary files /dev/null and b/data/datasets/image_classification/__pycache__/gtsrb.cpython-38.pyc differ diff --git a/data/datasets/image_classification/__pycache__/imagenet.cpython-38.pyc b/data/datasets/image_classification/__pycache__/imagenet.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c80249e5e282451cc4c6e780a1ee49bc7d62d2c2 Binary files /dev/null and b/data/datasets/image_classification/__pycache__/imagenet.cpython-38.pyc differ diff --git a/data/datasets/image_classification/__pycache__/imagenet_a.cpython-38.pyc b/data/datasets/image_classification/__pycache__/imagenet_a.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d70a334a6b09a7369e4466ca64a811418fba9f47 Binary files /dev/null and b/data/datasets/image_classification/__pycache__/imagenet_a.cpython-38.pyc differ diff --git a/data/datasets/image_classification/__pycache__/mnist.cpython-38.pyc b/data/datasets/image_classification/__pycache__/mnist.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e51ef2380d17fe3b4f1247a10367c0b2f82435cb Binary files /dev/null and b/data/datasets/image_classification/__pycache__/mnist.cpython-38.pyc differ diff --git a/data/datasets/image_classification/__pycache__/mnist_single.cpython-38.pyc b/data/datasets/image_classification/__pycache__/mnist_single.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d71108bf1122625cbe574f5b90dbca0da7b6e370 Binary files /dev/null and b/data/datasets/image_classification/__pycache__/mnist_single.cpython-38.pyc differ diff --git a/data/datasets/image_classification/__pycache__/stl10.cpython-38.pyc b/data/datasets/image_classification/__pycache__/stl10.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9cc23df49667cbf7c66b98dfc80bedd8ef4c5a78 Binary files /dev/null and b/data/datasets/image_classification/__pycache__/stl10.cpython-38.pyc differ diff --git a/data/datasets/image_classification/__pycache__/stl10_single.cpython-38.pyc b/data/datasets/image_classification/__pycache__/stl10_single.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2e20f4418bab5d78d562f57cf130d90cbd6645a9 Binary files /dev/null and b/data/datasets/image_classification/__pycache__/stl10_single.cpython-38.pyc differ diff --git a/data/datasets/image_classification/__pycache__/supervisely_person_cls.cpython-38.pyc b/data/datasets/image_classification/__pycache__/supervisely_person_cls.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a9776b60faa7187ab940781bbd764218d3acbdf8 Binary files /dev/null and b/data/datasets/image_classification/__pycache__/supervisely_person_cls.cpython-38.pyc differ diff --git a/data/datasets/image_classification/__pycache__/svhn.cpython-38.pyc b/data/datasets/image_classification/__pycache__/svhn.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..273071a93fcaf1076787246f372d3dbe97590dfc Binary files /dev/null and b/data/datasets/image_classification/__pycache__/svhn.cpython-38.pyc differ diff --git a/data/datasets/image_classification/__pycache__/svhn_single.cpython-38.pyc b/data/datasets/image_classification/__pycache__/svhn_single.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..86b9b88f0ec4703f2bd7622f0ea54a61e2a020d3 Binary files /dev/null and b/data/datasets/image_classification/__pycache__/svhn_single.cpython-38.pyc differ diff --git a/data/datasets/image_classification/__pycache__/synsigns.cpython-38.pyc b/data/datasets/image_classification/__pycache__/synsigns.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..025bec45ec498ca891f626740817f25a7262590d Binary files /dev/null and b/data/datasets/image_classification/__pycache__/synsigns.cpython-38.pyc differ diff --git a/data/datasets/image_classification/__pycache__/usps.cpython-38.pyc b/data/datasets/image_classification/__pycache__/usps.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7f50e26d977156efea58c5d1d33107c143b1a41e Binary files /dev/null and b/data/datasets/image_classification/__pycache__/usps.cpython-38.pyc differ diff --git a/data/datasets/image_classification/__pycache__/usps_single.cpython-38.pyc b/data/datasets/image_classification/__pycache__/usps_single.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7e8463bb7e73894fb4ff0332256db5364d2f59fc Binary files /dev/null and b/data/datasets/image_classification/__pycache__/usps_single.cpython-38.pyc differ diff --git a/data/datasets/image_classification/aid.py b/data/datasets/image_classification/aid.py new file mode 100644 index 0000000000000000000000000000000000000000..2d1fe024d2bac37b19b48c5533f0e96914c053fa --- /dev/null +++ b/data/datasets/image_classification/aid.py @@ -0,0 +1,37 @@ +from ..data_aug import imagenet_like_image_train_aug, imagenet_like_image_test_aug +from ..ab_dataset import ABDataset +from ..dataset_split import train_val_split, train_val_test_split +from torchvision.datasets import ImageFolder +import os +from typing import Dict, List, Optional +from torchvision.transforms import Compose + +from ..registery import dataset_register + +@dataset_register( + name='AID', + classes=['airport', 'bare land', 'baseball field', 'beach', 'bridge', 'center', 'church', 'commercial', 'dense residential', 'desert', 'farmland', 'forest', 'industrial', 'meadow', 'medium residential', 'mountain', 'park', 'parking', 'playground', 'pond', 'port', 'railway station', 'resort', 'river', 'school', 'sparse residential', 'square', 'stadium', 'storage tanks', 'viaduct'], + task_type='Image Classification', + object_type='Remote Sensing', + class_aliases=[], + shift_type=None +) + +class AID(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform: Optional[Compose], + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + if transform is None: + transform = imagenet_like_image_train_aug() if split == 'train' else imagenet_like_image_test_aug() + self.transform = transform + #root_dir = os.path.join(root_dir, 'train' if split != 'test' else 'val') + dataset = ImageFolder(root_dir, transform=transform) + + if len(ignore_classes) > 0: + ignore_classes_idx = [classes.index(c) for c in ignore_classes] + dataset.samples = [s for s in dataset.samples if s[1] not in ignore_classes_idx] + + if idx_map is not None: + dataset.samples = [(s[0], idx_map[s[1]]) if s[1] in idx_map.keys() else s for s in dataset.samples] + + dataset = train_val_test_split(dataset, split) + return dataset \ No newline at end of file diff --git a/data/datasets/image_classification/baidu_person_cls.py b/data/datasets/image_classification/baidu_person_cls.py new file mode 100644 index 0000000000000000000000000000000000000000..61871e096bb3feb82a5b2ff98054bc83a265b761 --- /dev/null +++ b/data/datasets/image_classification/baidu_person_cls.py @@ -0,0 +1,41 @@ +from ..data_aug import imagenet_like_image_train_aug, imagenet_like_image_test_aug +from ..ab_dataset import ABDataset +from ..dataset_split import train_val_split, train_val_test_split +from torchvision.datasets import ImageFolder +import os +from typing import Dict, List, Optional +from torchvision.transforms import Compose + +from ..registery import dataset_register + +# with open(os.path.join(os.path.dirname(__file__), 'fruits360_classes.txt'), 'r') as f: +# classes = [line.split(':')[0].strip('"') for line in f.readlines()] +# assert len(classes) == 131 + +@dataset_register( + name='BaiduPersonCls', + classes=['seg_person'], + task_type='Image Classification', + object_type='Person', + class_aliases=[], + shift_type=None +) + +class BaiduPersonCls(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform: Optional[Compose], + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + if transform is None: + transform = imagenet_like_image_train_aug() if split == 'train' else imagenet_like_image_test_aug() + self.transform = transform + #root_dir = os.path.join(root_dir, 'train' if split != 'test' else 'val') + dataset = ImageFolder(root_dir, transform=transform) + + if len(ignore_classes) > 0: + ignore_classes_idx = [classes.index(c) for c in ignore_classes] + dataset.samples = [s for s in dataset.samples if s[1] not in ignore_classes_idx] + + if idx_map is not None: + dataset.samples = [(s[0], idx_map[s[1]]) if s[1] in idx_map.keys() else s for s in dataset.samples] + + dataset = train_val_test_split(dataset, split) + return dataset diff --git a/data/datasets/image_classification/caltech256.py b/data/datasets/image_classification/caltech256.py new file mode 100644 index 0000000000000000000000000000000000000000..6b68a740e23d4e9b080b6da8428d9959a6940cc2 --- /dev/null +++ b/data/datasets/image_classification/caltech256.py @@ -0,0 +1,39 @@ +from ..data_aug import imagenet_like_image_train_aug, imagenet_like_image_test_aug +from ..ab_dataset import ABDataset +from ..dataset_split import train_val_split, train_val_test_split +from torchvision.datasets import ImageFolder +import os +from typing import Dict, List, Optional +from torchvision.transforms import Compose + +from ..registery import dataset_register + + +classes = ['001.ak47', '002.american-flag', '003.backpack', '004.baseball-bat', '005.baseball-glove', '006.basketball-hoop', '007.bat', '008.bathtub', '009.bear', '010.beer-mug', '011.billiards', '012.binoculars', '013.birdbath', '014.blimp', '015.bonsai-101', '016.boom-box', '017.bowling-ball', '018.bowling-pin', '019.boxing-glove', '020.brain-101', '021.breadmaker', '022.buddha-101', '023.bulldozer', '024.butterfly', '025.cactus', '026.cake', '027.calculator', '028.camel', '029.cannon', '030.canoe', '031.car-tire', '032.cartman', '033.cd', '034.centipede', '035.cereal-box', '036.chandelier-101', '037.chess-board', '038.chimp', '039.chopsticks', '040.cockroach', '041.coffee-mug', '042.coffin', '043.coin', '044.comet', '045.computer-keyboard', '046.computer-monitor', '047.computer-mouse', '048.conch', '049.cormorant', '050.covered-wagon', '051.cowboy-hat', '052.crab-101', '053.desk-globe', '054.diamond-ring', '055.dice', '056.dog', '057.dolphin-101', '058.doorknob', '059.drinking-straw', '060.duck', '061.dumb-bell', '062.eiffel-tower', '063.electric-guitar-101', '064.elephant-101', '065.elk', '066.ewer-101', '067.eyeglasses', '068.fern', '069.fighter-jet', '070.fire-extinguisher', '071.fire-hydrant', '072.fire-truck', '073.fireworks', '074.flashlight', '075.floppy-disk', '076.football-helmet', '077.french-horn', '078.fried-egg', '079.frisbee', '080.frog', '081.frying-pan', '082.galaxy', '083.gas-pump', '084.giraffe', '085.goat', '086.golden-gate-bridge', '087.goldfish', '088.golf-ball', '089.goose', '090.gorilla', '091.grand-piano-101', '092.grapes', '093.grasshopper', '094.guitar-pick', '095.hamburger', '096.hammock', '097.harmonica', '098.harp', '099.harpsichord', '100.hawksbill-101', '101.head-phones', '102.helicopter-101', '103.hibiscus', '104.homer-simpson', '105.horse', '106.horseshoe-crab', '107.hot-air-balloon', '108.hot-dog', '109.hot-tub', '110.hourglass', '111.house-fly', '112.human-skeleton', '113.hummingbird', '114.ibis-101', '115.ice-cream-cone', '116.iguana', '117.ipod', '118.iris', '119.jesus-christ', '120.joy-stick', '121.kangaroo-101', '122.kayak', '123.ketch-101', '124.killer-whale', '125.knife', '126.ladder', '127.laptop-101', '128.lathe', '129.leopards-101', '130.license-plate', '131.lightbulb', '132.light-house', '133.lightning', '134.llama-101', '135.mailbox', '136.mandolin', '137.mars', '138.mattress', '139.megaphone', '140.menorah-101', '141.microscope', '142.microwave', '143.minaret', '144.minotaur', '145.motorbikes-101', '146.mountain-bike', '147.mushroom', '148.mussels', '149.necktie', '150.octopus', '151.ostrich', '152.owl', '153.palm-pilot', '154.palm-tree', '155.paperclip', '156.paper-shredder', '157.pci-card', '158.penguin', '159.people', '160.pez-dispenser', '161.photocopier', '162.picnic-table', '163.playing-card', '164.porcupine', '165.pram', '166.praying-mantis', '167.pyramid', '168.raccoon', '169.radio-telescope', '170.rainbow', '171.refrigerator', '172.revolver-101', '173.rifle', '174.rotary-phone', '175.roulette-wheel', '176.saddle', '177.saturn', '178.school-bus', '179.scorpion-101', '180.screwdriver', '181.segway', '182.self-propelled-lawn-mower', '183.sextant', '184.sheet-music', '185.skateboard', '186.skunk', '187.skyscraper', '188.smokestack', '189.snail', '190.snake', '191.sneaker', '192.snowmobile', '193.soccer-ball', '194.socks', '195.soda-can', '196.spaghetti', '197.speed-boat', '198.spider', '199.spoon', '200.stained-glass', '201.starfish-101', '202.steering-wheel', '203.stirrups', '204.sunflower-101', '205.superman', '206.sushi', '207.swan', '208.swiss-army-knife', '209.sword', '210.syringe', '211.tambourine', '212.teapot', '213.teddy-bear', '214.teepee', '215.telephone-box', '216.tennis-ball', '217.tennis-court', '218.tennis-racket', '219.theodolite', '220.toaster', '221.tomato', '222.tombstone', '223.top-hat', '224.touring-bike', '225.tower-pisa', '226.traffic-light', '227.treadmill', '228.triceratops', '229.tricycle', '230.trilobite-101', '231.tripod', '232.t-shirt', '233.tuning-fork', '234.tweezer', '235.umbrella-101', '236.unicorn', '237.vcr', '238.video-projector', '239.washing-machine', '240.watch-101', '241.waterfall', '242.watermelon', '243.welding-mask', '244.wheelbarrow', '245.windmill', '246.wine-bottle', '247.xylophone', '248.yarmulke', '249.yo-yo', '250.zebra', '251.airplanes-101', '252.car-side-101', '253.faces-easy-101', '254.greyhound', '255.tennis-shoes', '256.toad', '257.clutter'] +classes = [c.split('.')[1] for c in classes] + +@dataset_register( + name='Caltech256', + classes=classes, + task_type='Image Classification', + object_type='Generic Object', + class_aliases=[], + shift_type=None +) +class Caltech256(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform: Optional[Compose], + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + if transform is None: + transform = imagenet_like_image_train_aug() if split == 'train' else imagenet_like_image_test_aug() + self.transform = transform + dataset = ImageFolder(root_dir, transform=transform) + + if len(ignore_classes) > 0: + ignore_classes_idx = [classes.index(c) for c in ignore_classes] + dataset.samples = [s for s in dataset.samples if s[1] not in ignore_classes_idx] + + if idx_map is not None: + dataset.samples = [(s[0], idx_map[s[1]]) if s[1] in idx_map.keys() else s for s in dataset.samples] + + dataset = train_val_test_split(dataset, split) + return dataset diff --git a/data/datasets/image_classification/cifar10.py b/data/datasets/image_classification/cifar10.py new file mode 100644 index 0000000000000000000000000000000000000000..c6a56fd878d2f315b93543dc8a2ba10a5a5af093 --- /dev/null +++ b/data/datasets/image_classification/cifar10.py @@ -0,0 +1,47 @@ +from ..data_aug import cifar_like_image_train_aug, cifar_like_image_test_aug +from ..ab_dataset import ABDataset +from ..dataset_split import train_val_split +from torchvision.datasets import CIFAR10 as RawCIFAR10 +import numpy as np +from typing import Dict, List, Optional +from torchvision.transforms import Compose +from utils.common.others import HiddenPrints + +from ..registery import dataset_register + + +@dataset_register( + name='CIFAR10', + classes=['airplane', 'automobile', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck'], + task_type='Image Classification', + object_type='Generic Object', + # class_aliases=[['automobile', 'car']], + class_aliases=[], + shift_type=None +) +class CIFAR10(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform: Optional[Compose], + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + if transform is None: + transform = cifar_like_image_train_aug() if split == 'train' else cifar_like_image_test_aug() + self.transform = transform + with HiddenPrints(): + dataset = RawCIFAR10(root_dir, split != 'test', transform=transform, download=True) + + dataset.targets = np.asarray(dataset.targets) + if len(ignore_classes) > 0: + for ignore_class in ignore_classes: + dataset.data = dataset.data[dataset.targets != classes.index(ignore_class)] + dataset.targets = dataset.targets[dataset.targets != classes.index(ignore_class)] + + if idx_map is not None: + # note: the code below seems correct but has bug! + # for old_idx, new_idx in idx_map.items(): + # dataset.targets[dataset.targets == old_idx] = new_idx + + for ti, t in enumerate(dataset.targets): + dataset.targets[ti] = idx_map[t] + + if split != 'test': + dataset = train_val_split(dataset, split) + return dataset diff --git a/data/datasets/image_classification/cifar10_single.py b/data/datasets/image_classification/cifar10_single.py new file mode 100644 index 0000000000000000000000000000000000000000..ccfc476ee7676b54c7577b2d68a4edb679f8f23a --- /dev/null +++ b/data/datasets/image_classification/cifar10_single.py @@ -0,0 +1,39 @@ +from ..data_aug import cifar_like_image_train_aug, cifar_like_image_test_aug +from ..ab_dataset import ABDataset +from ..dataset_split import train_val_test_split +# from torchvision.datasets import CIFAR10 as RawCIFAR10 +from torchvision.datasets import ImageFolder +import numpy as np +from typing import Dict, List, Optional +from torchvision.transforms import Compose +from utils.common.others import HiddenPrints + +from ..registery import dataset_register + + +@dataset_register( + name='CIFAR10-single', + classes=['airplane', 'automobile', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck'], + task_type='Image Classification', + object_type='Generic Object', + # class_aliases=[['automobile', 'car']], + class_aliases=[], + shift_type=None +) +class CIFAR10Single(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform: Optional[Compose], + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + if transform is None: + transform = cifar_like_image_train_aug() if split == 'train' else cifar_like_image_test_aug() + self.transform = transform + dataset = ImageFolder(root_dir, transform=transform) + + if len(ignore_classes) > 0: + ignore_classes_idx = [classes.index(c) for c in ignore_classes] + dataset.samples = [s for s in dataset.samples if s[1] not in ignore_classes_idx] + + if idx_map is not None: + dataset.samples = [(s[0], idx_map[s[1]]) if s[1] in idx_map.keys() else s for s in dataset.samples] + + dataset = train_val_test_split(dataset, split) + return dataset \ No newline at end of file diff --git a/data/datasets/image_classification/cityscapes_cls.py b/data/datasets/image_classification/cityscapes_cls.py new file mode 100644 index 0000000000000000000000000000000000000000..2631e5b50b1a62b7eb11bac30a25a32b9da96fcf --- /dev/null +++ b/data/datasets/image_classification/cityscapes_cls.py @@ -0,0 +1,72 @@ +from ..data_aug import imagenet_like_image_train_aug, imagenet_like_image_test_aug +from ..ab_dataset import ABDataset +from ..dataset_split import train_val_split, train_val_test_split +from torchvision.datasets import ImageFolder +import os +from typing import Dict, List, Optional +from torchvision.transforms import Compose + +from ..registery import dataset_register + +# with open(os.path.join(os.path.dirname(__file__), 'fruits360_classes.txt'), 'r') as f: +# classes = [line.split(':')[0].strip('"') for line in f.readlines()] +# assert len(classes) == 131 + + +# gta_classes = [ +# 'road', 'sidewalk', 'building', 'wall', +# 'fence', 'pole', 'light', 'sign', +# 'vegetation', 'terrain', 'sky', 'people', # person +# 'rider', 'car', 'truck', 'bus', 'train', +# 'motocycle', 'bicycle' +# ] +# cityscapes_classes = [] + +# ignore_label = 255 +# m = {-1: ignore_label, 0: ignore_label, 1: ignore_label, 2: ignore_label, +# 3: ignore_label, 4: ignore_label, 5: ignore_label, 6: ignore_label, +# 7: 0, 8: 1, 9: ignore_label, 10: ignore_label, 11: 2, 12: 3, 13: 4, +# 14: ignore_label, 15: ignore_label, 16: ignore_label, 17: 5, +# 18: ignore_label, 19: 6, 20: 7, 21: 8, 22: 9, 23: 10, 24: 11, 25: 12, 26: 13, 27: 14, +# 28: 15, 29: ignore_label, 30: ignore_label, 31: 16, 32: 17, 33: 18} + +# for ci, c in enumerate(gta_classes): +# for k, v in m.items(): +# if v == ci: +# cityscapes_classes += [c] +# print(cityscapes_classes) +# exit() + +@dataset_register( + name='CityscapesCls', + classes=[ + 'road', 'sidewalk', 'building', 'wall', + 'fence', 'pole', 'light', 'sign', + 'vegetation', 'terrain', 'sky', 'people', # person + 'rider', 'car', 'truck', 'bus', 'train', + 'motocycle', 'bicycle' + ], + task_type='Image Classification', + object_type='Autonomous Driving', + class_aliases=[], + shift_type=None +) + +class CityscapesCls(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform: Optional[Compose], + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + if transform is None: + transform = imagenet_like_image_train_aug() if split == 'train' else imagenet_like_image_test_aug() + self.transform = transform + #root_dir = os.path.join(root_dir, 'train' if split != 'test' else 'val') + dataset = ImageFolder(root_dir, transform=transform) + + if len(ignore_classes) > 0: + ignore_classes_idx = [classes.index(c) for c in ignore_classes] + dataset.samples = [s for s in dataset.samples if s[1] not in ignore_classes_idx] + + if idx_map is not None: + dataset.samples = [(s[0], idx_map[s[1]]) if s[1] in idx_map.keys() else s for s in dataset.samples] + + dataset = train_val_test_split(dataset, split) + return dataset diff --git a/data/datasets/image_classification/coco_cls.py b/data/datasets/image_classification/coco_cls.py new file mode 100644 index 0000000000000000000000000000000000000000..5df6b972589d62f719b60e9ed6ba77ba6aff7d38 --- /dev/null +++ b/data/datasets/image_classification/coco_cls.py @@ -0,0 +1,40 @@ +from ..data_aug import imagenet_like_image_train_aug, imagenet_like_image_test_aug +from ..ab_dataset import ABDataset +from ..dataset_split import train_val_split, train_val_test_split +from torchvision.datasets import ImageFolder +import os +from typing import Dict, List, Optional +from torchvision.transforms import Compose + +from ..registery import dataset_register + +categories = [{"supercategory": "person","id": 1,"name": "person"},{"supercategory": "vehicle","id": 2,"name": "bicycle"},{"supercategory": "vehicle","id": 3,"name": "car"},{"supercategory": "vehicle","id": 4,"name": "motorcycle"},{"supercategory": "vehicle","id": 5,"name": "airplane"},{"supercategory": "vehicle","id": 6,"name": "bus"},{"supercategory": "vehicle","id": 7,"name": "train"},{"supercategory": "vehicle","id": 8,"name": "truck"},{"supercategory": "vehicle","id": 9,"name": "boat"},{"supercategory": "outdoor","id": 10,"name": "traffic light"},{"supercategory": "outdoor","id": 11,"name": "fire hydrant"},{"supercategory": "outdoor","id": 13,"name": "stop sign"},{"supercategory": "outdoor","id": 14,"name": "parking meter"},{"supercategory": "outdoor","id": 15,"name": "bench"},{"supercategory": "animal","id": 16,"name": "bird"},{"supercategory": "animal","id": 17,"name": "cat"},{"supercategory": "animal","id": 18,"name": "dog"},{"supercategory": "animal","id": 19,"name": "horse"},{"supercategory": "animal","id": 20,"name": "sheep"},{"supercategory": "animal","id": 21,"name": "cow"},{"supercategory": "animal","id": 22,"name": "elephant"},{"supercategory": "animal","id": 23,"name": "bear"},{"supercategory": "animal","id": 24,"name": "zebra"},{"supercategory": "animal","id": 25,"name": "giraffe"},{"supercategory": "accessory","id": 27,"name": "backpack"},{"supercategory": "accessory","id": 28,"name": "umbrella"},{"supercategory": "accessory","id": 31,"name": "handbag"},{"supercategory": "accessory","id": 32,"name": "tie"},{"supercategory": "accessory","id": 33,"name": "suitcase"},{"supercategory": "sports","id": 34,"name": "frisbee"},{"supercategory": "sports","id": 35,"name": "skis"},{"supercategory": "sports","id": 36,"name": "snowboard"},{"supercategory": "sports","id": 37,"name": "sports ball"},{"supercategory": "sports","id": 38,"name": "kite"},{"supercategory": "sports","id": 39,"name": "baseball bat"},{"supercategory": "sports","id": 40,"name": "baseball glove"},{"supercategory": "sports","id": 41,"name": "skateboard"},{"supercategory": "sports","id": 42,"name": "surfboard"},{"supercategory": "sports","id": 43,"name": "tennis racket"},{"supercategory": "kitchen","id": 44,"name": "bottle"},{"supercategory": "kitchen","id": 46,"name": "wine glass"},{"supercategory": "kitchen","id": 47,"name": "cup"},{"supercategory": "kitchen","id": 48,"name": "fork"},{"supercategory": "kitchen","id": 49,"name": "knife"},{"supercategory": "kitchen","id": 50,"name": "spoon"},{"supercategory": "kitchen","id": 51,"name": "bowl"},{"supercategory": "food","id": 52,"name": "banana"},{"supercategory": "food","id": 53,"name": "apple"},{"supercategory": "food","id": 54,"name": "sandwich"},{"supercategory": "food","id": 55,"name": "orange"},{"supercategory": "food","id": 56,"name": "broccoli"},{"supercategory": "food","id": 57,"name": "carrot"},{"supercategory": "food","id": 58,"name": "hot dog"},{"supercategory": "food","id": 59,"name": "pizza"},{"supercategory": "food","id": 60,"name": "donut"},{"supercategory": "food","id": 61,"name": "cake"},{"supercategory": "furniture","id": 62,"name": "chair"},{"supercategory": "furniture","id": 63,"name": "couch"},{"supercategory": "furniture","id": 64,"name": "potted plant"},{"supercategory": "furniture","id": 65,"name": "bed"},{"supercategory": "furniture","id": 67,"name": "dining table"},{"supercategory": "furniture","id": 70,"name": "toilet"},{"supercategory": "electronic","id": 72,"name": "tv"},{"supercategory": "electronic","id": 73,"name": "laptop"},{"supercategory": "electronic","id": 74,"name": "mouse"},{"supercategory": "electronic","id": 75,"name": "remote"},{"supercategory": "electronic","id": 76,"name": "keyboard"},{"supercategory": "electronic","id": 77,"name": "cell phone"},{"supercategory": "appliance","id": 78,"name": "microwave"},{"supercategory": "appliance","id": 79,"name": "oven"},{"supercategory": "appliance","id": 80,"name": "toaster"},{"supercategory": "appliance","id": 81,"name": "sink"},{"supercategory": "appliance","id": 82,"name": "refrigerator"},{"supercategory": "indoor","id": 84,"name": "book"},{"supercategory": "indoor","id": 85,"name": "clock"},{"supercategory": "indoor","id": 86,"name": "vase"},{"supercategory": "indoor","id": 87,"name": "scissors"},{"supercategory": "indoor","id": 88,"name": "teddy bear"},{"supercategory": "indoor","id": 89,"name": "hair drier"},{"supercategory": "indoor","id": 90,"name": "toothbrush"}] +classes = [i['name'] for i in categories] + +@dataset_register( + name='COCOCls', + classes=classes, + task_type='Image Classification', + object_type='Generic Object', + class_aliases=[], + shift_type=None +) + +class COCOCls(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform: Optional[Compose], + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + if transform is None: + transform = imagenet_like_image_train_aug() if split == 'train' else imagenet_like_image_test_aug() + self.transform = transform + #root_dir = os.path.join(root_dir, 'train' if split != 'test' else 'val') + dataset = ImageFolder(root_dir, transform=transform) + + if len(ignore_classes) > 0: + ignore_classes_idx = [classes.index(c) for c in ignore_classes] + dataset.samples = [s for s in dataset.samples if s[1] not in ignore_classes_idx] + + if idx_map is not None: + dataset.samples = [(s[0], idx_map[s[1]]) if s[1] in idx_map.keys() else s for s in dataset.samples] + + dataset = train_val_test_split(dataset, split) + return dataset diff --git a/data/datasets/image_classification/domainnet_real.py b/data/datasets/image_classification/domainnet_real.py new file mode 100644 index 0000000000000000000000000000000000000000..c25c2b758887065a2769be798f6fea1f02bdeee5 --- /dev/null +++ b/data/datasets/image_classification/domainnet_real.py @@ -0,0 +1,36 @@ +from ..data_aug import imagenet_like_image_train_aug, imagenet_like_image_test_aug +from ..ab_dataset import ABDataset +from ..dataset_split import train_val_split, train_val_test_split +from torchvision.datasets import ImageFolder +import os +from typing import Dict, List, Optional +from torchvision.transforms import Compose + +from ..registery import dataset_register + + +@dataset_register( + name='DomainNet (real)', + classes=['The_Eiffel_Tower', 'The_Great_Wall_of_China', 'The_Mona_Lisa', 'aircraft_carrier', 'airplane', 'alarm_clock', 'ambulance', 'angel', 'animal_migration', 'ant', 'anvil', 'apple', 'arm', 'asparagus', 'axe', 'backpack', 'banana', 'bandage', 'barn', 'baseball', 'baseball_bat', 'basket', 'basketball', 'bat', 'bathtub', 'beach', 'bear', 'beard', 'bed', 'bee', 'belt', 'bench', 'bicycle', 'binoculars', 'bird', 'birthday_cake', 'blackberry', 'blueberry', 'book', 'boomerang', 'bottlecap', 'bowtie', 'bracelet', 'brain', 'bread', 'bridge', 'broccoli', 'broom', 'bucket', 'bulldozer', 'bus', 'bush', 'butterfly', 'cactus', 'cake', 'calculator', 'calendar', 'camel', 'camera', 'camouflage', 'campfire', 'candle', 'cannon', 'canoe', 'car', 'carrot', 'castle', 'cat', 'ceiling_fan', 'cell_phone', 'cello', 'chair', 'chandelier', 'church', 'circle', 'clarinet', 'clock', 'cloud', 'coffee_cup', 'compass', 'computer', 'cookie', 'cooler', 'couch', 'cow', 'crab', 'crayon', 'crocodile', 'crown', 'cruise_ship', 'cup', 'diamond', 'dishwasher', 'diving_board', 'dog', 'dolphin', 'donut', 'door', 'dragon', 'dresser', 'drill', 'drums', 'duck', 'dumbbell', 'ear', 'elbow', 'elephant', 'envelope', 'eraser', 'eye', 'eyeglasses', 'face', 'fan', 'feather', 'fence', 'finger', 'fire_hydrant', 'fireplace', 'firetruck', 'fish', 'flamingo', 'flashlight', 'flip_flops', 'floor_lamp', 'flower', 'flying_saucer', 'foot', 'fork', 'frog', 'frying_pan', 'garden', 'garden_hose', 'giraffe', 'goatee', 'golf_club', 'grapes', 'grass', 'guitar', 'hamburger', 'hammer', 'hand', 'harp', 'hat', 'headphones', 'hedgehog', 'helicopter', 'helmet', 'hexagon', 'hockey_puck', 'hockey_stick', 'horse', 'hospital', 'hot_air_balloon', 'hot_dog', 'hot_tub', 'hourglass', 'house', 'house_plant', 'hurricane', 'ice_cream', 'jacket', 'jail', 'kangaroo', 'key', 'keyboard', 'knee', 'knife', 'ladder', 'lantern', 'laptop', 'leaf', 'leg', 'light_bulb', 'lighter', 'lighthouse', 'lightning', 'line', 'lion', 'lipstick', 'lobster', 'lollipop', 'mailbox', 'map', 'marker', 'matches', 'megaphone', 'mermaid', 'microphone', 'microwave', 'monkey', 'moon', 'mosquito', 'motorbike', 'mountain', 'mouse', 'moustache', 'mouth', 'mug', 'mushroom', 'nail', 'necklace', 'nose', 'ocean', 'octagon', 'octopus', 'onion', 'oven', 'owl', 'paint_can', 'paintbrush', 'palm_tree', 'panda', 'pants', 'paper_clip', 'parachute', 'parrot', 'passport', 'peanut', 'pear', 'peas', 'pencil', 'penguin', 'piano', 'pickup_truck', 'picture_frame', 'pig', 'pillow', 'pineapple', 'pizza', 'pliers', 'police_car', 'pond', 'pool', 'popsicle', 'postcard', 'potato', 'power_outlet', 'purse', 'rabbit', 'raccoon', 'radio', 'rain', 'rainbow', 'rake', 'remote_control', 'rhinoceros', 'rifle', 'river', 'roller_coaster', 'rollerskates', 'sailboat', 'sandwich', 'saw', 'saxophone', 'school_bus', 'scissors', 'scorpion', 'screwdriver', 'sea_turtle', 'see_saw', 'shark', 'sheep', 'shoe', 'shorts', 'shovel', 'sink', 'skateboard', 'skull', 'skyscraper', 'sleeping_bag', 'smiley_face', 'snail', 'snake', 'snorkel', 'snowflake', 'snowman', 'soccer_ball', 'sock', 'speedboat', 'spider', 'spoon', 'spreadsheet', 'square', 'squiggle', 'squirrel', 'stairs', 'star', 'steak', 'stereo', 'stethoscope', 'stitches', 'stop_sign', 'stove', 'strawberry', 'streetlight', 'string_bean', 'submarine', 'suitcase', 'sun', 'swan', 'sweater', 'swing_set', 'sword', 'syringe', 't-shirt', 'table', 'teapot', 'teddy-bear', 'telephone', 'television', 'tennis_racquet', 'tent', 'tiger', 'toaster', 'toe', 'toilet', 'tooth', 'toothbrush', 'toothpaste', 'tornado', 'tractor', 'traffic_light', 'train', 'tree', 'triangle', 'trombone', 'truck', 'trumpet', 'umbrella', 'underwear', 'van', 'vase', 'violin', 'washing_machine', 'watermelon', 'waterslide', 'whale', 'wheel', 'windmill', 'wine_bottle', 'wine_glass', 'wristwatch', 'yoga', 'zebra', 'zigzag'], + task_type='Image Classification', + object_type='Generic Object', + class_aliases=[], + shift_type=None +) +class DomainNetReal(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform: Optional[Compose], + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + if transform is None: + transform = imagenet_like_image_train_aug() if split == 'train' else imagenet_like_image_test_aug() + self.transform = transform + dataset = ImageFolder(root_dir, transform=transform) + + if len(ignore_classes) > 0: + ignore_classes_idx = [classes.index(c) for c in ignore_classes] + dataset.samples = [s for s in dataset.samples if s[1] not in ignore_classes_idx] + + if idx_map is not None: + dataset.samples = [(s[0], idx_map[s[1]]) if s[1] in idx_map.keys() else s for s in dataset.samples] + + dataset = train_val_test_split(dataset, split) + return dataset \ No newline at end of file diff --git a/data/datasets/image_classification/emnist.py b/data/datasets/image_classification/emnist.py new file mode 100644 index 0000000000000000000000000000000000000000..26055807bef8087b8d64a22c6f9c5ed2a61db235 --- /dev/null +++ b/data/datasets/image_classification/emnist.py @@ -0,0 +1,47 @@ +import enum +from ..data_aug import one_d_image_test_aug, one_d_image_train_aug +from ..ab_dataset import ABDataset +from ..dataset_split import train_val_split +from torchvision.datasets import EMNIST as RawEMNIST +import string +import numpy as np +from typing import Dict, List, Optional +from torchvision.transforms import Compose + +from ..registery import dataset_register + + +@dataset_register( + name='EMNIST', + classes=list(string.digits + string.ascii_letters), + class_aliases=[], + task_type='Image Classification', + object_type='Digit and Letter', + shift_type=None +) +class EMNIST(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform: Optional[Compose], + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + if transform is None: + transform = one_d_image_train_aug() if split == 'train' else one_d_image_test_aug() + self.transform = transform + dataset = RawEMNIST(root_dir, 'byclass', train=split != 'test', transform=transform, download=True) + + dataset.targets = np.asarray(dataset.targets) + + if len(ignore_classes) > 0: + for ignore_class in ignore_classes: + dataset.data = dataset.data[dataset.targets != classes.index(ignore_class)] + dataset.targets = dataset.targets[dataset.targets != classes.index(ignore_class)] + + if idx_map is not None: + # note: the code below seems correct but has bug! + # for old_idx, new_idx in idx_map.items(): + # dataset.targets[dataset.targets == old_idx] = new_idx + + for ti, t in enumerate(dataset.targets): + dataset.targets[ti] = idx_map[t] + + if split != 'test': + dataset = train_val_split(dataset, split) + return dataset diff --git a/data/datasets/image_classification/eurosat.py b/data/datasets/image_classification/eurosat.py new file mode 100644 index 0000000000000000000000000000000000000000..2db441a336f1cd1675fb0d6070104bbd495400c2 --- /dev/null +++ b/data/datasets/image_classification/eurosat.py @@ -0,0 +1,37 @@ +from ..data_aug import imagenet_like_image_train_aug, imagenet_like_image_test_aug +from ..ab_dataset import ABDataset +from ..dataset_split import train_val_split, train_val_test_split +from torchvision.datasets import ImageFolder +import os +from typing import Dict, List, Optional +from torchvision.transforms import Compose + +from ..registery import dataset_register + +@dataset_register( + name='EuroSAT', + classes=['AnnualCrop', 'Forest', 'HerbaceousVegetation', 'Highway', 'Industrial', 'Pasture', 'PermanentCrop', 'Residential', 'River', 'SeaLake'], + task_type='Image Classification', + object_type='Remote Sensing', + class_aliases=[], + shift_type=None +) + +class EuroSAT(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform: Optional[Compose], + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + if transform is None: + transform = imagenet_like_image_train_aug() if split == 'train' else imagenet_like_image_test_aug() + self.transform = transform + #root_dir = os.path.join(root_dir, 'train' if split != 'test' else 'val') + dataset = ImageFolder(root_dir, transform=transform) + + if len(ignore_classes) > 0: + ignore_classes_idx = [classes.index(c) for c in ignore_classes] + dataset.samples = [s for s in dataset.samples if s[1] not in ignore_classes_idx] + + if idx_map is not None: + dataset.samples = [(s[0], idx_map[s[1]]) if s[1] in idx_map.keys() else s for s in dataset.samples] + + dataset = train_val_test_split(dataset, split) + return dataset diff --git a/data/datasets/image_classification/fashionmnist.py b/data/datasets/image_classification/fashionmnist.py new file mode 100644 index 0000000000000000000000000000000000000000..d2016dbb3f4fd9ac07521115c89609cbff6fe04f --- /dev/null +++ b/data/datasets/image_classification/fashionmnist.py @@ -0,0 +1,38 @@ +from ..data_aug import imagenet_like_image_train_aug, imagenet_like_image_test_aug +from ..ab_dataset import ABDataset +from ..dataset_split import train_val_split, train_val_test_split +from torchvision.datasets import ImageFolder +import os +from typing import Dict, List, Optional +from torchvision.transforms import Compose + +from ..registery import dataset_register + +@dataset_register( + name='FashionMNIST', + classes=['t-shirt', 'trouser', 'pullover', 'dress', 'coat', 'sandal', 'shirt', 'sneaker', 'bag', 'ankle boot'], + task_type='Image Classification', + object_type='Fashion Product', + class_aliases=[], + shift_type=None +) + +class FashionMNIST(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform: Optional[Compose], + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + if transform is None: + transform = imagenet_like_image_train_aug() if split == 'train' else imagenet_like_image_test_aug() + self.transform = transform + #root_dir = os.path.join(root_dir, 'train' if split != 'test' else 'val') + dataset = ImageFolder(root_dir, transform=transform) + + if len(ignore_classes) > 0: + ignore_classes_idx = [classes.index(c) for c in ignore_classes] + dataset.samples = [s for s in dataset.samples if s[1] not in ignore_classes_idx] + + if idx_map is not None: + dataset.samples = [(s[0], idx_map[s[1]]) if s[1] in idx_map.keys() else s for s in dataset.samples] + + dataset = train_val_test_split(dataset, split) + return dataset + diff --git a/data/datasets/image_classification/fruits360.py b/data/datasets/image_classification/fruits360.py new file mode 100644 index 0000000000000000000000000000000000000000..0fd6e3c38d843db3b03446088d40e803af356a10 --- /dev/null +++ b/data/datasets/image_classification/fruits360.py @@ -0,0 +1,41 @@ +from ..data_aug import imagenet_like_image_train_aug, imagenet_like_image_test_aug +from ..ab_dataset import ABDataset +from ..dataset_split import train_val_split, train_val_test_split +from torchvision.datasets import ImageFolder +import os +from typing import Dict, List, Optional +from torchvision.transforms import Compose + +from ..registery import dataset_register + +with open(os.path.join(os.path.dirname(__file__), 'fruits360_classes.txt'), 'r') as f: + classes = [line.split(':')[0].strip('"') for line in f.readlines()] + assert len(classes) == 131 + +@dataset_register( + name='Fruits360', + classes=classes, + task_type='Image Classification', + object_type='Fruit and Vegatable', + class_aliases=[], + shift_type=None +) + +class Fruits360(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform: Optional[Compose], + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + if transform is None: + transform = imagenet_like_image_train_aug() if split == 'train' else imagenet_like_image_test_aug() + self.transform = transform + #root_dir = os.path.join(root_dir, 'train' if split != 'test' else 'val') + dataset = ImageFolder(root_dir, transform=transform) + + if len(ignore_classes) > 0: + ignore_classes_idx = [classes.index(c) for c in ignore_classes] + dataset.samples = [s for s in dataset.samples if s[1] not in ignore_classes_idx] + + if idx_map is not None: + dataset.samples = [(s[0], idx_map[s[1]]) if s[1] in idx_map.keys() else s for s in dataset.samples] + + dataset = train_val_test_split(dataset, split) + return dataset diff --git a/data/datasets/image_classification/fruits360_classes.txt b/data/datasets/image_classification/fruits360_classes.txt new file mode 100644 index 0000000000000000000000000000000000000000..e7d02978af135efe87896f2131a625f7af8ef7ae --- /dev/null +++ b/data/datasets/image_classification/fruits360_classes.txt @@ -0,0 +1,131 @@ +"Apple Braeburn": 164, +"Apple Crimson Snow": 148, +"Apple Golden 1": 160, +"Apple Golden 2": 164, +"Apple Golden 3": 161, +"Apple Granny Smith": 164, +"Apple Pink Lady": 152, +"Apple Red 1": 164, +"Apple Red 2": 164, +"Apple Red 3": 144, +"Apple Red Delicious": 166, +"Apple Red Yellow 1": 164, +"Apple Red Yellow 2": 219, +"Apricot": 164, +"Avocado": 143, +"Avocado ripe": 166, +"Banana": 166, +"Banana Lady Finger": 152, +"Banana Red": 166, +"Beetroot": 150, +"Blueberry": 154, +"Cactus fruit": 166, +"Cantaloupe 1": 164, +"Cantaloupe 2": 164, +"Carambula": 166, +"Cauliflower": 234, +"Cherry 1": 164, +"Cherry 2": 246, +"Cherry Rainier": 246, +"Cherry Wax Black": 164, +"Cherry Wax Red": 164, +"Cherry Wax Yellow": 164, +"Chestnut": 153, +"Clementine": 166, +"Cocos": 166, +"Corn": 150, +"Corn Husk": 154, +"Cucumber Ripe": 130, +"Cucumber Ripe 2": 156, +"Dates": 166, +"Eggplant": 156, +"Fig": 234, +"Ginger Root": 99, +"Granadilla": 166, +"Grape Blue": 328, +"Grape Pink": 164, +"Grape White": 166, +"Grape White 2": 166, +"Grape White 3": 164, +"Grape White 4": 158, +"Grapefruit Pink": 166, +"Grapefruit White": 164, +"Guava": 166, +"Hazelnut": 157, +"Huckleberry": 166, +"Kaki": 166, +"Kiwi": 156, +"Kohlrabi": 157, +"Kumquats": 166, +"Lemon": 164, +"Lemon Meyer": 166, +"Limes": 166, +"Lychee": 166, +"Mandarine": 166, +"Mango": 166, +"Mango Red": 142, +"Mangostan": 102, +"Maracuja": 166, +"Melon Piel de Sapo": 246, +"Mulberry": 164, +"Nectarine": 164, +"Nectarine Flat": 160, +"Nut Forest": 218, +"Nut Pecan": 178, +"Onion Red": 150, +"Onion Red Peeled": 155, +"Onion White": 146, +"Orange": 160, +"Papaya": 164, +"Passion Fruit": 166, +"Peach": 164, +"Peach 2": 246, +"Peach Flat": 164, +"Pear": 164, +"Pear 2": 232, +"Pear Abate": 166, +"Pear Forelle": 234, +"Pear Kaiser": 102, +"Pear Monster": 166, +"Pear Red": 222, +"Pear Stone": 237, +"Pear Williams": 166, +"Pepino": 166, +"Pepper Green": 148, +"Pepper Orange": 234, +"Pepper Red": 222, +"Pepper Yellow": 222, +"Physalis": 164, +"Physalis with Husk": 164, +"Pineapple": 166, +"Pineapple Mini": 163, +"Pitahaya Red": 166, +"Plum": 151, +"Plum 2": 142, +"Plum 3": 304, +"Pomegranate": 164, +"Pomelo Sweetie": 153, +"Potato Red": 150, +"Potato Red Washed": 151, +"Potato Sweet": 150, +"Potato White": 150, +"Quince": 166, +"Rambutan": 164, +"Raspberry": 166, +"Redcurrant": 164, +"Salak": 162, +"Strawberry": 164, +"Strawberry Wedge": 246, +"Tamarillo": 166, +"Tangelo": 166, +"Tomato 1": 246, +"Tomato 2": 225, +"Tomato 3": 246, +"Tomato 4": 160, +"Tomato Cherry Red": 164, +"Tomato Heart": 228, +"Tomato Maroon": 127, +"Tomato not Ripened": 158, +"Tomato Yellow": 153, +"Walnut": 249, +"Watermelon": 157 \ No newline at end of file diff --git a/data/datasets/image_classification/gta5_cls.py b/data/datasets/image_classification/gta5_cls.py new file mode 100644 index 0000000000000000000000000000000000000000..52a84768c994d39a919d7d6b99cde48240b45d14 --- /dev/null +++ b/data/datasets/image_classification/gta5_cls.py @@ -0,0 +1,47 @@ +from ..data_aug import imagenet_like_image_train_aug, imagenet_like_image_test_aug +from ..ab_dataset import ABDataset +from ..dataset_split import train_val_split, train_val_test_split +from torchvision.datasets import ImageFolder +import os +from typing import Dict, List, Optional +from torchvision.transforms import Compose + +from ..registery import dataset_register + +# with open(os.path.join(os.path.dirname(__file__), 'fruits360_classes.txt'), 'r') as f: +# classes = [line.split(':')[0].strip('"') for line in f.readlines()] +# assert len(classes) == 131 + +@dataset_register( + name='GTA5Cls', + classes=[ + 'road', 'sidewalk', 'building', 'wall', + 'fence', 'pole', 'light', 'sign', + 'vegetation', 'terrain', 'sky', 'people', # person + 'rider', 'car', 'truck', 'bus', 'train', + 'motocycle', 'bicycle' + ], + task_type='Image Classification', + object_type='Autonomous Driving', + class_aliases=[], + shift_type=None +) + +class GTA5Cls(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform: Optional[Compose], + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + if transform is None: + transform = imagenet_like_image_train_aug() if split == 'train' else imagenet_like_image_test_aug() + self.transform = transform + #root_dir = os.path.join(root_dir, 'train' if split != 'test' else 'val') + dataset = ImageFolder(root_dir, transform=transform) + + if len(ignore_classes) > 0: + ignore_classes_idx = [classes.index(c) for c in ignore_classes] + dataset.samples = [s for s in dataset.samples if s[1] not in ignore_classes_idx] + + if idx_map is not None: + dataset.samples = [(s[0], idx_map[s[1]]) if s[1] in idx_map.keys() else s for s in dataset.samples] + + dataset = train_val_test_split(dataset, split) + return dataset diff --git a/data/datasets/image_classification/gtsrb.py b/data/datasets/image_classification/gtsrb.py new file mode 100644 index 0000000000000000000000000000000000000000..7386d11ae18aa52d480ddc972f23fb31dfd22a42 --- /dev/null +++ b/data/datasets/image_classification/gtsrb.py @@ -0,0 +1,39 @@ +from ..data_aug import cifar_like_image_train_aug, cifar_like_image_test_aug, imagenet_like_image_test_aug, imagenet_like_image_train_aug +from ..ab_dataset import ABDataset +from ..dataset_split import train_val_split, train_val_test_split +from torchvision.datasets import ImageFolder +from typing import Dict, List, Optional +from torchvision.transforms import Compose +from utils.common.others import HiddenPrints +import os + +from ..registery import dataset_register + + +@dataset_register( + name='GTSRB', + classes=[f'{i:05d}' for i in range(42)], + task_type='Image Classification', + object_type='Traffic Sign', + class_aliases=[], + shift_type=None +) +class GTSRB(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform: Optional[Compose], + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + if transform is None: + transform = imagenet_like_image_train_aug() if split == 'train' else imagenet_like_image_test_aug() + self.transform = transform + + dataset = ImageFolder(os.path.join(root_dir, 'Final_Training/Images/') if split != 'test' else os.path.join(root_dir, 'Final_Test_ImageFolder/'), transform=transform) + + if len(ignore_classes) > 0: + ignore_classes_idx = [classes.index(c) for c in ignore_classes] + dataset.samples = [s for s in dataset.samples if s[1] not in ignore_classes_idx] + + if idx_map is not None: + dataset.samples = [(s[0], idx_map[s[1]]) if s[1] in idx_map.keys() else s for s in dataset.samples] + + if split != 'test': + dataset = train_val_split(dataset, split) + return dataset diff --git a/data/datasets/image_classification/imagenet.py b/data/datasets/image_classification/imagenet.py new file mode 100644 index 0000000000000000000000000000000000000000..1cf3d98be99e7e3b828f7f8e8b676ea8fe3599cc --- /dev/null +++ b/data/datasets/image_classification/imagenet.py @@ -0,0 +1,48 @@ +from ..data_aug import imagenet_like_image_train_aug, imagenet_like_image_test_aug +from ..ab_dataset import ABDataset +from ..dataset_split import train_val_split +from ..dataset_cache import get_dataset_cache_path, read_cached_dataset_status, cache_dataset_status +from torchvision.datasets import ImageFolder +import os +from typing import Dict, List, Optional +from torchvision.transforms import Compose + +from ..registery import dataset_register + + +with open(os.path.join(os.path.dirname(__file__), 'imagenet_classes.txt'), 'r') as f: + classes = [line.split(' ')[2].strip() for line in f.readlines()] + assert len(classes) == 1000 + +@dataset_register( + name='ImageNet', + classes=classes, + task_type='Image Classification', + object_type='Generic Object', + class_aliases=[], + shift_type=None +) +class ImageNet(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform: Optional[Compose], + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + if transform is None: + transform = imagenet_like_image_train_aug() if split == 'train' else imagenet_like_image_test_aug() + self.transform = transform + root_dir = os.path.join(root_dir, 'train' if split != 'test' else 'val') + dataset = ImageFolder(root_dir, transform=transform) + + cache_file_path = get_dataset_cache_path(root_dir, classes, ignore_classes, idx_map) + if os.path.exists(cache_file_path): + dataset.samples = read_cached_dataset_status(cache_file_path, 'ImageNet-' + split) + else: + if len(ignore_classes) > 0: + ignore_classes_idx = [classes.index(c) for c in ignore_classes] + dataset.samples = [s for s in dataset.samples if s[1] not in ignore_classes_idx] + if idx_map is not None: + dataset.samples = [(s[0], idx_map[s[1]]) if s[1] in idx_map.keys() else s for s in dataset.samples] + + cache_dataset_status(dataset.samples, cache_file_path, 'ImageNet-' + split) + + if split != 'test': + dataset = train_val_split(dataset, split) + return dataset diff --git a/data/datasets/image_classification/imagenet_a.py b/data/datasets/image_classification/imagenet_a.py new file mode 100644 index 0000000000000000000000000000000000000000..db5f6f8ddc68648f805f4bc2313281e754282719 --- /dev/null +++ b/data/datasets/image_classification/imagenet_a.py @@ -0,0 +1,45 @@ +from ..data_aug import imagenet_like_image_train_aug, imagenet_like_image_test_aug +from ..ab_dataset import ABDataset +from ..dataset_split import train_val_split +from torchvision.datasets import ImageFolder +import os +from typing import Dict, List, Optional +from torchvision.transforms import Compose + +from ..registery import dataset_register + + +with open(os.path.join(os.path.dirname(__file__), 'imagenet_classes.txt'), 'r') as f: + classes = [line.split(' ')[2].strip() for line in f.readlines()] + assert len(classes) == 1000 + +@dataset_register( + name='ImageNet-A', + classes=classes, + task_type='Image Classification', + object_type='Generic Object', + class_aliases=[], + shift_type={ + 'ImageNet': 'Adversarially Filtered Shifts' # for ImageNet, ImageNet-A causes "Adversarially Filtered Shifts" + } +) +class ImageNetA(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform: Optional[Compose], + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + # TODO: just for scenario building test + if transform is None: + transform = imagenet_like_image_train_aug() if split == 'train' else imagenet_like_image_test_aug() + self.transform = transform + root_dir = os.path.join(root_dir, 'train' if split != 'test' else 'val') + dataset = ImageFolder(root_dir, transform=transform) + + if len(ignore_classes) > 0: + ignore_classes_idx = [classes.index(c) for c in ignore_classes] + dataset.samples = [s for s in dataset.samples if s[1] not in ignore_classes_idx] + + if idx_map is not None: + dataset.samples = [(s[0], idx_map[s[1]]) if s[1] in idx_map.keys() else s for s in dataset.samples] + + if split != 'test': + dataset = train_val_split(dataset, split) + return dataset diff --git a/data/datasets/image_classification/imagenet_classes.txt b/data/datasets/image_classification/imagenet_classes.txt new file mode 100644 index 0000000000000000000000000000000000000000..0f1b228c88cb4d56e720e546f9b8b55062ceb4cd --- /dev/null +++ b/data/datasets/image_classification/imagenet_classes.txt @@ -0,0 +1,1000 @@ +n01440764 0 tench +n01443537 1 goldfish +n01484850 2 great_white_shark +n01491361 3 tiger_shark +n01494475 4 hammerhead +n01496331 5 electric_ray +n01498041 6 stingray +n01514668 7 cock +n01514859 8 hen +n01518878 9 ostrich +n01530575 10 brambling +n01531178 11 goldfinch +n01532829 12 house_finch +n01534433 13 junco +n01537544 14 indigo_bunting +n01558993 15 robin +n01560419 16 bulbul +n01580077 17 jay +n01582220 18 magpie +n01592084 19 chickadee +n01601694 20 water_ouzel +n01608432 21 kite +n01614925 22 bald_eagle +n01616318 23 vulture +n01622779 24 great_grey_owl +n01629819 25 European_fire_salamander +n01630670 26 common_newt +n01631663 27 eft +n01632458 28 spotted_salamander +n01632777 29 axolotl +n01641577 30 bullfrog +n01644373 31 tree_frog +n01644900 32 tailed_frog +n01664065 33 loggerhead +n01665541 34 leatherback_turtle +n01667114 35 mud_turtle +n01667778 36 terrapin +n01669191 37 box_turtle +n01675722 38 banded_gecko +n01677366 39 common_iguana +n01682714 40 American_chameleon +n01685808 41 whiptail +n01687978 42 agama +n01688243 43 frilled_lizard +n01689811 44 alligator_lizard +n01692333 45 Gila_monster +n01693334 46 green_lizard +n01694178 47 African_chameleon +n01695060 48 Komodo_dragon +n01697457 49 African_crocodile +n01698640 50 American_alligator +n01704323 51 triceratops +n01728572 52 thunder_snake +n01728920 53 ringneck_snake +n01729322 54 hognose_snake +n01729977 55 green_snake +n01734418 56 king_snake +n01735189 57 garter_snake +n01737021 58 water_snake +n01739381 59 vine_snake +n01740131 60 night_snake +n01742172 61 boa_constrictor +n01744401 62 rock_python +n01748264 63 Indian_cobra +n01749939 64 green_mamba +n01751748 65 sea_snake +n01753488 66 horned_viper +n01755581 67 diamondback +n01756291 68 sidewinder +n01768244 69 trilobite +n01770081 70 harvestman +n01770393 71 scorpion +n01773157 72 black_and_gold_garden_spider +n01773549 73 barn_spider +n01773797 74 garden_spider +n01774384 75 black_widow +n01774750 76 tarantula +n01775062 77 wolf_spider +n01776313 78 tick +n01784675 79 centipede +n01795545 80 black_grouse +n01796340 81 ptarmigan +n01797886 82 ruffed_grouse +n01798484 83 prairie_chicken +n01806143 84 peacock +n01806567 85 quail +n01807496 86 partridge +n01817953 87 African_grey +n01818515 88 macaw +n01819313 89 sulphur-crested_cockatoo +n01820546 90 lorikeet +n01824575 91 coucal +n01828970 92 bee_eater +n01829413 93 hornbill +n01833805 94 hummingbird +n01843065 95 jacamar +n01843383 96 toucan +n01847000 97 drake +n01855032 98 red-breasted_merganser +n01855672 99 goose +n01860187 100 black_swan +n01871265 101 tusker +n01872401 102 echidna +n01873310 103 platypus +n01877812 104 wallaby +n01882714 105 koala +n01883070 106 wombat +n01910747 107 jellyfish +n01914609 108 sea_anemone +n01917289 109 brain_coral +n01924916 110 flatworm +n01930112 111 nematode +n01943899 112 conch +n01944390 113 snail +n01945685 114 slug +n01950731 115 sea_slug +n01955084 116 chiton +n01968897 117 chambered_nautilus +n01978287 118 Dungeness_crab +n01978455 119 rock_crab +n01980166 120 fiddler_crab +n01981276 121 king_crab +n01983481 122 American_lobster +n01984695 123 spiny_lobster +n01985128 124 crayfish +n01986214 125 hermit_crab +n01990800 126 isopod +n02002556 127 white_stork +n02002724 128 black_stork +n02006656 129 spoonbill +n02007558 130 flamingo +n02009229 131 little_blue_heron +n02009912 132 American_egret +n02011460 133 bittern +n02012849 134 crane1 +n02013706 135 limpkin +n02017213 136 European_gallinule +n02018207 137 American_coot +n02018795 138 bustard +n02025239 139 ruddy_turnstone +n02027492 140 red-backed_sandpiper +n02028035 141 redshank +n02033041 142 dowitcher +n02037110 143 oystercatcher +n02051845 144 pelican +n02056570 145 king_penguin +n02058221 146 albatross +n02066245 147 grey_whale +n02071294 148 killer_whale +n02074367 149 dugong +n02077923 150 sea_lion +n02085620 151 Chihuahua +n02085782 152 Japanese_spaniel +n02085936 153 Maltese_dog +n02086079 154 Pekinese +n02086240 155 Shih-Tzu +n02086646 156 Blenheim_spaniel +n02086910 157 papillon +n02087046 158 toy_terrier +n02087394 159 Rhodesian_ridgeback +n02088094 160 Afghan_hound +n02088238 161 basset +n02088364 162 beagle +n02088466 163 bloodhound +n02088632 164 bluetick +n02089078 165 black-and-tan_coonhound +n02089867 166 Walker_hound +n02089973 167 English_foxhound +n02090379 168 redbone +n02090622 169 borzoi +n02090721 170 Irish_wolfhound +n02091032 171 Italian_greyhound +n02091134 172 whippet +n02091244 173 Ibizan_hound +n02091467 174 Norwegian_elkhound +n02091635 175 otterhound +n02091831 176 Saluki +n02092002 177 Scottish_deerhound +n02092339 178 Weimaraner +n02093256 179 Staffordshire_bullterrier +n02093428 180 American_Staffordshire_terrier +n02093647 181 Bedlington_terrier +n02093754 182 Border_terrier +n02093859 183 Kerry_blue_terrier +n02093991 184 Irish_terrier +n02094114 185 Norfolk_terrier +n02094258 186 Norwich_terrier +n02094433 187 Yorkshire_terrier +n02095314 188 wire-haired_fox_terrier +n02095570 189 Lakeland_terrier +n02095889 190 Sealyham_terrier +n02096051 191 Airedale +n02096177 192 cairn +n02096294 193 Australian_terrier +n02096437 194 Dandie_Dinmont +n02096585 195 Boston_bull +n02097047 196 miniature_schnauzer +n02097130 197 giant_schnauzer +n02097209 198 standard_schnauzer +n02097298 199 Scotch_terrier +n02097474 200 Tibetan_terrier +n02097658 201 silky_terrier +n02098105 202 soft-coated_wheaten_terrier +n02098286 203 West_Highland_white_terrier +n02098413 204 Lhasa +n02099267 205 flat-coated_retriever +n02099429 206 curly-coated_retriever +n02099601 207 golden_retriever +n02099712 208 Labrador_retriever +n02099849 209 Chesapeake_Bay_retriever +n02100236 210 German_short-haired_pointer +n02100583 211 vizsla +n02100735 212 English_setter +n02100877 213 Irish_setter +n02101006 214 Gordon_setter +n02101388 215 Brittany_spaniel +n02101556 216 clumber +n02102040 217 English_springer +n02102177 218 Welsh_springer_spaniel +n02102318 219 cocker_spaniel +n02102480 220 Sussex_spaniel +n02102973 221 Irish_water_spaniel +n02104029 222 kuvasz +n02104365 223 schipperke +n02105056 224 groenendael +n02105162 225 malinois +n02105251 226 briard +n02105412 227 kelpie +n02105505 228 komondor +n02105641 229 Old_English_sheepdog +n02105855 230 Shetland_sheepdog +n02106030 231 collie +n02106166 232 Border_collie +n02106382 233 Bouvier_des_Flandres +n02106550 234 Rottweiler +n02106662 235 German_shepherd +n02107142 236 Doberman +n02107312 237 miniature_pinscher +n02107574 238 Greater_Swiss_Mountain_dog +n02107683 239 Bernese_mountain_dog +n02107908 240 Appenzeller +n02108000 241 EntleBucher +n02108089 242 boxer +n02108422 243 bull_mastiff +n02108551 244 Tibetan_mastiff +n02108915 245 French_bulldog +n02109047 246 Great_Dane +n02109525 247 Saint_Bernard +n02109961 248 Eskimo_dog +n02110063 249 malamute +n02110185 250 Siberian_husky +n02110341 251 dalmatian +n02110627 252 affenpinscher +n02110806 253 basenji +n02110958 254 pug +n02111129 255 Leonberg +n02111277 256 Newfoundland +n02111500 257 Great_Pyrenees +n02111889 258 Samoyed +n02112018 259 Pomeranian +n02112137 260 chow +n02112350 261 keeshond +n02112706 262 Brabancon_griffon +n02113023 263 Pembroke +n02113186 264 Cardigan +n02113624 265 toy_poodle +n02113712 266 miniature_poodle +n02113799 267 standard_poodle +n02113978 268 Mexican_hairless +n02114367 269 timber_wolf +n02114548 270 white_wolf +n02114712 271 red_wolf +n02114855 272 coyote +n02115641 273 dingo +n02115913 274 dhole +n02116738 275 African_hunting_dog +n02117135 276 hyena +n02119022 277 red_fox +n02119789 278 kit_fox +n02120079 279 Arctic_fox +n02120505 280 grey_fox +n02123045 281 tabby +n02123159 282 tiger_cat +n02123394 283 Persian_cat +n02123597 284 Siamese_cat +n02124075 285 Egyptian_cat +n02125311 286 cougar +n02127052 287 lynx +n02128385 288 leopard +n02128757 289 snow_leopard +n02128925 290 jaguar +n02129165 291 lion +n02129604 292 tiger +n02130308 293 cheetah +n02132136 294 brown_bear +n02133161 295 American_black_bear +n02134084 296 ice_bear +n02134418 297 sloth_bear +n02137549 298 mongoose +n02138441 299 meerkat +n02165105 300 tiger_beetle +n02165456 301 ladybug +n02167151 302 ground_beetle +n02168699 303 long-horned_beetle +n02169497 304 leaf_beetle +n02172182 305 dung_beetle +n02174001 306 rhinoceros_beetle +n02177972 307 weevil +n02190166 308 fly +n02206856 309 bee +n02219486 310 ant +n02226429 311 grasshopper +n02229544 312 cricket +n02231487 313 walking_stick +n02233338 314 cockroach +n02236044 315 mantis +n02256656 316 cicada +n02259212 317 leafhopper +n02264363 318 lacewing +n02268443 319 dragonfly +n02268853 320 damselfly +n02276258 321 admiral +n02277742 322 ringlet +n02279972 323 monarch +n02280649 324 cabbage_butterfly +n02281406 325 sulphur_butterfly +n02281787 326 lycaenid +n02317335 327 starfish +n02319095 328 sea_urchin +n02321529 329 sea_cucumber +n02325366 330 wood_rabbit +n02326432 331 hare +n02328150 332 Angora +n02342885 333 hamster +n02346627 334 porcupine +n02356798 335 fox_squirrel +n02361337 336 marmot +n02363005 337 beaver +n02364673 338 guinea_pig +n02389026 339 sorrel +n02391049 340 zebra +n02395406 341 hog +n02396427 342 wild_boar +n02397096 343 warthog +n02398521 344 hippopotamus +n02403003 345 ox +n02408429 346 water_buffalo +n02410509 347 bison +n02412080 348 ram +n02415577 349 bighorn +n02417914 350 ibex +n02422106 351 hartebeest +n02422699 352 impala +n02423022 353 gazelle +n02437312 354 Arabian_camel +n02437616 355 llama +n02441942 356 weasel +n02442845 357 mink +n02443114 358 polecat +n02443484 359 black-footed_ferret +n02444819 360 otter +n02445715 361 skunk +n02447366 362 badger +n02454379 363 armadillo +n02457408 364 three-toed_sloth +n02480495 365 orangutan +n02480855 366 gorilla +n02481823 367 chimpanzee +n02483362 368 gibbon +n02483708 369 siamang +n02484975 370 guenon +n02486261 371 patas +n02486410 372 baboon +n02487347 373 macaque +n02488291 374 langur +n02488702 375 colobus +n02489166 376 proboscis_monkey +n02490219 377 marmoset +n02492035 378 capuchin +n02492660 379 howler_monkey +n02493509 380 titi +n02493793 381 spider_monkey +n02494079 382 squirrel_monkey +n02497673 383 Madagascar_cat +n02500267 384 indri +n02504013 385 Indian_elephant +n02504458 386 African_elephant +n02509815 387 lesser_panda +n02510455 388 giant_panda +n02514041 389 barracouta +n02526121 390 eel +n02536864 391 coho +n02606052 392 rock_beauty +n02607072 393 anemone_fish +n02640242 394 sturgeon +n02641379 395 gar +n02643566 396 lionfish +n02655020 397 puffer +n02666196 398 abacus +n02667093 399 abaya +n02669723 400 academic_gown +n02672831 401 accordion +n02676566 402 acoustic_guitar +n02687172 403 aircraft_carrier +n02690373 404 airliner +n02692877 405 airship +n02699494 406 altar +n02701002 407 ambulance +n02704792 408 amphibian +n02708093 409 analog_clock +n02727426 410 apiary +n02730930 411 apron +n02747177 412 ashcan +n02749479 413 assault_rifle +n02769748 414 backpack +n02776631 415 bakery +n02777292 416 balance_beam +n02782093 417 balloon +n02783161 418 ballpoint +n02786058 419 Band_Aid +n02787622 420 banjo +n02788148 421 bannister +n02790996 422 barbell +n02791124 423 barber_chair +n02791270 424 barbershop +n02793495 425 barn +n02794156 426 barometer +n02795169 427 barrel +n02797295 428 barrow +n02799071 429 baseball +n02802426 430 basketball +n02804414 431 bassinet +n02804610 432 bassoon +n02807133 433 bathing_cap +n02808304 434 bath_towel +n02808440 435 bathtub +n02814533 436 beach_wagon +n02814860 437 beacon +n02815834 438 beaker +n02817516 439 bearskin +n02823428 440 beer_bottle +n02823750 441 beer_glass +n02825657 442 bell_cote +n02834397 443 bib +n02835271 444 bicycle-built-for-two +n02837789 445 bikini +n02840245 446 binder +n02841315 447 binoculars +n02843684 448 birdhouse +n02859443 449 boathouse +n02860847 450 bobsled +n02865351 451 bolo_tie +n02869837 452 bonnet +n02870880 453 bookcase +n02871525 454 bookshop +n02877765 455 bottlecap +n02879718 456 bow +n02883205 457 bow_tie +n02892201 458 brass +n02892767 459 brassiere +n02894605 460 breakwater +n02895154 461 breastplate +n02906734 462 broom +n02909870 463 bucket +n02910353 464 buckle +n02916936 465 bulletproof_vest +n02917067 466 bullet_train +n02927161 467 butcher_shop +n02930766 468 cab +n02939185 469 caldron +n02948072 470 candle +n02950826 471 cannon +n02951358 472 canoe +n02951585 473 can_opener +n02963159 474 cardigan +n02965783 475 car_mirror +n02966193 476 carousel +n02966687 477 carpenter's_kit +n02971356 478 carton +n02974003 479 car_wheel +n02977058 480 cash_machine +n02978881 481 cassette +n02979186 482 cassette_player +n02980441 483 castle +n02981792 484 catamaran +n02988304 485 CD_player +n02992211 486 cello +n02992529 487 cellular_telephone +n02999410 488 chain +n03000134 489 chainlink_fence +n03000247 490 chain_mail +n03000684 491 chain_saw +n03014705 492 chest +n03016953 493 chiffonier +n03017168 494 chime +n03018349 495 china_cabinet +n03026506 496 Christmas_stocking +n03028079 497 church +n03032252 498 cinema +n03041632 499 cleaver +n03042490 500 cliff_dwelling +n03045698 501 cloak +n03047690 502 clog +n03062245 503 cocktail_shaker +n03063599 504 coffee_mug +n03063689 505 coffeepot +n03065424 506 coil +n03075370 507 combination_lock +n03085013 508 computer_keyboard +n03089624 509 confectionery +n03095699 510 container_ship +n03100240 511 convertible +n03109150 512 corkscrew +n03110669 513 cornet +n03124043 514 cowboy_boot +n03124170 515 cowboy_hat +n03125729 516 cradle +n03126707 517 crane2 +n03127747 518 crash_helmet +n03127925 519 crate +n03131574 520 crib +n03133878 521 Crock_Pot +n03134739 522 croquet_ball +n03141823 523 crutch +n03146219 524 cuirass +n03160309 525 dam +n03179701 526 desk +n03180011 527 desktop_computer +n03187595 528 dial_telephone +n03188531 529 diaper +n03196217 530 digital_clock +n03197337 531 digital_watch +n03201208 532 dining_table +n03207743 533 dishrag +n03207941 534 dishwasher +n03208938 535 disk_brake +n03216828 536 dock +n03218198 537 dogsled +n03220513 538 dome +n03223299 539 doormat +n03240683 540 drilling_platform +n03249569 541 drum +n03250847 542 drumstick +n03255030 543 dumbbell +n03259280 544 Dutch_oven +n03271574 545 electric_fan +n03272010 546 electric_guitar +n03272562 547 electric_locomotive +n03290653 548 entertainment_center +n03291819 549 envelope +n03297495 550 espresso_maker +n03314780 551 face_powder +n03325584 552 feather_boa +n03337140 553 file +n03344393 554 fireboat +n03345487 555 fire_engine +n03347037 556 fire_screen +n03355925 557 flagpole +n03372029 558 flute +n03376595 559 folding_chair +n03379051 560 football_helmet +n03384352 561 forklift +n03388043 562 fountain +n03388183 563 fountain_pen +n03388549 564 four-poster +n03393912 565 freight_car +n03394916 566 French_horn +n03400231 567 frying_pan +n03404251 568 fur_coat +n03417042 569 garbage_truck +n03424325 570 gasmask +n03425413 571 gas_pump +n03443371 572 goblet +n03444034 573 go-kart +n03445777 574 golf_ball +n03445924 575 golfcart +n03447447 576 gondola +n03447721 577 gong +n03450230 578 gown +n03452741 579 grand_piano +n03457902 580 greenhouse +n03459775 581 grille +n03461385 582 grocery_store +n03467068 583 guillotine +n03476684 584 hair_slide +n03476991 585 hair_spray +n03478589 586 half_track +n03481172 587 hammer +n03482405 588 hamper +n03483316 589 hand_blower +n03485407 590 hand-held_computer +n03485794 591 handkerchief +n03492542 592 hard_disc +n03494278 593 harmonica +n03495258 594 harp +n03496892 595 harvester +n03498962 596 hatchet +n03527444 597 holster +n03529860 598 home_theater +n03530642 599 honeycomb +n03532672 600 hook +n03534580 601 hoopskirt +n03535780 602 horizontal_bar +n03538406 603 horse_cart +n03544143 604 hourglass +n03584254 605 iPod +n03584829 606 iron +n03590841 607 jack-o'-lantern +n03594734 608 jean +n03594945 609 jeep +n03595614 610 jersey +n03598930 611 jigsaw_puzzle +n03599486 612 jinrikisha +n03602883 613 joystick +n03617480 614 kimono +n03623198 615 knee_pad +n03627232 616 knot +n03630383 617 lab_coat +n03633091 618 ladle +n03637318 619 lampshade +n03642806 620 laptop +n03649909 621 lawn_mower +n03657121 622 lens_cap +n03658185 623 letter_opener +n03661043 624 library +n03662601 625 lifeboat +n03666591 626 lighter +n03670208 627 limousine +n03673027 628 liner +n03676483 629 lipstick +n03680355 630 Loafer +n03690938 631 lotion +n03691459 632 loudspeaker +n03692522 633 loupe +n03697007 634 lumbermill +n03706229 635 magnetic_compass +n03709823 636 mailbag +n03710193 637 mailbox +n03710637 638 maillot1 +n03710721 639 maillot2 +n03717622 640 manhole_cover +n03720891 641 maraca +n03721384 642 marimba +n03724870 643 mask +n03729826 644 matchstick +n03733131 645 maypole +n03733281 646 maze +n03733805 647 measuring_cup +n03742115 648 medicine_chest +n03743016 649 megalith +n03759954 650 microphone +n03761084 651 microwave +n03763968 652 military_uniform +n03764736 653 milk_can +n03769881 654 minibus +n03770439 655 miniskirt +n03770679 656 minivan +n03773504 657 missile +n03775071 658 mitten +n03775546 659 mixing_bowl +n03776460 660 mobile_home +n03777568 661 Model_T +n03777754 662 modem +n03781244 663 monastery +n03782006 664 monitor +n03785016 665 moped +n03786901 666 mortar +n03787032 667 mortarboard +n03788195 668 mosque +n03788365 669 mosquito_net +n03791053 670 motor_scooter +n03792782 671 mountain_bike +n03792972 672 mountain_tent +n03793489 673 mouse +n03794056 674 mousetrap +n03796401 675 moving_van +n03803284 676 muzzle +n03804744 677 nail +n03814639 678 neck_brace +n03814906 679 necklace +n03825788 680 nipple +n03832673 681 notebook +n03837869 682 obelisk +n03838899 683 oboe +n03840681 684 ocarina +n03841143 685 odometer +n03843555 686 oil_filter +n03854065 687 organ +n03857828 688 oscilloscope +n03866082 689 overskirt +n03868242 690 oxcart +n03868863 691 oxygen_mask +n03871628 692 packet +n03873416 693 paddle +n03874293 694 paddlewheel +n03874599 695 padlock +n03876231 696 paintbrush +n03877472 697 pajama +n03877845 698 palace +n03884397 699 panpipe +n03887697 700 paper_towel +n03888257 701 parachute +n03888605 702 parallel_bars +n03891251 703 park_bench +n03891332 704 parking_meter +n03895866 705 passenger_car +n03899768 706 patio +n03902125 707 pay-phone +n03903868 708 pedestal +n03908618 709 pencil_box +n03908714 710 pencil_sharpener +n03916031 711 perfume +n03920288 712 Petri_dish +n03924679 713 photocopier +n03929660 714 pick +n03929855 715 pickelhaube +n03930313 716 picket_fence +n03930630 717 pickup +n03933933 718 pier +n03935335 719 piggy_bank +n03937543 720 pill_bottle +n03938244 721 pillow +n03942813 722 ping-pong_ball +n03944341 723 pinwheel +n03947888 724 pirate +n03950228 725 pitcher +n03954731 726 plane +n03956157 727 planetarium +n03958227 728 plastic_bag +n03961711 729 plate_rack +n03967562 730 plow +n03970156 731 plunger +n03976467 732 Polaroid_camera +n03976657 733 pole +n03977966 734 police_van +n03980874 735 poncho +n03982430 736 pool_table +n03983396 737 pop_bottle +n03991062 738 pot +n03992509 739 potter's_wheel +n03995372 740 power_drill +n03998194 741 prayer_rug +n04004767 742 printer +n04005630 743 prison +n04008634 744 projectile +n04009552 745 projector +n04019541 746 puck +n04023962 747 punching_bag +n04026417 748 purse +n04033901 749 quill +n04033995 750 quilt +n04037443 751 racer +n04039381 752 racket +n04040759 753 radiator +n04041544 754 radio +n04044716 755 radio_telescope +n04049303 756 rain_barrel +n04065272 757 recreational_vehicle +n04067472 758 reel +n04069434 759 reflex_camera +n04070727 760 refrigerator +n04074963 761 remote_control +n04081281 762 restaurant +n04086273 763 revolver +n04090263 764 rifle +n04099969 765 rocking_chair +n04111531 766 rotisserie +n04116512 767 rubber_eraser +n04118538 768 rugby_ball +n04118776 769 rule +n04120489 770 running_shoe +n04125021 771 safe +n04127249 772 safety_pin +n04131690 773 saltshaker +n04133789 774 sandal +n04136333 775 sarong +n04141076 776 sax +n04141327 777 scabbard +n04141975 778 scale +n04146614 779 school_bus +n04147183 780 schooner +n04149813 781 scoreboard +n04152593 782 screen +n04153751 783 screw +n04154565 784 screwdriver +n04162706 785 seat_belt +n04179913 786 sewing_machine +n04192698 787 shield +n04200800 788 shoe_shop +n04201297 789 shoji +n04204238 790 shopping_basket +n04204347 791 shopping_cart +n04208210 792 shovel +n04209133 793 shower_cap +n04209239 794 shower_curtain +n04228054 795 ski +n04229816 796 ski_mask +n04235860 797 sleeping_bag +n04238763 798 slide_rule +n04239074 799 sliding_door +n04243546 800 slot +n04251144 801 snorkel +n04252077 802 snowmobile +n04252225 803 snowplow +n04254120 804 soap_dispenser +n04254680 805 soccer_ball +n04254777 806 sock +n04258138 807 solar_dish +n04259630 808 sombrero +n04263257 809 soup_bowl +n04264628 810 space_bar +n04265275 811 space_heater +n04266014 812 space_shuttle +n04270147 813 spatula +n04273569 814 speedboat +n04275548 815 spider_web +n04277352 816 spindle +n04285008 817 sports_car +n04286575 818 spotlight +n04296562 819 stage +n04310018 820 steam_locomotive +n04311004 821 steel_arch_bridge +n04311174 822 steel_drum +n04317175 823 stethoscope +n04325704 824 stole +n04326547 825 stone_wall +n04328186 826 stopwatch +n04330267 827 stove +n04332243 828 strainer +n04335435 829 streetcar +n04336792 830 stretcher +n04344873 831 studio_couch +n04346328 832 stupa +n04347754 833 submarine +n04350905 834 suit +n04355338 835 sundial +n04355933 836 sunglass +n04356056 837 sunglasses +n04357314 838 sunscreen +n04366367 839 suspension_bridge +n04367480 840 swab +n04370456 841 sweatshirt +n04371430 842 swimming_trunks +n04371774 843 swing +n04372370 844 switch +n04376876 845 syringe +n04380533 846 table_lamp +n04389033 847 tank +n04392985 848 tape_player +n04398044 849 teapot +n04399382 850 teddy +n04404412 851 television +n04409515 852 tennis_ball +n04417672 853 thatch +n04418357 854 theater_curtain +n04423845 855 thimble +n04428191 856 thresher +n04429376 857 throne +n04435653 858 tile_roof +n04442312 859 toaster +n04443257 860 tobacco_shop +n04447861 861 toilet_seat +n04456115 862 torch +n04458633 863 totem_pole +n04461696 864 tow_truck +n04462240 865 toyshop +n04465501 866 tractor +n04467665 867 trailer_truck +n04476259 868 tray +n04479046 869 trench_coat +n04482393 870 tricycle +n04483307 871 trimaran +n04485082 872 tripod +n04486054 873 triumphal_arch +n04487081 874 trolleybus +n04487394 875 trombone +n04493381 876 tub +n04501370 877 turnstile +n04505470 878 typewriter_keyboard +n04507155 879 umbrella +n04509417 880 unicycle +n04515003 881 upright +n04517823 882 vacuum +n04522168 883 vase +n04523525 884 vault +n04525038 885 velvet +n04525305 886 vending_machine +n04532106 887 vestment +n04532670 888 viaduct +n04536866 889 violin +n04540053 890 volleyball +n04542943 891 waffle_iron +n04548280 892 wall_clock +n04548362 893 wallet +n04550184 894 wardrobe +n04552348 895 warplane +n04553703 896 washbasin +n04554684 897 washer +n04557648 898 water_bottle +n04560804 899 water_jug +n04562935 900 water_tower +n04579145 901 whiskey_jug +n04579432 902 whistle +n04584207 903 wig +n04589890 904 window_screen +n04590129 905 window_shade +n04591157 906 Windsor_tie +n04591713 907 wine_bottle +n04592741 908 wing +n04596742 909 wok +n04597913 910 wooden_spoon +n04599235 911 wool +n04604644 912 worm_fence +n04606251 913 wreck +n04612504 914 yawl +n04613696 915 yurt +n06359193 916 web_site +n06596364 917 comic_book +n06785654 918 crossword_puzzle +n06794110 919 street_sign +n06874185 920 traffic_light +n07248320 921 book_jacket +n07565083 922 menu +n07579787 923 plate +n07583066 924 guacamole +n07584110 925 consomme +n07590611 926 hot_pot +n07613480 927 trifle +n07614500 928 ice_cream +n07615774 929 ice_lolly +n07684084 930 French_loaf +n07693725 931 bagel +n07695742 932 pretzel +n07697313 933 cheeseburger +n07697537 934 hotdog +n07711569 935 mashed_potato +n07714571 936 head_cabbage +n07714990 937 broccoli +n07715103 938 cauliflower +n07716358 939 zucchini +n07716906 940 spaghetti_squash +n07717410 941 acorn_squash +n07717556 942 butternut_squash +n07718472 943 cucumber +n07718747 944 artichoke +n07720875 945 bell_pepper +n07730033 946 cardoon +n07734744 947 mushroom +n07742313 948 Granny_Smith +n07745940 949 strawberry +n07747607 950 orange +n07749582 951 lemon +n07753113 952 fig +n07753275 953 pineapple +n07753592 954 banana +n07754684 955 jackfruit +n07760859 956 custard_apple +n07768694 957 pomegranate +n07802026 958 hay +n07831146 959 carbonara +n07836838 960 chocolate_sauce +n07860988 961 dough +n07871810 962 meat_loaf +n07873807 963 pizza +n07875152 964 potpie +n07880968 965 burrito +n07892512 966 red_wine +n07920052 967 espresso +n07930864 968 cup +n07932039 969 eggnog +n09193705 970 alp +n09229709 971 bubble +n09246464 972 cliff +n09256479 973 coral_reef +n09288635 974 geyser +n09332890 975 lakeside +n09399592 976 promontory +n09421951 977 sandbar +n09428293 978 seashore +n09468604 979 valley +n09472597 980 volcano +n09835506 981 ballplayer +n10148035 982 groom +n10565667 983 scuba_diver +n11879895 984 rapeseed +n11939491 985 daisy +n12057211 986 yellow_lady's_slipper +n12144580 987 corn +n12267677 988 acorn +n12620546 989 hip +n12768682 990 buckeye +n12985857 991 coral_fungus +n12998815 992 agaric +n13037406 993 gyromitra +n13040303 994 stinkhorn +n13044778 995 earthstar +n13052670 996 hen-of-the-woods +n13054560 997 bolete +n13133613 998 ear +n15075141 999 toilet_tissue diff --git a/data/datasets/image_classification/imagewoof.py b/data/datasets/image_classification/imagewoof.py new file mode 100644 index 0000000000000000000000000000000000000000..82c2006dd40aa5a219f9f89b84afcd9d62265b23 --- /dev/null +++ b/data/datasets/image_classification/imagewoof.py @@ -0,0 +1,37 @@ +from ..data_aug import imagenet_like_image_train_aug, imagenet_like_image_test_aug +from ..ab_dataset import ABDataset +from ..dataset_split import train_val_split, train_val_test_split +from torchvision.datasets import ImageFolder +import os +from typing import Dict, List, Optional +from torchvision.transforms import Compose + +from ..registery import dataset_register + +@dataset_register( + name='Imagewoof', + classes=["Australian terrier", "Border terrier", "Samoyed", "Beagle", "Shih-Tzu", "English foxhound", "Rhodesian ridgeback", "Dingo", "Golden retriever", "Old English sheepdog"], + task_type='Image Classification', + object_type='Dog', + class_aliases=[], + shift_type=None +) + +class Imagewoof(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform: Optional[Compose], + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + if transform is None: + transform = imagenet_like_image_train_aug() if split == 'train' else imagenet_like_image_test_aug() + self.transform = transform + #root_dir = os.path.join(root_dir, 'train' if split != 'test' else 'val') + dataset = ImageFolder(root_dir, transform=transform) + + if len(ignore_classes) > 0: + ignore_classes_idx = [classes.index(c) for c in ignore_classes] + dataset.samples = [s for s in dataset.samples if s[1] not in ignore_classes_idx] + + if idx_map is not None: + dataset.samples = [(s[0], idx_map[s[1]]) if s[1] in idx_map.keys() else s for s in dataset.samples] + + dataset = train_val_test_split(dataset, split) + return dataset diff --git a/data/datasets/image_classification/mnist.py b/data/datasets/image_classification/mnist.py new file mode 100644 index 0000000000000000000000000000000000000000..02edaebdbbd66799c70ba9f76c9719d19c2986eb --- /dev/null +++ b/data/datasets/image_classification/mnist.py @@ -0,0 +1,47 @@ +from ..data_aug import one_d_image_test_aug, one_d_image_train_aug +from ..ab_dataset import ABDataset +from ..dataset_split import train_val_split +from torchvision.datasets import MNIST as RawMNIST +import numpy as np +from typing import Dict, List, Optional +from torchvision.transforms import Compose +from utils.common.others import HiddenPrints + +from ..registery import dataset_register + + +@dataset_register( + name='MNIST', + classes=[str(i) for i in range(10)], + task_type='Image Classification', + object_type='Digit and Letter', + class_aliases=[], + shift_type=None +) +class MNIST(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform: Optional[Compose], + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + if transform is None: + transform = one_d_image_train_aug() if split == 'train' else one_d_image_test_aug() + self.transform = transform + + with HiddenPrints(): + dataset = RawMNIST(root_dir, split != 'test', transform=transform, download=True) + + dataset.targets = np.asarray(dataset.targets) + if len(ignore_classes) > 0: + for ignore_class in ignore_classes: + dataset.data = dataset.data[dataset.targets != classes.index(ignore_class)] + dataset.targets = dataset.targets[dataset.targets != classes.index(ignore_class)] + + if idx_map is not None: + # note: the code below seems correct but has bug! + # for old_idx, new_idx in idx_map.items(): + # dataset.targets[dataset.targets == old_idx] = new_idx + + for ti, t in enumerate(dataset.targets): + dataset.targets[ti] = idx_map[t] + + if split != 'test': + dataset = train_val_split(dataset, split) + return dataset diff --git a/data/datasets/image_classification/mnist_single.py b/data/datasets/image_classification/mnist_single.py new file mode 100644 index 0000000000000000000000000000000000000000..70ed2e10f7de42dc93cf37e4948294e7caa0c533 --- /dev/null +++ b/data/datasets/image_classification/mnist_single.py @@ -0,0 +1,38 @@ +from ..data_aug import one_d_image_test_aug, one_d_image_train_aug +from ..ab_dataset import ABDataset +from ..dataset_split import train_val_test_split +from torchvision.datasets import ImageFolder +import numpy as np +from typing import Dict, List, Optional +from torchvision.transforms import Compose +from utils.common.others import HiddenPrints + +from ..registery import dataset_register + + +@dataset_register( + name='MNIST-single', + classes=[str(i) for i in range(10)], + task_type='Image Classification', + object_type='Digit and Letter', + class_aliases=[], + shift_type=None +) +class MNISTSingle(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform: Optional[Compose], + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + if transform is None: + transform = one_d_image_train_aug() if split == 'train' else one_d_image_test_aug() + self.transform = transform + + dataset = ImageFolder(root_dir, transform=transform) + + if len(ignore_classes) > 0: + ignore_classes_idx = [classes.index(c) for c in ignore_classes] + dataset.samples = [s for s in dataset.samples if s[1] not in ignore_classes_idx] + + if idx_map is not None: + dataset.samples = [(s[0], idx_map[s[1]]) if s[1] in idx_map.keys() else s for s in dataset.samples] + + dataset = train_val_test_split(dataset, split) + return dataset diff --git a/data/datasets/image_classification/stanfordcars.py b/data/datasets/image_classification/stanfordcars.py new file mode 100644 index 0000000000000000000000000000000000000000..e6d027a4a88087e99de0dd1b1aa5029ee62ef0d9 --- /dev/null +++ b/data/datasets/image_classification/stanfordcars.py @@ -0,0 +1,41 @@ +from ..data_aug import imagenet_like_image_train_aug, imagenet_like_image_test_aug +from ..ab_dataset import ABDataset +from ..dataset_split import train_val_split, train_val_test_split +from torchvision.datasets import ImageFolder +import os +from typing import Dict, List, Optional +from torchvision.transforms import Compose + +from ..registery import dataset_register + +with open(os.path.join(os.path.dirname(__file__), 'stanfordcars_classes.txt'), 'r') as f: + classes = [line.split(' ')[1].strip() for line in f.readlines()] + assert len(classes) == 196 + +@dataset_register( + name='Stanford_Cars', + classes=classes, + task_type='Image Classification', + object_type='Car', + class_aliases=[], + shift_type=None +) + +class Stanford_Cars(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform: Optional[Compose], + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + if transform is None: + transform = imagenet_like_image_train_aug() if split == 'train' else imagenet_like_image_test_aug() + self.transform = transform + #root_dir = os.path.join(root_dir, 'train' if split != 'test' else 'val') + dataset = ImageFolder(root_dir, transform=transform) + + if len(ignore_classes) > 0: + ignore_classes_idx = [classes.index(c) for c in ignore_classes] + dataset.samples = [s for s in dataset.samples if s[1] not in ignore_classes_idx] + + if idx_map is not None: + dataset.samples = [(s[0], idx_map[s[1]]) if s[1] in idx_map.keys() else s for s in dataset.samples] + + dataset = train_val_test_split(dataset, split) + return dataset diff --git a/data/datasets/image_classification/stanfordcars_classes.txt b/data/datasets/image_classification/stanfordcars_classes.txt new file mode 100644 index 0000000000000000000000000000000000000000..bf892e8515459ac75be77f50e0b987e97d7a4797 --- /dev/null +++ b/data/datasets/image_classification/stanfordcars_classes.txt @@ -0,0 +1,196 @@ +0 AM_General_Hummer_SUV_2000 +1 Acura_RL_Sedan_2012 +2 Acura_TL_Sedan_2012 +3 Acura_TL_Type-S_2008 +4 Acura_TSX_Sedan_2012 +5 Acura_Integra_Type_R_2001 +6 Acura_ZDX_Hatchback_2012 +7 Aston_Martin_V8_Vantage_Convertible_2012 +8 Aston_Martin_V8_Vantage_Coupe_2012 +9 Aston_Martin_Virage_Convertible_2012 +10 Aston_Martin_Virage_Coupe_2012 +11 Audi_RS_4_Convertible_2008 +12 Audi_A5_Coupe_2012 +13 Audi_TTS_Coupe_2012 +14 Audi_R8_Coupe_2012 +15 Audi_V8_Sedan_1994 +16 Audi_100_Sedan_1994 +17 Audi_100_Wagon_1994 +18 Audi_TT_Hatchback_2011 +19 Audi_S6_Sedan_2011 +20 Audi_S5_Convertible_2012 +21 Audi_S5_Coupe_2012 +22 Audi_S4_Sedan_2012 +23 Audi_S4_Sedan_2007 +24 Audi_TT_RS_Coupe_2012 +25 BMW_ActiveHybrid_5_Sedan_2012 +26 BMW_1_Series_Convertible_2012 +27 BMW_1_Series_Coupe_2012 +28 BMW_3_Series_Sedan_2012 +29 BMW_3_Series_Wagon_2012 +30 BMW_6_Series_Convertible_2007 +31 BMW_X5_SUV_2007 +32 BMW_X6_SUV_2012 +33 BMW_M3_Coupe_2012 +34 BMW_M5_Sedan_2010 +35 BMW_M6_Convertible_2010 +36 BMW_X3_SUV_2012 +37 BMW_Z4_Convertible_2012 +38 Bentley_Continental_Supersports_Conv._Convertible_2012 +39 Bentley_Arnage_Sedan_2009 +40 Bentley_Mulsanne_Sedan_2011 +41 Bentley_Continental_GT_Coupe_2012 +42 Bentley_Continental_GT_Coupe_2007 +43 Bentley_Continental_Flying_Spur_Sedan_2007 +44 Bugatti_Veyron_16.4_Convertible_2009 +45 Bugatti_Veyron_16.4_Coupe_2009 +46 Buick_Regal_GS_2012 +47 Buick_Rainier_SUV_2007 +48 Buick_Verano_Sedan_2012 +49 Buick_Enclave_SUV_2012 +50 Cadillac_CTS-V_Sedan_2012 +51 Cadillac_SRX_SUV_2012 +52 Cadillac_Escalade_EXT_Crew_Cab_2007 +53 Chevrolet_Silverado_1500_Hybrid_Crew_Cab_2012 +54 Chevrolet_Corvette_Convertible_2012 +55 Chevrolet_Corvette_ZR1_2012 +56 Chevrolet_Corvette_Ron_Fellows_Edition_Z06_2007 +57 Chevrolet_Traverse_SUV_2012 +58 Chevrolet_Camaro_Convertible_2012 +59 Chevrolet_HHR_SS_2010 +60 Chevrolet_Impala_Sedan_2007 +61 Chevrolet_Tahoe_Hybrid_SUV_2012 +62 Chevrolet_Sonic_Sedan_2012 +63 Chevrolet_Express_Cargo_Van_2007 +64 Chevrolet_Avalanche_Crew_Cab_2012 +65 Chevrolet_Cobalt_SS_2010 +66 Chevrolet_Malibu_Hybrid_Sedan_2010 +67 Chevrolet_TrailBlazer_SS_2009 +68 Chevrolet_Silverado_2500HD_Regular_Cab_2012 +69 Chevrolet_Silverado_1500_Classic_Extended_Cab_2007 +70 Chevrolet_Express_Van_2007 +71 Chevrolet_Monte_Carlo_Coupe_2007 +72 Chevrolet_Malibu_Sedan_2007 +73 Chevrolet_Silverado_1500_Extended_Cab_2012 +74 Chevrolet_Silverado_1500_Regular_Cab_2012 +75 Chrysler_Aspen_SUV_2009 +76 Chrysler_Sebring_Convertible_2010 +77 Chrysler_Town_and_Country_Minivan_2012 +78 Chrysler_300_SRT-8_2010 +79 Chrysler_Crossfire_Convertible_2008 +80 Chrysler_PT_Cruiser_Convertible_2008 +81 Daewoo_Nubira_Wagon_2002 +82 Dodge_Caliber_Wagon_2012 +83 Dodge_Caliber_Wagon_2007 +84 Dodge_Caravan_Minivan_1997 +85 Dodge_Ram_Pickup_3500_Crew_Cab_2010 +86 Dodge_Ram_Pickup_3500_Quad_Cab_2009 +87 Dodge_Sprinter_Cargo_Van_2009 +88 Dodge_Journey_SUV_2012 +89 Dodge_Dakota_Crew_Cab_2010 +90 Dodge_Dakota_Club_Cab_2007 +91 Dodge_Magnum_Wagon_2008 +92 Dodge_Challenger_SRT8_2011 +93 Dodge_Durango_SUV_2012 +94 Dodge_Durango_SUV_2007 +95 Dodge_Charger_Sedan_2012 +96 Dodge_Charger_SRT-8_2009 +97 Eagle_Talon_Hatchback_1998 +98 FIAT_500_Abarth_2012 +99 FIAT_500_Convertible_2012 +100 Ferrari_FF_Coupe_2012 +101 Ferrari_California_Convertible_2012 +102 Ferrari_458_Italia_Convertible_2012 +103 Ferrari_458_Italia_Coupe_2012 +104 Fisker_Karma_Sedan_2012 +105 Ford_F-450_Super_Duty_Crew_Cab_2012 +106 Ford_Mustang_Convertible_2007 +107 Ford_Freestar_Minivan_2007 +108 Ford_Expedition_EL_SUV_2009 +109 Ford_Edge_SUV_2012 +110 Ford_Ranger_SuperCab_2011 +111 Ford_GT_Coupe_2006 +112 Ford_F-150_Regular_Cab_2012 +113 Ford_F-150_Regular_Cab_2007 +114 Ford_Focus_Sedan_2007 +115 Ford_E-Series_Wagon_Van_2012 +116 Ford_Fiesta_Sedan_2012 +117 GMC_Terrain_SUV_2012 +118 GMC_Savana_Van_2012 +119 GMC_Yukon_Hybrid_SUV_2012 +120 GMC_Acadia_SUV_2012 +121 GMC_Canyon_Extended_Cab_2012 +122 Geo_Metro_Convertible_1993 +123 HUMMER_H3T_Crew_Cab_2010 +124 HUMMER_H2_SUT_Crew_Cab_2009 +125 Honda_Odyssey_Minivan_2012 +126 Honda_Odyssey_Minivan_2007 +127 Honda_Accord_Coupe_2012 +128 Honda_Accord_Sedan_2012 +129 Hyundai_Veloster_Hatchback_2012 +130 Hyundai_Santa_Fe_SUV_2012 +131 Hyundai_Tucson_SUV_2012 +132 Hyundai_Veracruz_SUV_2012 +133 Hyundai_Sonata_Hybrid_Sedan_2012 +134 Hyundai_Elantra_Sedan_2007 +135 Hyundai_Accent_Sedan_2012 +136 Hyundai_Genesis_Sedan_2012 +137 Hyundai_Sonata_Sedan_2012 +138 Hyundai_Elantra_Touring_Hatchback_2012 +139 Hyundai_Azera_Sedan_2012 +140 Infiniti_G_Coupe_IPL_2012 +141 Infiniti_QX56_SUV_2011 +142 Isuzu_Ascender_SUV_2008 +143 Jaguar_XK_XKR_2012 +144 Jeep_Patriot_SUV_2012 +145 Jeep_Wrangler_SUV_2012 +146 Jeep_Liberty_SUV_2012 +147 Jeep_Grand_Cherokee_SUV_2012 +148 Jeep_Compass_SUV_2012 +149 Lamborghini_Reventon_Coupe_2008 +150 Lamborghini_Aventador_Coupe_2012 +151 Lamborghini_Gallardo_LP_570-4_Superleggera_2012 +152 Lamborghini_Diablo_Coupe_2001 +153 Land_Rover_Range_Rover_SUV_2012 +154 Land_Rover_LR2_SUV_2012 +155 Lincoln_Town_Car_Sedan_2011 +156 MINI_Cooper_Roadster_Convertible_2012 +157 Maybach_Landaulet_Convertible_2012 +158 Mazda_Tribute_SUV_2011 +159 McLaren_MP4-12C_Coupe_2012 +160 Mercedes-Benz_300-Class_Convertible_1993 +161 Mercedes-Benz_C-Class_Sedan_2012 +162 Mercedes-Benz_SL-Class_Coupe_2009 +163 Mercedes-Benz_E-Class_Sedan_2012 +164 Mercedes-Benz_S-Class_Sedan_2012 +165 Mercedes-Benz_Sprinter_Van_2012 +166 Mitsubishi_Lancer_Sedan_2012 +167 Nissan_Leaf_Hatchback_2012 +168 Nissan_NV_Passenger_Van_2012 +169 Nissan_Juke_Hatchback_2012 +170 Nissan_240SX_Coupe_1998 +171 Plymouth_Neon_Coupe_1999 +172 Porsche_Panamera_Sedan_2012 +173 Ram_C/V_Cargo_Van_Minivan_2012 +174 Rolls-Royce_Phantom_Drophead_Coupe_Convertible_2012 +175 Rolls-Royce_Ghost_Sedan_2012 +176 Rolls-Royce_Phantom_Sedan_2012 +177 Scion_xD_Hatchback_2012 +178 Spyker_C8_Convertible_2009 +179 Spyker_C8_Coupe_2009 +180 Suzuki_Aerio_Sedan_2007 +181 Suzuki_Kizashi_Sedan_2012 +182 Suzuki_SX4_Hatchback_2012 +183 Suzuki_SX4_Sedan_2012 +184 Tesla_Model_S_Sedan_2012 +185 Toyota_Sequoia_SUV_2012 +186 Toyota_Camry_Sedan_2012 +187 Toyota_Corolla_Sedan_2012 +188 Toyota_4Runner_SUV_2012 +189 Volkswagen_Golf_Hatchback_2012 +190 Volkswagen_Golf_Hatchback_1991 +191 Volkswagen_Beetle_Hatchback_2012 +192 Volvo_C30_Hatchback_2012 +193 Volvo_240_Sedan_1993 +194 Volvo_XC90_SUV_2007 +195 smart_fortwo_Convertible_2012 diff --git a/data/datasets/image_classification/stl10.py b/data/datasets/image_classification/stl10.py new file mode 100644 index 0000000000000000000000000000000000000000..344857d14719cf0a0d781e184d557d5cd7282892 --- /dev/null +++ b/data/datasets/image_classification/stl10.py @@ -0,0 +1,45 @@ +from ..data_aug import cifar_like_image_train_aug, cifar_like_image_test_aug +from ..ab_dataset import ABDataset +from ..dataset_split import train_val_split +from torchvision.datasets import STL10 as RawSTL10 +from typing import Dict, List, Optional +from torchvision.transforms import Compose +from utils.common.others import HiddenPrints + +from ..registery import dataset_register + + +@dataset_register( + name='STL10', + classes=['airplane', 'bird', 'car', 'cat', 'deer', 'dog', 'horse', 'monkey', 'ship', 'truck'], + task_type='Image Classification', + object_type='Generic Object', + class_aliases=[], + shift_type=None +) +class STL10(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform: Optional[Compose], + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + if transform is None: + transform = cifar_like_image_train_aug() if split == 'train' else cifar_like_image_test_aug() + self.transform = transform + + with HiddenPrints(): + dataset = RawSTL10(root_dir, 'train' if split != 'test' else 'test', transform=transform, download=True) + + if len(ignore_classes) > 0: + for ignore_class in ignore_classes: + dataset.data = dataset.data[dataset.labels != classes.index(ignore_class)] + dataset.labels = dataset.labels[dataset.labels != classes.index(ignore_class)] + + if idx_map is not None: + # note: the code below seems correct but has bug! + # for old_idx, new_idx in idx_map.items(): + # dataset.targets[dataset.targets == old_idx] = new_idx + + for ti, t in enumerate(dataset.labels): + dataset.labels[ti] = idx_map[t] + + if split != 'test': + dataset = train_val_split(dataset, split) + return dataset diff --git a/data/datasets/image_classification/stl10_single.py b/data/datasets/image_classification/stl10_single.py new file mode 100644 index 0000000000000000000000000000000000000000..ad1ba43e2f0cd2d4793a27d72b8f99c91864e488 --- /dev/null +++ b/data/datasets/image_classification/stl10_single.py @@ -0,0 +1,37 @@ +from ..data_aug import cifar_like_image_train_aug, cifar_like_image_test_aug +from ..ab_dataset import ABDataset +from ..dataset_split import train_val_test_split +from torchvision.datasets import ImageFolder +from typing import Dict, List, Optional +from torchvision.transforms import Compose +from utils.common.others import HiddenPrints + +from ..registery import dataset_register + + +@dataset_register( + name='STL10-single', + classes=['airplane', 'bird', 'car', 'cat', 'deer', 'dog', 'horse', 'monkey', 'ship', 'truck'], + task_type='Image Classification', + object_type='Generic Object', + class_aliases=[], + shift_type=None +) +class STL10Single(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform: Optional[Compose], + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + if transform is None: + transform = cifar_like_image_train_aug() if split == 'train' else cifar_like_image_test_aug() + self.transform = transform + + dataset = ImageFolder(root_dir, transform=transform) + + if len(ignore_classes) > 0: + ignore_classes_idx = [classes.index(c) for c in ignore_classes] + dataset.samples = [s for s in dataset.samples if s[1] not in ignore_classes_idx] + + if idx_map is not None: + dataset.samples = [(s[0], idx_map[s[1]]) if s[1] in idx_map.keys() else s for s in dataset.samples] + + dataset = train_val_test_split(dataset, split) + return dataset diff --git a/data/datasets/image_classification/supervisely_person_cls.py b/data/datasets/image_classification/supervisely_person_cls.py new file mode 100644 index 0000000000000000000000000000000000000000..89ba88771a5bad0e4ef32e0a101fa677a6253e97 --- /dev/null +++ b/data/datasets/image_classification/supervisely_person_cls.py @@ -0,0 +1,41 @@ +from ..data_aug import imagenet_like_image_train_aug, imagenet_like_image_test_aug +from ..ab_dataset import ABDataset +from ..dataset_split import train_val_split, train_val_test_split +from torchvision.datasets import ImageFolder +import os +from typing import Dict, List, Optional +from torchvision.transforms import Compose + +from ..registery import dataset_register + +# with open(os.path.join(os.path.dirname(__file__), 'fruits360_classes.txt'), 'r') as f: +# classes = [line.split(':')[0].strip('"') for line in f.readlines()] +# assert len(classes) == 131 + +@dataset_register( + name='SuperviselyPersonCls', + classes=['seg_person'], + task_type='Image Classification', + object_type='Person', + class_aliases=[], + shift_type=None +) + +class SuperviselyPersonCls(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform: Optional[Compose], + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + if transform is None: + transform = imagenet_like_image_train_aug() if split == 'train' else imagenet_like_image_test_aug() + self.transform = transform + #root_dir = os.path.join(root_dir, 'train' if split != 'test' else 'val') + dataset = ImageFolder(root_dir, transform=transform) + + if len(ignore_classes) > 0: + ignore_classes_idx = [classes.index(c) for c in ignore_classes] + dataset.samples = [s for s in dataset.samples if s[1] not in ignore_classes_idx] + + if idx_map is not None: + dataset.samples = [(s[0], idx_map[s[1]]) if s[1] in idx_map.keys() else s for s in dataset.samples] + + dataset = train_val_test_split(dataset, split) + return dataset diff --git a/data/datasets/image_classification/svhn.py b/data/datasets/image_classification/svhn.py new file mode 100644 index 0000000000000000000000000000000000000000..0aae5e763623cbc3b2dadf7ef960da18c3ee49f8 --- /dev/null +++ b/data/datasets/image_classification/svhn.py @@ -0,0 +1,58 @@ +from ..data_aug import cifar_like_image_test_aug, cifar_like_image_train_aug +from ..ab_dataset import ABDataset +from ..dataset_split import train_val_split +from torchvision.datasets import SVHN as RawSVHN +import numpy as np +from typing import Dict, List, Optional +from torchvision import transforms +from torchvision.transforms import Compose +from utils.common.others import HiddenPrints + +from ..registery import dataset_register + + +@dataset_register( + name='SVHN', + classes=[str(i) for i in range(10)], + task_type='Image Classification', + object_type='Digit and Letter', + class_aliases=[], + shift_type=None +) +class SVHN(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform: Optional[Compose], + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + if transform is None: + mean, std = [0.5] * 3, [0.5] * 3 + transform = transforms.Compose([ + transforms.RandomCrop(32, padding=4), + transforms.ToTensor(), + transforms.Normalize(mean, std) + ]) if split == 'train' else \ + transforms.Compose([ + transforms.Resize(32), + transforms.ToTensor(), + transforms.Normalize(mean, std) + ]) + + self.transform = transform + + with HiddenPrints(): + dataset = RawSVHN(root_dir, 'train' if split != 'test' else 'test', transform=transform, download=True) + + if len(ignore_classes) > 0: + for ignore_class in ignore_classes: + dataset.data = dataset.data[dataset.labels != classes.index(ignore_class)] + dataset.labels = dataset.labels[dataset.labels != classes.index(ignore_class)] + + if idx_map is not None: + # note: the code below seems correct but has bug! + # for old_idx, new_idx in idx_map.items(): + # dataset.targets[dataset.targets == old_idx] = new_idx + + for ti, t in enumerate(dataset.labels): + dataset.labels[ti] = idx_map[t] + + if split != 'test': + dataset = train_val_split(dataset, split) + return dataset diff --git a/data/datasets/image_classification/svhn_single.py b/data/datasets/image_classification/svhn_single.py new file mode 100644 index 0000000000000000000000000000000000000000..9be4d45d5de362dedb5b02446a96a10b01821e4e --- /dev/null +++ b/data/datasets/image_classification/svhn_single.py @@ -0,0 +1,50 @@ +from ..data_aug import cifar_like_image_test_aug, cifar_like_image_train_aug +from ..ab_dataset import ABDataset +from ..dataset_split import train_val_test_split +from torchvision.datasets import ImageFolder +import numpy as np +from typing import Dict, List, Optional +from torchvision import transforms +from torchvision.transforms import Compose +from utils.common.others import HiddenPrints + +from ..registery import dataset_register + + +@dataset_register( + name='SVHN-single', + classes=[str(i) for i in range(10)], + task_type='Image Classification', + object_type='Digit and Letter', + class_aliases=[], + shift_type=None +) +class SVHNSingle(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform: Optional[Compose], + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + if transform is None: + mean, std = [0.5] * 3, [0.5] * 3 + transform = transforms.Compose([ + transforms.RandomCrop(32, padding=4), + transforms.ToTensor(), + transforms.Normalize(mean, std) + ]) if split == 'train' else \ + transforms.Compose([ + transforms.Resize(32), + transforms.ToTensor(), + transforms.Normalize(mean, std) + ]) + + self.transform = transform + + dataset = ImageFolder(root_dir, transform=transform) + + if len(ignore_classes) > 0: + ignore_classes_idx = [classes.index(c) for c in ignore_classes] + dataset.samples = [s for s in dataset.samples if s[1] not in ignore_classes_idx] + + if idx_map is not None: + dataset.samples = [(s[0], idx_map[s[1]]) if s[1] in idx_map.keys() else s for s in dataset.samples] + + dataset = train_val_test_split(dataset, split) + return dataset diff --git a/data/datasets/image_classification/synsigns.py b/data/datasets/image_classification/synsigns.py new file mode 100644 index 0000000000000000000000000000000000000000..e28c68d4df20cea0ca879af13138b3ceab04a9f2 --- /dev/null +++ b/data/datasets/image_classification/synsigns.py @@ -0,0 +1,42 @@ +from ..data_aug import cifar_like_image_train_aug, cifar_like_image_test_aug, imagenet_like_image_test_aug, imagenet_like_image_train_aug +from ..ab_dataset import ABDataset +from ..dataset_split import train_val_test_split +from torchvision.datasets import ImageFolder +from typing import Dict, List, Optional +from torchvision.transforms import Compose +from utils.common.others import HiddenPrints +from ..dataset_cache import get_dataset_cache_path, read_cached_dataset_status, cache_dataset_status +import os +from ..registery import dataset_register + + +@dataset_register( + name='SYNSIGNS', + classes=[f'{i:05d}' for i in range(42)], + task_type='Image Classification', + object_type='Traffic Sign', + class_aliases=[], + shift_type=None +) +class SYNSIGNS(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform: Optional[Compose], + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + if transform is None: + transform = imagenet_like_image_train_aug() if split == 'train' else imagenet_like_image_test_aug() + self.transform = transform + + dataset = ImageFolder(root_dir, transform=transform) + + cache_file_path = get_dataset_cache_path(root_dir, classes, ignore_classes, idx_map) + if os.path.exists(cache_file_path): + dataset.samples = read_cached_dataset_status(cache_file_path, 'SYNSIGNS-' + split) + else: + if len(ignore_classes) > 0: + ignore_classes_idx = [classes.index(c) for c in ignore_classes] + dataset.samples = [s for s in dataset.samples if s[1] not in ignore_classes_idx] + if idx_map is not None: + dataset.samples = [(s[0], idx_map[s[1]]) if s[1] in idx_map.keys() else s for s in dataset.samples] + cache_dataset_status(dataset.samples, cache_file_path, 'SYNSIGNS-' + split) + + dataset = train_val_test_split(dataset, split) + return dataset diff --git a/data/datasets/image_classification/usps.py b/data/datasets/image_classification/usps.py new file mode 100644 index 0000000000000000000000000000000000000000..939424a4064c743e6c973ed4042360978ed85208 --- /dev/null +++ b/data/datasets/image_classification/usps.py @@ -0,0 +1,44 @@ +from ..data_aug import one_d_image_test_aug, one_d_image_train_aug +from ..ab_dataset import ABDataset +from ..dataset_split import train_val_split +from torchvision.datasets import USPS as RawUSPS +import numpy as np +from typing import Dict, List, Optional +from torchvision.transforms import Compose + +from ..registery import dataset_register + + +@dataset_register( + name='USPS', + classes=[str(i) for i in range(10)], + task_type='Image Classification', + object_type='Digit and Letter', + class_aliases=[], + shift_type=None +) +class USPS(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform: Optional[Compose], + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + if transform is None: + transform = one_d_image_train_aug() if split == 'train' else one_d_image_test_aug() + self.transform = transform + dataset = RawUSPS(root_dir, split != 'test', transform=transform, download=True) + + dataset.targets = np.asarray(dataset.targets) + if len(ignore_classes) > 0: + for ignore_class in ignore_classes: + dataset.data = dataset.data[dataset.targets != classes.index(ignore_class)] + dataset.targets = dataset.targets[dataset.targets != classes.index(ignore_class)] + + if idx_map is not None: + # note: the code below seems correct but has bug! + # for old_idx, new_idx in idx_map.items(): + # dataset.targets[dataset.targets == old_idx] = new_idx + + for ti, t in enumerate(dataset.targets): + dataset.targets[ti] = idx_map[t] + + if split != 'test': + dataset = train_val_split(dataset, split) + return dataset diff --git a/data/datasets/image_classification/usps_single.py b/data/datasets/image_classification/usps_single.py new file mode 100644 index 0000000000000000000000000000000000000000..4c0cc5f7a826148b83c5f4fdc480668466f94b95 --- /dev/null +++ b/data/datasets/image_classification/usps_single.py @@ -0,0 +1,36 @@ +from ..data_aug import one_d_image_test_aug, one_d_image_train_aug +from ..ab_dataset import ABDataset +from ..dataset_split import train_val_test_split +from torchvision.datasets import ImageFolder +import numpy as np +from typing import Dict, List, Optional +from torchvision.transforms import Compose + +from ..registery import dataset_register + + +@dataset_register( + name='USPS-single', + classes=[str(i) for i in range(10)], + task_type='Image Classification', + object_type='Digit and Letter', + class_aliases=[], + shift_type=None +) +class USPSSingle(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform: Optional[Compose], + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + if transform is None: + transform = one_d_image_train_aug() if split == 'train' else one_d_image_test_aug() + self.transform = transform + dataset = ImageFolder(root_dir, transform=transform) + + if len(ignore_classes) > 0: + ignore_classes_idx = [classes.index(c) for c in ignore_classes] + dataset.samples = [s for s in dataset.samples if s[1] not in ignore_classes_idx] + + if idx_map is not None: + dataset.samples = [(s[0], idx_map[s[1]]) if s[1] in idx_map.keys() else s for s in dataset.samples] + + dataset = train_val_test_split(dataset, split) + return dataset \ No newline at end of file diff --git a/data/datasets/machine_translation/REFERENCES.md b/data/datasets/machine_translation/REFERENCES.md new file mode 100644 index 0000000000000000000000000000000000000000..7223c0f9c76d7a503d1edef944972656d8da80c5 --- /dev/null +++ b/data/datasets/machine_translation/REFERENCES.md @@ -0,0 +1,12 @@ +https://blog.csdn.net/tMb8Z9Vdm66wH68VX1/article/details/125827045 use BERT for machine translation + +https://ifwind.github.io/2021/08/26/BERT%E5%AE%9E%E6%88%98%E2%80%94%E2%80%94%EF%BC%881%EF%BC%89%E6%96%87%E6%9C%AC%E5%88%86%E7%B1%BB/#%E6%95%B0%E6%8D%AE%E9%A2%84%E5%A4%84%E7%90%86 machine translation tutorial + + +https://colab.research.google.com/drive/1WIk2bxglElfZewOHboPFNj8H44_VAyKE?usp=sharing#scrollTo=ZwQIEhKOrJpl # offical tutorial code + + + +# NOTE: +# official tutorial code (without trainer, writing from stractch) +https://github.com/huggingface/transformers/blob/66954ea25e342fd451c26ec1c295da0b8692086b/examples/pytorch/translation/run_translation_no_trainer.py \ No newline at end of file diff --git a/data/datasets/machine_translation/__init__.py b/data/datasets/machine_translation/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..7f495d0cebcdff91aa2f59eca56b25cdf5aaa700 --- /dev/null +++ b/data/datasets/machine_translation/__init__.py @@ -0,0 +1 @@ +from .universal_asc_19_domains import * diff --git a/data/datasets/machine_translation/__pycache__/__init__.cpython-38.pyc b/data/datasets/machine_translation/__pycache__/__init__.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3872c3d0410f82766401b68b512cafdc27901b00 Binary files /dev/null and b/data/datasets/machine_translation/__pycache__/__init__.cpython-38.pyc differ diff --git a/data/datasets/machine_translation/__pycache__/universal_asc_19_domains.cpython-38.pyc b/data/datasets/machine_translation/__pycache__/universal_asc_19_domains.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f679257be6d5962f9bc0ec2ec565ce87cea7dfa2 Binary files /dev/null and b/data/datasets/machine_translation/__pycache__/universal_asc_19_domains.cpython-38.pyc differ diff --git a/data/datasets/machine_translation/gen_label.py b/data/datasets/machine_translation/gen_label.py new file mode 100644 index 0000000000000000000000000000000000000000..d69ff36f6eefc096a90c4223f508903d79392ad3 --- /dev/null +++ b/data/datasets/machine_translation/gen_label.py @@ -0,0 +1,94 @@ +from utils.common.data_record import read_json, write_json +import requests +import random +import hashlib +import tqdm +import time + + +session = requests.Session() + + +def translate(sentence): + app_id = '20221004001369410' + salt = str(random.randint(1000000000, 9999999999)) + key = 'XEsBS6babmp9wz5bcoEs' + + sign = hashlib.md5(f'{app_id}{sentence}{salt}{key}'.encode('utf8')).hexdigest() + + response = requests.get( + 'https://fanyi-api.baidu.com/api/trans/vip/translate', + params={ + 'q': sentence, + 'from': 'en', + 'to': 'zh', + 'appid': app_id, + 'salt': salt, + 'sign': sign + } + ).json() + + if 'trans_result' not in response.keys(): + print(response) + raise RuntimeError + + return response['trans_result'][0]['src'], response['trans_result'][0]['dst'] + + +def gen_label_from_sen_cls_json(sen_cls_json_path): + # generate Chinese translation + + texts = [] + anns = read_json(sen_cls_json_path) + for v in anns.values(): + texts += [v['sentence']] + assert '\n' not in texts[-1] + + texts = list(set(texts)) + + res_json = [] + + for text in tqdm.tqdm(texts): + time.sleep(1.2) + + src_text, dst_text = translate(text) + res_json += [{ + 'src': src_text, + 'dst': dst_text + }] + + write_json(sen_cls_json_path + '.translate_data', res_json, backup=False) + + +if __name__ == '__main__': + # res = translate('I am a doctor.\nHello world!') + # print(res) + import os + + data_dir_paths = { + **{k: f'/data/zql/datasets/nlp_asc_19_domains/dat/absa/Bing5Domains/asc/{k.split("-")[1]}' + for k in ['HL5Domains-ApexAD2600Progressive', 'HL5Domains-CanonG3', 'HL5Domains-CreativeLabsNomadJukeboxZenXtra40GB', + 'HL5Domains-NikonCoolpix4300', 'HL5Domains-Nokia6610']}, + + **{k: f'/data/zql/datasets/nlp_asc_19_domains/dat/absa/Bing3Domains/asc/{k.split("-")[1]}' + for k in ['Liu3Domains-Computer', 'Liu3Domains-Router', 'Liu3Domains-Speaker']}, + + **{k: f'/data/zql/datasets/nlp_asc_19_domains/dat/absa/Bing9Domains/asc/{k.split("-")[1]}' + for k in [f'Ding9Domains-{d}' for d in os.listdir('/data/zql/datasets/nlp_asc_19_domains/dat/absa/Bing9Domains/asc')]}, + + **{f'SemEval-{k[0].upper()}{k[1:]}': f'/data/zql/datasets/nlp_asc_19_domains/dat/absa/XuSemEval/asc/14/{k}' + for k in ['laptop', 'rest']}, + } + + json_paths = [] + for p in data_dir_paths.values(): + json_paths += [os.path.join(p, f'{split}.json') for split in ['train', 'dev', 'test']] + + assert all([os.path.exists(p) for p in json_paths]) + + # print(len(json_paths)) + # exit() + + for p in tqdm.tqdm(json_paths[23:]): + print(p) + gen_label_from_sen_cls_json(p) \ No newline at end of file diff --git a/data/datasets/machine_translation/universal_asc_19_domains.py b/data/datasets/machine_translation/universal_asc_19_domains.py new file mode 100644 index 0000000000000000000000000000000000000000..60798b26e58f1fad4d2540e10269c2ba820e9ef4 --- /dev/null +++ b/data/datasets/machine_translation/universal_asc_19_domains.py @@ -0,0 +1,334 @@ +import os +import torch +from torch.utils.data import Dataset, DataLoader +from transformers import BertTokenizer +from torch.optim import Adam +from torch.nn import CrossEntropyLoss +from typing import Dict, List, Optional, Any +from utils.common.data_record import read_json +from ..sentiment_classification.global_bert_tokenizer import get_tokenizer + + +# 自定义数据集类 +class UniversalASC19DomainsTranslationDataset(Dataset): + def __init__(self, root_dir: str, split: str, transform: Any, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + + assert transform is None + + self.tokenizer = get_tokenizer() # 传入tokenizer对象 + self.srcs = [] + self.tgts = [] + self.max_length = None # 设置文本的最大长度 + + json_file_path = os.path.join(root_dir, f'{split if split != "val" else "dev"}.json.translate_data') + anns = read_json(json_file_path) + + # label_map = {'-': 0, '+': 1, 'negative': 0, 'positive': 1} + + # ignore_cls_indexes = [classes.index(c) for c in ignore_classes] + + for info in anns: + self.srcs += [info['src']] + self.tgts += [info['dst']] + + def __len__(self): + return len(self.srcs) + + def __getitem__(self, idx): + src = self.srcs[idx] + tgt = self.tgts[idx] + + encoded_src = self.tokenizer( + src, max_length=self.max_length, padding="max_length", truncation=True, return_tensors="pt" + ) + encoded_tgt = self.tokenizer( + tgt, max_length=self.max_length, padding="max_length", truncation=True, return_tensors="pt" + ) + + x = {key: tensor.squeeze(0) for key, tensor in encoded_src.items()} + + y = encoded_tgt['input_ids'][0] + y = torch.LongTensor([(int(l) if l != self.tokenizer.pad_token_id else -100) for l in y]) + + return x, y + + +from ..ab_dataset import ABDataset +from ..registery import dataset_register + +@dataset_register( + name='HL5Domains-ApexAD2600Progressive-Tr', + classes=['unknown'], + task_type='Machine Translation', + object_type='Generic', + class_aliases=[], + shift_type=None +) +class HL5Domains_ApexAD2600Progressive(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsTranslationDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + +@dataset_register( + name='HL5Domains-CanonG3-Tr', + classes=['unknown'], + task_type='Machine Translation', + object_type='Generic', + class_aliases=[], + shift_type=None +) +class HL5Domains_CanonG3(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsTranslationDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + +@dataset_register( + name='HL5Domains-CreativeLabsNomadJukeboxZenXtra40GB-Tr', + classes=['unknown'], + task_type='Machine Translation', + object_type='Generic', + class_aliases=[], + shift_type=None +) +class HL5Domains_CreativeLabsNomadJukeboxZenXtra40GB(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsTranslationDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + +@dataset_register( + name='HL5Domains-NikonCoolpix4300-Tr', + classes=['unknown'], + task_type='Machine Translation', + object_type='Generic', + class_aliases=[], + shift_type=None +) +class HL5Domains_NikonCoolpix4300(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsTranslationDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + +@dataset_register( + name='HL5Domains-Nokia6610-Tr', + classes=['unknown'], + task_type='Machine Translation', + object_type='Generic', + class_aliases=[], + shift_type=None +) +class HL5Domains_Nokia6610(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsTranslationDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + + +@dataset_register( + name='Liu3Domains-Computer-Tr', + classes=['unknown'], + task_type='Machine Translation', + object_type='Generic', + class_aliases=[], + shift_type=None +) +class Liu3Domains_Computer(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsTranslationDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + +@dataset_register( + name='Liu3Domains-Router-Tr', + classes=['unknown'], + task_type='Machine Translation', + object_type='Generic', + class_aliases=[], + shift_type=None +) +class Liu3Domains_Router(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsTranslationDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + +@dataset_register( + name='Liu3Domains-Speaker-Tr', + classes=['unknown'], + task_type='Machine Translation', + object_type='Generic', + class_aliases=[], + shift_type=None +) +class Liu3Domains_Speaker(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsTranslationDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + + +# import os +# for domain in os.listdir('/data/zql/datasets/nlp_asc_19_domains/dat/absa/Bing9Domains/asc'): +# print(f""" +# @dataset_register( +# name='Ding9Domains-{domain}', +# classes=['unknown'], +# task_type='Machine Translation', +# object_type='Generic', +# class_aliases=[], +# shift_type=None +# ) +# class Ding9Domains_{domain}(ABDataset): +# def create_dataset(self, root_dir: str, split: str, transform, +# classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): +# return UniversalASC19DomainsTranslationDataset(root_dir, split, transform, classes, ignore_classes, idx_map) +# """) + +@dataset_register( + name='Ding9Domains-DiaperChamp-Tr', + classes=['unknown'], + task_type='Machine Translation', + object_type='Generic', + class_aliases=[], + shift_type=None +) +class Ding9Domains_DiaperChamp(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsTranslationDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + + +@dataset_register( + name='Ding9Domains-Norton-Tr', + classes=['unknown'], + task_type='Machine Translation', + object_type='Generic', + class_aliases=[], + shift_type=None +) +class Ding9Domains_Norton(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsTranslationDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + + +@dataset_register( + name='Ding9Domains-LinksysRouter-Tr', + classes=['unknown'], + task_type='Machine Translation', + object_type='Generic', + class_aliases=[], + shift_type=None +) +class Ding9Domains_LinksysRouter(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsTranslationDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + + +@dataset_register( + name='Ding9Domains-MicroMP3-Tr', + classes=['unknown'], + task_type='Machine Translation', + object_type='Generic', + class_aliases=[], + shift_type=None +) +class Ding9Domains_MicroMP3(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsTranslationDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + + +@dataset_register( + name='Ding9Domains-Nokia6600-Tr', + classes=['unknown'], + task_type='Machine Translation', + object_type='Generic', + class_aliases=[], + shift_type=None +) +class Ding9Domains_Nokia6600(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsTranslationDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + + +@dataset_register( + name='Ding9Domains-CanonPowerShotSD500-Tr', + classes=['unknown'], + task_type='Machine Translation', + object_type='Generic', + class_aliases=[], + shift_type=None +) +class Ding9Domains_CanonPowerShotSD500(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsTranslationDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + + +@dataset_register( + name='Ding9Domains-ipod-Tr', + classes=['unknown'], + task_type='Machine Translation', + object_type='Generic', + class_aliases=[], + shift_type=None +) +class Ding9Domains_ipod(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsTranslationDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + + +@dataset_register( + name='Ding9Domains-HitachiRouter-Tr', + classes=['unknown'], + task_type='Machine Translation', + object_type='Generic', + class_aliases=[], + shift_type=None +) +class Ding9Domains_HitachiRouter(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsTranslationDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + + +@dataset_register( + name='Ding9Domains-CanonS100-Tr', + classes=['unknown'], + task_type='Machine Translation', + object_type='Generic', + class_aliases=[], + shift_type=None +) +class Ding9Domains_CanonS100(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsTranslationDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + + + +@dataset_register( + name='SemEval-Laptop-Tr', + classes=['unknown'], + task_type='Machine Translation', + object_type='Generic', + class_aliases=[], + shift_type=None +) +class SemEval_Laptop(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsTranslationDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + +@dataset_register( + name='SemEval-Rest-Tr', + classes=['unknown'], + task_type='Machine Translation', + object_type='Generic', + class_aliases=[], + shift_type=None +) +class SemEval_Rest(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsTranslationDataset(root_dir, split, transform, classes, ignore_classes, idx_map) \ No newline at end of file diff --git a/data/datasets/mm_image_classification/__init__.py b/data/datasets/mm_image_classification/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..c6084568a36eb09b6a273df9a65b1b731b86202a --- /dev/null +++ b/data/datasets/mm_image_classification/__init__.py @@ -0,0 +1,12 @@ +from .imagenet import ImageNet +from .imagenet_c import ImageNetCZoomBlur + +from .imagenet_32 import ImageNet32 +from .imagenet_128 import ImageNet128 +from .imagenet_72 import ImageNet72 + +from .cifar10 import CIFAR10MM +from .cifar10c import CIFAR10CMMxx + +from .stl10 import STL10MM +from .stl10c import STL10MMC \ No newline at end of file diff --git a/data/datasets/mm_image_classification/__pycache__/__init__.cpython-38.pyc b/data/datasets/mm_image_classification/__pycache__/__init__.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..32bf2bff0552b6ff23ca194028c0bb6254f515b5 Binary files /dev/null and b/data/datasets/mm_image_classification/__pycache__/__init__.cpython-38.pyc differ diff --git a/data/datasets/mm_image_classification/__pycache__/cifar10.cpython-38.pyc b/data/datasets/mm_image_classification/__pycache__/cifar10.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3fc7a86d8212de1131791997e7923a637ca0e517 Binary files /dev/null and b/data/datasets/mm_image_classification/__pycache__/cifar10.cpython-38.pyc differ diff --git a/data/datasets/mm_image_classification/__pycache__/cifar10c.cpython-38.pyc b/data/datasets/mm_image_classification/__pycache__/cifar10c.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..285794f10aa8e747595c0abe628d89726c194ca4 Binary files /dev/null and b/data/datasets/mm_image_classification/__pycache__/cifar10c.cpython-38.pyc differ diff --git a/data/datasets/mm_image_classification/__pycache__/imagenet.cpython-38.pyc b/data/datasets/mm_image_classification/__pycache__/imagenet.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2af33ffb91e10a3dbcd56af375a8e5c0944803bd Binary files /dev/null and b/data/datasets/mm_image_classification/__pycache__/imagenet.cpython-38.pyc differ diff --git a/data/datasets/mm_image_classification/__pycache__/imagenet_128.cpython-38.pyc b/data/datasets/mm_image_classification/__pycache__/imagenet_128.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ca8625d524873fa56771acd14667dc4597da480a Binary files /dev/null and b/data/datasets/mm_image_classification/__pycache__/imagenet_128.cpython-38.pyc differ diff --git a/data/datasets/mm_image_classification/__pycache__/imagenet_32.cpython-38.pyc b/data/datasets/mm_image_classification/__pycache__/imagenet_32.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0a299c76434a71c8ea89141144a9319e6988720a Binary files /dev/null and b/data/datasets/mm_image_classification/__pycache__/imagenet_32.cpython-38.pyc differ diff --git a/data/datasets/mm_image_classification/__pycache__/imagenet_72.cpython-38.pyc b/data/datasets/mm_image_classification/__pycache__/imagenet_72.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3d546da33e8146b07e989d6c06aef67b0163d7de Binary files /dev/null and b/data/datasets/mm_image_classification/__pycache__/imagenet_72.cpython-38.pyc differ diff --git a/data/datasets/mm_image_classification/__pycache__/imagenet_c.cpython-38.pyc b/data/datasets/mm_image_classification/__pycache__/imagenet_c.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..604042fd6c978a8949e96dedf97f0488d6492f03 Binary files /dev/null and b/data/datasets/mm_image_classification/__pycache__/imagenet_c.cpython-38.pyc differ diff --git a/data/datasets/mm_image_classification/__pycache__/mm_image_folder.cpython-38.pyc b/data/datasets/mm_image_classification/__pycache__/mm_image_folder.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a84624276691d87bb789947ba2f410b5505b8fd2 Binary files /dev/null and b/data/datasets/mm_image_classification/__pycache__/mm_image_folder.cpython-38.pyc differ diff --git a/data/datasets/mm_image_classification/__pycache__/stl10.cpython-38.pyc b/data/datasets/mm_image_classification/__pycache__/stl10.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4917718dfb2520ed56b013af9c3ecfb6cca204ad Binary files /dev/null and b/data/datasets/mm_image_classification/__pycache__/stl10.cpython-38.pyc differ diff --git a/data/datasets/mm_image_classification/__pycache__/stl10c.cpython-38.pyc b/data/datasets/mm_image_classification/__pycache__/stl10c.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8bfe35b46929f3b78ba0a8f193b3c26e695f42fd Binary files /dev/null and b/data/datasets/mm_image_classification/__pycache__/stl10c.cpython-38.pyc differ diff --git a/data/datasets/mm_image_classification/cifar10.py b/data/datasets/mm_image_classification/cifar10.py new file mode 100644 index 0000000000000000000000000000000000000000..c6c888f7abd7b253a4f5f3e6944b3faaaa923d41 --- /dev/null +++ b/data/datasets/mm_image_classification/cifar10.py @@ -0,0 +1,48 @@ +from ..data_aug import pil_image_to_tensor +from ..ab_dataset import ABDataset +from ..dataset_split import train_val_test_split +from ..dataset_cache import get_dataset_cache_path, read_cached_dataset_status, cache_dataset_status +from .mm_image_folder import MMImageFolder +from ..dataset_split import train_val_split +from torchvision.datasets import CIFAR10 as RawCIFAR10 +import os +from typing import Dict, List, Optional +from torchvision.transforms import Compose +from utils.common.others import HiddenPrints +import numpy as np +from ..registery import dataset_register + + + +@dataset_register( + name='CIFAR10-Single-MM', + classes=['airplane', 'automobile', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck'], + task_type='MM Image Classification', + object_type='Generic Object', + class_aliases=[], + shift_type=None +) +class CIFAR10MM(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform: Optional[Compose], + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + if transform is None: + transform = pil_image_to_tensor(32) + self.transform = transform + + # root_dir = os.path.join(root_dir, 'train' if split != 'test' else 'val') + dataset = MMImageFolder(root_dir, [c for c in classes if c not in ignore_classes], transform=transform) + + cache_file_path = get_dataset_cache_path(root_dir, classes, ignore_classes, idx_map) + if os.path.exists(cache_file_path): + dataset.samples = read_cached_dataset_status(cache_file_path, 'ImageNet-' + split) + else: + if len(ignore_classes) > 0: + ignore_classes_idx = [classes.index(c) for c in ignore_classes] + dataset.samples = [s for s in dataset.samples if s[1] not in ignore_classes_idx] + if idx_map is not None: + dataset.samples = [(s[0], idx_map[s[1]]) if s[1] in idx_map.keys() else s for s in dataset.samples] + + cache_dataset_status(dataset.samples, cache_file_path, 'ImageNet-' + split) + + dataset = train_val_test_split(dataset, split) + return dataset \ No newline at end of file diff --git a/data/datasets/mm_image_classification/cifar10_128.py b/data/datasets/mm_image_classification/cifar10_128.py new file mode 100644 index 0000000000000000000000000000000000000000..0c2ed3ea374a5e55a902b13c120794ea41f17b3a --- /dev/null +++ b/data/datasets/mm_image_classification/cifar10_128.py @@ -0,0 +1,48 @@ +from ..data_aug import pil_image_to_tensor +from ..ab_dataset import ABDataset +from ..dataset_split import train_val_test_split +from ..dataset_cache import get_dataset_cache_path, read_cached_dataset_status, cache_dataset_status +from .mm_image_folder import MMImageFolder +from ..dataset_split import train_val_split +from torchvision.datasets import CIFAR10 as RawCIFAR10 +import os +from typing import Dict, List, Optional +from torchvision.transforms import Compose +from utils.common.others import HiddenPrints +import numpy as np +from ..registery import dataset_register + + + +@dataset_register( + name='CIFAR10-Single-MM-128', + classes=['airplane', 'automobile', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck'], + task_type='MM Image Classification', + object_type='Generic Object', + class_aliases=[], + shift_type=None +) +class CIFAR10MM128(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform: Optional[Compose], + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + if transform is None: + transform = pil_image_to_tensor(128) + self.transform = transform + + # root_dir = os.path.join(root_dir, 'train' if split != 'test' else 'val') + dataset = MMImageFolder(root_dir, [c for c in classes if c not in ignore_classes], transform=transform) + + cache_file_path = get_dataset_cache_path(root_dir, classes, ignore_classes, idx_map) + if os.path.exists(cache_file_path): + dataset.samples = read_cached_dataset_status(cache_file_path, 'ImageNet-' + split) + else: + if len(ignore_classes) > 0: + ignore_classes_idx = [classes.index(c) for c in ignore_classes] + dataset.samples = [s for s in dataset.samples if s[1] not in ignore_classes_idx] + if idx_map is not None: + dataset.samples = [(s[0], idx_map[s[1]]) if s[1] in idx_map.keys() else s for s in dataset.samples] + + cache_dataset_status(dataset.samples, cache_file_path, 'ImageNet-' + split) + + dataset = train_val_test_split(dataset, split) + return dataset \ No newline at end of file diff --git a/data/datasets/mm_image_classification/cifar10c.py b/data/datasets/mm_image_classification/cifar10c.py new file mode 100644 index 0000000000000000000000000000000000000000..8d3417a3f8b31d17edba7a62ebed4b06730e1439 --- /dev/null +++ b/data/datasets/mm_image_classification/cifar10c.py @@ -0,0 +1,72 @@ +from ..data_aug import pil_image_to_tensor +from ..ab_dataset import ABDataset +from ..dataset_split import train_val_test_split +from ..dataset_cache import get_dataset_cache_path, read_cached_dataset_status, cache_dataset_status +from .mm_image_folder import MMImageFolder +from ..dataset_split import train_val_split +from torchvision.datasets import CIFAR10 as RawCIFAR10 +import os +from typing import Dict, List, Optional +from torchvision.transforms import Compose +from utils.common.others import HiddenPrints +import numpy as np +from ..registery import dataset_register + + +class _CIFAR10(RawCIFAR10): + def __init__( + self, + root: str, + classes_exclude_ignored, + train, + transform, + target_transform, + download: bool = False, + + ): + super().__init__(root, train, transform, target_transform, download) + self.classes_exclude_ignored = classes_exclude_ignored + + def __getitem__(self, index: int): + image, label = super().__getitem__(index) + + classes = self.classes_exclude_ignored + # text = f"a photo of a {classes[label].replace('-', ' ').replace('_', ' ').lower()}" # should be ground truth + text = f"a photo of {classes[label]}" # should be ground truth + + x = {'images': image, 'texts': text, 'for_training': False} + return x, label + + +@dataset_register( + name='CIFAR10C-Single-MM-xx', + classes=['airplane', 'automobile', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck'], + task_type='MM Image Classification', + object_type='Generic Object', + class_aliases=[], + shift_type=None +) +class CIFAR10CMMxx(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform: Optional[Compose], + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + if transform is None: + transform = pil_image_to_tensor(32) + self.transform = transform + + # root_dir = os.path.join(root_dir, 'train' if split != 'test' else 'val') + dataset = MMImageFolder(root_dir, [c for c in classes if c not in ignore_classes], transform=transform) + + cache_file_path = get_dataset_cache_path(root_dir, classes, ignore_classes, idx_map) + if os.path.exists(cache_file_path): + dataset.samples = read_cached_dataset_status(cache_file_path, 'ImageNet-' + split) + else: + if len(ignore_classes) > 0: + ignore_classes_idx = [classes.index(c) for c in ignore_classes] + dataset.samples = [s for s in dataset.samples if s[1] not in ignore_classes_idx] + if idx_map is not None: + dataset.samples = [(s[0], idx_map[s[1]]) if s[1] in idx_map.keys() else s for s in dataset.samples] + + cache_dataset_status(dataset.samples, cache_file_path, 'ImageNet-' + split) + + dataset = train_val_test_split(dataset, split) + return dataset \ No newline at end of file diff --git a/data/datasets/mm_image_classification/imagenet.py b/data/datasets/mm_image_classification/imagenet.py new file mode 100644 index 0000000000000000000000000000000000000000..aa0365fcd3e9a672e5bc4b7b797fbea0a684748f --- /dev/null +++ b/data/datasets/mm_image_classification/imagenet.py @@ -0,0 +1,48 @@ +from ..data_aug import pil_image_to_tensor +from ..ab_dataset import ABDataset +from ..dataset_split import train_val_split +from ..dataset_cache import get_dataset_cache_path, read_cached_dataset_status, cache_dataset_status +from .mm_image_folder import MMImageFolder +import os +from typing import Dict, List, Optional +from torchvision.transforms import Compose + +from ..registery import dataset_register + + +with open(os.path.join(os.path.dirname(__file__), 'imagenet_classes.txt'), 'r') as f: + classes = [line.split(' ')[2].strip() for line in f.readlines()] + assert len(classes) == 1000 + +@dataset_register( + name='ImageNet-MM', + classes=classes, + task_type='MM Image Classification', + object_type='Generic Object', + class_aliases=[], + shift_type=None +) +class ImageNet(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform: Optional[Compose], + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + if transform is None: + transform = pil_image_to_tensor() + self.transform = transform + root_dir = os.path.join(root_dir, 'train' if split != 'test' else 'val') + dataset = MMImageFolder(root_dir, [c for c in classes if c not in ignore_classes], transform=transform) + + cache_file_path = get_dataset_cache_path(root_dir, classes, ignore_classes, idx_map) + if os.path.exists(cache_file_path): + dataset.samples = read_cached_dataset_status(cache_file_path, 'ImageNet-' + split) + else: + if len(ignore_classes) > 0: + ignore_classes_idx = [classes.index(c) for c in ignore_classes] + dataset.samples = [s for s in dataset.samples if s[1] not in ignore_classes_idx] + if idx_map is not None: + dataset.samples = [(s[0], idx_map[s[1]]) if s[1] in idx_map.keys() else s for s in dataset.samples] + + cache_dataset_status(dataset.samples, cache_file_path, 'ImageNet-' + split) + + if split != 'test': + dataset = train_val_split(dataset, split) + return dataset diff --git a/data/datasets/mm_image_classification/imagenet_128.py b/data/datasets/mm_image_classification/imagenet_128.py new file mode 100644 index 0000000000000000000000000000000000000000..e1bada99476b254f3093a70e8cbbadfc542bf2a5 --- /dev/null +++ b/data/datasets/mm_image_classification/imagenet_128.py @@ -0,0 +1,48 @@ +from ..data_aug import pil_image_to_tensor +from ..ab_dataset import ABDataset +from ..dataset_split import train_val_split +from ..dataset_cache import get_dataset_cache_path, read_cached_dataset_status, cache_dataset_status +from .mm_image_folder import MMImageFolder +import os +from typing import Dict, List, Optional +from torchvision.transforms import Compose + +from ..registery import dataset_register + + +with open(os.path.join(os.path.dirname(__file__), 'imagenet_classes.txt'), 'r') as f: + classes = [line.split(' ')[2].strip() for line in f.readlines()] + assert len(classes) == 1000 + +@dataset_register( + name='ImageNet-MM-128', + classes=classes, + task_type='MM Image Classification', + object_type='Generic Object', + class_aliases=[], + shift_type=None +) +class ImageNet128(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform: Optional[Compose], + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + if transform is None: + transform = pil_image_to_tensor(128) + self.transform = transform + root_dir = os.path.join(root_dir, 'train' if split != 'test' else 'val') + dataset = MMImageFolder(root_dir, [c for c in classes if c not in ignore_classes], transform=transform) + + cache_file_path = get_dataset_cache_path(root_dir, classes, ignore_classes, idx_map) + if os.path.exists(cache_file_path): + dataset.samples = read_cached_dataset_status(cache_file_path, 'ImageNet-' + split) + else: + if len(ignore_classes) > 0: + ignore_classes_idx = [classes.index(c) for c in ignore_classes] + dataset.samples = [s for s in dataset.samples if s[1] not in ignore_classes_idx] + if idx_map is not None: + dataset.samples = [(s[0], idx_map[s[1]]) if s[1] in idx_map.keys() else s for s in dataset.samples] + + cache_dataset_status(dataset.samples, cache_file_path, 'ImageNet-' + split) + + if split != 'test': + dataset = train_val_split(dataset, split) + return dataset diff --git a/data/datasets/mm_image_classification/imagenet_32.py b/data/datasets/mm_image_classification/imagenet_32.py new file mode 100644 index 0000000000000000000000000000000000000000..09f85dc813e4bdde0c2d7128dc1cbb80dcde5675 --- /dev/null +++ b/data/datasets/mm_image_classification/imagenet_32.py @@ -0,0 +1,48 @@ +from ..data_aug import pil_image_to_tensor +from ..ab_dataset import ABDataset +from ..dataset_split import train_val_split +from ..dataset_cache import get_dataset_cache_path, read_cached_dataset_status, cache_dataset_status +from .mm_image_folder import MMImageFolder +import os +from typing import Dict, List, Optional +from torchvision.transforms import Compose + +from ..registery import dataset_register + + +with open(os.path.join(os.path.dirname(__file__), 'imagenet_classes.txt'), 'r') as f: + classes = [line.split(' ')[2].strip() for line in f.readlines()] + assert len(classes) == 1000 + +@dataset_register( + name='ImageNet-MM-32', + classes=classes, + task_type='MM Image Classification', + object_type='Generic Object', + class_aliases=[], + shift_type=None +) +class ImageNet32(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform: Optional[Compose], + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + if transform is None: + transform = pil_image_to_tensor(32) + self.transform = transform + root_dir = os.path.join(root_dir, 'train' if split != 'test' else 'val') + dataset = MMImageFolder(root_dir, [c for c in classes if c not in ignore_classes], transform=transform) + + cache_file_path = get_dataset_cache_path(root_dir, classes, ignore_classes, idx_map) + if os.path.exists(cache_file_path): + dataset.samples = read_cached_dataset_status(cache_file_path, 'ImageNet-' + split) + else: + if len(ignore_classes) > 0: + ignore_classes_idx = [classes.index(c) for c in ignore_classes] + dataset.samples = [s for s in dataset.samples if s[1] not in ignore_classes_idx] + if idx_map is not None: + dataset.samples = [(s[0], idx_map[s[1]]) if s[1] in idx_map.keys() else s for s in dataset.samples] + + cache_dataset_status(dataset.samples, cache_file_path, 'ImageNet-' + split) + + if split != 'test': + dataset = train_val_split(dataset, split) + return dataset diff --git a/data/datasets/mm_image_classification/imagenet_72.py b/data/datasets/mm_image_classification/imagenet_72.py new file mode 100644 index 0000000000000000000000000000000000000000..eb78be5ad5c3d3bf61ffc61eb60340a58dfdff27 --- /dev/null +++ b/data/datasets/mm_image_classification/imagenet_72.py @@ -0,0 +1,48 @@ +from ..data_aug import pil_image_to_tensor +from ..ab_dataset import ABDataset +from ..dataset_split import train_val_split +from ..dataset_cache import get_dataset_cache_path, read_cached_dataset_status, cache_dataset_status +from .mm_image_folder import MMImageFolder +import os +from typing import Dict, List, Optional +from torchvision.transforms import Compose + +from ..registery import dataset_register + + +with open(os.path.join(os.path.dirname(__file__), 'imagenet_classes.txt'), 'r') as f: + classes = [line.split(' ')[2].strip() for line in f.readlines()] + assert len(classes) == 1000 + +@dataset_register( + name='ImageNet-MM-72', + classes=classes, + task_type='MM Image Classification', + object_type='Generic Object', + class_aliases=[], + shift_type=None +) +class ImageNet72(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform: Optional[Compose], + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + if transform is None: + transform = pil_image_to_tensor(72) + self.transform = transform + root_dir = os.path.join(root_dir, 'train' if split != 'test' else 'val') + dataset = MMImageFolder(root_dir, [c for c in classes if c not in ignore_classes], transform=transform) + + cache_file_path = get_dataset_cache_path(root_dir, classes, ignore_classes, idx_map) + if os.path.exists(cache_file_path): + dataset.samples = read_cached_dataset_status(cache_file_path, 'ImageNet-' + split) + else: + if len(ignore_classes) > 0: + ignore_classes_idx = [classes.index(c) for c in ignore_classes] + dataset.samples = [s for s in dataset.samples if s[1] not in ignore_classes_idx] + if idx_map is not None: + dataset.samples = [(s[0], idx_map[s[1]]) if s[1] in idx_map.keys() else s for s in dataset.samples] + + cache_dataset_status(dataset.samples, cache_file_path, 'ImageNet-' + split) + + if split != 'test': + dataset = train_val_split(dataset, split) + return dataset diff --git a/data/datasets/mm_image_classification/imagenet_c.py b/data/datasets/mm_image_classification/imagenet_c.py new file mode 100644 index 0000000000000000000000000000000000000000..efef86540cf4cb2a38a941aba4b89d5c3ef3fa2b --- /dev/null +++ b/data/datasets/mm_image_classification/imagenet_c.py @@ -0,0 +1,48 @@ +from ..data_aug import pil_image_to_tensor +from ..ab_dataset import ABDataset +from ..dataset_split import train_val_split +from ..dataset_cache import get_dataset_cache_path, read_cached_dataset_status, cache_dataset_status +from .mm_image_folder import MMImageFolder +import os +from typing import Dict, List, Optional +from torchvision.transforms import Compose + +from ..registery import dataset_register + + +with open(os.path.join(os.path.dirname(__file__), 'imagenet_classes.txt'), 'r') as f: + classes = [line.split(' ')[2].strip() for line in f.readlines()] + assert len(classes) == 1000 + +@dataset_register( + name='ImageNet-C-ZoomBlur-MM', + classes=classes, + task_type='MM Image Classification', + object_type='Generic Object', + class_aliases=[], + shift_type=None +) +class ImageNetCZoomBlur(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform: Optional[Compose], + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + if transform is None: + transform = pil_image_to_tensor + self.transform = transform + root_dir = os.path.join(root_dir, 'train' if split != 'test' else 'val') + dataset = MMImageFolder(root_dir, [c for c in classes if c not in ignore_classes], transform=transform) + + cache_file_path = get_dataset_cache_path(root_dir, classes, ignore_classes, idx_map) + if os.path.exists(cache_file_path): + dataset.samples = read_cached_dataset_status(cache_file_path, 'ImageNet-' + split) + else: + if len(ignore_classes) > 0: + ignore_classes_idx = [classes.index(c) for c in ignore_classes] + dataset.samples = [s for s in dataset.samples if s[1] not in ignore_classes_idx] + if idx_map is not None: + dataset.samples = [(s[0], idx_map[s[1]]) if s[1] in idx_map.keys() else s for s in dataset.samples] + + cache_dataset_status(dataset.samples, cache_file_path, 'ImageNet-' + split) + + if split != 'test': + dataset = train_val_split(dataset, split) + return dataset diff --git a/data/datasets/mm_image_classification/imagenet_classes.txt b/data/datasets/mm_image_classification/imagenet_classes.txt new file mode 100644 index 0000000000000000000000000000000000000000..0f1b228c88cb4d56e720e546f9b8b55062ceb4cd --- /dev/null +++ b/data/datasets/mm_image_classification/imagenet_classes.txt @@ -0,0 +1,1000 @@ +n01440764 0 tench +n01443537 1 goldfish +n01484850 2 great_white_shark +n01491361 3 tiger_shark +n01494475 4 hammerhead +n01496331 5 electric_ray +n01498041 6 stingray +n01514668 7 cock +n01514859 8 hen +n01518878 9 ostrich +n01530575 10 brambling +n01531178 11 goldfinch +n01532829 12 house_finch +n01534433 13 junco +n01537544 14 indigo_bunting +n01558993 15 robin +n01560419 16 bulbul +n01580077 17 jay +n01582220 18 magpie +n01592084 19 chickadee +n01601694 20 water_ouzel +n01608432 21 kite +n01614925 22 bald_eagle +n01616318 23 vulture +n01622779 24 great_grey_owl +n01629819 25 European_fire_salamander +n01630670 26 common_newt +n01631663 27 eft +n01632458 28 spotted_salamander +n01632777 29 axolotl +n01641577 30 bullfrog +n01644373 31 tree_frog +n01644900 32 tailed_frog +n01664065 33 loggerhead +n01665541 34 leatherback_turtle +n01667114 35 mud_turtle +n01667778 36 terrapin +n01669191 37 box_turtle +n01675722 38 banded_gecko +n01677366 39 common_iguana +n01682714 40 American_chameleon +n01685808 41 whiptail +n01687978 42 agama +n01688243 43 frilled_lizard +n01689811 44 alligator_lizard +n01692333 45 Gila_monster +n01693334 46 green_lizard +n01694178 47 African_chameleon +n01695060 48 Komodo_dragon +n01697457 49 African_crocodile +n01698640 50 American_alligator +n01704323 51 triceratops +n01728572 52 thunder_snake +n01728920 53 ringneck_snake +n01729322 54 hognose_snake +n01729977 55 green_snake +n01734418 56 king_snake +n01735189 57 garter_snake +n01737021 58 water_snake +n01739381 59 vine_snake +n01740131 60 night_snake +n01742172 61 boa_constrictor +n01744401 62 rock_python +n01748264 63 Indian_cobra +n01749939 64 green_mamba +n01751748 65 sea_snake +n01753488 66 horned_viper +n01755581 67 diamondback +n01756291 68 sidewinder +n01768244 69 trilobite +n01770081 70 harvestman +n01770393 71 scorpion +n01773157 72 black_and_gold_garden_spider +n01773549 73 barn_spider +n01773797 74 garden_spider +n01774384 75 black_widow +n01774750 76 tarantula +n01775062 77 wolf_spider +n01776313 78 tick +n01784675 79 centipede +n01795545 80 black_grouse +n01796340 81 ptarmigan +n01797886 82 ruffed_grouse +n01798484 83 prairie_chicken +n01806143 84 peacock +n01806567 85 quail +n01807496 86 partridge +n01817953 87 African_grey +n01818515 88 macaw +n01819313 89 sulphur-crested_cockatoo +n01820546 90 lorikeet +n01824575 91 coucal +n01828970 92 bee_eater +n01829413 93 hornbill +n01833805 94 hummingbird +n01843065 95 jacamar +n01843383 96 toucan +n01847000 97 drake +n01855032 98 red-breasted_merganser +n01855672 99 goose +n01860187 100 black_swan +n01871265 101 tusker +n01872401 102 echidna +n01873310 103 platypus +n01877812 104 wallaby +n01882714 105 koala +n01883070 106 wombat +n01910747 107 jellyfish +n01914609 108 sea_anemone +n01917289 109 brain_coral +n01924916 110 flatworm +n01930112 111 nematode +n01943899 112 conch +n01944390 113 snail +n01945685 114 slug +n01950731 115 sea_slug +n01955084 116 chiton +n01968897 117 chambered_nautilus +n01978287 118 Dungeness_crab +n01978455 119 rock_crab +n01980166 120 fiddler_crab +n01981276 121 king_crab +n01983481 122 American_lobster +n01984695 123 spiny_lobster +n01985128 124 crayfish +n01986214 125 hermit_crab +n01990800 126 isopod +n02002556 127 white_stork +n02002724 128 black_stork +n02006656 129 spoonbill +n02007558 130 flamingo +n02009229 131 little_blue_heron +n02009912 132 American_egret +n02011460 133 bittern +n02012849 134 crane1 +n02013706 135 limpkin +n02017213 136 European_gallinule +n02018207 137 American_coot +n02018795 138 bustard +n02025239 139 ruddy_turnstone +n02027492 140 red-backed_sandpiper +n02028035 141 redshank +n02033041 142 dowitcher +n02037110 143 oystercatcher +n02051845 144 pelican +n02056570 145 king_penguin +n02058221 146 albatross +n02066245 147 grey_whale +n02071294 148 killer_whale +n02074367 149 dugong +n02077923 150 sea_lion +n02085620 151 Chihuahua +n02085782 152 Japanese_spaniel +n02085936 153 Maltese_dog +n02086079 154 Pekinese +n02086240 155 Shih-Tzu +n02086646 156 Blenheim_spaniel +n02086910 157 papillon +n02087046 158 toy_terrier +n02087394 159 Rhodesian_ridgeback +n02088094 160 Afghan_hound +n02088238 161 basset +n02088364 162 beagle +n02088466 163 bloodhound +n02088632 164 bluetick +n02089078 165 black-and-tan_coonhound +n02089867 166 Walker_hound +n02089973 167 English_foxhound +n02090379 168 redbone +n02090622 169 borzoi +n02090721 170 Irish_wolfhound +n02091032 171 Italian_greyhound +n02091134 172 whippet +n02091244 173 Ibizan_hound +n02091467 174 Norwegian_elkhound +n02091635 175 otterhound +n02091831 176 Saluki +n02092002 177 Scottish_deerhound +n02092339 178 Weimaraner +n02093256 179 Staffordshire_bullterrier +n02093428 180 American_Staffordshire_terrier +n02093647 181 Bedlington_terrier +n02093754 182 Border_terrier +n02093859 183 Kerry_blue_terrier +n02093991 184 Irish_terrier +n02094114 185 Norfolk_terrier +n02094258 186 Norwich_terrier +n02094433 187 Yorkshire_terrier +n02095314 188 wire-haired_fox_terrier +n02095570 189 Lakeland_terrier +n02095889 190 Sealyham_terrier +n02096051 191 Airedale +n02096177 192 cairn +n02096294 193 Australian_terrier +n02096437 194 Dandie_Dinmont +n02096585 195 Boston_bull +n02097047 196 miniature_schnauzer +n02097130 197 giant_schnauzer +n02097209 198 standard_schnauzer +n02097298 199 Scotch_terrier +n02097474 200 Tibetan_terrier +n02097658 201 silky_terrier +n02098105 202 soft-coated_wheaten_terrier +n02098286 203 West_Highland_white_terrier +n02098413 204 Lhasa +n02099267 205 flat-coated_retriever +n02099429 206 curly-coated_retriever +n02099601 207 golden_retriever +n02099712 208 Labrador_retriever +n02099849 209 Chesapeake_Bay_retriever +n02100236 210 German_short-haired_pointer +n02100583 211 vizsla +n02100735 212 English_setter +n02100877 213 Irish_setter +n02101006 214 Gordon_setter +n02101388 215 Brittany_spaniel +n02101556 216 clumber +n02102040 217 English_springer +n02102177 218 Welsh_springer_spaniel +n02102318 219 cocker_spaniel +n02102480 220 Sussex_spaniel +n02102973 221 Irish_water_spaniel +n02104029 222 kuvasz +n02104365 223 schipperke +n02105056 224 groenendael +n02105162 225 malinois +n02105251 226 briard +n02105412 227 kelpie +n02105505 228 komondor +n02105641 229 Old_English_sheepdog +n02105855 230 Shetland_sheepdog +n02106030 231 collie +n02106166 232 Border_collie +n02106382 233 Bouvier_des_Flandres +n02106550 234 Rottweiler +n02106662 235 German_shepherd +n02107142 236 Doberman +n02107312 237 miniature_pinscher +n02107574 238 Greater_Swiss_Mountain_dog +n02107683 239 Bernese_mountain_dog +n02107908 240 Appenzeller +n02108000 241 EntleBucher +n02108089 242 boxer +n02108422 243 bull_mastiff +n02108551 244 Tibetan_mastiff +n02108915 245 French_bulldog +n02109047 246 Great_Dane +n02109525 247 Saint_Bernard +n02109961 248 Eskimo_dog +n02110063 249 malamute +n02110185 250 Siberian_husky +n02110341 251 dalmatian +n02110627 252 affenpinscher +n02110806 253 basenji +n02110958 254 pug +n02111129 255 Leonberg +n02111277 256 Newfoundland +n02111500 257 Great_Pyrenees +n02111889 258 Samoyed +n02112018 259 Pomeranian +n02112137 260 chow +n02112350 261 keeshond +n02112706 262 Brabancon_griffon +n02113023 263 Pembroke +n02113186 264 Cardigan +n02113624 265 toy_poodle +n02113712 266 miniature_poodle +n02113799 267 standard_poodle +n02113978 268 Mexican_hairless +n02114367 269 timber_wolf +n02114548 270 white_wolf +n02114712 271 red_wolf +n02114855 272 coyote +n02115641 273 dingo +n02115913 274 dhole +n02116738 275 African_hunting_dog +n02117135 276 hyena +n02119022 277 red_fox +n02119789 278 kit_fox +n02120079 279 Arctic_fox +n02120505 280 grey_fox +n02123045 281 tabby +n02123159 282 tiger_cat +n02123394 283 Persian_cat +n02123597 284 Siamese_cat +n02124075 285 Egyptian_cat +n02125311 286 cougar +n02127052 287 lynx +n02128385 288 leopard +n02128757 289 snow_leopard +n02128925 290 jaguar +n02129165 291 lion +n02129604 292 tiger +n02130308 293 cheetah +n02132136 294 brown_bear +n02133161 295 American_black_bear +n02134084 296 ice_bear +n02134418 297 sloth_bear +n02137549 298 mongoose +n02138441 299 meerkat +n02165105 300 tiger_beetle +n02165456 301 ladybug +n02167151 302 ground_beetle +n02168699 303 long-horned_beetle +n02169497 304 leaf_beetle +n02172182 305 dung_beetle +n02174001 306 rhinoceros_beetle +n02177972 307 weevil +n02190166 308 fly +n02206856 309 bee +n02219486 310 ant +n02226429 311 grasshopper +n02229544 312 cricket +n02231487 313 walking_stick +n02233338 314 cockroach +n02236044 315 mantis +n02256656 316 cicada +n02259212 317 leafhopper +n02264363 318 lacewing +n02268443 319 dragonfly +n02268853 320 damselfly +n02276258 321 admiral +n02277742 322 ringlet +n02279972 323 monarch +n02280649 324 cabbage_butterfly +n02281406 325 sulphur_butterfly +n02281787 326 lycaenid +n02317335 327 starfish +n02319095 328 sea_urchin +n02321529 329 sea_cucumber +n02325366 330 wood_rabbit +n02326432 331 hare +n02328150 332 Angora +n02342885 333 hamster +n02346627 334 porcupine +n02356798 335 fox_squirrel +n02361337 336 marmot +n02363005 337 beaver +n02364673 338 guinea_pig +n02389026 339 sorrel +n02391049 340 zebra +n02395406 341 hog +n02396427 342 wild_boar +n02397096 343 warthog +n02398521 344 hippopotamus +n02403003 345 ox +n02408429 346 water_buffalo +n02410509 347 bison +n02412080 348 ram +n02415577 349 bighorn +n02417914 350 ibex +n02422106 351 hartebeest +n02422699 352 impala +n02423022 353 gazelle +n02437312 354 Arabian_camel +n02437616 355 llama +n02441942 356 weasel +n02442845 357 mink +n02443114 358 polecat +n02443484 359 black-footed_ferret +n02444819 360 otter +n02445715 361 skunk +n02447366 362 badger +n02454379 363 armadillo +n02457408 364 three-toed_sloth +n02480495 365 orangutan +n02480855 366 gorilla +n02481823 367 chimpanzee +n02483362 368 gibbon +n02483708 369 siamang +n02484975 370 guenon +n02486261 371 patas +n02486410 372 baboon +n02487347 373 macaque +n02488291 374 langur +n02488702 375 colobus +n02489166 376 proboscis_monkey +n02490219 377 marmoset +n02492035 378 capuchin +n02492660 379 howler_monkey +n02493509 380 titi +n02493793 381 spider_monkey +n02494079 382 squirrel_monkey +n02497673 383 Madagascar_cat +n02500267 384 indri +n02504013 385 Indian_elephant +n02504458 386 African_elephant +n02509815 387 lesser_panda +n02510455 388 giant_panda +n02514041 389 barracouta +n02526121 390 eel +n02536864 391 coho +n02606052 392 rock_beauty +n02607072 393 anemone_fish +n02640242 394 sturgeon +n02641379 395 gar +n02643566 396 lionfish +n02655020 397 puffer +n02666196 398 abacus +n02667093 399 abaya +n02669723 400 academic_gown +n02672831 401 accordion +n02676566 402 acoustic_guitar +n02687172 403 aircraft_carrier +n02690373 404 airliner +n02692877 405 airship +n02699494 406 altar +n02701002 407 ambulance +n02704792 408 amphibian +n02708093 409 analog_clock +n02727426 410 apiary +n02730930 411 apron +n02747177 412 ashcan +n02749479 413 assault_rifle +n02769748 414 backpack +n02776631 415 bakery +n02777292 416 balance_beam +n02782093 417 balloon +n02783161 418 ballpoint +n02786058 419 Band_Aid +n02787622 420 banjo +n02788148 421 bannister +n02790996 422 barbell +n02791124 423 barber_chair +n02791270 424 barbershop +n02793495 425 barn +n02794156 426 barometer +n02795169 427 barrel +n02797295 428 barrow +n02799071 429 baseball +n02802426 430 basketball +n02804414 431 bassinet +n02804610 432 bassoon +n02807133 433 bathing_cap +n02808304 434 bath_towel +n02808440 435 bathtub +n02814533 436 beach_wagon +n02814860 437 beacon +n02815834 438 beaker +n02817516 439 bearskin +n02823428 440 beer_bottle +n02823750 441 beer_glass +n02825657 442 bell_cote +n02834397 443 bib +n02835271 444 bicycle-built-for-two +n02837789 445 bikini +n02840245 446 binder +n02841315 447 binoculars +n02843684 448 birdhouse +n02859443 449 boathouse +n02860847 450 bobsled +n02865351 451 bolo_tie +n02869837 452 bonnet +n02870880 453 bookcase +n02871525 454 bookshop +n02877765 455 bottlecap +n02879718 456 bow +n02883205 457 bow_tie +n02892201 458 brass +n02892767 459 brassiere +n02894605 460 breakwater +n02895154 461 breastplate +n02906734 462 broom +n02909870 463 bucket +n02910353 464 buckle +n02916936 465 bulletproof_vest +n02917067 466 bullet_train +n02927161 467 butcher_shop +n02930766 468 cab +n02939185 469 caldron +n02948072 470 candle +n02950826 471 cannon +n02951358 472 canoe +n02951585 473 can_opener +n02963159 474 cardigan +n02965783 475 car_mirror +n02966193 476 carousel +n02966687 477 carpenter's_kit +n02971356 478 carton +n02974003 479 car_wheel +n02977058 480 cash_machine +n02978881 481 cassette +n02979186 482 cassette_player +n02980441 483 castle +n02981792 484 catamaran +n02988304 485 CD_player +n02992211 486 cello +n02992529 487 cellular_telephone +n02999410 488 chain +n03000134 489 chainlink_fence +n03000247 490 chain_mail +n03000684 491 chain_saw +n03014705 492 chest +n03016953 493 chiffonier +n03017168 494 chime +n03018349 495 china_cabinet +n03026506 496 Christmas_stocking +n03028079 497 church +n03032252 498 cinema +n03041632 499 cleaver +n03042490 500 cliff_dwelling +n03045698 501 cloak +n03047690 502 clog +n03062245 503 cocktail_shaker +n03063599 504 coffee_mug +n03063689 505 coffeepot +n03065424 506 coil +n03075370 507 combination_lock +n03085013 508 computer_keyboard +n03089624 509 confectionery +n03095699 510 container_ship +n03100240 511 convertible +n03109150 512 corkscrew +n03110669 513 cornet +n03124043 514 cowboy_boot +n03124170 515 cowboy_hat +n03125729 516 cradle +n03126707 517 crane2 +n03127747 518 crash_helmet +n03127925 519 crate +n03131574 520 crib +n03133878 521 Crock_Pot +n03134739 522 croquet_ball +n03141823 523 crutch +n03146219 524 cuirass +n03160309 525 dam +n03179701 526 desk +n03180011 527 desktop_computer +n03187595 528 dial_telephone +n03188531 529 diaper +n03196217 530 digital_clock +n03197337 531 digital_watch +n03201208 532 dining_table +n03207743 533 dishrag +n03207941 534 dishwasher +n03208938 535 disk_brake +n03216828 536 dock +n03218198 537 dogsled +n03220513 538 dome +n03223299 539 doormat +n03240683 540 drilling_platform +n03249569 541 drum +n03250847 542 drumstick +n03255030 543 dumbbell +n03259280 544 Dutch_oven +n03271574 545 electric_fan +n03272010 546 electric_guitar +n03272562 547 electric_locomotive +n03290653 548 entertainment_center +n03291819 549 envelope +n03297495 550 espresso_maker +n03314780 551 face_powder +n03325584 552 feather_boa +n03337140 553 file +n03344393 554 fireboat +n03345487 555 fire_engine +n03347037 556 fire_screen +n03355925 557 flagpole +n03372029 558 flute +n03376595 559 folding_chair +n03379051 560 football_helmet +n03384352 561 forklift +n03388043 562 fountain +n03388183 563 fountain_pen +n03388549 564 four-poster +n03393912 565 freight_car +n03394916 566 French_horn +n03400231 567 frying_pan +n03404251 568 fur_coat +n03417042 569 garbage_truck +n03424325 570 gasmask +n03425413 571 gas_pump +n03443371 572 goblet +n03444034 573 go-kart +n03445777 574 golf_ball +n03445924 575 golfcart +n03447447 576 gondola +n03447721 577 gong +n03450230 578 gown +n03452741 579 grand_piano +n03457902 580 greenhouse +n03459775 581 grille +n03461385 582 grocery_store +n03467068 583 guillotine +n03476684 584 hair_slide +n03476991 585 hair_spray +n03478589 586 half_track +n03481172 587 hammer +n03482405 588 hamper +n03483316 589 hand_blower +n03485407 590 hand-held_computer +n03485794 591 handkerchief +n03492542 592 hard_disc +n03494278 593 harmonica +n03495258 594 harp +n03496892 595 harvester +n03498962 596 hatchet +n03527444 597 holster +n03529860 598 home_theater +n03530642 599 honeycomb +n03532672 600 hook +n03534580 601 hoopskirt +n03535780 602 horizontal_bar +n03538406 603 horse_cart +n03544143 604 hourglass +n03584254 605 iPod +n03584829 606 iron +n03590841 607 jack-o'-lantern +n03594734 608 jean +n03594945 609 jeep +n03595614 610 jersey +n03598930 611 jigsaw_puzzle +n03599486 612 jinrikisha +n03602883 613 joystick +n03617480 614 kimono +n03623198 615 knee_pad +n03627232 616 knot +n03630383 617 lab_coat +n03633091 618 ladle +n03637318 619 lampshade +n03642806 620 laptop +n03649909 621 lawn_mower +n03657121 622 lens_cap +n03658185 623 letter_opener +n03661043 624 library +n03662601 625 lifeboat +n03666591 626 lighter +n03670208 627 limousine +n03673027 628 liner +n03676483 629 lipstick +n03680355 630 Loafer +n03690938 631 lotion +n03691459 632 loudspeaker +n03692522 633 loupe +n03697007 634 lumbermill +n03706229 635 magnetic_compass +n03709823 636 mailbag +n03710193 637 mailbox +n03710637 638 maillot1 +n03710721 639 maillot2 +n03717622 640 manhole_cover +n03720891 641 maraca +n03721384 642 marimba +n03724870 643 mask +n03729826 644 matchstick +n03733131 645 maypole +n03733281 646 maze +n03733805 647 measuring_cup +n03742115 648 medicine_chest +n03743016 649 megalith +n03759954 650 microphone +n03761084 651 microwave +n03763968 652 military_uniform +n03764736 653 milk_can +n03769881 654 minibus +n03770439 655 miniskirt +n03770679 656 minivan +n03773504 657 missile +n03775071 658 mitten +n03775546 659 mixing_bowl +n03776460 660 mobile_home +n03777568 661 Model_T +n03777754 662 modem +n03781244 663 monastery +n03782006 664 monitor +n03785016 665 moped +n03786901 666 mortar +n03787032 667 mortarboard +n03788195 668 mosque +n03788365 669 mosquito_net +n03791053 670 motor_scooter +n03792782 671 mountain_bike +n03792972 672 mountain_tent +n03793489 673 mouse +n03794056 674 mousetrap +n03796401 675 moving_van +n03803284 676 muzzle +n03804744 677 nail +n03814639 678 neck_brace +n03814906 679 necklace +n03825788 680 nipple +n03832673 681 notebook +n03837869 682 obelisk +n03838899 683 oboe +n03840681 684 ocarina +n03841143 685 odometer +n03843555 686 oil_filter +n03854065 687 organ +n03857828 688 oscilloscope +n03866082 689 overskirt +n03868242 690 oxcart +n03868863 691 oxygen_mask +n03871628 692 packet +n03873416 693 paddle +n03874293 694 paddlewheel +n03874599 695 padlock +n03876231 696 paintbrush +n03877472 697 pajama +n03877845 698 palace +n03884397 699 panpipe +n03887697 700 paper_towel +n03888257 701 parachute +n03888605 702 parallel_bars +n03891251 703 park_bench +n03891332 704 parking_meter +n03895866 705 passenger_car +n03899768 706 patio +n03902125 707 pay-phone +n03903868 708 pedestal +n03908618 709 pencil_box +n03908714 710 pencil_sharpener +n03916031 711 perfume +n03920288 712 Petri_dish +n03924679 713 photocopier +n03929660 714 pick +n03929855 715 pickelhaube +n03930313 716 picket_fence +n03930630 717 pickup +n03933933 718 pier +n03935335 719 piggy_bank +n03937543 720 pill_bottle +n03938244 721 pillow +n03942813 722 ping-pong_ball +n03944341 723 pinwheel +n03947888 724 pirate +n03950228 725 pitcher +n03954731 726 plane +n03956157 727 planetarium +n03958227 728 plastic_bag +n03961711 729 plate_rack +n03967562 730 plow +n03970156 731 plunger +n03976467 732 Polaroid_camera +n03976657 733 pole +n03977966 734 police_van +n03980874 735 poncho +n03982430 736 pool_table +n03983396 737 pop_bottle +n03991062 738 pot +n03992509 739 potter's_wheel +n03995372 740 power_drill +n03998194 741 prayer_rug +n04004767 742 printer +n04005630 743 prison +n04008634 744 projectile +n04009552 745 projector +n04019541 746 puck +n04023962 747 punching_bag +n04026417 748 purse +n04033901 749 quill +n04033995 750 quilt +n04037443 751 racer +n04039381 752 racket +n04040759 753 radiator +n04041544 754 radio +n04044716 755 radio_telescope +n04049303 756 rain_barrel +n04065272 757 recreational_vehicle +n04067472 758 reel +n04069434 759 reflex_camera +n04070727 760 refrigerator +n04074963 761 remote_control +n04081281 762 restaurant +n04086273 763 revolver +n04090263 764 rifle +n04099969 765 rocking_chair +n04111531 766 rotisserie +n04116512 767 rubber_eraser +n04118538 768 rugby_ball +n04118776 769 rule +n04120489 770 running_shoe +n04125021 771 safe +n04127249 772 safety_pin +n04131690 773 saltshaker +n04133789 774 sandal +n04136333 775 sarong +n04141076 776 sax +n04141327 777 scabbard +n04141975 778 scale +n04146614 779 school_bus +n04147183 780 schooner +n04149813 781 scoreboard +n04152593 782 screen +n04153751 783 screw +n04154565 784 screwdriver +n04162706 785 seat_belt +n04179913 786 sewing_machine +n04192698 787 shield +n04200800 788 shoe_shop +n04201297 789 shoji +n04204238 790 shopping_basket +n04204347 791 shopping_cart +n04208210 792 shovel +n04209133 793 shower_cap +n04209239 794 shower_curtain +n04228054 795 ski +n04229816 796 ski_mask +n04235860 797 sleeping_bag +n04238763 798 slide_rule +n04239074 799 sliding_door +n04243546 800 slot +n04251144 801 snorkel +n04252077 802 snowmobile +n04252225 803 snowplow +n04254120 804 soap_dispenser +n04254680 805 soccer_ball +n04254777 806 sock +n04258138 807 solar_dish +n04259630 808 sombrero +n04263257 809 soup_bowl +n04264628 810 space_bar +n04265275 811 space_heater +n04266014 812 space_shuttle +n04270147 813 spatula +n04273569 814 speedboat +n04275548 815 spider_web +n04277352 816 spindle +n04285008 817 sports_car +n04286575 818 spotlight +n04296562 819 stage +n04310018 820 steam_locomotive +n04311004 821 steel_arch_bridge +n04311174 822 steel_drum +n04317175 823 stethoscope +n04325704 824 stole +n04326547 825 stone_wall +n04328186 826 stopwatch +n04330267 827 stove +n04332243 828 strainer +n04335435 829 streetcar +n04336792 830 stretcher +n04344873 831 studio_couch +n04346328 832 stupa +n04347754 833 submarine +n04350905 834 suit +n04355338 835 sundial +n04355933 836 sunglass +n04356056 837 sunglasses +n04357314 838 sunscreen +n04366367 839 suspension_bridge +n04367480 840 swab +n04370456 841 sweatshirt +n04371430 842 swimming_trunks +n04371774 843 swing +n04372370 844 switch +n04376876 845 syringe +n04380533 846 table_lamp +n04389033 847 tank +n04392985 848 tape_player +n04398044 849 teapot +n04399382 850 teddy +n04404412 851 television +n04409515 852 tennis_ball +n04417672 853 thatch +n04418357 854 theater_curtain +n04423845 855 thimble +n04428191 856 thresher +n04429376 857 throne +n04435653 858 tile_roof +n04442312 859 toaster +n04443257 860 tobacco_shop +n04447861 861 toilet_seat +n04456115 862 torch +n04458633 863 totem_pole +n04461696 864 tow_truck +n04462240 865 toyshop +n04465501 866 tractor +n04467665 867 trailer_truck +n04476259 868 tray +n04479046 869 trench_coat +n04482393 870 tricycle +n04483307 871 trimaran +n04485082 872 tripod +n04486054 873 triumphal_arch +n04487081 874 trolleybus +n04487394 875 trombone +n04493381 876 tub +n04501370 877 turnstile +n04505470 878 typewriter_keyboard +n04507155 879 umbrella +n04509417 880 unicycle +n04515003 881 upright +n04517823 882 vacuum +n04522168 883 vase +n04523525 884 vault +n04525038 885 velvet +n04525305 886 vending_machine +n04532106 887 vestment +n04532670 888 viaduct +n04536866 889 violin +n04540053 890 volleyball +n04542943 891 waffle_iron +n04548280 892 wall_clock +n04548362 893 wallet +n04550184 894 wardrobe +n04552348 895 warplane +n04553703 896 washbasin +n04554684 897 washer +n04557648 898 water_bottle +n04560804 899 water_jug +n04562935 900 water_tower +n04579145 901 whiskey_jug +n04579432 902 whistle +n04584207 903 wig +n04589890 904 window_screen +n04590129 905 window_shade +n04591157 906 Windsor_tie +n04591713 907 wine_bottle +n04592741 908 wing +n04596742 909 wok +n04597913 910 wooden_spoon +n04599235 911 wool +n04604644 912 worm_fence +n04606251 913 wreck +n04612504 914 yawl +n04613696 915 yurt +n06359193 916 web_site +n06596364 917 comic_book +n06785654 918 crossword_puzzle +n06794110 919 street_sign +n06874185 920 traffic_light +n07248320 921 book_jacket +n07565083 922 menu +n07579787 923 plate +n07583066 924 guacamole +n07584110 925 consomme +n07590611 926 hot_pot +n07613480 927 trifle +n07614500 928 ice_cream +n07615774 929 ice_lolly +n07684084 930 French_loaf +n07693725 931 bagel +n07695742 932 pretzel +n07697313 933 cheeseburger +n07697537 934 hotdog +n07711569 935 mashed_potato +n07714571 936 head_cabbage +n07714990 937 broccoli +n07715103 938 cauliflower +n07716358 939 zucchini +n07716906 940 spaghetti_squash +n07717410 941 acorn_squash +n07717556 942 butternut_squash +n07718472 943 cucumber +n07718747 944 artichoke +n07720875 945 bell_pepper +n07730033 946 cardoon +n07734744 947 mushroom +n07742313 948 Granny_Smith +n07745940 949 strawberry +n07747607 950 orange +n07749582 951 lemon +n07753113 952 fig +n07753275 953 pineapple +n07753592 954 banana +n07754684 955 jackfruit +n07760859 956 custard_apple +n07768694 957 pomegranate +n07802026 958 hay +n07831146 959 carbonara +n07836838 960 chocolate_sauce +n07860988 961 dough +n07871810 962 meat_loaf +n07873807 963 pizza +n07875152 964 potpie +n07880968 965 burrito +n07892512 966 red_wine +n07920052 967 espresso +n07930864 968 cup +n07932039 969 eggnog +n09193705 970 alp +n09229709 971 bubble +n09246464 972 cliff +n09256479 973 coral_reef +n09288635 974 geyser +n09332890 975 lakeside +n09399592 976 promontory +n09421951 977 sandbar +n09428293 978 seashore +n09468604 979 valley +n09472597 980 volcano +n09835506 981 ballplayer +n10148035 982 groom +n10565667 983 scuba_diver +n11879895 984 rapeseed +n11939491 985 daisy +n12057211 986 yellow_lady's_slipper +n12144580 987 corn +n12267677 988 acorn +n12620546 989 hip +n12768682 990 buckeye +n12985857 991 coral_fungus +n12998815 992 agaric +n13037406 993 gyromitra +n13040303 994 stinkhorn +n13044778 995 earthstar +n13052670 996 hen-of-the-woods +n13054560 997 bolete +n13133613 998 ear +n15075141 999 toilet_tissue diff --git a/data/datasets/mm_image_classification/mm_image_folder.py b/data/datasets/mm_image_classification/mm_image_folder.py new file mode 100644 index 0000000000000000000000000000000000000000..782a671d7b41feb5b7e9ea2a5a429cf6f594ca59 --- /dev/null +++ b/data/datasets/mm_image_classification/mm_image_folder.py @@ -0,0 +1,27 @@ +from typing import Any, Callable, Optional +from torchvision.datasets import ImageFolder +from torchvision.datasets.folder import default_loader + + +class MMImageFolder(ImageFolder): + def __init__( + self, + root: str, + classes_exclude_ignored, + transform=None, + target_transform=None, + loader=default_loader, + is_valid_file=None, + ): + super().__init__(root, transform, target_transform, loader, is_valid_file) + self.classes_exclude_ignored = classes_exclude_ignored + + def __getitem__(self, index: int): + image, label = super().__getitem__(index) + + classes = self.classes_exclude_ignored + # text = f"a photo of a {classes[label].replace('-', ' ').replace('_', ' ').lower()}" # should be ground truth + text = f"a photo of a {classes[label]}" # should be ground truth + + x = {'images': image, 'texts': text, 'for_training': False} + return x, label diff --git a/data/datasets/mm_image_classification/stl10.py b/data/datasets/mm_image_classification/stl10.py new file mode 100644 index 0000000000000000000000000000000000000000..c78f72dde68e17a34b1a8d014d4980028d487504 --- /dev/null +++ b/data/datasets/mm_image_classification/stl10.py @@ -0,0 +1,48 @@ +from ..data_aug import pil_image_to_tensor +from ..ab_dataset import ABDataset +from ..dataset_split import train_val_test_split +from ..dataset_cache import get_dataset_cache_path, read_cached_dataset_status, cache_dataset_status +from .mm_image_folder import MMImageFolder +from ..dataset_split import train_val_split +from torchvision.datasets import CIFAR10 as RawCIFAR10 +import os +from typing import Dict, List, Optional +from torchvision.transforms import Compose +from utils.common.others import HiddenPrints +import numpy as np +from ..registery import dataset_register + + + +@dataset_register( + name='STL10-MM', + classes=['airplane', 'bird', 'car', 'cat', 'deer', 'dog', 'horse', 'monkey', 'ship', 'truck'], + task_type='MM Image Classification', + object_type='Generic Object', + class_aliases=[], + shift_type=None +) +class STL10MM(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform: Optional[Compose], + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + if transform is None: + transform = pil_image_to_tensor(224) + self.transform = transform + + # root_dir = os.path.join(root_dir, 'train' if split != 'test' else 'val') + dataset = MMImageFolder(root_dir, [c for c in classes if c not in ignore_classes], transform=transform) + + cache_file_path = get_dataset_cache_path(root_dir, classes, ignore_classes, idx_map) + if os.path.exists(cache_file_path): + dataset.samples = read_cached_dataset_status(cache_file_path, 'STL10-' + split) + else: + if len(ignore_classes) > 0: + ignore_classes_idx = [classes.index(c) for c in ignore_classes] + dataset.samples = [s for s in dataset.samples if s[1] not in ignore_classes_idx] + if idx_map is not None: + dataset.samples = [(s[0], idx_map[s[1]]) if s[1] in idx_map.keys() else s for s in dataset.samples] + + cache_dataset_status(dataset.samples, cache_file_path, 'STL10-' + split) + + dataset = train_val_test_split(dataset, split) + return dataset \ No newline at end of file diff --git a/data/datasets/mm_image_classification/stl10c.py b/data/datasets/mm_image_classification/stl10c.py new file mode 100644 index 0000000000000000000000000000000000000000..683c37440523888891cecf183e59ef0662bf73b2 --- /dev/null +++ b/data/datasets/mm_image_classification/stl10c.py @@ -0,0 +1,48 @@ +from ..data_aug import pil_image_to_tensor +from ..ab_dataset import ABDataset +from ..dataset_split import train_val_test_split +from ..dataset_cache import get_dataset_cache_path, read_cached_dataset_status, cache_dataset_status +from .mm_image_folder import MMImageFolder +from ..dataset_split import train_val_split +from torchvision.datasets import CIFAR10 as RawCIFAR10 +import os +from typing import Dict, List, Optional +from torchvision.transforms import Compose +from utils.common.others import HiddenPrints +import numpy as np +from ..registery import dataset_register + + + +@dataset_register( + name='STL10-MM-C', + classes=['airplane', 'bird', 'car', 'cat', 'deer', 'dog', 'horse', 'monkey', 'ship', 'truck'], + task_type='MM Image Classification', + object_type='Generic Object', + class_aliases=[], + shift_type=None +) +class STL10MMC(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform: Optional[Compose], + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + if transform is None: + transform = pil_image_to_tensor(224) + self.transform = transform + + # root_dir = os.path.join(root_dir, 'train' if split != 'test' else 'val') + dataset = MMImageFolder(root_dir, [c for c in classes if c not in ignore_classes], transform=transform) + + cache_file_path = get_dataset_cache_path(root_dir, classes, ignore_classes, idx_map) + if os.path.exists(cache_file_path): + dataset.samples = read_cached_dataset_status(cache_file_path, 'STL10-' + split) + else: + if len(ignore_classes) > 0: + ignore_classes_idx = [classes.index(c) for c in ignore_classes] + dataset.samples = [s for s in dataset.samples if s[1] not in ignore_classes_idx] + if idx_map is not None: + dataset.samples = [(s[0], idx_map[s[1]]) if s[1] in idx_map.keys() else s for s in dataset.samples] + + cache_dataset_status(dataset.samples, cache_file_path, 'STL10-' + split) + + dataset = train_val_test_split(dataset, split) + return dataset \ No newline at end of file diff --git a/data/datasets/object_detection/__init__.py b/data/datasets/object_detection/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9a43e5358d18181de81b1ccbb8cc7ebfa938a95c --- /dev/null +++ b/data/datasets/object_detection/__init__.py @@ -0,0 +1,9 @@ +from .voc2012 import VOC2012 +from .coco2017 import COCO2017, MM_COCO2017 +from .makeml_mask import MakeML_Mask +from .wi_mask import WI_Mask + +from .baidu_person_det import BaiduPersonDet +from .cityscapes_det import CityscapesDet, MM_CityscapesDet +from .gta5_det import GTA5Det +from .supervisely_person_det import SuperviselyPersonDet \ No newline at end of file diff --git a/data/datasets/object_detection/__pycache__/__init__.cpython-38.pyc b/data/datasets/object_detection/__pycache__/__init__.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..42489aaa04026c4d8a6804a98d464b562fae9d2a Binary files /dev/null and b/data/datasets/object_detection/__pycache__/__init__.cpython-38.pyc differ diff --git a/data/datasets/object_detection/__pycache__/baidu_person_det.cpython-38.pyc b/data/datasets/object_detection/__pycache__/baidu_person_det.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..372f661eb833a85564b8f3446c8660ce51b5c00f Binary files /dev/null and b/data/datasets/object_detection/__pycache__/baidu_person_det.cpython-38.pyc differ diff --git a/data/datasets/object_detection/__pycache__/cityscapes_det.cpython-38.pyc b/data/datasets/object_detection/__pycache__/cityscapes_det.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3325551fd8cdac6a4bacc63f39460c547b10339e Binary files /dev/null and b/data/datasets/object_detection/__pycache__/cityscapes_det.cpython-38.pyc differ diff --git a/data/datasets/object_detection/__pycache__/coco2017.cpython-38.pyc b/data/datasets/object_detection/__pycache__/coco2017.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8abe9ed656fda4da831a3824b62bfe98ea1aff9e Binary files /dev/null and b/data/datasets/object_detection/__pycache__/coco2017.cpython-38.pyc differ diff --git a/data/datasets/object_detection/__pycache__/gta5_det.cpython-38.pyc b/data/datasets/object_detection/__pycache__/gta5_det.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e0f45a06fa7ada96d4c38cd95189137325b770fc Binary files /dev/null and b/data/datasets/object_detection/__pycache__/gta5_det.cpython-38.pyc differ diff --git a/data/datasets/object_detection/__pycache__/makeml_mask.cpython-38.pyc b/data/datasets/object_detection/__pycache__/makeml_mask.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c5aa9cab2f1ff46d6a1bed463f92f3c3d10f5e92 Binary files /dev/null and b/data/datasets/object_detection/__pycache__/makeml_mask.cpython-38.pyc differ diff --git a/data/datasets/object_detection/__pycache__/supervisely_person_det.cpython-38.pyc b/data/datasets/object_detection/__pycache__/supervisely_person_det.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f6c9ee8f7df5a7c2079f27619d70de87eef010d2 Binary files /dev/null and b/data/datasets/object_detection/__pycache__/supervisely_person_det.cpython-38.pyc differ diff --git a/data/datasets/object_detection/__pycache__/voc2012.cpython-38.pyc b/data/datasets/object_detection/__pycache__/voc2012.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f15fb608aa01e4776991761493df2fae02a89b10 Binary files /dev/null and b/data/datasets/object_detection/__pycache__/voc2012.cpython-38.pyc differ diff --git a/data/datasets/object_detection/__pycache__/wi_mask.cpython-38.pyc b/data/datasets/object_detection/__pycache__/wi_mask.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..de975c3bc1247be1b46951d655fd497314a551db Binary files /dev/null and b/data/datasets/object_detection/__pycache__/wi_mask.cpython-38.pyc differ diff --git a/data/datasets/object_detection/baidu_person_det.py b/data/datasets/object_detection/baidu_person_det.py new file mode 100644 index 0000000000000000000000000000000000000000..45df74565b0ae252cb2d2928751da41c59f0fb7c --- /dev/null +++ b/data/datasets/object_detection/baidu_person_det.py @@ -0,0 +1,41 @@ +from ..ab_dataset import ABDataset +from ..dataset_split import train_val_split, train_val_test_split +from typing import Dict, List, Optional +from torchvision.transforms import Compose +from .yolox_data_util.api import get_default_yolox_coco_dataset, remap_dataset, ensure_index_start_from_1_and_successive, coco_train_val_test_split +import os + +from ..registery import dataset_register + + +@dataset_register( + name='BaiduPersonDet', + classes=[ + 'person' + ], + task_type='Object Detection', + object_type='Person', + class_aliases=[], + shift_type=None +) +class BaiduPersonDet(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform: Optional[Compose], + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + assert transform is None, \ + 'The implementation of object detection datasets is based on YOLOX (https://github.com/Megvii-BaseDetection/YOLOX) ' \ + 'where normal `torchvision.transforms` is not supported. You can re-implement the dataset to override default data aug.' + + ann_json_file_path = os.path.join(root_dir, 'coco_ann.json') + assert os.path.exists(ann_json_file_path), \ + f'Please put the COCO annotation JSON file in root_dir: `{root_dir}/coco_ann.json`.' + + ann_json_file_path = ensure_index_start_from_1_and_successive(ann_json_file_path) + ann_json_file_path = remap_dataset(ann_json_file_path, ignore_classes, idx_map) + ann_json_file_path = coco_train_val_test_split(ann_json_file_path, split) + + self.ann_json_file_path_for_split = ann_json_file_path + + dataset = get_default_yolox_coco_dataset(root_dir, ann_json_file_path, train=(split == 'train')) + + # dataset = train_val_test_split(dataset, split) + return dataset diff --git a/data/datasets/object_detection/cityscapes_det.py b/data/datasets/object_detection/cityscapes_det.py new file mode 100644 index 0000000000000000000000000000000000000000..f78d2c018d0a1ef081f3047b827acb8c17e43117 --- /dev/null +++ b/data/datasets/object_detection/cityscapes_det.py @@ -0,0 +1,71 @@ +from ..ab_dataset import ABDataset +from ..dataset_split import train_val_split, train_val_test_split +from typing import Dict, List, Optional +from torchvision.transforms import Compose +from .yolox_data_util.api import get_default_yolox_coco_dataset, get_yolox_coco_dataset_with_caption, remap_dataset, ensure_index_start_from_1_and_successive, coco_train_val_test_split +import os + +from ..registery import dataset_register + + +@dataset_register( + name='CityscapesDet', + classes=[ + 'car', 'bus' + ], + task_type='Object Detection', + object_type='Driving', + class_aliases=[], + shift_type=None +) +class CityscapesDet(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform: Optional[Compose], + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + assert transform is None, \ + 'The implementation of object detection datasets is based on YOLOX (https://github.com/Megvii-BaseDetection/YOLOX) ' \ + 'where normal `torchvision.transforms` is not supported. You can re-implement the dataset to override default data aug.' + + ann_json_file_path = os.path.join(root_dir, 'coco_ann.json') + assert os.path.exists(ann_json_file_path), \ + f'Please put the COCO annotation JSON file in root_dir: `{root_dir}/coco_ann.json`.' + + ann_json_file_path = ensure_index_start_from_1_and_successive(ann_json_file_path) + ann_json_file_path = remap_dataset(ann_json_file_path, ignore_classes, idx_map) + ann_json_file_path = coco_train_val_test_split(ann_json_file_path, split) + self.ann_json_file_path_for_split = ann_json_file_path + + dataset = get_default_yolox_coco_dataset(root_dir, ann_json_file_path, train=(split == 'train')) + + # dataset = train_val_test_split(dataset, split) + return dataset + +@dataset_register( + name='MM-CityscapesDet', + classes=[ + 'car', 'bus' + ], + task_type='MM Object Detection', + object_type='Driving', + class_aliases=[], + shift_type=None +) +class MM_CityscapesDet(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform: Optional[Compose], + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + # assert transform is None, \ + # 'The implementation of object detection datasets is based on YOLOX (https://github.com/Megvii-BaseDetection/YOLOX) ' \ + # 'where normal `torchvision.transforms` is not supported. You can re-implement the dataset to override default data aug.' + + ann_json_file_path = os.path.join(root_dir, 'coco_ann.json') + assert os.path.exists(ann_json_file_path), \ + f'Please put the COCO annotation JSON file in root_dir: `{root_dir}/coco_ann.json`.' + + ann_json_file_path = ensure_index_start_from_1_and_successive(ann_json_file_path) + ann_json_file_path = remap_dataset(ann_json_file_path, ignore_classes, idx_map) + ann_json_file_path = coco_train_val_test_split(ann_json_file_path, split) + self.ann_json_file_path_for_split = ann_json_file_path + + dataset = get_yolox_coco_dataset_with_caption(root_dir, ann_json_file_path, transform=transform, train=(split == 'train'), classes=classes) + + # dataset = train_val_test_split(dataset, split) + return dataset \ No newline at end of file diff --git a/data/datasets/object_detection/coco2017.py b/data/datasets/object_detection/coco2017.py new file mode 100644 index 0000000000000000000000000000000000000000..11a16033da47a9a34fd1832eb530304f44c1de5b --- /dev/null +++ b/data/datasets/object_detection/coco2017.py @@ -0,0 +1,90 @@ +from ..ab_dataset import ABDataset +from typing import Dict, List, Optional +from torchvision.transforms import Compose +from .yolox_data_util.api import get_default_yolox_coco_dataset, get_yolox_coco_dataset_with_caption, remap_dataset, ensure_index_start_from_1_and_successive, coco_train_val_split +import os + +from ..registery import dataset_register + + +categories = [{"supercategory": "person","id": 1,"name": "person"},{"supercategory": "vehicle","id": 2,"name": "bicycle"},{"supercategory": "vehicle","id": 3,"name": "car"},{"supercategory": "vehicle","id": 4,"name": "motorcycle"},{"supercategory": "vehicle","id": 5,"name": "airplane"},{"supercategory": "vehicle","id": 6,"name": "bus"},{"supercategory": "vehicle","id": 7,"name": "train"},{"supercategory": "vehicle","id": 8,"name": "truck"},{"supercategory": "vehicle","id": 9,"name": "boat"},{"supercategory": "outdoor","id": 10,"name": "traffic light"},{"supercategory": "outdoor","id": 11,"name": "fire hydrant"},{"supercategory": "outdoor","id": 13,"name": "stop sign"},{"supercategory": "outdoor","id": 14,"name": "parking meter"},{"supercategory": "outdoor","id": 15,"name": "bench"},{"supercategory": "animal","id": 16,"name": "bird"},{"supercategory": "animal","id": 17,"name": "cat"},{"supercategory": "animal","id": 18,"name": "dog"},{"supercategory": "animal","id": 19,"name": "horse"},{"supercategory": "animal","id": 20,"name": "sheep"},{"supercategory": "animal","id": 21,"name": "cow"},{"supercategory": "animal","id": 22,"name": "elephant"},{"supercategory": "animal","id": 23,"name": "bear"},{"supercategory": "animal","id": 24,"name": "zebra"},{"supercategory": "animal","id": 25,"name": "giraffe"},{"supercategory": "accessory","id": 27,"name": "backpack"},{"supercategory": "accessory","id": 28,"name": "umbrella"},{"supercategory": "accessory","id": 31,"name": "handbag"},{"supercategory": "accessory","id": 32,"name": "tie"},{"supercategory": "accessory","id": 33,"name": "suitcase"},{"supercategory": "sports","id": 34,"name": "frisbee"},{"supercategory": "sports","id": 35,"name": "skis"},{"supercategory": "sports","id": 36,"name": "snowboard"},{"supercategory": "sports","id": 37,"name": "sports ball"},{"supercategory": "sports","id": 38,"name": "kite"},{"supercategory": "sports","id": 39,"name": "baseball bat"},{"supercategory": "sports","id": 40,"name": "baseball glove"},{"supercategory": "sports","id": 41,"name": "skateboard"},{"supercategory": "sports","id": 42,"name": "surfboard"},{"supercategory": "sports","id": 43,"name": "tennis racket"},{"supercategory": "kitchen","id": 44,"name": "bottle"},{"supercategory": "kitchen","id": 46,"name": "wine glass"},{"supercategory": "kitchen","id": 47,"name": "cup"},{"supercategory": "kitchen","id": 48,"name": "fork"},{"supercategory": "kitchen","id": 49,"name": "knife"},{"supercategory": "kitchen","id": 50,"name": "spoon"},{"supercategory": "kitchen","id": 51,"name": "bowl"},{"supercategory": "food","id": 52,"name": "banana"},{"supercategory": "food","id": 53,"name": "apple"},{"supercategory": "food","id": 54,"name": "sandwich"},{"supercategory": "food","id": 55,"name": "orange"},{"supercategory": "food","id": 56,"name": "broccoli"},{"supercategory": "food","id": 57,"name": "carrot"},{"supercategory": "food","id": 58,"name": "hot dog"},{"supercategory": "food","id": 59,"name": "pizza"},{"supercategory": "food","id": 60,"name": "donut"},{"supercategory": "food","id": 61,"name": "cake"},{"supercategory": "furniture","id": 62,"name": "chair"},{"supercategory": "furniture","id": 63,"name": "couch"},{"supercategory": "furniture","id": 64,"name": "potted plant"},{"supercategory": "furniture","id": 65,"name": "bed"},{"supercategory": "furniture","id": 67,"name": "dining table"},{"supercategory": "furniture","id": 70,"name": "toilet"},{"supercategory": "electronic","id": 72,"name": "tv"},{"supercategory": "electronic","id": 73,"name": "laptop"},{"supercategory": "electronic","id": 74,"name": "mouse"},{"supercategory": "electronic","id": 75,"name": "remote"},{"supercategory": "electronic","id": 76,"name": "keyboard"},{"supercategory": "electronic","id": 77,"name": "cell phone"},{"supercategory": "appliance","id": 78,"name": "microwave"},{"supercategory": "appliance","id": 79,"name": "oven"},{"supercategory": "appliance","id": 80,"name": "toaster"},{"supercategory": "appliance","id": 81,"name": "sink"},{"supercategory": "appliance","id": 82,"name": "refrigerator"},{"supercategory": "indoor","id": 84,"name": "book"},{"supercategory": "indoor","id": 85,"name": "clock"},{"supercategory": "indoor","id": 86,"name": "vase"},{"supercategory": "indoor","id": 87,"name": "scissors"},{"supercategory": "indoor","id": 88,"name": "teddy bear"},{"supercategory": "indoor","id": 89,"name": "hair drier"},{"supercategory": "indoor","id": 90,"name": "toothbrush"}] +classes = [i['name'] for i in categories] + +@dataset_register( + name='COCO2017', + classes=classes, + task_type='Object Detection', + object_type='Generic Object', + class_aliases=[], + shift_type=None +) +class COCO2017(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform: Optional[Compose], + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + assert transform is None, \ + 'The implementation of object detection datasets is based on YOLOX (https://github.com/Megvii-BaseDetection/YOLOX) ' \ + 'where normal `torchvision.transforms` is not supported. You can re-implement the dataset to override default data aug.' + + if split == 'val': + split = 'test' + + root_dir = os.path.join(root_dir, 'train2017' if split != 'test' else 'val2017') + ann_json_file_path = os.path.join(root_dir, 'coco_ann.json') + assert os.path.exists(ann_json_file_path), \ + f'Please put the COCO annotation JSON file in root_dir: `{root_dir}/coco_ann.json`.' + + ann_json_file_path = ensure_index_start_from_1_and_successive(ann_json_file_path) + + ann_json_file_path = remap_dataset(ann_json_file_path, ignore_classes, idx_map) + + if split != 'test': + ann_json_file_path = coco_train_val_split(ann_json_file_path, split) + + # print('coco debug fixed ann_json_file_path') + # ann_json_file_path = '/data/zql/datasets/coco2017/coco/annotations/instances_val2017.json' + + + dataset = get_default_yolox_coco_dataset(root_dir, ann_json_file_path, train=(split == 'train')) + + + return dataset + +@dataset_register( + name='MM-COCO2017', + classes=classes, + task_type='MM Object Detection', + object_type='Generic Object', + class_aliases=[], + shift_type=None +) +class MM_COCO2017(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform: Optional[Compose], + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + # assert transform is None, \ + # 'The implementation of object detection datasets is based on YOLOX (https://github.com/Megvii-BaseDetection/YOLOX) ' \ + # 'where normal `torchvision.transforms` is not supported. You can re-implement the dataset to override default data aug.' + + if split == 'val': + split = 'test' + + root_dir = os.path.join(root_dir, 'train2017' if split != 'test' else 'val2017') + ann_json_file_path = os.path.join(root_dir, 'coco_ann.json') + + assert os.path.exists(ann_json_file_path), \ + f'Please put the COCO annotation JSON file in root_dir: `{root_dir}/coco_ann.json`.' + + ann_json_file_path = ensure_index_start_from_1_and_successive(ann_json_file_path) + + ann_json_file_path = remap_dataset(ann_json_file_path, ignore_classes, idx_map) + + if split != 'test': + ann_json_file_path = coco_train_val_split(ann_json_file_path, split) + + # print('coco debug fixed ann_json_file_path') + # ann_json_file_path = '/data/zql/datasets/coco2017/coco/annotations/instances_val2017.json' + self.ann_json_file_path_for_split = '/data/zql/datasets/coco2017/val2017/coco_ann.json' + + dataset = get_yolox_coco_dataset_with_caption(root_dir, ann_json_file_path, transform=transform, train=(split == 'train')) + self.num_classes = 80 + + return dataset diff --git a/data/datasets/object_detection/gta5_det.py b/data/datasets/object_detection/gta5_det.py new file mode 100644 index 0000000000000000000000000000000000000000..1a4a6c2c20933a97dca39cde599491ed2a891538 --- /dev/null +++ b/data/datasets/object_detection/gta5_det.py @@ -0,0 +1,83 @@ +from ..ab_dataset import ABDataset +from ..dataset_split import train_val_split, train_val_test_split +from typing import Dict, List, Optional +from torchvision.transforms import Compose +from .yolox_data_util.api import get_default_yolox_coco_dataset, get_yolox_coco_dataset_with_caption, remap_dataset, ensure_index_start_from_1_and_successive, coco_train_val_test_split +import os + +from ..registery import dataset_register + + +@dataset_register( + name='GTA5Det', + # classes=[ + # 'road', 'sidewalk', 'building', 'wall', + # 'fence', 'pole', 'light', 'sign', + # 'vegetation', 'terrain', 'sky', 'people', # person + # 'rider', 'car', 'truck', 'bus', 'train', + # 'motocycle', 'bicycle', '?' + # ], + classes=[ + 'car', 'bus' + ], + task_type='Object Detection', + object_type='Driving', + class_aliases=[], + shift_type=None +) +class GTA5Det(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform: Optional[Compose], + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + assert transform is None, \ + 'The implementation of object detection datasets is based on YOLOX (https://github.com/Megvii-BaseDetection/YOLOX) ' \ + 'where normal `torchvision.transforms` is not supported. You can re-implement the dataset to override default data aug.' + + ann_json_file_path = os.path.join(root_dir, 'coco_ann.json') + assert os.path.exists(ann_json_file_path), \ + f'Please put the COCO annotation JSON file in root_dir: `{root_dir}/coco_ann.json`.' + + ann_json_file_path = ensure_index_start_from_1_and_successive(ann_json_file_path) + ann_json_file_path = remap_dataset(ann_json_file_path, ignore_classes, idx_map) + ann_json_file_path = coco_train_val_test_split(ann_json_file_path, split) + + dataset = get_default_yolox_coco_dataset(root_dir, ann_json_file_path, train=(split == 'train')) + + # dataset = train_val_test_split(dataset, split) + return dataset + +@dataset_register( + name='MM-GTA5Det', + # classes=[ + # 'road', 'sidewalk', 'building', 'wall', + # 'fence', 'pole', 'light', 'sign', + # 'vegetation', 'terrain', 'sky', 'people', # person + # 'rider', 'car', 'truck', 'bus', 'train', + # 'motocycle', 'bicycle', '?' + # ], + classes=[ + 'car', 'bus' + ], + task_type='MM Object Detection', + object_type='Driving', + class_aliases=[], + shift_type=None +) +class MM_GTA5Det(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform: Optional[Compose], + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + # assert transform is None, \ + # 'The implementation of object detection datasets is based on YOLOX (https://github.com/Megvii-BaseDetection/YOLOX) ' \ + # 'where normal `torchvision.transforms` is not supported. You can re-implement the dataset to override default data aug.' + + ann_json_file_path = os.path.join(root_dir, 'coco_ann.json') + assert os.path.exists(ann_json_file_path), \ + f'Please put the COCO annotation JSON file in root_dir: `{root_dir}/coco_ann.json`.' + + ann_json_file_path = ensure_index_start_from_1_and_successive(ann_json_file_path) + ann_json_file_path = remap_dataset(ann_json_file_path, ignore_classes, idx_map) + ann_json_file_path = coco_train_val_test_split(ann_json_file_path, split) + self.ann_json_file_path_for_split = ann_json_file_path + dataset = get_yolox_coco_dataset_with_caption(root_dir, ann_json_file_path, transform=transform, train=(split == 'train'), classes=classes) + + # dataset = train_val_test_split(dataset, split) + return dataset \ No newline at end of file diff --git a/data/datasets/object_detection/makeml_mask.py b/data/datasets/object_detection/makeml_mask.py new file mode 100644 index 0000000000000000000000000000000000000000..ef0f77b4d709e03f884e60e6cea707c77bb5ad61 --- /dev/null +++ b/data/datasets/object_detection/makeml_mask.py @@ -0,0 +1,37 @@ +from ..ab_dataset import ABDataset +# from ..dataset_split import train_val_split, train_val_test_split +from typing import Dict, List, Optional +from torchvision.transforms import Compose +from .yolox_data_util.api import get_default_yolox_coco_dataset, remap_dataset, ensure_index_start_from_1_and_successive, coco_train_val_test_split +import os + +from ..registery import dataset_register + + +@dataset_register( + name='MakeML_Mask', + classes=['face_no_mask', 'face_with_mask'], + task_type='Object Detection', + object_type='Face Mask', + class_aliases=[], + shift_type=None +) +class MakeML_Mask(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform: Optional[Compose], + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + assert transform is None, \ + 'The implementation of object detection datasets is based on YOLOX (https://github.com/Megvii-BaseDetection/YOLOX) ' \ + 'where normal `torchvision.transforms` is not supported. You can re-implement the dataset to override default data aug.' + + ann_json_file_path = os.path.join(root_dir, 'coco_ann.json') + assert os.path.exists(ann_json_file_path), \ + f'Please put the COCO annotation JSON file in root_dir: `{root_dir}/coco_ann.json`.' + + ann_json_file_path = ensure_index_start_from_1_and_successive(ann_json_file_path) + ann_json_file_path = remap_dataset(ann_json_file_path, ignore_classes, idx_map) + ann_json_file_path = coco_train_val_test_split(ann_json_file_path, split) + + dataset = get_default_yolox_coco_dataset(root_dir, ann_json_file_path, train=(split == 'train')) + + + return dataset diff --git a/data/datasets/object_detection/supervisely_person_det.py b/data/datasets/object_detection/supervisely_person_det.py new file mode 100644 index 0000000000000000000000000000000000000000..d5b2d6167679d7f2fb98a6b60df89686973f506d --- /dev/null +++ b/data/datasets/object_detection/supervisely_person_det.py @@ -0,0 +1,39 @@ +from ..ab_dataset import ABDataset +from ..dataset_split import train_val_split, train_val_test_split +from typing import Dict, List, Optional +from torchvision.transforms import Compose +from .yolox_data_util.api import get_default_yolox_coco_dataset, remap_dataset, ensure_index_start_from_1_and_successive, coco_train_val_test_split +import os + +from ..registery import dataset_register + + +@dataset_register( + name='SuperviselyPersonDet', + classes=[ + 'person' + ], + task_type='Object Detection', + object_type='Person', + class_aliases=[], + shift_type=None +) +class SuperviselyPersonDet(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform: Optional[Compose], + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + assert transform is None, \ + 'The implementation of object detection datasets is based on YOLOX (https://github.com/Megvii-BaseDetection/YOLOX) ' \ + 'where normal `torchvision.transforms` is not supported. You can re-implement the dataset to override default data aug.' + + ann_json_file_path = os.path.join(root_dir, 'coco_ann.json') + assert os.path.exists(ann_json_file_path), \ + f'Please put the COCO annotation JSON file in root_dir: `{root_dir}/coco_ann.json`.' + + ann_json_file_path = ensure_index_start_from_1_and_successive(ann_json_file_path) + ann_json_file_path = remap_dataset(ann_json_file_path, ignore_classes, idx_map) + ann_json_file_path = coco_train_val_test_split(ann_json_file_path, split) + + dataset = get_default_yolox_coco_dataset(root_dir, ann_json_file_path, train=(split == 'train')) + + # dataset = train_val_test_split(dataset, split) + return dataset diff --git a/data/datasets/object_detection/voc2012.py b/data/datasets/object_detection/voc2012.py new file mode 100644 index 0000000000000000000000000000000000000000..f35c5a7caf4ce74b6d2e0f66624f66630be669c2 --- /dev/null +++ b/data/datasets/object_detection/voc2012.py @@ -0,0 +1,40 @@ +from ..ab_dataset import ABDataset +from ..dataset_split import train_val_split, train_val_test_split +from typing import Dict, List, Optional +from torchvision.transforms import Compose +from .yolox_data_util.api import get_default_yolox_coco_dataset, remap_dataset, ensure_index_start_from_1_and_successive, coco_train_val_test_split +import os + +from ..registery import dataset_register + + +categories = [{"supercategory": "none", "id": 0, "name": "boat"}, {"supercategory": "none", "id": 1, "name": "person"}, {"supercategory": "none", "id": 2, "name": "car"}, {"supercategory": "none", "id": 3, "name": "bus"}, {"supercategory": "none", "id": 4, "name": "horse"}, {"supercategory": "none", "id": 5, "name": "train"}, {"supercategory": "none", "id": 6, "name": "chair"}, {"supercategory": "none", "id": 7, "name": "aeroplane"}, {"supercategory": "none", "id": 8, "name": "dog"}, {"supercategory": "none", "id": 9, "name": "pottedplant"}, {"supercategory": "none", "id": 10, "name": "motorbike"}, {"supercategory": "none", "id": 11, "name": "cat"}, {"supercategory": "none", "id": 12, "name": "bicycle"}, {"supercategory": "none", "id": 13, "name": "bird"}, {"supercategory": "none", "id": 14, "name": "bottle"}, {"supercategory": "none", "id": 15, "name": "sofa"}, {"supercategory": "none", "id": 16, "name": "diningtable"}, {"supercategory": "none", "id": 17, "name": "tvmonitor"}, {"supercategory": "none", "id": 18, "name": "sheep"}, {"supercategory": "none", "id": 19, "name": "cow"}] +classes = [i['name'] for i in categories] + +@dataset_register( + name='VOC2012', + classes=classes, + task_type='Object Detection', + object_type='Generic Object', + class_aliases=[], + shift_type=None +) +class VOC2012(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform: Optional[Compose], + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + assert transform is None, \ + 'The implementation of object detection datasets is based on YOLOX (https://github.com/Megvii-BaseDetection/YOLOX) ' \ + 'where normal `torchvision.transforms` is not supported. You can re-implement the dataset to override default data aug.' + + ann_json_file_path = os.path.join(root_dir, 'coco_ann.json') + assert os.path.exists(ann_json_file_path), \ + f'Please put the COCO annotation JSON file in root_dir: `{root_dir}/coco_ann.json`.' + + ann_json_file_path = ensure_index_start_from_1_and_successive(ann_json_file_path) + ann_json_file_path = remap_dataset(ann_json_file_path, ignore_classes, idx_map) + ann_json_file_path = coco_train_val_test_split(ann_json_file_path, split) + + dataset = get_default_yolox_coco_dataset(root_dir, ann_json_file_path, train=(split == 'train')) + + # dataset = train_val_test_split(dataset, split) + return dataset diff --git a/data/datasets/object_detection/wi_mask.py b/data/datasets/object_detection/wi_mask.py new file mode 100644 index 0000000000000000000000000000000000000000..7bb34a4cfd71226db833e4ce0fe43e427507c45f --- /dev/null +++ b/data/datasets/object_detection/wi_mask.py @@ -0,0 +1,37 @@ +from ..ab_dataset import ABDataset +from ..dataset_split import train_val_split, train_val_test_split +from typing import Dict, List, Optional +from torchvision.transforms import Compose +from .yolox_data_util.api import get_default_yolox_coco_dataset, remap_dataset, ensure_index_start_from_1_and_successive, coco_train_val_test_split +import os + +from ..registery import dataset_register + + +@dataset_register( + name='WI_Mask', + classes=['face_no_mask', 'face_with_mask'], + task_type='Object Detection', + object_type='Face Mask', + class_aliases=[], + shift_type=None +) +class WI_Mask(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform: Optional[Compose], + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + assert transform is None, \ + 'The implementation of object detection datasets is based on YOLOX (https://github.com/Megvii-BaseDetection/YOLOX) ' \ + 'where normal `torchvision.transforms` is not supported. You can re-implement the dataset to override default data aug.' + + ann_json_file_path = os.path.join(root_dir, 'coco_ann.json') + assert os.path.exists(ann_json_file_path), \ + f'Please put the COCO annotation JSON file in root_dir: `{root_dir}/coco_ann.json`.' + + ann_json_file_path = ensure_index_start_from_1_and_successive(ann_json_file_path) + ann_json_file_path = remap_dataset(ann_json_file_path, ignore_classes, idx_map) + ann_json_file_path = coco_train_val_test_split(ann_json_file_path, split) + + dataset = get_default_yolox_coco_dataset(root_dir, ann_json_file_path, train=(split == 'train')) + + # dataset = train_val_test_split(dataset, split) + return dataset diff --git a/data/datasets/object_detection/yolox_data_util/NOTE.md b/data/datasets/object_detection/yolox_data_util/NOTE.md new file mode 100644 index 0000000000000000000000000000000000000000..ed9ee62c48497e3368352b2a37b466d36dc42381 --- /dev/null +++ b/data/datasets/object_detection/yolox_data_util/NOTE.md @@ -0,0 +1 @@ +This implementation is from https://github.com/Megvii-BaseDetection/YOLOX ! diff --git a/data/datasets/object_detection/yolox_data_util/__init__.py b/data/datasets/object_detection/yolox_data_util/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..aeaf4f930ab8b9890ca43ba031f5b035be623ccd --- /dev/null +++ b/data/datasets/object_detection/yolox_data_util/__init__.py @@ -0,0 +1,9 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +# Copyright (c) Megvii, Inc. and its affiliates. + +from .data_augment import TrainTransform, ValTransform +from .data_prefetcher import DataPrefetcher +from .dataloading import DataLoader, get_yolox_datadir, worker_init_reset_seed +from .datasets import * +from .samplers import InfiniteSampler, YoloBatchSampler diff --git a/data/datasets/object_detection/yolox_data_util/__pycache__/__init__.cpython-38.pyc b/data/datasets/object_detection/yolox_data_util/__pycache__/__init__.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4cd2717074cae063cc3a00b11a2d219bdd784374 Binary files /dev/null and b/data/datasets/object_detection/yolox_data_util/__pycache__/__init__.cpython-38.pyc differ diff --git a/data/datasets/object_detection/yolox_data_util/__pycache__/api.cpython-38.pyc b/data/datasets/object_detection/yolox_data_util/__pycache__/api.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..71ab02066a447f36278c6cd43fee30c383710273 Binary files /dev/null and b/data/datasets/object_detection/yolox_data_util/__pycache__/api.cpython-38.pyc differ diff --git a/data/datasets/object_detection/yolox_data_util/__pycache__/data_augment.cpython-38.pyc b/data/datasets/object_detection/yolox_data_util/__pycache__/data_augment.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e8e6fa159dca17326f75242cf32a27e477328de8 Binary files /dev/null and b/data/datasets/object_detection/yolox_data_util/__pycache__/data_augment.cpython-38.pyc differ diff --git a/data/datasets/object_detection/yolox_data_util/__pycache__/data_prefetcher.cpython-38.pyc b/data/datasets/object_detection/yolox_data_util/__pycache__/data_prefetcher.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d7ff29943d6513aafaba79f9c272b2a0a6c60df9 Binary files /dev/null and b/data/datasets/object_detection/yolox_data_util/__pycache__/data_prefetcher.cpython-38.pyc differ diff --git a/data/datasets/object_detection/yolox_data_util/__pycache__/dataloading.cpython-38.pyc b/data/datasets/object_detection/yolox_data_util/__pycache__/dataloading.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3b6c022620d7b9d79a8d2045e1c72d071223cff2 Binary files /dev/null and b/data/datasets/object_detection/yolox_data_util/__pycache__/dataloading.cpython-38.pyc differ diff --git a/data/datasets/object_detection/yolox_data_util/__pycache__/norm_categories_index.cpython-38.pyc b/data/datasets/object_detection/yolox_data_util/__pycache__/norm_categories_index.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d0253e6f6b9a94e4eeca4011711b1d227c82abbc Binary files /dev/null and b/data/datasets/object_detection/yolox_data_util/__pycache__/norm_categories_index.cpython-38.pyc differ diff --git a/data/datasets/object_detection/yolox_data_util/__pycache__/samplers.cpython-38.pyc b/data/datasets/object_detection/yolox_data_util/__pycache__/samplers.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8e4579fcfddd59e0b440fb8db0d01cfccc0035f9 Binary files /dev/null and b/data/datasets/object_detection/yolox_data_util/__pycache__/samplers.cpython-38.pyc differ diff --git a/data/datasets/object_detection/yolox_data_util/api.py b/data/datasets/object_detection/yolox_data_util/api.py new file mode 100644 index 0000000000000000000000000000000000000000..95b69042ae44a7ab477908465c9f7dc326a1f4c5 --- /dev/null +++ b/data/datasets/object_detection/yolox_data_util/api.py @@ -0,0 +1,306 @@ +from curses import raw +from .data_augment import TrainTransform, ValTransform +from .datasets.coco import COCODataset +from .datasets.mm_coco import MM_COCODataset +from .datasets.mosaicdetection import MosaicDetection +from utils.common.others import HiddenPrints +import os +import json +from tqdm import tqdm +from utils.common.log import logger + +from .norm_categories_index import ensure_index_start_from_1_and_successive + + +def get_default_yolox_coco_dataset(data_dir, json_file_path, img_size=416, train=True): + logger.info(f'[get yolox dataset] "{json_file_path}"') + + if train: + with HiddenPrints(): + dataset = COCODataset( + data_dir=data_dir, + json_file=json_file_path, + name='', + img_size=(img_size, img_size), + preproc=TrainTransform( + max_labels=50, + flip_prob=0.5, + hsv_prob=1.0), + cache=False, + ) + # dataset = COCODataset( + # data_dir=data_dir, + # json_file=json_file_path, + # name='', + # img_size=(img_size, img_size), + # preproc=ValTransform(legacy=False), + # ) + + dataset = MosaicDetection( + dataset, + mosaic=True, + img_size=(img_size, img_size), + preproc=TrainTransform( + max_labels=120, + flip_prob=0.5, + hsv_prob=1.0), + degrees=10.0, + translate=0.1, + mosaic_scale=(0.1, 2), + mixup_scale=(0.5, 1.5), + shear=2.0, + enable_mixup=True, + mosaic_prob=1.0, + mixup_prob=1.0, + only_return_xy=True + ) + + else: + with HiddenPrints(): + dataset = COCODataset( + data_dir=data_dir, + json_file=json_file_path, + name='', + img_size=(img_size, img_size), + preproc=ValTransform(legacy=False), + ) + + # print(json_file_path, len(dataset)) + + return dataset + +def get_yolox_coco_dataset_with_caption(data_dir, json_file_path, img_size=416, transform=None, train=True, classes=None): + logger.info(f'[get yolox dataset] "{json_file_path}"') + + if train: + with HiddenPrints(): + dataset = COCODataset( + data_dir=data_dir, + json_file=json_file_path, + name='', + img_size=(img_size, img_size), + preproc=TrainTransform( + max_labels=50, + flip_prob=0.5, + hsv_prob=1.0), + cache=False, + ) + # dataset = COCODataset( + # data_dir=data_dir, + # json_file=json_file_path, + # name='', + # img_size=(img_size, img_size), + # preproc=ValTransform(legacy=False), + # ) + coco = dataset.coco + dataset = MosaicDetection( + dataset, + mosaic=True, + img_size=(img_size, img_size), + preproc=TrainTransform( + max_labels=120, + flip_prob=0.5, + hsv_prob=1.0), + degrees=10.0, + translate=0.1, + mosaic_scale=(0.1, 2), + mixup_scale=(0.5, 1.5), + shear=2.0, + enable_mixup=True, + mosaic_prob=1.0, + mixup_prob=1.0, + only_return_xy=True + ) + dataset = MM_COCODataset(dataset, transform=transform, split='train', coco=coco, classes=classes) + else: + with HiddenPrints(): + dataset = COCODataset( + data_dir=data_dir, + json_file=json_file_path, + name='', + img_size=(img_size, img_size), + preproc=ValTransform(legacy=False), + ) + dataset = MM_COCODataset(dataset, transform=transform, split='val', coco=dataset.coco, classes=classes) + # print(json_file_path, len(dataset)) + + return dataset + +import hashlib + +def _hash(o): + if isinstance(o, list): + o = sorted(o) + elif isinstance(o, dict): + o = {k: o[k] for k in sorted(o)} + elif isinstance(o, set): + o = sorted(list(o)) + # else: + # print(type(o)) + + obj = hashlib.md5() + obj.update(str(o).encode('utf-8')) + return obj.hexdigest() + + +DEBUG = True + + +def remap_dataset(json_file_path, ignore_classes, category_idx_map): + # k and v in category_idx_map indicates the index of categories, not 'id' of categories + ignore_classes = sorted(list(ignore_classes)) + # print(ignore_classes, category_idx_map) + + if len(ignore_classes) == 0 and category_idx_map is None: + return json_file_path + + # hash_str = '_'.join(ignore_classes) + str(category_idx_map) + hash_str = _hash(f'yolox_dataset_cache_{_hash(ignore_classes)}_{_hash(category_idx_map)}') + cached_json_file_path = f'{json_file_path}.{hash(hash_str)}' + + # TODO: + if os.path.exists(cached_json_file_path): + if DEBUG: + os.remove(cached_json_file_path) + else: + logger.info(f'get cached dataset in {cached_json_file_path}') + return cached_json_file_path + + with open(json_file_path, 'r') as f: + raw_ann = json.load(f) + id_to_idx_map = {c['id']: i for i, c in enumerate(raw_ann['categories'])} + + ignore_classes_id = [c['id'] for c in raw_ann['categories'] if c['name'] in ignore_classes] + raw_ann['categories'] = [c for c in raw_ann['categories'] if c['id'] not in ignore_classes_id] + raw_ann['annotations'] = [ann for ann in raw_ann['annotations'] if ann['category_id'] not in ignore_classes_id] + ann_img_map = {ann['image_id']: 1 for ann in raw_ann['annotations']} + raw_ann['images'] = [img for img in raw_ann['images'] if img['id'] in ann_img_map.keys()] + + # print(category_idx_map, id_to_idx_map) + # NOTE: category idx starts from 0 or 1? 1 + # NOTE: reshuffle "categories" + new_categories = [{"id": i, "name": f"dummy-{i}"} for i in range(int(os.environ['_ZQL_NUMC']))] + for c in raw_ann['categories']: + # print(c) + # print(id_to_idx_map, c['id'], category_idx_map) + new_idx = category_idx_map[id_to_idx_map[c['id']]] + new_categories[new_idx] = c + c['id'] = new_idx + raw_ann['categories'] = new_categories + for ann in raw_ann['annotations']: + ann['category_id'] = category_idx_map[id_to_idx_map[ann['category_id']]] + if 'segmentation' in ann: + del ann['segmentation'] + + with open(cached_json_file_path, 'w') as f: + json.dump(raw_ann, f) + + return cached_json_file_path + + +def coco_split(ann_json_file_path, ratio=0.8): + if os.path.exists(ann_json_file_path + f'.{ratio}.split1') and not DEBUG: + return ann_json_file_path + f'.{ratio}.split1', ann_json_file_path + f'.{ratio}.split2' + + with open(ann_json_file_path, 'r') as f: + raw_ann = json.load(f) + + import copy + import torch + res_ann1, res_ann2 = copy.deepcopy(raw_ann), copy.deepcopy(raw_ann) + + images = raw_ann['images'] + + cache_images_path = ann_json_file_path + '.tmp-cached-shuffled-images' + if True: + import random + random.shuffle(images) + torch.save(images, cache_images_path) + else: + images = torch.load(cache_images_path) + + images1, images2 = images[0: int(len(images) * ratio)], images[int(len(images) * ratio): ] + images1_id, images2_id = {i['id']: 0 for i in images1}, {i['id']: 0 for i in images2} + ann1 = [ann for ann in raw_ann['annotations'] if ann['image_id'] in images1_id.keys()] + ann2 = [ann for ann in raw_ann['annotations'] if ann['image_id'] in images2_id.keys()] + + res_ann1['images'] = images1 + res_ann1['annotations'] = ann1 + res_ann2['images'] = images2 + res_ann2['annotations'] = ann2 + + from utils.common.data_record import write_json + write_json(ann_json_file_path + f'.{ratio}.split1', res_ann1, indent=0, backup=False) + write_json(ann_json_file_path + f'.{ratio}.split2', res_ann2, indent=0, backup=False) + + return ann_json_file_path + f'.{ratio}.split1', ann_json_file_path + f'.{ratio}.split2' + + +def coco_train_val_test_split(ann_json_file_path, split): + train_ann_p, test_ann_p = coco_split(ann_json_file_path) + if split == 'test': + return test_ann_p + train_ann_p, val_ann_p = coco_split(train_ann_p) + return train_ann_p if split == 'train' else val_ann_p + + +def coco_train_val_split(train_ann_p, split): + train_ann_p, val_ann_p = coco_split(train_ann_p) + return train_ann_p if split == 'train' else val_ann_p + + +def visualize_coco_dataset(dataset, num_images, res_save_p, cxcy): + from torchvision.transforms import ToTensor + from torchvision.utils import make_grid + from PIL import Image, ImageDraw + import matplotlib.pyplot as plt + import numpy as np + + def draw_bbox(img, bbox, label, f): + # if f: + # img = np.uint8(img.transpose(1, 2, 0)) + img = Image.fromarray(img) + draw = ImageDraw.Draw(img) + draw.rectangle(bbox, outline=(255, 0, 0), width=6) + draw.text((bbox[0], bbox[1]), label) + return np.array(img) + + d = dataset.dataset + if d.__class__.__name__ == 'MosaicDetection': + d = d._dataset + class_ids = d.class_ids # category_id + def get_cname(label): + return d.coco.loadCats(class_ids[int(label)])[0]['name'] + + def cxcywh2xyxy(bbox): + cx, cy, w, h = bbox + x1, y1 = cx - w/2, cy - h/2 + x2, y2 = cx + w/2, cy + h/2 + return x1, y1, x2, y2 + + xs = [] + import random + for image_i in range(num_images): + x, y = dataset[random.randint(0, len(dataset) - 1)][:2] + x = np.uint8(x.transpose(1, 2, 0)) + + for label_i, label_info in enumerate(y): + if sum(label_info[1:]) == 0: # pad label + break + + label, bbox = label_info[0], label_info[1:] + + if cxcy: + bbox = cxcywh2xyxy(bbox) + + x = draw_bbox(x, bbox, str(label) + '-' + get_cname(label), label_i == 0) + # print(x.shape) + xs += [x] + + xs = [ToTensor()(x) for x in xs] + grid = make_grid(xs, normalize=True, nrow=2) + plt.axis('off') + img = grid.permute(1, 2, 0).numpy() + plt.imshow(img) + plt.savefig(res_save_p, dpi=300) + plt.clf() diff --git a/data/datasets/object_detection/yolox_data_util/data_augment.py b/data/datasets/object_detection/yolox_data_util/data_augment.py new file mode 100644 index 0000000000000000000000000000000000000000..4acd55e82a92174951e1ad0b6cc869b1f3bd2284 --- /dev/null +++ b/data/datasets/object_detection/yolox_data_util/data_augment.py @@ -0,0 +1,269 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +# Copyright (c) Megvii, Inc. and its affiliates. +""" +Data augmentation functionality. Passed as callable transformations to +Dataset classes. + +The data augmentation procedures were interpreted from @weiliu89's SSD paper +http://arxiv.org/abs/1512.02325 +""" + +import math +import random + +import cv2 +import numpy as np + +# from models.yolox.yolox.utils import xyxy2cxcywh + + +def xyxy2cxcywh(bboxes): + bboxes[:, 2] = bboxes[:, 2] - bboxes[:, 0] + bboxes[:, 3] = bboxes[:, 3] - bboxes[:, 1] + bboxes[:, 0] = bboxes[:, 0] + bboxes[:, 2] * 0.5 + bboxes[:, 1] = bboxes[:, 1] + bboxes[:, 3] * 0.5 + return bboxes + + + +def augment_hsv(img, hgain=5, sgain=30, vgain=30): + hsv_augs = np.random.uniform(-1, 1, 3) * [hgain, sgain, vgain] # random gains + hsv_augs *= np.random.randint(0, 2, 3) # random selection of h, s, v + hsv_augs = hsv_augs.astype(np.int16) + img_hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV).astype(np.int16) + + img_hsv[..., 0] = (img_hsv[..., 0] + hsv_augs[0]) % 180 + img_hsv[..., 1] = np.clip(img_hsv[..., 1] + hsv_augs[1], 0, 255) + img_hsv[..., 2] = np.clip(img_hsv[..., 2] + hsv_augs[2], 0, 255) + + cv2.cvtColor(img_hsv.astype(img.dtype), cv2.COLOR_HSV2BGR, dst=img) # no return needed + + +def get_aug_params(value, center=0): + if isinstance(value, float): + return random.uniform(center - value, center + value) + elif len(value) == 2: + return random.uniform(value[0], value[1]) + else: + raise ValueError( + "Affine params should be either a sequence containing two values\ + or single float values. Got {}".format(value) + ) + + +def get_affine_matrix( + target_size, + degrees=10, + translate=0.1, + scales=0.1, + shear=10, +): + twidth, theight = target_size + + # Rotation and Scale + angle = get_aug_params(degrees) + scale = get_aug_params(scales, center=1.0) + + if scale <= 0.0: + raise ValueError("Argument scale should be positive") + + R = cv2.getRotationMatrix2D(angle=angle, center=(0, 0), scale=scale) + + M = np.ones([2, 3]) + # Shear + shear_x = math.tan(get_aug_params(shear) * math.pi / 180) + shear_y = math.tan(get_aug_params(shear) * math.pi / 180) + + M[0] = R[0] + shear_y * R[1] + M[1] = R[1] + shear_x * R[0] + + # Translation + translation_x = get_aug_params(translate) * twidth # x translation (pixels) + translation_y = get_aug_params(translate) * theight # y translation (pixels) + + M[0, 2] = translation_x + M[1, 2] = translation_y + + return M, scale + + +def apply_affine_to_bboxes(targets, target_size, M, scale): + num_gts = len(targets) + + # warp corner points + twidth, theight = target_size + corner_points = np.ones((4 * num_gts, 3)) + corner_points[:, :2] = targets[:, [0, 1, 2, 3, 0, 3, 2, 1]].reshape( + 4 * num_gts, 2 + ) # x1y1, x2y2, x1y2, x2y1 + corner_points = corner_points @ M.T # apply affine transform + corner_points = corner_points.reshape(num_gts, 8) + + # create new boxes + corner_xs = corner_points[:, 0::2] + corner_ys = corner_points[:, 1::2] + new_bboxes = ( + np.concatenate( + (corner_xs.min(1), corner_ys.min(1), corner_xs.max(1), corner_ys.max(1)) + ) + .reshape(4, num_gts) + .T + ) + + # clip boxes + new_bboxes[:, 0::2] = new_bboxes[:, 0::2].clip(0, twidth) + new_bboxes[:, 1::2] = new_bboxes[:, 1::2].clip(0, theight) + + targets[:, :4] = new_bboxes + + return targets + + +def random_affine( + img, + targets=(), + target_size=(640, 640), + degrees=10, + translate=0.1, + scales=0.1, + shear=10, +): + M, scale = get_affine_matrix(target_size, degrees, translate, scales, shear) + + img = cv2.warpAffine(img, M, dsize=target_size, borderValue=(114, 114, 114)) + + # Transform label coordinates + if len(targets) > 0: + targets = apply_affine_to_bboxes(targets, target_size, M, scale) + + return img, targets + + +def _mirror(image, boxes, prob=0.5): + _, width, _ = image.shape + if random.random() < prob: + image = image[:, ::-1] + boxes[:, 0::2] = width - boxes[:, 2::-2] + return image, boxes + + +def preproc(img, input_size, swap=(2, 0, 1)): + if len(img.shape) == 3: + padded_img = np.ones((input_size[0], input_size[1], 3), dtype=np.uint8) * 114 + else: + padded_img = np.ones(input_size, dtype=np.uint8) * 114 + + r = min(input_size[0] / img.shape[0], input_size[1] / img.shape[1]) + resized_img = cv2.resize( + img, + (int(img.shape[1] * r), int(img.shape[0] * r)), + interpolation=cv2.INTER_LINEAR, + ).astype(np.uint8) + padded_img[: int(img.shape[0] * r), : int(img.shape[1] * r)] = resized_img + + padded_img = padded_img.transpose(swap) + padded_img = np.ascontiguousarray(padded_img, dtype=np.float32) + return padded_img, r + + +class TrainTransform: + def __init__(self, max_labels=50, flip_prob=0.5, hsv_prob=1.0): + self.max_labels = max_labels + self.flip_prob = flip_prob + self.hsv_prob = hsv_prob + + def __call__(self, image, targets, input_dim): + boxes = targets[:, :4].copy() + labels = targets[:, 4].copy() + if len(boxes) == 0: + targets = np.zeros((self.max_labels, 5), dtype=np.float32) + image, r_o = preproc(image, input_dim) + return image, targets + + image_o = image.copy() + targets_o = targets.copy() + height_o, width_o, _ = image_o.shape + boxes_o = targets_o[:, :4] + labels_o = targets_o[:, 4] + # bbox_o: [xyxy] to [c_x,c_y,w,h] + boxes_o = xyxy2cxcywh(boxes_o) + + if random.random() < self.hsv_prob: + augment_hsv(image) + image_t, boxes = _mirror(image, boxes, self.flip_prob) + height, width, _ = image_t.shape + image_t, r_ = preproc(image_t, input_dim) + # boxes [xyxy] 2 [cx,cy,w,h] + boxes = xyxy2cxcywh(boxes) + boxes *= r_ + + mask_b = np.minimum(boxes[:, 2], boxes[:, 3]) > 1 + boxes_t = boxes[mask_b] + labels_t = labels[mask_b] + + if len(boxes_t) == 0: + image_t, r_o = preproc(image_o, input_dim) + boxes_o *= r_o + boxes_t = boxes_o + labels_t = labels_o + + labels_t = np.expand_dims(labels_t, 1) + + targets_t = np.hstack((labels_t, boxes_t)) + padded_labels = np.zeros((self.max_labels, 5)) + padded_labels[range(len(targets_t))[: self.max_labels]] = targets_t[ + : self.max_labels + ] + padded_labels = np.ascontiguousarray(padded_labels, dtype=np.float32) + return image_t, padded_labels + + +class ValTransform: + """ + Defines the transformations that should be applied to test PIL image + for input into the network + + dimension -> tensorize -> color adj + + Arguments: + resize (int): input dimension to SSD + rgb_means ((int,int,int)): average RGB of the dataset + (104,117,123) + swap ((int,int,int)): final order of channels + + Returns: + transform (transform) : callable transform to be applied to test/val + data + """ + + def __init__(self, swap=(2, 0, 1), legacy=False): + self.swap = swap + self.legacy = legacy + + # assume input is cv2 img for now + def __call__(self, img, res, input_size): + max_labels = 80 + + targets = res + boxes = targets[:, :4].copy() + labels = targets[:, 4].copy() + labels = np.expand_dims(labels, 1) + targets = np.hstack((labels, boxes)) + padded_labels = np.zeros((max_labels, 5)) + padded_labels[range(len(targets))[: max_labels]] = targets[ + : max_labels + ] + padded_labels = np.ascontiguousarray(padded_labels, dtype=np.float32) + + img, _ = preproc(img, input_size, self.swap) + if self.legacy: + img = img[::-1, :, :].copy() + img /= 255.0 + img -= np.array([0.485, 0.456, 0.406]).reshape(3, 1, 1) + img /= np.array([0.229, 0.224, 0.225]).reshape(3, 1, 1) + + # return img, np.zeros((1, 5)) + + # return real labels for visualization + return img, padded_labels diff --git a/data/datasets/object_detection/yolox_data_util/data_prefetcher.py b/data/datasets/object_detection/yolox_data_util/data_prefetcher.py new file mode 100644 index 0000000000000000000000000000000000000000..a118cf4e4ef968c9cf89a72457ede8c63bdf2cce --- /dev/null +++ b/data/datasets/object_detection/yolox_data_util/data_prefetcher.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +# Copyright (c) Megvii, Inc. and its affiliates. + +import torch + + +class DataPrefetcher: + """ + DataPrefetcher is inspired by code of following file: + https://github.com/NVIDIA/apex/blob/master/examples/imagenet/main_amp.py + It could speedup your pytorch dataloader. For more information, please check + https://github.com/NVIDIA/apex/issues/304#issuecomment-493562789. + """ + + def __init__(self, loader): + self.loader = iter(loader) + self.stream = torch.cuda.Stream() + self.input_cuda = self._input_cuda_for_image + self.record_stream = DataPrefetcher._record_stream_for_image + self.preload() + + def preload(self): + try: + self.next_input, self.next_target, _, _ = next(self.loader) + except StopIteration: + self.next_input = None + self.next_target = None + return + + with torch.cuda.stream(self.stream): + self.input_cuda() + self.next_target = self.next_target.cuda(non_blocking=True) + + def next(self): + torch.cuda.current_stream().wait_stream(self.stream) + input = self.next_input + target = self.next_target + if input is not None: + self.record_stream(input) + if target is not None: + target.record_stream(torch.cuda.current_stream()) + self.preload() + return input, target + + def _input_cuda_for_image(self): + self.next_input = self.next_input.cuda(non_blocking=True) + + @staticmethod + def _record_stream_for_image(input): + input.record_stream(torch.cuda.current_stream()) diff --git a/data/datasets/object_detection/yolox_data_util/dataloading.py b/data/datasets/object_detection/yolox_data_util/dataloading.py new file mode 100644 index 0000000000000000000000000000000000000000..6fecf3f06abe908ea5f0d84fba85d2e230257512 --- /dev/null +++ b/data/datasets/object_detection/yolox_data_util/dataloading.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +# Copyright (c) Megvii, Inc. and its affiliates. + +import os +import random +import uuid + +import numpy as np + +import torch +from torch.utils.data.dataloader import DataLoader as torchDataLoader +from torch.utils.data.dataloader import default_collate + +from .samplers import YoloBatchSampler + + +def get_yolox_datadir(): + """ + get dataset dir of YOLOX. If environment variable named `YOLOX_DATADIR` is set, + this function will return value of the environment variable. Otherwise, use data + """ + yolox_datadir = os.getenv("YOLOX_DATADIR", None) + if yolox_datadir is None: + import yolox + + yolox_path = os.path.dirname(os.path.dirname(yolox.__file__)) + yolox_datadir = os.path.join(yolox_path, "datasets") + return yolox_datadir + + +class DataLoader(torchDataLoader): + """ + Lightnet dataloader that enables on the fly resizing of the images. + See :class:`torch.utils.data.DataLoader` for more information on the arguments. + Check more on the following website: + https://gitlab.com/EAVISE/lightnet/-/blob/master/lightnet/data/_dataloading.py + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.__initialized = False + shuffle = False + batch_sampler = None + if len(args) > 5: + shuffle = args[2] + sampler = args[3] + batch_sampler = args[4] + elif len(args) > 4: + shuffle = args[2] + sampler = args[3] + if "batch_sampler" in kwargs: + batch_sampler = kwargs["batch_sampler"] + elif len(args) > 3: + shuffle = args[2] + if "sampler" in kwargs: + sampler = kwargs["sampler"] + if "batch_sampler" in kwargs: + batch_sampler = kwargs["batch_sampler"] + else: + if "shuffle" in kwargs: + shuffle = kwargs["shuffle"] + if "sampler" in kwargs: + sampler = kwargs["sampler"] + if "batch_sampler" in kwargs: + batch_sampler = kwargs["batch_sampler"] + + # Use custom BatchSampler + if batch_sampler is None: + if sampler is None: + if shuffle: + sampler = torch.utils.data.sampler.RandomSampler(self.dataset) + # sampler = torch.utils.data.DistributedSampler(self.dataset) + else: + sampler = torch.utils.data.sampler.SequentialSampler(self.dataset) + batch_sampler = YoloBatchSampler( + sampler, + self.batch_size, + self.drop_last, + input_dimension=self.dataset.input_dim, + ) + # batch_sampler = IterationBasedBatchSampler(batch_sampler, num_iterations = + + self.batch_sampler = batch_sampler + + self.__initialized = True + + def close_mosaic(self): + self.batch_sampler.mosaic = False + + +def list_collate(batch): + """ + Function that collates lists or tuples together into one list (of lists/tuples). + Use this as the collate function in a Dataloader, if you want to have a list of + items as an output, as opposed to tensors (eg. Brambox.boxes). + """ + items = list(zip(*batch)) + + for i in range(len(items)): + if isinstance(items[i][0], (list, tuple)): + items[i] = list(items[i]) + else: + items[i] = default_collate(items[i]) + + return items + + +def worker_init_reset_seed(worker_id): + seed = uuid.uuid4().int % 2**32 + random.seed(seed) + torch.set_rng_state(torch.manual_seed(seed).get_state()) + np.random.seed(seed) diff --git a/data/datasets/object_detection/yolox_data_util/datasets/__init__.py b/data/datasets/object_detection/yolox_data_util/datasets/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..58b39a83bf4809fdd9c0e3f66cb2c37f70955502 --- /dev/null +++ b/data/datasets/object_detection/yolox_data_util/datasets/__init__.py @@ -0,0 +1,9 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +# Copyright (c) Megvii, Inc. and its affiliates. + +from .coco import COCODataset +from .coco_classes import COCO_CLASSES +from .datasets_wrapper import ConcatDataset, Dataset, MixConcatDataset +from .mosaicdetection import MosaicDetection +# from .voc import VOCDetection diff --git a/data/datasets/object_detection/yolox_data_util/datasets/__pycache__/__init__.cpython-38.pyc b/data/datasets/object_detection/yolox_data_util/datasets/__pycache__/__init__.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..153562ff9f278a2483caac2de943c31951d3e1c3 Binary files /dev/null and b/data/datasets/object_detection/yolox_data_util/datasets/__pycache__/__init__.cpython-38.pyc differ diff --git a/data/datasets/object_detection/yolox_data_util/datasets/__pycache__/coco.cpython-38.pyc b/data/datasets/object_detection/yolox_data_util/datasets/__pycache__/coco.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..fe63428a67705e534fb23ef87efef4873d687221 Binary files /dev/null and b/data/datasets/object_detection/yolox_data_util/datasets/__pycache__/coco.cpython-38.pyc differ diff --git a/data/datasets/object_detection/yolox_data_util/datasets/__pycache__/coco_classes.cpython-38.pyc b/data/datasets/object_detection/yolox_data_util/datasets/__pycache__/coco_classes.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b509d2eda20d0203f8f1299d31d398ba04df5bfe Binary files /dev/null and b/data/datasets/object_detection/yolox_data_util/datasets/__pycache__/coco_classes.cpython-38.pyc differ diff --git a/data/datasets/object_detection/yolox_data_util/datasets/__pycache__/datasets_wrapper.cpython-38.pyc b/data/datasets/object_detection/yolox_data_util/datasets/__pycache__/datasets_wrapper.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..368bedb2b3215374cbdf68e57353baaa910d4d2f Binary files /dev/null and b/data/datasets/object_detection/yolox_data_util/datasets/__pycache__/datasets_wrapper.cpython-38.pyc differ diff --git a/data/datasets/object_detection/yolox_data_util/datasets/__pycache__/mm_coco.cpython-38.pyc b/data/datasets/object_detection/yolox_data_util/datasets/__pycache__/mm_coco.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..dbc96e776e35d57633c8a13ce598e18526c9d8b9 Binary files /dev/null and b/data/datasets/object_detection/yolox_data_util/datasets/__pycache__/mm_coco.cpython-38.pyc differ diff --git a/data/datasets/object_detection/yolox_data_util/datasets/__pycache__/mosaicdetection.cpython-38.pyc b/data/datasets/object_detection/yolox_data_util/datasets/__pycache__/mosaicdetection.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7583346ce943fde00f67579269c2206062312429 Binary files /dev/null and b/data/datasets/object_detection/yolox_data_util/datasets/__pycache__/mosaicdetection.cpython-38.pyc differ diff --git a/data/datasets/object_detection/yolox_data_util/datasets/coco.py b/data/datasets/object_detection/yolox_data_util/datasets/coco.py new file mode 100644 index 0000000000000000000000000000000000000000..31b912d4ba7254845ea47790f6fc46ed8deab760 --- /dev/null +++ b/data/datasets/object_detection/yolox_data_util/datasets/coco.py @@ -0,0 +1,256 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +# Copyright (c) Megvii, Inc. and its affiliates. + +import os +# from loguru import logger +from logging import Logger +logger = Logger('yolox') +import cv2 +import numpy as np +from pycocotools.coco import COCO + +from ..dataloading import get_yolox_datadir +from .datasets_wrapper import Dataset + + +def remove_useless_info(coco): + """ + Remove useless info in coco dataset. COCO object is modified inplace. + This function is mainly used for saving memory (save about 30% mem). + """ + if isinstance(coco, COCO): + dataset = coco.dataset + dataset.pop("info", None) + dataset.pop("licenses", None) + for img in dataset["images"]: + img.pop("license", None) + img.pop("coco_url", None) + img.pop("date_captured", None) + img.pop("flickr_url", None) + if "annotations" in coco.dataset: + for anno in coco.dataset["annotations"]: + anno.pop("segmentation", None) + +from maskrcnn_benchmark.structures.bounding_box import BoxList +from PIL import Image, ImageDraw +class TMPDataset(Dataset): + def __init__(self): + ori_image = Image.open('new_impl/cv/glip/object_detection/000000103759.jpg').convert("RGB") + image = transform(np.asarray(ori_image)[:, :, [2, 1, 0]]) + text = 'orange. umbrella. ' + targets = BoxList(torch.FloatTensor([[0., 0., 0., 0.]]), image_size=image.size()[1:], mode='xyxy') + targets.add_field('caption', text) + targets.add_field('tokens_positive', run_ner(text)) + targets.add_field('labels', torch.LongTensor([0])) + samples = [{'images' : [image], 'targets' : [targets]}] + + def __getitem__(self, index): + return samples['images'], samples['targets'] + + def __len__(self): + return 1 + +class COCODataset(Dataset): + """ + COCO dataset class. + """ + + def __init__( + self, + data_dir=None, + json_file="instances_train2017.json", + name="train2017", + img_size=(416, 416), + preproc=None, + cache=False, + ): + """ + COCO dataset initialization. Annotation data are read into memory by COCO API. + Args: + data_dir (str): dataset root directory + json_file (str): COCO json file name + name (str): COCO data name (e.g. 'train2017' or 'val2017') + img_size (int): target image size after pre-processing + preproc: data augmentation strategy + """ + super().__init__(img_size) + if data_dir is None: + data_dir = os.path.join(get_yolox_datadir(), "COCO") + self.data_dir = data_dir + self.json_file = json_file + self.coco = COCO(os.path.join(self.data_dir, "annotations", self.json_file)) + remove_useless_info(self.coco) + self.ids = self.coco.getImgIds() + self.class_ids = sorted(self.coco.getCatIds()) + cats = self.coco.loadCats(self.coco.getCatIds()) + self._classes = tuple([c["name"] for c in cats]) + self.imgs = None + self.name = name + self.img_size = img_size + self.preproc = preproc + self.annotations = self._load_coco_annotations() + if cache: + self._cache_images() + + def __len__(self): + return len(self.ids) + + def __del__(self): + del self.imgs + + def _load_coco_annotations(self): + return [self.load_anno_from_ids(_ids) for _ids in self.ids] + + def _cache_images(self): + logger.warning( + "\n********************************************************************************\n" + "You are using cached images in RAM to accelerate training.\n" + "This requires large system RAM.\n" + "Make sure you have 200G+ RAM and 136G available disk space for training COCO.\n" + "********************************************************************************\n" + ) + max_h = self.img_size[0] + max_w = self.img_size[1] + cache_file = os.path.join(self.data_dir, f"img_resized_cache_{self.name}.array") + if not os.path.exists(cache_file): + logger.info( + "Caching images for the first time. This might take about 20 minutes for COCO" + ) + self.imgs = np.memmap( + cache_file, + shape=(len(self.ids), max_h, max_w, 3), + dtype=np.uint8, + mode="w+", + ) + from tqdm import tqdm + from multiprocessing.pool import ThreadPool + + NUM_THREADs = min(8, os.cpu_count()) + loaded_images = ThreadPool(NUM_THREADs).imap( + lambda x: self.load_resized_img(x), + range(len(self.annotations)), + ) + pbar = tqdm(enumerate(loaded_images), total=len(self.annotations)) + for k, out in pbar: + self.imgs[k][: out.shape[0], : out.shape[1], :] = out.copy() + self.imgs.flush() + pbar.close() + else: + logger.warning( + "You are using cached imgs! Make sure your dataset is not changed!!\n" + "Everytime the self.input_size is changed in your exp file, you need to delete\n" + "the cached data and re-generate them.\n" + ) + + logger.info("Loading cached imgs...") + self.imgs = np.memmap( + cache_file, + shape=(len(self.ids), max_h, max_w, 3), + dtype=np.uint8, + mode="r+", + ) + + def load_anno_from_ids(self, id_): + im_ann = self.coco.loadImgs(id_)[0] + width = im_ann["width"] + height = im_ann["height"] + anno_ids = self.coco.getAnnIds(imgIds=[int(id_)], iscrowd=False) + annotations = self.coco.loadAnns(anno_ids) + objs = [] + for obj in annotations: + x1 = np.max((0, obj["bbox"][0])) + y1 = np.max((0, obj["bbox"][1])) + x2 = np.min((width, x1 + np.max((0, obj["bbox"][2])))) + y2 = np.min((height, y1 + np.max((0, obj["bbox"][3])))) + if obj["area"] > 0 and x2 >= x1 and y2 >= y1: + obj["clean_bbox"] = [x1, y1, x2, y2] + objs.append(obj) + + num_objs = len(objs) + + res = np.zeros((num_objs, 5)) +# {'file_name': '000000103759.jpg', 'height': 480, 'width': 640, 'id': 103759} + for ix, obj in enumerate(objs): + cls = self.class_ids.index(obj["category_id"]) + res[ix, 0:4] = obj["clean_bbox"] + res[ix, 4] = cls + + r = min(self.img_size[0] / height, self.img_size[1] / width) + res[:, :4] *= r + + img_info = (height, width) + resized_info = (int(height * r), int(width * r)) + + file_name = ( + im_ann["file_name"] + if "file_name" in im_ann + else "{:012}".format(id_) + ".jpg" + ) + + return (res, img_info, resized_info, file_name) + + def load_anno(self, index): + return self.annotations[index][0] + + def load_resized_img(self, index): + img = self.load_image(index) + r = min(self.img_size[0] / img.shape[0], self.img_size[1] / img.shape[1]) + resized_img = cv2.resize( + img, + (int(img.shape[1] * r), int(img.shape[0] * r)), + interpolation=cv2.INTER_LINEAR, + ).astype(np.uint8) + return resized_img + + def load_image(self, index): + file_name = self.annotations[index][3] + + img_file = os.path.join(self.data_dir, self.name, file_name) + + img = cv2.imread(img_file) + assert img is not None, img_file + + return img + + def pull_item(self, index): + id_ = self.ids[index] + + res, img_info, resized_info, _ = self.annotations[index] + + + if self.imgs is not None: + pad_img = self.imgs[index] + img = pad_img[: resized_info[0], : resized_info[1], :].copy() + else: + img = self.load_resized_img(index) + + return img, res.copy(), img_info, np.array([id_]) + + @Dataset.mosaic_getitem + def __getitem__(self, index): + """ + One image / label pair for the given index is picked up and pre-processed. + + Args: + index (int): data index + + Returns: + img (numpy.ndarray): pre-processed image + padded_labels (torch.Tensor): pre-processed label data. + The shape is :math:`[max_labels, 5]`. + each label consists of [class, xc, yc, w, h]: + class (float): class index. + xc, yc (float) : center of bbox whose values range from 0 to 1. + w, h (float) : size of bbox whose values range from 0 to 1. + info_img : tuple of h, w. + h, w (int): original shape of the image + img_id (int): same as the input index. Used for evaluation. + """ + img, target, img_info, img_id = self.pull_item(index) + + if self.preproc is not None: + img, target = self.preproc(img, target, self.input_dim) + + return img, target, img_info, img_id + diff --git a/data/datasets/object_detection/yolox_data_util/datasets/coco_classes.py b/data/datasets/object_detection/yolox_data_util/datasets/coco_classes.py new file mode 100644 index 0000000000000000000000000000000000000000..17f5cbe6e86ed4fc8378760da71f8349a6406ff1 --- /dev/null +++ b/data/datasets/object_detection/yolox_data_util/datasets/coco_classes.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +# Copyright (c) Megvii, Inc. and its affiliates. + +COCO_CLASSES = ( + "person", + "bicycle", + "car", + "motorcycle", + "airplane", + "bus", + "train", + "truck", + "boat", + "traffic light", + "fire hydrant", + "stop sign", + "parking meter", + "bench", + "bird", + "cat", + "dog", + "horse", + "sheep", + "cow", + "elephant", + "bear", + "zebra", + "giraffe", + "backpack", + "umbrella", + "handbag", + "tie", + "suitcase", + "frisbee", + "skis", + "snowboard", + "sports ball", + "kite", + "baseball bat", + "baseball glove", + "skateboard", + "surfboard", + "tennis racket", + "bottle", + "wine glass", + "cup", + "fork", + "knife", + "spoon", + "bowl", + "banana", + "apple", + "sandwich", + "orange", + "broccoli", + "carrot", + "hot dog", + "pizza", + "donut", + "cake", + "chair", + "couch", + "potted plant", + "bed", + "dining table", + "toilet", + "tv", + "laptop", + "mouse", + "remote", + "keyboard", + "cell phone", + "microwave", + "oven", + "toaster", + "sink", + "refrigerator", + "book", + "clock", + "vase", + "scissors", + "teddy bear", + "hair drier", + "toothbrush", +) diff --git a/data/datasets/object_detection/yolox_data_util/datasets/datasets_wrapper.py b/data/datasets/object_detection/yolox_data_util/datasets/datasets_wrapper.py new file mode 100644 index 0000000000000000000000000000000000000000..f85a121d5a655560568f839019ffd4ac7309e287 --- /dev/null +++ b/data/datasets/object_detection/yolox_data_util/datasets/datasets_wrapper.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +# Copyright (c) Megvii, Inc. and its affiliates. + +import bisect +from functools import wraps + +from torch.utils.data.dataset import ConcatDataset as torchConcatDataset +from torch.utils.data.dataset import Dataset as torchDataset + + +class ConcatDataset(torchConcatDataset): + def __init__(self, datasets): + super(ConcatDataset, self).__init__(datasets) + if hasattr(self.datasets[0], "input_dim"): + self._input_dim = self.datasets[0].input_dim + self.input_dim = self.datasets[0].input_dim + + def pull_item(self, idx): + if idx < 0: + if -idx > len(self): + raise ValueError( + "absolute value of index should not exceed dataset length" + ) + idx = len(self) + idx + dataset_idx = bisect.bisect_right(self.cumulative_sizes, idx) + if dataset_idx == 0: + sample_idx = idx + else: + sample_idx = idx - self.cumulative_sizes[dataset_idx - 1] + return self.datasets[dataset_idx].pull_item(sample_idx) + + +class MixConcatDataset(torchConcatDataset): + def __init__(self, datasets): + super(MixConcatDataset, self).__init__(datasets) + if hasattr(self.datasets[0], "input_dim"): + self._input_dim = self.datasets[0].input_dim + self.input_dim = self.datasets[0].input_dim + + def __getitem__(self, index): + + if not isinstance(index, int): + idx = index[1] + if idx < 0: + if -idx > len(self): + raise ValueError( + "absolute value of index should not exceed dataset length" + ) + idx = len(self) + idx + dataset_idx = bisect.bisect_right(self.cumulative_sizes, idx) + if dataset_idx == 0: + sample_idx = idx + else: + sample_idx = idx - self.cumulative_sizes[dataset_idx - 1] + if not isinstance(index, int): + index = (index[0], sample_idx, index[2]) + + return self.datasets[dataset_idx][index] + + +class Dataset(torchDataset): + """ This class is a subclass of the base :class:`torch.utils.data.Dataset`, + that enables on the fly resizing of the ``input_dim``. + + Args: + input_dimension (tuple): (width,height) tuple with default dimensions of the network + """ + + def __init__(self, input_dimension, mosaic=True): + super().__init__() + self.__input_dim = input_dimension[:2] + self.enable_mosaic = mosaic + + @property + def input_dim(self): + """ + Dimension that can be used by transforms to set the correct image size, etc. + This allows transforms to have a single source of truth + for the input dimension of the network. + + Return: + list: Tuple containing the current width,height + """ + if hasattr(self, "_input_dim"): + return self._input_dim + return self.__input_dim + + @staticmethod + def mosaic_getitem(getitem_fn): + """ + Decorator method that needs to be used around the ``__getitem__`` method. |br| + This decorator enables the closing mosaic + + Example: + >>> class CustomSet(ln.data.Dataset): + ... def __len__(self): + ... return 10 + ... @ln.data.Dataset.mosaic_getitem + ... def __getitem__(self, index): + ... return self.enable_mosaic + """ + + @wraps(getitem_fn) + def wrapper(self, index): + if not isinstance(index, int): + self.enable_mosaic = index[0] + index = index[1] + + ret_val = getitem_fn(self, index) + + return ret_val + + return wrapper diff --git a/data/datasets/object_detection/yolox_data_util/datasets/mm_coco.py b/data/datasets/object_detection/yolox_data_util/datasets/mm_coco.py new file mode 100644 index 0000000000000000000000000000000000000000..ddea420db02e617ac4c30da13ffb23604702b2f6 --- /dev/null +++ b/data/datasets/object_detection/yolox_data_util/datasets/mm_coco.py @@ -0,0 +1,89 @@ +import torch +from .datasets_wrapper import Dataset +from .coco_classes import COCO_CLASSES +from maskrcnn_benchmark.structures.bounding_box import BoxList + +def cxcywh2xyxy(bboxes): + #bboxes:(cx,cy,w,h) + bboxes[:, 0] = bboxes[:, 0] - bboxes[:, 2] * 0.5 + bboxes[:, 1] = bboxes[:, 1] - bboxes[:, 3] * 0.5 + bboxes[:, 2] = bboxes[:, 2] + bboxes[:, 0] + bboxes[:, 3] = bboxes[:, 3] + bboxes[:, 1] + return bboxes + +class MM_COCODataset(Dataset): + def __init__(self, dataset, transform, split, coco=None, classes=None): + self.cocods = dataset + self.coco = coco + self.transform = transform + self.split = split + catIds = self.coco.getCatIds() + cats = self.coco.loadCats(catIds) + if classes is None: + self.cls_names = [cat['name'] for cat in cats] + else: + self.cls_names = classes + + def __getitem__(self, index): + if self.split == 'train': + img, target = self.cocods[index] + else: + img, target, img_size, idx = self.cocods[index] + + img = self.transform(torch.Tensor(img)) + + # self.json_category_id_to_contiguous_id = { + # v: i + 1 for i, v in enumerate(self.coco.getCatIds()) + # } + # self.contiguous_category_id_to_json_id = { + # v: k for k, v in self.json_category_id_to_contiguous_id.items() + # } + + num_bbox = 0 + for item in target: + if item[1] == 0 and item[2] == 0 and item[3] == 0 and item[4] == 0: + break + num_bbox += 1 + + # [0:'zebra', + # 1:'backpack', + # 2:'umbrella', + # 3:'kite', + # 4:'cup', + # 5:'banana', + # 6:'orange', + # 7:'broccoli', + # 8:'pizza', + # 9:'laptop', + # 10:'mouse', + # 11:'microwave', + # 12:'toaster', + # 13:'refrigerator', + # 14:'vase'] + if self.split == 'train': + new_target = BoxList(cxcywh2xyxy(target[:num_bbox, 1:]), self.cocods.input_dim, mode='xyxy') + else: + new_target = BoxList(target[:num_bbox, 1:], self.cocods.img_size, mode='xyxy') + + labels = target[:num_bbox, 0] + my_labels = [] + + caption_string = ". ".join(self.cls_names) + ". " + tokens_positive = [] + + for label in labels: + word = self.cls_names[int(label)] + st = caption_string.find(word) + my_labels.append(label + 1) + tokens_positive.append([[st, st + len(word)]]) + + new_target.add_field('labels', torch.LongTensor(my_labels)) + new_target.add_field('caption', caption_string) + new_target.add_field('tokens_positive', tokens_positive) + if self.split == 'train': + return img, new_target + else: + return img, new_target, img_size, idx + + def __len__(self): + return self.cocods.__len__() \ No newline at end of file diff --git a/data/datasets/object_detection/yolox_data_util/datasets/mosaicdetection.py b/data/datasets/object_detection/yolox_data_util/datasets/mosaicdetection.py new file mode 100644 index 0000000000000000000000000000000000000000..8b2ac86189f314c2dbde092885e2ff5b46151896 --- /dev/null +++ b/data/datasets/object_detection/yolox_data_util/datasets/mosaicdetection.py @@ -0,0 +1,287 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +# Copyright (c) Megvii, Inc. and its affiliates. + +import random + +import cv2 +import numpy as np + +# from models.yolox.yolox.utils import adjust_box_anns, get_local_rank + +from ..data_augment import random_affine +from .datasets_wrapper import Dataset + + +import functools +import os +import pickle +import time +from contextlib import contextmanager +# from loguru import logger + +import numpy as np + +import torch +from torch import distributed as dist + +def adjust_box_anns(bbox, scale_ratio, padw, padh, w_max, h_max): + bbox[:, 0::2] = np.clip(bbox[:, 0::2] * scale_ratio + padw, 0, w_max) + bbox[:, 1::2] = np.clip(bbox[:, 1::2] * scale_ratio + padh, 0, h_max) + return bbox + +_LOCAL_PROCESS_GROUP = None + +def get_rank() -> int: + if not dist.is_available(): + return 0 + if not dist.is_initialized(): + return 0 + return dist.get_rank() + +def get_local_rank() -> int: + """ + Returns: + The rank of the current process within the local (per-machine) process group. + """ + if _LOCAL_PROCESS_GROUP is None: + return get_rank() + + if not dist.is_available(): + return 0 + if not dist.is_initialized(): + return 0 + return dist.get_rank(group=_LOCAL_PROCESS_GROUP) + + +def get_mosaic_coordinate(mosaic_image, mosaic_index, xc, yc, w, h, input_h, input_w): + # TODO update doc + # index0 to top left part of image + if mosaic_index == 0: + x1, y1, x2, y2 = max(xc - w, 0), max(yc - h, 0), xc, yc + small_coord = w - (x2 - x1), h - (y2 - y1), w, h + # index1 to top right part of image + elif mosaic_index == 1: + x1, y1, x2, y2 = xc, max(yc - h, 0), min(xc + w, input_w * 2), yc + small_coord = 0, h - (y2 - y1), min(w, x2 - x1), h + # index2 to bottom left part of image + elif mosaic_index == 2: + x1, y1, x2, y2 = max(xc - w, 0), yc, xc, min(input_h * 2, yc + h) + small_coord = w - (x2 - x1), 0, w, min(y2 - y1, h) + # index2 to bottom right part of image + elif mosaic_index == 3: + x1, y1, x2, y2 = xc, yc, min(xc + w, input_w * 2), min(input_h * 2, yc + h) # noqa + small_coord = 0, 0, min(w, x2 - x1), min(y2 - y1, h) + return (x1, y1, x2, y2), small_coord + + +class MosaicDetection(Dataset): + """Detection dataset wrapper that performs mixup for normal dataset.""" + + def __init__( + self, dataset, img_size, mosaic=True, preproc=None, + degrees=10.0, translate=0.1, mosaic_scale=(0.5, 1.5), + mixup_scale=(0.5, 1.5), shear=2.0, enable_mixup=True, + mosaic_prob=1.0, mixup_prob=1.0, only_return_xy=False, preprocess=None, + use_caption=False, *args + ): + """ + + Args: + dataset(Dataset) : Pytorch dataset object. + img_size (tuple): + mosaic (bool): enable mosaic augmentation or not. + preproc (func): + degrees (float): + translate (float): + mosaic_scale (tuple): + mixup_scale (tuple): + shear (float): + enable_mixup (bool): + *args(tuple) : Additional arguments for mixup random sampler. + """ + super().__init__(img_size, mosaic=mosaic) + self._dataset = dataset + self.preproc = preproc + self.degrees = degrees + self.translate = translate + self.scale = mosaic_scale + self.shear = shear + self.mixup_scale = mixup_scale + self.enable_mosaic = mosaic + self.enable_mixup = enable_mixup + self.mosaic_prob = mosaic_prob + self.mixup_prob = mixup_prob + self.only_return_xy = only_return_xy + self.preprocess = preprocess + self.local_rank = get_local_rank() + + def __len__(self): + return len(self._dataset) + + @Dataset.mosaic_getitem + def __getitem__(self, idx): + if self.enable_mosaic and random.random() < self.mosaic_prob: + mosaic_labels = [] + input_dim = self._dataset.input_dim + input_h, input_w = input_dim[0], input_dim[1] + + # yc, xc = s, s # mosaic center x, y + yc = int(random.uniform(0.5 * input_h, 1.5 * input_h)) + xc = int(random.uniform(0.5 * input_w, 1.5 * input_w)) + + # 3 additional image indices + indices = [idx] + [random.randint(0, len(self._dataset) - 1) for _ in range(3)] + + for i_mosaic, index in enumerate(indices): + img, _labels, _, img_id = self._dataset.pull_item(index) + h0, w0 = img.shape[:2] # orig hw + scale = min(1. * input_h / h0, 1. * input_w / w0) + img = cv2.resize( + img, (int(w0 * scale), int(h0 * scale)), interpolation=cv2.INTER_LINEAR + ) + # generate output mosaic image + (h, w, c) = img.shape[:3] + if i_mosaic == 0: + mosaic_img = np.full((input_h * 2, input_w * 2, c), 114, dtype=np.uint8) + + # suffix l means large image, while s means small image in mosaic aug. + (l_x1, l_y1, l_x2, l_y2), (s_x1, s_y1, s_x2, s_y2) = get_mosaic_coordinate( + mosaic_img, i_mosaic, xc, yc, w, h, input_h, input_w + ) + + mosaic_img[l_y1:l_y2, l_x1:l_x2] = img[s_y1:s_y2, s_x1:s_x2] + padw, padh = l_x1 - s_x1, l_y1 - s_y1 + + labels = _labels.copy() + # Normalized xywh to pixel xyxy format + if _labels.size > 0: + labels[:, 0] = scale * _labels[:, 0] + padw + labels[:, 1] = scale * _labels[:, 1] + padh + labels[:, 2] = scale * _labels[:, 2] + padw + labels[:, 3] = scale * _labels[:, 3] + padh + mosaic_labels.append(labels) + + if len(mosaic_labels): + mosaic_labels = np.concatenate(mosaic_labels, 0) + np.clip(mosaic_labels[:, 0], 0, 2 * input_w, out=mosaic_labels[:, 0]) + np.clip(mosaic_labels[:, 1], 0, 2 * input_h, out=mosaic_labels[:, 1]) + np.clip(mosaic_labels[:, 2], 0, 2 * input_w, out=mosaic_labels[:, 2]) + np.clip(mosaic_labels[:, 3], 0, 2 * input_h, out=mosaic_labels[:, 3]) + + mosaic_img, mosaic_labels = random_affine( + mosaic_img, + mosaic_labels, + target_size=(input_w, input_h), + degrees=self.degrees, + translate=self.translate, + scales=self.scale, + shear=self.shear, + ) + + # ----------------------------------------------------------------- + # CopyPaste: https://arxiv.org/abs/2012.07177 + # ----------------------------------------------------------------- + if ( + self.enable_mixup + and not len(mosaic_labels) == 0 + and random.random() < self.mixup_prob + ): + mosaic_img, mosaic_labels = self.mixup(mosaic_img, mosaic_labels, self.input_dim) + mix_img, padded_labels = self.preproc(mosaic_img, mosaic_labels, self.input_dim) + img_info = (mix_img.shape[1], mix_img.shape[0]) + + # ----------------------------------------------------------------- + # img_info and img_id are not used for training. + # They are also hard to be specified on a mosaic image. + # ----------------------------------------------------------------- + if self.preprocess is not None: + mix_img, padded_labels = self.preprocess(mix_img, padded_labels, (320, 320)) + if self.only_return_xy: + return mix_img, padded_labels + + return mix_img, caption, padded_labels, img_info, img_id + + else: + self._dataset._input_dim = self.input_dim + img, label, img_info, img_id = self._dataset.pull_item(idx) + img, label = self.preproc(img, label, self.input_dim) + if self.preprocess is not None: + img, label = self.preprocess(img, label, (320, 320)) + if self.only_return_xy: + return img, label + return img, label, img_info, img_id + + def mixup(self, origin_img, origin_labels, input_dim): + jit_factor = random.uniform(*self.mixup_scale) + FLIP = random.uniform(0, 1) > 0.5 + cp_labels = [] + while len(cp_labels) == 0: + cp_index = random.randint(0, self.__len__() - 1) + cp_labels = self._dataset.load_anno(cp_index) + img, cp_labels, _, _ = self._dataset.pull_item(cp_index) + + if len(img.shape) == 3: + cp_img = np.ones((input_dim[0], input_dim[1], 3), dtype=np.uint8) * 114 + else: + cp_img = np.ones(input_dim, dtype=np.uint8) * 114 + + cp_scale_ratio = min(input_dim[0] / img.shape[0], input_dim[1] / img.shape[1]) + resized_img = cv2.resize( + img, + (int(img.shape[1] * cp_scale_ratio), int(img.shape[0] * cp_scale_ratio)), + interpolation=cv2.INTER_LINEAR, + ) + + cp_img[ + : int(img.shape[0] * cp_scale_ratio), : int(img.shape[1] * cp_scale_ratio) + ] = resized_img + + cp_img = cv2.resize( + cp_img, + (int(cp_img.shape[1] * jit_factor), int(cp_img.shape[0] * jit_factor)), + ) + cp_scale_ratio *= jit_factor + + if FLIP: + cp_img = cp_img[:, ::-1, :] + + origin_h, origin_w = cp_img.shape[:2] + target_h, target_w = origin_img.shape[:2] + padded_img = np.zeros( + (max(origin_h, target_h), max(origin_w, target_w), 3), dtype=np.uint8 + ) + padded_img[:origin_h, :origin_w] = cp_img + + x_offset, y_offset = 0, 0 + if padded_img.shape[0] > target_h: + y_offset = random.randint(0, padded_img.shape[0] - target_h - 1) + if padded_img.shape[1] > target_w: + x_offset = random.randint(0, padded_img.shape[1] - target_w - 1) + padded_cropped_img = padded_img[ + y_offset: y_offset + target_h, x_offset: x_offset + target_w + ] + + cp_bboxes_origin_np = adjust_box_anns( + cp_labels[:, :4].copy(), cp_scale_ratio, 0, 0, origin_w, origin_h + ) + if FLIP: + cp_bboxes_origin_np[:, 0::2] = ( + origin_w - cp_bboxes_origin_np[:, 0::2][:, ::-1] + ) + cp_bboxes_transformed_np = cp_bboxes_origin_np.copy() + cp_bboxes_transformed_np[:, 0::2] = np.clip( + cp_bboxes_transformed_np[:, 0::2] - x_offset, 0, target_w + ) + cp_bboxes_transformed_np[:, 1::2] = np.clip( + cp_bboxes_transformed_np[:, 1::2] - y_offset, 0, target_h + ) + + cls_labels = cp_labels[:, 4:5].copy() + box_labels = cp_bboxes_transformed_np + labels = np.hstack((box_labels, cls_labels)) + origin_labels = np.vstack((origin_labels, labels)) + origin_img = origin_img.astype(np.float32) + origin_img = 0.5 * origin_img + 0.5 * padded_cropped_img.astype(np.float32) + + return origin_img.astype(np.uint8), origin_labels diff --git a/data/datasets/object_detection/yolox_data_util/norm_categories_index.py b/data/datasets/object_detection/yolox_data_util/norm_categories_index.py new file mode 100644 index 0000000000000000000000000000000000000000..108bf89b5c0493051c73e1c8030d8020f1a482ca --- /dev/null +++ b/data/datasets/object_detection/yolox_data_util/norm_categories_index.py @@ -0,0 +1,51 @@ +from copy import deepcopy +import json +from tqdm import tqdm +import os + + +p1 = '/data/zql/datasets/coco2017/train2017/coco_ann.json' +p2 = '/data/zql/datasets/face_mask/WI/Medical mask/Medical mask/Medical Mask/images/coco_ann.json' +p3 = '/data/zql/datasets/face_mask/make_ml/images/coco_ann.json' +p4 = '/data/datasets/VOCdevkit/VOC2012/JPEGImages/coco_ann.json' + +# for p in [p1, p2, p3, p4]: +# if os.path.exists(p + '.normed'): +# os.remove(p + '.normed') + + +def ensure_index_start_from_1_and_successive(p): + return p + + if os.path.exists(p + '.normed'): + return p + '.normed' + + with open(p, 'r') as f: + data = json.load(f) + + need_norm = False + for i, c in enumerate(data['categories']): + if i + 1 != c['id']: + need_norm = True + break + if not need_norm: + return p + + categories_map = {} + + new_categories = [] + for i, c in tqdm(enumerate(data['categories']), desc='norm index in categories'): + new_categories += [deepcopy(c)] + new_categories[-1]['id'] = i + 1 + categories_map[c['id']] = i + 1 + data['categories'] = new_categories + + new_annotations = [] + for ann in tqdm(data['annotations'], desc='norm index in annotations'): + new_annotations += [deepcopy(ann)] + new_annotations[-1]['category_id'] = categories_map[ann['category_id']] + data['annotations'] = new_annotations + + with open(p + '.normed', 'w') as f: + json.dump(data, f) + return p + '.normed' diff --git a/data/datasets/object_detection/yolox_data_util/samplers.py b/data/datasets/object_detection/yolox_data_util/samplers.py new file mode 100644 index 0000000000000000000000000000000000000000..6b7ea38d3cd5bc0c906229b48ceaa51483173c42 --- /dev/null +++ b/data/datasets/object_detection/yolox_data_util/samplers.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +# Copyright (c) Megvii, Inc. and its affiliates. + +import itertools +from typing import Optional + +import torch +import torch.distributed as dist +from torch.utils.data.sampler import BatchSampler as torchBatchSampler +from torch.utils.data.sampler import Sampler + + +class YoloBatchSampler(torchBatchSampler): + """ + This batch sampler will generate mini-batches of (mosaic, index) tuples from another sampler. + It works just like the :class:`torch.utils.data.sampler.BatchSampler`, + but it will turn on/off the mosaic aug. + """ + + def __init__(self, *args, mosaic=True, **kwargs): + super().__init__(*args, **kwargs) + self.mosaic = mosaic + + def __iter__(self): + for batch in super().__iter__(): + yield [(self.mosaic, idx) for idx in batch] + + +class InfiniteSampler(Sampler): + """ + In training, we only care about the "infinite stream" of training data. + So this sampler produces an infinite stream of indices and + all workers cooperate to correctly shuffle the indices and sample different indices. + The samplers in each worker effectively produces `indices[worker_id::num_workers]` + where `indices` is an infinite stream of indices consisting of + `shuffle(range(size)) + shuffle(range(size)) + ...` (if shuffle is True) + or `range(size) + range(size) + ...` (if shuffle is False) + """ + + def __init__( + self, + size: int, + shuffle: bool = True, + seed: Optional[int] = 0, + rank=0, + world_size=1, + ): + """ + Args: + size (int): the total number of data of the underlying dataset to sample from + shuffle (bool): whether to shuffle the indices or not + seed (int): the initial seed of the shuffle. Must be the same + across all workers. If None, will use a random seed shared + among workers (require synchronization among all workers). + """ + self._size = size + assert size > 0 + self._shuffle = shuffle + self._seed = int(seed) + + if dist.is_available() and dist.is_initialized(): + self._rank = dist.get_rank() + self._world_size = dist.get_world_size() + else: + self._rank = rank + self._world_size = world_size + + def __iter__(self): + start = self._rank + yield from itertools.islice( + self._infinite_indices(), start, None, self._world_size + ) + + def _infinite_indices(self): + g = torch.Generator() + g.manual_seed(self._seed) + while True: + if self._shuffle: + yield from torch.randperm(self._size, generator=g) + else: + yield from torch.arange(self._size) + + def __len__(self): + return self._size // self._world_size diff --git a/data/datasets/pos_tagging/__init__.py b/data/datasets/pos_tagging/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..4c56617f98f9609b9ea20a2147df56ef2f1af4d4 --- /dev/null +++ b/data/datasets/pos_tagging/__init__.py @@ -0,0 +1,2 @@ +from .universal_asc_19_domains import * +# from .universal_asc_19_domains_for_gpt_neo import * diff --git a/data/datasets/pos_tagging/__pycache__/__init__.cpython-38.pyc b/data/datasets/pos_tagging/__pycache__/__init__.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b58670a6402c4df9306346f02278a79a4a65c0b9 Binary files /dev/null and b/data/datasets/pos_tagging/__pycache__/__init__.cpython-38.pyc differ diff --git a/data/datasets/pos_tagging/__pycache__/universal_asc_19_domains.cpython-38.pyc b/data/datasets/pos_tagging/__pycache__/universal_asc_19_domains.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..66f5b320a5273ec91413845eec71dc12b70a99fb Binary files /dev/null and b/data/datasets/pos_tagging/__pycache__/universal_asc_19_domains.cpython-38.pyc differ diff --git a/data/datasets/pos_tagging/__pycache__/universal_asc_19_domains_for_gpt_neo.cpython-38.pyc b/data/datasets/pos_tagging/__pycache__/universal_asc_19_domains_for_gpt_neo.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5f27de073b5515b31b2ebdb338bcf978ae04c191 Binary files /dev/null and b/data/datasets/pos_tagging/__pycache__/universal_asc_19_domains_for_gpt_neo.cpython-38.pyc differ diff --git a/data/datasets/pos_tagging/gen_label.py b/data/datasets/pos_tagging/gen_label.py new file mode 100644 index 0000000000000000000000000000000000000000..c40424db152a35e70b3a87ae3f68574416a1b1dc --- /dev/null +++ b/data/datasets/pos_tagging/gen_label.py @@ -0,0 +1,113 @@ +from utils.common.data_record import read_json, write_json +import requests +import random +import hashlib +import tqdm +import time +import nltk +# nltk.download('punkt') +# nltk.download('averaged_perceptron_tagger') +# https://blog.csdn.net/weixin_44826203/article/details/107484634 + +from data.datasets.sentiment_classification.global_bert_tokenizer import get_tokenizer +bert_tokenizer = get_tokenizer() + + +pos_tag_set = set() + +def gen_label_from_sen_cls_json(sen_cls_json_path): + texts = [] + anns = read_json(sen_cls_json_path) + for v in anns.values(): + texts += [v['sentence']] + assert '\n' not in texts[-1] + + texts = list(set(texts)) + + res_json = [] + + + # for text in tqdm.tqdm(texts): + # text = "When I put it to use for my daughter 's graduation party in longer lengths , the speaker wire worked as expected , even out of doors . I like it . " + # words = bert_tokenizer._tokenize(text) + # pos_tags = nltk.pos_tag(words) + + # print(text, pos_tags, len(pos_tags)) + + # exit() + + + for text in tqdm.tqdm(texts): + # text = "When I put it to use for my daughter 's graduation party in longer lengths , the speaker wire worked as expected , even out of doors . I like it . " + + # tag for whole text + words_whole_text = bert_tokenizer._tokenize(text) + pos_tags_trial = nltk.pos_tag(words_whole_text) + + # tag for splited sentences + sentences = nltk.sent_tokenize(text) + + pos_tags = [] + for sentence in sentences: + # words = nltk.word_tokenize(sentence) + words = bert_tokenizer._tokenize(sentence) + pos_tags += nltk.pos_tag(words) + + if len(pos_tags_trial) != len(pos_tags): + pos_tags = pos_tags_trial + + res_json += [{ + 'sentence': text, + 'tags': [t[1] for t in pos_tags] + }] + + for tag in pos_tags: + pos_tag_set.add(tag[1]) + + write_json(sen_cls_json_path + '.token_cls_data', res_json, backup=False) + +if __name__ == '__main__': + # res = translate('I am a doctor.\nHello world!') + # print(res) + import os + + data_dir_paths = { + **{k: f'/data/zql/datasets/nlp_asc_19_domains/dat/absa/Bing5Domains/asc/{k.split("-")[1]}' + for k in ['HL5Domains-ApexAD2600Progressive', 'HL5Domains-CanonG3', 'HL5Domains-CreativeLabsNomadJukeboxZenXtra40GB', + 'HL5Domains-NikonCoolpix4300', 'HL5Domains-Nokia6610']}, + + **{k: f'/data/zql/datasets/nlp_asc_19_domains/dat/absa/Bing3Domains/asc/{k.split("-")[1]}' + for k in ['Liu3Domains-Computer', 'Liu3Domains-Router', 'Liu3Domains-Speaker']}, + + **{k: f'/data/zql/datasets/nlp_asc_19_domains/dat/absa/Bing9Domains/asc/{k.split("-")[1]}' + for k in [f'Ding9Domains-{d}' for d in os.listdir('/data/zql/datasets/nlp_asc_19_domains/dat/absa/Bing9Domains/asc')]}, + + **{f'SemEval-{k[0].upper()}{k[1:]}': f'/data/zql/datasets/nlp_asc_19_domains/dat/absa/XuSemEval/asc/14/{k}' + for k in ['laptop', 'rest']}, + } + + json_paths = [] + for p in data_dir_paths.values(): + json_paths += [os.path.join(p, f'{split}.json') for split in ['train', 'dev', 'test']] + + assert all([os.path.exists(p) for p in json_paths]) + + # print(len(json_paths)) + # exit() + + # for p in tqdm.tqdm(json_paths): + # print(p) + # gen_label_from_sen_cls_json(p) + + # print(pos_tag_set) + + """ + ['FW', 'NNPS', '$', 'RBR', 'DT', 'VBG', 'EX', '.', 'JJS', 'RB', 'RP', 'JJR', '#', 'IN', 'VBZ', 'VB', 'NNP', 'WRB', 'JJ', 'POS', 'WP', 'RBS', 'VBN', 'UH', 'PRP$', 'NN', 'VBD', '(', 'NNS', 'WDT', 'MD', ',', 'SYM', 'TO', 'VBP', 'LS', 'PDT', 'CD', ')', ':', "''", 'PRP', 'CC'] + """ + + tags = ['FW', 'NNPS', '$', 'RBR', 'DT', 'VBG', 'EX', '.', 'JJS', 'RB', 'RP', 'JJR', '#', 'IN', 'VBZ', 'VB', 'NNP', 'WRB', 'JJ', 'POS', 'WP', 'RBS', 'VBN', 'UH', 'PRP$', 'NN', 'VBD', '(', 'NNS', 'WDT', 'MD', ',', 'SYM', 'TO', 'VBP', 'LS', 'PDT', 'CD', ')', ':', "''", 'PRP', 'CC'] + print(sorted(tags)) + + """ + ['CC', 'CD', 'DT', 'EX', 'FW', 'IN', 'JJ', 'JJR', 'JJS', 'LS', 'MD', 'NN', 'NNP', 'NNPS', 'NNS', 'PDT', 'POS', 'PRP', 'PRP$', 'RB', 'RBR', 'RBS', 'RP', 'SYM', 'TO', 'UH', 'VB', 'VBD', 'VBG', 'VBN', 'VBP', 'VBZ', 'WDT', 'WP', 'WRB', '#', '$', "''", '(', ')', ',', '.', ':'] + """ \ No newline at end of file diff --git a/data/datasets/pos_tagging/universal_asc_19_domains.py b/data/datasets/pos_tagging/universal_asc_19_domains.py new file mode 100644 index 0000000000000000000000000000000000000000..4e3b4e1dd34ad3edeabb16a70210001afb343748 --- /dev/null +++ b/data/datasets/pos_tagging/universal_asc_19_domains.py @@ -0,0 +1,336 @@ +import os +import torch +from torch.utils.data import Dataset, DataLoader +from transformers import BertTokenizer +from torch.optim import Adam +from torch.nn import CrossEntropyLoss +from typing import Dict, List, Optional, Any +from utils.common.data_record import read_json +from ..sentiment_classification.global_bert_tokenizer import get_tokenizer + + +TAGS = ['CC', 'CD', 'DT', 'EX', 'FW', 'IN', 'JJ', 'JJR', 'JJS', 'LS', 'MD', 'NN', 'NNP', 'NNPS', 'NNS', 'PDT', 'POS', 'PRP', 'PRP$', 'RB', 'RBR', 'RBS', 'RP', 'SYM', 'TO', 'UH', 'VB', 'VBD', 'VBG', 'VBN', 'VBP', 'VBZ', 'WDT', 'WP', 'WRB', '#', '$', "''", '(', ')', ',', '.', ':'] + + +# 自定义数据集类 +class UniversalASC19DomainsTokenClsDataset(Dataset): + + def __init__(self, root_dir: str, split: str, transform: Any, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + + assert transform is None + + self.tokenizer = get_tokenizer() # 传入tokenizer对象 + self.srcs = [] + self.tgts = [] + self.max_length = None # 设置文本的最大长度 + + json_file_path = os.path.join(root_dir, f'{split if split != "val" else "dev"}.json.token_cls_data') + anns = read_json(json_file_path) + + # label_map = {'-': 0, '+': 1, 'negative': 0, 'positive': 1} + + # ignore_cls_indexes = [classes.index(c) for c in ignore_classes] + + for info in anns: + self.srcs += [info['sentence']] + + label = [TAGS.index(t) for t in info['tags']] + label = label + [-100] * (512 - len(label)) + self.tgts += [label] + + def __len__(self): + return len(self.srcs) + + def __getitem__(self, idx): + text = self.srcs[idx] + label = self.tgts[idx] + + encoded_input = self.tokenizer.encode_plus( + text, max_length=self.max_length, padding="max_length", truncation=True, return_tensors="pt" + ) + + x = {key: tensor.squeeze(0) for key, tensor in encoded_input.items()} + x['return_dict'] = False + + return x, torch.tensor(label) + + +from ..ab_dataset import ABDataset +from ..registery import dataset_register + +@dataset_register( + name='HL5Domains-ApexAD2600Progressive-TokenCls', + classes=TAGS, + task_type='POS Tagging', + object_type='Generic', + class_aliases=[], + shift_type=None +) +class HL5Domains_ApexAD2600Progressive(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsTokenClsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + +@dataset_register( + name='HL5Domains-CanonG3-TokenCls', + classes=TAGS, + task_type='POS Tagging', + object_type='Generic', + class_aliases=[], + shift_type=None +) +class HL5Domains_CanonG3(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsTokenClsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + +@dataset_register( + name='HL5Domains-CreativeLabsNomadJukeboxZenXtra40GB-TokenCls', + classes=TAGS, + task_type='POS Tagging', + object_type='Generic', + class_aliases=[], + shift_type=None +) +class HL5Domains_CreativeLabsNomadJukeboxZenXtra40GB(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsTokenClsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + +@dataset_register( + name='HL5Domains-NikonCoolpix4300-TokenCls', + classes=TAGS, + task_type='POS Tagging', + object_type='Generic', + class_aliases=[], + shift_type=None +) +class HL5Domains_NikonCoolpix4300(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsTokenClsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + +@dataset_register( + name='HL5Domains-Nokia6610-TokenCls', + classes=TAGS, + task_type='POS Tagging', + object_type='Generic', + class_aliases=[], + shift_type=None +) +class HL5Domains_Nokia6610(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsTokenClsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + + +@dataset_register( + name='Liu3Domains-Computer-TokenCls', + classes=TAGS, + task_type='POS Tagging', + object_type='Generic', + class_aliases=[], + shift_type=None +) +class Liu3Domains_Computer(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsTokenClsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + +@dataset_register( + name='Liu3Domains-Router-TokenCls', + classes=TAGS, + task_type='POS Tagging', + object_type='Generic', + class_aliases=[], + shift_type=None +) +class Liu3Domains_Router(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsTokenClsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + +@dataset_register( + name='Liu3Domains-Speaker-TokenCls', + classes=TAGS, + task_type='POS Tagging', + object_type='Generic', + class_aliases=[], + shift_type=None +) +class Liu3Domains_Speaker(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsTokenClsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + + +# import os +# for domain in os.listdir('/data/zql/datasets/nlp_asc_19_domains/dat/absa/Bing9Domains/asc'): +# print(f""" +# @dataset_register( +# name='Ding9Domains-{domain}', +# classes=TAGS, +# task_type='POS Tagging', +# object_type='Generic', +# class_aliases=[], +# shift_type=None +# ) +# class Ding9Domains_{domain}(ABDataset): +# def create_dataset(self, root_dir: str, split: str, transform, +# classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): +# return UniversalASC19DomainsTokenClsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) +# """) + +@dataset_register( + name='Ding9Domains-DiaperChamp-TokenCls', + classes=TAGS, + task_type='POS Tagging', + object_type='Generic', + class_aliases=[], + shift_type=None +) +class Ding9Domains_DiaperChamp(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsTokenClsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + + +@dataset_register( + name='Ding9Domains-Norton-TokenCls', + classes=TAGS, + task_type='POS Tagging', + object_type='Generic', + class_aliases=[], + shift_type=None +) +class Ding9Domains_Norton(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsTokenClsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + + +@dataset_register( + name='Ding9Domains-LinksysRouter-TokenCls', + classes=TAGS, + task_type='POS Tagging', + object_type='Generic', + class_aliases=[], + shift_type=None +) +class Ding9Domains_LinksysRouter(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsTokenClsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + + +@dataset_register( + name='Ding9Domains-MicroMP3-TokenCls', + classes=TAGS, + task_type='POS Tagging', + object_type='Generic', + class_aliases=[], + shift_type=None +) +class Ding9Domains_MicroMP3(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsTokenClsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + + +@dataset_register( + name='Ding9Domains-Nokia6600-TokenCls', + classes=TAGS, + task_type='POS Tagging', + object_type='Generic', + class_aliases=[], + shift_type=None +) +class Ding9Domains_Nokia6600(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsTokenClsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + + +@dataset_register( + name='Ding9Domains-CanonPowerShotSD500-TokenCls', + classes=TAGS, + task_type='POS Tagging', + object_type='Generic', + class_aliases=[], + shift_type=None +) +class Ding9Domains_CanonPowerShotSD500(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsTokenClsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + + +@dataset_register( + name='Ding9Domains-ipod-TokenCls', + classes=TAGS, + task_type='POS Tagging', + object_type='Generic', + class_aliases=[], + shift_type=None +) +class Ding9Domains_ipod(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsTokenClsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + + +@dataset_register( + name='Ding9Domains-HitachiRouter-TokenCls', + classes=TAGS, + task_type='POS Tagging', + object_type='Generic', + class_aliases=[], + shift_type=None +) +class Ding9Domains_HitachiRouter(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsTokenClsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + + +@dataset_register( + name='Ding9Domains-CanonS100-TokenCls', + classes=TAGS, + task_type='POS Tagging', + object_type='Generic', + class_aliases=[], + shift_type=None +) +class Ding9Domains_CanonS100(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsTokenClsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + + + +@dataset_register( + name='SemEval-Laptop-TokenCls', + classes=TAGS, + task_type='POS Tagging', + object_type='Generic', + class_aliases=[], + shift_type=None +) +class SemEval_Laptop(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsTokenClsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + +@dataset_register( + name='SemEval-Rest-TokenCls', + classes=TAGS, + task_type='POS Tagging', + object_type='Generic', + class_aliases=[], + shift_type=None +) +class SemEval_Rest(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsTokenClsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) \ No newline at end of file diff --git a/data/datasets/pos_tagging/universal_asc_19_domains_for_gpt_neo.py b/data/datasets/pos_tagging/universal_asc_19_domains_for_gpt_neo.py new file mode 100644 index 0000000000000000000000000000000000000000..d64d854a779ae0e9380573148e584496e6a1216b --- /dev/null +++ b/data/datasets/pos_tagging/universal_asc_19_domains_for_gpt_neo.py @@ -0,0 +1,342 @@ +import os +import torch +from torch.utils.data import Dataset, DataLoader +from transformers import BertTokenizer +from torch.optim import Adam +from torch.nn import CrossEntropyLoss +from typing import Dict, List, Optional, Any +from utils.common.data_record import read_json +# from ..sentiment_classification.global_bert_tokenizer import get_tokenizer + +from transformers import GPT2Tokenizer +tokenizer = GPT2Tokenizer.from_pretrained('experiments/elasticdnn/gpt_series/ckpts/gpt2') +# print(tokenizer.config) + +TAGS = ['CC', 'CD', 'DT', 'EX', 'FW', 'IN', 'JJ', 'JJR', 'JJS', 'LS', 'MD', 'NN', 'NNP', 'NNPS', 'NNS', 'PDT', 'POS', 'PRP', 'PRP$', 'RB', 'RBR', 'RBS', 'RP', 'SYM', 'TO', 'UH', 'VB', 'VBD', 'VBG', 'VBN', 'VBP', 'VBZ', 'WDT', 'WP', 'WRB', '#', '$', "''", '(', ')', ',', '.', ':'] + + +# 自定义数据集类 +class UniversalASC19DomainsTokenClsDataset(Dataset): + + def __init__(self, root_dir: str, split: str, transform: Any, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + + assert transform is None + + self.tokenizer = tokenizer # 传入tokenizer对象 + tokenizer.pad_token = tokenizer.eos_token + + self.srcs = [] + self.tgts = [] + self.max_length = 512 # 设置文本的最大长度 + + json_file_path = os.path.join(root_dir, f'{split if split != "val" else "dev"}.json.token_cls_data') + anns = read_json(json_file_path) + + # label_map = {'-': 0, '+': 1, 'negative': 0, 'positive': 1} + + # ignore_cls_indexes = [classes.index(c) for c in ignore_classes] + + for info in anns: + self.srcs += [info['sentence']] + + label = [TAGS.index(t) for t in info['tags']] + label = label + [-100] * (self.max_length - len(label)) + self.tgts += [label] + + def __len__(self): + return len(self.srcs) + + def __getitem__(self, idx): + text = self.srcs[idx] + label = self.tgts[idx] + + encoded_input = self.tokenizer.encode_plus( + text, max_length=self.max_length, padding="max_length", truncation=True, return_tensors="pt" + ) + + x = {key: tensor.squeeze(0) for key, tensor in encoded_input.items()} + # print(x['input_ids'].size()) + x['return_dict'] = False + + return x, torch.tensor(label) + + +from ..ab_dataset import ABDataset +from ..registery import dataset_register + +@dataset_register( + name='HL5Domains-ApexAD2600Progressive-TokenCls-GPTNeo', + classes=TAGS, + task_type='POS Tagging', + object_type='Generic', + class_aliases=[], + shift_type=None +) +class HL5Domains_ApexAD2600Progressive_GPTNeo(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsTokenClsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + +@dataset_register( + name='HL5Domains-CanonG3-TokenCls-GPTNeo', + classes=TAGS, + task_type='POS Tagging', + object_type='Generic', + class_aliases=[], + shift_type=None +) +class HL5Domains_CanonG3_GPTNeo(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsTokenClsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + +@dataset_register( + name='HL5Domains-CreativeLabsNomadJukeboxZenXtra40GB-TokenCls-GPTNeo', + classes=TAGS, + task_type='POS Tagging', + object_type='Generic', + class_aliases=[], + shift_type=None +) +class HL5Domains_CreativeLabsNomadJukeboxZenXtra40GB_GPTNeo(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsTokenClsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + +@dataset_register( + name='HL5Domains-NikonCoolpix4300-TokenCls-GPTNeo', + classes=TAGS, + task_type='POS Tagging', + object_type='Generic', + class_aliases=[], + shift_type=None +) +class HL5Domains_NikonCoolpix4300_GPTNeo(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsTokenClsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + +@dataset_register( + name='HL5Domains-Nokia6610-TokenCls-GPTNeo', + classes=TAGS, + task_type='POS Tagging', + object_type='Generic', + class_aliases=[], + shift_type=None +) +class HL5Domains_Nokia6610_GPTNeo(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsTokenClsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + + +@dataset_register( + name='Liu3Domains-Computer-TokenCls-GPTNeo', + classes=TAGS, + task_type='POS Tagging', + object_type='Generic', + class_aliases=[], + shift_type=None +) +class Liu3Domains_Computer_GPTNeo(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsTokenClsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + +@dataset_register( + name='Liu3Domains-Router-TokenCls-GPTNeo', + classes=TAGS, + task_type='POS Tagging', + object_type='Generic', + class_aliases=[], + shift_type=None +) +class Liu3Domains_Router_GPTNeo(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsTokenClsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + +@dataset_register( + name='Liu3Domains-Speaker-TokenCls-GPTNeo', + classes=TAGS, + task_type='POS Tagging', + object_type='Generic', + class_aliases=[], + shift_type=None +) +class Liu3Domains_Speaker_GPTNeo(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsTokenClsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + + +# import os +# for domain in os.listdir('/data/zql/datasets/nlp_asc_19_domains/dat/absa/Bing9Domains/asc'): +# print(f""" +# @dataset_register( +# name='Ding9Domains-{domain}', +# classes=TAGS, +# task_type='POS Tagging', +# object_type='Generic', +# class_aliases=[], +# shift_type=None +# ) +# class Ding9Domains_{domain}_GPTNeo(ABDataset): +# def create_dataset(self, root_dir: str, split: str, transform, +# classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): +# return UniversalASC19DomainsTokenClsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) +# """) + +@dataset_register( + name='Ding9Domains-DiaperChamp-TokenCls-GPTNeo', + classes=TAGS, + task_type='POS Tagging', + object_type='Generic', + class_aliases=[], + shift_type=None +) +class Ding9Domains_DiaperChamp_GPTNeo(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsTokenClsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + + +@dataset_register( + name='Ding9Domains-Norton-TokenCls-GPTNeo', + classes=TAGS, + task_type='POS Tagging', + object_type='Generic', + class_aliases=[], + shift_type=None +) +class Ding9Domains_Norton_GPTNeo(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsTokenClsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + + +@dataset_register( + name='Ding9Domains-LinksysRouter-TokenCls-GPTNeo', + classes=TAGS, + task_type='POS Tagging', + object_type='Generic', + class_aliases=[], + shift_type=None +) +class Ding9Domains_LinksysRouter_GPTNeo(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsTokenClsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + + +@dataset_register( + name='Ding9Domains-MicroMP3-TokenCls-GPTNeo', + classes=TAGS, + task_type='POS Tagging', + object_type='Generic', + class_aliases=[], + shift_type=None +) +class Ding9Domains_MicroMP3_GPTNeo(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsTokenClsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + + +@dataset_register( + name='Ding9Domains-Nokia6600-TokenCls-GPTNeo', + classes=TAGS, + task_type='POS Tagging', + object_type='Generic', + class_aliases=[], + shift_type=None +) +class Ding9Domains_Nokia6600_GPTNeo(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsTokenClsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + + +@dataset_register( + name='Ding9Domains-CanonPowerShotSD500-TokenCls-GPTNeo', + classes=TAGS, + task_type='POS Tagging', + object_type='Generic', + class_aliases=[], + shift_type=None +) +class Ding9Domains_CanonPowerShotSD500_GPTNeo(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsTokenClsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + + +@dataset_register( + name='Ding9Domains-ipod-TokenCls-GPTNeo', + classes=TAGS, + task_type='POS Tagging', + object_type='Generic', + class_aliases=[], + shift_type=None +) +class Ding9Domains_ipod_GPTNeo(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsTokenClsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + + +@dataset_register( + name='Ding9Domains-HitachiRouter-TokenCls-GPTNeo', + classes=TAGS, + task_type='POS Tagging', + object_type='Generic', + class_aliases=[], + shift_type=None +) +class Ding9Domains_HitachiRouter_GPTNeo(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsTokenClsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + + +@dataset_register( + name='Ding9Domains-CanonS100-TokenCls-GPTNeo', + classes=TAGS, + task_type='POS Tagging', + object_type='Generic', + class_aliases=[], + shift_type=None +) +class Ding9Domains_CanonS100_GPTNeo(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsTokenClsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + + + +@dataset_register( + name='SemEval-Laptop-TokenCls-GPTNeo', + classes=TAGS, + task_type='POS Tagging', + object_type='Generic', + class_aliases=[], + shift_type=None +) +class SemEval_Laptop_GPTNeo(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsTokenClsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + +@dataset_register( + name='SemEval-Rest-TokenCls-GPTNeo', + classes=TAGS, + task_type='POS Tagging', + object_type='Generic', + class_aliases=[], + shift_type=None +) +class SemEval_Rest_GPTNeo(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsTokenClsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) \ No newline at end of file diff --git a/data/datasets/registery.py b/data/datasets/registery.py new file mode 100644 index 0000000000000000000000000000000000000000..b76f8f05a8534760395089084b5b118e5f1d8ef1 --- /dev/null +++ b/data/datasets/registery.py @@ -0,0 +1,37 @@ + +from typing import Dict, Type + +from .ab_dataset import ABDataset + + +static_dataset_registery = {} + +# (NIPS'20) Measuring Robustness to Natural Distribution Shifts in Image Classification +POSSIBLE_SHIFT_TYPES = ['Consistency Shifts', 'Dataset Shifts', 'Adversarially Filtered Shifts', + 'Image Corruptions', 'Geometric Transformations', 'Style Transfer', 'Adversarial Examples'] + +def dataset_register(name, classes, task_type, object_type, class_aliases=[], shift_type: Dict[str, str]=None): + assert shift_type is None or len(shift_type.keys()) == 1 + if shift_type is not None: + assert list(shift_type.values())[0] in POSSIBLE_SHIFT_TYPES + + class _Register: + def __init__(self, func: Type[ABDataset]): + self.func = func + static_dataset_registery[name] = (self, classes, task_type, object_type, class_aliases, shift_type) + + def __call__(self, *args, **kwargs): + res = self.func(*args, **kwargs) + + res.name = name + res.classes = classes + res.class_aliases = class_aliases + res.shift_type = shift_type + res.task_type = task_type + res.object_type = object_type + + res.build() + + return res + + return _Register diff --git a/data/datasets/semantic_segmentation/__init__.py b/data/datasets/semantic_segmentation/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..72212a1c655b19a4c070596ee4bb6d9c3f1d6823 --- /dev/null +++ b/data/datasets/semantic_segmentation/__init__.py @@ -0,0 +1,4 @@ +from .gta5 import GTA5 +from .cityscapes import Cityscapes +from .baidu_person import BaiduPerson +from .supervisely_person import SuperviselyPerson diff --git a/data/datasets/semantic_segmentation/__pycache__/__init__.cpython-38.pyc b/data/datasets/semantic_segmentation/__pycache__/__init__.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f1de6a99b4111df69da31cc178f5138c0955399e Binary files /dev/null and b/data/datasets/semantic_segmentation/__pycache__/__init__.cpython-38.pyc differ diff --git a/data/datasets/semantic_segmentation/__pycache__/baidu_person.cpython-38.pyc b/data/datasets/semantic_segmentation/__pycache__/baidu_person.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..242009a0527ff382906792b77436a5394ddb821b Binary files /dev/null and b/data/datasets/semantic_segmentation/__pycache__/baidu_person.cpython-38.pyc differ diff --git a/data/datasets/semantic_segmentation/__pycache__/cityscapes.cpython-38.pyc b/data/datasets/semantic_segmentation/__pycache__/cityscapes.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a7e6b7dc10db8bc45bcc8ca9a3dc1d118bf13b6d Binary files /dev/null and b/data/datasets/semantic_segmentation/__pycache__/cityscapes.cpython-38.pyc differ diff --git a/data/datasets/semantic_segmentation/__pycache__/common_dataset.cpython-38.pyc b/data/datasets/semantic_segmentation/__pycache__/common_dataset.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..39e950d11e93f8b134f19327e65a9e73fe5d6ba9 Binary files /dev/null and b/data/datasets/semantic_segmentation/__pycache__/common_dataset.cpython-38.pyc differ diff --git a/data/datasets/semantic_segmentation/__pycache__/gta5.cpython-38.pyc b/data/datasets/semantic_segmentation/__pycache__/gta5.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9d0e02b13b3859dcb89195931c53200ec40ddd03 Binary files /dev/null and b/data/datasets/semantic_segmentation/__pycache__/gta5.cpython-38.pyc differ diff --git a/data/datasets/semantic_segmentation/__pycache__/supervisely_person.cpython-38.pyc b/data/datasets/semantic_segmentation/__pycache__/supervisely_person.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ffcbf3c190619353893ba9f0cff31fb43909216f Binary files /dev/null and b/data/datasets/semantic_segmentation/__pycache__/supervisely_person.cpython-38.pyc differ diff --git a/data/datasets/semantic_segmentation/baidu_person.py b/data/datasets/semantic_segmentation/baidu_person.py new file mode 100644 index 0000000000000000000000000000000000000000..6fbff6cfb3400af72a0c85f6d5ec1e8738b25e14 --- /dev/null +++ b/data/datasets/semantic_segmentation/baidu_person.py @@ -0,0 +1,75 @@ +from ..data_aug import cityscapes_like_image_train_aug, cityscapes_like_image_test_aug, cityscapes_like_label_aug +from .common_dataset import CommonDataset +from ..ab_dataset import ABDataset +from ..dataset_split import train_val_test_split +import numpy as np +from typing import Dict, List, Optional +from torchvision.transforms import Compose, Lambda +import os + +from ..registery import dataset_register + + +@dataset_register( + name='BaiduPerson', + classes=[ + 'person', 'background' + ], + task_type='Semantic Segmentation', + object_type='Person', + # class_aliases=[['automobile', 'car']], + class_aliases=[], + shift_type=None +) +class BaiduPerson(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform: Optional[Compose], + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + if transform is None: + x_transform = cityscapes_like_image_train_aug() if split == 'train' else cityscapes_like_image_test_aug() + y_transform = cityscapes_like_label_aug() + self.transform = x_transform + else: + x_transform = transform + y_transform = cityscapes_like_label_aug() + + images_path, labels_path = [], [] + for p in os.listdir(os.path.join(root_dir, 'images')): + images_path += [os.path.join(root_dir, 'images', p)] + labels_path += [os.path.join(root_dir, 'profiles', p.split('.')[0] + '-profile.jpg')] + + idx_map_in_y_transform = {i: i for i in range(len(classes))} + + # dataset.targets = np.asarray(dataset.targets) + if len(ignore_classes) > 0: + for ignore_class in ignore_classes: + # dataset.data = dataset.data[dataset.targets != classes.index(ignore_class)] + # dataset.targets = dataset.targets[dataset.targets != classes.index(ignore_class)] + idx_map_in_y_transform[ignore_class] = 255 + + if idx_map is not None: + # note: the code below seems correct but has bug! + # for old_idx, new_idx in idx_map.items(): + # dataset.targets[dataset.targets == old_idx] = new_idx + + # for ti, t in enumerate(dataset.targets): + # dataset.targets[ti] = idx_map[t] + + for k, v in idx_map.items(): + idx_map_in_y_transform[k] = v + + def map_class(x): + x[x > 1] = 1 # 0: background; 1: person + + for k, v in idx_map_in_y_transform.items(): + x[x == k] = v + return x + + y_transform = Compose([ + *y_transform.transforms, + Lambda(lambda x: map_class(x)) + ]) + + dataset = CommonDataset(images_path, labels_path, x_transform=x_transform, y_transform=y_transform) + + dataset = train_val_test_split(dataset, split) + return dataset diff --git a/data/datasets/semantic_segmentation/cityscapes.py b/data/datasets/semantic_segmentation/cityscapes.py new file mode 100644 index 0000000000000000000000000000000000000000..1ca767466ded2cb6cfae245879a913f2ba91787a --- /dev/null +++ b/data/datasets/semantic_segmentation/cityscapes.py @@ -0,0 +1,95 @@ +from ..data_aug import cityscapes_like_image_train_aug, cityscapes_like_image_test_aug, cityscapes_like_label_aug +from torchvision.datasets import Cityscapes as RawCityscapes +from ..ab_dataset import ABDataset +from ..dataset_split import train_val_test_split +import numpy as np +from typing import Dict, List, Optional +from torchvision.transforms import Compose, Lambda +import os + +from ..registery import dataset_register + + +@dataset_register( + name='Cityscapes', + classes=[ + 'road', 'sidewalk', 'building', 'wall', + 'fence', 'pole', 'light', 'sign', + 'vegetation', 'terrain', 'sky', 'people', # person + 'rider', 'car', 'truck', 'bus', 'train', + 'motocycle', 'bicycle' + ], + task_type='Semantic Segmentation', + object_type='Autonomous Driving', + # class_aliases=[['automobile', 'car']], + class_aliases=[], + shift_type=None +) +class Cityscapes(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform: Optional[Compose], + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + if transform is None: + x_transform = cityscapes_like_image_train_aug() if split == 'train' else cityscapes_like_image_test_aug() + y_transform = cityscapes_like_label_aug() + self.transform = x_transform + else: + x_transform = transform + y_transform = cityscapes_like_label_aug() + + # images_path, labels_path = [], [] + # for p in os.listdir(os.path.join(root_dir, 'images')): + # p = os.path.join(root_dir, 'images', p) + # if not p.endswith('png'): + # continue + # images_path += [p] + # labels_path += [p.replace('images', 'labels_gt')] + + ignore_label = 255 + raw_idx_map_in_y_transform = {-1: ignore_label, 0: ignore_label, 1: ignore_label, 2: ignore_label, + 3: ignore_label, 4: ignore_label, 5: ignore_label, 6: ignore_label, + 7: 0, 8: 1, 9: ignore_label, 10: ignore_label, 11: 2, 12: 3, 13: 4, + 14: ignore_label, 15: ignore_label, 16: ignore_label, 17: 5, + 18: ignore_label, 19: 6, 20: 7, 21: 8, 22: 9, 23: 10, 24: 11, 25: 12, 26: 13, 27: 14, + 28: 15, 29: ignore_label, 30: ignore_label, 31: 16, 32: 17, 33: 18} + + idx_map_in_y_transform = {i: i for i in range(len(classes))} + idx_map_in_y_transform[255] = 255 + + # dataset.targets = np.asarray(dataset.targets) + if len(ignore_classes) > 0: + for ignore_class in ignore_classes: + # dataset.data = dataset.data[dataset.targets != classes.index(ignore_class)] + # dataset.targets = dataset.targets[dataset.targets != classes.index(ignore_class)] + idx_map_in_y_transform[ignore_class] = 255 + + if idx_map is not None: + # note: the code below seems correct but has bug! + # for old_idx, new_idx in idx_map.items(): + # dataset.targets[dataset.targets == old_idx] = new_idx + + # for ti, t in enumerate(dataset.targets): + # dataset.targets[ti] = idx_map[t] + + for k, v in idx_map.items(): + idx_map_in_y_transform[k] = v + + # merge idx map + final_idx_map_in_y_transform = {} + for k1, v1 in raw_idx_map_in_y_transform.items(): + final_idx_map_in_y_transform[k1] = idx_map_in_y_transform[v1] + idx_map_in_y_transform = final_idx_map_in_y_transform + + def map_class(x): + for k, v in idx_map_in_y_transform.items(): + x[x == k] = v + return x + + y_transform = Compose([ + *y_transform.transforms, + Lambda(lambda x: map_class(x)) + ]) + + dataset = RawCityscapes(root_dir, target_type='semantic', + transform=x_transform, target_transform=y_transform) + dataset = train_val_test_split(dataset, split) + return dataset diff --git a/data/datasets/semantic_segmentation/common_dataset.py b/data/datasets/semantic_segmentation/common_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..936e38862e32bc20f24f84b95eca5296d921448d --- /dev/null +++ b/data/datasets/semantic_segmentation/common_dataset.py @@ -0,0 +1,53 @@ +from torch.utils.data import Dataset +import os +from torchvision.datasets.folder import default_loader +import torchvision.transforms as T +import torch +import numpy as np +from PIL import Image + + +class CommonDataset(Dataset): + def __init__(self, images_path, labels_path, x_transform, y_transform): + self.imgs_path = images_path + self.labels_path = labels_path + + # for p in os.listdir(os.path.join(image_dir)): + # p = os.path.join(dataset_project_dir, 'images', p) + # if not p.endswith('png'): + # continue + # self.imgs_path += [p] + # self.labels_path += [p.replace('images', 'labels_gt')] + + # self.x_transform = T.Compose( + # [ + # T.Resize((224, 224)), + # T.ToTensor(), + # T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), + # ] + # ) + # self.y_transform = T.Compose( + # [ + # T.Resize((224, 224)), + # T.Lambda(lambda x: torch.from_numpy(np.array(x)).long()) + # ] + # ) + self.x_transform = x_transform + self.y_transform = y_transform + + + def __len__(self): + return len(self.imgs_path) + + def __getitem__(self, idx): + x_path = os.path.join(self.imgs_path[idx]) + y_path = os.path.join(self.labels_path[idx]) + + x = default_loader(x_path) + # y = default_loader(y_path) + y = Image.open(y_path).convert('L') + + x = self.x_transform(x) + y = self.y_transform(y) + + return x, y diff --git a/data/datasets/semantic_segmentation/gta5.py b/data/datasets/semantic_segmentation/gta5.py new file mode 100644 index 0000000000000000000000000000000000000000..bddd89307f654c51deeaa8659296cf5e4474d37a --- /dev/null +++ b/data/datasets/semantic_segmentation/gta5.py @@ -0,0 +1,88 @@ +from ..data_aug import cityscapes_like_image_train_aug, cityscapes_like_image_test_aug, cityscapes_like_label_aug +from .common_dataset import CommonDataset +from ..ab_dataset import ABDataset +from ..dataset_split import train_val_test_split +import numpy as np +from typing import Dict, List, Optional +from torchvision.transforms import Compose, Lambda +import os + +from ..registery import dataset_register + + +@dataset_register( + name='GTA5', + classes=[ + 'road', 'sidewalk', 'building', 'wall', + 'fence', 'pole', 'light', 'sign', + 'vegetation', 'terrain', 'sky', 'people', # person + 'rider', 'car', 'truck', 'bus', 'train', + 'motocycle', 'bicycle' + ], + task_type='Semantic Segmentation', + object_type='Autonomous Driving', + # class_aliases=[['automobile', 'car']], + class_aliases=[], + shift_type=None +) +class GTA5(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform: Optional[Compose], + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + + # x_transform, y_transform = transform + # if x_transform is None: + # x_transform = cityscapes_like_image_train_aug() if split == 'train' else cityscapes_like_image_test_aug() + # self.transform = x_transform + # if y_transform is None: + # y_transform = cityscapes_like_label_aug() + + if transform is None: + x_transform = cityscapes_like_image_train_aug() if split == 'train' else cityscapes_like_image_test_aug() + y_transform = cityscapes_like_label_aug() + self.transform = x_transform + else: + x_transform = transform + y_transform = cityscapes_like_label_aug() + + images_path, labels_path = [], [] + for p in os.listdir(os.path.join(root_dir, 'images')): + p = os.path.join(root_dir, 'images', p) + if not p.endswith('png'): + continue + images_path += [p] + labels_path += [p.replace('images', 'labels_gt')] + + idx_map_in_y_transform = {i: i for i in range(len(classes))} + + # dataset.targets = np.asarray(dataset.targets) + if len(ignore_classes) > 0: + for ignore_class in ignore_classes: + # dataset.data = dataset.data[dataset.targets != classes.index(ignore_class)] + # dataset.targets = dataset.targets[dataset.targets != classes.index(ignore_class)] + idx_map_in_y_transform[ignore_class] = 255 + + if idx_map is not None: + # note: the code below seems correct but has bug! + # for old_idx, new_idx in idx_map.items(): + # dataset.targets[dataset.targets == old_idx] = new_idx + + # for ti, t in enumerate(dataset.targets): + # dataset.targets[ti] = idx_map[t] + + for k, v in idx_map.items(): + idx_map_in_y_transform[k] = v + + def map_class(x): + for k, v in idx_map_in_y_transform.items(): + x[x == k] = v + return x + + y_transform = Compose([ + *y_transform.transforms, + Lambda(lambda x: map_class(x)) + ]) + + dataset = CommonDataset(images_path, labels_path, x_transform=x_transform, y_transform=y_transform) + + dataset = train_val_test_split(dataset, split) + return dataset diff --git a/data/datasets/semantic_segmentation/supervisely_person.py b/data/datasets/semantic_segmentation/supervisely_person.py new file mode 100644 index 0000000000000000000000000000000000000000..a0f34afe37bcc49102e16d501d6cabaf5c4d74b6 --- /dev/null +++ b/data/datasets/semantic_segmentation/supervisely_person.py @@ -0,0 +1,79 @@ +from ..data_aug import cityscapes_like_image_train_aug, cityscapes_like_image_test_aug, cityscapes_like_label_aug +from .common_dataset import CommonDataset +from ..ab_dataset import ABDataset +from ..dataset_split import train_val_test_split +import numpy as np +from typing import Dict, List, Optional +from torchvision.transforms import Compose, Lambda +import os + +from ..registery import dataset_register + + +@dataset_register( + name='SuperviselyPerson', + classes=[ + 'background', 'person' + ], + task_type='Semantic Segmentation', + object_type='Person', + # class_aliases=[['automobile', 'car']], + class_aliases=[], + shift_type=None +) +class SuperviselyPerson(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform: Optional[Compose], + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + if transform is None: + x_transform = cityscapes_like_image_train_aug() if split == 'train' else cityscapes_like_image_test_aug() + y_transform = cityscapes_like_label_aug() + self.transform = x_transform + else: + x_transform = transform + y_transform = cityscapes_like_label_aug() + + images_path, labels_path = [], [] + for p in os.listdir(root_dir): + if p.startswith('ds'): + p1 = os.path.join(root_dir, p, 'img') + images_path += [(p, os.path.join(p1, n)) for n in os.listdir(p1)] + + for dsi, img_p in images_path: + target_p = os.path.join(root_dir, p, dsi, img_p.split('/')[-1]) + labels_path += [target_p] + images_path = [i[1] for i in images_path] + + idx_map_in_y_transform = {i: i for i in range(len(classes))} + + # dataset.targets = np.asarray(dataset.targets) + if len(ignore_classes) > 0: + for ignore_class in ignore_classes: + # dataset.data = dataset.data[dataset.targets != classes.index(ignore_class)] + # dataset.targets = dataset.targets[dataset.targets != classes.index(ignore_class)] + idx_map_in_y_transform[ignore_class] = 255 + + if idx_map is not None: + # note: the code below seems correct but has bug! + # for old_idx, new_idx in idx_map.items(): + # dataset.targets[dataset.targets == old_idx] = new_idx + + # for ti, t in enumerate(dataset.targets): + # dataset.targets[ti] = idx_map[t] + + for k, v in idx_map.items(): + idx_map_in_y_transform[k] = v + + def map_class(x): + for k, v in idx_map_in_y_transform.items(): + x[x == k] = v + return x + + y_transform = Compose([ + *y_transform.transforms, + Lambda(lambda x: map_class(x)) + ]) + + dataset = CommonDataset(images_path, labels_path, x_transform=x_transform, y_transform=y_transform) + + dataset = train_val_test_split(dataset, split) + return dataset diff --git a/data/datasets/sentiment_classification/REFERENCES.md b/data/datasets/sentiment_classification/REFERENCES.md new file mode 100644 index 0000000000000000000000000000000000000000..0bed634f78a3c96d4e3c31b6aa0355295fdd61c6 --- /dev/null +++ b/data/datasets/sentiment_classification/REFERENCES.md @@ -0,0 +1 @@ +https://juejin.cn/post/7249286405023825977 # how to use custom dataset diff --git a/data/datasets/sentiment_classification/__init__.py b/data/datasets/sentiment_classification/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..7e5fc865860cd4070b158457728558a36fed22a1 --- /dev/null +++ b/data/datasets/sentiment_classification/__init__.py @@ -0,0 +1,8 @@ +from .universal_asc_19_domains import * +from .newsgroup import Newsgroup, Newsgroup2, Newsgroup3 +# from .universal_asc_19_domains_for_gpt import * +# from .newsgroup_for_gpt import * + +# from .newsgroup_for_gpt_with_aug1 import * +# from .newsgroup_for_gpt_with_aug2 import * +# from .newsgroup_for_gpt_with_aug3 import * diff --git a/data/datasets/sentiment_classification/__pycache__/__init__.cpython-38.pyc b/data/datasets/sentiment_classification/__pycache__/__init__.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..bd261026fe5a66c3235babdb6a0dc68ce0f7f4bd Binary files /dev/null and b/data/datasets/sentiment_classification/__pycache__/__init__.cpython-38.pyc differ diff --git a/data/datasets/sentiment_classification/__pycache__/global_bert_tokenizer.cpython-38.pyc b/data/datasets/sentiment_classification/__pycache__/global_bert_tokenizer.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..20ad9a2627ab5b3e44cb79689adc11ee59d58113 Binary files /dev/null and b/data/datasets/sentiment_classification/__pycache__/global_bert_tokenizer.cpython-38.pyc differ diff --git a/data/datasets/sentiment_classification/__pycache__/newsgroup.cpython-38.pyc b/data/datasets/sentiment_classification/__pycache__/newsgroup.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..27298a3f5747f90f73834d299d5dd1f1c48972f3 Binary files /dev/null and b/data/datasets/sentiment_classification/__pycache__/newsgroup.cpython-38.pyc differ diff --git a/data/datasets/sentiment_classification/__pycache__/universal_asc_19_domains.cpython-38.pyc b/data/datasets/sentiment_classification/__pycache__/universal_asc_19_domains.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e31ad867c82bfd9a761d84a079af9601b834b7c5 Binary files /dev/null and b/data/datasets/sentiment_classification/__pycache__/universal_asc_19_domains.cpython-38.pyc differ diff --git a/data/datasets/sentiment_classification/global_bert_tokenizer.py b/data/datasets/sentiment_classification/global_bert_tokenizer.py new file mode 100644 index 0000000000000000000000000000000000000000..1608a75c881dd453ba145e82df18536735bdbe36 --- /dev/null +++ b/data/datasets/sentiment_classification/global_bert_tokenizer.py @@ -0,0 +1,16 @@ +from transformers import AutoTokenizer +from utils.common.log import logger +import os + +tokenizer = None +bert_model_tag = os.environ['bert_path'] if 'bert_path' in os.environ.keys() else '/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/data/datasets/sentiment_classification/mobilebert-uncased' + + +def get_tokenizer(): + global tokenizer + + + if tokenizer is None: + logger.info(f'init bert tokenizer for sen cls (using {bert_model_tag})') + tokenizer = AutoTokenizer.from_pretrained(bert_model_tag) + return tokenizer diff --git a/data/datasets/sentiment_classification/mobilebert-uncased/tokenizer.json b/data/datasets/sentiment_classification/mobilebert-uncased/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..949a6f013d67eb8a5b4b5b46026217b888021b88 --- /dev/null +++ b/data/datasets/sentiment_classification/mobilebert-uncased/tokenizer.json @@ -0,0 +1 @@ +{"version":"1.0","truncation":null,"padding":null,"added_tokens":[{"id":0,"special":true,"content":"[PAD]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":100,"special":true,"content":"[UNK]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":101,"special":true,"content":"[CLS]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":102,"special":true,"content":"[SEP]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":103,"special":true,"content":"[MASK]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false}],"normalizer":{"type":"BertNormalizer","clean_text":true,"handle_chinese_chars":true,"strip_accents":null,"lowercase":true},"pre_tokenizer":{"type":"BertPreTokenizer"},"post_processor":{"type":"TemplateProcessing","single":[{"SpecialToken":{"id":"[CLS]","type_id":0}},{"Sequence":{"id":"A","type_id":0}},{"SpecialToken":{"id":"[SEP]","type_id":0}}],"pair":[{"SpecialToken":{"id":"[CLS]","type_id":0}},{"Sequence":{"id":"A","type_id":0}},{"SpecialToken":{"id":"[SEP]","type_id":0}},{"Sequence":{"id":"B","type_id":1}},{"SpecialToken":{"id":"[SEP]","type_id":1}}],"special_tokens":{"[CLS]":{"id":"[CLS]","ids":[101],"tokens":["[CLS]"]},"[SEP]":{"id":"[SEP]","ids":[102],"tokens":["[SEP]"]}}},"decoder":{"type":"WordPiece","prefix":"##","cleanup":true},"model":{"unk_token":"[UNK]","continuing_subword_prefix":"##","max_input_chars_per_word":100,"vocab":{"[PAD]":0,"[unused0]":1,"[unused1]":2,"[unused2]":3,"[unused3]":4,"[unused4]":5,"[unused5]":6,"[unused6]":7,"[unused7]":8,"[unused8]":9,"[unused9]":10,"[unused10]":11,"[unused11]":12,"[unused12]":13,"[unused13]":14,"[unused14]":15,"[unused15]":16,"[unused16]":17,"[unused17]":18,"[unused18]":19,"[unused19]":20,"[unused20]":21,"[unused21]":22,"[unused22]":23,"[unused23]":24,"[unused24]":25,"[unused25]":26,"[unused26]":27,"[unused27]":28,"[unused28]":29,"[unused29]":30,"[unused30]":31,"[unused31]":32,"[unused32]":33,"[unused33]":34,"[unused34]":35,"[unused35]":36,"[unused36]":37,"[unused37]":38,"[unused38]":39,"[unused39]":40,"[unused40]":41,"[unused41]":42,"[unused42]":43,"[unused43]":44,"[unused44]":45,"[unused45]":46,"[unused46]":47,"[unused47]":48,"[unused48]":49,"[unused49]":50,"[unused50]":51,"[unused51]":52,"[unused52]":53,"[unused53]":54,"[unused54]":55,"[unused55]":56,"[unused56]":57,"[unused57]":58,"[unused58]":59,"[unused59]":60,"[unused60]":61,"[unused61]":62,"[unused62]":63,"[unused63]":64,"[unused64]":65,"[unused65]":66,"[unused66]":67,"[unused67]":68,"[unused68]":69,"[unused69]":70,"[unused70]":71,"[unused71]":72,"[unused72]":73,"[unused73]":74,"[unused74]":75,"[unused75]":76,"[unused76]":77,"[unused77]":78,"[unused78]":79,"[unused79]":80,"[unused80]":81,"[unused81]":82,"[unused82]":83,"[unused83]":84,"[unused84]":85,"[unused85]":86,"[unused86]":87,"[unused87]":88,"[unused88]":89,"[unused89]":90,"[unused90]":91,"[unused91]":92,"[unused92]":93,"[unused93]":94,"[unused94]":95,"[unused95]":96,"[unused96]":97,"[unused97]":98,"[unused98]":99,"[UNK]":100,"[CLS]":101,"[SEP]":102,"[MASK]":103,"[unused99]":104,"[unused100]":105,"[unused101]":106,"[unused102]":107,"[unused103]":108,"[unused104]":109,"[unused105]":110,"[unused106]":111,"[unused107]":112,"[unused108]":113,"[unused109]":114,"[unused110]":115,"[unused111]":116,"[unused112]":117,"[unused113]":118,"[unused114]":119,"[unused115]":120,"[unused116]":121,"[unused117]":122,"[unused118]":123,"[unused119]":124,"[unused120]":125,"[unused121]":126,"[unused122]":127,"[unused123]":128,"[unused124]":129,"[unused125]":130,"[unused126]":131,"[unused127]":132,"[unused128]":133,"[unused129]":134,"[unused130]":135,"[unused131]":136,"[unused132]":137,"[unused133]":138,"[unused134]":139,"[unused135]":140,"[unused136]":141,"[unused137]":142,"[unused138]":143,"[unused139]":144,"[unused140]":145,"[unused141]":146,"[unused142]":147,"[unused143]":148,"[unused144]":149,"[unused145]":150,"[unused146]":151,"[unused147]":152,"[unused148]":153,"[unused149]":154,"[unused150]":155,"[unused151]":156,"[unused152]":157,"[unused153]":158,"[unused154]":159,"[unused155]":160,"[unused156]":161,"[unused157]":162,"[unused158]":163,"[unused159]":164,"[unused160]":165,"[unused161]":166,"[unused162]":167,"[unused163]":168,"[unused164]":169,"[unused165]":170,"[unused166]":171,"[unused167]":172,"[unused168]":173,"[unused169]":174,"[unused170]":175,"[unused171]":176,"[unused172]":177,"[unused173]":178,"[unused174]":179,"[unused175]":180,"[unused176]":181,"[unused177]":182,"[unused178]":183,"[unused179]":184,"[unused180]":185,"[unused181]":186,"[unused182]":187,"[unused183]":188,"[unused184]":189,"[unused185]":190,"[unused186]":191,"[unused187]":192,"[unused188]":193,"[unused189]":194,"[unused190]":195,"[unused191]":196,"[unused192]":197,"[unused193]":198,"[unused194]":199,"[unused195]":200,"[unused196]":201,"[unused197]":202,"[unused198]":203,"[unused199]":204,"[unused200]":205,"[unused201]":206,"[unused202]":207,"[unused203]":208,"[unused204]":209,"[unused205]":210,"[unused206]":211,"[unused207]":212,"[unused208]":213,"[unused209]":214,"[unused210]":215,"[unused211]":216,"[unused212]":217,"[unused213]":218,"[unused214]":219,"[unused215]":220,"[unused216]":221,"[unused217]":222,"[unused218]":223,"[unused219]":224,"[unused220]":225,"[unused221]":226,"[unused222]":227,"[unused223]":228,"[unused224]":229,"[unused225]":230,"[unused226]":231,"[unused227]":232,"[unused228]":233,"[unused229]":234,"[unused230]":235,"[unused231]":236,"[unused232]":237,"[unused233]":238,"[unused234]":239,"[unused235]":240,"[unused236]":241,"[unused237]":242,"[unused238]":243,"[unused239]":244,"[unused240]":245,"[unused241]":246,"[unused242]":247,"[unused243]":248,"[unused244]":249,"[unused245]":250,"[unused246]":251,"[unused247]":252,"[unused248]":253,"[unused249]":254,"[unused250]":255,"[unused251]":256,"[unused252]":257,"[unused253]":258,"[unused254]":259,"[unused255]":260,"[unused256]":261,"[unused257]":262,"[unused258]":263,"[unused259]":264,"[unused260]":265,"[unused261]":266,"[unused262]":267,"[unused263]":268,"[unused264]":269,"[unused265]":270,"[unused266]":271,"[unused267]":272,"[unused268]":273,"[unused269]":274,"[unused270]":275,"[unused271]":276,"[unused272]":277,"[unused273]":278,"[unused274]":279,"[unused275]":280,"[unused276]":281,"[unused277]":282,"[unused278]":283,"[unused279]":284,"[unused280]":285,"[unused281]":286,"[unused282]":287,"[unused283]":288,"[unused284]":289,"[unused285]":290,"[unused286]":291,"[unused287]":292,"[unused288]":293,"[unused289]":294,"[unused290]":295,"[unused291]":296,"[unused292]":297,"[unused293]":298,"[unused294]":299,"[unused295]":300,"[unused296]":301,"[unused297]":302,"[unused298]":303,"[unused299]":304,"[unused300]":305,"[unused301]":306,"[unused302]":307,"[unused303]":308,"[unused304]":309,"[unused305]":310,"[unused306]":311,"[unused307]":312,"[unused308]":313,"[unused309]":314,"[unused310]":315,"[unused311]":316,"[unused312]":317,"[unused313]":318,"[unused314]":319,"[unused315]":320,"[unused316]":321,"[unused317]":322,"[unused318]":323,"[unused319]":324,"[unused320]":325,"[unused321]":326,"[unused322]":327,"[unused323]":328,"[unused324]":329,"[unused325]":330,"[unused326]":331,"[unused327]":332,"[unused328]":333,"[unused329]":334,"[unused330]":335,"[unused331]":336,"[unused332]":337,"[unused333]":338,"[unused334]":339,"[unused335]":340,"[unused336]":341,"[unused337]":342,"[unused338]":343,"[unused339]":344,"[unused340]":345,"[unused341]":346,"[unused342]":347,"[unused343]":348,"[unused344]":349,"[unused345]":350,"[unused346]":351,"[unused347]":352,"[unused348]":353,"[unused349]":354,"[unused350]":355,"[unused351]":356,"[unused352]":357,"[unused353]":358,"[unused354]":359,"[unused355]":360,"[unused356]":361,"[unused357]":362,"[unused358]":363,"[unused359]":364,"[unused360]":365,"[unused361]":366,"[unused362]":367,"[unused363]":368,"[unused364]":369,"[unused365]":370,"[unused366]":371,"[unused367]":372,"[unused368]":373,"[unused369]":374,"[unused370]":375,"[unused371]":376,"[unused372]":377,"[unused373]":378,"[unused374]":379,"[unused375]":380,"[unused376]":381,"[unused377]":382,"[unused378]":383,"[unused379]":384,"[unused380]":385,"[unused381]":386,"[unused382]":387,"[unused383]":388,"[unused384]":389,"[unused385]":390,"[unused386]":391,"[unused387]":392,"[unused388]":393,"[unused389]":394,"[unused390]":395,"[unused391]":396,"[unused392]":397,"[unused393]":398,"[unused394]":399,"[unused395]":400,"[unused396]":401,"[unused397]":402,"[unused398]":403,"[unused399]":404,"[unused400]":405,"[unused401]":406,"[unused402]":407,"[unused403]":408,"[unused404]":409,"[unused405]":410,"[unused406]":411,"[unused407]":412,"[unused408]":413,"[unused409]":414,"[unused410]":415,"[unused411]":416,"[unused412]":417,"[unused413]":418,"[unused414]":419,"[unused415]":420,"[unused416]":421,"[unused417]":422,"[unused418]":423,"[unused419]":424,"[unused420]":425,"[unused421]":426,"[unused422]":427,"[unused423]":428,"[unused424]":429,"[unused425]":430,"[unused426]":431,"[unused427]":432,"[unused428]":433,"[unused429]":434,"[unused430]":435,"[unused431]":436,"[unused432]":437,"[unused433]":438,"[unused434]":439,"[unused435]":440,"[unused436]":441,"[unused437]":442,"[unused438]":443,"[unused439]":444,"[unused440]":445,"[unused441]":446,"[unused442]":447,"[unused443]":448,"[unused444]":449,"[unused445]":450,"[unused446]":451,"[unused447]":452,"[unused448]":453,"[unused449]":454,"[unused450]":455,"[unused451]":456,"[unused452]":457,"[unused453]":458,"[unused454]":459,"[unused455]":460,"[unused456]":461,"[unused457]":462,"[unused458]":463,"[unused459]":464,"[unused460]":465,"[unused461]":466,"[unused462]":467,"[unused463]":468,"[unused464]":469,"[unused465]":470,"[unused466]":471,"[unused467]":472,"[unused468]":473,"[unused469]":474,"[unused470]":475,"[unused471]":476,"[unused472]":477,"[unused473]":478,"[unused474]":479,"[unused475]":480,"[unused476]":481,"[unused477]":482,"[unused478]":483,"[unused479]":484,"[unused480]":485,"[unused481]":486,"[unused482]":487,"[unused483]":488,"[unused484]":489,"[unused485]":490,"[unused486]":491,"[unused487]":492,"[unused488]":493,"[unused489]":494,"[unused490]":495,"[unused491]":496,"[unused492]":497,"[unused493]":498,"[unused494]":499,"[unused495]":500,"[unused496]":501,"[unused497]":502,"[unused498]":503,"[unused499]":504,"[unused500]":505,"[unused501]":506,"[unused502]":507,"[unused503]":508,"[unused504]":509,"[unused505]":510,"[unused506]":511,"[unused507]":512,"[unused508]":513,"[unused509]":514,"[unused510]":515,"[unused511]":516,"[unused512]":517,"[unused513]":518,"[unused514]":519,"[unused515]":520,"[unused516]":521,"[unused517]":522,"[unused518]":523,"[unused519]":524,"[unused520]":525,"[unused521]":526,"[unused522]":527,"[unused523]":528,"[unused524]":529,"[unused525]":530,"[unused526]":531,"[unused527]":532,"[unused528]":533,"[unused529]":534,"[unused530]":535,"[unused531]":536,"[unused532]":537,"[unused533]":538,"[unused534]":539,"[unused535]":540,"[unused536]":541,"[unused537]":542,"[unused538]":543,"[unused539]":544,"[unused540]":545,"[unused541]":546,"[unused542]":547,"[unused543]":548,"[unused544]":549,"[unused545]":550,"[unused546]":551,"[unused547]":552,"[unused548]":553,"[unused549]":554,"[unused550]":555,"[unused551]":556,"[unused552]":557,"[unused553]":558,"[unused554]":559,"[unused555]":560,"[unused556]":561,"[unused557]":562,"[unused558]":563,"[unused559]":564,"[unused560]":565,"[unused561]":566,"[unused562]":567,"[unused563]":568,"[unused564]":569,"[unused565]":570,"[unused566]":571,"[unused567]":572,"[unused568]":573,"[unused569]":574,"[unused570]":575,"[unused571]":576,"[unused572]":577,"[unused573]":578,"[unused574]":579,"[unused575]":580,"[unused576]":581,"[unused577]":582,"[unused578]":583,"[unused579]":584,"[unused580]":585,"[unused581]":586,"[unused582]":587,"[unused583]":588,"[unused584]":589,"[unused585]":590,"[unused586]":591,"[unused587]":592,"[unused588]":593,"[unused589]":594,"[unused590]":595,"[unused591]":596,"[unused592]":597,"[unused593]":598,"[unused594]":599,"[unused595]":600,"[unused596]":601,"[unused597]":602,"[unused598]":603,"[unused599]":604,"[unused600]":605,"[unused601]":606,"[unused602]":607,"[unused603]":608,"[unused604]":609,"[unused605]":610,"[unused606]":611,"[unused607]":612,"[unused608]":613,"[unused609]":614,"[unused610]":615,"[unused611]":616,"[unused612]":617,"[unused613]":618,"[unused614]":619,"[unused615]":620,"[unused616]":621,"[unused617]":622,"[unused618]":623,"[unused619]":624,"[unused620]":625,"[unused621]":626,"[unused622]":627,"[unused623]":628,"[unused624]":629,"[unused625]":630,"[unused626]":631,"[unused627]":632,"[unused628]":633,"[unused629]":634,"[unused630]":635,"[unused631]":636,"[unused632]":637,"[unused633]":638,"[unused634]":639,"[unused635]":640,"[unused636]":641,"[unused637]":642,"[unused638]":643,"[unused639]":644,"[unused640]":645,"[unused641]":646,"[unused642]":647,"[unused643]":648,"[unused644]":649,"[unused645]":650,"[unused646]":651,"[unused647]":652,"[unused648]":653,"[unused649]":654,"[unused650]":655,"[unused651]":656,"[unused652]":657,"[unused653]":658,"[unused654]":659,"[unused655]":660,"[unused656]":661,"[unused657]":662,"[unused658]":663,"[unused659]":664,"[unused660]":665,"[unused661]":666,"[unused662]":667,"[unused663]":668,"[unused664]":669,"[unused665]":670,"[unused666]":671,"[unused667]":672,"[unused668]":673,"[unused669]":674,"[unused670]":675,"[unused671]":676,"[unused672]":677,"[unused673]":678,"[unused674]":679,"[unused675]":680,"[unused676]":681,"[unused677]":682,"[unused678]":683,"[unused679]":684,"[unused680]":685,"[unused681]":686,"[unused682]":687,"[unused683]":688,"[unused684]":689,"[unused685]":690,"[unused686]":691,"[unused687]":692,"[unused688]":693,"[unused689]":694,"[unused690]":695,"[unused691]":696,"[unused692]":697,"[unused693]":698,"[unused694]":699,"[unused695]":700,"[unused696]":701,"[unused697]":702,"[unused698]":703,"[unused699]":704,"[unused700]":705,"[unused701]":706,"[unused702]":707,"[unused703]":708,"[unused704]":709,"[unused705]":710,"[unused706]":711,"[unused707]":712,"[unused708]":713,"[unused709]":714,"[unused710]":715,"[unused711]":716,"[unused712]":717,"[unused713]":718,"[unused714]":719,"[unused715]":720,"[unused716]":721,"[unused717]":722,"[unused718]":723,"[unused719]":724,"[unused720]":725,"[unused721]":726,"[unused722]":727,"[unused723]":728,"[unused724]":729,"[unused725]":730,"[unused726]":731,"[unused727]":732,"[unused728]":733,"[unused729]":734,"[unused730]":735,"[unused731]":736,"[unused732]":737,"[unused733]":738,"[unused734]":739,"[unused735]":740,"[unused736]":741,"[unused737]":742,"[unused738]":743,"[unused739]":744,"[unused740]":745,"[unused741]":746,"[unused742]":747,"[unused743]":748,"[unused744]":749,"[unused745]":750,"[unused746]":751,"[unused747]":752,"[unused748]":753,"[unused749]":754,"[unused750]":755,"[unused751]":756,"[unused752]":757,"[unused753]":758,"[unused754]":759,"[unused755]":760,"[unused756]":761,"[unused757]":762,"[unused758]":763,"[unused759]":764,"[unused760]":765,"[unused761]":766,"[unused762]":767,"[unused763]":768,"[unused764]":769,"[unused765]":770,"[unused766]":771,"[unused767]":772,"[unused768]":773,"[unused769]":774,"[unused770]":775,"[unused771]":776,"[unused772]":777,"[unused773]":778,"[unused774]":779,"[unused775]":780,"[unused776]":781,"[unused777]":782,"[unused778]":783,"[unused779]":784,"[unused780]":785,"[unused781]":786,"[unused782]":787,"[unused783]":788,"[unused784]":789,"[unused785]":790,"[unused786]":791,"[unused787]":792,"[unused788]":793,"[unused789]":794,"[unused790]":795,"[unused791]":796,"[unused792]":797,"[unused793]":798,"[unused794]":799,"[unused795]":800,"[unused796]":801,"[unused797]":802,"[unused798]":803,"[unused799]":804,"[unused800]":805,"[unused801]":806,"[unused802]":807,"[unused803]":808,"[unused804]":809,"[unused805]":810,"[unused806]":811,"[unused807]":812,"[unused808]":813,"[unused809]":814,"[unused810]":815,"[unused811]":816,"[unused812]":817,"[unused813]":818,"[unused814]":819,"[unused815]":820,"[unused816]":821,"[unused817]":822,"[unused818]":823,"[unused819]":824,"[unused820]":825,"[unused821]":826,"[unused822]":827,"[unused823]":828,"[unused824]":829,"[unused825]":830,"[unused826]":831,"[unused827]":832,"[unused828]":833,"[unused829]":834,"[unused830]":835,"[unused831]":836,"[unused832]":837,"[unused833]":838,"[unused834]":839,"[unused835]":840,"[unused836]":841,"[unused837]":842,"[unused838]":843,"[unused839]":844,"[unused840]":845,"[unused841]":846,"[unused842]":847,"[unused843]":848,"[unused844]":849,"[unused845]":850,"[unused846]":851,"[unused847]":852,"[unused848]":853,"[unused849]":854,"[unused850]":855,"[unused851]":856,"[unused852]":857,"[unused853]":858,"[unused854]":859,"[unused855]":860,"[unused856]":861,"[unused857]":862,"[unused858]":863,"[unused859]":864,"[unused860]":865,"[unused861]":866,"[unused862]":867,"[unused863]":868,"[unused864]":869,"[unused865]":870,"[unused866]":871,"[unused867]":872,"[unused868]":873,"[unused869]":874,"[unused870]":875,"[unused871]":876,"[unused872]":877,"[unused873]":878,"[unused874]":879,"[unused875]":880,"[unused876]":881,"[unused877]":882,"[unused878]":883,"[unused879]":884,"[unused880]":885,"[unused881]":886,"[unused882]":887,"[unused883]":888,"[unused884]":889,"[unused885]":890,"[unused886]":891,"[unused887]":892,"[unused888]":893,"[unused889]":894,"[unused890]":895,"[unused891]":896,"[unused892]":897,"[unused893]":898,"[unused894]":899,"[unused895]":900,"[unused896]":901,"[unused897]":902,"[unused898]":903,"[unused899]":904,"[unused900]":905,"[unused901]":906,"[unused902]":907,"[unused903]":908,"[unused904]":909,"[unused905]":910,"[unused906]":911,"[unused907]":912,"[unused908]":913,"[unused909]":914,"[unused910]":915,"[unused911]":916,"[unused912]":917,"[unused913]":918,"[unused914]":919,"[unused915]":920,"[unused916]":921,"[unused917]":922,"[unused918]":923,"[unused919]":924,"[unused920]":925,"[unused921]":926,"[unused922]":927,"[unused923]":928,"[unused924]":929,"[unused925]":930,"[unused926]":931,"[unused927]":932,"[unused928]":933,"[unused929]":934,"[unused930]":935,"[unused931]":936,"[unused932]":937,"[unused933]":938,"[unused934]":939,"[unused935]":940,"[unused936]":941,"[unused937]":942,"[unused938]":943,"[unused939]":944,"[unused940]":945,"[unused941]":946,"[unused942]":947,"[unused943]":948,"[unused944]":949,"[unused945]":950,"[unused946]":951,"[unused947]":952,"[unused948]":953,"[unused949]":954,"[unused950]":955,"[unused951]":956,"[unused952]":957,"[unused953]":958,"[unused954]":959,"[unused955]":960,"[unused956]":961,"[unused957]":962,"[unused958]":963,"[unused959]":964,"[unused960]":965,"[unused961]":966,"[unused962]":967,"[unused963]":968,"[unused964]":969,"[unused965]":970,"[unused966]":971,"[unused967]":972,"[unused968]":973,"[unused969]":974,"[unused970]":975,"[unused971]":976,"[unused972]":977,"[unused973]":978,"[unused974]":979,"[unused975]":980,"[unused976]":981,"[unused977]":982,"[unused978]":983,"[unused979]":984,"[unused980]":985,"[unused981]":986,"[unused982]":987,"[unused983]":988,"[unused984]":989,"[unused985]":990,"[unused986]":991,"[unused987]":992,"[unused988]":993,"[unused989]":994,"[unused990]":995,"[unused991]":996,"[unused992]":997,"[unused993]":998,"!":999,"\"":1000,"#":1001,"$":1002,"%":1003,"&":1004,"'":1005,"(":1006,")":1007,"*":1008,"+":1009,",":1010,"-":1011,".":1012,"/":1013,"0":1014,"1":1015,"2":1016,"3":1017,"4":1018,"5":1019,"6":1020,"7":1021,"8":1022,"9":1023,":":1024,";":1025,"<":1026,"=":1027,">":1028,"?":1029,"@":1030,"[":1031,"\\":1032,"]":1033,"^":1034,"_":1035,"`":1036,"a":1037,"b":1038,"c":1039,"d":1040,"e":1041,"f":1042,"g":1043,"h":1044,"i":1045,"j":1046,"k":1047,"l":1048,"m":1049,"n":1050,"o":1051,"p":1052,"q":1053,"r":1054,"s":1055,"t":1056,"u":1057,"v":1058,"w":1059,"x":1060,"y":1061,"z":1062,"{":1063,"|":1064,"}":1065,"~":1066,"¡":1067,"¢":1068,"£":1069,"¤":1070,"¥":1071,"¦":1072,"§":1073,"¨":1074,"©":1075,"ª":1076,"«":1077,"¬":1078,"®":1079,"°":1080,"±":1081,"²":1082,"³":1083,"´":1084,"µ":1085,"¶":1086,"·":1087,"¹":1088,"º":1089,"»":1090,"¼":1091,"½":1092,"¾":1093,"¿":1094,"×":1095,"ß":1096,"æ":1097,"ð":1098,"÷":1099,"ø":1100,"þ":1101,"đ":1102,"ħ":1103,"ı":1104,"ł":1105,"ŋ":1106,"œ":1107,"ƒ":1108,"ɐ":1109,"ɑ":1110,"ɒ":1111,"ɔ":1112,"ɕ":1113,"ə":1114,"ɛ":1115,"ɡ":1116,"ɣ":1117,"ɨ":1118,"ɪ":1119,"ɫ":1120,"ɬ":1121,"ɯ":1122,"ɲ":1123,"ɴ":1124,"ɹ":1125,"ɾ":1126,"ʀ":1127,"ʁ":1128,"ʂ":1129,"ʃ":1130,"ʉ":1131,"ʊ":1132,"ʋ":1133,"ʌ":1134,"ʎ":1135,"ʐ":1136,"ʑ":1137,"ʒ":1138,"ʔ":1139,"ʰ":1140,"ʲ":1141,"ʳ":1142,"ʷ":1143,"ʸ":1144,"ʻ":1145,"ʼ":1146,"ʾ":1147,"ʿ":1148,"ˈ":1149,"ː":1150,"ˡ":1151,"ˢ":1152,"ˣ":1153,"ˤ":1154,"α":1155,"β":1156,"γ":1157,"δ":1158,"ε":1159,"ζ":1160,"η":1161,"θ":1162,"ι":1163,"κ":1164,"λ":1165,"μ":1166,"ν":1167,"ξ":1168,"ο":1169,"π":1170,"ρ":1171,"ς":1172,"σ":1173,"τ":1174,"υ":1175,"φ":1176,"χ":1177,"ψ":1178,"ω":1179,"а":1180,"б":1181,"в":1182,"г":1183,"д":1184,"е":1185,"ж":1186,"з":1187,"и":1188,"к":1189,"л":1190,"м":1191,"н":1192,"о":1193,"п":1194,"р":1195,"с":1196,"т":1197,"у":1198,"ф":1199,"х":1200,"ц":1201,"ч":1202,"ш":1203,"щ":1204,"ъ":1205,"ы":1206,"ь":1207,"э":1208,"ю":1209,"я":1210,"ђ":1211,"є":1212,"і":1213,"ј":1214,"љ":1215,"њ":1216,"ћ":1217,"ӏ":1218,"ա":1219,"բ":1220,"գ":1221,"դ":1222,"ե":1223,"թ":1224,"ի":1225,"լ":1226,"կ":1227,"հ":1228,"մ":1229,"յ":1230,"ն":1231,"ո":1232,"պ":1233,"ս":1234,"վ":1235,"տ":1236,"ր":1237,"ւ":1238,"ք":1239,"־":1240,"א":1241,"ב":1242,"ג":1243,"ד":1244,"ה":1245,"ו":1246,"ז":1247,"ח":1248,"ט":1249,"י":1250,"ך":1251,"כ":1252,"ל":1253,"ם":1254,"מ":1255,"ן":1256,"נ":1257,"ס":1258,"ע":1259,"ף":1260,"פ":1261,"ץ":1262,"צ":1263,"ק":1264,"ר":1265,"ש":1266,"ת":1267,"،":1268,"ء":1269,"ا":1270,"ب":1271,"ة":1272,"ت":1273,"ث":1274,"ج":1275,"ح":1276,"خ":1277,"د":1278,"ذ":1279,"ر":1280,"ز":1281,"س":1282,"ش":1283,"ص":1284,"ض":1285,"ط":1286,"ظ":1287,"ع":1288,"غ":1289,"ـ":1290,"ف":1291,"ق":1292,"ك":1293,"ل":1294,"م":1295,"ن":1296,"ه":1297,"و":1298,"ى":1299,"ي":1300,"ٹ":1301,"پ":1302,"چ":1303,"ک":1304,"گ":1305,"ں":1306,"ھ":1307,"ہ":1308,"ی":1309,"ے":1310,"अ":1311,"आ":1312,"उ":1313,"ए":1314,"क":1315,"ख":1316,"ग":1317,"च":1318,"ज":1319,"ट":1320,"ड":1321,"ण":1322,"त":1323,"थ":1324,"द":1325,"ध":1326,"न":1327,"प":1328,"ब":1329,"भ":1330,"म":1331,"य":1332,"र":1333,"ल":1334,"व":1335,"श":1336,"ष":1337,"स":1338,"ह":1339,"ा":1340,"ि":1341,"ी":1342,"ो":1343,"।":1344,"॥":1345,"ং":1346,"অ":1347,"আ":1348,"ই":1349,"উ":1350,"এ":1351,"ও":1352,"ক":1353,"খ":1354,"গ":1355,"চ":1356,"ছ":1357,"জ":1358,"ট":1359,"ড":1360,"ণ":1361,"ত":1362,"থ":1363,"দ":1364,"ধ":1365,"ন":1366,"প":1367,"ব":1368,"ভ":1369,"ম":1370,"য":1371,"র":1372,"ল":1373,"শ":1374,"ষ":1375,"স":1376,"হ":1377,"া":1378,"ি":1379,"ী":1380,"ে":1381,"க":1382,"ச":1383,"ட":1384,"த":1385,"ந":1386,"ன":1387,"ப":1388,"ம":1389,"ய":1390,"ர":1391,"ல":1392,"ள":1393,"வ":1394,"ா":1395,"ி":1396,"ு":1397,"ே":1398,"ை":1399,"ನ":1400,"ರ":1401,"ಾ":1402,"ක":1403,"ය":1404,"ර":1405,"ල":1406,"ව":1407,"ා":1408,"ก":1409,"ง":1410,"ต":1411,"ท":1412,"น":1413,"พ":1414,"ม":1415,"ย":1416,"ร":1417,"ล":1418,"ว":1419,"ส":1420,"อ":1421,"า":1422,"เ":1423,"་":1424,"།":1425,"ག":1426,"ང":1427,"ད":1428,"ན":1429,"པ":1430,"བ":1431,"མ":1432,"འ":1433,"ར":1434,"ལ":1435,"ས":1436,"မ":1437,"ა":1438,"ბ":1439,"გ":1440,"დ":1441,"ე":1442,"ვ":1443,"თ":1444,"ი":1445,"კ":1446,"ლ":1447,"მ":1448,"ნ":1449,"ო":1450,"რ":1451,"ს":1452,"ტ":1453,"უ":1454,"ᄀ":1455,"ᄂ":1456,"ᄃ":1457,"ᄅ":1458,"ᄆ":1459,"ᄇ":1460,"ᄉ":1461,"ᄊ":1462,"ᄋ":1463,"ᄌ":1464,"ᄎ":1465,"ᄏ":1466,"ᄐ":1467,"ᄑ":1468,"ᄒ":1469,"ᅡ":1470,"ᅢ":1471,"ᅥ":1472,"ᅦ":1473,"ᅧ":1474,"ᅩ":1475,"ᅪ":1476,"ᅭ":1477,"ᅮ":1478,"ᅯ":1479,"ᅲ":1480,"ᅳ":1481,"ᅴ":1482,"ᅵ":1483,"ᆨ":1484,"ᆫ":1485,"ᆯ":1486,"ᆷ":1487,"ᆸ":1488,"ᆼ":1489,"ᴬ":1490,"ᴮ":1491,"ᴰ":1492,"ᴵ":1493,"ᴺ":1494,"ᵀ":1495,"ᵃ":1496,"ᵇ":1497,"ᵈ":1498,"ᵉ":1499,"ᵍ":1500,"ᵏ":1501,"ᵐ":1502,"ᵒ":1503,"ᵖ":1504,"ᵗ":1505,"ᵘ":1506,"ᵢ":1507,"ᵣ":1508,"ᵤ":1509,"ᵥ":1510,"ᶜ":1511,"ᶠ":1512,"‐":1513,"‑":1514,"‒":1515,"–":1516,"—":1517,"―":1518,"‖":1519,"‘":1520,"’":1521,"‚":1522,"“":1523,"”":1524,"„":1525,"†":1526,"‡":1527,"•":1528,"…":1529,"‰":1530,"′":1531,"″":1532,"›":1533,"‿":1534,"⁄":1535,"⁰":1536,"ⁱ":1537,"⁴":1538,"⁵":1539,"⁶":1540,"⁷":1541,"⁸":1542,"⁹":1543,"⁺":1544,"⁻":1545,"ⁿ":1546,"₀":1547,"₁":1548,"₂":1549,"₃":1550,"₄":1551,"₅":1552,"₆":1553,"₇":1554,"₈":1555,"₉":1556,"₊":1557,"₍":1558,"₎":1559,"ₐ":1560,"ₑ":1561,"ₒ":1562,"ₓ":1563,"ₕ":1564,"ₖ":1565,"ₗ":1566,"ₘ":1567,"ₙ":1568,"ₚ":1569,"ₛ":1570,"ₜ":1571,"₤":1572,"₩":1573,"€":1574,"₱":1575,"₹":1576,"ℓ":1577,"№":1578,"ℝ":1579,"™":1580,"⅓":1581,"⅔":1582,"←":1583,"↑":1584,"→":1585,"↓":1586,"↔":1587,"↦":1588,"⇄":1589,"⇌":1590,"⇒":1591,"∂":1592,"∅":1593,"∆":1594,"∇":1595,"∈":1596,"−":1597,"∗":1598,"∘":1599,"√":1600,"∞":1601,"∧":1602,"∨":1603,"∩":1604,"∪":1605,"≈":1606,"≡":1607,"≤":1608,"≥":1609,"⊂":1610,"⊆":1611,"⊕":1612,"⊗":1613,"⋅":1614,"─":1615,"│":1616,"■":1617,"▪":1618,"●":1619,"★":1620,"☆":1621,"☉":1622,"♠":1623,"♣":1624,"♥":1625,"♦":1626,"♭":1627,"♯":1628,"⟨":1629,"⟩":1630,"ⱼ":1631,"⺩":1632,"⺼":1633,"⽥":1634,"、":1635,"。":1636,"〈":1637,"〉":1638,"《":1639,"》":1640,"「":1641,"」":1642,"『":1643,"』":1644,"〜":1645,"あ":1646,"い":1647,"う":1648,"え":1649,"お":1650,"か":1651,"き":1652,"く":1653,"け":1654,"こ":1655,"さ":1656,"し":1657,"す":1658,"せ":1659,"そ":1660,"た":1661,"ち":1662,"っ":1663,"つ":1664,"て":1665,"と":1666,"な":1667,"に":1668,"ぬ":1669,"ね":1670,"の":1671,"は":1672,"ひ":1673,"ふ":1674,"へ":1675,"ほ":1676,"ま":1677,"み":1678,"む":1679,"め":1680,"も":1681,"や":1682,"ゆ":1683,"よ":1684,"ら":1685,"り":1686,"る":1687,"れ":1688,"ろ":1689,"を":1690,"ん":1691,"ァ":1692,"ア":1693,"ィ":1694,"イ":1695,"ウ":1696,"ェ":1697,"エ":1698,"オ":1699,"カ":1700,"キ":1701,"ク":1702,"ケ":1703,"コ":1704,"サ":1705,"シ":1706,"ス":1707,"セ":1708,"タ":1709,"チ":1710,"ッ":1711,"ツ":1712,"テ":1713,"ト":1714,"ナ":1715,"ニ":1716,"ノ":1717,"ハ":1718,"ヒ":1719,"フ":1720,"ヘ":1721,"ホ":1722,"マ":1723,"ミ":1724,"ム":1725,"メ":1726,"モ":1727,"ャ":1728,"ュ":1729,"ョ":1730,"ラ":1731,"リ":1732,"ル":1733,"レ":1734,"ロ":1735,"ワ":1736,"ン":1737,"・":1738,"ー":1739,"一":1740,"三":1741,"上":1742,"下":1743,"不":1744,"世":1745,"中":1746,"主":1747,"久":1748,"之":1749,"也":1750,"事":1751,"二":1752,"五":1753,"井":1754,"京":1755,"人":1756,"亻":1757,"仁":1758,"介":1759,"代":1760,"仮":1761,"伊":1762,"会":1763,"佐":1764,"侍":1765,"保":1766,"信":1767,"健":1768,"元":1769,"光":1770,"八":1771,"公":1772,"内":1773,"出":1774,"分":1775,"前":1776,"劉":1777,"力":1778,"加":1779,"勝":1780,"北":1781,"区":1782,"十":1783,"千":1784,"南":1785,"博":1786,"原":1787,"口":1788,"古":1789,"史":1790,"司":1791,"合":1792,"吉":1793,"同":1794,"名":1795,"和":1796,"囗":1797,"四":1798,"国":1799,"國":1800,"土":1801,"地":1802,"坂":1803,"城":1804,"堂":1805,"場":1806,"士":1807,"夏":1808,"外":1809,"大":1810,"天":1811,"太":1812,"夫":1813,"奈":1814,"女":1815,"子":1816,"学":1817,"宀":1818,"宇":1819,"安":1820,"宗":1821,"定":1822,"宣":1823,"宮":1824,"家":1825,"宿":1826,"寺":1827,"將":1828,"小":1829,"尚":1830,"山":1831,"岡":1832,"島":1833,"崎":1834,"川":1835,"州":1836,"巿":1837,"帝":1838,"平":1839,"年":1840,"幸":1841,"广":1842,"弘":1843,"張":1844,"彳":1845,"後":1846,"御":1847,"德":1848,"心":1849,"忄":1850,"志":1851,"忠":1852,"愛":1853,"成":1854,"我":1855,"戦":1856,"戸":1857,"手":1858,"扌":1859,"政":1860,"文":1861,"新":1862,"方":1863,"日":1864,"明":1865,"星":1866,"春":1867,"昭":1868,"智":1869,"曲":1870,"書":1871,"月":1872,"有":1873,"朝":1874,"木":1875,"本":1876,"李":1877,"村":1878,"東":1879,"松":1880,"林":1881,"森":1882,"楊":1883,"樹":1884,"橋":1885,"歌":1886,"止":1887,"正":1888,"武":1889,"比":1890,"氏":1891,"民":1892,"水":1893,"氵":1894,"氷":1895,"永":1896,"江":1897,"沢":1898,"河":1899,"治":1900,"法":1901,"海":1902,"清":1903,"漢":1904,"瀬":1905,"火":1906,"版":1907,"犬":1908,"王":1909,"生":1910,"田":1911,"男":1912,"疒":1913,"発":1914,"白":1915,"的":1916,"皇":1917,"目":1918,"相":1919,"省":1920,"真":1921,"石":1922,"示":1923,"社":1924,"神":1925,"福":1926,"禾":1927,"秀":1928,"秋":1929,"空":1930,"立":1931,"章":1932,"竹":1933,"糹":1934,"美":1935,"義":1936,"耳":1937,"良":1938,"艹":1939,"花":1940,"英":1941,"華":1942,"葉":1943,"藤":1944,"行":1945,"街":1946,"西":1947,"見":1948,"訁":1949,"語":1950,"谷":1951,"貝":1952,"貴":1953,"車":1954,"軍":1955,"辶":1956,"道":1957,"郎":1958,"郡":1959,"部":1960,"都":1961,"里":1962,"野":1963,"金":1964,"鈴":1965,"镇":1966,"長":1967,"門":1968,"間":1969,"阝":1970,"阿":1971,"陳":1972,"陽":1973,"雄":1974,"青":1975,"面":1976,"風":1977,"食":1978,"香":1979,"馬":1980,"高":1981,"龍":1982,"龸":1983,"fi":1984,"fl":1985,"！":1986,"（":1987,"）":1988,"，":1989,"－":1990,"．":1991,"／":1992,"：":1993,"？":1994,"～":1995,"the":1996,"of":1997,"and":1998,"in":1999,"to":2000,"was":2001,"he":2002,"is":2003,"as":2004,"for":2005,"on":2006,"with":2007,"that":2008,"it":2009,"his":2010,"by":2011,"at":2012,"from":2013,"her":2014,"##s":2015,"she":2016,"you":2017,"had":2018,"an":2019,"were":2020,"but":2021,"be":2022,"this":2023,"are":2024,"not":2025,"my":2026,"they":2027,"one":2028,"which":2029,"or":2030,"have":2031,"him":2032,"me":2033,"first":2034,"all":2035,"also":2036,"their":2037,"has":2038,"up":2039,"who":2040,"out":2041,"been":2042,"when":2043,"after":2044,"there":2045,"into":2046,"new":2047,"two":2048,"its":2049,"##a":2050,"time":2051,"would":2052,"no":2053,"what":2054,"about":2055,"said":2056,"we":2057,"over":2058,"then":2059,"other":2060,"so":2061,"more":2062,"##e":2063,"can":2064,"if":2065,"like":2066,"back":2067,"them":2068,"only":2069,"some":2070,"could":2071,"##i":2072,"where":2073,"just":2074,"##ing":2075,"during":2076,"before":2077,"##n":2078,"do":2079,"##o":2080,"made":2081,"school":2082,"through":2083,"than":2084,"now":2085,"years":2086,"most":2087,"world":2088,"may":2089,"between":2090,"down":2091,"well":2092,"three":2093,"##d":2094,"year":2095,"while":2096,"will":2097,"##ed":2098,"##r":2099,"##y":2100,"later":2101,"##t":2102,"city":2103,"under":2104,"around":2105,"did":2106,"such":2107,"being":2108,"used":2109,"state":2110,"people":2111,"part":2112,"know":2113,"against":2114,"your":2115,"many":2116,"second":2117,"university":2118,"both":2119,"national":2120,"##er":2121,"these":2122,"don":2123,"known":2124,"off":2125,"way":2126,"until":2127,"re":2128,"how":2129,"even":2130,"get":2131,"head":2132,"...":2133,"didn":2134,"##ly":2135,"team":2136,"american":2137,"because":2138,"de":2139,"##l":2140,"born":2141,"united":2142,"film":2143,"since":2144,"still":2145,"long":2146,"work":2147,"south":2148,"us":2149,"became":2150,"any":2151,"high":2152,"again":2153,"day":2154,"family":2155,"see":2156,"right":2157,"man":2158,"eyes":2159,"house":2160,"season":2161,"war":2162,"states":2163,"including":2164,"took":2165,"life":2166,"north":2167,"same":2168,"each":2169,"called":2170,"name":2171,"much":2172,"place":2173,"however":2174,"go":2175,"four":2176,"group":2177,"another":2178,"found":2179,"won":2180,"area":2181,"here":2182,"going":2183,"10":2184,"away":2185,"series":2186,"left":2187,"home":2188,"music":2189,"best":2190,"make":2191,"hand":2192,"number":2193,"company":2194,"several":2195,"never":2196,"last":2197,"john":2198,"000":2199,"very":2200,"album":2201,"take":2202,"end":2203,"good":2204,"too":2205,"following":2206,"released":2207,"game":2208,"played":2209,"little":2210,"began":2211,"district":2212,"##m":2213,"old":2214,"want":2215,"those":2216,"side":2217,"held":2218,"own":2219,"early":2220,"county":2221,"ll":2222,"league":2223,"use":2224,"west":2225,"##u":2226,"face":2227,"think":2228,"##es":2229,"2010":2230,"government":2231,"##h":2232,"march":2233,"came":2234,"small":2235,"general":2236,"town":2237,"june":2238,"##on":2239,"line":2240,"based":2241,"something":2242,"##k":2243,"september":2244,"thought":2245,"looked":2246,"along":2247,"international":2248,"2011":2249,"air":2250,"july":2251,"club":2252,"went":2253,"january":2254,"october":2255,"our":2256,"august":2257,"april":2258,"york":2259,"12":2260,"few":2261,"2012":2262,"2008":2263,"east":2264,"show":2265,"member":2266,"college":2267,"2009":2268,"father":2269,"public":2270,"##us":2271,"come":2272,"men":2273,"five":2274,"set":2275,"station":2276,"church":2277,"##c":2278,"next":2279,"former":2280,"november":2281,"room":2282,"party":2283,"located":2284,"december":2285,"2013":2286,"age":2287,"got":2288,"2007":2289,"##g":2290,"system":2291,"let":2292,"love":2293,"2006":2294,"though":2295,"every":2296,"2014":2297,"look":2298,"song":2299,"water":2300,"century":2301,"without":2302,"body":2303,"black":2304,"night":2305,"within":2306,"great":2307,"women":2308,"single":2309,"ve":2310,"building":2311,"large":2312,"population":2313,"river":2314,"named":2315,"band":2316,"white":2317,"started":2318,"##an":2319,"once":2320,"15":2321,"20":2322,"should":2323,"18":2324,"2015":2325,"service":2326,"top":2327,"built":2328,"british":2329,"open":2330,"death":2331,"king":2332,"moved":2333,"local":2334,"times":2335,"children":2336,"february":2337,"book":2338,"why":2339,"11":2340,"door":2341,"need":2342,"president":2343,"order":2344,"final":2345,"road":2346,"wasn":2347,"although":2348,"due":2349,"major":2350,"died":2351,"village":2352,"third":2353,"knew":2354,"2016":2355,"asked":2356,"turned":2357,"st":2358,"wanted":2359,"say":2360,"##p":2361,"together":2362,"received":2363,"main":2364,"son":2365,"served":2366,"different":2367,"##en":2368,"behind":2369,"himself":2370,"felt":2371,"members":2372,"power":2373,"football":2374,"law":2375,"voice":2376,"play":2377,"##in":2378,"near":2379,"park":2380,"history":2381,"30":2382,"having":2383,"2005":2384,"16":2385,"##man":2386,"saw":2387,"mother":2388,"##al":2389,"army":2390,"point":2391,"front":2392,"help":2393,"english":2394,"street":2395,"art":2396,"late":2397,"hands":2398,"games":2399,"award":2400,"##ia":2401,"young":2402,"14":2403,"put":2404,"published":2405,"country":2406,"division":2407,"across":2408,"told":2409,"13":2410,"often":2411,"ever":2412,"french":2413,"london":2414,"center":2415,"six":2416,"red":2417,"2017":2418,"led":2419,"days":2420,"include":2421,"light":2422,"25":2423,"find":2424,"tell":2425,"among":2426,"species":2427,"really":2428,"according":2429,"central":2430,"half":2431,"2004":2432,"form":2433,"original":2434,"gave":2435,"office":2436,"making":2437,"enough":2438,"lost":2439,"full":2440,"opened":2441,"must":2442,"included":2443,"live":2444,"given":2445,"german":2446,"player":2447,"run":2448,"business":2449,"woman":2450,"community":2451,"cup":2452,"might":2453,"million":2454,"land":2455,"2000":2456,"court":2457,"development":2458,"17":2459,"short":2460,"round":2461,"ii":2462,"km":2463,"seen":2464,"class":2465,"story":2466,"always":2467,"become":2468,"sure":2469,"research":2470,"almost":2471,"director":2472,"council":2473,"la":2474,"##2":2475,"career":2476,"things":2477,"using":2478,"island":2479,"##z":2480,"couldn":2481,"car":2482,"##is":2483,"24":2484,"close":2485,"force":2486,"##1":2487,"better":2488,"free":2489,"support":2490,"control":2491,"field":2492,"students":2493,"2003":2494,"education":2495,"married":2496,"##b":2497,"nothing":2498,"worked":2499,"others":2500,"record":2501,"big":2502,"inside":2503,"level":2504,"anything":2505,"continued":2506,"give":2507,"james":2508,"##3":2509,"military":2510,"established":2511,"non":2512,"returned":2513,"feel":2514,"does":2515,"title":2516,"written":2517,"thing":2518,"feet":2519,"william":2520,"far":2521,"co":2522,"association":2523,"hard":2524,"already":2525,"2002":2526,"##ra":2527,"championship":2528,"human":2529,"western":2530,"100":2531,"##na":2532,"department":2533,"hall":2534,"role":2535,"various":2536,"production":2537,"21":2538,"19":2539,"heart":2540,"2001":2541,"living":2542,"fire":2543,"version":2544,"##ers":2545,"##f":2546,"television":2547,"royal":2548,"##4":2549,"produced":2550,"working":2551,"act":2552,"case":2553,"society":2554,"region":2555,"present":2556,"radio":2557,"period":2558,"looking":2559,"least":2560,"total":2561,"keep":2562,"england":2563,"wife":2564,"program":2565,"per":2566,"brother":2567,"mind":2568,"special":2569,"22":2570,"##le":2571,"am":2572,"works":2573,"soon":2574,"##6":2575,"political":2576,"george":2577,"services":2578,"taken":2579,"created":2580,"##7":2581,"further":2582,"able":2583,"reached":2584,"david":2585,"union":2586,"joined":2587,"upon":2588,"done":2589,"important":2590,"social":2591,"information":2592,"either":2593,"##ic":2594,"##x":2595,"appeared":2596,"position":2597,"ground":2598,"lead":2599,"rock":2600,"dark":2601,"election":2602,"23":2603,"board":2604,"france":2605,"hair":2606,"course":2607,"arms":2608,"site":2609,"police":2610,"girl":2611,"instead":2612,"real":2613,"sound":2614,"##v":2615,"words":2616,"moment":2617,"##te":2618,"someone":2619,"##8":2620,"summer":2621,"project":2622,"announced":2623,"san":2624,"less":2625,"wrote":2626,"past":2627,"followed":2628,"##5":2629,"blue":2630,"founded":2631,"al":2632,"finally":2633,"india":2634,"taking":2635,"records":2636,"america":2637,"##ne":2638,"1999":2639,"design":2640,"considered":2641,"northern":2642,"god":2643,"stop":2644,"battle":2645,"toward":2646,"european":2647,"outside":2648,"described":2649,"track":2650,"today":2651,"playing":2652,"language":2653,"28":2654,"call":2655,"26":2656,"heard":2657,"professional":2658,"low":2659,"australia":2660,"miles":2661,"california":2662,"win":2663,"yet":2664,"green":2665,"##ie":2666,"trying":2667,"blood":2668,"##ton":2669,"southern":2670,"science":2671,"maybe":2672,"everything":2673,"match":2674,"square":2675,"27":2676,"mouth":2677,"video":2678,"race":2679,"recorded":2680,"leave":2681,"above":2682,"##9":2683,"daughter":2684,"points":2685,"space":2686,"1998":2687,"museum":2688,"change":2689,"middle":2690,"common":2691,"##0":2692,"move":2693,"tv":2694,"post":2695,"##ta":2696,"lake":2697,"seven":2698,"tried":2699,"elected":2700,"closed":2701,"ten":2702,"paul":2703,"minister":2704,"##th":2705,"months":2706,"start":2707,"chief":2708,"return":2709,"canada":2710,"person":2711,"sea":2712,"release":2713,"similar":2714,"modern":2715,"brought":2716,"rest":2717,"hit":2718,"formed":2719,"mr":2720,"##la":2721,"1997":2722,"floor":2723,"event":2724,"doing":2725,"thomas":2726,"1996":2727,"robert":2728,"care":2729,"killed":2730,"training":2731,"star":2732,"week":2733,"needed":2734,"turn":2735,"finished":2736,"railway":2737,"rather":2738,"news":2739,"health":2740,"sent":2741,"example":2742,"ran":2743,"term":2744,"michael":2745,"coming":2746,"currently":2747,"yes":2748,"forces":2749,"despite":2750,"gold":2751,"areas":2752,"50":2753,"stage":2754,"fact":2755,"29":2756,"dead":2757,"says":2758,"popular":2759,"2018":2760,"originally":2761,"germany":2762,"probably":2763,"developed":2764,"result":2765,"pulled":2766,"friend":2767,"stood":2768,"money":2769,"running":2770,"mi":2771,"signed":2772,"word":2773,"songs":2774,"child":2775,"eventually":2776,"met":2777,"tour":2778,"average":2779,"teams":2780,"minutes":2781,"festival":2782,"current":2783,"deep":2784,"kind":2785,"1995":2786,"decided":2787,"usually":2788,"eastern":2789,"seemed":2790,"##ness":2791,"episode":2792,"bed":2793,"added":2794,"table":2795,"indian":2796,"private":2797,"charles":2798,"route":2799,"available":2800,"idea":2801,"throughout":2802,"centre":2803,"addition":2804,"appointed":2805,"style":2806,"1994":2807,"books":2808,"eight":2809,"construction":2810,"press":2811,"mean":2812,"wall":2813,"friends":2814,"remained":2815,"schools":2816,"study":2817,"##ch":2818,"##um":2819,"institute":2820,"oh":2821,"chinese":2822,"sometimes":2823,"events":2824,"possible":2825,"1992":2826,"australian":2827,"type":2828,"brown":2829,"forward":2830,"talk":2831,"process":2832,"food":2833,"debut":2834,"seat":2835,"performance":2836,"committee":2837,"features":2838,"character":2839,"arts":2840,"herself":2841,"else":2842,"lot":2843,"strong":2844,"russian":2845,"range":2846,"hours":2847,"peter":2848,"arm":2849,"##da":2850,"morning":2851,"dr":2852,"sold":2853,"##ry":2854,"quickly":2855,"directed":2856,"1993":2857,"guitar":2858,"china":2859,"##w":2860,"31":2861,"list":2862,"##ma":2863,"performed":2864,"media":2865,"uk":2866,"players":2867,"smile":2868,"##rs":2869,"myself":2870,"40":2871,"placed":2872,"coach":2873,"province":2874,"towards":2875,"wouldn":2876,"leading":2877,"whole":2878,"boy":2879,"official":2880,"designed":2881,"grand":2882,"census":2883,"##el":2884,"europe":2885,"attack":2886,"japanese":2887,"henry":2888,"1991":2889,"##re":2890,"##os":2891,"cross":2892,"getting":2893,"alone":2894,"action":2895,"lower":2896,"network":2897,"wide":2898,"washington":2899,"japan":2900,"1990":2901,"hospital":2902,"believe":2903,"changed":2904,"sister":2905,"##ar":2906,"hold":2907,"gone":2908,"sir":2909,"hadn":2910,"ship":2911,"##ka":2912,"studies":2913,"academy":2914,"shot":2915,"rights":2916,"below":2917,"base":2918,"bad":2919,"involved":2920,"kept":2921,"largest":2922,"##ist":2923,"bank":2924,"future":2925,"especially":2926,"beginning":2927,"mark":2928,"movement":2929,"section":2930,"female":2931,"magazine":2932,"plan":2933,"professor":2934,"lord":2935,"longer":2936,"##ian":2937,"sat":2938,"walked":2939,"hill":2940,"actually":2941,"civil":2942,"energy":2943,"model":2944,"families":2945,"size":2946,"thus":2947,"aircraft":2948,"completed":2949,"includes":2950,"data":2951,"captain":2952,"##or":2953,"fight":2954,"vocals":2955,"featured":2956,"richard":2957,"bridge":2958,"fourth":2959,"1989":2960,"officer":2961,"stone":2962,"hear":2963,"##ism":2964,"means":2965,"medical":2966,"groups":2967,"management":2968,"self":2969,"lips":2970,"competition":2971,"entire":2972,"lived":2973,"technology":2974,"leaving":2975,"federal":2976,"tournament":2977,"bit":2978,"passed":2979,"hot":2980,"independent":2981,"awards":2982,"kingdom":2983,"mary":2984,"spent":2985,"fine":2986,"doesn":2987,"reported":2988,"##ling":2989,"jack":2990,"fall":2991,"raised":2992,"itself":2993,"stay":2994,"true":2995,"studio":2996,"1988":2997,"sports":2998,"replaced":2999,"paris":3000,"systems":3001,"saint":3002,"leader":3003,"theatre":3004,"whose":3005,"market":3006,"capital":3007,"parents":3008,"spanish":3009,"canadian":3010,"earth":3011,"##ity":3012,"cut":3013,"degree":3014,"writing":3015,"bay":3016,"christian":3017,"awarded":3018,"natural":3019,"higher":3020,"bill":3021,"##as":3022,"coast":3023,"provided":3024,"previous":3025,"senior":3026,"ft":3027,"valley":3028,"organization":3029,"stopped":3030,"onto":3031,"countries":3032,"parts":3033,"conference":3034,"queen":3035,"security":3036,"interest":3037,"saying":3038,"allowed":3039,"master":3040,"earlier":3041,"phone":3042,"matter":3043,"smith":3044,"winning":3045,"try":3046,"happened":3047,"moving":3048,"campaign":3049,"los":3050,"##ley":3051,"breath":3052,"nearly":3053,"mid":3054,"1987":3055,"certain":3056,"girls":3057,"date":3058,"italian":3059,"african":3060,"standing":3061,"fell":3062,"artist":3063,"##ted":3064,"shows":3065,"deal":3066,"mine":3067,"industry":3068,"1986":3069,"##ng":3070,"everyone":3071,"republic":3072,"provide":3073,"collection":3074,"library":3075,"student":3076,"##ville":3077,"primary":3078,"owned":3079,"older":3080,"via":3081,"heavy":3082,"1st":3083,"makes":3084,"##able":3085,"attention":3086,"anyone":3087,"africa":3088,"##ri":3089,"stated":3090,"length":3091,"ended":3092,"fingers":3093,"command":3094,"staff":3095,"skin":3096,"foreign":3097,"opening":3098,"governor":3099,"okay":3100,"medal":3101,"kill":3102,"sun":3103,"cover":3104,"job":3105,"1985":3106,"introduced":3107,"chest":3108,"hell":3109,"feeling":3110,"##ies":3111,"success":3112,"meet":3113,"reason":3114,"standard":3115,"meeting":3116,"novel":3117,"1984":3118,"trade":3119,"source":3120,"buildings":3121,"##land":3122,"rose":3123,"guy":3124,"goal":3125,"##ur":3126,"chapter":3127,"native":3128,"husband":3129,"previously":3130,"unit":3131,"limited":3132,"entered":3133,"weeks":3134,"producer":3135,"operations":3136,"mountain":3137,"takes":3138,"covered":3139,"forced":3140,"related":3141,"roman":3142,"complete":3143,"successful":3144,"key":3145,"texas":3146,"cold":3147,"##ya":3148,"channel":3149,"1980":3150,"traditional":3151,"films":3152,"dance":3153,"clear":3154,"approximately":3155,"500":3156,"nine":3157,"van":3158,"prince":3159,"question":3160,"active":3161,"tracks":3162,"ireland":3163,"regional":3164,"silver":3165,"author":3166,"personal":3167,"sense":3168,"operation":3169,"##ine":3170,"economic":3171,"1983":3172,"holding":3173,"twenty":3174,"isbn":3175,"additional":3176,"speed":3177,"hour":3178,"edition":3179,"regular":3180,"historic":3181,"places":3182,"whom":3183,"shook":3184,"movie":3185,"km²":3186,"secretary":3187,"prior":3188,"report":3189,"chicago":3190,"read":3191,"foundation":3192,"view":3193,"engine":3194,"scored":3195,"1982":3196,"units":3197,"ask":3198,"airport":3199,"property":3200,"ready":3201,"immediately":3202,"lady":3203,"month":3204,"listed":3205,"contract":3206,"##de":3207,"manager":3208,"themselves":3209,"lines":3210,"##ki":3211,"navy":3212,"writer":3213,"meant":3214,"##ts":3215,"runs":3216,"##ro":3217,"practice":3218,"championships":3219,"singer":3220,"glass":3221,"commission":3222,"required":3223,"forest":3224,"starting":3225,"culture":3226,"generally":3227,"giving":3228,"access":3229,"attended":3230,"test":3231,"couple":3232,"stand":3233,"catholic":3234,"martin":3235,"caught":3236,"executive":3237,"##less":3238,"eye":3239,"##ey":3240,"thinking":3241,"chair":3242,"quite":3243,"shoulder":3244,"1979":3245,"hope":3246,"decision":3247,"plays":3248,"defeated":3249,"municipality":3250,"whether":3251,"structure":3252,"offered":3253,"slowly":3254,"pain":3255,"ice":3256,"direction":3257,"##ion":3258,"paper":3259,"mission":3260,"1981":3261,"mostly":3262,"200":3263,"noted":3264,"individual":3265,"managed":3266,"nature":3267,"lives":3268,"plant":3269,"##ha":3270,"helped":3271,"except":3272,"studied":3273,"computer":3274,"figure":3275,"relationship":3276,"issue":3277,"significant":3278,"loss":3279,"die":3280,"smiled":3281,"gun":3282,"ago":3283,"highest":3284,"1972":3285,"##am":3286,"male":3287,"bring":3288,"goals":3289,"mexico":3290,"problem":3291,"distance":3292,"commercial":3293,"completely":3294,"location":3295,"annual":3296,"famous":3297,"drive":3298,"1976":3299,"neck":3300,"1978":3301,"surface":3302,"caused":3303,"italy":3304,"understand":3305,"greek":3306,"highway":3307,"wrong":3308,"hotel":3309,"comes":3310,"appearance":3311,"joseph":3312,"double":3313,"issues":3314,"musical":3315,"companies":3316,"castle":3317,"income":3318,"review":3319,"assembly":3320,"bass":3321,"initially":3322,"parliament":3323,"artists":3324,"experience":3325,"1974":3326,"particular":3327,"walk":3328,"foot":3329,"engineering":3330,"talking":3331,"window":3332,"dropped":3333,"##ter":3334,"miss":3335,"baby":3336,"boys":3337,"break":3338,"1975":3339,"stars":3340,"edge":3341,"remember":3342,"policy":3343,"carried":3344,"train":3345,"stadium":3346,"bar":3347,"sex":3348,"angeles":3349,"evidence":3350,"##ge":3351,"becoming":3352,"assistant":3353,"soviet":3354,"1977":3355,"upper":3356,"step":3357,"wing":3358,"1970":3359,"youth":3360,"financial":3361,"reach":3362,"##ll":3363,"actor":3364,"numerous":3365,"##se":3366,"##st":3367,"nodded":3368,"arrived":3369,"##ation":3370,"minute":3371,"##nt":3372,"believed":3373,"sorry":3374,"complex":3375,"beautiful":3376,"victory":3377,"associated":3378,"temple":3379,"1968":3380,"1973":3381,"chance":3382,"perhaps":3383,"metal":3384,"##son":3385,"1945":3386,"bishop":3387,"##et":3388,"lee":3389,"launched":3390,"particularly":3391,"tree":3392,"le":3393,"retired":3394,"subject":3395,"prize":3396,"contains":3397,"yeah":3398,"theory":3399,"empire":3400,"##ce":3401,"suddenly":3402,"waiting":3403,"trust":3404,"recording":3405,"##to":3406,"happy":3407,"terms":3408,"camp":3409,"champion":3410,"1971":3411,"religious":3412,"pass":3413,"zealand":3414,"names":3415,"2nd":3416,"port":3417,"ancient":3418,"tom":3419,"corner":3420,"represented":3421,"watch":3422,"legal":3423,"anti":3424,"justice":3425,"cause":3426,"watched":3427,"brothers":3428,"45":3429,"material":3430,"changes":3431,"simply":3432,"response":3433,"louis":3434,"fast":3435,"##ting":3436,"answer":3437,"60":3438,"historical":3439,"1969":3440,"stories":3441,"straight":3442,"create":3443,"feature":3444,"increased":3445,"rate":3446,"administration":3447,"virginia":3448,"el":3449,"activities":3450,"cultural":3451,"overall":3452,"winner":3453,"programs":3454,"basketball":3455,"legs":3456,"guard":3457,"beyond":3458,"cast":3459,"doctor":3460,"mm":3461,"flight":3462,"results":3463,"remains":3464,"cost":3465,"effect":3466,"winter":3467,"##ble":3468,"larger":3469,"islands":3470,"problems":3471,"chairman":3472,"grew":3473,"commander":3474,"isn":3475,"1967":3476,"pay":3477,"failed":3478,"selected":3479,"hurt":3480,"fort":3481,"box":3482,"regiment":3483,"majority":3484,"journal":3485,"35":3486,"edward":3487,"plans":3488,"##ke":3489,"##ni":3490,"shown":3491,"pretty":3492,"irish":3493,"characters":3494,"directly":3495,"scene":3496,"likely":3497,"operated":3498,"allow":3499,"spring":3500,"##j":3501,"junior":3502,"matches":3503,"looks":3504,"mike":3505,"houses":3506,"fellow":3507,"##tion":3508,"beach":3509,"marriage":3510,"##ham":3511,"##ive":3512,"rules":3513,"oil":3514,"65":3515,"florida":3516,"expected":3517,"nearby":3518,"congress":3519,"sam":3520,"peace":3521,"recent":3522,"iii":3523,"wait":3524,"subsequently":3525,"cell":3526,"##do":3527,"variety":3528,"serving":3529,"agreed":3530,"please":3531,"poor":3532,"joe":3533,"pacific":3534,"attempt":3535,"wood":3536,"democratic":3537,"piece":3538,"prime":3539,"##ca":3540,"rural":3541,"mile":3542,"touch":3543,"appears":3544,"township":3545,"1964":3546,"1966":3547,"soldiers":3548,"##men":3549,"##ized":3550,"1965":3551,"pennsylvania":3552,"closer":3553,"fighting":3554,"claimed":3555,"score":3556,"jones":3557,"physical":3558,"editor":3559,"##ous":3560,"filled":3561,"genus":3562,"specific":3563,"sitting":3564,"super":3565,"mom":3566,"##va":3567,"therefore":3568,"supported":3569,"status":3570,"fear":3571,"cases":3572,"store":3573,"meaning":3574,"wales":3575,"minor":3576,"spain":3577,"tower":3578,"focus":3579,"vice":3580,"frank":3581,"follow":3582,"parish":3583,"separate":3584,"golden":3585,"horse":3586,"fifth":3587,"remaining":3588,"branch":3589,"32":3590,"presented":3591,"stared":3592,"##id":3593,"uses":3594,"secret":3595,"forms":3596,"##co":3597,"baseball":3598,"exactly":3599,"##ck":3600,"choice":3601,"note":3602,"discovered":3603,"travel":3604,"composed":3605,"truth":3606,"russia":3607,"ball":3608,"color":3609,"kiss":3610,"dad":3611,"wind":3612,"continue":3613,"ring":3614,"referred":3615,"numbers":3616,"digital":3617,"greater":3618,"##ns":3619,"metres":3620,"slightly":3621,"direct":3622,"increase":3623,"1960":3624,"responsible":3625,"crew":3626,"rule":3627,"trees":3628,"troops":3629,"##no":3630,"broke":3631,"goes":3632,"individuals":3633,"hundred":3634,"weight":3635,"creek":3636,"sleep":3637,"memory":3638,"defense":3639,"provides":3640,"ordered":3641,"code":3642,"value":3643,"jewish":3644,"windows":3645,"1944":3646,"safe":3647,"judge":3648,"whatever":3649,"corps":3650,"realized":3651,"growing":3652,"pre":3653,"##ga":3654,"cities":3655,"alexander":3656,"gaze":3657,"lies":3658,"spread":3659,"scott":3660,"letter":3661,"showed":3662,"situation":3663,"mayor":3664,"transport":3665,"watching":3666,"workers":3667,"extended":3668,"##li":3669,"expression":3670,"normal":3671,"##ment":3672,"chart":3673,"multiple":3674,"border":3675,"##ba":3676,"host":3677,"##ner":3678,"daily":3679,"mrs":3680,"walls":3681,"piano":3682,"##ko":3683,"heat":3684,"cannot":3685,"##ate":3686,"earned":3687,"products":3688,"drama":3689,"era":3690,"authority":3691,"seasons":3692,"join":3693,"grade":3694,"##io":3695,"sign":3696,"difficult":3697,"machine":3698,"1963":3699,"territory":3700,"mainly":3701,"##wood":3702,"stations":3703,"squadron":3704,"1962":3705,"stepped":3706,"iron":3707,"19th":3708,"##led":3709,"serve":3710,"appear":3711,"sky":3712,"speak":3713,"broken":3714,"charge":3715,"knowledge":3716,"kilometres":3717,"removed":3718,"ships":3719,"article":3720,"campus":3721,"simple":3722,"##ty":3723,"pushed":3724,"britain":3725,"##ve":3726,"leaves":3727,"recently":3728,"cd":3729,"soft":3730,"boston":3731,"latter":3732,"easy":3733,"acquired":3734,"poland":3735,"##sa":3736,"quality":3737,"officers":3738,"presence":3739,"planned":3740,"nations":3741,"mass":3742,"broadcast":3743,"jean":3744,"share":3745,"image":3746,"influence":3747,"wild":3748,"offer":3749,"emperor":3750,"electric":3751,"reading":3752,"headed":3753,"ability":3754,"promoted":3755,"yellow":3756,"ministry":3757,"1942":3758,"throat":3759,"smaller":3760,"politician":3761,"##by":3762,"latin":3763,"spoke":3764,"cars":3765,"williams":3766,"males":3767,"lack":3768,"pop":3769,"80":3770,"##ier":3771,"acting":3772,"seeing":3773,"consists":3774,"##ti":3775,"estate":3776,"1961":3777,"pressure":3778,"johnson":3779,"newspaper":3780,"jr":3781,"chris":3782,"olympics":3783,"online":3784,"conditions":3785,"beat":3786,"elements":3787,"walking":3788,"vote":3789,"##field":3790,"needs":3791,"carolina":3792,"text":3793,"featuring":3794,"global":3795,"block":3796,"shirt":3797,"levels":3798,"francisco":3799,"purpose":3800,"females":3801,"et":3802,"dutch":3803,"duke":3804,"ahead":3805,"gas":3806,"twice":3807,"safety":3808,"serious":3809,"turning":3810,"highly":3811,"lieutenant":3812,"firm":3813,"maria":3814,"amount":3815,"mixed":3816,"daniel":3817,"proposed":3818,"perfect":3819,"agreement":3820,"affairs":3821,"3rd":3822,"seconds":3823,"contemporary":3824,"paid":3825,"1943":3826,"prison":3827,"save":3828,"kitchen":3829,"label":3830,"administrative":3831,"intended":3832,"constructed":3833,"academic":3834,"nice":3835,"teacher":3836,"races":3837,"1956":3838,"formerly":3839,"corporation":3840,"ben":3841,"nation":3842,"issued":3843,"shut":3844,"1958":3845,"drums":3846,"housing":3847,"victoria":3848,"seems":3849,"opera":3850,"1959":3851,"graduated":3852,"function":3853,"von":3854,"mentioned":3855,"picked":3856,"build":3857,"recognized":3858,"shortly":3859,"protection":3860,"picture":3861,"notable":3862,"exchange":3863,"elections":3864,"1980s":3865,"loved":3866,"percent":3867,"racing":3868,"fish":3869,"elizabeth":3870,"garden":3871,"volume":3872,"hockey":3873,"1941":3874,"beside":3875,"settled":3876,"##ford":3877,"1940":3878,"competed":3879,"replied":3880,"drew":3881,"1948":3882,"actress":3883,"marine":3884,"scotland":3885,"steel":3886,"glanced":3887,"farm":3888,"steve":3889,"1957":3890,"risk":3891,"tonight":3892,"positive":3893,"magic":3894,"singles":3895,"effects":3896,"gray":3897,"screen":3898,"dog":3899,"##ja":3900,"residents":3901,"bus":3902,"sides":3903,"none":3904,"secondary":3905,"literature":3906,"polish":3907,"destroyed":3908,"flying":3909,"founder":3910,"households":3911,"1939":3912,"lay":3913,"reserve":3914,"usa":3915,"gallery":3916,"##ler":3917,"1946":3918,"industrial":3919,"younger":3920,"approach":3921,"appearances":3922,"urban":3923,"ones":3924,"1950":3925,"finish":3926,"avenue":3927,"powerful":3928,"fully":3929,"growth":3930,"page":3931,"honor":3932,"jersey":3933,"projects":3934,"advanced":3935,"revealed":3936,"basic":3937,"90":3938,"infantry":3939,"pair":3940,"equipment":3941,"visit":3942,"33":3943,"evening":3944,"search":3945,"grant":3946,"effort":3947,"solo":3948,"treatment":3949,"buried":3950,"republican":3951,"primarily":3952,"bottom":3953,"owner":3954,"1970s":3955,"israel":3956,"gives":3957,"jim":3958,"dream":3959,"bob":3960,"remain":3961,"spot":3962,"70":3963,"notes":3964,"produce":3965,"champions":3966,"contact":3967,"ed":3968,"soul":3969,"accepted":3970,"ways":3971,"del":3972,"##ally":3973,"losing":3974,"split":3975,"price":3976,"capacity":3977,"basis":3978,"trial":3979,"questions":3980,"##ina":3981,"1955":3982,"20th":3983,"guess":3984,"officially":3985,"memorial":3986,"naval":3987,"initial":3988,"##ization":3989,"whispered":3990,"median":3991,"engineer":3992,"##ful":3993,"sydney":3994,"##go":3995,"columbia":3996,"strength":3997,"300":3998,"1952":3999,"tears":4000,"senate":4001,"00":4002,"card":4003,"asian":4004,"agent":4005,"1947":4006,"software":4007,"44":4008,"draw":4009,"warm":4010,"supposed":4011,"com":4012,"pro":4013,"##il":4014,"transferred":4015,"leaned":4016,"##at":4017,"candidate":4018,"escape":4019,"mountains":4020,"asia":4021,"potential":4022,"activity":4023,"entertainment":4024,"seem":4025,"traffic":4026,"jackson":4027,"murder":4028,"36":4029,"slow":4030,"product":4031,"orchestra":4032,"haven":4033,"agency":4034,"bbc":4035,"taught":4036,"website":4037,"comedy":4038,"unable":4039,"storm":4040,"planning":4041,"albums":4042,"rugby":4043,"environment":4044,"scientific":4045,"grabbed":4046,"protect":4047,"##hi":4048,"boat":4049,"typically":4050,"1954":4051,"1953":4052,"damage":4053,"principal":4054,"divided":4055,"dedicated":4056,"mount":4057,"ohio":4058,"##berg":4059,"pick":4060,"fought":4061,"driver":4062,"##der":4063,"empty":4064,"shoulders":4065,"sort":4066,"thank":4067,"berlin":4068,"prominent":4069,"account":4070,"freedom":4071,"necessary":4072,"efforts":4073,"alex":4074,"headquarters":4075,"follows":4076,"alongside":4077,"des":4078,"simon":4079,"andrew":4080,"suggested":4081,"operating":4082,"learning":4083,"steps":4084,"1949":4085,"sweet":4086,"technical":4087,"begin":4088,"easily":4089,"34":4090,"teeth":4091,"speaking":4092,"settlement":4093,"scale":4094,"##sh":4095,"renamed":4096,"ray":4097,"max":4098,"enemy":4099,"semi":4100,"joint":4101,"compared":4102,"##rd":4103,"scottish":4104,"leadership":4105,"analysis":4106,"offers":4107,"georgia":4108,"pieces":4109,"captured":4110,"animal":4111,"deputy":4112,"guest":4113,"organized":4114,"##lin":4115,"tony":4116,"combined":4117,"method":4118,"challenge":4119,"1960s":4120,"huge":4121,"wants":4122,"battalion":4123,"sons":4124,"rise":4125,"crime":4126,"types":4127,"facilities":4128,"telling":4129,"path":4130,"1951":4131,"platform":4132,"sit":4133,"1990s":4134,"##lo":4135,"tells":4136,"assigned":4137,"rich":4138,"pull":4139,"##ot":4140,"commonly":4141,"alive":4142,"##za":4143,"letters":4144,"concept":4145,"conducted":4146,"wearing":4147,"happen":4148,"bought":4149,"becomes":4150,"holy":4151,"gets":4152,"ocean":4153,"defeat":4154,"languages":4155,"purchased":4156,"coffee":4157,"occurred":4158,"titled":4159,"##q":4160,"declared":4161,"applied":4162,"sciences":4163,"concert":4164,"sounds":4165,"jazz":4166,"brain":4167,"##me":4168,"painting":4169,"fleet":4170,"tax":4171,"nick":4172,"##ius":4173,"michigan":4174,"count":4175,"animals":4176,"leaders":4177,"episodes":4178,"##line":4179,"content":4180,"##den":4181,"birth":4182,"##it":4183,"clubs":4184,"64":4185,"palace":4186,"critical":4187,"refused":4188,"fair":4189,"leg":4190,"laughed":4191,"returning":4192,"surrounding":4193,"participated":4194,"formation":4195,"lifted":4196,"pointed":4197,"connected":4198,"rome":4199,"medicine":4200,"laid":4201,"taylor":4202,"santa":4203,"powers":4204,"adam":4205,"tall":4206,"shared":4207,"focused":4208,"knowing":4209,"yards":4210,"entrance":4211,"falls":4212,"##wa":4213,"calling":4214,"##ad":4215,"sources":4216,"chosen":4217,"beneath":4218,"resources":4219,"yard":4220,"##ite":4221,"nominated":4222,"silence":4223,"zone":4224,"defined":4225,"##que":4226,"gained":4227,"thirty":4228,"38":4229,"bodies":4230,"moon":4231,"##ard":4232,"adopted":4233,"christmas":4234,"widely":4235,"register":4236,"apart":4237,"iran":4238,"premier":4239,"serves":4240,"du":4241,"unknown":4242,"parties":4243,"##les":4244,"generation":4245,"##ff":4246,"continues":4247,"quick":4248,"fields":4249,"brigade":4250,"quiet":4251,"teaching":4252,"clothes":4253,"impact":4254,"weapons":4255,"partner":4256,"flat":4257,"theater":4258,"supreme":4259,"1938":4260,"37":4261,"relations":4262,"##tor":4263,"plants":4264,"suffered":4265,"1936":4266,"wilson":4267,"kids":4268,"begins":4269,"##age":4270,"1918":4271,"seats":4272,"armed":4273,"internet":4274,"models":4275,"worth":4276,"laws":4277,"400":4278,"communities":4279,"classes":4280,"background":4281,"knows":4282,"thanks":4283,"quarter":4284,"reaching":4285,"humans":4286,"carry":4287,"killing":4288,"format":4289,"kong":4290,"hong":4291,"setting":4292,"75":4293,"architecture":4294,"disease":4295,"railroad":4296,"inc":4297,"possibly":4298,"wish":4299,"arthur":4300,"thoughts":4301,"harry":4302,"doors":4303,"density":4304,"##di":4305,"crowd":4306,"illinois":4307,"stomach":4308,"tone":4309,"unique":4310,"reports":4311,"anyway":4312,"##ir":4313,"liberal":4314,"der":4315,"vehicle":4316,"thick":4317,"dry":4318,"drug":4319,"faced":4320,"largely":4321,"facility":4322,"theme":4323,"holds":4324,"creation":4325,"strange":4326,"colonel":4327,"##mi":4328,"revolution":4329,"bell":4330,"politics":4331,"turns":4332,"silent":4333,"rail":4334,"relief":4335,"independence":4336,"combat":4337,"shape":4338,"write":4339,"determined":4340,"sales":4341,"learned":4342,"4th":4343,"finger":4344,"oxford":4345,"providing":4346,"1937":4347,"heritage":4348,"fiction":4349,"situated":4350,"designated":4351,"allowing":4352,"distribution":4353,"hosted":4354,"##est":4355,"sight":4356,"interview":4357,"estimated":4358,"reduced":4359,"##ria":4360,"toronto":4361,"footballer":4362,"keeping":4363,"guys":4364,"damn":4365,"claim":4366,"motion":4367,"sport":4368,"sixth":4369,"stayed":4370,"##ze":4371,"en":4372,"rear":4373,"receive":4374,"handed":4375,"twelve":4376,"dress":4377,"audience":4378,"granted":4379,"brazil":4380,"##well":4381,"spirit":4382,"##ated":4383,"noticed":4384,"etc":4385,"olympic":4386,"representative":4387,"eric":4388,"tight":4389,"trouble":4390,"reviews":4391,"drink":4392,"vampire":4393,"missing":4394,"roles":4395,"ranked":4396,"newly":4397,"household":4398,"finals":4399,"wave":4400,"critics":4401,"##ee":4402,"phase":4403,"massachusetts":4404,"pilot":4405,"unlike":4406,"philadelphia":4407,"bright":4408,"guns":4409,"crown":4410,"organizations":4411,"roof":4412,"42":4413,"respectively":4414,"clearly":4415,"tongue":4416,"marked":4417,"circle":4418,"fox":4419,"korea":4420,"bronze":4421,"brian":4422,"expanded":4423,"sexual":4424,"supply":4425,"yourself":4426,"inspired":4427,"labour":4428,"fc":4429,"##ah":4430,"reference":4431,"vision":4432,"draft":4433,"connection":4434,"brand":4435,"reasons":4436,"1935":4437,"classic":4438,"driving":4439,"trip":4440,"jesus":4441,"cells":4442,"entry":4443,"1920":4444,"neither":4445,"trail":4446,"claims":4447,"atlantic":4448,"orders":4449,"labor":4450,"nose":4451,"afraid":4452,"identified":4453,"intelligence":4454,"calls":4455,"cancer":4456,"attacked":4457,"passing":4458,"stephen":4459,"positions":4460,"imperial":4461,"grey":4462,"jason":4463,"39":4464,"sunday":4465,"48":4466,"swedish":4467,"avoid":4468,"extra":4469,"uncle":4470,"message":4471,"covers":4472,"allows":4473,"surprise":4474,"materials":4475,"fame":4476,"hunter":4477,"##ji":4478,"1930":4479,"citizens":4480,"figures":4481,"davis":4482,"environmental":4483,"confirmed":4484,"shit":4485,"titles":4486,"di":4487,"performing":4488,"difference":4489,"acts":4490,"attacks":4491,"##ov":4492,"existing":4493,"votes":4494,"opportunity":4495,"nor":4496,"shop":4497,"entirely":4498,"trains":4499,"opposite":4500,"pakistan":4501,"##pa":4502,"develop":4503,"resulted":4504,"representatives":4505,"actions":4506,"reality":4507,"pressed":4508,"##ish":4509,"barely":4510,"wine":4511,"conversation":4512,"faculty":4513,"northwest":4514,"ends":4515,"documentary":4516,"nuclear":4517,"stock":4518,"grace":4519,"sets":4520,"eat":4521,"alternative":4522,"##ps":4523,"bag":4524,"resulting":4525,"creating":4526,"surprised":4527,"cemetery":4528,"1919":4529,"drop":4530,"finding":4531,"sarah":4532,"cricket":4533,"streets":4534,"tradition":4535,"ride":4536,"1933":4537,"exhibition":4538,"target":4539,"ear":4540,"explained":4541,"rain":4542,"composer":4543,"injury":4544,"apartment":4545,"municipal":4546,"educational":4547,"occupied":4548,"netherlands":4549,"clean":4550,"billion":4551,"constitution":4552,"learn":4553,"1914":4554,"maximum":4555,"classical":4556,"francis":4557,"lose":4558,"opposition":4559,"jose":4560,"ontario":4561,"bear":4562,"core":4563,"hills":4564,"rolled":4565,"ending":4566,"drawn":4567,"permanent":4568,"fun":4569,"##tes":4570,"##lla":4571,"lewis":4572,"sites":4573,"chamber":4574,"ryan":4575,"##way":4576,"scoring":4577,"height":4578,"1934":4579,"##house":4580,"lyrics":4581,"staring":4582,"55":4583,"officials":4584,"1917":4585,"snow":4586,"oldest":4587,"##tic":4588,"orange":4589,"##ger":4590,"qualified":4591,"interior":4592,"apparently":4593,"succeeded":4594,"thousand":4595,"dinner":4596,"lights":4597,"existence":4598,"fans":4599,"heavily":4600,"41":4601,"greatest":4602,"conservative":4603,"send":4604,"bowl":4605,"plus":4606,"enter":4607,"catch":4608,"##un":4609,"economy":4610,"duty":4611,"1929":4612,"speech":4613,"authorities":4614,"princess":4615,"performances":4616,"versions":4617,"shall":4618,"graduate":4619,"pictures":4620,"effective":4621,"remembered":4622,"poetry":4623,"desk":4624,"crossed":4625,"starring":4626,"starts":4627,"passenger":4628,"sharp":4629,"##ant":4630,"acres":4631,"ass":4632,"weather":4633,"falling":4634,"rank":4635,"fund":4636,"supporting":4637,"check":4638,"adult":4639,"publishing":4640,"heads":4641,"cm":4642,"southeast":4643,"lane":4644,"##burg":4645,"application":4646,"bc":4647,"##ura":4648,"les":4649,"condition":4650,"transfer":4651,"prevent":4652,"display":4653,"ex":4654,"regions":4655,"earl":4656,"federation":4657,"cool":4658,"relatively":4659,"answered":4660,"besides":4661,"1928":4662,"obtained":4663,"portion":4664,"##town":4665,"mix":4666,"##ding":4667,"reaction":4668,"liked":4669,"dean":4670,"express":4671,"peak":4672,"1932":4673,"##tte":4674,"counter":4675,"religion":4676,"chain":4677,"rare":4678,"miller":4679,"convention":4680,"aid":4681,"lie":4682,"vehicles":4683,"mobile":4684,"perform":4685,"squad":4686,"wonder":4687,"lying":4688,"crazy":4689,"sword":4690,"##ping":4691,"attempted":4692,"centuries":4693,"weren":4694,"philosophy":4695,"category":4696,"##ize":4697,"anna":4698,"interested":4699,"47":4700,"sweden":4701,"wolf":4702,"frequently":4703,"abandoned":4704,"kg":4705,"literary":4706,"alliance":4707,"task":4708,"entitled":4709,"##ay":4710,"threw":4711,"promotion":4712,"factory":4713,"tiny":4714,"soccer":4715,"visited":4716,"matt":4717,"fm":4718,"achieved":4719,"52":4720,"defence":4721,"internal":4722,"persian":4723,"43":4724,"methods":4725,"##ging":4726,"arrested":4727,"otherwise":4728,"cambridge":4729,"programming":4730,"villages":4731,"elementary":4732,"districts":4733,"rooms":4734,"criminal":4735,"conflict":4736,"worry":4737,"trained":4738,"1931":4739,"attempts":4740,"waited":4741,"signal":4742,"bird":4743,"truck":4744,"subsequent":4745,"programme":4746,"##ol":4747,"ad":4748,"49":4749,"communist":4750,"details":4751,"faith":4752,"sector":4753,"patrick":4754,"carrying":4755,"laugh":4756,"##ss":4757,"controlled":4758,"korean":4759,"showing":4760,"origin":4761,"fuel":4762,"evil":4763,"1927":4764,"##ent":4765,"brief":4766,"identity":4767,"darkness":4768,"address":4769,"pool":4770,"missed":4771,"publication":4772,"web":4773,"planet":4774,"ian":4775,"anne":4776,"wings":4777,"invited":4778,"##tt":4779,"briefly":4780,"standards":4781,"kissed":4782,"##be":4783,"ideas":4784,"climate":4785,"causing":4786,"walter":4787,"worse":4788,"albert":4789,"articles":4790,"winners":4791,"desire":4792,"aged":4793,"northeast":4794,"dangerous":4795,"gate":4796,"doubt":4797,"1922":4798,"wooden":4799,"multi":4800,"##ky":4801,"poet":4802,"rising":4803,"funding":4804,"46":4805,"communications":4806,"communication":4807,"violence":4808,"copies":4809,"prepared":4810,"ford":4811,"investigation":4812,"skills":4813,"1924":4814,"pulling":4815,"electronic":4816,"##ak":4817,"##ial":4818,"##han":4819,"containing":4820,"ultimately":4821,"offices":4822,"singing":4823,"understanding":4824,"restaurant":4825,"tomorrow":4826,"fashion":4827,"christ":4828,"ward":4829,"da":4830,"pope":4831,"stands":4832,"5th":4833,"flow":4834,"studios":4835,"aired":4836,"commissioned":4837,"contained":4838,"exist":4839,"fresh":4840,"americans":4841,"##per":4842,"wrestling":4843,"approved":4844,"kid":4845,"employed":4846,"respect":4847,"suit":4848,"1925":4849,"angel":4850,"asking":4851,"increasing":4852,"frame":4853,"angry":4854,"selling":4855,"1950s":4856,"thin":4857,"finds":4858,"##nd":4859,"temperature":4860,"statement":4861,"ali":4862,"explain":4863,"inhabitants":4864,"towns":4865,"extensive":4866,"narrow":4867,"51":4868,"jane":4869,"flowers":4870,"images":4871,"promise":4872,"somewhere":4873,"object":4874,"fly":4875,"closely":4876,"##ls":4877,"1912":4878,"bureau":4879,"cape":4880,"1926":4881,"weekly":4882,"presidential":4883,"legislative":4884,"1921":4885,"##ai":4886,"##au":4887,"launch":4888,"founding":4889,"##ny":4890,"978":4891,"##ring":4892,"artillery":4893,"strike":4894,"un":4895,"institutions":4896,"roll":4897,"writers":4898,"landing":4899,"chose":4900,"kevin":4901,"anymore":4902,"pp":4903,"##ut":4904,"attorney":4905,"fit":4906,"dan":4907,"billboard":4908,"receiving":4909,"agricultural":4910,"breaking":4911,"sought":4912,"dave":4913,"admitted":4914,"lands":4915,"mexican":4916,"##bury":4917,"charlie":4918,"specifically":4919,"hole":4920,"iv":4921,"howard":4922,"credit":4923,"moscow":4924,"roads":4925,"accident":4926,"1923":4927,"proved":4928,"wear":4929,"struck":4930,"hey":4931,"guards":4932,"stuff":4933,"slid":4934,"expansion":4935,"1915":4936,"cat":4937,"anthony":4938,"##kin":4939,"melbourne":4940,"opposed":4941,"sub":4942,"southwest":4943,"architect":4944,"failure":4945,"plane":4946,"1916":4947,"##ron":4948,"map":4949,"camera":4950,"tank":4951,"listen":4952,"regarding":4953,"wet":4954,"introduction":4955,"metropolitan":4956,"link":4957,"ep":4958,"fighter":4959,"inch":4960,"grown":4961,"gene":4962,"anger":4963,"fixed":4964,"buy":4965,"dvd":4966,"khan":4967,"domestic":4968,"worldwide":4969,"chapel":4970,"mill":4971,"functions":4972,"examples":4973,"##head":4974,"developing":4975,"1910":4976,"turkey":4977,"hits":4978,"pocket":4979,"antonio":4980,"papers":4981,"grow":4982,"unless":4983,"circuit":4984,"18th":4985,"concerned":4986,"attached":4987,"journalist":4988,"selection":4989,"journey":4990,"converted":4991,"provincial":4992,"painted":4993,"hearing":4994,"aren":4995,"bands":4996,"negative":4997,"aside":4998,"wondered":4999,"knight":5000,"lap":5001,"survey":5002,"ma":5003,"##ow":5004,"noise":5005,"billy":5006,"##ium":5007,"shooting":5008,"guide":5009,"bedroom":5010,"priest":5011,"resistance":5012,"motor":5013,"homes":5014,"sounded":5015,"giant":5016,"##mer":5017,"150":5018,"scenes":5019,"equal":5020,"comic":5021,"patients":5022,"hidden":5023,"solid":5024,"actual":5025,"bringing":5026,"afternoon":5027,"touched":5028,"funds":5029,"wedding":5030,"consisted":5031,"marie":5032,"canal":5033,"sr":5034,"kim":5035,"treaty":5036,"turkish":5037,"recognition":5038,"residence":5039,"cathedral":5040,"broad":5041,"knees":5042,"incident":5043,"shaped":5044,"fired":5045,"norwegian":5046,"handle":5047,"cheek":5048,"contest":5049,"represent":5050,"##pe":5051,"representing":5052,"beauty":5053,"##sen":5054,"birds":5055,"advantage":5056,"emergency":5057,"wrapped":5058,"drawing":5059,"notice":5060,"pink":5061,"broadcasting":5062,"##ong":5063,"somehow":5064,"bachelor":5065,"seventh":5066,"collected":5067,"registered":5068,"establishment":5069,"alan":5070,"assumed":5071,"chemical":5072,"personnel":5073,"roger":5074,"retirement":5075,"jeff":5076,"portuguese":5077,"wore":5078,"tied":5079,"device":5080,"threat":5081,"progress":5082,"advance":5083,"##ised":5084,"banks":5085,"hired":5086,"manchester":5087,"nfl":5088,"teachers":5089,"structures":5090,"forever":5091,"##bo":5092,"tennis":5093,"helping":5094,"saturday":5095,"sale":5096,"applications":5097,"junction":5098,"hip":5099,"incorporated":5100,"neighborhood":5101,"dressed":5102,"ceremony":5103,"##ds":5104,"influenced":5105,"hers":5106,"visual":5107,"stairs":5108,"decades":5109,"inner":5110,"kansas":5111,"hung":5112,"hoped":5113,"gain":5114,"scheduled":5115,"downtown":5116,"engaged":5117,"austria":5118,"clock":5119,"norway":5120,"certainly":5121,"pale":5122,"protected":5123,"1913":5124,"victor":5125,"employees":5126,"plate":5127,"putting":5128,"surrounded":5129,"##ists":5130,"finishing":5131,"blues":5132,"tropical":5133,"##ries":5134,"minnesota":5135,"consider":5136,"philippines":5137,"accept":5138,"54":5139,"retrieved":5140,"1900":5141,"concern":5142,"anderson":5143,"properties":5144,"institution":5145,"gordon":5146,"successfully":5147,"vietnam":5148,"##dy":5149,"backing":5150,"outstanding":5151,"muslim":5152,"crossing":5153,"folk":5154,"producing":5155,"usual":5156,"demand":5157,"occurs":5158,"observed":5159,"lawyer":5160,"educated":5161,"##ana":5162,"kelly":5163,"string":5164,"pleasure":5165,"budget":5166,"items":5167,"quietly":5168,"colorado":5169,"philip":5170,"typical":5171,"##worth":5172,"derived":5173,"600":5174,"survived":5175,"asks":5176,"mental":5177,"##ide":5178,"56":5179,"jake":5180,"jews":5181,"distinguished":5182,"ltd":5183,"1911":5184,"sri":5185,"extremely":5186,"53":5187,"athletic":5188,"loud":5189,"thousands":5190,"worried":5191,"shadow":5192,"transportation":5193,"horses":5194,"weapon":5195,"arena":5196,"importance":5197,"users":5198,"tim":5199,"objects":5200,"contributed":5201,"dragon":5202,"douglas":5203,"aware":5204,"senator":5205,"johnny":5206,"jordan":5207,"sisters":5208,"engines":5209,"flag":5210,"investment":5211,"samuel":5212,"shock":5213,"capable":5214,"clark":5215,"row":5216,"wheel":5217,"refers":5218,"session":5219,"familiar":5220,"biggest":5221,"wins":5222,"hate":5223,"maintained":5224,"drove":5225,"hamilton":5226,"request":5227,"expressed":5228,"injured":5229,"underground":5230,"churches":5231,"walker":5232,"wars":5233,"tunnel":5234,"passes":5235,"stupid":5236,"agriculture":5237,"softly":5238,"cabinet":5239,"regarded":5240,"joining":5241,"indiana":5242,"##ea":5243,"##ms":5244,"push":5245,"dates":5246,"spend":5247,"behavior":5248,"woods":5249,"protein":5250,"gently":5251,"chase":5252,"morgan":5253,"mention":5254,"burning":5255,"wake":5256,"combination":5257,"occur":5258,"mirror":5259,"leads":5260,"jimmy":5261,"indeed":5262,"impossible":5263,"singapore":5264,"paintings":5265,"covering":5266,"##nes":5267,"soldier":5268,"locations":5269,"attendance":5270,"sell":5271,"historian":5272,"wisconsin":5273,"invasion":5274,"argued":5275,"painter":5276,"diego":5277,"changing":5278,"egypt":5279,"##don":5280,"experienced":5281,"inches":5282,"##ku":5283,"missouri":5284,"vol":5285,"grounds":5286,"spoken":5287,"switzerland":5288,"##gan":5289,"reform":5290,"rolling":5291,"ha":5292,"forget":5293,"massive":5294,"resigned":5295,"burned":5296,"allen":5297,"tennessee":5298,"locked":5299,"values":5300,"improved":5301,"##mo":5302,"wounded":5303,"universe":5304,"sick":5305,"dating":5306,"facing":5307,"pack":5308,"purchase":5309,"user":5310,"##pur":5311,"moments":5312,"##ul":5313,"merged":5314,"anniversary":5315,"1908":5316,"coal":5317,"brick":5318,"understood":5319,"causes":5320,"dynasty":5321,"queensland":5322,"establish":5323,"stores":5324,"crisis":5325,"promote":5326,"hoping":5327,"views":5328,"cards":5329,"referee":5330,"extension":5331,"##si":5332,"raise":5333,"arizona":5334,"improve":5335,"colonial":5336,"formal":5337,"charged":5338,"##rt":5339,"palm":5340,"lucky":5341,"hide":5342,"rescue":5343,"faces":5344,"95":5345,"feelings":5346,"candidates":5347,"juan":5348,"##ell":5349,"goods":5350,"6th":5351,"courses":5352,"weekend":5353,"59":5354,"luke":5355,"cash":5356,"fallen":5357,"##om":5358,"delivered":5359,"affected":5360,"installed":5361,"carefully":5362,"tries":5363,"swiss":5364,"hollywood":5365,"costs":5366,"lincoln":5367,"responsibility":5368,"##he":5369,"shore":5370,"file":5371,"proper":5372,"normally":5373,"maryland":5374,"assistance":5375,"jump":5376,"constant":5377,"offering":5378,"friendly":5379,"waters":5380,"persons":5381,"realize":5382,"contain":5383,"trophy":5384,"800":5385,"partnership":5386,"factor":5387,"58":5388,"musicians":5389,"cry":5390,"bound":5391,"oregon":5392,"indicated":5393,"hero":5394,"houston":5395,"medium":5396,"##ure":5397,"consisting":5398,"somewhat":5399,"##ara":5400,"57":5401,"cycle":5402,"##che":5403,"beer":5404,"moore":5405,"frederick":5406,"gotten":5407,"eleven":5408,"worst":5409,"weak":5410,"approached":5411,"arranged":5412,"chin":5413,"loan":5414,"universal":5415,"bond":5416,"fifteen":5417,"pattern":5418,"disappeared":5419,"##ney":5420,"translated":5421,"##zed":5422,"lip":5423,"arab":5424,"capture":5425,"interests":5426,"insurance":5427,"##chi":5428,"shifted":5429,"cave":5430,"prix":5431,"warning":5432,"sections":5433,"courts":5434,"coat":5435,"plot":5436,"smell":5437,"feed":5438,"golf":5439,"favorite":5440,"maintain":5441,"knife":5442,"vs":5443,"voted":5444,"degrees":5445,"finance":5446,"quebec":5447,"opinion":5448,"translation":5449,"manner":5450,"ruled":5451,"operate":5452,"productions":5453,"choose":5454,"musician":5455,"discovery":5456,"confused":5457,"tired":5458,"separated":5459,"stream":5460,"techniques":5461,"committed":5462,"attend":5463,"ranking":5464,"kings":5465,"throw":5466,"passengers":5467,"measure":5468,"horror":5469,"fan":5470,"mining":5471,"sand":5472,"danger":5473,"salt":5474,"calm":5475,"decade":5476,"dam":5477,"require":5478,"runner":5479,"##ik":5480,"rush":5481,"associate":5482,"greece":5483,"##ker":5484,"rivers":5485,"consecutive":5486,"matthew":5487,"##ski":5488,"sighed":5489,"sq":5490,"documents":5491,"steam":5492,"edited":5493,"closing":5494,"tie":5495,"accused":5496,"1905":5497,"##ini":5498,"islamic":5499,"distributed":5500,"directors":5501,"organisation":5502,"bruce":5503,"7th":5504,"breathing":5505,"mad":5506,"lit":5507,"arrival":5508,"concrete":5509,"taste":5510,"08":5511,"composition":5512,"shaking":5513,"faster":5514,"amateur":5515,"adjacent":5516,"stating":5517,"1906":5518,"twin":5519,"flew":5520,"##ran":5521,"tokyo":5522,"publications":5523,"##tone":5524,"obviously":5525,"ridge":5526,"storage":5527,"1907":5528,"carl":5529,"pages":5530,"concluded":5531,"desert":5532,"driven":5533,"universities":5534,"ages":5535,"terminal":5536,"sequence":5537,"borough":5538,"250":5539,"constituency":5540,"creative":5541,"cousin":5542,"economics":5543,"dreams":5544,"margaret":5545,"notably":5546,"reduce":5547,"montreal":5548,"mode":5549,"17th":5550,"ears":5551,"saved":5552,"jan":5553,"vocal":5554,"##ica":5555,"1909":5556,"andy":5557,"##jo":5558,"riding":5559,"roughly":5560,"threatened":5561,"##ise":5562,"meters":5563,"meanwhile":5564,"landed":5565,"compete":5566,"repeated":5567,"grass":5568,"czech":5569,"regularly":5570,"charges":5571,"tea":5572,"sudden":5573,"appeal":5574,"##ung":5575,"solution":5576,"describes":5577,"pierre":5578,"classification":5579,"glad":5580,"parking":5581,"##ning":5582,"belt":5583,"physics":5584,"99":5585,"rachel":5586,"add":5587,"hungarian":5588,"participate":5589,"expedition":5590,"damaged":5591,"gift":5592,"childhood":5593,"85":5594,"fifty":5595,"##red":5596,"mathematics":5597,"jumped":5598,"letting":5599,"defensive":5600,"mph":5601,"##ux":5602,"##gh":5603,"testing":5604,"##hip":5605,"hundreds":5606,"shoot":5607,"owners":5608,"matters":5609,"smoke":5610,"israeli":5611,"kentucky":5612,"dancing":5613,"mounted":5614,"grandfather":5615,"emma":5616,"designs":5617,"profit":5618,"argentina":5619,"##gs":5620,"truly":5621,"li":5622,"lawrence":5623,"cole":5624,"begun":5625,"detroit":5626,"willing":5627,"branches":5628,"smiling":5629,"decide":5630,"miami":5631,"enjoyed":5632,"recordings":5633,"##dale":5634,"poverty":5635,"ethnic":5636,"gay":5637,"##bi":5638,"gary":5639,"arabic":5640,"09":5641,"accompanied":5642,"##one":5643,"##ons":5644,"fishing":5645,"determine":5646,"residential":5647,"acid":5648,"##ary":5649,"alice":5650,"returns":5651,"starred":5652,"mail":5653,"##ang":5654,"jonathan":5655,"strategy":5656,"##ue":5657,"net":5658,"forty":5659,"cook":5660,"businesses":5661,"equivalent":5662,"commonwealth":5663,"distinct":5664,"ill":5665,"##cy":5666,"seriously":5667,"##ors":5668,"##ped":5669,"shift":5670,"harris":5671,"replace":5672,"rio":5673,"imagine":5674,"formula":5675,"ensure":5676,"##ber":5677,"additionally":5678,"scheme":5679,"conservation":5680,"occasionally":5681,"purposes":5682,"feels":5683,"favor":5684,"##and":5685,"##ore":5686,"1930s":5687,"contrast":5688,"hanging":5689,"hunt":5690,"movies":5691,"1904":5692,"instruments":5693,"victims":5694,"danish":5695,"christopher":5696,"busy":5697,"demon":5698,"sugar":5699,"earliest":5700,"colony":5701,"studying":5702,"balance":5703,"duties":5704,"##ks":5705,"belgium":5706,"slipped":5707,"carter":5708,"05":5709,"visible":5710,"stages":5711,"iraq":5712,"fifa":5713,"##im":5714,"commune":5715,"forming":5716,"zero":5717,"07":5718,"continuing":5719,"talked":5720,"counties":5721,"legend":5722,"bathroom":5723,"option":5724,"tail":5725,"clay":5726,"daughters":5727,"afterwards":5728,"severe":5729,"jaw":5730,"visitors":5731,"##ded":5732,"devices":5733,"aviation":5734,"russell":5735,"kate":5736,"##vi":5737,"entering":5738,"subjects":5739,"##ino":5740,"temporary":5741,"swimming":5742,"forth":5743,"smooth":5744,"ghost":5745,"audio":5746,"bush":5747,"operates":5748,"rocks":5749,"movements":5750,"signs":5751,"eddie":5752,"##tz":5753,"ann":5754,"voices":5755,"honorary":5756,"06":5757,"memories":5758,"dallas":5759,"pure":5760,"measures":5761,"racial":5762,"promised":5763,"66":5764,"harvard":5765,"ceo":5766,"16th":5767,"parliamentary":5768,"indicate":5769,"benefit":5770,"flesh":5771,"dublin":5772,"louisiana":5773,"1902":5774,"1901":5775,"patient":5776,"sleeping":5777,"1903":5778,"membership":5779,"coastal":5780,"medieval":5781,"wanting":5782,"element":5783,"scholars":5784,"rice":5785,"62":5786,"limit":5787,"survive":5788,"makeup":5789,"rating":5790,"definitely":5791,"collaboration":5792,"obvious":5793,"##tan":5794,"boss":5795,"ms":5796,"baron":5797,"birthday":5798,"linked":5799,"soil":5800,"diocese":5801,"##lan":5802,"ncaa":5803,"##mann":5804,"offensive":5805,"shell":5806,"shouldn":5807,"waist":5808,"##tus":5809,"plain":5810,"ross":5811,"organ":5812,"resolution":5813,"manufacturing":5814,"adding":5815,"relative":5816,"kennedy":5817,"98":5818,"whilst":5819,"moth":5820,"marketing":5821,"gardens":5822,"crash":5823,"72":5824,"heading":5825,"partners":5826,"credited":5827,"carlos":5828,"moves":5829,"cable":5830,"##zi":5831,"marshall":5832,"##out":5833,"depending":5834,"bottle":5835,"represents":5836,"rejected":5837,"responded":5838,"existed":5839,"04":5840,"jobs":5841,"denmark":5842,"lock":5843,"##ating":5844,"treated":5845,"graham":5846,"routes":5847,"talent":5848,"commissioner":5849,"drugs":5850,"secure":5851,"tests":5852,"reign":5853,"restored":5854,"photography":5855,"##gi":5856,"contributions":5857,"oklahoma":5858,"designer":5859,"disc":5860,"grin":5861,"seattle":5862,"robin":5863,"paused":5864,"atlanta":5865,"unusual":5866,"##gate":5867,"praised":5868,"las":5869,"laughing":5870,"satellite":5871,"hungary":5872,"visiting":5873,"##sky":5874,"interesting":5875,"factors":5876,"deck":5877,"poems":5878,"norman":5879,"##water":5880,"stuck":5881,"speaker":5882,"rifle":5883,"domain":5884,"premiered":5885,"##her":5886,"dc":5887,"comics":5888,"actors":5889,"01":5890,"reputation":5891,"eliminated":5892,"8th":5893,"ceiling":5894,"prisoners":5895,"script":5896,"##nce":5897,"leather":5898,"austin":5899,"mississippi":5900,"rapidly":5901,"admiral":5902,"parallel":5903,"charlotte":5904,"guilty":5905,"tools":5906,"gender":5907,"divisions":5908,"fruit":5909,"##bs":5910,"laboratory":5911,"nelson":5912,"fantasy":5913,"marry":5914,"rapid":5915,"aunt":5916,"tribe":5917,"requirements":5918,"aspects":5919,"suicide":5920,"amongst":5921,"adams":5922,"bone":5923,"ukraine":5924,"abc":5925,"kick":5926,"sees":5927,"edinburgh":5928,"clothing":5929,"column":5930,"rough":5931,"gods":5932,"hunting":5933,"broadway":5934,"gathered":5935,"concerns":5936,"##ek":5937,"spending":5938,"ty":5939,"12th":5940,"snapped":5941,"requires":5942,"solar":5943,"bones":5944,"cavalry":5945,"##tta":5946,"iowa":5947,"drinking":5948,"waste":5949,"index":5950,"franklin":5951,"charity":5952,"thompson":5953,"stewart":5954,"tip":5955,"flash":5956,"landscape":5957,"friday":5958,"enjoy":5959,"singh":5960,"poem":5961,"listening":5962,"##back":5963,"eighth":5964,"fred":5965,"differences":5966,"adapted":5967,"bomb":5968,"ukrainian":5969,"surgery":5970,"corporate":5971,"masters":5972,"anywhere":5973,"##more":5974,"waves":5975,"odd":5976,"sean":5977,"portugal":5978,"orleans":5979,"dick":5980,"debate":5981,"kent":5982,"eating":5983,"puerto":5984,"cleared":5985,"96":5986,"expect":5987,"cinema":5988,"97":5989,"guitarist":5990,"blocks":5991,"electrical":5992,"agree":5993,"involving":5994,"depth":5995,"dying":5996,"panel":5997,"struggle":5998,"##ged":5999,"peninsula":6000,"adults":6001,"novels":6002,"emerged":6003,"vienna":6004,"metro":6005,"debuted":6006,"shoes":6007,"tamil":6008,"songwriter":6009,"meets":6010,"prove":6011,"beating":6012,"instance":6013,"heaven":6014,"scared":6015,"sending":6016,"marks":6017,"artistic":6018,"passage":6019,"superior":6020,"03":6021,"significantly":6022,"shopping":6023,"##tive":6024,"retained":6025,"##izing":6026,"malaysia":6027,"technique":6028,"cheeks":6029,"##ola":6030,"warren":6031,"maintenance":6032,"destroy":6033,"extreme":6034,"allied":6035,"120":6036,"appearing":6037,"##yn":6038,"fill":6039,"advice":6040,"alabama":6041,"qualifying":6042,"policies":6043,"cleveland":6044,"hat":6045,"battery":6046,"smart":6047,"authors":6048,"10th":6049,"soundtrack":6050,"acted":6051,"dated":6052,"lb":6053,"glance":6054,"equipped":6055,"coalition":6056,"funny":6057,"outer":6058,"ambassador":6059,"roy":6060,"possibility":6061,"couples":6062,"campbell":6063,"dna":6064,"loose":6065,"ethan":6066,"supplies":6067,"1898":6068,"gonna":6069,"88":6070,"monster":6071,"##res":6072,"shake":6073,"agents":6074,"frequency":6075,"springs":6076,"dogs":6077,"practices":6078,"61":6079,"gang":6080,"plastic":6081,"easier":6082,"suggests":6083,"gulf":6084,"blade":6085,"exposed":6086,"colors":6087,"industries":6088,"markets":6089,"pan":6090,"nervous":6091,"electoral":6092,"charts":6093,"legislation":6094,"ownership":6095,"##idae":6096,"mac":6097,"appointment":6098,"shield":6099,"copy":6100,"assault":6101,"socialist":6102,"abbey":6103,"monument":6104,"license":6105,"throne":6106,"employment":6107,"jay":6108,"93":6109,"replacement":6110,"charter":6111,"cloud":6112,"powered":6113,"suffering":6114,"accounts":6115,"oak":6116,"connecticut":6117,"strongly":6118,"wright":6119,"colour":6120,"crystal":6121,"13th":6122,"context":6123,"welsh":6124,"networks":6125,"voiced":6126,"gabriel":6127,"jerry":6128,"##cing":6129,"forehead":6130,"mp":6131,"##ens":6132,"manage":6133,"schedule":6134,"totally":6135,"remix":6136,"##ii":6137,"forests":6138,"occupation":6139,"print":6140,"nicholas":6141,"brazilian":6142,"strategic":6143,"vampires":6144,"engineers":6145,"76":6146,"roots":6147,"seek":6148,"correct":6149,"instrumental":6150,"und":6151,"alfred":6152,"backed":6153,"hop":6154,"##des":6155,"stanley":6156,"robinson":6157,"traveled":6158,"wayne":6159,"welcome":6160,"austrian":6161,"achieve":6162,"67":6163,"exit":6164,"rates":6165,"1899":6166,"strip":6167,"whereas":6168,"##cs":6169,"sing":6170,"deeply":6171,"adventure":6172,"bobby":6173,"rick":6174,"jamie":6175,"careful":6176,"components":6177,"cap":6178,"useful":6179,"personality":6180,"knee":6181,"##shi":6182,"pushing":6183,"hosts":6184,"02":6185,"protest":6186,"ca":6187,"ottoman":6188,"symphony":6189,"##sis":6190,"63":6191,"boundary":6192,"1890":6193,"processes":6194,"considering":6195,"considerable":6196,"tons":6197,"##work":6198,"##ft":6199,"##nia":6200,"cooper":6201,"trading":6202,"dear":6203,"conduct":6204,"91":6205,"illegal":6206,"apple":6207,"revolutionary":6208,"holiday":6209,"definition":6210,"harder":6211,"##van":6212,"jacob":6213,"circumstances":6214,"destruction":6215,"##lle":6216,"popularity":6217,"grip":6218,"classified":6219,"liverpool":6220,"donald":6221,"baltimore":6222,"flows":6223,"seeking":6224,"honour":6225,"approval":6226,"92":6227,"mechanical":6228,"till":6229,"happening":6230,"statue":6231,"critic":6232,"increasingly":6233,"immediate":6234,"describe":6235,"commerce":6236,"stare":6237,"##ster":6238,"indonesia":6239,"meat":6240,"rounds":6241,"boats":6242,"baker":6243,"orthodox":6244,"depression":6245,"formally":6246,"worn":6247,"naked":6248,"claire":6249,"muttered":6250,"sentence":6251,"11th":6252,"emily":6253,"document":6254,"77":6255,"criticism":6256,"wished":6257,"vessel":6258,"spiritual":6259,"bent":6260,"virgin":6261,"parker":6262,"minimum":6263,"murray":6264,"lunch":6265,"danny":6266,"printed":6267,"compilation":6268,"keyboards":6269,"false":6270,"blow":6271,"belonged":6272,"68":6273,"raising":6274,"78":6275,"cutting":6276,"##board":6277,"pittsburgh":6278,"##up":6279,"9th":6280,"shadows":6281,"81":6282,"hated":6283,"indigenous":6284,"jon":6285,"15th":6286,"barry":6287,"scholar":6288,"ah":6289,"##zer":6290,"oliver":6291,"##gy":6292,"stick":6293,"susan":6294,"meetings":6295,"attracted":6296,"spell":6297,"romantic":6298,"##ver":6299,"ye":6300,"1895":6301,"photo":6302,"demanded":6303,"customers":6304,"##ac":6305,"1896":6306,"logan":6307,"revival":6308,"keys":6309,"modified":6310,"commanded":6311,"jeans":6312,"##ious":6313,"upset":6314,"raw":6315,"phil":6316,"detective":6317,"hiding":6318,"resident":6319,"vincent":6320,"##bly":6321,"experiences":6322,"diamond":6323,"defeating":6324,"coverage":6325,"lucas":6326,"external":6327,"parks":6328,"franchise":6329,"helen":6330,"bible":6331,"successor":6332,"percussion":6333,"celebrated":6334,"il":6335,"lift":6336,"profile":6337,"clan":6338,"romania":6339,"##ied":6340,"mills":6341,"##su":6342,"nobody":6343,"achievement":6344,"shrugged":6345,"fault":6346,"1897":6347,"rhythm":6348,"initiative":6349,"breakfast":6350,"carbon":6351,"700":6352,"69":6353,"lasted":6354,"violent":6355,"74":6356,"wound":6357,"ken":6358,"killer":6359,"gradually":6360,"filmed":6361,"°c":6362,"dollars":6363,"processing":6364,"94":6365,"remove":6366,"criticized":6367,"guests":6368,"sang":6369,"chemistry":6370,"##vin":6371,"legislature":6372,"disney":6373,"##bridge":6374,"uniform":6375,"escaped":6376,"integrated":6377,"proposal":6378,"purple":6379,"denied":6380,"liquid":6381,"karl":6382,"influential":6383,"morris":6384,"nights":6385,"stones":6386,"intense":6387,"experimental":6388,"twisted":6389,"71":6390,"84":6391,"##ld":6392,"pace":6393,"nazi":6394,"mitchell":6395,"ny":6396,"blind":6397,"reporter":6398,"newspapers":6399,"14th":6400,"centers":6401,"burn":6402,"basin":6403,"forgotten":6404,"surviving":6405,"filed":6406,"collections":6407,"monastery":6408,"losses":6409,"manual":6410,"couch":6411,"description":6412,"appropriate":6413,"merely":6414,"tag":6415,"missions":6416,"sebastian":6417,"restoration":6418,"replacing":6419,"triple":6420,"73":6421,"elder":6422,"julia":6423,"warriors":6424,"benjamin":6425,"julian":6426,"convinced":6427,"stronger":6428,"amazing":6429,"declined":6430,"versus":6431,"merchant":6432,"happens":6433,"output":6434,"finland":6435,"bare":6436,"barbara":6437,"absence":6438,"ignored":6439,"dawn":6440,"injuries":6441,"##port":6442,"producers":6443,"##ram":6444,"82":6445,"luis":6446,"##ities":6447,"kw":6448,"admit":6449,"expensive":6450,"electricity":6451,"nba":6452,"exception":6453,"symbol":6454,"##ving":6455,"ladies":6456,"shower":6457,"sheriff":6458,"characteristics":6459,"##je":6460,"aimed":6461,"button":6462,"ratio":6463,"effectively":6464,"summit":6465,"angle":6466,"jury":6467,"bears":6468,"foster":6469,"vessels":6470,"pants":6471,"executed":6472,"evans":6473,"dozen":6474,"advertising":6475,"kicked":6476,"patrol":6477,"1889":6478,"competitions":6479,"lifetime":6480,"principles":6481,"athletics":6482,"##logy":6483,"birmingham":6484,"sponsored":6485,"89":6486,"rob":6487,"nomination":6488,"1893":6489,"acoustic":6490,"##sm":6491,"creature":6492,"longest":6493,"##tra":6494,"credits":6495,"harbor":6496,"dust":6497,"josh":6498,"##so":6499,"territories":6500,"milk":6501,"infrastructure":6502,"completion":6503,"thailand":6504,"indians":6505,"leon":6506,"archbishop":6507,"##sy":6508,"assist":6509,"pitch":6510,"blake":6511,"arrangement":6512,"girlfriend":6513,"serbian":6514,"operational":6515,"hence":6516,"sad":6517,"scent":6518,"fur":6519,"dj":6520,"sessions":6521,"hp":6522,"refer":6523,"rarely":6524,"##ora":6525,"exists":6526,"1892":6527,"##ten":6528,"scientists":6529,"dirty":6530,"penalty":6531,"burst":6532,"portrait":6533,"seed":6534,"79":6535,"pole":6536,"limits":6537,"rival":6538,"1894":6539,"stable":6540,"alpha":6541,"grave":6542,"constitutional":6543,"alcohol":6544,"arrest":6545,"flower":6546,"mystery":6547,"devil":6548,"architectural":6549,"relationships":6550,"greatly":6551,"habitat":6552,"##istic":6553,"larry":6554,"progressive":6555,"remote":6556,"cotton":6557,"##ics":6558,"##ok":6559,"preserved":6560,"reaches":6561,"##ming":6562,"cited":6563,"86":6564,"vast":6565,"scholarship":6566,"decisions":6567,"cbs":6568,"joy":6569,"teach":6570,"1885":6571,"editions":6572,"knocked":6573,"eve":6574,"searching":6575,"partly":6576,"participation":6577,"gap":6578,"animated":6579,"fate":6580,"excellent":6581,"##ett":6582,"na":6583,"87":6584,"alternate":6585,"saints":6586,"youngest":6587,"##ily":6588,"climbed":6589,"##ita":6590,"##tors":6591,"suggest":6592,"##ct":6593,"discussion":6594,"staying":6595,"choir":6596,"lakes":6597,"jacket":6598,"revenue":6599,"nevertheless":6600,"peaked":6601,"instrument":6602,"wondering":6603,"annually":6604,"managing":6605,"neil":6606,"1891":6607,"signing":6608,"terry":6609,"##ice":6610,"apply":6611,"clinical":6612,"brooklyn":6613,"aim":6614,"catherine":6615,"fuck":6616,"farmers":6617,"figured":6618,"ninth":6619,"pride":6620,"hugh":6621,"evolution":6622,"ordinary":6623,"involvement":6624,"comfortable":6625,"shouted":6626,"tech":6627,"encouraged":6628,"taiwan":6629,"representation":6630,"sharing":6631,"##lia":6632,"##em":6633,"panic":6634,"exact":6635,"cargo":6636,"competing":6637,"fat":6638,"cried":6639,"83":6640,"1920s":6641,"occasions":6642,"pa":6643,"cabin":6644,"borders":6645,"utah":6646,"marcus":6647,"##isation":6648,"badly":6649,"muscles":6650,"##ance":6651,"victorian":6652,"transition":6653,"warner":6654,"bet":6655,"permission":6656,"##rin":6657,"slave":6658,"terrible":6659,"similarly":6660,"shares":6661,"seth":6662,"uefa":6663,"possession":6664,"medals":6665,"benefits":6666,"colleges":6667,"lowered":6668,"perfectly":6669,"mall":6670,"transit":6671,"##ye":6672,"##kar":6673,"publisher":6674,"##ened":6675,"harrison":6676,"deaths":6677,"elevation":6678,"##ae":6679,"asleep":6680,"machines":6681,"sigh":6682,"ash":6683,"hardly":6684,"argument":6685,"occasion":6686,"parent":6687,"leo":6688,"decline":6689,"1888":6690,"contribution":6691,"##ua":6692,"concentration":6693,"1000":6694,"opportunities":6695,"hispanic":6696,"guardian":6697,"extent":6698,"emotions":6699,"hips":6700,"mason":6701,"volumes":6702,"bloody":6703,"controversy":6704,"diameter":6705,"steady":6706,"mistake":6707,"phoenix":6708,"identify":6709,"violin":6710,"##sk":6711,"departure":6712,"richmond":6713,"spin":6714,"funeral":6715,"enemies":6716,"1864":6717,"gear":6718,"literally":6719,"connor":6720,"random":6721,"sergeant":6722,"grab":6723,"confusion":6724,"1865":6725,"transmission":6726,"informed":6727,"op":6728,"leaning":6729,"sacred":6730,"suspended":6731,"thinks":6732,"gates":6733,"portland":6734,"luck":6735,"agencies":6736,"yours":6737,"hull":6738,"expert":6739,"muscle":6740,"layer":6741,"practical":6742,"sculpture":6743,"jerusalem":6744,"latest":6745,"lloyd":6746,"statistics":6747,"deeper":6748,"recommended":6749,"warrior":6750,"arkansas":6751,"mess":6752,"supports":6753,"greg":6754,"eagle":6755,"1880":6756,"recovered":6757,"rated":6758,"concerts":6759,"rushed":6760,"##ano":6761,"stops":6762,"eggs":6763,"files":6764,"premiere":6765,"keith":6766,"##vo":6767,"delhi":6768,"turner":6769,"pit":6770,"affair":6771,"belief":6772,"paint":6773,"##zing":6774,"mate":6775,"##ach":6776,"##ev":6777,"victim":6778,"##ology":6779,"withdrew":6780,"bonus":6781,"styles":6782,"fled":6783,"##ud":6784,"glasgow":6785,"technologies":6786,"funded":6787,"nbc":6788,"adaptation":6789,"##ata":6790,"portrayed":6791,"cooperation":6792,"supporters":6793,"judges":6794,"bernard":6795,"justin":6796,"hallway":6797,"ralph":6798,"##ick":6799,"graduating":6800,"controversial":6801,"distant":6802,"continental":6803,"spider":6804,"bite":6805,"##ho":6806,"recognize":6807,"intention":6808,"mixing":6809,"##ese":6810,"egyptian":6811,"bow":6812,"tourism":6813,"suppose":6814,"claiming":6815,"tiger":6816,"dominated":6817,"participants":6818,"vi":6819,"##ru":6820,"nurse":6821,"partially":6822,"tape":6823,"##rum":6824,"psychology":6825,"##rn":6826,"essential":6827,"touring":6828,"duo":6829,"voting":6830,"civilian":6831,"emotional":6832,"channels":6833,"##king":6834,"apparent":6835,"hebrew":6836,"1887":6837,"tommy":6838,"carrier":6839,"intersection":6840,"beast":6841,"hudson":6842,"##gar":6843,"##zo":6844,"lab":6845,"nova":6846,"bench":6847,"discuss":6848,"costa":6849,"##ered":6850,"detailed":6851,"behalf":6852,"drivers":6853,"unfortunately":6854,"obtain":6855,"##lis":6856,"rocky":6857,"##dae":6858,"siege":6859,"friendship":6860,"honey":6861,"##rian":6862,"1861":6863,"amy":6864,"hang":6865,"posted":6866,"governments":6867,"collins":6868,"respond":6869,"wildlife":6870,"preferred":6871,"operator":6872,"##po":6873,"laura":6874,"pregnant":6875,"videos":6876,"dennis":6877,"suspected":6878,"boots":6879,"instantly":6880,"weird":6881,"automatic":6882,"businessman":6883,"alleged":6884,"placing":6885,"throwing":6886,"ph":6887,"mood":6888,"1862":6889,"perry":6890,"venue":6891,"jet":6892,"remainder":6893,"##lli":6894,"##ci":6895,"passion":6896,"biological":6897,"boyfriend":6898,"1863":6899,"dirt":6900,"buffalo":6901,"ron":6902,"segment":6903,"fa":6904,"abuse":6905,"##era":6906,"genre":6907,"thrown":6908,"stroke":6909,"colored":6910,"stress":6911,"exercise":6912,"displayed":6913,"##gen":6914,"struggled":6915,"##tti":6916,"abroad":6917,"dramatic":6918,"wonderful":6919,"thereafter":6920,"madrid":6921,"component":6922,"widespread":6923,"##sed":6924,"tale":6925,"citizen":6926,"todd":6927,"monday":6928,"1886":6929,"vancouver":6930,"overseas":6931,"forcing":6932,"crying":6933,"descent":6934,"##ris":6935,"discussed":6936,"substantial":6937,"ranks":6938,"regime":6939,"1870":6940,"provinces":6941,"switch":6942,"drum":6943,"zane":6944,"ted":6945,"tribes":6946,"proof":6947,"lp":6948,"cream":6949,"researchers":6950,"volunteer":6951,"manor":6952,"silk":6953,"milan":6954,"donated":6955,"allies":6956,"venture":6957,"principle":6958,"delivery":6959,"enterprise":6960,"##ves":6961,"##ans":6962,"bars":6963,"traditionally":6964,"witch":6965,"reminded":6966,"copper":6967,"##uk":6968,"pete":6969,"inter":6970,"links":6971,"colin":6972,"grinned":6973,"elsewhere":6974,"competitive":6975,"frequent":6976,"##oy":6977,"scream":6978,"##hu":6979,"tension":6980,"texts":6981,"submarine":6982,"finnish":6983,"defending":6984,"defend":6985,"pat":6986,"detail":6987,"1884":6988,"affiliated":6989,"stuart":6990,"themes":6991,"villa":6992,"periods":6993,"tool":6994,"belgian":6995,"ruling":6996,"crimes":6997,"answers":6998,"folded":6999,"licensed":7000,"resort":7001,"demolished":7002,"hans":7003,"lucy":7004,"1881":7005,"lion":7006,"traded":7007,"photographs":7008,"writes":7009,"craig":7010,"##fa":7011,"trials":7012,"generated":7013,"beth":7014,"noble":7015,"debt":7016,"percentage":7017,"yorkshire":7018,"erected":7019,"ss":7020,"viewed":7021,"grades":7022,"confidence":7023,"ceased":7024,"islam":7025,"telephone":7026,"retail":7027,"##ible":7028,"chile":7029,"m²":7030,"roberts":7031,"sixteen":7032,"##ich":7033,"commented":7034,"hampshire":7035,"innocent":7036,"dual":7037,"pounds":7038,"checked":7039,"regulations":7040,"afghanistan":7041,"sung":7042,"rico":7043,"liberty":7044,"assets":7045,"bigger":7046,"options":7047,"angels":7048,"relegated":7049,"tribute":7050,"wells":7051,"attending":7052,"leaf":7053,"##yan":7054,"butler":7055,"romanian":7056,"forum":7057,"monthly":7058,"lisa":7059,"patterns":7060,"gmina":7061,"##tory":7062,"madison":7063,"hurricane":7064,"rev":7065,"##ians":7066,"bristol":7067,"##ula":7068,"elite":7069,"valuable":7070,"disaster":7071,"democracy":7072,"awareness":7073,"germans":7074,"freyja":7075,"##ins":7076,"loop":7077,"absolutely":7078,"paying":7079,"populations":7080,"maine":7081,"sole":7082,"prayer":7083,"spencer":7084,"releases":7085,"doorway":7086,"bull":7087,"##ani":7088,"lover":7089,"midnight":7090,"conclusion":7091,"##sson":7092,"thirteen":7093,"lily":7094,"mediterranean":7095,"##lt":7096,"nhl":7097,"proud":7098,"sample":7099,"##hill":7100,"drummer":7101,"guinea":7102,"##ova":7103,"murphy":7104,"climb":7105,"##ston":7106,"instant":7107,"attributed":7108,"horn":7109,"ain":7110,"railways":7111,"steven":7112,"##ao":7113,"autumn":7114,"ferry":7115,"opponent":7116,"root":7117,"traveling":7118,"secured":7119,"corridor":7120,"stretched":7121,"tales":7122,"sheet":7123,"trinity":7124,"cattle":7125,"helps":7126,"indicates":7127,"manhattan":7128,"murdered":7129,"fitted":7130,"1882":7131,"gentle":7132,"grandmother":7133,"mines":7134,"shocked":7135,"vegas":7136,"produces":7137,"##light":7138,"caribbean":7139,"##ou":7140,"belong":7141,"continuous":7142,"desperate":7143,"drunk":7144,"historically":7145,"trio":7146,"waved":7147,"raf":7148,"dealing":7149,"nathan":7150,"bat":7151,"murmured":7152,"interrupted":7153,"residing":7154,"scientist":7155,"pioneer":7156,"harold":7157,"aaron":7158,"##net":7159,"delta":7160,"attempting":7161,"minority":7162,"mini":7163,"believes":7164,"chorus":7165,"tend":7166,"lots":7167,"eyed":7168,"indoor":7169,"load":7170,"shots":7171,"updated":7172,"jail":7173,"##llo":7174,"concerning":7175,"connecting":7176,"wealth":7177,"##ved":7178,"slaves":7179,"arrive":7180,"rangers":7181,"sufficient":7182,"rebuilt":7183,"##wick":7184,"cardinal":7185,"flood":7186,"muhammad":7187,"whenever":7188,"relation":7189,"runners":7190,"moral":7191,"repair":7192,"viewers":7193,"arriving":7194,"revenge":7195,"punk":7196,"assisted":7197,"bath":7198,"fairly":7199,"breathe":7200,"lists":7201,"innings":7202,"illustrated":7203,"whisper":7204,"nearest":7205,"voters":7206,"clinton":7207,"ties":7208,"ultimate":7209,"screamed":7210,"beijing":7211,"lions":7212,"andre":7213,"fictional":7214,"gathering":7215,"comfort":7216,"radar":7217,"suitable":7218,"dismissed":7219,"hms":7220,"ban":7221,"pine":7222,"wrist":7223,"atmosphere":7224,"voivodeship":7225,"bid":7226,"timber":7227,"##ned":7228,"##nan":7229,"giants":7230,"##ane":7231,"cameron":7232,"recovery":7233,"uss":7234,"identical":7235,"categories":7236,"switched":7237,"serbia":7238,"laughter":7239,"noah":7240,"ensemble":7241,"therapy":7242,"peoples":7243,"touching":7244,"##off":7245,"locally":7246,"pearl":7247,"platforms":7248,"everywhere":7249,"ballet":7250,"tables":7251,"lanka":7252,"herbert":7253,"outdoor":7254,"toured":7255,"derek":7256,"1883":7257,"spaces":7258,"contested":7259,"swept":7260,"1878":7261,"exclusive":7262,"slight":7263,"connections":7264,"##dra":7265,"winds":7266,"prisoner":7267,"collective":7268,"bangladesh":7269,"tube":7270,"publicly":7271,"wealthy":7272,"thai":7273,"##ys":7274,"isolated":7275,"select":7276,"##ric":7277,"insisted":7278,"pen":7279,"fortune":7280,"ticket":7281,"spotted":7282,"reportedly":7283,"animation":7284,"enforcement":7285,"tanks":7286,"110":7287,"decides":7288,"wider":7289,"lowest":7290,"owen":7291,"##time":7292,"nod":7293,"hitting":7294,"##hn":7295,"gregory":7296,"furthermore":7297,"magazines":7298,"fighters":7299,"solutions":7300,"##ery":7301,"pointing":7302,"requested":7303,"peru":7304,"reed":7305,"chancellor":7306,"knights":7307,"mask":7308,"worker":7309,"eldest":7310,"flames":7311,"reduction":7312,"1860":7313,"volunteers":7314,"##tis":7315,"reporting":7316,"##hl":7317,"wire":7318,"advisory":7319,"endemic":7320,"origins":7321,"settlers":7322,"pursue":7323,"knock":7324,"consumer":7325,"1876":7326,"eu":7327,"compound":7328,"creatures":7329,"mansion":7330,"sentenced":7331,"ivan":7332,"deployed":7333,"guitars":7334,"frowned":7335,"involves":7336,"mechanism":7337,"kilometers":7338,"perspective":7339,"shops":7340,"maps":7341,"terminus":7342,"duncan":7343,"alien":7344,"fist":7345,"bridges":7346,"##pers":7347,"heroes":7348,"fed":7349,"derby":7350,"swallowed":7351,"##ros":7352,"patent":7353,"sara":7354,"illness":7355,"characterized":7356,"adventures":7357,"slide":7358,"hawaii":7359,"jurisdiction":7360,"##op":7361,"organised":7362,"##side":7363,"adelaide":7364,"walks":7365,"biology":7366,"se":7367,"##ties":7368,"rogers":7369,"swing":7370,"tightly":7371,"boundaries":7372,"##rie":7373,"prepare":7374,"implementation":7375,"stolen":7376,"##sha":7377,"certified":7378,"colombia":7379,"edwards":7380,"garage":7381,"##mm":7382,"recalled":7383,"##ball":7384,"rage":7385,"harm":7386,"nigeria":7387,"breast":7388,"##ren":7389,"furniture":7390,"pupils":7391,"settle":7392,"##lus":7393,"cuba":7394,"balls":7395,"client":7396,"alaska":7397,"21st":7398,"linear":7399,"thrust":7400,"celebration":7401,"latino":7402,"genetic":7403,"terror":7404,"##cia":7405,"##ening":7406,"lightning":7407,"fee":7408,"witness":7409,"lodge":7410,"establishing":7411,"skull":7412,"##ique":7413,"earning":7414,"hood":7415,"##ei":7416,"rebellion":7417,"wang":7418,"sporting":7419,"warned":7420,"missile":7421,"devoted":7422,"activist":7423,"porch":7424,"worship":7425,"fourteen":7426,"package":7427,"1871":7428,"decorated":7429,"##shire":7430,"housed":7431,"##ock":7432,"chess":7433,"sailed":7434,"doctors":7435,"oscar":7436,"joan":7437,"treat":7438,"garcia":7439,"harbour":7440,"jeremy":7441,"##ire":7442,"traditions":7443,"dominant":7444,"jacques":7445,"##gon":7446,"##wan":7447,"relocated":7448,"1879":7449,"amendment":7450,"sized":7451,"companion":7452,"simultaneously":7453,"volleyball":7454,"spun":7455,"acre":7456,"increases":7457,"stopping":7458,"loves":7459,"belongs":7460,"affect":7461,"drafted":7462,"tossed":7463,"scout":7464,"battles":7465,"1875":7466,"filming":7467,"shoved":7468,"munich":7469,"tenure":7470,"vertical":7471,"romance":7472,"pc":7473,"##cher":7474,"argue":7475,"##ical":7476,"craft":7477,"ranging":7478,"www":7479,"opens":7480,"honest":7481,"tyler":7482,"yesterday":7483,"virtual":7484,"##let":7485,"muslims":7486,"reveal":7487,"snake":7488,"immigrants":7489,"radical":7490,"screaming":7491,"speakers":7492,"firing":7493,"saving":7494,"belonging":7495,"ease":7496,"lighting":7497,"prefecture":7498,"blame":7499,"farmer":7500,"hungry":7501,"grows":7502,"rubbed":7503,"beam":7504,"sur":7505,"subsidiary":7506,"##cha":7507,"armenian":7508,"sao":7509,"dropping":7510,"conventional":7511,"##fer":7512,"microsoft":7513,"reply":7514,"qualify":7515,"spots":7516,"1867":7517,"sweat":7518,"festivals":7519,"##ken":7520,"immigration":7521,"physician":7522,"discover":7523,"exposure":7524,"sandy":7525,"explanation":7526,"isaac":7527,"implemented":7528,"##fish":7529,"hart":7530,"initiated":7531,"connect":7532,"stakes":7533,"presents":7534,"heights":7535,"householder":7536,"pleased":7537,"tourist":7538,"regardless":7539,"slip":7540,"closest":7541,"##ction":7542,"surely":7543,"sultan":7544,"brings":7545,"riley":7546,"preparation":7547,"aboard":7548,"slammed":7549,"baptist":7550,"experiment":7551,"ongoing":7552,"interstate":7553,"organic":7554,"playoffs":7555,"##ika":7556,"1877":7557,"130":7558,"##tar":7559,"hindu":7560,"error":7561,"tours":7562,"tier":7563,"plenty":7564,"arrangements":7565,"talks":7566,"trapped":7567,"excited":7568,"sank":7569,"ho":7570,"athens":7571,"1872":7572,"denver":7573,"welfare":7574,"suburb":7575,"athletes":7576,"trick":7577,"diverse":7578,"belly":7579,"exclusively":7580,"yelled":7581,"1868":7582,"##med":7583,"conversion":7584,"##ette":7585,"1874":7586,"internationally":7587,"computers":7588,"conductor":7589,"abilities":7590,"sensitive":7591,"hello":7592,"dispute":7593,"measured":7594,"globe":7595,"rocket":7596,"prices":7597,"amsterdam":7598,"flights":7599,"tigers":7600,"inn":7601,"municipalities":7602,"emotion":7603,"references":7604,"3d":7605,"##mus":7606,"explains":7607,"airlines":7608,"manufactured":7609,"pm":7610,"archaeological":7611,"1873":7612,"interpretation":7613,"devon":7614,"comment":7615,"##ites":7616,"settlements":7617,"kissing":7618,"absolute":7619,"improvement":7620,"suite":7621,"impressed":7622,"barcelona":7623,"sullivan":7624,"jefferson":7625,"towers":7626,"jesse":7627,"julie":7628,"##tin":7629,"##lu":7630,"grandson":7631,"hi":7632,"gauge":7633,"regard":7634,"rings":7635,"interviews":7636,"trace":7637,"raymond":7638,"thumb":7639,"departments":7640,"burns":7641,"serial":7642,"bulgarian":7643,"scores":7644,"demonstrated":7645,"##ix":7646,"1866":7647,"kyle":7648,"alberta":7649,"underneath":7650,"romanized":7651,"##ward":7652,"relieved":7653,"acquisition":7654,"phrase":7655,"cliff":7656,"reveals":7657,"han":7658,"cuts":7659,"merger":7660,"custom":7661,"##dar":7662,"nee":7663,"gilbert":7664,"graduation":7665,"##nts":7666,"assessment":7667,"cafe":7668,"difficulty":7669,"demands":7670,"swung":7671,"democrat":7672,"jennifer":7673,"commons":7674,"1940s":7675,"grove":7676,"##yo":7677,"completing":7678,"focuses":7679,"sum":7680,"substitute":7681,"bearing":7682,"stretch":7683,"reception":7684,"##py":7685,"reflected":7686,"essentially":7687,"destination":7688,"pairs":7689,"##ched":7690,"survival":7691,"resource":7692,"##bach":7693,"promoting":7694,"doubles":7695,"messages":7696,"tear":7697,"##down":7698,"##fully":7699,"parade":7700,"florence":7701,"harvey":7702,"incumbent":7703,"partial":7704,"framework":7705,"900":7706,"pedro":7707,"frozen":7708,"procedure":7709,"olivia":7710,"controls":7711,"##mic":7712,"shelter":7713,"personally":7714,"temperatures":7715,"##od":7716,"brisbane":7717,"tested":7718,"sits":7719,"marble":7720,"comprehensive":7721,"oxygen":7722,"leonard":7723,"##kov":7724,"inaugural":7725,"iranian":7726,"referring":7727,"quarters":7728,"attitude":7729,"##ivity":7730,"mainstream":7731,"lined":7732,"mars":7733,"dakota":7734,"norfolk":7735,"unsuccessful":7736,"##°":7737,"explosion":7738,"helicopter":7739,"congressional":7740,"##sing":7741,"inspector":7742,"bitch":7743,"seal":7744,"departed":7745,"divine":7746,"##ters":7747,"coaching":7748,"examination":7749,"punishment":7750,"manufacturer":7751,"sink":7752,"columns":7753,"unincorporated":7754,"signals":7755,"nevada":7756,"squeezed":7757,"dylan":7758,"dining":7759,"photos":7760,"martial":7761,"manuel":7762,"eighteen":7763,"elevator":7764,"brushed":7765,"plates":7766,"ministers":7767,"ivy":7768,"congregation":7769,"##len":7770,"slept":7771,"specialized":7772,"taxes":7773,"curve":7774,"restricted":7775,"negotiations":7776,"likes":7777,"statistical":7778,"arnold":7779,"inspiration":7780,"execution":7781,"bold":7782,"intermediate":7783,"significance":7784,"margin":7785,"ruler":7786,"wheels":7787,"gothic":7788,"intellectual":7789,"dependent":7790,"listened":7791,"eligible":7792,"buses":7793,"widow":7794,"syria":7795,"earn":7796,"cincinnati":7797,"collapsed":7798,"recipient":7799,"secrets":7800,"accessible":7801,"philippine":7802,"maritime":7803,"goddess":7804,"clerk":7805,"surrender":7806,"breaks":7807,"playoff":7808,"database":7809,"##ified":7810,"##lon":7811,"ideal":7812,"beetle":7813,"aspect":7814,"soap":7815,"regulation":7816,"strings":7817,"expand":7818,"anglo":7819,"shorter":7820,"crosses":7821,"retreat":7822,"tough":7823,"coins":7824,"wallace":7825,"directions":7826,"pressing":7827,"##oon":7828,"shipping":7829,"locomotives":7830,"comparison":7831,"topics":7832,"nephew":7833,"##mes":7834,"distinction":7835,"honors":7836,"travelled":7837,"sierra":7838,"ibn":7839,"##over":7840,"fortress":7841,"sa":7842,"recognised":7843,"carved":7844,"1869":7845,"clients":7846,"##dan":7847,"intent":7848,"##mar":7849,"coaches":7850,"describing":7851,"bread":7852,"##ington":7853,"beaten":7854,"northwestern":7855,"##ona":7856,"merit":7857,"youtube":7858,"collapse":7859,"challenges":7860,"em":7861,"historians":7862,"objective":7863,"submitted":7864,"virus":7865,"attacking":7866,"drake":7867,"assume":7868,"##ere":7869,"diseases":7870,"marc":7871,"stem":7872,"leeds":7873,"##cus":7874,"##ab":7875,"farming":7876,"glasses":7877,"##lock":7878,"visits":7879,"nowhere":7880,"fellowship":7881,"relevant":7882,"carries":7883,"restaurants":7884,"experiments":7885,"101":7886,"constantly":7887,"bases":7888,"targets":7889,"shah":7890,"tenth":7891,"opponents":7892,"verse":7893,"territorial":7894,"##ira":7895,"writings":7896,"corruption":7897,"##hs":7898,"instruction":7899,"inherited":7900,"reverse":7901,"emphasis":7902,"##vic":7903,"employee":7904,"arch":7905,"keeps":7906,"rabbi":7907,"watson":7908,"payment":7909,"uh":7910,"##ala":7911,"nancy":7912,"##tre":7913,"venice":7914,"fastest":7915,"sexy":7916,"banned":7917,"adrian":7918,"properly":7919,"ruth":7920,"touchdown":7921,"dollar":7922,"boards":7923,"metre":7924,"circles":7925,"edges":7926,"favour":7927,"comments":7928,"ok":7929,"travels":7930,"liberation":7931,"scattered":7932,"firmly":7933,"##ular":7934,"holland":7935,"permitted":7936,"diesel":7937,"kenya":7938,"den":7939,"originated":7940,"##ral":7941,"demons":7942,"resumed":7943,"dragged":7944,"rider":7945,"##rus":7946,"servant":7947,"blinked":7948,"extend":7949,"torn":7950,"##ias":7951,"##sey":7952,"input":7953,"meal":7954,"everybody":7955,"cylinder":7956,"kinds":7957,"camps":7958,"##fe":7959,"bullet":7960,"logic":7961,"##wn":7962,"croatian":7963,"evolved":7964,"healthy":7965,"fool":7966,"chocolate":7967,"wise":7968,"preserve":7969,"pradesh":7970,"##ess":7971,"respective":7972,"1850":7973,"##ew":7974,"chicken":7975,"artificial":7976,"gross":7977,"corresponding":7978,"convicted":7979,"cage":7980,"caroline":7981,"dialogue":7982,"##dor":7983,"narrative":7984,"stranger":7985,"mario":7986,"br":7987,"christianity":7988,"failing":7989,"trent":7990,"commanding":7991,"buddhist":7992,"1848":7993,"maurice":7994,"focusing":7995,"yale":7996,"bike":7997,"altitude":7998,"##ering":7999,"mouse":8000,"revised":8001,"##sley":8002,"veteran":8003,"##ig":8004,"pulls":8005,"theology":8006,"crashed":8007,"campaigns":8008,"legion":8009,"##ability":8010,"drag":8011,"excellence":8012,"customer":8013,"cancelled":8014,"intensity":8015,"excuse":8016,"##lar":8017,"liga":8018,"participating":8019,"contributing":8020,"printing":8021,"##burn":8022,"variable":8023,"##rk":8024,"curious":8025,"bin":8026,"legacy":8027,"renaissance":8028,"##my":8029,"symptoms":8030,"binding":8031,"vocalist":8032,"dancer":8033,"##nie":8034,"grammar":8035,"gospel":8036,"democrats":8037,"ya":8038,"enters":8039,"sc":8040,"diplomatic":8041,"hitler":8042,"##ser":8043,"clouds":8044,"mathematical":8045,"quit":8046,"defended":8047,"oriented":8048,"##heim":8049,"fundamental":8050,"hardware":8051,"impressive":8052,"equally":8053,"convince":8054,"confederate":8055,"guilt":8056,"chuck":8057,"sliding":8058,"##ware":8059,"magnetic":8060,"narrowed":8061,"petersburg":8062,"bulgaria":8063,"otto":8064,"phd":8065,"skill":8066,"##ama":8067,"reader":8068,"hopes":8069,"pitcher":8070,"reservoir":8071,"hearts":8072,"automatically":8073,"expecting":8074,"mysterious":8075,"bennett":8076,"extensively":8077,"imagined":8078,"seeds":8079,"monitor":8080,"fix":8081,"##ative":8082,"journalism":8083,"struggling":8084,"signature":8085,"ranch":8086,"encounter":8087,"photographer":8088,"observation":8089,"protests":8090,"##pin":8091,"influences":8092,"##hr":8093,"calendar":8094,"##all":8095,"cruz":8096,"croatia":8097,"locomotive":8098,"hughes":8099,"naturally":8100,"shakespeare":8101,"basement":8102,"hook":8103,"uncredited":8104,"faded":8105,"theories":8106,"approaches":8107,"dare":8108,"phillips":8109,"filling":8110,"fury":8111,"obama":8112,"##ain":8113,"efficient":8114,"arc":8115,"deliver":8116,"min":8117,"raid":8118,"breeding":8119,"inducted":8120,"leagues":8121,"efficiency":8122,"axis":8123,"montana":8124,"eagles":8125,"##ked":8126,"supplied":8127,"instructions":8128,"karen":8129,"picking":8130,"indicating":8131,"trap":8132,"anchor":8133,"practically":8134,"christians":8135,"tomb":8136,"vary":8137,"occasional":8138,"electronics":8139,"lords":8140,"readers":8141,"newcastle":8142,"faint":8143,"innovation":8144,"collect":8145,"situations":8146,"engagement":8147,"160":8148,"claude":8149,"mixture":8150,"##feld":8151,"peer":8152,"tissue":8153,"logo":8154,"lean":8155,"##ration":8156,"°f":8157,"floors":8158,"##ven":8159,"architects":8160,"reducing":8161,"##our":8162,"##ments":8163,"rope":8164,"1859":8165,"ottawa":8166,"##har":8167,"samples":8168,"banking":8169,"declaration":8170,"proteins":8171,"resignation":8172,"francois":8173,"saudi":8174,"advocate":8175,"exhibited":8176,"armor":8177,"twins":8178,"divorce":8179,"##ras":8180,"abraham":8181,"reviewed":8182,"jo":8183,"temporarily":8184,"matrix":8185,"physically":8186,"pulse":8187,"curled":8188,"##ena":8189,"difficulties":8190,"bengal":8191,"usage":8192,"##ban":8193,"annie":8194,"riders":8195,"certificate":8196,"##pi":8197,"holes":8198,"warsaw":8199,"distinctive":8200,"jessica":8201,"##mon":8202,"mutual":8203,"1857":8204,"customs":8205,"circular":8206,"eugene":8207,"removal":8208,"loaded":8209,"mere":8210,"vulnerable":8211,"depicted":8212,"generations":8213,"dame":8214,"heir":8215,"enormous":8216,"lightly":8217,"climbing":8218,"pitched":8219,"lessons":8220,"pilots":8221,"nepal":8222,"ram":8223,"google":8224,"preparing":8225,"brad":8226,"louise":8227,"renowned":8228,"##₂":8229,"liam":8230,"##ably":8231,"plaza":8232,"shaw":8233,"sophie":8234,"brilliant":8235,"bills":8236,"##bar":8237,"##nik":8238,"fucking":8239,"mainland":8240,"server":8241,"pleasant":8242,"seized":8243,"veterans":8244,"jerked":8245,"fail":8246,"beta":8247,"brush":8248,"radiation":8249,"stored":8250,"warmth":8251,"southeastern":8252,"nate":8253,"sin":8254,"raced":8255,"berkeley":8256,"joke":8257,"athlete":8258,"designation":8259,"trunk":8260,"##low":8261,"roland":8262,"qualification":8263,"archives":8264,"heels":8265,"artwork":8266,"receives":8267,"judicial":8268,"reserves":8269,"##bed":8270,"woke":8271,"installation":8272,"abu":8273,"floating":8274,"fake":8275,"lesser":8276,"excitement":8277,"interface":8278,"concentrated":8279,"addressed":8280,"characteristic":8281,"amanda":8282,"saxophone":8283,"monk":8284,"auto":8285,"##bus":8286,"releasing":8287,"egg":8288,"dies":8289,"interaction":8290,"defender":8291,"ce":8292,"outbreak":8293,"glory":8294,"loving":8295,"##bert":8296,"sequel":8297,"consciousness":8298,"http":8299,"awake":8300,"ski":8301,"enrolled":8302,"##ress":8303,"handling":8304,"rookie":8305,"brow":8306,"somebody":8307,"biography":8308,"warfare":8309,"amounts":8310,"contracts":8311,"presentation":8312,"fabric":8313,"dissolved":8314,"challenged":8315,"meter":8316,"psychological":8317,"lt":8318,"elevated":8319,"rally":8320,"accurate":8321,"##tha":8322,"hospitals":8323,"undergraduate":8324,"specialist":8325,"venezuela":8326,"exhibit":8327,"shed":8328,"nursing":8329,"protestant":8330,"fluid":8331,"structural":8332,"footage":8333,"jared":8334,"consistent":8335,"prey":8336,"##ska":8337,"succession":8338,"reflect":8339,"exile":8340,"lebanon":8341,"wiped":8342,"suspect":8343,"shanghai":8344,"resting":8345,"integration":8346,"preservation":8347,"marvel":8348,"variant":8349,"pirates":8350,"sheep":8351,"rounded":8352,"capita":8353,"sailing":8354,"colonies":8355,"manuscript":8356,"deemed":8357,"variations":8358,"clarke":8359,"functional":8360,"emerging":8361,"boxing":8362,"relaxed":8363,"curse":8364,"azerbaijan":8365,"heavyweight":8366,"nickname":8367,"editorial":8368,"rang":8369,"grid":8370,"tightened":8371,"earthquake":8372,"flashed":8373,"miguel":8374,"rushing":8375,"##ches":8376,"improvements":8377,"boxes":8378,"brooks":8379,"180":8380,"consumption":8381,"molecular":8382,"felix":8383,"societies":8384,"repeatedly":8385,"variation":8386,"aids":8387,"civic":8388,"graphics":8389,"professionals":8390,"realm":8391,"autonomous":8392,"receiver":8393,"delayed":8394,"workshop":8395,"militia":8396,"chairs":8397,"trump":8398,"canyon":8399,"##point":8400,"harsh":8401,"extending":8402,"lovely":8403,"happiness":8404,"##jan":8405,"stake":8406,"eyebrows":8407,"embassy":8408,"wellington":8409,"hannah":8410,"##ella":8411,"sony":8412,"corners":8413,"bishops":8414,"swear":8415,"cloth":8416,"contents":8417,"xi":8418,"namely":8419,"commenced":8420,"1854":8421,"stanford":8422,"nashville":8423,"courage":8424,"graphic":8425,"commitment":8426,"garrison":8427,"##bin":8428,"hamlet":8429,"clearing":8430,"rebels":8431,"attraction":8432,"literacy":8433,"cooking":8434,"ruins":8435,"temples":8436,"jenny":8437,"humanity":8438,"celebrate":8439,"hasn":8440,"freight":8441,"sixty":8442,"rebel":8443,"bastard":8444,"##art":8445,"newton":8446,"##ada":8447,"deer":8448,"##ges":8449,"##ching":8450,"smiles":8451,"delaware":8452,"singers":8453,"##ets":8454,"approaching":8455,"assists":8456,"flame":8457,"##ph":8458,"boulevard":8459,"barrel":8460,"planted":8461,"##ome":8462,"pursuit":8463,"##sia":8464,"consequences":8465,"posts":8466,"shallow":8467,"invitation":8468,"rode":8469,"depot":8470,"ernest":8471,"kane":8472,"rod":8473,"concepts":8474,"preston":8475,"topic":8476,"chambers":8477,"striking":8478,"blast":8479,"arrives":8480,"descendants":8481,"montgomery":8482,"ranges":8483,"worlds":8484,"##lay":8485,"##ari":8486,"span":8487,"chaos":8488,"praise":8489,"##ag":8490,"fewer":8491,"1855":8492,"sanctuary":8493,"mud":8494,"fbi":8495,"##ions":8496,"programmes":8497,"maintaining":8498,"unity":8499,"harper":8500,"bore":8501,"handsome":8502,"closure":8503,"tournaments":8504,"thunder":8505,"nebraska":8506,"linda":8507,"facade":8508,"puts":8509,"satisfied":8510,"argentine":8511,"dale":8512,"cork":8513,"dome":8514,"panama":8515,"##yl":8516,"1858":8517,"tasks":8518,"experts":8519,"##ates":8520,"feeding":8521,"equation":8522,"##las":8523,"##ida":8524,"##tu":8525,"engage":8526,"bryan":8527,"##ax":8528,"um":8529,"quartet":8530,"melody":8531,"disbanded":8532,"sheffield":8533,"blocked":8534,"gasped":8535,"delay":8536,"kisses":8537,"maggie":8538,"connects":8539,"##non":8540,"sts":8541,"poured":8542,"creator":8543,"publishers":8544,"##we":8545,"guided":8546,"ellis":8547,"extinct":8548,"hug":8549,"gaining":8550,"##ord":8551,"complicated":8552,"##bility":8553,"poll":8554,"clenched":8555,"investigate":8556,"##use":8557,"thereby":8558,"quantum":8559,"spine":8560,"cdp":8561,"humor":8562,"kills":8563,"administered":8564,"semifinals":8565,"##du":8566,"encountered":8567,"ignore":8568,"##bu":8569,"commentary":8570,"##maker":8571,"bother":8572,"roosevelt":8573,"140":8574,"plains":8575,"halfway":8576,"flowing":8577,"cultures":8578,"crack":8579,"imprisoned":8580,"neighboring":8581,"airline":8582,"##ses":8583,"##view":8584,"##mate":8585,"##ec":8586,"gather":8587,"wolves":8588,"marathon":8589,"transformed":8590,"##ill":8591,"cruise":8592,"organisations":8593,"carol":8594,"punch":8595,"exhibitions":8596,"numbered":8597,"alarm":8598,"ratings":8599,"daddy":8600,"silently":8601,"##stein":8602,"queens":8603,"colours":8604,"impression":8605,"guidance":8606,"liu":8607,"tactical":8608,"##rat":8609,"marshal":8610,"della":8611,"arrow":8612,"##ings":8613,"rested":8614,"feared":8615,"tender":8616,"owns":8617,"bitter":8618,"advisor":8619,"escort":8620,"##ides":8621,"spare":8622,"farms":8623,"grants":8624,"##ene":8625,"dragons":8626,"encourage":8627,"colleagues":8628,"cameras":8629,"##und":8630,"sucked":8631,"pile":8632,"spirits":8633,"prague":8634,"statements":8635,"suspension":8636,"landmark":8637,"fence":8638,"torture":8639,"recreation":8640,"bags":8641,"permanently":8642,"survivors":8643,"pond":8644,"spy":8645,"predecessor":8646,"bombing":8647,"coup":8648,"##og":8649,"protecting":8650,"transformation":8651,"glow":8652,"##lands":8653,"##book":8654,"dug":8655,"priests":8656,"andrea":8657,"feat":8658,"barn":8659,"jumping":8660,"##chen":8661,"##ologist":8662,"##con":8663,"casualties":8664,"stern":8665,"auckland":8666,"pipe":8667,"serie":8668,"revealing":8669,"ba":8670,"##bel":8671,"trevor":8672,"mercy":8673,"spectrum":8674,"yang":8675,"consist":8676,"governing":8677,"collaborated":8678,"possessed":8679,"epic":8680,"comprises":8681,"blew":8682,"shane":8683,"##ack":8684,"lopez":8685,"honored":8686,"magical":8687,"sacrifice":8688,"judgment":8689,"perceived":8690,"hammer":8691,"mtv":8692,"baronet":8693,"tune":8694,"das":8695,"missionary":8696,"sheets":8697,"350":8698,"neutral":8699,"oral":8700,"threatening":8701,"attractive":8702,"shade":8703,"aims":8704,"seminary":8705,"##master":8706,"estates":8707,"1856":8708,"michel":8709,"wounds":8710,"refugees":8711,"manufacturers":8712,"##nic":8713,"mercury":8714,"syndrome":8715,"porter":8716,"##iya":8717,"##din":8718,"hamburg":8719,"identification":8720,"upstairs":8721,"purse":8722,"widened":8723,"pause":8724,"cared":8725,"breathed":8726,"affiliate":8727,"santiago":8728,"prevented":8729,"celtic":8730,"fisher":8731,"125":8732,"recruited":8733,"byzantine":8734,"reconstruction":8735,"farther":8736,"##mp":8737,"diet":8738,"sake":8739,"au":8740,"spite":8741,"sensation":8742,"##ert":8743,"blank":8744,"separation":8745,"105":8746,"##hon":8747,"vladimir":8748,"armies":8749,"anime":8750,"##lie":8751,"accommodate":8752,"orbit":8753,"cult":8754,"sofia":8755,"archive":8756,"##ify":8757,"##box":8758,"founders":8759,"sustained":8760,"disorder":8761,"honours":8762,"northeastern":8763,"mia":8764,"crops":8765,"violet":8766,"threats":8767,"blanket":8768,"fires":8769,"canton":8770,"followers":8771,"southwestern":8772,"prototype":8773,"voyage":8774,"assignment":8775,"altered":8776,"moderate":8777,"protocol":8778,"pistol":8779,"##eo":8780,"questioned":8781,"brass":8782,"lifting":8783,"1852":8784,"math":8785,"authored":8786,"##ual":8787,"doug":8788,"dimensional":8789,"dynamic":8790,"##san":8791,"1851":8792,"pronounced":8793,"grateful":8794,"quest":8795,"uncomfortable":8796,"boom":8797,"presidency":8798,"stevens":8799,"relating":8800,"politicians":8801,"chen":8802,"barrier":8803,"quinn":8804,"diana":8805,"mosque":8806,"tribal":8807,"cheese":8808,"palmer":8809,"portions":8810,"sometime":8811,"chester":8812,"treasure":8813,"wu":8814,"bend":8815,"download":8816,"millions":8817,"reforms":8818,"registration":8819,"##osa":8820,"consequently":8821,"monitoring":8822,"ate":8823,"preliminary":8824,"brandon":8825,"invented":8826,"ps":8827,"eaten":8828,"exterior":8829,"intervention":8830,"ports":8831,"documented":8832,"log":8833,"displays":8834,"lecture":8835,"sally":8836,"favourite":8837,"##itz":8838,"vermont":8839,"lo":8840,"invisible":8841,"isle":8842,"breed":8843,"##ator":8844,"journalists":8845,"relay":8846,"speaks":8847,"backward":8848,"explore":8849,"midfielder":8850,"actively":8851,"stefan":8852,"procedures":8853,"cannon":8854,"blond":8855,"kenneth":8856,"centered":8857,"servants":8858,"chains":8859,"libraries":8860,"malcolm":8861,"essex":8862,"henri":8863,"slavery":8864,"##hal":8865,"facts":8866,"fairy":8867,"coached":8868,"cassie":8869,"cats":8870,"washed":8871,"cop":8872,"##fi":8873,"announcement":8874,"item":8875,"2000s":8876,"vinyl":8877,"activated":8878,"marco":8879,"frontier":8880,"growled":8881,"curriculum":8882,"##das":8883,"loyal":8884,"accomplished":8885,"leslie":8886,"ritual":8887,"kenny":8888,"##00":8889,"vii":8890,"napoleon":8891,"hollow":8892,"hybrid":8893,"jungle":8894,"stationed":8895,"friedrich":8896,"counted":8897,"##ulated":8898,"platinum":8899,"theatrical":8900,"seated":8901,"col":8902,"rubber":8903,"glen":8904,"1840":8905,"diversity":8906,"healing":8907,"extends":8908,"id":8909,"provisions":8910,"administrator":8911,"columbus":8912,"##oe":8913,"tributary":8914,"te":8915,"assured":8916,"org":8917,"##uous":8918,"prestigious":8919,"examined":8920,"lectures":8921,"grammy":8922,"ronald":8923,"associations":8924,"bailey":8925,"allan":8926,"essays":8927,"flute":8928,"believing":8929,"consultant":8930,"proceedings":8931,"travelling":8932,"1853":8933,"kit":8934,"kerala":8935,"yugoslavia":8936,"buddy":8937,"methodist":8938,"##ith":8939,"burial":8940,"centres":8941,"batman":8942,"##nda":8943,"discontinued":8944,"bo":8945,"dock":8946,"stockholm":8947,"lungs":8948,"severely":8949,"##nk":8950,"citing":8951,"manga":8952,"##ugh":8953,"steal":8954,"mumbai":8955,"iraqi":8956,"robot":8957,"celebrity":8958,"bride":8959,"broadcasts":8960,"abolished":8961,"pot":8962,"joel":8963,"overhead":8964,"franz":8965,"packed":8966,"reconnaissance":8967,"johann":8968,"acknowledged":8969,"introduce":8970,"handled":8971,"doctorate":8972,"developments":8973,"drinks":8974,"alley":8975,"palestine":8976,"##nis":8977,"##aki":8978,"proceeded":8979,"recover":8980,"bradley":8981,"grain":8982,"patch":8983,"afford":8984,"infection":8985,"nationalist":8986,"legendary":8987,"##ath":8988,"interchange":8989,"virtually":8990,"gen":8991,"gravity":8992,"exploration":8993,"amber":8994,"vital":8995,"wishes":8996,"powell":8997,"doctrine":8998,"elbow":8999,"screenplay":9000,"##bird":9001,"contribute":9002,"indonesian":9003,"pet":9004,"creates":9005,"##com":9006,"enzyme":9007,"kylie":9008,"discipline":9009,"drops":9010,"manila":9011,"hunger":9012,"##ien":9013,"layers":9014,"suffer":9015,"fever":9016,"bits":9017,"monica":9018,"keyboard":9019,"manages":9020,"##hood":9021,"searched":9022,"appeals":9023,"##bad":9024,"testament":9025,"grande":9026,"reid":9027,"##war":9028,"beliefs":9029,"congo":9030,"##ification":9031,"##dia":9032,"si":9033,"requiring":9034,"##via":9035,"casey":9036,"1849":9037,"regret":9038,"streak":9039,"rape":9040,"depends":9041,"syrian":9042,"sprint":9043,"pound":9044,"tourists":9045,"upcoming":9046,"pub":9047,"##xi":9048,"tense":9049,"##els":9050,"practiced":9051,"echo":9052,"nationwide":9053,"guild":9054,"motorcycle":9055,"liz":9056,"##zar":9057,"chiefs":9058,"desired":9059,"elena":9060,"bye":9061,"precious":9062,"absorbed":9063,"relatives":9064,"booth":9065,"pianist":9066,"##mal":9067,"citizenship":9068,"exhausted":9069,"wilhelm":9070,"##ceae":9071,"##hed":9072,"noting":9073,"quarterback":9074,"urge":9075,"hectares":9076,"##gue":9077,"ace":9078,"holly":9079,"##tal":9080,"blonde":9081,"davies":9082,"parked":9083,"sustainable":9084,"stepping":9085,"twentieth":9086,"airfield":9087,"galaxy":9088,"nest":9089,"chip":9090,"##nell":9091,"tan":9092,"shaft":9093,"paulo":9094,"requirement":9095,"##zy":9096,"paradise":9097,"tobacco":9098,"trans":9099,"renewed":9100,"vietnamese":9101,"##cker":9102,"##ju":9103,"suggesting":9104,"catching":9105,"holmes":9106,"enjoying":9107,"md":9108,"trips":9109,"colt":9110,"holder":9111,"butterfly":9112,"nerve":9113,"reformed":9114,"cherry":9115,"bowling":9116,"trailer":9117,"carriage":9118,"goodbye":9119,"appreciate":9120,"toy":9121,"joshua":9122,"interactive":9123,"enabled":9124,"involve":9125,"##kan":9126,"collar":9127,"determination":9128,"bunch":9129,"facebook":9130,"recall":9131,"shorts":9132,"superintendent":9133,"episcopal":9134,"frustration":9135,"giovanni":9136,"nineteenth":9137,"laser":9138,"privately":9139,"array":9140,"circulation":9141,"##ovic":9142,"armstrong":9143,"deals":9144,"painful":9145,"permit":9146,"discrimination":9147,"##wi":9148,"aires":9149,"retiring":9150,"cottage":9151,"ni":9152,"##sta":9153,"horizon":9154,"ellen":9155,"jamaica":9156,"ripped":9157,"fernando":9158,"chapters":9159,"playstation":9160,"patron":9161,"lecturer":9162,"navigation":9163,"behaviour":9164,"genes":9165,"georgian":9166,"export":9167,"solomon":9168,"rivals":9169,"swift":9170,"seventeen":9171,"rodriguez":9172,"princeton":9173,"independently":9174,"sox":9175,"1847":9176,"arguing":9177,"entity":9178,"casting":9179,"hank":9180,"criteria":9181,"oakland":9182,"geographic":9183,"milwaukee":9184,"reflection":9185,"expanding":9186,"conquest":9187,"dubbed":9188,"##tv":9189,"halt":9190,"brave":9191,"brunswick":9192,"doi":9193,"arched":9194,"curtis":9195,"divorced":9196,"predominantly":9197,"somerset":9198,"streams":9199,"ugly":9200,"zoo":9201,"horrible":9202,"curved":9203,"buenos":9204,"fierce":9205,"dictionary":9206,"vector":9207,"theological":9208,"unions":9209,"handful":9210,"stability":9211,"chan":9212,"punjab":9213,"segments":9214,"##lly":9215,"altar":9216,"ignoring":9217,"gesture":9218,"monsters":9219,"pastor":9220,"##stone":9221,"thighs":9222,"unexpected":9223,"operators":9224,"abruptly":9225,"coin":9226,"compiled":9227,"associates":9228,"improving":9229,"migration":9230,"pin":9231,"##ose":9232,"compact":9233,"collegiate":9234,"reserved":9235,"##urs":9236,"quarterfinals":9237,"roster":9238,"restore":9239,"assembled":9240,"hurry":9241,"oval":9242,"##cies":9243,"1846":9244,"flags":9245,"martha":9246,"##del":9247,"victories":9248,"sharply":9249,"##rated":9250,"argues":9251,"deadly":9252,"neo":9253,"drawings":9254,"symbols":9255,"performer":9256,"##iel":9257,"griffin":9258,"restrictions":9259,"editing":9260,"andrews":9261,"java":9262,"journals":9263,"arabia":9264,"compositions":9265,"dee":9266,"pierce":9267,"removing":9268,"hindi":9269,"casino":9270,"runway":9271,"civilians":9272,"minds":9273,"nasa":9274,"hotels":9275,"##zation":9276,"refuge":9277,"rent":9278,"retain":9279,"potentially":9280,"conferences":9281,"suburban":9282,"conducting":9283,"##tto":9284,"##tions":9285,"##tle":9286,"descended":9287,"massacre":9288,"##cal":9289,"ammunition":9290,"terrain":9291,"fork":9292,"souls":9293,"counts":9294,"chelsea":9295,"durham":9296,"drives":9297,"cab":9298,"##bank":9299,"perth":9300,"realizing":9301,"palestinian":9302,"finn":9303,"simpson":9304,"##dal":9305,"betty":9306,"##ule":9307,"moreover":9308,"particles":9309,"cardinals":9310,"tent":9311,"evaluation":9312,"extraordinary":9313,"##oid":9314,"inscription":9315,"##works":9316,"wednesday":9317,"chloe":9318,"maintains":9319,"panels":9320,"ashley":9321,"trucks":9322,"##nation":9323,"cluster":9324,"sunlight":9325,"strikes":9326,"zhang":9327,"##wing":9328,"dialect":9329,"canon":9330,"##ap":9331,"tucked":9332,"##ws":9333,"collecting":9334,"##mas":9335,"##can":9336,"##sville":9337,"maker":9338,"quoted":9339,"evan":9340,"franco":9341,"aria":9342,"buying":9343,"cleaning":9344,"eva":9345,"closet":9346,"provision":9347,"apollo":9348,"clinic":9349,"rat":9350,"##ez":9351,"necessarily":9352,"ac":9353,"##gle":9354,"##ising":9355,"venues":9356,"flipped":9357,"cent":9358,"spreading":9359,"trustees":9360,"checking":9361,"authorized":9362,"##sco":9363,"disappointed":9364,"##ado":9365,"notion":9366,"duration":9367,"trumpet":9368,"hesitated":9369,"topped":9370,"brussels":9371,"rolls":9372,"theoretical":9373,"hint":9374,"define":9375,"aggressive":9376,"repeat":9377,"wash":9378,"peaceful":9379,"optical":9380,"width":9381,"allegedly":9382,"mcdonald":9383,"strict":9384,"copyright":9385,"##illa":9386,"investors":9387,"mar":9388,"jam":9389,"witnesses":9390,"sounding":9391,"miranda":9392,"michelle":9393,"privacy":9394,"hugo":9395,"harmony":9396,"##pp":9397,"valid":9398,"lynn":9399,"glared":9400,"nina":9401,"102":9402,"headquartered":9403,"diving":9404,"boarding":9405,"gibson":9406,"##ncy":9407,"albanian":9408,"marsh":9409,"routine":9410,"dealt":9411,"enhanced":9412,"er":9413,"intelligent":9414,"substance":9415,"targeted":9416,"enlisted":9417,"discovers":9418,"spinning":9419,"observations":9420,"pissed":9421,"smoking":9422,"rebecca":9423,"capitol":9424,"visa":9425,"varied":9426,"costume":9427,"seemingly":9428,"indies":9429,"compensation":9430,"surgeon":9431,"thursday":9432,"arsenal":9433,"westminster":9434,"suburbs":9435,"rid":9436,"anglican":9437,"##ridge":9438,"knots":9439,"foods":9440,"alumni":9441,"lighter":9442,"fraser":9443,"whoever":9444,"portal":9445,"scandal":9446,"##ray":9447,"gavin":9448,"advised":9449,"instructor":9450,"flooding":9451,"terrorist":9452,"##ale":9453,"teenage":9454,"interim":9455,"senses":9456,"duck":9457,"teen":9458,"thesis":9459,"abby":9460,"eager":9461,"overcome":9462,"##ile":9463,"newport":9464,"glenn":9465,"rises":9466,"shame":9467,"##cc":9468,"prompted":9469,"priority":9470,"forgot":9471,"bomber":9472,"nicolas":9473,"protective":9474,"360":9475,"cartoon":9476,"katherine":9477,"breeze":9478,"lonely":9479,"trusted":9480,"henderson":9481,"richardson":9482,"relax":9483,"banner":9484,"candy":9485,"palms":9486,"remarkable":9487,"##rio":9488,"legends":9489,"cricketer":9490,"essay":9491,"ordained":9492,"edmund":9493,"rifles":9494,"trigger":9495,"##uri":9496,"##away":9497,"sail":9498,"alert":9499,"1830":9500,"audiences":9501,"penn":9502,"sussex":9503,"siblings":9504,"pursued":9505,"indianapolis":9506,"resist":9507,"rosa":9508,"consequence":9509,"succeed":9510,"avoided":9511,"1845":9512,"##ulation":9513,"inland":9514,"##tie":9515,"##nna":9516,"counsel":9517,"profession":9518,"chronicle":9519,"hurried":9520,"##una":9521,"eyebrow":9522,"eventual":9523,"bleeding":9524,"innovative":9525,"cure":9526,"##dom":9527,"committees":9528,"accounting":9529,"con":9530,"scope":9531,"hardy":9532,"heather":9533,"tenor":9534,"gut":9535,"herald":9536,"codes":9537,"tore":9538,"scales":9539,"wagon":9540,"##oo":9541,"luxury":9542,"tin":9543,"prefer":9544,"fountain":9545,"triangle":9546,"bonds":9547,"darling":9548,"convoy":9549,"dried":9550,"traced":9551,"beings":9552,"troy":9553,"accidentally":9554,"slam":9555,"findings":9556,"smelled":9557,"joey":9558,"lawyers":9559,"outcome":9560,"steep":9561,"bosnia":9562,"configuration":9563,"shifting":9564,"toll":9565,"brook":9566,"performers":9567,"lobby":9568,"philosophical":9569,"construct":9570,"shrine":9571,"aggregate":9572,"boot":9573,"cox":9574,"phenomenon":9575,"savage":9576,"insane":9577,"solely":9578,"reynolds":9579,"lifestyle":9580,"##ima":9581,"nationally":9582,"holdings":9583,"consideration":9584,"enable":9585,"edgar":9586,"mo":9587,"mama":9588,"##tein":9589,"fights":9590,"relegation":9591,"chances":9592,"atomic":9593,"hub":9594,"conjunction":9595,"awkward":9596,"reactions":9597,"currency":9598,"finale":9599,"kumar":9600,"underwent":9601,"steering":9602,"elaborate":9603,"gifts":9604,"comprising":9605,"melissa":9606,"veins":9607,"reasonable":9608,"sunshine":9609,"chi":9610,"solve":9611,"trails":9612,"inhabited":9613,"elimination":9614,"ethics":9615,"huh":9616,"ana":9617,"molly":9618,"consent":9619,"apartments":9620,"layout":9621,"marines":9622,"##ces":9623,"hunters":9624,"bulk":9625,"##oma":9626,"hometown":9627,"##wall":9628,"##mont":9629,"cracked":9630,"reads":9631,"neighbouring":9632,"withdrawn":9633,"admission":9634,"wingspan":9635,"damned":9636,"anthology":9637,"lancashire":9638,"brands":9639,"batting":9640,"forgive":9641,"cuban":9642,"awful":9643,"##lyn":9644,"104":9645,"dimensions":9646,"imagination":9647,"##ade":9648,"dante":9649,"##ship":9650,"tracking":9651,"desperately":9652,"goalkeeper":9653,"##yne":9654,"groaned":9655,"workshops":9656,"confident":9657,"burton":9658,"gerald":9659,"milton":9660,"circus":9661,"uncertain":9662,"slope":9663,"copenhagen":9664,"sophia":9665,"fog":9666,"philosopher":9667,"portraits":9668,"accent":9669,"cycling":9670,"varying":9671,"gripped":9672,"larvae":9673,"garrett":9674,"specified":9675,"scotia":9676,"mature":9677,"luther":9678,"kurt":9679,"rap":9680,"##kes":9681,"aerial":9682,"750":9683,"ferdinand":9684,"heated":9685,"es":9686,"transported":9687,"##shan":9688,"safely":9689,"nonetheless":9690,"##orn":9691,"##gal":9692,"motors":9693,"demanding":9694,"##sburg":9695,"startled":9696,"##brook":9697,"ally":9698,"generate":9699,"caps":9700,"ghana":9701,"stained":9702,"demo":9703,"mentions":9704,"beds":9705,"ap":9706,"afterward":9707,"diary":9708,"##bling":9709,"utility":9710,"##iro":9711,"richards":9712,"1837":9713,"conspiracy":9714,"conscious":9715,"shining":9716,"footsteps":9717,"observer":9718,"cyprus":9719,"urged":9720,"loyalty":9721,"developer":9722,"probability":9723,"olive":9724,"upgraded":9725,"gym":9726,"miracle":9727,"insects":9728,"graves":9729,"1844":9730,"ourselves":9731,"hydrogen":9732,"amazon":9733,"katie":9734,"tickets":9735,"poets":9736,"##pm":9737,"planes":9738,"##pan":9739,"prevention":9740,"witnessed":9741,"dense":9742,"jin":9743,"randy":9744,"tang":9745,"warehouse":9746,"monroe":9747,"bang":9748,"archived":9749,"elderly":9750,"investigations":9751,"alec":9752,"granite":9753,"mineral":9754,"conflicts":9755,"controlling":9756,"aboriginal":9757,"carlo":9758,"##zu":9759,"mechanics":9760,"stan":9761,"stark":9762,"rhode":9763,"skirt":9764,"est":9765,"##berry":9766,"bombs":9767,"respected":9768,"##horn":9769,"imposed":9770,"limestone":9771,"deny":9772,"nominee":9773,"memphis":9774,"grabbing":9775,"disabled":9776,"##als":9777,"amusement":9778,"aa":9779,"frankfurt":9780,"corn":9781,"referendum":9782,"varies":9783,"slowed":9784,"disk":9785,"firms":9786,"unconscious":9787,"incredible":9788,"clue":9789,"sue":9790,"##zhou":9791,"twist":9792,"##cio":9793,"joins":9794,"idaho":9795,"chad":9796,"developers":9797,"computing":9798,"destroyer":9799,"103":9800,"mortal":9801,"tucker":9802,"kingston":9803,"choices":9804,"yu":9805,"carson":9806,"1800":9807,"os":9808,"whitney":9809,"geneva":9810,"pretend":9811,"dimension":9812,"staged":9813,"plateau":9814,"maya":9815,"##une":9816,"freestyle":9817,"##bc":9818,"rovers":9819,"hiv":9820,"##ids":9821,"tristan":9822,"classroom":9823,"prospect":9824,"##hus":9825,"honestly":9826,"diploma":9827,"lied":9828,"thermal":9829,"auxiliary":9830,"feast":9831,"unlikely":9832,"iata":9833,"##tel":9834,"morocco":9835,"pounding":9836,"treasury":9837,"lithuania":9838,"considerably":9839,"1841":9840,"dish":9841,"1812":9842,"geological":9843,"matching":9844,"stumbled":9845,"destroying":9846,"marched":9847,"brien":9848,"advances":9849,"cake":9850,"nicole":9851,"belle":9852,"settling":9853,"measuring":9854,"directing":9855,"##mie":9856,"tuesday":9857,"bassist":9858,"capabilities":9859,"stunned":9860,"fraud":9861,"torpedo":9862,"##list":9863,"##phone":9864,"anton":9865,"wisdom":9866,"surveillance":9867,"ruined":9868,"##ulate":9869,"lawsuit":9870,"healthcare":9871,"theorem":9872,"halls":9873,"trend":9874,"aka":9875,"horizontal":9876,"dozens":9877,"acquire":9878,"lasting":9879,"swim":9880,"hawk":9881,"gorgeous":9882,"fees":9883,"vicinity":9884,"decrease":9885,"adoption":9886,"tactics":9887,"##ography":9888,"pakistani":9889,"##ole":9890,"draws":9891,"##hall":9892,"willie":9893,"burke":9894,"heath":9895,"algorithm":9896,"integral":9897,"powder":9898,"elliott":9899,"brigadier":9900,"jackie":9901,"tate":9902,"varieties":9903,"darker":9904,"##cho":9905,"lately":9906,"cigarette":9907,"specimens":9908,"adds":9909,"##ree":9910,"##ensis":9911,"##inger":9912,"exploded":9913,"finalist":9914,"cia":9915,"murders":9916,"wilderness":9917,"arguments":9918,"nicknamed":9919,"acceptance":9920,"onwards":9921,"manufacture":9922,"robertson":9923,"jets":9924,"tampa":9925,"enterprises":9926,"blog":9927,"loudly":9928,"composers":9929,"nominations":9930,"1838":9931,"ai":9932,"malta":9933,"inquiry":9934,"automobile":9935,"hosting":9936,"viii":9937,"rays":9938,"tilted":9939,"grief":9940,"museums":9941,"strategies":9942,"furious":9943,"euro":9944,"equality":9945,"cohen":9946,"poison":9947,"surrey":9948,"wireless":9949,"governed":9950,"ridiculous":9951,"moses":9952,"##esh":9953,"##room":9954,"vanished":9955,"##ito":9956,"barnes":9957,"attract":9958,"morrison":9959,"istanbul":9960,"##iness":9961,"absent":9962,"rotation":9963,"petition":9964,"janet":9965,"##logical":9966,"satisfaction":9967,"custody":9968,"deliberately":9969,"observatory":9970,"comedian":9971,"surfaces":9972,"pinyin":9973,"novelist":9974,"strictly":9975,"canterbury":9976,"oslo":9977,"monks":9978,"embrace":9979,"ibm":9980,"jealous":9981,"photograph":9982,"continent":9983,"dorothy":9984,"marina":9985,"doc":9986,"excess":9987,"holden":9988,"allegations":9989,"explaining":9990,"stack":9991,"avoiding":9992,"lance":9993,"storyline":9994,"majesty":9995,"poorly":9996,"spike":9997,"dos":9998,"bradford":9999,"raven":10000,"travis":10001,"classics":10002,"proven":10003,"voltage":10004,"pillow":10005,"fists":10006,"butt":10007,"1842":10008,"interpreted":10009,"##car":10010,"1839":10011,"gage":10012,"telegraph":10013,"lens":10014,"promising":10015,"expelled":10016,"casual":10017,"collector":10018,"zones":10019,"##min":10020,"silly":10021,"nintendo":10022,"##kh":10023,"##bra":10024,"downstairs":10025,"chef":10026,"suspicious":10027,"afl":10028,"flies":10029,"vacant":10030,"uganda":10031,"pregnancy":10032,"condemned":10033,"lutheran":10034,"estimates":10035,"cheap":10036,"decree":10037,"saxon":10038,"proximity":10039,"stripped":10040,"idiot":10041,"deposits":10042,"contrary":10043,"presenter":10044,"magnus":10045,"glacier":10046,"im":10047,"offense":10048,"edwin":10049,"##ori":10050,"upright":10051,"##long":10052,"bolt":10053,"##ois":10054,"toss":10055,"geographical":10056,"##izes":10057,"environments":10058,"delicate":10059,"marking":10060,"abstract":10061,"xavier":10062,"nails":10063,"windsor":10064,"plantation":10065,"occurring":10066,"equity":10067,"saskatchewan":10068,"fears":10069,"drifted":10070,"sequences":10071,"vegetation":10072,"revolt":10073,"##stic":10074,"1843":10075,"sooner":10076,"fusion":10077,"opposing":10078,"nato":10079,"skating":10080,"1836":10081,"secretly":10082,"ruin":10083,"lease":10084,"##oc":10085,"edit":10086,"##nne":10087,"flora":10088,"anxiety":10089,"ruby":10090,"##ological":10091,"##mia":10092,"tel":10093,"bout":10094,"taxi":10095,"emmy":10096,"frost":10097,"rainbow":10098,"compounds":10099,"foundations":10100,"rainfall":10101,"assassination":10102,"nightmare":10103,"dominican":10104,"##win":10105,"achievements":10106,"deserve":10107,"orlando":10108,"intact":10109,"armenia":10110,"##nte":10111,"calgary":10112,"valentine":10113,"106":10114,"marion":10115,"proclaimed":10116,"theodore":10117,"bells":10118,"courtyard":10119,"thigh":10120,"gonzalez":10121,"console":10122,"troop":10123,"minimal":10124,"monte":10125,"everyday":10126,"##ence":10127,"##if":10128,"supporter":10129,"terrorism":10130,"buck":10131,"openly":10132,"presbyterian":10133,"activists":10134,"carpet":10135,"##iers":10136,"rubbing":10137,"uprising":10138,"##yi":10139,"cute":10140,"conceived":10141,"legally":10142,"##cht":10143,"millennium":10144,"cello":10145,"velocity":10146,"ji":10147,"rescued":10148,"cardiff":10149,"1835":10150,"rex":10151,"concentrate":10152,"senators":10153,"beard":10154,"rendered":10155,"glowing":10156,"battalions":10157,"scouts":10158,"competitors":10159,"sculptor":10160,"catalogue":10161,"arctic":10162,"ion":10163,"raja":10164,"bicycle":10165,"wow":10166,"glancing":10167,"lawn":10168,"##woman":10169,"gentleman":10170,"lighthouse":10171,"publish":10172,"predicted":10173,"calculated":10174,"##val":10175,"variants":10176,"##gne":10177,"strain":10178,"##ui":10179,"winston":10180,"deceased":10181,"##nus":10182,"touchdowns":10183,"brady":10184,"caleb":10185,"sinking":10186,"echoed":10187,"crush":10188,"hon":10189,"blessed":10190,"protagonist":10191,"hayes":10192,"endangered":10193,"magnitude":10194,"editors":10195,"##tine":10196,"estimate":10197,"responsibilities":10198,"##mel":10199,"backup":10200,"laying":10201,"consumed":10202,"sealed":10203,"zurich":10204,"lovers":10205,"frustrated":10206,"##eau":10207,"ahmed":10208,"kicking":10209,"mit":10210,"treasurer":10211,"1832":10212,"biblical":10213,"refuse":10214,"terrified":10215,"pump":10216,"agrees":10217,"genuine":10218,"imprisonment":10219,"refuses":10220,"plymouth":10221,"##hen":10222,"lou":10223,"##nen":10224,"tara":10225,"trembling":10226,"antarctic":10227,"ton":10228,"learns":10229,"##tas":10230,"crap":10231,"crucial":10232,"faction":10233,"atop":10234,"##borough":10235,"wrap":10236,"lancaster":10237,"odds":10238,"hopkins":10239,"erik":10240,"lyon":10241,"##eon":10242,"bros":10243,"##ode":10244,"snap":10245,"locality":10246,"tips":10247,"empress":10248,"crowned":10249,"cal":10250,"acclaimed":10251,"chuckled":10252,"##ory":10253,"clara":10254,"sends":10255,"mild":10256,"towel":10257,"##fl":10258,"##day":10259,"##а":10260,"wishing":10261,"assuming":10262,"interviewed":10263,"##bal":10264,"##die":10265,"interactions":10266,"eden":10267,"cups":10268,"helena":10269,"##lf":10270,"indie":10271,"beck":10272,"##fire":10273,"batteries":10274,"filipino":10275,"wizard":10276,"parted":10277,"##lam":10278,"traces":10279,"##born":10280,"rows":10281,"idol":10282,"albany":10283,"delegates":10284,"##ees":10285,"##sar":10286,"discussions":10287,"##ex":10288,"notre":10289,"instructed":10290,"belgrade":10291,"highways":10292,"suggestion":10293,"lauren":10294,"possess":10295,"orientation":10296,"alexandria":10297,"abdul":10298,"beats":10299,"salary":10300,"reunion":10301,"ludwig":10302,"alright":10303,"wagner":10304,"intimate":10305,"pockets":10306,"slovenia":10307,"hugged":10308,"brighton":10309,"merchants":10310,"cruel":10311,"stole":10312,"trek":10313,"slopes":10314,"repairs":10315,"enrollment":10316,"politically":10317,"underlying":10318,"promotional":10319,"counting":10320,"boeing":10321,"##bb":10322,"isabella":10323,"naming":10324,"##и":10325,"keen":10326,"bacteria":10327,"listing":10328,"separately":10329,"belfast":10330,"ussr":10331,"450":10332,"lithuanian":10333,"anybody":10334,"ribs":10335,"sphere":10336,"martinez":10337,"cock":10338,"embarrassed":10339,"proposals":10340,"fragments":10341,"nationals":10342,"##fs":10343,"##wski":10344,"premises":10345,"fin":10346,"1500":10347,"alpine":10348,"matched":10349,"freely":10350,"bounded":10351,"jace":10352,"sleeve":10353,"##af":10354,"gaming":10355,"pier":10356,"populated":10357,"evident":10358,"##like":10359,"frances":10360,"flooded":10361,"##dle":10362,"frightened":10363,"pour":10364,"trainer":10365,"framed":10366,"visitor":10367,"challenging":10368,"pig":10369,"wickets":10370,"##fold":10371,"infected":10372,"email":10373,"##pes":10374,"arose":10375,"##aw":10376,"reward":10377,"ecuador":10378,"oblast":10379,"vale":10380,"ch":10381,"shuttle":10382,"##usa":10383,"bach":10384,"rankings":10385,"forbidden":10386,"cornwall":10387,"accordance":10388,"salem":10389,"consumers":10390,"bruno":10391,"fantastic":10392,"toes":10393,"machinery":10394,"resolved":10395,"julius":10396,"remembering":10397,"propaganda":10398,"iceland":10399,"bombardment":10400,"tide":10401,"contacts":10402,"wives":10403,"##rah":10404,"concerto":10405,"macdonald":10406,"albania":10407,"implement":10408,"daisy":10409,"tapped":10410,"sudan":10411,"helmet":10412,"angela":10413,"mistress":10414,"##lic":10415,"crop":10416,"sunk":10417,"finest":10418,"##craft":10419,"hostile":10420,"##ute":10421,"##tsu":10422,"boxer":10423,"fr":10424,"paths":10425,"adjusted":10426,"habit":10427,"ballot":10428,"supervision":10429,"soprano":10430,"##zen":10431,"bullets":10432,"wicked":10433,"sunset":10434,"regiments":10435,"disappear":10436,"lamp":10437,"performs":10438,"app":10439,"##gia":10440,"##oa":10441,"rabbit":10442,"digging":10443,"incidents":10444,"entries":10445,"##cion":10446,"dishes":10447,"##oi":10448,"introducing":10449,"##ati":10450,"##fied":10451,"freshman":10452,"slot":10453,"jill":10454,"tackles":10455,"baroque":10456,"backs":10457,"##iest":10458,"lone":10459,"sponsor":10460,"destiny":10461,"altogether":10462,"convert":10463,"##aro":10464,"consensus":10465,"shapes":10466,"demonstration":10467,"basically":10468,"feminist":10469,"auction":10470,"artifacts":10471,"##bing":10472,"strongest":10473,"twitter":10474,"halifax":10475,"2019":10476,"allmusic":10477,"mighty":10478,"smallest":10479,"precise":10480,"alexandra":10481,"viola":10482,"##los":10483,"##ille":10484,"manuscripts":10485,"##illo":10486,"dancers":10487,"ari":10488,"managers":10489,"monuments":10490,"blades":10491,"barracks":10492,"springfield":10493,"maiden":10494,"consolidated":10495,"electron":10496,"##end":10497,"berry":10498,"airing":10499,"wheat":10500,"nobel":10501,"inclusion":10502,"blair":10503,"payments":10504,"geography":10505,"bee":10506,"cc":10507,"eleanor":10508,"react":10509,"##hurst":10510,"afc":10511,"manitoba":10512,"##yu":10513,"su":10514,"lineup":10515,"fitness":10516,"recreational":10517,"investments":10518,"airborne":10519,"disappointment":10520,"##dis":10521,"edmonton":10522,"viewing":10523,"##row":10524,"renovation":10525,"##cast":10526,"infant":10527,"bankruptcy":10528,"roses":10529,"aftermath":10530,"pavilion":10531,"##yer":10532,"carpenter":10533,"withdrawal":10534,"ladder":10535,"##hy":10536,"discussing":10537,"popped":10538,"reliable":10539,"agreements":10540,"rochester":10541,"##abad":10542,"curves":10543,"bombers":10544,"220":10545,"rao":10546,"reverend":10547,"decreased":10548,"choosing":10549,"107":10550,"stiff":10551,"consulting":10552,"naples":10553,"crawford":10554,"tracy":10555,"ka":10556,"ribbon":10557,"cops":10558,"##lee":10559,"crushed":10560,"deciding":10561,"unified":10562,"teenager":10563,"accepting":10564,"flagship":10565,"explorer":10566,"poles":10567,"sanchez":10568,"inspection":10569,"revived":10570,"skilled":10571,"induced":10572,"exchanged":10573,"flee":10574,"locals":10575,"tragedy":10576,"swallow":10577,"loading":10578,"hanna":10579,"demonstrate":10580,"##ela":10581,"salvador":10582,"flown":10583,"contestants":10584,"civilization":10585,"##ines":10586,"wanna":10587,"rhodes":10588,"fletcher":10589,"hector":10590,"knocking":10591,"considers":10592,"##ough":10593,"nash":10594,"mechanisms":10595,"sensed":10596,"mentally":10597,"walt":10598,"unclear":10599,"##eus":10600,"renovated":10601,"madame":10602,"##cks":10603,"crews":10604,"governmental":10605,"##hin":10606,"undertaken":10607,"monkey":10608,"##ben":10609,"##ato":10610,"fatal":10611,"armored":10612,"copa":10613,"caves":10614,"governance":10615,"grasp":10616,"perception":10617,"certification":10618,"froze":10619,"damp":10620,"tugged":10621,"wyoming":10622,"##rg":10623,"##ero":10624,"newman":10625,"##lor":10626,"nerves":10627,"curiosity":10628,"graph":10629,"115":10630,"##ami":10631,"withdraw":10632,"tunnels":10633,"dull":10634,"meredith":10635,"moss":10636,"exhibits":10637,"neighbors":10638,"communicate":10639,"accuracy":10640,"explored":10641,"raiders":10642,"republicans":10643,"secular":10644,"kat":10645,"superman":10646,"penny":10647,"criticised":10648,"##tch":10649,"freed":10650,"update":10651,"conviction":10652,"wade":10653,"ham":10654,"likewise":10655,"delegation":10656,"gotta":10657,"doll":10658,"promises":10659,"technological":10660,"myth":10661,"nationality":10662,"resolve":10663,"convent":10664,"##mark":10665,"sharon":10666,"dig":10667,"sip":10668,"coordinator":10669,"entrepreneur":10670,"fold":10671,"##dine":10672,"capability":10673,"councillor":10674,"synonym":10675,"blown":10676,"swan":10677,"cursed":10678,"1815":10679,"jonas":10680,"haired":10681,"sofa":10682,"canvas":10683,"keeper":10684,"rivalry":10685,"##hart":10686,"rapper":10687,"speedway":10688,"swords":10689,"postal":10690,"maxwell":10691,"estonia":10692,"potter":10693,"recurring":10694,"##nn":10695,"##ave":10696,"errors":10697,"##oni":10698,"cognitive":10699,"1834":10700,"##²":10701,"claws":10702,"nadu":10703,"roberto":10704,"bce":10705,"wrestler":10706,"ellie":10707,"##ations":10708,"infinite":10709,"ink":10710,"##tia":10711,"presumably":10712,"finite":10713,"staircase":10714,"108":10715,"noel":10716,"patricia":10717,"nacional":10718,"##cation":10719,"chill":10720,"eternal":10721,"tu":10722,"preventing":10723,"prussia":10724,"fossil":10725,"limbs":10726,"##logist":10727,"ernst":10728,"frog":10729,"perez":10730,"rene":10731,"##ace":10732,"pizza":10733,"prussian":10734,"##ios":10735,"##vy":10736,"molecules":10737,"regulatory":10738,"answering":10739,"opinions":10740,"sworn":10741,"lengths":10742,"supposedly":10743,"hypothesis":10744,"upward":10745,"habitats":10746,"seating":10747,"ancestors":10748,"drank":10749,"yield":10750,"hd":10751,"synthesis":10752,"researcher":10753,"modest":10754,"##var":10755,"mothers":10756,"peered":10757,"voluntary":10758,"homeland":10759,"##the":10760,"acclaim":10761,"##igan":10762,"static":10763,"valve":10764,"luxembourg":10765,"alto":10766,"carroll":10767,"fe":10768,"receptor":10769,"norton":10770,"ambulance":10771,"##tian":10772,"johnston":10773,"catholics":10774,"depicting":10775,"jointly":10776,"elephant":10777,"gloria":10778,"mentor":10779,"badge":10780,"ahmad":10781,"distinguish":10782,"remarked":10783,"councils":10784,"precisely":10785,"allison":10786,"advancing":10787,"detection":10788,"crowded":10789,"##10":10790,"cooperative":10791,"ankle":10792,"mercedes":10793,"dagger":10794,"surrendered":10795,"pollution":10796,"commit":10797,"subway":10798,"jeffrey":10799,"lesson":10800,"sculptures":10801,"provider":10802,"##fication":10803,"membrane":10804,"timothy":10805,"rectangular":10806,"fiscal":10807,"heating":10808,"teammate":10809,"basket":10810,"particle":10811,"anonymous":10812,"deployment":10813,"##ple":10814,"missiles":10815,"courthouse":10816,"proportion":10817,"shoe":10818,"sec":10819,"##ller":10820,"complaints":10821,"forbes":10822,"blacks":10823,"abandon":10824,"remind":10825,"sizes":10826,"overwhelming":10827,"autobiography":10828,"natalie":10829,"##awa":10830,"risks":10831,"contestant":10832,"countryside":10833,"babies":10834,"scorer":10835,"invaded":10836,"enclosed":10837,"proceed":10838,"hurling":10839,"disorders":10840,"##cu":10841,"reflecting":10842,"continuously":10843,"cruiser":10844,"graduates":10845,"freeway":10846,"investigated":10847,"ore":10848,"deserved":10849,"maid":10850,"blocking":10851,"phillip":10852,"jorge":10853,"shakes":10854,"dove":10855,"mann":10856,"variables":10857,"lacked":10858,"burden":10859,"accompanying":10860,"que":10861,"consistently":10862,"organizing":10863,"provisional":10864,"complained":10865,"endless":10866,"##rm":10867,"tubes":10868,"juice":10869,"georges":10870,"krishna":10871,"mick":10872,"labels":10873,"thriller":10874,"##uch":10875,"laps":10876,"arcade":10877,"sage":10878,"snail":10879,"##table":10880,"shannon":10881,"fi":10882,"laurence":10883,"seoul":10884,"vacation":10885,"presenting":10886,"hire":10887,"churchill":10888,"surprisingly":10889,"prohibited":10890,"savannah":10891,"technically":10892,"##oli":10893,"170":10894,"##lessly":10895,"testimony":10896,"suited":10897,"speeds":10898,"toys":10899,"romans":10900,"mlb":10901,"flowering":10902,"measurement":10903,"talented":10904,"kay":10905,"settings":10906,"charleston":10907,"expectations":10908,"shattered":10909,"achieving":10910,"triumph":10911,"ceremonies":10912,"portsmouth":10913,"lanes":10914,"mandatory":10915,"loser":10916,"stretching":10917,"cologne":10918,"realizes":10919,"seventy":10920,"cornell":10921,"careers":10922,"webb":10923,"##ulating":10924,"americas":10925,"budapest":10926,"ava":10927,"suspicion":10928,"##ison":10929,"yo":10930,"conrad":10931,"##hai":10932,"sterling":10933,"jessie":10934,"rector":10935,"##az":10936,"1831":10937,"transform":10938,"organize":10939,"loans":10940,"christine":10941,"volcanic":10942,"warrant":10943,"slender":10944,"summers":10945,"subfamily":10946,"newer":10947,"danced":10948,"dynamics":10949,"rhine":10950,"proceeds":10951,"heinrich":10952,"gastropod":10953,"commands":10954,"sings":10955,"facilitate":10956,"easter":10957,"ra":10958,"positioned":10959,"responses":10960,"expense":10961,"fruits":10962,"yanked":10963,"imported":10964,"25th":10965,"velvet":10966,"vic":10967,"primitive":10968,"tribune":10969,"baldwin":10970,"neighbourhood":10971,"donna":10972,"rip":10973,"hay":10974,"pr":10975,"##uro":10976,"1814":10977,"espn":10978,"welcomed":10979,"##aria":10980,"qualifier":10981,"glare":10982,"highland":10983,"timing":10984,"##cted":10985,"shells":10986,"eased":10987,"geometry":10988,"louder":10989,"exciting":10990,"slovakia":10991,"##sion":10992,"##iz":10993,"##lot":10994,"savings":10995,"prairie":10996,"##ques":10997,"marching":10998,"rafael":10999,"tonnes":11000,"##lled":11001,"curtain":11002,"preceding":11003,"shy":11004,"heal":11005,"greene":11006,"worthy":11007,"##pot":11008,"detachment":11009,"bury":11010,"sherman":11011,"##eck":11012,"reinforced":11013,"seeks":11014,"bottles":11015,"contracted":11016,"duchess":11017,"outfit":11018,"walsh":11019,"##sc":11020,"mickey":11021,"##ase":11022,"geoffrey":11023,"archer":11024,"squeeze":11025,"dawson":11026,"eliminate":11027,"invention":11028,"##enberg":11029,"neal":11030,"##eth":11031,"stance":11032,"dealer":11033,"coral":11034,"maple":11035,"retire":11036,"polo":11037,"simplified":11038,"##ht":11039,"1833":11040,"hid":11041,"watts":11042,"backwards":11043,"jules":11044,"##oke":11045,"genesis":11046,"mt":11047,"frames":11048,"rebounds":11049,"burma":11050,"woodland":11051,"moist":11052,"santos":11053,"whispers":11054,"drained":11055,"subspecies":11056,"##aa":11057,"streaming":11058,"ulster":11059,"burnt":11060,"correspondence":11061,"maternal":11062,"gerard":11063,"denis":11064,"stealing":11065,"##load":11066,"genius":11067,"duchy":11068,"##oria":11069,"inaugurated":11070,"momentum":11071,"suits":11072,"placement":11073,"sovereign":11074,"clause":11075,"thames":11076,"##hara":11077,"confederation":11078,"reservation":11079,"sketch":11080,"yankees":11081,"lets":11082,"rotten":11083,"charm":11084,"hal":11085,"verses":11086,"ultra":11087,"commercially":11088,"dot":11089,"salon":11090,"citation":11091,"adopt":11092,"winnipeg":11093,"mist":11094,"allocated":11095,"cairo":11096,"##boy":11097,"jenkins":11098,"interference":11099,"objectives":11100,"##wind":11101,"1820":11102,"portfolio":11103,"armoured":11104,"sectors":11105,"##eh":11106,"initiatives":11107,"##world":11108,"integrity":11109,"exercises":11110,"robe":11111,"tap":11112,"ab":11113,"gazed":11114,"##tones":11115,"distracted":11116,"rulers":11117,"111":11118,"favorable":11119,"jerome":11120,"tended":11121,"cart":11122,"factories":11123,"##eri":11124,"diplomat":11125,"valued":11126,"gravel":11127,"charitable":11128,"##try":11129,"calvin":11130,"exploring":11131,"chang":11132,"shepherd":11133,"terrace":11134,"pdf":11135,"pupil":11136,"##ural":11137,"reflects":11138,"ups":11139,"##rch":11140,"governors":11141,"shelf":11142,"depths":11143,"##nberg":11144,"trailed":11145,"crest":11146,"tackle":11147,"##nian":11148,"##ats":11149,"hatred":11150,"##kai":11151,"clare":11152,"makers":11153,"ethiopia":11154,"longtime":11155,"detected":11156,"embedded":11157,"lacking":11158,"slapped":11159,"rely":11160,"thomson":11161,"anticipation":11162,"iso":11163,"morton":11164,"successive":11165,"agnes":11166,"screenwriter":11167,"straightened":11168,"philippe":11169,"playwright":11170,"haunted":11171,"licence":11172,"iris":11173,"intentions":11174,"sutton":11175,"112":11176,"logical":11177,"correctly":11178,"##weight":11179,"branded":11180,"licked":11181,"tipped":11182,"silva":11183,"ricky":11184,"narrator":11185,"requests":11186,"##ents":11187,"greeted":11188,"supernatural":11189,"cow":11190,"##wald":11191,"lung":11192,"refusing":11193,"employer":11194,"strait":11195,"gaelic":11196,"liner":11197,"##piece":11198,"zoe":11199,"sabha":11200,"##mba":11201,"driveway":11202,"harvest":11203,"prints":11204,"bates":11205,"reluctantly":11206,"threshold":11207,"algebra":11208,"ira":11209,"wherever":11210,"coupled":11211,"240":11212,"assumption":11213,"picks":11214,"##air":11215,"designers":11216,"raids":11217,"gentlemen":11218,"##ean":11219,"roller":11220,"blowing":11221,"leipzig":11222,"locks":11223,"screw":11224,"dressing":11225,"strand":11226,"##lings":11227,"scar":11228,"dwarf":11229,"depicts":11230,"##nu":11231,"nods":11232,"##mine":11233,"differ":11234,"boris":11235,"##eur":11236,"yuan":11237,"flip":11238,"##gie":11239,"mob":11240,"invested":11241,"questioning":11242,"applying":11243,"##ture":11244,"shout":11245,"##sel":11246,"gameplay":11247,"blamed":11248,"illustrations":11249,"bothered":11250,"weakness":11251,"rehabilitation":11252,"##of":11253,"##zes":11254,"envelope":11255,"rumors":11256,"miners":11257,"leicester":11258,"subtle":11259,"kerry":11260,"##ico":11261,"ferguson":11262,"##fu":11263,"premiership":11264,"ne":11265,"##cat":11266,"bengali":11267,"prof":11268,"catches":11269,"remnants":11270,"dana":11271,"##rily":11272,"shouting":11273,"presidents":11274,"baltic":11275,"ought":11276,"ghosts":11277,"dances":11278,"sailors":11279,"shirley":11280,"fancy":11281,"dominic":11282,"##bie":11283,"madonna":11284,"##rick":11285,"bark":11286,"buttons":11287,"gymnasium":11288,"ashes":11289,"liver":11290,"toby":11291,"oath":11292,"providence":11293,"doyle":11294,"evangelical":11295,"nixon":11296,"cement":11297,"carnegie":11298,"embarked":11299,"hatch":11300,"surroundings":11301,"guarantee":11302,"needing":11303,"pirate":11304,"essence":11305,"##bee":11306,"filter":11307,"crane":11308,"hammond":11309,"projected":11310,"immune":11311,"percy":11312,"twelfth":11313,"##ult":11314,"regent":11315,"doctoral":11316,"damon":11317,"mikhail":11318,"##ichi":11319,"lu":11320,"critically":11321,"elect":11322,"realised":11323,"abortion":11324,"acute":11325,"screening":11326,"mythology":11327,"steadily":11328,"##fc":11329,"frown":11330,"nottingham":11331,"kirk":11332,"wa":11333,"minneapolis":11334,"##rra":11335,"module":11336,"algeria":11337,"mc":11338,"nautical":11339,"encounters":11340,"surprising":11341,"statues":11342,"availability":11343,"shirts":11344,"pie":11345,"alma":11346,"brows":11347,"munster":11348,"mack":11349,"soup":11350,"crater":11351,"tornado":11352,"sanskrit":11353,"cedar":11354,"explosive":11355,"bordered":11356,"dixon":11357,"planets":11358,"stamp":11359,"exam":11360,"happily":11361,"##bble":11362,"carriers":11363,"kidnapped":11364,"##vis":11365,"accommodation":11366,"emigrated":11367,"##met":11368,"knockout":11369,"correspondent":11370,"violation":11371,"profits":11372,"peaks":11373,"lang":11374,"specimen":11375,"agenda":11376,"ancestry":11377,"pottery":11378,"spelling":11379,"equations":11380,"obtaining":11381,"ki":11382,"linking":11383,"1825":11384,"debris":11385,"asylum":11386,"##20":11387,"buddhism":11388,"teddy":11389,"##ants":11390,"gazette":11391,"##nger":11392,"##sse":11393,"dental":11394,"eligibility":11395,"utc":11396,"fathers":11397,"averaged":11398,"zimbabwe":11399,"francesco":11400,"coloured":11401,"hissed":11402,"translator":11403,"lynch":11404,"mandate":11405,"humanities":11406,"mackenzie":11407,"uniforms":11408,"lin":11409,"##iana":11410,"##gio":11411,"asset":11412,"mhz":11413,"fitting":11414,"samantha":11415,"genera":11416,"wei":11417,"rim":11418,"beloved":11419,"shark":11420,"riot":11421,"entities":11422,"expressions":11423,"indo":11424,"carmen":11425,"slipping":11426,"owing":11427,"abbot":11428,"neighbor":11429,"sidney":11430,"##av":11431,"rats":11432,"recommendations":11433,"encouraging":11434,"squadrons":11435,"anticipated":11436,"commanders":11437,"conquered":11438,"##oto":11439,"donations":11440,"diagnosed":11441,"##mond":11442,"divide":11443,"##iva":11444,"guessed":11445,"decoration":11446,"vernon":11447,"auditorium":11448,"revelation":11449,"conversations":11450,"##kers":11451,"##power":11452,"herzegovina":11453,"dash":11454,"alike":11455,"protested":11456,"lateral":11457,"herman":11458,"accredited":11459,"mg":11460,"##gent":11461,"freeman":11462,"mel":11463,"fiji":11464,"crow":11465,"crimson":11466,"##rine":11467,"livestock":11468,"##pped":11469,"humanitarian":11470,"bored":11471,"oz":11472,"whip":11473,"##lene":11474,"##ali":11475,"legitimate":11476,"alter":11477,"grinning":11478,"spelled":11479,"anxious":11480,"oriental":11481,"wesley":11482,"##nin":11483,"##hole":11484,"carnival":11485,"controller":11486,"detect":11487,"##ssa":11488,"bowed":11489,"educator":11490,"kosovo":11491,"macedonia":11492,"##sin":11493,"occupy":11494,"mastering":11495,"stephanie":11496,"janeiro":11497,"para":11498,"unaware":11499,"nurses":11500,"noon":11501,"135":11502,"cam":11503,"hopefully":11504,"ranger":11505,"combine":11506,"sociology":11507,"polar":11508,"rica":11509,"##eer":11510,"neill":11511,"##sman":11512,"holocaust":11513,"##ip":11514,"doubled":11515,"lust":11516,"1828":11517,"109":11518,"decent":11519,"cooling":11520,"unveiled":11521,"##card":11522,"1829":11523,"nsw":11524,"homer":11525,"chapman":11526,"meyer":11527,"##gin":11528,"dive":11529,"mae":11530,"reagan":11531,"expertise":11532,"##gled":11533,"darwin":11534,"brooke":11535,"sided":11536,"prosecution":11537,"investigating":11538,"comprised":11539,"petroleum":11540,"genres":11541,"reluctant":11542,"differently":11543,"trilogy":11544,"johns":11545,"vegetables":11546,"corpse":11547,"highlighted":11548,"lounge":11549,"pension":11550,"unsuccessfully":11551,"elegant":11552,"aided":11553,"ivory":11554,"beatles":11555,"amelia":11556,"cain":11557,"dubai":11558,"sunny":11559,"immigrant":11560,"babe":11561,"click":11562,"##nder":11563,"underwater":11564,"pepper":11565,"combining":11566,"mumbled":11567,"atlas":11568,"horns":11569,"accessed":11570,"ballad":11571,"physicians":11572,"homeless":11573,"gestured":11574,"rpm":11575,"freak":11576,"louisville":11577,"corporations":11578,"patriots":11579,"prizes":11580,"rational":11581,"warn":11582,"modes":11583,"decorative":11584,"overnight":11585,"din":11586,"troubled":11587,"phantom":11588,"##ort":11589,"monarch":11590,"sheer":11591,"##dorf":11592,"generals":11593,"guidelines":11594,"organs":11595,"addresses":11596,"##zon":11597,"enhance":11598,"curling":11599,"parishes":11600,"cord":11601,"##kie":11602,"linux":11603,"caesar":11604,"deutsche":11605,"bavaria":11606,"##bia":11607,"coleman":11608,"cyclone":11609,"##eria":11610,"bacon":11611,"petty":11612,"##yama":11613,"##old":11614,"hampton":11615,"diagnosis":11616,"1824":11617,"throws":11618,"complexity":11619,"rita":11620,"disputed":11621,"##₃":11622,"pablo":11623,"##sch":11624,"marketed":11625,"trafficking":11626,"##ulus":11627,"examine":11628,"plague":11629,"formats":11630,"##oh":11631,"vault":11632,"faithful":11633,"##bourne":11634,"webster":11635,"##ox":11636,"highlights":11637,"##ient":11638,"##ann":11639,"phones":11640,"vacuum":11641,"sandwich":11642,"modeling":11643,"##gated":11644,"bolivia":11645,"clergy":11646,"qualities":11647,"isabel":11648,"##nas":11649,"##ars":11650,"wears":11651,"screams":11652,"reunited":11653,"annoyed":11654,"bra":11655,"##ancy":11656,"##rate":11657,"differential":11658,"transmitter":11659,"tattoo":11660,"container":11661,"poker":11662,"##och":11663,"excessive":11664,"resides":11665,"cowboys":11666,"##tum":11667,"augustus":11668,"trash":11669,"providers":11670,"statute":11671,"retreated":11672,"balcony":11673,"reversed":11674,"void":11675,"storey":11676,"preceded":11677,"masses":11678,"leap":11679,"laughs":11680,"neighborhoods":11681,"wards":11682,"schemes":11683,"falcon":11684,"santo":11685,"battlefield":11686,"pad":11687,"ronnie":11688,"thread":11689,"lesbian":11690,"venus":11691,"##dian":11692,"beg":11693,"sandstone":11694,"daylight":11695,"punched":11696,"gwen":11697,"analog":11698,"stroked":11699,"wwe":11700,"acceptable":11701,"measurements":11702,"dec":11703,"toxic":11704,"##kel":11705,"adequate":11706,"surgical":11707,"economist":11708,"parameters":11709,"varsity":11710,"##sberg":11711,"quantity":11712,"ella":11713,"##chy":11714,"##rton":11715,"countess":11716,"generating":11717,"precision":11718,"diamonds":11719,"expressway":11720,"ga":11721,"##ı":11722,"1821":11723,"uruguay":11724,"talents":11725,"galleries":11726,"expenses":11727,"scanned":11728,"colleague":11729,"outlets":11730,"ryder":11731,"lucien":11732,"##ila":11733,"paramount":11734,"##bon":11735,"syracuse":11736,"dim":11737,"fangs":11738,"gown":11739,"sweep":11740,"##sie":11741,"toyota":11742,"missionaries":11743,"websites":11744,"##nsis":11745,"sentences":11746,"adviser":11747,"val":11748,"trademark":11749,"spells":11750,"##plane":11751,"patience":11752,"starter":11753,"slim":11754,"##borg":11755,"toe":11756,"incredibly":11757,"shoots":11758,"elliot":11759,"nobility":11760,"##wyn":11761,"cowboy":11762,"endorsed":11763,"gardner":11764,"tendency":11765,"persuaded":11766,"organisms":11767,"emissions":11768,"kazakhstan":11769,"amused":11770,"boring":11771,"chips":11772,"themed":11773,"##hand":11774,"llc":11775,"constantinople":11776,"chasing":11777,"systematic":11778,"guatemala":11779,"borrowed":11780,"erin":11781,"carey":11782,"##hard":11783,"highlands":11784,"struggles":11785,"1810":11786,"##ifying":11787,"##ced":11788,"wong":11789,"exceptions":11790,"develops":11791,"enlarged":11792,"kindergarten":11793,"castro":11794,"##ern":11795,"##rina":11796,"leigh":11797,"zombie":11798,"juvenile":11799,"##most":11800,"consul":11801,"##nar":11802,"sailor":11803,"hyde":11804,"clarence":11805,"intensive":11806,"pinned":11807,"nasty":11808,"useless":11809,"jung":11810,"clayton":11811,"stuffed":11812,"exceptional":11813,"ix":11814,"apostolic":11815,"230":11816,"transactions":11817,"##dge":11818,"exempt":11819,"swinging":11820,"cove":11821,"religions":11822,"##ash":11823,"shields":11824,"dairy":11825,"bypass":11826,"190":11827,"pursuing":11828,"bug":11829,"joyce":11830,"bombay":11831,"chassis":11832,"southampton":11833,"chat":11834,"interact":11835,"redesignated":11836,"##pen":11837,"nascar":11838,"pray":11839,"salmon":11840,"rigid":11841,"regained":11842,"malaysian":11843,"grim":11844,"publicity":11845,"constituted":11846,"capturing":11847,"toilet":11848,"delegate":11849,"purely":11850,"tray":11851,"drift":11852,"loosely":11853,"striker":11854,"weakened":11855,"trinidad":11856,"mitch":11857,"itv":11858,"defines":11859,"transmitted":11860,"ming":11861,"scarlet":11862,"nodding":11863,"fitzgerald":11864,"fu":11865,"narrowly":11866,"sp":11867,"tooth":11868,"standings":11869,"virtue":11870,"##₁":11871,"##wara":11872,"##cting":11873,"chateau":11874,"gloves":11875,"lid":11876,"##nel":11877,"hurting":11878,"conservatory":11879,"##pel":11880,"sinclair":11881,"reopened":11882,"sympathy":11883,"nigerian":11884,"strode":11885,"advocated":11886,"optional":11887,"chronic":11888,"discharge":11889,"##rc":11890,"suck":11891,"compatible":11892,"laurel":11893,"stella":11894,"shi":11895,"fails":11896,"wage":11897,"dodge":11898,"128":11899,"informal":11900,"sorts":11901,"levi":11902,"buddha":11903,"villagers":11904,"##aka":11905,"chronicles":11906,"heavier":11907,"summoned":11908,"gateway":11909,"3000":11910,"eleventh":11911,"jewelry":11912,"translations":11913,"accordingly":11914,"seas":11915,"##ency":11916,"fiber":11917,"pyramid":11918,"cubic":11919,"dragging":11920,"##ista":11921,"caring":11922,"##ops":11923,"android":11924,"contacted":11925,"lunar":11926,"##dt":11927,"kai":11928,"lisbon":11929,"patted":11930,"1826":11931,"sacramento":11932,"theft":11933,"madagascar":11934,"subtropical":11935,"disputes":11936,"ta":11937,"holidays":11938,"piper":11939,"willow":11940,"mare":11941,"cane":11942,"itunes":11943,"newfoundland":11944,"benny":11945,"companions":11946,"dong":11947,"raj":11948,"observe":11949,"roar":11950,"charming":11951,"plaque":11952,"tibetan":11953,"fossils":11954,"enacted":11955,"manning":11956,"bubble":11957,"tina":11958,"tanzania":11959,"##eda":11960,"##hir":11961,"funk":11962,"swamp":11963,"deputies":11964,"cloak":11965,"ufc":11966,"scenario":11967,"par":11968,"scratch":11969,"metals":11970,"anthem":11971,"guru":11972,"engaging":11973,"specially":11974,"##boat":11975,"dialects":11976,"nineteen":11977,"cecil":11978,"duet":11979,"disability":11980,"messenger":11981,"unofficial":11982,"##lies":11983,"defunct":11984,"eds":11985,"moonlight":11986,"drainage":11987,"surname":11988,"puzzle":11989,"honda":11990,"switching":11991,"conservatives":11992,"mammals":11993,"knox":11994,"broadcaster":11995,"sidewalk":11996,"cope":11997,"##ried":11998,"benson":11999,"princes":12000,"peterson":12001,"##sal":12002,"bedford":12003,"sharks":12004,"eli":12005,"wreck":12006,"alberto":12007,"gasp":12008,"archaeology":12009,"lgbt":12010,"teaches":12011,"securities":12012,"madness":12013,"compromise":12014,"waving":12015,"coordination":12016,"davidson":12017,"visions":12018,"leased":12019,"possibilities":12020,"eighty":12021,"jun":12022,"fernandez":12023,"enthusiasm":12024,"assassin":12025,"sponsorship":12026,"reviewer":12027,"kingdoms":12028,"estonian":12029,"laboratories":12030,"##fy":12031,"##nal":12032,"applies":12033,"verb":12034,"celebrations":12035,"##zzo":12036,"rowing":12037,"lightweight":12038,"sadness":12039,"submit":12040,"mvp":12041,"balanced":12042,"dude":12043,"##vas":12044,"explicitly":12045,"metric":12046,"magnificent":12047,"mound":12048,"brett":12049,"mohammad":12050,"mistakes":12051,"irregular":12052,"##hing":12053,"##ass":12054,"sanders":12055,"betrayed":12056,"shipped":12057,"surge":12058,"##enburg":12059,"reporters":12060,"termed":12061,"georg":12062,"pity":12063,"verbal":12064,"bulls":12065,"abbreviated":12066,"enabling":12067,"appealed":12068,"##are":12069,"##atic":12070,"sicily":12071,"sting":12072,"heel":12073,"sweetheart":12074,"bart":12075,"spacecraft":12076,"brutal":12077,"monarchy":12078,"##tter":12079,"aberdeen":12080,"cameo":12081,"diane":12082,"##ub":12083,"survivor":12084,"clyde":12085,"##aries":12086,"complaint":12087,"##makers":12088,"clarinet":12089,"delicious":12090,"chilean":12091,"karnataka":12092,"coordinates":12093,"1818":12094,"panties":12095,"##rst":12096,"pretending":12097,"ar":12098,"dramatically":12099,"kiev":12100,"bella":12101,"tends":12102,"distances":12103,"113":12104,"catalog":12105,"launching":12106,"instances":12107,"telecommunications":12108,"portable":12109,"lindsay":12110,"vatican":12111,"##eim":12112,"angles":12113,"aliens":12114,"marker":12115,"stint":12116,"screens":12117,"bolton":12118,"##rne":12119,"judy":12120,"wool":12121,"benedict":12122,"plasma":12123,"europa":12124,"spark":12125,"imaging":12126,"filmmaker":12127,"swiftly":12128,"##een":12129,"contributor":12130,"##nor":12131,"opted":12132,"stamps":12133,"apologize":12134,"financing":12135,"butter":12136,"gideon":12137,"sophisticated":12138,"alignment":12139,"avery":12140,"chemicals":12141,"yearly":12142,"speculation":12143,"prominence":12144,"professionally":12145,"##ils":12146,"immortal":12147,"institutional":12148,"inception":12149,"wrists":12150,"identifying":12151,"tribunal":12152,"derives":12153,"gains":12154,"##wo":12155,"papal":12156,"preference":12157,"linguistic":12158,"vince":12159,"operative":12160,"brewery":12161,"##ont":12162,"unemployment":12163,"boyd":12164,"##ured":12165,"##outs":12166,"albeit":12167,"prophet":12168,"1813":12169,"bi":12170,"##rr":12171,"##face":12172,"##rad":12173,"quarterly":12174,"asteroid":12175,"cleaned":12176,"radius":12177,"temper":12178,"##llen":12179,"telugu":12180,"jerk":12181,"viscount":12182,"menu":12183,"##ote":12184,"glimpse":12185,"##aya":12186,"yacht":12187,"hawaiian":12188,"baden":12189,"##rl":12190,"laptop":12191,"readily":12192,"##gu":12193,"monetary":12194,"offshore":12195,"scots":12196,"watches":12197,"##yang":12198,"##arian":12199,"upgrade":12200,"needle":12201,"xbox":12202,"lea":12203,"encyclopedia":12204,"flank":12205,"fingertips":12206,"##pus":12207,"delight":12208,"teachings":12209,"confirm":12210,"roth":12211,"beaches":12212,"midway":12213,"winters":12214,"##iah":12215,"teasing":12216,"daytime":12217,"beverly":12218,"gambling":12219,"bonnie":12220,"##backs":12221,"regulated":12222,"clement":12223,"hermann":12224,"tricks":12225,"knot":12226,"##shing":12227,"##uring":12228,"##vre":12229,"detached":12230,"ecological":12231,"owed":12232,"specialty":12233,"byron":12234,"inventor":12235,"bats":12236,"stays":12237,"screened":12238,"unesco":12239,"midland":12240,"trim":12241,"affection":12242,"##ander":12243,"##rry":12244,"jess":12245,"thoroughly":12246,"feedback":12247,"##uma":12248,"chennai":12249,"strained":12250,"heartbeat":12251,"wrapping":12252,"overtime":12253,"pleaded":12254,"##sworth":12255,"mon":12256,"leisure":12257,"oclc":12258,"##tate":12259,"##ele":12260,"feathers":12261,"angelo":12262,"thirds":12263,"nuts":12264,"surveys":12265,"clever":12266,"gill":12267,"commentator":12268,"##dos":12269,"darren":12270,"rides":12271,"gibraltar":12272,"##nc":12273,"##mu":12274,"dissolution":12275,"dedication":12276,"shin":12277,"meals":12278,"saddle":12279,"elvis":12280,"reds":12281,"chaired":12282,"taller":12283,"appreciation":12284,"functioning":12285,"niece":12286,"favored":12287,"advocacy":12288,"robbie":12289,"criminals":12290,"suffolk":12291,"yugoslav":12292,"passport":12293,"constable":12294,"congressman":12295,"hastings":12296,"vera":12297,"##rov":12298,"consecrated":12299,"sparks":12300,"ecclesiastical":12301,"confined":12302,"##ovich":12303,"muller":12304,"floyd":12305,"nora":12306,"1822":12307,"paved":12308,"1827":12309,"cumberland":12310,"ned":12311,"saga":12312,"spiral":12313,"##flow":12314,"appreciated":12315,"yi":12316,"collaborative":12317,"treating":12318,"similarities":12319,"feminine":12320,"finishes":12321,"##ib":12322,"jade":12323,"import":12324,"##nse":12325,"##hot":12326,"champagne":12327,"mice":12328,"securing":12329,"celebrities":12330,"helsinki":12331,"attributes":12332,"##gos":12333,"cousins":12334,"phases":12335,"ache":12336,"lucia":12337,"gandhi":12338,"submission":12339,"vicar":12340,"spear":12341,"shine":12342,"tasmania":12343,"biting":12344,"detention":12345,"constitute":12346,"tighter":12347,"seasonal":12348,"##gus":12349,"terrestrial":12350,"matthews":12351,"##oka":12352,"effectiveness":12353,"parody":12354,"philharmonic":12355,"##onic":12356,"1816":12357,"strangers":12358,"encoded":12359,"consortium":12360,"guaranteed":12361,"regards":12362,"shifts":12363,"tortured":12364,"collision":12365,"supervisor":12366,"inform":12367,"broader":12368,"insight":12369,"theaters":12370,"armour":12371,"emeritus":12372,"blink":12373,"incorporates":12374,"mapping":12375,"##50":12376,"##ein":12377,"handball":12378,"flexible":12379,"##nta":12380,"substantially":12381,"generous":12382,"thief":12383,"##own":12384,"carr":12385,"loses":12386,"1793":12387,"prose":12388,"ucla":12389,"romeo":12390,"generic":12391,"metallic":12392,"realization":12393,"damages":12394,"mk":12395,"commissioners":12396,"zach":12397,"default":12398,"##ther":12399,"helicopters":12400,"lengthy":12401,"stems":12402,"spa":12403,"partnered":12404,"spectators":12405,"rogue":12406,"indication":12407,"penalties":12408,"teresa":12409,"1801":12410,"sen":12411,"##tric":12412,"dalton":12413,"##wich":12414,"irving":12415,"photographic":12416,"##vey":12417,"dell":12418,"deaf":12419,"peters":12420,"excluded":12421,"unsure":12422,"##vable":12423,"patterson":12424,"crawled":12425,"##zio":12426,"resided":12427,"whipped":12428,"latvia":12429,"slower":12430,"ecole":12431,"pipes":12432,"employers":12433,"maharashtra":12434,"comparable":12435,"va":12436,"textile":12437,"pageant":12438,"##gel":12439,"alphabet":12440,"binary":12441,"irrigation":12442,"chartered":12443,"choked":12444,"antoine":12445,"offs":12446,"waking":12447,"supplement":12448,"##wen":12449,"quantities":12450,"demolition":12451,"regain":12452,"locate":12453,"urdu":12454,"folks":12455,"alt":12456,"114":12457,"##mc":12458,"scary":12459,"andreas":12460,"whites":12461,"##ava":12462,"classrooms":12463,"mw":12464,"aesthetic":12465,"publishes":12466,"valleys":12467,"guides":12468,"cubs":12469,"johannes":12470,"bryant":12471,"conventions":12472,"affecting":12473,"##itt":12474,"drain":12475,"awesome":12476,"isolation":12477,"prosecutor":12478,"ambitious":12479,"apology":12480,"captive":12481,"downs":12482,"atmospheric":12483,"lorenzo":12484,"aisle":12485,"beef":12486,"foul":12487,"##onia":12488,"kidding":12489,"composite":12490,"disturbed":12491,"illusion":12492,"natives":12493,"##ffer":12494,"emi":12495,"rockets":12496,"riverside":12497,"wartime":12498,"painters":12499,"adolf":12500,"melted":12501,"##ail":12502,"uncertainty":12503,"simulation":12504,"hawks":12505,"progressed":12506,"meantime":12507,"builder":12508,"spray":12509,"breach":12510,"unhappy":12511,"regina":12512,"russians":12513,"##urg":12514,"determining":12515,"##tation":12516,"tram":12517,"1806":12518,"##quin":12519,"aging":12520,"##12":12521,"1823":12522,"garion":12523,"rented":12524,"mister":12525,"diaz":12526,"terminated":12527,"clip":12528,"1817":12529,"depend":12530,"nervously":12531,"disco":12532,"owe":12533,"defenders":12534,"shiva":12535,"notorious":12536,"disbelief":12537,"shiny":12538,"worcester":12539,"##gation":12540,"##yr":12541,"trailing":12542,"undertook":12543,"islander":12544,"belarus":12545,"limitations":12546,"watershed":12547,"fuller":12548,"overlooking":12549,"utilized":12550,"raphael":12551,"1819":12552,"synthetic":12553,"breakdown":12554,"klein":12555,"##nate":12556,"moaned":12557,"memoir":12558,"lamb":12559,"practicing":12560,"##erly":12561,"cellular":12562,"arrows":12563,"exotic":12564,"##graphy":12565,"witches":12566,"117":12567,"charted":12568,"rey":12569,"hut":12570,"hierarchy":12571,"subdivision":12572,"freshwater":12573,"giuseppe":12574,"aloud":12575,"reyes":12576,"qatar":12577,"marty":12578,"sideways":12579,"utterly":12580,"sexually":12581,"jude":12582,"prayers":12583,"mccarthy":12584,"softball":12585,"blend":12586,"damien":12587,"##gging":12588,"##metric":12589,"wholly":12590,"erupted":12591,"lebanese":12592,"negro":12593,"revenues":12594,"tasted":12595,"comparative":12596,"teamed":12597,"transaction":12598,"labeled":12599,"maori":12600,"sovereignty":12601,"parkway":12602,"trauma":12603,"gran":12604,"malay":12605,"121":12606,"advancement":12607,"descendant":12608,"2020":12609,"buzz":12610,"salvation":12611,"inventory":12612,"symbolic":12613,"##making":12614,"antarctica":12615,"mps":12616,"##gas":12617,"##bro":12618,"mohammed":12619,"myanmar":12620,"holt":12621,"submarines":12622,"tones":12623,"##lman":12624,"locker":12625,"patriarch":12626,"bangkok":12627,"emerson":12628,"remarks":12629,"predators":12630,"kin":12631,"afghan":12632,"confession":12633,"norwich":12634,"rental":12635,"emerge":12636,"advantages":12637,"##zel":12638,"rca":12639,"##hold":12640,"shortened":12641,"storms":12642,"aidan":12643,"##matic":12644,"autonomy":12645,"compliance":12646,"##quet":12647,"dudley":12648,"atp":12649,"##osis":12650,"1803":12651,"motto":12652,"documentation":12653,"summary":12654,"professors":12655,"spectacular":12656,"christina":12657,"archdiocese":12658,"flashing":12659,"innocence":12660,"remake":12661,"##dell":12662,"psychic":12663,"reef":12664,"scare":12665,"employ":12666,"rs":12667,"sticks":12668,"meg":12669,"gus":12670,"leans":12671,"##ude":12672,"accompany":12673,"bergen":12674,"tomas":12675,"##iko":12676,"doom":12677,"wages":12678,"pools":12679,"##nch":12680,"##bes":12681,"breasts":12682,"scholarly":12683,"alison":12684,"outline":12685,"brittany":12686,"breakthrough":12687,"willis":12688,"realistic":12689,"##cut":12690,"##boro":12691,"competitor":12692,"##stan":12693,"pike":12694,"picnic":12695,"icon":12696,"designing":12697,"commercials":12698,"washing":12699,"villain":12700,"skiing":12701,"micro":12702,"costumes":12703,"auburn":12704,"halted":12705,"executives":12706,"##hat":12707,"logistics":12708,"cycles":12709,"vowel":12710,"applicable":12711,"barrett":12712,"exclaimed":12713,"eurovision":12714,"eternity":12715,"ramon":12716,"##umi":12717,"##lls":12718,"modifications":12719,"sweeping":12720,"disgust":12721,"##uck":12722,"torch":12723,"aviv":12724,"ensuring":12725,"rude":12726,"dusty":12727,"sonic":12728,"donovan":12729,"outskirts":12730,"cu":12731,"pathway":12732,"##band":12733,"##gun":12734,"##lines":12735,"disciplines":12736,"acids":12737,"cadet":12738,"paired":12739,"##40":12740,"sketches":12741,"##sive":12742,"marriages":12743,"##⁺":12744,"folding":12745,"peers":12746,"slovak":12747,"implies":12748,"admired":12749,"##beck":12750,"1880s":12751,"leopold":12752,"instinct":12753,"attained":12754,"weston":12755,"megan":12756,"horace":12757,"##ination":12758,"dorsal":12759,"ingredients":12760,"evolutionary":12761,"##its":12762,"complications":12763,"deity":12764,"lethal":12765,"brushing":12766,"levy":12767,"deserted":12768,"institutes":12769,"posthumously":12770,"delivering":12771,"telescope":12772,"coronation":12773,"motivated":12774,"rapids":12775,"luc":12776,"flicked":12777,"pays":12778,"volcano":12779,"tanner":12780,"weighed":12781,"##nica":12782,"crowds":12783,"frankie":12784,"gifted":12785,"addressing":12786,"granddaughter":12787,"winding":12788,"##rna":12789,"constantine":12790,"gomez":12791,"##front":12792,"landscapes":12793,"rudolf":12794,"anthropology":12795,"slate":12796,"werewolf":12797,"##lio":12798,"astronomy":12799,"circa":12800,"rouge":12801,"dreaming":12802,"sack":12803,"knelt":12804,"drowned":12805,"naomi":12806,"prolific":12807,"tracked":12808,"freezing":12809,"herb":12810,"##dium":12811,"agony":12812,"randall":12813,"twisting":12814,"wendy":12815,"deposit":12816,"touches":12817,"vein":12818,"wheeler":12819,"##bbled":12820,"##bor":12821,"batted":12822,"retaining":12823,"tire":12824,"presently":12825,"compare":12826,"specification":12827,"daemon":12828,"nigel":12829,"##grave":12830,"merry":12831,"recommendation":12832,"czechoslovakia":12833,"sandra":12834,"ng":12835,"roma":12836,"##sts":12837,"lambert":12838,"inheritance":12839,"sheikh":12840,"winchester":12841,"cries":12842,"examining":12843,"##yle":12844,"comeback":12845,"cuisine":12846,"nave":12847,"##iv":12848,"ko":12849,"retrieve":12850,"tomatoes":12851,"barker":12852,"polished":12853,"defining":12854,"irene":12855,"lantern":12856,"personalities":12857,"begging":12858,"tract":12859,"swore":12860,"1809":12861,"175":12862,"##gic":12863,"omaha":12864,"brotherhood":12865,"##rley":12866,"haiti":12867,"##ots":12868,"exeter":12869,"##ete":12870,"##zia":12871,"steele":12872,"dumb":12873,"pearson":12874,"210":12875,"surveyed":12876,"elisabeth":12877,"trends":12878,"##ef":12879,"fritz":12880,"##rf":12881,"premium":12882,"bugs":12883,"fraction":12884,"calmly":12885,"viking":12886,"##birds":12887,"tug":12888,"inserted":12889,"unusually":12890,"##ield":12891,"confronted":12892,"distress":12893,"crashing":12894,"brent":12895,"turks":12896,"resign":12897,"##olo":12898,"cambodia":12899,"gabe":12900,"sauce":12901,"##kal":12902,"evelyn":12903,"116":12904,"extant":12905,"clusters":12906,"quarry":12907,"teenagers":12908,"luna":12909,"##lers":12910,"##ister":12911,"affiliation":12912,"drill":12913,"##ashi":12914,"panthers":12915,"scenic":12916,"libya":12917,"anita":12918,"strengthen":12919,"inscriptions":12920,"##cated":12921,"lace":12922,"sued":12923,"judith":12924,"riots":12925,"##uted":12926,"mint":12927,"##eta":12928,"preparations":12929,"midst":12930,"dub":12931,"challenger":12932,"##vich":12933,"mock":12934,"cf":12935,"displaced":12936,"wicket":12937,"breaths":12938,"enables":12939,"schmidt":12940,"analyst":12941,"##lum":12942,"ag":12943,"highlight":12944,"automotive":12945,"axe":12946,"josef":12947,"newark":12948,"sufficiently":12949,"resembles":12950,"50th":12951,"##pal":12952,"flushed":12953,"mum":12954,"traits":12955,"##ante":12956,"commodore":12957,"incomplete":12958,"warming":12959,"titular":12960,"ceremonial":12961,"ethical":12962,"118":12963,"celebrating":12964,"eighteenth":12965,"cao":12966,"lima":12967,"medalist":12968,"mobility":12969,"strips":12970,"snakes":12971,"##city":12972,"miniature":12973,"zagreb":12974,"barton":12975,"escapes":12976,"umbrella":12977,"automated":12978,"doubted":12979,"differs":12980,"cooled":12981,"georgetown":12982,"dresden":12983,"cooked":12984,"fade":12985,"wyatt":12986,"rna":12987,"jacobs":12988,"carlton":12989,"abundant":12990,"stereo":12991,"boost":12992,"madras":12993,"inning":12994,"##hia":12995,"spur":12996,"ip":12997,"malayalam":12998,"begged":12999,"osaka":13000,"groan":13001,"escaping":13002,"charging":13003,"dose":13004,"vista":13005,"##aj":13006,"bud":13007,"papa":13008,"communists":13009,"advocates":13010,"edged":13011,"tri":13012,"##cent":13013,"resemble":13014,"peaking":13015,"necklace":13016,"fried":13017,"montenegro":13018,"saxony":13019,"goose":13020,"glances":13021,"stuttgart":13022,"curator":13023,"recruit":13024,"grocery":13025,"sympathetic":13026,"##tting":13027,"##fort":13028,"127":13029,"lotus":13030,"randolph":13031,"ancestor":13032,"##rand":13033,"succeeding":13034,"jupiter":13035,"1798":13036,"macedonian":13037,"##heads":13038,"hiking":13039,"1808":13040,"handing":13041,"fischer":13042,"##itive":13043,"garbage":13044,"node":13045,"##pies":13046,"prone":13047,"singular":13048,"papua":13049,"inclined":13050,"attractions":13051,"italia":13052,"pouring":13053,"motioned":13054,"grandma":13055,"garnered":13056,"jacksonville":13057,"corp":13058,"ego":13059,"ringing":13060,"aluminum":13061,"##hausen":13062,"ordering":13063,"##foot":13064,"drawer":13065,"traders":13066,"synagogue":13067,"##play":13068,"##kawa":13069,"resistant":13070,"wandering":13071,"fragile":13072,"fiona":13073,"teased":13074,"var":13075,"hardcore":13076,"soaked":13077,"jubilee":13078,"decisive":13079,"exposition":13080,"mercer":13081,"poster":13082,"valencia":13083,"hale":13084,"kuwait":13085,"1811":13086,"##ises":13087,"##wr":13088,"##eed":13089,"tavern":13090,"gamma":13091,"122":13092,"johan":13093,"##uer":13094,"airways":13095,"amino":13096,"gil":13097,"##ury":13098,"vocational":13099,"domains":13100,"torres":13101,"##sp":13102,"generator":13103,"folklore":13104,"outcomes":13105,"##keeper":13106,"canberra":13107,"shooter":13108,"fl":13109,"beams":13110,"confrontation":13111,"##lling":13112,"##gram":13113,"feb":13114,"aligned":13115,"forestry":13116,"pipeline":13117,"jax":13118,"motorway":13119,"conception":13120,"decay":13121,"##tos":13122,"coffin":13123,"##cott":13124,"stalin":13125,"1805":13126,"escorted":13127,"minded":13128,"##nam":13129,"sitcom":13130,"purchasing":13131,"twilight":13132,"veronica":13133,"additions":13134,"passive":13135,"tensions":13136,"straw":13137,"123":13138,"frequencies":13139,"1804":13140,"refugee":13141,"cultivation":13142,"##iate":13143,"christie":13144,"clary":13145,"bulletin":13146,"crept":13147,"disposal":13148,"##rich":13149,"##zong":13150,"processor":13151,"crescent":13152,"##rol":13153,"bmw":13154,"emphasized":13155,"whale":13156,"nazis":13157,"aurora":13158,"##eng":13159,"dwelling":13160,"hauled":13161,"sponsors":13162,"toledo":13163,"mega":13164,"ideology":13165,"theatres":13166,"tessa":13167,"cerambycidae":13168,"saves":13169,"turtle":13170,"cone":13171,"suspects":13172,"kara":13173,"rusty":13174,"yelling":13175,"greeks":13176,"mozart":13177,"shades":13178,"cocked":13179,"participant":13180,"##tro":13181,"shire":13182,"spit":13183,"freeze":13184,"necessity":13185,"##cos":13186,"inmates":13187,"nielsen":13188,"councillors":13189,"loaned":13190,"uncommon":13191,"omar":13192,"peasants":13193,"botanical":13194,"offspring":13195,"daniels":13196,"formations":13197,"jokes":13198,"1794":13199,"pioneers":13200,"sigma":13201,"licensing":13202,"##sus":13203,"wheelchair":13204,"polite":13205,"1807":13206,"liquor":13207,"pratt":13208,"trustee":13209,"##uta":13210,"forewings":13211,"balloon":13212,"##zz":13213,"kilometre":13214,"camping":13215,"explicit":13216,"casually":13217,"shawn":13218,"foolish":13219,"teammates":13220,"nm":13221,"hassan":13222,"carrie":13223,"judged":13224,"satisfy":13225,"vanessa":13226,"knives":13227,"selective":13228,"cnn":13229,"flowed":13230,"##lice":13231,"eclipse":13232,"stressed":13233,"eliza":13234,"mathematician":13235,"cease":13236,"cultivated":13237,"##roy":13238,"commissions":13239,"browns":13240,"##ania":13241,"destroyers":13242,"sheridan":13243,"meadow":13244,"##rius":13245,"minerals":13246,"##cial":13247,"downstream":13248,"clash":13249,"gram":13250,"memoirs":13251,"ventures":13252,"baha":13253,"seymour":13254,"archie":13255,"midlands":13256,"edith":13257,"fare":13258,"flynn":13259,"invite":13260,"canceled":13261,"tiles":13262,"stabbed":13263,"boulder":13264,"incorporate":13265,"amended":13266,"camden":13267,"facial":13268,"mollusk":13269,"unreleased":13270,"descriptions":13271,"yoga":13272,"grabs":13273,"550":13274,"raises":13275,"ramp":13276,"shiver":13277,"##rose":13278,"coined":13279,"pioneering":13280,"tunes":13281,"qing":13282,"warwick":13283,"tops":13284,"119":13285,"melanie":13286,"giles":13287,"##rous":13288,"wandered":13289,"##inal":13290,"annexed":13291,"nov":13292,"30th":13293,"unnamed":13294,"##ished":13295,"organizational":13296,"airplane":13297,"normandy":13298,"stoke":13299,"whistle":13300,"blessing":13301,"violations":13302,"chased":13303,"holders":13304,"shotgun":13305,"##ctic":13306,"outlet":13307,"reactor":13308,"##vik":13309,"tires":13310,"tearing":13311,"shores":13312,"fortified":13313,"mascot":13314,"constituencies":13315,"nc":13316,"columnist":13317,"productive":13318,"tibet":13319,"##rta":13320,"lineage":13321,"hooked":13322,"oct":13323,"tapes":13324,"judging":13325,"cody":13326,"##gger":13327,"hansen":13328,"kashmir":13329,"triggered":13330,"##eva":13331,"solved":13332,"cliffs":13333,"##tree":13334,"resisted":13335,"anatomy":13336,"protesters":13337,"transparent":13338,"implied":13339,"##iga":13340,"injection":13341,"mattress":13342,"excluding":13343,"##mbo":13344,"defenses":13345,"helpless":13346,"devotion":13347,"##elli":13348,"growl":13349,"liberals":13350,"weber":13351,"phenomena":13352,"atoms":13353,"plug":13354,"##iff":13355,"mortality":13356,"apprentice":13357,"howe":13358,"convincing":13359,"aaa":13360,"swimmer":13361,"barber":13362,"leone":13363,"promptly":13364,"sodium":13365,"def":13366,"nowadays":13367,"arise":13368,"##oning":13369,"gloucester":13370,"corrected":13371,"dignity":13372,"norm":13373,"erie":13374,"##ders":13375,"elders":13376,"evacuated":13377,"sylvia":13378,"compression":13379,"##yar":13380,"hartford":13381,"pose":13382,"backpack":13383,"reasoning":13384,"accepts":13385,"24th":13386,"wipe":13387,"millimetres":13388,"marcel":13389,"##oda":13390,"dodgers":13391,"albion":13392,"1790":13393,"overwhelmed":13394,"aerospace":13395,"oaks":13396,"1795":13397,"showcase":13398,"acknowledge":13399,"recovering":13400,"nolan":13401,"ashe":13402,"hurts":13403,"geology":13404,"fashioned":13405,"disappearance":13406,"farewell":13407,"swollen":13408,"shrug":13409,"marquis":13410,"wimbledon":13411,"124":13412,"rue":13413,"1792":13414,"commemorate":13415,"reduces":13416,"experiencing":13417,"inevitable":13418,"calcutta":13419,"intel":13420,"##court":13421,"murderer":13422,"sticking":13423,"fisheries":13424,"imagery":13425,"bloom":13426,"280":13427,"brake":13428,"##inus":13429,"gustav":13430,"hesitation":13431,"memorable":13432,"po":13433,"viral":13434,"beans":13435,"accidents":13436,"tunisia":13437,"antenna":13438,"spilled":13439,"consort":13440,"treatments":13441,"aye":13442,"perimeter":13443,"##gard":13444,"donation":13445,"hostage":13446,"migrated":13447,"banker":13448,"addiction":13449,"apex":13450,"lil":13451,"trout":13452,"##ously":13453,"conscience":13454,"##nova":13455,"rams":13456,"sands":13457,"genome":13458,"passionate":13459,"troubles":13460,"##lets":13461,"##set":13462,"amid":13463,"##ibility":13464,"##ret":13465,"higgins":13466,"exceed":13467,"vikings":13468,"##vie":13469,"payne":13470,"##zan":13471,"muscular":13472,"##ste":13473,"defendant":13474,"sucking":13475,"##wal":13476,"ibrahim":13477,"fuselage":13478,"claudia":13479,"vfl":13480,"europeans":13481,"snails":13482,"interval":13483,"##garh":13484,"preparatory":13485,"statewide":13486,"tasked":13487,"lacrosse":13488,"viktor":13489,"##lation":13490,"angola":13491,"##hra":13492,"flint":13493,"implications":13494,"employs":13495,"teens":13496,"patrons":13497,"stall":13498,"weekends":13499,"barriers":13500,"scrambled":13501,"nucleus":13502,"tehran":13503,"jenna":13504,"parsons":13505,"lifelong":13506,"robots":13507,"displacement":13508,"5000":13509,"##bles":13510,"precipitation":13511,"##gt":13512,"knuckles":13513,"clutched":13514,"1802":13515,"marrying":13516,"ecology":13517,"marx":13518,"accusations":13519,"declare":13520,"scars":13521,"kolkata":13522,"mat":13523,"meadows":13524,"bermuda":13525,"skeleton":13526,"finalists":13527,"vintage":13528,"crawl":13529,"coordinate":13530,"affects":13531,"subjected":13532,"orchestral":13533,"mistaken":13534,"##tc":13535,"mirrors":13536,"dipped":13537,"relied":13538,"260":13539,"arches":13540,"candle":13541,"##nick":13542,"incorporating":13543,"wildly":13544,"fond":13545,"basilica":13546,"owl":13547,"fringe":13548,"rituals":13549,"whispering":13550,"stirred":13551,"feud":13552,"tertiary":13553,"slick":13554,"goat":13555,"honorable":13556,"whereby":13557,"skip":13558,"ricardo":13559,"stripes":13560,"parachute":13561,"adjoining":13562,"submerged":13563,"synthesizer":13564,"##gren":13565,"intend":13566,"positively":13567,"ninety":13568,"phi":13569,"beaver":13570,"partition":13571,"fellows":13572,"alexis":13573,"prohibition":13574,"carlisle":13575,"bizarre":13576,"fraternity":13577,"##bre":13578,"doubts":13579,"icy":13580,"cbc":13581,"aquatic":13582,"sneak":13583,"sonny":13584,"combines":13585,"airports":13586,"crude":13587,"supervised":13588,"spatial":13589,"merge":13590,"alfonso":13591,"##bic":13592,"corrupt":13593,"scan":13594,"undergo":13595,"##ams":13596,"disabilities":13597,"colombian":13598,"comparing":13599,"dolphins":13600,"perkins":13601,"##lish":13602,"reprinted":13603,"unanimous":13604,"bounced":13605,"hairs":13606,"underworld":13607,"midwest":13608,"semester":13609,"bucket":13610,"paperback":13611,"miniseries":13612,"coventry":13613,"demise":13614,"##leigh":13615,"demonstrations":13616,"sensor":13617,"rotating":13618,"yan":13619,"##hler":13620,"arrange":13621,"soils":13622,"##idge":13623,"hyderabad":13624,"labs":13625,"##dr":13626,"brakes":13627,"grandchildren":13628,"##nde":13629,"negotiated":13630,"rover":13631,"ferrari":13632,"continuation":13633,"directorate":13634,"augusta":13635,"stevenson":13636,"counterpart":13637,"gore":13638,"##rda":13639,"nursery":13640,"rican":13641,"ave":13642,"collectively":13643,"broadly":13644,"pastoral":13645,"repertoire":13646,"asserted":13647,"discovering":13648,"nordic":13649,"styled":13650,"fiba":13651,"cunningham":13652,"harley":13653,"middlesex":13654,"survives":13655,"tumor":13656,"tempo":13657,"zack":13658,"aiming":13659,"lok":13660,"urgent":13661,"##rade":13662,"##nto":13663,"devils":13664,"##ement":13665,"contractor":13666,"turin":13667,"##wl":13668,"##ool":13669,"bliss":13670,"repaired":13671,"simmons":13672,"moan":13673,"astronomical":13674,"cr":13675,"negotiate":13676,"lyric":13677,"1890s":13678,"lara":13679,"bred":13680,"clad":13681,"angus":13682,"pbs":13683,"##ience":13684,"engineered":13685,"posed":13686,"##lk":13687,"hernandez":13688,"possessions":13689,"elbows":13690,"psychiatric":13691,"strokes":13692,"confluence":13693,"electorate":13694,"lifts":13695,"campuses":13696,"lava":13697,"alps":13698,"##ep":13699,"##ution":13700,"##date":13701,"physicist":13702,"woody":13703,"##page":13704,"##ographic":13705,"##itis":13706,"juliet":13707,"reformation":13708,"sparhawk":13709,"320":13710,"complement":13711,"suppressed":13712,"jewel":13713,"##½":13714,"floated":13715,"##kas":13716,"continuity":13717,"sadly":13718,"##ische":13719,"inability":13720,"melting":13721,"scanning":13722,"paula":13723,"flour":13724,"judaism":13725,"safer":13726,"vague":13727,"##lm":13728,"solving":13729,"curb":13730,"##stown":13731,"financially":13732,"gable":13733,"bees":13734,"expired":13735,"miserable":13736,"cassidy":13737,"dominion":13738,"1789":13739,"cupped":13740,"145":13741,"robbery":13742,"facto":13743,"amos":13744,"warden":13745,"resume":13746,"tallest":13747,"marvin":13748,"ing":13749,"pounded":13750,"usd":13751,"declaring":13752,"gasoline":13753,"##aux":13754,"darkened":13755,"270":13756,"650":13757,"sophomore":13758,"##mere":13759,"erection":13760,"gossip":13761,"televised":13762,"risen":13763,"dial":13764,"##eu":13765,"pillars":13766,"##link":13767,"passages":13768,"profound":13769,"##tina":13770,"arabian":13771,"ashton":13772,"silicon":13773,"nail":13774,"##ead":13775,"##lated":13776,"##wer":13777,"##hardt":13778,"fleming":13779,"firearms":13780,"ducked":13781,"circuits":13782,"blows":13783,"waterloo":13784,"titans":13785,"##lina":13786,"atom":13787,"fireplace":13788,"cheshire":13789,"financed":13790,"activation":13791,"algorithms":13792,"##zzi":13793,"constituent":13794,"catcher":13795,"cherokee":13796,"partnerships":13797,"sexuality":13798,"platoon":13799,"tragic":13800,"vivian":13801,"guarded":13802,"whiskey":13803,"meditation":13804,"poetic":13805,"##late":13806,"##nga":13807,"##ake":13808,"porto":13809,"listeners":13810,"dominance":13811,"kendra":13812,"mona":13813,"chandler":13814,"factions":13815,"22nd":13816,"salisbury":13817,"attitudes":13818,"derivative":13819,"##ido":13820,"##haus":13821,"intake":13822,"paced":13823,"javier":13824,"illustrator":13825,"barrels":13826,"bias":13827,"cockpit":13828,"burnett":13829,"dreamed":13830,"ensuing":13831,"##anda":13832,"receptors":13833,"someday":13834,"hawkins":13835,"mattered":13836,"##lal":13837,"slavic":13838,"1799":13839,"jesuit":13840,"cameroon":13841,"wasted":13842,"tai":13843,"wax":13844,"lowering":13845,"victorious":13846,"freaking":13847,"outright":13848,"hancock":13849,"librarian":13850,"sensing":13851,"bald":13852,"calcium":13853,"myers":13854,"tablet":13855,"announcing":13856,"barack":13857,"shipyard":13858,"pharmaceutical":13859,"##uan":13860,"greenwich":13861,"flush":13862,"medley":13863,"patches":13864,"wolfgang":13865,"pt":13866,"speeches":13867,"acquiring":13868,"exams":13869,"nikolai":13870,"##gg":13871,"hayden":13872,"kannada":13873,"##type":13874,"reilly":13875,"##pt":13876,"waitress":13877,"abdomen":13878,"devastated":13879,"capped":13880,"pseudonym":13881,"pharmacy":13882,"fulfill":13883,"paraguay":13884,"1796":13885,"clicked":13886,"##trom":13887,"archipelago":13888,"syndicated":13889,"##hman":13890,"lumber":13891,"orgasm":13892,"rejection":13893,"clifford":13894,"lorraine":13895,"advent":13896,"mafia":13897,"rodney":13898,"brock":13899,"##ght":13900,"##used":13901,"##elia":13902,"cassette":13903,"chamberlain":13904,"despair":13905,"mongolia":13906,"sensors":13907,"developmental":13908,"upstream":13909,"##eg":13910,"##alis":13911,"spanning":13912,"165":13913,"trombone":13914,"basque":13915,"seeded":13916,"interred":13917,"renewable":13918,"rhys":13919,"leapt":13920,"revision":13921,"molecule":13922,"##ages":13923,"chord":13924,"vicious":13925,"nord":13926,"shivered":13927,"23rd":13928,"arlington":13929,"debts":13930,"corpus":13931,"sunrise":13932,"bays":13933,"blackburn":13934,"centimetres":13935,"##uded":13936,"shuddered":13937,"gm":13938,"strangely":13939,"gripping":13940,"cartoons":13941,"isabelle":13942,"orbital":13943,"##ppa":13944,"seals":13945,"proving":13946,"##lton":13947,"refusal":13948,"strengthened":13949,"bust":13950,"assisting":13951,"baghdad":13952,"batsman":13953,"portrayal":13954,"mara":13955,"pushes":13956,"spears":13957,"og":13958,"##cock":13959,"reside":13960,"nathaniel":13961,"brennan":13962,"1776":13963,"confirmation":13964,"caucus":13965,"##worthy":13966,"markings":13967,"yemen":13968,"nobles":13969,"ku":13970,"lazy":13971,"viewer":13972,"catalan":13973,"encompasses":13974,"sawyer":13975,"##fall":13976,"sparked":13977,"substances":13978,"patents":13979,"braves":13980,"arranger":13981,"evacuation":13982,"sergio":13983,"persuade":13984,"dover":13985,"tolerance":13986,"penguin":13987,"cum":13988,"jockey":13989,"insufficient":13990,"townships":13991,"occupying":13992,"declining":13993,"plural":13994,"processed":13995,"projection":13996,"puppet":13997,"flanders":13998,"introduces":13999,"liability":14000,"##yon":14001,"gymnastics":14002,"antwerp":14003,"taipei":14004,"hobart":14005,"candles":14006,"jeep":14007,"wes":14008,"observers":14009,"126":14010,"chaplain":14011,"bundle":14012,"glorious":14013,"##hine":14014,"hazel":14015,"flung":14016,"sol":14017,"excavations":14018,"dumped":14019,"stares":14020,"sh":14021,"bangalore":14022,"triangular":14023,"icelandic":14024,"intervals":14025,"expressing":14026,"turbine":14027,"##vers":14028,"songwriting":14029,"crafts":14030,"##igo":14031,"jasmine":14032,"ditch":14033,"rite":14034,"##ways":14035,"entertaining":14036,"comply":14037,"sorrow":14038,"wrestlers":14039,"basel":14040,"emirates":14041,"marian":14042,"rivera":14043,"helpful":14044,"##some":14045,"caution":14046,"downward":14047,"networking":14048,"##atory":14049,"##tered":14050,"darted":14051,"genocide":14052,"emergence":14053,"replies":14054,"specializing":14055,"spokesman":14056,"convenient":14057,"unlocked":14058,"fading":14059,"augustine":14060,"concentrations":14061,"resemblance":14062,"elijah":14063,"investigator":14064,"andhra":14065,"##uda":14066,"promotes":14067,"bean":14068,"##rrell":14069,"fleeing":14070,"wan":14071,"simone":14072,"announcer":14073,"##ame":14074,"##bby":14075,"lydia":14076,"weaver":14077,"132":14078,"residency":14079,"modification":14080,"##fest":14081,"stretches":14082,"##ast":14083,"alternatively":14084,"nat":14085,"lowe":14086,"lacks":14087,"##ented":14088,"pam":14089,"tile":14090,"concealed":14091,"inferior":14092,"abdullah":14093,"residences":14094,"tissues":14095,"vengeance":14096,"##ided":14097,"moisture":14098,"peculiar":14099,"groove":14100,"zip":14101,"bologna":14102,"jennings":14103,"ninja":14104,"oversaw":14105,"zombies":14106,"pumping":14107,"batch":14108,"livingston":14109,"emerald":14110,"installations":14111,"1797":14112,"peel":14113,"nitrogen":14114,"rama":14115,"##fying":14116,"##star":14117,"schooling":14118,"strands":14119,"responding":14120,"werner":14121,"##ost":14122,"lime":14123,"casa":14124,"accurately":14125,"targeting":14126,"##rod":14127,"underway":14128,"##uru":14129,"hemisphere":14130,"lester":14131,"##yard":14132,"occupies":14133,"2d":14134,"griffith":14135,"angrily":14136,"reorganized":14137,"##owing":14138,"courtney":14139,"deposited":14140,"##dd":14141,"##30":14142,"estadio":14143,"##ifies":14144,"dunn":14145,"exiled":14146,"##ying":14147,"checks":14148,"##combe":14149,"##о":14150,"##fly":14151,"successes":14152,"unexpectedly":14153,"blu":14154,"assessed":14155,"##flower":14156,"##ه":14157,"observing":14158,"sacked":14159,"spiders":14160,"kn":14161,"##tail":14162,"mu":14163,"nodes":14164,"prosperity":14165,"audrey":14166,"divisional":14167,"155":14168,"broncos":14169,"tangled":14170,"adjust":14171,"feeds":14172,"erosion":14173,"paolo":14174,"surf":14175,"directory":14176,"snatched":14177,"humid":14178,"admiralty":14179,"screwed":14180,"gt":14181,"reddish":14182,"##nese":14183,"modules":14184,"trench":14185,"lamps":14186,"bind":14187,"leah":14188,"bucks":14189,"competes":14190,"##nz":14191,"##form":14192,"transcription":14193,"##uc":14194,"isles":14195,"violently":14196,"clutching":14197,"pga":14198,"cyclist":14199,"inflation":14200,"flats":14201,"ragged":14202,"unnecessary":14203,"##hian":14204,"stubborn":14205,"coordinated":14206,"harriet":14207,"baba":14208,"disqualified":14209,"330":14210,"insect":14211,"wolfe":14212,"##fies":14213,"reinforcements":14214,"rocked":14215,"duel":14216,"winked":14217,"embraced":14218,"bricks":14219,"##raj":14220,"hiatus":14221,"defeats":14222,"pending":14223,"brightly":14224,"jealousy":14225,"##xton":14226,"##hm":14227,"##uki":14228,"lena":14229,"gdp":14230,"colorful":14231,"##dley":14232,"stein":14233,"kidney":14234,"##shu":14235,"underwear":14236,"wanderers":14237,"##haw":14238,"##icus":14239,"guardians":14240,"m³":14241,"roared":14242,"habits":14243,"##wise":14244,"permits":14245,"gp":14246,"uranium":14247,"punished":14248,"disguise":14249,"bundesliga":14250,"elise":14251,"dundee":14252,"erotic":14253,"partisan":14254,"pi":14255,"collectors":14256,"float":14257,"individually":14258,"rendering":14259,"behavioral":14260,"bucharest":14261,"ser":14262,"hare":14263,"valerie":14264,"corporal":14265,"nutrition":14266,"proportional":14267,"##isa":14268,"immense":14269,"##kis":14270,"pavement":14271,"##zie":14272,"##eld":14273,"sutherland":14274,"crouched":14275,"1775":14276,"##lp":14277,"suzuki":14278,"trades":14279,"endurance":14280,"operas":14281,"crosby":14282,"prayed":14283,"priory":14284,"rory":14285,"socially":14286,"##urn":14287,"gujarat":14288,"##pu":14289,"walton":14290,"cube":14291,"pasha":14292,"privilege":14293,"lennon":14294,"floods":14295,"thorne":14296,"waterfall":14297,"nipple":14298,"scouting":14299,"approve":14300,"##lov":14301,"minorities":14302,"voter":14303,"dwight":14304,"extensions":14305,"assure":14306,"ballroom":14307,"slap":14308,"dripping":14309,"privileges":14310,"rejoined":14311,"confessed":14312,"demonstrating":14313,"patriotic":14314,"yell":14315,"investor":14316,"##uth":14317,"pagan":14318,"slumped":14319,"squares":14320,"##cle":14321,"##kins":14322,"confront":14323,"bert":14324,"embarrassment":14325,"##aid":14326,"aston":14327,"urging":14328,"sweater":14329,"starr":14330,"yuri":14331,"brains":14332,"williamson":14333,"commuter":14334,"mortar":14335,"structured":14336,"selfish":14337,"exports":14338,"##jon":14339,"cds":14340,"##him":14341,"unfinished":14342,"##rre":14343,"mortgage":14344,"destinations":14345,"##nagar":14346,"canoe":14347,"solitary":14348,"buchanan":14349,"delays":14350,"magistrate":14351,"fk":14352,"##pling":14353,"motivation":14354,"##lier":14355,"##vier":14356,"recruiting":14357,"assess":14358,"##mouth":14359,"malik":14360,"antique":14361,"1791":14362,"pius":14363,"rahman":14364,"reich":14365,"tub":14366,"zhou":14367,"smashed":14368,"airs":14369,"galway":14370,"xii":14371,"conditioning":14372,"honduras":14373,"discharged":14374,"dexter":14375,"##pf":14376,"lionel":14377,"129":14378,"debates":14379,"lemon":14380,"tiffany":14381,"volunteered":14382,"dom":14383,"dioxide":14384,"procession":14385,"devi":14386,"sic":14387,"tremendous":14388,"advertisements":14389,"colts":14390,"transferring":14391,"verdict":14392,"hanover":14393,"decommissioned":14394,"utter":14395,"relate":14396,"pac":14397,"racism":14398,"##top":14399,"beacon":14400,"limp":14401,"similarity":14402,"terra":14403,"occurrence":14404,"ant":14405,"##how":14406,"becky":14407,"capt":14408,"updates":14409,"armament":14410,"richie":14411,"pal":14412,"##graph":14413,"halloween":14414,"mayo":14415,"##ssen":14416,"##bone":14417,"cara":14418,"serena":14419,"fcc":14420,"dolls":14421,"obligations":14422,"##dling":14423,"violated":14424,"lafayette":14425,"jakarta":14426,"exploitation":14427,"##ime":14428,"infamous":14429,"iconic":14430,"##lah":14431,"##park":14432,"kitty":14433,"moody":14434,"reginald":14435,"dread":14436,"spill":14437,"crystals":14438,"olivier":14439,"modeled":14440,"bluff":14441,"equilibrium":14442,"separating":14443,"notices":14444,"ordnance":14445,"extinction":14446,"onset":14447,"cosmic":14448,"attachment":14449,"sammy":14450,"expose":14451,"privy":14452,"anchored":14453,"##bil":14454,"abbott":14455,"admits":14456,"bending":14457,"baritone":14458,"emmanuel":14459,"policeman":14460,"vaughan":14461,"winged":14462,"climax":14463,"dresses":14464,"denny":14465,"polytechnic":14466,"mohamed":14467,"burmese":14468,"authentic":14469,"nikki":14470,"genetics":14471,"grandparents":14472,"homestead":14473,"gaza":14474,"postponed":14475,"metacritic":14476,"una":14477,"##sby":14478,"##bat":14479,"unstable":14480,"dissertation":14481,"##rial":14482,"##cian":14483,"curls":14484,"obscure":14485,"uncovered":14486,"bronx":14487,"praying":14488,"disappearing":14489,"##hoe":14490,"prehistoric":14491,"coke":14492,"turret":14493,"mutations":14494,"nonprofit":14495,"pits":14496,"monaco":14497,"##ي":14498,"##usion":14499,"prominently":14500,"dispatched":14501,"podium":14502,"##mir":14503,"uci":14504,"##uation":14505,"133":14506,"fortifications":14507,"birthplace":14508,"kendall":14509,"##lby":14510,"##oll":14511,"preacher":14512,"rack":14513,"goodman":14514,"##rman":14515,"persistent":14516,"##ott":14517,"countless":14518,"jaime":14519,"recorder":14520,"lexington":14521,"persecution":14522,"jumps":14523,"renewal":14524,"wagons":14525,"##11":14526,"crushing":14527,"##holder":14528,"decorations":14529,"##lake":14530,"abundance":14531,"wrath":14532,"laundry":14533,"£1":14534,"garde":14535,"##rp":14536,"jeanne":14537,"beetles":14538,"peasant":14539,"##sl":14540,"splitting":14541,"caste":14542,"sergei":14543,"##rer":14544,"##ema":14545,"scripts":14546,"##ively":14547,"rub":14548,"satellites":14549,"##vor":14550,"inscribed":14551,"verlag":14552,"scrapped":14553,"gale":14554,"packages":14555,"chick":14556,"potato":14557,"slogan":14558,"kathleen":14559,"arabs":14560,"##culture":14561,"counterparts":14562,"reminiscent":14563,"choral":14564,"##tead":14565,"rand":14566,"retains":14567,"bushes":14568,"dane":14569,"accomplish":14570,"courtesy":14571,"closes":14572,"##oth":14573,"slaughter":14574,"hague":14575,"krakow":14576,"lawson":14577,"tailed":14578,"elias":14579,"ginger":14580,"##ttes":14581,"canopy":14582,"betrayal":14583,"rebuilding":14584,"turf":14585,"##hof":14586,"frowning":14587,"allegiance":14588,"brigades":14589,"kicks":14590,"rebuild":14591,"polls":14592,"alias":14593,"nationalism":14594,"td":14595,"rowan":14596,"audition":14597,"bowie":14598,"fortunately":14599,"recognizes":14600,"harp":14601,"dillon":14602,"horrified":14603,"##oro":14604,"renault":14605,"##tics":14606,"ropes":14607,"##α":14608,"presumed":14609,"rewarded":14610,"infrared":14611,"wiping":14612,"accelerated":14613,"illustration":14614,"##rid":14615,"presses":14616,"practitioners":14617,"badminton":14618,"##iard":14619,"detained":14620,"##tera":14621,"recognizing":14622,"relates":14623,"misery":14624,"##sies":14625,"##tly":14626,"reproduction":14627,"piercing":14628,"potatoes":14629,"thornton":14630,"esther":14631,"manners":14632,"hbo":14633,"##aan":14634,"ours":14635,"bullshit":14636,"ernie":14637,"perennial":14638,"sensitivity":14639,"illuminated":14640,"rupert":14641,"##jin":14642,"##iss":14643,"##ear":14644,"rfc":14645,"nassau":14646,"##dock":14647,"staggered":14648,"socialism":14649,"##haven":14650,"appointments":14651,"nonsense":14652,"prestige":14653,"sharma":14654,"haul":14655,"##tical":14656,"solidarity":14657,"gps":14658,"##ook":14659,"##rata":14660,"igor":14661,"pedestrian":14662,"##uit":14663,"baxter":14664,"tenants":14665,"wires":14666,"medication":14667,"unlimited":14668,"guiding":14669,"impacts":14670,"diabetes":14671,"##rama":14672,"sasha":14673,"pas":14674,"clive":14675,"extraction":14676,"131":14677,"continually":14678,"constraints":14679,"##bilities":14680,"sonata":14681,"hunted":14682,"sixteenth":14683,"chu":14684,"planting":14685,"quote":14686,"mayer":14687,"pretended":14688,"abs":14689,"spat":14690,"##hua":14691,"ceramic":14692,"##cci":14693,"curtains":14694,"pigs":14695,"pitching":14696,"##dad":14697,"latvian":14698,"sore":14699,"dayton":14700,"##sted":14701,"##qi":14702,"patrols":14703,"slice":14704,"playground":14705,"##nted":14706,"shone":14707,"stool":14708,"apparatus":14709,"inadequate":14710,"mates":14711,"treason":14712,"##ija":14713,"desires":14714,"##liga":14715,"##croft":14716,"somalia":14717,"laurent":14718,"mir":14719,"leonardo":14720,"oracle":14721,"grape":14722,"obliged":14723,"chevrolet":14724,"thirteenth":14725,"stunning":14726,"enthusiastic":14727,"##ede":14728,"accounted":14729,"concludes":14730,"currents":14731,"basil":14732,"##kovic":14733,"drought":14734,"##rica":14735,"mai":14736,"##aire":14737,"shove":14738,"posting":14739,"##shed":14740,"pilgrimage":14741,"humorous":14742,"packing":14743,"fry":14744,"pencil":14745,"wines":14746,"smells":14747,"144":14748,"marilyn":14749,"aching":14750,"newest":14751,"clung":14752,"bon":14753,"neighbours":14754,"sanctioned":14755,"##pie":14756,"mug":14757,"##stock":14758,"drowning":14759,"##mma":14760,"hydraulic":14761,"##vil":14762,"hiring":14763,"reminder":14764,"lilly":14765,"investigators":14766,"##ncies":14767,"sour":14768,"##eous":14769,"compulsory":14770,"packet":14771,"##rion":14772,"##graphic":14773,"##elle":14774,"cannes":14775,"##inate":14776,"depressed":14777,"##rit":14778,"heroic":14779,"importantly":14780,"theresa":14781,"##tled":14782,"conway":14783,"saturn":14784,"marginal":14785,"rae":14786,"##xia":14787,"corresponds":14788,"royce":14789,"pact":14790,"jasper":14791,"explosives":14792,"packaging":14793,"aluminium":14794,"##ttered":14795,"denotes":14796,"rhythmic":14797,"spans":14798,"assignments":14799,"hereditary":14800,"outlined":14801,"originating":14802,"sundays":14803,"lad":14804,"reissued":14805,"greeting":14806,"beatrice":14807,"##dic":14808,"pillar":14809,"marcos":14810,"plots":14811,"handbook":14812,"alcoholic":14813,"judiciary":14814,"avant":14815,"slides":14816,"extract":14817,"masculine":14818,"blur":14819,"##eum":14820,"##force":14821,"homage":14822,"trembled":14823,"owens":14824,"hymn":14825,"trey":14826,"omega":14827,"signaling":14828,"socks":14829,"accumulated":14830,"reacted":14831,"attic":14832,"theo":14833,"lining":14834,"angie":14835,"distraction":14836,"primera":14837,"talbot":14838,"##key":14839,"1200":14840,"ti":14841,"creativity":14842,"billed":14843,"##hey":14844,"deacon":14845,"eduardo":14846,"identifies":14847,"proposition":14848,"dizzy":14849,"gunner":14850,"hogan":14851,"##yam":14852,"##pping":14853,"##hol":14854,"ja":14855,"##chan":14856,"jensen":14857,"reconstructed":14858,"##berger":14859,"clearance":14860,"darius":14861,"##nier":14862,"abe":14863,"harlem":14864,"plea":14865,"dei":14866,"circled":14867,"emotionally":14868,"notation":14869,"fascist":14870,"neville":14871,"exceeded":14872,"upwards":14873,"viable":14874,"ducks":14875,"##fo":14876,"workforce":14877,"racer":14878,"limiting":14879,"shri":14880,"##lson":14881,"possesses":14882,"1600":14883,"kerr":14884,"moths":14885,"devastating":14886,"laden":14887,"disturbing":14888,"locking":14889,"##cture":14890,"gal":14891,"fearing":14892,"accreditation":14893,"flavor":14894,"aide":14895,"1870s":14896,"mountainous":14897,"##baum":14898,"melt":14899,"##ures":14900,"motel":14901,"texture":14902,"servers":14903,"soda":14904,"##mb":14905,"herd":14906,"##nium":14907,"erect":14908,"puzzled":14909,"hum":14910,"peggy":14911,"examinations":14912,"gould":14913,"testified":14914,"geoff":14915,"ren":14916,"devised":14917,"sacks":14918,"##law":14919,"denial":14920,"posters":14921,"grunted":14922,"cesar":14923,"tutor":14924,"ec":14925,"gerry":14926,"offerings":14927,"byrne":14928,"falcons":14929,"combinations":14930,"ct":14931,"incoming":14932,"pardon":14933,"rocking":14934,"26th":14935,"avengers":14936,"flared":14937,"mankind":14938,"seller":14939,"uttar":14940,"loch":14941,"nadia":14942,"stroking":14943,"exposing":14944,"##hd":14945,"fertile":14946,"ancestral":14947,"instituted":14948,"##has":14949,"noises":14950,"prophecy":14951,"taxation":14952,"eminent":14953,"vivid":14954,"pol":14955,"##bol":14956,"dart":14957,"indirect":14958,"multimedia":14959,"notebook":14960,"upside":14961,"displaying":14962,"adrenaline":14963,"referenced":14964,"geometric":14965,"##iving":14966,"progression":14967,"##ddy":14968,"blunt":14969,"announce":14970,"##far":14971,"implementing":14972,"##lav":14973,"aggression":14974,"liaison":14975,"cooler":14976,"cares":14977,"headache":14978,"plantations":14979,"gorge":14980,"dots":14981,"impulse":14982,"thickness":14983,"ashamed":14984,"averaging":14985,"kathy":14986,"obligation":14987,"precursor":14988,"137":14989,"fowler":14990,"symmetry":14991,"thee":14992,"225":14993,"hears":14994,"##rai":14995,"undergoing":14996,"ads":14997,"butcher":14998,"bowler":14999,"##lip":15000,"cigarettes":15001,"subscription":15002,"goodness":15003,"##ically":15004,"browne":15005,"##hos":15006,"##tech":15007,"kyoto":15008,"donor":15009,"##erty":15010,"damaging":15011,"friction":15012,"drifting":15013,"expeditions":15014,"hardened":15015,"prostitution":15016,"152":15017,"fauna":15018,"blankets":15019,"claw":15020,"tossing":15021,"snarled":15022,"butterflies":15023,"recruits":15024,"investigative":15025,"coated":15026,"healed":15027,"138":15028,"communal":15029,"hai":15030,"xiii":15031,"academics":15032,"boone":15033,"psychologist":15034,"restless":15035,"lahore":15036,"stephens":15037,"mba":15038,"brendan":15039,"foreigners":15040,"printer":15041,"##pc":15042,"ached":15043,"explode":15044,"27th":15045,"deed":15046,"scratched":15047,"dared":15048,"##pole":15049,"cardiac":15050,"1780":15051,"okinawa":15052,"proto":15053,"commando":15054,"compelled":15055,"oddly":15056,"electrons":15057,"##base":15058,"replica":15059,"thanksgiving":15060,"##rist":15061,"sheila":15062,"deliberate":15063,"stafford":15064,"tidal":15065,"representations":15066,"hercules":15067,"ou":15068,"##path":15069,"##iated":15070,"kidnapping":15071,"lenses":15072,"##tling":15073,"deficit":15074,"samoa":15075,"mouths":15076,"consuming":15077,"computational":15078,"maze":15079,"granting":15080,"smirk":15081,"razor":15082,"fixture":15083,"ideals":15084,"inviting":15085,"aiden":15086,"nominal":15087,"##vs":15088,"issuing":15089,"julio":15090,"pitt":15091,"ramsey":15092,"docks":15093,"##oss":15094,"exhaust":15095,"##owed":15096,"bavarian":15097,"draped":15098,"anterior":15099,"mating":15100,"ethiopian":15101,"explores":15102,"noticing":15103,"##nton":15104,"discarded":15105,"convenience":15106,"hoffman":15107,"endowment":15108,"beasts":15109,"cartridge":15110,"mormon":15111,"paternal":15112,"probe":15113,"sleeves":15114,"interfere":15115,"lump":15116,"deadline":15117,"##rail":15118,"jenks":15119,"bulldogs":15120,"scrap":15121,"alternating":15122,"justified":15123,"reproductive":15124,"nam":15125,"seize":15126,"descending":15127,"secretariat":15128,"kirby":15129,"coupe":15130,"grouped":15131,"smash":15132,"panther":15133,"sedan":15134,"tapping":15135,"##18":15136,"lola":15137,"cheer":15138,"germanic":15139,"unfortunate":15140,"##eter":15141,"unrelated":15142,"##fan":15143,"subordinate":15144,"##sdale":15145,"suzanne":15146,"advertisement":15147,"##ility":15148,"horsepower":15149,"##lda":15150,"cautiously":15151,"discourse":15152,"luigi":15153,"##mans":15154,"##fields":15155,"noun":15156,"prevalent":15157,"mao":15158,"schneider":15159,"everett":15160,"surround":15161,"governorate":15162,"kira":15163,"##avia":15164,"westward":15165,"##take":15166,"misty":15167,"rails":15168,"sustainability":15169,"134":15170,"unused":15171,"##rating":15172,"packs":15173,"toast":15174,"unwilling":15175,"regulate":15176,"thy":15177,"suffrage":15178,"nile":15179,"awe":15180,"assam":15181,"definitions":15182,"travelers":15183,"affordable":15184,"##rb":15185,"conferred":15186,"sells":15187,"undefeated":15188,"beneficial":15189,"torso":15190,"basal":15191,"repeating":15192,"remixes":15193,"##pass":15194,"bahrain":15195,"cables":15196,"fang":15197,"##itated":15198,"excavated":15199,"numbering":15200,"statutory":15201,"##rey":15202,"deluxe":15203,"##lian":15204,"forested":15205,"ramirez":15206,"derbyshire":15207,"zeus":15208,"slamming":15209,"transfers":15210,"astronomer":15211,"banana":15212,"lottery":15213,"berg":15214,"histories":15215,"bamboo":15216,"##uchi":15217,"resurrection":15218,"posterior":15219,"bowls":15220,"vaguely":15221,"##thi":15222,"thou":15223,"preserving":15224,"tensed":15225,"offence":15226,"##inas":15227,"meyrick":15228,"callum":15229,"ridden":15230,"watt":15231,"langdon":15232,"tying":15233,"lowland":15234,"snorted":15235,"daring":15236,"truman":15237,"##hale":15238,"##girl":15239,"aura":15240,"overly":15241,"filing":15242,"weighing":15243,"goa":15244,"infections":15245,"philanthropist":15246,"saunders":15247,"eponymous":15248,"##owski":15249,"latitude":15250,"perspectives":15251,"reviewing":15252,"mets":15253,"commandant":15254,"radial":15255,"##kha":15256,"flashlight":15257,"reliability":15258,"koch":15259,"vowels":15260,"amazed":15261,"ada":15262,"elaine":15263,"supper":15264,"##rth":15265,"##encies":15266,"predator":15267,"debated":15268,"soviets":15269,"cola":15270,"##boards":15271,"##nah":15272,"compartment":15273,"crooked":15274,"arbitrary":15275,"fourteenth":15276,"##ctive":15277,"havana":15278,"majors":15279,"steelers":15280,"clips":15281,"profitable":15282,"ambush":15283,"exited":15284,"packers":15285,"##tile":15286,"nude":15287,"cracks":15288,"fungi":15289,"##е":15290,"limb":15291,"trousers":15292,"josie":15293,"shelby":15294,"tens":15295,"frederic":15296,"##ος":15297,"definite":15298,"smoothly":15299,"constellation":15300,"insult":15301,"baton":15302,"discs":15303,"lingering":15304,"##nco":15305,"conclusions":15306,"lent":15307,"staging":15308,"becker":15309,"grandpa":15310,"shaky":15311,"##tron":15312,"einstein":15313,"obstacles":15314,"sk":15315,"adverse":15316,"elle":15317,"economically":15318,"##moto":15319,"mccartney":15320,"thor":15321,"dismissal":15322,"motions":15323,"readings":15324,"nostrils":15325,"treatise":15326,"##pace":15327,"squeezing":15328,"evidently":15329,"prolonged":15330,"1783":15331,"venezuelan":15332,"je":15333,"marguerite":15334,"beirut":15335,"takeover":15336,"shareholders":15337,"##vent":15338,"denise":15339,"digit":15340,"airplay":15341,"norse":15342,"##bbling":15343,"imaginary":15344,"pills":15345,"hubert":15346,"blaze":15347,"vacated":15348,"eliminating":15349,"##ello":15350,"vine":15351,"mansfield":15352,"##tty":15353,"retrospective":15354,"barrow":15355,"borne":15356,"clutch":15357,"bail":15358,"forensic":15359,"weaving":15360,"##nett":15361,"##witz":15362,"desktop":15363,"citadel":15364,"promotions":15365,"worrying":15366,"dorset":15367,"ieee":15368,"subdivided":15369,"##iating":15370,"manned":15371,"expeditionary":15372,"pickup":15373,"synod":15374,"chuckle":15375,"185":15376,"barney":15377,"##rz":15378,"##ffin":15379,"functionality":15380,"karachi":15381,"litigation":15382,"meanings":15383,"uc":15384,"lick":15385,"turbo":15386,"anders":15387,"##ffed":15388,"execute":15389,"curl":15390,"oppose":15391,"ankles":15392,"typhoon":15393,"##د":15394,"##ache":15395,"##asia":15396,"linguistics":15397,"compassion":15398,"pressures":15399,"grazing":15400,"perfection":15401,"##iting":15402,"immunity":15403,"monopoly":15404,"muddy":15405,"backgrounds":15406,"136":15407,"namibia":15408,"francesca":15409,"monitors":15410,"attracting":15411,"stunt":15412,"tuition":15413,"##ии":15414,"vegetable":15415,"##mates":15416,"##quent":15417,"mgm":15418,"jen":15419,"complexes":15420,"forts":15421,"##ond":15422,"cellar":15423,"bites":15424,"seventeenth":15425,"royals":15426,"flemish":15427,"failures":15428,"mast":15429,"charities":15430,"##cular":15431,"peruvian":15432,"capitals":15433,"macmillan":15434,"ipswich":15435,"outward":15436,"frigate":15437,"postgraduate":15438,"folds":15439,"employing":15440,"##ouse":15441,"concurrently":15442,"fiery":15443,"##tai":15444,"contingent":15445,"nightmares":15446,"monumental":15447,"nicaragua":15448,"##kowski":15449,"lizard":15450,"mal":15451,"fielding":15452,"gig":15453,"reject":15454,"##pad":15455,"harding":15456,"##ipe":15457,"coastline":15458,"##cin":15459,"##nos":15460,"beethoven":15461,"humphrey":15462,"innovations":15463,"##tam":15464,"##nge":15465,"norris":15466,"doris":15467,"solicitor":15468,"huang":15469,"obey":15470,"141":15471,"##lc":15472,"niagara":15473,"##tton":15474,"shelves":15475,"aug":15476,"bourbon":15477,"curry":15478,"nightclub":15479,"specifications":15480,"hilton":15481,"##ndo":15482,"centennial":15483,"dispersed":15484,"worm":15485,"neglected":15486,"briggs":15487,"sm":15488,"font":15489,"kuala":15490,"uneasy":15491,"plc":15492,"##nstein":15493,"##bound":15494,"##aking":15495,"##burgh":15496,"awaiting":15497,"pronunciation":15498,"##bbed":15499,"##quest":15500,"eh":15501,"optimal":15502,"zhu":15503,"raped":15504,"greens":15505,"presided":15506,"brenda":15507,"worries":15508,"##life":15509,"venetian":15510,"marxist":15511,"turnout":15512,"##lius":15513,"refined":15514,"braced":15515,"sins":15516,"grasped":15517,"sunderland":15518,"nickel":15519,"speculated":15520,"lowell":15521,"cyrillic":15522,"communism":15523,"fundraising":15524,"resembling":15525,"colonists":15526,"mutant":15527,"freddie":15528,"usc":15529,"##mos":15530,"gratitude":15531,"##run":15532,"mural":15533,"##lous":15534,"chemist":15535,"wi":15536,"reminds":15537,"28th":15538,"steals":15539,"tess":15540,"pietro":15541,"##ingen":15542,"promoter":15543,"ri":15544,"microphone":15545,"honoured":15546,"rai":15547,"sant":15548,"##qui":15549,"feather":15550,"##nson":15551,"burlington":15552,"kurdish":15553,"terrorists":15554,"deborah":15555,"sickness":15556,"##wed":15557,"##eet":15558,"hazard":15559,"irritated":15560,"desperation":15561,"veil":15562,"clarity":15563,"##rik":15564,"jewels":15565,"xv":15566,"##gged":15567,"##ows":15568,"##cup":15569,"berkshire":15570,"unfair":15571,"mysteries":15572,"orchid":15573,"winced":15574,"exhaustion":15575,"renovations":15576,"stranded":15577,"obe":15578,"infinity":15579,"##nies":15580,"adapt":15581,"redevelopment":15582,"thanked":15583,"registry":15584,"olga":15585,"domingo":15586,"noir":15587,"tudor":15588,"ole":15589,"##atus":15590,"commenting":15591,"behaviors":15592,"##ais":15593,"crisp":15594,"pauline":15595,"probable":15596,"stirling":15597,"wigan":15598,"##bian":15599,"paralympics":15600,"panting":15601,"surpassed":15602,"##rew":15603,"luca":15604,"barred":15605,"pony":15606,"famed":15607,"##sters":15608,"cassandra":15609,"waiter":15610,"carolyn":15611,"exported":15612,"##orted":15613,"andres":15614,"destructive":15615,"deeds":15616,"jonah":15617,"castles":15618,"vacancy":15619,"suv":15620,"##glass":15621,"1788":15622,"orchard":15623,"yep":15624,"famine":15625,"belarusian":15626,"sprang":15627,"##forth":15628,"skinny":15629,"##mis":15630,"administrators":15631,"rotterdam":15632,"zambia":15633,"zhao":15634,"boiler":15635,"discoveries":15636,"##ride":15637,"##physics":15638,"lucius":15639,"disappointing":15640,"outreach":15641,"spoon":15642,"##frame":15643,"qualifications":15644,"unanimously":15645,"enjoys":15646,"regency":15647,"##iidae":15648,"stade":15649,"realism":15650,"veterinary":15651,"rodgers":15652,"dump":15653,"alain":15654,"chestnut":15655,"castile":15656,"censorship":15657,"rumble":15658,"gibbs":15659,"##itor":15660,"communion":15661,"reggae":15662,"inactivated":15663,"logs":15664,"loads":15665,"##houses":15666,"homosexual":15667,"##iano":15668,"ale":15669,"informs":15670,"##cas":15671,"phrases":15672,"plaster":15673,"linebacker":15674,"ambrose":15675,"kaiser":15676,"fascinated":15677,"850":15678,"limerick":15679,"recruitment":15680,"forge":15681,"mastered":15682,"##nding":15683,"leinster":15684,"rooted":15685,"threaten":15686,"##strom":15687,"borneo":15688,"##hes":15689,"suggestions":15690,"scholarships":15691,"propeller":15692,"documentaries":15693,"patronage":15694,"coats":15695,"constructing":15696,"invest":15697,"neurons":15698,"comet":15699,"entirety":15700,"shouts":15701,"identities":15702,"annoying":15703,"unchanged":15704,"wary":15705,"##antly":15706,"##ogy":15707,"neat":15708,"oversight":15709,"##kos":15710,"phillies":15711,"replay":15712,"constance":15713,"##kka":15714,"incarnation":15715,"humble":15716,"skies":15717,"minus":15718,"##acy":15719,"smithsonian":15720,"##chel":15721,"guerrilla":15722,"jar":15723,"cadets":15724,"##plate":15725,"surplus":15726,"audit":15727,"##aru":15728,"cracking":15729,"joanna":15730,"louisa":15731,"pacing":15732,"##lights":15733,"intentionally":15734,"##iri":15735,"diner":15736,"nwa":15737,"imprint":15738,"australians":15739,"tong":15740,"unprecedented":15741,"bunker":15742,"naive":15743,"specialists":15744,"ark":15745,"nichols":15746,"railing":15747,"leaked":15748,"pedal":15749,"##uka":15750,"shrub":15751,"longing":15752,"roofs":15753,"v8":15754,"captains":15755,"neural":15756,"tuned":15757,"##ntal":15758,"##jet":15759,"emission":15760,"medina":15761,"frantic":15762,"codex":15763,"definitive":15764,"sid":15765,"abolition":15766,"intensified":15767,"stocks":15768,"enrique":15769,"sustain":15770,"genoa":15771,"oxide":15772,"##written":15773,"clues":15774,"cha":15775,"##gers":15776,"tributaries":15777,"fragment":15778,"venom":15779,"##rity":15780,"##ente":15781,"##sca":15782,"muffled":15783,"vain":15784,"sire":15785,"laos":15786,"##ingly":15787,"##hana":15788,"hastily":15789,"snapping":15790,"surfaced":15791,"sentiment":15792,"motive":15793,"##oft":15794,"contests":15795,"approximate":15796,"mesa":15797,"luckily":15798,"dinosaur":15799,"exchanges":15800,"propelled":15801,"accord":15802,"bourne":15803,"relieve":15804,"tow":15805,"masks":15806,"offended":15807,"##ues":15808,"cynthia":15809,"##mmer":15810,"rains":15811,"bartender":15812,"zinc":15813,"reviewers":15814,"lois":15815,"##sai":15816,"legged":15817,"arrogant":15818,"rafe":15819,"rosie":15820,"comprise":15821,"handicap":15822,"blockade":15823,"inlet":15824,"lagoon":15825,"copied":15826,"drilling":15827,"shelley":15828,"petals":15829,"##inian":15830,"mandarin":15831,"obsolete":15832,"##inated":15833,"onward":15834,"arguably":15835,"productivity":15836,"cindy":15837,"praising":15838,"seldom":15839,"busch":15840,"discusses":15841,"raleigh":15842,"shortage":15843,"ranged":15844,"stanton":15845,"encouragement":15846,"firstly":15847,"conceded":15848,"overs":15849,"temporal":15850,"##uke":15851,"cbe":15852,"##bos":15853,"woo":15854,"certainty":15855,"pumps":15856,"##pton":15857,"stalked":15858,"##uli":15859,"lizzie":15860,"periodic":15861,"thieves":15862,"weaker":15863,"##night":15864,"gases":15865,"shoving":15866,"chooses":15867,"wc":15868,"##chemical":15869,"prompting":15870,"weights":15871,"##kill":15872,"robust":15873,"flanked":15874,"sticky":15875,"hu":15876,"tuberculosis":15877,"##eb":15878,"##eal":15879,"christchurch":15880,"resembled":15881,"wallet":15882,"reese":15883,"inappropriate":15884,"pictured":15885,"distract":15886,"fixing":15887,"fiddle":15888,"giggled":15889,"burger":15890,"heirs":15891,"hairy":15892,"mechanic":15893,"torque":15894,"apache":15895,"obsessed":15896,"chiefly":15897,"cheng":15898,"logging":15899,"##tag":15900,"extracted":15901,"meaningful":15902,"numb":15903,"##vsky":15904,"gloucestershire":15905,"reminding":15906,"##bay":15907,"unite":15908,"##lit":15909,"breeds":15910,"diminished":15911,"clown":15912,"glove":15913,"1860s":15914,"##ن":15915,"##ug":15916,"archibald":15917,"focal":15918,"freelance":15919,"sliced":15920,"depiction":15921,"##yk":15922,"organism":15923,"switches":15924,"sights":15925,"stray":15926,"crawling":15927,"##ril":15928,"lever":15929,"leningrad":15930,"interpretations":15931,"loops":15932,"anytime":15933,"reel":15934,"alicia":15935,"delighted":15936,"##ech":15937,"inhaled":15938,"xiv":15939,"suitcase":15940,"bernie":15941,"vega":15942,"licenses":15943,"northampton":15944,"exclusion":15945,"induction":15946,"monasteries":15947,"racecourse":15948,"homosexuality":15949,"##right":15950,"##sfield":15951,"##rky":15952,"dimitri":15953,"michele":15954,"alternatives":15955,"ions":15956,"commentators":15957,"genuinely":15958,"objected":15959,"pork":15960,"hospitality":15961,"fencing":15962,"stephan":15963,"warships":15964,"peripheral":15965,"wit":15966,"drunken":15967,"wrinkled":15968,"quentin":15969,"spends":15970,"departing":15971,"chung":15972,"numerical":15973,"spokesperson":15974,"##zone":15975,"johannesburg":15976,"caliber":15977,"killers":15978,"##udge":15979,"assumes":15980,"neatly":15981,"demographic":15982,"abigail":15983,"bloc":15984,"##vel":15985,"mounting":15986,"##lain":15987,"bentley":15988,"slightest":15989,"xu":15990,"recipients":15991,"##jk":15992,"merlin":15993,"##writer":15994,"seniors":15995,"prisons":15996,"blinking":15997,"hindwings":15998,"flickered":15999,"kappa":16000,"##hel":16001,"80s":16002,"strengthening":16003,"appealing":16004,"brewing":16005,"gypsy":16006,"mali":16007,"lashes":16008,"hulk":16009,"unpleasant":16010,"harassment":16011,"bio":16012,"treaties":16013,"predict":16014,"instrumentation":16015,"pulp":16016,"troupe":16017,"boiling":16018,"mantle":16019,"##ffe":16020,"ins":16021,"##vn":16022,"dividing":16023,"handles":16024,"verbs":16025,"##onal":16026,"coconut":16027,"senegal":16028,"340":16029,"thorough":16030,"gum":16031,"momentarily":16032,"##sto":16033,"cocaine":16034,"panicked":16035,"destined":16036,"##turing":16037,"teatro":16038,"denying":16039,"weary":16040,"captained":16041,"mans":16042,"##hawks":16043,"##code":16044,"wakefield":16045,"bollywood":16046,"thankfully":16047,"##16":16048,"cyril":16049,"##wu":16050,"amendments":16051,"##bahn":16052,"consultation":16053,"stud":16054,"reflections":16055,"kindness":16056,"1787":16057,"internally":16058,"##ovo":16059,"tex":16060,"mosaic":16061,"distribute":16062,"paddy":16063,"seeming":16064,"143":16065,"##hic":16066,"piers":16067,"##15":16068,"##mura":16069,"##verse":16070,"popularly":16071,"winger":16072,"kang":16073,"sentinel":16074,"mccoy":16075,"##anza":16076,"covenant":16077,"##bag":16078,"verge":16079,"fireworks":16080,"suppress":16081,"thrilled":16082,"dominate":16083,"##jar":16084,"swansea":16085,"##60":16086,"142":16087,"reconciliation":16088,"##ndi":16089,"stiffened":16090,"cue":16091,"dorian":16092,"##uf":16093,"damascus":16094,"amor":16095,"ida":16096,"foremost":16097,"##aga":16098,"porsche":16099,"unseen":16100,"dir":16101,"##had":16102,"##azi":16103,"stony":16104,"lexi":16105,"melodies":16106,"##nko":16107,"angular":16108,"integer":16109,"podcast":16110,"ants":16111,"inherent":16112,"jaws":16113,"justify":16114,"persona":16115,"##olved":16116,"josephine":16117,"##nr":16118,"##ressed":16119,"customary":16120,"flashes":16121,"gala":16122,"cyrus":16123,"glaring":16124,"backyard":16125,"ariel":16126,"physiology":16127,"greenland":16128,"html":16129,"stir":16130,"avon":16131,"atletico":16132,"finch":16133,"methodology":16134,"ked":16135,"##lent":16136,"mas":16137,"catholicism":16138,"townsend":16139,"branding":16140,"quincy":16141,"fits":16142,"containers":16143,"1777":16144,"ashore":16145,"aragon":16146,"##19":16147,"forearm":16148,"poisoning":16149,"##sd":16150,"adopting":16151,"conquer":16152,"grinding":16153,"amnesty":16154,"keller":16155,"finances":16156,"evaluate":16157,"forged":16158,"lankan":16159,"instincts":16160,"##uto":16161,"guam":16162,"bosnian":16163,"photographed":16164,"workplace":16165,"desirable":16166,"protector":16167,"##dog":16168,"allocation":16169,"intently":16170,"encourages":16171,"willy":16172,"##sten":16173,"bodyguard":16174,"electro":16175,"brighter":16176,"##ν":16177,"bihar":16178,"##chev":16179,"lasts":16180,"opener":16181,"amphibious":16182,"sal":16183,"verde":16184,"arte":16185,"##cope":16186,"captivity":16187,"vocabulary":16188,"yields":16189,"##tted":16190,"agreeing":16191,"desmond":16192,"pioneered":16193,"##chus":16194,"strap":16195,"campaigned":16196,"railroads":16197,"##ович":16198,"emblem":16199,"##dre":16200,"stormed":16201,"501":16202,"##ulous":16203,"marijuana":16204,"northumberland":16205,"##gn":16206,"##nath":16207,"bowen":16208,"landmarks":16209,"beaumont":16210,"##qua":16211,"danube":16212,"##bler":16213,"attorneys":16214,"th":16215,"ge":16216,"flyers":16217,"critique":16218,"villains":16219,"cass":16220,"mutation":16221,"acc":16222,"##0s":16223,"colombo":16224,"mckay":16225,"motif":16226,"sampling":16227,"concluding":16228,"syndicate":16229,"##rell":16230,"neon":16231,"stables":16232,"ds":16233,"warnings":16234,"clint":16235,"mourning":16236,"wilkinson":16237,"##tated":16238,"merrill":16239,"leopard":16240,"evenings":16241,"exhaled":16242,"emil":16243,"sonia":16244,"ezra":16245,"discrete":16246,"stove":16247,"farrell":16248,"fifteenth":16249,"prescribed":16250,"superhero":16251,"##rier":16252,"worms":16253,"helm":16254,"wren":16255,"##duction":16256,"##hc":16257,"expo":16258,"##rator":16259,"hq":16260,"unfamiliar":16261,"antony":16262,"prevents":16263,"acceleration":16264,"fiercely":16265,"mari":16266,"painfully":16267,"calculations":16268,"cheaper":16269,"ign":16270,"clifton":16271,"irvine":16272,"davenport":16273,"mozambique":16274,"##np":16275,"pierced":16276,"##evich":16277,"wonders":16278,"##wig":16279,"##cate":16280,"##iling":16281,"crusade":16282,"ware":16283,"##uel":16284,"enzymes":16285,"reasonably":16286,"mls":16287,"##coe":16288,"mater":16289,"ambition":16290,"bunny":16291,"eliot":16292,"kernel":16293,"##fin":16294,"asphalt":16295,"headmaster":16296,"torah":16297,"aden":16298,"lush":16299,"pins":16300,"waived":16301,"##care":16302,"##yas":16303,"joao":16304,"substrate":16305,"enforce":16306,"##grad":16307,"##ules":16308,"alvarez":16309,"selections":16310,"epidemic":16311,"tempted":16312,"##bit":16313,"bremen":16314,"translates":16315,"ensured":16316,"waterfront":16317,"29th":16318,"forrest":16319,"manny":16320,"malone":16321,"kramer":16322,"reigning":16323,"cookies":16324,"simpler":16325,"absorption":16326,"205":16327,"engraved":16328,"##ffy":16329,"evaluated":16330,"1778":16331,"haze":16332,"146":16333,"comforting":16334,"crossover":16335,"##abe":16336,"thorn":16337,"##rift":16338,"##imo":16339,"##pop":16340,"suppression":16341,"fatigue":16342,"cutter":16343,"##tr":16344,"201":16345,"wurttemberg":16346,"##orf":16347,"enforced":16348,"hovering":16349,"proprietary":16350,"gb":16351,"samurai":16352,"syllable":16353,"ascent":16354,"lacey":16355,"tick":16356,"lars":16357,"tractor":16358,"merchandise":16359,"rep":16360,"bouncing":16361,"defendants":16362,"##yre":16363,"huntington":16364,"##ground":16365,"##oko":16366,"standardized":16367,"##hor":16368,"##hima":16369,"assassinated":16370,"nu":16371,"predecessors":16372,"rainy":16373,"liar":16374,"assurance":16375,"lyrical":16376,"##uga":16377,"secondly":16378,"flattened":16379,"ios":16380,"parameter":16381,"undercover":16382,"##mity":16383,"bordeaux":16384,"punish":16385,"ridges":16386,"markers":16387,"exodus":16388,"inactive":16389,"hesitate":16390,"debbie":16391,"nyc":16392,"pledge":16393,"savoy":16394,"nagar":16395,"offset":16396,"organist":16397,"##tium":16398,"hesse":16399,"marin":16400,"converting":16401,"##iver":16402,"diagram":16403,"propulsion":16404,"pu":16405,"validity":16406,"reverted":16407,"supportive":16408,"##dc":16409,"ministries":16410,"clans":16411,"responds":16412,"proclamation":16413,"##inae":16414,"##ø":16415,"##rea":16416,"ein":16417,"pleading":16418,"patriot":16419,"sf":16420,"birch":16421,"islanders":16422,"strauss":16423,"hates":16424,"##dh":16425,"brandenburg":16426,"concession":16427,"rd":16428,"##ob":16429,"1900s":16430,"killings":16431,"textbook":16432,"antiquity":16433,"cinematography":16434,"wharf":16435,"embarrassing":16436,"setup":16437,"creed":16438,"farmland":16439,"inequality":16440,"centred":16441,"signatures":16442,"fallon":16443,"370":16444,"##ingham":16445,"##uts":16446,"ceylon":16447,"gazing":16448,"directive":16449,"laurie":16450,"##tern":16451,"globally":16452,"##uated":16453,"##dent":16454,"allah":16455,"excavation":16456,"threads":16457,"##cross":16458,"148":16459,"frantically":16460,"icc":16461,"utilize":16462,"determines":16463,"respiratory":16464,"thoughtful":16465,"receptions":16466,"##dicate":16467,"merging":16468,"chandra":16469,"seine":16470,"147":16471,"builders":16472,"builds":16473,"diagnostic":16474,"dev":16475,"visibility":16476,"goddamn":16477,"analyses":16478,"dhaka":16479,"cho":16480,"proves":16481,"chancel":16482,"concurrent":16483,"curiously":16484,"canadians":16485,"pumped":16486,"restoring":16487,"1850s":16488,"turtles":16489,"jaguar":16490,"sinister":16491,"spinal":16492,"traction":16493,"declan":16494,"vows":16495,"1784":16496,"glowed":16497,"capitalism":16498,"swirling":16499,"install":16500,"universidad":16501,"##lder":16502,"##oat":16503,"soloist":16504,"##genic":16505,"##oor":16506,"coincidence":16507,"beginnings":16508,"nissan":16509,"dip":16510,"resorts":16511,"caucasus":16512,"combustion":16513,"infectious":16514,"##eno":16515,"pigeon":16516,"serpent":16517,"##itating":16518,"conclude":16519,"masked":16520,"salad":16521,"jew":16522,"##gr":16523,"surreal":16524,"toni":16525,"##wc":16526,"harmonica":16527,"151":16528,"##gins":16529,"##etic":16530,"##coat":16531,"fishermen":16532,"intending":16533,"bravery":16534,"##wave":16535,"klaus":16536,"titan":16537,"wembley":16538,"taiwanese":16539,"ransom":16540,"40th":16541,"incorrect":16542,"hussein":16543,"eyelids":16544,"jp":16545,"cooke":16546,"dramas":16547,"utilities":16548,"##etta":16549,"##print":16550,"eisenhower":16551,"principally":16552,"granada":16553,"lana":16554,"##rak":16555,"openings":16556,"concord":16557,"##bl":16558,"bethany":16559,"connie":16560,"morality":16561,"sega":16562,"##mons":16563,"##nard":16564,"earnings":16565,"##kara":16566,"##cine":16567,"wii":16568,"communes":16569,"##rel":16570,"coma":16571,"composing":16572,"softened":16573,"severed":16574,"grapes":16575,"##17":16576,"nguyen":16577,"analyzed":16578,"warlord":16579,"hubbard":16580,"heavenly":16581,"behave":16582,"slovenian":16583,"##hit":16584,"##ony":16585,"hailed":16586,"filmmakers":16587,"trance":16588,"caldwell":16589,"skye":16590,"unrest":16591,"coward":16592,"likelihood":16593,"##aging":16594,"bern":16595,"sci":16596,"taliban":16597,"honolulu":16598,"propose":16599,"##wang":16600,"1700":16601,"browser":16602,"imagining":16603,"cobra":16604,"contributes":16605,"dukes":16606,"instinctively":16607,"conan":16608,"violinist":16609,"##ores":16610,"accessories":16611,"gradual":16612,"##amp":16613,"quotes":16614,"sioux":16615,"##dating":16616,"undertake":16617,"intercepted":16618,"sparkling":16619,"compressed":16620,"139":16621,"fungus":16622,"tombs":16623,"haley":16624,"imposing":16625,"rests":16626,"degradation":16627,"lincolnshire":16628,"retailers":16629,"wetlands":16630,"tulsa":16631,"distributor":16632,"dungeon":16633,"nun":16634,"greenhouse":16635,"convey":16636,"atlantis":16637,"aft":16638,"exits":16639,"oman":16640,"dresser":16641,"lyons":16642,"##sti":16643,"joking":16644,"eddy":16645,"judgement":16646,"omitted":16647,"digits":16648,"##cts":16649,"##game":16650,"juniors":16651,"##rae":16652,"cents":16653,"stricken":16654,"une":16655,"##ngo":16656,"wizards":16657,"weir":16658,"breton":16659,"nan":16660,"technician":16661,"fibers":16662,"liking":16663,"royalty":16664,"##cca":16665,"154":16666,"persia":16667,"terribly":16668,"magician":16669,"##rable":16670,"##unt":16671,"vance":16672,"cafeteria":16673,"booker":16674,"camille":16675,"warmer":16676,"##static":16677,"consume":16678,"cavern":16679,"gaps":16680,"compass":16681,"contemporaries":16682,"foyer":16683,"soothing":16684,"graveyard":16685,"maj":16686,"plunged":16687,"blush":16688,"##wear":16689,"cascade":16690,"demonstrates":16691,"ordinance":16692,"##nov":16693,"boyle":16694,"##lana":16695,"rockefeller":16696,"shaken":16697,"banjo":16698,"izzy":16699,"##ense":16700,"breathless":16701,"vines":16702,"##32":16703,"##eman":16704,"alterations":16705,"chromosome":16706,"dwellings":16707,"feudal":16708,"mole":16709,"153":16710,"catalonia":16711,"relics":16712,"tenant":16713,"mandated":16714,"##fm":16715,"fridge":16716,"hats":16717,"honesty":16718,"patented":16719,"raul":16720,"heap":16721,"cruisers":16722,"accusing":16723,"enlightenment":16724,"infants":16725,"wherein":16726,"chatham":16727,"contractors":16728,"zen":16729,"affinity":16730,"hc":16731,"osborne":16732,"piston":16733,"156":16734,"traps":16735,"maturity":16736,"##rana":16737,"lagos":16738,"##zal":16739,"peering":16740,"##nay":16741,"attendant":16742,"dealers":16743,"protocols":16744,"subset":16745,"prospects":16746,"biographical":16747,"##cre":16748,"artery":16749,"##zers":16750,"insignia":16751,"nuns":16752,"endured":16753,"##eration":16754,"recommend":16755,"schwartz":16756,"serbs":16757,"berger":16758,"cromwell":16759,"crossroads":16760,"##ctor":16761,"enduring":16762,"clasped":16763,"grounded":16764,"##bine":16765,"marseille":16766,"twitched":16767,"abel":16768,"choke":16769,"https":16770,"catalyst":16771,"moldova":16772,"italians":16773,"##tist":16774,"disastrous":16775,"wee":16776,"##oured":16777,"##nti":16778,"wwf":16779,"nope":16780,"##piration":16781,"##asa":16782,"expresses":16783,"thumbs":16784,"167":16785,"##nza":16786,"coca":16787,"1781":16788,"cheating":16789,"##ption":16790,"skipped":16791,"sensory":16792,"heidelberg":16793,"spies":16794,"satan":16795,"dangers":16796,"semifinal":16797,"202":16798,"bohemia":16799,"whitish":16800,"confusing":16801,"shipbuilding":16802,"relies":16803,"surgeons":16804,"landings":16805,"ravi":16806,"baku":16807,"moor":16808,"suffix":16809,"alejandro":16810,"##yana":16811,"litre":16812,"upheld":16813,"##unk":16814,"rajasthan":16815,"##rek":16816,"coaster":16817,"insists":16818,"posture":16819,"scenarios":16820,"etienne":16821,"favoured":16822,"appoint":16823,"transgender":16824,"elephants":16825,"poked":16826,"greenwood":16827,"defences":16828,"fulfilled":16829,"militant":16830,"somali":16831,"1758":16832,"chalk":16833,"potent":16834,"##ucci":16835,"migrants":16836,"wink":16837,"assistants":16838,"nos":16839,"restriction":16840,"activism":16841,"niger":16842,"##ario":16843,"colon":16844,"shaun":16845,"##sat":16846,"daphne":16847,"##erated":16848,"swam":16849,"congregations":16850,"reprise":16851,"considerations":16852,"magnet":16853,"playable":16854,"xvi":16855,"##р":16856,"overthrow":16857,"tobias":16858,"knob":16859,"chavez":16860,"coding":16861,"##mers":16862,"propped":16863,"katrina":16864,"orient":16865,"newcomer":16866,"##suke":16867,"temperate":16868,"##pool":16869,"farmhouse":16870,"interrogation":16871,"##vd":16872,"committing":16873,"##vert":16874,"forthcoming":16875,"strawberry":16876,"joaquin":16877,"macau":16878,"ponds":16879,"shocking":16880,"siberia":16881,"##cellular":16882,"chant":16883,"contributors":16884,"##nant":16885,"##ologists":16886,"sped":16887,"absorb":16888,"hail":16889,"1782":16890,"spared":16891,"##hore":16892,"barbados":16893,"karate":16894,"opus":16895,"originates":16896,"saul":16897,"##xie":16898,"evergreen":16899,"leaped":16900,"##rock":16901,"correlation":16902,"exaggerated":16903,"weekday":16904,"unification":16905,"bump":16906,"tracing":16907,"brig":16908,"afb":16909,"pathways":16910,"utilizing":16911,"##ners":16912,"mod":16913,"mb":16914,"disturbance":16915,"kneeling":16916,"##stad":16917,"##guchi":16918,"100th":16919,"pune":16920,"##thy":16921,"decreasing":16922,"168":16923,"manipulation":16924,"miriam":16925,"academia":16926,"ecosystem":16927,"occupational":16928,"rbi":16929,"##lem":16930,"rift":16931,"##14":16932,"rotary":16933,"stacked":16934,"incorporation":16935,"awakening":16936,"generators":16937,"guerrero":16938,"racist":16939,"##omy":16940,"cyber":16941,"derivatives":16942,"culminated":16943,"allie":16944,"annals":16945,"panzer":16946,"sainte":16947,"wikipedia":16948,"pops":16949,"zu":16950,"austro":16951,"##vate":16952,"algerian":16953,"politely":16954,"nicholson":16955,"mornings":16956,"educate":16957,"tastes":16958,"thrill":16959,"dartmouth":16960,"##gating":16961,"db":16962,"##jee":16963,"regan":16964,"differing":16965,"concentrating":16966,"choreography":16967,"divinity":16968,"##media":16969,"pledged":16970,"alexandre":16971,"routing":16972,"gregor":16973,"madeline":16974,"##idal":16975,"apocalypse":16976,"##hora":16977,"gunfire":16978,"culminating":16979,"elves":16980,"fined":16981,"liang":16982,"lam":16983,"programmed":16984,"tar":16985,"guessing":16986,"transparency":16987,"gabrielle":16988,"##gna":16989,"cancellation":16990,"flexibility":16991,"##lining":16992,"accession":16993,"shea":16994,"stronghold":16995,"nets":16996,"specializes":16997,"##rgan":16998,"abused":16999,"hasan":17000,"sgt":17001,"ling":17002,"exceeding":17003,"##₄":17004,"admiration":17005,"supermarket":17006,"##ark":17007,"photographers":17008,"specialised":17009,"tilt":17010,"resonance":17011,"hmm":17012,"perfume":17013,"380":17014,"sami":17015,"threatens":17016,"garland":17017,"botany":17018,"guarding":17019,"boiled":17020,"greet":17021,"puppy":17022,"russo":17023,"supplier":17024,"wilmington":17025,"vibrant":17026,"vijay":17027,"##bius":17028,"paralympic":17029,"grumbled":17030,"paige":17031,"faa":17032,"licking":17033,"margins":17034,"hurricanes":17035,"##gong":17036,"fest":17037,"grenade":17038,"ripping":17039,"##uz":17040,"counseling":17041,"weigh":17042,"##sian":17043,"needles":17044,"wiltshire":17045,"edison":17046,"costly":17047,"##not":17048,"fulton":17049,"tramway":17050,"redesigned":17051,"staffordshire":17052,"cache":17053,"gasping":17054,"watkins":17055,"sleepy":17056,"candidacy":17057,"##group":17058,"monkeys":17059,"timeline":17060,"throbbing":17061,"##bid":17062,"##sos":17063,"berth":17064,"uzbekistan":17065,"vanderbilt":17066,"bothering":17067,"overturned":17068,"ballots":17069,"gem":17070,"##iger":17071,"sunglasses":17072,"subscribers":17073,"hooker":17074,"compelling":17075,"ang":17076,"exceptionally":17077,"saloon":17078,"stab":17079,"##rdi":17080,"carla":17081,"terrifying":17082,"rom":17083,"##vision":17084,"coil":17085,"##oids":17086,"satisfying":17087,"vendors":17088,"31st":17089,"mackay":17090,"deities":17091,"overlooked":17092,"ambient":17093,"bahamas":17094,"felipe":17095,"olympia":17096,"whirled":17097,"botanist":17098,"advertised":17099,"tugging":17100,"##dden":17101,"disciples":17102,"morales":17103,"unionist":17104,"rites":17105,"foley":17106,"morse":17107,"motives":17108,"creepy":17109,"##₀":17110,"soo":17111,"##sz":17112,"bargain":17113,"highness":17114,"frightening":17115,"turnpike":17116,"tory":17117,"reorganization":17118,"##cer":17119,"depict":17120,"biographer":17121,"##walk":17122,"unopposed":17123,"manifesto":17124,"##gles":17125,"institut":17126,"emile":17127,"accidental":17128,"kapoor":17129,"##dam":17130,"kilkenny":17131,"cortex":17132,"lively":17133,"##13":17134,"romanesque":17135,"jain":17136,"shan":17137,"cannons":17138,"##ood":17139,"##ske":17140,"petrol":17141,"echoing":17142,"amalgamated":17143,"disappears":17144,"cautious":17145,"proposes":17146,"sanctions":17147,"trenton":17148,"##ر":17149,"flotilla":17150,"aus":17151,"contempt":17152,"tor":17153,"canary":17154,"cote":17155,"theirs":17156,"##hun":17157,"conceptual":17158,"deleted":17159,"fascinating":17160,"paso":17161,"blazing":17162,"elf":17163,"honourable":17164,"hutchinson":17165,"##eiro":17166,"##outh":17167,"##zin":17168,"surveyor":17169,"tee":17170,"amidst":17171,"wooded":17172,"reissue":17173,"intro":17174,"##ono":17175,"cobb":17176,"shelters":17177,"newsletter":17178,"hanson":17179,"brace":17180,"encoding":17181,"confiscated":17182,"dem":17183,"caravan":17184,"marino":17185,"scroll":17186,"melodic":17187,"cows":17188,"imam":17189,"##adi":17190,"##aneous":17191,"northward":17192,"searches":17193,"biodiversity":17194,"cora":17195,"310":17196,"roaring":17197,"##bers":17198,"connell":17199,"theologian":17200,"halo":17201,"compose":17202,"pathetic":17203,"unmarried":17204,"dynamo":17205,"##oot":17206,"az":17207,"calculation":17208,"toulouse":17209,"deserves":17210,"humour":17211,"nr":17212,"forgiveness":17213,"tam":17214,"undergone":17215,"martyr":17216,"pamela":17217,"myths":17218,"whore":17219,"counselor":17220,"hicks":17221,"290":17222,"heavens":17223,"battleship":17224,"electromagnetic":17225,"##bbs":17226,"stellar":17227,"establishments":17228,"presley":17229,"hopped":17230,"##chin":17231,"temptation":17232,"90s":17233,"wills":17234,"nas":17235,"##yuan":17236,"nhs":17237,"##nya":17238,"seminars":17239,"##yev":17240,"adaptations":17241,"gong":17242,"asher":17243,"lex":17244,"indicator":17245,"sikh":17246,"tobago":17247,"cites":17248,"goin":17249,"##yte":17250,"satirical":17251,"##gies":17252,"characterised":17253,"correspond":17254,"bubbles":17255,"lure":17256,"participates":17257,"##vid":17258,"eruption":17259,"skate":17260,"therapeutic":17261,"1785":17262,"canals":17263,"wholesale":17264,"defaulted":17265,"sac":17266,"460":17267,"petit":17268,"##zzled":17269,"virgil":17270,"leak":17271,"ravens":17272,"256":17273,"portraying":17274,"##yx":17275,"ghetto":17276,"creators":17277,"dams":17278,"portray":17279,"vicente":17280,"##rington":17281,"fae":17282,"namesake":17283,"bounty":17284,"##arium":17285,"joachim":17286,"##ota":17287,"##iser":17288,"aforementioned":17289,"axle":17290,"snout":17291,"depended":17292,"dismantled":17293,"reuben":17294,"480":17295,"##ibly":17296,"gallagher":17297,"##lau":17298,"##pd":17299,"earnest":17300,"##ieu":17301,"##iary":17302,"inflicted":17303,"objections":17304,"##llar":17305,"asa":17306,"gritted":17307,"##athy":17308,"jericho":17309,"##sea":17310,"##was":17311,"flick":17312,"underside":17313,"ceramics":17314,"undead":17315,"substituted":17316,"195":17317,"eastward":17318,"undoubtedly":17319,"wheeled":17320,"chimney":17321,"##iche":17322,"guinness":17323,"cb":17324,"##ager":17325,"siding":17326,"##bell":17327,"traitor":17328,"baptiste":17329,"disguised":17330,"inauguration":17331,"149":17332,"tipperary":17333,"choreographer":17334,"perched":17335,"warmed":17336,"stationary":17337,"eco":17338,"##ike":17339,"##ntes":17340,"bacterial":17341,"##aurus":17342,"flores":17343,"phosphate":17344,"##core":17345,"attacker":17346,"invaders":17347,"alvin":17348,"intersects":17349,"a1":17350,"indirectly":17351,"immigrated":17352,"businessmen":17353,"cornelius":17354,"valves":17355,"narrated":17356,"pill":17357,"sober":17358,"ul":17359,"nationale":17360,"monastic":17361,"applicants":17362,"scenery":17363,"##jack":17364,"161":17365,"motifs":17366,"constitutes":17367,"cpu":17368,"##osh":17369,"jurisdictions":17370,"sd":17371,"tuning":17372,"irritation":17373,"woven":17374,"##uddin":17375,"fertility":17376,"gao":17377,"##erie":17378,"antagonist":17379,"impatient":17380,"glacial":17381,"hides":17382,"boarded":17383,"denominations":17384,"interception":17385,"##jas":17386,"cookie":17387,"nicola":17388,"##tee":17389,"algebraic":17390,"marquess":17391,"bahn":17392,"parole":17393,"buyers":17394,"bait":17395,"turbines":17396,"paperwork":17397,"bestowed":17398,"natasha":17399,"renee":17400,"oceans":17401,"purchases":17402,"157":17403,"vaccine":17404,"215":17405,"##tock":17406,"fixtures":17407,"playhouse":17408,"integrate":17409,"jai":17410,"oswald":17411,"intellectuals":17412,"##cky":17413,"booked":17414,"nests":17415,"mortimer":17416,"##isi":17417,"obsession":17418,"sept":17419,"##gler":17420,"##sum":17421,"440":17422,"scrutiny":17423,"simultaneous":17424,"squinted":17425,"##shin":17426,"collects":17427,"oven":17428,"shankar":17429,"penned":17430,"remarkably":17431,"##я":17432,"slips":17433,"luggage":17434,"spectral":17435,"1786":17436,"collaborations":17437,"louie":17438,"consolidation":17439,"##ailed":17440,"##ivating":17441,"420":17442,"hoover":17443,"blackpool":17444,"harness":17445,"ignition":17446,"vest":17447,"tails":17448,"belmont":17449,"mongol":17450,"skinner":17451,"##nae":17452,"visually":17453,"mage":17454,"derry":17455,"##tism":17456,"##unce":17457,"stevie":17458,"transitional":17459,"##rdy":17460,"redskins":17461,"drying":17462,"prep":17463,"prospective":17464,"##21":17465,"annoyance":17466,"oversee":17467,"##loaded":17468,"fills":17469,"##books":17470,"##iki":17471,"announces":17472,"fda":17473,"scowled":17474,"respects":17475,"prasad":17476,"mystic":17477,"tucson":17478,"##vale":17479,"revue":17480,"springer":17481,"bankrupt":17482,"1772":17483,"aristotle":17484,"salvatore":17485,"habsburg":17486,"##geny":17487,"dal":17488,"natal":17489,"nut":17490,"pod":17491,"chewing":17492,"darts":17493,"moroccan":17494,"walkover":17495,"rosario":17496,"lenin":17497,"punjabi":17498,"##ße":17499,"grossed":17500,"scattering":17501,"wired":17502,"invasive":17503,"hui":17504,"polynomial":17505,"corridors":17506,"wakes":17507,"gina":17508,"portrays":17509,"##cratic":17510,"arid":17511,"retreating":17512,"erich":17513,"irwin":17514,"sniper":17515,"##dha":17516,"linen":17517,"lindsey":17518,"maneuver":17519,"butch":17520,"shutting":17521,"socio":17522,"bounce":17523,"commemorative":17524,"postseason":17525,"jeremiah":17526,"pines":17527,"275":17528,"mystical":17529,"beads":17530,"bp":17531,"abbas":17532,"furnace":17533,"bidding":17534,"consulted":17535,"assaulted":17536,"empirical":17537,"rubble":17538,"enclosure":17539,"sob":17540,"weakly":17541,"cancel":17542,"polly":17543,"yielded":17544,"##emann":17545,"curly":17546,"prediction":17547,"battered":17548,"70s":17549,"vhs":17550,"jacqueline":17551,"render":17552,"sails":17553,"barked":17554,"detailing":17555,"grayson":17556,"riga":17557,"sloane":17558,"raging":17559,"##yah":17560,"herbs":17561,"bravo":17562,"##athlon":17563,"alloy":17564,"giggle":17565,"imminent":17566,"suffers":17567,"assumptions":17568,"waltz":17569,"##itate":17570,"accomplishments":17571,"##ited":17572,"bathing":17573,"remixed":17574,"deception":17575,"prefix":17576,"##emia":17577,"deepest":17578,"##tier":17579,"##eis":17580,"balkan":17581,"frogs":17582,"##rong":17583,"slab":17584,"##pate":17585,"philosophers":17586,"peterborough":17587,"grains":17588,"imports":17589,"dickinson":17590,"rwanda":17591,"##atics":17592,"1774":17593,"dirk":17594,"lan":17595,"tablets":17596,"##rove":17597,"clone":17598,"##rice":17599,"caretaker":17600,"hostilities":17601,"mclean":17602,"##gre":17603,"regimental":17604,"treasures":17605,"norms":17606,"impose":17607,"tsar":17608,"tango":17609,"diplomacy":17610,"variously":17611,"complain":17612,"192":17613,"recognise":17614,"arrests":17615,"1779":17616,"celestial":17617,"pulitzer":17618,"##dus":17619,"bing":17620,"libretto":17621,"##moor":17622,"adele":17623,"splash":17624,"##rite":17625,"expectation":17626,"lds":17627,"confronts":17628,"##izer":17629,"spontaneous":17630,"harmful":17631,"wedge":17632,"entrepreneurs":17633,"buyer":17634,"##ope":17635,"bilingual":17636,"translate":17637,"rugged":17638,"conner":17639,"circulated":17640,"uae":17641,"eaton":17642,"##gra":17643,"##zzle":17644,"lingered":17645,"lockheed":17646,"vishnu":17647,"reelection":17648,"alonso":17649,"##oom":17650,"joints":17651,"yankee":17652,"headline":17653,"cooperate":17654,"heinz":17655,"laureate":17656,"invading":17657,"##sford":17658,"echoes":17659,"scandinavian":17660,"##dham":17661,"hugging":17662,"vitamin":17663,"salute":17664,"micah":17665,"hind":17666,"trader":17667,"##sper":17668,"radioactive":17669,"##ndra":17670,"militants":17671,"poisoned":17672,"ratified":17673,"remark":17674,"campeonato":17675,"deprived":17676,"wander":17677,"prop":17678,"##dong":17679,"outlook":17680,"##tani":17681,"##rix":17682,"##eye":17683,"chiang":17684,"darcy":17685,"##oping":17686,"mandolin":17687,"spice":17688,"statesman":17689,"babylon":17690,"182":17691,"walled":17692,"forgetting":17693,"afro":17694,"##cap":17695,"158":17696,"giorgio":17697,"buffer":17698,"##polis":17699,"planetary":17700,"##gis":17701,"overlap":17702,"terminals":17703,"kinda":17704,"centenary":17705,"##bir":17706,"arising":17707,"manipulate":17708,"elm":17709,"ke":17710,"1770":17711,"ak":17712,"##tad":17713,"chrysler":17714,"mapped":17715,"moose":17716,"pomeranian":17717,"quad":17718,"macarthur":17719,"assemblies":17720,"shoreline":17721,"recalls":17722,"stratford":17723,"##rted":17724,"noticeable":17725,"##evic":17726,"imp":17727,"##rita":17728,"##sque":17729,"accustomed":17730,"supplying":17731,"tents":17732,"disgusted":17733,"vogue":17734,"sipped":17735,"filters":17736,"khz":17737,"reno":17738,"selecting":17739,"luftwaffe":17740,"mcmahon":17741,"tyne":17742,"masterpiece":17743,"carriages":17744,"collided":17745,"dunes":17746,"exercised":17747,"flare":17748,"remembers":17749,"muzzle":17750,"##mobile":17751,"heck":17752,"##rson":17753,"burgess":17754,"lunged":17755,"middleton":17756,"boycott":17757,"bilateral":17758,"##sity":17759,"hazardous":17760,"lumpur":17761,"multiplayer":17762,"spotlight":17763,"jackets":17764,"goldman":17765,"liege":17766,"porcelain":17767,"rag":17768,"waterford":17769,"benz":17770,"attracts":17771,"hopeful":17772,"battling":17773,"ottomans":17774,"kensington":17775,"baked":17776,"hymns":17777,"cheyenne":17778,"lattice":17779,"levine":17780,"borrow":17781,"polymer":17782,"clashes":17783,"michaels":17784,"monitored":17785,"commitments":17786,"denounced":17787,"##25":17788,"##von":17789,"cavity":17790,"##oney":17791,"hobby":17792,"akin":17793,"##holders":17794,"futures":17795,"intricate":17796,"cornish":17797,"patty":17798,"##oned":17799,"illegally":17800,"dolphin":17801,"##lag":17802,"barlow":17803,"yellowish":17804,"maddie":17805,"apologized":17806,"luton":17807,"plagued":17808,"##puram":17809,"nana":17810,"##rds":17811,"sway":17812,"fanny":17813,"łodz":17814,"##rino":17815,"psi":17816,"suspicions":17817,"hanged":17818,"##eding":17819,"initiate":17820,"charlton":17821,"##por":17822,"nak":17823,"competent":17824,"235":17825,"analytical":17826,"annex":17827,"wardrobe":17828,"reservations":17829,"##rma":17830,"sect":17831,"162":17832,"fairfax":17833,"hedge":17834,"piled":17835,"buckingham":17836,"uneven":17837,"bauer":17838,"simplicity":17839,"snyder":17840,"interpret":17841,"accountability":17842,"donors":17843,"moderately":17844,"byrd":17845,"continents":17846,"##cite":17847,"##max":17848,"disciple":17849,"hr":17850,"jamaican":17851,"ping":17852,"nominees":17853,"##uss":17854,"mongolian":17855,"diver":17856,"attackers":17857,"eagerly":17858,"ideological":17859,"pillows":17860,"miracles":17861,"apartheid":17862,"revolver":17863,"sulfur":17864,"clinics":17865,"moran":17866,"163":17867,"##enko":17868,"ile":17869,"katy":17870,"rhetoric":17871,"##icated":17872,"chronology":17873,"recycling":17874,"##hrer":17875,"elongated":17876,"mughal":17877,"pascal":17878,"profiles":17879,"vibration":17880,"databases":17881,"domination":17882,"##fare":17883,"##rant":17884,"matthias":17885,"digest":17886,"rehearsal":17887,"polling":17888,"weiss":17889,"initiation":17890,"reeves":17891,"clinging":17892,"flourished":17893,"impress":17894,"ngo":17895,"##hoff":17896,"##ume":17897,"buckley":17898,"symposium":17899,"rhythms":17900,"weed":17901,"emphasize":17902,"transforming":17903,"##taking":17904,"##gence":17905,"##yman":17906,"accountant":17907,"analyze":17908,"flicker":17909,"foil":17910,"priesthood":17911,"voluntarily":17912,"decreases":17913,"##80":17914,"##hya":17915,"slater":17916,"sv":17917,"charting":17918,"mcgill":17919,"##lde":17920,"moreno":17921,"##iu":17922,"besieged":17923,"zur":17924,"robes":17925,"##phic":17926,"admitting":17927,"api":17928,"deported":17929,"turmoil":17930,"peyton":17931,"earthquakes":17932,"##ares":17933,"nationalists":17934,"beau":17935,"clair":17936,"brethren":17937,"interrupt":17938,"welch":17939,"curated":17940,"galerie":17941,"requesting":17942,"164":17943,"##ested":17944,"impending":17945,"steward":17946,"viper":17947,"##vina":17948,"complaining":17949,"beautifully":17950,"brandy":17951,"foam":17952,"nl":17953,"1660":17954,"##cake":17955,"alessandro":17956,"punches":17957,"laced":17958,"explanations":17959,"##lim":17960,"attribute":17961,"clit":17962,"reggie":17963,"discomfort":17964,"##cards":17965,"smoothed":17966,"whales":17967,"##cene":17968,"adler":17969,"countered":17970,"duffy":17971,"disciplinary":17972,"widening":17973,"recipe":17974,"reliance":17975,"conducts":17976,"goats":17977,"gradient":17978,"preaching":17979,"##shaw":17980,"matilda":17981,"quasi":17982,"striped":17983,"meridian":17984,"cannabis":17985,"cordoba":17986,"certificates":17987,"##agh":17988,"##tering":17989,"graffiti":17990,"hangs":17991,"pilgrims":17992,"repeats":17993,"##ych":17994,"revive":17995,"urine":17996,"etat":17997,"##hawk":17998,"fueled":17999,"belts":18000,"fuzzy":18001,"susceptible":18002,"##hang":18003,"mauritius":18004,"salle":18005,"sincere":18006,"beers":18007,"hooks":18008,"##cki":18009,"arbitration":18010,"entrusted":18011,"advise":18012,"sniffed":18013,"seminar":18014,"junk":18015,"donnell":18016,"processors":18017,"principality":18018,"strapped":18019,"celia":18020,"mendoza":18021,"everton":18022,"fortunes":18023,"prejudice":18024,"starving":18025,"reassigned":18026,"steamer":18027,"##lund":18028,"tuck":18029,"evenly":18030,"foreman":18031,"##ffen":18032,"dans":18033,"375":18034,"envisioned":18035,"slit":18036,"##xy":18037,"baseman":18038,"liberia":18039,"rosemary":18040,"##weed":18041,"electrified":18042,"periodically":18043,"potassium":18044,"stride":18045,"contexts":18046,"sperm":18047,"slade":18048,"mariners":18049,"influx":18050,"bianca":18051,"subcommittee":18052,"##rane":18053,"spilling":18054,"icao":18055,"estuary":18056,"##nock":18057,"delivers":18058,"iphone":18059,"##ulata":18060,"isa":18061,"mira":18062,"bohemian":18063,"dessert":18064,"##sbury":18065,"welcoming":18066,"proudly":18067,"slowing":18068,"##chs":18069,"musee":18070,"ascension":18071,"russ":18072,"##vian":18073,"waits":18074,"##psy":18075,"africans":18076,"exploit":18077,"##morphic":18078,"gov":18079,"eccentric":18080,"crab":18081,"peck":18082,"##ull":18083,"entrances":18084,"formidable":18085,"marketplace":18086,"groom":18087,"bolted":18088,"metabolism":18089,"patton":18090,"robbins":18091,"courier":18092,"payload":18093,"endure":18094,"##ifier":18095,"andes":18096,"refrigerator":18097,"##pr":18098,"ornate":18099,"##uca":18100,"ruthless":18101,"illegitimate":18102,"masonry":18103,"strasbourg":18104,"bikes":18105,"adobe":18106,"##³":18107,"apples":18108,"quintet":18109,"willingly":18110,"niche":18111,"bakery":18112,"corpses":18113,"energetic":18114,"##cliffe":18115,"##sser":18116,"##ards":18117,"177":18118,"centimeters":18119,"centro":18120,"fuscous":18121,"cretaceous":18122,"rancho":18123,"##yde":18124,"andrei":18125,"telecom":18126,"tottenham":18127,"oasis":18128,"ordination":18129,"vulnerability":18130,"presiding":18131,"corey":18132,"cp":18133,"penguins":18134,"sims":18135,"##pis":18136,"malawi":18137,"piss":18138,"##48":18139,"correction":18140,"##cked":18141,"##ffle":18142,"##ryn":18143,"countdown":18144,"detectives":18145,"psychiatrist":18146,"psychedelic":18147,"dinosaurs":18148,"blouse":18149,"##get":18150,"choi":18151,"vowed":18152,"##oz":18153,"randomly":18154,"##pol":18155,"49ers":18156,"scrub":18157,"blanche":18158,"bruins":18159,"dusseldorf":18160,"##using":18161,"unwanted":18162,"##ums":18163,"212":18164,"dominique":18165,"elevations":18166,"headlights":18167,"om":18168,"laguna":18169,"##oga":18170,"1750":18171,"famously":18172,"ignorance":18173,"shrewsbury":18174,"##aine":18175,"ajax":18176,"breuning":18177,"che":18178,"confederacy":18179,"greco":18180,"overhaul":18181,"##screen":18182,"paz":18183,"skirts":18184,"disagreement":18185,"cruelty":18186,"jagged":18187,"phoebe":18188,"shifter":18189,"hovered":18190,"viruses":18191,"##wes":18192,"mandy":18193,"##lined":18194,"##gc":18195,"landlord":18196,"squirrel":18197,"dashed":18198,"##ι":18199,"ornamental":18200,"gag":18201,"wally":18202,"grange":18203,"literal":18204,"spurs":18205,"undisclosed":18206,"proceeding":18207,"yin":18208,"##text":18209,"billie":18210,"orphan":18211,"spanned":18212,"humidity":18213,"indy":18214,"weighted":18215,"presentations":18216,"explosions":18217,"lucian":18218,"##tary":18219,"vaughn":18220,"hindus":18221,"##anga":18222,"##hell":18223,"psycho":18224,"171":18225,"daytona":18226,"protects":18227,"efficiently":18228,"rematch":18229,"sly":18230,"tandem":18231,"##oya":18232,"rebranded":18233,"impaired":18234,"hee":18235,"metropolis":18236,"peach":18237,"godfrey":18238,"diaspora":18239,"ethnicity":18240,"prosperous":18241,"gleaming":18242,"dar":18243,"grossing":18244,"playback":18245,"##rden":18246,"stripe":18247,"pistols":18248,"##tain":18249,"births":18250,"labelled":18251,"##cating":18252,"172":18253,"rudy":18254,"alba":18255,"##onne":18256,"aquarium":18257,"hostility":18258,"##gb":18259,"##tase":18260,"shudder":18261,"sumatra":18262,"hardest":18263,"lakers":18264,"consonant":18265,"creeping":18266,"demos":18267,"homicide":18268,"capsule":18269,"zeke":18270,"liberties":18271,"expulsion":18272,"pueblo":18273,"##comb":18274,"trait":18275,"transporting":18276,"##ddin":18277,"##neck":18278,"##yna":18279,"depart":18280,"gregg":18281,"mold":18282,"ledge":18283,"hangar":18284,"oldham":18285,"playboy":18286,"termination":18287,"analysts":18288,"gmbh":18289,"romero":18290,"##itic":18291,"insist":18292,"cradle":18293,"filthy":18294,"brightness":18295,"slash":18296,"shootout":18297,"deposed":18298,"bordering":18299,"##truct":18300,"isis":18301,"microwave":18302,"tumbled":18303,"sheltered":18304,"cathy":18305,"werewolves":18306,"messy":18307,"andersen":18308,"convex":18309,"clapped":18310,"clinched":18311,"satire":18312,"wasting":18313,"edo":18314,"vc":18315,"rufus":18316,"##jak":18317,"mont":18318,"##etti":18319,"poznan":18320,"##keeping":18321,"restructuring":18322,"transverse":18323,"##rland":18324,"azerbaijani":18325,"slovene":18326,"gestures":18327,"roommate":18328,"choking":18329,"shear":18330,"##quist":18331,"vanguard":18332,"oblivious":18333,"##hiro":18334,"disagreed":18335,"baptism":18336,"##lich":18337,"coliseum":18338,"##aceae":18339,"salvage":18340,"societe":18341,"cory":18342,"locke":18343,"relocation":18344,"relying":18345,"versailles":18346,"ahl":18347,"swelling":18348,"##elo":18349,"cheerful":18350,"##word":18351,"##edes":18352,"gin":18353,"sarajevo":18354,"obstacle":18355,"diverted":18356,"##nac":18357,"messed":18358,"thoroughbred":18359,"fluttered":18360,"utrecht":18361,"chewed":18362,"acquaintance":18363,"assassins":18364,"dispatch":18365,"mirza":18366,"##wart":18367,"nike":18368,"salzburg":18369,"swell":18370,"yen":18371,"##gee":18372,"idle":18373,"ligue":18374,"samson":18375,"##nds":18376,"##igh":18377,"playful":18378,"spawned":18379,"##cise":18380,"tease":18381,"##case":18382,"burgundy":18383,"##bot":18384,"stirring":18385,"skeptical":18386,"interceptions":18387,"marathi":18388,"##dies":18389,"bedrooms":18390,"aroused":18391,"pinch":18392,"##lik":18393,"preferences":18394,"tattoos":18395,"buster":18396,"digitally":18397,"projecting":18398,"rust":18399,"##ital":18400,"kitten":18401,"priorities":18402,"addison":18403,"pseudo":18404,"##guard":18405,"dusk":18406,"icons":18407,"sermon":18408,"##psis":18409,"##iba":18410,"bt":18411,"##lift":18412,"##xt":18413,"ju":18414,"truce":18415,"rink":18416,"##dah":18417,"##wy":18418,"defects":18419,"psychiatry":18420,"offences":18421,"calculate":18422,"glucose":18423,"##iful":18424,"##rized":18425,"##unda":18426,"francaise":18427,"##hari":18428,"richest":18429,"warwickshire":18430,"carly":18431,"1763":18432,"purity":18433,"redemption":18434,"lending":18435,"##cious":18436,"muse":18437,"bruises":18438,"cerebral":18439,"aero":18440,"carving":18441,"##name":18442,"preface":18443,"terminology":18444,"invade":18445,"monty":18446,"##int":18447,"anarchist":18448,"blurred":18449,"##iled":18450,"rossi":18451,"treats":18452,"guts":18453,"shu":18454,"foothills":18455,"ballads":18456,"undertaking":18457,"premise":18458,"cecilia":18459,"affiliates":18460,"blasted":18461,"conditional":18462,"wilder":18463,"minors":18464,"drone":18465,"rudolph":18466,"buffy":18467,"swallowing":18468,"horton":18469,"attested":18470,"##hop":18471,"rutherford":18472,"howell":18473,"primetime":18474,"livery":18475,"penal":18476,"##bis":18477,"minimize":18478,"hydro":18479,"wrecked":18480,"wrought":18481,"palazzo":18482,"##gling":18483,"cans":18484,"vernacular":18485,"friedman":18486,"nobleman":18487,"shale":18488,"walnut":18489,"danielle":18490,"##ection":18491,"##tley":18492,"sears":18493,"##kumar":18494,"chords":18495,"lend":18496,"flipping":18497,"streamed":18498,"por":18499,"dracula":18500,"gallons":18501,"sacrifices":18502,"gamble":18503,"orphanage":18504,"##iman":18505,"mckenzie":18506,"##gible":18507,"boxers":18508,"daly":18509,"##balls":18510,"##ان":18511,"208":18512,"##ific":18513,"##rative":18514,"##iq":18515,"exploited":18516,"slated":18517,"##uity":18518,"circling":18519,"hillary":18520,"pinched":18521,"goldberg":18522,"provost":18523,"campaigning":18524,"lim":18525,"piles":18526,"ironically":18527,"jong":18528,"mohan":18529,"successors":18530,"usaf":18531,"##tem":18532,"##ught":18533,"autobiographical":18534,"haute":18535,"preserves":18536,"##ending":18537,"acquitted":18538,"comparisons":18539,"203":18540,"hydroelectric":18541,"gangs":18542,"cypriot":18543,"torpedoes":18544,"rushes":18545,"chrome":18546,"derive":18547,"bumps":18548,"instability":18549,"fiat":18550,"pets":18551,"##mbe":18552,"silas":18553,"dye":18554,"reckless":18555,"settler":18556,"##itation":18557,"info":18558,"heats":18559,"##writing":18560,"176":18561,"canonical":18562,"maltese":18563,"fins":18564,"mushroom":18565,"stacy":18566,"aspen":18567,"avid":18568,"##kur":18569,"##loading":18570,"vickers":18571,"gaston":18572,"hillside":18573,"statutes":18574,"wilde":18575,"gail":18576,"kung":18577,"sabine":18578,"comfortably":18579,"motorcycles":18580,"##rgo":18581,"169":18582,"pneumonia":18583,"fetch":18584,"##sonic":18585,"axel":18586,"faintly":18587,"parallels":18588,"##oop":18589,"mclaren":18590,"spouse":18591,"compton":18592,"interdisciplinary":18593,"miner":18594,"##eni":18595,"181":18596,"clamped":18597,"##chal":18598,"##llah":18599,"separates":18600,"versa":18601,"##mler":18602,"scarborough":18603,"labrador":18604,"##lity":18605,"##osing":18606,"rutgers":18607,"hurdles":18608,"como":18609,"166":18610,"burt":18611,"divers":18612,"##100":18613,"wichita":18614,"cade":18615,"coincided":18616,"##erson":18617,"bruised":18618,"mla":18619,"##pper":18620,"vineyard":18621,"##ili":18622,"##brush":18623,"notch":18624,"mentioning":18625,"jase":18626,"hearted":18627,"kits":18628,"doe":18629,"##acle":18630,"pomerania":18631,"##ady":18632,"ronan":18633,"seizure":18634,"pavel":18635,"problematic":18636,"##zaki":18637,"domenico":18638,"##ulin":18639,"catering":18640,"penelope":18641,"dependence":18642,"parental":18643,"emilio":18644,"ministerial":18645,"atkinson":18646,"##bolic":18647,"clarkson":18648,"chargers":18649,"colby":18650,"grill":18651,"peeked":18652,"arises":18653,"summon":18654,"##aged":18655,"fools":18656,"##grapher":18657,"faculties":18658,"qaeda":18659,"##vial":18660,"garner":18661,"refurbished":18662,"##hwa":18663,"geelong":18664,"disasters":18665,"nudged":18666,"bs":18667,"shareholder":18668,"lori":18669,"algae":18670,"reinstated":18671,"rot":18672,"##ades":18673,"##nous":18674,"invites":18675,"stainless":18676,"183":18677,"inclusive":18678,"##itude":18679,"diocesan":18680,"til":18681,"##icz":18682,"denomination":18683,"##xa":18684,"benton":18685,"floral":18686,"registers":18687,"##ider":18688,"##erman":18689,"##kell":18690,"absurd":18691,"brunei":18692,"guangzhou":18693,"hitter":18694,"retaliation":18695,"##uled":18696,"##eve":18697,"blanc":18698,"nh":18699,"consistency":18700,"contamination":18701,"##eres":18702,"##rner":18703,"dire":18704,"palermo":18705,"broadcasters":18706,"diaries":18707,"inspire":18708,"vols":18709,"brewer":18710,"tightening":18711,"ky":18712,"mixtape":18713,"hormone":18714,"##tok":18715,"stokes":18716,"##color":18717,"##dly":18718,"##ssi":18719,"pg":18720,"##ometer":18721,"##lington":18722,"sanitation":18723,"##tility":18724,"intercontinental":18725,"apps":18726,"##adt":18727,"¹⁄₂":18728,"cylinders":18729,"economies":18730,"favourable":18731,"unison":18732,"croix":18733,"gertrude":18734,"odyssey":18735,"vanity":18736,"dangling":18737,"##logists":18738,"upgrades":18739,"dice":18740,"middleweight":18741,"practitioner":18742,"##ight":18743,"206":18744,"henrik":18745,"parlor":18746,"orion":18747,"angered":18748,"lac":18749,"python":18750,"blurted":18751,"##rri":18752,"sensual":18753,"intends":18754,"swings":18755,"angled":18756,"##phs":18757,"husky":18758,"attain":18759,"peerage":18760,"precinct":18761,"textiles":18762,"cheltenham":18763,"shuffled":18764,"dai":18765,"confess":18766,"tasting":18767,"bhutan":18768,"##riation":18769,"tyrone":18770,"segregation":18771,"abrupt":18772,"ruiz":18773,"##rish":18774,"smirked":18775,"blackwell":18776,"confidential":18777,"browning":18778,"amounted":18779,"##put":18780,"vase":18781,"scarce":18782,"fabulous":18783,"raided":18784,"staple":18785,"guyana":18786,"unemployed":18787,"glider":18788,"shay":18789,"##tow":18790,"carmine":18791,"troll":18792,"intervene":18793,"squash":18794,"superstar":18795,"##uce":18796,"cylindrical":18797,"len":18798,"roadway":18799,"researched":18800,"handy":18801,"##rium":18802,"##jana":18803,"meta":18804,"lao":18805,"declares":18806,"##rring":18807,"##tadt":18808,"##elin":18809,"##kova":18810,"willem":18811,"shrubs":18812,"napoleonic":18813,"realms":18814,"skater":18815,"qi":18816,"volkswagen":18817,"##ł":18818,"tad":18819,"hara":18820,"archaeologist":18821,"awkwardly":18822,"eerie":18823,"##kind":18824,"wiley":18825,"##heimer":18826,"##24":18827,"titus":18828,"organizers":18829,"cfl":18830,"crusaders":18831,"lama":18832,"usb":18833,"vent":18834,"enraged":18835,"thankful":18836,"occupants":18837,"maximilian":18838,"##gaard":18839,"possessing":18840,"textbooks":18841,"##oran":18842,"collaborator":18843,"quaker":18844,"##ulo":18845,"avalanche":18846,"mono":18847,"silky":18848,"straits":18849,"isaiah":18850,"mustang":18851,"surged":18852,"resolutions":18853,"potomac":18854,"descend":18855,"cl":18856,"kilograms":18857,"plato":18858,"strains":18859,"saturdays":18860,"##olin":18861,"bernstein":18862,"##ype":18863,"holstein":18864,"ponytail":18865,"##watch":18866,"belize":18867,"conversely":18868,"heroine":18869,"perpetual":18870,"##ylus":18871,"charcoal":18872,"piedmont":18873,"glee":18874,"negotiating":18875,"backdrop":18876,"prologue":18877,"##jah":18878,"##mmy":18879,"pasadena":18880,"climbs":18881,"ramos":18882,"sunni":18883,"##holm":18884,"##tner":18885,"##tri":18886,"anand":18887,"deficiency":18888,"hertfordshire":18889,"stout":18890,"##avi":18891,"aperture":18892,"orioles":18893,"##irs":18894,"doncaster":18895,"intrigued":18896,"bombed":18897,"coating":18898,"otis":18899,"##mat":18900,"cocktail":18901,"##jit":18902,"##eto":18903,"amir":18904,"arousal":18905,"sar":18906,"##proof":18907,"##act":18908,"##ories":18909,"dixie":18910,"pots":18911,"##bow":18912,"whereabouts":18913,"159":18914,"##fted":18915,"drains":18916,"bullying":18917,"cottages":18918,"scripture":18919,"coherent":18920,"fore":18921,"poe":18922,"appetite":18923,"##uration":18924,"sampled":18925,"##ators":18926,"##dp":18927,"derrick":18928,"rotor":18929,"jays":18930,"peacock":18931,"installment":18932,"##rro":18933,"advisors":18934,"##coming":18935,"rodeo":18936,"scotch":18937,"##mot":18938,"##db":18939,"##fen":18940,"##vant":18941,"ensued":18942,"rodrigo":18943,"dictatorship":18944,"martyrs":18945,"twenties":18946,"##н":18947,"towed":18948,"incidence":18949,"marta":18950,"rainforest":18951,"sai":18952,"scaled":18953,"##cles":18954,"oceanic":18955,"qualifiers":18956,"symphonic":18957,"mcbride":18958,"dislike":18959,"generalized":18960,"aubrey":18961,"colonization":18962,"##iation":18963,"##lion":18964,"##ssing":18965,"disliked":18966,"lublin":18967,"salesman":18968,"##ulates":18969,"spherical":18970,"whatsoever":18971,"sweating":18972,"avalon":18973,"contention":18974,"punt":18975,"severity":18976,"alderman":18977,"atari":18978,"##dina":18979,"##grant":18980,"##rop":18981,"scarf":18982,"seville":18983,"vertices":18984,"annexation":18985,"fairfield":18986,"fascination":18987,"inspiring":18988,"launches":18989,"palatinate":18990,"regretted":18991,"##rca":18992,"feral":18993,"##iom":18994,"elk":18995,"nap":18996,"olsen":18997,"reddy":18998,"yong":18999,"##leader":19000,"##iae":19001,"garment":19002,"transports":19003,"feng":19004,"gracie":19005,"outrage":19006,"viceroy":19007,"insides":19008,"##esis":19009,"breakup":19010,"grady":19011,"organizer":19012,"softer":19013,"grimaced":19014,"222":19015,"murals":19016,"galicia":19017,"arranging":19018,"vectors":19019,"##rsten":19020,"bas":19021,"##sb":19022,"##cens":19023,"sloan":19024,"##eka":19025,"bitten":19026,"ara":19027,"fender":19028,"nausea":19029,"bumped":19030,"kris":19031,"banquet":19032,"comrades":19033,"detector":19034,"persisted":19035,"##llan":19036,"adjustment":19037,"endowed":19038,"cinemas":19039,"##shot":19040,"sellers":19041,"##uman":19042,"peek":19043,"epa":19044,"kindly":19045,"neglect":19046,"simpsons":19047,"talon":19048,"mausoleum":19049,"runaway":19050,"hangul":19051,"lookout":19052,"##cic":19053,"rewards":19054,"coughed":19055,"acquainted":19056,"chloride":19057,"##ald":19058,"quicker":19059,"accordion":19060,"neolithic":19061,"##qa":19062,"artemis":19063,"coefficient":19064,"lenny":19065,"pandora":19066,"tx":19067,"##xed":19068,"ecstasy":19069,"litter":19070,"segunda":19071,"chairperson":19072,"gemma":19073,"hiss":19074,"rumor":19075,"vow":19076,"nasal":19077,"antioch":19078,"compensate":19079,"patiently":19080,"transformers":19081,"##eded":19082,"judo":19083,"morrow":19084,"penis":19085,"posthumous":19086,"philips":19087,"bandits":19088,"husbands":19089,"denote":19090,"flaming":19091,"##any":19092,"##phones":19093,"langley":19094,"yorker":19095,"1760":19096,"walters":19097,"##uo":19098,"##kle":19099,"gubernatorial":19100,"fatty":19101,"samsung":19102,"leroy":19103,"outlaw":19104,"##nine":19105,"unpublished":19106,"poole":19107,"jakob":19108,"##ᵢ":19109,"##ₙ":19110,"crete":19111,"distorted":19112,"superiority":19113,"##dhi":19114,"intercept":19115,"crust":19116,"mig":19117,"claus":19118,"crashes":19119,"positioning":19120,"188":19121,"stallion":19122,"301":19123,"frontal":19124,"armistice":19125,"##estinal":19126,"elton":19127,"aj":19128,"encompassing":19129,"camel":19130,"commemorated":19131,"malaria":19132,"woodward":19133,"calf":19134,"cigar":19135,"penetrate":19136,"##oso":19137,"willard":19138,"##rno":19139,"##uche":19140,"illustrate":19141,"amusing":19142,"convergence":19143,"noteworthy":19144,"##lma":19145,"##rva":19146,"journeys":19147,"realise":19148,"manfred":19149,"##sable":19150,"410":19151,"##vocation":19152,"hearings":19153,"fiance":19154,"##posed":19155,"educators":19156,"provoked":19157,"adjusting":19158,"##cturing":19159,"modular":19160,"stockton":19161,"paterson":19162,"vlad":19163,"rejects":19164,"electors":19165,"selena":19166,"maureen":19167,"##tres":19168,"uber":19169,"##rce":19170,"swirled":19171,"##num":19172,"proportions":19173,"nanny":19174,"pawn":19175,"naturalist":19176,"parma":19177,"apostles":19178,"awoke":19179,"ethel":19180,"wen":19181,"##bey":19182,"monsoon":19183,"overview":19184,"##inating":19185,"mccain":19186,"rendition":19187,"risky":19188,"adorned":19189,"##ih":19190,"equestrian":19191,"germain":19192,"nj":19193,"conspicuous":19194,"confirming":19195,"##yoshi":19196,"shivering":19197,"##imeter":19198,"milestone":19199,"rumours":19200,"flinched":19201,"bounds":19202,"smacked":19203,"token":19204,"##bei":19205,"lectured":19206,"automobiles":19207,"##shore":19208,"impacted":19209,"##iable":19210,"nouns":19211,"nero":19212,"##leaf":19213,"ismail":19214,"prostitute":19215,"trams":19216,"##lace":19217,"bridget":19218,"sud":19219,"stimulus":19220,"impressions":19221,"reins":19222,"revolves":19223,"##oud":19224,"##gned":19225,"giro":19226,"honeymoon":19227,"##swell":19228,"criterion":19229,"##sms":19230,"##uil":19231,"libyan":19232,"prefers":19233,"##osition":19234,"211":19235,"preview":19236,"sucks":19237,"accusation":19238,"bursts":19239,"metaphor":19240,"diffusion":19241,"tolerate":19242,"faye":19243,"betting":19244,"cinematographer":19245,"liturgical":19246,"specials":19247,"bitterly":19248,"humboldt":19249,"##ckle":19250,"flux":19251,"rattled":19252,"##itzer":19253,"archaeologists":19254,"odor":19255,"authorised":19256,"marshes":19257,"discretion":19258,"##ов":19259,"alarmed":19260,"archaic":19261,"inverse":19262,"##leton":19263,"explorers":19264,"##pine":19265,"drummond":19266,"tsunami":19267,"woodlands":19268,"##minate":19269,"##tland":19270,"booklet":19271,"insanity":19272,"owning":19273,"insert":19274,"crafted":19275,"calculus":19276,"##tore":19277,"receivers":19278,"##bt":19279,"stung":19280,"##eca":19281,"##nched":19282,"prevailing":19283,"travellers":19284,"eyeing":19285,"lila":19286,"graphs":19287,"##borne":19288,"178":19289,"julien":19290,"##won":19291,"morale":19292,"adaptive":19293,"therapist":19294,"erica":19295,"cw":19296,"libertarian":19297,"bowman":19298,"pitches":19299,"vita":19300,"##ional":19301,"crook":19302,"##ads":19303,"##entation":19304,"caledonia":19305,"mutiny":19306,"##sible":19307,"1840s":19308,"automation":19309,"##ß":19310,"flock":19311,"##pia":19312,"ironic":19313,"pathology":19314,"##imus":19315,"remarried":19316,"##22":19317,"joker":19318,"withstand":19319,"energies":19320,"##att":19321,"shropshire":19322,"hostages":19323,"madeleine":19324,"tentatively":19325,"conflicting":19326,"mateo":19327,"recipes":19328,"euros":19329,"ol":19330,"mercenaries":19331,"nico":19332,"##ndon":19333,"albuquerque":19334,"augmented":19335,"mythical":19336,"bel":19337,"freud":19338,"##child":19339,"cough":19340,"##lica":19341,"365":19342,"freddy":19343,"lillian":19344,"genetically":19345,"nuremberg":19346,"calder":19347,"209":19348,"bonn":19349,"outdoors":19350,"paste":19351,"suns":19352,"urgency":19353,"vin":19354,"restraint":19355,"tyson":19356,"##cera":19357,"##selle":19358,"barrage":19359,"bethlehem":19360,"kahn":19361,"##par":19362,"mounts":19363,"nippon":19364,"barony":19365,"happier":19366,"ryu":19367,"makeshift":19368,"sheldon":19369,"blushed":19370,"castillo":19371,"barking":19372,"listener":19373,"taped":19374,"bethel":19375,"fluent":19376,"headlines":19377,"pornography":19378,"rum":19379,"disclosure":19380,"sighing":19381,"mace":19382,"doubling":19383,"gunther":19384,"manly":19385,"##plex":19386,"rt":19387,"interventions":19388,"physiological":19389,"forwards":19390,"emerges":19391,"##tooth":19392,"##gny":19393,"compliment":19394,"rib":19395,"recession":19396,"visibly":19397,"barge":19398,"faults":19399,"connector":19400,"exquisite":19401,"prefect":19402,"##rlin":19403,"patio":19404,"##cured":19405,"elevators":19406,"brandt":19407,"italics":19408,"pena":19409,"173":19410,"wasp":19411,"satin":19412,"ea":19413,"botswana":19414,"graceful":19415,"respectable":19416,"##jima":19417,"##rter":19418,"##oic":19419,"franciscan":19420,"generates":19421,"##dl":19422,"alfredo":19423,"disgusting":19424,"##olate":19425,"##iously":19426,"sherwood":19427,"warns":19428,"cod":19429,"promo":19430,"cheryl":19431,"sino":19432,"##ة":19433,"##escu":19434,"twitch":19435,"##zhi":19436,"brownish":19437,"thom":19438,"ortiz":19439,"##dron":19440,"densely":19441,"##beat":19442,"carmel":19443,"reinforce":19444,"##bana":19445,"187":19446,"anastasia":19447,"downhill":19448,"vertex":19449,"contaminated":19450,"remembrance":19451,"harmonic":19452,"homework":19453,"##sol":19454,"fiancee":19455,"gears":19456,"olds":19457,"angelica":19458,"loft":19459,"ramsay":19460,"quiz":19461,"colliery":19462,"sevens":19463,"##cape":19464,"autism":19465,"##hil":19466,"walkway":19467,"##boats":19468,"ruben":19469,"abnormal":19470,"ounce":19471,"khmer":19472,"##bbe":19473,"zachary":19474,"bedside":19475,"morphology":19476,"punching":19477,"##olar":19478,"sparrow":19479,"convinces":19480,"##35":19481,"hewitt":19482,"queer":19483,"remastered":19484,"rods":19485,"mabel":19486,"solemn":19487,"notified":19488,"lyricist":19489,"symmetric":19490,"##xide":19491,"174":19492,"encore":19493,"passports":19494,"wildcats":19495,"##uni":19496,"baja":19497,"##pac":19498,"mildly":19499,"##ease":19500,"bleed":19501,"commodity":19502,"mounds":19503,"glossy":19504,"orchestras":19505,"##omo":19506,"damian":19507,"prelude":19508,"ambitions":19509,"##vet":19510,"awhile":19511,"remotely":19512,"##aud":19513,"asserts":19514,"imply":19515,"##iques":19516,"distinctly":19517,"modelling":19518,"remedy":19519,"##dded":19520,"windshield":19521,"dani":19522,"xiao":19523,"##endra":19524,"audible":19525,"powerplant":19526,"1300":19527,"invalid":19528,"elemental":19529,"acquisitions":19530,"##hala":19531,"immaculate":19532,"libby":19533,"plata":19534,"smuggling":19535,"ventilation":19536,"denoted":19537,"minh":19538,"##morphism":19539,"430":19540,"differed":19541,"dion":19542,"kelley":19543,"lore":19544,"mocking":19545,"sabbath":19546,"spikes":19547,"hygiene":19548,"drown":19549,"runoff":19550,"stylized":19551,"tally":19552,"liberated":19553,"aux":19554,"interpreter":19555,"righteous":19556,"aba":19557,"siren":19558,"reaper":19559,"pearce":19560,"millie":19561,"##cier":19562,"##yra":19563,"gaius":19564,"##iso":19565,"captures":19566,"##ttering":19567,"dorm":19568,"claudio":19569,"##sic":19570,"benches":19571,"knighted":19572,"blackness":19573,"##ored":19574,"discount":19575,"fumble":19576,"oxidation":19577,"routed":19578,"##ς":19579,"novak":19580,"perpendicular":19581,"spoiled":19582,"fracture":19583,"splits":19584,"##urt":19585,"pads":19586,"topology":19587,"##cats":19588,"axes":19589,"fortunate":19590,"offenders":19591,"protestants":19592,"esteem":19593,"221":19594,"broadband":19595,"convened":19596,"frankly":19597,"hound":19598,"prototypes":19599,"isil":19600,"facilitated":19601,"keel":19602,"##sher":19603,"sahara":19604,"awaited":19605,"bubba":19606,"orb":19607,"prosecutors":19608,"186":19609,"hem":19610,"520":19611,"##xing":19612,"relaxing":19613,"remnant":19614,"romney":19615,"sorted":19616,"slalom":19617,"stefano":19618,"ulrich":19619,"##active":19620,"exemption":19621,"folder":19622,"pauses":19623,"foliage":19624,"hitchcock":19625,"epithet":19626,"204":19627,"criticisms":19628,"##aca":19629,"ballistic":19630,"brody":19631,"hinduism":19632,"chaotic":19633,"youths":19634,"equals":19635,"##pala":19636,"pts":19637,"thicker":19638,"analogous":19639,"capitalist":19640,"improvised":19641,"overseeing":19642,"sinatra":19643,"ascended":19644,"beverage":19645,"##tl":19646,"straightforward":19647,"##kon":19648,"curran":19649,"##west":19650,"bois":19651,"325":19652,"induce":19653,"surveying":19654,"emperors":19655,"sax":19656,"unpopular":19657,"##kk":19658,"cartoonist":19659,"fused":19660,"##mble":19661,"unto":19662,"##yuki":19663,"localities":19664,"##cko":19665,"##ln":19666,"darlington":19667,"slain":19668,"academie":19669,"lobbying":19670,"sediment":19671,"puzzles":19672,"##grass":19673,"defiance":19674,"dickens":19675,"manifest":19676,"tongues":19677,"alumnus":19678,"arbor":19679,"coincide":19680,"184":19681,"appalachian":19682,"mustafa":19683,"examiner":19684,"cabaret":19685,"traumatic":19686,"yves":19687,"bracelet":19688,"draining":19689,"heroin":19690,"magnum":19691,"baths":19692,"odessa":19693,"consonants":19694,"mitsubishi":19695,"##gua":19696,"kellan":19697,"vaudeville":19698,"##fr":19699,"joked":19700,"null":19701,"straps":19702,"probation":19703,"##ław":19704,"ceded":19705,"interfaces":19706,"##pas":19707,"##zawa":19708,"blinding":19709,"viet":19710,"224":19711,"rothschild":19712,"museo":19713,"640":19714,"huddersfield":19715,"##vr":19716,"tactic":19717,"##storm":19718,"brackets":19719,"dazed":19720,"incorrectly":19721,"##vu":19722,"reg":19723,"glazed":19724,"fearful":19725,"manifold":19726,"benefited":19727,"irony":19728,"##sun":19729,"stumbling":19730,"##rte":19731,"willingness":19732,"balkans":19733,"mei":19734,"wraps":19735,"##aba":19736,"injected":19737,"##lea":19738,"gu":19739,"syed":19740,"harmless":19741,"##hammer":19742,"bray":19743,"takeoff":19744,"poppy":19745,"timor":19746,"cardboard":19747,"astronaut":19748,"purdue":19749,"weeping":19750,"southbound":19751,"cursing":19752,"stalls":19753,"diagonal":19754,"##neer":19755,"lamar":19756,"bryce":19757,"comte":19758,"weekdays":19759,"harrington":19760,"##uba":19761,"negatively":19762,"##see":19763,"lays":19764,"grouping":19765,"##cken":19766,"##henko":19767,"affirmed":19768,"halle":19769,"modernist":19770,"##lai":19771,"hodges":19772,"smelling":19773,"aristocratic":19774,"baptized":19775,"dismiss":19776,"justification":19777,"oilers":19778,"##now":19779,"coupling":19780,"qin":19781,"snack":19782,"healer":19783,"##qing":19784,"gardener":19785,"layla":19786,"battled":19787,"formulated":19788,"stephenson":19789,"gravitational":19790,"##gill":19791,"##jun":19792,"1768":19793,"granny":19794,"coordinating":19795,"suites":19796,"##cd":19797,"##ioned":19798,"monarchs":19799,"##cote":19800,"##hips":19801,"sep":19802,"blended":19803,"apr":19804,"barrister":19805,"deposition":19806,"fia":19807,"mina":19808,"policemen":19809,"paranoid":19810,"##pressed":19811,"churchyard":19812,"covert":19813,"crumpled":19814,"creep":19815,"abandoning":19816,"tr":19817,"transmit":19818,"conceal":19819,"barr":19820,"understands":19821,"readiness":19822,"spire":19823,"##cology":19824,"##enia":19825,"##erry":19826,"610":19827,"startling":19828,"unlock":19829,"vida":19830,"bowled":19831,"slots":19832,"##nat":19833,"##islav":19834,"spaced":19835,"trusting":19836,"admire":19837,"rig":19838,"##ink":19839,"slack":19840,"##70":19841,"mv":19842,"207":19843,"casualty":19844,"##wei":19845,"classmates":19846,"##odes":19847,"##rar":19848,"##rked":19849,"amherst":19850,"furnished":19851,"evolve":19852,"foundry":19853,"menace":19854,"mead":19855,"##lein":19856,"flu":19857,"wesleyan":19858,"##kled":19859,"monterey":19860,"webber":19861,"##vos":19862,"wil":19863,"##mith":19864,"##на":19865,"bartholomew":19866,"justices":19867,"restrained":19868,"##cke":19869,"amenities":19870,"191":19871,"mediated":19872,"sewage":19873,"trenches":19874,"ml":19875,"mainz":19876,"##thus":19877,"1800s":19878,"##cula":19879,"##inski":19880,"caine":19881,"bonding":19882,"213":19883,"converts":19884,"spheres":19885,"superseded":19886,"marianne":19887,"crypt":19888,"sweaty":19889,"ensign":19890,"historia":19891,"##br":19892,"spruce":19893,"##post":19894,"##ask":19895,"forks":19896,"thoughtfully":19897,"yukon":19898,"pamphlet":19899,"ames":19900,"##uter":19901,"karma":19902,"##yya":19903,"bryn":19904,"negotiation":19905,"sighs":19906,"incapable":19907,"##mbre":19908,"##ntial":19909,"actresses":19910,"taft":19911,"##mill":19912,"luce":19913,"prevailed":19914,"##amine":19915,"1773":19916,"motionless":19917,"envoy":19918,"testify":19919,"investing":19920,"sculpted":19921,"instructors":19922,"provence":19923,"kali":19924,"cullen":19925,"horseback":19926,"##while":19927,"goodwin":19928,"##jos":19929,"gaa":19930,"norte":19931,"##ldon":19932,"modify":19933,"wavelength":19934,"abd":19935,"214":19936,"skinned":19937,"sprinter":19938,"forecast":19939,"scheduling":19940,"marries":19941,"squared":19942,"tentative":19943,"##chman":19944,"boer":19945,"##isch":19946,"bolts":19947,"swap":19948,"fisherman":19949,"assyrian":19950,"impatiently":19951,"guthrie":19952,"martins":19953,"murdoch":19954,"194":19955,"tanya":19956,"nicely":19957,"dolly":19958,"lacy":19959,"med":19960,"##45":19961,"syn":19962,"decks":19963,"fashionable":19964,"millionaire":19965,"##ust":19966,"surfing":19967,"##ml":19968,"##ision":19969,"heaved":19970,"tammy":19971,"consulate":19972,"attendees":19973,"routinely":19974,"197":19975,"fuse":19976,"saxophonist":19977,"backseat":19978,"malaya":19979,"##lord":19980,"scowl":19981,"tau":19982,"##ishly":19983,"193":19984,"sighted":19985,"steaming":19986,"##rks":19987,"303":19988,"911":19989,"##holes":19990,"##hong":19991,"ching":19992,"##wife":19993,"bless":19994,"conserved":19995,"jurassic":19996,"stacey":19997,"unix":19998,"zion":19999,"chunk":20000,"rigorous":20001,"blaine":20002,"198":20003,"peabody":20004,"slayer":20005,"dismay":20006,"brewers":20007,"nz":20008,"##jer":20009,"det":20010,"##glia":20011,"glover":20012,"postwar":20013,"int":20014,"penetration":20015,"sylvester":20016,"imitation":20017,"vertically":20018,"airlift":20019,"heiress":20020,"knoxville":20021,"viva":20022,"##uin":20023,"390":20024,"macon":20025,"##rim":20026,"##fighter":20027,"##gonal":20028,"janice":20029,"##orescence":20030,"##wari":20031,"marius":20032,"belongings":20033,"leicestershire":20034,"196":20035,"blanco":20036,"inverted":20037,"preseason":20038,"sanity":20039,"sobbing":20040,"##due":20041,"##elt":20042,"##dled":20043,"collingwood":20044,"regeneration":20045,"flickering":20046,"shortest":20047,"##mount":20048,"##osi":20049,"feminism":20050,"##lat":20051,"sherlock":20052,"cabinets":20053,"fumbled":20054,"northbound":20055,"precedent":20056,"snaps":20057,"##mme":20058,"researching":20059,"##akes":20060,"guillaume":20061,"insights":20062,"manipulated":20063,"vapor":20064,"neighbour":20065,"sap":20066,"gangster":20067,"frey":20068,"f1":20069,"stalking":20070,"scarcely":20071,"callie":20072,"barnett":20073,"tendencies":20074,"audi":20075,"doomed":20076,"assessing":20077,"slung":20078,"panchayat":20079,"ambiguous":20080,"bartlett":20081,"##etto":20082,"distributing":20083,"violating":20084,"wolverhampton":20085,"##hetic":20086,"swami":20087,"histoire":20088,"##urus":20089,"liable":20090,"pounder":20091,"groin":20092,"hussain":20093,"larsen":20094,"popping":20095,"surprises":20096,"##atter":20097,"vie":20098,"curt":20099,"##station":20100,"mute":20101,"relocate":20102,"musicals":20103,"authorization":20104,"richter":20105,"##sef":20106,"immortality":20107,"tna":20108,"bombings":20109,"##press":20110,"deteriorated":20111,"yiddish":20112,"##acious":20113,"robbed":20114,"colchester":20115,"cs":20116,"pmid":20117,"ao":20118,"verified":20119,"balancing":20120,"apostle":20121,"swayed":20122,"recognizable":20123,"oxfordshire":20124,"retention":20125,"nottinghamshire":20126,"contender":20127,"judd":20128,"invitational":20129,"shrimp":20130,"uhf":20131,"##icient":20132,"cleaner":20133,"longitudinal":20134,"tanker":20135,"##mur":20136,"acronym":20137,"broker":20138,"koppen":20139,"sundance":20140,"suppliers":20141,"##gil":20142,"4000":20143,"clipped":20144,"fuels":20145,"petite":20146,"##anne":20147,"landslide":20148,"helene":20149,"diversion":20150,"populous":20151,"landowners":20152,"auspices":20153,"melville":20154,"quantitative":20155,"##xes":20156,"ferries":20157,"nicky":20158,"##llus":20159,"doo":20160,"haunting":20161,"roche":20162,"carver":20163,"downed":20164,"unavailable":20165,"##pathy":20166,"approximation":20167,"hiroshima":20168,"##hue":20169,"garfield":20170,"valle":20171,"comparatively":20172,"keyboardist":20173,"traveler":20174,"##eit":20175,"congestion":20176,"calculating":20177,"subsidiaries":20178,"##bate":20179,"serb":20180,"modernization":20181,"fairies":20182,"deepened":20183,"ville":20184,"averages":20185,"##lore":20186,"inflammatory":20187,"tonga":20188,"##itch":20189,"co₂":20190,"squads":20191,"##hea":20192,"gigantic":20193,"serum":20194,"enjoyment":20195,"retailer":20196,"verona":20197,"35th":20198,"cis":20199,"##phobic":20200,"magna":20201,"technicians":20202,"##vati":20203,"arithmetic":20204,"##sport":20205,"levin":20206,"##dation":20207,"amtrak":20208,"chow":20209,"sienna":20210,"##eyer":20211,"backstage":20212,"entrepreneurship":20213,"##otic":20214,"learnt":20215,"tao":20216,"##udy":20217,"worcestershire":20218,"formulation":20219,"baggage":20220,"hesitant":20221,"bali":20222,"sabotage":20223,"##kari":20224,"barren":20225,"enhancing":20226,"murmur":20227,"pl":20228,"freshly":20229,"putnam":20230,"syntax":20231,"aces":20232,"medicines":20233,"resentment":20234,"bandwidth":20235,"##sier":20236,"grins":20237,"chili":20238,"guido":20239,"##sei":20240,"framing":20241,"implying":20242,"gareth":20243,"lissa":20244,"genevieve":20245,"pertaining":20246,"admissions":20247,"geo":20248,"thorpe":20249,"proliferation":20250,"sato":20251,"bela":20252,"analyzing":20253,"parting":20254,"##gor":20255,"awakened":20256,"##isman":20257,"huddled":20258,"secrecy":20259,"##kling":20260,"hush":20261,"gentry":20262,"540":20263,"dungeons":20264,"##ego":20265,"coasts":20266,"##utz":20267,"sacrificed":20268,"##chule":20269,"landowner":20270,"mutually":20271,"prevalence":20272,"programmer":20273,"adolescent":20274,"disrupted":20275,"seaside":20276,"gee":20277,"trusts":20278,"vamp":20279,"georgie":20280,"##nesian":20281,"##iol":20282,"schedules":20283,"sindh":20284,"##market":20285,"etched":20286,"hm":20287,"sparse":20288,"bey":20289,"beaux":20290,"scratching":20291,"gliding":20292,"unidentified":20293,"216":20294,"collaborating":20295,"gems":20296,"jesuits":20297,"oro":20298,"accumulation":20299,"shaping":20300,"mbe":20301,"anal":20302,"##xin":20303,"231":20304,"enthusiasts":20305,"newscast":20306,"##egan":20307,"janata":20308,"dewey":20309,"parkinson":20310,"179":20311,"ankara":20312,"biennial":20313,"towering":20314,"dd":20315,"inconsistent":20316,"950":20317,"##chet":20318,"thriving":20319,"terminate":20320,"cabins":20321,"furiously":20322,"eats":20323,"advocating":20324,"donkey":20325,"marley":20326,"muster":20327,"phyllis":20328,"leiden":20329,"##user":20330,"grassland":20331,"glittering":20332,"iucn":20333,"loneliness":20334,"217":20335,"memorandum":20336,"armenians":20337,"##ddle":20338,"popularized":20339,"rhodesia":20340,"60s":20341,"lame":20342,"##illon":20343,"sans":20344,"bikini":20345,"header":20346,"orbits":20347,"##xx":20348,"##finger":20349,"##ulator":20350,"sharif":20351,"spines":20352,"biotechnology":20353,"strolled":20354,"naughty":20355,"yates":20356,"##wire":20357,"fremantle":20358,"milo":20359,"##mour":20360,"abducted":20361,"removes":20362,"##atin":20363,"humming":20364,"wonderland":20365,"##chrome":20366,"##ester":20367,"hume":20368,"pivotal":20369,"##rates":20370,"armand":20371,"grams":20372,"believers":20373,"elector":20374,"rte":20375,"apron":20376,"bis":20377,"scraped":20378,"##yria":20379,"endorsement":20380,"initials":20381,"##llation":20382,"eps":20383,"dotted":20384,"hints":20385,"buzzing":20386,"emigration":20387,"nearer":20388,"##tom":20389,"indicators":20390,"##ulu":20391,"coarse":20392,"neutron":20393,"protectorate":20394,"##uze":20395,"directional":20396,"exploits":20397,"pains":20398,"loire":20399,"1830s":20400,"proponents":20401,"guggenheim":20402,"rabbits":20403,"ritchie":20404,"305":20405,"hectare":20406,"inputs":20407,"hutton":20408,"##raz":20409,"verify":20410,"##ako":20411,"boilers":20412,"longitude":20413,"##lev":20414,"skeletal":20415,"yer":20416,"emilia":20417,"citrus":20418,"compromised":20419,"##gau":20420,"pokemon":20421,"prescription":20422,"paragraph":20423,"eduard":20424,"cadillac":20425,"attire":20426,"categorized":20427,"kenyan":20428,"weddings":20429,"charley":20430,"##bourg":20431,"entertain":20432,"monmouth":20433,"##lles":20434,"nutrients":20435,"davey":20436,"mesh":20437,"incentive":20438,"practised":20439,"ecosystems":20440,"kemp":20441,"subdued":20442,"overheard":20443,"##rya":20444,"bodily":20445,"maxim":20446,"##nius":20447,"apprenticeship":20448,"ursula":20449,"##fight":20450,"lodged":20451,"rug":20452,"silesian":20453,"unconstitutional":20454,"patel":20455,"inspected":20456,"coyote":20457,"unbeaten":20458,"##hak":20459,"34th":20460,"disruption":20461,"convict":20462,"parcel":20463,"##cl":20464,"##nham":20465,"collier":20466,"implicated":20467,"mallory":20468,"##iac":20469,"##lab":20470,"susannah":20471,"winkler":20472,"##rber":20473,"shia":20474,"phelps":20475,"sediments":20476,"graphical":20477,"robotic":20478,"##sner":20479,"adulthood":20480,"mart":20481,"smoked":20482,"##isto":20483,"kathryn":20484,"clarified":20485,"##aran":20486,"divides":20487,"convictions":20488,"oppression":20489,"pausing":20490,"burying":20491,"##mt":20492,"federico":20493,"mathias":20494,"eileen":20495,"##tana":20496,"kite":20497,"hunched":20498,"##acies":20499,"189":20500,"##atz":20501,"disadvantage":20502,"liza":20503,"kinetic":20504,"greedy":20505,"paradox":20506,"yokohama":20507,"dowager":20508,"trunks":20509,"ventured":20510,"##gement":20511,"gupta":20512,"vilnius":20513,"olaf":20514,"##thest":20515,"crimean":20516,"hopper":20517,"##ej":20518,"progressively":20519,"arturo":20520,"mouthed":20521,"arrondissement":20522,"##fusion":20523,"rubin":20524,"simulcast":20525,"oceania":20526,"##orum":20527,"##stra":20528,"##rred":20529,"busiest":20530,"intensely":20531,"navigator":20532,"cary":20533,"##vine":20534,"##hini":20535,"##bies":20536,"fife":20537,"rowe":20538,"rowland":20539,"posing":20540,"insurgents":20541,"shafts":20542,"lawsuits":20543,"activate":20544,"conor":20545,"inward":20546,"culturally":20547,"garlic":20548,"265":20549,"##eering":20550,"eclectic":20551,"##hui":20552,"##kee":20553,"##nl":20554,"furrowed":20555,"vargas":20556,"meteorological":20557,"rendezvous":20558,"##aus":20559,"culinary":20560,"commencement":20561,"##dition":20562,"quota":20563,"##notes":20564,"mommy":20565,"salaries":20566,"overlapping":20567,"mule":20568,"##iology":20569,"##mology":20570,"sums":20571,"wentworth":20572,"##isk":20573,"##zione":20574,"mainline":20575,"subgroup":20576,"##illy":20577,"hack":20578,"plaintiff":20579,"verdi":20580,"bulb":20581,"differentiation":20582,"engagements":20583,"multinational":20584,"supplemented":20585,"bertrand":20586,"caller":20587,"regis":20588,"##naire":20589,"##sler":20590,"##arts":20591,"##imated":20592,"blossom":20593,"propagation":20594,"kilometer":20595,"viaduct":20596,"vineyards":20597,"##uate":20598,"beckett":20599,"optimization":20600,"golfer":20601,"songwriters":20602,"seminal":20603,"semitic":20604,"thud":20605,"volatile":20606,"evolving":20607,"ridley":20608,"##wley":20609,"trivial":20610,"distributions":20611,"scandinavia":20612,"jiang":20613,"##ject":20614,"wrestled":20615,"insistence":20616,"##dio":20617,"emphasizes":20618,"napkin":20619,"##ods":20620,"adjunct":20621,"rhyme":20622,"##ricted":20623,"##eti":20624,"hopeless":20625,"surrounds":20626,"tremble":20627,"32nd":20628,"smoky":20629,"##ntly":20630,"oils":20631,"medicinal":20632,"padded":20633,"steer":20634,"wilkes":20635,"219":20636,"255":20637,"concessions":20638,"hue":20639,"uniquely":20640,"blinded":20641,"landon":20642,"yahoo":20643,"##lane":20644,"hendrix":20645,"commemorating":20646,"dex":20647,"specify":20648,"chicks":20649,"##ggio":20650,"intercity":20651,"1400":20652,"morley":20653,"##torm":20654,"highlighting":20655,"##oting":20656,"pang":20657,"oblique":20658,"stalled":20659,"##liner":20660,"flirting":20661,"newborn":20662,"1769":20663,"bishopric":20664,"shaved":20665,"232":20666,"currie":20667,"##ush":20668,"dharma":20669,"spartan":20670,"##ooped":20671,"favorites":20672,"smug":20673,"novella":20674,"sirens":20675,"abusive":20676,"creations":20677,"espana":20678,"##lage":20679,"paradigm":20680,"semiconductor":20681,"sheen":20682,"##rdo":20683,"##yen":20684,"##zak":20685,"nrl":20686,"renew":20687,"##pose":20688,"##tur":20689,"adjutant":20690,"marches":20691,"norma":20692,"##enity":20693,"ineffective":20694,"weimar":20695,"grunt":20696,"##gat":20697,"lordship":20698,"plotting":20699,"expenditure":20700,"infringement":20701,"lbs":20702,"refrain":20703,"av":20704,"mimi":20705,"mistakenly":20706,"postmaster":20707,"1771":20708,"##bara":20709,"ras":20710,"motorsports":20711,"tito":20712,"199":20713,"subjective":20714,"##zza":20715,"bully":20716,"stew":20717,"##kaya":20718,"prescott":20719,"1a":20720,"##raphic":20721,"##zam":20722,"bids":20723,"styling":20724,"paranormal":20725,"reeve":20726,"sneaking":20727,"exploding":20728,"katz":20729,"akbar":20730,"migrant":20731,"syllables":20732,"indefinitely":20733,"##ogical":20734,"destroys":20735,"replaces":20736,"applause":20737,"##phine":20738,"pest":20739,"##fide":20740,"218":20741,"articulated":20742,"bertie":20743,"##thing":20744,"##cars":20745,"##ptic":20746,"courtroom":20747,"crowley":20748,"aesthetics":20749,"cummings":20750,"tehsil":20751,"hormones":20752,"titanic":20753,"dangerously":20754,"##ibe":20755,"stadion":20756,"jaenelle":20757,"auguste":20758,"ciudad":20759,"##chu":20760,"mysore":20761,"partisans":20762,"##sio":20763,"lucan":20764,"philipp":20765,"##aly":20766,"debating":20767,"henley":20768,"interiors":20769,"##rano":20770,"##tious":20771,"homecoming":20772,"beyonce":20773,"usher":20774,"henrietta":20775,"prepares":20776,"weeds":20777,"##oman":20778,"ely":20779,"plucked":20780,"##pire":20781,"##dable":20782,"luxurious":20783,"##aq":20784,"artifact":20785,"password":20786,"pasture":20787,"juno":20788,"maddy":20789,"minsk":20790,"##dder":20791,"##ologies":20792,"##rone":20793,"assessments":20794,"martian":20795,"royalist":20796,"1765":20797,"examines":20798,"##mani":20799,"##rge":20800,"nino":20801,"223":20802,"parry":20803,"scooped":20804,"relativity":20805,"##eli":20806,"##uting":20807,"##cao":20808,"congregational":20809,"noisy":20810,"traverse":20811,"##agawa":20812,"strikeouts":20813,"nickelodeon":20814,"obituary":20815,"transylvania":20816,"binds":20817,"depictions":20818,"polk":20819,"trolley":20820,"##yed":20821,"##lard":20822,"breeders":20823,"##under":20824,"dryly":20825,"hokkaido":20826,"1762":20827,"strengths":20828,"stacks":20829,"bonaparte":20830,"connectivity":20831,"neared":20832,"prostitutes":20833,"stamped":20834,"anaheim":20835,"gutierrez":20836,"sinai":20837,"##zzling":20838,"bram":20839,"fresno":20840,"madhya":20841,"##86":20842,"proton":20843,"##lena":20844,"##llum":20845,"##phon":20846,"reelected":20847,"wanda":20848,"##anus":20849,"##lb":20850,"ample":20851,"distinguishing":20852,"##yler":20853,"grasping":20854,"sermons":20855,"tomato":20856,"bland":20857,"stimulation":20858,"avenues":20859,"##eux":20860,"spreads":20861,"scarlett":20862,"fern":20863,"pentagon":20864,"assert":20865,"baird":20866,"chesapeake":20867,"ir":20868,"calmed":20869,"distortion":20870,"fatalities":20871,"##olis":20872,"correctional":20873,"pricing":20874,"##astic":20875,"##gina":20876,"prom":20877,"dammit":20878,"ying":20879,"collaborate":20880,"##chia":20881,"welterweight":20882,"33rd":20883,"pointer":20884,"substitution":20885,"bonded":20886,"umpire":20887,"communicating":20888,"multitude":20889,"paddle":20890,"##obe":20891,"federally":20892,"intimacy":20893,"##insky":20894,"betray":20895,"ssr":20896,"##lett":20897,"##lean":20898,"##lves":20899,"##therapy":20900,"airbus":20901,"##tery":20902,"functioned":20903,"ud":20904,"bearer":20905,"biomedical":20906,"netflix":20907,"##hire":20908,"##nca":20909,"condom":20910,"brink":20911,"ik":20912,"##nical":20913,"macy":20914,"##bet":20915,"flap":20916,"gma":20917,"experimented":20918,"jelly":20919,"lavender":20920,"##icles":20921,"##ulia":20922,"munro":20923,"##mian":20924,"##tial":20925,"rye":20926,"##rle":20927,"60th":20928,"gigs":20929,"hottest":20930,"rotated":20931,"predictions":20932,"fuji":20933,"bu":20934,"##erence":20935,"##omi":20936,"barangay":20937,"##fulness":20938,"##sas":20939,"clocks":20940,"##rwood":20941,"##liness":20942,"cereal":20943,"roe":20944,"wight":20945,"decker":20946,"uttered":20947,"babu":20948,"onion":20949,"xml":20950,"forcibly":20951,"##df":20952,"petra":20953,"sarcasm":20954,"hartley":20955,"peeled":20956,"storytelling":20957,"##42":20958,"##xley":20959,"##ysis":20960,"##ffa":20961,"fibre":20962,"kiel":20963,"auditor":20964,"fig":20965,"harald":20966,"greenville":20967,"##berries":20968,"geographically":20969,"nell":20970,"quartz":20971,"##athic":20972,"cemeteries":20973,"##lr":20974,"crossings":20975,"nah":20976,"holloway":20977,"reptiles":20978,"chun":20979,"sichuan":20980,"snowy":20981,"660":20982,"corrections":20983,"##ivo":20984,"zheng":20985,"ambassadors":20986,"blacksmith":20987,"fielded":20988,"fluids":20989,"hardcover":20990,"turnover":20991,"medications":20992,"melvin":20993,"academies":20994,"##erton":20995,"ro":20996,"roach":20997,"absorbing":20998,"spaniards":20999,"colton":21000,"##founded":21001,"outsider":21002,"espionage":21003,"kelsey":21004,"245":21005,"edible":21006,"##ulf":21007,"dora":21008,"establishes":21009,"##sham":21010,"##tries":21011,"contracting":21012,"##tania":21013,"cinematic":21014,"costello":21015,"nesting":21016,"##uron":21017,"connolly":21018,"duff":21019,"##nology":21020,"mma":21021,"##mata":21022,"fergus":21023,"sexes":21024,"gi":21025,"optics":21026,"spectator":21027,"woodstock":21028,"banning":21029,"##hee":21030,"##fle":21031,"differentiate":21032,"outfielder":21033,"refinery":21034,"226":21035,"312":21036,"gerhard":21037,"horde":21038,"lair":21039,"drastically":21040,"##udi":21041,"landfall":21042,"##cheng":21043,"motorsport":21044,"odi":21045,"##achi":21046,"predominant":21047,"quay":21048,"skins":21049,"##ental":21050,"edna":21051,"harshly":21052,"complementary":21053,"murdering":21054,"##aves":21055,"wreckage":21056,"##90":21057,"ono":21058,"outstretched":21059,"lennox":21060,"munitions":21061,"galen":21062,"reconcile":21063,"470":21064,"scalp":21065,"bicycles":21066,"gillespie":21067,"questionable":21068,"rosenberg":21069,"guillermo":21070,"hostel":21071,"jarvis":21072,"kabul":21073,"volvo":21074,"opium":21075,"yd":21076,"##twined":21077,"abuses":21078,"decca":21079,"outpost":21080,"##cino":21081,"sensible":21082,"neutrality":21083,"##64":21084,"ponce":21085,"anchorage":21086,"atkins":21087,"turrets":21088,"inadvertently":21089,"disagree":21090,"libre":21091,"vodka":21092,"reassuring":21093,"weighs":21094,"##yal":21095,"glide":21096,"jumper":21097,"ceilings":21098,"repertory":21099,"outs":21100,"stain":21101,"##bial":21102,"envy":21103,"##ucible":21104,"smashing":21105,"heightened":21106,"policing":21107,"hyun":21108,"mixes":21109,"lai":21110,"prima":21111,"##ples":21112,"celeste":21113,"##bina":21114,"lucrative":21115,"intervened":21116,"kc":21117,"manually":21118,"##rned":21119,"stature":21120,"staffed":21121,"bun":21122,"bastards":21123,"nairobi":21124,"priced":21125,"##auer":21126,"thatcher":21127,"##kia":21128,"tripped":21129,"comune":21130,"##ogan":21131,"##pled":21132,"brasil":21133,"incentives":21134,"emanuel":21135,"hereford":21136,"musica":21137,"##kim":21138,"benedictine":21139,"biennale":21140,"##lani":21141,"eureka":21142,"gardiner":21143,"rb":21144,"knocks":21145,"sha":21146,"##ael":21147,"##elled":21148,"##onate":21149,"efficacy":21150,"ventura":21151,"masonic":21152,"sanford":21153,"maize":21154,"leverage":21155,"##feit":21156,"capacities":21157,"santana":21158,"##aur":21159,"novelty":21160,"vanilla":21161,"##cter":21162,"##tour":21163,"benin":21164,"##oir":21165,"##rain":21166,"neptune":21167,"drafting":21168,"tallinn":21169,"##cable":21170,"humiliation":21171,"##boarding":21172,"schleswig":21173,"fabian":21174,"bernardo":21175,"liturgy":21176,"spectacle":21177,"sweeney":21178,"pont":21179,"routledge":21180,"##tment":21181,"cosmos":21182,"ut":21183,"hilt":21184,"sleek":21185,"universally":21186,"##eville":21187,"##gawa":21188,"typed":21189,"##dry":21190,"favors":21191,"allegheny":21192,"glaciers":21193,"##rly":21194,"recalling":21195,"aziz":21196,"##log":21197,"parasite":21198,"requiem":21199,"auf":21200,"##berto":21201,"##llin":21202,"illumination":21203,"##breaker":21204,"##issa":21205,"festivities":21206,"bows":21207,"govern":21208,"vibe":21209,"vp":21210,"333":21211,"sprawled":21212,"larson":21213,"pilgrim":21214,"bwf":21215,"leaping":21216,"##rts":21217,"##ssel":21218,"alexei":21219,"greyhound":21220,"hoarse":21221,"##dler":21222,"##oration":21223,"seneca":21224,"##cule":21225,"gaping":21226,"##ulously":21227,"##pura":21228,"cinnamon":21229,"##gens":21230,"##rricular":21231,"craven":21232,"fantasies":21233,"houghton":21234,"engined":21235,"reigned":21236,"dictator":21237,"supervising":21238,"##oris":21239,"bogota":21240,"commentaries":21241,"unnatural":21242,"fingernails":21243,"spirituality":21244,"tighten":21245,"##tm":21246,"canadiens":21247,"protesting":21248,"intentional":21249,"cheers":21250,"sparta":21251,"##ytic":21252,"##iere":21253,"##zine":21254,"widen":21255,"belgarath":21256,"controllers":21257,"dodd":21258,"iaaf":21259,"navarre":21260,"##ication":21261,"defect":21262,"squire":21263,"steiner":21264,"whisky":21265,"##mins":21266,"560":21267,"inevitably":21268,"tome":21269,"##gold":21270,"chew":21271,"##uid":21272,"##lid":21273,"elastic":21274,"##aby":21275,"streaked":21276,"alliances":21277,"jailed":21278,"regal":21279,"##ined":21280,"##phy":21281,"czechoslovak":21282,"narration":21283,"absently":21284,"##uld":21285,"bluegrass":21286,"guangdong":21287,"quran":21288,"criticizing":21289,"hose":21290,"hari":21291,"##liest":21292,"##owa":21293,"skier":21294,"streaks":21295,"deploy":21296,"##lom":21297,"raft":21298,"bose":21299,"dialed":21300,"huff":21301,"##eira":21302,"haifa":21303,"simplest":21304,"bursting":21305,"endings":21306,"ib":21307,"sultanate":21308,"##titled":21309,"franks":21310,"whitman":21311,"ensures":21312,"sven":21313,"##ggs":21314,"collaborators":21315,"forster":21316,"organising":21317,"ui":21318,"banished":21319,"napier":21320,"injustice":21321,"teller":21322,"layered":21323,"thump":21324,"##otti":21325,"roc":21326,"battleships":21327,"evidenced":21328,"fugitive":21329,"sadie":21330,"robotics":21331,"##roud":21332,"equatorial":21333,"geologist":21334,"##iza":21335,"yielding":21336,"##bron":21337,"##sr":21338,"internationale":21339,"mecca":21340,"##diment":21341,"sbs":21342,"skyline":21343,"toad":21344,"uploaded":21345,"reflective":21346,"undrafted":21347,"lal":21348,"leafs":21349,"bayern":21350,"##dai":21351,"lakshmi":21352,"shortlisted":21353,"##stick":21354,"##wicz":21355,"camouflage":21356,"donate":21357,"af":21358,"christi":21359,"lau":21360,"##acio":21361,"disclosed":21362,"nemesis":21363,"1761":21364,"assemble":21365,"straining":21366,"northamptonshire":21367,"tal":21368,"##asi":21369,"bernardino":21370,"premature":21371,"heidi":21372,"42nd":21373,"coefficients":21374,"galactic":21375,"reproduce":21376,"buzzed":21377,"sensations":21378,"zionist":21379,"monsieur":21380,"myrtle":21381,"##eme":21382,"archery":21383,"strangled":21384,"musically":21385,"viewpoint":21386,"antiquities":21387,"bei":21388,"trailers":21389,"seahawks":21390,"cured":21391,"pee":21392,"preferring":21393,"tasmanian":21394,"lange":21395,"sul":21396,"##mail":21397,"##working":21398,"colder":21399,"overland":21400,"lucivar":21401,"massey":21402,"gatherings":21403,"haitian":21404,"##smith":21405,"disapproval":21406,"flaws":21407,"##cco":21408,"##enbach":21409,"1766":21410,"npr":21411,"##icular":21412,"boroughs":21413,"creole":21414,"forums":21415,"techno":21416,"1755":21417,"dent":21418,"abdominal":21419,"streetcar":21420,"##eson":21421,"##stream":21422,"procurement":21423,"gemini":21424,"predictable":21425,"##tya":21426,"acheron":21427,"christoph":21428,"feeder":21429,"fronts":21430,"vendor":21431,"bernhard":21432,"jammu":21433,"tumors":21434,"slang":21435,"##uber":21436,"goaltender":21437,"twists":21438,"curving":21439,"manson":21440,"vuelta":21441,"mer":21442,"peanut":21443,"confessions":21444,"pouch":21445,"unpredictable":21446,"allowance":21447,"theodor":21448,"vascular":21449,"##factory":21450,"bala":21451,"authenticity":21452,"metabolic":21453,"coughing":21454,"nanjing":21455,"##cea":21456,"pembroke":21457,"##bard":21458,"splendid":21459,"36th":21460,"ff":21461,"hourly":21462,"##ahu":21463,"elmer":21464,"handel":21465,"##ivate":21466,"awarding":21467,"thrusting":21468,"dl":21469,"experimentation":21470,"##hesion":21471,"##46":21472,"caressed":21473,"entertained":21474,"steak":21475,"##rangle":21476,"biologist":21477,"orphans":21478,"baroness":21479,"oyster":21480,"stepfather":21481,"##dridge":21482,"mirage":21483,"reefs":21484,"speeding":21485,"##31":21486,"barons":21487,"1764":21488,"227":21489,"inhabit":21490,"preached":21491,"repealed":21492,"##tral":21493,"honoring":21494,"boogie":21495,"captives":21496,"administer":21497,"johanna":21498,"##imate":21499,"gel":21500,"suspiciously":21501,"1767":21502,"sobs":21503,"##dington":21504,"backbone":21505,"hayward":21506,"garry":21507,"##folding":21508,"##nesia":21509,"maxi":21510,"##oof":21511,"##ppe":21512,"ellison":21513,"galileo":21514,"##stand":21515,"crimea":21516,"frenzy":21517,"amour":21518,"bumper":21519,"matrices":21520,"natalia":21521,"baking":21522,"garth":21523,"palestinians":21524,"##grove":21525,"smack":21526,"conveyed":21527,"ensembles":21528,"gardening":21529,"##manship":21530,"##rup":21531,"##stituting":21532,"1640":21533,"harvesting":21534,"topography":21535,"jing":21536,"shifters":21537,"dormitory":21538,"##carriage":21539,"##lston":21540,"ist":21541,"skulls":21542,"##stadt":21543,"dolores":21544,"jewellery":21545,"sarawak":21546,"##wai":21547,"##zier":21548,"fences":21549,"christy":21550,"confinement":21551,"tumbling":21552,"credibility":21553,"fir":21554,"stench":21555,"##bria":21556,"##plication":21557,"##nged":21558,"##sam":21559,"virtues":21560,"##belt":21561,"marjorie":21562,"pba":21563,"##eem":21564,"##made":21565,"celebrates":21566,"schooner":21567,"agitated":21568,"barley":21569,"fulfilling":21570,"anthropologist":21571,"##pro":21572,"restrict":21573,"novi":21574,"regulating":21575,"##nent":21576,"padres":21577,"##rani":21578,"##hesive":21579,"loyola":21580,"tabitha":21581,"milky":21582,"olson":21583,"proprietor":21584,"crambidae":21585,"guarantees":21586,"intercollegiate":21587,"ljubljana":21588,"hilda":21589,"##sko":21590,"ignorant":21591,"hooded":21592,"##lts":21593,"sardinia":21594,"##lidae":21595,"##vation":21596,"frontman":21597,"privileged":21598,"witchcraft":21599,"##gp":21600,"jammed":21601,"laude":21602,"poking":21603,"##than":21604,"bracket":21605,"amazement":21606,"yunnan":21607,"##erus":21608,"maharaja":21609,"linnaeus":21610,"264":21611,"commissioning":21612,"milano":21613,"peacefully":21614,"##logies":21615,"akira":21616,"rani":21617,"regulator":21618,"##36":21619,"grasses":21620,"##rance":21621,"luzon":21622,"crows":21623,"compiler":21624,"gretchen":21625,"seaman":21626,"edouard":21627,"tab":21628,"buccaneers":21629,"ellington":21630,"hamlets":21631,"whig":21632,"socialists":21633,"##anto":21634,"directorial":21635,"easton":21636,"mythological":21637,"##kr":21638,"##vary":21639,"rhineland":21640,"semantic":21641,"taut":21642,"dune":21643,"inventions":21644,"succeeds":21645,"##iter":21646,"replication":21647,"branched":21648,"##pired":21649,"jul":21650,"prosecuted":21651,"kangaroo":21652,"penetrated":21653,"##avian":21654,"middlesbrough":21655,"doses":21656,"bleak":21657,"madam":21658,"predatory":21659,"relentless":21660,"##vili":21661,"reluctance":21662,"##vir":21663,"hailey":21664,"crore":21665,"silvery":21666,"1759":21667,"monstrous":21668,"swimmers":21669,"transmissions":21670,"hawthorn":21671,"informing":21672,"##eral":21673,"toilets":21674,"caracas":21675,"crouch":21676,"kb":21677,"##sett":21678,"295":21679,"cartel":21680,"hadley":21681,"##aling":21682,"alexia":21683,"yvonne":21684,"##biology":21685,"cinderella":21686,"eton":21687,"superb":21688,"blizzard":21689,"stabbing":21690,"industrialist":21691,"maximus":21692,"##gm":21693,"##orus":21694,"groves":21695,"maud":21696,"clade":21697,"oversized":21698,"comedic":21699,"##bella":21700,"rosen":21701,"nomadic":21702,"fulham":21703,"montane":21704,"beverages":21705,"galaxies":21706,"redundant":21707,"swarm":21708,"##rot":21709,"##folia":21710,"##llis":21711,"buckinghamshire":21712,"fen":21713,"bearings":21714,"bahadur":21715,"##rom":21716,"gilles":21717,"phased":21718,"dynamite":21719,"faber":21720,"benoit":21721,"vip":21722,"##ount":21723,"##wd":21724,"booking":21725,"fractured":21726,"tailored":21727,"anya":21728,"spices":21729,"westwood":21730,"cairns":21731,"auditions":21732,"inflammation":21733,"steamed":21734,"##rocity":21735,"##acion":21736,"##urne":21737,"skyla":21738,"thereof":21739,"watford":21740,"torment":21741,"archdeacon":21742,"transforms":21743,"lulu":21744,"demeanor":21745,"fucked":21746,"serge":21747,"##sor":21748,"mckenna":21749,"minas":21750,"entertainer":21751,"##icide":21752,"caress":21753,"originate":21754,"residue":21755,"##sty":21756,"1740":21757,"##ilised":21758,"##org":21759,"beech":21760,"##wana":21761,"subsidies":21762,"##ghton":21763,"emptied":21764,"gladstone":21765,"ru":21766,"firefighters":21767,"voodoo":21768,"##rcle":21769,"het":21770,"nightingale":21771,"tamara":21772,"edmond":21773,"ingredient":21774,"weaknesses":21775,"silhouette":21776,"285":21777,"compatibility":21778,"withdrawing":21779,"hampson":21780,"##mona":21781,"anguish":21782,"giggling":21783,"##mber":21784,"bookstore":21785,"##jiang":21786,"southernmost":21787,"tilting":21788,"##vance":21789,"bai":21790,"economical":21791,"rf":21792,"briefcase":21793,"dreadful":21794,"hinted":21795,"projections":21796,"shattering":21797,"totaling":21798,"##rogate":21799,"analogue":21800,"indicted":21801,"periodical":21802,"fullback":21803,"##dman":21804,"haynes":21805,"##tenberg":21806,"##ffs":21807,"##ishment":21808,"1745":21809,"thirst":21810,"stumble":21811,"penang":21812,"vigorous":21813,"##ddling":21814,"##kor":21815,"##lium":21816,"octave":21817,"##ove":21818,"##enstein":21819,"##inen":21820,"##ones":21821,"siberian":21822,"##uti":21823,"cbn":21824,"repeal":21825,"swaying":21826,"##vington":21827,"khalid":21828,"tanaka":21829,"unicorn":21830,"otago":21831,"plastered":21832,"lobe":21833,"riddle":21834,"##rella":21835,"perch":21836,"##ishing":21837,"croydon":21838,"filtered":21839,"graeme":21840,"tripoli":21841,"##ossa":21842,"crocodile":21843,"##chers":21844,"sufi":21845,"mined":21846,"##tung":21847,"inferno":21848,"lsu":21849,"##phi":21850,"swelled":21851,"utilizes":21852,"£2":21853,"cale":21854,"periodicals":21855,"styx":21856,"hike":21857,"informally":21858,"coop":21859,"lund":21860,"##tidae":21861,"ala":21862,"hen":21863,"qui":21864,"transformations":21865,"disposed":21866,"sheath":21867,"chickens":21868,"##cade":21869,"fitzroy":21870,"sas":21871,"silesia":21872,"unacceptable":21873,"odisha":21874,"1650":21875,"sabrina":21876,"pe":21877,"spokane":21878,"ratios":21879,"athena":21880,"massage":21881,"shen":21882,"dilemma":21883,"##drum":21884,"##riz":21885,"##hul":21886,"corona":21887,"doubtful":21888,"niall":21889,"##pha":21890,"##bino":21891,"fines":21892,"cite":21893,"acknowledging":21894,"bangor":21895,"ballard":21896,"bathurst":21897,"##resh":21898,"huron":21899,"mustered":21900,"alzheimer":21901,"garments":21902,"kinase":21903,"tyre":21904,"warship":21905,"##cp":21906,"flashback":21907,"pulmonary":21908,"braun":21909,"cheat":21910,"kamal":21911,"cyclists":21912,"constructions":21913,"grenades":21914,"ndp":21915,"traveller":21916,"excuses":21917,"stomped":21918,"signalling":21919,"trimmed":21920,"futsal":21921,"mosques":21922,"relevance":21923,"##wine":21924,"wta":21925,"##23":21926,"##vah":21927,"##lter":21928,"hoc":21929,"##riding":21930,"optimistic":21931,"##´s":21932,"deco":21933,"sim":21934,"interacting":21935,"rejecting":21936,"moniker":21937,"waterways":21938,"##ieri":21939,"##oku":21940,"mayors":21941,"gdansk":21942,"outnumbered":21943,"pearls":21944,"##ended":21945,"##hampton":21946,"fairs":21947,"totals":21948,"dominating":21949,"262":21950,"notions":21951,"stairway":21952,"compiling":21953,"pursed":21954,"commodities":21955,"grease":21956,"yeast":21957,"##jong":21958,"carthage":21959,"griffiths":21960,"residual":21961,"amc":21962,"contraction":21963,"laird":21964,"sapphire":21965,"##marine":21966,"##ivated":21967,"amalgamation":21968,"dissolve":21969,"inclination":21970,"lyle":21971,"packaged":21972,"altitudes":21973,"suez":21974,"canons":21975,"graded":21976,"lurched":21977,"narrowing":21978,"boasts":21979,"guise":21980,"wed":21981,"enrico":21982,"##ovsky":21983,"rower":21984,"scarred":21985,"bree":21986,"cub":21987,"iberian":21988,"protagonists":21989,"bargaining":21990,"proposing":21991,"trainers":21992,"voyages":21993,"vans":21994,"fishes":21995,"##aea":21996,"##ivist":21997,"##verance":21998,"encryption":21999,"artworks":22000,"kazan":22001,"sabre":22002,"cleopatra":22003,"hepburn":22004,"rotting":22005,"supremacy":22006,"mecklenburg":22007,"##brate":22008,"burrows":22009,"hazards":22010,"outgoing":22011,"flair":22012,"organizes":22013,"##ctions":22014,"scorpion":22015,"##usions":22016,"boo":22017,"234":22018,"chevalier":22019,"dunedin":22020,"slapping":22021,"##34":22022,"ineligible":22023,"pensions":22024,"##38":22025,"##omic":22026,"manufactures":22027,"emails":22028,"bismarck":22029,"238":22030,"weakening":22031,"blackish":22032,"ding":22033,"mcgee":22034,"quo":22035,"##rling":22036,"northernmost":22037,"xx":22038,"manpower":22039,"greed":22040,"sampson":22041,"clicking":22042,"##ange":22043,"##horpe":22044,"##inations":22045,"##roving":22046,"torre":22047,"##eptive":22048,"##moral":22049,"symbolism":22050,"38th":22051,"asshole":22052,"meritorious":22053,"outfits":22054,"splashed":22055,"biographies":22056,"sprung":22057,"astros":22058,"##tale":22059,"302":22060,"737":22061,"filly":22062,"raoul":22063,"nw":22064,"tokugawa":22065,"linden":22066,"clubhouse":22067,"##apa":22068,"tracts":22069,"romano":22070,"##pio":22071,"putin":22072,"tags":22073,"##note":22074,"chained":22075,"dickson":22076,"gunshot":22077,"moe":22078,"gunn":22079,"rashid":22080,"##tails":22081,"zipper":22082,"##bas":22083,"##nea":22084,"contrasted":22085,"##ply":22086,"##udes":22087,"plum":22088,"pharaoh":22089,"##pile":22090,"aw":22091,"comedies":22092,"ingrid":22093,"sandwiches":22094,"subdivisions":22095,"1100":22096,"mariana":22097,"nokia":22098,"kamen":22099,"hz":22100,"delaney":22101,"veto":22102,"herring":22103,"##words":22104,"possessive":22105,"outlines":22106,"##roup":22107,"siemens":22108,"stairwell":22109,"rc":22110,"gallantry":22111,"messiah":22112,"palais":22113,"yells":22114,"233":22115,"zeppelin":22116,"##dm":22117,"bolivar":22118,"##cede":22119,"smackdown":22120,"mckinley":22121,"##mora":22122,"##yt":22123,"muted":22124,"geologic":22125,"finely":22126,"unitary":22127,"avatar":22128,"hamas":22129,"maynard":22130,"rees":22131,"bog":22132,"contrasting":22133,"##rut":22134,"liv":22135,"chico":22136,"disposition":22137,"pixel":22138,"##erate":22139,"becca":22140,"dmitry":22141,"yeshiva":22142,"narratives":22143,"##lva":22144,"##ulton":22145,"mercenary":22146,"sharpe":22147,"tempered":22148,"navigate":22149,"stealth":22150,"amassed":22151,"keynes":22152,"##lini":22153,"untouched":22154,"##rrie":22155,"havoc":22156,"lithium":22157,"##fighting":22158,"abyss":22159,"graf":22160,"southward":22161,"wolverine":22162,"balloons":22163,"implements":22164,"ngos":22165,"transitions":22166,"##icum":22167,"ambushed":22168,"concacaf":22169,"dormant":22170,"economists":22171,"##dim":22172,"costing":22173,"csi":22174,"rana":22175,"universite":22176,"boulders":22177,"verity":22178,"##llon":22179,"collin":22180,"mellon":22181,"misses":22182,"cypress":22183,"fluorescent":22184,"lifeless":22185,"spence":22186,"##ulla":22187,"crewe":22188,"shepard":22189,"pak":22190,"revelations":22191,"##م":22192,"jolly":22193,"gibbons":22194,"paw":22195,"##dro":22196,"##quel":22197,"freeing":22198,"##test":22199,"shack":22200,"fries":22201,"palatine":22202,"##51":22203,"##hiko":22204,"accompaniment":22205,"cruising":22206,"recycled":22207,"##aver":22208,"erwin":22209,"sorting":22210,"synthesizers":22211,"dyke":22212,"realities":22213,"sg":22214,"strides":22215,"enslaved":22216,"wetland":22217,"##ghan":22218,"competence":22219,"gunpowder":22220,"grassy":22221,"maroon":22222,"reactors":22223,"objection":22224,"##oms":22225,"carlson":22226,"gearbox":22227,"macintosh":22228,"radios":22229,"shelton":22230,"##sho":22231,"clergyman":22232,"prakash":22233,"254":22234,"mongols":22235,"trophies":22236,"oricon":22237,"228":22238,"stimuli":22239,"twenty20":22240,"cantonese":22241,"cortes":22242,"mirrored":22243,"##saurus":22244,"bhp":22245,"cristina":22246,"melancholy":22247,"##lating":22248,"enjoyable":22249,"nuevo":22250,"##wny":22251,"downfall":22252,"schumacher":22253,"##ind":22254,"banging":22255,"lausanne":22256,"rumbled":22257,"paramilitary":22258,"reflex":22259,"ax":22260,"amplitude":22261,"migratory":22262,"##gall":22263,"##ups":22264,"midi":22265,"barnard":22266,"lastly":22267,"sherry":22268,"##hp":22269,"##nall":22270,"keystone":22271,"##kra":22272,"carleton":22273,"slippery":22274,"##53":22275,"coloring":22276,"foe":22277,"socket":22278,"otter":22279,"##rgos":22280,"mats":22281,"##tose":22282,"consultants":22283,"bafta":22284,"bison":22285,"topping":22286,"##km":22287,"490":22288,"primal":22289,"abandonment":22290,"transplant":22291,"atoll":22292,"hideous":22293,"mort":22294,"pained":22295,"reproduced":22296,"tae":22297,"howling":22298,"##turn":22299,"unlawful":22300,"billionaire":22301,"hotter":22302,"poised":22303,"lansing":22304,"##chang":22305,"dinamo":22306,"retro":22307,"messing":22308,"nfc":22309,"domesday":22310,"##mina":22311,"blitz":22312,"timed":22313,"##athing":22314,"##kley":22315,"ascending":22316,"gesturing":22317,"##izations":22318,"signaled":22319,"tis":22320,"chinatown":22321,"mermaid":22322,"savanna":22323,"jameson":22324,"##aint":22325,"catalina":22326,"##pet":22327,"##hers":22328,"cochrane":22329,"cy":22330,"chatting":22331,"##kus":22332,"alerted":22333,"computation":22334,"mused":22335,"noelle":22336,"majestic":22337,"mohawk":22338,"campo":22339,"octagonal":22340,"##sant":22341,"##hend":22342,"241":22343,"aspiring":22344,"##mart":22345,"comprehend":22346,"iona":22347,"paralyzed":22348,"shimmering":22349,"swindon":22350,"rhone":22351,"##eley":22352,"reputed":22353,"configurations":22354,"pitchfork":22355,"agitation":22356,"francais":22357,"gillian":22358,"lipstick":22359,"##ilo":22360,"outsiders":22361,"pontifical":22362,"resisting":22363,"bitterness":22364,"sewer":22365,"rockies":22366,"##edd":22367,"##ucher":22368,"misleading":22369,"1756":22370,"exiting":22371,"galloway":22372,"##nging":22373,"risked":22374,"##heart":22375,"246":22376,"commemoration":22377,"schultz":22378,"##rka":22379,"integrating":22380,"##rsa":22381,"poses":22382,"shrieked":22383,"##weiler":22384,"guineas":22385,"gladys":22386,"jerking":22387,"owls":22388,"goldsmith":22389,"nightly":22390,"penetrating":22391,"##unced":22392,"lia":22393,"##33":22394,"ignited":22395,"betsy":22396,"##aring":22397,"##thorpe":22398,"follower":22399,"vigorously":22400,"##rave":22401,"coded":22402,"kiran":22403,"knit":22404,"zoology":22405,"tbilisi":22406,"##28":22407,"##bered":22408,"repository":22409,"govt":22410,"deciduous":22411,"dino":22412,"growling":22413,"##bba":22414,"enhancement":22415,"unleashed":22416,"chanting":22417,"pussy":22418,"biochemistry":22419,"##eric":22420,"kettle":22421,"repression":22422,"toxicity":22423,"nrhp":22424,"##arth":22425,"##kko":22426,"##bush":22427,"ernesto":22428,"commended":22429,"outspoken":22430,"242":22431,"mca":22432,"parchment":22433,"sms":22434,"kristen":22435,"##aton":22436,"bisexual":22437,"raked":22438,"glamour":22439,"navajo":22440,"a2":22441,"conditioned":22442,"showcased":22443,"##hma":22444,"spacious":22445,"youthful":22446,"##esa":22447,"usl":22448,"appliances":22449,"junta":22450,"brest":22451,"layne":22452,"conglomerate":22453,"enchanted":22454,"chao":22455,"loosened":22456,"picasso":22457,"circulating":22458,"inspect":22459,"montevideo":22460,"##centric":22461,"##kti":22462,"piazza":22463,"spurred":22464,"##aith":22465,"bari":22466,"freedoms":22467,"poultry":22468,"stamford":22469,"lieu":22470,"##ect":22471,"indigo":22472,"sarcastic":22473,"bahia":22474,"stump":22475,"attach":22476,"dvds":22477,"frankenstein":22478,"lille":22479,"approx":22480,"scriptures":22481,"pollen":22482,"##script":22483,"nmi":22484,"overseen":22485,"##ivism":22486,"tides":22487,"proponent":22488,"newmarket":22489,"inherit":22490,"milling":22491,"##erland":22492,"centralized":22493,"##rou":22494,"distributors":22495,"credentials":22496,"drawers":22497,"abbreviation":22498,"##lco":22499,"##xon":22500,"downing":22501,"uncomfortably":22502,"ripe":22503,"##oes":22504,"erase":22505,"franchises":22506,"##ever":22507,"populace":22508,"##bery":22509,"##khar":22510,"decomposition":22511,"pleas":22512,"##tet":22513,"daryl":22514,"sabah":22515,"##stle":22516,"##wide":22517,"fearless":22518,"genie":22519,"lesions":22520,"annette":22521,"##ogist":22522,"oboe":22523,"appendix":22524,"nair":22525,"dripped":22526,"petitioned":22527,"maclean":22528,"mosquito":22529,"parrot":22530,"rpg":22531,"hampered":22532,"1648":22533,"operatic":22534,"reservoirs":22535,"##tham":22536,"irrelevant":22537,"jolt":22538,"summarized":22539,"##fp":22540,"medallion":22541,"##taff":22542,"##−":22543,"clawed":22544,"harlow":22545,"narrower":22546,"goddard":22547,"marcia":22548,"bodied":22549,"fremont":22550,"suarez":22551,"altering":22552,"tempest":22553,"mussolini":22554,"porn":22555,"##isms":22556,"sweetly":22557,"oversees":22558,"walkers":22559,"solitude":22560,"grimly":22561,"shrines":22562,"hk":22563,"ich":22564,"supervisors":22565,"hostess":22566,"dietrich":22567,"legitimacy":22568,"brushes":22569,"expressive":22570,"##yp":22571,"dissipated":22572,"##rse":22573,"localized":22574,"systemic":22575,"##nikov":22576,"gettysburg":22577,"##js":22578,"##uaries":22579,"dialogues":22580,"muttering":22581,"251":22582,"housekeeper":22583,"sicilian":22584,"discouraged":22585,"##frey":22586,"beamed":22587,"kaladin":22588,"halftime":22589,"kidnap":22590,"##amo":22591,"##llet":22592,"1754":22593,"synonymous":22594,"depleted":22595,"instituto":22596,"insulin":22597,"reprised":22598,"##opsis":22599,"clashed":22600,"##ctric":22601,"interrupting":22602,"radcliffe":22603,"insisting":22604,"medici":22605,"1715":22606,"ejected":22607,"playfully":22608,"turbulent":22609,"##47":22610,"starvation":22611,"##rini":22612,"shipment":22613,"rebellious":22614,"petersen":22615,"verification":22616,"merits":22617,"##rified":22618,"cakes":22619,"##charged":22620,"1757":22621,"milford":22622,"shortages":22623,"spying":22624,"fidelity":22625,"##aker":22626,"emitted":22627,"storylines":22628,"harvested":22629,"seismic":22630,"##iform":22631,"cheung":22632,"kilda":22633,"theoretically":22634,"barbie":22635,"lynx":22636,"##rgy":22637,"##tius":22638,"goblin":22639,"mata":22640,"poisonous":22641,"##nburg":22642,"reactive":22643,"residues":22644,"obedience":22645,"##евич":22646,"conjecture":22647,"##rac":22648,"401":22649,"hating":22650,"sixties":22651,"kicker":22652,"moaning":22653,"motown":22654,"##bha":22655,"emancipation":22656,"neoclassical":22657,"##hering":22658,"consoles":22659,"ebert":22660,"professorship":22661,"##tures":22662,"sustaining":22663,"assaults":22664,"obeyed":22665,"affluent":22666,"incurred":22667,"tornadoes":22668,"##eber":22669,"##zow":22670,"emphasizing":22671,"highlanders":22672,"cheated":22673,"helmets":22674,"##ctus":22675,"internship":22676,"terence":22677,"bony":22678,"executions":22679,"legislators":22680,"berries":22681,"peninsular":22682,"tinged":22683,"##aco":22684,"1689":22685,"amplifier":22686,"corvette":22687,"ribbons":22688,"lavish":22689,"pennant":22690,"##lander":22691,"worthless":22692,"##chfield":22693,"##forms":22694,"mariano":22695,"pyrenees":22696,"expenditures":22697,"##icides":22698,"chesterfield":22699,"mandir":22700,"tailor":22701,"39th":22702,"sergey":22703,"nestled":22704,"willed":22705,"aristocracy":22706,"devotees":22707,"goodnight":22708,"raaf":22709,"rumored":22710,"weaponry":22711,"remy":22712,"appropriations":22713,"harcourt":22714,"burr":22715,"riaa":22716,"##lence":22717,"limitation":22718,"unnoticed":22719,"guo":22720,"soaking":22721,"swamps":22722,"##tica":22723,"collapsing":22724,"tatiana":22725,"descriptive":22726,"brigham":22727,"psalm":22728,"##chment":22729,"maddox":22730,"##lization":22731,"patti":22732,"caliph":22733,"##aja":22734,"akron":22735,"injuring":22736,"serra":22737,"##ganj":22738,"basins":22739,"##sari":22740,"astonished":22741,"launcher":22742,"##church":22743,"hilary":22744,"wilkins":22745,"sewing":22746,"##sf":22747,"stinging":22748,"##fia":22749,"##ncia":22750,"underwood":22751,"startup":22752,"##ition":22753,"compilations":22754,"vibrations":22755,"embankment":22756,"jurist":22757,"##nity":22758,"bard":22759,"juventus":22760,"groundwater":22761,"kern":22762,"palaces":22763,"helium":22764,"boca":22765,"cramped":22766,"marissa":22767,"soto":22768,"##worm":22769,"jae":22770,"princely":22771,"##ggy":22772,"faso":22773,"bazaar":22774,"warmly":22775,"##voking":22776,"229":22777,"pairing":22778,"##lite":22779,"##grate":22780,"##nets":22781,"wien":22782,"freaked":22783,"ulysses":22784,"rebirth":22785,"##alia":22786,"##rent":22787,"mummy":22788,"guzman":22789,"jimenez":22790,"stilled":22791,"##nitz":22792,"trajectory":22793,"tha":22794,"woken":22795,"archival":22796,"professions":22797,"##pts":22798,"##pta":22799,"hilly":22800,"shadowy":22801,"shrink":22802,"##bolt":22803,"norwood":22804,"glued":22805,"migrate":22806,"stereotypes":22807,"devoid":22808,"##pheus":22809,"625":22810,"evacuate":22811,"horrors":22812,"infancy":22813,"gotham":22814,"knowles":22815,"optic":22816,"downloaded":22817,"sachs":22818,"kingsley":22819,"parramatta":22820,"darryl":22821,"mor":22822,"##onale":22823,"shady":22824,"commence":22825,"confesses":22826,"kan":22827,"##meter":22828,"##placed":22829,"marlborough":22830,"roundabout":22831,"regents":22832,"frigates":22833,"io":22834,"##imating":22835,"gothenburg":22836,"revoked":22837,"carvings":22838,"clockwise":22839,"convertible":22840,"intruder":22841,"##sche":22842,"banged":22843,"##ogo":22844,"vicky":22845,"bourgeois":22846,"##mony":22847,"dupont":22848,"footing":22849,"##gum":22850,"pd":22851,"##real":22852,"buckle":22853,"yun":22854,"penthouse":22855,"sane":22856,"720":22857,"serviced":22858,"stakeholders":22859,"neumann":22860,"bb":22861,"##eers":22862,"comb":22863,"##gam":22864,"catchment":22865,"pinning":22866,"rallies":22867,"typing":22868,"##elles":22869,"forefront":22870,"freiburg":22871,"sweetie":22872,"giacomo":22873,"widowed":22874,"goodwill":22875,"worshipped":22876,"aspirations":22877,"midday":22878,"##vat":22879,"fishery":22880,"##trick":22881,"bournemouth":22882,"turk":22883,"243":22884,"hearth":22885,"ethanol":22886,"guadalajara":22887,"murmurs":22888,"sl":22889,"##uge":22890,"afforded":22891,"scripted":22892,"##hta":22893,"wah":22894,"##jn":22895,"coroner":22896,"translucent":22897,"252":22898,"memorials":22899,"puck":22900,"progresses":22901,"clumsy":22902,"##race":22903,"315":22904,"candace":22905,"recounted":22906,"##27":22907,"##slin":22908,"##uve":22909,"filtering":22910,"##mac":22911,"howl":22912,"strata":22913,"heron":22914,"leveled":22915,"##ays":22916,"dubious":22917,"##oja":22918,"##т":22919,"##wheel":22920,"citations":22921,"exhibiting":22922,"##laya":22923,"##mics":22924,"##pods":22925,"turkic":22926,"##lberg":22927,"injunction":22928,"##ennial":22929,"##mit":22930,"antibodies":22931,"##44":22932,"organise":22933,"##rigues":22934,"cardiovascular":22935,"cushion":22936,"inverness":22937,"##zquez":22938,"dia":22939,"cocoa":22940,"sibling":22941,"##tman":22942,"##roid":22943,"expanse":22944,"feasible":22945,"tunisian":22946,"algiers":22947,"##relli":22948,"rus":22949,"bloomberg":22950,"dso":22951,"westphalia":22952,"bro":22953,"tacoma":22954,"281":22955,"downloads":22956,"##ours":22957,"konrad":22958,"duran":22959,"##hdi":22960,"continuum":22961,"jett":22962,"compares":22963,"legislator":22964,"secession":22965,"##nable":22966,"##gues":22967,"##zuka":22968,"translating":22969,"reacher":22970,"##gley":22971,"##ła":22972,"aleppo":22973,"##agi":22974,"tc":22975,"orchards":22976,"trapping":22977,"linguist":22978,"versatile":22979,"drumming":22980,"postage":22981,"calhoun":22982,"superiors":22983,"##mx":22984,"barefoot":22985,"leary":22986,"##cis":22987,"ignacio":22988,"alfa":22989,"kaplan":22990,"##rogen":22991,"bratislava":22992,"mori":22993,"##vot":22994,"disturb":22995,"haas":22996,"313":22997,"cartridges":22998,"gilmore":22999,"radiated":23000,"salford":23001,"tunic":23002,"hades":23003,"##ulsive":23004,"archeological":23005,"delilah":23006,"magistrates":23007,"auditioned":23008,"brewster":23009,"charters":23010,"empowerment":23011,"blogs":23012,"cappella":23013,"dynasties":23014,"iroquois":23015,"whipping":23016,"##krishna":23017,"raceway":23018,"truths":23019,"myra":23020,"weaken":23021,"judah":23022,"mcgregor":23023,"##horse":23024,"mic":23025,"refueling":23026,"37th":23027,"burnley":23028,"bosses":23029,"markus":23030,"premio":23031,"query":23032,"##gga":23033,"dunbar":23034,"##economic":23035,"darkest":23036,"lyndon":23037,"sealing":23038,"commendation":23039,"reappeared":23040,"##mun":23041,"addicted":23042,"ezio":23043,"slaughtered":23044,"satisfactory":23045,"shuffle":23046,"##eves":23047,"##thic":23048,"##uj":23049,"fortification":23050,"warrington":23051,"##otto":23052,"resurrected":23053,"fargo":23054,"mane":23055,"##utable":23056,"##lei":23057,"##space":23058,"foreword":23059,"ox":23060,"##aris":23061,"##vern":23062,"abrams":23063,"hua":23064,"##mento":23065,"sakura":23066,"##alo":23067,"uv":23068,"sentimental":23069,"##skaya":23070,"midfield":23071,"##eses":23072,"sturdy":23073,"scrolls":23074,"macleod":23075,"##kyu":23076,"entropy":23077,"##lance":23078,"mitochondrial":23079,"cicero":23080,"excelled":23081,"thinner":23082,"convoys":23083,"perceive":23084,"##oslav":23085,"##urable":23086,"systematically":23087,"grind":23088,"burkina":23089,"287":23090,"##tagram":23091,"ops":23092,"##aman":23093,"guantanamo":23094,"##cloth":23095,"##tite":23096,"forcefully":23097,"wavy":23098,"##jou":23099,"pointless":23100,"##linger":23101,"##tze":23102,"layton":23103,"portico":23104,"superficial":23105,"clerical":23106,"outlaws":23107,"##hism":23108,"burials":23109,"muir":23110,"##inn":23111,"creditors":23112,"hauling":23113,"rattle":23114,"##leg":23115,"calais":23116,"monde":23117,"archers":23118,"reclaimed":23119,"dwell":23120,"wexford":23121,"hellenic":23122,"falsely":23123,"remorse":23124,"##tek":23125,"dough":23126,"furnishings":23127,"##uttered":23128,"gabon":23129,"neurological":23130,"novice":23131,"##igraphy":23132,"contemplated":23133,"pulpit":23134,"nightstand":23135,"saratoga":23136,"##istan":23137,"documenting":23138,"pulsing":23139,"taluk":23140,"##firmed":23141,"busted":23142,"marital":23143,"##rien":23144,"disagreements":23145,"wasps":23146,"##yes":23147,"hodge":23148,"mcdonnell":23149,"mimic":23150,"fran":23151,"pendant":23152,"dhabi":23153,"musa":23154,"##nington":23155,"congratulations":23156,"argent":23157,"darrell":23158,"concussion":23159,"losers":23160,"regrets":23161,"thessaloniki":23162,"reversal":23163,"donaldson":23164,"hardwood":23165,"thence":23166,"achilles":23167,"ritter":23168,"##eran":23169,"demonic":23170,"jurgen":23171,"prophets":23172,"goethe":23173,"eki":23174,"classmate":23175,"buff":23176,"##cking":23177,"yank":23178,"irrational":23179,"##inging":23180,"perished":23181,"seductive":23182,"qur":23183,"sourced":23184,"##crat":23185,"##typic":23186,"mustard":23187,"ravine":23188,"barre":23189,"horizontally":23190,"characterization":23191,"phylogenetic":23192,"boise":23193,"##dit":23194,"##runner":23195,"##tower":23196,"brutally":23197,"intercourse":23198,"seduce":23199,"##bbing":23200,"fay":23201,"ferris":23202,"ogden":23203,"amar":23204,"nik":23205,"unarmed":23206,"##inator":23207,"evaluating":23208,"kyrgyzstan":23209,"sweetness":23210,"##lford":23211,"##oki":23212,"mccormick":23213,"meiji":23214,"notoriety":23215,"stimulate":23216,"disrupt":23217,"figuring":23218,"instructional":23219,"mcgrath":23220,"##zoo":23221,"groundbreaking":23222,"##lto":23223,"flinch":23224,"khorasan":23225,"agrarian":23226,"bengals":23227,"mixer":23228,"radiating":23229,"##sov":23230,"ingram":23231,"pitchers":23232,"nad":23233,"tariff":23234,"##cript":23235,"tata":23236,"##codes":23237,"##emi":23238,"##ungen":23239,"appellate":23240,"lehigh":23241,"##bled":23242,"##giri":23243,"brawl":23244,"duct":23245,"texans":23246,"##ciation":23247,"##ropolis":23248,"skipper":23249,"speculative":23250,"vomit":23251,"doctrines":23252,"stresses":23253,"253":23254,"davy":23255,"graders":23256,"whitehead":23257,"jozef":23258,"timely":23259,"cumulative":23260,"haryana":23261,"paints":23262,"appropriately":23263,"boon":23264,"cactus":23265,"##ales":23266,"##pid":23267,"dow":23268,"legions":23269,"##pit":23270,"perceptions":23271,"1730":23272,"picturesque":23273,"##yse":23274,"periphery":23275,"rune":23276,"wr":23277,"##aha":23278,"celtics":23279,"sentencing":23280,"whoa":23281,"##erin":23282,"confirms":23283,"variance":23284,"425":23285,"moines":23286,"mathews":23287,"spade":23288,"rave":23289,"m1":23290,"fronted":23291,"fx":23292,"blending":23293,"alleging":23294,"reared":23295,"##gl":23296,"237":23297,"##paper":23298,"grassroots":23299,"eroded":23300,"##free":23301,"##physical":23302,"directs":23303,"ordeal":23304,"##sław":23305,"accelerate":23306,"hacker":23307,"rooftop":23308,"##inia":23309,"lev":23310,"buys":23311,"cebu":23312,"devote":23313,"##lce":23314,"specialising":23315,"##ulsion":23316,"choreographed":23317,"repetition":23318,"warehouses":23319,"##ryl":23320,"paisley":23321,"tuscany":23322,"analogy":23323,"sorcerer":23324,"hash":23325,"huts":23326,"shards":23327,"descends":23328,"exclude":23329,"nix":23330,"chaplin":23331,"gaga":23332,"ito":23333,"vane":23334,"##drich":23335,"causeway":23336,"misconduct":23337,"limo":23338,"orchestrated":23339,"glands":23340,"jana":23341,"##kot":23342,"u2":23343,"##mple":23344,"##sons":23345,"branching":23346,"contrasts":23347,"scoop":23348,"longed":23349,"##virus":23350,"chattanooga":23351,"##75":23352,"syrup":23353,"cornerstone":23354,"##tized":23355,"##mind":23356,"##iaceae":23357,"careless":23358,"precedence":23359,"frescoes":23360,"##uet":23361,"chilled":23362,"consult":23363,"modelled":23364,"snatch":23365,"peat":23366,"##thermal":23367,"caucasian":23368,"humane":23369,"relaxation":23370,"spins":23371,"temperance":23372,"##lbert":23373,"occupations":23374,"lambda":23375,"hybrids":23376,"moons":23377,"mp3":23378,"##oese":23379,"247":23380,"rolf":23381,"societal":23382,"yerevan":23383,"ness":23384,"##ssler":23385,"befriended":23386,"mechanized":23387,"nominate":23388,"trough":23389,"boasted":23390,"cues":23391,"seater":23392,"##hom":23393,"bends":23394,"##tangle":23395,"conductors":23396,"emptiness":23397,"##lmer":23398,"eurasian":23399,"adriatic":23400,"tian":23401,"##cie":23402,"anxiously":23403,"lark":23404,"propellers":23405,"chichester":23406,"jock":23407,"ev":23408,"2a":23409,"##holding":23410,"credible":23411,"recounts":23412,"tori":23413,"loyalist":23414,"abduction":23415,"##hoot":23416,"##redo":23417,"nepali":23418,"##mite":23419,"ventral":23420,"tempting":23421,"##ango":23422,"##crats":23423,"steered":23424,"##wice":23425,"javelin":23426,"dipping":23427,"laborers":23428,"prentice":23429,"looming":23430,"titanium":23431,"##ː":23432,"badges":23433,"emir":23434,"tensor":23435,"##ntation":23436,"egyptians":23437,"rash":23438,"denies":23439,"hawthorne":23440,"lombard":23441,"showers":23442,"wehrmacht":23443,"dietary":23444,"trojan":23445,"##reus":23446,"welles":23447,"executing":23448,"horseshoe":23449,"lifeboat":23450,"##lak":23451,"elsa":23452,"infirmary":23453,"nearing":23454,"roberta":23455,"boyer":23456,"mutter":23457,"trillion":23458,"joanne":23459,"##fine":23460,"##oked":23461,"sinks":23462,"vortex":23463,"uruguayan":23464,"clasp":23465,"sirius":23466,"##block":23467,"accelerator":23468,"prohibit":23469,"sunken":23470,"byu":23471,"chronological":23472,"diplomats":23473,"ochreous":23474,"510":23475,"symmetrical":23476,"1644":23477,"maia":23478,"##tology":23479,"salts":23480,"reigns":23481,"atrocities":23482,"##ия":23483,"hess":23484,"bared":23485,"issn":23486,"##vyn":23487,"cater":23488,"saturated":23489,"##cycle":23490,"##isse":23491,"sable":23492,"voyager":23493,"dyer":23494,"yusuf":23495,"##inge":23496,"fountains":23497,"wolff":23498,"##39":23499,"##nni":23500,"engraving":23501,"rollins":23502,"atheist":23503,"ominous":23504,"##ault":23505,"herr":23506,"chariot":23507,"martina":23508,"strung":23509,"##fell":23510,"##farlane":23511,"horrific":23512,"sahib":23513,"gazes":23514,"saetan":23515,"erased":23516,"ptolemy":23517,"##olic":23518,"flushing":23519,"lauderdale":23520,"analytic":23521,"##ices":23522,"530":23523,"navarro":23524,"beak":23525,"gorilla":23526,"herrera":23527,"broom":23528,"guadalupe":23529,"raiding":23530,"sykes":23531,"311":23532,"bsc":23533,"deliveries":23534,"1720":23535,"invasions":23536,"carmichael":23537,"tajikistan":23538,"thematic":23539,"ecumenical":23540,"sentiments":23541,"onstage":23542,"##rians":23543,"##brand":23544,"##sume":23545,"catastrophic":23546,"flanks":23547,"molten":23548,"##arns":23549,"waller":23550,"aimee":23551,"terminating":23552,"##icing":23553,"alternately":23554,"##oche":23555,"nehru":23556,"printers":23557,"outraged":23558,"##eving":23559,"empires":23560,"template":23561,"banners":23562,"repetitive":23563,"za":23564,"##oise":23565,"vegetarian":23566,"##tell":23567,"guiana":23568,"opt":23569,"cavendish":23570,"lucknow":23571,"synthesized":23572,"##hani":23573,"##mada":23574,"finalized":23575,"##ctable":23576,"fictitious":23577,"mayoral":23578,"unreliable":23579,"##enham":23580,"embracing":23581,"peppers":23582,"rbis":23583,"##chio":23584,"##neo":23585,"inhibition":23586,"slashed":23587,"togo":23588,"orderly":23589,"embroidered":23590,"safari":23591,"salty":23592,"236":23593,"barron":23594,"benito":23595,"totaled":23596,"##dak":23597,"pubs":23598,"simulated":23599,"caden":23600,"devin":23601,"tolkien":23602,"momma":23603,"welding":23604,"sesame":23605,"##ept":23606,"gottingen":23607,"hardness":23608,"630":23609,"shaman":23610,"temeraire":23611,"620":23612,"adequately":23613,"pediatric":23614,"##kit":23615,"ck":23616,"assertion":23617,"radicals":23618,"composure":23619,"cadence":23620,"seafood":23621,"beaufort":23622,"lazarus":23623,"mani":23624,"warily":23625,"cunning":23626,"kurdistan":23627,"249":23628,"cantata":23629,"##kir":23630,"ares":23631,"##41":23632,"##clusive":23633,"nape":23634,"townland":23635,"geared":23636,"insulted":23637,"flutter":23638,"boating":23639,"violate":23640,"draper":23641,"dumping":23642,"malmo":23643,"##hh":23644,"##romatic":23645,"firearm":23646,"alta":23647,"bono":23648,"obscured":23649,"##clave":23650,"exceeds":23651,"panorama":23652,"unbelievable":23653,"##train":23654,"preschool":23655,"##essed":23656,"disconnected":23657,"installing":23658,"rescuing":23659,"secretaries":23660,"accessibility":23661,"##castle":23662,"##drive":23663,"##ifice":23664,"##film":23665,"bouts":23666,"slug":23667,"waterway":23668,"mindanao":23669,"##buro":23670,"##ratic":23671,"halves":23672,"##ل":23673,"calming":23674,"liter":23675,"maternity":23676,"adorable":23677,"bragg":23678,"electrification":23679,"mcc":23680,"##dote":23681,"roxy":23682,"schizophrenia":23683,"##body":23684,"munoz":23685,"kaye":23686,"whaling":23687,"239":23688,"mil":23689,"tingling":23690,"tolerant":23691,"##ago":23692,"unconventional":23693,"volcanoes":23694,"##finder":23695,"deportivo":23696,"##llie":23697,"robson":23698,"kaufman":23699,"neuroscience":23700,"wai":23701,"deportation":23702,"masovian":23703,"scraping":23704,"converse":23705,"##bh":23706,"hacking":23707,"bulge":23708,"##oun":23709,"administratively":23710,"yao":23711,"580":23712,"amp":23713,"mammoth":23714,"booster":23715,"claremont":23716,"hooper":23717,"nomenclature":23718,"pursuits":23719,"mclaughlin":23720,"melinda":23721,"##sul":23722,"catfish":23723,"barclay":23724,"substrates":23725,"taxa":23726,"zee":23727,"originals":23728,"kimberly":23729,"packets":23730,"padma":23731,"##ality":23732,"borrowing":23733,"ostensibly":23734,"solvent":23735,"##bri":23736,"##genesis":23737,"##mist":23738,"lukas":23739,"shreveport":23740,"veracruz":23741,"##ь":23742,"##lou":23743,"##wives":23744,"cheney":23745,"tt":23746,"anatolia":23747,"hobbs":23748,"##zyn":23749,"cyclic":23750,"radiant":23751,"alistair":23752,"greenish":23753,"siena":23754,"dat":23755,"independents":23756,"##bation":23757,"conform":23758,"pieter":23759,"hyper":23760,"applicant":23761,"bradshaw":23762,"spores":23763,"telangana":23764,"vinci":23765,"inexpensive":23766,"nuclei":23767,"322":23768,"jang":23769,"nme":23770,"soho":23771,"spd":23772,"##ign":23773,"cradled":23774,"receptionist":23775,"pow":23776,"##43":23777,"##rika":23778,"fascism":23779,"##ifer":23780,"experimenting":23781,"##ading":23782,"##iec":23783,"##region":23784,"345":23785,"jocelyn":23786,"maris":23787,"stair":23788,"nocturnal":23789,"toro":23790,"constabulary":23791,"elgin":23792,"##kker":23793,"msc":23794,"##giving":23795,"##schen":23796,"##rase":23797,"doherty":23798,"doping":23799,"sarcastically":23800,"batter":23801,"maneuvers":23802,"##cano":23803,"##apple":23804,"##gai":23805,"##git":23806,"intrinsic":23807,"##nst":23808,"##stor":23809,"1753":23810,"showtime":23811,"cafes":23812,"gasps":23813,"lviv":23814,"ushered":23815,"##thed":23816,"fours":23817,"restart":23818,"astonishment":23819,"transmitting":23820,"flyer":23821,"shrugs":23822,"##sau":23823,"intriguing":23824,"cones":23825,"dictated":23826,"mushrooms":23827,"medial":23828,"##kovsky":23829,"##elman":23830,"escorting":23831,"gaped":23832,"##26":23833,"godfather":23834,"##door":23835,"##sell":23836,"djs":23837,"recaptured":23838,"timetable":23839,"vila":23840,"1710":23841,"3a":23842,"aerodrome":23843,"mortals":23844,"scientology":23845,"##orne":23846,"angelina":23847,"mag":23848,"convection":23849,"unpaid":23850,"insertion":23851,"intermittent":23852,"lego":23853,"##nated":23854,"endeavor":23855,"kota":23856,"pereira":23857,"##lz":23858,"304":23859,"bwv":23860,"glamorgan":23861,"insults":23862,"agatha":23863,"fey":23864,"##cend":23865,"fleetwood":23866,"mahogany":23867,"protruding":23868,"steamship":23869,"zeta":23870,"##arty":23871,"mcguire":23872,"suspense":23873,"##sphere":23874,"advising":23875,"urges":23876,"##wala":23877,"hurriedly":23878,"meteor":23879,"gilded":23880,"inline":23881,"arroyo":23882,"stalker":23883,"##oge":23884,"excitedly":23885,"revered":23886,"##cure":23887,"earle":23888,"introductory":23889,"##break":23890,"##ilde":23891,"mutants":23892,"puff":23893,"pulses":23894,"reinforcement":23895,"##haling":23896,"curses":23897,"lizards":23898,"stalk":23899,"correlated":23900,"##fixed":23901,"fallout":23902,"macquarie":23903,"##unas":23904,"bearded":23905,"denton":23906,"heaving":23907,"802":23908,"##ocation":23909,"winery":23910,"assign":23911,"dortmund":23912,"##lkirk":23913,"everest":23914,"invariant":23915,"charismatic":23916,"susie":23917,"##elling":23918,"bled":23919,"lesley":23920,"telegram":23921,"sumner":23922,"bk":23923,"##ogen":23924,"##к":23925,"wilcox":23926,"needy":23927,"colbert":23928,"duval":23929,"##iferous":23930,"##mbled":23931,"allotted":23932,"attends":23933,"imperative":23934,"##hita":23935,"replacements":23936,"hawker":23937,"##inda":23938,"insurgency":23939,"##zee":23940,"##eke":23941,"casts":23942,"##yla":23943,"680":23944,"ives":23945,"transitioned":23946,"##pack":23947,"##powering":23948,"authoritative":23949,"baylor":23950,"flex":23951,"cringed":23952,"plaintiffs":23953,"woodrow":23954,"##skie":23955,"drastic":23956,"ape":23957,"aroma":23958,"unfolded":23959,"commotion":23960,"nt":23961,"preoccupied":23962,"theta":23963,"routines":23964,"lasers":23965,"privatization":23966,"wand":23967,"domino":23968,"ek":23969,"clenching":23970,"nsa":23971,"strategically":23972,"showered":23973,"bile":23974,"handkerchief":23975,"pere":23976,"storing":23977,"christophe":23978,"insulting":23979,"316":23980,"nakamura":23981,"romani":23982,"asiatic":23983,"magdalena":23984,"palma":23985,"cruises":23986,"stripping":23987,"405":23988,"konstantin":23989,"soaring":23990,"##berman":23991,"colloquially":23992,"forerunner":23993,"havilland":23994,"incarcerated":23995,"parasites":23996,"sincerity":23997,"##utus":23998,"disks":23999,"plank":24000,"saigon":24001,"##ining":24002,"corbin":24003,"homo":24004,"ornaments":24005,"powerhouse":24006,"##tlement":24007,"chong":24008,"fastened":24009,"feasibility":24010,"idf":24011,"morphological":24012,"usable":24013,"##nish":24014,"##zuki":24015,"aqueduct":24016,"jaguars":24017,"keepers":24018,"##flies":24019,"aleksandr":24020,"faust":24021,"assigns":24022,"ewing":24023,"bacterium":24024,"hurled":24025,"tricky":24026,"hungarians":24027,"integers":24028,"wallis":24029,"321":24030,"yamaha":24031,"##isha":24032,"hushed":24033,"oblivion":24034,"aviator":24035,"evangelist":24036,"friars":24037,"##eller":24038,"monograph":24039,"ode":24040,"##nary":24041,"airplanes":24042,"labourers":24043,"charms":24044,"##nee":24045,"1661":24046,"hagen":24047,"tnt":24048,"rudder":24049,"fiesta":24050,"transcript":24051,"dorothea":24052,"ska":24053,"inhibitor":24054,"maccabi":24055,"retorted":24056,"raining":24057,"encompassed":24058,"clauses":24059,"menacing":24060,"1642":24061,"lineman":24062,"##gist":24063,"vamps":24064,"##ape":24065,"##dick":24066,"gloom":24067,"##rera":24068,"dealings":24069,"easing":24070,"seekers":24071,"##nut":24072,"##pment":24073,"helens":24074,"unmanned":24075,"##anu":24076,"##isson":24077,"basics":24078,"##amy":24079,"##ckman":24080,"adjustments":24081,"1688":24082,"brutality":24083,"horne":24084,"##zell":24085,"sui":24086,"##55":24087,"##mable":24088,"aggregator":24089,"##thal":24090,"rhino":24091,"##drick":24092,"##vira":24093,"counters":24094,"zoom":24095,"##01":24096,"##rting":24097,"mn":24098,"montenegrin":24099,"packard":24100,"##unciation":24101,"##♭":24102,"##kki":24103,"reclaim":24104,"scholastic":24105,"thugs":24106,"pulsed":24107,"##icia":24108,"syriac":24109,"quan":24110,"saddam":24111,"banda":24112,"kobe":24113,"blaming":24114,"buddies":24115,"dissent":24116,"##lusion":24117,"##usia":24118,"corbett":24119,"jaya":24120,"delle":24121,"erratic":24122,"lexie":24123,"##hesis":24124,"435":24125,"amiga":24126,"hermes":24127,"##pressing":24128,"##leen":24129,"chapels":24130,"gospels":24131,"jamal":24132,"##uating":24133,"compute":24134,"revolving":24135,"warp":24136,"##sso":24137,"##thes":24138,"armory":24139,"##eras":24140,"##gol":24141,"antrim":24142,"loki":24143,"##kow":24144,"##asian":24145,"##good":24146,"##zano":24147,"braid":24148,"handwriting":24149,"subdistrict":24150,"funky":24151,"pantheon":24152,"##iculate":24153,"concurrency":24154,"estimation":24155,"improper":24156,"juliana":24157,"##his":24158,"newcomers":24159,"johnstone":24160,"staten":24161,"communicated":24162,"##oco":24163,"##alle":24164,"sausage":24165,"stormy":24166,"##stered":24167,"##tters":24168,"superfamily":24169,"##grade":24170,"acidic":24171,"collateral":24172,"tabloid":24173,"##oped":24174,"##rza":24175,"bladder":24176,"austen":24177,"##ellant":24178,"mcgraw":24179,"##hay":24180,"hannibal":24181,"mein":24182,"aquino":24183,"lucifer":24184,"wo":24185,"badger":24186,"boar":24187,"cher":24188,"christensen":24189,"greenberg":24190,"interruption":24191,"##kken":24192,"jem":24193,"244":24194,"mocked":24195,"bottoms":24196,"cambridgeshire":24197,"##lide":24198,"sprawling":24199,"##bbly":24200,"eastwood":24201,"ghent":24202,"synth":24203,"##buck":24204,"advisers":24205,"##bah":24206,"nominally":24207,"hapoel":24208,"qu":24209,"daggers":24210,"estranged":24211,"fabricated":24212,"towels":24213,"vinnie":24214,"wcw":24215,"misunderstanding":24216,"anglia":24217,"nothin":24218,"unmistakable":24219,"##dust":24220,"##lova":24221,"chilly":24222,"marquette":24223,"truss":24224,"##edge":24225,"##erine":24226,"reece":24227,"##lty":24228,"##chemist":24229,"##connected":24230,"272":24231,"308":24232,"41st":24233,"bash":24234,"raion":24235,"waterfalls":24236,"##ump":24237,"##main":24238,"labyrinth":24239,"queue":24240,"theorist":24241,"##istle":24242,"bharatiya":24243,"flexed":24244,"soundtracks":24245,"rooney":24246,"leftist":24247,"patrolling":24248,"wharton":24249,"plainly":24250,"alleviate":24251,"eastman":24252,"schuster":24253,"topographic":24254,"engages":24255,"immensely":24256,"unbearable":24257,"fairchild":24258,"1620":24259,"dona":24260,"lurking":24261,"parisian":24262,"oliveira":24263,"ia":24264,"indictment":24265,"hahn":24266,"bangladeshi":24267,"##aster":24268,"vivo":24269,"##uming":24270,"##ential":24271,"antonia":24272,"expects":24273,"indoors":24274,"kildare":24275,"harlan":24276,"##logue":24277,"##ogenic":24278,"##sities":24279,"forgiven":24280,"##wat":24281,"childish":24282,"tavi":24283,"##mide":24284,"##orra":24285,"plausible":24286,"grimm":24287,"successively":24288,"scooted":24289,"##bola":24290,"##dget":24291,"##rith":24292,"spartans":24293,"emery":24294,"flatly":24295,"azure":24296,"epilogue":24297,"##wark":24298,"flourish":24299,"##iny":24300,"##tracted":24301,"##overs":24302,"##oshi":24303,"bestseller":24304,"distressed":24305,"receipt":24306,"spitting":24307,"hermit":24308,"topological":24309,"##cot":24310,"drilled":24311,"subunit":24312,"francs":24313,"##layer":24314,"eel":24315,"##fk":24316,"##itas":24317,"octopus":24318,"footprint":24319,"petitions":24320,"ufo":24321,"##say":24322,"##foil":24323,"interfering":24324,"leaking":24325,"palo":24326,"##metry":24327,"thistle":24328,"valiant":24329,"##pic":24330,"narayan":24331,"mcpherson":24332,"##fast":24333,"gonzales":24334,"##ym":24335,"##enne":24336,"dustin":24337,"novgorod":24338,"solos":24339,"##zman":24340,"doin":24341,"##raph":24342,"##patient":24343,"##meyer":24344,"soluble":24345,"ashland":24346,"cuffs":24347,"carole":24348,"pendleton":24349,"whistling":24350,"vassal":24351,"##river":24352,"deviation":24353,"revisited":24354,"constituents":24355,"rallied":24356,"rotate":24357,"loomed":24358,"##eil":24359,"##nting":24360,"amateurs":24361,"augsburg":24362,"auschwitz":24363,"crowns":24364,"skeletons":24365,"##cona":24366,"bonnet":24367,"257":24368,"dummy":24369,"globalization":24370,"simeon":24371,"sleeper":24372,"mandal":24373,"differentiated":24374,"##crow":24375,"##mare":24376,"milne":24377,"bundled":24378,"exasperated":24379,"talmud":24380,"owes":24381,"segregated":24382,"##feng":24383,"##uary":24384,"dentist":24385,"piracy":24386,"props":24387,"##rang":24388,"devlin":24389,"##torium":24390,"malicious":24391,"paws":24392,"##laid":24393,"dependency":24394,"##ergy":24395,"##fers":24396,"##enna":24397,"258":24398,"pistons":24399,"rourke":24400,"jed":24401,"grammatical":24402,"tres":24403,"maha":24404,"wig":24405,"512":24406,"ghostly":24407,"jayne":24408,"##achal":24409,"##creen":24410,"##ilis":24411,"##lins":24412,"##rence":24413,"designate":24414,"##with":24415,"arrogance":24416,"cambodian":24417,"clones":24418,"showdown":24419,"throttle":24420,"twain":24421,"##ception":24422,"lobes":24423,"metz":24424,"nagoya":24425,"335":24426,"braking":24427,"##furt":24428,"385":24429,"roaming":24430,"##minster":24431,"amin":24432,"crippled":24433,"##37":24434,"##llary":24435,"indifferent":24436,"hoffmann":24437,"idols":24438,"intimidating":24439,"1751":24440,"261":24441,"influenza":24442,"memo":24443,"onions":24444,"1748":24445,"bandage":24446,"consciously":24447,"##landa":24448,"##rage":24449,"clandestine":24450,"observes":24451,"swiped":24452,"tangle":24453,"##ener":24454,"##jected":24455,"##trum":24456,"##bill":24457,"##lta":24458,"hugs":24459,"congresses":24460,"josiah":24461,"spirited":24462,"##dek":24463,"humanist":24464,"managerial":24465,"filmmaking":24466,"inmate":24467,"rhymes":24468,"debuting":24469,"grimsby":24470,"ur":24471,"##laze":24472,"duplicate":24473,"vigor":24474,"##tf":24475,"republished":24476,"bolshevik":24477,"refurbishment":24478,"antibiotics":24479,"martini":24480,"methane":24481,"newscasts":24482,"royale":24483,"horizons":24484,"levant":24485,"iain":24486,"visas":24487,"##ischen":24488,"paler":24489,"##around":24490,"manifestation":24491,"snuck":24492,"alf":24493,"chop":24494,"futile":24495,"pedestal":24496,"rehab":24497,"##kat":24498,"bmg":24499,"kerman":24500,"res":24501,"fairbanks":24502,"jarrett":24503,"abstraction":24504,"saharan":24505,"##zek":24506,"1746":24507,"procedural":24508,"clearer":24509,"kincaid":24510,"sash":24511,"luciano":24512,"##ffey":24513,"crunch":24514,"helmut":24515,"##vara":24516,"revolutionaries":24517,"##tute":24518,"creamy":24519,"leach":24520,"##mmon":24521,"1747":24522,"permitting":24523,"nes":24524,"plight":24525,"wendell":24526,"##lese":24527,"contra":24528,"ts":24529,"clancy":24530,"ipa":24531,"mach":24532,"staples":24533,"autopsy":24534,"disturbances":24535,"nueva":24536,"karin":24537,"pontiac":24538,"##uding":24539,"proxy":24540,"venerable":24541,"haunt":24542,"leto":24543,"bergman":24544,"expands":24545,"##helm":24546,"wal":24547,"##pipe":24548,"canning":24549,"celine":24550,"cords":24551,"obesity":24552,"##enary":24553,"intrusion":24554,"planner":24555,"##phate":24556,"reasoned":24557,"sequencing":24558,"307":24559,"harrow":24560,"##chon":24561,"##dora":24562,"marred":24563,"mcintyre":24564,"repay":24565,"tarzan":24566,"darting":24567,"248":24568,"harrisburg":24569,"margarita":24570,"repulsed":24571,"##hur":24572,"##lding":24573,"belinda":24574,"hamburger":24575,"novo":24576,"compliant":24577,"runways":24578,"bingham":24579,"registrar":24580,"skyscraper":24581,"ic":24582,"cuthbert":24583,"improvisation":24584,"livelihood":24585,"##corp":24586,"##elial":24587,"admiring":24588,"##dened":24589,"sporadic":24590,"believer":24591,"casablanca":24592,"popcorn":24593,"##29":24594,"asha":24595,"shovel":24596,"##bek":24597,"##dice":24598,"coiled":24599,"tangible":24600,"##dez":24601,"casper":24602,"elsie":24603,"resin":24604,"tenderness":24605,"rectory":24606,"##ivision":24607,"avail":24608,"sonar":24609,"##mori":24610,"boutique":24611,"##dier":24612,"guerre":24613,"bathed":24614,"upbringing":24615,"vaulted":24616,"sandals":24617,"blessings":24618,"##naut":24619,"##utnant":24620,"1680":24621,"306":24622,"foxes":24623,"pia":24624,"corrosion":24625,"hesitantly":24626,"confederates":24627,"crystalline":24628,"footprints":24629,"shapiro":24630,"tirana":24631,"valentin":24632,"drones":24633,"45th":24634,"microscope":24635,"shipments":24636,"texted":24637,"inquisition":24638,"wry":24639,"guernsey":24640,"unauthorized":24641,"resigning":24642,"760":24643,"ripple":24644,"schubert":24645,"stu":24646,"reassure":24647,"felony":24648,"##ardo":24649,"brittle":24650,"koreans":24651,"##havan":24652,"##ives":24653,"dun":24654,"implicit":24655,"tyres":24656,"##aldi":24657,"##lth":24658,"magnolia":24659,"##ehan":24660,"##puri":24661,"##poulos":24662,"aggressively":24663,"fei":24664,"gr":24665,"familiarity":24666,"##poo":24667,"indicative":24668,"##trust":24669,"fundamentally":24670,"jimmie":24671,"overrun":24672,"395":24673,"anchors":24674,"moans":24675,"##opus":24676,"britannia":24677,"armagh":24678,"##ggle":24679,"purposely":24680,"seizing":24681,"##vao":24682,"bewildered":24683,"mundane":24684,"avoidance":24685,"cosmopolitan":24686,"geometridae":24687,"quartermaster":24688,"caf":24689,"415":24690,"chatter":24691,"engulfed":24692,"gleam":24693,"purge":24694,"##icate":24695,"juliette":24696,"jurisprudence":24697,"guerra":24698,"revisions":24699,"##bn":24700,"casimir":24701,"brew":24702,"##jm":24703,"1749":24704,"clapton":24705,"cloudy":24706,"conde":24707,"hermitage":24708,"278":24709,"simulations":24710,"torches":24711,"vincenzo":24712,"matteo":24713,"##rill":24714,"hidalgo":24715,"booming":24716,"westbound":24717,"accomplishment":24718,"tentacles":24719,"unaffected":24720,"##sius":24721,"annabelle":24722,"flopped":24723,"sloping":24724,"##litz":24725,"dreamer":24726,"interceptor":24727,"vu":24728,"##loh":24729,"consecration":24730,"copying":24731,"messaging":24732,"breaker":24733,"climates":24734,"hospitalized":24735,"1752":24736,"torino":24737,"afternoons":24738,"winfield":24739,"witnessing":24740,"##teacher":24741,"breakers":24742,"choirs":24743,"sawmill":24744,"coldly":24745,"##ege":24746,"sipping":24747,"haste":24748,"uninhabited":24749,"conical":24750,"bibliography":24751,"pamphlets":24752,"severn":24753,"edict":24754,"##oca":24755,"deux":24756,"illnesses":24757,"grips":24758,"##pl":24759,"rehearsals":24760,"sis":24761,"thinkers":24762,"tame":24763,"##keepers":24764,"1690":24765,"acacia":24766,"reformer":24767,"##osed":24768,"##rys":24769,"shuffling":24770,"##iring":24771,"##shima":24772,"eastbound":24773,"ionic":24774,"rhea":24775,"flees":24776,"littered":24777,"##oum":24778,"rocker":24779,"vomiting":24780,"groaning":24781,"champ":24782,"overwhelmingly":24783,"civilizations":24784,"paces":24785,"sloop":24786,"adoptive":24787,"##tish":24788,"skaters":24789,"##vres":24790,"aiding":24791,"mango":24792,"##joy":24793,"nikola":24794,"shriek":24795,"##ignon":24796,"pharmaceuticals":24797,"##mg":24798,"tuna":24799,"calvert":24800,"gustavo":24801,"stocked":24802,"yearbook":24803,"##urai":24804,"##mana":24805,"computed":24806,"subsp":24807,"riff":24808,"hanoi":24809,"kelvin":24810,"hamid":24811,"moors":24812,"pastures":24813,"summons":24814,"jihad":24815,"nectar":24816,"##ctors":24817,"bayou":24818,"untitled":24819,"pleasing":24820,"vastly":24821,"republics":24822,"intellect":24823,"##η":24824,"##ulio":24825,"##tou":24826,"crumbling":24827,"stylistic":24828,"sb":24829,"##ی":24830,"consolation":24831,"frequented":24832,"h₂o":24833,"walden":24834,"widows":24835,"##iens":24836,"404":24837,"##ignment":24838,"chunks":24839,"improves":24840,"288":24841,"grit":24842,"recited":24843,"##dev":24844,"snarl":24845,"sociological":24846,"##arte":24847,"##gul":24848,"inquired":24849,"##held":24850,"bruise":24851,"clube":24852,"consultancy":24853,"homogeneous":24854,"hornets":24855,"multiplication":24856,"pasta":24857,"prick":24858,"savior":24859,"##grin":24860,"##kou":24861,"##phile":24862,"yoon":24863,"##gara":24864,"grimes":24865,"vanishing":24866,"cheering":24867,"reacting":24868,"bn":24869,"distillery":24870,"##quisite":24871,"##vity":24872,"coe":24873,"dockyard":24874,"massif":24875,"##jord":24876,"escorts":24877,"voss":24878,"##valent":24879,"byte":24880,"chopped":24881,"hawke":24882,"illusions":24883,"workings":24884,"floats":24885,"##koto":24886,"##vac":24887,"kv":24888,"annapolis":24889,"madden":24890,"##onus":24891,"alvaro":24892,"noctuidae":24893,"##cum":24894,"##scopic":24895,"avenge":24896,"steamboat":24897,"forte":24898,"illustrates":24899,"erika":24900,"##trip":24901,"570":24902,"dew":24903,"nationalities":24904,"bran":24905,"manifested":24906,"thirsty":24907,"diversified":24908,"muscled":24909,"reborn":24910,"##standing":24911,"arson":24912,"##lessness":24913,"##dran":24914,"##logram":24915,"##boys":24916,"##kushima":24917,"##vious":24918,"willoughby":24919,"##phobia":24920,"286":24921,"alsace":24922,"dashboard":24923,"yuki":24924,"##chai":24925,"granville":24926,"myspace":24927,"publicized":24928,"tricked":24929,"##gang":24930,"adjective":24931,"##ater":24932,"relic":24933,"reorganisation":24934,"enthusiastically":24935,"indications":24936,"saxe":24937,"##lassified":24938,"consolidate":24939,"iec":24940,"padua":24941,"helplessly":24942,"ramps":24943,"renaming":24944,"regulars":24945,"pedestrians":24946,"accents":24947,"convicts":24948,"inaccurate":24949,"lowers":24950,"mana":24951,"##pati":24952,"barrie":24953,"bjp":24954,"outta":24955,"someplace":24956,"berwick":24957,"flanking":24958,"invoked":24959,"marrow":24960,"sparsely":24961,"excerpts":24962,"clothed":24963,"rei":24964,"##ginal":24965,"wept":24966,"##straße":24967,"##vish":24968,"alexa":24969,"excel":24970,"##ptive":24971,"membranes":24972,"aquitaine":24973,"creeks":24974,"cutler":24975,"sheppard":24976,"implementations":24977,"ns":24978,"##dur":24979,"fragrance":24980,"budge":24981,"concordia":24982,"magnesium":24983,"marcelo":24984,"##antes":24985,"gladly":24986,"vibrating":24987,"##rral":24988,"##ggles":24989,"montrose":24990,"##omba":24991,"lew":24992,"seamus":24993,"1630":24994,"cocky":24995,"##ament":24996,"##uen":24997,"bjorn":24998,"##rrick":24999,"fielder":25000,"fluttering":25001,"##lase":25002,"methyl":25003,"kimberley":25004,"mcdowell":25005,"reductions":25006,"barbed":25007,"##jic":25008,"##tonic":25009,"aeronautical":25010,"condensed":25011,"distracting":25012,"##promising":25013,"huffed":25014,"##cala":25015,"##sle":25016,"claudius":25017,"invincible":25018,"missy":25019,"pious":25020,"balthazar":25021,"ci":25022,"##lang":25023,"butte":25024,"combo":25025,"orson":25026,"##dication":25027,"myriad":25028,"1707":25029,"silenced":25030,"##fed":25031,"##rh":25032,"coco":25033,"netball":25034,"yourselves":25035,"##oza":25036,"clarify":25037,"heller":25038,"peg":25039,"durban":25040,"etudes":25041,"offender":25042,"roast":25043,"blackmail":25044,"curvature":25045,"##woods":25046,"vile":25047,"309":25048,"illicit":25049,"suriname":25050,"##linson":25051,"overture":25052,"1685":25053,"bubbling":25054,"gymnast":25055,"tucking":25056,"##mming":25057,"##ouin":25058,"maldives":25059,"##bala":25060,"gurney":25061,"##dda":25062,"##eased":25063,"##oides":25064,"backside":25065,"pinto":25066,"jars":25067,"racehorse":25068,"tending":25069,"##rdial":25070,"baronetcy":25071,"wiener":25072,"duly":25073,"##rke":25074,"barbarian":25075,"cupping":25076,"flawed":25077,"##thesis":25078,"bertha":25079,"pleistocene":25080,"puddle":25081,"swearing":25082,"##nob":25083,"##tically":25084,"fleeting":25085,"prostate":25086,"amulet":25087,"educating":25088,"##mined":25089,"##iti":25090,"##tler":25091,"75th":25092,"jens":25093,"respondents":25094,"analytics":25095,"cavaliers":25096,"papacy":25097,"raju":25098,"##iente":25099,"##ulum":25100,"##tip":25101,"funnel":25102,"271":25103,"disneyland":25104,"##lley":25105,"sociologist":25106,"##iam":25107,"2500":25108,"faulkner":25109,"louvre":25110,"menon":25111,"##dson":25112,"276":25113,"##ower":25114,"afterlife":25115,"mannheim":25116,"peptide":25117,"referees":25118,"comedians":25119,"meaningless":25120,"##anger":25121,"##laise":25122,"fabrics":25123,"hurley":25124,"renal":25125,"sleeps":25126,"##bour":25127,"##icle":25128,"breakout":25129,"kristin":25130,"roadside":25131,"animator":25132,"clover":25133,"disdain":25134,"unsafe":25135,"redesign":25136,"##urity":25137,"firth":25138,"barnsley":25139,"portage":25140,"reset":25141,"narrows":25142,"268":25143,"commandos":25144,"expansive":25145,"speechless":25146,"tubular":25147,"##lux":25148,"essendon":25149,"eyelashes":25150,"smashwords":25151,"##yad":25152,"##bang":25153,"##claim":25154,"craved":25155,"sprinted":25156,"chet":25157,"somme":25158,"astor":25159,"wrocław":25160,"orton":25161,"266":25162,"bane":25163,"##erving":25164,"##uing":25165,"mischief":25166,"##amps":25167,"##sund":25168,"scaling":25169,"terre":25170,"##xious":25171,"impairment":25172,"offenses":25173,"undermine":25174,"moi":25175,"soy":25176,"contiguous":25177,"arcadia":25178,"inuit":25179,"seam":25180,"##tops":25181,"macbeth":25182,"rebelled":25183,"##icative":25184,"##iot":25185,"590":25186,"elaborated":25187,"frs":25188,"uniformed":25189,"##dberg":25190,"259":25191,"powerless":25192,"priscilla":25193,"stimulated":25194,"980":25195,"qc":25196,"arboretum":25197,"frustrating":25198,"trieste":25199,"bullock":25200,"##nified":25201,"enriched":25202,"glistening":25203,"intern":25204,"##adia":25205,"locus":25206,"nouvelle":25207,"ollie":25208,"ike":25209,"lash":25210,"starboard":25211,"ee":25212,"tapestry":25213,"headlined":25214,"hove":25215,"rigged":25216,"##vite":25217,"pollock":25218,"##yme":25219,"thrive":25220,"clustered":25221,"cas":25222,"roi":25223,"gleamed":25224,"olympiad":25225,"##lino":25226,"pressured":25227,"regimes":25228,"##hosis":25229,"##lick":25230,"ripley":25231,"##ophone":25232,"kickoff":25233,"gallon":25234,"rockwell":25235,"##arable":25236,"crusader":25237,"glue":25238,"revolutions":25239,"scrambling":25240,"1714":25241,"grover":25242,"##jure":25243,"englishman":25244,"aztec":25245,"263":25246,"contemplating":25247,"coven":25248,"ipad":25249,"preach":25250,"triumphant":25251,"tufts":25252,"##esian":25253,"rotational":25254,"##phus":25255,"328":25256,"falkland":25257,"##brates":25258,"strewn":25259,"clarissa":25260,"rejoin":25261,"environmentally":25262,"glint":25263,"banded":25264,"drenched":25265,"moat":25266,"albanians":25267,"johor":25268,"rr":25269,"maestro":25270,"malley":25271,"nouveau":25272,"shaded":25273,"taxonomy":25274,"v6":25275,"adhere":25276,"bunk":25277,"airfields":25278,"##ritan":25279,"1741":25280,"encompass":25281,"remington":25282,"tran":25283,"##erative":25284,"amelie":25285,"mazda":25286,"friar":25287,"morals":25288,"passions":25289,"##zai":25290,"breadth":25291,"vis":25292,"##hae":25293,"argus":25294,"burnham":25295,"caressing":25296,"insider":25297,"rudd":25298,"##imov":25299,"##mini":25300,"##rso":25301,"italianate":25302,"murderous":25303,"textual":25304,"wainwright":25305,"armada":25306,"bam":25307,"weave":25308,"timer":25309,"##taken":25310,"##nh":25311,"fra":25312,"##crest":25313,"ardent":25314,"salazar":25315,"taps":25316,"tunis":25317,"##ntino":25318,"allegro":25319,"gland":25320,"philanthropic":25321,"##chester":25322,"implication":25323,"##optera":25324,"esq":25325,"judas":25326,"noticeably":25327,"wynn":25328,"##dara":25329,"inched":25330,"indexed":25331,"crises":25332,"villiers":25333,"bandit":25334,"royalties":25335,"patterned":25336,"cupboard":25337,"interspersed":25338,"accessory":25339,"isla":25340,"kendrick":25341,"entourage":25342,"stitches":25343,"##esthesia":25344,"headwaters":25345,"##ior":25346,"interlude":25347,"distraught":25348,"draught":25349,"1727":25350,"##basket":25351,"biased":25352,"sy":25353,"transient":25354,"triad":25355,"subgenus":25356,"adapting":25357,"kidd":25358,"shortstop":25359,"##umatic":25360,"dimly":25361,"spiked":25362,"mcleod":25363,"reprint":25364,"nellie":25365,"pretoria":25366,"windmill":25367,"##cek":25368,"singled":25369,"##mps":25370,"273":25371,"reunite":25372,"##orous":25373,"747":25374,"bankers":25375,"outlying":25376,"##omp":25377,"##ports":25378,"##tream":25379,"apologies":25380,"cosmetics":25381,"patsy":25382,"##deh":25383,"##ocks":25384,"##yson":25385,"bender":25386,"nantes":25387,"serene":25388,"##nad":25389,"lucha":25390,"mmm":25391,"323":25392,"##cius":25393,"##gli":25394,"cmll":25395,"coinage":25396,"nestor":25397,"juarez":25398,"##rook":25399,"smeared":25400,"sprayed":25401,"twitching":25402,"sterile":25403,"irina":25404,"embodied":25405,"juveniles":25406,"enveloped":25407,"miscellaneous":25408,"cancers":25409,"dq":25410,"gulped":25411,"luisa":25412,"crested":25413,"swat":25414,"donegal":25415,"ref":25416,"##anov":25417,"##acker":25418,"hearst":25419,"mercantile":25420,"##lika":25421,"doorbell":25422,"ua":25423,"vicki":25424,"##alla":25425,"##som":25426,"bilbao":25427,"psychologists":25428,"stryker":25429,"sw":25430,"horsemen":25431,"turkmenistan":25432,"wits":25433,"##national":25434,"anson":25435,"mathew":25436,"screenings":25437,"##umb":25438,"rihanna":25439,"##agne":25440,"##nessy":25441,"aisles":25442,"##iani":25443,"##osphere":25444,"hines":25445,"kenton":25446,"saskatoon":25447,"tasha":25448,"truncated":25449,"##champ":25450,"##itan":25451,"mildred":25452,"advises":25453,"fredrik":25454,"interpreting":25455,"inhibitors":25456,"##athi":25457,"spectroscopy":25458,"##hab":25459,"##kong":25460,"karim":25461,"panda":25462,"##oia":25463,"##nail":25464,"##vc":25465,"conqueror":25466,"kgb":25467,"leukemia":25468,"##dity":25469,"arrivals":25470,"cheered":25471,"pisa":25472,"phosphorus":25473,"shielded":25474,"##riated":25475,"mammal":25476,"unitarian":25477,"urgently":25478,"chopin":25479,"sanitary":25480,"##mission":25481,"spicy":25482,"drugged":25483,"hinges":25484,"##tort":25485,"tipping":25486,"trier":25487,"impoverished":25488,"westchester":25489,"##caster":25490,"267":25491,"epoch":25492,"nonstop":25493,"##gman":25494,"##khov":25495,"aromatic":25496,"centrally":25497,"cerro":25498,"##tively":25499,"##vio":25500,"billions":25501,"modulation":25502,"sedimentary":25503,"283":25504,"facilitating":25505,"outrageous":25506,"goldstein":25507,"##eak":25508,"##kt":25509,"ld":25510,"maitland":25511,"penultimate":25512,"pollard":25513,"##dance":25514,"fleets":25515,"spaceship":25516,"vertebrae":25517,"##nig":25518,"alcoholism":25519,"als":25520,"recital":25521,"##bham":25522,"##ference":25523,"##omics":25524,"m2":25525,"##bm":25526,"trois":25527,"##tropical":25528,"##в":25529,"commemorates":25530,"##meric":25531,"marge":25532,"##raction":25533,"1643":25534,"670":25535,"cosmetic":25536,"ravaged":25537,"##ige":25538,"catastrophe":25539,"eng":25540,"##shida":25541,"albrecht":25542,"arterial":25543,"bellamy":25544,"decor":25545,"harmon":25546,"##rde":25547,"bulbs":25548,"synchronized":25549,"vito":25550,"easiest":25551,"shetland":25552,"shielding":25553,"wnba":25554,"##glers":25555,"##ssar":25556,"##riam":25557,"brianna":25558,"cumbria":25559,"##aceous":25560,"##rard":25561,"cores":25562,"thayer":25563,"##nsk":25564,"brood":25565,"hilltop":25566,"luminous":25567,"carts":25568,"keynote":25569,"larkin":25570,"logos":25571,"##cta":25572,"##ا":25573,"##mund":25574,"##quay":25575,"lilith":25576,"tinted":25577,"277":25578,"wrestle":25579,"mobilization":25580,"##uses":25581,"sequential":25582,"siam":25583,"bloomfield":25584,"takahashi":25585,"274":25586,"##ieving":25587,"presenters":25588,"ringo":25589,"blazed":25590,"witty":25591,"##oven":25592,"##ignant":25593,"devastation":25594,"haydn":25595,"harmed":25596,"newt":25597,"therese":25598,"##peed":25599,"gershwin":25600,"molina":25601,"rabbis":25602,"sudanese":25603,"001":25604,"innate":25605,"restarted":25606,"##sack":25607,"##fus":25608,"slices":25609,"wb":25610,"##shah":25611,"enroll":25612,"hypothetical":25613,"hysterical":25614,"1743":25615,"fabio":25616,"indefinite":25617,"warped":25618,"##hg":25619,"exchanging":25620,"525":25621,"unsuitable":25622,"##sboro":25623,"gallo":25624,"1603":25625,"bret":25626,"cobalt":25627,"homemade":25628,"##hunter":25629,"mx":25630,"operatives":25631,"##dhar":25632,"terraces":25633,"durable":25634,"latch":25635,"pens":25636,"whorls":25637,"##ctuated":25638,"##eaux":25639,"billing":25640,"ligament":25641,"succumbed":25642,"##gly":25643,"regulators":25644,"spawn":25645,"##brick":25646,"##stead":25647,"filmfare":25648,"rochelle":25649,"##nzo":25650,"1725":25651,"circumstance":25652,"saber":25653,"supplements":25654,"##nsky":25655,"##tson":25656,"crowe":25657,"wellesley":25658,"carrot":25659,"##9th":25660,"##movable":25661,"primate":25662,"drury":25663,"sincerely":25664,"topical":25665,"##mad":25666,"##rao":25667,"callahan":25668,"kyiv":25669,"smarter":25670,"tits":25671,"undo":25672,"##yeh":25673,"announcements":25674,"anthologies":25675,"barrio":25676,"nebula":25677,"##islaus":25678,"##shaft":25679,"##tyn":25680,"bodyguards":25681,"2021":25682,"assassinate":25683,"barns":25684,"emmett":25685,"scully":25686,"##mah":25687,"##yd":25688,"##eland":25689,"##tino":25690,"##itarian":25691,"demoted":25692,"gorman":25693,"lashed":25694,"prized":25695,"adventist":25696,"writ":25697,"##gui":25698,"alla":25699,"invertebrates":25700,"##ausen":25701,"1641":25702,"amman":25703,"1742":25704,"align":25705,"healy":25706,"redistribution":25707,"##gf":25708,"##rize":25709,"insulation":25710,"##drop":25711,"adherents":25712,"hezbollah":25713,"vitro":25714,"ferns":25715,"yanking":25716,"269":25717,"php":25718,"registering":25719,"uppsala":25720,"cheerleading":25721,"confines":25722,"mischievous":25723,"tully":25724,"##ross":25725,"49th":25726,"docked":25727,"roam":25728,"stipulated":25729,"pumpkin":25730,"##bry":25731,"prompt":25732,"##ezer":25733,"blindly":25734,"shuddering":25735,"craftsmen":25736,"frail":25737,"scented":25738,"katharine":25739,"scramble":25740,"shaggy":25741,"sponge":25742,"helix":25743,"zaragoza":25744,"279":25745,"##52":25746,"43rd":25747,"backlash":25748,"fontaine":25749,"seizures":25750,"posse":25751,"cowan":25752,"nonfiction":25753,"telenovela":25754,"wwii":25755,"hammered":25756,"undone":25757,"##gpur":25758,"encircled":25759,"irs":25760,"##ivation":25761,"artefacts":25762,"oneself":25763,"searing":25764,"smallpox":25765,"##belle":25766,"##osaurus":25767,"shandong":25768,"breached":25769,"upland":25770,"blushing":25771,"rankin":25772,"infinitely":25773,"psyche":25774,"tolerated":25775,"docking":25776,"evicted":25777,"##col":25778,"unmarked":25779,"##lving":25780,"gnome":25781,"lettering":25782,"litres":25783,"musique":25784,"##oint":25785,"benevolent":25786,"##jal":25787,"blackened":25788,"##anna":25789,"mccall":25790,"racers":25791,"tingle":25792,"##ocene":25793,"##orestation":25794,"introductions":25795,"radically":25796,"292":25797,"##hiff":25798,"##باد":25799,"1610":25800,"1739":25801,"munchen":25802,"plead":25803,"##nka":25804,"condo":25805,"scissors":25806,"##sight":25807,"##tens":25808,"apprehension":25809,"##cey":25810,"##yin":25811,"hallmark":25812,"watering":25813,"formulas":25814,"sequels":25815,"##llas":25816,"aggravated":25817,"bae":25818,"commencing":25819,"##building":25820,"enfield":25821,"prohibits":25822,"marne":25823,"vedic":25824,"civilized":25825,"euclidean":25826,"jagger":25827,"beforehand":25828,"blasts":25829,"dumont":25830,"##arney":25831,"##nem":25832,"740":25833,"conversions":25834,"hierarchical":25835,"rios":25836,"simulator":25837,"##dya":25838,"##lellan":25839,"hedges":25840,"oleg":25841,"thrusts":25842,"shadowed":25843,"darby":25844,"maximize":25845,"1744":25846,"gregorian":25847,"##nded":25848,"##routed":25849,"sham":25850,"unspecified":25851,"##hog":25852,"emory":25853,"factual":25854,"##smo":25855,"##tp":25856,"fooled":25857,"##rger":25858,"ortega":25859,"wellness":25860,"marlon":25861,"##oton":25862,"##urance":25863,"casket":25864,"keating":25865,"ley":25866,"enclave":25867,"##ayan":25868,"char":25869,"influencing":25870,"jia":25871,"##chenko":25872,"412":25873,"ammonia":25874,"erebidae":25875,"incompatible":25876,"violins":25877,"cornered":25878,"##arat":25879,"grooves":25880,"astronauts":25881,"columbian":25882,"rampant":25883,"fabrication":25884,"kyushu":25885,"mahmud":25886,"vanish":25887,"##dern":25888,"mesopotamia":25889,"##lete":25890,"ict":25891,"##rgen":25892,"caspian":25893,"kenji":25894,"pitted":25895,"##vered":25896,"999":25897,"grimace":25898,"roanoke":25899,"tchaikovsky":25900,"twinned":25901,"##analysis":25902,"##awan":25903,"xinjiang":25904,"arias":25905,"clemson":25906,"kazakh":25907,"sizable":25908,"1662":25909,"##khand":25910,"##vard":25911,"plunge":25912,"tatum":25913,"vittorio":25914,"##nden":25915,"cholera":25916,"##dana":25917,"##oper":25918,"bracing":25919,"indifference":25920,"projectile":25921,"superliga":25922,"##chee":25923,"realises":25924,"upgrading":25925,"299":25926,"porte":25927,"retribution":25928,"##vies":25929,"nk":25930,"stil":25931,"##resses":25932,"ama":25933,"bureaucracy":25934,"blackberry":25935,"bosch":25936,"testosterone":25937,"collapses":25938,"greer":25939,"##pathic":25940,"ioc":25941,"fifties":25942,"malls":25943,"##erved":25944,"bao":25945,"baskets":25946,"adolescents":25947,"siegfried":25948,"##osity":25949,"##tosis":25950,"mantra":25951,"detecting":25952,"existent":25953,"fledgling":25954,"##cchi":25955,"dissatisfied":25956,"gan":25957,"telecommunication":25958,"mingled":25959,"sobbed":25960,"6000":25961,"controversies":25962,"outdated":25963,"taxis":25964,"##raus":25965,"fright":25966,"slams":25967,"##lham":25968,"##fect":25969,"##tten":25970,"detectors":25971,"fetal":25972,"tanned":25973,"##uw":25974,"fray":25975,"goth":25976,"olympian":25977,"skipping":25978,"mandates":25979,"scratches":25980,"sheng":25981,"unspoken":25982,"hyundai":25983,"tracey":25984,"hotspur":25985,"restrictive":25986,"##buch":25987,"americana":25988,"mundo":25989,"##bari":25990,"burroughs":25991,"diva":25992,"vulcan":25993,"##6th":25994,"distinctions":25995,"thumping":25996,"##ngen":25997,"mikey":25998,"sheds":25999,"fide":26000,"rescues":26001,"springsteen":26002,"vested":26003,"valuation":26004,"##ece":26005,"##ely":26006,"pinnacle":26007,"rake":26008,"sylvie":26009,"##edo":26010,"almond":26011,"quivering":26012,"##irus":26013,"alteration":26014,"faltered":26015,"##wad":26016,"51st":26017,"hydra":26018,"ticked":26019,"##kato":26020,"recommends":26021,"##dicated":26022,"antigua":26023,"arjun":26024,"stagecoach":26025,"wilfred":26026,"trickle":26027,"pronouns":26028,"##pon":26029,"aryan":26030,"nighttime":26031,"##anian":26032,"gall":26033,"pea":26034,"stitch":26035,"##hei":26036,"leung":26037,"milos":26038,"##dini":26039,"eritrea":26040,"nexus":26041,"starved":26042,"snowfall":26043,"kant":26044,"parasitic":26045,"cot":26046,"discus":26047,"hana":26048,"strikers":26049,"appleton":26050,"kitchens":26051,"##erina":26052,"##partisan":26053,"##itha":26054,"##vius":26055,"disclose":26056,"metis":26057,"##channel":26058,"1701":26059,"tesla":26060,"##vera":26061,"fitch":26062,"1735":26063,"blooded":26064,"##tila":26065,"decimal":26066,"##tang":26067,"##bai":26068,"cyclones":26069,"eun":26070,"bottled":26071,"peas":26072,"pensacola":26073,"basha":26074,"bolivian":26075,"crabs":26076,"boil":26077,"lanterns":26078,"partridge":26079,"roofed":26080,"1645":26081,"necks":26082,"##phila":26083,"opined":26084,"patting":26085,"##kla":26086,"##lland":26087,"chuckles":26088,"volta":26089,"whereupon":26090,"##nche":26091,"devout":26092,"euroleague":26093,"suicidal":26094,"##dee":26095,"inherently":26096,"involuntary":26097,"knitting":26098,"nasser":26099,"##hide":26100,"puppets":26101,"colourful":26102,"courageous":26103,"southend":26104,"stills":26105,"miraculous":26106,"hodgson":26107,"richer":26108,"rochdale":26109,"ethernet":26110,"greta":26111,"uniting":26112,"prism":26113,"umm":26114,"##haya":26115,"##itical":26116,"##utation":26117,"deterioration":26118,"pointe":26119,"prowess":26120,"##ropriation":26121,"lids":26122,"scranton":26123,"billings":26124,"subcontinent":26125,"##koff":26126,"##scope":26127,"brute":26128,"kellogg":26129,"psalms":26130,"degraded":26131,"##vez":26132,"stanisław":26133,"##ructured":26134,"ferreira":26135,"pun":26136,"astonishing":26137,"gunnar":26138,"##yat":26139,"arya":26140,"prc":26141,"gottfried":26142,"##tight":26143,"excursion":26144,"##ographer":26145,"dina":26146,"##quil":26147,"##nare":26148,"huffington":26149,"illustrious":26150,"wilbur":26151,"gundam":26152,"verandah":26153,"##zard":26154,"naacp":26155,"##odle":26156,"constructive":26157,"fjord":26158,"kade":26159,"##naud":26160,"generosity":26161,"thrilling":26162,"baseline":26163,"cayman":26164,"frankish":26165,"plastics":26166,"accommodations":26167,"zoological":26168,"##fting":26169,"cedric":26170,"qb":26171,"motorized":26172,"##dome":26173,"##otted":26174,"squealed":26175,"tackled":26176,"canucks":26177,"budgets":26178,"situ":26179,"asthma":26180,"dail":26181,"gabled":26182,"grasslands":26183,"whimpered":26184,"writhing":26185,"judgments":26186,"##65":26187,"minnie":26188,"pv":26189,"##carbon":26190,"bananas":26191,"grille":26192,"domes":26193,"monique":26194,"odin":26195,"maguire":26196,"markham":26197,"tierney":26198,"##estra":26199,"##chua":26200,"libel":26201,"poke":26202,"speedy":26203,"atrium":26204,"laval":26205,"notwithstanding":26206,"##edly":26207,"fai":26208,"kala":26209,"##sur":26210,"robb":26211,"##sma":26212,"listings":26213,"luz":26214,"supplementary":26215,"tianjin":26216,"##acing":26217,"enzo":26218,"jd":26219,"ric":26220,"scanner":26221,"croats":26222,"transcribed":26223,"##49":26224,"arden":26225,"cv":26226,"##hair":26227,"##raphy":26228,"##lver":26229,"##uy":26230,"357":26231,"seventies":26232,"staggering":26233,"alam":26234,"horticultural":26235,"hs":26236,"regression":26237,"timbers":26238,"blasting":26239,"##ounded":26240,"montagu":26241,"manipulating":26242,"##cit":26243,"catalytic":26244,"1550":26245,"troopers":26246,"##meo":26247,"condemnation":26248,"fitzpatrick":26249,"##oire":26250,"##roved":26251,"inexperienced":26252,"1670":26253,"castes":26254,"##lative":26255,"outing":26256,"314":26257,"dubois":26258,"flicking":26259,"quarrel":26260,"ste":26261,"learners":26262,"1625":26263,"iq":26264,"whistled":26265,"##class":26266,"282":26267,"classify":26268,"tariffs":26269,"temperament":26270,"355":26271,"folly":26272,"liszt":26273,"##yles":26274,"immersed":26275,"jordanian":26276,"ceasefire":26277,"apparel":26278,"extras":26279,"maru":26280,"fished":26281,"##bio":26282,"harta":26283,"stockport":26284,"assortment":26285,"craftsman":26286,"paralysis":26287,"transmitters":26288,"##cola":26289,"blindness":26290,"##wk":26291,"fatally":26292,"proficiency":26293,"solemnly":26294,"##orno":26295,"repairing":26296,"amore":26297,"groceries":26298,"ultraviolet":26299,"##chase":26300,"schoolhouse":26301,"##tua":26302,"resurgence":26303,"nailed":26304,"##otype":26305,"##×":26306,"ruse":26307,"saliva":26308,"diagrams":26309,"##tructing":26310,"albans":26311,"rann":26312,"thirties":26313,"1b":26314,"antennas":26315,"hilarious":26316,"cougars":26317,"paddington":26318,"stats":26319,"##eger":26320,"breakaway":26321,"ipod":26322,"reza":26323,"authorship":26324,"prohibiting":26325,"scoffed":26326,"##etz":26327,"##ttle":26328,"conscription":26329,"defected":26330,"trondheim":26331,"##fires":26332,"ivanov":26333,"keenan":26334,"##adan":26335,"##ciful":26336,"##fb":26337,"##slow":26338,"locating":26339,"##ials":26340,"##tford":26341,"cadiz":26342,"basalt":26343,"blankly":26344,"interned":26345,"rags":26346,"rattling":26347,"##tick":26348,"carpathian":26349,"reassured":26350,"sync":26351,"bum":26352,"guildford":26353,"iss":26354,"staunch":26355,"##onga":26356,"astronomers":26357,"sera":26358,"sofie":26359,"emergencies":26360,"susquehanna":26361,"##heard":26362,"duc":26363,"mastery":26364,"vh1":26365,"williamsburg":26366,"bayer":26367,"buckled":26368,"craving":26369,"##khan":26370,"##rdes":26371,"bloomington":26372,"##write":26373,"alton":26374,"barbecue":26375,"##bians":26376,"justine":26377,"##hri":26378,"##ndt":26379,"delightful":26380,"smartphone":26381,"newtown":26382,"photon":26383,"retrieval":26384,"peugeot":26385,"hissing":26386,"##monium":26387,"##orough":26388,"flavors":26389,"lighted":26390,"relaunched":26391,"tainted":26392,"##games":26393,"##lysis":26394,"anarchy":26395,"microscopic":26396,"hopping":26397,"adept":26398,"evade":26399,"evie":26400,"##beau":26401,"inhibit":26402,"sinn":26403,"adjustable":26404,"hurst":26405,"intuition":26406,"wilton":26407,"cisco":26408,"44th":26409,"lawful":26410,"lowlands":26411,"stockings":26412,"thierry":26413,"##dalen":26414,"##hila":26415,"##nai":26416,"fates":26417,"prank":26418,"tb":26419,"maison":26420,"lobbied":26421,"provocative":26422,"1724":26423,"4a":26424,"utopia":26425,"##qual":26426,"carbonate":26427,"gujarati":26428,"purcell":26429,"##rford":26430,"curtiss":26431,"##mei":26432,"overgrown":26433,"arenas":26434,"mediation":26435,"swallows":26436,"##rnik":26437,"respectful":26438,"turnbull":26439,"##hedron":26440,"##hope":26441,"alyssa":26442,"ozone":26443,"##ʻi":26444,"ami":26445,"gestapo":26446,"johansson":26447,"snooker":26448,"canteen":26449,"cuff":26450,"declines":26451,"empathy":26452,"stigma":26453,"##ags":26454,"##iner":26455,"##raine":26456,"taxpayers":26457,"gui":26458,"volga":26459,"##wright":26460,"##copic":26461,"lifespan":26462,"overcame":26463,"tattooed":26464,"enactment":26465,"giggles":26466,"##ador":26467,"##camp":26468,"barrington":26469,"bribe":26470,"obligatory":26471,"orbiting":26472,"peng":26473,"##enas":26474,"elusive":26475,"sucker":26476,"##vating":26477,"cong":26478,"hardship":26479,"empowered":26480,"anticipating":26481,"estrada":26482,"cryptic":26483,"greasy":26484,"detainees":26485,"planck":26486,"sudbury":26487,"plaid":26488,"dod":26489,"marriott":26490,"kayla":26491,"##ears":26492,"##vb":26493,"##zd":26494,"mortally":26495,"##hein":26496,"cognition":26497,"radha":26498,"319":26499,"liechtenstein":26500,"meade":26501,"richly":26502,"argyle":26503,"harpsichord":26504,"liberalism":26505,"trumpets":26506,"lauded":26507,"tyrant":26508,"salsa":26509,"tiled":26510,"lear":26511,"promoters":26512,"reused":26513,"slicing":26514,"trident":26515,"##chuk":26516,"##gami":26517,"##lka":26518,"cantor":26519,"checkpoint":26520,"##points":26521,"gaul":26522,"leger":26523,"mammalian":26524,"##tov":26525,"##aar":26526,"##schaft":26527,"doha":26528,"frenchman":26529,"nirvana":26530,"##vino":26531,"delgado":26532,"headlining":26533,"##eron":26534,"##iography":26535,"jug":26536,"tko":26537,"1649":26538,"naga":26539,"intersections":26540,"##jia":26541,"benfica":26542,"nawab":26543,"##suka":26544,"ashford":26545,"gulp":26546,"##deck":26547,"##vill":26548,"##rug":26549,"brentford":26550,"frazier":26551,"pleasures":26552,"dunne":26553,"potsdam":26554,"shenzhen":26555,"dentistry":26556,"##tec":26557,"flanagan":26558,"##dorff":26559,"##hear":26560,"chorale":26561,"dinah":26562,"prem":26563,"quezon":26564,"##rogated":26565,"relinquished":26566,"sutra":26567,"terri":26568,"##pani":26569,"flaps":26570,"##rissa":26571,"poly":26572,"##rnet":26573,"homme":26574,"aback":26575,"##eki":26576,"linger":26577,"womb":26578,"##kson":26579,"##lewood":26580,"doorstep":26581,"orthodoxy":26582,"threaded":26583,"westfield":26584,"##rval":26585,"dioceses":26586,"fridays":26587,"subsided":26588,"##gata":26589,"loyalists":26590,"##biotic":26591,"##ettes":26592,"letterman":26593,"lunatic":26594,"prelate":26595,"tenderly":26596,"invariably":26597,"souza":26598,"thug":26599,"winslow":26600,"##otide":26601,"furlongs":26602,"gogh":26603,"jeopardy":26604,"##runa":26605,"pegasus":26606,"##umble":26607,"humiliated":26608,"standalone":26609,"tagged":26610,"##roller":26611,"freshmen":26612,"klan":26613,"##bright":26614,"attaining":26615,"initiating":26616,"transatlantic":26617,"logged":26618,"viz":26619,"##uance":26620,"1723":26621,"combatants":26622,"intervening":26623,"stephane":26624,"chieftain":26625,"despised":26626,"grazed":26627,"317":26628,"cdc":26629,"galveston":26630,"godzilla":26631,"macro":26632,"simulate":26633,"##planes":26634,"parades":26635,"##esses":26636,"960":26637,"##ductive":26638,"##unes":26639,"equator":26640,"overdose":26641,"##cans":26642,"##hosh":26643,"##lifting":26644,"joshi":26645,"epstein":26646,"sonora":26647,"treacherous":26648,"aquatics":26649,"manchu":26650,"responsive":26651,"##sation":26652,"supervisory":26653,"##christ":26654,"##llins":26655,"##ibar":26656,"##balance":26657,"##uso":26658,"kimball":26659,"karlsruhe":26660,"mab":26661,"##emy":26662,"ignores":26663,"phonetic":26664,"reuters":26665,"spaghetti":26666,"820":26667,"almighty":26668,"danzig":26669,"rumbling":26670,"tombstone":26671,"designations":26672,"lured":26673,"outset":26674,"##felt":26675,"supermarkets":26676,"##wt":26677,"grupo":26678,"kei":26679,"kraft":26680,"susanna":26681,"##blood":26682,"comprehension":26683,"genealogy":26684,"##aghan":26685,"##verted":26686,"redding":26687,"##ythe":26688,"1722":26689,"bowing":26690,"##pore":26691,"##roi":26692,"lest":26693,"sharpened":26694,"fulbright":26695,"valkyrie":26696,"sikhs":26697,"##unds":26698,"swans":26699,"bouquet":26700,"merritt":26701,"##tage":26702,"##venting":26703,"commuted":26704,"redhead":26705,"clerks":26706,"leasing":26707,"cesare":26708,"dea":26709,"hazy":26710,"##vances":26711,"fledged":26712,"greenfield":26713,"servicemen":26714,"##gical":26715,"armando":26716,"blackout":26717,"dt":26718,"sagged":26719,"downloadable":26720,"intra":26721,"potion":26722,"pods":26723,"##4th":26724,"##mism":26725,"xp":26726,"attendants":26727,"gambia":26728,"stale":26729,"##ntine":26730,"plump":26731,"asteroids":26732,"rediscovered":26733,"buds":26734,"flea":26735,"hive":26736,"##neas":26737,"1737":26738,"classifications":26739,"debuts":26740,"##eles":26741,"olympus":26742,"scala":26743,"##eurs":26744,"##gno":26745,"##mute":26746,"hummed":26747,"sigismund":26748,"visuals":26749,"wiggled":26750,"await":26751,"pilasters":26752,"clench":26753,"sulfate":26754,"##ances":26755,"bellevue":26756,"enigma":26757,"trainee":26758,"snort":26759,"##sw":26760,"clouded":26761,"denim":26762,"##rank":26763,"##rder":26764,"churning":26765,"hartman":26766,"lodges":26767,"riches":26768,"sima":26769,"##missible":26770,"accountable":26771,"socrates":26772,"regulates":26773,"mueller":26774,"##cr":26775,"1702":26776,"avoids":26777,"solids":26778,"himalayas":26779,"nutrient":26780,"pup":26781,"##jevic":26782,"squat":26783,"fades":26784,"nec":26785,"##lates":26786,"##pina":26787,"##rona":26788,"##ου":26789,"privateer":26790,"tequila":26791,"##gative":26792,"##mpton":26793,"apt":26794,"hornet":26795,"immortals":26796,"##dou":26797,"asturias":26798,"cleansing":26799,"dario":26800,"##rries":26801,"##anta":26802,"etymology":26803,"servicing":26804,"zhejiang":26805,"##venor":26806,"##nx":26807,"horned":26808,"erasmus":26809,"rayon":26810,"relocating":26811,"£10":26812,"##bags":26813,"escalated":26814,"promenade":26815,"stubble":26816,"2010s":26817,"artisans":26818,"axial":26819,"liquids":26820,"mora":26821,"sho":26822,"yoo":26823,"##tsky":26824,"bundles":26825,"oldies":26826,"##nally":26827,"notification":26828,"bastion":26829,"##ths":26830,"sparkle":26831,"##lved":26832,"1728":26833,"leash":26834,"pathogen":26835,"highs":26836,"##hmi":26837,"immature":26838,"880":26839,"gonzaga":26840,"ignatius":26841,"mansions":26842,"monterrey":26843,"sweets":26844,"bryson":26845,"##loe":26846,"polled":26847,"regatta":26848,"brightest":26849,"pei":26850,"rosy":26851,"squid":26852,"hatfield":26853,"payroll":26854,"addict":26855,"meath":26856,"cornerback":26857,"heaviest":26858,"lodging":26859,"##mage":26860,"capcom":26861,"rippled":26862,"##sily":26863,"barnet":26864,"mayhem":26865,"ymca":26866,"snuggled":26867,"rousseau":26868,"##cute":26869,"blanchard":26870,"284":26871,"fragmented":26872,"leighton":26873,"chromosomes":26874,"risking":26875,"##md":26876,"##strel":26877,"##utter":26878,"corinne":26879,"coyotes":26880,"cynical":26881,"hiroshi":26882,"yeomanry":26883,"##ractive":26884,"ebook":26885,"grading":26886,"mandela":26887,"plume":26888,"agustin":26889,"magdalene":26890,"##rkin":26891,"bea":26892,"femme":26893,"trafford":26894,"##coll":26895,"##lun":26896,"##tance":26897,"52nd":26898,"fourier":26899,"upton":26900,"##mental":26901,"camilla":26902,"gust":26903,"iihf":26904,"islamabad":26905,"longevity":26906,"##kala":26907,"feldman":26908,"netting":26909,"##rization":26910,"endeavour":26911,"foraging":26912,"mfa":26913,"orr":26914,"##open":26915,"greyish":26916,"contradiction":26917,"graz":26918,"##ruff":26919,"handicapped":26920,"marlene":26921,"tweed":26922,"oaxaca":26923,"spp":26924,"campos":26925,"miocene":26926,"pri":26927,"configured":26928,"cooks":26929,"pluto":26930,"cozy":26931,"pornographic":26932,"##entes":26933,"70th":26934,"fairness":26935,"glided":26936,"jonny":26937,"lynne":26938,"rounding":26939,"sired":26940,"##emon":26941,"##nist":26942,"remade":26943,"uncover":26944,"##mack":26945,"complied":26946,"lei":26947,"newsweek":26948,"##jured":26949,"##parts":26950,"##enting":26951,"##pg":26952,"293":26953,"finer":26954,"guerrillas":26955,"athenian":26956,"deng":26957,"disused":26958,"stepmother":26959,"accuse":26960,"gingerly":26961,"seduction":26962,"521":26963,"confronting":26964,"##walker":26965,"##going":26966,"gora":26967,"nostalgia":26968,"sabres":26969,"virginity":26970,"wrenched":26971,"##minated":26972,"syndication":26973,"wielding":26974,"eyre":26975,"##56":26976,"##gnon":26977,"##igny":26978,"behaved":26979,"taxpayer":26980,"sweeps":26981,"##growth":26982,"childless":26983,"gallant":26984,"##ywood":26985,"amplified":26986,"geraldine":26987,"scrape":26988,"##ffi":26989,"babylonian":26990,"fresco":26991,"##rdan":26992,"##kney":26993,"##position":26994,"1718":26995,"restricting":26996,"tack":26997,"fukuoka":26998,"osborn":26999,"selector":27000,"partnering":27001,"##dlow":27002,"318":27003,"gnu":27004,"kia":27005,"tak":27006,"whitley":27007,"gables":27008,"##54":27009,"##mania":27010,"mri":27011,"softness":27012,"immersion":27013,"##bots":27014,"##evsky":27015,"1713":27016,"chilling":27017,"insignificant":27018,"pcs":27019,"##uis":27020,"elites":27021,"lina":27022,"purported":27023,"supplemental":27024,"teaming":27025,"##americana":27026,"##dding":27027,"##inton":27028,"proficient":27029,"rouen":27030,"##nage":27031,"##rret":27032,"niccolo":27033,"selects":27034,"##bread":27035,"fluffy":27036,"1621":27037,"gruff":27038,"knotted":27039,"mukherjee":27040,"polgara":27041,"thrash":27042,"nicholls":27043,"secluded":27044,"smoothing":27045,"thru":27046,"corsica":27047,"loaf":27048,"whitaker":27049,"inquiries":27050,"##rrier":27051,"##kam":27052,"indochina":27053,"289":27054,"marlins":27055,"myles":27056,"peking":27057,"##tea":27058,"extracts":27059,"pastry":27060,"superhuman":27061,"connacht":27062,"vogel":27063,"##ditional":27064,"##het":27065,"##udged":27066,"##lash":27067,"gloss":27068,"quarries":27069,"refit":27070,"teaser":27071,"##alic":27072,"##gaon":27073,"20s":27074,"materialized":27075,"sling":27076,"camped":27077,"pickering":27078,"tung":27079,"tracker":27080,"pursuant":27081,"##cide":27082,"cranes":27083,"soc":27084,"##cini":27085,"##typical":27086,"##viere":27087,"anhalt":27088,"overboard":27089,"workout":27090,"chores":27091,"fares":27092,"orphaned":27093,"stains":27094,"##logie":27095,"fenton":27096,"surpassing":27097,"joyah":27098,"triggers":27099,"##itte":27100,"grandmaster":27101,"##lass":27102,"##lists":27103,"clapping":27104,"fraudulent":27105,"ledger":27106,"nagasaki":27107,"##cor":27108,"##nosis":27109,"##tsa":27110,"eucalyptus":27111,"tun":27112,"##icio":27113,"##rney":27114,"##tara":27115,"dax":27116,"heroism":27117,"ina":27118,"wrexham":27119,"onboard":27120,"unsigned":27121,"##dates":27122,"moshe":27123,"galley":27124,"winnie":27125,"droplets":27126,"exiles":27127,"praises":27128,"watered":27129,"noodles":27130,"##aia":27131,"fein":27132,"adi":27133,"leland":27134,"multicultural":27135,"stink":27136,"bingo":27137,"comets":27138,"erskine":27139,"modernized":27140,"canned":27141,"constraint":27142,"domestically":27143,"chemotherapy":27144,"featherweight":27145,"stifled":27146,"##mum":27147,"darkly":27148,"irresistible":27149,"refreshing":27150,"hasty":27151,"isolate":27152,"##oys":27153,"kitchener":27154,"planners":27155,"##wehr":27156,"cages":27157,"yarn":27158,"implant":27159,"toulon":27160,"elects":27161,"childbirth":27162,"yue":27163,"##lind":27164,"##lone":27165,"cn":27166,"rightful":27167,"sportsman":27168,"junctions":27169,"remodeled":27170,"specifies":27171,"##rgh":27172,"291":27173,"##oons":27174,"complimented":27175,"##urgent":27176,"lister":27177,"ot":27178,"##logic":27179,"bequeathed":27180,"cheekbones":27181,"fontana":27182,"gabby":27183,"##dial":27184,"amadeus":27185,"corrugated":27186,"maverick":27187,"resented":27188,"triangles":27189,"##hered":27190,"##usly":27191,"nazareth":27192,"tyrol":27193,"1675":27194,"assent":27195,"poorer":27196,"sectional":27197,"aegean":27198,"##cous":27199,"296":27200,"nylon":27201,"ghanaian":27202,"##egorical":27203,"##weig":27204,"cushions":27205,"forbid":27206,"fusiliers":27207,"obstruction":27208,"somerville":27209,"##scia":27210,"dime":27211,"earrings":27212,"elliptical":27213,"leyte":27214,"oder":27215,"polymers":27216,"timmy":27217,"atm":27218,"midtown":27219,"piloted":27220,"settles":27221,"continual":27222,"externally":27223,"mayfield":27224,"##uh":27225,"enrichment":27226,"henson":27227,"keane":27228,"persians":27229,"1733":27230,"benji":27231,"braden":27232,"pep":27233,"324":27234,"##efe":27235,"contenders":27236,"pepsi":27237,"valet":27238,"##isches":27239,"298":27240,"##asse":27241,"##earing":27242,"goofy":27243,"stroll":27244,"##amen":27245,"authoritarian":27246,"occurrences":27247,"adversary":27248,"ahmedabad":27249,"tangent":27250,"toppled":27251,"dorchester":27252,"1672":27253,"modernism":27254,"marxism":27255,"islamist":27256,"charlemagne":27257,"exponential":27258,"racks":27259,"unicode":27260,"brunette":27261,"mbc":27262,"pic":27263,"skirmish":27264,"##bund":27265,"##lad":27266,"##powered":27267,"##yst":27268,"hoisted":27269,"messina":27270,"shatter":27271,"##ctum":27272,"jedi":27273,"vantage":27274,"##music":27275,"##neil":27276,"clemens":27277,"mahmoud":27278,"corrupted":27279,"authentication":27280,"lowry":27281,"nils":27282,"##washed":27283,"omnibus":27284,"wounding":27285,"jillian":27286,"##itors":27287,"##opped":27288,"serialized":27289,"narcotics":27290,"handheld":27291,"##arm":27292,"##plicity":27293,"intersecting":27294,"stimulating":27295,"##onis":27296,"crate":27297,"fellowships":27298,"hemingway":27299,"casinos":27300,"climatic":27301,"fordham":27302,"copeland":27303,"drip":27304,"beatty":27305,"leaflets":27306,"robber":27307,"brothel":27308,"madeira":27309,"##hedral":27310,"sphinx":27311,"ultrasound":27312,"##vana":27313,"valor":27314,"forbade":27315,"leonid":27316,"villas":27317,"##aldo":27318,"duane":27319,"marquez":27320,"##cytes":27321,"disadvantaged":27322,"forearms":27323,"kawasaki":27324,"reacts":27325,"consular":27326,"lax":27327,"uncles":27328,"uphold":27329,"##hopper":27330,"concepcion":27331,"dorsey":27332,"lass":27333,"##izan":27334,"arching":27335,"passageway":27336,"1708":27337,"researches":27338,"tia":27339,"internationals":27340,"##graphs":27341,"##opers":27342,"distinguishes":27343,"javanese":27344,"divert":27345,"##uven":27346,"plotted":27347,"##listic":27348,"##rwin":27349,"##erik":27350,"##tify":27351,"affirmative":27352,"signifies":27353,"validation":27354,"##bson":27355,"kari":27356,"felicity":27357,"georgina":27358,"zulu":27359,"##eros":27360,"##rained":27361,"##rath":27362,"overcoming":27363,"##dot":27364,"argyll":27365,"##rbin":27366,"1734":27367,"chiba":27368,"ratification":27369,"windy":27370,"earls":27371,"parapet":27372,"##marks":27373,"hunan":27374,"pristine":27375,"astrid":27376,"punta":27377,"##gart":27378,"brodie":27379,"##kota":27380,"##oder":27381,"malaga":27382,"minerva":27383,"rouse":27384,"##phonic":27385,"bellowed":27386,"pagoda":27387,"portals":27388,"reclamation":27389,"##gur":27390,"##odies":27391,"##⁄₄":27392,"parentheses":27393,"quoting":27394,"allergic":27395,"palette":27396,"showcases":27397,"benefactor":27398,"heartland":27399,"nonlinear":27400,"##tness":27401,"bladed":27402,"cheerfully":27403,"scans":27404,"##ety":27405,"##hone":27406,"1666":27407,"girlfriends":27408,"pedersen":27409,"hiram":27410,"sous":27411,"##liche":27412,"##nator":27413,"1683":27414,"##nery":27415,"##orio":27416,"##umen":27417,"bobo":27418,"primaries":27419,"smiley":27420,"##cb":27421,"unearthed":27422,"uniformly":27423,"fis":27424,"metadata":27425,"1635":27426,"ind":27427,"##oted":27428,"recoil":27429,"##titles":27430,"##tura":27431,"##ια":27432,"406":27433,"hilbert":27434,"jamestown":27435,"mcmillan":27436,"tulane":27437,"seychelles":27438,"##frid":27439,"antics":27440,"coli":27441,"fated":27442,"stucco":27443,"##grants":27444,"1654":27445,"bulky":27446,"accolades":27447,"arrays":27448,"caledonian":27449,"carnage":27450,"optimism":27451,"puebla":27452,"##tative":27453,"##cave":27454,"enforcing":27455,"rotherham":27456,"seo":27457,"dunlop":27458,"aeronautics":27459,"chimed":27460,"incline":27461,"zoning":27462,"archduke":27463,"hellenistic":27464,"##oses":27465,"##sions":27466,"candi":27467,"thong":27468,"##ople":27469,"magnate":27470,"rustic":27471,"##rsk":27472,"projective":27473,"slant":27474,"##offs":27475,"danes":27476,"hollis":27477,"vocalists":27478,"##ammed":27479,"congenital":27480,"contend":27481,"gesellschaft":27482,"##ocating":27483,"##pressive":27484,"douglass":27485,"quieter":27486,"##cm":27487,"##kshi":27488,"howled":27489,"salim":27490,"spontaneously":27491,"townsville":27492,"buena":27493,"southport":27494,"##bold":27495,"kato":27496,"1638":27497,"faerie":27498,"stiffly":27499,"##vus":27500,"##rled":27501,"297":27502,"flawless":27503,"realising":27504,"taboo":27505,"##7th":27506,"bytes":27507,"straightening":27508,"356":27509,"jena":27510,"##hid":27511,"##rmin":27512,"cartwright":27513,"berber":27514,"bertram":27515,"soloists":27516,"411":27517,"noses":27518,"417":27519,"coping":27520,"fission":27521,"hardin":27522,"inca":27523,"##cen":27524,"1717":27525,"mobilized":27526,"vhf":27527,"##raf":27528,"biscuits":27529,"curate":27530,"##85":27531,"##anial":27532,"331":27533,"gaunt":27534,"neighbourhoods":27535,"1540":27536,"##abas":27537,"blanca":27538,"bypassed":27539,"sockets":27540,"behold":27541,"coincidentally":27542,"##bane":27543,"nara":27544,"shave":27545,"splinter":27546,"terrific":27547,"##arion":27548,"##erian":27549,"commonplace":27550,"juris":27551,"redwood":27552,"waistband":27553,"boxed":27554,"caitlin":27555,"fingerprints":27556,"jennie":27557,"naturalized":27558,"##ired":27559,"balfour":27560,"craters":27561,"jody":27562,"bungalow":27563,"hugely":27564,"quilt":27565,"glitter":27566,"pigeons":27567,"undertaker":27568,"bulging":27569,"constrained":27570,"goo":27571,"##sil":27572,"##akh":27573,"assimilation":27574,"reworked":27575,"##person":27576,"persuasion":27577,"##pants":27578,"felicia":27579,"##cliff":27580,"##ulent":27581,"1732":27582,"explodes":27583,"##dun":27584,"##inium":27585,"##zic":27586,"lyman":27587,"vulture":27588,"hog":27589,"overlook":27590,"begs":27591,"northwards":27592,"ow":27593,"spoil":27594,"##urer":27595,"fatima":27596,"favorably":27597,"accumulate":27598,"sargent":27599,"sorority":27600,"corresponded":27601,"dispersal":27602,"kochi":27603,"toned":27604,"##imi":27605,"##lita":27606,"internacional":27607,"newfound":27608,"##agger":27609,"##lynn":27610,"##rigue":27611,"booths":27612,"peanuts":27613,"##eborg":27614,"medicare":27615,"muriel":27616,"nur":27617,"##uram":27618,"crates":27619,"millennia":27620,"pajamas":27621,"worsened":27622,"##breakers":27623,"jimi":27624,"vanuatu":27625,"yawned":27626,"##udeau":27627,"carousel":27628,"##hony":27629,"hurdle":27630,"##ccus":27631,"##mounted":27632,"##pod":27633,"rv":27634,"##eche":27635,"airship":27636,"ambiguity":27637,"compulsion":27638,"recapture":27639,"##claiming":27640,"arthritis":27641,"##osomal":27642,"1667":27643,"asserting":27644,"ngc":27645,"sniffing":27646,"dade":27647,"discontent":27648,"glendale":27649,"ported":27650,"##amina":27651,"defamation":27652,"rammed":27653,"##scent":27654,"fling":27655,"livingstone":27656,"##fleet":27657,"875":27658,"##ppy":27659,"apocalyptic":27660,"comrade":27661,"lcd":27662,"##lowe":27663,"cessna":27664,"eine":27665,"persecuted":27666,"subsistence":27667,"demi":27668,"hoop":27669,"reliefs":27670,"710":27671,"coptic":27672,"progressing":27673,"stemmed":27674,"perpetrators":27675,"1665":27676,"priestess":27677,"##nio":27678,"dobson":27679,"ebony":27680,"rooster":27681,"itf":27682,"tortricidae":27683,"##bbon":27684,"##jian":27685,"cleanup":27686,"##jean":27687,"##øy":27688,"1721":27689,"eighties":27690,"taxonomic":27691,"holiness":27692,"##hearted":27693,"##spar":27694,"antilles":27695,"showcasing":27696,"stabilized":27697,"##nb":27698,"gia":27699,"mascara":27700,"michelangelo":27701,"dawned":27702,"##uria":27703,"##vinsky":27704,"extinguished":27705,"fitz":27706,"grotesque":27707,"£100":27708,"##fera":27709,"##loid":27710,"##mous":27711,"barges":27712,"neue":27713,"throbbed":27714,"cipher":27715,"johnnie":27716,"##a1":27717,"##mpt":27718,"outburst":27719,"##swick":27720,"spearheaded":27721,"administrations":27722,"c1":27723,"heartbreak":27724,"pixels":27725,"pleasantly":27726,"##enay":27727,"lombardy":27728,"plush":27729,"##nsed":27730,"bobbie":27731,"##hly":27732,"reapers":27733,"tremor":27734,"xiang":27735,"minogue":27736,"substantive":27737,"hitch":27738,"barak":27739,"##wyl":27740,"kwan":27741,"##encia":27742,"910":27743,"obscene":27744,"elegance":27745,"indus":27746,"surfer":27747,"bribery":27748,"conserve":27749,"##hyllum":27750,"##masters":27751,"horatio":27752,"##fat":27753,"apes":27754,"rebound":27755,"psychotic":27756,"##pour":27757,"iteration":27758,"##mium":27759,"##vani":27760,"botanic":27761,"horribly":27762,"antiques":27763,"dispose":27764,"paxton":27765,"##hli":27766,"##wg":27767,"timeless":27768,"1704":27769,"disregard":27770,"engraver":27771,"hounds":27772,"##bau":27773,"##version":27774,"looted":27775,"uno":27776,"facilitates":27777,"groans":27778,"masjid":27779,"rutland":27780,"antibody":27781,"disqualification":27782,"decatur":27783,"footballers":27784,"quake":27785,"slacks":27786,"48th":27787,"rein":27788,"scribe":27789,"stabilize":27790,"commits":27791,"exemplary":27792,"tho":27793,"##hort":27794,"##chison":27795,"pantry":27796,"traversed":27797,"##hiti":27798,"disrepair":27799,"identifiable":27800,"vibrated":27801,"baccalaureate":27802,"##nnis":27803,"csa":27804,"interviewing":27805,"##iensis":27806,"##raße":27807,"greaves":27808,"wealthiest":27809,"343":27810,"classed":27811,"jogged":27812,"£5":27813,"##58":27814,"##atal":27815,"illuminating":27816,"knicks":27817,"respecting":27818,"##uno":27819,"scrubbed":27820,"##iji":27821,"##dles":27822,"kruger":27823,"moods":27824,"growls":27825,"raider":27826,"silvia":27827,"chefs":27828,"kam":27829,"vr":27830,"cree":27831,"percival":27832,"##terol":27833,"gunter":27834,"counterattack":27835,"defiant":27836,"henan":27837,"ze":27838,"##rasia":27839,"##riety":27840,"equivalence":27841,"submissions":27842,"##fra":27843,"##thor":27844,"bautista":27845,"mechanically":27846,"##heater":27847,"cornice":27848,"herbal":27849,"templar":27850,"##mering":27851,"outputs":27852,"ruining":27853,"ligand":27854,"renumbered":27855,"extravagant":27856,"mika":27857,"blockbuster":27858,"eta":27859,"insurrection":27860,"##ilia":27861,"darkening":27862,"ferocious":27863,"pianos":27864,"strife":27865,"kinship":27866,"##aer":27867,"melee":27868,"##anor":27869,"##iste":27870,"##may":27871,"##oue":27872,"decidedly":27873,"weep":27874,"##jad":27875,"##missive":27876,"##ppel":27877,"354":27878,"puget":27879,"unease":27880,"##gnant":27881,"1629":27882,"hammering":27883,"kassel":27884,"ob":27885,"wessex":27886,"##lga":27887,"bromwich":27888,"egan":27889,"paranoia":27890,"utilization":27891,"##atable":27892,"##idad":27893,"contradictory":27894,"provoke":27895,"##ols":27896,"##ouring":27897,"##tangled":27898,"knesset":27899,"##very":27900,"##lette":27901,"plumbing":27902,"##sden":27903,"##¹":27904,"greensboro":27905,"occult":27906,"sniff":27907,"338":27908,"zev":27909,"beaming":27910,"gamer":27911,"haggard":27912,"mahal":27913,"##olt":27914,"##pins":27915,"mendes":27916,"utmost":27917,"briefing":27918,"gunnery":27919,"##gut":27920,"##pher":27921,"##zh":27922,"##rok":27923,"1679":27924,"khalifa":27925,"sonya":27926,"##boot":27927,"principals":27928,"urbana":27929,"wiring":27930,"##liffe":27931,"##minating":27932,"##rrado":27933,"dahl":27934,"nyu":27935,"skepticism":27936,"np":27937,"townspeople":27938,"ithaca":27939,"lobster":27940,"somethin":27941,"##fur":27942,"##arina":27943,"##−1":27944,"freighter":27945,"zimmerman":27946,"biceps":27947,"contractual":27948,"##herton":27949,"amend":27950,"hurrying":27951,"subconscious":27952,"##anal":27953,"336":27954,"meng":27955,"clermont":27956,"spawning":27957,"##eia":27958,"##lub":27959,"dignitaries":27960,"impetus":27961,"snacks":27962,"spotting":27963,"twigs":27964,"##bilis":27965,"##cz":27966,"##ouk":27967,"libertadores":27968,"nic":27969,"skylar":27970,"##aina":27971,"##firm":27972,"gustave":27973,"asean":27974,"##anum":27975,"dieter":27976,"legislatures":27977,"flirt":27978,"bromley":27979,"trolls":27980,"umar":27981,"##bbies":27982,"##tyle":27983,"blah":27984,"parc":27985,"bridgeport":27986,"crank":27987,"negligence":27988,"##nction":27989,"46th":27990,"constantin":27991,"molded":27992,"bandages":27993,"seriousness":27994,"00pm":27995,"siegel":27996,"carpets":27997,"compartments":27998,"upbeat":27999,"statehood":28000,"##dner":28001,"##edging":28002,"marko":28003,"730":28004,"platt":28005,"##hane":28006,"paving":28007,"##iy":28008,"1738":28009,"abbess":28010,"impatience":28011,"limousine":28012,"nbl":28013,"##talk":28014,"441":28015,"lucille":28016,"mojo":28017,"nightfall":28018,"robbers":28019,"##nais":28020,"karel":28021,"brisk":28022,"calves":28023,"replicate":28024,"ascribed":28025,"telescopes":28026,"##olf":28027,"intimidated":28028,"##reen":28029,"ballast":28030,"specialization":28031,"##sit":28032,"aerodynamic":28033,"caliphate":28034,"rainer":28035,"visionary":28036,"##arded":28037,"epsilon":28038,"##aday":28039,"##onte":28040,"aggregation":28041,"auditory":28042,"boosted":28043,"reunification":28044,"kathmandu":28045,"loco":28046,"robyn":28047,"402":28048,"acknowledges":28049,"appointing":28050,"humanoid":28051,"newell":28052,"redeveloped":28053,"restraints":28054,"##tained":28055,"barbarians":28056,"chopper":28057,"1609":28058,"italiana":28059,"##lez":28060,"##lho":28061,"investigates":28062,"wrestlemania":28063,"##anies":28064,"##bib":28065,"690":28066,"##falls":28067,"creaked":28068,"dragoons":28069,"gravely":28070,"minions":28071,"stupidity":28072,"volley":28073,"##harat":28074,"##week":28075,"musik":28076,"##eries":28077,"##uously":28078,"fungal":28079,"massimo":28080,"semantics":28081,"malvern":28082,"##ahl":28083,"##pee":28084,"discourage":28085,"embryo":28086,"imperialism":28087,"1910s":28088,"profoundly":28089,"##ddled":28090,"jiangsu":28091,"sparkled":28092,"stat":28093,"##holz":28094,"sweatshirt":28095,"tobin":28096,"##iction":28097,"sneered":28098,"##cheon":28099,"##oit":28100,"brit":28101,"causal":28102,"smyth":28103,"##neuve":28104,"diffuse":28105,"perrin":28106,"silvio":28107,"##ipes":28108,"##recht":28109,"detonated":28110,"iqbal":28111,"selma":28112,"##nism":28113,"##zumi":28114,"roasted":28115,"##riders":28116,"tay":28117,"##ados":28118,"##mament":28119,"##mut":28120,"##rud":28121,"840":28122,"completes":28123,"nipples":28124,"cfa":28125,"flavour":28126,"hirsch":28127,"##laus":28128,"calderon":28129,"sneakers":28130,"moravian":28131,"##ksha":28132,"1622":28133,"rq":28134,"294":28135,"##imeters":28136,"bodo":28137,"##isance":28138,"##pre":28139,"##ronia":28140,"anatomical":28141,"excerpt":28142,"##lke":28143,"dh":28144,"kunst":28145,"##tablished":28146,"##scoe":28147,"biomass":28148,"panted":28149,"unharmed":28150,"gael":28151,"housemates":28152,"montpellier":28153,"##59":28154,"coa":28155,"rodents":28156,"tonic":28157,"hickory":28158,"singleton":28159,"##taro":28160,"451":28161,"1719":28162,"aldo":28163,"breaststroke":28164,"dempsey":28165,"och":28166,"rocco":28167,"##cuit":28168,"merton":28169,"dissemination":28170,"midsummer":28171,"serials":28172,"##idi":28173,"haji":28174,"polynomials":28175,"##rdon":28176,"gs":28177,"enoch":28178,"prematurely":28179,"shutter":28180,"taunton":28181,"£3":28182,"##grating":28183,"##inates":28184,"archangel":28185,"harassed":28186,"##asco":28187,"326":28188,"archway":28189,"dazzling":28190,"##ecin":28191,"1736":28192,"sumo":28193,"wat":28194,"##kovich":28195,"1086":28196,"honneur":28197,"##ently":28198,"##nostic":28199,"##ttal":28200,"##idon":28201,"1605":28202,"403":28203,"1716":28204,"blogger":28205,"rents":28206,"##gnan":28207,"hires":28208,"##ikh":28209,"##dant":28210,"howie":28211,"##rons":28212,"handler":28213,"retracted":28214,"shocks":28215,"1632":28216,"arun":28217,"duluth":28218,"kepler":28219,"trumpeter":28220,"##lary":28221,"peeking":28222,"seasoned":28223,"trooper":28224,"##mara":28225,"laszlo":28226,"##iciencies":28227,"##rti":28228,"heterosexual":28229,"##inatory":28230,"##ssion":28231,"indira":28232,"jogging":28233,"##inga":28234,"##lism":28235,"beit":28236,"dissatisfaction":28237,"malice":28238,"##ately":28239,"nedra":28240,"peeling":28241,"##rgeon":28242,"47th":28243,"stadiums":28244,"475":28245,"vertigo":28246,"##ains":28247,"iced":28248,"restroom":28249,"##plify":28250,"##tub":28251,"illustrating":28252,"pear":28253,"##chner":28254,"##sibility":28255,"inorganic":28256,"rappers":28257,"receipts":28258,"watery":28259,"##kura":28260,"lucinda":28261,"##oulos":28262,"reintroduced":28263,"##8th":28264,"##tched":28265,"gracefully":28266,"saxons":28267,"nutritional":28268,"wastewater":28269,"rained":28270,"favourites":28271,"bedrock":28272,"fisted":28273,"hallways":28274,"likeness":28275,"upscale":28276,"##lateral":28277,"1580":28278,"blinds":28279,"prequel":28280,"##pps":28281,"##tama":28282,"deter":28283,"humiliating":28284,"restraining":28285,"tn":28286,"vents":28287,"1659":28288,"laundering":28289,"recess":28290,"rosary":28291,"tractors":28292,"coulter":28293,"federer":28294,"##ifiers":28295,"##plin":28296,"persistence":28297,"##quitable":28298,"geschichte":28299,"pendulum":28300,"quakers":28301,"##beam":28302,"bassett":28303,"pictorial":28304,"buffet":28305,"koln":28306,"##sitor":28307,"drills":28308,"reciprocal":28309,"shooters":28310,"##57":28311,"##cton":28312,"##tees":28313,"converge":28314,"pip":28315,"dmitri":28316,"donnelly":28317,"yamamoto":28318,"aqua":28319,"azores":28320,"demographics":28321,"hypnotic":28322,"spitfire":28323,"suspend":28324,"wryly":28325,"roderick":28326,"##rran":28327,"sebastien":28328,"##asurable":28329,"mavericks":28330,"##fles":28331,"##200":28332,"himalayan":28333,"prodigy":28334,"##iance":28335,"transvaal":28336,"demonstrators":28337,"handcuffs":28338,"dodged":28339,"mcnamara":28340,"sublime":28341,"1726":28342,"crazed":28343,"##efined":28344,"##till":28345,"ivo":28346,"pondered":28347,"reconciled":28348,"shrill":28349,"sava":28350,"##duk":28351,"bal":28352,"cad":28353,"heresy":28354,"jaipur":28355,"goran":28356,"##nished":28357,"341":28358,"lux":28359,"shelly":28360,"whitehall":28361,"##hre":28362,"israelis":28363,"peacekeeping":28364,"##wled":28365,"1703":28366,"demetrius":28367,"ousted":28368,"##arians":28369,"##zos":28370,"beale":28371,"anwar":28372,"backstroke":28373,"raged":28374,"shrinking":28375,"cremated":28376,"##yck":28377,"benign":28378,"towing":28379,"wadi":28380,"darmstadt":28381,"landfill":28382,"parana":28383,"soothe":28384,"colleen":28385,"sidewalks":28386,"mayfair":28387,"tumble":28388,"hepatitis":28389,"ferrer":28390,"superstructure":28391,"##gingly":28392,"##urse":28393,"##wee":28394,"anthropological":28395,"translators":28396,"##mies":28397,"closeness":28398,"hooves":28399,"##pw":28400,"mondays":28401,"##roll":28402,"##vita":28403,"landscaping":28404,"##urized":28405,"purification":28406,"sock":28407,"thorns":28408,"thwarted":28409,"jalan":28410,"tiberius":28411,"##taka":28412,"saline":28413,"##rito":28414,"confidently":28415,"khyber":28416,"sculptors":28417,"##ij":28418,"brahms":28419,"hammersmith":28420,"inspectors":28421,"battista":28422,"fivb":28423,"fragmentation":28424,"hackney":28425,"##uls":28426,"arresting":28427,"exercising":28428,"antoinette":28429,"bedfordshire":28430,"##zily":28431,"dyed":28432,"##hema":28433,"1656":28434,"racetrack":28435,"variability":28436,"##tique":28437,"1655":28438,"austrians":28439,"deteriorating":28440,"madman":28441,"theorists":28442,"aix":28443,"lehman":28444,"weathered":28445,"1731":28446,"decreed":28447,"eruptions":28448,"1729":28449,"flaw":28450,"quinlan":28451,"sorbonne":28452,"flutes":28453,"nunez":28454,"1711":28455,"adored":28456,"downwards":28457,"fable":28458,"rasped":28459,"1712":28460,"moritz":28461,"mouthful":28462,"renegade":28463,"shivers":28464,"stunts":28465,"dysfunction":28466,"restrain":28467,"translit":28468,"327":28469,"pancakes":28470,"##avio":28471,"##cision":28472,"##tray":28473,"351":28474,"vial":28475,"##lden":28476,"bain":28477,"##maid":28478,"##oxide":28479,"chihuahua":28480,"malacca":28481,"vimes":28482,"##rba":28483,"##rnier":28484,"1664":28485,"donnie":28486,"plaques":28487,"##ually":28488,"337":28489,"bangs":28490,"floppy":28491,"huntsville":28492,"loretta":28493,"nikolay":28494,"##otte":28495,"eater":28496,"handgun":28497,"ubiquitous":28498,"##hett":28499,"eras":28500,"zodiac":28501,"1634":28502,"##omorphic":28503,"1820s":28504,"##zog":28505,"cochran":28506,"##bula":28507,"##lithic":28508,"warring":28509,"##rada":28510,"dalai":28511,"excused":28512,"blazers":28513,"mcconnell":28514,"reeling":28515,"bot":28516,"este":28517,"##abi":28518,"geese":28519,"hoax":28520,"taxon":28521,"##bla":28522,"guitarists":28523,"##icon":28524,"condemning":28525,"hunts":28526,"inversion":28527,"moffat":28528,"taekwondo":28529,"##lvis":28530,"1624":28531,"stammered":28532,"##rest":28533,"##rzy":28534,"sousa":28535,"fundraiser":28536,"marylebone":28537,"navigable":28538,"uptown":28539,"cabbage":28540,"daniela":28541,"salman":28542,"shitty":28543,"whimper":28544,"##kian":28545,"##utive":28546,"programmers":28547,"protections":28548,"rm":28549,"##rmi":28550,"##rued":28551,"forceful":28552,"##enes":28553,"fuss":28554,"##tao":28555,"##wash":28556,"brat":28557,"oppressive":28558,"reykjavik":28559,"spartak":28560,"ticking":28561,"##inkles":28562,"##kiewicz":28563,"adolph":28564,"horst":28565,"maui":28566,"protege":28567,"straighten":28568,"cpc":28569,"landau":28570,"concourse":28571,"clements":28572,"resultant":28573,"##ando":28574,"imaginative":28575,"joo":28576,"reactivated":28577,"##rem":28578,"##ffled":28579,"##uising":28580,"consultative":28581,"##guide":28582,"flop":28583,"kaitlyn":28584,"mergers":28585,"parenting":28586,"somber":28587,"##vron":28588,"supervise":28589,"vidhan":28590,"##imum":28591,"courtship":28592,"exemplified":28593,"harmonies":28594,"medallist":28595,"refining":28596,"##rrow":28597,"##ка":28598,"amara":28599,"##hum":28600,"780":28601,"goalscorer":28602,"sited":28603,"overshadowed":28604,"rohan":28605,"displeasure":28606,"secretive":28607,"multiplied":28608,"osman":28609,"##orth":28610,"engravings":28611,"padre":28612,"##kali":28613,"##veda":28614,"miniatures":28615,"mis":28616,"##yala":28617,"clap":28618,"pali":28619,"rook":28620,"##cana":28621,"1692":28622,"57th":28623,"antennae":28624,"astro":28625,"oskar":28626,"1628":28627,"bulldog":28628,"crotch":28629,"hackett":28630,"yucatan":28631,"##sure":28632,"amplifiers":28633,"brno":28634,"ferrara":28635,"migrating":28636,"##gree":28637,"thanking":28638,"turing":28639,"##eza":28640,"mccann":28641,"ting":28642,"andersson":28643,"onslaught":28644,"gaines":28645,"ganga":28646,"incense":28647,"standardization":28648,"##mation":28649,"sentai":28650,"scuba":28651,"stuffing":28652,"turquoise":28653,"waivers":28654,"alloys":28655,"##vitt":28656,"regaining":28657,"vaults":28658,"##clops":28659,"##gizing":28660,"digger":28661,"furry":28662,"memorabilia":28663,"probing":28664,"##iad":28665,"payton":28666,"rec":28667,"deutschland":28668,"filippo":28669,"opaque":28670,"seamen":28671,"zenith":28672,"afrikaans":28673,"##filtration":28674,"disciplined":28675,"inspirational":28676,"##merie":28677,"banco":28678,"confuse":28679,"grafton":28680,"tod":28681,"##dgets":28682,"championed":28683,"simi":28684,"anomaly":28685,"biplane":28686,"##ceptive":28687,"electrode":28688,"##para":28689,"1697":28690,"cleavage":28691,"crossbow":28692,"swirl":28693,"informant":28694,"##lars":28695,"##osta":28696,"afi":28697,"bonfire":28698,"spec":28699,"##oux":28700,"lakeside":28701,"slump":28702,"##culus":28703,"##lais":28704,"##qvist":28705,"##rrigan":28706,"1016":28707,"facades":28708,"borg":28709,"inwardly":28710,"cervical":28711,"xl":28712,"pointedly":28713,"050":28714,"stabilization":28715,"##odon":28716,"chests":28717,"1699":28718,"hacked":28719,"ctv":28720,"orthogonal":28721,"suzy":28722,"##lastic":28723,"gaulle":28724,"jacobite":28725,"rearview":28726,"##cam":28727,"##erted":28728,"ashby":28729,"##drik":28730,"##igate":28731,"##mise":28732,"##zbek":28733,"affectionately":28734,"canine":28735,"disperse":28736,"latham":28737,"##istles":28738,"##ivar":28739,"spielberg":28740,"##orin":28741,"##idium":28742,"ezekiel":28743,"cid":28744,"##sg":28745,"durga":28746,"middletown":28747,"##cina":28748,"customized":28749,"frontiers":28750,"harden":28751,"##etano":28752,"##zzy":28753,"1604":28754,"bolsheviks":28755,"##66":28756,"coloration":28757,"yoko":28758,"##bedo":28759,"briefs":28760,"slabs":28761,"debra":28762,"liquidation":28763,"plumage":28764,"##oin":28765,"blossoms":28766,"dementia":28767,"subsidy":28768,"1611":28769,"proctor":28770,"relational":28771,"jerseys":28772,"parochial":28773,"ter":28774,"##ici":28775,"esa":28776,"peshawar":28777,"cavalier":28778,"loren":28779,"cpi":28780,"idiots":28781,"shamrock":28782,"1646":28783,"dutton":28784,"malabar":28785,"mustache":28786,"##endez":28787,"##ocytes":28788,"referencing":28789,"terminates":28790,"marche":28791,"yarmouth":28792,"##sop":28793,"acton":28794,"mated":28795,"seton":28796,"subtly":28797,"baptised":28798,"beige":28799,"extremes":28800,"jolted":28801,"kristina":28802,"telecast":28803,"##actic":28804,"safeguard":28805,"waldo":28806,"##baldi":28807,"##bular":28808,"endeavors":28809,"sloppy":28810,"subterranean":28811,"##ensburg":28812,"##itung":28813,"delicately":28814,"pigment":28815,"tq":28816,"##scu":28817,"1626":28818,"##ound":28819,"collisions":28820,"coveted":28821,"herds":28822,"##personal":28823,"##meister":28824,"##nberger":28825,"chopra":28826,"##ricting":28827,"abnormalities":28828,"defective":28829,"galician":28830,"lucie":28831,"##dilly":28832,"alligator":28833,"likened":28834,"##genase":28835,"burundi":28836,"clears":28837,"complexion":28838,"derelict":28839,"deafening":28840,"diablo":28841,"fingered":28842,"champaign":28843,"dogg":28844,"enlist":28845,"isotope":28846,"labeling":28847,"mrna":28848,"##erre":28849,"brilliance":28850,"marvelous":28851,"##ayo":28852,"1652":28853,"crawley":28854,"ether":28855,"footed":28856,"dwellers":28857,"deserts":28858,"hamish":28859,"rubs":28860,"warlock":28861,"skimmed":28862,"##lizer":28863,"870":28864,"buick":28865,"embark":28866,"heraldic":28867,"irregularities":28868,"##ajan":28869,"kiara":28870,"##kulam":28871,"##ieg":28872,"antigen":28873,"kowalski":28874,"##lge":28875,"oakley":28876,"visitation":28877,"##mbit":28878,"vt":28879,"##suit":28880,"1570":28881,"murderers":28882,"##miento":28883,"##rites":28884,"chimneys":28885,"##sling":28886,"condemn":28887,"custer":28888,"exchequer":28889,"havre":28890,"##ghi":28891,"fluctuations":28892,"##rations":28893,"dfb":28894,"hendricks":28895,"vaccines":28896,"##tarian":28897,"nietzsche":28898,"biking":28899,"juicy":28900,"##duced":28901,"brooding":28902,"scrolling":28903,"selangor":28904,"##ragan":28905,"352":28906,"annum":28907,"boomed":28908,"seminole":28909,"sugarcane":28910,"##dna":28911,"departmental":28912,"dismissing":28913,"innsbruck":28914,"arteries":28915,"ashok":28916,"batavia":28917,"daze":28918,"kun":28919,"overtook":28920,"##rga":28921,"##tlan":28922,"beheaded":28923,"gaddafi":28924,"holm":28925,"electronically":28926,"faulty":28927,"galilee":28928,"fractures":28929,"kobayashi":28930,"##lized":28931,"gunmen":28932,"magma":28933,"aramaic":28934,"mala":28935,"eastenders":28936,"inference":28937,"messengers":28938,"bf":28939,"##qu":28940,"407":28941,"bathrooms":28942,"##vere":28943,"1658":28944,"flashbacks":28945,"ideally":28946,"misunderstood":28947,"##jali":28948,"##weather":28949,"mendez":28950,"##grounds":28951,"505":28952,"uncanny":28953,"##iii":28954,"1709":28955,"friendships":28956,"##nbc":28957,"sacrament":28958,"accommodated":28959,"reiterated":28960,"logistical":28961,"pebbles":28962,"thumped":28963,"##escence":28964,"administering":28965,"decrees":28966,"drafts":28967,"##flight":28968,"##cased":28969,"##tula":28970,"futuristic":28971,"picket":28972,"intimidation":28973,"winthrop":28974,"##fahan":28975,"interfered":28976,"339":28977,"afar":28978,"francoise":28979,"morally":28980,"uta":28981,"cochin":28982,"croft":28983,"dwarfs":28984,"##bruck":28985,"##dents":28986,"##nami":28987,"biker":28988,"##hner":28989,"##meral":28990,"nano":28991,"##isen":28992,"##ometric":28993,"##pres":28994,"##ан":28995,"brightened":28996,"meek":28997,"parcels":28998,"securely":28999,"gunners":29000,"##jhl":29001,"##zko":29002,"agile":29003,"hysteria":29004,"##lten":29005,"##rcus":29006,"bukit":29007,"champs":29008,"chevy":29009,"cuckoo":29010,"leith":29011,"sadler":29012,"theologians":29013,"welded":29014,"##section":29015,"1663":29016,"jj":29017,"plurality":29018,"xander":29019,"##rooms":29020,"##formed":29021,"shredded":29022,"temps":29023,"intimately":29024,"pau":29025,"tormented":29026,"##lok":29027,"##stellar":29028,"1618":29029,"charred":29030,"ems":29031,"essen":29032,"##mmel":29033,"alarms":29034,"spraying":29035,"ascot":29036,"blooms":29037,"twinkle":29038,"##abia":29039,"##apes":29040,"internment":29041,"obsidian":29042,"##chaft":29043,"snoop":29044,"##dav":29045,"##ooping":29046,"malibu":29047,"##tension":29048,"quiver":29049,"##itia":29050,"hays":29051,"mcintosh":29052,"travers":29053,"walsall":29054,"##ffie":29055,"1623":29056,"beverley":29057,"schwarz":29058,"plunging":29059,"structurally":29060,"m3":29061,"rosenthal":29062,"vikram":29063,"##tsk":29064,"770":29065,"ghz":29066,"##onda":29067,"##tiv":29068,"chalmers":29069,"groningen":29070,"pew":29071,"reckon":29072,"unicef":29073,"##rvis":29074,"55th":29075,"##gni":29076,"1651":29077,"sulawesi":29078,"avila":29079,"cai":29080,"metaphysical":29081,"screwing":29082,"turbulence":29083,"##mberg":29084,"augusto":29085,"samba":29086,"56th":29087,"baffled":29088,"momentary":29089,"toxin":29090,"##urian":29091,"##wani":29092,"aachen":29093,"condoms":29094,"dali":29095,"steppe":29096,"##3d":29097,"##app":29098,"##oed":29099,"##year":29100,"adolescence":29101,"dauphin":29102,"electrically":29103,"inaccessible":29104,"microscopy":29105,"nikita":29106,"##ega":29107,"atv":29108,"##cel":29109,"##enter":29110,"##oles":29111,"##oteric":29112,"##ы":29113,"accountants":29114,"punishments":29115,"wrongly":29116,"bribes":29117,"adventurous":29118,"clinch":29119,"flinders":29120,"southland":29121,"##hem":29122,"##kata":29123,"gough":29124,"##ciency":29125,"lads":29126,"soared":29127,"##ה":29128,"undergoes":29129,"deformation":29130,"outlawed":29131,"rubbish":29132,"##arus":29133,"##mussen":29134,"##nidae":29135,"##rzburg":29136,"arcs":29137,"##ingdon":29138,"##tituted":29139,"1695":29140,"wheelbase":29141,"wheeling":29142,"bombardier":29143,"campground":29144,"zebra":29145,"##lices":29146,"##oj":29147,"##bain":29148,"lullaby":29149,"##ecure":29150,"donetsk":29151,"wylie":29152,"grenada":29153,"##arding":29154,"##ης":29155,"squinting":29156,"eireann":29157,"opposes":29158,"##andra":29159,"maximal":29160,"runes":29161,"##broken":29162,"##cuting":29163,"##iface":29164,"##ror":29165,"##rosis":29166,"additive":29167,"britney":29168,"adultery":29169,"triggering":29170,"##drome":29171,"detrimental":29172,"aarhus":29173,"containment":29174,"jc":29175,"swapped":29176,"vichy":29177,"##ioms":29178,"madly":29179,"##oric":29180,"##rag":29181,"brant":29182,"##ckey":29183,"##trix":29184,"1560":29185,"1612":29186,"broughton":29187,"rustling":29188,"##stems":29189,"##uder":29190,"asbestos":29191,"mentoring":29192,"##nivorous":29193,"finley":29194,"leaps":29195,"##isan":29196,"apical":29197,"pry":29198,"slits":29199,"substitutes":29200,"##dict":29201,"intuitive":29202,"fantasia":29203,"insistent":29204,"unreasonable":29205,"##igen":29206,"##vna":29207,"domed":29208,"hannover":29209,"margot":29210,"ponder":29211,"##zziness":29212,"impromptu":29213,"jian":29214,"lc":29215,"rampage":29216,"stemming":29217,"##eft":29218,"andrey":29219,"gerais":29220,"whichever":29221,"amnesia":29222,"appropriated":29223,"anzac":29224,"clicks":29225,"modifying":29226,"ultimatum":29227,"cambrian":29228,"maids":29229,"verve":29230,"yellowstone":29231,"##mbs":29232,"conservatoire":29233,"##scribe":29234,"adherence":29235,"dinners":29236,"spectra":29237,"imperfect":29238,"mysteriously":29239,"sidekick":29240,"tatar":29241,"tuba":29242,"##aks":29243,"##ifolia":29244,"distrust":29245,"##athan":29246,"##zle":29247,"c2":29248,"ronin":29249,"zac":29250,"##pse":29251,"celaena":29252,"instrumentalist":29253,"scents":29254,"skopje":29255,"##mbling":29256,"comical":29257,"compensated":29258,"vidal":29259,"condor":29260,"intersect":29261,"jingle":29262,"wavelengths":29263,"##urrent":29264,"mcqueen":29265,"##izzly":29266,"carp":29267,"weasel":29268,"422":29269,"kanye":29270,"militias":29271,"postdoctoral":29272,"eugen":29273,"gunslinger":29274,"##ɛ":29275,"faux":29276,"hospice":29277,"##for":29278,"appalled":29279,"derivation":29280,"dwarves":29281,"##elis":29282,"dilapidated":29283,"##folk":29284,"astoria":29285,"philology":29286,"##lwyn":29287,"##otho":29288,"##saka":29289,"inducing":29290,"philanthropy":29291,"##bf":29292,"##itative":29293,"geek":29294,"markedly":29295,"sql":29296,"##yce":29297,"bessie":29298,"indices":29299,"rn":29300,"##flict":29301,"495":29302,"frowns":29303,"resolving":29304,"weightlifting":29305,"tugs":29306,"cleric":29307,"contentious":29308,"1653":29309,"mania":29310,"rms":29311,"##miya":29312,"##reate":29313,"##ruck":29314,"##tucket":29315,"bien":29316,"eels":29317,"marek":29318,"##ayton":29319,"##cence":29320,"discreet":29321,"unofficially":29322,"##ife":29323,"leaks":29324,"##bber":29325,"1705":29326,"332":29327,"dung":29328,"compressor":29329,"hillsborough":29330,"pandit":29331,"shillings":29332,"distal":29333,"##skin":29334,"381":29335,"##tat":29336,"##you":29337,"nosed":29338,"##nir":29339,"mangrove":29340,"undeveloped":29341,"##idia":29342,"textures":29343,"##inho":29344,"##500":29345,"##rise":29346,"ae":29347,"irritating":29348,"nay":29349,"amazingly":29350,"bancroft":29351,"apologetic":29352,"compassionate":29353,"kata":29354,"symphonies":29355,"##lovic":29356,"airspace":29357,"##lch":29358,"930":29359,"gifford":29360,"precautions":29361,"fulfillment":29362,"sevilla":29363,"vulgar":29364,"martinique":29365,"##urities":29366,"looting":29367,"piccolo":29368,"tidy":29369,"##dermott":29370,"quadrant":29371,"armchair":29372,"incomes":29373,"mathematicians":29374,"stampede":29375,"nilsson":29376,"##inking":29377,"##scan":29378,"foo":29379,"quarterfinal":29380,"##ostal":29381,"shang":29382,"shouldered":29383,"squirrels":29384,"##owe":29385,"344":29386,"vinegar":29387,"##bner":29388,"##rchy":29389,"##systems":29390,"delaying":29391,"##trics":29392,"ars":29393,"dwyer":29394,"rhapsody":29395,"sponsoring":29396,"##gration":29397,"bipolar":29398,"cinder":29399,"starters":29400,"##olio":29401,"##urst":29402,"421":29403,"signage":29404,"##nty":29405,"aground":29406,"figurative":29407,"mons":29408,"acquaintances":29409,"duets":29410,"erroneously":29411,"soyuz":29412,"elliptic":29413,"recreated":29414,"##cultural":29415,"##quette":29416,"##ssed":29417,"##tma":29418,"##zcz":29419,"moderator":29420,"scares":29421,"##itaire":29422,"##stones":29423,"##udence":29424,"juniper":29425,"sighting":29426,"##just":29427,"##nsen":29428,"britten":29429,"calabria":29430,"ry":29431,"bop":29432,"cramer":29433,"forsyth":29434,"stillness":29435,"##л":29436,"airmen":29437,"gathers":29438,"unfit":29439,"##umber":29440,"##upt":29441,"taunting":29442,"##rip":29443,"seeker":29444,"streamlined":29445,"##bution":29446,"holster":29447,"schumann":29448,"tread":29449,"vox":29450,"##gano":29451,"##onzo":29452,"strive":29453,"dil":29454,"reforming":29455,"covent":29456,"newbury":29457,"predicting":29458,"##orro":29459,"decorate":29460,"tre":29461,"##puted":29462,"andover":29463,"ie":29464,"asahi":29465,"dept":29466,"dunkirk":29467,"gills":29468,"##tori":29469,"buren":29470,"huskies":29471,"##stis":29472,"##stov":29473,"abstracts":29474,"bets":29475,"loosen":29476,"##opa":29477,"1682":29478,"yearning":29479,"##glio":29480,"##sir":29481,"berman":29482,"effortlessly":29483,"enamel":29484,"napoli":29485,"persist":29486,"##peration":29487,"##uez":29488,"attache":29489,"elisa":29490,"b1":29491,"invitations":29492,"##kic":29493,"accelerating":29494,"reindeer":29495,"boardwalk":29496,"clutches":29497,"nelly":29498,"polka":29499,"starbucks":29500,"##kei":29501,"adamant":29502,"huey":29503,"lough":29504,"unbroken":29505,"adventurer":29506,"embroidery":29507,"inspecting":29508,"stanza":29509,"##ducted":29510,"naia":29511,"taluka":29512,"##pone":29513,"##roids":29514,"chases":29515,"deprivation":29516,"florian":29517,"##jing":29518,"##ppet":29519,"earthly":29520,"##lib":29521,"##ssee":29522,"colossal":29523,"foreigner":29524,"vet":29525,"freaks":29526,"patrice":29527,"rosewood":29528,"triassic":29529,"upstate":29530,"##pkins":29531,"dominates":29532,"ata":29533,"chants":29534,"ks":29535,"vo":29536,"##400":29537,"##bley":29538,"##raya":29539,"##rmed":29540,"555":29541,"agra":29542,"infiltrate":29543,"##ailing":29544,"##ilation":29545,"##tzer":29546,"##uppe":29547,"##werk":29548,"binoculars":29549,"enthusiast":29550,"fujian":29551,"squeak":29552,"##avs":29553,"abolitionist":29554,"almeida":29555,"boredom":29556,"hampstead":29557,"marsden":29558,"rations":29559,"##ands":29560,"inflated":29561,"334":29562,"bonuses":29563,"rosalie":29564,"patna":29565,"##rco":29566,"329":29567,"detachments":29568,"penitentiary":29569,"54th":29570,"flourishing":29571,"woolf":29572,"##dion":29573,"##etched":29574,"papyrus":29575,"##lster":29576,"##nsor":29577,"##toy":29578,"bobbed":29579,"dismounted":29580,"endelle":29581,"inhuman":29582,"motorola":29583,"tbs":29584,"wince":29585,"wreath":29586,"##ticus":29587,"hideout":29588,"inspections":29589,"sanjay":29590,"disgrace":29591,"infused":29592,"pudding":29593,"stalks":29594,"##urbed":29595,"arsenic":29596,"leases":29597,"##hyl":29598,"##rrard":29599,"collarbone":29600,"##waite":29601,"##wil":29602,"dowry":29603,"##bant":29604,"##edance":29605,"genealogical":29606,"nitrate":29607,"salamanca":29608,"scandals":29609,"thyroid":29610,"necessitated":29611,"##!":29612,"##\"":29613,"###":29614,"##$":29615,"##%":29616,"##&":29617,"##'":29618,"##(":29619,"##)":29620,"##*":29621,"##+":29622,"##,":29623,"##-":29624,"##.":29625,"##/":29626,"##:":29627,"##;":29628,"##<":29629,"##=":29630,"##>":29631,"##?":29632,"##@":29633,"##[":29634,"##\\":29635,"##]":29636,"##^":29637,"##_":29638,"##`":29639,"##{":29640,"##|":29641,"##}":29642,"##~":29643,"##¡":29644,"##¢":29645,"##£":29646,"##¤":29647,"##¥":29648,"##¦":29649,"##§":29650,"##¨":29651,"##©":29652,"##ª":29653,"##«":29654,"##¬":29655,"##®":29656,"##±":29657,"##´":29658,"##µ":29659,"##¶":29660,"##·":29661,"##º":29662,"##»":29663,"##¼":29664,"##¾":29665,"##¿":29666,"##æ":29667,"##ð":29668,"##÷":29669,"##þ":29670,"##đ":29671,"##ħ":29672,"##ŋ":29673,"##œ":29674,"##ƒ":29675,"##ɐ":29676,"##ɑ":29677,"##ɒ":29678,"##ɔ":29679,"##ɕ":29680,"##ə":29681,"##ɡ":29682,"##ɣ":29683,"##ɨ":29684,"##ɪ":29685,"##ɫ":29686,"##ɬ":29687,"##ɯ":29688,"##ɲ":29689,"##ɴ":29690,"##ɹ":29691,"##ɾ":29692,"##ʀ":29693,"##ʁ":29694,"##ʂ":29695,"##ʃ":29696,"##ʉ":29697,"##ʊ":29698,"##ʋ":29699,"##ʌ":29700,"##ʎ":29701,"##ʐ":29702,"##ʑ":29703,"##ʒ":29704,"##ʔ":29705,"##ʰ":29706,"##ʲ":29707,"##ʳ":29708,"##ʷ":29709,"##ʸ":29710,"##ʻ":29711,"##ʼ":29712,"##ʾ":29713,"##ʿ":29714,"##ˈ":29715,"##ˡ":29716,"##ˢ":29717,"##ˣ":29718,"##ˤ":29719,"##β":29720,"##γ":29721,"##δ":29722,"##ε":29723,"##ζ":29724,"##θ":29725,"##κ":29726,"##λ":29727,"##μ":29728,"##ξ":29729,"##ο":29730,"##π":29731,"##ρ":29732,"##σ":29733,"##τ":29734,"##υ":29735,"##φ":29736,"##χ":29737,"##ψ":29738,"##ω":29739,"##б":29740,"##г":29741,"##д":29742,"##ж":29743,"##з":29744,"##м":29745,"##п":29746,"##с":29747,"##у":29748,"##ф":29749,"##х":29750,"##ц":29751,"##ч":29752,"##ш":29753,"##щ":29754,"##ъ":29755,"##э":29756,"##ю":29757,"##ђ":29758,"##є":29759,"##і":29760,"##ј":29761,"##љ":29762,"##њ":29763,"##ћ":29764,"##ӏ":29765,"##ա":29766,"##բ":29767,"##գ":29768,"##դ":29769,"##ե":29770,"##թ":29771,"##ի":29772,"##լ":29773,"##կ":29774,"##հ":29775,"##մ":29776,"##յ":29777,"##ն":29778,"##ո":29779,"##պ":29780,"##ս":29781,"##վ":29782,"##տ":29783,"##ր":29784,"##ւ":29785,"##ք":29786,"##־":29787,"##א":29788,"##ב":29789,"##ג":29790,"##ד":29791,"##ו":29792,"##ז":29793,"##ח":29794,"##ט":29795,"##י":29796,"##ך":29797,"##כ":29798,"##ל":29799,"##ם":29800,"##מ":29801,"##ן":29802,"##נ":29803,"##ס":29804,"##ע":29805,"##ף":29806,"##פ":29807,"##ץ":29808,"##צ":29809,"##ק":29810,"##ר":29811,"##ש":29812,"##ת":29813,"##،":29814,"##ء":29815,"##ب":29816,"##ت":29817,"##ث":29818,"##ج":29819,"##ح":29820,"##خ":29821,"##ذ":29822,"##ز":29823,"##س":29824,"##ش":29825,"##ص":29826,"##ض":29827,"##ط":29828,"##ظ":29829,"##ع":29830,"##غ":29831,"##ـ":29832,"##ف":29833,"##ق":29834,"##ك":29835,"##و":29836,"##ى":29837,"##ٹ":29838,"##پ":29839,"##چ":29840,"##ک":29841,"##گ":29842,"##ں":29843,"##ھ":29844,"##ہ":29845,"##ے":29846,"##अ":29847,"##आ":29848,"##उ":29849,"##ए":29850,"##क":29851,"##ख":29852,"##ग":29853,"##च":29854,"##ज":29855,"##ट":29856,"##ड":29857,"##ण":29858,"##त":29859,"##थ":29860,"##द":29861,"##ध":29862,"##न":29863,"##प":29864,"##ब":29865,"##भ":29866,"##म":29867,"##य":29868,"##र":29869,"##ल":29870,"##व":29871,"##श":29872,"##ष":29873,"##स":29874,"##ह":29875,"##ा":29876,"##ि":29877,"##ी":29878,"##ो":29879,"##।":29880,"##॥":29881,"##ং":29882,"##অ":29883,"##আ":29884,"##ই":29885,"##উ":29886,"##এ":29887,"##ও":29888,"##ক":29889,"##খ":29890,"##গ":29891,"##চ":29892,"##ছ":29893,"##জ":29894,"##ট":29895,"##ড":29896,"##ণ":29897,"##ত":29898,"##থ":29899,"##দ":29900,"##ধ":29901,"##ন":29902,"##প":29903,"##ব":29904,"##ভ":29905,"##ম":29906,"##য":29907,"##র":29908,"##ল":29909,"##শ":29910,"##ষ":29911,"##স":29912,"##হ":29913,"##া":29914,"##ি":29915,"##ী":29916,"##ে":29917,"##க":29918,"##ச":29919,"##ட":29920,"##த":29921,"##ந":29922,"##ன":29923,"##ப":29924,"##ம":29925,"##ய":29926,"##ர":29927,"##ல":29928,"##ள":29929,"##வ":29930,"##ா":29931,"##ி":29932,"##ு":29933,"##ே":29934,"##ை":29935,"##ನ":29936,"##ರ":29937,"##ಾ":29938,"##ක":29939,"##ය":29940,"##ර":29941,"##ල":29942,"##ව":29943,"##ා":29944,"##ก":29945,"##ง":29946,"##ต":29947,"##ท":29948,"##น":29949,"##พ":29950,"##ม":29951,"##ย":29952,"##ร":29953,"##ล":29954,"##ว":29955,"##ส":29956,"##อ":29957,"##า":29958,"##เ":29959,"##་":29960,"##།":29961,"##ག":29962,"##ང":29963,"##ད":29964,"##ན":29965,"##པ":29966,"##བ":29967,"##མ":29968,"##འ":29969,"##ར":29970,"##ལ":29971,"##ས":29972,"##မ":29973,"##ა":29974,"##ბ":29975,"##გ":29976,"##დ":29977,"##ე":29978,"##ვ":29979,"##თ":29980,"##ი":29981,"##კ":29982,"##ლ":29983,"##მ":29984,"##ნ":29985,"##ო":29986,"##რ":29987,"##ს":29988,"##ტ":29989,"##უ":29990,"##ᄀ":29991,"##ᄂ":29992,"##ᄃ":29993,"##ᄅ":29994,"##ᄆ":29995,"##ᄇ":29996,"##ᄉ":29997,"##ᄊ":29998,"##ᄋ":29999,"##ᄌ":30000,"##ᄎ":30001,"##ᄏ":30002,"##ᄐ":30003,"##ᄑ":30004,"##ᄒ":30005,"##ᅡ":30006,"##ᅢ":30007,"##ᅥ":30008,"##ᅦ":30009,"##ᅧ":30010,"##ᅩ":30011,"##ᅪ":30012,"##ᅭ":30013,"##ᅮ":30014,"##ᅯ":30015,"##ᅲ":30016,"##ᅳ":30017,"##ᅴ":30018,"##ᅵ":30019,"##ᆨ":30020,"##ᆫ":30021,"##ᆯ":30022,"##ᆷ":30023,"##ᆸ":30024,"##ᆼ":30025,"##ᴬ":30026,"##ᴮ":30027,"##ᴰ":30028,"##ᴵ":30029,"##ᴺ":30030,"##ᵀ":30031,"##ᵃ":30032,"##ᵇ":30033,"##ᵈ":30034,"##ᵉ":30035,"##ᵍ":30036,"##ᵏ":30037,"##ᵐ":30038,"##ᵒ":30039,"##ᵖ":30040,"##ᵗ":30041,"##ᵘ":30042,"##ᵣ":30043,"##ᵤ":30044,"##ᵥ":30045,"##ᶜ":30046,"##ᶠ":30047,"##‐":30048,"##‑":30049,"##‒":30050,"##–":30051,"##—":30052,"##―":30053,"##‖":30054,"##‘":30055,"##’":30056,"##‚":30057,"##“":30058,"##”":30059,"##„":30060,"##†":30061,"##‡":30062,"##•":30063,"##…":30064,"##‰":30065,"##′":30066,"##″":30067,"##›":30068,"##‿":30069,"##⁄":30070,"##⁰":30071,"##ⁱ":30072,"##⁴":30073,"##⁵":30074,"##⁶":30075,"##⁷":30076,"##⁸":30077,"##⁹":30078,"##⁻":30079,"##ⁿ":30080,"##₅":30081,"##₆":30082,"##₇":30083,"##₈":30084,"##₉":30085,"##₊":30086,"##₍":30087,"##₎":30088,"##ₐ":30089,"##ₑ":30090,"##ₒ":30091,"##ₓ":30092,"##ₕ":30093,"##ₖ":30094,"##ₗ":30095,"##ₘ":30096,"##ₚ":30097,"##ₛ":30098,"##ₜ":30099,"##₤":30100,"##₩":30101,"##€":30102,"##₱":30103,"##₹":30104,"##ℓ":30105,"##№":30106,"##ℝ":30107,"##™":30108,"##⅓":30109,"##⅔":30110,"##←":30111,"##↑":30112,"##→":30113,"##↓":30114,"##↔":30115,"##↦":30116,"##⇄":30117,"##⇌":30118,"##⇒":30119,"##∂":30120,"##∅":30121,"##∆":30122,"##∇":30123,"##∈":30124,"##∗":30125,"##∘":30126,"##√":30127,"##∞":30128,"##∧":30129,"##∨":30130,"##∩":30131,"##∪":30132,"##≈":30133,"##≡":30134,"##≤":30135,"##≥":30136,"##⊂":30137,"##⊆":30138,"##⊕":30139,"##⊗":30140,"##⋅":30141,"##─":30142,"##│":30143,"##■":30144,"##▪":30145,"##●":30146,"##★":30147,"##☆":30148,"##☉":30149,"##♠":30150,"##♣":30151,"##♥":30152,"##♦":30153,"##♯":30154,"##⟨":30155,"##⟩":30156,"##ⱼ":30157,"##⺩":30158,"##⺼":30159,"##⽥":30160,"##、":30161,"##。":30162,"##〈":30163,"##〉":30164,"##《":30165,"##》":30166,"##「":30167,"##」":30168,"##『":30169,"##』":30170,"##〜":30171,"##あ":30172,"##い":30173,"##う":30174,"##え":30175,"##お":30176,"##か":30177,"##き":30178,"##く":30179,"##け":30180,"##こ":30181,"##さ":30182,"##し":30183,"##す":30184,"##せ":30185,"##そ":30186,"##た":30187,"##ち":30188,"##っ":30189,"##つ":30190,"##て":30191,"##と":30192,"##な":30193,"##に":30194,"##ぬ":30195,"##ね":30196,"##の":30197,"##は":30198,"##ひ":30199,"##ふ":30200,"##へ":30201,"##ほ":30202,"##ま":30203,"##み":30204,"##む":30205,"##め":30206,"##も":30207,"##や":30208,"##ゆ":30209,"##よ":30210,"##ら":30211,"##り":30212,"##る":30213,"##れ":30214,"##ろ":30215,"##を":30216,"##ん":30217,"##ァ":30218,"##ア":30219,"##ィ":30220,"##イ":30221,"##ウ":30222,"##ェ":30223,"##エ":30224,"##オ":30225,"##カ":30226,"##キ":30227,"##ク":30228,"##ケ":30229,"##コ":30230,"##サ":30231,"##シ":30232,"##ス":30233,"##セ":30234,"##タ":30235,"##チ":30236,"##ッ":30237,"##ツ":30238,"##テ":30239,"##ト":30240,"##ナ":30241,"##ニ":30242,"##ノ":30243,"##ハ":30244,"##ヒ":30245,"##フ":30246,"##ヘ":30247,"##ホ":30248,"##マ":30249,"##ミ":30250,"##ム":30251,"##メ":30252,"##モ":30253,"##ャ":30254,"##ュ":30255,"##ョ":30256,"##ラ":30257,"##リ":30258,"##ル":30259,"##レ":30260,"##ロ":30261,"##ワ":30262,"##ン":30263,"##・":30264,"##ー":30265,"##一":30266,"##三":30267,"##上":30268,"##下":30269,"##不":30270,"##世":30271,"##中":30272,"##主":30273,"##久":30274,"##之":30275,"##也":30276,"##事":30277,"##二":30278,"##五":30279,"##井":30280,"##京":30281,"##人":30282,"##亻":30283,"##仁":30284,"##介":30285,"##代":30286,"##仮":30287,"##伊":30288,"##会":30289,"##佐":30290,"##侍":30291,"##保":30292,"##信":30293,"##健":30294,"##元":30295,"##光":30296,"##八":30297,"##公":30298,"##内":30299,"##出":30300,"##分":30301,"##前":30302,"##劉":30303,"##力":30304,"##加":30305,"##勝":30306,"##北":30307,"##区":30308,"##十":30309,"##千":30310,"##南":30311,"##博":30312,"##原":30313,"##口":30314,"##古":30315,"##史":30316,"##司":30317,"##合":30318,"##吉":30319,"##同":30320,"##名":30321,"##和":30322,"##囗":30323,"##四":30324,"##国":30325,"##國":30326,"##土":30327,"##地":30328,"##坂":30329,"##城":30330,"##堂":30331,"##場":30332,"##士":30333,"##夏":30334,"##外":30335,"##大":30336,"##天":30337,"##太":30338,"##夫":30339,"##奈":30340,"##女":30341,"##子":30342,"##学":30343,"##宀":30344,"##宇":30345,"##安":30346,"##宗":30347,"##定":30348,"##宣":30349,"##宮":30350,"##家":30351,"##宿":30352,"##寺":30353,"##將":30354,"##小":30355,"##尚":30356,"##山":30357,"##岡":30358,"##島":30359,"##崎":30360,"##川":30361,"##州":30362,"##巿":30363,"##帝":30364,"##平":30365,"##年":30366,"##幸":30367,"##广":30368,"##弘":30369,"##張":30370,"##彳":30371,"##後":30372,"##御":30373,"##德":30374,"##心":30375,"##忄":30376,"##志":30377,"##忠":30378,"##愛":30379,"##成":30380,"##我":30381,"##戦":30382,"##戸":30383,"##手":30384,"##扌":30385,"##政":30386,"##文":30387,"##新":30388,"##方":30389,"##日":30390,"##明":30391,"##星":30392,"##春":30393,"##昭":30394,"##智":30395,"##曲":30396,"##書":30397,"##月":30398,"##有":30399,"##朝":30400,"##木":30401,"##本":30402,"##李":30403,"##村":30404,"##東":30405,"##松":30406,"##林":30407,"##森":30408,"##楊":30409,"##樹":30410,"##橋":30411,"##歌":30412,"##止":30413,"##正":30414,"##武":30415,"##比":30416,"##氏":30417,"##民":30418,"##水":30419,"##氵":30420,"##氷":30421,"##永":30422,"##江":30423,"##沢":30424,"##河":30425,"##治":30426,"##法":30427,"##海":30428,"##清":30429,"##漢":30430,"##瀬":30431,"##火":30432,"##版":30433,"##犬":30434,"##王":30435,"##生":30436,"##田":30437,"##男":30438,"##疒":30439,"##発":30440,"##白":30441,"##的":30442,"##皇":30443,"##目":30444,"##相":30445,"##省":30446,"##真":30447,"##石":30448,"##示":30449,"##社":30450,"##神":30451,"##福":30452,"##禾":30453,"##秀":30454,"##秋":30455,"##空":30456,"##立":30457,"##章":30458,"##竹":30459,"##糹":30460,"##美":30461,"##義":30462,"##耳":30463,"##良":30464,"##艹":30465,"##花":30466,"##英":30467,"##華":30468,"##葉":30469,"##藤":30470,"##行":30471,"##街":30472,"##西":30473,"##見":30474,"##訁":30475,"##語":30476,"##谷":30477,"##貝":30478,"##貴":30479,"##車":30480,"##軍":30481,"##辶":30482,"##道":30483,"##郎":30484,"##郡":30485,"##部":30486,"##都":30487,"##里":30488,"##野":30489,"##金":30490,"##鈴":30491,"##镇":30492,"##長":30493,"##門":30494,"##間":30495,"##阝":30496,"##阿":30497,"##陳":30498,"##陽":30499,"##雄":30500,"##青":30501,"##面":30502,"##風":30503,"##食":30504,"##香":30505,"##馬":30506,"##高":30507,"##龍":30508,"##龸":30509,"##fi":30510,"##fl":30511,"##！":30512,"##（":30513,"##）":30514,"##，":30515,"##－":30516,"##．":30517,"##／":30518,"##：":30519,"##？":30520,"##～":30521}}} \ No newline at end of file diff --git a/data/datasets/sentiment_classification/mobilebert-uncased/vocab.txt b/data/datasets/sentiment_classification/mobilebert-uncased/vocab.txt new file mode 100644 index 0000000000000000000000000000000000000000..fb140275c155a9c7c5a3b3e0e77a9e839594a938 --- /dev/null +++ b/data/datasets/sentiment_classification/mobilebert-uncased/vocab.txt @@ -0,0 +1,30522 @@ +[PAD] +[unused0] +[unused1] +[unused2] +[unused3] +[unused4] +[unused5] +[unused6] +[unused7] +[unused8] +[unused9] +[unused10] +[unused11] +[unused12] +[unused13] +[unused14] +[unused15] +[unused16] +[unused17] +[unused18] +[unused19] +[unused20] +[unused21] +[unused22] +[unused23] +[unused24] +[unused25] +[unused26] +[unused27] +[unused28] +[unused29] +[unused30] +[unused31] +[unused32] +[unused33] +[unused34] +[unused35] +[unused36] +[unused37] +[unused38] +[unused39] +[unused40] +[unused41] +[unused42] +[unused43] +[unused44] +[unused45] +[unused46] +[unused47] +[unused48] +[unused49] +[unused50] +[unused51] +[unused52] +[unused53] +[unused54] +[unused55] +[unused56] +[unused57] +[unused58] +[unused59] +[unused60] +[unused61] +[unused62] +[unused63] +[unused64] +[unused65] +[unused66] +[unused67] +[unused68] +[unused69] +[unused70] +[unused71] +[unused72] +[unused73] +[unused74] +[unused75] +[unused76] +[unused77] +[unused78] +[unused79] +[unused80] +[unused81] +[unused82] +[unused83] +[unused84] +[unused85] +[unused86] +[unused87] +[unused88] +[unused89] +[unused90] +[unused91] +[unused92] +[unused93] +[unused94] +[unused95] +[unused96] +[unused97] +[unused98] +[UNK] +[CLS] +[SEP] +[MASK] +[unused99] +[unused100] +[unused101] +[unused102] +[unused103] +[unused104] +[unused105] +[unused106] +[unused107] +[unused108] +[unused109] +[unused110] +[unused111] +[unused112] +[unused113] +[unused114] +[unused115] +[unused116] +[unused117] +[unused118] +[unused119] +[unused120] +[unused121] +[unused122] +[unused123] +[unused124] +[unused125] +[unused126] +[unused127] +[unused128] +[unused129] +[unused130] +[unused131] +[unused132] +[unused133] +[unused134] +[unused135] +[unused136] +[unused137] +[unused138] +[unused139] +[unused140] +[unused141] +[unused142] +[unused143] +[unused144] +[unused145] +[unused146] +[unused147] +[unused148] +[unused149] +[unused150] +[unused151] +[unused152] +[unused153] +[unused154] +[unused155] +[unused156] +[unused157] +[unused158] +[unused159] +[unused160] +[unused161] +[unused162] +[unused163] +[unused164] +[unused165] +[unused166] +[unused167] +[unused168] +[unused169] +[unused170] +[unused171] +[unused172] +[unused173] +[unused174] +[unused175] +[unused176] +[unused177] +[unused178] +[unused179] +[unused180] +[unused181] +[unused182] +[unused183] +[unused184] +[unused185] +[unused186] +[unused187] +[unused188] +[unused189] +[unused190] +[unused191] +[unused192] +[unused193] +[unused194] +[unused195] +[unused196] +[unused197] +[unused198] +[unused199] +[unused200] +[unused201] +[unused202] +[unused203] +[unused204] +[unused205] +[unused206] +[unused207] +[unused208] +[unused209] +[unused210] +[unused211] +[unused212] +[unused213] +[unused214] +[unused215] +[unused216] +[unused217] +[unused218] +[unused219] +[unused220] +[unused221] +[unused222] +[unused223] +[unused224] +[unused225] +[unused226] +[unused227] +[unused228] +[unused229] +[unused230] +[unused231] +[unused232] +[unused233] +[unused234] +[unused235] +[unused236] +[unused237] +[unused238] +[unused239] +[unused240] +[unused241] +[unused242] +[unused243] +[unused244] +[unused245] +[unused246] +[unused247] +[unused248] +[unused249] +[unused250] +[unused251] +[unused252] +[unused253] +[unused254] +[unused255] +[unused256] +[unused257] +[unused258] +[unused259] +[unused260] +[unused261] +[unused262] +[unused263] +[unused264] +[unused265] +[unused266] +[unused267] +[unused268] +[unused269] +[unused270] +[unused271] +[unused272] +[unused273] +[unused274] +[unused275] +[unused276] +[unused277] +[unused278] +[unused279] +[unused280] +[unused281] +[unused282] +[unused283] +[unused284] +[unused285] +[unused286] +[unused287] +[unused288] +[unused289] +[unused290] +[unused291] +[unused292] +[unused293] +[unused294] +[unused295] +[unused296] +[unused297] +[unused298] +[unused299] +[unused300] +[unused301] +[unused302] +[unused303] +[unused304] +[unused305] +[unused306] +[unused307] +[unused308] +[unused309] +[unused310] +[unused311] +[unused312] +[unused313] +[unused314] +[unused315] +[unused316] +[unused317] +[unused318] +[unused319] +[unused320] +[unused321] +[unused322] +[unused323] +[unused324] +[unused325] +[unused326] +[unused327] +[unused328] +[unused329] +[unused330] +[unused331] +[unused332] +[unused333] +[unused334] +[unused335] +[unused336] +[unused337] +[unused338] +[unused339] +[unused340] +[unused341] +[unused342] +[unused343] +[unused344] +[unused345] +[unused346] +[unused347] +[unused348] +[unused349] +[unused350] +[unused351] +[unused352] +[unused353] +[unused354] +[unused355] +[unused356] +[unused357] +[unused358] +[unused359] +[unused360] +[unused361] +[unused362] +[unused363] +[unused364] +[unused365] +[unused366] +[unused367] +[unused368] +[unused369] +[unused370] +[unused371] +[unused372] +[unused373] +[unused374] +[unused375] +[unused376] +[unused377] +[unused378] +[unused379] +[unused380] +[unused381] +[unused382] +[unused383] +[unused384] +[unused385] +[unused386] +[unused387] +[unused388] +[unused389] +[unused390] +[unused391] +[unused392] +[unused393] +[unused394] +[unused395] +[unused396] +[unused397] +[unused398] +[unused399] +[unused400] +[unused401] +[unused402] +[unused403] +[unused404] +[unused405] +[unused406] +[unused407] +[unused408] +[unused409] +[unused410] +[unused411] +[unused412] +[unused413] +[unused414] +[unused415] +[unused416] +[unused417] +[unused418] +[unused419] +[unused420] +[unused421] +[unused422] +[unused423] +[unused424] +[unused425] +[unused426] +[unused427] +[unused428] +[unused429] +[unused430] +[unused431] +[unused432] +[unused433] +[unused434] +[unused435] +[unused436] +[unused437] +[unused438] +[unused439] +[unused440] +[unused441] +[unused442] +[unused443] +[unused444] +[unused445] +[unused446] +[unused447] +[unused448] +[unused449] +[unused450] +[unused451] +[unused452] +[unused453] +[unused454] +[unused455] +[unused456] +[unused457] +[unused458] +[unused459] +[unused460] +[unused461] +[unused462] +[unused463] +[unused464] +[unused465] +[unused466] +[unused467] +[unused468] +[unused469] +[unused470] +[unused471] +[unused472] +[unused473] +[unused474] +[unused475] +[unused476] +[unused477] +[unused478] +[unused479] +[unused480] +[unused481] +[unused482] +[unused483] +[unused484] +[unused485] +[unused486] +[unused487] +[unused488] +[unused489] +[unused490] +[unused491] +[unused492] +[unused493] +[unused494] +[unused495] +[unused496] +[unused497] +[unused498] +[unused499] +[unused500] +[unused501] +[unused502] +[unused503] +[unused504] +[unused505] +[unused506] +[unused507] +[unused508] +[unused509] +[unused510] +[unused511] +[unused512] +[unused513] +[unused514] +[unused515] +[unused516] +[unused517] +[unused518] +[unused519] +[unused520] +[unused521] +[unused522] +[unused523] +[unused524] +[unused525] +[unused526] +[unused527] +[unused528] +[unused529] +[unused530] +[unused531] +[unused532] +[unused533] +[unused534] +[unused535] +[unused536] +[unused537] +[unused538] +[unused539] +[unused540] +[unused541] +[unused542] +[unused543] +[unused544] +[unused545] +[unused546] +[unused547] +[unused548] +[unused549] +[unused550] +[unused551] +[unused552] +[unused553] +[unused554] +[unused555] +[unused556] +[unused557] +[unused558] +[unused559] +[unused560] +[unused561] +[unused562] +[unused563] +[unused564] +[unused565] +[unused566] +[unused567] +[unused568] +[unused569] +[unused570] +[unused571] +[unused572] +[unused573] +[unused574] +[unused575] +[unused576] +[unused577] +[unused578] +[unused579] +[unused580] +[unused581] +[unused582] +[unused583] +[unused584] +[unused585] +[unused586] +[unused587] +[unused588] +[unused589] +[unused590] +[unused591] +[unused592] +[unused593] +[unused594] +[unused595] +[unused596] +[unused597] +[unused598] +[unused599] +[unused600] +[unused601] +[unused602] +[unused603] +[unused604] +[unused605] +[unused606] +[unused607] +[unused608] +[unused609] +[unused610] +[unused611] +[unused612] +[unused613] +[unused614] +[unused615] +[unused616] +[unused617] +[unused618] +[unused619] +[unused620] +[unused621] +[unused622] +[unused623] +[unused624] +[unused625] +[unused626] +[unused627] +[unused628] +[unused629] +[unused630] +[unused631] +[unused632] +[unused633] +[unused634] +[unused635] +[unused636] +[unused637] +[unused638] +[unused639] +[unused640] +[unused641] +[unused642] +[unused643] +[unused644] +[unused645] +[unused646] +[unused647] +[unused648] +[unused649] +[unused650] +[unused651] +[unused652] +[unused653] +[unused654] +[unused655] +[unused656] +[unused657] +[unused658] +[unused659] +[unused660] +[unused661] +[unused662] +[unused663] +[unused664] +[unused665] +[unused666] +[unused667] +[unused668] +[unused669] +[unused670] +[unused671] +[unused672] +[unused673] +[unused674] +[unused675] +[unused676] +[unused677] +[unused678] +[unused679] +[unused680] +[unused681] +[unused682] +[unused683] +[unused684] +[unused685] +[unused686] +[unused687] +[unused688] +[unused689] +[unused690] +[unused691] +[unused692] +[unused693] +[unused694] +[unused695] +[unused696] +[unused697] +[unused698] +[unused699] +[unused700] +[unused701] +[unused702] +[unused703] +[unused704] +[unused705] +[unused706] +[unused707] +[unused708] +[unused709] +[unused710] +[unused711] +[unused712] +[unused713] +[unused714] +[unused715] +[unused716] +[unused717] +[unused718] +[unused719] +[unused720] +[unused721] +[unused722] +[unused723] +[unused724] +[unused725] +[unused726] +[unused727] +[unused728] +[unused729] +[unused730] +[unused731] +[unused732] +[unused733] +[unused734] +[unused735] +[unused736] +[unused737] +[unused738] +[unused739] +[unused740] +[unused741] +[unused742] +[unused743] +[unused744] +[unused745] +[unused746] +[unused747] +[unused748] +[unused749] +[unused750] +[unused751] +[unused752] +[unused753] +[unused754] +[unused755] +[unused756] +[unused757] +[unused758] +[unused759] +[unused760] +[unused761] +[unused762] +[unused763] +[unused764] +[unused765] +[unused766] +[unused767] +[unused768] +[unused769] +[unused770] +[unused771] +[unused772] +[unused773] +[unused774] +[unused775] +[unused776] +[unused777] +[unused778] +[unused779] +[unused780] +[unused781] +[unused782] +[unused783] +[unused784] +[unused785] +[unused786] +[unused787] +[unused788] +[unused789] +[unused790] +[unused791] +[unused792] +[unused793] +[unused794] +[unused795] +[unused796] +[unused797] +[unused798] +[unused799] +[unused800] +[unused801] +[unused802] +[unused803] +[unused804] +[unused805] +[unused806] +[unused807] +[unused808] +[unused809] +[unused810] +[unused811] +[unused812] +[unused813] +[unused814] +[unused815] +[unused816] +[unused817] +[unused818] +[unused819] +[unused820] +[unused821] +[unused822] +[unused823] +[unused824] +[unused825] +[unused826] +[unused827] +[unused828] +[unused829] +[unused830] +[unused831] +[unused832] +[unused833] +[unused834] +[unused835] +[unused836] +[unused837] +[unused838] +[unused839] +[unused840] +[unused841] +[unused842] +[unused843] +[unused844] +[unused845] +[unused846] +[unused847] +[unused848] +[unused849] +[unused850] +[unused851] +[unused852] +[unused853] +[unused854] +[unused855] +[unused856] +[unused857] +[unused858] +[unused859] +[unused860] +[unused861] +[unused862] +[unused863] +[unused864] +[unused865] +[unused866] +[unused867] +[unused868] +[unused869] +[unused870] +[unused871] +[unused872] +[unused873] +[unused874] +[unused875] +[unused876] +[unused877] +[unused878] +[unused879] +[unused880] +[unused881] +[unused882] +[unused883] +[unused884] +[unused885] +[unused886] +[unused887] +[unused888] +[unused889] +[unused890] +[unused891] +[unused892] +[unused893] +[unused894] +[unused895] +[unused896] +[unused897] +[unused898] +[unused899] +[unused900] +[unused901] +[unused902] +[unused903] +[unused904] +[unused905] +[unused906] +[unused907] +[unused908] +[unused909] +[unused910] +[unused911] +[unused912] +[unused913] +[unused914] +[unused915] +[unused916] +[unused917] +[unused918] +[unused919] +[unused920] +[unused921] +[unused922] +[unused923] +[unused924] +[unused925] +[unused926] +[unused927] +[unused928] +[unused929] +[unused930] +[unused931] +[unused932] +[unused933] +[unused934] +[unused935] +[unused936] +[unused937] +[unused938] +[unused939] +[unused940] +[unused941] +[unused942] +[unused943] +[unused944] +[unused945] +[unused946] +[unused947] +[unused948] +[unused949] +[unused950] +[unused951] +[unused952] +[unused953] +[unused954] +[unused955] +[unused956] +[unused957] +[unused958] +[unused959] +[unused960] +[unused961] +[unused962] +[unused963] +[unused964] +[unused965] +[unused966] +[unused967] +[unused968] +[unused969] +[unused970] +[unused971] +[unused972] +[unused973] +[unused974] +[unused975] +[unused976] +[unused977] +[unused978] +[unused979] +[unused980] +[unused981] +[unused982] +[unused983] +[unused984] +[unused985] +[unused986] +[unused987] +[unused988] +[unused989] +[unused990] +[unused991] +[unused992] +[unused993] +! +" +# +$ +% +& +' +( +) +* ++ +, +- +. +/ +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +: +; +< += +> +? +@ +[ +\ +] +^ +_ +` +a +b +c +d +e +f +g +h +i +j +k +l +m +n +o +p +q +r +s +t +u +v +w +x +y +z +{ +| +} +~ +¡ +¢ +£ +¤ +¥ +¦ +§ +¨ +© +ª +« +¬ +® +° +± +² +³ +´ +µ +¶ +· +¹ +º +» +¼ +½ +¾ +¿ +× +ß +æ +ð +÷ +ø +þ +đ +ħ +ı +ł +ŋ +œ +ƒ +ɐ +ɑ +ɒ +ɔ +ɕ +ə +ɛ +ɡ +ɣ +ɨ +ɪ +ɫ +ɬ +ɯ +ɲ +ɴ +ɹ +ɾ +ʀ +ʁ +ʂ +ʃ +ʉ +ʊ +ʋ +ʌ +ʎ +ʐ +ʑ +ʒ +ʔ +ʰ +ʲ +ʳ +ʷ +ʸ +ʻ +ʼ +ʾ +ʿ +ˈ +ː +ˡ +ˢ +ˣ +ˤ +α +β +γ +δ +ε +ζ +η +θ +ι +κ +λ +μ +ν +ξ +ο +π +ρ +ς +σ +τ +υ +φ +χ +ψ +ω +а +б +в +г +д +е +ж +з +и +к +л +м +н +о +п +р +с +т +у +ф +х +ц +ч +ш +щ +ъ +ы +ь +э +ю +я +ђ +є +і +ј +љ +њ +ћ +ӏ +ա +բ +գ +դ +ե +թ +ի +լ +կ +հ +մ +յ +ն +ո +պ +ս +վ +տ +ր +ւ +ք +־ +א +ב +ג +ד +ה +ו +ז +ח +ט +י +ך +כ +ל +ם +מ +ן +נ +ס +ע +ף +פ +ץ +צ +ק +ר +ש +ת +، +ء +ا +ب +ة +ت +ث +ج +ح +خ +د +ذ +ر +ز +س +ش +ص +ض +ط +ظ +ع +غ +ـ +ف +ق +ك +ل +م +ن +ه +و +ى +ي +ٹ +پ +چ +ک +گ +ں +ھ +ہ +ی +ے +अ +आ +उ +ए +क +ख +ग +च +ज +ट +ड +ण +त +थ +द +ध +न +प +ब +भ +म +य +र +ल +व +श +ष +स +ह +ा +ि +ी +ो +। +॥ +ং +অ +আ +ই +উ +এ +ও +ক +খ +গ +চ +ছ +জ +ট +ড +ণ +ত +থ +দ +ধ +ন +প +ব +ভ +ম +য +র +ল +শ +ষ +স +হ +া +ি +ী +ে +க +ச +ட +த +ந +ன +ப +ம +ய +ர +ல +ள +வ +ா +ி +ு +ே +ை +ನ +ರ +ಾ +ක +ය +ර +ල +ව +ා +ก +ง +ต +ท +น +พ +ม +ย +ร +ล +ว +ส +อ +า +เ +་ +། +ག +ང +ད +ན +པ +བ +མ +འ +ར +ལ +ས +မ +ა +ბ +გ +დ +ე +ვ +თ +ი +კ +ლ +მ +ნ +ო +რ +ს +ტ +უ +ᄀ +ᄂ +ᄃ +ᄅ +ᄆ +ᄇ +ᄉ +ᄊ +ᄋ +ᄌ +ᄎ +ᄏ +ᄐ +ᄑ +ᄒ +ᅡ +ᅢ +ᅥ +ᅦ +ᅧ +ᅩ +ᅪ +ᅭ +ᅮ +ᅯ +ᅲ +ᅳ +ᅴ +ᅵ +ᆨ +ᆫ +ᆯ +ᆷ +ᆸ +ᆼ +ᴬ +ᴮ +ᴰ +ᴵ +ᴺ +ᵀ +ᵃ +ᵇ +ᵈ +ᵉ +ᵍ +ᵏ +ᵐ +ᵒ +ᵖ +ᵗ +ᵘ +ᵢ +ᵣ +ᵤ +ᵥ +ᶜ +ᶠ +‐ +‑ +‒ +– +— +― +‖ +‘ +’ +‚ +“ +” +„ +† +‡ +• +… +‰ +′ +″ +› +‿ +⁄ +⁰ +ⁱ +⁴ +⁵ +⁶ +⁷ +⁸ +⁹ +⁺ +⁻ +ⁿ +₀ +₁ +₂ +₃ +₄ +₅ +₆ +₇ +₈ +₉ +₊ +₍ +₎ +ₐ +ₑ +ₒ +ₓ +ₕ +ₖ +ₗ +ₘ +ₙ +ₚ +ₛ +ₜ +₤ +₩ +€ +₱ +₹ +ℓ +№ +ℝ +™ +⅓ +⅔ +← +↑ +→ +↓ +↔ +↦ +⇄ +⇌ +⇒ +∂ +∅ +∆ +∇ +∈ +− +∗ +∘ +√ +∞ +∧ +∨ +∩ +∪ +≈ +≡ +≤ +≥ +⊂ +⊆ +⊕ +⊗ +⋅ +─ +│ +■ +▪ +● +★ +☆ +☉ +♠ +♣ +♥ +♦ +♭ +♯ +⟨ +⟩ +ⱼ +⺩ +⺼ +⽥ +、 +。 +〈 +〉 +《 +》 +「 +」 +『 +』 +〜 +あ +い +う +え +お +か +き +く +け +こ +さ +し +す +せ +そ +た +ち +っ +つ +て +と +な +に +ぬ +ね +の +は +ひ +ふ +へ +ほ +ま +み +む +め +も +や +ゆ +よ +ら +り +る +れ +ろ +を +ん +ァ +ア +ィ +イ +ウ +ェ +エ +オ +カ +キ +ク +ケ +コ +サ +シ +ス +セ +タ +チ +ッ +ツ +テ +ト +ナ +ニ +ノ +ハ +ヒ +フ +ヘ +ホ +マ +ミ +ム +メ +モ +ャ +ュ +ョ +ラ +リ +ル +レ +ロ +ワ +ン +・ +ー +一 +三 +上 +下 +不 +世 +中 +主 +久 +之 +也 +事 +二 +五 +井 +京 +人 +亻 +仁 +介 +代 +仮 +伊 +会 +佐 +侍 +保 +信 +健 +元 +光 +八 +公 +内 +出 +分 +前 +劉 +力 +加 +勝 +北 +区 +十 +千 +南 +博 +原 +口 +古 +史 +司 +合 +吉 +同 +名 +和 +囗 +四 +国 +國 +土 +地 +坂 +城 +堂 +場 +士 +夏 +外 +大 +天 +太 +夫 +奈 +女 +子 +学 +宀 +宇 +安 +宗 +定 +宣 +宮 +家 +宿 +寺 +將 +小 +尚 +山 +岡 +島 +崎 +川 +州 +巿 +帝 +平 +年 +幸 +广 +弘 +張 +彳 +後 +御 +德 +心 +忄 +志 +忠 +愛 +成 +我 +戦 +戸 +手 +扌 +政 +文 +新 +方 +日 +明 +星 +春 +昭 +智 +曲 +書 +月 +有 +朝 +木 +本 +李 +村 +東 +松 +林 +森 +楊 +樹 +橋 +歌 +止 +正 +武 +比 +氏 +民 +水 +氵 +氷 +永 +江 +沢 +河 +治 +法 +海 +清 +漢 +瀬 +火 +版 +犬 +王 +生 +田 +男 +疒 +発 +白 +的 +皇 +目 +相 +省 +真 +石 +示 +社 +神 +福 +禾 +秀 +秋 +空 +立 +章 +竹 +糹 +美 +義 +耳 +良 +艹 +花 +英 +華 +葉 +藤 +行 +街 +西 +見 +訁 +語 +谷 +貝 +貴 +車 +軍 +辶 +道 +郎 +郡 +部 +都 +里 +野 +金 +鈴 +镇 +長 +門 +間 +阝 +阿 +陳 +陽 +雄 +青 +面 +風 +食 +香 +馬 +高 +龍 +龸 +fi +fl +！ +（ +） +， +－ +． +／ +： +？ +～ +the +of +and +in +to +was +he +is +as +for +on +with +that +it +his +by +at +from +her +##s +she +you +had +an +were +but +be +this +are +not +my +they +one +which +or +have +him +me +first +all +also +their +has +up +who +out +been +when +after +there +into +new +two +its +##a +time +would +no +what +about +said +we +over +then +other +so +more +##e +can +if +like +back +them +only +some +could +##i +where +just +##ing +during +before +##n +do +##o +made +school +through +than +now +years +most +world +may +between +down +well +three +##d +year +while +will +##ed +##r +##y +later +##t +city +under +around +did +such +being +used +state +people +part +know +against +your +many +second +university +both +national +##er +these +don +known +off +way +until +re +how +even +get +head +... +didn +##ly +team +american +because +de +##l +born +united +film +since +still +long +work +south +us +became +any +high +again +day +family +see +right +man +eyes +house +season +war +states +including +took +life +north +same +each +called +name +much +place +however +go +four +group +another +found +won +area +here +going +10 +away +series +left +home +music +best +make +hand +number +company +several +never +last +john +000 +very +album +take +end +good +too +following +released +game +played +little +began +district +##m +old +want +those +side +held +own +early +county +ll +league +use +west +##u +face +think +##es +2010 +government +##h +march +came +small +general +town +june +##on +line +based +something +##k +september +thought +looked +along +international +2011 +air +july +club +went +january +october +our +august +april +york +12 +few +2012 +2008 +east +show +member +college +2009 +father +public +##us +come +men +five +set +station +church +##c +next +former +november +room +party +located +december +2013 +age +got +2007 +##g +system +let +love +2006 +though +every +2014 +look +song +water +century +without +body +black +night +within +great +women +single +ve +building +large +population +river +named +band +white +started +##an +once +15 +20 +should +18 +2015 +service +top +built +british +open +death +king +moved +local +times +children +february +book +why +11 +door +need +president +order +final +road +wasn +although +due +major +died +village +third +knew +2016 +asked +turned +st +wanted +say +##p +together +received +main +son +served +different +##en +behind +himself +felt +members +power +football +law +voice +play +##in +near +park +history +30 +having +2005 +16 +##man +saw +mother +##al +army +point +front +help +english +street +art +late +hands +games +award +##ia +young +14 +put +published +country +division +across +told +13 +often +ever +french +london +center +six +red +2017 +led +days +include +light +25 +find +tell +among +species +really +according +central +half +2004 +form +original +gave +office +making +enough +lost +full +opened +must +included +live +given +german +player +run +business +woman +community +cup +might +million +land +2000 +court +development +17 +short +round +ii +km +seen +class +story +always +become +sure +research +almost +director +council +la +##2 +career +things +using +island +##z +couldn +car +##is +24 +close +force +##1 +better +free +support +control +field +students +2003 +education +married +##b +nothing +worked +others +record +big +inside +level +anything +continued +give +james +##3 +military +established +non +returned +feel +does +title +written +thing +feet +william +far +co +association +hard +already +2002 +##ra +championship +human +western +100 +##na +department +hall +role +various +production +21 +19 +heart +2001 +living +fire +version +##ers +##f +television +royal +##4 +produced +working +act +case +society +region +present +radio +period +looking +least +total +keep +england +wife +program +per +brother +mind +special +22 +##le +am +works +soon +##6 +political +george +services +taken +created +##7 +further +able +reached +david +union +joined +upon +done +important +social +information +either +##ic +##x +appeared +position +ground +lead +rock +dark +election +23 +board +france +hair +course +arms +site +police +girl +instead +real +sound +##v +words +moment +##te +someone +##8 +summer +project +announced +san +less +wrote +past +followed +##5 +blue +founded +al +finally +india +taking +records +america +##ne +1999 +design +considered +northern +god +stop +battle +toward +european +outside +described +track +today +playing +language +28 +call +26 +heard +professional +low +australia +miles +california +win +yet +green +##ie +trying +blood +##ton +southern +science +maybe +everything +match +square +27 +mouth +video +race +recorded +leave +above +##9 +daughter +points +space +1998 +museum +change +middle +common +##0 +move +tv +post +##ta +lake +seven +tried +elected +closed +ten +paul +minister +##th +months +start +chief +return +canada +person +sea +release +similar +modern +brought +rest +hit +formed +mr +##la +1997 +floor +event +doing +thomas +1996 +robert +care +killed +training +star +week +needed +turn +finished +railway +rather +news +health +sent +example +ran +term +michael +coming +currently +yes +forces +despite +gold +areas +50 +stage +fact +29 +dead +says +popular +2018 +originally +germany +probably +developed +result +pulled +friend +stood +money +running +mi +signed +word +songs +child +eventually +met +tour +average +teams +minutes +festival +current +deep +kind +1995 +decided +usually +eastern +seemed +##ness +episode +bed +added +table +indian +private +charles +route +available +idea +throughout +centre +addition +appointed +style +1994 +books +eight +construction +press +mean +wall +friends +remained +schools +study +##ch +##um +institute +oh +chinese +sometimes +events +possible +1992 +australian +type +brown +forward +talk +process +food +debut +seat +performance +committee +features +character +arts +herself +else +lot +strong +russian +range +hours +peter +arm +##da +morning +dr +sold +##ry +quickly +directed +1993 +guitar +china +##w +31 +list +##ma +performed +media +uk +players +smile +##rs +myself +40 +placed +coach +province +towards +wouldn +leading +whole +boy +official +designed +grand +census +##el +europe +attack +japanese +henry +1991 +##re +##os +cross +getting +alone +action +lower +network +wide +washington +japan +1990 +hospital +believe +changed +sister +##ar +hold +gone +sir +hadn +ship +##ka +studies +academy +shot +rights +below +base +bad +involved +kept +largest +##ist +bank +future +especially +beginning +mark +movement +section +female +magazine +plan +professor +lord +longer +##ian +sat +walked +hill +actually +civil +energy +model +families +size +thus +aircraft +completed +includes +data +captain +##or +fight +vocals +featured +richard +bridge +fourth +1989 +officer +stone +hear +##ism +means +medical +groups +management +self +lips +competition +entire +lived +technology +leaving +federal +tournament +bit +passed +hot +independent +awards +kingdom +mary +spent +fine +doesn +reported +##ling +jack +fall +raised +itself +stay +true +studio +1988 +sports +replaced +paris +systems +saint +leader +theatre +whose +market +capital +parents +spanish +canadian +earth +##ity +cut +degree +writing +bay +christian +awarded +natural +higher +bill +##as +coast +provided +previous +senior +ft +valley +organization +stopped +onto +countries +parts +conference +queen +security +interest +saying +allowed +master +earlier +phone +matter +smith +winning +try +happened +moving +campaign +los +##ley +breath +nearly +mid +1987 +certain +girls +date +italian +african +standing +fell +artist +##ted +shows +deal +mine +industry +1986 +##ng +everyone +republic +provide +collection +library +student +##ville +primary +owned +older +via +heavy +1st +makes +##able +attention +anyone +africa +##ri +stated +length +ended +fingers +command +staff +skin +foreign +opening +governor +okay +medal +kill +sun +cover +job +1985 +introduced +chest +hell +feeling +##ies +success +meet +reason +standard +meeting +novel +1984 +trade +source +buildings +##land +rose +guy +goal +##ur +chapter +native +husband +previously +unit +limited +entered +weeks +producer +operations +mountain +takes +covered +forced +related +roman +complete +successful +key +texas +cold +##ya +channel +1980 +traditional +films +dance +clear +approximately +500 +nine +van +prince +question +active +tracks +ireland +regional +silver +author +personal +sense +operation +##ine +economic +1983 +holding +twenty +isbn +additional +speed +hour +edition +regular +historic +places +whom +shook +movie +km² +secretary +prior +report +chicago +read +foundation +view +engine +scored +1982 +units +ask +airport +property +ready +immediately +lady +month +listed +contract +##de +manager +themselves +lines +##ki +navy +writer +meant +##ts +runs +##ro +practice +championships +singer +glass +commission +required +forest +starting +culture +generally +giving +access +attended +test +couple +stand +catholic +martin +caught +executive +##less +eye +##ey +thinking +chair +quite +shoulder +1979 +hope +decision +plays +defeated +municipality +whether +structure +offered +slowly +pain +ice +direction +##ion +paper +mission +1981 +mostly +200 +noted +individual +managed +nature +lives +plant +##ha +helped +except +studied +computer +figure +relationship +issue +significant +loss +die +smiled +gun +ago +highest +1972 +##am +male +bring +goals +mexico +problem +distance +commercial +completely +location +annual +famous +drive +1976 +neck +1978 +surface +caused +italy +understand +greek +highway +wrong +hotel +comes +appearance +joseph +double +issues +musical +companies +castle +income +review +assembly +bass +initially +parliament +artists +experience +1974 +particular +walk +foot +engineering +talking +window +dropped +##ter +miss +baby +boys +break +1975 +stars +edge +remember +policy +carried +train +stadium +bar +sex +angeles +evidence +##ge +becoming +assistant +soviet +1977 +upper +step +wing +1970 +youth +financial +reach +##ll +actor +numerous +##se +##st +nodded +arrived +##ation +minute +##nt +believed +sorry +complex +beautiful +victory +associated +temple +1968 +1973 +chance +perhaps +metal +##son +1945 +bishop +##et +lee +launched +particularly +tree +le +retired +subject +prize +contains +yeah +theory +empire +##ce +suddenly +waiting +trust +recording +##to +happy +terms +camp +champion +1971 +religious +pass +zealand +names +2nd +port +ancient +tom +corner +represented +watch +legal +anti +justice +cause +watched +brothers +45 +material +changes +simply +response +louis +fast +##ting +answer +60 +historical +1969 +stories +straight +create +feature +increased +rate +administration +virginia +el +activities +cultural +overall +winner +programs +basketball +legs +guard +beyond +cast +doctor +mm +flight +results +remains +cost +effect +winter +##ble +larger +islands +problems +chairman +grew +commander +isn +1967 +pay +failed +selected +hurt +fort +box +regiment +majority +journal +35 +edward +plans +##ke +##ni +shown +pretty +irish +characters +directly +scene +likely +operated +allow +spring +##j +junior +matches +looks +mike +houses +fellow +##tion +beach +marriage +##ham +##ive +rules +oil +65 +florida +expected +nearby +congress +sam +peace +recent +iii +wait +subsequently +cell +##do +variety +serving +agreed +please +poor +joe +pacific +attempt +wood +democratic +piece +prime +##ca +rural +mile +touch +appears +township +1964 +1966 +soldiers +##men +##ized +1965 +pennsylvania +closer +fighting +claimed +score +jones +physical +editor +##ous +filled +genus +specific +sitting +super +mom +##va +therefore +supported +status +fear +cases +store +meaning +wales +minor +spain +tower +focus +vice +frank +follow +parish +separate +golden +horse +fifth +remaining +branch +32 +presented +stared +##id +uses +secret +forms +##co +baseball +exactly +##ck +choice +note +discovered +travel +composed +truth +russia +ball +color +kiss +dad +wind +continue +ring +referred +numbers +digital +greater +##ns +metres +slightly +direct +increase +1960 +responsible +crew +rule +trees +troops +##no +broke +goes +individuals +hundred +weight +creek +sleep +memory +defense +provides +ordered +code +value +jewish +windows +1944 +safe +judge +whatever +corps +realized +growing +pre +##ga +cities +alexander +gaze +lies +spread +scott +letter +showed +situation +mayor +transport +watching +workers +extended +##li +expression +normal +##ment +chart +multiple +border +##ba +host +##ner +daily +mrs +walls +piano +##ko +heat +cannot +##ate +earned +products +drama +era +authority +seasons +join +grade +##io +sign +difficult +machine +1963 +territory +mainly +##wood +stations +squadron +1962 +stepped +iron +19th +##led +serve +appear +sky +speak +broken +charge +knowledge +kilometres +removed +ships +article +campus +simple +##ty +pushed +britain +##ve +leaves +recently +cd +soft +boston +latter +easy +acquired +poland +##sa +quality +officers +presence +planned +nations +mass +broadcast +jean +share +image +influence +wild +offer +emperor +electric +reading +headed +ability +promoted +yellow +ministry +1942 +throat +smaller +politician +##by +latin +spoke +cars +williams +males +lack +pop +80 +##ier +acting +seeing +consists +##ti +estate +1961 +pressure +johnson +newspaper +jr +chris +olympics +online +conditions +beat +elements +walking +vote +##field +needs +carolina +text +featuring +global +block +shirt +levels +francisco +purpose +females +et +dutch +duke +ahead +gas +twice +safety +serious +turning +highly +lieutenant +firm +maria +amount +mixed +daniel +proposed +perfect +agreement +affairs +3rd +seconds +contemporary +paid +1943 +prison +save +kitchen +label +administrative +intended +constructed +academic +nice +teacher +races +1956 +formerly +corporation +ben +nation +issued +shut +1958 +drums +housing +victoria +seems +opera +1959 +graduated +function +von +mentioned +picked +build +recognized +shortly +protection +picture +notable +exchange +elections +1980s +loved +percent +racing +fish +elizabeth +garden +volume +hockey +1941 +beside +settled +##ford +1940 +competed +replied +drew +1948 +actress +marine +scotland +steel +glanced +farm +steve +1957 +risk +tonight +positive +magic +singles +effects +gray +screen +dog +##ja +residents +bus +sides +none +secondary +literature +polish +destroyed +flying +founder +households +1939 +lay +reserve +usa +gallery +##ler +1946 +industrial +younger +approach +appearances +urban +ones +1950 +finish +avenue +powerful +fully +growth +page +honor +jersey +projects +advanced +revealed +basic +90 +infantry +pair +equipment +visit +33 +evening +search +grant +effort +solo +treatment +buried +republican +primarily +bottom +owner +1970s +israel +gives +jim +dream +bob +remain +spot +70 +notes +produce +champions +contact +ed +soul +accepted +ways +del +##ally +losing +split +price +capacity +basis +trial +questions +##ina +1955 +20th +guess +officially +memorial +naval +initial +##ization +whispered +median +engineer +##ful +sydney +##go +columbia +strength +300 +1952 +tears +senate +00 +card +asian +agent +1947 +software +44 +draw +warm +supposed +com +pro +##il +transferred +leaned +##at +candidate +escape +mountains +asia +potential +activity +entertainment +seem +traffic +jackson +murder +36 +slow +product +orchestra +haven +agency +bbc +taught +website +comedy +unable +storm +planning +albums +rugby +environment +scientific +grabbed +protect +##hi +boat +typically +1954 +1953 +damage +principal +divided +dedicated +mount +ohio +##berg +pick +fought +driver +##der +empty +shoulders +sort +thank +berlin +prominent +account +freedom +necessary +efforts +alex +headquarters +follows +alongside +des +simon +andrew +suggested +operating +learning +steps +1949 +sweet +technical +begin +easily +34 +teeth +speaking +settlement +scale +##sh +renamed +ray +max +enemy +semi +joint +compared +##rd +scottish +leadership +analysis +offers +georgia +pieces +captured +animal +deputy +guest +organized +##lin +tony +combined +method +challenge +1960s +huge +wants +battalion +sons +rise +crime +types +facilities +telling +path +1951 +platform +sit +1990s +##lo +tells +assigned +rich +pull +##ot +commonly +alive +##za +letters +concept +conducted +wearing +happen +bought +becomes +holy +gets +ocean +defeat +languages +purchased +coffee +occurred +titled +##q +declared +applied +sciences +concert +sounds +jazz +brain +##me +painting +fleet +tax +nick +##ius +michigan +count +animals +leaders +episodes +##line +content +##den +birth +##it +clubs +64 +palace +critical +refused +fair +leg +laughed +returning +surrounding +participated +formation +lifted +pointed +connected +rome +medicine +laid +taylor +santa +powers +adam +tall +shared +focused +knowing +yards +entrance +falls +##wa +calling +##ad +sources +chosen +beneath +resources +yard +##ite +nominated +silence +zone +defined +##que +gained +thirty +38 +bodies +moon +##ard +adopted +christmas +widely +register +apart +iran +premier +serves +du +unknown +parties +##les +generation +##ff +continues +quick +fields +brigade +quiet +teaching +clothes +impact +weapons +partner +flat +theater +supreme +1938 +37 +relations +##tor +plants +suffered +1936 +wilson +kids +begins +##age +1918 +seats +armed +internet +models +worth +laws +400 +communities +classes +background +knows +thanks +quarter +reaching +humans +carry +killing +format +kong +hong +setting +75 +architecture +disease +railroad +inc +possibly +wish +arthur +thoughts +harry +doors +density +##di +crowd +illinois +stomach +tone +unique +reports +anyway +##ir +liberal +der +vehicle +thick +dry +drug +faced +largely +facility +theme +holds +creation +strange +colonel +##mi +revolution +bell +politics +turns +silent +rail +relief +independence +combat +shape +write +determined +sales +learned +4th +finger +oxford +providing +1937 +heritage +fiction +situated +designated +allowing +distribution +hosted +##est +sight +interview +estimated +reduced +##ria +toronto +footballer +keeping +guys +damn +claim +motion +sport +sixth +stayed +##ze +en +rear +receive +handed +twelve +dress +audience +granted +brazil +##well +spirit +##ated +noticed +etc +olympic +representative +eric +tight +trouble +reviews +drink +vampire +missing +roles +ranked +newly +household +finals +wave +critics +##ee +phase +massachusetts +pilot +unlike +philadelphia +bright +guns +crown +organizations +roof +42 +respectively +clearly +tongue +marked +circle +fox +korea +bronze +brian +expanded +sexual +supply +yourself +inspired +labour +fc +##ah +reference +vision +draft +connection +brand +reasons +1935 +classic +driving +trip +jesus +cells +entry +1920 +neither +trail +claims +atlantic +orders +labor +nose +afraid +identified +intelligence +calls +cancer +attacked +passing +stephen +positions +imperial +grey +jason +39 +sunday +48 +swedish +avoid +extra +uncle +message +covers +allows +surprise +materials +fame +hunter +##ji +1930 +citizens +figures +davis +environmental +confirmed +shit +titles +di +performing +difference +acts +attacks +##ov +existing +votes +opportunity +nor +shop +entirely +trains +opposite +pakistan +##pa +develop +resulted +representatives +actions +reality +pressed +##ish +barely +wine +conversation +faculty +northwest +ends +documentary +nuclear +stock +grace +sets +eat +alternative +##ps +bag +resulting +creating +surprised +cemetery +1919 +drop +finding +sarah +cricket +streets +tradition +ride +1933 +exhibition +target +ear +explained +rain +composer +injury +apartment +municipal +educational +occupied +netherlands +clean +billion +constitution +learn +1914 +maximum +classical +francis +lose +opposition +jose +ontario +bear +core +hills +rolled +ending +drawn +permanent +fun +##tes +##lla +lewis +sites +chamber +ryan +##way +scoring +height +1934 +##house +lyrics +staring +55 +officials +1917 +snow +oldest +##tic +orange +##ger +qualified +interior +apparently +succeeded +thousand +dinner +lights +existence +fans +heavily +41 +greatest +conservative +send +bowl +plus +enter +catch +##un +economy +duty +1929 +speech +authorities +princess +performances +versions +shall +graduate +pictures +effective +remembered +poetry +desk +crossed +starring +starts +passenger +sharp +##ant +acres +ass +weather +falling +rank +fund +supporting +check +adult +publishing +heads +cm +southeast +lane +##burg +application +bc +##ura +les +condition +transfer +prevent +display +ex +regions +earl +federation +cool +relatively +answered +besides +1928 +obtained +portion +##town +mix +##ding +reaction +liked +dean +express +peak +1932 +##tte +counter +religion +chain +rare +miller +convention +aid +lie +vehicles +mobile +perform +squad +wonder +lying +crazy +sword +##ping +attempted +centuries +weren +philosophy +category +##ize +anna +interested +47 +sweden +wolf +frequently +abandoned +kg +literary +alliance +task +entitled +##ay +threw +promotion +factory +tiny +soccer +visited +matt +fm +achieved +52 +defence +internal +persian +43 +methods +##ging +arrested +otherwise +cambridge +programming +villages +elementary +districts +rooms +criminal +conflict +worry +trained +1931 +attempts +waited +signal +bird +truck +subsequent +programme +##ol +ad +49 +communist +details +faith +sector +patrick +carrying +laugh +##ss +controlled +korean +showing +origin +fuel +evil +1927 +##ent +brief +identity +darkness +address +pool +missed +publication +web +planet +ian +anne +wings +invited +##tt +briefly +standards +kissed +##be +ideas +climate +causing +walter +worse +albert +articles +winners +desire +aged +northeast +dangerous +gate +doubt +1922 +wooden +multi +##ky +poet +rising +funding +46 +communications +communication +violence +copies +prepared +ford +investigation +skills +1924 +pulling +electronic +##ak +##ial +##han +containing +ultimately +offices +singing +understanding +restaurant +tomorrow +fashion +christ +ward +da +pope +stands +5th +flow +studios +aired +commissioned +contained +exist +fresh +americans +##per +wrestling +approved +kid +employed +respect +suit +1925 +angel +asking +increasing +frame +angry +selling +1950s +thin +finds +##nd +temperature +statement +ali +explain +inhabitants +towns +extensive +narrow +51 +jane +flowers +images +promise +somewhere +object +fly +closely +##ls +1912 +bureau +cape +1926 +weekly +presidential +legislative +1921 +##ai +##au +launch +founding +##ny +978 +##ring +artillery +strike +un +institutions +roll +writers +landing +chose +kevin +anymore +pp +##ut +attorney +fit +dan +billboard +receiving +agricultural +breaking +sought +dave +admitted +lands +mexican +##bury +charlie +specifically +hole +iv +howard +credit +moscow +roads +accident +1923 +proved +wear +struck +hey +guards +stuff +slid +expansion +1915 +cat +anthony +##kin +melbourne +opposed +sub +southwest +architect +failure +plane +1916 +##ron +map +camera +tank +listen +regarding +wet +introduction +metropolitan +link +ep +fighter +inch +grown +gene +anger +fixed +buy +dvd +khan +domestic +worldwide +chapel +mill +functions +examples +##head +developing +1910 +turkey +hits +pocket +antonio +papers +grow +unless +circuit +18th +concerned +attached +journalist +selection +journey +converted +provincial +painted +hearing +aren +bands +negative +aside +wondered +knight +lap +survey +ma +##ow +noise +billy +##ium +shooting +guide +bedroom +priest +resistance +motor +homes +sounded +giant +##mer +150 +scenes +equal +comic +patients +hidden +solid +actual +bringing +afternoon +touched +funds +wedding +consisted +marie +canal +sr +kim +treaty +turkish +recognition +residence +cathedral +broad +knees +incident +shaped +fired +norwegian +handle +cheek +contest +represent +##pe +representing +beauty +##sen +birds +advantage +emergency +wrapped +drawing +notice +pink +broadcasting +##ong +somehow +bachelor +seventh +collected +registered +establishment +alan +assumed +chemical +personnel +roger +retirement +jeff +portuguese +wore +tied +device +threat +progress +advance +##ised +banks +hired +manchester +nfl +teachers +structures +forever +##bo +tennis +helping +saturday +sale +applications +junction +hip +incorporated +neighborhood +dressed +ceremony +##ds +influenced +hers +visual +stairs +decades +inner +kansas +hung +hoped +gain +scheduled +downtown +engaged +austria +clock +norway +certainly +pale +protected +1913 +victor +employees +plate +putting +surrounded +##ists +finishing +blues +tropical +##ries +minnesota +consider +philippines +accept +54 +retrieved +1900 +concern +anderson +properties +institution +gordon +successfully +vietnam +##dy +backing +outstanding +muslim +crossing +folk +producing +usual +demand +occurs +observed +lawyer +educated +##ana +kelly +string +pleasure +budget +items +quietly +colorado +philip +typical +##worth +derived +600 +survived +asks +mental +##ide +56 +jake +jews +distinguished +ltd +1911 +sri +extremely +53 +athletic +loud +thousands +worried +shadow +transportation +horses +weapon +arena +importance +users +tim +objects +contributed +dragon +douglas +aware +senator +johnny +jordan +sisters +engines +flag +investment +samuel +shock +capable +clark +row +wheel +refers +session +familiar +biggest +wins +hate +maintained +drove +hamilton +request +expressed +injured +underground +churches +walker +wars +tunnel +passes +stupid +agriculture +softly +cabinet +regarded +joining +indiana +##ea +##ms +push +dates +spend +behavior +woods +protein +gently +chase +morgan +mention +burning +wake +combination +occur +mirror +leads +jimmy +indeed +impossible +singapore +paintings +covering +##nes +soldier +locations +attendance +sell +historian +wisconsin +invasion +argued +painter +diego +changing +egypt +##don +experienced +inches +##ku +missouri +vol +grounds +spoken +switzerland +##gan +reform +rolling +ha +forget +massive +resigned +burned +allen +tennessee +locked +values +improved +##mo +wounded +universe +sick +dating +facing +pack +purchase +user +##pur +moments +##ul +merged +anniversary +1908 +coal +brick +understood +causes +dynasty +queensland +establish +stores +crisis +promote +hoping +views +cards +referee +extension +##si +raise +arizona +improve +colonial +formal +charged +##rt +palm +lucky +hide +rescue +faces +95 +feelings +candidates +juan +##ell +goods +6th +courses +weekend +59 +luke +cash +fallen +##om +delivered +affected +installed +carefully +tries +swiss +hollywood +costs +lincoln +responsibility +##he +shore +file +proper +normally +maryland +assistance +jump +constant +offering +friendly +waters +persons +realize +contain +trophy +800 +partnership +factor +58 +musicians +cry +bound +oregon +indicated +hero +houston +medium +##ure +consisting +somewhat +##ara +57 +cycle +##che +beer +moore +frederick +gotten +eleven +worst +weak +approached +arranged +chin +loan +universal +bond +fifteen +pattern +disappeared +##ney +translated +##zed +lip +arab +capture +interests +insurance +##chi +shifted +cave +prix +warning +sections +courts +coat +plot +smell +feed +golf +favorite +maintain +knife +vs +voted +degrees +finance +quebec +opinion +translation +manner +ruled +operate +productions +choose +musician +discovery +confused +tired +separated +stream +techniques +committed +attend +ranking +kings +throw +passengers +measure +horror +fan +mining +sand +danger +salt +calm +decade +dam +require +runner +##ik +rush +associate +greece +##ker +rivers +consecutive +matthew +##ski +sighed +sq +documents +steam +edited +closing +tie +accused +1905 +##ini +islamic +distributed +directors +organisation +bruce +7th +breathing +mad +lit +arrival +concrete +taste +08 +composition +shaking +faster +amateur +adjacent +stating +1906 +twin +flew +##ran +tokyo +publications +##tone +obviously +ridge +storage +1907 +carl +pages +concluded +desert +driven +universities +ages +terminal +sequence +borough +250 +constituency +creative +cousin +economics +dreams +margaret +notably +reduce +montreal +mode +17th +ears +saved +jan +vocal +##ica +1909 +andy +##jo +riding +roughly +threatened +##ise +meters +meanwhile +landed +compete +repeated +grass +czech +regularly +charges +tea +sudden +appeal +##ung +solution +describes +pierre +classification +glad +parking +##ning +belt +physics +99 +rachel +add +hungarian +participate +expedition +damaged +gift +childhood +85 +fifty +##red +mathematics +jumped +letting +defensive +mph +##ux +##gh +testing +##hip +hundreds +shoot +owners +matters +smoke +israeli +kentucky +dancing +mounted +grandfather +emma +designs +profit +argentina +##gs +truly +li +lawrence +cole +begun +detroit +willing +branches +smiling +decide +miami +enjoyed +recordings +##dale +poverty +ethnic +gay +##bi +gary +arabic +09 +accompanied +##one +##ons +fishing +determine +residential +acid +##ary +alice +returns +starred +mail +##ang +jonathan +strategy +##ue +net +forty +cook +businesses +equivalent +commonwealth +distinct +ill +##cy +seriously +##ors +##ped +shift +harris +replace +rio +imagine +formula +ensure +##ber +additionally +scheme +conservation +occasionally +purposes +feels +favor +##and +##ore +1930s +contrast +hanging +hunt +movies +1904 +instruments +victims +danish +christopher +busy +demon +sugar +earliest +colony +studying +balance +duties +##ks +belgium +slipped +carter +05 +visible +stages +iraq +fifa +##im +commune +forming +zero +07 +continuing +talked +counties +legend +bathroom +option +tail +clay +daughters +afterwards +severe +jaw +visitors +##ded +devices +aviation +russell +kate +##vi +entering +subjects +##ino +temporary +swimming +forth +smooth +ghost +audio +bush +operates +rocks +movements +signs +eddie +##tz +ann +voices +honorary +06 +memories +dallas +pure +measures +racial +promised +66 +harvard +ceo +16th +parliamentary +indicate +benefit +flesh +dublin +louisiana +1902 +1901 +patient +sleeping +1903 +membership +coastal +medieval +wanting +element +scholars +rice +62 +limit +survive +makeup +rating +definitely +collaboration +obvious +##tan +boss +ms +baron +birthday +linked +soil +diocese +##lan +ncaa +##mann +offensive +shell +shouldn +waist +##tus +plain +ross +organ +resolution +manufacturing +adding +relative +kennedy +98 +whilst +moth +marketing +gardens +crash +72 +heading +partners +credited +carlos +moves +cable +##zi +marshall +##out +depending +bottle +represents +rejected +responded +existed +04 +jobs +denmark +lock +##ating +treated +graham +routes +talent +commissioner +drugs +secure +tests +reign +restored +photography +##gi +contributions +oklahoma +designer +disc +grin +seattle +robin +paused +atlanta +unusual +##gate +praised +las +laughing +satellite +hungary +visiting +##sky +interesting +factors +deck +poems +norman +##water +stuck +speaker +rifle +domain +premiered +##her +dc +comics +actors +01 +reputation +eliminated +8th +ceiling +prisoners +script +##nce +leather +austin +mississippi +rapidly +admiral +parallel +charlotte +guilty +tools +gender +divisions +fruit +##bs +laboratory +nelson +fantasy +marry +rapid +aunt +tribe +requirements +aspects +suicide +amongst +adams +bone +ukraine +abc +kick +sees +edinburgh +clothing +column +rough +gods +hunting +broadway +gathered +concerns +##ek +spending +ty +12th +snapped +requires +solar +bones +cavalry +##tta +iowa +drinking +waste +index +franklin +charity +thompson +stewart +tip +flash +landscape +friday +enjoy +singh +poem +listening +##back +eighth +fred +differences +adapted +bomb +ukrainian +surgery +corporate +masters +anywhere +##more +waves +odd +sean +portugal +orleans +dick +debate +kent +eating +puerto +cleared +96 +expect +cinema +97 +guitarist +blocks +electrical +agree +involving +depth +dying +panel +struggle +##ged +peninsula +adults +novels +emerged +vienna +metro +debuted +shoes +tamil +songwriter +meets +prove +beating +instance +heaven +scared +sending +marks +artistic +passage +superior +03 +significantly +shopping +##tive +retained +##izing +malaysia +technique +cheeks +##ola +warren +maintenance +destroy +extreme +allied +120 +appearing +##yn +fill +advice +alabama +qualifying +policies +cleveland +hat +battery +smart +authors +10th +soundtrack +acted +dated +lb +glance +equipped +coalition +funny +outer +ambassador +roy +possibility +couples +campbell +dna +loose +ethan +supplies +1898 +gonna +88 +monster +##res +shake +agents +frequency +springs +dogs +practices +61 +gang +plastic +easier +suggests +gulf +blade +exposed +colors +industries +markets +pan +nervous +electoral +charts +legislation +ownership +##idae +mac +appointment +shield +copy +assault +socialist +abbey +monument +license +throne +employment +jay +93 +replacement +charter +cloud +powered +suffering +accounts +oak +connecticut +strongly +wright +colour +crystal +13th +context +welsh +networks +voiced +gabriel +jerry +##cing +forehead +mp +##ens +manage +schedule +totally +remix +##ii +forests +occupation +print +nicholas +brazilian +strategic +vampires +engineers +76 +roots +seek +correct +instrumental +und +alfred +backed +hop +##des +stanley +robinson +traveled +wayne +welcome +austrian +achieve +67 +exit +rates +1899 +strip +whereas +##cs +sing +deeply +adventure +bobby +rick +jamie +careful +components +cap +useful +personality +knee +##shi +pushing +hosts +02 +protest +ca +ottoman +symphony +##sis +63 +boundary +1890 +processes +considering +considerable +tons +##work +##ft +##nia +cooper +trading +dear +conduct +91 +illegal +apple +revolutionary +holiday +definition +harder +##van +jacob +circumstances +destruction +##lle +popularity +grip +classified +liverpool +donald +baltimore +flows +seeking +honour +approval +92 +mechanical +till +happening +statue +critic +increasingly +immediate +describe +commerce +stare +##ster +indonesia +meat +rounds +boats +baker +orthodox +depression +formally +worn +naked +claire +muttered +sentence +11th +emily +document +77 +criticism +wished +vessel +spiritual +bent +virgin +parker +minimum +murray +lunch +danny +printed +compilation +keyboards +false +blow +belonged +68 +raising +78 +cutting +##board +pittsburgh +##up +9th +shadows +81 +hated +indigenous +jon +15th +barry +scholar +ah +##zer +oliver +##gy +stick +susan +meetings +attracted +spell +romantic +##ver +ye +1895 +photo +demanded +customers +##ac +1896 +logan +revival +keys +modified +commanded +jeans +##ious +upset +raw +phil +detective +hiding +resident +vincent +##bly +experiences +diamond +defeating +coverage +lucas +external +parks +franchise +helen +bible +successor +percussion +celebrated +il +lift +profile +clan +romania +##ied +mills +##su +nobody +achievement +shrugged +fault +1897 +rhythm +initiative +breakfast +carbon +700 +69 +lasted +violent +74 +wound +ken +killer +gradually +filmed +°c +dollars +processing +94 +remove +criticized +guests +sang +chemistry +##vin +legislature +disney +##bridge +uniform +escaped +integrated +proposal +purple +denied +liquid +karl +influential +morris +nights +stones +intense +experimental +twisted +71 +84 +##ld +pace +nazi +mitchell +ny +blind +reporter +newspapers +14th +centers +burn +basin +forgotten +surviving +filed +collections +monastery +losses +manual +couch +description +appropriate +merely +tag +missions +sebastian +restoration +replacing +triple +73 +elder +julia +warriors +benjamin +julian +convinced +stronger +amazing +declined +versus +merchant +happens +output +finland +bare +barbara +absence +ignored +dawn +injuries +##port +producers +##ram +82 +luis +##ities +kw +admit +expensive +electricity +nba +exception +symbol +##ving +ladies +shower +sheriff +characteristics +##je +aimed +button +ratio +effectively +summit +angle +jury +bears +foster +vessels +pants +executed +evans +dozen +advertising +kicked +patrol +1889 +competitions +lifetime +principles +athletics +##logy +birmingham +sponsored +89 +rob +nomination +1893 +acoustic +##sm +creature +longest +##tra +credits +harbor +dust +josh +##so +territories +milk +infrastructure +completion +thailand +indians +leon +archbishop +##sy +assist +pitch +blake +arrangement +girlfriend +serbian +operational +hence +sad +scent +fur +dj +sessions +hp +refer +rarely +##ora +exists +1892 +##ten +scientists +dirty +penalty +burst +portrait +seed +79 +pole +limits +rival +1894 +stable +alpha +grave +constitutional +alcohol +arrest +flower +mystery +devil +architectural +relationships +greatly +habitat +##istic +larry +progressive +remote +cotton +##ics +##ok +preserved +reaches +##ming +cited +86 +vast +scholarship +decisions +cbs +joy +teach +1885 +editions +knocked +eve +searching +partly +participation +gap +animated +fate +excellent +##ett +na +87 +alternate +saints +youngest +##ily +climbed +##ita +##tors +suggest +##ct +discussion +staying +choir +lakes +jacket +revenue +nevertheless +peaked +instrument +wondering +annually +managing +neil +1891 +signing +terry +##ice +apply +clinical +brooklyn +aim +catherine +fuck +farmers +figured +ninth +pride +hugh +evolution +ordinary +involvement +comfortable +shouted +tech +encouraged +taiwan +representation +sharing +##lia +##em +panic +exact +cargo +competing +fat +cried +83 +1920s +occasions +pa +cabin +borders +utah +marcus +##isation +badly +muscles +##ance +victorian +transition +warner +bet +permission +##rin +slave +terrible +similarly +shares +seth +uefa +possession +medals +benefits +colleges +lowered +perfectly +mall +transit +##ye +##kar +publisher +##ened +harrison +deaths +elevation +##ae +asleep +machines +sigh +ash +hardly +argument +occasion +parent +leo +decline +1888 +contribution +##ua +concentration +1000 +opportunities +hispanic +guardian +extent +emotions +hips +mason +volumes +bloody +controversy +diameter +steady +mistake +phoenix +identify +violin +##sk +departure +richmond +spin +funeral +enemies +1864 +gear +literally +connor +random +sergeant +grab +confusion +1865 +transmission +informed +op +leaning +sacred +suspended +thinks +gates +portland +luck +agencies +yours +hull +expert +muscle +layer +practical +sculpture +jerusalem +latest +lloyd +statistics +deeper +recommended +warrior +arkansas +mess +supports +greg +eagle +1880 +recovered +rated +concerts +rushed +##ano +stops +eggs +files +premiere +keith +##vo +delhi +turner +pit +affair +belief +paint +##zing +mate +##ach +##ev +victim +##ology +withdrew +bonus +styles +fled +##ud +glasgow +technologies +funded +nbc +adaptation +##ata +portrayed +cooperation +supporters +judges +bernard +justin +hallway +ralph +##ick +graduating +controversial +distant +continental +spider +bite +##ho +recognize +intention +mixing +##ese +egyptian +bow +tourism +suppose +claiming +tiger +dominated +participants +vi +##ru +nurse +partially +tape +##rum +psychology +##rn +essential +touring +duo +voting +civilian +emotional +channels +##king +apparent +hebrew +1887 +tommy +carrier +intersection +beast +hudson +##gar +##zo +lab +nova +bench +discuss +costa +##ered +detailed +behalf +drivers +unfortunately +obtain +##lis +rocky +##dae +siege +friendship +honey +##rian +1861 +amy +hang +posted +governments +collins +respond +wildlife +preferred +operator +##po +laura +pregnant +videos +dennis +suspected +boots +instantly +weird +automatic +businessman +alleged +placing +throwing +ph +mood +1862 +perry +venue +jet +remainder +##lli +##ci +passion +biological +boyfriend +1863 +dirt +buffalo +ron +segment +fa +abuse +##era +genre +thrown +stroke +colored +stress +exercise +displayed +##gen +struggled +##tti +abroad +dramatic +wonderful +thereafter +madrid +component +widespread +##sed +tale +citizen +todd +monday +1886 +vancouver +overseas +forcing +crying +descent +##ris +discussed +substantial +ranks +regime +1870 +provinces +switch +drum +zane +ted +tribes +proof +lp +cream +researchers +volunteer +manor +silk +milan +donated +allies +venture +principle +delivery +enterprise +##ves +##ans +bars +traditionally +witch +reminded +copper +##uk +pete +inter +links +colin +grinned +elsewhere +competitive +frequent +##oy +scream +##hu +tension +texts +submarine +finnish +defending +defend +pat +detail +1884 +affiliated +stuart +themes +villa +periods +tool +belgian +ruling +crimes +answers +folded +licensed +resort +demolished +hans +lucy +1881 +lion +traded +photographs +writes +craig +##fa +trials +generated +beth +noble +debt +percentage +yorkshire +erected +ss +viewed +grades +confidence +ceased +islam +telephone +retail +##ible +chile +m² +roberts +sixteen +##ich +commented +hampshire +innocent +dual +pounds +checked +regulations +afghanistan +sung +rico +liberty +assets +bigger +options +angels +relegated +tribute +wells +attending +leaf +##yan +butler +romanian +forum +monthly +lisa +patterns +gmina +##tory +madison +hurricane +rev +##ians +bristol +##ula +elite +valuable +disaster +democracy +awareness +germans +freyja +##ins +loop +absolutely +paying +populations +maine +sole +prayer +spencer +releases +doorway +bull +##ani +lover +midnight +conclusion +##sson +thirteen +lily +mediterranean +##lt +nhl +proud +sample +##hill +drummer +guinea +##ova +murphy +climb +##ston +instant +attributed +horn +ain +railways +steven +##ao +autumn +ferry +opponent +root +traveling +secured +corridor +stretched +tales +sheet +trinity +cattle +helps +indicates +manhattan +murdered +fitted +1882 +gentle +grandmother +mines +shocked +vegas +produces +##light +caribbean +##ou +belong +continuous +desperate +drunk +historically +trio +waved +raf +dealing +nathan +bat +murmured +interrupted +residing +scientist +pioneer +harold +aaron +##net +delta +attempting +minority +mini +believes +chorus +tend +lots +eyed +indoor +load +shots +updated +jail +##llo +concerning +connecting +wealth +##ved +slaves +arrive +rangers +sufficient +rebuilt +##wick +cardinal +flood +muhammad +whenever +relation +runners +moral +repair +viewers +arriving +revenge +punk +assisted +bath +fairly +breathe +lists +innings +illustrated +whisper +nearest +voters +clinton +ties +ultimate +screamed +beijing +lions +andre +fictional +gathering +comfort +radar +suitable +dismissed +hms +ban +pine +wrist +atmosphere +voivodeship +bid +timber +##ned +##nan +giants +##ane +cameron +recovery +uss +identical +categories +switched +serbia +laughter +noah +ensemble +therapy +peoples +touching +##off +locally +pearl +platforms +everywhere +ballet +tables +lanka +herbert +outdoor +toured +derek +1883 +spaces +contested +swept +1878 +exclusive +slight +connections +##dra +winds +prisoner +collective +bangladesh +tube +publicly +wealthy +thai +##ys +isolated +select +##ric +insisted +pen +fortune +ticket +spotted +reportedly +animation +enforcement +tanks +110 +decides +wider +lowest +owen +##time +nod +hitting +##hn +gregory +furthermore +magazines +fighters +solutions +##ery +pointing +requested +peru +reed +chancellor +knights +mask +worker +eldest +flames +reduction +1860 +volunteers +##tis +reporting +##hl +wire +advisory +endemic +origins +settlers +pursue +knock +consumer +1876 +eu +compound +creatures +mansion +sentenced +ivan +deployed +guitars +frowned +involves +mechanism +kilometers +perspective +shops +maps +terminus +duncan +alien +fist +bridges +##pers +heroes +fed +derby +swallowed +##ros +patent +sara +illness +characterized +adventures +slide +hawaii +jurisdiction +##op +organised +##side +adelaide +walks +biology +se +##ties +rogers +swing +tightly +boundaries +##rie +prepare +implementation +stolen +##sha +certified +colombia +edwards +garage +##mm +recalled +##ball +rage +harm +nigeria +breast +##ren +furniture +pupils +settle +##lus +cuba +balls +client +alaska +21st +linear +thrust +celebration +latino +genetic +terror +##cia +##ening +lightning +fee +witness +lodge +establishing +skull +##ique +earning +hood +##ei +rebellion +wang +sporting +warned +missile +devoted +activist +porch +worship +fourteen +package +1871 +decorated +##shire +housed +##ock +chess +sailed +doctors +oscar +joan +treat +garcia +harbour +jeremy +##ire +traditions +dominant +jacques +##gon +##wan +relocated +1879 +amendment +sized +companion +simultaneously +volleyball +spun +acre +increases +stopping +loves +belongs +affect +drafted +tossed +scout +battles +1875 +filming +shoved +munich +tenure +vertical +romance +pc +##cher +argue +##ical +craft +ranging +www +opens +honest +tyler +yesterday +virtual +##let +muslims +reveal +snake +immigrants +radical +screaming +speakers +firing +saving +belonging +ease +lighting +prefecture +blame +farmer +hungry +grows +rubbed +beam +sur +subsidiary +##cha +armenian +sao +dropping +conventional +##fer +microsoft +reply +qualify +spots +1867 +sweat +festivals +##ken +immigration +physician +discover +exposure +sandy +explanation +isaac +implemented +##fish +hart +initiated +connect +stakes +presents +heights +householder +pleased +tourist +regardless +slip +closest +##ction +surely +sultan +brings +riley +preparation +aboard +slammed +baptist +experiment +ongoing +interstate +organic +playoffs +##ika +1877 +130 +##tar +hindu +error +tours +tier +plenty +arrangements +talks +trapped +excited +sank +ho +athens +1872 +denver +welfare +suburb +athletes +trick +diverse +belly +exclusively +yelled +1868 +##med +conversion +##ette +1874 +internationally +computers +conductor +abilities +sensitive +hello +dispute +measured +globe +rocket +prices +amsterdam +flights +tigers +inn +municipalities +emotion +references +3d +##mus +explains +airlines +manufactured +pm +archaeological +1873 +interpretation +devon +comment +##ites +settlements +kissing +absolute +improvement +suite +impressed +barcelona +sullivan +jefferson +towers +jesse +julie +##tin +##lu +grandson +hi +gauge +regard +rings +interviews +trace +raymond +thumb +departments +burns +serial +bulgarian +scores +demonstrated +##ix +1866 +kyle +alberta +underneath +romanized +##ward +relieved +acquisition +phrase +cliff +reveals +han +cuts +merger +custom +##dar +nee +gilbert +graduation +##nts +assessment +cafe +difficulty +demands +swung +democrat +jennifer +commons +1940s +grove +##yo +completing +focuses +sum +substitute +bearing +stretch +reception +##py +reflected +essentially +destination +pairs +##ched +survival +resource +##bach +promoting +doubles +messages +tear +##down +##fully +parade +florence +harvey +incumbent +partial +framework +900 +pedro +frozen +procedure +olivia +controls +##mic +shelter +personally +temperatures +##od +brisbane +tested +sits +marble +comprehensive +oxygen +leonard +##kov +inaugural +iranian +referring +quarters +attitude +##ivity +mainstream +lined +mars +dakota +norfolk +unsuccessful +##° +explosion +helicopter +congressional +##sing +inspector +bitch +seal +departed +divine +##ters +coaching +examination +punishment +manufacturer +sink +columns +unincorporated +signals +nevada +squeezed +dylan +dining +photos +martial +manuel +eighteen +elevator +brushed +plates +ministers +ivy +congregation +##len +slept +specialized +taxes +curve +restricted +negotiations +likes +statistical +arnold +inspiration +execution +bold +intermediate +significance +margin +ruler +wheels +gothic +intellectual +dependent +listened +eligible +buses +widow +syria +earn +cincinnati +collapsed +recipient +secrets +accessible +philippine +maritime +goddess +clerk +surrender +breaks +playoff +database +##ified +##lon +ideal +beetle +aspect +soap +regulation +strings +expand +anglo +shorter +crosses +retreat +tough +coins +wallace +directions +pressing +##oon +shipping +locomotives +comparison +topics +nephew +##mes +distinction +honors +travelled +sierra +ibn +##over +fortress +sa +recognised +carved +1869 +clients +##dan +intent +##mar +coaches +describing +bread +##ington +beaten +northwestern +##ona +merit +youtube +collapse +challenges +em +historians +objective +submitted +virus +attacking +drake +assume +##ere +diseases +marc +stem +leeds +##cus +##ab +farming +glasses +##lock +visits +nowhere +fellowship +relevant +carries +restaurants +experiments +101 +constantly +bases +targets +shah +tenth +opponents +verse +territorial +##ira +writings +corruption +##hs +instruction +inherited +reverse +emphasis +##vic +employee +arch +keeps +rabbi +watson +payment +uh +##ala +nancy +##tre +venice +fastest +sexy +banned +adrian +properly +ruth +touchdown +dollar +boards +metre +circles +edges +favour +comments +ok +travels +liberation +scattered +firmly +##ular +holland +permitted +diesel +kenya +den +originated +##ral +demons +resumed +dragged +rider +##rus +servant +blinked +extend +torn +##ias +##sey +input +meal +everybody +cylinder +kinds +camps +##fe +bullet +logic +##wn +croatian +evolved +healthy +fool +chocolate +wise +preserve +pradesh +##ess +respective +1850 +##ew +chicken +artificial +gross +corresponding +convicted +cage +caroline +dialogue +##dor +narrative +stranger +mario +br +christianity +failing +trent +commanding +buddhist +1848 +maurice +focusing +yale +bike +altitude +##ering +mouse +revised +##sley +veteran +##ig +pulls +theology +crashed +campaigns +legion +##ability +drag +excellence +customer +cancelled +intensity +excuse +##lar +liga +participating +contributing +printing +##burn +variable +##rk +curious +bin +legacy +renaissance +##my +symptoms +binding +vocalist +dancer +##nie +grammar +gospel +democrats +ya +enters +sc +diplomatic +hitler +##ser +clouds +mathematical +quit +defended +oriented +##heim +fundamental +hardware +impressive +equally +convince +confederate +guilt +chuck +sliding +##ware +magnetic +narrowed +petersburg +bulgaria +otto +phd +skill +##ama +reader +hopes +pitcher +reservoir +hearts +automatically +expecting +mysterious +bennett +extensively +imagined +seeds +monitor +fix +##ative +journalism +struggling +signature +ranch +encounter +photographer +observation +protests +##pin +influences +##hr +calendar +##all +cruz +croatia +locomotive +hughes +naturally +shakespeare +basement +hook +uncredited +faded +theories +approaches +dare +phillips +filling +fury +obama +##ain +efficient +arc +deliver +min +raid +breeding +inducted +leagues +efficiency +axis +montana +eagles +##ked +supplied +instructions +karen +picking +indicating +trap +anchor +practically +christians +tomb +vary +occasional +electronics +lords +readers +newcastle +faint +innovation +collect +situations +engagement +160 +claude +mixture +##feld +peer +tissue +logo +lean +##ration +°f +floors +##ven +architects +reducing +##our +##ments +rope +1859 +ottawa +##har +samples +banking +declaration +proteins +resignation +francois +saudi +advocate +exhibited +armor +twins +divorce +##ras +abraham +reviewed +jo +temporarily +matrix +physically +pulse +curled +##ena +difficulties +bengal +usage +##ban +annie +riders +certificate +##pi +holes +warsaw +distinctive +jessica +##mon +mutual +1857 +customs +circular +eugene +removal +loaded +mere +vulnerable +depicted +generations +dame +heir +enormous +lightly +climbing +pitched +lessons +pilots +nepal +ram +google +preparing +brad +louise +renowned +##₂ +liam +##ably +plaza +shaw +sophie +brilliant +bills +##bar +##nik +fucking +mainland +server +pleasant +seized +veterans +jerked +fail +beta +brush +radiation +stored +warmth +southeastern +nate +sin +raced +berkeley +joke +athlete +designation +trunk +##low +roland +qualification +archives +heels +artwork +receives +judicial +reserves +##bed +woke +installation +abu +floating +fake +lesser +excitement +interface +concentrated +addressed +characteristic +amanda +saxophone +monk +auto +##bus +releasing +egg +dies +interaction +defender +ce +outbreak +glory +loving +##bert +sequel +consciousness +http +awake +ski +enrolled +##ress +handling +rookie +brow +somebody +biography +warfare +amounts +contracts +presentation +fabric +dissolved +challenged +meter +psychological +lt +elevated +rally +accurate +##tha +hospitals +undergraduate +specialist +venezuela +exhibit +shed +nursing +protestant +fluid +structural +footage +jared +consistent +prey +##ska +succession +reflect +exile +lebanon +wiped +suspect +shanghai +resting +integration +preservation +marvel +variant +pirates +sheep +rounded +capita +sailing +colonies +manuscript +deemed +variations +clarke +functional +emerging +boxing +relaxed +curse +azerbaijan +heavyweight +nickname +editorial +rang +grid +tightened +earthquake +flashed +miguel +rushing +##ches +improvements +boxes +brooks +180 +consumption +molecular +felix +societies +repeatedly +variation +aids +civic +graphics +professionals +realm +autonomous +receiver +delayed +workshop +militia +chairs +trump +canyon +##point +harsh +extending +lovely +happiness +##jan +stake +eyebrows +embassy +wellington +hannah +##ella +sony +corners +bishops +swear +cloth +contents +xi +namely +commenced +1854 +stanford +nashville +courage +graphic +commitment +garrison +##bin +hamlet +clearing +rebels +attraction +literacy +cooking +ruins +temples +jenny +humanity +celebrate +hasn +freight +sixty +rebel +bastard +##art +newton +##ada +deer +##ges +##ching +smiles +delaware +singers +##ets +approaching +assists +flame +##ph +boulevard +barrel +planted +##ome +pursuit +##sia +consequences +posts +shallow +invitation +rode +depot +ernest +kane +rod +concepts +preston +topic +chambers +striking +blast +arrives +descendants +montgomery +ranges +worlds +##lay +##ari +span +chaos +praise +##ag +fewer +1855 +sanctuary +mud +fbi +##ions +programmes +maintaining +unity +harper +bore +handsome +closure +tournaments +thunder +nebraska +linda +facade +puts +satisfied +argentine +dale +cork +dome +panama +##yl +1858 +tasks +experts +##ates +feeding +equation +##las +##ida +##tu +engage +bryan +##ax +um +quartet +melody +disbanded +sheffield +blocked +gasped +delay +kisses +maggie +connects +##non +sts +poured +creator +publishers +##we +guided +ellis +extinct +hug +gaining +##ord +complicated +##bility +poll +clenched +investigate +##use +thereby +quantum +spine +cdp +humor +kills +administered +semifinals +##du +encountered +ignore +##bu +commentary +##maker +bother +roosevelt +140 +plains +halfway +flowing +cultures +crack +imprisoned +neighboring +airline +##ses +##view +##mate +##ec +gather +wolves +marathon +transformed +##ill +cruise +organisations +carol +punch +exhibitions +numbered +alarm +ratings +daddy +silently +##stein +queens +colours +impression +guidance +liu +tactical +##rat +marshal +della +arrow +##ings +rested +feared +tender +owns +bitter +advisor +escort +##ides +spare +farms +grants +##ene +dragons +encourage +colleagues +cameras +##und +sucked +pile +spirits +prague +statements +suspension +landmark +fence +torture +recreation +bags +permanently +survivors +pond +spy +predecessor +bombing +coup +##og +protecting +transformation +glow +##lands +##book +dug +priests +andrea +feat +barn +jumping +##chen +##ologist +##con +casualties +stern +auckland +pipe +serie +revealing +ba +##bel +trevor +mercy +spectrum +yang +consist +governing +collaborated +possessed +epic +comprises +blew +shane +##ack +lopez +honored +magical +sacrifice +judgment +perceived +hammer +mtv +baronet +tune +das +missionary +sheets +350 +neutral +oral +threatening +attractive +shade +aims +seminary +##master +estates +1856 +michel +wounds +refugees +manufacturers +##nic +mercury +syndrome +porter +##iya +##din +hamburg +identification +upstairs +purse +widened +pause +cared +breathed +affiliate +santiago +prevented +celtic +fisher +125 +recruited +byzantine +reconstruction +farther +##mp +diet +sake +au +spite +sensation +##ert +blank +separation +105 +##hon +vladimir +armies +anime +##lie +accommodate +orbit +cult +sofia +archive +##ify +##box +founders +sustained +disorder +honours +northeastern +mia +crops +violet +threats +blanket +fires +canton +followers +southwestern +prototype +voyage +assignment +altered +moderate +protocol +pistol +##eo +questioned +brass +lifting +1852 +math +authored +##ual +doug +dimensional +dynamic +##san +1851 +pronounced +grateful +quest +uncomfortable +boom +presidency +stevens +relating +politicians +chen +barrier +quinn +diana +mosque +tribal +cheese +palmer +portions +sometime +chester +treasure +wu +bend +download +millions +reforms +registration +##osa +consequently +monitoring +ate +preliminary +brandon +invented +ps +eaten +exterior +intervention +ports +documented +log +displays +lecture +sally +favourite +##itz +vermont +lo +invisible +isle +breed +##ator +journalists +relay +speaks +backward +explore +midfielder +actively +stefan +procedures +cannon +blond +kenneth +centered +servants +chains +libraries +malcolm +essex +henri +slavery +##hal +facts +fairy +coached +cassie +cats +washed +cop +##fi +announcement +item +2000s +vinyl +activated +marco +frontier +growled +curriculum +##das +loyal +accomplished +leslie +ritual +kenny +##00 +vii +napoleon +hollow +hybrid +jungle +stationed +friedrich +counted +##ulated +platinum +theatrical +seated +col +rubber +glen +1840 +diversity +healing +extends +id +provisions +administrator +columbus +##oe +tributary +te +assured +org +##uous +prestigious +examined +lectures +grammy +ronald +associations +bailey +allan +essays +flute +believing +consultant +proceedings +travelling +1853 +kit +kerala +yugoslavia +buddy +methodist +##ith +burial +centres +batman +##nda +discontinued +bo +dock +stockholm +lungs +severely +##nk +citing +manga +##ugh +steal +mumbai +iraqi +robot +celebrity +bride +broadcasts +abolished +pot +joel +overhead +franz +packed +reconnaissance +johann +acknowledged +introduce +handled +doctorate +developments +drinks +alley +palestine +##nis +##aki +proceeded +recover +bradley +grain +patch +afford +infection +nationalist +legendary +##ath +interchange +virtually +gen +gravity +exploration +amber +vital +wishes +powell +doctrine +elbow +screenplay +##bird +contribute +indonesian +pet +creates +##com +enzyme +kylie +discipline +drops +manila +hunger +##ien +layers +suffer +fever +bits +monica +keyboard +manages +##hood +searched +appeals +##bad +testament +grande +reid +##war +beliefs +congo +##ification +##dia +si +requiring +##via +casey +1849 +regret +streak +rape +depends +syrian +sprint +pound +tourists +upcoming +pub +##xi +tense +##els +practiced +echo +nationwide +guild +motorcycle +liz +##zar +chiefs +desired +elena +bye +precious +absorbed +relatives +booth +pianist +##mal +citizenship +exhausted +wilhelm +##ceae +##hed +noting +quarterback +urge +hectares +##gue +ace +holly +##tal +blonde +davies +parked +sustainable +stepping +twentieth +airfield +galaxy +nest +chip +##nell +tan +shaft +paulo +requirement +##zy +paradise +tobacco +trans +renewed +vietnamese +##cker +##ju +suggesting +catching +holmes +enjoying +md +trips +colt +holder +butterfly +nerve +reformed +cherry +bowling +trailer +carriage +goodbye +appreciate +toy +joshua +interactive +enabled +involve +##kan +collar +determination +bunch +facebook +recall +shorts +superintendent +episcopal +frustration +giovanni +nineteenth +laser +privately +array +circulation +##ovic +armstrong +deals +painful +permit +discrimination +##wi +aires +retiring +cottage +ni +##sta +horizon +ellen +jamaica +ripped +fernando +chapters +playstation +patron +lecturer +navigation +behaviour +genes +georgian +export +solomon +rivals +swift +seventeen +rodriguez +princeton +independently +sox +1847 +arguing +entity +casting +hank +criteria +oakland +geographic +milwaukee +reflection +expanding +conquest +dubbed +##tv +halt +brave +brunswick +doi +arched +curtis +divorced +predominantly +somerset +streams +ugly +zoo +horrible +curved +buenos +fierce +dictionary +vector +theological +unions +handful +stability +chan +punjab +segments +##lly +altar +ignoring +gesture +monsters +pastor +##stone +thighs +unexpected +operators +abruptly +coin +compiled +associates +improving +migration +pin +##ose +compact +collegiate +reserved +##urs +quarterfinals +roster +restore +assembled +hurry +oval +##cies +1846 +flags +martha +##del +victories +sharply +##rated +argues +deadly +neo +drawings +symbols +performer +##iel +griffin +restrictions +editing +andrews +java +journals +arabia +compositions +dee +pierce +removing +hindi +casino +runway +civilians +minds +nasa +hotels +##zation +refuge +rent +retain +potentially +conferences +suburban +conducting +##tto +##tions +##tle +descended +massacre +##cal +ammunition +terrain +fork +souls +counts +chelsea +durham +drives +cab +##bank +perth +realizing +palestinian +finn +simpson +##dal +betty +##ule +moreover +particles +cardinals +tent +evaluation +extraordinary +##oid +inscription +##works +wednesday +chloe +maintains +panels +ashley +trucks +##nation +cluster +sunlight +strikes +zhang +##wing +dialect +canon +##ap +tucked +##ws +collecting +##mas +##can +##sville +maker +quoted +evan +franco +aria +buying +cleaning +eva +closet +provision +apollo +clinic +rat +##ez +necessarily +ac +##gle +##ising +venues +flipped +cent +spreading +trustees +checking +authorized +##sco +disappointed +##ado +notion +duration +trumpet +hesitated +topped +brussels +rolls +theoretical +hint +define +aggressive +repeat +wash +peaceful +optical +width +allegedly +mcdonald +strict +copyright +##illa +investors +mar +jam +witnesses +sounding +miranda +michelle +privacy +hugo +harmony +##pp +valid +lynn +glared +nina +102 +headquartered +diving +boarding +gibson +##ncy +albanian +marsh +routine +dealt +enhanced +er +intelligent +substance +targeted +enlisted +discovers +spinning +observations +pissed +smoking +rebecca +capitol +visa +varied +costume +seemingly +indies +compensation +surgeon +thursday +arsenal +westminster +suburbs +rid +anglican +##ridge +knots +foods +alumni +lighter +fraser +whoever +portal +scandal +##ray +gavin +advised +instructor +flooding +terrorist +##ale +teenage +interim +senses +duck +teen +thesis +abby +eager +overcome +##ile +newport +glenn +rises +shame +##cc +prompted +priority +forgot +bomber +nicolas +protective +360 +cartoon +katherine +breeze +lonely +trusted +henderson +richardson +relax +banner +candy +palms +remarkable +##rio +legends +cricketer +essay +ordained +edmund +rifles +trigger +##uri +##away +sail +alert +1830 +audiences +penn +sussex +siblings +pursued +indianapolis +resist +rosa +consequence +succeed +avoided +1845 +##ulation +inland +##tie +##nna +counsel +profession +chronicle +hurried +##una +eyebrow +eventual +bleeding +innovative +cure +##dom +committees +accounting +con +scope +hardy +heather +tenor +gut +herald +codes +tore +scales +wagon +##oo +luxury +tin +prefer +fountain +triangle +bonds +darling +convoy +dried +traced +beings +troy +accidentally +slam +findings +smelled +joey +lawyers +outcome +steep +bosnia +configuration +shifting +toll +brook +performers +lobby +philosophical +construct +shrine +aggregate +boot +cox +phenomenon +savage +insane +solely +reynolds +lifestyle +##ima +nationally +holdings +consideration +enable +edgar +mo +mama +##tein +fights +relegation +chances +atomic +hub +conjunction +awkward +reactions +currency +finale +kumar +underwent +steering +elaborate +gifts +comprising +melissa +veins +reasonable +sunshine +chi +solve +trails +inhabited +elimination +ethics +huh +ana +molly +consent +apartments +layout +marines +##ces +hunters +bulk +##oma +hometown +##wall +##mont +cracked +reads +neighbouring +withdrawn +admission +wingspan +damned +anthology +lancashire +brands +batting +forgive +cuban +awful +##lyn +104 +dimensions +imagination +##ade +dante +##ship +tracking +desperately +goalkeeper +##yne +groaned +workshops +confident +burton +gerald +milton +circus +uncertain +slope +copenhagen +sophia +fog +philosopher +portraits +accent +cycling +varying +gripped +larvae +garrett +specified +scotia +mature +luther +kurt +rap +##kes +aerial +750 +ferdinand +heated +es +transported +##shan +safely +nonetheless +##orn +##gal +motors +demanding +##sburg +startled +##brook +ally +generate +caps +ghana +stained +demo +mentions +beds +ap +afterward +diary +##bling +utility +##iro +richards +1837 +conspiracy +conscious +shining +footsteps +observer +cyprus +urged +loyalty +developer +probability +olive +upgraded +gym +miracle +insects +graves +1844 +ourselves +hydrogen +amazon +katie +tickets +poets +##pm +planes +##pan +prevention +witnessed +dense +jin +randy +tang +warehouse +monroe +bang +archived +elderly +investigations +alec +granite +mineral +conflicts +controlling +aboriginal +carlo +##zu +mechanics +stan +stark +rhode +skirt +est +##berry +bombs +respected +##horn +imposed +limestone +deny +nominee +memphis +grabbing +disabled +##als +amusement +aa +frankfurt +corn +referendum +varies +slowed +disk +firms +unconscious +incredible +clue +sue +##zhou +twist +##cio +joins +idaho +chad +developers +computing +destroyer +103 +mortal +tucker +kingston +choices +yu +carson +1800 +os +whitney +geneva +pretend +dimension +staged +plateau +maya +##une +freestyle +##bc +rovers +hiv +##ids +tristan +classroom +prospect +##hus +honestly +diploma +lied +thermal +auxiliary +feast +unlikely +iata +##tel +morocco +pounding +treasury +lithuania +considerably +1841 +dish +1812 +geological +matching +stumbled +destroying +marched +brien +advances +cake +nicole +belle +settling +measuring +directing +##mie +tuesday +bassist +capabilities +stunned +fraud +torpedo +##list +##phone +anton +wisdom +surveillance +ruined +##ulate +lawsuit +healthcare +theorem +halls +trend +aka +horizontal +dozens +acquire +lasting +swim +hawk +gorgeous +fees +vicinity +decrease +adoption +tactics +##ography +pakistani +##ole +draws +##hall +willie +burke +heath +algorithm +integral +powder +elliott +brigadier +jackie +tate +varieties +darker +##cho +lately +cigarette +specimens +adds +##ree +##ensis +##inger +exploded +finalist +cia +murders +wilderness +arguments +nicknamed +acceptance +onwards +manufacture +robertson +jets +tampa +enterprises +blog +loudly +composers +nominations +1838 +ai +malta +inquiry +automobile +hosting +viii +rays +tilted +grief +museums +strategies +furious +euro +equality +cohen +poison +surrey +wireless +governed +ridiculous +moses +##esh +##room +vanished +##ito +barnes +attract +morrison +istanbul +##iness +absent +rotation +petition +janet +##logical +satisfaction +custody +deliberately +observatory +comedian +surfaces +pinyin +novelist +strictly +canterbury +oslo +monks +embrace +ibm +jealous +photograph +continent +dorothy +marina +doc +excess +holden +allegations +explaining +stack +avoiding +lance +storyline +majesty +poorly +spike +dos +bradford +raven +travis +classics +proven +voltage +pillow +fists +butt +1842 +interpreted +##car +1839 +gage +telegraph +lens +promising +expelled +casual +collector +zones +##min +silly +nintendo +##kh +##bra +downstairs +chef +suspicious +afl +flies +vacant +uganda +pregnancy +condemned +lutheran +estimates +cheap +decree +saxon +proximity +stripped +idiot +deposits +contrary +presenter +magnus +glacier +im +offense +edwin +##ori +upright +##long +bolt +##ois +toss +geographical +##izes +environments +delicate +marking +abstract +xavier +nails +windsor +plantation +occurring +equity +saskatchewan +fears +drifted +sequences +vegetation +revolt +##stic +1843 +sooner +fusion +opposing +nato +skating +1836 +secretly +ruin +lease +##oc +edit +##nne +flora +anxiety +ruby +##ological +##mia +tel +bout +taxi +emmy +frost +rainbow +compounds +foundations +rainfall +assassination +nightmare +dominican +##win +achievements +deserve +orlando +intact +armenia +##nte +calgary +valentine +106 +marion +proclaimed +theodore +bells +courtyard +thigh +gonzalez +console +troop +minimal +monte +everyday +##ence +##if +supporter +terrorism +buck +openly +presbyterian +activists +carpet +##iers +rubbing +uprising +##yi +cute +conceived +legally +##cht +millennium +cello +velocity +ji +rescued +cardiff +1835 +rex +concentrate +senators +beard +rendered +glowing +battalions +scouts +competitors +sculptor +catalogue +arctic +ion +raja +bicycle +wow +glancing +lawn +##woman +gentleman +lighthouse +publish +predicted +calculated +##val +variants +##gne +strain +##ui +winston +deceased +##nus +touchdowns +brady +caleb +sinking +echoed +crush +hon +blessed +protagonist +hayes +endangered +magnitude +editors +##tine +estimate +responsibilities +##mel +backup +laying +consumed +sealed +zurich +lovers +frustrated +##eau +ahmed +kicking +mit +treasurer +1832 +biblical +refuse +terrified +pump +agrees +genuine +imprisonment +refuses +plymouth +##hen +lou +##nen +tara +trembling +antarctic +ton +learns +##tas +crap +crucial +faction +atop +##borough +wrap +lancaster +odds +hopkins +erik +lyon +##eon +bros +##ode +snap +locality +tips +empress +crowned +cal +acclaimed +chuckled +##ory +clara +sends +mild +towel +##fl +##day +##а +wishing +assuming +interviewed +##bal +##die +interactions +eden +cups +helena +##lf +indie +beck +##fire +batteries +filipino +wizard +parted +##lam +traces +##born +rows +idol +albany +delegates +##ees +##sar +discussions +##ex +notre +instructed +belgrade +highways +suggestion +lauren +possess +orientation +alexandria +abdul +beats +salary +reunion +ludwig +alright +wagner +intimate +pockets +slovenia +hugged +brighton +merchants +cruel +stole +trek +slopes +repairs +enrollment +politically +underlying +promotional +counting +boeing +##bb +isabella +naming +##и +keen +bacteria +listing +separately +belfast +ussr +450 +lithuanian +anybody +ribs +sphere +martinez +cock +embarrassed +proposals +fragments +nationals +##fs +##wski +premises +fin +1500 +alpine +matched +freely +bounded +jace +sleeve +##af +gaming +pier +populated +evident +##like +frances +flooded +##dle +frightened +pour +trainer +framed +visitor +challenging +pig +wickets +##fold +infected +email +##pes +arose +##aw +reward +ecuador +oblast +vale +ch +shuttle +##usa +bach +rankings +forbidden +cornwall +accordance +salem +consumers +bruno +fantastic +toes +machinery +resolved +julius +remembering +propaganda +iceland +bombardment +tide +contacts +wives +##rah +concerto +macdonald +albania +implement +daisy +tapped +sudan +helmet +angela +mistress +##lic +crop +sunk +finest +##craft +hostile +##ute +##tsu +boxer +fr +paths +adjusted +habit +ballot +supervision +soprano +##zen +bullets +wicked +sunset +regiments +disappear +lamp +performs +app +##gia +##oa +rabbit +digging +incidents +entries +##cion +dishes +##oi +introducing +##ati +##fied +freshman +slot +jill +tackles +baroque +backs +##iest +lone +sponsor +destiny +altogether +convert +##aro +consensus +shapes +demonstration +basically +feminist +auction +artifacts +##bing +strongest +twitter +halifax +2019 +allmusic +mighty +smallest +precise +alexandra +viola +##los +##ille +manuscripts +##illo +dancers +ari +managers +monuments +blades +barracks +springfield +maiden +consolidated +electron +##end +berry +airing +wheat +nobel +inclusion +blair +payments +geography +bee +cc +eleanor +react +##hurst +afc +manitoba +##yu +su +lineup +fitness +recreational +investments +airborne +disappointment +##dis +edmonton +viewing +##row +renovation +##cast +infant +bankruptcy +roses +aftermath +pavilion +##yer +carpenter +withdrawal +ladder +##hy +discussing +popped +reliable +agreements +rochester +##abad +curves +bombers +220 +rao +reverend +decreased +choosing +107 +stiff +consulting +naples +crawford +tracy +ka +ribbon +cops +##lee +crushed +deciding +unified +teenager +accepting +flagship +explorer +poles +sanchez +inspection +revived +skilled +induced +exchanged +flee +locals +tragedy +swallow +loading +hanna +demonstrate +##ela +salvador +flown +contestants +civilization +##ines +wanna +rhodes +fletcher +hector +knocking +considers +##ough +nash +mechanisms +sensed +mentally +walt +unclear +##eus +renovated +madame +##cks +crews +governmental +##hin +undertaken +monkey +##ben +##ato +fatal +armored +copa +caves +governance +grasp +perception +certification +froze +damp +tugged +wyoming +##rg +##ero +newman +##lor +nerves +curiosity +graph +115 +##ami +withdraw +tunnels +dull +meredith +moss +exhibits +neighbors +communicate +accuracy +explored +raiders +republicans +secular +kat +superman +penny +criticised +##tch +freed +update +conviction +wade +ham +likewise +delegation +gotta +doll +promises +technological +myth +nationality +resolve +convent +##mark +sharon +dig +sip +coordinator +entrepreneur +fold +##dine +capability +councillor +synonym +blown +swan +cursed +1815 +jonas +haired +sofa +canvas +keeper +rivalry +##hart +rapper +speedway +swords +postal +maxwell +estonia +potter +recurring +##nn +##ave +errors +##oni +cognitive +1834 +##² +claws +nadu +roberto +bce +wrestler +ellie +##ations +infinite +ink +##tia +presumably +finite +staircase +108 +noel +patricia +nacional +##cation +chill +eternal +tu +preventing +prussia +fossil +limbs +##logist +ernst +frog +perez +rene +##ace +pizza +prussian +##ios +##vy +molecules +regulatory +answering +opinions +sworn +lengths +supposedly +hypothesis +upward +habitats +seating +ancestors +drank +yield +hd +synthesis +researcher +modest +##var +mothers +peered +voluntary +homeland +##the +acclaim +##igan +static +valve +luxembourg +alto +carroll +fe +receptor +norton +ambulance +##tian +johnston +catholics +depicting +jointly +elephant +gloria +mentor +badge +ahmad +distinguish +remarked +councils +precisely +allison +advancing +detection +crowded +##10 +cooperative +ankle +mercedes +dagger +surrendered +pollution +commit +subway +jeffrey +lesson +sculptures +provider +##fication +membrane +timothy +rectangular +fiscal +heating +teammate +basket +particle +anonymous +deployment +##ple +missiles +courthouse +proportion +shoe +sec +##ller +complaints +forbes +blacks +abandon +remind +sizes +overwhelming +autobiography +natalie +##awa +risks +contestant +countryside +babies +scorer +invaded +enclosed +proceed +hurling +disorders +##cu +reflecting +continuously +cruiser +graduates +freeway +investigated +ore +deserved +maid +blocking +phillip +jorge +shakes +dove +mann +variables +lacked +burden +accompanying +que +consistently +organizing +provisional +complained +endless +##rm +tubes +juice +georges +krishna +mick +labels +thriller +##uch +laps +arcade +sage +snail +##table +shannon +fi +laurence +seoul +vacation +presenting +hire +churchill +surprisingly +prohibited +savannah +technically +##oli +170 +##lessly +testimony +suited +speeds +toys +romans +mlb +flowering +measurement +talented +kay +settings +charleston +expectations +shattered +achieving +triumph +ceremonies +portsmouth +lanes +mandatory +loser +stretching +cologne +realizes +seventy +cornell +careers +webb +##ulating +americas +budapest +ava +suspicion +##ison +yo +conrad +##hai +sterling +jessie +rector +##az +1831 +transform +organize +loans +christine +volcanic +warrant +slender +summers +subfamily +newer +danced +dynamics +rhine +proceeds +heinrich +gastropod +commands +sings +facilitate +easter +ra +positioned +responses +expense +fruits +yanked +imported +25th +velvet +vic +primitive +tribune +baldwin +neighbourhood +donna +rip +hay +pr +##uro +1814 +espn +welcomed +##aria +qualifier +glare +highland +timing +##cted +shells +eased +geometry +louder +exciting +slovakia +##sion +##iz +##lot +savings +prairie +##ques +marching +rafael +tonnes +##lled +curtain +preceding +shy +heal +greene +worthy +##pot +detachment +bury +sherman +##eck +reinforced +seeks +bottles +contracted +duchess +outfit +walsh +##sc +mickey +##ase +geoffrey +archer +squeeze +dawson +eliminate +invention +##enberg +neal +##eth +stance +dealer +coral +maple +retire +polo +simplified +##ht +1833 +hid +watts +backwards +jules +##oke +genesis +mt +frames +rebounds +burma +woodland +moist +santos +whispers +drained +subspecies +##aa +streaming +ulster +burnt +correspondence +maternal +gerard +denis +stealing +##load +genius +duchy +##oria +inaugurated +momentum +suits +placement +sovereign +clause +thames +##hara +confederation +reservation +sketch +yankees +lets +rotten +charm +hal +verses +ultra +commercially +dot +salon +citation +adopt +winnipeg +mist +allocated +cairo +##boy +jenkins +interference +objectives +##wind +1820 +portfolio +armoured +sectors +##eh +initiatives +##world +integrity +exercises +robe +tap +ab +gazed +##tones +distracted +rulers +111 +favorable +jerome +tended +cart +factories +##eri +diplomat +valued +gravel +charitable +##try +calvin +exploring +chang +shepherd +terrace +pdf +pupil +##ural +reflects +ups +##rch +governors +shelf +depths +##nberg +trailed +crest +tackle +##nian +##ats +hatred +##kai +clare +makers +ethiopia +longtime +detected +embedded +lacking +slapped +rely +thomson +anticipation +iso +morton +successive +agnes +screenwriter +straightened +philippe +playwright +haunted +licence +iris +intentions +sutton +112 +logical +correctly +##weight +branded +licked +tipped +silva +ricky +narrator +requests +##ents +greeted +supernatural +cow +##wald +lung +refusing +employer +strait +gaelic +liner +##piece +zoe +sabha +##mba +driveway +harvest +prints +bates +reluctantly +threshold +algebra +ira +wherever +coupled +240 +assumption +picks +##air +designers +raids +gentlemen +##ean +roller +blowing +leipzig +locks +screw +dressing +strand +##lings +scar +dwarf +depicts +##nu +nods +##mine +differ +boris +##eur +yuan +flip +##gie +mob +invested +questioning +applying +##ture +shout +##sel +gameplay +blamed +illustrations +bothered +weakness +rehabilitation +##of +##zes +envelope +rumors +miners +leicester +subtle +kerry +##ico +ferguson +##fu +premiership +ne +##cat +bengali +prof +catches +remnants +dana +##rily +shouting +presidents +baltic +ought +ghosts +dances +sailors +shirley +fancy +dominic +##bie +madonna +##rick +bark +buttons +gymnasium +ashes +liver +toby +oath +providence +doyle +evangelical +nixon +cement +carnegie +embarked +hatch +surroundings +guarantee +needing +pirate +essence +##bee +filter +crane +hammond +projected +immune +percy +twelfth +##ult +regent +doctoral +damon +mikhail +##ichi +lu +critically +elect +realised +abortion +acute +screening +mythology +steadily +##fc +frown +nottingham +kirk +wa +minneapolis +##rra +module +algeria +mc +nautical +encounters +surprising +statues +availability +shirts +pie +alma +brows +munster +mack +soup +crater +tornado +sanskrit +cedar +explosive +bordered +dixon +planets +stamp +exam +happily +##bble +carriers +kidnapped +##vis +accommodation +emigrated +##met +knockout +correspondent +violation +profits +peaks +lang +specimen +agenda +ancestry +pottery +spelling +equations +obtaining +ki +linking +1825 +debris +asylum +##20 +buddhism +teddy +##ants +gazette +##nger +##sse +dental +eligibility +utc +fathers +averaged +zimbabwe +francesco +coloured +hissed +translator +lynch +mandate +humanities +mackenzie +uniforms +lin +##iana +##gio +asset +mhz +fitting +samantha +genera +wei +rim +beloved +shark +riot +entities +expressions +indo +carmen +slipping +owing +abbot +neighbor +sidney +##av +rats +recommendations +encouraging +squadrons +anticipated +commanders +conquered +##oto +donations +diagnosed +##mond +divide +##iva +guessed +decoration +vernon +auditorium +revelation +conversations +##kers +##power +herzegovina +dash +alike +protested +lateral +herman +accredited +mg +##gent +freeman +mel +fiji +crow +crimson +##rine +livestock +##pped +humanitarian +bored +oz +whip +##lene +##ali +legitimate +alter +grinning +spelled +anxious +oriental +wesley +##nin +##hole +carnival +controller +detect +##ssa +bowed +educator +kosovo +macedonia +##sin +occupy +mastering +stephanie +janeiro +para +unaware +nurses +noon +135 +cam +hopefully +ranger +combine +sociology +polar +rica +##eer +neill +##sman +holocaust +##ip +doubled +lust +1828 +109 +decent +cooling +unveiled +##card +1829 +nsw +homer +chapman +meyer +##gin +dive +mae +reagan +expertise +##gled +darwin +brooke +sided +prosecution +investigating +comprised +petroleum +genres +reluctant +differently +trilogy +johns +vegetables +corpse +highlighted +lounge +pension +unsuccessfully +elegant +aided +ivory +beatles +amelia +cain +dubai +sunny +immigrant +babe +click +##nder +underwater +pepper +combining +mumbled +atlas +horns +accessed +ballad +physicians +homeless +gestured +rpm +freak +louisville +corporations +patriots +prizes +rational +warn +modes +decorative +overnight +din +troubled +phantom +##ort +monarch +sheer +##dorf +generals +guidelines +organs +addresses +##zon +enhance +curling +parishes +cord +##kie +linux +caesar +deutsche +bavaria +##bia +coleman +cyclone +##eria +bacon +petty +##yama +##old +hampton +diagnosis +1824 +throws +complexity +rita +disputed +##₃ +pablo +##sch +marketed +trafficking +##ulus +examine +plague +formats +##oh +vault +faithful +##bourne +webster +##ox +highlights +##ient +##ann +phones +vacuum +sandwich +modeling +##gated +bolivia +clergy +qualities +isabel +##nas +##ars +wears +screams +reunited +annoyed +bra +##ancy +##rate +differential +transmitter +tattoo +container +poker +##och +excessive +resides +cowboys +##tum +augustus +trash +providers +statute +retreated +balcony +reversed +void +storey +preceded +masses +leap +laughs +neighborhoods +wards +schemes +falcon +santo +battlefield +pad +ronnie +thread +lesbian +venus +##dian +beg +sandstone +daylight +punched +gwen +analog +stroked +wwe +acceptable +measurements +dec +toxic +##kel +adequate +surgical +economist +parameters +varsity +##sberg +quantity +ella +##chy +##rton +countess +generating +precision +diamonds +expressway +ga +##ı +1821 +uruguay +talents +galleries +expenses +scanned +colleague +outlets +ryder +lucien +##ila +paramount +##bon +syracuse +dim +fangs +gown +sweep +##sie +toyota +missionaries +websites +##nsis +sentences +adviser +val +trademark +spells +##plane +patience +starter +slim +##borg +toe +incredibly +shoots +elliot +nobility +##wyn +cowboy +endorsed +gardner +tendency +persuaded +organisms +emissions +kazakhstan +amused +boring +chips +themed +##hand +llc +constantinople +chasing +systematic +guatemala +borrowed +erin +carey +##hard +highlands +struggles +1810 +##ifying +##ced +wong +exceptions +develops +enlarged +kindergarten +castro +##ern +##rina +leigh +zombie +juvenile +##most +consul +##nar +sailor +hyde +clarence +intensive +pinned +nasty +useless +jung +clayton +stuffed +exceptional +ix +apostolic +230 +transactions +##dge +exempt +swinging +cove +religions +##ash +shields +dairy +bypass +190 +pursuing +bug +joyce +bombay +chassis +southampton +chat +interact +redesignated +##pen +nascar +pray +salmon +rigid +regained +malaysian +grim +publicity +constituted +capturing +toilet +delegate +purely +tray +drift +loosely +striker +weakened +trinidad +mitch +itv +defines +transmitted +ming +scarlet +nodding +fitzgerald +fu +narrowly +sp +tooth +standings +virtue +##₁ +##wara +##cting +chateau +gloves +lid +##nel +hurting +conservatory +##pel +sinclair +reopened +sympathy +nigerian +strode +advocated +optional +chronic +discharge +##rc +suck +compatible +laurel +stella +shi +fails +wage +dodge +128 +informal +sorts +levi +buddha +villagers +##aka +chronicles +heavier +summoned +gateway +3000 +eleventh +jewelry +translations +accordingly +seas +##ency +fiber +pyramid +cubic +dragging +##ista +caring +##ops +android +contacted +lunar +##dt +kai +lisbon +patted +1826 +sacramento +theft +madagascar +subtropical +disputes +ta +holidays +piper +willow +mare +cane +itunes +newfoundland +benny +companions +dong +raj +observe +roar +charming +plaque +tibetan +fossils +enacted +manning +bubble +tina +tanzania +##eda +##hir +funk +swamp +deputies +cloak +ufc +scenario +par +scratch +metals +anthem +guru +engaging +specially +##boat +dialects +nineteen +cecil +duet +disability +messenger +unofficial +##lies +defunct +eds +moonlight +drainage +surname +puzzle +honda +switching +conservatives +mammals +knox +broadcaster +sidewalk +cope +##ried +benson +princes +peterson +##sal +bedford +sharks +eli +wreck +alberto +gasp +archaeology +lgbt +teaches +securities +madness +compromise +waving +coordination +davidson +visions +leased +possibilities +eighty +jun +fernandez +enthusiasm +assassin +sponsorship +reviewer +kingdoms +estonian +laboratories +##fy +##nal +applies +verb +celebrations +##zzo +rowing +lightweight +sadness +submit +mvp +balanced +dude +##vas +explicitly +metric +magnificent +mound +brett +mohammad +mistakes +irregular +##hing +##ass +sanders +betrayed +shipped +surge +##enburg +reporters +termed +georg +pity +verbal +bulls +abbreviated +enabling +appealed +##are +##atic +sicily +sting +heel +sweetheart +bart +spacecraft +brutal +monarchy +##tter +aberdeen +cameo +diane +##ub +survivor +clyde +##aries +complaint +##makers +clarinet +delicious +chilean +karnataka +coordinates +1818 +panties +##rst +pretending +ar +dramatically +kiev +bella +tends +distances +113 +catalog +launching +instances +telecommunications +portable +lindsay +vatican +##eim +angles +aliens +marker +stint +screens +bolton +##rne +judy +wool +benedict +plasma +europa +spark +imaging +filmmaker +swiftly +##een +contributor +##nor +opted +stamps +apologize +financing +butter +gideon +sophisticated +alignment +avery +chemicals +yearly +speculation +prominence +professionally +##ils +immortal +institutional +inception +wrists +identifying +tribunal +derives +gains +##wo +papal +preference +linguistic +vince +operative +brewery +##ont +unemployment +boyd +##ured +##outs +albeit +prophet +1813 +bi +##rr +##face +##rad +quarterly +asteroid +cleaned +radius +temper +##llen +telugu +jerk +viscount +menu +##ote +glimpse +##aya +yacht +hawaiian +baden +##rl +laptop +readily +##gu +monetary +offshore +scots +watches +##yang +##arian +upgrade +needle +xbox +lea +encyclopedia +flank +fingertips +##pus +delight +teachings +confirm +roth +beaches +midway +winters +##iah +teasing +daytime +beverly +gambling +bonnie +##backs +regulated +clement +hermann +tricks +knot +##shing +##uring +##vre +detached +ecological +owed +specialty +byron +inventor +bats +stays +screened +unesco +midland +trim +affection +##ander +##rry +jess +thoroughly +feedback +##uma +chennai +strained +heartbeat +wrapping +overtime +pleaded +##sworth +mon +leisure +oclc +##tate +##ele +feathers +angelo +thirds +nuts +surveys +clever +gill +commentator +##dos +darren +rides +gibraltar +##nc +##mu +dissolution +dedication +shin +meals +saddle +elvis +reds +chaired +taller +appreciation +functioning +niece +favored +advocacy +robbie +criminals +suffolk +yugoslav +passport +constable +congressman +hastings +vera +##rov +consecrated +sparks +ecclesiastical +confined +##ovich +muller +floyd +nora +1822 +paved +1827 +cumberland +ned +saga +spiral +##flow +appreciated +yi +collaborative +treating +similarities +feminine +finishes +##ib +jade +import +##nse +##hot +champagne +mice +securing +celebrities +helsinki +attributes +##gos +cousins +phases +ache +lucia +gandhi +submission +vicar +spear +shine +tasmania +biting +detention +constitute +tighter +seasonal +##gus +terrestrial +matthews +##oka +effectiveness +parody +philharmonic +##onic +1816 +strangers +encoded +consortium +guaranteed +regards +shifts +tortured +collision +supervisor +inform +broader +insight +theaters +armour +emeritus +blink +incorporates +mapping +##50 +##ein +handball +flexible +##nta +substantially +generous +thief +##own +carr +loses +1793 +prose +ucla +romeo +generic +metallic +realization +damages +mk +commissioners +zach +default +##ther +helicopters +lengthy +stems +spa +partnered +spectators +rogue +indication +penalties +teresa +1801 +sen +##tric +dalton +##wich +irving +photographic +##vey +dell +deaf +peters +excluded +unsure +##vable +patterson +crawled +##zio +resided +whipped +latvia +slower +ecole +pipes +employers +maharashtra +comparable +va +textile +pageant +##gel +alphabet +binary +irrigation +chartered +choked +antoine +offs +waking +supplement +##wen +quantities +demolition +regain +locate +urdu +folks +alt +114 +##mc +scary +andreas +whites +##ava +classrooms +mw +aesthetic +publishes +valleys +guides +cubs +johannes +bryant +conventions +affecting +##itt +drain +awesome +isolation +prosecutor +ambitious +apology +captive +downs +atmospheric +lorenzo +aisle +beef +foul +##onia +kidding +composite +disturbed +illusion +natives +##ffer +emi +rockets +riverside +wartime +painters +adolf +melted +##ail +uncertainty +simulation +hawks +progressed +meantime +builder +spray +breach +unhappy +regina +russians +##urg +determining +##tation +tram +1806 +##quin +aging +##12 +1823 +garion +rented +mister +diaz +terminated +clip +1817 +depend +nervously +disco +owe +defenders +shiva +notorious +disbelief +shiny +worcester +##gation +##yr +trailing +undertook +islander +belarus +limitations +watershed +fuller +overlooking +utilized +raphael +1819 +synthetic +breakdown +klein +##nate +moaned +memoir +lamb +practicing +##erly +cellular +arrows +exotic +##graphy +witches +117 +charted +rey +hut +hierarchy +subdivision +freshwater +giuseppe +aloud +reyes +qatar +marty +sideways +utterly +sexually +jude +prayers +mccarthy +softball +blend +damien +##gging +##metric +wholly +erupted +lebanese +negro +revenues +tasted +comparative +teamed +transaction +labeled +maori +sovereignty +parkway +trauma +gran +malay +121 +advancement +descendant +2020 +buzz +salvation +inventory +symbolic +##making +antarctica +mps +##gas +##bro +mohammed +myanmar +holt +submarines +tones +##lman +locker +patriarch +bangkok +emerson +remarks +predators +kin +afghan +confession +norwich +rental +emerge +advantages +##zel +rca +##hold +shortened +storms +aidan +##matic +autonomy +compliance +##quet +dudley +atp +##osis +1803 +motto +documentation +summary +professors +spectacular +christina +archdiocese +flashing +innocence +remake +##dell +psychic +reef +scare +employ +rs +sticks +meg +gus +leans +##ude +accompany +bergen +tomas +##iko +doom +wages +pools +##nch +##bes +breasts +scholarly +alison +outline +brittany +breakthrough +willis +realistic +##cut +##boro +competitor +##stan +pike +picnic +icon +designing +commercials +washing +villain +skiing +micro +costumes +auburn +halted +executives +##hat +logistics +cycles +vowel +applicable +barrett +exclaimed +eurovision +eternity +ramon +##umi +##lls +modifications +sweeping +disgust +##uck +torch +aviv +ensuring +rude +dusty +sonic +donovan +outskirts +cu +pathway +##band +##gun +##lines +disciplines +acids +cadet +paired +##40 +sketches +##sive +marriages +##⁺ +folding +peers +slovak +implies +admired +##beck +1880s +leopold +instinct +attained +weston +megan +horace +##ination +dorsal +ingredients +evolutionary +##its +complications +deity +lethal +brushing +levy +deserted +institutes +posthumously +delivering +telescope +coronation +motivated +rapids +luc +flicked +pays +volcano +tanner +weighed +##nica +crowds +frankie +gifted +addressing +granddaughter +winding +##rna +constantine +gomez +##front +landscapes +rudolf +anthropology +slate +werewolf +##lio +astronomy +circa +rouge +dreaming +sack +knelt +drowned +naomi +prolific +tracked +freezing +herb +##dium +agony +randall +twisting +wendy +deposit +touches +vein +wheeler +##bbled +##bor +batted +retaining +tire +presently +compare +specification +daemon +nigel +##grave +merry +recommendation +czechoslovakia +sandra +ng +roma +##sts +lambert +inheritance +sheikh +winchester +cries +examining +##yle +comeback +cuisine +nave +##iv +ko +retrieve +tomatoes +barker +polished +defining +irene +lantern +personalities +begging +tract +swore +1809 +175 +##gic +omaha +brotherhood +##rley +haiti +##ots +exeter +##ete +##zia +steele +dumb +pearson +210 +surveyed +elisabeth +trends +##ef +fritz +##rf +premium +bugs +fraction +calmly +viking +##birds +tug +inserted +unusually +##ield +confronted +distress +crashing +brent +turks +resign +##olo +cambodia +gabe +sauce +##kal +evelyn +116 +extant +clusters +quarry +teenagers +luna +##lers +##ister +affiliation +drill +##ashi +panthers +scenic +libya +anita +strengthen +inscriptions +##cated +lace +sued +judith +riots +##uted +mint +##eta +preparations +midst +dub +challenger +##vich +mock +cf +displaced +wicket +breaths +enables +schmidt +analyst +##lum +ag +highlight +automotive +axe +josef +newark +sufficiently +resembles +50th +##pal +flushed +mum +traits +##ante +commodore +incomplete +warming +titular +ceremonial +ethical +118 +celebrating +eighteenth +cao +lima +medalist +mobility +strips +snakes +##city +miniature +zagreb +barton +escapes +umbrella +automated +doubted +differs +cooled +georgetown +dresden +cooked +fade +wyatt +rna +jacobs +carlton +abundant +stereo +boost +madras +inning +##hia +spur +ip +malayalam +begged +osaka +groan +escaping +charging +dose +vista +##aj +bud +papa +communists +advocates +edged +tri +##cent +resemble +peaking +necklace +fried +montenegro +saxony +goose +glances +stuttgart +curator +recruit +grocery +sympathetic +##tting +##fort +127 +lotus +randolph +ancestor +##rand +succeeding +jupiter +1798 +macedonian +##heads +hiking +1808 +handing +fischer +##itive +garbage +node +##pies +prone +singular +papua +inclined +attractions +italia +pouring +motioned +grandma +garnered +jacksonville +corp +ego +ringing +aluminum +##hausen +ordering +##foot +drawer +traders +synagogue +##play +##kawa +resistant +wandering +fragile +fiona +teased +var +hardcore +soaked +jubilee +decisive +exposition +mercer +poster +valencia +hale +kuwait +1811 +##ises +##wr +##eed +tavern +gamma +122 +johan +##uer +airways +amino +gil +##ury +vocational +domains +torres +##sp +generator +folklore +outcomes +##keeper +canberra +shooter +fl +beams +confrontation +##lling +##gram +feb +aligned +forestry +pipeline +jax +motorway +conception +decay +##tos +coffin +##cott +stalin +1805 +escorted +minded +##nam +sitcom +purchasing +twilight +veronica +additions +passive +tensions +straw +123 +frequencies +1804 +refugee +cultivation +##iate +christie +clary +bulletin +crept +disposal +##rich +##zong +processor +crescent +##rol +bmw +emphasized +whale +nazis +aurora +##eng +dwelling +hauled +sponsors +toledo +mega +ideology +theatres +tessa +cerambycidae +saves +turtle +cone +suspects +kara +rusty +yelling +greeks +mozart +shades +cocked +participant +##tro +shire +spit +freeze +necessity +##cos +inmates +nielsen +councillors +loaned +uncommon +omar +peasants +botanical +offspring +daniels +formations +jokes +1794 +pioneers +sigma +licensing +##sus +wheelchair +polite +1807 +liquor +pratt +trustee +##uta +forewings +balloon +##zz +kilometre +camping +explicit +casually +shawn +foolish +teammates +nm +hassan +carrie +judged +satisfy +vanessa +knives +selective +cnn +flowed +##lice +eclipse +stressed +eliza +mathematician +cease +cultivated +##roy +commissions +browns +##ania +destroyers +sheridan +meadow +##rius +minerals +##cial +downstream +clash +gram +memoirs +ventures +baha +seymour +archie +midlands +edith +fare +flynn +invite +canceled +tiles +stabbed +boulder +incorporate +amended +camden +facial +mollusk +unreleased +descriptions +yoga +grabs +550 +raises +ramp +shiver +##rose +coined +pioneering +tunes +qing +warwick +tops +119 +melanie +giles +##rous +wandered +##inal +annexed +nov +30th +unnamed +##ished +organizational +airplane +normandy +stoke +whistle +blessing +violations +chased +holders +shotgun +##ctic +outlet +reactor +##vik +tires +tearing +shores +fortified +mascot +constituencies +nc +columnist +productive +tibet +##rta +lineage +hooked +oct +tapes +judging +cody +##gger +hansen +kashmir +triggered +##eva +solved +cliffs +##tree +resisted +anatomy +protesters +transparent +implied +##iga +injection +mattress +excluding +##mbo +defenses +helpless +devotion +##elli +growl +liberals +weber +phenomena +atoms +plug +##iff +mortality +apprentice +howe +convincing +aaa +swimmer +barber +leone +promptly +sodium +def +nowadays +arise +##oning +gloucester +corrected +dignity +norm +erie +##ders +elders +evacuated +sylvia +compression +##yar +hartford +pose +backpack +reasoning +accepts +24th +wipe +millimetres +marcel +##oda +dodgers +albion +1790 +overwhelmed +aerospace +oaks +1795 +showcase +acknowledge +recovering +nolan +ashe +hurts +geology +fashioned +disappearance +farewell +swollen +shrug +marquis +wimbledon +124 +rue +1792 +commemorate +reduces +experiencing +inevitable +calcutta +intel +##court +murderer +sticking +fisheries +imagery +bloom +280 +brake +##inus +gustav +hesitation +memorable +po +viral +beans +accidents +tunisia +antenna +spilled +consort +treatments +aye +perimeter +##gard +donation +hostage +migrated +banker +addiction +apex +lil +trout +##ously +conscience +##nova +rams +sands +genome +passionate +troubles +##lets +##set +amid +##ibility +##ret +higgins +exceed +vikings +##vie +payne +##zan +muscular +##ste +defendant +sucking +##wal +ibrahim +fuselage +claudia +vfl +europeans +snails +interval +##garh +preparatory +statewide +tasked +lacrosse +viktor +##lation +angola +##hra +flint +implications +employs +teens +patrons +stall +weekends +barriers +scrambled +nucleus +tehran +jenna +parsons +lifelong +robots +displacement +5000 +##bles +precipitation +##gt +knuckles +clutched +1802 +marrying +ecology +marx +accusations +declare +scars +kolkata +mat +meadows +bermuda +skeleton +finalists +vintage +crawl +coordinate +affects +subjected +orchestral +mistaken +##tc +mirrors +dipped +relied +260 +arches +candle +##nick +incorporating +wildly +fond +basilica +owl +fringe +rituals +whispering +stirred +feud +tertiary +slick +goat +honorable +whereby +skip +ricardo +stripes +parachute +adjoining +submerged +synthesizer +##gren +intend +positively +ninety +phi +beaver +partition +fellows +alexis +prohibition +carlisle +bizarre +fraternity +##bre +doubts +icy +cbc +aquatic +sneak +sonny +combines +airports +crude +supervised +spatial +merge +alfonso +##bic +corrupt +scan +undergo +##ams +disabilities +colombian +comparing +dolphins +perkins +##lish +reprinted +unanimous +bounced +hairs +underworld +midwest +semester +bucket +paperback +miniseries +coventry +demise +##leigh +demonstrations +sensor +rotating +yan +##hler +arrange +soils +##idge +hyderabad +labs +##dr +brakes +grandchildren +##nde +negotiated +rover +ferrari +continuation +directorate +augusta +stevenson +counterpart +gore +##rda +nursery +rican +ave +collectively +broadly +pastoral +repertoire +asserted +discovering +nordic +styled +fiba +cunningham +harley +middlesex +survives +tumor +tempo +zack +aiming +lok +urgent +##rade +##nto +devils +##ement +contractor +turin +##wl +##ool +bliss +repaired +simmons +moan +astronomical +cr +negotiate +lyric +1890s +lara +bred +clad +angus +pbs +##ience +engineered +posed +##lk +hernandez +possessions +elbows +psychiatric +strokes +confluence +electorate +lifts +campuses +lava +alps +##ep +##ution +##date +physicist +woody +##page +##ographic +##itis +juliet +reformation +sparhawk +320 +complement +suppressed +jewel +##½ +floated +##kas +continuity +sadly +##ische +inability +melting +scanning +paula +flour +judaism +safer +vague +##lm +solving +curb +##stown +financially +gable +bees +expired +miserable +cassidy +dominion +1789 +cupped +145 +robbery +facto +amos +warden +resume +tallest +marvin +ing +pounded +usd +declaring +gasoline +##aux +darkened +270 +650 +sophomore +##mere +erection +gossip +televised +risen +dial +##eu +pillars +##link +passages +profound +##tina +arabian +ashton +silicon +nail +##ead +##lated +##wer +##hardt +fleming +firearms +ducked +circuits +blows +waterloo +titans +##lina +atom +fireplace +cheshire +financed +activation +algorithms +##zzi +constituent +catcher +cherokee +partnerships +sexuality +platoon +tragic +vivian +guarded +whiskey +meditation +poetic +##late +##nga +##ake +porto +listeners +dominance +kendra +mona +chandler +factions +22nd +salisbury +attitudes +derivative +##ido +##haus +intake +paced +javier +illustrator +barrels +bias +cockpit +burnett +dreamed +ensuing +##anda +receptors +someday +hawkins +mattered +##lal +slavic +1799 +jesuit +cameroon +wasted +tai +wax +lowering +victorious +freaking +outright +hancock +librarian +sensing +bald +calcium +myers +tablet +announcing +barack +shipyard +pharmaceutical +##uan +greenwich +flush +medley +patches +wolfgang +pt +speeches +acquiring +exams +nikolai +##gg +hayden +kannada +##type +reilly +##pt +waitress +abdomen +devastated +capped +pseudonym +pharmacy +fulfill +paraguay +1796 +clicked +##trom +archipelago +syndicated +##hman +lumber +orgasm +rejection +clifford +lorraine +advent +mafia +rodney +brock +##ght +##used +##elia +cassette +chamberlain +despair +mongolia +sensors +developmental +upstream +##eg +##alis +spanning +165 +trombone +basque +seeded +interred +renewable +rhys +leapt +revision +molecule +##ages +chord +vicious +nord +shivered +23rd +arlington +debts +corpus +sunrise +bays +blackburn +centimetres +##uded +shuddered +gm +strangely +gripping +cartoons +isabelle +orbital +##ppa +seals +proving +##lton +refusal +strengthened +bust +assisting +baghdad +batsman +portrayal +mara +pushes +spears +og +##cock +reside +nathaniel +brennan +1776 +confirmation +caucus +##worthy +markings +yemen +nobles +ku +lazy +viewer +catalan +encompasses +sawyer +##fall +sparked +substances +patents +braves +arranger +evacuation +sergio +persuade +dover +tolerance +penguin +cum +jockey +insufficient +townships +occupying +declining +plural +processed +projection +puppet +flanders +introduces +liability +##yon +gymnastics +antwerp +taipei +hobart +candles +jeep +wes +observers +126 +chaplain +bundle +glorious +##hine +hazel +flung +sol +excavations +dumped +stares +sh +bangalore +triangular +icelandic +intervals +expressing +turbine +##vers +songwriting +crafts +##igo +jasmine +ditch +rite +##ways +entertaining +comply +sorrow +wrestlers +basel +emirates +marian +rivera +helpful +##some +caution +downward +networking +##atory +##tered +darted +genocide +emergence +replies +specializing +spokesman +convenient +unlocked +fading +augustine +concentrations +resemblance +elijah +investigator +andhra +##uda +promotes +bean +##rrell +fleeing +wan +simone +announcer +##ame +##bby +lydia +weaver +132 +residency +modification +##fest +stretches +##ast +alternatively +nat +lowe +lacks +##ented +pam +tile +concealed +inferior +abdullah +residences +tissues +vengeance +##ided +moisture +peculiar +groove +zip +bologna +jennings +ninja +oversaw +zombies +pumping +batch +livingston +emerald +installations +1797 +peel +nitrogen +rama +##fying +##star +schooling +strands +responding +werner +##ost +lime +casa +accurately +targeting +##rod +underway +##uru +hemisphere +lester +##yard +occupies +2d +griffith +angrily +reorganized +##owing +courtney +deposited +##dd +##30 +estadio +##ifies +dunn +exiled +##ying +checks +##combe +##о +##fly +successes +unexpectedly +blu +assessed +##flower +##ه +observing +sacked +spiders +kn +##tail +mu +nodes +prosperity +audrey +divisional +155 +broncos +tangled +adjust +feeds +erosion +paolo +surf +directory +snatched +humid +admiralty +screwed +gt +reddish +##nese +modules +trench +lamps +bind +leah +bucks +competes +##nz +##form +transcription +##uc +isles +violently +clutching +pga +cyclist +inflation +flats +ragged +unnecessary +##hian +stubborn +coordinated +harriet +baba +disqualified +330 +insect +wolfe +##fies +reinforcements +rocked +duel +winked +embraced +bricks +##raj +hiatus +defeats +pending +brightly +jealousy +##xton +##hm +##uki +lena +gdp +colorful +##dley +stein +kidney +##shu +underwear +wanderers +##haw +##icus +guardians +m³ +roared +habits +##wise +permits +gp +uranium +punished +disguise +bundesliga +elise +dundee +erotic +partisan +pi +collectors +float +individually +rendering +behavioral +bucharest +ser +hare +valerie +corporal +nutrition +proportional +##isa +immense +##kis +pavement +##zie +##eld +sutherland +crouched +1775 +##lp +suzuki +trades +endurance +operas +crosby +prayed +priory +rory +socially +##urn +gujarat +##pu +walton +cube +pasha +privilege +lennon +floods +thorne +waterfall +nipple +scouting +approve +##lov +minorities +voter +dwight +extensions +assure +ballroom +slap +dripping +privileges +rejoined +confessed +demonstrating +patriotic +yell +investor +##uth +pagan +slumped +squares +##cle +##kins +confront +bert +embarrassment +##aid +aston +urging +sweater +starr +yuri +brains +williamson +commuter +mortar +structured +selfish +exports +##jon +cds +##him +unfinished +##rre +mortgage +destinations +##nagar +canoe +solitary +buchanan +delays +magistrate +fk +##pling +motivation +##lier +##vier +recruiting +assess +##mouth +malik +antique +1791 +pius +rahman +reich +tub +zhou +smashed +airs +galway +xii +conditioning +honduras +discharged +dexter +##pf +lionel +129 +debates +lemon +tiffany +volunteered +dom +dioxide +procession +devi +sic +tremendous +advertisements +colts +transferring +verdict +hanover +decommissioned +utter +relate +pac +racism +##top +beacon +limp +similarity +terra +occurrence +ant +##how +becky +capt +updates +armament +richie +pal +##graph +halloween +mayo +##ssen +##bone +cara +serena +fcc +dolls +obligations +##dling +violated +lafayette +jakarta +exploitation +##ime +infamous +iconic +##lah +##park +kitty +moody +reginald +dread +spill +crystals +olivier +modeled +bluff +equilibrium +separating +notices +ordnance +extinction +onset +cosmic +attachment +sammy +expose +privy +anchored +##bil +abbott +admits +bending +baritone +emmanuel +policeman +vaughan +winged +climax +dresses +denny +polytechnic +mohamed +burmese +authentic +nikki +genetics +grandparents +homestead +gaza +postponed +metacritic +una +##sby +##bat +unstable +dissertation +##rial +##cian +curls +obscure +uncovered +bronx +praying +disappearing +##hoe +prehistoric +coke +turret +mutations +nonprofit +pits +monaco +##ي +##usion +prominently +dispatched +podium +##mir +uci +##uation +133 +fortifications +birthplace +kendall +##lby +##oll +preacher +rack +goodman +##rman +persistent +##ott +countless +jaime +recorder +lexington +persecution +jumps +renewal +wagons +##11 +crushing +##holder +decorations +##lake +abundance +wrath +laundry +£1 +garde +##rp +jeanne +beetles +peasant +##sl +splitting +caste +sergei +##rer +##ema +scripts +##ively +rub +satellites +##vor +inscribed +verlag +scrapped +gale +packages +chick +potato +slogan +kathleen +arabs +##culture +counterparts +reminiscent +choral +##tead +rand +retains +bushes +dane +accomplish +courtesy +closes +##oth +slaughter +hague +krakow +lawson +tailed +elias +ginger +##ttes +canopy +betrayal +rebuilding +turf +##hof +frowning +allegiance +brigades +kicks +rebuild +polls +alias +nationalism +td +rowan +audition +bowie +fortunately +recognizes +harp +dillon +horrified +##oro +renault +##tics +ropes +##α +presumed +rewarded +infrared +wiping +accelerated +illustration +##rid +presses +practitioners +badminton +##iard +detained +##tera +recognizing +relates +misery +##sies +##tly +reproduction +piercing +potatoes +thornton +esther +manners +hbo +##aan +ours +bullshit +ernie +perennial +sensitivity +illuminated +rupert +##jin +##iss +##ear +rfc +nassau +##dock +staggered +socialism +##haven +appointments +nonsense +prestige +sharma +haul +##tical +solidarity +gps +##ook +##rata +igor +pedestrian +##uit +baxter +tenants +wires +medication +unlimited +guiding +impacts +diabetes +##rama +sasha +pas +clive +extraction +131 +continually +constraints +##bilities +sonata +hunted +sixteenth +chu +planting +quote +mayer +pretended +abs +spat +##hua +ceramic +##cci +curtains +pigs +pitching +##dad +latvian +sore +dayton +##sted +##qi +patrols +slice +playground +##nted +shone +stool +apparatus +inadequate +mates +treason +##ija +desires +##liga +##croft +somalia +laurent +mir +leonardo +oracle +grape +obliged +chevrolet +thirteenth +stunning +enthusiastic +##ede +accounted +concludes +currents +basil +##kovic +drought +##rica +mai +##aire +shove +posting +##shed +pilgrimage +humorous +packing +fry +pencil +wines +smells +144 +marilyn +aching +newest +clung +bon +neighbours +sanctioned +##pie +mug +##stock +drowning +##mma +hydraulic +##vil +hiring +reminder +lilly +investigators +##ncies +sour +##eous +compulsory +packet +##rion +##graphic +##elle +cannes +##inate +depressed +##rit +heroic +importantly +theresa +##tled +conway +saturn +marginal +rae +##xia +corresponds +royce +pact +jasper +explosives +packaging +aluminium +##ttered +denotes +rhythmic +spans +assignments +hereditary +outlined +originating +sundays +lad +reissued +greeting +beatrice +##dic +pillar +marcos +plots +handbook +alcoholic +judiciary +avant +slides +extract +masculine +blur +##eum +##force +homage +trembled +owens +hymn +trey +omega +signaling +socks +accumulated +reacted +attic +theo +lining +angie +distraction +primera +talbot +##key +1200 +ti +creativity +billed +##hey +deacon +eduardo +identifies +proposition +dizzy +gunner +hogan +##yam +##pping +##hol +ja +##chan +jensen +reconstructed +##berger +clearance +darius +##nier +abe +harlem +plea +dei +circled +emotionally +notation +fascist +neville +exceeded +upwards +viable +ducks +##fo +workforce +racer +limiting +shri +##lson +possesses +1600 +kerr +moths +devastating +laden +disturbing +locking +##cture +gal +fearing +accreditation +flavor +aide +1870s +mountainous +##baum +melt +##ures +motel +texture +servers +soda +##mb +herd +##nium +erect +puzzled +hum +peggy +examinations +gould +testified +geoff +ren +devised +sacks +##law +denial +posters +grunted +cesar +tutor +ec +gerry +offerings +byrne +falcons +combinations +ct +incoming +pardon +rocking +26th +avengers +flared +mankind +seller +uttar +loch +nadia +stroking +exposing +##hd +fertile +ancestral +instituted +##has +noises +prophecy +taxation +eminent +vivid +pol +##bol +dart +indirect +multimedia +notebook +upside +displaying +adrenaline +referenced +geometric +##iving +progression +##ddy +blunt +announce +##far +implementing +##lav +aggression +liaison +cooler +cares +headache +plantations +gorge +dots +impulse +thickness +ashamed +averaging +kathy +obligation +precursor +137 +fowler +symmetry +thee +225 +hears +##rai +undergoing +ads +butcher +bowler +##lip +cigarettes +subscription +goodness +##ically +browne +##hos +##tech +kyoto +donor +##erty +damaging +friction +drifting +expeditions +hardened +prostitution +152 +fauna +blankets +claw +tossing +snarled +butterflies +recruits +investigative +coated +healed +138 +communal +hai +xiii +academics +boone +psychologist +restless +lahore +stephens +mba +brendan +foreigners +printer +##pc +ached +explode +27th +deed +scratched +dared +##pole +cardiac +1780 +okinawa +proto +commando +compelled +oddly +electrons +##base +replica +thanksgiving +##rist +sheila +deliberate +stafford +tidal +representations +hercules +ou +##path +##iated +kidnapping +lenses +##tling +deficit +samoa +mouths +consuming +computational +maze +granting +smirk +razor +fixture +ideals +inviting +aiden +nominal +##vs +issuing +julio +pitt +ramsey +docks +##oss +exhaust +##owed +bavarian +draped +anterior +mating +ethiopian +explores +noticing +##nton +discarded +convenience +hoffman +endowment +beasts +cartridge +mormon +paternal +probe +sleeves +interfere +lump +deadline +##rail +jenks +bulldogs +scrap +alternating +justified +reproductive +nam +seize +descending +secretariat +kirby +coupe +grouped +smash +panther +sedan +tapping +##18 +lola +cheer +germanic +unfortunate +##eter +unrelated +##fan +subordinate +##sdale +suzanne +advertisement +##ility +horsepower +##lda +cautiously +discourse +luigi +##mans +##fields +noun +prevalent +mao +schneider +everett +surround +governorate +kira +##avia +westward +##take +misty +rails +sustainability +134 +unused +##rating +packs +toast +unwilling +regulate +thy +suffrage +nile +awe +assam +definitions +travelers +affordable +##rb +conferred +sells +undefeated +beneficial +torso +basal +repeating +remixes +##pass +bahrain +cables +fang +##itated +excavated +numbering +statutory +##rey +deluxe +##lian +forested +ramirez +derbyshire +zeus +slamming +transfers +astronomer +banana +lottery +berg +histories +bamboo +##uchi +resurrection +posterior +bowls +vaguely +##thi +thou +preserving +tensed +offence +##inas +meyrick +callum +ridden +watt +langdon +tying +lowland +snorted +daring +truman +##hale +##girl +aura +overly +filing +weighing +goa +infections +philanthropist +saunders +eponymous +##owski +latitude +perspectives +reviewing +mets +commandant +radial +##kha +flashlight +reliability +koch +vowels +amazed +ada +elaine +supper +##rth +##encies +predator +debated +soviets +cola +##boards +##nah +compartment +crooked +arbitrary +fourteenth +##ctive +havana +majors +steelers +clips +profitable +ambush +exited +packers +##tile +nude +cracks +fungi +##е +limb +trousers +josie +shelby +tens +frederic +##ος +definite +smoothly +constellation +insult +baton +discs +lingering +##nco +conclusions +lent +staging +becker +grandpa +shaky +##tron +einstein +obstacles +sk +adverse +elle +economically +##moto +mccartney +thor +dismissal +motions +readings +nostrils +treatise +##pace +squeezing +evidently +prolonged +1783 +venezuelan +je +marguerite +beirut +takeover +shareholders +##vent +denise +digit +airplay +norse +##bbling +imaginary +pills +hubert +blaze +vacated +eliminating +##ello +vine +mansfield +##tty +retrospective +barrow +borne +clutch +bail +forensic +weaving +##nett +##witz +desktop +citadel +promotions +worrying +dorset +ieee +subdivided +##iating +manned +expeditionary +pickup +synod +chuckle +185 +barney +##rz +##ffin +functionality +karachi +litigation +meanings +uc +lick +turbo +anders +##ffed +execute +curl +oppose +ankles +typhoon +##د +##ache +##asia +linguistics +compassion +pressures +grazing +perfection +##iting +immunity +monopoly +muddy +backgrounds +136 +namibia +francesca +monitors +attracting +stunt +tuition +##ии +vegetable +##mates +##quent +mgm +jen +complexes +forts +##ond +cellar +bites +seventeenth +royals +flemish +failures +mast +charities +##cular +peruvian +capitals +macmillan +ipswich +outward +frigate +postgraduate +folds +employing +##ouse +concurrently +fiery +##tai +contingent +nightmares +monumental +nicaragua +##kowski +lizard +mal +fielding +gig +reject +##pad +harding +##ipe +coastline +##cin +##nos +beethoven +humphrey +innovations +##tam +##nge +norris +doris +solicitor +huang +obey +141 +##lc +niagara +##tton +shelves +aug +bourbon +curry +nightclub +specifications +hilton +##ndo +centennial +dispersed +worm +neglected +briggs +sm +font +kuala +uneasy +plc +##nstein +##bound +##aking +##burgh +awaiting +pronunciation +##bbed +##quest +eh +optimal +zhu +raped +greens +presided +brenda +worries +##life +venetian +marxist +turnout +##lius +refined +braced +sins +grasped +sunderland +nickel +speculated +lowell +cyrillic +communism +fundraising +resembling +colonists +mutant +freddie +usc +##mos +gratitude +##run +mural +##lous +chemist +wi +reminds +28th +steals +tess +pietro +##ingen +promoter +ri +microphone +honoured +rai +sant +##qui +feather +##nson +burlington +kurdish +terrorists +deborah +sickness +##wed +##eet +hazard +irritated +desperation +veil +clarity +##rik +jewels +xv +##gged +##ows +##cup +berkshire +unfair +mysteries +orchid +winced +exhaustion +renovations +stranded +obe +infinity +##nies +adapt +redevelopment +thanked +registry +olga +domingo +noir +tudor +ole +##atus +commenting +behaviors +##ais +crisp +pauline +probable +stirling +wigan +##bian +paralympics +panting +surpassed +##rew +luca +barred +pony +famed +##sters +cassandra +waiter +carolyn +exported +##orted +andres +destructive +deeds +jonah +castles +vacancy +suv +##glass +1788 +orchard +yep +famine +belarusian +sprang +##forth +skinny +##mis +administrators +rotterdam +zambia +zhao +boiler +discoveries +##ride +##physics +lucius +disappointing +outreach +spoon +##frame +qualifications +unanimously +enjoys +regency +##iidae +stade +realism +veterinary +rodgers +dump +alain +chestnut +castile +censorship +rumble +gibbs +##itor +communion +reggae +inactivated +logs +loads +##houses +homosexual +##iano +ale +informs +##cas +phrases +plaster +linebacker +ambrose +kaiser +fascinated +850 +limerick +recruitment +forge +mastered +##nding +leinster +rooted +threaten +##strom +borneo +##hes +suggestions +scholarships +propeller +documentaries +patronage +coats +constructing +invest +neurons +comet +entirety +shouts +identities +annoying +unchanged +wary +##antly +##ogy +neat +oversight +##kos +phillies +replay +constance +##kka +incarnation +humble +skies +minus +##acy +smithsonian +##chel +guerrilla +jar +cadets +##plate +surplus +audit +##aru +cracking +joanna +louisa +pacing +##lights +intentionally +##iri +diner +nwa +imprint +australians +tong +unprecedented +bunker +naive +specialists +ark +nichols +railing +leaked +pedal +##uka +shrub +longing +roofs +v8 +captains +neural +tuned +##ntal +##jet +emission +medina +frantic +codex +definitive +sid +abolition +intensified +stocks +enrique +sustain +genoa +oxide +##written +clues +cha +##gers +tributaries +fragment +venom +##rity +##ente +##sca +muffled +vain +sire +laos +##ingly +##hana +hastily +snapping +surfaced +sentiment +motive +##oft +contests +approximate +mesa +luckily +dinosaur +exchanges +propelled +accord +bourne +relieve +tow +masks +offended +##ues +cynthia +##mmer +rains +bartender +zinc +reviewers +lois +##sai +legged +arrogant +rafe +rosie +comprise +handicap +blockade +inlet +lagoon +copied +drilling +shelley +petals +##inian +mandarin +obsolete +##inated +onward +arguably +productivity +cindy +praising +seldom +busch +discusses +raleigh +shortage +ranged +stanton +encouragement +firstly +conceded +overs +temporal +##uke +cbe +##bos +woo +certainty +pumps +##pton +stalked +##uli +lizzie +periodic +thieves +weaker +##night +gases +shoving +chooses +wc +##chemical +prompting +weights +##kill +robust +flanked +sticky +hu +tuberculosis +##eb +##eal +christchurch +resembled +wallet +reese +inappropriate +pictured +distract +fixing +fiddle +giggled +burger +heirs +hairy +mechanic +torque +apache +obsessed +chiefly +cheng +logging +##tag +extracted +meaningful +numb +##vsky +gloucestershire +reminding +##bay +unite +##lit +breeds +diminished +clown +glove +1860s +##ن +##ug +archibald +focal +freelance +sliced +depiction +##yk +organism +switches +sights +stray +crawling +##ril +lever +leningrad +interpretations +loops +anytime +reel +alicia +delighted +##ech +inhaled +xiv +suitcase +bernie +vega +licenses +northampton +exclusion +induction +monasteries +racecourse +homosexuality +##right +##sfield +##rky +dimitri +michele +alternatives +ions +commentators +genuinely +objected +pork +hospitality +fencing +stephan +warships +peripheral +wit +drunken +wrinkled +quentin +spends +departing +chung +numerical +spokesperson +##zone +johannesburg +caliber +killers +##udge +assumes +neatly +demographic +abigail +bloc +##vel +mounting +##lain +bentley +slightest +xu +recipients +##jk +merlin +##writer +seniors +prisons +blinking +hindwings +flickered +kappa +##hel +80s +strengthening +appealing +brewing +gypsy +mali +lashes +hulk +unpleasant +harassment +bio +treaties +predict +instrumentation +pulp +troupe +boiling +mantle +##ffe +ins +##vn +dividing +handles +verbs +##onal +coconut +senegal +340 +thorough +gum +momentarily +##sto +cocaine +panicked +destined +##turing +teatro +denying +weary +captained +mans +##hawks +##code +wakefield +bollywood +thankfully +##16 +cyril +##wu +amendments +##bahn +consultation +stud +reflections +kindness +1787 +internally +##ovo +tex +mosaic +distribute +paddy +seeming +143 +##hic +piers +##15 +##mura +##verse +popularly +winger +kang +sentinel +mccoy +##anza +covenant +##bag +verge +fireworks +suppress +thrilled +dominate +##jar +swansea +##60 +142 +reconciliation +##ndi +stiffened +cue +dorian +##uf +damascus +amor +ida +foremost +##aga +porsche +unseen +dir +##had +##azi +stony +lexi +melodies +##nko +angular +integer +podcast +ants +inherent +jaws +justify +persona +##olved +josephine +##nr +##ressed +customary +flashes +gala +cyrus +glaring +backyard +ariel +physiology +greenland +html +stir +avon +atletico +finch +methodology +ked +##lent +mas +catholicism +townsend +branding +quincy +fits +containers +1777 +ashore +aragon +##19 +forearm +poisoning +##sd +adopting +conquer +grinding +amnesty +keller +finances +evaluate +forged +lankan +instincts +##uto +guam +bosnian +photographed +workplace +desirable +protector +##dog +allocation +intently +encourages +willy +##sten +bodyguard +electro +brighter +##ν +bihar +##chev +lasts +opener +amphibious +sal +verde +arte +##cope +captivity +vocabulary +yields +##tted +agreeing +desmond +pioneered +##chus +strap +campaigned +railroads +##ович +emblem +##dre +stormed +501 +##ulous +marijuana +northumberland +##gn +##nath +bowen +landmarks +beaumont +##qua +danube +##bler +attorneys +th +ge +flyers +critique +villains +cass +mutation +acc +##0s +colombo +mckay +motif +sampling +concluding +syndicate +##rell +neon +stables +ds +warnings +clint +mourning +wilkinson +##tated +merrill +leopard +evenings +exhaled +emil +sonia +ezra +discrete +stove +farrell +fifteenth +prescribed +superhero +##rier +worms +helm +wren +##duction +##hc +expo +##rator +hq +unfamiliar +antony +prevents +acceleration +fiercely +mari +painfully +calculations +cheaper +ign +clifton +irvine +davenport +mozambique +##np +pierced +##evich +wonders +##wig +##cate +##iling +crusade +ware +##uel +enzymes +reasonably +mls +##coe +mater +ambition +bunny +eliot +kernel +##fin +asphalt +headmaster +torah +aden +lush +pins +waived +##care +##yas +joao +substrate +enforce +##grad +##ules +alvarez +selections +epidemic +tempted +##bit +bremen +translates +ensured +waterfront +29th +forrest +manny +malone +kramer +reigning +cookies +simpler +absorption +205 +engraved +##ffy +evaluated +1778 +haze +146 +comforting +crossover +##abe +thorn +##rift +##imo +##pop +suppression +fatigue +cutter +##tr +201 +wurttemberg +##orf +enforced +hovering +proprietary +gb +samurai +syllable +ascent +lacey +tick +lars +tractor +merchandise +rep +bouncing +defendants +##yre +huntington +##ground +##oko +standardized +##hor +##hima +assassinated +nu +predecessors +rainy +liar +assurance +lyrical +##uga +secondly +flattened +ios +parameter +undercover +##mity +bordeaux +punish +ridges +markers +exodus +inactive +hesitate +debbie +nyc +pledge +savoy +nagar +offset +organist +##tium +hesse +marin +converting +##iver +diagram +propulsion +pu +validity +reverted +supportive +##dc +ministries +clans +responds +proclamation +##inae +##ø +##rea +ein +pleading +patriot +sf +birch +islanders +strauss +hates +##dh +brandenburg +concession +rd +##ob +1900s +killings +textbook +antiquity +cinematography +wharf +embarrassing +setup +creed +farmland +inequality +centred +signatures +fallon +370 +##ingham +##uts +ceylon +gazing +directive +laurie +##tern +globally +##uated +##dent +allah +excavation +threads +##cross +148 +frantically +icc +utilize +determines +respiratory +thoughtful +receptions +##dicate +merging +chandra +seine +147 +builders +builds +diagnostic +dev +visibility +goddamn +analyses +dhaka +cho +proves +chancel +concurrent +curiously +canadians +pumped +restoring +1850s +turtles +jaguar +sinister +spinal +traction +declan +vows +1784 +glowed +capitalism +swirling +install +universidad +##lder +##oat +soloist +##genic +##oor +coincidence +beginnings +nissan +dip +resorts +caucasus +combustion +infectious +##eno +pigeon +serpent +##itating +conclude +masked +salad +jew +##gr +surreal +toni +##wc +harmonica +151 +##gins +##etic +##coat +fishermen +intending +bravery +##wave +klaus +titan +wembley +taiwanese +ransom +40th +incorrect +hussein +eyelids +jp +cooke +dramas +utilities +##etta +##print +eisenhower +principally +granada +lana +##rak +openings +concord +##bl +bethany +connie +morality +sega +##mons +##nard +earnings +##kara +##cine +wii +communes +##rel +coma +composing +softened +severed +grapes +##17 +nguyen +analyzed +warlord +hubbard +heavenly +behave +slovenian +##hit +##ony +hailed +filmmakers +trance +caldwell +skye +unrest +coward +likelihood +##aging +bern +sci +taliban +honolulu +propose +##wang +1700 +browser +imagining +cobra +contributes +dukes +instinctively +conan +violinist +##ores +accessories +gradual +##amp +quotes +sioux +##dating +undertake +intercepted +sparkling +compressed +139 +fungus +tombs +haley +imposing +rests +degradation +lincolnshire +retailers +wetlands +tulsa +distributor +dungeon +nun +greenhouse +convey +atlantis +aft +exits +oman +dresser +lyons +##sti +joking +eddy +judgement +omitted +digits +##cts +##game +juniors +##rae +cents +stricken +une +##ngo +wizards +weir +breton +nan +technician +fibers +liking +royalty +##cca +154 +persia +terribly +magician +##rable +##unt +vance +cafeteria +booker +camille +warmer +##static +consume +cavern +gaps +compass +contemporaries +foyer +soothing +graveyard +maj +plunged +blush +##wear +cascade +demonstrates +ordinance +##nov +boyle +##lana +rockefeller +shaken +banjo +izzy +##ense +breathless +vines +##32 +##eman +alterations +chromosome +dwellings +feudal +mole +153 +catalonia +relics +tenant +mandated +##fm +fridge +hats +honesty +patented +raul +heap +cruisers +accusing +enlightenment +infants +wherein +chatham +contractors +zen +affinity +hc +osborne +piston +156 +traps +maturity +##rana +lagos +##zal +peering +##nay +attendant +dealers +protocols +subset +prospects +biographical +##cre +artery +##zers +insignia +nuns +endured +##eration +recommend +schwartz +serbs +berger +cromwell +crossroads +##ctor +enduring +clasped +grounded +##bine +marseille +twitched +abel +choke +https +catalyst +moldova +italians +##tist +disastrous +wee +##oured +##nti +wwf +nope +##piration +##asa +expresses +thumbs +167 +##nza +coca +1781 +cheating +##ption +skipped +sensory +heidelberg +spies +satan +dangers +semifinal +202 +bohemia +whitish +confusing +shipbuilding +relies +surgeons +landings +ravi +baku +moor +suffix +alejandro +##yana +litre +upheld +##unk +rajasthan +##rek +coaster +insists +posture +scenarios +etienne +favoured +appoint +transgender +elephants +poked +greenwood +defences +fulfilled +militant +somali +1758 +chalk +potent +##ucci +migrants +wink +assistants +nos +restriction +activism +niger +##ario +colon +shaun +##sat +daphne +##erated +swam +congregations +reprise +considerations +magnet +playable +xvi +##р +overthrow +tobias +knob +chavez +coding +##mers +propped +katrina +orient +newcomer +##suke +temperate +##pool +farmhouse +interrogation +##vd +committing +##vert +forthcoming +strawberry +joaquin +macau +ponds +shocking +siberia +##cellular +chant +contributors +##nant +##ologists +sped +absorb +hail +1782 +spared +##hore +barbados +karate +opus +originates +saul +##xie +evergreen +leaped +##rock +correlation +exaggerated +weekday +unification +bump +tracing +brig +afb +pathways +utilizing +##ners +mod +mb +disturbance +kneeling +##stad +##guchi +100th +pune +##thy +decreasing +168 +manipulation +miriam +academia +ecosystem +occupational +rbi +##lem +rift +##14 +rotary +stacked +incorporation +awakening +generators +guerrero +racist +##omy +cyber +derivatives +culminated +allie +annals +panzer +sainte +wikipedia +pops +zu +austro +##vate +algerian +politely +nicholson +mornings +educate +tastes +thrill +dartmouth +##gating +db +##jee +regan +differing +concentrating +choreography +divinity +##media +pledged +alexandre +routing +gregor +madeline +##idal +apocalypse +##hora +gunfire +culminating +elves +fined +liang +lam +programmed +tar +guessing +transparency +gabrielle +##gna +cancellation +flexibility +##lining +accession +shea +stronghold +nets +specializes +##rgan +abused +hasan +sgt +ling +exceeding +##₄ +admiration +supermarket +##ark +photographers +specialised +tilt +resonance +hmm +perfume +380 +sami +threatens +garland +botany +guarding +boiled +greet +puppy +russo +supplier +wilmington +vibrant +vijay +##bius +paralympic +grumbled +paige +faa +licking +margins +hurricanes +##gong +fest +grenade +ripping +##uz +counseling +weigh +##sian +needles +wiltshire +edison +costly +##not +fulton +tramway +redesigned +staffordshire +cache +gasping +watkins +sleepy +candidacy +##group +monkeys +timeline +throbbing +##bid +##sos +berth +uzbekistan +vanderbilt +bothering +overturned +ballots +gem +##iger +sunglasses +subscribers +hooker +compelling +ang +exceptionally +saloon +stab +##rdi +carla +terrifying +rom +##vision +coil +##oids +satisfying +vendors +31st +mackay +deities +overlooked +ambient +bahamas +felipe +olympia +whirled +botanist +advertised +tugging +##dden +disciples +morales +unionist +rites +foley +morse +motives +creepy +##₀ +soo +##sz +bargain +highness +frightening +turnpike +tory +reorganization +##cer +depict +biographer +##walk +unopposed +manifesto +##gles +institut +emile +accidental +kapoor +##dam +kilkenny +cortex +lively +##13 +romanesque +jain +shan +cannons +##ood +##ske +petrol +echoing +amalgamated +disappears +cautious +proposes +sanctions +trenton +##ر +flotilla +aus +contempt +tor +canary +cote +theirs +##hun +conceptual +deleted +fascinating +paso +blazing +elf +honourable +hutchinson +##eiro +##outh +##zin +surveyor +tee +amidst +wooded +reissue +intro +##ono +cobb +shelters +newsletter +hanson +brace +encoding +confiscated +dem +caravan +marino +scroll +melodic +cows +imam +##adi +##aneous +northward +searches +biodiversity +cora +310 +roaring +##bers +connell +theologian +halo +compose +pathetic +unmarried +dynamo +##oot +az +calculation +toulouse +deserves +humour +nr +forgiveness +tam +undergone +martyr +pamela +myths +whore +counselor +hicks +290 +heavens +battleship +electromagnetic +##bbs +stellar +establishments +presley +hopped +##chin +temptation +90s +wills +nas +##yuan +nhs +##nya +seminars +##yev +adaptations +gong +asher +lex +indicator +sikh +tobago +cites +goin +##yte +satirical +##gies +characterised +correspond +bubbles +lure +participates +##vid +eruption +skate +therapeutic +1785 +canals +wholesale +defaulted +sac +460 +petit +##zzled +virgil +leak +ravens +256 +portraying +##yx +ghetto +creators +dams +portray +vicente +##rington +fae +namesake +bounty +##arium +joachim +##ota +##iser +aforementioned +axle +snout +depended +dismantled +reuben +480 +##ibly +gallagher +##lau +##pd +earnest +##ieu +##iary +inflicted +objections +##llar +asa +gritted +##athy +jericho +##sea +##was +flick +underside +ceramics +undead +substituted +195 +eastward +undoubtedly +wheeled +chimney +##iche +guinness +cb +##ager +siding +##bell +traitor +baptiste +disguised +inauguration +149 +tipperary +choreographer +perched +warmed +stationary +eco +##ike +##ntes +bacterial +##aurus +flores +phosphate +##core +attacker +invaders +alvin +intersects +a1 +indirectly +immigrated +businessmen +cornelius +valves +narrated +pill +sober +ul +nationale +monastic +applicants +scenery +##jack +161 +motifs +constitutes +cpu +##osh +jurisdictions +sd +tuning +irritation +woven +##uddin +fertility +gao +##erie +antagonist +impatient +glacial +hides +boarded +denominations +interception +##jas +cookie +nicola +##tee +algebraic +marquess +bahn +parole +buyers +bait +turbines +paperwork +bestowed +natasha +renee +oceans +purchases +157 +vaccine +215 +##tock +fixtures +playhouse +integrate +jai +oswald +intellectuals +##cky +booked +nests +mortimer +##isi +obsession +sept +##gler +##sum +440 +scrutiny +simultaneous +squinted +##shin +collects +oven +shankar +penned +remarkably +##я +slips +luggage +spectral +1786 +collaborations +louie +consolidation +##ailed +##ivating +420 +hoover +blackpool +harness +ignition +vest +tails +belmont +mongol +skinner +##nae +visually +mage +derry +##tism +##unce +stevie +transitional +##rdy +redskins +drying +prep +prospective +##21 +annoyance +oversee +##loaded +fills +##books +##iki +announces +fda +scowled +respects +prasad +mystic +tucson +##vale +revue +springer +bankrupt +1772 +aristotle +salvatore +habsburg +##geny +dal +natal +nut +pod +chewing +darts +moroccan +walkover +rosario +lenin +punjabi +##ße +grossed +scattering +wired +invasive +hui +polynomial +corridors +wakes +gina +portrays +##cratic +arid +retreating +erich +irwin +sniper +##dha +linen +lindsey +maneuver +butch +shutting +socio +bounce +commemorative +postseason +jeremiah +pines +275 +mystical +beads +bp +abbas +furnace +bidding +consulted +assaulted +empirical +rubble +enclosure +sob +weakly +cancel +polly +yielded +##emann +curly +prediction +battered +70s +vhs +jacqueline +render +sails +barked +detailing +grayson +riga +sloane +raging +##yah +herbs +bravo +##athlon +alloy +giggle +imminent +suffers +assumptions +waltz +##itate +accomplishments +##ited +bathing +remixed +deception +prefix +##emia +deepest +##tier +##eis +balkan +frogs +##rong +slab +##pate +philosophers +peterborough +grains +imports +dickinson +rwanda +##atics +1774 +dirk +lan +tablets +##rove +clone +##rice +caretaker +hostilities +mclean +##gre +regimental +treasures +norms +impose +tsar +tango +diplomacy +variously +complain +192 +recognise +arrests +1779 +celestial +pulitzer +##dus +bing +libretto +##moor +adele +splash +##rite +expectation +lds +confronts +##izer +spontaneous +harmful +wedge +entrepreneurs +buyer +##ope +bilingual +translate +rugged +conner +circulated +uae +eaton +##gra +##zzle +lingered +lockheed +vishnu +reelection +alonso +##oom +joints +yankee +headline +cooperate +heinz +laureate +invading +##sford +echoes +scandinavian +##dham +hugging +vitamin +salute +micah +hind +trader +##sper +radioactive +##ndra +militants +poisoned +ratified +remark +campeonato +deprived +wander +prop +##dong +outlook +##tani +##rix +##eye +chiang +darcy +##oping +mandolin +spice +statesman +babylon +182 +walled +forgetting +afro +##cap +158 +giorgio +buffer +##polis +planetary +##gis +overlap +terminals +kinda +centenary +##bir +arising +manipulate +elm +ke +1770 +ak +##tad +chrysler +mapped +moose +pomeranian +quad +macarthur +assemblies +shoreline +recalls +stratford +##rted +noticeable +##evic +imp +##rita +##sque +accustomed +supplying +tents +disgusted +vogue +sipped +filters +khz +reno +selecting +luftwaffe +mcmahon +tyne +masterpiece +carriages +collided +dunes +exercised +flare +remembers +muzzle +##mobile +heck +##rson +burgess +lunged +middleton +boycott +bilateral +##sity +hazardous +lumpur +multiplayer +spotlight +jackets +goldman +liege +porcelain +rag +waterford +benz +attracts +hopeful +battling +ottomans +kensington +baked +hymns +cheyenne +lattice +levine +borrow +polymer +clashes +michaels +monitored +commitments +denounced +##25 +##von +cavity +##oney +hobby +akin +##holders +futures +intricate +cornish +patty +##oned +illegally +dolphin +##lag +barlow +yellowish +maddie +apologized +luton +plagued +##puram +nana +##rds +sway +fanny +łodz +##rino +psi +suspicions +hanged +##eding +initiate +charlton +##por +nak +competent +235 +analytical +annex +wardrobe +reservations +##rma +sect +162 +fairfax +hedge +piled +buckingham +uneven +bauer +simplicity +snyder +interpret +accountability +donors +moderately +byrd +continents +##cite +##max +disciple +hr +jamaican +ping +nominees +##uss +mongolian +diver +attackers +eagerly +ideological +pillows +miracles +apartheid +revolver +sulfur +clinics +moran +163 +##enko +ile +katy +rhetoric +##icated +chronology +recycling +##hrer +elongated +mughal +pascal +profiles +vibration +databases +domination +##fare +##rant +matthias +digest +rehearsal +polling +weiss +initiation +reeves +clinging +flourished +impress +ngo +##hoff +##ume +buckley +symposium +rhythms +weed +emphasize +transforming +##taking +##gence +##yman +accountant +analyze +flicker +foil +priesthood +voluntarily +decreases +##80 +##hya +slater +sv +charting +mcgill +##lde +moreno +##iu +besieged +zur +robes +##phic +admitting +api +deported +turmoil +peyton +earthquakes +##ares +nationalists +beau +clair +brethren +interrupt +welch +curated +galerie +requesting +164 +##ested +impending +steward +viper +##vina +complaining +beautifully +brandy +foam +nl +1660 +##cake +alessandro +punches +laced +explanations +##lim +attribute +clit +reggie +discomfort +##cards +smoothed +whales +##cene +adler +countered +duffy +disciplinary +widening +recipe +reliance +conducts +goats +gradient +preaching +##shaw +matilda +quasi +striped +meridian +cannabis +cordoba +certificates +##agh +##tering +graffiti +hangs +pilgrims +repeats +##ych +revive +urine +etat +##hawk +fueled +belts +fuzzy +susceptible +##hang +mauritius +salle +sincere +beers +hooks +##cki +arbitration +entrusted +advise +sniffed +seminar +junk +donnell +processors +principality +strapped +celia +mendoza +everton +fortunes +prejudice +starving +reassigned +steamer +##lund +tuck +evenly +foreman +##ffen +dans +375 +envisioned +slit +##xy +baseman +liberia +rosemary +##weed +electrified +periodically +potassium +stride +contexts +sperm +slade +mariners +influx +bianca +subcommittee +##rane +spilling +icao +estuary +##nock +delivers +iphone +##ulata +isa +mira +bohemian +dessert +##sbury +welcoming +proudly +slowing +##chs +musee +ascension +russ +##vian +waits +##psy +africans +exploit +##morphic +gov +eccentric +crab +peck +##ull +entrances +formidable +marketplace +groom +bolted +metabolism +patton +robbins +courier +payload +endure +##ifier +andes +refrigerator +##pr +ornate +##uca +ruthless +illegitimate +masonry +strasbourg +bikes +adobe +##³ +apples +quintet +willingly +niche +bakery +corpses +energetic +##cliffe +##sser +##ards +177 +centimeters +centro +fuscous +cretaceous +rancho +##yde +andrei +telecom +tottenham +oasis +ordination +vulnerability +presiding +corey +cp +penguins +sims +##pis +malawi +piss +##48 +correction +##cked +##ffle +##ryn +countdown +detectives +psychiatrist +psychedelic +dinosaurs +blouse +##get +choi +vowed +##oz +randomly +##pol +49ers +scrub +blanche +bruins +dusseldorf +##using +unwanted +##ums +212 +dominique +elevations +headlights +om +laguna +##oga +1750 +famously +ignorance +shrewsbury +##aine +ajax +breuning +che +confederacy +greco +overhaul +##screen +paz +skirts +disagreement +cruelty +jagged +phoebe +shifter +hovered +viruses +##wes +mandy +##lined +##gc +landlord +squirrel +dashed +##ι +ornamental +gag +wally +grange +literal +spurs +undisclosed +proceeding +yin +##text +billie +orphan +spanned +humidity +indy +weighted +presentations +explosions +lucian +##tary +vaughn +hindus +##anga +##hell +psycho +171 +daytona +protects +efficiently +rematch +sly +tandem +##oya +rebranded +impaired +hee +metropolis +peach +godfrey +diaspora +ethnicity +prosperous +gleaming +dar +grossing +playback +##rden +stripe +pistols +##tain +births +labelled +##cating +172 +rudy +alba +##onne +aquarium +hostility +##gb +##tase +shudder +sumatra +hardest +lakers +consonant +creeping +demos +homicide +capsule +zeke +liberties +expulsion +pueblo +##comb +trait +transporting +##ddin +##neck +##yna +depart +gregg +mold +ledge +hangar +oldham +playboy +termination +analysts +gmbh +romero +##itic +insist +cradle +filthy +brightness +slash +shootout +deposed +bordering +##truct +isis +microwave +tumbled +sheltered +cathy +werewolves +messy +andersen +convex +clapped +clinched +satire +wasting +edo +vc +rufus +##jak +mont +##etti +poznan +##keeping +restructuring +transverse +##rland +azerbaijani +slovene +gestures +roommate +choking +shear +##quist +vanguard +oblivious +##hiro +disagreed +baptism +##lich +coliseum +##aceae +salvage +societe +cory +locke +relocation +relying +versailles +ahl +swelling +##elo +cheerful +##word +##edes +gin +sarajevo +obstacle +diverted +##nac +messed +thoroughbred +fluttered +utrecht +chewed +acquaintance +assassins +dispatch +mirza +##wart +nike +salzburg +swell +yen +##gee +idle +ligue +samson +##nds +##igh +playful +spawned +##cise +tease +##case +burgundy +##bot +stirring +skeptical +interceptions +marathi +##dies +bedrooms +aroused +pinch +##lik +preferences +tattoos +buster +digitally +projecting +rust +##ital +kitten +priorities +addison +pseudo +##guard +dusk +icons +sermon +##psis +##iba +bt +##lift +##xt +ju +truce +rink +##dah +##wy +defects +psychiatry +offences +calculate +glucose +##iful +##rized +##unda +francaise +##hari +richest +warwickshire +carly +1763 +purity +redemption +lending +##cious +muse +bruises +cerebral +aero +carving +##name +preface +terminology +invade +monty +##int +anarchist +blurred +##iled +rossi +treats +guts +shu +foothills +ballads +undertaking +premise +cecilia +affiliates +blasted +conditional +wilder +minors +drone +rudolph +buffy +swallowing +horton +attested +##hop +rutherford +howell +primetime +livery +penal +##bis +minimize +hydro +wrecked +wrought +palazzo +##gling +cans +vernacular +friedman +nobleman +shale +walnut +danielle +##ection +##tley +sears +##kumar +chords +lend +flipping +streamed +por +dracula +gallons +sacrifices +gamble +orphanage +##iman +mckenzie +##gible +boxers +daly +##balls +##ان +208 +##ific +##rative +##iq +exploited +slated +##uity +circling +hillary +pinched +goldberg +provost +campaigning +lim +piles +ironically +jong +mohan +successors +usaf +##tem +##ught +autobiographical +haute +preserves +##ending +acquitted +comparisons +203 +hydroelectric +gangs +cypriot +torpedoes +rushes +chrome +derive +bumps +instability +fiat +pets +##mbe +silas +dye +reckless +settler +##itation +info +heats +##writing +176 +canonical +maltese +fins +mushroom +stacy +aspen +avid +##kur +##loading +vickers +gaston +hillside +statutes +wilde +gail +kung +sabine +comfortably +motorcycles +##rgo +169 +pneumonia +fetch +##sonic +axel +faintly +parallels +##oop +mclaren +spouse +compton +interdisciplinary +miner +##eni +181 +clamped +##chal +##llah +separates +versa +##mler +scarborough +labrador +##lity +##osing +rutgers +hurdles +como +166 +burt +divers +##100 +wichita +cade +coincided +##erson +bruised +mla +##pper +vineyard +##ili +##brush +notch +mentioning +jase +hearted +kits +doe +##acle +pomerania +##ady +ronan +seizure +pavel +problematic +##zaki +domenico +##ulin +catering +penelope +dependence +parental +emilio +ministerial +atkinson +##bolic +clarkson +chargers +colby +grill +peeked +arises +summon +##aged +fools +##grapher +faculties +qaeda +##vial +garner +refurbished +##hwa +geelong +disasters +nudged +bs +shareholder +lori +algae +reinstated +rot +##ades +##nous +invites +stainless +183 +inclusive +##itude +diocesan +til +##icz +denomination +##xa +benton +floral +registers +##ider +##erman +##kell +absurd +brunei +guangzhou +hitter +retaliation +##uled +##eve +blanc +nh +consistency +contamination +##eres +##rner +dire +palermo +broadcasters +diaries +inspire +vols +brewer +tightening +ky +mixtape +hormone +##tok +stokes +##color +##dly +##ssi +pg +##ometer +##lington +sanitation +##tility +intercontinental +apps +##adt +¹⁄₂ +cylinders +economies +favourable +unison +croix +gertrude +odyssey +vanity +dangling +##logists +upgrades +dice +middleweight +practitioner +##ight +206 +henrik +parlor +orion +angered +lac +python +blurted +##rri +sensual +intends +swings +angled +##phs +husky +attain +peerage +precinct +textiles +cheltenham +shuffled +dai +confess +tasting +bhutan +##riation +tyrone +segregation +abrupt +ruiz +##rish +smirked +blackwell +confidential +browning +amounted +##put +vase +scarce +fabulous +raided +staple +guyana +unemployed +glider +shay +##tow +carmine +troll +intervene +squash +superstar +##uce +cylindrical +len +roadway +researched +handy +##rium +##jana +meta +lao +declares +##rring +##tadt +##elin +##kova +willem +shrubs +napoleonic +realms +skater +qi +volkswagen +##ł +tad +hara +archaeologist +awkwardly +eerie +##kind +wiley +##heimer +##24 +titus +organizers +cfl +crusaders +lama +usb +vent +enraged +thankful +occupants +maximilian +##gaard +possessing +textbooks +##oran +collaborator +quaker +##ulo +avalanche +mono +silky +straits +isaiah +mustang +surged +resolutions +potomac +descend +cl +kilograms +plato +strains +saturdays +##olin +bernstein +##ype +holstein +ponytail +##watch +belize +conversely +heroine +perpetual +##ylus +charcoal +piedmont +glee +negotiating +backdrop +prologue +##jah +##mmy +pasadena +climbs +ramos +sunni +##holm +##tner +##tri +anand +deficiency +hertfordshire +stout +##avi +aperture +orioles +##irs +doncaster +intrigued +bombed +coating +otis +##mat +cocktail +##jit +##eto +amir +arousal +sar +##proof +##act +##ories +dixie +pots +##bow +whereabouts +159 +##fted +drains +bullying +cottages +scripture +coherent +fore +poe +appetite +##uration +sampled +##ators +##dp +derrick +rotor +jays +peacock +installment +##rro +advisors +##coming +rodeo +scotch +##mot +##db +##fen +##vant +ensued +rodrigo +dictatorship +martyrs +twenties +##н +towed +incidence +marta +rainforest +sai +scaled +##cles +oceanic +qualifiers +symphonic +mcbride +dislike +generalized +aubrey +colonization +##iation +##lion +##ssing +disliked +lublin +salesman +##ulates +spherical +whatsoever +sweating +avalon +contention +punt +severity +alderman +atari +##dina +##grant +##rop +scarf +seville +vertices +annexation +fairfield +fascination +inspiring +launches +palatinate +regretted +##rca +feral +##iom +elk +nap +olsen +reddy +yong +##leader +##iae +garment +transports +feng +gracie +outrage +viceroy +insides +##esis +breakup +grady +organizer +softer +grimaced +222 +murals +galicia +arranging +vectors +##rsten +bas +##sb +##cens +sloan +##eka +bitten +ara +fender +nausea +bumped +kris +banquet +comrades +detector +persisted +##llan +adjustment +endowed +cinemas +##shot +sellers +##uman +peek +epa +kindly +neglect +simpsons +talon +mausoleum +runaway +hangul +lookout +##cic +rewards +coughed +acquainted +chloride +##ald +quicker +accordion +neolithic +##qa +artemis +coefficient +lenny +pandora +tx +##xed +ecstasy +litter +segunda +chairperson +gemma +hiss +rumor +vow +nasal +antioch +compensate +patiently +transformers +##eded +judo +morrow +penis +posthumous +philips +bandits +husbands +denote +flaming +##any +##phones +langley +yorker +1760 +walters +##uo +##kle +gubernatorial +fatty +samsung +leroy +outlaw +##nine +unpublished +poole +jakob +##ᵢ +##ₙ +crete +distorted +superiority +##dhi +intercept +crust +mig +claus +crashes +positioning +188 +stallion +301 +frontal +armistice +##estinal +elton +aj +encompassing +camel +commemorated +malaria +woodward +calf +cigar +penetrate +##oso +willard +##rno +##uche +illustrate +amusing +convergence +noteworthy +##lma +##rva +journeys +realise +manfred +##sable +410 +##vocation +hearings +fiance +##posed +educators +provoked +adjusting +##cturing +modular +stockton +paterson +vlad +rejects +electors +selena +maureen +##tres +uber +##rce +swirled +##num +proportions +nanny +pawn +naturalist +parma +apostles +awoke +ethel +wen +##bey +monsoon +overview +##inating +mccain +rendition +risky +adorned +##ih +equestrian +germain +nj +conspicuous +confirming +##yoshi +shivering +##imeter +milestone +rumours +flinched +bounds +smacked +token +##bei +lectured +automobiles +##shore +impacted +##iable +nouns +nero +##leaf +ismail +prostitute +trams +##lace +bridget +sud +stimulus +impressions +reins +revolves +##oud +##gned +giro +honeymoon +##swell +criterion +##sms +##uil +libyan +prefers +##osition +211 +preview +sucks +accusation +bursts +metaphor +diffusion +tolerate +faye +betting +cinematographer +liturgical +specials +bitterly +humboldt +##ckle +flux +rattled +##itzer +archaeologists +odor +authorised +marshes +discretion +##ов +alarmed +archaic +inverse +##leton +explorers +##pine +drummond +tsunami +woodlands +##minate +##tland +booklet +insanity +owning +insert +crafted +calculus +##tore +receivers +##bt +stung +##eca +##nched +prevailing +travellers +eyeing +lila +graphs +##borne +178 +julien +##won +morale +adaptive +therapist +erica +cw +libertarian +bowman +pitches +vita +##ional +crook +##ads +##entation +caledonia +mutiny +##sible +1840s +automation +##ß +flock +##pia +ironic +pathology +##imus +remarried +##22 +joker +withstand +energies +##att +shropshire +hostages +madeleine +tentatively +conflicting +mateo +recipes +euros +ol +mercenaries +nico +##ndon +albuquerque +augmented +mythical +bel +freud +##child +cough +##lica +365 +freddy +lillian +genetically +nuremberg +calder +209 +bonn +outdoors +paste +suns +urgency +vin +restraint +tyson +##cera +##selle +barrage +bethlehem +kahn +##par +mounts +nippon +barony +happier +ryu +makeshift +sheldon +blushed +castillo +barking +listener +taped +bethel +fluent +headlines +pornography +rum +disclosure +sighing +mace +doubling +gunther +manly +##plex +rt +interventions +physiological +forwards +emerges +##tooth +##gny +compliment +rib +recession +visibly +barge +faults +connector +exquisite +prefect +##rlin +patio +##cured +elevators +brandt +italics +pena +173 +wasp +satin +ea +botswana +graceful +respectable +##jima +##rter +##oic +franciscan +generates +##dl +alfredo +disgusting +##olate +##iously +sherwood +warns +cod +promo +cheryl +sino +##ة +##escu +twitch +##zhi +brownish +thom +ortiz +##dron +densely +##beat +carmel +reinforce +##bana +187 +anastasia +downhill +vertex +contaminated +remembrance +harmonic +homework +##sol +fiancee +gears +olds +angelica +loft +ramsay +quiz +colliery +sevens +##cape +autism +##hil +walkway +##boats +ruben +abnormal +ounce +khmer +##bbe +zachary +bedside +morphology +punching +##olar +sparrow +convinces +##35 +hewitt +queer +remastered +rods +mabel +solemn +notified +lyricist +symmetric +##xide +174 +encore +passports +wildcats +##uni +baja +##pac +mildly +##ease +bleed +commodity +mounds +glossy +orchestras +##omo +damian +prelude +ambitions +##vet +awhile +remotely +##aud +asserts +imply +##iques +distinctly +modelling +remedy +##dded +windshield +dani +xiao +##endra +audible +powerplant +1300 +invalid +elemental +acquisitions +##hala +immaculate +libby +plata +smuggling +ventilation +denoted +minh +##morphism +430 +differed +dion +kelley +lore +mocking +sabbath +spikes +hygiene +drown +runoff +stylized +tally +liberated +aux +interpreter +righteous +aba +siren +reaper +pearce +millie +##cier +##yra +gaius +##iso +captures +##ttering +dorm +claudio +##sic +benches +knighted +blackness +##ored +discount +fumble +oxidation +routed +##ς +novak +perpendicular +spoiled +fracture +splits +##urt +pads +topology +##cats +axes +fortunate +offenders +protestants +esteem +221 +broadband +convened +frankly +hound +prototypes +isil +facilitated +keel +##sher +sahara +awaited +bubba +orb +prosecutors +186 +hem +520 +##xing +relaxing +remnant +romney +sorted +slalom +stefano +ulrich +##active +exemption +folder +pauses +foliage +hitchcock +epithet +204 +criticisms +##aca +ballistic +brody +hinduism +chaotic +youths +equals +##pala +pts +thicker +analogous +capitalist +improvised +overseeing +sinatra +ascended +beverage +##tl +straightforward +##kon +curran +##west +bois +325 +induce +surveying +emperors +sax +unpopular +##kk +cartoonist +fused +##mble +unto +##yuki +localities +##cko +##ln +darlington +slain +academie +lobbying +sediment +puzzles +##grass +defiance +dickens +manifest +tongues +alumnus +arbor +coincide +184 +appalachian +mustafa +examiner +cabaret +traumatic +yves +bracelet +draining +heroin +magnum +baths +odessa +consonants +mitsubishi +##gua +kellan +vaudeville +##fr +joked +null +straps +probation +##ław +ceded +interfaces +##pas +##zawa +blinding +viet +224 +rothschild +museo +640 +huddersfield +##vr +tactic +##storm +brackets +dazed +incorrectly +##vu +reg +glazed +fearful +manifold +benefited +irony +##sun +stumbling +##rte +willingness +balkans +mei +wraps +##aba +injected +##lea +gu +syed +harmless +##hammer +bray +takeoff +poppy +timor +cardboard +astronaut +purdue +weeping +southbound +cursing +stalls +diagonal +##neer +lamar +bryce +comte +weekdays +harrington +##uba +negatively +##see +lays +grouping +##cken +##henko +affirmed +halle +modernist +##lai +hodges +smelling +aristocratic +baptized +dismiss +justification +oilers +##now +coupling +qin +snack +healer +##qing +gardener +layla +battled +formulated +stephenson +gravitational +##gill +##jun +1768 +granny +coordinating +suites +##cd +##ioned +monarchs +##cote +##hips +sep +blended +apr +barrister +deposition +fia +mina +policemen +paranoid +##pressed +churchyard +covert +crumpled +creep +abandoning +tr +transmit +conceal +barr +understands +readiness +spire +##cology +##enia +##erry +610 +startling +unlock +vida +bowled +slots +##nat +##islav +spaced +trusting +admire +rig +##ink +slack +##70 +mv +207 +casualty +##wei +classmates +##odes +##rar +##rked +amherst +furnished +evolve +foundry +menace +mead +##lein +flu +wesleyan +##kled +monterey +webber +##vos +wil +##mith +##на +bartholomew +justices +restrained +##cke +amenities +191 +mediated +sewage +trenches +ml +mainz +##thus +1800s +##cula +##inski +caine +bonding +213 +converts +spheres +superseded +marianne +crypt +sweaty +ensign +historia +##br +spruce +##post +##ask +forks +thoughtfully +yukon +pamphlet +ames +##uter +karma +##yya +bryn +negotiation +sighs +incapable +##mbre +##ntial +actresses +taft +##mill +luce +prevailed +##amine +1773 +motionless +envoy +testify +investing +sculpted +instructors +provence +kali +cullen +horseback +##while +goodwin +##jos +gaa +norte +##ldon +modify +wavelength +abd +214 +skinned +sprinter +forecast +scheduling +marries +squared +tentative +##chman +boer +##isch +bolts +swap +fisherman +assyrian +impatiently +guthrie +martins +murdoch +194 +tanya +nicely +dolly +lacy +med +##45 +syn +decks +fashionable +millionaire +##ust +surfing +##ml +##ision +heaved +tammy +consulate +attendees +routinely +197 +fuse +saxophonist +backseat +malaya +##lord +scowl +tau +##ishly +193 +sighted +steaming +##rks +303 +911 +##holes +##hong +ching +##wife +bless +conserved +jurassic +stacey +unix +zion +chunk +rigorous +blaine +198 +peabody +slayer +dismay +brewers +nz +##jer +det +##glia +glover +postwar +int +penetration +sylvester +imitation +vertically +airlift +heiress +knoxville +viva +##uin +390 +macon +##rim +##fighter +##gonal +janice +##orescence +##wari +marius +belongings +leicestershire +196 +blanco +inverted +preseason +sanity +sobbing +##due +##elt +##dled +collingwood +regeneration +flickering +shortest +##mount +##osi +feminism +##lat +sherlock +cabinets +fumbled +northbound +precedent +snaps +##mme +researching +##akes +guillaume +insights +manipulated +vapor +neighbour +sap +gangster +frey +f1 +stalking +scarcely +callie +barnett +tendencies +audi +doomed +assessing +slung +panchayat +ambiguous +bartlett +##etto +distributing +violating +wolverhampton +##hetic +swami +histoire +##urus +liable +pounder +groin +hussain +larsen +popping +surprises +##atter +vie +curt +##station +mute +relocate +musicals +authorization +richter +##sef +immortality +tna +bombings +##press +deteriorated +yiddish +##acious +robbed +colchester +cs +pmid +ao +verified +balancing +apostle +swayed +recognizable +oxfordshire +retention +nottinghamshire +contender +judd +invitational +shrimp +uhf +##icient +cleaner +longitudinal +tanker +##mur +acronym +broker +koppen +sundance +suppliers +##gil +4000 +clipped +fuels +petite +##anne +landslide +helene +diversion +populous +landowners +auspices +melville +quantitative +##xes +ferries +nicky +##llus +doo +haunting +roche +carver +downed +unavailable +##pathy +approximation +hiroshima +##hue +garfield +valle +comparatively +keyboardist +traveler +##eit +congestion +calculating +subsidiaries +##bate +serb +modernization +fairies +deepened +ville +averages +##lore +inflammatory +tonga +##itch +co₂ +squads +##hea +gigantic +serum +enjoyment +retailer +verona +35th +cis +##phobic +magna +technicians +##vati +arithmetic +##sport +levin +##dation +amtrak +chow +sienna +##eyer +backstage +entrepreneurship +##otic +learnt +tao +##udy +worcestershire +formulation +baggage +hesitant +bali +sabotage +##kari +barren +enhancing +murmur +pl +freshly +putnam +syntax +aces +medicines +resentment +bandwidth +##sier +grins +chili +guido +##sei +framing +implying +gareth +lissa +genevieve +pertaining +admissions +geo +thorpe +proliferation +sato +bela +analyzing +parting +##gor +awakened +##isman +huddled +secrecy +##kling +hush +gentry +540 +dungeons +##ego +coasts +##utz +sacrificed +##chule +landowner +mutually +prevalence +programmer +adolescent +disrupted +seaside +gee +trusts +vamp +georgie +##nesian +##iol +schedules +sindh +##market +etched +hm +sparse +bey +beaux +scratching +gliding +unidentified +216 +collaborating +gems +jesuits +oro +accumulation +shaping +mbe +anal +##xin +231 +enthusiasts +newscast +##egan +janata +dewey +parkinson +179 +ankara +biennial +towering +dd +inconsistent +950 +##chet +thriving +terminate +cabins +furiously +eats +advocating +donkey +marley +muster +phyllis +leiden +##user +grassland +glittering +iucn +loneliness +217 +memorandum +armenians +##ddle +popularized +rhodesia +60s +lame +##illon +sans +bikini +header +orbits +##xx +##finger +##ulator +sharif +spines +biotechnology +strolled +naughty +yates +##wire +fremantle +milo +##mour +abducted +removes +##atin +humming +wonderland +##chrome +##ester +hume +pivotal +##rates +armand +grams +believers +elector +rte +apron +bis +scraped +##yria +endorsement +initials +##llation +eps +dotted +hints +buzzing +emigration +nearer +##tom +indicators +##ulu +coarse +neutron +protectorate +##uze +directional +exploits +pains +loire +1830s +proponents +guggenheim +rabbits +ritchie +305 +hectare +inputs +hutton +##raz +verify +##ako +boilers +longitude +##lev +skeletal +yer +emilia +citrus +compromised +##gau +pokemon +prescription +paragraph +eduard +cadillac +attire +categorized +kenyan +weddings +charley +##bourg +entertain +monmouth +##lles +nutrients +davey +mesh +incentive +practised +ecosystems +kemp +subdued +overheard +##rya +bodily +maxim +##nius +apprenticeship +ursula +##fight +lodged +rug +silesian +unconstitutional +patel +inspected +coyote +unbeaten +##hak +34th +disruption +convict +parcel +##cl +##nham +collier +implicated +mallory +##iac +##lab +susannah +winkler +##rber +shia +phelps +sediments +graphical +robotic +##sner +adulthood +mart +smoked +##isto +kathryn +clarified +##aran +divides +convictions +oppression +pausing +burying +##mt +federico +mathias +eileen +##tana +kite +hunched +##acies +189 +##atz +disadvantage +liza +kinetic +greedy +paradox +yokohama +dowager +trunks +ventured +##gement +gupta +vilnius +olaf +##thest +crimean +hopper +##ej +progressively +arturo +mouthed +arrondissement +##fusion +rubin +simulcast +oceania +##orum +##stra +##rred +busiest +intensely +navigator +cary +##vine +##hini +##bies +fife +rowe +rowland +posing +insurgents +shafts +lawsuits +activate +conor +inward +culturally +garlic +265 +##eering +eclectic +##hui +##kee +##nl +furrowed +vargas +meteorological +rendezvous +##aus +culinary +commencement +##dition +quota +##notes +mommy +salaries +overlapping +mule +##iology +##mology +sums +wentworth +##isk +##zione +mainline +subgroup +##illy +hack +plaintiff +verdi +bulb +differentiation +engagements +multinational +supplemented +bertrand +caller +regis +##naire +##sler +##arts +##imated +blossom +propagation +kilometer +viaduct +vineyards +##uate +beckett +optimization +golfer +songwriters +seminal +semitic +thud +volatile +evolving +ridley +##wley +trivial +distributions +scandinavia +jiang +##ject +wrestled +insistence +##dio +emphasizes +napkin +##ods +adjunct +rhyme +##ricted +##eti +hopeless +surrounds +tremble +32nd +smoky +##ntly +oils +medicinal +padded +steer +wilkes +219 +255 +concessions +hue +uniquely +blinded +landon +yahoo +##lane +hendrix +commemorating +dex +specify +chicks +##ggio +intercity +1400 +morley +##torm +highlighting +##oting +pang +oblique +stalled +##liner +flirting +newborn +1769 +bishopric +shaved +232 +currie +##ush +dharma +spartan +##ooped +favorites +smug +novella +sirens +abusive +creations +espana +##lage +paradigm +semiconductor +sheen +##rdo +##yen +##zak +nrl +renew +##pose +##tur +adjutant +marches +norma +##enity +ineffective +weimar +grunt +##gat +lordship +plotting +expenditure +infringement +lbs +refrain +av +mimi +mistakenly +postmaster +1771 +##bara +ras +motorsports +tito +199 +subjective +##zza +bully +stew +##kaya +prescott +1a +##raphic +##zam +bids +styling +paranormal +reeve +sneaking +exploding +katz +akbar +migrant +syllables +indefinitely +##ogical +destroys +replaces +applause +##phine +pest +##fide +218 +articulated +bertie +##thing +##cars +##ptic +courtroom +crowley +aesthetics +cummings +tehsil +hormones +titanic +dangerously +##ibe +stadion +jaenelle +auguste +ciudad +##chu +mysore +partisans +##sio +lucan +philipp +##aly +debating +henley +interiors +##rano +##tious +homecoming +beyonce +usher +henrietta +prepares +weeds +##oman +ely +plucked +##pire +##dable +luxurious +##aq +artifact +password +pasture +juno +maddy +minsk +##dder +##ologies +##rone +assessments +martian +royalist +1765 +examines +##mani +##rge +nino +223 +parry +scooped +relativity +##eli +##uting +##cao +congregational +noisy +traverse +##agawa +strikeouts +nickelodeon +obituary +transylvania +binds +depictions +polk +trolley +##yed +##lard +breeders +##under +dryly +hokkaido +1762 +strengths +stacks +bonaparte +connectivity +neared +prostitutes +stamped +anaheim +gutierrez +sinai +##zzling +bram +fresno +madhya +##86 +proton +##lena +##llum +##phon +reelected +wanda +##anus +##lb +ample +distinguishing +##yler +grasping +sermons +tomato +bland +stimulation +avenues +##eux +spreads +scarlett +fern +pentagon +assert +baird +chesapeake +ir +calmed +distortion +fatalities +##olis +correctional +pricing +##astic +##gina +prom +dammit +ying +collaborate +##chia +welterweight +33rd +pointer +substitution +bonded +umpire +communicating +multitude +paddle +##obe +federally +intimacy +##insky +betray +ssr +##lett +##lean +##lves +##therapy +airbus +##tery +functioned +ud +bearer +biomedical +netflix +##hire +##nca +condom +brink +ik +##nical +macy +##bet +flap +gma +experimented +jelly +lavender +##icles +##ulia +munro +##mian +##tial +rye +##rle +60th +gigs +hottest +rotated +predictions +fuji +bu +##erence +##omi +barangay +##fulness +##sas +clocks +##rwood +##liness +cereal +roe +wight +decker +uttered +babu +onion +xml +forcibly +##df +petra +sarcasm +hartley +peeled +storytelling +##42 +##xley +##ysis +##ffa +fibre +kiel +auditor +fig +harald +greenville +##berries +geographically +nell +quartz +##athic +cemeteries +##lr +crossings +nah +holloway +reptiles +chun +sichuan +snowy +660 +corrections +##ivo +zheng +ambassadors +blacksmith +fielded +fluids +hardcover +turnover +medications +melvin +academies +##erton +ro +roach +absorbing +spaniards +colton +##founded +outsider +espionage +kelsey +245 +edible +##ulf +dora +establishes +##sham +##tries +contracting +##tania +cinematic +costello +nesting +##uron +connolly +duff +##nology +mma +##mata +fergus +sexes +gi +optics +spectator +woodstock +banning +##hee +##fle +differentiate +outfielder +refinery +226 +312 +gerhard +horde +lair +drastically +##udi +landfall +##cheng +motorsport +odi +##achi +predominant +quay +skins +##ental +edna +harshly +complementary +murdering +##aves +wreckage +##90 +ono +outstretched +lennox +munitions +galen +reconcile +470 +scalp +bicycles +gillespie +questionable +rosenberg +guillermo +hostel +jarvis +kabul +volvo +opium +yd +##twined +abuses +decca +outpost +##cino +sensible +neutrality +##64 +ponce +anchorage +atkins +turrets +inadvertently +disagree +libre +vodka +reassuring +weighs +##yal +glide +jumper +ceilings +repertory +outs +stain +##bial +envy +##ucible +smashing +heightened +policing +hyun +mixes +lai +prima +##ples +celeste +##bina +lucrative +intervened +kc +manually +##rned +stature +staffed +bun +bastards +nairobi +priced +##auer +thatcher +##kia +tripped +comune +##ogan +##pled +brasil +incentives +emanuel +hereford +musica +##kim +benedictine +biennale +##lani +eureka +gardiner +rb +knocks +sha +##ael +##elled +##onate +efficacy +ventura +masonic +sanford +maize +leverage +##feit +capacities +santana +##aur +novelty +vanilla +##cter +##tour +benin +##oir +##rain +neptune +drafting +tallinn +##cable +humiliation +##boarding +schleswig +fabian +bernardo +liturgy +spectacle +sweeney +pont +routledge +##tment +cosmos +ut +hilt +sleek +universally +##eville +##gawa +typed +##dry +favors +allegheny +glaciers +##rly +recalling +aziz +##log +parasite +requiem +auf +##berto +##llin +illumination +##breaker +##issa +festivities +bows +govern +vibe +vp +333 +sprawled +larson +pilgrim +bwf +leaping +##rts +##ssel +alexei +greyhound +hoarse +##dler +##oration +seneca +##cule +gaping +##ulously +##pura +cinnamon +##gens +##rricular +craven +fantasies +houghton +engined +reigned +dictator +supervising +##oris +bogota +commentaries +unnatural +fingernails +spirituality +tighten +##tm +canadiens +protesting +intentional +cheers +sparta +##ytic +##iere +##zine +widen +belgarath +controllers +dodd +iaaf +navarre +##ication +defect +squire +steiner +whisky +##mins +560 +inevitably +tome +##gold +chew +##uid +##lid +elastic +##aby +streaked +alliances +jailed +regal +##ined +##phy +czechoslovak +narration +absently +##uld +bluegrass +guangdong +quran +criticizing +hose +hari +##liest +##owa +skier +streaks +deploy +##lom +raft +bose +dialed +huff +##eira +haifa +simplest +bursting +endings +ib +sultanate +##titled +franks +whitman +ensures +sven +##ggs +collaborators +forster +organising +ui +banished +napier +injustice +teller +layered +thump +##otti +roc +battleships +evidenced +fugitive +sadie +robotics +##roud +equatorial +geologist +##iza +yielding +##bron +##sr +internationale +mecca +##diment +sbs +skyline +toad +uploaded +reflective +undrafted +lal +leafs +bayern +##dai +lakshmi +shortlisted +##stick +##wicz +camouflage +donate +af +christi +lau +##acio +disclosed +nemesis +1761 +assemble +straining +northamptonshire +tal +##asi +bernardino +premature +heidi +42nd +coefficients +galactic +reproduce +buzzed +sensations +zionist +monsieur +myrtle +##eme +archery +strangled +musically +viewpoint +antiquities +bei +trailers +seahawks +cured +pee +preferring +tasmanian +lange +sul +##mail +##working +colder +overland +lucivar +massey +gatherings +haitian +##smith +disapproval +flaws +##cco +##enbach +1766 +npr +##icular +boroughs +creole +forums +techno +1755 +dent +abdominal +streetcar +##eson +##stream +procurement +gemini +predictable +##tya +acheron +christoph +feeder +fronts +vendor +bernhard +jammu +tumors +slang +##uber +goaltender +twists +curving +manson +vuelta +mer +peanut +confessions +pouch +unpredictable +allowance +theodor +vascular +##factory +bala +authenticity +metabolic +coughing +nanjing +##cea +pembroke +##bard +splendid +36th +ff +hourly +##ahu +elmer +handel +##ivate +awarding +thrusting +dl +experimentation +##hesion +##46 +caressed +entertained +steak +##rangle +biologist +orphans +baroness +oyster +stepfather +##dridge +mirage +reefs +speeding +##31 +barons +1764 +227 +inhabit +preached +repealed +##tral +honoring +boogie +captives +administer +johanna +##imate +gel +suspiciously +1767 +sobs +##dington +backbone +hayward +garry +##folding +##nesia +maxi +##oof +##ppe +ellison +galileo +##stand +crimea +frenzy +amour +bumper +matrices +natalia +baking +garth +palestinians +##grove +smack +conveyed +ensembles +gardening +##manship +##rup +##stituting +1640 +harvesting +topography +jing +shifters +dormitory +##carriage +##lston +ist +skulls +##stadt +dolores +jewellery +sarawak +##wai +##zier +fences +christy +confinement +tumbling +credibility +fir +stench +##bria +##plication +##nged +##sam +virtues +##belt +marjorie +pba +##eem +##made +celebrates +schooner +agitated +barley +fulfilling +anthropologist +##pro +restrict +novi +regulating +##nent +padres +##rani +##hesive +loyola +tabitha +milky +olson +proprietor +crambidae +guarantees +intercollegiate +ljubljana +hilda +##sko +ignorant +hooded +##lts +sardinia +##lidae +##vation +frontman +privileged +witchcraft +##gp +jammed +laude +poking +##than +bracket +amazement +yunnan +##erus +maharaja +linnaeus +264 +commissioning +milano +peacefully +##logies +akira +rani +regulator +##36 +grasses +##rance +luzon +crows +compiler +gretchen +seaman +edouard +tab +buccaneers +ellington +hamlets +whig +socialists +##anto +directorial +easton +mythological +##kr +##vary +rhineland +semantic +taut +dune +inventions +succeeds +##iter +replication +branched +##pired +jul +prosecuted +kangaroo +penetrated +##avian +middlesbrough +doses +bleak +madam +predatory +relentless +##vili +reluctance +##vir +hailey +crore +silvery +1759 +monstrous +swimmers +transmissions +hawthorn +informing +##eral +toilets +caracas +crouch +kb +##sett +295 +cartel +hadley +##aling +alexia +yvonne +##biology +cinderella +eton +superb +blizzard +stabbing +industrialist +maximus +##gm +##orus +groves +maud +clade +oversized +comedic +##bella +rosen +nomadic +fulham +montane +beverages +galaxies +redundant +swarm +##rot +##folia +##llis +buckinghamshire +fen +bearings +bahadur +##rom +gilles +phased +dynamite +faber +benoit +vip +##ount +##wd +booking +fractured +tailored +anya +spices +westwood +cairns +auditions +inflammation +steamed +##rocity +##acion +##urne +skyla +thereof +watford +torment +archdeacon +transforms +lulu +demeanor +fucked +serge +##sor +mckenna +minas +entertainer +##icide +caress +originate +residue +##sty +1740 +##ilised +##org +beech +##wana +subsidies +##ghton +emptied +gladstone +ru +firefighters +voodoo +##rcle +het +nightingale +tamara +edmond +ingredient +weaknesses +silhouette +285 +compatibility +withdrawing +hampson +##mona +anguish +giggling +##mber +bookstore +##jiang +southernmost +tilting +##vance +bai +economical +rf +briefcase +dreadful +hinted +projections +shattering +totaling +##rogate +analogue +indicted +periodical +fullback +##dman +haynes +##tenberg +##ffs +##ishment +1745 +thirst +stumble +penang +vigorous +##ddling +##kor +##lium +octave +##ove +##enstein +##inen +##ones +siberian +##uti +cbn +repeal +swaying +##vington +khalid +tanaka +unicorn +otago +plastered +lobe +riddle +##rella +perch +##ishing +croydon +filtered +graeme +tripoli +##ossa +crocodile +##chers +sufi +mined +##tung +inferno +lsu +##phi +swelled +utilizes +£2 +cale +periodicals +styx +hike +informally +coop +lund +##tidae +ala +hen +qui +transformations +disposed +sheath +chickens +##cade +fitzroy +sas +silesia +unacceptable +odisha +1650 +sabrina +pe +spokane +ratios +athena +massage +shen +dilemma +##drum +##riz +##hul +corona +doubtful +niall +##pha +##bino +fines +cite +acknowledging +bangor +ballard +bathurst +##resh +huron +mustered +alzheimer +garments +kinase +tyre +warship +##cp +flashback +pulmonary +braun +cheat +kamal +cyclists +constructions +grenades +ndp +traveller +excuses +stomped +signalling +trimmed +futsal +mosques +relevance +##wine +wta +##23 +##vah +##lter +hoc +##riding +optimistic +##´s +deco +sim +interacting +rejecting +moniker +waterways +##ieri +##oku +mayors +gdansk +outnumbered +pearls +##ended +##hampton +fairs +totals +dominating +262 +notions +stairway +compiling +pursed +commodities +grease +yeast +##jong +carthage +griffiths +residual +amc +contraction +laird +sapphire +##marine +##ivated +amalgamation +dissolve +inclination +lyle +packaged +altitudes +suez +canons +graded +lurched +narrowing +boasts +guise +wed +enrico +##ovsky +rower +scarred +bree +cub +iberian +protagonists +bargaining +proposing +trainers +voyages +vans +fishes +##aea +##ivist +##verance +encryption +artworks +kazan +sabre +cleopatra +hepburn +rotting +supremacy +mecklenburg +##brate +burrows +hazards +outgoing +flair +organizes +##ctions +scorpion +##usions +boo +234 +chevalier +dunedin +slapping +##34 +ineligible +pensions +##38 +##omic +manufactures +emails +bismarck +238 +weakening +blackish +ding +mcgee +quo +##rling +northernmost +xx +manpower +greed +sampson +clicking +##ange +##horpe +##inations +##roving +torre +##eptive +##moral +symbolism +38th +asshole +meritorious +outfits +splashed +biographies +sprung +astros +##tale +302 +737 +filly +raoul +nw +tokugawa +linden +clubhouse +##apa +tracts +romano +##pio +putin +tags +##note +chained +dickson +gunshot +moe +gunn +rashid +##tails +zipper +##bas +##nea +contrasted +##ply +##udes +plum +pharaoh +##pile +aw +comedies +ingrid +sandwiches +subdivisions +1100 +mariana +nokia +kamen +hz +delaney +veto +herring +##words +possessive +outlines +##roup +siemens +stairwell +rc +gallantry +messiah +palais +yells +233 +zeppelin +##dm +bolivar +##cede +smackdown +mckinley +##mora +##yt +muted +geologic +finely +unitary +avatar +hamas +maynard +rees +bog +contrasting +##rut +liv +chico +disposition +pixel +##erate +becca +dmitry +yeshiva +narratives +##lva +##ulton +mercenary +sharpe +tempered +navigate +stealth +amassed +keynes +##lini +untouched +##rrie +havoc +lithium +##fighting +abyss +graf +southward +wolverine +balloons +implements +ngos +transitions +##icum +ambushed +concacaf +dormant +economists +##dim +costing +csi +rana +universite +boulders +verity +##llon +collin +mellon +misses +cypress +fluorescent +lifeless +spence +##ulla +crewe +shepard +pak +revelations +##م +jolly +gibbons +paw +##dro +##quel +freeing +##test +shack +fries +palatine +##51 +##hiko +accompaniment +cruising +recycled +##aver +erwin +sorting +synthesizers +dyke +realities +sg +strides +enslaved +wetland +##ghan +competence +gunpowder +grassy +maroon +reactors +objection +##oms +carlson +gearbox +macintosh +radios +shelton +##sho +clergyman +prakash +254 +mongols +trophies +oricon +228 +stimuli +twenty20 +cantonese +cortes +mirrored +##saurus +bhp +cristina +melancholy +##lating +enjoyable +nuevo +##wny +downfall +schumacher +##ind +banging +lausanne +rumbled +paramilitary +reflex +ax +amplitude +migratory +##gall +##ups +midi +barnard +lastly +sherry +##hp +##nall +keystone +##kra +carleton +slippery +##53 +coloring +foe +socket +otter +##rgos +mats +##tose +consultants +bafta +bison +topping +##km +490 +primal +abandonment +transplant +atoll +hideous +mort +pained +reproduced +tae +howling +##turn +unlawful +billionaire +hotter +poised +lansing +##chang +dinamo +retro +messing +nfc +domesday +##mina +blitz +timed +##athing +##kley +ascending +gesturing +##izations +signaled +tis +chinatown +mermaid +savanna +jameson +##aint +catalina +##pet +##hers +cochrane +cy +chatting +##kus +alerted +computation +mused +noelle +majestic +mohawk +campo +octagonal +##sant +##hend +241 +aspiring +##mart +comprehend +iona +paralyzed +shimmering +swindon +rhone +##eley +reputed +configurations +pitchfork +agitation +francais +gillian +lipstick +##ilo +outsiders +pontifical +resisting +bitterness +sewer +rockies +##edd +##ucher +misleading +1756 +exiting +galloway +##nging +risked +##heart +246 +commemoration +schultz +##rka +integrating +##rsa +poses +shrieked +##weiler +guineas +gladys +jerking +owls +goldsmith +nightly +penetrating +##unced +lia +##33 +ignited +betsy +##aring +##thorpe +follower +vigorously +##rave +coded +kiran +knit +zoology +tbilisi +##28 +##bered +repository +govt +deciduous +dino +growling +##bba +enhancement +unleashed +chanting +pussy +biochemistry +##eric +kettle +repression +toxicity +nrhp +##arth +##kko +##bush +ernesto +commended +outspoken +242 +mca +parchment +sms +kristen +##aton +bisexual +raked +glamour +navajo +a2 +conditioned +showcased +##hma +spacious +youthful +##esa +usl +appliances +junta +brest +layne +conglomerate +enchanted +chao +loosened +picasso +circulating +inspect +montevideo +##centric +##kti +piazza +spurred +##aith +bari +freedoms +poultry +stamford +lieu +##ect +indigo +sarcastic +bahia +stump +attach +dvds +frankenstein +lille +approx +scriptures +pollen +##script +nmi +overseen +##ivism +tides +proponent +newmarket +inherit +milling +##erland +centralized +##rou +distributors +credentials +drawers +abbreviation +##lco +##xon +downing +uncomfortably +ripe +##oes +erase +franchises +##ever +populace +##bery +##khar +decomposition +pleas +##tet +daryl +sabah +##stle +##wide +fearless +genie +lesions +annette +##ogist +oboe +appendix +nair +dripped +petitioned +maclean +mosquito +parrot +rpg +hampered +1648 +operatic +reservoirs +##tham +irrelevant +jolt +summarized +##fp +medallion +##taff +##− +clawed +harlow +narrower +goddard +marcia +bodied +fremont +suarez +altering +tempest +mussolini +porn +##isms +sweetly +oversees +walkers +solitude +grimly +shrines +hk +ich +supervisors +hostess +dietrich +legitimacy +brushes +expressive +##yp +dissipated +##rse +localized +systemic +##nikov +gettysburg +##js +##uaries +dialogues +muttering +251 +housekeeper +sicilian +discouraged +##frey +beamed +kaladin +halftime +kidnap +##amo +##llet +1754 +synonymous +depleted +instituto +insulin +reprised +##opsis +clashed +##ctric +interrupting +radcliffe +insisting +medici +1715 +ejected +playfully +turbulent +##47 +starvation +##rini +shipment +rebellious +petersen +verification +merits +##rified +cakes +##charged +1757 +milford +shortages +spying +fidelity +##aker +emitted +storylines +harvested +seismic +##iform +cheung +kilda +theoretically +barbie +lynx +##rgy +##tius +goblin +mata +poisonous +##nburg +reactive +residues +obedience +##евич +conjecture +##rac +401 +hating +sixties +kicker +moaning +motown +##bha +emancipation +neoclassical +##hering +consoles +ebert +professorship +##tures +sustaining +assaults +obeyed +affluent +incurred +tornadoes +##eber +##zow +emphasizing +highlanders +cheated +helmets +##ctus +internship +terence +bony +executions +legislators +berries +peninsular +tinged +##aco +1689 +amplifier +corvette +ribbons +lavish +pennant +##lander +worthless +##chfield +##forms +mariano +pyrenees +expenditures +##icides +chesterfield +mandir +tailor +39th +sergey +nestled +willed +aristocracy +devotees +goodnight +raaf +rumored +weaponry +remy +appropriations +harcourt +burr +riaa +##lence +limitation +unnoticed +guo +soaking +swamps +##tica +collapsing +tatiana +descriptive +brigham +psalm +##chment +maddox +##lization +patti +caliph +##aja +akron +injuring +serra +##ganj +basins +##sari +astonished +launcher +##church +hilary +wilkins +sewing +##sf +stinging +##fia +##ncia +underwood +startup +##ition +compilations +vibrations +embankment +jurist +##nity +bard +juventus +groundwater +kern +palaces +helium +boca +cramped +marissa +soto +##worm +jae +princely +##ggy +faso +bazaar +warmly +##voking +229 +pairing +##lite +##grate +##nets +wien +freaked +ulysses +rebirth +##alia +##rent +mummy +guzman +jimenez +stilled +##nitz +trajectory +tha +woken +archival +professions +##pts +##pta +hilly +shadowy +shrink +##bolt +norwood +glued +migrate +stereotypes +devoid +##pheus +625 +evacuate +horrors +infancy +gotham +knowles +optic +downloaded +sachs +kingsley +parramatta +darryl +mor +##onale +shady +commence +confesses +kan +##meter +##placed +marlborough +roundabout +regents +frigates +io +##imating +gothenburg +revoked +carvings +clockwise +convertible +intruder +##sche +banged +##ogo +vicky +bourgeois +##mony +dupont +footing +##gum +pd +##real +buckle +yun +penthouse +sane +720 +serviced +stakeholders +neumann +bb +##eers +comb +##gam +catchment +pinning +rallies +typing +##elles +forefront +freiburg +sweetie +giacomo +widowed +goodwill +worshipped +aspirations +midday +##vat +fishery +##trick +bournemouth +turk +243 +hearth +ethanol +guadalajara +murmurs +sl +##uge +afforded +scripted +##hta +wah +##jn +coroner +translucent +252 +memorials +puck +progresses +clumsy +##race +315 +candace +recounted +##27 +##slin +##uve +filtering +##mac +howl +strata +heron +leveled +##ays +dubious +##oja +##т +##wheel +citations +exhibiting +##laya +##mics +##pods +turkic +##lberg +injunction +##ennial +##mit +antibodies +##44 +organise +##rigues +cardiovascular +cushion +inverness +##zquez +dia +cocoa +sibling +##tman +##roid +expanse +feasible +tunisian +algiers +##relli +rus +bloomberg +dso +westphalia +bro +tacoma +281 +downloads +##ours +konrad +duran +##hdi +continuum +jett +compares +legislator +secession +##nable +##gues +##zuka +translating +reacher +##gley +##ła +aleppo +##agi +tc +orchards +trapping +linguist +versatile +drumming +postage +calhoun +superiors +##mx +barefoot +leary +##cis +ignacio +alfa +kaplan +##rogen +bratislava +mori +##vot +disturb +haas +313 +cartridges +gilmore +radiated +salford +tunic +hades +##ulsive +archeological +delilah +magistrates +auditioned +brewster +charters +empowerment +blogs +cappella +dynasties +iroquois +whipping +##krishna +raceway +truths +myra +weaken +judah +mcgregor +##horse +mic +refueling +37th +burnley +bosses +markus +premio +query +##gga +dunbar +##economic +darkest +lyndon +sealing +commendation +reappeared +##mun +addicted +ezio +slaughtered +satisfactory +shuffle +##eves +##thic +##uj +fortification +warrington +##otto +resurrected +fargo +mane +##utable +##lei +##space +foreword +ox +##aris +##vern +abrams +hua +##mento +sakura +##alo +uv +sentimental +##skaya +midfield +##eses +sturdy +scrolls +macleod +##kyu +entropy +##lance +mitochondrial +cicero +excelled +thinner +convoys +perceive +##oslav +##urable +systematically +grind +burkina +287 +##tagram +ops +##aman +guantanamo +##cloth +##tite +forcefully +wavy +##jou +pointless +##linger +##tze +layton +portico +superficial +clerical +outlaws +##hism +burials +muir +##inn +creditors +hauling +rattle +##leg +calais +monde +archers +reclaimed +dwell +wexford +hellenic +falsely +remorse +##tek +dough +furnishings +##uttered +gabon +neurological +novice +##igraphy +contemplated +pulpit +nightstand +saratoga +##istan +documenting +pulsing +taluk +##firmed +busted +marital +##rien +disagreements +wasps +##yes +hodge +mcdonnell +mimic +fran +pendant +dhabi +musa +##nington +congratulations +argent +darrell +concussion +losers +regrets +thessaloniki +reversal +donaldson +hardwood +thence +achilles +ritter +##eran +demonic +jurgen +prophets +goethe +eki +classmate +buff +##cking +yank +irrational +##inging +perished +seductive +qur +sourced +##crat +##typic +mustard +ravine +barre +horizontally +characterization +phylogenetic +boise +##dit +##runner +##tower +brutally +intercourse +seduce +##bbing +fay +ferris +ogden +amar +nik +unarmed +##inator +evaluating +kyrgyzstan +sweetness +##lford +##oki +mccormick +meiji +notoriety +stimulate +disrupt +figuring +instructional +mcgrath +##zoo +groundbreaking +##lto +flinch +khorasan +agrarian +bengals +mixer +radiating +##sov +ingram +pitchers +nad +tariff +##cript +tata +##codes +##emi +##ungen +appellate +lehigh +##bled +##giri +brawl +duct +texans +##ciation +##ropolis +skipper +speculative +vomit +doctrines +stresses +253 +davy +graders +whitehead +jozef +timely +cumulative +haryana +paints +appropriately +boon +cactus +##ales +##pid +dow +legions +##pit +perceptions +1730 +picturesque +##yse +periphery +rune +wr +##aha +celtics +sentencing +whoa +##erin +confirms +variance +425 +moines +mathews +spade +rave +m1 +fronted +fx +blending +alleging +reared +##gl +237 +##paper +grassroots +eroded +##free +##physical +directs +ordeal +##sław +accelerate +hacker +rooftop +##inia +lev +buys +cebu +devote +##lce +specialising +##ulsion +choreographed +repetition +warehouses +##ryl +paisley +tuscany +analogy +sorcerer +hash +huts +shards +descends +exclude +nix +chaplin +gaga +ito +vane +##drich +causeway +misconduct +limo +orchestrated +glands +jana +##kot +u2 +##mple +##sons +branching +contrasts +scoop +longed +##virus +chattanooga +##75 +syrup +cornerstone +##tized +##mind +##iaceae +careless +precedence +frescoes +##uet +chilled +consult +modelled +snatch +peat +##thermal +caucasian +humane +relaxation +spins +temperance +##lbert +occupations +lambda +hybrids +moons +mp3 +##oese +247 +rolf +societal +yerevan +ness +##ssler +befriended +mechanized +nominate +trough +boasted +cues +seater +##hom +bends +##tangle +conductors +emptiness +##lmer +eurasian +adriatic +tian +##cie +anxiously +lark +propellers +chichester +jock +ev +2a +##holding +credible +recounts +tori +loyalist +abduction +##hoot +##redo +nepali +##mite +ventral +tempting +##ango +##crats +steered +##wice +javelin +dipping +laborers +prentice +looming +titanium +##ː +badges +emir +tensor +##ntation +egyptians +rash +denies +hawthorne +lombard +showers +wehrmacht +dietary +trojan +##reus +welles +executing +horseshoe +lifeboat +##lak +elsa +infirmary +nearing +roberta +boyer +mutter +trillion +joanne +##fine +##oked +sinks +vortex +uruguayan +clasp +sirius +##block +accelerator +prohibit +sunken +byu +chronological +diplomats +ochreous +510 +symmetrical +1644 +maia +##tology +salts +reigns +atrocities +##ия +hess +bared +issn +##vyn +cater +saturated +##cycle +##isse +sable +voyager +dyer +yusuf +##inge +fountains +wolff +##39 +##nni +engraving +rollins +atheist +ominous +##ault +herr +chariot +martina +strung +##fell +##farlane +horrific +sahib +gazes +saetan +erased +ptolemy +##olic +flushing +lauderdale +analytic +##ices +530 +navarro +beak +gorilla +herrera +broom +guadalupe +raiding +sykes +311 +bsc +deliveries +1720 +invasions +carmichael +tajikistan +thematic +ecumenical +sentiments +onstage +##rians +##brand +##sume +catastrophic +flanks +molten +##arns +waller +aimee +terminating +##icing +alternately +##oche +nehru +printers +outraged +##eving +empires +template +banners +repetitive +za +##oise +vegetarian +##tell +guiana +opt +cavendish +lucknow +synthesized +##hani +##mada +finalized +##ctable +fictitious +mayoral +unreliable +##enham +embracing +peppers +rbis +##chio +##neo +inhibition +slashed +togo +orderly +embroidered +safari +salty +236 +barron +benito +totaled +##dak +pubs +simulated +caden +devin +tolkien +momma +welding +sesame +##ept +gottingen +hardness +630 +shaman +temeraire +620 +adequately +pediatric +##kit +ck +assertion +radicals +composure +cadence +seafood +beaufort +lazarus +mani +warily +cunning +kurdistan +249 +cantata +##kir +ares +##41 +##clusive +nape +townland +geared +insulted +flutter +boating +violate +draper +dumping +malmo +##hh +##romatic +firearm +alta +bono +obscured +##clave +exceeds +panorama +unbelievable +##train +preschool +##essed +disconnected +installing +rescuing +secretaries +accessibility +##castle +##drive +##ifice +##film +bouts +slug +waterway +mindanao +##buro +##ratic +halves +##ل +calming +liter +maternity +adorable +bragg +electrification +mcc +##dote +roxy +schizophrenia +##body +munoz +kaye +whaling +239 +mil +tingling +tolerant +##ago +unconventional +volcanoes +##finder +deportivo +##llie +robson +kaufman +neuroscience +wai +deportation +masovian +scraping +converse +##bh +hacking +bulge +##oun +administratively +yao +580 +amp +mammoth +booster +claremont +hooper +nomenclature +pursuits +mclaughlin +melinda +##sul +catfish +barclay +substrates +taxa +zee +originals +kimberly +packets +padma +##ality +borrowing +ostensibly +solvent +##bri +##genesis +##mist +lukas +shreveport +veracruz +##ь +##lou +##wives +cheney +tt +anatolia +hobbs +##zyn +cyclic +radiant +alistair +greenish +siena +dat +independents +##bation +conform +pieter +hyper +applicant +bradshaw +spores +telangana +vinci +inexpensive +nuclei +322 +jang +nme +soho +spd +##ign +cradled +receptionist +pow +##43 +##rika +fascism +##ifer +experimenting +##ading +##iec +##region +345 +jocelyn +maris +stair +nocturnal +toro +constabulary +elgin +##kker +msc +##giving +##schen +##rase +doherty +doping +sarcastically +batter +maneuvers +##cano +##apple +##gai +##git +intrinsic +##nst +##stor +1753 +showtime +cafes +gasps +lviv +ushered +##thed +fours +restart +astonishment +transmitting +flyer +shrugs +##sau +intriguing +cones +dictated +mushrooms +medial +##kovsky +##elman +escorting +gaped +##26 +godfather +##door +##sell +djs +recaptured +timetable +vila +1710 +3a +aerodrome +mortals +scientology +##orne +angelina +mag +convection +unpaid +insertion +intermittent +lego +##nated +endeavor +kota +pereira +##lz +304 +bwv +glamorgan +insults +agatha +fey +##cend +fleetwood +mahogany +protruding +steamship +zeta +##arty +mcguire +suspense +##sphere +advising +urges +##wala +hurriedly +meteor +gilded +inline +arroyo +stalker +##oge +excitedly +revered +##cure +earle +introductory +##break +##ilde +mutants +puff +pulses +reinforcement +##haling +curses +lizards +stalk +correlated +##fixed +fallout +macquarie +##unas +bearded +denton +heaving +802 +##ocation +winery +assign +dortmund +##lkirk +everest +invariant +charismatic +susie +##elling +bled +lesley +telegram +sumner +bk +##ogen +##к +wilcox +needy +colbert +duval +##iferous +##mbled +allotted +attends +imperative +##hita +replacements +hawker +##inda +insurgency +##zee +##eke +casts +##yla +680 +ives +transitioned +##pack +##powering +authoritative +baylor +flex +cringed +plaintiffs +woodrow +##skie +drastic +ape +aroma +unfolded +commotion +nt +preoccupied +theta +routines +lasers +privatization +wand +domino +ek +clenching +nsa +strategically +showered +bile +handkerchief +pere +storing +christophe +insulting +316 +nakamura +romani +asiatic +magdalena +palma +cruises +stripping +405 +konstantin +soaring +##berman +colloquially +forerunner +havilland +incarcerated +parasites +sincerity +##utus +disks +plank +saigon +##ining +corbin +homo +ornaments +powerhouse +##tlement +chong +fastened +feasibility +idf +morphological +usable +##nish +##zuki +aqueduct +jaguars +keepers +##flies +aleksandr +faust +assigns +ewing +bacterium +hurled +tricky +hungarians +integers +wallis +321 +yamaha +##isha +hushed +oblivion +aviator +evangelist +friars +##eller +monograph +ode +##nary +airplanes +labourers +charms +##nee +1661 +hagen +tnt +rudder +fiesta +transcript +dorothea +ska +inhibitor +maccabi +retorted +raining +encompassed +clauses +menacing +1642 +lineman +##gist +vamps +##ape +##dick +gloom +##rera +dealings +easing +seekers +##nut +##pment +helens +unmanned +##anu +##isson +basics +##amy +##ckman +adjustments +1688 +brutality +horne +##zell +sui +##55 +##mable +aggregator +##thal +rhino +##drick +##vira +counters +zoom +##01 +##rting +mn +montenegrin +packard +##unciation +##♭ +##kki +reclaim +scholastic +thugs +pulsed +##icia +syriac +quan +saddam +banda +kobe +blaming +buddies +dissent +##lusion +##usia +corbett +jaya +delle +erratic +lexie +##hesis +435 +amiga +hermes +##pressing +##leen +chapels +gospels +jamal +##uating +compute +revolving +warp +##sso +##thes +armory +##eras +##gol +antrim +loki +##kow +##asian +##good +##zano +braid +handwriting +subdistrict +funky +pantheon +##iculate +concurrency +estimation +improper +juliana +##his +newcomers +johnstone +staten +communicated +##oco +##alle +sausage +stormy +##stered +##tters +superfamily +##grade +acidic +collateral +tabloid +##oped +##rza +bladder +austen +##ellant +mcgraw +##hay +hannibal +mein +aquino +lucifer +wo +badger +boar +cher +christensen +greenberg +interruption +##kken +jem +244 +mocked +bottoms +cambridgeshire +##lide +sprawling +##bbly +eastwood +ghent +synth +##buck +advisers +##bah +nominally +hapoel +qu +daggers +estranged +fabricated +towels +vinnie +wcw +misunderstanding +anglia +nothin +unmistakable +##dust +##lova +chilly +marquette +truss +##edge +##erine +reece +##lty +##chemist +##connected +272 +308 +41st +bash +raion +waterfalls +##ump +##main +labyrinth +queue +theorist +##istle +bharatiya +flexed +soundtracks +rooney +leftist +patrolling +wharton +plainly +alleviate +eastman +schuster +topographic +engages +immensely +unbearable +fairchild +1620 +dona +lurking +parisian +oliveira +ia +indictment +hahn +bangladeshi +##aster +vivo +##uming +##ential +antonia +expects +indoors +kildare +harlan +##logue +##ogenic +##sities +forgiven +##wat +childish +tavi +##mide +##orra +plausible +grimm +successively +scooted +##bola +##dget +##rith +spartans +emery +flatly +azure +epilogue +##wark +flourish +##iny +##tracted +##overs +##oshi +bestseller +distressed +receipt +spitting +hermit +topological +##cot +drilled +subunit +francs +##layer +eel +##fk +##itas +octopus +footprint +petitions +ufo +##say +##foil +interfering +leaking +palo +##metry +thistle +valiant +##pic +narayan +mcpherson +##fast +gonzales +##ym +##enne +dustin +novgorod +solos +##zman +doin +##raph +##patient +##meyer +soluble +ashland +cuffs +carole +pendleton +whistling +vassal +##river +deviation +revisited +constituents +rallied +rotate +loomed +##eil +##nting +amateurs +augsburg +auschwitz +crowns +skeletons +##cona +bonnet +257 +dummy +globalization +simeon +sleeper +mandal +differentiated +##crow +##mare +milne +bundled +exasperated +talmud +owes +segregated +##feng +##uary +dentist +piracy +props +##rang +devlin +##torium +malicious +paws +##laid +dependency +##ergy +##fers +##enna +258 +pistons +rourke +jed +grammatical +tres +maha +wig +512 +ghostly +jayne +##achal +##creen +##ilis +##lins +##rence +designate +##with +arrogance +cambodian +clones +showdown +throttle +twain +##ception +lobes +metz +nagoya +335 +braking +##furt +385 +roaming +##minster +amin +crippled +##37 +##llary +indifferent +hoffmann +idols +intimidating +1751 +261 +influenza +memo +onions +1748 +bandage +consciously +##landa +##rage +clandestine +observes +swiped +tangle +##ener +##jected +##trum +##bill +##lta +hugs +congresses +josiah +spirited +##dek +humanist +managerial +filmmaking +inmate +rhymes +debuting +grimsby +ur +##laze +duplicate +vigor +##tf +republished +bolshevik +refurbishment +antibiotics +martini +methane +newscasts +royale +horizons +levant +iain +visas +##ischen +paler +##around +manifestation +snuck +alf +chop +futile +pedestal +rehab +##kat +bmg +kerman +res +fairbanks +jarrett +abstraction +saharan +##zek +1746 +procedural +clearer +kincaid +sash +luciano +##ffey +crunch +helmut +##vara +revolutionaries +##tute +creamy +leach +##mmon +1747 +permitting +nes +plight +wendell +##lese +contra +ts +clancy +ipa +mach +staples +autopsy +disturbances +nueva +karin +pontiac +##uding +proxy +venerable +haunt +leto +bergman +expands +##helm +wal +##pipe +canning +celine +cords +obesity +##enary +intrusion +planner +##phate +reasoned +sequencing +307 +harrow +##chon +##dora +marred +mcintyre +repay +tarzan +darting +248 +harrisburg +margarita +repulsed +##hur +##lding +belinda +hamburger +novo +compliant +runways +bingham +registrar +skyscraper +ic +cuthbert +improvisation +livelihood +##corp +##elial +admiring +##dened +sporadic +believer +casablanca +popcorn +##29 +asha +shovel +##bek +##dice +coiled +tangible +##dez +casper +elsie +resin +tenderness +rectory +##ivision +avail +sonar +##mori +boutique +##dier +guerre +bathed +upbringing +vaulted +sandals +blessings +##naut +##utnant +1680 +306 +foxes +pia +corrosion +hesitantly +confederates +crystalline +footprints +shapiro +tirana +valentin +drones +45th +microscope +shipments +texted +inquisition +wry +guernsey +unauthorized +resigning +760 +ripple +schubert +stu +reassure +felony +##ardo +brittle +koreans +##havan +##ives +dun +implicit +tyres +##aldi +##lth +magnolia +##ehan +##puri +##poulos +aggressively +fei +gr +familiarity +##poo +indicative +##trust +fundamentally +jimmie +overrun +395 +anchors +moans +##opus +britannia +armagh +##ggle +purposely +seizing +##vao +bewildered +mundane +avoidance +cosmopolitan +geometridae +quartermaster +caf +415 +chatter +engulfed +gleam +purge +##icate +juliette +jurisprudence +guerra +revisions +##bn +casimir +brew +##jm +1749 +clapton +cloudy +conde +hermitage +278 +simulations +torches +vincenzo +matteo +##rill +hidalgo +booming +westbound +accomplishment +tentacles +unaffected +##sius +annabelle +flopped +sloping +##litz +dreamer +interceptor +vu +##loh +consecration +copying +messaging +breaker +climates +hospitalized +1752 +torino +afternoons +winfield +witnessing +##teacher +breakers +choirs +sawmill +coldly +##ege +sipping +haste +uninhabited +conical +bibliography +pamphlets +severn +edict +##oca +deux +illnesses +grips +##pl +rehearsals +sis +thinkers +tame +##keepers +1690 +acacia +reformer +##osed +##rys +shuffling +##iring +##shima +eastbound +ionic +rhea +flees +littered +##oum +rocker +vomiting +groaning +champ +overwhelmingly +civilizations +paces +sloop +adoptive +##tish +skaters +##vres +aiding +mango +##joy +nikola +shriek +##ignon +pharmaceuticals +##mg +tuna +calvert +gustavo +stocked +yearbook +##urai +##mana +computed +subsp +riff +hanoi +kelvin +hamid +moors +pastures +summons +jihad +nectar +##ctors +bayou +untitled +pleasing +vastly +republics +intellect +##η +##ulio +##tou +crumbling +stylistic +sb +##ی +consolation +frequented +h₂o +walden +widows +##iens +404 +##ignment +chunks +improves +288 +grit +recited +##dev +snarl +sociological +##arte +##gul +inquired +##held +bruise +clube +consultancy +homogeneous +hornets +multiplication +pasta +prick +savior +##grin +##kou +##phile +yoon +##gara +grimes +vanishing +cheering +reacting +bn +distillery +##quisite +##vity +coe +dockyard +massif +##jord +escorts +voss +##valent +byte +chopped +hawke +illusions +workings +floats +##koto +##vac +kv +annapolis +madden +##onus +alvaro +noctuidae +##cum +##scopic +avenge +steamboat +forte +illustrates +erika +##trip +570 +dew +nationalities +bran +manifested +thirsty +diversified +muscled +reborn +##standing +arson +##lessness +##dran +##logram +##boys +##kushima +##vious +willoughby +##phobia +286 +alsace +dashboard +yuki +##chai +granville +myspace +publicized +tricked +##gang +adjective +##ater +relic +reorganisation +enthusiastically +indications +saxe +##lassified +consolidate +iec +padua +helplessly +ramps +renaming +regulars +pedestrians +accents +convicts +inaccurate +lowers +mana +##pati +barrie +bjp +outta +someplace +berwick +flanking +invoked +marrow +sparsely +excerpts +clothed +rei +##ginal +wept +##straße +##vish +alexa +excel +##ptive +membranes +aquitaine +creeks +cutler +sheppard +implementations +ns +##dur +fragrance +budge +concordia +magnesium +marcelo +##antes +gladly +vibrating +##rral +##ggles +montrose +##omba +lew +seamus +1630 +cocky +##ament +##uen +bjorn +##rrick +fielder +fluttering +##lase +methyl +kimberley +mcdowell +reductions +barbed +##jic +##tonic +aeronautical +condensed +distracting +##promising +huffed +##cala +##sle +claudius +invincible +missy +pious +balthazar +ci +##lang +butte +combo +orson +##dication +myriad +1707 +silenced +##fed +##rh +coco +netball +yourselves +##oza +clarify +heller +peg +durban +etudes +offender +roast +blackmail +curvature +##woods +vile +309 +illicit +suriname +##linson +overture +1685 +bubbling +gymnast +tucking +##mming +##ouin +maldives +##bala +gurney +##dda +##eased +##oides +backside +pinto +jars +racehorse +tending +##rdial +baronetcy +wiener +duly +##rke +barbarian +cupping +flawed +##thesis +bertha +pleistocene +puddle +swearing +##nob +##tically +fleeting +prostate +amulet +educating +##mined +##iti +##tler +75th +jens +respondents +analytics +cavaliers +papacy +raju +##iente +##ulum +##tip +funnel +271 +disneyland +##lley +sociologist +##iam +2500 +faulkner +louvre +menon +##dson +276 +##ower +afterlife +mannheim +peptide +referees +comedians +meaningless +##anger +##laise +fabrics +hurley +renal +sleeps +##bour +##icle +breakout +kristin +roadside +animator +clover +disdain +unsafe +redesign +##urity +firth +barnsley +portage +reset +narrows +268 +commandos +expansive +speechless +tubular +##lux +essendon +eyelashes +smashwords +##yad +##bang +##claim +craved +sprinted +chet +somme +astor +wrocław +orton +266 +bane +##erving +##uing +mischief +##amps +##sund +scaling +terre +##xious +impairment +offenses +undermine +moi +soy +contiguous +arcadia +inuit +seam +##tops +macbeth +rebelled +##icative +##iot +590 +elaborated +frs +uniformed +##dberg +259 +powerless +priscilla +stimulated +980 +qc +arboretum +frustrating +trieste +bullock +##nified +enriched +glistening +intern +##adia +locus +nouvelle +ollie +ike +lash +starboard +ee +tapestry +headlined +hove +rigged +##vite +pollock +##yme +thrive +clustered +cas +roi +gleamed +olympiad +##lino +pressured +regimes +##hosis +##lick +ripley +##ophone +kickoff +gallon +rockwell +##arable +crusader +glue +revolutions +scrambling +1714 +grover +##jure +englishman +aztec +263 +contemplating +coven +ipad +preach +triumphant +tufts +##esian +rotational +##phus +328 +falkland +##brates +strewn +clarissa +rejoin +environmentally +glint +banded +drenched +moat +albanians +johor +rr +maestro +malley +nouveau +shaded +taxonomy +v6 +adhere +bunk +airfields +##ritan +1741 +encompass +remington +tran +##erative +amelie +mazda +friar +morals +passions +##zai +breadth +vis +##hae +argus +burnham +caressing +insider +rudd +##imov +##mini +##rso +italianate +murderous +textual +wainwright +armada +bam +weave +timer +##taken +##nh +fra +##crest +ardent +salazar +taps +tunis +##ntino +allegro +gland +philanthropic +##chester +implication +##optera +esq +judas +noticeably +wynn +##dara +inched +indexed +crises +villiers +bandit +royalties +patterned +cupboard +interspersed +accessory +isla +kendrick +entourage +stitches +##esthesia +headwaters +##ior +interlude +distraught +draught +1727 +##basket +biased +sy +transient +triad +subgenus +adapting +kidd +shortstop +##umatic +dimly +spiked +mcleod +reprint +nellie +pretoria +windmill +##cek +singled +##mps +273 +reunite +##orous +747 +bankers +outlying +##omp +##ports +##tream +apologies +cosmetics +patsy +##deh +##ocks +##yson +bender +nantes +serene +##nad +lucha +mmm +323 +##cius +##gli +cmll +coinage +nestor +juarez +##rook +smeared +sprayed +twitching +sterile +irina +embodied +juveniles +enveloped +miscellaneous +cancers +dq +gulped +luisa +crested +swat +donegal +ref +##anov +##acker +hearst +mercantile +##lika +doorbell +ua +vicki +##alla +##som +bilbao +psychologists +stryker +sw +horsemen +turkmenistan +wits +##national +anson +mathew +screenings +##umb +rihanna +##agne +##nessy +aisles +##iani +##osphere +hines +kenton +saskatoon +tasha +truncated +##champ +##itan +mildred +advises +fredrik +interpreting +inhibitors +##athi +spectroscopy +##hab +##kong +karim +panda +##oia +##nail +##vc +conqueror +kgb +leukemia +##dity +arrivals +cheered +pisa +phosphorus +shielded +##riated +mammal +unitarian +urgently +chopin +sanitary +##mission +spicy +drugged +hinges +##tort +tipping +trier +impoverished +westchester +##caster +267 +epoch +nonstop +##gman +##khov +aromatic +centrally +cerro +##tively +##vio +billions +modulation +sedimentary +283 +facilitating +outrageous +goldstein +##eak +##kt +ld +maitland +penultimate +pollard +##dance +fleets +spaceship +vertebrae +##nig +alcoholism +als +recital +##bham +##ference +##omics +m2 +##bm +trois +##tropical +##в +commemorates +##meric +marge +##raction +1643 +670 +cosmetic +ravaged +##ige +catastrophe +eng +##shida +albrecht +arterial +bellamy +decor +harmon +##rde +bulbs +synchronized +vito +easiest +shetland +shielding +wnba +##glers +##ssar +##riam +brianna +cumbria +##aceous +##rard +cores +thayer +##nsk +brood +hilltop +luminous +carts +keynote +larkin +logos +##cta +##ا +##mund +##quay +lilith +tinted +277 +wrestle +mobilization +##uses +sequential +siam +bloomfield +takahashi +274 +##ieving +presenters +ringo +blazed +witty +##oven +##ignant +devastation +haydn +harmed +newt +therese +##peed +gershwin +molina +rabbis +sudanese +001 +innate +restarted +##sack +##fus +slices +wb +##shah +enroll +hypothetical +hysterical +1743 +fabio +indefinite +warped +##hg +exchanging +525 +unsuitable +##sboro +gallo +1603 +bret +cobalt +homemade +##hunter +mx +operatives +##dhar +terraces +durable +latch +pens +whorls +##ctuated +##eaux +billing +ligament +succumbed +##gly +regulators +spawn +##brick +##stead +filmfare +rochelle +##nzo +1725 +circumstance +saber +supplements +##nsky +##tson +crowe +wellesley +carrot +##9th +##movable +primate +drury +sincerely +topical +##mad +##rao +callahan +kyiv +smarter +tits +undo +##yeh +announcements +anthologies +barrio +nebula +##islaus +##shaft +##tyn +bodyguards +2021 +assassinate +barns +emmett +scully +##mah +##yd +##eland +##tino +##itarian +demoted +gorman +lashed +prized +adventist +writ +##gui +alla +invertebrates +##ausen +1641 +amman +1742 +align +healy +redistribution +##gf +##rize +insulation +##drop +adherents +hezbollah +vitro +ferns +yanking +269 +php +registering +uppsala +cheerleading +confines +mischievous +tully +##ross +49th +docked +roam +stipulated +pumpkin +##bry +prompt +##ezer +blindly +shuddering +craftsmen +frail +scented +katharine +scramble +shaggy +sponge +helix +zaragoza +279 +##52 +43rd +backlash +fontaine +seizures +posse +cowan +nonfiction +telenovela +wwii +hammered +undone +##gpur +encircled +irs +##ivation +artefacts +oneself +searing +smallpox +##belle +##osaurus +shandong +breached +upland +blushing +rankin +infinitely +psyche +tolerated +docking +evicted +##col +unmarked +##lving +gnome +lettering +litres +musique +##oint +benevolent +##jal +blackened +##anna +mccall +racers +tingle +##ocene +##orestation +introductions +radically +292 +##hiff +##باد +1610 +1739 +munchen +plead +##nka +condo +scissors +##sight +##tens +apprehension +##cey +##yin +hallmark +watering +formulas +sequels +##llas +aggravated +bae +commencing +##building +enfield +prohibits +marne +vedic +civilized +euclidean +jagger +beforehand +blasts +dumont +##arney +##nem +740 +conversions +hierarchical +rios +simulator +##dya +##lellan +hedges +oleg +thrusts +shadowed +darby +maximize +1744 +gregorian +##nded +##routed +sham +unspecified +##hog +emory +factual +##smo +##tp +fooled +##rger +ortega +wellness +marlon +##oton +##urance +casket +keating +ley +enclave +##ayan +char +influencing +jia +##chenko +412 +ammonia +erebidae +incompatible +violins +cornered +##arat +grooves +astronauts +columbian +rampant +fabrication +kyushu +mahmud +vanish +##dern +mesopotamia +##lete +ict +##rgen +caspian +kenji +pitted +##vered +999 +grimace +roanoke +tchaikovsky +twinned +##analysis +##awan +xinjiang +arias +clemson +kazakh +sizable +1662 +##khand +##vard +plunge +tatum +vittorio +##nden +cholera +##dana +##oper +bracing +indifference +projectile +superliga +##chee +realises +upgrading +299 +porte +retribution +##vies +nk +stil +##resses +ama +bureaucracy +blackberry +bosch +testosterone +collapses +greer +##pathic +ioc +fifties +malls +##erved +bao +baskets +adolescents +siegfried +##osity +##tosis +mantra +detecting +existent +fledgling +##cchi +dissatisfied +gan +telecommunication +mingled +sobbed +6000 +controversies +outdated +taxis +##raus +fright +slams +##lham +##fect +##tten +detectors +fetal +tanned +##uw +fray +goth +olympian +skipping +mandates +scratches +sheng +unspoken +hyundai +tracey +hotspur +restrictive +##buch +americana +mundo +##bari +burroughs +diva +vulcan +##6th +distinctions +thumping +##ngen +mikey +sheds +fide +rescues +springsteen +vested +valuation +##ece +##ely +pinnacle +rake +sylvie +##edo +almond +quivering +##irus +alteration +faltered +##wad +51st +hydra +ticked +##kato +recommends +##dicated +antigua +arjun +stagecoach +wilfred +trickle +pronouns +##pon +aryan +nighttime +##anian +gall +pea +stitch +##hei +leung +milos +##dini +eritrea +nexus +starved +snowfall +kant +parasitic +cot +discus +hana +strikers +appleton +kitchens +##erina +##partisan +##itha +##vius +disclose +metis +##channel +1701 +tesla +##vera +fitch +1735 +blooded +##tila +decimal +##tang +##bai +cyclones +eun +bottled +peas +pensacola +basha +bolivian +crabs +boil +lanterns +partridge +roofed +1645 +necks +##phila +opined +patting +##kla +##lland +chuckles +volta +whereupon +##nche +devout +euroleague +suicidal +##dee +inherently +involuntary +knitting +nasser +##hide +puppets +colourful +courageous +southend +stills +miraculous +hodgson +richer +rochdale +ethernet +greta +uniting +prism +umm +##haya +##itical +##utation +deterioration +pointe +prowess +##ropriation +lids +scranton +billings +subcontinent +##koff +##scope +brute +kellogg +psalms +degraded +##vez +stanisław +##ructured +ferreira +pun +astonishing +gunnar +##yat +arya +prc +gottfried +##tight +excursion +##ographer +dina +##quil +##nare +huffington +illustrious +wilbur +gundam +verandah +##zard +naacp +##odle +constructive +fjord +kade +##naud +generosity +thrilling +baseline +cayman +frankish +plastics +accommodations +zoological +##fting +cedric +qb +motorized +##dome +##otted +squealed +tackled +canucks +budgets +situ +asthma +dail +gabled +grasslands +whimpered +writhing +judgments +##65 +minnie +pv +##carbon +bananas +grille +domes +monique +odin +maguire +markham +tierney +##estra +##chua +libel +poke +speedy +atrium +laval +notwithstanding +##edly +fai +kala +##sur +robb +##sma +listings +luz +supplementary +tianjin +##acing +enzo +jd +ric +scanner +croats +transcribed +##49 +arden +cv +##hair +##raphy +##lver +##uy +357 +seventies +staggering +alam +horticultural +hs +regression +timbers +blasting +##ounded +montagu +manipulating +##cit +catalytic +1550 +troopers +##meo +condemnation +fitzpatrick +##oire +##roved +inexperienced +1670 +castes +##lative +outing +314 +dubois +flicking +quarrel +ste +learners +1625 +iq +whistled +##class +282 +classify +tariffs +temperament +355 +folly +liszt +##yles +immersed +jordanian +ceasefire +apparel +extras +maru +fished +##bio +harta +stockport +assortment +craftsman +paralysis +transmitters +##cola +blindness +##wk +fatally +proficiency +solemnly +##orno +repairing +amore +groceries +ultraviolet +##chase +schoolhouse +##tua +resurgence +nailed +##otype +##× +ruse +saliva +diagrams +##tructing +albans +rann +thirties +1b +antennas +hilarious +cougars +paddington +stats +##eger +breakaway +ipod +reza +authorship +prohibiting +scoffed +##etz +##ttle +conscription +defected +trondheim +##fires +ivanov +keenan +##adan +##ciful +##fb +##slow +locating +##ials +##tford +cadiz +basalt +blankly +interned +rags +rattling +##tick +carpathian +reassured +sync +bum +guildford +iss +staunch +##onga +astronomers +sera +sofie +emergencies +susquehanna +##heard +duc +mastery +vh1 +williamsburg +bayer +buckled +craving +##khan +##rdes +bloomington +##write +alton +barbecue +##bians +justine +##hri +##ndt +delightful +smartphone +newtown +photon +retrieval +peugeot +hissing +##monium +##orough +flavors +lighted +relaunched +tainted +##games +##lysis +anarchy +microscopic +hopping +adept +evade +evie +##beau +inhibit +sinn +adjustable +hurst +intuition +wilton +cisco +44th +lawful +lowlands +stockings +thierry +##dalen +##hila +##nai +fates +prank +tb +maison +lobbied +provocative +1724 +4a +utopia +##qual +carbonate +gujarati +purcell +##rford +curtiss +##mei +overgrown +arenas +mediation +swallows +##rnik +respectful +turnbull +##hedron +##hope +alyssa +ozone +##ʻi +ami +gestapo +johansson +snooker +canteen +cuff +declines +empathy +stigma +##ags +##iner +##raine +taxpayers +gui +volga +##wright +##copic +lifespan +overcame +tattooed +enactment +giggles +##ador +##camp +barrington +bribe +obligatory +orbiting +peng +##enas +elusive +sucker +##vating +cong +hardship +empowered +anticipating +estrada +cryptic +greasy +detainees +planck +sudbury +plaid +dod +marriott +kayla +##ears +##vb +##zd +mortally +##hein +cognition +radha +319 +liechtenstein +meade +richly +argyle +harpsichord +liberalism +trumpets +lauded +tyrant +salsa +tiled +lear +promoters +reused +slicing +trident +##chuk +##gami +##lka +cantor +checkpoint +##points +gaul +leger +mammalian +##tov +##aar +##schaft +doha +frenchman +nirvana +##vino +delgado +headlining +##eron +##iography +jug +tko +1649 +naga +intersections +##jia +benfica +nawab +##suka +ashford +gulp +##deck +##vill +##rug +brentford +frazier +pleasures +dunne +potsdam +shenzhen +dentistry +##tec +flanagan +##dorff +##hear +chorale +dinah +prem +quezon +##rogated +relinquished +sutra +terri +##pani +flaps +##rissa +poly +##rnet +homme +aback +##eki +linger +womb +##kson +##lewood +doorstep +orthodoxy +threaded +westfield +##rval +dioceses +fridays +subsided +##gata +loyalists +##biotic +##ettes +letterman +lunatic +prelate +tenderly +invariably +souza +thug +winslow +##otide +furlongs +gogh +jeopardy +##runa +pegasus +##umble +humiliated +standalone +tagged +##roller +freshmen +klan +##bright +attaining +initiating +transatlantic +logged +viz +##uance +1723 +combatants +intervening +stephane +chieftain +despised +grazed +317 +cdc +galveston +godzilla +macro +simulate +##planes +parades +##esses +960 +##ductive +##unes +equator +overdose +##cans +##hosh +##lifting +joshi +epstein +sonora +treacherous +aquatics +manchu +responsive +##sation +supervisory +##christ +##llins +##ibar +##balance +##uso +kimball +karlsruhe +mab +##emy +ignores +phonetic +reuters +spaghetti +820 +almighty +danzig +rumbling +tombstone +designations +lured +outset +##felt +supermarkets +##wt +grupo +kei +kraft +susanna +##blood +comprehension +genealogy +##aghan +##verted +redding +##ythe +1722 +bowing +##pore +##roi +lest +sharpened +fulbright +valkyrie +sikhs +##unds +swans +bouquet +merritt +##tage +##venting +commuted +redhead +clerks +leasing +cesare +dea +hazy +##vances +fledged +greenfield +servicemen +##gical +armando +blackout +dt +sagged +downloadable +intra +potion +pods +##4th +##mism +xp +attendants +gambia +stale +##ntine +plump +asteroids +rediscovered +buds +flea +hive +##neas +1737 +classifications +debuts +##eles +olympus +scala +##eurs +##gno +##mute +hummed +sigismund +visuals +wiggled +await +pilasters +clench +sulfate +##ances +bellevue +enigma +trainee +snort +##sw +clouded +denim +##rank +##rder +churning +hartman +lodges +riches +sima +##missible +accountable +socrates +regulates +mueller +##cr +1702 +avoids +solids +himalayas +nutrient +pup +##jevic +squat +fades +nec +##lates +##pina +##rona +##ου +privateer +tequila +##gative +##mpton +apt +hornet +immortals +##dou +asturias +cleansing +dario +##rries +##anta +etymology +servicing +zhejiang +##venor +##nx +horned +erasmus +rayon +relocating +£10 +##bags +escalated +promenade +stubble +2010s +artisans +axial +liquids +mora +sho +yoo +##tsky +bundles +oldies +##nally +notification +bastion +##ths +sparkle +##lved +1728 +leash +pathogen +highs +##hmi +immature +880 +gonzaga +ignatius +mansions +monterrey +sweets +bryson +##loe +polled +regatta +brightest +pei +rosy +squid +hatfield +payroll +addict +meath +cornerback +heaviest +lodging +##mage +capcom +rippled +##sily +barnet +mayhem +ymca +snuggled +rousseau +##cute +blanchard +284 +fragmented +leighton +chromosomes +risking +##md +##strel +##utter +corinne +coyotes +cynical +hiroshi +yeomanry +##ractive +ebook +grading +mandela +plume +agustin +magdalene +##rkin +bea +femme +trafford +##coll +##lun +##tance +52nd +fourier +upton +##mental +camilla +gust +iihf +islamabad +longevity +##kala +feldman +netting +##rization +endeavour +foraging +mfa +orr +##open +greyish +contradiction +graz +##ruff +handicapped +marlene +tweed +oaxaca +spp +campos +miocene +pri +configured +cooks +pluto +cozy +pornographic +##entes +70th +fairness +glided +jonny +lynne +rounding +sired +##emon +##nist +remade +uncover +##mack +complied +lei +newsweek +##jured +##parts +##enting +##pg +293 +finer +guerrillas +athenian +deng +disused +stepmother +accuse +gingerly +seduction +521 +confronting +##walker +##going +gora +nostalgia +sabres +virginity +wrenched +##minated +syndication +wielding +eyre +##56 +##gnon +##igny +behaved +taxpayer +sweeps +##growth +childless +gallant +##ywood +amplified +geraldine +scrape +##ffi +babylonian +fresco +##rdan +##kney +##position +1718 +restricting +tack +fukuoka +osborn +selector +partnering +##dlow +318 +gnu +kia +tak +whitley +gables +##54 +##mania +mri +softness +immersion +##bots +##evsky +1713 +chilling +insignificant +pcs +##uis +elites +lina +purported +supplemental +teaming +##americana +##dding +##inton +proficient +rouen +##nage +##rret +niccolo +selects +##bread +fluffy +1621 +gruff +knotted +mukherjee +polgara +thrash +nicholls +secluded +smoothing +thru +corsica +loaf +whitaker +inquiries +##rrier +##kam +indochina +289 +marlins +myles +peking +##tea +extracts +pastry +superhuman +connacht +vogel +##ditional +##het +##udged +##lash +gloss +quarries +refit +teaser +##alic +##gaon +20s +materialized +sling +camped +pickering +tung +tracker +pursuant +##cide +cranes +soc +##cini +##typical +##viere +anhalt +overboard +workout +chores +fares +orphaned +stains +##logie +fenton +surpassing +joyah +triggers +##itte +grandmaster +##lass +##lists +clapping +fraudulent +ledger +nagasaki +##cor +##nosis +##tsa +eucalyptus +tun +##icio +##rney +##tara +dax +heroism +ina +wrexham +onboard +unsigned +##dates +moshe +galley +winnie +droplets +exiles +praises +watered +noodles +##aia +fein +adi +leland +multicultural +stink +bingo +comets +erskine +modernized +canned +constraint +domestically +chemotherapy +featherweight +stifled +##mum +darkly +irresistible +refreshing +hasty +isolate +##oys +kitchener +planners +##wehr +cages +yarn +implant +toulon +elects +childbirth +yue +##lind +##lone +cn +rightful +sportsman +junctions +remodeled +specifies +##rgh +291 +##oons +complimented +##urgent +lister +ot +##logic +bequeathed +cheekbones +fontana +gabby +##dial +amadeus +corrugated +maverick +resented +triangles +##hered +##usly +nazareth +tyrol +1675 +assent +poorer +sectional +aegean +##cous +296 +nylon +ghanaian +##egorical +##weig +cushions +forbid +fusiliers +obstruction +somerville +##scia +dime +earrings +elliptical +leyte +oder +polymers +timmy +atm +midtown +piloted +settles +continual +externally +mayfield +##uh +enrichment +henson +keane +persians +1733 +benji +braden +pep +324 +##efe +contenders +pepsi +valet +##isches +298 +##asse +##earing +goofy +stroll +##amen +authoritarian +occurrences +adversary +ahmedabad +tangent +toppled +dorchester +1672 +modernism +marxism +islamist +charlemagne +exponential +racks +unicode +brunette +mbc +pic +skirmish +##bund +##lad +##powered +##yst +hoisted +messina +shatter +##ctum +jedi +vantage +##music +##neil +clemens +mahmoud +corrupted +authentication +lowry +nils +##washed +omnibus +wounding +jillian +##itors +##opped +serialized +narcotics +handheld +##arm +##plicity +intersecting +stimulating +##onis +crate +fellowships +hemingway +casinos +climatic +fordham +copeland +drip +beatty +leaflets +robber +brothel +madeira +##hedral +sphinx +ultrasound +##vana +valor +forbade +leonid +villas +##aldo +duane +marquez +##cytes +disadvantaged +forearms +kawasaki +reacts +consular +lax +uncles +uphold +##hopper +concepcion +dorsey +lass +##izan +arching +passageway +1708 +researches +tia +internationals +##graphs +##opers +distinguishes +javanese +divert +##uven +plotted +##listic +##rwin +##erik +##tify +affirmative +signifies +validation +##bson +kari +felicity +georgina +zulu +##eros +##rained +##rath +overcoming +##dot +argyll +##rbin +1734 +chiba +ratification +windy +earls +parapet +##marks +hunan +pristine +astrid +punta +##gart +brodie +##kota +##oder +malaga +minerva +rouse +##phonic +bellowed +pagoda +portals +reclamation +##gur +##odies +##⁄₄ +parentheses +quoting +allergic +palette +showcases +benefactor +heartland +nonlinear +##tness +bladed +cheerfully +scans +##ety +##hone +1666 +girlfriends +pedersen +hiram +sous +##liche +##nator +1683 +##nery +##orio +##umen +bobo +primaries +smiley +##cb +unearthed +uniformly +fis +metadata +1635 +ind +##oted +recoil +##titles +##tura +##ια +406 +hilbert +jamestown +mcmillan +tulane +seychelles +##frid +antics +coli +fated +stucco +##grants +1654 +bulky +accolades +arrays +caledonian +carnage +optimism +puebla +##tative +##cave +enforcing +rotherham +seo +dunlop +aeronautics +chimed +incline +zoning +archduke +hellenistic +##oses +##sions +candi +thong +##ople +magnate +rustic +##rsk +projective +slant +##offs +danes +hollis +vocalists +##ammed +congenital +contend +gesellschaft +##ocating +##pressive +douglass +quieter +##cm +##kshi +howled +salim +spontaneously +townsville +buena +southport +##bold +kato +1638 +faerie +stiffly +##vus +##rled +297 +flawless +realising +taboo +##7th +bytes +straightening +356 +jena +##hid +##rmin +cartwright +berber +bertram +soloists +411 +noses +417 +coping +fission +hardin +inca +##cen +1717 +mobilized +vhf +##raf +biscuits +curate +##85 +##anial +331 +gaunt +neighbourhoods +1540 +##abas +blanca +bypassed +sockets +behold +coincidentally +##bane +nara +shave +splinter +terrific +##arion +##erian +commonplace +juris +redwood +waistband +boxed +caitlin +fingerprints +jennie +naturalized +##ired +balfour +craters +jody +bungalow +hugely +quilt +glitter +pigeons +undertaker +bulging +constrained +goo +##sil +##akh +assimilation +reworked +##person +persuasion +##pants +felicia +##cliff +##ulent +1732 +explodes +##dun +##inium +##zic +lyman +vulture +hog +overlook +begs +northwards +ow +spoil +##urer +fatima +favorably +accumulate +sargent +sorority +corresponded +dispersal +kochi +toned +##imi +##lita +internacional +newfound +##agger +##lynn +##rigue +booths +peanuts +##eborg +medicare +muriel +nur +##uram +crates +millennia +pajamas +worsened +##breakers +jimi +vanuatu +yawned +##udeau +carousel +##hony +hurdle +##ccus +##mounted +##pod +rv +##eche +airship +ambiguity +compulsion +recapture +##claiming +arthritis +##osomal +1667 +asserting +ngc +sniffing +dade +discontent +glendale +ported +##amina +defamation +rammed +##scent +fling +livingstone +##fleet +875 +##ppy +apocalyptic +comrade +lcd +##lowe +cessna +eine +persecuted +subsistence +demi +hoop +reliefs +710 +coptic +progressing +stemmed +perpetrators +1665 +priestess +##nio +dobson +ebony +rooster +itf +tortricidae +##bbon +##jian +cleanup +##jean +##øy +1721 +eighties +taxonomic +holiness +##hearted +##spar +antilles +showcasing +stabilized +##nb +gia +mascara +michelangelo +dawned +##uria +##vinsky +extinguished +fitz +grotesque +£100 +##fera +##loid +##mous +barges +neue +throbbed +cipher +johnnie +##a1 +##mpt +outburst +##swick +spearheaded +administrations +c1 +heartbreak +pixels +pleasantly +##enay +lombardy +plush +##nsed +bobbie +##hly +reapers +tremor +xiang +minogue +substantive +hitch +barak +##wyl +kwan +##encia +910 +obscene +elegance +indus +surfer +bribery +conserve +##hyllum +##masters +horatio +##fat +apes +rebound +psychotic +##pour +iteration +##mium +##vani +botanic +horribly +antiques +dispose +paxton +##hli +##wg +timeless +1704 +disregard +engraver +hounds +##bau +##version +looted +uno +facilitates +groans +masjid +rutland +antibody +disqualification +decatur +footballers +quake +slacks +48th +rein +scribe +stabilize +commits +exemplary +tho +##hort +##chison +pantry +traversed +##hiti +disrepair +identifiable +vibrated +baccalaureate +##nnis +csa +interviewing +##iensis +##raße +greaves +wealthiest +343 +classed +jogged +£5 +##58 +##atal +illuminating +knicks +respecting +##uno +scrubbed +##iji +##dles +kruger +moods +growls +raider +silvia +chefs +kam +vr +cree +percival +##terol +gunter +counterattack +defiant +henan +ze +##rasia +##riety +equivalence +submissions +##fra +##thor +bautista +mechanically +##heater +cornice +herbal +templar +##mering +outputs +ruining +ligand +renumbered +extravagant +mika +blockbuster +eta +insurrection +##ilia +darkening +ferocious +pianos +strife +kinship +##aer +melee +##anor +##iste +##may +##oue +decidedly +weep +##jad +##missive +##ppel +354 +puget +unease +##gnant +1629 +hammering +kassel +ob +wessex +##lga +bromwich +egan +paranoia +utilization +##atable +##idad +contradictory +provoke +##ols +##ouring +##tangled +knesset +##very +##lette +plumbing +##sden +##¹ +greensboro +occult +sniff +338 +zev +beaming +gamer +haggard +mahal +##olt +##pins +mendes +utmost +briefing +gunnery +##gut +##pher +##zh +##rok +1679 +khalifa +sonya +##boot +principals +urbana +wiring +##liffe +##minating +##rrado +dahl +nyu +skepticism +np +townspeople +ithaca +lobster +somethin +##fur +##arina +##−1 +freighter +zimmerman +biceps +contractual +##herton +amend +hurrying +subconscious +##anal +336 +meng +clermont +spawning +##eia +##lub +dignitaries +impetus +snacks +spotting +twigs +##bilis +##cz +##ouk +libertadores +nic +skylar +##aina +##firm +gustave +asean +##anum +dieter +legislatures +flirt +bromley +trolls +umar +##bbies +##tyle +blah +parc +bridgeport +crank +negligence +##nction +46th +constantin +molded +bandages +seriousness +00pm +siegel +carpets +compartments +upbeat +statehood +##dner +##edging +marko +730 +platt +##hane +paving +##iy +1738 +abbess +impatience +limousine +nbl +##talk +441 +lucille +mojo +nightfall +robbers +##nais +karel +brisk +calves +replicate +ascribed +telescopes +##olf +intimidated +##reen +ballast +specialization +##sit +aerodynamic +caliphate +rainer +visionary +##arded +epsilon +##aday +##onte +aggregation +auditory +boosted +reunification +kathmandu +loco +robyn +402 +acknowledges +appointing +humanoid +newell +redeveloped +restraints +##tained +barbarians +chopper +1609 +italiana +##lez +##lho +investigates +wrestlemania +##anies +##bib +690 +##falls +creaked +dragoons +gravely +minions +stupidity +volley +##harat +##week +musik +##eries +##uously +fungal +massimo +semantics +malvern +##ahl +##pee +discourage +embryo +imperialism +1910s +profoundly +##ddled +jiangsu +sparkled +stat +##holz +sweatshirt +tobin +##iction +sneered +##cheon +##oit +brit +causal +smyth +##neuve +diffuse +perrin +silvio +##ipes +##recht +detonated +iqbal +selma +##nism +##zumi +roasted +##riders +tay +##ados +##mament +##mut +##rud +840 +completes +nipples +cfa +flavour +hirsch +##laus +calderon +sneakers +moravian +##ksha +1622 +rq +294 +##imeters +bodo +##isance +##pre +##ronia +anatomical +excerpt +##lke +dh +kunst +##tablished +##scoe +biomass +panted +unharmed +gael +housemates +montpellier +##59 +coa +rodents +tonic +hickory +singleton +##taro +451 +1719 +aldo +breaststroke +dempsey +och +rocco +##cuit +merton +dissemination +midsummer +serials +##idi +haji +polynomials +##rdon +gs +enoch +prematurely +shutter +taunton +£3 +##grating +##inates +archangel +harassed +##asco +326 +archway +dazzling +##ecin +1736 +sumo +wat +##kovich +1086 +honneur +##ently +##nostic +##ttal +##idon +1605 +403 +1716 +blogger +rents +##gnan +hires +##ikh +##dant +howie +##rons +handler +retracted +shocks +1632 +arun +duluth +kepler +trumpeter +##lary +peeking +seasoned +trooper +##mara +laszlo +##iciencies +##rti +heterosexual +##inatory +##ssion +indira +jogging +##inga +##lism +beit +dissatisfaction +malice +##ately +nedra +peeling +##rgeon +47th +stadiums +475 +vertigo +##ains +iced +restroom +##plify +##tub +illustrating +pear +##chner +##sibility +inorganic +rappers +receipts +watery +##kura +lucinda +##oulos +reintroduced +##8th +##tched +gracefully +saxons +nutritional +wastewater +rained +favourites +bedrock +fisted +hallways +likeness +upscale +##lateral +1580 +blinds +prequel +##pps +##tama +deter +humiliating +restraining +tn +vents +1659 +laundering +recess +rosary +tractors +coulter +federer +##ifiers +##plin +persistence +##quitable +geschichte +pendulum +quakers +##beam +bassett +pictorial +buffet +koln +##sitor +drills +reciprocal +shooters +##57 +##cton +##tees +converge +pip +dmitri +donnelly +yamamoto +aqua +azores +demographics +hypnotic +spitfire +suspend +wryly +roderick +##rran +sebastien +##asurable +mavericks +##fles +##200 +himalayan +prodigy +##iance +transvaal +demonstrators +handcuffs +dodged +mcnamara +sublime +1726 +crazed +##efined +##till +ivo +pondered +reconciled +shrill +sava +##duk +bal +cad +heresy +jaipur +goran +##nished +341 +lux +shelly +whitehall +##hre +israelis +peacekeeping +##wled +1703 +demetrius +ousted +##arians +##zos +beale +anwar +backstroke +raged +shrinking +cremated +##yck +benign +towing +wadi +darmstadt +landfill +parana +soothe +colleen +sidewalks +mayfair +tumble +hepatitis +ferrer +superstructure +##gingly +##urse +##wee +anthropological +translators +##mies +closeness +hooves +##pw +mondays +##roll +##vita +landscaping +##urized +purification +sock +thorns +thwarted +jalan +tiberius +##taka +saline +##rito +confidently +khyber +sculptors +##ij +brahms +hammersmith +inspectors +battista +fivb +fragmentation +hackney +##uls +arresting +exercising +antoinette +bedfordshire +##zily +dyed +##hema +1656 +racetrack +variability +##tique +1655 +austrians +deteriorating +madman +theorists +aix +lehman +weathered +1731 +decreed +eruptions +1729 +flaw +quinlan +sorbonne +flutes +nunez +1711 +adored +downwards +fable +rasped +1712 +moritz +mouthful +renegade +shivers +stunts +dysfunction +restrain +translit +327 +pancakes +##avio +##cision +##tray +351 +vial +##lden +bain +##maid +##oxide +chihuahua +malacca +vimes +##rba +##rnier +1664 +donnie +plaques +##ually +337 +bangs +floppy +huntsville +loretta +nikolay +##otte +eater +handgun +ubiquitous +##hett +eras +zodiac +1634 +##omorphic +1820s +##zog +cochran +##bula +##lithic +warring +##rada +dalai +excused +blazers +mcconnell +reeling +bot +este +##abi +geese +hoax +taxon +##bla +guitarists +##icon +condemning +hunts +inversion +moffat +taekwondo +##lvis +1624 +stammered +##rest +##rzy +sousa +fundraiser +marylebone +navigable +uptown +cabbage +daniela +salman +shitty +whimper +##kian +##utive +programmers +protections +rm +##rmi +##rued +forceful +##enes +fuss +##tao +##wash +brat +oppressive +reykjavik +spartak +ticking +##inkles +##kiewicz +adolph +horst +maui +protege +straighten +cpc +landau +concourse +clements +resultant +##ando +imaginative +joo +reactivated +##rem +##ffled +##uising +consultative +##guide +flop +kaitlyn +mergers +parenting +somber +##vron +supervise +vidhan +##imum +courtship +exemplified +harmonies +medallist +refining +##rrow +##ка +amara +##hum +780 +goalscorer +sited +overshadowed +rohan +displeasure +secretive +multiplied +osman +##orth +engravings +padre +##kali +##veda +miniatures +mis +##yala +clap +pali +rook +##cana +1692 +57th +antennae +astro +oskar +1628 +bulldog +crotch +hackett +yucatan +##sure +amplifiers +brno +ferrara +migrating +##gree +thanking +turing +##eza +mccann +ting +andersson +onslaught +gaines +ganga +incense +standardization +##mation +sentai +scuba +stuffing +turquoise +waivers +alloys +##vitt +regaining +vaults +##clops +##gizing +digger +furry +memorabilia +probing +##iad +payton +rec +deutschland +filippo +opaque +seamen +zenith +afrikaans +##filtration +disciplined +inspirational +##merie +banco +confuse +grafton +tod +##dgets +championed +simi +anomaly +biplane +##ceptive +electrode +##para +1697 +cleavage +crossbow +swirl +informant +##lars +##osta +afi +bonfire +spec +##oux +lakeside +slump +##culus +##lais +##qvist +##rrigan +1016 +facades +borg +inwardly +cervical +xl +pointedly +050 +stabilization +##odon +chests +1699 +hacked +ctv +orthogonal +suzy +##lastic +gaulle +jacobite +rearview +##cam +##erted +ashby +##drik +##igate +##mise +##zbek +affectionately +canine +disperse +latham +##istles +##ivar +spielberg +##orin +##idium +ezekiel +cid +##sg +durga +middletown +##cina +customized +frontiers +harden +##etano +##zzy +1604 +bolsheviks +##66 +coloration +yoko +##bedo +briefs +slabs +debra +liquidation +plumage +##oin +blossoms +dementia +subsidy +1611 +proctor +relational +jerseys +parochial +ter +##ici +esa +peshawar +cavalier +loren +cpi +idiots +shamrock +1646 +dutton +malabar +mustache +##endez +##ocytes +referencing +terminates +marche +yarmouth +##sop +acton +mated +seton +subtly +baptised +beige +extremes +jolted +kristina +telecast +##actic +safeguard +waldo +##baldi +##bular +endeavors +sloppy +subterranean +##ensburg +##itung +delicately +pigment +tq +##scu +1626 +##ound +collisions +coveted +herds +##personal +##meister +##nberger +chopra +##ricting +abnormalities +defective +galician +lucie +##dilly +alligator +likened +##genase +burundi +clears +complexion +derelict +deafening +diablo +fingered +champaign +dogg +enlist +isotope +labeling +mrna +##erre +brilliance +marvelous +##ayo +1652 +crawley +ether +footed +dwellers +deserts +hamish +rubs +warlock +skimmed +##lizer +870 +buick +embark +heraldic +irregularities +##ajan +kiara +##kulam +##ieg +antigen +kowalski +##lge +oakley +visitation +##mbit +vt +##suit +1570 +murderers +##miento +##rites +chimneys +##sling +condemn +custer +exchequer +havre +##ghi +fluctuations +##rations +dfb +hendricks +vaccines +##tarian +nietzsche +biking +juicy +##duced +brooding +scrolling +selangor +##ragan +352 +annum +boomed +seminole +sugarcane +##dna +departmental +dismissing +innsbruck +arteries +ashok +batavia +daze +kun +overtook +##rga +##tlan +beheaded +gaddafi +holm +electronically +faulty +galilee +fractures +kobayashi +##lized +gunmen +magma +aramaic +mala +eastenders +inference +messengers +bf +##qu +407 +bathrooms +##vere +1658 +flashbacks +ideally +misunderstood +##jali +##weather +mendez +##grounds +505 +uncanny +##iii +1709 +friendships +##nbc +sacrament +accommodated +reiterated +logistical +pebbles +thumped +##escence +administering +decrees +drafts +##flight +##cased +##tula +futuristic +picket +intimidation +winthrop +##fahan +interfered +339 +afar +francoise +morally +uta +cochin +croft +dwarfs +##bruck +##dents +##nami +biker +##hner +##meral +nano +##isen +##ometric +##pres +##ан +brightened +meek +parcels +securely +gunners +##jhl +##zko +agile +hysteria +##lten +##rcus +bukit +champs +chevy +cuckoo +leith +sadler +theologians +welded +##section +1663 +jj +plurality +xander +##rooms +##formed +shredded +temps +intimately +pau +tormented +##lok +##stellar +1618 +charred +ems +essen +##mmel +alarms +spraying +ascot +blooms +twinkle +##abia +##apes +internment +obsidian +##chaft +snoop +##dav +##ooping +malibu +##tension +quiver +##itia +hays +mcintosh +travers +walsall +##ffie +1623 +beverley +schwarz +plunging +structurally +m3 +rosenthal +vikram +##tsk +770 +ghz +##onda +##tiv +chalmers +groningen +pew +reckon +unicef +##rvis +55th +##gni +1651 +sulawesi +avila +cai +metaphysical +screwing +turbulence +##mberg +augusto +samba +56th +baffled +momentary +toxin +##urian +##wani +aachen +condoms +dali +steppe +##3d +##app +##oed +##year +adolescence +dauphin +electrically +inaccessible +microscopy +nikita +##ega +atv +##cel +##enter +##oles +##oteric +##ы +accountants +punishments +wrongly +bribes +adventurous +clinch +flinders +southland +##hem +##kata +gough +##ciency +lads +soared +##ה +undergoes +deformation +outlawed +rubbish +##arus +##mussen +##nidae +##rzburg +arcs +##ingdon +##tituted +1695 +wheelbase +wheeling +bombardier +campground +zebra +##lices +##oj +##bain +lullaby +##ecure +donetsk +wylie +grenada +##arding +##ης +squinting +eireann +opposes +##andra +maximal +runes +##broken +##cuting +##iface +##ror +##rosis +additive +britney +adultery +triggering +##drome +detrimental +aarhus +containment +jc +swapped +vichy +##ioms +madly +##oric +##rag +brant +##ckey +##trix +1560 +1612 +broughton +rustling +##stems +##uder +asbestos +mentoring +##nivorous +finley +leaps +##isan +apical +pry +slits +substitutes +##dict +intuitive +fantasia +insistent +unreasonable +##igen +##vna +domed +hannover +margot +ponder +##zziness +impromptu +jian +lc +rampage +stemming +##eft +andrey +gerais +whichever +amnesia +appropriated +anzac +clicks +modifying +ultimatum +cambrian +maids +verve +yellowstone +##mbs +conservatoire +##scribe +adherence +dinners +spectra +imperfect +mysteriously +sidekick +tatar +tuba +##aks +##ifolia +distrust +##athan +##zle +c2 +ronin +zac +##pse +celaena +instrumentalist +scents +skopje +##mbling +comical +compensated +vidal +condor +intersect +jingle +wavelengths +##urrent +mcqueen +##izzly +carp +weasel +422 +kanye +militias +postdoctoral +eugen +gunslinger +##ɛ +faux +hospice +##for +appalled +derivation +dwarves +##elis +dilapidated +##folk +astoria +philology +##lwyn +##otho +##saka +inducing +philanthropy +##bf +##itative +geek +markedly +sql +##yce +bessie +indices +rn +##flict +495 +frowns +resolving +weightlifting +tugs +cleric +contentious +1653 +mania +rms +##miya +##reate +##ruck +##tucket +bien +eels +marek +##ayton +##cence +discreet +unofficially +##ife +leaks +##bber +1705 +332 +dung +compressor +hillsborough +pandit +shillings +distal +##skin +381 +##tat +##you +nosed +##nir +mangrove +undeveloped +##idia +textures +##inho +##500 +##rise +ae +irritating +nay +amazingly +bancroft +apologetic +compassionate +kata +symphonies +##lovic +airspace +##lch +930 +gifford +precautions +fulfillment +sevilla +vulgar +martinique +##urities +looting +piccolo +tidy +##dermott +quadrant +armchair +incomes +mathematicians +stampede +nilsson +##inking +##scan +foo +quarterfinal +##ostal +shang +shouldered +squirrels +##owe +344 +vinegar +##bner +##rchy +##systems +delaying +##trics +ars +dwyer +rhapsody +sponsoring +##gration +bipolar +cinder +starters +##olio +##urst +421 +signage +##nty +aground +figurative +mons +acquaintances +duets +erroneously +soyuz +elliptic +recreated +##cultural +##quette +##ssed +##tma +##zcz +moderator +scares +##itaire +##stones +##udence +juniper +sighting +##just +##nsen +britten +calabria +ry +bop +cramer +forsyth +stillness +##л +airmen +gathers +unfit +##umber +##upt +taunting +##rip +seeker +streamlined +##bution +holster +schumann +tread +vox +##gano +##onzo +strive +dil +reforming +covent +newbury +predicting +##orro +decorate +tre +##puted +andover +ie +asahi +dept +dunkirk +gills +##tori +buren +huskies +##stis +##stov +abstracts +bets +loosen +##opa +1682 +yearning +##glio +##sir +berman +effortlessly +enamel +napoli +persist +##peration +##uez +attache +elisa +b1 +invitations +##kic +accelerating +reindeer +boardwalk +clutches +nelly +polka +starbucks +##kei +adamant +huey +lough +unbroken +adventurer +embroidery +inspecting +stanza +##ducted +naia +taluka +##pone +##roids +chases +deprivation +florian +##jing +##ppet +earthly +##lib +##ssee +colossal +foreigner +vet +freaks +patrice +rosewood +triassic +upstate +##pkins +dominates +ata +chants +ks +vo +##400 +##bley +##raya +##rmed +555 +agra +infiltrate +##ailing +##ilation +##tzer +##uppe +##werk +binoculars +enthusiast +fujian +squeak +##avs +abolitionist +almeida +boredom +hampstead +marsden +rations +##ands +inflated +334 +bonuses +rosalie +patna +##rco +329 +detachments +penitentiary +54th +flourishing +woolf +##dion +##etched +papyrus +##lster +##nsor +##toy +bobbed +dismounted +endelle +inhuman +motorola +tbs +wince +wreath +##ticus +hideout +inspections +sanjay +disgrace +infused +pudding +stalks +##urbed +arsenic +leases +##hyl +##rrard +collarbone +##waite +##wil +dowry +##bant +##edance +genealogical +nitrate +salamanca +scandals +thyroid +necessitated +##! +##" +### +##$ +##% +##& +##' +##( +##) +##* +##+ +##, +##- +##. +##/ +##: +##; +##< +##= +##> +##? +##@ +##[ +##\ +##] +##^ +##_ +##` +##{ +##| +##} +##~ +##¡ +##¢ +##£ +##¤ +##¥ +##¦ +##§ +##¨ +##© +##ª +##« +##¬ +##® +##± +##´ +##µ +##¶ +##· +##º +##» +##¼ +##¾ +##¿ +##æ +##ð +##÷ +##þ +##đ +##ħ +##ŋ +##œ +##ƒ +##ɐ +##ɑ +##ɒ +##ɔ +##ɕ +##ə +##ɡ +##ɣ +##ɨ +##ɪ +##ɫ +##ɬ +##ɯ +##ɲ +##ɴ +##ɹ +##ɾ +##ʀ +##ʁ +##ʂ +##ʃ +##ʉ +##ʊ +##ʋ +##ʌ +##ʎ +##ʐ +##ʑ +##ʒ +##ʔ +##ʰ +##ʲ +##ʳ +##ʷ +##ʸ +##ʻ +##ʼ +##ʾ +##ʿ +##ˈ +##ˡ +##ˢ +##ˣ +##ˤ +##β +##γ +##δ +##ε +##ζ +##θ +##κ +##λ +##μ +##ξ +##ο +##π +##ρ +##σ +##τ +##υ +##φ +##χ +##ψ +##ω +##б +##г +##д +##ж +##з +##м +##п +##с +##у +##ф +##х +##ц +##ч +##ш +##щ +##ъ +##э +##ю +##ђ +##є +##і +##ј +##љ +##њ +##ћ +##ӏ +##ա +##բ +##գ +##դ +##ե +##թ +##ի +##լ +##կ +##հ +##մ +##յ +##ն +##ո +##պ +##ս +##վ +##տ +##ր +##ւ +##ք +##־ +##א +##ב +##ג +##ד +##ו +##ז +##ח +##ט +##י +##ך +##כ +##ל +##ם +##מ +##ן +##נ +##ס +##ע +##ף +##פ +##ץ +##צ +##ק +##ר +##ש +##ת +##، +##ء +##ب +##ت +##ث +##ج +##ح +##خ +##ذ +##ز +##س +##ش +##ص +##ض +##ط +##ظ +##ع +##غ +##ـ +##ف +##ق +##ك +##و +##ى +##ٹ +##پ +##چ +##ک +##گ +##ں +##ھ +##ہ +##ے +##अ +##आ +##उ +##ए +##क +##ख +##ग +##च +##ज +##ट +##ड +##ण +##त +##थ +##द +##ध +##न +##प +##ब +##भ +##म +##य +##र +##ल +##व +##श +##ष +##स +##ह +##ा +##ि +##ी +##ो +##। +##॥ +##ং +##অ +##আ +##ই +##উ +##এ +##ও +##ক +##খ +##গ +##চ +##ছ +##জ +##ট +##ড +##ণ +##ত +##থ +##দ +##ধ +##ন +##প +##ব +##ভ +##ম +##য +##র +##ল +##শ +##ষ +##স +##হ +##া +##ি +##ী +##ে +##க +##ச +##ட +##த +##ந +##ன +##ப +##ம +##ய +##ர +##ல +##ள +##வ +##ா +##ி +##ு +##ே +##ை +##ನ +##ರ +##ಾ +##ක +##ය +##ර +##ල +##ව +##ා +##ก +##ง +##ต +##ท +##น +##พ +##ม +##ย +##ร +##ล +##ว +##ส +##อ +##า +##เ +##་ +##། +##ག +##ང +##ད +##ན +##པ +##བ +##མ +##འ +##ར +##ལ +##ས +##မ +##ა +##ბ +##გ +##დ +##ე +##ვ +##თ +##ი +##კ +##ლ +##მ +##ნ +##ო +##რ +##ს +##ტ +##უ +##ᄀ +##ᄂ +##ᄃ +##ᄅ +##ᄆ +##ᄇ +##ᄉ +##ᄊ +##ᄋ +##ᄌ +##ᄎ +##ᄏ +##ᄐ +##ᄑ +##ᄒ +##ᅡ +##ᅢ +##ᅥ +##ᅦ +##ᅧ +##ᅩ +##ᅪ +##ᅭ +##ᅮ +##ᅯ +##ᅲ +##ᅳ +##ᅴ +##ᅵ +##ᆨ +##ᆫ +##ᆯ +##ᆷ +##ᆸ +##ᆼ +##ᴬ +##ᴮ +##ᴰ +##ᴵ +##ᴺ +##ᵀ +##ᵃ +##ᵇ +##ᵈ +##ᵉ +##ᵍ +##ᵏ +##ᵐ +##ᵒ +##ᵖ +##ᵗ +##ᵘ +##ᵣ +##ᵤ +##ᵥ +##ᶜ +##ᶠ +##‐ +##‑ +##‒ +##– +##— +##― +##‖ +##‘ +##’ +##‚ +##“ +##” +##„ +##† +##‡ +##• +##… +##‰ +##′ +##″ +##› +##‿ +##⁄ +##⁰ +##ⁱ +##⁴ +##⁵ +##⁶ +##⁷ +##⁸ +##⁹ +##⁻ +##ⁿ +##₅ +##₆ +##₇ +##₈ +##₉ +##₊ +##₍ +##₎ +##ₐ +##ₑ +##ₒ +##ₓ +##ₕ +##ₖ +##ₗ +##ₘ +##ₚ +##ₛ +##ₜ +##₤ +##₩ +##€ +##₱ +##₹ +##ℓ +##№ +##ℝ +##™ +##⅓ +##⅔ +##← +##↑ +##→ +##↓ +##↔ +##↦ +##⇄ +##⇌ +##⇒ +##∂ +##∅ +##∆ +##∇ +##∈ +##∗ +##∘ +##√ +##∞ +##∧ +##∨ +##∩ +##∪ +##≈ +##≡ +##≤ +##≥ +##⊂ +##⊆ +##⊕ +##⊗ +##⋅ +##─ +##│ +##■ +##▪ +##● +##★ +##☆ +##☉ +##♠ +##♣ +##♥ +##♦ +##♯ +##⟨ +##⟩ +##ⱼ +##⺩ +##⺼ +##⽥ +##、 +##。 +##〈 +##〉 +##《 +##》 +##「 +##」 +##『 +##』 +##〜 +##あ +##い +##う +##え +##お +##か +##き +##く +##け +##こ +##さ +##し +##す +##せ +##そ +##た +##ち +##っ +##つ +##て +##と +##な +##に +##ぬ +##ね +##の +##は +##ひ +##ふ +##へ +##ほ +##ま +##み +##む +##め +##も +##や +##ゆ +##よ +##ら +##り +##る +##れ +##ろ +##を +##ん +##ァ +##ア +##ィ +##イ +##ウ +##ェ +##エ +##オ +##カ +##キ +##ク +##ケ +##コ +##サ +##シ +##ス +##セ +##タ +##チ +##ッ +##ツ +##テ +##ト +##ナ +##ニ +##ノ +##ハ +##ヒ +##フ +##ヘ +##ホ +##マ +##ミ +##ム +##メ +##モ +##ャ +##ュ +##ョ +##ラ +##リ +##ル +##レ +##ロ +##ワ +##ン +##・ +##ー +##一 +##三 +##上 +##下 +##不 +##世 +##中 +##主 +##久 +##之 +##也 +##事 +##二 +##五 +##井 +##京 +##人 +##亻 +##仁 +##介 +##代 +##仮 +##伊 +##会 +##佐 +##侍 +##保 +##信 +##健 +##元 +##光 +##八 +##公 +##内 +##出 +##分 +##前 +##劉 +##力 +##加 +##勝 +##北 +##区 +##十 +##千 +##南 +##博 +##原 +##口 +##古 +##史 +##司 +##合 +##吉 +##同 +##名 +##和 +##囗 +##四 +##国 +##國 +##土 +##地 +##坂 +##城 +##堂 +##場 +##士 +##夏 +##外 +##大 +##天 +##太 +##夫 +##奈 +##女 +##子 +##学 +##宀 +##宇 +##安 +##宗 +##定 +##宣 +##宮 +##家 +##宿 +##寺 +##將 +##小 +##尚 +##山 +##岡 +##島 +##崎 +##川 +##州 +##巿 +##帝 +##平 +##年 +##幸 +##广 +##弘 +##張 +##彳 +##後 +##御 +##德 +##心 +##忄 +##志 +##忠 +##愛 +##成 +##我 +##戦 +##戸 +##手 +##扌 +##政 +##文 +##新 +##方 +##日 +##明 +##星 +##春 +##昭 +##智 +##曲 +##書 +##月 +##有 +##朝 +##木 +##本 +##李 +##村 +##東 +##松 +##林 +##森 +##楊 +##樹 +##橋 +##歌 +##止 +##正 +##武 +##比 +##氏 +##民 +##水 +##氵 +##氷 +##永 +##江 +##沢 +##河 +##治 +##法 +##海 +##清 +##漢 +##瀬 +##火 +##版 +##犬 +##王 +##生 +##田 +##男 +##疒 +##発 +##白 +##的 +##皇 +##目 +##相 +##省 +##真 +##石 +##示 +##社 +##神 +##福 +##禾 +##秀 +##秋 +##空 +##立 +##章 +##竹 +##糹 +##美 +##義 +##耳 +##良 +##艹 +##花 +##英 +##華 +##葉 +##藤 +##行 +##街 +##西 +##見 +##訁 +##語 +##谷 +##貝 +##貴 +##車 +##軍 +##辶 +##道 +##郎 +##郡 +##部 +##都 +##里 +##野 +##金 +##鈴 +##镇 +##長 +##門 +##間 +##阝 +##阿 +##陳 +##陽 +##雄 +##青 +##面 +##風 +##食 +##香 +##馬 +##高 +##龍 +##龸 +##fi +##fl +##！ +##（ +##） +##， +##－ +##． +##／ +##： +##？ +##～ diff --git a/data/datasets/sentiment_classification/newsgroup.py b/data/datasets/sentiment_classification/newsgroup.py new file mode 100644 index 0000000000000000000000000000000000000000..a9515b87325dd824ed5b3aed45a06c43041a020f --- /dev/null +++ b/data/datasets/sentiment_classification/newsgroup.py @@ -0,0 +1,113 @@ +from sklearn.datasets import fetch_20newsgroups +# from pprint import pprint +# newsgroups_train = fetch_20newsgroups(subset='train') +# print(newsgroups_train.target_names) +# print(newsgroups_train['data'][0]) +import os +import torch +from torch.utils.data import Dataset, DataLoader +from transformers import BertTokenizer, BertForSequenceClassification +from torch.optim import Adam +from torch.nn import CrossEntropyLoss +from typing import Dict, List, Optional, Any +from utils.common.data_record import read_json +from .global_bert_tokenizer import get_tokenizer + + +class NewsgroupDomainsDataset(Dataset): + def __init__(self, root_dir: str, split: str, transform: Any, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + + assert transform is None + + self.tokenizer = get_tokenizer() # 传入tokenizer对象 + self.texts = [] + self.labels = [] + self.max_length = None # 设置文本的最大长度 + + # json_file_path = os.path.join(root_dir, f'{split if split != "val" else "dev"}.json') + # anns = read_json(json_file_path) + + # label_map = {'-': 0, '+': 1, 'negative': 0, 'positive': 1} + + ignore_cls_indexes = [classes.index(c) for c in ignore_classes] + + # for v in anns.values(): + # if v['polarity'] not in label_map.keys(): + # continue + + # cls = label_map[v['polarity']] + + # if cls in ignore_cls_indexes: + # continue + + # self.texts += [v['sentence']] + # self.labels += [idx_map[cls] if idx_map is not None else cls] + + if split == 'val': + split = 'test' + data = fetch_20newsgroups(subset=split) + + self.texts = [i for _i, i in enumerate(data['data']) if data['target'][_i] not in ignore_cls_indexes] + self.labels = [i for i in data['target'] if i not in ignore_cls_indexes] + self.labels = [idx_map[i] if idx_map is not None else i for i in self.labels] + + def __len__(self): + return len(self.texts) + + def __getitem__(self, idx): + text = self.texts[idx] + label = self.labels[idx] + + encoded_input = self.tokenizer.encode_plus( + text, max_length=self.max_length, padding="max_length", truncation=True, return_tensors="pt" + ) + + x = {key: tensor.squeeze(0) for key, tensor in encoded_input.items()} + x['return_dict'] = False + return x, torch.tensor(label) + + +from ..ab_dataset import ABDataset +from ..registery import dataset_register + +@dataset_register( + name='Newsgroup', + classes=['alt.atheism', 'comp.graphics', 'comp.os.ms-windows.misc', 'comp.sys.ibm.pc.hardware', 'comp.sys.mac.hardware', 'comp.windows.x', 'misc.forsale', 'rec.autos', 'rec.motorcycles', 'rec.sport.baseball', 'rec.sport.hockey', 'sci.crypt', 'sci.electronics', 'sci.med', 'sci.space', 'soc.religion.christian', 'talk.politics.guns', 'talk.politics.mideast', 'talk.politics.misc', 'talk.religion.misc'], + task_type='Sentiment Classification', + object_type='News', + class_aliases=[], + shift_type=None +) +class Newsgroup(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return NewsgroupDomainsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + + +@dataset_register( + name='Newsgroup2', + classes=['alt.atheism', 'comp.graphics', 'comp.os.ms-windows.misc', 'comp.sys.ibm.pc.hardware', 'comp.sys.mac.hardware', 'comp.windows.x', 'misc.forsale', 'rec.autos', 'rec.motorcycles', 'rec.sport.baseball', 'rec.sport.hockey', 'sci.crypt', 'sci.electronics', 'sci.med', 'sci.space', 'soc.religion.christian', 'talk.politics.guns', 'talk.politics.mideast', 'talk.politics.misc', 'talk.religion.misc'], + task_type='Sentiment Classification', + object_type='News', + class_aliases=[], + shift_type=None +) +class Newsgroup2(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return NewsgroupDomainsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + + +@dataset_register( + name='Newsgroup3', + classes=['alt.atheism', 'comp.graphics', 'comp.os.ms-windows.misc', 'comp.sys.ibm.pc.hardware', 'comp.sys.mac.hardware', 'comp.windows.x', 'misc.forsale', 'rec.autos', 'rec.motorcycles', 'rec.sport.baseball', 'rec.sport.hockey', 'sci.crypt', 'sci.electronics', 'sci.med', 'sci.space', 'soc.religion.christian', 'talk.politics.guns', 'talk.politics.mideast', 'talk.politics.misc', 'talk.religion.misc'], + task_type='Sentiment Classification', + object_type='News', + class_aliases=[], + shift_type=None +) +class Newsgroup3(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return NewsgroupDomainsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) \ No newline at end of file diff --git a/data/datasets/sentiment_classification/newsgroup_for_gpt.py b/data/datasets/sentiment_classification/newsgroup_for_gpt.py new file mode 100644 index 0000000000000000000000000000000000000000..cb718ac936405f3915036aa0bab6558502d30330 --- /dev/null +++ b/data/datasets/sentiment_classification/newsgroup_for_gpt.py @@ -0,0 +1,122 @@ +from sklearn.datasets import fetch_20newsgroups +# from pprint import pprint +# newsgroups_train = fetch_20newsgroups(subset='train') +# print(newsgroups_train.target_names) +# print(newsgroups_train['data'][0]) + +# print(1111) + + +import os +import torch +from torch.utils.data import Dataset, DataLoader +from transformers import BertTokenizer, BertForSequenceClassification +from torch.optim import Adam +from torch.nn import CrossEntropyLoss +from typing import Dict, List, Optional, Any +from utils.common.data_record import read_json +# from .global_bert_tokenizer import get_tokenizer + +from transformers import GPT2Tokenizer +tokenizer = GPT2Tokenizer.from_pretrained(f'experiments/elasticdnn/gpt_neo/{os.environ["gpt_neo_series_id"]}') + + +class NewsgroupDomainsDataset(Dataset): + def __init__(self, root_dir: str, split: str, transform: Any, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + + assert transform is None + + self.tokenizer = tokenizer # 传入tokenizer对象\ + tokenizer.pad_token = tokenizer.eos_token + + self.texts = [] + self.labels = [] + self.max_length = 512 # 设置文本的最大长度 + + # json_file_path = os.path.join(root_dir, f'{split if split != "val" else "dev"}.json') + # anns = read_json(json_file_path) + + # label_map = {'-': 0, '+': 1, 'negative': 0, 'positive': 1} + + ignore_cls_indexes = [classes.index(c) for c in ignore_classes] + + # for v in anns.values(): + # if v['polarity'] not in label_map.keys(): + # continue + + # cls = label_map[v['polarity']] + + # if cls in ignore_cls_indexes: + # continue + + # self.texts += [v['sentence']] + # self.labels += [idx_map[cls] if idx_map is not None else cls] + + if split == 'val': + split = 'test' + data = fetch_20newsgroups(subset=split) + + self.texts = [i for _i, i in enumerate(data['data']) if data['target'][_i] not in ignore_cls_indexes] + self.labels = [i for i in data['target'] if i not in ignore_cls_indexes] + self.labels = [idx_map[i] if idx_map is not None else i for i in self.labels] + + def __len__(self): + return len(self.texts) + + def __getitem__(self, idx): + text = self.texts[idx] + label = self.labels[idx] + + encoded_input = self.tokenizer.encode_plus( + text, max_length=self.max_length, padding="max_length", truncation=True, return_tensors="pt" + ) + + x = {key: tensor.squeeze(0) for key, tensor in encoded_input.items()} + x['return_dict'] = False + return x, torch.tensor(label) + + +from ..ab_dataset import ABDataset +from ..registery import dataset_register + +@dataset_register( + name='Newsgroup-GPT', + classes=['alt.atheism', 'comp.graphics', 'comp.os.ms-windows.misc', 'comp.sys.ibm.pc.hardware', 'comp.sys.mac.hardware', 'comp.windows.x', 'misc.forsale', 'rec.autos', 'rec.motorcycles', 'rec.sport.baseball', 'rec.sport.hockey', 'sci.crypt', 'sci.electronics', 'sci.med', 'sci.space', 'soc.religion.christian', 'talk.politics.guns', 'talk.politics.mideast', 'talk.politics.misc', 'talk.religion.misc'], + task_type='Sentiment Classification', + object_type='News', + class_aliases=[], + shift_type=None +) +class NewsgroupGPT(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return NewsgroupDomainsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + + +@dataset_register( + name='Newsgroup2-GPT', + classes=['alt.atheism', 'comp.graphics', 'comp.os.ms-windows.misc', 'comp.sys.ibm.pc.hardware', 'comp.sys.mac.hardware', 'comp.windows.x', 'misc.forsale', 'rec.autos', 'rec.motorcycles', 'rec.sport.baseball', 'rec.sport.hockey', 'sci.crypt', 'sci.electronics', 'sci.med', 'sci.space', 'soc.religion.christian', 'talk.politics.guns', 'talk.politics.mideast', 'talk.politics.misc', 'talk.religion.misc'], + task_type='Sentiment Classification', + object_type='News', + class_aliases=[], + shift_type=None +) +class Newsgroup2GPT(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return NewsgroupDomainsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + + +@dataset_register( + name='Newsgroup3', + classes=['alt.atheism', 'comp.graphics', 'comp.os.ms-windows.misc', 'comp.sys.ibm.pc.hardware', 'comp.sys.mac.hardware', 'comp.windows.x', 'misc.forsale', 'rec.autos', 'rec.motorcycles', 'rec.sport.baseball', 'rec.sport.hockey', 'sci.crypt', 'sci.electronics', 'sci.med', 'sci.space', 'soc.religion.christian', 'talk.politics.guns', 'talk.politics.mideast', 'talk.politics.misc', 'talk.religion.misc'], + task_type='Sentiment Classification', + object_type='News', + class_aliases=[], + shift_type=None +) +class Newsgroup3(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return NewsgroupDomainsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) \ No newline at end of file diff --git a/data/datasets/sentiment_classification/newsgroup_for_gpt_with_aug1.py b/data/datasets/sentiment_classification/newsgroup_for_gpt_with_aug1.py new file mode 100644 index 0000000000000000000000000000000000000000..793cc0c041859f717316aceb77f529a68e65905f --- /dev/null +++ b/data/datasets/sentiment_classification/newsgroup_for_gpt_with_aug1.py @@ -0,0 +1,137 @@ +from sklearn.datasets import fetch_20newsgroups +# from pprint import pprint +# newsgroups_train = fetch_20newsgroups(subset='train') +# print(newsgroups_train.target_names) +# print(newsgroups_train['data'][0]) + +# print(1111) + +import os +import torch +from torch.utils.data import Dataset, DataLoader +from transformers import BertTokenizer, BertForSequenceClassification +from torch.optim import Adam +from torch.nn import CrossEntropyLoss +from typing import Dict, List, Optional, Any +from utils.common.data_record import read_json +# from .global_bert_tokenizer import get_tokenizer + +from transformers import GPT2Tokenizer +tokenizer = GPT2Tokenizer.from_pretrained(f'experiments/elasticdnn/gpt_neo/{os.environ["gpt_neo_series_id"]}') + +import nlpaug.augmenter.char as nac +import nlpaug.augmenter.word as naw +import nlpaug.augmenter.sentence as nas +import nlpaug.flow as nafc +from nlpaug.util import Action + +# from nlpaug.util.file.download import DownloadUtil +# DownloadUtil.download_word2vec(dest_dir='.') # Download word2vec model + +# aug = naw.WordEmbsAug( +# model_type='word2vec', model_path='GoogleNews-vectors-negative300.bin', +# action="insert") +aug = naw.SynonymAug(aug_src='wordnet') + + +class NewsgroupDomainsDataset(Dataset): + def __init__(self, root_dir: str, split: str, transform: Any, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + + assert transform is None + + self.tokenizer = tokenizer # 传入tokenizer对象\ + tokenizer.pad_token = tokenizer.eos_token + + self.texts = [] + self.labels = [] + self.max_length = 512 # 设置文本的最大长度 + + # json_file_path = os.path.join(root_dir, f'{split if split != "val" else "dev"}.json') + # anns = read_json(json_file_path) + + # label_map = {'-': 0, '+': 1, 'negative': 0, 'positive': 1} + + ignore_cls_indexes = [classes.index(c) for c in ignore_classes] + + # for v in anns.values(): + # if v['polarity'] not in label_map.keys(): + # continue + + # cls = label_map[v['polarity']] + + # if cls in ignore_cls_indexes: + # continue + + # self.texts += [v['sentence']] + # self.labels += [idx_map[cls] if idx_map is not None else cls] + + if split == 'val': + split = 'test' + data = fetch_20newsgroups(subset=split) + + self.texts = [i for _i, i in enumerate(data['data']) if data['target'][_i] not in ignore_cls_indexes] + self.labels = [i for i in data['target'] if i not in ignore_cls_indexes] + self.labels = [idx_map[i] if idx_map is not None else i for i in self.labels] + + def __len__(self): + return len(self.texts) + + def __getitem__(self, idx): + text = self.texts[idx] + label = self.labels[idx] + + text = aug.augment(text) + + encoded_input = self.tokenizer.encode_plus( + text, max_length=self.max_length, padding="max_length", truncation=True, return_tensors="pt" + ) + + x = {key: tensor.squeeze(0) for key, tensor in encoded_input.items()} + x['return_dict'] = False + return x, torch.tensor(label) + + +from ..ab_dataset import ABDataset +from ..registery import dataset_register + +@dataset_register( + name='Newsgroup-GPT-Aug1', + classes=['alt.atheism', 'comp.graphics', 'comp.os.ms-windows.misc', 'comp.sys.ibm.pc.hardware', 'comp.sys.mac.hardware', 'comp.windows.x', 'misc.forsale', 'rec.autos', 'rec.motorcycles', 'rec.sport.baseball', 'rec.sport.hockey', 'sci.crypt', 'sci.electronics', 'sci.med', 'sci.space', 'soc.religion.christian', 'talk.politics.guns', 'talk.politics.mideast', 'talk.politics.misc', 'talk.religion.misc'], + task_type='Sentiment Classification', + object_type='News', + class_aliases=[], + shift_type=None +) +class NewsgroupGPTAug1(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return NewsgroupDomainsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + + +# @dataset_register( +# name='Newsgroup2-GPT', +# classes=['alt.atheism', 'comp.graphics', 'comp.os.ms-windows.misc', 'comp.sys.ibm.pc.hardware', 'comp.sys.mac.hardware', 'comp.windows.x', 'misc.forsale', 'rec.autos', 'rec.motorcycles', 'rec.sport.baseball', 'rec.sport.hockey', 'sci.crypt', 'sci.electronics', 'sci.med', 'sci.space', 'soc.religion.christian', 'talk.politics.guns', 'talk.politics.mideast', 'talk.politics.misc', 'talk.religion.misc'], +# task_type='Sentiment Classification', +# object_type='News', +# class_aliases=[], +# shift_type=None +# ) +# class Newsgroup2GPT(ABDataset): +# def create_dataset(self, root_dir: str, split: str, transform, +# classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): +# return NewsgroupDomainsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + + +# @dataset_register( +# name='Newsgroup3', +# classes=['alt.atheism', 'comp.graphics', 'comp.os.ms-windows.misc', 'comp.sys.ibm.pc.hardware', 'comp.sys.mac.hardware', 'comp.windows.x', 'misc.forsale', 'rec.autos', 'rec.motorcycles', 'rec.sport.baseball', 'rec.sport.hockey', 'sci.crypt', 'sci.electronics', 'sci.med', 'sci.space', 'soc.religion.christian', 'talk.politics.guns', 'talk.politics.mideast', 'talk.politics.misc', 'talk.religion.misc'], +# task_type='Sentiment Classification', +# object_type='News', +# class_aliases=[], +# shift_type=None +# ) +# class Newsgroup3(ABDataset): +# def create_dataset(self, root_dir: str, split: str, transform, +# classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): +# return NewsgroupDomainsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) \ No newline at end of file diff --git a/data/datasets/sentiment_classification/newsgroup_for_gpt_with_aug2.py b/data/datasets/sentiment_classification/newsgroup_for_gpt_with_aug2.py new file mode 100644 index 0000000000000000000000000000000000000000..eb876e8eba8ecf93c51331317914a95f1e1e6a70 --- /dev/null +++ b/data/datasets/sentiment_classification/newsgroup_for_gpt_with_aug2.py @@ -0,0 +1,130 @@ +from sklearn.datasets import fetch_20newsgroups +# from pprint import pprint +# newsgroups_train = fetch_20newsgroups(subset='train') +# print(newsgroups_train.target_names) +# print(newsgroups_train['data'][0]) +import os +import torch +from torch.utils.data import Dataset, DataLoader +from transformers import BertTokenizer, BertForSequenceClassification +from torch.optim import Adam +from torch.nn import CrossEntropyLoss +from typing import Dict, List, Optional, Any +from utils.common.data_record import read_json +# from .global_bert_tokenizer import get_tokenizer + +from transformers import GPT2Tokenizer +tokenizer = GPT2Tokenizer.from_pretrained(f'experiments/elasticdnn/gpt_neo/{os.environ["gpt_neo_series_id"]}') + + +import nlpaug.augmenter.char as nac +import nlpaug.augmenter.word as naw +import nlpaug.augmenter.sentence as nas +import nlpaug.flow as nafc +from nlpaug.util import Action + +aug = nac.RandomCharAug(action="substitute") + + + +class NewsgroupDomainsDataset(Dataset): + def __init__(self, root_dir: str, split: str, transform: Any, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + + assert transform is None + + self.tokenizer = tokenizer # 传入tokenizer对象\ + tokenizer.pad_token = tokenizer.eos_token + + self.texts = [] + self.labels = [] + self.max_length = 512 # 设置文本的最大长度 + + # json_file_path = os.path.join(root_dir, f'{split if split != "val" else "dev"}.json') + # anns = read_json(json_file_path) + + # label_map = {'-': 0, '+': 1, 'negative': 0, 'positive': 1} + + ignore_cls_indexes = [classes.index(c) for c in ignore_classes] + + # for v in anns.values(): + # if v['polarity'] not in label_map.keys(): + # continue + + # cls = label_map[v['polarity']] + + # if cls in ignore_cls_indexes: + # continue + + # self.texts += [v['sentence']] + # self.labels += [idx_map[cls] if idx_map is not None else cls] + + if split == 'val': + split = 'test' + data = fetch_20newsgroups(subset=split) + + self.texts = [i for _i, i in enumerate(data['data']) if data['target'][_i] not in ignore_cls_indexes] + self.labels = [i for i in data['target'] if i not in ignore_cls_indexes] + self.labels = [idx_map[i] if idx_map is not None else i for i in self.labels] + + def __len__(self): + return len(self.texts) + + def __getitem__(self, idx): + text = self.texts[idx] + label = self.labels[idx] + + text = aug.augment(text) + + encoded_input = self.tokenizer.encode_plus( + text, max_length=self.max_length, padding="max_length", truncation=True, return_tensors="pt" + ) + + x = {key: tensor.squeeze(0) for key, tensor in encoded_input.items()} + x['return_dict'] = False + return x, torch.tensor(label) + + +from ..ab_dataset import ABDataset +from ..registery import dataset_register + +@dataset_register( + name='Newsgroup-GPT-Aug2', + classes=['alt.atheism', 'comp.graphics', 'comp.os.ms-windows.misc', 'comp.sys.ibm.pc.hardware', 'comp.sys.mac.hardware', 'comp.windows.x', 'misc.forsale', 'rec.autos', 'rec.motorcycles', 'rec.sport.baseball', 'rec.sport.hockey', 'sci.crypt', 'sci.electronics', 'sci.med', 'sci.space', 'soc.religion.christian', 'talk.politics.guns', 'talk.politics.mideast', 'talk.politics.misc', 'talk.religion.misc'], + task_type='Sentiment Classification', + object_type='News', + class_aliases=[], + shift_type=None +) +class NewsgroupGPTAug2(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return NewsgroupDomainsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + + +# @dataset_register( +# name='Newsgroup2-GPT', +# classes=['alt.atheism', 'comp.graphics', 'comp.os.ms-windows.misc', 'comp.sys.ibm.pc.hardware', 'comp.sys.mac.hardware', 'comp.windows.x', 'misc.forsale', 'rec.autos', 'rec.motorcycles', 'rec.sport.baseball', 'rec.sport.hockey', 'sci.crypt', 'sci.electronics', 'sci.med', 'sci.space', 'soc.religion.christian', 'talk.politics.guns', 'talk.politics.mideast', 'talk.politics.misc', 'talk.religion.misc'], +# task_type='Sentiment Classification', +# object_type='News', +# class_aliases=[], +# shift_type=None +# ) +# class Newsgroup2GPT(ABDataset): +# def create_dataset(self, root_dir: str, split: str, transform, +# classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): +# return NewsgroupDomainsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + + +# @dataset_register( +# name='Newsgroup3', +# classes=['alt.atheism', 'comp.graphics', 'comp.os.ms-windows.misc', 'comp.sys.ibm.pc.hardware', 'comp.sys.mac.hardware', 'comp.windows.x', 'misc.forsale', 'rec.autos', 'rec.motorcycles', 'rec.sport.baseball', 'rec.sport.hockey', 'sci.crypt', 'sci.electronics', 'sci.med', 'sci.space', 'soc.religion.christian', 'talk.politics.guns', 'talk.politics.mideast', 'talk.politics.misc', 'talk.religion.misc'], +# task_type='Sentiment Classification', +# object_type='News', +# class_aliases=[], +# shift_type=None +# ) +# class Newsgroup3(ABDataset): +# def create_dataset(self, root_dir: str, split: str, transform, +# classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): +# return NewsgroupDomainsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) \ No newline at end of file diff --git a/data/datasets/sentiment_classification/newsgroup_for_gpt_with_aug3.py b/data/datasets/sentiment_classification/newsgroup_for_gpt_with_aug3.py new file mode 100644 index 0000000000000000000000000000000000000000..c1de22b4a20da7c890fc1c0798df548c6e25f835 --- /dev/null +++ b/data/datasets/sentiment_classification/newsgroup_for_gpt_with_aug3.py @@ -0,0 +1,130 @@ +from sklearn.datasets import fetch_20newsgroups +# from pprint import pprint +# newsgroups_train = fetch_20newsgroups(subset='train') +# print(newsgroups_train.target_names) +# print(newsgroups_train['data'][0]) +import os +import torch +from torch.utils.data import Dataset, DataLoader +from transformers import BertTokenizer, BertForSequenceClassification +from torch.optim import Adam +from torch.nn import CrossEntropyLoss +from typing import Dict, List, Optional, Any +from utils.common.data_record import read_json +# from .global_bert_tokenizer import get_tokenizer + +from transformers import GPT2Tokenizer +tokenizer = GPT2Tokenizer.from_pretrained(f'experiments/elasticdnn/gpt_neo/{os.environ["gpt_neo_series_id"]}') + + +import nlpaug.augmenter.char as nac +import nlpaug.augmenter.word as naw +import nlpaug.augmenter.sentence as nas +import nlpaug.flow as nafc +from nlpaug.util import Action + +aug = naw.RandomWordAug(action="swap") + + + +class NewsgroupDomainsDataset(Dataset): + def __init__(self, root_dir: str, split: str, transform: Any, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + + assert transform is None + + self.tokenizer = tokenizer # 传入tokenizer对象\ + tokenizer.pad_token = tokenizer.eos_token + + self.texts = [] + self.labels = [] + self.max_length = 512 # 设置文本的最大长度 + + # json_file_path = os.path.join(root_dir, f'{split if split != "val" else "dev"}.json') + # anns = read_json(json_file_path) + + # label_map = {'-': 0, '+': 1, 'negative': 0, 'positive': 1} + + ignore_cls_indexes = [classes.index(c) for c in ignore_classes] + + # for v in anns.values(): + # if v['polarity'] not in label_map.keys(): + # continue + + # cls = label_map[v['polarity']] + + # if cls in ignore_cls_indexes: + # continue + + # self.texts += [v['sentence']] + # self.labels += [idx_map[cls] if idx_map is not None else cls] + + if split == 'val': + split = 'test' + data = fetch_20newsgroups(subset=split) + + self.texts = [i for _i, i in enumerate(data['data']) if data['target'][_i] not in ignore_cls_indexes] + self.labels = [i for i in data['target'] if i not in ignore_cls_indexes] + self.labels = [idx_map[i] if idx_map is not None else i for i in self.labels] + + def __len__(self): + return len(self.texts) + + def __getitem__(self, idx): + text = self.texts[idx] + label = self.labels[idx] + + text = aug.augment(text) + + encoded_input = self.tokenizer.encode_plus( + text, max_length=self.max_length, padding="max_length", truncation=True, return_tensors="pt" + ) + + x = {key: tensor.squeeze(0) for key, tensor in encoded_input.items()} + x['return_dict'] = False + return x, torch.tensor(label) + + +from ..ab_dataset import ABDataset +from ..registery import dataset_register + +@dataset_register( + name='Newsgroup-GPT-Aug3', + classes=['alt.atheism', 'comp.graphics', 'comp.os.ms-windows.misc', 'comp.sys.ibm.pc.hardware', 'comp.sys.mac.hardware', 'comp.windows.x', 'misc.forsale', 'rec.autos', 'rec.motorcycles', 'rec.sport.baseball', 'rec.sport.hockey', 'sci.crypt', 'sci.electronics', 'sci.med', 'sci.space', 'soc.religion.christian', 'talk.politics.guns', 'talk.politics.mideast', 'talk.politics.misc', 'talk.religion.misc'], + task_type='Sentiment Classification', + object_type='News', + class_aliases=[], + shift_type=None +) +class NewsgroupGPTAug3(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return NewsgroupDomainsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + + +# @dataset_register( +# name='Newsgroup2-GPT', +# classes=['alt.atheism', 'comp.graphics', 'comp.os.ms-windows.misc', 'comp.sys.ibm.pc.hardware', 'comp.sys.mac.hardware', 'comp.windows.x', 'misc.forsale', 'rec.autos', 'rec.motorcycles', 'rec.sport.baseball', 'rec.sport.hockey', 'sci.crypt', 'sci.electronics', 'sci.med', 'sci.space', 'soc.religion.christian', 'talk.politics.guns', 'talk.politics.mideast', 'talk.politics.misc', 'talk.religion.misc'], +# task_type='Sentiment Classification', +# object_type='News', +# class_aliases=[], +# shift_type=None +# ) +# class Newsgroup2GPT(ABDataset): +# def create_dataset(self, root_dir: str, split: str, transform, +# classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): +# return NewsgroupDomainsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + + +# @dataset_register( +# name='Newsgroup3', +# classes=['alt.atheism', 'comp.graphics', 'comp.os.ms-windows.misc', 'comp.sys.ibm.pc.hardware', 'comp.sys.mac.hardware', 'comp.windows.x', 'misc.forsale', 'rec.autos', 'rec.motorcycles', 'rec.sport.baseball', 'rec.sport.hockey', 'sci.crypt', 'sci.electronics', 'sci.med', 'sci.space', 'soc.religion.christian', 'talk.politics.guns', 'talk.politics.mideast', 'talk.politics.misc', 'talk.religion.misc'], +# task_type='Sentiment Classification', +# object_type='News', +# class_aliases=[], +# shift_type=None +# ) +# class Newsgroup3(ABDataset): +# def create_dataset(self, root_dir: str, split: str, transform, +# classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): +# return NewsgroupDomainsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) \ No newline at end of file diff --git a/data/datasets/sentiment_classification/universal_asc_19_domains.py b/data/datasets/sentiment_classification/universal_asc_19_domains.py new file mode 100644 index 0000000000000000000000000000000000000000..f026bafb9e0e8b423fad38c6ccc9eb3f594c7da7 --- /dev/null +++ b/data/datasets/sentiment_classification/universal_asc_19_domains.py @@ -0,0 +1,336 @@ +import os +import torch +from torch.utils.data import Dataset, DataLoader +from transformers import BertTokenizer, BertForSequenceClassification +from torch.optim import Adam +from torch.nn import CrossEntropyLoss +from typing import Dict, List, Optional, Any +from utils.common.data_record import read_json +from .global_bert_tokenizer import get_tokenizer + + +# 自定义数据集类 +class UniversalASC19DomainsDataset(Dataset): + def __init__(self, root_dir: str, split: str, transform: Any, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + + assert transform is None + + self.tokenizer = get_tokenizer() # 传入tokenizer对象 + self.texts = [] + self.labels = [] + self.max_length = 512 # 设置文本的最大长度 + + json_file_path = os.path.join(root_dir, f'{split if split != "val" else "dev"}.json') + anns = read_json(json_file_path) + + label_map = {'-': 0, '+': 1, 'negative': 0, 'positive': 1} + + ignore_cls_indexes = [classes.index(c) for c in ignore_classes] + + for v in anns.values(): + if v['polarity'] not in label_map.keys(): + continue + + cls = label_map[v['polarity']] + + if cls in ignore_cls_indexes: + continue + + self.texts += [v['sentence']] + self.labels += [idx_map[cls] if idx_map is not None else cls] + + def __len__(self): + return len(self.texts) + + def __getitem__(self, idx): + text = self.texts[idx] + label = self.labels[idx] + + encoded_input = self.tokenizer.encode_plus( + text, max_length=self.max_length, padding="max_length", truncation=True, return_tensors="pt" + ) + + x = {key: tensor.squeeze(0) for key, tensor in encoded_input.items()} + x['return_dict'] = False + return x, torch.tensor(label) + + +from ..ab_dataset import ABDataset +from ..registery import dataset_register + +@dataset_register( + name='HL5Domains-ApexAD2600Progressive', + classes=['negative', 'positive'], + task_type='Sentiment Classification', + object_type='Emotion', + class_aliases=[], + shift_type=None +) +class HL5Domains_ApexAD2600Progressive(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + +@dataset_register( + name='HL5Domains-CanonG3', + classes=['negative', 'positive'], + task_type='Sentiment Classification', + object_type='Emotion', + class_aliases=[], + shift_type=None +) +class HL5Domains_CanonG3(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + +@dataset_register( + name='HL5Domains-CreativeLabsNomadJukeboxZenXtra40GB', + classes=['negative', 'positive'], + task_type='Sentiment Classification', + object_type='Emotion', + class_aliases=[], + shift_type=None +) +class HL5Domains_CreativeLabsNomadJukeboxZenXtra40GB(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + +@dataset_register( + name='HL5Domains-NikonCoolpix4300', + classes=['negative', 'positive'], + task_type='Sentiment Classification', + object_type='Emotion', + class_aliases=[], + shift_type=None +) +class HL5Domains_NikonCoolpix4300(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + +@dataset_register( + name='HL5Domains-Nokia6610', + classes=['negative', 'positive'], + task_type='Sentiment Classification', + object_type='Emotion', + class_aliases=[], + shift_type=None +) +class HL5Domains_Nokia6610(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + + +@dataset_register( + name='Liu3Domains-Computer', + classes=['negative', 'positive'], + task_type='Sentiment Classification', + object_type='Emotion', + class_aliases=[], + shift_type=None +) +class Liu3Domains_Computer(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + +@dataset_register( + name='Liu3Domains-Router', + classes=['negative', 'positive'], + task_type='Sentiment Classification', + object_type='Emotion', + class_aliases=[], + shift_type=None +) +class Liu3Domains_Router(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + +@dataset_register( + name='Liu3Domains-Speaker', + classes=['negative', 'positive'], + task_type='Sentiment Classification', + object_type='Emotion', + class_aliases=[], + shift_type=None +) +class Liu3Domains_Speaker(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + + +# import os +# for domain in os.listdir('/data/zql/datasets/nlp_asc_19_domains/dat/absa/Bing9Domains/asc'): +# print(f""" +# @dataset_register( +# name='Ding9Domains-{domain}', +# classes=['negative', 'positive'], +# task_type='Sentiment Classification', +# object_type='Emotion', +# class_aliases=[], +# shift_type=None +# ) +# class Ding9Domains_{domain}(ABDataset): +# def create_dataset(self, root_dir: str, split: str, transform, +# classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): +# return UniversalASC19DomainsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) +# """) + +@dataset_register( + name='Ding9Domains-DiaperChamp', + classes=['negative', 'positive'], + task_type='Sentiment Classification', + object_type='Emotion', + class_aliases=[], + shift_type=None +) +class Ding9Domains_DiaperChamp(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + + +@dataset_register( + name='Ding9Domains-Norton', + classes=['negative', 'positive'], + task_type='Sentiment Classification', + object_type='Emotion', + class_aliases=[], + shift_type=None +) +class Ding9Domains_Norton(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + + +@dataset_register( + name='Ding9Domains-LinksysRouter', + classes=['negative', 'positive'], + task_type='Sentiment Classification', + object_type='Emotion', + class_aliases=[], + shift_type=None +) +class Ding9Domains_LinksysRouter(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + + +@dataset_register( + name='Ding9Domains-MicroMP3', + classes=['negative', 'positive'], + task_type='Sentiment Classification', + object_type='Emotion', + class_aliases=[], + shift_type=None +) +class Ding9Domains_MicroMP3(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + + +@dataset_register( + name='Ding9Domains-Nokia6600', + classes=['negative', 'positive'], + task_type='Sentiment Classification', + object_type='Emotion', + class_aliases=[], + shift_type=None +) +class Ding9Domains_Nokia6600(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + + +@dataset_register( + name='Ding9Domains-CanonPowerShotSD500', + classes=['negative', 'positive'], + task_type='Sentiment Classification', + object_type='Emotion', + class_aliases=[], + shift_type=None +) +class Ding9Domains_CanonPowerShotSD500(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + + +@dataset_register( + name='Ding9Domains-ipod', + classes=['negative', 'positive'], + task_type='Sentiment Classification', + object_type='Emotion', + class_aliases=[], + shift_type=None +) +class Ding9Domains_ipod(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + + +@dataset_register( + name='Ding9Domains-HitachiRouter', + classes=['negative', 'positive'], + task_type='Sentiment Classification', + object_type='Emotion', + class_aliases=[], + shift_type=None +) +class Ding9Domains_HitachiRouter(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + + +@dataset_register( + name='Ding9Domains-CanonS100', + classes=['negative', 'positive'], + task_type='Sentiment Classification', + object_type='Emotion', + class_aliases=[], + shift_type=None +) +class Ding9Domains_CanonS100(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + + + +@dataset_register( + name='SemEval-Laptop', + classes=['negative', 'positive'], + task_type='Sentiment Classification', + object_type='Emotion', + class_aliases=[], + shift_type=None +) +class SemEval_Laptop(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + +@dataset_register( + name='SemEval-Rest', + classes=['negative', 'positive'], + task_type='Sentiment Classification', + object_type='Emotion', + class_aliases=[], + shift_type=None +) +class SemEval_Rest(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) \ No newline at end of file diff --git a/data/datasets/sentiment_classification/universal_asc_19_domains_for_gpt.py b/data/datasets/sentiment_classification/universal_asc_19_domains_for_gpt.py new file mode 100644 index 0000000000000000000000000000000000000000..ee49ec35584b9eede6211f09762b194dfbfa6407 --- /dev/null +++ b/data/datasets/sentiment_classification/universal_asc_19_domains_for_gpt.py @@ -0,0 +1,341 @@ +import os +import torch +from torch.utils.data import Dataset, DataLoader +from transformers import BertTokenizer, BertForSequenceClassification +from torch.optim import Adam +from torch.nn import CrossEntropyLoss +from typing import Dict, List, Optional, Any +from utils.common.data_record import read_json +# from .global_bert_tokenizer import get_tokenizer + +from transformers import GPT2Tokenizer +tokenizer = GPT2Tokenizer.from_pretrained(f'experiments/elasticdnn/gpt_neo/{os.environ["gpt_neo_series_id"]}') + + +# 自定义数据集类 +class UniversalASC19DomainsDataset(Dataset): + def __init__(self, root_dir: str, split: str, transform: Any, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + + assert transform is None + + self.tokenizer = tokenizer # 传入tokenizer对象 + tokenizer.pad_token = tokenizer.eos_token + + self.texts = [] + self.labels = [] + self.max_length = 512 # 设置文本的最大长度 + + json_file_path = os.path.join(root_dir, f'{split if split != "val" else "dev"}.json') + anns = read_json(json_file_path) + + label_map = {'-': 0, '+': 1, 'negative': 0, 'positive': 1} + + ignore_cls_indexes = [classes.index(c) for c in ignore_classes] + + for v in anns.values(): + if v['polarity'] not in label_map.keys(): + continue + + cls = label_map[v['polarity']] + + if cls in ignore_cls_indexes: + continue + + self.texts += [v['sentence']] + self.labels += [idx_map[cls] if idx_map is not None else cls] + + def __len__(self): + return len(self.texts) + + def __getitem__(self, idx): + text = self.texts[idx] + label = self.labels[idx] + + encoded_input = self.tokenizer.encode_plus( + text, max_length=self.max_length, padding="max_length", truncation=True, return_tensors="pt" + ) + + x = {key: tensor.squeeze(0) for key, tensor in encoded_input.items()} + x['return_dict'] = False + return x, torch.tensor(label) + + +from ..ab_dataset import ABDataset +from ..registery import dataset_register + +@dataset_register( + name='HL5Domains-ApexAD2600Progressive-GPTNeo', + classes=['negative', 'positive'], + task_type='Sentiment Classification', + object_type='Emotion', + class_aliases=[], + shift_type=None +) +class HL5Domains_ApexAD2600Progressive_GPTNeo(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + +@dataset_register( + name='HL5Domains-CanonG3-GPTNeo', + classes=['negative', 'positive'], + task_type='Sentiment Classification', + object_type='Emotion', + class_aliases=[], + shift_type=None +) +class HL5Domains_CanonG3_GPTNeo(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + +@dataset_register( + name='HL5Domains-CreativeLabsNomadJukeboxZenXtra40GB-GPTNeo', + classes=['negative', 'positive'], + task_type='Sentiment Classification', + object_type='Emotion', + class_aliases=[], + shift_type=None +) +class HL5Domains_CreativeLabsNomadJukeboxZenXtra40GB_GPTNeo(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + +@dataset_register( + name='HL5Domains-NikonCoolpix4300-GPTNeo', + classes=['negative', 'positive'], + task_type='Sentiment Classification', + object_type='Emotion', + class_aliases=[], + shift_type=None +) +class HL5Domains_NikonCoolpix4300_GPTNeo(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + +@dataset_register( + name='HL5Domains-Nokia6610-GPTNeo', + classes=['negative', 'positive'], + task_type='Sentiment Classification', + object_type='Emotion', + class_aliases=[], + shift_type=None +) +class HL5Domains_Nokia6610_GPTNeo(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + + +@dataset_register( + name='Liu3Domains-Computer-GPTNeo', + classes=['negative', 'positive'], + task_type='Sentiment Classification', + object_type='Emotion', + class_aliases=[], + shift_type=None +) +class Liu3Domains_Computer_GPTNeo(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + +@dataset_register( + name='Liu3Domains-Router-GPTNeo', + classes=['negative', 'positive'], + task_type='Sentiment Classification', + object_type='Emotion', + class_aliases=[], + shift_type=None +) +class Liu3Domains_Router_GPTNeo(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + +@dataset_register( + name='Liu3Domains-Speaker-GPTNeo', + classes=['negative', 'positive'], + task_type='Sentiment Classification', + object_type='Emotion', + class_aliases=[], + shift_type=None +) +class Liu3Domains_Speaker_GPTNeo(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + + +# import os +# for domain in os.listdir('/data/zql/datasets/nlp_asc_19_domains/dat/absa/Bing9Domains/asc'): +# print(f""" +# @dataset_register( +# name='Ding9Domains-{domain}', +# classes=['negative', 'positive'], +# task_type='Sentiment Classification', +# object_type='Emotion', +# class_aliases=[], +# shift_type=None +# ) +# class Ding9Domains_{domain}(ABDataset): +# def create_dataset(self, root_dir: str, split: str, transform, +# classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): +# return UniversalASC19DomainsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) +# """) + +@dataset_register( + name='Ding9Domains-DiaperChamp-GPTNeo', + classes=['negative', 'positive'], + task_type='Sentiment Classification', + object_type='Emotion', + class_aliases=[], + shift_type=None +) +class Ding9Domains_DiaperChamp_GPTNeo(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + + +@dataset_register( + name='Ding9Domains-Norton-GPTNeo', + classes=['negative', 'positive'], + task_type='Sentiment Classification', + object_type='Emotion', + class_aliases=[], + shift_type=None +) +class Ding9Domains_Norton_GPTNeo(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + + +@dataset_register( + name='Ding9Domains-LinksysRouter-GPTNeo', + classes=['negative', 'positive'], + task_type='Sentiment Classification', + object_type='Emotion', + class_aliases=[], + shift_type=None +) +class Ding9Domains_LinksysRouter_GPTNeo(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + + +@dataset_register( + name='Ding9Domains-MicroMP3-GPTNeo', + classes=['negative', 'positive'], + task_type='Sentiment Classification', + object_type='Emotion', + class_aliases=[], + shift_type=None +) +class Ding9Domains_MicroMP3_GPTNeo(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + + +@dataset_register( + name='Ding9Domains-Nokia6600-GPTNeo', + classes=['negative', 'positive'], + task_type='Sentiment Classification', + object_type='Emotion', + class_aliases=[], + shift_type=None +) +class Ding9Domains_Nokia6600_GPTNeo(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + + +@dataset_register( + name='Ding9Domains-CanonPowerShotSD500-GPTNeo', + classes=['negative', 'positive'], + task_type='Sentiment Classification', + object_type='Emotion', + class_aliases=[], + shift_type=None +) +class Ding9Domains_CanonPowerShotSD500_GPTNeo(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + + +@dataset_register( + name='Ding9Domains-ipod-GPTNeo', + classes=['negative', 'positive'], + task_type='Sentiment Classification', + object_type='Emotion', + class_aliases=[], + shift_type=None +) +class Ding9Domains_ipod_GPTNeo(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + + +@dataset_register( + name='Ding9Domains-HitachiRouter-GPTNeo', + classes=['negative', 'positive'], + task_type='Sentiment Classification', + object_type='Emotion', + class_aliases=[], + shift_type=None +) +class Ding9Domains_HitachiRouter_GPTNeo(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + + +@dataset_register( + name='Ding9Domains-CanonS100-GPTNeo', + classes=['negative', 'positive'], + task_type='Sentiment Classification', + object_type='Emotion', + class_aliases=[], + shift_type=None +) +class Ding9Domains_CanonS100_GPTNeo(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + + + +@dataset_register( + name='SemEval-Laptop-GPTNeo', + classes=['negative', 'positive'], + task_type='Sentiment Classification', + object_type='Emotion', + class_aliases=[], + shift_type=None +) +class SemEval_Laptop_GPTNeo(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) + +@dataset_register( + name='SemEval-Rest-GPTNeo', + classes=['negative', 'positive'], + task_type='Sentiment Classification', + object_type='Emotion', + class_aliases=[], + shift_type=None +) +class SemEval_Rest_GPTNeo(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return UniversalASC19DomainsDataset(root_dir, split, transform, classes, ignore_classes, idx_map) \ No newline at end of file diff --git a/data/datasets/text_generation/__init__.py b/data/datasets/text_generation/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..622f54de9ad8214a930b617f40b4e666f1ea075a --- /dev/null +++ b/data/datasets/text_generation/__init__.py @@ -0,0 +1,5 @@ +from .alpaca import * +from .no_robots import * +from .law_task import * +from .finance_task import * +from .medicine_task import * \ No newline at end of file diff --git a/data/datasets/text_generation/__pycache__/__init__.cpython-38.pyc b/data/datasets/text_generation/__pycache__/__init__.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..93af7994a954a1be692e179053cc7fb22ad27cd0 Binary files /dev/null and b/data/datasets/text_generation/__pycache__/__init__.cpython-38.pyc differ diff --git a/data/datasets/text_generation/__pycache__/alpaca.cpython-38.pyc b/data/datasets/text_generation/__pycache__/alpaca.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4f2dc071357c6f63c6f7c3ef5d506cd3c245b7bc Binary files /dev/null and b/data/datasets/text_generation/__pycache__/alpaca.cpython-38.pyc differ diff --git a/data/datasets/text_generation/__pycache__/finance_task.cpython-38.pyc b/data/datasets/text_generation/__pycache__/finance_task.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..46f96a811cd1d9a5a45a2a364f2cb78b33583018 Binary files /dev/null and b/data/datasets/text_generation/__pycache__/finance_task.cpython-38.pyc differ diff --git a/data/datasets/text_generation/__pycache__/law_task.cpython-38.pyc b/data/datasets/text_generation/__pycache__/law_task.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0b046bfa3c14d198038e5953292357c59b7e9d1c Binary files /dev/null and b/data/datasets/text_generation/__pycache__/law_task.cpython-38.pyc differ diff --git a/data/datasets/text_generation/__pycache__/medicine_task.cpython-38.pyc b/data/datasets/text_generation/__pycache__/medicine_task.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d384f301f288473604e69c73634195c7ee741707 Binary files /dev/null and b/data/datasets/text_generation/__pycache__/medicine_task.cpython-38.pyc differ diff --git a/data/datasets/text_generation/__pycache__/no_robots.cpython-38.pyc b/data/datasets/text_generation/__pycache__/no_robots.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..069da98ec44cf15cdb5e266c3d03a0074245ba24 Binary files /dev/null and b/data/datasets/text_generation/__pycache__/no_robots.cpython-38.pyc differ diff --git a/data/datasets/text_generation/alpaca.py b/data/datasets/text_generation/alpaca.py new file mode 100644 index 0000000000000000000000000000000000000000..9b8885fe48bc7972b29d9e026df51be1005fc38f --- /dev/null +++ b/data/datasets/text_generation/alpaca.py @@ -0,0 +1,111 @@ +import os +import torch +from torch.utils.data import Dataset, DataLoader +from transformers import BertTokenizer, BertForSequenceClassification +from torch.optim import Adam +from torch.nn import CrossEntropyLoss +from typing import Dict, List, Optional, Any +from utils.common.data_record import read_json +# from .global_bert_tokenizer import get_tokenizer + +from transformers import GPT2Tokenizer +# gpt_neo_series_id = '1.3B_ckpt' +# os.environ['gpt_neo_series_id'] = gpt_neo_series_id + +class AlpacaBase(Dataset): + def __init__(self, root_dir: str, split: str, transform: Any, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + + assert transform is None + + self.tokenizer = GPT2Tokenizer.from_pretrained(f'experiments/elasticdnn/gpt_neo/{os.environ["gpt_neo_series_id"]}') # 传入tokenizer对象 + self.tokenizer.pad_token = self.tokenizer.eos_token + + self.texts = [] + self.labels = [] + self.idx_map = [] + self.ignore_classes = [] + self.max_length = 768 # 设置文本的最大长度 + + json_file_path = os.path.join(root_dir, f'data.json') + anns = read_json(json_file_path) + + for v in anns: + if len(v['input']) != 0: + txt = f"""Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request. + +### Instruction: +{v['instruction']} + +### Input: +{v['input']} + +### Response: +""" + else: + txt = f"""Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request. + +### Instruction: +{v['instruction']} + +### Response: +""" + self.texts.append(txt) + self.labels.append(v['output']) + + def __len__(self): + return len(self.texts) + + def setSplit(self, split): + self.split = split + + def __getitem__(self, idx): + if self.split == 'val': + self.tokenizer.padding_side = "left" + text = self.texts[idx] + label = self.labels[idx] + + encoded_input = self.tokenizer.encode_plus( + text, max_length=self.max_length - 128, padding="max_length", truncation=True, return_tensors="pt" + ) + label = self.tokenizer.encode_plus( + label, max_length=128, padding="max_length", truncation=True, return_tensors="pt" + ) + x = {key: tensor.squeeze(0) for key, tensor in encoded_input.items()} + y = label['input_ids'].squeeze(0) + x['labels'] = y + x['len'] = len(text) + x['return_dict'] = False + return x + else: + self.tokenizer.padding_side = "right" + text = self.texts[idx] + self.labels[idx] + self.tokenizer.eos_token + + encoded_input = self.tokenizer.encode_plus( + text, max_length=self.max_length, padding="max_length", truncation=True, return_tensors="pt" + ) + x = {key: tensor.squeeze(0) for key, tensor in encoded_input.items()} + x['labels'] = x['input_ids'].clone() + x['labels'][x['labels'] == self.tokenizer.pad_token_id] = -100 + for i, v in enumerate(x['labels']): + if v == -100: + x['labels'][i] = self.tokenizer.eos_token_id + break + x['return_dict'] = False + return x + +from ..ab_dataset import ABDataset +from ..registery import dataset_register + +@dataset_register( + name='Alpaca', + classes=[], + task_type='Text Generation', + object_type=None, + class_aliases=[], + shift_type=None +) +class Alpaca(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return AlpacaBase(root_dir, split, transform, classes, ignore_classes, idx_map) \ No newline at end of file diff --git a/data/datasets/text_generation/finance_task.py b/data/datasets/text_generation/finance_task.py new file mode 100644 index 0000000000000000000000000000000000000000..3935d99b8394c1e4c853130d9908d4b07661dd2d --- /dev/null +++ b/data/datasets/text_generation/finance_task.py @@ -0,0 +1,122 @@ +from datasets import load_dataset +import os +import torch +from torch.utils.data import Dataset, DataLoader +from transformers import BertTokenizer, BertForSequenceClassification +from torch.optim import Adam +from torch.nn import CrossEntropyLoss +from typing import Dict, List, Optional, Any +from utils.common.data_record import read_json +from itertools import chain +import random +import json +# from .global_bert_tokenizer import get_tokenizer + +from transformers import GPT2Tokenizer +# gpt_neo_series_id = '1.3B_ckpt' +# os.environ['gpt_neo_series_id'] = gpt_neo_series_id +class Finance_taskbase(Dataset): + def __init__(self, root_dir: str, split: str, transform: Any, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + + assert transform is None + rate = 0.8 + self.tokenizer = GPT2Tokenizer.from_pretrained(f'experiments/elasticdnn/gpt_neo/{os.environ["gpt_neo_series_id"]}') + special_tokens = {"pad_token":"<|pad|>"}#, "sep_token":"<|sep|>", "bos_token":"<|bos|>"} + self.tokenizer.add_special_tokens(special_tokens) + self.tokenizer.pad_token = "<|pad|>" # 传入tokenizer对象 + # self.tokenizer.pad_token = self.tokenizer.eos_token + self.tokenizer.sep_token = self.tokenizer.eos_token + self.msgs = [] + self.idx_map = [] + self.ignore_classes = [] + self.max_length = 768 # 设置文本的最大长度 + self.split = split + json_file_path = os.path.join(root_dir, f'{split}.json') + if not os.path.exists(json_file_path): + anns = read_json(os.path.join(root_dir, f'data.json')) + random.shuffle(anns) + train_anns = anns[:int(len(anns) * rate)] + test_anns = anns[int(len(anns) * rate):] + train_file_path = os.path.join(root_dir, f'train.json') + test_file_path = os.path.join(root_dir, f'val.json') + with open(train_file_path, 'w') as f: + json.dump(train_anns, f) + with open(test_file_path, 'w') as f: + json.dump(test_anns, f) + + anns = read_json(json_file_path) + self.questions = [] + self.answers = [] + + for line in anns: + quest = line['input'] + " " + ans = line['options'][line['gold_index']] + self.questions.append(quest) + self.answers.append(ans) + + def __len__(self): + return len(self.questions) + + def __getitem__(self, idx): + bos, eos, pad, sep = self.tokenizer.bos_token_id, self.tokenizer.eos_token_id, self.tokenizer.pad_token_id, self.tokenizer.sep_token_id + if self.split == 'val': + self.tokenizer.padding_side = "left" + input_ids = [] + labels = [] + input_ids = self.tokenizer.encode(self.questions[idx]) + if len(input_ids) > self.max_length - 128: + return {'return_dict': True} + leng = len(self.tokenizer.decode(input_ids)) + input_ids = [pad] * (self.max_length - 128 - len(input_ids)) + input_ids + labels = self.tokenizer.encode(self.answers[idx], max_length=128, padding="max_length", truncation=True) + if len(labels) > 128: + return {'return_dict': True} + x = { + "input_ids": torch.tensor(input_ids), + "labels": torch.tensor(labels), + 'return_dict': True, + 'len': leng + } + return x + else: + self.tokenizer.padding_side = "right" + input_ids = [] + labels = [] + input_ids = self.tokenizer.encode(self.questions[idx]) + labels = [-100] * len(input_ids) + self.tokenizer.encode(self.answers[idx]) + [eos] + # labels = input_ids + self.tokenizer.encode(target) + [eos] + input_ids += self.tokenizer.encode(self.answers[idx]) + [eos] + + if len(input_ids) > self.max_length: + return {'return_dict': True} + attention_mask = [1] * len(input_ids) + [0] * (self.max_length - len(input_ids)) + # labels = [[-100] * (len(token_type_ids) - len(self.tokenizer.encode(target)) - 1)] + [self.tokenizer.encode(target)] + [[eos]] + labels += [-100] * (self.max_length - len(input_ids)) + input_ids += [pad] * (self.max_length - len(input_ids)) + x = { + "input_ids": torch.tensor(input_ids), + "attention_mask": torch.tensor(attention_mask), + "labels": torch.tensor(labels), + 'return_dict': True + } + return x + +from ..ab_dataset import ABDataset +from ..registery import dataset_register + +@dataset_register( + name='Finance_task', + classes=['None'], + task_type='Text Generation', + object_type=None, + class_aliases=[], + shift_type=None +) +class Finance_task(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return Finance_taskbase(root_dir, split, transform, classes, ignore_classes, idx_map) + +# a = Finance_taskbase('/data/zql/datasets/finance_task', 'train', None, None, None, None) +# a.__getitem__(0) \ No newline at end of file diff --git a/data/datasets/text_generation/law_task.py b/data/datasets/text_generation/law_task.py new file mode 100644 index 0000000000000000000000000000000000000000..288eceaccf5d9ff6e708073e6aad082fa2551a09 --- /dev/null +++ b/data/datasets/text_generation/law_task.py @@ -0,0 +1,123 @@ +from datasets import load_dataset +import os +import torch +from torch.utils.data import Dataset, DataLoader +from transformers import BertTokenizer, BertForSequenceClassification +from torch.optim import Adam +from torch.nn import CrossEntropyLoss +from typing import Dict, List, Optional, Any +from utils.common.data_record import read_json +from itertools import chain +import random +import json +# from .global_bert_tokenizer import get_tokenizer + +from transformers import GPT2Tokenizer +# gpt_neo_series_id = '1.3B_ckpt' +# os.environ['gpt_neo_series_id'] = gpt_neo_series_id +class Law_taskbase(Dataset): + def __init__(self, root_dir: str, split: str, transform: Any, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + + assert transform is None + rate = 0.8 + self.tokenizer = GPT2Tokenizer.from_pretrained(f'experiments/elasticdnn/gpt_neo/{os.environ["gpt_neo_series_id"]}') + special_tokens = {"pad_token":"<|pad|>"}#, "sep_token":"<|sep|>", "bos_token":"<|bos|>"} + self.tokenizer.add_special_tokens(special_tokens) + self.tokenizer.pad_token = "<|pad|>" # 传入tokenizer对象 + # self.tokenizer.pad_token = self.tokenizer.eos_token + self.tokenizer.sep_token = self.tokenizer.eos_token + self.msgs = [] + self.idx_map = [] + self.ignore_classes = [] + self.max_length = 768 # 设置文本的最大长度 + self.split = split + json_file_path = os.path.join(root_dir, f'{split}.json') + if not os.path.exists(json_file_path): + anns = read_json(os.path.join(root_dir, f'data.json')) + random.shuffle(anns) + train_anns = anns[:int(len(anns) * rate)] + test_anns = anns[int(len(anns) * rate):] + train_file_path = os.path.join(root_dir, f'train.json') + test_file_path = os.path.join(root_dir, f'val.json') + with open(train_file_path, 'w') as f: + json.dump(train_anns, f) + with open(test_file_path, 'w') as f: + json.dump(test_anns, f) + + anns = read_json(json_file_path) + self.questions = [] + self.answers = [] + + for line in anns: + tmp = line['output'].split(' ') + quest = line['input_options'][line['gold_index']] + tmp[0] + ans = ' '.join(tmp[1:]) + self.questions.append(quest) + self.answers.append(ans) + + def __len__(self): + return len(self.questions) + + def __getitem__(self, idx): + bos, eos, pad, sep = self.tokenizer.bos_token_id, self.tokenizer.eos_token_id, self.tokenizer.pad_token_id, self.tokenizer.sep_token_id + if self.split == 'val': + self.tokenizer.padding_side = "left" + input_ids = [] + labels = [] + input_ids = self.tokenizer.encode("Q: ") + self.tokenizer.encode(self.questions[idx] + '\n\n') + self.tokenizer.encode("A: ") + if len(input_ids) > self.max_length - 128: + return {'return_dict': True} + leng = len(self.tokenizer.decode(input_ids)) + input_ids = [pad] * (self.max_length - 128 - len(input_ids)) + input_ids + labels = self.tokenizer.encode(self.answers[idx], max_length=128, padding="max_length", truncation=True) + if len(labels) > 128: + return {'return_dict': True} + x = { + "input_ids": torch.tensor(input_ids), + "labels": torch.tensor(labels), + 'return_dict': True, + 'len': leng + } + return x + else: + self.tokenizer.padding_side = "right" + input_ids = [] + labels = [] + input_ids = self.tokenizer.encode("Q: ") + self.tokenizer.encode(self.questions[idx] + '\n\n') + self.tokenizer.encode("A: ") + labels = [-100] * len(input_ids) + self.tokenizer.encode(self.answers[idx]) + [eos] + # labels = input_ids + self.tokenizer.encode(target) + [eos] + input_ids += self.tokenizer.encode(self.answers[idx]) + [eos] + + if len(input_ids) > self.max_length: + return {'return_dict': True} + attention_mask = [1] * len(input_ids) + [0] * (self.max_length - len(input_ids)) + # labels = [[-100] * (len(token_type_ids) - len(self.tokenizer.encode(target)) - 1)] + [self.tokenizer.encode(target)] + [[eos]] + labels += [-100] * (self.max_length - len(input_ids)) + input_ids += [pad] * (self.max_length - len(input_ids)) + x = { + "input_ids": torch.tensor(input_ids), + "attention_mask": torch.tensor(attention_mask), + "labels": torch.tensor(labels), + 'return_dict': True + } + return x + +from ..ab_dataset import ABDataset +from ..registery import dataset_register + +@dataset_register( + name='Law_task', + classes=['None'], + task_type='Text Generation', + object_type=None, + class_aliases=[], + shift_type=None +) +class Law_task(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return Law_taskbase(root_dir, split, transform, classes, ignore_classes, idx_map) + +# a = Law_taskbase('/data/zql/datasets/law_task', 'val', None, None, None, None) +# a.__getitem__(0) \ No newline at end of file diff --git a/data/datasets/text_generation/medicine_task.py b/data/datasets/text_generation/medicine_task.py new file mode 100644 index 0000000000000000000000000000000000000000..5fee54c18c49ebfce2ac87b7542413b435f92943 --- /dev/null +++ b/data/datasets/text_generation/medicine_task.py @@ -0,0 +1,130 @@ +from datasets import load_dataset +import os +import torch +from torch.utils.data import Dataset, DataLoader +from transformers import BertTokenizer, BertForSequenceClassification +from torch.optim import Adam +from torch.nn import CrossEntropyLoss +from typing import Dict, List, Optional, Any +from utils.common.data_record import read_json +from itertools import chain +import random +import json +# from .global_bert_tokenizer import get_tokenizer + +from transformers import GPT2Tokenizer +# gpt_neo_series_id = '1.3B_ckpt' +# os.environ['gpt_neo_series_id'] = gpt_neo_series_id +class Medicine_taskbase(Dataset): + def __init__(self, root_dir: str, split: str, transform: Any, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + + assert transform is None + rate = 0.8 + self.tokenizer = GPT2Tokenizer.from_pretrained(f'experiments/elasticdnn/gpt_neo/{os.environ["gpt_neo_series_id"]}') + special_tokens = {"pad_token":"<|pad|>"}#, "sep_token":"<|sep|>", "bos_token":"<|bos|>"} + self.tokenizer.add_special_tokens(special_tokens) + self.tokenizer.pad_token = "<|pad|>" # 传入tokenizer对象 + # self.tokenizer.pad_token = self.tokenizer.eos_token + self.tokenizer.sep_token = self.tokenizer.eos_token + self.msgs = [] + self.idx_map = [] + self.ignore_classes = [] + self.max_length = 768 # 设置文本的最大长度 + self.split = split + json_file_path = os.path.join(root_dir, f'{split}.json') + if not os.path.exists(json_file_path): + anns = read_json(os.path.join(root_dir, f'data.json')) + random.shuffle(anns) + train_anns = anns[:int(len(anns) * rate)] + test_anns = anns[int(len(anns) * rate):] + train_file_path = os.path.join(root_dir, f'train.json') + test_file_path = os.path.join(root_dir, f'val.json') + with open(train_file_path, 'w') as f: + json.dump(train_anns, f) + with open(test_file_path, 'w') as f: + json.dump(test_anns, f) + + anns = read_json(json_file_path) + self.questions = [] + self.answers = [] + + for line in anns: + quest = line['input'] + if 'Q: ' in quest: + quest = quest.replace('Q: ', '') + quest = quest.replace('A:', '') + if 'Question: ' in quest: + quest = quest.replace('Question: ', '') + quest = quest.replace('Answer:', '') + quest = quest.strip('\n') + "\nOptions:\n" + '\n'.join(line['options']) + + ans = line['options'][line['gold_index']] + self.questions.append(quest) + self.answers.append(ans) + + def __len__(self): + return len(self.questions) + + def __getitem__(self, idx): + bos, eos, pad, sep = self.tokenizer.bos_token_id, self.tokenizer.eos_token_id, self.tokenizer.pad_token_id, self.tokenizer.sep_token_id + if self.split == 'val': + self.tokenizer.padding_side = "left" + input_ids = [] + labels = [] + input_ids = self.tokenizer.encode("Q: ") + self.tokenizer.encode(self.questions[idx] + '\n\n') + self.tokenizer.encode("A: ") + if len(input_ids) > self.max_length - 128: + return {'return_dict': True} + leng = len(self.tokenizer.decode(input_ids)) + input_ids = [pad] * (self.max_length - 128 - len(input_ids)) + input_ids + labels = self.tokenizer.encode(self.answers[idx], max_length=128, padding="max_length", truncation=True) + if len(labels) > 128: + return {'return_dict': True} + x = { + "input_ids": torch.tensor(input_ids), + "labels": torch.tensor(labels), + 'return_dict': True, + 'len': leng + } + return x + else: + self.tokenizer.padding_side = "right" + input_ids = [] + labels = [] + input_ids = self.tokenizer.encode("Q: ") + self.tokenizer.encode(self.questions[idx] + '\n\n') + self.tokenizer.encode("A: ") + labels = [-100] * len(input_ids) + self.tokenizer.encode(self.answers[idx]) + [eos] + # labels = input_ids + self.tokenizer.encode(target) + [eos] + input_ids += self.tokenizer.encode(self.answers[idx]) + [eos] + + if len(input_ids) > self.max_length: + return {'return_dict': True} + attention_mask = [1] * len(input_ids) + [0] * (self.max_length - len(input_ids)) + # labels = [[-100] * (len(token_type_ids) - len(self.tokenizer.encode(target)) - 1)] + [self.tokenizer.encode(target)] + [[eos]] + labels += [-100] * (self.max_length - len(input_ids)) + input_ids += [pad] * (self.max_length - len(input_ids)) + x = { + "input_ids": torch.tensor(input_ids), + "attention_mask": torch.tensor(attention_mask), + "labels": torch.tensor(labels), + 'return_dict': True + } + return x + +from ..ab_dataset import ABDataset +from ..registery import dataset_register + +@dataset_register( + name='Medicine_task', + classes=['None'], + task_type='Text Generation', + object_type=None, + class_aliases=[], + shift_type=None +) +class Medicine_task(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return Medicine_taskbase(root_dir, split, transform, classes, ignore_classes, idx_map) + +# a = Medicine_taskbase('/data/zql/datasets/medicine_task', 'train', None, None, None, None) +# a.__getitem__(0) \ No newline at end of file diff --git a/data/datasets/text_generation/no_robots.py b/data/datasets/text_generation/no_robots.py new file mode 100644 index 0000000000000000000000000000000000000000..564303261b715420e65768813b0b9d28e1db471f --- /dev/null +++ b/data/datasets/text_generation/no_robots.py @@ -0,0 +1,130 @@ +from datasets import load_dataset +import os +import torch +from torch.utils.data import Dataset, DataLoader +from transformers import BertTokenizer, BertForSequenceClassification +from torch.optim import Adam +from torch.nn import CrossEntropyLoss +from typing import Dict, List, Optional, Any +from utils.common.data_record import read_json +from itertools import chain +# from .global_bert_tokenizer import get_tokenizer + +from transformers import GPT2Tokenizer +# gpt_neo_series_id = '1.3B_ckpt' +# os.environ['gpt_neo_series_id'] = gpt_neo_series_id +class No_Robotsbase(Dataset): + def __init__(self, root_dir: str, split: str, transform: Any, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + + assert transform is None + + self.tokenizer = GPT2Tokenizer.from_pretrained(f'experiments/elasticdnn/gpt_neo/{os.environ["gpt_neo_series_id"]}') + special_tokens = {"pad_token":"<|pad|>"}#, "sep_token":"<|sep|>", "bos_token":"<|bos|>"} + self.tokenizer.add_special_tokens(special_tokens) + self.tokenizer.pad_token = "<|pad|>" # 传入tokenizer对象 + # self.tokenizer.pad_token = self.tokenizer.eos_token + self.tokenizer.sep_token = self.tokenizer.eos_token + self.msgs = [] + self.idx_map = [] + self.ignore_classes = [] + self.max_length = 768 # 设置文本的最大长度 + self.split = split + + dataset = load_dataset(root_dir, split=('test' if split == 'val' else split)) + for line in dataset: + for i, msg in enumerate(line['messages']): + if msg['role'] == 'assistant': + self.msgs.append(line['messages'][:i + 1]) + if self.split == 'val': + self.msgs = self.msgs[:100] + + + def __len__(self): + return len(self.msgs) + + def __getitem__(self, idx): + bos, eos, pad, sep = self.tokenizer.bos_token_id, self.tokenizer.eos_token_id, self.tokenizer.pad_token_id, self.tokenizer.sep_token_id + role_tti = {'user': 0, 'assistant': 1, 'system': 2, 'pad': 3} + role_sgn = {'user': "Q: ", 'assistant': "A: "} + context_list = [con['content'] for con in self.msgs[idx]] + role_list = [con['role'] for con in self.msgs[idx]] + if self.split == 'val': + self.tokenizer.padding_side = "left" + input_ids = [] + labels = [] + for id, utter in enumerate(context_list[:-1]): + if role_list[id] == 'system': + tmp = self.tokenizer.encode(utter + '\n\n') + else: + tmp = self.tokenizer.encode(role_sgn[role_list[id]]) + self.tokenizer.encode(utter + '\n\n') + input_ids += tmp + input_ids += self.tokenizer.encode(role_sgn[role_list[len(context_list) - 1]]) + if len(input_ids) > self.max_length - 128: + return {'return_dict': True} + leng = len(self.tokenizer.decode(input_ids)) + input_ids = [pad] * (self.max_length - 128 - len(input_ids)) + input_ids + labels = self.tokenizer.encode(context_list[-1], max_length=128, padding="max_length", truncation=True) + if len(labels) > 128: + return {'return_dict': True} + x = { + "input_ids": torch.tensor(input_ids), + "labels": torch.tensor(labels), + 'return_dict': True, + 'len': leng + } + return x + else: + self.tokenizer.padding_side = "right" + target = context_list[-1] + input_ids = [] + labels = [] + for id, utter in enumerate(context_list[:-1]): + if role_list[id] == 'system': + tmp = self.tokenizer.encode(utter + '\n\n') + else: + tmp = self.tokenizer.encode(role_sgn[role_list[id]]) + self.tokenizer.encode(utter + '\n\n') + input_ids += tmp + input_ids += self.tokenizer.encode(role_sgn[role_list[len(context_list) - 1]]) + labels = [-100] * len(input_ids) + self.tokenizer.encode(target) + [eos] + # labels = input_ids + self.tokenizer.encode(target) + [eos] + input_ids += self.tokenizer.encode(target) + [eos] + + # token_type_ids = [[role_tti[role_list[i]]] * (len(self.tokenizer.encode(utter)) + len(self.tokenizer.encode(role_sgn[role_list[i]]))) for i, utter in enumerate(context_list)] + # token_type_ids += [[role_tti[role_list[-1]]]] + # lm_labels = [[pad] * (len(list(chain(*input_ids))) - len(self.tokenizer.encode(target)) - 1)] + [self.tokenizer.encode(target)] + [eos] + # input_ids = list(chain(*input_ids)) + if len(input_ids) > self.max_length: + return {'return_dict': True} + # token_type_ids = list(chain(*token_type_ids)) + attention_mask = [1] * len(input_ids) + [0] * (self.max_length - len(input_ids)) + # labels = [[-100] * (len(token_type_ids) - len(self.tokenizer.encode(target)) - 1)] + [self.tokenizer.encode(target)] + [[eos]] + # labels = list(chain(*labels)) + # labels = input_ids.copy() + labels += [-100] * (self.max_length - len(input_ids)) + input_ids += [pad] * (self.max_length - len(input_ids)) + # token_type_ids += [role_tti['pad']] * (self.max_length - len(token_type_ids)) + x = { + "input_ids": torch.tensor(input_ids), + # "token_type_ids": torch.tensor(token_type_ids), + "attention_mask": torch.tensor(attention_mask), + "labels": torch.tensor(labels), + 'return_dict': True + } + return x + +from ..ab_dataset import ABDataset +from ..registery import dataset_register + +@dataset_register( + name='No_robots', + classes=['None'], + task_type='Text Generation', + object_type=None, + class_aliases=[], + shift_type=None +) +class No_Robots(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform, + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + return No_Robotsbase(root_dir, split, transform, classes, ignore_classes, idx_map) \ No newline at end of file diff --git a/data/datasets/visual_question_answering/__init__.py b/data/datasets/visual_question_answering/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..70128bca25e946918520612628906ef2cb2c5447 --- /dev/null +++ b/data/datasets/visual_question_answering/__init__.py @@ -0,0 +1 @@ +from .vqav2 import VQAv2_split2, VQAv2_split1, VQAv2_split1_c_gaussian_noise, VQAv2_split1_c_shot_noise, VQAv2_split1_c_impulse_noise, VQAv2_split1_c_defocus_blur, VQAv2_split1_c_glass_blur, VQAv2_split1_c_motion_blur, VQAv2_split1_c_zoom_blur, VQAv2_split1_c_snow, VQAv2_split1_c_frost, VQAv2_split1_c_fog, VQAv2_split1_c_brightness, VQAv2_split1_c_contrast, VQAv2_split1_c_elastic_transform, VQAv2_split1_c_pixelate, VQAv2_split1_c_jpeg_compression, VQAv2_split1_c_speckle_noise, VQAv2_split1_c_gaussian_blur, VQAv2_split1_c_spatter, VQAv2_split1_c_saturate diff --git a/data/datasets/visual_question_answering/__pycache__/__init__.cpython-38.pyc b/data/datasets/visual_question_answering/__pycache__/__init__.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..606c066644c912c8c879ab7d018004325ad6db4c Binary files /dev/null and b/data/datasets/visual_question_answering/__pycache__/__init__.cpython-38.pyc differ diff --git a/data/datasets/visual_question_answering/__pycache__/vqav2.cpython-38.pyc b/data/datasets/visual_question_answering/__pycache__/vqav2.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b4094f6291da48d4dcef7458d3bc2c801c725bac Binary files /dev/null and b/data/datasets/visual_question_answering/__pycache__/vqav2.cpython-38.pyc differ diff --git a/data/datasets/visual_question_answering/generate_c_image/imagenet_c/__init__.py b/data/datasets/visual_question_answering/generate_c_image/imagenet_c/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d41c63097cd95261663a5e31dbb43ef3db3be586 --- /dev/null +++ b/data/datasets/visual_question_answering/generate_c_image/imagenet_c/__init__.py @@ -0,0 +1,35 @@ +import numpy as np +from PIL import Image +from .corruptions import * + +corruption_tuple = (gaussian_noise, shot_noise, impulse_noise, defocus_blur, + glass_blur, motion_blur, zoom_blur, snow, frost, fog, + brightness, contrast, elastic_transform, pixelate, jpeg_compression, + speckle_noise, gaussian_blur, spatter, saturate) + +corruption_dict = {corr_func.__name__: corr_func for corr_func in corruption_tuple} + + +def corrupt(x, severity=1, corruption_name=None, corruption_number=-1): + """ + :param x: image to corrupt; a 224x224x3 numpy array in [0, 255] + :param severity: strength with which to corrupt x; an integer in [0, 5] + :param corruption_name: specifies which corruption function to call; + must be one of 'gaussian_noise', 'shot_noise', 'impulse_noise', 'defocus_blur', + 'glass_blur', 'motion_blur', 'zoom_blur', 'snow', 'frost', 'fog', + 'brightness', 'contrast', 'elastic_transform', 'pixelate', 'jpeg_compression', + 'speckle_noise', 'gaussian_blur', 'spatter', 'saturate'; + the last four are validation functions + :param corruption_number: the position of the corruption_name in the above list; + an integer in [0, 18]; useful for easy looping; 15, 16, 17, 18 are validation corruption numbers + :return: the image x corrupted by a corruption function at the given severity; same shape as input + """ + + if corruption_name: + x_corrupted = corruption_dict[corruption_name](Image.fromarray(x), severity) + elif corruption_number != -1: + x_corrupted = corruption_tuple[corruption_number](Image.fromarray(x), severity) + else: + raise ValueError("Either corruption_name or corruption_number must be passed") + + return np.uint8(x_corrupted) diff --git a/data/datasets/visual_question_answering/generate_c_image/imagenet_c/__pycache__/__init__.cpython-38.pyc b/data/datasets/visual_question_answering/generate_c_image/imagenet_c/__pycache__/__init__.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6324bad8e9d06454ac15f4aceacbcf7a8afc75f3 Binary files /dev/null and b/data/datasets/visual_question_answering/generate_c_image/imagenet_c/__pycache__/__init__.cpython-38.pyc differ diff --git a/data/datasets/visual_question_answering/generate_c_image/imagenet_c/__pycache__/corruptions.cpython-38.pyc b/data/datasets/visual_question_answering/generate_c_image/imagenet_c/__pycache__/corruptions.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..bac32040226dd971661f84da32ca693a503c7ba5 Binary files /dev/null and b/data/datasets/visual_question_answering/generate_c_image/imagenet_c/__pycache__/corruptions.cpython-38.pyc differ diff --git a/data/datasets/visual_question_answering/generate_c_image/imagenet_c/corruptions.py b/data/datasets/visual_question_answering/generate_c_image/imagenet_c/corruptions.py new file mode 100644 index 0000000000000000000000000000000000000000..a6dbcff3f88f3ea75c320c74abfc411b062e736f --- /dev/null +++ b/data/datasets/visual_question_answering/generate_c_image/imagenet_c/corruptions.py @@ -0,0 +1,439 @@ +# -*- coding: utf-8 -*- + +import numpy as np +from PIL import Image + +# /////////////// Corruption Helpers /////////////// + +import skimage as sk +from skimage.filters import gaussian +from io import BytesIO +from wand.image import Image as WandImage +from wand.api import library as wandlibrary +import wand.color as WandColor +import ctypes +from PIL import Image as PILImage +import cv2 +cv2.setNumThreads(0) +from scipy.ndimage import zoom as scizoom +from scipy.ndimage.interpolation import map_coordinates +import warnings +import os +from pkg_resources import resource_filename + +warnings.simplefilter("ignore", UserWarning) + + +def disk(radius, alias_blur=0.1, dtype=np.float32): + if radius <= 8: + L = np.arange(-8, 8 + 1) + ksize = (3, 3) + else: + L = np.arange(-radius, radius + 1) + ksize = (5, 5) + X, Y = np.meshgrid(L, L) + aliased_disk = np.array((X ** 2 + Y ** 2) <= radius ** 2, dtype=dtype) + aliased_disk /= np.sum(aliased_disk) + + cv2.setNumThreads(0) + + # supersample disk to antialias + return cv2.GaussianBlur(aliased_disk, ksize=ksize, sigmaX=alias_blur) + + +# Tell Python about the C method +wandlibrary.MagickMotionBlurImage.argtypes = (ctypes.c_void_p, # wand + ctypes.c_double, # radius + ctypes.c_double, # sigma + ctypes.c_double) # angle + + +# Extend wand.image.Image class to include method signature +class MotionImage(WandImage): + def motion_blur(self, radius=0.0, sigma=0.0, angle=0.0): + wandlibrary.MagickMotionBlurImage(self.wand, radius, sigma, angle) + + +# modification of https://github.com/FLHerne/mapgen/blob/master/diamondsquare.py +def plasma_fractal(mapsize=256, wibbledecay=3): + """ + Generate a heightmap using diamond-square algorithm. + Return square 2d array, side length 'mapsize', of floats in range 0-255. + 'mapsize' must be a power of two. + """ + assert (mapsize & (mapsize - 1) == 0) + maparray = np.empty((mapsize, mapsize), dtype=np.float_) + maparray[0, 0] = 0 + stepsize = mapsize + wibble = 100 + + def wibbledmean(array): + return array / 4 + wibble * np.random.uniform(-wibble, wibble, array.shape) + + def fillsquares(): + """For each square of points stepsize apart, + calculate middle value as mean of points + wibble""" + cornerref = maparray[0:mapsize:stepsize, 0:mapsize:stepsize] + squareaccum = cornerref + np.roll(cornerref, shift=-1, axis=0) + squareaccum += np.roll(squareaccum, shift=-1, axis=1) + maparray[stepsize // 2:mapsize:stepsize, + stepsize // 2:mapsize:stepsize] = wibbledmean(squareaccum) + + def filldiamonds(): + """For each diamond of points stepsize apart, + calculate middle value as mean of points + wibble""" + mapsize = maparray.shape[0] + drgrid = maparray[stepsize // 2:mapsize:stepsize, stepsize // 2:mapsize:stepsize] + ulgrid = maparray[0:mapsize:stepsize, 0:mapsize:stepsize] + ldrsum = drgrid + np.roll(drgrid, 1, axis=0) + lulsum = ulgrid + np.roll(ulgrid, -1, axis=1) + ltsum = ldrsum + lulsum + maparray[0:mapsize:stepsize, stepsize // 2:mapsize:stepsize] = wibbledmean(ltsum) + tdrsum = drgrid + np.roll(drgrid, 1, axis=1) + tulsum = ulgrid + np.roll(ulgrid, -1, axis=0) + ttsum = tdrsum + tulsum + maparray[stepsize // 2:mapsize:stepsize, 0:mapsize:stepsize] = wibbledmean(ttsum) + + while stepsize >= 2: + fillsquares() + filldiamonds() + stepsize //= 2 + wibble /= wibbledecay + + maparray -= maparray.min() + return maparray / maparray.max() + + +def clipped_zoom(img, zoom_factor): + h = img.shape[0] + # ceil crop height(= crop width) + ch = int(np.ceil(h / float(zoom_factor))) + + top = (h - ch) // 2 + img = scizoom(img[top:top + ch, top:top + ch], (zoom_factor, zoom_factor, 1), order=1) + # trim off any extra pixels + trim_top = (img.shape[0] - h) // 2 + + return img[trim_top:trim_top + h, trim_top:trim_top + h] + + +# /////////////// End Corruption Helpers /////////////// + + +# /////////////// Corruptions /////////////// + +def gaussian_noise(x, severity=1): + c = [.08, .12, 0.18, 0.26, 0.38][severity - 1] + + x = np.array(x) / 255. + return np.clip(x + np.random.normal(size=x.shape, scale=c), 0, 1) * 255 + + +def shot_noise(x, severity=1): + c = [60, 25, 12, 5, 3][severity - 1] + + x = np.array(x) / 255. + return np.clip(np.random.poisson(x * c) / float(c), 0, 1) * 255 + + +def impulse_noise(x, severity=1): + c = [.03, .06, .09, 0.17, 0.27][severity - 1] + + x = sk.util.random_noise(np.array(x) / 255., mode='s&p', amount=c) + return np.clip(x, 0, 1) * 255 + + +def speckle_noise(x, severity=1): + c = [.15, .2, 0.35, 0.45, 0.6][severity - 1] + + x = np.array(x) / 255. + return np.clip(x + x * np.random.normal(size=x.shape, scale=c), 0, 1) * 255 + + +def fgsm(x, source_net, severity=1): + c = [8, 16, 32, 64, 128][severity - 1] + + x = V(x, requires_grad=True) + logits = source_net(x) + source_net.zero_grad() + loss = F.cross_entropy(logits, V(logits.data.max(1)[1].squeeze_()), size_average=False) + loss.backward() + + return standardize(torch.clamp(unstandardize(x.data) + c / 255. * unstandardize(torch.sign(x.grad.data)), 0, 1)) + + +def gaussian_blur(x, severity=1): + c = [1, 2, 3, 4, 6][severity - 1] + + x = gaussian(np.array(x) / 255., sigma=c) + return np.clip(x, 0, 1) * 255 + + +def glass_blur(x, severity=1): + # sigma, max_delta, iterations + c = [(0.7, 1, 2), (0.9, 2, 1), (1, 2, 3), (1.1, 3, 2), (1.5, 4, 2)][severity - 1] + + # x = np.uint8(gaussian(np.array(x) / 255., sigma=c[0], multichannel=True) * 255) + x = np.uint8(gaussian(np.array(x) / 255., sigma=c[0]) * 255) + + # locally shuffle pixels + for i in range(c[2]): + for h in range(224 - c[1], c[1], -1): + for w in range(224 - c[1], c[1], -1): + dx, dy = np.random.randint(-c[1], c[1], size=(2,)) + h_prime, w_prime = h + dy, w + dx + # swap + x[h, w], x[h_prime, w_prime] = x[h_prime, w_prime], x[h, w] + + return np.clip(gaussian(x / 255., sigma=c[0]), 0, 1) * 255 + + +def defocus_blur(x, severity=1): + c = [(3, 0.1), (4, 0.5), (6, 0.5), (8, 0.5), (10, 0.5)][severity - 1] + + x = np.array(x) / 255. + kernel = disk(radius=c[0], alias_blur=c[1]) + + channels = [] + for d in range(3): + cv2.setNumThreads(0) + channels.append(cv2.filter2D(x[:, :, d], -1, kernel)) + channels = np.array(channels).transpose((1, 2, 0)) # 3x224x224 -> 224x224x3 + + return np.clip(channels, 0, 1) * 255 + + +def motion_blur(x, severity=1): + c = [(10, 3), (15, 5), (15, 8), (15, 12), (20, 15)][severity - 1] + + output = BytesIO() + x.save(output, format='PNG') + x = MotionImage(blob=output.getvalue()) + + x.motion_blur(radius=c[0], sigma=c[1], angle=np.random.uniform(-45, 45)) + + cv2.setNumThreads(0) + x = cv2.imdecode(np.fromstring(x.make_blob(), np.uint8), + cv2.IMREAD_UNCHANGED) + + if x.shape != (224, 224): + return np.clip(x[..., [2, 1, 0]], 0, 255) # BGR to RGB + else: # greyscale to RGB + return np.clip(np.array([x, x, x]).transpose((1, 2, 0)), 0, 255) + + +def zoom_blur(x, severity=1): + c = [np.arange(1, 1.11, 0.01), + np.arange(1, 1.16, 0.01), + np.arange(1, 1.21, 0.02), + np.arange(1, 1.26, 0.02), + np.arange(1, 1.31, 0.03)][severity - 1] + + x = (np.array(x) / 255.).astype(np.float32) + out = np.zeros_like(x) + for zoom_factor in c: + out += clipped_zoom(x, zoom_factor) + + x = (x + out) / (len(c) + 1) + return np.clip(x, 0, 1) * 255 + + +def fog(x, severity=1): + c = [(1.5, 2), (2., 2), (2.5, 1.7), (2.5, 1.5), (3., 1.4)][severity - 1] + + x = np.array(x) / 255. + max_val = x.max() + x += c[0] * plasma_fractal(wibbledecay=c[1])[:224, :224][..., np.newaxis] + return np.clip(x * max_val / (max_val + c[0]), 0, 1) * 255 + + +def frost(x, severity=1): + c = [(1, 0.4), + (0.8, 0.6), + (0.7, 0.7), + (0.65, 0.7), + (0.6, 0.75)][severity - 1] + idx = np.random.randint(5) + filename = [resource_filename(__name__, 'frost/frost1.png'), + resource_filename(__name__, 'frost/frost2.png'), + resource_filename(__name__, 'frost/frost3.png'), + resource_filename(__name__, 'frost/frost4.jpg'), + resource_filename(__name__, 'frost/frost5.jpg'), + resource_filename(__name__, 'frost/frost6.jpg')][idx] + + cv2.setNumThreads(0) + frost = cv2.imread(filename) + # randomly crop and convert to rgb + x_start, y_start = np.random.randint(0, frost.shape[0] - 224), np.random.randint(0, frost.shape[1] - 224) + frost = frost[x_start:x_start + 224, y_start:y_start + 224][..., [2, 1, 0]] + + return np.clip(c[0] * np.array(x) + c[1] * frost, 0, 255) + + +def snow(x, severity=1): + c = [(0.1, 0.3, 3, 0.5, 10, 4, 0.8), + (0.2, 0.3, 2, 0.5, 12, 4, 0.7), + (0.55, 0.3, 4, 0.9, 12, 8, 0.7), + (0.55, 0.3, 4.5, 0.85, 12, 8, 0.65), + (0.55, 0.3, 2.5, 0.85, 12, 12, 0.55)][severity - 1] + + x = np.array(x, dtype=np.float32) / 255. + snow_layer = np.random.normal(size=x.shape[:2], loc=c[0], scale=c[1]) # [:2] for monochrome + + snow_layer = clipped_zoom(snow_layer[..., np.newaxis], c[2]) + snow_layer[snow_layer < c[3]] = 0 + + snow_layer = PILImage.fromarray((np.clip(snow_layer.squeeze(), 0, 1) * 255).astype(np.uint8), mode='L') + output = BytesIO() + snow_layer.save(output, format='PNG') + snow_layer = MotionImage(blob=output.getvalue()) + + snow_layer.motion_blur(radius=c[4], sigma=c[5], angle=np.random.uniform(-135, -45)) + cv2.setNumThreads(0) + snow_layer = cv2.imdecode(np.fromstring(snow_layer.make_blob(), np.uint8), + cv2.IMREAD_UNCHANGED) / 255. + snow_layer = snow_layer[..., np.newaxis] + + x = c[6] * x + (1 - c[6]) * np.maximum(x, cv2.cvtColor(x, cv2.COLOR_RGB2GRAY).reshape(224, 224, 1) * 1.5 + 0.5) + return np.clip(x + snow_layer + np.rot90(snow_layer, k=2), 0, 1) * 255 + + +def spatter(x, severity=1): + cv2.setNumThreads(0) + + c = [(0.65, 0.3, 4, 0.69, 0.6, 0), + (0.65, 0.3, 3, 0.68, 0.6, 0), + (0.65, 0.3, 2, 0.68, 0.5, 0), + (0.65, 0.3, 1, 0.65, 1.5, 1), + (0.67, 0.4, 1, 0.65, 1.5, 1)][severity - 1] + x = np.array(x, dtype=np.float32) / 255. + + liquid_layer = np.random.normal(size=x.shape[:2], loc=c[0], scale=c[1]) + + liquid_layer = gaussian(liquid_layer, sigma=c[2]) + liquid_layer[liquid_layer < c[3]] = 0 + if c[5] == 0: + liquid_layer = (liquid_layer * 255).astype(np.uint8) + dist = 255 - cv2.Canny(liquid_layer, 50, 150) + dist = cv2.distanceTransform(dist, cv2.DIST_L2, 5) + _, dist = cv2.threshold(dist, 20, 20, cv2.THRESH_TRUNC) + dist = cv2.blur(dist, (3, 3)).astype(np.uint8) + dist = cv2.equalizeHist(dist) + ker = np.array([[-2, -1, 0], [-1, 1, 1], [0, 1, 2]]) + dist = cv2.filter2D(dist, cv2.CV_8U, ker) + dist = cv2.blur(dist, (3, 3)).astype(np.float32) + + m = cv2.cvtColor(liquid_layer * dist, cv2.COLOR_GRAY2BGRA) + m /= np.max(m, axis=(0, 1)) + m *= c[4] + + # water is pale turqouise + color = np.concatenate((175 / 255. * np.ones_like(m[..., :1]), + 238 / 255. * np.ones_like(m[..., :1]), + 238 / 255. * np.ones_like(m[..., :1])), axis=2) + + color = cv2.cvtColor(color, cv2.COLOR_BGR2BGRA) + x = cv2.cvtColor(x, cv2.COLOR_BGR2BGRA) + + return cv2.cvtColor(np.clip(x + m * color, 0, 1), cv2.COLOR_BGRA2BGR) * 255 + else: + m = np.where(liquid_layer > c[3], 1, 0) + m = gaussian(m.astype(np.float32), sigma=c[4]) + m[m < 0.8] = 0 + + # mud brown + color = np.concatenate((63 / 255. * np.ones_like(x[..., :1]), + 42 / 255. * np.ones_like(x[..., :1]), + 20 / 255. * np.ones_like(x[..., :1])), axis=2) + + color *= m[..., np.newaxis] + x *= (1 - m[..., np.newaxis]) + + return np.clip(x + color, 0, 1) * 255 + + +def contrast(x, severity=1): + c = [0.4, .3, .2, .1, .05][severity - 1] + + x = np.array(x) / 255. + means = np.mean(x, axis=(0, 1), keepdims=True) + return np.clip((x - means) * c + means, 0, 1) * 255 + + +def brightness(x, severity=1): + c = [.1, .2, .3, .4, .5][severity - 1] + + x = np.array(x) / 255. + x = sk.color.rgb2hsv(x) + x[:, :, 2] = np.clip(x[:, :, 2] + c, 0, 1) + x = sk.color.hsv2rgb(x) + + return np.clip(x, 0, 1) * 255 + + +def saturate(x, severity=1): + c = [(0.3, 0), (0.1, 0), (2, 0), (5, 0.1), (20, 0.2)][severity - 1] + + x = np.array(x) / 255. + x = sk.color.rgb2hsv(x) + x[:, :, 1] = np.clip(x[:, :, 1] * c[0] + c[1], 0, 1) + x = sk.color.hsv2rgb(x) + + return np.clip(x, 0, 1) * 255 + + +def jpeg_compression(x, severity=1): + c = [25, 18, 15, 10, 7][severity - 1] + + output = BytesIO() + x.save(output, 'JPEG', quality=c) + x = PILImage.open(output) + + return x + + +def pixelate(x, severity=1): + c = [0.6, 0.5, 0.4, 0.3, 0.25][severity - 1] + + x = x.resize((int(224 * c), int(224 * c)), PILImage.BOX) + x = x.resize((224, 224), PILImage.BOX) + + return x + + +# mod of https://gist.github.com/erniejunior/601cdf56d2b424757de5 +def elastic_transform(image, severity=1): + c = [(244 * 2, 244 * 0.7, 244 * 0.1), # 244 should have been 224, but ultimately nothing is incorrect + (244 * 2, 244 * 0.08, 244 * 0.2), + (244 * 0.05, 244 * 0.01, 244 * 0.02), + (244 * 0.07, 244 * 0.01, 244 * 0.02), + (244 * 0.12, 244 * 0.01, 244 * 0.02)][severity - 1] + + cv2.setNumThreads(0) + + image = np.array(image, dtype=np.float32) / 255. + shape = image.shape + shape_size = shape[:2] + + # random affine + center_square = np.float32(shape_size) // 2 + square_size = min(shape_size) // 3 + pts1 = np.float32([center_square + square_size, + [center_square[0] + square_size, center_square[1] - square_size], + center_square - square_size]) + pts2 = pts1 + np.random.uniform(-c[2], c[2], size=pts1.shape).astype(np.float32) + M = cv2.getAffineTransform(pts1, pts2) + image = cv2.warpAffine(image, M, shape_size[::-1], borderMode=cv2.BORDER_REFLECT_101) + + dx = (gaussian(np.random.uniform(-1, 1, size=shape[:2]), + c[1], mode='reflect', truncate=3) * c[0]).astype(np.float32) + dy = (gaussian(np.random.uniform(-1, 1, size=shape[:2]), + c[1], mode='reflect', truncate=3) * c[0]).astype(np.float32) + dx, dy = dx[..., np.newaxis], dy[..., np.newaxis] + + x, y, z = np.meshgrid(np.arange(shape[1]), np.arange(shape[0]), np.arange(shape[2])) + indices = np.reshape(y + dy, (-1, 1)), np.reshape(x + dx, (-1, 1)), np.reshape(z, (-1, 1)) + return np.clip(map_coordinates(image, indices, order=1, mode='reflect').reshape(shape), 0, 1) * 255 + + +# /////////////// End Corruptions /////////////// diff --git a/data/datasets/visual_question_answering/generate_c_image/imagenet_c/frost/frost1.png b/data/datasets/visual_question_answering/generate_c_image/imagenet_c/frost/frost1.png new file mode 100644 index 0000000000000000000000000000000000000000..c9edf9b6e1a2744d15af615af641f2aa48aa89c2 --- /dev/null +++ b/data/datasets/visual_question_answering/generate_c_image/imagenet_c/frost/frost1.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff9f907860bd7a835d459e32f9d588062b7f61ee267343cc7222b56753a14755 +size 1199930 diff --git a/data/datasets/visual_question_answering/generate_c_image/imagenet_c/frost/frost2.png b/data/datasets/visual_question_answering/generate_c_image/imagenet_c/frost/frost2.png new file mode 100644 index 0000000000000000000000000000000000000000..48f7a861ffa41b6d7496b701fef96d5edf739282 Binary files /dev/null and b/data/datasets/visual_question_answering/generate_c_image/imagenet_c/frost/frost2.png differ diff --git a/data/datasets/visual_question_answering/generate_c_image/imagenet_c/frost/frost3.png b/data/datasets/visual_question_answering/generate_c_image/imagenet_c/frost/frost3.png new file mode 100644 index 0000000000000000000000000000000000000000..d47f9d25f41251ee9a66b294c0bbfad6053017c9 Binary files /dev/null and b/data/datasets/visual_question_answering/generate_c_image/imagenet_c/frost/frost3.png differ diff --git a/data/datasets/visual_question_answering/generate_c_image/imagenet_c/frost/frost4.jpg b/data/datasets/visual_question_answering/generate_c_image/imagenet_c/frost/frost4.jpg new file mode 100644 index 0000000000000000000000000000000000000000..f8b0c413176d70150b593e029d84b4a88c21dd4b Binary files /dev/null and b/data/datasets/visual_question_answering/generate_c_image/imagenet_c/frost/frost4.jpg differ diff --git a/data/datasets/visual_question_answering/generate_c_image/imagenet_c/frost/frost5.jpg b/data/datasets/visual_question_answering/generate_c_image/imagenet_c/frost/frost5.jpg new file mode 100644 index 0000000000000000000000000000000000000000..95dc9056926d8201df760535f9bb9112f012e862 Binary files /dev/null and b/data/datasets/visual_question_answering/generate_c_image/imagenet_c/frost/frost5.jpg differ diff --git a/data/datasets/visual_question_answering/generate_c_image/imagenet_c/frost/frost6.jpg b/data/datasets/visual_question_answering/generate_c_image/imagenet_c/frost/frost6.jpg new file mode 100644 index 0000000000000000000000000000000000000000..14e5d58e762a5d0808df9fa6494fd6d78ee4409b Binary files /dev/null and b/data/datasets/visual_question_answering/generate_c_image/imagenet_c/frost/frost6.jpg differ diff --git a/data/datasets/visual_question_answering/generate_c_image/make_vqa_c.py b/data/datasets/visual_question_answering/generate_c_image/make_vqa_c.py new file mode 100644 index 0000000000000000000000000000000000000000..d87ab9e47669506c14e04eb55d5c29ceca46f8ab --- /dev/null +++ b/data/datasets/visual_question_answering/generate_c_image/make_vqa_c.py @@ -0,0 +1,16 @@ +import os +import numpy as np +from PIL import Image +from data.datasets.visual_question_answering.generate_c_image.imagenet_c import corrupt + + +imgs_dir = '/data/zql/datasets/vqav2/train2014' + +for img_p in os.listdir(imgs_dir): + img = np.array(Image.open(os.path.join(imgs_dir, img_p))) + print(img.shape) + + c_img = corrupt(img, severity=1, corruption_name='gaussian_noise') + print(c_img.shape) + + break \ No newline at end of file diff --git a/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/condensenet_converted.py b/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/condensenet_converted.py new file mode 100644 index 0000000000000000000000000000000000000000..0ed15724da4abd191fd49fdc39a186943d579c67 --- /dev/null +++ b/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/condensenet_converted.py @@ -0,0 +1,121 @@ +from __future__ import absolute_import +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import division + +import torch +import torch.nn as nn +from torch.autograd import Variable +import math +from layers import ShuffleLayer, Conv, CondenseConv, CondenseLinear + +__all__ = ['CondenseNet'] + +class _DenseLayer(nn.Module): + def __init__(self, in_channels, growth_rate, args): + super(_DenseLayer, self).__init__() + self.group_1x1 = args.group_1x1 + self.group_3x3 = args.group_3x3 + ### 1x1 conv i --> b*k + self.conv_1 = CondenseConv(in_channels, args.bottleneck * growth_rate, + kernel_size=1, groups=self.group_1x1) + ### 3x3 conv b*k-->k + self.conv_2 = Conv(args.bottleneck * growth_rate, growth_rate, + kernel_size=3, padding=1, groups=self.group_3x3) + + def forward(self, x): + x_ = x + x = self.conv_1(x) + x = self.conv_2(x) + return torch.cat([x_, x], 1) + + +class _DenseBlock(nn.Sequential): + def __init__(self, num_layers, in_channels, growth_rate, args): + super(_DenseBlock, self).__init__() + for i in range(num_layers): + layer = _DenseLayer(in_channels + i * growth_rate, growth_rate, args) + self.add_module('denselayer_%d' % (i + 1), layer) + + +class _Transition(nn.Module): + def __init__(self, in_channels, args): + super(_Transition, self).__init__() + self.pool = nn.AvgPool2d(kernel_size=2, stride=2) + + def forward(self, x): + x = self.pool(x) + return x + + +class CondenseNet(nn.Module): + def __init__(self, args): + + super(CondenseNet, self).__init__() + + self.stages = args.stages + self.growth = args.growth + assert len(self.stages) == len(self.growth) + self.args = args + self.progress = 0.0 + if args.data in ['cifar10', 'cifar100']: + self.init_stride = 1 + self.pool_size = 8 + else: + self.init_stride = 2 + self.pool_size = 7 + + self.features = nn.Sequential() + ### Initial nChannels should be 3 + self.num_features = 2 * self.growth[0] + ### Dense-block 1 (224x224) + self.features.add_module('init_conv', nn.Conv2d(3, self.num_features, + kernel_size=3, + stride=self.init_stride, + padding=1, + bias=False)) + for i in range(len(self.stages)): + ### Dense-block i + self.add_block(i) + ### Linear layer + self.classifier = CondenseLinear(self.num_features, args.num_classes, + 0.5) + ### initialize + for m in self.modules(): + if isinstance(m, nn.Conv2d): + n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels + m.weight.data.normal_(0, math.sqrt(2. / n)) + elif isinstance(m, nn.BatchNorm2d): + m.weight.data.fill_(1) + m.bias.data.zero_() + elif isinstance(m, nn.Linear): + m.bias.data.zero_() + + def add_block(self, i): + ### Check if ith is the last one + last = (i == len(self.stages) - 1) + block = _DenseBlock( + num_layers=self.stages[i], + in_channels=self.num_features, + growth_rate=self.growth[i], + args=self.args, + ) + self.features.add_module('denseblock_%d' % (i + 1), block) + self.num_features += self.stages[i] * self.growth[i] + if not last: + trans = _Transition(in_channels=self.num_features, + args=self.args) + self.features.add_module('transition_%d' % (i + 1), trans) + else: + self.features.add_module('norm_last', + nn.BatchNorm2d(self.num_features)) + self.features.add_module('relu_last', + nn.ReLU(inplace=True)) + self.features.add_module('pool_last', + nn.AvgPool2d(self.pool_size)) + + def forward(self, x, progress=None): + features = self.features(x) + out = features.view(features.size(0), -1) + out = self.classifier(out) + return out diff --git a/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/create_c/224.sh b/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/create_c/224.sh new file mode 100644 index 0000000000000000000000000000000000000000..a8ed3498584878943a522048ae02dac778957bb6 --- /dev/null +++ b/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/create_c/224.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +source ~/new_begin.sh + +python make_imagenet_c.py + diff --git a/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/create_c/299.sh b/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/create_c/299.sh new file mode 100644 index 0000000000000000000000000000000000000000..2e5429a82f2d9b9c58ca185b8751fd51a5a69285 --- /dev/null +++ b/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/create_c/299.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +source ~/new_begin.sh + +python make_imagenet_c_inception.py + diff --git a/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/create_c/64.sh b/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/create_c/64.sh new file mode 100644 index 0000000000000000000000000000000000000000..267d8811c51f30cd1256c55660b620f67335e167 --- /dev/null +++ b/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/create_c/64.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +source ~/new_begin.sh + +python make_imagenet_64_c.py + diff --git a/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/create_c/frost1.png b/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/create_c/frost1.png new file mode 100644 index 0000000000000000000000000000000000000000..c9edf9b6e1a2744d15af615af641f2aa48aa89c2 --- /dev/null +++ b/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/create_c/frost1.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff9f907860bd7a835d459e32f9d588062b7f61ee267343cc7222b56753a14755 +size 1199930 diff --git a/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/create_c/frost2.png b/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/create_c/frost2.png new file mode 100644 index 0000000000000000000000000000000000000000..48f7a861ffa41b6d7496b701fef96d5edf739282 Binary files /dev/null and b/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/create_c/frost2.png differ diff --git a/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/create_c/frost3.png b/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/create_c/frost3.png new file mode 100644 index 0000000000000000000000000000000000000000..d47f9d25f41251ee9a66b294c0bbfad6053017c9 Binary files /dev/null and b/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/create_c/frost3.png differ diff --git a/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/create_c/frost4.jpg b/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/create_c/frost4.jpg new file mode 100644 index 0000000000000000000000000000000000000000..f8b0c413176d70150b593e029d84b4a88c21dd4b Binary files /dev/null and b/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/create_c/frost4.jpg differ diff --git a/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/create_c/frost5.jpg b/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/create_c/frost5.jpg new file mode 100644 index 0000000000000000000000000000000000000000..95dc9056926d8201df760535f9bb9112f012e862 Binary files /dev/null and b/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/create_c/frost5.jpg differ diff --git a/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/create_c/frost6.jpg b/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/create_c/frost6.jpg new file mode 100644 index 0000000000000000000000000000000000000000..14e5d58e762a5d0808df9fa6494fd6d78ee4409b Binary files /dev/null and b/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/create_c/frost6.jpg differ diff --git a/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/create_c/make_cifar_c.py b/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/create_c/make_cifar_c.py new file mode 100644 index 0000000000000000000000000000000000000000..fa10c095c9a3b4ab70b7f37251c9311469ecb3d8 --- /dev/null +++ b/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/create_c/make_cifar_c.py @@ -0,0 +1,465 @@ +# -*- coding: utf-8 -*- + +import os +from PIL import Image +import os.path +import time +import torch +import torchvision.datasets as dset +import torchvision.transforms as trn +import torch.utils.data as data +import numpy as np + +from PIL import Image + + +# /////////////// Distortion Helpers /////////////// + +import skimage as sk +from skimage.filters import gaussian +from io import BytesIO +from wand.image import Image as WandImage +from wand.api import library as wandlibrary +import wand.color as WandColor +import ctypes +from PIL import Image as PILImage +import cv2 +from scipy.ndimage import zoom as scizoom +from scipy.ndimage.interpolation import map_coordinates +import warnings + +warnings.simplefilter("ignore", UserWarning) + + +def disk(radius, alias_blur=0.1, dtype=np.float32): + if radius <= 8: + L = np.arange(-8, 8 + 1) + ksize = (3, 3) + else: + L = np.arange(-radius, radius + 1) + ksize = (5, 5) + X, Y = np.meshgrid(L, L) + aliased_disk = np.array((X ** 2 + Y ** 2) <= radius ** 2, dtype=dtype) + aliased_disk /= np.sum(aliased_disk) + + # supersample disk to antialias + return cv2.GaussianBlur(aliased_disk, ksize=ksize, sigmaX=alias_blur) + + +# Tell Python about the C method +wandlibrary.MagickMotionBlurImage.argtypes = (ctypes.c_void_p, # wand + ctypes.c_double, # radius + ctypes.c_double, # sigma + ctypes.c_double) # angle + + +# Extend wand.image.Image class to include method signature +class MotionImage(WandImage): + def motion_blur(self, radius=0.0, sigma=0.0, angle=0.0): + wandlibrary.MagickMotionBlurImage(self.wand, radius, sigma, angle) + + +# modification of https://github.com/FLHerne/mapgen/blob/master/diamondsquare.py +def plasma_fractal(mapsize=32, wibbledecay=3): + """ + Generate a heightmap using diamond-square algorithm. + Return square 2d array, side length 'mapsize', of floats in range 0-255. + 'mapsize' must be a power of two. + """ + assert (mapsize & (mapsize - 1) == 0) + maparray = np.empty((mapsize, mapsize), dtype=np.float_) + maparray[0, 0] = 0 + stepsize = mapsize + wibble = 100 + + def wibbledmean(array): + return array / 4 + wibble * np.random.uniform(-wibble, wibble, array.shape) + + def fillsquares(): + """For each square of points stepsize apart, + calculate middle value as mean of points + wibble""" + cornerref = maparray[0:mapsize:stepsize, 0:mapsize:stepsize] + squareaccum = cornerref + np.roll(cornerref, shift=-1, axis=0) + squareaccum += np.roll(squareaccum, shift=-1, axis=1) + maparray[stepsize // 2:mapsize:stepsize, + stepsize // 2:mapsize:stepsize] = wibbledmean(squareaccum) + + def filldiamonds(): + """For each diamond of points stepsize apart, + calculate middle value as mean of points + wibble""" + mapsize = maparray.shape[0] + drgrid = maparray[stepsize // 2:mapsize:stepsize, stepsize // 2:mapsize:stepsize] + ulgrid = maparray[0:mapsize:stepsize, 0:mapsize:stepsize] + ldrsum = drgrid + np.roll(drgrid, 1, axis=0) + lulsum = ulgrid + np.roll(ulgrid, -1, axis=1) + ltsum = ldrsum + lulsum + maparray[0:mapsize:stepsize, stepsize // 2:mapsize:stepsize] = wibbledmean(ltsum) + tdrsum = drgrid + np.roll(drgrid, 1, axis=1) + tulsum = ulgrid + np.roll(ulgrid, -1, axis=0) + ttsum = tdrsum + tulsum + maparray[stepsize // 2:mapsize:stepsize, 0:mapsize:stepsize] = wibbledmean(ttsum) + + while stepsize >= 2: + fillsquares() + filldiamonds() + stepsize //= 2 + wibble /= wibbledecay + + maparray -= maparray.min() + return maparray / maparray.max() + + +def clipped_zoom(img, zoom_factor): + h = img.shape[0] + # ceil crop height(= crop width) + ch = int(np.ceil(h / zoom_factor)) + + top = (h - ch) // 2 + img = scizoom(img[top:top + ch, top:top + ch], (zoom_factor, zoom_factor, 1), order=1) + # trim off any extra pixels + trim_top = (img.shape[0] - h) // 2 + + return img[trim_top:trim_top + h, trim_top:trim_top + h] + + +# /////////////// End Distortion Helpers /////////////// + + +# /////////////// Distortions /////////////// + +def gaussian_noise(x, severity=1): + c = [0.04, 0.06, .08, .09, .10][severity - 1] + + x = np.array(x) / 255. + return np.clip(x + np.random.normal(size=x.shape, scale=c), 0, 1) * 255 + + +def shot_noise(x, severity=1): + c = [500, 250, 100, 75, 50][severity - 1] + + x = np.array(x) / 255. + return np.clip(np.random.poisson(x * c) / c, 0, 1) * 255 + + +def impulse_noise(x, severity=1): + c = [.01, .02, .03, .05, .07][severity - 1] + + x = sk.util.random_noise(np.array(x) / 255., mode='s&p', amount=c) + return np.clip(x, 0, 1) * 255 + + +def speckle_noise(x, severity=1): + c = [.06, .1, .12, .16, .2][severity - 1] + + x = np.array(x) / 255. + return np.clip(x + x * np.random.normal(size=x.shape, scale=c), 0, 1) * 255 + + +def gaussian_blur(x, severity=1): + c = [.4, .6, 0.7, .8, 1][severity - 1] + + x = gaussian(np.array(x) / 255., sigma=c, multichannel=True) + return np.clip(x, 0, 1) * 255 + + +def glass_blur(x, severity=1): + # sigma, max_delta, iterations + c = [(0.05,1,1), (0.25,1,1), (0.4,1,1), (0.25,1,2), (0.4,1,2)][severity - 1] + + x = np.uint8(gaussian(np.array(x) / 255., sigma=c[0], multichannel=True) * 255) + + # locally shuffle pixels + for i in range(c[2]): + for h in range(32 - c[1], c[1], -1): + for w in range(32 - c[1], c[1], -1): + dx, dy = np.random.randint(-c[1], c[1], size=(2,)) + h_prime, w_prime = h + dy, w + dx + # swap + x[h, w], x[h_prime, w_prime] = x[h_prime, w_prime], x[h, w] + + return np.clip(gaussian(x / 255., sigma=c[0], multichannel=True), 0, 1) * 255 + + +def defocus_blur(x, severity=1): + c = [(0.3, 0.4), (0.4, 0.5), (0.5, 0.6), (1, 0.2), (1.5, 0.1)][severity - 1] + + x = np.array(x) / 255. + kernel = disk(radius=c[0], alias_blur=c[1]) + + channels = [] + for d in range(3): + channels.append(cv2.filter2D(x[:, :, d], -1, kernel)) + channels = np.array(channels).transpose((1, 2, 0)) # 3x32x32 -> 32x32x3 + + return np.clip(channels, 0, 1) * 255 + + +def motion_blur(x, severity=1): + c = [(6,1), (6,1.5), (6,2), (8,2), (9,2.5)][severity - 1] + + output = BytesIO() + x.save(output, format='PNG') + x = MotionImage(blob=output.getvalue()) + + x.motion_blur(radius=c[0], sigma=c[1], angle=np.random.uniform(-45, 45)) + + x = cv2.imdecode(np.fromstring(x.make_blob(), np.uint8), + cv2.IMREAD_UNCHANGED) + + if x.shape != (32, 32): + return np.clip(x[..., [2, 1, 0]], 0, 255) # BGR to RGB + else: # greyscale to RGB + return np.clip(np.array([x, x, x]).transpose((1, 2, 0)), 0, 255) + + +def zoom_blur(x, severity=1): + c = [np.arange(1, 1.06, 0.01), np.arange(1, 1.11, 0.01), np.arange(1, 1.16, 0.01), + np.arange(1, 1.21, 0.01), np.arange(1, 1.26, 0.01)][severity - 1] + + x = (np.array(x) / 255.).astype(np.float32) + out = np.zeros_like(x) + for zoom_factor in c: + out += clipped_zoom(x, zoom_factor) + + x = (x + out) / (len(c) + 1) + return np.clip(x, 0, 1) * 255 + + +def fog(x, severity=1): + c = [(.2,3), (.5,3), (0.75,2.5), (1,2), (1.5,1.75)][severity - 1] + + x = np.array(x) / 255. + max_val = x.max() + x += c[0] * plasma_fractal(wibbledecay=c[1])[:32, :32][..., np.newaxis] + return np.clip(x * max_val / (max_val + c[0]), 0, 1) * 255 + + +def frost(x, severity=1): + c = [(1, 0.2), (1, 0.3), (0.9, 0.4), (0.85, 0.4), (0.75, 0.45)][severity - 1] + idx = np.random.randint(5) + filename = ['./frost1.png', './frost2.png', './frost3.png', './frost4.jpg', './frost5.jpg', './frost6.jpg'][idx] + frost = cv2.imread(filename) + frost = cv2.resize(frost, (0, 0), fx=0.2, fy=0.2) + # randomly crop and convert to rgb + x_start, y_start = np.random.randint(0, frost.shape[0] - 32), np.random.randint(0, frost.shape[1] - 32) + frost = frost[x_start:x_start + 32, y_start:y_start + 32][..., [2, 1, 0]] + + return np.clip(c[0] * np.array(x) + c[1] * frost, 0, 255) + + +def snow(x, severity=1): + c = [(0.1,0.2,1,0.6,8,3,0.95), + (0.1,0.2,1,0.5,10,4,0.9), + (0.15,0.3,1.75,0.55,10,4,0.9), + (0.25,0.3,2.25,0.6,12,6,0.85), + (0.3,0.3,1.25,0.65,14,12,0.8)][severity - 1] + + x = np.array(x, dtype=np.float32) / 255. + snow_layer = np.random.normal(size=x.shape[:2], loc=c[0], scale=c[1]) # [:2] for monochrome + + snow_layer = clipped_zoom(snow_layer[..., np.newaxis], c[2]) + snow_layer[snow_layer < c[3]] = 0 + + snow_layer = PILImage.fromarray((np.clip(snow_layer.squeeze(), 0, 1) * 255).astype(np.uint8), mode='L') + output = BytesIO() + snow_layer.save(output, format='PNG') + snow_layer = MotionImage(blob=output.getvalue()) + + snow_layer.motion_blur(radius=c[4], sigma=c[5], angle=np.random.uniform(-135, -45)) + + snow_layer = cv2.imdecode(np.fromstring(snow_layer.make_blob(), np.uint8), + cv2.IMREAD_UNCHANGED) / 255. + snow_layer = snow_layer[..., np.newaxis] + + x = c[6] * x + (1 - c[6]) * np.maximum(x, cv2.cvtColor(x, cv2.COLOR_RGB2GRAY).reshape(32, 32, 1) * 1.5 + 0.5) + return np.clip(x + snow_layer + np.rot90(snow_layer, k=2), 0, 1) * 255 + + +def spatter(x, severity=1): + c = [(0.62,0.1,0.7,0.7,0.5,0), + (0.65,0.1,0.8,0.7,0.5,0), + (0.65,0.3,1,0.69,0.5,0), + (0.65,0.1,0.7,0.69,0.6,1), + (0.65,0.1,0.5,0.68,0.6,1)][severity - 1] + x = np.array(x, dtype=np.float32) / 255. + + liquid_layer = np.random.normal(size=x.shape[:2], loc=c[0], scale=c[1]) + + liquid_layer = gaussian(liquid_layer, sigma=c[2]) + liquid_layer[liquid_layer < c[3]] = 0 + if c[5] == 0: + liquid_layer = (liquid_layer * 255).astype(np.uint8) + dist = 255 - cv2.Canny(liquid_layer, 50, 150) + dist = cv2.distanceTransform(dist, cv2.DIST_L2, 5) + _, dist = cv2.threshold(dist, 20, 20, cv2.THRESH_TRUNC) + dist = cv2.blur(dist, (3, 3)).astype(np.uint8) + dist = cv2.equalizeHist(dist) + # ker = np.array([[-1,-2,-3],[-2,0,0],[-3,0,1]], dtype=np.float32) + # ker -= np.mean(ker) + ker = np.array([[-2, -1, 0], [-1, 1, 1], [0, 1, 2]]) + dist = cv2.filter2D(dist, cv2.CV_8U, ker) + dist = cv2.blur(dist, (3, 3)).astype(np.float32) + + m = cv2.cvtColor(liquid_layer * dist, cv2.COLOR_GRAY2BGRA) + m /= np.max(m, axis=(0, 1)) + m *= c[4] + + # water is pale turqouise + color = np.concatenate((175 / 255. * np.ones_like(m[..., :1]), + 238 / 255. * np.ones_like(m[..., :1]), + 238 / 255. * np.ones_like(m[..., :1])), axis=2) + + color = cv2.cvtColor(color, cv2.COLOR_BGR2BGRA) + x = cv2.cvtColor(x, cv2.COLOR_BGR2BGRA) + + return cv2.cvtColor(np.clip(x + m * color, 0, 1), cv2.COLOR_BGRA2BGR) * 255 + else: + m = np.where(liquid_layer > c[3], 1, 0) + m = gaussian(m.astype(np.float32), sigma=c[4]) + m[m < 0.8] = 0 + # m = np.abs(m) ** (1/c[4]) + + # mud brown + color = np.concatenate((63 / 255. * np.ones_like(x[..., :1]), + 42 / 255. * np.ones_like(x[..., :1]), + 20 / 255. * np.ones_like(x[..., :1])), axis=2) + + color *= m[..., np.newaxis] + x *= (1 - m[..., np.newaxis]) + + return np.clip(x + color, 0, 1) * 255 + + +def contrast(x, severity=1): + c = [.75, .5, .4, .3, 0.15][severity - 1] + + x = np.array(x) / 255. + means = np.mean(x, axis=(0, 1), keepdims=True) + return np.clip((x - means) * c + means, 0, 1) * 255 + + +def brightness(x, severity=1): + c = [.05, .1, .15, .2, .3][severity - 1] + + x = np.array(x) / 255. + x = sk.color.rgb2hsv(x) + x[:, :, 2] = np.clip(x[:, :, 2] + c, 0, 1) + x = sk.color.hsv2rgb(x) + + return np.clip(x, 0, 1) * 255 + + +def saturate(x, severity=1): + c = [(0.3, 0), (0.1, 0), (1.5, 0), (2, 0.1), (2.5, 0.2)][severity - 1] + + x = np.array(x) / 255. + x = sk.color.rgb2hsv(x) + x[:, :, 1] = np.clip(x[:, :, 1] * c[0] + c[1], 0, 1) + x = sk.color.hsv2rgb(x) + + return np.clip(x, 0, 1) * 255 + + +def jpeg_compression(x, severity=1): + c = [80, 65, 58, 50, 40][severity - 1] + + output = BytesIO() + x.save(output, 'JPEG', quality=c) + x = PILImage.open(output) + + return x + + +def pixelate(x, severity=1): + c = [0.95, 0.9, 0.85, 0.75, 0.65][severity - 1] + + x = x.resize((int(32 * c), int(32 * c)), PILImage.BOX) + x = x.resize((32, 32), PILImage.BOX) + + return x + + +# mod of https://gist.github.com/erniejunior/601cdf56d2b424757de5 +def elastic_transform(image, severity=1): + IMSIZE = 32 + c = [(IMSIZE*0, IMSIZE*0, IMSIZE*0.08), + (IMSIZE*0.05, IMSIZE*0.2, IMSIZE*0.07), + (IMSIZE*0.08, IMSIZE*0.06, IMSIZE*0.06), + (IMSIZE*0.1, IMSIZE*0.04, IMSIZE*0.05), + (IMSIZE*0.1, IMSIZE*0.03, IMSIZE*0.03)][severity - 1] + + image = np.array(image, dtype=np.float32) / 255. + shape = image.shape + shape_size = shape[:2] + + # random affine + center_square = np.float32(shape_size) // 2 + square_size = min(shape_size) // 3 + pts1 = np.float32([center_square + square_size, + [center_square[0] + square_size, center_square[1] - square_size], + center_square - square_size]) + pts2 = pts1 + np.random.uniform(-c[2], c[2], size=pts1.shape).astype(np.float32) + M = cv2.getAffineTransform(pts1, pts2) + image = cv2.warpAffine(image, M, shape_size[::-1], borderMode=cv2.BORDER_REFLECT_101) + + dx = (gaussian(np.random.uniform(-1, 1, size=shape[:2]), + c[1], mode='reflect', truncate=3) * c[0]).astype(np.float32) + dy = (gaussian(np.random.uniform(-1, 1, size=shape[:2]), + c[1], mode='reflect', truncate=3) * c[0]).astype(np.float32) + dx, dy = dx[..., np.newaxis], dy[..., np.newaxis] + + x, y, z = np.meshgrid(np.arange(shape[1]), np.arange(shape[0]), np.arange(shape[2])) + indices = np.reshape(y + dy, (-1, 1)), np.reshape(x + dx, (-1, 1)), np.reshape(z, (-1, 1)) + return np.clip(map_coordinates(image, indices, order=1, mode='reflect').reshape(shape), 0, 1) * 255 + + +# /////////////// End Distortions /////////////// + +import collections + +print('Using CIFAR-10 data') + +d = collections.OrderedDict() +d['Gaussian Noise'] = gaussian_noise +d['Shot Noise'] = shot_noise +d['Impulse Noise'] = impulse_noise +d['Defocus Blur'] = defocus_blur +d['Glass Blur'] = glass_blur +d['Motion Blur'] = motion_blur +d['Zoom Blur'] = zoom_blur +d['Snow'] = snow +d['Frost'] = frost +d['Fog'] = fog +d['Brightness'] = brightness +d['Contrast'] = contrast +d['Elastic'] = elastic_transform +d['Pixelate'] = pixelate +d['JPEG'] = jpeg_compression + +d['Speckle Noise'] = speckle_noise +d['Gaussian Blur'] = gaussian_blur +d['Spatter'] = spatter +d['Saturate'] = saturate + + +test_data = dset.CIFAR10('/share/data/vision-greg/cifarpy', train=False) +convert_img = trn.Compose([trn.ToTensor(), trn.ToPILImage()]) + + +for method_name in d.keys(): + print('Creating images for the corruption', method_name) + cifar_c, labels = [], [] + + for severity in range(1,6): + corruption = lambda clean_img: d[method_name](clean_img, severity) + + for img, label in zip(test_data.data, test_data.targets): + labels.append(label) + cifar_c.append(np.uint8(corruption(convert_img(img)))) + + np.save('/share/data/vision-greg2/users/dan/datasets/CIFAR-10-C/' + d[method_name].__name__ + '.npy', + np.array(cifar_c).astype(np.uint8)) + + np.save('/share/data/vision-greg2/users/dan/datasets/CIFAR-10-C/labels.npy', + np.array(labels).astype(np.uint8)) + diff --git a/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/create_c/make_imagenet_64_c.py b/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/create_c/make_imagenet_64_c.py new file mode 100644 index 0000000000000000000000000000000000000000..1af23f7ed811d5bc8e401f9028893af32b6b4b0a --- /dev/null +++ b/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/create_c/make_imagenet_64_c.py @@ -0,0 +1,584 @@ +# -*- coding: utf-8 -*- + +import os +from PIL import Image +import os.path +import time +import torch +import torchvision.datasets as dset +import torchvision.transforms as trn +import torch.utils.data as data +import numpy as np + +from PIL import Image + +# /////////////// Data Loader /////////////// + + +IMG_EXTENSIONS = ['.jpg', '.jpeg', '.png', '.ppm', '.bmp', '.pgm'] + + +def is_image_file(filename): + """Checks if a file is an image. + Args: + filename (string): path to a file + Returns: + bool: True if the filename ends with a known image extension + """ + filename_lower = filename.lower() + return any(filename_lower.endswith(ext) for ext in IMG_EXTENSIONS) + + +def find_classes(dir): + classes = [d for d in os.listdir(dir) if os.path.isdir(os.path.join(dir, d))] + classes.sort() + class_to_idx = {classes[i]: i for i in range(len(classes))} + return classes, class_to_idx + + +def make_dataset(dir, class_to_idx): + images = [] + dir = os.path.expanduser(dir) + for target in sorted(os.listdir(dir)): + d = os.path.join(dir, target) + if not os.path.isdir(d): + continue + + for root, _, fnames in sorted(os.walk(d)): + for fname in sorted(fnames): + if is_image_file(fname): + path = os.path.join(root, fname) + item = (path, class_to_idx[target]) + images.append(item) + + return images + + +def pil_loader(path): + # open path as file to avoid ResourceWarning (https://github.com/python-pillow/Pillow/issues/835) + with open(path, 'rb') as f: + img = Image.open(f) + return img.convert('RGB') + + +def accimage_loader(path): + import accimage + try: + return accimage.Image(path) + except IOError: + # Potentially a decoding problem, fall back to PIL.Image + return pil_loader(path) + + +def default_loader(path): + from torchvision import get_image_backend + if get_image_backend() == 'accimage': + return accimage_loader(path) + else: + return pil_loader(path) + + +class DistortImageFolder(data.Dataset): + def __init__(self, root, method, severity, transform=None, target_transform=None, + loader=default_loader): + classes, class_to_idx = find_classes(root) + imgs = make_dataset(root, class_to_idx) + if len(imgs) == 0: + raise (RuntimeError("Found 0 images in subfolders of: " + root + "\n" + + "Supported image extensions are: " + ",".join(IMG_EXTENSIONS))) + + self.root = root + self.method = method + self.severity = severity + self.imgs = imgs + self.classes = classes + self.class_to_idx = class_to_idx + self.idx_to_class = {v: k for k, v in class_to_idx.items()} + self.transform = transform + self.target_transform = target_transform + self.loader = loader + + def __getitem__(self, index): + path, target = self.imgs[index] + img = self.loader(path) + if self.transform is not None: + img = self.transform(img) + img = self.method(img, self.severity) + if self.target_transform is not None: + target = self.target_transform(target) + + save_path = '/share/data/lang/users/dan/ImageNet-64x64-C/' + self.method.__name__ + \ + '/' + str(self.severity) + '/' + self.idx_to_class[target] + + if not os.path.exists(save_path): + os.makedirs(save_path) + + save_path += path[path.rindex('/'):] + + Image.fromarray(np.uint8(img)).save(save_path, quality=85, optimize=True) + + return 0 # we do not care about returning the data + + def __len__(self): + return len(self.imgs) + + +# /////////////// Distortion Helpers /////////////// + +import skimage as sk +from skimage.filters import gaussian +from io import BytesIO +from wand.image import Image as WandImage +from wand.api import library as wandlibrary +import wand.color as WandColor +import ctypes +from PIL import Image as PILImage +import cv2 +from scipy.ndimage import zoom as scizoom +from scipy.ndimage.interpolation import map_coordinates +import warnings + +warnings.simplefilter("ignore", UserWarning) + + +def auc(errs): # area under the alteration error curve + area = 0 + for i in range(1, len(errs)): + area += (errs[i] + errs[i - 1]) / 2 + area /= len(errs) - 1 + return area + + +def disk(radius, alias_blur=0.1, dtype=np.float32): + if radius <= 8: + L = np.arange(-8, 8 + 1) + ksize = (3, 3) + else: + L = np.arange(-radius, radius + 1) + ksize = (5, 5) + X, Y = np.meshgrid(L, L) + aliased_disk = np.array((X ** 2 + Y ** 2) <= radius ** 2, dtype=dtype) + aliased_disk /= np.sum(aliased_disk) + + # supersample disk to antialias + return cv2.GaussianBlur(aliased_disk, ksize=ksize, sigmaX=alias_blur) + + +# Tell Python about the C method +wandlibrary.MagickMotionBlurImage.argtypes = (ctypes.c_void_p, # wand + ctypes.c_double, # radius + ctypes.c_double, # sigma + ctypes.c_double) # angle + + +# Extend wand.image.Image class to include method signature +class MotionImage(WandImage): + def motion_blur(self, radius=0.0, sigma=0.0, angle=0.0): + wandlibrary.MagickMotionBlurImage(self.wand, radius, sigma, angle) + + +# modification of https://github.com/FLHerne/mapgen/blob/master/diamondsquare.py +def plasma_fractal(mapsize=64, wibbledecay=3): + """ + Generate a heightmap using diamond-square algorithm. + Return square 2d array, side length 'mapsize', of floats in range 0-255. + 'mapsize' must be a power of two. + """ + assert (mapsize & (mapsize - 1) == 0) + maparray = np.empty((mapsize, mapsize), dtype=np.float_) + maparray[0, 0] = 0 + stepsize = mapsize + wibble = 100 + + def wibbledmean(array): + return array / 4 + wibble * np.random.uniform(-wibble, wibble, array.shape) + + def fillsquares(): + """For each square of points stepsize apart, + calculate middle value as mean of points + wibble""" + cornerref = maparray[0:mapsize:stepsize, 0:mapsize:stepsize] + squareaccum = cornerref + np.roll(cornerref, shift=-1, axis=0) + squareaccum += np.roll(squareaccum, shift=-1, axis=1) + maparray[stepsize // 2:mapsize:stepsize, + stepsize // 2:mapsize:stepsize] = wibbledmean(squareaccum) + + def filldiamonds(): + """For each diamond of points stepsize apart, + calculate middle value as mean of points + wibble""" + mapsize = maparray.shape[0] + drgrid = maparray[stepsize // 2:mapsize:stepsize, stepsize // 2:mapsize:stepsize] + ulgrid = maparray[0:mapsize:stepsize, 0:mapsize:stepsize] + ldrsum = drgrid + np.roll(drgrid, 1, axis=0) + lulsum = ulgrid + np.roll(ulgrid, -1, axis=1) + ltsum = ldrsum + lulsum + maparray[0:mapsize:stepsize, stepsize // 2:mapsize:stepsize] = wibbledmean(ltsum) + tdrsum = drgrid + np.roll(drgrid, 1, axis=1) + tulsum = ulgrid + np.roll(ulgrid, -1, axis=0) + ttsum = tdrsum + tulsum + maparray[stepsize // 2:mapsize:stepsize, 0:mapsize:stepsize] = wibbledmean(ttsum) + + while stepsize >= 2: + fillsquares() + filldiamonds() + stepsize //= 2 + wibble /= wibbledecay + + maparray -= maparray.min() + return maparray / maparray.max() + + +def clipped_zoom(img, zoom_factor): + h = img.shape[0] + # ceil crop height(= crop width) + ch = int(np.ceil(h / zoom_factor)) + + top = (h - ch) // 2 + img = scizoom(img[top:top + ch, top:top + ch], (zoom_factor, zoom_factor, 1), order=1) + # trim off any extra pixels + trim_top = (img.shape[0] - h) // 2 + + return img[trim_top:trim_top + h, trim_top:trim_top + h] + + +# /////////////// End Distortion Helpers /////////////// + + +# /////////////// Distortions /////////////// + +def gaussian_noise(x, severity=1): + c = [0.04, 0.08, .12, .15, .18][severity - 1] + + x = np.array(x) / 255. + return np.clip(x + np.random.normal(size=x.shape, scale=c), 0, 1) * 255 + + +def shot_noise(x, severity=1): + c = [250, 100, 50, 30, 15][severity - 1] + + x = np.array(x) / 255. + return np.clip(np.random.poisson(x * c) / c, 0, 1) * 255 + + +def impulse_noise(x, severity=1): + c = [.01, .02, .05, .08, .14][severity - 1] + + x = sk.util.random_noise(np.array(x) / 255., mode='s&p', amount=c) + return np.clip(x, 0, 1) * 255 + + +def speckle_noise(x, severity=1): + c = [.15, .2, 0.25, 0.3, 0.35][severity - 1] + + x = np.array(x) / 255. + return np.clip(x + x * np.random.normal(size=x.shape, scale=c), 0, 1) * 255 + + +def gaussian_blur(x, severity=1): + c = [.5, .75, 1, 1.25, 1.5][severity - 1] + + x = gaussian(np.array(x) / 255., sigma=c, multichannel=True) + return np.clip(x, 0, 1) * 255 + + +def glass_blur(x, severity=1): + # sigma, max_delta, iterations + c = [(0.1,1,1), (0.5,1,1), (0.6,1,2), (0.7,2,1), (0.9,2,2)][severity - 1] + + x = np.uint8(gaussian(np.array(x) / 255., sigma=c[0], multichannel=True) * 255) + + # locally shuffle pixels + for i in range(c[2]): + for h in range(64 - c[1], c[1], -1): + for w in range(64 - c[1], c[1], -1): + dx, dy = np.random.randint(-c[1], c[1], size=(2,)) + h_prime, w_prime = h + dy, w + dx + # swap + x[h, w], x[h_prime, w_prime] = x[h_prime, w_prime], x[h, w] + + return np.clip(gaussian(x / 255., sigma=c[0], multichannel=True), 0, 1) * 255 + + +def defocus_blur(x, severity=1): + c = [(0.5, 0.6), (1, 0.1), (1.5, 0.1), (2.5, 0.01), (3, 0.1)][severity - 1] + + x = np.array(x) / 255. + kernel = disk(radius=c[0], alias_blur=c[1]) + + channels = [] + for d in range(3): + channels.append(cv2.filter2D(x[:, :, d], -1, kernel)) + channels = np.array(channels).transpose((1, 2, 0)) # 3x64x64 -> 64x64x3 + + return np.clip(channels, 0, 1) * 255 + + +def motion_blur(x, severity=1): + c = [(10,1), (10,1.5), (10,2), (10,2.5), (12,3)][severity - 1] + + output = BytesIO() + x.save(output, format='PNG') + x = MotionImage(blob=output.getvalue()) + + x.motion_blur(radius=c[0], sigma=c[1], angle=np.random.uniform(-45, 45)) + + x = cv2.imdecode(np.fromstring(x.make_blob(), np.uint8), + cv2.IMREAD_UNCHANGED) + + if x.shape != (64, 64): + return np.clip(x[..., [2, 1, 0]], 0, 255) # BGR to RGB + else: # greyscale to RGB + return np.clip(np.array([x, x, x]).transpose((1, 2, 0)), 0, 255) + + +def zoom_blur(x, severity=1): + c = [np.arange(1, 1.06, 0.01), np.arange(1, 1.11, 0.01), np.arange(1, 1.16, 0.01), + np.arange(1, 1.21, 0.01), np.arange(1, 1.26, 0.01)][severity - 1] + + x = (np.array(x) / 255.).astype(np.float32) + out = np.zeros_like(x) + for zoom_factor in c: + out += clipped_zoom(x, zoom_factor) + + x = (x + out) / (len(c) + 1) + return np.clip(x, 0, 1) * 255 + + +def fog(x, severity=1): + c = [(.4,3), (.7,3), (1,2.5), (1.5,2), (2,1.75)][severity - 1] + + x = np.array(x) / 255. + max_val = x.max() + x += c[0] * plasma_fractal(wibbledecay=c[1])[:64, :64][..., np.newaxis] + return np.clip(x * max_val / (max_val + c[0]), 0, 1) * 255 + + +def frost(x, severity=1): + c = [(1, 0.3), (0.9, 0.4), (0.8, 0.45), (0.75, 0.5), (0.7, 0.55)][severity - 1] + idx = np.random.randint(5) + filename = ['./frost1.png', './frost2.png', './frost3.png', './frost4.jpg', './frost5.jpg', './frost6.jpg'][idx] + frost = cv2.imread(filename) + frost = cv2.resize(frost, (0, 0), fx=0.3, fy=0.3) + # randomly crop and convert to rgb + x_start, y_start = np.random.randint(0, frost.shape[0] - 64), np.random.randint(0, frost.shape[1] - 64) + frost = frost[x_start:x_start + 64, y_start:y_start + 64][..., [2, 1, 0]] + + return np.clip(c[0] * np.array(x) + c[1] * frost, 0, 255) + + +def snow(x, severity=1): + c = [(0.1,0.2,1,0.6,8,3,0.8), + (0.1,0.2,1,0.5,10,4,0.8), + (0.15,0.3,1.75,0.55,10,4,0.7), + (0.25,0.3,2.25,0.6,12,6,0.65), + (0.3,0.3,1.25,0.65,14,12,0.6)][severity - 1] + + x = np.array(x, dtype=np.float32) / 255. + snow_layer = np.random.normal(size=x.shape[:2], loc=c[0], scale=c[1]) # [:2] for monochrome + + snow_layer = clipped_zoom(snow_layer[..., np.newaxis], c[2]) + snow_layer[snow_layer < c[3]] = 0 + + snow_layer = PILImage.fromarray((np.clip(snow_layer.squeeze(), 0, 1) * 255).astype(np.uint8), mode='L') + output = BytesIO() + snow_layer.save(output, format='PNG') + snow_layer = MotionImage(blob=output.getvalue()) + + snow_layer.motion_blur(radius=c[4], sigma=c[5], angle=np.random.uniform(-135, -45)) + + snow_layer = cv2.imdecode(np.fromstring(snow_layer.make_blob(), np.uint8), + cv2.IMREAD_UNCHANGED) / 255. + snow_layer = snow_layer[..., np.newaxis] + + x = c[6] * x + (1 - c[6]) * np.maximum(x, cv2.cvtColor(x, cv2.COLOR_RGB2GRAY).reshape(64, 64, 1) * 1.5 + 0.5) + return np.clip(x + snow_layer + np.rot90(snow_layer, k=2), 0, 1) * 255 + + +def spatter(x, severity=1): + c = [(0.62,0.1,0.7,0.7,0.6,0), + (0.65,0.1,0.8,0.7,0.6,0), + (0.65,0.3,1,0.69,0.6,0), + (0.65,0.1,0.7,0.68,0.6,1), + (0.65,0.1,0.5,0.67,0.6,1)][severity - 1] + x = np.array(x, dtype=np.float32) / 255. + + liquid_layer = np.random.normal(size=x.shape[:2], loc=c[0], scale=c[1]) + + liquid_layer = gaussian(liquid_layer, sigma=c[2]) + liquid_layer[liquid_layer < c[3]] = 0 + if c[5] == 0: + liquid_layer = (liquid_layer * 255).astype(np.uint8) + dist = 255 - cv2.Canny(liquid_layer, 50, 150) + dist = cv2.distanceTransform(dist, cv2.DIST_L2, 5) + _, dist = cv2.threshold(dist, 20, 20, cv2.THRESH_TRUNC) + dist = cv2.blur(dist, (3, 3)).astype(np.uint8) + dist = cv2.equalizeHist(dist) + # ker = np.array([[-1,-2,-3],[-2,0,0],[-3,0,1]], dtype=np.float32) + # ker -= np.mean(ker) + ker = np.array([[-2, -1, 0], [-1, 1, 1], [0, 1, 2]]) + dist = cv2.filter2D(dist, cv2.CV_8U, ker) + dist = cv2.blur(dist, (3, 3)).astype(np.float32) + + m = cv2.cvtColor(liquid_layer * dist, cv2.COLOR_GRAY2BGRA) + m /= np.max(m, axis=(0, 1)) + m *= c[4] + + # water is pale turqouise + color = np.concatenate((175 / 255. * np.ones_like(m[..., :1]), + 238 / 255. * np.ones_like(m[..., :1]), + 238 / 255. * np.ones_like(m[..., :1])), axis=2) + + color = cv2.cvtColor(color, cv2.COLOR_BGR2BGRA) + x = cv2.cvtColor(x, cv2.COLOR_BGR2BGRA) + + return cv2.cvtColor(np.clip(x + m * color, 0, 1), cv2.COLOR_BGRA2BGR) * 255 + else: + m = np.where(liquid_layer > c[3], 1, 0) + m = gaussian(m.astype(np.float32), sigma=c[4]) + m[m < 0.8] = 0 + # m = np.abs(m) ** (1/c[4]) + + # mud brown + color = np.concatenate((63 / 255. * np.ones_like(x[..., :1]), + 42 / 255. * np.ones_like(x[..., :1]), + 20 / 255. * np.ones_like(x[..., :1])), axis=2) + + color *= m[..., np.newaxis] + x *= (1 - m[..., np.newaxis]) + + return np.clip(x + color, 0, 1) * 255 + + +def contrast(x, severity=1): + c = [.4, .3, .2, .1, 0.05][severity - 1] + + x = np.array(x) / 255. + means = np.mean(x, axis=(0, 1), keepdims=True) + return np.clip((x - means) * c + means, 0, 1) * 255 + + +def brightness(x, severity=1): + c = [.1, .2, .3, .4, .5][severity - 1] + + x = np.array(x) / 255. + x = sk.color.rgb2hsv(x) + x[:, :, 2] = np.clip(x[:, :, 2] + c, 0, 1) + x = sk.color.hsv2rgb(x) + + return np.clip(x, 0, 1) * 255 + + +def saturate(x, severity=1): + c = [(0.3, 0), (0.1, 0), (2, 0), (5, 0.1), (30, 0.2)][severity - 1] + + x = np.array(x) / 255. + x = sk.color.rgb2hsv(x) + x[:, :, 1] = np.clip(x[:, :, 1] * c[0] + c[1], 0, 1) + x = sk.color.hsv2rgb(x) + + return np.clip(x, 0, 1) * 255 + + +def jpeg_compression(x, severity=1): + c = [65, 58, 50, 40, 25][severity - 1] + + output = BytesIO() + x.save(output, 'JPEG', quality=c) + x = PILImage.open(output) + + return x + + +def pixelate(x, severity=1): + c = [0.9, 0.8, 0.7, 0.6, 0.5][severity - 1] + + x = x.resize((int(64 * c), int(64 * c)), PILImage.BOX) + x = x.resize((64, 64), PILImage.BOX) + + return x + + +# mod of https://gist.github.com/erniejunior/601cdf56d2b424757de5 +def elastic_transform(image, severity=1): + IMSIZE = 64 + c = [(IMSIZE*0, IMSIZE*0, IMSIZE*0.08), + (IMSIZE*0.05, IMSIZE*0.3, IMSIZE*0.06), + (IMSIZE*0.1, IMSIZE*0.08, IMSIZE*0.06), + (IMSIZE*0.1, IMSIZE*0.03, IMSIZE*0.03), + (IMSIZE*0.16, IMSIZE*0.03, IMSIZE*0.02)][severity - 1] + + image = np.array(image, dtype=np.float32) / 255. + shape = image.shape + shape_size = shape[:2] + + # random affine + center_square = np.float32(shape_size) // 2 + square_size = min(shape_size) // 3 + pts1 = np.float32([center_square + square_size, + [center_square[0] + square_size, center_square[1] - square_size], + center_square - square_size]) + pts2 = pts1 + np.random.uniform(-c[2], c[2], size=pts1.shape).astype(np.float32) + M = cv2.getAffineTransform(pts1, pts2) + image = cv2.warpAffine(image, M, shape_size[::-1], borderMode=cv2.BORDER_REFLECT_101) + + dx = (gaussian(np.random.uniform(-1, 1, size=shape[:2]), + c[1], mode='reflect', truncate=3) * c[0]).astype(np.float32) + dy = (gaussian(np.random.uniform(-1, 1, size=shape[:2]), + c[1], mode='reflect', truncate=3) * c[0]).astype(np.float32) + dx, dy = dx[..., np.newaxis], dy[..., np.newaxis] + + x, y, z = np.meshgrid(np.arange(shape[1]), np.arange(shape[0]), np.arange(shape[2])) + indices = np.reshape(y + dy, (-1, 1)), np.reshape(x + dx, (-1, 1)), np.reshape(z, (-1, 1)) + return np.clip(map_coordinates(image, indices, order=1, mode='reflect').reshape(shape), 0, 1) * 255 + + +# /////////////// End Distortions /////////////// + + +# /////////////// Further Setup /////////////// + + +def save_distorted(method=gaussian_noise): + for severity in range(1, 6): + print(method.__name__, severity) + distorted_dataset = DistortImageFolder( + root="/share/data/vision-greg/ImageNet/clsloc/images/val", + method=method, severity=severity, + transform=trn.Compose([trn.Resize((64, 64))])) + distorted_dataset_loader = torch.utils.data.DataLoader( + distorted_dataset, batch_size=100, shuffle=False, num_workers=6) + + for _ in distorted_dataset_loader: continue + + +# /////////////// End Further Setup /////////////// + + +# /////////////// Display Results /////////////// +import collections + +print('\nUsing ImageNet data') + +d = collections.OrderedDict() +# d['Gaussian Noise'] = gaussian_noise +# d['Shot Noise'] = shot_noise +# d['Impulse Noise'] = impulse_noise +# d['Defocus Blur'] = defocus_blur +# d['Glass Blur'] = glass_blur +# d['Motion Blur'] = motion_blur +# d['Zoom Blur'] = zoom_blur +# d['Snow'] = snow +d['Frost'] = frost +d['Fog'] = fog +d['Brightness'] = brightness +d['Contrast'] = contrast +# d['Elastic'] = elastic_transform +# d['Pixelate'] = pixelate +# d['JPEG'] = jpeg_compression +# +# d['Speckle Noise'] = speckle_noise +# d['Gaussian Blur'] = gaussian_blur +# d['Spatter'] = spatter +# d['Saturate'] = saturate + +for method_name in d.keys(): + save_distorted(d[method_name]) diff --git a/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/create_c/make_imagenet_c.py b/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/create_c/make_imagenet_c.py new file mode 100644 index 0000000000000000000000000000000000000000..01e0913506be91be59258e3293499b8cafa6329f --- /dev/null +++ b/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/create_c/make_imagenet_c.py @@ -0,0 +1,621 @@ +# -*- coding: utf-8 -*- + +import os +from PIL import Image +import os.path +import time +import torch +import torchvision.datasets as dset +import torchvision.transforms as trn +import torch.utils.data as data +import numpy as np + +from PIL import Image + +# /////////////// Data Loader /////////////// + + +IMG_EXTENSIONS = ['.jpg', '.jpeg', '.png', '.ppm', '.bmp', '.pgm'] + + +def is_image_file(filename): + """Checks if a file is an image. + Args: + filename (string): path to a file + Returns: + bool: True if the filename ends with a known image extension + """ + filename_lower = filename.lower() + return any(filename_lower.endswith(ext) for ext in IMG_EXTENSIONS) + + +def find_classes(dir): + classes = [d for d in os.listdir(dir) if os.path.isdir(os.path.join(dir, d))] + classes.sort() + class_to_idx = {classes[i]: i for i in range(len(classes))} + return classes, class_to_idx + + +def make_dataset(dir, class_to_idx): + images = [] + dir = os.path.expanduser(dir) + for target in sorted(os.listdir(dir)): + d = os.path.join(dir, target) + if not os.path.isdir(d): + continue + + for root, _, fnames in sorted(os.walk(d)): + for fname in sorted(fnames): + if is_image_file(fname): + path = os.path.join(root, fname) + item = (path, class_to_idx[target]) + images.append(item) + + return images + + +def pil_loader(path): + # open path as file to avoid ResourceWarning (https://github.com/python-pillow/Pillow/issues/835) + with open(path, 'rb') as f: + img = Image.open(f) + return img.convert('RGB') + + +def accimage_loader(path): + import accimage + try: + return accimage.Image(path) + except IOError: + # Potentially a decoding problem, fall back to PIL.Image + return pil_loader(path) + + +def default_loader(path): + from torchvision import get_image_backend + if get_image_backend() == 'accimage': + return accimage_loader(path) + else: + return pil_loader(path) + + +class DistortImageFolder(data.Dataset): + def __init__(self, root, method, severity, transform=None, target_transform=None, + loader=default_loader): + classes, class_to_idx = find_classes(root) + imgs = make_dataset(root, class_to_idx) + if len(imgs) == 0: + raise (RuntimeError("Found 0 images in subfolders of: " + root + "\n" + "Supported image extensions are: " + ",".join( + IMG_EXTENSIONS))) + + self.root = root + self.method = method + self.severity = severity + self.imgs = imgs + self.classes = classes + self.class_to_idx = class_to_idx + self.idx_to_class = {v: k for k, v in class_to_idx.items()} + self.transform = transform + self.target_transform = target_transform + self.loader = loader + + def __getitem__(self, index): + path, target = self.imgs[index] + img = self.loader(path) + if self.transform is not None: + img = self.transform(img) + img = self.method(img, self.severity) + if self.target_transform is not None: + target = self.target_transform(target) + + save_path = '/share/data/vision-greg/DistortedImageNet/JPEG/' + self.method.__name__ + \ + '/' + str(self.severity) + '/' + self.idx_to_class[target] + + if not os.path.exists(save_path): + os.makedirs(save_path) + + save_path += path[path.rindex('/'):] + + Image.fromarray(np.uint8(img)).save(save_path, quality=85, optimize=True) + + return 0 # we do not care about returning the data + + def __len__(self): + return len(self.imgs) + + +# /////////////// Distortion Helpers /////////////// + +import skimage as sk +from skimage.filters import gaussian +from io import BytesIO +from wand.image import Image as WandImage +from wand.api import library as wandlibrary +import wand.color as WandColor +import ctypes +from PIL import Image as PILImage +import cv2 +from scipy.ndimage import zoom as scizoom +from scipy.ndimage.interpolation import map_coordinates +import warnings + +warnings.simplefilter("ignore", UserWarning) + + +def auc(errs): # area under the alteration error curve + area = 0 + for i in range(1, len(errs)): + area += (errs[i] + errs[i - 1]) / 2 + area /= len(errs) - 1 + return area + + +def disk(radius, alias_blur=0.1, dtype=np.float32): + if radius <= 8: + L = np.arange(-8, 8 + 1) + ksize = (3, 3) + else: + L = np.arange(-radius, radius + 1) + ksize = (5, 5) + X, Y = np.meshgrid(L, L) + aliased_disk = np.array((X ** 2 + Y ** 2) <= radius ** 2, dtype=dtype) + aliased_disk /= np.sum(aliased_disk) + + # supersample disk to antialias + return cv2.GaussianBlur(aliased_disk, ksize=ksize, sigmaX=alias_blur) + + +# Tell Python about the C method +wandlibrary.MagickMotionBlurImage.argtypes = (ctypes.c_void_p, # wand + ctypes.c_double, # radius + ctypes.c_double, # sigma + ctypes.c_double) # angle + + +# Extend wand.image.Image class to include method signature +class MotionImage(WandImage): + def motion_blur(self, radius=0.0, sigma=0.0, angle=0.0): + wandlibrary.MagickMotionBlurImage(self.wand, radius, sigma, angle) + + +# modification of https://github.com/FLHerne/mapgen/blob/master/diamondsquare.py +def plasma_fractal(mapsize=256, wibbledecay=3): + """ + Generate a heightmap using diamond-square algorithm. + Return square 2d array, side length 'mapsize', of floats in range 0-255. + 'mapsize' must be a power of two. + """ + assert (mapsize & (mapsize - 1) == 0) + maparray = np.empty((mapsize, mapsize), dtype=np.float_) + maparray[0, 0] = 0 + stepsize = mapsize + wibble = 100 + + def wibbledmean(array): + return array / 4 + wibble * np.random.uniform(-wibble, wibble, array.shape) + + def fillsquares(): + """For each square of points stepsize apart, + calculate middle value as mean of points + wibble""" + cornerref = maparray[0:mapsize:stepsize, 0:mapsize:stepsize] + squareaccum = cornerref + np.roll(cornerref, shift=-1, axis=0) + squareaccum += np.roll(squareaccum, shift=-1, axis=1) + maparray[stepsize // 2:mapsize:stepsize, + stepsize // 2:mapsize:stepsize] = wibbledmean(squareaccum) + + def filldiamonds(): + """For each diamond of points stepsize apart, + calculate middle value as mean of points + wibble""" + mapsize = maparray.shape[0] + drgrid = maparray[stepsize // 2:mapsize:stepsize, stepsize // 2:mapsize:stepsize] + ulgrid = maparray[0:mapsize:stepsize, 0:mapsize:stepsize] + ldrsum = drgrid + np.roll(drgrid, 1, axis=0) + lulsum = ulgrid + np.roll(ulgrid, -1, axis=1) + ltsum = ldrsum + lulsum + maparray[0:mapsize:stepsize, stepsize // 2:mapsize:stepsize] = wibbledmean(ltsum) + tdrsum = drgrid + np.roll(drgrid, 1, axis=1) + tulsum = ulgrid + np.roll(ulgrid, -1, axis=0) + ttsum = tdrsum + tulsum + maparray[stepsize // 2:mapsize:stepsize, 0:mapsize:stepsize] = wibbledmean(ttsum) + + while stepsize >= 2: + fillsquares() + filldiamonds() + stepsize //= 2 + wibble /= wibbledecay + + maparray -= maparray.min() + return maparray / maparray.max() + + +def clipped_zoom(img, zoom_factor): + h = img.shape[0] + # ceil crop height(= crop width) + ch = int(np.ceil(h / zoom_factor)) + + top = (h - ch) // 2 + img = scizoom(img[top:top + ch, top:top + ch], (zoom_factor, zoom_factor, 1), order=1) + # trim off any extra pixels + trim_top = (img.shape[0] - h) // 2 + + return img[trim_top:trim_top + h, trim_top:trim_top + h] + + +# /////////////// End Distortion Helpers /////////////// + + +# /////////////// Distortions /////////////// + +def gaussian_noise(x, severity=1): + c = [.08, .12, 0.18, 0.26, 0.38][severity - 1] + + x = np.array(x) / 255. + return np.clip(x + np.random.normal(size=x.shape, scale=c), 0, 1) * 255 + + +def shot_noise(x, severity=1): + c = [60, 25, 12, 5, 3][severity - 1] + + x = np.array(x) / 255. + return np.clip(np.random.poisson(x * c) / c, 0, 1) * 255 + + +def impulse_noise(x, severity=1): + c = [.03, .06, .09, 0.17, 0.27][severity - 1] + + x = sk.util.random_noise(np.array(x) / 255., mode='s&p', amount=c) + return np.clip(x, 0, 1) * 255 + + +def speckle_noise(x, severity=1): + c = [.15, .2, 0.35, 0.45, 0.6][severity - 1] + + x = np.array(x) / 255. + return np.clip(x + x * np.random.normal(size=x.shape, scale=c), 0, 1) * 255 + + +def fgsm(x, source_net, severity=1): + c = [8, 16, 32, 64, 128][severity - 1] + + x = V(x, requires_grad=True) + logits = source_net(x) + source_net.zero_grad() + loss = F.cross_entropy(logits, V(logits.data.max(1)[1].squeeze_()), size_average=False) + loss.backward() + + return standardize(torch.clamp(unstandardize(x.data) + c / 255. * unstandardize(torch.sign(x.grad.data)), 0, 1)) + + +def gaussian_blur(x, severity=1): + c = [1, 2, 3, 4, 6][severity - 1] + + x = gaussian(np.array(x) / 255., sigma=c, multichannel=True) + return np.clip(x, 0, 1) * 255 + + +def glass_blur(x, severity=1): + # sigma, max_delta, iterations + c = [(0.7, 1, 2), (0.9, 2, 1), (1, 2, 3), (1.1, 3, 2), (1.5, 4, 2)][severity - 1] + + x = np.uint8(gaussian(np.array(x) / 255., sigma=c[0], multichannel=True) * 255) + + # locally shuffle pixels + for i in range(c[2]): + for h in range(224 - c[1], c[1], -1): + for w in range(224 - c[1], c[1], -1): + dx, dy = np.random.randint(-c[1], c[1], size=(2,)) + h_prime, w_prime = h + dy, w + dx + # swap + x[h, w], x[h_prime, w_prime] = x[h_prime, w_prime], x[h, w] + + return np.clip(gaussian(x / 255., sigma=c[0], multichannel=True), 0, 1) * 255 + + +def defocus_blur(x, severity=1): + c = [(3, 0.1), (4, 0.5), (6, 0.5), (8, 0.5), (10, 0.5)][severity - 1] + + x = np.array(x) / 255. + kernel = disk(radius=c[0], alias_blur=c[1]) + + channels = [] + for d in range(3): + channels.append(cv2.filter2D(x[:, :, d], -1, kernel)) + channels = np.array(channels).transpose((1, 2, 0)) # 3x224x224 -> 224x224x3 + + return np.clip(channels, 0, 1) * 255 + + +def motion_blur(x, severity=1): + c = [(10, 3), (15, 5), (15, 8), (15, 12), (20, 15)][severity - 1] + + output = BytesIO() + x.save(output, format='PNG') + x = MotionImage(blob=output.getvalue()) + + x.motion_blur(radius=c[0], sigma=c[1], angle=np.random.uniform(-45, 45)) + + x = cv2.imdecode(np.fromstring(x.make_blob(), np.uint8), + cv2.IMREAD_UNCHANGED) + + if x.shape != (224, 224): + return np.clip(x[..., [2, 1, 0]], 0, 255) # BGR to RGB + else: # greyscale to RGB + return np.clip(np.array([x, x, x]).transpose((1, 2, 0)), 0, 255) + + +def zoom_blur(x, severity=1): + c = [np.arange(1, 1.11, 0.01), + np.arange(1, 1.16, 0.01), + np.arange(1, 1.21, 0.02), + np.arange(1, 1.26, 0.02), + np.arange(1, 1.31, 0.03)][severity - 1] + + x = (np.array(x) / 255.).astype(np.float32) + out = np.zeros_like(x) + for zoom_factor in c: + out += clipped_zoom(x, zoom_factor) + + x = (x + out) / (len(c) + 1) + return np.clip(x, 0, 1) * 255 + + +# def barrel(x, severity=1): +# c = [(0,0.03,0.03), (0.05,0.05,0.05), (0.1,0.1,0.1), +# (0.2,0.2,0.2), (0.1,0.3,0.6)][severity - 1] +# +# output = BytesIO() +# x.save(output, format='PNG') +# +# x = WandImage(blob=output.getvalue()) +# x.distort('barrel', c) +# +# x = cv2.imdecode(np.fromstring(x.make_blob(), np.uint8), +# cv2.IMREAD_UNCHANGED) +# +# if x.shape != (224, 224): +# return np.clip(x[..., [2, 1, 0]], 0, 255) # BGR to RGB +# else: # greyscale to RGB +# return np.clip(np.array([x, x, x]).transpose((1, 2, 0)), 0, 255) + + +def fog(x, severity=1): + c = [(1.5, 2), (2, 2), (2.5, 1.7), (2.5, 1.5), (3, 1.4)][severity - 1] + + x = np.array(x) / 255. + max_val = x.max() + x += c[0] * plasma_fractal(wibbledecay=c[1])[:224, :224][..., np.newaxis] + return np.clip(x * max_val / (max_val + c[0]), 0, 1) * 255 + + +def frost(x, severity=1): + c = [(1, 0.4), + (0.8, 0.6), + (0.7, 0.7), + (0.65, 0.7), + (0.6, 0.75)][severity - 1] + idx = np.random.randint(5) + filename = ['./frost1.png', './frost2.png', './frost3.png', './frost4.jpg', './frost5.jpg', './frost6.jpg'][idx] + frost = cv2.imread(filename) + # randomly crop and convert to rgb + x_start, y_start = np.random.randint(0, frost.shape[0] - 224), np.random.randint(0, frost.shape[1] - 224) + frost = frost[x_start:x_start + 224, y_start:y_start + 224][..., [2, 1, 0]] + + return np.clip(c[0] * np.array(x) + c[1] * frost, 0, 255) + + +def snow(x, severity=1): + c = [(0.1, 0.3, 3, 0.5, 10, 4, 0.8), + (0.2, 0.3, 2, 0.5, 12, 4, 0.7), + (0.55, 0.3, 4, 0.9, 12, 8, 0.7), + (0.55, 0.3, 4.5, 0.85, 12, 8, 0.65), + (0.55, 0.3, 2.5, 0.85, 12, 12, 0.55)][severity - 1] + + x = np.array(x, dtype=np.float32) / 255. + snow_layer = np.random.normal(size=x.shape[:2], loc=c[0], scale=c[1]) # [:2] for monochrome + + snow_layer = clipped_zoom(snow_layer[..., np.newaxis], c[2]) + snow_layer[snow_layer < c[3]] = 0 + + snow_layer = PILImage.fromarray((np.clip(snow_layer.squeeze(), 0, 1) * 255).astype(np.uint8), mode='L') + output = BytesIO() + snow_layer.save(output, format='PNG') + snow_layer = MotionImage(blob=output.getvalue()) + + snow_layer.motion_blur(radius=c[4], sigma=c[5], angle=np.random.uniform(-135, -45)) + + snow_layer = cv2.imdecode(np.fromstring(snow_layer.make_blob(), np.uint8), + cv2.IMREAD_UNCHANGED) / 255. + snow_layer = snow_layer[..., np.newaxis] + + x = c[6] * x + (1 - c[6]) * np.maximum(x, cv2.cvtColor(x, cv2.COLOR_RGB2GRAY).reshape(224, 224, 1) * 1.5 + 0.5) + return np.clip(x + snow_layer + np.rot90(snow_layer, k=2), 0, 1) * 255 + + +def spatter(x, severity=1): + c = [(0.65, 0.3, 4, 0.69, 0.6, 0), + (0.65, 0.3, 3, 0.68, 0.6, 0), + (0.65, 0.3, 2, 0.68, 0.5, 0), + (0.65, 0.3, 1, 0.65, 1.5, 1), + (0.67, 0.4, 1, 0.65, 1.5, 1)][severity - 1] + x = np.array(x, dtype=np.float32) / 255. + + liquid_layer = np.random.normal(size=x.shape[:2], loc=c[0], scale=c[1]) + + liquid_layer = gaussian(liquid_layer, sigma=c[2]) + liquid_layer[liquid_layer < c[3]] = 0 + if c[5] == 0: + liquid_layer = (liquid_layer * 255).astype(np.uint8) + dist = 255 - cv2.Canny(liquid_layer, 50, 150) + dist = cv2.distanceTransform(dist, cv2.DIST_L2, 5) + _, dist = cv2.threshold(dist, 20, 20, cv2.THRESH_TRUNC) + dist = cv2.blur(dist, (3, 3)).astype(np.uint8) + dist = cv2.equalizeHist(dist) + # ker = np.array([[-1,-2,-3],[-2,0,0],[-3,0,1]], dtype=np.float32) + # ker -= np.mean(ker) + ker = np.array([[-2, -1, 0], [-1, 1, 1], [0, 1, 2]]) + dist = cv2.filter2D(dist, cv2.CV_8U, ker) + dist = cv2.blur(dist, (3, 3)).astype(np.float32) + + m = cv2.cvtColor(liquid_layer * dist, cv2.COLOR_GRAY2BGRA) + m /= np.max(m, axis=(0, 1)) + m *= c[4] + + # water is pale turqouise + color = np.concatenate((175 / 255. * np.ones_like(m[..., :1]), + 238 / 255. * np.ones_like(m[..., :1]), + 238 / 255. * np.ones_like(m[..., :1])), axis=2) + + color = cv2.cvtColor(color, cv2.COLOR_BGR2BGRA) + x = cv2.cvtColor(x, cv2.COLOR_BGR2BGRA) + + return cv2.cvtColor(np.clip(x + m * color, 0, 1), cv2.COLOR_BGRA2BGR) * 255 + else: + m = np.where(liquid_layer > c[3], 1, 0) + m = gaussian(m.astype(np.float32), sigma=c[4]) + m[m < 0.8] = 0 + # m = np.abs(m) ** (1/c[4]) + + # mud brown + color = np.concatenate((63 / 255. * np.ones_like(x[..., :1]), + 42 / 255. * np.ones_like(x[..., :1]), + 20 / 255. * np.ones_like(x[..., :1])), axis=2) + + color *= m[..., np.newaxis] + x *= (1 - m[..., np.newaxis]) + + return np.clip(x + color, 0, 1) * 255 + + +def contrast(x, severity=1): + c = [0.4, .3, .2, .1, .05][severity - 1] + + x = np.array(x) / 255. + means = np.mean(x, axis=(0, 1), keepdims=True) + return np.clip((x - means) * c + means, 0, 1) * 255 + + +def brightness(x, severity=1): + c = [.1, .2, .3, .4, .5][severity - 1] + + x = np.array(x) / 255. + x = sk.color.rgb2hsv(x) + x[:, :, 2] = np.clip(x[:, :, 2] + c, 0, 1) + x = sk.color.hsv2rgb(x) + + return np.clip(x, 0, 1) * 255 + + +def saturate(x, severity=1): + c = [(0.3, 0), (0.1, 0), (2, 0), (5, 0.1), (20, 0.2)][severity - 1] + + x = np.array(x) / 255. + x = sk.color.rgb2hsv(x) + x[:, :, 1] = np.clip(x[:, :, 1] * c[0] + c[1], 0, 1) + x = sk.color.hsv2rgb(x) + + return np.clip(x, 0, 1) * 255 + + +def jpeg_compression(x, severity=1): + c = [25, 18, 15, 10, 7][severity - 1] + + output = BytesIO() + x.save(output, 'JPEG', quality=c) + x = PILImage.open(output) + + return x + + +def pixelate(x, severity=1): + c = [0.6, 0.5, 0.4, 0.3, 0.25][severity - 1] + + x = x.resize((int(224 * c), int(224 * c)), PILImage.BOX) + x = x.resize((224, 224), PILImage.BOX) + + return x + + +# mod of https://gist.github.com/erniejunior/601cdf56d2b424757de5 +def elastic_transform(image, severity=1): + c = [(244 * 2, 244 * 0.7, 244 * 0.1), # 244 should have been 224, but ultimately nothing is incorrect + (244 * 2, 244 * 0.08, 244 * 0.2), + (244 * 0.05, 244 * 0.01, 244 * 0.02), + (244 * 0.07, 244 * 0.01, 244 * 0.02), + (244 * 0.12, 244 * 0.01, 244 * 0.02)][severity - 1] + + image = np.array(image, dtype=np.float32) / 255. + shape = image.shape + shape_size = shape[:2] + + # random affine + center_square = np.float32(shape_size) // 2 + square_size = min(shape_size) // 3 + pts1 = np.float32([center_square + square_size, + [center_square[0] + square_size, center_square[1] - square_size], + center_square - square_size]) + pts2 = pts1 + np.random.uniform(-c[2], c[2], size=pts1.shape).astype(np.float32) + M = cv2.getAffineTransform(pts1, pts2) + image = cv2.warpAffine(image, M, shape_size[::-1], borderMode=cv2.BORDER_REFLECT_101) + + dx = (gaussian(np.random.uniform(-1, 1, size=shape[:2]), + c[1], mode='reflect', truncate=3) * c[0]).astype(np.float32) + dy = (gaussian(np.random.uniform(-1, 1, size=shape[:2]), + c[1], mode='reflect', truncate=3) * c[0]).astype(np.float32) + dx, dy = dx[..., np.newaxis], dy[..., np.newaxis] + + x, y, z = np.meshgrid(np.arange(shape[1]), np.arange(shape[0]), np.arange(shape[2])) + indices = np.reshape(y + dy, (-1, 1)), np.reshape(x + dx, (-1, 1)), np.reshape(z, (-1, 1)) + return np.clip(map_coordinates(image, indices, order=1, mode='reflect').reshape(shape), 0, 1) * 255 + + +# /////////////// End Distortions /////////////// + + +# /////////////// Further Setup /////////////// + + +def save_distorted(method=gaussian_noise): + for severity in range(1, 6): + print(method.__name__, severity) + distorted_dataset = DistortImageFolder( + root="/share/data/vision-greg/ImageNet/clsloc/images/val", + method=method, severity=severity, + transform=trn.Compose([trn.Resize(256), trn.CenterCrop(224)])) + distorted_dataset_loader = torch.utils.data.DataLoader( + distorted_dataset, batch_size=100, shuffle=False, num_workers=4) + + for _ in distorted_dataset_loader: continue + + +# /////////////// End Further Setup /////////////// + + +# /////////////// Display Results /////////////// +import collections + +print('\nUsing ImageNet data') + +d = collections.OrderedDict() +d['Gaussian Noise'] = gaussian_noise +d['Shot Noise'] = shot_noise +d['Impulse Noise'] = impulse_noise +d['Defocus Blur'] = defocus_blur +d['Glass Blur'] = glass_blur +d['Motion Blur'] = motion_blur +d['Zoom Blur'] = zoom_blur +d['Snow'] = snow +d['Frost'] = frost +d['Fog'] = fog +d['Brightness'] = brightness +d['Contrast'] = contrast +d['Elastic'] = elastic_transform +d['Pixelate'] = pixelate +d['JPEG'] = jpeg_compression + +d['Speckle Noise'] = speckle_noise +d['Gaussian Blur'] = gaussian_blur +d['Spatter'] = spatter +d['Saturate'] = saturate + +for method_name in d.keys(): + save_distorted(d[method_name]) diff --git a/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/create_c/make_imagenet_c_inception.py b/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/create_c/make_imagenet_c_inception.py new file mode 100644 index 0000000000000000000000000000000000000000..9f1dc6c22b6db2a824bd9c6f3ae5f90c9ef67dc0 --- /dev/null +++ b/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/create_c/make_imagenet_c_inception.py @@ -0,0 +1,635 @@ +# -*- coding: utf-8 -*- + +import os +from PIL import Image +import os.path +import time +import torch +import torchvision.datasets as dset +import torchvision.transforms as trn +import torch.utils.data as data +import numpy as np + +from PIL import Image +import numbers + +# /////////////// Data Loader /////////////// + + +def resize(img, size, interpolation=Image.BILINEAR): + if isinstance(size, int): + w, h = img.size + if (w <= h and w == size) or (h <= w and h == size): + return img + if w < h: + ow = size + oh = int(size * h / w) + return img.resize((ow, oh), interpolation) + else: + oh = size + ow = int(size * w / h) + return img.resize((ow, oh), interpolation) + else: + return img.resize(size[::-1], interpolation) + + +def center_crop(img, output_size): + w, h = img.size + # Case of single value provided + if isinstance(output_size, numbers.Number): + # Float case: constraint for fraction must be from 0 to 1.0 + if isinstance(output_size, float): + if not 0.0 < output_size <= 1.0: + raise ValueError("Invalid float output size. Range is (0.0, 1.0]") + output_size = (output_size, output_size) + th, tw = int(h * output_size[0]), int(w * output_size[1]) + elif isinstance(output_size, int): + output_size = (output_size, output_size) + th, tw = output_size + # Case of tuple of values provided + else: + if isinstance(output_size[0], float): + th, tw = int(h * output_size[0]), int(w * output_size[1]) + elif isinstance(output_size[0], int): + th, tw = output_size + + i = int(round((h - th) / 2.)) + j = int(round((w - tw) / 2.)) + return img.crop((j, i, j + tw, i + th)) + + +def resized_center_crop(img, scale=0.875, size=(299, 299), interpolation=Image.BILINEAR): + img = center_crop(img, scale) + img = resize(img, size, interpolation) + + return img + + +IMG_EXTENSIONS = ['.jpg', '.jpeg', '.png', '.ppm', '.bmp', '.pgm'] + + +def is_image_file(filename): + """Checks if a file is an image. + Args: + filename (string): path to a file + Returns: + bool: True if the filename ends with a known image extension + """ + filename_lower = filename.lower() + return any(filename_lower.endswith(ext) for ext in IMG_EXTENSIONS) + + +def find_classes(dir): + classes = [d for d in os.listdir(dir) if os.path.isdir(os.path.join(dir, d))] + classes.sort() + class_to_idx = {classes[i]: i for i in range(len(classes))} + return classes, class_to_idx + + +def make_dataset(dir, class_to_idx): + images = [] + dir = os.path.expanduser(dir) + for target in sorted(os.listdir(dir)): + d = os.path.join(dir, target) + if not os.path.isdir(d): + continue + + for root, _, fnames in sorted(os.walk(d)): + for fname in sorted(fnames): + if is_image_file(fname): + path = os.path.join(root, fname) + item = (path, class_to_idx[target]) + images.append(item) + + return images + + +def pil_loader(path): + # open path as file to avoid ResourceWarning (https://github.com/python-pillow/Pillow/issues/835) + with open(path, 'rb') as f: + img = Image.open(f) + return img.convert('RGB') + + +def accimage_loader(path): + import accimage + try: + return accimage.Image(path) + except IOError: + # Potentially a decoding problem, fall back to PIL.Image + return pil_loader(path) + + +def default_loader(path): + from torchvision import get_image_backend + if get_image_backend() == 'accimage': + return accimage_loader(path) + else: + return pil_loader(path) + + +class DistortImageFolder(data.Dataset): + def __init__(self, root, method, severity, transform=None, target_transform=None, + loader=default_loader): + classes, class_to_idx = find_classes(root) + imgs = make_dataset(root, class_to_idx) + if len(imgs) == 0: + raise (RuntimeError("Found 0 images in subfolders of: " + root + "\n" + + "Supported image extensions are: " + ",".join(IMG_EXTENSIONS))) + + self.root = root + self.method = method + self.severity = severity + self.imgs = imgs + self.classes = classes + self.class_to_idx = class_to_idx + self.idx_to_class = {v: k for k, v in class_to_idx.items()} + self.transform = transform + self.target_transform = target_transform + self.loader = loader + + def __getitem__(self, index): + path, target = self.imgs[index] + img = self.loader(path) + if self.transform is not None: + img = self.transform(img) + img = self.method(img, self.severity) + if self.target_transform is not None: + target = self.target_transform(target) + + save_path = '/share/data/lang/users/dan/ImageNet-C-299/' + self.method.__name__ + \ + '/' + str(self.severity) + '/' + self.idx_to_class[target] + + if not os.path.exists(save_path): + os.makedirs(save_path) + + save_path += path[path.rindex('/'):] + + Image.fromarray(np.uint8(img)).save(save_path, quality=85, optimize=True) + + return 0 # we do not care about returning the data + + def __len__(self): + return len(self.imgs) + + +# /////////////// Distortion Helpers /////////////// + +import skimage as sk +from skimage.filters import gaussian +from io import BytesIO +from wand.image import Image as WandImage +from wand.api import library as wandlibrary +import wand.color as WandColor +import ctypes +from PIL import Image as PILImage +import cv2 +from scipy.ndimage import zoom as scizoom +from scipy.ndimage.interpolation import map_coordinates +import warnings + +warnings.simplefilter("ignore", UserWarning) + + +def auc(errs): # area under the alteration error curve + area = 0 + for i in range(1, len(errs)): + area += (errs[i] + errs[i - 1]) / 2 + area /= len(errs) - 1 + return area + + +def disk(radius, alias_blur=0.1, dtype=np.float32): + if radius <= 8: + L = np.arange(-8, 8 + 1) + ksize = (3, 3) + else: + L = np.arange(-radius, radius + 1) + ksize = (5, 5) + X, Y = np.meshgrid(L, L) + aliased_disk = np.array((X ** 2 + Y ** 2) <= radius ** 2, dtype=dtype) + aliased_disk /= np.sum(aliased_disk) + + # supersample disk to antialias + return cv2.GaussianBlur(aliased_disk, ksize=ksize, sigmaX=alias_blur) + + +# Tell Python about the C method +wandlibrary.MagickMotionBlurImage.argtypes = (ctypes.c_void_p, # wand + ctypes.c_double, # radius + ctypes.c_double, # sigma + ctypes.c_double) # angle + + +# Extend wand.image.Image class to include method signature +class MotionImage(WandImage): + def motion_blur(self, radius=0.0, sigma=0.0, angle=0.0): + wandlibrary.MagickMotionBlurImage(self.wand, radius, sigma, angle) + + +# modification of https://github.com/FLHerne/mapgen/blob/master/diamondsquare.py +def plasma_fractal(mapsize=512, wibbledecay=3): + """ + Generate a heightmap using diamond-square algorithm. + Return square 2d array, side length 'mapsize', of floats in range 0-255. + 'mapsize' must be a power of two. + """ + assert (mapsize & (mapsize - 1) == 0) + maparray = np.empty((mapsize, mapsize), dtype=np.float_) + maparray[0, 0] = 0 + stepsize = mapsize + wibble = 100 + + def wibbledmean(array): + return array / 4 + wibble * np.random.uniform(-wibble, wibble, array.shape) + + def fillsquares(): + """For each square of points stepsize apart, + calculate middle value as mean of points + wibble""" + cornerref = maparray[0:mapsize:stepsize, 0:mapsize:stepsize] + squareaccum = cornerref + np.roll(cornerref, shift=-1, axis=0) + squareaccum += np.roll(squareaccum, shift=-1, axis=1) + maparray[stepsize // 2:mapsize:stepsize, + stepsize // 2:mapsize:stepsize] = wibbledmean(squareaccum) + + def filldiamonds(): + """For each diamond of points stepsize apart, + calculate middle value as mean of points + wibble""" + mapsize = maparray.shape[0] + drgrid = maparray[stepsize // 2:mapsize:stepsize, stepsize // 2:mapsize:stepsize] + ulgrid = maparray[0:mapsize:stepsize, 0:mapsize:stepsize] + ldrsum = drgrid + np.roll(drgrid, 1, axis=0) + lulsum = ulgrid + np.roll(ulgrid, -1, axis=1) + ltsum = ldrsum + lulsum + maparray[0:mapsize:stepsize, stepsize // 2:mapsize:stepsize] = wibbledmean(ltsum) + tdrsum = drgrid + np.roll(drgrid, 1, axis=1) + tulsum = ulgrid + np.roll(ulgrid, -1, axis=0) + ttsum = tdrsum + tulsum + maparray[stepsize // 2:mapsize:stepsize, 0:mapsize:stepsize] = wibbledmean(ttsum) + + while stepsize >= 2: + fillsquares() + filldiamonds() + stepsize //= 2 + wibble /= wibbledecay + + maparray -= maparray.min() + return maparray / maparray.max() + + +def clipped_zoom(img, zoom_factor): + h = img.shape[0] + # ceil crop height(= crop width) + ch = int(np.ceil(h / zoom_factor)) + + top = (h - ch) // 2 + img = scizoom(img[top:top + ch, top:top + ch], (zoom_factor, zoom_factor, 1), order=1) + # trim off any extra pixels + trim_top = (img.shape[0] - h) // 2 + + return img[trim_top:trim_top + h, trim_top:trim_top + h] + + +# /////////////// End Distortion Helpers /////////////// + + +# /////////////// Distortions /////////////// + +def gaussian_noise(x, severity=1): + c = [.09, .13, 0.19, 0.27, 0.40][severity - 1] + + x = np.array(x) / 255. + return np.clip(x + np.random.normal(size=x.shape, scale=c), 0, 1) * 255 + + +def shot_noise(x, severity=1): + c = [50, 20, 10, 5, 3][severity - 1] + + x = np.array(x) / 255. + return np.clip(np.random.poisson(x * c) / c, 0, 1) * 255 + + +def impulse_noise(x, severity=1): + c = [.04, .07, .10, 0.18, 0.28][severity - 1] + + x = sk.util.random_noise(np.array(x) / 255., mode='s&p', amount=c) + return np.clip(x, 0, 1) * 255 + + +def speckle_noise(x, severity=1): + c = [.16, .21, 0.36, 0.46, 0.61][severity - 1] + + x = np.array(x) / 255. + return np.clip(x + x * np.random.normal(size=x.shape, scale=c), 0, 1) * 255 + + +def gaussian_blur(x, severity=1): + c = [1.5, 2.5, 3.5, 4.5, 6.5][severity - 1] + + x = gaussian(np.array(x) / 255., sigma=c, multichannel=True) + return np.clip(x, 0, 1) * 255 + + +def glass_blur(x, severity=1): + # sigma, max_delta, iterations + c = [(0.8,1,3), (1,3,1), (1.1,2,4), (1.2,4,2), (1.6,5,2)][severity - 1] + + x = np.uint8(gaussian(np.array(x) / 255., sigma=c[0], multichannel=True) * 255) + + # locally shuffle pixels + for i in range(c[2]): + for h in range(299 - c[1], c[1], -1): + for w in range(299 - c[1], c[1], -1): + dx, dy = np.random.randint(-c[1], c[1], size=(2,)) + h_prime, w_prime = h + dy, w + dx + # swap + x[h, w], x[h_prime, w_prime] = x[h_prime, w_prime], x[h, w] + + return np.clip(gaussian(x / 255., sigma=c[0], multichannel=True), 0, 1) * 255 + + +def defocus_blur(x, severity=1): + c = [(4, 0.1), (5,0.5), (7,0.5), (10,0.5), (12,0.5)][severity - 1] + + x = np.array(x) / 255. + kernel = disk(radius=c[0], alias_blur=c[1]) + + channels = [] + for d in range(3): + channels.append(cv2.filter2D(x[:, :, d], -1, kernel)) + channels = np.array(channels).transpose((1, 2, 0)) # 3x299x299 -> 299x299x3 + + return np.clip(channels, 0, 1) * 255 + + +def motion_blur(x, severity=1): + c = [(12,4), (17,6), (17, 9), (17,13), (22,16)][severity - 1] + + output = BytesIO() + x.save(output, format='PNG') + x = MotionImage(blob=output.getvalue()) + + x.motion_blur(radius=c[0], sigma=c[1], angle=np.random.uniform(-45, 45)) + + x = cv2.imdecode(np.fromstring(x.make_blob(), np.uint8), + cv2.IMREAD_UNCHANGED) + + if x.shape != (299, 299): + return np.clip(x[..., [2, 1, 0]], 0, 255) # BGR to RGB + else: # greyscale to RGB + return np.clip(np.array([x, x, x]).transpose((1, 2, 0)), 0, 255) + + +def zoom_blur(x, severity=1): + c = [np.arange(1, 1.11, 0.01), + np.arange(1, 1.16, 0.01), + np.arange(1, 1.21, 0.02), + np.arange(1, 1.26, 0.02), + np.arange(1, 1.31, 0.03)][severity - 1] + + x = (np.array(x) / 255.).astype(np.float32) + out = np.zeros_like(x) + for zoom_factor in c: + out += clipped_zoom(x, zoom_factor) + + x = (x + out) / (len(c) + 1) + return np.clip(x, 0, 1) * 255 + + +def fog(x, severity=1): + c = [(1.5, 2), (2, 2), (2.5, 1.8), (2.5, 1.6), (3, 1.5)][severity - 1] + + x = np.array(x) / 255. + max_val = x.max() + x += c[0] * plasma_fractal(wibbledecay=c[1])[:299, :299][..., np.newaxis] + return np.clip(x * max_val / (max_val + c[0]), 0, 1) * 255 + + +def frost(x, severity=1): + c = [(1, 0.4), (0.8, 0.6), (0.7, 0.7), (0.65, 0.7), (0.6, 0.75)][severity - 1] + idx = np.random.randint(5) + filename = ['./frost1.png', './frost2.png', './frost3.png', './frost4.jpg', './frost5.jpg', './frost6.jpg'][idx] + frost = cv2.imread(filename) + # randomly crop and convert to rgb + x_start, y_start = np.random.randint(0, frost.shape[0] - 299), np.random.randint(0, frost.shape[1] - 299) + frost = frost[x_start:x_start + 299, y_start:y_start + 299][..., [2, 1, 0]] + + return np.clip(c[0] * np.array(x) + c[1] * frost, 0, 255) + + +def snow(x, severity=1): + c = [(0.1, 0.3, 3, 0.5, 10, 4, 0.8), + (0.2, 0.3, 2, 0.5, 12, 4, 0.7), + (0.55, 0.3, 4, 0.9, 12, 8, 0.7), + (0.55, 0.3, 4.5, 0.85, 12, 8, 0.65), + (0.55, 0.3, 2.5, 0.85, 12, 12, 0.55)][severity - 1] + + x = np.array(x, dtype=np.float32) / 255. + snow_layer = np.random.normal(size=x.shape[:2], loc=c[0], scale=c[1]) # [:2] for monochrome + + snow_layer = clipped_zoom(snow_layer[..., np.newaxis], c[2]) + snow_layer[snow_layer < c[3]] = 0 + + snow_layer = PILImage.fromarray((np.clip(snow_layer.squeeze(), 0, 1) * 255).astype(np.uint8), mode='L') + output = BytesIO() + snow_layer.save(output, format='PNG') + snow_layer = MotionImage(blob=output.getvalue()) + + snow_layer.motion_blur(radius=c[4], sigma=c[5], angle=np.random.uniform(-135, -45)) + + snow_layer = cv2.imdecode(np.fromstring(snow_layer.make_blob(), np.uint8), + cv2.IMREAD_UNCHANGED) / 255. + snow_layer = snow_layer[..., np.newaxis] + + x = c[6] * x + (1 - c[6]) * np.maximum(x, cv2.cvtColor(x, cv2.COLOR_RGB2GRAY).reshape(299, 299, 1) * 1.5 + 0.5) + return np.clip(x + snow_layer + np.rot90(snow_layer, k=2), 0, 1) * 255 + + +def spatter(x, severity=1): + c = [(0.65,0.3,4,0.69,0.9,0), + (0.65,0.3,3.5,0.68,0.9,0), + (0.65,0.3,3,0.68,0.8,0), + (0.65,0.3,1.2,0.65,1.8,1), + (0.67,0.4,1.2,0.65,1.8,1)][severity - 1] + x = np.array(x, dtype=np.float32) / 255. + + liquid_layer = np.random.normal(size=x.shape[:2], loc=c[0], scale=c[1]) + + liquid_layer = gaussian(liquid_layer, sigma=c[2]) + liquid_layer[liquid_layer < c[3]] = 0 + if c[5] == 0: + liquid_layer = (liquid_layer * 255).astype(np.uint8) + dist = 255 - cv2.Canny(liquid_layer, 50, 150) + dist = cv2.distanceTransform(dist, cv2.DIST_L2, 5) + _, dist = cv2.threshold(dist, 20, 20, cv2.THRESH_TRUNC) + dist = cv2.blur(dist, (3, 3)).astype(np.uint8) + dist = cv2.equalizeHist(dist) + # ker = np.array([[-1,-2,-3],[-2,0,0],[-3,0,1]], dtype=np.float32) + # ker -= np.mean(ker) + ker = np.array([[-2, -1, 0], [-1, 1, 1], [0, 1, 2]]) + dist = cv2.filter2D(dist, cv2.CV_8U, ker) + dist = cv2.blur(dist, (3, 3)).astype(np.float32) + + m = cv2.cvtColor(liquid_layer * dist, cv2.COLOR_GRAY2BGRA) + m /= np.max(m, axis=(0, 1)) + m *= c[4] + + # water is pale turqouise + color = np.concatenate((175 / 255. * np.ones_like(m[..., :1]), + 238 / 255. * np.ones_like(m[..., :1]), + 238 / 255. * np.ones_like(m[..., :1])), axis=2) + + color = cv2.cvtColor(color, cv2.COLOR_BGR2BGRA) + x = cv2.cvtColor(x, cv2.COLOR_BGR2BGRA) + + return cv2.cvtColor(np.clip(x + m * color, 0, 1), cv2.COLOR_BGRA2BGR) * 255 + else: + m = np.where(liquid_layer > c[3], 1, 0) + m = gaussian(m.astype(np.float32), sigma=c[4]) + m[m < 0.8] = 0 + # m = np.abs(m) ** (1/c[4]) + + # mud brown + color = np.concatenate((63 / 255. * np.ones_like(x[..., :1]), + 42 / 255. * np.ones_like(x[..., :1]), + 20 / 255. * np.ones_like(x[..., :1])), axis=2) + + color *= m[..., np.newaxis] + x *= (1 - m[..., np.newaxis]) + + return np.clip(x + color, 0, 1) * 255 + + +def contrast(x, severity=1): + c = [0.4, .3, .2, .1, .05][severity - 1] + + x = np.array(x) / 255. + means = np.mean(x, axis=(0, 1), keepdims=True) + return np.clip((x - means) * c + means, 0, 1) * 255 + + +def brightness(x, severity=1): + c = [.1, .2, .3, .4, .5][severity - 1] + + x = np.array(x) / 255. + x = sk.color.rgb2hsv(x) + x[:, :, 2] = np.clip(x[:, :, 2] + c, 0, 1) + x = sk.color.hsv2rgb(x) + + return np.clip(x, 0, 1) * 255 + + +def saturate(x, severity=1): + c = [(0.3, 0), (0.1, 0), (2, 0), (5, 0.1), (20, 0.2)][severity - 1] + + x = np.array(x) / 255. + x = sk.color.rgb2hsv(x) + x[:, :, 1] = np.clip(x[:, :, 1] * c[0] + c[1], 0, 1) + x = sk.color.hsv2rgb(x) + + return np.clip(x, 0, 1) * 255 + + +def jpeg_compression(x, severity=1): + c = [23, 16, 13, 8, 5][severity - 1] + + output = BytesIO() + x.save(output, 'JPEG', quality=c) + x = PILImage.open(output) + + return x + + +def pixelate(x, severity=1): + c = [0.5, 0.4, 0.3, 0.25, 0.2][severity - 1] + + x = x.resize((int(299 * c), int(299 * c)), PILImage.BOX) + x = x.resize((299, 299), PILImage.BOX) + + return x + + +# mod of https://gist.github.com/erniejunior/601cdf56d2b424757de5 +def elastic_transform(image, severity=1): + c = [(360 * 2, 360 * 0.7, 360 * 0.1), + (360 * 2, 360 * 0.08, 360 * 0.2), + (360 * 0.05, 360 * 0.01, 360 * 0.02), + (360 * 0.07, 360 * 0.01, 360 * 0.02), + (360 * 0.12, 360 * 0.01, 360 * 0.02)][severity - 1] + + image = np.array(image, dtype=np.float32) / 255. + shape = image.shape + shape_size = shape[:2] + + # random affine + center_square = np.float32(shape_size) // 2 + square_size = min(shape_size) // 3 + pts1 = np.float32([center_square + square_size, + [center_square[0] + square_size, center_square[1] - square_size], + center_square - square_size]) + pts2 = pts1 + np.random.uniform(-c[2], c[2], size=pts1.shape).astype(np.float32) + M = cv2.getAffineTransform(pts1, pts2) + image = cv2.warpAffine(image, M, shape_size[::-1], borderMode=cv2.BORDER_REFLECT_101) + + dx = (gaussian(np.random.uniform(-1, 1, size=shape[:2]), + c[1], mode='reflect', truncate=3) * c[0]).astype(np.float32) + dy = (gaussian(np.random.uniform(-1, 1, size=shape[:2]), + c[1], mode='reflect', truncate=3) * c[0]).astype(np.float32) + dx, dy = dx[..., np.newaxis], dy[..., np.newaxis] + + x, y, z = np.meshgrid(np.arange(shape[1]), np.arange(shape[0]), np.arange(shape[2])) + indices = np.reshape(y + dy, (-1, 1)), np.reshape(x + dx, (-1, 1)), np.reshape(z, (-1, 1)) + return np.clip(map_coordinates(image, indices, order=1, mode='reflect').reshape(shape), 0, 1) * 255 + + +# /////////////// End Distortions /////////////// + + +# /////////////// Further Setup /////////////// + + +def save_distorted(method=gaussian_noise): + for severity in range(1, 6): + print(method.__name__, severity) + distorted_dataset = DistortImageFolder( + root="/share/data/vision-greg/ImageNet/clsloc/images/val", + method=method, severity=severity, + transform=resized_center_crop) + distorted_dataset_loader = torch.utils.data.DataLoader( + distorted_dataset, batch_size=100, shuffle=False, num_workers=4) + + for _ in distorted_dataset_loader: continue + + +# /////////////// End Further Setup /////////////// + + +# /////////////// Display Results /////////////// +import collections + +print('\nUsing ImageNet data') + +d = collections.OrderedDict() +# d['Gaussian Noise'] = gaussian_noise +# d['Shot Noise'] = shot_noise +# d['Impulse Noise'] = impulse_noise +# d['Defocus Blur'] = defocus_blur +# d['Glass Blur'] = glass_blur +# d['Motion Blur'] = motion_blur +# d['Zoom Blur'] = zoom_blur +# d['Snow'] = snow +# d['Frost'] = frost +# d['Fog'] = fog +# d['Brightness'] = brightness +# d['Contrast'] = contrast +# d['Elastic'] = elastic_transform +# d['Pixelate'] = pixelate +# d['JPEG'] = jpeg_compression +# +d['Speckle Noise'] = speckle_noise +d['Gaussian Blur'] = gaussian_blur +d['Spatter'] = spatter +d['Saturate'] = saturate + +for method_name in d.keys(): + save_distorted(d[method_name]) diff --git a/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/create_c/make_tinyimagenet_c.py b/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/create_c/make_tinyimagenet_c.py new file mode 100644 index 0000000000000000000000000000000000000000..cbb5dc44be77cf235f2791031d2d113c45a9b11d --- /dev/null +++ b/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/create_c/make_tinyimagenet_c.py @@ -0,0 +1,584 @@ +# -*- coding: utf-8 -*- + +import os +from PIL import Image +import os.path +import time +import torch +import torchvision.datasets as dset +import torchvision.transforms as trn +import torch.utils.data as data +import numpy as np + +from PIL import Image + +# /////////////// Data Loader /////////////// + + +IMG_EXTENSIONS = ['.jpg', '.jpeg', '.png', '.ppm', '.bmp', '.pgm'] + + +def is_image_file(filename): + """Checks if a file is an image. + Args: + filename (string): path to a file + Returns: + bool: True if the filename ends with a known image extension + """ + filename_lower = filename.lower() + return any(filename_lower.endswith(ext) for ext in IMG_EXTENSIONS) + + +def find_classes(dir): + classes = [d for d in os.listdir(dir) if os.path.isdir(os.path.join(dir, d))] + classes.sort() + class_to_idx = {classes[i]: i for i in range(len(classes))} + return classes, class_to_idx + + +def make_dataset(dir, class_to_idx): + images = [] + dir = os.path.expanduser(dir) + for target in sorted(os.listdir(dir)): + d = os.path.join(dir, target) + if not os.path.isdir(d): + continue + + for root, _, fnames in sorted(os.walk(d)): + for fname in sorted(fnames): + if is_image_file(fname): + path = os.path.join(root, fname) + item = (path, class_to_idx[target]) + images.append(item) + + return images + + +def pil_loader(path): + # open path as file to avoid ResourceWarning (https://github.com/python-pillow/Pillow/issues/835) + with open(path, 'rb') as f: + img = Image.open(f) + return img.convert('RGB') + + +def accimage_loader(path): + import accimage + try: + return accimage.Image(path) + except IOError: + # Potentially a decoding problem, fall back to PIL.Image + return pil_loader(path) + + +def default_loader(path): + from torchvision import get_image_backend + if get_image_backend() == 'accimage': + return accimage_loader(path) + else: + return pil_loader(path) + + +class DistortImageFolder(data.Dataset): + def __init__(self, root, method, severity, transform=None, target_transform=None, + loader=default_loader): + classes, class_to_idx = find_classes(root) + imgs = make_dataset(root, class_to_idx) + if len(imgs) == 0: + raise (RuntimeError("Found 0 images in subfolders of: " + root + "\n" + + "Supported image extensions are: " + ",".join(IMG_EXTENSIONS))) + + self.root = root + self.method = method + self.severity = severity + self.imgs = imgs + self.classes = classes + self.class_to_idx = class_to_idx + self.idx_to_class = {v: k for k, v in class_to_idx.items()} + self.transform = transform + self.target_transform = target_transform + self.loader = loader + + def __getitem__(self, index): + path, target = self.imgs[index] + img = self.loader(path) + if self.transform is not None: + img = self.transform(img) + img = self.method(img, self.severity) + if self.target_transform is not None: + target = self.target_transform(target) + + save_path = '/share/data/lang/users/dan/Tiny-ImageNet-C/' + self.method.__name__ + \ + '/' + str(self.severity) + '/' + self.idx_to_class[target] + + if not os.path.exists(save_path): + os.makedirs(save_path) + + save_path += path[path.rindex('/'):] + + Image.fromarray(np.uint8(img)).save(save_path, quality=85, optimize=True) + + return 0 # we do not care about returning the data + + def __len__(self): + return len(self.imgs) + + +# /////////////// Distortion Helpers /////////////// + +import skimage as sk +from skimage.filters import gaussian +from io import BytesIO +from wand.image import Image as WandImage +from wand.api import library as wandlibrary +import wand.color as WandColor +import ctypes +from PIL import Image as PILImage +import cv2 +from scipy.ndimage import zoom as scizoom +from scipy.ndimage.interpolation import map_coordinates +import warnings + +warnings.simplefilter("ignore", UserWarning) + + +def auc(errs): # area under the alteration error curve + area = 0 + for i in range(1, len(errs)): + area += (errs[i] + errs[i - 1]) / 2 + area /= len(errs) - 1 + return area + + +def disk(radius, alias_blur=0.1, dtype=np.float32): + if radius <= 8: + L = np.arange(-8, 8 + 1) + ksize = (3, 3) + else: + L = np.arange(-radius, radius + 1) + ksize = (5, 5) + X, Y = np.meshgrid(L, L) + aliased_disk = np.array((X ** 2 + Y ** 2) <= radius ** 2, dtype=dtype) + aliased_disk /= np.sum(aliased_disk) + + # supersample disk to antialias + return cv2.GaussianBlur(aliased_disk, ksize=ksize, sigmaX=alias_blur) + + +# Tell Python about the C method +wandlibrary.MagickMotionBlurImage.argtypes = (ctypes.c_void_p, # wand + ctypes.c_double, # radius + ctypes.c_double, # sigma + ctypes.c_double) # angle + + +# Extend wand.image.Image class to include method signature +class MotionImage(WandImage): + def motion_blur(self, radius=0.0, sigma=0.0, angle=0.0): + wandlibrary.MagickMotionBlurImage(self.wand, radius, sigma, angle) + + +# modification of https://github.com/FLHerne/mapgen/blob/master/diamondsquare.py +def plasma_fractal(mapsize=64, wibbledecay=3): + """ + Generate a heightmap using diamond-square algorithm. + Return square 2d array, side length 'mapsize', of floats in range 0-255. + 'mapsize' must be a power of two. + """ + assert (mapsize & (mapsize - 1) == 0) + maparray = np.empty((mapsize, mapsize), dtype=np.float_) + maparray[0, 0] = 0 + stepsize = mapsize + wibble = 100 + + def wibbledmean(array): + return array / 4 + wibble * np.random.uniform(-wibble, wibble, array.shape) + + def fillsquares(): + """For each square of points stepsize apart, + calculate middle value as mean of points + wibble""" + cornerref = maparray[0:mapsize:stepsize, 0:mapsize:stepsize] + squareaccum = cornerref + np.roll(cornerref, shift=-1, axis=0) + squareaccum += np.roll(squareaccum, shift=-1, axis=1) + maparray[stepsize // 2:mapsize:stepsize, + stepsize // 2:mapsize:stepsize] = wibbledmean(squareaccum) + + def filldiamonds(): + """For each diamond of points stepsize apart, + calculate middle value as mean of points + wibble""" + mapsize = maparray.shape[0] + drgrid = maparray[stepsize // 2:mapsize:stepsize, stepsize // 2:mapsize:stepsize] + ulgrid = maparray[0:mapsize:stepsize, 0:mapsize:stepsize] + ldrsum = drgrid + np.roll(drgrid, 1, axis=0) + lulsum = ulgrid + np.roll(ulgrid, -1, axis=1) + ltsum = ldrsum + lulsum + maparray[0:mapsize:stepsize, stepsize // 2:mapsize:stepsize] = wibbledmean(ltsum) + tdrsum = drgrid + np.roll(drgrid, 1, axis=1) + tulsum = ulgrid + np.roll(ulgrid, -1, axis=0) + ttsum = tdrsum + tulsum + maparray[stepsize // 2:mapsize:stepsize, 0:mapsize:stepsize] = wibbledmean(ttsum) + + while stepsize >= 2: + fillsquares() + filldiamonds() + stepsize //= 2 + wibble /= wibbledecay + + maparray -= maparray.min() + return maparray / maparray.max() + + +def clipped_zoom(img, zoom_factor): + h = img.shape[0] + # ceil crop height(= crop width) + ch = int(np.ceil(h / zoom_factor)) + + top = (h - ch) // 2 + img = scizoom(img[top:top + ch, top:top + ch], (zoom_factor, zoom_factor, 1), order=1) + # trim off any extra pixels + trim_top = (img.shape[0] - h) // 2 + + return img[trim_top:trim_top + h, trim_top:trim_top + h] + + +# /////////////// End Distortion Helpers /////////////// + + +# /////////////// Distortions /////////////// + +def gaussian_noise(x, severity=1): + c = [0.04, 0.08, .12, .15, .18][severity - 1] + + x = np.array(x) / 255. + return np.clip(x + np.random.normal(size=x.shape, scale=c), 0, 1) * 255 + + +def shot_noise(x, severity=1): + c = [250, 100, 50, 30, 15][severity - 1] + + x = np.array(x) / 255. + return np.clip(np.random.poisson(x * c) / c, 0, 1) * 255 + + +def impulse_noise(x, severity=1): + c = [.01, .02, .05, .08, .14][severity - 1] + + x = sk.util.random_noise(np.array(x) / 255., mode='s&p', amount=c) + return np.clip(x, 0, 1) * 255 + + +def speckle_noise(x, severity=1): + c = [.15, .2, 0.25, 0.3, 0.35][severity - 1] + + x = np.array(x) / 255. + return np.clip(x + x * np.random.normal(size=x.shape, scale=c), 0, 1) * 255 + + +def gaussian_blur(x, severity=1): + c = [.5, .75, 1, 1.25, 1.5][severity - 1] + + x = gaussian(np.array(x) / 255., sigma=c, multichannel=True) + return np.clip(x, 0, 1) * 255 + + +def glass_blur(x, severity=1): + # sigma, max_delta, iterations + c = [(0.1,1,1), (0.5,1,1), (0.6,1,2), (0.7,2,1), (0.9,2,2)][severity - 1] + + x = np.uint8(gaussian(np.array(x) / 255., sigma=c[0], multichannel=True) * 255) + + # locally shuffle pixels + for i in range(c[2]): + for h in range(64 - c[1], c[1], -1): + for w in range(64 - c[1], c[1], -1): + dx, dy = np.random.randint(-c[1], c[1], size=(2,)) + h_prime, w_prime = h + dy, w + dx + # swap + x[h, w], x[h_prime, w_prime] = x[h_prime, w_prime], x[h, w] + + return np.clip(gaussian(x / 255., sigma=c[0], multichannel=True), 0, 1) * 255 + + +def defocus_blur(x, severity=1): + c = [(0.5, 0.6), (1, 0.1), (1.5, 0.1), (2.5, 0.01), (3, 0.1)][severity - 1] + + x = np.array(x) / 255. + kernel = disk(radius=c[0], alias_blur=c[1]) + + channels = [] + for d in range(3): + channels.append(cv2.filter2D(x[:, :, d], -1, kernel)) + channels = np.array(channels).transpose((1, 2, 0)) # 3x64x64 -> 64x64x3 + + return np.clip(channels, 0, 1) * 255 + + +def motion_blur(x, severity=1): + c = [(10,1), (10,1.5), (10,2), (10,2.5), (12,3)][severity - 1] + + output = BytesIO() + x.save(output, format='PNG') + x = MotionImage(blob=output.getvalue()) + + x.motion_blur(radius=c[0], sigma=c[1], angle=np.random.uniform(-45, 45)) + + x = cv2.imdecode(np.fromstring(x.make_blob(), np.uint8), + cv2.IMREAD_UNCHANGED) + + if x.shape != (64, 64): + return np.clip(x[..., [2, 1, 0]], 0, 255) # BGR to RGB + else: # greyscale to RGB + return np.clip(np.array([x, x, x]).transpose((1, 2, 0)), 0, 255) + + +def zoom_blur(x, severity=1): + c = [np.arange(1, 1.06, 0.01), np.arange(1, 1.11, 0.01), np.arange(1, 1.16, 0.01), + np.arange(1, 1.21, 0.01), np.arange(1, 1.26, 0.01)][severity - 1] + + x = (np.array(x) / 255.).astype(np.float32) + out = np.zeros_like(x) + for zoom_factor in c: + out += clipped_zoom(x, zoom_factor) + + x = (x + out) / (len(c) + 1) + return np.clip(x, 0, 1) * 255 + + +def fog(x, severity=1): + c = [(.4,3), (.7,3), (1,2.5), (1.5,2), (2,1.75)][severity - 1] + + x = np.array(x) / 255. + max_val = x.max() + x += c[0] * plasma_fractal(wibbledecay=c[1])[:64, :64][..., np.newaxis] + return np.clip(x * max_val / (max_val + c[0]), 0, 1) * 255 + + +def frost(x, severity=1): + c = [(1, 0.3), (0.9, 0.4), (0.8, 0.45), (0.75, 0.5), (0.7, 0.55)][severity - 1] + idx = np.random.randint(5) + filename = ['./frost1.png', './frost2.png', './frost3.png', './frost4.jpg', './frost5.jpg', './frost6.jpg'][idx] + frost = cv2.imread(filename) + frost = cv2.resize(frost, (0, 0), fx=0.3, fy=0.3) + # randomly crop and convert to rgb + x_start, y_start = np.random.randint(0, frost.shape[0] - 64), np.random.randint(0, frost.shape[1] - 64) + frost = frost[x_start:x_start + 64, y_start:y_start + 64][..., [2, 1, 0]] + + return np.clip(c[0] * np.array(x) + c[1] * frost, 0, 255) + + +def snow(x, severity=1): + c = [(0.1,0.2,1,0.6,8,3,0.8), + (0.1,0.2,1,0.5,10,4,0.8), + (0.15,0.3,1.75,0.55,10,4,0.7), + (0.25,0.3,2.25,0.6,12,6,0.65), + (0.3,0.3,1.25,0.65,14,12,0.6)][severity - 1] + + x = np.array(x, dtype=np.float32) / 255. + snow_layer = np.random.normal(size=x.shape[:2], loc=c[0], scale=c[1]) # [:2] for monochrome + + snow_layer = clipped_zoom(snow_layer[..., np.newaxis], c[2]) + snow_layer[snow_layer < c[3]] = 0 + + snow_layer = PILImage.fromarray((np.clip(snow_layer.squeeze(), 0, 1) * 255).astype(np.uint8), mode='L') + output = BytesIO() + snow_layer.save(output, format='PNG') + snow_layer = MotionImage(blob=output.getvalue()) + + snow_layer.motion_blur(radius=c[4], sigma=c[5], angle=np.random.uniform(-135, -45)) + + snow_layer = cv2.imdecode(np.fromstring(snow_layer.make_blob(), np.uint8), + cv2.IMREAD_UNCHANGED) / 255. + snow_layer = snow_layer[..., np.newaxis] + + x = c[6] * x + (1 - c[6]) * np.maximum(x, cv2.cvtColor(x, cv2.COLOR_RGB2GRAY).reshape(64, 64, 1) * 1.5 + 0.5) + return np.clip(x + snow_layer + np.rot90(snow_layer, k=2), 0, 1) * 255 + + +def spatter(x, severity=1): + c = [(0.62,0.1,0.7,0.7,0.6,0), + (0.65,0.1,0.8,0.7,0.6,0), + (0.65,0.3,1,0.69,0.6,0), + (0.65,0.1,0.7,0.68,0.6,1), + (0.65,0.1,0.5,0.67,0.6,1)][severity - 1] + x = np.array(x, dtype=np.float32) / 255. + + liquid_layer = np.random.normal(size=x.shape[:2], loc=c[0], scale=c[1]) + + liquid_layer = gaussian(liquid_layer, sigma=c[2]) + liquid_layer[liquid_layer < c[3]] = 0 + if c[5] == 0: + liquid_layer = (liquid_layer * 255).astype(np.uint8) + dist = 255 - cv2.Canny(liquid_layer, 50, 150) + dist = cv2.distanceTransform(dist, cv2.DIST_L2, 5) + _, dist = cv2.threshold(dist, 20, 20, cv2.THRESH_TRUNC) + dist = cv2.blur(dist, (3, 3)).astype(np.uint8) + dist = cv2.equalizeHist(dist) + # ker = np.array([[-1,-2,-3],[-2,0,0],[-3,0,1]], dtype=np.float32) + # ker -= np.mean(ker) + ker = np.array([[-2, -1, 0], [-1, 1, 1], [0, 1, 2]]) + dist = cv2.filter2D(dist, cv2.CV_8U, ker) + dist = cv2.blur(dist, (3, 3)).astype(np.float32) + + m = cv2.cvtColor(liquid_layer * dist, cv2.COLOR_GRAY2BGRA) + m /= np.max(m, axis=(0, 1)) + m *= c[4] + + # water is pale turqouise + color = np.concatenate((175 / 255. * np.ones_like(m[..., :1]), + 238 / 255. * np.ones_like(m[..., :1]), + 238 / 255. * np.ones_like(m[..., :1])), axis=2) + + color = cv2.cvtColor(color, cv2.COLOR_BGR2BGRA) + x = cv2.cvtColor(x, cv2.COLOR_BGR2BGRA) + + return cv2.cvtColor(np.clip(x + m * color, 0, 1), cv2.COLOR_BGRA2BGR) * 255 + else: + m = np.where(liquid_layer > c[3], 1, 0) + m = gaussian(m.astype(np.float32), sigma=c[4]) + m[m < 0.8] = 0 + # m = np.abs(m) ** (1/c[4]) + + # mud brown + color = np.concatenate((63 / 255. * np.ones_like(x[..., :1]), + 42 / 255. * np.ones_like(x[..., :1]), + 20 / 255. * np.ones_like(x[..., :1])), axis=2) + + color *= m[..., np.newaxis] + x *= (1 - m[..., np.newaxis]) + + return np.clip(x + color, 0, 1) * 255 + + +def contrast(x, severity=1): + c = [.4, .3, .2, .1, 0.05][severity - 1] + + x = np.array(x) / 255. + means = np.mean(x, axis=(0, 1), keepdims=True) + return np.clip((x - means) * c + means, 0, 1) * 255 + + +def brightness(x, severity=1): + c = [.1, .2, .3, .4, .5][severity - 1] + + x = np.array(x) / 255. + x = sk.color.rgb2hsv(x) + x[:, :, 2] = np.clip(x[:, :, 2] + c, 0, 1) + x = sk.color.hsv2rgb(x) + + return np.clip(x, 0, 1) * 255 + + +def saturate(x, severity=1): + c = [(0.3, 0), (0.1, 0), (2, 0), (5, 0.1), (30, 0.2)][severity - 1] + + x = np.array(x) / 255. + x = sk.color.rgb2hsv(x) + x[:, :, 1] = np.clip(x[:, :, 1] * c[0] + c[1], 0, 1) + x = sk.color.hsv2rgb(x) + + return np.clip(x, 0, 1) * 255 + + +def jpeg_compression(x, severity=1): + c = [65, 58, 50, 40, 25][severity - 1] + + output = BytesIO() + x.save(output, 'JPEG', quality=c) + x = PILImage.open(output) + + return x + + +def pixelate(x, severity=1): + c = [0.9, 0.8, 0.7, 0.6, 0.5][severity - 1] + + x = x.resize((int(64 * c), int(64 * c)), PILImage.BOX) + x = x.resize((64, 64), PILImage.BOX) + + return x + + +# mod of https://gist.github.com/erniejunior/601cdf56d2b424757de5 +def elastic_transform(image, severity=1): + IMSIZE = 64 + c = [(IMSIZE*0, IMSIZE*0, IMSIZE*0.08), + (IMSIZE*0.05, IMSIZE*0.3, IMSIZE*0.06), + (IMSIZE*0.1, IMSIZE*0.08, IMSIZE*0.06), + (IMSIZE*0.1, IMSIZE*0.03, IMSIZE*0.03), + (IMSIZE*0.16, IMSIZE*0.03, IMSIZE*0.02)][severity - 1] + + image = np.array(image, dtype=np.float32) / 255. + shape = image.shape + shape_size = shape[:2] + + # random affine + center_square = np.float32(shape_size) // 2 + square_size = min(shape_size) // 3 + pts1 = np.float32([center_square + square_size, + [center_square[0] + square_size, center_square[1] - square_size], + center_square - square_size]) + pts2 = pts1 + np.random.uniform(-c[2], c[2], size=pts1.shape).astype(np.float32) + M = cv2.getAffineTransform(pts1, pts2) + image = cv2.warpAffine(image, M, shape_size[::-1], borderMode=cv2.BORDER_REFLECT_101) + + dx = (gaussian(np.random.uniform(-1, 1, size=shape[:2]), + c[1], mode='reflect', truncate=3) * c[0]).astype(np.float32) + dy = (gaussian(np.random.uniform(-1, 1, size=shape[:2]), + c[1], mode='reflect', truncate=3) * c[0]).astype(np.float32) + dx, dy = dx[..., np.newaxis], dy[..., np.newaxis] + + x, y, z = np.meshgrid(np.arange(shape[1]), np.arange(shape[0]), np.arange(shape[2])) + indices = np.reshape(y + dy, (-1, 1)), np.reshape(x + dx, (-1, 1)), np.reshape(z, (-1, 1)) + return np.clip(map_coordinates(image, indices, order=1, mode='reflect').reshape(shape), 0, 1) * 255 + + +# /////////////// End Distortions /////////////// + + +# /////////////// Further Setup /////////////// + + +def save_distorted(method=gaussian_noise): + for severity in range(1, 6): + print(method.__name__, severity) + distorted_dataset = DistortImageFolder( + root="./imagenet_val_bbox_crop/", + method=method, severity=severity, + transform=trn.Compose([trn.Resize((64, 64))])) + distorted_dataset_loader = torch.utils.data.DataLoader( + distorted_dataset, batch_size=100, shuffle=False, num_workers=6) + + for _ in distorted_dataset_loader: continue + + +# /////////////// End Further Setup /////////////// + + +# /////////////// Display Results /////////////// +import collections + +print('\nUsing ImageNet data') + +d = collections.OrderedDict() +# d['Gaussian Noise'] = gaussian_noise +# d['Shot Noise'] = shot_noise +# d['Impulse Noise'] = impulse_noise +# d['Defocus Blur'] = defocus_blur +# d['Glass Blur'] = glass_blur +# d['Motion Blur'] = motion_blur +# d['Zoom Blur'] = zoom_blur +# d['Snow'] = snow +d['Frost'] = frost +# d['Fog'] = fog +# d['Brightness'] = brightness +# d['Contrast'] = contrast +# d['Elastic'] = elastic_transform +# d['Pixelate'] = pixelate +# d['JPEG'] = jpeg_compression + +# d['Speckle Noise'] = speckle_noise +# d['Gaussian Blur'] = gaussian_blur +# d['Spatter'] = spatter +# d['Saturate'] = saturate + +for method_name in d.keys(): + save_distorted(d[method_name]) diff --git a/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/create_c/tiny.sh b/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/create_c/tiny.sh new file mode 100644 index 0000000000000000000000000000000000000000..bbba84bc41ce242c7bd5f0fa537439ea782c052d --- /dev/null +++ b/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/create_c/tiny.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +source ~/new_begin.sh + +python make_tinyimagenet_c.py + diff --git a/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/densenet_cosine_264_k48.py b/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/densenet_cosine_264_k48.py new file mode 100644 index 0000000000000000000000000000000000000000..8276999e3853f129c880f1e00ca37536c23530b5 --- /dev/null +++ b/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/densenet_cosine_264_k48.py @@ -0,0 +1,1485 @@ + +import torch +import torch.nn as nn +import torch.legacy.nn as lnn + +from functools import reduce +from torch.autograd import Variable + +class LambdaBase(nn.Sequential): + def __init__(self, fn, *args): + super(LambdaBase, self).__init__(*args) + self.lambda_func = fn + + def forward_prepare(self, input): + output = [] + for module in self._modules.values(): + output.append(module(input)) + return output if output else input + +class Lambda(LambdaBase): + def forward(self, input): + return self.lambda_func(self.forward_prepare(input)) + +class LambdaMap(LambdaBase): + def forward(self, input): + return list(map(self.lambda_func,self.forward_prepare(input))) + +class LambdaReduce(LambdaBase): + def forward(self, input): + return reduce(self.lambda_func,self.forward_prepare(input)) + + +densenet_cosine_264_k48 = nn.Sequential( # Sequential, + nn.Conv2d(3,96,(7, 7),(2, 2),(3, 3),1,1,bias=False), + nn.BatchNorm2d(96), + nn.ReLU(), + nn.MaxPool2d((3, 3),(2, 2),(1, 1)), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(96), + nn.ReLU(), + nn.Conv2d(96,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(144), + nn.ReLU(), + nn.Conv2d(144,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(240), + nn.ReLU(), + nn.Conv2d(240,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(288), + nn.ReLU(), + nn.Conv2d(288,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(336), + nn.ReLU(), + nn.Conv2d(336,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + nn.BatchNorm2d(384), + nn.ReLU(), + nn.Conv2d(384,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.AvgPool2d((2, 2),(2, 2)), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(240), + nn.ReLU(), + nn.Conv2d(240,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(288), + nn.ReLU(), + nn.Conv2d(288,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(336), + nn.ReLU(), + nn.Conv2d(336,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(384), + nn.ReLU(), + nn.Conv2d(384,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(432), + nn.ReLU(), + nn.Conv2d(432,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(480), + nn.ReLU(), + nn.Conv2d(480,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(528), + nn.ReLU(), + nn.Conv2d(528,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(576), + nn.ReLU(), + nn.Conv2d(576,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(624), + nn.ReLU(), + nn.Conv2d(624,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(672), + nn.ReLU(), + nn.Conv2d(672,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(720), + nn.ReLU(), + nn.Conv2d(720,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + nn.BatchNorm2d(768), + nn.ReLU(), + nn.Conv2d(768,384,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.AvgPool2d((2, 2),(2, 2)), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(384), + nn.ReLU(), + nn.Conv2d(384,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(432), + nn.ReLU(), + nn.Conv2d(432,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(480), + nn.ReLU(), + nn.Conv2d(480,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(528), + nn.ReLU(), + nn.Conv2d(528,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(576), + nn.ReLU(), + nn.Conv2d(576,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(624), + nn.ReLU(), + nn.Conv2d(624,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(672), + nn.ReLU(), + nn.Conv2d(672,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(720), + nn.ReLU(), + nn.Conv2d(720,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(768), + nn.ReLU(), + nn.Conv2d(768,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(816), + nn.ReLU(), + nn.Conv2d(816,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(864), + nn.ReLU(), + nn.Conv2d(864,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(912), + nn.ReLU(), + nn.Conv2d(912,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(960), + nn.ReLU(), + nn.Conv2d(960,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(1008), + nn.ReLU(), + nn.Conv2d(1008,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(1056), + nn.ReLU(), + nn.Conv2d(1056,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(1104), + nn.ReLU(), + nn.Conv2d(1104,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(1152), + nn.ReLU(), + nn.Conv2d(1152,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(1200), + nn.ReLU(), + nn.Conv2d(1200,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(1248), + nn.ReLU(), + nn.Conv2d(1248,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(1296), + nn.ReLU(), + nn.Conv2d(1296,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(1344), + nn.ReLU(), + nn.Conv2d(1344,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(1392), + nn.ReLU(), + nn.Conv2d(1392,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(1440), + nn.ReLU(), + nn.Conv2d(1440,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(1488), + nn.ReLU(), + nn.Conv2d(1488,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(1536), + nn.ReLU(), + nn.Conv2d(1536,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(1584), + nn.ReLU(), + nn.Conv2d(1584,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(1632), + nn.ReLU(), + nn.Conv2d(1632,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(1680), + nn.ReLU(), + nn.Conv2d(1680,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(1728), + nn.ReLU(), + nn.Conv2d(1728,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(1776), + nn.ReLU(), + nn.Conv2d(1776,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(1824), + nn.ReLU(), + nn.Conv2d(1824,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(1872), + nn.ReLU(), + nn.Conv2d(1872,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(1920), + nn.ReLU(), + nn.Conv2d(1920,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(1968), + nn.ReLU(), + nn.Conv2d(1968,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(2016), + nn.ReLU(), + nn.Conv2d(2016,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(2064), + nn.ReLU(), + nn.Conv2d(2064,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(2112), + nn.ReLU(), + nn.Conv2d(2112,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(2160), + nn.ReLU(), + nn.Conv2d(2160,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(2208), + nn.ReLU(), + nn.Conv2d(2208,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(2256), + nn.ReLU(), + nn.Conv2d(2256,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(2304), + nn.ReLU(), + nn.Conv2d(2304,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(2352), + nn.ReLU(), + nn.Conv2d(2352,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(2400), + nn.ReLU(), + nn.Conv2d(2400,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(2448), + nn.ReLU(), + nn.Conv2d(2448,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(2496), + nn.ReLU(), + nn.Conv2d(2496,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(2544), + nn.ReLU(), + nn.Conv2d(2544,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(2592), + nn.ReLU(), + nn.Conv2d(2592,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(2640), + nn.ReLU(), + nn.Conv2d(2640,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(2688), + nn.ReLU(), + nn.Conv2d(2688,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(2736), + nn.ReLU(), + nn.Conv2d(2736,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(2784), + nn.ReLU(), + nn.Conv2d(2784,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(2832), + nn.ReLU(), + nn.Conv2d(2832,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(2880), + nn.ReLU(), + nn.Conv2d(2880,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(2928), + nn.ReLU(), + nn.Conv2d(2928,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(2976), + nn.ReLU(), + nn.Conv2d(2976,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(3024), + nn.ReLU(), + nn.Conv2d(3024,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(3072), + nn.ReLU(), + nn.Conv2d(3072,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(3120), + nn.ReLU(), + nn.Conv2d(3120,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(3168), + nn.ReLU(), + nn.Conv2d(3168,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(3216), + nn.ReLU(), + nn.Conv2d(3216,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(3264), + nn.ReLU(), + nn.Conv2d(3264,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(3312), + nn.ReLU(), + nn.Conv2d(3312,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(3360), + nn.ReLU(), + nn.Conv2d(3360,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(3408), + nn.ReLU(), + nn.Conv2d(3408,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + nn.BatchNorm2d(3456), + nn.ReLU(), + nn.Conv2d(3456,1728,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.AvgPool2d((2, 2),(2, 2)), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(1728), + nn.ReLU(), + nn.Conv2d(1728,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(1776), + nn.ReLU(), + nn.Conv2d(1776,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(1824), + nn.ReLU(), + nn.Conv2d(1824,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(1872), + nn.ReLU(), + nn.Conv2d(1872,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(1920), + nn.ReLU(), + nn.Conv2d(1920,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(1968), + nn.ReLU(), + nn.Conv2d(1968,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(2016), + nn.ReLU(), + nn.Conv2d(2016,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(2064), + nn.ReLU(), + nn.Conv2d(2064,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(2112), + nn.ReLU(), + nn.Conv2d(2112,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(2160), + nn.ReLU(), + nn.Conv2d(2160,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(2208), + nn.ReLU(), + nn.Conv2d(2208,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(2256), + nn.ReLU(), + nn.Conv2d(2256,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(2304), + nn.ReLU(), + nn.Conv2d(2304,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(2352), + nn.ReLU(), + nn.Conv2d(2352,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(2400), + nn.ReLU(), + nn.Conv2d(2400,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(2448), + nn.ReLU(), + nn.Conv2d(2448,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(2496), + nn.ReLU(), + nn.Conv2d(2496,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(2544), + nn.ReLU(), + nn.Conv2d(2544,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(2592), + nn.ReLU(), + nn.Conv2d(2592,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(2640), + nn.ReLU(), + nn.Conv2d(2640,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(2688), + nn.ReLU(), + nn.Conv2d(2688,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(2736), + nn.ReLU(), + nn.Conv2d(2736,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(2784), + nn.ReLU(), + nn.Conv2d(2784,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(2832), + nn.ReLU(), + nn.Conv2d(2832,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(2880), + nn.ReLU(), + nn.Conv2d(2880,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(2928), + nn.ReLU(), + nn.Conv2d(2928,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(2976), + nn.ReLU(), + nn.Conv2d(2976,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(3024), + nn.ReLU(), + nn.Conv2d(3024,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(3072), + nn.ReLU(), + nn.Conv2d(3072,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(3120), + nn.ReLU(), + nn.Conv2d(3120,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(3168), + nn.ReLU(), + nn.Conv2d(3168,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(3216), + nn.ReLU(), + nn.Conv2d(3216,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(3264), + nn.ReLU(), + nn.Conv2d(3264,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(3312), + nn.ReLU(), + nn.Conv2d(3312,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(3360), + nn.ReLU(), + nn.Conv2d(3360,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(3408), + nn.ReLU(), + nn.Conv2d(3408,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(3456), + nn.ReLU(), + nn.Conv2d(3456,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(3504), + nn.ReLU(), + nn.Conv2d(3504,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(3552), + nn.ReLU(), + nn.Conv2d(3552,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(3600), + nn.ReLU(), + nn.Conv2d(3600,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(3648), + nn.ReLU(), + nn.Conv2d(3648,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(3696), + nn.ReLU(), + nn.Conv2d(3696,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(3744), + nn.ReLU(), + nn.Conv2d(3744,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(3792), + nn.ReLU(), + nn.Conv2d(3792,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(3840), + nn.ReLU(), + nn.Conv2d(3840,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(3888), + nn.ReLU(), + nn.Conv2d(3888,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(3936), + nn.ReLU(), + nn.Conv2d(3936,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + LambdaReduce(lambda x,y,dim=1: torch.cat((x,y),dim), # Concat, + Lambda(lambda x: x), # Identity, + nn.Sequential( # Sequential, + nn.BatchNorm2d(3984), + nn.ReLU(), + nn.Conv2d(3984,192,(1, 1),(1, 1),(0, 0),1,1,bias=False), + nn.BatchNorm2d(192), + nn.ReLU(), + nn.Conv2d(192,48,(3, 3),(1, 1),(1, 1),1,1,bias=False), + ), + ), + nn.BatchNorm2d(4032), + nn.ReLU(), + nn.AvgPool2d((7, 7),(7, 7)), + Lambda(lambda x: x.view(x.size(0),-1)), # Reshape, + nn.Sequential(Lambda(lambda x: x.view(1,-1) if 1==len(x.size()) else x ),nn.Linear(4032,1000)), # Linear, +) \ No newline at end of file diff --git a/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/imagenet.sh b/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/imagenet.sh new file mode 100644 index 0000000000000000000000000000000000000000..54dcda0a6f6b878ff93913504444621e9cbc3db0 --- /dev/null +++ b/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/imagenet.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +python test.py -m resnet50 diff --git a/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/imagenet_c/MANIFEST.in b/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/imagenet_c/MANIFEST.in new file mode 100644 index 0000000000000000000000000000000000000000..deb7fc47a24fe7a7669a1248093c896391a607d4 --- /dev/null +++ b/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/imagenet_c/MANIFEST.in @@ -0,0 +1,2 @@ +include imagenet_c/frost/* + diff --git a/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/imagenet_c/README.md b/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/imagenet_c/README.md new file mode 100644 index 0000000000000000000000000000000000000000..efa8e01a5c17aae1a3c20b1604476f4534554739 --- /dev/null +++ b/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/imagenet_c/README.md @@ -0,0 +1,33 @@ +# ImageNet-C Corruption Functions + +With this package, it is possible to corrupt an image with ImageNet-C corruptions. +These functions are exposed with the function ```corrupt```. + +Try +``` +from imagenet_c import corrupt + +corrupt(, corruption_number=0) +``` + +The ```corrupt``` function looks like +``` +def corrupt(x, severity=1, corruption_name=None, corruption_number=-1): + """ + :param x: image to corrupt; a 224x224x3 numpy array in [0, 255] + :param severity: strength with which to corrupt x; an integer in [0, 5] + :param corruption_name: specifies which corruption function to call; + must be one of 'gaussian_noise', 'shot_noise', 'impulse_noise', 'defocus_blur', + 'glass_blur', 'motion_blur', 'zoom_blur', 'snow', 'frost', 'fog', + 'brightness', 'contrast', 'elastic_transform', 'pixelate', 'jpeg_compression', + 'speckle_noise', 'gaussian_blur', 'spatter', 'saturate'; + the last four are validation functions + :param corruption_number: the position of the corruption_name in the above list; + an integer in [0, 18]; useful for easy looping; 15, 16, 17, 18 are validation corruption numbers + :return: the image x corrupted by a corruption function at the given severity; same shape as input + """ + ... + +``` + +The "frost" corruption only works should the package be [installed from source](https://github.com/hendrycks/robustness/issues/4#issuecomment-427226016). diff --git a/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/imagenet_c/imagenet_c/__init__.py b/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/imagenet_c/imagenet_c/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d41c63097cd95261663a5e31dbb43ef3db3be586 --- /dev/null +++ b/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/imagenet_c/imagenet_c/__init__.py @@ -0,0 +1,35 @@ +import numpy as np +from PIL import Image +from .corruptions import * + +corruption_tuple = (gaussian_noise, shot_noise, impulse_noise, defocus_blur, + glass_blur, motion_blur, zoom_blur, snow, frost, fog, + brightness, contrast, elastic_transform, pixelate, jpeg_compression, + speckle_noise, gaussian_blur, spatter, saturate) + +corruption_dict = {corr_func.__name__: corr_func for corr_func in corruption_tuple} + + +def corrupt(x, severity=1, corruption_name=None, corruption_number=-1): + """ + :param x: image to corrupt; a 224x224x3 numpy array in [0, 255] + :param severity: strength with which to corrupt x; an integer in [0, 5] + :param corruption_name: specifies which corruption function to call; + must be one of 'gaussian_noise', 'shot_noise', 'impulse_noise', 'defocus_blur', + 'glass_blur', 'motion_blur', 'zoom_blur', 'snow', 'frost', 'fog', + 'brightness', 'contrast', 'elastic_transform', 'pixelate', 'jpeg_compression', + 'speckle_noise', 'gaussian_blur', 'spatter', 'saturate'; + the last four are validation functions + :param corruption_number: the position of the corruption_name in the above list; + an integer in [0, 18]; useful for easy looping; 15, 16, 17, 18 are validation corruption numbers + :return: the image x corrupted by a corruption function at the given severity; same shape as input + """ + + if corruption_name: + x_corrupted = corruption_dict[corruption_name](Image.fromarray(x), severity) + elif corruption_number != -1: + x_corrupted = corruption_tuple[corruption_number](Image.fromarray(x), severity) + else: + raise ValueError("Either corruption_name or corruption_number must be passed") + + return np.uint8(x_corrupted) diff --git a/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/imagenet_c/imagenet_c/corruptions.py b/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/imagenet_c/imagenet_c/corruptions.py new file mode 100644 index 0000000000000000000000000000000000000000..f0bf2d495dc9514e8e6f800d1d2a753f781f9dd8 --- /dev/null +++ b/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/imagenet_c/imagenet_c/corruptions.py @@ -0,0 +1,427 @@ +# -*- coding: utf-8 -*- + +import numpy as np +from PIL import Image + +# /////////////// Corruption Helpers /////////////// + +import skimage as sk +from skimage.filters import gaussian +from io import BytesIO +from wand.image import Image as WandImage +from wand.api import library as wandlibrary +import wand.color as WandColor +import ctypes +from PIL import Image as PILImage +import cv2 +from scipy.ndimage import zoom as scizoom +from scipy.ndimage.interpolation import map_coordinates +import warnings +import os +from pkg_resources import resource_filename + +warnings.simplefilter("ignore", UserWarning) + + +def disk(radius, alias_blur=0.1, dtype=np.float32): + if radius <= 8: + L = np.arange(-8, 8 + 1) + ksize = (3, 3) + else: + L = np.arange(-radius, radius + 1) + ksize = (5, 5) + X, Y = np.meshgrid(L, L) + aliased_disk = np.array((X ** 2 + Y ** 2) <= radius ** 2, dtype=dtype) + aliased_disk /= np.sum(aliased_disk) + + # supersample disk to antialias + return cv2.GaussianBlur(aliased_disk, ksize=ksize, sigmaX=alias_blur) + + +# Tell Python about the C method +wandlibrary.MagickMotionBlurImage.argtypes = (ctypes.c_void_p, # wand + ctypes.c_double, # radius + ctypes.c_double, # sigma + ctypes.c_double) # angle + + +# Extend wand.image.Image class to include method signature +class MotionImage(WandImage): + def motion_blur(self, radius=0.0, sigma=0.0, angle=0.0): + wandlibrary.MagickMotionBlurImage(self.wand, radius, sigma, angle) + + +# modification of https://github.com/FLHerne/mapgen/blob/master/diamondsquare.py +def plasma_fractal(mapsize=256, wibbledecay=3): + """ + Generate a heightmap using diamond-square algorithm. + Return square 2d array, side length 'mapsize', of floats in range 0-255. + 'mapsize' must be a power of two. + """ + assert (mapsize & (mapsize - 1) == 0) + maparray = np.empty((mapsize, mapsize), dtype=np.float_) + maparray[0, 0] = 0 + stepsize = mapsize + wibble = 100 + + def wibbledmean(array): + return array / 4 + wibble * np.random.uniform(-wibble, wibble, array.shape) + + def fillsquares(): + """For each square of points stepsize apart, + calculate middle value as mean of points + wibble""" + cornerref = maparray[0:mapsize:stepsize, 0:mapsize:stepsize] + squareaccum = cornerref + np.roll(cornerref, shift=-1, axis=0) + squareaccum += np.roll(squareaccum, shift=-1, axis=1) + maparray[stepsize // 2:mapsize:stepsize, + stepsize // 2:mapsize:stepsize] = wibbledmean(squareaccum) + + def filldiamonds(): + """For each diamond of points stepsize apart, + calculate middle value as mean of points + wibble""" + mapsize = maparray.shape[0] + drgrid = maparray[stepsize // 2:mapsize:stepsize, stepsize // 2:mapsize:stepsize] + ulgrid = maparray[0:mapsize:stepsize, 0:mapsize:stepsize] + ldrsum = drgrid + np.roll(drgrid, 1, axis=0) + lulsum = ulgrid + np.roll(ulgrid, -1, axis=1) + ltsum = ldrsum + lulsum + maparray[0:mapsize:stepsize, stepsize // 2:mapsize:stepsize] = wibbledmean(ltsum) + tdrsum = drgrid + np.roll(drgrid, 1, axis=1) + tulsum = ulgrid + np.roll(ulgrid, -1, axis=0) + ttsum = tdrsum + tulsum + maparray[stepsize // 2:mapsize:stepsize, 0:mapsize:stepsize] = wibbledmean(ttsum) + + while stepsize >= 2: + fillsquares() + filldiamonds() + stepsize //= 2 + wibble /= wibbledecay + + maparray -= maparray.min() + return maparray / maparray.max() + + +def clipped_zoom(img, zoom_factor): + h = img.shape[0] + # ceil crop height(= crop width) + ch = int(np.ceil(h / float(zoom_factor))) + + top = (h - ch) // 2 + img = scizoom(img[top:top + ch, top:top + ch], (zoom_factor, zoom_factor, 1), order=1) + # trim off any extra pixels + trim_top = (img.shape[0] - h) // 2 + + return img[trim_top:trim_top + h, trim_top:trim_top + h] + + +# /////////////// End Corruption Helpers /////////////// + + +# /////////////// Corruptions /////////////// + +def gaussian_noise(x, severity=1): + c = [.08, .12, 0.18, 0.26, 0.38][severity - 1] + + x = np.array(x) / 255. + return np.clip(x + np.random.normal(size=x.shape, scale=c), 0, 1) * 255 + + +def shot_noise(x, severity=1): + c = [60, 25, 12, 5, 3][severity - 1] + + x = np.array(x) / 255. + return np.clip(np.random.poisson(x * c) / float(c), 0, 1) * 255 + + +def impulse_noise(x, severity=1): + c = [.03, .06, .09, 0.17, 0.27][severity - 1] + + x = sk.util.random_noise(np.array(x) / 255., mode='s&p', amount=c) + return np.clip(x, 0, 1) * 255 + + +def speckle_noise(x, severity=1): + c = [.15, .2, 0.35, 0.45, 0.6][severity - 1] + + x = np.array(x) / 255. + return np.clip(x + x * np.random.normal(size=x.shape, scale=c), 0, 1) * 255 + + +def fgsm(x, source_net, severity=1): + c = [8, 16, 32, 64, 128][severity - 1] + + x = V(x, requires_grad=True) + logits = source_net(x) + source_net.zero_grad() + loss = F.cross_entropy(logits, V(logits.data.max(1)[1].squeeze_()), size_average=False) + loss.backward() + + return standardize(torch.clamp(unstandardize(x.data) + c / 255. * unstandardize(torch.sign(x.grad.data)), 0, 1)) + + +def gaussian_blur(x, severity=1): + c = [1, 2, 3, 4, 6][severity - 1] + + x = gaussian(np.array(x) / 255., sigma=c, multichannel=True) + return np.clip(x, 0, 1) * 255 + + +def glass_blur(x, severity=1): + # sigma, max_delta, iterations + c = [(0.7, 1, 2), (0.9, 2, 1), (1, 2, 3), (1.1, 3, 2), (1.5, 4, 2)][severity - 1] + + x = np.uint8(gaussian(np.array(x) / 255., sigma=c[0], multichannel=True) * 255) + + # locally shuffle pixels + for i in range(c[2]): + for h in range(224 - c[1], c[1], -1): + for w in range(224 - c[1], c[1], -1): + dx, dy = np.random.randint(-c[1], c[1], size=(2,)) + h_prime, w_prime = h + dy, w + dx + # swap + x[h, w], x[h_prime, w_prime] = x[h_prime, w_prime], x[h, w] + + return np.clip(gaussian(x / 255., sigma=c[0], multichannel=True), 0, 1) * 255 + + +def defocus_blur(x, severity=1): + c = [(3, 0.1), (4, 0.5), (6, 0.5), (8, 0.5), (10, 0.5)][severity - 1] + + x = np.array(x) / 255. + kernel = disk(radius=c[0], alias_blur=c[1]) + + channels = [] + for d in range(3): + channels.append(cv2.filter2D(x[:, :, d], -1, kernel)) + channels = np.array(channels).transpose((1, 2, 0)) # 3x224x224 -> 224x224x3 + + return np.clip(channels, 0, 1) * 255 + + +def motion_blur(x, severity=1): + c = [(10, 3), (15, 5), (15, 8), (15, 12), (20, 15)][severity - 1] + + output = BytesIO() + x.save(output, format='PNG') + x = MotionImage(blob=output.getvalue()) + + x.motion_blur(radius=c[0], sigma=c[1], angle=np.random.uniform(-45, 45)) + + x = cv2.imdecode(np.fromstring(x.make_blob(), np.uint8), + cv2.IMREAD_UNCHANGED) + + if x.shape != (224, 224): + return np.clip(x[..., [2, 1, 0]], 0, 255) # BGR to RGB + else: # greyscale to RGB + return np.clip(np.array([x, x, x]).transpose((1, 2, 0)), 0, 255) + + +def zoom_blur(x, severity=1): + c = [np.arange(1, 1.11, 0.01), + np.arange(1, 1.16, 0.01), + np.arange(1, 1.21, 0.02), + np.arange(1, 1.26, 0.02), + np.arange(1, 1.31, 0.03)][severity - 1] + + x = (np.array(x) / 255.).astype(np.float32) + out = np.zeros_like(x) + for zoom_factor in c: + out += clipped_zoom(x, zoom_factor) + + x = (x + out) / (len(c) + 1) + return np.clip(x, 0, 1) * 255 + + +def fog(x, severity=1): + c = [(1.5, 2), (2., 2), (2.5, 1.7), (2.5, 1.5), (3., 1.4)][severity - 1] + + x = np.array(x) / 255. + max_val = x.max() + x += c[0] * plasma_fractal(wibbledecay=c[1])[:224, :224][..., np.newaxis] + return np.clip(x * max_val / (max_val + c[0]), 0, 1) * 255 + + +def frost(x, severity=1): + c = [(1, 0.4), + (0.8, 0.6), + (0.7, 0.7), + (0.65, 0.7), + (0.6, 0.75)][severity - 1] + idx = np.random.randint(5) + filename = [resource_filename(__name__, 'frost/frost1.png'), + resource_filename(__name__, 'frost/frost2.png'), + resource_filename(__name__, 'frost/frost3.png'), + resource_filename(__name__, 'frost/frost4.jpg'), + resource_filename(__name__, 'frost/frost5.jpg'), + resource_filename(__name__, 'frost/frost6.jpg')][idx] + frost = cv2.imread(filename) + # randomly crop and convert to rgb + x_start, y_start = np.random.randint(0, frost.shape[0] - 224), np.random.randint(0, frost.shape[1] - 224) + frost = frost[x_start:x_start + 224, y_start:y_start + 224][..., [2, 1, 0]] + + return np.clip(c[0] * np.array(x) + c[1] * frost, 0, 255) + + +def snow(x, severity=1): + c = [(0.1, 0.3, 3, 0.5, 10, 4, 0.8), + (0.2, 0.3, 2, 0.5, 12, 4, 0.7), + (0.55, 0.3, 4, 0.9, 12, 8, 0.7), + (0.55, 0.3, 4.5, 0.85, 12, 8, 0.65), + (0.55, 0.3, 2.5, 0.85, 12, 12, 0.55)][severity - 1] + + x = np.array(x, dtype=np.float32) / 255. + snow_layer = np.random.normal(size=x.shape[:2], loc=c[0], scale=c[1]) # [:2] for monochrome + + snow_layer = clipped_zoom(snow_layer[..., np.newaxis], c[2]) + snow_layer[snow_layer < c[3]] = 0 + + snow_layer = PILImage.fromarray((np.clip(snow_layer.squeeze(), 0, 1) * 255).astype(np.uint8), mode='L') + output = BytesIO() + snow_layer.save(output, format='PNG') + snow_layer = MotionImage(blob=output.getvalue()) + + snow_layer.motion_blur(radius=c[4], sigma=c[5], angle=np.random.uniform(-135, -45)) + + snow_layer = cv2.imdecode(np.fromstring(snow_layer.make_blob(), np.uint8), + cv2.IMREAD_UNCHANGED) / 255. + snow_layer = snow_layer[..., np.newaxis] + + x = c[6] * x + (1 - c[6]) * np.maximum(x, cv2.cvtColor(x, cv2.COLOR_RGB2GRAY).reshape(224, 224, 1) * 1.5 + 0.5) + return np.clip(x + snow_layer + np.rot90(snow_layer, k=2), 0, 1) * 255 + + +def spatter(x, severity=1): + c = [(0.65, 0.3, 4, 0.69, 0.6, 0), + (0.65, 0.3, 3, 0.68, 0.6, 0), + (0.65, 0.3, 2, 0.68, 0.5, 0), + (0.65, 0.3, 1, 0.65, 1.5, 1), + (0.67, 0.4, 1, 0.65, 1.5, 1)][severity - 1] + x = np.array(x, dtype=np.float32) / 255. + + liquid_layer = np.random.normal(size=x.shape[:2], loc=c[0], scale=c[1]) + + liquid_layer = gaussian(liquid_layer, sigma=c[2]) + liquid_layer[liquid_layer < c[3]] = 0 + if c[5] == 0: + liquid_layer = (liquid_layer * 255).astype(np.uint8) + dist = 255 - cv2.Canny(liquid_layer, 50, 150) + dist = cv2.distanceTransform(dist, cv2.DIST_L2, 5) + _, dist = cv2.threshold(dist, 20, 20, cv2.THRESH_TRUNC) + dist = cv2.blur(dist, (3, 3)).astype(np.uint8) + dist = cv2.equalizeHist(dist) + ker = np.array([[-2, -1, 0], [-1, 1, 1], [0, 1, 2]]) + dist = cv2.filter2D(dist, cv2.CV_8U, ker) + dist = cv2.blur(dist, (3, 3)).astype(np.float32) + + m = cv2.cvtColor(liquid_layer * dist, cv2.COLOR_GRAY2BGRA) + m /= np.max(m, axis=(0, 1)) + m *= c[4] + + # water is pale turqouise + color = np.concatenate((175 / 255. * np.ones_like(m[..., :1]), + 238 / 255. * np.ones_like(m[..., :1]), + 238 / 255. * np.ones_like(m[..., :1])), axis=2) + + color = cv2.cvtColor(color, cv2.COLOR_BGR2BGRA) + x = cv2.cvtColor(x, cv2.COLOR_BGR2BGRA) + + return cv2.cvtColor(np.clip(x + m * color, 0, 1), cv2.COLOR_BGRA2BGR) * 255 + else: + m = np.where(liquid_layer > c[3], 1, 0) + m = gaussian(m.astype(np.float32), sigma=c[4]) + m[m < 0.8] = 0 + + # mud brown + color = np.concatenate((63 / 255. * np.ones_like(x[..., :1]), + 42 / 255. * np.ones_like(x[..., :1]), + 20 / 255. * np.ones_like(x[..., :1])), axis=2) + + color *= m[..., np.newaxis] + x *= (1 - m[..., np.newaxis]) + + return np.clip(x + color, 0, 1) * 255 + + +def contrast(x, severity=1): + c = [0.4, .3, .2, .1, .05][severity - 1] + + x = np.array(x) / 255. + means = np.mean(x, axis=(0, 1), keepdims=True) + return np.clip((x - means) * c + means, 0, 1) * 255 + + +def brightness(x, severity=1): + c = [.1, .2, .3, .4, .5][severity - 1] + + x = np.array(x) / 255. + x = sk.color.rgb2hsv(x) + x[:, :, 2] = np.clip(x[:, :, 2] + c, 0, 1) + x = sk.color.hsv2rgb(x) + + return np.clip(x, 0, 1) * 255 + + +def saturate(x, severity=1): + c = [(0.3, 0), (0.1, 0), (2, 0), (5, 0.1), (20, 0.2)][severity - 1] + + x = np.array(x) / 255. + x = sk.color.rgb2hsv(x) + x[:, :, 1] = np.clip(x[:, :, 1] * c[0] + c[1], 0, 1) + x = sk.color.hsv2rgb(x) + + return np.clip(x, 0, 1) * 255 + + +def jpeg_compression(x, severity=1): + c = [25, 18, 15, 10, 7][severity - 1] + + output = BytesIO() + x.save(output, 'JPEG', quality=c) + x = PILImage.open(output) + + return x + + +def pixelate(x, severity=1): + c = [0.6, 0.5, 0.4, 0.3, 0.25][severity - 1] + + x = x.resize((int(224 * c), int(224 * c)), PILImage.BOX) + x = x.resize((224, 224), PILImage.BOX) + + return x + + +# mod of https://gist.github.com/erniejunior/601cdf56d2b424757de5 +def elastic_transform(image, severity=1): + c = [(244 * 2, 244 * 0.7, 244 * 0.1), # 244 should have been 224, but ultimately nothing is incorrect + (244 * 2, 244 * 0.08, 244 * 0.2), + (244 * 0.05, 244 * 0.01, 244 * 0.02), + (244 * 0.07, 244 * 0.01, 244 * 0.02), + (244 * 0.12, 244 * 0.01, 244 * 0.02)][severity - 1] + + image = np.array(image, dtype=np.float32) / 255. + shape = image.shape + shape_size = shape[:2] + + # random affine + center_square = np.float32(shape_size) // 2 + square_size = min(shape_size) // 3 + pts1 = np.float32([center_square + square_size, + [center_square[0] + square_size, center_square[1] - square_size], + center_square - square_size]) + pts2 = pts1 + np.random.uniform(-c[2], c[2], size=pts1.shape).astype(np.float32) + M = cv2.getAffineTransform(pts1, pts2) + image = cv2.warpAffine(image, M, shape_size[::-1], borderMode=cv2.BORDER_REFLECT_101) + + dx = (gaussian(np.random.uniform(-1, 1, size=shape[:2]), + c[1], mode='reflect', truncate=3) * c[0]).astype(np.float32) + dy = (gaussian(np.random.uniform(-1, 1, size=shape[:2]), + c[1], mode='reflect', truncate=3) * c[0]).astype(np.float32) + dx, dy = dx[..., np.newaxis], dy[..., np.newaxis] + + x, y, z = np.meshgrid(np.arange(shape[1]), np.arange(shape[0]), np.arange(shape[2])) + indices = np.reshape(y + dy, (-1, 1)), np.reshape(x + dx, (-1, 1)), np.reshape(z, (-1, 1)) + return np.clip(map_coordinates(image, indices, order=1, mode='reflect').reshape(shape), 0, 1) * 255 + + +# /////////////// End Corruptions /////////////// diff --git a/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/imagenet_c/imagenet_c/frost/frost1.png b/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/imagenet_c/imagenet_c/frost/frost1.png new file mode 100644 index 0000000000000000000000000000000000000000..c9edf9b6e1a2744d15af615af641f2aa48aa89c2 --- /dev/null +++ b/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/imagenet_c/imagenet_c/frost/frost1.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff9f907860bd7a835d459e32f9d588062b7f61ee267343cc7222b56753a14755 +size 1199930 diff --git a/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/imagenet_c/imagenet_c/frost/frost2.png b/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/imagenet_c/imagenet_c/frost/frost2.png new file mode 100644 index 0000000000000000000000000000000000000000..48f7a861ffa41b6d7496b701fef96d5edf739282 Binary files /dev/null and b/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/imagenet_c/imagenet_c/frost/frost2.png differ diff --git a/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/imagenet_c/imagenet_c/frost/frost3.png b/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/imagenet_c/imagenet_c/frost/frost3.png new file mode 100644 index 0000000000000000000000000000000000000000..d47f9d25f41251ee9a66b294c0bbfad6053017c9 Binary files /dev/null and b/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/imagenet_c/imagenet_c/frost/frost3.png differ diff --git a/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/imagenet_c/imagenet_c/frost/frost4.jpg b/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/imagenet_c/imagenet_c/frost/frost4.jpg new file mode 100644 index 0000000000000000000000000000000000000000..f8b0c413176d70150b593e029d84b4a88c21dd4b Binary files /dev/null and b/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/imagenet_c/imagenet_c/frost/frost4.jpg differ diff --git a/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/imagenet_c/imagenet_c/frost/frost5.jpg b/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/imagenet_c/imagenet_c/frost/frost5.jpg new file mode 100644 index 0000000000000000000000000000000000000000..95dc9056926d8201df760535f9bb9112f012e862 Binary files /dev/null and b/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/imagenet_c/imagenet_c/frost/frost5.jpg differ diff --git a/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/imagenet_c/imagenet_c/frost/frost6.jpg b/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/imagenet_c/imagenet_c/frost/frost6.jpg new file mode 100644 index 0000000000000000000000000000000000000000..14e5d58e762a5d0808df9fa6494fd6d78ee4409b Binary files /dev/null and b/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/imagenet_c/imagenet_c/frost/frost6.jpg differ diff --git a/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/imagenet_c/new_release_instructions.md b/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/imagenet_c/new_release_instructions.md new file mode 100644 index 0000000000000000000000000000000000000000..d62841c3db60174ed27df52a7003c2c9e84b859a --- /dev/null +++ b/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/imagenet_c/new_release_instructions.md @@ -0,0 +1,9 @@ +### Cutting a new release to pypi + +Bump the version in setup.py + +Run the following +```bash +pip install twine +rm -rf dist && python setup.py sdist bdist_wheel && twine upload dist/* +``` diff --git a/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/imagenet_c/requirements.txt b/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/imagenet_c/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..6e464cf0286f6502387e1437d615c92dc732f3f6 --- /dev/null +++ b/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/imagenet_c/requirements.txt @@ -0,0 +1,6 @@ +numpy==1.14.0 +Pillow==5.0.0 +scikit-image==0.13.1 +Wand==0.4.4 +opencv-python==3.4.2.16 +scipy==1.0.0 diff --git a/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/imagenet_c/setup.py b/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/imagenet_c/setup.py new file mode 100644 index 0000000000000000000000000000000000000000..bc8198a6ad2f7f67511dbc3137cc164fc39b3245 --- /dev/null +++ b/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/imagenet_c/setup.py @@ -0,0 +1,34 @@ +import setuptools +from glob import glob + +with open("README.md", "r") as fh: + long_description = fh.read() + +setuptools.setup( + name="imagenet_c", + version="0.0.2", + author="Dan Hendrycks", + author_email="hendrycks@berkeley.edu", + description="Access to ImageNet-C corruption functions", + long_description=long_description, + long_description_content_type="text/markdown", + url="https://github.com/hendrycks/robustness/tree/master/ImageNet-C/imagenet_c", + packages=setuptools.find_packages(), + package_data={ + "imagenet_c.frost": [ + "frost1.png", "frost2.png", "frost3.png", + "frost4.jpg", "frost5.jpg", "frost6.jpg" + ], + }, + install_requires=[ + 'wand ~= 0.4', + 'opencv-python ~= 3.4', + ], + #include_package_data=True, + classifiers=[ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + ], +) + diff --git a/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/layers.py b/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/layers.py new file mode 100644 index 0000000000000000000000000000000000000000..d9611c360706ede20c585b4b5649deddbca7ef45 --- /dev/null +++ b/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/layers.py @@ -0,0 +1,257 @@ +from __future__ import absolute_import +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import division + +import torch +import torch.nn as nn +from torch.autograd import Variable +import torch.nn.functional as F + + +class LearnedGroupConv(nn.Module): + global_progress = 0.0 + def __init__(self, in_channels, out_channels, kernel_size, stride=1, + padding=0, dilation=1, groups=1, + condense_factor=None, dropout_rate=0.): + super(LearnedGroupConv, self).__init__() + self.norm = nn.BatchNorm2d(in_channels) + self.relu = nn.ReLU(inplace=True) + self.dropout_rate = dropout_rate + if self.dropout_rate > 0: + self.drop = nn.Dropout(dropout_rate, inplace=False) + self.conv = nn.Conv2d(in_channels, out_channels, kernel_size, stride, + padding, dilation, groups=1, bias=False) + self.in_channels = in_channels + self.out_channels = out_channels + self.groups = groups + self.condense_factor = condense_factor + if self.condense_factor is None: + self.condense_factor = self.groups + ### Parameters that should be carefully used + self.register_buffer('_count', torch.zeros(1)) + self.register_buffer('_stage', torch.zeros(1)) + self.register_buffer('_mask', torch.ones(self.conv.weight.size())) + ### Check if arguments are valid + assert self.in_channels % self.groups == 0, "group number can not be divided by input channels" + assert self.in_channels % self.condense_factor == 0, "condensation factor can not be divided by input channels" + assert self.out_channels % self.groups == 0, "group number can not be divided by output channels" + + def forward(self, x): + self._check_drop() + x = self.norm(x) + x = self.relu(x) + if self.dropout_rate > 0: + x = self.drop(x) + ### Masked output + weight = self.conv.weight * self.mask + return F.conv2d(x, weight, None, self.conv.stride, + self.conv.padding, self.conv.dilation, 1) + + def _check_drop(self): + progress = LearnedGroupConv.global_progress + delta = 0 + ### Get current stage + for i in range(self.condense_factor - 1): + if progress * 2 < (i + 1) / (self.condense_factor - 1): + stage = i + break + else: + stage = self.condense_factor - 1 + ### Check for dropping + if not self._at_stage(stage): + self.stage = stage + delta = self.in_channels // self.condense_factor + if delta > 0: + self._dropping(delta) + return + + def _dropping(self, delta): + weight = self.conv.weight * self.mask + ### Sum up all kernels + ### Assume only apply to 1x1 conv to speed up + assert weight.size()[-1] == 1 + weight = weight.abs().squeeze() + assert weight.size()[0] == self.out_channels + assert weight.size()[1] == self.in_channels + d_out = self.out_channels // self.groups + ### Shuffle weight + weight = weight.view(d_out, self.groups, self.in_channels) + weight = weight.transpose(0, 1).contiguous() + weight = weight.view(self.out_channels, self.in_channels) + ### Sort and drop + for i in range(self.groups): + wi = weight[i * d_out:(i + 1) * d_out, :] + ### Take corresponding delta index + di = wi.sum(0).sort()[1][self.count:self.count + delta] + for d in di.data: + self._mask[i::self.groups, d, :, :].fill_(0) + self.count = self.count + delta + + @property + def count(self): + return int(self._count[0]) + + @count.setter + def count(self, val): + self._count.fill_(val) + + @property + def stage(self): + return int(self._stage[0]) + + @stage.setter + def stage(self, val): + self._stage.fill_(val) + + @property + def mask(self): + return Variable(self._mask) + + def _at_stage(self, stage): + return (self._stage == stage).all() + + @property + def lasso_loss(self): + if self._at_stage(self.groups - 1): + return 0 + weight = self.conv.weight * self.mask + ### Assume only apply to 1x1 conv to speed up + assert weight.size()[-1] == 1 + weight = weight.squeeze().pow(2) + d_out = self.out_channels // self.groups + ### Shuffle weight + weight = weight.view(d_out, self.groups, self.in_channels) + weight = weight.sum(0).clamp(min=1e-6).sqrt() + return weight.sum() + + +def ShuffleLayer(x, groups): + batchsize, num_channels, height, width = x.data.size() + channels_per_group = num_channels // groups + ### reshape + x = x.view(batchsize, groups, + channels_per_group, height, width) + ### transpose + x = torch.transpose(x, 1, 2).contiguous() + ### flatten + x = x.view(batchsize, -1, height, width) + return x + + +class CondensingLinear(nn.Module): + def __init__(self, model, drop_rate=0.5): + super(CondensingLinear, self).__init__() + self.in_features = int(model.in_features*drop_rate) + self.out_features = model.out_features + self.linear = nn.Linear(self.in_features, self.out_features) + self.register_buffer('index', torch.LongTensor(self.in_features)) + _, index = model.weight.data.abs().sum(0).sort() + index = index[model.in_features-self.in_features:] + self.linear.bias.data = model.bias.data.clone() + for i in range(self.in_features): + self.index[i] = index[i] + self.linear.weight.data[:, i] = model.weight.data[:, index[i]] + + def forward(self, x): + x = torch.index_select(x, 1, Variable(self.index)) + x = self.linear(x) + return x + + +class CondensingConv(nn.Module): + def __init__(self, model): + super(CondensingConv, self).__init__() + self.in_channels = model.conv.in_channels \ + * model.groups // model.condense_factor + self.out_channels = model.conv.out_channels + self.groups = model.groups + self.condense_factor = model.condense_factor + self.norm = nn.BatchNorm2d(self.in_channels) + self.relu = nn.ReLU(inplace=True) + self.conv = nn.Conv2d(self.in_channels, self.out_channels, + kernel_size=model.conv.kernel_size, + padding=model.conv.padding, + groups=self.groups, + bias=False, + stride=model.conv.stride) + self.register_buffer('index', torch.LongTensor(self.in_channels)) + index = 0 + mask = model._mask.mean(-1).mean(-1) + for i in range(self.groups): + for j in range(model.conv.in_channels): + if index < (self.in_channels // self.groups) * (i + 1) \ + and mask[i, j] == 1: + for k in range(self.out_channels // self.groups): + idx_i = int(k + i * (self.out_channels // self.groups)) + idx_j = index % (self.in_channels // self.groups) + self.conv.weight.data[idx_i, idx_j, :, :] = \ + model.conv.weight.data[int(i + k * self.groups), j, :, :] + self.norm.weight.data[index] = model.norm.weight.data[j] + self.norm.bias.data[index] = model.norm.bias.data[j] + self.norm.running_mean[index] = model.norm.running_mean[j] + self.norm.running_var[index] = model.norm.running_var[j] + self.index[index] = j + index += 1 + + def forward(self, x): + x = torch.index_select(x, 1, Variable(self.index)) + x = self.norm(x) + x = self.relu(x) + x = self.conv(x) + x = ShuffleLayer(x, self.groups) + return x + + +class CondenseLinear(nn.Module): + def __init__(self, in_features, out_features, drop_rate=0.5): + super(CondenseLinear, self).__init__() + self.in_features = int(in_features*drop_rate) + self.out_features = out_features + self.linear = nn.Linear(self.in_features, self.out_features) + self.register_buffer('index', torch.LongTensor(self.in_features)) + + def forward(self, x): + x = torch.index_select(x, 1, Variable(self.index)) + x = self.linear(x) + return x + + +class CondenseConv(nn.Module): + def __init__(self, in_channels, out_channels, kernel_size, + stride=1, padding=0, groups=1): + super(CondenseConv, self).__init__() + self.in_channels = in_channels + self.out_channels = out_channels + self.groups = groups + self.norm = nn.BatchNorm2d(self.in_channels) + self.relu = nn.ReLU(inplace=True) + self.conv = nn.Conv2d(self.in_channels, self.out_channels, + kernel_size=kernel_size, + stride=stride, + padding=padding, + groups=self.groups, + bias=False) + self.register_buffer('index', torch.LongTensor(self.in_channels)) + self.index.fill_(0) + + def forward(self, x): + x = torch.index_select(x, 1, Variable(self.index)) + x = self.norm(x) + x = self.relu(x) + x = self.conv(x) + x = ShuffleLayer(x, self.groups) + return x + + +class Conv(nn.Sequential): + def __init__(self, in_channels, out_channels, kernel_size, + stride=1, padding=0, groups=1): + super(Conv, self).__init__() + self.add_module('norm', nn.BatchNorm2d(in_channels)) + self.add_module('relu', nn.ReLU(inplace=True)) + self.add_module('conv', nn.Conv2d(in_channels, out_channels, + kernel_size=kernel_size, + stride=stride, + padding=padding, bias=False, + groups=groups)) diff --git a/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/test.py b/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/test.py new file mode 100644 index 0000000000000000000000000000000000000000..891968edbac1c02da76abb6c7bcdacb0a4f47624 --- /dev/null +++ b/data/datasets/visual_question_answering/generate_c_image/robustness-master/ImageNet-C/test.py @@ -0,0 +1,288 @@ +# -*- coding: utf-8 -*- + +import argparse +import os +import time +import torch +from torch.autograd import Variable as V +import torch.nn.functional as F +import torch.backends.cudnn as cudnn +import torchvision.datasets as dset +import torchvision.transforms as trn +import torchvision.models as models +import torch.utils.model_zoo as model_zoo +import numpy as np +from resnext_50_32x4d import resnext_50_32x4d +from resnext_101_32x4d import resnext_101_32x4d +from resnext_101_64x4d import resnext_101_64x4d +from densenet_cosine_264_k48 import densenet_cosine_264_k48 +from condensenet_converted import CondenseNet + +parser = argparse.ArgumentParser(description='Evaluates robustness of various nets on ImageNet', + formatter_class=argparse.ArgumentDefaultsHelpFormatter) +# Architecture +parser.add_argument('--model-name', '-m', type=str, + choices=['alexnet', 'squeezenet1.0', 'squeezenet1.1', 'condensenet4', 'condensenet8', + 'vgg11', 'vgg', 'vggbn', + 'densenet121', 'densenet169', 'densenet201', 'densenet161', 'densenet264', + 'resnet18', 'resnet34', 'resnet50', 'resnet101', 'resnet152', + 'resnext50', 'resnext101', 'resnext101_64']) +# Acceleration +parser.add_argument('--ngpu', type=int, default=1, help='0 = CPU.') +args = parser.parse_args() +print(args) + +# /////////////// Model Setup /////////////// + +if args.model_name == 'alexnet': + net = models.AlexNet() + net.load_state_dict(model_zoo.load_url('https://download.pytorch.org/models/alexnet-owt-4df8aa71.pth', + model_dir='/share/data/lang/users/dan/.torch/models')) + args.test_bs = 256 + +elif args.model_name == 'squeezenet1.0': + net = models.SqueezeNet(version=1.0) + net.load_state_dict(model_zoo.load_url('https://download.pytorch.org/models/squeezenet1_0-a815701f.pth', + model_dir='/share/data/lang/users/dan/.torch/models')) + args.test_bs = 256 + +elif args.model_name == 'squeezenet1.1': + net = models.SqueezeNet(version=1.1) + net.load_state_dict(model_zoo.load_url('https://download.pytorch.org/models/squeezenet1_1-f364aa15.pth', + model_dir='/share/data/lang/users/dan/.torch/models')) + args.test_bs = 256 + +elif args.model_name == 'condensenet4': + args.evaluate = True + args.stages = [4,6,8,10,8] + args.growth = [8,16,32,64,128] + args.data = 'imagenet' + args.num_classes = 1000 + args.bottleneck = 4 + args.group_1x1 = 4 + args.group_3x3 = 4 + args.reduction = 0.5 + args.condense_factor = 4 + net = CondenseNet(args) + state_dict = torch.load('./converted_condensenet_4.pth')['state_dict'] + for i in range(len(state_dict)): + name, v = state_dict.popitem(False) + state_dict[name[7:]] = v # remove 'module.' in key beginning + net.load_state_dict(state_dict) + args.test_bs = 256 + +elif args.model_name == 'condensenet8': + args.evaluate = True + args.stages = [4,6,8,10,8] + args.growth = [8,16,32,64,128] + args.data = 'imagenet' + args.num_classes = 1000 + args.bottleneck = 4 + args.group_1x1 = 8 + args.group_3x3 = 8 + args.reduction = 0.5 + args.condense_factor = 8 + net = CondenseNet(args) + state_dict = torch.load('./converted_condensenet_8.pth')['state_dict'] + for i in range(len(state_dict)): + name, v = state_dict.popitem(False) + state_dict[name[7:]] = v # remove 'module.' in key beginning + net.load_state_dict(state_dict) + args.test_bs = 256 + +elif 'vgg' in args.model_name: + if 'bn' not in args.model_name: + net = models.vgg19() + net.load_state_dict(model_zoo.load_url('https://download.pytorch.org/models/vgg19-dcbb9e9d.pth', + model_dir='/share/data/lang/users/dan/.torch/models')) + elif '11' in args.model_name: + net = models.vgg11() + net.load_state_dict(model_zoo.load_url('https://download.pytorch.org/models/vgg11-bbd30ac9.pth', + model_dir='/share/data/lang/users/dan/.torch/models')) + else: + net = models.vgg19_bn() + net.load_state_dict(model_zoo.load_url('https://download.pytorch.org/models/vgg19_bn-c79401a0.pth', + model_dir='/share/data/lang/users/dan/.torch/models')) + args.test_bs = 64 + +elif args.model_name == 'densenet121': + net = models.densenet121() + net.load_state_dict(model_zoo.load_url('https://download.pytorch.org/models/densenet121-a639ec97.pth', + model_dir='/share/data/lang/users/dan/.torch/models')) + args.test_bs = 128 + +elif args.model_name == 'densenet169': + net = models.densenet169() + net.load_state_dict(model_zoo.load_url('https://download.pytorch.org/models/densenet169-6f0f7f60.pth', + model_dir='/share/data/lang/users/dan/.torch/models')) + args.test_bs = 128 + +elif args.model_name == 'densenet201': + net = models.densenet201() + net.load_state_dict(model_zoo.load_url('https://download.pytorch.org/models/densenet201-c1103571.pth', + model_dir='/share/data/lang/users/dan/.torch/models')) + args.test_bs = 64 + +elif args.model_name == 'densenet161': + net = models.densenet161() + net.load_state_dict(model_zoo.load_url('https://download.pytorch.org/models/densenet161-8d451a50.pth', + model_dir='/share/data/lang/users/dan/.torch/models')) + args.test_bs = 64 + +elif args.model_name == 'densenet264': + net = densenet_cosine_264_k48 + net.load_state_dict(model_zoo.load_url('https://download.pytorch.org/models/densenet_cosine_264_k48.pth', + model_dir='/share/data/lang/users/dan/.torch/models')) + args.test_bs = 64 + +elif args.model_name == 'resnet18': + net = models.resnet18() + net.load_state_dict(model_zoo.load_url('https://download.pytorch.org/models/resnet18-5c106cde.pth', + model_dir='/share/data/lang/users/dan/.torch/models')) + args.test_bs = 256 + +elif args.model_name == 'resnet34': + net = models.resnet34() + net.load_state_dict(model_zoo.load_url('https://download.pytorch.org/models/resnet34-333f7ec4.pth', + model_dir='/share/data/lang/users/dan/.torch/models')) + args.test_bs = 128 + +elif args.model_name == 'resnet50': + net = models.resnet50() + net.load_state_dict(model_zoo.load_url('https://download.pytorch.org/models/resnet50-19c8e357.pth', + model_dir='/share/data/lang/users/dan/.torch/models')) + args.test_bs = 128 + +elif args.model_name == 'resnet101': + net = models.resnet101() + net.load_state_dict(model_zoo.load_url('https://download.pytorch.org/models/resnet101-5d3b4d8f.pth', + model_dir='/share/data/lang/users/dan/.torch/models')) + args.test_bs = 128 + +elif args.model_name == 'resnet152': + net = models.resnet152() + net.load_state_dict(model_zoo.load_url('https://download.pytorch.org/models/resnet152-b121ed2d.pth', + model_dir='/share/data/lang/users/dan/.torch/models')) + args.test_bs = 64 + +elif args.model_name == 'resnext50': + net = resnext_50_32x4d + net.load_state_dict(torch.load('/share/data/lang/users/dan/.torch/models/resnext_50_32x4d.pth')) + args.test_bs = 64 + +elif args.model_name == 'resnext101': + net = resnext_101_32x4d + net.load_state_dict(torch.load('/share/data/lang/users/dan/.torch/models/resnext_101_32x4d.pth')) + args.test_bs = 64 + +elif args.model_name == 'resnext101_64': + net = resnext_101_64x4d + net.load_state_dict(torch.load('/share/data/lang/users/dan/.torch/models/resnext_101_64x4d.pth')) + args.test_bs = 64 + +args.prefetch = 4 + +for p in net.parameters(): + p.volatile = True + +if args.ngpu > 1: + net = torch.nn.DataParallel(net, device_ids=list(range(args.ngpu))) + +if args.ngpu > 0: + net.cuda() + +torch.manual_seed(1) +np.random.seed(1) +if args.ngpu > 0: + torch.cuda.manual_seed(1) + +net.eval() +cudnn.benchmark = True # fire on all cylinders + +print('Model Loaded') + +# /////////////// Data Loader /////////////// + +mean = [0.485, 0.456, 0.406] +std = [0.229, 0.224, 0.225] + +clean_loader = torch.utils.data.DataLoader(dset.ImageFolder( + root="/share/data/vision-greg/ImageNet/clsloc/images/val", + transform=trn.Compose([trn.Resize(256), trn.CenterCrop(224), trn.ToTensor(), trn.Normalize(mean, std)])), + batch_size=args.test_bs, shuffle=False, num_workers=args.prefetch, pin_memory=True) + + +# /////////////// Further Setup /////////////// + +def auc(errs): # area under the distortion-error curve + area = 0 + for i in range(1, len(errs)): + area += (errs[i] + errs[i - 1]) / 2 + area /= len(errs) - 1 + return area + + +# correct = 0 +# for batch_idx, (data, target) in enumerate(clean_loader): +# data = V(data.cuda(), volatile=True) +# +# output = net(data) +# +# pred = output.data.max(1)[1] +# correct += pred.eq(target.cuda()).sum() +# +# clean_error = 1 - correct / len(clean_loader.dataset) +# print('Clean dataset error (%): {:.2f}'.format(100 * clean_error)) + + +def show_performance(distortion_name): + errs = [] + + for severity in range(1, 6): + distorted_dataset = dset.ImageFolder( + root='/share/data/vision-greg/DistortedImageNet/JPEG/' + distortion_name + '/' + str(severity), + transform=trn.Compose([trn.CenterCrop(224), trn.ToTensor(), trn.Normalize(mean, std)])) + + distorted_dataset_loader = torch.utils.data.DataLoader( + distorted_dataset, batch_size=args.test_bs, shuffle=False, num_workers=args.prefetch, pin_memory=True) + + correct = 0 + for batch_idx, (data, target) in enumerate(distorted_dataset_loader): + data = V(data.cuda(), volatile=True) + + output = net(data) + + pred = output.data.max(1)[1] + correct += pred.eq(target.cuda()).sum() + + errs.append(1 - 1.*correct / len(distorted_dataset)) + + print('\n=Average', tuple(errs)) + return np.mean(errs) + + +# /////////////// End Further Setup /////////////// + + +# /////////////// Display Results /////////////// +import collections + +print('\nUsing ImageNet data') + +distortions = [ + 'gaussian_noise', 'shot_noise', 'impulse_noise', + 'defocus_blur', 'glass_blur', 'motion_blur', 'zoom_blur', + 'snow', 'frost', 'fog', 'brightness', + 'contrast', 'elastic_transform', 'pixelate', 'jpeg_compression', + 'speckle_noise', 'gaussian_blur', 'spatter', 'saturate' +] + +error_rates = [] +for distortion_name in distortions: + rate = show_performance(distortion_name) + error_rates.append(rate) + print('Distortion: {:15s} | CE (unnormalized) (%): {:.2f}'.format(distortion_name, 100 * rate)) + + +print('mCE (unnormalized by AlexNet errors) (%): {:.2f}'.format(100 * np.mean(error_rates))) + diff --git a/data/datasets/visual_question_answering/generate_c_image/robustness-master/LICENSE b/data/datasets/visual_question_answering/generate_c_image/robustness-master/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..261eeb9e9f8b2b4b0d119366dda99c6fd7d35c64 --- /dev/null +++ b/data/datasets/visual_question_answering/generate_c_image/robustness-master/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/data/datasets/visual_question_answering/generate_c_image/robustness-master/README.md b/data/datasets/visual_question_answering/generate_c_image/robustness-master/README.md new file mode 100644 index 0000000000000000000000000000000000000000..b6fb6af5f8ef141f948fcde4dd74356732f6a641 --- /dev/null +++ b/data/datasets/visual_question_answering/generate_c_image/robustness-master/README.md @@ -0,0 +1,95 @@ +# Benchmarking Neural Network Robustness to Common Corruptions and Perturbations + +This repository contains the datasets and some code for the paper [Benchmarking Neural Network Robustness to Common Corruptions and Perturbations](https://arxiv.org/abs/1903.12261) (ICLR 2019) by Dan Hendrycks and Thomas Dietterich. + +Requires Python 3+ and PyTorch 0.3+. For evaluation, please download the data from the links below. + +## ImageNet-C + + + +[Download ImageNet-C here.](https://zenodo.org/record/2235448) [(Mirror.)](https://drive.google.com/drive/folders/1HDVw6CmX3HiG0ODFtI75iIfBDxSiSz2K?usp=sharing) + +[Download Tiny ImageNet-C here.](https://zenodo.org/record/2536630) [(Mirror.)](https://berkeley.box.com/s/6zt1qzwm34hgdzcvi45svsb10zspop8a) + +Tiny ImageNet-C has 200 classes with images of size 64x64, while ImageNet-C has all 1000 classes where each image is the standard size. For even quicker experimentation, there is [CIFAR-10-C](https://zenodo.org/record/2535967) and [CIFAR-100-C](https://zenodo.org/record/3555552). Evaluation using the JPEGs above is strongly prefered to computing the corruptions in memory, so that evaluation is deterministic and consistent. + +## ImageNet-C Leaderboard + +ImageNet-C Robustness with a ResNet-50 Backbone trained on ImageNet-1K and evaluated on 224x224x3 images. + +| Method | Reference | Standalone? | mCE | Clean Error | +|-------------------------------|------------------------------------------------------------------------|:-------:|:-------:| :-------:| +| [DeepAugment+AugMix](https://github.com/hendrycks/imagenet-r) | [Hendrycks et al.](https://arxiv.org/abs/2006.16241) | No | 53.6% | 24.2% +| [Assemble-ResNet50](https://github.com/clovaai/assembled-cnn) | [Lee et al.](https://arxiv.org/abs/2001.06268) | No | 56.5% | 17.90% +| [ANT (3x3)](https://github.com/bethgelab/game-of-noise) | [Rusak and Schott et al.](https://arxiv.org/abs/2001.06057) | Yes | 63% | 23.9% +| BlurAfterConv | [Vasconcelos et al.](https://arxiv.org/abs/2011.10675)| Yes | 64.9% | 21.2% +| [AugMix](https://github.com/google-research/augmix) | [Hendrycks and Mu et al.](https://arxiv.org/pdf/1912.02781.pdf) (ICLR 2020) | Yes | 65.3% | 22.47% +| [Stylized ImageNet](https://github.com/rgeirhos/Stylized-ImageNet) | [Geirhos et al.](https://arxiv.org/pdf/1811.12231.pdf) (ICLR 2019) | Yes | 69.3% | 25.41% +| [Group-wise Inhibition+DataAug](https://github.com/LinusWu/TENET_Training) | [Liu et al.](https://openaccess.thecvf.com/content/ICCV2021/papers/Liu_Group-Wise_Inhibition_Based_Feature_Regularization_for_Robust_Classification_ICCV_2021_paper.pdf) (ICCV2021) | No | 69.6% | 22.4% +| Patch Uniform | [Lopes et al.](https://arxiv.org/abs/1906.02611) | Yes | 74.3% | 24.5% +| [Group-wise Inhibition](https://github.com/LinusWu/TENET_Training) | [Liu et al.](https://openaccess.thecvf.com/content/ICCV2021/papers/Liu_Group-Wise_Inhibition_Based_Feature_Regularization_for_Robust_Classification_ICCV_2021_paper.pdf) (ICCV2021) | Yes | 75.3% | 23.1% +| ResNet-50 Baseline | | N/A | 76.7% | 23.85% + +"Standalone" indicates whether the method is a combination of techniques or a standalone/single method. Combining methods and proposing standalone methods are both valuable but not necessarily commensurable. + +Be sure to check each paper for results on all 15 corruptions, as some of these techniques improve robustness on all corruptions, some methods help on some corruptions and hurt on others, and some are exceptional against noise corruptions. Other backbones can obtain better results. For example, a vanilla ResNeXt-101 has an mCE of 62.2%. Note Lopes et al. have a ResNet-50 backbone with an mCE of [80.6](https://openreview.net/pdf?id=S1gmrxHFvB#page=7&zoom=100,144,580), so their improvement is larger than what is immediately suggested by the table. + +Submit a pull request if you beat the state-of-the-art on ImageNet-C with a ResNet-50 backbone. + +### UPDATE: New Robustness Benchmarks + +For other distribution shift benchmarks like ImageNet-C, consider datasets like [ImageNet-A](https://github.com/hendrycks/natural-adv-examples) or [ImageNet-R](https://github.com/hendrycks/imagenet-r/). + +ImageNet-A contains real-world, unmodified natural images that cause model accuracy to substantially degrade. ImageNet-R(endition) has 30,000 renditions of ImageNet classes cocering art, cartoons, deviantart, graffiti, embroidery, graphics, origami, paintings, patterns, plastic objects, plush objects, sculptures, sketches, tattoos, toys, and video games. + + +## Calculating the mCE + +This spreadsheet shows how to calculate the [mean Corruption Error](https://docs.google.com/spreadsheets/d/1RwqofJPHhtdRPG-dDO7wPp-aGn-AmwmU5-rpvTzrMHw/edit?usp=sharing). + +## ImageNet-P + + + +ImageNet-P sequences are MP4s not GIFs. The spatter perturbation sequence is a validation sequence. + +[Download Tiny ImageNet-P here.](https://zenodo.org/record/2536630) [(Mirror.)](https://berkeley.box.com/s/19m2ppji0xsqgtkrs95329bqftbvncx9) + +[Download ImageNet-P here.](https://zenodo.org/record/3565846) [(Mirror.)](https://drive.google.com/drive/folders/1vRrDaWA6-_GaUZqOmovWrr4W34aiSLu7?usp=sharing) + +[Download CIFAR-10-P here.](https://drive.google.com/drive/folders/1dY1_zeRyjMKdfmMbQ_uK8O1i0MVI9UbZ?usp=sharing) + +[Download CIFAR-100-P here.](https://drive.google.com/drive/folders/1Tpssw4Vn6X_4hmIW8KK5LzkKCZpAaae8?usp=sharing) + +## ImageNet-P Leaderboard + +ImageNet-P Perturbation Robustness with a ResNet-50 Backbone + +| Method | Reference | mFR | mT5D | +|-------------------------------------|--------------------------------------------------------------------|:-------:|:-------:| +| [AugMix](https://github.com/google-research/augmix) | [Hendrycks and Mu et al.](https://arxiv.org/pdf/1912.02781.pdf) (ICLR 2020) | 37.4% | | +| Low Pass Filter Pooling (bin-5) | [Zhang](https://arxiv.org/abs/1904.11486) (ICML 2019) | 51.2% | 71.9% | +| ResNet-50 Baseline | | 58.0% | 78.4% | + +Submit a pull request if you beat the state-of-the-art on ImageNet-P. + +## Citation + +If you find this useful in your research, please consider citing: + + @article{hendrycks2019robustness, + title={Benchmarking Neural Network Robustness to Common Corruptions and Perturbations}, + author={Dan Hendrycks and Thomas Dietterich}, + journal={Proceedings of the International Conference on Learning Representations}, + year={2019} + } + +Part of the code was contributed by [Tom Brown](https://github.com/nottombrown). + +## Icons-50 (From an Older Draft) + + + +Download Icons-50 [here](https://berkeley.box.com/s/jcem6ik7rxr6594lg99kmrdo01ue6yjt) or [here.](https://drive.google.com/drive/folders/16_kaFo3uUoS-U8FTDm4nUh6Vo21UVnJX?usp=sharing) + diff --git a/data/datasets/visual_question_answering/generate_c_image/robustness-master/assets/icons-50.png b/data/datasets/visual_question_answering/generate_c_image/robustness-master/assets/icons-50.png new file mode 100644 index 0000000000000000000000000000000000000000..11b4732e80ea2ced92c01d52314e211d9a4e81c8 Binary files /dev/null and b/data/datasets/visual_question_answering/generate_c_image/robustness-master/assets/icons-50.png differ diff --git a/data/datasets/visual_question_answering/generate_c_image/robustness-master/assets/imagenet-c.png b/data/datasets/visual_question_answering/generate_c_image/robustness-master/assets/imagenet-c.png new file mode 100644 index 0000000000000000000000000000000000000000..fc181d0a33005fe371bcdbdab031d91ad225f177 Binary files /dev/null and b/data/datasets/visual_question_answering/generate_c_image/robustness-master/assets/imagenet-c.png differ diff --git a/data/datasets/visual_question_answering/generate_c_image/robustness-master/assets/spatter.gif b/data/datasets/visual_question_answering/generate_c_image/robustness-master/assets/spatter.gif new file mode 100644 index 0000000000000000000000000000000000000000..388ad07e6d48f49d4da4f24abb13627f848633b0 --- /dev/null +++ b/data/datasets/visual_question_answering/generate_c_image/robustness-master/assets/spatter.gif @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3e72e4fd8cad91eb8c4dde9e8251ca66bfd76b9f60b67fa411f1521a71fd5fad +size 1015591 diff --git a/data/datasets/visual_question_answering/generate_c_image/robustness-master/assets/tilt.gif b/data/datasets/visual_question_answering/generate_c_image/robustness-master/assets/tilt.gif new file mode 100644 index 0000000000000000000000000000000000000000..dab79ed1b65a84930ce1b32a4dc64b287d18b873 --- /dev/null +++ b/data/datasets/visual_question_answering/generate_c_image/robustness-master/assets/tilt.gif @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9f42cd01272ecf89e5b57935130191f408139a82f5091fd76f318e076b48c339 +size 1151135 diff --git a/data/datasets/visual_question_answering/generate_c_image/robustness-master/assets/translate.gif b/data/datasets/visual_question_answering/generate_c_image/robustness-master/assets/translate.gif new file mode 100644 index 0000000000000000000000000000000000000000..3a848ae3ed47f963569c7f41517309696191e5a0 --- /dev/null +++ b/data/datasets/visual_question_answering/generate_c_image/robustness-master/assets/translate.gif @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:770824edec8fa93e13b622e092402c895003a0be053da6585be15a36f7df7bc3 +size 1519142 diff --git a/data/datasets/visual_question_answering/generate_c_image/robustness-master/old/Icons-50/models/augment.py b/data/datasets/visual_question_answering/generate_c_image/robustness-master/old/Icons-50/models/augment.py new file mode 100644 index 0000000000000000000000000000000000000000..ea0d7e3c452970aa8da42d3d6e11efca3657b6a7 --- /dev/null +++ b/data/datasets/visual_question_answering/generate_c_image/robustness-master/old/Icons-50/models/augment.py @@ -0,0 +1,53 @@ +from torchvision.transforms import * + +from PIL import Image +import random +import math +import numpy as np +import torch + +class RandomErasing(object): + ''' + Class that performs Random Erasing in Random Erasing Data Augmentation by Zhong et al. + ------------------------------------------------------------------------------------- + probability: The probability that the operation will be performed. + sl: min erasing area + sh: max erasing area + r1: min aspect ratio + mean: erasing value + ------------------------------------------------------------------------------------- + ''' + def __init__(self, probability = 0.5, sl = 0.02, sh = 0.4, r1 = 0.3, mean=[0.5, 0.5, 0.5]): + self.probability = probability + self.mean = mean + self.sl = sl + self.sh = sh + self.r1 = r1 + + def __call__(self, img): + + if random.uniform(0, 1) > self.probability: + return img + + for attempt in range(100): + area = img.size()[1] * img.size()[2] + + target_area = random.uniform(self.sl, self.sh) * area + aspect_ratio = random.uniform(self.r1, 1/self.r1) + + h = int(round(math.sqrt(target_area * aspect_ratio))) + w = int(round(math.sqrt(target_area / aspect_ratio))) + + if w < img.size()[2] and h < img.size()[1]: + x1 = random.randint(0, img.size()[1] - h) + y1 = random.randint(0, img.size()[2] - w) + if img.size()[0] == 3: + img[0, x1:x1+h, y1:y1+w] = self.mean[0] + img[1, x1:x1+h, y1:y1+w] = self.mean[1] + img[2, x1:x1+h, y1:y1+w] = self.mean[2] + else: + img[0, x1:x1+h, y1:y1+w] = self.mean[0] + return img + + return img + diff --git a/data/datasets/visual_question_answering/generate_c_image/robustness-master/old/Icons-50/models/densenet.py b/data/datasets/visual_question_answering/generate_c_image/robustness-master/old/Icons-50/models/densenet.py new file mode 100644 index 0000000000000000000000000000000000000000..61fb55733051574450b476fa8b132664720877f7 --- /dev/null +++ b/data/datasets/visual_question_answering/generate_c_image/robustness-master/old/Icons-50/models/densenet.py @@ -0,0 +1,189 @@ +# coding: utf-8 + +import torch +import torch.nn as nn +import torch.nn.functional as F + + +def initialize_weights(m): + if isinstance(m, nn.Conv2d): + nn.init.kaiming_normal_(m.weight.data, mode='fan_out') + elif isinstance(m, nn.BatchNorm2d): + m.weight.data.fill_(1) + m.bias.data.zero_() + elif isinstance(m, nn.Linear): + m.bias.data.zero_() + + +class BasicBlock(nn.Module): + def __init__(self, in_channels, out_channels, drop_rate): + super(BasicBlock, self).__init__() + + self.drop_rate = drop_rate + + self.bn = nn.BatchNorm2d(in_channels) + self.conv = nn.Conv2d( + in_channels, + out_channels, + kernel_size=3, + stride=1, + padding=1, + bias=False) + + def forward(self, x): + y = self.conv(F.relu(self.bn(x), inplace=True)) + if self.drop_rate > 0: + y = F.dropout( + y, p=self.drop_rate, training=self.training, inplace=False) + return torch.cat([x, y], dim=1) + + +class BottleneckBlock(nn.Module): + def __init__(self, in_channels, out_channels, drop_rate): + super(BottleneckBlock, self).__init__() + + self.drop_rate = drop_rate + + bottleneck_channels = out_channels * 4 + + self.bn1 = nn.BatchNorm2d(in_channels) + self.conv1 = nn.Conv2d( + in_channels, + bottleneck_channels, + kernel_size=1, + stride=1, + padding=0, + bias=False) + + self.bn2 = nn.BatchNorm2d(bottleneck_channels) + self.conv2 = nn.Conv2d( + bottleneck_channels, + out_channels, + kernel_size=3, + stride=1, + padding=1, + bias=False) + + def forward(self, x): + y = self.conv1(F.relu(self.bn1(x), inplace=True)) + if self.drop_rate > 0: + y = F.dropout( + y, p=self.drop_rate, training=self.training, inplace=False) + y = self.conv2(F.relu(self.bn2(y), inplace=True)) + if self.drop_rate > 0: + y = F.dropout( + y, p=self.drop_rate, training=self.training, inplace=False) + return torch.cat([x, y], dim=1) + + +class TransitionBlock(nn.Module): + def __init__(self, in_channels, out_channels, drop_rate): + super(TransitionBlock, self).__init__() + + self.drop_rate = drop_rate + + self.bn = nn.BatchNorm2d(in_channels) + self.conv = nn.Conv2d( + in_channels, + out_channels, + kernel_size=1, + stride=1, + padding=0, + bias=False) + + def forward(self, x): + x = self.conv(F.relu(self.bn(x), inplace=True)) + if self.drop_rate > 0: + x = F.dropout( + x, p=self.drop_rate, training=self.training, inplace=False) + x = F.avg_pool2d(x, kernel_size=2, stride=2) + return x + + +class DenseNet(nn.Module): + def __init__(self, config): + super(DenseNet, self).__init__() + + input_shape = config['input_shape'] + n_classes = config['n_classes'] + + block_type = config['block_type'] + depth = config['depth'] + self.growth_rate = config['growth_rate'] + self.drop_rate = config['drop_rate'] + self.compression_rate = config['compression_rate'] + + assert block_type in ['basic', 'bottleneck'] + if block_type == 'basic': + block = BasicBlock + n_blocks_per_stage = (depth - 4) // 3 + assert n_blocks_per_stage * 3 + 4 == depth + else: + block = BottleneckBlock + n_blocks_per_stage = (depth - 4) // 6 + assert n_blocks_per_stage * 6 + 4 == depth + + in_channels = [2 * self.growth_rate] + for index in range(3): + denseblock_out_channels = int( + in_channels[-1] + n_blocks_per_stage * self.growth_rate) + if index < 2: + transitionblock_out_channels = int( + denseblock_out_channels * self.compression_rate) + else: + transitionblock_out_channels = denseblock_out_channels + in_channels.append(transitionblock_out_channels) + + self.conv = nn.Conv2d( + input_shape[1], + in_channels[0], + kernel_size=3, + stride=1, + padding=1, + bias=False) + self.stage1 = self._make_stage(in_channels[0], n_blocks_per_stage, + block, True) + self.stage2 = self._make_stage(in_channels[1], n_blocks_per_stage, + block, True) + self.stage3 = self._make_stage(in_channels[2], n_blocks_per_stage, + block, False) + self.bn = nn.BatchNorm2d(in_channels[3]) + + # compute conv feature size + self.feature_size = self._forward_conv( + torch.zeros(*input_shape)).view(-1).shape[0] + + self.fc = nn.Linear(self.feature_size, n_classes) + + # initialize weights + self.apply(initialize_weights) + + def _make_stage(self, in_channels, n_blocks, block, add_transition_block): + stage = nn.Sequential() + for index in range(n_blocks): + stage.add_module('block{}'.format(index + 1), + block(in_channels + index * self.growth_rate, + self.growth_rate, self.drop_rate)) + if add_transition_block: + in_channels = int(in_channels + n_blocks * self.growth_rate) + out_channels = int(in_channels * self.compression_rate) + stage.add_module('transition', + TransitionBlock(in_channels, out_channels, + self.drop_rate)) + return stage + + def _forward_conv(self, x): + x = self.conv(x) + x = self.stage1(x) + x = self.stage2(x) + x = self.stage3(x) + x = F.relu(self.bn(x), inplace=True) + x = F.adaptive_avg_pool2d(x, output_size=1) + return x + + def forward(self, x): + x = self._forward_conv(x) + x = x.view(x.size(0), -1) + x = self.fc(x) + return x + diff --git a/data/datasets/visual_question_answering/generate_c_image/robustness-master/old/Icons-50/models/msdnet.py b/data/datasets/visual_question_answering/generate_c_image/robustness-master/old/Icons-50/models/msdnet.py new file mode 100644 index 0000000000000000000000000000000000000000..7ffa7707ec311a62c324b0746fe07d99dc7d4110 --- /dev/null +++ b/data/datasets/visual_question_answering/generate_c_image/robustness-master/old/Icons-50/models/msdnet.py @@ -0,0 +1,733 @@ +import torch +import torch.nn as nn +import math + + +class _DynamicInputDenseBlock(nn.Module): + + def __init__(self, conv_modules, debug): + super(_DynamicInputDenseBlock, self).__init__() + self.conv_modules = conv_modules + self.debug = debug + + def forward(self, x): + """ + Use the first element as raw input, and stream the rest of + the inputs through the list of modules, then apply concatenation. + expect x to be [identity, first input, second input, ..] + and len(x) - len(self.conv_modules) = 1 for identity + + :param x: Input + :return: Concatenation of the input with 1 or more module outputs + """ + if self.debug: + for i, t in enumerate(x): + print("Current input size[{}]: {}".format(i, + t.size())) + + # Init output + out = x[0] + + # Apply all given modules and return output + for calc, m in enumerate(self.conv_modules): + out = torch.cat([out, m(x[calc + 1])], 1) + + if self.debug: + print("Working on input number: %s" % calc) + print("Added: ", m(x[calc + 1]).size()) + print("Current out size {}".format(out.size())) + + return out + + +class MSDLayer(nn.Module): + + def __init__(self, in_channels, out_channels, + in_scales, out_scales, orig_scales, args): + """ + Creates a regular/transition MSDLayer. this layer uses DenseNet like concatenation on each scale, + and performs spatial reduction between scales. if input and output scales are different, than this + class creates a transition layer and the first layer (with the largest spatial size) is dropped. + + :param current_channels: number of input channels + :param in_scales: number of input scales + :param out_scales: number of output scales + :param orig_scales: number of scales in the first layer of the MSDNet + :param args: other arguments + """ + super(MSDLayer, self).__init__() + + # Init vars + self.current_channels = in_channels + self.out_channels = out_channels + self.in_scales = in_scales + self.out_scales = out_scales + self.orig_scales = orig_scales + self.args = args + self.bottleneck = args.msd_bottleneck + self.bottleneck_factor = args.msd_bottleneck_factor + self.growth_factor = self.args.msd_growth_factor + self.debug = self.args.debug + + # Define Conv2d/GCN params + self.use_gcn = args.msd_all_gcn + self.conv_l, self.ks, self.pad = get_conv_params(self.use_gcn, args) + + # Calculate number of channels to drop and number of + # all dropped channels + self.to_drop = in_scales - out_scales + self.dropped = orig_scales - out_scales # Use this as an offset + self.subnets = self.get_subnets() + + def get_subnets(self): + """ + Builds the different scales of the MSD network layer. + + :return: A list of scale modules + """ + subnets = nn.ModuleList() + + # If this is a transition layer + if self.to_drop: + # Create a reduced feature map for the first scale + # self.dropped > 0 since out_scales < in_scales < orig_scales + in_channels1 = self.current_channels *\ + self.growth_factor[self.dropped - 1] + in_channels2 = self.current_channels *\ + self.growth_factor[self.dropped] + out_channels = self.out_channels *\ + self.growth_factor[self.dropped] + bn_width1 = self.bottleneck_factor[self.dropped - 1] + bn_width2 = self.bottleneck_factor[self.dropped] + subnets.append(self.build_down_densenet(in_channels1, + in_channels2, + out_channels, + self.bottleneck, + bn_width1, + bn_width2)) + else: + # Create a normal first scale + in_channels = self.current_channels *\ + self.growth_factor[self.dropped] + out_channels = self.out_channels *\ + self.growth_factor[self.dropped] + bn_width = self.bottleneck_factor[self.dropped] + subnets.append(self.build_densenet(in_channels, + out_channels, + self.bottleneck, + bn_width)) + + + # Build second+ scales + for scale in range(1, self.out_scales): + in_channels1 = self.current_channels *\ + self.growth_factor[self.dropped + scale - 1] + in_channels2 = self.current_channels *\ + self.growth_factor[self.dropped + scale] + out_channels = self.out_channels *\ + self.growth_factor[self.dropped + scale] + bn_width1 = self.bottleneck_factor[self.dropped + scale - 1] + bn_width2 = self.bottleneck_factor[self.dropped + scale] + subnets.append(self.build_down_densenet(in_channels1, + in_channels2, + out_channels, + self.bottleneck, + bn_width1, + bn_width2)) + + return subnets + + def build_down_densenet(self, in_channels1, in_channels2, out_channels, + bottleneck, bn_width1, bn_width2): + """ + Builds a scale sub-network for scales 2 and up. + + :param in_channels1: number of same scale input channels + :param in_channels2: number of upper scale input channels + :param out_channels: number of output channels + :param bottleneck: A flag to perform a channel dimension bottleneck + :param bn_width1: The first input width of the bottleneck factor + :param bn_width2: The first input width of the bottleneck factor + :return: A scale module + """ + conv_module1 = self.convolve(in_channels1, int(out_channels/2), 'down', + bottleneck, bn_width1) + conv_module2 = self.convolve(in_channels2, int(out_channels/2), 'normal', + bottleneck, bn_width2) + conv_modules = [conv_module1, conv_module2] + return _DynamicInputDenseBlock(nn.ModuleList(conv_modules), + self.debug) + + def build_densenet(self, in_channels, out_channels, bottleneck, bn_width): + """ + Builds a scale sub-network for the first layer + + :param in_channels: number of input channels + :param out_channels: number of output channels + :param bottleneck: A flag to perform a channel dimension bottleneck + :param bn_width: The width of the bottleneck factor + :return: A scale module + """ + conv_module = self.convolve(in_channels, out_channels, 'normal', + bottleneck, bn_width) + return _DynamicInputDenseBlock(nn.ModuleList([conv_module]), + self.debug) + + def convolve(self, in_channels, out_channels, conv_type, + bottleneck, bn_width=4): + """ + Doing the main convolution of a specific scale in the + MSD network + + :param in_channels: number of input channels + :param out_channels: number of output channels + :param conv_type: convolution type + :param bottleneck: A flag to perform a channel dimension bottleneck + :param bn_width: The width of the bottleneck factor + :return: A Sequential module of the main convolution + """ + conv = nn.Sequential() + tmp_channels = in_channels + + # Bottleneck before the convolution + if bottleneck: + tmp_channels = int(min([in_channels, bn_width * out_channels])) + conv.add_module('Bottleneck_1x1', nn.Conv2d(in_channels, + tmp_channels, + kernel_size=1, + stride=1, + padding=0)) + conv.add_module('Bottleneck_BN', nn.BatchNorm2d(tmp_channels)) + conv.add_module('Bottleneck_ReLU', nn.ReLU(inplace=True)) + if conv_type == 'normal': + conv.add_module('Spatial_forward', self.conv_l(tmp_channels, + out_channels, + kernel_size=self.ks, + stride=1, + padding=self.pad)) + elif conv_type == 'down': + conv.add_module('Spatial_down', self.conv_l(tmp_channels, out_channels, + kernel_size=self.ks, + stride=2, + padding=self.pad)) + else: # Leaving an option to change the main conv type + raise NotImplementedError + + conv.add_module('BN_out', nn.BatchNorm2d(out_channels)) + conv.add_module('ReLU_out', nn.ReLU(inplace=True)) + return conv + + def forward(self, x): + cur_input = [] + outputs = [] + + # Prepare the different scales' inputs of the + # current transition/regular layer + if self.to_drop: # Transition + for scale in range(0, self.out_scales): + last_same_scale = x[self.to_drop + scale] + last_upper_scale = x[self.to_drop + scale - 1] + cur_input.append([last_same_scale, + last_upper_scale, + last_same_scale]) + else: # Regular + + # Add first scale's input + cur_input.append([x[0], x[0]]) + + # Add second+ scales' input + for scale in range(1, self.out_scales): + last_same_scale = x[scale] + last_upper_scale = x[scale - 1] + cur_input.append([last_same_scale, + last_upper_scale, + last_same_scale]) + + # Flow inputs in subnets and fill outputs + for scale in range(0, self.out_scales): + outputs.append(self.subnets[scale](cur_input[scale])) + + return outputs + + +class MSDFirstLayer(nn.Module): + + def __init__(self, in_channels, out_channels, num_scales, args): + """ + Creates the first layer of the MSD network, which takes + an input tensor (image) and generates a list of size num_scales + with deeper features with smaller (spatial) dimensions. + + :param in_channels: number of input channels to the first layer + :param out_channels: number of output channels in the first scale + :param num_scales: number of output scales in the first layer + :param args: other arguments + """ + super(MSDFirstLayer, self).__init__() + + # Init params + self.in_channels = in_channels + self.out_channels = out_channels + self.num_scales = num_scales + self.args = args + self.use_gcn = args.msd_gcn + self.conv_l, self.ks, self.pad = get_conv_params(self.use_gcn, args) + if self.use_gcn: + print('| First layer with GCN |') + else: + print('| First layer without GCN |') + + self.subnets = self.create_modules() + + def create_modules(self): + + # Create first scale features + modules = nn.ModuleList() + if 'cifar' in self.args.data: + current_channels = int(self.out_channels * + self.args.msd_growth_factor[0]) + + current_m = nn.Sequential( + self.conv_l(self.in_channels, + current_channels, kernel_size=self.ks, + stride=1, padding=self.pad), + nn.BatchNorm2d(current_channels), + nn.ReLU(inplace=True) + ) + modules.append(current_m) + else: + raise NotImplementedError + + # Create second scale features and down + for scale in range(1, self.num_scales): + + # Calculate desired output channels + out_channels = int(self.out_channels * + self.args.msd_growth_factor[scale]) + + # Use a strided convolution to create next scale features + current_m = nn.Sequential( + self.conv_l(current_channels, out_channels, + kernel_size=self.ks, + stride=2, padding=self.pad), + nn.BatchNorm2d(out_channels), + nn.ReLU(inplace=True) + ) + + # Use the output channels size for the next scale + current_channels = out_channels + + # Append module + modules.append(current_m) + + return modules + + def forward(self, x): + output = [None] * self.num_scales + current_input = x + for scale in range(0, self.num_scales): + + # Use upper scale as an input + if scale > 0: + current_input = output[scale-1] + output[scale] = self.subnets[scale](current_input) + return output + + +class Transition(nn.Sequential): + + def __init__(self, channels_in, channels_out, + out_scales, offset, growth_factor, args): + """ + Performs 1x1 convolution to increase channels size after reducing a spatial size reduction + in transition layer. + + :param channels_in: channels before the transition + :param channels_out: channels after reduction + :param out_scales: number of scales after the transition + :param offset: gap between original number of scales to out_scales + :param growth_factor: densenet channel growth factor + :return: A Parallel trainable array with the scales after channel + reduction + """ + + super(Transition, self).__init__() + self.args = args + + # Define a parallel stream for the different scales + self.scales = nn.ModuleList() + for i in range(0, out_scales): + cur_in = channels_in * growth_factor[offset + i] + cur_out = channels_out * growth_factor[offset + i] + self.scales.append(self.conv1x1(cur_in, cur_out)) + + def conv1x1(self, in_channels, out_channels): + """ + Inner function to define the basic operation + + :param in_channels: number of input channels + :param out_channels: number of output channels + :return: A Sequential module to perform 1x1 convolution + """ + scale = nn.Sequential( + nn.Conv2d(in_channels, out_channels, + kernel_size=1, stride=1, padding=0), + nn.BatchNorm2d(out_channels), + nn.ReLU(inplace=True) + ) + + return scale + + def forward(self, x): + """ + Propegate output through different scales. + + :param x: input to the transition layer + :return: list of scales' outputs + """ + if self.args.debug: + print ("In transition forward!") + + output = [] + for scale, scale_net in enumerate(self.scales): + if self.args.debug: + print ("Size of x[{}]: {}".format(scale, x[scale].size())) + print ("scale_net[0]: {}".format(scale_net[0])) + output.append(scale_net(x[scale])) + + return output + + +class CifarClassifier(nn.Module): + + def __init__(self, num_channels, num_classes): + """ + Classifier of a cifar10/100 image. + + :param num_channels: Number of input channels to the classifier + :param num_classes: Number of classes to classify + """ + + super(CifarClassifier, self).__init__() + self.inner_channels = 128 + + self.features = nn.Sequential( + nn.Conv2d(num_channels, self.inner_channels, kernel_size=3, + stride=2, padding=1), + nn.BatchNorm2d(self.inner_channels), + nn.ReLU(inplace=True), + nn.Conv2d(self.inner_channels, self.inner_channels, kernel_size=3, + stride=2, padding=1), + nn.BatchNorm2d(self.inner_channels), + nn.ReLU(inplace=True), + nn.AvgPool2d(2, 2) + ) + + self.classifier = nn.Linear(self.inner_channels, num_classes) + + def forward(self, x): + """ + Drive features to classification. + + :param x: Input of the lowest scale of the last layer of + the last block + :return: Cifar object classification result + """ + + x = self.features(x) + x = x.view(x.size(0), self.inner_channels) + x = self.classifier(x) + return x + + +class GCN(nn.Module): + + def __init__(self, in_channels, out_channels, kernel_size=7, stride=1, padding=1): + """ + Global convolutional network module implementation + + :param in_channels: number of input channels + :param out_channels: number of output channels + :param kernel_size: size of conv kernel + :param stride: stride to use in the conv parts + :param padding: padding to use in the conv parts + :param share_weights: use shared weights for every side of GCN + """ + super(GCN, self).__init__() + self.conv_l1 = nn.Conv2d(in_channels, out_channels, kernel_size=(kernel_size, 1), + padding=(padding, 0), stride=(stride, 1)) + self.conv_l2 = nn.Conv2d(out_channels, out_channels, kernel_size=(1, kernel_size), + padding=(0, padding), stride=(1, stride)) + self.conv_r1 = nn.Conv2d(in_channels, out_channels, kernel_size=(1, kernel_size), + padding=(0, padding), stride=(1, stride)) + self.conv_r2 = nn.Conv2d(out_channels, out_channels, kernel_size=(kernel_size, 1), + padding=(padding, 0), stride=(stride, 1)) + + + def forward(self, x): + + if GCN.share_weights: + + # Prepare input and state + self.conv_l1.shared = 2 + self.conv_l2.shared = 2 + xt = x.transpose(2,3) + + # Left convs + xl = self.conv_l1(x) + xl = self.conv_l2(xl) + + # Right convs + xrt = self.conv_l1(xt) + xrt = self.conv_l2(xrt) + xr = xrt.transpose(2,3) + else: + + # Left convs + xl = self.conv_l1(x) + xl = self.conv_l2(xl) + + # Right convs + xr = self.conv_r1(x) + xr = self.conv_r2(xr) + + return xl + xr + +def get_conv_params(use_gcn, args): + """ + Calculates and returns the convulotion parameters + + :param use_gcn: flag to use GCN or not + :param args: user defined arguments + :return: convolution type, kernel size and padding + """ + + if use_gcn: + GCN.share_weights = args.msd_share_weights + conv_l = GCN + ks = args.msd_gcn_kernel + else: + conv_l = nn.Conv2d + ks = args.msd_kernel + pad = int(math.floor(ks / 2)) + return conv_l, ks, pad + + +class MSDNet(nn.Module): + + def __init__(self, args): + """ + The main module for Multi Scale Dense Network. + It holds the different blocks with layers and classifiers of the MSDNet layers + + :param args: Network argument + """ + + super(MSDNet, self).__init__() + + # Init arguments + self.args = args + self.base = self.args.msd_base + self.step = self.args.msd_step + self.step_mode = self.args.msd_stepmode + self.msd_prune = self.args.msd_prune + self.num_blocks = self.args.msd_blocks + self.reduction_rate = self.args.reduction + self.growth = self.args.msd_growth + self.growth_factor = args.msd_growth_factor + self.bottleneck = self.args.msd_bottleneck + self.bottleneck_factor = args.msd_bottleneck_factor + + + # Set progress + if args.data in ['cifar10', 'cifar100']: + self.image_channels = 3 + self.num_channels = 32 + self.num_scales = 3 + self.num_classes = int(args.data.strip('cifar')) + else: + raise NotImplementedError + + # Init MultiScale graph and fill with Blocks and Classifiers + print('| MSDNet-Block {}-{}-{}'.format(self.num_blocks, + self.step, + self.args.data)) + (self.num_layers, self.steps) = self.calc_steps() + + print('Building network with the steps: {}'.format(self.steps)) + self.cur_layer = 1 + self.cur_transition_layer = 1 + self.subnets = nn.ModuleList(self.build_modules(self.num_channels)) + + # initialize + for m in self.subnets: + self.init_weights(m) + if hasattr(m,'__iter__'): + for sub_m in m: + self.init_weights(sub_m) + + def init_weights(self, m): + if isinstance(m, nn.Conv2d): + n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels + m.weight.data.normal_(0, math.sqrt(2. / n)) + elif isinstance(m, nn.BatchNorm2d): + m.weight.data.fill_(1) + m.bias.data.zero_() + elif isinstance(m, nn.Linear): + m.bias.data.zero_() + + def calc_steps(self): + """Calculates the number of layers required in each + Block and the total number of layers, according to + the step and stepmod. + + :return: number of total layers and list of layers/steps per blocks + """ + + # Init steps array + steps = [None]*self.num_blocks + steps[0] = num_layers = self.base + + # Fill steps and num_layers + for i in range(1, self.num_blocks): + + # Take even steps or calc next linear growth of a step + steps[i] = (self.step_mode == 'even' and self.step) or \ + self.step*(i-1)+1 + num_layers += steps[i] + + return num_layers, steps + + def build_modules(self, num_channels): + """Builds all blocks and classifiers and add it + into an array in the order of the format: + [[block]*num_blocks [classifier]*num_blocks] + where the i'th block corresponds to the (i+num_block) classifier. + + :param num_channels: number of input channels + :return: An array with all blocks and classifiers + """ + + # Init the blocks & classifiers data structure + modules = [None] * self.num_blocks * 2 + for i in range(0, self.num_blocks): + print ('|-----------------Block {:0>2d}----------------|'.format(i+1)) + + # Add block + modules[i], num_channels = self.create_block(num_channels, i) + + # Calculate the last scale (smallest) channels size + channels_in_last_layer = num_channels *\ + self.growth_factor[self.num_scales] + + # Add a classifier that belongs to the i'th block + modules[i + self.num_blocks] = \ + CifarClassifier(channels_in_last_layer, self.num_classes) + return modules + + def create_block(self, num_channels, block_num): + ''' + :param num_channels: number of input channels to the block + :param block_num: the number of the block (among all blocks) + :return: A sequential container with steps[block_num] MSD layers + ''' + + block = nn.Sequential() + + # Add the first layer if needed + if block_num == 0: + block.add_module('MSD_first', MSDFirstLayer(self.image_channels, + num_channels, + self.num_scales, + self.args)) + + # Add regular layers + current_channels = num_channels + for _ in range(0, self.steps[block_num]): + + # Calculate in and out scales of the layer (use paper heuristics) + if self.msd_prune == 'max': + interval = math.ceil(self.num_layers/ + self.num_scales) + in_scales = int(self.num_scales - \ + math.floor((max(0, self.cur_layer - 2))/interval)) + out_scales = int(self.num_scales - \ + math.floor((self.cur_layer - 1)/interval)) + else: + raise NotImplementedError + + self.print_layer(in_scales, out_scales) + self.cur_layer += 1 + + # Add an MSD layer + block.add_module('MSD_layer_{}'.format(self.cur_layer - 1), + MSDLayer(current_channels, + self.growth, + in_scales, + out_scales, + self.num_scales, + self.args)) + + # Increase number of channel (as in densenet pattern) + current_channels += self.growth + + # Add a transition layer if required + if (self.msd_prune == 'max' and in_scales > out_scales and + self.reduction_rate): + + # Calculate scales transition and add a Transition layer + offset = self.num_scales - out_scales + new_channels = int(math.floor(current_channels* + self.reduction_rate)) + block.add_module('Transition', Transition( + current_channels, new_channels, out_scales, + offset, self.growth_factor, self.args)) + print('| Transition layer {} was added! |'. + format(self.cur_transition_layer)) + current_channels = new_channels + + # Increment counters + self.cur_transition_layer += 1 + + elif self.msd_prune != 'max': + raise NotImplementedError + + return block, current_channels + + def print_layer(self, in_scales, out_scales): + print('| Layer {:0>2d} input scales {} output scales {} |'. + format(self.cur_layer, in_scales, out_scales)) + + def forward(self, x, progress=None): + """ + Propagate Input image in all blocks of MSD layers and classifiers + and return a list of classifications + + :param x: Input image / batch + :return: a list of classification outputs + """ + + outputs = [None] * self.num_blocks + cur_input = x + for block_num in range(0, self.num_blocks): + + # Get the current block's output + if self.args.debug: + print("") + print("Forwarding to block %s:" % str(block_num + 1)) + block = self.subnets[block_num] + cur_input = block_output = block(cur_input) + + # Classify and add current output + if self.args.debug: + print("- Getting %s block's output" % str(block_num + 1)) + for s, b in enumerate(block_output): + print("- Output size of this block's scale {}: ".format(s), + b.size()) + class_output = \ + self.subnets[block_num+self.num_blocks](block_output[-1]) + outputs[block_num] = class_output + + return outputs + + diff --git a/data/datasets/visual_question_answering/generate_c_image/robustness-master/old/Icons-50/models/resnet.py b/data/datasets/visual_question_answering/generate_c_image/robustness-master/old/Icons-50/models/resnet.py new file mode 100644 index 0000000000000000000000000000000000000000..dbbfed8680452a5d30a3751138a30feeb4470385 --- /dev/null +++ b/data/datasets/visual_question_answering/generate_c_image/robustness-master/old/Icons-50/models/resnet.py @@ -0,0 +1,199 @@ +# coding: utf-8 + +import torch +import torch.nn as nn +import torch.nn.functional as F + + +def initialize_weights(module): + if isinstance(module, nn.Conv2d): + nn.init.kaiming_normal_(module.weight.data, mode='fan_out') + elif isinstance(module, nn.BatchNorm2d): + module.weight.data.fill_(1) + module.bias.data.zero_() + elif isinstance(module, nn.Linear): + module.bias.data.zero_() + + +class BasicBlock(nn.Module): + expansion = 1 + + def __init__(self, in_channels, out_channels, stride): + super(BasicBlock, self).__init__() + + self.conv1 = nn.Conv2d( + in_channels, + out_channels, + kernel_size=3, + stride=stride, # downsample with first conv + padding=1, + bias=False) + self.bn1 = nn.BatchNorm2d(out_channels) + self.conv2 = nn.Conv2d( + out_channels, + out_channels, + kernel_size=3, + stride=1, + padding=1, + bias=False) + self.bn2 = nn.BatchNorm2d(out_channels) + + self.shortcut = nn.Sequential() + if in_channels != out_channels: + self.shortcut.add_module( + 'conv', + nn.Conv2d( + in_channels, + out_channels, + kernel_size=1, + stride=stride, # downsample + padding=0, + bias=False)) + self.shortcut.add_module('bn', nn.BatchNorm2d(out_channels)) # BN + + def forward(self, x): + y = F.relu(self.bn1(self.conv1(x)), inplace=True) + y = self.bn2(self.conv2(y)) + y += self.shortcut(x) + y = F.relu(y, inplace=True) # apply ReLU after addition + return y + + +class BottleneckBlock(nn.Module): + expansion = 4 + + def __init__(self, in_channels, out_channels, stride): + super(BottleneckBlock, self).__init__() + + bottleneck_channels = out_channels // self.expansion + + self.conv1 = nn.Conv2d( + in_channels, + bottleneck_channels, + kernel_size=1, + stride=1, + padding=0, + bias=False) + self.bn1 = nn.BatchNorm2d(bottleneck_channels) + + self.conv2 = nn.Conv2d( + bottleneck_channels, + bottleneck_channels, + kernel_size=3, + stride=stride, # downsample with 3x3 conv + padding=1, + bias=False) + self.bn2 = nn.BatchNorm2d(bottleneck_channels) + + self.conv3 = nn.Conv2d( + bottleneck_channels, + out_channels, + kernel_size=1, + stride=1, + padding=0, + bias=False) + self.bn3 = nn.BatchNorm2d(out_channels) + + self.shortcut = nn.Sequential() # identity + if in_channels != out_channels: + self.shortcut.add_module( + 'conv', + nn.Conv2d( + in_channels, + out_channels, + kernel_size=1, + stride=stride, # downsample + padding=0, + bias=False)) + self.shortcut.add_module('bn', nn.BatchNorm2d(out_channels)) # BN + + def forward(self, x): + y = F.relu(self.bn1(self.conv1(x)), inplace=True) + y = F.relu(self.bn2(self.conv2(y)), inplace=True) + y = self.bn3(self.conv3(y)) # not apply ReLU + y += self.shortcut(x) + y = F.relu(y, inplace=True) # apply ReLU after addition + return y + + +class ResNet(nn.Module): + def __init__(self, config): + super(ResNet, self).__init__() + + input_shape = config['input_shape'] + n_classes = config['n_classes'] + + base_channels = config['base_channels'] + block_type = config['block_type'] + depth = config['depth'] + + assert block_type in ['basic', 'bottleneck'] + if block_type == 'basic': + block = BasicBlock + n_blocks_per_stage = (depth - 2) // 6 + assert n_blocks_per_stage * 6 + 2 == depth + else: + block = BottleneckBlock + n_blocks_per_stage = (depth - 2) // 9 + assert n_blocks_per_stage * 9 + 2 == depth + + n_channels = [ + base_channels, + base_channels * 2 * block.expansion, + base_channels * 4 * block.expansion + ] + + self.conv = nn.Conv2d( + input_shape[1], + n_channels[0], + kernel_size=3, + stride=1, + padding=1, + bias=False) + self.bn = nn.BatchNorm2d(base_channels) + + self.stage1 = self._make_stage( + n_channels[0], n_channels[0], n_blocks_per_stage, block, stride=1) + self.stage2 = self._make_stage( + n_channels[0], n_channels[1], n_blocks_per_stage, block, stride=2) + self.stage3 = self._make_stage( + n_channels[1], n_channels[2], n_blocks_per_stage, block, stride=2) + + # compute conv feature size + self.feature_size = self._forward_conv( + torch.zeros(*input_shape)).view(-1).shape[0] + + self.fc = nn.Linear(self.feature_size, n_classes) + + # initialize weights + self.apply(initialize_weights) + + def _make_stage(self, in_channels, out_channels, n_blocks, block, stride): + stage = nn.Sequential() + for index in range(n_blocks): + block_name = 'block{}'.format(index + 1) + if index == 0: + stage.add_module(block_name, + block(in_channels, + out_channels, + stride=stride)) + else: + stage.add_module(block_name, + block(out_channels, + out_channels, + stride=1)) + return stage + + def _forward_conv(self, x): + x = F.relu(self.bn(self.conv(x)), inplace=True) + x = self.stage1(x) + x = self.stage2(x) + x = self.stage3(x) + x = F.adaptive_avg_pool2d(x, output_size=1) + return x + + def forward(self, x): + x = self._forward_conv(x) + x = x.view(x.size(0), -1) + x = self.fc(x) + return x diff --git a/data/datasets/visual_question_answering/generate_c_image/robustness-master/old/Icons-50/models/resnext.py b/data/datasets/visual_question_answering/generate_c_image/robustness-master/old/Icons-50/models/resnext.py new file mode 100644 index 0000000000000000000000000000000000000000..705dd9a5eb9566342a9e7850c42f39671a7bbb62 --- /dev/null +++ b/data/datasets/visual_question_answering/generate_c_image/robustness-master/old/Icons-50/models/resnext.py @@ -0,0 +1,158 @@ +# coding: utf-8 + +import torch +import torch.nn as nn +import torch.nn.functional as F + + +def initialize_weights(m): + if isinstance(m, nn.Conv2d): + nn.init.kaiming_normal_(m.weight.data, mode='fan_out') + elif isinstance(m, nn.BatchNorm2d): + m.weight.data.fill_(1) + m.bias.data.zero_() + elif isinstance(m, nn.Linear): + m.bias.data.zero_() + + +class BottleneckBlock(nn.Module): + expansion = 4 + + def __init__(self, in_channels, out_channels, stride, cardinality): + super(BottleneckBlock, self).__init__() + + bottleneck_channels = cardinality * out_channels // self.expansion + + self.conv1 = nn.Conv2d( + in_channels, + bottleneck_channels, + kernel_size=1, + stride=1, + padding=0, + bias=False) + self.bn1 = nn.BatchNorm2d(bottleneck_channels) + + self.conv2 = nn.Conv2d( + bottleneck_channels, + bottleneck_channels, + kernel_size=3, + stride=stride, # downsample with 3x3 conv + padding=1, + groups=cardinality, + bias=False) + self.bn2 = nn.BatchNorm2d(bottleneck_channels) + + self.conv3 = nn.Conv2d( + bottleneck_channels, + out_channels, + kernel_size=1, + stride=1, + padding=0, + bias=False) + self.bn3 = nn.BatchNorm2d(out_channels) + + self.shortcut = nn.Sequential() # identity + if in_channels != out_channels: + self.shortcut.add_module( + 'conv', + nn.Conv2d( + in_channels, + out_channels, + kernel_size=1, + stride=stride, # downsample + padding=0, + bias=False)) + self.shortcut.add_module('bn', nn.BatchNorm2d(out_channels)) # BN + + def forward(self, x): + y = F.relu(self.bn1(self.conv1(x)), inplace=True) + y = F.relu(self.bn2(self.conv2(y)), inplace=True) + y = self.bn3(self.conv3(y)) # not apply ReLU + y += self.shortcut(x) + y = F.relu(y, inplace=True) # apply ReLU after addition + return y + + +class ResNeXt(nn.Module): + def __init__(self, config): + super(ResNeXt, self).__init__() + + input_shape = config['input_shape'] + n_classes = config['n_classes'] + + base_channels = config['base_channels'] + depth = config['depth'] + self.cardinality = config['cardinality'] + + n_blocks_per_stage = (depth - 2) // 9 + assert n_blocks_per_stage * 9 + 2 == depth + block = BottleneckBlock + + n_channels = [ + base_channels, + base_channels * block.expansion, + base_channels * 2 * block.expansion, + base_channels * 4 * block.expansion + ] + + self.conv = nn.Conv2d( + input_shape[1], + n_channels[0], + kernel_size=3, + stride=1, + padding=1, + bias=False) + self.bn = nn.BatchNorm2d(n_channels[0]) + + self.stage1 = self._make_stage( + n_channels[0], n_channels[1], n_blocks_per_stage, stride=1) + self.stage2 = self._make_stage( + n_channels[1], n_channels[2], n_blocks_per_stage, stride=2) + self.stage3 = self._make_stage( + n_channels[2], n_channels[3], n_blocks_per_stage, stride=2) + + with torch.no_grad(): + # compute conv feature size + self.feature_size = self._forward_conv( + torch.zeros(*input_shape)).view(-1).shape[0] + + self.fc = nn.Linear(self.feature_size, n_classes) + + # initialize weights + self.apply(initialize_weights) + + def _make_stage(self, in_channels, out_channels, n_blocks, stride): + stage = nn.Sequential() + for index in range(n_blocks): + block_name = 'block{}'.format(index + 1) + if index == 0: + stage.add_module( + block_name, + BottleneckBlock( + in_channels, + out_channels, + stride, # downsample + self.cardinality)) + else: + stage.add_module( + block_name, + BottleneckBlock( + out_channels, + out_channels, + 1, # no downsampling + self.cardinality)) + return stage + + def _forward_conv(self, x): + x = F.relu(self.bn(self.conv(x)), inplace=True) + x = self.stage1(x) + x = self.stage2(x) + x = self.stage3(x) + x = F.adaptive_avg_pool2d(x, output_size=1) + return x + + def forward(self, x): + x = self._forward_conv(x) + x = x.view(x.size(0), -1) + x = self.fc(x) + return x diff --git a/data/datasets/visual_question_answering/generate_c_image/robustness-master/old/Icons-50/models/shake_shake.py b/data/datasets/visual_question_answering/generate_c_image/robustness-master/old/Icons-50/models/shake_shake.py new file mode 100644 index 0000000000000000000000000000000000000000..a62f61597ef1a90aed17e2e2a1119798dc32030a --- /dev/null +++ b/data/datasets/visual_question_answering/generate_c_image/robustness-master/old/Icons-50/models/shake_shake.py @@ -0,0 +1,234 @@ +# coding: utf-8 + +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.autograd import Function + + +class ShakeFunction(Function): + @staticmethod + def forward(ctx, x1, x2, alpha, beta): + ctx.save_for_backward(x1, x2, alpha, beta) + + y = x1 * alpha.data + x2 * (1 - alpha.data) + return y + + @staticmethod + def backward(ctx, grad_output): + x1, x2, alpha, beta = ctx.saved_variables + grad_x1 = grad_x2 = grad_alpha = grad_beta = None + + if ctx.needs_input_grad[0]: + grad_x1 = grad_output * beta + if ctx.needs_input_grad[1]: + grad_x2 = grad_output * (1 - beta) + + return grad_x1, grad_x2, grad_alpha, grad_beta + + +shake_function = ShakeFunction.apply + + +def get_alpha_beta(batch_size, shake_config, is_cuda): + forward_shake, backward_shake, shake_image = shake_config + + if forward_shake and not shake_image: + alpha = torch.rand(1) + elif forward_shake and shake_image: + alpha = torch.rand(batch_size).view(batch_size, 1, 1, 1) + else: + alpha = torch.tensor(0.5) + + if backward_shake and not shake_image: + beta = torch.rand(1) + elif backward_shake and shake_image: + beta = torch.rand(batch_size).view(batch_size, 1, 1, 1) + else: + beta = torch.tensor(0.5) + + if is_cuda: + alpha, beta = alpha.cuda(), beta.cuda() + + return alpha, beta + + +def initialize_weights(module): + if isinstance(module, nn.Conv2d): + nn.init.kaiming_normal(module.weight.data, mode='fan_out') + elif isinstance(module, nn.BatchNorm2d): + module.weight.data.fill_(1) + module.bias.data.zero_() + elif isinstance(module, nn.Linear): + module.bias.data.zero_() + + +class ResidualPath(nn.Module): + def __init__(self, in_channels, out_channels, stride): + super(ResidualPath, self).__init__() + + self.conv1 = nn.Conv2d( + in_channels, + out_channels, + kernel_size=3, + stride=stride, + padding=1, + bias=False, + ) + self.bn1 = nn.BatchNorm2d(out_channels) + self.conv2 = nn.Conv2d( + out_channels, + out_channels, + kernel_size=3, + stride=1, + padding=1, + bias=False) + self.bn2 = nn.BatchNorm2d(out_channels) + + def forward(self, x): + x = F.relu(x, inplace=False) + x = F.relu(self.bn1(self.conv1(x)), inplace=False) + x = self.bn2(self.conv2(x)) + return x + + +class DownsamplingShortcut(nn.Module): + def __init__(self, in_channels): + super(DownsamplingShortcut, self).__init__() + self.conv1 = nn.Conv2d( + in_channels, + in_channels, + kernel_size=1, + stride=1, + padding=0, + bias=False) + self.conv2 = nn.Conv2d( + in_channels, + in_channels, + kernel_size=1, + stride=1, + padding=0, + bias=False) + self.bn = nn.BatchNorm2d(in_channels * 2) + + def forward(self, x): + x = F.relu(x, inplace=False) + y1 = F.avg_pool2d(x, kernel_size=1, stride=2, padding=0) + y1 = self.conv1(y1) + + y2 = F.pad(x[:, :, 1:, 1:], (0, 1, 0, 1)) + y2 = F.avg_pool2d(y2, kernel_size=1, stride=2, padding=0) + y2 = self.conv2(y2) + + z = torch.cat([y1, y2], dim=1) + z = self.bn(z) + + return z + + +class BasicBlock(nn.Module): + def __init__(self, in_channels, out_channels, stride, shake_config): + super(BasicBlock, self).__init__() + + self.shake_config = shake_config + + self.residual_path1 = ResidualPath(in_channels, out_channels, stride) + self.residual_path2 = ResidualPath(in_channels, out_channels, stride) + + self.shortcut = nn.Sequential() + if in_channels != out_channels: + self.shortcut.add_module('downsample', + DownsamplingShortcut(in_channels)) + + def forward(self, x): + x1 = self.residual_path1(x) + x2 = self.residual_path2(x) + + if self.training: + shake_config = self.shake_config + else: + shake_config = (False, False, False) + + alpha, beta = get_alpha_beta(x.size(0), shake_config, x.is_cuda) + y = shake_function(x1, x2, alpha, beta) + + return self.shortcut(x) + y + + +class ResNeXt(nn.Module): + def __init__(self, config): + super(ResNeXt, self).__init__() + + input_shape = config['input_shape'] + n_classes = config['n_classes'] + + base_channels = config['base_channels'] + depth = config['depth'] + self.shake_config = (config['shake_forward'], config['shake_backward'], + config['shake_image']) + + block = BasicBlock + n_blocks_per_stage = (depth - 2) // 6 + assert n_blocks_per_stage * 6 + 2 == depth + + n_channels = [base_channels, base_channels * 2, base_channels * 4] + + self.conv = nn.Conv2d( + input_shape[1], + n_channels[0], + kernel_size=3, + stride=1, + padding=1, + bias=False) + self.bn = nn.BatchNorm2d(base_channels) + + self.stage1 = self._make_stage( + n_channels[0], n_channels[0], n_blocks_per_stage, block, stride=1) + self.stage2 = self._make_stage( + n_channels[0], n_channels[1], n_blocks_per_stage, block, stride=2) + self.stage3 = self._make_stage( + n_channels[1], n_channels[2], n_blocks_per_stage, block, stride=2) + + # compute conv feature size + self.feature_size = self._forward_conv( + torch.zeros(*input_shape)).view(-1).shape[0] + + self.fc = nn.Linear(self.feature_size, n_classes) + + # initialize weights + self.apply(initialize_weights) + + def _make_stage(self, in_channels, out_channels, n_blocks, block, stride): + stage = nn.Sequential() + for index in range(n_blocks): + block_name = 'block{}'.format(index + 1) + if index == 0: + stage.add_module(block_name, + block( + in_channels, + out_channels, + stride=stride, + shake_config=self.shake_config)) + else: + stage.add_module(block_name, + block( + out_channels, + out_channels, + stride=1, + shake_config=self.shake_config)) + return stage + + def _forward_conv(self, x): + x = F.relu(self.bn(self.conv(x)), inplace=True) + x = self.stage1(x) + x = self.stage2(x) + x = self.stage3(x) + x = F.adaptive_avg_pool2d(x, output_size=1) + return x + + def forward(self, x): + x = self._forward_conv(x) + x = x.view(x.size(0), -1) + x = self.fc(x) + return x + diff --git a/data/datasets/visual_question_answering/generate_c_image/robustness-master/old/Icons-50/models/wrn.py b/data/datasets/visual_question_answering/generate_c_image/robustness-master/old/Icons-50/models/wrn.py new file mode 100644 index 0000000000000000000000000000000000000000..58c09e169d881d82a8ccbb87944ff4df02150a1c --- /dev/null +++ b/data/datasets/visual_question_answering/generate_c_image/robustness-master/old/Icons-50/models/wrn.py @@ -0,0 +1,96 @@ +import math +import torch +import torch.nn as nn +import torch.nn.functional as F + + +class BasicBlock(nn.Module): + def __init__(self, in_planes, out_planes, stride, dropRate=0.0): + super(BasicBlock, self).__init__() + self.bn1 = nn.BatchNorm2d(in_planes) + self.relu1 = nn.ReLU(inplace=True) + self.conv1 = nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride, + padding=1, bias=False) + self.bn2 = nn.BatchNorm2d(out_planes) + self.relu2 = nn.ReLU(inplace=True) + self.conv2 = nn.Conv2d(out_planes, out_planes, kernel_size=3, stride=1, + padding=1, bias=False) + self.droprate = dropRate + self.equalInOut = (in_planes == out_planes) + self.convShortcut = (not self.equalInOut) and nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride, + padding=0, bias=False) or None + + def forward(self, x): + if not self.equalInOut: + x = self.relu1(self.bn1(x)) + else: + out = self.relu1(self.bn1(x)) + if self.equalInOut: + out = self.relu2(self.bn2(self.conv1(out))) + else: + out = self.relu2(self.bn2(self.conv1(x))) + if self.droprate > 0: + out = F.dropout(out, p=self.droprate, training=self.training) + out = self.conv2(out) + if not self.equalInOut: + return torch.add(self.convShortcut(x), out) + else: + return torch.add(x, out) + + +class NetworkBlock(nn.Module): + def __init__(self, nb_layers, in_planes, out_planes, block, stride, dropRate=0.0): + super(NetworkBlock, self).__init__() + self.layer = self._make_layer(block, in_planes, out_planes, nb_layers, stride, dropRate) + + def _make_layer(self, block, in_planes, out_planes, nb_layers, stride, dropRate): + layers = [] + for i in range(nb_layers): + layers.append(block(i == 0 and in_planes or out_planes, out_planes, i == 0 and stride or 1, dropRate)) + return nn.Sequential(*layers) + + def forward(self, x): + return self.layer(x) + + +class WideResNet(nn.Module): + def __init__(self, depth, num_classes, widen_factor=1, dropRate=0.0): + super(WideResNet, self).__init__() + nChannels = [16, 16 * widen_factor, 32 * widen_factor, 64 * widen_factor] + assert ((depth - 4) % 6 == 0) + n = (depth - 4) // 6 + block = BasicBlock + # 1st conv before any network block + self.conv1 = nn.Conv2d(3, nChannels[0], kernel_size=3, stride=1, + padding=1, bias=False) + # 1st block + self.block1 = NetworkBlock(n, nChannels[0], nChannels[1], block, 1, dropRate) + # 2nd block + self.block2 = NetworkBlock(n, nChannels[1], nChannels[2], block, 2, dropRate) + # 3rd block + self.block3 = NetworkBlock(n, nChannels[2], nChannels[3], block, 2, dropRate) + # global average pooling and classifier + self.bn1 = nn.BatchNorm2d(nChannels[3]) + self.relu = nn.ReLU(inplace=True) + self.fc = nn.Linear(nChannels[3], num_classes) + self.nChannels = nChannels[3] + + for m in self.modules(): + if isinstance(m, nn.Conv2d): + n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels + m.weight.data.normal_(0, math.sqrt(2. / n)) + elif isinstance(m, nn.BatchNorm2d): + m.weight.data.fill_(1) + m.bias.data.zero_() + elif isinstance(m, nn.Linear): + m.bias.data.zero_() + + def forward(self, x): + out = self.conv1(x) + out = self.block1(out) + out = self.block2(out) + out = self.block3(out) + out = self.relu(self.bn1(out)) + out = F.avg_pool2d(out, 8) + out = out.view(-1, self.nChannels) + return self.fc(out) diff --git a/data/datasets/visual_question_answering/generate_c_image/robustness-master/old/Icons-50/train.py b/data/datasets/visual_question_answering/generate_c_image/robustness-master/old/Icons-50/train.py new file mode 100644 index 0000000000000000000000000000000000000000..865c83c2b91424b0f444b482014ed2dca23e76fd --- /dev/null +++ b/data/datasets/visual_question_answering/generate_c_image/robustness-master/old/Icons-50/train.py @@ -0,0 +1,341 @@ +# -*- coding: utf-8 -*- +import os +import numpy as np +import argparse +import math +import time +import torch +import torch.backends.cudnn as cudnn +import torchvision.transforms as trn +import torchvision.datasets as dset +import torch.nn.functional as F +from models.augment import RandomErasing + +parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) +parser.add_argument('--test-style', type=str, default='microsoft') +parser.add_argument('--traditional', action='store_true', help='Test classification performance not robustness.') +parser.add_argument('--subtype', action='store_true', help='Test subtype robustness.') +parser.add_argument('--c100', action='store_true', help='Test classification performance on CIFAR-100 not robustness.') +# Optimization options +parser.add_argument('--epochs', '-e', type=int, default=50, help='Number of epochs to train.') +parser.add_argument('--learning_rate', '-lr', type=float, default=0.1, help='The initial learning rate.') +parser.add_argument('--batch_size', '-b', type=int, default=128, help='Batch size.') +parser.add_argument('--valid_size', '-v', type=int, default=0, help='Number of validation examples to hold out.') +parser.add_argument('--test_bs', type=int, default=250) +parser.add_argument('--momentum', type=float, default=0.9, help='Momentum.') +parser.add_argument('--decay', '-d', type=float, default=0.0001, help='Weight decay (L2 penalty).') +# Checkpoints +parser.add_argument('--save', '-s', type=str, default='./snapshots', help='Folder to save checkpoints.') +parser.add_argument('--load', '-l', type=str, default='', help='Checkpoint path to resume / test.') +parser.add_argument('--test', '-t', action='store_true', help='Test only flag.') +# Architecture +parser.add_argument('--model', default='resnext', type=str) +# Acceleration +parser.add_argument('--ngpu', type=int, default=1, help='0 = CPU.') +parser.add_argument('--prefetch', type=int, default=2, help='Pre-fetching threads.') +args = parser.parse_args() +args.dataset = 'icons' + +state = {k: v for k, v in args._get_kwargs()} + +print(state) + +# set seeds +torch.manual_seed(1) +np.random.seed(1) +torch.cuda.manual_seed(1) + + +# /////////////// Dataset Loading /////////////// + +if args.c100: + # mean and standard deviation of channels of CIFAR-10 images + mean = [x / 255 for x in [125.3, 123.0, 113.9]] + std = [x / 255 for x in [63.0, 62.1, 66.7]] + + train_transform = trn.Compose([trn.RandomHorizontalFlip(), trn.RandomCrop(32, padding=4), + trn.ToTensor(), trn.Normalize(mean, std)]) + test_transform = trn.Compose([trn.ToTensor(), trn.Normalize(mean, std)]) + + train_data = dset.CIFAR100('/share/data/vision-greg/cifarpy', train=True, transform=train_transform, download=False) + test_data = dset.CIFAR100('/share/data/vision-greg/cifarpy', train=False, transform=test_transform, download=False) + num_classes = 100 + +else: + train_data = dset.ImageFolder('/share/data/vision-greg/DistortedImageNet/Icons-50', + transform=trn.Compose([trn.Resize((32, 32)), trn.RandomHorizontalFlip(), + trn.RandomCrop(32, padding=4), trn.ToTensor(), + # RandomErasing() + ])) + test_data = dset.ImageFolder('/share/data/vision-greg/DistortedImageNet/Icons-50', + transform=trn.Compose([trn.Resize((32, 32)), trn.ToTensor()])) + num_classes = 50 + + if args.traditional: + filtered_imgs = [] + for img in train_data.samples: + img_name = img[0] + if '_2' not in img_name: + filtered_imgs.append(img) + + train_data.samples = filtered_imgs[:] + + filtered_imgs = [] + for img in test_data.samples: + img_name = img[0] + if '_2' in img_name: + filtered_imgs.append(img) + + test_data.samples = filtered_imgs[:] + + elif args.subtype: + test_subclasses = ( + "small_airplane", "top_with_upwards_arrow_above", "soccer_ball", "duck", "hatching_chick", "crossed_swords", + "passenger_ship", "ledger", "books", "derelict_house_building", "convenience_store", "rabbit_face", + "cartwheel_type_6", "mantelpiece_clock", "watch", "sun_behind_cloud_with_rain", "wine_glass", + "face_throwing_a_kiss", "e_mail_symbol", "family_man_boy", "family_man_girl", "family_man_boy_boy", + "family_man_girl_boy", "family_man_girl_girl", "monorail", "leopard", "chequered_flag", "tulip", + "womans_sandal", + "victory_hand", "womans_hat", "broken_heart", "unified_ideograph_5408", "circled_ideograph_accept", + "closed_lock_with_key", "open_mailbox_with_lowered_flag", "shark", "military_medal", + "banknote_with_dollar_sign", + "monkey", "crescent_moon", "mount_fuji", "mobile_phone_off", "no_smoking_symbol", "glowing_star", + "evergreen_tree", + "umbrella_with_rain_drops", "racing_car", "factory_worker", "pencil" + ) + + filtered_imgs = [] + for img in train_data.samples: + img_name = img[0] + in_test_subclass = False + for subclass in test_subclasses: + if subclass in img_name: + in_test_subclass = True + break + if in_test_subclass is False: + filtered_imgs.append(img) + + train_data.samples = filtered_imgs[:] + + filtered_imgs = [] + for img in test_data.samples: + img_name = img[0] + in_test_subclass = False + for subclass in test_subclasses: + if subclass in img_name: + in_test_subclass = True + break + if in_test_subclass is True: + filtered_imgs.append(img) + + test_data.samples = filtered_imgs[:] + + else: + filtered_imgs = [] + for img in train_data.samples: + img_name = img[0] + if args.test_style not in img_name: + filtered_imgs.append(img) + + train_data.samples = filtered_imgs[:] + + filtered_imgs = [] + for img in test_data.samples: + img_name = img[0] + if args.test_style in img_name: + filtered_imgs.append(img) + + test_data.samples = filtered_imgs[:] + +train_loader = torch.utils.data.DataLoader( + train_data, batch_size=args.batch_size, shuffle=True, + num_workers=args.prefetch, pin_memory=True) +test_loader = torch.utils.data.DataLoader( + test_data, batch_size=args.test_bs, shuffle=False, + num_workers=args.prefetch, pin_memory=True) + +# /////////////// Model Setup /////////////// + +# Create model + +if args.model == 'resnext': + from models.resnext import ResNeXt + net = ResNeXt({'input_shape': (1,3,32,32), 'n_classes': num_classes, + 'base_channels': 32, 'depth': 29, 'cardinality': 8}) +if 'shake' in args.model: + from models.shake_shake import ResNeXt + net = ResNeXt({'input_shape': (1,3,32,32), 'n_classes': num_classes, + 'base_channels': 96, 'depth': 26, "shake_forward": True, + "shake_backward": True, "shake_image": True}) + args.epochs = 500 + print('Overwriting epochs parameter; now the value is', args.epochs) +elif args.model == 'wrn' or 'wide' in args.model: + from models.wrn import WideResNet + net = WideResNet(16, num_classes, 4, dropRate=0.3) + # args.decay = 5e-4 + # print('Overwriting decay parameter; now the value is', args.decay) +elif args.model == 'resnet': + from models.resnet import ResNet + net = ResNet({'input_shape': (1,3,32,32), 'n_classes': num_classes, + 'base_channels': 16, 'block_type': 'basic', 'depth': 20}) +elif args.model == 'densenet': + from models.densenet import DenseNet + net = DenseNet({'input_shape': (1,3,32,32), 'n_classes': num_classes, + "depth": 40, "block_type": "bottleneck", "growth_rate": 24, + "drop_rate": 0.0, "compression_rate": 1}) # 1 is turns compression off + +start_epoch = 0 +# Restore model +if args.load != '': + for i in range(1000 - 1, -1, -1): + model_name = os.path.join(args.load, args.dataset + '_epoch_' + str(i) + '.pt') + if os.path.isfile(model_name): + net.load_state_dict(torch.load(model_name)) + print('Model restored! Epoch:', i) + start_epoch = i + 1 + break + if start_epoch == 0: + assert False, "could not resume" + + +if args.ngpu > 1: + net = torch.nn.DataParallel(net, device_ids=list(range(args.ngpu))) + +if args.ngpu > 0: + net.cuda() + torch.cuda.manual_seed(1) + +cudnn.benchmark = True # fire on all cylinders + +optimizer = torch.optim.SGD(net.parameters(), + state['learning_rate'], momentum=state['momentum'], + weight_decay=state['decay'], nesterov=True) + + +def cosine_annealing(step, total_steps, lr_max, lr_min): + return lr_min + (lr_max - lr_min) * 0.5 * ( + 1 + np.cos(step / total_steps * np.pi)) + + +scheduler = torch.optim.lr_scheduler.LambdaLR( + optimizer, + lr_lambda=lambda step: cosine_annealing( + step, + args.epochs * len(train_loader), + 1, # since lr_lambda computes multiplicative factor + 1e-6 / args.learning_rate)) + + +# /////////////// Training /////////////// + +from tqdm import tqdm + + +def train(): + net.train() # enter train mode + loss_avg = 0.0 + for batch_idx, (data, target) in enumerate(train_loader): + + data, target = data.requires_grad_().cuda(), target.requires_grad_().cuda() + + # forward + x = net(data) + + # backward + scheduler.step() + optimizer.zero_grad() + loss = F.cross_entropy(x, target) + loss.backward() + optimizer.step() + + # exponential moving average + loss_avg = loss_avg * 0.8 + float(loss.data) * 0.2 + + state['train_loss'] = loss_avg + + +# test function +def test(): + net.eval() + loss_avg = 0.0 + correct = 0 + with torch.no_grad(): + for batch_idx, (data, target) in enumerate(test_loader): + data, target = data.cuda(), target.cuda() + + # forward + output = net(data) + loss = F.cross_entropy(output, target) + + # accuracy + pred = output.data.max(1)[1] + correct += pred.eq(target.data).sum().item() + + # test loss average + loss_avg += float(loss.data) + + state['test_loss'] = loss_avg / len(test_loader) + state['test_accuracy'] = correct / len(test_loader.dataset) + + +if args.test: + test() + print(state) + exit() + + +# Make save directory +if not os.path.exists(args.save): + os.makedirs(args.save) +if not os.path.isdir(args.save): + raise Exception('%s is not a dir' % args.save) + +experiment_indicator = '' +if args.traditional: + experiment_indicator = '_tradition' +elif args.c100: + experiment_indicator = '_c100' +elif args.subtype: + experiment_indicator = '_subtype' + +with open(os.path.join(args.save, args.model + experiment_indicator + '_results.csv'), 'w') as f: + f.write('epoch,time(s),train_loss,test_loss,test_error(%)\n') + + +print('Beginning Training\n') + +# Main loop +for epoch in range(start_epoch, args.epochs): + state['epoch'] = epoch + + begin_epoch = time.time() + + train() + test() + + # # Save model + # torch.save(net.state_dict(), + # os.path.join(args.save, args.dataset + '_epoch_' + str(epoch) + '.pt')) + # # Let us not waste space and delete the previous model + # try: os.remove(os.path.join(args.save, args.dataset + '_epoch_' + str(epoch - 1) + '.pt')) + # except: True + + # Show results + + with open(os.path.join(args.save, args.model + experiment_indicator + '_results.csv'), 'a') as f: + f.write('%03d,%05d,%0.6f,%0.5f,%0.2f,\n' % ( + (epoch + 1), + time.time() - begin_epoch, + state['train_loss'], + state['test_loss'], + 100 - 100. * state['test_accuracy'], + )) + + print('Epoch {0:3d} | Time {1:5d} | Train Loss {2:.4f} | Test Loss {3:.3f} | Test Error {4:.2f}'.format( + (epoch + 1), + int(time.time() - begin_epoch), + state['train_loss'], + state['test_loss'], + 100 - 100. * state['test_accuracy']) + ) + diff --git a/data/datasets/visual_question_answering/generate_c_image/robustness-master/old/Icons-50/train.sh b/data/datasets/visual_question_answering/generate_c_image/robustness-master/old/Icons-50/train.sh new file mode 100644 index 0000000000000000000000000000000000000000..4d5cb75d08bd9bdf53a02acb98f39dc1150fad40 --- /dev/null +++ b/data/datasets/visual_question_answering/generate_c_image/robustness-master/old/Icons-50/train.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +source ~/new_begin.sh + +python train.py --model $1 --subtype #--traditional # --c100 + diff --git a/data/datasets/visual_question_answering/generate_c_image/robustness-master/old/auxiliary/CIFAR100/train.py b/data/datasets/visual_question_answering/generate_c_image/robustness-master/old/auxiliary/CIFAR100/train.py new file mode 100644 index 0000000000000000000000000000000000000000..9fd012c22160f76052b94fc02ac4080cd9696a98 --- /dev/null +++ b/data/datasets/visual_question_answering/generate_c_image/robustness-master/old/auxiliary/CIFAR100/train.py @@ -0,0 +1,275 @@ +# -*- coding: utf-8 -*- +import numpy as np +import sys +import os +import pickle +import argparse +import math +import time +import torch +import torch.nn as nn +import torch.backends.cudnn as cudnn +import torchvision.transforms as trn +import torchvision.datasets as dset +import torch.nn.functional as F +from torch.autograd import Variable as V +from cifar_resnet import WideResNet + +parser = argparse.ArgumentParser(description='Trains a CIFAR-100 Classifier', + formatter_class=argparse.ArgumentDefaultsHelpFormatter) +# Optimization options +parser.add_argument('--epochs', '-e', type=int, default=100, help='Number of epochs to train.') +parser.add_argument('--learning_rate', '-lr', type=float, default=0.1, help='The initial learning rate.') +parser.add_argument('--batch_size', '-b', type=int, default=128, help='Batch size.') +parser.add_argument('--test_bs', type=int, default=200) +parser.add_argument('--momentum', '-m', type=float, default=0.9, help='Momentum.') +parser.add_argument('--decay', '-d', type=float, default=0.0005, help='Weight decay (L2 penalty).') +# Checkpoints +parser.add_argument('--save', '-s', type=str, default='./snapshots', help='Folder to save checkpoints.') +parser.add_argument('--load', '-l', type=str, default='', help='Checkpoint path to resume / test.') +parser.add_argument('--test', '-t', action='store_true', help='Test only flag.') +# Architecture +parser.add_argument('--layers', default=40, type=int, help='total number of layers (default: 28)') +parser.add_argument('--widen-factor', default=2, type=int, help='widen factor (default: 10)') +parser.add_argument('--droprate', default=0.3, type=float, help='dropout probability (default: 0.0)') +# Acceleration +parser.add_argument('--ngpu', type=int, default=1, help='0 = CPU.') +parser.add_argument('--prefetch', type=int, default=4, help='Pre-fetching threads.') +args = parser.parse_args() +args.dataset = 'cifar100' + +torch.manual_seed(1) +np.random.seed(1) + +state = {k: v for k, v in args._get_kwargs()} +state['tt'] = 0 # SGDR variable +state['init_learning_rate'] = args.learning_rate + + +# mean and standard deviation of channels of CIFAR-10 images +mean = [x / 255 for x in [125.3, 123.0, 113.9]] +std = [x / 255 for x in [63.0, 62.1, 66.7]] + +train_transform = trn.Compose([trn.RandomHorizontalFlip(), trn.RandomCrop(32, padding=4), + trn.ToTensor(), trn.Normalize(mean, std)]) +test_transform = trn.Compose([trn.ToTensor(), trn.Normalize(mean, std)]) + + +train_data = dset.CIFAR100('/share/data/vision-greg/cifarpy', train=True, transform=train_transform, download=False) +test_data = dset.CIFAR100('/share/data/vision-greg/cifarpy', train=False, transform=test_transform, download=False) +test_data_out = dset.CIFAR100('/share/data/vision-greg/cifarpy', train=False, transform=test_transform, download=False) +num_classes = 20 + +# d = dict.fromkeys([i for i in range(20)]) +# for i in range(len(coarse)): +# if d[coarse[i]] is None: d[coarse[i]] = [] +# if fine[i] not in d[coarse[i]]: +# d[coarse[i]].append(fine[i]) + +coarse_to_fine =\ + {0: [72, 4, 95, 30, 55], 1: [73, 32, 67, 91, 1], 2: [92, 70, 82, 54, 62], + 3: [16, 61, 9, 10, 28], 4: [51, 0, 53, 57, 83], 5: [40, 39, 22, 87, 86], + 6: [20, 25, 94, 84, 5], 7: [14, 24, 6, 7, 18], 8: [43, 97, 42, 3, 88], + 9: [37, 17, 76, 12, 68], 10: [49, 33, 71, 23, 60], 11: [15, 21, 19, 31, 38], + 12: [75, 63, 66, 64, 34], 13: [77, 26, 45, 99, 79], 14: [11, 2, 35, 46, 98], + 15: [29, 93, 27, 78, 44], 16: [65, 50, 74, 36, 80], 17: [56, 52, 47, 59, 96], + 18: [8, 58, 90, 13, 48], 19: [81, 69, 41, 89, 85]} + +# {v: k for k, v in coarse_to_fine.items()} +fine_to_coarse = dict((v,k) for k in coarse_to_fine for v in coarse_to_fine[k]) + +train_in_data = [] +train_in_labels = [] +for i in range(len(train_data)): + fine = train_data.train_labels[i] + + if coarse_to_fine[fine_to_coarse[fine]].index(fine) > 0: # 0, 1, 2, 3 + train_in_data.append(train_data.train_data[i]) + train_in_labels.append(fine_to_coarse[fine]) + +train_in_data = np.array(train_in_data) +train_data.train_data = train_in_data +train_data.train_labels = train_in_labels + + +test_in_data = [] +test_in_labels = [] +test_out_data = [] +test_out_labels = [] +for i in range(len(test_data)): + fine = test_data.test_labels[i] + + if coarse_to_fine[fine_to_coarse[fine]].index(fine) > 0: + test_in_data.append(test_data.test_data[i]) + test_in_labels.append(fine_to_coarse[fine]) + else: + test_out_data.append(test_data.test_data[i]) + test_out_labels.append(fine_to_coarse[fine]) + +test_in_data = np.array(test_in_data) +test_data.test_data = test_in_data +test_data.test_labels = test_in_labels + +test_out_data = np.array(test_out_data) +test_data_out.test_data = test_out_data +test_data_out.test_labels = test_out_labels + + +train_loader = torch.utils.data.DataLoader( + train_data, batch_size=args.batch_size, shuffle=True, + num_workers=args.prefetch, pin_memory=True) +test_loader = torch.utils.data.DataLoader( + test_data, batch_size=args.test_bs, shuffle=False, + num_workers=args.prefetch, pin_memory=True) +test_out_loader = torch.utils.data.DataLoader( + test_data_out, batch_size=args.test_bs, shuffle=False, + num_workers=args.prefetch, pin_memory=True) + +print('Number of in examples:',len(test_loader.dataset)) +print('Number of out examples:', len(test_out_loader.dataset)) + +# Create model +net = WideResNet(args.layers, num_classes, args.widen_factor, dropRate=args.droprate) + +start_epoch = 0 + +# # Restore model +# if args.load != '': +# for i in range(1000 - 1, -1, -1): +# model_name = os.path.join(args.load, args.dataset + '_model_epoch' + str(i) + '.pytorch') +# if os.path.isfile(model_name): +# net.load_state_dict(torch.load(model_name)) +# print('Model restored! Epoch:', i) +# start_epoch = i + 1 +# break +# if start_epoch == 0: +# assert False, "could not resume" + + +if args.ngpu > 1: + net = torch.nn.DataParallel(net, device_ids=list(range(args.ngpu))) + +if args.ngpu > 0: + net.cuda() + torch.cuda.manual_seed(1) + +cudnn.benchmark = True # fire on all cylinders + +optimizer = torch.optim.SGD(net.parameters(), state['learning_rate'], momentum=state['momentum'], + weight_decay=state['decay'], nesterov=True) + +from tqdm import tqdm + +def train(): + net.train() # enter train mode + loss_avg = 0.0 + for batch_idx, (data, target) in enumerate(train_loader): + t = torch.from_numpy(np.random.beta(1,1, size=data.size(0)).astype(np.float32)).view(-1,1,1,1) + perm = torch.from_numpy(np.random.permutation(data.size(0))).long() + + data, target = V((t * data + (1 - t) * data[perm]).cuda()), V(target.cuda()) + + # forward + x = net(data) + + # backward + optimizer.zero_grad() + t = V(t.view(-1).cuda()) + loss = (t * F.cross_entropy(x, target, reduce=False)).mean() +\ + ((1 - t) * F.cross_entropy(x, target[V(perm.cuda())], reduce=False)).mean() + loss.backward() + optimizer.step() + + # exponential moving average + loss_avg = loss_avg * 0.8 + loss.data[0] * 0.2 + + dt = math.pi / float(args.epochs) + state['tt'] += float(dt) / (len(train_loader.dataset) / float(args.batch_size)) + if state['tt'] >= math.pi - 0.01: + state['tt'] = math.pi - 0.01 + curT = math.pi / 2.0 + state['tt'] + new_lr = args.learning_rate * (1.0 + math.sin(curT)) / 2.0 # lr_min = 0, lr_max = lr + state['learning_rate'] = new_lr + for param_group in optimizer.param_groups: + param_group['lr'] = state['learning_rate'] + + state['train_loss'] = loss_avg + + +# test function +def test(): + net.eval() + loss_avg = 0.0 + correct = 0 + for batch_idx, (data, target) in enumerate(test_loader): + data, target = V(data.cuda(), volatile=True), V(target.cuda(), volatile=True) + + # forward + output = net(data) + loss = F.cross_entropy(output, target) + + # accuracy + pred = output.data.max(1)[1] + correct += pred.eq(target.data).sum() + + # test loss average + loss_avg += loss.data[0] + + state['test_loss'] = loss_avg / len(test_loader) + state['test_accuracy'] = correct / len(test_loader.dataset) + + +def test_out(): + net.eval() + loss_avg = 0.0 + correct = 0 + for batch_idx, (data, target) in enumerate(test_out_loader): + data, target = V(data.cuda(), volatile=True), V(target.cuda(), volatile=True) + + # forward + output = net(data) + loss = F.cross_entropy(output, target) + + # accuracy + pred = output.data.max(1)[1] + correct += pred.eq(target.data).sum() + + # test loss average + loss_avg += loss.data[0] + + state['test_out_loss'] = loss_avg / len(test_out_loader) + state['test_out_accuracy'] = correct / len(test_out_loader.dataset) + + +if args.test: + test() + print(state) + exit() + +state['learning_rate'] = state['init_learning_rate'] + +print('Beginning Training') +# Main loop +best_accuracy = 0.0 +for epoch in range(start_epoch, args.epochs): + for param_group in optimizer.param_groups: + param_group['lr'] = state['learning_rate'] + state['tt'] = math.pi / float(args.epochs) * epoch + + state['epoch'] = epoch + + begin_epoch = time.time() + train() + print('Epoch', epoch, '| Time Spent:', round(time.time() - begin_epoch, 4)) + + test() + test_out() + + torch.save(net.state_dict(), + os.path.join(args.save, args.dataset + '_model_subclass_epoch' + str(epoch) + '.pytorch')) + # Let us not waste space and delete the previous model + # We do not overwrite the model because we need the epoch number + try: os.remove(os.path.join(args.save, args.dataset + '_model_subclass_epoch' + str(epoch - 1) + '.pytorch')) + except: True + + print(state) diff --git a/data/datasets/visual_question_answering/generate_c_image/robustness-master/old/auxiliary/ImageNet22K/test.py b/data/datasets/visual_question_answering/generate_c_image/robustness-master/old/auxiliary/ImageNet22K/test.py new file mode 100644 index 0000000000000000000000000000000000000000..650ba647bcc91c7b73dc9f5010eabfc2fc2264b7 --- /dev/null +++ b/data/datasets/visual_question_answering/generate_c_image/robustness-master/old/auxiliary/ImageNet22K/test.py @@ -0,0 +1,983 @@ +# -*- coding: utf-8 -*- +import numpy as np +import sys +import os +import pickle +import argparse +import math +import time +import torch +import torch.nn as nn +import torch.backends.cudnn as cudnn +import torchvision.transforms as trn +import torchvision.datasets as dset +import torch.nn.functional as F +from torch.autograd import Variable as V +import torchvision.models as models +import torch.utils.model_zoo as model_zoo +from resnext_101_64x4d import resnext_101_64x4d + +parser = argparse.ArgumentParser(description='Trains an ImageNet Superclass Classifier', + formatter_class=argparse.ArgumentDefaultsHelpFormatter) +parser.add_argument('--model-name', '-m', type=str, + choices=['alexnet', 'squeezenet1.1', 'vgg11', 'vgg', 'vggbn', 'resnet18', 'resnet50', 'resnext']) +# Optimization options +parser.add_argument('--test_bs', type=int, default=100) +# Checkpoints +parser.add_argument('--load', '-l', type=str, default='./snapshots', help='Checkpoint path to resume / test.') +parser.add_argument('--test', '-t', action='store_true', help='Test only flag.') +# Acceleration +parser.add_argument('--ngpu', type=int, default=1, help='0 = CPU.') +parser.add_argument('--prefetch', type=int, default=4, help='Pre-fetching threads.') +args = parser.parse_args() +state = {k: v for k, v in args._get_kwargs()} + +torch.manual_seed(1) +np.random.seed(1) + + +im1k_classes = ( + # amphibian n01627424 + ('n01641577', 'n01644373', 'n01644900', 'n01629819', 'n01630670', 'n01631663', 'n01632458', 'n01632777'), + # appliance n02729837 + ('n03483316', 'n04179913', 'n03584829', 'n03297495', 'n03761084', 'n03259280', 'n04111531', 'n04442312', 'n04542943', 'n04517823', 'n03207941', 'n04070727', 'n04554684'), + # aquatic mammal n02062017 + ('n02074367', 'n02077923', 'n02071294', 'n02066245'), + # bird n01503061 + ('n01514668', 'n01514859', 'n01518878', 'n01530575', 'n01531178', 'n01532829', 'n01534433', 'n01537544', 'n01558993', 'n01560419', 'n01580077', 'n01582220', 'n01592084', 'n01601694', 'n01608432', 'n01614925', 'n01616318', 'n01622779', 'n01795545', 'n01796340', 'n01797886', 'n01798484', 'n01806143', 'n01806567', 'n01807496', 'n01817953', 'n01818515', 'n01819313', 'n01820546', 'n01824575', 'n01828970', 'n01829413', 'n01833805', 'n01843065', 'n01843383', 'n01855672', 'n01847000', 'n01860187', 'n02002556', 'n02002724', 'n02006656', 'n02007558', 'n02009912', 'n02009229', 'n02011460', 'n02012849', 'n02013706', 'n02018207', 'n02018795', 'n02025239', 'n02027492', 'n02028035', 'n02033041', 'n02037110', 'n02017213', 'n02051845', 'n02056570', 'n02058221'), + # bear n02131653 + ('n02132136', 'n02134084', 'n02133161', 'n02134418'), + # beverage n07881800 + ('n07920052', 'n07892512', 'n07932039', 'n07930864'), + # big cat n02127808 + ('n02128925', 'n02129604', 'n02128385', 'n02128757', 'n02129165', 'n02130308'), + # building n02913152 + ('n02793495', 'n03457902', 'n03877845', 'n03781244', 'n03661043', 'n02727426', 'n02859443', 'n03028079', 'n03788195', 'n04346328', 'n03956157', 'n04081281', 'n03032252', 'n03529860'), + # cat n02121620 + ('n02124075', 'n02123394', 'n02123159', 'n02123597', 'n02123045', 'n02125311', 'n02127052'), + # clothing n03051540 + ('n03724870', 'n04584207', 'n03623198', 'n02730930', 'n04162706', 'n02669723', 'n04532106', 'n02916936', 'n04371430', 'n03710721', 'n02837789', 'n04350905', 'n03594734', 'n03325584', 'n04325704', 'n02883205', 'n04591157', 'n02865351', 'n03534580', 'n03770439', 'n03866082', 'n04136333', 'n03980874', 'n03404251', 'n04479046', 'n03630383', 'n04370456', 'n02963159', 'n03617480', 'n02667093', 'n03595614', 'n02892767', 'n03188531', 'n03775071', 'n03127747', 'n03379051', 'n02807133', 'n03787032', 'n04209133', 'n02869837', 'n02817516', 'n03124170', 'n04259630', 'n03710637', 'n04254777', 'n03026506', 'n03763968', 'n03877472', 'n03594734', 'n03450230', 'n02892767'), + # dog n02084071 + ('n02100583', 'n02100236', 'n02100735', 'n02101006', 'n02100877', 'n02102973', 'n02102177', 'n02102040', 'n02101556', 'n02101388', 'n02102318', 'n02102480', 'n02099601', 'n02099849', 'n02099429', 'n02099267', 'n02099712', 'n02096294', 'n02095314', 'n02098105', 'n02095889', 'n02095570', 'n02096437', 'n02096051', 'n02098413', 'n02094433', 'n02098286', 'n02097130', 'n02097047', 'n02097209', 'n02093256', 'n02093428', 'n02094114', 'n02096177', 'n02093859', 'n02097298', 'n02096585', 'n02093647', 'n02093991', 'n02097658', 'n02094258', 'n02097474', 'n02093754', 'n02090622', 'n02090721', 'n02092002', 'n02089078', 'n02089867', 'n02089973', 'n02092339', 'n02091635', 'n02088466', 'n02091467', 'n02091831', 'n02088094', 'n02091134', 'n02091032', 'n02088364', 'n02088238', 'n02088632', 'n02090379', 'n02091244', 'n02087394', 'n02110341', 'n02113186', 'n02113023', 'n02113978', 'n02111277', 'n02113712', 'n02113624', 'n02113799', 'n02110806', 'n02111129', 'n02112706', 'n02110958', 'n02109047', 'n02105641', 'n02106382', 'n02106550', 'n02105505', 'n02106030', 'n02106166', 'n02105162', 'n02105056', 'n02105855', 'n02105412', 'n02105251', 'n02106662', 'n02104365', 'n02107142', 'n02110627', 'n02107312', 'n02104029', 'n02110185', 'n02110063', 'n02108089', 'n02108422', 'n02109961', 'n02108000', 'n02107683', 'n02107574', 'n02107908', 'n02109525', 'n02108551', 'n02108915', 'n02112018', 'n02112350', 'n02112137', 'n02111889', 'n02111500', 'n02086910', 'n02086646', 'n02086079', 'n02085936', 'n02087046', 'n02085782', 'n02086240', 'n02085620'), + # electronic equipment n03278248 + ('n03584254', 'n03777754', 'n03782006', 'n03857828', 'n04004767', 'n03085013', 'n03602883', 'n02979186', 'n02992529', 'n03902125', 'n03187595', 'n04392985', 'n02988304'), + # fish n02512053 + ('n01496331', 'n01498041', 'n01484850', 'n01491361', 'n01494475', 'n02514041', 'n02536864', 'n01440764', 'n01443537', 'n02526121', 'n02536864', 'n02606052', 'n02607072', 'n02643566', 'n02655020', 'n02640242', 'n02641379'), + # footwear n03380867 + ('n04133789', 'n04120489', 'n03680355', 'n03124043', 'n03047690'), + # fruit n13134947 + ('n07742313', 'n07745940', 'n07747607', 'n07749582', 'n07753113', 'n07753275', 'n07753592', 'n07754684', 'n07760859', 'n07768694', 'n12267677', 'n12620546', 'n13133613', 'n11879895', 'n12144580', 'n12768682', 'n07742313'), + # fungus n12992868 + ('n13052670', 'n13044778', 'n12985857', 'n13040303', 'n13037406', 'n13054560', 'n12998815'), + # geological formation n09287968 + ('n09246464', 'n09468604', 'n09193705', 'n09472597', 'n09399592', 'n09421951', 'n09256479', 'n09332890', 'n09428293', 'n09288635'), + # hoofed animal n02370806 + ('n02391049', 'n02389026', 'n02437312', 'n02412080', 'n02423022', 'n02422699', 'n02422106', 'n02415577', 'n02417914', 'n02410509', 'n02408429', 'n02403003', 'n02398521', 'n02437616', 'n02396427', 'n02397096', 'n02395406'), + # insect n02159955 + ('n02165105', 'n02165456', 'n02167151', 'n02168699', 'n02169497', 'n02172182', 'n02174001', 'n02177972', 'n02190166', 'n02219486', 'n02206856', 'n02226429', 'n02229544', 'n02231487', 'n02233338', 'n02236044', 'n02256656', 'n02259212', 'n02264363', 'n02268443', 'n02268853', 'n02276258', 'n02277742', 'n02279972', 'n02280649', 'n02281406', 'n02281787'), + # musical instrument n03800933 + ('n02672831', 'n03854065', 'n03452741', 'n04515003', 'n03452741', 'n04515003', 'n03017168', 'n03249569', 'n03447721', 'n03720891', 'n03721384', 'n04311174', 'n03452741', 'n04515003', 'n02787622', 'n02992211', 'n04536866', 'n03495258', 'n02676566', 'n03272010', 'n03854065', 'n03110669', 'n03394916', 'n04487394', 'n02672831', 'n03494278', 'n03840681', 'n03884397', 'n02804610', 'n03838899', 'n04141076', 'n03372029'), + # primate n02469914 + ('n02500267', 'n02497673', 'n02483708', 'n02483362', 'n02480495', 'n02481823', 'n02480855', 'n02488702', 'n02484975', 'n02489166', 'n02486261', 'n02486410', 'n02487347', 'n02488291', 'n02493509', 'n02494079', 'n02493793', 'n02492035', 'n02492660', 'n02490219'), + # reptile n01661091 + ('n01664065', 'n01665541', 'n01667114', 'n01667778', 'n01669191', 'n01675722', 'n01677366', 'n01682714', 'n01685808', 'n01687978', 'n01688243', 'n01689811', 'n01692333', 'n01693334', 'n01694178', 'n01695060', 'n01704323', 'n01698640', 'n01697457', 'n01728572', 'n01728920', 'n01729322', 'n01729977', 'n01734418', 'n01735189', 'n01737021', 'n01739381', 'n01740131', 'n01742172', 'n01744401', 'n01748264', 'n01749939', 'n01751748', 'n01753488', 'n01755581', 'n01756291'), + # utensil n04516672 + ('n03133878', 'n03400231', 'n04596742', 'n02939185', 'n03063689', 'n04398044', 'n04270147'), + # vegetable n07707451 + ('n07711569', 'n07720875', 'n07711569', 'n07714571', 'n07714990', 'n07715103', 'n07717410', 'n07717556', 'n07716358', 'n07716906', 'n07718472', 'n07718747', 'n07730033', 'n07734744'), + # vehicle n04576211 + ('n02835271', 'n03792782', 'n03393912', 'n03895866', 'n02797295', 'n04204347', 'n03791053', 'n04389033', 'n03384352', 'n03272562', 'n04310018', 'n02704792', 'n02701002', 'n02814533', 'n02930766', 'n03100240', 'n03594945', 'n03670208', 'n03770679', 'n03777568', 'n04037443', 'n04285008', 'n03444034', 'n03445924', 'n03785016', 'n04252225', 'n03345487', 'n03417042', 'n03930630', 'n04461696', 'n04467665', 'n03796401', 'n03770679', 'n03977966', 'n04065272', 'n04335435', 'n03478589', 'n04389033', 'n04252077', 'n04465501', 'n03776460', 'n04482393', 'n04509417', 'n03538406', 'n03599486', 'n03868242') +) + + +# this excludes ImageNet-1K classes +im22k_classes = ( + # amphibian n01627424 + ( + 'n01627424', 'n01639765', 'n01640846', 'n01641206', 'n01641391', 'n01641739', 'n01641930', 'n01642097', 'n01642257', + 'n01642391', 'n01642539', 'n01642943', 'n01643255', 'n01643507', 'n01643896', 'n01645466', 'n01645776', 'n01646292', + 'n01646388', 'n01646555', 'n01646648', 'n01646802', 'n01646902', 'n01647033', 'n01647180', 'n01647303', 'n01647466', + 'n01647640', 'n01648139', 'n01648356', 'n01648620', 'n01649170', 'n01649412', 'n01649556', 'n01649726', 'n01650167', + 'n01650690', 'n01650901', 'n01651059', 'n01651285', 'n01651487', 'n01651641', 'n01651778', 'n01652026', 'n01652297', + 'n01653026', 'n01653223', 'n01653509', 'n01653773', 'n01654083', 'n01654637', 'n01654863', 'n01628331', 'n01628770', + 'n01629276', 'n01629962', 'n01630148', 'n01630284', 'n01630901', 'n01631175', 'n01631354', 'n01631512', 'n01632047', + 'n01632308', 'n01632601', 'n01632952', 'n01633406', 'n01633781', 'n01634227', 'n01634522', 'n01635027', 'n01635176', + 'n01635480', 'n01636127', 'n01636352', 'n01636510', 'n01636829', 'n01637112', 'n01637338', 'n01637615', 'n01637932', + 'n01638194', 'n01638329', 'n01638722', 'n01639187', 'n01655344'), + # appliance n02729837 + ( + 'n02729837', 'n03251766', 'n03050655', 'n03717285', 'n04277826', 'n04496726', 'n04607242', 'n03528263', 'n04174101', + 'n03150511', 'n04309833', 'n04475631', 'n03620052', 'n03063338', 'n04219185', 'n03212114', 'n03378174', 'n03543254', + 'n03557692', 'n03862676', 'n02905036', 'n03425241', 'n04388473', 'n04330340', 'n03102371', 'n03273740', 'n03425595', + 'n03991202', 'n04003241', 'n04280487', 'n04442441', 'n04488857', 'n03534776', 'n04580493', 'n03050655', 'n03717285', + 'n04277826', 'n04496726', 'n04607242', 'n03273740', 'n03102654', 'n03273913', 'n03557590', 'n03170635'), + # aquatic mammal n02062017 + ( + 'n02062017', 'n02073250', 'n02073831', 'n02074726', 'n02075927', 'n02076196', 'n02076779', 'n02078574', 'n02078292', + 'n02078738', 'n02079005', 'n02077658', 'n02077787', 'n02077152', 'n02077384', 'n02076402', 'n02079389', 'n02081060', + 'n02079851', 'n02080146', 'n02080415', 'n02080713', 'n02081571', 'n02081798', 'n02081927', 'n02062430', 'n02062744', + 'n02066707', 'n02068974', 'n02072040', 'n02069701', 'n02069974', 'n02070174', 'n02072798', 'n02069412', 'n02071028', + 'n02070430', 'n02070624', 'n02070776', 'n02071636', 'n02067240', 'n02068206', 'n02068541', 'n02067768', 'n02067603', + 'n02072493', 'n02063224', 'n02063662', 'n02064000', 'n02064816', 'n02064338', 'n02065026', 'n02065407', 'n02065263', + 'n02065726'), + # bird n01503061 + ( + 'n01503061', 'n01503976', 'n01514752', 'n01514926', 'n01515078', 'n01515217', 'n01515303', 'n01516212', 'n01517389', + 'n01517565', 'n01519563', 'n01519873', 'n01520576', 'n01521399', 'n01521756', 'n01522450', 'n01523105', 'n01517966', + 'n01524359', 'n01525720', 'n01526521', 'n01526766', 'n01527194', 'n01527347', 'n01527617', 'n01527917', 'n01528396', + 'n01528654', 'n01528845', 'n01529672', 'n01530439', 'n01531344', 'n01531512', 'n01531639', 'n01531811', 'n01531971', + 'n01532325', 'n01532511', 'n01533000', 'n01533339', 'n01533481', 'n01533651', 'n01533893', 'n01534155', 'n01534582', + 'n01534762', 'n01535140', 'n01535469', 'n01535690', 'n01536035', 'n01536186', 'n01536334', 'n01536644', 'n01536780', + 'n01537134', 'n01537895', 'n01538059', 'n01538200', 'n01538362', 'n01538630', 'n01540233', 'n01540566', 'n01540832', + 'n01541102', 'n01541386', 'n01541760', 'n01541922', 'n01542433', 'n01542168', 'n01544704', 'n01538955', 'n01539272', + 'n01542786', 'n01543175', 'n01543383', 'n01543632', 'n01543936', 'n01544208', 'n01544389', 'n01555809', 'n01556182', + 'n01556514', 'n01557185', 'n01557962', 'n01558149', 'n01558307', 'n01558461', 'n01558594', 'n01558765', 'n01559160', + 'n01559477', 'n01559639', 'n01559804', 'n01560105', 'n01560280', 'n01560636', 'n01560793', 'n01560935', 'n01561181', + 'n01561452', 'n01561732', 'n01562014', 'n01562265', 'n01562451', 'n01563128', 'n01563449', 'n01563746', 'n01563945', + 'n01564101', 'n01564217', 'n01564394', 'n01564773', 'n01565345', 'n01565599', 'n01565930', 'n01566207', 'n01564914', + 'n01565078', 'n01567133', 'n01567678', 'n01567879', 'n01568132', 'n01568294', 'n01568720', 'n01568892', 'n01569060', + 'n01569262', 'n01569423', 'n01569566', 'n01569836', 'n01569971', 'n01570267', 'n01570421', 'n01570676', 'n01570839', + 'n01566645', 'n01571410', 'n01571904', 'n01572328', 'n01572489', 'n01572654', 'n01572782', 'n01573074', 'n01573240', + 'n01573360', 'n01573627', 'n01573898', 'n01574045', 'n01574390', 'n01574560', 'n01574801', 'n01575117', 'n01575401', + 'n01575745', 'n01576076', 'n01576358', 'n01576695', 'n01577035', 'n01577458', 'n01577659', 'n01577941', 'n01578180', + 'n01578575', 'n01579028', 'n01579149', 'n01579260', 'n01579410', 'n01579578', 'n01579729', 'n01580379', 'n01580490', + 'n01580772', 'n01580870', 'n01581166', 'n01581434', 'n01581730', 'n01581874', 'n01581984', 'n01582398', 'n01582498', + 'n01582856', 'n01583209', 'n01583495', 'n01583828', 'n01586941', 'n01587278', 'n01587526', 'n01587834', 'n01588002', + 'n01588431', 'n01588725', 'n01588996', 'n01589286', 'n01589718', 'n01589893', 'n01590220', 'n01591005', 'n01591123', + 'n01591301', 'n01591697', 'n01592257', 'n01592540', 'n01592387', 'n01592694', 'n01593028', 'n01593282', 'n01593553', + 'n01594004', 'n01594372', 'n01594787', 'n01594968', 'n01595168', 'n01595450', 'n01595624', 'n01595974', 'n01596273', + 'n01596608', 'n01597022', 'n01597336', 'n01597737', 'n01597906', 'n01598074', 'n01598271', 'n01598588', 'n01598988', + 'n01599159', 'n01599269', 'n01599388', 'n01599556', 'n01599741', 'n01600085', 'n01600341', 'n01600657', 'n01601410', + 'n01601068', 'n01602080', 'n01602209', 'n01602630', 'n01602832', 'n01603000', 'n01603152', 'n01603600', 'n01603812', + 'n01603953', 'n01539573', 'n01539925', 'n01540090', 'n01545574', 'n01546039', 'n01546506', 'n01546921', 'n01547832', + 'n01548301', 'n01548492', 'n01548694', 'n01548865', 'n01549053', 'n01549430', 'n01549641', 'n01549886', 'n01550172', + 'n01550761', 'n01551080', 'n01551300', 'n01552034', 'n01552333', 'n01555305', 'n01551711', 'n01552813', 'n01553142', + 'n01553527', 'n01553762', 'n01554017', 'n01554448', 'n01555004', 'n01584225', 'n01584695', 'n01584853', 'n01585121', + 'n01585287', 'n01585422', 'n01585715', 'n01586020', 'n01586374', 'n01524761', 'n01604330', 'n01604968', 'n01605630', + 'n01606097', 'n01606177', 'n01606522', 'n01606672', 'n01606809', 'n01606978', 'n01607309', 'n01607429', 'n01607600', + 'n01607812', 'n01607962', 'n01608265', 'n01608814', 'n01609062', 'n01609391', 'n01609751', 'n01609956', 'n01610100', + 'n01610226', 'n01610552', 'n01610955', 'n01611472', 'n01611674', 'n01611800', 'n01611969', 'n01612122', 'n01612275', + 'n01612476', 'n01612628', 'n01612955', 'n01613177', 'n01616086', 'n01613294', 'n01613807', 'n01614038', 'n01614343', + 'n01614556', 'n01615121', 'n01615303', 'n01615458', 'n01615703', 'n01616551', 'n01616764', 'n01617095', 'n01617443', + 'n01617766', 'n01618082', 'n01618922', 'n01619310', 'n01619536', 'n01619835', 'n01620135', 'n01620414', 'n01620735', + 'n01618503', 'n01621127', 'n01621635', 'n01622120', 'n01622352', 'n01622483', 'n01622959', 'n01623110', 'n01623425', + 'n01623615', 'n01623706', 'n01624115', 'n01624212', 'n01623880', 'n01624305', 'n01624537', 'n01624833', 'n01625121', + 'n01625562', 'n01789386', 'n01789740', 'n01790171', 'n01790304', 'n01790398', 'n01790557', 'n01790711', 'n01790812', + 'n01791625', 'n01792042', 'n01792158', 'n01792429', 'n01792530', 'n01792640', 'n01792808', 'n01792955', 'n01793085', + 'n01793159', 'n01793249', 'n01793340', 'n01793435', 'n01793565', 'n01793715', 'n01791954', 'n01794158', 'n01794344', + 'n01809106', 'n01809371', 'n01791107', 'n01791314', 'n01791388', 'n01791463', 'n01794651', 'n01799302', 'n01800195', + 'n01799679', 'n01800424', 'n01800633', 'n01801088', 'n01801479', 'n01801672', 'n01801876', 'n01802159', 'n01809752', + 'n01810700', 'n01811243', 'n01811909', 'n01812187', 'n01812337', 'n01813385', 'n01813532', 'n01813658', 'n01813948', + 'n01814217', 'n01812662', 'n01812866', 'n01813088', 'n01814370', 'n01814620', 'n01814755', 'n01814921', 'n01815036', + 'n01814549', 'n01815270', 'n01815601', 'n01816017', 'n01816140', 'n01816474', 'n02153203', 'n01795088', 'n01795735', + 'n01795900', 'n01796019', 'n01796105', 'n01796519', 'n01796729', 'n01797020', 'n01797307', 'n01797601', 'n01798168', + 'n01798706', 'n01798839', 'n01798979', 'n01802721', 'n01803078', 'n01803362', 'n01803641', 'n01803893', 'n01804163', + 'n01805321', 'n01805801', 'n01806061', 'n01806297', 'n01806364', 'n01806467', 'n01807105', 'n01804478', 'n01804653', + 'n01804921', 'n01805070', 'n01806847', 'n01807828', 'n01808140', 'n01808291', 'n01808596', 'n01810268', 'n01816887', + 'n01817263', 'n01817346', 'n01818299', 'n01818832', 'n01819115', 'n01819465', 'n01819734', 'n01820052', 'n01820348', + 'n01820801', 'n01821076', 'n01821203', 'n01821554', 'n01821869', 'n01822300', 'n01822602', 'n01823013', 'n01823414', + 'n01823740', 'n01824035', 'n01824344', 'n01824749', 'n01825278', 'n01825930', 'n01826364', 'n01826680', 'n01826844', + 'n01827403', 'n01827793', 'n01828096', 'n01828556', 'n01829869', 'n01830042', 'n01830479', 'n01830915', 'n01831360', + 'n01831712', 'n01832167', 'n01832493', 'n01832813', 'n01833112', 'n01833415', 'n01834177', 'n01834540', 'n01835276', + 'n01835769', 'n01835918', 'n01836087', 'n01836673', 'n01837072', 'n01837526', 'n01838038', 'n01838598', 'n01839086', + 'n01839330', 'n01839598', 'n01839750', 'n01839949', 'n01840120', 'n01840412', 'n01840775', 'n01841102', 'n01841288', + 'n01841441', 'n01841679', 'n01841943', 'n01842235', 'n01842504', 'n01842788', 'n01843719', 'n01844231', 'n01844551', + 'n01844746', 'n01844917', 'n01845132', 'n01860497', 'n01860864', 'n01861148', 'n01861330', 'n01845477', 'n01856072', + 'n01856155', 'n01856380', 'n01856553', 'n01856890', 'n01857079', 'n01857325', 'n01857512', 'n01857632', 'n01857851', + 'n01846331', 'n01847089', 'n01847170', 'n01847253', 'n01847407', 'n01847806', 'n01847978', 'n01848123', 'n01848323', + 'n01848453', 'n01848555', 'n01848648', 'n01848840', 'n01848976', 'n01849157', 'n01849466', 'n01849676', 'n01849863', + 'n01850192', 'n01850373', 'n01850553', 'n01850873', 'n01851038', 'n01851207', 'n01851375', 'n01851731', 'n01851573', + 'n01851895', 'n01852142', 'n01852329', 'n01852400', 'n01852671', 'n01852861', 'n01853195', 'n01853498', 'n01853870', + 'n01854415', 'n01858441', 'n01858281', 'n01858780', 'n01858845', 'n01858906', 'n01859190', 'n01859325', 'n01859496', + 'n01859689', 'n01859852', 'n01860002', 'n02000954', 'n02002075', 'n02003037', 'n02003204', 'n02003577', 'n02003839', + 'n02004131', 'n02004492', 'n02004855', 'n02005399', 'n02005790', 'n02006063', 'n02006364', 'n02007284', 'n02006985', + 'n02008041', 'n02008497', 'n02008643', 'n02008796', 'n02009380', 'n02009508', 'n02009750', 'n02010272', 'n02010453', + 'n02010728', 'n02011016', 'n02011281', 'n02011805', 'n02011943', 'n02012185', 'n02013177', 'n02013567', 'n02014237', + 'n02014524', 'n02014941', 'n02015357', 'n02015554', 'n02015797', 'n02016066', 'n02017725', 'n02018027', 'n02018368', + 'n02019190', 'n02019438', 'n02019929', 'n02020219', 'n02020578', 'n02021050', 'n02021281', 'n02022684', 'n02023341', + 'n02023855', 'n02023992', 'n02024185', 'n02024479', 'n02024763', 'n02025043', 'n02025389', 'n02026059', 'n02026948', + 'n02027075', 'n02027357', 'n02027897', 'n02028175', 'n02028342', 'n02028451', 'n02028727', 'n02028900', 'n02029087', + 'n02029378', 'n02029706', 'n02030035', 'n02030224', 'n02030287', 'n02030837', 'n02030568', 'n02026629', 'n02030996', + 'n02031298', 'n02031585', 'n02031934', 'n02032222', 'n02032355', 'n02032480', 'n02032769', 'n02033324', 'n02033208', + 'n02033561', 'n02033779', 'n02033882', 'n02034129', 'n02034295', 'n02034661', 'n02034971', 'n02035210', 'n02035402', + 'n02035656', 'n02036053', 'n02036228', 'n02036711', 'n02037464', 'n02037869', 'n02038141', 'n02038466', 'n02038993', + 'n02039171', 'n02039780', 'n02039497', 'n02040266', 'n02016358', 'n02016659', 'n02016816', 'n02016956', 'n02017475', + 'n02021795', 'n02040505', 'n02041085', 'n02043063', 'n02043333', 'n02041246', 'n02041678', 'n02041875', 'n02042046', + 'n02042180', 'n02042472', 'n02042759', 'n02043808', 'n02044178', 'n02044778', 'n02044908', 'n02044517', 'n02045369', + 'n02045596', 'n02045864', 'n02046171', 'n02046759', 'n02046939', 'n02047045', 'n02047260', 'n02047411', 'n02047517', + 'n02047614', 'n02047975', 'n02048115', 'n02048353', 'n02048698', 'n02049088', 'n02049532', 'n02050004', 'n02050313', + 'n02050442', 'n02050586', 'n02050809', 'n02051059', 'n02051474', 'n02052365', 'n02052204', 'n02052775', 'n02053083', + 'n02053425', 'n02053584', 'n02054036', 'n02054502', 'n02054711', 'n02055107', 'n02055658', 'n02055803', 'n02056228', + 'n02056728', 'n02057035', 'n02057330', 'n02057731', 'n02057898', 'n02058594', 'n02058747', 'n02059162', 'n02059541', + 'n02059852', 'n02060133', 'n02060411', 'n02060569', 'n02060889', 'n02061217', 'n02061560', 'n02061853', 'n02511730'), + # bear n02131653 + ('n02131653', 'n02133704', 'n02132788', 'n02132466', 'n02132580', 'n02132320', 'n02133400', 'n01322983'), + # beverage n07881800 + ( + 'n07881800', 'n07925966', 'n07926346', 'n07926250', 'n07926442', 'n07933274', 'n07934373', 'n07933652', 'n07933799', + 'n07933891', 'n07934032', 'n07934152', 'n07934282', 'n07913180', 'n07929519', 'n07920349', 'n07731122', 'n07731284', + 'n07731436', 'n07921239', 'n07919665', 'n07919572', 'n07919441', 'n07920872', 'n07920222', 'n07919787', 'n07919894', + 'n07920540', 'n07920663', 'n07929940', 'n07926785', 'n07922764', 'n07890970', 'n07891309', 'n07883251', 'n07883661', + 'n07883384', 'n07883510', 'n07882420', 'n07924033', 'n07925116', 'n07924366', 'n07924443', 'n07924747', 'n07924560', + 'n07924655', 'n07924276', 'n07924834', 'n07924955', 'n07914271', 'n07891189', 'n07921455', 'n07921615', 'n07921834', + 'n07921948', 'n07922041', 'n07914128', 'n07936263', 'n07936093', 'n07935737', 'n07936745', 'n14941787', 'n07936979', + 'n07937069', 'n07936459', 'n07936548', 'n07914006', 'n07927197', 'n07928887', 'n07927512', 'n07927836', 'n07928367', + 'n07927931', 'n07928696', 'n07928790', 'n07928163', 'n07928264', 'n07927716', 'n07929172', 'n07928578', 'n07928998', + 'n07928488', 'n07884567', 'n07886176', 'n07885705', 'n07922607', 'n07905618', 'n07886572', 'n07890617', 'n07886849', + 'n07889510', 'n07888465', 'n07888816', 'n07890068', 'n07889814', 'n07890352', 'n07890540', 'n07889990', 'n07890226', + 'n07887099', 'n07887192', 'n07887634', 'n07887967', 'n07889274', 'n07888058', 'n07887461', 'n07888229', 'n07887304', + 'n07890750', 'n07890890', 'n07932323', 'n07932454', 'n07921615', 'n07922147', 'n07922512', 'n07886463', 'n07901587', + 'n07902336', 'n07904934', 'n07905474', 'n07903101', 'n07903208', 'n07904293', 'n07903962', 'n07904072', 'n07902443', + 'n07903731', 'n07903841', 'n07903643', 'n07903543', 'n07906284', 'n07906718', 'n07906572', 'n07907429', 'n07907831', + 'n07907161', 'n07907342', 'n07907548', 'n07909593', 'n07906877', 'n07902520', 'n07902937', 'n07904395', 'n07904760', + 'n07902698', 'n07904637', 'n07902799', 'n07907037', 'n07906111', 'n07905038', 'n07905296', 'n07904865', 'n07905386', + 'n07905770', 'n07905979', 'n07907943', 'n07910656', 'n07909714', 'n07908812', 'n07908647', 'n07908567', 'n07908411', + 'n07910245', 'n07910379', 'n07909504', 'n07911249', 'n07909811', 'n07909954', 'n07910048', 'n07910152', 'n07909593', + 'n07908923', 'n07909129', 'n07910970', 'n07910538', 'n07911061', 'n07909231', 'n07909362', 'n07910799', 'n07925808', + 'n07902121', 'n07891726', 'n07900225', 'n07892813', 'n07894551', 'n07896893', 'n07898443', 'n07897975', 'n07899769', + 'n07898247', 'n07899533', 'n07895100', 'n07897200', 'n07897438', 'n07897600', 'n07894703', 'n07899660', 'n07895962', + 'n07894799', 'n07899899', 'n07896765', 'n07894451', 'n07900406', 'n07901355', 'n07900825', 'n07900616', 'n07900734', + 'n07901457', 'n07900958', 'n07899976', 'n07893528', 'n07893642', 'n07893792', 'n07896060', 'n07896560', 'n07897750', + 'n07897865', 'n07895710', 'n07895839', 'n07895435', 'n07898117', 'n07898333', 'n07894965', 'n07894102', 'n07895595', + 'n07894298', 'n07896165', 'n07897116', 'n07896422', 'n07892418', 'n07893891', 'n07894551', 'n07894703', 'n07894102', + 'n07899108', 'n07899434', 'n07899292', 'n07926920', 'n07927070', 'n07913300', 'n07898745', 'n07899003', 'n07895237', + 'n07895435', 'n07898117', 'n07894298', 'n07893253', 'n07896994', 'n07893425', 'n07896287', 'n07898617', 'n07896661', + 'n07898895', 'n07886057', 'n07911371', 'n07918879', 'n07930554', 'n07931280', 'n07930205', 'n07931001', 'n07931096', + 'n07931733', 'n07930062', 'n07931870', 'n07912211', 'n07913882', 'n07917618', 'n07917951', 'n07917874', 'n07917791', + 'n07915491', 'n07915094', 'n07913774', 'n07917507', 'n07919165', 'n07912093', 'n07911677', 'n07916041', 'n07916319', + 'n07917392', 'n07915918', 'n07915366', 'n07931612', 'n07913393', 'n07913537', 'n07916437', 'n07914413', 'n07914586', + 'n07914686', 'n07916582', 'n07918028', 'n07918193', 'n07913644', 'n07917133', 'n07915618', 'n07915800', 'n07917272', + 'n07931452', 'n07916183', 'n07915213', 'n07918309', 'n07914995', 'n07930315', 'n07914887', 'n07918706', 'n07930433', + 'n07914777', 'n07932614', 'n07932762', 'n07891433', 'n07886317', 'n07891095', 'n07844042', 'n07844786', 'n07846143', + 'n07846274', 'n07845775', 'n07921360', 'n07846014', 'n07845421', 'n07847047', 'n07846359', 'n07845335', 'n07846471', + 'n07846557', 'n07845702', 'n07846688', 'n07845495', 'n07846802', 'n07846938', 'n07845863', 'n07845166', 'n07845571', + 'n07919310', 'n07933530'), + # big cat n02127808 + ( + 'n02127808', 'n02129991', 'n02130545', 'n02130925', 'n02129923', 'n02129837', 'n01323068', 'n02128598', 'n02128669', + 'n02129463', 'n02129530', 'n01322898', 'n02130086'), + # building n02913152 + ( + 'n02913152', 'n02666943', 'n02726681', 'n04409384', 'n02734725', 'n02763604', 'n02806992', 'n02882190', 'n02993546', + 'n02940385', 'n03078506', 'n03089753', 'n03097362', 'n04077889', 'n04175574', 'n04177931', 'n04343511', 'n03007444', + 'n03224893', 'n03479397', 'n03322570', 'n03123809', 'n04441662', 'n03016389', 'n04294879', 'n03326371', 'n03413428', + 'n02977936', 'n03430418', 'n03449564', 'n02956699', 'n03005033', 'n03121431', 'n03152303', 'n03203806', 'n03093427', + 'n03282295', 'n04305210', 'n04461437', 'n03092166', 'n13252672', 'n03478756', 'n03036022', 'n03466839', 'n03437184', + 'n03698723', 'n03479121', 'n03479266', 'n03542333', 'n03541696', 'n02961035', 'n03561573', 'n03989898', 'n04097085', + 'n03790755', 'n03788498', 'n04080705', 'n04095109', 'n04229737', 'n08640531', 'n08560295', 'n08652376', 'n03542605', + 'n03544360', 'n02814338', 'n02857477', 'n02820085', 'n02919792', 'n02932400', 'n03686924', 'n03002816', 'n03007297', + 'n03118969', 'n03010915', 'n03158186', 'n04202142', 'n04354026', 'n04535252', 'n04535370', 'n03180865', 'n03219483', + 'n03257210', 'n03322836', 'n03428090', 'n03685640', 'n03465605', 'n03474352', 'n03685486', 'n03685820', 'n03367321', + 'n03713151', 'n03719053', 'n03718458', 'n03878066', 'n04305323', 'n04052658', 'n04079244', 'n03121040', 'n03166514', + 'n03718935', 'n02695627', 'n03892557', 'n03439348', 'n04073948', 'n03099454', 'n02667478', 'n02667379', 'n03396580', + 'n03635032', 'n04005197', 'n04115256', 'n02907873', 'n04413969', 'n04125541', 'n04131368', 'n04255899', 'n04258438', + 'n04465050', 'n04535524', 'n03545150', 'n02806875', 'n04350235', 'n03121298', 'n03333610', 'n03736269', 'n03837698', + 'n04022708', 'n04022866', 'n04246731', 'n03739518', 'n03043274', 'n03210552', 'n03540595', 'n03129471', 'n03730334', + 'n03762982', 'n03333349', 'n03770316', 'n03785499', 'n03130233', 'n03402941', 'n03839671', 'n03842012', 'n03859280', + 'n03055857', 'n03416489', 'n02968074', 'n03610524', 'n03860404', 'n04187547', 'n03056288', 'n04452757', 'n04598318', + 'n03872016', 'n03953416', 'n02833040', 'n03007130', 'n03006788', 'n03633341', 'n04214413', 'n02667576', 'n02801184', + 'n02984061', 'n03772077', 'n02984203', 'n03618982', 'n03099622', 'n03724756', 'n04210390', 'n04374735', 'n04407435', + 'n03602365', 'n03884778', 'n03999160', 'n02844214', 'n02892499', 'n02897389', 'n02935658', 'n02936281', 'n03155178', + 'n03297644', 'n03298089', 'n02935891', 'n02760099', 'n02952485', 'n02952674', 'n03199647', 'n03456548', 'n03459914', + 'n03497100', 'n03697552', 'n04111414', 'n04307878', 'n04398497', 'n04081699', 'n04093625', 'n03558176', 'n03557360', + 'n04104500', 'n02801047', 'n04112654', 'n04118635', 'n04146050', 'n03092314', 'n03801671', 'n02746978', 'n03165616', + 'n04217546', 'n04233124', 'n04343740', 'n04395875', 'n02823586', 'n02910241', 'n04018399', 'n02696165', 'n03393017', + 'n04055700', 'n07888378', 'n04400109', 'n04407686', 'n04614655', 'n04417809', 'n03798982', 'n03202481', 'n03678729', + 'n03801533', 'n03849814', 'n04581595', 'n03726233'), + # cat n02121620 + ( + 'n02121620', 'n02121808', 'n02122298', 'n02123478', 'n02122725', 'n02124484', 'n02124157', 'n02122878', 'n02123917', + 'n02122510', 'n02124313', 'n02123242', 'n02122430', 'n02124623', 'n02125081', 'n02125872', 'n02125010', 'n02126787', + 'n02126317', 'n02125494', 'n02125689', 'n02126028', 'n02126640', 'n02126139'), + # clothing n03051540 + ( + 'n03051540', 'n04385079', 'n02756098', 'n03859958', 'n04489695', 'n02834506', 'n03964611', 'n03289985', 'n04129766', + 'n03114041', 'n03113657', 'n03320519', 'n03340923', 'n04355115', 'n03786194', 'n03114236', 'n03206718', 'n03320519', + 'n03319457', 'n03221059', 'n02726017', 'n03384891', 'n02780704', 'n03239054', 'n03201638', 'n03201776', 'n04285803', + 'n04120695', 'n03472672', 'n03476083', 'n02683454', 'n04459018', 'n03917327', 'n03538957', 'n03263338', 'n03113835', + 'n02669534', 'n04092168', 'n03473966', 'n03398153', 'n03835941', 'n03781467', 'n03474167', 'n02846141', 'n03692379', + 'n03692136', 'n03692272', 'n03692004', 'n04243142', 'n04600912', 'n03863108', 'n03655720', 'n03815482', 'n03068998', + 'n03406759', 'n04015204', 'n04194127', 'n03268645', 'n03216402', 'n03121897', 'n03863262', 'n03604763', 'n04607640', + 'n02752615', 'n04429038', 'n03015478', 'n02841847', 'n04423552', 'n04001845', 'n02720576', 'n04266375', 'n02863340', + 'n02738859', 'n03386870', 'n04207903', 'n03521771', 'n02671780', 'n02827606', 'n04125853', 'n04132465', 'n02778294', + 'n02972397', 'n02786611', 'n03527565', 'n03781683', 'n03405595', 'n03859495', 'n03450516', 'n03434830', 'n03010795', + 'n03880129', 'n02694966', 'n04364994', 'n03981340', 'n03548930', 'n02979516', 'n04264233', 'n04446162', 'n02814774', + 'n02742322', 'n04552097', 'n02855925', 'n03419014', 'n04332580', 'n04488530', 'n02922578', 'n04531873', 'n03608504', + 'n04371563', 'n04574067', 'n04219580', 'n02896294', 'n03186199', 'n03226538', 'n04222470', 'n03943920', 'n02925519', + 'n04427715', 'n04504141', 'n04615644', 'n04233832', 'n03885669', 'n03657511', 'n04046277', 'n03789794', 'n04123317', + 'n03502331', 'n03314884', 'n03826039', 'n04612159', 'n02998841', 'n04605572', 'n04489008', 'n03660124', 'n04337287', + 'n03877674', 'n03600285', 'n02896442', 'n02902816', 'n03970363', 'n04491934', 'n02910864', 'n03107488', 'n03688605', + 'n03019434', 'n03884554', 'n04370288', 'n04480303', 'n03029296', 'n04132158', 'n04233715', 'n03903733', 'n04205318', + 'n03653833', 'n02831595', 'n03543112', 'n02825442', 'n03357081', 'n03745487', 'n03487642', 'n03450734', 'n04143897', + 'n03332173', 'n03610992', 'n03814817', 'n03505504', 'n03520493', 'n03615655', 'n02766168', 'n04122262', 'n04495698', + 'n03719743', 'n03900028', 'n04060448', 'n03797182', 'n03648219', 'n03816005', 'n03815615', 'n03845990', 'n02847631', + 'n04339191', 'n03388323', 'n03128085', 'n02747063', 'n03814727', 'n03913930', 'n04325804', 'n04370774', 'n04160261', + 'n04570532', 'n03605598', 'n04104770', 'n04230808', 'n03454442', 'n03617312', 'n03429003', 'n03205669', 'n03732458', + 'n02780815', 'n03402511', 'n03146777', 'n03649003', 'n03523506', 'n03863923', 'n03588216', 'n03045337', 'n03595055', + 'n04497570', 'n03472796', 'n04361937', 'n04378489', 'n03619275', 'n03607923', 'n02956883', 'n02936402', 'n02923535', + 'n03103904', 'n03880032', 'n02955767', 'n04440597', 'n03021228', 'n03719560', 'n03906789', 'n03219859', 'n04605446', + 'n04445040', 'n04445154', 'n04186455', 'n04173907', 'n03998333', 'n03849943', 'n03057021', 'n03703463', 'n03770954', + 'n04122492', 'n04455579', 'n04049405', 'n03844815', 'n02921406', 'n03254046', 'n03057841', 'n03456665', 'n04506402', + 'n04365229', 'n02957008', 'n03589791', 'n03238286', 'n02867966', 'n03595264', 'n02788462', 'n03221540', 'n02864504', + 'n04123026', 'n03548320', 'n03226375', 'n03751269', 'n04222307', 'n03696909', 'n02925385', 'n02850358', 'n03219966', + 'n03720005', 'n04368496', 'n02820675', 'n03902756', 'n03891051', 'n04230387', 'n02937010', 'n03301175', 'n03829857', + 'n03604536', 'n03228254', 'n04123448', 'n03398228', 'n04003359', 'n04363777', 'n04187970', 'n02885233', 'n04252560', + 'n04357531', 'n04367950', 'n04370048', 'n04021028', 'n04502197', 'n03655072', 'n04269822', 'n03410938', 'n04027935', + 'n03006903', 'n04097866', 'n02807616', 'n03237992', 'n04085574', 'n04197391', 'n04390577', 'n03238879', 'n04602956', + 'n03978966', 'n03476542', 'n03163381', 'n03629231', 'n02943964', 'n04502197', 'n04172904', 'n04508163', 'n04223299', + 'n02944146', 'n04508489', 'n02837887', 'n04426427', 'n02854739', 'n03885028', 'n02901114', 'n03234164', 'n04514241', + 'n03387323', 'n04103665', 'n03112869', 'n03885788', 'n02863014', 'n03013580', 'n04508949', 'n03688192', 'n03673450', + 'n04509171', 'n03824381', 'n04231905', 'n02930214', 'n03920737', 'n03132776', 'n03421324', 'n03688707', 'n03585875', + 'n03404149', 'n03540090', 'n03456186', 'n03490324', 'n03796974', 'n03441112', 'n02811204', 'n03429771', 'n03447075', + 'n03616979', 'n03429682', 'n03502509', 'n03531281', 'n03124474', 'n02941845', 'n03513137', 'n04265428', 'n04229107', + 'n02811350', 'n03504205', 'n03492922', 'n02954340', 'n04129688', 'n04228693', 'n03103563', 'n03061893', 'n04387095', + 'n02831237', 'n03331077', 'n04596116', 'n03610682', 'n02799323', 'n03824284', 'n02843909', 'n04232153', 'n03065243', + 'n02816768', 'n04612026', 'n04556408', 'n03607527', 'n02776825', 'n03776167', 'n03420801', 'n03049924', 'n02941228', + 'n03439631', 'n04455048', 'n04585318', 'n03597317', 'n04432203', 'n03138669', 'n03497657', 'n04356595', 'n03950899', + 'n04354589', 'n04354589', 'n04248507', 'n03984643', 'n02987379', 'n03256631', 'n03061050', 'n02834642', 'n04482177', + 'n04456011', 'n02859184', 'n04208582', 'n02881757', 'n03404360', 'n02818135', 'n04505888', 'n04264361', 'n02945964', + 'n03237416', 'n03766322', 'n03931885', 'n03046029', 'n03937835', 'n03028785', 'n03170872', 'n03325941', 'n04441528', + 'n03607186', 'n04498389', 'n04335693', 'n03381126', 'n03540267', 'n04434932', 'n03885904', 'n02713218', 'n04378956', + 'n03622931', 'n02736798', 'n02752496', 'n04323819', 'n04360914', 'n03622931', 'n03836976', 'n02874336', 'n04532022', + 'n04059157', 'n04509592', 'n03605504', 'n04071393', 'n03402188', 'n03239259', 'n03237212', 'n03846234', 'n02811719', + 'n03615563', 'n03324928', 'n03825080', 'n04235771', 'n03824381', 'n03824999', 'n03625943', 'n02728440', 'n04603872', + 'n03660124', 'n04602956', 'n03746330', 'n02752615', 'n02887489', 'n03237416', 'n04596852', 'n03236735', 'n03619196', + 'n03788914', 'n04197878', 'n03604400', 'n02936570', 'n03013438', 'n03062015', 'n02781121', 'n02898585', 'n03201638', + 'n04397645', 'n04334105', 'n03057724', 'n03205574', 'n04355511', 'n03978815', 'n03786096', 'n04136161', 'n03817647', + 'n02908123', 'n02944075', 'n02697221', 'n04453666', 'n02861387', 'n02926426', 'n03480579', 'n02854926', 'n03418749', + 'n03467254', 'n04198562', 'n03762238', 'n04514241', 'n03464053', 'n04241249', 'n03036469', 'n03036341', 'n03797264'), + # dog n02084071 + ( + 'n02084071', 'n02084732', 'n02087122', 'n02098550', 'n02099997', 'n02100399', 'n02098806', 'n02101108', 'n02101670', + 'n02102605', 'n02102806', 'n02101861', 'n02103181', 'n02098906', 'n02099029', 'n02089232', 'n02089468', 'n02092468', + 'n02095050', 'n02095212', 'n02095412', 'n02095727', 'n02096756', 'n02093056', 'n02097786', 'n02097967', 'n02094562', + 'n02094721', 'n02094931', 'n02087314', 'n02087551', 'n02090253', 'n02090475', 'n02088839', 'n02088992', 'n02089555', + 'n02089725', 'n02092173', 'n02090827', 'n02090129', 'n02088745', 'n02110532', 'n02084861', 'n02085118', 'n02085019', + 'n02112826', 'n02085272', 'n02113335', 'n02113892', 'n02112497', 'n02103406', 'n02109150', 'n02109256', 'n02104523', + 'n02104882', 'n02103841', 'n02106966', 'n02104280', 'n02104184', 'n02106854', 'n02109687', 'n02109811', 'n02107420', + 'n02108254', 'n02109391', 'n02108672', 'n02111626', 'n02085374', 'n02086346', 'n02086753', 'n02086478', 'n01322604'), + # electronic equipment n03278248 + ( + 'n03278248', 'n02757462', 'n04077430', 'n03517760', 'n04315948', 'n04546340', 'n03436182', 'n04030965', 'n03584400', + 'n04064213', 'n04043411', 'n04413419', 'n04075291', 'n02676670', 'n03181293', 'n03143400', 'n04142731', 'n03034405', + 'n03775388', 'n04176528', 'n04060647', 'n03205304', 'n03447894', 'n04042204', 'n04137773', 'n04043733', 'n03144156', + 'n03516996', 'n03046921', 'n04027367', 'n04405907', 'n04472726', 'n04138131', 'n04406552', 'n03592931', 'n04045085', + 'n04269668', 'n04405762', 'n02705944', 'n02872529', 'n02756854', 'n03724176', 'n03424204', 'n04405540', 'n04404997', + 'n02942349', 'n02995345', 'n03916720', 'n03225777', 'n04595285', 'n03571942', 'n02909285', 'n03861048', 'n03163973', + 'n04143140', 'n03781787', 'n02962938', 'n03278914', 'n03963294', 'n03293741', 'n03656957', 'n04392526', 'n02979074', + 'n04401088', 'n03488438', 'n03306869', 'n04044498', 'n03179910', 'n04270371', 'n03842377'), + # fish n02512053 + ( + 'n02512053', 'n01316579', 'n02599958', 'n02600298', 'n02600503', 'n02600798', 'n01480516', 'n01482071', 'n01495701', + 'n01497118', 'n01497413', 'n01497738', 'n01498406', 'n01498699', 'n01498989', 'n01499396', 'n01499732', 'n01500091', + 'n01500854', 'n01500476', 'n01501160', 'n01501641', 'n01501777', 'n01501948', 'n01502101', 'n01482330', 'n01483021', + 'n01483522', 'n01483830', 'n01484097', 'n01484285', 'n01484447', 'n01484562', 'n01485479', 'n01486010', 'n01486540', + 'n01486838', 'n01487506', 'n01488038', 'n01488918', 'n01489501', 'n01489709', 'n01489920', 'n01490112', 'n01490360', + 'n01490670', 'n01491006', 'n01491661', 'n01491874', 'n01492357', 'n01492569', 'n01492708', 'n01492860', 'n01493146', + 'n01493541', 'n01493829', 'n01494041', 'n01494757', 'n01494882', 'n01495006', 'n01495493', 'n01480880', 'n01481331', + 'n01481498', 'n02512752', 'n02512830', 'n02512938', 'n02513355', 'n02530421', 'n02530637', 'n02530831', 'n02530999', + 'n02532028', 'n02532272', 'n02532451', 'n02532602', 'n02532918', 'n02532786', 'n02534734', 'n02535163', 'n02535258', + 'n02535537', 'n02535759', 'n02536165', 'n02536456', 'n02537085', 'n02537319', 'n02537716', 'n02537525', 'n02538010', + 'n02538216', 'n02538985', 'n02539424', 'n02539573', 'n02539894', 'n02567334', 'n02567633', 'n02568087', 'n02568447', + 'n02568959', 'n02569484', 'n02569631', 'n02569905', 'n02570164', 'n02586543', 'n02587051', 'n02587300', 'n02587479', + 'n02587618', 'n02587877', 'n02626762', 'n02627037', 'n02627292', 'n02627532', 'n02663849', 'n02664285', 'n02664642', + 'n02665250', 'n02513248', 'n02513560', 'n02513727', 'n02530052', 'n02530188', 'n02535080', 'n02513805', 'n02513939', + 'n02515214', 'n02515713', 'n02516188', 'n02516776', 'n02528163', 'n02538985', 'n02539424', 'n02539573', 'n02539894', + 'n01429172', 'n01438208', 'n01438581', 'n01439121', 'n01439514', 'n01439808', 'n01441117', 'n01441272', 'n01441425', + 'n01441910', 'n01442450', 'n01442710', 'n01442972', 'n01443243', 'n01443831', 'n01444339', 'n01444783', 'n01445429', + 'n01445857', 'n01446152', 'n01446589', 'n01446760', 'n01447139', 'n01447331', 'n01447658', 'n01447946', 'n01448291', + 'n01448594', 'n01448951', 'n01449374', 'n01449712', 'n01449980', 'n02583567', 'n02583890', 'n02584145', 'n02584449', + 'n02517442', 'n02517938', 'n02518622', 'n02518324', 'n02519148', 'n02519340', 'n02519472', 'n02519686', 'n02519862', + 'n02520147', 'n02520525', 'n02520810', 'n02521646', 'n02522399', 'n02522637', 'n02522722', 'n02522866', 'n02523427', + 'n02523110', 'n02523877', 'n02524202', 'n02524524', 'n02524928', 'n02524659', 'n02525382', 'n02525703', 'n02526425', + 'n02526818', 'n02527057', 'n02527271', 'n02527622', 'n02529293', 'n02529772', 'n02530421', 'n02530637', 'n02530831', + 'n02530999', 'n02532028', 'n02532272', 'n02532451', 'n02532602', 'n02532918', 'n02532786', 'n02531114', 'n02531625', + 'n02533209', 'n02533545', 'n02533834', 'n02534165', 'n02534559', 'n02534734', 'n02535163', 'n02535258', 'n02535537', + 'n02535759', 'n02536165', 'n02536456', 'n02537085', 'n02537319', 'n02537716', 'n02537525', 'n02538010', 'n02538216', + 'n02538406', 'n02538562', 'n02540412', 'n02540983', 'n02541257', 'n02541687', 'n02542017', 'n02542432', 'n02542958', + 'n02543255', 'n02543565', 'n02544274', 'n02545841', 'n02546028', 'n02546331', 'n02546627', 'n02547014', 'n01454545', + 'n01455778', 'n01456137', 'n01456454', 'n01456756', 'n01457082', 'n01457407', 'n01457852', 'n02549989', 'n02550203', + 'n02550460', 'n02550655', 'n02551134', 'n02551668', 'n02552171', 'n01450661', 'n01450950', 'n01451115', 'n01451295', + 'n01451426', 'n01451863', 'n01452345', 'n01453087', 'n01453475', 'n01453742', 'n01454856', 'n01455461', 'n01455317', + 'n02547733', 'n02548247', 'n02548689', 'n02548884', 'n02549248', 'n02549376', 'n02554730', 'n02599958', 'n02600298', + 'n02600503', 'n02600798', 'n02586543', 'n02587051', 'n02587300', 'n02587479', 'n02587618', 'n02587877', 'n02555863', + 'n02556846', 'n02557182', 'n02557318', 'n02557591', 'n02557749', 'n02558206', 'n02558860', 'n02559144', 'n02559383', + 'n02559862', 'n02560110', 'n02561108', 'n02561381', 'n02561514', 'n02561661', 'n02561803', 'n02561937', 'n02562315', + 'n02562796', 'n02562971', 'n02563079', 'n02563182', 'n01440467', 'n02563792', 'n02563949', 'n02563648', 'n02564403', + 'n02564720', 'n02564935', 'n02565072', 'n02565324', 'n02565573', 'n02564270', 'n02566109', 'n02567334', 'n02567633', + 'n02568087', 'n02568447', 'n02568959', 'n02566489', 'n02566665', 'n02570484', 'n02570838', 'n02571167', 'n02571652', + 'n02571810', 'n02572196', 'n02572484', 'n02573249', 'n02573704', 'n02574271', 'n02576223', 'n02576575', 'n02576906', + 'n02577041', 'n02577164', 'n02577403', 'n02577662', 'n02577952', 'n02578771', 'n02578928', 'n02579303', 'n02578233', + 'n02578454', 'n02579557', 'n02579762', 'n02579928', 'n02580336', 'n02580679', 'n02580830', 'n02581108', 'n02581482', + 'n02581642', 'n02581957', 'n02582220', 'n02582349', 'n02585872', 'n02586238', 'n02588286', 'n02588794', 'n02588945', + 'n02589062', 'n02589196', 'n02589316', 'n02589623', 'n02589796', 'n02590094', 'n02590495', 'n02592055', 'n02592371', + 'n02593019', 'n02590702', 'n02582721', 'n02590987', 'n02591330', 'n02592734', 'n02593453', 'n02593679', 'n02591613', + 'n02591911', 'n02593191', 'n02594250', 'n02594942', 'n02595056', 'n02595339', 'n02595702', 'n02596067', 'n02596252', + 'n02596381', 'n02596720', 'n02597004', 'n02598573', 'n02598878', 'n02597367', 'n02597608', 'n02597818', 'n02597972', + 'n02598134', 'n02599052', 'n02599557', 'n02599347', 'n02601344', 'n02601767', 'n02601921', 'n02602059', 'n02604157', + 'n02604480', 'n02604954', 'n02605316', 'n02605703', 'n02605936', 'n02606384', 'n02606751', 'n02607201', 'n02607470', + 'n02607862', 'n02608284', 'n02608547', 'n02608860', 'n02608996', 'n02609302', 'n02609823', 'n02610066', 'n02610373', + 'n02610664', 'n02610980', 'n02611561', 'n02611898', 'n02612167', 'n02613181', 'n02613572', 'n02613820', 'n02614140', + 'n02614482', 'n02614653', 'n02614978', 'n02615298', 'n02616128', 'n02616397', 'n02616851', 'n02617537', 'n02618094', + 'n02619165', 'n02619550', 'n02619861', 'n02620167', 'n02620578', 'n02621258', 'n02621908', 'n02622249', 'n02622547', + 'n02622712', 'n02622955', 'n02623445', 'n02626762', 'n02627037', 'n02627292', 'n02627532', 'n02624167', 'n02624551', + 'n02624807', 'n02624987', 'n02625258', 'n02625612', 'n02627835', 'n02628062', 'n02628259', 'n02628600', 'n02629230', + 'n02629716', 'n02630281', 'n02630615', 'n02630739', 'n02631041', 'n02632039', 'n02632494', 'n02633422', 'n02633677', + 'n02633977', 'n02634545', 'n02635154', 'n02635580', 'n02636170', 'n02636405', 'n02636550', 'n02636854', 'n02637179', + 'n02637475', 'n02637977', 'n02574910', 'n02575325', 'n02575590', 'n02602405', 'n02602760', 'n02603317', 'n02603540', + 'n02618513', 'n02618827', 'n02642107', 'n02553028', 'n02642644', 'n02643112', 'n02643316', 'n02643836', 'n02644113', + 'n02644360', 'n02644501', 'n02644665', 'n02644817', 'n02645538', 'n02645691', 'n02645953', 'n02646667', 'n02646892', + 'n02648035', 'n02648625', 'n02648916', 'n02649218', 'n02649546', 'n02650050', 'n02650413', 'n02650541', 'n02651060', + 'n02652132', 'n02652668', 'n02653145', 'n02653497', 'n02653786', 'n02654112', 'n02654425', 'n02654745', 'n02655523', + 'n02655848', 'n02656032', 'n02656301', 'n02656670', 'n02656969', 'n02657368', 'n02663849', 'n02664285', 'n02664642', + 'n02665250', 'n02657694', 'n02658079', 'n02658531', 'n02658811', 'n02659176', 'n02659478', 'n02659808', 'n02660091', + 'n02660519', 'n02660640', 'n02660208', 'n02661017', 'n02661473', 'n02661618', 'n02662239', 'n02662397', 'n02662559', + 'n02662825', 'n02662993', 'n02663211', 'n02663485', 'n02603862', 'n02638596', 'n02639087', 'n02639605', 'n02639922', + 'n02640857', 'n02640626', 'n02556373'), + # footwear n03380867 + ( + 'n03380867', 'n04199027', 'n03297103', 'n04156411', 'n03547530', 'n04027706', 'n03364008', 'n04386664', 'n03027625', + 'n04239786', 'n04122578', 'n03090710', 'n02904927', 'n02713364', 'n04593524', 'n02938218', 'n02855701', 'n03411079', + 'n04545748', 'n03868406', 'n04124370', 'n02882894', 'n02767147', 'n04570118', 'n03776877', 'n03364156', 'n03041449', + 'n03472535', 'n03967270', 'n03025250', 'n04022332', 'n04272389', 'n04546081', 'n03361550', 'n04241394', 'n03798061', + 'n02873733', 'n02872752', 'n04228581', 'n04089666', 'n03600475', 'n03516844', 'n03521544', 'n04542715', 'n02925666', + 'n04116294', 'n03865949', 'n02735538'), + # fruit n13134947 + ( + 'n13134947', 'n07705931', 'n07738105', 'n07738224', 'n07739035', 'n07739125', 'n07739344', 'n07739506', 'n07739923', + 'n07740033', 'n07740220', 'n07740342', 'n07740461', 'n07740597', 'n07740744', 'n07740855', 'n07740954', 'n07741138', + 'n07741235', 'n07741357', 'n07741461', 'n07740115', 'n07741623', 'n07741706', 'n07741804', 'n07741888', 'n07742012', + 'n07742224', 'n07742415', 'n07742513', 'n07742605', 'n07742704', 'n07743224', 'n07743384', 'n07743544', 'n07743723', + 'n07743902', 'n07744057', 'n07744246', 'n07744430', 'n07744559', 'n07744682', 'n07744811', 'n07745046', 'n07745197', + 'n07745357', 'n07745466', 'n07745661', 'n07746038', 'n07746186', 'n07746334', 'n07767171', 'n07746551', 'n07746749', + 'n07746910', 'n07747055', 'n07747811', 'n07748753', 'n07748912', 'n07749095', 'n07749192', 'n07749312', 'n07747951', + 'n07748157', 'n07748276', 'n07748416', 'n07749446', 'n07749731', 'n07749870', 'n07749969', 'n07750146', 'n07750299', + 'n07750449', 'n07748574', 'n07750872', 'n07751004', 'n07751148', 'n07751280', 'n07751451', 'n07751737', 'n07751858', + 'n07751977', 'n07752109', 'n07752264', 'n07752377', 'n07752514', 'n07752602', 'n07752664', 'n07752782', 'n07752874', + 'n07752966', 'n07753448', 'n07753743', 'n07753980', 'n07754155', 'n07754279', 'n07754451', 'n07755262', 'n07755411', + 'n07755619', 'n07755707', 'n07755929', 'n07756096', 'n07756325', 'n07756499', 'n07756838', 'n07756641', 'n07756951', + 'n07757132', 'n07757312', 'n07757511', 'n07757602', 'n07757753', 'n07757874', 'n07757990', 'n07758125', 'n07758260', + 'n07758407', 'n07758680', 'n07759424', 'n07759576', 'n07759691', 'n07758950', 'n07759194', 'n07759324', 'n07759816', + 'n07760070', 'n07760153', 'n07760297', 'n07760395', 'n07760501', 'n07760673', 'n07760755', 'n07761141', 'n07761309', + 'n07761611', 'n07761777', 'n07761954', 'n07762114', 'n07762244', 'n07762373', 'n07762534', 'n07762740', 'n07762913', + 'n07763107', 'n07763290', 'n07763483', 'n07763629', 'n07763792', 'n07763987', 'n07764155', 'n07764315', 'n07764486', + 'n07764630', 'n07764847', 'n07765073', 'n07765208', 'n07765361', 'n07765517', 'n07765612', 'n07765728', 'n07765862', + 'n07765999', 'n07766173', 'n07766409', 'n07766530', 'n07766723', 'n07766891', 'n07767002', 'n07767847', 'n07768068', + 'n07768139', 'n07768230', 'n07768318', 'n07768590', 'n07768858', 'n07769102', 'n07769306', 'n07769584', 'n07769731', + 'n07769886', 'n07770034', 'n07770180', 'n07770439', 'n11636835', 'n11700279', 'n12036067', 'n12036226', 'n12158031', + 'n12815838', 'n12162758', 'n12193334', 'n12301445', 'n12642090', 'n12644283', 'n12647787', 'n12650805', 'n12658481', + 'n12144313', 'n13135692', 'n13135832', 'n07770571', 'n07770763', 'n07770869', 'n07775197', 'n07814634', 'n07929351', + 'n11685091', 'n11689197', 'n11689367', 'n11689483', 'n11689678', 'n11689815', 'n11689957', 'n15086247', 'n11946313', + 'n12156819', 'n11823305', 'n12123648', 'n12142357', 'n12157056', 'n12157179', 'n12585373', 'n12592839', 'n12593341', + 'n12696830', 'n12928819', 'n13136316', 'n11750173', 'n11766046', 'n12487058', 'n12493426', 'n12532564', 'n12576323', + 'n13136556', 'n07737081', 'n07737594', 'n07737745', 'n07750586', 'n07750736', 'n07769465', 'n07771082', 'n07771212', + 'n07771539', 'n07771405', 'n07771731', 'n07771891', 'n07772026', 'n07772147', 'n07772274', 'n07772413', 'n07772788', + 'n07772935', 'n07774182', 'n07774295', 'n07774596', 'n07774719', 'n07774842', 'n07775050', 'n11612235', 'n12197601', + 'n12280364', 'n12590715', 'n13136781', 'n13137409', 'n07743902', 'n11723986', 'n13137951', 'n13137672', 'n13137225', + 'n13138308', 'n07751004', 'n07767344', 'n07767709', 'n07767549', 'n13138658', 'n07744811', 'n13138155', 'n13138842', + 'n07739125', 'n07739344', 'n13139055', 'n11748002', 'n12515925', 'n12544539', 'n12560282', 'n12560621', 'n12561594', + 'n12578916', 'n11748811', 'n11766432', 'n12172364', 'n13139321', 'n13139482', 'n13140367', 'n13141415', 'n13150378', + 'n13150592'), + # fungus n12992868 + ( + 'n12992868', 'n13035241', 'n13035707', 'n13036312', 'n13035925', 'n13036116', 'n13036804', 'n12982468', 'n12982590', + 'n12985420', 'n13079419', 'n13079567', 'n13001930', 'n13023134', 'n12987056', 'n12987535', 'n12988158', 'n12988341', + 'n12988572', 'n12991184', 'n12989938', 'n12991837', 'n12989007', 'n12987423', 'n12992177', 'n12990597', 'n13027879', + 'n13081999', 'n12979829', 'n13077295', 'n12963628', 'n12980840', 'n12981301', 'n12981443', 'n12981086', 'n12969131', + 'n12969670', 'n12969425', 'n12969927', 'n13025647', 'n13026015', 'n13025854', 'n13046669', 'n13081229', 'n13078021', + 'n13002209', 'n13079073', 'n13015509', 'n13060190', 'n13062421', 'n13061348', 'n13061704', 'n13061471', 'n13061172', + 'n13080306', 'n12964920', 'n12970733', 'n13080866', 'n13018906', 'n13048447', 'n12966945', 'n13066129', 'n13067672', + 'n13067532', 'n13068434', 'n13067191', 'n13067330', 'n13066979', 'n13068255', 'n13068917', 'n13069224', 'n13068735', + 'n13066448', 'n12971400', 'n12971804', 'n12972136', 'n13046130', 'n13045210', 'n13045975', 'n12978076', 'n12965626', + 'n12965951', 'n13034555', 'n13045594', 'n12985773', 'n13042982', 'n13041312', 'n13040629', 'n13040796', 'n13082568', + 'n13016076', 'n13024012', 'n12983873', 'n13024500', 'n13024653', 'n12983961', 'n12984489', 'n12984595', 'n12984267', + 'n13028611', 'n13029122', 'n13031323', 'n13029326', 'n13031474', 'n13029760', 'n13031193', 'n13030616', 'n13029610', + 'n13030852', 'n13028937', 'n12982915', 'n13047862', 'n13016289', 'n13077033', 'n12973791', 'n12973937', 'n12980080', + 'n12973443', 'n12981954', 'n12995601', 'n12966804', 'n13063269', 'n13065514', 'n13065089', 'n13064111', 'n13064457', + 'n13035389', 'n13038577', 'n13043926', 'n13044375', 'n13027557', 'n12968136', 'n12968309', 'n13037585', 'n13037805', + 'n13038376', 'n13038068', 'n13038744', 'n13069773', 'n13054073', 'n13039349', 'n12970193', 'n12970293', 'n13042316', + 'n13042134', 'n13041943', 'n12986227', 'n13056799', 'n13059298', 'n13056135', 'n13058272', 'n13056349', 'n13057054', + 'n13055577', 'n13057422', 'n13058608', 'n13058037', 'n13055792', 'n13060017', 'n13055423', 'n13055949', 'n13057242', + 'n13057639', 'n13059657', 'n13056607', 'n12997654', 'n13049953', 'n13052931', 'n13050940', 'n13053608', 'n13050705', + 'n13050397', 'n13051346', 'n13052248', 'n13052014', 'n13011595', 'n12997919', 'n13032115', 'n13033879', 'n13032381', + 'n13033577', 'n13032618', 'n13034062', 'n13032923', 'n13033134', 'n13033396', 'n13002750', 'n13232106', 'n13003254', + 'n13001366', 'n13008839', 'n13004826', 'n13075020', 'n13009656', 'n13009244', 'n13020481', 'n13003712', 'n13003522', + 'n13018407', 'n13004640', 'n13022210', 'n13017240', 'n13002925', 'n13074814', 'n13008157', 'n13020964', 'n13017979', + 'n13232363', 'n13017610', 'n13076405', 'n13012469', 'n13076831', 'n13017439', 'n13007629', 'n13232779', 'n13020191', + 'n13004423', 'n13013534', 'n13014097', 'n13014581', 'n13014741', 'n13014879', 'n13014409', 'n13013965', 'n13014265', + 'n13006631', 'n13005984', 'n13010951', 'n13008485', 'n13021867', 'n13000891', 'n13013764', 'n13021543', 'n13017789', + 'n13009429', 'n13005329', 'n13006171', 'n13010694', 'n13075441', 'n13001206', 'n13017102', 'n13076041', 'n13018088', + 'n13021332', 'n13018232', 'n13070308', 'n13072350', 'n13072209', 'n13070875', 'n13072863', 'n13072031', 'n13073703', + 'n13072528', 'n13071815', 'n13072706', 'n13073055', 'n13071371', 'n13071553', 'n13075684', 'n13075847', 'n13004992', + 'n13021689', 'n13231678', 'n13001041', 'n13009085', 'n13008689', 'n13001529', 'n13012253', 'n13231919', 'n13019496', + 'n13074619', 'n13006894', 'n13019835', 'n13075272', 'n13003061', 'n13012973', 'n13011221', 'n13019643', 'n13076643', + 'n13008315', 'n13021166', 'n13007417', 'n13015688', 'n12974987', 'n12975804', 'n12976198', 'n12976554', 'n12983654', + 'n12979316', 'n13226871', 'n13034788', 'n12983048'), + # geological formation n09287968 + ( + 'n09287968', 'n09201998', 'n09217230', 'n09393524', 'n09238926', 'n09239302', 'n09257843', 'n09294877', 'n09259025', + 'n09398677', 'n09264803', 'n09266604', 'n09283866', 'n09309292', 'n09289331', 'n09194227', 'n09255070', 'n09396608', + 'n09391774', 'n09308572', 'n09295210', 'n09308743', 'n09309046', 'n09309168', 'n09331251', 'n09348460', 'n09357447', + 'n09362316', 'n09366017', 'n09215437', 'n09245515', 'n09421031', 'n09457979', 'n09330378', 'n09376526', 'n09186592', + 'n09415671', 'n09448690', 'n09259219', 'n09249155', 'n09344324', 'n09304750', 'n09230041', 'n09474765', 'n09290350', + 'n09214269', 'n09226869', 'n09268007', 'n09402944', 'n09422190', 'n09425019', 'n09454744', 'n09398076', 'n09403086', + 'n09344198', 'n09335809', 'n09422631', 'n09435739', 'n09452291', 'n09262690', 'n09289596', 'n09300306', 'n09206896', + 'n09269882', 'n09474010', 'n09305031', 'n09375606', 'n09405787', 'n09290444', 'n09295946', 'n09233446', 'n09410224', + 'n09366317', 'n09302616', 'n09269341', 'n09453008', 'n09351905', 'n09456207', 'n09303008', 'n09230202', 'n09283405', + 'n09326662', 'n09199101', 'n09327077', 'n09357346', 'n09459979', 'n09359803', 'n09218641', 'n09362945', 'n09396465', + 'n09409512', 'n09213434', 'n09224725', 'n09421799', 'n09433312', 'n09214060', 'n09270735', 'n09429630', 'n09274305', + 'n09337253', 'n09219233', 'n09406793', 'n09210862', 'n09214916', 'n09411295', 'n09452760', 'n09376786', 'n09403734', + 'n09409752', 'n09205509', 'n09433442', 'n08596076', 'n09335693', 'n09428628', 'n09458269', 'n09447666', 'n09437454', + 'n09206985', 'n09466678', 'n09213565', 'n09475925', 'n09415584', 'n09233603', 'n09248153', 'n09265620', 'n09269472', + 'n09445008', 'n09274152', 'n09303528', 'n09228055', 'n09361517', 'n09391644', 'n09436444', 'n09305898', 'n09454153', + 'n09470222', 'n09472413', 'n09344724', 'n09231117', 'n09474412', 'n09476123'), + # hoofed animal n02370806 + ( + 'n02370806', 'n02373336', 'n02374149', 'n02390258', 'n02391617', 'n02390101', 'n02389346', 'n02389559', 'n02389779', + 'n02389865', 'n02390015', 'n02389943', 'n02390454', 'n02390834', 'n02390938', 'n02390640', 'n02390738', 'n02391234', + 'n02391373', 'n02391508', 'n02374451', 'n02382948', 'n02385776', 'n02385580', 'n02383231', 'n02385098', 'n02388143', + 'n02385214', 'n02385676', 'n02388276', 'n02388453', 'n02375862', 'n02385898', 'n02389261', 'n02382204', 'n02380335', + 'n02382039', 'n02380583', 'n02380745', 'n02381460', 'n02381831', 'n02381609', 'n02386310', 'n02386496', 'n02386853', + 'n02387452', 'n02387346', 'n02387887', 'n02386968', 'n02387093', 'n02386746', 'n02382338', 'n02387254', 'n02382132', + 'n02375302', 'n02388917', 'n02388588', 'n02376918', 'n02377181', 'n02377291', 'n02377388', 'n02387983', 'n02380464', + 'n02388832', 'n02386014', 'n02386224', 'n02386141', 'n02382437', 'n02382850', 'n02382750', 'n02382635', 'n02377480', + 'n02377603', 'n02389128', 'n02387722', 'n02375757', 'n02377703', 'n02379430', 'n02381364', 'n02378870', 'n02379908', + 'n02379081', 'n02379743', 'n02379630', 'n02378415', 'n02378625', 'n02378755', 'n02378541', 'n02378299', 'n02378969', + 'n02381004', 'n02380052', 'n02381119', 'n02379183', 'n02381261', 'n02379329', 'n02378149', 'n02388735', 'n02375438', + 'n02384741', 'n02393580', 'n02393807', 'n02393940', 'n02391994', 'n02392824', 'n02392434', 'n02393161', 'n02392555', + 'n02394477', 'n02438580', 'n02397529', 'n02397744', 'n02397987', 'n02437136', 'n02437482', 'n02399000', 'n02429456', + 'n02401031', 'n02418064', 'n02418465', 'n02419336', 'n02419056', 'n02419634', 'n02418770', 'n02411206', 'n02411705', + 'n02412210', 'n02413131', 'n02413917', 'n02414043', 'n02414442', 'n02413593', 'n02414209', 'n02414290', 'n02413717', + 'n02413824', 'n02413484', 'n02411999', 'n02413050', 'n02419796', 'n02424085', 'n02424695', 'n02425228', 'n02424909', + 'n02423218', 'n02423589', 'n02423362', 'n02421449', 'n02425887', 'n02421136', 'n02421792', 'n02427724', 'n02427576', + 'n02427470', 'n02428349', 'n02428508', 'n02426813', 'n02427032', 'n02427183', 'n02425086', 'n02424305', 'n02424589', + 'n02424486', 'n02420828', 'n02422391', 'n02428089', 'n02426176', 'n02420509', 'n02426481', 'n02425532', 'n02414578', + 'n02414763', 'n02416104', 'n02415130', 'n02414904', 'n02415435', 'n02415829', 'n02415253', 'n02416519', 'n02417534', + 'n02417785', 'n02417663', 'n02417070', 'n02417387', 'n02417242', 'n02416820', 'n02416964', 'n02416880', 'n02410702', + 'n02410900', 'n02407959', 'n02409202', 'n02409508', 'n02409038', 'n02408817', 'n02408660', 'n02402010', 'n02404573', + 'n02404906', 'n02402425', 'n02403231', 'n02403153', 'n02403325', 'n02405692', 'n02406859', 'n02403454', 'n02405577', + 'n02406952', 'n02406046', 'n02404186', 'n02406174', 'n02402175', 'n02405101', 'n02405302', 'n02405440', 'n02409870', + 'n02428842', 'n02430045', 'n02431976', 'n02430830', 'n02435216', 'n02432511', 'n02432704', 'n02435517', 'n02434712', + 'n02431122', 'n02431542', 'n02431337', 'n02431441', 'n02432291', 'n02433318', 'n02433925', 'n02434190', 'n02434415', + 'n02431628', 'n02430748', 'n02432983', 'n02431785', 'n02434954', 'n02433546', 'n02433729', 'n02435853', 'n02436224', + 'n02436353', 'n02436645', 'n02439033', 'n02439398', 'n02437971', 'n02438272', 'n02438173', 'n02395003', 'n02395931', + 'n02396014', 'n02396088', 'n02396796', 'n02396157', 'n02372140'), + # insect n02159955 + ( + 'n02159955', 'n02160947', 'n02161225', 'n02161338', 'n02161457', 'n02161588', 'n02162561', 'n02163008', 'n02163297', + 'n02164464', 'n02165877', 'n02166229', 'n02166567', 'n02166826', 'n02167505', 'n02167820', 'n02167944', 'n02168245', + 'n02168427', 'n02169023', 'n02169218', 'n02169705', 'n02169974', 'n02170400', 'n02170599', 'n02170738', 'n02170993', + 'n02171164', 'n02171453', 'n02171869', 'n02172518', 'n02172678', 'n02172761', 'n02172870', 'n02173113', 'n02173373', + 'n02173784', 'n02174355', 'n02174659', 'n02175014', 'n02175569', 'n02175916', 'n02176261', 'n02176439', 'n02176747', + 'n02177196', 'n02177506', 'n02177775', 'n02178411', 'n02178717', 'n02181235', 'n02181724', 'n02182045', 'n02182355', + 'n02182642', 'n02182930', 'n02179012', 'n02179192', 'n02179340', 'n02180233', 'n02179891', 'n02180427', 'n02180875', + 'n02183096', 'n02183507', 'n02183857', 'n02184473', 'n02184589', 'n02184720', 'n02185167', 'n02185481', 'n02186153', + 'n02186717', 'n02187150', 'n02187279', 'n02187554', 'n02187900', 'n02188699', 'n02189363', 'n02189670', 'n02190790', + 'n02191273', 'n02191773', 'n02191979', 'n02192252', 'n02192513', 'n02192814', 'n02193009', 'n02193163', 'n02194249', + 'n02194750', 'n02195091', 'n02195526', 'n02195819', 'n02199502', 'n02196119', 'n02196344', 'n02196896', 'n02197185', + 'n02197689', 'n02197877', 'n02198532', 'n02198859', 'n02199170', 'n02200198', 'n02200630', 'n02200850', 'n02201000', + 'n02201497', 'n02201626', 'n02202006', 'n02202124', 'n02202287', 'n02202678', 'n02203152', 'n02203978', 'n02204249', + 'n02205673', 'n02203592', 'n02204722', 'n02204907', 'n02205219', 'n02198129', 'n02206270', 'n02220055', 'n02220225', + 'n02220518', 'n02220804', 'n02221083', 'n02221414', 'n02221571', 'n02221715', 'n02221820', 'n02222035', 'n02222582', + 'n02222321', 'n02207179', 'n02208280', 'n02208498', 'n02208848', 'n02208979', 'n02209111', 'n02209354', 'n02209624', + 'n02209964', 'n02210427', 'n02210921', 'n02211444', 'n02211627', 'n02211896', 'n02212062', 'n02212602', 'n02212958', + 'n02214096', 'n02213107', 'n02213239', 'n02213663', 'n02213788', 'n02213543', 'n02214341', 'n02214499', 'n02214773', + 'n02215161', 'n02215621', 'n02215770', 'n02216211', 'n02216365', 'n02216740', 'n02214660', 'n02217563', 'n02218134', + 'n02218371', 'n02218713', 'n02219015', 'n02207449', 'n02207805', 'n02207647', 'n02223266', 'n02223520', 'n02225798', + 'n02224023', 'n02224713', 'n02225081', 'n02226183', 'n02226821', 'n02226970', 'n02227247', 'n02227604', 'n02227966', + 'n02228341', 'n02228697', 'n02229156', 'n02229765', 'n02230023', 'n02230187', 'n02230480', 'n02230634', 'n02231052', + 'n02231803', 'n02232223', 'n02233943', 'n02234355', 'n02234570', 'n02234848', 'n02235205', 'n02236241', 'n02236355', + 'n02236896', 'n02237424', 'n02237581', 'n02237868', 'n02238235', 'n02238358', 'n02238594', 'n02238887', 'n02239192', + 'n02239528', 'n02239774', 'n02240068', 'n02240517', 'n02241008', 'n02241426', 'n02241569', 'n02241799', 'n02242137', + 'n02242455', 'n02243209', 'n02243562', 'n02243878', 'n02244173', 'n02244515', 'n02244797', 'n02245111', 'n02245443', + 'n02246011', 'n02246628', 'n02246941', 'n02247216', 'n02247511', 'n02247655', 'n02248062', 'n02248368', 'n02248510', + 'n02248887', 'n02249134', 'n02249515', 'n02249809', 'n02250280', 'n02250822', 'n02251067', 'n02251233', 'n02251593', + 'n02251775', 'n02252226', 'n02252799', 'n02252972', 'n02253127', 'n02253264', 'n02253494', 'n02253715', 'n02253913', + 'n02254246', 'n02254697', 'n02254901', 'n02255023', 'n02255391', 'n02256172', 'n02257003', 'n02257284', 'n02257715', + 'n02257985', 'n02258198', 'n02258508', 'n02258629', 'n02259377', 'n02259708', 'n02259987', 'n02260421', 'n02260863', + 'n02261063', 'n02261419', 'n02261757', 'n02262178', 'n02262449', 'n02262803', 'n02263378', 'n02264021', 'n02264885', + 'n02265330', 'n02266050', 'n02266421', 'n02266864', 'n02267208', 'n02267483', 'n02268148', 'n02269196', 'n02269340', + 'n02270011', 'n02270200', 'n02270945', 'n02270623', 'n02271222', 'n02271570', 'n02271897', 'n02272286', 'n02272552', + 'n02272871', 'n02273392', 'n02274024', 'n02274259', 'n02274822', 'n02275560', 'n02275773', 'n02276078', 'n02276355', + 'n02276749', 'n02276902', 'n02277094', 'n02277268', 'n02277422', 'n02278024', 'n02278210', 'n02278463', 'n02278839', + 'n02278980', 'n02279257', 'n02279637', 'n02280458', 'n02281015', 'n02281136', 'n02281267', 'n02282257', 'n02282385', + 'n02282553', 'n02282903', 'n02283077', 'n02283201', 'n02283617', 'n02283951', 'n02284224', 'n02284611', 'n02284884', + 'n02285179', 'n02285548', 'n02286089', 'n02286425', 'n02286654', 'n02287004', 'n02287352', 'n02287622', 'n02288789', + 'n02289307', 'n02289610', 'n02289988', 'n02290340', 'n02290664', 'n02290870', 'n02291220', 'n02291572', 'n02291748', + 'n02292085', 'n02292401', 'n02292692', 'n02293352', 'n02293868', 'n02294097', 'n02294407', 'n02295064', 'n02295870', + 'n02296021', 'n02296276', 'n02296612', 'n02297294', 'n02297819', 'n02298095', 'n02298541', 'n02299039', 'n02299378', + 'n02299846', 'n02300173', 'n02300554', 'n02301452', 'n02301935', 'n02302244', 'n02302459', 'n02303585', 'n02305085', + 'n02302969', 'n02303284', 'n02304036', 'n02304432', 'n02304657', 'n02304797', 'n02305407', 'n02305636', 'n02305929', + 'n02306433', 'n02306825', 'n02307515', 'n02307910', 'n02308471', 'n02308618', 'n02307176', 'n02312427', 'n02312640', + 'n02312912', 'n02313008', 'n02207345'), + # musical instrument n03800933 + ( + 'n03800933', 'n02795978', 'n02803349', 'n02803934', 'n02804123', 'n02804252', 'n03301568', 'n03512030', 'n02940706', + 'n03279153', 'n03273551', 'n04376400', 'n04419642', 'n03597916', 'n03614532', 'n04376400', 'n02990758', 'n03038870', + 'n03039015', 'n03496296', 'n04278247', 'n04537436', 'n03928116', 'n02766792', 'n03086457', 'n03738066', 'n04278353', + 'n03801353', 'n03915437', 'n03928116', 'n02766792', 'n03086457', 'n03738066', 'n04278353', 'n02869249', 'n02965529', + 'n03483230', 'n03157348', 'n03518829', 'n04614844', 'n02803666', 'n02869737', 'n04249415', 'n04382334', 'n04387201', + 'n04387400', 'n04410086', 'n04436542', 'n03440682', 'n03612965', 'n03633632', 'n04049753', 'n04480853', 'n04532831', + 'n04338517', 'n03038870', 'n03039015', 'n03496296', 'n04278247', 'n04537436', 'n03928116', 'n02766792', 'n03086457', + 'n03738066', 'n04278353', 'n02880546', 'n02803934', 'n04536153', 'n04536465', 'n04536595', 'n04536765', 'n04536335', + 'n02700895', 'n03465500', 'n04330998', 'n03025886', 'n02776978', 'n02682407', 'n03699280', 'n03698360', 'n03716966', + 'n03716887', 'n03254862', 'n03467517', 'n02804123', 'n03035832', 'n03499907', 'n04506289', 'n03628215', 'n04016846', + 'n04132603', 'n04224842', 'n04615226', 'n03254737', 'n04586932', 'n02891788', 'n02804252', 'n03301568', 'n03512030', + 'n02793089', 'n02912894', 'n04174500', 'n03369276', 'n04141198', 'n04123123', 'n03393324', 'n02701730', 'n03086670', + 'n02786736', 'n03494537', 'n03609397', 'n03854815', 'n03254625', 'n04067231', 'n04542595', 'n04263950', 'n04542474', + 'n04067143', 'n03945615', 'n02775483', 'n03800371', 'n03006626', 'n03245724', 'n03343354', 'n03355468', 'n03945459', + 'n03912218', 'n03950647', 'n03989777', 'n04579667', 'n04598582', 'n02817799', 'n03228016', 'n03096439', 'n04410365', + 'n03288742', 'n03628831', 'n03510866', 'n03800485', 'n03839172', 'n03839276', 'n04186624', 'n03393199', 'n04222847', + 'n03037709', 'n02803539', 'n02803809', 'n02834027', 'n03537550', 'n03334492', 'n03831757', 'n03929091'), + # primate n02469914 + ( + 'n02469914', 'n02496913', 'n02499808', 'n02499022', 'n02498153', 'n02499568', 'n02499316', 'n02498743', 'n02500596', + 'n02470238', 'n02470709', 'n02471300', 'n02478875', 'n02479332', 'n02471762', 'n02473983', 'n02478239', 'n02472293', + 'n02472987', 'n02474777', 'n02475358', 'n02475669', 'n02473307', 'n02473720', 'n02473857', 'n02475078', 'n02474605', + 'n02474110', 'n10528148', 'n02474282', 'n02476219', 'n02476870', 'n02477187', 'n02477329', 'n02477516', 'n02476567', + 'n02477028', 'n02477782', 'n02473554', 'n02501583', 'n02502006', 'n02501923', 'n02496052', 'n02470325', 'n02470899', + 'n02483092', 'n02480153', 'n02484322', 'n02484473', 'n02485988', 'n02488894', 'n02485225', 'n02485688', 'n02485371', + 'n02485536', 'n02486657', 'n02487079', 'n02486908', 'n02487847', 'n02488003', 'n02487547', 'n02487675', 'n02488415', + 'n02489589', 'n02494383', 'n02493224', 'n02492948', 'n02492356', 'n02490597', 'n02490811', 'n02491107'), + # reptile n01661091 + ( + 'n01661091', 'n01661592', 'n01662622', 'n01662784', 'n01663401', 'n01663782', 'n01664369', 'n01664492', 'n01664674', + 'n01664990', 'n01665932', 'n01666585', 'n01666228', 'n01667432', 'n01668091', 'n01668436', 'n01668665', 'n01668892', + 'n01669372', 'n01669654', 'n01670092', 'n01670535', 'n01670802', 'n01671125', 'n01671479', 'n01671705', 'n01672032', + 'n01672432', 'n01672611', 'n01661818', 'n01673282', 'n01674216', 'n01674464', 'n01674990', 'n01675352', 'n01676755', + 'n01677747', 'n01678043', 'n01678343', 'n01678657', 'n01679005', 'n01679307', 'n01679626', 'n01679962', 'n01680264', + 'n01680478', 'n01680655', 'n01680813', 'n01680983', 'n01681328', 'n01681653', 'n01681940', 'n01682172', 'n01682435', + 'n01683201', 'n01683558', 'n01684133', 'n01684578', 'n01684741', 'n01685439', 'n01686044', 'n01686220', 'n01686403', + 'n01686609', 'n01686808', 'n01687128', 'n01687290', 'n01687665', 'n01688961', 'n01689081', 'n01689411', 'n01690149', + 'n01690466', 'n01691217', 'n01691652', 'n01691951', 'n01692523', 'n01692864', 'n01693175', 'n01693783', 'n01694311', + 'n01694709', 'n01694955', 'n01701551', 'n01701859', 'n01702256', 'n01702479', 'n01703011', 'n01703161', 'n01703569', + 'n01704103', 'n01704626', 'n01705010', 'n01705591', 'n01705934', 'n01707294', 'n01708106', 'n01708998', 'n01709484', + 'n01709876', 'n01712008', 'n01712752', 'n01713170', 'n01713764', 'n01714231', 'n01715888', 'n01717016', 'n01717229', + 'n01717467', 'n01718096', 'n01718414', 'n01710177', 'n01711160', 'n01722998', 'n01723579', 'n01724231', 'n01724840', + 'n01725086', 'n01725713', 'n01726203', 'n01696633', 'n01698434', 'n01698782', 'n01697178', 'n01697611', 'n01697749', + 'n01697978', 'n01699040', 'n01699254', 'n01699675', 'n01726692', 'n01727646', 'n01728266', 'n01729672', 'n01730185', + 'n01730307', 'n01730563', 'n01730812', 'n01730960', 'n01731137', 'n01731277', 'n01731545', 'n01731764', 'n01731941', + 'n01732093', 'n01732244', 'n01732614', 'n01732789', 'n01732989', 'n01733214', 'n01733466', 'n01733757', 'n01733957', + 'n01734104', 'n01734637', 'n01734808', 'n01735439', 'n01735577', 'n01735728', 'n01736032', 'n01736375', 'n01736796', + 'n01737472', 'n01737728', 'n01737875', 'n01738065', 'n01738306', 'n01738601', 'n01738731', 'n01739094', 'n01739647', + 'n01739871', 'n01741232', 'n01741442', 'n01740551', 'n01740885', 'n01741562', 'n01741943', 'n01742447', 'n01742821', + 'n01743086', 'n01743605', 'n01743936', 'n01744100', 'n01744270', 'n01744555', 'n01745125', 'n01745484', 'n01745902', + 'n01746191', 'n01746359', 'n01746952', 'n01747285', 'n01747589', 'n01747885', 'n01748389', 'n01748686', 'n01748906', + 'n01749244', 'n01749582', 'n01749742', 'n01750167', 'n01750437', 'n01750743', 'n01751036', 'n01751215', 'n01751472', + 'n01752165', 'n01752585', 'n01752736', 'n01753032', 'n01753180', 'n01753959', 'n01754370', 'n01754533', 'n01754876', + 'n01755740', 'n01755952', 'n01756089', 'n01756508', 'n01756733', 'n01756916', 'n01757115', 'n01757343', 'n01757677', + 'n01757901', 'n01758141', 'n01662060', 'n01719403', 'n01721174', 'n01721898', 'n01722670'), + # utensil n04516672 + ( + 'n04516672', 'n02997607', 'n03262519', 'n03173270', 'n03317788', 'n03713436', 'n04414101', 'n04414319', 'n03984234', + 'n03018209', 'n02869155', 'n03125588', 'n04282992', 'n03992703', 'n02684248', 'n03698226', 'n04570214', 'n04326676', + 'n03104512', 'n03403643', 'n03621049', 'n03012499', 'n03101517', 'n03101986', 'n02805283', 'n02999138', 'n03101156', + 'n03983712', 'n03101796', 'n03284981', 'n03047799', 'n03453231', 'n03458422', 'n03459328', 'n03880531', 'n03242390', + 'n03271765', 'n04275283', 'n03846677', 'n03900301', 'n04097760', 'n04138977', 'n03226254', 'n04317325', 'n03972372', + 'n03990474', 'n03242506', 'n03915118', 'n03216562', 'n03259401', 'n03612814', 'n04397768', 'n03992975', 'n04139140', + 'n04324297', 'n04516214', 'n03064250', 'n04132985', 'n04399158', 'n04229959', 'n04309548', 'n04500060', 'n03352961', + 'n03881305', 'n03454885', 'n03621377', 'n03724417', 'n03767966', 'n03775199', 'n02850732', 'n03266371', 'n03272940', + 'n04578934', 'n04088441', 'n04103206', 'n04293119', 'n04059516', 'n04396902', 'n04175039'), + # vegetable n07707451 + ( + 'n07707451', 'n07708124', 'n07708398', 'n07708798', 'n07709046', 'n07724943', 'n07725158', 'n07726796', 'n07727048', + 'n07727140', 'n07727252', 'n07727377', 'n07727458', 'n07727578', 'n07727868', 'n07728053', 'n07728181', 'n07728284', + 'n07728391', 'n07728585', 'n07728708', 'n07728804', 'n07729000', 'n07729142', 'n07729225', 'n07729384', 'n07729828', + 'n07727741', 'n07729485', 'n07729926', 'n07725255', 'n07725376', 'n07725531', 'n07725789', 'n07725888', 'n07726009', + 'n07725663', 'n07726230', 'n07726386', 'n07726095', 'n07726672', 'n07709172', 'n07709333', 'n07709701', 'n07719437', + 'n07719616', 'n07719756', 'n07719980', 'n07720277', 'n07723330', 'n07723559', 'n07723753', 'n07723968', 'n07724078', + 'n07724173', 'n07724269', 'n07724492', 'n07724654', 'n07724819', 'n07730855', 'n07731006', 'n07731587', 'n07731767', + 'n07732747', 'n07732904', 'n07733005', 'n07733124', 'n07820036', 'n07733217', 'n07733712', 'n07733847', 'n07736256', + 'n07736371', 'n07736527', 'n07736692', 'n07710007', 'n07710616', 'n07710952', 'n07711371', 'n07711080', 'n07711232', + 'n07711799', 'n07713074', 'n07720442', 'n07720615', 'n07721018', 'n07721118', 'n07721195', 'n07721325', 'n07722052', + 'n07721456', 'n07721678', 'n07721833', 'n07721942', 'n07734017', 'n07734183', 'n07734292', 'n07734417', 'n07734555', + 'n07710283', 'n07710616', 'n07710952', 'n07711371', 'n07711907', 'n07712063', 'n07712267', 'n07719058', 'n07719839', + 'n07720084', 'n07720185', 'n07730207', 'n07730708', 'n07735052', 'n07735179', 'n07735294', 'n07735404', 'n07735687', + 'n07735803', 'n07735981', 'n07736087', 'n07736813', 'n07713267', 'n07713395', 'n07735687', 'n07713763', 'n07713895', + 'n07714078', 'n07714188', 'n07714287', 'n07714448', 'n07714895', 'n07714802', 'n07715221', 'n07715407', 'n07733567', + 'n07715561', 'n07717070', 'n07717714', 'n07717858', 'n07718068', 'n07718195', 'n07718329', 'n07715721', 'n07716034', + 'n07716203', 'n07716504', 'n07716649', 'n07716750', 'n07718671', 'n07718920', 'n07719213', 'n07719330', 'n07722217', + 'n07722390', 'n07722485', 'n07722666', 'n07722763', 'n07722888', 'n07723177', 'n07723039', 'n07730406', 'n07730562', + 'n07733394', 'n07735510', 'n07736971', 'n07768423', 'n07817871'), + # vehicle n04576211 + ( + 'n04576211', 'n02766534', 'n02804515', 'n02834778', 'n03853924', 'n04026813', 'n04126066', 'n04524716', 'n02869563', + 'n02959942', 'n02775039', 'n02932523', 'n03053976', 'n02885108', 'n04322692', 'n02986066', 'n03056097', 'n03360731', + 'n04070964', 'n04389521', 'n03465320', 'n03483971', 'n03710294', 'n03200357', 'n03828020', 'n04020912', 'n04236001', + 'n04246855', 'n04409279', 'n03484083', 'n02729222', 'n03490119', 'n03648431', 'n04176068', 'n04397027', 'n03897634', + 'n03538634', 'n02968473', 'n02794474', 'n02907296', 'n02909706', 'n02912557', 'n02931013', 'n02966068', 'n03002555', + 'n03009111', 'n03037590', 'n03055670', 'n04297098', 'n03247351', 'n03435991', 'n03436656', 'n03389889', 'n03492087', + 'n03638014', 'n03989199', 'n04302863', 'n04365112', 'n04486616', 'n03009269', 'n03669245', 'n04353573', 'n04103364', + 'n04149374', 'n04170037', 'n03919096', 'n04062807', 'n04566561', 'n02740533', 'n03886053', 'n02739889', 'n02740061', + 'n02740300', 'n02749292', 'n04389718', 'n03684823', 'n03025165', 'n03193597', 'n03193260', 'n03193423', 'n03585778', + 'n03939565', 'n04211219', 'n04373428', 'n04389854', 'n04465358', 'n03791235', 'n04368695', 'n02854630', 'n02958343', + 'n03404012', 'n04201733', 'n03472937', 'n03079136', 'n03119396', 'n03141065', 'n03881534', 'n03268790', 'n03421669', + 'n03493219', 'n03498781', 'n03539103', 'n03543394', 'n02831335', 'n03680512', 'n03770085', 'n03870105', 'n04322801', + 'n03342961', 'n04097373', 'n04166281', 'n04285965', 'n04302988', 'n04347119', 'n04459122', 'n04516354', 'n03389761', + 'n03506880', 'n03790512', 'n03769722', 'n04466871', 'n04490091', 'n03256166', 'n03632852', 'n03690473', 'n04465666', + 'n04474035', 'n04520170', 'n02871314', 'n03173929', 'n03648667', 'n03764822', 'n03884639', 'n03896419', 'n02946348', + 'n04520382', 'n03256788', 'n03538300', 'n04464852', 'n03886053', 'n02983507', 'n04250599', 'n04229007', 'n02916179', + 'n02712643', 'n04225987', 'n04467099', 'n02946509', 'n03904433', 'n04543158', 'n02787120', 'n02970849', 'n03217739', + 'n03255899', 'n04497249', 'n03235979', 'n03594010', 'n03981924', 'n04558059', 'n04560502', 'n03027505', 'n03122295', + 'n03765467', 'n04543924', 'n04563020', 'n04543509', 'n04571800') +) + +num_classes = 25 + +# mean and standard deviation of channels of ImageNet images +mean = [0.485, 0.456, 0.406] +std = [0.229, 0.224, 0.225] + +test_transform = trn.Compose([trn.Resize(256), trn.CenterCrop(224), trn.ToTensor(), trn.Normalize(mean, std)]) + +test_data_in = dset.ImageFolder( + root="/share/data/vision-greg/ImageNet/clsloc/images/val", + transform=test_transform) + + +print('Loading ImageNet-22K') +# some ImageNet-22K images are broken JPEGs + + +def my_collate(batch): + batch = list(filter(lambda x: x is not None, batch)) + return torch.utils.data.dataloader.default_collate(batch) + + +class MyImageFolder(dset.ImageFolder): + __init__ = dset.ImageFolder.__init__ + + def __getitem__(self, index): + try: + return super(MyImageFolder, self).__getitem__(index) + except OSError: + pass + + +test_data_out = MyImageFolder(root="/share/data/vision-greg/ImageNet/clsloc/images/val", + transform=test_transform) +test_data_out.root = '/share/data/vision-greg/ImageNet22k' +test_data_out.class_to_idx = pickle.load(open(test_data_out.root + '/class_to_idx.p', "rb")) +test_data_out.classes = pickle.load(open(test_data_out.root + '/classes.p', "rb")) +# test_data_out.imgs = pickle.load(open(test_data_out.root + '/imgs.p', "rb")) +print('Loaded ImageNet-22K') + + +print('Filtering Data') +im1k_classes_flat = tuple(np.concatenate(im1k_classes)) +test_data_in.imgs = [x for x in test_data_in.imgs if + x[0][x[0].index('val/') + 4:x[0].index('val/') + 4 + len('n07718472')] in im1k_classes_flat] + +in_idx_to_superidx = {} +for c in im1k_classes_flat: + for i in range(len(im1k_classes)): # range(num_classes) + t = im1k_classes[i] + if c in t: + in_idx_to_superidx[test_data_in.class_to_idx[c]] = i + break +test_data_in.target_transform = lambda x: in_idx_to_superidx[x] + +im22k_classes_flat = tuple(np.concatenate(im22k_classes)) +# test_data_out.imgs = [x for x in test_data_out.imgs if +# x[0][x[0].index('22k/images/') + 11:x[0].index('22k/images/') + 11 + len('n07718472')] +# in im22k_classes_flat] +test_data_out.imgs = pickle.load(open('./imgs.p', "rb")) + +out_idx_to_superidx = {} +for c in im22k_classes_flat: + for i in range(len(im22k_classes)): # range(num_classes) + t = im22k_classes[i] + if c in t: + out_idx_to_superidx[test_data_out.class_to_idx[c]] = i + break +test_data_out.target_transform = lambda x: out_idx_to_superidx[x] + +test_loader_in = torch.utils.data.DataLoader( + test_data_in, batch_size=args.test_bs, shuffle=False, + num_workers=args.prefetch, pin_memory=True) +test_loader_out = torch.utils.data.DataLoader( + test_data_out, batch_size=args.test_bs, shuffle=False, + num_workers=args.prefetch, pin_memory=True, collate_fn=my_collate) + + +print('Initializing Network') + + +class FineTuneModel(nn.Module): + """ + This freezes the weights of all layers except the last one. + + Arguments: + original_model: Model to finetune + arch: Name of model architecture + num_classes: Number of classes to tune for + """ + + def __init__(self, original_model, arch, num_classes): + super(FineTuneModel, self).__init__() + + if arch.startswith('alexnet') or arch.startswith('vgg'): + self.features = original_model.features + self.fc = nn.Sequential(*list(original_model.classifier.children())[:-1]) + self.classifier = nn.Sequential( + nn.Linear(4096, num_classes) + ) + elif arch.startswith('resnet') or arch.startswith('resnext'): + # Everything except the last linear layer + self.features = nn.Sequential(*list(original_model.children())[:-1]) + if arch == 'resnet18': + self.classifier = nn.Sequential( + nn.Linear(512, num_classes) + ) + else: + self.classifier = nn.Sequential( + nn.Linear(2048, num_classes) + ) + else: + raise ("Finetuning not supported on this architecture yet. Feel free to add") + + self.unfreeze(False) # Freeze weights except last layer + + def unfreeze(self, unfreeze): + # Freeze those weights + for p in self.features.parameters(): + p.requires_grad = unfreeze + if hasattr(self, 'fc'): + for p in self.fc.parameters(): + p.requires_grad = unfreeze + + def forward(self, x): + f = self.features(x) + if hasattr(self, 'fc'): + f = f.view(f.size(0), -1) + f = self.fc(f) + f = f.view(f.size(0), -1) + y = self.classifier(f) + return y + + +if args.model_name == 'alexnet': + net = models.AlexNet() + net.load_state_dict(model_zoo.load_url('https://download.pytorch.org/models/alexnet-owt-4df8aa71.pth', + model_dir='/share/data/lang/users/dan/.torch/models')) + net = FineTuneModel(net, 'alexnet', num_classes) + +elif args.model_name == 'squeezenet1.1': + net = models.SqueezeNet(version=1.1) + net.load_state_dict(model_zoo.load_url('https://download.pytorch.org/models/squeezenet1_1-f364aa15.pth', + model_dir='/share/data/lang/users/dan/.torch/models')) + net.classifier = nn.Sequential( + nn.Dropout(p=0.5), + nn.Conv2d(512, num_classes, kernel_size=1), + nn.ReLU(inplace=True), + nn.AvgPool2d(13) + ) + net.forward = lambda x: net.classifier(net.features(x)).view(x.size(0), num_classes) + + for p in net.features.parameters(): + p.requires_grad = False + +elif 'vgg' in args.model_name: + if 'bn' not in args.model_name: + net = models.vgg19() + net.load_state_dict(model_zoo.load_url('https://download.pytorch.org/models/vgg19-dcbb9e9d.pth', + model_dir='/share/data/lang/users/dan/.torch/models')) + elif '11' in args.model_name: + net = models.vgg11() + net.load_state_dict(model_zoo.load_url('https://download.pytorch.org/models/vgg11-bbd30ac9.pth', + model_dir='/share/data/lang/users/dan/.torch/models')) + else: + net = models.vgg19_bn() + net.load_state_dict(model_zoo.load_url('https://download.pytorch.org/models/vgg19_bn-c79401a0.pth', + model_dir='/share/data/lang/users/dan/.torch/models')) + net = FineTuneModel(net, 'vgg', num_classes) + +elif args.model_name == 'resnet18': + net = models.resnet18() + net.load_state_dict(model_zoo.load_url('https://download.pytorch.org/models/resnet18-5c106cde.pth', + model_dir='/share/data/lang/users/dan/.torch/models')) + net = FineTuneModel(net, 'resnet18', num_classes) + +elif args.model_name == 'resnet50': + net = models.resnet50() + net.load_state_dict(model_zoo.load_url('https://download.pytorch.org/models/resnet50-19c8e357.pth', + model_dir='/share/data/lang/users/dan/.torch/models')) + net = FineTuneModel(net, 'resnet50', num_classes) + +elif args.model_name == 'resnext': + net = resnext_101_64x4d + net.load_state_dict(torch.load('/share/data/lang/users/dan/.torch/models/resnext_101_64x4d.pth')) + net = FineTuneModel(net, 'resnext', num_classes) + +start_epoch = 0 +# Restore model +if args.load != '': + for i in range(1000 - 1, -1, -1): + model_name = os.path.join(args.load, args.model_name + '_superclass_epoch' + str(i) + '.pth') + if os.path.isfile(model_name): + snapshot = torch.load(model_name) + net.load_state_dict(snapshot['state_dict']) + print('Model restored! Epoch:', i) + start_epoch = i + 1 + break + if start_epoch == 0: + assert False, "could not resume" + + +if args.ngpu > 1: + net = torch.nn.DataParallel(net, device_ids=list(range(args.ngpu))) + +if args.ngpu > 0: + net.cuda() + torch.cuda.manual_seed(1) + +for p in net.parameters(): + p.volatile = True + +cudnn.benchmark = True # fire on all cylinders + + +# test function +def test_in(): + net.eval() + loss_avg = 0.0 + correct = 0 + for batch_idx, (data, target) in enumerate(test_loader_in): + data, target = V(data.cuda(), volatile=True), V(target.cuda(), volatile=True) + + # forward + output = net(data) + loss = F.cross_entropy(output, target) + + # accuracy + pred = output.data.max(1)[1] + correct += pred.eq(target.data).sum() + + # test loss average + loss_avg += loss.data[0] + + state['test_in_loss'] = loss_avg / len(test_loader_in) + state['test_in_accuracy'] = correct / len(test_loader_in.dataset) + + +# test function +def test_out(): + print('Testing unseen classes') + net.eval() + correct = 0 + for batch_idx, (data, target) in enumerate(test_loader_out): + data, target = V(data.cuda(), volatile=True), V(target.cuda(), volatile=True) + + # forward + output = net(data) + + # accuracy + pred = output.data.max(1)[1] + correct += pred.eq(target.data).sum() + + state['test_out_accuracy'] = correct / len(test_loader_out.dataset) + + +test_in() +test_out() +print(state) diff --git a/data/datasets/visual_question_answering/generate_c_image/robustness-master/old/auxiliary/ImageNet22K/train.py b/data/datasets/visual_question_answering/generate_c_image/robustness-master/old/auxiliary/ImageNet22K/train.py new file mode 100644 index 0000000000000000000000000000000000000000..8b78052e0465a86a9d96a237432183d3e4d98de8 --- /dev/null +++ b/data/datasets/visual_question_answering/generate_c_image/robustness-master/old/auxiliary/ImageNet22K/train.py @@ -0,0 +1,1074 @@ +# -*- coding: utf-8 -*- +import numpy as np +import sys +import os +import pickle +import argparse +import math +import time +import torch +import torch.nn as nn +import torch.backends.cudnn as cudnn +import torchvision.transforms as trn +import torchvision.datasets as dset +import torch.nn.functional as F +from torch.autograd import Variable as V +import torchvision.models as models +import torch.utils.model_zoo as model_zoo +from resnext_101_64x4d import resnext_101_64x4d + +parser = argparse.ArgumentParser(description='Trains an ImageNet Superclass Classifier', + formatter_class=argparse.ArgumentDefaultsHelpFormatter) +parser.add_argument('--model-name', '-m', type=str, + choices=['alexnet', 'squeezenet1.1', 'vgg11', 'vgg', 'vggbn', + 'resnet18', 'resnet50', 'resnext']) +# Optimization options +parser.add_argument('--epochs', '-e', type=int, default=1, help='Number of epochs to train.') +parser.add_argument('--learning_rate', '-lr', type=float, default=0.01, help='The initial learning rate.') +parser.add_argument('--batch_size', '-b', type=int, default=128, help='Batch size.') +parser.add_argument('--test_bs', type=int, default=200) +parser.add_argument('--momentum', type=float, default=0.9, help='Momentum.') +parser.add_argument('--decay', '-d', type=float, default=0.0005, help='Weight decay (L2 penalty).') +# Checkpoints +parser.add_argument('--save', '-s', type=str, default='./snapshots', help='Folder to save checkpoints.') +parser.add_argument('--load', '-l', type=str, default='', help='Checkpoint path to resume / test.') +parser.add_argument('--test', '-t', action='store_true', help='Test only flag.') +# Acceleration +parser.add_argument('--ngpu', type=int, default=1, help='0 = CPU.') +parser.add_argument('--prefetch', type=int, default=4, help='Pre-fetching threads.') +args = parser.parse_args() + +torch.manual_seed(1) +np.random.seed(1) + +state = {k: v for k, v in args._get_kwargs()} +state['tt'] = 0 # SGDR variable +state['init_learning_rate'] = args.learning_rate + + +im1k_classes = ( + # amphibian n01627424 + ('n01641577', 'n01644373', 'n01644900', 'n01629819', 'n01630670', 'n01631663', 'n01632458', 'n01632777'), + # appliance n02729837 + ('n03483316', 'n04179913', 'n03584829', 'n03297495', 'n03761084', 'n03259280', 'n04111531', 'n04442312', 'n04542943', 'n04517823', 'n03207941', 'n04070727', 'n04554684'), + # aquatic mammal n02062017 + ('n02074367', 'n02077923', 'n02071294', 'n02066245'), + # bird n01503061 + ('n01514668', 'n01514859', 'n01518878', 'n01530575', 'n01531178', 'n01532829', 'n01534433', 'n01537544', 'n01558993', 'n01560419', 'n01580077', 'n01582220', 'n01592084', 'n01601694', 'n01608432', 'n01614925', 'n01616318', 'n01622779', 'n01795545', 'n01796340', 'n01797886', 'n01798484', 'n01806143', 'n01806567', 'n01807496', 'n01817953', 'n01818515', 'n01819313', 'n01820546', 'n01824575', 'n01828970', 'n01829413', 'n01833805', 'n01843065', 'n01843383', 'n01855672', 'n01847000', 'n01860187', 'n02002556', 'n02002724', 'n02006656', 'n02007558', 'n02009912', 'n02009229', 'n02011460', 'n02012849', 'n02013706', 'n02018207', 'n02018795', 'n02025239', 'n02027492', 'n02028035', 'n02033041', 'n02037110', 'n02017213', 'n02051845', 'n02056570', 'n02058221'), + # bear n02131653 + ('n02132136', 'n02134084', 'n02133161', 'n02134418'), + # beverage n07881800 + ('n07920052', 'n07892512', 'n07932039', 'n07930864'), + # big cat n02127808 + ('n02128925', 'n02129604', 'n02128385', 'n02128757', 'n02129165', 'n02130308'), + # building n02913152 + ('n02793495', 'n03457902', 'n03877845', 'n03781244', 'n03661043', 'n02727426', 'n02859443', 'n03028079', 'n03788195', 'n04346328', 'n03956157', 'n04081281', 'n03032252', 'n03529860'), + # cat n02121620 + ('n02124075', 'n02123394', 'n02123159', 'n02123597', 'n02123045', 'n02125311', 'n02127052'), + # clothing n03051540 + ('n03724870', 'n04584207', 'n03623198', 'n02730930', 'n04162706', 'n02669723', 'n04532106', 'n02916936', 'n04371430', 'n03710721', 'n02837789', 'n04350905', 'n03594734', 'n03325584', 'n04325704', 'n02883205', 'n04591157', 'n02865351', 'n03534580', 'n03770439', 'n03866082', 'n04136333', 'n03980874', 'n03404251', 'n04479046', 'n03630383', 'n04370456', 'n02963159', 'n03617480', 'n02667093', 'n03595614', 'n02892767', 'n03188531', 'n03775071', 'n03127747', 'n03379051', 'n02807133', 'n03787032', 'n04209133', 'n02869837', 'n02817516', 'n03124170', 'n04259630', 'n03710637', 'n04254777', 'n03026506', 'n03763968', 'n03877472', 'n03594734', 'n03450230', 'n02892767'), + # dog n02084071 + ('n02100583', 'n02100236', 'n02100735', 'n02101006', 'n02100877', 'n02102973', 'n02102177', 'n02102040', 'n02101556', 'n02101388', 'n02102318', 'n02102480', 'n02099601', 'n02099849', 'n02099429', 'n02099267', 'n02099712', 'n02096294', 'n02095314', 'n02098105', 'n02095889', 'n02095570', 'n02096437', 'n02096051', 'n02098413', 'n02094433', 'n02098286', 'n02097130', 'n02097047', 'n02097209', 'n02093256', 'n02093428', 'n02094114', 'n02096177', 'n02093859', 'n02097298', 'n02096585', 'n02093647', 'n02093991', 'n02097658', 'n02094258', 'n02097474', 'n02093754', 'n02090622', 'n02090721', 'n02092002', 'n02089078', 'n02089867', 'n02089973', 'n02092339', 'n02091635', 'n02088466', 'n02091467', 'n02091831', 'n02088094', 'n02091134', 'n02091032', 'n02088364', 'n02088238', 'n02088632', 'n02090379', 'n02091244', 'n02087394', 'n02110341', 'n02113186', 'n02113023', 'n02113978', 'n02111277', 'n02113712', 'n02113624', 'n02113799', 'n02110806', 'n02111129', 'n02112706', 'n02110958', 'n02109047', 'n02105641', 'n02106382', 'n02106550', 'n02105505', 'n02106030', 'n02106166', 'n02105162', 'n02105056', 'n02105855', 'n02105412', 'n02105251', 'n02106662', 'n02104365', 'n02107142', 'n02110627', 'n02107312', 'n02104029', 'n02110185', 'n02110063', 'n02108089', 'n02108422', 'n02109961', 'n02108000', 'n02107683', 'n02107574', 'n02107908', 'n02109525', 'n02108551', 'n02108915', 'n02112018', 'n02112350', 'n02112137', 'n02111889', 'n02111500', 'n02086910', 'n02086646', 'n02086079', 'n02085936', 'n02087046', 'n02085782', 'n02086240', 'n02085620'), + # electronic equipment n03278248 + ('n03584254', 'n03777754', 'n03782006', 'n03857828', 'n04004767', 'n03085013', 'n03602883', 'n02979186', 'n02992529', 'n03902125', 'n03187595', 'n04392985', 'n02988304'), + # fish n02512053 + ('n01496331', 'n01498041', 'n01484850', 'n01491361', 'n01494475', 'n02514041', 'n02536864', 'n01440764', 'n01443537', 'n02526121', 'n02536864', 'n02606052', 'n02607072', 'n02643566', 'n02655020', 'n02640242', 'n02641379'), + # footwear n03380867 + ('n04133789', 'n04120489', 'n03680355', 'n03124043', 'n03047690'), + # fruit n13134947 + ('n07742313', 'n07745940', 'n07747607', 'n07749582', 'n07753113', 'n07753275', 'n07753592', 'n07754684', 'n07760859', 'n07768694', 'n12267677', 'n12620546', 'n13133613', 'n11879895', 'n12144580', 'n12768682', 'n07742313'), + # fungus n12992868 + ('n13052670', 'n13044778', 'n12985857', 'n13040303', 'n13037406', 'n13054560', 'n12998815'), + # geological formation n09287968 + ('n09246464', 'n09468604', 'n09193705', 'n09472597', 'n09399592', 'n09421951', 'n09256479', 'n09332890', 'n09428293', 'n09288635'), + # hoofed animal n02370806 + ('n02391049', 'n02389026', 'n02437312', 'n02412080', 'n02423022', 'n02422699', 'n02422106', 'n02415577', 'n02417914', 'n02410509', 'n02408429', 'n02403003', 'n02398521', 'n02437616', 'n02396427', 'n02397096', 'n02395406'), + # insect n02159955 + ('n02165105', 'n02165456', 'n02167151', 'n02168699', 'n02169497', 'n02172182', 'n02174001', 'n02177972', 'n02190166', 'n02219486', 'n02206856', 'n02226429', 'n02229544', 'n02231487', 'n02233338', 'n02236044', 'n02256656', 'n02259212', 'n02264363', 'n02268443', 'n02268853', 'n02276258', 'n02277742', 'n02279972', 'n02280649', 'n02281406', 'n02281787'), + # musical instrument n03800933 + ('n02672831', 'n03854065', 'n03452741', 'n04515003', 'n03452741', 'n04515003', 'n03017168', 'n03249569', 'n03447721', 'n03720891', 'n03721384', 'n04311174', 'n03452741', 'n04515003', 'n02787622', 'n02992211', 'n04536866', 'n03495258', 'n02676566', 'n03272010', 'n03854065', 'n03110669', 'n03394916', 'n04487394', 'n02672831', 'n03494278', 'n03840681', 'n03884397', 'n02804610', 'n03838899', 'n04141076', 'n03372029'), + # primate n02469914 + ('n02500267', 'n02497673', 'n02483708', 'n02483362', 'n02480495', 'n02481823', 'n02480855', 'n02488702', 'n02484975', 'n02489166', 'n02486261', 'n02486410', 'n02487347', 'n02488291', 'n02493509', 'n02494079', 'n02493793', 'n02492035', 'n02492660', 'n02490219'), + # reptile n01661091 + ('n01664065', 'n01665541', 'n01667114', 'n01667778', 'n01669191', 'n01675722', 'n01677366', 'n01682714', 'n01685808', 'n01687978', 'n01688243', 'n01689811', 'n01692333', 'n01693334', 'n01694178', 'n01695060', 'n01704323', 'n01698640', 'n01697457', 'n01728572', 'n01728920', 'n01729322', 'n01729977', 'n01734418', 'n01735189', 'n01737021', 'n01739381', 'n01740131', 'n01742172', 'n01744401', 'n01748264', 'n01749939', 'n01751748', 'n01753488', 'n01755581', 'n01756291'), + # utensil n04516672 + ('n03133878', 'n03400231', 'n04596742', 'n02939185', 'n03063689', 'n04398044', 'n04270147'), + # vegetable n07707451 + ('n07711569', 'n07720875', 'n07711569', 'n07714571', 'n07714990', 'n07715103', 'n07717410', 'n07717556', 'n07716358', 'n07716906', 'n07718472', 'n07718747', 'n07730033', 'n07734744'), + # vehicle n04576211 + ('n02835271', 'n03792782', 'n03393912', 'n03895866', 'n02797295', 'n04204347', 'n03791053', 'n04389033', 'n03384352', 'n03272562', 'n04310018', 'n02704792', 'n02701002', 'n02814533', 'n02930766', 'n03100240', 'n03594945', 'n03670208', 'n03770679', 'n03777568', 'n04037443', 'n04285008', 'n03444034', 'n03445924', 'n03785016', 'n04252225', 'n03345487', 'n03417042', 'n03930630', 'n04461696', 'n04467665', 'n03796401', 'n03770679', 'n03977966', 'n04065272', 'n04335435', 'n03478589', 'n04389033', 'n04252077', 'n04465501', 'n03776460', 'n04482393', 'n04509417', 'n03538406', 'n03599486', 'n03868242') +) + +# this excludes ImageNet-1K classes + + +im22k_classes = ( + # amphibian n01627424 + ( + 'n01627424', 'n01639765', 'n01640846', 'n01641206', 'n01641391', 'n01641739', 'n01641930', 'n01642097', 'n01642257', + 'n01642391', 'n01642539', 'n01642943', 'n01643255', 'n01643507', 'n01643896', 'n01645466', 'n01645776', 'n01646292', + 'n01646388', 'n01646555', 'n01646648', 'n01646802', 'n01646902', 'n01647033', 'n01647180', 'n01647303', 'n01647466', + 'n01647640', 'n01648139', 'n01648356', 'n01648620', 'n01649170', 'n01649412', 'n01649556', 'n01649726', 'n01650167', + 'n01650690', 'n01650901', 'n01651059', 'n01651285', 'n01651487', 'n01651641', 'n01651778', 'n01652026', 'n01652297', + 'n01653026', 'n01653223', 'n01653509', 'n01653773', 'n01654083', 'n01654637', 'n01654863', 'n01628331', 'n01628770', + 'n01629276', 'n01629962', 'n01630148', 'n01630284', 'n01630901', 'n01631175', 'n01631354', 'n01631512', 'n01632047', + 'n01632308', 'n01632601', 'n01632952', 'n01633406', 'n01633781', 'n01634227', 'n01634522', 'n01635027', 'n01635176', + 'n01635480', 'n01636127', 'n01636352', 'n01636510', 'n01636829', 'n01637112', 'n01637338', 'n01637615', 'n01637932', + 'n01638194', 'n01638329', 'n01638722', 'n01639187', 'n01655344'), + # appliance n02729837 + ( + 'n02729837', 'n03251766', 'n03050655', 'n03717285', 'n04277826', 'n04496726', 'n04607242', 'n03528263', 'n04174101', + 'n03150511', 'n04309833', 'n04475631', 'n03620052', 'n03063338', 'n04219185', 'n03212114', 'n03378174', 'n03543254', + 'n03557692', 'n03862676', 'n02905036', 'n03425241', 'n04388473', 'n04330340', 'n03102371', 'n03273740', 'n03425595', + 'n03991202', 'n04003241', 'n04280487', 'n04442441', 'n04488857', 'n03534776', 'n04580493', 'n03050655', 'n03717285', + 'n04277826', 'n04496726', 'n04607242', 'n03273740', 'n03102654', 'n03273913', 'n03557590', 'n03170635'), + # aquatic mammal n02062017 + ( + 'n02062017', 'n02073250', 'n02073831', 'n02074726', 'n02075927', 'n02076196', 'n02076779', 'n02078574', 'n02078292', + 'n02078738', 'n02079005', 'n02077658', 'n02077787', 'n02077152', 'n02077384', 'n02076402', 'n02079389', 'n02081060', + 'n02079851', 'n02080146', 'n02080415', 'n02080713', 'n02081571', 'n02081798', 'n02081927', 'n02062430', 'n02062744', + 'n02066707', 'n02068974', 'n02072040', 'n02069701', 'n02069974', 'n02070174', 'n02072798', 'n02069412', 'n02071028', + 'n02070430', 'n02070624', 'n02070776', 'n02071636', 'n02067240', 'n02068206', 'n02068541', 'n02067768', 'n02067603', + 'n02072493', 'n02063224', 'n02063662', 'n02064000', 'n02064816', 'n02064338', 'n02065026', 'n02065407', 'n02065263', + 'n02065726'), + # bird n01503061 + ( + 'n01503061', 'n01503976', 'n01514752', 'n01514926', 'n01515078', 'n01515217', 'n01515303', 'n01516212', 'n01517389', + 'n01517565', 'n01519563', 'n01519873', 'n01520576', 'n01521399', 'n01521756', 'n01522450', 'n01523105', 'n01517966', + 'n01524359', 'n01525720', 'n01526521', 'n01526766', 'n01527194', 'n01527347', 'n01527617', 'n01527917', 'n01528396', + 'n01528654', 'n01528845', 'n01529672', 'n01530439', 'n01531344', 'n01531512', 'n01531639', 'n01531811', 'n01531971', + 'n01532325', 'n01532511', 'n01533000', 'n01533339', 'n01533481', 'n01533651', 'n01533893', 'n01534155', 'n01534582', + 'n01534762', 'n01535140', 'n01535469', 'n01535690', 'n01536035', 'n01536186', 'n01536334', 'n01536644', 'n01536780', + 'n01537134', 'n01537895', 'n01538059', 'n01538200', 'n01538362', 'n01538630', 'n01540233', 'n01540566', 'n01540832', + 'n01541102', 'n01541386', 'n01541760', 'n01541922', 'n01542433', 'n01542168', 'n01544704', 'n01538955', 'n01539272', + 'n01542786', 'n01543175', 'n01543383', 'n01543632', 'n01543936', 'n01544208', 'n01544389', 'n01555809', 'n01556182', + 'n01556514', 'n01557185', 'n01557962', 'n01558149', 'n01558307', 'n01558461', 'n01558594', 'n01558765', 'n01559160', + 'n01559477', 'n01559639', 'n01559804', 'n01560105', 'n01560280', 'n01560636', 'n01560793', 'n01560935', 'n01561181', + 'n01561452', 'n01561732', 'n01562014', 'n01562265', 'n01562451', 'n01563128', 'n01563449', 'n01563746', 'n01563945', + 'n01564101', 'n01564217', 'n01564394', 'n01564773', 'n01565345', 'n01565599', 'n01565930', 'n01566207', 'n01564914', + 'n01565078', 'n01567133', 'n01567678', 'n01567879', 'n01568132', 'n01568294', 'n01568720', 'n01568892', 'n01569060', + 'n01569262', 'n01569423', 'n01569566', 'n01569836', 'n01569971', 'n01570267', 'n01570421', 'n01570676', 'n01570839', + 'n01566645', 'n01571410', 'n01571904', 'n01572328', 'n01572489', 'n01572654', 'n01572782', 'n01573074', 'n01573240', + 'n01573360', 'n01573627', 'n01573898', 'n01574045', 'n01574390', 'n01574560', 'n01574801', 'n01575117', 'n01575401', + 'n01575745', 'n01576076', 'n01576358', 'n01576695', 'n01577035', 'n01577458', 'n01577659', 'n01577941', 'n01578180', + 'n01578575', 'n01579028', 'n01579149', 'n01579260', 'n01579410', 'n01579578', 'n01579729', 'n01580379', 'n01580490', + 'n01580772', 'n01580870', 'n01581166', 'n01581434', 'n01581730', 'n01581874', 'n01581984', 'n01582398', 'n01582498', + 'n01582856', 'n01583209', 'n01583495', 'n01583828', 'n01586941', 'n01587278', 'n01587526', 'n01587834', 'n01588002', + 'n01588431', 'n01588725', 'n01588996', 'n01589286', 'n01589718', 'n01589893', 'n01590220', 'n01591005', 'n01591123', + 'n01591301', 'n01591697', 'n01592257', 'n01592540', 'n01592387', 'n01592694', 'n01593028', 'n01593282', 'n01593553', + 'n01594004', 'n01594372', 'n01594787', 'n01594968', 'n01595168', 'n01595450', 'n01595624', 'n01595974', 'n01596273', + 'n01596608', 'n01597022', 'n01597336', 'n01597737', 'n01597906', 'n01598074', 'n01598271', 'n01598588', 'n01598988', + 'n01599159', 'n01599269', 'n01599388', 'n01599556', 'n01599741', 'n01600085', 'n01600341', 'n01600657', 'n01601410', + 'n01601068', 'n01602080', 'n01602209', 'n01602630', 'n01602832', 'n01603000', 'n01603152', 'n01603600', 'n01603812', + 'n01603953', 'n01539573', 'n01539925', 'n01540090', 'n01545574', 'n01546039', 'n01546506', 'n01546921', 'n01547832', + 'n01548301', 'n01548492', 'n01548694', 'n01548865', 'n01549053', 'n01549430', 'n01549641', 'n01549886', 'n01550172', + 'n01550761', 'n01551080', 'n01551300', 'n01552034', 'n01552333', 'n01555305', 'n01551711', 'n01552813', 'n01553142', + 'n01553527', 'n01553762', 'n01554017', 'n01554448', 'n01555004', 'n01584225', 'n01584695', 'n01584853', 'n01585121', + 'n01585287', 'n01585422', 'n01585715', 'n01586020', 'n01586374', 'n01524761', 'n01604330', 'n01604968', 'n01605630', + 'n01606097', 'n01606177', 'n01606522', 'n01606672', 'n01606809', 'n01606978', 'n01607309', 'n01607429', 'n01607600', + 'n01607812', 'n01607962', 'n01608265', 'n01608814', 'n01609062', 'n01609391', 'n01609751', 'n01609956', 'n01610100', + 'n01610226', 'n01610552', 'n01610955', 'n01611472', 'n01611674', 'n01611800', 'n01611969', 'n01612122', 'n01612275', + 'n01612476', 'n01612628', 'n01612955', 'n01613177', 'n01616086', 'n01613294', 'n01613807', 'n01614038', 'n01614343', + 'n01614556', 'n01615121', 'n01615303', 'n01615458', 'n01615703', 'n01616551', 'n01616764', 'n01617095', 'n01617443', + 'n01617766', 'n01618082', 'n01618922', 'n01619310', 'n01619536', 'n01619835', 'n01620135', 'n01620414', 'n01620735', + 'n01618503', 'n01621127', 'n01621635', 'n01622120', 'n01622352', 'n01622483', 'n01622959', 'n01623110', 'n01623425', + 'n01623615', 'n01623706', 'n01624115', 'n01624212', 'n01623880', 'n01624305', 'n01624537', 'n01624833', 'n01625121', + 'n01625562', 'n01789386', 'n01789740', 'n01790171', 'n01790304', 'n01790398', 'n01790557', 'n01790711', 'n01790812', + 'n01791625', 'n01792042', 'n01792158', 'n01792429', 'n01792530', 'n01792640', 'n01792808', 'n01792955', 'n01793085', + 'n01793159', 'n01793249', 'n01793340', 'n01793435', 'n01793565', 'n01793715', 'n01791954', 'n01794158', 'n01794344', + 'n01809106', 'n01809371', 'n01791107', 'n01791314', 'n01791388', 'n01791463', 'n01794651', 'n01799302', 'n01800195', + 'n01799679', 'n01800424', 'n01800633', 'n01801088', 'n01801479', 'n01801672', 'n01801876', 'n01802159', 'n01809752', + 'n01810700', 'n01811243', 'n01811909', 'n01812187', 'n01812337', 'n01813385', 'n01813532', 'n01813658', 'n01813948', + 'n01814217', 'n01812662', 'n01812866', 'n01813088', 'n01814370', 'n01814620', 'n01814755', 'n01814921', 'n01815036', + 'n01814549', 'n01815270', 'n01815601', 'n01816017', 'n01816140', 'n01816474', 'n02153203', 'n01795088', 'n01795735', + 'n01795900', 'n01796019', 'n01796105', 'n01796519', 'n01796729', 'n01797020', 'n01797307', 'n01797601', 'n01798168', + 'n01798706', 'n01798839', 'n01798979', 'n01802721', 'n01803078', 'n01803362', 'n01803641', 'n01803893', 'n01804163', + 'n01805321', 'n01805801', 'n01806061', 'n01806297', 'n01806364', 'n01806467', 'n01807105', 'n01804478', 'n01804653', + 'n01804921', 'n01805070', 'n01806847', 'n01807828', 'n01808140', 'n01808291', 'n01808596', 'n01810268', 'n01816887', + 'n01817263', 'n01817346', 'n01818299', 'n01818832', 'n01819115', 'n01819465', 'n01819734', 'n01820052', 'n01820348', + 'n01820801', 'n01821076', 'n01821203', 'n01821554', 'n01821869', 'n01822300', 'n01822602', 'n01823013', 'n01823414', + 'n01823740', 'n01824035', 'n01824344', 'n01824749', 'n01825278', 'n01825930', 'n01826364', 'n01826680', 'n01826844', + 'n01827403', 'n01827793', 'n01828096', 'n01828556', 'n01829869', 'n01830042', 'n01830479', 'n01830915', 'n01831360', + 'n01831712', 'n01832167', 'n01832493', 'n01832813', 'n01833112', 'n01833415', 'n01834177', 'n01834540', 'n01835276', + 'n01835769', 'n01835918', 'n01836087', 'n01836673', 'n01837072', 'n01837526', 'n01838038', 'n01838598', 'n01839086', + 'n01839330', 'n01839598', 'n01839750', 'n01839949', 'n01840120', 'n01840412', 'n01840775', 'n01841102', 'n01841288', + 'n01841441', 'n01841679', 'n01841943', 'n01842235', 'n01842504', 'n01842788', 'n01843719', 'n01844231', 'n01844551', + 'n01844746', 'n01844917', 'n01845132', 'n01860497', 'n01860864', 'n01861148', 'n01861330', 'n01845477', 'n01856072', + 'n01856155', 'n01856380', 'n01856553', 'n01856890', 'n01857079', 'n01857325', 'n01857512', 'n01857632', 'n01857851', + 'n01846331', 'n01847089', 'n01847170', 'n01847253', 'n01847407', 'n01847806', 'n01847978', 'n01848123', 'n01848323', + 'n01848453', 'n01848555', 'n01848648', 'n01848840', 'n01848976', 'n01849157', 'n01849466', 'n01849676', 'n01849863', + 'n01850192', 'n01850373', 'n01850553', 'n01850873', 'n01851038', 'n01851207', 'n01851375', 'n01851731', 'n01851573', + 'n01851895', 'n01852142', 'n01852329', 'n01852400', 'n01852671', 'n01852861', 'n01853195', 'n01853498', 'n01853870', + 'n01854415', 'n01858441', 'n01858281', 'n01858780', 'n01858845', 'n01858906', 'n01859190', 'n01859325', 'n01859496', + 'n01859689', 'n01859852', 'n01860002', 'n02000954', 'n02002075', 'n02003037', 'n02003204', 'n02003577', 'n02003839', + 'n02004131', 'n02004492', 'n02004855', 'n02005399', 'n02005790', 'n02006063', 'n02006364', 'n02007284', 'n02006985', + 'n02008041', 'n02008497', 'n02008643', 'n02008796', 'n02009380', 'n02009508', 'n02009750', 'n02010272', 'n02010453', + 'n02010728', 'n02011016', 'n02011281', 'n02011805', 'n02011943', 'n02012185', 'n02013177', 'n02013567', 'n02014237', + 'n02014524', 'n02014941', 'n02015357', 'n02015554', 'n02015797', 'n02016066', 'n02017725', 'n02018027', 'n02018368', + 'n02019190', 'n02019438', 'n02019929', 'n02020219', 'n02020578', 'n02021050', 'n02021281', 'n02022684', 'n02023341', + 'n02023855', 'n02023992', 'n02024185', 'n02024479', 'n02024763', 'n02025043', 'n02025389', 'n02026059', 'n02026948', + 'n02027075', 'n02027357', 'n02027897', 'n02028175', 'n02028342', 'n02028451', 'n02028727', 'n02028900', 'n02029087', + 'n02029378', 'n02029706', 'n02030035', 'n02030224', 'n02030287', 'n02030837', 'n02030568', 'n02026629', 'n02030996', + 'n02031298', 'n02031585', 'n02031934', 'n02032222', 'n02032355', 'n02032480', 'n02032769', 'n02033324', 'n02033208', + 'n02033561', 'n02033779', 'n02033882', 'n02034129', 'n02034295', 'n02034661', 'n02034971', 'n02035210', 'n02035402', + 'n02035656', 'n02036053', 'n02036228', 'n02036711', 'n02037464', 'n02037869', 'n02038141', 'n02038466', 'n02038993', + 'n02039171', 'n02039780', 'n02039497', 'n02040266', 'n02016358', 'n02016659', 'n02016816', 'n02016956', 'n02017475', + 'n02021795', 'n02040505', 'n02041085', 'n02043063', 'n02043333', 'n02041246', 'n02041678', 'n02041875', 'n02042046', + 'n02042180', 'n02042472', 'n02042759', 'n02043808', 'n02044178', 'n02044778', 'n02044908', 'n02044517', 'n02045369', + 'n02045596', 'n02045864', 'n02046171', 'n02046759', 'n02046939', 'n02047045', 'n02047260', 'n02047411', 'n02047517', + 'n02047614', 'n02047975', 'n02048115', 'n02048353', 'n02048698', 'n02049088', 'n02049532', 'n02050004', 'n02050313', + 'n02050442', 'n02050586', 'n02050809', 'n02051059', 'n02051474', 'n02052365', 'n02052204', 'n02052775', 'n02053083', + 'n02053425', 'n02053584', 'n02054036', 'n02054502', 'n02054711', 'n02055107', 'n02055658', 'n02055803', 'n02056228', + 'n02056728', 'n02057035', 'n02057330', 'n02057731', 'n02057898', 'n02058594', 'n02058747', 'n02059162', 'n02059541', + 'n02059852', 'n02060133', 'n02060411', 'n02060569', 'n02060889', 'n02061217', 'n02061560', 'n02061853', 'n02511730'), + # bear n02131653 + ('n02131653', 'n02133704', 'n02132788', 'n02132466', 'n02132580', 'n02132320', 'n02133400', 'n01322983'), + # beverage n07881800 + ( + 'n07881800', 'n07925966', 'n07926346', 'n07926250', 'n07926442', 'n07933274', 'n07934373', 'n07933652', 'n07933799', + 'n07933891', 'n07934032', 'n07934152', 'n07934282', 'n07913180', 'n07929519', 'n07920349', 'n07731122', 'n07731284', + 'n07731436', 'n07921239', 'n07919665', 'n07919572', 'n07919441', 'n07920872', 'n07920222', 'n07919787', 'n07919894', + 'n07920540', 'n07920663', 'n07929940', 'n07926785', 'n07922764', 'n07890970', 'n07891309', 'n07883251', 'n07883661', + 'n07883384', 'n07883510', 'n07882420', 'n07924033', 'n07925116', 'n07924366', 'n07924443', 'n07924747', 'n07924560', + 'n07924655', 'n07924276', 'n07924834', 'n07924955', 'n07914271', 'n07891189', 'n07921455', 'n07921615', 'n07921834', + 'n07921948', 'n07922041', 'n07914128', 'n07936263', 'n07936093', 'n07935737', 'n07936745', 'n14941787', 'n07936979', + 'n07937069', 'n07936459', 'n07936548', 'n07914006', 'n07927197', 'n07928887', 'n07927512', 'n07927836', 'n07928367', + 'n07927931', 'n07928696', 'n07928790', 'n07928163', 'n07928264', 'n07927716', 'n07929172', 'n07928578', 'n07928998', + 'n07928488', 'n07884567', 'n07886176', 'n07885705', 'n07922607', 'n07905618', 'n07886572', 'n07890617', 'n07886849', + 'n07889510', 'n07888465', 'n07888816', 'n07890068', 'n07889814', 'n07890352', 'n07890540', 'n07889990', 'n07890226', + 'n07887099', 'n07887192', 'n07887634', 'n07887967', 'n07889274', 'n07888058', 'n07887461', 'n07888229', 'n07887304', + 'n07890750', 'n07890890', 'n07932323', 'n07932454', 'n07921615', 'n07922147', 'n07922512', 'n07886463', 'n07901587', + 'n07902336', 'n07904934', 'n07905474', 'n07903101', 'n07903208', 'n07904293', 'n07903962', 'n07904072', 'n07902443', + 'n07903731', 'n07903841', 'n07903643', 'n07903543', 'n07906284', 'n07906718', 'n07906572', 'n07907429', 'n07907831', + 'n07907161', 'n07907342', 'n07907548', 'n07909593', 'n07906877', 'n07902520', 'n07902937', 'n07904395', 'n07904760', + 'n07902698', 'n07904637', 'n07902799', 'n07907037', 'n07906111', 'n07905038', 'n07905296', 'n07904865', 'n07905386', + 'n07905770', 'n07905979', 'n07907943', 'n07910656', 'n07909714', 'n07908812', 'n07908647', 'n07908567', 'n07908411', + 'n07910245', 'n07910379', 'n07909504', 'n07911249', 'n07909811', 'n07909954', 'n07910048', 'n07910152', 'n07909593', + 'n07908923', 'n07909129', 'n07910970', 'n07910538', 'n07911061', 'n07909231', 'n07909362', 'n07910799', 'n07925808', + 'n07902121', 'n07891726', 'n07900225', 'n07892813', 'n07894551', 'n07896893', 'n07898443', 'n07897975', 'n07899769', + 'n07898247', 'n07899533', 'n07895100', 'n07897200', 'n07897438', 'n07897600', 'n07894703', 'n07899660', 'n07895962', + 'n07894799', 'n07899899', 'n07896765', 'n07894451', 'n07900406', 'n07901355', 'n07900825', 'n07900616', 'n07900734', + 'n07901457', 'n07900958', 'n07899976', 'n07893528', 'n07893642', 'n07893792', 'n07896060', 'n07896560', 'n07897750', + 'n07897865', 'n07895710', 'n07895839', 'n07895435', 'n07898117', 'n07898333', 'n07894965', 'n07894102', 'n07895595', + 'n07894298', 'n07896165', 'n07897116', 'n07896422', 'n07892418', 'n07893891', 'n07894551', 'n07894703', 'n07894102', + 'n07899108', 'n07899434', 'n07899292', 'n07926920', 'n07927070', 'n07913300', 'n07898745', 'n07899003', 'n07895237', + 'n07895435', 'n07898117', 'n07894298', 'n07893253', 'n07896994', 'n07893425', 'n07896287', 'n07898617', 'n07896661', + 'n07898895', 'n07886057', 'n07911371', 'n07918879', 'n07930554', 'n07931280', 'n07930205', 'n07931001', 'n07931096', + 'n07931733', 'n07930062', 'n07931870', 'n07912211', 'n07913882', 'n07917618', 'n07917951', 'n07917874', 'n07917791', + 'n07915491', 'n07915094', 'n07913774', 'n07917507', 'n07919165', 'n07912093', 'n07911677', 'n07916041', 'n07916319', + 'n07917392', 'n07915918', 'n07915366', 'n07931612', 'n07913393', 'n07913537', 'n07916437', 'n07914413', 'n07914586', + 'n07914686', 'n07916582', 'n07918028', 'n07918193', 'n07913644', 'n07917133', 'n07915618', 'n07915800', 'n07917272', + 'n07931452', 'n07916183', 'n07915213', 'n07918309', 'n07914995', 'n07930315', 'n07914887', 'n07918706', 'n07930433', + 'n07914777', 'n07932614', 'n07932762', 'n07891433', 'n07886317', 'n07891095', 'n07844042', 'n07844786', 'n07846143', + 'n07846274', 'n07845775', 'n07921360', 'n07846014', 'n07845421', 'n07847047', 'n07846359', 'n07845335', 'n07846471', + 'n07846557', 'n07845702', 'n07846688', 'n07845495', 'n07846802', 'n07846938', 'n07845863', 'n07845166', 'n07845571', + 'n07919310', 'n07933530'), + # big cat n02127808 + ( + 'n02127808', 'n02129991', 'n02130545', 'n02130925', 'n02129923', 'n02129837', 'n01323068', 'n02128598', 'n02128669', + 'n02129463', 'n02129530', 'n01322898', 'n02130086'), + # building n02913152 + ( + 'n02913152', 'n02666943', 'n02726681', 'n04409384', 'n02734725', 'n02763604', 'n02806992', 'n02882190', 'n02993546', + 'n02940385', 'n03078506', 'n03089753', 'n03097362', 'n04077889', 'n04175574', 'n04177931', 'n04343511', 'n03007444', + 'n03224893', 'n03479397', 'n03322570', 'n03123809', 'n04441662', 'n03016389', 'n04294879', 'n03326371', 'n03413428', + 'n02977936', 'n03430418', 'n03449564', 'n02956699', 'n03005033', 'n03121431', 'n03152303', 'n03203806', 'n03093427', + 'n03282295', 'n04305210', 'n04461437', 'n03092166', 'n13252672', 'n03478756', 'n03036022', 'n03466839', 'n03437184', + 'n03698723', 'n03479121', 'n03479266', 'n03542333', 'n03541696', 'n02961035', 'n03561573', 'n03989898', 'n04097085', + 'n03790755', 'n03788498', 'n04080705', 'n04095109', 'n04229737', 'n08640531', 'n08560295', 'n08652376', 'n03542605', + 'n03544360', 'n02814338', 'n02857477', 'n02820085', 'n02919792', 'n02932400', 'n03686924', 'n03002816', 'n03007297', + 'n03118969', 'n03010915', 'n03158186', 'n04202142', 'n04354026', 'n04535252', 'n04535370', 'n03180865', 'n03219483', + 'n03257210', 'n03322836', 'n03428090', 'n03685640', 'n03465605', 'n03474352', 'n03685486', 'n03685820', 'n03367321', + 'n03713151', 'n03719053', 'n03718458', 'n03878066', 'n04305323', 'n04052658', 'n04079244', 'n03121040', 'n03166514', + 'n03718935', 'n02695627', 'n03892557', 'n03439348', 'n04073948', 'n03099454', 'n02667478', 'n02667379', 'n03396580', + 'n03635032', 'n04005197', 'n04115256', 'n02907873', 'n04413969', 'n04125541', 'n04131368', 'n04255899', 'n04258438', + 'n04465050', 'n04535524', 'n03545150', 'n02806875', 'n04350235', 'n03121298', 'n03333610', 'n03736269', 'n03837698', + 'n04022708', 'n04022866', 'n04246731', 'n03739518', 'n03043274', 'n03210552', 'n03540595', 'n03129471', 'n03730334', + 'n03762982', 'n03333349', 'n03770316', 'n03785499', 'n03130233', 'n03402941', 'n03839671', 'n03842012', 'n03859280', + 'n03055857', 'n03416489', 'n02968074', 'n03610524', 'n03860404', 'n04187547', 'n03056288', 'n04452757', 'n04598318', + 'n03872016', 'n03953416', 'n02833040', 'n03007130', 'n03006788', 'n03633341', 'n04214413', 'n02667576', 'n02801184', + 'n02984061', 'n03772077', 'n02984203', 'n03618982', 'n03099622', 'n03724756', 'n04210390', 'n04374735', 'n04407435', + 'n03602365', 'n03884778', 'n03999160', 'n02844214', 'n02892499', 'n02897389', 'n02935658', 'n02936281', 'n03155178', + 'n03297644', 'n03298089', 'n02935891', 'n02760099', 'n02952485', 'n02952674', 'n03199647', 'n03456548', 'n03459914', + 'n03497100', 'n03697552', 'n04111414', 'n04307878', 'n04398497', 'n04081699', 'n04093625', 'n03558176', 'n03557360', + 'n04104500', 'n02801047', 'n04112654', 'n04118635', 'n04146050', 'n03092314', 'n03801671', 'n02746978', 'n03165616', + 'n04217546', 'n04233124', 'n04343740', 'n04395875', 'n02823586', 'n02910241', 'n04018399', 'n02696165', 'n03393017', + 'n04055700', 'n07888378', 'n04400109', 'n04407686', 'n04614655', 'n04417809', 'n03798982', 'n03202481', 'n03678729', + 'n03801533', 'n03849814', 'n04581595', 'n03726233'), + # cat n02121620 + ( + 'n02121620', 'n02121808', 'n02122298', 'n02123478', 'n02122725', 'n02124484', 'n02124157', 'n02122878', 'n02123917', + 'n02122510', 'n02124313', 'n02123242', 'n02122430', 'n02124623', 'n02125081', 'n02125872', 'n02125010', 'n02126787', + 'n02126317', 'n02125494', 'n02125689', 'n02126028', 'n02126640', 'n02126139'), + # clothing n03051540 + ( + 'n03051540', 'n04385079', 'n02756098', 'n03859958', 'n04489695', 'n02834506', 'n03964611', 'n03289985', 'n04129766', + 'n03114041', 'n03113657', 'n03320519', 'n03340923', 'n04355115', 'n03786194', 'n03114236', 'n03206718', 'n03320519', + 'n03319457', 'n03221059', 'n02726017', 'n03384891', 'n02780704', 'n03239054', 'n03201638', 'n03201776', 'n04285803', + 'n04120695', 'n03472672', 'n03476083', 'n02683454', 'n04459018', 'n03917327', 'n03538957', 'n03263338', 'n03113835', + 'n02669534', 'n04092168', 'n03473966', 'n03398153', 'n03835941', 'n03781467', 'n03474167', 'n02846141', 'n03692379', + 'n03692136', 'n03692272', 'n03692004', 'n04243142', 'n04600912', 'n03863108', 'n03655720', 'n03815482', 'n03068998', + 'n03406759', 'n04015204', 'n04194127', 'n03268645', 'n03216402', 'n03121897', 'n03863262', 'n03604763', 'n04607640', + 'n02752615', 'n04429038', 'n03015478', 'n02841847', 'n04423552', 'n04001845', 'n02720576', 'n04266375', 'n02863340', + 'n02738859', 'n03386870', 'n04207903', 'n03521771', 'n02671780', 'n02827606', 'n04125853', 'n04132465', 'n02778294', + 'n02972397', 'n02786611', 'n03527565', 'n03781683', 'n03405595', 'n03859495', 'n03450516', 'n03434830', 'n03010795', + 'n03880129', 'n02694966', 'n04364994', 'n03981340', 'n03548930', 'n02979516', 'n04264233', 'n04446162', 'n02814774', + 'n02742322', 'n04552097', 'n02855925', 'n03419014', 'n04332580', 'n04488530', 'n02922578', 'n04531873', 'n03608504', + 'n04371563', 'n04574067', 'n04219580', 'n02896294', 'n03186199', 'n03226538', 'n04222470', 'n03943920', 'n02925519', + 'n04427715', 'n04504141', 'n04615644', 'n04233832', 'n03885669', 'n03657511', 'n04046277', 'n03789794', 'n04123317', + 'n03502331', 'n03314884', 'n03826039', 'n04612159', 'n02998841', 'n04605572', 'n04489008', 'n03660124', 'n04337287', + 'n03877674', 'n03600285', 'n02896442', 'n02902816', 'n03970363', 'n04491934', 'n02910864', 'n03107488', 'n03688605', + 'n03019434', 'n03884554', 'n04370288', 'n04480303', 'n03029296', 'n04132158', 'n04233715', 'n03903733', 'n04205318', + 'n03653833', 'n02831595', 'n03543112', 'n02825442', 'n03357081', 'n03745487', 'n03487642', 'n03450734', 'n04143897', + 'n03332173', 'n03610992', 'n03814817', 'n03505504', 'n03520493', 'n03615655', 'n02766168', 'n04122262', 'n04495698', + 'n03719743', 'n03900028', 'n04060448', 'n03797182', 'n03648219', 'n03816005', 'n03815615', 'n03845990', 'n02847631', + 'n04339191', 'n03388323', 'n03128085', 'n02747063', 'n03814727', 'n03913930', 'n04325804', 'n04370774', 'n04160261', + 'n04570532', 'n03605598', 'n04104770', 'n04230808', 'n03454442', 'n03617312', 'n03429003', 'n03205669', 'n03732458', + 'n02780815', 'n03402511', 'n03146777', 'n03649003', 'n03523506', 'n03863923', 'n03588216', 'n03045337', 'n03595055', + 'n04497570', 'n03472796', 'n04361937', 'n04378489', 'n03619275', 'n03607923', 'n02956883', 'n02936402', 'n02923535', + 'n03103904', 'n03880032', 'n02955767', 'n04440597', 'n03021228', 'n03719560', 'n03906789', 'n03219859', 'n04605446', + 'n04445040', 'n04445154', 'n04186455', 'n04173907', 'n03998333', 'n03849943', 'n03057021', 'n03703463', 'n03770954', + 'n04122492', 'n04455579', 'n04049405', 'n03844815', 'n02921406', 'n03254046', 'n03057841', 'n03456665', 'n04506402', + 'n04365229', 'n02957008', 'n03589791', 'n03238286', 'n02867966', 'n03595264', 'n02788462', 'n03221540', 'n02864504', + 'n04123026', 'n03548320', 'n03226375', 'n03751269', 'n04222307', 'n03696909', 'n02925385', 'n02850358', 'n03219966', + 'n03720005', 'n04368496', 'n02820675', 'n03902756', 'n03891051', 'n04230387', 'n02937010', 'n03301175', 'n03829857', + 'n03604536', 'n03228254', 'n04123448', 'n03398228', 'n04003359', 'n04363777', 'n04187970', 'n02885233', 'n04252560', + 'n04357531', 'n04367950', 'n04370048', 'n04021028', 'n04502197', 'n03655072', 'n04269822', 'n03410938', 'n04027935', + 'n03006903', 'n04097866', 'n02807616', 'n03237992', 'n04085574', 'n04197391', 'n04390577', 'n03238879', 'n04602956', + 'n03978966', 'n03476542', 'n03163381', 'n03629231', 'n02943964', 'n04502197', 'n04172904', 'n04508163', 'n04223299', + 'n02944146', 'n04508489', 'n02837887', 'n04426427', 'n02854739', 'n03885028', 'n02901114', 'n03234164', 'n04514241', + 'n03387323', 'n04103665', 'n03112869', 'n03885788', 'n02863014', 'n03013580', 'n04508949', 'n03688192', 'n03673450', + 'n04509171', 'n03824381', 'n04231905', 'n02930214', 'n03920737', 'n03132776', 'n03421324', 'n03688707', 'n03585875', + 'n03404149', 'n03540090', 'n03456186', 'n03490324', 'n03796974', 'n03441112', 'n02811204', 'n03429771', 'n03447075', + 'n03616979', 'n03429682', 'n03502509', 'n03531281', 'n03124474', 'n02941845', 'n03513137', 'n04265428', 'n04229107', + 'n02811350', 'n03504205', 'n03492922', 'n02954340', 'n04129688', 'n04228693', 'n03103563', 'n03061893', 'n04387095', + 'n02831237', 'n03331077', 'n04596116', 'n03610682', 'n02799323', 'n03824284', 'n02843909', 'n04232153', 'n03065243', + 'n02816768', 'n04612026', 'n04556408', 'n03607527', 'n02776825', 'n03776167', 'n03420801', 'n03049924', 'n02941228', + 'n03439631', 'n04455048', 'n04585318', 'n03597317', 'n04432203', 'n03138669', 'n03497657', 'n04356595', 'n03950899', + 'n04354589', 'n04354589', 'n04248507', 'n03984643', 'n02987379', 'n03256631', 'n03061050', 'n02834642', 'n04482177', + 'n04456011', 'n02859184', 'n04208582', 'n02881757', 'n03404360', 'n02818135', 'n04505888', 'n04264361', 'n02945964', + 'n03237416', 'n03766322', 'n03931885', 'n03046029', 'n03937835', 'n03028785', 'n03170872', 'n03325941', 'n04441528', + 'n03607186', 'n04498389', 'n04335693', 'n03381126', 'n03540267', 'n04434932', 'n03885904', 'n02713218', 'n04378956', + 'n03622931', 'n02736798', 'n02752496', 'n04323819', 'n04360914', 'n03622931', 'n03836976', 'n02874336', 'n04532022', + 'n04059157', 'n04509592', 'n03605504', 'n04071393', 'n03402188', 'n03239259', 'n03237212', 'n03846234', 'n02811719', + 'n03615563', 'n03324928', 'n03825080', 'n04235771', 'n03824381', 'n03824999', 'n03625943', 'n02728440', 'n04603872', + 'n03660124', 'n04602956', 'n03746330', 'n02752615', 'n02887489', 'n03237416', 'n04596852', 'n03236735', 'n03619196', + 'n03788914', 'n04197878', 'n03604400', 'n02936570', 'n03013438', 'n03062015', 'n02781121', 'n02898585', 'n03201638', + 'n04397645', 'n04334105', 'n03057724', 'n03205574', 'n04355511', 'n03978815', 'n03786096', 'n04136161', 'n03817647', + 'n02908123', 'n02944075', 'n02697221', 'n04453666', 'n02861387', 'n02926426', 'n03480579', 'n02854926', 'n03418749', + 'n03467254', 'n04198562', 'n03762238', 'n04514241', 'n03464053', 'n04241249', 'n03036469', 'n03036341', 'n03797264'), + # dog n02084071 + ( + 'n02084071', 'n02084732', 'n02087122', 'n02098550', 'n02099997', 'n02100399', 'n02098806', 'n02101108', 'n02101670', + 'n02102605', 'n02102806', 'n02101861', 'n02103181', 'n02098906', 'n02099029', 'n02089232', 'n02089468', 'n02092468', + 'n02095050', 'n02095212', 'n02095412', 'n02095727', 'n02096756', 'n02093056', 'n02097786', 'n02097967', 'n02094562', + 'n02094721', 'n02094931', 'n02087314', 'n02087551', 'n02090253', 'n02090475', 'n02088839', 'n02088992', 'n02089555', + 'n02089725', 'n02092173', 'n02090827', 'n02090129', 'n02088745', 'n02110532', 'n02084861', 'n02085118', 'n02085019', + 'n02112826', 'n02085272', 'n02113335', 'n02113892', 'n02112497', 'n02103406', 'n02109150', 'n02109256', 'n02104523', + 'n02104882', 'n02103841', 'n02106966', 'n02104280', 'n02104184', 'n02106854', 'n02109687', 'n02109811', 'n02107420', + 'n02108254', 'n02109391', 'n02108672', 'n02111626', 'n02085374', 'n02086346', 'n02086753', 'n02086478', 'n01322604'), + # electronic equipment n03278248 + ( + 'n03278248', 'n02757462', 'n04077430', 'n03517760', 'n04315948', 'n04546340', 'n03436182', 'n04030965', 'n03584400', + 'n04064213', 'n04043411', 'n04413419', 'n04075291', 'n02676670', 'n03181293', 'n03143400', 'n04142731', 'n03034405', + 'n03775388', 'n04176528', 'n04060647', 'n03205304', 'n03447894', 'n04042204', 'n04137773', 'n04043733', 'n03144156', + 'n03516996', 'n03046921', 'n04027367', 'n04405907', 'n04472726', 'n04138131', 'n04406552', 'n03592931', 'n04045085', + 'n04269668', 'n04405762', 'n02705944', 'n02872529', 'n02756854', 'n03724176', 'n03424204', 'n04405540', 'n04404997', + 'n02942349', 'n02995345', 'n03916720', 'n03225777', 'n04595285', 'n03571942', 'n02909285', 'n03861048', 'n03163973', + 'n04143140', 'n03781787', 'n02962938', 'n03278914', 'n03963294', 'n03293741', 'n03656957', 'n04392526', 'n02979074', + 'n04401088', 'n03488438', 'n03306869', 'n04044498', 'n03179910', 'n04270371', 'n03842377'), + # fish n02512053 + ( + 'n02512053', 'n01316579', 'n02599958', 'n02600298', 'n02600503', 'n02600798', 'n01480516', 'n01482071', 'n01495701', + 'n01497118', 'n01497413', 'n01497738', 'n01498406', 'n01498699', 'n01498989', 'n01499396', 'n01499732', 'n01500091', + 'n01500854', 'n01500476', 'n01501160', 'n01501641', 'n01501777', 'n01501948', 'n01502101', 'n01482330', 'n01483021', + 'n01483522', 'n01483830', 'n01484097', 'n01484285', 'n01484447', 'n01484562', 'n01485479', 'n01486010', 'n01486540', + 'n01486838', 'n01487506', 'n01488038', 'n01488918', 'n01489501', 'n01489709', 'n01489920', 'n01490112', 'n01490360', + 'n01490670', 'n01491006', 'n01491661', 'n01491874', 'n01492357', 'n01492569', 'n01492708', 'n01492860', 'n01493146', + 'n01493541', 'n01493829', 'n01494041', 'n01494757', 'n01494882', 'n01495006', 'n01495493', 'n01480880', 'n01481331', + 'n01481498', 'n02512752', 'n02512830', 'n02512938', 'n02513355', 'n02530421', 'n02530637', 'n02530831', 'n02530999', + 'n02532028', 'n02532272', 'n02532451', 'n02532602', 'n02532918', 'n02532786', 'n02534734', 'n02535163', 'n02535258', + 'n02535537', 'n02535759', 'n02536165', 'n02536456', 'n02537085', 'n02537319', 'n02537716', 'n02537525', 'n02538010', + 'n02538216', 'n02538985', 'n02539424', 'n02539573', 'n02539894', 'n02567334', 'n02567633', 'n02568087', 'n02568447', + 'n02568959', 'n02569484', 'n02569631', 'n02569905', 'n02570164', 'n02586543', 'n02587051', 'n02587300', 'n02587479', + 'n02587618', 'n02587877', 'n02626762', 'n02627037', 'n02627292', 'n02627532', 'n02663849', 'n02664285', 'n02664642', + 'n02665250', 'n02513248', 'n02513560', 'n02513727', 'n02530052', 'n02530188', 'n02535080', 'n02513805', 'n02513939', + 'n02515214', 'n02515713', 'n02516188', 'n02516776', 'n02528163', 'n02538985', 'n02539424', 'n02539573', 'n02539894', + 'n01429172', 'n01438208', 'n01438581', 'n01439121', 'n01439514', 'n01439808', 'n01441117', 'n01441272', 'n01441425', + 'n01441910', 'n01442450', 'n01442710', 'n01442972', 'n01443243', 'n01443831', 'n01444339', 'n01444783', 'n01445429', + 'n01445857', 'n01446152', 'n01446589', 'n01446760', 'n01447139', 'n01447331', 'n01447658', 'n01447946', 'n01448291', + 'n01448594', 'n01448951', 'n01449374', 'n01449712', 'n01449980', 'n02583567', 'n02583890', 'n02584145', 'n02584449', + 'n02517442', 'n02517938', 'n02518622', 'n02518324', 'n02519148', 'n02519340', 'n02519472', 'n02519686', 'n02519862', + 'n02520147', 'n02520525', 'n02520810', 'n02521646', 'n02522399', 'n02522637', 'n02522722', 'n02522866', 'n02523427', + 'n02523110', 'n02523877', 'n02524202', 'n02524524', 'n02524928', 'n02524659', 'n02525382', 'n02525703', 'n02526425', + 'n02526818', 'n02527057', 'n02527271', 'n02527622', 'n02529293', 'n02529772', 'n02530421', 'n02530637', 'n02530831', + 'n02530999', 'n02532028', 'n02532272', 'n02532451', 'n02532602', 'n02532918', 'n02532786', 'n02531114', 'n02531625', + 'n02533209', 'n02533545', 'n02533834', 'n02534165', 'n02534559', 'n02534734', 'n02535163', 'n02535258', 'n02535537', + 'n02535759', 'n02536165', 'n02536456', 'n02537085', 'n02537319', 'n02537716', 'n02537525', 'n02538010', 'n02538216', + 'n02538406', 'n02538562', 'n02540412', 'n02540983', 'n02541257', 'n02541687', 'n02542017', 'n02542432', 'n02542958', + 'n02543255', 'n02543565', 'n02544274', 'n02545841', 'n02546028', 'n02546331', 'n02546627', 'n02547014', 'n01454545', + 'n01455778', 'n01456137', 'n01456454', 'n01456756', 'n01457082', 'n01457407', 'n01457852', 'n02549989', 'n02550203', + 'n02550460', 'n02550655', 'n02551134', 'n02551668', 'n02552171', 'n01450661', 'n01450950', 'n01451115', 'n01451295', + 'n01451426', 'n01451863', 'n01452345', 'n01453087', 'n01453475', 'n01453742', 'n01454856', 'n01455461', 'n01455317', + 'n02547733', 'n02548247', 'n02548689', 'n02548884', 'n02549248', 'n02549376', 'n02554730', 'n02599958', 'n02600298', + 'n02600503', 'n02600798', 'n02586543', 'n02587051', 'n02587300', 'n02587479', 'n02587618', 'n02587877', 'n02555863', + 'n02556846', 'n02557182', 'n02557318', 'n02557591', 'n02557749', 'n02558206', 'n02558860', 'n02559144', 'n02559383', + 'n02559862', 'n02560110', 'n02561108', 'n02561381', 'n02561514', 'n02561661', 'n02561803', 'n02561937', 'n02562315', + 'n02562796', 'n02562971', 'n02563079', 'n02563182', 'n01440467', 'n02563792', 'n02563949', 'n02563648', 'n02564403', + 'n02564720', 'n02564935', 'n02565072', 'n02565324', 'n02565573', 'n02564270', 'n02566109', 'n02567334', 'n02567633', + 'n02568087', 'n02568447', 'n02568959', 'n02566489', 'n02566665', 'n02570484', 'n02570838', 'n02571167', 'n02571652', + 'n02571810', 'n02572196', 'n02572484', 'n02573249', 'n02573704', 'n02574271', 'n02576223', 'n02576575', 'n02576906', + 'n02577041', 'n02577164', 'n02577403', 'n02577662', 'n02577952', 'n02578771', 'n02578928', 'n02579303', 'n02578233', + 'n02578454', 'n02579557', 'n02579762', 'n02579928', 'n02580336', 'n02580679', 'n02580830', 'n02581108', 'n02581482', + 'n02581642', 'n02581957', 'n02582220', 'n02582349', 'n02585872', 'n02586238', 'n02588286', 'n02588794', 'n02588945', + 'n02589062', 'n02589196', 'n02589316', 'n02589623', 'n02589796', 'n02590094', 'n02590495', 'n02592055', 'n02592371', + 'n02593019', 'n02590702', 'n02582721', 'n02590987', 'n02591330', 'n02592734', 'n02593453', 'n02593679', 'n02591613', + 'n02591911', 'n02593191', 'n02594250', 'n02594942', 'n02595056', 'n02595339', 'n02595702', 'n02596067', 'n02596252', + 'n02596381', 'n02596720', 'n02597004', 'n02598573', 'n02598878', 'n02597367', 'n02597608', 'n02597818', 'n02597972', + 'n02598134', 'n02599052', 'n02599557', 'n02599347', 'n02601344', 'n02601767', 'n02601921', 'n02602059', 'n02604157', + 'n02604480', 'n02604954', 'n02605316', 'n02605703', 'n02605936', 'n02606384', 'n02606751', 'n02607201', 'n02607470', + 'n02607862', 'n02608284', 'n02608547', 'n02608860', 'n02608996', 'n02609302', 'n02609823', 'n02610066', 'n02610373', + 'n02610664', 'n02610980', 'n02611561', 'n02611898', 'n02612167', 'n02613181', 'n02613572', 'n02613820', 'n02614140', + 'n02614482', 'n02614653', 'n02614978', 'n02615298', 'n02616128', 'n02616397', 'n02616851', 'n02617537', 'n02618094', + 'n02619165', 'n02619550', 'n02619861', 'n02620167', 'n02620578', 'n02621258', 'n02621908', 'n02622249', 'n02622547', + 'n02622712', 'n02622955', 'n02623445', 'n02626762', 'n02627037', 'n02627292', 'n02627532', 'n02624167', 'n02624551', + 'n02624807', 'n02624987', 'n02625258', 'n02625612', 'n02627835', 'n02628062', 'n02628259', 'n02628600', 'n02629230', + 'n02629716', 'n02630281', 'n02630615', 'n02630739', 'n02631041', 'n02632039', 'n02632494', 'n02633422', 'n02633677', + 'n02633977', 'n02634545', 'n02635154', 'n02635580', 'n02636170', 'n02636405', 'n02636550', 'n02636854', 'n02637179', + 'n02637475', 'n02637977', 'n02574910', 'n02575325', 'n02575590', 'n02602405', 'n02602760', 'n02603317', 'n02603540', + 'n02618513', 'n02618827', 'n02642107', 'n02553028', 'n02642644', 'n02643112', 'n02643316', 'n02643836', 'n02644113', + 'n02644360', 'n02644501', 'n02644665', 'n02644817', 'n02645538', 'n02645691', 'n02645953', 'n02646667', 'n02646892', + 'n02648035', 'n02648625', 'n02648916', 'n02649218', 'n02649546', 'n02650050', 'n02650413', 'n02650541', 'n02651060', + 'n02652132', 'n02652668', 'n02653145', 'n02653497', 'n02653786', 'n02654112', 'n02654425', 'n02654745', 'n02655523', + 'n02655848', 'n02656032', 'n02656301', 'n02656670', 'n02656969', 'n02657368', 'n02663849', 'n02664285', 'n02664642', + 'n02665250', 'n02657694', 'n02658079', 'n02658531', 'n02658811', 'n02659176', 'n02659478', 'n02659808', 'n02660091', + 'n02660519', 'n02660640', 'n02660208', 'n02661017', 'n02661473', 'n02661618', 'n02662239', 'n02662397', 'n02662559', + 'n02662825', 'n02662993', 'n02663211', 'n02663485', 'n02603862', 'n02638596', 'n02639087', 'n02639605', 'n02639922', + 'n02640857', 'n02640626', 'n02556373'), + # footwear n03380867 + ( + 'n03380867', 'n04199027', 'n03297103', 'n04156411', 'n03547530', 'n04027706', 'n03364008', 'n04386664', 'n03027625', + 'n04239786', 'n04122578', 'n03090710', 'n02904927', 'n02713364', 'n04593524', 'n02938218', 'n02855701', 'n03411079', + 'n04545748', 'n03868406', 'n04124370', 'n02882894', 'n02767147', 'n04570118', 'n03776877', 'n03364156', 'n03041449', + 'n03472535', 'n03967270', 'n03025250', 'n04022332', 'n04272389', 'n04546081', 'n03361550', 'n04241394', 'n03798061', + 'n02873733', 'n02872752', 'n04228581', 'n04089666', 'n03600475', 'n03516844', 'n03521544', 'n04542715', 'n02925666', + 'n04116294', 'n03865949', 'n02735538'), + # fruit n13134947 + ( + 'n13134947', 'n07705931', 'n07738105', 'n07738224', 'n07739035', 'n07739125', 'n07739344', 'n07739506', 'n07739923', + 'n07740033', 'n07740220', 'n07740342', 'n07740461', 'n07740597', 'n07740744', 'n07740855', 'n07740954', 'n07741138', + 'n07741235', 'n07741357', 'n07741461', 'n07740115', 'n07741623', 'n07741706', 'n07741804', 'n07741888', 'n07742012', + 'n07742224', 'n07742415', 'n07742513', 'n07742605', 'n07742704', 'n07743224', 'n07743384', 'n07743544', 'n07743723', + 'n07743902', 'n07744057', 'n07744246', 'n07744430', 'n07744559', 'n07744682', 'n07744811', 'n07745046', 'n07745197', + 'n07745357', 'n07745466', 'n07745661', 'n07746038', 'n07746186', 'n07746334', 'n07767171', 'n07746551', 'n07746749', + 'n07746910', 'n07747055', 'n07747811', 'n07748753', 'n07748912', 'n07749095', 'n07749192', 'n07749312', 'n07747951', + 'n07748157', 'n07748276', 'n07748416', 'n07749446', 'n07749731', 'n07749870', 'n07749969', 'n07750146', 'n07750299', + 'n07750449', 'n07748574', 'n07750872', 'n07751004', 'n07751148', 'n07751280', 'n07751451', 'n07751737', 'n07751858', + 'n07751977', 'n07752109', 'n07752264', 'n07752377', 'n07752514', 'n07752602', 'n07752664', 'n07752782', 'n07752874', + 'n07752966', 'n07753448', 'n07753743', 'n07753980', 'n07754155', 'n07754279', 'n07754451', 'n07755262', 'n07755411', + 'n07755619', 'n07755707', 'n07755929', 'n07756096', 'n07756325', 'n07756499', 'n07756838', 'n07756641', 'n07756951', + 'n07757132', 'n07757312', 'n07757511', 'n07757602', 'n07757753', 'n07757874', 'n07757990', 'n07758125', 'n07758260', + 'n07758407', 'n07758680', 'n07759424', 'n07759576', 'n07759691', 'n07758950', 'n07759194', 'n07759324', 'n07759816', + 'n07760070', 'n07760153', 'n07760297', 'n07760395', 'n07760501', 'n07760673', 'n07760755', 'n07761141', 'n07761309', + 'n07761611', 'n07761777', 'n07761954', 'n07762114', 'n07762244', 'n07762373', 'n07762534', 'n07762740', 'n07762913', + 'n07763107', 'n07763290', 'n07763483', 'n07763629', 'n07763792', 'n07763987', 'n07764155', 'n07764315', 'n07764486', + 'n07764630', 'n07764847', 'n07765073', 'n07765208', 'n07765361', 'n07765517', 'n07765612', 'n07765728', 'n07765862', + 'n07765999', 'n07766173', 'n07766409', 'n07766530', 'n07766723', 'n07766891', 'n07767002', 'n07767847', 'n07768068', + 'n07768139', 'n07768230', 'n07768318', 'n07768590', 'n07768858', 'n07769102', 'n07769306', 'n07769584', 'n07769731', + 'n07769886', 'n07770034', 'n07770180', 'n07770439', 'n11636835', 'n11700279', 'n12036067', 'n12036226', 'n12158031', + 'n12815838', 'n12162758', 'n12193334', 'n12301445', 'n12642090', 'n12644283', 'n12647787', 'n12650805', 'n12658481', + 'n12144313', 'n13135692', 'n13135832', 'n07770571', 'n07770763', 'n07770869', 'n07775197', 'n07814634', 'n07929351', + 'n11685091', 'n11689197', 'n11689367', 'n11689483', 'n11689678', 'n11689815', 'n11689957', 'n15086247', 'n11946313', + 'n12156819', 'n11823305', 'n12123648', 'n12142357', 'n12157056', 'n12157179', 'n12585373', 'n12592839', 'n12593341', + 'n12696830', 'n12928819', 'n13136316', 'n11750173', 'n11766046', 'n12487058', 'n12493426', 'n12532564', 'n12576323', + 'n13136556', 'n07737081', 'n07737594', 'n07737745', 'n07750586', 'n07750736', 'n07769465', 'n07771082', 'n07771212', + 'n07771539', 'n07771405', 'n07771731', 'n07771891', 'n07772026', 'n07772147', 'n07772274', 'n07772413', 'n07772788', + 'n07772935', 'n07774182', 'n07774295', 'n07774596', 'n07774719', 'n07774842', 'n07775050', 'n11612235', 'n12197601', + 'n12280364', 'n12590715', 'n13136781', 'n13137409', 'n07743902', 'n11723986', 'n13137951', 'n13137672', 'n13137225', + 'n13138308', 'n07751004', 'n07767344', 'n07767709', 'n07767549', 'n13138658', 'n07744811', 'n13138155', 'n13138842', + 'n07739125', 'n07739344', 'n13139055', 'n11748002', 'n12515925', 'n12544539', 'n12560282', 'n12560621', 'n12561594', + 'n12578916', 'n11748811', 'n11766432', 'n12172364', 'n13139321', 'n13139482', 'n13140367', 'n13141415', 'n13150378', + 'n13150592'), + # fungus n12992868 + ( + 'n12992868', 'n13035241', 'n13035707', 'n13036312', 'n13035925', 'n13036116', 'n13036804', 'n12982468', 'n12982590', + 'n12985420', 'n13079419', 'n13079567', 'n13001930', 'n13023134', 'n12987056', 'n12987535', 'n12988158', 'n12988341', + 'n12988572', 'n12991184', 'n12989938', 'n12991837', 'n12989007', 'n12987423', 'n12992177', 'n12990597', 'n13027879', + 'n13081999', 'n12979829', 'n13077295', 'n12963628', 'n12980840', 'n12981301', 'n12981443', 'n12981086', 'n12969131', + 'n12969670', 'n12969425', 'n12969927', 'n13025647', 'n13026015', 'n13025854', 'n13046669', 'n13081229', 'n13078021', + 'n13002209', 'n13079073', 'n13015509', 'n13060190', 'n13062421', 'n13061348', 'n13061704', 'n13061471', 'n13061172', + 'n13080306', 'n12964920', 'n12970733', 'n13080866', 'n13018906', 'n13048447', 'n12966945', 'n13066129', 'n13067672', + 'n13067532', 'n13068434', 'n13067191', 'n13067330', 'n13066979', 'n13068255', 'n13068917', 'n13069224', 'n13068735', + 'n13066448', 'n12971400', 'n12971804', 'n12972136', 'n13046130', 'n13045210', 'n13045975', 'n12978076', 'n12965626', + 'n12965951', 'n13034555', 'n13045594', 'n12985773', 'n13042982', 'n13041312', 'n13040629', 'n13040796', 'n13082568', + 'n13016076', 'n13024012', 'n12983873', 'n13024500', 'n13024653', 'n12983961', 'n12984489', 'n12984595', 'n12984267', + 'n13028611', 'n13029122', 'n13031323', 'n13029326', 'n13031474', 'n13029760', 'n13031193', 'n13030616', 'n13029610', + 'n13030852', 'n13028937', 'n12982915', 'n13047862', 'n13016289', 'n13077033', 'n12973791', 'n12973937', 'n12980080', + 'n12973443', 'n12981954', 'n12995601', 'n12966804', 'n13063269', 'n13065514', 'n13065089', 'n13064111', 'n13064457', + 'n13035389', 'n13038577', 'n13043926', 'n13044375', 'n13027557', 'n12968136', 'n12968309', 'n13037585', 'n13037805', + 'n13038376', 'n13038068', 'n13038744', 'n13069773', 'n13054073', 'n13039349', 'n12970193', 'n12970293', 'n13042316', + 'n13042134', 'n13041943', 'n12986227', 'n13056799', 'n13059298', 'n13056135', 'n13058272', 'n13056349', 'n13057054', + 'n13055577', 'n13057422', 'n13058608', 'n13058037', 'n13055792', 'n13060017', 'n13055423', 'n13055949', 'n13057242', + 'n13057639', 'n13059657', 'n13056607', 'n12997654', 'n13049953', 'n13052931', 'n13050940', 'n13053608', 'n13050705', + 'n13050397', 'n13051346', 'n13052248', 'n13052014', 'n13011595', 'n12997919', 'n13032115', 'n13033879', 'n13032381', + 'n13033577', 'n13032618', 'n13034062', 'n13032923', 'n13033134', 'n13033396', 'n13002750', 'n13232106', 'n13003254', + 'n13001366', 'n13008839', 'n13004826', 'n13075020', 'n13009656', 'n13009244', 'n13020481', 'n13003712', 'n13003522', + 'n13018407', 'n13004640', 'n13022210', 'n13017240', 'n13002925', 'n13074814', 'n13008157', 'n13020964', 'n13017979', + 'n13232363', 'n13017610', 'n13076405', 'n13012469', 'n13076831', 'n13017439', 'n13007629', 'n13232779', 'n13020191', + 'n13004423', 'n13013534', 'n13014097', 'n13014581', 'n13014741', 'n13014879', 'n13014409', 'n13013965', 'n13014265', + 'n13006631', 'n13005984', 'n13010951', 'n13008485', 'n13021867', 'n13000891', 'n13013764', 'n13021543', 'n13017789', + 'n13009429', 'n13005329', 'n13006171', 'n13010694', 'n13075441', 'n13001206', 'n13017102', 'n13076041', 'n13018088', + 'n13021332', 'n13018232', 'n13070308', 'n13072350', 'n13072209', 'n13070875', 'n13072863', 'n13072031', 'n13073703', + 'n13072528', 'n13071815', 'n13072706', 'n13073055', 'n13071371', 'n13071553', 'n13075684', 'n13075847', 'n13004992', + 'n13021689', 'n13231678', 'n13001041', 'n13009085', 'n13008689', 'n13001529', 'n13012253', 'n13231919', 'n13019496', + 'n13074619', 'n13006894', 'n13019835', 'n13075272', 'n13003061', 'n13012973', 'n13011221', 'n13019643', 'n13076643', + 'n13008315', 'n13021166', 'n13007417', 'n13015688', 'n12974987', 'n12975804', 'n12976198', 'n12976554', 'n12983654', + 'n12979316', 'n13226871', 'n13034788', 'n12983048'), + # geological formation n09287968 + ( + 'n09287968', 'n09201998', 'n09217230', 'n09393524', 'n09238926', 'n09239302', 'n09257843', 'n09294877', 'n09259025', + 'n09398677', 'n09264803', 'n09266604', 'n09283866', 'n09309292', 'n09289331', 'n09194227', 'n09255070', 'n09396608', + 'n09391774', 'n09308572', 'n09295210', 'n09308743', 'n09309046', 'n09309168', 'n09331251', 'n09348460', 'n09357447', + 'n09362316', 'n09366017', 'n09215437', 'n09245515', 'n09421031', 'n09457979', 'n09330378', 'n09376526', 'n09186592', + 'n09415671', 'n09448690', 'n09259219', 'n09249155', 'n09344324', 'n09304750', 'n09230041', 'n09474765', 'n09290350', + 'n09214269', 'n09226869', 'n09268007', 'n09402944', 'n09422190', 'n09425019', 'n09454744', 'n09398076', 'n09403086', + 'n09344198', 'n09335809', 'n09422631', 'n09435739', 'n09452291', 'n09262690', 'n09289596', 'n09300306', 'n09206896', + 'n09269882', 'n09474010', 'n09305031', 'n09375606', 'n09405787', 'n09290444', 'n09295946', 'n09233446', 'n09410224', + 'n09366317', 'n09302616', 'n09269341', 'n09453008', 'n09351905', 'n09456207', 'n09303008', 'n09230202', 'n09283405', + 'n09326662', 'n09199101', 'n09327077', 'n09357346', 'n09459979', 'n09359803', 'n09218641', 'n09362945', 'n09396465', + 'n09409512', 'n09213434', 'n09224725', 'n09421799', 'n09433312', 'n09214060', 'n09270735', 'n09429630', 'n09274305', + 'n09337253', 'n09219233', 'n09406793', 'n09210862', 'n09214916', 'n09411295', 'n09452760', 'n09376786', 'n09403734', + 'n09409752', 'n09205509', 'n09433442', 'n08596076', 'n09335693', 'n09428628', 'n09458269', 'n09447666', 'n09437454', + 'n09206985', 'n09466678', 'n09213565', 'n09475925', 'n09415584', 'n09233603', 'n09248153', 'n09265620', 'n09269472', + 'n09445008', 'n09274152', 'n09303528', 'n09228055', 'n09361517', 'n09391644', 'n09436444', 'n09305898', 'n09454153', + 'n09470222', 'n09472413', 'n09344724', 'n09231117', 'n09474412', 'n09476123'), + # hoofed animal n02370806 + ( + 'n02370806', 'n02373336', 'n02374149', 'n02390258', 'n02391617', 'n02390101', 'n02389346', 'n02389559', 'n02389779', + 'n02389865', 'n02390015', 'n02389943', 'n02390454', 'n02390834', 'n02390938', 'n02390640', 'n02390738', 'n02391234', + 'n02391373', 'n02391508', 'n02374451', 'n02382948', 'n02385776', 'n02385580', 'n02383231', 'n02385098', 'n02388143', + 'n02385214', 'n02385676', 'n02388276', 'n02388453', 'n02375862', 'n02385898', 'n02389261', 'n02382204', 'n02380335', + 'n02382039', 'n02380583', 'n02380745', 'n02381460', 'n02381831', 'n02381609', 'n02386310', 'n02386496', 'n02386853', + 'n02387452', 'n02387346', 'n02387887', 'n02386968', 'n02387093', 'n02386746', 'n02382338', 'n02387254', 'n02382132', + 'n02375302', 'n02388917', 'n02388588', 'n02376918', 'n02377181', 'n02377291', 'n02377388', 'n02387983', 'n02380464', + 'n02388832', 'n02386014', 'n02386224', 'n02386141', 'n02382437', 'n02382850', 'n02382750', 'n02382635', 'n02377480', + 'n02377603', 'n02389128', 'n02387722', 'n02375757', 'n02377703', 'n02379430', 'n02381364', 'n02378870', 'n02379908', + 'n02379081', 'n02379743', 'n02379630', 'n02378415', 'n02378625', 'n02378755', 'n02378541', 'n02378299', 'n02378969', + 'n02381004', 'n02380052', 'n02381119', 'n02379183', 'n02381261', 'n02379329', 'n02378149', 'n02388735', 'n02375438', + 'n02384741', 'n02393580', 'n02393807', 'n02393940', 'n02391994', 'n02392824', 'n02392434', 'n02393161', 'n02392555', + 'n02394477', 'n02438580', 'n02397529', 'n02397744', 'n02397987', 'n02437136', 'n02437482', 'n02399000', 'n02429456', + 'n02401031', 'n02418064', 'n02418465', 'n02419336', 'n02419056', 'n02419634', 'n02418770', 'n02411206', 'n02411705', + 'n02412210', 'n02413131', 'n02413917', 'n02414043', 'n02414442', 'n02413593', 'n02414209', 'n02414290', 'n02413717', + 'n02413824', 'n02413484', 'n02411999', 'n02413050', 'n02419796', 'n02424085', 'n02424695', 'n02425228', 'n02424909', + 'n02423218', 'n02423589', 'n02423362', 'n02421449', 'n02425887', 'n02421136', 'n02421792', 'n02427724', 'n02427576', + 'n02427470', 'n02428349', 'n02428508', 'n02426813', 'n02427032', 'n02427183', 'n02425086', 'n02424305', 'n02424589', + 'n02424486', 'n02420828', 'n02422391', 'n02428089', 'n02426176', 'n02420509', 'n02426481', 'n02425532', 'n02414578', + 'n02414763', 'n02416104', 'n02415130', 'n02414904', 'n02415435', 'n02415829', 'n02415253', 'n02416519', 'n02417534', + 'n02417785', 'n02417663', 'n02417070', 'n02417387', 'n02417242', 'n02416820', 'n02416964', 'n02416880', 'n02410702', + 'n02410900', 'n02407959', 'n02409202', 'n02409508', 'n02409038', 'n02408817', 'n02408660', 'n02402010', 'n02404573', + 'n02404906', 'n02402425', 'n02403231', 'n02403153', 'n02403325', 'n02405692', 'n02406859', 'n02403454', 'n02405577', + 'n02406952', 'n02406046', 'n02404186', 'n02406174', 'n02402175', 'n02405101', 'n02405302', 'n02405440', 'n02409870', + 'n02428842', 'n02430045', 'n02431976', 'n02430830', 'n02435216', 'n02432511', 'n02432704', 'n02435517', 'n02434712', + 'n02431122', 'n02431542', 'n02431337', 'n02431441', 'n02432291', 'n02433318', 'n02433925', 'n02434190', 'n02434415', + 'n02431628', 'n02430748', 'n02432983', 'n02431785', 'n02434954', 'n02433546', 'n02433729', 'n02435853', 'n02436224', + 'n02436353', 'n02436645', 'n02439033', 'n02439398', 'n02437971', 'n02438272', 'n02438173', 'n02395003', 'n02395931', + 'n02396014', 'n02396088', 'n02396796', 'n02396157', 'n02372140'), + # insect n02159955 + ( + 'n02159955', 'n02160947', 'n02161225', 'n02161338', 'n02161457', 'n02161588', 'n02162561', 'n02163008', 'n02163297', + 'n02164464', 'n02165877', 'n02166229', 'n02166567', 'n02166826', 'n02167505', 'n02167820', 'n02167944', 'n02168245', + 'n02168427', 'n02169023', 'n02169218', 'n02169705', 'n02169974', 'n02170400', 'n02170599', 'n02170738', 'n02170993', + 'n02171164', 'n02171453', 'n02171869', 'n02172518', 'n02172678', 'n02172761', 'n02172870', 'n02173113', 'n02173373', + 'n02173784', 'n02174355', 'n02174659', 'n02175014', 'n02175569', 'n02175916', 'n02176261', 'n02176439', 'n02176747', + 'n02177196', 'n02177506', 'n02177775', 'n02178411', 'n02178717', 'n02181235', 'n02181724', 'n02182045', 'n02182355', + 'n02182642', 'n02182930', 'n02179012', 'n02179192', 'n02179340', 'n02180233', 'n02179891', 'n02180427', 'n02180875', + 'n02183096', 'n02183507', 'n02183857', 'n02184473', 'n02184589', 'n02184720', 'n02185167', 'n02185481', 'n02186153', + 'n02186717', 'n02187150', 'n02187279', 'n02187554', 'n02187900', 'n02188699', 'n02189363', 'n02189670', 'n02190790', + 'n02191273', 'n02191773', 'n02191979', 'n02192252', 'n02192513', 'n02192814', 'n02193009', 'n02193163', 'n02194249', + 'n02194750', 'n02195091', 'n02195526', 'n02195819', 'n02199502', 'n02196119', 'n02196344', 'n02196896', 'n02197185', + 'n02197689', 'n02197877', 'n02198532', 'n02198859', 'n02199170', 'n02200198', 'n02200630', 'n02200850', 'n02201000', + 'n02201497', 'n02201626', 'n02202006', 'n02202124', 'n02202287', 'n02202678', 'n02203152', 'n02203978', 'n02204249', + 'n02205673', 'n02203592', 'n02204722', 'n02204907', 'n02205219', 'n02198129', 'n02206270', 'n02220055', 'n02220225', + 'n02220518', 'n02220804', 'n02221083', 'n02221414', 'n02221571', 'n02221715', 'n02221820', 'n02222035', 'n02222582', + 'n02222321', 'n02207179', 'n02208280', 'n02208498', 'n02208848', 'n02208979', 'n02209111', 'n02209354', 'n02209624', + 'n02209964', 'n02210427', 'n02210921', 'n02211444', 'n02211627', 'n02211896', 'n02212062', 'n02212602', 'n02212958', + 'n02214096', 'n02213107', 'n02213239', 'n02213663', 'n02213788', 'n02213543', 'n02214341', 'n02214499', 'n02214773', + 'n02215161', 'n02215621', 'n02215770', 'n02216211', 'n02216365', 'n02216740', 'n02214660', 'n02217563', 'n02218134', + 'n02218371', 'n02218713', 'n02219015', 'n02207449', 'n02207805', 'n02207647', 'n02223266', 'n02223520', 'n02225798', + 'n02224023', 'n02224713', 'n02225081', 'n02226183', 'n02226821', 'n02226970', 'n02227247', 'n02227604', 'n02227966', + 'n02228341', 'n02228697', 'n02229156', 'n02229765', 'n02230023', 'n02230187', 'n02230480', 'n02230634', 'n02231052', + 'n02231803', 'n02232223', 'n02233943', 'n02234355', 'n02234570', 'n02234848', 'n02235205', 'n02236241', 'n02236355', + 'n02236896', 'n02237424', 'n02237581', 'n02237868', 'n02238235', 'n02238358', 'n02238594', 'n02238887', 'n02239192', + 'n02239528', 'n02239774', 'n02240068', 'n02240517', 'n02241008', 'n02241426', 'n02241569', 'n02241799', 'n02242137', + 'n02242455', 'n02243209', 'n02243562', 'n02243878', 'n02244173', 'n02244515', 'n02244797', 'n02245111', 'n02245443', + 'n02246011', 'n02246628', 'n02246941', 'n02247216', 'n02247511', 'n02247655', 'n02248062', 'n02248368', 'n02248510', + 'n02248887', 'n02249134', 'n02249515', 'n02249809', 'n02250280', 'n02250822', 'n02251067', 'n02251233', 'n02251593', + 'n02251775', 'n02252226', 'n02252799', 'n02252972', 'n02253127', 'n02253264', 'n02253494', 'n02253715', 'n02253913', + 'n02254246', 'n02254697', 'n02254901', 'n02255023', 'n02255391', 'n02256172', 'n02257003', 'n02257284', 'n02257715', + 'n02257985', 'n02258198', 'n02258508', 'n02258629', 'n02259377', 'n02259708', 'n02259987', 'n02260421', 'n02260863', + 'n02261063', 'n02261419', 'n02261757', 'n02262178', 'n02262449', 'n02262803', 'n02263378', 'n02264021', 'n02264885', + 'n02265330', 'n02266050', 'n02266421', 'n02266864', 'n02267208', 'n02267483', 'n02268148', 'n02269196', 'n02269340', + 'n02270011', 'n02270200', 'n02270945', 'n02270623', 'n02271222', 'n02271570', 'n02271897', 'n02272286', 'n02272552', + 'n02272871', 'n02273392', 'n02274024', 'n02274259', 'n02274822', 'n02275560', 'n02275773', 'n02276078', 'n02276355', + 'n02276749', 'n02276902', 'n02277094', 'n02277268', 'n02277422', 'n02278024', 'n02278210', 'n02278463', 'n02278839', + 'n02278980', 'n02279257', 'n02279637', 'n02280458', 'n02281015', 'n02281136', 'n02281267', 'n02282257', 'n02282385', + 'n02282553', 'n02282903', 'n02283077', 'n02283201', 'n02283617', 'n02283951', 'n02284224', 'n02284611', 'n02284884', + 'n02285179', 'n02285548', 'n02286089', 'n02286425', 'n02286654', 'n02287004', 'n02287352', 'n02287622', 'n02288789', + 'n02289307', 'n02289610', 'n02289988', 'n02290340', 'n02290664', 'n02290870', 'n02291220', 'n02291572', 'n02291748', + 'n02292085', 'n02292401', 'n02292692', 'n02293352', 'n02293868', 'n02294097', 'n02294407', 'n02295064', 'n02295870', + 'n02296021', 'n02296276', 'n02296612', 'n02297294', 'n02297819', 'n02298095', 'n02298541', 'n02299039', 'n02299378', + 'n02299846', 'n02300173', 'n02300554', 'n02301452', 'n02301935', 'n02302244', 'n02302459', 'n02303585', 'n02305085', + 'n02302969', 'n02303284', 'n02304036', 'n02304432', 'n02304657', 'n02304797', 'n02305407', 'n02305636', 'n02305929', + 'n02306433', 'n02306825', 'n02307515', 'n02307910', 'n02308471', 'n02308618', 'n02307176', 'n02312427', 'n02312640', + 'n02312912', 'n02313008', 'n02207345'), + # musical instrument n03800933 + ( + 'n03800933', 'n02795978', 'n02803349', 'n02803934', 'n02804123', 'n02804252', 'n03301568', 'n03512030', 'n02940706', + 'n03279153', 'n03273551', 'n04376400', 'n04419642', 'n03597916', 'n03614532', 'n04376400', 'n02990758', 'n03038870', + 'n03039015', 'n03496296', 'n04278247', 'n04537436', 'n03928116', 'n02766792', 'n03086457', 'n03738066', 'n04278353', + 'n03801353', 'n03915437', 'n03928116', 'n02766792', 'n03086457', 'n03738066', 'n04278353', 'n02869249', 'n02965529', + 'n03483230', 'n03157348', 'n03518829', 'n04614844', 'n02803666', 'n02869737', 'n04249415', 'n04382334', 'n04387201', + 'n04387400', 'n04410086', 'n04436542', 'n03440682', 'n03612965', 'n03633632', 'n04049753', 'n04480853', 'n04532831', + 'n04338517', 'n03038870', 'n03039015', 'n03496296', 'n04278247', 'n04537436', 'n03928116', 'n02766792', 'n03086457', + 'n03738066', 'n04278353', 'n02880546', 'n02803934', 'n04536153', 'n04536465', 'n04536595', 'n04536765', 'n04536335', + 'n02700895', 'n03465500', 'n04330998', 'n03025886', 'n02776978', 'n02682407', 'n03699280', 'n03698360', 'n03716966', + 'n03716887', 'n03254862', 'n03467517', 'n02804123', 'n03035832', 'n03499907', 'n04506289', 'n03628215', 'n04016846', + 'n04132603', 'n04224842', 'n04615226', 'n03254737', 'n04586932', 'n02891788', 'n02804252', 'n03301568', 'n03512030', + 'n02793089', 'n02912894', 'n04174500', 'n03369276', 'n04141198', 'n04123123', 'n03393324', 'n02701730', 'n03086670', + 'n02786736', 'n03494537', 'n03609397', 'n03854815', 'n03254625', 'n04067231', 'n04542595', 'n04263950', 'n04542474', + 'n04067143', 'n03945615', 'n02775483', 'n03800371', 'n03006626', 'n03245724', 'n03343354', 'n03355468', 'n03945459', + 'n03912218', 'n03950647', 'n03989777', 'n04579667', 'n04598582', 'n02817799', 'n03228016', 'n03096439', 'n04410365', + 'n03288742', 'n03628831', 'n03510866', 'n03800485', 'n03839172', 'n03839276', 'n04186624', 'n03393199', 'n04222847', + 'n03037709', 'n02803539', 'n02803809', 'n02834027', 'n03537550', 'n03334492', 'n03831757', 'n03929091'), + # primate n02469914 + ( + 'n02469914', 'n02496913', 'n02499808', 'n02499022', 'n02498153', 'n02499568', 'n02499316', 'n02498743', 'n02500596', + 'n02470238', 'n02470709', 'n02471300', 'n02478875', 'n02479332', 'n02471762', 'n02473983', 'n02478239', 'n02472293', + 'n02472987', 'n02474777', 'n02475358', 'n02475669', 'n02473307', 'n02473720', 'n02473857', 'n02475078', 'n02474605', + 'n02474110', 'n10528148', 'n02474282', 'n02476219', 'n02476870', 'n02477187', 'n02477329', 'n02477516', 'n02476567', + 'n02477028', 'n02477782', 'n02473554', 'n02501583', 'n02502006', 'n02501923', 'n02496052', 'n02470325', 'n02470899', + 'n02483092', 'n02480153', 'n02484322', 'n02484473', 'n02485988', 'n02488894', 'n02485225', 'n02485688', 'n02485371', + 'n02485536', 'n02486657', 'n02487079', 'n02486908', 'n02487847', 'n02488003', 'n02487547', 'n02487675', 'n02488415', + 'n02489589', 'n02494383', 'n02493224', 'n02492948', 'n02492356', 'n02490597', 'n02490811', 'n02491107'), + # reptile n01661091 + ( + 'n01661091', 'n01661592', 'n01662622', 'n01662784', 'n01663401', 'n01663782', 'n01664369', 'n01664492', 'n01664674', + 'n01664990', 'n01665932', 'n01666585', 'n01666228', 'n01667432', 'n01668091', 'n01668436', 'n01668665', 'n01668892', + 'n01669372', 'n01669654', 'n01670092', 'n01670535', 'n01670802', 'n01671125', 'n01671479', 'n01671705', 'n01672032', + 'n01672432', 'n01672611', 'n01661818', 'n01673282', 'n01674216', 'n01674464', 'n01674990', 'n01675352', 'n01676755', + 'n01677747', 'n01678043', 'n01678343', 'n01678657', 'n01679005', 'n01679307', 'n01679626', 'n01679962', 'n01680264', + 'n01680478', 'n01680655', 'n01680813', 'n01680983', 'n01681328', 'n01681653', 'n01681940', 'n01682172', 'n01682435', + 'n01683201', 'n01683558', 'n01684133', 'n01684578', 'n01684741', 'n01685439', 'n01686044', 'n01686220', 'n01686403', + 'n01686609', 'n01686808', 'n01687128', 'n01687290', 'n01687665', 'n01688961', 'n01689081', 'n01689411', 'n01690149', + 'n01690466', 'n01691217', 'n01691652', 'n01691951', 'n01692523', 'n01692864', 'n01693175', 'n01693783', 'n01694311', + 'n01694709', 'n01694955', 'n01701551', 'n01701859', 'n01702256', 'n01702479', 'n01703011', 'n01703161', 'n01703569', + 'n01704103', 'n01704626', 'n01705010', 'n01705591', 'n01705934', 'n01707294', 'n01708106', 'n01708998', 'n01709484', + 'n01709876', 'n01712008', 'n01712752', 'n01713170', 'n01713764', 'n01714231', 'n01715888', 'n01717016', 'n01717229', + 'n01717467', 'n01718096', 'n01718414', 'n01710177', 'n01711160', 'n01722998', 'n01723579', 'n01724231', 'n01724840', + 'n01725086', 'n01725713', 'n01726203', 'n01696633', 'n01698434', 'n01698782', 'n01697178', 'n01697611', 'n01697749', + 'n01697978', 'n01699040', 'n01699254', 'n01699675', 'n01726692', 'n01727646', 'n01728266', 'n01729672', 'n01730185', + 'n01730307', 'n01730563', 'n01730812', 'n01730960', 'n01731137', 'n01731277', 'n01731545', 'n01731764', 'n01731941', + 'n01732093', 'n01732244', 'n01732614', 'n01732789', 'n01732989', 'n01733214', 'n01733466', 'n01733757', 'n01733957', + 'n01734104', 'n01734637', 'n01734808', 'n01735439', 'n01735577', 'n01735728', 'n01736032', 'n01736375', 'n01736796', + 'n01737472', 'n01737728', 'n01737875', 'n01738065', 'n01738306', 'n01738601', 'n01738731', 'n01739094', 'n01739647', + 'n01739871', 'n01741232', 'n01741442', 'n01740551', 'n01740885', 'n01741562', 'n01741943', 'n01742447', 'n01742821', + 'n01743086', 'n01743605', 'n01743936', 'n01744100', 'n01744270', 'n01744555', 'n01745125', 'n01745484', 'n01745902', + 'n01746191', 'n01746359', 'n01746952', 'n01747285', 'n01747589', 'n01747885', 'n01748389', 'n01748686', 'n01748906', + 'n01749244', 'n01749582', 'n01749742', 'n01750167', 'n01750437', 'n01750743', 'n01751036', 'n01751215', 'n01751472', + 'n01752165', 'n01752585', 'n01752736', 'n01753032', 'n01753180', 'n01753959', 'n01754370', 'n01754533', 'n01754876', + 'n01755740', 'n01755952', 'n01756089', 'n01756508', 'n01756733', 'n01756916', 'n01757115', 'n01757343', 'n01757677', + 'n01757901', 'n01758141', 'n01662060', 'n01719403', 'n01721174', 'n01721898', 'n01722670'), + # utensil n04516672 + ( + 'n04516672', 'n02997607', 'n03262519', 'n03173270', 'n03317788', 'n03713436', 'n04414101', 'n04414319', 'n03984234', + 'n03018209', 'n02869155', 'n03125588', 'n04282992', 'n03992703', 'n02684248', 'n03698226', 'n04570214', 'n04326676', + 'n03104512', 'n03403643', 'n03621049', 'n03012499', 'n03101517', 'n03101986', 'n02805283', 'n02999138', 'n03101156', + 'n03983712', 'n03101796', 'n03284981', 'n03047799', 'n03453231', 'n03458422', 'n03459328', 'n03880531', 'n03242390', + 'n03271765', 'n04275283', 'n03846677', 'n03900301', 'n04097760', 'n04138977', 'n03226254', 'n04317325', 'n03972372', + 'n03990474', 'n03242506', 'n03915118', 'n03216562', 'n03259401', 'n03612814', 'n04397768', 'n03992975', 'n04139140', + 'n04324297', 'n04516214', 'n03064250', 'n04132985', 'n04399158', 'n04229959', 'n04309548', 'n04500060', 'n03352961', + 'n03881305', 'n03454885', 'n03621377', 'n03724417', 'n03767966', 'n03775199', 'n02850732', 'n03266371', 'n03272940', + 'n04578934', 'n04088441', 'n04103206', 'n04293119', 'n04059516', 'n04396902', 'n04175039'), + # vegetable n07707451 + ( + 'n07707451', 'n07708124', 'n07708398', 'n07708798', 'n07709046', 'n07724943', 'n07725158', 'n07726796', 'n07727048', + 'n07727140', 'n07727252', 'n07727377', 'n07727458', 'n07727578', 'n07727868', 'n07728053', 'n07728181', 'n07728284', + 'n07728391', 'n07728585', 'n07728708', 'n07728804', 'n07729000', 'n07729142', 'n07729225', 'n07729384', 'n07729828', + 'n07727741', 'n07729485', 'n07729926', 'n07725255', 'n07725376', 'n07725531', 'n07725789', 'n07725888', 'n07726009', + 'n07725663', 'n07726230', 'n07726386', 'n07726095', 'n07726672', 'n07709172', 'n07709333', 'n07709701', 'n07719437', + 'n07719616', 'n07719756', 'n07719980', 'n07720277', 'n07723330', 'n07723559', 'n07723753', 'n07723968', 'n07724078', + 'n07724173', 'n07724269', 'n07724492', 'n07724654', 'n07724819', 'n07730855', 'n07731006', 'n07731587', 'n07731767', + 'n07732747', 'n07732904', 'n07733005', 'n07733124', 'n07820036', 'n07733217', 'n07733712', 'n07733847', 'n07736256', + 'n07736371', 'n07736527', 'n07736692', 'n07710007', 'n07710616', 'n07710952', 'n07711371', 'n07711080', 'n07711232', + 'n07711799', 'n07713074', 'n07720442', 'n07720615', 'n07721018', 'n07721118', 'n07721195', 'n07721325', 'n07722052', + 'n07721456', 'n07721678', 'n07721833', 'n07721942', 'n07734017', 'n07734183', 'n07734292', 'n07734417', 'n07734555', + 'n07710283', 'n07710616', 'n07710952', 'n07711371', 'n07711907', 'n07712063', 'n07712267', 'n07719058', 'n07719839', + 'n07720084', 'n07720185', 'n07730207', 'n07730708', 'n07735052', 'n07735179', 'n07735294', 'n07735404', 'n07735687', + 'n07735803', 'n07735981', 'n07736087', 'n07736813', 'n07713267', 'n07713395', 'n07735687', 'n07713763', 'n07713895', + 'n07714078', 'n07714188', 'n07714287', 'n07714448', 'n07714895', 'n07714802', 'n07715221', 'n07715407', 'n07733567', + 'n07715561', 'n07717070', 'n07717714', 'n07717858', 'n07718068', 'n07718195', 'n07718329', 'n07715721', 'n07716034', + 'n07716203', 'n07716504', 'n07716649', 'n07716750', 'n07718671', 'n07718920', 'n07719213', 'n07719330', 'n07722217', + 'n07722390', 'n07722485', 'n07722666', 'n07722763', 'n07722888', 'n07723177', 'n07723039', 'n07730406', 'n07730562', + 'n07733394', 'n07735510', 'n07736971', 'n07768423', 'n07817871'), + # vehicle n04576211 + ( + 'n04576211', 'n02766534', 'n02804515', 'n02834778', 'n03853924', 'n04026813', 'n04126066', 'n04524716', 'n02869563', + 'n02959942', 'n02775039', 'n02932523', 'n03053976', 'n02885108', 'n04322692', 'n02986066', 'n03056097', 'n03360731', + 'n04070964', 'n04389521', 'n03465320', 'n03483971', 'n03710294', 'n03200357', 'n03828020', 'n04020912', 'n04236001', + 'n04246855', 'n04409279', 'n03484083', 'n02729222', 'n03490119', 'n03648431', 'n04176068', 'n04397027', 'n03897634', + 'n03538634', 'n02968473', 'n02794474', 'n02907296', 'n02909706', 'n02912557', 'n02931013', 'n02966068', 'n03002555', + 'n03009111', 'n03037590', 'n03055670', 'n04297098', 'n03247351', 'n03435991', 'n03436656', 'n03389889', 'n03492087', + 'n03638014', 'n03989199', 'n04302863', 'n04365112', 'n04486616', 'n03009269', 'n03669245', 'n04353573', 'n04103364', + 'n04149374', 'n04170037', 'n03919096', 'n04062807', 'n04566561', 'n02740533', 'n03886053', 'n02739889', 'n02740061', + 'n02740300', 'n02749292', 'n04389718', 'n03684823', 'n03025165', 'n03193597', 'n03193260', 'n03193423', 'n03585778', + 'n03939565', 'n04211219', 'n04373428', 'n04389854', 'n04465358', 'n03791235', 'n04368695', 'n02854630', 'n02958343', + 'n03404012', 'n04201733', 'n03472937', 'n03079136', 'n03119396', 'n03141065', 'n03881534', 'n03268790', 'n03421669', + 'n03493219', 'n03498781', 'n03539103', 'n03543394', 'n02831335', 'n03680512', 'n03770085', 'n03870105', 'n04322801', + 'n03342961', 'n04097373', 'n04166281', 'n04285965', 'n04302988', 'n04347119', 'n04459122', 'n04516354', 'n03389761', + 'n03506880', 'n03790512', 'n03769722', 'n04466871', 'n04490091', 'n03256166', 'n03632852', 'n03690473', 'n04465666', + 'n04474035', 'n04520170', 'n02871314', 'n03173929', 'n03648667', 'n03764822', 'n03884639', 'n03896419', 'n02946348', + 'n04520382', 'n03256788', 'n03538300', 'n04464852', 'n03886053', 'n02983507', 'n04250599', 'n04229007', 'n02916179', + 'n02712643', 'n04225987', 'n04467099', 'n02946509', 'n03904433', 'n04543158', 'n02787120', 'n02970849', 'n03217739', + 'n03255899', 'n04497249', 'n03235979', 'n03594010', 'n03981924', 'n04558059', 'n04560502', 'n03027505', 'n03122295', + 'n03765467', 'n04543924', 'n04563020', 'n04543509', 'n04571800') +) + +num_classes = 25 + +# mean and standard deviation of channels of ImageNet images +mean = [0.485, 0.456, 0.406] +std = [0.229, 0.224, 0.225] + +train_transform = trn.Compose([trn.Resize(256), trn.RandomCrop(224), trn.RandomHorizontalFlip(), + trn.ToTensor(), trn.Normalize(mean, std)]) +test_transform = trn.Compose([trn.Resize(256), trn.CenterCrop(224), trn.ToTensor(), trn.Normalize(mean, std)]) + +train_data = dset.ImageFolder( + root="/share/data/vision-greg/ImageNet/clsloc/images/train", + transform=train_transform) +test_data_in = dset.ImageFolder( + root="/share/data/vision-greg/ImageNet/clsloc/images/val", + transform=test_transform) + + +# print('Loading ImageNet-22K') +# # some ImageNet-22K images are broken JPEGs +# +# +# def my_collate(batch): +# batch = list(filter(lambda x: x is not None, batch)) +# return torch.utils.data.dataloader.default_collate(batch) +# +# +# class MyImageFolder(dset.ImageFolder): +# __init__ = dset.ImageFolder.__init__ +# +# def __getitem__(self, index): +# try: +# return super(MyImageFolder, self).__getitem__(index) +# except Exception as e: +# print(e) +# +# +# test_data_out = MyImageFolder(root="/share/data/vision-greg/ImageNet/clsloc/images/val", +# transform=trn.Compose([trn.Resize(256), trn.CenterCrop(224), +# trn.ToTensor(), trn.Normalize(mean, std)])) +# test_data_out.root = '/share/data/vision-greg/ImageNet22k' +# test_data_out.class_to_idx = pickle.load(open(test_data_out.root + '/class_to_idx.p', "rb")) +# test_data_out.classes = pickle.load(open(test_data_out.root + '/classes.p', "rb")) +# test_data_out.imgs = pickle.load(open(test_data_out.root + '/imgs.p', "rb")) +# print('Loaded ImageNet-22K') + + +print('Filtering Data') +im1k_classes_flat = tuple(np.concatenate(im1k_classes)) +train_data.imgs = [x for x in train_data.imgs if + x[0][x[0].index('train/') + 6:x[0].index('train/') + 6 + len('n07718472')] in im1k_classes_flat] +test_data_in.imgs = [x for x in test_data_in.imgs if + x[0][x[0].index('val/') + 4:x[0].index('val/') + 4 + len('n07718472')] in im1k_classes_flat] + +in_idx_to_superidx = {} +for c in im1k_classes_flat: + for i in range(len(im1k_classes)): # range(num_classes) + t = im1k_classes[i] + if c in t: + in_idx_to_superidx[train_data.class_to_idx[c]] = i + break +train_data.target_transform = lambda x: in_idx_to_superidx[x] +test_data_in.target_transform = lambda x: in_idx_to_superidx[x] + +# im22k_classes_flat = tuple(np.concatenate(im22k_classes)) +# test_data_out.imgs = [x for x in test_data_out.imgs if +# x[0][x[0].index('22k/images/') + 11:x[0].index('22k/images/') + 11 + len('n07718472')] +# in im22k_classes_flat] +# +# out_idx_to_superidx = {} +# for c in im22k_classes_flat: +# for i in range(len(im22k_classes)): # range(num_classes) +# t = im22k_classes[i] +# if c in t: +# out_idx_to_superidx[test_data_out.class_to_idx[c]] = i +# break +# test_data_out.target_transform = lambda x: out_idx_to_superidx[x] + +train_loader = torch.utils.data.DataLoader( + train_data, batch_size=args.batch_size, shuffle=True, + num_workers=args.prefetch, pin_memory=True) +test_loader_in = torch.utils.data.DataLoader( + test_data_in, batch_size=args.test_bs, shuffle=False, + num_workers=args.prefetch, pin_memory=True) +# test_loader_out = torch.utils.data.DataLoader( +# test_data_out, batch_size=args.test_bs, shuffle=False, +# num_workers=args.prefetch, pin_memory=True, collate_fn=my_collate) + + +print('Initializing Network') + + +class FineTuneModel(nn.Module): + """ + This freezes the weights of all layers except the last one. + + Arguments: + original_model: Model to finetune + arch: Name of model architecture + num_classes: Number of classes to tune for + """ + + def __init__(self, original_model, arch, num_classes): + super(FineTuneModel, self).__init__() + + if arch.startswith('alexnet') or arch.startswith('vgg'): + self.features = original_model.features + self.fc = nn.Sequential(*list(original_model.classifier.children())[:-1]) + self.classifier = nn.Sequential( + nn.Linear(4096, num_classes) + ) + elif arch.startswith('resnet') or arch.startswith('resnext'): + # Everything except the last linear layer + self.features = nn.Sequential(*list(original_model.children())[:-1]) + if arch == 'resnet18': + self.classifier = nn.Sequential( + nn.Linear(512, num_classes) + ) + else: + self.classifier = nn.Sequential( + nn.Linear(2048, num_classes) + ) + else: + raise ("Finetuning not supported on this architecture yet. Feel free to add") + + self.unfreeze(False) # Freeze weights except last layer + + def unfreeze(self, unfreeze): + # Freeze those weights + for p in self.features.parameters(): + p.requires_grad = unfreeze + if hasattr(self, 'fc'): + for p in self.fc.parameters(): + p.requires_grad = unfreeze + + def forward(self, x): + f = self.features(x) + if hasattr(self, 'fc'): + f = f.view(f.size(0), -1) + f = self.fc(f) + f = f.view(f.size(0), -1) + y = self.classifier(f) + return y + + +if args.model_name == 'alexnet': + net = models.AlexNet() + net.load_state_dict(model_zoo.load_url('https://download.pytorch.org/models/alexnet-owt-4df8aa71.pth', + model_dir='/share/data/lang/users/dan/.torch/models')) + net = FineTuneModel(net, 'alexnet', num_classes) + +elif args.model_name == 'squeezenet1.1': + net = models.SqueezeNet(version=1.1) + net.load_state_dict(model_zoo.load_url('https://download.pytorch.org/models/squeezenet1_1-f364aa15.pth', + model_dir='/share/data/lang/users/dan/.torch/models')) + + net.classifier = nn.Sequential( + nn.Dropout(p=0.5), + nn.Conv2d(512, num_classes, kernel_size=1), + nn.ReLU(inplace=True), + nn.AvgPool2d(13) + ) + net.forward = lambda x: net.classifier(net.features(x)).view(x.size(0), num_classes) + + for p in net.features.parameters(): + p.requires_grad = False + +elif 'vgg' in args.model_name: + if 'bn' not in args.model_name: + net = models.vgg19() + net.load_state_dict(model_zoo.load_url('https://download.pytorch.org/models/vgg19-dcbb9e9d.pth', + model_dir='/share/data/lang/users/dan/.torch/models')) + elif '11' in args.model_name: + net = models.vgg11() + net.load_state_dict(model_zoo.load_url('https://download.pytorch.org/models/vgg11-bbd30ac9.pth', + model_dir='/share/data/lang/users/dan/.torch/models')) + else: + net = models.vgg19_bn() + net.load_state_dict(model_zoo.load_url('https://download.pytorch.org/models/vgg19_bn-c79401a0.pth', + model_dir='/share/data/lang/users/dan/.torch/models')) + net = FineTuneModel(net, 'vgg', num_classes) + +elif args.model_name == 'resnet18': + net = models.resnet18() + net.load_state_dict(model_zoo.load_url('https://download.pytorch.org/models/resnet18-5c106cde.pth', + model_dir='/share/data/lang/users/dan/.torch/models')) + net = FineTuneModel(net, 'resnet18', num_classes) + +elif args.model_name == 'resnet50': + net = models.resnet50() + net.load_state_dict(model_zoo.load_url('https://download.pytorch.org/models/resnet50-19c8e357.pth', + model_dir='/share/data/lang/users/dan/.torch/models')) + net = FineTuneModel(net, 'resnet50', num_classes) + +elif args.model_name == 'resnext': + net = resnext_101_64x4d + net.load_state_dict(torch.load('/share/data/lang/users/dan/.torch/models/resnext_101_64x4d.pth')) + net = FineTuneModel(net, 'resnext', num_classes) + +print(args.model_name) +print(net) + +optimizer = torch.optim.SGD(filter(lambda p: p.requires_grad, net.parameters()), + state['learning_rate'], momentum=state['momentum'], + weight_decay=state['decay'], nesterov=True) + +start_epoch = 0 +# # Restore model +# if args.load != '': +# for i in range(1000 - 1, -1, -1): +# model_name = os.path.join(args.load, 'superclass_model_epoch' + str(i) + '.pth') +# if os.path.isfile(model_name): +# snapshot = torch.load(model_name) +# net.load_state_dict(checkpoint['state_dict']) +# optimizer.load_state_dict(checkpoint['optimizer']) +# print('Model restored! Epoch:', i) +# start_epoch = i + 1 +# break +# if start_epoch == 0: +# assert False, "could not resume" + + +if args.ngpu > 1: + net = torch.nn.DataParallel(net, device_ids=list(range(args.ngpu))) + +if args.ngpu > 0: + net.cuda() + torch.cuda.manual_seed(1) + +cudnn.benchmark = True # fire on all cylinders + +from tqdm import tqdm + +def train(): + net.train() # enter train mode + loss_avg = 0.0 + for batch_idx, (data, target) in enumerate(train_loader): + data, target = V(data.cuda()), V(target.cuda()) + + # forward + x = net(data) + + # backward + optimizer.zero_grad() + loss = F.cross_entropy(x, target) + loss.backward() + optimizer.step() + + # exponential moving average + loss_avg = loss_avg * 0.8 + loss.data[0] * 0.2 + + dt = math.pi / float(args.epochs) + state['tt'] += float(dt) / (len(train_loader.dataset) / float(args.batch_size)) + if state['tt'] >= math.pi - 0.01: + state['tt'] = math.pi - 0.01 + curT = math.pi / 2.0 + state['tt'] + new_lr = args.learning_rate * (1.0 + math.sin(curT)) / 2.0 # lr_min = 0, lr_max = lr + state['learning_rate'] = new_lr + for param_group in optimizer.param_groups: + param_group['lr'] = state['learning_rate'] + + state['train_loss'] = loss_avg + + +# test function +def test_in(): + net.eval() + loss_avg = 0.0 + correct = 0 + for batch_idx, (data, target) in enumerate(test_loader_in): + data, target = V(data.cuda(), volatile=True), V(target.cuda(), volatile=True) + + # forward + output = net(data) + loss = F.cross_entropy(output, target) + + # accuracy + pred = output.data.max(1)[1] + correct += pred.eq(target.data).sum() + + # test loss average + loss_avg += loss.data[0] + + state['test_in_loss'] = loss_avg / len(test_loader_in) + state['test_in_accuracy'] = correct / len(test_loader_in.dataset) + + +# # test function +# def test_out(): +# net.eval() +# correct = 0 +# for batch_idx, (data, target) in enumerate(test_loader_out): +# data, target = V(data.cuda(), volatile=True), V(target.cuda(), volatile=True) +# +# # forward +# output = net(data) +# +# # accuracy +# pred = output.data.max(1)[1] +# correct += pred.eq(target.data).sum() +# +# state['test_out_accuracy'] = correct / len(test_loader_out.dataset) + + +if args.test: + test_in() + # test_out() + print(state) + exit() + +state['learning_rate'] = state['init_learning_rate'] + +print('Beginning Training') +# Main loop +best_accuracy = 0.0 +for epoch in range(start_epoch, args.epochs): + for param_group in optimizer.param_groups: + param_group['lr'] = state['learning_rate'] + state['tt'] = math.pi / float(args.epochs) * epoch + + state['epoch'] = epoch + + begin_epoch = time.time() + train() + print('Epoch', epoch, '| Time Spent:', round(time.time() - begin_epoch, 4)) + + torch.save({ + 'state_dict': net.state_dict(), + 'optimizer': optimizer.state_dict() + }, os.path.join(args.save, args.model_name + '_superclass_epoch' + str(epoch) + '.pth')) + # Let us not waste space and delete the previous model + # We do not overwrite the model because we need the epoch number + try: os.remove(os.path.join(args.save, args.model_name + '_superclass_epoch' + str(epoch - 1) + '.pth')) + except: True + + test_in() + # test_out() + + print(state) diff --git a/data/datasets/visual_question_answering/generate_vqa_c_code.py b/data/datasets/visual_question_answering/generate_vqa_c_code.py new file mode 100644 index 0000000000000000000000000000000000000000..ebc7ff8b72a32cd66425b3be5f6d59d90647c1cb --- /dev/null +++ b/data/datasets/visual_question_answering/generate_vqa_c_code.py @@ -0,0 +1,29 @@ +template = """ +@dataset_register( + name='VQAv2_split1_c_{}', + classes=all_classes[0: 100], + task_type='Visual Question Answering', + object_type='Generic Object', + class_aliases=[], + shift_type=None +) +class VQAv2_split1_c_{}(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform: Optional[Compose], + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + + if transform is None: + transform = None + self.transform = transform + dataset = _VQAv2_split1_c(root_dir, split, "{}", classes, ignore_classes, idx_map) + return dataset +""" + + +# for c in 'gaussian_noise, shot_noise, impulse_noise, defocus_blur, glass_blur, motion_blur, zoom_blur, snow, frost, fog, brightness, contrast, elastic_transform, pixelate, jpeg_compression, speckle_noise, gaussian_blur, spatter, saturate'.split(', '): +# print(template.format(c, c, c)) +# print() + # break + + +classes_name = [f'VQAv2_split1_c_{c}' for c in 'gaussian_noise, shot_noise, impulse_noise, defocus_blur, glass_blur, motion_blur, zoom_blur, snow, frost, fog, brightness, contrast, elastic_transform, pixelate, jpeg_compression, speckle_noise, gaussian_blur, spatter, saturate'.split(', ')] +print(', '.join(classes_name)) \ No newline at end of file diff --git a/data/datasets/visual_question_answering/get_embeds.py b/data/datasets/visual_question_answering/get_embeds.py new file mode 100644 index 0000000000000000000000000000000000000000..d2b4ed55d3c990f7e1f2b9de085a2e4e13228d06 --- /dev/null +++ b/data/datasets/visual_question_answering/get_embeds.py @@ -0,0 +1,17 @@ +from .utils import Config +from .modeling_frcnn import GeneralizedRCNN +from .processing_image import Preprocess +from PIL import Image +from torchvision.transforms import Resize +import torch + +def get_visual_embeds(image_path): + config = Config.from_pretrained('data/datasets/visual_question_answering/frcnn') + frcnn = GeneralizedRCNN.from_pretrained('data/datasets/visual_question_answering/frcnn',config = config) + image_preprocess = Preprocess(config) + # image = Image.open('2.jpg').convert('RGB') + # image = Resize((224,224))(image) + images, sizes, scales_yx = image_preprocess(image_path) + output_dict = frcnn(images,sizes,scales_yx=scales_yx,padding="max_detections",max_detections=config.max_detections,return_tensors = "pt") + features = output_dict.get("roi_features") + return features \ No newline at end of file diff --git a/data/datasets/visual_question_answering/glossary.py b/data/datasets/visual_question_answering/glossary.py new file mode 100644 index 0000000000000000000000000000000000000000..81fddf30c92ad11d48ed2a9698488fa20724d563 --- /dev/null +++ b/data/datasets/visual_question_answering/glossary.py @@ -0,0 +1,190 @@ +import re + +contractions = { + "aint": "ain't", + "arent": "aren't", + "cant": "can't", + "couldve": "could've", + "couldnt": "couldn't", + "couldn'tve": "couldn't've", + "couldnt've": "couldn't've", + "didnt": "didn't", + "doesnt": "doesn't", + "dont": "don't", + "hadnt": "hadn't", + "hadnt've": "hadn't've", + "hadn'tve": "hadn't've", + "hasnt": "hasn't", + "havent": "haven't", + "hed": "he'd", + "hed've": "he'd've", + "he'dve": "he'd've", + "hes": "he's", + "howd": "how'd", + "howll": "how'll", + "hows": "how's", + "Id've": "I'd've", + "I'dve": "I'd've", + "Im": "I'm", + "Ive": "I've", + "isnt": "isn't", + "itd": "it'd", + "itd've": "it'd've", + "it'dve": "it'd've", + "itll": "it'll", + "let's": "let's", + "maam": "ma'am", + "mightnt": "mightn't", + "mightnt've": "mightn't've", + "mightn'tve": "mightn't've", + "mightve": "might've", + "mustnt": "mustn't", + "mustve": "must've", + "neednt": "needn't", + "notve": "not've", + "oclock": "o'clock", + "oughtnt": "oughtn't", + "ow's'at": "'ow's'at", + "'ows'at": "'ow's'at", + "'ow'sat": "'ow's'at", + "shant": "shan't", + "shed've": "she'd've", + "she'dve": "she'd've", + "she's": "she's", + "shouldve": "should've", + "shouldnt": "shouldn't", + "shouldnt've": "shouldn't've", + "shouldn'tve": "shouldn't've", + "somebody'd": "somebodyd", + "somebodyd've": "somebody'd've", + "somebody'dve": "somebody'd've", + "somebodyll": "somebody'll", + "somebodys": "somebody's", + "someoned": "someone'd", + "someoned've": "someone'd've", + "someone'dve": "someone'd've", + "someonell": "someone'll", + "someones": "someone's", + "somethingd": "something'd", + "somethingd've": "something'd've", + "something'dve": "something'd've", + "somethingll": "something'll", + "thats": "that's", + "thered": "there'd", + "thered've": "there'd've", + "there'dve": "there'd've", + "therere": "there're", + "theres": "there's", + "theyd": "they'd", + "theyd've": "they'd've", + "they'dve": "they'd've", + "theyll": "they'll", + "theyre": "they're", + "theyve": "they've", + "twas": "'twas", + "wasnt": "wasn't", + "wed've": "we'd've", + "we'dve": "we'd've", + "weve": "we've", + "werent": "weren't", + "whatll": "what'll", + "whatre": "what're", + "whats": "what's", + "whatve": "what've", + "whens": "when's", + "whered": "where'd", + "wheres": "where's", + "whereve": "where've", + "whod": "who'd", + "whod've": "who'd've", + "who'dve": "who'd've", + "wholl": "who'll", + "whos": "who's", + "whove": "who've", + "whyll": "why'll", + "whyre": "why're", + "whys": "why's", + "wont": "won't", + "wouldve": "would've", + "wouldnt": "wouldn't", + "wouldnt've": "wouldn't've", + "wouldn'tve": "wouldn't've", + "yall": "y'all", + "yall'll": "y'all'll", + "y'allll": "y'all'll", + "yall'd've": "y'all'd've", + "y'alld've": "y'all'd've", + "y'all'dve": "y'all'd've", + "youd": "you'd", + "youd've": "you'd've", + "you'dve": "you'd've", + "youll": "you'll", + "youre": "you're", + "youve": "you've", +} + +manual_map = { + "none": "0", + "zero": "0", + "one": "1", + "two": "2", + "three": "3", + "four": "4", + "five": "5", + "six": "6", + "seven": "7", + "eight": "8", + "nine": "9", + "ten": "10", +} +articles = ["a", "an", "the"] +period_strip = re.compile("(?!<=\d)(\.)(?!\d)") +comma_strip = re.compile("(\d)(\,)(\d)") +punct = [ + ";", + r"/", + "[", + "]", + '"', + "{", + "}", + "(", + ")", + "=", + "+", + "\\", + "_", + "-", + ">", + "<", + "@", + "`", + ",", + "?", + "!", +] + + +def normalize_word(token): + _token = token + for p in punct: + if (p + " " in token or " " + p in token) or ( + re.search(comma_strip, token) != None + ): + _token = _token.replace(p, "") + else: + _token = _token.replace(p, " ") + token = period_strip.sub("", _token, re.UNICODE) + + _token = [] + temp = token.lower().split() + for word in temp: + word = manual_map.setdefault(word, word) + if word not in articles: + _token.append(word) + for i, word in enumerate(_token): + if word in contractions: + _token[i] = contractions[word] + token = " ".join(_token) + token = token.replace(",", "") + return token diff --git a/data/datasets/visual_question_answering/modeling_frcnn.py b/data/datasets/visual_question_answering/modeling_frcnn.py new file mode 100644 index 0000000000000000000000000000000000000000..df1ec245bfd05dccaa6fab0e7b6a00e649f4817f --- /dev/null +++ b/data/datasets/visual_question_answering/modeling_frcnn.py @@ -0,0 +1,1921 @@ +""" + coding=utf-8 + Copyright 2018, Antonio Mendoza Hao Tan, Mohit Bansal + Adapted From Facebook Inc, Detectron2 && Huggingface Co. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License.import copy + """ +import itertools +import math +import os +from abc import ABCMeta, abstractmethod +from collections import OrderedDict, namedtuple +from typing import Dict, List, Tuple + +import numpy as np +import torch +from torch import nn +from torch.nn.modules.batchnorm import BatchNorm2d +from torchvision.ops import RoIPool +from torchvision.ops.boxes import batched_nms, nms + +from .utils import WEIGHTS_NAME, Config, cached_path, hf_bucket_url, is_remote_url, load_checkpoint + + +# other: +def norm_box(boxes, raw_sizes): + if not isinstance(boxes, torch.Tensor): + normalized_boxes = boxes.copy() + else: + normalized_boxes = boxes.clone() + normalized_boxes[:, :, (0, 2)] /= raw_sizes[:, 1] + normalized_boxes[:, :, (1, 3)] /= raw_sizes[:, 0] + return normalized_boxes + + +def pad_list_tensors( + list_tensors, + preds_per_image, + max_detections=None, + return_tensors=None, + padding=None, + pad_value=0, + location=None, +): + """ + location will always be cpu for np tensors + """ + if location is None: + location = "cpu" + assert return_tensors in {"pt", "np", None} + assert padding in {"max_detections", "max_batch", None} + new = [] + if padding is None: + if return_tensors is None: + return list_tensors + elif return_tensors == "pt": + if not isinstance(list_tensors, torch.Tensor): + return torch.stack(list_tensors).to(location) + else: + return list_tensors.to(location) + else: + if not isinstance(list_tensors, list): + return np.array(list_tensors.to(location)) + else: + return list_tensors.to(location) + if padding == "max_detections": + assert max_detections is not None, "specify max number of detections per batch" + elif padding == "max_batch": + max_detections = max(preds_per_image) + for i in range(len(list_tensors)): + too_small = False + tensor_i = list_tensors.pop(0) + if tensor_i.ndim < 2: + too_small = True + tensor_i = tensor_i.unsqueeze(-1) + assert isinstance(tensor_i, torch.Tensor) + tensor_i = nn.functional.pad( + input=tensor_i, + pad=(0, 0, 0, max_detections - preds_per_image[i]), + mode="constant", + value=pad_value, + ) + if too_small: + tensor_i = tensor_i.squeeze(-1) + if return_tensors is None: + if location == "cpu": + tensor_i = tensor_i.cpu() + tensor_i = tensor_i.tolist() + if return_tensors == "np": + if location == "cpu": + tensor_i = tensor_i.cpu() + tensor_i = tensor_i.numpy() + else: + if location == "cpu": + tensor_i = tensor_i.cpu() + new.append(tensor_i) + if return_tensors == "np": + return np.stack(new, axis=0) + elif return_tensors == "pt" and not isinstance(new, torch.Tensor): + return torch.stack(new, dim=0) + else: + return list_tensors + + +def do_nms(boxes, scores, image_shape, score_thresh, nms_thresh, mind, maxd): + scores = scores[:, :-1] + num_bbox_reg_classes = boxes.shape[1] // 4 + # Convert to Boxes to use the `clip` function ... + boxes = boxes.reshape(-1, 4) + _clip_box(boxes, image_shape) + boxes = boxes.view(-1, num_bbox_reg_classes, 4) # R x C x 4 + + # Select max scores + max_scores, max_classes = scores.max(1) # R x C --> R + num_objs = boxes.size(0) + boxes = boxes.view(-1, 4) + idxs = torch.arange(num_objs).to(boxes.device) * num_bbox_reg_classes + max_classes + max_boxes = boxes[idxs] # Select max boxes according to the max scores. + + # Apply NMS + keep = nms(max_boxes, max_scores, nms_thresh) + keep = keep[:maxd] + if keep.shape[-1] >= mind and keep.shape[-1] <= maxd: + max_boxes, max_scores = max_boxes[keep], max_scores[keep] + classes = max_classes[keep] + return max_boxes, max_scores, classes, keep + else: + return None + + +# Helper Functions +def _clip_box(tensor, box_size: Tuple[int, int]): + assert torch.isfinite(tensor).all(), "Box tensor contains infinite or NaN!" + h, w = box_size + tensor[:, 0].clamp_(min=0, max=w) + tensor[:, 1].clamp_(min=0, max=h) + tensor[:, 2].clamp_(min=0, max=w) + tensor[:, 3].clamp_(min=0, max=h) + + +def _nonempty_boxes(box, threshold: float = 0.0) -> torch.Tensor: + widths = box[:, 2] - box[:, 0] + heights = box[:, 3] - box[:, 1] + keep = (widths > threshold) & (heights > threshold) + return keep + + +def get_norm(norm, out_channels): + if isinstance(norm, str): + if len(norm) == 0: + return None + norm = { + "BN": BatchNorm2d, + "GN": lambda channels: nn.GroupNorm(32, channels), + "nnSyncBN": nn.SyncBatchNorm, # keep for debugging + "": lambda x: x, + }[norm] + return norm(out_channels) + + +def _create_grid_offsets(size: List[int], stride: int, offset: float, device): + grid_height, grid_width = size + shifts_x = torch.arange( + offset * stride, + grid_width * stride, + step=stride, + dtype=torch.float32, + device=device, + ) + shifts_y = torch.arange( + offset * stride, + grid_height * stride, + step=stride, + dtype=torch.float32, + device=device, + ) + + shift_y, shift_x = torch.meshgrid(shifts_y, shifts_x) + shift_x = shift_x.reshape(-1) + shift_y = shift_y.reshape(-1) + return shift_x, shift_y + + +def build_backbone(cfg): + input_shape = ShapeSpec(channels=len(cfg.MODEL.PIXEL_MEAN)) + norm = cfg.RESNETS.NORM + stem = BasicStem( + in_channels=input_shape.channels, + out_channels=cfg.RESNETS.STEM_OUT_CHANNELS, + norm=norm, + caffe_maxpool=cfg.MODEL.MAX_POOL, + ) + freeze_at = cfg.BACKBONE.FREEZE_AT + + if freeze_at >= 1: + for p in stem.parameters(): + p.requires_grad = False + + out_features = cfg.RESNETS.OUT_FEATURES + depth = cfg.RESNETS.DEPTH + num_groups = cfg.RESNETS.NUM_GROUPS + width_per_group = cfg.RESNETS.WIDTH_PER_GROUP + bottleneck_channels = num_groups * width_per_group + in_channels = cfg.RESNETS.STEM_OUT_CHANNELS + out_channels = cfg.RESNETS.RES2_OUT_CHANNELS + stride_in_1x1 = cfg.RESNETS.STRIDE_IN_1X1 + res5_dilation = cfg.RESNETS.RES5_DILATION + assert res5_dilation in {1, 2}, "res5_dilation cannot be {}.".format(res5_dilation) + + num_blocks_per_stage = {50: [3, 4, 6, 3], 101: [3, 4, 23, 3], 152: [3, 8, 36, 3]}[depth] + + stages = [] + out_stage_idx = [{"res2": 2, "res3": 3, "res4": 4, "res5": 5}[f] for f in out_features] + max_stage_idx = max(out_stage_idx) + for idx, stage_idx in enumerate(range(2, max_stage_idx + 1)): + dilation = res5_dilation if stage_idx == 5 else 1 + first_stride = 1 if idx == 0 or (stage_idx == 5 and dilation == 2) else 2 + stage_kargs = { + "num_blocks": num_blocks_per_stage[idx], + "first_stride": first_stride, + "in_channels": in_channels, + "bottleneck_channels": bottleneck_channels, + "out_channels": out_channels, + "num_groups": num_groups, + "norm": norm, + "stride_in_1x1": stride_in_1x1, + "dilation": dilation, + } + + stage_kargs["block_class"] = BottleneckBlock + blocks = ResNet.make_stage(**stage_kargs) + in_channels = out_channels + out_channels *= 2 + bottleneck_channels *= 2 + + if freeze_at >= stage_idx: + for block in blocks: + block.freeze() + stages.append(blocks) + + return ResNet(stem, stages, out_features=out_features) + + +def find_top_rpn_proposals( + proposals, + pred_objectness_logits, + images, + image_sizes, + nms_thresh, + pre_nms_topk, + post_nms_topk, + min_box_side_len, + training, +): + """Args: + proposals (list[Tensor]): (L, N, Hi*Wi*A, 4). + pred_objectness_logits: tensors of length L. + nms_thresh (float): IoU threshold to use for NMS + pre_nms_topk (int): before nms + post_nms_topk (int): after nms + min_box_side_len (float): minimum proposal box side + training (bool): True if proposals are to be used in training, + Returns: + results (List[Dict]): stores post_nms_topk object proposals for image i. + """ + num_images = len(images) + device = proposals[0].device + + # 1. Select top-k anchor for every level and every image + topk_scores = [] # #lvl Tensor, each of shape N x topk + topk_proposals = [] + level_ids = [] # #lvl Tensor, each of shape (topk,) + batch_idx = torch.arange(num_images, device=device) + for level_id, proposals_i, logits_i in zip(itertools.count(), proposals, pred_objectness_logits): + Hi_Wi_A = logits_i.shape[1] + num_proposals_i = min(pre_nms_topk, Hi_Wi_A) + + # sort is faster than topk (https://github.com/pytorch/pytorch/issues/22812) + # topk_scores_i, topk_idx = logits_i.topk(num_proposals_i, dim=1) + logits_i, idx = logits_i.sort(descending=True, dim=1) + topk_scores_i = logits_i[batch_idx, :num_proposals_i] + topk_idx = idx[batch_idx, :num_proposals_i] + + # each is N x topk + topk_proposals_i = proposals_i[batch_idx[:, None], topk_idx] # N x topk x 4 + + topk_proposals.append(topk_proposals_i) + topk_scores.append(topk_scores_i) + level_ids.append(torch.full((num_proposals_i,), level_id, dtype=torch.int64, device=device)) + + # 2. Concat all levels together + topk_scores = torch.cat(topk_scores, dim=1) + topk_proposals = torch.cat(topk_proposals, dim=1) + level_ids = torch.cat(level_ids, dim=0) + + # if I change to batched_nms, I wonder if this will make a difference + # 3. For each image, run a per-level NMS, and choose topk results. + results = [] + for n, image_size in enumerate(image_sizes): + boxes = topk_proposals[n] + scores_per_img = topk_scores[n] + # I will have to take a look at the boxes clip method + _clip_box(boxes, image_size) + # filter empty boxes + keep = _nonempty_boxes(boxes, threshold=min_box_side_len) + lvl = level_ids + if keep.sum().item() != len(boxes): + boxes, scores_per_img, lvl = ( + boxes[keep], + scores_per_img[keep], + level_ids[keep], + ) + + keep = batched_nms(boxes, scores_per_img, lvl, nms_thresh) + keep = keep[:post_nms_topk] + + res = (boxes[keep], scores_per_img[keep]) + results.append(res) + + # I wonder if it would be possible for me to pad all these things. + return results + + +def subsample_labels(labels, num_samples, positive_fraction, bg_label): + """ + Returns: + pos_idx, neg_idx (Tensor): + 1D vector of indices. The total length of both is `num_samples` or fewer. + """ + positive = torch.nonzero((labels != -1) & (labels != bg_label)).squeeze(1) + negative = torch.nonzero(labels == bg_label).squeeze(1) + + num_pos = int(num_samples * positive_fraction) + # protect against not enough positive examples + num_pos = min(positive.numel(), num_pos) + num_neg = num_samples - num_pos + # protect against not enough negative examples + num_neg = min(negative.numel(), num_neg) + + # randomly select positive and negative examples + perm1 = torch.randperm(positive.numel(), device=positive.device)[:num_pos] + perm2 = torch.randperm(negative.numel(), device=negative.device)[:num_neg] + + pos_idx = positive[perm1] + neg_idx = negative[perm2] + return pos_idx, neg_idx + + +def add_ground_truth_to_proposals(gt_boxes, proposals): + raise NotImplementedError() + + +def add_ground_truth_to_proposals_single_image(gt_boxes, proposals): + raise NotImplementedError() + + +def _fmt_box_list(box_tensor, batch_index: int): + repeated_index = torch.full( + (len(box_tensor), 1), + batch_index, + dtype=box_tensor.dtype, + device=box_tensor.device, + ) + return torch.cat((repeated_index, box_tensor), dim=1) + + +def convert_boxes_to_pooler_format(box_lists: List[torch.Tensor]): + pooler_fmt_boxes = torch.cat( + [_fmt_box_list(box_list, i) for i, box_list in enumerate(box_lists)], + dim=0, + ) + return pooler_fmt_boxes + + +def assign_boxes_to_levels( + box_lists: List[torch.Tensor], + min_level: int, + max_level: int, + canonical_box_size: int, + canonical_level: int, +): + box_sizes = torch.sqrt(torch.cat([boxes.area() for boxes in box_lists])) + # Eqn.(1) in FPN paper + level_assignments = torch.floor(canonical_level + torch.log2(box_sizes / canonical_box_size + 1e-8)) + # clamp level to (min, max), in case the box size is too large or too small + # for the available feature maps + level_assignments = torch.clamp(level_assignments, min=min_level, max=max_level) + return level_assignments.to(torch.int64) - min_level + + +# Helper Classes +class _NewEmptyTensorOp(torch.autograd.Function): + @staticmethod + def forward(ctx, x, new_shape): + ctx.shape = x.shape + return x.new_empty(new_shape) + + @staticmethod + def backward(ctx, grad): + shape = ctx.shape + return _NewEmptyTensorOp.apply(grad, shape), None + + +class ShapeSpec(namedtuple("_ShapeSpec", ["channels", "height", "width", "stride"])): + def __new__(cls, *, channels=None, height=None, width=None, stride=None): + return super().__new__(cls, channels, height, width, stride) + + +class Box2BoxTransform(object): + """ + This R-CNN transformation scales the box's width and height + by exp(dw), exp(dh) and shifts a box's center by the offset + (dx * width, dy * height). + """ + + def __init__(self, weights: Tuple[float, float, float, float], scale_clamp: float = None): + """ + Args: + weights (4-element tuple): Scaling factors that are applied to the + (dx, dy, dw, dh) deltas. In Fast R-CNN, these were originally set + such that the deltas have unit variance; now they are treated as + hyperparameters of the system. + scale_clamp (float): When predicting deltas, the predicted box scaling + factors (dw and dh) are clamped such that they are <= scale_clamp. + """ + self.weights = weights + if scale_clamp is not None: + self.scale_clamp = scale_clamp + else: + """ + Value for clamping large dw and dh predictions. + The heuristic is that we clamp such that dw and dh are no larger + than what would transform a 16px box into a 1000px box + (based on a small anchor, 16px, and a typical image size, 1000px). + """ + self.scale_clamp = math.log(1000.0 / 16) + + def get_deltas(self, src_boxes, target_boxes): + """ + Get box regression transformation deltas (dx, dy, dw, dh) that can be used + to transform the `src_boxes` into the `target_boxes`. That is, the relation + ``target_boxes == self.apply_deltas(deltas, src_boxes)`` is true (unless + any delta is too large and is clamped). + Args: + src_boxes (Tensor): source boxes, e.g., object proposals + target_boxes (Tensor): target of the transformation, e.g., ground-truth + boxes. + """ + assert isinstance(src_boxes, torch.Tensor), type(src_boxes) + assert isinstance(target_boxes, torch.Tensor), type(target_boxes) + + src_widths = src_boxes[:, 2] - src_boxes[:, 0] + src_heights = src_boxes[:, 3] - src_boxes[:, 1] + src_ctr_x = src_boxes[:, 0] + 0.5 * src_widths + src_ctr_y = src_boxes[:, 1] + 0.5 * src_heights + + target_widths = target_boxes[:, 2] - target_boxes[:, 0] + target_heights = target_boxes[:, 3] - target_boxes[:, 1] + target_ctr_x = target_boxes[:, 0] + 0.5 * target_widths + target_ctr_y = target_boxes[:, 1] + 0.5 * target_heights + + wx, wy, ww, wh = self.weights + dx = wx * (target_ctr_x - src_ctr_x) / src_widths + dy = wy * (target_ctr_y - src_ctr_y) / src_heights + dw = ww * torch.log(target_widths / src_widths) + dh = wh * torch.log(target_heights / src_heights) + + deltas = torch.stack((dx, dy, dw, dh), dim=1) + assert (src_widths > 0).all().item(), "Input boxes to Box2BoxTransform are not valid!" + return deltas + + def apply_deltas(self, deltas, boxes): + """ + Apply transformation `deltas` (dx, dy, dw, dh) to `boxes`. + Args: + deltas (Tensor): transformation deltas of shape (N, k*4), where k >= 1. + deltas[i] represents k potentially different class-specific + box transformations for the single box boxes[i]. + boxes (Tensor): boxes to transform, of shape (N, 4) + """ + boxes = boxes.to(deltas.dtype) + + widths = boxes[:, 2] - boxes[:, 0] + heights = boxes[:, 3] - boxes[:, 1] + ctr_x = boxes[:, 0] + 0.5 * widths + ctr_y = boxes[:, 1] + 0.5 * heights + + wx, wy, ww, wh = self.weights + dx = deltas[:, 0::4] / wx + dy = deltas[:, 1::4] / wy + dw = deltas[:, 2::4] / ww + dh = deltas[:, 3::4] / wh + + # Prevent sending too large values into torch.exp() + dw = torch.clamp(dw, max=self.scale_clamp) + dh = torch.clamp(dh, max=self.scale_clamp) + + pred_ctr_x = dx * widths[:, None] + ctr_x[:, None] + pred_ctr_y = dy * heights[:, None] + ctr_y[:, None] + pred_w = torch.exp(dw) * widths[:, None] + pred_h = torch.exp(dh) * heights[:, None] + + pred_boxes = torch.zeros_like(deltas) + pred_boxes[:, 0::4] = pred_ctr_x - 0.5 * pred_w # x1 + pred_boxes[:, 1::4] = pred_ctr_y - 0.5 * pred_h # y1 + pred_boxes[:, 2::4] = pred_ctr_x + 0.5 * pred_w # x2 + pred_boxes[:, 3::4] = pred_ctr_y + 0.5 * pred_h # y2 + return pred_boxes + + +class Matcher(object): + """ + This class assigns to each predicted "element" (e.g., a box) a ground-truth + element. Each predicted element will have exactly zero or one matches; each + ground-truth element may be matched to zero or more predicted elements. + The matching is determined by the MxN match_quality_matrix, that characterizes + how well each (ground-truth, prediction)-pair match each other. For example, + if the elements are boxes, this matrix may contain box intersection-over-union + overlap values. + The matcher returns (a) a vector of length N containing the index of the + ground-truth element m in [0, M) that matches to prediction n in [0, N). + (b) a vector of length N containing the labels for each prediction. + """ + + def __init__( + self, + thresholds: List[float], + labels: List[int], + allow_low_quality_matches: bool = False, + ): + """ + Args: + thresholds (list): a list of thresholds used to stratify predictions + into levels. + labels (list): a list of values to label predictions belonging at + each level. A label can be one of {-1, 0, 1} signifying + {ignore, negative class, positive class}, respectively. + allow_low_quality_matches (bool): if True, produce additional matches or predictions with maximum match quality lower than high_threshold. + For example, thresholds = [0.3, 0.5] labels = [0, -1, 1] All predictions with iou < 0.3 will be marked with 0 and + thus will be considered as false positives while training. All predictions with 0.3 <= iou < 0.5 will be marked with -1 and + thus will be ignored. All predictions with 0.5 <= iou will be marked with 1 and thus will be considered as true positives. + """ + thresholds = thresholds[:] + assert thresholds[0] > 0 + thresholds.insert(0, -float("inf")) + thresholds.append(float("inf")) + assert all(low <= high for (low, high) in zip(thresholds[:-1], thresholds[1:])) + assert all(label_i in [-1, 0, 1] for label_i in labels) + assert len(labels) == len(thresholds) - 1 + self.thresholds = thresholds + self.labels = labels + self.allow_low_quality_matches = allow_low_quality_matches + + def __call__(self, match_quality_matrix): + """ + Args: + match_quality_matrix (Tensor[float]): an MxN tensor, containing the pairwise quality between M ground-truth elements and N predicted + elements. All elements must be >= 0 (due to the us of `torch.nonzero` for selecting indices in :meth:`set_low_quality_matches_`). + Returns: + matches (Tensor[int64]): a vector of length N, where matches[i] is a matched ground-truth index in [0, M) + match_labels (Tensor[int8]): a vector of length N, where pred_labels[i] indicates true or false positive or ignored + """ + assert match_quality_matrix.dim() == 2 + if match_quality_matrix.numel() == 0: + default_matches = match_quality_matrix.new_full((match_quality_matrix.size(1),), 0, dtype=torch.int64) + # When no gt boxes exist, we define IOU = 0 and therefore set labels + # to `self.labels[0]`, which usually defaults to background class 0 + # To choose to ignore instead, + # can make labels=[-1,0,-1,1] + set appropriate thresholds + default_match_labels = match_quality_matrix.new_full( + (match_quality_matrix.size(1),), self.labels[0], dtype=torch.int8 + ) + return default_matches, default_match_labels + + assert torch.all(match_quality_matrix >= 0) + + # match_quality_matrix is M (gt) x N (predicted) + # Max over gt elements (dim 0) to find best gt candidate for each prediction + matched_vals, matches = match_quality_matrix.max(dim=0) + + match_labels = matches.new_full(matches.size(), 1, dtype=torch.int8) + + for l, low, high in zip(self.labels, self.thresholds[:-1], self.thresholds[1:]): + low_high = (matched_vals >= low) & (matched_vals < high) + match_labels[low_high] = l + + if self.allow_low_quality_matches: + self.set_low_quality_matches_(match_labels, match_quality_matrix) + + return matches, match_labels + + def set_low_quality_matches_(self, match_labels, match_quality_matrix): + """ + Produce additional matches for predictions that have only low-quality matches. + Specifically, for each ground-truth G find the set of predictions that have + maximum overlap with it (including ties); for each prediction in that set, if + it is unmatched, then match it to the ground-truth G. + This function implements the RPN assignment case (i) + in Sec. 3.1.2 of Faster R-CNN. + """ + # For each gt, find the prediction with which it has highest quality + highest_quality_foreach_gt, _ = match_quality_matrix.max(dim=1) + # Find the highest quality match available, even if it is low, including ties. + # Note that the matches qualities must be positive due to the use of + # `torch.nonzero`. + of_quality_inds = match_quality_matrix == highest_quality_foreach_gt[:, None] + if of_quality_inds.dim() == 0: + (_, pred_inds_with_highest_quality) = of_quality_inds.unsqueeze(0).nonzero().unbind(1) + else: + (_, pred_inds_with_highest_quality) = of_quality_inds.nonzero().unbind(1) + match_labels[pred_inds_with_highest_quality] = 1 + + +class RPNOutputs(object): + def __init__( + self, + box2box_transform, + anchor_matcher, + batch_size_per_image, + positive_fraction, + images, + pred_objectness_logits, + pred_anchor_deltas, + anchors, + boundary_threshold=0, + gt_boxes=None, + smooth_l1_beta=0.0, + ): + """ + Args: + box2box_transform (Box2BoxTransform): :class:`Box2BoxTransform` instance for anchor-proposal transformations. + anchor_matcher (Matcher): :class:`Matcher` instance for matching anchors to ground-truth boxes; used to determine training labels. + batch_size_per_image (int): number of proposals to sample when training + positive_fraction (float): target fraction of sampled proposals that should be positive + images (ImageList): :class:`ImageList` instance representing N input images + pred_objectness_logits (list[Tensor]): A list of L elements. Element i is a tensor of shape (N, A, Hi, W) + pred_anchor_deltas (list[Tensor]): A list of L elements. Element i is a tensor of shape (N, A*4, Hi, Wi) + anchors (list[torch.Tensor]): nested list of boxes. anchors[i][j] at (n, l) stores anchor array for feature map l + boundary_threshold (int): if >= 0, then anchors that extend beyond the image boundary by more than boundary_thresh are not used in training. + gt_boxes (list[Boxes], optional): A list of N elements. + smooth_l1_beta (float): The transition point between L1 and L2 lossn. When set to 0, the loss becomes L1. When +inf, it is ignored + """ + self.box2box_transform = box2box_transform + self.anchor_matcher = anchor_matcher + self.batch_size_per_image = batch_size_per_image + self.positive_fraction = positive_fraction + self.pred_objectness_logits = pred_objectness_logits + self.pred_anchor_deltas = pred_anchor_deltas + + self.anchors = anchors + self.gt_boxes = gt_boxes + self.num_feature_maps = len(pred_objectness_logits) + self.num_images = len(images) + self.boundary_threshold = boundary_threshold + self.smooth_l1_beta = smooth_l1_beta + + def _get_ground_truth(self): + raise NotImplementedError() + + def predict_proposals(self): + # pred_anchor_deltas: (L, N, ? Hi, Wi) + # anchors:(N, L, -1, B) + # here we loop over specific feature map, NOT images + proposals = [] + anchors = self.anchors.transpose(0, 1) + for anchors_i, pred_anchor_deltas_i in zip(anchors, self.pred_anchor_deltas): + B = anchors_i.size(-1) + N, _, Hi, Wi = pred_anchor_deltas_i.shape + anchors_i = anchors_i.flatten(start_dim=0, end_dim=1) + pred_anchor_deltas_i = pred_anchor_deltas_i.view(N, -1, B, Hi, Wi).permute(0, 3, 4, 1, 2).reshape(-1, B) + proposals_i = self.box2box_transform.apply_deltas(pred_anchor_deltas_i, anchors_i) + # Append feature map proposals with shape (N, Hi*Wi*A, B) + proposals.append(proposals_i.view(N, -1, B)) + proposals = torch.stack(proposals) + return proposals + + def predict_objectness_logits(self): + """ + Returns: + pred_objectness_logits (list[Tensor]) -> (N, Hi*Wi*A). + """ + pred_objectness_logits = [ + # Reshape: (N, A, Hi, Wi) -> (N, Hi, Wi, A) -> (N, Hi*Wi*A) + score.permute(0, 2, 3, 1).reshape(self.num_images, -1) + for score in self.pred_objectness_logits + ] + return pred_objectness_logits + + +# Main Classes +class Conv2d(nn.Conv2d): + def __init__(self, *args, **kwargs): + norm = kwargs.pop("norm", None) + activation = kwargs.pop("activation", None) + super().__init__(*args, **kwargs) + + self.norm = norm + self.activation = activation + + def forward(self, x): + if x.numel() == 0 and self.training: + assert not isinstance(self.norm, nn.SyncBatchNorm) + if x.numel() == 0: + assert not isinstance(self.norm, nn.GroupNorm) + output_shape = [ + (i + 2 * p - (di * (k - 1) + 1)) // s + 1 + for i, p, di, k, s in zip( + x.shape[-2:], + self.padding, + self.dilation, + self.kernel_size, + self.stride, + ) + ] + output_shape = [x.shape[0], self.weight.shape[0]] + output_shape + empty = _NewEmptyTensorOp.apply(x, output_shape) + if self.training: + _dummy = sum(x.view(-1)[0] for x in self.parameters()) * 0.0 + return empty + _dummy + else: + return empty + + x = super().forward(x) + if self.norm is not None: + x = self.norm(x) + if self.activation is not None: + x = self.activation(x) + return x + + +class LastLevelMaxPool(nn.Module): + """ + This module is used in the original FPN to generate a downsampled P6 feature from P5. + """ + + def __init__(self): + super().__init__() + self.num_levels = 1 + self.in_feature = "p5" + + def forward(self, x): + return [nn.functional.max_pool2d(x, kernel_size=1, stride=2, padding=0)] + + +class LastLevelP6P7(nn.Module): + """ + This module is used in RetinaNet to generate extra layers, P6 and P7 from C5 feature. + """ + + def __init__(self, in_channels, out_channels): + super().__init__() + self.num_levels = 2 + self.in_feature = "res5" + self.p6 = nn.Conv2d(in_channels, out_channels, 3, 2, 1) + self.p7 = nn.Conv2d(out_channels, out_channels, 3, 2, 1) + + def forward(self, c5): + p6 = self.p6(c5) + p7 = self.p7(nn.functional.relu(p6)) + return [p6, p7] + + +class BasicStem(nn.Module): + def __init__(self, in_channels=3, out_channels=64, norm="BN", caffe_maxpool=False): + super().__init__() + self.conv1 = Conv2d( + in_channels, + out_channels, + kernel_size=7, + stride=2, + padding=3, + bias=False, + norm=get_norm(norm, out_channels), + ) + self.caffe_maxpool = caffe_maxpool + # use pad 1 instead of pad zero + + def forward(self, x): + x = self.conv1(x) + x = nn.functional.relu_(x) + if self.caffe_maxpool: + x = nn.functional.max_pool2d(x, kernel_size=3, stride=2, padding=0, ceil_mode=True) + else: + x = nn.functional.max_pool2d(x, kernel_size=3, stride=2, padding=1) + return x + + @property + def out_channels(self): + return self.conv1.out_channels + + @property + def stride(self): + return 4 # = stride 2 conv -> stride 2 max pool + + +class ResNetBlockBase(nn.Module): + def __init__(self, in_channels, out_channels, stride): + super().__init__() + self.in_channels = in_channels + self.out_channels = out_channels + self.stride = stride + + def freeze(self): + for p in self.parameters(): + p.requires_grad = False + return self + + +class BottleneckBlock(ResNetBlockBase): + def __init__( + self, + in_channels, + out_channels, + bottleneck_channels, + stride=1, + num_groups=1, + norm="BN", + stride_in_1x1=False, + dilation=1, + ): + super().__init__(in_channels, out_channels, stride) + + if in_channels != out_channels: + self.shortcut = Conv2d( + in_channels, + out_channels, + kernel_size=1, + stride=stride, + bias=False, + norm=get_norm(norm, out_channels), + ) + else: + self.shortcut = None + + # The original MSRA ResNet models have stride in the first 1x1 conv + # The subsequent fb.torch.resnet and Caffe2 ResNe[X]t implementations have + # stride in the 3x3 conv + stride_1x1, stride_3x3 = (stride, 1) if stride_in_1x1 else (1, stride) + + self.conv1 = Conv2d( + in_channels, + bottleneck_channels, + kernel_size=1, + stride=stride_1x1, + bias=False, + norm=get_norm(norm, bottleneck_channels), + ) + + self.conv2 = Conv2d( + bottleneck_channels, + bottleneck_channels, + kernel_size=3, + stride=stride_3x3, + padding=1 * dilation, + bias=False, + groups=num_groups, + dilation=dilation, + norm=get_norm(norm, bottleneck_channels), + ) + + self.conv3 = Conv2d( + bottleneck_channels, + out_channels, + kernel_size=1, + bias=False, + norm=get_norm(norm, out_channels), + ) + + def forward(self, x): + out = self.conv1(x) + out = nn.functional.relu_(out) + + out = self.conv2(out) + out = nn.functional.relu_(out) + + out = self.conv3(out) + + if self.shortcut is not None: + shortcut = self.shortcut(x) + else: + shortcut = x + + out += shortcut + out = nn.functional.relu_(out) + return out + + +class Backbone(nn.Module, metaclass=ABCMeta): + def __init__(self): + super().__init__() + + @abstractmethod + def forward(self): + pass + + @property + def size_divisibility(self): + """ + Some backbones require the input height and width to be divisible by a specific integer. This is + typically true for encoder / decoder type networks with lateral connection (e.g., FPN) for which feature maps need to match + dimension in the "bottom up" and "top down" paths. Set to 0 if no specific input size divisibility is required. + """ + return 0 + + def output_shape(self): + return { + name: ShapeSpec( + channels=self._out_feature_channels[name], + stride=self._out_feature_strides[name], + ) + for name in self._out_features + } + + @property + def out_features(self): + """deprecated""" + return self._out_features + + @property + def out_feature_strides(self): + """deprecated""" + return {f: self._out_feature_strides[f] for f in self._out_features} + + @property + def out_feature_channels(self): + """deprecated""" + return {f: self._out_feature_channels[f] for f in self._out_features} + + +class ResNet(Backbone): + def __init__(self, stem, stages, num_classes=None, out_features=None): + """ + Args: + stem (nn.Module): a stem module + stages (list[list[ResNetBlock]]): several (typically 4) stages, each contains multiple :class:`ResNetBlockBase`. + num_classes (None or int): if None, will not perform classification. + out_features (list[str]): name of the layers whose outputs should be returned in forward. Can be anything in: + "stem", "linear", or "res2" ... If None, will return the output of the last layer. + """ + super(ResNet, self).__init__() + self.stem = stem + self.num_classes = num_classes + + current_stride = self.stem.stride + self._out_feature_strides = {"stem": current_stride} + self._out_feature_channels = {"stem": self.stem.out_channels} + + self.stages_and_names = [] + for i, blocks in enumerate(stages): + for block in blocks: + assert isinstance(block, ResNetBlockBase), block + curr_channels = block.out_channels + stage = nn.Sequential(*blocks) + name = "res" + str(i + 2) + self.add_module(name, stage) + self.stages_and_names.append((stage, name)) + self._out_feature_strides[name] = current_stride = int( + current_stride * np.prod([k.stride for k in blocks]) + ) + self._out_feature_channels[name] = blocks[-1].out_channels + + if num_classes is not None: + self.avgpool = nn.AdaptiveAvgPool2d((1, 1)) + self.linear = nn.Linear(curr_channels, num_classes) + + # Sec 5.1 in "Accurate, Large Minibatch SGD: Training ImageNet in 1 Hour": + # "The 1000-way fully-connected layer is initialized by + # drawing weights from a zero-mean Gaussian with std of 0.01." + nn.init.normal_(self.linear.weight, stddev=0.01) + name = "linear" + + if out_features is None: + out_features = [name] + self._out_features = out_features + assert len(self._out_features) + children = [x[0] for x in self.named_children()] + for out_feature in self._out_features: + assert out_feature in children, "Available children: {}".format(", ".join(children)) + + def forward(self, x): + outputs = {} + x = self.stem(x) + if "stem" in self._out_features: + outputs["stem"] = x + for stage, name in self.stages_and_names: + x = stage(x) + if name in self._out_features: + outputs[name] = x + if self.num_classes is not None: + x = self.avgpool(x) + x = self.linear(x) + if "linear" in self._out_features: + outputs["linear"] = x + return outputs + + def output_shape(self): + return { + name: ShapeSpec( + channels=self._out_feature_channels[name], + stride=self._out_feature_strides[name], + ) + for name in self._out_features + } + + @staticmethod + def make_stage( + block_class, + num_blocks, + first_stride=None, + *, + in_channels, + out_channels, + **kwargs, + ): + """ + Usually, layers that produce the same feature map spatial size + are defined as one "stage". + Under such definition, stride_per_block[1:] should all be 1. + """ + if first_stride is not None: + assert "stride" not in kwargs and "stride_per_block" not in kwargs + kwargs["stride_per_block"] = [first_stride] + [1] * (num_blocks - 1) + blocks = [] + for i in range(num_blocks): + curr_kwargs = {} + for k, v in kwargs.items(): + if k.endswith("_per_block"): + assert ( + len(v) == num_blocks + ), f"Argument '{k}' of make_stage should have the same length as num_blocks={num_blocks}." + newk = k[: -len("_per_block")] + assert newk not in kwargs, f"Cannot call make_stage with both {k} and {newk}!" + curr_kwargs[newk] = v[i] + else: + curr_kwargs[k] = v + + blocks.append(block_class(in_channels=in_channels, out_channels=out_channels, **curr_kwargs)) + in_channels = out_channels + + return blocks + + +class ROIPooler(nn.Module): + """ + Region of interest feature map pooler that supports pooling from one or more + feature maps. + """ + + def __init__( + self, + output_size, + scales, + sampling_ratio, + canonical_box_size=224, + canonical_level=4, + ): + super().__init__() + # assumption that stride is a power of 2. + min_level = -math.log2(scales[0]) + max_level = -math.log2(scales[-1]) + + # a bunch of testing + assert math.isclose(min_level, int(min_level)) and math.isclose(max_level, int(max_level)) + assert len(scales) == max_level - min_level + 1, "not pyramid" + assert 0 < min_level and min_level <= max_level + if isinstance(output_size, int): + output_size = (output_size, output_size) + assert len(output_size) == 2 and isinstance(output_size[0], int) and isinstance(output_size[1], int) + if len(scales) > 1: + assert min_level <= canonical_level and canonical_level <= max_level + assert canonical_box_size > 0 + + self.output_size = output_size + self.min_level = int(min_level) + self.max_level = int(max_level) + self.level_poolers = nn.ModuleList(RoIPool(output_size, spatial_scale=scale) for scale in scales) + self.canonical_level = canonical_level + self.canonical_box_size = canonical_box_size + + def forward(self, feature_maps, boxes): + """ + Args: + feature_maps: List[torch.Tensor(N,C,W,H)] + box_lists: list[torch.Tensor]) + Returns: + A tensor of shape(N*B, Channels, output_size, output_size) + """ + x = list(feature_maps.values()) + num_level_assignments = len(self.level_poolers) + assert len(x) == num_level_assignments and len(boxes) == x[0].size(0) + + pooler_fmt_boxes = convert_boxes_to_pooler_format(boxes) + + if num_level_assignments == 1: + return self.level_poolers[0](x[0], pooler_fmt_boxes) + + level_assignments = assign_boxes_to_levels( + boxes, + self.min_level, + self.max_level, + self.canonical_box_size, + self.canonical_level, + ) + + num_boxes = len(pooler_fmt_boxes) + num_channels = x[0].shape[1] + output_size = self.output_size[0] + + dtype, device = x[0].dtype, x[0].device + output = torch.zeros( + (num_boxes, num_channels, output_size, output_size), + dtype=dtype, + device=device, + ) + + for level, (x_level, pooler) in enumerate(zip(x, self.level_poolers)): + inds = torch.nonzero(level_assignments == level).squeeze(1) + pooler_fmt_boxes_level = pooler_fmt_boxes[inds] + output[inds] = pooler(x_level, pooler_fmt_boxes_level) + + return output + + +class ROIOutputs(object): + def __init__(self, cfg, training=False): + self.smooth_l1_beta = cfg.ROI_BOX_HEAD.SMOOTH_L1_BETA + self.box2box_transform = Box2BoxTransform(weights=cfg.ROI_BOX_HEAD.BBOX_REG_WEIGHTS) + self.training = training + self.score_thresh = cfg.ROI_HEADS.SCORE_THRESH_TEST + self.min_detections = cfg.MIN_DETECTIONS + self.max_detections = cfg.MAX_DETECTIONS + + nms_thresh = cfg.ROI_HEADS.NMS_THRESH_TEST + if not isinstance(nms_thresh, list): + nms_thresh = [nms_thresh] + self.nms_thresh = nms_thresh + + def _predict_boxes(self, proposals, box_deltas, preds_per_image): + num_pred = box_deltas.size(0) + B = proposals[0].size(-1) + K = box_deltas.size(-1) // B + box_deltas = box_deltas.view(num_pred * K, B) + proposals = torch.cat(proposals, dim=0).unsqueeze(-2).expand(num_pred, K, B) + proposals = proposals.reshape(-1, B) + boxes = self.box2box_transform.apply_deltas(box_deltas, proposals) + return boxes.view(num_pred, K * B).split(preds_per_image, dim=0) + + def _predict_objs(self, obj_logits, preds_per_image): + probs = nn.functional.softmax(obj_logits, dim=-1) + probs = probs.split(preds_per_image, dim=0) + return probs + + def _predict_attrs(self, attr_logits, preds_per_image): + attr_logits = attr_logits[..., :-1].softmax(-1) + attr_probs, attrs = attr_logits.max(-1) + return attr_probs.split(preds_per_image, dim=0), attrs.split(preds_per_image, dim=0) + + @torch.no_grad() + def inference( + self, + obj_logits, + attr_logits, + box_deltas, + pred_boxes, + features, + sizes, + scales=None, + ): + # only the pred boxes is the + preds_per_image = [p.size(0) for p in pred_boxes] + boxes_all = self._predict_boxes(pred_boxes, box_deltas, preds_per_image) + obj_scores_all = self._predict_objs(obj_logits, preds_per_image) # list of length N + attr_probs_all, attrs_all = self._predict_attrs(attr_logits, preds_per_image) + features = features.split(preds_per_image, dim=0) + + # fun for each image too, also I can experiment and do multiple images + final_results = [] + zipped = zip(boxes_all, obj_scores_all, attr_probs_all, attrs_all, sizes) + for i, (boxes, obj_scores, attr_probs, attrs, size) in enumerate(zipped): + for nms_t in self.nms_thresh: + outputs = do_nms( + boxes, + obj_scores, + size, + self.score_thresh, + nms_t, + self.min_detections, + self.max_detections, + ) + if outputs is not None: + max_boxes, max_scores, classes, ids = outputs + break + + if scales is not None: + scale_yx = scales[i] + max_boxes[:, 0::2] *= scale_yx[1] + max_boxes[:, 1::2] *= scale_yx[0] + + final_results.append( + ( + max_boxes, + classes, + max_scores, + attrs[ids], + attr_probs[ids], + features[i][ids], + ) + ) + boxes, classes, class_probs, attrs, attr_probs, roi_features = map(list, zip(*final_results)) + return boxes, classes, class_probs, attrs, attr_probs, roi_features + + def training(self, obj_logits, attr_logits, box_deltas, pred_boxes, features, sizes): + pass + + def __call__( + self, + obj_logits, + attr_logits, + box_deltas, + pred_boxes, + features, + sizes, + scales=None, + ): + if self.training: + raise NotImplementedError() + return self.inference( + obj_logits, + attr_logits, + box_deltas, + pred_boxes, + features, + sizes, + scales=scales, + ) + + +class Res5ROIHeads(nn.Module): + """ + ROIHeads perform all per-region computation in an R-CNN. + It contains logic of cropping the regions, extract per-region features + (by the res-5 block in this case), and make per-region predictions. + """ + + def __init__(self, cfg, input_shape): + super().__init__() + self.batch_size_per_image = cfg.RPN.BATCH_SIZE_PER_IMAGE + self.positive_sample_fraction = cfg.ROI_HEADS.POSITIVE_FRACTION + self.in_features = cfg.ROI_HEADS.IN_FEATURES + self.num_classes = cfg.ROI_HEADS.NUM_CLASSES + self.proposal_append_gt = cfg.ROI_HEADS.PROPOSAL_APPEND_GT + self.feature_strides = {k: v.stride for k, v in input_shape.items()} + self.feature_channels = {k: v.channels for k, v in input_shape.items()} + self.cls_agnostic_bbox_reg = cfg.ROI_BOX_HEAD.CLS_AGNOSTIC_BBOX_REG + self.stage_channel_factor = 2**3 # res5 is 8x res2 + self.out_channels = cfg.RESNETS.RES2_OUT_CHANNELS * self.stage_channel_factor + + # self.proposal_matcher = Matcher( + # cfg.ROI_HEADS.IOU_THRESHOLDS, + # cfg.ROI_HEADS.IOU_LABELS, + # allow_low_quality_matches=False, + # ) + + pooler_resolution = cfg.ROI_BOX_HEAD.POOLER_RESOLUTION + pooler_scales = (1.0 / self.feature_strides[self.in_features[0]],) + sampling_ratio = cfg.ROI_BOX_HEAD.POOLER_SAMPLING_RATIO + res5_halve = cfg.ROI_BOX_HEAD.RES5HALVE + use_attr = cfg.ROI_BOX_HEAD.ATTR + num_attrs = cfg.ROI_BOX_HEAD.NUM_ATTRS + + self.pooler = ROIPooler( + output_size=pooler_resolution, + scales=pooler_scales, + sampling_ratio=sampling_ratio, + ) + + self.res5 = self._build_res5_block(cfg) + if not res5_halve: + """ + Modifications for VG in RoI heads: + 1. Change the stride of conv1 and shortcut in Res5.Block1 from 2 to 1 + 2. Modifying all conv2 with (padding: 1 --> 2) and (dilation: 1 --> 2) + """ + self.res5[0].conv1.stride = (1, 1) + self.res5[0].shortcut.stride = (1, 1) + for i in range(3): + self.res5[i].conv2.padding = (2, 2) + self.res5[i].conv2.dilation = (2, 2) + + self.box_predictor = FastRCNNOutputLayers( + self.out_channels, + self.num_classes, + self.cls_agnostic_bbox_reg, + use_attr=use_attr, + num_attrs=num_attrs, + ) + + def _build_res5_block(self, cfg): + stage_channel_factor = self.stage_channel_factor # res5 is 8x res2 + num_groups = cfg.RESNETS.NUM_GROUPS + width_per_group = cfg.RESNETS.WIDTH_PER_GROUP + bottleneck_channels = num_groups * width_per_group * stage_channel_factor + out_channels = self.out_channels + stride_in_1x1 = cfg.RESNETS.STRIDE_IN_1X1 + norm = cfg.RESNETS.NORM + + blocks = ResNet.make_stage( + BottleneckBlock, + 3, + first_stride=2, + in_channels=out_channels // 2, + bottleneck_channels=bottleneck_channels, + out_channels=out_channels, + num_groups=num_groups, + norm=norm, + stride_in_1x1=stride_in_1x1, + ) + return nn.Sequential(*blocks) + + def _shared_roi_transform(self, features, boxes): + x = self.pooler(features, boxes) + return self.res5(x) + + def forward(self, features, proposal_boxes, gt_boxes=None): + if self.training: + """ + see https://github.com/airsplay/py-bottom-up-attention/\ + blob/master/detectron2/modeling/roi_heads/roi_heads.py + """ + raise NotImplementedError() + + assert not proposal_boxes[0].requires_grad + box_features = self._shared_roi_transform(features, proposal_boxes) + feature_pooled = box_features.mean(dim=[2, 3]) # pooled to 1x1 + obj_logits, attr_logits, pred_proposal_deltas = self.box_predictor(feature_pooled) + return obj_logits, attr_logits, pred_proposal_deltas, feature_pooled + + +class AnchorGenerator(nn.Module): + """ + For a set of image sizes and feature maps, computes a set of anchors. + """ + + def __init__(self, cfg, input_shape: List[ShapeSpec]): + super().__init__() + sizes = cfg.ANCHOR_GENERATOR.SIZES + aspect_ratios = cfg.ANCHOR_GENERATOR.ASPECT_RATIOS + self.strides = [x.stride for x in input_shape] + self.offset = cfg.ANCHOR_GENERATOR.OFFSET + assert 0.0 <= self.offset < 1.0, self.offset + + """ + sizes (list[list[int]]): sizes[i] is the list of anchor sizes for feat map i + 1. given in absolute lengths in units of the input image; + 2. they do not dynamically scale if the input image size changes. + aspect_ratios (list[list[float]]) + strides (list[int]): stride of each input feature. + """ + + self.num_features = len(self.strides) + self.cell_anchors = nn.ParameterList(self._calculate_anchors(sizes, aspect_ratios)) + self._spacial_feat_dim = 4 + + def _calculate_anchors(self, sizes, aspect_ratios): + # If one size (or aspect ratio) is specified and there are multiple feature + # maps, then we "broadcast" anchors of that single size (or aspect ratio) + if len(sizes) == 1: + sizes *= self.num_features + if len(aspect_ratios) == 1: + aspect_ratios *= self.num_features + assert self.num_features == len(sizes) + assert self.num_features == len(aspect_ratios) + + cell_anchors = [self.generate_cell_anchors(s, a).float() for s, a in zip(sizes, aspect_ratios)] + + return cell_anchors + + @property + def box_dim(self): + return self._spacial_feat_dim + + @property + def num_cell_anchors(self): + """ + Returns: + list[int]: Each int is the number of anchors at every pixel location, on that feature map. + """ + return [len(cell_anchors) for cell_anchors in self.cell_anchors] + + def grid_anchors(self, grid_sizes): + anchors = [] + for size, stride, base_anchors in zip(grid_sizes, self.strides, self.cell_anchors): + shift_x, shift_y = _create_grid_offsets(size, stride, self.offset, base_anchors.device) + shifts = torch.stack((shift_x, shift_y, shift_x, shift_y), dim=1) + + anchors.append((shifts.view(-1, 1, 4) + base_anchors.view(1, -1, 4)).reshape(-1, 4)) + + return anchors + + def generate_cell_anchors(self, sizes=(32, 64, 128, 256, 512), aspect_ratios=(0.5, 1, 2)): + """ + anchors are continuous geometric rectangles + centered on one feature map point sample. + We can later build the set of anchors + for the entire feature map by tiling these tensors + """ + + anchors = [] + for size in sizes: + area = size**2.0 + for aspect_ratio in aspect_ratios: + w = math.sqrt(area / aspect_ratio) + h = aspect_ratio * w + x0, y0, x1, y1 = -w / 2.0, -h / 2.0, w / 2.0, h / 2.0 + anchors.append([x0, y0, x1, y1]) + return nn.Parameter(torch.tensor(anchors)) + + def forward(self, features): + """ + Args: + features List[torch.Tensor]: list of feature maps on which to generate anchors. + Returns: + torch.Tensor: a list of #image elements. + """ + num_images = features[0].size(0) + grid_sizes = [feature_map.shape[-2:] for feature_map in features] + anchors_over_all_feature_maps = self.grid_anchors(grid_sizes) + anchors_over_all_feature_maps = torch.stack(anchors_over_all_feature_maps) + return anchors_over_all_feature_maps.unsqueeze(0).repeat_interleave(num_images, dim=0) + + +class RPNHead(nn.Module): + """ + RPN classification and regression heads. Uses a 3x3 conv to produce a shared + hidden state from which one 1x1 conv predicts objectness logits for each anchor + and a second 1x1 conv predicts bounding-box deltas specifying how to deform + each anchor into an object proposal. + """ + + def __init__(self, cfg, input_shape: List[ShapeSpec]): + super().__init__() + + # Standard RPN is shared across levels: + in_channels = [s.channels for s in input_shape] + assert len(set(in_channels)) == 1, "Each level must have the same channel!" + in_channels = in_channels[0] + + anchor_generator = AnchorGenerator(cfg, input_shape) + num_cell_anchors = anchor_generator.num_cell_anchors + box_dim = anchor_generator.box_dim + assert len(set(num_cell_anchors)) == 1, "Each level must have the same number of cell anchors" + num_cell_anchors = num_cell_anchors[0] + + if cfg.PROPOSAL_GENERATOR.HIDDEN_CHANNELS == -1: + hid_channels = in_channels + else: + hid_channels = cfg.PROPOSAL_GENERATOR.HIDDEN_CHANNELS + # Modifications for VG in RPN (modeling/proposal_generator/rpn.py) + # Use hidden dim instead fo the same dim as Res4 (in_channels) + + # 3x3 conv for the hidden representation + self.conv = nn.Conv2d(in_channels, hid_channels, kernel_size=3, stride=1, padding=1) + # 1x1 conv for predicting objectness logits + self.objectness_logits = nn.Conv2d(hid_channels, num_cell_anchors, kernel_size=1, stride=1) + # 1x1 conv for predicting box2box transform deltas + self.anchor_deltas = nn.Conv2d(hid_channels, num_cell_anchors * box_dim, kernel_size=1, stride=1) + + for layer in [self.conv, self.objectness_logits, self.anchor_deltas]: + nn.init.normal_(layer.weight, std=0.01) + nn.init.constant_(layer.bias, 0) + + def forward(self, features): + """ + Args: + features (list[Tensor]): list of feature maps + """ + pred_objectness_logits = [] + pred_anchor_deltas = [] + for x in features: + t = nn.functional.relu(self.conv(x)) + pred_objectness_logits.append(self.objectness_logits(t)) + pred_anchor_deltas.append(self.anchor_deltas(t)) + return pred_objectness_logits, pred_anchor_deltas + + +class RPN(nn.Module): + """ + Region Proposal Network, introduced by the Faster R-CNN paper. + """ + + def __init__(self, cfg, input_shape: Dict[str, ShapeSpec]): + super().__init__() + + self.min_box_side_len = cfg.PROPOSAL_GENERATOR.MIN_SIZE + self.in_features = cfg.RPN.IN_FEATURES + self.nms_thresh = cfg.RPN.NMS_THRESH + self.batch_size_per_image = cfg.RPN.BATCH_SIZE_PER_IMAGE + self.positive_fraction = cfg.RPN.POSITIVE_FRACTION + self.smooth_l1_beta = cfg.RPN.SMOOTH_L1_BETA + self.loss_weight = cfg.RPN.LOSS_WEIGHT + + self.pre_nms_topk = { + True: cfg.RPN.PRE_NMS_TOPK_TRAIN, + False: cfg.RPN.PRE_NMS_TOPK_TEST, + } + self.post_nms_topk = { + True: cfg.RPN.POST_NMS_TOPK_TRAIN, + False: cfg.RPN.POST_NMS_TOPK_TEST, + } + self.boundary_threshold = cfg.RPN.BOUNDARY_THRESH + + self.anchor_generator = AnchorGenerator(cfg, [input_shape[f] for f in self.in_features]) + self.box2box_transform = Box2BoxTransform(weights=cfg.RPN.BBOX_REG_WEIGHTS) + self.anchor_matcher = Matcher( + cfg.RPN.IOU_THRESHOLDS, + cfg.RPN.IOU_LABELS, + allow_low_quality_matches=True, + ) + self.rpn_head = RPNHead(cfg, [input_shape[f] for f in self.in_features]) + + def training(self, images, image_shapes, features, gt_boxes): + pass + + def inference(self, outputs, images, image_shapes, features, gt_boxes=None): + outputs = find_top_rpn_proposals( + outputs.predict_proposals(), + outputs.predict_objectness_logits(), + images, + image_shapes, + self.nms_thresh, + self.pre_nms_topk[self.training], + self.post_nms_topk[self.training], + self.min_box_side_len, + self.training, + ) + + results = [] + for img in outputs: + im_boxes, img_box_logits = img + img_box_logits, inds = img_box_logits.sort(descending=True) + im_boxes = im_boxes[inds] + results.append((im_boxes, img_box_logits)) + + (proposal_boxes, logits) = tuple(map(list, zip(*results))) + return proposal_boxes, logits + + def forward(self, images, image_shapes, features, gt_boxes=None): + """ + Args: + images (torch.Tensor): input images of length `N` + features (dict[str: Tensor]) + gt_instances + """ + # features is dict, key = block level, v = feature_map + features = [features[f] for f in self.in_features] + pred_objectness_logits, pred_anchor_deltas = self.rpn_head(features) + anchors = self.anchor_generator(features) + outputs = RPNOutputs( + self.box2box_transform, + self.anchor_matcher, + self.batch_size_per_image, + self.positive_fraction, + images, + pred_objectness_logits, + pred_anchor_deltas, + anchors, + self.boundary_threshold, + gt_boxes, + self.smooth_l1_beta, + ) + # For RPN-only models, the proposals are the final output + + if self.training: + raise NotImplementedError() + return self.training(outputs, images, image_shapes, features, gt_boxes) + else: + return self.inference(outputs, images, image_shapes, features, gt_boxes) + + +class FastRCNNOutputLayers(nn.Module): + """ + Two linear layers for predicting Fast R-CNN outputs: + (1) proposal-to-detection box regression deltas + (2) classification scores + """ + + def __init__( + self, + input_size, + num_classes, + cls_agnostic_bbox_reg, + box_dim=4, + use_attr=False, + num_attrs=-1, + ): + """ + Args: + input_size (int): channels, or (channels, height, width) + num_classes (int) + cls_agnostic_bbox_reg (bool) + box_dim (int) + """ + super().__init__() + + if not isinstance(input_size, int): + input_size = np.prod(input_size) + + # (do + 1 for background class) + self.cls_score = nn.Linear(input_size, num_classes + 1) + num_bbox_reg_classes = 1 if cls_agnostic_bbox_reg else num_classes + self.bbox_pred = nn.Linear(input_size, num_bbox_reg_classes * box_dim) + + self.use_attr = use_attr + if use_attr: + """ + Modifications for VG in RoI heads + Embedding: {num_classes + 1} --> {input_size // 8} + Linear: {input_size + input_size // 8} --> {input_size // 4} + Linear: {input_size // 4} --> {num_attrs + 1} + """ + self.cls_embedding = nn.Embedding(num_classes + 1, input_size // 8) + self.fc_attr = nn.Linear(input_size + input_size // 8, input_size // 4) + self.attr_score = nn.Linear(input_size // 4, num_attrs + 1) + + nn.init.normal_(self.cls_score.weight, std=0.01) + nn.init.normal_(self.bbox_pred.weight, std=0.001) + for item in [self.cls_score, self.bbox_pred]: + nn.init.constant_(item.bias, 0) + + def forward(self, roi_features): + if roi_features.dim() > 2: + roi_features = torch.flatten(roi_features, start_dim=1) + scores = self.cls_score(roi_features) + proposal_deltas = self.bbox_pred(roi_features) + if self.use_attr: + _, max_class = scores.max(-1) # [b, c] --> [b] + cls_emb = self.cls_embedding(max_class) # [b] --> [b, 256] + roi_features = torch.cat([roi_features, cls_emb], -1) # [b, 2048] + [b, 256] --> [b, 2304] + roi_features = self.fc_attr(roi_features) + roi_features = nn.functional.relu(roi_features) + attr_scores = self.attr_score(roi_features) + return scores, attr_scores, proposal_deltas + else: + return scores, proposal_deltas + + +class GeneralizedRCNN(nn.Module): + def __init__(self, cfg): + super().__init__() + + self.device = torch.device(cfg.MODEL.DEVICE) + self.backbone = build_backbone(cfg) + self.proposal_generator = RPN(cfg, self.backbone.output_shape()) + self.roi_heads = Res5ROIHeads(cfg, self.backbone.output_shape()) + self.roi_outputs = ROIOutputs(cfg) + self.to(self.device) + + @classmethod + def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): + config = kwargs.pop("config", None) + state_dict = kwargs.pop("state_dict", None) + cache_dir = kwargs.pop("cache_dir", None) + from_tf = kwargs.pop("from_tf", False) + force_download = kwargs.pop("force_download", False) + resume_download = kwargs.pop("resume_download", False) + proxies = kwargs.pop("proxies", None) + local_files_only = kwargs.pop("local_files_only", False) + use_cdn = kwargs.pop("use_cdn", True) + + # Load config if we don't provide a configuration + if not isinstance(config, Config): + config_path = config if config is not None else pretrained_model_name_or_path + # try: + config = Config.from_pretrained( + config_path, + cache_dir=cache_dir, + force_download=force_download, + resume_download=resume_download, + proxies=proxies, + local_files_only=local_files_only, + ) + + # Load model + if pretrained_model_name_or_path is not None: + if os.path.isdir(pretrained_model_name_or_path): + if os.path.isfile(os.path.join(pretrained_model_name_or_path, WEIGHTS_NAME)): + # Load from a PyTorch checkpoint + archive_file = os.path.join(pretrained_model_name_or_path, WEIGHTS_NAME) + else: + raise EnvironmentError( + "Error no file named {} found in directory {} ".format( + WEIGHTS_NAME, + pretrained_model_name_or_path, + ) + ) + elif os.path.isfile(pretrained_model_name_or_path) or is_remote_url(pretrained_model_name_or_path): + archive_file = pretrained_model_name_or_path + elif os.path.isfile(pretrained_model_name_or_path + ".index"): + assert ( + from_tf + ), "We found a TensorFlow checkpoint at {}, please set from_tf to True to load from this checkpoint".format( + pretrained_model_name_or_path + ".index" + ) + archive_file = pretrained_model_name_or_path + ".index" + else: + archive_file = hf_bucket_url( + pretrained_model_name_or_path, + filename=WEIGHTS_NAME, + use_cdn=use_cdn, + ) + + try: + # Load from URL or cache if already cached + resolved_archive_file = cached_path( + archive_file, + cache_dir=cache_dir, + force_download=force_download, + proxies=proxies, + resume_download=resume_download, + local_files_only=local_files_only, + ) + if resolved_archive_file is None: + raise EnvironmentError + except EnvironmentError: + msg = f"Can't load weights for '{pretrained_model_name_or_path}'." + raise EnvironmentError(msg) + + if resolved_archive_file == archive_file: + print("loading weights file {}".format(archive_file)) + else: + print("loading weights file {} from cache at {}".format(archive_file, resolved_archive_file)) + else: + resolved_archive_file = None + + # Instantiate model. + model = cls(config) + + if state_dict is None: + try: + try: + state_dict = torch.load(resolved_archive_file, map_location="cpu") + except Exception: + state_dict = load_checkpoint(resolved_archive_file) + + except Exception: + raise OSError( + "Unable to load weights from pytorch checkpoint file. " + "If you tried to load a PyTorch model from a TF 2.0 checkpoint, please set from_tf=True. " + ) + + missing_keys = [] + unexpected_keys = [] + error_msgs = [] + + # Convert old format to new format if needed from a PyTorch state_dict + old_keys = [] + new_keys = [] + for key in state_dict.keys(): + new_key = None + if "gamma" in key: + new_key = key.replace("gamma", "weight") + if "beta" in key: + new_key = key.replace("beta", "bias") + if new_key: + old_keys.append(key) + new_keys.append(new_key) + for old_key, new_key in zip(old_keys, new_keys): + state_dict[new_key] = state_dict.pop(old_key) + + # copy state_dict so _load_from_state_dict can modify it + metadata = getattr(state_dict, "_metadata", None) + state_dict = state_dict.copy() + if metadata is not None: + state_dict._metadata = metadata + + model_to_load = model + model_to_load.load_state_dict(state_dict) + + if model.__class__.__name__ != model_to_load.__class__.__name__: + base_model_state_dict = model_to_load.state_dict().keys() + head_model_state_dict_without_base_prefix = [ + key.split(cls.base_model_prefix + ".")[-1] for key in model.state_dict().keys() + ] + missing_keys.extend(head_model_state_dict_without_base_prefix - base_model_state_dict) + + if len(unexpected_keys) > 0: + print( + f"Some weights of the model checkpoint at {pretrained_model_name_or_path} were not used when" + f" initializing {model.__class__.__name__}: {unexpected_keys}\n- This IS expected if you are" + f" initializing {model.__class__.__name__} from the checkpoint of a model trained on another task or" + " with another architecture (e.g. initializing a BertForSequenceClassification model from a" + " BertForPreTraining model).\n- This IS NOT expected if you are initializing" + f" {model.__class__.__name__} from the checkpoint of a model that you expect to be exactly identical" + " (initializing a BertForSequenceClassification model from a BertForSequenceClassification model)." + ) + else: + print(f"All model checkpoint weights were used when initializing {model.__class__.__name__}.\n") + if len(missing_keys) > 0: + print( + f"Some weights of {model.__class__.__name__} were not initialized from the model checkpoint at" + f" {pretrained_model_name_or_path} and are newly initialized: {missing_keys}\nYou should probably" + " TRAIN this model on a down-stream task to be able to use it for predictions and inference." + ) + else: + print( + f"All the weights of {model.__class__.__name__} were initialized from the model checkpoint at" + f" {pretrained_model_name_or_path}.\nIf your task is similar to the task the model of the checkpoint" + f" was trained on, you can already use {model.__class__.__name__} for predictions without further" + " training." + ) + if len(error_msgs) > 0: + raise RuntimeError( + "Error(s) in loading state_dict for {}:\n\t{}".format( + model.__class__.__name__, "\n\t".join(error_msgs) + ) + ) + # Set model in evaluation mode to deactivate DropOut modules by default + model.eval() + + return model + + def forward( + self, + images, + image_shapes, + gt_boxes=None, + proposals=None, + scales_yx=None, + **kwargs, + ): + """ + kwargs: + max_detections (int), return_tensors {"np", "pt", None}, padding {None, + "max_detections"}, pad_value (int), location = {"cuda", "cpu"} + """ + if self.training: + raise NotImplementedError() + return self.inference( + images=images, + image_shapes=image_shapes, + gt_boxes=gt_boxes, + proposals=proposals, + scales_yx=scales_yx, + **kwargs, + ) + + @torch.no_grad() + def inference( + self, + images, + image_shapes, + gt_boxes=None, + proposals=None, + scales_yx=None, + **kwargs, + ): + # run images through backbone + original_sizes = image_shapes * scales_yx + features = self.backbone(images) + + # generate proposals if none are available + if proposals is None: + proposal_boxes, _ = self.proposal_generator(images, image_shapes, features, gt_boxes) + else: + assert proposals is not None + + # pool object features from either gt_boxes, or from proposals + obj_logits, attr_logits, box_deltas, feature_pooled = self.roi_heads(features, proposal_boxes, gt_boxes) + + # prepare FRCNN Outputs and select top proposals + boxes, classes, class_probs, attrs, attr_probs, roi_features = self.roi_outputs( + obj_logits=obj_logits, + attr_logits=attr_logits, + box_deltas=box_deltas, + pred_boxes=proposal_boxes, + features=feature_pooled, + sizes=image_shapes, + scales=scales_yx, + ) + + # will we pad??? + subset_kwargs = { + "max_detections": kwargs.get("max_detections", None), + "return_tensors": kwargs.get("return_tensors", None), + "pad_value": kwargs.get("pad_value", 0), + "padding": kwargs.get("padding", None), + } + preds_per_image = torch.tensor([p.size(0) for p in boxes]) + boxes = pad_list_tensors(boxes, preds_per_image, **subset_kwargs) + classes = pad_list_tensors(classes, preds_per_image, **subset_kwargs) + class_probs = pad_list_tensors(class_probs, preds_per_image, **subset_kwargs) + attrs = pad_list_tensors(attrs, preds_per_image, **subset_kwargs) + attr_probs = pad_list_tensors(attr_probs, preds_per_image, **subset_kwargs) + roi_features = pad_list_tensors(roi_features, preds_per_image, **subset_kwargs) + subset_kwargs["padding"] = None + preds_per_image = pad_list_tensors(preds_per_image, None, **subset_kwargs) + sizes = pad_list_tensors(image_shapes, None, **subset_kwargs) + normalized_boxes = norm_box(boxes, original_sizes) + return OrderedDict( + { + "obj_ids": classes, + "obj_probs": class_probs, + "attr_ids": attrs, + "attr_probs": attr_probs, + "boxes": boxes, + "sizes": sizes, + "preds_per_image": preds_per_image, + "roi_features": roi_features, + "normalized_boxes": normalized_boxes, + } + ) diff --git a/data/datasets/visual_question_answering/prepare.py b/data/datasets/visual_question_answering/prepare.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/data/datasets/visual_question_answering/processing_image.py b/data/datasets/visual_question_answering/processing_image.py new file mode 100644 index 0000000000000000000000000000000000000000..d32db29ccb3ba18e35cdf52f8ec77e2cb93a3b65 --- /dev/null +++ b/data/datasets/visual_question_answering/processing_image.py @@ -0,0 +1,151 @@ +""" + coding=utf-8 + Copyright 2018, Antonio Mendoza Hao Tan, Mohit Bansal + Adapted From Facebook Inc, Detectron2 + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License.import copy + """ +import sys +from typing import Tuple + +import numpy as np +import torch +from PIL import Image +from torch import nn + +from transformers.image_utils import PILImageResampling +from .utils import img_tensorize + + +class ResizeShortestEdge: + def __init__(self, short_edge_length, max_size=sys.maxsize): + """ + Args: + short_edge_length (list[min, max]) + max_size (int): maximum allowed longest edge length. + """ + self.interp_method = "bilinear" + self.max_size = max_size + self.short_edge_length = short_edge_length + + def __call__(self, imgs): + img_augs = [] + for img in imgs: + h, w = img.shape[:2] + # later: provide list and randomly choose index for resize + size = np.random.randint(self.short_edge_length[0], self.short_edge_length[1] + 1) + if size == 0: + return img + scale = size * 1.0 / min(h, w) + if h < w: + newh, neww = size, scale * w + else: + newh, neww = scale * h, size + if max(newh, neww) > self.max_size: + scale = self.max_size * 1.0 / max(newh, neww) + newh = newh * scale + neww = neww * scale + neww = int(neww + 0.5) + newh = int(newh + 0.5) + + if img.dtype == np.uint8: + pil_image = Image.fromarray(img) + pil_image = pil_image.resize((neww, newh), PILImageResampling.BILINEAR) + img = np.asarray(pil_image) + else: + img = img.permute(2, 0, 1).unsqueeze(0) # 3, 0, 1) # hw(c) -> nchw + img = nn.functional.interpolate( + img, (newh, neww), mode=self.interp_method, align_corners=False + ).squeeze(0) + img_augs.append(img) + + return img_augs + + +class Preprocess: + def __init__(self, cfg): + self.aug = ResizeShortestEdge([cfg.INPUT.MIN_SIZE_TEST, cfg.INPUT.MIN_SIZE_TEST], cfg.INPUT.MAX_SIZE_TEST) + self.input_format = cfg.INPUT.FORMAT + self.size_divisibility = cfg.SIZE_DIVISIBILITY + self.pad_value = cfg.PAD_VALUE + self.max_image_size = cfg.INPUT.MAX_SIZE_TEST + self.device = cfg.MODEL.DEVICE + self.pixel_std = torch.tensor(cfg.MODEL.PIXEL_STD).to(self.device).view(len(cfg.MODEL.PIXEL_STD), 1, 1) + self.pixel_mean = torch.tensor(cfg.MODEL.PIXEL_MEAN).to(self.device).view(len(cfg.MODEL.PIXEL_STD), 1, 1) + self.normalizer = lambda x: (x - self.pixel_mean) / self.pixel_std + + def pad(self, images): + max_size = tuple(max(s) for s in zip(*[img.shape for img in images])) + image_sizes = [im.shape[-2:] for im in images] + images = [ + nn.functional.pad( + im, + [0, max_size[-1] - size[1], 0, max_size[-2] - size[0]], + value=self.pad_value, + ) + for size, im in zip(image_sizes, images) + ] + + return torch.stack(images), torch.tensor(image_sizes) + + def __call__(self, images, single_image=False): + with torch.no_grad(): + if not isinstance(images, list): + images = [images] + if single_image: + assert len(images) == 1 + for i in range(len(images)): + if isinstance(images[i], torch.Tensor): + images.insert(i, images.pop(i).to(self.device).float()) + elif not isinstance(images[i], torch.Tensor): + images.insert( + i, + torch.as_tensor(img_tensorize(images.pop(i), input_format=self.input_format)) + #torch.as_tensor(images.pop(i)) + .to(self.device) + .float(), + ) + # resize smallest edge + raw_sizes = torch.tensor([im.shape[:2] for im in images]) + images = self.aug(images) + # transpose images and convert to torch tensors + # images = [torch.as_tensor(i.astype("float32")).permute(2, 0, 1).to(self.device) for i in images] + # now normalize before pad to avoid useless arithmetic + images = [self.normalizer(x) for x in images] + # now pad them to do the following operations + images, sizes = self.pad(images) + # Normalize + + if self.size_divisibility > 0: + raise NotImplementedError() + # pad + scales_yx = torch.true_divide(raw_sizes, sizes) + if single_image: + return images[0], sizes[0], scales_yx[0] + else: + return images, sizes, scales_yx + + +def _scale_box(boxes, scale_yx): + boxes[:, 0::2] *= scale_yx[:, 1] + boxes[:, 1::2] *= scale_yx[:, 0] + return boxes + + +def _clip_box(tensor, box_size: Tuple[int, int]): + assert torch.isfinite(tensor).all(), "Box tensor contains infinite or NaN!" + h, w = box_size + tensor[:, 0].clamp_(min=0, max=w) + tensor[:, 1].clamp_(min=0, max=h) + tensor[:, 2].clamp_(min=0, max=w) + tensor[:, 3].clamp_(min=0, max=h) diff --git a/data/datasets/visual_question_answering/split_train_as_train_val.py b/data/datasets/visual_question_answering/split_train_as_train_val.py new file mode 100644 index 0000000000000000000000000000000000000000..b359bb1f85496439977289083784c902e9414fc4 --- /dev/null +++ b/data/datasets/visual_question_answering/split_train_as_train_val.py @@ -0,0 +1,24 @@ +from utils.common.data_record import read_json, write_json +import random +import copy +from utils.dl.common.env import set_random_seed + +set_random_seed(1) + + +question_data = read_json("/data/zql/datasets/vqav2/Questions/v2_OpenEnded_mscoco_train2014_questions.json") + +train_val_split_ratio = 0.8 + +num_train = int(len(question_data["questions"]) * train_val_split_ratio) + +train_question_data = copy.deepcopy(question_data) +val_question_data = copy.deepcopy(question_data) + +train_question_data["questions"] = train_question_data["questions"][:num_train] +val_question_data["questions"] = val_question_data["questions"][num_train:] + +write_json("/data/zql/datasets/vqav2/Questions/v2_OpenEnded_mscoco_train2014_questions_train.json.my_train_split", train_question_data) +write_json("/data/zql/datasets/vqav2/Questions/v2_OpenEnded_mscoco_train2014_questions_train.json.my_val_split", val_question_data) + + diff --git a/data/datasets/visual_question_answering/test.py b/data/datasets/visual_question_answering/test.py new file mode 100644 index 0000000000000000000000000000000000000000..3a1d9c6639c05c0653bcbc581184a36595c64442 --- /dev/null +++ b/data/datasets/visual_question_answering/test.py @@ -0,0 +1,99 @@ +from utils.common.data_record import read_json, write_json +from data.datasets.visual_question_answering.glossary import normalize_word +from collections import defaultdict, Counter +from tqdm import tqdm + +ann_data = read_json('/data/zql/datasets/vqav2/Annotations/v2_mscoco_train2014_annotations.json') +question_data = read_json('/data/zql/datasets/vqav2/Questions/v2_OpenEnded_mscoco_train2014_questions.json') + + + +question_to_id = {} + +for q in tqdm(question_data['questions']): + question_to_id[q['question_id']] = q['question'] + + +classes_set = [] +for ann in ann_data['annotations']: + classes_set += [normalize_word(ann['multiple_choice_answer'])] +counter = {k: v for k, v in Counter(classes_set).items() if v >= 9} +ans2label = {k: i for i, k in enumerate(counter.keys())} +label2ans = list(counter.keys()) + + +# print(list(ans2label.keys())) +# exit() + +available_classes = list(ans2label.values()) +classes_split_1 = available_classes[0: 100] +classes_split_2 = available_classes[100: ] + +print(classes_split_1) + +dataset_info_1 = [] # (image_file_path, question, labels, scores) +dataset_info_2 = [] # (image_file_path, question, labels, scores) + +def get_score(occurences): + if occurences == 0: + return 0.0 + elif occurences == 1: + return 0.3 + elif occurences == 2: + return 0.6 + elif occurences == 3: + return 0.9 + else: + return 1.0 + + +ii = 0 + +pbar = tqdm(ann_data['annotations']) +for q in pbar: + answers = q["answers"] + answer_count = {} + for answer in answers: + answer_ = answer["answer"] + answer_count[answer_] = answer_count.get(answer_, 0) + 1 + + labels = [] + scores = [] + for answer in answer_count: + if answer not in ans2label: + continue + labels.append(ans2label[answer]) + score = get_score(answer_count[answer]) + scores.append(score) + + if len(labels) == 0: + continue + + # annotations[q["image_id"]][q["question_id"]].append( + # {"labels": labels, "scores": scores,} + # ) + + # full_label = [0] * len(ans2label) + # for label_idx, score in zip(labels, scores): + # full_label[label_idx] = score + + if all([label in classes_split_1 for label in labels]): + dataset_info_1 += [(q["image_id"], question_to_id[q["question_id"]], labels, scores)] + elif all([label in classes_split_2 for label in labels]): + dataset_info_2 += [(q["image_id"], question_to_id[q["question_id"]], [ii - 100 for ii in labels], scores)] + else: + # print('ignore') + pass + + # dataset_info += [(q["image_id"], question_to_id[q["question_id"]], labels, scores)] + + pbar.set_description(f'# samples: {len(dataset_info_1)}, {len(dataset_info_2)}') + # print(dataset_info[-1]) + # break + + # if ii < 10: + # print(dataset_info[-1]) + # ii += 1 + +write_json('/data/zql/datasets/vqav2/label1.json', dataset_info_1) +write_json('/data/zql/datasets/vqav2/label2.json', dataset_info_2) diff --git a/data/datasets/visual_question_answering/utils.py b/data/datasets/visual_question_answering/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..2fc6ea2062efd2412dbd121f2f72c8aec75d36cf --- /dev/null +++ b/data/datasets/visual_question_answering/utils.py @@ -0,0 +1,554 @@ +""" + coding=utf-8 + Copyright 2018, Antonio Mendoza Hao Tan, Mohit Bansal, Huggingface team :) + Adapted From Facebook Inc, Detectron2 + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License.import copy + """ + +import copy +import fnmatch +import json +import os +import pickle as pkl +import shutil +import sys +import tarfile +import tempfile +from collections import OrderedDict +from contextlib import contextmanager +from functools import partial +from hashlib import sha256 +from io import BytesIO +from pathlib import Path +from urllib.parse import urlparse +from zipfile import ZipFile, is_zipfile + +import cv2 +import numpy as np +import requests +import wget +from filelock import FileLock +from PIL import Image +from tqdm.auto import tqdm +from yaml import Loader, dump, load + + +try: + import torch + + _torch_available = True +except ImportError: + _torch_available = False + + +try: + from torch.hub import _get_torch_home + + torch_cache_home = _get_torch_home() +except ImportError: + torch_cache_home = os.path.expanduser( + os.getenv("TORCH_HOME", os.path.join(os.getenv("XDG_CACHE_HOME", "~/.cache"), "torch")) + ) + +default_cache_path = os.path.join(torch_cache_home, "transformers") + +CLOUDFRONT_DISTRIB_PREFIX = "https://cdn.huggingface.co" +S3_BUCKET_PREFIX = "https://s3.amazonaws.com/models.huggingface.co/bert" +PATH = "/".join(str(Path(__file__).resolve()).split("/")[:-1]) +CONFIG = os.path.join(PATH, "config.yaml") +ATTRIBUTES = os.path.join(PATH, "attributes.txt") +OBJECTS = os.path.join(PATH, "objects.txt") +PYTORCH_PRETRAINED_BERT_CACHE = os.getenv("PYTORCH_PRETRAINED_BERT_CACHE", default_cache_path) +PYTORCH_TRANSFORMERS_CACHE = os.getenv("PYTORCH_TRANSFORMERS_CACHE", PYTORCH_PRETRAINED_BERT_CACHE) +TRANSFORMERS_CACHE = os.getenv("TRANSFORMERS_CACHE", PYTORCH_TRANSFORMERS_CACHE) +WEIGHTS_NAME = "pytorch_model.bin" +CONFIG_NAME = "config.yaml" + + +def load_labels(objs=OBJECTS, attrs=ATTRIBUTES): + vg_classes = [] + with open(objs) as f: + for object in f.readlines(): + vg_classes.append(object.split(",")[0].lower().strip()) + + vg_attrs = [] + with open(attrs) as f: + for object in f.readlines(): + vg_attrs.append(object.split(",")[0].lower().strip()) + return vg_classes, vg_attrs + + +def load_checkpoint(ckp): + r = OrderedDict() + with open(ckp, "rb") as f: + ckp = pkl.load(f)["model"] + for k in copy.deepcopy(list(ckp.keys())): + v = ckp.pop(k) + if isinstance(v, np.ndarray): + v = torch.tensor(v) + else: + assert isinstance(v, torch.tensor), type(v) + r[k] = v + return r + + +class Config: + _pointer = {} + + def __init__(self, dictionary: dict, name: str = "root", level=0): + self._name = name + self._level = level + d = {} + for k, v in dictionary.items(): + if v is None: + raise ValueError() + k = copy.deepcopy(k) + v = copy.deepcopy(v) + if isinstance(v, dict): + v = Config(v, name=k, level=level + 1) + d[k] = v + setattr(self, k, v) + + self._pointer = d + + def __repr__(self): + return str(list((self._pointer.keys()))) + + def __setattr__(self, key, val): + self.__dict__[key] = val + self.__dict__[key.upper()] = val + levels = key.split(".") + last_level = len(levels) - 1 + pointer = self._pointer + if len(levels) > 1: + for i, l in enumerate(levels): + if hasattr(self, l) and isinstance(getattr(self, l), Config): + setattr(getattr(self, l), ".".join(levels[i:]), val) + if l == last_level: + pointer[l] = val + else: + pointer = pointer[l] + + def to_dict(self): + return self._pointer + + def dump_yaml(self, data, file_name): + with open(f"{file_name}", "w") as stream: + dump(data, stream) + + def dump_json(self, data, file_name): + with open(f"{file_name}", "w") as stream: + json.dump(data, stream) + + @staticmethod + def load_yaml(config): + with open(config) as stream: + data = load(stream, Loader=Loader) + return data + + def __str__(self): + t = " " + if self._name != "root": + r = f"{t * (self._level-1)}{self._name}:\n" + else: + r = "" + level = self._level + for i, (k, v) in enumerate(self._pointer.items()): + if isinstance(v, Config): + r += f"{t * (self._level)}{v}\n" + self._level += 1 + else: + r += f"{t * (self._level)}{k}: {v} ({type(v).__name__})\n" + self._level = level + return r[:-1] + + @classmethod + def from_pretrained(cls, pretrained_model_name_or_path: str, **kwargs): + config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs) + return cls(config_dict) + + @classmethod + def get_config_dict(cls, pretrained_model_name_or_path: str, **kwargs): + cache_dir = kwargs.pop("cache_dir", None) + force_download = kwargs.pop("force_download", False) + resume_download = kwargs.pop("resume_download", False) + proxies = kwargs.pop("proxies", None) + local_files_only = kwargs.pop("local_files_only", False) + + if os.path.isdir(pretrained_model_name_or_path): + config_file = os.path.join(pretrained_model_name_or_path, CONFIG_NAME) + elif os.path.isfile(pretrained_model_name_or_path) or is_remote_url(pretrained_model_name_or_path): + config_file = pretrained_model_name_or_path + else: + config_file = hf_bucket_url(pretrained_model_name_or_path, filename=CONFIG_NAME, use_cdn=False) + + try: + # Load from URL or cache if already cached + resolved_config_file = cached_path( + config_file, + cache_dir=cache_dir, + force_download=force_download, + proxies=proxies, + resume_download=resume_download, + local_files_only=local_files_only, + ) + # Load config dict + if resolved_config_file is None: + raise EnvironmentError + + config_file = Config.load_yaml(resolved_config_file) + + except EnvironmentError: + msg = "Can't load config for" + raise EnvironmentError(msg) + + if resolved_config_file == config_file: + print("loading configuration file from path") + else: + print("loading configuration file cache") + + return Config.load_yaml(resolved_config_file), kwargs + + +# quick compare tensors +def compare(in_tensor): + out_tensor = torch.load("dump.pt", map_location=in_tensor.device) + n1 = in_tensor.numpy() + n2 = out_tensor.numpy()[0] + print(n1.shape, n1[0, 0, :5]) + print(n2.shape, n2[0, 0, :5]) + assert np.allclose(n1, n2, rtol=0.01, atol=0.1), ( + f"{sum([1 for x in np.isclose(n1, n2, rtol=0.01, atol=0.1).flatten() if x is False])/len(n1.flatten())*100:.4f} %" + " element-wise mismatch" + ) + raise Exception("tensors are all good") + + # Hugging face functions below + + +def is_remote_url(url_or_filename): + parsed = urlparse(url_or_filename) + return parsed.scheme in ("http", "https") + + +def hf_bucket_url(model_id: str, filename: str, use_cdn=True) -> str: + endpoint = CLOUDFRONT_DISTRIB_PREFIX if use_cdn else S3_BUCKET_PREFIX + legacy_format = "/" not in model_id + if legacy_format: + return f"{endpoint}/{model_id}-{filename}" + else: + return f"{endpoint}/{model_id}/{filename}" + + +def http_get( + url, + temp_file, + proxies=None, + resume_size=0, + user_agent=None, +): + ua = "python/{}".format(sys.version.split()[0]) + if _torch_available: + ua += "; torch/{}".format(torch.__version__) + if isinstance(user_agent, dict): + ua += "; " + "; ".join("{}/{}".format(k, v) for k, v in user_agent.items()) + elif isinstance(user_agent, str): + ua += "; " + user_agent + headers = {"user-agent": ua} + if resume_size > 0: + headers["Range"] = "bytes=%d-" % (resume_size,) + response = requests.get(url, stream=True, proxies=proxies, headers=headers) + if response.status_code == 416: # Range not satisfiable + return + content_length = response.headers.get("Content-Length") + total = resume_size + int(content_length) if content_length is not None else None + progress = tqdm( + unit="B", + unit_scale=True, + total=total, + initial=resume_size, + desc="Downloading", + ) + for chunk in response.iter_content(chunk_size=1024): + if chunk: # filter out keep-alive new chunks + progress.update(len(chunk)) + temp_file.write(chunk) + progress.close() + + +def get_from_cache( + url, + cache_dir=None, + force_download=False, + proxies=None, + etag_timeout=10, + resume_download=False, + user_agent=None, + local_files_only=False, +): + if cache_dir is None: + cache_dir = TRANSFORMERS_CACHE + if isinstance(cache_dir, Path): + cache_dir = str(cache_dir) + + os.makedirs(cache_dir, exist_ok=True) + + etag = None + if not local_files_only: + try: + response = requests.head(url, allow_redirects=True, proxies=proxies, timeout=etag_timeout) + if response.status_code == 200: + etag = response.headers.get("ETag") + except (EnvironmentError, requests.exceptions.Timeout): + # etag is already None + pass + + filename = url_to_filename(url, etag) + + # get cache path to put the file + cache_path = os.path.join(cache_dir, filename) + + # etag is None = we don't have a connection, or url doesn't exist, or is otherwise inaccessible. + # try to get the last downloaded one + if etag is None: + if os.path.exists(cache_path): + return cache_path + else: + matching_files = [ + file + for file in fnmatch.filter(os.listdir(cache_dir), filename + ".*") + if not file.endswith(".json") and not file.endswith(".lock") + ] + if len(matching_files) > 0: + return os.path.join(cache_dir, matching_files[-1]) + else: + # If files cannot be found and local_files_only=True, + # the models might've been found if local_files_only=False + # Notify the user about that + if local_files_only: + raise ValueError( + "Cannot find the requested files in the cached path and outgoing traffic has been" + " disabled. To enable model look-ups and downloads online, set 'local_files_only'" + " to False." + ) + return None + + # From now on, etag is not None. + if os.path.exists(cache_path) and not force_download: + return cache_path + + # Prevent parallel downloads of the same file with a lock. + lock_path = cache_path + ".lock" + with FileLock(lock_path): + # If the download just completed while the lock was activated. + if os.path.exists(cache_path) and not force_download: + # Even if returning early like here, the lock will be released. + return cache_path + + if resume_download: + incomplete_path = cache_path + ".incomplete" + + @contextmanager + def _resumable_file_manager(): + with open(incomplete_path, "a+b") as f: + yield f + + temp_file_manager = _resumable_file_manager + if os.path.exists(incomplete_path): + resume_size = os.stat(incomplete_path).st_size + else: + resume_size = 0 + else: + temp_file_manager = partial(tempfile.NamedTemporaryFile, dir=cache_dir, delete=False) + resume_size = 0 + + # Download to temporary file, then copy to cache dir once finished. + # Otherwise you get corrupt cache entries if the download gets interrupted. + with temp_file_manager() as temp_file: + print( + "%s not found in cache or force_download set to True, downloading to %s", + url, + temp_file.name, + ) + + http_get( + url, + temp_file, + proxies=proxies, + resume_size=resume_size, + user_agent=user_agent, + ) + + os.replace(temp_file.name, cache_path) + + meta = {"url": url, "etag": etag} + meta_path = cache_path + ".json" + with open(meta_path, "w") as meta_file: + json.dump(meta, meta_file) + + return cache_path + + +def url_to_filename(url, etag=None): + url_bytes = url.encode("utf-8") + url_hash = sha256(url_bytes) + filename = url_hash.hexdigest() + + if etag: + etag_bytes = etag.encode("utf-8") + etag_hash = sha256(etag_bytes) + filename += "." + etag_hash.hexdigest() + + if url.endswith(".h5"): + filename += ".h5" + + return filename + + +def cached_path( + url_or_filename, + cache_dir=None, + force_download=False, + proxies=None, + resume_download=False, + user_agent=None, + extract_compressed_file=False, + force_extract=False, + local_files_only=False, +): + if cache_dir is None: + cache_dir = TRANSFORMERS_CACHE + if isinstance(url_or_filename, Path): + url_or_filename = str(url_or_filename) + if isinstance(cache_dir, Path): + cache_dir = str(cache_dir) + + if is_remote_url(url_or_filename): + # URL, so get it from the cache (downloading if necessary) + output_path = get_from_cache( + url_or_filename, + cache_dir=cache_dir, + force_download=force_download, + proxies=proxies, + resume_download=resume_download, + user_agent=user_agent, + local_files_only=local_files_only, + ) + elif os.path.exists(url_or_filename): + # File, and it exists. + output_path = url_or_filename + elif urlparse(url_or_filename).scheme == "": + # File, but it doesn't exist. + raise EnvironmentError("file {} not found".format(url_or_filename)) + else: + # Something unknown + raise ValueError("unable to parse {} as a URL or as a local path".format(url_or_filename)) + + if extract_compressed_file: + if not is_zipfile(output_path) and not tarfile.is_tarfile(output_path): + return output_path + + # Path where we extract compressed archives + # We avoid '.' in dir name and add "-extracted" at the end: "./model.zip" => "./model-zip-extracted/" + output_dir, output_file = os.path.split(output_path) + output_extract_dir_name = output_file.replace(".", "-") + "-extracted" + output_path_extracted = os.path.join(output_dir, output_extract_dir_name) + + if os.path.isdir(output_path_extracted) and os.listdir(output_path_extracted) and not force_extract: + return output_path_extracted + + # Prevent parallel extractions + lock_path = output_path + ".lock" + with FileLock(lock_path): + shutil.rmtree(output_path_extracted, ignore_errors=True) + os.makedirs(output_path_extracted) + if is_zipfile(output_path): + with ZipFile(output_path, "r") as zip_file: + zip_file.extractall(output_path_extracted) + zip_file.close() + elif tarfile.is_tarfile(output_path): + tar_file = tarfile.open(output_path) + tar_file.extractall(output_path_extracted) + tar_file.close() + else: + raise EnvironmentError("Archive format of {} could not be identified".format(output_path)) + + return output_path_extracted + + return output_path + + +def get_data(query, delim=","): + assert isinstance(query, str) + if os.path.isfile(query): + with open(query) as f: + data = eval(f.read()) + else: + req = requests.get(query) + try: + data = requests.json() + except Exception: + data = req.content.decode() + assert data is not None, "could not connect" + try: + data = eval(data) + except Exception: + data = data.split("\n") + req.close() + return data + + +def get_image_from_url(url): + response = requests.get(url) + img = np.array(Image.open(BytesIO(response.content))) + return img + + +# to load legacy frcnn checkpoint from detectron +def load_frcnn_pkl_from_url(url): + fn = url.split("/")[-1] + if fn not in os.listdir(os.getcwd()): + wget.download(url) + with open(fn, "rb") as stream: + weights = pkl.load(stream) + model = weights.pop("model") + new = {} + for k, v in model.items(): + new[k] = torch.from_numpy(v) + if "running_var" in k: + zero = torch.tensor([0]) + k2 = k.replace("running_var", "num_batches_tracked") + new[k2] = zero + return new + + +def get_demo_path(): + print(f"{os.path.abspath(os.path.join(PATH, os.pardir))}/demo.ipynb") + + +def img_tensorize(im, input_format="RGB"): + assert isinstance(im, str) + if os.path.isfile(im): + img = cv2.imread(im) + else: + img = get_image_from_url(im) + assert img is not None, f"could not connect to: {im}" + img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) + if input_format == "RGB": + img = img[:, :, ::-1] + return img + + +def chunk(images, batch=1): + return (images[i : i + batch] for i in range(0, len(images), batch)) diff --git a/data/datasets/visual_question_answering/vqav2.py b/data/datasets/visual_question_answering/vqav2.py new file mode 100644 index 0000000000000000000000000000000000000000..aae934bc09befb5a1755c96d871fc4924f73e25f --- /dev/null +++ b/data/datasets/visual_question_answering/vqav2.py @@ -0,0 +1,916 @@ +from ..data_aug import pil_image_to_tensor +from ..ab_dataset import ABDataset +from ..dataset_split import train_val_test_split +from ..dataset_cache import get_dataset_cache_path, read_cached_dataset_status, cache_dataset_status +# from .mm_image_folder import MMImageFolder +from ..dataset_split import train_val_split +# from torchvision.datasets import CIFAR10 as RawCIFAR10 +import os +from typing import Dict, List, Optional +from torchvision.transforms import Compose, Resize +from utils.common.others import HiddenPrints +import numpy as np +from ..registery import dataset_register +import torch +from transformers import ViltProcessor, ViltForQuestionAnswering +from PIL import Image +from utils.common.log import logger +from utils.common.data_record import read_json +import cv2 + +all_classes = ['net', 'pitcher', 'orange', 'yes', 'white', 'skiing', 'red', 'frisbee', 'brushing teeth', 'no', 'black and white', 'skateboard', '1', 'blue', 'green', 'motorcycle', 'gray', '2', 'purse', 'skis', 'poles', 'surfboard', 'dog', 'on', 'office', 'large', 'very big', 'laptop', 'vent', 'computer', 'black', 'bear', '3', 'wii', 'glasses', 'tree', 'eating', 'log', '5', 'left', 'living room', 'pink', 'right', 'railing', 'grass', 'wire', '10 years', 'knife', 'cake', 'banana', 'chef', 'vanilla', '4', 'outdoor', 'mustard', 'bun', 'clouds', 'dock', 'brown', 'silver', 'refrigerator', 'square', 'teddy', 'elm', 'stripes', 'baseball', 'catcher', 'beer', 'bottom', 'north', 'nike', 'yellow and white', 'morning', 'elephant', 'red and white', 'propeller', 'tan', 'wall', 'clock', 'table', '0', 'wood', 'christmas', 'spinach', 'thick', 'bag', 'leaves', 'necklace', '6', 'bathroom', 'shower', 'towel', 'solid', 'referee', 'wilson', 'e', '24', 'hat', 'grazing', 'sheep', '10', 'tag', 'spanish', 'hot dog', 'plate', 'lunch', 'butter', 'peppers', 'onions', 'very', 'pig', 'sweet', 'flowers', 'floral', 'yellow', 'window', '7', 'pizza', 'car', '', 'cargo', 'stairs', 'abstract', 'rug', 'baseball cap', 'texting', 'pole', 'crosswalk', 'nothing', 'urban', 'bus', 'light', 'afternoon', 'boat', 'cheese', 'paper', 'real', 'sun', 'birthday', 'words', 'inside', 'shadows', 'tomato', 'evergreen', '100 feet', 'trees', 'building', 'hay', 'ski pole', 'walking', 'ice', 'laundry', 'pepsi', 'good', '1:50', 'purple', '13', 'africa', 'teddy bears', 'socks', 'giraffe', 'soccer', 'blue and yellow', 'zebras', 'cupcake', 'broccoli', 'parking lot', 'cows', 'herding', 'on table', 'fish', 'nightstand', '50', 'overcast', 'cross', 'toaster oven', 'tile', '11:55', 'red and yellow', 'nowhere', 'hair dryer', 'truck', '11', 'people', 'rectangle', 'hot dogs', 'party', '12:55', 'apron', 'kitchen', 'cooking', 'ring', '1 way', 'stop', 'neither', 'many', 'female', 'brushing', 'tie', 'tennis racket', 'knife and fork', 'restaurant', 'cat', 'bed', 'sand', 'ocean', 'cold', 'kites', 'cumulus', 'standing', 'male', 'star', 'tracks', 'chocolate', 'round', 'fork and knife', 'yankees', 'pictures', 'dots', 'bird', 'parrot', 'red white and blue', 'man', 'metal', 'fence', 'snowboarding', 'pine', 'snow', 'shorts', 'swim', 'wine', 'brick', 'no parking', 'children', 'beef', 'phone', 'english', 'cell phone', 'pink and yellow', 'clear', 'watermelon', 'bedroom', 'fork', 'cow', 'rackets', 'tennis rackets', '8', 'collar', 'tennis', 'playing tennis', 'skirt', '30', 'polka dot', 'beach', 'horse', 'grill', 'african american', 'down', 'street', 'in air', 'sweater', 'yellow and blue', 'park', 'spectators', 'parasailing', '31', 'river', '55', 'shadow', 'winter', 'chicken', 'tea', 'evening', 'dusk', 'ski resort', 'helmet', 'bench', 'resting', 'elephants', 'southwest', 'usa', 'cars', 'town', 'bananas', 'umbrella', 'container', 'woman', 'on counter', 'salad', 'striped', 'motel', 'vertical', 'oranges', 'hot sauce', 'bottle', 'juice', 'eyes', 'ground', 'backpack', 'black and yellow', 'forward', 'jackets', '1 on right', 'green and yellow', 'playing baseball', 'riding', 'sitting', 'carrot', 'basket', 'seagull', 'ski poles', 'p', 'parking', 'street light', 'strap', 'bike', 'riding bike', 'poodle', 'shoes', 'carpet', 'lettuce', 'food', '1 foot', 'roses', 'mountains', 'scissors', 'camera', 'beige', 'beard', 'cutting', 'baby', 'tape', 'watch', 'never', 'taking picture', 'eggs', 'syrup', 'sandwich', 'water skiing', 'microphone', 'back', 'bears', 'donuts', 'w', 'sky', 'double decker', 'england', 'surfing', 'running', 'shirt', 'barn', 'weather vane', 'white and blue', 'fishing', 'bridge', 'los angeles', 'open', 'red sox', 'bat', 'plane', 'white and green', 'transportation', 'sunny', 'bus stop', 'city', 'brown and white', 'bicycle', 'crow', 'magazines', 'daisy', '14', 'old', 'curtains', 'snowboard', 'dinosaur', 'racing', 'asphalt', 'court', 'plastic', 'circle', 'red and blue', 'zebra', '12', 'biplane', 'shallow', 'brazil', 'logo', '2:20', 'electric', 'motion', 'toothbrushes', 'orange and white', '66', 'spoon', 'toyota', 'tennis shoes', '46', 'second', 'no 1', 'iphone', 'friend', 'apple', '15', 'tiger', 'glove', 'airplane', 'bow', 'air france', 'passengers', 'tv', 'on building', '3:55', 'victorian', 'steeple', 'happy', 'skateboarding', 'fruit', 'cutting board', 'cantaloupe', 'kiwi', 'sliced', 'heart', 'water', 'rainy', 'carrots', 'giraffes', 'eat', 'ramp', 'lab', 'field', 'horizontal', 'birds', 'home', 'shrimp', '12 feet', 'girl', 'modern', 'dell', 'boots', 'sunglasses', 'black and orange', 'yellow and black', 'gloves', 'hp', 'desk', 'both', 'sign', 'on street', '2000', 'cirrus', 'ceiling', 'fluorescent', 'up', '9', 'boys', 'playing soccer', 'american', 'passenger', 'turn', 'palm', 'wedding', 'branch', 'parrots', 'air force', 'on tracks', 'small', 'dirty', 'france', 'honda', '2.00', 'vase', 'flying', 'driving', 'tissue', 'protest', 'corona', 'twin', 'clothes', 't shirt', 'window sill', 'wild', 'noon', 'caution', 'spring', 'raining', 'cane', 'school', 'windsurfing', 'parachute', 'black and red', '25', 'background', 'toaster', 'planes', 'yellow and red', 'spatula', '10:10', 'ivory', 'train', 'highway', 'off', 'on track', 'electricity', 'italy', 'dinner', 'sink', 'squares', '5 ft', 'parked', 'store', 'dress', 'signs', 'football', 'rugby', 'stainless steel', 'dirt', 'blue and white', 'klm', 'house', 'unknown', 'ford', 'reading', 'chair', 'mountain', 'alive', 'water skis', 'picture', 'parade', 'trailer', 'boating', 'holding it', 'shade', 'cloth', 'candle', 'hose', 'hand', '3:25', 'on sidewalk', 'poster', 'downhill', 'reflection', 'summer', 'pickles', 'halloween', 'bats', 'london', 'zoo', 'surfer', 'racket', 'flickr', 'cutting hair', 'strawberries', 'mushroom', 'teddy bear', 'big', 'suitcase', 'veggie', 'pepper', 'houses', '70', 'toshiba', 'triangle', 'boxes', 'photograph', 'smoke', 'engine', 'camel', 'sidewalk', 'left 1', 'red and green', '4:35', 'on couch', 'candy', 'homemade', 'mouse', 'box', 'movie', '45', 'strawberry', 'fridge', 'full', 'vegetables', 'bright', 'play', 'remote', 'pond', 'savannah', 'celery', 'concrete', 'semi', 'scania', 'safety', 'posing', 'fabric', 'laying', 'couch', 'blueberries', 'handle', 'pipe', 'stick', 'steak', 'chain link', 'barbed wire', 'mozzarella', 'soda', 'fire hydrant', 'cat food', 'pepperoni', 'lot', 'licking', 'red and black', 'clay', 'tennis court', 'jumping', 'potatoes', 'toothbrush', 'kite', 'flying kite', 'broken', 'black and silver', 'lap', 'outside', '44', 'delta', 'greyhound', 'talking on phone', 'bad', 'kettle', '35', 'motorcycles', 'produce', 'steering wheel', '18', 'humans', 'coffee', 'white and brown', 'fall', 'bread', 'cherry', '4:30', 'flag', 'night', 'lamp', 'cucumber', 'porcelain', 'oval', 'museum', 'rain', 'sprinkles', '20', 'kids', 'bracelet', 'sneakers', 'mask', 'mickey mouse', 'very high', 'costume', 'cabbage', 'paint', 'lighting', 'young', 'air conditioner', 'wooden', 'board', 'beets', '16', 'lights', 'ladder', 'glass', 'fries', 'steamed', 'shepherd', 'cotton', 'suit', 'goatee', 'on his head', 'print', 'happy birthday', 'forks', 'travel', 'maple', '200', 'oil', 'jeans', 'can', 'chopsticks', 'on wall', 'construction', '36', 'chinese', 'festival', 'gas', 'throwing', 'circus', 'wires', 'not possible', 'plates', 'sugar', 'in', "women's", 'door', 'volleyball', 'serving', 'ponytail', 'business', 'decoration', 'santa', 'flat', 'barrel', '12:15', 'candles', 'free', 'hair', 'ball', 'stop sign', 'wetsuit', 'green and black', 'foreground', 'stands', 'china airlines', 'flower', '300', 'on bench', 'plaster', 'phones', 'sailboat', 'apples', 'road', 'recently', 'cones', 'cactus', 'rice', 'vegetarian', 'donut', 'ketchup', 'police', 'mirror', 'rock', 'meat', 'blinds', 'cell phones', 'china', 'rust', '7:25', 'stone', 'vans', 'middle', 'eagle', '9:30', 'ping pong', 'microwave', 'gmc', 'umbrellas', 'wrist', 'laughing', 'boy', 'next to toilet', 'tabby', 'petting', 'south', '40', 'checkered', 'slow', 'cardboard', 'windows', 'croissant', 'plain', 'cookie', 'on ground', 'low', 'water bottle', 'goggles', 'turkey', 'shut', 'kite flying', 'bowl', 'smile', 'in bowl', 'bush', 'cloudy', 'top left', 'skateboarder', 'coca cola', 'pan', 'drinking', 'short', 'floor', 'thanksgiving', 'radio', 'drink', 'on toilet', 'bike rack', 'bleachers', 'train tracks', 'horses', 'far', 'top', 'toilet', 'in water', 'private', 'nature', 'commercial', 'stroller', 'power', 'stuffed animals', 'uniforms', 'japan', 'faucet', 'green and orange', 'corn', 'white and yellow', 'mercedes', 'in sky', 'tarp', 'indian', 'counter', 'multicolored', 'polar', 'go', 'no number', 'swimming', 'bridle', 'cowboy', 'olives', 'pizza cutter', 'british airways', 'nighttime', 'australia', 'tiles', 'pug', 'wicker', 'british', 'us airways express', 'burton', 'christmas tree', 'napkin', 'writing', 'rocks', 'hello kitty', 'gold', 'fan', 'skateboards', 'day', 'on floor', '2008', 'dark', 'flying kites', 'rural', 'olympics', 'bmw', '34', 'denim', 'typing', 'for fun', 'steel', 'watching tv', 'driver', 'grapes', 'f', 'angels', 'roof', 'handlebars', 'train station', 'public', 'oak', 'sleeping', 'canada', 'air canada', 'on top', 'tired', 'blonde', 'cups', 'little', 'adidas', '10 feet', 'white and gray', 'leaf', 'fisheye', 'forest', 'war', 'octagon', 'raspberry', 'helmets', 'united states', '29', 'noodles', 'van', 'long', 'traveling', 'luggage', 'airport', 'single', 'pitching', 'dugout', 'garbage', 'happiness', 'cigarette', 'on tower', 'antelope', 'graffiti', 'skating', 'on road', 'curved', 'washington', 'ski lift', 'athletics', 'brace', 'squatting', 'catching', 'batter', 'batting', 'game', 'towards', '33', 'sliding', 'makeup', 'japanese', 'person', 'pirates', 'plaid', 'rose', 'daytime', 'keyboard', 'surfboards', 'hummingbird', 'ollie', '11:30', 'clock tower', 'san francisco', 'stopping', 'tags', 'samsung', 'computers', 'cabinets', 'talking', 'asparagus', '5 years', 'adult', 'rabbit', 'empty', 'softball', '1st', 'playing', 'chairs', 'farm', 'cross country', 'dump truck', 'women', 'snowboarder', 'tall', 'monkey', 'fire', 'books', 'cessna', 'chandelier', 'dunkin donuts', 'beans', 'relish', 'parking meter', 'ducks', 'sandals', 'doughnut', 'lighthouse', 'yacht', 'german shepherd', 'raw', 'chain', '2 feet', 'pedestal', 'mutt', 'race', 'poor', 'cat and dog', 'station', 'printer', 'daisies', 'front', 'gravel', 'grassy', 'pigeons', 'dogs', 'in car', 'life', 'wii remotes', 'suv', 'leather', 'bottom right', 'peace', 'blanket', 'frisbees', '12:30', 'scooter', 'going', 'analog', 'america', 'pitbull', 'relaxing', 'paddle boarding', 'white and pink', 'ride', 'side', 'on desk', 'on chair', '2012', 'multi', 'straight', 'big ben', 'closed', '3 feet', 'waves', 'buoy', 'trash can', 'medium', 'very tall', 'yamaha', 'sunlight', 'hit ball', 'dry', 'coke', 'gym', 'orange and black', 'center', 'rope', 'flip flops', 'siamese', 'crafts', 'color', 'italian', 'playing frisbee', 'skate park', 'orange juice', 'windowsill', 'thumb', 'pie', 'toast', 'no hat', 'benches', 'diamond', 'blender', 'avocado', 'television', 'speakers', 'pony', 'baseball field', 'pavement', 'not there', 'diamonds', '4 feet', 'goalie', 'soccer ball', 'runway', 'video game', 'gaming', 'casual', 'green and white', 'toilet brush', 'working', 'pickup', 'girls', 'remotes', 'pasta', 'hood', 'braves', 'skier', 'motorola', '17', 'b', '100', 'hospital', 'wagon', 'milk', 'ferry', 'rainbow', 'on bed', 'toward', '1:30', '19', 'mercedes benz', 'supreme', 'thin', 'platform', 'thai', 'storage', 'swan', 'peach', '10:05', 'dome', 'chiquita', '2:00', 'mountain dew', '23', 'knives', 'street sign', 'on beach', 'playing wii', 'stickers', 'yogurt', 'on grass', '9:45', 'gatorade', 'umpire', '37', 'desktop', 'desserts', 'main', 'boston', 'fell', 'top right', 'case', 'asleep', 'over', 'grapefruit', 'breakfast', 'headphones', 'freight', 'cup', 'sweatband', 'nobody', 'lamps', '9:25', 'scarf', 'on fridge', 'moving', 'fresh', 'blue jay', 'chihuahua', 'ceramic', 'mushrooms', 'on plate', 'human', 'power lines', 'hotel', 'map', 'earring', 'boarding', 'warm', 'napkins', 'brown and black', 'broom', 'basketball', 'papers', 'sad', 'kickstand', '60', 'shoulder', 'sleep', 'footprints', 'tunnel', '1990', 'hats', '6 inches', 'ham', 'bacon', 'church', '53', 'pineapple', 'at camera', 'red bull', 'pilot', 'tattoo', 'work', 'polar bear', 'taking off', 'website', '22', '4:00', 'coffee maker', 'fast', 'fur', 'rubber', 'tongs', 'german', 'germany', 'toy', '3:20', 'calm', 'pots', 'fruits', '9:20', 'drawer', 'oven', 'soup', 'stove', 'heels', 'wind', 'island', 'blood', 'leg', 'theater', 'tennis racquet', '21', 'gothic', '2:35', 'wii remote', 'turning', '20 feet', 'ears', 'fun', 'to right', 'child', 'fly', 'head', 'drywall', 'pier', 'feeding giraffe', 'in vase', 'burger', 'easter', 'onion', 'uniform', 'guitar', 'time', 'tomatoes', 'ship', 'tulips', 'glaze', 'tent', 'market', 'bandana', 'still', "don't know", 'piano', 'mouth', 'run', 'sparrow', 'lines', 'vest', '1950', 'jet', 'sepia', '2015', 'busy', 'dessert', '75', 'finch', 'pastries', 'outdoors', 'bakery', 'clean', 'ipod', 'tablecloth', 'looking at phone', 'in front', 'food truck', 'face', 'swinging', 'safari', '500', 'volkswagen', '2010', 'shelves', 'riding horses', '2016', 'towels', 'lemon', 'straw', 'bamboo', '5 feet', 'hardwood', 'h', 'meter', 'charging', 'bald', 'caucasian', 'man on left', 'stand', '27', 'dining room', 'sandwiches', '32', 'apartment', 'tower', 'virgin', 'out', 'white and red', "i don't know", 'chains', 'legs', 'goats', 's', 'dresser', 'camper', 'half', 'decorative', 'hawaiian', 'wheel', 'florida', 'reds', 'washington dc', 'moon', 'conference', 'screen', 'controller', 'robin', 'men', 'protection', 'harley davidson', 'coal', 'mustache', 'smiling', 'pedestrians', 'me', 'tray', 'monitor', 'bell', 'landscape', 'club', 'toothpick', 'seagulls', 'bowtie', 'lake', 'steam', 'surf', 'baseball glove', 'blinders', 'woods', 'shearing', 'dad', 'mixer', 'pot', 'blending', 'identification', 'owl', 'wine glass', 'new york', 'yarn', 'tennis ball', 'ice cream', 'chevrolet', 'shirt and tie', 'taking selfie', 'blue and green', "he isn't", 'cutting cake', 'east', 'setting', '7 eleven', 'stars', 'jockey', 'jacket', 'book', 'gray and white', 'pen', 'red white blue', 'above', 'alaska', 'tongue', 'feathers', 'k', 'camping', 'corner', 'away', 'ski', 'texas', 'fire truck', 'sailboats', 'jump', 'walk', 'spray paint', 'loading', 'united', '1000', 'roman numerals', 'surprise', '3rd', 'first', 'side of road', 'dodgers', 'airplanes', 'unsure', 'russian', 'wet', '5 star', 'blankets', 'natural', 'across street', 'smartphone', 'duck', 'sausage', 'paris', 'newspaper', 'pants', 'spices', 'pillow', 'to left', 'snowboards', 'colgate', 'on elephant', 'string', 'horns', '2:40', "men's", 'cobblestone', 'regular', 'staring', '28', 'barber shop', 'cut', 'x', 'above sink', 'above stove', 'dishes', 'dalmatian', 'watching', 'glazed', '5:25', 'messy', 'wallet', 'tuna', 'grilled', 'french', 'green and blue', 'sunflowers', 'wool', 'cabinet', 'shell', 'foil', 'bottles', 'bar', 'king', 'paper towels', 'friends', 'beagle', 'school bus', 'laptops', 'snowing', 'cement', 'pc', 'accident', 'stuffed animal', 'balance', 'white and black', 'cleats', 'on sink', 'pool', 'mom', 'downtown', 'asian', 'heater', 'bathing', '193', 'against wall', 'canopy', 'berries', 'military', 'pickle', 'clams', 'seafood', 'in box', 'boats', 'lizard', 'lemonade', 'm', 'soft', 'country', 'for sale', 'arm', 'listening', 'curly', 'play tennis', 'hands', 'cereal', 'blue and red', 'robe', 'soap', 'trains', 'throwing frisbee', 'smoking', 'india', 'headband', 'not very', 'westin', 'serve', 'bicycles', "can't tell", 'visibility', 'ana', 'reins', 'rodeo', 'riding motorcycle', 'mexico', 'mother', 'african', 'left and right', 'button', 'earrings', 'blackberry', 'cell', '10:00', 'harness', 'pillows', 'vegetable', 'tablet', 'fern', 'cats', 'golden retriever', 'goat', 'tractor', "valentine's day", 'hearts', 'khaki', 'man on right', "mcdonald's", 'arriving', 'husky', 'on skateboard', 'vases', 'coat', 'beanie', 'coming', 'granite', 'sports', 'leash', 'balls', 'blurry', 'baseball bat', 'mug', 'eiffel tower', 'worms', 'trash', 'terrier', 'painting', 'rooster', '42', 'jones', 'state farm', 'balloon', 'trunk', 'coach', 't', 'playing game', 'fireplace', 'behind clouds', 'uphill', 'motocross', 'sony', 'magazine', 'kitesurfing', 'catching frisbee', 'catch frisbee', 'bud light', 'fighting', '1 on left', 'very old', 'hallway', 'lexus', 'wii controller', '5:45', 'catholic', 'muffin', 'traffic light', 'grocery', 'shelf', '2:25', 'honey', 'plants', 'oars', 'foggy', "nathan's", 'cord', 'yard', '48', 'chimney', 'calico', 'suits', 'sideways', 'animals', 'black and blue', 'bikini', 'photographer', 'queen', '1:00', '12:05', 'horseback riding', 'awake', 'bunny', '12:00', 'continental', 'rye', 'family', 'lots', 'owner', 'palm tree', 'design', 'far right', 'tire', 'younger', 'biking', 'giants', 'caramel', 'polo', 'emirates', 'magnets', 'mat', 'ivy', 'cakes', 'bob', 'asia', 'graduation', 'cauliflower', 'c', 'rough', 'air', 'windy', 'victoria', 'trick', 'labrador', 'on left', 'yellow and green', 'butterfly', 'fake', 'on napkin', 'bricks', 'wine glasses', 'detroit', "man's", 'parsley', 'art', 'subway', 'wave', 'placemat', 'hydrant', 'sofa', 'pigeon', 'all', 'branches', 'plant', 'to eat', 'zucchini', 'feta', 'mouse pad', 'cloud', 'toilet paper', 'pumpkin', 'rowing', 'handicap', 'seeds', 'fly kite', 'chicago', 'marble', 'frame', '150', 'rocky', 'sauce', "it's not", 'control', 'high chair', 'playstation', 'xbox', 'roman', 'land', '1:35', 'lifeguard', 'size', 'bull', 'goose', '8 feet', 'recessed', 'statue', 'index', 'phillies', 'strike', 'mirrors', 'pointing', 'farmer', 'collie', 'motorbike', 'lanes', 'bikes', 'gas station', 'logs', 'smaller', 'desert', 'yield', 'flags', 'stool', 'kitten', 'doll', 'daffodils', 'letters', 'dishwasher', 'nuts', '2013', 'persian', 'swim trunks', 'deep', 'doubles', 'in field', 'wristband', 'wheels', 'baking', '4:15', '11:00', 'ear', '2007', '51', 'frog', 'boogie board', 'hungry', 'by window', 'ambulance', 'pigtails', 'microsoft', 'on man', 'laying down', '3:00', 'taxi', 'pedestrian', 'landing', 'numbers', '38', 'stones', 'clocks', 'new', 'picnic', 'fog', 'buffalo', 'under armour', 'orioles', 'bags', 'golden gate', 'castle', 'canoe', 'selfie', 'cream', 'floating', 'indoor', 'antique', 'aluminum', 'peas', 'sun hat', 'on right', 'flour', 'under sink', 'fashion', 'fedora', 'shells', '1 hour', 'puppy', 'motor', '120', 'sail', 'mexican', 'dead end', 'paddle', 'shop', 'boxing', 'birthday cake', 'chalk', 'style', 'nissan', 'sticker', 'north face', 'squash', 'not sure', 'seat', 'himself', 'circles', 'san diego', 'kia', 'mattress', 'obama', 'lamb', 'american flag', 'climbing', 'skull and crossbones', 'roast beef', 'visor', 'double', '52', 'high', 'stagecoach', 'cart', 'feeding', 'eaten', 'cone', 'smoothie', 'golf', 'colorado', 'electronics', '5:15', 'bowling', 'players', 'ketchup and mustard', 'styrofoam', '6 feet', 'hawk', 'cheddar', 'arabic', 'shower curtain', 'army', 'salmon', 'hanging', 'whole', 'behind fence', 'bars', 'moss', 'no dog', 'traffic', 'r', 'countryside', 'directions', 'cooked', 'aa', '6:45', '4 way', 'stripe', 'brand', 'baseball player', 'bunk', 'coleslaw', 'europe', 'dead', 'arch', 'scrambled', 'clothing', 'closet', 'egg', 'suitcases', 'indoors', 'tires', 'lilies', 'cafe', 'toothpaste', 'in background', 'tarmac', 'painted', 'sunset', 'orange and yellow', 'zebra and giraffe', 'ladybug', 'hills', 'tail', 'couple', 'kawasaki', 'smooth', 'powdered sugar', 'pedestrian crossing', 'french fries', 'teeth', 'ribbon', 'saddle', 'on train', '39', 'curb', 'tow', 'shark', 'white and orange', 'gravy', 'curtain', 'lime', 'skull', 'crossing', 'peacock', 'neck', 'hit', 'dragon', 'tissues', 'basil', 'waving', 'helicopter', 'mud', 'us', 'red and gray', 'sunflower', 'wallpaper', '11:20', 'seattle', 'bookshelf', 'looking', '1 inch', 'harley', 'urinal', 'navy', 'fedex', 'rays', 'deck', 'coaster', '1:20', '4:20', '5:00', 'jp morgan', 'palm trees', 'tub', 'pens', '2 people', 'speaker', 'hamburger', 'green beans', "it isn't", '10:20', 'buildings', 'on shelf', 'orange and blue', '90', 'north america', 'arrow', 'news', 'tropicana', 'formal', 'in grass', 'thumbs up', 'clip', 'tennis player', 'pastry', 'nose', 'pacifier', '11:35', 'different teams', 'cardinals', 'bagel', 'huge', 'out of focus', 'cook', 'wheat', 'photo', 'sedan', 'lanyard', 'pink and white', 'sesame', 'space', 'warning', 'snowy', 'tater tots', 'tropical', 'grandfather', 'mac', 'pajamas', '350', 'casserole', 'pelican', '2009', 'clydesdale', 'tow truck', 'belt', 'west', 'omelet', 'heavy', 'crown', 'in corner', 'hexagon', 'mound', 'iris', 'g', '2:15', '3:10', 'drawing', 'only', 'washing', 'nokia', 'windsor', 'icing', 'several', 'no smoking', 'kayak', 'frosting', 'jetblue', 'shoe', 'britain', 'ties', 'bank', 'camouflage', 'privacy', 'bib', 'blue and gray', 'looking out window', 'falling', 'bucket', 'cupcakes', 'throw ball', 'garden', 'almonds', 'starbucks', 'all way', 'home plate', 'base', 'toys', '1 in front', 'foot', 'california', 'towing', 'cheesecake', 'bushes', 'bow tie', 'down street', '2011', 'police officer', 'windmill', 'taking pictures', 'cleaning', 'on pole', 'main street', 'catch ball', 'mario', 'track', 'garage', "they aren't", 'tents', 'tattoos', '2:45', 'wheelchair', 'money', 'top hat', 'willow', 'brushing hair', '80', 'green and red', 'barrier', 'hiking', 'tank top', 'lufthansa', 'menu', 'forehand', 'wii controllers', 'hundreds', 'water ski', 'furniture', 'paisley', 'pizza hut', 'hill', 'prom', 'tiara', 'students', 'information', 'hazy', 'canon', 'bird feeder', 'crane', 'dr pepper', 'logitech', '2:10', 'all of them', 'utensils', 'telephone', 'converse', 'bone', 'jeep', 'nursing', 'krispy kreme', 'ranch', 'polka dots', 'railroad crossing', 'shirts', 'feeder', 'above toilet', 'unclear', 'below', '43', 'spoons', 'calendar', 'mint', 'spiderman', 'lg', 'concert', 'coats', 'lady', 'dodge', 'flat screen', '10:30', 'music', 'polar bears', 'riding horse', 'cookies', 'hot', 'behind', 'dole', '26', 'pans', 'love', 'winnie pooh', 'copyright', '2 hours', 'snowsuit', 'kissing', 'backhand', 'swans', 'nintendo', 'direction', 'waiting', 'mohawk', 'rail', 'hoodie', 'feet', '106', '10:55', 'coins', 'mitt', 'room', 'adults', 'cameras', 'marker', 'sled', 'conductor', 'farmers market', 'toiletries', 'blue and black', 'sprite', 'bank of america', 'heat', 'emergency', 'hard', '41', '6:00', 'in his hand', 'cluttered', 'grizzly', 'not', 'in hand', 'under table', 'd', 'hitting ball', 'photography', 'intersection', 'backwards', 'crocs', 'chips', 'harry potter', 'hawaii', 'half full', 'carriage', 'curious', 'geese', 'pork', 'l', 'sidecar', 'penguin', 'to see', 'pocket', 'steps', 'cubs', 'junk', 'deer', 'ottoman', 'salt', 'condiments', '1:55', 'post', 'bulldog', 'notebook', 'no cat', 'jets', 'knee pads', 'throw frisbee', 'drinks', 'leopard', 'grape', 'wine tasting', 'baskets', 'santa hat', 'chest', 'sewing', 'on car', 'sony ericsson', 'peeing', 'tour', 'fire extinguisher', 'lemons', 'wiimote', 'guitar hero', 'stopped', 'library', 'blue and pink', 'choppy', 'sailing', 'brush', 'jelly', 'dairy queen', 'shaking hands', 'ge', 'tigers', 'tokyo', 'buses', 'pink and blue', 'singles', 'iron', "don't walk", 'classroom', 'harbor', 'residential', 'joshua', 'uk', 'burgers', 'lace', 'overalls', 'ram', 'dancing', '47', 'shed', 'lid', "he's not", 'amtrak', 'ostrich', 'bathtub', '2:50', 'mall', 'slow down', 'hammer time', 'octopus', 'crib', 'broadway', 'pottery', 'wavy', 'holding phone', 'tusks', 'dining', 'packing', 'thomas', 'budweiser', 'beijing', '11:10', 'wide', 'slope', 'black and gray', 'chili', 'siblings', 'kayaking', 'captivity', 'rack', 'panda', 'pelicans', 'genetics', 'not in service', 'v', 'on laptop', 'gone', 'tying tie', 'scale', 'lily', 'cool', 'n', 'toilets', 'tree branch', 'copper', '870', 'shopping', 'batman', 'black and brown', 'legos', 'drinking water', 'burrito', 'spiral', 'ibm', 'tools', 'cherries', 'maple leaf', 'vines', 'sushi', 'baker', 'globe', 'wireless', 'compaq', 'do not enter', '1:05', 'advertisement', 'movement', 'model', 'hammock', 'swing', 'sheet', 'google', 'right 1', 'haircut', 'exit', 'tim hortons', 'lego', 'cucumbers', 'potato', 'egg salad', 'controllers', 'upside down', 'lion', 'camo', 'dirt bike', 'playing video games', 'crates', 'horizontally', 'plunger', 'radiator', 'in basket', 'cap', 'living', 'briefcase', 'ascending', 'flip phone', '101', 'gun', 'foam', 'serious', 'pancakes', 'heineken', 'driveway', 'cleaner', 'delivery', 'commuter', 'apple and banana', 'chase', 'trucks', 'trunks', '64', 'slacks', 'skiers', 'carrot cake', 'holding', 'surfers', 'horse racing', 'orchid', 'leaving', 'pitch', 'crest', 'miami', 'bus station', 'take off', 'diesel', 'pm', 'wetsuits', '7:35', 'tie dye', 'baked', 'life jacket', 'grilled cheese', 'meatballs', 'monster', 'smiley face', 'keys', 'straight ahead', 'badminton', 'end', '5:05', '10:50', 'each other', 'weeds', 'tinkerbell', 'rottweiler', 'apartments', 'sweatshirt', 'shore', 'switzerland', '65', 'jar', 'skate', 'raspberries', 'singing', 'on bus', 'carnations', 'descending', 'hsbc', 'space needle', 'skatepark', 'kenmore', 'db', "baby's breath", 'shelter', '1980', 'no left turn', '9:05', 'pipes', 'donkey', 'mitsubishi', 'tell time', 'outfield', 'flip', 'stadium', 'heinz', 'distance', 'macaroni', 'on plane', 'triumph', '4:50', 'on stove', 'shih tzu', 'fried', 'sunrise', '2nd', 'suzuki', 'traffic lights', 'hitting', 'healthy', 'tulip', 'right side', 'on sign', 'maroon', '5:40', 'michigan', 'close', 'license plate', 'sniffing', '1:15', 'cardinal', 'older', 'nest', 'colored', 'in back', 'formica', 'roundabout', 'drain', 'drying', '11:25', 'westjet', 'us air force', 'comcast', 'soon', 'futon', 'braid', 'us airways', '49', 'red velvet', 'sas', 'cosmo', '100 year party ct', 'in cabbage town'] + +class _VQA_split1(torch.utils.data.Dataset): + def __init__(self, root_dir, split, classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + + + # NOTE: so tricky + if root_dir.endswith('vv'): + root_dir = root_dir[0: -2] + self.full_label_dim = 700 + else: + self.full_label_dim = None + + self.root_dir = root_dir + self.data = read_json(os.path.join(root_dir, 'label1.json')) + + n = int(len(self.data) * 0.8) + + if split == 'train': + self.data = self.data[: n] + elif split in ('test', 'val'): + self.data = self.data[n: ] + + # logger.info(f'Loaded {len(self.data)} samples for {split} split') + + ignore_classes_idx = [classes.index(c) for c in ignore_classes] + + new_self_data = [] + for i, d in enumerate(self.data): + should_ignore = False + for label_idx in d[2]: + if label_idx in ignore_classes_idx: + should_ignore = True + break + if not should_ignore: + # full_label = [0] * (len(classes) - len(ignore_classes)) + new_labels = [idx_map[l] if idx_map is not None else l for l in d[2]] + # for l in zip(new_labels, d[3]): + # full_label[l[0]] = l[1] + + new_self_data.append( + (d[0], d[1], new_labels, d[3]) + ) + self.data = new_self_data + + logger.info(f'Loaded {len(self.data)} samples for {split} split (after ignoring some classes)') + + #self.processor = ViltProcessor.from_pretrained('dandelin/vilt-b32-mlm') + from transformers import AutoTokenizer,AutoProcessor,GitProcessor + self.processor = AutoProcessor.from_pretrained('new_impl/mm/Vis_bert/QuestionAnswering/VisBert_pretrained',model_max_length = 30) + self.classes = classes + self.ignore_classes = ignore_classes + self.idx_map = idx_map + + def __len__(self): + return len(self.data) + + def __getitem__(self, idx): + + cv2.setNumThreads(0) + + image_id, question, labels, scores = self.data[idx] + + image_path = os.path.join(self.root_dir, f'train2014/COCO_train2014_{str(image_id).zfill(12)}.jpg') + image = Image.open(image_path).convert('RGB') + image = Resize((224, 224))(image) + #encoding = self.processor(image, question, padding='max_length', return_tensors="pt") + encoding = self.processor(images = image,text = question,return_tensors = "pt",padding = "max_length") + + # label = self.processor(text="2", return_tensors="pt").input_ids + # print(label) + for k in ['input_ids', 'attention_mask', 'pixel_values']: + encoding[k] = encoding[k][0] + label1 = 0 + max = 0 + text = '1' + full_label = [0] * (len(self.classes) if self.full_label_dim is None else self.full_label_dim) + for label, score in zip(labels, scores): + if score > max: + label1 = self.processor(text = all_classes[label],return_tensors = "pt",padding = 'max_length').input_ids + max = score + text = all_classes[label] + full_label[label] = score + full_label = torch.FloatTensor(full_label) + inputs = encoding + inputs["labels"] = label1 + inputs['labels'] = inputs['labels'][0] + return encoding, full_label + #return encoding, inputs , text#这里的这个text就是文本答案 + +from data.datasets.visual_question_answering.generate_c_image.imagenet_c import corrupt + +class _VQA_split1_c(torch.utils.data.Dataset): + def __init__(self, root_dir, split, corruption_name, classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + + # NOTE: so tricky + if root_dir.endswith('vv'): + root_dir = root_dir[0: -2] + self.full_label_dim = 700 + else: + self.full_label_dim = None + + self.root_dir = root_dir + self.data = read_json(os.path.join(root_dir, 'label1.json')) + self.corruption_name = corruption_name + + n = int(len(self.data) * 0.8) + + if split == 'train': + self.data = self.data[: n] + elif split in ('test', 'val'): + self.data = self.data[n: ] + + logger.info(f'Loaded {len(self.data)} samples for {split} split') + + ignore_classes_idx = [classes.index(c) for c in ignore_classes] + + new_self_data = [] + for i, d in enumerate(self.data): + should_ignore = False + for label_idx in d[2]: + if label_idx in ignore_classes_idx: + should_ignore = True + break + if not should_ignore: + # full_label = [0] * (len(classes) - len(ignore_classes)) + new_labels = [idx_map[l] if idx_map is not None else l for l in d[2]] + # for l in zip(new_labels, d[3]): + # full_label[l[0]] = l[1] + + new_self_data.append( + (d[0], d[1], new_labels, d[3]) + ) + self.data = new_self_data + + logger.info(f'Loaded {len(self.data)} samples for {split} split (after ignoring some classes)') + + #self.processor = ViltProcessor.from_pretrained('dandelin/vilt-b32-mlm') + from transformers import AutoTokenizer,AutoProcessor + self.processor = AutoProcessor.from_pretrained('new_impl/mm/Vis_bert/QuestionAnswering/VisBert_pretrained',model_max_length = 30) + + self.classes = classes + self.ignore_classes = ignore_classes + self.idx_map = idx_map + + def __len__(self): + return len(self.data) + + def __getitem__(self, idx): + + cv2.setNumThreads(0) + + image_id, question, labels, scores = self.data[idx] + + image_path = os.path.join(self.root_dir, f'train2014/COCO_train2014_{str(image_id).zfill(12)}.jpg') + image = Image.open(image_path).convert('RGB') + image = Resize((224, 224))(image) + + # key + image = Image.fromarray(corrupt(np.array(image), severity=5, corruption_name=self.corruption_name)) + + #encoding = self.processor(image, question, padding='max_length', return_tensors="pt") + encoding = self.processor(images = image, text = question,return_tensors = "pt",padding = "max_length") + + for k in ['input_ids', 'attention_mask', 'pixel_values']: + encoding[k] = encoding[k][0] + label1 = 0 + max = 0 + text = '1' + full_label = [0] * (len(self.classes) if self.full_label_dim is None else self.full_label_dim) + for label, score in zip(labels, scores): + if score > max: + labels = self.processor(text = all_classes[label],return_tensors = "pt",padding = "max_length").input_ids + max = score + text = all_classes[label] + full_label[label] = score + full_label = torch.FloatTensor(full_label) + inputs = encoding + inputs["labels"] = label1 + inputs['labels'] = inputs['labels'][0] + return encoding, full_label + #return encoding , inputs , text + + +class _VQAv2_split1(torch.utils.data.Dataset): + def __init__(self, root_dir, split, classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + + + # NOTE: so tricky + if root_dir.endswith('vv'): + root_dir = root_dir[0: -2] + self.full_label_dim = 700 + else: + self.full_label_dim = None + + self.root_dir = root_dir + self.data = read_json(os.path.join(root_dir, 'label1.json')) + + n = int(len(self.data) * 0.8) + + if split == 'train': + self.data = self.data[: n] + elif split in ('test', 'val'): + self.data = self.data[n: ] + + # logger.info(f'Loaded {len(self.data)} samples for {split} split') + + ignore_classes_idx = [classes.index(c) for c in ignore_classes] + + new_self_data = [] + for i, d in enumerate(self.data): + should_ignore = False + for label_idx in d[2]: + if label_idx in ignore_classes_idx: + should_ignore = True + break + if not should_ignore: + # full_label = [0] * (len(classes) - len(ignore_classes)) + new_labels = [idx_map[l] if idx_map is not None else l for l in d[2]] + # for l in zip(new_labels, d[3]): + # full_label[l[0]] = l[1] + + new_self_data.append( + (d[0], d[1], new_labels, d[3]) + ) + self.data = new_self_data + + logger.info(f'Loaded {len(self.data)} samples for {split} split (after ignoring some classes)') + + self.processor = ViltProcessor.from_pretrained('dandelin/vilt-b32-mlm') + #self.processor = ViltProcessor.from_pretrained('new_impl/mm/Vis_bert/QuestionAnswering/vilt',model_max_length = 40) + self.classes = classes + self.ignore_classes = ignore_classes + self.idx_map = idx_map + + def __len__(self): + return len(self.data) + + def __getitem__(self, idx): + + cv2.setNumThreads(0) + + image_id, question, labels, scores = self.data[idx] + + image_path = os.path.join(self.root_dir, f'train2014/COCO_train2014_{str(image_id).zfill(12)}.jpg') + image = Image.open(image_path).convert('RGB') + image = Resize((224, 224))(image) + encoding = self.processor(image, question, padding='max_length', return_tensors="pt") + for k in ['input_ids', 'token_type_ids', 'attention_mask', 'pixel_values', 'pixel_mask']: + encoding[k] = encoding[k][0] + + full_label = [0] * (len(self.classes) if self.full_label_dim is None else self.full_label_dim) + for label, score in zip(labels, scores): + full_label[label] = score + full_label = torch.FloatTensor(full_label) + + return encoding, full_label + + +from data.datasets.visual_question_answering.generate_c_image.imagenet_c import corrupt + +class _VQAv2_split1_c(torch.utils.data.Dataset): + def __init__(self, root_dir, split, corruption_name, classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + + # NOTE: so tricky + if root_dir.endswith('vv'): + root_dir = root_dir[0: -2] + self.full_label_dim = 700 + else: + self.full_label_dim = None + + self.root_dir = root_dir + self.data = read_json(os.path.join(root_dir, 'label1.json')) + self.corruption_name = corruption_name + + n = int(len(self.data) * 0.8) + + if split == 'train': + self.data = self.data[: n] + elif split in ('test', 'val'): + self.data = self.data[n: ] + + logger.info(f'Loaded {len(self.data)} samples for {split} split') + + ignore_classes_idx = [classes.index(c) for c in ignore_classes] + + new_self_data = [] + for i, d in enumerate(self.data): + should_ignore = False + for label_idx in d[2]: + if label_idx in ignore_classes_idx: + should_ignore = True + break + if not should_ignore: + # full_label = [0] * (len(classes) - len(ignore_classes)) + new_labels = [idx_map[l] if idx_map is not None else l for l in d[2]] + # for l in zip(new_labels, d[3]): + # full_label[l[0]] = l[1] + + new_self_data.append( + (d[0], d[1], new_labels, d[3]) + ) + self.data = new_self_data + + logger.info(f'Loaded {len(self.data)} samples for {split} split (after ignoring some classes)') + + self.processor = ViltProcessor.from_pretrained('dandelin/vilt-b32-mlm') + #self.processor = ViltProcessor.from_pretrained('new_impl/mm/Vis_bert/QuestionAnswering/vilt',model_max_length = 40) + self.classes = classes + self.ignore_classes = ignore_classes + self.idx_map = idx_map + + def __len__(self): + return len(self.data) + + def __getitem__(self, idx): + + cv2.setNumThreads(0) + + image_id, question, labels, scores = self.data[idx] + + image_path = os.path.join(self.root_dir, f'train2014/COCO_train2014_{str(image_id).zfill(12)}.jpg') + image = Image.open(image_path).convert('RGB') + image = Resize((224, 224))(image) + + # key + image = Image.fromarray(corrupt(np.array(image), severity=5, corruption_name=self.corruption_name)) + + encoding = self.processor(image, question, padding='max_length', return_tensors="pt") + + for k in ['input_ids', 'token_type_ids', 'attention_mask', 'pixel_values', 'pixel_mask']: + encoding[k] = encoding[k][0] + + full_label = [0] * (len(self.classes) if self.full_label_dim is None else self.full_label_dim) + for label, score in zip(labels, scores): + full_label[label] = score + full_label = torch.FloatTensor(full_label) + + return encoding, full_label + + +class _VQAv2_split2(torch.utils.data.Dataset): + def __init__(self, root_dir, split, classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + + # NOTE: so tricky + if root_dir.endswith('vv'): + root_dir = root_dir[0: -2] + self.full_label_dim = 700 + else: + self.full_label_dim = None + + + self.root_dir = root_dir + self.data = read_json(os.path.join(root_dir, 'label2.json')) + + n = int(len(self.data) * 0.8) + + if split == 'train': + self.data = self.data[: n] + elif split in ('test', 'val'): + self.data = self.data[n: ] + + # logger.info(f'Loaded {len(self.data)} samples for {split} split') + + ignore_classes_idx = [classes.index(c) for c in ignore_classes] + + new_self_data = [] + for i, d in enumerate(self.data): + should_ignore = False + for label_idx in d[2]: + if label_idx in ignore_classes_idx: + should_ignore = True + break + if not should_ignore: + # print(idx_map) + # full_label = [0] * (len(classes) - len(ignore_classes)) + new_labels = [idx_map[l] if idx_map is not None else l for l in d[2]] + # for l in zip(new_labels, d[3]): + # full_label[l[0]] = l[1] + + new_self_data.append( + (d[0], d[1], new_labels, d[3]) + ) + self.data = new_self_data + + logger.info(f'Loaded {len(self.data)} samples for {split} split (after ignoring some classes)') + + self.processor = ViltProcessor.from_pretrained('dandelin/vilt-b32-mlm') + + self.classes = classes + self.ignore_classes = ignore_classes + self.idx_map = idx_map + + def __len__(self): + return len(self.data) + + def __getitem__(self, idx): + + cv2.setNumThreads(0) + + image_id, question, labels, scores = self.data[idx] + + image_path = os.path.join(self.root_dir, f'train2014/COCO_train2014_{str(image_id).zfill(12)}.jpg') + image = Image.open(image_path).convert('RGB') + image = Resize((224, 224))(image) + encoding = self.processor(image, question, padding='max_length', return_tensors="pt") + + for k in ['input_ids', 'token_type_ids', 'attention_mask', 'pixel_values', 'pixel_mask']: + encoding[k] = encoding[k][0] + + full_label = [0] * (len(self.classes) if self.full_label_dim is None else self.full_label_dim) + for label, score in zip(labels, scores): + full_label[label] = score + full_label = torch.FloatTensor(full_label) + + return encoding, full_label + + + +all_classes = ['net', 'pitcher', 'orange', 'yes', 'white', 'skiing', 'red', 'frisbee', 'brushing teeth', 'no', 'black and white', 'skateboard', '1', 'blue', 'green', 'motorcycle', 'gray', '2', 'purse', 'skis', 'poles', 'surfboard', 'dog', 'on', 'office', 'large', 'very big', 'laptop', 'vent', 'computer', 'black', 'bear', '3', 'wii', 'glasses', 'tree', 'eating', 'log', '5', 'left', 'living room', 'pink', 'right', 'railing', 'grass', 'wire', '10 years', 'knife', 'cake', 'banana', 'chef', 'vanilla', '4', 'outdoor', 'mustard', 'bun', 'clouds', 'dock', 'brown', 'silver', 'refrigerator', 'square', 'teddy', 'elm', 'stripes', 'baseball', 'catcher', 'beer', 'bottom', 'north', 'nike', 'yellow and white', 'morning', 'elephant', 'red and white', 'propeller', 'tan', 'wall', 'clock', 'table', '0', 'wood', 'christmas', 'spinach', 'thick', 'bag', 'leaves', 'necklace', '6', 'bathroom', 'shower', 'towel', 'solid', 'referee', 'wilson', 'e', '24', 'hat', 'grazing', 'sheep', '10', 'tag', 'spanish', 'hot dog', 'plate', 'lunch', 'butter', 'peppers', 'onions', 'very', 'pig', 'sweet', 'flowers', 'floral', 'yellow', 'window', '7', 'pizza', 'car', '', 'cargo', 'stairs', 'abstract', 'rug', 'baseball cap', 'texting', 'pole', 'crosswalk', 'nothing', 'urban', 'bus', 'light', 'afternoon', 'boat', 'cheese', 'paper', 'real', 'sun', 'birthday', 'words', 'inside', 'shadows', 'tomato', 'evergreen', '100 feet', 'trees', 'building', 'hay', 'ski pole', 'walking', 'ice', 'laundry', 'pepsi', 'good', '1:50', 'purple', '13', 'africa', 'teddy bears', 'socks', 'giraffe', 'soccer', 'blue and yellow', 'zebras', 'cupcake', 'broccoli', 'parking lot', 'cows', 'herding', 'on table', 'fish', 'nightstand', '50', 'overcast', 'cross', 'toaster oven', 'tile', '11:55', 'red and yellow', 'nowhere', 'hair dryer', 'truck', '11', 'people', 'rectangle', 'hot dogs', 'party', '12:55', 'apron', 'kitchen', 'cooking', 'ring', '1 way', 'stop', 'neither', 'many', 'female', 'brushing', 'tie', 'tennis racket', 'knife and fork', 'restaurant', 'cat', 'bed', 'sand', 'ocean', 'cold', 'kites', 'cumulus', 'standing', 'male', 'star', 'tracks', 'chocolate', 'round', 'fork and knife', 'yankees', 'pictures', 'dots', 'bird', 'parrot', 'red white and blue', 'man', 'metal', 'fence', 'snowboarding', 'pine', 'snow', 'shorts', 'swim', 'wine', 'brick', 'no parking', 'children', 'beef', 'phone', 'english', 'cell phone', 'pink and yellow', 'clear', 'watermelon', 'bedroom', 'fork', 'cow', 'rackets', 'tennis rackets', '8', 'collar', 'tennis', 'playing tennis', 'skirt', '30', 'polka dot', 'beach', 'horse', 'grill', 'african american', 'down', 'street', 'in air', 'sweater', 'yellow and blue', 'park', 'spectators', 'parasailing', '31', 'river', '55', 'shadow', 'winter', 'chicken', 'tea', 'evening', 'dusk', 'ski resort', 'helmet', 'bench', 'resting', 'elephants', 'southwest', 'usa', 'cars', 'town', 'bananas', 'umbrella', 'container', 'woman', 'on counter', 'salad', 'striped', 'motel', 'vertical', 'oranges', 'hot sauce', 'bottle', 'juice', 'eyes', 'ground', 'backpack', 'black and yellow', 'forward', 'jackets', '1 on right', 'green and yellow', 'playing baseball', 'riding', 'sitting', 'carrot', 'basket', 'seagull', 'ski poles', 'p', 'parking', 'street light', 'strap', 'bike', 'riding bike', 'poodle', 'shoes', 'carpet', 'lettuce', 'food', '1 foot', 'roses', 'mountains', 'scissors', 'camera', 'beige', 'beard', 'cutting', 'baby', 'tape', 'watch', 'never', 'taking picture', 'eggs', 'syrup', 'sandwich', 'water skiing', 'microphone', 'back', 'bears', 'donuts', 'w', 'sky', 'double decker', 'england', 'surfing', 'running', 'shirt', 'barn', 'weather vane', 'white and blue', 'fishing', 'bridge', 'los angeles', 'open', 'red sox', 'bat', 'plane', 'white and green', 'transportation', 'sunny', 'bus stop', 'city', 'brown and white', 'bicycle', 'crow', 'magazines', 'daisy', '14', 'old', 'curtains', 'snowboard', 'dinosaur', 'racing', 'asphalt', 'court', 'plastic', 'circle', 'red and blue', 'zebra', '12', 'biplane', 'shallow', 'brazil', 'logo', '2:20', 'electric', 'motion', 'toothbrushes', 'orange and white', '66', 'spoon', 'toyota', 'tennis shoes', '46', 'second', 'no 1', 'iphone', 'friend', 'apple', '15', 'tiger', 'glove', 'airplane', 'bow', 'air france', 'passengers', 'tv', 'on building', '3:55', 'victorian', 'steeple', 'happy', 'skateboarding', 'fruit', 'cutting board', 'cantaloupe', 'kiwi', 'sliced', 'heart', 'water', 'rainy', 'carrots', 'giraffes', 'eat', 'ramp', 'lab', 'field', 'horizontal', 'birds', 'home', 'shrimp', '12 feet', 'girl', 'modern', 'dell', 'boots', 'sunglasses', 'black and orange', 'yellow and black', 'gloves', 'hp', 'desk', 'both', 'sign', 'on street', '2000', 'cirrus', 'ceiling', 'fluorescent', 'up', '9', 'boys', 'playing soccer', 'american', 'passenger', 'turn', 'palm', 'wedding', 'branch', 'parrots', 'air force', 'on tracks', 'small', 'dirty', 'france', 'honda', '2.00', 'vase', 'flying', 'driving', 'tissue', 'protest', 'corona', 'twin', 'clothes', 't shirt', 'window sill', 'wild', 'noon', 'caution', 'spring', 'raining', 'cane', 'school', 'windsurfing', 'parachute', 'black and red', '25', 'background', 'toaster', 'planes', 'yellow and red', 'spatula', '10:10', 'ivory', 'train', 'highway', 'off', 'on track', 'electricity', 'italy', 'dinner', 'sink', 'squares', '5 ft', 'parked', 'store', 'dress', 'signs', 'football', 'rugby', 'stainless steel', 'dirt', 'blue and white', 'klm', 'house', 'unknown', 'ford', 'reading', 'chair', 'mountain', 'alive', 'water skis', 'picture', 'parade', 'trailer', 'boating', 'holding it', 'shade', 'cloth', 'candle', 'hose', 'hand', '3:25', 'on sidewalk', 'poster', 'downhill', 'reflection', 'summer', 'pickles', 'halloween', 'bats', 'london', 'zoo', 'surfer', 'racket', 'flickr', 'cutting hair', 'strawberries', 'mushroom', 'teddy bear', 'big', 'suitcase', 'veggie', 'pepper', 'houses', '70', 'toshiba', 'triangle', 'boxes', 'photograph', 'smoke', 'engine', 'camel', 'sidewalk', 'left 1', 'red and green', '4:35', 'on couch', 'candy', 'homemade', 'mouse', 'box', 'movie', '45', 'strawberry', 'fridge', 'full', 'vegetables', 'bright', 'play', 'remote', 'pond', 'savannah', 'celery', 'concrete', 'semi', 'scania', 'safety', 'posing', 'fabric', 'laying', 'couch', 'blueberries', 'handle', 'pipe', 'stick', 'steak', 'chain link', 'barbed wire', 'mozzarella', 'soda', 'fire hydrant', 'cat food', 'pepperoni', 'lot', 'licking', 'red and black', 'clay', 'tennis court', 'jumping', 'potatoes', 'toothbrush', 'kite', 'flying kite', 'broken', 'black and silver', 'lap', 'outside', '44', 'delta', 'greyhound', 'talking on phone', 'bad', 'kettle', '35', 'motorcycles', 'produce', 'steering wheel', '18', 'humans', 'coffee', 'white and brown', 'fall', 'bread', 'cherry', '4:30', 'flag', 'night', 'lamp', 'cucumber', 'porcelain', 'oval', 'museum', 'rain', 'sprinkles', '20', 'kids', 'bracelet', 'sneakers', 'mask', 'mickey mouse', 'very high', 'costume', 'cabbage', 'paint', 'lighting', 'young', 'air conditioner', 'wooden', 'board', 'beets', '16', 'lights', 'ladder', 'glass', 'fries', 'steamed', 'shepherd', 'cotton', 'suit', 'goatee', 'on his head', 'print', 'happy birthday', 'forks', 'travel', 'maple', '200', 'oil', 'jeans', 'can', 'chopsticks', 'on wall', 'construction', '36', 'chinese', 'festival', 'gas', 'throwing', 'circus', 'wires', 'not possible', 'plates', 'sugar', 'in', "women's", 'door', 'volleyball', 'serving', 'ponytail', 'business', 'decoration', 'santa', 'flat', 'barrel', '12:15', 'candles', 'free', 'hair', 'ball', 'stop sign', 'wetsuit', 'green and black', 'foreground', 'stands', 'china airlines', 'flower', '300', 'on bench', 'plaster', 'phones', 'sailboat', 'apples', 'road', 'recently', 'cones', 'cactus', 'rice', 'vegetarian', 'donut', 'ketchup', 'police', 'mirror', 'rock', 'meat', 'blinds', 'cell phones', 'china', 'rust', '7:25', 'stone', 'vans', 'middle', 'eagle', '9:30', 'ping pong', 'microwave', 'gmc', 'umbrellas', 'wrist', 'laughing', 'boy', 'next to toilet', 'tabby', 'petting', 'south', '40', 'checkered', 'slow', 'cardboard', 'windows', 'croissant', 'plain', 'cookie', 'on ground', 'low', 'water bottle', 'goggles', 'turkey', 'shut', 'kite flying', 'bowl', 'smile', 'in bowl', 'bush', 'cloudy', 'top left', 'skateboarder', 'coca cola', 'pan', 'drinking', 'short', 'floor', 'thanksgiving', 'radio', 'drink', 'on toilet', 'bike rack', 'bleachers', 'train tracks', 'horses', 'far', 'top', 'toilet', 'in water', 'private', 'nature', 'commercial', 'stroller', 'power', 'stuffed animals', 'uniforms', 'japan', 'faucet', 'green and orange', 'corn', 'white and yellow', 'mercedes', 'in sky', 'tarp', 'indian', 'counter', 'multicolored', 'polar', 'go', 'no number', 'swimming', 'bridle', 'cowboy', 'olives', 'pizza cutter', 'british airways', 'nighttime', 'australia', 'tiles', 'pug', 'wicker', 'british', 'us airways express', 'burton', 'christmas tree', 'napkin', 'writing', 'rocks', 'hello kitty', 'gold', 'fan', 'skateboards', 'day', 'on floor', '2008', 'dark', 'flying kites', 'rural', 'olympics', 'bmw', '34', 'denim', 'typing', 'for fun', 'steel', 'watching tv', 'driver', 'grapes', 'f', 'angels', 'roof', 'handlebars', 'train station', 'public', 'oak', 'sleeping', 'canada', 'air canada', 'on top', 'tired', 'blonde', 'cups', 'little', 'adidas', '10 feet', 'white and gray', 'leaf', 'fisheye', 'forest', 'war', 'octagon', 'raspberry', 'helmets', 'united states', '29', 'noodles', 'van', 'long', 'traveling', 'luggage', 'airport', 'single', 'pitching', 'dugout', 'garbage', 'happiness', 'cigarette', 'on tower', 'antelope', 'graffiti', 'skating', 'on road', 'curved', 'washington', 'ski lift', 'athletics', 'brace', 'squatting', 'catching', 'batter', 'batting', 'game', 'towards', '33', 'sliding', 'makeup', 'japanese', 'person', 'pirates', 'plaid', 'rose', 'daytime', 'keyboard', 'surfboards', 'hummingbird', 'ollie', '11:30', 'clock tower', 'san francisco', 'stopping', 'tags', 'samsung', 'computers', 'cabinets', 'talking', 'asparagus', '5 years', 'adult', 'rabbit', 'empty', 'softball', '1st', 'playing', 'chairs', 'farm', 'cross country', 'dump truck', 'women', 'snowboarder', 'tall', 'monkey', 'fire', 'books', 'cessna', 'chandelier', 'dunkin donuts', 'beans', 'relish', 'parking meter', 'ducks', 'sandals', 'doughnut', 'lighthouse', 'yacht', 'german shepherd', 'raw', 'chain', '2 feet', 'pedestal', 'mutt', 'race', 'poor', 'cat and dog', 'station', 'printer', 'daisies', 'front', 'gravel', 'grassy', 'pigeons', 'dogs', 'in car', 'life', 'wii remotes', 'suv', 'leather', 'bottom right', 'peace', 'blanket', 'frisbees', '12:30', 'scooter', 'going', 'analog', 'america', 'pitbull', 'relaxing', 'paddle boarding', 'white and pink', 'ride', 'side', 'on desk', 'on chair', '2012', 'multi', 'straight', 'big ben', 'closed', '3 feet', 'waves', 'buoy', 'trash can', 'medium', 'very tall', 'yamaha', 'sunlight', 'hit ball', 'dry', 'coke', 'gym', 'orange and black', 'center', 'rope', 'flip flops', 'siamese', 'crafts', 'color', 'italian', 'playing frisbee', 'skate park', 'orange juice', 'windowsill', 'thumb', 'pie', 'toast', 'no hat', 'benches', 'diamond', 'blender', 'avocado', 'television', 'speakers', 'pony', 'baseball field', 'pavement', 'not there', 'diamonds', '4 feet', 'goalie', 'soccer ball', 'runway', 'video game', 'gaming', 'casual', 'green and white', 'toilet brush', 'working', 'pickup', 'girls', 'remotes', 'pasta', 'hood', 'braves', 'skier', 'motorola', '17', 'b', '100', 'hospital', 'wagon', 'milk', 'ferry', 'rainbow', 'on bed', 'toward', '1:30', '19', 'mercedes benz', 'supreme', 'thin', 'platform', 'thai', 'storage', 'swan', 'peach', '10:05', 'dome', 'chiquita', '2:00', 'mountain dew', '23', 'knives', 'street sign', 'on beach', 'playing wii', 'stickers', 'yogurt', 'on grass', '9:45', 'gatorade', 'umpire', '37', 'desktop', 'desserts', 'main', 'boston', 'fell', 'top right', 'case', 'asleep', 'over', 'grapefruit', 'breakfast', 'headphones', 'freight', 'cup', 'sweatband', 'nobody', 'lamps', '9:25', 'scarf', 'on fridge', 'moving', 'fresh', 'blue jay', 'chihuahua', 'ceramic', 'mushrooms', 'on plate', 'human', 'power lines', 'hotel', 'map', 'earring', 'boarding', 'warm', 'napkins', 'brown and black', 'broom', 'basketball', 'papers', 'sad', 'kickstand', '60', 'shoulder', 'sleep', 'footprints', 'tunnel', '1990', 'hats', '6 inches', 'ham', 'bacon', 'church', '53', 'pineapple', 'at camera', 'red bull', 'pilot', 'tattoo', 'work', 'polar bear', 'taking off', 'website', '22', '4:00', 'coffee maker', 'fast', 'fur', 'rubber', 'tongs', 'german', 'germany', 'toy', '3:20', 'calm', 'pots', 'fruits', '9:20', 'drawer', 'oven', 'soup', 'stove', 'heels', 'wind', 'island', 'blood', 'leg', 'theater', 'tennis racquet', '21', 'gothic', '2:35', 'wii remote', 'turning', '20 feet', 'ears', 'fun', 'to right', 'child', 'fly', 'head', 'drywall', 'pier', 'feeding giraffe', 'in vase', 'burger', 'easter', 'onion', 'uniform', 'guitar', 'time', 'tomatoes', 'ship', 'tulips', 'glaze', 'tent', 'market', 'bandana', 'still', "don't know", 'piano', 'mouth', 'run', 'sparrow', 'lines', 'vest', '1950', 'jet', 'sepia', '2015', 'busy', 'dessert', '75', 'finch', 'pastries', 'outdoors', 'bakery', 'clean', 'ipod', 'tablecloth', 'looking at phone', 'in front', 'food truck', 'face', 'swinging', 'safari', '500', 'volkswagen', '2010', 'shelves', 'riding horses', '2016', 'towels', 'lemon', 'straw', 'bamboo', '5 feet', 'hardwood', 'h', 'meter', 'charging', 'bald', 'caucasian', 'man on left', 'stand', '27', 'dining room', 'sandwiches', '32', 'apartment', 'tower', 'virgin', 'out', 'white and red', "i don't know", 'chains', 'legs', 'goats', 's', 'dresser', 'camper', 'half', 'decorative', 'hawaiian', 'wheel', 'florida', 'reds', 'washington dc', 'moon', 'conference', 'screen', 'controller', 'robin', 'men', 'protection', 'harley davidson', 'coal', 'mustache', 'smiling', 'pedestrians', 'me', 'tray', 'monitor', 'bell', 'landscape', 'club', 'toothpick', 'seagulls', 'bowtie', 'lake', 'steam', 'surf', 'baseball glove', 'blinders', 'woods', 'shearing', 'dad', 'mixer', 'pot', 'blending', 'identification', 'owl', 'wine glass', 'new york', 'yarn', 'tennis ball', 'ice cream', 'chevrolet', 'shirt and tie', 'taking selfie', 'blue and green', "he isn't", 'cutting cake', 'east', 'setting', '7 eleven', 'stars', 'jockey', 'jacket', 'book', 'gray and white', 'pen', 'red white blue', 'above', 'alaska', 'tongue', 'feathers', 'k', 'camping', 'corner', 'away', 'ski', 'texas', 'fire truck', 'sailboats', 'jump', 'walk', 'spray paint', 'loading', 'united', '1000', 'roman numerals', 'surprise', '3rd', 'first', 'side of road', 'dodgers', 'airplanes', 'unsure', 'russian', 'wet', '5 star', 'blankets', 'natural', 'across street', 'smartphone', 'duck', 'sausage', 'paris', 'newspaper', 'pants', 'spices', 'pillow', 'to left', 'snowboards', 'colgate', 'on elephant', 'string', 'horns', '2:40', "men's", 'cobblestone', 'regular', 'staring', '28', 'barber shop', 'cut', 'x', 'above sink', 'above stove', 'dishes', 'dalmatian', 'watching', 'glazed', '5:25', 'messy', 'wallet', 'tuna', 'grilled', 'french', 'green and blue', 'sunflowers', 'wool', 'cabinet', 'shell', 'foil', 'bottles', 'bar', 'king', 'paper towels', 'friends', 'beagle', 'school bus', 'laptops', 'snowing', 'cement', 'pc', 'accident', 'stuffed animal', 'balance', 'white and black', 'cleats', 'on sink', 'pool', 'mom', 'downtown', 'asian', 'heater', 'bathing', '193', 'against wall', 'canopy', 'berries', 'military', 'pickle', 'clams', 'seafood', 'in box', 'boats', 'lizard', 'lemonade', 'm', 'soft', 'country', 'for sale', 'arm', 'listening', 'curly', 'play tennis', 'hands', 'cereal', 'blue and red', 'robe', 'soap', 'trains', 'throwing frisbee', 'smoking', 'india', 'headband', 'not very', 'westin', 'serve', 'bicycles', "can't tell", 'visibility', 'ana', 'reins', 'rodeo', 'riding motorcycle', 'mexico', 'mother', 'african', 'left and right', 'button', 'earrings', 'blackberry', 'cell', '10:00', 'harness', 'pillows', 'vegetable', 'tablet', 'fern', 'cats', 'golden retriever', 'goat', 'tractor', "valentine's day", 'hearts', 'khaki', 'man on right', "mcdonald's", 'arriving', 'husky', 'on skateboard', 'vases', 'coat', 'beanie', 'coming', 'granite', 'sports', 'leash', 'balls', 'blurry', 'baseball bat', 'mug', 'eiffel tower', 'worms', 'trash', 'terrier', 'painting', 'rooster', '42', 'jones', 'state farm', 'balloon', 'trunk', 'coach', 't', 'playing game', 'fireplace', 'behind clouds', 'uphill', 'motocross', 'sony', 'magazine', 'kitesurfing', 'catching frisbee', 'catch frisbee', 'bud light', 'fighting', '1 on left', 'very old', 'hallway', 'lexus', 'wii controller', '5:45', 'catholic', 'muffin', 'traffic light', 'grocery', 'shelf', '2:25', 'honey', 'plants', 'oars', 'foggy', "nathan's", 'cord', 'yard', '48', 'chimney', 'calico', 'suits', 'sideways', 'animals', 'black and blue', 'bikini', 'photographer', 'queen', '1:00', '12:05', 'horseback riding', 'awake', 'bunny', '12:00', 'continental', 'rye', 'family', 'lots', 'owner', 'palm tree', 'design', 'far right', 'tire', 'younger', 'biking', 'giants', 'caramel', 'polo', 'emirates', 'magnets', 'mat', 'ivy', 'cakes', 'bob', 'asia', 'graduation', 'cauliflower', 'c', 'rough', 'air', 'windy', 'victoria', 'trick', 'labrador', 'on left', 'yellow and green', 'butterfly', 'fake', 'on napkin', 'bricks', 'wine glasses', 'detroit', "man's", 'parsley', 'art', 'subway', 'wave', 'placemat', 'hydrant', 'sofa', 'pigeon', 'all', 'branches', 'plant', 'to eat', 'zucchini', 'feta', 'mouse pad', 'cloud', 'toilet paper', 'pumpkin', 'rowing', 'handicap', 'seeds', 'fly kite', 'chicago', 'marble', 'frame', '150', 'rocky', 'sauce', "it's not", 'control', 'high chair', 'playstation', 'xbox', 'roman', 'land', '1:35', 'lifeguard', 'size', 'bull', 'goose', '8 feet', 'recessed', 'statue', 'index', 'phillies', 'strike', 'mirrors', 'pointing', 'farmer', 'collie', 'motorbike', 'lanes', 'bikes', 'gas station', 'logs', 'smaller', 'desert', 'yield', 'flags', 'stool', 'kitten', 'doll', 'daffodils', 'letters', 'dishwasher', 'nuts', '2013', 'persian', 'swim trunks', 'deep', 'doubles', 'in field', 'wristband', 'wheels', 'baking', '4:15', '11:00', 'ear', '2007', '51', 'frog', 'boogie board', 'hungry', 'by window', 'ambulance', 'pigtails', 'microsoft', 'on man', 'laying down', '3:00', 'taxi', 'pedestrian', 'landing', 'numbers', '38', 'stones', 'clocks', 'new', 'picnic', 'fog', 'buffalo', 'under armour', 'orioles', 'bags', 'golden gate', 'castle', 'canoe', 'selfie', 'cream', 'floating', 'indoor', 'antique', 'aluminum', 'peas', 'sun hat', 'on right', 'flour', 'under sink', 'fashion', 'fedora', 'shells', '1 hour', 'puppy', 'motor', '120', 'sail', 'mexican', 'dead end', 'paddle', 'shop', 'boxing', 'birthday cake', 'chalk', 'style', 'nissan', 'sticker', 'north face', 'squash', 'not sure', 'seat', 'himself', 'circles', 'san diego', 'kia', 'mattress', 'obama', 'lamb', 'american flag', 'climbing', 'skull and crossbones', 'roast beef', 'visor', 'double', '52', 'high', 'stagecoach', 'cart', 'feeding', 'eaten', 'cone', 'smoothie', 'golf', 'colorado', 'electronics', '5:15', 'bowling', 'players', 'ketchup and mustard', 'styrofoam', '6 feet', 'hawk', 'cheddar', 'arabic', 'shower curtain', 'army', 'salmon', 'hanging', 'whole', 'behind fence', 'bars', 'moss', 'no dog', 'traffic', 'r', 'countryside', 'directions', 'cooked', 'aa', '6:45', '4 way', 'stripe', 'brand', 'baseball player', 'bunk', 'coleslaw', 'europe', 'dead', 'arch', 'scrambled', 'clothing', 'closet', 'egg', 'suitcases', 'indoors', 'tires', 'lilies', 'cafe', 'toothpaste', 'in background', 'tarmac', 'painted', 'sunset', 'orange and yellow', 'zebra and giraffe', 'ladybug', 'hills', 'tail', 'couple', 'kawasaki', 'smooth', 'powdered sugar', 'pedestrian crossing', 'french fries', 'teeth', 'ribbon', 'saddle', 'on train', '39', 'curb', 'tow', 'shark', 'white and orange', 'gravy', 'curtain', 'lime', 'skull', 'crossing', 'peacock', 'neck', 'hit', 'dragon', 'tissues', 'basil', 'waving', 'helicopter', 'mud', 'us', 'red and gray', 'sunflower', 'wallpaper', '11:20', 'seattle', 'bookshelf', 'looking', '1 inch', 'harley', 'urinal', 'navy', 'fedex', 'rays', 'deck', 'coaster', '1:20', '4:20', '5:00', 'jp morgan', 'palm trees', 'tub', 'pens', '2 people', 'speaker', 'hamburger', 'green beans', "it isn't", '10:20', 'buildings', 'on shelf', 'orange and blue', '90', 'north america', 'arrow', 'news', 'tropicana', 'formal', 'in grass', 'thumbs up', 'clip', 'tennis player', 'pastry', 'nose', 'pacifier', '11:35', 'different teams', 'cardinals', 'bagel', 'huge', 'out of focus', 'cook', 'wheat', 'photo', 'sedan', 'lanyard', 'pink and white', 'sesame', 'space', 'warning', 'snowy', 'tater tots', 'tropical', 'grandfather', 'mac', 'pajamas', '350', 'casserole', 'pelican', '2009', 'clydesdale', 'tow truck', 'belt', 'west', 'omelet', 'heavy', 'crown', 'in corner', 'hexagon', 'mound', 'iris', 'g', '2:15', '3:10', 'drawing', 'only', 'washing', 'nokia', 'windsor', 'icing', 'several', 'no smoking', 'kayak', 'frosting', 'jetblue', 'shoe', 'britain', 'ties', 'bank', 'camouflage', 'privacy', 'bib', 'blue and gray', 'looking out window', 'falling', 'bucket', 'cupcakes', 'throw ball', 'garden', 'almonds', 'starbucks', 'all way', 'home plate', 'base', 'toys', '1 in front', 'foot', 'california', 'towing', 'cheesecake', 'bushes', 'bow tie', 'down street', '2011', 'police officer', 'windmill', 'taking pictures', 'cleaning', 'on pole', 'main street', 'catch ball', 'mario', 'track', 'garage', "they aren't", 'tents', 'tattoos', '2:45', 'wheelchair', 'money', 'top hat', 'willow', 'brushing hair', '80', 'green and red', 'barrier', 'hiking', 'tank top', 'lufthansa', 'menu', 'forehand', 'wii controllers', 'hundreds', 'water ski', 'furniture', 'paisley', 'pizza hut', 'hill', 'prom', 'tiara', 'students', 'information', 'hazy', 'canon', 'bird feeder', 'crane', 'dr pepper', 'logitech', '2:10', 'all of them', 'utensils', 'telephone', 'converse', 'bone', 'jeep', 'nursing', 'krispy kreme', 'ranch', 'polka dots', 'railroad crossing', 'shirts', 'feeder', 'above toilet', 'unclear', 'below', '43', 'spoons', 'calendar', 'mint', 'spiderman', 'lg', 'concert', 'coats', 'lady', 'dodge', 'flat screen', '10:30', 'music', 'polar bears', 'riding horse', 'cookies', 'hot', 'behind', 'dole', '26', 'pans', 'love', 'winnie pooh', 'copyright', '2 hours', 'snowsuit', 'kissing', 'backhand', 'swans', 'nintendo', 'direction', 'waiting', 'mohawk', 'rail', 'hoodie', 'feet', '106', '10:55', 'coins', 'mitt', 'room', 'adults', 'cameras', 'marker', 'sled', 'conductor', 'farmers market', 'toiletries', 'blue and black', 'sprite', 'bank of america', 'heat', 'emergency', 'hard', '41', '6:00', 'in his hand', 'cluttered', 'grizzly', 'not', 'in hand', 'under table', 'd', 'hitting ball', 'photography', 'intersection', 'backwards', 'crocs', 'chips', 'harry potter', 'hawaii', 'half full', 'carriage', 'curious', 'geese', 'pork', 'l', 'sidecar', 'penguin', 'to see', 'pocket', 'steps', 'cubs', 'junk', 'deer', 'ottoman', 'salt', 'condiments', '1:55', 'post', 'bulldog', 'notebook', 'no cat', 'jets', 'knee pads', 'throw frisbee', 'drinks', 'leopard', 'grape', 'wine tasting', 'baskets', 'santa hat', 'chest', 'sewing', 'on car', 'sony ericsson', 'peeing', 'tour', 'fire extinguisher', 'lemons', 'wiimote', 'guitar hero', 'stopped', 'library', 'blue and pink', 'choppy', 'sailing', 'brush', 'jelly', 'dairy queen', 'shaking hands', 'ge', 'tigers', 'tokyo', 'buses', 'pink and blue', 'singles', 'iron', "don't walk", 'classroom', 'harbor', 'residential', 'joshua', 'uk', 'burgers', 'lace', 'overalls', 'ram', 'dancing', '47', 'shed', 'lid', "he's not", 'amtrak', 'ostrich', 'bathtub', '2:50', 'mall', 'slow down', 'hammer time', 'octopus', 'crib', 'broadway', 'pottery', 'wavy', 'holding phone', 'tusks', 'dining', 'packing', 'thomas', 'budweiser', 'beijing', '11:10', 'wide', 'slope', 'black and gray', 'chili', 'siblings', 'kayaking', 'captivity', 'rack', 'panda', 'pelicans', 'genetics', 'not in service', 'v', 'on laptop', 'gone', 'tying tie', 'scale', 'lily', 'cool', 'n', 'toilets', 'tree branch', 'copper', '870', 'shopping', 'batman', 'black and brown', 'legos', 'drinking water', 'burrito', 'spiral', 'ibm', 'tools', 'cherries', 'maple leaf', 'vines', 'sushi', 'baker', 'globe', 'wireless', 'compaq', 'do not enter', '1:05', 'advertisement', 'movement', 'model', 'hammock', 'swing', 'sheet', 'google', 'right 1', 'haircut', 'exit', 'tim hortons', 'lego', 'cucumbers', 'potato', 'egg salad', 'controllers', 'upside down', 'lion', 'camo', 'dirt bike', 'playing video games', 'crates', 'horizontally', 'plunger', 'radiator', 'in basket', 'cap', 'living', 'briefcase', 'ascending', 'flip phone', '101', 'gun', 'foam', 'serious', 'pancakes', 'heineken', 'driveway', 'cleaner', 'delivery', 'commuter', 'apple and banana', 'chase', 'trucks', 'trunks', '64', 'slacks', 'skiers', 'carrot cake', 'holding', 'surfers', 'horse racing', 'orchid', 'leaving', 'pitch', 'crest', 'miami', 'bus station', 'take off', 'diesel', 'pm', 'wetsuits', '7:35', 'tie dye', 'baked', 'life jacket', 'grilled cheese', 'meatballs', 'monster', 'smiley face', 'keys', 'straight ahead', 'badminton', 'end', '5:05', '10:50', 'each other', 'weeds', 'tinkerbell', 'rottweiler', 'apartments', 'sweatshirt', 'shore', 'switzerland', '65', 'jar', 'skate', 'raspberries', 'singing', 'on bus', 'carnations', 'descending', 'hsbc', 'space needle', 'skatepark', 'kenmore', 'db', "baby's breath", 'shelter', '1980', 'no left turn', '9:05', 'pipes', 'donkey', 'mitsubishi', 'tell time', 'outfield', 'flip', 'stadium', 'heinz', 'distance', 'macaroni', 'on plane', 'triumph', '4:50', 'on stove', 'shih tzu', 'fried', 'sunrise', '2nd', 'suzuki', 'traffic lights', 'hitting', 'healthy', 'tulip', 'right side', 'on sign', 'maroon', '5:40', 'michigan', 'close', 'license plate', 'sniffing', '1:15', 'cardinal', 'older', 'nest', 'colored', 'in back', 'formica', 'roundabout', 'drain', 'drying', '11:25', 'westjet', 'us air force', 'comcast', 'soon', 'futon', 'braid', 'us airways', '49', 'red velvet', 'sas', 'cosmo', '100 year party ct', 'in cabbage town'] + +@dataset_register( + name='VQA_split1', + classes=all_classes[0: 100], + task_type='Visual Question Answering', + object_type='Generic Object', + class_aliases=[], + shift_type=None +) +class VQAv2_split1(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform: Optional[Compose], + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + if transform is None: + transform = None + self.transform = transform + dataset = _VQA_split1(root_dir, split, classes, ignore_classes, idx_map) + return dataset + +@dataset_register( + name='VQA_split1_c', + classes=all_classes[0: 100], + task_type='Visual Question Answering', + object_type='Generic Object', + class_aliases=[], + shift_type=None +) +class VQAv2_split1_c(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform: Optional[Compose], + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + + if transform is None: + transform = None + self.transform = transform + dataset = _VQA_split1_c(root_dir, split, "gaussian_noise",classes, ignore_classes, idx_map) + return dataset + + +@dataset_register( + name='VQAv2_split1', + classes=all_classes[0: 100], + task_type='Visual Question Answering', + object_type='Generic Object', + class_aliases=[], + shift_type=None +) +class VQAv2_split1(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform: Optional[Compose], + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + if transform is None: + transform = None + self.transform = transform + dataset = _VQAv2_split1(root_dir, split, classes, ignore_classes, idx_map) + return dataset + + +@dataset_register( + name='VQAv2_split1_c', + classes=all_classes[0: 100], + task_type='Visual Question Answering', + object_type='Generic Object', + class_aliases=[], + shift_type=None +) +class VQAv2_split1_c(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform: Optional[Compose], + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + + if transform is None: + transform = None + self.transform = transform + dataset = _VQAv2_split1_c(root_dir, split, classes, ignore_classes, idx_map) + return dataset + + +@dataset_register( + name='VQAv2_split2', + classes=all_classes[100: ], + task_type='Visual Question Answering', + object_type='Generic Object', + class_aliases=[], + shift_type=None +) +class VQAv2_split2(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform: Optional[Compose], + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + + print(len(all_classes), len(ignore_classes)) + + if transform is None: + transform = None + self.transform = transform + dataset = _VQAv2_split2(root_dir, split, classes, ignore_classes, idx_map) + return dataset + + + + +@dataset_register( + name='VQAv2_split1_c_gaussian_noise', + classes=all_classes[0: 100], + task_type='Visual Question Answering', + object_type='Generic Object', + class_aliases=[], + shift_type=None +) +class VQAv2_split1_c_gaussian_noise(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform: Optional[Compose], + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + + if transform is None: + transform = None + self.transform = transform + dataset = _VQAv2_split1_c(root_dir, split, "gaussian_noise", classes, ignore_classes, idx_map) + return dataset + + + +@dataset_register( + name='VQAv2_split1_c_shot_noise', + classes=all_classes[0: 100], + task_type='Visual Question Answering', + object_type='Generic Object', + class_aliases=[], + shift_type=None +) +class VQAv2_split1_c_shot_noise(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform: Optional[Compose], + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + + if transform is None: + transform = None + self.transform = transform + dataset = _VQAv2_split1_c(root_dir, split, "shot_noise", classes, ignore_classes, idx_map) + return dataset + + + +@dataset_register( + name='VQAv2_split1_c_impulse_noise', + classes=all_classes[0: 100], + task_type='Visual Question Answering', + object_type='Generic Object', + class_aliases=[], + shift_type=None +) +class VQAv2_split1_c_impulse_noise(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform: Optional[Compose], + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + + if transform is None: + transform = None + self.transform = transform + dataset = _VQAv2_split1_c(root_dir, split, "impulse_noise", classes, ignore_classes, idx_map) + return dataset + + + +@dataset_register( + name='VQAv2_split1_c_defocus_blur', + classes=all_classes[0: 100], + task_type='Visual Question Answering', + object_type='Generic Object', + class_aliases=[], + shift_type=None +) +class VQAv2_split1_c_defocus_blur(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform: Optional[Compose], + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + + if transform is None: + transform = None + self.transform = transform + dataset = _VQAv2_split1_c(root_dir, split, "defocus_blur", classes, ignore_classes, idx_map) + return dataset + + + +@dataset_register( + name='VQAv2_split1_c_glass_blur', + classes=all_classes[0: 100], + task_type='Visual Question Answering', + object_type='Generic Object', + class_aliases=[], + shift_type=None +) +class VQAv2_split1_c_glass_blur(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform: Optional[Compose], + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + + if transform is None: + transform = None + self.transform = transform + dataset = _VQAv2_split1_c(root_dir, split, "glass_blur", classes, ignore_classes, idx_map) + return dataset + + + +@dataset_register( + name='VQAv2_split1_c_motion_blur', + classes=all_classes[0: 100], + task_type='Visual Question Answering', + object_type='Generic Object', + class_aliases=[], + shift_type=None +) +class VQAv2_split1_c_motion_blur(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform: Optional[Compose], + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + + if transform is None: + transform = None + self.transform = transform + dataset = _VQAv2_split1_c(root_dir, split, "motion_blur", classes, ignore_classes, idx_map) + return dataset + + + +@dataset_register( + name='VQAv2_split1_c_zoom_blur', + classes=all_classes[0: 100], + task_type='Visual Question Answering', + object_type='Generic Object', + class_aliases=[], + shift_type=None +) +class VQAv2_split1_c_zoom_blur(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform: Optional[Compose], + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + + if transform is None: + transform = None + self.transform = transform + dataset = _VQAv2_split1_c(root_dir, split, "zoom_blur", classes, ignore_classes, idx_map) + return dataset + + + +@dataset_register( + name='VQAv2_split1_c_snow', + classes=all_classes[0: 100], + task_type='Visual Question Answering', + object_type='Generic Object', + class_aliases=[], + shift_type=None +) +class VQAv2_split1_c_snow(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform: Optional[Compose], + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + + if transform is None: + transform = None + self.transform = transform + dataset = _VQAv2_split1_c(root_dir, split, "snow", classes, ignore_classes, idx_map) + return dataset + + + +@dataset_register( + name='VQAv2_split1_c_frost', + classes=all_classes[0: 100], + task_type='Visual Question Answering', + object_type='Generic Object', + class_aliases=[], + shift_type=None +) +class VQAv2_split1_c_frost(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform: Optional[Compose], + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + + if transform is None: + transform = None + self.transform = transform + dataset = _VQAv2_split1_c(root_dir, split, "frost", classes, ignore_classes, idx_map) + return dataset + + + +@dataset_register( + name='VQAv2_split1_c_fog', + classes=all_classes[0: 100], + task_type='Visual Question Answering', + object_type='Generic Object', + class_aliases=[], + shift_type=None +) +class VQAv2_split1_c_fog(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform: Optional[Compose], + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + + if transform is None: + transform = None + self.transform = transform + dataset = _VQAv2_split1_c(root_dir, split, "fog", classes, ignore_classes, idx_map) + return dataset + + + +@dataset_register( + name='VQAv2_split1_c_brightness', + classes=all_classes[0: 100], + task_type='Visual Question Answering', + object_type='Generic Object', + class_aliases=[], + shift_type=None +) +class VQAv2_split1_c_brightness(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform: Optional[Compose], + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + + if transform is None: + transform = None + self.transform = transform + dataset = _VQAv2_split1_c(root_dir, split, "brightness", classes, ignore_classes, idx_map) + return dataset + + + +@dataset_register( + name='VQAv2_split1_c_contrast', + classes=all_classes[0: 100], + task_type='Visual Question Answering', + object_type='Generic Object', + class_aliases=[], + shift_type=None +) +class VQAv2_split1_c_contrast(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform: Optional[Compose], + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + + if transform is None: + transform = None + self.transform = transform + dataset = _VQAv2_split1_c(root_dir, split, "contrast", classes, ignore_classes, idx_map) + return dataset + + + +@dataset_register( + name='VQAv2_split1_c_elastic_transform', + classes=all_classes[0: 100], + task_type='Visual Question Answering', + object_type='Generic Object', + class_aliases=[], + shift_type=None +) +class VQAv2_split1_c_elastic_transform(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform: Optional[Compose], + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + + if transform is None: + transform = None + self.transform = transform + dataset = _VQAv2_split1_c(root_dir, split, "elastic_transform", classes, ignore_classes, idx_map) + return dataset + + + +@dataset_register( + name='VQAv2_split1_c_pixelate', + classes=all_classes[0: 100], + task_type='Visual Question Answering', + object_type='Generic Object', + class_aliases=[], + shift_type=None +) +class VQAv2_split1_c_pixelate(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform: Optional[Compose], + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + + if transform is None: + transform = None + self.transform = transform + dataset = _VQAv2_split1_c(root_dir, split, "pixelate", classes, ignore_classes, idx_map) + return dataset + + + +@dataset_register( + name='VQAv2_split1_c_jpeg_compression', + classes=all_classes[0: 100], + task_type='Visual Question Answering', + object_type='Generic Object', + class_aliases=[], + shift_type=None +) +class VQAv2_split1_c_jpeg_compression(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform: Optional[Compose], + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + + if transform is None: + transform = None + self.transform = transform + dataset = _VQAv2_split1_c(root_dir, split, "jpeg_compression", classes, ignore_classes, idx_map) + return dataset + + + +@dataset_register( + name='VQAv2_split1_c_speckle_noise', + classes=all_classes[0: 100], + task_type='Visual Question Answering', + object_type='Generic Object', + class_aliases=[], + shift_type=None +) +class VQAv2_split1_c_speckle_noise(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform: Optional[Compose], + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + + if transform is None: + transform = None + self.transform = transform + dataset = _VQAv2_split1_c(root_dir, split, "speckle_noise", classes, ignore_classes, idx_map) + return dataset + + + +@dataset_register( + name='VQAv2_split1_c_gaussian_blur', + classes=all_classes[0: 100], + task_type='Visual Question Answering', + object_type='Generic Object', + class_aliases=[], + shift_type=None +) +class VQAv2_split1_c_gaussian_blur(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform: Optional[Compose], + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + + if transform is None: + transform = None + self.transform = transform + dataset = _VQAv2_split1_c(root_dir, split, "gaussian_blur", classes, ignore_classes, idx_map) + return dataset + + + +@dataset_register( + name='VQAv2_split1_c_spatter', + classes=all_classes[0: 100], + task_type='Visual Question Answering', + object_type='Generic Object', + class_aliases=[], + shift_type=None +) +class VQAv2_split1_c_spatter(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform: Optional[Compose], + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + + if transform is None: + transform = None + self.transform = transform + dataset = _VQAv2_split1_c(root_dir, split, "spatter", classes, ignore_classes, idx_map) + return dataset + + + +@dataset_register( + name='VQAv2_split1_c_saturate', + classes=all_classes[0: 100], + task_type='Visual Question Answering', + object_type='Generic Object', + class_aliases=[], + shift_type=None +) +class VQAv2_split1_c_saturate(ABDataset): + def create_dataset(self, root_dir: str, split: str, transform: Optional[Compose], + classes: List[str], ignore_classes: List[str], idx_map: Optional[Dict[int, int]]): + + if transform is None: + transform = None + self.transform = transform + dataset = _VQAv2_split1_c(root_dir, split, "saturate", classes, ignore_classes, idx_map) + return dataset + + diff --git a/data/datasets/visual_question_answering/write_vqa.py b/data/datasets/visual_question_answering/write_vqa.py new file mode 100644 index 0000000000000000000000000000000000000000..3cc3699c3e600ff733c4e6becbb9b9a84003afc0 --- /dev/null +++ b/data/datasets/visual_question_answering/write_vqa.py @@ -0,0 +1,205 @@ +import json +import pandas as pd +import pyarrow as pa +import random +import os + +from tqdm import tqdm +from glob import glob +from collections import defaultdict, Counter +from .glossary import normalize_word + + +def get_score(occurences): + if occurences == 0: + return 0.0 + elif occurences == 1: + return 0.3 + elif occurences == 2: + return 0.6 + elif occurences == 3: + return 0.9 + else: + return 1.0 + + +def path2rest(path, split, annotations, label2ans): + iid = int(path.split("/")[-1].split("_")[-1][:-4]) + + with open(path, "rb") as fp: + binary = fp.read() + + _annot = annotations[split][iid] + _annot = list(_annot.items()) + qids, qas = [a[0] for a in _annot], [a[1] for a in _annot] + questions = [qa[0] for qa in qas] + answers = [qa[1] for qa in qas] if "test" not in split else list(list()) + answer_labels = ( + [a["labels"] for a in answers] if "test" not in split else list(list()) + ) + answer_scores = ( + [a["scores"] for a in answers] if "test" not in split else list(list()) + ) + answers = ( + [[label2ans[l] for l in al] for al in answer_labels] + if "test" not in split + else list(list()) + ) + + return [binary, questions, answers, answer_labels, answer_scores, iid, qids, split] + + +def make_arrow(root, dataset_root): + with open(f"{root}/v2_OpenEnded_mscoco_train2014_questions.json", "r") as fp: + questions_train2014 = json.load(fp)["questions"] + with open(f"{root}/v2_OpenEnded_mscoco_val2014_questions.json", "r") as fp: + questions_val2014 = json.load(fp)["questions"] + with open(f"{root}/v2_OpenEnded_mscoco_test2015_questions.json", "r") as fp: + questions_test2015 = json.load(fp)["questions"] + with open(f"{root}/v2_OpenEnded_mscoco_test-dev2015_questions.json", "r") as fp: + questions_test_dev2015 = json.load(fp)["questions"] + + with open(f"{root}/v2_mscoco_train2014_annotations.json", "r") as fp: + annotations_train2014 = json.load(fp)["annotations"] + with open(f"{root}/v2_mscoco_val2014_annotations.json", "r") as fp: + annotations_val2014 = json.load(fp)["annotations"] + + annotations = dict() + + for split, questions in zip( + ["train", "val", "test", "test-dev"], + [ + questions_train2014, + questions_val2014, + questions_test2015, + questions_test_dev2015, + ], + ): + _annot = defaultdict(dict) + for q in tqdm(questions): + _annot[q["image_id"]][q["question_id"]] = [q["question"]] + + annotations[split] = _annot + + all_major_answers = list() + + for split, annots in zip( + ["train", "val"], [annotations_train2014, annotations_val2014], + ): + _annot = annotations[split] + for q in tqdm(annots): + all_major_answers.append(q["multiple_choice_answer"]) + + all_major_answers = [normalize_word(word) for word in tqdm(all_major_answers)] + counter = {k: v for k, v in Counter(all_major_answers).items() if v >= 9} + ans2label = {k: i for i, k in enumerate(counter.keys())} + label2ans = list(counter.keys()) + + for split, annots in zip( + ["train", "val"], [annotations_train2014, annotations_val2014], + ): + _annot = annotations[split] + for q in tqdm(annots): + answers = q["answers"] + answer_count = {} + for answer in answers: + answer_ = answer["answer"] + answer_count[answer_] = answer_count.get(answer_, 0) + 1 + + labels = [] + scores = [] + for answer in answer_count: + if answer not in ans2label: + continue + labels.append(ans2label[answer]) + score = get_score(answer_count[answer]) + scores.append(score) + + _annot[q["image_id"]][q["question_id"]].append( + {"labels": labels, "scores": scores,} + ) + + for split in ["train", "val"]: + filtered_annot = dict() + for ik, iv in annotations[split].items(): + new_q = dict() + for qk, qv in iv.items(): + if len(qv[1]["labels"]) != 0: + new_q[qk] = qv + if len(new_q) != 0: + filtered_annot[ik] = new_q + annotations[split] = filtered_annot + + for split in [ + "train", + "val", + "test", + "test-dev", + ]: + annot = annotations[split] + split_name = { + "train": "train2014", + "val": "val2014", + "test": "test2015", + "test-dev": "test2015", + }[split] + paths = list(glob(f"{root}/{split_name}/*.jpg")) + random.shuffle(paths) + annot_paths = [ + path + for path in paths + if int(path.split("/")[-1].split("_")[-1][:-4]) in annot + ] + + if len(paths) == len(annot_paths): + print("all images have caption annotations") + else: + print("not all images have caption annotations") + print( + len(paths), len(annot_paths), len(annot), + ) + + bs = [ + path2rest(path, split, annotations, label2ans) for path in tqdm(annot_paths) + ] + + dataframe = pd.DataFrame( + bs, + columns=[ + "image", + "questions", + "answers", + "answer_labels", + "answer_scores", + "image_id", + "question_id", + "split", + ], + ) + + table = pa.Table.from_pandas(dataframe) + + os.makedirs(dataset_root, exist_ok=True) + with pa.OSFile(f"{dataset_root}/vqav2_{split}.arrow", "wb") as sink: + with pa.RecordBatchFileWriter(sink, table.schema) as writer: + writer.write_table(table) + + table = pa.ipc.RecordBatchFileReader( + pa.memory_map(f"{dataset_root}/vqav2_val.arrow", "r") + ).read_all() + + pdtable = table.to_pandas() + + df1 = pdtable[:-1000] + df2 = pdtable[-1000:] + + df1 = pa.Table.from_pandas(df1) + df2 = pa.Table.from_pandas(df2) + + with pa.OSFile(f"{dataset_root}/vqav2_trainable_val.arrow", "wb") as sink: + with pa.RecordBatchFileWriter(sink, df1.schema) as writer: + writer.write_table(df1) + + with pa.OSFile(f"{dataset_root}/vqav2_rest_val.arrow", "wb") as sink: + with pa.RecordBatchFileWriter(sink, df2.schema) as writer: + writer.write_table(df2) \ No newline at end of file diff --git a/data/visualize.py b/data/visualize.py new file mode 100644 index 0000000000000000000000000000000000000000..f02fc7b2ff62ffe403cad28147c0dc4dc28c447c --- /dev/null +++ b/data/visualize.py @@ -0,0 +1,231 @@ +from .datasets.ab_dataset import ABDataset +import matplotlib.pyplot as plt +from torchvision.utils import make_grid +import math +import torch + + +def visualize_classes_image_classification(dataset: ABDataset, class_to_idx_map, rename_map, + fig_save_path: str, num_imgs_per_class=2, max_num_classes=20, + unknown_class_idx=None): + + idx_to_images = {} + idx_to_class = {} + idx_to_original_idx = {} + + reach_max_num_class_limit = False + for i, (c, idx) in enumerate(class_to_idx_map.items()): + if unknown_class_idx is not None and idx == unknown_class_idx: + continue + + idx_to_images[idx] = [] + idx_to_class[idx] = c + idx_to_original_idx[idx] = dataset.raw_classes.index(c) + + if unknown_class_idx is not None and len(idx_to_images.keys()) == max_num_classes - 1: + reach_max_num_class_limit = True + break + if unknown_class_idx is None and len(idx_to_images.keys()) == max_num_classes: + reach_max_num_class_limit = True + break + + if unknown_class_idx is not None: + idx_to_images[unknown_class_idx] = [] + idx_to_class[unknown_class_idx] = ['(unknown classes)'] + + full_flags = {k: False for k in idx_to_images.keys()} + + i = 0 + while True: + x, y = dataset[i] + i += 1 + y = int(y) + + if full_flags[y]: + continue + + idx_to_images[y] += [x] + if len(idx_to_images[y]) == num_imgs_per_class: + full_flags[y] = True + + if all(full_flags.values()): + break + + shown_num_classes = len(idx_to_images.keys()) + if reach_max_num_class_limit: + shown_num_classes += 1 + num_cols = 3 + num_rows = math.ceil(shown_num_classes / num_cols) + + plt.figure(figsize=(6.4, 4.8 * num_rows // 2)) + + draw_i = 1 + for class_idx, imgs in idx_to_images.items(): + class_name = idx_to_class[class_idx] + + grid = make_grid(imgs, normalize=True) + plt.subplot(num_rows, num_cols, draw_i) + draw_i += 1 + + plt.axis('off') + img = grid.permute(1, 2, 0).numpy() + plt.imshow(img) + + if unknown_class_idx is not None and class_idx == unknown_class_idx: + plt.title(f'(unknown classes)\n' + f'current index: {class_idx}') + else: + class_i = idx_to_original_idx[class_idx] + if class_name in rename_map.keys(): + renamed_class = rename_map[class_name] + plt.title(f'{class_i}-th original class\n' + f'"{class_name}" (→ "{renamed_class}")\n' + f'current index: {class_idx}') + else: + plt.title(f'{class_i}-th original class\n' + f'"{class_name}"\n' + f'current index: {class_idx}') + + if reach_max_num_class_limit: + plt.subplot(num_rows, num_cols, draw_i) + plt.axis('off') + plt.imshow(torch.ones_like(grid).permute(1, 2, 0).numpy()) + plt.title(f'(Show up to {max_num_classes} classes...)') + + plt.tight_layout() + plt.savefig(fig_save_path, dpi=300) + plt.clf() + + +def visualize_classes_in_object_detection(dataset: ABDataset, class_to_idx_map, rename_map, + fig_save_path: str, num_imgs_per_class=2, max_num_classes=20, + unknown_class_idx=None): + + idx_to_images = {} + idx_to_class = {} + idx_to_original_idx = {} + + reach_max_num_class_limit = False + for i, (c, idx) in enumerate(class_to_idx_map.items()): + if unknown_class_idx is not None and idx == unknown_class_idx: + continue + + idx_to_images[idx] = [] + idx_to_class[idx] = c + idx_to_original_idx[idx] = dataset.raw_classes.index(c) + + if unknown_class_idx is not None and len(idx_to_images.keys()) == max_num_classes - 1: + reach_max_num_class_limit = True + break + if unknown_class_idx is None and len(idx_to_images.keys()) == max_num_classes: + reach_max_num_class_limit = True + break + + if unknown_class_idx is not None: + idx_to_images[unknown_class_idx] = [] + idx_to_class[unknown_class_idx] = ['(unknown classes)'] + + full_flags = {k: False for k in idx_to_images.keys()} + + # print(idx_to_images.keys()) + + ii = 0 + + import time + start_time = time.time() + + while True: + # print(dataset[i]) + x, y = dataset[ii][:2] + ii += 1 + + cur_map = {} + + for label_info in y: + if sum(label_info[1:]) == 0: # pad label + break + + ci = label_info[0] + print(f'cur ci: {ci}') + # print(ci, label_info) + + if ci in cur_map.keys(): + continue # do not visualize multiple objects in an image + + if len(idx_to_images[ci]) == num_imgs_per_class: + full_flags[ci] = True + break + + idx_to_images[ci] += [(x, label_info[1:])] + print(f'add image, ci: {ci}') + cur_map[ci] = 1 + + if time.time() - start_time > 40: + break + + if sum(list(full_flags.values())) > len(full_flags.values()) * 0.7: + break + + shown_num_classes = len(idx_to_images.keys()) + if reach_max_num_class_limit: + shown_num_classes += 1 + num_cols = 3 + num_rows = math.ceil(shown_num_classes / num_cols) + + plt.figure(figsize=(6.4, 4.8 * num_rows // 2)) + + from torchvision.transforms import ToTensor + from PIL import Image, ImageDraw + import numpy as np + + def draw_bbox(img, bbox): + img = Image.fromarray(np.uint8(img.transpose(1, 2, 0))) + draw = ImageDraw.Draw(img) + draw.rectangle(bbox, outline=(255, 0, 0), width=6) + return np.array(img) + + draw_i = 1 + for class_idx, imgs in idx_to_images.items(): + if len(imgs) == 0: + draw_i += 1 + continue + + imgs, bboxes = [img[0] for img in imgs], [img[1] for img in imgs] + class_name = idx_to_class[class_idx] + + # draw bbox + imgs = [draw_bbox(img, bbox) for img, bbox in zip(imgs, bboxes)] + imgs = [ToTensor()(img) for img in imgs] + + grid = make_grid(imgs, normalize=True) + plt.subplot(num_rows, num_cols, draw_i) + draw_i += 1 + + plt.axis('off') + img = grid.permute(1, 2, 0).numpy() + plt.imshow(img) + + if unknown_class_idx is not None and class_idx == unknown_class_idx: + plt.title(f'(unknown classes)\n' + f'current index: {class_idx}') + else: + class_i = idx_to_original_idx[class_idx] + if class_name in rename_map.keys(): + renamed_class = rename_map[class_name] + plt.title(f'{class_i}-th original class\n' + f'"{class_name}" (→ "{renamed_class}")\n' + f'current index: {class_idx}') + else: + plt.title(f'{class_i}-th original class\n' + f'"{class_name}"\n' + f'current index: {class_idx}') + + if reach_max_num_class_limit: + plt.subplot(num_rows, num_cols, draw_i) + plt.axis('off') + plt.imshow(torch.ones_like(grid).permute(1, 2, 0).numpy()) + plt.title(f'(Show up to {max_num_classes} classes...)') + + plt.tight_layout() + plt.savefig(fig_save_path, dpi=300) + plt.clf() diff --git a/det_lora_map50_0.5432.png b/det_lora_map50_0.5432.png new file mode 100644 index 0000000000000000000000000000000000000000..c718ea8e8bfc41791ffd0bf7871708eee4c8a3ba Binary files /dev/null and b/det_lora_map50_0.5432.png differ diff --git a/det_md_w_fbs_index_map50_0.4167.png b/det_md_w_fbs_index_map50_0.4167.png new file mode 100644 index 0000000000000000000000000000000000000000..ef39bb2d956312c3f5c74913626828668afe2f52 Binary files /dev/null and b/det_md_w_fbs_index_map50_0.4167.png differ diff --git a/det_md_wo_fbs_map50_0.3926.png b/det_md_wo_fbs_map50_0.3926.png new file mode 100644 index 0000000000000000000000000000000000000000..97d499310f73d508447f59a77067177e75fda290 Binary files /dev/null and b/det_md_wo_fbs_map50_0.3926.png differ diff --git a/det_online.png b/det_online.png new file mode 100644 index 0000000000000000000000000000000000000000..0761b6649dfd5f7bfc253b4b0d48b6592fdc813f Binary files /dev/null and b/det_online.png differ diff --git a/dnns/bert/__init__.py b/dnns/bert/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..2becbb7a0ade09e14c8f5adac2cfb06f07416430 --- /dev/null +++ b/dnns/bert/__init__.py @@ -0,0 +1,85 @@ +from transformers import BertTokenizer, BertModel, BertConfig +from utils.dl.common.model import set_module +from torch import nn +import torch +from utils.common.log import logger + + +bert_model_tag = 'bert-base-multilingual-cased' + + +class BertForSenCls(nn.Module): + def __init__(self, num_classes): + super(BertForSenCls, self).__init__() + + logger.info(f'init bert for sen cls (using {bert_model_tag})') + self.bert = BertModel.from_pretrained(bert_model_tag) + self.classifier = nn.Linear(768, num_classes) + + def forward(self, **x): + x['return_dict'] = False + + pool_output = self.bert(**x)[-1] + + return self.classifier(pool_output) + + +class BertForTokenCls(nn.Module): + def __init__(self, num_classes): + super(BertForTokenCls, self).__init__() + + logger.info(f'init bert for token cls (using {bert_model_tag})') + self.bert = BertModel.from_pretrained(bert_model_tag) + self.classifier = nn.Linear(768, num_classes) + + def forward(self, **x): + x['return_dict'] = False + + pool_output = self.bert(**x)[0] + + return self.classifier(pool_output) + + +class BertForTranslation(nn.Module): + def __init__(self): + super(BertForTranslation, self).__init__() + + self.bert = BertModel.from_pretrained(bert_model_tag) + + vocab_size = BertConfig.from_pretrained(bert_model_tag).vocab_size + self.decoder = nn.Linear(768, vocab_size) + + logger.info(f'init bert for sen cls (using {bert_model_tag}), vocab size {vocab_size}') + + # https://github.com/huggingface/transformers/blob/66954ea25e342fd451c26ec1c295da0b8692086b/src/transformers/models/bert_generation/modeling_bert_generation.py#L594 + self.decoder.weight.data.normal_(mean=0.0, std=0.02) + + def forward(self, **x): + x['return_dict'] = False + + seq_output = self.bert(**x)[0] + + return self.decoder(seq_output) + + +def bert_base_sen_cls(num_classes): + return BertForSenCls(num_classes) + + +def bert_base_token_cls(num_classes): + return BertForTokenCls(num_classes) + + +def bert_base_translation(no_bert_pooler=False): + # return BertForTranslation() + from transformers import BertTokenizer, BertModel, BertConfig, EncoderDecoderModel, BertGenerationDecoder + encoder = BertModel.from_pretrained(bert_model_tag) + model = BertGenerationDecoder.from_pretrained(bert_model_tag) + model.bert = encoder + + if no_bert_pooler: + logger.info('replace pooler with nn.Identity()') + encoder.pooler = nn.Identity() + + return model + \ No newline at end of file diff --git a/dnns/bert/vit_like_bert.py b/dnns/bert/vit_like_bert.py new file mode 100644 index 0000000000000000000000000000000000000000..3207ceea1510ecafef8036f5c7f8ae01596c8cde --- /dev/null +++ b/dnns/bert/vit_like_bert.py @@ -0,0 +1,277 @@ +from timm.models.vision_transformer import VisionTransformer, Mlp, Block, PatchEmbed, PatchDropout, named_apply, \ + init_weights_vit_timm, get_init_weights_vit, _load_weights, checkpoint_seq + +import torch +from torch import nn + +from functools import partial +from typing import Union, Tuple, Callable, Optional + +import logging +import math +from collections import OrderedDict +from functools import partial +from typing import Callable, List, Optional, Sequence, Tuple, Union + +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.utils.checkpoint +from torch.jit import Final + +from timm.data import IMAGENET_DEFAULT_MEAN, IMAGENET_DEFAULT_STD, IMAGENET_INCEPTION_MEAN, IMAGENET_INCEPTION_STD, \ + OPENAI_CLIP_MEAN, OPENAI_CLIP_STD +from timm.layers import PatchEmbed, Mlp, DropPath, trunc_normal_, lecun_normal_, resample_patch_embed, \ + resample_abs_pos_embed, RmsNorm, PatchDropout, use_fused_attn, SwiGLUPacked + + +class ViTLikeBERT(nn.Module): + + def __init__( + self, + img_size: Union[int, Tuple[int, int]] = 224, + patch_size: Union[int, Tuple[int, int]] = 16, + in_chans: int = 3, + num_classes: int = 1000, + global_pool: str = 'token', + embed_dim: int = 768, + depth: int = 12, + num_heads: int = 12, + mlp_ratio: float = 4., + qkv_bias: bool = True, + qk_norm: bool = False, + init_values: Optional[float] = None, + class_token: bool = True, + no_embed_class: bool = False, + pre_norm: bool = False, + fc_norm: Optional[bool] = None, + drop_rate: float = 0., + pos_drop_rate: float = 0., + patch_drop_rate: float = 0., + proj_drop_rate: float = 0., + attn_drop_rate: float = 0., + drop_path_rate: float = 0., + weight_init: str = '', + embed_layer: Callable = PatchEmbed, + norm_layer: Optional[Callable] = None, + act_layer: Optional[Callable] = None, + block_fn: Callable = Block, + mlp_layer: Callable = Mlp, + ): + """ + Args: + img_size: Input image size. + patch_size: Patch size. + in_chans: Number of image input channels. + num_classes: Mumber of classes for classification head. + global_pool: Type of global pooling for final sequence (default: 'token'). + embed_dim: Transformer embedding dimension. + depth: Depth of transformer. + num_heads: Number of attention heads. + mlp_ratio: Ratio of mlp hidden dim to embedding dim. + qkv_bias: Enable bias for qkv projections if True. + init_values: Layer-scale init values (layer-scale enabled if not None). + class_token: Use class token. + fc_norm: Pre head norm after pool (instead of before), if None, enabled when global_pool == 'avg'. + drop_rate: Head dropout rate. + pos_drop_rate: Position embedding dropout rate. + attn_drop_rate: Attention dropout rate. + drop_path_rate: Stochastic depth rate. + weight_init: Weight initialization scheme. + embed_layer: Patch embedding layer. + norm_layer: Normalization layer. + act_layer: MLP activation layer. + block_fn: Transformer block layer. + """ + super().__init__() + assert global_pool in ('', 'avg', 'token') + assert class_token or global_pool != 'token' + use_fc_norm = global_pool == 'avg' if fc_norm is None else fc_norm + norm_layer = norm_layer or partial(nn.LayerNorm, eps=1e-6) + act_layer = act_layer or nn.GELU + + self.num_classes = num_classes + self.global_pool = global_pool + self.num_features = self.embed_dim = embed_dim # num_features for consistency with other models + self.num_prefix_tokens = 1 if class_token else 0 + self.no_embed_class = no_embed_class + self.grad_checkpointing = False + + self.patch_embed = embed_layer( + img_size=img_size, + patch_size=patch_size, + in_chans=in_chans, + embed_dim=embed_dim, + bias=not pre_norm, # disable bias if pre-norm is used (e.g. CLIP) + ) + num_patches = self.patch_embed.num_patches + + self.cls_token = nn.Parameter(torch.zeros(1, 1, embed_dim)) if class_token else None + embed_len = num_patches if no_embed_class else num_patches + self.num_prefix_tokens + self.pos_embed = nn.Parameter(torch.randn(1, embed_len, embed_dim) * .02) + self.pos_drop = nn.Dropout(p=pos_drop_rate) + if patch_drop_rate > 0: + self.patch_drop = PatchDropout( + patch_drop_rate, + num_prefix_tokens=self.num_prefix_tokens, + ) + else: + self.patch_drop = nn.Identity() + self.norm_pre = norm_layer(embed_dim) if pre_norm else nn.Identity() + + dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth)] # stochastic depth decay rule + self.blocks = nn.Sequential(*[ + block_fn( + dim=embed_dim, + num_heads=num_heads, + mlp_ratio=mlp_ratio, + qkv_bias=qkv_bias, + qk_norm=qk_norm, + init_values=init_values, + proj_drop=proj_drop_rate, + attn_drop=attn_drop_rate, + drop_path=dpr[i], + norm_layer=norm_layer, + act_layer=act_layer, + mlp_layer=mlp_layer, + ) + for i in range(depth)]) + self.norm = norm_layer(embed_dim) if not use_fc_norm else nn.Identity() + + # Classifier Head + self.fc_norm = norm_layer(embed_dim) if use_fc_norm else nn.Identity() + self.head_drop = nn.Dropout(drop_rate) + self.head = nn.Linear(self.embed_dim, num_classes) if num_classes > 0 else nn.Identity() + + if weight_init != 'skip': + self.init_weights(weight_init) + + def init_weights(self, mode=''): + assert mode in ('jax', 'jax_nlhb', 'moco', '') + head_bias = -math.log(self.num_classes) if 'nlhb' in mode else 0. + trunc_normal_(self.pos_embed, std=.02) + if self.cls_token is not None: + nn.init.normal_(self.cls_token, std=1e-6) + named_apply(get_init_weights_vit(mode, head_bias), self) + + def _init_weights(self, m): + # this fn left here for compat with downstream users + init_weights_vit_timm(m) + + @torch.jit.ignore() + def load_pretrained(self, checkpoint_path, prefix=''): + _load_weights(self, checkpoint_path, prefix) + + @torch.jit.ignore + def no_weight_decay(self): + return {'pos_embed', 'cls_token', 'dist_token'} + + @torch.jit.ignore + def group_matcher(self, coarse=False): + return dict( + stem=r'^cls_token|pos_embed|patch_embed', # stem and embed + blocks=[(r'^blocks\.(\d+)', None), (r'^norm', (99999,))] + ) + + @torch.jit.ignore + def set_grad_checkpointing(self, enable=True): + self.grad_checkpointing = enable + + @torch.jit.ignore + def get_classifier(self): + return self.head + + def reset_classifier(self, num_classes: int, global_pool=None): + self.num_classes = num_classes + if global_pool is not None: + assert global_pool in ('', 'avg', 'token') + self.global_pool = global_pool + self.head = nn.Linear(self.embed_dim, num_classes) if num_classes > 0 else nn.Identity() + + def _pos_embed(self, x): + if self.no_embed_class: + # deit-3, updated JAX (big vision) + # position embedding does not overlap with class token, add then concat + x = x + self.pos_embed + if self.cls_token is not None: + x = torch.cat((self.cls_token.expand(x.shape[0], -1, -1), x), dim=1) + else: + # original timm, JAX, and deit vit impl + # pos_embed has entry for class token, concat then add + if self.cls_token is not None: + x = torch.cat((self.cls_token.expand(x.shape[0], -1, -1), x), dim=1) + x = x + self.pos_embed + return self.pos_drop(x) + + def _intermediate_layers( + self, + x: torch.Tensor, + n: Union[int, Sequence] = 1, + ): + outputs, num_blocks = [], len(self.blocks) + take_indices = set(range(num_blocks - n, num_blocks) if isinstance(n, int) else n) + + # forward pass + x = self.patch_embed(x) + x = self._pos_embed(x) + x = self.patch_drop(x) + x = self.norm_pre(x) + for i, blk in enumerate(self.blocks): + x = blk(x) + if i in take_indices: + outputs.append(x) + + return outputs + + def get_intermediate_layers( + self, + x: torch.Tensor, + n: Union[int, Sequence] = 1, + reshape: bool = False, + return_class_token: bool = False, + norm: bool = False, + ) -> Tuple[Union[torch.Tensor, Tuple[torch.Tensor]]]: + """ Intermediate layer accessor (NOTE: This is a WIP experiment). + Inspired by DINO / DINOv2 interface + """ + # take last n blocks if n is an int, if in is a sequence, select by matching indices + outputs = self._intermediate_layers(x, n) + if norm: + outputs = [self.norm(out) for out in outputs] + class_tokens = [out[:, 0:self.num_prefix_tokens] for out in outputs] + outputs = [out[:, self.num_prefix_tokens:] for out in outputs] + + if reshape: + grid_size = self.patch_embed.grid_size + outputs = [ + out.reshape(x.shape[0], grid_size[0], grid_size[1], -1).permute(0, 3, 1, 2).contiguous() + for out in outputs + ] + + if return_class_token: + return tuple(zip(outputs, class_tokens)) + return tuple(outputs) + + def forward_features(self, x): + x = self.patch_embed(x) + x = self._pos_embed(x) + x = self.patch_drop(x) + x = self.norm_pre(x) + if self.grad_checkpointing and not torch.jit.is_scripting(): + x = checkpoint_seq(self.blocks, x) + else: + x = self.blocks(x) + x = self.norm(x) + return x + + def forward_head(self, x, pre_logits: bool = False): + if self.global_pool: + x = x[:, self.num_prefix_tokens:].mean(dim=1) if self.global_pool == 'avg' else x[:, 0] + x = self.fc_norm(x) + x = self.head_drop(x) + return x if pre_logits else self.head(x) + + def forward(self, x): + x = self.forward_features(x) + x = self.forward_head(x) + return x diff --git a/dnns/clip/__init__.py b/dnns/clip/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..3958ba6967b8cfc4842df7675be6eb97dbd6ddbd --- /dev/null +++ b/dnns/clip/__init__.py @@ -0,0 +1,169 @@ +import timm +from timm.models._factory import load_checkpoint +import torch +import os +from typing import List, Union +from torch import nn +from torch.jit import Final +from einops import rearrange, repeat +from einops.layers.torch import Rearrange +from utils.dl.common.model import get_model_device, set_module +import torch.nn.functional as F +from utils.common.log import logger + +from transformers import CLIPProcessor, CLIPModel, CLIPVisionConfig, CLIPConfig +from dnns.clip.custom_clip import CLIPModelCanReceiveTextEmbeds + +import torch.nn.functional as F + + +class Clip_ViTB16(nn.Module): + def __init__(self, img_size): + super(Clip_ViTB16, self).__init__() + + self.processor = CLIPProcessor.from_pretrained("openai/clip-vit-base-patch16") + self.model: CLIPModel = CLIPModelCanReceiveTextEmbeds.from_pretrained("openai/clip-vit-base-patch16") + + self.img_size = img_size + + # reconstruct xx + vm_embed = self.model.vision_model.embeddings + raw_num_patches = vm_embed.num_patches + vm_embed.num_patches = (img_size // self.model.vision_model.embeddings.patch_size) ** 2 + vm_embed.num_positions = vm_embed.num_patches + 1 + vm_embed.register_buffer("position_ids", torch.arange(vm_embed.num_positions).expand((1, -1)), persistent=False) + + logger.info(f'due to changed input image size ({img_size}), num patches are updated from {raw_num_patches} to {vm_embed.num_patches}') + + self.first_inference = True + + def forward(self, images, texts: Union[List[List[str]], torch.Tensor], for_training, disable_return_loss=False, only_return_logits_per_text=False, no_grad_text=False): + + if isinstance(texts[0], str): + inputs = self.processor(text=texts, images=images, return_tensors="pt", padding=True) + else: + # input embeds instead of input ids + # however, original CLIP cannot receive Tensor as input + inputs = self.processor(images=images, return_tensors="pt") + inputs['attention_mask'] = torch.ones((texts.size(0), texts.size(1))) + inputs['input_embeds'] = texts + + if for_training and not disable_return_loss: + inputs['return_loss'] = True + else: + inputs['return_loss'] = False + + inputs['only_return_logits_per_text'] = only_return_logits_per_text + inputs['no_grad_text'] = no_grad_text + + for k, v in inputs.items(): + if isinstance(v, torch.Tensor): + inputs[k] = v.to('cuda') + + if self.first_inference: + logger.info(f'before input size: {inputs["pixel_values"].size()}') + + # print(inputs.keys()) + # print(inputs['pixel_values'].size()) + inputs['pixel_values'] = F.interpolate(inputs['pixel_values'], size=(self.img_size, self.img_size)) + # print(inputs['pixel_values'].size()) + + if self.first_inference: + logger.info(f'after input size: {inputs["pixel_values"].size()}') + self.first_inference = False + + return self.model(**inputs) + +# @torch.no_grad() +# def clip_vit_b_16(): +# # https://huggingface.co/openai/clip-vit-base-patch16 +# model = CLIPModelCanReceiveTextEmbeds.from_pretrained("openai/clip-vit-base-patch16") +# processor = CLIPProcessor.from_pretrained("openai/clip-vit-base-patch16") + +# print(model) + +# from PIL import Image +# import requests +# image = Image.open('/data/zql/datasets/Caltech-256/data/caltech256/256_ObjectCategories/003.backpack/003_0001.jpg') +# inputs = processor(text=["a photo of a dog", "a photo of a backpack", "a photo of a cat"], images=image, return_tensors="pt", padding=True) +# print(inputs) + +# from utils.dl.common.model import LayerActivation2, get_module +# input_embed_hook = LayerActivation2(get_module(model, 'text_model.embeddings')) +# outputs = model(**inputs) +# logits_per_image = outputs.logits_per_image # this is the image-text similarity score +# probs = logits_per_image.softmax(dim=1) +# print(probs) + +# input_embed = input_embed_hook.output +# input_embed_hook.remove() + +# torch.save(input_embed, os.path.join(os.path.dirname(__file__), './test_input_embed.pth')) + +# print('embed', input_embed.size()) + +# del inputs['input_ids'] +# inputs['input_embeds'] = input_embed +# outputs = model(**inputs) +# logits_per_image = outputs.logits_per_image # this is the image-text similarity score +# probs = logits_per_image.softmax(dim=1) +# print(probs) + + +@torch.no_grad() +def clip_vit_b_16(img_size): + # https://huggingface.co/openai/clip-vit-base-patch16 + return Clip_ViTB16(img_size) + + + + +if __name__ == '__main__': + model = clip_vit_b_16().cuda() + # print(model) + # exit() + + + # config = CLIPConfig.from_pretrained('openai/clip-vit-base-patch16') + # print(config) + + # # test 1: single image inference + # from PIL import Image + # import requests + # image = Image.open('/data/zql/datasets/Caltech-256/data/caltech256/256_ObjectCategories/003.backpack/003_0001.jpg') + # text = ["a photo of a dog", "a photo of a backpack", "a photo of a cat"] + + # o = model(image, text, False) + # print(o) + # print(o.logits_per_image.softmax(dim=1)) + + # o = model(image, torch.load('dnns/clip/test_input_embed.pth'), False) + # # print(o) + # print(o.logits_per_image.softmax(dim=1)) + # exit() + + # test 2: normal training using clip loss (batch) + from data import get_dataset, build_dataloader + from torchvision.transforms import Compose, ToTensor, Resize + dataset = get_dataset('Caltech256', '/data/zql/datasets/Caltech-256/data/caltech256/256_ObjectCategories/', 'train', transform=Compose([ + Resize((32, 32)), ToTensor() + ])) + dataloader = build_dataloader(dataset, 8, 0, True, None) + + from PIL import Image + import requests + images, labels = next(iter(dataloader)) + + # torch.save(images, 'dnns/clip/test_image.pth') + classes = dataset.classes + text = [f"a photo of a {classes[i]}" for i in labels] # should be ground truth + print(text) + print(images.size()) + + o = model(images, text, True) + print(o) + print(o.logits_per_image.softmax(dim=1)) + + # o = model(image, torch.load('dnns/clip/test_input_embed.pth'), False) + # # print(o) + # print(o.logits_per_image.softmax(dim=1)) \ No newline at end of file diff --git a/dnns/clip/custom_clip.py b/dnns/clip/custom_clip.py new file mode 100644 index 0000000000000000000000000000000000000000..a89edbd1bf715b4f06cda60b83c481b69d156e4c --- /dev/null +++ b/dnns/clip/custom_clip.py @@ -0,0 +1,177 @@ +from typing import Optional, Tuple, Union +import torch +from transformers.modeling_outputs import BaseModelOutputWithPooling +from transformers.models.clip.configuration_clip import CLIPConfig +from transformers.models.clip.modeling_clip import CLIPModel, CLIPTextTransformer, _make_causal_mask, _expand_mask, clip_loss, CLIPOutput + + +class CLIPTextTransformerCanReceiveEmbed(CLIPTextTransformer): + def forward(self, + input_ids: Optional[torch.Tensor] = None, + input_embeds: Optional[torch.Tensor] = None, # NOTE + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.Tensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None,) -> Union[Tuple, BaseModelOutputWithPooling]: + + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + if input_embeds is None: + if input_ids is None: + raise ValueError("You have to specify input_ids") + + input_shape = input_ids.size() + input_ids = input_ids.view(-1, input_shape[-1]) + + hidden_states = self.embeddings(input_ids=input_ids, position_ids=position_ids) + else: + hidden_states = input_embeds + input_shape = torch.Size([hidden_states.size(0), hidden_states.size(1)]) + + # CLIP's text model uses causal mask, prepare it here. + # https://github.com/openai/CLIP/blob/cfcffb90e69f37bf2ff1e988237a0fbe41f33c04/clip/model.py#L324 + # print(input_shape) + causal_attention_mask = _make_causal_mask(input_shape, hidden_states.dtype, device=hidden_states.device) + # expand attention_mask + if attention_mask is not None: + # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len] + attention_mask = _expand_mask(attention_mask, hidden_states.dtype) + + encoder_outputs = self.encoder( + inputs_embeds=hidden_states, + attention_mask=attention_mask, + causal_attention_mask=causal_attention_mask, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + last_hidden_state = encoder_outputs[0] + last_hidden_state = self.final_layer_norm(last_hidden_state) + + # text_embeds.shape = [batch_size, sequence_length, transformer.width] + # take features from the eot embedding (eot_token is the highest number in each sequence) + # eot embedding pos: input_ids.to(dtype=torch.int, device=last_hidden_state.device).argmax(dim=-1) + # casting to torch.int for onnx compatibility: argmax doesn't support int64 inputs with opset 14 + + if input_ids is not None: + eos_embedding_pos = input_ids.to(dtype=torch.int, device=last_hidden_state.device).argmax(dim=-1) + # print(input_ids, eos_embedding_pos) + else: + # pass + # TODO: is there any exception? + eos_embedding_pos = torch.tensor([input_embeds.size(1) - 1] * input_embeds.size(0), device=last_hidden_state.device) + + pooled_output = last_hidden_state[ + torch.arange(last_hidden_state.shape[0], device=last_hidden_state.device), + eos_embedding_pos + ] + + if not return_dict: + return (last_hidden_state, pooled_output) + encoder_outputs[1:] + + return BaseModelOutputWithPooling( + last_hidden_state=last_hidden_state, + pooler_output=pooled_output, + hidden_states=encoder_outputs.hidden_states, + attentions=encoder_outputs.attentions, + ) + + +class CLIPModelCanReceiveTextEmbeds(CLIPModel): + def __init__(self, config: CLIPConfig): + super().__init__(config) + + self.text_model = CLIPTextTransformerCanReceiveEmbed(config.text_config) + + def forward( + self, + input_ids: Optional[torch.LongTensor] = None, + input_embeds: Optional[torch.LongTensor] = None, + pixel_values: Optional[torch.FloatTensor] = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + return_loss: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + only_return_logits_per_text = False, + no_grad_text = False + ) -> Union[Tuple, CLIPOutput]: + + # Use CLIP model's config for some fields (if specified) instead of those of vision & text components. + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + vision_outputs = self.vision_model( + pixel_values=pixel_values, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + if no_grad_text: + with torch.no_grad(): + text_outputs = self.text_model( + input_ids=input_ids, + input_embeds=input_embeds, + attention_mask=attention_mask, + position_ids=position_ids, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + else: + text_outputs = self.text_model( + input_ids=input_ids, + input_embeds=input_embeds, + attention_mask=attention_mask, + position_ids=position_ids, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + image_embeds = vision_outputs[1] + image_embeds = self.visual_projection(image_embeds) + + text_embeds = text_outputs[1] + text_embeds = self.text_projection(text_embeds) + + # normalized features + image_embeds = image_embeds / image_embeds.norm(p=2, dim=-1, keepdim=True) + text_embeds = text_embeds / text_embeds.norm(p=2, dim=-1, keepdim=True) + + # cosine similarity as logits + logit_scale = self.logit_scale.exp() + logits_per_text = torch.matmul(text_embeds, image_embeds.t()) * logit_scale + logits_per_image = logits_per_text.t() + + if only_return_logits_per_text: + return logits_per_text + + loss = None + if return_loss: + loss = clip_loss(logits_per_text) + + if not return_dict: + output = (logits_per_image, logits_per_text, text_embeds, image_embeds, text_outputs, vision_outputs) + return ((loss,) + output) if loss is not None else output + + return CLIPOutput( + loss=loss, + logits_per_image=logits_per_image, + logits_per_text=logits_per_text, + text_embeds=text_embeds, + image_embeds=image_embeds, + text_model_output=text_outputs, + vision_model_output=vision_outputs, + ) \ No newline at end of file diff --git a/dnns/deeplabv3/head.py b/dnns/deeplabv3/head.py new file mode 100644 index 0000000000000000000000000000000000000000..1256b46b2dc7d74a40ace70dafb86bbddc717c0a --- /dev/null +++ b/dnns/deeplabv3/head.py @@ -0,0 +1,54 @@ +from torch import nn +from einops import rearrange +import torch.nn.functional as F + +from utils.dl.common.model import get_super_module + + +class DecoderLinear(nn.Module): + def __init__(self, n_cls, patch_size, d_encoder, im_size): + super(DecoderLinear, self).__init__() + + self.d_encoder = d_encoder + self.patch_size = patch_size + self.n_cls = n_cls + self.im_size = im_size + + self.head = nn.Linear(self.d_encoder, n_cls) + + def debug(self): + print(self.head, id(self), 'debug()') + + def forward(self, x): + # print('inside debug') + # self.debug() + x = x[:, 1:] # remove cls token + # print(x.size()) + + H, W = self.im_size + GS = H // self.patch_size + # print(H, W, GS, self.patch_size) + # print('head', self.head.weight.size(), x.size()) + # print(self.head, 'debug()') + x = self.head(x) + # print(x.size()) + + # (b, HW//ps**2, ps_c) + x = rearrange(x, "b (h w) c -> b c h w", h=GS) + + # print(x.size()) + + masks = x + masks = F.upsample(masks, size=(H, W), mode="bilinear") + + # print(masks.size()) + + return masks + + +def modify_forward_head(): + from types import MethodType + from timm.models.vision_transformer import VisionTransformer + def forward_head(self, x, pre_logits: bool = False): + return self.head(x) + VisionTransformer.forward_head = MethodType(forward_head, VisionTransformer) diff --git a/dnns/deeplabv3/init_from_vit.py b/dnns/deeplabv3/init_from_vit.py new file mode 100644 index 0000000000000000000000000000000000000000..e06651ddb52407df88a8859c93add2a1d488be70 --- /dev/null +++ b/dnns/deeplabv3/init_from_vit.py @@ -0,0 +1,15 @@ +from typing import Callable, Optional, Tuple, Union +from timm.layers import Mlp, PatchEmbed +from timm.models.vision_transformer import Block, VisionTransformer +from .head import DecoderLinear + + +class ViTForSeg(VisionTransformer): + def __init__(self, img_size: int | Tuple[int, int] = 224, patch_size: int | Tuple[int, int] = 16, in_chans: int = 3, num_classes: int = 1000, global_pool: str = 'token', embed_dim: int = 768, depth: int = 12, num_heads: int = 12, mlp_ratio: float = 4, qkv_bias: bool = True, qk_norm: bool = False, init_values: float | None = None, class_token: bool = True, no_embed_class: bool = False, pre_norm: bool = False, fc_norm: bool | None = None, drop_rate: float = 0, pos_drop_rate: float = 0, patch_drop_rate: float = 0, proj_drop_rate: float = 0, attn_drop_rate: float = 0, drop_path_rate: float = 0, weight_init: str = '', embed_layer: Callable[..., Any] = ..., norm_layer: Callable[..., Any] | None = None, act_layer: Callable[..., Any] | None = None, block_fn: Callable[..., Any] = ..., mlp_layer: Callable[..., Any] = ...): + super().__init__(img_size, patch_size, in_chans, num_classes, global_pool, embed_dim, depth, num_heads, mlp_ratio, qkv_bias, qk_norm, init_values, class_token, no_embed_class, pre_norm, fc_norm, drop_rate, pos_drop_rate, patch_drop_rate, proj_drop_rate, attn_drop_rate, drop_path_rate, weight_init, embed_layer, norm_layer, act_layer, block_fn, mlp_layer) + self.head = DecoderLinear(num) + def forward_head(self, x, pre_logits: bool = False): + return self.head(x) + + def init_from_vit(self, vit): + self.load_state_dict(vit.state_dict(), strict=False) diff --git a/dnns/vilt/__init__.py b/dnns/vilt/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..0305219d5fd8ec2668f58c1827cb49c822071ade --- /dev/null +++ b/dnns/vilt/__init__.py @@ -0,0 +1,63 @@ +import timm +from timm.models._factory import load_checkpoint +import torch +import os +from typing import List, Union +from torch import nn +from torch.jit import Final +from einops import rearrange, repeat +from einops.layers.torch import Rearrange +from utils.dl.common.model import get_model_device, set_module +import torch.nn.functional as F +from utils.common.log import logger + +from transformers import ViltModel, ViltForQuestionAnswering +import torch.nn.functional as F + + + +def vilt_b_32(num_classes): + """ + Vilt for VQA + + settings based on the dataset VQAv2 (3129 classes): + + 1. use half of classes for LoRA adaptation + 2. use this half of classes for DA evaluation (using corruptions for generating domain shifts), + and use another half of classes for CL evaluation. + """ + + #model = ViltForQuestionAnswering.from_pretrained('dandelin/vilt-b32-mlm-itm') + model = ViltForQuestionAnswering.from_pretrained('new_impl/mm/Vis_bert/QuestionAnswering/vilt') + linear = model.classifier[3] + new_linear = nn.Linear(linear.in_features, num_classes, bias=True) + set_module(model, 'classifier.3', new_linear) + + return model + + +if __name__ == '__main__': + model = vilt_b_32(1565) + + print(model) + + from transformers import ViltProcessor, ViltModel + from PIL import Image + import requests + + # prepare image and text + url = "http://images.cocodataset.org/val2017/000000039769.jpg" + image = Image.open(requests.get(url, stream=True).raw) + text = "hello world" + + processor = ViltProcessor.from_pretrained("dandelin/vilt-b32-mlm") + model = ViltModel.from_pretrained("dandelin/vilt-b32-mlm-itm") + + inputs = processor(image, text, return_tensors="pt") + + print(inputs) + + outputs = model(**inputs) + last_hidden_states = outputs.last_hidden_state + + print(last_hidden_states.shape) \ No newline at end of file diff --git a/dnns/vit/__init__.py b/dnns/vit/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..8ab0736efaa4c135ab7a3d16f1f890219e0e6ba3 --- /dev/null +++ b/dnns/vit/__init__.py @@ -0,0 +1,349 @@ +import timm +from timm.models._factory import load_checkpoint +import torch +import os +from torch import nn +from torch.jit import Final +from einops import rearrange, repeat +from einops.layers.torch import Rearrange +from utils.dl.common.model import get_model_device, set_module +import torch.nn.functional as F +from utils.common.log import logger + + +# class SoftmaxIgnoringZero(nn.Module): +# def __init__(self): +# super(SoftmaxIgnoringZero, self).__init__() + +# def forward(self, x: torch.Tensor): +# # non_zero_x_indexes = x.nonzero(as_tuple=True)[0] +# # non_zero_x = x[non_zero_x_indexes] +# # non_zero_x_softmax = F.softmax(non_zero_x, self.dim, _stacklevel=5) +# # res = torch.zeros_like(x) + +# # original: e^i / \sum_i e^i +# # ignoring zero: e^i +# # print(x) + +# non_zero_mask = x != 0 + +# if non_zero_mask.sum() == x.numel(): +# return F.softmax(x, -1) + +# t = non_zero_mask.sum(-1) +# assert t.view(-1).unique().size(0) == 1, f'{t.view(-1).unique()}, {x.size()}' # all vectors in the softmaxed dim has the same number of 0 +# # assert t.view(-1).unique().size(0) <= 2, f'{t.view(-1).unique()}, {x.size()}' # all vectors in the softmaxed dim has the same number of 0 or has no 0 +# non_zero_x = torch.masked_select(x, non_zero_mask) + +# non_zero_x = non_zero_x.view(*(list(x.size())[0: -1] + [t.view(-1)[0].item()])) + +# # print(non_zero_x) + +# non_zero_x_softmax = F.softmax(non_zero_x, -1) + +# a = x.nonzero(as_tuple=True)[-1] +# a = a.view(*non_zero_x_softmax.size()) +# x = x.scatter(x.dim() - 1, a, non_zero_x_softmax) + +# return x + + +class SoftmaxIgnoringZero(nn.Module): + def __init__(self): + super(SoftmaxIgnoringZero, self).__init__() + + def f(self, x): + # return x / (x + 1e-8) + return 1. + + def forward(self, x: torch.Tensor): + res = F.softmax(x, -1) + return res * self.f(x) + + +class PrunableAttention(nn.Module): + """ + https://github.com/lucidrains/vit-pytorch + """ + def __init__(self, dim, heads = 8, dim_head = 64, dropout = 0., qkv_bias = False): + super().__init__() + self.inner_dim = inner_dim = dim_head * heads + project_out = not (heads == 1 and dim_head == dim) + + self.num_heads = heads + self.scale = dim_head ** -0.5 + + self.attend = nn.Softmax(dim = -1) + self.dropout = nn.Dropout(dropout) + + self.qkv = nn.Linear(dim, inner_dim * 3, bias = qkv_bias) + + # self.proj = nn.Sequential( + # nn.Linear(inner_dim, dim), + # nn.Dropout(dropout) + # ) if project_out else nn.Identity() + + self.proj = nn.Linear(inner_dim, dim) if project_out else nn.Identity() + self.proj_dropout = nn.Dropout(dropout) + + def forward(self, x): + # qkv = self.qkv(x).chunk(3, dim = -1) + raw_qkv = self.qkv(x) + + self.inner_dim = (raw_qkv.size(-1) - self.proj.in_features) // 2 + qkv = raw_qkv[:, :, 0: self.inner_dim], raw_qkv[:, :, self.inner_dim: self.inner_dim * 2], raw_qkv[:, :, self.inner_dim * 2:] + + # print('v', qkv[0].size(), qkv[0].sum((0, 1))[0: 10], qkv[0].sum((0, 1)).nonzero(as_tuple=True)[0].size()) + + # raw_v = qkv[2] + # print('after_fbs_q, after_fbs_k', qkv[0].sum((0, 1))[0: 10], qkv[0].sum((0, 1)).nonzero(as_tuple=True)[0].size(), + # qkv[1].sum((0, 1))[0: 10], qkv[1].sum((0, 1)).nonzero(as_tuple=True)[0].size(),) + # print('after_fbs_v', raw_v.size(), raw_v.sum((0, 1))[0: 10], raw_v.sum((0, 1)).nonzero(as_tuple=True)[0].size()) + # print('q, before rearrage', qkv[0].size()) + q, k, v = qkv + # print('raw qkv size', q.size(), k.size(), v.size()) + # exit() + q, k, v = map(lambda t: rearrange(t, 'b n (h d) -> b h n d', h = self.num_heads), qkv) + # print('raw qkv size', q.size(), k.size(), v.size()) + + dots = torch.matmul(q, k.transpose(-1, -2)) * self.scale + + # print('q, k, dots, after rearrage', q.size(), k.transpose(-1, -2).size(), dots.size()) + + attn = self.attend(dots) + # attn = dots + attn = self.dropout(attn) + + # print(attn) + # print('attn', attn.size(), attn.sum((0, 1))[0: 10], attn.sum((0, 1)).nonzero(as_tuple=True)[0].size()) + # print('attn', attn.size(), attn.sum((0, 1))[0: 10], attn.sum((0, 1)).nonzero(as_tuple=True)[0].size()) + # print('v2', v.size()) + out = torch.matmul(attn, v) + # print('out1', out.size()) + # NOTE: just for trial debug + # out = v + + # print('out before rerange', out.size()) + + # print(v.size(), v) + # exit() + + out = rearrange(out, 'b h n d -> b n (h d)') + + # print('out', out.size(), out.sum((0, 1))[0: 10], out.sum((0, 1)).nonzero(as_tuple=True)[0].size()) + # exit() + + res = self.proj_dropout(self.proj(out)) + + # res = self.proj_dropout( + # F.linear(self.proj.weight.T, out.T, self.proj.bias) + # ) + # print(self.proj, self.proj_dropout) + # print('res', res.size(), res.sum((0, 1))[0: 10], res.sum((0, 1)).nonzero(as_tuple=True)[0].size()) + + return res + + +def make_attention_prunable(vit): + for block in vit.blocks: + attn = block.attn + + assert attn.attn_drop.p == attn.proj_drop.p + + prunable_attn = PrunableAttention( + dim=attn.head_dim * attn.num_heads, + heads=attn.num_heads, + dim_head=attn.head_dim, + dropout=attn.attn_drop.p, + qkv_bias=attn.qkv.bias is not None + ) + prunable_attn.qkv.weight.copy_(attn.qkv.weight) + if attn.qkv.bias is not None: + prunable_attn.qkv.bias.copy_(attn.qkv.bias) + prunable_attn.proj.weight.copy_(attn.proj.weight) + prunable_attn.proj.bias.copy_(attn.proj.bias) + + set_module(block, 'attn', prunable_attn) + + +@torch.no_grad() +def vit_l_16(pretrained=True, num_classes=None) -> nn.Module: + # https://huggingface.co/timm/vit_large_patch16_224.augreg_in21k_ft_in1k + res = timm.create_model('vit_large_patch16_224.augreg_in21k_ft_in1k', + num_classes=num_classes) + + if pretrained: + checkpoint_path = os.path.join(os.path.dirname(__file__), + 'weights/vit_large_patch16_224.augreg_in21k_ft_in1k.bin') + def filter_fn(state_dict, _): + if num_classes is None: # use fine-tuned in1k fc head + return state_dict + else: # use a new linear + del state_dict['head.weight'] + del state_dict['head.bias'] + return state_dict + + load_checkpoint(res, checkpoint_path, strict=False, filter_fn=filter_fn) + + res.eval() + input_sample = torch.rand(2, 3, 224, 224) + o1 = res(input_sample) + + make_attention_prunable(res) + res.eval() + o2 = res(input_sample) + + assert ((o1 - o2) ** 2).sum() < 1e-5 + return res + + +from timm.models.vision_transformer import VisionTransformer + +@torch.no_grad() +def vit_b_16(pretrained=True, num_classes=None) -> VisionTransformer: + # https://huggingface.co/timm/vit_base_patch16_224.augreg_in21k_ft_in1k + res = timm.create_model('vit_base_patch16_224.augreg_in21k_ft_in1k', + num_classes=num_classes) + + if pretrained: + checkpoint_path = os.path.join(os.path.dirname(__file__), + 'weights/vit_base_patch16_224.augreg_in21k_ft_in1k.bin') + def filter_fn(state_dict, _): + if num_classes is None: # use fine-tuned in1k fc head + return state_dict + else: # use a new linear + del state_dict['head.weight'] + del state_dict['head.bias'] + return state_dict + + load_checkpoint(res, checkpoint_path, strict=False, filter_fn=filter_fn) + + res.eval() + input_sample = torch.rand(2, 3, 224, 224) + o1 = res(input_sample) + + logger.info(f'make attention prunable') + make_attention_prunable(res) + # logger.info(f'make softmax prunable') + # make_softmax_prunable(res) + + res.eval() + o2 = res(input_sample) + # print(((o1 - o2) ** 2).sum()) + assert ((o1 - o2) ** 2).sum() < 1e-5 + return res + + +def make_softmax_prunable(model): + model.eval() + input_sample = torch.rand(2, 3, 224, 224).to(get_model_device(model)) + o1 = model(input_sample) + + for name, module in model.named_modules(): + if isinstance(module, nn.Softmax): + set_module(model, name, SoftmaxIgnoringZero()) + logger.info(f'make softmax {name} prunable') + + model.eval() + o2 = model(input_sample) + assert ((o1 - o2) ** 2).sum() < 1e-5 + return model + + +if __name__ == '__main__': + model = vit_l_16() + model(torch.rand((1, 3, 224, 224))) + + + # from utils.dl.common.data_loader import ImageNetDataLoader + # _, test_loader = ImageNetDataLoader('/data/zql/datasets/imagenet2012/train', '/data/zql/datasets/imagenet2012/val', 512, 8) + + # import torch + # import tqdm + # import torch.nn.functional as F + # def get_accuracy(model, dataloader=test_loader, device='cuda'): + # acc = 0 + # sample_num = 0 + + # model.eval() + # model = model.to(device) + + # with torch.no_grad(): + # pbar = tqdm.tqdm(enumerate(dataloader), total=len(dataloader), dynamic_ncols=True, leave=False) + # for batch_index, (x, y) in pbar: + # x, y = x.to(device), y.to(device) + # output = model(x) + # pred = F.softmax(output, dim=1).argmax(dim=1) + # correct = torch.eq(pred, y).sum().item() + # acc += correct + # sample_num += len(y) + + # pbar.set_description(f'cur_batch_total: {len(y)}, cur_batch_correct: {correct}, ' + # f'cur_batch_acc: {(correct / len(y)):.4f}') + + # acc /= sample_num + # return acc + + # model = model.cuda() + # print(f'vit_l_16 im1k acc: {get_accuracy(model, test_loader, "cuda")}') + + + # softmax = SoftmaxIgnoringZero() + + # x = torch.tensor([[[1, 0, 3], [2, 2, 0]]] * 2).float() + # print(softmax(x)) + + + # model = vit_b_16(True) + # print(get_accuracy(model)) + + # for name, module in model.named_modules(): + # if isinstance(module, nn.Softmax): + # set_module(model, name, SoftmaxIgnoringZero()) + # print(f'{name}') + + # # print(model) + # print(get_accuracy(model)) + + # softmax = SoftmaxIgnoringZero() + # linear = nn.Linear(20, 10) + + # net = nn.Sequential(linear, softmax) + + # optimizer = torch.optim.SGD(net.parameters(), lr=10, momentum=0.9) + + # x = torch.rand((64, 20)) + # y_g = torch.rand((64, 10)) + + # for _ in range(100): + # y = net(x) + # # print(y) + + # loss = F.mse_loss(y, y_g) + + # optimizer.zero_grad() + # loss.backward() + + # # print(linear.weight.grad) + + # optimizer.step() + + # print(loss) + + + softmax = SoftmaxIgnoringZero() + + x = torch.tensor([ + [1, 0, 2], + [4, 0, 9], + [0, 0, 0], + [1, 1, 1] + ]).float() + print(softmax(x)) + + + x = torch.tensor([ + [1, 2], + [4, 9], + ]).float() + print(softmax(x)) \ No newline at end of file diff --git a/dnns/vit/__pycache__/__init__.cpython-38.pyc b/dnns/vit/__pycache__/__init__.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8b7668cc2f06a1fac9578806630bc96a5c3c69a6 Binary files /dev/null and b/dnns/vit/__pycache__/__init__.cpython-38.pyc differ diff --git a/dnns/vit/weights/download.txt b/dnns/vit/weights/download.txt new file mode 100644 index 0000000000000000000000000000000000000000..67697de3fcac56c25202ccaacaffcb6434ec7dcd --- /dev/null +++ b/dnns/vit/weights/download.txt @@ -0,0 +1 @@ +download the weight from https://huggingface.co/timm/vit_base_patch16_224.augreg2_in21k_ft_in1k/tree/main \ No newline at end of file diff --git a/dnns/yolov3/coco_evaluator.py b/dnns/yolov3/coco_evaluator.py new file mode 100644 index 0000000000000000000000000000000000000000..7c960fdebaff5f4ab16b9e1bfe5e4d3c53db2ea9 --- /dev/null +++ b/dnns/yolov3/coco_evaluator.py @@ -0,0 +1,372 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +# Copyright (c) Megvii, Inc. and its affiliates. + +import contextlib +import io +import itertools +import json +import tempfile +import time +from loguru import logger +from tabulate import tabulate +from tqdm import tqdm + +import numpy as np + +import torch + +# from core.common.dnn.detection.yolox.yolox.data.datasets import COCO_CLASSES +from .utils import ( + gather, + is_main_process, + postprocess, + synchronize, + time_synchronized, + xyxy2xywh +) +# from core.common.dnn.detection.yolox.yolox.layers import COCOeval_opt as COCOeval + + +def per_class_AR_table(coco_eval, class_names, headers=["class", "AR"], colums=6): + per_class_AR = {} + recalls = coco_eval.eval["recall"] + # dimension of recalls: [TxKxAxM] + # recall has dims (iou, cls, area range, max dets) + assert len(class_names) == recalls.shape[1] + + for idx, name in enumerate(class_names): + recall = recalls[:, idx, 0, -1] + recall = recall[recall > -1] + ar = np.mean(recall) if recall.size else float("nan") + per_class_AR[name] = float(ar * 100) + + num_cols = min(colums, len(per_class_AR) * len(headers)) + result_pair = [x for pair in per_class_AR.items() for x in pair] + row_pair = itertools.zip_longest(*[result_pair[i::num_cols] for i in range(num_cols)]) + table_headers = headers * (num_cols // len(headers)) + table = tabulate( + row_pair, tablefmt="pipe", floatfmt=".3f", headers=table_headers, numalign="left", + ) + return table + + +def per_class_AP_table(coco_eval, class_names, headers=["class", "AP"], colums=6): + per_class_AP = {} + precisions = coco_eval.eval["precision"] + # dimension of precisions: [TxRxKxAxM] + # precision has dims (iou, recall, cls, area range, max dets) + assert len(class_names) == precisions.shape[2] + + for idx, name in enumerate(class_names): + # area range index 0: all area ranges + # max dets index -1: typically 100 per image + precision = precisions[:, :, idx, 0, -1] + precision = precision[precision > -1] + ap = np.mean(precision) if precision.size else float("nan") + per_class_AP[name] = float(ap * 100) + + num_cols = min(colums, len(per_class_AP) * len(headers)) + result_pair = [x for pair in per_class_AP.items() for x in pair] + row_pair = itertools.zip_longest(*[result_pair[i::num_cols] for i in range(num_cols)]) + table_headers = headers * (num_cols // len(headers)) + table = tabulate( + row_pair, tablefmt="pipe", floatfmt=".3f", headers=table_headers, numalign="left", + ) + return table + + +class COCOEvaluator: + """ + COCO AP Evaluation class. All the data in the val2017 dataset are processed + and evaluated by COCO API. + """ + + def __init__( + self, + dataloader, + img_size: int, + confthre: float, + nmsthre: float, + num_classes: int, + testdev: bool = False, + per_class_AP: bool = False, + per_class_AR: bool = False, + ): + """ + Args: + dataloader (Dataloader): evaluate dataloader. + img_size: image size after preprocess. images are resized + to squares whose shape is (img_size, img_size). + confthre: confidence threshold ranging from 0 to 1, which + is defined in the config file. + nmsthre: IoU threshold of non-max supression ranging from 0 to 1. + per_class_AP: Show per class AP during evalution or not. Default to False. + per_class_AR: Show per class AR during evalution or not. Default to False. + """ + self.dataloader = dataloader + self.img_size = img_size + self.confthre = confthre + self.nmsthre = nmsthre + self.num_classes = num_classes + self.testdev = testdev + self.per_class_AP = per_class_AP + self.per_class_AR = per_class_AR + + def evaluate( + self, + model, + distributed=False, + half=False, + trt_file=None, + decoder=None, + test_size=None, + ): + """ + COCO average precision (AP) Evaluation. Iterate inference on the test dataset + and the results are evaluated by COCO API. + + NOTE: This function will change training mode to False, please save states if needed. + + Args: + model : model to evaluate. + + Returns: + ap50_95 (float) : COCO AP of IoU=50:95 + ap50 (float) : COCO AP of IoU=50 + summary (sr): summary info of evaluation. + """ + # TODO half to amp_test + tensor_type = torch.cuda.HalfTensor if half else torch.cuda.FloatTensor + model = model.eval() + if half: + model = model.half() + ids = [] + data_list = [] + progress_bar = iter if is_main_process() else iter + + inference_time = 0 + nms_time = 0 + n_samples = max(len(self.dataloader) - 1, 1) + + if trt_file is not None: + from torch2trt import TRTModule + + model_trt = TRTModule() + model_trt.load_state_dict(torch.load(trt_file)) + + x = torch.ones(1, 3, test_size[0], test_size[1]).cuda() + model(x) + model = model_trt + + import tqdm + for cur_iter, (imgs, _, info_imgs, ids) in tqdm.tqdm(enumerate( + progress_bar(self.dataloader) + ), dynamic_ncols=True, leave=False, total=len(self.dataloader)): + with torch.no_grad(): + imgs = imgs.type(tensor_type) + + # skip the the last iters since batchsize might be not enough for batch inference + is_time_record = cur_iter < len(self.dataloader) - 1 + if is_time_record: + start = time.time() + + outputs = model(imgs) + if decoder is not None: + outputs = decoder(outputs, dtype=outputs.type()) + + if is_time_record: + infer_end = time_synchronized() + inference_time += infer_end - start + + outputs = postprocess( + outputs, self.num_classes, self.confthre, self.nmsthre + ) + if is_time_record: + nms_end = time_synchronized() + nms_time += nms_end - infer_end + + data_list.extend(self.convert_to_coco_format(outputs, info_imgs, ids, imgs)) + + statistics = torch.cuda.FloatTensor([inference_time, nms_time, n_samples]) + if distributed: + data_list = gather(data_list, dst=0) + data_list = list(itertools.chain(*data_list)) + torch.distributed.reduce(statistics, dst=0) + + eval_results = self.evaluate_prediction(data_list, statistics) + synchronize() + return eval_results + + def convert_to_coco_format(self, outputs, info_imgs, ids, input_imgs=None): + data_list = [] + img_i = 0 + for (output, img_h, img_w, img_id, img) in zip( + outputs, info_imgs[0], info_imgs[1], ids, input_imgs + ): + if output is None: + continue + output = output.cpu() + + bboxes = output[:, 0:4] + + # preprocessing: resize + scale = min( + self.img_size[0] / float(img_h), self.img_size[1] / float(img_w) + ) + bboxes /= scale + bboxes = xyxy2xywh(bboxes) + + cls = output[:, 6] + scores = output[:, 4] * output[:, 5] + for ind in range(bboxes.shape[0]): + # print(self.dataloader.dataset.class_ids, cls[ind]) + # implemented by queyu, 2022/08/08 + _d = self.dataloader.dataset + if _d.__class__.__name__ == 'MergedDataset': + # _d = _d.datasets[0] + raise NotImplementedError + from data import ABDataset + if _d.__class__.__name__ == '_AugWrapperForDataset': + _d = _d.raw_dataset + if isinstance(_d, ABDataset): + _d = _d.dataset + if _d.__class__.__name__ == '_SplitDataset': + raise NotImplementedError + _d = _d.underlying_dataset + + class_ids = _d.class_ids + if int(cls[ind]) >= len(class_ids): + raise RuntimeError + label = self.dataloader.dataset.class_ids[-1] + else: + label = class_ids[int(cls[ind])] + pred_data = { + "image_id": int(img_id), + "category_id": label, + "bbox": bboxes[ind].numpy().tolist(), + "score": scores[ind].numpy().item(), + "segmentation": [], + } # COCO json format + data_list.append(pred_data) + + # TODO: debug + # img = input_imgs[ind] + + + # from torchvision.transforms import ToTensor, ToPILImage + # from torchvision.utils import make_grid + # from PIL import Image, ImageDraw + # import matplotlib.pyplot as plt + # import numpy as np + # def draw_bbox(img, bbox, label, f): + # # if f: + # # img = np.uint8(img.permute(1, 2, 0)) + # # img = Image.fromarray(img) + # img = ToPILImage()(img) + # draw = ImageDraw.Draw(img) + # draw.rectangle(bbox, outline=(255, 0, 0), width=6) + # draw.text((bbox[0], bbox[1]), label) + # return ToTensor()(np.array(img)) + + # def xywh2xyxy(bbox): + # x, y, w, h = bbox + # x1, y1 = x, y + # x2, y2 = x + w, y + h + # return x1, y1, x2, y2 + + # img = draw_bbox(img, xywh2xyxy(bboxes[ind].numpy()), str(label), True) + + # img = make_grid([img], 1, normalize=True) + # plt.axis('off') + # img = img.permute(1, 2, 0).numpy() + # plt.imshow(img) + # plt.savefig(f'./tmp-coco-eval-{ind}.png') + # plt.clf() + # img_i += 1 + + # exit(0) + return data_list + + def evaluate_prediction(self, data_dict, statistics): + if not is_main_process(): + return 0, 0, None + + # logger.info("Evaluate in main process...") + + annType = ["segm", "bbox", "keypoints"] + + inference_time = statistics[0].item() + nms_time = statistics[1].item() + n_samples = statistics[2].item() + + a_infer_time = 1000 * inference_time / (n_samples * self.dataloader.batch_size) + a_nms_time = 1000 * nms_time / (n_samples * self.dataloader.batch_size) + + time_info = ", ".join( + [ + "Average {} time: {:.2f} ms".format(k, v) + for k, v in zip( + ["forward", "NMS", "inference"], + [a_infer_time, a_nms_time, (a_infer_time + a_nms_time)], + ) + ] + ) + + info = time_info + "\n" + + # Evaluate the Dt (detection) json comparing with the ground truth + if len(data_dict) > 0: + # cocoGt = self.dataloader.dataset.coco + _d = self.dataloader.dataset + if _d.__class__.__name__ == 'MergedDataset': + # _d = _d.datasets[0] + raise NotImplementedError + from data import ABDataset + if _d.__class__.__name__ == '_AugWrapperForDataset': + _d = _d.raw_dataset + if isinstance(_d, ABDataset): + _d = _d.dataset + if _d.__class__.__name__ == '_SplitDataset': + raise NotImplementedError + _d = _d.underlying_dataset + + cocoGt = _d.coco + + # implemented by queyu, 2022/08/08 + # make cocoGt's label += y_offset + # cocoGt: COCOAPI + + # TODO: since pycocotools can't process dict in py36, write data to json file. + if self.testdev: + json.dump(data_dict, open("./yolox_testdev_2017.json", "w")) + cocoDt = cocoGt.loadRes("./yolox_testdev_2017.json") + else: + _, tmp = tempfile.mkstemp() + json.dump(data_dict, open(tmp, "w")) + cocoDt = cocoGt.loadRes(tmp) + # try: + # from core.common.dnn.detection.yolox.yolox.layers import COCOeval_opt as COCOeval + # except ImportError: + from pycocotools.cocoeval import COCOeval + + logger.warning("Use standard COCOeval.") + + cocoEval = COCOeval(cocoGt, cocoDt, annType[1]) + cocoEval.evaluate() + cocoEval.accumulate() + redirect_string = io.StringIO() + with contextlib.redirect_stdout(redirect_string): + cocoEval.summarize() + info += redirect_string.getvalue() + cat_ids = list(cocoGt.cats.keys()) + cat_names = [cocoGt.cats[catId]['name'] for catId in sorted(cat_ids)] + if self.per_class_AP: + AP_table = per_class_AP_table(cocoEval, class_names=cat_names) + info += "per class AP:\n" + AP_table + "\n" + if self.per_class_AR: + AR_table = per_class_AR_table(cocoEval, class_names=cat_names) + info += "per class AR:\n" + AR_table + "\n" + return cocoEval.stats[0], cocoEval.stats[1], info + else: + return 0, 0, info diff --git a/dnns/yolov3/head.py b/dnns/yolov3/head.py new file mode 100644 index 0000000000000000000000000000000000000000..1eca44bae9019b1438e01d433afbef0addf0b784 --- /dev/null +++ b/dnns/yolov3/head.py @@ -0,0 +1,676 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +# Copyright (c) Megvii Inc. All rights reserved. + +import math +from loguru import logger + +import torch +import torch.nn as nn +import torch.nn.functional as F + +# from core.common.dnn.detection.yolox.yolox.utils import bboxes_iou + +from .losses import IOUloss +from .network_blocks import BaseConv, DWConv + + +def bboxes_iou(bboxes_a, bboxes_b, xyxy=True): + if bboxes_a.shape[1] != 4 or bboxes_b.shape[1] != 4: + raise IndexError + + if xyxy: + tl = torch.max(bboxes_a[:, None, :2], bboxes_b[:, :2]) + br = torch.min(bboxes_a[:, None, 2:], bboxes_b[:, 2:]) + area_a = torch.prod(bboxes_a[:, 2:] - bboxes_a[:, :2], 1) + area_b = torch.prod(bboxes_b[:, 2:] - bboxes_b[:, :2], 1) + else: + tl = torch.max( + (bboxes_a[:, None, :2] - bboxes_a[:, None, 2:] / 2), + (bboxes_b[:, :2] - bboxes_b[:, 2:] / 2), + ) + br = torch.min( + (bboxes_a[:, None, :2] + bboxes_a[:, None, 2:] / 2), + (bboxes_b[:, :2] + bboxes_b[:, 2:] / 2), + ) + + area_a = torch.prod(bboxes_a[:, 2:], 1) + area_b = torch.prod(bboxes_b[:, 2:], 1) + en = (tl < br).type(tl.type()).prod(dim=2) + area_i = torch.prod(br - tl, 2) * en # * ((tl < br).all()) + return area_i / (area_a[:, None] + area_b - area_i) + + +class YOLOXHead(nn.Module): + def __init__( + self, + num_classes, + width=1.0, + strides=[8, 16, 32], + in_channels=[256, 512, 1024], + act="silu", + depthwise=False, + ): + """ + Args: + act (str): activation type of conv. Defalut value: "silu". + depthwise (bool): whether apply depthwise conv in conv branch. Defalut value: False. + """ + super().__init__() + + self.n_anchors = 1 + self.num_classes = num_classes + self.decode_in_inference = True # for deploy, set to False + + self.cls_convs = nn.ModuleList() + self.reg_convs = nn.ModuleList() + self.cls_preds = nn.ModuleList() + self.reg_preds = nn.ModuleList() + self.obj_preds = nn.ModuleList() + self.stems = nn.ModuleList() + Conv = DWConv if depthwise else BaseConv + + for i in range(len(in_channels)): + self.stems.append( + BaseConv( + in_channels=int(in_channels[i] * width), + out_channels=int(256 * width), + ksize=1, + stride=1, + act=act, + ) + ) + self.cls_convs.append( + nn.Sequential( + *[ + Conv( + in_channels=int(256 * width), + out_channels=int(256 * width), + ksize=3, + stride=1, + act=act, + ), + Conv( + in_channels=int(256 * width), + out_channels=int(256 * width), + ksize=3, + stride=1, + act=act, + ), + ] + ) + ) + self.reg_convs.append( + nn.Sequential( + *[ + Conv( + in_channels=int(256 * width), + out_channels=int(256 * width), + ksize=3, + stride=1, + act=act, + ), + Conv( + in_channels=int(256 * width), + out_channels=int(256 * width), + ksize=3, + stride=1, + act=act, + ), + ] + ) + ) + self.cls_preds.append( + nn.Conv2d( + in_channels=int(256 * width), + out_channels=self.n_anchors * self.num_classes, + kernel_size=1, + stride=1, + padding=0, + ) + ) + self.reg_preds.append( + nn.Conv2d( + in_channels=int(256 * width), + out_channels=4, + kernel_size=1, + stride=1, + padding=0, + ) + ) + self.obj_preds.append( + nn.Conv2d( + in_channels=int(256 * width), + out_channels=self.n_anchors * 1, + kernel_size=1, + stride=1, + padding=0, + ) + ) + + self.use_l1 = False + self.l1_loss = nn.L1Loss(reduction="none") + self.bcewithlog_loss = nn.BCEWithLogitsLoss(reduction="none") + self.iou_loss = IOUloss(reduction="none") + self.strides = strides + self.grids = [torch.zeros(1)] * len(in_channels) + + def initialize_biases(self, prior_prob): + for conv in self.cls_preds: + b = conv.bias.view(self.n_anchors, -1) + b.data.fill_(-math.log((1 - prior_prob) / prior_prob)) + conv.bias = torch.nn.Parameter(b.view(-1), requires_grad=True) + + for conv in self.obj_preds: + b = conv.bias.view(self.n_anchors, -1) + b.data.fill_(-math.log((1 - prior_prob) / prior_prob)) + conv.bias = torch.nn.Parameter(b.view(-1), requires_grad=True) + + def forward(self, xin, labels=None, imgs=None): + if isinstance(xin, list): + xin, labels, imgs = xin[0], xin[1], xin[2] + + outputs = [] + origin_preds = [] + x_shifts = [] + y_shifts = [] + expanded_strides = [] + + for k, (cls_conv, reg_conv, stride_this_level, x) in enumerate( + zip(self.cls_convs, self.reg_convs, self.strides, xin) + ): + x = self.stems[k](x) + cls_x = x + reg_x = x + + cls_feat = cls_conv(cls_x) + cls_output = self.cls_preds[k](cls_feat) + # print('cls output size', cls_output.size()) + + reg_feat = reg_conv(reg_x) + reg_output = self.reg_preds[k](reg_feat) + obj_output = self.obj_preds[k](reg_feat) + + # if self.training: + if labels is not None: + output = torch.cat([reg_output, obj_output, cls_output], 1) + output, grid = self.get_output_and_grid( + output, k, stride_this_level, xin[0].type() + ) + x_shifts.append(grid[:, :, 0]) + y_shifts.append(grid[:, :, 1]) + expanded_strides.append( + torch.zeros(1, grid.shape[1]) + .fill_(stride_this_level) + .type_as(xin[0]) + ) + if self.use_l1: + batch_size = reg_output.shape[0] + hsize, wsize = reg_output.shape[-2:] + reg_output = reg_output.view( + batch_size, self.n_anchors, 4, hsize, wsize + ) + reg_output = reg_output.permute(0, 1, 3, 4, 2).reshape( + batch_size, -1, 4 + ) + origin_preds.append(reg_output.clone()) + + else: + output = torch.cat( + [reg_output, obj_output.sigmoid(), cls_output.sigmoid()], 1 + ) + + outputs.append(output) + # print(111) + # if self.training: + if labels is not None: + return self.get_losses( + imgs, + x_shifts, + y_shifts, + expanded_strides, + labels, + torch.cat(outputs, 1), + origin_preds, + dtype=xin[0].dtype, + ) + else: + self.hw = [x.shape[-2:] for x in outputs] + # [batch, n_anchors_all, 85] + outputs = torch.cat( + [x.flatten(start_dim=2) for x in outputs], dim=2 + ).permute(0, 2, 1) + if self.decode_in_inference: + return self.decode_outputs(outputs, dtype=xin[0].type()) + else: + # print(outputs.size()) + return outputs + + def get_output_and_grid(self, output, k, stride, dtype): + grid = self.grids[k] + + batch_size = output.shape[0] + n_ch = 5 + self.num_classes + hsize, wsize = output.shape[-2:] + if grid.shape[2:4] != output.shape[2:4]: + # yv, xv = torch.meshgrid([torch.arange(hsize), torch.arange(wsize)], indexing="ij") + yv, xv = torch.meshgrid([torch.arange(hsize), torch.arange(wsize)]) # fixed bu queyu, 2022/03/14 + grid = torch.stack((xv, yv), 2).view(1, 1, hsize, wsize, 2).type(dtype) + self.grids[k] = grid + + output = output.view(batch_size, self.n_anchors, n_ch, hsize, wsize) + output = output.permute(0, 1, 3, 4, 2).reshape( + batch_size, self.n_anchors * hsize * wsize, -1 + ) + grid = grid.view(1, -1, 2) + output[..., :2] = (output[..., :2] + grid) * stride + output[..., 2:4] = torch.exp(output[..., 2:4]) * stride + return output, grid + + def decode_outputs(self, outputs, dtype): + grids = [] + strides = [] + for (hsize, wsize), stride in zip(self.hw, self.strides): + # yv, xv = torch.meshgrid([torch.arange(hsize), torch.arange(wsize)], indexing="ij") + yv, xv = torch.meshgrid([torch.arange(hsize), torch.arange(wsize)]) # fixed by queyu, 2022/03/14 + grid = torch.stack((xv, yv), 2).view(1, -1, 2) + grids.append(grid) + shape = grid.shape[:2] + strides.append(torch.full((*shape, 1), stride)) + + grids = torch.cat(grids, dim=1).type(dtype) + strides = torch.cat(strides, dim=1).type(dtype) + + outputs[..., :2] = (outputs[..., :2] + grids) * strides + outputs[..., 2:4] = torch.exp(outputs[..., 2:4]) * strides + return outputs + + def get_losses( + self, + imgs, + x_shifts, + y_shifts, + expanded_strides, + labels, + outputs, + origin_preds, + dtype, + ): + bbox_preds = outputs[:, :, :4] # [batch, n_anchors_all, 4] + obj_preds = outputs[:, :, 4].unsqueeze(-1) # [batch, n_anchors_all, 1] + cls_preds = outputs[:, :, 5:] # [batch, n_anchors_all, n_cls] + + # calculate targets + nlabel = (labels.sum(dim=2) > 0).sum(dim=1) # number of objects + + total_num_anchors = outputs.shape[1] + x_shifts = torch.cat(x_shifts, 1) # [1, n_anchors_all] + y_shifts = torch.cat(y_shifts, 1) # [1, n_anchors_all] + expanded_strides = torch.cat(expanded_strides, 1) + if self.use_l1: + origin_preds = torch.cat(origin_preds, 1) + + cls_targets = [] + reg_targets = [] + l1_targets = [] + obj_targets = [] + fg_masks = [] + + num_fg = 0.0 + num_gts = 0.0 + + for batch_idx in range(outputs.shape[0]): + num_gt = int(nlabel[batch_idx]) + num_gts += num_gt + if num_gt == 0: + cls_target = outputs.new_zeros((0, self.num_classes)) + reg_target = outputs.new_zeros((0, 4)) + l1_target = outputs.new_zeros((0, 4)) + obj_target = outputs.new_zeros((total_num_anchors, 1)) + fg_mask = outputs.new_zeros(total_num_anchors).bool() + else: + gt_bboxes_per_image = labels[batch_idx, :num_gt, 1:5] + gt_classes = labels[batch_idx, :num_gt, 0] + bboxes_preds_per_image = bbox_preds[batch_idx] + + try: + ( + gt_matched_classes, + fg_mask, + pred_ious_this_matching, + matched_gt_inds, + num_fg_img, + ) = self.get_assignments( # noqa + batch_idx, + num_gt, + total_num_anchors, + gt_bboxes_per_image, + gt_classes, + bboxes_preds_per_image, + expanded_strides, + x_shifts, + y_shifts, + cls_preds, + bbox_preds, + obj_preds, + labels, + imgs, + ) + except RuntimeError: + logger.error( + "OOM RuntimeError is raised due to the huge memory cost during label assignment. \ + CPU mode is applied in this batch. If you want to avoid this issue, \ + try to reduce the batch size or image size." + ) + torch.cuda.empty_cache() + ( + gt_matched_classes, + fg_mask, + pred_ious_this_matching, + matched_gt_inds, + num_fg_img, + ) = self.get_assignments( # noqa + batch_idx, + num_gt, + total_num_anchors, + gt_bboxes_per_image, + gt_classes, + bboxes_preds_per_image, + expanded_strides, + x_shifts, + y_shifts, + cls_preds, + bbox_preds, + obj_preds, + labels, + imgs, + "cpu", + ) + + torch.cuda.empty_cache() + num_fg += num_fg_img + + cls_target = F.one_hot( + gt_matched_classes.to(torch.int64), self.num_classes + ) * pred_ious_this_matching.unsqueeze(-1) + obj_target = fg_mask.unsqueeze(-1) + reg_target = gt_bboxes_per_image[matched_gt_inds] + if self.use_l1: + l1_target = self.get_l1_target( + outputs.new_zeros((num_fg_img, 4)), + gt_bboxes_per_image[matched_gt_inds], + expanded_strides[0][fg_mask], + x_shifts=x_shifts[0][fg_mask], + y_shifts=y_shifts[0][fg_mask], + ) + + cls_targets.append(cls_target) + reg_targets.append(reg_target) + obj_targets.append(obj_target.to(dtype)) + fg_masks.append(fg_mask) + if self.use_l1: + l1_targets.append(l1_target) + + cls_targets = torch.cat(cls_targets, 0) + reg_targets = torch.cat(reg_targets, 0) + obj_targets = torch.cat(obj_targets, 0) + fg_masks = torch.cat(fg_masks, 0) + if self.use_l1: + l1_targets = torch.cat(l1_targets, 0) + + num_fg = max(num_fg, 1) + loss_iou = ( + self.iou_loss(bbox_preds.view(-1, 4)[fg_masks], reg_targets) + ).sum() / num_fg + loss_obj = ( + self.bcewithlog_loss(obj_preds.view(-1, 1), obj_targets) + ).sum() / num_fg + loss_cls = ( + self.bcewithlog_loss( + cls_preds.view(-1, self.num_classes)[fg_masks], cls_targets + ) + ).sum() / num_fg + if self.use_l1: + loss_l1 = ( + self.l1_loss(origin_preds.view(-1, 4)[fg_masks], l1_targets) + ).sum() / num_fg + else: + loss_l1 = 0.0 + + reg_weight = 5.0 + loss = reg_weight * loss_iou + loss_obj + loss_cls + loss_l1 + + return ( + loss, + reg_weight * loss_iou, + loss_obj, + loss_cls, + loss_l1, + num_fg / max(num_gts, 1), + ) + + def get_l1_target(self, l1_target, gt, stride, x_shifts, y_shifts, eps=1e-8): + l1_target[:, 0] = gt[:, 0] / stride - x_shifts + l1_target[:, 1] = gt[:, 1] / stride - y_shifts + l1_target[:, 2] = torch.log(gt[:, 2] / stride + eps) + l1_target[:, 3] = torch.log(gt[:, 3] / stride + eps) + return l1_target + + @torch.no_grad() + def get_assignments( + self, + batch_idx, + num_gt, + total_num_anchors, + gt_bboxes_per_image, + gt_classes, + bboxes_preds_per_image, + expanded_strides, + x_shifts, + y_shifts, + cls_preds, + bbox_preds, + obj_preds, + labels, + imgs, + mode="gpu", + ): + + if mode == "cpu": + print("------------CPU Mode for This Batch-------------") + gt_bboxes_per_image = gt_bboxes_per_image.cpu().float() + bboxes_preds_per_image = bboxes_preds_per_image.cpu().float() + gt_classes = gt_classes.cpu().float() + expanded_strides = expanded_strides.cpu().float() + x_shifts = x_shifts.cpu() + y_shifts = y_shifts.cpu() + + fg_mask, is_in_boxes_and_center = self.get_in_boxes_info( + gt_bboxes_per_image, + expanded_strides, + x_shifts, + y_shifts, + total_num_anchors, + num_gt, + ) + + bboxes_preds_per_image = bboxes_preds_per_image[fg_mask] + cls_preds_ = cls_preds[batch_idx][fg_mask] + obj_preds_ = obj_preds[batch_idx][fg_mask] + num_in_boxes_anchor = bboxes_preds_per_image.shape[0] + + if mode == "cpu": + gt_bboxes_per_image = gt_bboxes_per_image.cpu() + bboxes_preds_per_image = bboxes_preds_per_image.cpu() + + pair_wise_ious = bboxes_iou(gt_bboxes_per_image, bboxes_preds_per_image, False) + + gt_cls_per_image = ( + F.one_hot(gt_classes.to(torch.int64), self.num_classes) + .float() + .unsqueeze(1) + .repeat(1, num_in_boxes_anchor, 1) + ) + pair_wise_ious_loss = -torch.log(pair_wise_ious + 1e-8) + + if mode == "cpu": + cls_preds_, obj_preds_ = cls_preds_.cpu(), obj_preds_.cpu() + + with torch.cuda.amp.autocast(enabled=False): + cls_preds_ = ( + cls_preds_.float().unsqueeze(0).repeat(num_gt, 1, 1).sigmoid_() + * obj_preds_.float().unsqueeze(0).repeat(num_gt, 1, 1).sigmoid_() + ) + pair_wise_cls_loss = F.binary_cross_entropy( + cls_preds_.sqrt_(), gt_cls_per_image, reduction="none" + ).sum(-1) + del cls_preds_ + + cost = ( + pair_wise_cls_loss + + 3.0 * pair_wise_ious_loss + + 100000.0 * (~is_in_boxes_and_center) + ) + + ( + num_fg, + gt_matched_classes, + pred_ious_this_matching, + matched_gt_inds, + ) = self.dynamic_k_matching(cost, pair_wise_ious, gt_classes, num_gt, fg_mask) + del pair_wise_cls_loss, cost, pair_wise_ious, pair_wise_ious_loss + + if mode == "cpu": + gt_matched_classes = gt_matched_classes.cuda() + fg_mask = fg_mask.cuda() + pred_ious_this_matching = pred_ious_this_matching.cuda() + matched_gt_inds = matched_gt_inds.cuda() + + return ( + gt_matched_classes, + fg_mask, + pred_ious_this_matching, + matched_gt_inds, + num_fg, + ) + + def get_in_boxes_info( + self, + gt_bboxes_per_image, + expanded_strides, + x_shifts, + y_shifts, + total_num_anchors, + num_gt, + ): + expanded_strides_per_image = expanded_strides[0] + x_shifts_per_image = x_shifts[0] * expanded_strides_per_image + y_shifts_per_image = y_shifts[0] * expanded_strides_per_image + x_centers_per_image = ( + (x_shifts_per_image + 0.5 * expanded_strides_per_image) + .unsqueeze(0) + .repeat(num_gt, 1) + ) # [n_anchor] -> [n_gt, n_anchor] + y_centers_per_image = ( + (y_shifts_per_image + 0.5 * expanded_strides_per_image) + .unsqueeze(0) + .repeat(num_gt, 1) + ) + + gt_bboxes_per_image_l = ( + (gt_bboxes_per_image[:, 0] - 0.5 * gt_bboxes_per_image[:, 2]) + .unsqueeze(1) + .repeat(1, total_num_anchors) + ) + gt_bboxes_per_image_r = ( + (gt_bboxes_per_image[:, 0] + 0.5 * gt_bboxes_per_image[:, 2]) + .unsqueeze(1) + .repeat(1, total_num_anchors) + ) + gt_bboxes_per_image_t = ( + (gt_bboxes_per_image[:, 1] - 0.5 * gt_bboxes_per_image[:, 3]) + .unsqueeze(1) + .repeat(1, total_num_anchors) + ) + gt_bboxes_per_image_b = ( + (gt_bboxes_per_image[:, 1] + 0.5 * gt_bboxes_per_image[:, 3]) + .unsqueeze(1) + .repeat(1, total_num_anchors) + ) + + b_l = x_centers_per_image - gt_bboxes_per_image_l + b_r = gt_bboxes_per_image_r - x_centers_per_image + b_t = y_centers_per_image - gt_bboxes_per_image_t + b_b = gt_bboxes_per_image_b - y_centers_per_image + bbox_deltas = torch.stack([b_l, b_t, b_r, b_b], 2) + + is_in_boxes = bbox_deltas.min(dim=-1).values > 0.0 + is_in_boxes_all = is_in_boxes.sum(dim=0) > 0 + # in fixed center + + center_radius = 2.5 + + gt_bboxes_per_image_l = (gt_bboxes_per_image[:, 0]).unsqueeze(1).repeat( + 1, total_num_anchors + ) - center_radius * expanded_strides_per_image.unsqueeze(0) + gt_bboxes_per_image_r = (gt_bboxes_per_image[:, 0]).unsqueeze(1).repeat( + 1, total_num_anchors + ) + center_radius * expanded_strides_per_image.unsqueeze(0) + gt_bboxes_per_image_t = (gt_bboxes_per_image[:, 1]).unsqueeze(1).repeat( + 1, total_num_anchors + ) - center_radius * expanded_strides_per_image.unsqueeze(0) + gt_bboxes_per_image_b = (gt_bboxes_per_image[:, 1]).unsqueeze(1).repeat( + 1, total_num_anchors + ) + center_radius * expanded_strides_per_image.unsqueeze(0) + + c_l = x_centers_per_image - gt_bboxes_per_image_l + c_r = gt_bboxes_per_image_r - x_centers_per_image + c_t = y_centers_per_image - gt_bboxes_per_image_t + c_b = gt_bboxes_per_image_b - y_centers_per_image + center_deltas = torch.stack([c_l, c_t, c_r, c_b], 2) + is_in_centers = center_deltas.min(dim=-1).values > 0.0 + is_in_centers_all = is_in_centers.sum(dim=0) > 0 + + # in boxes and in centers + is_in_boxes_anchor = is_in_boxes_all | is_in_centers_all + + is_in_boxes_and_center = ( + is_in_boxes[:, is_in_boxes_anchor] & is_in_centers[:, is_in_boxes_anchor] + ) + return is_in_boxes_anchor, is_in_boxes_and_center + + def dynamic_k_matching(self, cost, pair_wise_ious, gt_classes, num_gt, fg_mask): + # Dynamic K + # --------------------------------------------------------------- + matching_matrix = torch.zeros_like(cost, dtype=torch.uint8) + + ious_in_boxes_matrix = pair_wise_ious + n_candidate_k = min(10, ious_in_boxes_matrix.size(1)) + topk_ious, _ = torch.topk(ious_in_boxes_matrix, n_candidate_k, dim=1) + dynamic_ks = torch.clamp(topk_ious.sum(1).int(), min=1) + dynamic_ks = dynamic_ks.tolist() + for gt_idx in range(num_gt): + _, pos_idx = torch.topk( + cost[gt_idx], k=dynamic_ks[gt_idx], largest=False + ) + matching_matrix[gt_idx][pos_idx] = 1 + + del topk_ious, dynamic_ks, pos_idx + + anchor_matching_gt = matching_matrix.sum(0) + if (anchor_matching_gt > 1).sum() > 0: + _, cost_argmin = torch.min(cost[:, anchor_matching_gt > 1], dim=0) + matching_matrix[:, anchor_matching_gt > 1] *= 0 + matching_matrix[cost_argmin, anchor_matching_gt > 1] = 1 + fg_mask_inboxes = matching_matrix.sum(0) > 0 + num_fg = fg_mask_inboxes.sum().item() + + fg_mask[fg_mask.clone()] = fg_mask_inboxes + + matched_gt_inds = matching_matrix[:, fg_mask_inboxes].argmax(0) + gt_matched_classes = gt_classes[matched_gt_inds] + + pred_ious_this_matching = (matching_matrix * pair_wise_ious).sum(0)[ + fg_mask_inboxes + ] + return num_fg, gt_matched_classes, pred_ious_this_matching, matched_gt_inds diff --git a/dnns/yolov3/losses.py b/dnns/yolov3/losses.py new file mode 100644 index 0000000000000000000000000000000000000000..77b4d8ef7660880031f4ef23c82ba3a85b6fd254 --- /dev/null +++ b/dnns/yolov3/losses.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +# Copyright (c) Megvii Inc. All rights reserved. + +import torch +import torch.nn as nn + + +class IOUloss(nn.Module): + def __init__(self, reduction="none", loss_type="iou"): + super(IOUloss, self).__init__() + self.reduction = reduction + self.loss_type = loss_type + + def forward(self, pred, target): + assert pred.shape[0] == target.shape[0] + + pred = pred.view(-1, 4) + target = target.view(-1, 4) + tl = torch.max( + (pred[:, :2] - pred[:, 2:] / 2), (target[:, :2] - target[:, 2:] / 2) + ) + br = torch.min( + (pred[:, :2] + pred[:, 2:] / 2), (target[:, :2] + target[:, 2:] / 2) + ) + + area_p = torch.prod(pred[:, 2:], 1) + area_g = torch.prod(target[:, 2:], 1) + + en = (tl < br).type(tl.type()).prod(dim=1) + area_i = torch.prod(br - tl, 1) * en + area_u = area_p + area_g - area_i + iou = (area_i) / (area_u + 1e-16) + + if self.loss_type == "iou": + loss = 1 - iou ** 2 + elif self.loss_type == "giou": + c_tl = torch.min( + (pred[:, :2] - pred[:, 2:] / 2), (target[:, :2] - target[:, 2:] / 2) + ) + c_br = torch.max( + (pred[:, :2] + pred[:, 2:] / 2), (target[:, :2] + target[:, 2:] / 2) + ) + area_c = torch.prod(c_br - c_tl, 1) + giou = iou - (area_c - area_u) / area_c.clamp(1e-16) + loss = 1 - giou.clamp(min=-1.0, max=1.0) + + if self.reduction == "mean": + loss = loss.mean() + elif self.reduction == "sum": + loss = loss.sum() + + return loss diff --git a/dnns/yolov3/network_blocks.py b/dnns/yolov3/network_blocks.py new file mode 100644 index 0000000000000000000000000000000000000000..68aacfc33208eab072422e0647742006984dfdfd --- /dev/null +++ b/dnns/yolov3/network_blocks.py @@ -0,0 +1,210 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +# Copyright (c) Megvii Inc. All rights reserved. + +import torch +import torch.nn as nn + + +class SiLU(nn.Module): + """export-friendly version of nn.SiLU()""" + + @staticmethod + def forward(x): + return x * torch.sigmoid(x) + + +def get_activation(name="silu", inplace=True): + if name == "silu": + module = nn.SiLU(inplace=inplace) + elif name == "relu": + module = nn.ReLU(inplace=inplace) + elif name == "lrelu": + module = nn.LeakyReLU(0.1, inplace=inplace) + else: + raise AttributeError("Unsupported act type: {}".format(name)) + return module + + +class BaseConv(nn.Module): + """A Conv2d -> Batchnorm -> silu/leaky relu block""" + + def __init__( + self, in_channels, out_channels, ksize, stride, groups=1, bias=False, act="silu" + ): + super().__init__() + # same padding + pad = (ksize - 1) // 2 + self.conv = nn.Conv2d( + in_channels, + out_channels, + kernel_size=ksize, + stride=stride, + padding=pad, + groups=groups, + bias=bias, + ) + self.bn = nn.BatchNorm2d(out_channels) + self.act = get_activation(act, inplace=True) + + def forward(self, x): + return self.act(self.bn(self.conv(x))) + + def fuseforward(self, x): + return self.act(self.conv(x)) + + +class DWConv(nn.Module): + """Depthwise Conv + Conv""" + + def __init__(self, in_channels, out_channels, ksize, stride=1, act="silu"): + super().__init__() + self.dconv = BaseConv( + in_channels, + in_channels, + ksize=ksize, + stride=stride, + groups=in_channels, + act=act, + ) + self.pconv = BaseConv( + in_channels, out_channels, ksize=1, stride=1, groups=1, act=act + ) + + def forward(self, x): + x = self.dconv(x) + return self.pconv(x) + + +class Bottleneck(nn.Module): + # Standard bottleneck + def __init__( + self, + in_channels, + out_channels, + shortcut=True, + expansion=0.5, + depthwise=False, + act="silu", + ): + super().__init__() + hidden_channels = int(out_channels * expansion) + Conv = DWConv if depthwise else BaseConv + self.conv1 = BaseConv(in_channels, hidden_channels, 1, stride=1, act=act) + self.conv2 = Conv(hidden_channels, out_channels, 3, stride=1, act=act) + self.use_add = shortcut and in_channels == out_channels + + def forward(self, x): + y = self.conv2(self.conv1(x)) + if self.use_add: + y = y + x + return y + + +class ResLayer(nn.Module): + "Residual layer with `in_channels` inputs." + + def __init__(self, in_channels: int): + super().__init__() + mid_channels = in_channels // 2 + self.layer1 = BaseConv( + in_channels, mid_channels, ksize=1, stride=1, act="lrelu" + ) + self.layer2 = BaseConv( + mid_channels, in_channels, ksize=3, stride=1, act="lrelu" + ) + + def forward(self, x): + out = self.layer2(self.layer1(x)) + return x + out + + +class SPPBottleneck(nn.Module): + """Spatial pyramid pooling layer used in YOLOv3-SPP""" + + def __init__( + self, in_channels, out_channels, kernel_sizes=(5, 9, 13), activation="silu" + ): + super().__init__() + hidden_channels = in_channels // 2 + self.conv1 = BaseConv(in_channels, hidden_channels, 1, stride=1, act=activation) + self.m = nn.ModuleList( + [ + nn.MaxPool2d(kernel_size=ks, stride=1, padding=ks // 2) + for ks in kernel_sizes + ] + ) + conv2_channels = hidden_channels * (len(kernel_sizes) + 1) + self.conv2 = BaseConv(conv2_channels, out_channels, 1, stride=1, act=activation) + + def forward(self, x): + x = self.conv1(x) + x = torch.cat([x] + [m(x) for m in self.m], dim=1) + x = self.conv2(x) + return x + + +class CSPLayer(nn.Module): + """C3 in yolov5, CSP Bottleneck with 3 convolutions""" + + def __init__( + self, + in_channels, + out_channels, + n=1, + shortcut=True, + expansion=0.5, + depthwise=False, + act="silu", + ): + """ + Args: + in_channels (int): input channels. + out_channels (int): output channels. + n (int): number of Bottlenecks. Default value: 1. + """ + # ch_in, ch_out, number, shortcut, groups, expansion + super().__init__() + hidden_channels = int(out_channels * expansion) # hidden channels + self.conv1 = BaseConv(in_channels, hidden_channels, 1, stride=1, act=act) + self.conv2 = BaseConv(in_channels, hidden_channels, 1, stride=1, act=act) + self.conv3 = BaseConv(2 * hidden_channels, out_channels, 1, stride=1, act=act) + module_list = [ + Bottleneck( + hidden_channels, hidden_channels, shortcut, 1.0, depthwise, act=act + ) + for _ in range(n) + ] + self.m = nn.Sequential(*module_list) + + def forward(self, x): + x_1 = self.conv1(x) + x_2 = self.conv2(x) + x_1 = self.m(x_1) + x = torch.cat((x_1, x_2), dim=1) + return self.conv3(x) + + +class Focus(nn.Module): + """Focus width and height information into channel space.""" + + def __init__(self, in_channels, out_channels, ksize=1, stride=1, act="silu"): + super().__init__() + self.conv = BaseConv(in_channels * 4, out_channels, ksize, stride, act=act) + + def forward(self, x): + # shape of x (b,c,w,h) -> y(b,4c,w/2,h/2) + patch_top_left = x[..., ::2, ::2] + patch_top_right = x[..., ::2, 1::2] + patch_bot_left = x[..., 1::2, ::2] + patch_bot_right = x[..., 1::2, 1::2] + x = torch.cat( + ( + patch_top_left, + patch_bot_left, + patch_top_right, + patch_bot_right, + ), + dim=1, + ) + return self.conv(x) diff --git a/dnns/yolov3/utils/__init__.py b/dnns/yolov3/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..ff8db0dbc2473d2ee7c2b9ad8261183171f08ab1 --- /dev/null +++ b/dnns/yolov3/utils/__init__.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +# Copyright (c) Megvii Inc. All rights reserved. + +from .allreduce_norm import * +from .boxes import * +from .checkpoint import load_ckpt, save_checkpoint +from .demo_utils import * +from .dist import * +from .ema import * +from .logger import WandbLogger, setup_logger +from .lr_scheduler import LRScheduler +from .metric import * +from .model_utils import * +from .setup_env import * +from .visualize import * diff --git a/dnns/yolov3/utils/allreduce_norm.py b/dnns/yolov3/utils/allreduce_norm.py new file mode 100644 index 0000000000000000000000000000000000000000..142c76c78061db6e2c5f4b899bcc5e2f2214f010 --- /dev/null +++ b/dnns/yolov3/utils/allreduce_norm.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +# Copyright (c) Megvii Inc. All rights reserved. + +import pickle +from collections import OrderedDict + +import torch +from torch import distributed as dist +from torch import nn + +from .dist import _get_global_gloo_group, get_world_size + +ASYNC_NORM = ( + nn.BatchNorm1d, + nn.BatchNorm2d, + nn.BatchNorm3d, + nn.InstanceNorm1d, + nn.InstanceNorm2d, + nn.InstanceNorm3d, +) + +__all__ = [ + "get_async_norm_states", + "pyobj2tensor", + "tensor2pyobj", + "all_reduce", + "all_reduce_norm", +] + + +def get_async_norm_states(module): + async_norm_states = OrderedDict() + for name, child in module.named_modules(): + if isinstance(child, ASYNC_NORM): + for k, v in child.state_dict().items(): + async_norm_states[".".join([name, k])] = v + return async_norm_states + + +def pyobj2tensor(pyobj, device="cuda"): + """serialize picklable python object to tensor""" + storage = torch.ByteStorage.from_buffer(pickle.dumps(pyobj)) + return torch.ByteTensor(storage).to(device=device) + + +def tensor2pyobj(tensor): + """deserialize tensor to picklable python object""" + return pickle.loads(tensor.cpu().numpy().tobytes()) + + +def _get_reduce_op(op_name): + return { + "sum": dist.ReduceOp.SUM, + "mean": dist.ReduceOp.SUM, + }[op_name.lower()] + + +def all_reduce(py_dict, op="sum", group=None): + """ + Apply all reduce function for python dict object. + NOTE: make sure that every py_dict has the same keys and values are in the same shape. + + Args: + py_dict (dict): dict to apply all reduce op. + op (str): operator, could be "sum" or "mean". + """ + world_size = get_world_size() + if world_size == 1: + return py_dict + if group is None: + group = _get_global_gloo_group() + if dist.get_world_size(group) == 1: + return py_dict + + # all reduce logic across different devices. + py_key = list(py_dict.keys()) + py_key_tensor = pyobj2tensor(py_key) + dist.broadcast(py_key_tensor, src=0) + py_key = tensor2pyobj(py_key_tensor) + + tensor_shapes = [py_dict[k].shape for k in py_key] + tensor_numels = [py_dict[k].numel() for k in py_key] + + flatten_tensor = torch.cat([py_dict[k].flatten() for k in py_key]) + dist.all_reduce(flatten_tensor, op=_get_reduce_op(op)) + if op == "mean": + flatten_tensor /= world_size + + split_tensors = [ + x.reshape(shape) + for x, shape in zip(torch.split(flatten_tensor, tensor_numels), tensor_shapes) + ] + return OrderedDict({k: v for k, v in zip(py_key, split_tensors)}) + + +def all_reduce_norm(module): + """ + All reduce norm statistics in different devices. + """ + states = get_async_norm_states(module) + states = all_reduce(states, op="mean") + module.load_state_dict(states, strict=False) diff --git a/dnns/yolov3/utils/boxes.py b/dnns/yolov3/utils/boxes.py new file mode 100644 index 0000000000000000000000000000000000000000..a8287a2c243be235cd6504a3b375bde48d3270da --- /dev/null +++ b/dnns/yolov3/utils/boxes.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +# Copyright (c) Megvii Inc. All rights reserved. + +import numpy as np + +import torch +import torchvision + +__all__ = [ + "filter_box", + "postprocess", + "bboxes_iou", + "matrix_iou", + "adjust_box_anns", + "xyxy2xywh", + "xyxy2cxcywh", +] + + +def filter_box(output, scale_range): + """ + output: (N, 5+class) shape + """ + min_scale, max_scale = scale_range + w = output[:, 2] - output[:, 0] + h = output[:, 3] - output[:, 1] + keep = (w * h > min_scale * min_scale) & (w * h < max_scale * max_scale) + return output[keep] + + +def postprocess(prediction, num_classes, conf_thre=0.7, nms_thre=0.45, class_agnostic=False): + box_corner = prediction.new(prediction.shape) + box_corner[:, :, 0] = prediction[:, :, 0] - prediction[:, :, 2] / 2 + box_corner[:, :, 1] = prediction[:, :, 1] - prediction[:, :, 3] / 2 + box_corner[:, :, 2] = prediction[:, :, 0] + prediction[:, :, 2] / 2 + box_corner[:, :, 3] = prediction[:, :, 1] + prediction[:, :, 3] / 2 + prediction[:, :, :4] = box_corner[:, :, :4] + + output = [None for _ in range(len(prediction))] + for i, image_pred in enumerate(prediction): + + # If none are remaining => process next image + if not image_pred.size(0): + continue + # Get score and class with highest confidence + class_conf, class_pred = torch.max(image_pred[:, 5: 5 + num_classes], 1, keepdim=True) + + # implemented by queyu, 2022/08/08 + # print(class_pred) + # class_pred -= y_offset + # class_pred[class_pred < 0] = 0 + # print(class_pred) + # exit(0) + + conf_mask = (image_pred[:, 4] * class_conf.squeeze() >= conf_thre).squeeze() + # Detections ordered as (x1, y1, x2, y2, obj_conf, class_conf, class_pred) + detections = torch.cat((image_pred[:, :5], class_conf, class_pred.float()), 1) + detections = detections[conf_mask] + if not detections.size(0): + continue + + if class_agnostic: + nms_out_index = torchvision.ops.nms( + detections[:, :4], + detections[:, 4] * detections[:, 5], + nms_thre, + ) + else: + nms_out_index = torchvision.ops.batched_nms( + detections[:, :4], + detections[:, 4] * detections[:, 5], + detections[:, 6], + nms_thre, + ) + + detections = detections[nms_out_index] + if output[i] is None: + output[i] = detections + else: + output[i] = torch.cat((output[i], detections)) + + return output + + +def bboxes_iou(bboxes_a, bboxes_b, xyxy=True): + if bboxes_a.shape[1] != 4 or bboxes_b.shape[1] != 4: + raise IndexError + + if xyxy: + tl = torch.max(bboxes_a[:, None, :2], bboxes_b[:, :2]) + br = torch.min(bboxes_a[:, None, 2:], bboxes_b[:, 2:]) + area_a = torch.prod(bboxes_a[:, 2:] - bboxes_a[:, :2], 1) + area_b = torch.prod(bboxes_b[:, 2:] - bboxes_b[:, :2], 1) + else: + tl = torch.max( + (bboxes_a[:, None, :2] - bboxes_a[:, None, 2:] / 2), + (bboxes_b[:, :2] - bboxes_b[:, 2:] / 2), + ) + br = torch.min( + (bboxes_a[:, None, :2] + bboxes_a[:, None, 2:] / 2), + (bboxes_b[:, :2] + bboxes_b[:, 2:] / 2), + ) + + area_a = torch.prod(bboxes_a[:, 2:], 1) + area_b = torch.prod(bboxes_b[:, 2:], 1) + en = (tl < br).type(tl.type()).prod(dim=2) + area_i = torch.prod(br - tl, 2) * en # * ((tl < br).all()) + return area_i / (area_a[:, None] + area_b - area_i) + + +def matrix_iou(a, b): + """ + return iou of a and b, numpy version for data augenmentation + """ + lt = np.maximum(a[:, np.newaxis, :2], b[:, :2]) + rb = np.minimum(a[:, np.newaxis, 2:], b[:, 2:]) + + area_i = np.prod(rb - lt, axis=2) * (lt < rb).all(axis=2) + area_a = np.prod(a[:, 2:] - a[:, :2], axis=1) + area_b = np.prod(b[:, 2:] - b[:, :2], axis=1) + return area_i / (area_a[:, np.newaxis] + area_b - area_i + 1e-12) + + +def adjust_box_anns(bbox, scale_ratio, padw, padh, w_max, h_max): + bbox[:, 0::2] = np.clip(bbox[:, 0::2] * scale_ratio + padw, 0, w_max) + bbox[:, 1::2] = np.clip(bbox[:, 1::2] * scale_ratio + padh, 0, h_max) + return bbox + + +def xyxy2xywh(bboxes): + bboxes[:, 2] = bboxes[:, 2] - bboxes[:, 0] + bboxes[:, 3] = bboxes[:, 3] - bboxes[:, 1] + return bboxes + + +def xyxy2cxcywh(bboxes): + bboxes[:, 2] = bboxes[:, 2] - bboxes[:, 0] + bboxes[:, 3] = bboxes[:, 3] - bboxes[:, 1] + bboxes[:, 0] = bboxes[:, 0] + bboxes[:, 2] * 0.5 + bboxes[:, 1] = bboxes[:, 1] + bboxes[:, 3] * 0.5 + return bboxes diff --git a/dnns/yolov3/utils/checkpoint.py b/dnns/yolov3/utils/checkpoint.py new file mode 100644 index 0000000000000000000000000000000000000000..a0c200e41da9ad8b720369a2181c9642724622ca --- /dev/null +++ b/dnns/yolov3/utils/checkpoint.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +# Copyright (c) Megvii Inc. All rights reserved. +import os +import shutil +from loguru import logger + +import torch + + +def load_ckpt(model, ckpt): + model_state_dict = model.state_dict() + load_dict = {} + for key_model, v in model_state_dict.items(): + if key_model not in ckpt: + logger.warning( + "{} is not in the ckpt. Please double check and see if this is desired.".format( + key_model + ) + ) + continue + v_ckpt = ckpt[key_model] + if v.shape != v_ckpt.shape: + logger.warning( + "Shape of {} in checkpoint is {}, while shape of {} in model is {}.".format( + key_model, v_ckpt.shape, key_model, v.shape + ) + ) + continue + load_dict[key_model] = v_ckpt + + model.load_state_dict(load_dict, strict=False) + return model + + +def save_checkpoint(state, is_best, save_dir, model_name=""): + if not os.path.exists(save_dir): + os.makedirs(save_dir) + filename = os.path.join(save_dir, model_name + "_ckpt.pth") + torch.save(state, filename) + if is_best: + best_filename = os.path.join(save_dir, "best_ckpt.pth") + shutil.copyfile(filename, best_filename) diff --git a/dnns/yolov3/utils/demo_utils.py b/dnns/yolov3/utils/demo_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..71222379497bd4a57d464afb63baebe43e9c447b --- /dev/null +++ b/dnns/yolov3/utils/demo_utils.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +# Copyright (c) Megvii Inc. All rights reserved. + +import os + +import numpy as np + +__all__ = ["mkdir", "nms", "multiclass_nms", "demo_postprocess"] + + +def mkdir(path): + if not os.path.exists(path): + os.makedirs(path) + + +def nms(boxes, scores, nms_thr): + """Single class NMS implemented in Numpy.""" + x1 = boxes[:, 0] + y1 = boxes[:, 1] + x2 = boxes[:, 2] + y2 = boxes[:, 3] + + areas = (x2 - x1 + 1) * (y2 - y1 + 1) + order = scores.argsort()[::-1] + + keep = [] + while order.size > 0: + i = order[0] + keep.append(i) + xx1 = np.maximum(x1[i], x1[order[1:]]) + yy1 = np.maximum(y1[i], y1[order[1:]]) + xx2 = np.minimum(x2[i], x2[order[1:]]) + yy2 = np.minimum(y2[i], y2[order[1:]]) + + w = np.maximum(0.0, xx2 - xx1 + 1) + h = np.maximum(0.0, yy2 - yy1 + 1) + inter = w * h + ovr = inter / (areas[i] + areas[order[1:]] - inter) + + inds = np.where(ovr <= nms_thr)[0] + order = order[inds + 1] + + return keep + + +def multiclass_nms(boxes, scores, nms_thr, score_thr, class_agnostic=True): + """Multiclass NMS implemented in Numpy""" + if class_agnostic: + nms_method = multiclass_nms_class_agnostic + else: + nms_method = multiclass_nms_class_aware + return nms_method(boxes, scores, nms_thr, score_thr) + + +def multiclass_nms_class_aware(boxes, scores, nms_thr, score_thr): + """Multiclass NMS implemented in Numpy. Class-aware version.""" + final_dets = [] + num_classes = scores.shape[1] + for cls_ind in range(num_classes): + cls_scores = scores[:, cls_ind] + valid_score_mask = cls_scores > score_thr + if valid_score_mask.sum() == 0: + continue + else: + valid_scores = cls_scores[valid_score_mask] + valid_boxes = boxes[valid_score_mask] + keep = nms(valid_boxes, valid_scores, nms_thr) + if len(keep) > 0: + cls_inds = np.ones((len(keep), 1)) * cls_ind + dets = np.concatenate( + [valid_boxes[keep], valid_scores[keep, None], cls_inds], 1 + ) + final_dets.append(dets) + if len(final_dets) == 0: + return None + return np.concatenate(final_dets, 0) + + +def multiclass_nms_class_agnostic(boxes, scores, nms_thr, score_thr): + """Multiclass NMS implemented in Numpy. Class-agnostic version.""" + cls_inds = scores.argmax(1) + cls_scores = scores[np.arange(len(cls_inds)), cls_inds] + + valid_score_mask = cls_scores > score_thr + if valid_score_mask.sum() == 0: + return None + valid_scores = cls_scores[valid_score_mask] + valid_boxes = boxes[valid_score_mask] + valid_cls_inds = cls_inds[valid_score_mask] + keep = nms(valid_boxes, valid_scores, nms_thr) + if keep: + dets = np.concatenate( + [valid_boxes[keep], valid_scores[keep, None], valid_cls_inds[keep, None]], 1 + ) + return dets + + +def demo_postprocess(outputs, img_size, p6=False): + + grids = [] + expanded_strides = [] + + if not p6: + strides = [8, 16, 32] + else: + strides = [8, 16, 32, 64] + + hsizes = [img_size[0] // stride for stride in strides] + wsizes = [img_size[1] // stride for stride in strides] + + for hsize, wsize, stride in zip(hsizes, wsizes, strides): + xv, yv = np.meshgrid(np.arange(wsize), np.arange(hsize)) + grid = np.stack((xv, yv), 2).reshape(1, -1, 2) + grids.append(grid) + shape = grid.shape[:2] + expanded_strides.append(np.full((*shape, 1), stride)) + + grids = np.concatenate(grids, 1) + expanded_strides = np.concatenate(expanded_strides, 1) + outputs[..., :2] = (outputs[..., :2] + grids) * expanded_strides + outputs[..., 2:4] = np.exp(outputs[..., 2:4]) * expanded_strides + + return outputs diff --git a/dnns/yolov3/utils/dist.py b/dnns/yolov3/utils/dist.py new file mode 100644 index 0000000000000000000000000000000000000000..e389f42f89d4f3b2d14fbe336e8bf4a0b92add1c --- /dev/null +++ b/dnns/yolov3/utils/dist.py @@ -0,0 +1,287 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +# This file mainly comes from +# https://github.com/facebookresearch/detectron2/blob/master/detectron2/utils/comm.py +# Copyright (c) Facebook, Inc. and its affiliates. +# Copyright (c) Megvii Inc. All rights reserved. +""" +This file contains primitives for multi-gpu communication. +This is useful when doing distributed training. +""" + +import functools +import os +import pickle +import time +from contextlib import contextmanager +from loguru import logger + +import numpy as np + +import torch +from torch import distributed as dist + +__all__ = [ + "get_num_devices", + "wait_for_the_master", + "is_main_process", + "synchronize", + "get_world_size", + "get_rank", + "get_local_rank", + "get_local_size", + "time_synchronized", + "gather", + "all_gather", +] + +_LOCAL_PROCESS_GROUP = None + + +def get_num_devices(): + gpu_list = os.getenv('CUDA_VISIBLE_DEVICES', None) + if gpu_list is not None: + return len(gpu_list.split(',')) + else: + devices_list_info = os.popen("nvidia-smi -L") + devices_list_info = devices_list_info.read().strip().split("\n") + return len(devices_list_info) + + +@contextmanager +def wait_for_the_master(local_rank: int): + """ + Make all processes waiting for the master to do some task. + """ + if local_rank > 0: + dist.barrier() + yield + if local_rank == 0: + if not dist.is_available(): + return + if not dist.is_initialized(): + return + else: + dist.barrier() + + +def synchronize(): + """ + Helper function to synchronize (barrier) among all processes when using distributed training + """ + if not dist.is_available(): + return + if not dist.is_initialized(): + return + world_size = dist.get_world_size() + if world_size == 1: + return + dist.barrier() + + +def get_world_size() -> int: + if not dist.is_available(): + return 1 + if not dist.is_initialized(): + return 1 + return dist.get_world_size() + + +def get_rank() -> int: + if not dist.is_available(): + return 0 + if not dist.is_initialized(): + return 0 + return dist.get_rank() + + +def get_local_rank() -> int: + """ + Returns: + The rank of the current process within the local (per-machine) process group. + """ + if _LOCAL_PROCESS_GROUP is None: + return get_rank() + + if not dist.is_available(): + return 0 + if not dist.is_initialized(): + return 0 + return dist.get_rank(group=_LOCAL_PROCESS_GROUP) + + +def get_local_size() -> int: + """ + Returns: + The size of the per-machine process group, i.e. the number of processes per machine. + """ + if not dist.is_available(): + return 1 + if not dist.is_initialized(): + return 1 + return dist.get_world_size(group=_LOCAL_PROCESS_GROUP) + + +def is_main_process() -> bool: + return get_rank() == 0 + + +@functools.lru_cache() +def _get_global_gloo_group(): + """ + Return a process group based on gloo backend, containing all the ranks + The result is cached. + """ + if dist.get_backend() == "nccl": + return dist.new_group(backend="gloo") + else: + return dist.group.WORLD + + +def _serialize_to_tensor(data, group): + backend = dist.get_backend(group) + assert backend in ["gloo", "nccl"] + device = torch.device("cpu" if backend == "gloo" else "cuda") + + buffer = pickle.dumps(data) + if len(buffer) > 1024 ** 3: + logger.warning( + "Rank {} trying to all-gather {:.2f} GB of data on device {}".format( + get_rank(), len(buffer) / (1024 ** 3), device + ) + ) + storage = torch.ByteStorage.from_buffer(buffer) + tensor = torch.ByteTensor(storage).to(device=device) + return tensor + + +def _pad_to_largest_tensor(tensor, group): + """ + Returns: + list[int]: size of the tensor, on each rank + Tensor: padded tensor that has the max size + """ + world_size = dist.get_world_size(group=group) + assert ( + world_size >= 1 + ), "comm.gather/all_gather must be called from ranks within the given group!" + local_size = torch.tensor([tensor.numel()], dtype=torch.int64, device=tensor.device) + size_list = [ + torch.zeros([1], dtype=torch.int64, device=tensor.device) + for _ in range(world_size) + ] + dist.all_gather(size_list, local_size, group=group) + size_list = [int(size.item()) for size in size_list] + + max_size = max(size_list) + + # we pad the tensor because torch all_gather does not support + # gathering tensors of different shapes + if local_size != max_size: + padding = torch.zeros( + (max_size - local_size,), dtype=torch.uint8, device=tensor.device + ) + tensor = torch.cat((tensor, padding), dim=0) + return size_list, tensor + + +def all_gather(data, group=None): + """ + Run all_gather on arbitrary picklable data (not necessarily tensors). + + Args: + data: any picklable object + group: a torch process group. By default, will use a group which + contains all ranks on gloo backend. + Returns: + list[data]: list of data gathered from each rank + """ + if get_world_size() == 1: + return [data] + if group is None: + group = _get_global_gloo_group() + if dist.get_world_size(group) == 1: + return [data] + + tensor = _serialize_to_tensor(data, group) + + size_list, tensor = _pad_to_largest_tensor(tensor, group) + max_size = max(size_list) + + # receiving Tensor from all ranks + tensor_list = [ + torch.empty((max_size,), dtype=torch.uint8, device=tensor.device) + for _ in size_list + ] + dist.all_gather(tensor_list, tensor, group=group) + + data_list = [] + for size, tensor in zip(size_list, tensor_list): + buffer = tensor.cpu().numpy().tobytes()[:size] + data_list.append(pickle.loads(buffer)) + + return data_list + + +def gather(data, dst=0, group=None): + """ + Run gather on arbitrary picklable data (not necessarily tensors). + + Args: + data: any picklable object + dst (int): destination rank + group: a torch process group. By default, will use a group which + contains all ranks on gloo backend. + + Returns: + list[data]: on dst, a list of data gathered from each rank. Otherwise, + an empty list. + """ + if get_world_size() == 1: + return [data] + if group is None: + group = _get_global_gloo_group() + if dist.get_world_size(group=group) == 1: + return [data] + rank = dist.get_rank(group=group) + + tensor = _serialize_to_tensor(data, group) + size_list, tensor = _pad_to_largest_tensor(tensor, group) + + # receiving Tensor from all ranks + if rank == dst: + max_size = max(size_list) + tensor_list = [ + torch.empty((max_size,), dtype=torch.uint8, device=tensor.device) + for _ in size_list + ] + dist.gather(tensor, tensor_list, dst=dst, group=group) + + data_list = [] + for size, tensor in zip(size_list, tensor_list): + buffer = tensor.cpu().numpy().tobytes()[:size] + data_list.append(pickle.loads(buffer)) + return data_list + else: + dist.gather(tensor, [], dst=dst, group=group) + return [] + + +def shared_random_seed(): + """ + Returns: + int: a random number that is the same across all workers. + If workers need a shared RNG, they can use this shared seed to + create one. + All workers must call this function, otherwise it will deadlock. + """ + ints = np.random.randint(2 ** 31) + all_ints = all_gather(ints) + return all_ints[0] + + +def time_synchronized(): + """pytorch-accurate time""" + if torch.cuda.is_available(): + torch.cuda.synchronize() + return time.time() diff --git a/dnns/yolov3/utils/ema.py b/dnns/yolov3/utils/ema.py new file mode 100644 index 0000000000000000000000000000000000000000..73acbca6796d3cdd07397e657167acdbd5a57647 --- /dev/null +++ b/dnns/yolov3/utils/ema.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +# Copyright (c) Megvii Inc. All rights reserved. +import math +from copy import deepcopy + +import torch +import torch.nn as nn + +__all__ = ["ModelEMA", "is_parallel"] + + +def is_parallel(model): + """check if model is in parallel mode.""" + parallel_type = ( + nn.parallel.DataParallel, + nn.parallel.DistributedDataParallel, + ) + return isinstance(model, parallel_type) + + +class ModelEMA: + """ + Model Exponential Moving Average from https://github.com/rwightman/pytorch-image-models + Keep a moving average of everything in the model state_dict (parameters and buffers). + This is intended to allow functionality like + https://www.tensorflow.org/api_docs/python/tf/train/ExponentialMovingAverage + A smoothed version of the weights is necessary for some training schemes to perform well. + This class is sensitive where it is initialized in the sequence of model init, + GPU assignment and distributed training wrappers. + """ + + def __init__(self, model, decay=0.9999, updates=0): + """ + Args: + model (nn.Module): model to apply EMA. + decay (float): ema decay reate. + updates (int): counter of EMA updates. + """ + # Create EMA(FP32) + self.ema = deepcopy(model.module if is_parallel(model) else model).eval() + self.updates = updates + # decay exponential ramp (to help early epochs) + self.decay = lambda x: decay * (1 - math.exp(-x / 2000)) + for p in self.ema.parameters(): + p.requires_grad_(False) + + def update(self, model): + # Update EMA parameters + with torch.no_grad(): + self.updates += 1 + d = self.decay(self.updates) + + msd = ( + model.module.state_dict() if is_parallel(model) else model.state_dict() + ) # model state_dict + for k, v in self.ema.state_dict().items(): + if v.dtype.is_floating_point: + v *= d + v += (1.0 - d) * msd[k].detach() diff --git a/dnns/yolov3/utils/logger.py b/dnns/yolov3/utils/logger.py new file mode 100644 index 0000000000000000000000000000000000000000..f426e64164915e2708cce2ed6eaee79e9f54109f --- /dev/null +++ b/dnns/yolov3/utils/logger.py @@ -0,0 +1,217 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +# Copyright (c) Megvii Inc. All rights reserved. + +import inspect +import os +import sys +from loguru import logger + +import torch + + +def get_caller_name(depth=0): + """ + Args: + depth (int): Depth of caller conext, use 0 for caller depth. + Default value: 0. + + Returns: + str: module name of the caller + """ + # the following logic is a little bit faster than inspect.stack() logic + frame = inspect.currentframe().f_back + for _ in range(depth): + frame = frame.f_back + + return frame.f_globals["__name__"] + + +class StreamToLoguru: + """ + stream object that redirects writes to a logger instance. + """ + + def __init__(self, level="INFO", caller_names=("apex", "pycocotools")): + """ + Args: + level(str): log level string of loguru. Default value: "INFO". + caller_names(tuple): caller names of redirected module. + Default value: (apex, pycocotools). + """ + self.level = level + self.linebuf = "" + self.caller_names = caller_names + + def write(self, buf): + full_name = get_caller_name(depth=1) + module_name = full_name.rsplit(".", maxsplit=-1)[0] + if module_name in self.caller_names: + for line in buf.rstrip().splitlines(): + # use caller level log + logger.opt(depth=2).log(self.level, line.rstrip()) + else: + sys.__stdout__.write(buf) + + def flush(self): + pass + + +def redirect_sys_output(log_level="INFO"): + redirect_logger = StreamToLoguru(log_level) + sys.stderr = redirect_logger + sys.stdout = redirect_logger + + +def setup_logger(save_dir, distributed_rank=0, filename="log.txt", mode="a"): + """setup logger for training and testing. + Args: + save_dir(str): location to save log file + distributed_rank(int): device rank when multi-gpu environment + filename (string): log save name. + mode(str): log file write mode, `append` or `override`. default is `a`. + + Return: + logger instance. + """ + loguru_format = ( + "{time:YYYY-MM-DD HH:mm:ss} | " + "{level: <8} | " + "{name}:{line} - {message}" + ) + + logger.remove() + save_file = os.path.join(save_dir, filename) + if mode == "o" and os.path.exists(save_file): + os.remove(save_file) + # only keep logger in rank0 process + if distributed_rank == 0: + logger.add( + sys.stderr, + format=loguru_format, + level="INFO", + enqueue=True, + ) + logger.add(save_file) + + # redirect stdout/stderr to loguru + redirect_sys_output("INFO") + + +class WandbLogger(object): + """ + Log training runs, datasets, models, and predictions to Weights & Biases. + This logger sends information to W&B at wandb.ai. + By default, this information includes hyperparameters, + system configuration and metrics, model metrics, + and basic data metrics and analyses. + + For more information, please refer to: + https://docs.wandb.ai/guides/track + """ + def __init__(self, + project=None, + name=None, + id=None, + entity=None, + save_dir=None, + config=None, + **kwargs): + """ + Args: + project (str): wandb project name. + name (str): wandb run name. + id (str): wandb run id. + entity (str): wandb entity name. + save_dir (str): save directory. + config (dict): config dict. + **kwargs: other kwargs. + """ + try: + import wandb + self.wandb = wandb + except ModuleNotFoundError: + raise ModuleNotFoundError( + "wandb is not installed." + "Please install wandb using pip install wandb" + ) + + self.project = project + self.name = name + self.id = id + self.save_dir = save_dir + self.config = config + self.kwargs = kwargs + self.entity = entity + self._run = None + self._wandb_init = dict( + project=self.project, + name=self.name, + id=self.id, + entity=self.entity, + dir=self.save_dir, + resume="allow" + ) + self._wandb_init.update(**kwargs) + + _ = self.run + + if self.config: + self.run.config.update(self.config) + self.run.define_metric("epoch") + self.run.define_metric("val/", step_metric="epoch") + + @property + def run(self): + if self._run is None: + if self.wandb.run is not None: + logger.info( + "There is a wandb run already in progress " + "and newly created instances of `WandbLogger` will reuse" + " this run. If this is not desired, call `wandb.finish()`" + "before instantiating `WandbLogger`." + ) + self._run = self.wandb.run + else: + self._run = self.wandb.init(**self._wandb_init) + return self._run + + def log_metrics(self, metrics, step=None): + """ + Args: + metrics (dict): metrics dict. + step (int): step number. + """ + + for k, v in metrics.items(): + if isinstance(v, torch.Tensor): + metrics[k] = v.item() + + if step is not None: + self.run.log(metrics, step=step) + else: + self.run.log(metrics) + + def save_checkpoint(self, save_dir, model_name, is_best): + """ + Args: + save_dir (str): save directory. + model_name (str): model name. + is_best (bool): whether the model is the best model. + """ + filename = os.path.join(save_dir, model_name + "_ckpt.pth") + artifact = self.wandb.Artifact( + name=f"model-{self.run.id}", + type="model" + ) + artifact.add_file(filename, name="model_ckpt.pth") + + aliases = ["latest"] + + if is_best: + aliases.append("best") + + self.run.log_artifact(artifact, aliases=aliases) + + def finish(self): + self.run.finish() diff --git a/dnns/yolov3/utils/lr_scheduler.py b/dnns/yolov3/utils/lr_scheduler.py new file mode 100644 index 0000000000000000000000000000000000000000..2a1513c22abb5bbcb65447f6ee4bbadebfa9d43f --- /dev/null +++ b/dnns/yolov3/utils/lr_scheduler.py @@ -0,0 +1,205 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +# Copyright (c) Megvii Inc. All rights reserved. + +import math +from functools import partial + + +class LRScheduler: + def __init__(self, name, lr, iters_per_epoch, total_epochs, **kwargs): + """ + Supported lr schedulers: [cos, warmcos, multistep] + + Args: + lr (float): learning rate. + iters_per_peoch (int): number of iterations in one epoch. + total_epochs (int): number of epochs in training. + kwargs (dict): + - cos: None + - warmcos: [warmup_epochs, warmup_lr_start (default 1e-6)] + - multistep: [milestones (epochs), gamma (default 0.1)] + """ + + self.lr = lr + self.iters_per_epoch = iters_per_epoch + self.total_epochs = total_epochs + self.total_iters = iters_per_epoch * total_epochs + + self.__dict__.update(kwargs) + + self.lr_func = self._get_lr_func(name) + + def update_lr(self, iters): + return self.lr_func(iters) + + def _get_lr_func(self, name): + if name == "cos": # cosine lr schedule + lr_func = partial(cos_lr, self.lr, self.total_iters) + elif name == "warmcos": + warmup_total_iters = self.iters_per_epoch * self.warmup_epochs + warmup_lr_start = getattr(self, "warmup_lr_start", 1e-6) + lr_func = partial( + warm_cos_lr, + self.lr, + self.total_iters, + warmup_total_iters, + warmup_lr_start, + ) + elif name == "yoloxwarmcos": + warmup_total_iters = self.iters_per_epoch * self.warmup_epochs + no_aug_iters = self.iters_per_epoch * self.no_aug_epochs + warmup_lr_start = getattr(self, "warmup_lr_start", 0) + min_lr_ratio = getattr(self, "min_lr_ratio", 0.2) + lr_func = partial( + yolox_warm_cos_lr, + self.lr, + min_lr_ratio, + self.total_iters, + warmup_total_iters, + warmup_lr_start, + no_aug_iters, + ) + elif name == "yoloxsemiwarmcos": + warmup_lr_start = getattr(self, "warmup_lr_start", 0) + min_lr_ratio = getattr(self, "min_lr_ratio", 0.2) + warmup_total_iters = self.iters_per_epoch * self.warmup_epochs + no_aug_iters = self.iters_per_epoch * self.no_aug_epochs + normal_iters = self.iters_per_epoch * self.semi_epoch + semi_iters = self.iters_per_epoch_semi * ( + self.total_epochs - self.semi_epoch - self.no_aug_epochs + ) + lr_func = partial( + yolox_semi_warm_cos_lr, + self.lr, + min_lr_ratio, + warmup_lr_start, + self.total_iters, + normal_iters, + no_aug_iters, + warmup_total_iters, + semi_iters, + self.iters_per_epoch, + self.iters_per_epoch_semi, + ) + elif name == "multistep": # stepwise lr schedule + milestones = [ + int(self.total_iters * milestone / self.total_epochs) + for milestone in self.milestones + ] + gamma = getattr(self, "gamma", 0.1) + lr_func = partial(multistep_lr, self.lr, milestones, gamma) + else: + raise ValueError("Scheduler version {} not supported.".format(name)) + return lr_func + + +def cos_lr(lr, total_iters, iters): + """Cosine learning rate""" + lr *= 0.5 * (1.0 + math.cos(math.pi * iters / total_iters)) + return lr + + +def warm_cos_lr(lr, total_iters, warmup_total_iters, warmup_lr_start, iters): + """Cosine learning rate with warm up.""" + if iters <= warmup_total_iters: + lr = (lr - warmup_lr_start) * iters / float( + warmup_total_iters + ) + warmup_lr_start + else: + lr *= 0.5 * ( + 1.0 + + math.cos( + math.pi + * (iters - warmup_total_iters) + / (total_iters - warmup_total_iters) + ) + ) + return lr + + +def yolox_warm_cos_lr( + lr, + min_lr_ratio, + total_iters, + warmup_total_iters, + warmup_lr_start, + no_aug_iter, + iters, +): + """Cosine learning rate with warm up.""" + min_lr = lr * min_lr_ratio + if iters <= warmup_total_iters: + # lr = (lr - warmup_lr_start) * iters / float(warmup_total_iters) + warmup_lr_start + lr = (lr - warmup_lr_start) * pow( + iters / float(warmup_total_iters), 2 + ) + warmup_lr_start + elif iters >= total_iters - no_aug_iter: + lr = min_lr + else: + lr = min_lr + 0.5 * (lr - min_lr) * ( + 1.0 + + math.cos( + math.pi + * (iters - warmup_total_iters) + / (total_iters - warmup_total_iters - no_aug_iter) + ) + ) + return lr + + +def yolox_semi_warm_cos_lr( + lr, + min_lr_ratio, + warmup_lr_start, + total_iters, + normal_iters, + no_aug_iters, + warmup_total_iters, + semi_iters, + iters_per_epoch, + iters_per_epoch_semi, + iters, +): + """Cosine learning rate with warm up.""" + min_lr = lr * min_lr_ratio + if iters <= warmup_total_iters: + # lr = (lr - warmup_lr_start) * iters / float(warmup_total_iters) + warmup_lr_start + lr = (lr - warmup_lr_start) * pow( + iters / float(warmup_total_iters), 2 + ) + warmup_lr_start + elif iters >= normal_iters + semi_iters: + lr = min_lr + elif iters <= normal_iters: + lr = min_lr + 0.5 * (lr - min_lr) * ( + 1.0 + + math.cos( + math.pi + * (iters - warmup_total_iters) + / (total_iters - warmup_total_iters - no_aug_iters) + ) + ) + else: + lr = min_lr + 0.5 * (lr - min_lr) * ( + 1.0 + + math.cos( + math.pi + * ( + normal_iters + - warmup_total_iters + + (iters - normal_iters) + * iters_per_epoch + * 1.0 + / iters_per_epoch_semi + ) + / (total_iters - warmup_total_iters - no_aug_iters) + ) + ) + return lr + + +def multistep_lr(lr, milestones, gamma, iters): + """MultiStep learning rate""" + for milestone in milestones: + lr *= gamma if iters >= milestone else 1.0 + return lr diff --git a/dnns/yolov3/utils/metric.py b/dnns/yolov3/utils/metric.py new file mode 100644 index 0000000000000000000000000000000000000000..10f0e631f9996bb50ac72539b7a6dc91d9560932 --- /dev/null +++ b/dnns/yolov3/utils/metric.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# Copyright (c) Megvii Inc. All rights reserved. +import functools +import os +import time +from collections import defaultdict, deque + +import numpy as np + +import torch + +__all__ = [ + "AverageMeter", + "MeterBuffer", + "get_total_and_free_memory_in_Mb", + "occupy_mem", + "gpu_mem_usage", +] + + +def get_total_and_free_memory_in_Mb(cuda_device): + devices_info_str = os.popen( + "nvidia-smi --query-gpu=memory.total,memory.used --format=csv,nounits,noheader" + ) + devices_info = devices_info_str.read().strip().split("\n") + if "CUDA_VISIBLE_DEVICES" in os.environ: + visible_devices = os.environ["CUDA_VISIBLE_DEVICES"].split(',') + cuda_device = int(visible_devices[cuda_device]) + total, used = devices_info[int(cuda_device)].split(",") + return int(total), int(used) + + +def occupy_mem(cuda_device, mem_ratio=0.9): + """ + pre-allocate gpu memory for training to avoid memory Fragmentation. + """ + total, used = get_total_and_free_memory_in_Mb(cuda_device) + max_mem = int(total * mem_ratio) + block_mem = max_mem - used + x = torch.cuda.FloatTensor(256, 1024, block_mem) + del x + time.sleep(5) + + +def gpu_mem_usage(): + """ + Compute the GPU memory usage for the current device (MB). + """ + mem_usage_bytes = torch.cuda.max_memory_allocated() + return mem_usage_bytes / (1024 * 1024) + + +class AverageMeter: + """Track a series of values and provide access to smoothed values over a + window or the global series average. + """ + + def __init__(self, window_size=50): + self._deque = deque(maxlen=window_size) + self._total = 0.0 + self._count = 0 + + def update(self, value): + self._deque.append(value) + self._count += 1 + self._total += value + + @property + def median(self): + d = np.array(list(self._deque)) + return np.median(d) + + @property + def avg(self): + # if deque is empty, nan will be returned. + d = np.array(list(self._deque)) + return d.mean() + + @property + def global_avg(self): + return self._total / max(self._count, 1e-5) + + @property + def latest(self): + return self._deque[-1] if len(self._deque) > 0 else None + + @property + def total(self): + return self._total + + def reset(self): + self._deque.clear() + self._total = 0.0 + self._count = 0 + + def clear(self): + self._deque.clear() + + +class MeterBuffer(defaultdict): + """Computes and stores the average and current value""" + + def __init__(self, window_size=20): + factory = functools.partial(AverageMeter, window_size=window_size) + super().__init__(factory) + + def reset(self): + for v in self.values(): + v.reset() + + def get_filtered_meter(self, filter_key="time"): + return {k: v for k, v in self.items() if filter_key in k} + + def update(self, values=None, **kwargs): + if values is None: + values = {} + values.update(kwargs) + for k, v in values.items(): + if isinstance(v, torch.Tensor): + v = v.detach() + self[k].update(v) + + def clear_meters(self): + for v in self.values(): + v.clear() diff --git a/dnns/yolov3/utils/model_utils.py b/dnns/yolov3/utils/model_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..4b9b99291bc252d86dd9abb35519534ea9d10faa --- /dev/null +++ b/dnns/yolov3/utils/model_utils.py @@ -0,0 +1,185 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +# Copyright (c) Megvii Inc. All rights reserved. + +import contextlib +from copy import deepcopy +from typing import Sequence + +import torch +import torch.nn as nn +from thop import profile + +__all__ = [ + "fuse_conv_and_bn", + "fuse_model", + "get_model_info", + "replace_module", + "freeze_module", + "adjust_status", +] + + +def get_model_info(model: nn.Module, tsize: Sequence[int]) -> str: + stride = 64 + img = torch.zeros((1, 3, stride, stride), device=next(model.parameters()).device) + flops, params = profile(deepcopy(model), inputs=(img,), verbose=False) + params /= 1e6 + flops /= 1e9 + flops *= tsize[0] * tsize[1] / stride / stride * 2 # Gflops + info = "Params: {:.2f}M, Gflops: {:.2f}".format(params, flops) + return info + + +def fuse_conv_and_bn(conv: nn.Conv2d, bn: nn.BatchNorm2d) -> nn.Conv2d: + """ + Fuse convolution and batchnorm layers. + check more info on https://tehnokv.com/posts/fusing-batchnorm-and-conv/ + + Args: + conv (nn.Conv2d): convolution to fuse. + bn (nn.BatchNorm2d): batchnorm to fuse. + + Returns: + nn.Conv2d: fused convolution behaves the same as the input conv and bn. + """ + fusedconv = ( + nn.Conv2d( + conv.in_channels, + conv.out_channels, + kernel_size=conv.kernel_size, + stride=conv.stride, + padding=conv.padding, + groups=conv.groups, + bias=True, + ) + .requires_grad_(False) + .to(conv.weight.device) + ) + + # prepare filters + w_conv = conv.weight.clone().view(conv.out_channels, -1) + w_bn = torch.diag(bn.weight.div(torch.sqrt(bn.eps + bn.running_var))) + fusedconv.weight.copy_(torch.mm(w_bn, w_conv).view(fusedconv.weight.shape)) + + # prepare spatial bias + b_conv = ( + torch.zeros(conv.weight.size(0), device=conv.weight.device) + if conv.bias is None + else conv.bias + ) + b_bn = bn.bias - bn.weight.mul(bn.running_mean).div( + torch.sqrt(bn.running_var + bn.eps) + ) + fusedconv.bias.copy_(torch.mm(w_bn, b_conv.reshape(-1, 1)).reshape(-1) + b_bn) + + return fusedconv + + +def fuse_model(model: nn.Module) -> nn.Module: + """fuse conv and bn in model + + Args: + model (nn.Module): model to fuse + + Returns: + nn.Module: fused model + """ + from yolox.models.network_blocks import BaseConv + + for m in model.modules(): + if type(m) is BaseConv and hasattr(m, "bn"): + m.conv = fuse_conv_and_bn(m.conv, m.bn) # update conv + delattr(m, "bn") # remove batchnorm + m.forward = m.fuseforward # update forward + return model + + +def replace_module(module, replaced_module_type, new_module_type, replace_func=None) -> nn.Module: + """ + Replace given type in module to a new type. mostly used in deploy. + + Args: + module (nn.Module): model to apply replace operation. + replaced_module_type (Type): module type to be replaced. + new_module_type (Type) + replace_func (function): python function to describe replace logic. Defalut value None. + + Returns: + model (nn.Module): module that already been replaced. + """ + + def default_replace_func(replaced_module_type, new_module_type): + return new_module_type() + + if replace_func is None: + replace_func = default_replace_func + + model = module + if isinstance(module, replaced_module_type): + model = replace_func(replaced_module_type, new_module_type) + else: # recurrsively replace + for name, child in module.named_children(): + new_child = replace_module(child, replaced_module_type, new_module_type) + if new_child is not child: # child is already replaced + model.add_module(name, new_child) + + return model + + +def freeze_module(module: nn.Module, name=None) -> nn.Module: + """freeze module inplace + + Args: + module (nn.Module): module to freeze. + name (str, optional): name to freeze. If not given, freeze the whole module. + Note that fuzzy match is not supported. Defaults to None. + + Examples: + freeze the backbone of model + >>> freeze_moudle(model.backbone) + + or freeze the backbone of model by name + >>> freeze_moudle(model, name="backbone") + """ + for param_name, parameter in module.named_parameters(): + if name is None or name in param_name: + parameter.requires_grad = False + + # ensure module like BN and dropout are freezed + for module_name, sub_module in module.named_modules(): + # actually there are no needs to call eval for every single sub_module + if name is None or name in module_name: + sub_module.eval() + + return module + + +@contextlib.contextmanager +def adjust_status(module: nn.Module, training: bool = False) -> nn.Module: + """Adjust module to training/eval mode temporarily. + + Args: + module (nn.Module): module to adjust status. + training (bool): training mode to set. True for train mode, False fro eval mode. + + Examples: + >>> with adjust_status(model, training=False): + ... model(data) + """ + status = {} + + def backup_status(module): + for m in module.modules(): + # save prev status to dict + status[m] = m.training + m.training = training + + def recover_status(module): + for m in module.modules(): + # recover prev status from dict + m.training = status.pop(m) + + backup_status(module) + yield module + recover_status(module) diff --git a/dnns/yolov3/utils/setup_env.py b/dnns/yolov3/utils/setup_env.py new file mode 100644 index 0000000000000000000000000000000000000000..45289f3245f09e48395ad419d17efffe6846b05c --- /dev/null +++ b/dnns/yolov3/utils/setup_env.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +# Copyright (c) Megvii Inc. All rights reserved. + +import os +import subprocess +from loguru import logger + +import cv2 + +from .dist import get_world_size, is_main_process + +__all__ = ["configure_nccl", "configure_module", "configure_omp"] + + +def configure_nccl(): + """Configure multi-machine environment variables of NCCL.""" + os.environ["NCCL_LAUNCH_MODE"] = "PARALLEL" + os.environ["NCCL_IB_HCA"] = subprocess.getoutput( + "pushd /sys/class/infiniband/ > /dev/null; for i in mlx5_*; " + "do cat $i/ports/1/gid_attrs/types/* 2>/dev/null " + "| grep v >/dev/null && echo $i ; done; popd > /dev/null" + ) + os.environ["NCCL_IB_GID_INDEX"] = "3" + os.environ["NCCL_IB_TC"] = "106" + + +def configure_omp(num_threads=1): + """ + If OMP_NUM_THREADS is not configured and world_size is greater than 1, + Configure OMP_NUM_THREADS environment variables of NCCL to `num_thread`. + + Args: + num_threads (int): value of `OMP_NUM_THREADS` to set. + """ + # We set OMP_NUM_THREADS=1 by default, which achieves the best speed on our machines + # feel free to change it for better performance. + if "OMP_NUM_THREADS" not in os.environ and get_world_size() > 1: + os.environ["OMP_NUM_THREADS"] = str(num_threads) + if is_main_process(): + logger.info( + "\n***************************************************************\n" + "We set `OMP_NUM_THREADS` for each process to {} to speed up.\n" + "please further tune the variable for optimal performance.\n" + "***************************************************************".format( + os.environ["OMP_NUM_THREADS"] + ) + ) + + +def configure_module(ulimit_value=8192): + """ + Configure pytorch module environment. setting of ulimit and cv2 will be set. + + Args: + ulimit_value(int): default open file number on linux. Default value: 8192. + """ + # system setting + try: + import resource + + rlimit = resource.getrlimit(resource.RLIMIT_NOFILE) + resource.setrlimit(resource.RLIMIT_NOFILE, (ulimit_value, rlimit[1])) + except Exception: + # Exception might be raised in Windows OS or rlimit reaches max limit number. + # However, set rlimit value might not be necessary. + pass + + # cv2 + # multiprocess might be harmful on performance of torch dataloader + os.environ["OPENCV_OPENCL_RUNTIME"] = "disabled" + try: + cv2.setNumThreads(0) + cv2.ocl.setUseOpenCL(False) + except Exception: + # cv2 version mismatch might rasie exceptions. + pass diff --git a/dnns/yolov3/utils/visualize.py b/dnns/yolov3/utils/visualize.py new file mode 100644 index 0000000000000000000000000000000000000000..e714a3ee73699141fb4cd8d131d541a6e6625ed6 --- /dev/null +++ b/dnns/yolov3/utils/visualize.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +# Copyright (c) Megvii Inc. All rights reserved. + +import cv2 +import numpy as np + +__all__ = ["vis"] + + +def vis(img, boxes, scores, cls_ids, conf=0.5, class_names=None): + + for i in range(len(boxes)): + box = boxes[i] + cls_id = int(cls_ids[i]) + score = scores[i] + if score < conf: + continue + x0 = int(box[0]) + y0 = int(box[1]) + x1 = int(box[2]) + y1 = int(box[3]) + + color = (_COLORS[cls_id] * 255).astype(np.uint8).tolist() + text = '{}:{:.1f}%'.format(class_names[cls_id], score * 100) + txt_color = (0, 0, 0) if np.mean(_COLORS[cls_id]) > 0.5 else (255, 255, 255) + font = cv2.FONT_HERSHEY_SIMPLEX + + txt_size = cv2.getTextSize(text, font, 0.4, 1)[0] + cv2.rectangle(img, (x0, y0), (x1, y1), color, 2) + + txt_bk_color = (_COLORS[cls_id] * 255 * 0.7).astype(np.uint8).tolist() + cv2.rectangle( + img, + (x0, y0 + 1), + (x0 + txt_size[0] + 1, y0 + int(1.5*txt_size[1])), + txt_bk_color, + -1 + ) + cv2.putText(img, text, (x0, y0 + txt_size[1]), font, 0.4, txt_color, thickness=1) + + return img + + +_COLORS = np.array( + [ + 0.000, 0.447, 0.741, + 0.850, 0.325, 0.098, + 0.929, 0.694, 0.125, + 0.494, 0.184, 0.556, + 0.466, 0.674, 0.188, + 0.301, 0.745, 0.933, + 0.635, 0.078, 0.184, + 0.300, 0.300, 0.300, + 0.600, 0.600, 0.600, + 1.000, 0.000, 0.000, + 1.000, 0.500, 0.000, + 0.749, 0.749, 0.000, + 0.000, 1.000, 0.000, + 0.000, 0.000, 1.000, + 0.667, 0.000, 1.000, + 0.333, 0.333, 0.000, + 0.333, 0.667, 0.000, + 0.333, 1.000, 0.000, + 0.667, 0.333, 0.000, + 0.667, 0.667, 0.000, + 0.667, 1.000, 0.000, + 1.000, 0.333, 0.000, + 1.000, 0.667, 0.000, + 1.000, 1.000, 0.000, + 0.000, 0.333, 0.500, + 0.000, 0.667, 0.500, + 0.000, 1.000, 0.500, + 0.333, 0.000, 0.500, + 0.333, 0.333, 0.500, + 0.333, 0.667, 0.500, + 0.333, 1.000, 0.500, + 0.667, 0.000, 0.500, + 0.667, 0.333, 0.500, + 0.667, 0.667, 0.500, + 0.667, 1.000, 0.500, + 1.000, 0.000, 0.500, + 1.000, 0.333, 0.500, + 1.000, 0.667, 0.500, + 1.000, 1.000, 0.500, + 0.000, 0.333, 1.000, + 0.000, 0.667, 1.000, + 0.000, 1.000, 1.000, + 0.333, 0.000, 1.000, + 0.333, 0.333, 1.000, + 0.333, 0.667, 1.000, + 0.333, 1.000, 1.000, + 0.667, 0.000, 1.000, + 0.667, 0.333, 1.000, + 0.667, 0.667, 1.000, + 0.667, 1.000, 1.000, + 1.000, 0.000, 1.000, + 1.000, 0.333, 1.000, + 1.000, 0.667, 1.000, + 0.333, 0.000, 0.000, + 0.500, 0.000, 0.000, + 0.667, 0.000, 0.000, + 0.833, 0.000, 0.000, + 1.000, 0.000, 0.000, + 0.000, 0.167, 0.000, + 0.000, 0.333, 0.000, + 0.000, 0.500, 0.000, + 0.000, 0.667, 0.000, + 0.000, 0.833, 0.000, + 0.000, 1.000, 0.000, + 0.000, 0.000, 0.167, + 0.000, 0.000, 0.333, + 0.000, 0.000, 0.500, + 0.000, 0.000, 0.667, + 0.000, 0.000, 0.833, + 0.000, 0.000, 1.000, + 0.000, 0.000, 0.000, + 0.143, 0.143, 0.143, + 0.286, 0.286, 0.286, + 0.429, 0.429, 0.429, + 0.571, 0.571, 0.571, + 0.714, 0.714, 0.714, + 0.857, 0.857, 0.857, + 0.000, 0.447, 0.741, + 0.314, 0.717, 0.741, + 0.50, 0.5, 0 + ] +).astype(np.float32).reshape(-1, 3) diff --git a/dnns/yolov3/vit_yolov3.py b/dnns/yolov3/vit_yolov3.py new file mode 100644 index 0000000000000000000000000000000000000000..755d6251c2bbe7520ae895d02aab8ad07209c022 --- /dev/null +++ b/dnns/yolov3/vit_yolov3.py @@ -0,0 +1,470 @@ +import torch +from torch import nn +from timm.models.vision_transformer import VisionTransformer +from functools import partial +from einops import rearrange +from dnns.yolov3.yolo_fpn import YOLOFPN +from dnns.yolov3.head import YOLOXHead +from utils.dl.common.model import set_module, get_module +from types import MethodType +import os +from utils.common.log import logger + + +class VisionTransformerYOLOv3(VisionTransformer): + def forward_head(self, x): + # print(222) + return self.head(x) + + def forward_features(self, x): + # print(111) + return self._intermediate_layers(x, n=[len(self.blocks) // 3 - 1, len(self.blocks) // 3 * 2 - 1, len(self.blocks) - 1]) + + def forward(self, x, targets=None): + features = self.forward_features(x) + return self.head(x, features, targets) + + @staticmethod + def init_from_vit(vit: VisionTransformer): + res = VisionTransformerYOLOv3() + + for attr in dir(vit): + # if str(attr) not in ['forward_head', 'forward_features'] and not attr.startswith('__'): + if isinstance(getattr(vit, attr), nn.Module): + # print(attr) + try: + setattr(res, attr, getattr(vit, attr)) + except Exception as e: + print(attr, str(e)) + + return res + + +class Norm2d(nn.Module): + def __init__(self, embed_dim): + super().__init__() + self.ln = nn.LayerNorm(embed_dim, eps=1e-6) + def forward(self, x): + x = x.permute(0, 2, 3, 1) + x = self.ln(x) + x = x.permute(0, 3, 1, 2).contiguous() + return x + + +class ViTYOLOv3Head(nn.Module): + def __init__(self, im_size, patch_size, patch_dim, num_classes, use_bigger_fpns, cls_vit_ckpt_path, init_head): + super(ViTYOLOv3Head, self).__init__() + + self.im_size = im_size + self.patch_size = patch_size + + # target_patch_dim: [256, 512, 512] + # self.change_patchs_dim = nn.ModuleList([nn.Linear(patch_dim, target_patch_dim) for target_patch_dim in [256, 512, 512]]) + + # # input: (1, target_patch_dim, 14, 14) + # # target feature size: {40, 20, 10} + # self.change_features_size = nn.ModuleList([ + # self.get_change_feature_size(cin, cout, t) for t, cin, cout in zip([40, 20, 10], [256, 512, 512], [256, 512, 512]) + # ]) + + embed_dim = 768 + self.before_fpns = nn.ModuleList([ + # nn.Sequential( + # nn.ConvTranspose2d(embed_dim, embed_dim, kernel_size=2, stride=2), + # nn.GroupNorm(embed_dim), + # nn.GELU(), + # nn.ConvTranspose2d(embed_dim, embed_dim, kernel_size=2, stride=2), + # ), + + nn.Sequential( + nn.ConvTranspose2d(embed_dim, embed_dim, kernel_size=2, stride=2), + ), + + nn.Identity(), + + nn.MaxPool2d(kernel_size=2, stride=2) + ]) + + if use_bigger_fpns == 1: + logger.info('use 421x fpns') + self.before_fpns = nn.ModuleList([ + + nn.Sequential( + nn.ConvTranspose2d(embed_dim, embed_dim, kernel_size=2, stride=2), + Norm2d(embed_dim), + nn.GELU(), + nn.ConvTranspose2d(embed_dim, embed_dim, kernel_size=2, stride=2), + ), + + nn.Sequential( + nn.ConvTranspose2d(embed_dim, embed_dim, kernel_size=2, stride=2), + ), + + nn.Identity(), + + # nn.MaxPool2d(kernel_size=2, stride=2) + ]) + if use_bigger_fpns == -1: + logger.info('use 1/0.5/0.25x fpns') + self.before_fpns = nn.ModuleList([ + + # nn.Sequential( + # nn.ConvTranspose2d(embed_dim, embed_dim, kernel_size=2, stride=2), + # ), + + nn.Identity(), + + nn.MaxPool2d(kernel_size=2, stride=2), + + nn.Sequential(nn.MaxPool2d(kernel_size=2, stride=2), nn.MaxPool2d(kernel_size=2, stride=2)) + ]) + + # self.fpn = YOLOFPN() + self.fpn = nn.Identity() + self.head = YOLOXHead(num_classes, in_channels=[768, 768, 768], act='lrelu') + + if init_head: + logger.info('init head') + self.load_pretrained_weight(cls_vit_ckpt_path) + else: + logger.info('do not init head') + + def load_pretrained_weight(self, cls_vit_ckpt_path): + ckpt = torch.load(os.path.join(os.path.dirname(__file__), 'yolox_darknet.pth')) + # for k in [f'head.cls_preds.{i}.{j}' for i in [0, 1, 2] for j in ['weight', 'bias']]: + # del ckpt['model'][k] + removed_k = [f'head.cls_preds.{i}.{j}' for i in [0, 1, 2] for j in ['weight', 'bias']] + for k, v in ckpt['model'].items(): + if 'backbone.backbone' in k: + removed_k += [k] + if 'head.stems' in k and 'conv.weight' in k: + removed_k += [k] + for k in removed_k: + del ckpt['model'][k] + # print(ckpt['model'].keys()) + + new_state_dict = {} + for k, v in ckpt['model'].items(): + new_k = k.replace('backbone', 'fpn') + new_state_dict[new_k] = v + + # cls_vit_ckpt = torch.load(cls_vit_ckpt_path) + # for k, v in cls_vit_ckpt['main'].named_parameters(): + # if not 'qkv.abs' not in k: + # continue + + # new_state_dict[k] = v + # logger.info(f'load {k} from cls vit ckpt') + + self.load_state_dict(new_state_dict, strict=False) + + def get_change_feature_size(self, in_channels, out_channels, target_size): + H, W = self.im_size + GS = H // self.patch_size # 14 + + if target_size == GS: + return nn.Identity() + elif target_size < GS: + return nn.AdaptiveMaxPool2d((target_size, target_size)) + else: + return { + 20: nn.Sequential( + nn.ConvTranspose2d(in_channels, out_channels, kernel_size=(3, 3), stride=2, padding=0), + # nn.BatchNorm2d(out_channels), + # nn.ReLU(), + nn.AdaptiveMaxPool2d((target_size, target_size)) + ), + 40: nn.Sequential( + nn.ConvTranspose2d(in_channels, out_channels, kernel_size=(3, 3), stride=3, padding=1), + # nn.BatchNorm2d(out_channels), + # nn.ReLU(), + ) + }[target_size] + + def forward(self, input_images, x, targets=None): + # print(111) + # NOTE: YOLOX backbone (w/o FPN) output, or FPN input: {'dark3': torch.Size([4, 256, 40, 40]), 'dark4': torch.Size([4, 512, 20, 20]), 'dark5': torch.Size([4, 512, 10, 10])} + # NOTE: YOLOXHead input: [torch.Size([4, 128, 40, 40]), torch.Size([4, 256, 20, 20]), torch.Size([4, 512, 10, 10])] + # print(x) + + # print([i.size() for i in x]) + x = [i[:, 1:] for i in x] + x = [i.permute(0, 2, 1).reshape(input_images.size(0), -1, 14, 14) for i in x] # 14 is hardcode, obtained from timm.layers.patch_embed.py + # print([i.size() for i in x]) + # exit() + + # NOTE: old + # x[0]: torch.Size([1, 196, 768]) + # H, W = self.im_size + # GS = H // self.patch_size # 14 + # xs = [cpd(x) for x, cpd in zip(xs, self.change_patchs_dim)] # (1, 196, target_patch_dim) + # xs = [rearrange(x, "b (h w) c -> b c h w", h=GS) for x in xs] # (1, target_patch_dim, 14, 14) + + # xs = [cfs(x) for x, cfs in zip(xs, self.change_features_size)] + # print([i.size() for i in xs]) + # ---------------- + + xs = [before_fpn(x[-1]) for i, before_fpn in zip(x, self.before_fpns)] + + # print([i.size() for i in xs]) + # exit() + # [torch.Size([1, 768, 28, 28]), torch.Size([1, 768, 14, 14]), torch.Size([1, 768, 7, 7])] + + + xs = self.fpn(xs) + # print('before head', [i.size() for i in xs]) + xs = tuple(xs) + + if targets is not None: + loss, iou_loss, conf_loss, cls_loss, l1_loss, num_fg = self.head(xs, targets, input_images) + return { + "total_loss": loss, + "iou_loss": iou_loss, + "l1_loss": l1_loss, + "conf_loss": conf_loss, + "cls_loss": cls_loss, + "num_fg": num_fg, + } + + return self.head(xs) + + +class ViTYOLOv3Head2(nn.Module): + def __init__(self, im_size, patch_size, patch_dim, num_classes, use_bigger_fpns): + super(ViTYOLOv3Head2, self).__init__() + + self.im_size = im_size + self.patch_size = patch_size + + # target_patch_dim: [256, 512, 512] + # self.change_patchs_dim = nn.ModuleList([nn.Linear(patch_dim, target_patch_dim) for target_patch_dim in [256, 512, 512]]) + + # # input: (1, target_patch_dim, 14, 14) + # # target feature size: {40, 20, 10} + # self.change_features_size = nn.ModuleList([ + # self.get_change_feature_size(cin, cout, t) for t, cin, cout in zip([40, 20, 10], [256, 512, 512], [256, 512, 512]) + # ]) + + embed_dim = 768 + self.before_fpns = nn.ModuleList([ + # nn.Sequential( + # nn.ConvTranspose2d(embed_dim, embed_dim, kernel_size=2, stride=2), + # nn.GroupNorm(embed_dim), + # nn.GELU(), + # nn.ConvTranspose2d(embed_dim, embed_dim, kernel_size=2, stride=2), + # ), + + nn.Sequential( + nn.ConvTranspose2d(embed_dim, embed_dim, kernel_size=2, stride=2), + ), + + nn.Identity(), + + nn.MaxPool2d(kernel_size=2, stride=2) + ]) + + if use_bigger_fpns: + logger.info('use 8/4/2x fpns') + self.before_fpns = nn.ModuleList([ + + nn.Sequential( + nn.ConvTranspose2d(embed_dim, embed_dim, kernel_size=2, stride=2), + Norm2d(embed_dim), + nn.GELU(), + nn.ConvTranspose2d(embed_dim, embed_dim, kernel_size=2, stride=2), + ), + + nn.Sequential( + nn.ConvTranspose2d(embed_dim, embed_dim, kernel_size=2, stride=2), + ), + + nn.Identity(), + + # nn.MaxPool2d(kernel_size=2, stride=2) + ]) + + # self.fpn = YOLOFPN() + self.fpn = nn.Identity() + self.head = YOLOXHead(num_classes, in_channels=[768, 768, 768], act='lrelu') + + self.load_pretrained_weight() + + def load_pretrained_weight(self): + ckpt = torch.load(os.path.join(os.path.dirname(__file__), 'yolox_darknet.pth')) + # for k in [f'head.cls_preds.{i}.{j}' for i in [0, 1, 2] for j in ['weight', 'bias']]: + # del ckpt['model'][k] + removed_k = [f'head.cls_preds.{i}.{j}' for i in [0, 1, 2] for j in ['weight', 'bias']] + for k, v in ckpt['model'].items(): + if 'backbone.backbone' in k: + removed_k += [k] + if 'head.stems' in k and 'conv.weight' in k: + removed_k += [k] + for k in removed_k: + del ckpt['model'][k] + # print(ckpt['model'].keys()) + + new_state_dict = {} + for k, v in ckpt['model'].items(): + new_k = k.replace('backbone', 'fpn') + new_state_dict[new_k] = v + self.load_state_dict(new_state_dict, strict=False) + + def get_change_feature_size(self, in_channels, out_channels, target_size): + H, W = self.im_size + GS = H // self.patch_size # 14 + + if target_size == GS: + return nn.Identity() + elif target_size < GS: + return nn.AdaptiveMaxPool2d((target_size, target_size)) + else: + return { + 20: nn.Sequential( + nn.ConvTranspose2d(in_channels, out_channels, kernel_size=(3, 3), stride=2, padding=0), + # nn.BatchNorm2d(out_channels), + # nn.ReLU(), + nn.AdaptiveMaxPool2d((target_size, target_size)) + ), + 40: nn.Sequential( + nn.ConvTranspose2d(in_channels, out_channels, kernel_size=(3, 3), stride=3, padding=1), + # nn.BatchNorm2d(out_channels), + # nn.ReLU(), + ) + }[target_size] + + def forward(self, input_images, x, targets=None): + # print(111) + # NOTE: YOLOX backbone (w/o FPN) output, or FPN input: {'dark3': torch.Size([4, 256, 40, 40]), 'dark4': torch.Size([4, 512, 20, 20]), 'dark5': torch.Size([4, 512, 10, 10])} + # NOTE: YOLOXHead input: [torch.Size([4, 128, 40, 40]), torch.Size([4, 256, 20, 20]), torch.Size([4, 512, 10, 10])] + # print(x) + + # print([i.size() for i in x]) + x = [i[:, 1:] for i in x] + x = [i.permute(0, 2, 1).reshape(input_images.size(0), -1, 14, 14) for i in x] # 14 is hardcode, obtained from timm.layers.patch_embed.py + # print([i.size() for i in x]) + # exit() + + # NOTE: old + # x[0]: torch.Size([1, 196, 768]) + # H, W = self.im_size + # GS = H // self.patch_size # 14 + # xs = [cpd(x) for x, cpd in zip(xs, self.change_patchs_dim)] # (1, 196, target_patch_dim) + # xs = [rearrange(x, "b (h w) c -> b c h w", h=GS) for x in xs] # (1, target_patch_dim, 14, 14) + + # xs = [cfs(x) for x, cfs in zip(xs, self.change_features_size)] + # print([i.size() for i in xs]) + # ---------------- + + xs = [before_fpn(i) for i, before_fpn in zip(x, self.before_fpns)] + + # print([i.size() for i in xs]) + # exit() + # [torch.Size([1, 768, 28, 28]), torch.Size([1, 768, 14, 14]), torch.Size([1, 768, 7, 7])] + + + xs = self.fpn(xs) + # print('before head', [i.size() for i in xs]) + xs = tuple(xs) + + if targets is not None: + loss, iou_loss, conf_loss, cls_loss, l1_loss, num_fg = self.head(xs, targets, input_images) + return { + "total_loss": loss, + "iou_loss": iou_loss, + "l1_loss": l1_loss, + "conf_loss": conf_loss, + "cls_loss": cls_loss, + "num_fg": num_fg, + } + + return self.head(xs) + + +def _forward_head(self, x): + return self.head(x) + + +# def ensure_forward_head_obj_repoint(self): +# self.forward_head = MethodType(_forward_head, self) + + +@torch.no_grad() +def make_vit_yolov3(vit: VisionTransformer, samples: torch.Tensor, patch_size, patch_dim, num_classes, + use_bigger_fpns=False, use_multi_layer_feature=False, cls_vit_ckpt_path=None, init_head=False): + + assert cls_vit_ckpt_path is None + + # vit -> fpn -> head + + # modify vit.forward() to make it output middle features + # vit.forward_features = partial(vit._intermediate_layers, + # n=[len(vit.blocks) // 3 - 1, len(vit.blocks) // 3 * 2 - 1, len(vit.blocks) - 1]) + # vit.forward_head = _forward_head + # vit.__deepcopy__ = MethodType(ensure_forward_head_obj_repoint, vit) + + vit = VisionTransformerYOLOv3.init_from_vit(vit) + + if not use_multi_layer_feature: + set_module(vit, 'head', ViTYOLOv3Head( + im_size=(samples.size(2), samples.size(3)), + patch_size=patch_size, + patch_dim=patch_dim, + num_classes=num_classes, + use_bigger_fpns=use_bigger_fpns, + cls_vit_ckpt_path=cls_vit_ckpt_path, + init_head=init_head + )) + + + else: + raise NotImplementedError + logger.info('use multi layer feature') + set_module(vit, 'head', ViTYOLOv3Head2( + im_size=(samples.size(2), samples.size(3)), + patch_size=patch_size, + patch_dim=patch_dim, + num_classes=num_classes, + use_bigger_fpns=use_bigger_fpns, + cls_vit_ckpt_path=cls_vit_ckpt_path + )) + + # print(vit) + + vit.eval() + output = vit(samples) + # print([oo.size() for oo in output]) + assert len(output) == samples.size(0) and output[0].size(1) == num_classes + 5, f'{[oo.size() for oo in output]}, {num_classes}' + + return vit + + +if __name__ == '__main__': + from dnns.vit import vit_b_16 + vit_b_16 = vit_b_16() + make_vit_yolov3(vit_b_16, torch.rand((1, 3, 224, 224)), 16, 768, 20) + exit() + + from types import MethodType + + class Student(object): + pass + + def set_name(self, name): + self.name = name + + def get_name(self): + print(self.name) + + s1 = Student() + #将方法绑定到s1和s2实例中 + s1.set_name = MethodType(set_name, s1) + s1.get_name = MethodType(get_name, s1) + s1.set_name('s1') + + from copy import deepcopy + s2 = deepcopy(s1) + s2.get_name() + + s2.set_name('s2') + s1.get_name() + s2.get_name() \ No newline at end of file diff --git a/dnns/yolov3/yolo_fpn.py b/dnns/yolov3/yolo_fpn.py new file mode 100644 index 0000000000000000000000000000000000000000..1aa786704b0844bcf90380aaebed20689ed0c02f --- /dev/null +++ b/dnns/yolov3/yolo_fpn.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +# Copyright (c) Megvii Inc. All rights reserved. + +import torch +import torch.nn as nn + +from .network_blocks import BaseConv + + +class YOLOFPN(nn.Module): + """ + YOLOFPN module. Darknet 53 is the default backbone of this model. + """ + + def __init__( + self + ): + super().__init__() + + # self.backbone = Darknet(depth) + # self.in_features = in_features + + # out 1 + self.out1_cbl = self._make_cbl(512, 256, 1) + self.out1 = self._make_embedding([256, 512], 512 + 256) + + # out 2 + self.out2_cbl = self._make_cbl(256, 128, 1) + self.out2 = self._make_embedding([128, 256], 256 + 128) + + # upsample + self.upsample = nn.Upsample(scale_factor=2, mode="nearest") + + def _make_cbl(self, _in, _out, ks): + return BaseConv(_in, _out, ks, stride=1, act="lrelu") + + def _make_embedding(self, filters_list, in_filters): + m = nn.Sequential( + *[ + self._make_cbl(in_filters, filters_list[0], 1), + self._make_cbl(filters_list[0], filters_list[1], 3), + self._make_cbl(filters_list[1], filters_list[0], 1), + self._make_cbl(filters_list[0], filters_list[1], 3), + self._make_cbl(filters_list[1], filters_list[0], 1), + ] + ) + return m + + def load_pretrained_model(self, filename="./weights/darknet53.mix.pth"): + with open(filename, "rb") as f: + state_dict = torch.load(f, map_location="cpu") + print("loading pretrained weights...") + self.backbone.load_state_dict(state_dict) + + def forward(self, backbone_out_features): + """ + Args: + inputs (Tensor): input image. + + Returns: + Tuple[Tensor]: FPN output features.. + """ + # backbone + # out_features = self.backbone(inputs) + + out_features = backbone_out_features + x2, x1, x0 = out_features + + # yolo branch 1 + x1_in = self.out1_cbl(x0) + x1_in = self.upsample(x1_in) + x1_in = torch.cat([x1_in, x1], 1) + out_dark4 = self.out1(x1_in) + + # yolo branch 2 + x2_in = self.out2_cbl(out_dark4) + x2_in = self.upsample(x2_in) + x2_in = torch.cat([x2_in, x2], 1) + out_dark3 = self.out2(x2_in) + + outputs = (out_dark3, out_dark4, x0) + return outputs \ No newline at end of file diff --git a/experiments/__init__.py b/experiments/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/experiments/__pycache__/__init__.cpython-38.pyc b/experiments/__pycache__/__init__.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..fbe309ce23c285ec7629a2812488bfc882c9271d Binary files /dev/null and b/experiments/__pycache__/__init__.cpython-38.pyc differ diff --git a/experiments/elasticdnn/bert_base/offline/fm_lora/pos/pos.py b/experiments/elasticdnn/bert_base/offline/fm_lora/pos/pos.py new file mode 100644 index 0000000000000000000000000000000000000000..591c7294304aee5021f51c9a155628a1cdb2eacd --- /dev/null +++ b/experiments/elasticdnn/bert_base/offline/fm_lora/pos/pos.py @@ -0,0 +1,110 @@ +import torch +from methods.elasticdnn.api.model import ElasticDNN_OfflineTokenClsFMModel, ElasticDNN_OfflineTokenClsMDModel +from methods.elasticdnn.api.algs.fm_lora import ElasticDNN_FMLoRAAlg +from methods.elasticdnn.model.base import ElasticDNNUtil +from methods.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from methods.elasticdnn.pipeline.offline.fm_lora.bert import FMLoRA_Bert_Util +from methods.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from methods.elasticdnn.pipeline.offline.fm_to_md.vit import FM_to_MD_ViT_Util +from methods.elasticdnn.model.vit import ElasticViTUtil +from utils.dl.common.model import LayerActivation, get_module, get_parameter, set_module +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +from utils.common.log import logger +import torch.nn.functional as F +import sys + + +class ElasticDNN_Bert_OfflineTokenClsFMModel(ElasticDNN_OfflineTokenClsFMModel): + def generate_md_by_reducing_width(self, reducing_width_ratio, samples: torch.Tensor): + # return FM_to_MD_ViT_Util().init_md_from_fm_by_reducing_width_with_perf_test(self.models_dict['main'], + # reducing_width_ratio, samples) + raise NotImplementedError + + def get_feature_hook(self) -> LayerActivation: + return LayerActivation(get_module(self.models_dict['main'], 'classifier'), True, self.device) + + def get_elastic_dnn_util(self) -> ElasticDNNUtil: + raise NotImplementedError + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + self.to_train_mode() + o = self.infer(x) + return F.cross_entropy(o.view(-1, o.size(-1)), y.view(-1)) + + def get_lora_util(self) -> FMLoRA_Util: + return FMLoRA_Bert_Util() + + def get_task_head_params(self): + head = get_module(self.models_dict['main'], 'classifier') + params_name = {k for k, v in head.named_parameters()} + logger.info(f'task head params: {params_name}') + return list(head.parameters()) + + +class ElasticDNN_Bert_OfflineTokenClsMDModel(ElasticDNN_OfflineTokenClsMDModel): + def get_feature_hook(self) -> LayerActivation: + return LayerActivation(get_module(self.models_dict['main'], 'classifier'), True, self.device) + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + self.to_train_mode() + return F.cross_entropy(self.infer(x), y) + + +if __name__ == '__main__': + from utils.dl.common.env import set_random_seed + set_random_seed(1) + + scenario = build_scenario( + source_datasets_name=['HL5Domains-ApexAD2600Progressive-TokenCls', 'HL5Domains-CanonG3-TokenCls', 'HL5Domains-CreativeLabsNomadJukeboxZenXtra40GB-TokenCls', + 'HL5Domains-NikonCoolpix4300-TokenCls', 'HL5Domains-Nokia6610-TokenCls'], + target_datasets_order=['Liu3Domains-Computer-TokenCls'] * 1, # TODO + da_mode='close_set', + data_dirs={ + **{k: f'/data/zql/datasets/nlp_asc_19_domains/dat/absa/Bing5Domains/asc/{k.split("-")[1]}' + for k in ['HL5Domains-ApexAD2600Progressive-TokenCls', 'HL5Domains-CanonG3-TokenCls', 'HL5Domains-CreativeLabsNomadJukeboxZenXtra40GB-TokenCls', + 'HL5Domains-NikonCoolpix4300-TokenCls', 'HL5Domains-Nokia6610-TokenCls']}, + 'Liu3Domains-Computer-TokenCls': '' + }, + ) + + # 2. init model + device = 'cuda' + from dnns.bert import bert_base_token_cls + model = bert_base_token_cls(num_classes=scenario.num_classes) + + fm_models_dict_path = save_models_dict_for_init({ + 'main': model + }, __file__, 'fm_bert_base_pretrained_with_token_cls_head') + + fm_model = ElasticDNN_Bert_OfflineTokenClsFMModel('fm', fm_models_dict_path, device) + + # 3. init alg + models = { + 'fm': fm_model + } + fm_lora_alg = ElasticDNN_FMLoRAAlg(models, get_res_save_dir(__file__, sys.argv[1])) + + + # 4. run alg + from utils.dl.common.lr_scheduler import get_linear_schedule_with_warmup + fm_lora_alg.run(scenario, hyps={ + 'launch_tbboard': False, + + 'samples_size': {'input_ids': torch.tensor([[ 101, 5672, 2033, 2011, 2151, 3793, 2017, 1005, 1040, 2066, 1012, 102]]).to(device), + 'token_type_ids': torch.tensor([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]).to(device), + 'attention_mask': torch.tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]).to(device), 'return_dict': False}, + + 'ab_r': 8, + 'train_batch_size': 8, + 'val_batch_size': 16, + 'num_workers': 16, + 'optimizer': 'AdamW', + 'optimizer_args': {'lr': 1e-4, 'betas': [0.9, 0.999]}, + 'scheduler': 'LambdaLR', + 'scheduler_args': {'lr_lambda': get_linear_schedule_with_warmup(10000, 310000)}, + 'num_iters': 320000, + 'val_freq': 400, + + # 'fm_lora_ckpt_path': 'experiments/elasticdnn/vit_b_16/offline/fm_lora/cls/results/cls.py/20230607/999995-234355-TokenClsial/models/fm_best.pt' + }) \ No newline at end of file diff --git a/experiments/elasticdnn/bert_base/offline/fm_lora/se_cls/se_cls.py b/experiments/elasticdnn/bert_base/offline/fm_lora/se_cls/se_cls.py new file mode 100644 index 0000000000000000000000000000000000000000..4b51cdca3b83cd71678e0721cdcf0f297c44229b --- /dev/null +++ b/experiments/elasticdnn/bert_base/offline/fm_lora/se_cls/se_cls.py @@ -0,0 +1,109 @@ +import torch +from methods.elasticdnn.api.model import ElasticDNN_OfflineSenClsFMModel, ElasticDNN_OfflineSenClsMDModel +from methods.elasticdnn.api.algs.fm_lora import ElasticDNN_FMLoRAAlg +from methods.elasticdnn.model.base import ElasticDNNUtil +from methods.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from methods.elasticdnn.pipeline.offline.fm_lora.bert import FMLoRA_Bert_Util +from methods.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from methods.elasticdnn.pipeline.offline.fm_to_md.vit import FM_to_MD_ViT_Util +from methods.elasticdnn.model.vit import ElasticViTUtil +from utils.dl.common.model import LayerActivation, get_module, get_parameter, set_module +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +from utils.common.log import logger +import torch.nn.functional as F +import sys + + +class ElasticDNN_Bert_OfflineSenClsFMModel(ElasticDNN_OfflineSenClsFMModel): + def generate_md_by_reducing_width(self, reducing_width_ratio, samples: torch.Tensor): + # return FM_to_MD_ViT_Util().init_md_from_fm_by_reducing_width_with_perf_test(self.models_dict['main'], + # reducing_width_ratio, samples) + raise NotImplementedError + + def get_feature_hook(self) -> LayerActivation: + return LayerActivation(get_module(self.models_dict['main'], 'classifier'), True, self.device) + + def get_elastic_dnn_util(self) -> ElasticDNNUtil: + raise NotImplementedError + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + self.to_train_mode() + return F.cross_entropy(self.infer(x), y) + + def get_lora_util(self) -> FMLoRA_Util: + return FMLoRA_Bert_Util() + + def get_task_head_params(self): + head = get_module(self.models_dict['main'], 'classifier') + params_name = {k for k, v in head.named_parameters()} + logger.info(f'task head params: {params_name}') + return list(head.parameters()) + + +class ElasticDNN_ViT_OfflineDetMDModel(ElasticDNN_OfflineSenClsMDModel): + def get_feature_hook(self) -> LayerActivation: + return LayerActivation(get_module(self.models_dict['main'], 'classifier'), True, self.device) + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + self.to_train_mode() + return F.cross_entropy(self.infer(x), y) + + +if __name__ == '__main__': + from utils.dl.common.env import set_random_seed + set_random_seed(1) + + scenario = build_scenario( + source_datasets_name=['HL5Domains-ApexAD2600Progressive', 'HL5Domains-CanonG3', 'HL5Domains-CreativeLabsNomadJukeboxZenXtra40GB', + 'HL5Domains-NikonCoolpix4300', 'HL5Domains-Nokia6610'], + target_datasets_order=['Liu3Domains-Computer'] * 1, # TODO + da_mode='close_set', + data_dirs={ + **{k: f'/data/zql/datasets/nlp_asc_19_domains/dat/absa/Bing5Domains/asc/{k.split("-")[1]}' + for k in ['HL5Domains-ApexAD2600Progressive', 'HL5Domains-CanonG3', 'HL5Domains-CreativeLabsNomadJukeboxZenXtra40GB', + 'HL5Domains-NikonCoolpix4300', 'HL5Domains-Nokia6610']}, + 'Liu3Domains-Computer': '' + }, + ) + + # 2. init model + device = 'cuda' + from dnns.bert import bert_base_sen_cls + model = bert_base_sen_cls(num_classes=scenario.num_classes) + + fm_models_dict_path = save_models_dict_for_init({ + 'main': model + }, __file__, 'fm_bert_base_pretrained_with_sen_cls_head') + + fm_model = ElasticDNN_Bert_OfflineSenClsFMModel('fm', fm_models_dict_path, device) + + # 3. init alg + models = { + 'fm': fm_model + } + fm_lora_alg = ElasticDNN_FMLoRAAlg(models, get_res_save_dir(__file__, sys.argv[1])) + + + # 4. run alg + from utils.dl.common.lr_scheduler import get_linear_schedule_with_warmup + fm_lora_alg.run(scenario, hyps={ + 'launch_tbboard': False, + + 'samples_size': {'input_ids': torch.tensor([[ 101, 5672, 2033, 2011, 2151, 3793, 2017, 1005, 1040, 2066, 1012, 102]]).to(device), + 'token_type_ids': torch.tensor([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]).to(device), + 'attention_mask': torch.tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]).to(device), 'return_dict': False}, + + 'ab_r': 8, + 'train_batch_size': 8, + 'val_batch_size': 16, + 'num_workers': 16, + 'optimizer': 'AdamW', + 'optimizer_args': {'lr': 1e-4, 'betas': [0.9, 0.999]}, + 'scheduler': 'LambdaLR', + 'scheduler_args': {'lr_lambda': get_linear_schedule_with_warmup(10000, 310000)}, + 'num_iters': 320000, + 'val_freq': 400, + + # 'fm_lora_ckpt_path': 'experiments/elasticdnn/vit_b_16/offline/fm_lora/cls/results/cls.py/20230607/999995-234355-trial/models/fm_best.pt' + }) \ No newline at end of file diff --git a/experiments/elasticdnn/bert_base/offline/fm_to_md/pos/pos_md_w_fbs_index.py b/experiments/elasticdnn/bert_base/offline/fm_to_md/pos/pos_md_w_fbs_index.py new file mode 100644 index 0000000000000000000000000000000000000000..314431d10c8816cc80828d770e088be59586a0d6 --- /dev/null +++ b/experiments/elasticdnn/bert_base/offline/fm_to_md/pos/pos_md_w_fbs_index.py @@ -0,0 +1,186 @@ +import torch +import sys +from torch import nn +from methods.elasticdnn.api.model import ElasticDNN_OfflineTokenClsFMModel, ElasticDNN_OfflineTokenClsMDModel +from methods.elasticdnn.api.algs.md_pretraining_wo_fbs import ElasticDNN_MDPretrainingWoFBSAlg +from methods.elasticdnn.model.base import ElasticDNNUtil +from methods.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from methods.elasticdnn.pipeline.offline.fm_to_md.bert import FM_to_MD_Bert_Util +from methods.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from methods.elasticdnn.pipeline.offline.fm_lora.bert import FMLoRA_Bert_Util +from methods.elasticdnn.model.bert import ElasticBertUtil +from utils.dl.common.model import LayerActivation2, get_module, get_parameter +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +from utils.dl.common.loss import CrossEntropyLossSoft +import torch.nn.functional as F +from utils.common.log import logger + + +class ElasticDNN_Bert_OfflineTokenClsFMModel(ElasticDNN_OfflineTokenClsFMModel): + def generate_md_by_reducing_width(self, reducing_width_ratio, samples: torch.Tensor): # TODO: + return FM_to_MD_Bert_Util().init_md_from_fm_by_reducing_width_with_perf_test(self.models_dict['main'], + reducing_width_ratio, samples) + # raise NotImplementedError + + def get_feature_hook(self) -> LayerActivation2: + return LayerActivation2(get_module(self.models_dict['main'], 'classifier')) + + def get_elastic_dnn_util(self) -> ElasticDNNUtil: # TODO: + return ElasticBertUtil() + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + self.to_train_mode() + o = self.infer(x) + return F.cross_entropy(o.view(-1, o.size(-1)), y.view(-1)) + + def get_lora_util(self) -> FMLoRA_Util: + return FMLoRA_Bert_Util() + + def get_task_head_params(self): + head = get_module(self.models_dict['main'], 'classifier') + params_name = {k for k, v in head.named_parameters()} + logger.info(f'task head params: {params_name}') + return list(head.parameters()) + + +class ElasticDNN_Bert_OfflineTokenClsMDModel(ElasticDNN_OfflineTokenClsMDModel): + def __init__(self, name: str, models_dict_path: str, device: str): + super().__init__(name, models_dict_path, device) + + self.distill_criterion = CrossEntropyLossSoft() + + def get_feature_hook(self) -> LayerActivation2: + return LayerActivation2(get_module(self.models_dict['main'], 'classifier')) + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + self.to_train_mode() + o = self.infer(x) + return F.cross_entropy(o.view(-1, o.size(-1)), y.view(-1)) + + def get_distill_loss(self, student_output, teacher_output): + # print(student_output, teacher_output) + return self.distill_criterion(student_output.view(-1, student_output.size(-1)), teacher_output.view(-1, teacher_output.size(-1))) + + def get_matched_param_of_fm(self, self_param_name, fm: nn.Module): # TODO: + if any([k in self_param_name for k in ['fbs', 'ab', 'embeddings']]): + return None + + p = get_parameter(self.models_dict['main'], self_param_name) + if p.dim() == 0: + return None + + elif p.dim() == 1 and 'LayerNorm' in self_param_name and 'weight' in self_param_name: + # if self_param_name.startswith('norm'): + # return None + return get_parameter(fm, self_param_name) + + # 1. xx.query.weight -> xx.query.fc.weight and xx.query.ab.0/1 + if ('query' in self_param_name or 'key' in self_param_name or \ + 'value' in self_param_name) and ('weight' in self_param_name): + ss = self_param_name.split('.') + # raise NotImplementedError() # TODO: + fm_qkv_name = '.'.join(ss[0: -1]) + '.fc' + fm_qkv = get_module(fm, fm_qkv_name) + + fm_abs_name = '.'.join(ss[0: -1]) + '.ab' + fm_abs = get_module(fm, fm_abs_name) + + return torch.cat([ + fm_qkv.weight.data, # task-agnositc params + fm_abs[1].weight @ fm_abs[0].weight + ], dim=0) + + elif ('query' in self_param_name or 'key' in self_param_name or \ + 'value' in self_param_name) and ('bias' in self_param_name): + ss = self_param_name.split('.') + + fm_qkv_name = '.'.join(ss[0: -1]) + '.fc.bias' + return get_parameter(fm, fm_qkv_name) + + elif 'intermediate.dense' in self_param_name: + fm_param_name = self_param_name.replace('.linear', '') + return get_parameter(fm, fm_param_name) + + else: + return get_parameter(fm, self_param_name) + + + +if __name__ == '__main__': + from utils.dl.common.env import set_random_seed + set_random_seed(1) + + # 3. init scenario + scenario = build_scenario( + source_datasets_name=['HL5Domains-ApexAD2600Progressive-TokenCls', 'HL5Domains-CanonG3-TokenCls', 'HL5Domains-CreativeLabsNomadJukeboxZenXtra40GB-TokenCls', + 'HL5Domains-NikonCoolpix4300-TokenCls', 'HL5Domains-Nokia6610-TokenCls'], + target_datasets_order=['Liu3Domains-Computer-TokenCls'] * 1, # TODO + da_mode='close_set', + data_dirs={ + **{k: f'/data/zql/datasets/nlp_asc_19_domains/dat/absa/Bing5Domains/asc/{k.split("-")[1]}' + for k in ['HL5Domains-ApexAD2600Progressive-TokenCls', 'HL5Domains-CanonG3-TokenCls', 'HL5Domains-CreativeLabsNomadJukeboxZenXtra40GB-TokenCls', + 'HL5Domains-NikonCoolpix4300-TokenCls', 'HL5Domains-Nokia6610-TokenCls']}, + 'Liu3Domains-Computer-TokenCls': '' + }, + ) + + # 1. init model + # from dnns.deeplabv3.head import modify_forward_head + # modify_forward_head() # TODO: bring a bug + # from dnns.vit import vit_b_16 + fm_models_dict_path = 'experiments/elasticdnn/bert_base/offline/fm_to_md/pos/results/pos_md_wo_fbs.py/20230703/999997-181910/models/fm_best.pt' + fm_models = torch.load(fm_models_dict_path) + fm_models_dict_path = save_models_dict_for_init(fm_models, __file__, 'fm_bert_base_secls_lora') + md_models_dict_path = save_models_dict_for_init( + torch.load('experiments/elasticdnn/bert_base/offline/fm_to_md/pos/results/pos_md_wo_fbs.py/20230703/999997-181910/models/md_best.pt'), + __file__, 'md_bert_base_secls_raw_pretrained') + device = 'cuda' + + fm_model = ElasticDNN_Bert_OfflineTokenClsFMModel('fm', fm_models_dict_path, device) + md_model = ElasticDNN_Bert_OfflineTokenClsMDModel('md', md_models_dict_path, device) + + # 2. init alg + models = { + 'fm': fm_model, + 'md': md_model + } + from methods.elasticdnn.api.algs.md_pretraining_index_v2_train_index_and_md import ElasticDNN_MDPretrainingIndexAlg + fm_to_md_alg = ElasticDNN_MDPretrainingIndexAlg(models, get_res_save_dir(__file__, sys.argv[1])) + + from utils.dl.common.lr_scheduler import get_linear_schedule_with_warmup + fm_to_md_alg.run(scenario, hyps={ + 'launch_tbboard': False, + + 'samples_size': {'input_ids': torch.tensor([[ 101, 5672, 2033, 2011, 2151, 3793, 2017, 1005, 1040, 2066, 1012, 102]]).to(device), + 'token_type_ids': torch.tensor([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]).to(device), + 'attention_mask': torch.tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]).to(device), 'return_dict': False}, + + 'FBS_r': 16, + 'FBS_ignore_layers': [], + + 'train_batch_size': 8, + 'val_batch_size': 16, + 'num_workers': 16, + 'optimizer': 'AdamW', + 'optimizer_args': {'lr': 1e-4, 'betas': [0.9, 0.999]}, + 'scheduler': 'LambdaLR', + 'scheduler_args': {'lr_lambda': get_linear_schedule_with_warmup(10000, 70000)}, + + 'indexes_optimizer_args': {'lr': 3e-3, 'momentum': 0.9, 'weight_decay': 5e-4}, + + 'num_iters': 80000, + 'val_freq': 400, + + 'max_sparsity': 0.9, + 'min_sparsity': 0.0, + 'l1_reg_loss_weight': 1e-9, + 'index_loss_weight': 1e-4, + 'val_num_sparsities': 4, + + 'bn_cal_num_iters': 0, + + 'index_init': 'zero', + 'index_guided_linear_comb_split_size': 512 + }) + \ No newline at end of file diff --git a/experiments/elasticdnn/bert_base/offline/fm_to_md/pos/pos_md_wo_fbs.py b/experiments/elasticdnn/bert_base/offline/fm_to_md/pos/pos_md_wo_fbs.py new file mode 100644 index 0000000000000000000000000000000000000000..a4e0cdf01fde704ad1bca9d5f09053f9a32a7a48 --- /dev/null +++ b/experiments/elasticdnn/bert_base/offline/fm_to_md/pos/pos_md_wo_fbs.py @@ -0,0 +1,155 @@ +import torch +import sys +from torch import nn +from methods.elasticdnn.api.model import ElasticDNN_OfflineTokenClsFMModel, ElasticDNN_OfflineTokenClsMDModel +from methods.elasticdnn.api.algs.md_pretraining_wo_fbs import ElasticDNN_MDPretrainingWoFBSAlg +from methods.elasticdnn.model.base import ElasticDNNUtil +from methods.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from methods.elasticdnn.pipeline.offline.fm_to_md.bert import FM_to_MD_Bert_Util +from methods.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from methods.elasticdnn.pipeline.offline.fm_lora.bert import FMLoRA_Bert_Util +from methods.elasticdnn.model.bert import ElasticBertUtil +from utils.dl.common.model import LayerActivation2, get_module, get_parameter +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +from utils.dl.common.loss import CrossEntropyLossSoft +import torch.nn.functional as F +from utils.common.log import logger + + +class ElasticDNN_Bert_OfflineTokenClsFMModel(ElasticDNN_OfflineTokenClsFMModel): + def generate_md_by_reducing_width(self, reducing_width_ratio, samples: torch.Tensor): # TODO: + return FM_to_MD_Bert_Util().init_md_from_fm_by_reducing_width_with_perf_test(self.models_dict['main'], + reducing_width_ratio, samples) + # raise NotImplementedError + + def get_feature_hook(self) -> LayerActivation2: + return LayerActivation2(get_module(self.models_dict['main'], 'classifier')) + + def get_elastic_dnn_util(self) -> ElasticDNNUtil: # TODO: + raise ElasticBertUtil() + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + self.to_train_mode() + o = self.infer(x) + return F.cross_entropy(o.view(-1, o.size(-1)), y.view(-1)) + + def get_lora_util(self) -> FMLoRA_Util: + return FMLoRA_Bert_Util() + + def get_task_head_params(self): + head = get_module(self.models_dict['main'], 'classifier') + params_name = {k for k, v in head.named_parameters()} + logger.info(f'task head params: {params_name}') + return list(head.parameters()) + + +class ElasticDNN_Bert_OfflineTokenClsMDModel(ElasticDNN_OfflineTokenClsMDModel): + def __init__(self, name: str, models_dict_path: str, device: str): + super().__init__(name, models_dict_path, device) + + self.distill_criterion = CrossEntropyLossSoft() + + def get_feature_hook(self) -> LayerActivation2: + return LayerActivation2(get_module(self.models_dict['main'], 'classifier')) + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + self.to_train_mode() + o = self.infer(x) + return F.cross_entropy(o.view(-1, o.size(-1)), y.view(-1)) + + def get_distill_loss(self, student_output, teacher_output): + # print(student_output, teacher_output) + return self.distill_criterion(student_output.view(-1, student_output.size(-1)), teacher_output.view(-1, teacher_output.size(-1))) + + def get_matched_param_of_fm(self, self_param_name, fm: nn.Module): # TODO: + if any([k in self_param_name for k in ['fbs', 'embeddings']]): + return None + + # 1. xx.qkv.to_qkv.yy to xx.qkv.qkv.aa and xx.qkv.abs.zz + if 'query' in self_param_name or 'key' in self_param_name or 'value' in self_param_name: + ss = self_param_name.split('.') + raise NotImplementedError() # TODO: + fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv' + fm_qkv = get_module(fm, fm_qkv_name) + + fm_abs_name = '.'.join(ss[0: -2]) + '.abs' + fm_abs = get_module(fm, fm_abs_name) + + return torch.cat([ + fm_qkv.weight.data, # task-agnositc params + torch.cat([(_abs[0].weight.T @ _abs[1].weight.T).T for _abs in fm_abs], dim=0) # task-specific params (LoRA) + ], dim=0) + + elif 'to_qkv.bias' in self_param_name: + ss = self_param_name.split('.') + + fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv.bias' + return get_parameter(fm, fm_qkv_name) + + elif 'mlp.fc1' in self_param_name: + fm_param_name = self_param_name.replace('.linear', '') + return get_parameter(fm, fm_param_name) + + else: + return get_parameter(fm, self_param_name) + + +if __name__ == '__main__': + from utils.dl.common.env import set_random_seed + set_random_seed(1) + + # 3. init scenario + scenario = build_scenario( + source_datasets_name=['HL5Domains-ApexAD2600Progressive-TokenCls', 'HL5Domains-CanonG3-TokenCls', 'HL5Domains-CreativeLabsNomadJukeboxZenXtra40GB-TokenCls', + 'HL5Domains-NikonCoolpix4300-TokenCls', 'HL5Domains-Nokia6610-TokenCls'], + target_datasets_order=['Liu3Domains-Computer-TokenCls'] * 1, # TODO + da_mode='close_set', + data_dirs={ + **{k: f'/data/zql/datasets/nlp_asc_19_domains/dat/absa/Bing5Domains/asc/{k.split("-")[1]}' + for k in ['HL5Domains-ApexAD2600Progressive-TokenCls', 'HL5Domains-CanonG3-TokenCls', 'HL5Domains-CreativeLabsNomadJukeboxZenXtra40GB-TokenCls', + 'HL5Domains-NikonCoolpix4300-TokenCls', 'HL5Domains-Nokia6610-TokenCls']}, + 'Liu3Domains-Computer-TokenCls': '' + }, + ) + + # 1. init model + fm_models_dict_path = 'experiments/elasticdnn/bert_base/offline/fm_lora/pos/results/pos.py/20230630/999989-113655-trial/models/fm_best.pt' + fm_models = torch.load(fm_models_dict_path) + fm_models_dict_path = save_models_dict_for_init(fm_models, __file__, 'fm_bert_base_tokencls_lora') + md_models_dict_path = save_models_dict_for_init({ + 'main': -1 + }, __file__, 'md_bert_base_none') + device = 'cuda' + + fm_model = ElasticDNN_Bert_OfflineTokenClsFMModel('fm', fm_models_dict_path, device) + md_model = ElasticDNN_Bert_OfflineTokenClsMDModel('md', md_models_dict_path, device) + + # 2. init alg + models = { + 'fm': fm_model, + 'md': md_model + } + fm_to_md_alg = ElasticDNN_MDPretrainingWoFBSAlg(models, get_res_save_dir(__file__, None)) + + from utils.dl.common.lr_scheduler import get_linear_schedule_with_warmup + fm_to_md_alg.run(scenario, hyps={ + 'launch_tbboard': False, + + 'samples_size': {'input_ids': torch.tensor([[ 101, 5672, 2033, 2011, 2151, 3793, 2017, 1005, 1040, 2066, 1012, 102]]).to(device), + 'token_type_ids': torch.tensor([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]).to(device), + 'attention_mask': torch.tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]).to(device), 'return_dict': False}, + 'generate_md_width_ratio': 4, + + 'train_batch_size': 8, + 'val_batch_size': 16, + 'num_workers': 16, + 'optimizer': 'AdamW', + 'optimizer_args': {'lr': 1e-4, 'betas': [0.9, 0.999]}, + 'scheduler': 'LambdaLR', + 'scheduler_args': {'lr_lambda': get_linear_schedule_with_warmup(10000, 70000)}, + 'num_iters': 80000, + 'val_freq': 4000, + 'distill_loss_weight': 1.0 + }) + \ No newline at end of file diff --git a/experiments/elasticdnn/bert_base/offline/fm_to_md/se_cls/secls_md_w_fbs_index.py b/experiments/elasticdnn/bert_base/offline/fm_to_md/se_cls/secls_md_w_fbs_index.py new file mode 100644 index 0000000000000000000000000000000000000000..5c9365980d1316bf06edae845f5c1c895ddb3aa9 --- /dev/null +++ b/experiments/elasticdnn/bert_base/offline/fm_to_md/se_cls/secls_md_w_fbs_index.py @@ -0,0 +1,184 @@ +import torch +import sys +from torch import nn +from methods.elasticdnn.api.model import ElasticDNN_OfflineSenClsFMModel, ElasticDNN_OfflineSenClsMDModel +from methods.elasticdnn.api.algs.md_pretraining_wo_fbs import ElasticDNN_MDPretrainingWoFBSAlg +from methods.elasticdnn.model.base import ElasticDNNUtil +from methods.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from methods.elasticdnn.pipeline.offline.fm_to_md.bert import FM_to_MD_Bert_Util +from methods.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from methods.elasticdnn.pipeline.offline.fm_lora.bert import FMLoRA_Bert_Util +from methods.elasticdnn.model.bert import ElasticBertUtil +from utils.dl.common.model import LayerActivation2, get_module, get_parameter +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +from utils.dl.common.loss import CrossEntropyLossSoft +import torch.nn.functional as F +from utils.common.log import logger + + +class ElasticDNN_Bert_OfflineSenClsFMModel(ElasticDNN_OfflineSenClsFMModel): + def generate_md_by_reducing_width(self, reducing_width_ratio, samples: torch.Tensor): # TODO: + return FM_to_MD_Bert_Util().init_md_from_fm_by_reducing_width_with_perf_test(self.models_dict['main'], + reducing_width_ratio, samples) + # raise NotImplementedError + + def get_feature_hook(self) -> LayerActivation2: + return LayerActivation2(get_module(self.models_dict['main'], 'classifier')) + + def get_elastic_dnn_util(self) -> ElasticDNNUtil: # TODO: + return ElasticBertUtil() + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + self.to_train_mode() + return F.cross_entropy(self.infer(x), y) + + def get_lora_util(self) -> FMLoRA_Util: + return FMLoRA_Bert_Util() + + def get_task_head_params(self): + head = get_module(self.models_dict['main'], 'classifier') + params_name = {k for k, v in head.named_parameters()} + logger.info(f'task head params: {params_name}') + return list(head.parameters()) + + +class ElasticDNN_Bert_OfflineSenClsMDModel(ElasticDNN_OfflineSenClsMDModel): + def __init__(self, name: str, models_dict_path: str, device: str): + super().__init__(name, models_dict_path, device) + + self.distill_criterion = CrossEntropyLossSoft() + + def get_feature_hook(self) -> LayerActivation2: + return LayerActivation2(get_module(self.models_dict['main'], 'classifier')) + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + self.to_train_mode() + return F.cross_entropy(self.infer(x), y) + + def get_distill_loss(self, student_output, teacher_output): + # print(student_output, teacher_output) + return self.distill_criterion(student_output, teacher_output) + + def get_matched_param_of_fm(self, self_param_name, fm: nn.Module): # TODO: + if any([k in self_param_name for k in ['fbs', 'ab', 'embeddings']]): + return None + + p = get_parameter(self.models_dict['main'], self_param_name) + if p.dim() == 0: + return None + + elif p.dim() == 1 and 'LayerNorm' in self_param_name and 'weight' in self_param_name: + # if self_param_name.startswith('norm'): + # return None + return get_parameter(fm, self_param_name) + + # 1. xx.query.weight -> xx.query.fc.weight and xx.query.ab.0/1 + if ('query' in self_param_name or 'key' in self_param_name or \ + 'value' in self_param_name) and ('weight' in self_param_name): + ss = self_param_name.split('.') + # raise NotImplementedError() # TODO: + fm_qkv_name = '.'.join(ss[0: -1]) + '.fc' + fm_qkv = get_module(fm, fm_qkv_name) + + fm_abs_name = '.'.join(ss[0: -1]) + '.ab' + fm_abs = get_module(fm, fm_abs_name) + + return torch.cat([ + fm_qkv.weight.data, # task-agnositc params + fm_abs[1].weight @ fm_abs[0].weight + ], dim=0) + + elif ('query' in self_param_name or 'key' in self_param_name or \ + 'value' in self_param_name) and ('bias' in self_param_name): + ss = self_param_name.split('.') + + fm_qkv_name = '.'.join(ss[0: -1]) + '.fc.bias' + return get_parameter(fm, fm_qkv_name) + + elif 'intermediate.dense' in self_param_name: + fm_param_name = self_param_name.replace('.linear', '') + return get_parameter(fm, fm_param_name) + + else: + return get_parameter(fm, self_param_name) + + + +if __name__ == '__main__': + from utils.dl.common.env import set_random_seed + set_random_seed(1) + + # 3. init scenario + scenario = build_scenario( + source_datasets_name=['HL5Domains-ApexAD2600Progressive', 'HL5Domains-CanonG3', 'HL5Domains-CreativeLabsNomadJukeboxZenXtra40GB', + 'HL5Domains-NikonCoolpix4300', 'HL5Domains-Nokia6610'], + target_datasets_order=['Liu3Domains-Computer'] * 1, # TODO + da_mode='close_set', + data_dirs={ + **{k: f'/data/zql/datasets/nlp_asc_19_domains/dat/absa/Bing5Domains/asc/{k.split("-")[1]}' + for k in ['HL5Domains-ApexAD2600Progressive', 'HL5Domains-CanonG3', 'HL5Domains-CreativeLabsNomadJukeboxZenXtra40GB', + 'HL5Domains-NikonCoolpix4300', 'HL5Domains-Nokia6610']}, + 'Liu3Domains-Computer': '' + }, + ) + + # 1. init model + # from dnns.deeplabv3.head import modify_forward_head + # modify_forward_head() # TODO: bring a bug + # from dnns.vit import vit_b_16 + fm_models_dict_path = 'experiments/elasticdnn/bert_base/offline/fm_to_md/se_cls/results/secls_md_wo_fbs.py/20230703/999971-162231/models/fm_best.pt' + fm_models = torch.load(fm_models_dict_path) + fm_models_dict_path = save_models_dict_for_init(fm_models, __file__, 'fm_bert_base_secls_lora') + md_models_dict_path = save_models_dict_for_init( + torch.load('experiments/elasticdnn/bert_base/offline/fm_to_md/se_cls/results/secls_md_wo_fbs.py/20230703/999971-162231/models/md_best.pt'), + __file__, 'md_bert_base_secls_raw_pretrained') + device = 'cuda' + + fm_model = ElasticDNN_Bert_OfflineSenClsFMModel('fm', fm_models_dict_path, device) + md_model = ElasticDNN_Bert_OfflineSenClsMDModel('md', md_models_dict_path, device) + + # 2. init alg + models = { + 'fm': fm_model, + 'md': md_model + } + from methods.elasticdnn.api.algs.md_pretraining_index_v2_train_index_and_md import ElasticDNN_MDPretrainingIndexAlg + fm_to_md_alg = ElasticDNN_MDPretrainingIndexAlg(models, get_res_save_dir(__file__, sys.argv[1])) + + from utils.dl.common.lr_scheduler import get_linear_schedule_with_warmup + fm_to_md_alg.run(scenario, hyps={ + 'launch_tbboard': False, + + 'samples_size': {'input_ids': torch.tensor([[ 101, 5672, 2033, 2011, 2151, 3793, 2017, 1005, 1040, 2066, 1012, 102]]).to(device), + 'token_type_ids': torch.tensor([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]).to(device), + 'attention_mask': torch.tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]).to(device), 'return_dict': False}, + + 'FBS_r': 16, + 'FBS_ignore_layers': [], + + 'train_batch_size': 8, + 'val_batch_size': 16, + 'num_workers': 16, + 'optimizer': 'AdamW', + 'optimizer_args': {'lr': 1e-4, 'betas': [0.9, 0.999]}, + 'scheduler': 'LambdaLR', + 'scheduler_args': {'lr_lambda': get_linear_schedule_with_warmup(10000, 70000)}, + + 'indexes_optimizer_args': {'lr': 3e-3, 'momentum': 0.9, 'weight_decay': 5e-4}, + + 'num_iters': 80000, + 'val_freq': 400, + + 'max_sparsity': 0.9, + 'min_sparsity': 0.0, + 'l1_reg_loss_weight': 1e-9, + 'index_loss_weight': 1e-4, + 'val_num_sparsities': 4, + + 'bn_cal_num_iters': 0, + + 'index_init': 'zero', + 'index_guided_linear_comb_split_size': 512 + }) + \ No newline at end of file diff --git a/experiments/elasticdnn/bert_base/offline/fm_to_md/se_cls/secls_md_wo_fbs.py b/experiments/elasticdnn/bert_base/offline/fm_to_md/se_cls/secls_md_wo_fbs.py new file mode 100644 index 0000000000000000000000000000000000000000..3ddc12831486ac9fb4ca5cbc91c03db2a45dc246 --- /dev/null +++ b/experiments/elasticdnn/bert_base/offline/fm_to_md/se_cls/secls_md_wo_fbs.py @@ -0,0 +1,153 @@ +import torch +import sys +from torch import nn +from methods.elasticdnn.api.model import ElasticDNN_OfflineSenClsFMModel, ElasticDNN_OfflineSenClsMDModel +from methods.elasticdnn.api.algs.md_pretraining_wo_fbs import ElasticDNN_MDPretrainingWoFBSAlg +from methods.elasticdnn.model.base import ElasticDNNUtil +from methods.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from methods.elasticdnn.pipeline.offline.fm_to_md.bert import FM_to_MD_Bert_Util +from methods.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from methods.elasticdnn.pipeline.offline.fm_lora.bert import FMLoRA_Bert_Util +from methods.elasticdnn.model.bert import ElasticBertUtil +from utils.dl.common.model import LayerActivation2, get_module, get_parameter +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +from utils.dl.common.loss import CrossEntropyLossSoft +import torch.nn.functional as F +from utils.common.log import logger + + +class ElasticDNN_Bert_OfflineSenClsFMModel(ElasticDNN_OfflineSenClsFMModel): + def generate_md_by_reducing_width(self, reducing_width_ratio, samples: torch.Tensor): # TODO: + return FM_to_MD_Bert_Util().init_md_from_fm_by_reducing_width_with_perf_test(self.models_dict['main'], + reducing_width_ratio, samples) + # raise NotImplementedError + + def get_feature_hook(self) -> LayerActivation2: + return LayerActivation2(get_module(self.models_dict['main'], 'classifier')) + + def get_elastic_dnn_util(self) -> ElasticDNNUtil: # TODO: + raise ElasticBertUtil() + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + self.to_train_mode() + return F.cross_entropy(self.infer(x), y) + + def get_lora_util(self) -> FMLoRA_Util: + return FMLoRA_Bert_Util() + + def get_task_head_params(self): + head = get_module(self.models_dict['main'], 'classifier') + params_name = {k for k, v in head.named_parameters()} + logger.info(f'task head params: {params_name}') + return list(head.parameters()) + + +class ElasticDNN_Bert_OfflineSenClsMDModel(ElasticDNN_OfflineSenClsMDModel): + def __init__(self, name: str, models_dict_path: str, device: str): + super().__init__(name, models_dict_path, device) + + self.distill_criterion = CrossEntropyLossSoft() + + def get_feature_hook(self) -> LayerActivation2: + return LayerActivation2(get_module(self.models_dict['main'], 'classifier')) + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + self.to_train_mode() + return F.cross_entropy(self.infer(x), y) + + def get_distill_loss(self, student_output, teacher_output): + # print(student_output, teacher_output) + return self.distill_criterion(student_output, teacher_output) + + def get_matched_param_of_fm(self, self_param_name, fm: nn.Module): # TODO: + if any([k in self_param_name for k in ['fbs', 'embeddings']]): + return None + + # 1. xx.qkv.to_qkv.yy to xx.qkv.qkv.aa and xx.qkv.abs.zz + if 'query' in self_param_name or 'key' in self_param_name or 'value' in self_param_name: + ss = self_param_name.split('.') + raise NotImplementedError() # TODO: + fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv' + fm_qkv = get_module(fm, fm_qkv_name) + + fm_abs_name = '.'.join(ss[0: -2]) + '.abs' + fm_abs = get_module(fm, fm_abs_name) + + return torch.cat([ + fm_qkv.weight.data, # task-agnositc params + torch.cat([(_abs[0].weight.T @ _abs[1].weight.T).T for _abs in fm_abs], dim=0) # task-specific params (LoRA) + ], dim=0) + + elif 'to_qkv.bias' in self_param_name: + ss = self_param_name.split('.') + + fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv.bias' + return get_parameter(fm, fm_qkv_name) + + elif 'mlp.fc1' in self_param_name: + fm_param_name = self_param_name.replace('.linear', '') + return get_parameter(fm, fm_param_name) + + else: + return get_parameter(fm, self_param_name) + + +if __name__ == '__main__': + from utils.dl.common.env import set_random_seed + set_random_seed(1) + + # 3. init scenario + scenario = build_scenario( + source_datasets_name=['HL5Domains-ApexAD2600Progressive', 'HL5Domains-CanonG3', 'HL5Domains-CreativeLabsNomadJukeboxZenXtra40GB', + 'HL5Domains-NikonCoolpix4300', 'HL5Domains-Nokia6610'], + target_datasets_order=['Liu3Domains-Computer'] * 1, # TODO + da_mode='close_set', + data_dirs={ + **{k: f'/data/zql/datasets/nlp_asc_19_domains/dat/absa/Bing5Domains/asc/{k.split("-")[1]}' + for k in ['HL5Domains-ApexAD2600Progressive', 'HL5Domains-CanonG3', 'HL5Domains-CreativeLabsNomadJukeboxZenXtra40GB', + 'HL5Domains-NikonCoolpix4300', 'HL5Domains-Nokia6610']}, + 'Liu3Domains-Computer': '' + }, + ) + + # 1. init model + fm_models_dict_path = 'experiments/elasticdnn/bert_base/offline/fm_lora/se_cls/results/se_cls.py/20230628/star_multilinguial_ok_999981-190501-multi_lin_bert_trial/models/fm_best.pt' + fm_models = torch.load(fm_models_dict_path) + fm_models_dict_path = save_models_dict_for_init(fm_models, __file__, 'fm_bert_base_secls_lora') + md_models_dict_path = save_models_dict_for_init({ + 'main': -1 + }, __file__, 'md_bert_base_none') + device = 'cuda' + + fm_model = ElasticDNN_Bert_OfflineSenClsFMModel('fm', fm_models_dict_path, device) + md_model = ElasticDNN_Bert_OfflineSenClsMDModel('md', md_models_dict_path, device) + + # 2. init alg + models = { + 'fm': fm_model, + 'md': md_model + } + fm_to_md_alg = ElasticDNN_MDPretrainingWoFBSAlg(models, get_res_save_dir(__file__, None)) + + from utils.dl.common.lr_scheduler import get_linear_schedule_with_warmup + fm_to_md_alg.run(scenario, hyps={ + 'launch_tbboard': False, + + 'samples_size': {'input_ids': torch.tensor([[ 101, 5672, 2033, 2011, 2151, 3793, 2017, 1005, 1040, 2066, 1012, 102]]).to(device), + 'token_type_ids': torch.tensor([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]).to(device), + 'attention_mask': torch.tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]).to(device), 'return_dict': False}, + 'generate_md_width_ratio': 4, + + 'train_batch_size': 8, + 'val_batch_size': 16, + 'num_workers': 16, + 'optimizer': 'AdamW', + 'optimizer_args': {'lr': 1e-4, 'betas': [0.9, 0.999]}, + 'scheduler': 'LambdaLR', + 'scheduler_args': {'lr_lambda': get_linear_schedule_with_warmup(10000, 70000)}, + 'num_iters': 80000, + 'val_freq': 4000, + 'distill_loss_weight': 1.0 + }) + \ No newline at end of file diff --git a/experiments/elasticdnn/bert_base/online/pos/model.py b/experiments/elasticdnn/bert_base/online/pos/model.py new file mode 100644 index 0000000000000000000000000000000000000000..b5aea488b686f3edd0b748bbca5daa4f1fd0f622 --- /dev/null +++ b/experiments/elasticdnn/bert_base/online/pos/model.py @@ -0,0 +1,251 @@ +from typing import List +from data.dataloader import build_dataloader +# from methods.elasticdnn.api.online_model import ElasticDNN_OnlineModel +from methods.elasticdnn.api.online_model_v2 import ElasticDNN_OnlineModel + +import torch +import sys +from torch import nn +from methods.elasticdnn.api.model import ElasticDNN_OfflineSegFMModel, ElasticDNN_OfflineSegMDModel +from methods.elasticdnn.api.algs.md_pretraining_wo_fbs import ElasticDNN_MDPretrainingWoFBSAlg +from methods.elasticdnn.model.base import ElasticDNNUtil +from methods.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from methods.elasticdnn.pipeline.offline.fm_to_md.vit import FM_to_MD_ViT_Util +from methods.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from methods.elasticdnn.pipeline.offline.fm_lora.vit import FMLoRA_ViT_Util +from methods.elasticdnn.model.bert import ElasticBertUtil +from utils.common.file import ensure_dir +from utils.dl.common.model import LayerActivation, get_module, get_parameter +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +from utils.dl.common.loss import CrossEntropyLossSoft +import torch.nn.functional as F +from utils.dl.common.env import create_tbwriter +import os +from utils.common.log import logger +from utils.common.data_record import write_json +# from methods.shot.shot import OnlineShotModel +from methods.feat_align.main import OnlineFeatAlignModel, FeatAlignAlg +import tqdm +from methods.feat_align.mmd import mmd_rbf + +class ElasticDNN_POSOnlineModel(ElasticDNN_OnlineModel): + def get_accuracy(self, test_loader, *args, **kwargs): + acc = 0 + sample_num = 0 + + self.to_eval_mode() + + with torch.no_grad(): + pbar = tqdm.tqdm(enumerate(test_loader), total=len(test_loader), dynamic_ncols=True, leave=False) + for batch_index, (x, y) in pbar: + for k, v in x.items(): + if isinstance(v, torch.Tensor): + x[k] = v.to(self.device) + # print(x) + y = y.to(self.device) + output = self.infer(x) + + # torch.Size([16, 512, 43]) torch.Size([16, 512]) + + for oi, yi, xi in zip(output, y, x['input_ids']): + # oi: 512, 43; yi: 512 + seq_len = xi.nonzero().size(0) + + # print(output.size(), y.size()) + + pred = F.softmax(oi, dim=-1).argmax(dim=-1) + correct = torch.eq(pred[1: seq_len], yi[1: seq_len]).sum().item() + + # print(output.size(), y.size()) + + acc += correct + sample_num += seq_len + + # pbar.set_description(f'seq_len: {seq_len}, cur_seq_acc: {(correct / seq_len):.4f}') + + acc /= sample_num + return acc + + def get_elastic_dnn_util(self) -> ElasticDNNUtil: + return ElasticBertUtil() + + def get_fm_matched_param_of_md_param(self, md_param_name): + # only between qkv.weight, norm.weight/bias + self_param_name = md_param_name + fm = self.models_dict['fm'] + if any([k in self_param_name for k in ['fbs', 'ab', 'embeddings']]): + return None + + p = get_parameter(self.models_dict['md'], self_param_name) + if p.dim() == 0: + return None + elif p.dim() == 1 and 'LayerNorm' in self_param_name and 'weight' in self_param_name: + return get_parameter(fm, self_param_name) + + # 1. xx.qkv.to_qkv.yy to xx.qkv.qkv.aa and xx.qkv.abs.zz + if ('query' in self_param_name or 'key' in self_param_name or \ + 'value' in self_param_name) and ('weight' in self_param_name): + + ss = self_param_name.split('.') + + fm_qkv_name = '.'.join(ss[0: -1]) + '.fc' + fm_qkv = get_module(fm, fm_qkv_name) + + fm_abs_name = '.'.join(ss[0: -1]) + '.ab' + fm_abs = get_module(fm, fm_abs_name) + + # NOTE: unrecoverable operation! multiply LoRA parameters to allow it being updated in update_fm_param() + # TODO: if fm will be used for inference, _mul_lora_weight will not be applied! + if not hasattr(fm_abs, '_mul_lora_weight'): + logger.debug(f'set _mul_lora_weight in {fm_abs_name}') + setattr(fm_abs, '_mul_lora_weight', + nn.Parameter(fm_abs[1].weight @ fm_abs[0].weight)) + + return torch.cat([ + fm_qkv.weight.data, # task-agnositc params + fm_abs._mul_lora_weight.data # task-specific params (LoRA) + ], dim=0) + + # elif 'to_qkv.bias' in self_param_name: + # ss = self_param_name.split('.') + + # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv.bias' + # return get_parameter(fm, fm_qkv_name) + + elif 'dense' in self_param_name and 'weight' in self_param_name: + fm_param_name = self_param_name.replace('.linear', '') + return get_parameter(fm, fm_param_name) + + # elif 'mlp.fc2' in self_param_name and 'weight' in self_param_name: + # fm_param_name = self_param_name + # return get_parameter(fm, fm_param_name) + + else: + # return get_parameter(fm, self_param_name) + return None + + def update_fm_param(self, md_param_name, cal_new_fm_param_by_md_param): + if not ('query' in md_param_name or 'key' in md_param_name or 'value' in md_param_name): + matched_fm_param_ref = self.get_fm_matched_param_of_md_param(md_param_name) + matched_fm_param_ref.copy_(cal_new_fm_param_by_md_param) + else: + new_fm_attn_weight, new_fm_lora_weight = torch.chunk(cal_new_fm_param_by_md_param, 2, 0) + + ss = md_param_name.split('.') + fm = self.models_dict['fm'] + + # update task-agnostic parameters + fm_qkv_name = '.'.join(ss[0: -1]) + '.fc' + fm_qkv = get_module(fm, fm_qkv_name) + fm_qkv.weight.data.copy_(new_fm_attn_weight) + + # update task-specific parameters + fm_abs_name = '.'.join(ss[0: -1]) + '.ab' + fm_abs = get_module(fm, fm_abs_name) + fm_abs._mul_lora_weight.data.copy_(new_fm_lora_weight) # TODO: this will not be applied in inference! + + def get_md_matched_param_of_fm_param(self, fm_param_name): + return super().get_md_matched_param_of_fm_param(fm_param_name) + + def get_md_matched_param_of_sd_param(self, sd_param_name): + # raise NotImplementedError + + # only between qkv.weight, norm.weight/bias + self_param_name = sd_param_name + md = self.models_dict['md'] + if any([k in self_param_name for k in ['fbs', 'ab', 'embeddings']]): + return None + + p = get_parameter(self.models_dict['sd'], self_param_name) + if p.dim() == 0: + return None + elif p.dim() == 1 and 'LayerNorm' in self_param_name and 'weight' in self_param_name: + return get_parameter(md, self_param_name) + + # 1. xx.qkv.to_qkv.yy to xx.qkv.qkv.aa and xx.qkv.abs.zz + if ('query' in self_param_name or 'key' in self_param_name or \ + 'value' in self_param_name) and ('weight' in self_param_name): + + + return get_parameter(md, self_param_name) # NOTE: no fbs in qkv! + + # elif 'to_qkv.bias' in self_param_name: + # ss = self_param_name.split('.') + + # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv.bias' + # return get_parameter(fm, fm_qkv_name) + + elif 'intermediate.dense.0.weight' in self_param_name: + fm_param_name = '.'.join(self_param_name.split('.')[0: -2]) + '.linear.weight' + return get_parameter(md, fm_param_name) + + elif 'output.dense' in self_param_name and 'weight' in self_param_name: + fm_param_name = self_param_name + return get_parameter(md, fm_param_name) + + else: + # return get_parameter(fm, self_param_name) + return None + + def get_task_head_params(self): + head = get_module(self.models_dict['sd'], 'classifier') + return list(head.parameters()) + +class POSOnlineFeatAlignModel(OnlineFeatAlignModel): + def get_trained_params(self): + qkv_and_norm_params = [p for n, p in self.models_dict['main'].named_parameters() if 'query' in n or 'key' in n or 'value' in n or 'dense' in n or 'LayerNorm' in n] + return qkv_and_norm_params + + def get_feature_hook(self): + return LayerActivation(get_module(self.models_dict['main'], 'classifier'), False, self.device) + + def forward_to_get_task_loss(self, x, y): + self.to_train_mode() + o = self.infer(x) + return F.cross_entropy(o.view(-1, o.size(-1)), y.view(-1)) + + def get_mmd_loss(self, f1, f2): + # print(f1.size()) + # return mmd_rbf(f1.mean(1).flatten(1), f2.mean(1).flatten(1)) + return mmd_rbf(f1.flatten(1), f2.flatten(1)) + + def infer(self, x, *args, **kwargs): + return self.models_dict['main'](**x) + + def get_accuracy(self, test_loader, *args, **kwargs): + acc = 0 + sample_num = 0 + + self.to_eval_mode() + + with torch.no_grad(): + pbar = tqdm.tqdm(enumerate(test_loader), total=len(test_loader), dynamic_ncols=True, leave=False) + for batch_index, (x, y) in pbar: + for k, v in x.items(): + if isinstance(v, torch.Tensor): + x[k] = v.to(self.device) + # print(x) + y = y.to(self.device) + output = self.infer(x) + + # torch.Size([16, 512, 43]) torch.Size([16, 512]) + + for oi, yi, xi in zip(output, y, x['input_ids']): + # oi: 512, 43; yi: 512 + seq_len = xi.nonzero().size(0) + + # print(output.size(), y.size()) + + pred = F.softmax(oi, dim=-1).argmax(dim=-1) + correct = torch.eq(pred[1: seq_len], yi[1: seq_len]).sum().item() + + # print(output.size(), y.size()) + + acc += correct + sample_num += seq_len + + # pbar.set_description(f'seq_len: {seq_len}, cur_seq_acc: {(correct / seq_len):.4f}') + + acc /= sample_num + return acc \ No newline at end of file diff --git a/experiments/elasticdnn/bert_base/online/pos/pos.py b/experiments/elasticdnn/bert_base/online/pos/pos.py new file mode 100644 index 0000000000000000000000000000000000000000..c1d1f4b4087b054fc6c2b29a69eb031d33f5891d --- /dev/null +++ b/experiments/elasticdnn/bert_base/online/pos/pos.py @@ -0,0 +1,108 @@ +from typing import List +from data.dataloader import build_dataloader +# from methods.elasticdnn.api.online_model import ElasticDNN_OnlineModel +from methods.elasticdnn.api.online_model_v2 import ElasticDNN_OnlineModel + +import torch +import sys +from torch import nn +from methods.elasticdnn.api.model import ElasticDNN_OfflineSegFMModel, ElasticDNN_OfflineSegMDModel +from methods.elasticdnn.api.algs.md_pretraining_wo_fbs import ElasticDNN_MDPretrainingWoFBSAlg +from methods.elasticdnn.model.base import ElasticDNNUtil +from methods.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from methods.elasticdnn.pipeline.offline.fm_to_md.vit import FM_to_MD_ViT_Util +from methods.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from methods.elasticdnn.pipeline.offline.fm_lora.vit import FMLoRA_ViT_Util +from methods.elasticdnn.model.vit import ElasticViTUtil +from utils.common.file import ensure_dir +from utils.dl.common.model import LayerActivation, get_module, get_parameter +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +from utils.dl.common.loss import CrossEntropyLossSoft +import torch.nn.functional as F +from utils.dl.common.env import create_tbwriter +import os +from utils.common.log import logger +from utils.common.data_record import write_json +# from methods.shot.shot import OnlineShotModel +from methods.feat_align.main import OnlineFeatAlignModel, FeatAlignAlg +import tqdm +from methods.feat_align.mmd import mmd_rbf +from experiments.utils.elasticfm_da import init_online_model, elasticfm_da + +device = 'cuda' +app_name = 'pos' +sd_sparsity = 0.8 + +settings = { + 'involve_fm': True +} + +scenario = build_scenario( + source_datasets_name=[i + '-TokenCls' for i in ['HL5Domains-ApexAD2600Progressive', 'HL5Domains-CanonG3', 'HL5Domains-CreativeLabsNomadJukeboxZenXtra40GB', + 'HL5Domains-NikonCoolpix4300', 'HL5Domains-Nokia6610']], + target_datasets_order=[i + '-TokenCls' for i in ['Liu3Domains-Computer', 'Liu3Domains-Router', 'Liu3Domains-Speaker', + 'Ding9Domains-DiaperChamp', 'Ding9Domains-Norton', 'Ding9Domains-LinksysRouter', + 'Ding9Domains-MicroMP3', 'Ding9Domains-Nokia6600', 'Ding9Domains-CanonPowerShotSD500', + 'Ding9Domains-ipod', 'Ding9Domains-HitachiRouter', 'Ding9Domains-CanonS100', + 'SemEval-Laptop', 'SemEval-Rest'] * 2 + ['Liu3Domains-Computer', 'Liu3Domains-Router']], + da_mode='close_set', + data_dirs={ + **{k: f'/data/zql/datasets/nlp_asc_19_domains/dat/absa/Bing5Domains/asc/{k.split("-")[1]}' + for k in [i + '-TokenCls' for i in ['HL5Domains-ApexAD2600Progressive', 'HL5Domains-CanonG3', 'HL5Domains-CreativeLabsNomadJukeboxZenXtra40GB', + 'HL5Domains-NikonCoolpix4300', 'HL5Domains-Nokia6610']]}, + + **{k: f'/data/zql/datasets/nlp_asc_19_domains/dat/absa/Bing3Domains/asc/{k.split("-")[1]}' + for k in [i + '-TokenCls' for i in ['Liu3Domains-Computer', 'Liu3Domains-Router', 'Liu3Domains-Speaker']]}, + + **{k: f'/data/zql/datasets/nlp_asc_19_domains/dat/absa/Bing9Domains/asc/{k.split("-")[1]}' + for k in [i + '-TokenCls' for i in ['Ding9Domains-DiaperChamp', 'Ding9Domains-Norton', 'Ding9Domains-LinksysRouter', + 'Ding9Domains-MicroMP3', 'Ding9Domains-Nokia6600', 'Ding9Domains-CanonPowerShotSD500', + 'Ding9Domains-ipod', 'Ding9Domains-HitachiRouter', 'Ding9Domains-CanonS100']]}, + + **{k: f'/data/zql/datasets/nlp_asc_19_domains/dat/absa/XuSemEval/asc/14/{k.split("-")[1].lower()}' + for k in [i + '-TokenCls' for i in ['SemEval-Laptop', 'SemEval-Rest']]}, + }, +) + + +from experiments.elasticdnn.bert_base.online.pos.model import ElasticDNN_POSOnlineModel +elasticfm_model = ElasticDNN_POSOnlineModel('pos', init_online_model( + 'experiments/elasticdnn/bert_base/offline/fm_to_md/pos/results/pos_md_w_fbs_index.py/20230704/999998-085253-trial/models/fm_best.pt', + 'experiments/elasticdnn/bert_base/offline/fm_to_md/pos/results/pos_md_w_fbs_index.py/20230704/999998-085253-trial/models/md_best.pt', + 'pos', __file__ +), device, { + 'md_to_fm_alpha': 0.1, + 'fm_to_md_alpha': 0.1 +}) + +da_alg = FeatAlignAlg +from experiments.elasticdnn.bert_base.online.pos.model import POSOnlineFeatAlignModel +da_model = POSOnlineFeatAlignModel +da_alg_hyp = { + 'train_batch_size': 16, + 'val_batch_size': 64, + 'num_workers': 8, + 'optimizer': 'SGD', + 'optimizer_args': {'lr': 1e-4, 'momentum': 0.9}, + 'scheduler': '', + 'scheduler_args': {}, + 'num_iters': 100, + 'val_freq': 20, + 'feat_align_loss_weight': 1.0, + 'sd_sparsity': 0.7 +} + + +elasticfm_da( + [app_name], + [scenario], + [elasticfm_model], + [da_alg], + [da_alg_hyp], + [da_model], + device, + settings, + __file__, + sys.argv[1] +) diff --git a/experiments/elasticdnn/bert_base/online/se_cls/model.py b/experiments/elasticdnn/bert_base/online/se_cls/model.py new file mode 100644 index 0000000000000000000000000000000000000000..499051d8cbb593802032573a8bdb9196a6a714e1 --- /dev/null +++ b/experiments/elasticdnn/bert_base/online/se_cls/model.py @@ -0,0 +1,228 @@ +from typing import List +from data.dataloader import build_dataloader +# from methods.elasticdnn.api.online_model import ElasticDNN_OnlineModel +from methods.elasticdnn.api.online_model_v2 import ElasticDNN_OnlineModel + +import torch +import sys +from torch import nn +from methods.elasticdnn.api.model import ElasticDNN_OfflineSegFMModel, ElasticDNN_OfflineSegMDModel +from methods.elasticdnn.api.algs.md_pretraining_wo_fbs import ElasticDNN_MDPretrainingWoFBSAlg +from methods.elasticdnn.model.base import ElasticDNNUtil +from methods.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from methods.elasticdnn.pipeline.offline.fm_to_md.vit import FM_to_MD_ViT_Util +from methods.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from methods.elasticdnn.pipeline.offline.fm_lora.vit import FMLoRA_ViT_Util +from methods.elasticdnn.model.bert import ElasticBertUtil +from utils.common.file import ensure_dir +from utils.dl.common.model import LayerActivation, get_module, get_parameter +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +from utils.dl.common.loss import CrossEntropyLossSoft +import torch.nn.functional as F +from utils.dl.common.env import create_tbwriter +import os +from utils.common.log import logger +from utils.common.data_record import write_json +# from methods.shot.shot import OnlineShotModel +from methods.feat_align.main import OnlineFeatAlignModel, FeatAlignAlg +import tqdm +from methods.feat_align.mmd import mmd_rbf + +class ElasticDNN_SeClsOnlineModel(ElasticDNN_OnlineModel): + def get_accuracy(self, test_loader, *args, **kwargs): + acc = 0 + sample_num = 0 + + self.to_eval_mode() + + with torch.no_grad(): + pbar = tqdm.tqdm(enumerate(test_loader), total=len(test_loader), dynamic_ncols=True, leave=False) + for batch_index, (x, y) in pbar: + for k, v in x.items(): + if isinstance(v, torch.Tensor): + x[k] = v.to(self.device) + y = y.to(self.device) + output = self.infer(x) + pred = F.softmax(output, dim=1).argmax(dim=1) + correct = torch.eq(pred, y).sum().item() + acc += correct + sample_num += len(y) + + pbar.set_description(f'cur_batch_total: {len(y)}, cur_batch_correct: {correct}, ' + f'cur_batch_acc: {(correct / len(y)):.4f}') + + acc /= sample_num + return acc + + def get_elastic_dnn_util(self) -> ElasticDNNUtil: + return ElasticBertUtil() + + def get_fm_matched_param_of_md_param(self, md_param_name): + # only between qkv.weight, norm.weight/bias + self_param_name = md_param_name + fm = self.models_dict['fm'] + if any([k in self_param_name for k in ['fbs', 'ab', 'embeddings']]): + return None + + p = get_parameter(self.models_dict['md'], self_param_name) + if p.dim() == 0: + return None + elif p.dim() == 1 and 'LayerNorm' in self_param_name and 'weight' in self_param_name: + return get_parameter(fm, self_param_name) + + # 1. xx.qkv.to_qkv.yy to xx.qkv.qkv.aa and xx.qkv.abs.zz + if ('query' in self_param_name or 'key' in self_param_name or \ + 'value' in self_param_name) and ('weight' in self_param_name): + + ss = self_param_name.split('.') + + fm_qkv_name = '.'.join(ss[0: -1]) + '.fc' + fm_qkv = get_module(fm, fm_qkv_name) + + fm_abs_name = '.'.join(ss[0: -1]) + '.ab' + fm_abs = get_module(fm, fm_abs_name) + + # NOTE: unrecoverable operation! multiply LoRA parameters to allow it being updated in update_fm_param() + # TODO: if fm will be used for inference, _mul_lora_weight will not be applied! + if not hasattr(fm_abs, '_mul_lora_weight'): + logger.debug(f'set _mul_lora_weight in {fm_abs_name}') + setattr(fm_abs, '_mul_lora_weight', + nn.Parameter(fm_abs[1].weight @ fm_abs[0].weight)) + + return torch.cat([ + fm_qkv.weight.data, # task-agnositc params + fm_abs._mul_lora_weight.data # task-specific params (LoRA) + ], dim=0) + + # elif 'to_qkv.bias' in self_param_name: + # ss = self_param_name.split('.') + + # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv.bias' + # return get_parameter(fm, fm_qkv_name) + + elif 'dense' in self_param_name and 'weight' in self_param_name: + fm_param_name = self_param_name.replace('.linear', '') + return get_parameter(fm, fm_param_name) + + # elif 'mlp.fc2' in self_param_name and 'weight' in self_param_name: + # fm_param_name = self_param_name + # return get_parameter(fm, fm_param_name) + + else: + # return get_parameter(fm, self_param_name) + return None + + def update_fm_param(self, md_param_name, cal_new_fm_param_by_md_param): + if not ('query' in md_param_name or 'key' in md_param_name or 'value' in md_param_name): + matched_fm_param_ref = self.get_fm_matched_param_of_md_param(md_param_name) + matched_fm_param_ref.copy_(cal_new_fm_param_by_md_param) + else: + new_fm_attn_weight, new_fm_lora_weight = torch.chunk(cal_new_fm_param_by_md_param, 2, 0) + + ss = md_param_name.split('.') + fm = self.models_dict['fm'] + + # update task-agnostic parameters + fm_qkv_name = '.'.join(ss[0: -1]) + '.fc' + fm_qkv = get_module(fm, fm_qkv_name) + fm_qkv.weight.data.copy_(new_fm_attn_weight) + + # update task-specific parameters + fm_abs_name = '.'.join(ss[0: -1]) + '.ab' + fm_abs = get_module(fm, fm_abs_name) + fm_abs._mul_lora_weight.data.copy_(new_fm_lora_weight) # TODO: this will not be applied in inference! + + def get_md_matched_param_of_fm_param(self, fm_param_name): + return super().get_md_matched_param_of_fm_param(fm_param_name) + + def get_md_matched_param_of_sd_param(self, sd_param_name): + # raise NotImplementedError + + # only between qkv.weight, norm.weight/bias + self_param_name = sd_param_name + md = self.models_dict['md'] + if any([k in self_param_name for k in ['fbs', 'ab', 'embeddings']]): + return None + + p = get_parameter(self.models_dict['sd'], self_param_name) + if p.dim() == 0: + return None + elif p.dim() == 1 and 'LayerNorm' in self_param_name and 'weight' in self_param_name: + return get_parameter(md, self_param_name) + + # 1. xx.qkv.to_qkv.yy to xx.qkv.qkv.aa and xx.qkv.abs.zz + if ('query' in self_param_name or 'key' in self_param_name or \ + 'value' in self_param_name) and ('weight' in self_param_name): + + + return get_parameter(md, self_param_name) # NOTE: no fbs in qkv! + + # elif 'to_qkv.bias' in self_param_name: + # ss = self_param_name.split('.') + + # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv.bias' + # return get_parameter(fm, fm_qkv_name) + + elif 'intermediate.dense.0.weight' in self_param_name: + fm_param_name = '.'.join(self_param_name.split('.')[0: -2]) + '.linear.weight' + return get_parameter(md, fm_param_name) + + elif 'output.dense' in self_param_name and 'weight' in self_param_name: + fm_param_name = self_param_name + return get_parameter(md, fm_param_name) + + else: + # return get_parameter(fm, self_param_name) + return None + + def get_task_head_params(self): + head = get_module(self.models_dict['sd'], 'classifier') + return list(head.parameters()) + + +class SeClsOnlineFeatAlignModel(OnlineFeatAlignModel): + def get_trained_params(self): + qkv_and_norm_params = [p for n, p in self.models_dict['main'].named_parameters() if 'query' in n or 'key' in n or 'value' in n or 'dense' in n or 'LayerNorm' in n] + return qkv_and_norm_params + + def get_feature_hook(self): + return LayerActivation(get_module(self.models_dict['main'], 'classifier'), False, self.device) + + def forward_to_get_task_loss(self, x, y): + return F.cross_entropy(self.infer(x), y) + + def get_mmd_loss(self, f1, f2): + return mmd_rbf(f1, f2) + + def infer(self, x, *args, **kwargs): + return self.models_dict['main'](**x) + + def get_accuracy(self, test_loader, *args, **kwargs): + acc = 0 + sample_num = 0 + + self.to_eval_mode() + + with torch.no_grad(): + pbar = tqdm.tqdm(enumerate(test_loader), total=len(test_loader), dynamic_ncols=True, leave=False) + for batch_index, (x, y) in pbar: + for k, v in x.items(): + if isinstance(v, torch.Tensor): + x[k] = v.to(self.device) + y = y.to(self.device) + output = self.infer(x) + pred = F.softmax(output, dim=1).argmax(dim=1) + correct = torch.eq(pred, y).sum().item() + + if batch_index == 0: + logger.info(f'{pred}, {y}') + + acc += correct + sample_num += len(y) + + pbar.set_description(f'cur_batch_total: {len(y)}, cur_batch_correct: {correct}, ' + f'cur_batch_acc: {(correct / len(y)):.4f}') + + acc /= sample_num + return acc \ No newline at end of file diff --git a/experiments/elasticdnn/bert_base/online/se_cls/se_cls.py b/experiments/elasticdnn/bert_base/online/se_cls/se_cls.py new file mode 100644 index 0000000000000000000000000000000000000000..9f686f71db7a91047ac92ae5b15d245ba24e1b62 --- /dev/null +++ b/experiments/elasticdnn/bert_base/online/se_cls/se_cls.py @@ -0,0 +1,110 @@ +from typing import List +from data.dataloader import build_dataloader +# from methods.elasticdnn.api.online_model import ElasticDNN_OnlineModel +from methods.elasticdnn.api.online_model_v2 import ElasticDNN_OnlineModel + +import torch +import sys +from torch import nn +from methods.elasticdnn.api.model import ElasticDNN_OfflineSegFMModel, ElasticDNN_OfflineSegMDModel +from methods.elasticdnn.api.algs.md_pretraining_wo_fbs import ElasticDNN_MDPretrainingWoFBSAlg +from methods.elasticdnn.model.base import ElasticDNNUtil +from methods.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from methods.elasticdnn.pipeline.offline.fm_to_md.vit import FM_to_MD_ViT_Util +from methods.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from methods.elasticdnn.pipeline.offline.fm_lora.vit import FMLoRA_ViT_Util +from methods.elasticdnn.model.vit import ElasticViTUtil +from utils.common.file import ensure_dir +from utils.dl.common.model import LayerActivation, get_module, get_parameter +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +from utils.dl.common.loss import CrossEntropyLossSoft +import torch.nn.functional as F +from utils.dl.common.env import create_tbwriter +import os +from utils.common.log import logger +from utils.common.data_record import write_json +# from methods.shot.shot import OnlineShotModel +from methods.gem.gem_el_bert import GEMAlg +import tqdm +from methods.feat_align.mmd import mmd_rbf +from experiments.utils.elasticfm_da import init_online_model, elasticfm_da + +device = 'cuda' +app_name = 'secls' +sd_sparsity = 0.8 + +settings = { + 'involve_fm': True +} + +scenario = build_scenario( + source_datasets_name=['HL5Domains-ApexAD2600Progressive', 'HL5Domains-CanonG3', 'HL5Domains-CreativeLabsNomadJukeboxZenXtra40GB', + 'HL5Domains-NikonCoolpix4300', 'HL5Domains-Nokia6610'], + target_datasets_order=['Liu3Domains-Computer', 'Liu3Domains-Router', 'Liu3Domains-Speaker', + 'Ding9Domains-DiaperChamp', 'Ding9Domains-Norton', 'Ding9Domains-LinksysRouter', + 'Ding9Domains-MicroMP3', 'Ding9Domains-Nokia6600', 'Ding9Domains-CanonPowerShotSD500', + 'Ding9Domains-ipod', 'Ding9Domains-HitachiRouter', 'Ding9Domains-CanonS100', + 'SemEval-Laptop', 'SemEval-Rest'] * 2 + ['Liu3Domains-Computer', 'Liu3Domains-Router'], + da_mode='close_set', + data_dirs={ + **{k: f'/data/zql/datasets/nlp_asc_19_domains/dat/absa/Bing5Domains/asc/{k.split("-")[1]}' + for k in ['HL5Domains-ApexAD2600Progressive', 'HL5Domains-CanonG3', 'HL5Domains-CreativeLabsNomadJukeboxZenXtra40GB', + 'HL5Domains-NikonCoolpix4300', 'HL5Domains-Nokia6610']}, + + **{k: f'/data/zql/datasets/nlp_asc_19_domains/dat/absa/Bing3Domains/asc/{k.split("-")[1]}' + for k in ['Liu3Domains-Computer', 'Liu3Domains-Router', 'Liu3Domains-Speaker']}, + + **{k: f'/data/zql/datasets/nlp_asc_19_domains/dat/absa/Bing9Domains/asc/{k.split("-")[1]}' + for k in ['Ding9Domains-DiaperChamp', 'Ding9Domains-Norton', 'Ding9Domains-LinksysRouter', + 'Ding9Domains-MicroMP3', 'Ding9Domains-Nokia6600', 'Ding9Domains-CanonPowerShotSD500', + 'Ding9Domains-ipod', 'Ding9Domains-HitachiRouter', 'Ding9Domains-CanonS100']}, + + **{k: f'/data/zql/datasets/nlp_asc_19_domains/dat/absa/XuSemEval/asc/14/{k.split("-")[1].lower()}' + for k in ['SemEval-Laptop', 'SemEval-Rest']}, + }, +) + +from experiments.elasticdnn.bert_base.online.se_cls_cl.model import ElasticDNN_SeClsOnlineModel +elasticfm_model = ElasticDNN_SeClsOnlineModel('secls', init_online_model( + 'experiments/elasticdnn/bert_base/offline/fm_to_md/se_cls/results/secls_md_w_fbs_index.py/20230704/999994-085209-logic_verify/models/fm_best.pt', + 'experiments/elasticdnn/bert_base/offline/fm_to_md/se_cls/results/secls_md_w_fbs_index.py/20230704/999994-085209-logic_verify/models/md_best.pt', + 'cls', __file__ +), device, { + 'md_to_fm_alpha': 0.2, + 'fm_to_md_alpha': 0.2 +}) + +da_alg = GEMAlg +from experiments.elasticdnn.bert_base.online.se_cls_cl.model import SeClsOnlineGEMModel +da_model = SeClsOnlineGEMModel +da_alg_hyp = { + 'train_batch_size': 16, + 'val_batch_size': 64, + 'num_workers': 8, + 'optimizer': 'AdamW', + 'optimizer_args': {'lr': 1e-4, 'betas': [0.9, 0.999], 'weight_decay': 0.01}, + 'scheduler': '', + 'scheduler_args': {}, + 'num_iters': 100, + 'val_freq': 20, + 'n_memories': 16, + 'n_inputs': 3 * 224 * 224, + 'margin': 0.5, + 'num_my_iters': 0, + 'sd_sparsity': 0.7 +} + + +elasticfm_da( + [app_name], + [scenario], + [elasticfm_model], + [da_alg], + [da_alg_hyp], + [da_model], + device, + settings, + __file__, + sys.argv[1] +) diff --git a/experiments/elasticdnn/bert_base/online/se_cls_cl/model.py b/experiments/elasticdnn/bert_base/online/se_cls_cl/model.py new file mode 100644 index 0000000000000000000000000000000000000000..0b479756ab6bd31725a4062d47ff4ced371726cc --- /dev/null +++ b/experiments/elasticdnn/bert_base/online/se_cls_cl/model.py @@ -0,0 +1,341 @@ +from typing import List +from data.dataloader import build_dataloader +# from methods.elasticdnn.api.online_model import ElasticDNN_OnlineModel +from methods.elasticdnn.api.online_model_v2 import ElasticDNN_OnlineModel + +import torch +import sys +from torch import nn +from methods.elasticdnn.api.model import ElasticDNN_OfflineSegFMModel, ElasticDNN_OfflineSegMDModel +from methods.elasticdnn.api.algs.md_pretraining_wo_fbs import ElasticDNN_MDPretrainingWoFBSAlg +from methods.elasticdnn.model.base import ElasticDNNUtil +from methods.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from methods.elasticdnn.pipeline.offline.fm_to_md.vit import FM_to_MD_ViT_Util +from methods.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from methods.elasticdnn.pipeline.offline.fm_lora.vit import FMLoRA_ViT_Util +from methods.elasticdnn.model.bert import ElasticBertUtil +from utils.common.file import ensure_dir +from utils.dl.common.model import LayerActivation, get_module, get_parameter, set_module +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +from utils.dl.common.loss import CrossEntropyLossSoft +import torch.nn.functional as F +from utils.dl.common.env import create_tbwriter +import os +from utils.common.log import logger +from utils.common.data_record import write_json +# from methods.shot.shot import OnlineShotModel +from methods.feat_align.main import OnlineFeatAlignModel, FeatAlignAlg +import tqdm +from methods.feat_align.mmd import mmd_rbf + +class ElasticDNN_SeClsOnlineModel(ElasticDNN_OnlineModel): + + @torch.no_grad() + def sd_feedback_to_md(self, after_da_sd, unpruned_indexes_of_layers): + self.models_dict['sd'] = after_da_sd + self.before_da_md = deepcopy(self.models_dict['md']) + + logger.info('\n\nsurrogate DNN feedback to master DNN...\n\n') + # one-to-one + + cur_unpruned_indexes = None + cur_unpruned_indexes_name = None + + for p_name, p in self.models_dict['sd'].named_parameters(): + matched_md_param = self.get_md_matched_param_of_sd_param(p_name) + logger.debug(f'if feedback: {p_name}') + if matched_md_param is None: + continue + logger.debug(f'start feedback: {p_name}, {p.size()} -> {matched_md_param.size()}') + # average + # setattr(matched_md_module, matched_md_param_name, (matched_md_param + p) / 2.) + + if p_name in unpruned_indexes_of_layers.keys(): + cur_unpruned_indexes = unpruned_indexes_of_layers[p_name] + cur_unpruned_indexes_name = p_name + + if p.size() != matched_md_param.size(): + logger.debug(f'cur unpruned indexes: {cur_unpruned_indexes_name}, {cur_unpruned_indexes.size()}') + + if p.dim() == 1: # norm + new_p = deepcopy(matched_md_param) + new_p[cur_unpruned_indexes] = p + elif p.dim() == 2: # linear + if p.size(0) < matched_md_param.size(0): # output pruned + new_p = deepcopy(matched_md_param) + new_p[cur_unpruned_indexes] = p + else: # input pruned + new_p = deepcopy(matched_md_param) + new_p[:, cur_unpruned_indexes] = p + p = new_p + + assert p.size() == matched_md_param.size(), f'{p.size()}, {matched_md_param.size()}' + + if 'classifier' in p_name: + continue + # if False: + # self.last_trained_cls_indexes + assert hasattr(self, 'last_trained_cls_indexes') + print(self.last_trained_cls_indexes) + + diff = self._compute_diff(matched_md_param, p) + # matched_md_param[self.last_trained_cls_indexes].copy_(p[self.last_trained_cls_indexes.to(self.device)]) + matched_md_param.copy_(p) + logger.debug(f'SPECIFIC FOR CL HEAD | end feedback: {p_name}, diff: {diff:.6f}') + else: + diff = self._compute_diff(matched_md_param, (matched_md_param + p) / 2.) + matched_md_param.copy_((matched_md_param + p) / 2.) + logger.debug(f'end feedback: {p_name}, diff: {diff:.6f}') + + def add_cls_in_head(self, num_cls): + head: nn.Linear = get_module(self.models_dict['md'], 'classifier') + + new_head = nn.Linear(head.in_features, head.out_features + num_cls, head.bias is not None, device=self.device) + + # nn.init.zeros_(new_head.weight.data) + # nn.init.zeros_(new_head.bias.data) + + new_head.weight.data[0: head.out_features] = deepcopy(head.weight.data) + new_head.bias.data[0: head.out_features] = deepcopy(head.bias.data) + set_module(self.models_dict['md'], 'classifier', new_head) + set_module(self.models_dict['fm'], 'classifier', new_head) + + + def get_accuracy(self, test_loader, *args, **kwargs): + acc = 0 + sample_num = 0 + + self.to_eval_mode() + + with torch.no_grad(): + pbar = tqdm.tqdm(enumerate(test_loader), total=len(test_loader), dynamic_ncols=True, leave=False) + for batch_index, (x, y) in pbar: + for k, v in x.items(): + if isinstance(v, torch.Tensor): + x[k] = v.to(self.device) + y = y.to(self.device) + output = self.infer(x) + pred = F.softmax(output, dim=1).argmax(dim=1) + correct = torch.eq(pred, y).sum().item() + acc += correct + sample_num += len(y) + + pbar.set_description(f'cur_batch_total: {len(y)}, cur_batch_correct: {correct}, ' + f'cur_batch_acc: {(correct / len(y)):.4f}') + + acc /= sample_num + return acc + + def get_elastic_dnn_util(self) -> ElasticDNNUtil: + return ElasticBertUtil() + + def get_fm_matched_param_of_md_param(self, md_param_name): + # only between qkv.weight, norm.weight/bias + self_param_name = md_param_name + fm = self.models_dict['fm'] + if any([k in self_param_name for k in ['fbs', 'ab', 'embeddings']]): + return None + + p = get_parameter(self.models_dict['md'], self_param_name) + if p.dim() == 0: + return None + elif p.dim() == 1 and 'LayerNorm' in self_param_name and 'weight' in self_param_name: + return get_parameter(fm, self_param_name) + + # 1. xx.qkv.to_qkv.yy to xx.qkv.qkv.aa and xx.qkv.abs.zz + if ('query' in self_param_name or 'key' in self_param_name or \ + 'value' in self_param_name) and ('weight' in self_param_name): + + ss = self_param_name.split('.') + + fm_qkv_name = '.'.join(ss[0: -1]) + '.fc' + fm_qkv = get_module(fm, fm_qkv_name) + + fm_abs_name = '.'.join(ss[0: -1]) + '.ab' + fm_abs = get_module(fm, fm_abs_name) + + # NOTE: unrecoverable operation! multiply LoRA parameters to allow it being updated in update_fm_param() + # TODO: if fm will be used for inference, _mul_lora_weight will not be applied! + if not hasattr(fm_abs, '_mul_lora_weight'): + logger.debug(f'set _mul_lora_weight in {fm_abs_name}') + setattr(fm_abs, '_mul_lora_weight', + nn.Parameter(fm_abs[1].weight @ fm_abs[0].weight)) + + return torch.cat([ + fm_qkv.weight.data, # task-agnositc params + fm_abs._mul_lora_weight.data # task-specific params (LoRA) + ], dim=0) + + # elif 'to_qkv.bias' in self_param_name: + # ss = self_param_name.split('.') + + # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv.bias' + # return get_parameter(fm, fm_qkv_name) + + elif 'dense' in self_param_name and 'weight' in self_param_name: + fm_param_name = self_param_name.replace('.linear', '') + return get_parameter(fm, fm_param_name) + + # elif 'mlp.fc2' in self_param_name and 'weight' in self_param_name: + # fm_param_name = self_param_name + # return get_parameter(fm, fm_param_name) + + else: + # return get_parameter(fm, self_param_name) + return None + + def update_fm_param(self, md_param_name, cal_new_fm_param_by_md_param): + if not ('query' in md_param_name or 'key' in md_param_name or 'value' in md_param_name): + matched_fm_param_ref = self.get_fm_matched_param_of_md_param(md_param_name) + matched_fm_param_ref.copy_(cal_new_fm_param_by_md_param) + else: + new_fm_attn_weight, new_fm_lora_weight = torch.chunk(cal_new_fm_param_by_md_param, 2, 0) + + ss = md_param_name.split('.') + fm = self.models_dict['fm'] + + # update task-agnostic parameters + fm_qkv_name = '.'.join(ss[0: -1]) + '.fc' + fm_qkv = get_module(fm, fm_qkv_name) + fm_qkv.weight.data.copy_(new_fm_attn_weight) + + # update task-specific parameters + fm_abs_name = '.'.join(ss[0: -1]) + '.ab' + fm_abs = get_module(fm, fm_abs_name) + fm_abs._mul_lora_weight.data.copy_(new_fm_lora_weight) # TODO: this will not be applied in inference! + + def get_md_matched_param_of_fm_param(self, fm_param_name): + return super().get_md_matched_param_of_fm_param(fm_param_name) + + def get_md_matched_param_of_sd_param(self, sd_param_name): + # raise NotImplementedError + + # only between qkv.weight, norm.weight/bias + self_param_name = sd_param_name + md = self.models_dict['md'] + if any([k in self_param_name for k in ['fbs', 'ab', 'embeddings']]): + return None + + p = get_parameter(self.models_dict['sd'], self_param_name) + if p.dim() == 0: + return None + elif p.dim() == 1 and 'LayerNorm' in self_param_name and 'weight' in self_param_name: + return get_parameter(md, self_param_name) + + if 'classifier' in self_param_name: + return get_parameter(md, self_param_name) + + # 1. xx.qkv.to_qkv.yy to xx.qkv.qkv.aa and xx.qkv.abs.zz + if ('query' in self_param_name or 'key' in self_param_name or \ + 'value' in self_param_name) and ('weight' in self_param_name): + + + return get_parameter(md, self_param_name) # NOTE: no fbs in qkv! + + # elif 'to_qkv.bias' in self_param_name: + # ss = self_param_name.split('.') + + # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv.bias' + # return get_parameter(fm, fm_qkv_name) + + elif 'intermediate.dense.0.weight' in self_param_name: + fm_param_name = '.'.join(self_param_name.split('.')[0: -2]) + '.linear.weight' + return get_parameter(md, fm_param_name) + + elif 'output.dense' in self_param_name and 'weight' in self_param_name: + fm_param_name = self_param_name + return get_parameter(md, fm_param_name) + + else: + # return get_parameter(fm, self_param_name) + return None + + def get_task_head_params(self): + head = get_module(self.models_dict['sd'], 'classifier') + return list(head.parameters()) + + +from methods.gem.gem_el_bert import OnlineGEMModel +import tqdm +from methods.feat_align.mmd import mmd_rbf +from copy import deepcopy + + +class SeClsOnlineGEMModel(OnlineGEMModel): + def get_trained_params(self): + qkv_and_norm_params = [p for n, p in self.models_dict['main'].named_parameters() if 'query' in n or 'key' in n or 'value' in n or 'dense' in n or 'LayerNorm' in n] + return qkv_and_norm_params + + def forward_to_get_task_loss(self, x, y): + return F.cross_entropy(self.infer(x), y) + + def add_cls_in_head(self, num_cls): + return + + head: nn.Linear = get_module(self.models_dict['main'], 'head') + + new_head = nn.Linear(head.in_features, head.out_features + num_cls, head.bias is not None, device=self.device) + new_head.weight.data[0: head.out_features] = deepcopy(head.weight.data) + new_head.bias.data[0: head.out_features] = deepcopy(head.bias.data) + set_module(self.models_dict['main'], 'head', new_head) + + def infer(self, x, *args, **kwargs): + return self.models_dict['main'](**x) + + def get_accuracy(self, test_loader, *args, **kwargs): + _d = test_loader.dataset + from data import build_dataloader, split_dataset + if _d.__class__.__name__ == '_SplitDataset' and _d.underlying_dataset.__class__.__name__ == 'MergedDataset': # necessary for CL + print('\neval on merged datasets') + + merged_full_dataset = _d.underlying_dataset.datasets + ratio = len(_d.keys) / len(_d.underlying_dataset) + + if int(len(_d) * ratio) == 0: + ratio = 1. + # print(ratio) + # bs = + # test_loaders = [build_dataloader(split_dataset(d, min(max(test_loader.batch_size, int(len(d) * ratio)), len(d)))[0], # TODO: this might be overlapped with train dataset + # min(test_loader.batch_size, int(len(d) * ratio)), + # test_loader.num_workers, False, None) for d in merged_full_dataset] + + test_loaders = [] + for d in merged_full_dataset: + n = int(len(d) * ratio) + if n == 0: + n = len(d) + sub_dataset = split_dataset(d, min(max(test_loader.batch_size, n), len(d)))[0] + loader = build_dataloader(sub_dataset, min(test_loader.batch_size, n), test_loader.num_workers, False, None) + test_loaders += [loader] + + accs = [self.get_accuracy(loader) for loader in test_loaders] + print(accs) + return sum(accs) / len(accs) + + acc = 0 + sample_num = 0 + + self.to_eval_mode() + + with torch.no_grad(): + pbar = tqdm.tqdm(enumerate(test_loader), total=len(test_loader), dynamic_ncols=True, leave=False) + for batch_index, (x, y) in pbar: + for k, v in x.items(): + if isinstance(v, torch.Tensor): + x[k] = v.to(self.device) + y = y.to(self.device) + output = self.infer(x) + pred = F.softmax(output, dim=1).argmax(dim=1) + correct = torch.eq(pred, y).sum().item() + acc += correct + sample_num += len(y) + + # if batch_index == 0: + # print(pred, y) + + pbar.set_description(f'cur_batch_total: {len(y)}, cur_batch_correct: {correct}, ' + f'cur_batch_acc: {(correct / len(y)):.4f}') + + acc /= sample_num + return acc \ No newline at end of file diff --git a/experiments/elasticdnn/bert_base/online/se_cls_cl/secls_cl.py b/experiments/elasticdnn/bert_base/online/se_cls_cl/secls_cl.py new file mode 100644 index 0000000000000000000000000000000000000000..12a5f747c53ae94797728c6b09f6a01efeaa266a --- /dev/null +++ b/experiments/elasticdnn/bert_base/online/se_cls_cl/secls_cl.py @@ -0,0 +1,120 @@ +from typing import List +from data.dataloader import build_dataloader +# from methods.elasticdnn.api.online_model import ElasticDNN_OnlineModel +from methods.elasticdnn.api.online_model_v2 import ElasticDNN_OnlineModel + +import torch +import sys +from torch import nn +from methods.elasticdnn.api.model import ElasticDNN_OfflineSegFMModel, ElasticDNN_OfflineSegMDModel +from methods.elasticdnn.api.algs.md_pretraining_wo_fbs import ElasticDNN_MDPretrainingWoFBSAlg +from methods.elasticdnn.model.base import ElasticDNNUtil +from methods.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from methods.elasticdnn.pipeline.offline.fm_to_md.vit import FM_to_MD_ViT_Util +from methods.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from methods.elasticdnn.pipeline.offline.fm_lora.vit import FMLoRA_ViT_Util +from methods.elasticdnn.model.vit import ElasticViTUtil +from utils.common.file import ensure_dir +from utils.dl.common.model import LayerActivation, get_module, get_parameter +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +from utils.dl.common.loss import CrossEntropyLossSoft +import torch.nn.functional as F +from utils.dl.common.env import create_tbwriter +import os +from utils.common.log import logger +from utils.common.data_record import write_json +# from methods.shot.shot import OnlineShotModel +from methods.gem.gem_el_bert import GEMAlg +import tqdm +from methods.feat_align.mmd import mmd_rbf +from experiments.utils.elasticfm_cl import init_online_model, elasticfm_cl +from data import build_cl_scenario, build_scenario + +device = 'cuda' +app_name = 'se_cls' +sd_sparsity = 0.8 + +settings = { + 'involve_fm': True +} + +scenario = scenario = build_scenario( + source_datasets_name=['HL5Domains-ApexAD2600Progressive', 'HL5Domains-CanonG3', 'HL5Domains-CreativeLabsNomadJukeboxZenXtra40GB', + 'HL5Domains-NikonCoolpix4300', 'HL5Domains-Nokia6610'], + target_datasets_order=['Liu3Domains-Computer', 'Liu3Domains-Router', 'Liu3Domains-Speaker', + 'Ding9Domains-DiaperChamp', 'Ding9Domains-Norton', 'Ding9Domains-LinksysRouter', + 'Ding9Domains-MicroMP3', 'Ding9Domains-Nokia6600', 'Ding9Domains-CanonPowerShotSD500', + 'Ding9Domains-ipod', 'Ding9Domains-HitachiRouter', 'Ding9Domains-CanonS100', + 'SemEval-Laptop', 'SemEval-Rest'] * 2 + ['Liu3Domains-Computer', 'Liu3Domains-Router'], + da_mode='close_set', + data_dirs={ + **{k: f'/data/zql/datasets/nlp_asc_19_domains/dat/absa/Bing5Domains/asc/{k.split("-")[1]}' + for k in ['HL5Domains-ApexAD2600Progressive', 'HL5Domains-CanonG3', 'HL5Domains-CreativeLabsNomadJukeboxZenXtra40GB', + 'HL5Domains-NikonCoolpix4300', 'HL5Domains-Nokia6610']}, + + **{k: f'/data/zql/datasets/nlp_asc_19_domains/dat/absa/Bing3Domains/asc/{k.split("-")[1]}' + for k in ['Liu3Domains-Computer', 'Liu3Domains-Router', 'Liu3Domains-Speaker']}, + + **{k: f'/data/zql/datasets/nlp_asc_19_domains/dat/absa/Bing9Domains/asc/{k.split("-")[1]}' + for k in ['Ding9Domains-DiaperChamp', 'Ding9Domains-Norton', 'Ding9Domains-LinksysRouter', + 'Ding9Domains-MicroMP3', 'Ding9Domains-Nokia6600', 'Ding9Domains-CanonPowerShotSD500', + 'Ding9Domains-ipod', 'Ding9Domains-HitachiRouter', 'Ding9Domains-CanonS100']}, + + **{k: f'/data/zql/datasets/nlp_asc_19_domains/dat/absa/XuSemEval/asc/14/{k.split("-")[1].lower()}' + for k in ['SemEval-Laptop', 'SemEval-Rest']}, + }, +) +scenario = build_cl_scenario( + da_scenario=scenario, + target_datasets_name=['Newsgroup'] * 8, + num_classes_per_task=5, + max_num_tasks=30, + data_dirs={ + 'Newsgroup': '' + }, +) + +from experiments.elasticdnn.bert_base.online.se_cls_cl.model import ElasticDNN_SeClsOnlineModel +elasticfm_model = ElasticDNN_SeClsOnlineModel('secls', init_online_model( + 'experiments/elasticdnn/bert_base/offline/fm_to_md/se_cls/results/secls_md_w_fbs_index.py/20230704/999994-085209-logic_verify/models/fm_best.pt', + 'experiments/elasticdnn/bert_base/offline/fm_to_md/se_cls/results/secls_md_w_fbs_index.py/20230704/999994-085209-logic_verify/models/md_best.pt', + 'cls', __file__ +), device, { + 'md_to_fm_alpha': 0.2, + 'fm_to_md_alpha': 0.2 +}) + +da_alg = GEMAlg +from experiments.elasticdnn.bert_base.online.se_cls_cl.model import SeClsOnlineGEMModel +da_model = SeClsOnlineGEMModel +da_alg_hyp = { + 'train_batch_size': 16, + 'val_batch_size': 64, + 'num_workers': 8, + 'optimizer': 'AdamW', + 'optimizer_args': {'lr': 1e-4, 'betas': [0.9, 0.999], 'weight_decay': 0.01}, + 'scheduler': '', + 'scheduler_args': {}, + 'num_iters': 100, + 'val_freq': 20, + 'n_memories': 64, + 'n_inputs': 3 * 224 * 224, + 'margin': 0.5, + 'num_my_iters': 0, + 'sd_sparsity': sd_sparsity, +} + + +elasticfm_cl( + [app_name], + [scenario], + [elasticfm_model], + [da_alg], + [da_alg_hyp], + [da_model], + device, + settings, + __file__, + sys.argv[1] +) diff --git a/experiments/elasticdnn/gpt_neo/1.3B_ckpt/config.json b/experiments/elasticdnn/gpt_neo/1.3B_ckpt/config.json new file mode 100644 index 0000000000000000000000000000000000000000..1aac412fd4a63c7d9e7efe3f5771042b2e0c6166 --- /dev/null +++ b/experiments/elasticdnn/gpt_neo/1.3B_ckpt/config.json @@ -0,0 +1,72 @@ +{ + "activation_function": "gelu_new", + "architectures": [ + "GPTNeoForCausalLM" + ], + "attention_dropout": 0, + "attention_layers": [ + "global", + "local", + "global", + "local", + "global", + "local", + "global", + "local", + "global", + "local", + "global", + "local", + "global", + "local", + "global", + "local", + "global", + "local", + "global", + "local", + "global", + "local", + "global", + "local" + ], + "attention_types": [ + [ + [ + "global", + "local" + ], + 12 + ] + ], + "bos_token_id": 50256, + "embed_dropout": 0, + "eos_token_id": 50256, + "gradient_checkpointing": false, + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": null, + "layer_norm_epsilon": 1e-05, + "max_position_embeddings": 2048, + "model_type": "gpt_neo", + "num_heads": 16, + "num_layers": 24, + "resid_dropout": 0, + "summary_activation": null, + "summary_first_dropout": 0.1, + "summary_proj_to_labels": true, + "summary_type": "cls_index", + "summary_use_proj": true, + "task_specific_params": { + "text-generation": { + "do_sample": true, + "max_length": 50, + "temperature": 0.9 + } + }, + "tokenizer_class": "GPT2Tokenizer", + "transformers_version": "4.9.0.dev0", + "use_cache": true, + "vocab_size": 50257, + "window_size": 256 +} \ No newline at end of file diff --git a/experiments/elasticdnn/gpt_neo/1.3B_ckpt/merges.txt b/experiments/elasticdnn/gpt_neo/1.3B_ckpt/merges.txt new file mode 100644 index 0000000000000000000000000000000000000000..6636bda4a1fd7a63653dffb22683b8162c8de956 --- /dev/null +++ b/experiments/elasticdnn/gpt_neo/1.3B_ckpt/merges.txt @@ -0,0 +1,50001 @@ +#version: 0.2 - Trained by `huggingface/tokenizers` +Ġ t +Ġ a +h e +i n +r e +o n +Ġt he +e r +Ġ s +a t +Ġ w +Ġ o +e n +Ġ c +i t +i s +a n +o r +e s +Ġ b +e d +Ġ f +in g +Ġ p +o u +Ġa n +a l +a r +Ġt o +Ġ m +Ġo f +Ġ in +Ġ d +Ġ h +Ġan d +i c +a s +l e +Ġt h +i on +o m +l l +en t +Ġ n +Ġ l +s t +Ġ re +v e +Ġ e +r o +l y +Ġb e +Ġ g +Ġ T +c t +Ġ S +i d +o t +Ġ I +u t +e t +Ġ A +Ġ is +Ġ on +i m +a m +o w +a y +a d +s e +Ġth at +Ġ C +i g +Ġf or +a c +Ġ y +v er +u r +Ġ u +l d +Ġs t +Ġ M +' s +Ġ he +Ġ it +at ion +it h +i r +c e +Ġy ou +i l +Ġ B +Ġw h +o l +Ġ P +Ġw ith +Ġ 1 +t er +c h +Ġa s +Ġw e +Ġ ( +n d +i ll +Ġ D +i f +Ġ 2 +a g +er s +k e +Ġ " +Ġ H +e m +Ġc on +Ġ W +Ġ R +he r +Ġw as +Ġ r +o d +Ġ F +u l +at e +Ġa t +r i +p p +o re +ĠT he +Ġs e +u s +Ġp ro +Ġh a +u m +Ġa re +Ġd e +a in +an d +Ġo r +ig h +es t +is t +a b +r om +Ġ N +t h +Ġc om +Ġ G +u n +o p +0 0 +Ġ L +Ġn ot +es s +Ġe x +Ġ v +re s +Ġ E +e w +it y +an t +Ġb y +e l +o s +or t +o c +q u +Ġf rom +Ġha ve +Ġs u +i ve +ou ld +Ġs h +Ġth is +n t +r a +p e +igh t +ar t +m ent +Ġa l +u st +en d +- - +al l +Ġ O +ac k +Ġc h +Ġ le +i es +re d +ar d +â Ģ +ou t +Ġ J +Ġa b +e ar +i v +al ly +ou r +o st +g h +p t +Ġp l +as t +Ġc an +a k +om e +u d +T he +Ġh is +Ġd o +Ġg o +Ġh as +g e +' t +Ġ U +r ou +Ġs a +Ġ j +Ġb ut +Ġw or +Ġa ll +e ct +Ġ k +am e +Ġw ill +o k +Ġw he +Ġthe y +id e +0 1 +f f +ic h +p l +t her +Ġt r +. . +Ġin t +i e +u re +ag e +Ġn e +i al +a p +in e +ic e +Ġm e +Ġo ut +an s +on e +on g +ion s +Ġwh o +Ġ K +Ġu p +Ġthe ir +Ġa d +Ġ 3 +Ġu s +at ed +ou s +Ġm ore +u e +o g +ĠS t +in d +i ke +Ġs o +im e +p er +. " +b er +i z +a ct +Ġon e +Ġsa id +Ġ - +a re +Ġyou r +c c +ĠT h +Ġc l +e p +a ke +ab le +i p +Ġcon t +Ġwh ich +i a +Ġ im +Ġab out +Ġwe re +ver y +u b +Ġh ad +Ġ en +Ġcom p +, " +ĠI n +Ġu n +Ġa g +i re +ac e +a u +ar y +Ġw ould +as s +r y +Ġ âĢ +c l +o ok +e re +s o +Ġ V +ig n +i b +Ġof f +Ġt e +v en +Ġ Y +i le +o se +it e +or m +Ġ2 01 +Ġre s +Ġm an +Ġp er +Ġo ther +or d +ul t +Ġbe en +Ġl ike +as e +an ce +k s +ay s +ow n +en ce +Ġd is +ct ion +Ġan y +Ġa pp +Ġs p +in t +res s +ation s +a il +Ġ 4 +ic al +Ġthe m +Ġhe r +ou nt +ĠC h +Ġa r +Ġ if +Ġthe re +Ġp e +Ġy ear +a v +Ġm y +Ġs ome +Ġwhe n +ou gh +ac h +Ġth an +r u +on d +ic k +Ġo ver +ve l +Ġ qu +Ċ Ċ +Ġs c +re at +re e +ĠI t +ou nd +p ort +Ġal so +Ġp art +f ter +Ġk n +Ġbe c +Ġt ime +en s +Ġ 5 +op le +Ġwh at +Ġn o +d u +m er +an g +Ġn ew +-- -- +Ġg et +or y +it ion +ing s +Ġj ust +Ġint o +Ġ 0 +ent s +o ve +t e +Ġpe ople +Ġp re +Ġit s +Ġre c +Ġt w +i an +ir st +ar k +or s +Ġwor k +ad e +o b +Ġs he +Ġo ur +w n +in k +l ic +Ġ1 9 +ĠH e +is h +nd er +au se +Ġh im +on s +Ġ [ +Ġ ro +f orm +i ld +at es +ver s +Ġon ly +o ll +Ġs pe +c k +e ll +am p +Ġa cc +Ġb l +i ous +ur n +f t +o od +Ġh ow +he d +Ġ ' +Ġa fter +a w +Ġat t +o v +n e +Ġpl ay +er v +ic t +Ġc ould +it t +Ġa m +Ġf irst +Ġ 6 +Ġa ct +Ġ $ +e c +h ing +u al +u ll +Ġcom m +o y +o ld +c es +at er +Ġf e +Ġbe t +w e +if f +Ġtw o +oc k +Ġb ack +) . +id ent +Ġu nder +rou gh +se l +x t +Ġm ay +rou nd +Ġp o +p h +is s +Ġd es +Ġm ost +Ġd id +Ġad d +j ect +Ġin c +f ore +Ġp ol +on t +Ġag ain +cl ud +ter n +Ġkn ow +Ġne ed +Ġcon s +Ġc o +Ġ . +Ġw ant +Ġse e +Ġ 7 +n ing +i ew +ĠTh is +c ed +Ġe ven +Ġin d +t y +ĠW e +at h +Ġthe se +Ġp r +Ġu se +Ġbec ause +Ġf l +n g +Ġn ow +ĠâĢ ĵ +c om +is e +Ġm ake +Ġthe n +ow er +Ġe very +ĠU n +Ġse c +os s +u ch +Ġe m +Ġ = +ĠR e +i ed +r it +Ġin v +le ct +Ġsu pp +at ing +Ġl ook +m an +pe ct +Ġ 8 +ro w +Ġb u +Ġwhe re +if ic +Ġyear s +i ly +Ġd iff +Ġsh ould +Ġre m +T h +I n +Ġe v +d ay +' re +ri b +Ġre l +s s +Ġde f +Ġr ight +Ġs y +) , +l es +00 0 +he n +Ġth rough +ĠT r +_ _ +Ġw ay +Ġd on +Ġ , +Ġ1 0 +as ed +Ġas s +ub lic +Ġre g +ĠA nd +i x +Ġ very +Ġin clud +ot her +Ġim p +ot h +Ġsu b +ĠâĢ Ķ +Ġbe ing +ar g +ĠW h += = +ib le +Ġdo es +an ge +r am +Ġ 9 +er t +p s +it ed +ation al +Ġb r +Ġd own +Ġman y +ak ing +Ġc all +ur ing +it ies +Ġp h +ic s +al s +Ġde c +at ive +en er +Ġbe fore +il ity +Ġwe ll +Ġm uch +ers on +Ġth ose +Ġsu ch +Ġ ke +Ġ end +ĠB ut +as on +t ing +Ġl ong +e f +Ġth ink +y s +Ġbe l +Ġs m +it s +a x +Ġo wn +Ġpro v +Ġs et +if e +ment s +b le +w ard +Ġsh ow +Ġp res +m s +om et +Ġo b +Ġs ay +ĠS h +t s +f ul +Ġe ff +Ġg u +Ġin st +u nd +re n +c ess +Ġ ent +ĠY ou +Ġgo od +Ġst art +in ce +Ġm ade +t t +st em +ol og +u p +Ġ | +um p +Ġhe l +ver n +ul ar +u ally +Ġa c +Ġm on +Ġl ast +Ġ2 00 +1 0 +Ġst ud +u res +ĠA r +sel f +ar s +mer ic +u es +c y +Ġm in +oll ow +Ġc ol +i o +Ġm od +Ġc ount +ĠC om +he s +Ġf in +a ir +i er +âĢ Ķ +re ad +an k +at ch +e ver +Ġst r +Ġpo int +or k +ĠN ew +Ġs ur +o ol +al k +em ent +Ġus ed +ra ct +we en +Ġs ame +ou n +ĠA l +c i +Ġdiff ere +Ġwh ile +---- ---- +Ġg ame +ce pt +Ġs im +.. . +Ġin ter +e k +Ġre port +Ġpro du +Ġst ill +l ed +a h +Ġhe re +Ġwor ld +Ġth ough +Ġn um +ar ch +im es +al e +ĠS e +ĠI f +/ / +ĠL e +Ġre t +Ġre f +Ġtr ans +n er +ut ion +ter s +Ġt ake +ĠC l +Ġcon f +w ay +a ve +Ġgo ing +Ġs l +u g +ĠA meric +Ġspe c +Ġh and +Ġbet ween +ist s +ĠD e +o ot +I t +Ġe ar +Ġagain st +Ġh igh +g an +a z +at her +Ġex p +Ġo p +Ġin s +Ġg r +Ġhel p +Ġre qu +et s +in s +ĠP ro +is m +Ġf ound +l and +at a +us s +am es +Ġp erson +Ġg reat +p r +Ġs ign +ĠA n +' ve +Ġs omet +Ġs er +h ip +Ġr un +Ġ : +Ġt er +ire ct +Ġf ollow +Ġd et +ic es +Ġf ind +1 2 +Ġm em +Ġc r +e red +e x +Ġex t +ut h +en se +c o +Ġte am +v ing +ou se +as h +at t +v ed +Ġsy stem +ĠA s +d er +iv es +m in +Ġle ad +ĠB l +c ent +Ġa round +Ġgo vern +Ġc ur +vel op +an y +Ġc our +al th +ag es +iz e +Ġc ar +od e +Ġl aw +Ġre ad +' m +c on +Ġre al +Ġsupp ort +Ġ1 2 +.. .. +Ġre ally +n ess +Ġf act +Ġd ay +Ġb oth +y ing +Ġs erv +ĠF or +Ġth ree +Ġw om +Ġm ed +od y +ĠThe y +5 0 +Ġex per +t on +Ġe ach +ak es +Ġc he +Ġc re +in es +Ġre p +1 9 +g g +ill ion +Ġg rou +ut e +i k +W e +g et +E R +Ġm et +Ġs ays +o x +Ġd uring +er n +iz ed +a red +Ġf am +ic ally +Ġha pp +ĠI s +Ġch ar +m ed +v ent +Ġg ener +i ent +p le +i et +re nt +1 1 +v es +pt ion +Ġ2 0 +form ation +Ġc or +Ġoff ic +ie ld +Ġto o +is ion +Ġin f +Ġ Z +t he +o ad +Ġp ublic +Ġpro g +r ic +* * +Ġw ar +Ġp ower +v iew +Ġf ew +Ġl oc +Ġdiffere nt +Ġst ate +Ġhe ad +' ll +Ġp oss +Ġst at +re t +ant s +Ġv al +Ġis s +Ġc le +i vers +an c +Ġex pl +Ġan other +Ġ Q +Ġa v +th ing +n ce +W h +Ġch ild +Ġs ince +i red +l ess +Ġl ife +Ġde velop +itt le +Ġde p +Ġp ass +ã ĥ +Ġt urn +or n +Th is +b ers +ro ss +ĠA d +Ġf r +Ġres p +Ġsec ond +o h +Ġ / +Ġdis c +Ġ & +Ġsomet hing +Ġcomp le +Ġ ed +Ġf il +Ġmon th +a j +u c +Ġgovern ment +Ġwith out +Ġle g +Ġd ist +Ġp ut +Ġqu est +an n +Ġpro t +2 0 +Ġne ver +i ence +Ġle vel +Ġar t +Ġth ings +Ġm ight +Ġeff ect +Ġcont ro +Ġc ent +Ġ1 8 +Ġall ow +Ġbel ie +ch ool +ot t +Ġinc re +Ġfe el +Ġres ult +Ġl ot +Ġf un +ot e +Ġt y +ere st +Ġcont in +Ġus ing +Ġb ig +2 01 +Ġas k +Ġb est +Ġ ) +I N +Ġo pp +3 0 +Ġnum ber +in ess +S t +le ase +Ġc a +Ġm ust +Ġd irect +Ġg l +Ġ < +Ġop en +Ġp ost +Ġcom e +Ġse em +ord ing +Ġwe ek +ate ly +it al +Ġe l +ri end +Ġf ar +Ġt ra +in al +Ġp ri +ĠU S +Ġpl ace +Ġfor m +Ġto ld +" : +ain s +at ure +ĠTr ump +Ġst and +Ġ # +id er +ĠF r +Ġne xt +Ġs oc +Ġp ur +Ġle t +Ġl ittle +Ġh um +Ġ i +r on +1 5 +Ġ1 5 +Ġcomm un +Ġm ark +ĠThe re +Ġw r +ĠTh at +Ġin formation +w ays +Ġb us +a pp +Ġinv est +m e +Ġh ard +ain ed +e ad +Ġim port +Ġapp ro +Ġt est +Ġt ri +Ġre st +os ed +Ġf ull +Ġc are +ĠS p +Ġc ase +O N +Ġs k +Ġl ess +Ġ + +Ġpart ic +ĠP l +ab ly +u ck +is hed +ch n +b e +Ġl ist +at or +Ġto p +Ġad v +ĠB e +ru ct +Ġd em +r ation +l ing +g y +re en +g er +Ġh ome +Ġle ft +Ġbet ter +Ġd ata +Ġ1 1 +Ġatt ack +Ġpro ble +l ine +ard s +Ġbe h +r al +ĠH ow +ĠS he +ar ge +Ġ -- +: // +Ġb ro +ĠP h +at s +Ġbu ild +w w +id ed +a im +as es +en cy +Ġm ain +in ed +Ġinclud ing +Ġ { +Ġg ot +Ġint erest +Ġke ep +Ġ X +Ġe as +ain ing +Ġcl ass +âĢ ¦ +ĠN o +Ġv ar +Ġsm all +amp le +A T +Ġ ide +ĠS o +Ġre ce +Ġpol it +Ġm ov +Ġpl an +Ġper cent +iv ing +Ġc amp +Ġp ay +1 4 +s c +is ed +Ġu nt +one y +pl oy +== == +Ġdid n +ĠI nd +el s +ert ain +Ġp os +__ __ +i ver +Ġpro cess +Ġprog ram +if ied +ĠR ep +1 6 +u ro +olog y +at ter +in a +Ġn ame +ĠA ll +Ġf our +Ġret urn +v ious +b s +Ġcall ed +Ġm ove +ĠS c +ir d +Ġgrou p +Ġb re +Ġm en +Ġc ap +t en +e e +Ġd ri +le g +he re +uth or +Ġp at +Ġcur rent +id es +Ġp op +t o +ent ion +Ġal ways +Ġm il +Ġwom en +Ġ1 6 +Ġo ld +iv en +ra ph +ĠO r +r or +ent ly +Ġn ear +ĠE x +re am +s h +Ġ1 4 +Ġf ree +iss ion +st and +ĠC on +al ity +us ed +1 3 +Ġdes ign +Ġch ange +Ġch ang +Ġb o +Ġv is +em ber +Ġb ook +read y +Ġk ill +2 5 +pp ed +Ġa way +Ġab le +Ġcount ry +Ġcon st +ar n +Ġor der +A R +i or +i um +or th +1 8 +ail able +Ġs w +Ġm illion +Ġ1 3 +at ic +t ed +ĠG o +Ġo per +en g +Ġth ing +aj or +con om +ĠCom m +Ġwh y +u red +ur al +Ġs chool +b y +ĠM ar +Ġa ff +Ġd ays +Ġan n +us h +an e +I f +e g +Ġpro f +Ġhe alth +ou th +B ut +ion al +. , +Ġs ol +Ġal ready +Ġ3 0 +Ġchar act +H e +Ġf riend +E S +i ans +ic le +' d +ĠO n +Ġle ast +Ġp rom +Ġd r +Ġh ist +it her +Ġ est +i qu +1 7 +s on +Ġte ll +Ġt alk +oh n +o int +le ction +A N +Ġunt il +au gh +Ġl ater +Ġ ve +Ġv iew +end ing +iv ed +Ġwor d +w are +Ġc ost +Ġen ough +Ġg ive +ĠUn ited +Ġte chn +are nt +O R +Ġp ar +ĠD r +Ġ201 6 +r ist +er ing +Ġ  +Ġl arge +s ide +ac y +cc ess +Ġw in +Ġimport ant +Ġ19 9 +Ġdoes n +Ġ1 7 +Ġbus iness +Ġcle ar +Ġre se +" , +ur y +Ġe qu +as ter +al f +ĠAmeric an +n ect +Ġex pect +ivers ity +Ġo cc +ĠF l +Ġk ind +Ġme an +Ġp ast +Ġde v +Ġb as +le t +ra ft +Ġor gan +Ġde l +Ġper form +Ġst ory +Ġse ason +ĠC ol +Ġcl aim +Ġc ame +Ġwith in +Ġl ine +Ġpro ject +ĠA t +Ġcontro l +end ed +ĠS y +Ġa ir +iz ation +Ġ * +le y +Ġm oney +id d +Y ou +f or +Ġfam ily +Ġm aking +Ġb it +Ġpol ice +Ġhapp en +Ġ vers +on y +u ff +ĠW hen +Ġs it +ide o +l f +is on +Ġsu re +g in +Ġapp ear +Ġl ight +Ġ es +o f +Ġw ater +Ġt imes +n ot +Ġg row +Ġcomp any +ĠT e +ow s +Ġm ar +our ce +i ol +ar m +b r +Ġex ample +Ġcon c +Ġf ore +ĠT o +p ro +E N +ri es +Ġ2 5 +ĠC an +ne y +Ġact ually +Ġe ver +ur ity +ak en +ap s +Ġt ax +Ġm ajor +am a +Ġof ten +er al +Ġhum an +Ġj ob +is ter +Ġav ailable +oc r +en n +a id +iv id +Ġrec ord +? " +Ġs ing +ĠA m +id ence +Ġnew s +st er +Ġe conom +Ġfollow ing +ĠB r +is ing +Ġh our +m ost +um ent +Ġse x +Ġdes c +Ġbec ome +ĠE d +Ġto ok +Ġha ving +Ġprodu ct +a ult +A s +ar ing +Ġme ans +Ġh op +un e +Ġch o +Ġc ertain +Ġn on +Ġde al +2 4 +le ment +oc i +en e +Ġs ide +ĠP r +ĠM ay +Ġre ason +u ed +c hed +ul ation +Ġe lect +Ġoffic ial +Ġposs ible +Ġh old +and s +ot s +Ġc ity +or ies +Ġse ver +Ġchild ren +Ġon ce +Ġact iv +l er +Ġn ight +it ions +ĠJ ohn +a pe +pl ay +Ġd one +Ġl im +Ġwork ing +ĠP res +or ld +e b +ĠC o +Ġb ody +ail s +ut es +ĠM r +Ġwhe ther +Ġa uthor +ro p +Ġpro per +Ġse en +) ; +Ġf ac +ĠS u +Ġcon d +it ing +Ġcour se +Ġ } +-------- -------- +a ign +Ġev ent +Ġen g +Ġp ot +Ġin tern +i am +Ġsh ort +em pt +ã Ĥ +ĠG od +il ar +8 0 +Ġor ig +I S +our n +ab ility +it ive +Ġd am +Ġ1 00 +Ġp ress +Ġdo ing +Ġprot ect +r ing +Ġthough t +Ġquest ion +re w +ĠW ar +Ġsever al +ĠSt ate +Ġg iven +Ġf und +ĠT w +Ġw ent +an ces +w ork +p or +m y +4 0 +Ġar g +art ment +ust om +Ġpol ic +Ġme et +Ġc reat +2 2 +ĠSt ates +Ġg ames +ra w +ut ure +Ġunder stand +ur s +ĠO b +l ish +s y +Ġm akes +Ġw on +ag on +Ġh tt +Ġl ove +ent ial +Ġcomple te +p ar +ĠI m +A L +Ġacc ount + ł +ore d +ver t +Ġ ident +Ġ201 5 +Ġother s +ĠM in +i ber +ver age +The re +ition al +d d +Ġpro b +Ġyou ng +Ġal ong +Ġacc ording +Ġy et +Ġmem bers +ĠWh at +o id +ĠM an +A nd +Ġam ong +a i +Ġem ploy +ĠR es +Ġ > +Ġinv ol +Ġl ow +a f +ĠC ar +Ġh ig +ĠO ne +ĠS ec +in ation +Ġlike ly +Ġan t +ag ed +ĠR uss +Ġb en +Ġre le +F or +b ack +ĠN ot +Ġpres ident +b all +Ġacc ess +ivid ual +ĠD em +ĠE uro +6 0 +Ġkn own +ir l +ĠG r +Ġear ly +u se +iet y +âĢ ĵ +Ġf ight +Ġs ent +Ġto day +Ġmark et +" . +Ġb ased +Ġstr ong +ur ther +Ġde b +m ber +Ġproble m +Ġde ath +Ġsoc ial +im ate +A S +ort un +Ġcamp aign +er y +C h +Ġe y +i ally +Ġm us +w h +p os +Ġ er +Ġsa f +Ġmonth s +ir on +Ġv iol +Ġf ive +Ġst re +Ġplay ers +in c +al d +y ear +a un +Ġsu ccess +Ġpres ent +ere nce +Ġ201 4 +Ġsu gg +Ġpartic ular +Ġtr y +Ġsugg est +ĠCh rist +on es +Ġpri v +2 3 +Ġc rit +Ġl and +Ġloc al +if y +2 9 +Ġa ut +E D +ĠG u +Ġm ult +Ġpolit ical +Ġask ed +Ġfor mer +it ter +ri pt +Ġcl ose +Ġp ract +ĠY ork +Ġget ting +Ġac ross +Ġcom b +Ġbelie ve +Ġ z +Ġto get +Ġtoget her +ĠC ent +ir c +Ġind ividual +ĠM c +2 7 +is k +ĠE ng +Ġf ace +Ġ2 4 +Ġval ue +Ġare a +e v +Ġw rit +ĠPres ident +Ġv ot +Ġke y +Ġm om +p ut +Ġany thing +Ġexper ience +att le +Ġm ind +a ff +om m +Ġf uture +g ed +Ġc ut +Ġto t +it ch +Ġv ideo +Ġinvest ig +Ġn et +ĠM y +r ict +i en +. ) +Ġimp ro +th ough +ward s +Ġcon nect +ĠM ed +sel ves +ens ive +m b +o ber +at ors +A n +Ġ5 0 +Ġre du +res ent +Ġab ove +Ġf re +ĠEuro pe +s w +Ġam ount +ĠA pp +Ġe ither +Ġmil it +Ġan al +Ġf ail +ĠE n +al es +Ġspec ial +Ġbl ack +I T +c her +Ġlook ing +Ġf ire +y n +Ġal most +o on +Ġstud y +Ġm iss +c hes +ro wn +Ġt re +Ġcommun ity +Ġmed ia +Ġf ood +Ġcom es +ĠUn iversity +Ġsing le +Wh at +u ly +Ġh alf +ag ue +h od +ĠRep ublic +Ġstart ed +Ġqu ick +ot o +b ook +Ġiss ue +it or +Ġel se +Ġcons ider +2 6 +ro du +Ġt aken +2 8 +9 9 +ĠW ith +Ġtr ue +Ġw a +Ġtr ad +Ġag o +Ġm ess +ie f +Ġadd ed +o ke +Ġb ad +Ġf av +3 3 +Ġsim ilar +as k +ĠD on +Ġcharact er +ort s +ĠH ouse +Ġreport ed +Ġty pe +v al +i od +ĠHow ever +Ġt arg +Ġent ire +pp ing +Ġhist ory +Ġl ive +ff ic +.... .... +ed eral +Ġtr ying +Ġdisc uss +ĠH ar +ac es +l ished +Ġse lf +os p +re st +Ġro om +el t +Ġf all +ol ution +Ġe t +Ġ x +Ġis n +Ġide a +b o +Ġs ound +ĠD ep +Ġsome one +ci ally +ull y +Ġf oc +Ġob ject +if t +ap er +Ġplay er +Ġr ather +Ġserv ice +as hing +ĠD o +ĠP art +ru g +m on +p ly +Ġm or +Ġnot hing +Ġprov ide +I C +un g +Ġpart y +Ġex ist +Ġm ag +7 0 +Ġr ul +Ġh ouse +Ġbeh ind +Ġhow ever +ĠW orld +Ġs um +Ġapp lic +Ġ ; +Ġfun ction +g r +ĠP ol +Ġfr ont +2 00 +Ġser ies +Ġt em +Ġty p +ill s +Ġo pt +Ġpoint s +Ġbel ow +itt ed +Ġspec ific +Ġ201 7 +um b +Ġr a +Ġpre vious +Ġpre t +re me +Ġc ustom +Ġcour t +ĠM e +Ġre pl +Ġwho le +g o +c er +Ġt reat +ĠA ct +Ġprob ably +Ġle arn +end er +ĠA ss +Ġvers ion +n ow +Ġche ck +ĠC al +R E +min ist +O n +our ces +Ġben ef +Ġd oc +Ġdet er +Ġen c +Ġsu per +Ġadd ress +Ġv ict +Ġ201 3 +Ġme as +t r +Ġf ield +W hen +Ġsign ific +u ge +Ġfe at +Ġcomm on +l oad +Ġbe gin +Ġbr ing +Ġa ction +er man +Ġdesc rib +Ġind ust +Ġwant ed +ri ed +m ing +Ġatt empt +4 5 +f er +Ġd ue +ress ion +# # +Ġsh all +Ġs ix +o o +Ġst ep +Ġp ub +Ġhim self +Ġ2 3 +Ġc op +Ġd est +Ġst op +A C +ib ility +Ġl ab +ic ult +Ġhour s +Ġcre ate +Ġf urther +ĠAmeric a +ĠC ity +Ġd ou +he ad +S T +ĠN orth +c ing +Ġn ational +u le +ĠIn st +Ġt aking +ĠQ u +ir t +Ġre d +Ġrese arch +v iron +ĠG e +Ġbre ak +an a +Ġsp ace +ater ial +Ġrec ent +ĠA b +Ġgener al +Ġh it +Ġper iod +Ġevery thing +ive ly +Ġph ys +Ġsay ing +an ks +Ġc ou +Ġc ult +ac ed +e al +u ation +Ġc oun +l u +Ġinclud e +Ġpos ition +ĠA fter +ĠCan ad +ĠE m +Ġim m +ĠR ed +Ġp ick +Ġcom pl +Ġm atter +re g +e xt +ang u +is c +o le +a ut +Ġcomp et +e ed +f ect +Ġ2 1 +ĠS en +ĠThe se +as ing +Ġcan not +Ġin it +Ġrel ations +ac hed +Ġb ar +Ġ4 0 +ĠT H +Ġ201 2 +Ġv ol +Ġg round +Ġsec urity +Ġup d +il t +3 5 +Ġconc ern +ĠJ ust +Ġwh ite +Ġseem s +ĠH er +pe cially +i ents +Ġann oun +Ġf ig +ight s +Ġst ri +l ike +id s +Ġs us +Ġw atch +Ġ â +Ġw ind +ĠC ont +Ġit self +Ġm ass +A l +y le +iqu e +ĠN ational +Ġab s +Ġp ack +Ġout side +Ġan im +Ġp ain +et er +Ġman ag +du ct +og n +Ġ ] +ĠSe pt +se c +o ff +ĠJ an +Ġf oot +ad es +Ġth ird +Ġm ot +Ġev idence +int on +Ġth reat +a pt +pl es +c le +Ġl o +Ġde cl +Ġit em +med i +Ġrep resent +om b +am er +Ġsignific ant +og raph +s u +Ġc al +i res +00 00 +I D +A M +Ġsim ply +Ġlong er +Ġf ile +O T +c he +S o +ate g +or g +ĠH is +Ġen er +Ġd om +Ġup on +il i +": " +Ġthem selves +Ġcom ing +Ġqu ite +Ġdiff icult +ĠB ar +il ities +re l +end s +c ial +6 4 +Ġwom an +ra p +y r +Ġne cess +ip s +Ġte xt +Ġrequ ire +Ġmilit ary +Ġre view +Ġresp ons +7 5 +Ġsub ject +Ġinst ead +Ġiss ues +Ġg en +" ," +Ġmin utes +Ġwe ap +r ay +am ed +t ime +b l +H ow +Ġc ode +ĠS m +Ġhig her +ĠSt e +r is +Ġp age +Ġstud ents +ĠIn tern +Ġmet hod +ĠA ug +ĠP er +ĠA g +Ġpolic y +ĠS w +Ġex ec +Ġac cept +um e +rib ut +Ġword s +Ġfin al +Ġchang es +ĠDem ocr +Ġfriend s +Ġres pect +Ġe p +Ġcomp an +iv il +Ġdam age +** ** +og le +viron ment +Ġne g +ent al +Ġa p +Ġtot al +iv al +! " +l im +Ġneed s +Ġag re +Ġdevelop ment +Ġa ge +ip le +2 1 +Ġresult s +ĠA f +S h +Ġg un +ĠOb ama +ro ll +Ġ @ +Ġright s +ĠB rit +Ġrun ning +Ġwas n +Ġp ort +Ġr ate +Ġpret ty +Ġtarg et +Ġsa w +Ġc irc +Ġwor ks +ic ro +al t +o ver +ww w +Th at +l ier +Ġevery one +ud e +Ġp ie +idd le +ra el +Ġr ad +Ġbl ock +Ġw alk +T o +ã ģ +n es +ĠA ust +a ul +ro te +ĠS outh +ess ion +op h +Ġshow s +Ġs ite +Ġj o +Ġr isk +cl us +l t +Ġin j +id ing +ĠS pe +Ġch all +ir m +Ġ2 2 +itt ing +st r +Ġh y +L E +ke y +Ġbe gan +at ur +ashing ton +l am +ĠD av +b it +Ġs ize +ĠP ar +3 8 +ourn al +f ace +Ġdec ision +Ġl arg +Ġj ud +re ct +Ġcontin ue +ĠO ct +ove red +ĠI nt +==== ==== +Ġp arent +ĠW ill +Ġeas y +Ġd rug +ang er +Ġs ense +Ġd i +id ay +Ġener gy +ist ic +Ġass oci +ar ter +ob al +e ks +ĠE l +ur ch +Ġg irl +o e +it le +Ġ2 8 +ĠC he +Ġrequ est +Ġso on +Ġh ost +k y +Ġst ates +om es +Ġm aterial +le x +Ġmom ent +Ġan sw +on se +Ġes pecially +Ġn orm +Ġserv ices +p ite +r an +Ġro le +4 4 +) : +Ġc red +C l +____ ____ +Ġm at +Ġl og +ĠCl inton +O U +Ġoff ice +Ġ2 6 +Ġch arg +Ġtr ack +m a +Ġhe art +Ġb all +Ġperson al +Ġbuild ing +n a +s et +b ody +ĠBl ack +Ġincre ase +itt en +Ġneed ed +3 6 +3 2 += " +Ġl ost +Ġbec ame +Ġgrou ps +ĠM us +Ġw rote +ĠP e +Ġpro p +j oy +à © +ĠWh ite +Ġde ad +. ' +Ġhtt p +Ġwe bs +O S +Ġins ide +Ġwr ong +Ġstat ement +Ġ ... +y l +Ġfil m +Ġmus ic +Ġsh are +ific ation +Ġre lease +Ġfor ward +Ġst ay +Ġcomp ut +it te +s er +Ġorig inal +Ġc ard +Ġc and +Ġd iv +at ural +Ġfav or +O M +Ġc ases +us es +Ġse ction +Ġle ave +g ing +ov ed +ĠW ashington +3 9 +ĠG l +Ġrequ ired +act ion +ap an +o or +it er +ĠK ing +Ġcount ries +ĠG erman +ll ing +Ġ2 7 +3 4 +Ġquest ions +Ġpr im +Ġc ell +Ġsh oot +Ġany one +ĠW est +Ġaff ect +ep end +Ġon line +ĠIs rael +ĠSept ember +Ġab ility +Ġcont ent +is es +Ġre ve +Ġl aun +Ġind ic +Ġfor ce +c ast +Ġso ld +av ing +f l +Ġso ft +Ġcompan ies +ce ed +Ġart icle +Ġa ud +Ġre v +Ġed uc +Ġplay ing +0 5 +Ġhe ld +ct or +Ġrele ased +Ġf ederal +3 7 +Ġad minist +Ġinter view +Ġinst all +Ġrece ived +Ġs ource +u k +P h +Ġser ious +Ġcre ated +Ġc ause +Ġim medi +Ġdef in +u el +ĠDep artment +ct ions +ĠC our +ĠN ow +z e +it es +it ution +Ġl ate +Ġspe ak +n ers +Ġleg al +ar i +ĠC or +Ġwe eks +Ġmod el +Ġp red +Ġex act +B C +ĠB y +IN G +os ing +Ġt akes +Ġreg ard +Ġopp ortun +Ġpr ice +Ġ19 8 +ĠA pr +f ully +Ġor d +Ġproble ms +ru ction +h am +ĠC ount +le ge +Ġlead ers +E T +le v +Ġde ep +olog ical +es e +h aps +ĠS ome +Ġp ers +Ġcont ract +Ġrelations hip +s p +ou d +Ġb ase +4 8 +m it +A d +anc ial +Ġcons um +Ġpot ential +Ġl angu +re m +et h +Ġrel ig +ress ed +6 6 +Ġl ink +Ġl ower +ay er +ĠJ une +Ġf em +un t +er c +ur d +Ġcont act +Ġ ill +Ġm other +Ġest ab +h tt +ĠM arch +ĠB ro +ĠCh ina +Ġ2 9 +Ġs qu +Ġprov ided +Ġa verage +as ons +Ġ201 1 +Ġex am +l in +5 5 +n ed +Ġper fect +Ġt ou +al se +u x +Ġbu y +Ġsh ot +Ġcol lect +Ġph ot +Ġplay ed +Ġsur pr +Ġofficial s +Ġsim ple +av y +Ġindust ry +Ġhand s +g round +Ġp ull +Ġr ound +Ġus er +Ġr ange +u ary +Ġpriv ate +op s +e es +Ġw ays +ĠM ich +Ġve h +Ġex cept +Ġter ms +im um +pp er +I ON +ore s +ĠDr agon +ou l +Ġd en +Ġperform ance +Ġb ill +c il +4 7 +Ġen vironment +Ġex c +ad d +Ġwor th +Ġp ict +Ġch ance +Ġ201 8 +b or +Ġspe ed +ict ion +Ġal leg +ĠJ apan +at ory +re et +Ġm atch +ĠI I +Ġst ru +ord er +Ġst e +Ġl iving +Ġst ruct +in o +Ġse par +her n +Ġresp onse +Ġen joy +Ġv ia +A D +um ents +ace book +Ġmem ber +ib r +iz ing +Ġto ol +ĠM on +ĠWh ile +h ood +ĠA ng +ĠD ef +Ġoff er +T r +a ur +Ġturn ed +ĠJ uly +d own +an ced +Ġrec ently +ĠE ar +Ġc e +ĠSt ar +ĠC ong +rough t +Ġbl ood +Ġhop e +Ġcom ment +ain t +Ġar ri +il es +Ġpartic ip +ough t +ri ption +0 8 +4 9 +Ġg ave +Ġse lect +Ġkill ed +sy ch +Ġgo es +i j +Ġc oll +Ġimp act +at ives +ĠS er +0 9 +ĠAug ust +Ġb oy +d e +ĠD es +Ġf elt +U S +Ġexpect ed +Ġim age +ĠM ark +cc ording +o ice +E C +ĠM ag +en ed +h old +ĠP ost +Ġpre vent +N o +Ġinvol ved +Ġey es +Ġquick ly +A t +un k +Ġbeh av +Ġ ur +Ġl ed +c ome +e y +Ġcand id +Ġear lier +Ġfoc us +et y +P ro +led ge +ix ed +ill ed +Ġpop ular +A P +Ġset t +l ight +Ġvar ious +in ks +Ġlevel s +Ġro ad +ell ig +ab les +he l +itte e +ĠG ener +y pe +Ġhe ard +ic les +Ġm is +Ġus ers +ĠS an +Ġimpro ve +Ġf ather +Ġse arch +The y +v il +Ġprof ess +Ġkn ew +Ġl oss +Ġev ents +6 5 +Ġb illion +0 7 +0 2 +ĠNew s +ĠA M +Ġco ver +w here +ens ion +Ġb ott +Ġare as +en ces +op e +ĠTw itter +a el +Ġget s +ĠGo ogle +Ġs n +i ant +Ġv ote +Ġnear ly +Ġinclud ed +Ġrec ogn +z z +m m +al ed +Ġhappen ed +0 4 +Ġh ot +Ġwho se +Ġc ivil +Ġsu ff +o es +it iz +ĠSy ri +Ġresp ond +Ġh on +Ġfeat ures +Ġeconom ic +ĠApr il +r im +Ġtechn ology +Ġo ption +ag ing +Ġpur ch +R e +Ġl at +ch ie +is l +Ġrec omm +u f +Ġtr aining +Ġeffect s +Ġf ast +Ġ201 0 +Ġocc ur +Ġwebs ite +Ġem ail +Ġs ens +e ch +Ġo il +Ġinf lu +Ġcurrent ly +ĠS ch +ĠAd d +Ġgo al +Ġsc ient +Ġcon v +1 00 +em y +Ġdec ided +Ġtra vel +Ġm ention +L L +0 3 +Ġe lection +Ġph one +Ġlook s +Ġsit uation +Ġc y +Ġh or +b ed +ĠCour t +a ily +av es +Ġqu ality +ĠCom p +w ise +Ġt able +Ġst aff +ĠW ind +et t +Ġtri ed +ide red +Ġadd ition +Ġb ox +Ġl ack +ar ily +Ġw ide +Ġm id +Ġbo ard +ys is +Ġant i +h a +Ġd ig +en ing +Ġd ro +C on +6 8 +Ġsl ow +b ased +se qu +Ġp ath +E x +ak er +Ġwork ed +Ġp en +Ġeng ine +Ġlook ed +ĠSu per +ĠS erv +Ġvict im +U n +Ġproper ty +Ġint rodu +Ġexec ut +ĠP M +L e +Ġcol or +ĠM ore +Ġ6 0 +Ġnet work +Ġd ate +c ul +id ge +Ġext ra +3 1 +Ġs le +6 7 +Ġw ond +Ġreport s +j ust +ĠAust ral +Ġcap ital +Ġen s +Ġcomm and +Ġallow ed +Ġpre p +Ġca pt +h ib +Ġnum bers +ch an +Ġf air +m p +om s +Ġre ach +W ith +t ain +Ġbro ad +Ġcou ple +ec ause +ly ing +ĠF eb +Ġsc reen +Ġl ives +Ġpri or +ĠCong ress +A r +Ġappro ach +Ġe mer +ar ies +ĠD is +s erv +ĠN e +Ġbu ilt +c ies +Ġre pe +Ġrul es +for ce +ĠP al +Ġfin ancial +Ġcons idered +ĠCh ar +n ces +ĠI S +Ġb rought +Ġb i +i ers +ĠS im +O P +Ġproduct s +Ġvis it +Ġdoc ument +Ġcon duct +Ġcomplete ly +in ing +ĠCal if +ib ly +Ġwr itten +ĠT V +em ents +Ġd raw +O ne +Ġpub lished +Ġsec ret +r ain +he t +ĠF acebook +ond ay +ĠU p +Ġsex ual +Ġth ous +ĠP at +Ġ ess +Ġstand ard +Ġar m +g es +ect ion +Ġf ell +Ġfore ign +an i +ĠFr iday +Ġreg ular +in ary +Ġincre ased +Ġus ually +Ġdem on +Ġd ark +Ġadd itional +ro l +ĠO f +Ġprodu ction +! ! +und red +Ġintern ational +id ents +ĠF ree +rou p +Ġr ace +Ġm ach +Ġh uge +A ll +le ar +ove mber +Ġto wn +Ġatt ention +ĠO ff +y ond +ĠThe n +f ield +Ġter ror +ra z +ĠB o +Ġmeet ing +ĠP ark +Ġar rest +Ġf ear +Ġa w +ĠV al +or ing +' , +Ġext reme +ar r +Ġwork ers +A fter +Ġ3 1 +n et +am ent +Ġdirect ly +Ġpop ulation +ub e +ĠOct ober +ĠI N +ĠJan uary +5 9 +ĠDav id +Ġc ross +ce mber +ĠF irst +Ġmess age +ir it +Ġn ation +Ġp oll +is ions +Ġansw er +n y +is ode +Ġcar ry +ĠRuss ia +Ġhe ar +eng th +ro y +Ġn atural +in ally +Ġdo g +m itted +Ġtr ade +Ġsub st +Ġmult iple +ĠAf ric +Ġf ans +Ġs ort +Ġgl obal +ic ation +ĠW ed +ar a +Ġa chie +Ġlangu age +ve y +Ġt al +Ġnecess ary +Ġdet ails +Ġs en +ĠS und +ĠRe g +ĠR ec +0 6 +Ġs il +ress ive +Ġmed ical +un ch +orn ia +Ġu nd +f ort +oc ks +ĠM onday +ues day +c raft +7 7 +ur t +Ġ ver +ĠH ill +Ġrece ive +Ġmor ning +es tern +Ġb ank +Ġs at +ir th +ĠH igh +Ġdev ice +ĠTH E +ĠCent er +Ġsaf e +Ġp le +ĠCanad a +Ġsystem s +Ġass ist +Ġsur v +Ġb attle +ĠS oc +vert is +S he +Ġp aper +Ġgrow th +Ġc ast +S c +Ġpl ans +ll ed +Ġpart s +Ġw all +Ġmove ment +Ġpract ice +im ately +Ġdis play +Ġsomet imes +om p +ĠP aul +ĠY es +k ing +5 8 +o ly +Ġs on +Ġav oid +ok es +ĠJ ew +Ġto wards +as c +Ġ // +ĠK ore +Ġtalk ing +Ġcor rect +Ġsp ent +ic ks +i able +e ared +Ġter m +Ġwant s +om ing +Ġ ut +Ġdou b +Ġfor ces +Ġp lease +6 9 +ĠN ovember +at form +ond on +Ġon es +Ġimmedi ately +ĠRuss ian +ĠM et +Ġde g +Ġparent s +C H +ĠAmeric ans +al y +ĠM od +Ġsh own +Ġcond itions +Ġst uff +Ġre b +ĠY our +Ġinclud es +n own +ĠS am +Ġexper ien +m ission +ĠE ven +augh t +Ġannoun ced +ĠRepublic an +Ġdeter min +Ġdescrib ed +ĠCount y +( ) +Ġdo or +Ġchang ed +Ġne igh +ĠH ere +Ġcle an +Ġp an +ĠDe cember +ĠEurope an +ir ing +ap ter +Ġcl ub +ĠT uesday +Ġp aid +ĠN et +Ġattack s +Ġcharact ers +Ġal one +Ġdirect or +d om +Ġ3 5 +Ġl oad +Ġr out +ĠCalif ornia +Ġfin ally +Ġr ac +Ġcont r +Ġexact ly +res h +p ri +ĠIs lam +Ġn ature +Ġcare er +Ġlat est +Ġcon vers +ĠS l +p ose +ci ent +ĠIn c +iv ity +8 8 +ĠA tt +ĠM or +nes day +Ġwe ight +k en +Ġnot e +Ġteam s +Ġ \ +air s +ĠG reen +Ġh undred +on ent +Ġstre ng +Ġcons ist +ic ated +Ġreg ul +Ġl ic +ast ic +Ġt en +urs day +ellig ence +ous ly +ĠU K +B I +Ġcost s +Ġind epend +ĠA P +Ġnorm al +Ġh om +Ġob vious +Ġs we +Ġst ar +Ġread y +ac her +Ġimp lement +g est +Ġs ong +ĠG et +ĠL ab +Ġinterest ing +us ing +Ġg iving +ĠSund ay +Ġet c +Ġm iddle +Ġrem ember +r ight +os ition +ut ions +Ġm ax +4 6 +Ġyour self +Ġdem and +Ġtreat ment +Ġd anger +ĠC ons +Ġgu y +ĠBrit ish +Ġphys ical +Ġrel ated +Ġrem ain +Ġcould n +Ġref er +Ġc itiz +b ox +EN T +bo ard +Ġin n +I G +er o +ĠSt reet +osp ital +ren ch +cher s +Ġst ra +O L +ag er +ĠA N +Ġeas ily +I A +en ge +in y +Ġcl os +ock ed +Ġus es +ĠC oun +I m +u ild +? ? +m ore +Ġan g +Ġwr ite +ol ute +5 7 +Ġlead er +Ġread ing +< / +Ġaut om +est s +4 3 +Ġleg isl +ĠG old +Ġdesign ed +ĠS T +ĠLe g +a res +Ġbe aut +ĠT ex +Ġappear s +Ġstru gg +ĠR om +Ġ 00 +Ġcho ice +Ġparticular ly +ĠF rom +op er +ĠL ondon +ann ed +Ġallow s +ob ile +Ġdiffere nce +âĢ ¢ +ĠV iew +ĠWed nesday +Ġal though +Ġrel ative +Ġapplic ation +ate ver +Ġare n +Ġmy self +Ġim ag +Ġdis e +Ġsoc iety +Ġfre qu +ĠEng lish +Ġpo or +ĠD ay +Ġwrit ing +Ġse ven +Ġstart ing +Ġb ud +Ġpr int +ĠTr ans +uf act +ĠSt ud +n ew +Ġcr im +Ġg ives +Ġco ol +a e +i ance +ĠGener al +Ġthink ing +Ġsa ve +Ġlim ited +ĠPart y +Ġmean ing +p en +ow ers +ĠJ ack +E M +Ġn ice +ru pt +Ġg as +Ġe ight +Ġfe et +Ġeff ort +Ġ ign +ic it +B l +co in +Ġop in +Ġbr ain +Wh ile +he st +ĠTh ursday +Ġwould n +augh ter +Ġtou ch +le ments +Ġstud ies +Ġcent er +c ont +or ge +Ġcomput er +Ġinvestig ation +P l +or ks +Ġ200 8 +Ġincre asing +Ġst ore +Ġcom ments +Ġb al +m en +Ġdo ll +Ġl iber +Ġw ife +Ġlaw s +atur day +it ness +Ġmod ern +ĠS k +Ġadminist ration +Ġopportun ity +Ġs al +Ġpower ful +M y +Ġclaim s +ĠEar th +ord s +Ġt itle +Ġes c +n ame +N ot +om en +Ġbe yond +Ġc amer +Ġse ll +it ute +ear ch +Ġapp l +im ent +4 2 +ĠAr t +Ġun f +Ġviol ence +ur g +ĠE ast +Ġcomp ared +Ġopt ions +Ġthrough out +Ġv s +ig r +. [ +ac hes +7 8 +Ġfil es +F L +E L +ar ian +ĠJ ames +ĠA ir +an ch +Ġdet ail +Ġpie ce +P S +Ġn amed +Ġeduc ation +Ġdri ve +Ġitem s +Ġstud ent +ic ed +: : +ic o +Ġth row +Ġsc ene +Ġcomple x +Ġ200 9 +Ġpre c +ĠB re +7 9 +Ġcon cept +Ġstat us +am ing +Ġd ied +Ġknow ledge +Ġbegin ning +O D +ru ary +Ġcertain ly +Ġgu ys +Ġsl ight +in n +ound s +Ġf ine +Ġf at +ic ations +Ġper haps +ĠA nt +Ġinc ome +Ġhtt ps +Ġmajor ity +port s +st on +Ġgreat er +Ġfe ed +ent ially +Ġsaf ety +Ġun ique +and om +Ġg one +Ġshow ed +Ġhist or +Ġcoun ter +i us +id a +Ġlead ing +i pe +Ġs end +ĠDon ald +er ve +Ġdef ense +ines e +Ġy es +ĠF ire +ĠMus lim +ra q +Ġcontin ued +os h +Ġprov ides +Ġpr ison +ĠP re +Ġhapp y +Ġeconom y +Ġtr ust +ag s +ĠG ame +Ġweap ons +um an +ĠC le +it ation +Ġanal ysis +ĠT imes +Ġsc ience +- > +Ġfig ure +Ġdis app +ent y +Ġsoft ware +Ġu lt +Ġoffic ers +N ew +I s +Ġrem ains +ĠInd ia +Ġp sych +ri ef +Ġc at +es c +Ġob serv +Ġst age +ĠD ark +Ġent er +ch ange +Ġpass ed +Ġdes pite +ĠO ut +Ġmov ie +r s +Ġv oice +m ine +ĠPl ay +Ġto ward +ĠT er +Ġreg ion +Ġval ues +or ters +Ġm ount +Ġoffic er +ĠO ther +b an +Ġh ous +w ood +ro om +I V +ĠS un +se e +ĠO ver +ro g +9 0 +Ġl ay +ĠT ur +a wn +Ġpress ure +ĠS ub +Ġbook s +ed om +ĠS and +A A +ag o +Ġre asons +f ord +Ġactiv ity +U T +N ow +ĠSen ate +ce ll +n ight +Ġcall s +in ter +Ġlet ter +ĠR ob +ĠJ e +Ġcho ose +ĠL aw +G et +B e +Ġro b +Ġtyp es +Ġpl atform +Ġqu arter +R A +ĠT ime +Ġmay be +ĠC r +9 5 +p re +Ġmov ing +Ġl if +Ġgo ld +Ġs om +Ġpat ients +Ġtr uth +ĠK e +ur ance +ant ly +m ar +Ġchar ge +ĠG reat +Ġce le +---------------- ---------------- +Ġro ck +ro id +an cy +Ġcred it +a ud +B y +ĠE very +Ġmov ed +ing er +rib ution +Ġn ames +Ġstra ight +ĠHe alth +ĠW ell +Ġfe ature +Ġr ule +Ġsc he +in ated +ĠMich ael +ber g +4 1 +il ed +b and +Ġcl ick +ĠAng el +on ents +Â Ń +ĠI raq +ĠS aturday +Ġa ware +p art +Ġpat tern +O W +ĠL et +Ġgr ad +ign ed +Ġassoci ated +Ġst yle +n o +i ation +a ith +il ies +Ġst ories +ur ation +Ġindividual s +ĠâĢ ¦ +m iss +ĠAss oci +ish ing +ab y +Ġsum mer +ĠB en +Ġ3 2 +Ġar ch +ut y +ĠTex as +h ol +Ġfull y +Ġm ill +Ġfollow ed +ĠB ill +ĠInd ian +ĠSec ret +ĠB el +ĠFeb ruary +Ġjob s +Ġseem ed +ĠGo vern +i pped +Ġreal ity +Ġl ines +Ġp ark +Ġmeas ure +ĠO ur +I M +Ġbro ther +Ġgrow ing +Ġb an +Ġest im +Ġc ry +ĠS chool +Ġme chan +ĠO F +ĠWind ows +Ġr ates +ĠO h +Ġpos itive +Ġcult ure +ist ics +ic a +Ġh ar +y a +ite ly +i pp +Ġm ap +en cies +ĠWill iam +I I +ak ers +5 6 +ĠM art +ĠR em +Ġal tern +it ude +Ġco ach +row d +D on +Ġk ids +Ġj ournal +Ġcor por +Ġf alse +Ġwe b +Ġsle ep +Ġcont ain +Ġst o +Ġb ed +iver se +ĠR ich +ĠCh inese +Ġp un +Ġme ant +k nown +Ġnot ice +Ġfavor ite +a ven +Ġcond ition +Ġpur pose +) ) +Ġorgan ization +Ġchall eng +Ġman ufact +Ġsus p +ĠA c +Ġcrit ic +un es +uc lear +Ġm er +vent ion +Ġ8 0 +Ġm ist +ĠU s +ĠT or +htt p +ol f +Ġlarg er +Ġadv ant +Ġrese ar +Ġact ions +m l +Ġke pt +Ġa im +, ' +c ol +Ġbenef its +if ying +Ġact ual +ĠIntern ational +Ġveh icle +Ġch ief +Ġeff orts +ĠLe ague +ĠM ost +Ġwa it +Ġad ult +Ġover all +Ġspe ech +Ġhigh ly +Ġfem ale +Ġer ror +Ġeffect ive +5 4 +Ġenc our +w ell +Ġfail ed +Ġcons erv +Ġprogram s +Ġt rou +Ġa head +5 00 +vertis ement +I P +ĠF ound +p ir +Ġ % +Ġcr ime +and er +Ġloc ation +ĠI ran +Ġbehav ior +az ing +Ġr are +Ġem b +Ġca used +Ġsh ip +Ġact ive +Ġcont ribut +Ġg reen +Ġac qu +Ġref lect +ven ue +Ġf irm +Ġb irth +] . +Ġclear ly +Ġem ot +Ġag ency +ri age +Ġmem ory +9 8 +S A +ĠSe e +ac ing +C C +Ġbig gest +Ġr ap +Ġbas ic +Ġb and +e at +Ġsus pect +ĠM ac +Ġ9 0 +m ark +ist an +Ġsp read +am s +k i +as y +ra v +ĠR ober +Ġdemon str +r ated +Ġabs olute +Ġpl aces +Ġim pl +ibr ary +Ġc ards +Ġdest roy +Ġv irt +ve re +Ġapp eared +y an +p oint +Ġbe g +Ġtem per +s pe +ant ed +ear s +ĠD irect +Ġl ength +Ġbl og +am b +Ġint eg +Ġres ources +ac c +if ul +Ġsp ot +Ġfor ced +Ġthous ands +ĠMin ister +Ġqu al +ĠF rench +at ically +Ġgener ally +Ġdr ink +Ġth us +I L +od es +Ġappro pri +ĠRe ad +Ġwh om +Ġey e +Ġcol lege +Ġ4 5 +ire ction +Ġens ure +Ġapp arent +id ers +Ġrelig ious +Ġmin or +ol ic +Ġt ro +ĠWh y +rib ute +m et +Ġprim ary +Ġdevelop ed +Ġpe ace +Ġsk in +st e +av a +Ġbl ue +Ġfam ilies +Ġ ir +Ġapp ly +Ġin form +ĠSm ith +C T +i i +Ġlim it +Ġres ist +........ ........ +um n +Ġconf lic +Ġtw e +ud d +ĠT om +Ġl iter +qu e +b on +Ġha ir +Ġevent ually +Ġp us +Ġhelp ed +Ġag g +or ney +ĠApp le +Ġf it +ĠS ur +Ġpre m +Ġs ales +Ġsecond s +Ġstreng th +Ġfeel ing +¿ ½ +Ġt our +Ġknow s +o om +Ġex erc +Ġsom ew +ï ¿½ +> > +Ġsp okes +Ġide as +Ġreg ist +so ft +ĠD el +ĠP C +Ġpro pos +Ġlaun ch +Ġbott om +T H +ĠP lease +v est +it z +ĠIn ter +Ġsc ript +Ġr at +ar ning +Ġ il +ĠJ er +ĠA re +Ġwh atever +ok en +ci ence +Ġmod e +Ġag ree +Ġs ources +Ġinit ial +Ġrest rict +Ġwond er +us ion +## ## +ĠS il +vil le +Ġb urn +t w +as ion +Ġ £ +Ġn or +u ing +Ġre ached +Ġs un +Ġc ateg +ig ration +Ġc ook +Ġprom ot +Ġm ale +Ġcl imate +Ġf ix +Ġalleg ed +U R +all ed +Ġim ages +C ont +ot a +Ġschool s +i os +Ġd rop +Ġst ream +ĠM o +Ġprevious ly +al ing +Ġp et +Ġdou ble +Ġ( @ +ann el +Ġdef ault +t ies +Ġr ank +ĠD ec +ĠCoun cil +Ġweap on +Ġst ock +Ġanal y +ĠSt r +Ġpict ure +ĠPol ice +f erence +Ġcent ury +Ġcitiz ens +Ġon to +Ġexp and +Ġhe ro +ĠS ol +Ġw ild +Ġupd ate +Ġcustom ers +r ont +d ef +Ġl ik +Ġcrim inal +ĠChrist ian +S P +7 6 +Ġle aving +Ġother wise +ĠD ist +Ġbas is +5 2 +5 3 +ic ip +ĠB er +Ġrecomm end +Ġfl oor +Ġc rowd +ol es +Ġ7 0 +Ġcent ral +ĠE v +Ġd ream +Ġdown load +Ġconf ir +ĠTh om +Ġwind ow +Ġhapp ens +Ġun it +Ġt end +Ġs pl +Ġbec omes +Ġfight ing +Ġpred ict +ĠP ress +ĠP ower +Ġhe avy +ak ed +Ġf an +or ter +ate gy +B A +iz es +Ġsp end +H ere +Ġ200 7 +Ġad op +ĠH am +Ġfoot ball +ĠP ort +od ay +5 1 +amp ions +Ġtrans fer +h t +Ġ3 8 +ter m +ac ity +Ġb ur +] , +tern al +r ig +b ut +Ġthere fore +ĠB ecause +res p +re y +Ġm ission +S ome +Ġnot ed +Ġass um +Ġdise ase +Ġed it +Ġprog ress +r d +ĠB rown +oc al +Ġadd ing +Ġra ised +ĠAn y +Ġt ick +Ġsee ing +ĠPe ople +Ġagre ement +Ġser ver +Ġw at +Ġdeb ate +Ġsupp osed +il ing +Ġlarg est +Ġsuccess ful +ĠP ri +ĠDemocr atic +Ġj ump +ĠSyri a +Ġown ers +Ġoff ers +Ġshoot ing +Ġeff ic +se y +Ġha ven +ver se +te red +ĠL ight +im al +ĠB ig +Ġdef end +Ġbe at +Ġrecord s +% ) +Ġsc en +Ġemploy ees +Ġdev ices +he m +Ġcom mer +ĠM ex +Ġbenef it +ĠPro f +Ġil leg +Ġsur face +ĠAl so +Ġh arm +ing ly +w ide +ĠA lex +Ġsh ut +ĠC ur +Ġl ose +p m +Ġchall enge +se mb +Ġst ation +Ġint elligence +Ġacc ur +ĠFl or +Ġrequ ires +ĠM al +b um +Ġh ospital +Ġsp irit +Ġoff ered +Ġprodu ce +ĠComm un +Ġcreat ing +Ġcr is +s pect +Ġend ed +Ġd aily +Ġvot ers +land s +i as +i h +on a +Ġsm art +ĠOff ice +ĠL ord +ri al +ĠIntern et +Ġcirc um +Ġextreme ly +' . +Ġopin ion +ĠM il +Ġg ain +B S +ĠF in +y p +Ġuse ful +Ġbud get +Ġcom fort +is f +Ġback ground +el ine +Ġep isode +Ġen emy +Ġtri al +Ġestab lish +d ate +ĠC ap +Ġcontin ues +Ġshow ing +ĠUn ion +w ith +Ġpost ed +ĠSy stem +Ġe at +ri an +Ġr ise +ĠGerman y +il s +Ġsign ed +Ġv ill +Ġgr and +m or +ĠEng land +Ġproject s +um ber +Ġconf erence +z a +Ġrespons ible +ĠAr ab +Ġlearn ed +âĢĶ âĢĶ +i pping +ĠGe orge +O C +Ġreturn ed +ĠAustral ia +Ġb rief +Q u +Ġbr and +ill ing +ab led +Ġhig hest +Ġtr ain +ĠComm ission +wh ile +Ġn om +cept ion +Ġm ut +ĠBl ue +Ġinc ident +v ant +8 6 +ĠI D +Ġn uclear +7 4 +ĠL ike +ĠR E +ĠM icro +l i +m ail +Ġcharg es +8 9 +Ġad just +ad o +Ġear th +N A +Ġpr ices +P A +Ġd raft +Ġrun s +Ġcandid ate +ens es +Ġmanag ement +ĠPh il +ĠM iss +Ġte ach +g ram +Ġunderstand ing +a it +ic ago +A dd +ĠE p +sec ut +Ġsepar ate +Ġinst ance +Ġe th +Ġun less +**** **** +ĠF ore +in ate +Ġoper ations +S p +Ġf aith +g ar +ĠCh urch +ron ic +Ġconf ig +os ure +Ġactiv ities +Ġtrad itional +Ġ3 6 +Ġd irection +Ġmach ine +Ġsur round +Ġp ush +un ction +ĠE U +Ġeas ier +Ġarg ument +G B +Ġm icro +Ġsp ending +iz ations +Ġthe ory +ad ow +Ġcall ing +ĠL ast +Ġd er +Ġinflu ence +Ġcomm it +Ġph oto +Ġun c +ist ry +g n +ast e +ack s +Ġdis p +ad y +d o +ĠG ood +Ġ ` +Ġw ish +Ġreve aled +Âł Âł +l ig +Ġen force +ĠComm ittee +Ġche m +Ġmil es +Ġinterest ed +Ġsol ution +ic y +in ct +Ġ- > +ĠD et +Ġrem oved +Ġcomp ar +e ah +Ġpl ant +ĠS ince +Ġachie ve +Ġadvant age +Ġslight ly +b ing +Ġpl aced +u nder +201 5 +ĠM ad +Ġt im +os es +Ġc ru +ĠR ock +Ġmost ly +Ġneg ative +Ġset ting +Ġprodu ced +Ġm ur +Ġconnect ion +ĠM er +Ġdri ver +Ġexecut ive +Ġass ault +Ġb orn +ĠV er +t ained +Ġstruct ure +Ġredu ce +Ġdec ades +Ġd ed +u ke +ĠM any +idd en +Ġle ague +S e +Ġjo in +Ġdis co +Ġd ie +c ks +act ions +Ġass ess +ag n +Ġgo als +our s +I R +Ġsen ior +ill er +m od +ip ment +oc ol +u y +ĠQ ue +Ġpart ies +ir gin +Ġle arning +it able +Ġstre et +Ġcamer a +A pp +Ġsk ills +b re +c ious +Ġcele br +ĠFr anc +Ġexist ing +Ġwill ing +l or +Ġ id +ĠSp ace +Ġcrit ical +ĠL a +ortun ately +Ġser ve +Ġc old +Ġspec ies +T S +Ġanim als +ĠB ay +Ġold er +ĠU nder +est ic +ĠT re +Ġte acher +Ġpre fer +v is +Ġth read +ĠM att +Ġmanag er +ãĥ » +Ġprofess ional +ĠV ol +Ġnot es +The se +ul a +Ġf resh +ent ed +u zz +ed y +clus ion +ĠR el +Ġdoub t +E O +Ġopen ed +ĠB it +Ad vertisement +Ġgu ess +ĠU N +Ġse qu +Ġexpl ain +ott en +Ġatt ract +ak s +Ġstr ing +Ġcont ext +oss ible +ĠRepublic ans +Ġsol id +Ġc ities +Ġask ing +Ġr andom +u ps +ur ies +ar ant +dd en +g l +ĠFlor ida +Ġdep end +ĠSc ott +Ġ3 3 +Ġi T +ic on +Ġmention ed +Ġ2 000 +Ġclaim ed +Ġdefin itely +ul f +Ġc ore +Ġopen ing +ĠCon st +wh ich +ĠT ra +A G +7 2 +Ġbelie ved +ad a +Ġ4 8 +ĠSec urity +yr ight +ĠP et +ĠL ou +Ġhold ing +======== ======== +Ġ ice +Ġb row +Ġauthor ities +h ost +w ord +Ġsc ore +ĠD iv +Ġcell s +Ġtrans l +Ġneigh bor +Ġrem ove +u ct +Ġdist rict +ĠA ccording +Ġwor se +Ġconcern s +Ġpresident ial +Ġpolic ies +ĠH all +7 3 +Ġh us +A Y +Ġ200 6 +ĠJ ud +Ġindepend ent +ĠJust ice +ili ar +pr int +igh ter +Ġprotect ion +z en +Ġsu dden +h ouse +ĠJ es +P R +ĠIn f +Ġb ul +Ġ _ +ĠServ ice +ĠP R +Ġstr ategy +ff ect +Ġgirl s +Ġmiss ing +oy al +ĠTe am +ul ated +Ġd at +Ġpolit ics +ab or +A ccording +Ġspe ll +Ġg raph +ort hern +T C +A b +Ġlab or +is her +Ġk ick +ĠiT unes +Ġstep s +pos es +Ġsmall er +E n +ber t +Ġro ll +Ġresear chers +Ġcl osed +Ġtrans port +Ġlaw y +________ ________ +ĠCh icago +Ġas pect +Ġn one +Ġmar riage +9 6 +Ġe lements +ĠF re +ĠS al +Ġd ram +F C +t op +e qu +Ġhe aring +Ġsupport ed +Ġtest ing +co hol +Ġmass ive +Ġst ick +Ġgu ard +is co +ph one +F rom +How ever +Ġb order +Ġcop y +ograph y +l ist +7 1 +Ġown er +cl ass +ru it +r ate +ĠO nce +Ġdig ital +Ġt ask +ER S +Ġinc red +t es ++ + +ĠFr ance +Ġb reat +ow l +Ġiss ued +ĠW estern +Ġdet ect +Ġpart ners +Ġsh ared +ĠC all +Ġcan cer +ac he +rib e +Ġexpl ained +Ġhe at +{ " +Ġinvest ment +ĠB ook +Ġw ood +Ġtool s +ĠAl though +Ġbelie f +Ġcris is +Ġg e +ĠM P +Ġoper ation +ty pe +~ ~ +g a +Ġcont ains +ant a +Ġexp ress +ĠG roup +ĠJ ournal +k a +Ġam b +ĠUS A +Ġfind ing +Ġfund ing +h ow +Ġestab lished +ide os +Ġdeg ree +Ġdanger ous +ang ing +Ġfre edom +pp ort +out hern +Ġch urch +Ġc atch +ĠTw o +Ġpres ence +ĠGu ard +U p +Ġauthor ity +ĠPro ject +Ġbut ton +Ġcon sequ +Ġval id +Ġwe ak +Ġstart s +Ġref erence +ĠM em +" ) +U N +or age +ĠO pen +Ġcol lection +y m +g ency +Ġbeaut iful +ro s +Ġtell s +Ġwa iting +n el +Ġprov iding +ĠDemocr ats +Ġd aughter +Ġm aster +Ġpur poses +ĠJapan ese +Ġequ al +Ġturn s +Ġdoc uments +Ġwatch ing +R es +Ġr an +201 4 +Ġre ject +ĠKore a +Ġvictim s +Le vel +ere nces +Ġw itness +Ġ3 4 +Ġre form +com ing +Ġocc up +Ġc aught +Ġtra ffic +ad ing +Ġmod els +ar io +Ġserv ed +Ġb atter +u ate +ĠSecret ary +Ġagre ed +Ġtr uly +yn am +ĠR et +Ġun its +ĠRes earch +h and +az ine +ĠM ike +Ġvar iety +ot al +Ġam azing +Ġconfir med +Ġentire ly +Ġpurch ase +Ġe lement +Ġc ash +Ġdeter mine +D e +Ġc ars +ĠW all +â ĸ +Ġview s +Ġdrug s +Ġdep artment +ĠSt ep +u it +Ġ3 9 +as ure +ĠCl ass +Ġc overed +ĠB ank +Ġme re +u ana +Ġmult i +Ġm ix +Ġun like +lev ision +Ġsto pped +Ġs em +ĠG al +ul es +Ġwe l +ĠJohn son +l a +Ġsk ill +Ġbec oming +ri e +Ġappropri ate +f e +ell ow +ĠPro t +ul ate +oc ation +Ġweek end +od ies +Ġsit es +Ġanim al +ĠT im +Ġsc ale +Ġcharg ed +Ġinst ruct +ill a +Ġmethod s +Ġc ert +Ġjud ge +ĠH el +Ġdoll ars +Ġstand ing +ĠS qu +Ġdeb t +l iam +Ġdri ving +ĠS um +ĠEd ition +Ġal bum +and on +I F +ĠU k +6 3 +ad er +Ġcommer cial +es h +ĠGovern ment +Ġdisc overed +Ġout put +ĠHill ary +ĠCar ol +Ġ200 5 +Ġab use +anc ing +Ġsw itch +Ġann ual +T w +Ġst ated +ag ement +in ner +Ġdem ocr +Ġres idents +Ġallow ing +Ġfact ors +od d +Ġf uck +em ies +Ġoccur red +ot i +Ġn orth +ĠP ublic +Ġinj ury +Ġins urance +C L +oll y +ã Ģ +Ġrepe ated +Ġar ms +ang ed +Ġconst ruction +Ġf le +P U +ic ians +Ġfor ms +ĠMc C +ant ic +Ġm ental +p ire +Ġequ ipment +Ġf ant +Ġdiscuss ion +Ġregard ing +k in +ar p +Ġch air +og ue +Ġpro ceed +ĠI d +O ur +Ġmur der +M an +Ġ4 9 +as p +Ġsupp ly +Ġin put +Ġwe alth +liam ent +Ġpro ced +or ial +ĠSt at +ĠN FL +hen s +ĠInst itute +Ġput ting +ourn ament +et ic +Ġloc ated +Ġk id +er ia +r un +Ġpr inc +Ġ ! +go ing +ĠB et +Ġcl ot +Ġtell ing +Ġprop osed +i ot +or ry +Ġfund s +g ment +ĠL ife +Ġb aby +ĠB ack +Ġsp oke +Im age +Ġear n +ĠA T +g u +Ġex change +ĠL in +ov ing +Ġp air +M ore +az on +Ġarrest ed +Ġkill ing +c an +ĠC ard +y d +Ġident ified +Ġm obile +Ġthan ks +ony m +ĠF orm +Ġhundred s +ĠCh ris +ĠC at +Ġtre nd +h at +ĠA v +om an +Ġelect ric +ĠW il +S E +O f +Ġrest aur +ot ed +Ġtr ig +Ġn ine +Ġb omb +Wh y + ¯ +Ġco verage +Ġapp eal +ĠRober t +ĠS up +Ġfin ished +Ġfl ow +Ġdel iver +Ġcal cul +Ġphot os +Ġph il +Ġpie ces +Ġapp re +k es +Ġr ough +D o +Ġpart ner +Ġconcern ed +Ġ3 7 +ĠG en +C ol +ct ors +Ġ= > +st ate +Ġsuggest ed +ĠFor ce +C E +Ġher self +ĠPl an +w orks +o oth +ren cy +Ġcor ner +Ġhus band +Ġintern et +ĠA ut +em s +os en +ĠAt l +g en +Ġbal ance +6 2 +Ġsound s +te xt +Ġar r +ov es +Ġmill ions +Ġrad io +Ġsat isf +ĠD am +M r +G o +S pe +Ġcomb at +r ant +ĠG ree +Ġf uel +Ġdist ance +Ġtest s +Ġdec re +ĠE r +Ġman aged +D S +Ġt it +Ġmeas ures +ĠL iber +Ġatt end +as hed +ĠJ ose +ĠN ight +d it +ĠN ov +ĠE nd +out s +Ġgener ation +Ġadv oc +y th +Ġconvers ation +ĠS ky +act ive +ce l +ri er +ĠFr ank +Ġg ender +Ġcon cent +Ġcar ried +and a +ĠV irgin +Ġarri ved +ic ide +ad ed +Ġfail ure +Ġmin imum +le ts +Ġwor st +Ġkeep ing +Ġint ended +Ġilleg al +Ġsub sc +Ġdetermin ed +Ġtri p +Y es +Ġra ise +Ġ ~ +Ġfeel s +Ġpack age +ĠJ o +h i +201 6 +re al +Ġf ra +Ġsy mb +M e +uck y +p ret +ĠK h +ĠEd it +ĠWe b +em ic +ĠCol or +Ġjust ice +I nt +Ġfar m +ck now +" > +el ess +Ġredu ced +Ġ5 00 +x x +ĠR ad +ĠW ood +Ġcl in +Ġhy p +il er +ur a +k ins +8 5 +6 1 +ĠThe ir +ĠM ary +Ġs an +Ġno vel +ĠWh o +Ġcap acity +Ġimp ossible +Ġpl ays +Ġmin ister +ij uana +ic ate +ĠS et +Ġf ram +Ġ ing +Ġcommun ities +ĠF BI +it a +Ġb on +Ġstr ateg +Ġinterest s +l ock +g ers +m as +ĠAN D +Ġconflic t +Ġrequire ments +Ġs ac +Ġoper ating +in i +rel ated +Ġcomm itted +Ġrelative ly +Ġs outh +¯ ¯ +Ġaff ord +Ġident ity +Ġdec isions +Ġacc used +pl ace +Ġvict ory +o ch +i at +N ame +C om +t ion +ed s +Ġsee k +Ġt ight +ĠIm ages +Ġinit i +Ġhum ans +Ġfam iliar +Ġaud ience +Ġintern al +vent ure +Ġs ides +ĠT O +Ġd im +Ġcon clud +Ġapp oint +Ġenforce ment +ĠJ im +ĠAssoci ation +Ġcircum st +ĠCanad ian +Ġjo ined +Ġdiffere nces +ĠL os +Ġprot est +Ġtw ice +w in +Ġgl ass +ars h +ĠAr my +Ġexp ression +Ġdec ide +Ġplan ning +an ia +Ġhand le +ĠMicro soft +ĠN or +Ġmax imum +ĠRe v +Ġse a +Ġev al +Ġhel ps +re f +Ġb ound +Ġm outh +Ġstand ards +Ġcl im +ĠC amp +ĠF ox +cl es +Ġar my +ĠTe chn +ack ing +x y +S S +Ġ4 2 +Ġbu g +ĠUk rain +ĠM ax +ĠJ ones +ĠSh ow +l o +Ġplan et +Ġ7 5 +Ġwin ning +Ġf aster +Ġspe ct +Ġbro ken +T R +Ġdef ined +Ġhealth y +Ġcompet ition +htt ps +ĠIs land +ĠF e +Ġannoun ce +ĠC up +ĠInst ead +Ġcl ient +Ġposs ibly +se ction +ock et +l ook +Ġfin ish +Ġcre w +Ġres erv +Ġed itor +Ġh ate +Ġs ale +Ġcontro vers +Ġp ages +w ing +Ġnum er +Ġopp osition +Ġ200 4 +Ġref uge +Ġfl ight +Ġap art +ĠL at +A meric +ĠAfric a +Ġapplic ations +ĠPal est +ĠB ur +Ġg ar +ĠSoc ial +Ġup gr +Ġsh ape +Ġspe aking +ans ion +a o +ĠS n +Ġwor ry +ĠBrit ain +P lease +rou d +Ġh un +Ġintrodu ced +Ġd iet +I nd +ĠSec ond +Ġfun ctions +ut s +ĠE ach +ĠJe ff +Ġst ress +Ġaccount s +Ġgu arant +ĠAn n +ed ia +Ġhon est +Ġt ree +ĠAfric an +ĠB ush +} , +Ġs ch +ĠOn ly +Ġf if +ig an +Ġexerc ise +ĠEx p +Ġscient ists +Ġlegisl ation +ĠW ork +ĠS pr +à Ĥ +ĠH uman +Ġ è +Ġsur vey +Ġr ich +ri p +Ġmain tain +Ġfl o +Ġleaders hip +st ream +ĠIslam ic +Ġ 01 +ĠCol lege +Ġmag ic +ĠPr ime +Ġfig ures +201 7 +ind er +x ual +ĠDe ad +Ġabsolute ly +Ġfour th +Ġpresent ed +resp ond +rib le +Ġal cohol +at o +ĠD E +por ary +Ġgr ab +Ġvar i +Ġqu ant +ĠPh oto +Ġpl us +r ick +ar ks +Ġaltern ative +Ġp il +Ġappro x +th at +Ġobject s +ĠR o +ĠAnd roid +Ġsignificant ly +ĠR oad +k ay +R ead +av or +Ġa cknow +ĠH D +ĠS ing +O r +ĠM ont +Ġun s +pro f +Ġneg oti +ĠAr ch +ik i +Ġte levision +ĠJew ish +Ġcomm ittee +Ġmot or +Ġappear ance +Ġs itting +Ġstri ke +ĠD own +com p +ĠH ist +Ġf old +ac ement +ĠLou is +Ġbel ong +ĠâĢ ¢ +Ġm ort +Ġprep ared +Ġ6 4 +ĠM aster +Ġind eed +ĠD en +Ġre nt +T A +our ney +ar c +S u +9 7 +Ġadv ice +Ġchang ing +Ġlist ed +Ġlaun ched +is ation +ĠP eter +is hes +Ġl ived +ĠM el +ĠSup reme +ĠF ederal +Ġ) ; +ruct ure +Ġset s +Ġphil os +u ous +Ġ ł +Ġappl ied +ĠN OT +Ġhous ing +ĠM ount +Ġo dd +Ġsu st +D A +ffic ient +Ġ ? +ol ved +Ġp owers +Ġth r +Ġrem aining +ĠW ater +L C +Ġca uses +ãģ ® +Ġman ner +ad s +Ġsuggest s +Ġend s +stand ing +f ig +ĠD un +id th +Ġg ay +Ġter min +ĠAngel es +M S +Ġscient ific +Ġco al +ap ers +b ar +ĠThom as +Ġsy m +ĠR un +th is +P C +igr ants +Ġmin ute +ĠDist rict +cell ent +Ġle aves +Ġcomple ted +am in +Ġfoc used +Ġmon itor +Ġveh icles +M A +ĠM ass +ĠGr and +Ġaffect ed +itution al +Ġconst ruct +Ġfollow s +Ġt on +re ens +Ġh omes +ĠE xt +ĠLe vel +r ast +ĠI r +Ġel im +Ġlarge ly +ĠJ oe +Ġvot es +all s +Ġbusiness es +ĠFound ation +ĠCent ral +Ġy ards +Ġmaterial s +ul ner +Ġgu ide +Ġclos er +um s +Ġsp orts +ed er +J ust +Ġtax es +8 4 +ĠO ld +Ġdec ade +ol a +Ġv ir +Ġdro pped +Ġdel ay +it ect +Ġsec ure +ste in +le vel +Ġtre ated +Ġfil ed +ain e +Ġv an +Ġm ir +Ġcol umn +ict ed +e per +Ġro t +Ġcons ult +Ġent ry +Ġmar ijuana +ĠD ou +Ġapparent ly +ok ing +clus ive +Ġincre ases +an o +Ġspecific ally +Ġte le +ens ions +Ġrelig ion +ab ilities +Ġfr ame +ĠN ote +ĠLe e +Ġhelp ing +Ġed ge +ost on +Ġorgan izations +à ĥ +ĠB oth +hip s +Ġbig ger +Ġbo ost +ĠSt and +Ġro w +ul s +ab ase +Ġr id +L et +are n +ra ve +Ġst ret +P D +Ġv ision +Ġwe aring +Ġappre ci +Ġa ward +ĠU se +Ġfact or +w ar +ul ations +) ( +Ġg od +Ġter rit +Ġpar am +ast s +8 7 +Ġen emies +ĠG ames +F F +Ġacc ident +W ell +ĠMart in +T ER +Ġat h +ĠHe ll +Ġfor g +Ġve ter +ĠMed ic +f ree +Ġst ars +Ġexp ensive +Ġac ad +ra wn +ĠW he +Ġl ock +Ġform at +Ġsold iers +s m +Ġag ent +Ġrespons ibility +or a +ĠS cience +Ġrap id +Ġt ough +ĠJes us +Ġbelie ves +M L +Ġwe ar +le te +Ãĥ ÃĤ +ĠD ri +Ġcomm ission +ĠB ob +O h +ap ed +Ġwar m +ÃĥÃĤ ÃĥÃĤ +Ġ200 3 +ort ion +Ġhas n +ust er +Ġun ivers +ĠI ll +Ġk ing +olog ies +9 4 +ĠT em +ĠM os +Ġpat ient +ĠMex ico +ce an +ĠDe ath +ĠSand ers +y ou +ĠC ast +ĠComp any +pt y +Ġhappen ing +F P +ĠB attle +Ġb ought +A m +M od +U s +ut ers +ĠC re +ĠTh ose +Ġ4 4 +is er +Ġs oul +ĠT op +ĠHar ry +ĠA w +Ġse at +ff ee +Ġrev olution +Ġ( " +ĠD uring +et te +Ġr ing +Ġoff ensive +Ġreturn s +Ġv ideos +Ġdis cl +Ġfam ous +en ced +ĠS ign +ĠR iver +Ġ3 00 +P M +ĠB us +ĠC H +Ġcandid ates +ard en +Ġpercent age +Ġvis ual +Ġthan k +Ġtrou ble +ner gy +Ġ200 1 +Ġpro ve +ash ion +Ġen h +ĠL ong +U M +Ġconnect ed +Ġposs ibility +O ver +Ġexper t +Ġl ibrary +art s +ĠDirect or +Ġfell ow +9 2 +ir ty +Ġd ry +Ġsign s +ĠL ove +Ġqu iet +f oot +Ġp ure +ĠH un +Ġf illed +ph as +ĠE lect +end ment +ĠEx pl +Ġun able +n s +m o +Ġv ast +ob e +Ġident ify +app ing +ĠCarol ina +g ress +Ġpro te +Ġf ish +Ġcircumst ances +raz y +ĠPh ot +Ġb odies +ĠM ur +Ġdevelop ing +ĠA R +Ġexperien ced +Ġsubst ant +ĠBo ard +es ome +Ġdom estic +Ġcomb ined +ĠP ut +Ġchem ical +ĠCh ild +Ġpo ol +ĠC y +Ġe gg +c ons +st ers +Ġh urt +Ġmark ets +Ġconserv ative +Ġsupp orters +Ġag encies +id el +O b +ur b +Ġ4 3 +ĠDef ense +y e +ĠA p +du le +Ġtemper ature +Ġconduct ed +ĠCh ief +Ġpull ed +Ġf ol +L ast +ont o +os is +V ER +D es +ĠP an +F irst +Ġadv ance +Ġlic ense +r ors +ĠJ on +Ġimag ine +Ġhe ll +Ġf ixed +Ġinc or +os ite +ĠL og +ick en +] : +Ġsurpr ise +h ab +Ġc raft +ol t +ĠJ ul +Ġd ial +Ġrele vant +Ġent ered +Ġlead s +ĠA D +ĠCle an +Ġpict ures +ess or +Ġal t +Ġpay ing +P er +ĠMark et +Ġupd ates +am ily +ĠT ype +ĠH ome +Ġ5 5 +semb ly +rom e +8 3 +Ġgreat est +Ġhe ight +Ġhe av +ain ts +Ġlist en +as er +ĠS H +Ġcap able +ac le +Ġpers pect +in ating +Ġoff ering +ry pt +ĠDe velop +ab in +r c +Ġbr ight +al ty +ar row +Ġsupp l +ind ing +ack ed +gy pt +ĠAn other +p g +ĠVirgin ia +ĠL u +Ġpl anned +Ġp it +Ġswe et +T ype +ĠD i +Ġtyp ically +ĠFranc isco +Ġpro spect +ĠD an +Ġte en +re es +Ġsc hed +Ġh ol +Ġsc r +Ġlot s +l ife +Ġnews p +Ġfor get +ĠN one +ĠM iddle +ĠR yan +ed d +Ġse vere +Ġsu it +ll er +9 3 +Ġcor respond +Ġexpl os +u ations +Ġfl ag +g ame +r id +Ġpr in +ĠD ata +Ġde ploy +ĠEn ter +su it +gh an +ĠM en +Ġthough ts +Ġmat ters +Ġad apt +ĠA ri +Ġf ill +Ġfor th +Ġs am +Ġ4 1 +Ġpay ment +ĠH or +Ġsp ring +du c +Ġl osing +Ġbring ing +F O +al a +Ġdist ribution +he red +b our +ĠIsrael i +om a +Ġcomb ination +Ġpl enty +V E +C an +ĠH aw +Ġper man +ĠSpe cial +Ġto w +Ġsee king +Ġexam ples +Ġclass es +c r +Ġbe er +Ġmov es +ĠI P +ĠK n +Ġpan el +E ven +Ġproper ly +Ġr is +Ġpl ug +Ġestim ated +E very +Ġdef ensive +ag raph +Ġpre gn +Ġinst it +ĠV ict +Ġvol ume +Ġpos itions +Ġl inks +ĠPro gram +ĠWe ek +ag ues +Ġtrans form +k er +ĠC EO +Ġc as +Ġopp onent +Ġtwe et +ĠC ode +Ġsh op +Ġf ly +Ġtal ks +Ġb ag +Ph one +Ġa id +Ġpl ants +Ġ6 5 +Ġatt orney +ar ters +qu est +ĠMag ic +Ġbeg ins +Ġmy ster +Ġenvironment al +Ġst orage +N N +Ġm arg +Ġs ke +Ġmet al +ell y +Ġord ered +Ġrem ained +Ġl oved +Ġprom pt +Ġupd ated +Ġexper ts +Ġwalk ing +Ġan cient +Ġperform ed +AT E +Ġne ither +i ency +Ġmanufact ure +ĠP ak +Ġselect ed +Ġm ine +Ġult imately +Ġexpl an +Ġlab el +ĠServ ices +ribut ed +Tr ump +Ġsy n +ĠU lt +S C +Ġme at +Ġg iant +ĠW ars +ĠO N +Ġad m +Ġinter pret +Ġeven ing +Ġev il +ĠB oston +ĠW ild +Ġ à +ĠBit coin +ĠAm azon +D r +ĠIn formation +Ġobvious ly +Ġadv anced +Ph oto +ol ar +Ġwe ather +Ġsymb ol +Ġso le +Ġpot entially +ost er +Ġorig inally +m un +3 00 +az e +ess ions +Ġde ck +Ġst ood +Ġyou th +ĠB ern +R ep +ĠT est +Ġbas ically +ot ic +Ġinvol ve +ol it +ly n +S ee +Ġair craft +Ġconf irm +E W +Ġmess ages +ĠRich ard +Ġk it +Ġpro hib +Ġv ulner +is ters +Ġexist ence +Ġturn ing +ĠS P +Ġdes ire +Ġfl at +Ġm ent +se ason +ang es +Ġneighbor hood +ĠL ake +AT ION +Ġpoint ed +b ur +Ġinn ov +uc ks +U L +Ġprofess or +Ġexp ressed +A B +ic ious +Ġ200 2 +ĠDe v +Ġs ession +Ġb are +s en +Ġdis s +ĠC ath +ĠP ass +ĠP oint +Ġdo ctor +or row +ail ed +ĠR ub +ĠD C +ĠChar l +p erson +Ġwrit er +igh ters +ure au +Ġob lig +Ġrecord ed +Ġbro ke +Ġord ers +il ty +Ġmot ion +in ity +l aw +ad ium +Ġimm igration +Ġcontr ast +Ġb att +Ġex cellent +Ġtechn ical +am i +Ġt un +Ġcl oud +ĠY ear +ge on +Ġcre ation +Ġstr ange +Ġa uth +Ġfor t +b orn +Ġext ent +ĠT oday +ĠCl ub +Ġr ain +Ġs ample +Ġaccept ed +Ġt act +Ġf ired +ĠS on +Ġstand s +Ġb oot +Ġ4 7 +Ġstat ements +Ġvers ions +Ġse lling +ound ed +Ġ199 0 +Ġwere n +ĠW atch +Ġexper iment +P ost +Ġret ail +ul ed +In st +un te +ãĥ ¼ +Ġdep art +Ġb ond +i very +om pl +Ġre action +ĠSyri an +ĠP ac +app ed +ani el +D P +Ġres olution +Ġre act +Ġappro ved +on om +m ond +ĠO ffic +-- - +Ġrepl ace +Ġt ack +Ġsp ort +Ġch ain +Ġemer gency +r ad +ĠPalest in +Ġ4 6 +Ġautom atically +Ġrout e +Ġp al +Ġb anks +ĠPar is +ĠMed ia +ro ad +ic ing +i xt +ist ed +Ġg rew +Ġco ord +ĠW here +om in +Ġsub s +� � +Ġ ± +Ġcorpor ate +Ġse lection +n oon +ĠRep ort +c s +clud ing +ord ers +anc he +ĠIt s +Ġslow ly +ĠE gypt +ĠA cc +Ġcol le +iqu es +E X +Ġattempt s +ur l +ĠC ross +Ġfind ings +ĠS C +ĠO R +Ġind ex +ens ity +ĠW ay +ĠL and +Ġsh ock +d is +Ġd ynam +Ġc art +m osp +S ince +i est +ĠB oy +Ġst orm +ĠCont in +201 3 +he w +il it +Ġess ential +iqu id +O ther +ive red +Ġreason able +A ct +Ġsub sequ +ĠP ack +ĠF ort +Ġconsider ing +Ġun iversity +l og +Ġmar ried +Ġill ust +ĠTr ue +£ ı +Ġnumer ous +rast ructure +Ġserious ly +Ġrefer red +u a +Ġconsist ent +on na +ĠRe al +ru ption +ci ples +Ġfact s +9 1 +ot es +er g +The n +Ġacc ompl +N ote +Ġre venue +Ġpass ing +Ġm al +e en +ĠY et +Ġg ather +ter day +ew ork +ĠA uthor +P e +Ġopt im +Ġr ub +Ġè £ı +Ġun known +st one +Ġun ion +ol ve +Ġopportun ities +Ġbrow ser +ĠW al +ĠC ost +Ġreport ing +st s +p et +Ġs and +Ġsudden ly +Ġsurpr ising +ĠV R +Ġsomew hat +ĠB as +ult ure +iz z +ĠC D +Ġchalleng es +Ġsett ings +Ġexperien ces +ĠF ull +Ġcan n +Ġrece iving +ES T +Ġj oint +Ġcult ural +Ġa st +8 2 +as tern +ce ived +ĠC ru +Ġb ull +p ired +am m +Ġfac ing +p ower +Ġb oss +ĠH ol +Ġinst r +Ġincreasing ly +Ġsh ift +Ġstre ets +ĠWilliam s +ab b +Ġl ie +Ġl augh +ĠC a +P L +Ġadult s +Ġcustom er +Ġob tained +Ġsupport ing +ht ml +f ire +Ġdetail ed +Ġpick ed +ĠR ight +ld er +E E +st ood +ĠK im +Ġw ire +Ġs ight +Ġdevelop ers +Ġpers ons +Ġs ad +Ġc up +Ġwar ning +Ġboy s +l ong +Ġb ird +f o +Ġw al +Ġobserv ed +Ġz one +iven ess +Ġch annel +c ript +Ġref used +ĠAg ain +Ġsu c +Ġspokes man +ĠRe f +r ite +ou ston +ãĥ ³ +ĠS her +Ġact s +ĠN ame +Ġstrugg le +ar ry +omet imes +Ġdisc rim +H T +Ġcateg ory +Ġreal ize +Ġemploy ee +ĠAf ghan +en ger +Ġgun s +ĠSte ve +ĠM ot +ĠO l +ok ed +Ġth ick +Ġfair ly +ill y +Ġsur ve +ĠM at +we ight +â Ķ +Ġtro ops +Ġag ents +Ġbatter y +Ġmot iv +à ¡ +S ec +d en +o very +L S +Ġfl u +Ġconf ident +ĠO per +Ġem pty +Ġp hen +Ġse ctor +Ġexc ited +Ġrem ote +ap h +o en +Ġdestroy ed +Ġmor al +ĠH P +ĠR on +Ġd ress +ĠB at +Ġl it +ĠM S +Ġa f +H L +r um +is ms +Ġshould n +Ġsym pt +ĠTor onto +het ic +Ġcar bon +Ġinstall ed +Ġviol ent +Ġsol ar +j a +Ġpract ices +Ġr ide +ĠP enn +Ġimpro ved +Ġaud io +Ġbehav i +ĠP S +Ġe ating +D ata +ĠRe view +p ass +cl aim +u ated +ang ers +c hen +Ġproper ties +Ġany where +An other +Ġbl ow +ĠJack son +Ġp roud +Ġplan e +l ines +Ġsqu are +Ġpro of +ans as +Ġtalk ed +m akers +Ġs ister +Ġhold s +Ġres ident +Ġ= = +Ġresist ance +Ġspl it +Ġpro secut +Ġconf idence +res ents +Ġcut s +Ġexcept ion +Ġz ero +Get ty +Ġcop yright +Ġtot ally +orm al +ific ations +ĠAustral ian +Ġs ick +Ġ1 50 +Ġhouse hold +Ġfe es +Ġdri vers +og en +ĠN Y +Ġnecess arily +Ġregul ations +ear ing +s l +Ġperspect ive +c are +ic ial +H is +Ġesc ape +Ġsurpr ised +ĠV an +ur rent +Ġv ac +8 1 +ĠTh us +Ġem phas +ĠCh ampions +ĠI ce +Ġn arr +Ġhead s +Ġca using +b el +f ortunately +ĠM a +Ġtarg ets +ci pl +Ġafter noon +Ġadd s +ĠMay be +ĠF our +ess ed +ple te +Ġus ual +ch o +ing u +Ġwith d +ĠE nergy +ĠE conom +O O +Ġart icles +Ġinj ured +Ġman age +Ġexpl ains +Ġdi agn +R ec +at ures +Ġlink ed +Ġdiscuss ed +Ġexpl o +Ġocc asion +ath an +Ġopp osite +Ġfac es +Ġden ied +ĠK night +Ġn ut +Ġapprox imately +Ġdisapp oint +onym ous +ĠB est +ĠL o +ĠH y +ĠA ff +Ġvot ing +an while +ĠII I +Ġinstit utions +ag ram +ĠD aily +Ġdr ag +Ġnear by +Ġgu ilty +Ġcon ver +P re +s hip +Ġre ward +Ġphilos oph +ĠS S +u gh +Ġapp s +f riend +Ġu pper +Ġad vert +Ġs now +Ġfr ust +Ġour selves +F r +ĠD ie +amp ion +Ġdis miss +Ġc ere +Ġsign al +f rom +Ġ ). +Ġ5 2 +Ġcr imes +it ors +est ival +use um +Ġcoun cil +ĠS aud +M ay +ĠG un +ic ian +et her +Ġsu fficient +ĠH en +so le +Ġhistor ical +ĠF ar +ĠT urn +Ġp in +Ġsuc ceed +m at +ly mp +Ġtrad ition +ĠO k +Ġc ro +Ġdesc ription +al le +Ġsk y +T e +Ġwide ly +Ġw ave +Ġdefin ition +ĠJew s +Ġcy cle +Ġref ere +Ġbr ings +us al +Ġal ive +Ġfrequ ently +Ġint ention +ĠCont rol +l v +y stem +Ġpriv acy +g ent +ren ce +ĠQu est +ĠChrist mas +Ġr ail +Ġco oper +Ġtest ed +ĠC apt +as ks +Ġcomfort able +Ġdel ivered +sc ape +Ġdep th +ĠG OP +Ġwrit es +Ġass ets +Ġsa v +im ents +Ġtrans ition +Ġart ist +ĠL ook +Ġl ob +Ġcomp onents +ar ity +Ġwalk ed +Ġro ot +Ġparticip ants +Ġnot iced +Ġres c +Ġn av +ĠAd minist +d a +ut ral +pl ate +Ġimport ance +Ġass ert +ious ly +c ription +Ġinj uries +ĠChe ck +Ġregist ered +Ġint ent +Ġmiss ed +ograph ic +Ġsent ence +oun ter +Ġassist ance +ev in +Ġdat abase +Ġbuild ings +Ġclass ic +Ġth inks +ĠOh io +P r +ug g +Ġfe e +p an +Ġeffect ively +Ġfac ility +Ġbe ar +Ġch apter +Ġdog s +ĠCol umb +Ġl atter +it ial +Ġad mitted +T V +ĠGe org +Ġpost s +\ \ +Ġlawy er +Ġequ ival +Ġm and +Ġcontro lled +ĠW alk +ĠAnd rew +Ġmen u +am ental +Ġprotect ed +v a +Ġadminist r +or al +Ġre in +ĠS ar +Ġamount s +Ġn ative +ĠM oon +Ġrep resents +Ġab andon +Ġcarry ing +Ġt ank +m ary +Ġdecl ared +T ube +Ġh at +Ġpun ish +el lect +m es +Ġun iverse +ĠR od +ph y +Ġinf rastructure +Ġ5 1 +Ġopp osed +ow nt +c a +ĠM ake +Ġhard ware +Ġco ffee +R el +b al +w orld +ĠS af +ĠSe a +in als +Ġown ed +Ġh all +ers ion +Ġdescrib e +ĠP ot +Ġport ion +Ġat mosp +Ġgovern ments +Ġdep ending +Ġoff ense +Ġtr ick +aw a +ĠL ine +ĠV is +ĠH ard +ĠOr ig +ĠCl ick +Ġdes k +ĠVal ley +ĠS ov +Ġmov ies +Ġrem ark +Ġm ail +Ġcons cious +Ġrul ing +ĠR ights +Ġmed ic +he nt +ĠW omen +> < +Ġrepl aced +ĠP rem +ĠTh anks +Ġre new +ĠB all +if orm +Ġsh ots +C omm +Ġar med +Ġconst ant +Ġt aste +Ġreal ized +Ġbu ff +Ġm o +Ġeffic ient +M ost +or ation +if ies +Ġcommun ication +Ġfl ood +Ġconsequ ences +Ġany way +ig g +ĠG M +ĠTh ank +Ġ iron +Ġev olution +ĠC op +tw itter +Ġ9 5 +Ġrelationship s +ad el +ĠYou ng +Ġpropos al +ay ers +uild ing +ĠH ot +OR E +c os +Ġcoll abor +P G +ax y +Ġknow ing +Ġsupport s +ow ed +Ġcontrol s +Ġmere ly +um er +Ġath let +Ġf ashion +p ath +Ġg ift +Ġer a +AN D +Ġkind s +ĠKore an +Ġleg it +ul ous +Ġess entially +Ġthe rap +n ic +Ġsuff ered +Ġh ur +Ġprom ise +Ġex cess +Ġover w +Ġpr ime +ĠH ouston +er ry +ĠM s +R S +201 2 +Ġst ores +ĠO lymp +Ġj ourney +Al though +S ub +ĠE duc +ĠCh apter +Ġrequest s +Ġconsum ers +Ġt iny +Ġis ol +ĠF air +b a +ĠY OU +Ġcr ash +ce ler +Ġemot ional +Ġgood s +Ġelect ed +Ġmod er +ĠLin ux +Ġbl ocks +Ġis land +ĠSoc iety +Ġelect ions +Ġbroad cast +Ġche ap +Ġn ations +Ġse asons +4 00 +Ġwas te +ĠS at +Ġfield s +em ploy +Ġprof ile +Ġauth ors +AL L +ĠG ra +w est +ĠT y +Ġdeath s +Ġv acc +Ġfor med +Ġd u +Ġon going +ĠMuslim s +el f +ig ure +Ġass ume +ĠUkrain e +w ater +Ġco ast +Ġvot ed +g or +ĠA S +ĠMich igan +az a +ĠAr m +i ro +Ġf lex +as ters +' ' +Ġwel come +ar l +Ġloc ations +ig ation +ĠF il +Ġbu ying +Ġarch itect +Ġhard er +ĠC ub +Ġinter face +Ġrestaur ant +Ġdisco ver +Ġex ceed +Ġfav our +ger y +Ġd uty +Ġp itch +ad or +ĠM ach +b oy +Ġrespond ed +Ġext ended +her s +M any +ra id +if er +ĠIn s +S er +Ġmed ium +s he +ĠS ports +Ġmag azine +ut ation +Ġlim its +ĠG all +Ġex ternal +raz il +Ġyoung er +t le +Ġrem ind +ĠC ON +Ġimmedi ate +Ġh idden +Ġvol unte +Ġsim pl +od cast +Ġph ase +d r +Ġpl ot +Ġexp osure +R I +og rap +v in +an ish +ĠAc ad +ĠEng ine +Ġexp ansion +ĠP ay +Y our +Ġpus hed +ĠE ll +ĠHe ad +Ġmarket ing +ĠA C +k et +Ġh its +Ġg ro +ĠA ge +ĠSc ot +] [ +Ġst im +Ġi Phone +Ī Ĵ +Ġn arrow +ĠGet ty +ĠTur key +Ġperfect ly +Ġen able +ut ch +Ġprec ise +Ġreg ime +Ġsh if +Ġcomp ens +g un +d iv +Ġch osen +ĠK en +An y +Ġtre es +Ġrecomm ended +ĠR en +u able +ĠH T +F ollow +E G +ĠH and +ĠK enn +Ġarg uments +Ġex ists +Ġb ike +ĠCons erv +Ġbre aking +ĠG ar +Ġc razy +Ġvirt ual +ay lor +ix el +Ġ19 80 +Ġper mission +ĠSer ies +Ġconsum er +Ġclose ly +c alled +Ġ5 4 +Ġhop es +Ġar ray +ĠW in +ĠLab our +Ġsp ons +ĠI re +Ġp ow +Ġread ers +Ġemploy ment +Ġcreat ure +Ġresult ing +Ġaccur ate +Ġmom ents +Ġarg ued +Ġp ed +D uring +Ġ5 3 +ĠT al +Ġs ought +Ġsuff ering +Ġ icon +le e +Ġ( $ +al ian + ° +Ġp ra +Ġbon us +( " +k o +Ġact ing +D E +f all +Ġcompar ison +Ġsm ooth +ĠN AS +u pp +ĠJose ph +ep ing +ĠT ake +ĠM id +Ġs ending +f ast +ĠF all +Ġdeal ing +us er +ĠOr gan +C o +Ġatt ached +Ġse es +% . +Ġtyp ical +AR T +Ġfind s +ĠAs ia +um in +ĠC ore +ĠE nt +in ent +u ce +ĠBl ood +ĠN ever +Ġem ails +Ġhigh light +Ġconf ront +at us +ut ed +Ġun us +Ġtop ic +ĠAd am +Ġb le +at i +Ġunder stood +S et +st ruct +T P +Ġm ob +a a +ĠSt art +pect ed +se ll +Ġded icated +ĠC A +u an +Ġsong s +esc ription +Ġte ch +Ġr ape +Ġas ide +Ġgr ant +Ġ5 6 +s ub +Ġarg ue +Ġcont aining +Ġsche dule +Ġliber al +Ġpublic ly +Ġheav ily +ĠU t +in er +ĠS ection +ĠC are +we et +l s +D is +âĶ Ģ +ĠF ollow +B ack +ĠI T +Ġb es +j i +ĠH it +est ed +Ġevery body +ĠSw ed +Ġfem in +Ġfac ilities +Ġcon ven +C omp +ĠO S +c ore +Ġan x +Ġdiv ision +ĠC am +ĠSt an +m ates +Ġexpl ore +pl om +Ġsh ares +pl oad +an es +Ġide al +et ers +ĠB ase +Ġpl astic +Ġdist inct +ĠNet work +ĠSe attle +Ġtrad ing +ens us +int end +Ġex hib +Ġinit ially +ĠF ood +Ġthous and +ĠBus iness +act er +Ġpar agraph +Ġrough ly +Ġw ww +Ġcreat ive +ĠCon f +Ġconsum ption +Ġfil ms +ag an +Ġob tain +Ġt all +Ġt or +Ġacknow led +Ġg rown +al o +K E +Ġ4 00 +end ers +t aining +U G +Ġsu icide +Ġwat ched +ĠL ist +al i +re hens +Ġsurround ing +Ġp ip +Ġf lying +ĠJ ava +ord an +Ġserv ing +in ations +p ost +Ġsh o +A v +Ġj ail +z y +Ġ199 9 +Ġ< / +Ġliter ally +ĠS ir +Ġexp osed +Ġl ies +st ar +Ġb at +Ġear ned +ĠD ig +Ġspec ified +ĠSe ason +Ġdeg rees +Don ald +Ġcent re +Ġsh aring +Ġwin ter +ĠC O +C he +Ġ Î +M P +Ġun w +Ġfew er +ĠM ir +Ġsomew here +ĠK ey +Ġattack ed +ĠK ir +Ġdom ain +Ġstrong er +Ġ9 9 +Ġpen alty +I d +Sc ript +Ġdecl ined +Ġne ck +Ġfra ud +Ġcur rency +Ġr ising +R C +âĢ¦ âĢ¦ +H z +Ġt ab +Ġtal ent +n am +ĠN BA +Ġvill age +Ġleg s +ĠN ext +E d +Ġac id +Ġhy d +8 00 +Ġinvol ving +ĠIm age +ĠBe fore +F l +Ġyes terday +S ource +Ġterror ist +Ġsu p +Ġsy nt +ĠSaud i +Ġw est +Ġr u +b urg +Ġvis ible +Ġstru ck +r ison +Ġaw esome +Ġd rawn +Ġansw ers +ĠG irl +ĠR am +Ġthreat s +Ġdef eat +os it +Ġv ent +atur ally +Americ an +end a +ĠH oly +Ġr um +% , +c ase +ĠHist ory +ĠYou Tube +Ġsit uations +ĠD NA +S te +Ġsa ved +It em +Ġrec ip +olog ist +Ġfac ed +Ġel ig +O nce +ĠL i +u h +Ġmist ake +ĠDiv ision +ĠB ell +Ġsympt oms + ® +Ġdom in +Ġfall ing +Ġend ing +as hes +Ġmat ches +ĠOn line +Ġexplan ation +D ef +red it +Ġany more +ĠT otal +ĠF OR +us hed +Ġlet ters +Ġris ks +ĠO K +Ġreported ly +: \ +Ġpl ate +Ġsubject s +Ġattempt ed +if ier +ian a +Ġunlike ly +ĠTh ough +um a +ĠIn vest +ĠPr in +ic an +ĠD ar +ĠColor ado +au g +Ġve get +a os +ri a +Ġshe l +Ġmark ed +Ġ( ) +Ġsp r +p o +ĠL ink +Ġdef e +ĠJ r +Ġthem e +Ġpass ion +ĠP en +Ġinf o +iz er +Ġsh it +ĠC ivil +ap se +c re +Ġpo ly +Ġcomp onent +ĠChar les +ĠIre land +ĠPro v +Ġdo ctors +Ġgr anted +Ġpain t +Ġhon or +Ġsm oke +Ġpay ments +Ġprim arily +ĠKing dom +r ich +ate ll +Ġde als +Ġsched uled +Ġfund amental +Ġprote in +Ġnewsp aper +Ġcl ients +yth on +ĠD ate +h us +Ġfeed back +Ġstret ch +Ġc ock +Ġhot el +ĠQue en +Ġsu gar +Ġj u +Ġmil k +Ġappro val +ĠL ive +Ġequival ent +ef ully +Ġins ert +z ona +Ġext ension +d ri +J ohn +Ġacc omp +S m +ĠF und +Ġconst antly +Ġ` ` +Ġgener ated +ĠA ction +ĠP sych +ĠT ri +Ġrecogn ize +Ġv ary +ph a +ĠR a +d f +et ch +ĠSov iet +Tw o +Ġpattern s +Ġprof ession +an ing +T ime +ĠL im +Ġcol ors +ĠA z +ĠT R +Ġinf ect +Ġphen omen +Ġshe ll +Al so +Ġput s +Ġdel ivery +Ġbro wn +Ġprocess ing +Ġlight s +ess age +ĠBro ok +ĠA ud +l ation +Ġindust rial +L ike +ĠB razil +rou s +ES S +ĠL uc +Ġsome how +Ġ8 5 +Ġpro port +Ġpolit icians +Ġindic ate +Ġh ole +Ġtechn iques +Ġcompet itive +Ġph r +Ġv o +ist ent +ĠD ream +Ġcamp us +Ġaspect s +Ġhelp ful +Ġsh ield +or se +Ġtrig ger +m al +Ġ5 8 +Ġt ort +Ġperson ally +Ġt ag +Ġkeep s +ĠV ideo +Ġben ch +Ġg ap +a ire +Ġe ast +Ġrec overy +per ial +Ġprof it +ĠM ic +Ġ5 7 +Ġcol on +Ġstrong ly +st yle +Ġalleg ations +h an +Ġrep orters +j o +r ine +arg et +and al +Ġ0 3 +Ġfl ash +tr ans +Ġstr ict +Ġpark ing +ĠPak istan +Ġl i +Ġwe ird +ĠE ric +Ġreg ions +ĠJ un +Ġint ellect +ĠW H +od ing +rib utes +up id +ĠT it +Ġf inger +or ia +Ġe lev +ĠF ield +Ġcon clusion +; ; +Ġfeel ings +Ġext ensive +Ġm ixed +Ġne uro +v y +Ġhar ass +ĠC irc +ou ch +Ġterrit ory +Ġsuccess fully +M ar +Ġing red +Ġoverw hel +Ġl ayer +V iew +Ġall ies +ill ance +ĠTh ree +Ġb unch +Ġnorm ally +Ġnet works +Ġsac r +ĠC IA +b les +Ġch ose +Ġopp onents +Ġregard less +Ġfr anch +Ġpre f +ĠP o +Ġbr idge +ann a +ĠSil ver +Ġw age +p age +ri or +Ġrad ical +ĠL ittle +Ġman ip +Ġsecret ary +Ġg ang +D R +F A +Ġdec ent +ĠSp irit +Ġun cle +ĠDevelop ment +Ġinvest ors +Ġwall s +Ġpub lish +Ġgener ate +iss ions +c ar +Ġprom ote +Ġcut ting +Ġche st +Ġdrink ing +Ġcollect ed +Ġ7 2 +Ġhop ing +Ġem br +gor ith +Ġwar ned +Ġinstruct ions +O G +ĠD id +ĠAg ency +Ġg ear +Ġcritic ism +ĠF urther +Ġut il +ann y +R ed +Ġcoun sel +ĠAs ian +Ġredu ction +p ool +Ġteach ing +Ġdeep ly +i y +Ġestim ates +Ġcho ices +Ġperman ent +in em +ke l +Ġf asc +p se +f ile +ĠL ow +ĠP erson +Ġt ournament +st al +Ġm el +U ST +ĠR ay +az i +V al +Ġcont ained +ĠH olly +Ġw ake +Ġreve al +Ġprocess es +ĠIS IS +Ġ0 9 +Ġbl ind +Ġste el +ĠB ad +Ġcare fully +app y +ro it +Ġg aming +Ġhous es +ĠC oll +Ġtr uck +er m +Ġsc ored +Ġocc as +ret urn +b ound +v ar +Ġsh arp +Ġaf raid +ĠE X +am ber +c ific +Ġsche me +N C +ĠPol it +Ġdecl ine +Ġ199 8 +Ġpus hing +Ġposs ession +Ġpriv ile +Ġteacher s +Ġy ield +H A +ĠDav is +it led +#### #### +Ġr ig +ĠD aniel +ac on +Ġh ide +ut en +Ġcolle agues +Ġprin ciples +Ġl oud +Ġs in +ĠDem on +Ġst one +Ġ0 2 +Ġt aught +Ġter rible +Ġst uck +ĠPol icy +te en +Ġimplement ation +ĠB BC +ĠAP I +Ġwhe el +all as +Ġch ampions +ol ars +play er +Ġrepeated ly +ĠSt ill +Ġlik es +ast y +es ter +ĠCath olic +R L +Ġb ath +Ġno ise +t itle +Ġn orthern +P art +Ġmag n +Ġf ab +ĠAs h +Ġdis pl +Ġtick et +Ġm urd +Ġalong side +ĠMus ic +Ġr iver +ĠSte el +ĠC L +ĠPl ayer +ĠM ult +ow ing +re p +s ize +Ġt ur +ĠGeorg ia +isc al +ra ction +Ġc able +Ġ5 9 +Ġw ins +Ġup coming +Ġsurv ive +Ġins pired +ĠEduc ation +Ġstat istics +ĠF oot +iam i +Ġy ellow +ĠP age +. - +ĠH as +Ġur ban +Ġa x +es sel +\ " +Ġquarter back +Ġreg ister +ĠLab or +Ġab ilities +ĠF amily +Ġvar iable +ĠPr ice +Ġcont em +Ġth in +ĠE qu +d ata +Ġg otten +Ġconst it +Ġas ks +Ġt ail +Ġexc iting +ĠE ffect +ĠSp anish +Ġencour age +ins on +ĠA h +Ġcommit ment +C S +Ġr ally +Ġ: : +Ġsubs id +Ġsp in +Ġcapt ured +201 8 +Ġinn oc +Ġalleged ly +ĠC ome +Ġart ists +ĠN umber +Ġelect ronic +Ġreg ional +ap es +Ġw ra +Ġmy th +pr ise +ĠM iller +ĠC reat +ĠEp isode +b ell +Ġdirect ed +Ġext ract +Ġs orry +Ġv ice +ag ger +ĠSu pport +Ġ6 6 +ĠI ron +Ġwonder ful +Ġg ra +N et +ion e +E ng +Ġsh ips +ik es +ĠK evin +it ar +Ġactiv ists +tr ue +ĠAri zona +ent h +ĠDes pite +ĠS E +Ġha bit +ern el +Ġin qu +Ġab ortion +Ġv oid +Ġexpl icit +Ġeng aged +Ġang ry +Ġr ating +Ġfr ag +b ro +ick ing +d ev +Ġwor ried +Ġob ser +Ġap artment +ĠG T +Ġest ate +ĠConst itution +em on +ĠS now +Ġcount y +Ġdis ag +ĠStep hen +Ġimm igrants +w ind +ĠN ations +Ġfol ks +O ut +Ġg all +Ġtarget ed +Ġst ead +ĠB on +ĠL ib +Ġinform ed +Ġ12 0 +ch ain +idel ines +or ough +Ġdri ven +Ġregular ly +Ġbas ket +Ġprinc iple +oc ument +Ġst un +ib ilities +ĠRom an +ĠAb out +Ġal ert +Ġdemocr acy +Ġrepresent ed +H S +c ers +p arent +Ar t +p ack +Ġdi plom +re ts +ĠN O +Ġcapt ure +ĠAd v +Ħ ¢ +Ġannounce ment +ĠL ear +Ġh ook +Ġpur s +ĠS uch +ĠC amer +Ġrefuge es +ĠV e +P ol +Ġrecogn ized +l ib +Ġhad n +A ss +Ġpil ot +us hing +Ġreturn ing +Ġtra il +ĠSt one +Ġrout ine +Ġcour ts +Ġdes per +Ġfriend ly +ĠIt aly +Ġpl ed +Ġbreat h +Ġstud io +N S +Ġimp ressive +ĠAfghan istan +Ġf ing +Ġd ownt +ink ing +ĠR og +i ary +col or +se x +ar on +Ġf ault +ĠN ick +D own +ĠR ose +ĠS outhern +X X +is odes +L ist +6 00 +Ġout come +er r +Ġelse where +Ġret ire +Ġp ounds +ĠGl obal +Pe ople +Ġcommun ications +Ġlo an +Ġrat io +ĠEm pire +Ġg onna +Ġinv ent +D F +Ġ19 70 +ĠComm on +p at +Ġprom ised +Ġd inner +ĠH om +Ġcreat es +Ġoper ate +ver ty +ĠJ ordan +et ime +Ġsust ain +R eg +Ġincred ible +im a +Ġwar rant +Ġm m +A tt +Ġlaw suit +Ġreview s +it ure +ĠS ource +l ights +ĠF ord +Ġ6 3 +g roup +st ore +Ġfeat ured +Ġfore ver +Ġpo verty +ĠP op +ĠC NN +az z +ab is +ach ing +Ġl aid +ĠSu pp +Ġfil ter +en a +ĠCommun ity +Ġcreat ures +u ction +ĠR oyal +Ġassoci ation +ĠCon nect +ĠBr ad +âĸ Ī +l ers +the re +ĠG i +Ġval uable +AC K +ĠT aylor +Ġl iquid +ĠAtt orney +ĠCar l +ĠF inal +ag a +ĠWil son +B ecause +ĠProf essor +ak a +Ġincred ibly +r ance +! ) +R ef +s k +Ġsol utions +Ġatmosp here +Ġbl ame +um es +ĠN ob +C A +um ps +r ical +ĠPut in +ĠD est +or ic +ĠP A +Ġrespect ively +w an +Ġfif th +â Ħ¢ +ĠC ry +Ġgovern or +res ident +Ġpurch ased +Ġh ack +Ġint ense +ob s +Ġorig in +Ġdef ine +Ġcare ful +** * +Ġshould er +Cl ick +Ġt ied +Ġdest ruction +ou red +Ġno body +Ġh o +ĠEx per +Ġt ip +" ; +Ġtechn ique +Ġj ur +ĠP ok +b ow +Ġleg end +Ġacc ord +Ġbus y +ĠInt el +Ġh ang +ak i +. ] +âĢĶâĢĶ âĢĶâĢĶ +Ġsur gery +Ġrep rodu +Ġun iform +Ġscen es +c ode +Ġ6 2 +l isher +ĠH ave +ph ia +Ġcry pt +Ġrec on +Ġsc ream +Ġadop ted +Ġsc ores +N e +ĠIt alian +in cluding +B O +Ġindic ated +Ġent ertain +G u +T ext +i el +Ġtw enty +Ġeng age +off s +ĠPac ific +Ġsm ile +Ġperson nel +Ġto ler +Ġdo ors +Ġt one +Ġmach ines +Ġent ering +ten ance +C O +ĠJer sey +Ġfore st +Ġhor se +Ġcompl aint +ĠSpr ing +y o +ĠPl us +ed ing +ĠRet urn +qu arters +ial s +c ow +Ġacad emic +Ġf ruit +Ġ199 6 +og ether +Ġw ine +Ġpur su +ĠSte ven +Ġlic ens +Wh o +Ġclot hes +re ction +Ġsqu ad +Ġst able +Ġr aw +z ens +St ar +ut ies +anc er +Ġke ys +ĠM u +Ġcompl icated +ig er +ĠTe xt +Ġabs or +Ġ6 8 +Ġfun ny +Ġrel ief +ĠL ew +ĠC ook +Ġch art +Ġdraw ing +G E +Ġmod ule +ĠB ull +I LL +Ġs alt +0000 0000 +il le +Ġres ource +aw ay +adel phia +ĠB ru +Ġ6 7 +Ġsome body +Ġparticip ate +Ġro se +we red +Ġmus cle +Ġcons ent +Ġcontin uing +ĠGuard ian +ĠOr der +reg on +Ġre ar +Ġprov ision +Ġlik ed +ri ent +Ġb ra +Tr ans +Ġmeet ings +Ġto x +Ġcon vent +Ġaut o +Ġrec ording +ĠSo ft +00 1 +ĠR oll +Ġprogram ming +Ġp ic +Ġprov ed +Ġst ab +ĠA st +Ġca ption +ul ating +ĠAtt ack +Ġnew ly +Ġ199 7 +f r +Ġdis cipl +ĠGree k +Ġed ition +ĠDo es +ĠB ox +if le +ack et +Ġpass es +Ġgu est +Ġac celer +it als +U D +Ġaut hent +ĠR est +ov al +t a +u ine +Ġarm or +ĠT own +Ġcomp at +Ġinc hes +Des pite +Ġass ign +he rent +Ġprep are +ĠM eg +oc key +Ġdep ends +Ġtrack s +w atch +Ġl ists +ĠN orthern +Ġal ter +re c +ĠE astern +Ġcond em +Ġevery where +? ' +Ġaff ili +Ġf ought +": {" +Ġm ac +it arian +Ġsc ope +ĠA L +aw s +ar ms +Ġqu e +Ġenjoy ed +nes ota +Ġagg ressive +ĠSt ory +ĠI V +Ġrec ipe +Ġrare ly +ĠMed ical +val ue +ang el +ay ing +omet hing +Ġsub section +Ġs outhern +Ġfrequ ency +re te +roll ed +ult s +ĠN ic +Ġbeh alf +Ġsequ ence +ab et +Ġcontrovers ial +Ġcomp rom +Ġwork er +Ġmain ly +Ġal gorith +ĠM ajor +or ce +g ender +Ġorgan ized +Ġf ake +Ġconclud ed +ĠE D +ĠEx ec +r age +Ġch ances +ber ry +ĠTr ad +Ġconfig uration +Ġwithd raw +Ġf ro +ud es +ĠBro ther +ĠB rian +Ġtri es +Ġsam ples +Ġb id +ĠGold en +Ġphot ograph +if est +ĠD O +ĠPar liament +******** ******** +R em +Ġcont est +Ġsign ing +p x +ĠZ eal +âĶĢ âĶĢ +E ar +Ġex it +Be fore +ĠCor por +n ull +mon th +Ġrac ial +ott ed +ĠV eg +ĠRe uters +Ġsw ord +ps on +ĠRom ney +a ed +Ġt rib +Ġin ner +Ġprot ocol +ĠB i +ĠM iami +ever al +p ress +Ġsh ipping +ĠAm endment +ĠHow ard +con nect +ĠD isc +ĠJ ac +iam ond +ĠThere fore +s es +ĠPrin cess +ĠUS B +ĠAn th +Ġsurve illance +Ġap olog +Ġ6 1 +ow a +Ġf ulf +j s +Ġl uck +ust ed +Ġ § +n i +Ġant icip +em an +Ġwin ner +Ġsil ver +ll a +ic ity +Ġunus ual +Ġcr ack +Ġt ies +e z +Ġpract ical +Ġprov ince +ĠPl ace +Ġprior ity +IC E +Ġdescrib es +Ġbr anch +F orm +ask a +miss ions +b i +Ġp orn +ĠTur k +Ġent hus +Ġf ighters +Ġ0 8 +ĠDet roit +Ġfound ation +av id +A re +Ġjud gment +cl ing +Ġsol ve +ĠDes ign +W here +hes is +ĠT ro +a fter +Ġne utral +ĠPalestin ian +ĠHolly wood +Ġadv is +ĠN on +y es +ol is +Ġrep utation +Ġsm ell +Ġb read +ĠB ul +ĠBe ach +Ġclaim ing +Ġgen etic +Ġtechn ologies +Ġupgr ade +row s +Ġdevelop er +ĠJ osh +ĠDis ney +erv ed +ip al +Ġun ex +Ġbare ly +t hen +ĠP ub +Ġill ness +et ary +ĠB al +Ġp atch +Ġbut t +Ġst upid +ĠD og +ĠD allas +f ront +ie ce +Ġprot ests +Ġch at +oen ix +Ġw ing +Ġpar liament +Ġ7 7 +ose xual +Ġre nder +pt ions +ĠCo ast +os a +ĠG reg +h op +ĠMan agement +Ġbit coin +Ġrec over +Ġincor por +or ne +ĠUs ing +Ġpre ced +Ġthreat ened +Ġspirit ual +ĠE vent +ĠF red +Ġadvert ising +Ġimprove ments +ĠC ustom +Ġer rors +Ġsens itive +ĠN avy +Ġcre am +L ook +Ġex clusive +Ġcomp rehens +Ġde leg +Ġcon ce +Ġrem em +Ġstruct ures +Ġst ored +N D +Ġ1 000 +U P +ĠB udd +A F +w oman +ĠAcad emy +ð Ł +se a +Ġtem porary +Ab out +es ters +Ġtick ets +Ġposs ess +in ch +o z +Ġl a +Ġcontract s +Ġun p +Ġc ig +ĠK at +ult ural +as m +Ġmount ain +ĠCapt ain +St ep +m aking +ĠSp ain +Ġequ ally +Ġl ands +at ers +Ġreject ed +er a +im m +ri x +C D +Ġtrans action +g ener +less ly +Ġ| | +Ġc os +ĠHen ry +Ġprov isions +Ġg ained +Ġdirect ory +Ġra ising +ĠS ep +ol en +ond er +Ġcon sole +in st +Ġb om +Ġunc ertain +1 50 +ock ing +Ġmeas ured +Ġpl ain +Ġse ats +Ġd ict +S L +af e +Ġest imate +iz on +at hered +Ġcontribut ed +Ġep isodes +omm od +G r +AN T +Ġ6 9 +G ener +Ġ2 50 +vious ly +rog en +Ġterror ism +Ġmove ments +ent le +oun ce +ĠS oul +Ġpre v +ĠT able +act s +ri ors +t ab +Ġsuff er +Ġn erv +Ġmain stream +ĠW olf +Ġfranch ise +b at +Ġdem ands +Ġag enda +Ġdo zen +Ġclin ical +iz ard +ĠO p +t d +Ġvis ited +ĠPer haps +Ġact or +Ġde lic +Ġcont ribute +Ġin ject +ĠE s +ac co +Ġlist ening +Ġcon gress +epend ent +Ġprem ium +Ġ7 6 +ĠIr ish +Ġass igned +ĠPh ys +Ġworld wide +Ġnarr ative +ot ype +m ont +b ase +ĠB owl +ĠAdminist ration +Ġrel ation +ĠE V +C P +Ġco vers +Ġ7 8 +Ġcert ific +Ġgr ass +Ġ0 4 +pir acy +ir a +Ġengine ering +ĠM ars +Ġun employ +ĠFore ign +st ract +Ġv en +Ġst eal +Ġrepl ied +Ġult imate +Ġtit les +d ated +Ġj oy +a us +Ġhy per +ak u +Ġoffic ially +ĠPro duct +Ġdifficult y +per or +Ġresult ed +rib ed +l ink +wh o +~~ ~~ +ĠSpe ed +ĠV iet +W ind +ĠBar ack +Ġrestrict ions +ĠSh are +Ġ199 5 +ition ally +Ġbeaut y +op t +Ġm aps +ĠC R +ĠN ation +ĠCru z +W ill +Ġelectric ity +Ġor g +Ġb urd +Ġviol ation +Ġus age +Ġper mit +ĠCh ron +ĠF ant +Ġn aturally +Ġ0 7 +Ġth rown +ĠAw oken +Ġal ien +ĠHer o +ĠK ent +ĠR ick +ri ke +Ġp ace +}, {" +G L +Ġpo ison +ĠT ower +Ġform al +al ysis +Ġgen uine +Ġk il +a ver +Ġproced ure +ĠPro p +intend o +ĠM ain +as ant +Ġtr ained +G ame +ĠL oad +ĠM A +Ġcru cial +Ġle ts +ĠF R +Ġch ampion +1 01 +ĠCon ference +Ġwrit ers +Ġconnect ions +Ġo kay +ir ms +ĠR and +Ġenc ounter +ĠB uff +Ġachie ved +Ġche cks +isc ons +Ġassist ant +Ġwhen ever +ĠA ccess +ĠU r +b in +Ġcl ock +is p +op her +Ġb orrow +Ġm ad +Ġperson ality +on ly +IS T +ab ama +Ġg ains +Ġcommon ly +Ġter r +Ġhyp ot +Ġre ly +Ġt iss +iscons in +Ġrid ic +f unction +ĠO regon +Ġun com +r ating +el and +ĠN C +Ġm oon +ann on +Ġvulner able +ut ive +³³ ³³ +ĠRad io +Ġw estern +se ct +ĠT ony +Ġocc urs +ĠO s +ĠH on +Ã Ń +Ġv essel +ĠScot land +Ġdiscrim ination +Ġsubsequ ent +st ring +Ġfant asy +ĠSh adow +Ġtest im +W E +it i +r as +Ġbo at +Ġmar ks +Ġord inary +Ġre n +Ġrepresent ative +Ġpet ition +Ġ7 3 +Ġad venture +Ġign ore +ĠPhil adelphia +ĠS av +V P +Ġfact ory +Ġt asks +Ġdep ression +z ed +................ ................ +ĠSt orm +Ġc ogn +Ġelig ible +Ġredu cing +v ia +Ġ0 5 +Ġstri king +Ġdoll ar +h o +O V +Ġinstr ument +Ġphilosoph y +ĠMo ore +ĠA venue +Ġrul ed +ĠFr ont +IN E +ĠM ah +Ġscen ario +ĠNAS A +Ġen orm +Ġdeb ut +Ġte a +T oday +Ġabs ence +S im +Ġh am +le ep +Ġt ables +ĠHe art +M I +K e +re qu +V D +m ap +Ġchair man +Ġp ump +Ġrapid ly +v i +Ġsubstant ial +E P +d es +ch ant +ili pp +ĠS anta +ri ers +anche ster +L oad +ĠC ase +Ġsa ving +Ġ7 4 +ĠA FP +er ning +oun ced +ĠMin nesota +ĠW as +Ġrec ru +Ġassess ment +ĠB ron +U E +Ġdynam ic +Ġf urn +ul ator +Ġprop ag +h igh +Ġacc ommod +Ġst ack +ĠS us +w rit +Ġre ven +ĠGod d +ĠZeal and +ab s +Ġbr ut +Ġper pet +h ot +Ġhard ly +ĠB urn +ãĤ ¹ +Ġst y +Ġtrans actions +Ġg ate +Ġsc reens +Ġsub mitted +Ġ1 01 +Ġlangu ages +ugh t +em en +Ġfall s +Ġc oc +Ĥ ¬ +Ġstri kes +p a +Ġdel iber +ĠI M +Ġrel ax +ann els +ĠSen ator +Ġext rem +Ġ} , +ĠDe b +Ġbe ll +Ġdis order +c ut +Ġi OS +Ġl ocked +Ġem issions +Ġshort ly +" ] +ĠJud ge +ĠS ometimes +Ġr ival +Ġd ust +Ġreach ing +F ile +¯¯ ¯¯ +ino is +ĠJ ason +Ġs atell +are t +Ġst ations +Ġag ric +ĠTechn ology +com es +ĠUn fortunately +ĠChild ren +Ġappl ies +ast ed +Ġan ger +ail ability +ĠDam age +Ġcomp are +ĠStand ard +Ġaim ed +ĠB a +angu age +Ġreg ulation +Ġj ury +Ġair port +Ġse ctions +ĠPr ince +em ed +Ġmedic ine +Ġh itting +Ġsp ark +ol ves +Ġad s +St ate +Ġfood s +Ġrepl acement +Ġch icken +Ġlow est +Ġmind s +Ġinvol ves +u i +Ġarr ang +Ġproced ures +ĠWh ich +ivers ary +Ġb ills +Ġimprove ment +Ġin ev +Ġexpect ations +Ġintellect ual +Ġsp aces +Ġmechan ism +2 50 +bre ak +ĠZ e +ĠT enn +ĠB alt +Ġbar rel +Ġstat ic +man n +Pol ice +Ġt ips +Ġhand ling +c us +od ed +il ton +ir y +Ġjournal ists +our se +Ġcom ic +Ġnom ine +IT Y +Ġvers us +Ġlo op +Ġsur f +ĠInd ust +ĠHun ter +Ġbelief s +is an +Ġset up +Ġbre w +im age +Ġcomput ers +f ol +} ," +ĠMed al +Ġtax p +Ġdisplay ed +Ġg rav +Ġf iscal +M on +ĠMos cow +ĠK ong +ĠCent re +Ġcamer as +ĠMr s +ĠH ay +Ġa ver +ĠK elly +p y +Ġrequire ment +Ġent itled +omb ie +Ġsh adow +ag ic +ĠA k +Ġel ite +Ġdiv ided +Ġhead ing +Ġcop ies +Ġloss es +Ġv it +k ed +ĠB ry +Ġan s +ĠSte am +Ġrep orter +he im +ĠIt em +Ġsuper ior +d on +ere nt +à ¶ +Ġtherap y +Ġpe ak +ĠMod el +Ġl ying +Ġg am +z er +r itten +Ġrespons es +Ġconsider ation +ĠB ible +Ġl oyal +Ġinst ant +Ġp m +ĠFore st +à ¼ +Ġext end +Ġconv icted +Ġfound er +Ġconv in +ĠO ak +che ck +Ġsch olars +p ed +Ġover se +T op +c ount +ĠAr k + · +Ġ0 6 +ĠL A +m d +ĠLat in +im ental +ĠC PU +Ġsubst ance +Ġminor ity +Ġmanufact uring +E r +ocol ate +Ġatt ended +ĠMan ager +r ations +Ġappreci ate +om y +GB T +id ency +B L +Ġguarant ee +pos ition +Ġo cean +clud e +Ġhead ed +Ġt ape +Ġlo ose +Ġlog ic +Ġpro ven +Ġsp ir +Ġad mit +is a +Ġinvestig ate +Ġ199 4 +sy lv +ĠL ost +c est +Ġ7 1 +Ġrequest ed +Ġwind ows +ĠPok é +ĠWith out +M et +Ġbehavi our +Ġread er +Ġh ung +ĠKe ep +Ġro les +Ġimplement ed +Ġbl ank +Ġserv es +ĠJ ay +Ġc ited +ĠF riend +prof it +ap on +Ġrep air +it em +arr ass +Ġcrit ics +ad i +ĠF ather +Ġsh out +Ġf ool +Ġ8 8 +Ġprodu cing +Ġl ib +Ġround s +Ġcirc le +Ġpre par +Ġsub mit +Ġn ic +mor row +ãĥ « +U nder +Ġv ital +ater n +Ġpass word +Ġpublic ation +Ġprom inent +Ġspeak s +Ġb ars +Ġde eper +ĠM ill +port ed +Ġw id +Ġbut ter +Ġsm oking +Ġindic ates +K ey +rop ri +ĠF ile +all ing +ast ing +ĠR us +Ġad j +Ġ7 9 +av al +Ġpres um +bur gh +on ic +Ġf ur +Ġpoll s +ik a +Ġsecond ary +Ġmon ster +ig s +ĠCur rent +E vent +Ġowners hip +end ar +Ġarri ve +ĠT ax +Ġn ull +ĠPri v +Ġth ro +Ġk iss +c at +Ġup set +ang le +it ches +ect or +olog ists +ĠGal axy +Ġcor ruption +Ġh int +ent er +ĠH ospital +Ġgreat ly +Ġbeg un +es y +Ġso il +ĠAnt on +Ġmain tenance +ãĥ © +Ġdo zens +Ġhuman ity +ĠAl abama +Ġr om +w orth +ap ing +sylv ania +l ah +Ġg athered +G A +Ġattack ing +f ound +ĠSqu are +Ġar bit +ict ions +ĠW isconsin +Ġd ance +ĠS aint +arch y +Ġbase ball +Ġcontribut ions +Ġliter ature +Ġex ha +per ty +t est +Ġb ab +Ġcontain er +let ter +Ġfall en +Ġwebs ites +Ġbott le +ĠS ac +Ġbre ast +ĠP L +Ġveter an +Ġinterview s +ĠA le +Ġb anned +eng ers +ĠRev olution +in th +Ġconc erning +IV E +Ġexp enses +ĠMatt hew +ĠColumb ia +d s +ist ance +Ġent ity +.. ." +Ġrel iable +Ġpar alle +ĠChrist ians +Ġopin ions +Ġin du +l ow +Ġcompet e +Ġth orough +Ġemploy ed +Ġestablish ment +ig en +ĠC ro +Ġlawy ers +ĠSt ation +T E +ĠL ind +ĠP ur +it ary +Ġeffic iency +âĢ IJ +ĠL y +Ġm ask +Ġdis aster +Ġag es +ER E +es is +ĠH old +Ġcas ual +b led +Ġen abled +ĠEn vironment +ĠInt elligence +i per +ĠM ap +ĠB E +Ġemer ged +is dom +Ġc abin +Ġregist ration +Ġfing ers +Ġro ster +Ġfram ework +ĠDo ctor +et ts +Ġtransport ation +Ġaware ness +H er +Ġattempt ing +O ff +ĠSt ore +ÃĥÃĤÃĥÃĤ ÃĥÃĤÃĥÃĤ +ĠK now +Ġdef ence +Ġsc an +ĠT en +ĠCh air +ĠP H +ĠAtl anta +Ġfuck ing +Ġans wered +b n +ĠK ar +Ġcateg ories +Ġr ational +Ġc ust +Ġrob ot +Ġcorrect ly +Ġg if +Ġgraph ics +m ic +Ġground s +ĠO pp +i ate +Ġdist ributed +Ġsan ctions +Ġchalleng ing +ut o +Ġingred ients +Ġinv ited +Ġfound ed +ĠRe qu +d ed +Ġb owl +Ġbrother s +ĠH a +I O +Ġw ages +im ore +oc ial +Ġse ed +ative ly +Ġaddress es +ĠI owa +ab eth +Ġatt itude +is d +ch ild +Ġm ole +Ġdisco very +y ard +B r +Ġ8 2 +Ġsuppl ies +ell ing +Ġdist ingu +C R +Ġre cept +Ġ vert +Ġsw im +b ec +d oor +ĠY eah +Ġg al +Ġinter act +ĠE SP +ĠC S +amp s +Ġconvin ced +Ġobject ive +Ġdis h +ĠPhot os +l ad +Ġdownt own +o il +in ction +Ġto morrow +ĠC OM +Ġsurv ival +sh ot +Ġsett lement +C ons +ĠX box +int erest +ĠS M +arg o +en ess +Ġeth nic +b ered +M in +ĠT ok +Ġinc ent +ĠComm and +Ġmain tained +Ġbreak s +br idge +at ar +ag g +ĠF inally +un icip +ĠO nt +le ft +Ġrecogn ition +Ġ* / +ĠP ers +Ġwe lf +Ġaddress ed +ĠK ansas +Ġvir us +Ġwhere as +Ġp apers +ram s +ĠMin istry +Ġple asure +Ġacqu ired +Ġd uration +j pg +Ġcal m +ĠN HL +Ġburn ing +Ġfold er +ick ed +ĠP y +ĠIll inois +Cl ass +ĠGodd ess +Ġperform ing +Ġwelf are +j ar +In ter +Ġl in +Ġenh ance +Ġnot ion +f are +yp es +ĠAre a +Ġcann abis +ĠDie go +f s +ĠM anchester +com m +in ite +Ġcover ing +ĠS ound +Ġ19 60 +Ġ8 4 +e lect +z ing +Ġcitiz en +Ġph ones +Ġr aid +Ġign ored +ĠOb ject +Ġu pload +c ard +Ġmod ified +Ġroom s +ia h +r ange +he ast +ach us +Ġsuggest ing +âĢ ĭ +gr ade +E l +Ġclot hing +Ġr h +ĠH an +un ity +en cing +ĠAust in +sec ution +t ra +d em +ĠQ ual +Ġhe aven +Ġst ages +Ġw edd +pl us +ific ial +ĠIm m +ĠH o +iet ies +Ġphr ase +Ġbr ill +act ory +Ġprov iders +Ġsil ence +Ġa er +ĠA I +ĠAd venture +Ġplatform s +Ġdemonstr ated +Ġinter f +ing ton +Ġr aces +Ġgr ade +ult ane +ĠTh rough +f alse +Ġb ow +ĠA B +Ġfl avor +Ġhistor ic +g ov +Ġcol our +Ġview ed +ĠEm ail +el come +Ġinter vention +Ġd iversity +Ġperiod s +Ġre verse +ĠV ery +Ġqu ote +ĠLe ft +th rough +Ġsc rew +Ġland ing +Ġp ill +Ġw et +Ġprot esters +Ġrepe at +av ed +er k +Ġsal ary +ĠPenn sylvania +St ill +Ġmay or +Ġkit chen +Ġfeat uring +ĠM useum +ĠT ournament +ĠF al +Ġser vers +U C +Ġany body +im g +ĠTr ade +ixt ure +the less +Ġfin ance +Ġcl osing +ĠPat ri +i ac +ab el +Ġ> > +or ous +Ġf irms +sc reen +un a +Ġemb arrass +ul se +Ġlet ting +Ġth rew +ile y +Ġch annels +l an +ĠVeg as +Ġse ar +Ġfant astic +ar re +uzz le +ĠD er +Th ose +Ġsw ing +Ġshe et +ind ex +co ver +og an +Ġvari ables +ĠTe ch +Ġsp oken +ac hel +ĠD a +ĠMount ain +Ġload ed +Ġfoot age +vers ion +Ġun l +ĠPh oenix +Ġthrow ing +Ġf iring +Ġtrack ing +Ġw idth +Ġstrugg ling +ro oms +ot ion +Ġmonth ly +ĠSer ver +Ġegg s +op en +M C +Ġ199 3 +Ġh ired +Ġstay ed +ĠAll en +Ġst ro +Ġ9 8 +st ep +ĠTurk ish +Ġfab ric +ist ing +ĠD om +Ġd ates +Ġpr on +Ġbasket ball +Ġl ucky +ĠArab ia +Ġassum ed +est y +Ġaff airs +Ġgl ad +ĠInd eed +ĠF A +ĠW ord +Ġjo ining +if ice +p read +ir ts +ĠSe lect +Ġpop ulations +aw are +Ġn ose +Ġcompl aints +st art +Ġsc oring +Th anks +Ġmin ing +Ġvisit ors +S H +Ġdam aged +Ġcharacter istics +ĠP ent +D C +Ġ8 3 +ĠS ix +r ates +Ġfl ags +ĠB rew +d og +M ark +// // +Ġexec ution +Ġj oke +ph ones +Ġtestim ony +Ġob st +Q L +ĠC ut +Ġstud ied +ĠN intendo +ick et +ĠN BC +Ġl ad +ĠB ra +ĠM oh +Ġk ernel +Ġoverwhel ming +Ġag ed +Ġapplic able +ĠC ond +Ġroad s +ĠBl ock +m ade +od ge +Ġcomm ands +Ġoff ices +vel and +Ġt ut +Ġrece iver +ĠF ro +Ġsho pping +Ġi P +ĠSt re +ĠA BC +Ġentertain ment +ĠB ow +ort ed +M c +Ġread s +gr ad +ĠCol lect +Ġâ ĪĴ +ĠCap ital +eder ation +Ġemploy er +Ġinvolve ment +Ġanx iety +al ia +Ġro of +ĠAm ong +ĠDemocr at +Ġstat s +ĠV ill +Ġconst itutional +Ġrefer ring +itt y +Ġtack le +out ube +Ġback ed +ĠH ong +ĠBro ad +Ġe le +ĠO tt +Ġ199 2 +h our +achus etts +C al +Ġdefe ated +Ġ8 1 +es p +Ġseem ingly +w as +ĠJ enn +ĠK urd +Ġg ene +Ġdisc ount +R et +EC T +( ); +Ġclub s +Ġs id +ĠM arsh +Che ck +Ġp p +ĠE ag +ides pread +Ġbe ings +F T +Ġintrodu ction +ĠCh ange +AR D +Ġ1 10 +ad ows +ier ce +Ġme al +a uthor +ĠB ang +lah oma +Ġr anks +201 1 +?? ?? +m ax +Ġcoll apse +Ġop ens +Ġe cho +Ġs oph +Ġrac ist +Ġenorm ous +Ġw aves +Ġt ap +Ġcomprehens ive +. -- +ĠR oy +Ġfarm ers +Rel ated +a ired +ron es +ĠC rim +Ġproport ion +Ġdesign s +Ġnegoti ations +Ġvirt ually +ĠBat man +Ġwar n +Ġlegit imate +m ate +Ġcon vention +, , +net ic +ĠS D +Ġconsist ently +Ġcompens ation +Ġpunish ment +Ġy e +Ġt ie +ĠB ureau +ir lf +ĠB u +ĠA ren +ĠPh ilipp +Ġkn ife +Ġmem ories +ĠR oss +Ġang le +Ġ8 6 +ĠTh under +Ġre nd +ĠT our +Ġcount s +s ung +ĠIm p +Ġeduc ational +Ġaccess ible +C OM +Ġd rew +y er +G l +am ine +OR T +O B +I B +m aster +Ġtri als +og y +h ar +ĠTr ust +Ġprefer red +irlf riend +ĠN ev +Ġb in +Ġc ow +P age +Ġsign ature +ĠB L +7 00 +Ġret ired +Ġby tes +Ġneigh b +ĠLeg end +Ġdev ast +Ġsuspect ed +is ons +ĠPoké mon +sc ale +Ġcap abilities +Ġre vel +Ġche ese +d y +igr ant +Ġfail ing +b its +ĠHer oes +ĠG host +ĠS cient +Ġappoint ed +ur i +Ġinst itution +Ġexpand ed +g reg +Ġmonitor ing +Ġp odcast +Ġcoal ition +Ġ9 6 +J o +Ġst olen +ĠS ab +Ġstop s +Ġhol iday +Ġint r +C ar +Bl ack +ĠL GBT +Ġwar ming +ĠAnd erson +Ġ8 9 +Ġprodu cer +M ed +Ġaccur acy +ĠMar vel +iz abeth +ĠPat rick +m ony +Ġmin i +ac les +Ġover t +the y +Ġmembers hip +ĠV en +Ġex ch +Ġrem oval +ĠD ave +T Y +m ad +ĠF ind +Ġad equ +Ġe c +Ġte eth +Ġemot ion +Ġper m +Ġsole ly +d b +Ġextra ord +IG HT +c al +Ġgu idelines +Ġd ying +Ġsusp ended +ĠPrem ier +ĠAnth ony +el ve +Ġd ad +ĠE th +ĠFoot ball +Ġabandon ed +Ġ< < +Ġm arch +Ġhor ror +âĢ¦ " +Ġchild hood +Ġcampaign s +Ġl unch +ĠAl bert +bl ock +âĸĪ âĸĪ +ound ing +Ġb one +or gan +ad ers +ĠFl ash +ĠDri ve +Ġton ight +Ġw ars +ĠF L +Ġform ation +con st +New s +Ġcom pe +or ious +ĠSt aff +Ġdiscuss ions +ĠProt ection +ĠJ am +Ġcrit eria +Ġinstall ation +Ġaccompl ish +iz za +Ġpub lisher +Ġresc ue +ĠT ry +U LL +ĠS om +ĠH op +ore t +th s +ord on +Ġp ocket +ĠIn v +Down load +ĠCr ime +Ġb ene +ĠGu ide +ĠAs sembly +Ġparam eters +I E +ĠAlex ander +Ġconc ert +ĠSc he +Ġsh oes +Ġvis iting +Ġrec all +Ġb ub +Ġr ural +Ġconc rete +ĠR os +N ext +R uss +Ġlo ans +ĠSh ield +Ġtre m +hem at +k g +ĠHar ris +is ition +ĠM ove +ĠF C +Ġf ate +ĠCh o +Ġt ired +Ġprinc ipal +h ist +ien ces +ath y +Ġse vent +Ġm ood +Ġstrateg ic +Ġdise ases +Ġfor um +Ġtem por +Ġhead quarters +P ar +ig e +fl ix +Ġgu itar +Ġ9 4 +On ly +Ġrele ases +ro ph +================ ================ +Ġ6 00 +ĠContin ue +ig ate +ĠC rit +sy stem +Ġdis abled +Ġunex pected +ith ub +Ġuncle ar +ĠE st +Ġcontr ad +Ġstrateg ies +vent ures +Ġpass age +AM E +Ġimpro ving +Ġreve als +Ġdecre ase +ov a +Ġann oy +ĠSh ort +ĠL ibrary +Ġcy ber +n ell +ĠH ur +ĠC B +Ġphot ograp +U I +Ġs ed +G e +Ġ8 7 +Ġd iverse +Ġencour aged +Ġcons piracy +Ġbird s +Ġoper ator +Ġhand ful +Ġclass ified +? ) +Ġdram atic +Ġinvestig ators +it o +Ġw idespread +ĠR oom +-------------------------------- -------------------------------- +Ġcollect ive +Ġjournal ist +St ring +Ġtemper atures +il a +Ġgu id +Ġins pect +Ġmiss ile +ĠMay or +Ġman ual +Ġsim ultane +Ġrat ings +Ġsu ck +Ġ9 7 +Ġunivers al +Ġph arm +Ġdis rupt +ian o +A V +Ġf t +Ġstat ist +old s +ĠWalk er +ph p +Ġunder t +ĠL as +ish op +nt il +res hold +ĠWhe ther +M s +Ġden y +ĠCl oud +Ġprov ider +Ġsurv iv +ĠUp date +h as +Ġmist akes +ch arge +pl ed +r ity +Ġn ode +ĠMass achusetts +ool s +lic ation +Ġf ails +em ale +or i +back s +Ġsh irt +Ġ' ' +ĠN AT +Ġwat ers +els on +Ġe ase +Ġsc ar +Ġcont ents +m ind +Ġcont ribution +Ġsh r +Ġhand ed +Ġst ability +Ġtra ve +E m +Ġmir ror +12 3 +Ġwe igh +Ġf iction +ou ver +ist ant +r ition +ĠF ed +Ġphys ically +Ġst ake +ĠArt icle +ĠAr c +ĠLew is +ĠM ind +Ġdemonstr ate +Ġprof its +v ision +om ic +ol id +Ġbatt les +Ġdri ves +Ġeas tern +ĠS ony +!! ! +ar ation +v ard +ĠG L +port ation +Ġ9 2 +Ġlaw makers +Ġprotect ing +ĠE PA +Ġy eah +Ġsh ame +ol ph +e ven +x it +Ġatt ach +Ġrepresent ing +Ġob s +ĠUt ah +iff s +ĠFre edom +à ³ +A K +Ġinc idents +it age +Ġview ers +c d +Ġm ouse +Ġcl ar +Ġaccord ance +Ġb ot +c or +ĠSum mer +he ld +Ġinnoc ent +Ġiniti ative +ol s +________________ ________________ +Ġsp ots +p ace +Ġconvent ional +Ġcorpor ations +Ġblock ed +H D +at tered +Ġref ers +Ġbu ck +ĠDig ital +12 0 +Ġtop ics +T F +Ä ģ +br id +re ement +Ġunder lying +ĠM ember +Ġinvestig ating +Ġpregn ancy +Ġtouch down +ĠB and +ĠCall er +Ġinst ances +P P +w a +G ood +Ġ199 1 +ĠC old +Ġfear s +Ġrem arks +Ĩ Ĵ +at al +Ġm it +Ġexper iments +i pt +Col or +ind u +Up date +Ġ9 3 +A g +Ġ å +anc ouver +B oth +Ġjud ges +Ob ject +Ġst ere +umb n +Ġparticip ation +ĠSt ars +ĠJ ere +Ġweek ly +ĠB an +Ġconvers ations +ĠP itt +u z +ĠIndian a +ĠK ick +Ġinf ection +Ġhero es +Ġsett led +Ġstri p +Ġh al +Ġd ump +ĠS ci +Ġl es +Ġref erences +ĠU RL +ĠBr idge +Ġwant ing +For ce +Ġex clus +Me anwhile +m n +Ġg entle +m aker +sen al +ĠG ro +ou ri +ĠR ain +ĠAll iance +Ġl ift +el a +S D +ĠCle veland +Ġrank ed +Ġst adium +Ġdead ly +ä ¸ +Ġr iding +ar ia +ĠAr mor +Ġdocument ation +ĠGree ce +ree k +Ġl ens +ĠS a +Ġg ross +ĠE mer +ag ers +ĠD ub +ĠR h +ĠAM D +Ġarri val +Ġdes ert +Ġsupp lement +ĠRes p +Ġkn ee +Ġmarg in +f ont +og g +201 0 +ĠP ir +ĠP rom +iv als +Ġint ake +Ġdifferent ly +ug s +Ġb its +clud ed +Ġsearch ing +ĠD u +um ble +Ġfunction al +ĠBalt imore +ĠC ould +Ġdes ired +Ġcirc uit +ĠL yn +ĠG O +ĠF alse +re pre +' : +alt ies +Ġmin im +Ġdro ve +ĠSh ould +Ġh ip +Ġpro s +Ġut ility +ĠN ature +ĠM ode +P resident +o pp +r at +form ance +Ġconcent ration +Ġf ont +ĠB ud +Ġam id +Ġre vers +ĠM L +B ar +Ġinter action +Ġjur isd +Ġspell s +d ep +f il +Ġcivil ians +ut ter +ĠCo oper +ĠBel ow +Ġent rance +Ġcon vert +Ġcontrovers y +ow ered +Ġcontr ary +Ġar c +ĠExec utive +ĠOffic er +Ġpack ages +Ġprog ressive +w idth +Ġreserv ed +v ol +ĠSam sung +Ġprint ed +Ġcent ers +Ġintrodu ce +ĠKenn edy +Ġodd s +Ġsure ly +Ġindepend ence +Ġpass engers +repre ne +ĠBe h +Ġl oves +ĠESP N +Ġfac ilit +Ġident ical +Ġdo ct +Ġpartners hip +con f +ĠH ide +Ġconf used +ĠC ow +M en +Ġw rest +ĠIraq i +Ġh oles +ĠStud ies +Ġpregn ant +h ard +Ġsign als +I X +Ġpull ing +Ġgrad uate +Ġnomine e +D ate +Ġper mitted +Ġâ Ĥ¬ +ĠOk lahoma +St art +Ġauthor ized +Ġal arm +ĠC os +v an +Ġgener ations +c ular +Ġdr agon +ĠSoft ware +ĠEd ward +Ġcontro ller +S en +ge red +ĠV ik +Ġappro ached +Th ank +Ġcan ce +Ġform ula +ĠSm all +Ġweak ness +Ġr amp +it udes +j ud +Ġbrill iant +Ġacc us +s ource +Ġ8 00 +ĠE vil +S w +Ġhom eless +we ek +i ens +r ics +ĠTh ird +T O +Ġorgan ic +Ġpresent ation +ag h +ĠDown load +v ation +Ġas sembly +or able +hold ers +ĠBern ie +ĠHel p +Ġt ong +ĠF ight +Ġbe ach +B ook +ĠL ic +Ġr ush +ĠR ound +ou p +ĠMar x +Ġcalcul ated +ĠDe vil +ĠSar ah +Ġoccasion ally +Ġbul let +Av ailable +g ate +Ġ9 1 +Ġh osp +Ġprom ises +ĠH IV +ĠSt adium +ĠSt ock +ĠCorpor ation +g age +N G +ĠC redit +Ġs ne +ib l +Ġacc um +s uch +Ġterror ists +Ġconscious ness +ĠZ h +Ġdram a +ool a +pir ation +Ġlab our +ĠN in +Ġut ter +Ġdemocr atic +Ġass ass +il ation +Ġg est +Ġab road +Ġmet ab +Ġs orts +Ġfl av +U B +Ġm g +ĠNot hing +ĠO d +Ġmus ical +200 9 +Ġdro ps +oc ated +ater al +0000 00 +Ġg re +Ġequ ality +Ġburd en +Ġv ig +ĠLe ader +-------- ---- +Ġcere mony +Ġf ighter +Ġact ors +Ġ æ +am an +F i +Ġal ign +put er +Ġe lder +ĠN SA +Ġrepresent ation +ĠOnt ario +IT H +usal em +Ġharass ment +itz er +Ġsy mp +Ġbox es +ĠD R +Ġman ifest +at re +Ġ ^ +Ġd ies +le ton +Ġmiss ions +et he +Ġres olve +Ġfollow ers +Ġas c +Ġk m +l ord +am med +Ġsil ent +ĠAssoci ated +Ġtim ing +Ġprison ers +ĠK ings +ĠF ive +Ġtow er +Ġappro aches +Ġprecise ly +Ġb ureau +ĠM other +ĠI ss +Ġkey board +it ual +Ġfund ed +Ġstay ing +Ġpsych ological +Ġm ile +ĠLe on +ĠBar b +w ill +Ġw ider +ĠAtl antic +Ġt ill +ĠR ome +ro t +Ġaccomp an +Ġfl our +ac o +W orld +ĠExp ress +ĠY u +C or +Ġple ased +part y +Ġpoint ing +Ġinf lation +Ġro y +Ġ ), +ain er +Ġwedd ing +orm on +Ġrequ iring +Ġqual ified +Ġse gment +EN D +Ġs izes +e als +Ġcor rupt +ass ador +Ġcele b +Ġdream s +ĠM ess +Ġcheck ing +ĠV ersion +Ġprep aring +Ġact ively +ĠD iff +Ġl ux +ĠW inter +act eria +ĠN E +Ġdep uty +Ġtrans gender +Ġsum mary +Ġin her +er ies +ch ar +ĠY an +Ġkn ock +ĠP ath +Ġl ip +roll er +Ġimp ression +Ġcelebr ate +Ġsl ide +Ġgu ests +Ġcl ip +F S +Ġsav ings +Ġcapt ain +Ġleg acy +ĠDen ver +Ġw ounded +tab oola +AC T +Ġpurs ue +Ġo xy +Ġ q +Ġsem i +ĠN eed +ĠAff airs +Ġob sc +Ġcheck ed +Ġd ual +C ode +ĠM D +le m +ult y +Ġ © +ĠEl izabeth +Ġcent uries +ard ed +s rc +Ġev ident +enn is +at in +Ġunemploy ment +ĠMar io +Ġint im +Ch rist +Ġbi ological +Ġsold ier +ĠAdd ed +Ġm ath +ĠG il +Ġbi as +Ġd ating +ĠO cean +Ġm ice +M us +h ire +ĠT es +Ser ver +lim ited +S ize +Ġmet ers +Ġrock et +es see +Ġcertific ate +ĠIran ian +AS S +Ġgr id +D ec +Ġro lling +com mun +ĠSwed en +b ury +Ġtiss ue +Ġrac ism +ĠL ocal +Ġmyster y +Ġexam ine +Ġst em +Ġs its +Ġhop ed +ot ing +Ġdial ogue +Ġpers u +W atch +l ay +M AN +Ġch ronic +ĠPort land +mark et +ĠS EC +Ġparalle l +Ġsc andal +Ġcar ries +Ġphenomen on +h uman +ack er +ĠO x +Ġretire ment +tain ment +ov ie +ĠG ear +Ġd uties +Ġdo se +Ġsc roll +M B +in f +Ġsa uce +Ġland scape +red dit +ĠChampions hip +ĠRed dit +al id +Ġco in +Ġover s +Ġpost ing +ab out +Ġf el +and y +Ġb old +Ġfocus ing +e ffect +G R +Ġde emed +Ġrecommend ations +Ġste pped +Ġvot er +ĠDe ep +ĠInst agram +Ġmoder ate +ĠMary land +Ġrestrict ed +ĠM B +ĠCh all +Ġto b +Ġc ir +ĠO cc +ĠE ver +Ġcoll aps +IN FO += - +ĠP ict +ĠAcc ount +n c +Ġo ught +Ġex port +Ġdr unk +( ' +Ġw ise +ĠM ort +ne cess +Ġan cest +ĠInc re +Ġfrequ ent +m ir +Ġinterpret ation +Ġdepend ent +Ġco ins +ĠB ol +V ideo +ĠJust in +Ġfat al +Ġcook ing +Ġconf usion +ip her +Ġcust ody +ĠMor gan +om ach +ĠGovern or +Ġrestaur ants +el ing +Ġacknowled ged +Ġthe r +Ġgen es +ch ing +He y +Ġtact ics +ĠMex ican +Ġv end +Ġhe s +qu er +Ġnot ing +ĠCamer on +Ġtarget ing +ro ck +Ġcred its +Ġemot ions +Ġrepresent atives +new s +Ġlegisl ative +Ġrem oving +Ġtweet ed +ĠCar ter +ĠF ixed +Ġfor cing +Ġspeak er +Ġm ales +ĠViet nam +l ined +Ġconcept s +Ġvo ices +o ir +ĠT rib +W he +ĠJer usalem +ĠS ant +Ġc ul +Ġl ady +ĠHaw ai +Ġar ts +ĠIn n +ĠMach ine +ĠEm peror +Ġsl ot +g ly +ĠPro cess +II I +Ġathlet es +ĠTem ple +ĠRep resent +Ġpres c +Ġt ons +Ġgold en +Ġp unch +ĠG R +iver pool +Ġen act +Ġlob by +Ġm os +Ġpick ing +Ġlif etime +Ġcogn itive +E ach +z o +Ġd ub +Ġcons ists +ol n +Ġf estival +am ous +Ġint ellig +w ords +ĠSm art +Ġde le +Ġl apt +Ġmag ical +ĠS in +b us +ur ities +igh th +ĠRub y +ĠS ure +ol ving +Ġj un +O ST +Ġimp osed +Ġast ron +Ġcor rel +ĠN S +ĠK it +ĠF uture +b urn +Ġimm une +oc us +Ġcour ses +ĠSt ring +Ġle an +Ġg host +Ġout comes +Ġexp ense +Ġevery day +Ġaccept able +A h +Ġequ ipped +Ġor ange +F R +ĠD utch +Th ough +ĠR ank +Q U +ĠRober ts +wh at +re nd +Ġdisapp ear +Ġsp awn +ĠL am +o is +Ġdes erve +Ġmin imal +Ġnerv ous +ĠW ould +Ġro ok +ĠV ancouver +Ġres ign +sh ire +ĠW orks +ĠB uild +Ġafford able +ĠG ary +ĠAren a +Ġh anging +Ġimpl ications +ĠS ong +Ġmain taining +Ġgu ards +C ON +Ġder ived +Ġexecut ed +Ġthe ories +Ġqu oted +ĠAnd re +og a +sel ess +in fo +ĠBel g +Ġt ears +ĠSur v +Ġbirth day +ig ious +im mer +Ġspect rum +Ġarchitect ure +Ġrec ruit +arm a +T able +Ġmon sters +ĠG ov +Ġdest ination +Ġattract ive +Ġf oss +ĠMore over +Ġpres ents +TH E +Ġrep ly +pt on +Ġc um +Ġdel ight +Ġaffect s +Ġdon ations +ĠT oy +ĠH im +M ENT +Ġover come +it ched +ĠFant asy +ĠH at +ĠBe ast +b ott +Ġinvestig ations +R un +Ġhun ting +d i +f und +Ġs essions +est yle +Ġport ray +oid s +Y eah +Ġcommun icate +Ġcom edy +ĠY ang +Ġbel t +ĠMar ine +Ġpredict ed +Pl ay +Ġimportant ly +Ġremark able +Ġelim inate +D avid +Ġb ind +V ID +Ġadvoc ates +ĠG aza +im p +D B +ĠN a +ĠSim ilar +I ES +Ġchar ity +v as +m ath +Ġâ ĸ +ok er +nd um +Ġcap s +ĠH al +2 000 +e an +Ġfle et +Ġrec re +R ight +Ġsleep ing +ij ing +k ind +Ġdesign ated +à ¤ +Ġanim ation +ke e +ĠInt rodu +Ġ/ > +Ġdelay ed +Ġtrem end +Ġcur ious +U se +Ġle ct +d am +Ġinnov ation +ĠPoint s +Ġload ing +Ġdisp ute +ct ic +ird s +ĠB Y +Ġn urs +ĠVal ue +ION S +ĠH um +Ġtem plate +m ers +Ġappear ances +ĠEnter tainment +Ġtransl ation +Ġsa ke +Ġbene ath +Ġin hib +Ġe uro +abet es +Ġstud ying +ĠM as +Ġper ceived +Ġexam ined +Ġe ager +Ġco aches +Ġim per +ch i +Ġprodu ces +" ). +ĠEvery one +Ġm unicip +Ġg irlfriend +Ġh ire +ĠV ice +Ġsu itable +op y +Ġin equ +ĠD uke +f ish +f irst +ĠO bs +Ġinter ior +ĠBru ce +ĠR y +Ġanal ys +Ġconsider able +Ġfore cast +Ġf ert +ors hip +ĠD rug +ĠA LL +: " +th ur +ĠM ail +Ġball ot +Ġinst antly +ĠCh annel +Ġp icks +Ġ198 9 +Ġt ent +ol i +Ġcivil ian +b ling +ell o +b u +Ġin ch +Ġlog o +Ġcooper ation +Ġwal ks +Ġinvest ments +Ġimp rison +ĠF estival +ĠK y +Ġleg ally +Ġg ri +ch arg +S l +Ġthreat ening +du ction +fl ow +Ġdismiss ed +ibr aries +c ap +e le +ĠMc G +ĠHar vard +ĠConserv ative +ĠC BS +p ng +Ġro ots +ĠH aving +umb led +ĠF un +\ / +ĠS earch +ple x +Ġdiscuss ing +Ġcontin u +ĠT ai +ĠW ik +F ree +f it +Ġref use +Ġmanag ing +Ġsy nd +ip edia +w alk +Ġprofession als +Ġguid ance +Ġunivers ities +Ġas semb +unt u +F inally +AS E +ĠAut o +ĠH ad +Ġann iversary +L D +ĠD ur +ĠUlt imate +ih ad +pro duct +Ġtrans it +Ġrest ore +Ġexpl aining +Ġass et +Ġtransfer red +Ġbur st +ap olis +ĠMag azine +ĠC ra +ĠB R +gg ed +ĠH E +M ich +b et +ĠL ady +yl um +erv es +Ġme ets +wh ite +L og +Ġcorrespond ing +Ġins isted +G G +Ġsurround ed +Ġt ens +Ġl ane +Ġco inc +h ome +Ġexist ed +ect ed +ĠDou ble +lam m +Ġske pt +ex p +Ġper ception +ie v +ĠBe ing +o ft +Ġadop t +. : +] ; +Wind ows +Ġsatell ite +AS H +Ġinf ant +d escription +ĠMe anwhile +c m +oc a +ĠT reat +act or +Ġtob acco +ĠN orm +em ption +Ġfl esh +Ġj e +o op +ĠHe aven +Ġbe ating +an im +Ġgather ing +Ġcult iv +G O +ab e +ĠJon athan +ĠSaf ety +Ġbad ly +pro t +Ġcho osing +Ġcontact ed +Ġqu it +Ġdist ur +Ġst ir +Ġto ken +D et +ĠP a +Ġfunction ality +00 3 +s ome +Ġlimit ations +Ġmet h +b uild +con fig +N T +re ll +ble m +ĠM om +Ġveter ans +ĠH u +Ġtrend s +are r +ĠG iven +ĠCa ption +m ay +AS T +Ġwond ering +ĠCl ark +n ormal +Ġsepar ated +Ġdes p +st ic +b rew +Ġrel ating +ĠN ik +ĠF arm +Ġenthus i +g ood +d eb +Ġactiv ist +Ġm art +Ġexplos ion +ĠEconom ic +L ink +Ġins ight +Ġconven ient +Ġcounter part +su pport +ĠV irt +ag en +ĠTenn essee +ĠSim on +ĠA ward +OC K +ĠF igure +Ġoverse as +Ġpr ide +ĠC as +n ote +m g +C urrent +Ġdispl ays +cont ent +Ġtravel ing +Ġhosp itals +ĠFin ancial +ĠP ast +Ġdefend ant +Ġstream ing +m ble +ĠBer lin +uk i +Ġdist ribut +Ġant ib +Ġch ocolate +ĠCast le +Ġinter rupt +ĠR ow +Ġconvers ion +Ġbug s +ĠR ather +li est +L Y +ĠJe an +com mon +ak h +Ġ1 30 +ot ton +ĠDe an +Ġam endment +Ġgame play +ĠWar ren +od a +Ġhigh lights +Ġir re +ĠNAT O +Ġball s +Ġdemand ing +U RE +ĠL uke +F igure +st op +on ia +z one +iz ers +ĠW R +Ġaward ed +Ġregul atory +ĠH art +ĠS N +pl ing +Ġs our +ĠP ixel +us ive +Ġf et +ĠS ent +Ġautom atic +Ġf er +vern ment +ĠKh an +T ON +f ather +Ġextraord inary +th rop +ĠP ython +ĠG PU +Ġsex ually +Ġdesk top +it ivity +ĠAnton io +Ġo rient +Ġe ars +ob by +ous es +vertis ements +Ġmanufacture rs +ic ient +min ute +Ġconv iction +Ġg arden +p ublic +Ġsatisf ied +f old +O K +Ġin hab +ĠTh ink +Ġprogram me +Ġst omach +Ġcoord in +Ġh oly +Ġth reshold +Ġr het +Ġser ial +Ġemploy ers +ĠEvery thing +ra h +Ġb other +Ġbr ands +Val ue +ĠT ed +ĠPlan et +Ġp ink +ĠFurther more +s a +P E +re ck +ĠUS D +ot te +Ġ& & +Ġland ed +g ets +Ġprodu cers +Ġhealth care +Ġdomin ant +Ġdest ro +Ġam ended +ch ron +Ġf its +ĠSy d +ĠAuthor ity +AT CH +Ġfight s +ĠL LC +Ġ-- - +ĠCor p +Ġtox ic +spe cific +ĠC orn +ĠChe l +Ġtele phone +ĠP ant +Ġmyster ious +aun ch +od ox +med ia +Ġwitness es +ag u +Ġquestion ed +ĠBre xit +ĠRem ember +ene z +Ġend orse +iat ric +ĠId ent +Ġridic ulous +1 10 +Ġpr ayer +Ġscient ist +Ġ19 50 +ĠA qu +Ġunder ground +ĠU FC +m are +ĠL ater +w ich +Ġsubsc rib +Ġhost s +Ġer r +Ġgr ants +ant om +Ġsum mon +ear ly +ĠC lear +ĠPr im +Ġsusp ension +Ġguarant eed +app er +Ġr ice +ĠSe an +ĠSh in +Ġrefere ndum +Ġfl ed +r ust +Ġ3 60 +ter y +Ġsh ocked +B R +ĠO il +ĠAll ah +Ġpart ly +Ġign or +Ġtrans mission +Ġhom osexual +ivers al +Ġhop efully +ãĤ ¤ +Ġless on +L eg +Ġ .. +Y et +t able +app ropri +re tt +Ġbo ards +Ġincor rect +Ġb acteria +ar u +am ac +Ġsn ap +.' " +Ġpar ad +t em +he art +Ġav ailability +Ġw isdom +Ġ( + +Ġpri est +ĠÂł ĠÂł +O pen +Ġsp an +Ġparam eter +Ġconv ince +Ġ( %) +r ac +Ġf o +Ġsafe ly +Ġconver ted +ĠOlymp ic +Ġres erve +Ġhe aling +ĠM ine +M ax +Ġin herent +ĠGra ham +Ġinteg rated +D em +Ġpip eline +Ġapp lying +Ġem bed +ĠCharl ie +Ġc ave +200 8 +Ġcons ensus +Ġre wards +P al +ĠHT ML +Ġpopular ity +look ing +ĠSw ord +ĠAr ts +' ) +Ġelect ron +clus ions +Ġinteg rity +Ġexclus ively +Ġgr ace +Ġtort ure +Ġburn ed +tw o +Ġ18 0 +P rodu +Ġent reprene +raph ics +Ġg ym +ric ane +ĠT am +Ġadministr ative +Ġmanufacture r +Ġ vel +ĠN i +Ġisol ated +ĠMedic ine +Ġback up +Ġpromot ing +Ġcommand er +Ġfle e +ĠRus sell +Ġforg otten +ĠMiss ouri +Ġres idence +m ons +Ġrese mb +Ġw and +Ġmeaning ful +P T +Ġb ol +Ġhe lic +Ġwealth y +Ġr ifle +str ong +row ing +pl an +as ury +âĢ¦ . +Ġexpand ing +ĠHam ilton +Ġrece ives +S I +eat ures +ĠAn im +RE E +P ut +Ġbrief ly +ri ve +Ġstim ul +Ġ`` ( +Ġ __ +Ġch ip +Ġha z +Ġpri ze +ĠTh ings +AC E +ul in +d ict +ok u +Ġassoci ate +ock ets +y outube +St ory +ateg ory +Ġm ild +ail ing +ĠY e +O rig +ĠK a +or ig +Ġpropag anda +Ġan onymous +Ġstrugg led +Ġout rage +AT ED +ĠBe ijing +r ary +Ġle ather +Ġworld s +Ġbroad er +12 5 +id al +ĠBet ter +Ġt ear +E xt +Ġpropos als +Ġit er +ĠSqu ad +Ġvol unt +m i +D id +ĠP u +p in +Ġspeak ers +Ġb orders +Ġfig ured += ' +Ġsimultane ously +aed a +Ġcharg ing +Ġur ged +Ġcon j +25 6 +ĠG ordon +mer ce +Ġdocument ary +Sh are +it ol +ON E +ĠG arden +h att +ĠThom pson +ane ous +ap ore +Ġt anks +Ġless ons +tr ack +Ġout standing +Ġvolunte ers +Ġsp ray +Ġmanag ers +l arge +Ġcamp s +Ġart ificial +ĠR u +Ġb ags +th al +Ġcompat ible +ĠBl ade +Ġf ed +Ġarg ues +F I +Ġunf air +Ġcor n +Ġoff set +Ġdirect ions +Ġdisappoint ed +ĠCon vention +Ġview ing +M E +oc ity +Ġtown s +Ġlay ers +Ġro lled +Ġjump ed +Ġatt ribute +Ġun necess +inc oln +Ġsupp ose +ĠNet her +ch a +Ġbur ied +Ġsix th +B en +ress ing +OU R +Ġw ound +Ġcy cl +Ġmechan isms +Ġcongress ional +ĠE lement +Ġagre ements +Ġdec or +Ġclos est +ĠM it +Go ogle +} } +Ġm ixture +Ġflu id +S ign +ĠSch olar +Ġp ist +ask et +ab ling +Ġrac ing +he ro +ri el +ass y +Ġche aper +b en +Ġvert ical +amac are +ĠRead ing +g ments +Ġhelic op +Ġsacr ifice +ay a +p aren +V A +ĠL es +ĠStud io +Ġviol ations +ĠAn na +ac er +é ¾ +ĠR at +ĠBe ck +ĠD ick +ĠA CT +Ġcomp osition +Ġtext ure +ĠO wn +Ġsmart phone +ĠN A +Ġfor b +im port +Ġdef ending +il st +re r +Ġo h +ĠJere my +Ġbank ing +cept ions +Ġrespect ive +/ . +Ġdr inks +ĠW i +Ġb ands +ĠL iverpool +Ġg rip +ĠB uy +Ġopen ly +Ġreview ed +per t +Ġver ify +ĠCo le +ĠW ales +M O +Ġun pre +Ġshel ter +ĠIm perial +Ġgu i +ĠD ak +Ġsuggest ions +Ġexplicit ly +Ġsl ave +Ġblock chain +Ġcompet ing +Ġprom ising +S ON +Ġsoc cer +Ġconst itution +4 29 +Ġdist ract +ĠU ser +es ides +ĠMet hod +ĠTok yo +Ġaccompan ied +Cl ient +s ur +al og +Ġident ification +Ġinv asion +as ma +Ġindust ries +pp ers +Ġsub tle +ĠUn it +n atural +Ġsurv ived +Ġfl aw +ĺ ħ +ĠH oll +Ġdef icit +Ġtut orial +ĠCh ance +Ġarg uing +Ġcontem porary +Ġinteg ration +for ward +Ġt um +it is +Ġh iding +ĠD omin +ĠT an +ĠB uilding +ĠV in +Ġspokes person +ĠNot es +Ġemer ging +Ġprepar ation +Ġpro st +Ġsuspect s +Ġaut onom +D escription +Ġdeal t +ĠP ear +Ġstead y +Ġdecre ased +Ġso vere +ĠCl in +Ġgrad ually +ors es +ĠW AR +S erv +ãĤ ¢ +h r +Ġd irty +ĠB arn +ĠB C +Ġd il +Ġcal endar +Ġcompl iance +Ġch amber +b b +Ġpass enger +ate ful +ĠT itle +ĠSyd ney +ĠG ot +Ġdark ness +Ġdef ect +Ġpack ed +ass ion +Ġgod s +Ġh arsh +IC K +le ans +Ġalgorith m +Ġoxy gen +Ġvis its +Ġbl ade +Ġkil omet +ĠKent ucky +Ġkill er +P ack +enn y +Ġdiv ine +Ġnom ination +be ing +Ġeng ines +Ġc ats +Ġbuff er +ĠPh ill +Ġtra ff +AG E +Ġtong ue +Ġrad iation +ere r +m em +ĠExpl icit +é¾ į +Ġcou ples +Ġphys ics +ĠMc K +Ġpolit ically +aw ks +ĠBl oom +Ġwor ship +e ger +ut er +ĠF O +Ġmat hemat +Ġsent enced +Ġdis k +ĠM arg +Ġ/ * +P I +Ġoption al +Ġbab ies +Ġse eds +ĠScott ish +Ġth y +] ] +ĠHit ler +P H +ng th +Ġrec overed +ing e +Ġpow der +Ġl ips +Ġdesign er +Ġdis orders +Ġcour age +Ġch aos +" },{" +Ġcar rier +b ably +H igh +ĠR T +es ity +l en +Ġrout es +u ating +F il +N OT +w all +s burgh +Ġeng aging +ĠJava Script +ore r +li hood +Ġun ions +ĠF ederation +ĠTes la +Ġcomple tion +ĠT a +Ġprivile ge +ĠOr ange +Ġne ur +paren cy +Ġb ones +Ġtit led +Ġprosecut ors +ĠM E +Ġengine er +ĠUn iverse +ĠH ig +n ie +o ard +Ġheart s +ĠG re +uss ion +Ġmin istry +Ġpen et +ĠN ut +ĠO w +ĠX P +in stein +Ġbul k +S ystem +ic ism +ĠMarket able +Ġpre val +Ġpost er +Ġatt ending +ur able +Ġlicens ed +ĠG h +et ry +ĠTrad able +Ġbl ast +à ¤ +ĠTit an +ell ed +d ie +H ave +ĠFl ame +Ġprof ound +Ġparticip ating +Ġan ime +ĠE ss +Ġspec ify +Ġregard ed +ĠSpe ll +Ġs ons +own ed +Ġm erc +Ġexper imental +land o +h s +ĠDun geon +in os +Ġcomp ly +ĠSystem s +ar th +Ġse ized +l ocal +ĠGirl s +ud o +on ed +ĠF le +Ġconstruct ed +Ġhost ed +Ġsc ared +act ic +ĠIs lands +ĠM ORE +Ġbl ess +Ġblock ing +Ġch ips +Ġev ac +P s +Ġcorpor ation +Ġo x +Ġlight ing +Ġneighb ors +ĠU b +ar o +Ġbe ef +ĠU ber +F acebook +ar med +it ate +ĠR ating +ĠQu ick +Ġoccup ied +Ġaim s +ĠAdd itionally +ĠInt erest +Ġdram atically +Ġhe al +Ġpain ting +Ġengine ers +M M +ĠM ust +Ġquant ity +P aul +Ġearn ings +ĠPost s +st ra +ãĥ¼ ãĥ +Ġst ance +Ġdro pping +sc ript +Ġd ressed +M ake +Ġjust ify +ĠL td +Ġprompt ed +Ġscr ut +Ġspeed s +ĠGi ants +om er +ĠEd itor +Ġdescrib ing +ĠL ie +ment ed +Ġnow here +oc aly +Ġinst ruction +fort able +Ġent ities +Ġc m +ĠN atural +Ġinqu iry +Ġpress ed +iz ont +for ced +Ġra ises +ĠNet flix +ĠS ide +Ġout er +Ġamong st +im s +ows ki +Ġclim b +ne ver +Ġcomb ine +d ing +Ġcomp r +Ġsignific ance +Ġremem bered +ĠNev ada +ĠT el +ĠSc ar +ĠWar riors +ĠJ ane +Ġcou p +b as +Ġtermin al +, - +O H +Ġt ension +Ġw ings +ĠMy ster +�� �� +ĠUn like +val id +viron ments +ĠAl i +Ġn aked +book s +ĠM un +ĠG ulf +Ġd ensity +Ġdim in +Ġdesper ate +Ġpres idency +Ġ198 6 +h y +IN D +Ġun lock +im ens +Ġhand led +ĠE b +Ġdisapp eared +Ġgen re +Ġ198 8 +Ġdetermin ation +St ream +ik o +ap ters +Ġacknow ledge +J an +Ġcapital ism +P at +Ġ20 20 +Ġpain ful +Ġcur ve +Ġbom bs +st orm +ĠMet al +en cer +ĠF ig +ĠA aron +anc hes +Ġins piration +Ġexha ust +t ains +ash i +Ġdesc ript +Ġr itual +ĠChel sea +Ġpromot ion +ĠH ung +ĠW ard +iv a +ĠE T +Ġto ss +all ow +ĠFranc is +D ep +Ġhapp iness +ĠGl ass +Ġbet a +Ġstreng then +N E +o a +Ġbutt ons +ĠMur ray +Ġkick ed +Qu est +ĠT alk +ĠS everal +ĠZ ero +Ġdr one +ul k +Ġc am +ĠM obile +Ġprevent ing +Ġret ro +ĠA x +Ġcru el +Ġflo at +. ), +Ġfil ing +ĠGr ant +ĠB or +Ġr ib +Ġchampions hip +ĠM erc +Ġsty les +Ġc ake +Ġbuild s +ĠS elf +io x +Ġep ic +oy d +B el +ĠSt ew +. ( +ah u +ĠBe yond +Ġout s +Ġsol o +ĠT ree +Ġpres erve +Ġt ub +AR E +ro c +ĠIm pro +ĠW right +Ġbu nd +Ġtr aged +Ġoccas ional +b ian +Sec ond +r ons +Ġinter actions +form ed +s ing +Ġown s +Ġh ockey +Gener al +Ġlog ical +Ġexp end +Ġesc al +ĠGr iff +ĠC rown +ĠRes erve +Ġsto pping +Ġexc use +sec ond +Ġoper ated +Ġre aches +ĠMal ays +Ġpoll ution +ĠBrook lyn +Ġde lete +Ġhas h +Bl ock +ah a +âĢ ³ +Ġsh orter +p iece +> >> +ĠM ormon +t or +Ġpartic les +ĠB art +ry ption +Ġad min +Ġsqu ee +VID IA +Ġcreat or +iam eter +ic ular +N BC +Ġgrab bed +Ġn odd +Ġr ated +Ġrot ation +Ġgr asp +Ġexcess ive +ĠE C +ĠWh it +Ġinvent ory +ault s +ĠF B +Ġe cosystem +Ġbill ions +Ġvent ure +n amed +Ġdef ender +out e +Inst ead +ir able +W ar +Ġassum ption +Ġb ite +Ġearth qu +t ail +sp ace +Ġgif ts +boy s +Ġinev itable +Ġstruct ural +Ġbenef icial +Ġcompe lling +h ole +erv ation +Ġco at +o j +inc arn +ĠY ears +Ġdetermin ing +Ġrhet oric +Ġbound aries +Ġwh ites +A nt +add y +) - +ra ham +eter min +Ġhar vest +ĠCon c +Ġlapt op +ĠM atch +Ġenjoy ing +cc a +oll ar +Ġtri ps +Ġadd iction +ĠS ak +Ġpow ered +Ġc ous +ĠRuss ians +ie re +Ġret rie +qu ality +Ġdiff er +Ġking dom +ĠL aur +ĠCap itol +Ġcon clusions +ĠAl tern +ĠN av +Ġtrans parent +B ER +G roup +ĠCom plete +Ġinf er +Ġint rig +Ġins ane +R O +oph ob +is en +qu al +Mich ael +Ġm useum +ĠP ope +Ġres et +r ative +f ive +Ġagg reg +itte es +osit ory +Ġcar b +ĠRec ord +Ġdec ides +ĠF ix +Ġexcept ions +ĠCommission er +un s +ĠEnvironment al +Ġlegend ary +ist ence +Ġtun nel +k m +Ġins ult +Ġt roll +Ġsh ake +Ġdet ention +qu es +ĠCh rome +ĠF iles +Ġsub t +Ġprospect s +Ġpro l +re nder +pro of +Ġperform ances +St r +Ġh ref +ern ame +Ġachieve ment +Ġf ut +F ull +ĠLe ban +go ogle +ãĥ Ī +amp a +May be +Ġproject ed +ĠE mb +Ġcol leg +Ġa wards +Ġâ Ķ +G old +ĠBl ake +ĠR aj +if ting +Ġp ending +Ġinst inct +Ġdevelop ments +Con nect +ĠM and +ĠW ITH +ĠPhilipp ines +prof ile +Ġalt ogether +ĠB und +ĠT D +oo oo +amp ed +ip h +Ġste am +Ġold est +Ġdet ection +ul pt +Ġ ç +ĠWay ne +200 6 +f a +Ġcir cles +ĠF u +Ġdon ors +appropri ate +ĠDak ota +j amin +Ġmotiv ated +Ġpurch ases +ĠLouis iana +ĠS pl +Ġgl obe +Ġ10 5 +z ip +c all +Ġdepart ments +Ġsustain able +10 5 +ĠO P +if iers +Ġprevent ed +Ġinc omp +ĠComm ander +Ġdom inated +Ġ » +Ġinvest ed +Ġcomplex ity +Ġin cl +Ġens uring +Ġreal m +yn c +ĠInd ependent +r ained +ĠJ en +ĠFl ight +Ġat he +Ġspec ulation +ĠT E +oc ate +t ic +Ġpl aint +her ry +Ġto y +Ġ1 11 +Ġpl ates +st atus +ĠIs a +Ġdev oted +C op +ĠE S +25 5 +ur rency +M ain +Ġsl aves +Ġpe pper +Ġqu otes +Ġce iling +ĠF ish +Ġtrans formation +Ġfra ction +Ġadvant ages +Ġto ile +Ġstun ning +Ġmo ist +bre aking +s i +ĠL ocation +ĠMed ium +Ġtext s +Ġu gly +Ġb io +. âĢĶ +ĠB ased +Ġtr ains +ĠW ing +ĠAn cient +ĠRec ords +ĠH ope +Spe cial +ades h +ob i +[ / +Ġtempor arily +V er +h u +os er +Ġover night +Ġm amm +ĠTre asury +ĠV enezuel +ĠMeg a +Ġt ar +Ġexpect s +bl ack +or ph +\\ \\ +Ġaccept ance +Ġrad ar +s is +Ġjun ior +Ġfram es +Ġobserv ation +ac ies +P ower +ĠAdv anced +M ag +olog ically +ĠMe chan +Ġsent ences +Ġanaly sts +augh ters +force ment +Ġv ague +Ġcl ause +Ġdirect ors +Ġeval uate +Ġcabin et +M att +ĠClass ic +A ng +Ġcl er +ĠB uck +Ġresear cher +Ġ16 0 +Ġpoor ly +Ġexperien cing +ĠP ed +ĠMan hattan +Ġfre ed +Ġthem es +ad vant +Ġn in +Ġpra ise +10 4 +ĠLib ya +b est +Ġtrust ed +Ġce ase +Ġd ign +D irect +Ġbomb ing +Ġm igration +ĠSci ences +Ġmunicip al +ĠA verage +Ġgl ory +Ġreve aling +Ġare na +Ġuncertain ty +Ġbattle field +ia o +G od +Ġc inem +ra pe +el le +ap ons +Ġlist ing +Ġwa ited +Ġsp otted +ke ley +ĠAud io +e or +ard ing +idd ing +ig ma +ĠN eg +Ġl one +Ġ ---- +ex e +d eg +Ġtrans f +Ġwas h +Ġsl avery +Ġexpl oring +ĠW W +ats on +Ġen cl +l ies +ĠC reek +Ġwood en +Man ager +ĠBr and +um my +ĠAr thur +Ġbureau cr +Ġbl end +ar ians +F urther +Ġsupposed ly +Ġwind s +Ġ19 79 +Ġgrav ity +Ġanalys es +ĠTra vel +ĠV eter +Ġd umb +Ġaltern ate +g al +Ġconsum ed +Ġeffect iveness +.' ' +Ġpath s +ond a +L A +ĠStr ong +Ġen ables +Ġesc aped +Ġ" " +Ġ1 12 +Ġ198 3 +Ġsm iled +Ġtend ency +F ire +Ġp ars +ĠR oc +Ġl ake +Ġf itness +ĠA th +ĠH orn +Ġh ier +Ġimp ose +m other +Ġp ension +ic ut +bor ne +ic iary +. _ +ĠS U +Ġpol ar +is y +eng u +itial ized +AT A +w rite +Ġexerc ises +ĠD iamond +ot ypes +Ġharm ful +on z +Ġprint ing +st ory +Ġexpert ise +ĠG er +Ġtraged y +ĠF ly +Ġd ivid +amp ire +st ock +M em +Ġre ign +Ġun ve +Ġam end +ĠProp het +Ġmut ual +ĠF ac +Ġrepl acing +H ar +ĠCirc uit +Ġthro at +ĠSh ot +Ġbatter ies +Ġto ll +Ġaddress ing +ĠMedic aid +Ġp upp +ĠN ar +ol k +Ġequ ity +M R +ĠHis pan +ĠL arge +m id +D ev +Ġexp ed +Ġdem o +ĠMarsh all +erg us +Ġf iber +Ġdiv orce +ĠCre ate +Ġsl ower +ĠPark er +ĠStud ent +ĠTr aining +Ret urn +ĠT ru +Ġc ub +ĠRe ached +Ġpan ic +Ġqu arters +Ġre ct +Ġtreat ing +Ġr ats +ĠChristian ity +ol er +Ġsac red +Ġdecl are +ul ative +et ing +Ġdeliver ing +est one +Ġt el +ĠL arry +Ġmet a +ac cept +art z +ĠRog er +hand ed +Ġhead er +Ġtra pped +ĠCent ury +Ġkn ocked +ĠOx ford +Ġsurviv ors +b ot +Ġdemon stration +Ġd irt +Ġass ists +OM E +ĠD raft +ortun ate +fol io +pe red +ust ers +g t +ĠL ock +Ġjud icial +ver ted +Ġsec ured +out ing +ĠBook s +Ġhost ing +Ġlif ted +l ength +Ġj er +Ġwhe els +ĠR ange +umbn ails +Ġdiagn osis +te ch +ĠStew art +ĠP ract +Ġnation wide +Ġde ar +Ġoblig ations +Ġgrow s +Ġmand atory +Ġsusp icious +! ' +A pr +G reat +Ġmort gage +Ġprosecut or +Ġeditor ial +ĠK r +Ġprocess ed +ung le +Ġflex ibility +Ear lier +ĠC art +ĠS ug +Ġfoc uses +Ġstart up +Ġbre ach +ĠT ob +cy cle +ãĢ Į +ro se +Ġb izarre +ãĢ į +Ġveget ables +$ $ +Ġret reat +osh i +ĠSh op +ĠG round +ĠSt op +ĠHawai i +ĠA y +Per haps +ĠBe aut +uff er +enn a +Ġproduct ivity +F ixed +cont rol +Ġabs ent +ĠCamp aign +G reen +Ġident ifying +Ġreg ret +Ġpromot ed +ĠSe ven +Ġer u +ne ath +aug hed +ĠP in +ĠL iving +C ost +om atic +me ga +ĠN ig +oc y +Ġin box +Ġem pire +Ġhor izont +Ġbr anches +Ġmet aph +Act ive +ed i +ĠFil m +ĠS omething +Ġmod s +inc ial +ĠOrig inal +G en +Ġspir its +Ġear ning +H ist +Ġr iders +Ġsacr ific +M T +ĠV A +ĠS alt +Ġoccup ation +ĠM i +Ġdis g +lic t +Ġn it +Ġn odes +e em +ĠP ier +Ġhat red +ps y +ãĥ ī +Ġthe ater +Ġsophistic ated +Ġdef ended +Ġbes ides +Ġthorough ly +ĠMedic are +Ġbl amed +arent ly +Ġcry ing +F OR +pri v +Ġsing ing +ĠI l +Ġc ute +o ided +olit ical +ĠNe uro +å ¤ +Ġdon ation +ĠEag les +ĠG ive +T om +Ġsubstant ially +ĠLic ense +ĠJ a +Ġg rey +ĠAn imal +ĠE R +ĠU nd +Ġke en +Ġconclud e +ĠMississ ippi +Eng ine +ĠStud ios +P ress +o vers +ll ers +Ġ3 50 +ĠR angers +Ġr ou +ert o +E p +iss a +iv an +Ġse al +ĠReg ist +dis play +Ġwe aken +u um +ĠComm ons +ĠS ay +Ġcult ures +Ġl aughed +Ġsl ip +Ġtreat ments +iz able +m art +ĠR ice +Ġbe ast +Ġob esity +ĠLa ure +ig a +Wh ich +hold er +Ġelder ly +Ġp ays +Ġcompl ained +Ġc rop +Ġpro c +Ġexplos ive +ĠF an +ĠAr senal +A uthor +ef ul +Ġme als +Ġ( - +id ays +Ġimag ination +Ġann ually +Ġm s +as ures +H ead +ik h +m atic +Ġboy friend +ĠCom puter +Ġb ump +Ġsur ge +ĠCra ig +ĠKir k +D el +medi ate +Ġscen arios +ĠM ut +ĠSt ream +Ġcompet itors +Ù Ħ +ĠStan ford +ĠRes ources +az ed +b age +Ġorgan is +ĠRe lease +Ġsepar ately +Ġha bits +Ġmeasure ments +ĠCl ose +Ġaccomp any +Ġg ly +Ġt ang +ĠR ou +Ġplug in +Ġcon vey +ĠChall enge +oot s +j an +Ġcur s +ĠRel ations +ke eper +Ġapproach ing +p ing +Spe aking +Ġarrang ement +ĠV I +are ttes +Ġaffect ing +Ġperm its +b ecause +Ġu seless +ĠH us +!! !! +Ġdestro ying +Un fortunately +Ġfasc inating +S em +Ġelect oral +Ġtrans parency +ĠCh aos +Ġvolunte er +Ġstatist ical +Ġactiv ated +ro x +We b +H E +ĠHamp shire +is ive +M ap +Ġtr ash +ĠLaw rence +st ick +C r +Ġr ings +EX T +Ġoper ational +op es +D oes +ĠEv ans +Ġwitness ed +P ort +Ġlaunch ing +ec onom +w ear +ĠPart icip +um m +cul es +ĠR AM +ĠT un +Ġass ured +Ġb inary +Ġbet ray +Ġexpl oration +ĠF el +Ġad mission +it ated +S y +Ġav oided +ĠSim ulator +Ġcelebr ated +ĠElect ric +¥ ŀ +Ġcl uster +itzer land +he alth +L ine +ĠN ash +at on +Ġsp are +Ġenter prise +ĠD IS +clud es +Ġfl ights +Ġreg ards +ĠÃ Ĺ +h alf +Ġtr ucks +Ġcontact s +Ġunc ons +ĠCl imate +Ġimm ense +N EW +oc c +ect ive +Ġemb od +Ġpat rol +Ġbes ide +Ġv iable +Ġcre ep +Ġtrig gered +ver ning +Ġcompar able +q l +Ġg aining +ass es +Ġ( ); +ĠG rey +ĠM LS +s ized +Ġpros per +" ? +Ġpoll ing +Ġsh ar +ĠR C +Ġfire arm +or ient +Ġf ence +Ġvari ations +g iving +ĠP i +osp el +Ġpled ge +Ġc ure +Ġsp y +Ġviol ated +Ġr ushed +Ġstro ke +ĠBl og +sel s +ĠE c +,' ' +Ġp ale +ĠColl ins +ter ror +ĠCanad ians +Ġt une +Ġlabor atory +Ġn ons +t arian +Ġdis ability +ĠG am +Ġsing er +al g +ĠSen ior +Ġtrad ed +ĠWar rior +Ġinf ring +ĠFrank lin +Ġstr ain +ĠSwed ish +Ġsevent h +ĠB enn +ĠT ell +Ġsynd rome +Ġwond ered +id en +++ ++ +ig o +Ġpur ple +Ġjournal ism +Ġreb el +Ġf u +bl og +Ġinv ite +ren cies +ĠCont act +Is rael +ĠCont ent +Ġche er +Ġbed room +ĠEngine ering +ĠQue ens +Ġd well +ĠPlay Station +ĠD im +ĠCol on +l r +Ġoper ates +Ġmotiv ation +US A +ast ered +C ore +ĠTr uth +ol o +OS E +ĠMem ory +Ġpred ec +Ġan arch +Ġ19 20 +ĠY am +à ¨ +b id +Ġgr ateful +Ġexc itement +Ġtre asure +Ġlong est +ct ive +Ġdes erves +Ġreserv es +Ġcop s +ĠOtt awa +ĠEgypt ian +ank ed +Ġart if +Ġhypot hesis +: / +Ġpurch asing +Ġlove ly +H P +Ġdiv ide +Ġstrict ly +Ġquestion ing +Ġtaxp ayers +ĠJ oy +Ġroll s +ĠHe avy +Ġp orts +Ġmag netic +Ġinf lamm +Ġbr ush +t ics +â ĪĴ +Ġbott les +pp y +Ġp add +ãĤ ¯ +m illion +Ġdevast ating +Ġcomp iled +Ġmed ication +Ġtw elve +ĠPer ry +Sp ace +im b +y our +Ġle aked +ĠT ar +Ġun ity +Ġinfect ed +Ġtravel ed +ID E +ĠMc Donald +t xt +ĠPr inc +Ġinter ven +ĠTai wan +ĠP ow +Ġbe aring +ĠTh read +Ġz ones +iz ards +un ks +Ch apter +ll or +Ġ · +Ġw ounds +Ġdisc retion +Ġsucceed ed +ik ing +Ġicon ic +C all +Ġscreen ing +ĠM is +ict s +Ġmin isters +Ġsepar ation +Pl ayer +Ġb ip +Ġbel oved +Ġcount ing +ĠE ye +ar ound +ing ing +Ġtable t +Ġoff ence +in ance +h ave +ĠInf o +ĠNin ja +Ġprotect ive +ĠC ass +M ac +ĠQual ity +N orth +Ġ ic +ĠCub a +ĠChron icle +ĠPro perty +Ġfast est +ot os +ĠG erm +OW N +Ġbo om +ĠStan ley +ergus on +Ġcle ver +Ġent ers +m ode +ter ior +ĠS ens +Ġlin ear +AR K +Ġcomp aring +Ġpure ly +Ġsaf er +ĠPot ter +Ġc ups +R T +Ġgl uc +Ġatt ributed +Ġdu pl +ĠP ap +Ġprec ious +Ġp a +iction ary +ĠT ig +ĠTo o +ol utions +st an +Ġrob ots +Ġlob b +Ġstat ute +Ġprevent ion +w estern +16 0 +ĠAct ive +ĠMar ia +h al +N one +ell ar +ĠK B +ĠPart ners +ĠSing le +ĠFollow ing +ang o +ac ious +Ġth ou +Ġk g +Ġinflu ential +ĠFriend s +S ur +ain ted +Ġfor ums +Ġst arter +Ġcitizens hip +ĠE lection +on ge +ot ation +os ph +;; ;; +ut ical +p ur +ere n +Ġaccus ations +bit ious +ab bit +ĠOr d +Post ed +ir k +Ġsens itivity +ic he +ĠAm y +ĠF ab +Ġsum mit +Ġped est +Ġrub ber +Ġagric ultural +Ġcan cel +A E +Ġin aug +Ġcont am +Ġfirm ly +i w +st age +ĠK an +Ġt ier +Ġinv ention +Ġtransl ated +ĠR ules +B ox +Tw itter +ID S +Ġp izza +Ġdeb ug +ĠD rop +v s +Ġh orses +b ig +Ġb oring +Ġh ood +ĠMcC ain +at ched +ĠBro s +Ġsk ip +Ġess ay +st at +ĠLeg ends +Ġam munition +au c +Ġshoot er +Ġun h +Ġsuppl ied +Ġgener ic +ĠS K +ib an +yr ics +Ġ25 5 +Ġclim bing +Form er +Ġfl ip +Ġjump ing +Ġfrust ration +ĠTer ry +Ġneighborhood s +Ġmed ian +be an +Ġbr ains +Follow ing +Ġsh aped +Ġdraw s +Ġal tered +J ack +Ġrecip es +Ġsk illed +we alth +ach i +e lection +Ġbehavi ors +de als +ĠU ntil +F e +Ġdecl aration +mar ks +ĠBet ween +cel ona +Ġres on +Ġbub ble +Am ong +Ġim perial +G S +Ġfemin ist +200 5 +ĠK yle +Ġaccount ing +ĠTe le +ĠT yr +Ġconnect ing +Ġre hab +ĠP red +s im +Ġmeant ime +Ġphys ician +M W +ĠCamp bell +ĠBr andon +Ġcontribut ing +ĠR ule +ĠWe ight +ĠN ap +Ġinter active +Ġv ag +Ġhel met +ĠCom b +f our +Ġsh ipped +Ġcomple ting +ĠP D +PD ATE +Ġspread ing +Ġsc ary +erv ing +ĠG as +Ġfr ank +s chool +Ġrom antic +Ġstab il +R ob +Ġaccur ately +Ġac ute +ĠH ann +Ġsymbol s +Ġcivil ization +ĠA W +Ġlight ning +Ġcons iders +Ġven ue +Ġ × +Ġo ven +ĠS F +h is +Ġn u +ĠLear n +Ġpe oples +Ġst d +Ġsle e +Ġs lic +ĠStat istics +Ġcor ners +ĠB aker +Ġ: ) +ment ation +ol ver +Ġlaugh ing +ĠT odd +ond e +ĠH ills +Ġn uts +ĠW oman +pl ane +Ġl iver +ĠIn side +S orry +Ġagre es +Ġfund ament +ĠF isher +Ġa uction +Ġthread s +gl as +ĠBas ic +ĠN at +Ġlack ing +Ġceleb ration +j u +Ġs illy +E uro +Ġt att +ight y +cont rolled +T est +ĠSing h +Ġr age +Ġrh yth +o ffic +ĠPh antom +Ġhead lines +Ġrespond ing +ĠMor ning +Ġvit amin +Ġboot s +ĠS ite +al in +p i +Ġvir al +ĠU C +D ER +ĠSe x +Ġst ocks +c urrent +Ġch urches +ĠR are +ĠMur phy +Ġden ial +ĠG aming +Ġtou g +Ġn ick +Ġm akers +ĠRon ald +Ġgener ous +ĠD oc +ĠMor ris +Ġtransform ed +ĠN ormal +Ġ10 4 +ĠKick starter +ĠUp on +On line +ĠI RS +Ġw rap +Ġl oving +Ġarri ves +ĠD ue +Ġhe ter +ĠM ade +Ġrent al +Ġbelong s +Ġatt orneys +Ġcro ps +Ġmat ched +ul um +ol ine +10 9 +Ġdis par +Ġbuy ers +ĠCam bridge +Ġeth ics +rou ps +Ġjust ified +Ġmarg inal +Ġrespect ed +win ning +Ġnodd ed +ĠSer ge +ĠForm er +C raft +######## ######## +ĠWar ner +Ġd ash +et e +Ġent ert +ĠE scape +out heast +Ġkn ees +ĠB omb +Ġr ug +P ass +Ġatt itudes +go vernment +ĠPri or +Ġqual ities +Ġnot ification +ĠPh one +l ie +Ġanticip ated +ĠCom bat +ĠBar ry +Ġ198 2 +Us ers +on er +Ġcomput ing +ĠConnect icut +Ġless er +Ġpe ers +ĠC u +Ġtechn ically +Ġsub mission +ĠUn iversal +Ġman ually +our ge +Ġrespond ents +ĠB TC +ĠH ost +Ġf are +ĠB ird +Ġrece ipt +al so +Ġj ack +Ġagric ulture +Ġsk ull +Ġ! = +Ġpass ive +ĠC I +Ġsoc ieties +Ġremind ed +Ġinter ference +B uy +Ġâ ľ +g on +Ġscrut iny +ĠW itch +Ġconduct ing +Ġ ãĥ +Ġexch anges +ĠMit chell +Ġinhab it +Ġtw ist +B D +Ġwhere ver +group on +Ġj okes +ĠBen jamin +ĠR andom +fr ame +ĠL ions +Ġhighlight ed +ĠArk ansas +E nt +Ġp ile +Ġpre lim +g s +mind ed +Ġfel ony +ĠG A +ĠL uck +Ġpract ically +ĠB os +Ġact ress +D am +ĠB ou +Ġvis a +Ġembed ded +Ġhy brid +Ġear liest +Ġsoon er +s ocial +ĠH A +Ġste ep +Ġdis advant +Ġexplo it +ĠE gg +ĠUlt ra +Ġnecess ity +L ocal +ie ge +Ġd ated +Ġmass es +Ġsubsc ription +pl ess +Ġan onym +Ġpresum ably +Bl ue +The ir +asket ball +ĠPhil ip +Ġcom ed +load ed +r ane +Ġref lection +Ch ina +Ġext ends +Ġform ing +Ġund ers +200 1 +Ġgr at +Ġconcent rations +Ġins ulin +Ġsec ular +Ġwh ilst +Ġwin ners +Ad vertisements +Ġdeliber ately +ĠWork ing +Ġs ink +et ics +d ale +Ġmand ate +Ġg ram +Ġvac ation +Ġwarn ings +ri pp +ĠTH AT +Ġcomment ary +Ġint u +Ġa est +Ġreason ing +Ġbreak down +ĠZ ombie +Ġ-- > +ĠPolit ical +c ott +Ġthr ust +Ġtechn ological +Ġdec iding +Ġtraff icking +L ong +W elcome +pr ising +ĠCommun ications +Ġend ors +Ġsw ift +Ġmetab ol +co ins +res a +ĠHT TP +Ġen roll +ĠH appy +us r +int age +Ġ[ " +u ably +ĠM aterial +Ġrepe al +Se pt +k h +ĠMod i +Ġunder neath +ĠI L +sh ore +Ġdiagn osed +ace utical +Ġsh ower +au x +ĠSw itch +ĠStre ngth +Ġj ihad +n ational +Ġtra uma +uss y +on i +Ġcons olid +Ġcal ories +ĠF lynn +ag ged +16 8 +ĠP ink +Ġfulf ill +Ġch ains +Ġnot ably +ĠA V +L ife +ĠCh uck +m us +ĠUr ban +ĠH end +Ġdep osit +ĠS ad +Ġaff air +OR K +ie val +ĠF DA +Ġt rop +ĠOver all +Ġvirt ue +Ġsatisf action +au nd +Ġl un +ĠSw itzerland +ĠOper ation +pro cess +Ġsh ook +Ġcount ies +le ased +ĠCharl otte +1 12 +Ġtrans cript +Ġre dd +p ush +ĠHe y +ĠAn alysis +[ " +Ġaltern atives +ard less +Ġele ph +Ġpre jud +ĠLe af +H aving +ĠH ub +Ġexpress ions +ĠVol ume +Ġshock ing +ĠRed s +Ġread ily +Ġplan ets +ad ata +Ġcollaps ed +ĠMad rid +Ġir rit +i pper +ĠEn c +ĠW ire +Ġbu zz +ĠG P +ash a +Ġaccident ally +ur u +Ġfrust rated +ĠS A +Ġhung ry +ĠH uff +Ġlab els +ant o +ĠE P +Ġbar riers +) | +ĠBer keley +ĠJ ets +Ġp airs +ĠL an +J ames +ĠB ear +Ġhum or +ĠLiber ty +Ġmagn itude +Ġag ing +ĠM ason +Ġfriends hip +umb ling +Ġemer ge +Ġnewsp apers +Ġam bitious +ĠRich ards +atern al +Ġ198 1 +Ġcook ies +Ġsc ulpt +Ġpur suit +L ocation +Ġscript s +p c +Ġarrang ements +Ġd iameter +Ġl oses +am ation +Ġl iqu +ĠJ ake +aret te +Ġunderstand s +ĠZ en +v m +Ġappro ve +Ġw ip +Ġult ra +Ġint end +ĠD I +asc ular +Ġst ays +ĠK or +ĠK l +Ġinvest ing +L a +Ġbelie ving +b ad +m outh +Ġtaxp ayer +ãĥ ĥ +ĠQue bec +Ġl ap +ĠSw iss +d rop +Ġdr ain +ir i +et c +ft en +ĠN ex +Ġst raw +Ġscream ing +Ġcount ed +Ġdam aging +Ġamb assador +cent ury +Ġpro x +Ġarrest s +u v +il ateral +ĠCh arg +Ġpresc ribed +Ġindepend ently +Ġf ierce +ĠB aby +Ġb rave +Ġsu its += > +Ġbas eline +ĠR ate +Ġis lands +Ġ( ( +g reen +ix els +Ġname ly +ĠVill age +th an +am y +V ersion +g mail +ential s +ĠS ud +ĠMel bourne +Ġarri ving +Ġquant um +e ff +rop olitan +T ri +Ġfun eral +ĠI R +ÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤ ÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤ +ĠC ob +it ably +Ġt urb +Ġcomb o +Re view +Ġdeploy ment +u ity +ĠB ott +Ġinv isible +Ġrender ing +Ġunl ocked +Ġa qu +ĠVlad imir +Ġp ad +ĠBr ain +ĠLeg acy +dr agon +ĠKurd ish +Ġsound ed +Ġdet ained +ĠD M +g ary +Ġd aughters +Ġdistur bing +uk a +ĠPar ad +Ġt ast +Ġunf ortunate +Ġu l +em in +Ġattend ance +tr l +Ġpar ks +ĠMem orial +ĠAl ice +oth y +gu ard +ĠD ise +ĠSh an +ĠFor um +R ich +Ġshif ted +ue z +Ġl ighter +ĠMag n +Ġc od +S ch +ham mad +P ub +3 50 +ĠP okemon +Ġprot otype +Ġun re +B ase +ĠStud ents +ĠRep ly +ĠCommun ist +Ġg au +ĠTy ler +I Z +Ġparticip ated +Ġsup rem +ĠDet ails +Ġvessel s +ro d +Ġt ribe +ke ep +Ġassum ptions +Ġp ound +Ġcr ude +ĠAv ailable +Ġswim ming +Ġin clusion +Ġadv ances +c ulation +Ġconserv ation +Ġover d +ĠBuff alo +Art icle +ed ge +Ġaw a +ĠMad ison +Ġsid ew +Ġcat ast +ĠK rist +uc le +ĠHigh way +ĠTer ror +Ġactiv ation +Ġuncons cious +ĠSat an +ĠSus an +ill ery +Ġarr anged +i op +Ġrum ors +ur ring +th ink +ĠKe ith +ĠK ind +Ġavoid ing +by n +n ut +ĠSpe aker +r us +n ames +Ġgu ilt +ĠOlymp ics +Ġsa il +ĠM es +lev ant +ĠColumb us +a ft +C ity +S outh +ĠHar vey +ĠP un +S everal +Ġment ally +Ġimp ress +m ount +ĠUb untu +âĢĶâĢĶâĢĶâĢĶ âĢĶâĢĶâĢĶâĢĶ +ĠSuper man +ĠMP s +Ġintent ions +ĠR acing +Ġlike lihood +Ġ2 40 +T otal +Ġto ys +ĠW atson +Ġur ge +L ear +ĠP aper +Ġoccur ring +ĠB eng +ĠC ert +Ġst ones +T im +ĠTw in +z b +ĠD ynam +Ġpolit ician +k ens +ĠEnter prise +UT ERS +Ġab ol +Ġref resh +Ġarbit rary +pe ction +Ġtrou bles +Ġ} ); +t v +Ġpil ots +Ġdist ribute +Ġaud it +Ġp ause +orig inal +Ġr ivals + £ +F ig +T L +ab il +ry ing +L in +ion ed +l on +Ġf ancy +Ġcr ashed +Ġt ract +Ġshe d +Ġcons ume +B ased +down load +in it +Ġvolt age +Int rodu +Ġcondem ned +ĠFin ance +res pect +Ġex cluded +Ġestablish ing +her ic +Ġher itage +Ġspect acular +Ġun st +ĠSnow den +ĠL ane +S an +Ġprotect ions +st ruction +inc inn +Ġmac ro +C ustom +ios ity +Ġes p +Ġfunction ing +Ġm ush +Ġp uzzle +Ġeth ical +M al +Ġgo verning +ĠF erguson +Ġrest ored +Ġst ressed +ĠCoun ter +ĠK as +cl ip +AN S +Ġse iz +U K +by ss +old own +ap i +Ġperman ently +oun ters +W est +Th rough +L ight +at oes +Ġne at +Ġc ord +ure r +Ġsevere ly +ĠA ven +Ġinter rog +Ġtri ple +G iven +N umber +Ġar ise +Ġs her +pl ant +Ġfl ower +ĠC ou +Ġat e +Ġnew er +b ul +Ġmean while +ĠL air +Ġadjust ment +ĠCop yright +Ġd ivers +i ological +Ġgam ers +o at +Ġhistor ically +Ġanal og +Ġlong time +Ġpres cription +ĠM ist +ĠHy per +ĠM aine +ĠDe ity +Ġmulti pl +ĠRe incarn +ĠH yd +ĠP ic +S il +r ants +ĠC ris +. ; +( { +epend ence +Ġrec y +ate ur +Ġqu ad +Ġgl ob +Ġcon ced +te am +Ġcapital ist +ĠL ot +Ġroy al +ĠCy ber +Ġblack s +met ic +ri v +ĠD anny +Ġsp o +ĠR O +Ġanim ated +rypt ed +ĠDep uty +Ġrend ered +F E +Ġstre ak +Ġcloud s +ĠDou g +~~~~ ~~~~ +Ġdisc our +ĠVe h +Ġpsych ology +ĠJ ourney +Ġcry stal +ĠFro st +Ġsuspic ion +Ġrel ate +or us +ĠC rypt +ĠN VIDIA +com ed +ut ing +incinn ati +Ġvulner ability +ost ic +Ġisol ation +Ġcool ing +ĠCoal ition +Ġ1 19 +F our +ĠDe al +Ġâ ī +se mble +ram ent +ĠBar celona +Ġ10 2 +Ġcoc aine +ocaly pse +F eb +ogen ic +Ġmut ation +Ġcrypt oc +ĠK el +ĠG it +a is +Ġs isters +AN K +Ġactiv ate +T er +Ġd read +yl on +Ġprop ri +A ust +ĠDef ault +Ġout door +Ġshe er +ce ive +Ġg ently +Ð ¾ +Pro gram +Ġâ ĨĴ +Ġve gan +ĠCr us +Ġrespons ibilities +ĠH R +OL D +Ġprev ents +Ġst iff +ĠW ere +Ġathlet ic +ĠSc ore +Ġ) : +Ġcolumn s +ĠL oc +av ailable +ĠF ram +ĠS essions +Ġcompan ion +Ġpack s +14 0 +ĠKn ights +Ġf art +Ġstream s +Ġsh ore +Ġapp eals +ĠPer formance +h aul +ĠSt ra +ĠN ag +10 3 +ĠTrans portation +B B +E v +z an +P ublic +Ġtw in +uls ion +M ult +Ġelect ro +Ġstat ue +ation ally +ĠN ort +Ġins pection +/ * +ig ue +Ġcomp assion +ĠT ales +ĠSte in +ĠSc reen +ĠB ug +ĠL ion +g irl +Ġwithdraw al +Ġobject ives +Ġblood y +Ġprelim inary +Ġj acket +Ġdim ensions +ĠC ool +ĠOcc up +Ġw reck +Ġdoub led +ank ing +Ġ19 75 +Ġglass es +ĠW ang +pro v +P ath +connect ed +ĠMult i +ĠNor way +agon ist +Ġfe ared +Ġtouch ing +Ġarg uably +¯¯¯¯ ¯¯¯¯ +ĠNC AA +che m +Ġsp at +ĠW WE +ĠC el +ig ger +Ġattack er +ĠJo in +ob ject +ett a +Ġelim inated +d et +Ġdest ruct +ĠLuc as +ct uary +18 0 +ĠBr ady +ĠBl ues +B ay +au kee +Ġtim eline +Ġdeleg ates +w ritten +uff icient +Ġsh apes +Cop yright +ou ble +serv ice +Ġp ione +Ġcolleg es +Ġrow s +Ġsp ite +Ġassess ed +3 60 +Ġle ase +Ġconfident ial +ck er +ĠMan ning +ĠV oice +Ġse aled +Ġcalcul ate +N O +ĠAss istant +Ġteen ager +ul ent +ather ine +Ġm ock +Ġd iamond +Ġf est +Ġsw itched +Ġres ume +ĠPu erto +Ġl anes +ir ation +ĠSimilar ly +Ġro d +ĠS el +ĠPal ace +ĠLim ited +e ous +Ġvar iant +Ġw ard +Ġ) ) +Sh ow +OO K +A lex +ĠN ep +br is +ĠWik ipedia +Ġexcept ional +Ġman ages +ĠD raw +Ag ain +Ġco pper +ut t +Ġex ports +Ġport folio +Ġelev ated +R ated +ĠOther wise +ĠT act +ĠShe l +ĠT X +" âĢĶ +Ġres ur +ĠW a +ven ant +Ġmon etary +pe ople +E mail +Ġfif ty +ĠS weet +ĠMalays ia +Ġconf using +ĠR io +ud a +uten ant +" ); +Ġpra ised +Ġvol umes +t urn +Ġm ature +Ġnon profit +Ġpassion ate +ĠPriv ate +Ġ10 3 +Ġdesc end +ç ¥ŀ +uff y +head ed +Whe ther +ri en +ze ch +be it +Ġch rom +ĠMc M +Ġd ancing +Ġe leg +ĠNot iced +11 5 +Ġadvoc acy +ENT S +amb ling +ĠMin or +ĠF inn +Ġprior ities +Ġthere of +ĠSt age +ĠRog ers +Ġsubst itute +ĠJ ar +ĠJeff erson +Ġlight ly +10 2 +ĠL isa +u its +ys ical +Ġshif ts +Ġd rones +Ġwork place +Ġres id +ens ed +ah n +Ġpref erences +ser ver +Ġdeb ates +d oc +ĠGod s +Ġhelicop ter +Ġhon our +Ġconsider ably +ed ed +ĠF emale +ĠAn ne +Ġre un +ĠF ace +ĠHall ow +ĠBud get +Ġcondem n +Ġt ender +Pro f +ocr atic +ĠTurn er +ĠAg ric +Ġ19 76 +Ġa pt +d isc +ĠF ighter +ĠA ur +Ġgar bage +in put +ĠK arl +ĠOl iver +ĠL anguage +k n +N on +ĠCl ar +Ġtrad itions +Ġad vertisement +ĠS or +Ġarch ive +Ġvill ages +7 50 +Ġimplement ing +w aukee +Ġdiet ary +Ġswitch ing +Rep ublic +Ġvel ocity +Ġc it +ĠA wards +Ġfin ancing +Ġlast ed +) ] +Ġrem inder +P erson +Ġprec ision +Ġdesign ers +ĠF ried +ĠB order +Ġtr agic +Ġw ield +Ġiniti atives +ĠT ank +w er +Ġjo ins +R o +in ery +Ġar row +Ġgener ating +found er +Ġsear ches +Ġrandom ly +A ccess +Ġb atch +Ġp osed +l at +Ġpursu ing +as a +Ġtest ified +form ing +ĠSh ar +w iki +ĠE ither +S ometimes +Ġsen ators +ĠJohn ny +ĠTal iban +ĠG PS +":" / +ãģ® å +Ġanaly zed +ĠRub io +ĠMove ment +op ard +ii i +St and +f ight +Ġign oring +i ang +ĠG N +so ever +ĠST AT +Ġref using +Ġswe at +Ġb ay +P ORT +ir med +ak y +Ġdis pro +Ġlabel ed +Ġ10 8 +H ello +Ġple asant +ab a +Ġtri umph +Ġab oard +Ġinc om +ĠC row +le tt +Ġfol k +Ġch ase +` ` +ĠBr us +Ġte ens +c ue +Ġter rain +h yd +il ight +OR Y +Su pport +ew s +ll i +rain ts +ĠC and +Ġab used +ach ment +l arg +B as +ĠC ancer +Ġ19 78 +Ġsupp orter +ac cess +ĠTer min +ĠT ampa +ĠAN Y +Ġnew est +ĠCrim inal +ed u +Ġ19 30 +Ġadm its +Ġend e +Ġfail ures +ur ate +ful ness +cy cl +ĠSub ject +Ġinf inite +th ree +W A +p it +ĠInst all +R ad +ili ation +G M +Ġcontin ent +Ġaccommod ate +ĠCl ay +Ġp up +ĠF unction +Ġham mer +ĠAlbert a +Ġrev ised +Ġminor ities +Ġmeasure ment +Con nell +Ġdis able +ĠM ix +In cre +Ġfor k +ĠR osen +Ġimpl ies +umb lr +AN G +Ġprote ins +Ġagg ression +Ġfacilit ate +S N +Ġilleg ally +u er +Ġacad em +Ġp uzz +ĠSh ift +p ay +oll o +Ġaud iences +B uild +Ġno ble +Ġsynt ax +â ĺħ +Ġbe am +ĠB ed +ĠA ld +Ġorig ins +v ideo +Ġ19 77 +ĠAss ault +Ġgar age +Te am +Ġver dict +Ġd war +ĠVirt ual +e vent +Ke ep +Ġsent iment +Ġwild life +sh irt +Ġb urg +Ġrecommend ation +rep resent +Ġgall ery +own ers +Ġsch olar +Ġconven ience +ĠSw ift +Ġconv inc +C ap +Ġwar fare +ĠVis ual +Ġconst itute +Ġab ort +ĠWe ather +ĠLook ing +ĠH em +Ġmart ial +Ġinc oming +et ition +Ġtoler ance +ĠCre ated +Ġfl ows +ĠE lder +Ġsoul s +Ġf oul +ĠP ain +ĠC AN +Ġ2 20 +b c +he nd +Ġgen ius +R eal +ĠW r +omet er +p ad +Ġlim iting +ĠS i +ĠL ore +ĠAd ventures +Ġvar ied +D isc +f in +ĠPerson al +Ch ris +Ġinv ented +Ġd ive +ĠR ise +Ġo z +ĠCom ics +Ġexp ose +ĠRe b +let ters +s ite +im ated +Ġh acking +Ġeduc ated +ĠNob ody +Ġdep ri +Ġincent ive +ãĤ · +Ġovers ight +Ġtrib es +ĠBelg ium +Ġlicens ing +our t +Produ ct +ah l +ĠG em +Ġspecial ist +Ġc ra +ann ers +ĠCor byn +Ġ19 73 +RE AD +Ġsum mar +Ġover look +ĠApp lication +Ġin appropriate +Ġdownload ed +Q ue +ĠB ears +Ġth umb +ĠChar acter +ĠReincarn ated +ĠS id +Ġdemonstr ates +s ky +ĠBloom berg +ĠAr ray +ĠRes ults +ĠFour th +ĠED T +ĠO scar +c end +Ġ10 6 +ĠN ULL +ĠH ERE +m atch +ĠBr un +Ġgluc ose +ie g +eg u +Ġcert ified +Ġrel ie +Ġhuman itarian +Ġpr ayers +K ing +Ġn an +h ou +10 8 +ul u +Ġrenew able +Ġdistingu ish +Ġd ense +ĠV ent +ĠPack age +ĠB oss +Ġedit ors +Ġm igr +T ra +ĠPet ers +ĠAr ctic +200 4 +ĠC ape +Ġloc ally +Ġlast ing +Ġhand y +. ). +P an +ĠR ES +Ind ex +Ġt ensions +Ġformer ly +Ġide ological +Ġsens ors +Ġdeal ers +Ġdef ines +S k +Ġproceed s +Ġpro xy +az ines +ĠB ash +ĠP ad +ĠC raft +eal ous +Ġshe ets +omet ry +J une +cl ock +T T +ĠThe atre +ĠB uzz +Ġch apters +Ġmill enn +Ġd ough +ĠCongress ional +Ġimag ined +av ior +Ġclin ic +Ġ19 45 +Ġhold er +ro ot +oles ter +Ġrest art +B N +ĠHam as +ĠJ ob +Ġor b +Ġr am +Ġdiscl ose +Ġtransl ate +Ġimm igrant +Ġannoy ing +Ġtreat y +an ium +ĠTe a +ĠLeg ion +Ġcrowd s +ĠB ec +ĠA er +oh yd +B ro +Look ing +Ġl bs +Ġagg ress +Ġse am +Ġinter cept +ĠM I +mer cial +act iv +ĠC it +Ġdim ension +Ġconsist ency +Ġr ushing +ĠDou glas +Ġtr im +Inst all +ick er +Ġsh y +10 6 +Ġment ions +pe lled +ĠT ak +c ost +Ġclass room +Ġfort une +dri ven +Ġun le +ĠWhe el +Ġinvest or +ĠM asters +k it +Ġassoci ations +ĠEv olution +op ing +us cript +Ġprov incial +ĠWal ter +av i +S O +Ġun limited +Eng lish +ĠC ards +ĠEb ola +ne red +Ġreven ge +Ġout right +um per +Ġf itting +ĠSol id +Ġform ally +Ġproblem atic +Ġhaz ard +Ġenc ryption +Ġstraight forward +ĠA K +Ġp se +ĠOr b +ĠCh amber +ĠM ak +Cont ents +Ġloyal ty +Ġl yrics +ĠSy m +Ġwel comed +Ġcook ed +Ġmon op +Ġn urse +Ġmis leading +Ġe ternal +Ġshif ting +Ġ+ = +V is +Ġinst itutional +ill ary +Ġp ant +VER T +ĠA CC +ĠEn h +Ġinc on +ĠRE UTERS +Ġdon ated +âĢ¦âĢ¦ âĢ¦âĢ¦ +In tern +Ġexhib it +Ġt ire +ĠR ic +ĠCh ampion +ĠMu hammad +N ING +ĠSoc cer +Ġmob ility +Ġvary ing +ĠM ovie +Ġl ord +o ak +F ield +Ġve ctor +us ions +Ġsc rap +Ġen abling +m ake +T or +. * +| | +ĠWe bsite +ĠN PC +Ġsocial ist +ĠBill y +ĠAdd itional +Ġc argo +Ġfar ms +ĠSo on +ĠPri ze +Ġmid night +Ġ9 00 +se en +ĠSp ot +Ġshe ep +Ġspons ored +ĠH i +ĠJ ump +Ġ19 67 +Micro soft +ĠAg ent +Ġch arts +d ir +Ġadj acent +Ġtr icks +Ġman ga +Ġex agger +/ > +foot ball +ĠF CC +G C +ĠT ier +and ra +OU ND +% ), +Ġfru its +V C +ĠA A +R ober +Ġmid st +â Ĺ +ank a +Ġlegisl ature +ĠNe il +Ġtour ists +" " +ĠWar ning +ĠNever theless +ĠOffic ial +ĠWh atever +Ġm old +Ġdraft ed +Ġsubst ances +Ġbre ed +Ġt ags +ĠT ask +Ġver b +Ġmanufact ured +com ments +ĠPol ish +Pro v +Ġdetermin es +Ob ama +k ers +Ġutter ly +Ġse ct +sc he +ĠG ates +ĠCh ap +Ġal uminum +Ġz ombie +ĠT ouch +ĠU P +Ġsatisf y +Ġpred omin +asc ript +Ġelabor ate +Ġ19 68 +Ġmeas uring +ĠV ari +any ahu +Ġs ir +ul ates +id ges +ick ets +ĠSp encer +T M +oub ted +Ġpre y +Ġinstall ing +ĠC ab +re ed +re ated +Su pp +Ġwr ist +ĠK erry +10 7 +ĠK le +ĠR achel +Ġc otton +ĠA RE +ĠE le +Cont rol +Ġload s +ĠD od +an as +b one +Ġclass ical +ĠReg ional +ĠInt eg +V M +Ġdes ires +Ġaut ism +support ed +ĠM essage +Ġcomp act +writ er +Ġ10 9 +ĠHur ricane +c ision +Ġcy cles +Ġdr ill +Ġcolle ague +Ġm aker +G erman +Ġmist aken +S un +ĠG ay +Ġwhat soever +Ġsell s +ĠA irl +l iv +ĠO ption +Ġsol ved +Ġse ctors +Ġhorizont al +Ġequ ation +ĠSk ill +ĠB io +g ement +ĠSn ap +ĠLeg al +Ġtradem ark +Ġmake up +Ġassemb led +Ġsa ves +ĠHallow een +ĠVer mont +ĠFR OM +Ġfar ming +ĠP odcast +accept able +ĠHig her +Ġas leep +ull ivan +Ġrefere n +ĠLe v +Ġbul lets +ok o +H C +Ġst airs +Ġmain tains +ĠL ower +ĠV i +Ġmar ine +Ġac res +Ġcoordin ator +ĠJ oh +Ġcounterpart s +ĠBrother s +Ġind ict +b ra +Ġch unk +Ġc ents +H ome +ĠMon th +Ġaccording ly +if les +ĠGerm ans +ĠSy n +H ub +Ġey eb +âĶĢâĶĢ âĶĢâĶĢ +Ġr anges +ĠHoll and +ĠRob ot +f c +M ike +Ġpl asma +Ġsw ap +Ġath lete +ĠR ams +,' " +Ġinfect ions +Ġcor rid +Ġv ib +Ġpat ches +Ġtradition ally +Ġrevel ation +Ġswe ep +Ġgl ance +Ġin ex +200 3 +ĠR aw +work ing +os ures +ĠD at +ĠLyn ch +Ġle verage +ĠRe id +Ġcorrel ation +ian ces +av ascript +Ġrep ository +ret ty +Ġ19 72 +24 0 +Ġo un +p ol +ĠRe ed +Ġtact ical +is ite +App le +ĠQu inn +Ġrap ed +ill o +Euro pe +Ġalgorith ms +ĠRod rig +i u +Ġill um +Ġf ame +Ġintrodu cing +Ġdel ays +ĠRaid ers +Ġwh istle +Ġnovel s +ĠRe ally +Ġder iv +Ġpublic ations +ĠNe ither +ĠCom merce +Ġa ston +l anguage +Not es +ĠR oth +ĠF ear +Ġm ate +Ġpar ade +ĠQ B +Ġman eu +ĠC incinnati +m itting +Ġwa ist +ĠR ew +Ġdisc ont +Ð ° +Ġst aring +Ġal ias +Ġsec urities +Ġtoile t +ĠJ edi +Ġun law +v ised +//// //// +] ( +ĠWe iss +Ġpre st +ĠComp an +Ġmem o +ĠGr ace +J uly +ĠEl ite +cent er +ĠSt ay +Ġgal axy +Ġto oth +ĠS ettings +Ġsubject ed +ãĤ ¦ +Ġline back +Ġretail ers +ĠW ant +Ġd angers +A ir +Ġvolunt ary +ew ay +Ġinterpret ed +ot ine +à § +Ġp el +Serv ice +ĠEvent ually +Ġcare ers +Ġthreat en +Ġmem or +ĠBrad ley +anc ies +s n +ĠUn known +N ational +Ġsh adows +ail and +ĠD ash +Every one +izz ard +M arch += ( +Ġpull s +Ġstr anger +Ġback wards +ĠBern ard +imens ional +Ġch ron +Ġtheoret ical +k top +Ġw are +ĠInvest ig +ĠIn iti +ĠOper ations +o ven +oc ide +* / +Ġfl ames +ĠC ash +sh it +Ġc ab +ĠAn aly +ĠSe ah +Ġdefin ing +Ġorder ing +Ġimm un +Ġpers istent +AC H +Russ ian +m ans +Ġh ind +Ġphot ography + © +Ġh ug +Ġ10 7 +ĠH ence +i ots +ude au +Ġsubsid ies +Ġroutine ly +ĠDev ice +it ic +Ġdisg ust +land er +Ġ19 40 +Ġassign ment +ĠB esides +w ick +ĠD ust +us c +struct ed +11 1 +de velop +Ġf ond +Ġinter section +Ġdign ity +Ġcommission er +With out +re ach +Ġcart oon +Ġsc ales +ãĥ Ń +F IG +Ġsurve ys +ĠIndones ia +Ġart work +Ġun ch +Ġcy cling +un ct +au er +or ate +ĠOb viously +Ġcharacter ized +fe ld +Ġaff irm +Ġinn ings +Ġ é +Ġal iens +Ġcl oth +et ooth +ĠC ertain + § +Ġdig est +k now +ĠX L +Ġpredict ions +Ġd in +W AR +Ġafter math +Ex ample +ĠSu ccess +ĠTh r +IG N +Ġmin er +B us +Ġcl arity +heim er +ĠO UT +ĠS end +ĠCirc le +ĠD iet +Ġpron ounced +Ġcreat ors +Ġearthqu ake +atter y +ge ons +Ġo d +Ġlay ing +or p +U lt +pro ject +Ġunder min +Ġsequ el +S am +ĠDark ness +Ġre ception +b ull +Y S +ĠV ir +Ġsequ ences +ĠCo in +Ġout fit +ĠW ait +1 19 +Ġdel ivers +.... .. +Ġbl own +ĠE sc +ĠM ath +per m +ĠU l +Ġgl im +Ġfac ial +Ġgreen house +Ġto kens +/ - +ĠAnn ual +ĠON E +Ġteen age +ĠPhys ical +ĠL ang +ĠC elt +Ġsu ed +ivid ually +Ġpat ience +ch air +reg ular +Ġa ug +in v +ex cept +ĠL il +Ġn est +f d +s um +ĠCh ase +Russ ia +ĠJenn ifer +Ġoff season +Over all +F ore +Ġr iot +A ud +form er +Ġdefend ers +ĠC T +iot ic +rib ly +Ġautom ated +Ġpen is +Ġins ist +Ġdi agram +ĠS QL +ĠG arc +Ġw itch +cl ient +ier ra +am bers +Ġrec ount +f ar +V ery +oster one +Ġappreci ated +ĠPer fect +S ection +Ġd oses +oca ust +Ġcost ly +Ġg rams +ĠSh i +Ġwrest ling +Ġ19 71 +Ġtro phy +Ġn erve +ĠK az +ĠExper ience +Ġpled ged +Ġplay back +Ġcreat ivity +by e +Ġattack ers +Ġhold ers +ĠCo ach +ĠPh D +Ġtransf ers +Ġcol ored +ĠH indu +Ġd rown +Ġlist ened +ĠW A +ias m +P O +Ġappeal ing +Ġdiscl osed +ĠCh icken +ag ging +Ġple aded +Ġnav igation +ĠReturn s +Ġ[ [ +R OR +E A +Ġphotograp her +ĠR ider +ipp ers +Ġsl ice +Ġe rect +Ġhe d +iss ance +ĠVik ings +ur ious +Ġapp et +oubted ly +Ch ild +Ġauthent ic +o os +ĠM aking +Ġannoun cing +Ġb od +Ġmet er +ĠN ine +ĠR ogue +Ġwork force +Ġrenew ed +Ġorganis ations +ac s +P LE +Sh ort +Ġcomp ounds +ĠVis it +Ġen velop +ear th +Ġsupport ive +gg le +ĠBrus sels +ĠGu ild +Cre ate +RE L +Ġaver aged +Ġ19 69 +ri ages +Ġlength y +Ġforg ot +O kay +ĠE rd +Ġdeal er +Ġrec ession +D D +Ġdesper ately +Ġhun ger +Ġst icks +Ġm ph +ĠF aith +Ġintention ally +Ġdem ol +ue ller +ĠS ale +Ġde bris +s pring +Ġle ap +>> >> +Ġcontain ers +se lling +rane an +atter ing +Ġcomment ed +ĠC M +on ut +Ġwood s +es pecially +Ġorgan ize +iv ic +ĠWood s +ang a +s qu +Ġm aj +am on +Ġax is +Ġ19 74 +ĠDen mark +Ġwar rior +ĠP and +Ġout lined +ĠB O +ins ula +z illa +eb ook +Ġd are +Ġsear ched +Ġnav igate +S n +writ ing +Ġun ited +J apan +ĠHe brew +Ġfl ame +Ġrel ies +Ġcatch ing +ĠSh o +Ġimprison ment +Ġp ockets +Ġclos ure +ĠF am +t im +ade qu +Act ivity +Ġrecru iting +ĠW ATCH +ĠArgent ina +d est +Ġapolog ize +or o +Ġlack s +Ġtun ed +ĠGriff in +Ġinf amous +Ġcelebr ity +ss on +Ġ ---------------------------------------------------------------- +ĠIs is +ĠDis play +Ġcred ibility +Ġeconom ies +Ġhead line +ĠCow boys +Ġind ef +Ġl ately +Ġincent ives +but ton +ĠM ob +A ut +Ġres igned +ĠO m +c amp +Ġprof iles +Ġsche mes +olph ins +ay ed +Cl inton +en h +ĠY ahoo +Ġab st +Ġan k +su its +Ġw ished +ĠMar co +udd en +Ġsp here +ĠB ishop +Ġincorpor ated +ĠPl ant +11 4 +Ġh ated +p ic +Ġdon ate +Ġl ined +Ġbe ans +Ġsteal ing +Ġcost ume +Ġsher iff +Ġfor ty +Ġint act +Ġadapt ed +Ġtrave lling +b art +Ġnice ly +Ġdri ed +Ġsc al +os ity +NOT E +ĠB h +ĠBron cos +ĠI gn +Ġint imate +Ġchem istry +Ġopt imal +D eb +ĠGener ation +Ġ] , +ich i +ĠW ii +ĠYOU R +vent ions +W rite +Ġpop ul +un ning +ĠW or +V ol +Ġqu een +head s +K K +Ġanaly ze +op ic +ear chers +Ġd ot +leg raph +ast ically +Ġupgr ades +Ġca res +Ġext ending +Ġfree ze +Ġin ability +Ġorg ans +Ġpret end +Ġout let +11 3 +ol an +ĠM all +ul ing +t alk +Ġexpress ing +ĠAl ways +ĠBe gin +f iles +Ġlic enses +% % +ĠM itt +Ġfil ters +ĠMil waukee +G N +Ġunf old +M o +Ġnut rition +pp o +B o +Ġfound ing +Ġunder mine +Ġeas iest +ĠC zech +ĠM ack +Ġsexual ity +ĠN ixon +W in +ĠAr n +ĠK in +ãĤ £ +ic er +Ġfort un +Ġsurf aces +agh d +Ġcar riers +ĠP ART +ĠT ib +Ġinter val +Ġfrust rating +ĠSh ip +ĠAr med +ff e +Ġbo ats +ĠAb raham +in is +Ġsu ited +th read +i ov +ab ul +ĠVenezuel a +Ġto m +su per +Ġcast le +alth ough +iox ide +ec hes +Ġevolution ary +Ġnegoti ate +Ġconfront ed +Rem ember +Ġ17 0 +S uch +Ġ9 11 +m ult +ĠA byss +ur ry +ke es +spe c +ĠBarb ara +Ġbelong ing +Ġvill ain +ist ani +Ġaccount able +Ġport ions +ĠDe cl +U r +ĠK ate +g re +Ġmag azines +UC K +Ġregul ate +om on +ĠAl most +Ġover view +Ġsc ram +Ġl oot +ĠF itz +Ġcharacter istic +ĠSn ake +s ay +ĠR ico +Ġtra it +ĠJo ined +au cus +Ġadapt ation +ĠAirl ines +Ġarch ae +ĠI de +Ġb ikes +Ġliter ary +Ġinflu ences +ĠUs ed +C reat +Ġple a +ĠDef ence +ĠAss ass +Ġp ond +UL T +) " +Ġeval uated +Ġob taining +Ġdem ographic +Ġvig il +ale y +Ġsp ouse +ĠSeah awks +resp ons +ĠB elt +um atic +Ġr ises +run ner +ĠMichel le +Ġpot ent +r ace +ĠP AC +F ind +olester ol +IS S +ĠIntrodu ced +ress es +ign ment +O s +ĠT u +ĠDe x +ic ides +Ġspark ed +ĠLaur a +ĠBry ant +Ġsm iling +ĠNex us +Ġdefend ants +ĠCat al +Ġdis hes +sh aped +Ġpro long +m t +( $ +ãĢ Ĥ +Ġcalcul ations +ĠS ame +Ġp iv +H H +Ġcance lled +Ġgr in +Ġterrit ories +ist ically +C ome +ĠP arent +Pro ject +Ġneg lig +ĠPriv acy +Ġam mo +LE CT +olute ly +ĠEp ic +Ġmis under +w al +Apr il +m os +path y +ĠC arson +Ġalbum s +ĠE asy +Ġpist ol +< < +Ġ\ ( +t arget +hel p +Ġinter pre +cons cious +ĠH ousing +ĠJ oint +12 7 +Ġbe ers +s cience +ĠFire fox +effect ive +ĠC abin +ĠO kay +ĠApp lic +Ġspace craft +ĠS R +ve t +ĠStr ange +S B +Ġcor ps +iber al +e fficient +Ġpreval ence +Ġeconom ists +11 8 +Th read +ord able +OD E +ĠC ant +=- =- +if iable +ĠA round +Ġpo le +Ġwilling ness +CL A +ĠK id +Ġcomple ment +Ġsc attered +Ġin mates +Ġble eding +e very +Ġque ue +ĠTr ain +Ġh ij +Ġme lee +ple ted +Ġdig it +Ġg em +offic ial +Ġlif ting +Ð µ +Re qu +it utes +Ġpack aging +ĠWork ers +h ran +ĠLeban on +ol esc +Ġpun ished +ĠJ uan +Ġj am +ĠD ocument +Ġm apping +ic ates +Ġinev itably +Ġvan illa +ĠT on +Ġwat ches +Ġle agues +Ġiniti ated +deg ree +port ion +Ġrec alls +Ġru in +Ġm elt +I AN +Ġhe m +Ex p +Ġb aking +ĠCol omb +at ible +Ġrad ius +pl ug +ĠI F +et ically +Ġf ict +H ER +ĠT ap +atin um +Ġin k +Ġco h +ĠW izard +b oth +te x +Ġsp ends +ĠCurrent ly +ĠP it +Ġneur ons +ig nt +Ġr all +Ġbus es +b uilding +Ġadjust ments +Ġc ried +ibl ical +att ed +ĠZ ion +ĠM atter +Ġmed itation +ĠD ennis +Ġour s +ĠT ab +Ġrank ings +ort al +Ġad vers +Ġsur render +ĠG ob +ci um +om as +im eter +Ġmulti player +Ġhero in +Ġoptim istic +Ġindic ator +ĠBr ig +Ġgro cery +Ġapplic ant +ĠRock et +v id +Ex ception +p ent +Ġorgan izing +Ġenc ounters +ĠT OD +Ġjew el +S ave +ĠChrist ie +Ġhe ating +Ġl azy +ĠC P +Ġcous in +Con fig +Ġreg ener +Ġne arest +Ġachie ving +EN S +th row +ĠRich mond +ant le +200 2 +Ġan ten +b ird +13 3 +Ġn arc +r aint +un ny +ĠHispan ic +ourn aments +Ġprop he +ĠTh ailand +ĠT i +Ġinject ion +Ġinher it +rav is +Ġmed i +Ġwho ever +ĠDE BUG +G P +ĠH ud +C ard +p rom +Ġp or +Ġover head +L aw +Ġviol ate +Ġhe ated +Ġdescript ions +Ġachieve ments +ĠBe er +ĠQu ant +W as +Ġe ighth +ĠI v +Ġspecial ized +U PDATE +ĠD elta +P op +J ul +ĠAs k +oph y +Ġnews letters +ĠT ool +Ġg ard +ĠConf eder +ĠGM T +ĠAb bott +Ġimm unity +ĠV M +Is lam +Ġimpl icit +w d +Ġ19 44 +rav ity +omet ric +Ġsurv iving +ur ai +ĠPr ison +Ġr ust +ĠSk etch +Ġbe es +ĠThe ory +Ġmer it +T ex +ch at +Ġm im +Ġpast e +ĠK och +Ġignor ance +ĠSh oot +Ġbas ement +Un ited +ĠAd vis +he ight +Ġf oster +Ġdet ain +in formation +Ġne ural +' ; +Ġprov es +all ery +Ġinv itation +um bers +Ġc attle +Ġbicy cle +z i +Ġconsult ant +Ġap ology +ĠT iger +Ġ12 3 +99 9 +Ġind ividually +r t +ig ion +ĠBrazil ian +Ġdist urb +Ġentreprene urs +Ġfore sts +cer pt +pl ates +p her +clip se +Ġtw itter +Ġac ids +ograph ical +h um +ĠB ald +if ully +Ġcomp iler +ĠD A +Ġdon or +as i +Ġtrib al +l ash +ĠCon fig +Ġapplic ants +Ġsal aries +13 5 +Put in +ĠF ocus +ir s +Ġmisc onduct +ĠH az +Ġeat en +M obile +Mus lim +ĠMar cus +v iol +Ġfavor able +Ġst ub +ad in +ĠH ob +Ġfaith ful +Ġelectron ics +Ġvac uum +w ait +back ed +econom ic +d ist +Ġten ure +Ġsince re +ĠT ogether +ĠW ave +Ġprog ression +Ġden ying +Ġdist ress +br aska +th ird +Ġmix ing +Ġcolon ial +Ġpriv ately +Ġun rest +atern ity +Ġprem ises +ant i +greg ation +Ġlic ence +ĠH ind +ĠSam uel +Ġconvinc ing +ĠA ce +ĠR ust +ĠNet anyahu +Ġhand les +ĠP atch +orient ed +ah o +ĠG onz +Ġhack ers +claim er +Ġcustom s +ĠGr an +f ighters +Ġl uc +Ġman uscript +aren thood +Ġdev il +Ġwar riors +Ġoff enders +Will iam +Ġhol idays +Ġnight mare +Ġle ver +iff erent +St at +Ġexhib ition +put ed +ĠP ure +Ġal pha +Ġenthus iasm +ĠRepresent atives +E AR +ĠT yp +Ġwhe at +ĠAl f +Ġcor rection +Ġev angel +AT T +M iss +Ġs oup +Ġimpl ied +par am +Ġsex y +ĠL ux +Ġrep ublic +p atch +ab lish +Ġic ons +Ġfather s +ĠG ET +ĠCar ib +Ġregul ated +ĠCo hen +ĠBob by +Ġn er +Ġb ent +vent ory +ĠAl ong +ĠE ST +ĠWall ace +Ġmurd ers +r ise +ke ll +ĠCommon wealth +Ġn asty +et a +ĠM IT +Ġadminist ered +Ġgenuine ly +Ed itor +n ick +Ġhyd ro +**************** **************** +ĠB le +Ġfin es +Ġg orge +aus ible +r h +Ġapp le +ment ioned +Ġro pe +ot yp +H R +Ġdisappoint ing +Ġc age +n ik +Ġdoub ts +ĠF REE +print s +ĠM UST +Ġvend ors +ĠIn qu +Ġliber als +Ġcontract or +Ġup side +child ren +Ġtrick y +Ġregul ators +charg ed +l iter +Ġ *** +Ġreb ell +l ang +Ġloc als +Ġphys icians +Ġhe y +ar se +t m +ĠLe x +Ġbehavior al +success ful +F X +Ġbr ick +ov ic +Ġcon form +Ġreview ing +Ġins ights +Ġbi ology +ĠRem ove +ĠExt ra +Ġcomm itting +indu ced +ignt y +ig m +Ġat omic +Comm on +ĠE M +ĠP ere +ĠIt ems +e h +Ġpres erved +ĠH ood +Ġprison er +Ġbankrupt cy +Ġg ren +us hes +Ġexplo itation +Ġsign atures +Ġfin an +] ," +ĠM R +Ġme g +rem lin +Ġmusic ians +Ġselect ing +Ġexam ining +IN K +l ated +H i +Ġart ic +Ġp ets +Ġimp air +ĠM AN +Ġtable ts +in clude +R ange +Ġca ut +Ġlog s +Ġmount ing +Ġun aware +Ġdynam ics +ĠPalest ine +ĠQu arter +ĠPur ple +Ġm a +ĠIm port +Ġcollect ions +ci ation +Ġsuccess or +Ġcl one +Ġaim ing +Ġposs essed +Ġstick ing +Ġsh aking +Ġloc ate +ĠH ockey +T urn +17 0 +Ġfif teen +ĠHar rison +Ġcontinu ously +ĠT C +ĠVal ent +ĠRes cue +Ġby pass +am ount +Ġm ast +Ġprotect s +Ġart istic +Ġsomet ime +Ġsh oe +Ġshout ed +ific ant +et itive +ĠReg ister +ĠJ in +Ġconcent rated +ling ton +on ies +Ġgener ator +yr im +ĠAr men +Ġclear ing +id o +ĠT W +al ph +Ġlad ies +H ard +Ġdial og +Ġinput s +æ ľ +Ġpos es +Ġsl ots +ĠPrem ium +Ġle aks +Ġboss es +Ġ11 3 +c ourse +A cc +ĠNew ton +ĠAust ria +ĠM age +Ġte aches +ab ad +Ġwe ars +Ġc yl +Ġcur se +ĠS ales +ĠW ings +Ġp sy +Ġg aps +ĠIce land +ĠP interest +Ġland lord +Ġdefin itions +ĠK er +Ġsufficient ly +ĠP ence +ĠArch itect +Ġsur pass +Ġ11 4 +Ġsuper hero +ĠDise ase +Ġpri ests +ĠC ulture +Ġdefin itive +Ġsecret ly +ĠD ance +inst all +ch ief +ĠJess ica +W ould +Up dated +Ġlock er +ĠK ay +Ġmem orial +è ¦ +f at +Ġdis gu +Ġflav ors +ĠBase ball +ĠRes istance +Ġk icks +Ġen v +Ġteen agers +D ark +ĠC AR +Ġh alt +ĠL G +ĠGab riel +Ġfe ver +Ġs atur +Ġm all +Ġaffili ate +ĠS leep +ĠSpe cific +ĠV el +Ġj ar +ĠSac red +ĠEd wards +ĠA CL +Ġret ained +ĠG iant +Ġlim itation +in ces +Ġref usal +ĠT ale +ĠBut ler +Ġacc idents +ĠC SS +Ġimport ed +ĠCop y +Î ± +ER T +z el +Ġdiv isions +h ots +ĠAl b +ĠD S +Load er +W ashington +at isf +ĠCreat ive +\ . +ĠAut om +red ict +Ġrecept or +ĠCarl os +Met hod +ok a +Ġmal icious +Ġste pping +, [ +ĠD ad +Ġatt raction +ĠEffect s +ĠPir ate +ĠC er +ĠIndust ry +ĠR ud +Ġchar ter +Ġd ining +Ġins ists +Ġconfig ure +Ġ( # +ĠSim ple +ĠSc roll +UT C +17 5 +ĠK on +Ġmarket place +Ġ ãĤ +Ġref res +Ġg ates +er red +ĠP od +Ġbeh ave +Fr ank +n ode +Ġendors ed +he tt +as ive +ĠHom eland +Ġr ides +ĠLe ave +er ness +Ġflood ing +A FP +Ġris en +Ġcontin ually +Ġun anim +ĠCont ract +ĠP as +Ġgu ided +ĠCh ile +b d +Ġsu cc +pt ic +Ġcomm ittees +ĠL uther +ĠAny one +Ġs ab +12 4 +Ġp ixel +ĠB ak +ĠT ag +ĠBenn ett +En ter +sm all +ĠPresident ial +Ġp ul +Ġcontr ace +arch ive +Ġcoast al +ĠK ids +19 2 +âĢ ² +ick y +ING TON +Ġw olf +ĠSt alin +T ur +id get +am as +ĠUn less +Ġspons or +Ġmor ph +ĠCho ose +Ġrun ner +Ġun bel +Ġm ud +ĠMan a +Ġdub bed +Ġg odd +ure rs +wind ow +Ġrel ied +Ġcelebr ating +os c +Ġ13 5 +Ġlobb ying +Ġincom plete +Ġrestrict ion +Ġinc ap +it us +Ġexpect ation +ĠAp ollo +Ġint ens +Ġsyn c +G H +Ġmanip ulation +B Y +Ġspe ar +Ġbre asts +Ġvol can +il ia +M aterial +Ġform ats +ĠB ast +Ġparliament ary +Ġsn ake +Ġserv ants +ĠTr udeau +ĠGr im +ĠArab ic +ĠSC P +ĠBoy s +st ation +Ġprospect ive +ord e +in itialized +Ġb ored +AB LE +Ġaccess ed +Ġtax i +ĠShe ll +aid en +urs ed +in ates +ĠIns urance +ĠPet e +Sept ember +6 50 +Ġad ventures +ĠCo ver +Ġt ribute +Ġsk etch +Ġem power +Ġ Ø +ĠGl enn +ĠD aw += \" +ĠPolit ics +Ġgu ides +Ġd ioxide +ĠG ore +ĠBr ight +ĠS ierra +Ġval ued +c ond +Ġpo inter +Se lect +Ġrisk y +Ġabsor b +im ages +Ġref uses +Ġbon uses +__ _ +Ġh ilar +ĠF eatures +2 20 +ĠCollect or +F oot +Ġ19 64 +cul us +Ġd awn +Ġwork out +ĠL O +Ġphilosoph ical +ĠSand y +ĠYou th +Ġl iable +A f +bl ue +Ġovert urn +less ness +ĠTrib une +ĠIn g +Ġfact ories +Ġcat ches +Ġpr one +Ġmat rix +Ġlog in +Ġin acc +Ġex ert +s ys +Ġneed le +ĠQ ur +Ġnot ified +ould er +t x +Ġremind s +Ġpublisher s +Ġn ort +Ġg it +Ġfl ies +ĠEm ily +Ġflow ing +ĠAl ien +ĠStr ateg +Ġhard est +Ġmod ification +AP I +ĠM Y +Ġcr ashes +st airs +n umber +Ġur ging +ch annel +ĠFal con +Ġinhabit ants +Ġterr ifying +Ġutil ize +Ġban ner +Ġcig arettes +Ġsens es +ĠHol mes +Ġpract ition +ĠPhill ips +ott o +Ġcomp ile +Mod el +ĠK o +Ġ[ ] +Americ ans +ĠTer ms +Ġmed ications +ĠAn a +Ġfundament ally +ĠNot ice +Ġwe aker +Ġ 0000 +Ġgar lic +Ġout break +Ġeconom ist +ĠB irth +Ġobst acles +ar cer +ĠOr thodox +Ġplace bo +ĠC rew +asp berry +ĠAng els +Ġdis charge +Ġdestruct ive +11 7 +ĠR ising +Ġd airy +l ate +Ġcoll ision +ĠTig ers +ean or +ocument ed +ĠIn valid +Ġd ont +ĠL iter +ĠV a +Ġhyd rogen +Ġvari ants +ĠBrown s +Ġ19 65 +Ġind igenous +Ġtrad es +Ġremain der +Ġswe pt +ĠImp act +Ġred ist +Ġun int +grad uate +ãĥ ķ +ĠW ILL +ãģ® ç +ĠCrit ical +Ġf isher +Ġv icious +Ġrevers ed +Y ear +ĠS ox +Ġshoot ings +Ġfil ming +Ġtouchdown s +ai res +m el +Ġgrand father +Ġaffect ion +ing le +Ġover ly +Add itional +Ġsup reme +ĠGr ad +Ġsport ing +Ġmer cy +ĠBrook s +ount y +Ġperform s +Ġtight ly +Ġdem ons +Ġkill ings +Ġfact ion +ĠNov a +aut s +Ġund oubtedly +ar in +Ġunder way +ra k +Ġl iv +ĠReg ion +Ġbrief ing +s ers +cl oud +ĠM ik +us p +Ġpred iction +az or +Ġport able +ĠG and +Ġpresent ing +Ġ10 80 + » +ush i +ĠSp ark +there um +Ġjust ification +ĠN y +Ġcontract ors +ming ham +ĠSt yle +å ħ +ĠChron icles +ĠPict ure +Ġprov ing +Ġw ives +set t +Ġmole cules +ĠFair y +Ġconsist ing +Ġp ier +al one +in ition +Ġn ucle +j son +Ġg otta +Ġmob il +Ġver bal +ar ium +Ġmon ument +uck ed +Ġ25 6 +T ech +mine craft +ĠTr ack +Ġt ile +Ġcompat ibility +as is +Ġs add +Ġinstruct ed +ĠM ueller +Ġle thal +Ġhorm one +Ġor che +el se +Ġske let +Ġentert aining +Ġminim ize +ag ain +Ġunder go +Ġconst raints +Ġcig arette +ĠIslam ist +Ġtravel s +ĠPant hers +l ings +C are +Ġlaw suits +ur as +Ġcry st +Ġlow ered +Ġaer ial +Ġcomb inations +Ġha un +Ġch a +Ġv ine +Ġquant ities +Ġlink ing +b ank +Ġso y +B ill +ĠAngel a +Ġrecip ient +ĠProt est +Ġs ocket +Ġsolid arity +Ġâ Ĩ +m ill +Ġvar ies +ĠPak istani +Dr agon +Ġun e +Ġhor izon +³³³³ ³³³³ +Ġprov inces +Ġfrank ly +Ġenact ed +not es +[ ' +Ġ19 2 +ocr acy +Ġendorse ment +Ġover time +Tr ue +L ab +lic ted +ĠD NC +Ġbe ats +ĠJam ie +15 2 +ĠIN T +Cont act +Ġaccount ed +h ash +ĠPack ers +p ires +Ġles bian +Ġamend ments +Ġhop eful +ĠFin land +Ġspot light +Ġconfig ured +Ġtrou bled +Ġg aze +ĠCal gary +Ġrel iability +Ġins urg +sw er +b uy +ĠSk in +Ġp ixels +Ġhand gun +Ġpar as +Ġcateg or +ĠE L +ĠRe x +Ind eed +Ġkind a +Ġconj unction +ĠBry an +ĠMan ufact +y ang +Pl us +S QL +ish ment +Ġdom inate +Ġn ail +Ġo ath +Ġeru pt +ĠF ine +it bart +ĠCh ip +ĠAb d +ĠN am +Ġbuy er +Ġdiss ent +Le aks +Cont in +Ġr ider +ĠSome one +Ġill usion +c in +ĠBoe ing +Ġin adequ +ov ation +i ants +Ġreb uild +4 50 +ĠDest iny +S W +ĠT ill +H it +ia z +ĠBang l +acher s +ĠRe form +Ġse gments +Ġsystem atic +d c +ĠConserv atives +Ġport al +h or +ĠDragon bound +Ġdrag ged +om o +Ġthe e +ad vert +ĠRep orts +ĠE t +Ġbarrel s +Aug ust +Ġcompar isons +Ġhe x +Ġan throp +" [ +bor ough +ab i +Ġpict ured +play ing +ĠAdd ress +ĠMir ror +Sm ith +Ġt ires +ĠN PR +AA AA +Ġclass ification +ĠTh an +ĠH arm +ĠR A +Ġreject ion +min ation +Ġr anged +ĠF alls +D I +H ost +ãĤ ´ +ĠEx ample +list ed +th irds +Ġsaf egu +br and +Ġprob able +Can ada +IT ION +ĠQ aeda +Ġch ick +Ġimport s +h it +l oc +W W +Ġble w +Ġany time +Ġwh oles +ik ed +Ġcal culation +cre ate +ĠO ri +Ġupgr aded +Ġapp ar +ut ory +ĠM ol +B rit +ĠJ ong +IN AL +ĠStart ing +Ġd ice +urt le +Ġre lying +cl osure +Ġprof itable +Ġsl aughter +ĠMan ual +c aster +Ġ" $ +Ġfe ather +ĠSim ply +ie ves +Ġdeter ior +ĠPC I +Ġst amp +Ġfl aws +Ġsh ade +ham mer +Ġpass port +Ġcont ing +am el +Ġobser vers +Ġneg lect +ĠR B +ĠBrother hood +Ġskept ical +f amily +us k +Ġemotion ally +â Ļ +ĠBet a +ason able +id ity +ĠM ul +Ġkick ing +ĠC arm +oll ah +VERT IS +ĠAt hen +Ġlad der +ĠBul let +å £ +00 01 +ĠWild life +ĠM ask +ĠN an +R ev +Ġun acceptable +leg al +Ġcrowd ed +ag i +ĠC ox +j e +Ġmor ality +Ġfu els +Ġc ables +Ġman kind +ĠCarib bean +Ġanch or +Ġby te +ĠO ften +ĠO z +Ġcraft ed +Ġhistor ian +ĠW u +Ġtow ers +ĠCitiz ens +Ġhel m +Ġcred entials +Ġsing ular +ĠJes se +Ġtack les +Ġcont empt +Ġa fore +ĠSh adows +Ġn il +Ġur gent +app le +bl ood +Ġv on +Ġoff line +Ġbreat he +Ġj umps +Ġirre levant +ox ic +om al +import ant +J im +Ġgl oves +arm ing +dep th +Ġtal ents +ook ie +ĠS B +Ġpal m +uff s +est a +IG H +Ġcan on +ĠVer izon +ĠP le +Ġcou pled +vel t +Ġfundra ising +ĠGet ting +ĠD LC +Ġmathemat ical +ĠH S +ĠCard inals +te lling +Ġspons ors +Ġ Ï +ĠBull s +op tion +Ġprop ose +Ġmem orable +Ġembr aced +Ġdecl ining +He alth +ed a +Ġ} ; +Ġsp am +m ile +Ġpit cher +ĠE ight +Ġcar ing +ut ic +ro le +Ġair line +ernand ez +ĠAth let +Ġcert ification +ux e +rig er +Ġem pir +Ġsens ation +Ġdis m +Ġb olt +Ġev olve +H ouse +Ġconsult ation +ĠD uty +Ġtou ches +ĠN athan +Ġf aint +h ad +" ( +ĠCons umer +ĠExt reme +Ġ12 7 +ĠHer m +ĠSac rament +iz oph +Ġanx ious +ul ously +Ġsoc ially +ĠU TC +Ġsol ving +ĠLet ter +Hist ory +ed uc +Pr ice +) ); +Ġrel oad +am ic +Ġp ork +Ġdisc ourse +Ġt ournaments +ai ro +ĠK ur +ĠCost a +Ġviol ating +Ġinterf ere +Ġrecre ational +uff le +Ġspe eches +Ġneed ing +Ġremem bers +Ġcred ited +n ia +f ocused +amer a +Ġb ru +um bs +ĠCub an +Ġpreced ing +Ġnons ense +ac ial +Ġsmart phones +ĠSt ories +S ports +ĠEmer gency +oun cing +ef ined +Ġb er +Ġconsult ing +Ġm asters +he astern +." [ +ĠRun ning +Ġsus cept +ĠF eng +Americ a +pr ises +st itial +ĠWeek ly +ĠGreat er +mod ules +if ter +G raphics +ul er +Ġwho lly +Ġsupp ress +Ġconce aled +Ġhapp ily +Ġaccept s +ĠEn joy +Ġr ivers +ĠEx cept +2 25 +ĠN HS +ĠMc Connell +Ġp ussy +fer red +ut able +Ġatt ain +Ġ> = +Ġdepos its +roph ic +Ġnot orious +ĠSh aw +il itation +Ġepid emic +all ic +Ġsmall est +ov ich +Ġaccess ories +per ties +Ġsur plus +ĠMe ch +Ġamb ig +ĠImm igration +Ġch im +ev al +Ġpract icing +ĠMyster y +Ġdom ains +ĠSil icon +app s +Ġkilomet ers +e a +ĠSm ash +Ġwarrant y +Ġn ost +s il +re v +J on +ĠDub lin +Ġtast es +Ġb out +g reat +er ror +Ġsw itches +ĠB apt +D O +ok i +Ġsour ced +pro du +Ġattach ment +ĠIss ue +ĠQuest ion +Jo in +Ġf itted +Ġunlaw ful +^ ^ +ere k +Ġauthent ication +Ġst ole +Ġaccount ability +l abel +S earch +Ġal beit +atic an +fund ed +ĠAdd ing +ĠI Q +Ġsub mar +l it +a que +ĠLear ning +Ġint eger +M aster +ĠCh rom +Ġprem ier +O p +ĠLi u +Ġbl essed +ĠGl obe +ĠResp onse +Ġlegit im +ĠMer kel +Ġdispos al + ´ +Ġgau ge +pe at +Ġindu ced +Ġquestion able +arth y +ĠV it +ĠF eed +U ntil +U t +worth y +R Y +ĠH erald +ĠHam mer +Ġmed al +ĠR ivers +ĠH ack +Ġclar ify +Ġtrack ed +Ġautonom ous +Ġten ant +ĠQ atar +er ie +Ġgr im +ĠMon itor +Ġresist ant +ĠSpe c +ĠWell s +N AS +14 8 +Ġmin ers +iot ics +Ġmiss es +11 6 +g ian +g it +ĠE yes +p res +Ġgrad uated +Ġang el +Ġsyn chron +Ġefficient ly +Ġtrans mitted +H arry +Ġglob ally +EN CE +ĠMont ana +r aged +ĠPre vention +Ġp iss +ĠL l +Ġshe lf +ĠB JP +ĠTest ament +ĠL ate +ik er +ĠH app +ĠJul ian +h all +Ġsp ont +Ġshut down +Ġincons istent +Ġsubscrib ers +Ġske leton +ĠNe braska +Ġins pire +ĠV oid +F eed +Ġang les +ĠSpr ings +Ġbench mark +Ġvacc ines +izoph ren +se xual +uff ed +Ġsh ine +ĠK ath +Ġgest ure +ine a +Ġr ip +Ġopp ression +Ġcons cience +b t +ĠL um +Ġinc idence +ĠF a +w r +Ġmin eral +ĠSp urs +alk y +Ġth under +Ġop io +Be ing +ĠPal m +Ġwas ted +Ġl b +i aries +ĠIniti ative +Ġcur ric +Ġmark er +ĠMc L +Ġext ensions +ĠP v +ĠAr ms +Ġoffer ings +Ġdef enses +Ġvend or +Ġcontrad ict +ĠCol in +Ġredd it +Ġper ipher +12 2 +Ġs ins +E dit +IC T +So ft +ĠSh ah +Ġadministr ator +ĠT rip +Ġporn ography +Ġtu ition +in ence +ĠPro gress +Ġcat alog +Ġsu ite +Ġh ike +Ġreprodu ctive +eng ine +Ġd rought +ĠNo ah +Ġ2 30 +Ġd ude +Ġrelax ed +Ġpart ition +Ġparticip ant +Ġtel esc +Ġfe as +ĠF F +own er +Ġswe eping +Ġl enses +Ġmatch up +ĠRe pl +ourn als +Ġcred ible +Ġgrand mother +Ġther mal +Ġsubscrib ing +Ġident ities +col m +U CT +Ġreluct ant +us ers +ĠC ort +Ġassist ed +OS S +ATION S +IS H +Ġpharm aceutical +ic able +ad ian +ĠSon ic +ĠF ury +ĠM ong +A H +ĠPsych ology +Ġph osph +Ġtreat s +Ń Ķ +Ġstead ily +ĠHell o +Ġrel ates +Ġcl ue +Ex pl +a uth +Ġrev ision +Ġe ld +os ion +Ġbr on +14 4 +ri kes +Ġmin es +Ġblank et +ĠF ail +el ed +ĠIm agine +ĠPl anned +a ic +Re quest +M ad +ĠHor se +ĠEag le +Ġcap ac +15 7 +Ġl ing +ĠN ice +ĠP arenthood +min ster +og s +ens itive +Not hing +Ġcar n +F in +ĠP E +Ġr ifles +ĠL P +S and +Ġgui Active +Ġtour ist +C NN +Ġunve iled +Ġpredec essor +} { +u ber +Ġoff shore +Ġopt ical +ĠR ot +ĠPear l +et on +Ġst ared +Ġfart her +at ility +cont in +ĠG y +ĠF oster +ĠC oc +ri ents +Ġdesign ing +ĠEconom y +ON G +W omen +ĠN ancy +er ver +Ġmas cul +Ġcasual ties +Ġ2 25 +ĠS ullivan +ĠCh oice +Ġa ster +w s +Ġhot els +Ġconsider ations +Ġcou ch +ĠSt rip +ĠG n +Ġmanip ulate +l ied +Ġsynt hetic +Ġassault ed +Ġoff enses +ĠDra ke +Ġim pe +Oct ober +ĠHer itage +h l +ĠBl air +Un like +Ġg rief +Ġ4 50 +Ġopt ed +Ġresign ation +il o +Ġver se +ĠT omb +Ġu pt +Ġa ired +ĠH ook +ĠML B +Ġassum es +out ed +ĠV ers +Ġinfer ior +Ġbund le +ĠD NS +ograp her +Ġmult ip +ĠSoul s +Ġillust rated +Ġtact ic +Ġdress ing +Ġdu o +Con f +Ġrel ent +Ġc ant +Ġscar ce +Ġcand y +ĠC F +Ġaffili ated +Ġspr int +yl an +ĠGarc ia +Ġj unk +Pr int +ex ec +C rit +Ġport rait +ir ies +ĠOF F +Ġdisp utes +W R +L ove +ãģ Ħ +ĠRe yn +Ġh ipp +op ath +Ġflo ors +ĠFe el +Ġwor ries +Ġsett lements +ĠP os +Ġmos que +Ġfin als +Ġcr ushed +ĠPro bably +ĠB ot +ĠM ans +ĠPer iod +Ġsovere ignty +Ġsell er +Ġap ost +Ġam ateur +Ġd orm +Ġconsum ing +Ġarm our +ĠRo ose +Ġint ensive +Ġelim inating +ĠSun ni +ĠAle ppo +j in +Ġadv ise +p al +ĠH alo +Ġdes cent +Ġsimpl er +Ġbo oth +ST R +L ater +ĠC ave +== = +Ġm ol +Ġf ist +Ġshot gun +su pp +Ġrob bery +E ffect +Ġobsc ure +ĠProf essional +Ġemb assy +Ġmilit ant +Ġinc arcer +Ġgener ates +Ġlaun ches +Ġadministr ators +Ġsh aft +Ġcirc ular +Ġfresh man +ĠW es +ĠJo el +ĠD rew +ĠDun can +ĠApp arently +s ight +ĠIntern al +ĠInd ividual +ĠF E +Ġb ore +ĠM t +Ġbroad ly +ĠO ptions +ount ain +ip es +ĠV ideos +20 4 +Ġh ills +Ġsim ulation +Ġdisappoint ment +it an +ĠLabor atory +Ġup ward +Ġbound ary +Ġdark er +h art +Ġdomin ance +C ong +ĠOr acle +ĠL ords +Ġscholars hip +ĠVin cent +ed e +ĠR ah +Ġencour ages +ro v +Ġqu o +Ġprem ise +ĠCris is +ĠHol ocaust +Ġrhyth m +Ġmet ric +cl ub +Ġtransport ed +Ġn od +ĠP ist +Ġancest ors +ĠFred er +th umbnails +ĠC E +ON D +Ph il +ven ge +ĠProduct s +cast le +Ġqual ifying +ĠK aren +VERTIS EMENT +Ġmight y +Ġexplan ations +Ġfix ing +D i +Ġdecl aring +Ġanonym ity +Ġju ven +ĠN ord +ĠDo om +ĠAct ually +O k +ph is +ĠDes ert +Ġ11 6 +I K +ĠF M +Ġinc omes +V EL +ok ers +Ġpe cul +Ġlight weight +g ue +Ġacc ent +Ġincre ment +ĠCh an +Ġcompl aining +ĠB aghd +Ġmidfield er +Ġover haul +Pro cess +ĠH ollow +ĠTit ans +Sm all +man uel +ĠUn ity +ĠEv ents +S ty +Ġdispro portion +n esty +en es +ĠC od +Ġdemonstr ations +ĠCrim son +ĠO H +Ġen rolled +Ġc el +ĠBre tt +Ġa ide +Ġhe els +Ġbroad band +Ġmark ing +Ġw izard +ĠN J +ĠChief s +Ġingred ient +Ġd ug +ĠSh ut +urch ase +end or +Ġfar mer +ĠGold man +12 9 +15 5 +Or der +Ġl ion +i ably +Ġst ain +ar ray +ilit ary +ĠFA Q +Ġexpl oded +ĠMcC arthy +ĠT weet +ĠG reens +ek ing +l n +ens en +Ġmotor cycle +Ġpartic le +Ġch olesterol +B ron +Ġst air +Ġox id +Ġdes irable +ib les +Ġthe or +for cing +Ġpromot ional +ov o +b oot +ĠBon us +raw ling +Ġshort age +ĠP sy +Ġrecru ited +Ġinf ants +Ġtest osterone +Ġded uct +Ġdistinct ive +Ġfirm ware +bu ilt +14 5 +Ġexpl ored +Ġfact ions +Ġv ide +Ġtatt oo +Ġfinan cially +Ġfat igue +Ġproceed ing +const itutional +Ġmis er +Ġch airs +gg ing +ipp le +Ġd ent +Ġdis reg +ç Ķ +st ant +ll o +b ps +aken ing +Ġab normal +ĠE RA +å£ « +ĠH BO +ĠM AR +Ġcon cess +Ġserv ant +Ġas pir +l av +ĠPan el +am o +Ġprec ip +Ġrecord ings +Ġproceed ed +Ġcol ony +ĠT ang +ab lo +Ġstri pped +Le ft +to o +Ġpot atoes +Ġfin est +% ). +Ġc rap +ĠZ ach +ab ases +ĠG oth +Ġbillion aire +w olf +Ġsan ction +S K +Ġlog ged +P o +ey ed +un al +Ġcr icket +Ġarm ies +Ġunc overed +Cl oud +ó n +Ġreb ounds +Ġm es +O per +P ac +Ġnation ally +Ġinsert ed +p ict +Ġgovern ance +Ð ¸ +Ġprivile ges +G ET +Ġfavor ites +im ity +Ġlo ver +the m +em pl +Ġgorge ous +An n +Ġsl ipped +Ġve to +B ob +Ġsl im +u cc +ĠF ame +udden ly +Ġden ies +ĠM aur +Ġdist ances +Ġw anna +t ar +ĠS ER +Ġâ Ī +Ġle mon +at hetic +Ġlit eral +Ġdistingu ished +Ġansw ering +G I +Ġrelig ions +ĠPhil os +ĠL ay +Ġcomp os +ire ments +ĠK os +ine z +roll ing +Ġyoung est +and ise +ĠB orn +Ġalt ar +am ina +ĠB oot +v oc +Ġdig ging +Ġpress ures +Ġl en +26 4 +Ġassass ination +ĠBir mingham +ĠMy th +Ġsovere ign +ĠArt ist +ĠPhot ograph +Ġdep icted +Ġdisp ens +orth y +Ġamb ul +int eg +ĠC ele +ĠTib et +Ġhier archy +Ġc u +Ġpre season +ĠPet erson +Ġcol ours +Ġworry ing +Ġback ers +ĠPal mer +ĠÎ ¼ +Ġcontribut or +Ġhear ings +Ġur ine +Ġ Ù +ourge ois +Sim ilar +ĠZ immer +s omething +ĠUS C +Ġstrength s +ĠF I +Ġlog ging +As ked +ĠTh ai +in qu +ĠW alt +Ġcrew s +it ism +3 01 +Ġshar ply +um ed +Ġred irect +r ators +In f +ĠWe apons +Ġte asp +19 99 +L ive +ĠEs pecially +ĠS ter +ĠVeter ans +Ġint ro +other apy +Ġmal ware +Ġbre eding +Ġmole cular +ĠR oute +ĠCom ment +oc hem +Ġa in +Se ason +Ġlineback er +Ä « +ĠEconom ics +es ar +ĠL ives +ĠEm ma +Ġk in +ĠTer rit +Ġpl anted +ot on +ĠBut ter +ĠSp ons +P ER +Ġdun geon +Ġsymb olic +Ġfil med +Ġdi ets +Ġconclud es +Ġcertain ty +ĠForm at +Ġstr angers +form at +ĠPh ase +Ġcop ied +Ġmet res +ld a +ĠUs ers +Ġdeliber ate +Ġwas hed +ĠL ance +im ation +Ġimpro per +ĠGen esis +ick r +ĠK ush +Ġreal ise +Ġembarrass ing +alk ing +b ucks +Ġver ified +Ġout line +year s +ĠIn come +20 2 +Ġz ombies +F inal +ĠMill enn +Ġmod ifications +ĠV ision +ĠM oses +ver b +iter ranean +ĠJ et +Ġnav al +ĠA gg +Ġur l +Ġvict ories +Ġnon etheless +Ġinj ust +ĠF act +ç ļ +Ġins ufficient +re view +face book +Ġnegoti ating +Ġguarant ees +im en +uten berg +Ġg ambling +Ġcon gr +Load ing +Ġnever theless +Ġpres idents +ĠIndust rial +Ġ11 8 +Ġp oured +ĠT ory +Ġ17 5 +Ġ: = +Sc ott +ange red +T ok +Ġorgan izers +M at +ĠG rowth +Ġad ul +Ġens ures +Ġ11 7 +é¾į å +Ġmass acre +Ġgr ades +be fore +AD VERTISEMENT +ĠSl ow +ĠM MA +âĢĶ " +ĠV atican +Q aeda +Ġo we +66 66 +ĠS orry +ĠGr ass +Ġbackground s +Ġexha usted +Ġcl an +Ġcomprom ised +ĠE lf +ĠIsa ac +ens on +In vest +IF A +Ġinterrupt ed +ãĥī ãĥ© +Ġtw isted +ĠDrag ons +M ode +ĠK remlin +Ġfert il +he res +ph an +ĠN ode +f ed +ĠOr c +Ġunw illing +C ent +Ġprior it +Ġgrad uates +Ġsubject ive +Ġiss uing +ĠL t +Ġview er +Ġw oke +Th us +bro ok +Ġdep ressed +Ġbr acket +ĠG or +ĠFight ing +Ġstri ker +Rep ort +ĠPortug al +Ġne o +w ed +19 9 +Ġflee ing +sh adow +ident ified +US E +Ste am +Ġstret ched +Ġrevel ations +art ed +ĠD w +Ġalign ment +est on +ĠJ ared +S ep +Ġblog s +up date +g om +r isk +Ġcl ash +ĠH our +Ġrun time +Ġunw anted +Ġsc am +Ġr ack +Ġen light +on est +ĠF err +Ġconv ictions +Ġp iano +Ġcirc ulation +ĠW elcome +Ġback lash +ĠW ade +Ġrece ivers +ot ive +J eff +Ġnetwork ing +ĠPre p +ĠExpl orer +Ġlect ure +Ġupload ed +ĠMe at +B LE +ĠNaz is +ĠSy nd +st ud +ro ots +ri ans +Ġportray ed +Ġ ?? +ĠBudd ha +s un +Rober t +ĠCom plex +Ġover see +Ġste alth +T itle +ĠJ obs +ĠK um +Ġappreci ation +ĠM OD +Ġbas ics +Ġcl ips +Ġnurs ing +Ġpropos ition +Ġreal ised +ĠNY C +Ġall ocated +ri um +ar an +ĠPro duction +ĠV ote +Ġsm ugg +Ġhun ter +az er +ĠCh anges +Ġfl uct +y on +Ar ray +Ġk its +W ater +Ġuncom mon +Ġrest ing +ell s +w ould +Ġpurs ued +Ġassert ion +omet own +ĠMos ul +ĠPl atform +io let +Ġshare holders +Ġtra ils +P ay +ĠEn forcement +ty pes +ĠAn onymous +Ġsatisf ying +il ogy +Ġ( ' +w ave +c ity +Ste ve +Ġconfront ation +ĠE ld +C apt +ah an +ht m +ĠC trl +ON S +2 30 +if a +hold ing +Ġdelic ate +Ġj aw +ĠGo ing +or um +S al +Ġd ull +ĠB eth +Ġpr isons +Ġe go +ĠEl sa +avor ite +ĠG ang +ĠN uclear +Ġsp ider +ats u +Ġsam pling +Ġabsor bed +ĠPh arm +iet h +Ġbuck et +ĠRec omm +O F +ĠF actory +AN CE +Ġb acter +H as +ĠObs erv +12 1 +Ġprem iere +De velop +Ġcur rencies +C ast +Ġaccompany ing +ĠNash ville +Ġfat ty +ĠBre nd +Ġloc ks +Ġcent ered +ĠU T +augh s +or ie +ĠAff ordable +v ance +D L +em et +Ġthr one +ĠBlu etooth +Ġn aming +if ts +AD E +Ġcorrect ed +Ġprompt ly +ĠST R +Ġgen ome +Ġcop e +Ġval ley +Ġround ed +ĠK end +al ion +p ers +Ġtour ism +Ġst ark +v l +Ġblow ing +ĠSche dule +st d +Ġunh appy +Ġlit igation +ced es +Ġand roid +Ġinteg ral +ere rs +ud ed +t ax +Ġre iter +ĠMot ors +oci ated +Ġwond ers +ĠAp ost +uck ing +ĠRoose velt +f ram +Ġyield s +Ġconstit utes +aw k +Int erest +Ġinter im +Ġbreak through +ĠC her +Ġpro sec +ĠD j +ĠM T +Res p +ĠP T +Ġs perm +ed it +B T +Lin ux +count ry +le ague +Ġd ick +Ġo ct +Ġinsert ing +Ġsc ra +ĠBrew ing +Ġ19 66 +Ġrun ners +Ġpl un +id y +ĠD ian +Ġdys function +Ġex clusion +Ġdis gr +Ġincorpor ate +Ġrecon c +Ġnom inated +ĠAr cher +d raw +achel or +Ġwrit ings +Ġshall ow +Ġh ast +ĠB MW +ĠR S +Ġth igh +Ġ19 63 +Ġl amb +Ġfav ored +ag le +Ġcool er +ĠH ours +ĠG U +ĠOrig in +Ġglim pse +---------------- ---- +L im +Ġche ek +Ġj ealous +- ' +Ġhar ness +ĠPo ison +Ġdis abilities +ne apolis +Ġout look +Ġnot ify +ĠIndian apolis +Ġab rupt +ns ic +Ġenc rypted +Ġfor fe +reat h +Ġr abb +Ġfound ations +Ġcompl iment +ĠInter view +ĠS we +Ġad olesc +Ġmon itors +ĠSacrament o +Ġtime ly +Ġcontem pl +Ġposition ed +Ġpost ers +ph ies +iov ascular +v oid +ĠFif th +Ġinvestig ative +OU N +Ġinteg rate +ĠIN C +ish a +ibl ings +ĠRe quest +ĠRodrig uez +Ġsl ides +ĠD X +Ġfemin ism +Ġdat as +Ġb end +ir us +ĠNig eria +F ox +Ch ange +Ġair plane +ĠLad en +Ġpublic ity +ixt y +Ġcommit ments +Ġaggreg ate +Ġdisplay ing +ĠAr row +Ġ12 2 +Ġrespect s +and roid +s ix +ĠSh a +Ġrest oration +) \ +W S +oy s +Ġillust rate +with out +12 6 +ĠâĶ Ĥ +Ġpick up +n els +Ġ .... +f ood +ĠF en +) ? +Ġphenomen a +Ġcompan ions +ĠW rite +Ġsp ill +Ġbr idges +ĠUp dated +ĠF o +Ġinsect s +ASH INGTON +Ġsc are +il tr +ĠZh ang +Ġsever ity +Ġind ul +14 9 +ĠCo ffee +Ġnorm s +Ġp ulse +ĠF T +Ġhorr ific +ĠDest roy +ĠJ SON +Ġo live +Ġdiscuss es +R est +E lect +ĠW inn +ĠSurv iv +ĠH ait +S ure +op ed +Ġro oted +ĠS ke +ĠBron ze +Ġl ol +Def ault +Ġcommod ity +red ited +Ġliber tarian +Ġforb idden +Ġgr an +à ¨ +Ġl ag +en z +dri ve +Ġmathemat ics +Ġw ires +Ġcrit ically +Ġcarb ohyd +ĠChance llor +ĠEd die +Ġban ning +ĠF ri +Ġcompl ications +et ric +ĠBangl adesh +Ġband width +St op +ĠOrig inally +Ġhalf way +yn asty +sh ine +Ġt ales +rit ies +av ier +Ġspin ning +ĠWH O +Ġneighbour hood +b ach +Ġcommer ce +ĠS le +B U +Ġentreprene ur +Ġpecul iar +ĠCom ments +f re +3 20 +IC S +Ġimag ery +ĠCan on +ĠElect ronic +sh ort +( ( +D ig +Ġcomm em +u ced +Ġincl ined +ĠSum mon +Ġcl iff +ĠMed iterranean +Ġpo etry +Ġprosper ity +ĠRe ce +Ġp ills +m ember +Ġfin ale +un c +ĠG ig +ä ½ +Ġl od +Ġback ward +- + +ĠFor ward +Ġth ri +s ure +Ġso ap +ĠF X +R ES +ĠSe xual +oul os +Ġfool ish +Ġright eous +Ġco ff +terror ism +ust ain +ot er +Ġab uses +ne xt +Ġab usive +Ġthere after +Ġprohib ition +ĠS UP +Ġd ip +Ġr ipped +Ġinher ited +Ġb ats +st ru +G T +Ġflaw ed +ph abet +Ġf og +do ors +Ġim aging +Ġdig its +ĠHung ary +Ġar rog +Ġteach ings +Ġprotocol s +ĠB anks +à ¸ +p ound +ĠC urt +." ) +. / +Ġex emption +end ix +ĠM ull +Ġimpro ves +ĠG amer +d imensional +I con +ĠMarg aret +St atus +d ates +Ġint ends +Ġdep ict +Ġpark ed +J oe +ĠMar ines +chn ology +! ). +Ġjud ged +Ġwe ights +R ay +Ġapart ments +he ster +Ġrein force +Ġoff ender +occ up +Ġs ore +e pt +ĠPH P +ĠB row +Ġauthor ization +ĠR isk +ĠDel aware +ĠQ U +Ġnot ifications +Ġsun light +Ġex clude +d at +Ġm esh +ĠSud an +Ġbelong ed +Ġsub way +Ġno on +ĠInter ior +ol ics +ĠL akers +Ġc oding +Dis claimer +Cal if +O ld +Ġdis l +???? ? +Ġconfir ms +Ġrecruit ment +Ġhom icide +Cons ider +ĠJeff rey +ft y +} ; +Ġobject ion +do ing +ĠLe o +W ant +Ġgl ow +ĠClar ke +ĠNorm an +Ġver ification +Ġpack et +ĠForm ula +Ġpl ag +es ville +Ġshout ing +Ġo v +ĠR EC +ĠB ub +Ġn inth +Ġener g +Ġvalid ity +Ġup s +j ack +Ġneighbor ing +ĠN ec +ew orks +ĠH ab +are z +Ġsp ine +Ġevent ual +ĠLe aders +ĠC arn +Ġprob ation +Ġrom ance +ms g +ĠMechan ical +ER Y +R ock +Ġpart isan +N ode +ass ets +min ent +Ġforeign ers +Ġtest ify +ĠUs ually +l ords +ĠG ren +ĠPow ell +BI L +Ġs r +Ġadd ict +Ġshell s +Ġs igh +ĠY ale +tern ity +Ġ7 50 +E U +ĠR ifle +Ġpat ron +em a +ĠB annon +an ity +Ġtrop ical +ĠV II +c ross +Every thing +ĠIS O +Ġhum ble +ass ing +ĠF IG +Ġupd ating +ys on +Ġcal cium +Ġcompet ent +Ġste ering +Pro t +ĠS Y +ĠFin als +ĠR ug +15 9 +13 7 +ĠG olf +Ġ12 6 +Ġaccommod ation +ĠHug hes +Ġaest hetic +art isan +ĠTw ilight +Ġpr ince +ĠAgric ulture +ĠDis co +Ġpreced ent +Ġtyp ing +author ized +O ption +ĠA ub +l ishes +ach t +m ag +P eter +ĠU FO +mont on +ĠL ith +Ġa rom +Ġsec uring +Ġconf ined +priv ate +Ġsw ords +Ġmark ers +Ġmetab olic +se lect +ĠCur se +ĠO t +g ressive +Ġinc umb +ĠS aga +Ġpr iced +Ġclear ance +Cont ent +Ġdr illing +Ġnot ices +Ġb ourgeois +Ġv est +Ġcook ie +ĠGuard ians +ry s +in yl +Ġ12 4 +Ġpl ausible +on gh +ĠOd in +Ġconcept ion +ĠY uk +ĠBaghd ad +ĠFl ag +Aust ral +ĠI BM +Ġintern ationally +ĠWiki Leaks +I ED +Ġc yn +Ġcho oses +ĠP ill +Ġcomb ining +Ġrad i +ĠMoh ammed +def ense +atch ing +Sub ject +ic iency +Fr ame +Ġ{ " +Ġche ss +Ġtim er +19 0 +Ġt in +Ġord inance +emet ery +Ġacc using +Ġnotice able +Ġcent res +Ġl id +ĠM ills +img ur +Ġz oom +erg ic +Ġcomp ression +pr im +f ind +Ġsur g +Ġp and +ĠK ee +ĠCh ad +cell ence +oy le +Ġsocial ism +ĠT ravis +ĠM Hz +Ġgu ild +ALL Y +ĠSub scribe +ĠRel ated +Ġoccur rence +itch ing +Ġfict ional +Ġcr ush +ĠE A +c od +m ix +ĠTri ple +Ġretrie ve +Ġstimul us +Ġpsych iat +ĠDo or +Ġhomosexual ity +Ġelement ary +Ġcell ular +id ian +ĠL aun +Ġintrig uing +Ġfo am +ĠB ass +id i +its u +Ġass ure +Ġcongr at +Ġbusiness man +ĠBo ost +cl ose +Ġl ied +Ġsc iences +ĠO mega +ĠG raphics +Ġ< = +sp oken +Ġconnect ivity +S aturday +ĠAven gers +Ġto ggle +Ġank le +Ġnational ist +mod el +ĠP ool +ophob ia +V ar +ĠM ons +ator ies +Ġaggress ively +C lear +For ge +act ers +Ġhed ge +Ġpip es +Ġbl unt +Ġs q +Ġremote ly +W ed +as ers +Ġref riger +Ġt iles +Ġresc ued +Ġcompr ised +ins ky +Ġman if +avan augh +Ġprol ifer +Ġal igned +x ml +Ġtri v +Ġcoord ination +ĠP ER +ĠQu ote +13 4 +b f +ĠS aw +Ġtermin ation +Ġ19 0 +Ġadd itions +Ġtri o +Ġproject ions +Ġpositive ly +Ġin clusive +Ġmem br +19 90 +old er +Ġpract iced +ink le +Ar ch +Ġstar ters +ari us +Ġinter mediate +ĠBen ef +ĠK iller +Ġinter ventions +ĠK il +ĠF lying +In v +Ġprem ature +Ġpsych iatric +Ġind ie +Ġcoll ar +ĠRain bow +af i +Ġdis ruption +ĠFO X +cast ing +Ġmis dem +c ro +Ġw ipe +ard on +Ġb ast +ĠTom my +ĠRepresent ative +Ġbell y +ĠP O +ĠBre itbart +13 2 +Ġmess aging +Sh ould +Ref erences +ĠG RE +ist ical +L P +ĠC av +ĠC razy +Ġintu itive +ke eping +ĠM oss +Ġdiscont in +ĠMod ule +Ġun related +ĠPract ice +ĠTrans port +Ġstatist ically +orn s +Ġs ized +p u +Ġca f +ĠWorld s +ĠRod gers +ĠL un +ĠCom ic +l iving +Ġc ared +Ġclim bed +) { +Ġconsist ed +Ġmed ieval +fol k +Ġh acked +Ġd ire +ĠHerm ione +Ġt ended +ce ans +D aniel +w ent +Ġlegisl ators +Ġred es +g ames +Ġg n +am iliar +Ġ+ + +gg y +th reat +Ġmag net +Ġper ceive +Ġz ip +Ġindict ment +Ġcrit ique +g ard +ĠSaf e +ĠC ream +Ġad vent +ob a +Ġv owed +ous ands +Ġsk i +Ġabort ions +u art +Ġstun ned +Ġadv ancing +Ġlack ed +Ġ\ " +Ġsch izophren +Ġeleg ant +Ġconf erences +Ġcance led +ĠHud son +ĠHop efully +Ġtr ump +Ġfrequ encies +Ġmet eor +ĠJun ior +ĠFle et +ĠMal colm +ĠT ools +Ġ ........ +Ġh obby +ĠEurope ans +Ġ15 00 +ĠInt o +Ġs way +ĠApp ro +ĠCom pl +Comm unity +Ġt ide +ĠSum mit +ä » +Ġinter vals +ĠE ther +Ġhabit at +ĠSteven s +lish ing +ĠDom ain +Ġtrig gers +Ġch asing +Ġchar m +ĠFl ower +it ored +Ġbless ing +Ġtext ures +F ive +Ġliqu or +R P +F IN +Ġ19 62 +C AR +Un known +Ġres il +ĠL ily +Ġabund ance +Ġpredict able +r ar +Ġbull shit +le en +che t +M or +M uch +ä ¹ +Ġemphas ized +Ġcr ust +Ġprim itive +Ġenjoy able +ĠPict ures +Ġteam mate +pl er +ĠT ol +ĠK ane +Ġsummon ed +th y +ram a +ĠH onda +Ġreal izing +Ġquick er +Ġconcent rate +cle ar +Ġ2 10 +ĠErd ogan +ar is +Ġrespond s +ĠB I +Ġelig ibility +Ġpus hes +ĠId aho +Ġagg rav +Ġru ins +ur ations +Ġb ans +Ġan at +sh are +Ġgr ind +h in +um en +Ġut ilities +ĠYan kees +Ġdat abases +ĠD D +Ġdispl aced +Ġdepend encies +Ġstim ulation +h un +h ouses +ĠP retty +ĠRaven s +ĠTOD AY +Ġassoci ates +Ġthe rape +cl ed +Ġde er +Ġrep airs +rent ice +Ġrecept ors +Ġrem ed +ĠC e +Ġmar riages +Ġball ots +ĠSold ier +Ġhilar ious +op l +13 8 +Ġinherent ly +Ġignor ant +Ġb ounce +ĠE aster +REL ATED +ĠCur rency +E V +ãĥ ŀ +ĠLe ad +Ġdece ased +B rien +ĠMus k +J S +Ġmer ge +heart ed +c reat +m itt +m und +ĠâĢ ĭ +ĠB ag +Ġproject ion +Ġj ava +ĠStand ards +ĠLeon ard +Ġcoc onut +ĠPop ulation +Ġtra ject +Ġimp ly +Ġcur iosity +ĠD B +ĠF resh +ĠP or +Ġheav ier +ne ys +gom ery +Ġdes erved +Ġphr ases +ĠG C +Ġye ast +d esc +De ath +Ġreb oot +Ġmet adata +IC AL +Ġrep ay +ĠInd ependence +Ġsubur ban +ical s +Ġat op +Ġall ocation +gener ation +ĠG ram +Ġmoist ure +Ġp ine +ĠLiber als +Ġa ides +Ġund erest +ĠBer ry +Ġcere mon +3 70 +ast rous +ĠPir ates +Ġt ense +ĠIndust ries +ĠApp eals +ĠN ear +Ġè£ı ç +Ġlo vers +ĠC AP +ĠC raw +Ġg iants +Ġeffic acy +E lement +ĠBeh avior +ĠToy ota +Ġint est +P riv +A I +Ġmaneu ver +Ġperfect ion +Ġb ang +p aper +r ill +Ge orge +b order +in ters +ĠS eth +Ġcl ues +ĠLe vi +ĠRe venue +14 7 +Ġv apor +Ġfortun ate +Ġthreat ens +Ġve t +Ġdepend ency +ers ed +art icle +ĠBl izzard +Ġch lor +Ġmin us +ĠB ills +Ġcryptoc urrency +Ġmetabol ism +ter ing +Ġp estic +step s +ĠTre asure +ract ed +ĠConst ant +Ġtem p +13 9 +ĠDet ective +ur ally +Ġrecover ing +Ġcort ex +Ġ14 4 +cl osed +Ġprejud ice +aun ted +Ġstorm s +ĠN OW +Ġmach inery +Add ress +Ġcompe lled +27 0 +Ġdesp air +b ane +Ġveget able +Ġbed s +Lear n +Ġcolor ful +Ġsp ike +Ġmarg ins +Ġsymp athy +Ġworks hop +ĠC BC +S at +Ġburn s +ĠG ender +Ġ12 9 +ĠC able +Ġdeb ts +ĠThe resa +Ġreflect ing +Ġa irst +Ġr im +ram id +Ġweakness es +W rit +ogg le +t i +ĠCh arge +Ġwe ighed +Ġ( . +Ġl aughter +Ġrou ter +ĠDemocr acy +D ear +Ġhas ht +Ġd y +Ġhint s +run ning +Ġfin ishes +ar us +M ass +res ult +asc us +Ġv intage +Ġcon qu +Ġwild ly +ac ist +Ġl ingu +Ġprot agonist +st rom +te enth +ĠSol o +m ac +f illed +Ġre nown +it ives +Ġmot ive +ĠAnt ar +ĠM ann +ĠAd just +Ġrock ets +Ġtrou bling +e i +Ġorgan isms +ass is +Christ ian +Ġ14 5 +ĠH ass +Ġsw all +Ġw ax +ĠSurv ival +V S +ĠM urd +v d +stand ard +Ġdrag ons +Ġacceler ation +r ational +f inal +Ġp aired +ĠE thereum +Ġinterf aces +Ġres ent +Ġartif acts +Å « +are l +Ġcompet itor +ĠNich olas +ĠSur face +c pp +ĠT ot +Ġeconom ically +Ġorgan ised +Ġen forced +in ho +Ġvar ieties +Ġab dom +ĠBa iley +id av +ĠSal v +p aid +Ġalt itude +ess ert +ĠG utenberg +are a +op oulos +Ġprofess ors +igg s +ĠF ate +he y +Ġ3 000 +D ist +Ġtw ins +c ill +ĠM aps +Ġtra ps +Ġwe ed +ĠK iss +Ġy oga +Ġrecip ients +ĠWest minster +Ġpool s +ĠWal mart +18 8 +ĠSchool s +att ack +ĠAR M +par agraph +W arning +j l +Ġself ish +anche z +ĠHe ights +F re +ĠS oph +Ġ -------------------------------- +t ml +33 3 +Ġraid s +Ġsatell ites +KE Y +Ġlast s +Ñ Ĥ +In s +ĠD ame +Ġunp redict +// / +gh ai +Ġart illery +Ġcru ise +Ġg el +ĠCabin et +Ġbl ows +ĠE sp +Ġprox imity +ot he +ĠSk ills +ĠU pper +ob o +ĠN DP +Ġenjoy s +Ġrepe ating +ĠConst ruction +ĠQuest ions +H illary +Ġu int +Ġprocess ors +ĠGib son +ĠMult iple +q a +ĠB om +ĠM iles +vent ional +Ġhur ts +s kin +ĠA IDS +Ġadvis ers +ĠR oot +Ġmethod ology +ĠD ale +Ġdet on +ĠKnow ledge +sequ ently +Ġ12 1 +Ġconnect s +C y +ĠD anger +Ġcontribut ors +ĠB ent +Ġbr ass +ĠGun s +int o +ĠFort une +Ġbro ker +bal ance +Ġlength s +Ġv ic +Ġaver aging +Ġappropri ately +ĠCamer a +Ġsand wich +ĠCD C +Ġcoord inate +Ġnav ig +Ġgood ness +l aim +Ġbra ke +Ġextrem ist +ĠW ake +ĠM end +ĠT iny +ĠC OL +ĠR F +ĠD ual +ĠW ine +C ase +Ġref ined +Ġl amp +L ead +Ġb apt +ĠCar b +ĠS add +ĠMin neapolis +PD F +Ear ly +ĠH idden +I ts +ĠT IME +Ġp ap +Ġcommission ed +ĠF ew +ĠCol ts +ĠB ren +Ġbot hered +Ġlike wise +Ex per +ĠSch w +c ry +n n +ĠM itch +im on +M G +b m +UM P +r ays +Ġregist ry +Ġ2 70 +ach ine +re lla +ant ing +00 000 +Ġru ined +sp ot +Ġt a +Ġmaxim ize +Ġincon ven +D ead +H uman +En abled +ĠMar ie +Ġch ill +ĠParad ise +Ġstar ring +ĠLat ino +ĠProt ocol +ĠE VER +Ġsuppl iers +m essage +ĠBro ck +Ġser um +âĸĪâĸĪ âĸĪâĸĪ +Ġen comp +Ġamb ition +ues e +Ġar rows +And rew +Ġanten na +Ġ19 61 +ĠB ark +Ġb ool +ãĤ ª +ĠSt orage +Ġrail way +Ġtoug her +ĠC ad +Ġwas hing +P y +' ] +em bed +ĠMem phis +ack le +Ġfam ously +ĠF ortunately +ov ies +Ġmind set +Ġsne ak +ĠD h +RA W +ĠSim pson +Ġliv est +Ġland mark +Ġc ement +L ow +Ġthr illed +ĠCour se +in el +Ġch uck +id ate +gl obal +Ġwh it +Ġ � +ad ays +s ki +ĠS V +Ġvir uses +30 6 +ĠResp ons +Ġthe aters +ĠBr anch +ĠGene va +ĠM K +Ġunbel iev +Ġcommun ist +Orig inal +ĠRe ceived +ĠTrans fer +ĠAr g +In put +ĠStr ategy +Ġpal ace +the ning +D ri +Ġsent encing +umbn ail +Ġp ins +re cy +Ġs iblings +Get ting +ĠB U +ĠNorth west +Ġprolong ed +ĠSak ura +C omb +ĠB our +Ġinadequ ate +ĠK ash +Ġus ername +ĠImpro ve +Ġbatt ling +ĠM AC +Ġcurric ulum +Ġs oda +ĠC annon +Ġsens ible +sp ons +De cember +Ġw icked +ĠP engu +Ġdict ators +ĠHe arts +og yn +Ġsimilar ities +ĠSt ats +Ġh ollow +it ations +": [ +Ġh over +ĠList en +s ch +S und +Ġc ad +ĠPar ks +Ġl ur +Ġhy pe +ĠL em +N AME +is ure +Fr iday +Ġshoot s +Ġclos es +Ġd b +ĠR idge +ĠDiff erent +Ġrepl ies +ĠBroad way +op ers +Ġint oler +ĠZe us +akes pe +Ġpropri etary +Ġrequest ing +Ġcontro llers +ĠM IN +im edia +be cca +Ġexp ans +Ġoil s +B ot +ĠCh and +Ġpr inter +Ġto pped +ĠP OL +ĠEar lier +S ocial +av in +Ġdecre ases +ĠSe b +Ġspecific ations +ĠBl ast +ĠK urt +Ġfre el +B rown +Ġdil ig +ro e +ĠPro blem +ĠQu ad +Ġdecent ral +ĠV ector +an ut +Ġplug ins +ĠGreg ory +Ġfuck ed +el ines +ĠAmb assador +t ake +Ġcle ans +ong yang +An onymous +st ro +" } +al ine +ĠO dd +ĠE ug +2 16 +Ġbo il +ĠP owers +Ġnurs es +Ob viously +ĠTechn ical +Ġexceed ed +OR S +Ġextrem ists +Ġtr aces +ex pl +Ġcom r +ĠS ach +) / +Ġm asks +Ġsc i +B on +Ġreg ression +we gian +Ġadvis or +it ures +ĠV o +ex ample +ĠInst ruct +Ġs iege +Ġredu ctions +pt r +Ġstat utory +Ġrem oves +Ġp uck +red its +Ġbe e +Ġsal ad +Ġpromot ions +ĠJosh ua +with standing +ET H +ĠCh a +im us +Ġexpend iture +aun ting +Ġdelight ed +Ġ15 5 +be h +Ġcar pet +ĠSp art +Ġj ungle +l ists +Ġbull ying +ĠNob el +ĠGl en +Ġreferen ced +Ġintrodu ces +se in +Ġcho pped +gl ass +ĠW rest +Ġneutral ity +Ġâ Ļ +Ġinvestig ator +Ġshel ves +Ġun constitutional +Ġreprodu ction +Ġmer chant +m ia +Ġmet rics +Ġexplos ives +ĠSon ia +Ġbod ily +Ġthick ness +Ġpredomin antly +ĠAb ility +Ġmon itored +IC H +Ġ] . +ĠMart inez +Ġvis ibility +Ġqu eries +Ġgen ocide +ĠWar fare +Qu ery +Ġstud ios +Ġemb ry +Ġcorrid or +Ġclean ed +com plete +ĠM H +Ġenroll ment +ING S +Ġimpact ed +Ġdis astrous +ĠY un +ĠCl aire +ĠBas ically +y t +uster ity +Ġindirect ly +w ik +Ġd od +ĠCar r +Ġam p +Ġprohib it +ĠIn itial +ĠR d +ij i +Ġeduc ate +c orn +i ott +ĠBeaut y +Ġdetect ive +ĠCon n +s ince +Ġst agger +Ġob ese +Ġb ree +olog ic +is se +walk er +Ġbl ades +Ġlaw ful +fun c +ĠBeh ind +Ġappet ite +Ġ( * +Ġt ennis +Ġoff spring +Ġj ets +Ġstruct ured +Ġafore mentioned +N ov +Ġsc aling +f ill +Ġst ew +Ġcur b +ĠStep han +ed In +S F +ob ic +é ŃĶ +ou g +ĠM M +Ġgen etically +ope z +13 6 +Ġu mb +anc ers +Ġcoh ort +Ġmerch andise +Ġimp osing +ĠLegisl ature +ĠArch ive +iv ia +ĠN aval +Ġoff ences +Ġmir acle +Ġsn apped +Ġf oes +Ġextensive ly +ĠR af +Ġc ater +ed ience +K it +ĠB in +Ġrecomm ends +ĠC ities +Ġrig id +ĠRE AD +ĠNob le +ĠT ian +Ġcertific ates +ant is +o iler +ĠBudd hist +d id +Ġsurvey ed +Ġdown ward +Ġprint s +ĠMot ion +ron ics +ĠS ans +oss ibly +u ctions +Ġcolon ies +ĠDan ish +un it +Ġsp oil +Ġadvis ory +ber ries +Pl an +Ġspecific ation +op hers +ĠRes ource +Ġsh irts +prising ly +commun ications +Ġtriv ial +Ġmention ing +ise xual +Ġsupp lements +Ġsuper vision +B P +v or +Ġw it +Ġco oldown +Ġplaint iff +ĠReview s +ĠS ri +ĠM int +ĠSug ar +Ġafter ward +ĠPri est +ĠInvest ment +og ene +ĠT aking +Ġstretch ing +Ġinflamm ation +ĠTe hran +Ġl ining +Ġfree zing +ĠEnt ity +Ġins piring +spe cial +pr ice +Ġsu e +ĠP orter +oun ge +ET A +ĠD erek +ĠLu is +u o +ym ph +Ġex terior +ih il +ĠAsh ley +in ator +Ġnut rients +ĠTh rones +Ġfin ances +ĠIn spect +Ġspe cially +ĠRequ ired +ĠP TS +ĠViol ence +oint ed +sh ots +Ġex cerpt +co on +IN S +ĠG ri +Ġrecogn ised +We ek +You ng +Ġv om +is le +ĠCur ry +ĠBudd h +Ġnot ebook +Ġd urable +/ ? +ĠG ad +ĠP upp +Ġforg ive +p ark +Ġpersonal ities +an alysis +cl amation +Ġelev ator +Ġware house +ĠR ole +un n +Ġillust ration +ĠSc an +Ġatmosp heric +Im port +AN C +rict ed +f u +01 0 +Ġar che +Ġreward ed +akespe are +Ġintern ally +ĠR BI +alk er +Ġeleph ant +ow itz +ĠP izza +Ġbip artisan +é s +Ġslow ed +ĠSt ark +Ġover ride +OU S +Ġ3 20 +undred s +ĠDe ck +ĠC ensus +be e +14 6 +ot or +Ġ ip +Ġu b +oc ations +ĠBut ton +r ice +Ġc ripp +ff f +Ġorig inated +Ġoverwhel med +app a +Ġfore most +âĢ ij +ĠL EG +re lease +eat ured +at ches +Ġre ps +Ġl ending +ĠRe ference +ĠCl ient +16 5 +vent h +Com plete +ĠPat rol +Ġsw orn +c am +Ġshut tle +ĠR alph +Ġh ometown +- , +on al +ĠB P +å ı +Ġpersu ade +ĠAlex and +Ġcomb ines +Ġv ivid +ĠL ag +Ġenc oding +Ġsal vation +w en +ĠRec overy +i ya +Un iversity +ĠB iden +Ġbud gets +ĠTex ans +f its +Ġhon ored +Ġp ython +T D +## # +cl one +Ġbl ink +ĠL iquid +Ġunemploy ed +Ġcl ashes +ĠCoun sel +Ġdirect ing +Ġpun ct +ĠFal cons +Ġsh ark +ĠDam ascus +Ġje ans +Ġemb ark +Ġse ize +Ġup wards +2 80 +ĠE z +ĠAny thing +Ġex otic +l ower +ĠCreat or +ĠU m +Ġsubur bs +ber ger +ĠW end +Ġm int +ĠX X +ĠD ro +Ġsuff ers +Ġher b +t ree +Ġfrag ile +Ġflood ed +ĠAl cohol +ole an +ny der +ĠK O +F ram +Ġ13 6 +Ġow ed +ĠMe lee +ĠH ash +Ġwh isk +Ġsu do +r r +Qu ick +app ro +Ġi i +ĠEx amples +he e +Ġpromot es +per ature +k ar +ĠHon or +Ġs odium +ĠL if +ros so +intend ent +Ġcorrespond ent +F ound +sec ret +Ġident ifies +ag ne +Ġl ou +ĠP P +Ġcoinc idence +m ove +Ġmilit ia +Ġinf iltr +ĠPrim ary +Ġpitch ing +ĠI b +ĠGO OD +ãĤ ¸ +ĠW izards +ir al +ĠVen us +R R +ĠâĢ ķ +ĠCase y +Ġsad ly +Ġadm ire +Ġembarrass ed +c b +M el +Ġtub es +Ġbeaut ifully +ĠQueens land +Bel ow +re z +qu et +ple asant +Ġ « +C amp +Ġdec isive +19 98 +ĠL amb +ut ton +h n +ĠJ agu +au nder +ĠC ord +Ġcl erk +Ġca ffe +Ġwip ed +Ġre im +ĠMount ains +Ġimprison ed +Ġdevelop s +ĠP ra +Ġmodel ing +Any one +ance l +ĠS it +Ġshield s +Ġl awn +Ġcard iovascular +Ġdemonstr ating +Ġpar se +ĠIsrael is +Ġeuro s +14 3 +Ġgl orious +ins ki +ec d +Ġcondition ing +Ġhel pless +Ġmicro sc +ĠHar bor +Ġst akes +Ġ2 60 +Ġun equ +ĠFl oyd +Ġd amp +Ġappar atus +ĠLaw s +Ġcoun ters +Ġindu ce +at able +ĠAh med +Ġsl am +N ovember +Ġpers ist +Ġim minent +á n +Ġsh red +Ġph ases +ĠEd monton +ĠArm strong +ĠMe et +ĠK itty +Ñ Ģ +c irc +ĠAd ult +Ġa rose +ĠX en +D an +g ow +Ġsuper f +ĠAd mir +Ġend ure +Ġkey word +yr us +Ġy arn +Ġpath way +ĠHop kins +mid t +Ġcens orship +d ependent +Ġinstruct or +S ources +Ġto e +Ġball oon +N ob +Ġsw ear +ĠCast ro +Ġgl oss +ĠK avanaugh +Ġremark ably +Ph otos +ĠN om +ĠS outheast +y ers +Ġvalid ation +Ġcann on +ĠVict ory +ĠPier re +Ġcaut ious +Aud io +Ġf etch +ĠG ift +ĠH yp +Ġrem edy +Z E +Ġsc ent +Ġbe ard +ĠR ut +- " +Ġpat ents +H y +Ġun just +Ġpot ato +Ġforth coming +Ġche f +ĠR ift +aff e +ĠR OM +ĠL aunch +Ġp ads +ĠNe o +Ġon set +Ġsquee ze +s afe +Ġpref ix +ĠT M +ĠN early +ĠClin ical +ĠM ental +ot iation +ĠUn ic +ant ry +ĠC ir +Ġep it +à ¦ +Ġextract ed +verse ly +ri ad +Ġstr ains +Ġto ps +Ġpo em +ĠRand y +ĠMap le +TH ER +up iter +ĠSS D +ļ é +Ġun con +per ing +Ġsle pt +in ers +Ġunder water +ĠEv idence +g one +20 5 +Ġhistor ians +Ġsynt hesis +Ġf rog +b asketball +Ġvibr ant +Ġsub ord +Ġ3 65 +ĠD ial +Ġcooper ate +HA HA +Ġgreet ed +15 8 +Ġj azz +Ġinto x +ĠWalk ing +Ġsuper visor +ĠF usion +ĠMer cedes +s end +H am +s d +n l +Ġtour s +ĠF IFA +Ġcul p +g d +30 4 +Ġple as +Ġillust rates +ĠColomb ia +Ġhighlight ing +ĠSum mary +Ġexp osing +ĠD ru +Ġir ony +r itional +ĠCar roll +ĠEll is +P ict +ĠR apt +Ġad apter +Ġun m +Ġcor pse +Ġceleb rities +D en +at um +ĠAp ocalypse +ĠW ag +lin ing +Ġhorm ones +R ub +ĠX i +ĠV aults +20 8 +alky rie +inos aur +Ġfeed s +v ity +Ġdefe ating +W ait +Ġemphas ize +ĠSteel ers +yr inth +le ys +ĠWhe never +Current ly +ĠCl ock +Ġcollect ively +any on +ĠJ P +Ġment ality +Ġdownload s +Ġsurround ings +ĠBarn es +Ġflags hip +Ġindic ators +Ġgra pp +Jan uary +ĠElement al +ĠAthen a +ib al +Ġs ights +Ġcap ita +ĠTreat y +Ġvo iced +ĠG az +let te +Ġy a +Ġexp ired +Leg end +H ot +n ature +Ġunst able +Ġ2 80 +à º +Com ment +AL E +Ġquest s +Ġhand ler +n is +Ġvers atile +Ġconce al +enge ance +ĠInter active +Ġobs essed +ĠDog s +Ġcr acked +S ound +s v +ĠD ylan +ro ads +f x +ĠCath olics +ĠH ag +Ġsl ammed +Ġgl owing +s ale +Ġtiss ues +ĠCh i +ne e +Ġc her +s ic +ur rection +Ġb acon +ul atory +) ." +Ġir regular +FOR M +ass ed +Ġintention al +Ġcompens ate +ĠSpe aking +ĠS ets +15 3 +Ġconvent ions +b ands +em ade +Ġe cc +ĠWin ston +ĠAssass in +ĠBelg ian +Ġdepend ence +Ġnic he +Ġb ark +ĠJ azz +Ġdisadvant age +Ġgas oline +Ġ16 5 +çļ Ħ +ess a +mod ule +ang ular +O Y +ĠTreat ment +it as +ol ation +ĠArn old +Ġfe ud +ĠN est +Ġthe atre +ew ater +Ġmin ors +olic y +ĠH aven +div ision +Ġtr unk +F ar +ĠP ull +Ġcapt uring +Ġ18 00 +ĠTe en +Ġex empl +Ġclin ics +ĠB urg +Ġsubst it +Ġpay load +ĠL av +ĠT roy +ĠW itness +Ġfrag ments +Ġpass words +Ġg ospel +ĠG in +Ġten ants +ol ith +S ix +Pre vious +ĠAg es +ĠDar win +Ġbl at +Ġem pathy +sm ith +b ag +ĠE cho +ĠC amb +ĠM add +ĠB oo +Ġred e +ĠBurn ing +Ġsmooth ly +ĠAd rian +ĠV ampire +ĠMon sters +ste am +Sty le +M a +re a +ĠD war +aly st +urs or +Ġelim ination +Ġcrypt o +ch t +ĠE ternal +âĢ¦ ] +ĠS orce +I ll +N ER +Ġu h +Con clusion +w age +Ġresp ir +Ġrem inis +het ical +Ġg y +Ġutil ized +ic idal +Ġ19 00 +Ġhun ters +ĠSw an +ĠRe act +Ġvis itor +ĠThanks giving +30 8 +Post s +Ġh ips +19 97 +om ers +Ġkn ocking +ĠVeh icle +Ġt il +Ġ13 8 +Ġm i +ĠInvest igation +ĠKen ya +Ġcas ino +Ġmot ives +Ġreg ain +re x +Ġweek ends +Ġstab bed +bor o +Ġexplo ited +ĠHA VE +ĠTe levision +c ock +Ġprepar ations +Ġende av +ĠRem ote +ĠM aker +ĠPro du +ĠEv an +Ġinform ational +ĠLouis ville +15 4 +ĠDream s +Ġpl ots +ĠRun ner +Ġhur ting +Ġacad emy +ĠMont gomery +n m +ĠL anc +ĠAl z +2 10 +el ong +Ġretail er +Ġar ising +Ġrebell ion +Ġbl onde +play ed +Ġinstrument al +C ross +Ġret ention +Ġtherape utic +Ġse as +Ġinfant ry +ĠCl int +Ġprompt ing +Ġbit ch +Ġst ems +ĠK ra +Ġthe sis +ĠB og +ru ed +Ġk ings +Ġcl ay +ific ent +ĠY ES +ĠTh ing +ĠCub s +vey ard +els h +in arily +ĠE y +ĠRoll ing +Ġev olving +Ind ia +Ġrecogn izes +Ġgrad uation +is ers +Ġfert ility +ĠMil an +Comm and +Ġbox ing +Ġ19 43 +Ġgl uten +ĠEm ir +Ġid ol +Ġcon ceived +ĠCre ation +Mer it +udd y +uss ions +ĠLie utenant +iet al +Ġunch anged +ĠSc ale +ĠCrime a +ball s +ator ial +Ġdepth s +Ġempir ical +Ġtrans m +Ġuns afe +miss ible +com fort +15 6 +Ġmechan ic +00 2 +l ins +Ġsm oked +P os +Ġslow ing +Ġl av +Tex as +Ġche ating +ĠMet ropolitan +eth yl +Ġdiscover ing +as se +Ġpen cil +ĠPy ongyang +Ġclos et +ĠShe et +ĠEnt ry +ou stic +Ġmy st +er ate +ari at +Ġminer als +Ġmusic ian +ĠP ul +ĠM az +24 9 +Ġper missions +Ġ iv +en ary +ick ers +ĠB ing +he a +en able +Ġgri ev +Ġassert ed +ĠColon el +Ġaff idav +w o +Ġse ated +ĠR ide +Ġpaint ings +ĠP ix +Ġ13 7 +ish i +umb ai +g otten +ĠEar l +Ġin ning +Ġc ensus +Ġtrave lled +ĠCons ult +18 5 +b ind +Ġsimpl icity +Ġoverlook ed +ĠHelp ful +Ġmon key +Ġoverwhelming ly +Bl ood +ĠFl int +ĠJ ama +ĠPres ent +ĠR age +ĠT A +pt ive +Ġturn out +w ald +ĠD olphins +ĠV PN +Ġon ion +Ġcraft ing +m ma +ĠMerc ury +Ġarr ange +Ġalert s +ĠO T +zb ollah +Ġg ases +ĠRichards on +s al +l ar +Ġfro st +Ġlower ing +Ġacc laim +Ġstart ups +ĠG ain +ess ment +Ġguard ian +äº º +ĠP ie +ĠL inks +Ġmer its +Ġaw ake +Ġparent al +Ġexceed s +Ġid le +ĠPil ot +Ġe Bay +ĠAc cept +ipe g +C am +ĠK ot +Ġtrad ers +olit ics +unk er +ĠP ale +os i +an mar +Ġ19 47 +ĠF ell +est ial +it ating +G F +ĠS r +if ted +Ġconnect or +ĠB one +ill es +2 60 +h ma +Ġoverl ap +ĠGit Hub +Ġclean er +ĠBapt ist +ĠW AS +Ġlung s +Ñ ģ +ĠB UT +Ġc ite +Ġpit ched +reat ment +Ġtro phies +ĠN u +38 6 +ĠPr ide +Ġattend ees +[ ] +17 9 +Ġspat ial +Ġpri zes +ĠRel igion +Ġshow case +ĠC ategory +vid ia +T arget +Pro perty +? , +Ġf usion +p ie +ĠU CLA +Ġsound track +Ġprin cess +ĠC aval +sh ould +Ġlim bs +Back ground +Ġlone ly +Ġc ores +ĠT ail +she et +Ġ13 2 +R a +ãĤ « +ĠB olt +Ġbook ed +Ġadmin ister +Ġequ als +w y +Ġobserv ing +ĠBar on +ĠAd obe +Ġv irgin +ĠSocial ist +M ove +gh azi +ĠLind a +2 12 +Ġbre wing +Ġmerch ants +bur se +Ġdiv or +Ġmet als +ĠN er +Ġsum s +ĠEn emy +Ġen vision +Ġgrant ing +ĠH oney +ĠSk yrim +Ġsoc io +gr aded +Ġselect ive +W ASHINGTON +Ġ19 48 +ĠSir ius +ĠG ross +act ivity +ĠI van +Ġfur ious +BS D +ĠPre vious +Ġrespons ive +Ġchar itable +Ġle aning +ĠP ew +Ġviol ates +\\\\ \\\\ +ĠCom ing +w ire +Ġpo et +Ġres olutions +comm and +ĠPortug uese +Ġnick name +Ġde af +Feb ruary +Ġrecogn ise +Ġentire ty +Ġseason al +pl aced +ĠTe legraph +Ġmicro phone +our ing +Ġgr ains +Ġgovern ed +Ġpost p +ĠW aters +in ement +Ġund ocumented +ĠCom cast +Ġf ox +Ġassault s +re on +man y +ĠJen kins +ĠAny way +Ġassess ments +Ġdown s +ĠM ouse +Ġsuper b +k t +ĠD ow +Ġtax ation +4 01 +Ġsm iles +Ġundert aken +Ġex h +Ġenthusi astic +Ġtw ent +Ġgovernment al +Ġautonom y +ĠTechn ologies +ĠCh ain +Ġpreval ent +f b +Ġnic otine +og ram +j ob +Ġawa iting +ĠMen u +Ġdep uties +k ov +ish ops +But ton +ĠShan ghai +Ġdies el +ĠD uck +R yan +ĠPC s +N F +j ury +ent e +Ġinacc urate +edd y +Wh atever +Ġshow c +ĠN ad +od us +et r +Ġplaint iffs +ĠW OR +ĠAss ange +Ġpriv at +Ġpremium s +Ġt am +UR L +Ġel ites +ĠR anger +otten ham +ĠH off +ĠAt hens +Ġdefin ite +Ġs ighed +Ġeven ly +2 11 +ĠAm ber +ak ia +Ġmail ing +Ġcr ashing +ĠConfeder ate +ru gged +W al +ĠDep ths +Ġjuven ile +Ġreact or +Introdu ction +ĠDel uxe +19 95 +ĠS anchez +ĠM ead +iv able +: - +ĠPlan ning +ĠT rap +qu in +ĠProt ect +ve red +In formation +Ġkid ney +inn amon +l as +Ġpolic ing +Ġtoler ate +ĠQ i +Ġbi ased +F ort +ĠK i +s ave +Ġprivile ged +Ġbe asts +ĠGl as +ĠC inem +Ġcome back +Sund ay +Ġext inction +h ops +Ġtrans mit +Ġdoub les +ĠFl at +16 7 +Ġdis puted +Ġinjust ice +f oo +V ict +role um +ĠJul ie +Con text +ĠR arity +iss ue +Comp onent +Ġcounsel ing +an ne +d ark +Ġobject ions +u ilt +Ġg ast +Ġpl ac +Ġun used +ãĥ ĩ +ĠT rial +ĠJ as +hed ral +ob b +Ġtempor al +ĠPR O +ĠN W +ĠAnn iversary +L arge +Ġther m +Ġd avid +Ġsystem ic +ĠSh ir +m ut +ĠNe pt +add ress +Ġscan ning +Ġunderstand able +Ġcan vas +C at +ĠZ oo +Ġang els +L O +ĠStat ement +ĠS ig +ov able +ĠA way +sh aring +ocr ats +st ated +Ġweigh ing +N or +w ild +B ey +Ġaston ishing +ĠReyn olds +Ġop ener +Ġtrain er +Ġsurg ical +p n +Ġadjust ing +whe el +Ġf rown +erv ative +Ġsusp end +With in +te in +Ġobst acle +Ġliber ties +ym es +Ġur anium +ans om +an ol +ub a +ĠL oss +Ġa rous +ĠHend erson +W ow +s pl +c ur +ĠÂ Ń +Ġtheir s +Dam age +Ġdownload ing +Ġdisc ern +ĠSt o +ĠFl a +Ġh ath +ĠA j +Ġun pleasant +Europe an +exp ensive +Ġscreens hot +ĠU V +Ġall ied +ĠPers ian +Ġmonop oly +Ġat om +ĠReds kins +"> < +Ġcan cell +Ġcinem a +13 1 +f air +ĠAlf red +Ġd uck +arg s +22 3 +ĠIS I +Ġsign aling +in ar +Ġlaugh s +Ġfor wards +Ġreck less +Ġlisten ers +at ivity +Ġvast ly +n ant +L ess +ĠHun ting +ĠScient ific +IT ED +Ġkn ight +ĠH TC +us a +t mp +Ġr ude +ĠLegend ary +Ġar ises +B ad +ĠCl aim +pe g +Ġreal ities +Th ink +Ġ ° +Ġro de +Ġstri ve +Ġan ecd +Ġshort s +Ġhypot hes +Ġcoord inated +ĠGand hi +ĠF PS +R ED +Ġsuscept ible +Ġshr ink +ĠCh art +Hel p +Ġ ion +de ep +rib es +ĠK ai +ĠCustom er +Sum mary +Ġc ough +w ife +Ġl end +Ġposition ing +Ġlot tery +ĠC anyon +Ġf ade +Ġbron ze +ĠKenn y +Ġbo asts +ĠEnh anced +rec ord +Ġemer gence +Ġa kin +ĠB ert +it ous +âĸ ij +Ġst ip +Ġexch anged +om ore +als h +Ġreserv oir +Ġstand point +W M +Ġiniti ate +Ġdec ay +Ġbrew ery +Ġter ribly +Ġmort al +lev ard +Ġrev is +N I +el o +Ġconf ess +ĠMS NBC +Ġsub missions +Cont roller +Ġ20 2 +ĠR uth +} ); +ĠAz ure +Ġ ." +20 6 +ĠMarket ing +Ġl aund +ien cies +Ġrenown ed +ĠT rou +ĠN GO +ble ms +Ġterr ified +Ġwar ns +Ġper t +Ġuns ure +4 80 +ale z +ult z +ĠOut side +Ġst yl +ĠUnder ground +Ġp anc +Ġd ictionary +Ġf oe +rim inal +ĠNor wegian +Ġj ailed +Ġm aternal +é e +ĠLu cy +c op +Ch o +Ġuns igned +ĠZe lda +ĠIns ider +ĠContin ued +Ġ13 3 +ĠNar uto +ĠMajor ity +16 9 +ĠW o +ãĤ ĵ +Ġpast or +Ġinform al +Ð ½ +an throp +jo in +ãģ Ĺ +it ational +N P +ĠWrit ing +f n +ĠB ever +19 5 +Ġy elling +Ġdr astically +Ġe ject +Ġne ut +Ġth rive +ĠFre qu +ou x +Ġpossess es +ĠSen ators +ĠD ES +ĠSh akespeare +ĠFran co +ĠL B +uch i +Ġinc arn +Ġfound ers +F unction +Ġbright ness +ĠB T +Ġwh ale +ĠThe ater +m ass +ĠD oll +S omething +Ġecho ed +ĠHe x +c rit +af ia +Ġgodd ess +Ġele ven +ĠPre view +ĠAur ora +Ġ4 01 +uls ive +ĠLog an +in burgh +ĠCent ers +ĠON LY +ĠA id +Ġparad ox +Ġh urd +ĠL C +D ue +c ourt +Ġoff ended +Ġeval uating +ĠMatthew s +Ġto mb +Ġpay roll +Ġextra ction +ĠH ands +if i +Ġsuper natural +ĠCOM M +] = +dog s +Ġ5 12 +ĠMe eting +Rich ard +ĠMax imum +Ġide als +Th ings +m and +ĠReg ardless +Ġhum ili +b uffer +L ittle +ĠD ani +ĠN ak +Ġliber ation +ĠA be +ĠO L +Ġstuff ed +ac a +ind a +raph ic +Ġmos qu +Ġcampaign ing +Ġoccup y +S qu +r ina +ĠW el +ĠV S +Ġphys ic +Ġp uls +r int +oad ed +ET F +ĠArch ives +Ġven ues +h ner +ĠTur bo +Ġl ust +Ġappeal ed +que z +il ib +ĠTim othy +Ġo mn +d ro +Ġobs ession +ĠSav age +19 96 +Gl obal +J es +2 14 +Ġsl iding +Ġdisapp ro +ĠMag ical +Ġvolunt arily +g b +ane y +Ġprop het +ĠRe in +ĠJul ia +ĠW orth +aur us +Ġb ounds +ie u +)) ) +Ġcro re +ĠCitiz en +S ky +Ġcolumn ist +Ġseek ers +ond o +IS A +ĠL ength +Ġnost alg +Ġnew com +Ġdet rim +ent ric +3 75 +ĠG E +Ġaut op +Ġacadem ics +App Data +ĠS hen +Ġid iot +ĠTrans it +Ġteasp oon +W il +K O +ĠCom edy +> , +Ġpop ulated +W D +Ġp igs +ĠO culus +Ġsymp athetic +Ġmar athon +19 8 +Ġseiz ure +s ided +Ġd op +irt ual +L and +ĠFl oor +osa urs +... ] +Ġl os +Ġsubsid iary +E Y +ĠPart s +ĠSt ef +ĠJud iciary +Ġ13 4 +Ġmir rors +Ġk et +t imes +Ġneuro log +Ġc av +ĠGu est +Ġtum or +sc ill +ĠLl oyd +E st +Ġcle arer +Ġstere otypes +Ġd ur +not hing +Red dit +Ġnegoti ated +---------------- -------- +23 5 +Ġfl own +ĠSe oul +ĠRes ident +ĠS CH +Ġdisappear ance +ĠV ince +g rown +Ġgrab s +r il +ĠInf inite +ĠTw enty +Ġpedest rian +Ġjer sey +ĠF ur +ĠInf inity +ĠEll iott +Ġment or +Ġmor ally +Ġob ey +sec ure +iff e +Ġantib iotics +ang led +ĠFre eman +ĠIntrodu ction +J un +Ġm arsh +ic ans +ĠEV ENTS +och ond +W all +icult y +Ġmisdem eanor +Ġl y +Th omas +ĠRes olution +Ġanim ations +ĠD ry +Ġinter course +ĠNew castle +ĠH og +ĠEqu ipment +17 7 +Ġterrit orial +Ġarch ives +20 3 +Fil ter +ĠMun ich +Ġcommand ed +ĠW and +Ġpit ches +ĠCro at +Ġrat ios +ĠM its +Ġaccum ulated +ĠSpecific ally +Ġgentle man +acer b +Ġp enn +Ġa ka +ĠF uk +Ġinterven e +ĠRef uge +ĠAlz heimer +Ġsuccess ion +oh an +d oes +L ord +Ġsepar at +Ġcorrespond ence +Ġsh iny +P rior +Ġs ulf +Ġmiser able +Ġded ication +( ). +Ġspecial ists +Ġdefect s +ĠC ult +ĠX ia +Ġje opard +ĠO re +Ab ility +Ġle ar +Ġamb itions +ĠB MI +ĠArab s +Ġ19 42 +Ġpres ervation +ific ate +Ġash amed +l oss +ĠRest aur +Ġrese mble +Ġen rich +ĠK N +ĠCl an +fl oat +Ġplay able +IT T +Ġharm ony +arr ison +ĠWe instein +w ere +Ġpoison ing +ĠCom put +ĠWord Press +m ajor +ĠVal ve +F an +ĠTh row +ĠRom ans +ĠDep ression +ad os +Ġtort ured +Ġbal ancing +bott om +Ġacqu iring +ĠMon te +ard i +Ġa ura +Ġ# # +ĠStand ing +ĠAtl as +C F +Ġintr ins +ĠBen ghazi +Ġcamp ing +Ġt apped +bl ade +st rous +ĠR abb +ĠW ritten +t ip +ĠNe igh +ster dam +ĠAll ow +ĠHe aling +ĠR hod +n um +Ġcaffe ine +ĠPer cent +Ġbo o +Ġapp les +30 5 +Ġwel coming +Ġappl aud +Ġa usterity + ± +ĠRe ality +ef e +å ® +Ġsu cks +Ġtab s +ĠPay Pal +Ġback pack +Ġgif ted +abul ary +ĠSc out +ir teen +Ġch in +Ġo mitted +Ġnegative ly +Ġaccess ing +ĠE arn +Ġambul ance +Ġhead phones +Ġ20 5 +ĠRef resh +p resident +ĠKit chen +ĠEnt ered +ĠS nyder +00 5 +om ical +Ġborrow ed +ĠN em +Ġav iation +Ġst all +rim ination +Ġuniform s +it ime +ĠSim mons +ener gy +ab lished +y y +qual ified +Ġrall ies +ĠSt uart +fl ight +Ġgang s +r ag +Ġv ault +lu x +ĠCom par +Ġdesign ation +20 9 +ĠJ os +d ollar +z ero +Ġwell s +30 3 +Ġconstitu ents +Ġhe ck +Ġc ows +Ġcommand ers +Ġdifferent ial +ĠC atherine +29 9 +Ġval ve +Ġbr ace +Ġperspect ives +c ert +f act +icular ly +ĠMc N +pl anes +Ġint ric +Ġpe as +ov an +Ġtoss ed +ret ch +ĠL opez +Ġunf amiliar +de ath +ĠA part +ĠCh ang +Ġrelie ved +rop he +Ġair ports +Ġfre ak +ut il +M ill +ĠCh in +ĠOw en +m ale +ĠBro ken +ĠWind s +ro b +r ising +Ġfire fighters +Ġauthor itarian +Ġ14 8 +Bit coin +ex ternal +Ġbrow sers +iche ver +or ian +Ġun b +Ġpo ke +ĠZ ot +M id +ĠPop ular +Ġco vert +Ġcont ributes +Ġ6 50 +Ġcont ention +G ate +Ġcons oles +Ġchrom os +ĠI X +Ġvis ually +ĠE isen +Ġjewel ry +Ġdeleg ation +Ġacceler ate +ĠR iley +Ġsl ope +Ġind oor +it ially +Ġhuge ly +Ġtun nels +Ġfin ed +Ġdirect ive +Ġfore head +ustom ed +Ġsk ate +Mus ic +g as +Ġrecogn izing +am bo +Ġover weight +ĠGr ade +Ù Ĭ +Ġsound ing +Ġlock ing +ĠR EM +St ore +Ġexc av +ĠLike wise +ĠL ights +Ġel bow +ĠSupp ly +w ic +Ġhands ome +19 94 +C oll +Ġadequ ately +ĠAssoci ate +Ġstri ps +Ġcrack down +Ġmar vel +ĠK un +Ġpass ages +@@ @@ +ĠT all +Ġthought ful +names e +Ġprost itution +bus iness +Ġball istic +person al +c ig +iz ational +R ound +ĠÂłĠÂł ĠÂłĠÂł +ĠCole man +Ġadm itting +ĠPl ug +Ġbit coins +ĠSu z +Ġfair ness +Ġsupp lier +Ġcatast rophic +ĠHel en +o qu +M arc +ĠArt icles +g ie +Ġend angered +Ġdest iny +ĠVol t +ol ia +ax is +Ġche at +Ġun ified +IC O +qu ote +30 2 +ĠS ed +Ġsupp ression +Ġanaly zing +Ġsqu at +Ġfig uring +Ġcoordin ates +Ġch unks +Ġ19 46 +Ġsub p +Ġw iki +ĠFor bes +ĠJ upiter +ĠE rik +im er +ĠCom mercial +\ ) +Ġlegitim acy +Ġd ental +ĠMe an +Ġdefic its +5 50 +Orig inally +ĠHor ror +Ġcontam ination +ll ah +Ġconf isc +ĠCl are +T B +ĠF ailed +an ed +Ġrul er +ĠCont roller +Ġfemin ists +F ix +g ay +20 7 +Ġr abbit +Th ird +ownt own +Ġgl ue +Ġvol atile +Ġsh ining +Ġf oll +Ġimp aired +Ġsup ers +æ Ī +Ġcl utch +ļé ĨĴ +Ġpro let +Ġ( ! +Ġy elled +ĠK iev +ĠEr n +ĠSh ock +K B +Ġsit uated +qu ery +ĠN as +Ġan nex +char acter +ĠHol iday +Ġautom ation +ĠJ ill +ĠRem astered +Ġl inem +Ġwild erness +ĠHor izon +ĠGu inea +A Z +Ġmain land +Ġsec recy +LE ASE +Ġp unk +ĠProv ince +( ), +Spe ed +Ġhand ing +ĠSeb ast +S ir +r ase +Ġj ournals +Ġcon gest +ĠT ut +ir rel +Ġschizophren ia +Ġmis ogyn +health y +I ron +Ġreact ed +- $ +25 2 +Ġpl ural +Ġpl um +Ġbarg ain +Ġground ed +f inder +Ġdis se +ĠL az +O OD +Ġat roc +F actory +Ġmin ions +Ġo ri +ĠB rave +ĠP RE +ĠMy anmar +ĠH od +Ġexped ition +Ġexpl ode +ĠCo ord +Ġext r +ĠB rief +ĠAD HD +Ġhard core +feed ing +Ġd ile +ĠF ruit +Ġvacc ination +ĠM ao +osp here +Ġcont ests +- | +Ġf ren +isp here +R om +ĠSh arp +ĠTre nd +Ġdis connect +âĢ¢ âĢ¢ +Ġper secution +Ear th +Ġhealth ier +38 4 +Ġc ob +ĠTr inity +OW S +AN N +Ġspecial ty +Ġg ru +Ġcooper ative +wh y +Start ing +ĠIss ues +st re +ens or +Ġ18 5 +Ad v +! ? +ĠRe vel +em ia +ĠH ulk +Ġcelebr ations +ĠS ou +ra ud +ĠKle in +Ġun real +con text +Ġpartners hips +Ġadop ting +t ical +Ġspl ash +ĠHe zbollah +c ategory +cycl op +xt on +ĠD ot +urd y +t z +Ġenvelop e +ĠN L +â ķ +Ġwhere in +Spe c +18 4 +Ġte lev +al iation +Ġmyth s +å ° +Ġrig orous +Ġcommun icating +Ġobser ver +Ġre he +ĠW ash +Ġapolog ized +ĠT in +Ġexpend itures +work ers +d ocument +Ġhes itate +ĠLen in +Ġunpredict able +Ġrenew al +cl er +ok ia +ĠCON T +Ġpost season +Tok ens +Ġex acerb +Ġbet ting +Ġ14 7 +Ġelev ation +W ood +ĠSol omon +19 4 +00 4 +out put +Ġredu nd +ĠM umbai +Ġp H +Ġreprodu ce +ĠD uration +MA X +Ġb og +C BS +ĠBal ance +ĠS gt +ĠRec ent +Ġc d +Ġpo pped +Ġincomp et +pro p +ay an +g uy +Pac ific +Ġty r +Ġ{ { +ĠMy stic +ĠD ana +Ġmast urb +Ġge ometry +à ¢ +ĠCor rect +Ġtraject ory +Ġdistract ed +Ġf oo +ĠW elsh +L uc +m ith +Ġrug by +Ġrespir atory +Ġtri angle +Ġ2 15 +Ġunder graduate +ĠSuper ior +ch anging +_ - +Ġright ly +Ġrefere e +Ġluc rative +Ġun authorized +Ġresemb les +ĠGN U +ĠDer by +Ġpath ways +ĠL ed +Ġend urance +Ġst int +Ġcollect or +F ast +Ġd ots +Ġnational s +ĠSec urities +Ġwh ip +Par am +Ġlearn s +M agic +Ġdetail ing +m oon +Ġbroadcast ing +Ġb aked +26 5 +hol m +ĠS ah +ĠHus sein +ĠCourt esy +17 4 +Ġ14 6 +Ġge ographic +pe ace +Ġjud ging +ĠS tern +B ur +Ġstory line +G un +ĠSt ick +24 5 +30 7 +ãĤ´ ãĥ³ +ĠAdminist rator +Ġbur nt +Ġp ave +ch oes +Ex ec +Ġcamp uses +Res ult +Ġmut ations +ĠCh arter +Ġcapt ures +Ġcomp ares +Ġbad ge +S cient +Ġer ad +ier y +o i +ett es +ĠE state +Ġst rap +Ġproud ly +Ġf ried +Ġwithd rawn +ĠV oy +ph ony +It ems +ĠP ierce +b ard +Ġann otation +ant on +ill on +Im pro +... ) +Ġhapp ier +---- -- +ad just +Ġstaff ers +Ġactiv ism +Ġper f +Ġal right +N eed +Ġcomm ence +Ġopio id +ĠAm anda +E s +ĠP ars +ĠK aw +W orks +24 8 +Ġind o +t c +end ant +ĠM oto +Ġlegal ization +OT E +Ġtask ed +Ġt sp +ĠACT IONS +16 6 +Ġrefres hing +ĠN R +ĠPere z +Ġinfring ement +S Y +List en +in ning +k u +Ġrot ate +pro gram +ar ah +Des ign +Ġ( £ +Ġst oring +Ġwar rants +Ġjud gement +ĠB rist +us ually +ph oto +ĠR an +ĠP ine +Ġoutrage ous +ĠValent ine +lu ence +ĠEvery body +Al tern +Ġrele vance +Ġtermin ated +Ġd essert +Ġfulf illed +Ġprosecut ed +ĠW ords +Ġm igrant +Ġcultiv ation +ÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤ ÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤ +idel ity +ĠV ern +ĠLog in +Ġmetaph or +ĠT ip +Ġrecru its +ĠP ig +rib ing +Ġenthusi asts +ex per +Ġfright ening +ĠH air +ans on +str ate +Ġh i +He ight +Ġown ing +n one +Ġdis like +Ġkn ives +pher d +Ġloud ly +ĠAP Is +Dis play +ĠL ac +ĠUS S +ab l +ver ages +J ew +Ġ17 2 +ĠHist orical +at oon +ĠPhys ics +in tern +Ġwarm th +Ġto pp +D M +Ġgun man +Ġem peror +od i +ãĥ £ +in atory +ĠR ib +Ġ13 1 +ĠSat urn +ĠSh ining +Ġw aking +Qu otes +Ġcomed ian +en berg + ½ +Ġbelie vers +Ġpaper work +c ustom +Ġle v +Ġl ament +Ġpour ing +22 2 +p olitical +ĠSupp lement +m aid +Ġcruel ty +Ġt read +ys ics +A w +rit es +Ġmod ifier +ĠP osition +Ad am +l b +ub s +Ġimper fect +Ġcl usters +ĠEngine er +ĠC herry +Ġinaug uration +ĠS au +Ġembod iment +ĠUn cle +Ġover r +Ġexplos ions +c ule +ĠPrinc eton +ĠAndre a +Ġincorrect ly +Ġearn est +Ġpil gr +ĠS print +Ġslee ve +Ġhe ars +ĠAm azing +Ġbrow sing +ag in +Ġhom eland +Ġha w +Ġd iving +ist ered +17 8 +Ġbarg aining +ĠArc ade +Ġdeleg ate +ters on +................................ ................................ +ĠJackson ville +27 5 +Ġst agn +Ġad am +ĠSher man +C B +Ġsub urb +ĠFood s +Ġconver ting +ĠAr ist +Ġch ambers +l ove +Ġam ino +ĠG an +Ġmad ness +m c +ĠUS E +def ined +Ġul tr +ind ust +Ġw olves +l ance +Add itionally +Ġcr acks +as ia +ĠRe ason +ĠP ump +Ġaccident al +ĠL aser +ĠR id +Ġinitial ized +ell i +Ġun named +Ġn oun +ĠPass ed +Ġhost age +ĠEth iop +sh irts +Ġun rel +ĠEmb assy +Ġ19 41 +Ġat oms +Ġpur ported +16 4 +ĠF i +Ġgall ons +ĠMon ica +Ġp g +en ment +Ġsort ed +ĠG ospel +Ġhe ights +Ġtr aced +Ġunder going +She ll +Ġs acks +Ġproport ions +Ġhall uc +F ont +ac et +Ġwar mer +ĠIN TER +Ġgrab bing +Pl ug +Ġreal ization +ĠBur ke +Ġen chant +AT ER +ĠSe ed +Ġabund ant +F M +Ġc ivic +V s +is i +Ġv ow +Ġre per +ĠPartners hip +Ġpenet ration +Ġax e +Ġsh attered +ĠZ ombies +Ġv inyl +ĠAl ert +e on +Ġoblig ed +ĠIll ust +ĠPl aza +ĠFront ier +Ġdavid jl +ĠSer ial +ĠH av +ĠNut rition +B i +Ġâĸ Ī +ĠJ ays +lin ux +Ġhur ry +Ġv oy +Ġhop eless +ĠSte alth +Ġ ãģ +ess ors +tt le +b org +ĠSaf ari +f ell +Ġw ary +d ue +ĠAb ove +H a +E LL +Ġnot or +ĠW on +T oo +Ġoccup ations +Ġposs essions +Ġinv iting +Ġpred ators +Ġacceler ated +Ġ15 7 +uter te +ĠC ube +e ast +acc ount +G ive +Ġtrans plant +red ients +id able +Ġscreens hots +ĠG und +ĠF S +Ġtravel ers +Ġsens ory +ĠF iat +ĠRock ets +İ ĭ +_ { +F riend +Ġchar ming +AL S +Ġenjoy ment +m ph +Ġ5 000 +ĠRE G +Ù Ĩ +b ia +Ġcomp ilation +ro st +ĠV P +ĠSch ne +201 9 +Ġcop ying +M ORE +ĠFl ore +f alls +2 15 +t otal +Ġdis ciples +d ouble +Ġexceed ing +Ġsm ashed +Ġconcept ual +ĠRom ania +ĠB rent +ĠI CE +ĠT ou +Ġg rap +Ġn ails +18 9 +ãĥ ĺ +Ġproc ure +e ur +Ġconfir ming +ĠC ec +aw i +ĠEd en +Ġn g +Ġengine ered +at ics +Ġhook ed +Ġdisgust ing +ĠMur der +ãĤ ¿ +L ibrary +Ġ16 8 +Al most +hem atic +Men u +ĠNot re +ĠJ ur +Ġkidn apped +Ġhack er +ĠJ ade +Ġcreep y +Ġdraw ings +ĠSpons or +Ġcycl ists +ĠGob lin +Ġoptim ized +Ġst aged +ĠMc D +bet ween +A ge +en o +S ex +ĠW ide +n ings +av is +Ġincap able +ĠK ob +Ġreward ing +ĠL one +oles cent +Ġcontract ed +Ġstick y +J ose +B all +f est +ĠIn put +ĠRec ently +Ġto mat +squ are +App lication +Ġnit rogen +Ġdupl icate +ĠRec on +ĠD ear +L ondon +Ġint ra +Ġd ock +Ġout reach +ĠM illion +Ġmamm als +am pton +V AL +Ġsn aps +Ġd os +ĠWh ole +ĠRead y +T ry +ĠWinn ipeg +ear ance +Ġinc urred +ren ched +ĠNS W +il ot +rain e +Ġc ube +g ot +Ġrun way +etermin ed +ĠHaw ks +Ġsurviv or +ĠW ish +ĠD in +ĠDE F +ĠV ault +18 7 +Ġmush rooms +Ġcris p +be y +ĠDisco very +Ġdevelopment al +Ġparad igm +Ġcha otic +ĠT su +Ġ3 33 +b ons +Ġbacter ial +Ġcomm its +Ġcos mic +Ġme ga +oc ative +ĠP aint +ophob ic +Ġv ain +Ġcar ved +ĠTh ief +ĠG ul +ows hip +Ġc ites +ĠEd inburgh +Ġdimin ished +Ġacknowled ges +ĠK ills +Ġmic row +ĠHer a +Ġsen iors +Ġwhere by +H op +at ron +Ġun available +ĠN ate +Ġ4 80 +Ġsl ated +ĠRe becca +ĠB attery +Ġgram mar +Ġhead set +Ġcurs or +Ġex cluding +any e +aunder ing +eb in +Ġfeas ible +ĠPub lishing +ĠLab s +ĠCl iff +ĠFerr ari +Ġp ac +vis ible +mark ed +pe ll +Ġpol ite +Ġstagger ing +ĠGal actic +Ġsuper st +Ġpar an +ĠOffic ers +ãĢ ģ +Ġspecific s +ul us +23 9 +ĠP aste +AM P +ĠPan ama +ĠDe lete +angu ard +rest rial +Ġhero ic +ĠD y +ا ÙĦ +Ġincumb ent +Ġcr unch +t ro +Ġsc oop +Ġblog ger +Ġsell ers +ure n +Ġmedic ines +ĠC aps +ĠAnim ation +ox y +Ġout ward +Ġinqu iries +22 9 +Ġpsych ologist +ĠS ask +ev il +Ġcontam inated +ãĤ ¨ +he rence +Ġbrand ed +ĠAbd ul +z h +Ġparagraph s +Ġmin s +Ġcor related +er b +Ġimp art +Ġmil estone +ĠSol utions +ot le +Ġunder cover +Ġmar ched +ĠCharg ers +f ax +ĠSec rets +Ġr uth +we ather +Ġfemin ine +Ġsh am +Ġprest igious +igg ins +Ġs ung +hist ory +ett le +gg ie +Ġout dated +ol and +Ġper ceptions +ĠS ession +ĠDod gers +u j +ĠE ND +D oc +Ġdefic iency +Gr and +ĠJ oker +Ġretro spect +Ġdiagn ostic +Ġharm less +Ġro gue +ĠA val +E qu +Ġtrans c +ĠRoberts on +ĠDep ending +ĠBurn s +iv o +Ġhost ility +F eatures +ĵ ĺ +Ġdis comfort +ĠL CD +spec ified +ĠEx pect +3 40 +Ġimper ative +ĠReg ular +Ch inese +Ġstate wide +Ġsy mm +Ġlo ops +Ġaut umn +N ick +Ġsh aping +Ġqu ot +Ġc herry +ĠCross ref +è¦ ļéĨĴ +Stand ard +he ed +ĠD ell +ĠViet namese +Ġo st +ĠV alkyrie +O A +Ass ad +Ġreb ound +ĠTra ffic +pl aces +æ ĺ +ĠB uc +17 2 +Ġshel ters +Ġins isting +ĠCertain ly +ĠKenn eth +ĠT CP +Ġpen al +ĠRe play +he ard +Ġdial ect +iz a +ĠF Y +it cher +ĠD L +Ġspir al +Ġquarterback s +Ġh ull +Ġgo ogle +Ġto dd +ĠSter ling +ĠPl ate +Ġsp ying +mb ol +ĠReal m +ĠPro ced +ĠCr ash +Ġtermin ate +Ġprotest ing +C enter +gu ided +Ġun cover +Ġboy cott +Ġreal izes +s ound +Ġpret ending +ĠV as +19 80 +Ġfram ed +Ġ13 9 +Ġdesc ended +Ġrehab ilitation +Ġborrow ing +ĠB uch +Ġbl ur +R on +ĠFro zen +en za +Ch ief +ĠP oor +Ġtransl ates +M IN +Ġ2 12 +J ECT +Ġerupt ed +Ġsuccess es +S EC +Ġpl ague +Ġg ems +d oms +Ġstret ches +ĠSp y +Ġstory telling +C redit +ĠP ush +Ġtra ction +Ġin effective +ĠL una +Ġt apes +Ġanaly tics +erc ise +Ġprogram mes +ĠCar bon +Ġbeh old +he avy +ĠConserv ation +ĠF IR +Ġs ack +ter min +ric ks +Ġhous ed +Ġunus ually +I ce +Ġexecut ing +ĠMor oc +ed ay +Ġed itions +Ġsm arter +ĠB A +Ġout law +Ġvan ished +ib a +AL SE +ĠSil va +23 8 +C ould +Ġphilos opher +Ġevac uated +Sec ret +14 2 +Ġvis as +ãĤ ¬ +ĠM alt +ĠClear ly +ĠN iger +ĠC airo +ĠF ist +3 80 +ĠX ML +aut o +it ant +Ġrein forced +Rec ord +ĠSurviv or +G Hz +Ġscrew s +parent s +Ġo ceans +ma res +Ġbra kes +vas ive +Ġhell o +ĠS IM +rim p +Ġo re +ĠArm our +24 7 +Ġterr ific +Ġt ones +14 1 +ĠMin utes +Ep isode +Ġcur ves +Ġinflamm atory +Ġbat ting +ĠBeaut iful +L ay +Ġunp op +v able +Ġr iots +ĠTact ics +b augh +ĠC ock +Ġorg asm +ĠS as +Ġconstruct or +et z +G ov +Ġant agon +Ġthe at +Ġde eds +ha o +c uts +ĠMc Cl +Ġu m +ĠScient ists +Ġgrass roots +ys sey +"] => +Ġsurf aced +Ġsh ades +Ġneighb ours +Ġad vertis +oy a +Ġmer ged +Up on +Ġg ad +Ġanticip ate +Any way +Ġsl ogan +Ġdis respect +I ran +ĠT B +act ed +Ġsubp oen +medi ately +OO OO +Ġwa iver +Ġvulner abilities +ott esville +ĠHuff ington +J osh +ĠD H +M onday +ĠEll en +K now +x on +it ems +22 8 +Ġf ills +ĠN ike +Ġcum ulative +and als +I r +Ġ ì +Ġfr iction +ig ator +Ġsc ans +ĠVi enna +ld om +Ġperform ers +P rim +Ġb idding +M ur +Ġlean ed +ĠPri x +al ks +Ġ[ âĢ¦] +ĠTw itch +ĠDevelop er +ĠG ir +Ġcall back +Ab stract +Ġacc ustomed +Ġfreed oms +ĠP G +ur acy +Ġl ump +is man +,, ,, +19 92 +ĠR ED +Ġwor m +M atch +ĠPl atinum +I J +ĠOwn er +Tri via +com pl +Ġnew born +Ġfant as +O wn +Ġ19 59 +Ġsymp ath +Ġub iqu +Ġoutput s +Ġal lev +Ġpr ag +K evin +Ġfav ors +Ġbur ial +Ġn urt +so lete +c ache +Ġ15 6 +Ġunl ocks +te chn +M aking +Ġcon quer +ad ic +æ ĸ +Ġel f +Ġelect orate +ĠKurd s +ĠSt ack +ĠSam urai +Ġâ ĺħ +Ġ{ } +ĠS aid +ĠFall out +Ġkind ness +ĠCustom s +ĠBou levard +Ġhelicop ters +ot ics +ĠVe get +com ment +Ġcritic ised +Ġpol ished +ĠRem ix +ĠC ultural +Ġrec ons +Ġdo i +at em +Sc reen +Ġbar red +Com ments +ĠGener ally +Ġsl ap +7 20 +V ari +p ine +Ġem pt +Ġh ats +ĠPlay ing +l ab +a verage +form s +ĠC otton +Ġcan s +ĠD ON +ĠSom alia +C rypt +ĠIncre ases +E ver +mod ern +Ġsur geon +3 000 +Ġrandom ized +================================ ================================ +B ern +im pl +ĠC OR +Ġpro claim +th ouse +Ġto es +Ġam ple +Ġpres erving +Ġdis bel +gr and +B esides +Ġsil k +ĠPat tern +h m +Ġenter prises +Ġaffidav it +ĠAdvis ory +Ġadvert ised +ĠRel igious +se ctions +psy ch +ĠField s +aw ays +Ġhasht ag +ĠNight mare +Ġv ampire +Ġfore nsic +rosso ver +n ar +Ġn avy +Ġvac ant +ĠD uel +Ġhall way +Ġface book +ident ally +ĠN RA +Ġm att +Ġhur ricane +ĠKir by +ĠP uzzle +Ġsk irt +ou st +du llah +Ġanal ogy +in ion +Ġtomat oes +ĠN V +ĠPe ak +ĠMe yer +Ġappoint ments +Ġm asc +Ġal ley +re hend +Ġchar ities +Ġund o +Ġdest inations +ĠTest ing +"> " +c ats +* . +Ġgest ures +gener al +Le ague +Ġpack ets +ĠInspect or +ĠBer g +Ġfraud ulent +Ġcritic ize +F un +Ġbl aming +nd ra +Ġsl ash +ĠE ston +Ġpropos ing +Ġwh ales +Ġtherap ist +Ġsub set +Ġle isure +EL D +ĠC VE +ĠAct ivity +Ġcul min +sh op +ĠD AY +is cher +ĠAdmir al +ĠAtt acks +Ġ19 58 +Ġmem oir +Ġfold ed +Ġsex ist +Ġ15 3 +ĠL I +Ġread ings +Ġembarrass ment +ĠEmploy ment +w art +ch in +Ġcontin uation +l ia +Rec ently +Ġd uel +Ġevac uation +ĠKash mir +Ġdis position +ĠR ig +Ġbol ts +Ġins urers +4 67 +M ex +Ġret aliation +Ġmis ery +Ġunre asonable +r aining +I mm +ĠP U +em er +Ġgen ital +ãĤ ³ +ĠC andy +Ġon ions +ĠP att +lin er +Ġconced ed +Ġf a +Ġfor c +ĠH ernandez +ĠGe off +deb ian +ĠTe ams +Ġc ries +Ġhome owners +23 7 +A BC +Ġst itch +Ġstat istic +Ġhead ers +ĠBi ology +Ġmot ors +ĠG EN +ĠL ip +Ġh ates +Ġhe el +S elf +i pl +ED IT +ort ing +Ġann ot +ĠSpe ech +old emort +ĠJ avascript +ĠLe Bron +Ġfoot print +Ġf n +Ġseiz ures +n as +h ide +Ġ19 54 +ĠBe e +ĠDecl aration +ĠKat ie +Ġreserv ations +N R +f emale +Ġsatur ated +Ġb iblical +Ġtroll s +Dev ice +ph otos +Ġdr ums +ãĥīãĥ© ãĤ´ãĥ³ +N ight +f ighter +ĠH ak +ri ber +Ġc ush +Ġdiscipl inary +ba um +ĠG H +ĠSch midt +ilib rium +Ġs ixty +ĠKush ner +ro ts +Ġp und +ĠR ac +Ġspr ings +Ġcon ve +Bus iness +F all +Ġqual ifications +Ġvers es +Ġnarc iss +ĠK oh +ĠW ow +ĠCharl ottesville +ed o +Ġinterrog ation +ĠW ool +36 5 +B rian +Ġâľ ĵ +Ġalleg es +ond s +id ation +ĠJack ie +y u +Ġl akes +Ġworth while +Ġcryst als +ĠJud a +Ġcomp rehend +Ġfl ush +Ġabsor ption +ĠO C +Ġfright ened +ĠCh ocolate +Mart in +Ġbu ys +Ġbu cks +Ġapp ell +ĠChampions hips +Ġlist ener +ĠDef ensive +Ġc z +ud s +ĠM ate +Ġre play +Ġdecor ated +Ġs unk +ĠV IP +ĠAn k +Ġ19 5 +aa aa +Nob ody +ĠMil k +ĠG ur +ĠM k +ĠS ara +Ġse ating +ĠW id +Tr ack +Ġemploy s +Ġgig antic +AP P +ãĤ § +in ventory +Ġtow el +at che +l asting +ĠT L +Ġlat ency +Ġkn e +B er +me aning +Ġup held +Ġplay ground +Ġm ant +S ide +Ġstere o +Ġnorth west +Ġexception ally +Ġr ays +Ġrec urring +D rive +Ġup right +Ġab duct +ĠMar athon +Ġgood bye +Ġal phabet +h p +Ġcourt room +ring ton +ot hing +T ag +Ġdiplom ats +Ġbar bar +ĠAqu a +18 3 +33 33 +Ġmat urity +Ġinst ability +ĠAp ache +Ġ= == +Ġfast ing +ĠGr id +Mod Loader +Ġ15 2 +A bs +ĠOper ating +ett i +Ġacqu aint +Don nell +ĠK em +ĠFor ge +Ġarm ored +M il +Ġphilos ophers +in vest +Pl ayers +â Ī +Ġmy riad +Ġcomr ades +R ot +Ġremember ing +Ġcorrespond s +Ġprogram mers +ĠLyn n +Ġo lig +Ġco herent +yn chron +ĠChem ical +Ġj ugg +p air +post s +E ye +ĠIn ner +Ġsem ester +ott est +ĠEmir ates +ric anes +or ously +m its +ĠW is +Ġd odge +l ocation +Ġf aded +Am azon +ĠPro ceed +ĠIN FO +j ournal +ĠTru ck +T en +Ġ2 17 +Ġstat utes +m obile +ĠT ypes +Rec omm +b uster +pe x +Ġleg ends +Ġhead ache +f aced +ĠWi Fi +if ty +ĠH ER +Ġcirc uits +ER ROR +22 6 +ol in +Ġcyl inder +osp ace +ik ers +P rem +Qu ant +Ġconflic ting +Ġslight est +Ġfor ged +ion age +Step hen +ĠK ub +ĠOpp ortun +ĠHe al +Ġbl o +Ġrul ers +Ġh uh +Ġsubmar ine +f y +ass er +Ġallow ance +ĠKas ich +ĠT as +ĠAustral ians +Forge ModLoader +ĠâĨ ij +ĠMat rix +am ins +Ġ12 00 +ĠAc qu +23 6 +D ocument +ĠBre aking +19 3 +ĠSub st +ĠRoll er +ĠPro perties +ĠN I +t ier +Ġcr ushing +Ġadvoc ating +Further more +keep ers +Ġsex ism +x d +Ġcall er +ĠS ense +chie ve +ĠT F +Ġfuel ed +Ġreminis cent +Ġobs ess +ur st +Ġup hold +ĠF ans +het ics +Ġâ Ĺ +ĠB ath +Ġbe verage +Ġo scill +25 4 +Ġpol es +Ġgrad ual +Ġex ting +ĠS uff +ĠS uddenly +Ġlik ing +Ġ19 49 +un ciation +am ination +ĠO mar +ĠL V +ĠCon sequently +Ġsynt hes +ĠG IF +Ġp ains +Ġinteract ing +u ously +inc re +Ġrum or +ĠScient ology +19 7 +ĠZ ig +Ġspe lling +ĠA SS +Ġexting u +ms on +Ġg h +Ġremark ed +ĠStrateg ic +ĠM ON +å ¥ +g ae +ĠWH AT +E ric +ĠCamp us +Ġmeth ane +Ġimag in +J UST +ĠAl m +X T +i q +ĠR SS +Ġwrong doing +att a +Ġbig ot +Ġdemonstr ators +ĠCal vin +ĠV illa +Ġmembr ane +ĠAw esome +Ġbenef ic +26 8 +Ġmagn ificent +ĠL ots +G reg +ĠBor is +Ġdetain ees +ĠH erman +Ġwhis pered +Ġa we +Prof essor +fund ing +Ġphys iological +ĠDest ruction +Ġlim b +Ġmanip ulated +Ġbub bles +Ġpse ud +Ġhyd ra +ĠBrist ol +Ġst ellar +ĠExp ansion +ĠK ell +ĠInterest ingly +Ġm ans +Ġdrag ging +Ġec ological +ĠF it +Ġg ent +Ġbenef ited +ĠHait i +Ġpoly g +ãĥ İ +Ġ20 30 +Ġpro w +Ġrecon struction +Ġwas t +Ġpsych ic +ĠGree ks +Hand ler +16 2 +ĠP ulse +Ġsol icit +Ġsy s +Ġinflu x +ĠG entle +per cent +Ġprolifer ation +Ġtax able +Ġdisreg ard +Ġesc aping +Ġg inger +Ġwith stand +Ġdevast ated +ĠD ew +ser ies +Ġinject ed +ela ide +Ġturn over +he at +Ļ Ĥ +H appy +ĠSil ent +ãĤ Ń +iv ism +Ġir rational +AM A +Ġre ef +r ub +Ġ16 2 +Ġbank ers +ĠEth ics +v v +Ġcritic isms +K n +18 6 +M ovie +ĠT ories +Ġno od +Ġdist ortion +F alse +od ore +Ġt asty +Res earch +ĠU ID +- ) +Ġdivor ced +ĠM U +ĠHay es +ĠIs n +ian i +ĠH Q +Ġ" # +ign ant +Ġtra umatic +ĠL ing +H un +Ġsab ot +on line +r andom +Ġren amed +ra red +K A +d ead +é t +ĠAss istance +Ġse af +++++ ++++ +Ġse ldom +ĠWeb b +Ġbo olean +u let +Ġref rain +ĠDI Y +ru le +Ġshut ting +Ġutil izing +load ing +ĠPar am +co al +oot er +Ġattract ing +ĠD ol +Ġher s +ag netic +ĠRe ach +im o +Ġdisc arded +ĠP ip +01 5 +ü r +Ġm ug +Im agine +C OL +Ġcurs ed +ĠSh ows +ĠCurt is +ĠSach s +spe aking +ĠV ista +ĠFram ework +ong o +Ġsub reddit +Ġcr us +ĠO val +R ow +g rowing +Ġinstall ment +Ġgl ac +ĠAdv ance +EC K +ĠLGBT Q +LE Y +Ġac et +Ġsuccess ive +ĠNic ole +Ġ19 57 +Qu ote +Ġcircumst ance +ack ets +Ġ14 2 +ort ium +Ġguess ed +ĠFr ame +Ġperpet rators +ĠAv iation +ĠBen ch +Ġhand c +A p +Ġ19 56 +25 9 +r and +Net Message +d in +urt les +h ig +ĠV III +ff iti +ĠSw ords +b ial +Ġkidn apping +dev ice +Ġb arn +ĠEl i +auc as +S end +Con structed +Ġ ½ +Ġneed les +Ġad vertisements +Ġv ou +Ġexhib ited +ĠFort ress +As k +B erry +TY PE +Ġcan cers +ump ing +ĠTerrit ory +Ġpr ud +Ġn as +Ġathe ist +Ġbal ances +ãģ Ł +ĠSh awn +& & +Ġland sc +ĠR GB +Ġpet ty +Ġex cellence +Ġtransl ations +Ġpar cel +ĠChe v +E ast +ĠOut put +im i +Ġamb ient +ĠTh reat +Ġvill ains +Ġ5 50 +IC A +Ġtall er +Ġle aking +c up +Ġpol ish +Ġinfect ious +ĠK C +Ġ@ @ +back ground +Ġbureaucr acy +ĠS ai +un less +it ious +ĠSky pe +At l +ID ENT +00 8 +Ġhyp ocr +Ġpit chers +Ġguess ing +ĠF INAL +Bet ween +Ġvill agers +Ġ25 2 +f ashion +ĠTun is +Be h +ĠEx c +ĠM ID +28 8 +ĠHas kell +19 6 +ĠN OR +Ġspec s +Ġinv ari +Ġgl ut +ĠC ars +Ġimp ulse +Ġhon ors +g el +Ġjurisd ictions +ĠBund le +ul as +Calif ornia +ĠIncre ase +Ġp ear +Ġsing les +Ġc ues +Ġunder went +ĠW S +Ġexagger ated +Ġdub ious +Ġfl ashing +L OG +) ]. +J ournal +t g +V an +ĠI stanbul +ĠIn sp +ĠFrank en +D raw +Ġsad ness +Ġiron ic +ĠF ry +x c +Ġ16 4 +is ch +W ay +ĠProtest ant +h orn +Ġun aff +ĠV iv +ill as +ĠProduct ions +ĠH ogan +Ġper imeter +ĠS isters +Ġspont aneous +Ġdown side +Ġdescend ants +Ġor n +w orm +Japan ese +Ġ19 55 +Ġ15 1 +ĠDo ing +els en +umb les +Ġrad ically +ĠDr um +ĠB ach +Ġli abilities +ĠO B +ĠElement ary +Ġmem e +yn es +Ġfinger print +ĠGr ab +Ġundert ake +Mem bers +ĠRead er +ĠSim s +g od +Ġhypot hetical +s cient +ĠA J +Ġchar ism +Ġad missions +ĠMiss ile +tr ade +Ġexerc ising +ĠBack ground +W ritten +Ġvoc als +whe ther +Ġv i +ĠW inner +Ġl itter +ĠSh ooting +ST EM +ãĤ ¡ +ĠA FL +Ġvari ability +Ġe ats +ĠD PS +b row +Ġeleph ants +Ġstr at +Ġ Å +Ġsett lers +Matt hew +Ġin advert +H I +ĠIM F +ĠGo al +Ġnerv es +John son +ey e +ablish ment +Th ursday +BIL ITY +H ad +am oto +het amine +ep s +Ġmit ochond +Ġcomp ressed +ĠTre vor +ĠAnim als +T ool +L ock +Ġtwe ak +Ġpin ch +Ġcancell ation +P ot +Ġfoc al +ĠAst ron +17 3 +ĠA SC +ĠO THER +umn i +Ġdem ise +d l +Ù ħ +Sem itism +Ġcr acking +Ġcollabor ative +Ġexpl ores +s ql +Ġher bs +Ġconfig urations +m is +ĠRes ult +ace y +ĠSm oke +Ġsan ct +el ia +Ġdeg ener +Ġdeep est +Ġscream ed +Ġn ap +Soft ware +ĠST AR +E F +ĠX in +spons ored +mans hip +23 3 +Ġprim aries +Ġfilter ing +Ġas semble +m il +ĠMy ers +b ows +Ġpun ched +M ic +Ġinnov ations +Ġfun c +and o +Ġfr acking +ĠV ul +о Ð +osh op +ĠIm mun +Ġsett ling +Ġadolesc ents +Ġreb uilding +Ġtransform ing +Ġpar ole +Ġhar bor +Ġbook ing +ot ional +onge vity +ĠY o +b ug +Ġemer ges +ĠMethod s +ĠCh u +P res +ĠDun geons +Ġtra iling +ĠR um +ĠH ugh +å¤ © +ĠE ra +ĠBatt les +Res ults +ĠTr ading +Ġvers a +c ss +ax ies +he et +Ġgre ed +19 89 +Ġgard ens +Ġconting ent +P ark +ĠLeaf s +h ook +ro be +Ġdiplom acy +ĠF uel +ĠInv asion +Ġupgr ading +M ale +Ġe lic +Ġrelent less +ĠCo venant +ap esh +ĠT rop +T y +pro duction +art y +Ġpun ches +ak o +cyclop edia +ĠR abbit +ĠHD MI +Ġ14 1 +Ġf oil +Item Image +ĠF G +Ġimplement ations +ĠP om +ixt ures +Ġaw ait +Ġ3 30 +am us +Ġumb rella +Ġfore see +se par +Ġcircum cision +Ġperipher al +S ay +ĠExper t +In c +Ġwithd rew +ĠAnd ers +f ried +Ġradio active +ĠOp ening +Ġboard ing +ĠN D +Ġover throw +Act iv +W P +ĠAct s +× Ļ +Ġmot ions +v ic +ĠM ighty +ĠDef ender +a er +Ġthank ful +ĠK illing +ĠBr is +mo il +Ġpredict ing +26 6 +ch oice +Ġkill ers +Ġinc ub +ĠChe st +ather ing +Ġpro claimed +fl ower +oss om +umbled ore +ĠCy cling +ĠOccup y +AG ES +P en +ĠY ug +Ġpack aged +Ġheight ened +c ot +st ack +C ond +Ġst amps +m age +Ġpersu aded +Ġens l +ĠCard inal +Ġsol itary +Ġpossess ing +ĠC ork +Ġev id +ĠT ay +Ġbl ues +Ġextrem ism +Ġlun ar +Ġcl own +Te chn +Ġfest ivals +ĠPv P +ĠL ar +Ġconsequ ently +p resent +Ġsom eday +ç İĭ +ĠMet eor +Ġtour ing +c ulture +Ġbe aches +S hip +c ause +ĠFl ood +ãĥ ¯ +Ġpur ity +th ose +Ġem ission +b olt +Ġch ord +ĠScript ure +L u +Ġ$ { +cre ated +Other s +25 8 +Ġelement al +Ġannoy ed +ĠA E +d an +ĠS ag +Res earchers +Ġfair y +âĢĵ âĢĵ +======== ==== +Sm art +GG GG +Ġskelet ons +Ġpup ils +link ed +Ġur gency +en abled +ĠF uck +Ġcoun cill +r ab +U AL +T I +Ġlif es +Ġconf essed +B ug +Ġharm on +ĠCON FIG +ĠNe utral +D ouble +Ġst aple +ĠSH A +Brit ish +ĠSN P +AT OR +oc o +Ġswing ing +ge x +ole on +pl ain +ĠMiss ing +ĠTro phy +v ari +ran ch +Ġ3 01 +4 40 +00000000 00000000 +Ġrest oring +Ġha ul +uc ing +ner g +Ġfut ures +Ġstrateg ist +quest ion +Ġlater al +ĠB ard +Ġs or +ĠRhod es +ĠD owntown +????? - +ĠL it +ĠB ened +Ġco il +st reet +ĠPort al +FI LE +ĠG ru +* , +23 1 +ne um +Ġsuck ed +Ġr apper +Ġtend encies +ĠLaure n +cell aneous +26 7 +Ġbrow se +Ġover c +head er +o ise +Ġbe et +ĠG le +St ay +Ġm um +Ġtyp ed +Ġdiscount s +T alk +ĠO g +ex isting +ĠS ell +u ph +C I +ĠAust rian +ĠW arm +Ġdismiss al +Ġaver ages +c amera +Ġalleg iance +L AN +=" # +Ġcomment ators +ĠSet ting +ĠMid west +Ġpharm ac +ĠEX P +Ġstain less +Ch icago +Ġt an +24 4 +Ġcountry side +ĠV ac +29 5 +Ġpin ned +Ġcr ises +Ġstandard ized +T ask +ĠJ ail +ĠD ocker +col ored +f orth +" }, +Ġpat rons +Ġsp ice +Ġm ourn +ĠM ood +Ġlaund ry +Ġequ ip +ĠM ole +y ll +ĠTH C +n ation +ĠSher lock +Ġiss u +ĠK re +ĠAmeric as +ĠA AA +Ġsystem atically +Ġcont ra +ĠS ally +Ġrational e +Ġcar riage +Ġpe aks +Ġcontrad iction +ens ation +ĠFail ure +Ġpro ps +Ġnames pace +Ġc ove +field s +ãĤ ĭ +Ġw ool +ĠC atch +Ġpresum ed +ĠD iana +r agon +ig i +Ġh amm +Ġst unt +ĠG UI +ĠObserv atory +ĠSh ore +Ġsmell s +ann ah +Ġcock pit +ĠD uterte +8 50 +Ġopp ressed +bre aker +ĠCont ribut +ĠPer u +ĠMons anto +ĠAtt empt +Ġcommand ing +Ġfr idge +ĠR in +ĠChe ss +ual ity +Ġo l +Republic an +ĠGl ory +ĠW IN +.... ... +ag ent +read ing +Ġin h +J ones +Ġcl icks +al an +Ġ[ ]; +ĠMaj esty +ĠC ed +op us +ate l +à ª +AR C +ĠEc uador +ãĥ ł +ĠK uro +Ġritual s +Ġcapt ive +Ġoun ce +Ġdisag reement +Ġsl og +f uel +P et +M ail +Ġexerc ised +Ġsol ic +Ġrain fall +Ġdev otion +ĠAss essment +Ġrob otic +opt ions +ĠR P +ĠFam ilies +ĠFl ames +Ġassign ments +00 7 +aked own +Ġvoc abulary +Re illy +Ġc aval +g ars +Ġsupp ressed +ĠS ET +ĠJohn s +Ġwar p +bro ken +Ġstat ues +Ġadvoc ated +Ġ2 75 +Ġper il +om orph +ĠF emin +per fect +Ġh atch +L ib +5 12 +Ġlif elong +3 13 +Ġche eks +Ġnum bered +ĠM ug +B ody +ra vel +We ight +ĠJ ak +ĠHe ath +Ġkiss ing +ĠJ UST +Ġw aving +u pload +Ġins ider +ĠPro gressive +ĠFil ter +tt a +ĠBe am +Ġviol ently +ip ation +Ġskept icism +Ġ19 18 +ĠAnn ie +ĠS I +Ġgen etics +Ġon board +at l +ĠFried man +ĠB ri +cept ive +Ġpir ate +ĠRep orter +27 8 +Ġmyth ology +Ġe clipse +Ġsk ins +Ġgly ph +ing ham +F iles +C our +w omen +Ġreg imes +Ġphotograp hed +K at +ĠMA X +Offic ials +Ġunexpected ly +Ġimpress ions +F ront +;;;; ;;;; +Ġsuprem acy +Ġs ang +Ġaggrav ated +Ġabrupt ly +ĠS ector +Ġexc uses +Ġcost ing +ide press +St ack +ĠR NA +ob il +Ġghost s +ld on +at ibility +Top ics +Ġreim burse +ĠH M +ĠDe g +Ġth ief +y et +ogen esis +le aning +ĠK ol +ĠB asketball +Ġf i +ĠSee ing +Ġrecy cling +Ġ[ - +Cong ress +Ġlect ures +P sy +Ġne p +Ġm aid +Ġori ented +A X +Ġrespect ful +re ne +fl ush +ĠUn loaded +re quest +gr id +ĠAltern atively +ĠHug o +Ġdec ree +ĠBuddh ism +and um +And roid +ĠCong o +ĠJoy ce +Ġacknowled ging +hes ive +ĠTom orrow +ĠH iro +th ren +ĠM aced +Ġho ax +ĠIncre ased +ĠPr adesh +W ild +____ __ +16 1 +Ġa unt +Ġdistribut ing +ĠT ucker +ĠSS L +ĠW olves +B uilding +ou lt +ĠLu o +ĠY as +ĠSp ir +ĠSh ape +ĠCamb od +ĠIP v +Ġm l +Ġext rad +39 0 +ĠPenn y +d ream +Ġstation ed +opt ional +ew orthy +. +ĠWorks hop +ĠRet ail +ĠAv atar +6 25 +N a +ĠV C +ĠSec ure +M Y +19 88 +oss ip +Ġpro state +Ġund en +Ġg amer +ĠCont ents +ĠWar hammer +ĠSent inel +3 10 +Ġse gregation +ĠF lex +ĠM AY +Ġdr ills +ĠDrug s +Islam ic +Ġsp ur +Ġca fe +Ġimag inary +Ġgu iding +Ġsw ings +ĠThe me +ob y +Ġn ud +Ġbe gging +Ġstr ongh +Ġreject ing +Ġpedest rians +ĠPro spect +R are +s le +Ġconcess ions +ĠConst itutional +Ġbe ams +Ġfib ers +p oon +Ġinstinct s +pro perty +ĠB IG +Sand ers +im ates +Ġco ating +Ġcorps es +ĠTR UE +check ed +Ġ16 6 +A sh +ĠJ S +ĠF iction +Ġcommun al +Ġener getic +oooo oooo +Ġnow adays +IL D +ib o +ĠSU V +R en +Ġdwell ing +Sil ver +Ġt ally +ĠM oving +Ġcow ard +Ġgener als +Ġhorn s +Ġcirc ulated +Ġrob bed +ĠUn limited +Ġharass ed +Ġinhib it +Ġcomp oser +ĠSpot ify +Ġspread s +3 64 +Ġsu icidal +Ġno ises +ĠSt ur +Ġs aga +ĠK ag +is o +Ġtheoret ically +M oney +Ġsimilar ity +Ġslic ed +ut ils +ing es +" - +Ġan th +Ġimp ed +Mod ule +Through out +Ġmen us +comm ittee +and i +ob j +in av +f ired +ĠAb dullah +Ġund ead +Ġfont s +H old +EN G +Ġsustain ability +Ġfl ick +Ġr azor +ĠF est +ĠChar acters +Ġword ing +Ġpopul ist +Ġcritic izing +Ġm use +v ine +Ġcard board +Ġkind ly +Ġfr inge +ĠThe ft +icult ural +Ġgovern ors +Ġ ���� +Ġ16 3 +Ġtime out +ĠA uth +Child ren +A U +Ġred emption +ĠAl ger +Ġ19 14 +Ġw aved +Ġastron auts +og rams +Ġsw amp +ĠFinn ish +Ġcand le +Ġton nes +ut m +Ġr ay +Ġsp un +Ġfear ful +art icles +Ġca us +or ically +ĠRequ ires +ĠG ol +Ġpop e +Ġinaug ural +Ġg le +AD A +ĠIS IL +ĠOff ensive +Ġwatch dog +Ġbal con +ent ity +ĠH oo +Ġgall on +AC C +Ġdoub ling +Ġimpl ication +ĠS ight +Ġdoct r +---- --- +Ġ\ \ +Ġm alt +R oll +Ġâī ¥ +Ġrec ap +add ing +u ces +ĠB end +fig ure +Ġtur key +Ġsoc ietal +ĠT ickets +Ġcommer cially +Ġsp icy +Ġ2 16 +ĠR amp +Ġsuperior ity +à ¯ +ĠTr acker +C arl +ĠC oy +ĠPatri ot +Ġconsult ed +Ġlist ings +Ġsle w +reens hot +ĠG one +Ġ[ ...] +30 9 +Ġh ottest +Ø ± +Ġrock y +ĠD iaz +Ġmass age +Ġpar aly +Ġp ony +A z +Ġcart ridge +ĠN Z +Ġsn ack +ĠLam ar +ple ment +ĠLes lie +Ġm ater +Ġsn ipp +24 6 +Ġjoint ly +ĠBris bane +ĠiP od +Ġpump ing +Ġgo at +ĠSh aron +eal ing +Ġcor on +Ġan omal +rah im +ĠConnect ion +Ġsculpt ure +Ġsched uling +ĠD addy +at hing +Ġeyeb rows +Ġcur ved +Ġsent iments +Ġdraft ing +D rop +( [ +Ġnom inal +ĠLeaders hip +ĠG row +Ġ17 6 +Ġconstruct ive +iv ation +Ġcorrupt ed +ger ald +ĠC ros +ĠChe ster +ĠL ap +ãģ ª +OT H +D ATA +Ġal mond +pro bably +I mp +Ġfe ast +ĠWar craft +F lor +Ġcheck point +Ġtrans cription +Ġ20 4 +Ġtwe aks +Ġrel ieve +S cience +Ġperform er +Z one +Ġtur moil +ig ated +hib it +ĠC afe +the med +Ġflu or +ben ch +Ġde com +ĠU nt +ĠBar rett +ĠF acts +Ġt asting +ĠPTS D +ĠSe al +ĠJuda ism +ĠDynam ic +ĠC ors +V e +ĠM ing +ĠTrans form +v on +ĠDef enders +ĠTact ical +ĠV on +ĠUn ivers +Ġdist orted +ĠB reath +?' " +Ġag on +ĠDead ly +Ġl an +ĠCy cle +orn ed +Ġrel iably +Ġgl or +ĠMon key +ãĥ ¡ +Ġad ren +Ġmicrow ave +ĠAl ban +irc raft +dig it +sm art +ĠD read +¯¯¯¯¯¯¯¯ ¯¯¯¯¯¯¯¯ +{ { +ĠRoc hester +Ġsimpl ified +Ġinf licted +Ġtake over +Ġyour selves +ad itional +Ġmus cular +K S +Ġing en +T ax +ĠFe ature +27 7 +Ġcru c +Ġcr ate +Ġun identified +Ġacclaim ed +ĠM anga +ĠFr ances +ĠNep al +ĠG erald +ĠKu wait +Ġsl ain +ĠHe b +ĠG oku +ãģ® æ +28 6 +M rs +ĠC ody +ĠSan ctuary +01 6 +Ġdism ant +Ġdatas et +ĠH ond +b uck +ĠPat terson +Ġpal ette +ĠG D +ic ol +ĠL odge +Ġplanet ary +ak in +ĠRegist ered +ab we +ĠPeters burg +Ġha iled +ĠP iece +S che +ĠDO J +Ġen umer +18 1 +ĠObs erver +ĠB old +f ounded +com merce +Ġexplo its +ĠF inding +UR N +ĠS ne +ĠAc id +ay ette +ĠVal ues +Ġdr astic +Ġarchitect ural +Ġ" . +× ķ +ump ed +Ġwra pping +Ġwid ow +ĠSl ayer +l ace +on ce +German y +av oid +Ġtem ples +P AR +à ´ +ĠLuc ifer +ĠFl ickr +l ov +for ces +Ġsc outing +Ġlou der +tes y +Ġbefore hand +Ä ĵ +ĠNe on +ĠW ol +ĠTyp ically +ĠPolit ico +-+ -+ +Ġbuild er +Ġder ive +K ill +Ġp oker +Ġambig uous +Ġlif ts +Ġcy t +Ġrib s +ood le +ĠS ounds +h air +ĠSynd rome +t f +Ġproport ional +u id +Ġper taining +ĠKind le +ĠNeg ro +Ġreiter ated +ĠTon ight +oth s +ĠCorn ell +Ġo wing +Ġ20 8 +elf are +oc ating +ĠB irds +Sub scribe +Ġess ays +Ġburd ens +Ġillust rations +ar ious +ER AL +ĠCal cul +Ġx en +ĠLink edIn +ĠJ ung +Ġredes ign +Con nor +29 6 +Ġrevers al +ĠAd elaide +ĠL L +Ġs inking +Ġg um +US H +c apt +ĠGr imm +Ġfoot steps +ĠCB D +isp ers +Ġpro se +Wed nesday +ĠM ovies +ed in +Ġoverturn ed +Ġcontent ious +US B +~~~~~~~~ ~~~~~~~~ +ĠCo pper +Ġpoint less +N V +val ues +olph in +d ain +Ġdepos ited +ĠG W +Ġpreced ed +ĠCl a +ĠGo lem +ĠN im +ĠÎ ² +ĠEngine ers +m iddle +Ġfl att +oper ative +Ġcouncil s +imb abwe +el in +Ġstress ful +ĠL D +Ġres h +l ake +Ġwheel chair +ĠAltern ative +Ġoptim ize +oper ation +Ġpe ek +Ġones elf +ig il +Ġtrans itions +op athy +bl ank +Ġ16 9 +17 1 +________________________________ ________________________________ +Ġl aundering +En c +ĠD EC +Ġwork outs +Ġsp ikes +Ġdin osaurs +Ġdiscrim inatory +P ool +R ather +38 5 +R NA +tes ters +et o +ĠIdent ity +Ġve in +ĠBur ton +Ġarc ade +4 20 +Ult imately +ĠSad ly +à ° +p ill +Ġcub ic +ĠSpect rum +the se +st ates +Ġun official +h awks +ĠEVER Y +Ġrain bow +Ġincarcer ation +and ing +Ġsy ll +ĠEver ton +Ġ17 9 +ĠSer bia +Ġ18 9 +m eter +ĠMic key +Ġant iqu +Ġfact ual +ne ck +ĠN are +n orm +m ust +Ġhigh ways +Ġgl am +Ġdivid ing +ĠSquad ron +ĠMar tha +Ġbirth s +C over +//////// //////// +ĠW ong +Ph ot +ĠA LS +ri o +ĠNon etheless +ĠL emon +Ġ20 6 +ĠE E +Ġderiv ative +ĠWW II +v ote +Ġthere in +Ġsepar ating +44 6 +sy nc +ĠStre ets +Ġr att +Ġmunicip ality +ĠShort ly +Ġmon k +) ," +Ġscr ub +Ġoper atives +Ne ither +Pl ace +ĠLim it +F emale +ĠAct or +Char acter +Ġconstit uted +35 7 +Ġprotest ed +ĠSt raw +ĠHe ight +ild a +ĠTy ph +Ġflood s +Ġcos metic +W AY +pert ure +up on +t ons +ess ing +ĠP ocket +Ġro oft +ĠC aucas +Ġant idepress +Ġincomp atible +EC D +Ġoper a +ĠCont est +Ġgener ators +l ime +Def ense +19 87 +for um +Ġsav age +ĠHung arian +n z +Ġmet allic +Ġex pelled +Ġres idency +Ġdress es +66 6 +ĠC lement +f ires +C ategory +Ġge ek +al is +Ġc emetery +educ ated +Ġc rawl +ĠUn able +ĠT yson +ak is +Ġp ardon +ĠW ra +Ġstrengthen ed +ĠF ors +33 5 +ĠH C +ĠM ond +Ġvisual s +ĠBeat les +ett lement +Ġ ï +g ro +Ġb ash +Ġpo orest +Ġex cel +Ġaspir ations +ĠM unicip +ens ible +Ġceremon ies +Ġintimid ation +ĠCON TR +be ck +ĠK ap +as u +Ġtradem arks +ĠS ew +ĠComp etition +net work +ĠAr ri +ĠT et +Ro aming +W C +D at +Ġso b +Ġpair ing +Ġoverd ose +SA Y +ab er +Ġrev olt +ĠF ah +act ing +e q +est ation +F ight +ĠMar ks +27 3 +Ġ17 8 +R aw +ãģ ĭ +34 9 +bl ocks +Ġver ge +est ine +ĠPod esta +Ġinv asive +Ġprofound ly +ĠA o +e ach +Ġl est +inter pret +Ġshr inking +Ġerr one +Ġche es +ly s +ĠI vy +ĠDirect ory +Ġhint ed +V ICE +Ġcontact ing +ĠG ent +he i +Ġlabel ing +Ġmerc ury +ĠL ite +Ġexp ires +Ġdest abil +rit is +c u +Ġfeather s +Ġste er +Ġprogram med +ĠV ader +Go ing +ĠE lim +Ġy o +ĠMic he +Ġ20 3 +Ġslee ves +Ġb ully +ĠHum ans +36 8 +Ġcomp ress +ĠBan ner +AR S +Ġa while +Ġcal ib +Ġspons orship +ĠDiff iculty +ĠP apers +Ġident ifier +} . +Ġy og +ĠSh ia +Ġclean up +Ġvib e +int rodu +im ming +Austral ia +Ġout lines +ĠY outube +tr ain +ĠM akes +Ġde ported +Ġcent r +ĠD ug +ĠB oulder +ĠBuff y +Ġinj unction +ĠHar ley +ĠG roups +ĠD umbledore +ĠCl ara +Ġ" - +Ġsacrific ed +ep h +Sh adow +ib ling +Ġfreel ance +Ġevident ly +ph al +Ġret ains +M ir +Ġfin ite +d ar +ĠC ous +Ġrep aired +Ġperiod ic +Ġchampions hips +Ġaster oid +bl ind +Ġexpress ly +ĠAst ros +Ġsc aled +Ġge ographical +ĠRap ids +En joy +Ġel astic +ĠMoh amed +Mark et +be gin +Ġdisco vers +Ġtele communications +Ġscan ner +Ġen large +Ġsh arks +Ġpsy chedel +ĠRou ge +Ġsnap shot +is ine +X P +Ġpestic ides +ĠL SD +ĠDist ribution +re ally +Ġde gradation +Ġdisgu ise +Ġbi om +ĠEX T +Ġequ ations +Ġhaz ards +ĠComp ared +) * +Ġvirt ues +Ġeld ers +Ġenh ancing +ĠAc ross +er os +ang ling +Ġcomb ust +ucc i +Ġconc ussion +Ġcontrace ption +ĠK ang +Ġexpress es +Ġa ux +ĠP ione +Ġexhib its +Deb ug +OT AL +ĠAl ready +ĠWheel er +Ġexp ands +? : +Ġreconc iliation +Ġpir ates +Ġpur se +Ġdiscour age +Ġspect acle +R ank +Ġwra ps +ĠTh ought +Ġimp ending +O pp +ĠAng lo +ĠE UR +Ġscrew ed +ret ched +Ġencour agement +mod els +Ġconf use +mm m +ĠVit amin +âĸij âĸij +C ru +Ġkn ights +Ġdisc ard +Ġb ishops +ĠW ear +ĠGar rett +k an +ãĥ Ł +Ġmascul ine +cap ital +ĠA us +Ġfat ally +th anks +ĠA U +ĠG ut +12 00 +Ġ 00000000 +Ġsur rog +ĠBI OS +ra its +ĠWat ts +Ġresur rection +ĠElect oral +ĠT ips +4 000 +Ġnut rient +Ġdepict ing +Ġspr ink +Ġm uff +ĠL IM +ĠS ample +ps c +ib i +gener ated +Ġspec imens +Ġdiss atisf +Ġtail ored +Ġhold ings +ĠMonth ly +ĠE at +po ons +Ġne c +ĠC age +ĠLot us +ĠLan tern +Ġfront ier +Ġp ensions +Ġj oked +ĠHard y +=-=- =-=- +r ade +U ID +Ġr ails +Ġem it +Ġsl ate +Ġsm ug +Ġsp it +ĠCall s +ĠJac obs +f eat +ĠU E +Ġrest ruct +Ġregener ation +Ġenerg ies +ĠCon nor +OH N +ĠChe ese +Ġg er +Ġresur rect +man agement +N W +Ġpres ently +ĠBru ins +M ember +ĠM ang +id an +Ġboost ing +w yn ++ . +requ isite +ĠNY PD +ĠMe gan +ĠCond itions +Ġp ics +nes ium +ĠR ash +Ġ17 4 +ĠD ucks +Ġemb ro +z u +on ian +rel igious +Ġc raz +ĠAC A +ĠZ ucker +EM A +ĠPro s +We apon +ĠKn ox +ĠAr duino +Ġst ove +Ġheaven s +ĠP urchase +Ġher d +Ġfundra iser +Dig ital +5 000 +Ġprop onents +/ âĢĭ +Ġj elly +ĠVis a +Ġmon ks +Ġadvance ment +ĠW er +Ġ18 7 +e us +ert ility +Ġfet al +Ġ19 36 +L o +Ġout fits +Ġstair case +b omb +Ġcustom ized +cl air +T ree +Ġm apped +ĠConsider ing +ĠTor res +Ġmeth yl +Ġapprox imate +Ġdo om +ĠHans en +Ġc rossover +Ġstand alone +ä ¼ +Ġinv ites +Ġgra veyard +Ġh p +Donald Trump +Ġesc ort +G ar +Ġpredec essors +Ġh ay +Ġen zyme +ĠStra ight +vis ors +I ng +ane ously +ĠApp lied +Ġf ec +ĠDur ant +Ġout spoken +or b +Ġz eal +Ġdisgr ace +' ). +ĠChe ng +28 9 +ĠRen a +ĠSu icide +29 4 +Ġout raged +ĠNew man +ĠN vidia +ĠA ber +ĠB ers +Ġrecre ation +Wind ow +ĠD P +x e +Ġped oph +Ġfall out +ambo o +Ġpresent ations +ĠApp s +Ġh tml +3 45 +ĠX XX +Ġrub bing +ĠLe ather +Ġhum idity +se ys +est ablished +ĠUn its +64 6 +Ġrespect able +A uto +Ġthri ving +ĠInn ovation +ang s +Ext ra +reg ulation +29 8 +p ick +Ex amples +ĠC J +Att ack +Ġdr acon +L T +Ġstick er +re rs +Ġsun ny +I ss +reg ulated +d im +ĠAb stract +Ġhus bands +Off ice +om ination +it ars +AN GE +asc al +ĠK ris +ĠInf antry +Ġm alf +ĠA the +ĠR ally +bal anced +................ ........ +OU P +Ġmole cule +met ics +ĠSpl it +ĠInstruct ions +ĠN ights +c ards +Ġt ug +Ġcon e +å Ń +Ġt x +ĠDisc ussion +Ġcatast rophe +pp e +g io +Ġcommun ism +Ġhal ted +ĠGu ant +cle an +ĠSc hed +ĠK anye +Ġw ander +ĠSer iously +Ġ18 8 +enn ial +f ollow +product ive +ĠFl ow +ĠS ail +Ġc raw +Ġsim ulations +or u +ang les +ĠN olan +Ġmen stru +4 70 +Ġ20 7 +aj a +Ġcas ually +board ing +Ġ2 22 +ov y +ĠN umbers +um at +O E +28 7 +ĠCle mson +Ġcert s +Ġsl id +ĠT ribe +Ġto ast +Ġfort unes +Ġf als +ĠComm ittees +Ġg p +Ġf iery +ĠN ets +ĠAn ime +Pack age +ĠComp are +l aughter +in fect +Ġatroc ities +Ġjust ices +Ġins ults +ĠVern on +Ġsh aken +Ġperson a +est amp +36 7 +br ain +Ġexperiment ing +K en +ĠElect ronics +Ġ16 1 +dom ain +Ġgraph ical +b ishop +Ġwho pping +ĠEv angel +Ġadvertis ers +ĠSpe ar +Ġb ids +Ġdestro ys +ut z +Ġunders c +ĠAD D +Ġan ts +ĠC um +ipp les +ĠF ill +Ġgl anced +Ġind icted +ĠE ff +Ġmis con +ĠDes ktop +Ġab ide +ãĥ Ģ +ĠI o +ĠC oul +Ġcaps ule +ĠCh rys +M ON +Ġund es +ĠI RA +Ġc itation +Ġdict ate +ĠNet works +ĠConf lict +ĠSt uff +x a +is ec +ĠChem istry +Ġquarter ly +William s +an an +O pt +ĠAlexand ria +out heastern +ĠSpring field +ĠBlack s +Ġge ography +24 2 +Ġut most +ĠEx xon +ab outs +E VA +ĠEn able +ĠBar r +Ġdisag reed +ĠCy prus +Ġdement ia +Ġlab s +Ġubiqu itous +ĠLO VE +Ġconsolid ated +s r +Ġcream y +ĠTim ber +Reg ardless +ĠCert ificate +Ġ" ... +ogen ous +Capt ain +Ġinsult ing +ĠSor os +ĠInst r +ĠBulgar ia +bet ter +Ġsuck ing +ĠDavid son +at z +Ġcoll ateral +g if +Ġplag ued +ĠC ancel +ĠGard ner +R B +Ġsix teen +Rem ove +ur istic +c ook +R od +Ġcompr ising +f le +) âĢĶ +ĠVik ing +g rowth +agon al +Ġsr f +af ety +m ot +N early +st own +ĠF actor +Ġautom obile +Ġproced ural +m ask +amp ires +Ġdisapp ears +j ab +3 15 +Ġ19 51 +ne eded +Ġd aring +le ader +Ġp odium +Ġun healthy +Ġm und +Ġpy ramid +oc re +Ġkiss ed +Ġdream ed +ĠFant astic +ĠG ly +å Ĭ +Ġgreat ness +Ġsp ices +Ġmet ropolitan +Ġcomp uls +i ets +101 6 +ĠSh am +ĠP yr +fl ies +ĠMid night +Ġswall owed +Ġgen res +ĠL ucky +ĠRew ards +Ġdisp atch +ĠI PA +ĠApp ly +Ġa ven +al ities +3 12 +th ings +Ġ( ). +Ġm ates +ĠS z +ĠC OP +ol ate +O FF +Ġre charge +c aps +ĠYork er +ic one +Ġgal axies +ile aks +D ave +ĠP uzz +ĠCelt ic +ĠA FC +27 6 +ĠS ons +Ġaffirm ative +H or +Ġtutorial s +ĠC ITY +ĠR osa +ĠExt ension +Ser ies +Ġf ats +Ġr ab +l is +Ġun ic +Ġe ve +ĠSp in +Ġadul thood +ty p +Ġsect arian +Ġcheck out +ĠCy cl +S ingle +Ġmart yr +Ġch illing +88 8 +ou fl +Ġ] ; +Ġcongest ion +m k +ĠWhere as +Ġ19 38 +ur rencies +er ion +Ġbo ast +ĠPat ients +Ġch ap +ĠB D +real DonaldTrump +Ġexam ines +h ov +Ġstart ling +ĠBab ylon +w id +om ew +br ance +ĠOd yssey +w ig +Ġtor ch +ĠV ox +ĠMo z +ĠT roll +ĠAn s +Similar ly +ĠF ul +00 6 +Un less +ĠAl one +st ead +ĠPub lisher +r ights +t u +ĠDoes n +Ġprofession ally +Ġcl o +ic z +Ġste als +Ġ á +19 86 +Ġst urdy +ĠJoh ann +Ġmed als +Ġfil ings +ĠFr aser +d one +Ġmult inational +Ġf eder +Ġworth less +Ġp est +Yes terday +ank ind +Ġg ays +Ġb orne +ĠP OS +Pict ure +Ġpercent ages +25 1 +r ame +Ġpot ions +AM D +ĠLeban ese +Ġr ang +ĠL SU +ong s +Ġpen insula +ĠCl ause +AL K +oh a +ĠMac Book +Ġunanim ous +Ġl enders +Ġhang s +Ġfranch ises +ore rs +ĠUp dates +Ġisol ate +and ro +S oon +Ġdisrupt ive +ĠSur ve +Ġst itches +ĠSc orp +ĠDomin ion +Ġsupp lying +Ar g +Ġtur ret +ĠL uk +Ġbr ackets +* ) +ĠRevolution ary +ĠHon est +Ġnot icing +ĠSh annon +Ġafford ed +Ġth a +ĠJan et +! -- +ĠNare ndra +ĠPl ot +H ol +se ver +e enth +Ġobst ruction +Ġ10 24 +st aff +j as +or get +sc enes +l aughs +ĠF argo +cr ime +Ġorche str +Ġde let +ili ary +rie ved +Ġmilit ar +ĠGreen e +âĹ ı +ãģ ¦ +ĠGu ards +Ġunle ashed +ĠWe ber +Ġadjust able +Ġcal iber +Ġmotiv ations +Ġà ł +m Ah +ĠL anka +hand le +Ġp ent +ĠR av +ĠAng ular +ĠK au +umb ing +Ġphil anthrop +Ġde hyd +Ġtox icity +e er +ĠY ORK +w itz +å ¼ +ĠI E +commun ity +ĠA H +Ġret ali +Ġmass ively +ĠDani els +ĠD EL +Ġcar cin +Ur l +Ġrout ing +ĠNPC s +ĠR AF +ry ce +Ġwa ived +ĠGu atem +Every body +Ġco venant +Ġ17 3 +Ġrelax ing +Ġqu art +al most +Ġguard ed +ĠSold iers +ĠPL AY +Ġout going +L AND +Ġre write +ĠM OV +ĠIm per +ĠS olution +Ġphenomen al +Ġl ongevity +Ġimp at +ĠN issan +ir ie +Ġod or +ĠZ ar +ok s +Ġmilit ias +ĠSP EC +Ġtoler ated +ars er +ĠBrad ford ++ , +Ġsur real +s f +Can adian +Ġresemb lance +Ġcarbohyd rate +VI EW +Ġaccess ory +me al +larg est +ieg el +Some one +Ġtoug hest +os o +Ġfun nel +Ġcondemn ation +lu ent +Ġw ired +ĠSun set +Jes us +ĠP ST +ĠP ages +ĠTy coon +ĠP F +Ġselect ions +Ġ ठ+part isan +Ġhigh s +ĠR une +Ġcraft s +le ad +ĠParent s +Ġre claim +ek er +ĠAll ied +ae per +Ġlo oming +Ġbenefic iaries +ĠH ull +Stud ents +Jew ish +d j +Ġp act +tem plate +ĠOffic ials +ĠBay lor +Ġhe mp +Ġyouth s +ĠLevel s +ĠX iao +ĠC hes +Ġende avor +ĠRem oved +Ġhipp ocamp +H ell +ãĤ Ĭ +80 5 +Ġd inosaur +ĠWr ath +ĠIndones ian +Ġcalcul ator +ĠD ictionary +Ġ4 20 +ĠM AG +( _ +! , +t arians +Ġrestrict ing +rac use +Ġweek day +OU NT +Ġsh rugged +leg round +Ġb ald +ĠDo ctors +Ġt outed +ĠMax well +Ġ2 14 +Ġdiplom at +Ġrep ression +Ġconstitu ency +v ice +r anked +ĠNap oleon +g ang +ĠFore ver +t un +Ġbul b +ĠPD T +ĠC isco +V EN +Ġres umed +Ste ven +ĠManit oba +Ġfab ulous +ĠAg ents +19 84 +Ġam using +ĠMyster ies +Ġor thodox +fl oor +Ġquestion naire +Ġpenet rate +Ġfilm makers +ĠUn c +Ġst amped +Ġth irteen +Ġout field +Ġforward ed +Ġapp ra +Ġa ided +t ry +Ġunf ocused +ĠL iz +ĠWend y +ĠSc ene +Ch arg +Ġreject s +Ġleft ist +ĠProv idence +ĠBr id +reg n +Ġprophe cy +ĠL IVE +4 99 +Ġfor ge +ĠF ML +Ġintrins ic +ĠF rog +Ġw ont +ĠH olt +Ġfam ed +CL US +aeper nick +ĠH ate +ĠC ay +Ġregister ing +ort ality +rop y +ocaly ptic +a an +n av +Ġfasc ist +IF IED +Ġimpl icated +ĠRes ort +ĠChand ler +ĠBr ick +P in +ys c +Us age +ĠHel m +us ra +âĺħ âĺħ +ĠAb bas +Ġunanim ously +Ġke eper +Ġadd icted +?? ? +Ġhelm ets +Ġant ioxid +aps ed +80 8 +gi ene +Ġwa its +Ġmin ion +ra ved +ĠP orsche +Ġdream ing +Ġ17 1 +ĠC ain +Ġun for +ass o +ĠConfig uration +k un +hard t +Ġn ested +ĠL DS +L ES +Ġt ying +en os +Ġc ue +ĠMar qu +sk irts +Ġclick ed +Ġexp iration +ĠAccording ly +ĠW C +Ġbless ings +Ġaddict ive +ĠN arr +y x +ĠJagu ars +Ġrent s +ĠS iber +Ġt ipped +ous se +ĠFitz gerald +Ġhier arch +out ine +Ġwa velength +> . +ch id +ĠProcess ing +/ + +r anking +E asy +ĠConst ruct +Ġt et +ins ured +H UD +Ġqu oting +Ġcommun icated +in x +Ġin mate +Ġerect ed +ĠAbs olutely +ĠSure ly +Ġun im +ĠThr one +he id +Ġcl aws +Ġsuper star +ĠL enn +ĠWh is +U k +ab ol +Ġsk et +ĠN iet +Ġper ks +Ġaff inity +Ġopen ings +phas is +Ġdiscrim inate +T ip +v c +Ġgr inding +ĠJenn y +Ġast hma +hol es +ĠHom er +Ġreg isters +ĠGl ad +Ġcre ations +Ġlith ium +Ġappl ause +unt il +Just ice +ĠTur ks +Ġsc andals +Ġb ake +t ank +M ech +ĠMe ans +ĠM aid +Republic ans +is al +wind ows +ĠSant os +Ġveget ation +33 8 +t ri +Ġfl ux +ins ert +Ġclar ified +Ġmort g +ĠCh im +ĠT ort +Ġdiscl aim +met al +ĠAs ide +Ġindu ction +Ġinf l +Ġathe ists +amp h +Ġe ther +ĠV ital +ĠBu ilt +M ind +Ġweapon ry +S ET +Ġ18 6 +ad min +g am +cont ract +af a +Ġderiv atives +Ġsn acks +Ġch urn +E conom +Ġca pped +ĠUnder standing +ĠH ers +ĠI z +Ġd uct +I ENT +augh ty +Ġâľ Ķ +ĠN P +Ġsa iling +In itialized +Ġt ed +Ġreact ors +ĠL omb +Ġcho ke +ĠW orm +Ġadm iration +Ġsw ung +ens ibly +Ġr ash +ĠGo als +ĠImport ant +Sh ot +ĠR as +Ġtrain ers +ĠB un +Work ing +Ġhar med +ĠPand ora +ĠL TE +Ġmush room +ĠCH AR +ĠF ee +ĠM oy +B orn +ol iberal +ĠMart ial +Ġgentle men +Ġling ering +Offic ial +Ġgra ffiti +ĠN ames +D er +Ġqu int +ist rate +aze era +ĠNOT ICE +ĠFlore nce +Ġpay able +Ġdep icts +ĠSpe cies +He art +âĶĢâĶĢâĶĢâĶĢ âĶĢâĶĢâĶĢâĶĢ +Ġencl osed +Incre ases +D aily +ĠL is +Ġenact ment +ĠB acon +ĠSt eele +dem and +Ġ18 3 +Ġmouth s +Ġstr anded +Ġenhance ment +01 1 +ĠWh ats +Ġhe aled +en y +ĠR ab +Ġ3 40 +ĠLab yrinth +ro ach +ĠY osh +ĠCl ippers +Ġconcert s +Intern et +35 5 +Ġstick ers +Ġter med +ĠAx e +Ġgrand parents +Fr ance +ĠCl im +ĠU h +ul ic +Ġthr ill +cent ric +ĠOver view +ĠCond uct +Ġsubstant ive +Ġ18 2 +m ur +Ġstr ay +ĠCo ff +Ġrep etitive +ĠFor gotten +Ġqual ification +ew itness +ĠZ imbabwe +Ġsim ulated +ĠJ D +25 3 +ĠW are +Ġun sc +T imes +Ġsum mons +Ġdis connected +Ġ18 4 +ci us +ĠGu jar +od ka +Ġer ase +ĠTob acco +elect ed +Ġun cont +ĠShe pard +ĠL amp +Ġalert ed +Ġoper ative +arn a +u int +Ġneglig ence +ac ements +Ġsup ra +Ġprev ail +ĠSh ark +Ġbel ts +ãģ « +Ġt ighter +Engine ers +Ġin active +Ġexp onent +ĠWill ie +a ples +Ġhe ir +ĠH its +ian n +ĠS ays +Ġcurrent s +ĠBeng al +Ġar ist +B uffer +Ġbree ze +ĠWes ley +Col a +Ġpron oun +Ġde ed +ĠK ling +Ġof t +Ġinf lict +Ġpun ishing +Ġn m +ik u +OD UCT +01 4 +Ġsubsid y +ĠDE A +ĠHer bert +ĠJ al +B ank +Ġdef erred +Ġship ment +B ott +Ġal le +b earing +HT ML +Off line +Ġ2 13 +Ġscroll ing +Ġsc anned +ĠLib yan +ĠT OP +ch rom +d t +col umn +Psy NetMessage +Z ero +Ġtor so +0 50 +âķ IJ +Ġimp erson +ĠSchw artz +ud ic +Ġpiss ed +ĠS app +25 7 +ĠIS Ps +og l +Ġsuper vised +Ġad olescent +Ġatt ained +ĠDel ivery +ĠB unny +Ġ19 37 +Ġmini ature +Ġo s +Ġ3 70 +60 8 +ĠMour inho +Ġinn ate +Ġtem po +ĠN M +ĠFall en +00 9 +Ġprov ocative +Stream er +ĠBened ict +ĠBol she +Ġt urtle +ĠPC B +ĠEqu al +Direct or +ĠR end +Ġflu ids +Author ities +Ġcous ins +requ ency +ĠNeigh bor +s ets +sh ared +Char les +pass word +Ġg ears +Ġ2 11 +ĠHard ware +ri ka +Ġup stream +H om +Ġdisproportion ately +iv ities +Ġund efined +Ġelect rons +Ġcommem or +Event ually +Ġ> < +Ġir responsible +2 18 +ĠRe leased +ĠO VER +ĠI GN +ĠB read +st ellar +ĠS age +tt ed +dam age +ed ition +ĠPre c +Ġl ime +Ġconf inement +Ġcal orie +we apon +Ġdiff ering +ĠS ina +m ys +am d +Ġintric ate +k k +ĠP AT +ã o +st ones +lin ks +Ġr anch +Sem itic +Ġdifferent iate +ĠS inger +occup ied +Ġfort ress +c md +Ġinter ception +ĠAnk ara +Ġre pt +ĠSol itaire +Ġrem ake +p red +Ġd ared +aut ions +ĠB ACK +Run ning +Ġdebug ging +Ġgraph s +3 99 +ĠNig el +Ġb un +Ġpill ow +Ġprog ressed +fashion ed +Ġob edience +ER N +Ġrehe ars +C ell +t l +S her +Ġher ald +ĠPay ment +ĠC ory +ĠDe pt +Ġrep ent +ĠWe ak +uck land +Ġple asing +Ġshort ages +Ġjur ors +ĠK ab +q qa +Ant i +Ġw ow +ĠRC MP +Ġt sun +ĠS ic +Ġcomp rises +Ġsp ies +Ġprec inct +n u +Ġur ges +Ġtim ed +Ġstrip es +ĠB oots +Ġy en +Adv anced +Ġdisc rete +ĠArch angel +employ ment +D iff +Ġmon uments +Ġ20 9 +work er +Ġ19 6 +ĠI g +utter stock +T PS +J ac +Ġhomeless ness +Ġcomment ator +Ġrac ially +f ing +se ed +E le +ell ation +Ġeth anol +Ġpar ish +ĠD ong +ĠAw akening +Ġdev iation +ĠB earing +ĠTsu k +Ġrec ess +Ġl ymph +ĠCann abis +å ľ +ĠNEW S +Ġd ra +ĠStef an +ĠWr ong +ĠS AM +Ġloose ly +Ġinterpre ter +ĠPl ain +Go vernment +Ġbigot ry +Ġgren ades +ave z +pict ured +Ġmand ated +ĠMon k +ĠPed ro +Ġl ava +27 4 +Ġcyn ical +ĠScroll s +l ocks +M p +Ġcon gregation +orn ings +ph il +ĠI bid +Ġf erv +Ġdisapp earing +Ġarrog ant +sy n +ĠMa ver +ĠSu it +24 1 +Ġab bre +ack ers +P a +ĠY el +Whe never +Ġ23 5 +ĠV ine +ĠAn at +Ġext inct +LE T +Ġexecut able +V ERS +ox ide +D NA +ĠP rel +Ġresent ment +Ġcompr ise +ĠAv iv +Ġinter ceptions +Ġprol ific +IN A +ĠEr in +though t +2 19 +ĠPsychiat ry +un ky +chem ist +H o +ĠMcC oy +Ġbr icks +L os +ri ly +ĠUS SR +Ġr ud +Ġl aud +ĠW ise +ĠEmer ald +Ġrev ived +Ġdam ned +ĠRep air +id em +ct ica +Ġpatri arch +ĠN urs +me g +Ġcheap est +re ements +empt y +ĠCele br +Ġdepri vation +ch anted +ĠTh umbnails +E nergy +ĠEth an +ĠQ ing +Ġopp oses +W IND +v ik +ĠM au +ĠS UB +66 7 +G RE +ĠVol unte +nt on +C ook +å IJ +es que +Ġplum met +Ġsu ing +Ġpron ounce +Ġresist ing +ĠF ishing +ĠTri als +Ġy ell +Ġ3 10 +Ġin duct +Ġpersonal ized +oft en +R eb +EM BER +Ġview point +Ġexist ential +() ) +rem ove +MENT S +l asses +Ġev apor +Ġa isle +met a +Ġreflect ive +Ġentit lement +Ġdev ised +mus ic +asc ade +Ġwind ing +off set +Ġaccess ibility +ke red +Bet ter +ĠJohn ston +th inking +S now +ĠCroat ia +ĠAt omic +27 1 +34 8 +Ġtext book +ĠSix th +Ġ اÙĦ +Ġsl ider +ĠBur ger +b ol +S ync +Ġgrand children +Ġc erv ++ ) +Ġe ternity +Ġtweet ing +Ġspec ulative +Ġpiv otal +ĠW P +ĠT ER +ynam ic +Ġu pl +ĠC ats +per haps +Ġclass mates +Ġblat ant +' - +Ġl akh +ant ine +ĠB org +i om +/ ( +ĠAthlet ic +Ġs ar +OT A +ĠHoff man +Never theless +Ġad orable +Ġspawn ed +Ass ociated +ĠDom estic +Ġimpl ant +ĠLux em +ĠK ens +Ġp umps +ĠS AT +Att ributes +50 9 +av our +Ġcentral ized +ĠT N +Ġfresh ly +ĠA chieve +Ġouts iders +her ty +ĠRe e +ĠT owers +ĠD art +ak able +Ġm p +ĠHeaven ly +Ġr ipe +ĠCarol ine +ry an +Ġclass ics +Ġret iring +Ġ2 28 +Ġa h +Ġdeal ings +Ġpunch ing +ĠChap man +O ptions +max well +vol ume +Ġst al +Ġex ported +ĠQu ite +Ġnumer ical +B urn +F act +ĠKey stone +Ġtrend ing +Ġalter ing +ĠAfric ans +47 8 +ĠM N +ĠKn ock +Ġtempt ation +Ġprest ige +Over view +ĠTrad itional +ĠBah rain +Priv ate +ĠH OU +Ġbar r +ĠT at +C ube +US D +ĠGrand e +ĠG at +ĠFl o +Ġres ides +Ġind ec +vol ent +Ġperpet ual +ub es +Ġworld view +ĠQuant um +Ġfil tered +Ġen su +orget own +ERS ON +ĠM ild +37 9 +OT T +à ¥ +Ġvit amins +Ġrib bon +Ġsincere ly +ĠH in +Ġeight een +Ġcontradict ory +Ġgl aring +Ġexpect ancy +Ġcons pir +Ġmon strous +Ġ3 80 +re ci +Ġhand ic +Ġpump ed +Ġindic ative +Ġr app +Ġav ail +ĠLEG O +ĠMar ijuana +19 85 +ert on +Ġtwent ieth +################ ################ +ĠSw amp +Ġval uation +Ġaffili ates +adjust ed +ĠFac ility +26 2 +Ġenz ymes +itud inal +Ġimp rint +S ite +Ġinstall er +ĠT RA +m ology +lin ear +ĠCollect ive +ig ating +ĠT oken +Ġspec ulated +K N +ĠC ly +or ity +Ġdef er +Ġinspect ors +appro ved +R M +ĠSun s +Ġinform ing +ĠSy racuse +ib li +7 65 +Ġgl ove +Ġauthor ize +âĢ¦âĢ¦âĢ¦âĢ¦ âĢ¦âĢ¦âĢ¦âĢ¦ +ĠCru ise +Ġcontract ing +she ll +IF E +ĠJew el +p ract +ĠPhot oshop +ĠKnow ing +h arm +Ġattract ions +ad an +et us +01 8 +w agen +Al t +Ġmultip ly +Ġequ ilibrium +: { +ĠF ighters +ĠEd gar +Ġfour teen +Go vern +Ġmis use +Ġab using +Ġancest ry +ram er +64 4 +Ġwor ms +Ġthick er +ĠComb ine +Ġpeas ants +Ġv ind +Ġcon quest +Ġm ocked +Ġc innamon +ĠC ald +ĠGall up +Ġavoid ance +Ġincarn ation +ĠStr at +Ġt asted +ent a +ĠN eal +p ared +Ġtermin ology +ject ion +Scient ists +ĠIN S +ĠDe e +Ġdirect ories +R oad +ĠSh ap +br ight +ĠDirect ors +ĠCol umn +Ġb ob +Ġprefer ably +Ġgl itch +f urt +Ġe g +id is +C BC +Ġsur rendered +Ġtest ament +33 6 +ug gest +ĠN il +an other +Ġpat hetic +ĠDon na +Ġ2 18 +ĠA very +Ġwhis key +Ġf ixture +ĠCon quest +Ġbet s +O cc +ĠLe icester +] ." +Ġ) ); +Ġfl ashes +45 6 +Ġmask ed +ge bra +Ġcomput ed +che l +aud er +Ġdefe ats +ĠLiber ation +ĠOs ama +ĠV ive +Ch anges +Ch annel +Ġtar iffs +Ġm age +ĠS ax +Ġinadvert ently +ĠC RE +ĠRe aper +ink y +gr ading +Ġstere otyp +Ġcur l +ĠF ANT +Ġfram eworks +M om +ĠAn ch +Ġflav our +car bon +Ġperm itting +let cher +ĠMo zilla +ĠPark ing +ĠCh amp +Sc roll +Ġmurd erer +Ġrest ed +Ġow es +ĠP oss +AD D +IF F +res olution +ĠMin ing +Ġcompar ative +D im +Ġneighbour ing +ĠA ST +ĠT oxic +Ġbi ases +Ġgun fire +ur ous +ĠMom ent +19 83 +Ġper vasive +tt p +ĠNorm ally +r ir +S arah +ĠAlb any +Ġun sett +ĠS MS +ip ers +l ayer +ĠWh ites +up le +Ġtur bo +ĠLe eds +Ġthat s +ĠMin er +M ER +ĠRe ign +Ġper me +ĠBl itz +Ġ19 34 +Ġintimid ating +t ube +Ġecc entric +ab olic +box es +ĠAssoci ates +v otes +Ġsim ulate +um bo +aster y +Ġship ments +FF FF +an th +Ġseason ed +Ġexperiment ation +âĸ ł +law s +Me et +idd les +ant ics +R ating +IS IS +h ift +Ġfront s +b uf +01 7 +Ġun att +ĠD il +le ases +ĠGard ens +77 7 +t ouch +ve ll +45 8 +Ġ= ==== +s aving +Ġer osion +ĠQu in +Ġearn s +Ġaccomplish ment +ĠWe i +Ġ< [ +____ _ +Ġir rig +ĠT eddy +Ġconqu ered +ĠArm ored +Ġassert s +Ġmanip ulating +r é +Ġtranscript s +G allery +Ġplot ting +Ne il +Ġbetray al +load er +ĠS ul +Ġdispl acement +Ġroy alty +ĠW I +he it +ĠDev ices +alle l +Ġmunicipal ities +Ġcan al +St ars +ĠU AE +Ġ" âĢ¦ +ĠC U +ab ove +Ġreson ance +ĠguiActive Un +add ed +ĠBra ves +ĠI bn +Ġhere by +ĠB RE +Ġshare holder +ĠH ir +ĠJ i +Ġstrange ly +Ġadm ired +Ġpl ight +Ġb achelor +ĠP ole +cipl inary +T ony +ĠArmen ian +Ġun man +ĠZion ist +St age +isco ver +Ġautom otive +Ġs idelines +Ġsl ick +ĠRena issance +ĠF UN +Im ages +ĠH aj +Ġp ing +Ġshort cut +ĠBl vd +ĠLook s +Ġbur sts +Ġcl amp +Ġm ish +Ġsort ing +Ġpatri ot +Ġcorrect ness +ĠScand inav +ĠCaval iers +p ython +az ar +Ġ3 75 +ĠJa une +40 9 +Ġdetrim ental +Ġstab bing +Ġpoison ed +Ġf ountain +oc ent +or st +ĠMar i +Ġr ains +ĠO vers +ĠInst itution +ud get +AM Y +t ale +ĠK R +ĠPr ices +Ġhead aches +Ġlands l +ĠA ura +Bon us +ĠZ hao +ĠH ip +Ġhop s +ĠKurd istan +Ġexplo iting +ry n +Ġhypocr isy +op ening +Ġgun shot +Ġw ed +inter stitial +Inter stitial +Ġam en +Bre aking +Ġmarket ed +W ire +ĠC rowd +Contin ue +ĠK nown +ĠEffect ive +ore an +iz ons +Jose ph +Ġescal ation +us ername +Ġcur tain +AT ES +ĠP AR +ĠM iy +Ġcounter fe +l ene +Ġcont enders +d aily +ĠAs c +ĠPhill ip +most ly +Ġfil ename +he ne +Ġresemb ling +Ġst aging +ĠCh loe +Ġw iring +H on +ĠRen ew +ott age +ĠHy brid +m uch +Ġstro kes +Ġpolicy makers +AP TER +ĠArk ham +pl ot +Ġassist ants +Ġde port +ĠSe ga +Ġinflu enza +ĠC ursed +ĠK obe +Ġskin ny +Prov ider +ĠR ip +Ġincrement al +product s +B F +Ġd ome +ĠC redits +Ġlos ers +int s +ĠBet ty +ĠTal ent +ĠD AM +L v +E ss +Ġd ens +tem p +J udge +od ic +Ġ' ( +UR ES +ets k +V O +Ġretrie ved +Ġarchitect s +Ù ĩ +Ġeth ic +ĠSecond ary +st ocks +ad ia +Ġ3 25 +ĠOp inion +Ġsimultane ous +Ġd izz +ul p +Ġsmugg ling +ipp ery +R andom +f acing +ĠD as +Ġstock p +Ġdiscl osures +po inter +Ġcor al +ĠSe lection +ĠP ike +ival ent +Ġruth less +ĠR im +Ġensu ing +ĠExper iment +Ġcongress man +Ġbelie ver +Ġun specified +ĠM ord +Ġknowledge able +ĠV ERY +T X +Ġstra ps +Ġtur f +apesh ifter +Ġmar ital +Ġfl ock +ãģ Ĩ +26 3 +AM ES +ĠOpp osition +Ġtre asures +ĠG OD +Ġmodel ed +ĠWOR LD +Ġ( [ +ĠUs age +H F +Ġ$ ( +uss ed +Ġpione er +E ight +par se +b read +rit z +ĠMir anda +ĠK ant +++ ) +ore n +Ġprov oked +Ġbre eds +ĠIn cludes +ĠPast ebin +ĠFl ip +J ava +Ġbr ink +Ġrum ored +Ġun seen +Ġgar nered +ĠDef in +al ted +Ġtatt oos +Ġhes itation +is itions +ĠWe aver +ĠReport ing +Ġtherap ies +Ġconsult ants +Ġresid ual +ĠMal i +ĠRom a +i ago +ĠRes idents +ub i +Ġremed ies +Ġadapt ive +ĠAl ive +ĠBar cl +Ġwal lets +c rypt +etermin ation +ĠPel osi +Ġsl ipping +oton in +Ġall iances +pat rick +ir is +Ġor th +ĠPer kins +ĠDe V +ĠG ets +Ġdry ing +ge e +fore st +ĠFor get +ore m +33 9 +Ġvague ly +ĠD ion +ĠP orn +ĠH OW +Ġp neum +Ġrub ble +ĠT aste +enc ia +ĠG el +Ġd st +Ġ24 5 +ĠMoroc co +inf lamm +ĠTw ins +Ġb ots +d aughter +ĠB alk +Ġbre thren +Ġlog os +Ġgo bl +f ps +Ġsub division +Ġp awn +Ġsquee zed +Ġmor ale +ĠD W +' " +Ġkn ot +ook y +Ġdiv isive +Ġboost ed +ch y +ãĥ IJ +if act +Ġnewcom ers +ĠWrest ling +Ġsc outs +w olves +R at +Ġnin eteenth +ĠOs borne +St ats +Ġem powered +Ġpsych opath +ĠO EM +ugg age +ĠP K +ĠMoh ammad +P ak +Ġanarch ists +ĠExt ract +est hes +ĠStock holm +l oo +ĠG raph +Ġdeploy ing +ĠStr anger +ĠM old +Ġstaff er +Ġdiscount ed +uck le +ple ase +ĠLand ing +ÃŃ a +Ġ19 3 +Ġan te +Ġrep etition +Ġ+ /- +Ġpar ody +Ġlive ly +AA A +ĠHor us +Ġp its +ind ers +L OC +ĠVen ice +40 6 +ĠDis cover +â Ĩ +ellect ual +Ġp ens +Ġey el +ig uous +Im pl +Ġj oking +Ġinv al +ĠBel fast +Ġcredit ors +ĠSky walker +ov sky +Ġcease fire +Ġse als +is oft +) ). +ĠFel ix +IT S +Ġt resp +ĠBlock chain +ew are +ĠSch war +en ne +mount ed +ĠBe acon +les h +Ġimmense ly +Ġche ering +Em ploy +sc ene +ish ly +atche wan +ĠNic olas +Ġdr ained +ĠEx it +ĠAz erb +j un +Ġflo ated +u ania +De ep +Ġsuper v +Ġmyst ical +ĠD ollar +ĠApost le +ĠR EL +ĠProv ided +ĠB ucks +ãĥ ´ +cut ting +Ġenhance ments +ĠPengu ins +ĠIsa iah +Ġj erk +ĠW yn +Ġst alled +Ġcryptoc urrencies +ĠR oland +sing le +Ġl umin +ĠF ellow +ĠCap acity +ĠKaz akh +W N +Ġfin anced +38 9 +Ġt id +Ġcoll usion +ĠMy r +î Ģ +Sen ator +Ġped iatric +Ġneat ly +Ġsandwic hes +ĠArchitect ure +Ġt ucked +Ġbalcon y +Ġearthqu akes +qu ire +F uture +Ġhe fty +é Ĺ +Ġspecial izes +Ġstress es +Ġs ender +Ġmisunder standing +Ġep ile +Ġprov oke +ĠCol ors +Ġdis may +uk o +[ _ +58 6 +ne utral +Ġdon ating +ĠRand all +Mult i +Ġconvenient ly +ĠS ung +ĠC oca +Ġt ents +ĠAc celer +Ġpart nered +27 2 +ir ming +ĠB AS +s ometimes +Ġobject ed +ub ric +p osed +LC S +gr ass +Ġattribut able +V IS +Israel i +Ġrepe ats +ĠR M +v ag +ut a +in ous +Ġin ert +ĠMig uel +æ Ń +ĠHawai ian +B oard +Ġart ific +ĠAzerb ai +as io +ĠR ent +A IN +Ġappl iances +Ġnational ity +Ġass hole +ĠN eb +Ġnot ch +h ani +ĠBr ide +Av ailability +Ġintercept ed +Ġcontin ental +Ġsw elling +ĠPers pect +b ies +. < +ith metic +ĠL ara +Ġtempt ing +add r +Ġoversee ing +cl ad +ĠD V +ĠGing rich +Ġm un +ĠApp ropri +Ġalter ations +ĠPat reon +Ġha voc +Ġdiscipl ines +Ġnotor iously +aku ya +ier i +? ). +ĠW ent +Ġsil icon +Ġtre mb +Cont ainer +K nown +Ġmort ar +est e +ick a +Ar thur +ĠPre viously +ĠMart y +Ġsp arse +g ins +Ġin ward +ĠParticip ant +C opy +ĠM isc +Ġantib iotic +ĠRet ro +Ġel usive +Ġass ail +ĠBatt alion +ĠB ought +Ġdimin ish +ĠEuro pa +s ession +ĠDanger ous +ies el +Ġdisbel ief +Ġbl asts +ext reme +ĠBoy d +ĠProject s +ĠGu ys +Ġunder gone +Ġgr ill +ĠDw ight +Ġ19 7 +US ER +Ġfiles ystem +Ġcl ocks +T aylor +Ġwra pper +Ġfold ing +ous and +ĠPhilipp ine +ATION AL +ĠPer th +Ġas hes +Ġaccum ulate +ĠGate way +Sh op +orks hire +H an +ĠBar rel +ĠLe h +ĠX V +Ġwh im +Ġrep o +ĠC G +ĠM am +Ġincorpor ating +Ġbail out +Ġlingu istic +Ġdis integ +C LE +Ġcinem atic +ĠF iber +S yn +il ion +ĠCom pos +c hens +Ġne oc +Ġbo iled +F INE +on o +un cle +ik en +ĠB M +Î ¹ +Ġreceipt s +Ġdisp osed +ĠTh irty +ĠR ough +ĠA BS +Ġnot withstanding +oll en +# $ +Ġunrel iable +Ġbl oom +Ġmedi ocre +Ġtr am +ĠTas man +Ġsh akes +Ġmanifest o +ĠM W +Ġsatisf actory +Ġsh ores +Ġcomput ation +Ġassert ions +orm ons +ar ag +ab it +Dem ocrats +ĠL oot +ĠVol ks +ha ired +Ġgrav itational +S ing +ĠM iz +Ġthro ttle +Ġtyr anny +ĠView s +Ġrob ber +ĠMinor ity +Ġsh rine +sc ope +pur pose +Ġnucle us +our cing +ĠUS DA +ĠD HS +w ra +ĠBow ie +Sc ale +ĠB EL +x i +I ter +Ġ( ), +w right +Ġsail ors +ous ed +NAS A +ĠPro of +ĠMin eral +t oken +ĠF D +R ew +Ġe ll +6 30 +Ġchance llor +ĠG os +Ġamount ed +ĠRec re +ome z +ĠOpt im +ĠOl ive +Ġtrack er +ow ler +ĠUn ique +R oot +Ġmar itime +ĠQur an +ĠAd apt +Ġecosystem s +ĠRe peat +ĠS oy +ĠI MP +Ġgrad uating +and em +P ur +ĠRes et +ĠTr ick +ĠPh illy +ĠT ue +ĠMalays ian +Ġclim ax +Ġb ury +Ġcons pic +ĠSouth ampton +ĠFl owers +Ġesc orted +ĠEduc ational +ĠI RC +Ġbrut ally +e ating +Ġpill ar +ĠS ang +ĠJ ude +ar ling +ĠAm nesty +Ġrem inding +ĠAdminist rative +hes da +Ġfl ashed +ĠP BS +per ate +fe ature +Ġsw ipe +Ġgra ves +oult ry +26 1 +bre aks +ĠGu er +Ġsh rimp +ĠV oting +qu ist +Ġanaly tical +Ġtables poons +ĠS OU +Ġresear ched +Ġdisrupt ed +Ġj our +Ġrepl ica +Ġcart oons +b ians +} ) +c opy +G ot +ou ched +P UT +Ġsw arm +not ations +s aid +Ġreb uilt +Ġcollabor ate +Ġr aging +Ġn ar +Ġdem ographics +ĠD DR +Ġdist rust +oss ier +ĠK ro +Ġpump kin +Ġreg rets +Ġfatal ities +ĠL ens +ĠO le +p d +Ġpupp et +ĠOut look +ĠSt am +O l +F air +U U +Ġre written +Ä ± +Ġfasc inated +Ġve ctors +Ġtrib unal +u ay +ĠM ats +ĠCo ins +[ [ +Ġ18 1 +Ġrend ers +ĠK aepernick +Ġesp ionage +Ġsum m +Ġd itch +Acc ount +Ġspread sheet +Ġmut ant +p ast +40 7 +Ġd ye +Ġinit iation +Ġ4 000 +Ġpunish able +Ġth inner +ĠKh al +Ġinter medi +D un +ĠGoth am +Ġeager ly +Ġvag inal +p owers +V W +ĠWATCH ED +Ġpred ator +ams ung +Ġdispar ity +Ġ[ * +Ġam ph +Ġout skirts +ĠSpir its +Ġskelet al +Ð » +ĠR ear +Ġissu ance +ĠLog ic +re leased +Z Z +ĠB ound +Ent ry +Ġex its +is ol +ĠFound er +Ġw re +ĠGreen land +ĠM MO +t aker +IN C +ãģ ¾ +Ġhour ly +hen ko +Ġfantas ies +Ġdis ob +Ġdemol ition +ãĥ ĭ +Ġen listed +rat ulations +Ġmis guided +Ġens ured +Ġdiscour aged +m ort +Ġfl ank +Ġc ess +Ġreact s +ĠS ere +s ensitive +ĠSer pent +ass ad +Ġ24 7 +Ġcalm ly +b usters +Ġble ed +ĠSt ro +Ġamuse ment +ĠAntar ctica +Ġs cept +ĠG aw +a q +ason ic +Ġsp rawling +n ative +atur ated +ĠBattle field +IV ERS +E B +ĠG ems +ĠNorth western +ĠFil ms +ĠAut omatic +Ġappre hend +ãģ ¨ +Ġgui Name +Ġback end +Ġevid enced +ge ant +01 2 +ĠS iege +Ġexternal To +Ġunfocused Range +ĠguiActiveUn focused +Ġgui Icon +ĠexternalTo EVA +ĠexternalToEVA Only +F ri +ch ard +en aries +Ġchief s +Ġc f +ĠH UD +Ġcorro bor +Ġd B +ĠT aken +ĠPat ricia +ra il +ĠCh arm +ĠLiber tarian +rie ve +Person al +ĠO UR +ger ies +Ġdump ing +Ġneurolog ical +it imate +ĠClint ons +raft ed +ĠM olly +Ġtermin als +reg ister +Ġfl are +Ġenc oded +Ġautop sy +p el +m achine +Ġexempt ions +ĠRoy als +d istance +Ġdraft s +Ġl ame +ĠC unning +Ġsp ouses +ĠMark ets +ĠCar rier +Ġimp lying +ĠY ak +s id +Ġl oser +Ġvigil ant +Ġimpe achment +Ġaug mented +ĠEmploy ees +Ġunint ended +tern ally +ĠW att +Ġrecogn izable +ess im +æ Ŀ +Ġco ated +r ha +Ġlie utenant +ĠLegisl ation +pub lished +44 4 +01 3 +Ġide ally +ĠPass word +Ġsimpl ify +ĠMet a +ĠM RI +Ġple ading +organ ized +hand ler +Ġun ravel +cor rect +Ġ icy +Ġparan oid +Ġpass er +Ġinspect ions +of er +ĠHealth care +28 3 +ĠBr ut +iol a +for ge +ĠMed ieval +MS N +ie vers +ĠProgram ming +å ī +Ġ2 23 +m u +ĠC LE +ug a +Ġsho ppers +Ġinform ative +ĠPl ans +Ġsupplement ation +ĠT ests +ty ard +ocy tes +ĠVeg a +ĠGujar at +erman ent +Ex cept +ĠL OT +all a +ĠC umm +ĠO sw +Ġven om +ĠDeb t +ĠD OWN +Ġreun ion +Ġm uc +ĠRel ief +Ġge op +ĠðŁ ĺ +al ogue +An th +ech o +Ġcor ros +Ġrepl ication +ĠBl azing +ĠD aughter +Ġinf lic +ĠLind sey +Ù Ī +28 4 +Ex it +Ġgl oom +TA IN +Ġundermin ing +Ġadv ising +h idden +Ġover flow +Ġg or +urd ue +Ġe choes +enh agen +Ġimp uls +d rug +c ash +Ġas ync +Ġmir ac +at ts +p unk +Ġpiv ot +ĠLegisl ative +Ġblog gers +ĠCl aw +s burg +d yl +ĠRecomm end +Ġver te +Ġprohib iting +ĠPant her +Jon athan +Ġo min +Ġhate ful +28 1 +ĠOr che +ĠMurd och +down s +Ġas ymm +G ER +Al ways +Ġinform s +ĠW M +ĠP ony +ĠApp endix +ĠAr lington +J am +Ġmedic inal +ĠS lam +IT IES +Ġre aff +ĠR i +F G +S pring +b ool +Ġthigh s +Ġmark ings +ĠRa qqa +ĠL ak +p oll +ts ky +ĠMort y +ĠDef inition +Ġdeb unk +end ered +ĠLe one +a vers +Ġmortg ages +App arently +N ic +ha us +ĠTh ousands +au ld +Ġm ash +sh oot +Ġdi arr +Ġconscious ly +H ero +e as +ĠN aturally +ĠDestroy er +Ġdash board +serv ices +R og +Ġmillenn ials +Ġinv ade +- ( +Ġcomm issions +ĠA uckland +Ġbroadcast s +Ġfront al +Ġcr ank +ĠHist oric +Ġrum ours +CT V +Ġster il +Ġboost er +rock et +ãĤ ¼ +ut sche +ĠP I +Ġ2 33 +ĠProdu cer +ĠAnaly tics +Ġinval uable +Ġunint ention +ĠC Y +Ġscrut in +Ġg igg +Ġeng ulf +Ġprolet ariat +Ġh acks +ĠH ew +ar ak +ĠSl ime +ield ing +ag her +ĠEll iot +Ġtele com +Ġ2 19 +ult an +ĠAr bor +ĠSc outs +B an +Ġlifes pan +Ġbl asp +38 8 +Ġjud iciary +ĠContin ental +ask ing +Mc C +L ED +Ġbag gage +ĠSorce rer +Ġrem nants +ĠGriff ith +ets u +ĠSub aru +ĠPerson ality +des igned +ush ima +agn ar +Ġrec oil +Ġpass ions +\ ": +Ġte e +Ġabol ition +ĠCreat ing +j ac +Ġ19 4 +01 9 +Ġpill ars +ric hed +/ " +t k +Ġlive lihood +Ġro asted +ah on +ĠH utch +ass ert +Ġdivid end +Ġkn it +Ġd aunting +Ġdisturb ance +Ġsh ale +Ġcultiv ated +Ġrefriger ator +L B +ĠN ET +Ġcommercial s +Ġthink ers +45 5 +Ġch op +B road +Ġsuspic ions +Ġtag ged +l ifting +Ġsty lish +ĠShield s +Short ly +Ġt ails +A uth +ST E +ĠG AME +Ġse ism +ĠK is +olog ne +Ġcow ork +Ġforc ibly +Ġthy roid +ĠP B +AN E +mar ried +h orse +Ġpoly mer +ĠCh al +od or +DE BUG +ĠCon text +Ġbl iss +Ġpin point +ĠMat hemat +leg ram +ĠWeek end +Ġlab elled +Ġb art +it les +Ġest rogen +âĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶ âĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶ +" ' +Ġvis ibly +Ġouts ider +aid a +Are a +Ġdisse min +Ġdish onest +ĠCl osed +ĠBullet in +ĠRam sey +sw ord +ĠX I +our ced +S ame +34 6 +ĠRe pe +ĠK ou +c ake +em is +C ache +ĠMe aning +ĠEn light +onom y +Ġmanifest ation +sw orth +J ay +Ġch ore +ö r +D ream +Ġsanction ed +Ġcult urally +ĠA ra +N av +Ġthe ological +Ġstr ut +ĠV O +ĠHand book +Ġconstruct ing +Ġ ¶ +ĠBenef its +ĠPsych ological +s ac +å ¸ +p olicy +ĠMat ters +ĠReport ed +ĠBy te +Ġvit ro +ĠM aiden +Ġl am +ĠJenn ings +Ġgar ment +ĠRut gers +ĠStaff ord +ĠWell ington +Ġinter mitt +Ġn pm +Ġord eal +Ġplug ged +o oming +in ished +fram ework +Ġtim ber +Ġc ass +Ġ8 50 +il ess +ĠRed ux +7 68 +St re +Ġsurpass ed +w hel +Ġparalle ls +Ġve il +ĠG I +ĠR EST +Ġread iness +s ort +Ġmod ifying +ĠSl ate +ru ff +Ġmar ble +Ġinf rared +Ġaud itor +ĠFANT ASY +ĠP overty +ĠS PD +Ġ" ( +K y +RA Y +Ġexecut ions +ĠBever ly +ĠMarx ism +ĠBur st +ĠK ali +est ones +Clear ly +E ll +ãģ § +ĠProceed ings +T oken +IF IC +ñ a +Cent ral +ĠH aley +ĠD rama +Ġform ations +OR N +Book s +Ġdom inating +ĠFly ers +ĠCompan ion +Ġdiscipl ined +ĠYug oslav +ĠSpell s +Ġv engeance +Ġland lords +L en +ĠO gre +ano ia +Ġpier cing +Ġcon greg +Ġscore r +ob ia +Ġnic kel +ĠLear ns +Ġre jo +Ġmaster piece +Fl ash +Ġinhab ited +ĠOpen GL +ĠD ud +ĠI CO +Ġar ter +Ġpl ur +Ġmaster y +Ġlong standing +st ed +Ġw ines +Ġtelev ised +ĠSh rine +ĠBay ern +Ġâ ĵĺ +Ġencl osure +j ohn +Ġprophe ts +ĠRes urrection +ĠOrd ers +Ġun even +r als +Ġd wind +ĠL ah +ĠSl oven +37 8 +Ġins istence +aff le +ĠCl one +Ġhard ship +ĠCongress man +Ġple ad +Ġreview ers +Ġc ured +Ġ19 35 +as ley +f ake +ĠTh inking +yd ia +P ART +ĠD ota +o it +Ġwh ipped +Ġb ouncing +ĠHispan ics +com ings +Ġcann abin +ĠCh ambers +ĠZ ack +Option al +Ġco ats +Ġprow ess +ĠNort on +Ġplain ly +Ġfre ight +Ġinhib ition +Ġcl am +Ġ30 3 +ke f +ale igh +L uke +Ġpsych o +ator ium +M ED +Ġtreat ies +Ġind isc +Ġd c +OP S +Ġresil ient +ĠInter state +Ġsl ack +Ġmund ane +Ġestab lishes +35 9 +Ġstr ained +Ġn ond +S us +Ġcast e +ar ate +ie ving +Ġunfair ly +Ġpars er +on ial +urs ive +V ia +ĠOtt o +ĠAuthor ities +stro ke +K R +ĠMer cy +Ġfurn ished +Ġout set +Ġmet ic +19 82 +olith ic +ĠT ent +og ical +ĠA ircraft +Ġh ides +ĠBec ame +Ġeduc ators +re aching +Ġvol atility +Ġtodd ler +ĠNAS CAR +ĠTw elve +ĠHigh lights +Ġgra pe +Ġspl its +Ġpe asant +Ġre neg +ĠMS I +Tem p +st ars +Ġtre k +ĠHy de +b inding +Ġreal ism +Ġox ide +ĠH os +Ġmount s +Ġbit ing +Ġcollaps ing +Ġpost al +Ġmuse ums +Ġdet ached +Ġrespect ing +Ġmonop ol +Ġwork flow +ĠC ake +Tem plate +ĠOrgan isation +Ġpers istence +36 9 +C oming +B rad +Ġredund ant +ĠG TA +Ġb ending +Ġrev oked +Ġoff ending +Ġfram ing +Ġprint f +Comm un +mem bers +Out side +Ġconst rued +Ġc oded +F ORE +Ġch ast +Ch at +Ind ian +ĠY ard +? !" +ĠP orts +ĠX avier +ĠR ET +' ." +ĠBo at +iv ated +ich t +umer able +D s +ĠDun n +Ġcoff in +Ġsecure ly +ĠRapt ors +ĠB es +Install ation +Ġin ception +ĠHealth y +end ants +Ġpsych ologists +ĠShe ikh +c ultural +ĠBlack Berry +sh ift +F red +oc he +Ġc akes +ĠS EO +ĠG ian +ĠAs ians +og ging +e lement +Ġpund its +ĠV augh +ĠG avin +Ġh itter +Ġdrown ed +Ġch alk +ĠZ ika +Ġmeas les +80 2 +âĢ¦ .. +ĠAW S +] " +Ġdist ort +ĠM ast +Ġantib odies +ĠM ash +Mem ory +ĠUg anda +ĠPro b +Ġvom iting +ĠTurn s +Ġoccup ying +Ġev asion +ĠTher apy +Ġprom o +Ġelect r +Ġblue print +ĠD re +pr iced +ĠDep ot +Ġallev iate +ĠSom ali +m arg +n ine +Ġnostalg ia +ĠShe pherd +Ġcaval ry +Ġtor ped +ĠBlood y +x b +Ġs ank +Ġgo alt +report print +embed reportprint +clone embedreportprint +ĠIn itially +ĠF ischer +Ġnot eworthy +c ern +Ġin efficient +raw download +rawdownload cloneembedreportprint +c ation +ĠD ynasty +l ag +D ES +Ġdistinct ly +ĠEston ia +Ġopen ness +Ġg ossip +ru ck +W idth +ĠIb rahim +Ġpet roleum +Ġav atar +ĠH ed +ath a +ĠHog warts +Ġc aves +67 8 +Ġsafegu ard +ĠM og +iss on +ĠDur ham +sl aught +ĠGrad uate +Ġsub conscious +ĠEx cellent +ĠD um +---- - +Ġp iles +ĠW ORK +ĠG arn +ĠF ol +ĠAT M +Ġavoid s +ĠT ul +Ġble ak +EL Y +iv ist +light ly +P ers +ĠD ob +ĠL S +Ġins anity +Î µ +atal ie +En large +Ġtw ists +Ġfault y +Ġpir acy +Ġimp over +Ġrug ged +ĠF ashion +Ġs ands +' ? +sw ick +Ġn atives +Ġhe n +ĠNo ise +ãĥ Ĺ +Ġg reens +Ġfree zer +Ġd ynasty +ĠFather s +ĠNew ark +Ġarchae ological +Ġo t +ob ar +Ġblock ade +Ġall erg +L V +Ġdeb it +ĠR FC +ĠMil ton +ĠPress ure +Ġwill ingly +Ġdisproportion ate +Ġopp ressive +Ġdiamond s +Ġbelong ings +19 70 +Ġbell s +Ġimperial ism +Ġ2 27 +Ġexpl oding +ĠE clipse +Ġ19 19 +Ġr ant +Ġnom inations +34 7 +Ġpeace fully +ric a +ĠF UCK +Ġvib ration +mal ink +Ġro pes +ĠIv anka +ĠBrew ery +ĠBook er +ĠOw ens +go ers +Serv ices +ĠSn ape +Ġ19 1 +39 5 +Ġ2 99 +just ice +Ġb ri +Ġdisc s +Ġprom inently +Ġvul gar +Ġsk ipping +l ves +Ġtsun ami +37 4 +ĠU rug +ĠE id +rec ated +p hen +Ġfault s +ĠStart ed +9 50 +Ġp i +Ġdetect or +Ġbast ard +Ġvalid ated +Space Engineers +OUR CE +Ġ( ~ +Ġuns ur +Ġaff irmed +Ġfasc ism +Ġres olving +ĠCh avez +ĠC yn +Ġdet ract +L ost +Ġrig ged +Ġhom age +ĠBrun o +55 5 +ec a +Ġpress es +Ġhum our +Ġsp acing +Ġ' / +olk ien +C oun +OP ER +T re +S on +ĠCambod ia +ier re +m ong +o zy +Ġliquid ity +ĠSov iets +ĠFernand o +Ġ2 29 +Ġsl ug +ĠCatal an +elect ric +Ġsc enery +ĠH earth +Ġconst rained +Ġgoal ie +ĠGu idelines +ĠAm mo +ĠPear son +Ġtax ed +Ġfet us +Resp onse +ĠAlex is +th ia +G uy +Ġrecon struct +Ġextrem es +Ġconclud ing +ĠP eg +ook s +Ġded uctions +R ose +Ġground breaking +ĠT arg +ãĥ ģ +ĠRe ve +res ource +Ġmo ons +Ġelectrom agnetic +Ġamid st +ĠVik tor +N ESS +B ACK +Ġcomm ute +ĠAna heim +Ġfluct uations +6 40 +Ġnood les +ĠCop enhagen +ĠT ide +ĠGri zz +ĠS EE +Ġpip elines +Ġsc ars +end o +ag us +ĠE TF +/ # +ĠBec ome +44 8 +Ġvis c +ĠRecomm ended +Ġj umper +Ġcogn ition +Ġassass in +Ġwitness ing +ĠSet up +Ġl ac +v im +IS M +p ages +SS L +35 8 +Ġad ject +indust rial +l ore +cher y +Ġgl itter +Ġc alf +Flor ida +Ġspoil ers +Ġsucceed s +Ġch anting +Ġslog ans +ĠTr acy +Vis it +rol ogy +Ġm ornings +Ġline age +Ġs ip +Ġintense ly +Ġflour ish +ĠSle eping +ĠF em +or por +ĠK lan +ĠDar th +h ack +ĠNi elsen +Ġtum ors +Ġprocure ment +ĠY orkshire +Ġra ided +K Y +An na +Ġ// [ +ĠDis order +ĠMust ang +ĠW en +ĠTry ing +s q +Ġdeliver ies +Ġshut ter +Ġcere bral +Ġbip olar +ĠC N +l ass +j et +Ġdeb ating +> : +Ġe agle +gr ades +ĠD ixon +UG C +M AS +ĠDr aco +ĠMach ines +aff er +Ġem an + ² +pr on +ĠG ym +Ġcompar atively +ĠTrib unal +PR O +Ġle x +Ġfert ile +Ġdep ressing +Ġsuperf icial +ess ential +ĠHun ters +g p +Ġprom inence +L iber +ĠAn cest +ote chnology +Ġm ocking +ĠTra ff +ĸ ļ +Med ium +I raq +Ġpsychiat rist +Quant ity +ĠL ect +Ġno isy +5 20 +G Y +Ġsl apped +ĠM TV +Ġpar a +p ull +Mult iple +as her +Ġn our +ĠSe g +Spe ll +v ous +ord ial +Sen ior +ĠGold berg +ĠPl asma +ne ed +Ġmess enger +ere t +Ġteam ed +Ġliter acy +ĠLe ah +ĠD oyle +Ġem itted +U X +Ġev ade +Ġm aze +Ġwrong ly +ĠL ars +Ġstere otype +Ġpled ges +Ġarom a +ĠM ET +Ġac re +ĠO D +Ġf f +Ġbrew eries +ĠH ilton +und le +ĠK ak +ĠThank fully +ĠCan ucks +in ctions +ĠApp ears +Ġco er +Ġundermin ed +ro vers +And re +Ġbl aze +um ers +Ġfam ine +amp hetamine +ulk an +Am ount +Ġdesper ation +wik ipedia +develop ment +ĠCor inth +uss ia +Jack son +L I +N ative +R s +Oh io +ĠKath leen +F ortunately +Ġattend ant +ĠPre ferred +ĠDid n +ĠV s +M is +Ġrespond ent +Ġb oun +st able +Ġp aved +Ġunex pl +ĠChe ney +L M +ĠC ull +bl own +Ġconfront ing +oc ese +serv ing +W i +ĠLith uania +ann i +Ġst alk +h d +Ġv ener +AP H +ynchron ous +UR R +um ably +hist oric +H alf +H ay +Ġresil ience +spe ction +Ġabandon ing +O bs +ĠDeb bie +Ġgrad ient +ĠPl aint +ĠCan al +AR CH +Ġexpans ive +Ġfun g +Ġb ounced +U nd +Ġprec autions +Ġclar ification +Ġd agger +Ġgri ps +Ġ µ +ĠRiver a +ĠUnd ead +is ites +ĠFIR ST +ñ o +aud i +Ġhost ages +Ġcompl iant +Ġal umni +Se ven +Ġcyber security +e ither +Col lect +Ġinvari ably +ĠS oci +Ġlaw maker +Ġa le +ĠPerson ally +N azi +Ġcustom ization +ĠPro c +ĠSask atchewan +eat uring +Ġsp ared +Ġdiscontin ued +Ġcomput ational +ĠMotor ola +Ġsuprem acist +government al +Ġparad ise +ĠDown ing +ĠNik on +Ġcat alyst +ber ra +Tor onto +8 75 +bet a +ĠMac ron +Ġunreal istic +ve ctor +ĠVeh icles +it iveness +ĠR V +ĠCol bert +s in +o ji +ent in +ĠKr ish +hell o +ff ield +ok y +ĠT ate +Ġmap le +Ġa ids +chem ical +33 4 +n uts +ĠWar p +Ġx x +ĠRob b +umer ous +_- _ +ft ime +ĠV W +Ġw inger +ĠD ome +t ools +ĠP V +ĠGe orgetown +Ġg eared +Ġjihad ists +Ġc p +Ġster oids +M other +cler osis +ĠDR M +nes ia +Ġl inger +Ġimm ersive +ĠC OUN +Ġoutwe igh +ens ual +B and +Ġtransform s +mat ched +ps ons +ĠJud icial +f actor +Ġrefer ral +Ġodd ly +ĠW enger +B ring +ĠB ows +60 2 +IC LE +Ġl ions +ĠAcad emic +ĠTh orn +ĠRa ider +kef eller +St orage +L ower +ĠOr t +ĠEqu ality +AL T +ĠS OC +T ypes +Ġl yn +ĠAss et +co at +TP P +C VE +ĠPione er +app lication +Mod ern +ĠH K +En vironment +Al right +R ain +IP P +ĠShi ite +Ġm ound +ĠAb ilities +cond ition +St aff +Ġcompet ence +ĠM oor +ĠDi ablo +Ġwith held +Ġost ensibly +ĠB rom +Ġms g +Ġden omin +ĠRef erences +ĠF P +Ġplun ged +Ġp amph +m oving +cent ral +Ġdown right +Ġf ading +T al +T yp +ĠTh y +uk es +it he +Ġo ve +Ġbatt led +Ġseaf ood +Ġfig ur +ĠR D +c rop +Ġsqu ads +{ \ +à ¹ +ĠE h +Ġinterview ing +ĠQ in +Ġas piring +PL IC +Ġcla uses +ĠG ast +ĠN ir +Ġl uggage +Ġh ose +Ġsystem d +Ġdesc ending +ĠRev ised +ĠR ails +al ign +70 9 +33 7 +Ġf ug +charg ing +t ags +Ġut er +k ish +WAR NING +49 0 +prof its +Ġvoy age +Ġa ce +ĠV anguard +ĠT anks +ĠM uk +Ġ2 26 +S afe +Ar mor +Ġvolcan ic +Ġwom b +ĠM IL +Ġbegin ner +ĠRec ogn +ĠA AP +PL AY +) ! +Ġdetect ing +c n +Ġbre aches +Bas ically +ĠP ag +ĠMunicip al +ĠInd ie +ĠL af +ĠDis able +ĠOl son +Ġrest rained +Ġrul ings +Ġhum ane +ev ents +ĠCinem a +display Text +ĠH atch +action Date +onna issance +Ġassault ing +ĠL ug +CH AT +Ġvig orous +ĠPer se +Ġintoler ance +ĠSnap chat +ĠSh arks +Ġd ummy +ĠDi agn +ĠGu itar +im eters +40 3 +RE G +A x +Ġsepar ates +ĠMah m +Ġt v +j ah +O OL +C irc +ĠWinds or +uss ian +Ġintu ition +Ġdis dain +ĠDon ovan +Ġ2 21 +E mb +Ġcondem ning +Ġgener osity +zz y +Ġpant ies +ĠPre vent +Action Code +AN A +34 2 +external ActionCode +Ġspec ifying +Ġcryst all +J ere +Ġru pt +ĠApp rentice +Ġprof iling +Ð º +St rike +Ġsid eline +Ġoblig ated +Ġocc ult +Ġbureaucr atic +ant ically +rupt ed +neg ative +ĠEthiop ia +ĠC ivic +Ġins iders +el igible +ĠTV s +ĠB AR +ĠT I +i ologist +ĠA IR +Ġsubstit uted +Ar ab +ĠS aul +ĠY og +p rem +Ġbuild ers +Ġstation ary +Ġdoubt ful +Ġvig orously +Ġthr illing +Ph ysical +ĠCare y +ĠHyd ra +geon ing +ĠS ly +y ton +Ġborrow ers +ĠPark inson +Ġ ë +ĠJama ica +Ġsat ir +Ġinsurg ents +ĠF irm +Ġis ot +ĠK arn +our ning +ak ens +doc s +l ittle +ĠMon aco +CL ASS +Tur key +L y +ĠCon an +ass ic +Ġstar red +ĠPac ers +et ies +Ġt ipping +M oon +ĠR w +s ame +Ġcav ity +Ġgo of +ĠZ o +Sh ock +um mer +Ġemphas izes +Ġreg rett +Ġnovel ty +Ġen vy +ĠPass ive +r w +50 5 +Ġind ifferent +ĠR ica +ĠHim self +ĠFred die +Ġad ip +ä¸ Ģ +Ġbreak out +Ġhur ried +ĠHu ang +ĠD isk +Ġro aming +?????- ?????- +U V +ĠRick y +ĠS igma +Ġmarginal ized +Ġed its +Ġ30 4 +mem ory +Ġspec imen +29 3 +ãģ ¯ +Ġvert ically +Ġaud ition +ĠHe ck +Ġc aster +ĠHold ings +ad al +ĠC ron +ĠL iam +Ġdef lect +P ick +ĠDeb ug +RE F +Ġvers atility +ot hes +class ified +ĠMah ar +ĠH ort +C ounter +st asy +not iced +33 1 +ĠSh im +f uck +ĠB ie +Ġair ing +ĠPro tein +ĠHold ing +Ġspect ators +ili ated +ĠThat cher +n osis +ãĥ¼ ãĥ³ +Te le +B oston +ĠTem pl +st ay +Ġdecl arations +47 9 +Vol ume +ĠDesign er +ĠOver watch +id ae +Ġon wards +Ġn ets +ĠMan ila +part icularly +Ġpolit ic +o other +Ġport raits +Ġpave ment +c ffff +Ġs aints +Ġbegin ners +ES PN +Ġshort comings +âķIJ âķIJ +Ġcom et +ĠOrgan ic +qu el +Ġhospital ized +Bre ak +Ġpe el +dyl ib +asp x +ur ances +ĠT IM +P g +Ġread able +ĠMal ik +Ġm uzzle +Ġbench marks +d al +ĠV acc +ĠH icks +60 9 +ĠB iblical +he ng +Ġover load +ĠCivil ization +Ġimm oral +Ġf ries +ãĤ Ĵ +Ġreprodu ced +Ġform ulation +j ug +ire z +g ear +Ġco ached +Mp Server +ĠS J +ĠK w +In it +d eal +ĠO ro +ĠL oki +ĠSong s +Ġ23 2 +ĠLou ise +asion ally +Ġunc ond +olly wood +Ġprogress ives +ĠEn ough +ĠDo e +Ġwreck age +Ġbr ushed +ĠBase Type +Ġz oning +ish able +het ically +ĠC aucus +ĠH ue +Ġk arma +ĠSport ing +Ġtrad er +Ġseem ing +ĠCapt ure +4 30 +b ish +Ġt unes +Ġindo ors +ĠSp here +ĠD ancing +TER N +Ġno b +ĠG ST +m aps +Ġpe ppers +F it +Ġoverse es +ĠRabb i +ĠR uler +vert ising +off ice +xx x +Ġra ft +Ch anged +Ġtext books +L inks +ĠO mn +ãĢ ij +Ġinconven ience +ĠDon etsk += ~ +Ġimplicit ly +Ġboost s +ĠB ones +ĠBo om +Cour tesy +Ġsens ational +AN Y +Ġgre edy +ed en +Ġinex per +ĠL er +ĠV ale +Ġtight en +ĠE AR +ĠN um +Ġancest or +S ent +ĠH orde +urg ical +all ah +Ġsa p +amb a +ĠSp read +tw itch +Ġgrand son +Ġfract ure +Ġmoder ator +ĠSe venth +ĠRe verse +Ġestim ation +Cho ose +Ġpar ach +Ġbar ric +ãĢ IJ +Ġcomp ass +Ġall ergic +âĢ ķ +OT HER +err illa +Ġw agon +Ġz inc +Ġrub bed +ĠFull er +ĠLuxem bourg +ĠHoo ver +Ġli ar +ĠEven ing +ĠCob b +est eem +Ġselect or +ĠB rawl +is ance +ĠE k +Ġtro op +Ġg uts +ĠApp eal +ĠTibet an +Ġrout ines +ĠM ent +Ġsummar ized +steam apps +Ġtr anqu +Ġ19 29 +or an +ĠAut hent +Ġg maxwell +Ġappre hens +Ġpo ems +Ġsa usage +ĠWeb ster +ur us +Ġthem ed +Ġl ounge +Ġcharg er +Sp oiler +Ġsp illed +h og +ĠSu nder +ĠA in +ĠAng ry +Ġdis qual +ĠFrequ ency +ĠEther net +Ġhel per +Per cent +Ġhorr ifying +Ġa il +ĠAll an +EE E +ĠCross ing +44 9 +Ġh olog +ĠPuzz les +ĠGo es +eren n +60 4 +ãģ ı +ĠRaf ael +Ġatt en +ĠE manuel +Ġup ro +ĠSus p +P sych +ĠTr ainer +ĠN ES +ĠHun ts +bec ue +Ġcounsel or +R ule +Ġtox ins +Ġb anners +r ifice +Ġgreet ing +Ġfren zy +Ġall ocate +Ġ* ) +ex pr +50 3 +ĠCh ick +ĠT orn +Ġconsolid ation +ĠF letcher +sw itch +fr ac +cl ips +ĠMcK in +ĠLun ar +Mon th +IT CH +Ġscholar ly +rap ed +39 8 +Ġ19 10 +Ġe greg +Ġin secure +Ġvict orious +cffff cc +Ġsing led +Ġel ves +ĠW ond +bur st +Ġcam oufl +ĠBL ACK +Ġcondition ed +ç ī +ans wered +Ġcompuls ory +asc ist +Ġpodcast s +ĠFrank furt +bn b +Ġne oliberal +ĠKey board +ĠBel le +w arm +Ġtrust s +Ġins ured +ĠBu cc +us able +60 7 +ĠPl ains +Ġ18 90 +Ġsabot age +Ġlod ged +f elt +Ġg a +ĠN arc +ĠSal em +Ġsevent y +ĠBl ank +p ocket +Ġwhis per +Ġm ating +om ics +ĠSal man +ĠK ad +Ġan gered +Ġcoll isions +Ġextraord inarily +Ġcoerc ion +G host +b irds +è Ģ +k ok +Ġper missible +avor able +Ġpo inters +Ġdiss ip +ac i +Ġtheat rical +ĠCos mic +Ġforget ting +Ġfinal ized +å¤ § +y out +l ibrary +Ġbo oming +ĠBel ieve +ĠTe acher +ĠL iv +ĠGOOD MAN +ĠDomin ican +OR ED +ĠPart ies +Ġprecip itation +ĠSl ot +R oy +ĠComb ined +Ġinteg rating +Ġch rome +Ġintest inal +ĠRe bell +Ġmatch ups +Ġblock buster +ĠLore n +ĠLe vy +Ġpre aching +ĠS ending +ĠPur pose +ra x +f if +Ġauthor itative +ĠP ET +ast ical +Ġdish on +Ġchat ting +Ġ"$ :/ +Connect ion +Ġrecre ate +Ġdel inqu +Ġbro th +ĠD irty +ĠAd min +z man +Ġscholars hips +Ġ25 3 +cont act +als a +7 67 +c reen +abb age +Ġ19 15 +Ġbl ended +Ġal armed +L anguage +35 6 +Ġbl ends +ĠCh anged +W olf +Ġhe pat +Creat ing +Ġper secut +Ġsweet ness +art e +Ġforfe iture +ĠRober to +im pro +N FL +ĠMag net +Det ailed +Ġinsign ificant +ĠPOL IT +ĠBB Q +ĠC PS +Ġse aw +amin er +m L +end if +f inals +Ġ26 5 +u ish +Ġ} ) +ĠPro blems +Ġem blem +Ġserious ness +Ġpars ing +Ġsubst itution +Ġpress ured +Ġrecy cled +ale b +Rub y +Ġprof iciency +Dri ver +ĠW ester +: ' +AF TA +Ġm antle +ĠClay ton +fl ag +Ġpractition er +c overed +ĠSt ruct +add afi +4 25 +ĠTown ship +ĠHyd ro +Lou is +34 3 +Ġcond o +ĠT ao +Ġutil ization +Ġnause a +ĠDem s +rid ges +p ause +Ġform ulas +Ġchall enger +37 6 +Ġdefect ive +ĠRail way +ĠPub Med +Ġyog urt +l bs +ĠNor folk +OP E +ĠMood y +Ġdistribut or +Ġscroll s +Ġextract s +St an +Ġv iability +Ġexp oses +Ġstar vation +ĠStep s +ĠD odd +f ew +ST D +33 2 +Ġclos ures +Ġcomplement ary +ĠS asha +ump y +Ġmon et +Ġartic ulate +ĠDo ct +k iller +Ġsc rim +Ġ2 64 +Ġprost itutes +Ġse vered +Ġattach ments +Ġcool ed +L ev +ĠF alk +f ail +Ġpolic eman +ĠD ag +Ġpray ed +ĠK ernel +Ġcl ut +Ġc ath +Ġan omaly +St orm +em aker +ĠBreak fast +ul i +o ire +J J +h z +Oper ation +ĠS ick +35 4 +ĠGuatem ala +R ate +Ġexp osures +f aces +ĠArch ae +ra f +ĠM ia +Ġ20 25 +Ġop aque +Ġdisgu ised +ĠHead quarters +S ah +Ġp ots +9 78 +ĠM alf +Ġfrown ed +Ġpoison ous +ĠCon vers +ee ks +Ġcr ab +." " +Ġtre ason +Ġr anc +Ġescal ating +Ġwar r +Ġmob s +Ġl amps +ĠSun shine +ĠBrun swick +Ph ones +Ġspe lled +ĠSk ip +Ġ20 50 +Ġ19 11 +ĠPl uto +ĠAm end +Ġme ats +38 7 +Ġst omp +ĠZh ou +ĠLevi athan +ĠHaz ard +ad v +ĠOr well +Ġal oud +Ġb umper +ĠAn arch +ub untu +ĠSer ious +f itting +ĠOption al +ĠCec il +RE AM +Ġser otonin +Ġcultiv ate +ag ogue +} \ +Ġmos ques +ĠSun ny +Ġre active +rev olution +ĠL up +ĠFed ora +Ġdefense man +ĠV ID +ist ine +Ġdrown ing +ĠBroad casting +Ġthr iller +ĠS cy +Ġacceler ating +Ġdirect s +od ied +b ike +d uration +Ġpain fully +R edd +Ġproduct ions +Ġg ag +Ġwh ist +Ġs ock +Ġinf initely +ĠConc ern +ĠCit adel +Ġlie u +Ġcand les +ogene ous +arg er +Ġheaven ly +inflamm atory +Per formance +C s +ruct ose +az aki +Ġp essim +Ġinf erence +Ġpow d +ĠZ oe +Ġpain ts +Ġd azz +pt a +-------- --- +Ġins pir +ĠExper imental +ĠKn ife +reg or +b ors +Ġshow ers +rom eda +Ġs aint +Ġben ign +ĠJ iang +Ġenvision ed +Ġsh roud +IF T +H O +Ġsh uff +ĠI CC +Ġse greg +Ġrevis it +ighth ouse +L i +Ġsub strate +ĠSe as +ĠRew ard +ĠH ep +ĠBr ass +s bm +Ġelim inates +Ġst amina +ĠV AT +ĠLo an +Ġconst raint +Ġappropri ated +Ġp es +ĠA LE +r anging +Ġ40 4 +39 2 +Ġintellectual s +ach u +Ġrestruct uring +ĠLe vin +Ġrun es +Ġdelight ful +Ġcarbohyd rates +ĠMod els +ĠExp o +Ġtransport ing +all oc +Ġring ing +S amsung +Ġscarce ly +ĠURL s +ĠM AS +Ġprot otypes +Ġnarr ator +ĠCPU s +cd n +ĠBart on +Ġdecided ly +ĠSh u +ix ir +oc ious +ĠMy st +N intendo +Ġre use +Ġforg iven +F ew +in ical +n at +Ġseam less +ĠEv a +ĠE VE +ĠJ O +land ers +Ġso fter +neg ie +Ġtrans ient +Ġorb ital +Ġfulf il +ĠK om +Hop efully +Ġdynam ically +ĠHun ger +å Ľ +ĠArmen ia +el man +ber to +Ġp ige +ĠID s +lim it +Ġve ins +Ġso aring +p acks +Gold en +ĠCr ab +ist or +ĠR PM +Ġ$ $ +g ression +Ġjihad ist +Ġgam ble +Ġcare g +Ġinf lated +F ace +ĠFire arms +ĠEm manuel +â Ŀ +Ġsh ocks +gr ab +Ġspl end +ĠHP V +ab ortion +Ab ove +Ent ity +play ers +Ġcomm enced +ul ence +Ġfulfill ment +Ġembod iments +ĠW elfare +Ġha il +Ġ< @ +tt en +Ġcat cher +ĠJ azeera +Ġvolcan o +Ġstabil ize +ĠHand ler +Ġintens ified +ĠAb rams +Ġhum iliation +p aced +60 5 +ĠCent OS +Spe cific +Ġhe ed +ĠC AM +ĠGal ile +D ie +Ġabol ished +ĠThom son +ĠTe achers +ĠW ass +j ong +ĠIS BN +ĠAll ies +sh ake +å · +v ict +How ard +Ġde em +Ġexceed ingly +ĠSmart stocks +ib e +Ġdoor way +Ġcompet ed +ig mat +Ġnational ists +Ġg room +ĠKe en +Ġdispos able +de cl +ĠT olkien +ĠSche me +Ġb iod +Ġav id +ĠEl on +ag ar +ĠT SA +R oman +Ġartific ially +Ġadvis ors +X L +ĠInf erno +36 6 +Ġted ious +ĠPhot ography +ĠCar rie +Ġtro pe +ĠSand ra +Ġdec imal +Que en +ĠGund am +ĠO M +ote ch +N BA +Ġ19 32 +Ġent renched +ĠMar ion +Ġfr aternity +Lab our +Hen ry +Ġlat itude +E ither +Ġenh ances +ĠPot ential +Ġsh ines +id ad +Ġbread th +Ġcapac ities +ĠðŁ ĻĤ +ĠBron x +Ġsex es +Ġdifferent iation +Ġheavy weight +ĠT aj +d ra +Ġmigr ate +Ġexhaust ion +ĠR UN +els ius +ĠCu omo +Ġgu itars +Ġcl ones +ĠSom ew +ĠP ry +------------ - +Ġwarr anted +cy cles +Ġsalv age +Ġdis ks +R ANT +ĠNGO s +ĠMart ian +":[ {" +Ġadd icts +oj ure +il let +Ġamazing ly +art ments +p ixel +ĠGPU s +Lay out +è £ +ĠTam il +ĠBas il +Ġimpart ial +ĠSt ructure +f ork +b ryce +Ġr idge +ĠHamb urg +ri ous +Ġbl itz +cig arettes +Ġcan ned +40 2 +Ġiron ically +Ġcompassion ate +ĠHaw kins +. # +ĠCat hedral +Ġrall ied +in ternal +Ġqu ota +st akes +T EXT +m om +Ġcomple tes +Ġ23 8 +Ġsh rug +ãĥ ij +ĠN inth +Ġrev ise +ĠProv ider +Ġtre acher +Ġqu asi +ĠPR ES +Ġdep osition +Ġconfidential ity +iss ors +Ġim balance +Ġspan ning +Ġang ular +ĠC ul +commun ication +ĠNor a +ĠGen ius +op ter +Ġs acked +Sp ot +Ġfine ly +ĠCH R +28 2 +w aves +Pal est +ĠRo hing +N L +è ¿ +Ġsh itty +ĠSc alia +4 75 +Pro gress +Ġreferen cing +Ġclass rooms +ab ee +Ġs od +hes ion +70 8 +ĠZucker berg +ĠFin ish +ĠScot ia +ĠSav ior +ĠInstall ation +an tha +( - +Ġ30 2 +ĠP unk +Ġcr ater +yout u +Ġro ast +Ġinflu encing +Ġd up +ĠJ R +ĠG rav +Ġstat ure +Ġbath rooms +A side +W iki +me an +ĠZ ak +ĠOn es +ĠN ath +Ġhyper t +Ġcommence ment +C ivil +Ġmoder ately +Ġdistribut ors +Ġbreast feeding +Ġ9 80 +ĠS ik +ĠC ig +ĠAM ER +R IP +ĠCare er +ust ing +Ġmess ed +Ġe h +ĠJ ensen +/ $ +Ġblack mail +Ġconvers ions +Ġscientific ally +Ġmant ra +p aying +Ġiv ory +ĠCour ts +OU GH +aunt let +Ser ial +B row +ĠH undreds +3 23 +Ġpe e +Ġlin ux +Ġsub mer +ĠPrinc ipal +48 5 +ĠD SL +ĠCous ins +Ġdoctr ines +ĠAthlet ics +Ġ3 15 +ĠK arma +Ġatt ent +ur ger +Ġpresc ribe +Ġenc aps +ĠC ame +Ġsecret ive +ĠCr imes +d n +C lean +ĠEgypt ians +ĠCar penter +Ġ ll +H um +ĠMil o +Ġcapital ists +Ġbrief ed +T we +ĠBas in +elve t +M os +Ġplun ge +ĠKa iser +ĠFu j +ill in +Ġsafegu ards +Ġo ste +ĠOpportun ity +ĠM afia +ĠCall ing +ap a +ur ban +br ush +ill ard +c é +int elligence +ĠL ob +ĠDru id +Ġsm oother +Ġfoot ing +Ġmotor ists +arc ity +Ġmascul inity +Ġm ism +Ġabdom inal +ĠTa vern +ĠR oh +Ġesc apes +s igned +Anth ony +Ġsacrific ing +Ġintim acy +Ġan terior +ĠK od +Ġmot if +Ġg raz +Ġvisual ization +Ġguitar ist +ĠTro tsky +m agic +D ar +ĠMor i +Ġw ards +Ġtoile ts +l est +Ġtele port +ĠSund ays +ĠPl at +ET S +Ġe Sports +Pat rick +ĠK atherine +en ko +Ġhas sle +ĠM ick +gg les +Ġh ob +aint ain +Ġair borne +Ġsp ans +Ġch ili +Ġa perture +Ġvolunte ered +ĠInc ident +ĠF res +ĠVeter an +augh tered +ing o +Ġun insured +CL OSE +Ġf use +Ġer otic +Ġadvert ise +ra ising +Text ure +Ġatt ends +ĠRE AL +udd led +Ġsm oot +Ġ30 5 +ĠWill is +Ġbl ond +An alysis +ĠV T +on ica +Ġstrongh old +R F +N M +. >> +Ġprosper ous +Ġbo asted +29 2 +ĠManufact uring +PR ESS +g ren +Ġpharm acy +ĠRoc kefeller +k ai +Ġth umbs +ĠH ut +Ġmother board +Ġguard ians +ĠAl ter +ll ular +Ġsh ack +Ġwise ly +Ġback bone +erv a +Ġsu icides +ĠMcG regor +ij ah +E mer +ĠB rav +Ġdesign ate +P OST +produ ced +Ġcleans ing +irl wind +ex istent +ĠHum ph +ĠPay ne +Ġv ested +Å ¡ +Ġstring ent +ion a +Ġuns ub +Ġsum med +ĠHer cules +sub ject +ĠR agnar +ĠN os +Ġcharacter ization +Ġsav vy +ĠDaw son +ĠCas ino +Ġf ri +ĠBar rier +Ġmis information +Ġins ulation +Ġcorrid ors +Ġair planes +ĠNo ct +ah i +Ġ19 16 +k b +arm ac +Ġsh un +Ġsche ma +Ġhorr ified +Ġ23 9 +aund ers +N B +i ates +er ity +ĠSh ard +Ġr arity +Ġgroup ed +ĠGh ana +again st +ĠBi ological +ĠA ware +ow ell +Ï Ħ +ĠBe au +sh aw +H ack +ĠJul ius +US S +ol son +aun a +c ru +ĠMaur ice +ĠI k +Ġsequ encing +Ġradical s +Ġ( ?, +v irtual +Ġany ways +Ġreper c +Ġhand lers +Ġhes itant +é ĥ +ĠM F +ple mentation +ass ociated +Ġcampaign ed +ĠY ue +ut ations +ĠY oga +Ġsim mer +Ġro ds +Ġmel ody +Ġconv oy +v ideos +Ġscreen ed +N eg +ochem ical +Ġ( )) +Ġultr as +Ġant ip +ĠIsland ers +70 4 +Ġfet ish +Ġridic ulously +ĠK art +Ġmitochond rial +Ġinterf ering +Build er +Ġover fl +Ġac ne +ĠM ud +ĠK err +f lex +ĠPost al +ĠBalt ic +47 7 +ĠPers ons +our age +H B +ĠM use +ĠImm ortal +ĠDri ving +Ġpet itions +Ġsubsc ript +Ġs orce +ĠProcess or +ut on +S ony +Ġph on +Ġr aced +ĠAnth rop +Ġday time +ĠEx ercise +Add ing +Ġeng ages +ĠQual comm +Ġmir acles +Ġmem es +ĠDr ink +ĠOri oles +Ġhair s +ĠPol ar +ath om +Ġsl ippery +ĠR emy +Ġcar amel +ĠY EAR +Ġal k +I gn +a ution +ĠMer lin +ĠC ran +Ġap ologies +Ġ4 10 +Ġout ing +ĠMem ories +app ointed +Ġcount ered +u ld +pos ing +Ġfire wall +ĠW ast +ĠW et +work ed +se ller +Ġrepe aled +ere o +ass uming +BL IC +m ite +ĠCEO s +ĠChap el +ellig ent +________________ ________ +D og +Ġw art +Ġsubsc riber +s ports +Ġbe gged +ĠM V +Ġsem if +eth ical +Ġpre ach +Ġrev ital +Ġpun itive +Ġshort cuts +Ġinstit uted +ĠWars aw +Ġabdom en +ĠK ING +Ġsuper intendent +Ġf ry +ĠGe o +T OR +Ġcontrad ictions +apt ic +Ġlandsc apes +b ugs +Ġcl ust +Ġvol ley +c ribed +Ġt andem +Ġrob es +WH AT +Ġpromot er +Ġel oqu +review ed +ĠD K +ĠPl ato +Ġf ps +T ank +ĠDer rick +Ġpriorit ize +as per +ĠHond uras +ĠCom pleted +ne c +Ġm og +n ir +ĠMay o +DE F +st all +in ness +ĠVolks wagen +Ġprec aution +ĠM ell +i ak +ist ries +Ġ24 8 +Ġoverl apping +Sen ate +ĠEnh ance +res y +rac ial +OR TS +ĠM ormons +Str ong +ĠCo ch +Mex ico +ĠMad uro +Ġj ars +Ġcan e +W ik +oll a +iff erence +Ġphysic ist +ĠMag gie +Ġ28 5 +Ġdep iction +ĠMcL aren +J u +Ġsl ows +Ġcommission ers +ĠWill ow +ĠExpl os +hov ah +Ġtechn ician +Ġhom icides +ĠFl av +ĠTr uman +Ġ100 00 +u ctor +Ġsh ader +News letter +45 7 +Ġre ver +Ġhard ened +Ġwhere abouts +Ġrede velop +Ġcar bs +Ġtra vers +Ġsqu irrel +Ġfoll ower +Ġs ings +50 8 +Ġrabb its +emon ium +Ġdocument ing +Ġmisunder stood +) ' +R ick +gg ies +Ġprem ie +Ġsk ating +Ġpass ports +Ġf ists +aged don +H aw +AC P +0 80 +ĠThough ts +ĠCarl son +Ġpriest hood +h ua +Ġdun geons +ĠLo ans +Ġant is +Ġfamiliar ity +ĠS abb +op al +ĠIn k +st rike +Ġc ram +Ġlegal ized +Ġcu isine +Ġfib re +Tra vel +ĠMon ument +OD Y +eth y +Ġinter state +ĠP UR +em porary +ĠArab ian +develop ed +Ġsadd le +Ġg ithub +ĠOff er +ĠIS P +ro let +ĠSUP ER +ĠDen is +Ġmultipl ier +Ġstir red +Interest ingly +Ġcustom ary +Ġbill ed +he x +Ġmultipl ied +Ġfl ipping +ĠCros by +Ġfundament als +ia e +ĠPlay ed +ĠAt om +am azon +ĠFl am +ee z +activ ated +Ġtables poon +Ġliberal ism +ĠPal in +ĠP atel +N um +ĠT AM +Ġs urn +ĠRel oaded +Ġco ined +" ], +ĠCl ash +ĠAg u +Ġprag matic +ĠActiv ate +Ġ8 02 +Ġtrail ers +Ġsil hou +Ġprob es +Ġcirc us +ĠB ain +ĠLind say +ĠAb bey +Del ivery +Ġconcess ion +Ġgast ro +ĠSpr ite +Ä Ł +and el +Ġg imm +Ġaut obi +ĠT urtle +Ġwonder fully +ĠHar am +ĠWorld wide +ĠHand le +Ġtheor ists +Ġsle ek +ĠZh u +ograph ically +EG A +ĠOwn ers +ath s +ĠAntar ctic +n atal +=" " +fl ags +`` `` +Ġs ul +K h +Ġpot assium +Ġlinem an +Ġcere al +ĠSe asons +Ġ20 22 +Ġmat hematic +Ġastron omers +prof essional +Ġf ares +cknow led +Ġch i +Ġyoung sters +Ġmistaken ly +Ġhem isphere +ĠDiv inity +r one +Ġ" , +r ings +Ġattract s +v ana +å ¹ +C AP +Ġplay list +Ġpor ch +ãģ £ +Ġincorpor ates +Ġso ak +Ġassert ing +ĠTerror ism +ĠP ablo +J a +ces ter +Ġfear ing +ĠPr ayer +Ġescal ated +G W +Ġro be +ĠBright on +ac ists +ĠSym phony +ĠDwar f +ĠPar ade +ĠLe go +Ġinex pl +Ġl ords +le af +RA G +l iber +Ġcig ars +ĠJe hovah +60 6 +WIND OWS +ĠLiber ia +eb us +He avy +Ġl ubric +ĠR W +angu ages +Ġnarrow ed +com puter +ĠE mber +Ġmurder ing +Ġdown stream +ĠT uls +ĠT ables +Top ic +ĠAcc uracy += / +l ost +ĠRe i +Ġprogress es +b ear +Ġestablish ments +Just in +ĠPe ach +ĠG omez +å ¿ +ĠTri angle +Id ent +ĠH ive +Res ources +Ġmix es +ĠAss uming +M u +Ġhyp oc +Ġs ane +ĠW an +id ious +Su ccess +Ġ io +Ang el +Ġdanger ously +ĠCreat ure +W ORK +: [ +ĠKat rina +List ener +M iller +ĠId lib +h ang +Ġcircum vent +h ref +Ġcel estial +ĠWe eks +ĠP ug +ĠDal ton +Ġsubpoen a +uk u +Ġpers isted +pe i +old ing +ĠDoc uments +ĠH ast +ĠC ENT +Ġprim er +Ġsyn onymous +Ġn ib +om bs +Ġnot ation +ĠD ish +ĠAt mosp +Ġforb id +ĠAN G +pat tern +l os +Ġproject iles +b rown +." , +ĠVen om +Ġfierce ly +ub lished +ĠU ran +ĠNic arag +4 10 +ĠC AL +OT OS +ĠMir acle +ĠEn chant +Ġguard ing +app end +Att ach +Ġlevel ed +Ġcond oms +ih ilation +64 9 +Ġnight mares +ĠTHE Y +ĠST ART +ĠK inn +Ġroomm ate +Ġhy giene +o pping +J ob +Ġl vl +ĠV ER +ĠKe eping +ab etic +Ġformat ting +eral a +Ġrev isions +Ġres urg +T el +ĠGood man +35 3 +p od +Ġind isp +ĠTrans lation +Ġg own +ĠM und +Ġc is +Ġby stand +col lect +ĠPun jab +act ively +ĠG amb +te ll +Ġimport ing +g encies +Ġloc om +ĠBr ill +H oly +ĠBer ger +Ġshow down +Ġrespond ers +IL Y +Ġt akedown +le ted +Ġmat tered +Ġpredict ive +Ġover lay +G PU +ĠV ick +Ġconvey ed +T ab +pe er +Sc an +Ġdefensive ly +v ae +Ġappro ving +Ġt iers +ĠV ia +quer ade +ĠSaud is +Ġdemol ished +ĠProp he +Ġmon o +Ġhospital ity +H AM +ĠAri el +M OD +ĠTor ah +Ġbl ah +ĠBel arus +erent ial +ĠT uc +Ġbank er +39 7 +Ġmosqu it +ĠScient ist +ĠMus ical +Ġh ust +Sh ift +Ġtor ment +Ġstand off +E duc +ĠF og +Ġampl ifier +Sh ape +Inst ance +ĠCrit ics +Ġda emon +H ouston +Ġmatt ress +ĠID F +Ġobsc ene +ĠA mer +hett i +Ġcomp iling +35 2 +vere tt +ĠRed uction +ist ration +ĠBl essed +ĠB achelor +3 16 +Ġpr ank +ĠVul can +dd ing +Ġm ourning +ĠQu int +ĠBl aster +test ing +Ġsed iment +>> > +ĠE ternity +ĠWH ERE +ĠM aze +Ġreact ing +ĠAl v +oms day +ĠC RA +Ġtransl ator +Ġbog us +at u +We bsite +oll s +Ġbapt ism +Ġs ibling +ĠAut umn +ve z +ãģ® é +gu ards +Ge org +assad ors +ĠFre ud +Ġcontin ents +ĠReg istry +Bern ie +ĸļ 士 +Ġtoler ant +ĠU W +Ġhor ribly +99 5 +ĠMID I +Ġimpat ient +oc ado +er i +ĠWor st +ĠNor ris +ĠTalk ing +Ġdef ends +ens able +Ġ20 21 +Ġanat omy +L ew +Ġdraw er +ĠCan berra +Ġpatri otic +é¾įå ĸļ士 +ĠAv g +AR M +Ġundis closed +Ġfare well +45 9 +b able +ĠAll ison +OL OG +Ġcon co +t ight +ĠAC PI +ĠM ines +l ich +ĠâĶ ľ +represent ed +200 000 +Ġenthusi ast +OT S +b il +ĠIng redients +Ġinvent or +ĠMy SQL +³³ Âł +ĠAB OUT +with in +Ġm k +B ul +ĠF ake +Ġdracon ian +W a +hel m +ĠTer ran +erv ille +Ġcommon place +SI ZE +Ġ" < +re place +ograph s +ĠSE LECT +inc ible +ĠMost ly +ĠShe ffield +ĠID E +ugg le +Ġcit ations +h urst +ĠUn ix +Ġunle ash +ĠP iper +ĠN ano +Ġsucc umb +Ġreluct ance +Ġ25 00 +ĠMer chant +Ġwire t +Ġcomb os +ĠBirth day +Ġchar coal +ĠU PS +ĠFair fax +Ġdrive way +ĠT ek +ĠP itch +ove re +Ġtechn icians +ĠAct ual +fl ation +ĠF iscal +ĠEm pty +an amo +Ġmag nesium +Ġsl ut +Ġgrow ers +Invest igators +( ): +ĠS atellite +ĠKe ynes +miss ive +l ane +Ġb orough +3 44 +ĠTE AM +ĠBet hesda +C V +h ower +ĠR AD +Ġch ant +ĠR iy +Ġcompos itions +Ġmild ly +Ġmedd ling +Ġag ility +ane ers +5 01 +Ġsyn th +ling er +29 1 +Ġex claimed +Part y +Ġcont amin +ĠMan or +ĠResp ond +Ġpra ising +Ġman ners +fle et +Sum mer +ĠLy nd +ĠDef initely +gr im +Ġbow ling +st ri +ç Ľ +y nt +Ġmand ates +D IV +Ġreconc ile +view s +ĠDam on +vet te +F lo +ĠGreat est +il on +ic ia +Ġportray al +Ġcush ion +50 4 +19 79 +oss al +App lic +sc ription +Ġmit igation +AT S +p ac +Ġer ased +Ġdefic iencies +ĠHolland e +ĠX u +Ġb red +Ġpregn ancies +f emin +Ġem ph +Ġpl anners +Ġout per +utter ing +Ġperpet rator +Ġm otto +ĠEll ison +ĠNE VER +Ġadmitted ly +AR I +ĠAzerbai jan +Ġmill isec +Ġcombust ion +ĠBott le +ĠL und +ĠP s +ĠD ress +Ġfabric ated +Ġbat tered +Ġs idel +ĠNot ting +Fore ign +ĠJer ome +0 20 +ĠAr bit +Ġkn ots +ĠR IGHT +M oving +ãģ Ļ +Ġsur geries +Ġcour thouse +Ġm astered +Ġhover ing +ĠBr an +ĠAl ison +Ġsaf est +m ilitary +Ġbull ied +Ġbar rage +Read er +ES E +ĠGe ographic +T ools +3 14 +ĠGe ek +ro th +gl ers +ĠF IN +Ï ģ +ĠA ston +al tern +48 8 +Ġveter in +G amer +Ġint el +ren ches +Sh ield +Ġam nesty +ĠB har +Ġp iled +Ġhonor able +ĠInst itutes +Ġso aked +Ġcom a +ĠE FF +34 1 +by tes +ĠG mail +le in +ĠCanad iens +m aterial +I l +Ġinstruct ors +ĠK Y +Ġconce ive +ub b +ĠP ossible +Ġeas ing +ĠChrist ina +Ġcar ic +ĠHD R +R OM +Ġsho vel +de lete +Ġp uff +ĠCh anging +Ġseam lessly +Att ribute +Ġacqu isitions +ak ery +ĠE F +Ġaut istic +ĠT akes +ĠPow der +ĠSt ir +5 10 +ĠBub ble +sett ings +ĠF owler +Ġmust ard +Ġmore over +Ġcopyright ed +ĠLED s +15 00 +æ ī +ĠH IS +en f +Ġcust od +ĠH uck +G i +Ġim g +An swer +C t +j ay +ĠInf rastructure +Ġfeder ally +L oc +Ġmicro bes +Ġover run +dd s +ot ent +adi ator +>>>> >>>> +Ġtorn ado +Ġadj ud +Ġintrig ued +Ġs i +ĠRevel ation +pro gress +Ġburgl ary +ĠSai yan +ĠK athy +Ġser pent +ĠAndre as +Ġcomp el +ess ler +ĠPl astic +ĠAd vent +ĠPos itive +ĠQ t +ĠHind us +reg istered +ular ity +Ġrighteous ness +Ġdemon ic +u itive +ĠB DS +ĠGre gg +c ia +ĠCrus ade +ĠSina i +W ARE ++ ( +Ġme ll +Ġder ail +y ards +A st +Ġnotice ably +ĠO ber +R am +Ġun noticed +Ġse q +av age +T s +Ġ6 40 +Ġconced e +Ġ] ) +F ill +Ġcapt ivity +ĠImprove ment +ĠCrus ader +ara oh +M AP +æ Ĺ +Ġstr ide +al ways +F ly +N it +Ġal gae +ĠCook ing +ĠDo ors +Mal ley +Ġpolic emen +ãģ į +Ġastron aut +access ible +49 5 +ĠR AW +cl iffe +udic rous +Ġdep ended +al ach +Ġvent ures +ra ke +Ġt its +ĠH ou +Ġcond om +ormon al +Ġind ent +Ġupload ing +Foot note +Import ant +Ġ27 1 +Ġmind ful +Ġcont ends +C ra +Ġcal ibr +ĠO ECD +plug in +F at +ĠIS S +ĠDynam ics +ans en +68 6 +' ), +Ġsp rite +Ġhand held +ĠH ipp +=~ =~ +Tr ust +Ġsem antics +ĠBund es +ĠRen o +ĠLiter ature +s ense +G ary +ĠA eg +ĠTr in +EE K +Ġcler ic +ĠSS H +Ġch rist +Ġinv ading +ib u +Ġen um +aur a +Ġal lege +ĠInc redible +B BC +Ġth ru +Ġsa iled +Ġem ulate +Ġin security +Ġc rou +Ġaccommod ations +Ġincompet ent +Ġsl ips +ĠEarth qu +s ama +IL LE +Ġi Phones +as aki +Ġby e +Ġar d +Ġext ras +Ġsl aughtered +Ġcrowd funding +res so +Ġfil ib +ĠER ROR +ĠT LS +e gg +ĠIt al +Ġen list +ĠCatal onia +ĠSc ots +Ġser geant +Ġdiss olve +N H +Ġstand ings +ri que +I Q +Ġbenef iciary +Ġaqu arium +You Tube +ĠPower Shell +Ġbright est +ĠWar rant +S old +Writ ing +Ġbegin nings +ĠRes erved +ĠLatin os +head ing +Ġ4 40 +Ġrooft op +AT ING +Ġ3 90 +VP N +G s +k ernel +turn ed +Ġprefer able +Ġturn overs +ĠH els +S a +ĠShin ji +ve h +ĠMOD ULE +V iol +Ġex iting +Ġj ab +ĠVan illa +Ġac ron +ĠG ap +ber n +A k +ĠMc Gu +Ġend lessly +ĠFar age +ĠNo el +V a +M K +Ġbr ute +ĠK ru +ĠES V +ĠOl ivia +âĢ ł +ĠK af +Ġtrust ing +Ġh ots +3 24 +Ġmal aria +Ġj son +Ġp ounding +ort ment +Count ry +Ġpostp oned +Ġunequ iv +? ), +ĠRo oney +udd ing +ĠLe ap +ur rence +sh apeshifter +ĠH AS +os ate +Ġca vern +Ġconserv atism +ĠB AD +Ġmile age +Ġarrest ing +V aults +Ġmix er +Dem ocratic +ĠB enson +Ġauth ored +8 000 +Ġpro active +ĠSpirit ual +t re +Ġincarcer ated +ĠS ort +Ġpe aked +Ġwield ing +re ciation +×Ļ × +P atch +ĠEm my +Ġex qu +tt o +ĠRat io +ĠP icks +ĠG ry +ph ant +Ġf ret +Ġeth n +Ġarch ived +% - +c ases +ĠBl aze +Ġim b +c v +y ss +im ony +Ġcount down +Ġaw akening +ĠTunis ia +ĠRe fer +ĠM J +Ġun natural +ĠCar negie +iz en +ĠN uggets +he ss +Ġev ils +64 7 +Ġintrodu ctory +l oving +ĠMcM ahon +Ġambig uity +L abel +ĠAlm ighty +Ġcolor ing +ĠCl aus +set ting +N ULL +ĠF avorite +ĠS IG +> ( +ĠSh iva +ĠMay er +Ġstorm ed +ĠCo verage +we apons +igh am +Ġun answered +Ġle ve +Ġc oy +c as +b ags +as ured +Se attle +ĠSant orum +ser ious +Ġcourage ous +ĠS oup +Ġconfisc ated +Ġ// / +Ġuncon ventional +Ġmom s +ĠRohing ya +ĠOrche stra +ĠPot ion +Ġdisc redit +ĠF IL +f ixed +ĠDe er +do i +ĠDim ension +Ġbureaucr ats +et een +Ġaction Group +oh m +Ġb umps +ĠUt ility +Ġsubmar ines +ren heit +re search +ĠShap iro +Ġsket ches +Ġde ceptive +ĠV il +es ame +ĠEss entially +Ġramp age +isk y +Ġmut tered +th ritis +Ġ23 6 +f et +b ars +Ġpup il +ĠTh ou +o S +s ong +Ġfract ured +Ġre vert +pict ure +Ġcrit erion +us her +Ġreperc ussions +ĠV intage +ĠSuper intendent +Offic ers +Ġflag ged +Ġbl ames +Ġin verse +ograp hers +Ġmakes hift +Ġdev oid +Ġfoss ils +ĠArist otle +ĠFund s +Ġde pleted +ĠFl u +ĠY uan +Ġw oes +Ġlip id +Ġsit u +requ isites +Ġfurn ish +ĠSam ar +Ġshame ful +Ġadverse ly +Ġad ept +Ġrem orse +Ġmurder ous +uck les +ĠE SL +Ġ3 14 +s ent +Ġred ef +ĠC ache +ĠP urs +ig ans +Ġ4 60 +Ġpres criptions +Ġf res +F uck +ocr ates +Tw enty +ĠWe ird +ĠT oggle +ĠC alled +itiz ens +Ġp oultry +Ġharvest ing +ãĤ¦ ãĤ¹ +Bott om +Ġcaution ed +t n +39 6 +ĠNik ki +Ġeval uations +Ġharass ing +Ġbind ings +ĠMon etary +Ġhit ters +Ġadvers ary +un ts +Ġset back +Ġenc rypt +ĠC ait +Ġl ows +eng es +ĠN orn +Ġbul bs +Ġbott led +ĠVoy ager +3 17 +Ġsp heres +p olitics +Ġsubt ract +Ġsens ations +Ġapp alling +Ġ3 16 +Ġenvironment ally +ĠST EM +Ġpub lishes +5 60 +Ġdilig ence +48 4 +Ġadv ises +Ġpet rol +Ġimag ining +Ġpatrol s +ĠInt eger +ĠAs hes +act us +ĠRad iant +ĠL T +it ability +ht aking +Set ting +Ġnu anced +ĠRe ef +ĠDevelop ers +N i +pie ces +99 0 +Lic ense +Ġlow ers +ĠOtt oman +3 27 +oo o +Ġqu itting +mark ets +Beh ind +Ġbas in +Ġdoc s +an ie +fl ash +ct l +Ġcivil ized +ĠFuk ushima +"] ," +ĠK S +ĠHonest ly +ar at +Ġconstruct s +ĠL ans +ĠD ire +ĠLI KE +ĠTrou ble +Ġwith holding +ĠOb livion +Ġsan ity +any a +Con st +Ġgro cer +ĠC elsius +Ġrecount ed +ĠW ife +B order +ate red +h appy +Ġspo iler +Ġlog ically +H all +Ġsucceed ing +Ġpoly morph +Ġax es +ĠShot gun +ĠS lim +ĠPrin ciples +ĠL eth +art a +Ġsc or +Sc reenshot +Ġrelax ation +#$ #$ +Ġdeter rent +idd y +Ġpower less +Ġles bians +Ġch ords +ĠEd ited +se lected +Ġseparat ists +000 2 +Ġair space +Ġturn around +Ġc unning +P ATH +P oly +Ġbomb ed +Ġt ion +x s +Ġwith hold +Ġw aged +ĠLiber ties +Fl ag +Ġcomfort ing +45 4 +ĠI ris +are rs +Ġr ag +Ġrel ocated +ĠGu arant +Ġstrateg ically +Ġgam ma +uber ty +ĠLock heed +g res +Ġgr illed +ĠLow e +st ats +ĠR ocks +Ġsens ing +Ġrent ing +ĠGe ological +ا Ø +ot rop +Ġse w +Ġimproper ly +48 6 +Ġâĸ ł +Ġstar ving +ĠB j +Disc ussion +3 28 +ĠCom bo +ĠFix es +N AT +Ġstri ving +th ora +Ġharvest ed +ĠP ing +Ġplay ful +Ġaven ues +Ġoccup ational +Ġw akes +ĠCou rier +Ġdrum mer +ĠBrow ser +ĠH outh +it u +Ġapp arel +p aste +Ġhun ted +ĠSecond ly +l ain +X Y +ĠP IN +ic ons +Ġcock tails +Ġs izable +Ġhurd les +est inal +ĠRecre ation +Ġe co +64 8 +ĠD ied +m int +Ġfinger prints +Ġdis pose +ĠBos nia +ts y +22 00 +Ġins pected +ĠF ou +Ġf uss +Ġamb ush +ĠR ak +Ġmanif ested +Pro secut +Ġsuff ice +ren ces +Ġcompens ated +ĠC yrus +Ġgen us +ĠWolver ine +ĠTrend s +Ġh ikes +ĠSe en +Ġen rol +C old +Ġpol itely +ĠSl av +ĠRu pert +Ġey ewitness +ĠAl to +Ġun comp +Ġposter ior +M ust +ĠHer z +Ġprogress ively +Ġ23 4 +Ġind ifference +ĠCunning ham +Ġacadem ia +Ġse wer +Ġast ounding +ĠA ES +r ather +Ġeld est +Ġclim bs +ĠAdd s +Ġout cry +Ġcont ag +ĠH ouses +Ġpe pt +ĠMel ania +interest ed +ĠU CH +ĠR oots +ĠHub bard +ĠT BD +ĠRoman ian +fil ename +St one +ĠIm pl +Ġchromos ome +C le +d x +Ġscram bled +ĠP t +Ġ24 2 +OP LE +Ġtremend ously +St reet +Ġcra ving +Ġbund led +ĠR G +p ipe +Ġinj uring +Ġarc ane +Part icip +ĠHero ic +st y +Ġto pping +ĠTemp est +rent ices +b h +Ġpar anoia +ĠUnic ode +Ġegreg ious +Ġ\ ' +ĠOsw ald +Ġgra vel +ĠSim psons +Ġbl and +ĠGuant anamo +Writ er +lin ers +ĠD ice +J C +Ġpar ity +Ġs ided +Ġ23 7 +ĠPyr rha +at ters +d k +F ine +comp an +Ġform ulated +ĠId ol +il ers +hem oth +ĠF av +Ġintr usion +Ġcar rots +ĠL ayer +ĠH acker +Ġ ---------------- +Ġmoder ation +é ģ +oc oc +Ġcharacter ize +ĠTe resa +Ġsocio economic +Ġper k +ĠParticip ation +tr aining +ĠPaul o +ph ys +Ġtrust worthy +Ġembod ied +ĠMer ch +c urrency +ĠPrior ity +Ġte asing +Ġabsor bing +Ġunf inished +ĠCompar ison +Ġdis ple +writ ers +Ġprofess ions +ĠPengu in +Ġang rily +ĠL INK +68 8 +ĠCor respond +Ġprev ailed +Ġcart el +l p +as ms +ĠRed emption +ĠIslam ists +effect s +d ose +ĠL atter +ĠHal ifax +Ġv as +ĠTop ics +ĠN amed +advert ising +zz a +IC ES +Ġret arded +ach able +ĠPupp et +ĠItem Level +Ġret ract +Ġident ifiable +A aron +ĠB uster +s ol +hel le +as semb +H ope +r anged +B a +ĠP urch +é Ģ +ĠSir i +Ġarri vals +Ġ19 12 +Ġshort ened +Ġ3 12 +Ġdiscrep ancy +ĠTem perature +ĠWal ton +Ġkind erg +p olit +Ġrem ix +Ġconnect ors +ãĥĺ ãĥ© +ĠKazakh stan +dom inated +Ġsu gars +im ble +ĠPan ic +ĠDem and +ĠCol ony +on en +ĠM ER +7 75 +ur ia +aza ar +ĠDeg ree +P ri +Ġsun shine +Ġ25 1 +Ġpsychedel ic +Ġdigit ally +ĠBra un +Ġsh immer +Ġsh ave +ĠTel esc +ĠAst ral +ĠVenezuel an +ĠO G +Ġc rawling +Int eg +ĠFe ather +Ġunfold ing +Ġappropri ation +Ġè£ı è +ĠMob ility +ĠN ey +- . +b ilt +L IN +ĠT ube +ĠCon versely +Ġkey boards +ĠC ao +Ġover th +Ġla ure +>> \ +ĠV iper +ach a +Off set +ĠR aleigh +ĠJ ae +J ordan +j p +Ġtotal itarian +Connect or +Ġobserv es +ĠSpart an +ĠIm mediately +ĠSc al +C ool +Ġt aps +Ġro ar +P ast +Ġch ars +ĠB ender +ĠShe ldon +Ġpain ter +Ġbe acon +ĠCreat ures +Ġdownt urn +Ġh inder +ĠAnd romeda +à Ľ +cc oli +ĠF itness +et rical +Ġutil izes +Ġsen ate +Ġen semble +Ġche ers +T W +Ġaff luent +k il +ry lic +ord ering +Com puter +Ġgru esome +ost ics +ĠUb isoft +ĠKel ley +Ġw rench +Ġbourgeois ie +IB LE +ĠPrest on +w orn +ar ist +reat ing +Ġst ained +ar ine +Ġsl ime +EN N +Ġche sts +Ġground water +ann ot +ĠTr ay +ĠLoc ke +ĠC TR +Ġd udes +ĠEx ternal +ĠDec oder +Ġpar amed +ĠMed line +80 9 +ĠD inner +rup al +g z +ĠG um +ĠDem o +j ee +Ġd h +ber man +arch s +Ġen qu +ĠEp stein +Ġdevast ation +Ġfriends hips +ĠAr d +Ġ23 1 +ĠRub in +ĠDist ance +Ġsp urred +Ġd ossier +Ġover looking +\\\\\\\\ \\\\\\\\ +Fore st +ĠCom es +\ ", +ĠIran ians +Ġf ixtures +L aughs +Ġcur ry +ĠKing ston +Ġsqu ash +Ġcat alogue +Ġabnormal ities +Ġdigest ive +.... ..... +Ġsubord inate +og ly +Ġ24 9 +M iddle +Ġmass ac +Ġburg ers +Ġdown stairs +Ġ19 31 +39 4 +ĠV G +Ġl asers +ĠS ikh +ĠAlex a +der ived +Ġcycl ist +ãģ® éŃĶ +onel iness +!!!! !!!! +Ġbuff s +leg ate +Ġrap ing +Ġrecomm ending +ro red +Ġmult icultural +un ique +Ġbusiness men +Ġune asy +ĠM AP +Ġdisp ersed +cipl ine +J ess +ĠK erala +å § +Ġabst raction +Sur v +U h +Ġprin ters +ij a +ow der +Ġanalog ous +ĠA SP +af er +Ġunfold ed +Ġlevel ing +Ġbre ached +ĠH earing +Ġn at +Ġtransl ating +crit ical +Ġant agonist +ĠYes terday +Ġfuzz y +w ash +m ere +Ġbe wild +ĠM ae +V irgin +ph rase +Ġsign aled +ĠH IGH +Ġprot ester +Ġgar ner +unk nown +Ġk ay +Ġabduct ed +Ġst alking +am n +Ġdes erving +ĠR iv +ĠJ orge +Ġscratch ing +ĠS aving +ip ing +Ġte ase +Ġmission ary +ĠMor row +T IME +P resent +Ġchem otherapy +tern ess +ĠH omes +ĠP urdue +Ġst aunch +ĠWhit ney +ĠTH ERE +Î ¼ +iat us +ĠErn est +ĠDe ploy +Ġcove ted +F ML +ĠDial ogue +Ġex ited +f ruit +Ġner d +":" "," +Ġv ivo +ru ly +4 60 +ĠAm en +rehens ible +Ġâ ĺ +D IR +Ġad herence +Ġche w +ĠCo ke +ĠSerge i +dig ital +ĠNe ck +g ently +enth al +/ ) +Ġwe ary +Ġgu ise +ĠConc ord +ĠOn ion +at cher +Ġb inge +ĠDirect ive +Ġman ned +ans k +Ġill usions +Ġbillion aires +38 3 +oly n +odynam ic +ĠWhe at +ĠA lic +Ġcol oured +ĠN AFTA +ab o +Ġmac ros +ind ependent +s weet +Ġsp ac +ĠK abul +Ġ Ä +em e +Ġdict ated +Ġsh outs += { +Ġr ipping +ĠSh ay +ĠCr icket +direct ed +Ġanalys ed +ĠWAR RANT +ag ons +ĠBlaz ers +Ġche ered +Ġar ithmetic +ĠTan z +37 3 +ĠFl ags +Ġ29 5 +Ġw itches +ĠIn cluded +ĠG ained +ĠBl ades +G am +ĠSam antha +ĠAtl antis +ĠPr att +Ġspo iled +ĠI B +ĠRam irez +Pro bably +re ro +ĠN g +ĠWar lock +t p +Ġover he +Ġadministr ations +Ġt int +Ġreg iment +Ġpist ols +Ġblank ets +Ġep ist +Ġbowl s +Ġhydra ulic +Ġde an +Ġj ung +Ġasc end +70 5 +ĠSant iago +à ® +Ġun avoid +ĠSh aman +re b +Ġstem ming +99 8 +ĠM G +st icks +esthes ia +ER O +Ġmor bid +ĠGr ill +ĠP oe +any l +Ġdele ting +ĠSurve illance +Ġdirect ives +Ġiter ations +ĠR ox +ĠMil ky +F ather +Ġpat ented +44 7 +Ġprec ursor +Ġm aiden +ĠP hen +ĠVe gan +ĠPat ent +K elly +Redd itor +Ġn ods +Ġvent ilation +ĠSchwar z +Ġw izards +Ġomin ous +ĠHe ads +ĠB G +Ġl umber +ĠSp iel +Ġis Enabled +Ġancest ral +ĠSh ips +Ġwrest ler +ph i +Ġy uan +ĠRebell ion +Ġice berg +Ġmag ically +Ġdivers ion +ar ro +yth m +ĠR iders +ĠRob bie +ĠK ara +ĠMain tenance +ĠHer b +Ġhar ms +p acked +ĠFe instein +Ġmarry ing +Ġbl ending +ĠR ates +Ġ18 80 +Ġwr ink +ĠUn ch +ĠTor ch +desc ribed +Ġhuman oid +ilit ating +ĠCon v +ĠFe ld +IGH TS +Ġwhistlebl ower +ort mund +ets y +arre tt +ĠMon o +ĠI ke +ĠC NBC +ĠW AY +ĠMD MA +ĠIndividual s +Ġsupplement al +Ġpower house +ĠSt ru +F ocus +aph ael +ĠCol leg +att i +Z A +Ġp erenn +ĠSign ature +ĠRod ney +Ġcub es +idd led +ĠD ante +ĠIN V +iling ual +ĠC th +Ġso fa +Ġintimid ate +ĠR oe +ĠDi plom +ĠCount ries +ays on +Ġextrad ition +Ġdis abling +ĠCard iff +Ġmemor andum +ĠTr ace +Ġ?? ? +se ctor +ĠRou hani +ĠY ates +ĠFree ze +Ġbl adder +M otor +ĠProm ise +ant asy +Ġforesee able +ĠC ologne +cont ainer +ĠTre es +ĠG ors +ĠSin clair +Ġbar ring +key e +Ġsl ashed +ĠStat istical +é ĩ +Ġâĸ º +All ows +Ġhum ility +Ġdr illed +ĠF urn +44 3 +Ġse wage +Ġhome page +Ġcour tyard +Ġv ile +Ġsubsid iaries +aj o +direct ory +Ġam mon +V ers +charg es +Ġ} } +ĠCh ains +Ġ24 6 +n ob +Ġper cept +Ġg rit +Ġfisher men +ĠIraq is +ĠDIS TR +ĠF ULL +ĠEval uation +g raph +at ial +Ġcooper ating +Ġmel an +Ġenlight ened +Ġal i +t ailed +Ġsal ute +Ġweak est +ĠBull dogs +U A +ĠAll oy +Ġsem en +oc ene +ĠWilliam son +s pr +, âĢĶ +ĠG F +itt ens +Be at +ĠJ unk +iph ate +ĠFarm ers +ĠBit coins +ig ers +d h +ĠL oyal +p ayer +Ġentert ained +Ġpenn ed +Ġcoup on +Que ue +Ġweaken ing +c arry +Ġunderest imate +Ġshoot out +Ġcharism atic +ĠProced ure +Ġprud ent +in ances +Ġric hes +Ġcort ical +Ġstr ides +Ġd rib +ĠOil ers +5 40 +ĠPer form +ĠBang kok +Ġe uth +S ER +Ġsimpl istic +t ops +camp aign +Q uality +Ġimpover ished +ĠEisen hower +Ġaug ment +ĠH arden +Ġinterven ed +Ġlist ens +ĠK ok +Ġs age +Ġrub bish +ĠD ed +Ġm ull +pe lling +Ġvide ot +Produ ction +D J +m iah +Ġadapt ations +Ġmed ically +Ġboard ed +Ġarrog ance +Ġscra pped +Ġopp ress +FORM ATION +Ġj unction +4 15 +EE EE +S kill +Ġsub du +ĠSug gest +ĠP ett +Ġle tt +ĠMan ip +ĠC af +ĠCooper ation +T her +Ġreg ained +¶ æ +ref lect +Ġth ugs +ĠShel by +Ġdict ates +ĠWe iner +ĠH ale +Ġbatt leground +s child +Ġcond ol +h unt +osit ories +Ġacc uses +Fil ename +Ġsh ri +Ġmotiv ate +Ġreflect ions +N ull +ĠL obby +¥ µ +ĠS ATA +ĠBack up +Ñ ĥ +n in +ĠCor rection +Ġju icy +ut ra +ĠP ric +Ġrest raining +ĠAir bnb +ĠAr rest +Ġappropri ations +Ġsl opes +Ġmans laughter +Ġwork ings +ĠH uss +ĠF rey +Le ave +ĠHarm ony +ĠF eder +Ġ4 30 +Ġt rench +Ġglad ly +Ġbull pen +ĠG au +b ones +Ġgro ove +Ġpre text +ã ħĭ +Ġtransm itter +ĠComp onent +Ġunder age +ĠEm pires +T ile +Ġo y +ĠMar vin +ĠC AS +Ġbl oss +Ġrepl icated +ĠMar iners +Marc us +ĠBl ocks +Ġliber ated +Ġbutter fly +Fe el +Ġfer mentation +Ġyou tube +Ġoff end +ĠTer m +res ist +Ġcess ation +Ġinsurg ency +Ġb ir +ĠRa ise +59 5 +Ġhypothes es +50 2 +Ġpl aque +ocr at +Ġjack ets +ĠHuff Post +am ong +Ġconf er +48 7 +ĠL illy +Ġadapt ing +ĠF ay +Ġsh oved +ve c +Ġref ine +Ġg on +Ġgun men +z ai +ĠShut tle +ĠI zan +Ġ19 13 +Ġple thora +· · +Ġ5 10 +Ġp uberty +Ġ24 1 +ĠWe alth +ĠAl ma +ĠM EM +ĠAd ults +C as +pr ison +R ace +Ġwater proof +Ġathlet icism +Ġcapital ize +ĠJu ice +Ġillum inated +ĠP ascal +Ġirrit ation +ĠWitness es +ad le +ĠAst ro +Ġf ax +ĠEl vis +Prim ary +ĠL ich +ĠEl ves +Ġres iding +Ġst umble +3 19 +ĠP KK +Ġadvers aries +D OS +ĠR itual +Ġsm ear +Ġar son +ident al +Ġsc ant +Ġmon archy +Ġhal ftime +Ġresid ue +Ġind ign +ĠSh aun +ĠEl m +aur i +A ff +W ATCH +ĠLy on +hel ps +36 1 +Ġlobby ist +Ġdimin ishing +Ġout breaks +Ġgo ats +f avorite +ĠN ah +son ian +ĠBo oster +Ġsand box +ĠF are +ĠMalt a +Ġatt Rot +ĠM OR +ld e +Ġnavig ating +T ouch +Ġunt rue +ĠDis aster +Ġl udicrous +Pass word +ĠJ FK +blog spot +4 16 +ĠUN DER +ern al +Ġdelay ing +T OP +Ġimpl ants +ĠAV G +ĠH uge +att r +Ġjournal istic +ĠPe yton +ĠI A +R ap +go al +ĠProgram me +Ġsm ashing +w ives +print ln +ĠPl ague +in us +EE P +Ġcru iser +ĠPar ish +umin ium +Ġoccup ants +ĠJ ihad +m op +Ġp int +Ġhe ct +ĠMe cca +direct or +ĠFund ing +ĠM ixed +Ġst ag +T ier +Ġg ust +Ġbright ly +ors i +Ġup hill +R D +Ġles ions +ĠBund y +liv ious +Ġbi ologist +ĠFac ulty +ĠAuthor ization +Ġ24 4 +All ow +ï ¸ +ĠGi ul +Ġpert inent +ot aur +es se +ĠRo of +Ġunman ned +35 1 +ĠSh ak +ĠO rient +Ġend anger +D ir +Ġrepl en +ed ient +Ġtail or +Ġgad gets +Ġaud ible +âĺ Ĩ +N ice +Ġbomb ard +ĠR ape +Ġdef iance +ĠTW O +ĠFilip ino +Ġunaff ected +erv atives +Ġso ared +ĠBol ton +Ġcomprom ising +ĠBrew ers +R AL +ĠA HL +icy cle +Ġv ampires +Ġdi pped +oy er +ĠX III +Ġsidew ays +ĠW aste +ĠD iss +ĠâĶľ âĶĢâĶĢ +$ . +Ġhabit ats +ĠBe ef +tr uth +tr ained +spl it +R us +And y +ĠB ram +RE P +p id +è£ ħ +ĠMut ant +An im +ĠMar ina +Ġfut ile +hig hest +f requency +Ġepile psy +Ġcop ing +Ġconc ise +Ġtr acing +ĠS UN +pan el +ĠSoph ie +ĠCrow ley +ĠAd olf +ĠShoot er +Ġsh aky +ĠI G +ĠL ies +ĠBar ber +p kg +Ġupt ake +Ġpred atory +UL TS +/ ** +Ġintox icated +ĠWest brook +od der +he ment +Ġbas eman +AP D +st orage +ĠFif ty +ed itor +G EN +UT ION +ir ting +Ġse wing +r ift +Ġag ony +ĠS ands +Ġ25 4 +C ash +Ġl odge +Ġp unt +N atural +ĠIde as +Ġerrone ous +ĠSens or +ĠHann ity +Ġ19 21 +Ġm ould +ĠG on +kay a +Ġanonym ously +ĠK EY +Ġsim ulator +W inter +Ġstream ed +50 7 +? ", +Ġte ased +Ġco efficient +Ġwart ime +ĠTH R +' '. +ĠBank ing +mp ire +Ġf andom +Ġl ia +G a +Ġdown hill +Ġinterpre ting +Ind ividual +N orm +Ġjealous y +bit coin +Ġple asures +ĠToy s +ĠChev rolet +ĠAd visor +IZ E +Ġrecept ions +70 6 +C ro +Ġ26 2 +Ġcit rus +ir u +Review er +ject ed +U ES +an z +19 81 +ĠWork er +Ġcompl ied +ores cent +contin ental +T on +ĠPr ism +ĠShe ep +Ġ28 8 +n ox +ĠV og +O rd +Ġreal ms +te k +Ġirrig ation +Ġbicy cles +Ġelectron ically +p oly +t all +() ); +Ġaest hetics +ĠInteg rated +Expl ore +Ġd unk +47 6 +p ain +ĠJac ques +ĠD mit +Fram es +Ġreun ited +Ġhum id +D ro +P olitical +Ġyouth ful +Ġent ails +Ġmosqu ito +36 3 +spe cies +Ġcoord inating +ĠMay hem +ĠMagn us +M ount +Impro ved +ĠST ATE +ATT LE +Ġflow ed +Ġtack led +Ġfashion ed +Ġre organ +iv ari +f inger +Ġreluct antly +et ting +ĠV and +you ng +ĠGar land +Ġpresum ption +Ġamen ities +ĠPle asant +on ential +ĠO xy +Ġmor als +ĠY ah +Read y +Sim on +En h +D emon +Ġcl ich +Mon itor +ĠD U +Ġwel comes +Ġstand out +Ġdread ful +Ġban anas +Ġball oons +h ooting +bas ic +Ġsuff ix +Ġd uly +can o +Ch ain +at os +Ġgeop olitical +Ġ( & +ĠGem ini +ÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤ ÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤ +Ġacqu itted +L uck +prot ect +10 24 +Ġsc arcity +Ġmind fulness +ec ided +D N +pr ime +ĠPres idents +ĠVID EO +Ġ( âĪĴ +add ock +N OR +ĠP ru +p un +ĠL OL +)) )) +ĠL iqu +ĠS AS +Ġsty ling +Ġpunish ments +Ġnum b +Ġasc ertain +ĠRock ies +f lu +Th umbnail +Ġperpet rated +ĠSem i +Ġdis arm +ĠOld er +ĠEx ception +Ġexponent ially +ĠCommun ities +Ġabol ish +ĠPart ner +pt oms +Ġ7 77 +ĠFo ley +ĠC ases +Ġgre ase +ĠReb irth +G round +Ġ; ) +ĠDoct rine +ik ini +Y e +ĠBl ossom +Ġpers ists +b ill +Ġinf usion +Ġbud dies +9 11 +ĠPat ient +Ġdem os +Ġacquaint ance +ĠP aw +at ari +Ġx ml +Ġfasc ination +ĠSer ve +Ï Ĥ +br anded +Ġa z +Return s +Ġover shadow +Ġro am +Ġspeed y +n umbered +hel ial +Ġdisc iple +Ġass urances +g iven +pect ing +ĠN atalie +çĶ ° +Ġmosquit oes +rote in +Ġnumer ic +Ġindepend ents +Ġtrans itional +Ġreaction ary +ĠMech dragon +do ctor +Ġshort est +Ġsequ ential +ĠB ac +ĠAccount s +ãģ Į +ach y +ract ive +ĠReg iment +Ġbreat htaking +ffic iency +ĠB ates +Ġ3 11 +Ġward robe +ft s +ĠBer k +Sim ply +ĠRivers ide +iver ing +ident ial +lu cent +Ġen riched +ĠCon ver +ĠG iving +ãĥ Ļ +Ġlegal ize +ĠF TC +Ġfre aking +M ix +Ġter restrial +es ian +ci ents +W ing +LO AD +Ġled ge +ĠViol ent +ĠMet all +Ġ30 8 +Ġs outheastern +hett o +M eat +Ġslow down +Ġret reated +Jere my +end as +**** * +er ic +Ġre ins +opp able +ĠHuman ity +ear ances +rig an +C amera +Ġwa ivers +s oc +Ġalter ation +trans form +ĠC emetery +50 6 +Ġindef inite +Ġstim ulating +y g +60 3 +ĠS op +Ġdescript ive +Ph ase +ĠEd mund +Ġpneum onia +vent us +A mb +Ġlabor atories +ĠEx clusive +ug ar +W ere +Ġmalf unction +Ġhomosexual s +Ġ---- --- +un i +Ġturb ines +ĠEqu ity +D u +Ġmind ed +ĠR H +ĠBlack hawks +Ġfe ats +Ġ17 00 +re pl +36 2 +lad en +Ġindisp ensable +ly ss +tt i +Ġre el +Ġdiver ted +Ġlik eness +Ġsubscript ions +Ġfing ert +Ġfil thy +dest ruct +d raft +ĠBernard ino +l aunch +Ġper plex +ĠS UM +car b +Ġswe ater +ĠVent ure +ĠJ ag +ĠCele b +ĠV oters +Ġstead fast +Ġathlet ics +ĠHans on +ĠDr ac +Tr acker +Ġcomm end +ĠPres idency +ĠD ID +in formed +Ġweb page +P retty +Ġforce fully +ãĥĥ ãĤ¯ +Ġrel ocation +Ġsat ire +â ī +ĠSunder land +æ Ħ +V oice +???? ???? +Ġinform ant +Ġbow el +ĠUn iform +Ġ ..." +Ġpur ge +Ġpic nic +ĠU mb +ĠU PDATE +ĠSapp hire +ĠSt all +le arn +Ġobject ively +Ġob liter +Ġlooph ole +Ġjour neys +Ġo mission +Pro s +ĠSid ney +pl oma +Ġspray ed +Ġg uru +Ġtra itor +Ġtim et +Ġsn apping +ĠSe vent +urn al +ĠUk ip +Ġb owed +por al +l iberal +R os +Quest ions +i OS +Ġsummar ize +ST AT +Ġ18 50 +ap est +Ġl ender +ĠVari able +br inging +ĠL ORD +, ) +Ġcollaps es +x iety +ĠN ed +Y D +ĠSch a +Ġantib ody +Ġdis band +y re +ill usion +Ġro ver +s hed +ĠHiro sh +cc i +Ġcal am +ĠMort on +P interest +Ġ19 28 +ĠE uras +ord es +Ġf ences +ĠIn ventory +ĠVal encia +ĠU d +ĠT iff +Ġsqu e +Ġqu otation +Ġtroubles ome +er ker +QU EST +ĠKing doms +s outh +Ġle vy +Pr ince +ĠSt ing +Ġnick named +Ġapp e +Ġphot ographic +Ġcorp us +re ference +ĠT rog +U nt +) =( +ĠLat via +Ġactiv ating +Ġlicense e +Ġdispar ities +ĠNews letter +ãĥĥ ãĥĪ +Ġfree ing +ĠJe ep +ĠPer ception +ins k +Ġsil icone +ĠHay den +Le an +ĠSuz uki +ibr arian +66 8 +Ġsp or +Ġcorrel ations +ag hetti +Ġtu ber +ĠIP CC +il us +ĠV u +Ġwealth iest +ĠCarb uncle +an za +Ġfool ed +ĠZ ur +Ġd addy +ran o +il ian +Ġknock out +f man +requ ired +ĠWik ileaks +ĠD uffy +ON T +Ġins ol +ĠObject s +Ġb ou +ĠNord ic +ĠIns ert +sc an +Ġd ancers +Ġid iots +major ity +ĠNev ille +ĠFree BSD +Ġt art +pan ic +69 0 +Ġcoc oa +Ġsam pled +Ġlook up +Ind ust +Ġinject ions +gen re +Ġa u +Ġroad way +Ġgen itals +K ind +ĠEx aminer +ĠY az +F resh +Ġpar alysis +ĠAl uminum +Ġre ap +ok é +Ġsl oppy +ĠTun nel +pos ium +ner y +en ic +Ġher bal +ĠOut er +ĠBuild er +Ġinc ur +Ġide ologies +Ġback ups +cons uming +ĠDet ect +de ck +ĠKN OW +ĠG ret +ĠM IC +Ġtough ness +ĠEx hibit +Ġh ive +L es +ĠSCH OOL +ĠAt ari +ald e +ĠN ull +and estine +m ouse +Ġbrig ade +48 9 +Ġrev ol +ĠLaw son +ĠW ah +op oly +eb ted +ĠS aunders +Ġ3 13 +ĠW inc +Ġtab oo +ĠHel met +Ġw edge +ch ip +ĠT ina +b g +Ġinf uri +r n +Ġanomal ies +ĠSy nc +ĠEx am +ĠComm it +ĠDi ary +ĠALS O +ĠDe bor +omed ical +Ġcomprehens ion +6 55 +Ġempower ing +Ġ ire +Ġju ices +ĠE TH +ĠBox ing +=" / +Ġfacilit ated +p oke +ĠPars ons +ĠMod er +tra vel +Ġcivil izations +Ġliber tarians +Ġrun e +ĠCl arks +at hed +Ġcampaign ers +ĠDis patch +ĠFah renheit +ĠCap com +-------- -- +Ġl ace +Ġdr aining +Ġl iner +ĠArt ificial +é n +t ask +] ). +ĠGM O +ĠOper ator +ord inary +ĠInf luence +ĠU ps +Ġpot ency +uss en +osp ons +ĠSw im +ĠDead line +Un ity +Ġcul inary +Ġenlight enment +Ġwe arer +Ġmin ed +Ġp ly +Ġinc est +ĠDVD s +W alk +B TC +Tr ade +Ġdev al +ib and +ĠOvers ight +Palest inian +Ġd art +Ġm ul +L R +Ġrem ovable +ĠReal ms +ì Ŀ +Ġmisc ar +ĠV ulkan +68 5 +è re +ĠS ap +Ġmer ging +ĠCar ly +che ster +Ġbr isk +Ġlux urious +ĠGener ator +Ġbit terness +Ġed ible +Ġ24 3 +T G +Ġrect angle +With No +bel ow +J enn +Ġdark est +Ġh itch +Ġdos age +Ġsc aven +ĠK eller +ĠIllust rated +Certain ly +ĠMaver icks +Marg inal +Ġdiarr hea +Ġenorm ously +Ġ9 99 +sh r +qu art +Ġadam ant +ĠM ew +Ġren ovation +Ġcerv ical +ĠPercent age +en ers +ĠKim ber +Ġflo ats +Ġde x +ĠW itcher +ĠSwan sea +d m +Ġsal ty +y ellow +Ġca pe +ĠDr ain +ĠPaul a +ĠTol edo +les i +Mag azine +ĠW ick +ĠM n +ĠA ck +ĠR iding +AS ON +Ġhom ophobic +AR P +Ġwand ered +C PU +ood oo +ĠP ipe +Ġtight ening +ĠBut t +3 18 +Ġdesert ed +S ession +Ġfacilit ating +J ump +Ġemer gencies +OW ER +Ġexhaust ive +ĠAF TER +Ġheart beat +ĠLab el +ack y +ĠCert ified +ilt ration +Z e +ĠU tt +Ġ13 00 +Ġpres ume +ĠDis p +Ġsur ged +Ġdoll s +Col umb +Ġchim pan +ĠR azor +Ġt icks +Ġcouncill or +Ġpilgr image +ĠReb els +ĠQ C +ĠA uction +x ia +ik k +b red +Ġinsert ion +Ġco arse +d B +SE E +ĠZ ap +ĠF oo +Ġcontem por +ĠQuarter ly +ot ions +ĠAl chemist +ĠT rey +ĠDu o +S weet +80 4 +ĠGi ov +Ġfun n +N in +h off +Ġram ifications +Ġ19 22 +ĠExper ts +az es +Ġgar ments +ar ial +ĠN ab +Ġ25 7 +ĠV ed +Ġhum orous +ĠPom pe +Ġn ylon +Ġlur king +ĠSerge y +ĠMatt is +Ġmisogyn y +ĠComp onents +ĠWatch ing +ĠF olk +ract ical +B ush +Ġt aped +Ġgroup ing +Ġbe ads +Ġ20 48 +Ġcon du +quer que +Read ing +Ġgriev ances +Ult ra +Ġend point +H ig +ĠSt atic +ĠScar borough +L ua +ĠMess i +a qu +ĠPsy Net +ĠR udd +Ġa venue +v p +J er +Ġsh ady +ĠRes ist +ĠArt emis +Ġcare less +Ġbro kers +Ġtemper ament +Ġ5 20 +T ags +ĠTurn ing +Ġut tered +Ġp edd +Ġimpro vised +Ġ: ( +Ġtab l +Ġpl ains +16 00 +press ure +ĠEss ence +marg in +friend s +ĠRest oration +Ġpoll ut +ĠPok er +ĠAugust ine +ĠC IS +ĠSE AL +or ama +Ġth wart +se ek +Ġp agan + º +cp u +Ġg arn +Ġass ortment +ĠI LCS +t ower +Recomm ended +Ġun born +ĠRandom Redditor +ĠRandomRedditor WithNo +Ġparaly zed +Ġeru ption +Ġinter sect +ĠSt oke +ĠS co +B ind +å ¾ +ĠP NG +ĠNeg ative +ĠNO AA +Le on +Ġall oy +ĠL ama +ĠD iversity +5 75 +Ġunderest imated +ĠSc or +Ġm ural +Ġb usted +so on +l if +Ġnone x +Ġall ergy +ĠUnder world +ĠR ays +ĠBl asio +Ġh rs +ĠD ir +Ġ3 27 +by ter +Ġrepl acements +Ġactiv ates +ri ved +M H +Ġp ans +ĠH I +Ġlong itudinal +Ġnu isance +al er +Ġsw ell +ĠS igned +s ci +ĠIs les +ĠA GA +Ġdef iant +Ġson ic +oc on +K C +ĠA im +t ie +ah ah +Ġm L +D X +Ġb isc +ĠBill board +ĠSY STEM +NE Y +ga ard +Ġdist ressed +former ly +Al an +Ġche fs +Ġopt ics +ĠC omet +ĠAM C +Ġredes igned +irm ation +Ġsight ings +38 2 +3 11 +ĠW B +Ġcont raction +ĠT OTAL +D ual +Ġstart led +Ġunderstand ably +Ġsung lasses +ETH OD +Ġd ocker +Ġsurf ing +ĠH EL +ĠSl ack +ton es +Ġsh alt +Vis ual +49 8 +Dep artment +c ussion +Ġunrest ricted +Ġt ad +Ġre name +employ ed +Ġeduc ating +Ġgrin ned +bed room +ĠActiv ities +ĠV elvet +ĠSW AT +Ġsh uffle +ig or +Ġsatur ation +F inding +c ream +ic ter +Ġv odka +tr acking +te c +Ġfore ground +iest a +Ġve hement +ĠEC B +ĠT ie +E y +Ġt urtles +ĠRail road +ĠKat z +ĠFram es +Ġmen ace +ĠFell owship +ĠEss ential +ugg ish +Ġdri p +ch witz +ĠKy oto +s b +ĠN ina +Param eter +Ġal arms +ĠCl aud +Ġpione ering +Ġchief ly +ĠSc ream +Col lection +Ġthank fully +ĠRonald o +åŃ IJ +st rip +ĠDisney land +com mercial +See ing +S oul +Ġevac uate +Ġc iv +ĠAs he +Ġdiv ides +ĠD agger +rehens ive +Ġber ries +ĠD F +Ġs ushi +Ġplur ality +W I +Ġdisadvant aged +Ġbatt alion +ob iles +45 1 +Ġcl ing +Ġunden iable +ĠL ounge +Ġha unt +p he +Ġquant ify +Ġdiff ered +Ġ[* ] +ĠV iz +c um +sl ave +Ġvide og +Ġqu ar +Ġbund les +ĠAl onso +t ackle +Ġneur onal +Ġlandsl ide +conf irmed +ĠDep th +Ġrenew ables +B ear +ĠMaced onia +Ġjer seys +Ġb unk +ĠSp awn +ĠControl s +ĠBuch anan +Ġrobot ics +Ġemphas izing +ĠTut orial +h yp +ist on +Ġmonument al +æ ° +ĠCar ry +Ġt bsp +en ance +H ill +art hed +Ġro tten +De an +Ġtw isting +Ġgood will +Ġimm ersion +L iving +Ġbr ushes +ĠC GI +ĠAt k +tr aditional +Ġph antom +ĠSt amina +Ġexpans ions +ĠMar in +Ġembark ed +ĠE g +int estinal +ĠPE OPLE +ĠBo oth +ĠApp alach +Ġreleg ated +V T +M IT +Ġmust er +Ġwithdraw ing +Ġmicrosc ope +ĠG athering +ĠC rescent +ĠArgent ine +ĠDec re +ĠDomin ic +Ġbud s +ant age +ĠI on +Ġwid ened +ONS ORED +ĠGl oves +iann opoulos +raz en +fe el +Ġrepay ment +Ġhind sight +ĠRE ALLY +ĠPist ol +ĠBra h +Ġwat ts +Ġsurv ives +Ġfl urry +iss y +Al ert +ĠUrug uay +Ph oenix +S low +ĠG rave +ĠF ir +Ġmanage able +Ġtar iff +ĠU DP +ĠPist ons +ĠNiger ian +Ġstrike outs +Ġcos metics +whel ming +f ab +c ape +pro xy +Ġre think +Ġover coming +sim ple +Ġw oo +Ġdistract ing +ĠSt anton +ĠTuls a +ĠD ock +65 9 +Ġdisc ord +ĠEm acs +ĠV es +ĠR OB +Ġreass uring +Ġcons ortium +Muslim s +3 21 +Ġprompt s +se i +ĠH itch +imp osed +ĠF ool +Ġindisc rim +wr ong +bu querque +D avis +! ] +Ġtim eless +ĠNE ED +Ġpestic ide +Ġrally ing +ĠCal der +Ġå ¤ +Ġx p +ĠUn le +ĠEx port +lu aj +B uff +) [ +Ġsq or +S audi +Ġis tg +Ġindul ge +pro c +Ġdisg usted +Ġcomp ounded +Ġn em +Ġschool ing +ĠC ure +process ing +S ol +Ġpro verb +it ized +ĠAlv arez +Ġscar f +Ġrect angular +re ve +Ġh ormonal +ĠSt ress +itiz en +Ġ4 25 +girl s +ĠNo ir +ĠR app +Ġmar ches +ch urch +ĠUs es +Ġ40 5 +ĠBer m +Ġord inances +ĠJud gment +Charg es +ĠZ in +Ġdust y +Ġstraw berries +Ġper ce +ĠTh ur +ĠDebor ah +net flix +ĠLam bert +Ġam used +ĠGu ang +Y OU +R GB +ĠC CTV +Ġf iat +r ang +Ġf ederation +ĠM ant +ĠB ust +ĠM are +respect ive +ĠM igration +ĠB IT +59 0 +Ġpatriot ism +Ġout lining +reg ion +ĠJos é +Ġbl asting +ĠEz ra +B s +Ġundermin es +ĠSm ooth +Ġcl ashed +rad io +Ġtransition ing +ĠBucc aneers +ĠOw l +Ġplug s +Ġh iatus +ĠPin ball +Ġm ig +ĠNut r +ĠWolf e +Ġinteg ers +Ġor bits +ĠEd win +ĠDirect X +b ite +Ġbl azing +v r +Ed ge +ĠP ID +ex it +ĠCom ed +ĠPath finder +ĠGu id +ĠSign s +ĠZ er +ĠAg enda +Ġreimburse ment +M esh +i Phone +ĠMar cos +ĠS ites +h ate +en burg +Ġs ockets +p end +Bat man +v ir +ĠSH OW +Ġprovision al +con n +ĠDeath s +AT IVE +Pro file +sy m +J A +Ġnin ja +inst alled +id ates +eb ra +ĠOm aha +Ġse izing +ĠBe asts +Ġsal ts +M ission +Gener ally +ĠTr ilogy +he on +leg ates +Ġd ime +Ġf aire +par able +G raph +Ġtotal ing +Ġdiagram s +ĠYan uk +ple t +ĠMe h +Ġmyth ical +ĠStep hens +aut ical +ochem istry +Ġkil ograms +Ġel bows +anc ock +ĠB CE +ĠPr ague +Ġimpro v +ĠDev in +Ġ" \ +par alle +Ġsuprem acists +ĠB illion +Ġreg imen +inn acle +Ġrequ isite +ang an +ĠBur lington +ain ment +ĠObject ive +oms ky +G V +Ġun ilateral +Ġt c +Ġh ires +ment al +Ġinvol untary +Ġtrans pl +ĠASC II + ¨ +Ev ents +Ġdoub ted +ĠKa plan +ĠCour age +ig on +ĠMan aging +ĠT art +Ġfalse hood +ĠV iolet +Ġair s +Ġfertil izer +Brit ain +Ġaqu atic +ou f +W ords +ĠHart ford +Ġeven ings +ĠV engeance +qu ite +G all +ĠP ret +Ġp df +ĠL M +ĠSo chi +ĠInter cept +9 20 +Ġprofit ability +ĠId le +ĠMac Donald +ĠEst ablishment +um sy +Ġgather ings +ĠN aj +Charl ie +Ġas cent +ĠProt ector +Ġal gebra +Ġbi os +for ums +EL S +Introdu ced +Ġ3 35 +Ġastron omy +Cont ribut +ĠPol ic +Pl atform +Ġcontain ment +w rap +Ġcoron ary +ĠJ elly +man ager +Ġheart breaking +c air +ĠChe ro +c gi +Med ical +ĠAccount ability +! !" +oph ile +Ġpsych otic +ĠRest rict +Ġequ itable +iss ues +Ġ19 05 +ĠN ek +c ised +ĠTr acking +Ġo zone +Ġcook er +ros is +Ġre open +Ġinf inity +ĠPharm aceutical +ens ional +Att empt +ĠR ory +Mar co +Ġawa its +H OW +t reated +Ġbol st +Ġreve red +Ġp ods +opp ers +00 10 +Ġampl itude +ric an +SP ONSORED +Ġtrou sers +Ġhal ves +ĠK aine +ĠCut ler +ĠA UTH +Ġsplend id +Ġprevent ive +ĠDud ley +if acts +umin ati +ĠY in +Ġad mon +ĠV ag +Ġin verted +Ġhast ily +ĠH ague +L yn +Ġled ger +Ġastron omical +get ting +Ġcirc a +ĠC ic +ĠTenn is +Lim ited +Ġd ru +ĠBY U +Ġtrave llers +Ġp ane +ĠInt ro +Ġpatient ly +Ġa iding +Ġlo os +ĠT ough +Ġ29 3 +Ġconsum es +Source File +Ġ"" " +Ġbond ing +Ġtil ted +Ġmenstru al +ĠCel estial +UL AR +Plug in +Ġrisk ing +N az +ĠRiy adh +Ġacc redited +Ġsk irm +é Ľ +Ġexam iner +Ġmess ing +Ġnear ing +ĠC hern +ĠBeck ham +Ġsw apped +Ġgo ose +K ay +Ġlo fty +ĠWal let +Ġ[ ' +Ġap ocalypse +Ġb amboo +ĠSP ACE +ĠEl ena +Ġ30 6 +ac ons +Ġtight ened +Ġadolesc ence +Ġrain y +Ġvandal ism +ĠNew town +Ġcon ject +c akes +Ġche ated +Ġmoder ators +par ams +E FF +Ġdece it +ĠST L +ĠTanz ania +ĠR I +Ġ19 23 +ĠEx ile +the l +Ġthe olog +Ġquir ky +ĠIr vine +Ġneed y +or is +U m +K a +Ġmail box +3 22 +Ġb os +ĠPet ra +K ING +Ġenlarg ed +O ften +Ġbad ass +Ġ3 43 +ĠPl aces +ĠC AD +Ġpr istine +Ġinterven ing +d irection +Ġl az +ĠD SM +Ġproject ing +ĠF unk +ag og +pay ment +n ov +Ġch atter +AR B +Ġexam inations +ĠHouse hold +ĠG us +F ord +4 14 +B oss +Ġmy stic +Ġle aps +ĠB av +ul z +b udget +Foot ball +Ġsubsid ized +Ġfirst hand +Ġcoinc ide +oc ular +Con n +ĠColl abor +Ġfool s +am ura +ah ar +r ists +Ġsw ollen +Ġexp ended +ĠP au +s up +Ġsp ar +Ġkey note +s uff +Ġunequ al +Ġprogress ing +str ings +ĠGamer gate +Dis ney +ĠEle ven +om nia +Ġscript ed +Ġear ners +bro ther +ĠEn abled +æ ³ +Ġlar vae +ĠL OC +m ess +Wil son +ĠTem plate +success fully +Ġparam ount +Ġcamoufl age +Ġbind s +ĠQu iet +ĠSh utterstock +r ush +Ġmasc ot +fort une +ĠCol t +ĠBe yon +hab i +Ġha irc +Ġ26 7 +ĠDe us +Ġtw itch +Ġconcent rating +Ġn ipples +c ible +Ġg ir +N Z +M ath +n ih +Requ ired +Ġp onder +ĠS AN +Ġwedd ings +Ġl oneliness +N ES +ĠMah jong +69 5 +add le +ĠGar ner +ĠC OUR +Br idge +Ġsp ree +ĠCald well +Ġbri bery +Ġ���� ���� +plug ins +Ġr acket +Ġchamp agne +vers ible +V ote +Ġmod ifiers +May or +6 80 +Ġassemb lies +ĠS ultan +ĠN ing +ĠLad ies +Ġsulf ur +Ġor bs +Ġ---- - +____ ___ +ĠJournal ism +Ġes ports +Ġl ush +Ġh ue +Ġspect ral +H onest +ãĥ ı +Ġbus hes +Ġrein forcement +Ġre opened +ĠWhe els +ĠM org +rie ving +Ġaux iliary +Ġj Query +ĠB AT +tes que +Ġver tex +p ure +f rey +ãĤ º +d os +Ġty ph +Ġc ull +Ġe q +Ġdec on +Ġtoss ing +Ġdispar ate +ĠBr igham +print f +led ged +Ġsu nd +Ġco zy +Ġhepat itis +per forming +Ġav al +ĠG G +f uture +Ġpet ertodd +ĠKos ovo +Ġmagn ets +Al ready +ĠEd ison +ĠCe res +ĠRA ID +Ġbrill iance +57 6 +Ġder ives +Ġhypert ension +ĠÎ Ķ +Ġlamb da +Ġfl air +Ġmission aries +Ġrap es +ĠSt arter +ĠMon ths +Ġdef y +Ġseism ic +ĠR aphael +Ġeuro zone +65 6 +z sche +Ġscr atched +Ġb ows +ĠLenn on +ĠGa ia +Ġdri pping +f acts +A le +Ġfrog s +ĠBre ast +ogene ity +ĠProsecut or +Ġampl ified +ĠHod g +ĠF n +Th ousands +ĠNI H +ĠMonitor ing +FT WARE +ĠPri ebus +ĠG rowing +hun ter +Ġdiagn ose +ĠM ald +ĠL R +Ġcrown ed +Ġburst ing +Ġdiss olution +j avascript +Ġuseful ness +ĠExec ution +: ( +ĠIv ory +a ah +Ġpersecut ed +viol ence +ist as +ĠCr ate +Ġimpuls es +ĠSp ani +ed es +Hand le +ĠZ erg +think able +Last ly +Ġspont aneously +Ġinconven ient +Ġdismiss ing +Ġpl otted +Ġeight y +Ġ7 37 +r ish +ĠThor nton +ath am +Ġsit com +V en +Rec ipe +t el +l und +Ġcle ars +ĠSas uke +Ġ25 8 +Ġopt ing +Ġen raged +est hetic +ĠA e +uch s +Pre p +Fl ow +Ġrun off +ĠE ating +ĠG iles +ĠAct ing +res ources +ib aba +Ġr pm +Ġske wed +ĠBl anc +ĠS akuya +Ġhot ter +Ġ19 24 +op ian +ck o +Ġcr umbling +Ġcapt ains +ĠAppropri ations +le aders +dro pping +an uts +Ġrevers ing +ĠP ose +ĠS ek +Sc ot +ĠIde a +c ise +ĠSloven ia +Ġ3 17 +Do ctor +Ġcro cod +ald i +Se a +ĠFar rell +Ġmerc enaries +ĠR NC +ĠGu ess +Ġp acing +M achine +Streamer Bot +ĠChar ity +Ġ29 8 +Ġcann ons +ĠTob y +TPP StreamerBot +ĠPass ion +cf g +Th om +Ġbad ges +ĠBern stein +. âĢĵ +ĠP OP +ĠCon j +Ġinitial ization +Ġbiod iversity +D ub +Ġfeud al +Ġdisclaim er +Ġc row +Ġign ition +ar f +S HA +Ġk Hz +h azard +ĠArt ists +oe uv +67 9 +ĠRud y +N ine +ĠRam adan +å ½ +itt o +Ġadren aline +C ert +Ġsmell ed +Ġimp unity +Ġag endas +ĠRe born +ĠCon cent +ĠSe ems +Ġo mega +ĠDust in +Ġback er +ĠSau ce +ĠBoy le +W IN +Ġsp ins +Ġpa uses +u pt +Ġshred ded +Ġstra pped +ĠCor ruption +Ġscr atches +Ġn i +Ġatt ire +ĠS AF +Factory Reloaded +ĠI PS +Ġ( % +Ġsem inar +f ocus +c ivil +Ġ18 60 +int osh +Ġcontin ual +Ġabbre vi +ĠS ok +oc obo +X M +Ġfr antic +Ġunavoid able +Ġar tery +Ġannot ations +b ath +Cl imate +Ġd ors +ĠSl ide +co ord +ĠRel oad +ĠL DL +ĠLove craft +Ġunim agin +Ġresemb led +Ġbarr acks +n p +Ġsurrog ate +Ġcategor ized +ãĤ © +Ġvacc inated +Ġdrain age +Ġind ist +ĠWhats App +Ġ18 70 +oler ance +inv oke +am orph +Ġrecon nect +Ġem anc +Ġblind ness +Ġ12 80 +intern et +c ollar +Ġalt ru +Ġab yss +ĠT RI +65 7 +Ġinf used +HE AD +Ġforest ry +ĠWood y +ĠC i +w i +s am +78 4 +hol iday +Ġmog ul +ĠF ees +ĠD EN +In ternal +ur bed +f usc +at om +ĠIll usion +Ġpoll ed +Ġfl ap +Ġco ax +L GBT +An aly +ĠSect ions +ĠCalif orn +em n +Ġh ither +ĠN IGHT +Ġn ailed +ĠPip eline +39 1 +o of +ĠPr imal +vere nd +Ġsl ashing +Ġret ri +avi our +Ġdepart ing +g il +IS C +Ġmid way +Ġultras ound +Ġbeh aving +ĠT ara +class es +V irtual +ĠColon ial +Ġstri pping +Ġorchestr ated +ĠGra ves +45 2 +ĠIron ically +ĠWrit ers +Ġl ends +ĠMan z +Ġra ven +Ġoxid ative +Ġ26 6 +EL F +act ually +asc ar +D raft +Ġfavour able +Ġhumili ating +Ġf idelity +ĠH of +ĠX uan +49 6 +Ġlay ered +at is +79 0 +Ġpay check +it on +K ar +ĠVM ware +ĠFar mer +Ġserv ic +gl omer +Ġsl ump +ĠFab ric +ĠD OC +est ing +Ġreass ure +Ġph yl +v olt +it ory +R ules +Ġoxid ation +Ġpri zed +Ġmist ress +ĠDj ango +WAR N +å ij +Ġenc ode +ĠFeed back +Ġstupid ity +I an +ĠYugoslav ia +× ¨ +ac l +UT E +19 77 +Ġqual ifies +Ġpuls es +pret ty +Ġfro ze +Ġs s +Iter ator +Ġur gently +Ġm ailed +ĠCh am +Ġsust aining +Ġbas il +Ġpupp ies +il ant +ĠP LEASE +l ap +ace ous +F ear +ĠMaster y +aut omatic +ĠT AG +Ġant im +ag les +47 3 +fram es +Ġwh ispers +ĠWho ever +Ġbra very +ĠUK IP +ract ions +"" " +Ġt ame +Ġpart ed +every thing +CON T +Ġind ebted +Ġadd r +re k +IR ED +Ġem inent +cl inton +Ġo usted +Ġreview er +Ġmelt down +Ġre arr +ĠY ao +the real +aby te +Ġst umbling +Ġbat ches +Ġ25 9 +Ġcontrace ptive +Ġprost itute +ens is +De cl +ĠSt rikes +M ilitary +ĠO ath +v acc +pp ings +05 2 +Ġpart Name +amp ing +Rep orts +K I +CH R +Ġsubt ly +sw ers +Bl ake +us ual +Ġcontest ants +Ġcart ridges +ĠGRE AT +Ġbl ush +ĠâĢ º +47 2 +Ġreason ed +ãĥ ¤ +paralle led +Ġd yn +ag ate +Ġnight ly +å Ĩ +55 6 +Ġsem antic +ĠAdv oc +Ġ !! +Ġdisag rees +ĠB W +V eh +Ġharm ing +Ġembr aces +Ġstri ves +Ġin land +ĠK ard +Ġhe ats +ĠGin ny +ut an +ern aut +yl ene +ĠE lev +J D +Ġh ars +ĠStar r +Ġsk ysc +Ġcollabor ators +Us ually +Ġrev olutions +ĠSTAT S +Ġdism antle +Ġconfident ly +Ġkin etic +Al i +Ġpercent ile +Ġextract ing +ill ian +est ead +Ġphysic ists +ĠMarsh al +Ġfell owship +Ġd ashed +ĠU R +ĠSi oux +ĠComp act +am ide +P ython +ĠLe igh +ĠPharm ac +ist rates +her ical +Ġf ue +ĠE min +Ġ( { +ĠNeighbor hood +Ġdisrupt ing +ĠD up +Ġg land +ĠSe v +ĠMar ian +arg on +ĠD und +Ġ< !-- +Ġstr and +Ġstadium s +z os +Ġpsych osis +ĠR ack +Ġbrilliant ly +ï¸ ı +Ġsubmer ged +ĠInst it +ĠCh ow +Ġc ages +ĠH ats +ĠU rs +Ġdil uted +us at +ien ne +ĠMembers hip +ĠBur k +Ġ ie +Ġarche type +D rug +ult on +ĠSp ock +ĠMcK ay +ĠDep end +F eatured +S oc +19 78 +ĠB ere +Ġrelent lessly +Ġcripp ling +Ġar thritis +çĶ Ł +ĠTrop ical +ĠBul g +ĠCher yl +Ġadm irable +Ġsub title +Over ride +Ġorig inating +ĠC CP +Ġsw ore +ĠSo le +ĠDis orders +3 29 +Ġprocess ion +Ġref urb +Ġimm ersed +requ ently +Ġskept ics +Ġcer amic +m itter +en stein +b elt +ĠT IT +b idden +Ġf ir +m ist +> ] +Ġwe ave +ĠParad ox +Ġentr usted +ĠBarcl ays +Ġnovel ist +og ie +80 6 +Ġnin ety +Ġdisag reements +@@@@ @@@@ +ĠAus chwitz +c ars +ĠL ET +t ub +arant ine +P OS +Ġback story +Ġcheer ful +ĠR ag +ek a +bi ased +Ġinexper ienced +ak ra +ĠW itt +t an +Ġrap ist +Ġplate au +ch al +ĠInqu is +exp ression +Ġc ipher +Ġsh aving +add en +re ly +( \ +ism a +ĠReg ulatory +CH AR +ily n +N VIDIA +G U +Ġmur m +la us +Christ opher +Ġcontract ual +ĠPro xy +ĠJa ime +ĠMethod ist +Ġstew ards +st a +per ia +Ġphys iology +Ġbump ed +Ġf ructose +Austral ian +ĠMet allic +ĠMas querade +ar b +Ġprom ul +Ġdown fall +Ġbut cher +Ġb our +ĠIN FORMATION +ĠB is +pect s +ad ena +Ġcontempl ating +ar oo +cent ered +ĠPe aks +Us ed +Ġmod em +Ġg enders +Ġ8 000 +37 1 +Ġm aternity +ĠR az +Ġrock ing +Ġhandgun s +ĠD ACA +Aut om +ĠN ile +Ġtum ult +ĠBenef it +ĠAppro ach +works hop +ĠLe aving +G er +inst ead +Ġvibr ations +Ġrep ositories +49 7 +ĠA unt +ĠJ ub +ĠExp edition +Al pha +Ġs ans +Ġoverd ue +Ġoverc rowd +Ġlegisl atures +Ġp aternal +ĠLeon ardo +Ġexp ressive +Ġdistract ions +Ġsil enced +tr ust +Ġb iking +Ġ5 60 +Ġpropri et +Ġimp osition +Ġcon glomer +Ġ= ================================================================ +ĠTe aching +ĠY ose +int ensive +T own +Ġtroll ing +ĠGr ac +ĠAS US +Y o +Ġspecial s +ĠNep h +ĠGod zilla +Dat abase +ĠHe gel +Ġ27 2 +19 76 +ĠGl oria +Ġdis emb +ĠInvestig ations +ĠB ane +ag ements +St range +Ġtre asury +ĠPl ays +Ġundes irable +Ġwid ening +Ġverb ally +Ġinf ancy +Ġcut ter +f ml +Ġ21 00 +prot otype +f ine +Ġdec riminal +Ġdysfunction al +Ġbes ie +ĠErn st +z eb +Ġnort heastern +Ġa ust +por ate +ĠMar lins +Ġsegreg ated +ew orld +ĠMa her +Ġtra verse +Ġmon astery +ur gy +G ear +s and +Com pl +ĠE MP +Ġpl ent +ĠMer cer +Ġ27 6 +TA BLE +Config uration +H undreds +Ġpr ic +Ġcollabor ating +ĠPar amount +ĠCumm ings +Ġ( < +Ġrecord er +Ġfl ats +Ġ4 16 +wh ose +Font Size +ĠOr bit +Y R +Ġwr ists +Ġb akery +) } +ĠB ounty +ĠLanc aster +Ġend ings +acc ording +ĠSal am +e asy +75 5 +ĠBur r +ĠBarn ett +onom ous +Un ion +Ġpreced ence +ĠScholars hip +ĠU X +Ġroll out +Ġbo on +al m +ĠCan ter +æ µ +Ġround ing +Ġcl ad +Ġv ap +ĠF eatured +is ations +Ġ5 40 +pol ice +Ġunsett ling +Ġdr ifting +ĠLum ia +ĠObama Care +ĠF avor +Hy per +ĠRoth schild +ĠMil iband +an aly +ĠJul iet +H u +Ġrec alling +a head +69 6 +Ġunf avorable +Ġd ances +O x +Ġleg ality +Ġ40 3 +rom ancer +Ġinqu ire +ĠM oves +\ "> +ĠVari ant +ĠMess iah +ĠL CS +ĠBah á +75 6 +Ġeyeb row +Ġ ¥ +ĠMc F +ĠFort y +M as +Ġpan icked +Ġtransform ations +q q +Ġrev olves +ring e +ĠA i +ax e +Ġon ward +ĠC FR +ĠB are +log in +Ġliqu ids +Ġde comp +second ary +il an +ĠCon vert +ami ya +Ġprosecut ing +Ġâī ¡ +ĠYork ers +ĠByr ne +sl ow +aw ei +J ean +Ġ26 9 +ĠSky dragon +Ġ é +ĠNicarag ua +ĠHuck abee +ĠHigh ly +Ġamph ib +ĠPast or +ĠL ets +Ġbl urred +Ġvisc eral +ĠC BO +Ġcollabor ated +z ig +Leg al +Ġapart heid +Ġbr id +Ġpres et +ĠD ET +ĠAM A +× Ķ +arch ing +auc uses +build er +Ġpo etic +Ġem ulator +ĠMole cular +Ġhon oring +ise um +Ġtract or +ĠCl uster +ĠCal m +ared evil +Ġsidew alks +Ġviol in +Ġgeneral ized +ĠAle c +Ġemb argo +Ġfast ball +ĠHT TPS +ĠL ack +ĠCh ill +ri ver +C hel +ĠSw arm +ĠLev ine +ro ying +L aunch +Ġkick er +Ġadd itive +ĠDe als +W idget +cont aining +Ġescal ate +ĠOP EN +Ġtwe aked +Ġst ash +Ġsp arks +ĠEs sex +ĠE cc +Ġconv ict +Ġblog ging +I ER +ĠH L +Ġmurd erers +75 9 +ĠH ib +Ġde pl +ĠJ ord +S ac +Ġdis sect +ĠHow e +os her +Ġcustom izable +ĠFran z +Ġat ro +Ä ĩ +Ġ000 4 +Ġout post +R oss +Ġglyph osate +ĠHast ings +ĠBE FORE +Ġsh ove +o pped +ĠSc ala +Ġam ulet +an ian +Ġexacerb ated +Ġe ater +47 1 +UM E +Ġpul p +izont al +ĠZ am +ĠAT I +imm une +aby tes +Ġunnecess arily +ĠC AT +ĠAx is +Ġvisual ize +à ī +ĠRad ical +f m +Doc uments +ĠFor rest +Ġcontext ual +ĠSy mbol +Ġtent ative +ĠDO ES +ĠGood s +Ġintermitt ent +} : +medi ated +Ġridic ule +Ġathe ism +Ġpath ogens +ĠM um +Ġre introdu +Ġ30 7 +i HUD +Ġflash light +Ġsw earing +Ġp engu +B u +Ġrot ated +ĠCr ane +Ġ() ); +Ġfashion able +Ġendors ing +46 3 +) [ +Ġingest ion +Ġcook s +Ġ9 50 +ot omy +ĠIm am +Ġk a +Ġte aser +ĠGhost s +ĠãĤ µ +19 69 +Ï ĥ +ub by +Ġconver ter +zan ne +end e +ĠPre par +ĠNic kel +ĠChim era +h im +ĠTyr ann +ĠSabb ath +ĠNich ols +Ġra pt +ih ar +Ġshe lling +Ġillum inate +Ġdent ist +ut or +ĠInteg ration +Ġwh ims +ĠLiter ary +Be aut +Ġp archment +ag ara +Br and +Ġder og +âĢ¦ ) +ĠNor se +Ġunw itting +Ġc uc +Ġborder line +Ġupset ting +Ġrec ourse +Ġd raped +ĠRad ar +Ġcold er +ĠPep si +im inary +], [ +65 8 +V i +ĠF rem +ĠP es +Ġveter inary +ĠT ED +ĠEp idem +n ova +k id +Ġdev out +o ct +j ad +M oh +ĠP AY +Ġge ometric +Ġ3 23 +Ġcircum ference +ich ick +19 75 +ĠY uri +ĠSh all +ĠH over +un in +S pr +Ġg raft +ĠHapp iness +Ġdisadvant ages +att acks +Ġhub s +ĠStar Craft +é ĸ +Ġgall eries +ĠKor ra +Ġgrocer ies +ĠGors uch +Ġrap ists +Ġfun gi +ĠTyph oon +V ector +ĠEm press +b attle +4 68 +Ġparas ite +ĠBom ber +S G +ex ist +ĠP f +Ġun se +Ġsurge ons +B irth +ĠUn sure +ĠPrint ed +ĠBehavior al +ĠA ster +Pak istan +Ġun ethical +Ġs v +ĠIo T +Ġlay outs +P ain +Ġconst ants +ĠL W +ĠB ake +Ġtow els +Ġdeterior ation +ĠBol ivia +Ġblind ed +ĠW arden +ĠMist ress +Ġon stage +Ġcl ans +ĠB EST +19 60 +Ġant ique +Ġrhet orical +ĠPer cy +ĠRw anda +, . +B ruce +Ġtra umat +ĠParliament ary +Ġfoot note +id ia +ĠLear ned +se eking +gen ic +Ġdim ensional +H ide +èĢ ħ +Ġintrig ue +in se +Ġle ases +Ġapp rentices +w ashing +Ġ19 26 +V ILLE +Ġsw oop +s cl +Ġbed rooms +on ics +ĠCr unch +comp atible +Ġincap ac +ĠYemen i +ash tra +z hou +d anger +Ġmanifest ations +ĠDem ons +AA F +Secret ary +ACT ED +L OD +Ġam y +ra per +eth nic +4 17 +Ġpos itives +Ġ27 3 +ĠRefuge es +Ġus b +ĠV ald +odd y +ĠMahm oud +As ia +Ġskull s +ĠEx odus +ĠComp et +ĠL IC +ĠM ansion +ĠA me +Ġconsolid ate +storm s +ont ent +99 6 +Ġcl en +Ġm ummy +fl at +75 8 +ĠV OL +oter ic +n en +ĠMin ute +S ov +Ġfin er +R h +ly cer +Ġreinforce ments +ĠJohann es +ĠGall agher +Ġgym n +S uddenly +Ġext ortion +k r +i ator +T a +Ġhippocamp us +N PR +ĠComput ing +Ġsquare ly +Ġmod elling +ĠFor ums +ĠL isp +ĠKrish na +Ġ3 24 +Ġr ushes +Ġens ued +Ġcre eping +on te +n ai +il ater +ĠHorn ets +Ġob livious +IN ST +55 9 +Ġjeopard y +Ġdistingu ishing +j ured +Ġbeg s +sim ilar +ph ot +5 30 +ĠPark way +Ġs inks +ĠHearth stone +ib ur +ĠBat on +Av oid +Ġd ancer +Ġmag istrate +ary n +Ġdisturb ances +ĠRom ero +Ġpar aph +Ġmis chief +âĸ ĵ +ĠSh aria +Ġur inary +r oute +iv as +f itted +Ġeject ed +ĠAl buquerque +Ġ4 70 +Ġirrit ated +ĠZ ip +ĠB iol +à į +Ġden ounce +Ġbin aries +ĠVer se +Ġopp os +ĠKend rick +ĠG PL +Ġsp ew +ĠEl ijah +ĠE as +Ġdr ifted +so far +Ġannoy ance +ĠB ET +47 4 +ĠSt rongh +it ates +ĠCogn itive +oph one +ĠIdent ification +ocr ine +connect ion +Ġbox er +ĠAS D +ĠAre as +Y ang +t ch +ull ah +Ġdece ive +Comb at +ep isode +cre te +W itness +Ġcondol ences +ht ar +Ġhe als +Ġbuck ets +ĠLA W +B lu +Ġsl ab +ĠOR DER +oc l +att on +ĠSteven son +ĠG inger +ĠFriend ly +ĠVander bilt +sp irit +ig l +ĠReg arding +ĠPR OG +Ġse aling +start ing +Ġcard inal +ĠV ec +ĠBe ir +Ġmillisec onds +we ak +per se +Ġster ile +ĠCont emporary +ĠPh ant +ĠCl o +Ġout p +Ġex iled +Ġ27 7 +Ġself ie +Ġman ic +Ġn ano +ter ms +Alex ander +Ġres olves +Ġmillenn ia +Ġexpl odes +Ġconst ellation +Ġadul tery +m otion +D OC +Ġbroad casters +Ġkinderg arten +ĠMay weather +ĠE co +ich o +Ġ28 7 +l aun +Ġm ute +Ġdisc reet +Ġpres chool +Ġpre empt +De lete +ĠFre ed +P i +H K +Ġblock er +ĠC umber +Ġw rought +d ating +Ġins urer +Ġquot as +Ġpre ached +Ġev iction +ĠReg ina +ĠP ens +Ġsevent een +ĠN ass +D ick +Ġfold s +Ġd otted +ĠA ad +Un iversal +Ġp izz +ĠG uru +Ġso ils +Ġno vice +ĠNe ander +Ġst ool +Ġdeton ated +ĠPik achu +ĠMass ive +IV ER +ĠAb del +Ġsubdu ed +Ġtall est +Ġprec arious +Ġa y +r ification +ĠOb j +c ale +Ġun question +cul osis +ad as +igr ated +D ays +Ġque ens +ĠGaz ette +ĠCol our +ĠBow man +ĠJ J +ï ve +Ġdomin ates +Stud ent +Ġm u +Ġback log +ĠElect ro +Tr uth +48 3 +Ġcond ensed +r ules +ĠCons piracy +Ġacron ym +hand led +ĠMat te +j ri +ĠImp ossible +l ude +cre ation +Ġwar med +ĠSl ave +Ġmis led +Ġfer ment +ĠK ah +ink i +ke leton +cy l +ĠKar in +Hun ter +Reg ister +ĠSur rey +Ġst ares +ĠW idth +ĠN ay +ĠSk i +Ġblack list +uck et +Ġexp ulsion +im et +Ġret weet +vant age +Fe ature +Ġtro opers +Ġhom ers +9 69 +Ġconting ency +ĠW TC +ĠBrew er +fore ign +W are +S olar +Ġund ue +RE C +ulner able +path ic +ĠBo ise +Ġ3 22 +Ġarous ed +ĠY ing +ä¸ į +uel ess +Ġp as +Ġmor p +Ġfl oral +Ex press +ud ging +k B +ĠGr anted +Ø ¯ +ĠMich a +ĠGoth ic +ĠSPEC IAL +ĠRic ardo +F ran +Ġadminister ing +6 20 +por a +Ġ ® +Ġcomprom ises +Ġb itten +Ac cept +Th irty +Ð ² +Ġmater ially +ĠTer r +ig matic +ch ains +Ġdo ve +stad t +Mar vel +FA ULT +Ġwind shield +Ġ3 36 +ad ier +Ġsw apping +Ġflaw less +ĠPred ator +ĠMiche le +Ġprop ulsion +ĠPsych ic +Ġassign ing +Ġfabric ation +Ġbar ley +l ust +Ġtow ering +Ġalter cation +ĠBent ley +Sp here +Ġtun a +ĠClass es +Fre edom +un er +L ady +v oice +Ġcool est +or r +Ġpal p +$ { +Ġhyster ia +ĠMet atron +p ants +Ġspawn ing +Exper ts +ĠInvest ors +ĠAn archy +Ġshr unk +ĠVict im +Ġ28 9 +Ġec stasy +ĠB inding +58 5 +ĠMel ody +57 8 +ot ally +ĠE tsy +lig a +Ġapplaud ed +Ġswe ating +Ġredist ributed +Ġpop corn +Ġsem inal +f ur +ĠNeuro science +R and +ĠO st +ĠMadd en +ĠIncre asing +ĠDaw kins +ĠSub way +Ġar sen +cons erv +B UR +Ġsp iked +ĠLy ft +ĠImper ium +ĠDrop box +Ġfav oured +Ġencomp asses +gh ost +Ġins pires +Ġbur geoning +ĠY oshi +ĠVert ical +ĠAud itor +Ġint ending +Ġfilib uster +Bl oom +f ac +ĠCav s +ign ing +Ġcowork ers +ĠBarb arian +rem ember +FL AG +Ġaudit ory +ason ry +Col lege +Ġmut ed +gem ony +ob in +ĠPsych o +9 68 +Ġlav ish +Ġhierarch ical +ĠDr one +ou k +Ġcripp led +ĠMax im +Sl ot +Ġqu iz +ĠV id +if ling +Ġarchae ologists +Ġabandon ment +d ial +le on +ĠF as +T ed +Ġr aspberry +Ġmaneu vers +Ġbehavi ours +Ġins ure +Ġrem od +Sw itch +h oe +Ġsp aced +Ġafford ability +ĠF ern +not ation +ĠBal anced +Ġoccup ies +en vironment +Ġneck lace +Ġsed an +F U +ĠBrav o +Ġab users +ĠAn ita +met adata +ĠG ithub +ait o +ĠF aster +ĠWass erman +ĠF lesh +Ġth orn +r arily +ĠMer ry +w ine +Ġpopul ace +ĠL ann +Ġrepair ing +Ġpsy che +Ġmod ulation +aw aru +âĢĭ âĢĭ +ari j +Ġdecor ations +Ġapolog ise +ĠG arg +app ly +Ġgive away +ĠFl an +ĠWy att +U ber +Ġauthor ised +ĠMor al +HAHA HAHA +activ ate +Ġtorped o +ĠF AR +Ġam assed +ĠA ram +ark in +ĠVict ims +st ab +Ġo m +ĠE CO +Ġopio ids +Ġpurpose ly +ĠV est +Ġer g +at an +ĠSur gery +Ġcorrect ing +ĠOrt iz +ĠBe et +Ġrev oke +Ġfre eway +ĠH iggins +F ail +ĠFar ms +ĠAT P +h ound +Ġp oking +ĠCommun ists +mon ster +iment ary +Ġunlock ing +Ġunf it +we ed +en ario +at ical +ĠEnlight enment +ĠN G +ĠComp ensation +de en +ĠWid ow +ĠCind y +ĠAfter wards +Ġ6 000 +ikh ail +ag ically +Ġrat ified +Ġcasual ty +H OME +p sey +f ee +Ġspark ling +Ġd é +Ġconcert ed +C atal +Ġcomp lying +ĠA res +ĠD ent +Sh ut +Ġsk im +ad minist +Ġhost ilities +ĠG ins +Ġ6 08 +Ġm uddy +ĠMc Int +ĠDec ay +5 25 +Ġconspic uous +ĠEx posure +Ġresc ind +Ġwear able +Ġ3 28 +our met +ah s +ĠRob ots +Ġe clips +inst ance +ĠRE PORT +ĠApp l +0 30 +ĠSk ies +01 00 +Ġfall acy +S ocket +ĠRece iver +Ġsol ves +ĠButter fly +ĠSho pping +ĠFI RE +65 4 +Med ic +Ġsing ers +ĠNeed less +'' '' +isher s +ĠD ive +58 8 +Ġselect ively +Ġcl umsy +88 9 +Ġpurch aser +ear ned +ard y +Ġbenef iting +eng lish +Ġyield ing +ĠP our +Ġspin ach +Ġdel ve +ĠC rom +6 10 +Ġexport ing +ĠMA KE +Ġ26 3 +Ġg rop +Ġenv oy +ĠInqu iry +ĠLu igi +d ry +ĠT uring +Thumbnail Image +ĠVar iety +Ġfac et +Ġfl uffy +Ġexcerpt s +Ġsh orth +ĠOl sen +CL UD +Ġrel iant +ĠUN C +T our +Ġbat hing +Comp any +Ġglobal ization +P red +ĠMalf oy +Ġh oc +j am +craft ed +ĠBond s +ĠKiss inger +Eng land +Ġorder ly +cat entry +Ġ26 1 +Ġexch anging +ĠInt ent +ĠAmend ments +D OM +Ġst out +³³³³³³³³ ³³³³³³³³ +ĠAir bus +Ġ27 8 +hy de +P oll +Item ThumbnailImage +Ġlooph oles +ĠPill ar +Ġexpl or +St retch +A part +Ġun married +Lim it +ĠTransform ers +Ġintellect ually +unct ure +18 00 +Ġd arn +B razil +Ġleft over +ber us +f red +Mine craft +3 26 +ĠForm s +Ġproof s +ĠDes igned +Ġindex es +ĠSupp ose +EM S +ĠL oving +ĠBon nie +im ating +OT US +Ġconduct or +Ġbehav ed +ĠF ren +Ġsy nerg +Ġmillenn ium +Ġcater ing +ĠL auder +W r +ĠY iannopoulos +ĠAT F +Ġensl aved +Ġawaken ed +D VD +ĠED ITION +ĠConc ert +ĠChall enger +ĠH aku +umer ic +Ġdep recated +ĠSH AR +4 12 +Ġdy stop +Ġtremb ling +Ġdread ed +ĠSp ac +p adding +Re pl +ĠG arrison +M ini +Ġun paralleled +am ar +URR ENT +w reck +c ertain +t al +ĠC LS +app ings +Ġsens ed +Ġf encing +ĠPas o +ĠDes k +Ġsc off +Ġcontem plate +ĠL iga +l iquid +75 7 +Ġapp rentice +ĠUCH IJ +5 70 +ĠTh ousand +ĠIll um +Ġchampion ed +ãĤ Į +Ġelect ors +Ġ3 98 +ĠH ancock +round ed +ĠJ OHN +Ġuns atisf +Ġqual ifier +ĠGad get +EN E +Ġdead liest +ĠPl ants +Ġ ions +Ġacc ents +Ġtwe aking +Ġsh aved +F REE +ĠCh aser +Again st +9 60 +Ġmeth amphetamine +Ġnormal ized +Ġ$ \ +ĠPre cision +ĠGu am +Ġch oked +ĠX II +ĠCast ing +Tor rent +Ġscal p +ĠJagu ar +w it +Ġsem ic +ix ie +ĠG ould +Ġconf ines +N usra +ĠL on +ĠJ ugg +y cle +ĠCod ec +E gypt +Ġrest rain +ĠAl iens +Ġch oking +ĠD unk +ĠBell a +ab c +Ġsl ang +Ġneuro trans +s av +Ġempower ment +â ĨĴ +Ġclim bers +ĠM im +ĠF ra +ros se +Cap ital +ĠCth ulhu +Inter face +Ġprof icient +ĠIN TO +Ġ3 18 +ront al +5 80 +ĠDes pair +K enn +Ġscrim mage +ĠCo at +as ions +Ġwall paper +ĠJ ol +Ġresurg ence +Ġant iv +ĠB alls +² ¾ +Ġbuff ers +Ġsub system +ĠSt ellar +ĠL ung +A IDS +Ġerad icate +Ġblat antly +Ġbehav es +ĠN un +Ġant ics +ex port +DE V +w b +Ġph p +ĠInteg rity +Ġexplore r +Ġrev olving +auth ored +g ans +Ġbas k +Ġas ynchronous +å į +TH ING +69 8 +G ene +ĠR acer +ĠN ico +iss ued +Ġser mon +p ossibly +Ġsize of +Ġentrepreneur ial +ox in +ĠMin erva +Ġpl atoon +n os +ri ks +A UT +ĠAval anche +ĠDes c +ij 士 +ĠP oc +Ġconf erred +Î » +Ġpat ched +F BI +66 2 +Ġfract ures +Ġdetect s +Ġded icate +Ġconstitu ent +Ġcos mos +W T +Ġswe ats +Ġspr ung +b ara +s olid +Ġuns us +Ġbul ky +ĠPhilipp e +ĠFen rir +Ġtherap ists +ore al +^^ ^^ +Ġtotal ed +Ġboo ze +ĠR PC +Prosecut ors +Ġdis eng +ĠSh ared +Ġmotor cycles +Ġinvent ions +Ġlett uce +ĠMer ge +ĠJ C +Ġspiritual ity +ĠWAR NING +Ġunl ucky +ĠT ess +Ġtong ues +ĠD UI +T umblr +Ġle ans +Ġinv aders +Ġcan opy +ĠHur ricanes +ĠB ret +ĠAP PLIC +id ine +ick le +Reg arding +Ġve ggies +Ġe jac +ju ven +F ish +D EM +ĠD ino +Th row +ĠCheck ing +be ard +( & +Ġj ails +Ġh r +trans fer +iv ating +Ġfle ets +ĠIm ag +ĠMc Donnell +Ġsnipp et +Is a +ĠCh att +ĠSt ain +ĠSet FontSize +ĠO y +ĠMathemat ics +49 4 +Ġelectro ly +ĠG ott +ĠBr as +B OOK +ĠF inger +d ump +Ġmut ants +Ġrent als +Ġinter tw +Ġc reek +ail a +Bro ther +ĠDisc ord +pe e +raw ler +Ġcar p +Ġ27 9 +ãĤ· ãĥ£ +rel ations +Ġcontr asts +Col umn +Ġrec onnaissance +Ġun know +Ġl ooting +Ġregul ates +Ġopt imum +ĠChero kee +ĠA ry +Lat est +Ġroad side +Ġd anced +ĠUnic orn +A cknowled +Ġuncont roll +ĠM US +at io +ch ance +ha ven +VAL UE +Ġfavour ites +Ġceremon ial +b inary +pe ed +wood s +EM P +Ġv ascular +Ġcontempl ated +Ġbar ren +ĠL IST +Y ellow +ospons ors +Ġwhisk y +ĠM amm +ĠDeV os +min imum +H ung +44 2 +P ic +ĠSnap dragon +77 6 +Ġcar ving +Ġund ecided +Ġadvantage ous +Ġpal ms +ĠA Q +Ġst arch +L oop +Ġpadd le +Ġfl aming +ĠHor izons +An imation +bo ost +Ġprob abilities +ĠM ish +Ġex odus +ĠEditor ial +Ġfung us +Ġdissent ing +ĠDel icious +rog ram +ĠD yn +d isk +t om +Ġfab rics +ĠC ove +ĠB ans +Ġsoft en +ĠCON S +Ġin eligible +Ġestim ating +ĠLex ington +pract ice +of i +Ġshe dding +ĠN ope +Ġbreat hed +ĠCorinth ians +y ne +ek i +B ull +Ġatt aching +reens hots +Ġanaly se +ĠK appa +Ġuns ustainable +Ġinter pol +ank y +he mer +Ġprot agonists +Ġform atted +ĠBry ce +ĠAch illes +ĠAb edin +sh ock +Ġb um +b os +qu a +ĠW arn +q t +ĠDi abetes +8 64 +ĠIn visible +Ġvan ish +Ġtrans mitting +Ġmur ky +ĠFe i +Ġawa ited +ĠJur assic +umm ies +Ġmen acing +g all +C ath +B uilt +ild o +ĠV otes +Ġon t +Ġmun itions +ĠFre em +ÃŃ n +Ġdec ency +lo pp +ie ved +ĠG ord +Ġun thinkable +ĠNews week +Ġ3 21 +He at +Ġpresent er +ji ang +Ġpl ank +ĠAval on +Ġben z +ĠR out +Ġslam ming +ĠD ai +ou ter +ĠCook ie +ĠAlic ia +ge y +Ġvan ity +Ġow l +á µ +t ested +ĠAw akens +Ġcan v +Ġblind ly +ĠRid ley +ĠEm ails +Requ ires +ĠSer bian +ograp hed +if rame +eter ia +Ġaltern ating +qu iet +Ġsoc iology +ĠUn lock +ĠCommun ism +Ġo ps +Ġatt ribution +Ġab duction +ĠAb ram +Ġsidel ined +ĠB OOK +Ġref ining +ĠFe eling +ĠOs lo +ĠPru itt +r ack +ang ible +Ġcaut iously +ĠM ARK +eed s +M ouse +ĠStep h +ĠP air +S ab +99 7 +ĠBa al +B ec +Ġcomm a +ĠP all +ĠG ael +Ġmisunder stand +ĠP esh +Order able +Ġdis mal +ĠSh iny +% " +Ġreal istically +Ġpat io +ĠG w +ĠVirt ue +Ġexhaust ing +wh atever +oph ys +y ip +4 18 +Ad just +ĠWa iting +ess on +ĠMaz da +ĠDo zens +Ġstream lined +Ġincompet ence +ĠM eth +Ġeth os +ON ES +Ġincent iv +Ġgr itty +ĠBut cher +Head er +Ġexp onential +à Ł +Ġcorrel ate +Ġcons ensual +s ounding +R ing +Orig in +Ġcon clusive +fe et +ac ly +ĠF ernandez +Buy able +Ġd ucks +aunt lets +Ġel ong +Ġ28 6 +Ġsim ul +G as +ĠK irst +Ġprot r +ĠRob o +ĠAo E +op ol +Ġpsych ologically +sp in +ilater ally +ĠCon rad +W ave +44 1 +ĠAd vertisement +ĠHarm on +ĠOri ental +is Special +Ġpresum ptive +Ġw il +ĠK ier +ne a +Ġp pm +Ġhar bour +ĠW ired +comp any +Ġcor oner +atur days +ĠP roud +ĠN EXT +ĠFl ake +val ued +ce iver +Ġfra ught +Ġc asing +Ġrun away +Ġg in +ĠLaure nt +ĠHar lem +ĠCur iosity +qu ished +Ġneuro science +ĠH ulu +Ġborrow er +Ġpetition er +ĠCo oldown +W ARD +Ġinv oking +conf idence +For ward +Ġst s +pop ulation +Delivery Date +Fil m +ĠC ov +quick Ship +quickShip Available +prim ary +isSpecial Orderable +inventory Quantity +channel Availability +BO X +ĠMulti player +ĠJen ner +77 8 +ĠM d +Ġ~ /. +M N +Ġchild ish +Ġantioxid ant +ĠChrom ebook +Ġ27 4 +Ġscreen play +Ġadvent urous +ĠRelations hip +respons ive +ming ton +Ġcorner stone +ĠF ey +F IR +Ġrook ies +ĠF eaturing +Ġorig inate +Ġelectro des +ant es +Ġscript ures +Ġgl ued +Ġdiscont ent +Ġaff licted +lay out +B rave +Ġm osa +ĠQuant ity +ĠH ik +w inner +H ours +Ġent ail +ĠCell s +olog ue +Ġv il +Ġpre acher +Ġdecor ative +d ifferent +Ġprejud ices +ĠSm oking +ĠNotting ham +so Type +Ġrhyth ms +ĠAl ph +bl ast +Ste el +ĠDaniel le +Ġstr ife +Ġrem atch +so DeliveryDate +ĠF ork +t rip +ol ulu +hes es +C G +ĠPOLIT ICO +ost a +ĠDr ift +é¾įå ¥ +é¾įå¥ ij士 +Ġvet ting +ĠJin ping +ĠRec ession +Min or +ĠF raud +enf ranch +Ġconven ed +ĠNA ACP +ĠMill ions +ĠFarm ing +ĠW oo +ĠFl are +rit o +imm igrant +Ġvac ancy +ĠHE AD +ĠV aj +eg al +ĠV igil +Stud y +Ġru ining +Ġr acks +Ġhe ater +ĠRand olph +ĠBr ush +ĠT ir +Ø ¨ +Ġc ov +% ] +Ġrecount s +ĠO PT +ĠM elt +Ġtr uce +Ġcas inos +Ġcrus ade +Ġcarn age +Ġstri pe +ĠK yl +Text ures +Ġ6 98 +Ġpro clamation +Ġgood ies +Ġ........ .. +pro claimed +P olit +Ġtop ical +Ġspecial ize +ĠA min +g m +Ġanch ored +Ġbear ings +s ample +ĠHigh land +ĠAut ism +Ġmerc enary +Ġinterview er +L ER +ĠSom ers +Ġembry o +ĠAss y +Ġ28 1 +ĠEd iting +ĠCh osen +6 60 +Ġp ci +ĠThunder bolt +BI LL +Ġchuck led +jri wal +h of +Ġearth ly +() { +ind ependence +Ġdisp ers +ĠV endor +ĠG areth +Ġp als +P enn +ĠSub mit +ic um +Th u +Ġcl andestine +Ġcann ibal +ĠCl erk +E Stream +gal itarian +âĻ ¥ +g ew +Ġhor rend +ĠL ov +ĠRe action +ocr in +Class ic +Ġecho ing +Ġdiscl osing +ĠIns ight +og un +ĠInc arn +upload s +pp erc +guy en +Ġ19 01 +ĠB ars +68 7 +Ġb ribes +ĠFres no +ur at +ĠRe ese +Ġintr usive +Ġgri pping +ĠBlue print +ĠR asm +un ia +man aged +ĠHeb do +Ġ3 45 +Ġdec oding +Ġpo ets +Ġj aws +ĠF IGHT +am eless +ĠMead ows +ĠHar baugh +Inter view +ĠH osp +ĠB RA +Ġdelet ion +m ob +W alker +ĠMoon light +ĠJ ed +ĠSoph ia +Ġus ur +Ġfortun ately +ĠPut ting +ĠF old +Ġsan itation +Ġpart isans +IS ON +B ow +ĠCON C +ĠRed uced +ĠS utton +Ġtouch screen +Ġembry os +âĢ¢âĢ¢ âĢ¢âĢ¢ +ĠK rug +com bat +ĠPet roleum +Ġam d +ĠCos mos +Ġpresc ribing +Ġconform ity +ours es +Ġplent iful +Ġdis illusion +ĠEc ology +itt al +Ġf anc +Ġassass inated +regn ancy +Ġperenn ial +ĠBul lets +Ġst ale +Ġc ached +ĠJud ith +ĠDise ases +All en +Ġl as +Ġsh ards +ĠSu arez +ĠFriend ship +inter face +ĠSupp orters +add ons +46 2 +ĠIm ran +ĠW im +Ġnew found +ĠM b +An imal +Ġd arling +and e +Ġrh y +ĠTw isted +pos al +yn ski +Var ious +× ľ +ĠK iw +uy omi +Ġwell being +ĠL au +an os +Ġunm ist +Ġmac OS +Ġrest room +ĠOl iv +ĠAir ways +Ġtimet able +9 80 +Ġrad ios +v oy +ias co +Ġcloud y +ĠDraw ing +Any thing +Sy ria +ĠH ert +st aking +Ġun checked +Ġb razen +ĠN RS +69 7 +onom ic +est ablish +Ġl eng +Ġdi agonal +ĠF ior +L air +ĠSt ard +Ġdef icient +jo ining +be am +Ġomn ip +Ġbl ender +Ġsun rise +Mo ore +ĠF ault +ĠCost ume +ĠM ub +Fl ags +an se +Ġpay out +ĠGovern ors +ĠD illon +ĠBan ana +N ar +Ġtra iled +Ġimperial ist +um ann +ats uki +4 35 +ĠRoad s +Ġsl ur +ĠIde ally +Ġt renches +C trl +Ġmir rored +ĠZ el +ĠC rest +Comp at +ĠRoll s +sc rib +ĠTra ils +omet ers +w inter +Ġimm ortality +il ated +Ġcontrad icts +un iversal +ill ions +ĠM ama +opt im +AT URE +Ġge o +et ter +ĠCar lo +4 24 +Ġcanon ical +ĠStrongh old +n ear +Ġperf ume +Ġorche stra +od iac +Ġup he +Ġreign ing +vers ive +Ġc aucuses +ĠD EM +Ġinsult ed +Ġ---- -- +ĠCr ush +Ġroot ing +ĠWra ith +Ġwh ore +Ġto fu +C md +ĠB ree +Ġ$ _ +Ġr ive +ĠAd vertising +Ġw att +ĠH O +Ġpersu asive +ĠParam eters +Ġobserv ational +ĠN CT +ĠMo j +ĠSal on +Ġtr unc +Ġexqu isite +ĠMar a +Ġpo op +ĠAN N +Ex c +ĠWonder ful +ĠT aco +Ġhome owner +ĠSmith sonian +orpor ated +mm mm +Ġlo af +ĠYam ato +ĠInd o +Ġcl inging +á s +Ġimm utable +h ub +Or ange +Ġfingert ips +ĠWood en +ĠK idd +ĠJ PM +ĠDam n +C ow +c odes +48 2 +Ġiniti ating +ĠEl k +ĠCut ting +Ġabsent ee +ĠV ance +ĠLil ith +G UI +Ġobsc ured +Ġdwar ves +ĠCh op +ĠB oko +Val ues +Ġmult imedia +Ġbrew ed +Reg ular +CRIP TION +ĠMort al +Ġa pex +Ġtravel er +Ġbo ils +Ġspray ing +Rep resent +ĠStars hip +4 28 +Ġdisappro val +Ġshadow y +Ġlament ed +ĠRe place +ĠFran ç +67 7 +d or +Ġunst oppable +Ġcoh orts +gy n +ĠClass ics +ĠAm ph +Ġsl uggish +ĠAdd iction +ĠPad res +Ġins cription +Ġin human +min us +ĠJere miah +at ars +Ter ror +ĠT os +ĠSh arma +ast a +c atch +Ġpl umbing +ĠTim bers +Sh ar +H al +ĠO sc +Ġcou pling +hum ans +Ġsp onge +Ġid ols +ĠSp a +ĠAdv ocate +ĠBe ats +lu a +Ġtick ing +Ġload er +ĠG ron +8 10 +Ġstim ulated +Ġside bar +ĠManufact urer +ore And +19 73 +Ġpra ises +ĠFl ores +dis able +ĠElect rical +ra ise +E th +Ġmigr ated +Ġlect urer +K ids +ĠCa vern +Ġk ettle +Ġgly c +ĠMand ela +ĠF ully +å§ « +FIN EST +Ġsquee zing +ĠRy der +amp oo +oreAnd Online +Inst oreAndOnline +Buyable InstoreAndOnline +Ġcommem orate +ĠRamp age +Aust in +ĠSh roud +ĠRu ins +9 15 +ĠK H +Ġwater front +ĠE SC +b aby +ĠC out +ĠEm blem +Ġequival ents +49 2 +Un ique +ĠNiet zsche +brow ser +Ġim itation +ĠWere wolf +ĠKir in +ac as +' ," +Ġà ¾ +Review ed +Ġc unt +Ġvo ic +ĠLen ovo +Ġbond ed +48 1 +Ġinhib itors +Ġendeav ors +ĠHav ana +ĠSt out +ĠJ olly +A ctor +*/ ( +Ġoccur rences +ĠT ens +Incre ased +ĠACT ION +Ġ ãĢĮ +ĠRank ings +ĠB reat +Ġ30 9 +D ou +Ġimpact ing +ĠDuc hess +pre fix +Q B +Ġsummon ing +Ġbest owed +ĠKe pler +ĠPOW ER +c ube +ĠK its +ĠG rip +Ġop ium +Ġrep utable +t oc +ich ael +ĠR ipple +Ġcaf é +ĠZ oom +ĠBur ma +Ġwa ive +Ġst alls +Ġdem eanor +inc erity +Ġfluor ide +ĠSH OULD +Par is +Ġlong ing +Ġpl at +Ġgross ly +Ġbull s +Ġshowc asing +ex pected +ĠG addafi +engine ering +Re peat +ĠK ut +Ġconce ivable +Ġtrim med +osc ope +ĠCand idate +ĠT ears +rol og +Lew is +S UP +Ġroad map +Ġsal iva +Ġtrump et +Jim my +Ġmirac ulous +Ġcolon ization +Ġam put +ĠGN OME +ate ch +D ifferent +ĠE LE +ĠGovern ments +ĠA head +ãħĭ ãħĭ +word press +L IB +ĠIn clude +ĠDor othy +0 45 +ĠColomb ian +Ġle ased +88 4 +Ġde grading +ĠDa isy +i ations +Ġbapt ized +Ġsurn ame +co x +Ġblink ed +ãĥ ¢ +Ġpoll en +Ġder mat +Ġre gex +ĠNich olson +ĠE ater +ç ľ +rad or +Ġnarrow er +Ġhur ricanes +Ġhalluc inations +r idden +ISS ION +ĠFire fly +Ġattain ment +Ġnom inate +Ġav ocado +ĠM eredith +Ġt s +Ġreve rence +Ġe uph +Ġcr ates +ĠT EXT +Ġ4 43 +Ġ3 19 +J SON +iqu ette +Ġshort stop +ic key +Ġpro pelled +Ġap i +ĠTh ieves +77 9 +Ġovers aw +Ġcol i +ĠNic ola +Ġover cl +ik awa +ĠC yr +Ġ38 4 +78 9 +ĠAll ows +10 27 +Det roit +TR Y +set up +ĠSocial ism +Sov iet +s usp +ĠAP R +ĠShut down +Ġal uminium +zb ek +ĠL over +GGGG GGGG +Ġdemocr acies +Ġ19 08 +ĠMer rill +ĠFranco is +gd ala +Ġtraff ickers +ĠT il +ĠGo at +Ġsp ed +ĠRes erv +Ġpro d +55 2 +Ġc ac +ĠUn iv +ĠSch we +Ġsw irling +ĠWild erness +ĠEgg s +Ġsadd ened +Ġarch aic +H yd +Ġexcess ively +B RE +Ġaer ospace +ĠVo ices +Cra ig +Ġign ited +In itially +ĠMc A +Ġhand set +Ġreform ing +Ġfrust rations +ĠDead pool +ĠBel ichick +ract or +ĠRagnar ok +ĠD rupal +ĠApp roximately +19 20 +ĠHub ble +arm or +ĠSar as +ĠJon as +Ġnostalg ic +Ġfeas ibility +Sah aran +Ġorb iting +Ġ9 70 +R u +Ġsh in +ĠInvestig ators +Ġinconsist encies +ĠP AN +B G +Ġgraz ing +Ġdetect ors +ĠStart up +ĠFun ny +ĠNa omi +Consider ing +Ġh og +ut f +ce mic +Ġfort ified +ĠFun ctions +Ġcod ec +nut rition +H at +" ! +micro soft +55 8 +ĠTh in +ĠA CE +Al ias +ĠO PS +p apers +P K +ãĢ İ +Ġimpro bable +N orthern +equ al +Ġlook out +Ġty res +ĠMod ified +ĠK op +Abs olutely +Ġbuild up +sil ver +Ġaud i +Ġgro tesque +ĠSab er +ĠPres byter +ON Y +Ġglac iers +ĠSho als +ĠK ass +ĠH RC +ĠNic ol +ĠL unch +ĠF oss +âĸ Ĵ +AD RA +ĠOne Plus +o ing +ground s +Ġincident al +Ġdatas ets +68 9 +ĠClarks on +Ġassemb ling +ĠCorrect ions +Ġdrink ers +Ġqual ifiers +Ġle ash +Ġunf ounded +ĠH undred +Ġkick off +T i +Ġrecon cil +ĠGr ants +ĠCompl iance +ĠDexter ity +Ġ19 06 +w arn +D allas +Max imum +n ard +av ia +be aut +ens itivity +tr ace +Ġpione ers +ĠF ract +ãĢ ı +Ġpre cept +Ġgloss y +ĠI EEE +Ac ross +Ġ6 80 +S leep +che on +Ġsatir ical +ĠMin otaur +ĠCla ude +Ġr é +ape go +Ġcar rot +ĠSem in +ino a +Ġz o +Ind ependent +Ġdiagn oses +ĠC ue +M AR +Ġrend ition +ĠK ik +Ġpath ology +Ġselect s +Link edIn +Ġass ay +ĠD res +Ġtext ual +post ed +IT AL +ĠM aul +N eal +Ġinter connected +Ġerr atic +ĠVir us +Ġ5 30 +Ġenvironmental ists +ĠP helps +Ġeng agements +ĠIN ST +Ġeconom ical +nox ious +Ġg earing +izz y +Ġfavor ably +ĠMcG ill +T erm +Ġh anged +Ġball park +ĠRe yes +Ġbe ware +ĠP sal +ĠMass acre +q i +Ġin accessible +acly sm +Ġfr ay +ill ac +Ġbitter ly +ĠCert ification +Mich igan +Ġir respective +al ore +Em pty +Ġendorse ments +Ġund et +f g +equ ipped +Ġmerc iless +ĠC ust +Ġimm ature +Ġvou cher +ĠBlack well +Ñ ı +h awk +dis ciplinary +ile e +ĠMak oto +ĠD ude +ãĥĩ ãĤ£ +Y ears +Ġin ver +Ġsh aman +ĠY ong +ip el +ell en +ĠCath y +br ids +Ġs arc +65 1 +N ear +Ġground work +Ġam az +Ġ4 15 +ĠHunting ton +hew s +ĠB ung +Ġarbit rarily +ĠW it +ĠAl berto +Ġdis qualified +best os +46 1 +Ġp c +Ġ28 4 +ro bat +Rob in +Ġh ugs +ĠTrans ition +ĠOcc asionally +Ġ3 26 +ĠWh ilst +ĠLe y +Ġspaces hip +cs v +Ġun successfully +ĠA u +le ck +ĠWing ed +ĠGrizz lies +. � +Ġne arer +ĠSorce ress +ĠInd igo +El se +8 40 +let es +Co ach +Ġup bringing +ĠK es +Ġseparat ist +Ġrac ists +Ġch ained +Ġabst inence +lear ning +Ġrein stated +Ġsymm etry +Ġremind ers +ĠChe vy +Ġm ont +Ġexempl ary +ĠT OR +Z X +Ġqual itative +ĠSt amp +ĠSav annah +ĠRoss i +Ġp aed +Ġdispens aries +ĠWall s +ĠCh ronic +Ġcompliment ary +ĠBeir ut +Ġ+ --- +igs list +Ġcrypt ographic +mas ters +ĠCap itals +Ġmax imal +Ġent ropy +Point s +Ġcombat ants +l ip +ĠGl ob +ĠB MC +ph ase +th ank +HT TP +Ġcomm uter +Ġ\( \ +.. / +ĠReg ener +ĠDO I +ĠActiv ision +Ġsl it +os al +RE M +Ġch ants +Y u +Ke ys +Bre xit +ĠFor ced +Ari zona +Ġsquad ron +IS O +ĠMal one +Ġ3 38 +Ġcontrast ing +Ġt idal +Ġlib el +Ġimpl anted +Ġupro ar +ĠC ater +Ġpropos itions +M anchester +ĠEuro s +it amin +G il +ĠEl ven +ĠSe ek +ĠB ai +Ġredevelop ment +ĠTown s +ĠL ub +! ", +al on +K rist +Ġmeas urable +Ġimagin able +Ġapost les +Y N +7 60 +Ġster oid +Ġspecific ity +ĠL ocated +ĠBeck er +ĠE du +ĠDiet ary +uts ch +ĠMar ilyn +Ġbl ister +ĠM EP +ĠK oz +ĠC MS +y ahoo +ĠCar ney +Ġbo asting +ĠC aleb +By te +read s +ad en +Pro blem +ĠWood ward +S we +S up +ĠK GB +Set up +Ġtac it +Ġret ribution +Ġd ues +ĠM ü +. ? +ä¸ Ń +p ots +Ġcame o +ĠP AL +educ ation +A my +like ly +g ling +Ġconstitution ally +ĠHam m +ĠSpe ak +Ġwid gets +br ate +Ġcra ppy +ĠI ter +Ġanticip ating +ĠB out +P ixel +ĠY ep +ĠLaur ie +Ġh ut +Ġbullet in +ĠSal vation +Ġch ats +ear able +Honest ly +AL TH +onse qu +c ult +isco very +ovy ch +Ġse lves +ĠSat oshi +S ounds +Ġconver gence +ĠRosen berg +19 74 +Ġnas al +Ġfull est +Ġfer ocious +x us +ist e +AM S +Ġlobb ied +Ġso othing +ĠGun n +t oday +0 24 +Ġinspir ational +ĠN BN +p b +g ewater +or ah +all owed +ĠCol iseum +Ġspecial izing +Ġinsane ly +ĠT ape +del ay +Ġt arn +ĠP ound +Ġmel anch +Ġdeploy ments +il and +Ġless en +Ġfur ry +ĠUE FA +Ġblood shed +ĠMe ier +ither ing +Ġhe irs +ĠJ aw +ax ter +ĠPublic ations +Ġal ters +int ention +ĠWinc hester +d etermination +ĠLif etime +th in +Mon ster +7 80 +Ġapprox imation +Ġsuper markets +ĠSecond s +or os +h uge +Ġb ribe +ĠLIM ITED +un ed +Ġmis interpret +ĠIn jury +Ġ3 67 +Ġthreshold s +ĠCarn ival +Ġgastro intestinal +Ġguid eline +Ġde ceived +f eatures +Ġpurported ly +ĠRon nie +ĠNew t +Ġsp acious +as us +Ġsuperhero es +ĠCyn thia +le gged +k amp +ch io +Ġth umbnail +ĠShir ley +ill ation +Ġshe ds +ĠZ y +E PA +Ġdam s +Ġy awn +n ah +ĠPe ggy +ĠE rie +ĠJu ventus +ĠF ountain +r x +don ald +al bum +ĠComp rehensive +Ġc aching +ĠU z +ulner ability +ĠPrinc iple +ĠJ ian +ing ers +cast s +ĠOs iris +ch art +t ile +ĠTiff any +ĠPatt on +ĠWh ip +Ġovers ized +J e +ĠCind erella +ĠB orders +ĠDa esh +M ah +Ġdog ma +Ġcommun ists +v u +Coun cil +Ġfresh water +Ġw ounding +Ġdeb acle +Ġyoung ster +Ġthread ed +ĠB ots +ĠSav ings +ãģ Ĥ +ol ing +oh o +Ġillum ination +M RI +Ġlo osen +tr ump +ag ency +ur ion +Ġmoment arily +ĠCh un +ĠBud apest +ĠAl ley +D isk +Ġaston ished +ĠCon quer +ĠAccount ing +h aving +ĠWe in +ĠAl right +Ġrev olver +Ġdel usion +Ġrelic s +Ġad herent +qu ant +Ġhand made +or io +Ġcomb ating +c oded +Ġquad ru +re th +N ik +ĠTrib al +ĠMyster ious +Ġin hal +ĠWin ning +ĠClass ification +ch anged +Ġun ab +Ġsc orn +icip ated +w l +ond uctor +Ġrein forcing +ĠChild hood +an ova +Ġadventure r +Ġdoctor al +ĠStrateg ies +Ġengulf ed +ĠEnc ounter +Ġl ashes +Crit ical +ric ular +ĠU TF +oci ation +check ing +ĠConsult ing +Run time +per iod +ĠAs gard +Ġdist illed +ĠPas adena +ĠD ying +ĠCOUN TY +Ġgran ite +Ġsm ack +Ġparach ute +ĠS UR +Virgin ia +ĠF urious +78 7 +ĠO kin +Ġcam el +ĠM bps +19 72 +ĠCh ao +ĠC yan +j oice +ef er +ĠW rap +ĠDeb ate +S eg +Ġfore arm +ĠIgn ore +Ġtim estamp +Ġprob ing +ĠNo on +ĠGra il +f en +Ġdorm ant +ĠFirst ly +ĠE ighth +ĠH UN +ĠDes ire +or as +Girl s +ĠDes mond +z ar +am ines +O AD +exec ute +Ġbo obs +ĠAT L +_ ( +Chel sea +Ġmasturb ation +ĠCo C +Ġdestroy er +ĠCh omsky +Ġsc atter +ĠAss ets +79 6 +ĠC argo +Ġrecept ive +ĠSc ope +Ġmarket ers +Ġlaun chers +Ġax le +ĠSE A +se q +ĠM off +f inding +ĠGib bs +Georg ia +extreme ly +N J +Ġlab orers +st als +Ġmed iation +ĠH edge +at own +Ġi od +des pite +v ill +J ane +ex istence +Ġcoinc ided +ĠUt ilities +ĠChe ap +Ġlog istical +Ġcul mination +ĠNic otine +p ak +F older +Ġrod ents +st uff +Ġlaw fully +Ġreper to +io ch +j j +Dial ogue +HH HH +lic tion +Look s +Ġ29 7 +Ġtur rets +ĠAb andon +Ġinc ess +ĠTraff ord +Ġcur led +Ġprefer ring +Ġprivat ization +Ġir resist +ĠP anda +ĠSh ake +ĠMc Gr +ãĥ Ħ +und ers +Ġdiscrim inated +Ġbart ender +I LE +Atl antic +Ġprop ensity +ĠW iz +ĠG im +con ference +Ġrein forces +G h +w agon +Ġe erie +F al +Ġhug ged +rac ist +R IC +F u +Ġf iller +ĠSt ub +Ġeng raved +ĠWrest le +Ġimagin ative +ĠPe er +ĠFact ors +an us +ĠDrac ula +mon itor +Ġrou ters +ib ia +ĠBoo lean +end ale +ĠSl aughter +ĠSh ack +R FC +ĠSpiel berg +S ax +ĠPH OTO +ĠCl over +ĠR ae +Dep ending +ĠMem or +ar am +Ġpier ced +Ġcur tains +v ale +ĠInqu isition +ĠP oke +Ġforecast ing +Ġcompl ains +S ense +ĠHer mes +isc overed +Ġb ible +ĠMor ph +Ġg erm +78 5 +D ON +Ġcon gen +Ġcr ane +ĠD PR +Ġrespect fully +R oom +ĠN aw +ĠDal ai +re ason +ĠAng us +Educ ation +ĠTitan ic +Ë ľ +Ġo val +un ited +Ġthird s +Ġmoist ur +ĠC PC +M iami +Ġtent acles +ĠPol aris +ex c +ex clusive +ĠPra irie +Ġcol ossal +ĠBl end +sur prisingly +ÃŃ s +Ġindo ctr +Ġbas al +ĠMP EG +und o +Spl it +Develop ment +Ġlan tern +19 71 +Ġprov ocation +Ġang uish +ĠB ind +ĠLe ia +duc ers +ipp y +conserv ancy +Ġinitial ize +ĠTw ice +ĠSu k +Ġpred ic +Ġdi ploma +Ġsoc iop +Ing redients +Ġhamm ered +ĠIr ma +Q aida +Ġglim ps +ĠB ian +Ġst acking +Ġf end +gov track +Ġun n +dem ocratic +ig ree +Ġ5 80 +Ġ29 4 +Ġstraw berry +ID ER +Ġcher ished +ĠH ots +Ġinfer red +Ġ8 08 +ĠS ocrates +O regon +ĠR oses +ĠFO IA +Ġins ensitive +Ġ40 8 +Recomm end +ĠSh ine +Ġpain staking +UG E +ĠHell er +ĠEnter prises +I OR +ad j +N RS +L G +Ġalien ated +Ġacknowled gement +ĠA UD +ĠRen eg +Ġvou chers +Ġ9 60 +Ġm oot +ĠDim ensions +Ġc abbage +B right +g at +ĠK lu +Ġlat ent +Ġz e +ĠM eng +Ġdis perse +Ġpand emonium +H Q +Ġvirt uous +ĠLoc ations +ee per +prov ided +Ġse ams +ĠW T +iz o +PR OV +Ġtit anium +Ġrecol lection +Ġcr an +Ġ7 80 +ĠN F +49 1 +64 2 +p acking +59 8 +text ure +Sp ider +fre edom +cipl ed +ĠTAM ADRA +âĻ ¦ +aut hent +ĠW ANT +r ified +Ġr ites +Ġuter us +k iss +Ġâī ¤ +Ġsk illet +Ġdis enfranch +ĠGa al +Comp an +Ġage ing +gu ide +B alt +Ġiter ator +Ġdiscretion ary +t ips +Ġprim ates +ĠTechn ique +ĠPay ments +az el +ĠR OCK +stant ial +0 60 +Ġd mg +ĠJack ets +ĠPlay off +Ġnurs ery +ĠSy mb +art on +Ġannex ation +Color ado +Ġco ils +ĠSh oes +âĦ¢ : +ĠRo z +COM PLE +ĠEve rest +ĠTri umph +J oy +G rid +à ¼ +process or +ĠPros per +ĠSever us +ĠSelect ed +r g +ĠTay yip +St ra +Ġski ing +Ġ? ) +Ġpe g +Tes la +Ġtime frame +Ġmaster mind +ĠN B +scient ific +ĠSh it +gener ic +IN TER +N UM +Ġst roll +ĠEn ix +ĠM MR +ĠE MS +m ovie +Ĥ ª +Ġminim izing +idd ling +Ġilleg itimate +Ġprot otyp +Ġpremature ly +Ġmanual s +obb ies +ĠCass idy +D EC +des ktop +Ġaer os +Ġscreen ings +Ġdeb ilitating +ĠGr ind +nature conservancy +Ġf ades +ter mination +assets adobe +F actor +Ġdefinitive ly +P oké +ap ult +ĠLaf ayette +C orn +ĠCor al +Ġstagn ant +T ue +Ġdissatisf action +G ender +Ġkid neys +ĠG ow +ĠDef eat +ĠAsh ton +Ġcart els +Ġfore closure +ĠExpl ore +stre ngth +ot in +Ġveterin arian +Ġf umble +Ġpar ap +ĠSt rait +r ils +Ġpr ick +ĠBerm uda +ĠAm munition +skin ned +Ġab ound +ĠB raz +Ġshar per +ĠAsc ension +Ġ9 78 +Ġpreview s +Ġcommun ion +ĠX Y +Ġph ony +Ġnewcom er +Ġ3 32 +." ," +Ġredist ribution +Prot ect +ĠSo f +K al +Ġlip stick +w orst +Ġtang led +Ġretrospect ive +int eger +Ġvolunte ering +Ġ19 07 +Ġ -------------------- +ic hen +Ġunve iling +Ġsen seless +Ġfisher ies +\ - +Ġh inges +Ġcalcul us +My th +Ġund efeated +Ġoptim izations +Ġdep ress +Ġbill board +ĠY ad +ĠPy ramid +Is n +I de +Ġleg ion +ĠK ramer +ent anyl +Ġpenet rating +ĠHaw th +ĠPR ODUCT +ĠGer ard +ĠP act +ĠIn cluding +ĠEl ias +ĠEl aine +vis ual +Ġhum ming +Ġcond esc +ĠF asc +ä¸ Ĭ +Ġe galitarian +Ġdev s +ĠD ahl +O ps +D H +ĠB ounce +id ated +ald o +Ġrepublic an +Ġh amb +ĠS ett +ograph ies +CH APTER +Ġtrans sexual +Ġsky rocket +ans wer +Ġmark up +Ø ª +Ġhero ine +Comp are +ĠT av +Be ast +Ġsuccess ors +Ġna ïve +ĠBuck ley +st ress +me at +Ġdownload able +Ġindex ed +Ġsc aff +ĠL ump +ĠHom o +Stud io +In sp +Ġr acked +far ious +ĠPet ty +Ex ternal +Ġ19 09 +W ars +com mit +put ers +Ġun ob +ĠEr r +ĠE G +ĠAl am +ĠSiber ia +ĠAtmosp heric +IS TER +ĠSatan ic +trans lation +ĠL oud +tra umatic +l ique +Ġreson ate +ĠWel ch +Ġspark ing +ĠT OM +t one +Ġout l +Ġhandc uffed +ĠSer ie +8 01 +Ġland marks +ĠRee ves +Ġsoft ened +Ġdazz ling +ĠW anted +month s +Mag ikarp +Ġunt reated +ĠBed ford +M i +ĠDynam o +O re +79 5 +Ġwrong ful +Ġl ured +Ġcort isol +Ġve x +d rawn +ile t +Download ha +ĠF action +Ġlab yrinth +Ġhij acked +w aters +er ick +Ġsuper iors +ĠRow ling +ĠGu inness +Ġt d +99 2 +Ġune arthed +Ġcentr if +Ġsham eless +P od +ĠF ib +Ġ icing +Ġpredict or +Ġ29 2 +fore station +con struct +C and +@ # +Ġag itated +Ġre pr +OV A +Ġkn itting +ĠLim a +Ġf odder +68 4 +ĠPerson a +k l +7 01 +Ġbreak up +á ¸ +Ġapp alled +Ġantidepress ants +ĠSus sex +Har ris +ĠTher mal +ee ee +U pload +Ġg ulf +Ġdoor step +ĠSh ank +L U +ĠM EN +ĠP ond +s orry +Ġmis fortune +n ance +Ġb ona +M ut +Ġde graded +ĠL OG +ĠN ess +an imal +Ġa version +und own +Ġsupplement ed +ĠC ups +Ġ50 4 +Ġdep rive +ĠSpark le +Å Ĥ +ĠMed itation +auth ors +ĠSab an +ĠN aked +air d +ĠMand arin +ĠScript ures +ĠPerson nel +ĠMahar ashtra +Ġ19 03 +ĠP ai +ĠMir age +omb at +Access ory +Ġfrag mented +T ogether +Ġbelie vable +ĠGl adiator +al igned +ĠSl ug +M AT +Ġconvert ible +ĠBour bon +amer on +ĠRe hab +nt ax +Ġpowd ered +pill ar +Ġsm oker +ĠMans on +ĠB F +5 11 +ĠGood ell +ĠD AR +m ud +g art +Ġob edient +ĠTrans mission +ĠDon ation +8 80 +Ġbother ing +Material s +ãĤ ± +dest roy +Ġfore going +Ġanarch ism +ĠK ry +ice ps +Ġl ittered +ĠSch iff +Ġanecd otal +un its +Ġf ian +ĠSt im +ĠS OME +ĠInv aders +Ġbehaviour al +ĠVent ures +Ġsub lime +Ġfru ition +ĠPen alty +Ġcorros ion +¶ ħ +Ġlik ened +Ġbesie ged +ween ey +ĠCre ep +Ġlinem en +mult i +ic ably +ud der +Ġvital ity +Ġshort fall +ĠP ants +ap ist +H idden +ĠDro ps +med ical +Ġpron unciation +ĠN RL +Ġinsight ful +J V +ĠBe ard +ĠCh ou +Ġchar ms +Ġb ins +Ġamb assadors +ĠS aturdays +Ġinhib itor +ĠFr anch +6 01 +', ' +ĠCon or +art ney +ĠX peria +g rave +be es +ĠProtest ants +Ġso aking +ĠM andal +Ġph ased +Ġ6 60 +Ġsc ams +Ġbuzz ing +ĠItal ians +ĠLoren zo +ĠJ A +Ġhes itated +Ġcl iffs +ĠG OT +ingu ishable +Ġk o +Ġinter ruption +Z ip +Lear ning +Ġundersc ores +ĠBl ink +K u +57 9 +ĠAut ob +I RE +Ġwater ing +Ġpast ry +8 20 +Ġvision ary +ĠTempl ar +awa ited +Ġpist on +Ġant id +current ly +Ġp ard +Ġw aging +Ġnob ility +ĠY us +Ġinject ing +f aith +ĠP ASS +å º +Ġret ake +ĠPR OC +Ġcat hedral +b ash +Ġwrest lers +Ġpartner ing +Ġn oses +Ġ3 58 +Trans form +am en +Ġb outs +ĠId eal +ĠConstant in +Ġse p +ĠMon arch +att en +ĠPe oples +mod ified +Ġmor atorium +Ġpen chant +Ġoffensive ly +Ġprox ies +ok ane +ĠTaiwan ese +ĠP oo +ĠH OME +us ional +Ġver bs +ĠO man +vis ory +Ġpersu asion +Ġmult it +Ġsc issors +G ay +ow ay +oph ysical +l us +gn u +Ġap ocalyptic +Ġabsurd ity +Ġplay book +Ġautobi ography +I UM +Ġsne aking +ĠSim ulation +pp s +ell ery +Plan et +Ġright fully +Ġn iece +ĠN EC +ĠIP O +ĠDis closure +lean or +ous y +ST ER +Ġ28 2 +Cru z +Ch all +64 3 +ĠSurv ive +ĠF atal +ĠAm id +ap o +We apons +D EN +7 70 +ĠGreen wald +Ġlin en +al os +Ġpollut ants +ĠPCI e +k at +Ġp aw +ĠK raft +C hem +ĠTermin ator +Ġre incarn +Ġ] [ +ĠSe eds +Ġsilhou ette +ĠSt ores +Ġgro oming +ĠD irection +ĠIs abel +ĠBr idges +ðŁ ij +E ED +ĠM orsi +Ġval ves +ĠRank ed +ĠPh arma +ĠOrgan izations +Ġpenet rated +ĠRod ham +ĠProt oss +Ġove rest +Ġex asper +ĠT J +Ġ 000000 +Ġtrick le +Ġbour bon +WH O +Ġw retched +Ġmicrosc opic +Ġcheck list +Ġad orned +R oyal +Ad minist +ĠRet irement +ĠHig hest +We ather +ile ge +Ġincre ments +ĠC osponsors +Ġmas se +ĠS inn +r f +Ġh ordes +as sembly +75 4 +ĠNat asha +ĠTY PE +ĠGEN ERAL +Ġarr anging +Ġ40 7 +l ator +Ġg lean +Ġdisc redited +Ġclin icians +UN E +Ġachie ves +ĠEm erson +com plex += [ +Ġprincip ally +Ġfra il +p icked +Ġthan king +Ġre cl +ĠL AST +Ġsupp ressing +il ic +Ġantidepress ant +ĠLis bon +Ġth or +Ġsp a +Ġking doms +ĠPear ce +em o +Ġpl ung +Ġdiv est +Ġ ******************************** +b is +osp els +ad r +Sp irit +hall a +P ink +end ez +Ġresurrect ed +esc ape +ĠRosen stein +Ġge ological +Ġnecess ities +Ġcarn iv +ĠE lys +ĠBar ney +Ġ29 6 +dig y +ST ON +D OWN +Ġmil estones +Ġk er +Ġdismant ling +Ġre prim +Ġcross ings +19 45 +Ġpatri archy +Ġblasp hemy +Ġ3 59 +met ry +ĠOb esity +ĠDiff erences +bl ocking +ãĥķ ãĤ¡ +ich ita +ĠSab ha +ph alt +ĠCol o +ual a +effic ients +ĠMed ina +con sole +55 7 +ĠHann ibal +ĠHab it +ĠF ever +Ġthen ce +Ġsyn agogue +Ġessential s +Ġw ink +ĠTr ader +ID A +ĠSp oiler +ĠIceland ic +ĠHay ward +Ġpe ac +Ġmal ice +Ġflash back +Ġth w +Ġlay offs +L iquid +Ġtro oper +Ġh inge +ĠRead ers +Ph ill +ĠB auer +Cre ated +Ġaud its +ac compan +Ġunsus pecting +ier a +6666 6666 +Ġbro ch +Ġapprehend ed +ĠM alk +cer ning +ĠCod ex +O VER +M arsh +ĠD eng +ĠExp ression +Ġdisrespect ful +Ġasc ending +t ests +ĠPlaint iff +ster y +ĠAl ibaba +din and +ĠDem psey +Applic ations +mor al +Ġthrough put +Ġquar rel +Ġm ills +Ġhe mor +ĠC ASE +terror ist +st im +ifest yle +ro zen +CE PT +Ar k +u ci +lect ic +Ġirrit ating +she ets +A y +Ġrede emed +Ġhorn y +ĠTe ach +ĠS ear +dem ocracy +4 65 +ĠRest ore +Ġstand by +ĠP is +iff in +Ġsleep y +Ġextr ater +Ġcompl iments +Fram eworks +Ġinstall s +Ġb anging +sur face +found land +Ġmetaph ysical +Ġ28 3 +oul s +dev ices +Ar gs +ĠSac rifice +ĠMcC orm +es on +Cons ervative +ĠM ikhail +see ing +is ively +ĠRo oms +ĠGener ic +Ġenthusi astically +Ġgri pped +Ġcomed ic +ĠElectric ity +Ġgu errilla +Ġdec oration +ĠPerspect ive +Ġconsult ations +Ġun amb +Ġplag iar +Ġmagic ian +Ġe rection +ĠTour ism +or ied +ro xy +11 00 +T am +Ī è +Î ³ +× ª +ĠPred ators +Nit rome +Ġtelesc opes +project s +Ġun protected +Ġst ocked +ĠEnt reprene +nex pected +Ġwast ewater +V ill +Ġint imately +Ġi Cloud +ĠConst able +Ġspo of +Ġne farious +Ġfin s +Ġcens or +ĠMod es +ĠEs per +ar bon +Ġinter sections +Ġlaud ed +Ġphys i +Ġgener ously +ĠThe Nitrome +ĠTheNitrome Fan +Ġar isen +ĠÙ Ī +Ġg lands +ĠPav ilion +ĠGu pta +Ġuniform ly +Ġr amps +ri et +ĠWH EN +ĠVan essa +Ġrout ed +Ġlim p +ĠC PI +p ter +int uitive +Ġv aping +Ġexperiment ed +ĠOlymp us +ĠAm on +Ġsight ing +Ġinfiltr ate +ĠGentle man +Ġsign ings +ĠMe ow +ĠNav igation +che cks +4 33 +Ġel apsed +ĠBulg arian +esp ie +ĠS OM +d uring +Ġsp ills +anc a +ĠPly mouth +M AL +Ġdomest ically +ĠWater gate +ĠF AM +k illed +ed ited +ĠYour self +Ġsynchron ization +ĠPract ices +ST EP +Ġgen omes +ĠQ R +not ice +Ġloc ating +z in +Ġ3 29 +al cohol +Ġk itten +V o +Ġr inse +Ġgrapp le +ĠSc rew +ĠD ul +A IR +Ġle asing +ĠCaf é +Ġro ses +ĠRes pect +Ġmis lead +Ġperfect ed +Ġnud ity +Ġnon partisan +ĠCons umption +Report ing +Ġnu ances +Ġdeduct ible +ĠSh ots +Ġ3 77 +Ġæ ľ +ano oga +Ben ef +ĠB am +ĠS amp +if ix +Ġgal van +ĠMed als +rad ius +Ġno bles +Ġe aves +igr ate +K T +ĠHar bour +u ers +Ġrisk ed +re q +Ġneuro t +get table +ain a +Rom ney +Ġunder pin +Ġlo ft +ĠSub committee +ĠMong ol +b iz +Ġmanif ests +ass isted +ĠG aga +Ġsy nergy +Ġreligious ly +ĠPre f +ĠG erry +T AG +ĠCho i +4 66 +beh ind +ĠO u +Gold Magikarp +Ġhemor rh +R iver +Ġtend on +Ġinj ure +ĠF iona +Ġp ag +Ġag itation +|| || +ur an +ĠE SA +Ġest eem +Ġdod ging +Ġ4 12 +r ss +Ġce ases +ex cluding +Ġint akes +Ġinsert s +Ġemb old +ĠO ral +up uncture +4 11 +ĠUn ified +ĠDe le +Ġfurn ace +ĠCoy otes +ĠBr ach +L abor +Ġhand shake +Ġbru ises +Gr ade +éĹ ĺ +ĠGram my +ile en +St ates +ĠScandinav ian +ĠKard ash +8 66 +Ġeffort lessly +ĠDI RECT +ĠTH EN +ĠMe i +ert ation +19 68 +Ġgro in +w itch +Requ irements +98 5 +Ġroof s +Ġest ates +ĠH F +Ġha ha +Ġdense ly +ĠO CT +Ġpl astics +Ġincident ally +ĠTr acks +ĠTax es +Ġch anted +Ġforce ful +ĠBie ber +ĠK ahn +K ent +ĠC ot +lic ts +F ed +Ġhide ous +ĠVer d +ĠSynd icate +ĠIl legal +J et +ĠD AV +re asonable +c rew +Ġfundamental ist +Ġtruth ful +ĠJ ing +Ġl il +Ġdown ed +Ġen chanted +ĠPolic ies +ĠMcM aster +ĠH are +ides how +Ġpar ams +en cers +gorith m +Ġallow ances +Ġturb ulent +Ġcomplex ities +ĠK T +Ġ3 37 +ĠGen etic +F UN +D oug +t ick +Ġg igs +ument hal +Ġpatriarch al +Ġcal c +, ... +Ġc out +ĠGu an +Ġpath ological +ĠR ivals +Ġunder rated +Ġflu orescent +ĠJ iu +arna ev +ĠQu an +Ġ4 29 +Ġ ਠ+M ario +Con struct +ĠC itation +ĠR acial +ĠR SA +ĠF idel +Ġ3 95 +Person ally +C ause +à » +rad ical +in en +Ġvehement ly +ĠPap a +Ġintern ship +Ġfl akes +ĠRe ck +Luck ily +B ra +20 20 +rav ings +R N +W onder +Ser iously +Ġre usable +Ġpoll uted +ĠP eng +le igh +ind le +Ġcircuit ry +ĠMad onna +ĠB ART +Res idents +att ribute +Phil adelphia +Cl ub +Ġplan ner +Ġfr antically +Ġfaith fully +ĠTerrit ories +ĠL AT +ĠAnders en +an u +ĠP ARK +ĠS ora +i age +ĠPlay offs +ĠG CC +4 27 +Ġab norm +ĠL ever +Ġdisob edience +As ync +ĠShe a +V ert +Ġsk irts +ĠSaw yer +x p +Ġwors ening +Ġsc apego +ĠAng le +oth al +Ġtro ve +ĠSt y +ĠN guyen +mar ine +ide on +Dep ths +Bl og +ĠIll uminati +Ġtract s +Ġorgan ise +Ġo str +F s +Ġlever aging +ĠD aredevil +as ar +Ġl ang +Ġex termin +urs ions +ĠRom o +ãĤ¤ ãĥĪ +Ġcont ended +Ġencounter ing +ĠTable t +ĠAltern ate +sk ill +Ġswe ets +Ġco hesive +cap acity +Ġrep ud +Ġl izard +ro o +Ġpilgr ims +ĠR uff +ĠInstr ument +ĠLog o +uit ous +E H +Ġsales man +Ġank les +L ed +ĠPat ty +ud os +Own er +Ġdiscrep ancies +k j +M U +Ġuncond itional +Dragon Magazine +i ard +O ak +ĠConvers ation +be er +ĠOs aka +D elta +us ky +Ġsecret ion +Ġpl aza +Ġm ing +Ġde pletion +ĠM ous +ĠI TS +ĠH imal +ĠFle ming +Ġcyt ok +ĠH ick +Ġbat ters +ĠInt ellectual +6 75 +é r +IS ION +ĠQu entin +ĠCh apters +ih adi +Ġco aster +WAY S +ĠL izard +ĠY or +and ering +S kin +ha ust +ab by +Ġportray ing +Ġwield ed +d ash +Ġprop onent +Ġr ipple +Ġgrap hene +Ġfly er +Ġrec urrent +Ġdev ils +Ġwater fall +æĺ ¯ +go o +Text Color +Ġtam pering +IV ES +TR UMP +ĠAb el +ĠS AL +ĠHend ricks +ĠLu cius +b ots +Ġ40 96 +IST ORY +Gu est +ĠN X +in ant +Ben z +ĠLoad ed +ĠCle ver +t reatment +Ġta vern +Ġ3 39 +ĠT NT +ific antly +Tem perature +F el +Ġunder world +ĠJud ges +Ġ< + +Ġst ump +Ġoccup ancy +Ġab er +ĠF inder +) ", +ĠN unes +res et +in et +ect omy +Ġwell ness +ĠP eb +quart ered +and an +Ġneg atives +ĠTh iel +ĠCl ip +ĠL TD +Ġbl ight +Ġreperto ire +K yle +Ġqu er +ĠC es +Ġha pl +98 9 +ĠTh ames +isc opal +Des k +ivari ate +ĠEx cellence +found ation +Ġâ ĩ +X i +Ġmyster iously +esty les +Ġper ish +ĠEng els +ĠDE AD +09 0 +}} } +ĠUn real +Ġrest less +ID ES +orth odox +ĠInter mediate +Ġdin ners +ĠTr out +ĠSe ym +ĠHall s +og ged +Ġtraged ies +Ġdid nt +67 6 +Ġail ments +Ġobserv able +ĠV ide +ad apt +ĠD usk +Ġprofessional ism +ĠPres cott +ĠInd ies +p ox +ĠMe hran +W ide +Ġend emic +ĠPar an +B ird +Ġped als +ĠI U +ĠAdam ant +ĠH urt +Ġcorrel ates +urd en +Ġspons oring +cl imate +ĠUnivers ities +ĠK not +enn es +ĠDam ian +ĠAx el +S port +Ġbar b +ĠS no +sh own +ste en +ud ence +Ġnon violent +Ġhom ophobia +Ġbiom ass +ĠDet ail +Ġsrf N +ĠT une +accompan ied +I ENCE +Al bert +ĠMong o +z x +ĠCer berus +or bit +c ens +Ġsl ay +SH ARE +H Y +Ġb rawl +ĠPro be +Ġnonex istent +ĠClare nce +ĠBlack burn +Ġport als +ĠR ita +ĠRem ain +ĠLe vant +Ġtrick ed +ĠF erry +aver ing +ĠStraw berry +ĠAn swers +Ġhorrend ous +ĠA man +Supp lement +ĠT oad +Ġpe eled +Ġman oeuv +ĠU zbek +mond s +ĠH ector +Ġ40 2 +pe es +fix es +Ġd j +Ġres umes +Ġaccount ant +Ġadvers ity +Ġham pered +ĠL arson +Ġd oping +part s +H ur +Ġbe arded +Ġy r +ĠPlug in +å¥ ³ +Ġ/ ** +rol ley +Ġwaters hed +ĠSub mission +if lower +AS C +Ġcho ir +Ġsculpt ures +m A +incre asing +ai i +Ġsne akers +Ġconfront s +ĠEle phant +ĠEl ixir +Ġrec al +ĠT TL +w idget +ĠW ax +ĠGr ayson +Ġha irst +Ġhumili ated +ĠWAR N +app iness +ĠT TC +F uel +Ġpol io +Ġcomplex es +Ġbab e +ĠX IV +P F +). [ +P arts +Ġ4 35 +M eg +ĠY ards +ĠAL P +Ġy ells +Ġprin ces +Ġbull ies +ĠCapital ism +ex empt +FA Q +ĠSp onge +ĠAl a +Ġpleas antly +Ġbu f +Ġden ote +Ġunp ublished +Ġkne eling +asc a +Ġl apse +al ien +99 4 +Ġrefere es +ĠLaw yers +S anta +Ġpuzz ling +ĠProm etheus +ĠPh araoh +ĠDel ay +Ġfacilit ates +ĠC ES +Ġjew els +Ġbook let +ond ing +Ġpolar ization +ĠMor an +ĠSal ad +ĠS OS +ĠAdv ice +PH OTOS +IC AN +iat ures +ex press +ĠWonder land +ĠC ODE +ĠCL ASS +9 75 +Ġg rep +ĠD iesel +ĠGl ac +! ?" +Ġr m +o ine +disc rimination +ĠN urse +m allow +Ġv ortex +ĠCons ortium +Ġlarge Download +stra ight +augh lin +G rad +Ġpublic ized +ĠW aves +ĠRed d +Ġfest ivities +ĠM ane +ar ov +Ġfleet ing +ĠDr unk +ug en +C ele +Ġchromos omes +ĠD OT +-+-+ -+-+ +Ġbus iest +ĠBe aver +Sy rian +ĠK yr +k as +ĠCross Ref +19 50 +76 01 +Ġrepe aling +ĠWin ners +ĠMac ro +ĠD OD +bl ance +S ort +64 1 +Ġmet re +ĠD irk +Ġgo ggles +Ġdraw backs +Ġcomplain ant +Ġauthor izing +Ġantit rust +oper ated +Ġm ah +Ġexagger ation +Am azing +ĠSer aph +Ġha ze +w ow +Ġextingu ished +Ġcan yon +ĠB osh +Ġv ents +Ġsc rape +Cor rect +4 26 +Ġav g +Dem and +ĠâĪ ¼ +Ġmicrobi ota +"} ]," +ĠSt ev +B io +ĠPlan es +Ġsuggest ive +Ġdec ipher +ĠRefuge e +ĠKe jriwal +ĠGreen peace +Ġdecl ass +ĠSound ers +Ġth o +Ġdec rypt +Ġbr ushing +ĠJane iro +ip op +S i +8 77 +ĠGeoff rey +Ġc pu +ĠHaz el +Ġview points +Ġcris py +ĠNot ification +Ġsold er +ĠMod est +ĠHem isphere +Ġcass ette +in cludes +Ġident ifiers +ĠC ALL +in cent +T odd +ĠSwe ep +Ġ3 34 +b oss +Ġsm ir +gin x +Ġtown ship +Ġg rieving +ĠMos que +Net flix +AS ED +ĠMillenn ials +oc om +19 67 +Ġbold ly +s leep +Ġes che +arij uana +Ġsw irl +ĠPen al +Ġneglig ent +ĠStephen son +K ER +ĠZ oro +ris is +Ġlocal ization +ĠSeym our +ĠAng lic +red itation +prot ection +ĠPa ige +Ġo mit +ĠR ousse +ĠT ub +Ġinv itations +t ty +Ġm oss +ph ysical +C redits +Ġan archy +Ġchild care +Ġl ull +ĠM ek +ĠL anguages +lat est +ĠSan ford +Ġus ability +Ġdiff use +ĠD ATA +Ġsp rites +ĠVeget a +ĠProm otion +ãĥ¼ ãĤ¯ +rict ing +z ee +Tur kish +ĠTD s +pro ven +57 1 +Ġsmug glers +707 10 +Ġreform ed +ĠLo is +Ġun fl +ĠWITH OUT +ĠReturn ing +ann ie +ĠTom as +Fr anc +ĠProf it +ĠSER V +ĠR umble +ik uman +es an +Ġt esters +Ġgad get +Ġbrace let +ĠF SA +comp onent +Ġparamed ics +Ġj an +ĠRem em +ĠSk inner +Ġl ov +ĠQu ake +rom a +Ġfl ask +Pr inc +Ġover power +Ġlod ging +ĠK KK +ret te +Ġabsor bs +w rote +Ġ ," +K ings +ĠH ail +ĠFall ing +xt ap +ĠHel ena +ire ns +L arry +Ġpamph let +ĠC PR +G ro +ĠHirosh ima +Ġhol istic +". [ +Ġdet achment +Ġas pire +Ġcompl icit +ĠGreen wood +Ġresp awn +ĠSt upid +ĠFin ished +f al +b ass +Ġab hor +Ġmock ery +ĠFe ast +VID EO +Ġcon sec +ĠHung ry +P ull +ĠH ust +it ance +? ãĢį +) -- +ĠPar allel +con v +4 69 +ha ar +w ant +P aper +m ins +ĠTor o +ĠTR UMP +ĠR ai +D W +ĠW icked +ĠL ep +Ġfun ky +Ġdetrim ent +ios is +ache v +Ġde grade +im ilation +Ġret ard +Ġfrag mentation +Ġcow boy +ĠY PG +ĠH AL +Parent s +ĠS ieg +ĠStra uss +ĠRub ber +× IJ +Fr ag +Ġp t +Ġoption ally +ĠZ IP +ĠTrans cript +ĠD well +88 2 +M erc +ĠM OT +ãĥ¯ ãĥ³ +Ġhun ts +Ġexec utes +In cludes +Ġacid ic +ĠRespons ibility +ĠD umb +we i +And erson +ĠJas per +ight on +abs olutely +Ad ult +Ġpl under +Mor ning +ĠT ours +ĠD ane +Î º +ĠT EST +ĠG ina +Ġcan ine +aw an +Ġsocial ists +ĠS oda +Ġimp etus +ĠSupplement ary +oli ath +ĠKinn ikuman +mitted ly +second s +Ġorganis ers +Ġdocument aries +Vari able +GRE EN +Ġres orts +Ġbr agging +Ġ3 68 +Art ist +w k +bl ers +Un common +ĠRet rieved +Ġhect ares +Ġtox in +r ank +Ġfaith s +ĠG raphic +Ġve c +ĠL IA +Af rican +Ġard ent +end iary +L ake +ĠD OS +cient ious +ĠOk awaru +ĠAll y +ĠTim eline +D ash +ĠI c +contin ue +Ġt idy +Ġinstinct ively +ĠP ossibly +ĠOut door +ĠWould n +Ġl ich +ĠBr ay +ĠA X +Ġà ī +Ġ+ # +\ ' +Direct ory +ab iding +Ġf eral +ic ative +but t +Ġper verse +S alt +Ġwar ped +Ġnin eteen +Ġcabin ets +Ġsrf Attach +ĠSl oan +Ġpower ing +reg ation +F light +se vere +Ġst ren +Ġc og +ap ache +Ġâ Ŀ +Ġcaf eteria +p aces +ĠGrim oire +uton ium +Ġr aining +Ġcir cling +Ġlineback ers +c redit +Ġrep atri +ĠCam den +lic ense +Ġly ric +Ġdescript or +Ġval leys +Ġre q +Ġback stage +ĠPro hibition +ĠK et +Op ening +S ym +æĸ ¹ +Ġserv ings +Ġoverse en +Ġaster oids +ĠMod s +ĠSpr inger +ĠCont ainer +è » +ĠM ens +Ġmult im +Ġfire fighter +pe c +Ġchlor ine +Ð ¼ +end i +Ġsp aring +Ġpolyg amy +ĠR N +ĠP ell +Ġt igers +Ġflash y +ĠMad ame +S word +Ġpref rontal +Ġpre requisite +uc a +Ġw ifi +Ġmiscon ception +Ġharsh ly +ĠStream ing +ot om +ĠGiul iani +foot ed +Ġtub ing +ind ividual +z ek +n uclear +m ol +Ġright ful +49 3 +Ġspecial ization +Ġpassion ately +ĠVel ocity +ĠAv ailability +T enn +Ġl atch +ĠSome body +Ġhel ium +cl aw +Ġdi pping +XX X +Ġinter personal +7 10 +Ġsub ter +Ġbi ologists +ĠLight ing +Ġopt ic +Ġden im +end on +ĠC orm +Ġ3 41 +ĠC oup +Ġfear less +Ġal ot +ĠCliff ord +ĠRun time +ĠProv ision +up dated +lene ck +Ġneur on +Ġgrad ing +ĠC t +sequ ence +in ia +con cept +Ġro aring +ri val +ĠCaucas ian +Ġmon og +key es +Ġappell ate +Ġlia ison +EStream Frame +ĠPl um +! . +Ġsp herical +Ġper ished +Ġbl ot +Ġben ches +Ġ4 11 +Ġpione ered +Ġhur led +Jenn ifer +ĠYose mite +Ch air +Ġreef s +Ġelect or +ĠAnt hem +65 2 +Ġun install +Ġimp ede +Ġbl inking +Ġgot o +Dec re +A ren +Ġstabil ization +ĠDis abled +ĠYanuk ovych +Ġoutlaw ed +ĠVent ura +ten ess +Ġplant ation +Ġy acht +ĠHu awei +Ġsol vent +Ġgr acious +Ġcur iously +Ġcapac itor +Ġc x +ĠRef lex +Ph ys +ĠC f +pt in +cons ervative +Ġinv ocation +c our +F N +ĠNew ly +H our +As ian +ĠLe ading +ĠAer ospace +An ne +Ġpre natal +Ġdeterior ating +H CR +ĠNorm andy +ol ini +ĠAm bro +9 10 +Ġset backs +ĠT RE +Ġs ig +ĠSc ourge +59 7 +79 8 +Game play +Ġm sec +M X +Ġprice y +ĠL LP +aker u +Ġover arching +ĠB ale +Ġworld ly +Cl ark +Ġscen ic +Ġdisl iked +ĠCont rolled +T ickets +ĠE W +ab ies +ĠPl enty +Non etheless +Ġart isan +Trans fer +ĠF amous +Ġinf ield +ble y +Ġunres olved +ĠML A +ãĤ Ĥ +Cor rection +Ġdemocr at +ĠMore no +ro cal +il ings +Ġsail or +Ġr ife +h ung +Ġtrop es +Ġsn atched +ĠL IN +ĠB ib +ES A +ĠPre v +ĠCam el +run time +Ġob noxious +4 37 +Ġsum mers +Ġunexpl ained +ĠWal ters +cal iber +Ġg ull +ĠEnd urance +ä½ ľ +Ġ3 47 +Ir ish +Ġaer obic +Ġcr amped +ĠHon olulu +à © +us erc +ec ast +AC Y +ĠQu ery +ãĤ¹ ãĥĪ +Bet a +Ġsuscept ibility +ĠSh iv +ĠLim baugh +Ġà ĸ +ĠN XT +ĠM uss +ĠBrit ons +ES CO +EG IN +Ġ% % +Ġsec ession +ĠPat ron +ĠLu a +n aires +ĠJPM organ +us b +ocy te +Ġcouncill ors +ĠLi ang +f arm +Ġnerv ously +Ġattract iveness +ĠK ov +j ump +Pl ot +Ġst ains +ĠStat ue +ĠApost les +he ter +ĠSUP PORT +Ġoverwhel m +Y ES +Ġ29 1 +d ensity +Ġtra pping +M it +Ġf ide +ĠPam ela +atl antic +Dam n +Ġp ts +OP A +Ġserv icing +Ġoverfl owing +ul o +ĠE rit +t icket +light ing +ĠH mm +ãĥ¼ ãĥ« +im oto +Ġchuck le +4 23 +ãģ ķ +sh ape +Ġque ues +Ġanch ors +ãĤ¼ ãĤ¦ãĤ¹ +F er +Ġaw oke +Ġ6 66 +h ands +Ġdiver gence +Ġ50 5 +T ips +Ġdep ot +Ġske w +ĠDel iver +op ot +Ġdiv ul +ĠE B +uns igned +ĠUn i +X box +Ġfor ks +Ġ7 02 +å ¯ +Ġpromot ers +ĠV apor +Ġlev ied +sl ot +Ġpig ment +Ġcyl inders +C RE +Ġsn atch +Ġperpet ually +Ġl icking +ĠFe et +ĠKra ken +ĠHold en +ĠCLS ID +m r +Ġproject or +Ġden otes +Ġchap el +ĠTor rent +b ler +R oute +ĠDef endant +ĠPublisher s +ĠM ales +ĠInn ov +ĠAg ility +rit er +ty mology +st ores +L ind +Ġf olly +ĠZur ich +B le +Ġnurt ure +Ġcoast line +uch in +D omin +Ġfri vol +ĠCons olid +res ults +M J +Ġphyl ogen +Ġha uled +ĠW iley +ĠJess ie +ĠPrep are +ĠE ps +Ġtreasure r +I AS +Ġcolon ists +Ġin und +ĠWW F +ĠCon verted +6 000 +out side +ĠApp earance +ĠRel ic +ĠM ister +s aw +Ġresult ant +Ġadject ive +ĠLaure l +ĠHind i +b da +Pe ace +Ġreb irth +Ġmembr anes +Ġforward ing +Ġcoll ided +ĠCar olyn +K ansas +5 99 +ĠSolid GoldMagikarp +Be ck +Ġstress ing +ĠGo o +ĠCooper ative +Ġf s +ĠAr chie +L iter +ĠK lopp +J erry +Ġfoot wear +War ren +Ġsc ree +h are +Under standing +P ed +Ġanth ology +ĠAnn ounce +M ega +Ġflu ent +Ġbond age +ĠDisc ount +il ial +C art +ĠNight mares +Sh am +ĠB oll +uss ie +H ttp +Atl anta +Ġun recogn +ĠB id +Ġunder grad +Ġforg iving +ĠGl over +AAAA AAAA +4 45 +V G +pa io +kill ers +Ġrespons ibly +Ġmobil ize +Ġeffect ed +ĠL umin +Ġk ale +Ġinfring ing +ann ounced +Ġf itt +b atch +ĠT ackle +ĠL ime +ĠAP P +uke mia +Ġrub y +Ġex oner +ĠCas ual +0 70 +Ġpel vic +Ġautom ate +ĠK ear +ĠCoast al +Ġcre ed +Ġbored om +ĠSt un +ri ott +Ĥ İ +Ġregener ate +Ġcomed ians +ĠOP ER +Sp ons +id ium +on is +L ocated +05 7 +Ġsusp ense +ĠD ating +C ass +Ġneoc ons +ĠShin zo +Ġaw oken +ch rist +ĠMess ages +att led +ĠSpr ay +ĠSp ice +C W +Ġshield ing +ĠG aul +Am id +Ġparam ilitary +Ġmult if +ĠTan ner +il k +Ġgodd amn +g ements +Ġbe friend +m obi +Ġ3 88 +fold er +acc a +Ġins in +g ap +N ev +fif th +Ġpsychiat ry +b anks +TH IS +Ġhar b +ac qu +Ġfac ade +ĠPower Point +80 3 +Ġbl uff +Sh ares +Ġfavor ing +El izabeth +Ãį Ãį +Ġr anger +77 2 +ĠAr che +h ak +ĠGen etics +ĠF EMA +Ġev olves +Ġest e +ĠP ets +ĠM é +ĠInterest ing +ĠCanter bury +ch apter +ĠStar fleet +Sp anish +Ġdraw back +ĠNor wich +9 70 +n orth +ag anda +Ġtransform ative +ram ids +bi ology +ad ay +Ġpropag ation +ĠGam ma +ĠDen ise +ĠCalcul ator +ent imes +ĠB ett +Ġapp endix +ĠHD D +AK ING +Ġst igmat +Ġhol ster +Ġord inarily +Ch ance +ĠCont rary +Ġad hesive +Ġgather s +6 12 +re au +ony ms +ew ays +Ġindu ces +Ġinterchange able +se m +Wh it +Ġtr ance +Ġincorpor ation +ĠExt ras +Fin ancial +Ġawkward ly +ĠStur geon +ĠH Y +Norm ally +ĠEnd ing +ĠAss ist +enc rypted +Ġsub jug +Ġn os +Ġfan atic +C ub +C U +?" . +Ġirre versible +å Ĥ +03 1 +ĠH AR +sp read +ul ia += $ +Sc ope +L ots +Ġlif estyles +ol on +Ġf eds +Ġcongrat ulate +web kit +Ġindist inguishable +ĠSw ing +Ġcommand ments +qu ila +ab ella +m ethyl +ann abin +Ġo vere +Ġlob ster +ĠQU EST +ĠCONT IN +bern atorial +:::: :::: +ĠTra ve +ĠSam oa +AN I +75 2 +Ð ´ +userc ontent +ĠMod erate +y eah +ĠK itt +Ġwe e +Ġstuff ing +ĠInter vention +ĠD ign +Ġware houses +ĠF iji +Ġpel lets +Ġtake away +ĠT ABLE +ĠClass ical +col lection +Ġland fall +ĠMus cle +Ġsett les +ĠAD V +Ġ3 44 +L aura +Ġf ared +ĠPart ial +4 36 +oss ibility +ĠD aly +ĠT arant +ĠFu ji +am l +c ence +55 1 +ĠProced ures +ĠO CD +ĠU D +t in +Q UI +ach o +4 38 +Ġgl itches +Ġenchant ment +Ġcalcul ates +IR O +ĠH ua +alys es +ĠL ift +um o +Ġle apt +Ġhypothes ized +ĠGust av +it ans +VERS ION +æ ł +Rog er +Ġr and +ĠAd apter +Ġ3 31 +ĠPet ition +k ies +M ars +Ġunder cut +ze es +ĠLy ons +ĠDH CP +Miss ing +Ġretire es +Ġins idious +el i +> ) +. ãĢį +Ġfinal ists +ĠA ure +Ġacc user +Ġwas tes +ĠY s +ĠL ori +Ġconstitu encies +Ġsupp er +Ġmay hem +or ange +Ġmis placed +Ġmanager ial +Ġex ce +ĠCL I +Ġprim al +ĠL ent +Cry stal +h over +ĠN TS +end um +Ġd w +ĠAl c +n ostic +Ġpres erves +ĠTs arnaev +Ġtri pled +rel ative +Arc ade +k illing +ĠW EEK +ĠH anna +D ust +Com pleted +ģ « +Ġappro ves +ĠSur f +ĠLuther an +ven ants +Ġrobber ies +we ights +soft ware +at ana +ug al +Ġgrav y +ĠC ance +OLOG Y +ly ak +Ton ight +Ġunve il +Ġ19 04 +ĠMin ion +ent ious +st ice +pack ages +ĠG EAR +Ġg ol +ĠHutch inson +ĠProf ession +ĠG UN +ĠDiff erence +ĠTsuk uyomi +ĠLes bian +6 70 +Ġfug itive +ĠPlan etary +-------------------------------- ------------------------ +Ġacc rued +Ġch icks +Ġsto pp +Ġblock ers +C od +Ġcomment ers +ĠSomew here +ĠPhot ographer +the me +Ġmay oral +w u +Ġanten nas +Ġrev amped +ĠSubject s +it é +im ura +Ġentr ances +liter ally +Ġten ets +ĠO MG +ĠMP H +ĠDon key +ĠOff ense +Ġ" + +Sn ap +ĠAF B +Ġan imate +ĠS od +His panic +Ġinconsist ency +D b +F Y +Ex port +Ġa pe +Ġpear l +ib el +ĠPAC s +Ġ{ \ +Ġact u +ĠHS BC +camp us +Ġpay off +Ġde ities +ĠN ato +ou ple +Ġcens ored +ĠCl ojure +Ġconf ounding +en i +Ġreck on +op he +Ġspot ting +Ġsign ifies +Ġprop el +Ġfest ive +S uggest +Ġpled ging +ĠB erman +Ġrebell ious +Ġovershadow ed +Ġinfiltr ated +j obs +67 2 +Ġscal able +Ġdomin ion +ĠNew foundland +ĠMead ow +Ġpart itions +AM I +Ġsupplement ary +str ument +Ġhair y +Ġperpet uate +Ġnuts hell +ĠPot ato +ĠHob bit +Ġcur ses +Flo at +Ġquiet er +Ġfuel ing +Ġcaps ules +ĠL ust +ĠH aunted +Exec utive +Ġchild birth +G re +Ġrad iant +å İ +Ġm alls +Ġin ept +ĠWarrant y +Ġspect ator +E h +t hens +Ġculmin ating +æ © +ary a +ãĤ ® +ilit arian +ĠOR IG +ĠSp ending +pt ives +ĠS iren +ĠRec ording +ay ne +Ġv im +Ġspr ang +T ang +ĠM FT +mor ning +ĠWe ed +m peg +cess ion +ĠCh ung +7 30 +w arning +56 2 +handed ly +P oor +P olitics +: # +Ġp ian +Ġfec es +ĠDocument ation +Ġban ished +Ġ3 99 +ĠAR C +Ġhe inous +J ake +ĠAm ir +way ne +v re +os henko +Ġnotebook s +Ġfound ational +Ġmarvel ous +ixt ape +Ġwithdraw als +Ġh orde +ĠD habi +is able +ĠK D +Ġcontag ious +ĠD ip +ĠAr rows +Ġpronoun s +Ġmorph ine +ĠB US +68 2 +Ġk osher +fin ished +ĠInstr uments +Ġf used +yd en +ĠSal mon +F ab +aff ected +K EN +C ENT +Dom ain +Ġpoke mon +ĠDr inking +G rowing +ĠInvestig ative +ĠA ether +em i +Ġtabl oid +Ġrep ro +ĠNot withstanding +ĠBers erker +Ġdram as +Ġclich é +Ġb ung +ĠU RI +ĠD os +0 44 +Ġpast ors +Ġl s +Ġac rylic +aun ts +Ed ward +Ġmajor ities +B ang +Ġfield ing +ĠRepl acement +ĠAl chemy +pp ard +ĠRome o +ĠSan ct +ĠLav rov +ib ble +Inst ruct +Ġimp ractical +ĠPlay boy +ce phal +Ġsw aps +Ġk an +ĠThe o +Ġillust rating +Ġdismant led +ĠTrans gender +ĠG uth +UG H +Ġtriumph ant +Ġencomp ass +Ġbook mark +udd in +j er +Ġpred icate +ES H +Ġwhen ce +ĠAB E +Ġnon profits +Se qu +Ġdi abetic +Ġp end +Ġheart felt +sh i +Ġinter acts +ĠTele com +Ġbombard ment +dep ending +ĠLow ry +ĠAd mission +ĠBl ooming +ust ration +ene gger +B rew +Ġmol ten +ĠNer d +P IN +âĸ Ģ +ave ment +Ġtou red +Ġco efficients +ĠTray von +ans son +Ġsand y +t old +fl ows +Ġpop ulous +ĠT inder +ĠBl iss +R achel +Min imum +Ġcontest ant +ĠRed uce +ĠMor se +ĠGrass ley +ĠClick er +Ġexp r +Ġs incerity +Ġmar qu +Ġelic it +ĠPro position +ĠDemon ic +Ġtac os +G reek +Ġpost war +Ġin sofar +ĠP ork +Ġ35 2 +doctor al +walk ing +Ġmid term +ĠSam my +sight ed +ĠTR ANS +ic i +AL D +ĠUS L +ĠF ISA +ĠAm pl +ĠAlex andra +ine lli +Tr ain +Ġsign ify +ĠVers us +Ġob fusc +Ġk h +Ġagg ro +ĠRen ault +Ġ3 48 +5 18 +ox icity +0 22 +ĠTw ist +Ġgoof y +D ynamic +Ġbrief ings +m ight +8 99 +Ġderog atory +T ro +Ġfor ging +ĠKor an +ĠMar ried +ĠBuc s +Ġpal ate +ĠCon version +m able +4 13 +Ġ( _ +Ġs iph +ĠN EO +col lege +Ġmarg inally +Ġfl irt +ĠTra ps +ĠP ace +é »Ĵ +Ġgoalt ender +Ġforb ids +Ġcler ks +ĠT ant +ĠRobb ins +ĠPrint ing +Ġpremie red +Ġmagn ification +ĠT G +ĠR ouse +ĠM ock +odynam ics +Ġpre clude +ism o +ĠPul itzer +Ġaval anche +ĠK odi +rib une +ĠL ena +Elect ric +Ġref inery +Ġend owed +Ġcounsel ors +Ġd olphin +ĠM ith +Ġarm oured +hib ited +Beg in +ĠP W +O il +ĠV or +ĠShar if +ĠFraz ier +est ate +Ġj ams +Pro xy +Ġband its +ĠPresbyter ian +ĠPrem iere +t iny +ĠCru el +Test ing +Ġhom er +ĠV ERS +ĠPro l +ĠDep osit +ĠCoff in +Ġsemin ars +Ġs ql +ĠDef endants +Altern atively +ĠR ats +ç « +ethy st +' > +Ġiss uer +58 9 +Ġch aired +ĠAccess ories +man ent +Ġmar row +ĠPrim ordial +C N +Ġlimit less +ĠCarn age +Ġund rafted +q v +IN ESS +on ew +Ġco hesion +98 7 +Ġne cks +Ġfootball er +ĠG ER +Ġdetect able +ĠSupport ing +ĠCS V +oc ally +k Hz +Ġund e +Ġsh one +Ġbud ding +tra k +Stand ing +ĠStar craft +ĠKem p +Ben ch +Ġthw arted +ĠGround s +ath i +L isa +Dial og +ĠS X +V ision +Ġingen ious +Ù IJ +Ġfost ering +ĠZ a +ĠIn gram +Ġ" @ +N aturally +6 16 +0 35 +ĠF AC +H mm +55 4 +Ġacceler ator +ĠV end +Ġsun screen +Ġtuber culosis +rav iolet +ĠFunction al +ĠEr rors +ed ar +19 66 +ĠSpect re +ĠRec ipes +88 5 +ĠM ankind +L iverpool +Ġ| -- +Ġsubst itutes +ĠX T +w ired +Ġinc o +ĠAf gh +E va +ic c +S ong +K night +Ġdilig ently +ĠBroad cast +A id +Ġaf ar +ĠH MS +aton in +ĠGr ateful +Ġfire place +ĠOm ni +e uro +ĠF RE +ĠSh ib +ĠDig est +t oggle +Ġheads ets +Ġdiff usion +ĠSqu irrel +ĠF N +Ġdark ened +out her +Ġsleep s +ĠX er +gun s +Ġset ups +Ġpars ed +Ġmamm oth +ĠCur ious +g ob +ĠFitz patrick +ĠEm il +im ov +........ ..... +ĠB enny +Second ly +Ġheart y +Ġcons on +st ained +Ġgal actic +cl ave +Ġplummet ed +Ġp ests +Ġsw at +Ġrefer rals +ĠLion el +h oly +Ġunder dog +ĠSl ater +ĠProv ide +ĠAm ar +ress or +å Į +ong a +Ġtim id +Ġp iety +ĠD ek +Ġsur ging +az o +Ġ6 10 +Ġdes ks +ĠSp okane +ĠAn field +Ġwars hips +ĠCob ra +Ġar ming +clus ively +ĠBad ge +ag ascar +ĠPR ESS +ĠMcK enzie +ĠFer dinand +burn ing +Af ee +Ġtyr ann +ĠI w +ĠBo one +100 7 +ĠRe pt +Ċ Âł +Ġcar avan +ĠD ill +ĠBundes liga +Ch uck +Ġheal er +ãĥ¼ãĥ Ĩ +ĠH obby +Ġneg ate +Ġcrit iques +section al +mop olitan +Ġd x +Ġouts ourcing +ĠC ipher +t ap +Sh arp +Ġup beat +Ġhang ar +Ġcru ising +ĠNi agara +Ġ3 42 +ill us +ĠS v +Ġsubt itles +Ġsqu ared +Ġbook store +Ġrevolution aries +ĠCarl ton +ab al +Ut ah +Ġdesp ise +ĠU M +cons ider +aid o +Ġc arts +ĠT urtles +Tr aining +Ġhonor ary + ¢ +Ġtri angles +4 22 +Ġreprint ed +Ġgrace ful +ĠMong olia +Ġdisrupt ions +ĠB oh +Ġ3 49 +Ġdr ains +Ġcons ulate +Ġb ends +Ġm afia +ur on +ĠF ulton +m isc +Ġren al +Ġin action +ck ing +Ġphot ons +Ġbru ised +ĠC odes +og i +Ġn ests +ĠLove ly +ĠLib re +ĠD aryl +Ġ# ## +S ys +. ," +Ġfree zes +est ablishment +and owski +Ġcum bers +ĠSt arg +ĠBom bs +Ġleg ions +Ġhand writing +Ġgr un +ĠC ah +sequ ent +Ġm oth +ĠMS M +Ins ert +F if +Ġmot el +Ġdex ter +ĠB ild +hearted ly +Ġpro pe +ĠText ure +ĠJ unction +ynt hesis +oc ard +ĠVer a +ĠBar th +Ġμ g +Ġl ashed +Ġ35 1 +ĠZ amb +ĠSt aples +ĠCort ex +ĠCork er +Ġcontinu um +ĠWR ITE +unt a +rid or +Ġde ems +0 33 +ĠG OLD +p as +Ġrep ressive +ãĥĨ ãĤ£ +Ġbaff led +Sc ar +Ġc rave +Ġ ______ +Ġentrepreneurs hip +ĠDirector ate +Ġ' [ +Ġv ines +Ġasc ended +ĠGR OUP +ĠGood bye +Ġdo gged +ãĥ´ ãĤ¡ +Man ufact +Ġunimagin able +ri ots +ier rez +Ġrel ativity +ĠCraft ing +ra ught +ud en +c ookie +Ġassass ins +Ġdissatisf ied +ac ci +Ġcondu it +Sp read +ĠR ican +n ice +izz le +Ġsc ares +ĠWH Y +ph ans +5 35 +Ġprot racted +ĠKrist en +5 36 +ĠSc rib +ĠNe h +Ġtwent ies +Ġpredic ament +Ġhandc uffs +Ġfruit ful +ĠU L +ĠLud wig +Ġatt est +ĠBre aker +Ġbi ologically +ĠDeal er +Ġrenov ations +f w +ess en +Al ice +ĠHen ri +Ġun ilaterally +ĠS idd +h ai +ĠSt retch +S ales +Ġcumbers ome +ĠJ avier +Ġtrend y +Ġrot ting +ĠChall enges +Ġscra ps +Ġfac ets +ĠVer onica +ĠVer ge +ĠS ana +Al ien +ĠR ih +Ġrad ial +ect ar +Ġ6 30 +cl i +Mar ie +Ġwild fire +ĠCat o +h ander +Ġwait ress +Ġch ops +ĠS ECTION +Ġblunt ly +ĠCat alog +n ian +stud y +Ġpat rolling +ĠT enth +nex us +ĠN ON +op sy +Ġsc athing +s ie +Ġdeterior ated +V B +Naz is +Ġdep ictions +Ġauthent icated +ĠCon ce +k rit +Ġpromul g +ĠL ONG +U FC +ĠVis itors +ĠRec all +Ġrehab ilit +ĠSL I +Ġglac ier +ĠB ite +Ġ50 3 +Ġvom it +Ġfer mented +ĠKh alid +Ġgrad ed +ĠMag icka +ĠIch igo +power ful +ic ators +75 3 +Ġsh rew +Ġ35 6 +Ġlegal izing +Ġall otted +ĠArch demon +ith ing +igg urat +V OL +Le od +Ġo ily +Ġindu cing +Ġamy gdala +Ġadm ins +ĠAcqu isition +C AN +Ġsche matic +Ġmo an +ĠCamer oon +Ġt ink +Ġmer ry +Ġbutter flies +ĠGo ff +Ġworks pace +ĠCor ona +Ġj avascript +ĠD olphin +ĠCant or +4 64 +to e +AP S +ĠAg ing +Ġpadd ed +ĠZ heng +ĠHe ld +Ġest ranged +Ġ7 70 +. } +ĠDun ham +Ġsm okes +Ġcap itals +und ai +Sh in +ĠFound ing +Ġent itle +Ġcenter piece +D iscover +Ġthere to +al ert +ĠN ou +ĠAnaly st +l c +F H +FI ELD +ĠP OV +gr ay +Ġar cs +ĠH OT +Ġr s +Ġoblig atory +ĠArchitect s +ĠS ven +ĠF EC +0 200 +Christ mas +ĠAlban ia +rat om +58 7 +Ġhard ships +Ġaut os +ĠCharg es +Ġap es +Ġ3 76 +wal let +Ġintox ication +Ġgobl in +Ġ5 70 +++++++++ ++++++++ +ĠYel p +ĠMag netic +ĠBr iggs +R ail +Ġspawn s +ĠW iggins +Ġshowc ased +Ġres orted +ub en +Ġwh ipping +Ġim itate +Ġdigest ion +ĠUS PS +ĠG est +Ġye a +ĠT ight +ind al +ic as +` . +C AST +'' ; +ĠF et +opath ic +In valid +Ġregrett ed +Ġbro ccoli +ĠSc ores +e ve +Ġpost ings +Ġaccum ulating +Ġneed less +elf th +Ġmay ors +Ġsc rib +Ġanecd otes +Ġbot ched +ĠRib bon +ĠConstant ine +i uses +ess es +Ġdev ise +Comp ared +Ġp udding +Ġg arg +Ġev oke +79 7 +Ġdet ox +9 09 +ĠPie ces +ĠMcC artney +Ġmet ast +ĠK rypt +P OR +Ġt ending +ĠMerch ants +Pro of +ĠV arg +ĠPort able +ãĥ¼ãĥĨ ãĤ£ +B rain +25 00 +Ġfol iage +Ø ¹ +Ġment ors +ĠA ires +Ġminimal ist +Ġing ested +ĠTro jan +ĠQ ian +inv olved +0 27 +Ġer oded +RA FT +Ġbl urry +M ob +Ġbuff et +ĠFn atic +ae a +KN OWN +ĠIn it +s afety +en um +ACT ION +ĠCrus her +ĠD ates +Ġ ................ +c alling +ak ov +Ġvent ured +Ġ5 55 +au ga +H art +ĠA ero +M AC +Ġthin ly +Ġar ra +ST ATE +ild e +ĠJac qu +ĠFem ales +Ġthe orem +Ġ3 46 +Ġsmart est +ĠPU BLIC +ĠK ron +ĠB its +ĠV essel +ĠTele phone +Ġdec ap +Ġadj unct +ĠS EN +mer ga +Ġred acted +Ġpre historic +Ġexplan atory +ĠRun s +ĠUtt ar +ĠM anny +ĠAUTH OR +ĠUnle ashed +ĠBow ling +be ans +79 3 +Ġunivers es +Ġsens it +ĠK ung +re peat +ctr l +Ġp aced +Ġfull er +Cl ock +Ġrec omb +ĠF aul +ĠB unker +Ġpool ed +Ġan a +ĠM outh +LL OW +hum ane +Ġbull do +ĠMicha els +f am +Ġwreck ed +Ġport rays +ĠWh ale +ĠH es +Ġguess es +ĠBrow se +ĠL APD +Ġconsequ ential +ĠInn ocent +ĠD RAG +Ġtrans gress +ĠO aks +Ġtri via +ĠRes on +ĠA DS +-- + +ĠT oll +Ġgrasp ing +ĠTHE M +ĠT ags +ĠCon clusion +Ġpract icable +Ġho op +Ġunintention ally +Ġign ite +ĠM ov +ur ized +le hem +Ter min +Ġcolour ful +ĠLin ear +ĠEll ie +G y +Ġman power +Ġj s +Ġem oji +ĠSHAR ES +_ . +0000 7 +Ġsophistic ation +Ġunders core +Ġpract ise +Ġbl ob +op ens +Uk raine +Ke eping +Y C +J R +ult imate +Cl aim +Ġautom obiles +99 3 +ste el +Ġpart ing +ĠL ank +... ? +Ġ38 5 +Ġremem brance +Ġe ased +Ġcov ari +ĠS ind +Effect ive +Ġdisse mination +ĠMo ose +ĠCl apper +br ates +App ly +Ġinv is +Ġwors ened +âĢĶ - +Ġlegisl ator +ĠL ol +ĠRow e +Ġdealers hip +um ar +id ences +Ġinvestig ates +Ġc ascade +Ġbid der +ĠB EN +Iron ically +Ġpres iding +Ġd ing +Ġcontrad icted +Ġshut s +ĠF IX +Ġ3 66 +Dist rict +Ġsin ful +ĠChar isma +o ops +Ġtot ality +Ġrest itution +ĠOpt imus +ĠD ah +Ġcl ueless +urn ed +Ġnut rit +Ġland owners +Ġfl ushed +Ġbroad en +m ie +Ġprint ln +Ġn ig +ĠCorp us +J en +Ġprot o +ĠWik imedia +ĠPal o +C OR +Ġstory lines +Ġevangel icals +ĠDar rell +Ġrot or +ĠH W +sk illed +ery l +Ġbe gg +ĠBl umenthal +Ġwe aving +Ġdown wards +ĠJack et +ĠANG EL +Te chnology +Ġes oteric +alde hyde +Ġfur iously +Ġforeign er +We ak +CH O +ĠH ound +Exper ience +ĠPlay station +ĠM IA +ĠU ng +cl oth +ag all +Ġcal ming +iz ens +St ruct +ĠW itches +ĠCeleb ration +Ġ........ ...... +pt roller +ĠTC U +Ġb unny +ãĥ į +ut orial +Ġup scale +ĠSt a +ĠCol ossus +Ġchlor ide +ĠZ ac +ĠRe asons +ĠBrook ings +ĠWH ITE +][ / +ĠL ose +9 05 +Ġunders ide +ern els +Ġv ape +do zen +upp et +ĠST OP +mat ical +ĠStat ements +hed dar +P AC +Custom er +Ġmem os +ĠP J +end ars +ĠLim its +l augh +Ġstabil ized +ĠALE C +Y A +Up grade +al am +Ġtechn o +Ġan ew +fore seen +Ġcolleg iate +ĠPy ro +ĠD ism +Ġfront line +Ġammon ia +I U +Qu ite +John ny +ass in +G OP +ĠSt yles +ĠSovere ign +acter ial +5 49 +ĠR IP +ĠL ists +Ġ3 64 +ĠRece p +s ocket +ĠByr d +ĠCand le +An cient +Ġappell ant +en forcement +ace a +ans ki +Ġold s +88 6 +Ġsl urs +Ġem pires +Ġbuck le +Ġalien ation +ĠAber deen +Ġunic orn +Ġoverr iding +ĠL X +pp a +Ġdesp ised +ĠB ugs +ĠB ST +S outhern +5 33 +Ġhall mark +ĠPost er +Ġstem med +Ġprincip als +ĠT ECH +ĠSand wich +It aly +Ġche esy +ĠSet TextColor +ĠProt ective +ĠC ohn +J O +apt op +Re ason +Lead er +ĠUnder stand +ĠFr idays +ĠContin uous +Ġcl ipping +ĠR ye +Ġber th +tim er +ann is +re act +Ġbuff alo +ĠPar as +Ġ6 55 +Ġpres ided +ĠSun rise +Ġve ts +Ġcl oves +ĠMcC ull +Stre ngth +G AN +Ġill iter +ĠPric ing +l é +Ġresist or +Ġbr un +ĠSuff olk +Ñ ĭ +ĠL iver +Re leased +Ġwhat s +8 60 +ĠMe asures +Ġden ouncing +ĠRy zen +Ġsou ven +Ġcareg ivers +ch ini +ĠScar lett +Ġt rough +Cong ratulations +Ġtax is +ĠTrad ition +j it +Ġtable top +Ġhither to +Ġdis information +off ensive +h ra +ĠDISTR ICT +Ġcompl icate +chen ko +ĠRecon struction +Ġpalp able +Ġa usp +Ġ4 28 +Ġshowc ases +ĠPublic ation +know ledge +inn on +4 19 +Ġretri eval +and ers +Ġref ute +Ġinqu ired +g ur +Ġneg ativity +Ġcons erve +Ġafter life +Ġpres upp +ĠGill espie +Ġm t +ĠD N +T ap +Ġper pend +ĠS my +does n +Ġsp illing +Ġhyp ers +K ate +® , +ke pt +ĠP owered +Ġj a +ĠK lux +ard e +ab an +Ġ4 44 +Ġflatt ened +ĠImprove ments +urg a +ĠK und +Ġins cribed +Ġfac ult +Ġunpre pared +ĠCons umers +Ġsatisf ies +Ġpul monary +Ġinf iltration +Ġex ternally +Ġcongrat ulations +ag han +Ġair liner +Ġfl ung +Ġfly ers +G D +Ġsnipp ets +Ġrec ursive +Ġmaster ing +L ex +Ġovert ly +v g +Ġluck ily +Ġenc ro +ĠLanc et +ĠAbyss al +function al +Ġs ow +Ġsqu id +Ġnar ration +Ġn aughty +ĠHon our +ĠSpart ans +Ġsh atter +ĠTac oma +ĠCal ories +ĠR aces +Sub mit +Ġpurpose fully +w av +ĠY ok +F est +ĠG err +Met ro +Ġit iner +f amous +Ġ" { +in line +was her +Iss ue +ĠCL IENT +oz o +Vers ions +7 25 +ĠGl ock +Ġshield ed +ĠPC R +ENC Y +ĠWe ld +ĠSim pl +Ġredirect ed +ĠK ham +Ġ( > +Ġlab ou +Ġdi apers +ss l +Ġcell ar +organ isms +ore sc +ĠBer ks +did n +Sh ipping +C hest +Ġund one +Ġmillion aire +Ġc ords +ĠYoung er +appropri ately +Ġsequ els +u ve +ant icipated +Ġle wd +ĠSh irt +ĠDmit ry +V eter +Ġsl aying +ĠY ar +Ġcompl ication +I owa +ĠEric a +ĠBL M +g irlfriend +b odied +6 26 +19 63 +Ġintermedi ary +Ġcons olation +M ask +ĠSi em +ow an +Beg inning +Ġfix me +Ġculmin ated +Ġcon duc +ĠVolunte er +Ġpos itional +Ġgre ets +ĠDefin itions +Ġthink er +Ġingen uity +Ġfresh men +ĠMom ents +Ġ35 7 +ate urs +ĠFed Ex +s g +69 4 +Ġdwind ling +ĠBO X +sel age +Ġt mp +Ġst en +ĠS ut +Ġneighbourhood s +Ġclass mate +f ledged +Ġleft ists +Ġclim ates +ATH ER +ĠScy the +ul iffe +Ġs ag +Ġho pped +ĠF t +ĠE ck +ĠC K +ĠDo omsday +k ids +Ġgas ped +Ġmon iker +ĠL od +ĠC FL +t ions +r ums +fol ios +Ġm d +Ġunc anny +Ġtrans ports +ĠLab rador +Ġrail ways +Ġappl iance +ĠCTR L +æ Ģ +Pop ulation +ĠConfeder acy +Ġunb earable +Ġdors al +ĠIn form +op ted +ĠK ILL +Mar x +Ġhypoc ritical +q us +ĠN umerous +ĠGeorg ian +ĠAmbro se +ĠL och +Ġgu bernatorial +ĠX eon +ĠSupp orts +ens er +ee ly +ĠAven ger +19 65 +Ar my +Ġju xtap +Ġcho pping +ĠSpl ash +ĠS ustainable +ĠFin ch +Ġ18 61 +ict ive +at meal +ĠG ohan +Ġlights aber +ĠG PA +ug u +ĠRE PL +vari able +Ġher pes +Ġdesert s +ac iously +Ġsitu ational +week ly +ob l +Ġtext ile +ĠCorn wall +Ġcontrace ptives +ĠA ke +] - +ä¹ ĭ +: , +ĠW em +ĠB ihar +Ġ' . +Ġbe re +Ġanal ogue +ĠCook ies +Ġtake off +Whe el +Ġmaj estic +Ġcomm uting +0 23 +ĠCor pse +ass ment +min i +Ġgor illa +ĠAl as +ere e +Ġacquaint ances +ĠAd vantage +Ġspirit ually +Ġey ed +pm wiki +ĠE nder +Ġtrans lucent +Ġnight time +ĠIM AGES +5 45 +ĠK amp +ĠFre ak +Ġ ig +Port land +4 32 +ĠM ata +Ġmar ines +Ġh ors +ater asu +ĠAtt ribution +Ġ-------- - +Ġk ins +ĠBEL OW +++ + +Ġre eling +ol ed +Ġcl utter +ĠRel ative +Ġ4 27 +B US +Ġa vert +ĠChe ong +ĠA ble +ĠPry or +Develop er +Ġen cyclopedia +ĠUSA F +ĠG arry +Sp ain +Bl ocks +Ġexp osition +ĠGamer Gate +W OR +Ġstockp ile +Ġclot hed +ĠT one +ĠR ue +t umblr +Ġtreacher ous +Ġf rying +Ñ Į +ĠS ph +Ġrest raints +Ġemb odies +ĠG es +S afety +Ġnegoti ators +min ing +ĠAppalach ian +L OS +ĠJenn a +Ġpass ers +ç ĭ +sn ap +Ġshort en +creat or +Ġinn umerable +uther land +67 4 +ĠW OM +ĠAs cend +ĠArm ory +ĠTrans action +K ick +Ġsuit case +day Name +Ġwaste ful +mar riage +ĠMcC abe +ite ch +ĠO ss +Cl osure +ĠTreasure r +Ġindec ent +ĠD ull +Ġresid ences +19 59 +ĠS ettlement +Ham ilton +Ġself ies +ĠRank ing +ĠBark ley +ĠB ore +ĠW CS +ĠMar itime +ĠH uh +ĠForest ry +Ġcultiv ating +ĠBall ard +Ġg arrison +ĠSD L +9 30 +Ġnas cent +Ġirresist ible +Ġaw fully +\/ \/ +Ġequ ate +Ġanthrop ology +ĠSylv ia +Ġintest ine +Ġinnoc uous +cess ive +ag ra +ĠMet roid +G rant +8 55 +ģ ĸ +Ġ" _ +ãĥĥ ãĥī +Ġappra isal +ĠFred dy +04 6 +Ġ40 6 +Ġ18 30 +Ġd ocking +St atic +Ġp ont +ĠVolt age +ĠSt ead +ĠMort gage +ĠJon ah +Y L +CLASS IFIED +Ġas bestos +nik ov +Ġcoll agen +ĠOrb ital +P ocket +7 99 +Ġhy brids +inc hes +Ġinv oice +und y +Ġinequ alities +T rend +w ashed +B ALL +Ġluc id +ĠComment ary +Ġw itty +Br andon +Ġbru ising +Ġ6 20 +es cent +box ing +P OL +Ġ3 78 +R ect +Ġlic ences +ĠMcG ee +p ressed +D anny +Ġj ammed +ord inate +Ġle th +Ġdistingu ishes +ĠYam aha +IL S +ĠH ume +ĠC ategories +Rober ts +Ch art +Ġbeet le +ĠGra veyard +Ġ($ ) +o ÄŁ +Ġtw ilight +are lla +á ½ +Ġbooth s +ĠH HS +ĠFeld man +Ġexcav ation +Ġphilosoph ies +at ography +ĠGar age +te chnology +Ġunfor gettable +Ġver ifying +Ġsubord inates +E ls +Ġne b +G aming +EN A +ĠAchieve ment +it ters +ĠG abe +Ġd umps +for cer +Ġpo ignant +ĠM BA +ĠHe idi +ime i +Ġm ages +Ġliber ate +Ġcircum cised +ĠMer maid +ĠMat th +t ogether +ĠW ichita +Ġstore front +ĠAd in +V II +Four th +Ġexplore rs +W ER +Not able +Bro ok +m ens +F aith +-------- - +ĠJ ou +¬ ¼ +Ġpine apple +Ġam alg +el n +ark able +ĠãĤµ ãĥ¼ãĥĨãĤ£ +ĠãĤµãĥ¼ãĥĨãĤ£ ãĥ¯ãĥ³ +Ġov arian +ĠE choes +Ġhairc ut +Ġp av +Ġch illed +anas ia +Ġsty led +Ġd ab +ni per +Ġminister ial +ĠD UP +T an +Ġsul ph +ĠD eter +ĠBo hem +od an +Ġeduc ator +â ĵĺ +sp ir +Ch icken +ĠE leanor +Ġqu i +Ġheav iest +Ġgrasp ed +U RA +Ġcro oked +Jess ica +pro blem +Ġpred etermined +Ġman iac +Ġbreath s +ĠLauder dale +Ġh obbies +y z +Cr ime +Ġcharism a +d L +Ġle aping +Ġk ittens +Ang elo +ĠJ ACK +ĠSu zanne +Ġhal ting +ENT ION +Ġswall owing +ĠEarthqu ake +Ġeight eenth +ĠN IC +ĠIN F +ĠCons cious +Ġparticular s +circ le +7 40 +Ġbene volent +Ġ7 47 +Ġ4 90 +Ġr undown +ĠVal erie +ĠB UR +Ġcivil isation +ĠS chn +W B +ot ide +intern ational +Ġj ohn +Ġ19 02 +Ġpe anuts +Ġflav ored +k us +Ġro ared +Ġcut off +é £ +Ġorn ament +Ġarchitect ures +Ġ3 69 +ol or +ĠWild e +ĠC RC +ĠAdjust ed +Ġprov oking +land ish +Ġrational ity +Ġjust ifies +Ġdisp el +Ġa meric +ĠPol es +Ø © +Ġen vis +ĠD oodle +ä½ ¿ +igs aw +auld ron +Techn ical +T een +up hem +ĠX iang +Ġdetract ors +ĠZ i +ĠJournal ists +Ġconduc ive +ĠVolunte ers +Ġs d +Know ing +Ġtrans missions +ĠPL AN +ĠL IB +Ġall uded +Ġob e +Ġd ope +ĠGold stein +Ġwavelength s +ĠDest ination +nd a +ug i +Ġattent ive +ĠLe an +ral tar +Ġman g +mb uds +ak ings +b ender +Ġacc ol +Ġcraw led +N OW +Min nesota +Ġflour ished +ĠZ up +ĠSuper visor +ĠOliv ier +Ex cellent +Ġwid en +D one +Ġw ig +Ġmiscon ceptions +Cor p +W an +Ġvener able +ĠNot ably +ĠKling on +an imate +Bo ost +ĠS AY +miss ing +ibli ography +mel on +Ġpay day +Ø ³ +bo le +Ġve iled +ĠAl phabet +It alian +Ġever lasting +ĠR IS +ĠC ree +rom pt +Ġh ating +Ġgrin ning +Ġge ographically +OS H +Ġwe eping +ĠÂłĠÂłĠÂłĠÂł ĠÂłĠÂłĠÂłĠÂł +Ġimpe cc +Let ter +Ġblo ated +PL A +ĠFe in +Ġper sever +Th under +Ġa ur +ĠR L +Ġpit falls +âĸ º +Ġpredomin ant +Ġ5 25 +7 18 +AP E +7 14 +Ġfarm land +ĠQ iao +Ġv iolet +ĠBah amas +Ġinflic ting +ĠE fficiency +Ġhome brew +Ġundert ook +Ġcur ly +ĠHard ing +man ia +59 6 +Ġtem pered +Ġhar rowing +ĠP ledge +ĠFranken stein +è ª +M otion +Ġpredict ably +ĠExpl osion +oc using +er d +col o +FF ER +Ġback field +ĠV IDE +ue bl +N arr +ĠArg ument +Ġgen omic +Ġbout ique +Ġbatt ed +ĠB inary +Ġg amb +ĠRh ythm +67 3 +Ġa float +ĠOlymp ia +Y ING +Ġend if +is in +Ġwin ters +Ġsc attering +I v +D istance +Ġtr u +ĠCom fort +Ġne xus +Ġair flow +ĠByz antine +p ayers +con i +ĠB etsy +D eal +ĠN ug +ĠContin ent +red ibly +Ġoptim izing +al beit +Ġec static +ĠPro to +ç · +iv ot +âĸ Ħ +em p +rou nder +Ġcl out +ĠI ST +66 3 +ĠDoll ars +ĠD AC +Ġsubsc ribed +Ġrehears al +Ġam ps +ĠSh ang +es m +Ġspr inkle +Ġassail ant +ĠO o +ĠCoin base +T act +Ġret ina +Ġn uns +R ON +att o +Ġj ug +ĠSV G +Ġb ikini +ĠFI LE +ĠFound ers +ep ort +ĠK P +Ġrest ores +ĠTh ick +Ġash ore +Ġappro vals +R ender +M AG +G raham +ĠCort ana +ãĥ³ ãĤ¸ +ss h +or ians +ars ity +ĠInsp ired +u pper +Ġsign alling +Ġreb uke +Ġfl ares +Ġdownt ime +Stud ies +Ġstagn ation +ĠSequ ence +Ġgr unt +Ġass ures +ĠPL A +59 2 +Ġintra ven +d epend +Sus an +ĠManz iel +Man ia +Cont ract +Ġsl ams +Ġcult ured +Ġcred itor +L IST +ĠH UM +ĠChatt anooga +serv ed +Ġclo aked +ĠF TP +p owder +ĠSt ella +uct ive +Ġcheap ly +ĠMU CH +ĠGalile o +Ġsu ites +spe ech +Ġdeliber ations +ĠCh ips +« ĺ +Bal ance +ĠWyn ne +ĠAk ron +Ass et +Ġhon oured +Ġed ged +Like wise +anim ous +ĠW age +ĠEz ek +ad vertisement +ĠRT X +ĠM AD +Ġmigr ating +ĠS QU +Ġ4 75 +Ed ited +Ġshorth and +ĠBas ics +Ġcro tch +ĠEV EN +Ġv m +effic iency +Ġcal ves +ĠF rie +ĠBrill iant +Ġstri kers +Ġrepent ance +Ġarter ies +r l +B ed +h ap +Ġcrypt ography +ĠSab res +Ġ4 14 +vi ks +ih ara +aps es +T alking +Ġintertw ined +Ġdoc ks +Ġalle le +ĠArt ifact +ĠH IM +t orn +ç ķ +Ġop acity +ĠE ly +os uke +Ġn ipple +Ġhand written +ĠV K +ĠChamber lain +ĠLa os +ig raph +g row +Ġtr illions +Ġdescend ant +ĠSail or +as uring +Ġce ilings +ĠWare house +f lying +ĠGl ow +Ġn ont +Ġmiscar riage +Ġrig s +Ġmin istries +Ġelabor ated +Ġdel usional +ĠHum ane +Ġ3 79 +n ets +Ġblack out +add ers +Ġn p +ĠT ire +ro sc +Ġsub div +Ġlink age +Ġchron ological +ĠHER O +Ġres ettlement +ĠVin yl +Ġpast oral +ĠMob il +ĠBar bar +Co oldown +ĠF ritz +c riminal +re pe +Ġbell ig +ĠBre ed +Ġ4 18 +Ġsem blance +ij k +Ġcur tail +Ġclin ch +cont ained +ĠProm pt +ast on +Ġw i +Ġpursu its +5 15 +ĠGl oss +Ġfl ips +Ġcoup ons +Ġcl oning +ĠLike ly +Rem oved +ĠQu artz +r ices +ĠSpe ars +Ġp ious +Ġdep reciation +ĠD are +oun ces +am az +O nt +Ġp innacle +d ocker +0 26 +ĠW yr +ĠPro per +Ë Ī +n il +By tes +Ġseek er +t rial +Ġunf olds +ĠMar se +Ġextravag ant +ĠSurviv ors +RED ACTED +ĠSpeed way +ĠCra igslist +sub mit +ĠGener ations +Ġup holding +Ġblood stream +ĠMiss ions +ĠL awn +Ġlim bo +ene i +H uh +ĠWild cats +pre p +ĠMark us +ĠFor bidden +rit ic +IN O +Ġexhib iting +requ ent +ch uk +Ġhabit ual +ĠComp atibility +Dr ag +RIP T +uj ah +GR OUND +Ġdelinqu ent +Ġburn er +Ġcontempor aries +Ġgimm ick +load s +Ġno zzle +p odcast +ĠW ak +ĠStat en +ĠK uh +ãģ ĵ +inter rupted +Ġinv incible +ĠBurn ett +cig arette +ĠPeb ble +ĠTem porary +ĠMar ino +58 2 +Ġwast eland +ident ly +T x +Ġr ite +ĠPan asonic +ĠM iddles +ĠHort on +ae us +Ġc uring +Ġm ats +Ġadj ourn +Ġfears ome +pe z +bo ats +Ġpro pell +Ġconflic ted +ĠAng er +Ġinsurg ent +K arl +Ġco ales +Ġsouth western +Ġdis su +ĠO vert +******** **** +Ġbox ed +ĠBr une +aa a +Ġgard ening +ĠEng el +tr acks +Ġpur ified +Ġplace holder +ĠL ikes +Ġd an +G ab +Ġe ct +ĠF aw +ĠEl iot +Ġ' , +otrop ic +ĠRu in +hed on +Ġca ul +Ġa ft +ĠCad illac +gh a +ass ian +ud eb +ĠT ick +Ġadjust s +AR GET +5 37 +isc he +ant y +ĠFried rich +ĠBl izz +ĠA OL +Camp aign +Ġmamm al +ĠVe il +ĠK ev +ĠMaur it +ĠDam ien +N ation +E astern +Ġ{ : +Ġ= ================================ +Ġstereotyp ical +Ġatt ic +ĠCy borg +requ ire +Ġaward ing +ĠPap ua +bt n +b ent +B oo +Ġ( = +ĠX ander +ĠSomers et +Ġcatch y +Ġcert ify +STR UCT +Ġit al +Ġt ides +ĠBr ands +G ray +comp etitive +Ġcur ator +ĠD G +omin ium +ĠGM Os +ci ating +ĠCarm en +ow ard +Balt imore +Ġr gb +C u +Ġwip es +spe ll +IT NESS +Ġsummar izes +ĠRe vis +Ġwhistlebl owers +ĠBre ach +Ġcro chet +k os +ews ki +Ġrep et +Ġcrim son +ĠKar achi +read able +dim ension +ĠI gor +ild ed +ĠZ ed +ĠKe ane +ĠCos metic +DE P +Ġretreat ing +ĠU A +ens ical +Ġd usk +ĠDick ens +Ġaren as +ĠPass age +level s +Ġcur v +P ope +Ġch ores +ĠEl ise +ĠComp ass +b ub +Ġmamm alian +ĠSans krit +ĠAN C +ĠCr ack +Q ual +L aun +amp unk +Ġlearn ers +Ġglam orous +Ġfur the +erm ott +c and +Gener ic +Ġnarr ated +Ġdisorder ly +ĠTrans actions +ĠDet ention +ĠR oku +Ä į +Ġunder statement +ĠS aur +ĠRodrig o +ĠAS AP +S in +Ġre joice +Method s +Ġelectro de +Ġworsh ipped +Ġid i +ĠPhys icians +Ġpop up +Ġde ft +ĠRem oval +ĠBu enos +ver bs +Ġfun k +ush a +rict ion +ore a +ĠBang alore +ĠKen obi +zz i +Ġnorm ative +Ġgobl ins +Ġcaf es +ĠUN CLASSIFIED +ĠF ired +S IGN +Ġs clerosis +ĠV oter +ĠSon ny +ĠExt end +ĠEV s +Ar senal +Ġp si +Ġwid est +ĠT us +Ġlo oms +Ġjust ifying +ĠGr anger +è ¯ +Ref er +58 3 +Ġflour ishing +ab re +Ġr ave +ĠCont ra +Ġ18 98 +Add s +Ġf ul +ĠCo oke +some one += # +67 1 +Ġy ak +Ġar te +ĠMis cellaneous +ĠDet ection +ĠCl ancy +â ģ +ass ies +Ġval iant +ĠFemin ist +cor ruption +V el +P ear +Ġsucc inct +Ġquick est +k w +Ġsp itting +ĠL ibraries +åħ ī +ant z +D ad +ĠSpec ifications +rup ulous +and r +RES ULTS +Ġsnow ball +Ġpred is +ĠB axter +ĠNurs ing +ĠCh aff +s we +Ġout age +Ġnest ing +Ġnotor iety +tr igger +on ite +j on +Ġf ou +ook ed +ĠCelebr ity +re ality +Ġfat ig +Ġhug ging +Ġbother s +ĠPan zer +ĠCh andra +fig ured +Ġvol ts +ĠCloud s +Ġfee ble +ĠCur ve +ĠAs us +78 6 +abs or +ĠV ICE +ĠH ess +Ġmanufact ures +Ġgri zz +ĠPower ful +ac id +Ġsub sections +ĠKrug man +ĠAl ps +is u +Ġsequ est +ĠUlt ron +ĠT inker +ĠGo ose +Ġmism atch +Att orney +Ġmorph ology +ĠSix ers +ut tered +ĠE LECT +gr an +Rus sell +ĠG SL +Ġfort night +Ġ. ) +Ġapost le +pr one +el ist +Unt itled +ĠIm plementation +ist ors +Ġtank er +Ġpl ush +Ġattend ants +ĠT ik +ĠGreen wich +ĠY on +ĠSP L +cell s +unt led +S olution +ĠQu é +Ġvac ated +Ġupt ick +ĠMer idian +æ ĥ +ĠDr ill +9 25 +58 4 +Ġrenov ated +ĠKub rick +zy k +Ġl ousy +pp el +ohyd rate +ĠI zzy +lesi astical +CC C +ĠAj ax +Ġad apters +ĠPetra eus +Ġaffirm ation +ĠST OR +le ms +ad oes +ĠConstantin ople +Ġp onies +Ġl ighthouse +Ġadherent s +ĠBre es +omorph ic +Fight ing +Ġpl aster +ĠP VC +ĠOb st +Ġdear ly +ĠTo oth +icks on +Ġsh aming +P lex +A gg +ĠâĢ¦ " +Ġsub reddits +Ġpige on +ĠResident ial +ĠPass ing +Ġl um +ĠP ension +Ġpessim istic +Ġ4 32 +z inski +c ade +0 75 +Ġapolog ised +iy ah +Put ting +Ġgloom y +ĠLy me +=-=-=-=- =-=-=-=- +ĠT ome +ĠPsych iatric +ĠH IT +c ms +ap olog +Ġbreak er +Ġdeep en +Ġtheor ist +ĠHigh lands +Ġb aker +Ġst aples +Ġinterf ered +ĠAb ortion +jo ined +ch u +Ġform ulate +Ġvacc inations +Ġban ter +phe us +Ġoutfield er +ĠM eter +Ġ# #### +Ġ18 95 +Ġnarrow ing +ĠST ORY +f p +ĠC ST +ign ore +Ġproclaim ing +ĠR U +ĠB ALL +yn a +65 3 +Ġpos it +P RE +59 4 +ĠRegist rar +ĠPil grim +ic io +Ġpre tt +Ġlif eless +Ġ__ _ +Ne igh +ĠCh urches +orn o +Ġor cs +Ġkind red +ĠAud it +Ġmillenn ial +ĠPers ia +g ravity +ĠDis ability +ĠD ARK +W s +od on +Ġgrand daughter +ĠBro oke +ĠA DA +ER A +Ġpick ups +ĠWil kinson +ĠSh ards +ĠN K +Ġexp el +ĠKis lyak +Ġj argon +Ġpolar ized +ian e +Pub lisher +Ġreb utt +Ġapprehens ion +ĠK essler +Ġpr ism +F UL +19 64 +ĠL oll +ä ¿ +le thal +Å Ł +Ġg hetto +Ġb oulder +ĠSlow ly +ĠOsc ars +ĠInst ruction +ĠUl tr +ĠM oe +N ich +ĠP ATH +( * +ĠRE LEASE +un ing +rou se +en eg +Ġre imb +ĠDet ected +Do S +Ġster ling +Ġaggreg ation +ĠLone ly +ĠAtt end +hig her +Ġairst rike +ks on +SE LECT +Ġdef lation +ĠHer rera +C ole +rit ch +Ġadvis able +F ax +Ġwork around +Ġp id +mort em +ers en +Ġtyp o +Ġal um +78 2 +ĠJam al +script s +Ġcapt ives +ĠPres ence +ĠLie berman +angel o +Ġalcohol ism +ass i +Ġrec ite +Ġgap ing +Ġbask ets +ĠG ou +Brow ser +ne au +Ġcorrect ive +und a +sc oring +ĠX D +Ġfil ament +Ġdeep ening +ĠStain less +Int eger +Ġbu ggy +Ġten ancy +ĠMub arak +Ġt uple +ĠD roid +ĠS itting +Ġforfe it +ĠRasm ussen +ixt ies +es i +ĠKim mel +Ġmetic ulously +Ġap opt +ĠS eller +08 8 +ec ake +hem atically +T N +Ġmind less +Ġdig s +ĠAcc ord +ons ense +em ing +br ace +Ġe Book +ĠDist ribut +ĠInvest ments +w t +] ), +beh avior +56 3 +Ġbl inding +ĠPro testers +top ia +Ġreb orn +ĠKel vin +ĠDo ver +ĠD airy +ĠOut s +Ġ[ / +Ï Ģ +b p +ĠVan ity +ĠRec ap +ĠHOU SE +ĠF ACE +Ġ4 22 +69 2 +ĠAnt ioch +cook ed +Ġcoll ide +Ġa pr +Ġsle eper +ĠJar vis +Ġalternative ly +ĠLe aves +ĠM aw +Ġantiqu ity +ĠAdin ida +Ġab user +Poké mon +Ġass orted +ĠRev ision +ĠP iano +ĠG ideon +O cean +Ġsal on +Ġbust ling +ogn itive +ĠRah man +Ġwa iter +Ġpres ets +ĠO sh +ĠG HC +oper ator +Ġrept iles +Ġ4 13 +ĠG arr +ĠCh ak +Ġhas hes +Ġfail ings +Ġfolk lore +Ġab l +ĠC ena +ĠMac Arthur +ĠCOUR T +Ġperipher y +app ers +Ġreck oned +ĠInf lu +ĠC ET +Ġ3 72 +ĠDefin itive +ass ault +4 21 +Ġreservoir s +Ġd ives +ĠCo il +DA Q +Ġvivid ly +ĠR J +ĠBel lev +Ġec lectic +ĠShow down +ĠK M +ip ed +reet ings +ĠAs uka +L iberal +ĠÏ Ħ +Ġbystand ers +ĠGood win +uk ong +S it +ĠT rem +Ġcrim inally +ĠCirc us +ch rome +88 7 +Ġnan op +ĠOb i +ĠL OW +o gh +ĠAuth ors +ob yl +Ur ban +Ġt i +ĠWe ir +t rap +ag y +Ġparent heses +Ġout numbered +Ġcounter productive +ĠTob ias +ub is +P arser +ST AR +Ġsyn aptic +ĠG ears +Ġh iber +Ġdebunk ed +Ġex alted +aw atts +H OU +Ch urch +ĠPix ie +ĠU ri +ĠForm ation +ĠPred iction +C EO +Ġthro tt +ĠBrit ann +ĠMad agascar +ë ĭ +Ġbill boards +ĠRPG s +ĠBe es +complete ly +F IL +Ġdoes nt +ĠGreen berg +re ys +Ġsl ing +Ġempt ied +ĠPix ar +ĠDh arma +l uck +ingu ished +Ġend ot +Ġbab ys +05 9 +che st +r ats +Ġr idden +Ġbeet les +Ġillum inating +Ġfict itious +ĠProv incial +Ġ7 68 +Ġshe pherd +ĠR ender +Ġ18 96 +C rew +Ġmold ed +ĠXia omi +ĠSp iral +Ġdel im +Ġorgan ising +Ġho ops +ĠBe i +z hen +Ġfuck in +Ġdec ad +Ġun biased +am my +sw ing +Ġsmugg led +Ġk ios +ĠP ERSON +ĠInquis itor +Ġsnow y +Ġscrap ing +ĠBurg ess +P tr +ag ame +R W +Ġdro id +ĠL ys +ĠCass andra +Jac ob +Ġ35 4 +Ġpast ure +Ġfr anc +ĠScot ch +ĠEnd s +ĠI GF +def inition +Ġhyster ical +ĠBrown e +77 1 +Ġmobil ization +æ ķ +iqu eness +Th or +Ġspear headed +Ġembro iled +Ġconject ure +jud icial +Ch oice +Ġpaper back +P ir +Ġrec overs +ĠSur ge +ĠSh ogun +ĠPed iatrics +ãģ ł +Ġsweep s +ĠLabor atories +ĠP acks +al us +add in +Ġhead lights +g ra +Ev idence +COL OR +Ad min +Ĭ ± +Ġconco ct +s ufficient +Ġun marked +Ġrich ness +Ġdiss ertation +Ġseason ing +Ġg ib +ĠM ages +un ctions +ĠN id +che at +ĠTM Z +c itizens +ĠCatholic ism +n b +Ġdisemb ark +ĠPROG RAM +a ques +Ty ler +Or g +ĠSl ay +ĠN ero +ĠTown send +IN TON +te le +Ġmes mer +9 01 +Ġfire ball +ev idence +aff iliated +ĠFrench man +ĠAugust a +0 21 +Ġs led +Ġre used +ĠImmun ity +Ġwrest le +assemb led +Mar ia +Ġgun shots +ĠBarb ie +Ġcannabin oids +ĠTo ast +ĠK inder +IR D +Ġre juven +Ġg ore +Ġrupt ure +Ġbre aching +ĠCart oon +Ġ4 55 +ĠPale o +6 14 +Ġspe ars +ĠAm es +ab us +Mad ison +GR OUP +Ġab orted +y ah +Ġfel on +Ġcaus ation +Ġprep aid +Ġp itted +op lan +ĠShel ley +ĠRus so +ĠP agan +Ġwill fully +ĠCan aver +und rum +ĠSal ary +ĠAr paio +read er +ĠR ational +ĠOver se +ĠCa uses +Ġ* . +Ġw ob +Ke ith +ĠCons ent +man ac +77 3 +6 23 +Ġfate ful +et imes +Ġspir ited +ĠD ys +Ġhe gemony +Ġboy cot +ĠEn rique +em outh +Ġtim elines +ĠSah ara +ĠRel ax +ĠQuin cy +ĠLess ons +ĠE QU +SE A +N K +ĠCost co +Incre ase +Ġmotiv ating +ĠCh ong +am aru +ĠDiv ide +Ġped igree +ĠTasman ia +ĠPrel ude +L as +9 40 +57 4 +Ġch au +ĠSp iegel +un ic +-- > +ĠPhil ips +ĠKaf ka +Ġuphe aval +Ġsent imental +Ġsa x +ĠAk ira +ser ial +Mat rix +Ġelect ing +Ġcomment er +ĠNeb ula +ple ts +ĠNad u +ĠAd ren +Ġen shr +ĠR AND +fin ancial +ĠCly de +uther ford +Ġsign age +Ġde line +Ġphosph ate +rovers ial +f ascist +ĠV all +ĠBeth lehem +Ġfor s +Ġeng lish +S olid +N ature +Ġv a +ĠGu ests +Ġtant al +Ġauto immune +;;;;;;;; ;;;; +ĠTot ally +ĠO v +Ġdef ences +ĠCoc onut +Ġtranqu il +Ġpl oy +Ġflav ours +ĠFl ask +ãĤ¨ ãĥ« +ĠWest on +ĠVol vo +8 70 +Ġmicro phones +ver bal +R PG +Ġi ii +; } +0 28 +Ġhead lined +Ġprim ed +Ġho ard +ĠSh ad +ĠEN TER +Ġtri angular +Ġcap it +l ik +ĠAn cients +Ġl ash +Ġconv ol +Ġcolon el +en emy +G ra +Ġpub s +ut ters +Ġassign s +ĠPen et +ĠMon strous +ĠBow en +il ver +H aunted +ĠD ing +start ed +pl in +Ġcontamin ants +ĠDO E +ff en +ĠTechn ician +R y +Ġrob bers +Ġhot line +ĠGuard iola +ĠKau fman +row er +ĠDres den +ĠAl pine +E lf +Ġf mt +ĠS ard +urs es +g pu +Un ix +Ġunequiv ocally +ĠCitizens hip +qu ad +m ire +ĠS weeney +B attery +6 15 +Ġpanc akes +Ġo ats +M aps +ĠCont rast +mbuds man +ĠE PS +Ġsub committee +Ġsour cing +Ġs izing +ĠBuff er +ĠMand atory +Ġmoder ates +ĠPattern s +ĠCh ocobo +ĠZ an +ĠSTAT ES +ĠJud ging +ĠIn her +* : +Ġb il +ĠY en +Ġexh ilar +oll ower +z ers +Ġsn ug +max imum +Ġdesp icable +ĠP ACK +ĠAn nex +Ġsarcast ic +Ġlate x +Ġt amp +ĠS ao +b ah +ĠRe verend +ĠChin atown +ĠA UT +d ocumented +ĠGA BA +ĠCan aan +ĠÙ ħ +Ġgovern s +pre v +E sc +ĠEst imates +OS P +Ġendeav our +ĠCl osing +omet ime +every one +Ġwor sen +Ġsc anners +Ġdev iations +ĠRobot ics +ĠCom pton +Ġsorce rer +Ġend ogenous +Ġem ulation +ĠPier cing +ĠA ph +ĠS ocket +Ġb ould +ĠO U +ĠBorder lands +Ġ18 63 +G ordon +ĠW TO +Ġrestrict s +Ġmosa ic +Ġmel odies +ç Ħ +T ar +Ġdis son +ĠProv ides +Ġ ...... +b ek +F IX +Ġbro om +ans hip +Do ctors +Ġner ds +ĠReg ions +na issance +Ġmet e +Ġcre pt +pl ings +Ġgirlfriend s +kn it +ig ent +ow e +Ġus hered +ĠB az +M obil +4 34 +ĠPres ents +orig in +Ġins omnia +ĠA ux +4 39 +ĠCh ili +irs ch +G AME +Ġgest ation +alg ia +rom ising +$ , +c row +ĠIn spection +at omic +Rel ations +J OHN +rom an +ĠClock work +ĠBak r +m one +M ET +Ġthirst y +Ġb c +Ġfacult ies +R um +Ġnu ance +ĠD arius +ple ting +fter s +etch up +Reg istration +ĠK E +R ah +Ġpref erential +ĠL ash +ĠH H +Val id +ĠN AV +Ġstar ve +ĠG ong +z ynski +ĠAct ress +Ġw ik +Ġun accompanied +lv l +Br ide +AD S +ĠCommand o +ĠVaugh n +Wal let +Ġho pping +ĠV ie +Ġcave ats +Ġal as +if led +ab use +66 1 +Ġib n +Ġg ul +Ġrob bing +t il +IL A +Ġmit igating +Ġapt ly +Ġty rant +Ġmid day +ĠGil more +ĠDe cker +Ġ§ § +part ial +Ex actly +Ġphen otype +Ġ[+ ] +ĠP lex +ĠI ps +vers ions +Ġe book +Ġch ic +g ross +":" "},{" +ĠSur prisingly +M organ +Ġresid ues +ĠConf ederation +in feld +Ġl yr +mod erate +Ġperpend icular +V K +Ġsynchron ized +Ġrefres hed +Ġad ore +ĠTor ment +ol ina +Ġ26 00 +Item Tracker +Ġp ies +ĠF AT +ĠR HP +0 48 +ĠRES P +ĠB J +all ows +P and +Ġunw elcome +ĠV oc +ĠBast ard +ĠO W +ĠL AR +ĠHeal er +Environment al +ĠKen yan +ĠTr ance +ĠP ats +Ġali ases +ĠGar field +Ġcampaign er +Ġadvance ments +ĠOkin awa +ĠC oh +ows ky +Ġstar ved +Ġsize able +Ġ: -) +Ġm RNA +Ġsusp ensions +ist ar +Scot land +Pr in +-------------------------------- ---------------- +Ġ50 2 +Ġteasp oons +Ġ10 50 +Ġcoerc ive +ĠMason ic +edd ed +ĠPass enger +Ġl att +Ġbr aces +ĠSt eal +ĠNY T +ĠK ats +ĠCel est +ae z +T u +ĠCoul ter +ðŁ ĺ +Fl ickr +ĠWil mington +ith s +++ ; +Ġv ending +Ġneg ro +ĠPh i +ĠYellow stone +Call back +Ġsh ampoo +ĠSh ades +w at +Ġsuper human +Ġridic uled +Ġhol iest +om bo +Ġintern s +Ġh one +ĠPar agu +UR I +Ġd angling +ãĤ » +so v +ict ional +av ailability +Ġrev ocation +Ġd ow +in ic +ĠTHE IR +Ġis o +Ġout ings +ĠLeth al +Ġ) )) +Ġinacc ur +Ġout landish +Ġan us +let ico +id on +l ol +Ġun regulated +Ġsuccumb ed +Ġc uff +ĠWast eland +let al +Ġsub str +Ġcoff ers +Ġautom akers +ov i +ĠX ue +ĠDayton a +Ġjar ring +Ġf umes +Ġdisband ed +z ik +itt on +Ġstriking ly +Ġsp ores +Ad apter +.) : +ĠLynd on +ival ry +Ġor ally +Ġtumult uous +Ġdisple asure +Ġcon es +or rect +Ġappe ase +Ġder by +ĠTrip oli +ĠAl ess +Ġp oked +ĠGu ilty +v P +En ough +Ġorig inals +6 99 +Ġrabb i +Ġproverb ial +Ġpostp one +el ope +ĠMist y +Ġstaff ed +ĠUn employment +redit ary +Ġdilig ent +re comm +me asures +as in +8 25 +Ġpond s +Ġmm ol +ĠS AR +ĠC ARE +Ġ3 71 +Ġclen ched +ĠCors air +Ġcaric ature +z n +att ach +ĠSch ro +spe ak +p ainted +ĠS uc +ĠE NT +Ġcell ul +ĠP aid +di agn +WH ERE +Ġtext ed +B arn +Ġret racted +ĠRe ferred +S av +Ġup keep +Ġwork places +ĠTok ens +Ġampl ify +cl inical +Ġmult ic +mber g +Ġconvol uted +Reg ion +5 65 +ĠTop ic +Ġsn ail +Ġsal ine +Ġins urrection +ĠPet r +f orts +B AT +ĠNav ajo +Ġrud imentary +ĠLak sh +OND ON +Me asure +Ġtransform er +ĠGodd ard +Ġcoinc ides +ir in +R ex +ĠB ok +qu it +Ġshotgun s +Ġprolet arian +Ġsc orp +ĠAd a +5 14 +Ġsl ander +record ed +Ġemb ell +ris ome +Ġapolog izing +ĠMul cair +ĠGib raltar +Cl a +Ġall ot +ĠAtt ention +Ġ4 33 +le ave +Ġwh ine +ĠIss a +ĠFa ust +ĠBar ron +hen y +Ġvictim ized +J ews +Ġnurt uring +ett el +W inged +ĠSub tle +Ġflavor ful +ĠRep s +eng ed +call back +Ġdirection al +Ġcl asp +ĠDirect ions +plan et +icult ure +Hel per +ic ion +ac ia +Ġç ¥ŀ +Ġsur ges +Ġcan oe +ĠPrem iership +be en +Ġdef ied +ĠTro oper +Ġtrip od +Ġgas p +ĠE uph +ĠAd s +vern ight +high ly +R ole +Ġent angled +ĠZe it +6 18 +ĠRust y +Ġhaven s +ĠVaugh an +HA EL +ĠSER VICE +/ , +Ġstr icken +Ġdel usions +Ġb is +ĠH af +Ġgrat ification +Ġent icing +UN CH +Ad ams +ĠOL ED +ĠBeet le +Ġ18 99 +ĠSO FTWARE +ateg or +V L +ĠTot em +ĠG ators +AT URES +Ġimped ance +Reg istered +ĠC ary +ĠAer ial +on ne +en ium +Ġd red +ĠBe g +Ġconcurrent ly +Ġsuper power +ĠX an +j ew +imes ter +ĠDick inson +âĶ ģ +F la +Ġp ree +ĠRoll ins +© ¶æ +Ġden omination +ĠL ana +5 16 +Ġinc iting +sc ribed +j uries +ĠWond ers +app roximately +Ġsusp ending +Ġmountain ous +ĠL augh +oid al +N s +Det ect +) = +ĠL uthor +ĠSchwarz enegger +ĠMull er +ĠDev i +ec ycle +J ar +6 13 +ĠL ongh +B ah +ĠSP ORTS +n w +Ġref inement +Ġwater ways +Ġd iner +Bl ade +68 3 +F ac +Ġinitial s +Ġro g +Ġparan ormal +B UT +Ġ[ ( +ĠSw anson +ĠM esh +âĸ ¬ +Impro ve +ĠRad iation +ĠEst her +ĠE sk +ĠA ly +ik y +Ġir rad +ĠBuck ingham +Ġref ill +Ġ. _ +Re pe +CON CLUS +Ġdifferent iated +Ġchi rop +ĠAt kins +Pat tern +Ġexc ise +Ġcab al +N SA +ĠST A +ĠS IL +ĠPar aly +Ġr ye +ĠHow ell +ĠCount down +ness es +alys ed +Ġres ize +ãĤ ½ +Ġbudget ary +ĠStr as +w ang +Ġap iece +Ġprecinct s +Ġpe ach +Ġsky line +Ġ35 3 +pop ular +App earances +ĠMechan ics +ĠDev Online +S ullivan +Z en +Ġp u +op olis +5 44 +Ġde form +Ġcounter act +ĠL ange +Ġ4 17 +Con sole +77 4 +Ġnodd ing +Ġpopul ism +Ġhe p +Ġcoun selling +compl iance +U FF +Ġunden iably +Ġrail ing +ĠHor owitz +ĠSim one +ĠBung ie +Ġa k +ĠTal ks +x ff +fl ake +Cr ash +Ġsweat y +Ġban quet +ĠOFF IC +Ġinvent ive +Ġastron omer +ĠStam ford +ĠSc are +ĠGRE EN +olic ited +Ġr usher +Ġcent rist +ight ing +Ġsub class +Ġdis av +Ġdef und +ĠN anto +oci ate +m ast +Ġpac if +Ġm end +e ers +imm igration +ESS ION +Ġnumber ing +Ġlaugh able +ĠEnd ed +v iation +em ark +P itt +Ġmetic ulous +ĠL F +Ġcongrat ulated +ĠBir ch +Ġsway ed +Ġsemif inals +Ġhum ankind +m atter +ĠEqu ip +opa usal +S aid +ĠLay out +Ġvo icing +Ġth ug +Ġporn ographic +I PS +Ġmo aning +Ġgriev ance +Ġconf essions +esc al +TEXT URE +Aut hent +os aurus +P urchase +Ġreleg ation +al ter +ĠÂł Âł +Ġr iddled +Ġo gre +ĠLow ell +Occ up +E at +ĠHy der +ĠAdvis er +Com merce +H unt +ĠOr th +ĠComp etitive +ĠCL A +CD C +Ġsal ads +F le +Ġindustrial ized +` , +ĠO WN +Ġbec k +ĠPart icularly +oub t +Ġm M +ĠHuss ain +ĠChen nai +Ġ9 20 +Ġappoint ing +ĠCull en +,,,, ,,,, +Ġp ores +ver ified +Ġbi ochemical +em ate +Ġcoward ly +ĠHels inki +ĠEthiop ian +S OURCE +ER C +est ro +Ġbi otech +ĠS our +Ġbrew er +Bloom berg +Ġintens ify +Gl ass +an co +ĠF DR +gre SQL +ĠF ires +©¶æ ¥µ +ec o +100 1 +ĠHom eless +Ġinstant aneous +ĠH aste +ig el +D iamond +Ġp aving +Ġland fill +Ġd ads +h oun +: ] +Ġinc endiary +ĠLiving ston +ĠHil bert +ĠChe cks +st yles +in ators +ĠCl ive +ph rine +Ġchimpan zees +Ġp all +ĠJ M +ĠAad haar +ð Ŀ +Ġachie vable +dis abled +P ET +OOOO OOOO +M ot +Ġint angible +Ġbal let +ĠWe bs +ĠEst imated +Effect s +Ġb ailed +Josh ua +Ġturb ulence +Ġoccup ant +ĠDay light +Ġ36 1 +me et +Ġstat ically +Ġon look +Ġk i +il legal +Ġvel vet +Ġdehyd ration +Ġacqu ies +ĠRe z +ak ura +ĠU pton +at ro +Ġincomp rehensible +Ġback door +ĠRh ino +7 27 +Ġmath s +) + +Ġhe resy +Ġd f +ĠRoc he +ĠL ydia +Ġpanc reat +re ply +arre ll +Ġsolicit ation +Ġcirc adian +BI P +Ġfor ay +Ġcrypt ic +iz u +ime o +ĠTom ato +ĠH oms +ex amination +Ġqu arry +ĠVal iant +ĠJer icho +ĠIN CLUD +Ġ18 40 +5 19 +Ġres ists +Ġsnap shots +ĠSp ur +ĠAnt iqu +Log in +Ġbest selling +Ġant ic +ĠS utherland +ãĤ¢ ãĥ« +Ġ~ / +ĠP arm +è ĥ +P ages +int ensity +Ġimm obil +Ġ18 65 +zz o +Ġn ifty +Ġf entanyl +ĠPres ervation +op hen +Ġd arts +ĠD inosaur +po inters +ĠR ite +s uggest +aware ness +ĠSher idan +Ġst ances +Ġsor cery +Ġper jury +ĠNik ola +ie ver +Ġf iance +ĠJordan ian +ĠBall oon +Ġn ab +Ġk b +Ġhuman ities +ĠTan aka +hill ary +Ġconsult ancy +ĠZ ub +Ġrem ission +Ġconf id +CH Q +ĠF ug +Ġimpro vis +Y ep +/ _ +Ġunwilling ness +Ġport folios +05 5 +ĠInstruct or +aim an +Ġclaim ants +M bps +ĠBy e +re ceived +T weet +Ġind emn +ri z +am ara +N at +Ġeval uates +ĠL ur +ep ad +FO X +ĠTh ro +Ġrust y +Ġbed rock +ĠOp rah +J B +Ġmanip ulative +Ġwill ful +Ġrel apse +Ġext ant +The me +S ensor +ĠSt ability +go vern +Ġpo ppy +Ġkn ack +Ġins ulated +ĠT ile +ĠExt rem +Ġunt old +Ġconver ge +Ġref uel +ig roup +Ġdistort ions +Ġrav aged +Ġmechan ically +ĠRe illy +ĠN ose +ĠIncarn ation +ĠBeck y +abb ling +Ġt aco +Ġr ake +Ġmelanch oly +Ġillust rious +ĠDart mouth +Gu ide +ĠR azer +ĠBen z +Ult imate +ĠSur prise +Ġpage ant +off er +Who ever +Ġw iser +Ġchem ist +ĠHE LL +ĠBul k +Ġpl utonium +ĠCO VER +Ö ¼ +f ailed +Ġtire lessly +Ġinf ertility +ĠTr ident +ĠShow time +ĠC iv +V ice +requ ires +itt ance +Ġun controlled +interest ing +56 1 +Ġinnov ate +ateg ic +L ie +ĠS elling +U l +Ġsav ior +ĠT osh +Ġsw ast +P ASS +Ġr ink +Ġcard io +ĠI ro +ud i +Ġv antage +Ġv ans +ĠNi ño ++ = +Ġpropag ate +< ? +Ġmethod ological +204 39 +Ġtrig lycer +Ġing rained +ĠAn notations +arr anted +6 17 +ĠS odium +ĠA AC +techn ical +mult ipl +Ġ3 73 +å ĭ +Ġdec isively +Ġboost ers +Ġdessert s +ĠGren ade +Ġtest ifying +ĠSc ully +ID s +Ġlock down +ĠSc her +ĠR é +ĠWhit man +ĠRams ay +rem ote +Ġh ikers +ĠHy undai +Ġcons cientious +Ġcler ics +ĠSiber ian +ut i +is bury +Ġrel ayed +Ġqu artz +ĠC BI +seek ers +ull a +Ġweld ing +ĠSh al +ble acher +T ai +ĠSam son +Ġt umble +ĠInvest or +Ġsub contract +ĠShin ra +ow icz +j andro +d ad +Ġtermin ating +ĠNe ural +ä» £ +Ġleak age +ĠMid lands +ĠCaucas us +í ķ +c it +ll an +iv ably +ĠAlb ion +Ġ4 57 +Ġregist rations +Ġcomr ade +Ġclip board +0 47 +Ġdiscour aging +ĠO ops +Ad apt +Ġem path +n v +ĠPR OT +ĠDon n +ĠP ax +ĠB ayer +t is +Squ are +Ġfoot prints +part icip +ĠChile an +B rend +ind ucing +M agn +Ġclub house +ĠMagn um +Ġenc amp +ĠEth nic +uch a +ere y +Ġw atered +ĠCal ais +Ġcomplex ion +Ġsect s +Ġren ters +Ġbr as +oÄŁ an +Time out +Man agement +Ġinf ographic +P okemon +Cl ar +Ġloc ality +Ġfl ora +as el +P ont +Ġpop ulate +ĠO ng +Ġsubs istence +Ġa uctions +ĠMcA uliffe +ĠL OOK +br inger +Ġtit an +Ġmanif old +ĠâĹ ı +Ġcalibr ated +Ġcal iphate +ĠSH E +ĠCommission ers +ce ivable +j c +W inner +5 24 +Ġcond one +Other wise +Ġp iling +Ġem body +ĠCrime an +ut ics +ĠEx hibition +Ġ4 26 +e ering +Ġv ying +ĠH UGE +* =- +Ġprin cipled +à ¦ +Ġquir ks +ĠEdit ors +put ing +G ES +ĠF TA +ठ¾ +add on +ĠH AM +ĠFrie za +W oman +. $ +Ġc rib +ĠHer od +Ġtim ers +ĠSp aces +ĠMac intosh +at aka +Ġgl ide +Ġsmell ing +ĠB AL +Ġun su +Ġcond os +Ġbicy cl +ĠRev ival +55 3 +Ġjugg ling +H ug +ĠKardash ian +ĠBalk ans +mult iple +Ġnutrit ious +oc ry +19 00 +Ġinteg rates +Ġad joining +ĠF older +roll ment +ven ient +Ġu ber +y i +Ġwh iff +ĠJu ven +ĠB orough +net te +Ġb ilingual +ĠSp arks +ph thal +man ufact +Ġt outing +ĠPH I +Ke efe +Rew ard +Ġinf all +ĠTem per +typ ically +ĠNik ol +Ġregular s +Ġpseud onym +Ġexhib itions +Ġbl aster +Ġ40 9 +w arming +Ġrever ber +Ġrecip rocal +Ġ6 70 +ip ient +b ett +ĠBe gins +Ġit ching +ĠPh ar +Ass uming +Ġem itting +ĠML G +Ġbirth place +Ġt aunt +ĠL uffy +ĠAm it +Ġcir cled +ĠN ost +enn ett +Ġde forestation +ĠHist orically +ĠEvery day +Ġovert ake +79 2 +Ġn un +ĠLuc ia +Ġaccompan ies +ĠSe eking +ĠTr ash +an ism +R ogue +Ġnorth western +ĠSupplement al +ĠNY U +ĠF RI +ĠSat isf +x es +5 17 +Ġreass ured +Ġspor adic +Ġ7 01 +Ġmed ial +Ġcannabin oid +Ġbarbar ic +Ġep is +ĠExplos ive +ĠD ough +Ġuns olved +Support ed +Ġacknowled gment +sp awn +Ġkit chens +Ġ- = +talk ing +ic ist +ĠPeg asus +ĠPS U +Ġphot on +ĠAuthent ication +R G +@# & +76 2 +ĠCl air +Ġdi aper +Ġbr ist +ĠProsecut ors +ĠJ em +6 28 +ĠEvery where +ĠJean ne +equ ality +ãĥ© ãĥ³ +object s +ĠPel icans +Ġ39 2 +Ġbl u +b ys +ĠA go +Ġinstruction al +Ġdiscrim inating +ĠTR AN +ĠCorn el +ag os +Ġty re +Ġas piration +ĠBrid gewater +": - +! ". +ĠEn s +ĠCoc o +P ie +Ġdet ach +ĠC ouch +Ġphys ique +ĠOccup ations +osc opic +en ough +B uzz +App earance +Y P +Ġrac er +Ġcompl icity +r pm +T oy +Ġinterrupt s +ĠCat alyst +Ġut ilitarian +imp act +Ġsp aghetti +Ġp orous +Ġeste emed +Ġinc iner +ĠI OC +7 48 +Ġesp resso +ĠSm ile +abil ia +6 35 +Ġmathematic ian +Ġ4 24 +ĠK L +ĠH IP +Ġover heard +ĠT ud +ĠT ec +Ġqu izz +Ġfl attering +Ġcon n +âĢ İ +Ġatt aches +ĠR OS +ĠAC S +Ġt cp +ĠSh ame +sk ip +res pected +ĠTrin idad +gr ain +Ġfooth old +ĠUnch arted +ĠJul io +z l +av ored +ĠAn xiety +er rors +ĠCent auri +its ch +D addy +Ġclutch ing +ĠIm plement +ĠGut ierrez +Ġ7 60 +Ġtele portation +end ra +Ġrevers ible +st ros +Ad venture +08 3 +Ġliber ating +Ġas phalt +ĠSp end +AR DS +im sy +PR ES +ĠEmer ging +Ġwild fires +Ġtechn ologically +Ġem its +ĠART ICLE +Ġirregular ities +Ġcher ish +çī Ī +Ġst ink +ĠR ost +Econom ic +Ġcough ing +ĠMcC ann +pro perties +ilant ro +Ġreneg oti +Trans lation +Ġin quest +ĠGra pe +oot ers +gu i +ĠSwords man +ace ae +h itting +Ġr c +Ġexert ed +ĠS AP +it ent +Ġperil ous +Ġobsc urity +Ġassass inate +Ġab original +Ġresc uing +ĠSh attered +lock ing +all ion +Ch anging +ĠHar rington +ĠB ord +ĠAfgh ans +Jam ie +aret z +ĠAugust us +Ġ38 6 +8 30 +Ġj og +ok ingly +Tr igger +ĠH OR +Stat istics +Ġviewers hip +Ġadd itives +h ur +Ġmaxim izing +ĠR ove +ĠLou ie +ĠBuck et +ĠCHR IST +ou sel +Ġstre aks +ir ted +Ġt ert +Ġcolonial ism +Ġbur ying +y k +Cond ition +ĠDPR K +By Id +75 1 +âĹ ¼ +Ġwor risome +Ġvoc ational +sl ice +Ġsa ils +ĠCorrection al +95 4 +Ġt ul +K id +l uster +Ġfam ilial +ĠSp it +ĠEp iscopal +Specific ally +ĠVol cano +run s +q s +Ġve tted +Ġcram med +t rop +here r +Thank fully +Ġper cussion +Ġor anges +Ġround up +Ġ4 99 +x ious +Char acters +ĠZion ism +ĠR ao +ÃĽ ÃĽ +W F +Ġunintention al +ONE Y +Gr ab +Com mercial +Ġglut amate +ĠMcK enna +ru ciating +ning ton +ih u +Ch an +ĠSw ap +Ġleaf lets +Ġfunction ally +er ous +F arm +Ġcal oric +ĠLiter ally +con cert +Ġshe nan +Ġrep aid +ey es +Ġbas hing +ĠG orge +Ġcollabor ations +Ġun account +itch ie +Ġteam work +pp elin +Ġpip ing +Ġmin ced +Ġd iam +ri eg +Ġmasc ara +Ġsuck er +ĠMo ons +App s +ĠPe ck +Ġper v +ĠFl oat +o ley +ĠN ish +im ize +Ġarom atic +u in +end ish +! / +ĠB icycle +ĠAS IC +ile ged +ĠQuad ro +ios yn +Ġlock out +ĠW ink +SP EC +Attempt s +Ġseed ed +red o +ias is +Ġsn ag +ãĥķ ãĤ© +ãĤ ¶ +Ġground ing +Ġrelie ver +Ġfrivol ous +ĠG ifts +ĠF aces +Es pecially +Ġmicrobi ome +im ag +ĠSch l +ĠP les +ĠBle ach +ĠIr win +ĠE aton +ĠDisc iple +Ġmultipl ication +Ġcoer ced +Ġ4 19 +st h +E vil +B omb +Ġex orc +Ġstag gered +L ESS +Ġinert ia +ĠED IT +Ġgo b +Tr aditional +Ġclass y +Lear y +ĠP AGE +yr s +Ġtrans porter +Ġmat ured +Ġhij ab +Ġbi ome +Where as +Ġex termination +ĠT ues +ĠT akeru +ĠAud rey +er ial +ĠAd en +aff les +Ġnarciss istic +ĠB aird +UT F +I re +ĠCon nie +Ch amp +Ġwhis pering +ĠH att +D K +Ġdis infect +Ġdeduct ed +Ġpart ake +Ġdown grade +ĠEs ports +ĠContin uing +Ġdemocr atically +icro bial +itt a +Ġlim estone +Ġexempt ed +ĠFren zy +H erm +7 28 +Ġfled gling +Met a +765 61 +69 3 +% : +w ake +5 26 +ĠDis cipline +Ġvirgin ity +ĠLeg ions +ĠFrank ie +int ent +Ġrest rooms +ĠRou ter +da q +Ġobjection able +âĨ ij +w ark +ĠRah ul +g ain +activ ation +abs olute +ĠAccess ed +Ġ24 00 +ogg les +Ġsecond ly +ĠDEF ENSE +Ġpost age +wra pper +sh arp +7 29 +Ġcommun icates +Ġadd on +ĠMil itia +H ong +Ġsl umped +ĠJP EG +ĠI car +ad ish +68 1 +Ġmaj esty +ĠWolf gang +ĠEl astic +u per +Ġv iz +Ġunconscious ly +ĠST D +ĠS ass +Ġflower ing +ĠHel ic +ĠDra per +ĠAm ateur +Ġman ure +Ġdis ingen +ĠLe i +br ing +9 49 +Ġinhib ited +Ġhead quartered +Ġen igmatic +�� � +Ġred ress +R H +Ġratt led +Ġd iction +l io +ĠT BA +ĠSN AP +C alling +Ġfasc ists +ĠD ove +iew icz +0 36 +Ġco asts +ĠR ect +Ġ) ] +L ot +6 29 +ĠS EM +ĠPeters en +ĠExpl ain +ĠBo ards +ĠBe zos +ĠJ ournals +Ġ20 24 +p arser +Ġmist rust +Ġgr ate +ĠL ocked +bo a +S aint +g aming +Ġvow el +in ately +bl ow +All ah +Ġun matched +Ġb ordering +ĠExp end +n r +Or acle +rou ch +Ġcont iguous +ac us +Ġdist raught +58 1 +Ġanat omical +O X +ap ixel +8 33 +ĠPL US +Ġres usc +Ġab iding +57 3 +Ġvac ancies +Em ily +Ġhyp othal +ĠWer ner +ĠWe e +ĠDJ s +5 13 +Ġwitch craft +Ġac upuncture +ent ary +benef it +Product s +ĠP SP +ĠMP G +ĠJ inn +ĠJ arrett +Ġ4 45 +ĠIm aging +ĠP yth +Fin ish +Ġte x +Ġjuven iles +Ġhero ism +Ġdoubt less +ĠA ki +ĠT end +ĠPatri arch +Ġbit ters +ĠTele communications +it atively +ag na +Ġr g +ĠS OLD +Ġcomp ulsion +ĠN asa +ĠKath ryn +Ġmillion aires +Ġintrins ically +Ġbolst ered +time out +fl o +Ġtut or +p our +Stat ement +Ġ{ * +ĠRud olph +ĠKimber ly +rog ens +adi q +] + +Ġindign ation +Ġfract uring +ĠRe leases +ĠGr ain +pro tein +L ago +Ġvac ations +Ġboot ed +ĠTH REE +ĠH G +oresc ence +Ġt f +Ġso ar +iosyn cr +Ġgl ances +ĠSp oon +ĠJ ury +ĠCow boy +Ġcreat ively +Hig her +Ġsolic itor +Ġhaw k +ac io +89 6 +Ġsuperf lu +Ġbombs hell +ct ure +Ġbroker age +Ġraid ing +Ġf rench +Ġang led +Trans action +ĠGen ocide +u pe +ĠHait ian +57 2 +! : +Ġunwitting ly +iter ator +sc roll +Ġtall ied +Ġbi omedical +ĠC ARD +Ġe uphem +Ġbrain storm +a quin +K o +Mic helle +ĠR unes +ĠBall istic +ud ers +Ġmod esty +ĠiP ads +ĠEzek iel +Y E +Ġstars hip +Ġpower fully +Ġper l +ĠSh ade +ĠQu art +ĠE EG +Ġfisher man +OS ED +ĠTyp ical +df x +Ġmes hes +Ġet ched +worth iness +Ġtopp led +Ġ3 96 +or ius +We iss +Ġmy sql +ĠVal halla +Ù Ĵ +le asing +Ġrec omp +rap nel +S el +04 3 +Ġder ailed +ĠGu ides +IR T +Ġde human +ĠBritt any +" )) +Ġex claim +Ġb alk +Ġ8 40 +CLA IM +int el +L AB +Ġpe gged +Ġast roph +sm oking +Ġrig ging +Ġfix ation +Ġcat apult +ins ide +ĠC ascade +ĠBolshe vik +G aza +Dep th +Ġloud spe +Ġalmond s +me yer +l eness +j en +f resh +Ġunbeat en +ĠSqu id +ĠPres umably +Tim er +B W +Ġro sters +Ġell ipt +ĠHar riet +dat abase +ĠMut ual +ĠComm odore +uk ed +kn ife +ĠCOMM UN +h ya +Ġmel ts +arch ives +Ġrat ification +Ġmultip lying +Ġinter oper +Ġasc ert +w ings +ver ting +ĠScorp ion +ay e +ĠPorts mouth +ĠM TA +n it +iaz ep +Ġqu arantine +Ġslides how +Ġcent imeters +Ġsyn opsis +Ġsp ate +th irst +Ġnom inating +ĠMel vin +Pre view +Ġthro b +Ġgener ational +ĠRad ius +rest ling +put able +aw ar +N ECT +Ġunlaw fully +ĠRevel ations +Wik ipedia +sur v +Ġeye ing +ij n +ĠF W +Ġbr unt +Ġinter stellar +Ġcl itor +ĠCroat ian +ĠCh ic +ev a +ĠDis app +ĠA kin +iner ies +d ust +Interest ed +Ġgen esis +ĠE ucl +ö n +p icking +Ġmut ated +Ġdisappro ve +ĠHD L +Ġ6 25 +Ì ¶ +c ancer +Ġsqu ats +Ġle vers +Disc uss += ] +D ex +ĠVIDE OS +A UD +Ġtrans act +ĠKin ect +ĠK uala +ĠC yp +7 47 +Ġsh attering +Ġarsen ic +ĠInt ake +ĠAngel o +ĠQu it +ĠK he +Ġ18 93 +M aker +0 29 +ĠPain ting +Dis able +9 16 +Ġanal ges +Ġtact ile +Ġprop hes +Ġd iced +ĠTravel s +ĠHe ader +ĠClub s +Ass istant +Ġinc rim +Ġd ips +Ġcruc ifix +ĠShan ahan +ĠInter pret +Ġ40 90 +al ogy +abb a +Ġsimul ac +hus band +S IM +Ġrecy cle +uc er +ed ged +Ġre naissance +ĠBomb ay +Cath olic +ĠL INE +ĠCl othing +re ports +Ġpl aus +Ġd ag +ĠM ace +Z I +Ġintr uder +ĠVeter inary +g ru +Ġsne aky +ĠS ie +ĠC innamon +P OSE +Ġcou rier +ĠC NS +Ġemanc ipation +s it +Ġplay through +ĠFac ilities +v irt +ĠG auntlet +Thom pson +Ġunbeliev ably +Param eters +Ġst itching +ign e +ĠTH ESE +Priv acy +Ġshenan igans +Ġvit ri +ĠVal id +59 1 +Ń · +ĠProt otype +ink a +SC P +ĠT id +è Ī +old ed +Ġindividual ity +Ġbark ing +Ġm ars +ĠW D +Ġ8 20 +Ġt ir +Ġsl apping +Ġdisgr untled +ĠAng ola +ri us +ĠTorn ado +ĠTh urs +Ġcapt cha +Ġang st +ĠP og +ĠAssass ins +ĠAd idas +Ġjoy ful +Ġwh ining +Emer gency +Ġphosph orus +Ġatt rition +oph on +ĠTimber wolves +ĠJ ah +ĠBr inging +ĠW ad +ĠEn sure +oh l +ĠX ie +omm el +c mp +Ġz ipper +Ġrel at +ĠCor ridor +m ilo +T ING +Av g +Ġcro pped +] } +Ġr aged +ĠLump ur +ĠGuer rero +our ke +N ut +Ġoff sets +og lu +dr m +Ġmort als +lat able +Ġdismiss ive +ä¸ ī +Ġthro ats +Ġchips et +ĠSpot light +Catal og +art ist +G b +Ġch illy +Ġst oked +Ġ3 74 +W ard +L atin +Ġf iasco +Ġble ach +Ġb rav +Enh anced +Ġin oc +ĠFior ina +_ > +Ġle ukemia +Ġel uc +Ġannoun cer +ĠLith uan +ĠArm ageddon +å ĩ +Len in +ĠR uk +Ġpe pp +ĠRom antic +ĠP IT +ĠInter stellar +ĠAt kinson +R aid +J s +Go al +C ourse +Ġvan ishing +es ley +ĠR ounds +Els a +59 3 +Ġredund ancy +ĠST AND +Ġprop hetic +Ġhabit able +ry u +Ġfaint ly +M ODE +Ġfl anked +IR C +Aw esome +Ġsp urious +ĠZ ah +ĠMS G +Ġsh ading +Ġmotiv ational +ĠSant ana +ĠS PR +Ġexc ruciating +om ial +ĠM iko +ĠLe opard +A byss +Ġ[ | +d irty +Ġbath s +Ġdem oral +and re +P B +Ġun ification +Ġsac rament +Ġ[ & +Ġpric eless +Ġgel atin +Ġeman ating +ĠAll aah +98 6 +Ġout burst +Ġer as +ĠX VI +ĠSP I +O tt +ĠLaz arus +PL IED +F lying +blog s +W isconsin +R aven +Ġreb ate +Ġcreep s +ĠSp an +ĠPain ter +ĠKir a +ĠAm os +ĠCor vette +Cons umer +ĠRec over +ck i +Ġpes ky +ĠIn vention +Compan ies +Ġchalleng ers +ad emic +ĠUkrain ians +ĠNeuro log +ĠFors aken +Ġent rants +Ġemb attled +Ġdef unct +ĠGlac ier +Ġpo isons +ĠH orses +m akes +ĠD irt +Ġ4 23 +hh h +ĠTrans formation +QUI RE +................ .. +Ġtrave ller +ĠSe xy +ĠK ern +ip olar +Ġransom ware +oooooooo oooooooo +E c +rub y +Prof essional +ĠOut break +arg ument +G rey +ĠFif a +ĠCH O +ĠFOR M +ĠAm trak +- [ +Ġcr adle +Ġantioxid ants +ãģ®å ® +7 36 +ĠNAS L +ĠContribut ions +Ind iana +ĠST EP +C SS +Ġsal ient +Ġall ocations +yr ights +Ġm ashed +ĠCut ter +Sex ual +Ġp ounded +Ġfan base +Ġc asc +ĠTrans parency +Ġanaly tic +ĠSummon er +× ŀ +ĠAD C +det ail +Ġvan quished +Ġcr abs +ar ie +Dest roy +ĠS ack +Ġtrans istor +Al abama +ĠK oen +ĠFisher ies +c one +Ġannex ed +ĠM GM +es a +Ġf aked +ĠCong ratulations +Ġhind ered +Ġcorrection al +ĠI TV +lee ve +Ġin appropriately +lic ks +Ġtresp ass +Ġp aws +Ġnegoti ator +ĠChrist ensen +lim its +ĠDian ne +Ġeleg ance +ĠContract s +an ke +Ob j +Ġvigil ance +Ġcast les +ĠN AD +ĠHol o +Ġemph atically +ĠTit us +ĠServ ing +ĠRich ie +ĠP igs +5 68 +Ġanim osity +ĠAtt ributes +ĠU riel +M Q +my ra +ĠApplic ant +Ġpsychiat rists +ĠV ij +ĠAb by +ag ree +P ush +Ġk Wh +hib a +Ġinc ite +ĠWe asley +ĠTax i +minist ic +hy per +ĠF arn +Ġ6 01 +ĠNation wide +F ake +95 2 +Ġma ize +Ġinteract ed +Ġtransition ed +Ġparas itic +Ġharm onic +Ġdec aying +Ġbas eless +ns ics +Ġtrans pired +Ġabund antly +ĠFore nsic +Ġtread mill +ĠJ av +ab and +Ġssh d +Ġfront man +ĠJak arta +oll er +dro ps +ĠSERV ICES +rompt u +oph ical +h ospital +bled on +6 45 +Ġmid range +ĠEV ENT +cul ated +raw led +Ġper ched +Ġover board +ĠPe el +ĠP wr +ĠCar th +ĠCOM PLE +co e +sh all +Ġdeter rence +M ETHOD +ĠAbs ent +M EN +Ġs ill +ĠLE VEL +Y ork +Ġsin ners +ĠOP EC +ĠN ur +ĠDesign s +se lection +Ġunw orthy +CH A +Ġstreng thens +88 3 +ed ly +Ġslic ing +Ġmal nutrition +Ġfilm making +ĠPol k +ur ated +Ġ4 21 +bre akers +!' " +Ġwet lands +ĠDisc rimination +Ġallow able +Ġste ered +ĠSic ily +S AM +Ġmust ache +Ġm ids +Ġcl ipped +Ġcirc ulate +Ġbr ittle +ĠBuild ings +ra ised +ĠRound up +Ġwealth ier +Ġoverw rite +Ġover powered +ĠGerr ard +s ites +PD ATED +Ġacute ly +ĠGam ble +Ġp im +ĠK us +Typ ically +De ploy +ĠMoroc can +p otion +com be +Ġvigil ante +Ġ36 3 +St ew +ĠB agg +Ġres ided +ĠSp o +Ġrem nant +Ġempt iness +br ainer +Ġout patient +pri ority +Ġle ptin +ĠPay ton +ĠGle aming +ĠS hed +ĠPol o +ĠMormon ism +rest ricted +arl ane +w x +Ġcreat ine +ĠAn on +ĠST UD +ĠJ UL +ĠT ee +5 28 +08 9 +Ġhat ched +Dis patch +ĠCompos ite +Ġ45 1 +p uff +ĠX COM +ĠOr n +ĠTH ANK +END ED +ĠAshe ville +Ġà ľ +Ġman go +ĠS lightly +world ly +ĠW ander +ĠExp and +ĠCh r +M ist +Ġorthodox y +ĠUN ESCO +reg ate +Else where +k ie +ir led +Ġtopp le +Ġadopt ive +ĠLeg s +d ress +ĠS agan +b are +ĠGl ou +Cr unch +Ġhelp ers +Ġchron ically +ĠH uma +1 0000 +Ġaccommod ating +äº Ķ +Ġwrink les +Ġdod ged +four th +Ġpre con +Ġcompress or +ĠK are +Ġev ict +ĠWar wick +im ar +Ġmodern ization +Ġband wagon +Ġref uted +Ġnet ted +ĠNa ples +ĠGen ie +per ors +Ġfield ed +Ġde re +ĠPar ables +le es +Ġtr out +asp ers +Ġn ihil +Ġhapp iest +Ġflo ppy +ĠLo ft +ĠHe ard +Ġun ison +Ġl ug +ĠRed mond +class ic +Supp orters +SH IP +G MT +Ġfue lled +ç IJ +Ġd d +ĠEmin em +Ġ18 97 +NY SE +Ġsecret aries +ĠF IA +ĠCanaver al +F avorite +Ġp omp +Ġdetain ee +ers hip +aim on +i our +ĠA pex +Ġplant ations +am ia +ac ion +R ust +Ġtow ed +ĠTru ly +5 77 +Ġshel tered +r ider +W o +Ġl air +ĠInt elligent +impro ve +m atically +Ġet iquette +ad ra +all o +ĠJun o +any thing +ĠStru ggle +ĠPred ict +ĠGr imes +ĠAMER ICA +ct x +ĠSit uation +W OOD +Ġsol uble +me ier +Ġintoler able +ang ering +Ġun interrupted +Ġtool tip +Ġinterrog ated +Ġgun ned +ĠSne ak +æŃ ¦ +Ġt ether +Ġcr umble +L ens +Ġclust ered +ĠSy l +ĠHas an +Ġdystop ian +w ana +Ġjoy stick +ĠTh ib +amm u +Tom orrow +5 46 +Ġoverc ame +Ġminim ized +cept or +Run ner +ENG TH +ĠBrend a +ĠAchieve ments +Ġtor ches +Ġrapp ort +ĠInvestig ator +ĠHand ling +rel ation +g rey +8 15 +Ġk cal +ĠComm ands +d q +Ġcur ls +Ġbe arer +Ġcyn icism +it ri +ĠUse ful +B ee +D CS +Ġab ras +P ract +BIL ITIES +7 12 +Ġdebug ger +Ġdebt or +ĠL ia +ĠK ers +Ġexacerb ate +ĠSt acy +ĠB land +ĠSc enes +Ġbranch ing +âĸĪâĸĪâĸĪâĸĪ âĸĪâĸĪâĸĪâĸĪ +ape ake +Ġs alsa +Ġmish and +ĠKon ami +ĠN ib +Ġanecd ote +Ġagree able +Ï ī +ĠNath aniel +ĠHe isman +ĠB eware +Ġ18 86 +spect ive +69 1 +5 22 +Ġinhib its +Ġhas hing +Ġ18 89 +å° Ĩ +v ich +P ure +Ġsolid ly +Ġaspir in +im aru +Ġstreet car +ĠU CS +ĠJ udd +Ġflash backs +p ins +Ġ14 40 +ĠUN HCR +ĠSym ptoms +T IT +5 38 +F ra +% ); +Ġo oz +Ġcur few +Ġcal med +Ġparticip ates +Te X +Ġnons ensical +Ġfull back +ĠDe L +mon key +h ari +Ġmetabol ites +Ġloot ed +ĠAL WAYS +ĠB CC +L t +oc het +B one +Ġveto ed +Ġg cc +ĠCL ICK +Ġ18 88 +s af +Ġstiff ness +Ġlow ly +ĠGe h +vers on +ors et +Ġun foreseen +Ġan esthesia +ĠOpt ical +Ġrecon structed +ĠT up +sh ows +NEW S +ĠNewsp aper +ĠA SA +ter a +N umbers +Ġinexpl icable +× ij +Ġhard ness +unt arily +ĠA cer +grad ient +ARD IS +Ġwood land +Ġmetaph ors +ĠWem bley +ĠPa vel +phil is +Ġre writing +Ġpercept ual +Ġ10 70 +worm s +ĠDown s +Ġunsur prisingly +Ġtag ging +fl ame +Ġlit res +Ġboun ces +ĠB abe +sh ut +Ġoverd oses +ĠShe ila +ĠCh au +ĠBl ess +Capt ure +ĠSign ificant +ĠSc ion +Ġ38 9 +ĠMc H +ĠTitan ium +ĠMe al +amed a +ag ents +agg ressive +B illy +76 3 +ĠS aying +DER R +it one +Coll ins +B ound +Ġbol ted +ĠDM CA +95 3 +Ġun iqueness +Ġep igen +un ci +ant am +Ġreck oning +ch airs +OG R +ĠSen egal +Ġ18 62 +re levant +Ġ ¯ +Ġpharm acies +ĠG eral +v ier +Y an +OR PG +Ġrab id +b ending +ĠUN ITED +Ġ4 65 +As sembly +Ġwe ep +Ġbe hest +ĠMother s +ĠJ ace +h id +Ġwh irlwind +ĠUN IVERS +Ġut opian +Ġkidn ap +Ph ilipp +K in +89 3 +Ġlivest ream +ĠM ISS +Ġsub versive +ĠTechn iques +ĠJUST ICE +ĠB ASE +Ġ38 7 +Ġassail ants +ĠHard core +Ġsprink led +ĠP se +é ļ +print ed +ĠH au +OR GE +ĠT OUR +Ġl aced +Ġit ch +G iving +Ġport ed +78 1 +//////////////// //////////////// +bre eding +Ġlog ger +ĠH OL +inn ie +First ly +Ġembry onic +Ġdeleg ated +p ai +O IL +Ġcentr ally +ĠR x +ĠSc outing +D utch +Ġhe reditary +ĠCru iser +s at +5 29 +ĠMar riott +other mal +Ġprohib itions +E arn +ĠSt ab +ĠColleg es +ĠBel ief +st retched +ĠL H +ĠEntity Item +C IA +Ġun rem +Ġlaure ate +Ġdenomin ations +sum mary +h ler +S pect +ĠK laus +ĠBe ans +Ġins ur +ĠPA X +Ġfield er +ĠV et +ĠSp arrow +z ie +ĠS Q +ĠMond ays +ĠOff line +ĠLer ner +ĠExt ensions +Ire land +Ġpatron age +Ġcontrast ed +ĠMan ia +h irt +Mos cow +Ġcondem ns +ĠAn ge +Ġcomp osing +ĠPe pe +ĠP addock +Ġheter ogeneity +Ġide ologically +Ġf ishes +Ġcur sing +ĠR utherford +ĠFlo ating +ĠAm elia +Te a +Syn opsis +Ġstun ts +Ġbe ad +Ġstock ing +ĠM ILL +ob ook +mass ive +\ < +Ġh ump +ĠPref erences +Engine Debug +ge ist +ĠNiet o +ome ver +ish y +eval uate +col onial +Altern ative +ĠGo Pro +ĠV ortex +ĠNET WORK +ans ky +Sec ure +ĠTh rust +Sn ake +Ġparcel s +Ġsam urai +Ġactress es +N ap +M F +ifer ation +Be er +5 23 +ĠI ly +oint ment +P ing +Ġstri ped +ĠMell on +oss ession +Ġneut ron +end ium +Ġa ph +ĠFlav oring +Ġ38 3 +Ġrespons iveness +ĠJ indal +ĠHitch cock +Den ver +ĠDRAG ON +sm anship +ĠDu pl +Ġs ly +Ġweb cam +ĠTw ain +ĠDar ling +ili ate +cons umer +D IT +Ġnames ake +Ġun orthodox +Ġfun er +ĠPL oS +ĠCONTR OL +ozy g +ogl obin +F ACE +ER G +ĠD ia +ĠF iesta +ce le +0 34 +Ġencl ave +âĸ¬ âĸ¬ +on ement +al ist +M and +Ġhome grown +ĠF ancy +Ġconcept ions +ĠCont ains +ure en +Ġreiter ate +Ġme ager +Ġinstall ments +Sp awn +6 27 +Ġphot oc +ĠCab rera +ĠRos enthal +ĠLans ing +is ner +Ġinvest s +ĠUFO s +EX P +Hard ware +Ġtr agically +Ġconced es +ie ft +ch am +bor gh +ĠSch r +ĠMel anie +ĠH oy +Ġvisit ation +Ġid iosyncr +Ġfract ions +Ġfore skin +ob os +Ġpo aching +ĠVI EW +Ġstimul ates +ĠG ork +can on +M IC +ĠNem esis +ĠInd ra +ĠDM V +Ġ5 29 +Ġinspect ing +Ġgrand ma +ĠW hedon +ĠSh ant +ĠP urg +ik an +ĠT eg +ĠCL R +z ac +Vict oria +ĠVer ify +ion ics +Ġpart ying +ĠM ou +col our +Ġtestim onies +l ations +Ġpress uring +hi ro +ac ers +Ġf id +ang ler +ĠCS I +Ġhere after +Ġdiss idents +report ing +iph any +che v +Ġsol itude +Ġl obe +Ġind is +Ġcred ential +re cent +ad ult +ĠNir vana +ĠFranch ise +L ayer +H yp +ĠBerks hire +Ġwill s +t if +Ġtot em +ĠJud ah +rep air +Inst ant +5 48 +Ġemb assies +Ġbott leneck +Ġb ount +Ġtyp ew +ĠAl vin +j ing +im ilar +R ush +Ġbr im +ĠHEL P +A im +] ' +Ġpass ively +Ġbound ed +ĠR ated +Ġcriminal ity +Ġbiom ark +Ġdisp atcher +ĠTow ards +Ġ+ ++ +right eous +f rog +ĠP anc +C arter +0 32 +æ© Ł +Ġult raviolet +ĠLic ensed +ĠT ata +ĠBl essing +ĠG AM +Ġchem ically +ĠSe af +ĠRE LE +ĠMerc enary +capital ist +Ġform ulations +Ġann ihilation +ĠVer b +ĠAr gon +Ġun loaded +Ġmorp hed +Ġconqu ering +back er +I ELD +Ġtheft s +Ġfront runner +ĠRoy ale +ĠFund amental +el ight +C hip +necess ary +ay n +ĠSl ip +Ġ4 48 +cern ed +P ause +Ġshock ingly +ĠAB V +Ġcomp osure +7 33 +ĠMotors port +ah ime +Mur ray +M ach +Ġgr ids +Ġdeb ian +Ġfurther more +Ġdexter ity +ĠCollect ions +os lov +il age +b j +ĠMont eneg +Ġstrut Connector +Ġmassac res +Ġbrief s +fet ched +uv ian +ol ition +Fail ure +emon ic +Ġfl ared +Ġclaim ant +Ġc ures +Ġgive aways +ĠSubst ance +al ions +Ġcr inge +ĠK ul +Ġarist ocracy +ĠUl ster +ol ated +h ousing +ĠM IS +Ġgl ared +ĠWil helm +ne eds +lam bda +build ers +ĠV IS +Ġradi ator +ĠGhost busters +Ġ4 36 +act ual +Ġher ds +ç a +watch ing +Ġcounter ing +Ch arge +Ġchar red +Ġwar heads +Ġiod ine +ĠM acy +04 1 +Ġdepart ures +ĠS ins +Ġdy ed +ĠConcept s +g ado +7 13 +Ġquot ations +Ġg ist +ĠChrist y +Ġant igen +ĠHem p +ĠD rawn +ĠB arg +ez vous +Ġp aternity +Ġar du +ĠAnch orage +ĠR ik +Ġover loaded +ĠUs ername +ĠTam my +ĠN au +ĠCell ular +Ġw aning +Ġrod ent +ĠWor cester +il ts +ĠT ad +Ġdwell ings +Ġbull ish +4 31 +Ġretali ate +Ġmig raine +ĠChev ron +CH ECK +Ġdon key +c rim +SP A +ĠAn alog +Ġmarqu ee +ĠHa as +B ir +ĠGD DR +ĠDownload s +Ġwill power +ĠFor th +ĠRecord ed +Ġimp ossibility +ĠLog ged +ĠFr anks +ĠR att +in itions +Ġclean ers +Ġsore ly +Ġflick ering +ĠEx amination +c atching +allow een +Ms g +Ġdun no +F a +Ġdys ph +c razy +.' '. +Ġmain line +Ġc s +Ġp tr +ĠW ally +ig un +95 1 +ĠBig foot +f ights +Ġretrie ving +J r +Ġdupl ication +ĠExpl an +Ġrel ational +Ġqu aint +Ġbisc uits +Ġad o +Ġsh udder +Ġantid ote +blood ed +ks h +Ġsa uces +Ġrein vest +Ġdispens ary +ĠD iver +Ġ9 000 +stud ent +Ġin separ +esc ap +Ġtodd lers +ĠGP IO +ĠAss ignment +head ers +Ġlack luster +Ġab ack +95 6 +Ġtool bar +7 45 +Ġo ust +Ġcontempl ation +ĠPRES IDENT +Ġ4 58 +==== == +Ġguarantee ing +ĠHe ist +ĠCann es +Ļ ½ +Ġcollabor ator +ĠAm p +Ġg ou +ĠSH ALL +st ories +78 3 +Ġmobil ized +Ġbro od +ĠL U +ĠðŁ ij +Ġref in +ĠAnthrop ology +v ind +ill i +Ġwarrant ies +ĠB abel +Ġsw ath +Ġc aches +Ġantagon ists +art ifacts +Ġhot ly +ĠSt arts +ĠG ö +z ag +!! !!! +Ġsc ourge +Ġcons piring +ru its +re verse +ĠShe en +ĠJes uit +ĠGiov anni +ad ies +Ġbutt ocks +ear cher +ac an +Ġvolley ball +Ġshroud ed +Ġscore board +b ats +ĠI PM +Ġass es +Ġde regulation +ĠTe legram +ĠReb oot +Ġ7 000 +ĠCan ary +Ġk ernels +ĠFranç ois +ĠD uff +ĠP on +ĠLe ica +ĠGar min +Ġor phans +ĠClaud ia +Ġcal endars +ĠLe ilan +ent o +R ocket +Ġbr unch +ĠHaw king +ain ers +Ġsens ibilities +Ġk W +ĠK and +Ġre claimed +Ġinteresting ly +× © +rom y +J M +ĠEnhance ment +b ush +Sk ip +Ġrapp ers +Ġg azing +p edia +ath lon +Rev olution +Ġsn ipers +Ġre verted +Ġconglomer ate +T erry +79 4 +Ġhars her +Ġdes olate +ĠHit man +Comm ission +Ġ( / +âĢ¦ ." +Com par +Ġampl ification +om inated +Ġreg ress +ĠColl ider +Ġinform ants +Ġg azed diff --git a/experiments/elasticdnn/gpt_neo/1.3B_ckpt/special_tokens_map.json b/experiments/elasticdnn/gpt_neo/1.3B_ckpt/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..817762d631ad6f9c799f6b9dc713c46420e65546 --- /dev/null +++ b/experiments/elasticdnn/gpt_neo/1.3B_ckpt/special_tokens_map.json @@ -0,0 +1 @@ +{"bos_token": "<|endoftext|>", "eos_token": "<|endoftext|>", "unk_token": "<|endoftext|>"} \ No newline at end of file diff --git a/experiments/elasticdnn/gpt_neo/1.3B_ckpt/tokenizer_config.json b/experiments/elasticdnn/gpt_neo/1.3B_ckpt/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ddcabb4ad76163e2bae42b720c07d509c99494e0 --- /dev/null +++ b/experiments/elasticdnn/gpt_neo/1.3B_ckpt/tokenizer_config.json @@ -0,0 +1 @@ +{"unk_token": "<|endoftext|>", "bos_token": "<|endoftext|>", "eos_token": "<|endoftext|>", "add_prefix_space": false, "model_max_length": 2048, "special_tokens_map_file": null, "name_or_path": "gpt2"} \ No newline at end of file diff --git a/experiments/elasticdnn/gpt_neo/1.3B_ckpt/url.txt b/experiments/elasticdnn/gpt_neo/1.3B_ckpt/url.txt new file mode 100644 index 0000000000000000000000000000000000000000..7d66e441e2b1a90b57415e2685a54c406da832fd --- /dev/null +++ b/experiments/elasticdnn/gpt_neo/1.3B_ckpt/url.txt @@ -0,0 +1 @@ +https://huggingface.co/EleutherAI/gpt-neo-1.3B \ No newline at end of file diff --git a/experiments/elasticdnn/gpt_neo/1.3B_ckpt/vocab.json b/experiments/elasticdnn/gpt_neo/1.3B_ckpt/vocab.json new file mode 100644 index 0000000000000000000000000000000000000000..84ef7fb594b5c0979e48bdeddb60a0adef33df0b --- /dev/null +++ b/experiments/elasticdnn/gpt_neo/1.3B_ckpt/vocab.json @@ -0,0 +1 @@ +{"!":0,"\"":1,"#":2,"$":3,"%":4,"&":5,"'":6,"(":7,")":8,"*":9,"+":10,",":11,"-":12,".":13,"/":14,"0":15,"1":16,"2":17,"3":18,"4":19,"5":20,"6":21,"7":22,"8":23,"9":24,":":25,";":26,"<":27,"=":28,">":29,"?":30,"@":31,"A":32,"B":33,"C":34,"D":35,"E":36,"F":37,"G":38,"H":39,"I":40,"J":41,"K":42,"L":43,"M":44,"N":45,"O":46,"P":47,"Q":48,"R":49,"S":50,"T":51,"U":52,"V":53,"W":54,"X":55,"Y":56,"Z":57,"[":58,"\\":59,"]":60,"^":61,"_":62,"`":63,"a":64,"b":65,"c":66,"d":67,"e":68,"f":69,"g":70,"h":71,"i":72,"j":73,"k":74,"l":75,"m":76,"n":77,"o":78,"p":79,"q":80,"r":81,"s":82,"t":83,"u":84,"v":85,"w":86,"x":87,"y":88,"z":89,"{":90,"|":91,"}":92,"~":93,"¡":94,"¢":95,"£":96,"¤":97,"¥":98,"¦":99,"§":100,"¨":101,"©":102,"ª":103,"«":104,"¬":105,"®":106,"¯":107,"°":108,"±":109,"²":110,"³":111,"´":112,"µ":113,"¶":114,"·":115,"¸":116,"¹":117,"º":118,"»":119,"¼":120,"½":121,"¾":122,"¿":123,"À":124,"Á":125,"Â":126,"Ã":127,"Ä":128,"Å":129,"Æ":130,"Ç":131,"È":132,"É":133,"Ê":134,"Ë":135,"Ì":136,"Í":137,"Î":138,"Ï":139,"Ð":140,"Ñ":141,"Ò":142,"Ó":143,"Ô":144,"Õ":145,"Ö":146,"×":147,"Ø":148,"Ù":149,"Ú":150,"Û":151,"Ü":152,"Ý":153,"Þ":154,"ß":155,"à":156,"á":157,"â":158,"ã":159,"ä":160,"å":161,"æ":162,"ç":163,"è":164,"é":165,"ê":166,"ë":167,"ì":168,"í":169,"î":170,"ï":171,"ð":172,"ñ":173,"ò":174,"ó":175,"ô":176,"õ":177,"ö":178,"÷":179,"ø":180,"ù":181,"ú":182,"û":183,"ü":184,"ý":185,"þ":186,"ÿ":187,"Ā":188,"ā":189,"Ă":190,"ă":191,"Ą":192,"ą":193,"Ć":194,"ć":195,"Ĉ":196,"ĉ":197,"Ċ":198,"ċ":199,"Č":200,"č":201,"Ď":202,"ď":203,"Đ":204,"đ":205,"Ē":206,"ē":207,"Ĕ":208,"ĕ":209,"Ė":210,"ė":211,"Ę":212,"ę":213,"Ě":214,"ě":215,"Ĝ":216,"ĝ":217,"Ğ":218,"ğ":219,"Ġ":220,"ġ":221,"Ģ":222,"ģ":223,"Ĥ":224,"ĥ":225,"Ħ":226,"ħ":227,"Ĩ":228,"ĩ":229,"Ī":230,"ī":231,"Ĭ":232,"ĭ":233,"Į":234,"į":235,"İ":236,"ı":237,"IJ":238,"ij":239,"Ĵ":240,"ĵ":241,"Ķ":242,"ķ":243,"ĸ":244,"Ĺ":245,"ĺ":246,"Ļ":247,"ļ":248,"Ľ":249,"ľ":250,"Ŀ":251,"ŀ":252,"Ł":253,"ł":254,"Ń":255,"Ġt":256,"Ġa":257,"he":258,"in":259,"re":260,"on":261,"Ġthe":262,"er":263,"Ġs":264,"at":265,"Ġw":266,"Ġo":267,"en":268,"Ġc":269,"it":270,"is":271,"an":272,"or":273,"es":274,"Ġb":275,"ed":276,"Ġf":277,"ing":278,"Ġp":279,"ou":280,"Ġan":281,"al":282,"ar":283,"Ġto":284,"Ġm":285,"Ġof":286,"Ġin":287,"Ġd":288,"Ġh":289,"Ġand":290,"ic":291,"as":292,"le":293,"Ġth":294,"ion":295,"om":296,"ll":297,"ent":298,"Ġn":299,"Ġl":300,"st":301,"Ġre":302,"ve":303,"Ġe":304,"ro":305,"ly":306,"Ġbe":307,"Ġg":308,"ĠT":309,"ct":310,"ĠS":311,"id":312,"ot":313,"ĠI":314,"ut":315,"et":316,"ĠA":317,"Ġis":318,"Ġon":319,"im":320,"am":321,"ow":322,"ay":323,"ad":324,"se":325,"Ġthat":326,"ĠC":327,"ig":328,"Ġfor":329,"ac":330,"Ġy":331,"ver":332,"ur":333,"Ġu":334,"ld":335,"Ġst":336,"ĠM":337,"'s":338,"Ġhe":339,"Ġit":340,"ation":341,"ith":342,"ir":343,"ce":344,"Ġyou":345,"il":346,"ĠB":347,"Ġwh":348,"ol":349,"ĠP":350,"Ġwith":351,"Ġ1":352,"ter":353,"ch":354,"Ġas":355,"Ġwe":356,"Ġ(":357,"nd":358,"ill":359,"ĠD":360,"if":361,"Ġ2":362,"ag":363,"ers":364,"ke":365,"Ġ\"":366,"ĠH":367,"em":368,"Ġcon":369,"ĠW":370,"ĠR":371,"her":372,"Ġwas":373,"Ġr":374,"od":375,"ĠF":376,"ul":377,"ate":378,"Ġat":379,"ri":380,"pp":381,"ore":382,"ĠThe":383,"Ġse":384,"us":385,"Ġpro":386,"Ġha":387,"um":388,"Ġare":389,"Ġde":390,"ain":391,"and":392,"Ġor":393,"igh":394,"est":395,"ist":396,"ab":397,"rom":398,"ĠN":399,"th":400,"Ġcom":401,"ĠG":402,"un":403,"op":404,"00":405,"ĠL":406,"Ġnot":407,"ess":408,"Ġex":409,"Ġv":410,"res":411,"ĠE":412,"ew":413,"ity":414,"ant":415,"Ġby":416,"el":417,"os":418,"ort":419,"oc":420,"qu":421,"Ġfrom":422,"Ġhave":423,"Ġsu":424,"ive":425,"ould":426,"Ġsh":427,"Ġthis":428,"nt":429,"ra":430,"pe":431,"ight":432,"art":433,"ment":434,"Ġal":435,"ust":436,"end":437,"--":438,"all":439,"ĠO":440,"ack":441,"Ġch":442,"Ġle":443,"ies":444,"red":445,"ard":446,"âĢ":447,"out":448,"ĠJ":449,"Ġab":450,"ear":451,"iv":452,"ally":453,"our":454,"ost":455,"gh":456,"pt":457,"Ġpl":458,"ast":459,"Ġcan":460,"ak":461,"ome":462,"ud":463,"The":464,"Ġhis":465,"Ġdo":466,"Ġgo":467,"Ġhas":468,"ge":469,"'t":470,"ĠU":471,"rou":472,"Ġsa":473,"Ġj":474,"Ġbut":475,"Ġwor":476,"Ġall":477,"ect":478,"Ġk":479,"ame":480,"Ġwill":481,"ok":482,"Ġwhe":483,"Ġthey":484,"ide":485,"01":486,"ff":487,"ich":488,"pl":489,"ther":490,"Ġtr":491,"..":492,"Ġint":493,"ie":494,"ure":495,"age":496,"Ġne":497,"ial":498,"ap":499,"ine":500,"ice":501,"Ġme":502,"Ġout":503,"ans":504,"one":505,"ong":506,"ions":507,"Ġwho":508,"ĠK":509,"Ġup":510,"Ġtheir":511,"Ġad":512,"Ġ3":513,"Ġus":514,"ated":515,"ous":516,"Ġmore":517,"ue":518,"og":519,"ĠSt":520,"ind":521,"ike":522,"Ġso":523,"ime":524,"per":525,".\"":526,"ber":527,"iz":528,"act":529,"Ġone":530,"Ġsaid":531,"Ġ-":532,"are":533,"Ġyour":534,"cc":535,"ĠTh":536,"Ġcl":537,"ep":538,"ake":539,"able":540,"ip":541,"Ġcont":542,"Ġwhich":543,"ia":544,"Ġim":545,"Ġabout":546,"Ġwere":547,"very":548,"ub":549,"Ġhad":550,"Ġen":551,"Ġcomp":552,",\"":553,"ĠIn":554,"Ġun":555,"Ġag":556,"ire":557,"ace":558,"au":559,"ary":560,"Ġwould":561,"ass":562,"ry":563,"ĠâĢ":564,"cl":565,"ook":566,"ere":567,"so":568,"ĠV":569,"ign":570,"ib":571,"Ġoff":572,"Ġte":573,"ven":574,"ĠY":575,"ile":576,"ose":577,"ite":578,"orm":579,"Ġ201":580,"Ġres":581,"Ġman":582,"Ġper":583,"Ġother":584,"ord":585,"ult":586,"Ġbeen":587,"Ġlike":588,"ase":589,"ance":590,"ks":591,"ays":592,"own":593,"ence":594,"Ġdis":595,"ction":596,"Ġany":597,"Ġapp":598,"Ġsp":599,"int":600,"ress":601,"ations":602,"ail":603,"Ġ4":604,"ical":605,"Ġthem":606,"Ġher":607,"ount":608,"ĠCh":609,"Ġar":610,"Ġif":611,"Ġthere":612,"Ġpe":613,"Ġyear":614,"av":615,"Ġmy":616,"Ġsome":617,"Ġwhen":618,"ough":619,"ach":620,"Ġthan":621,"ru":622,"ond":623,"ick":624,"Ġover":625,"vel":626,"Ġqu":627,"ĊĊ":628,"Ġsc":629,"reat":630,"ree":631,"ĠIt":632,"ound":633,"port":634,"Ġalso":635,"Ġpart":636,"fter":637,"Ġkn":638,"Ġbec":639,"Ġtime":640,"ens":641,"Ġ5":642,"ople":643,"Ġwhat":644,"Ġno":645,"du":646,"mer":647,"ang":648,"Ġnew":649,"----":650,"Ġget":651,"ory":652,"ition":653,"ings":654,"Ġjust":655,"Ġinto":656,"Ġ0":657,"ents":658,"ove":659,"te":660,"Ġpeople":661,"Ġpre":662,"Ġits":663,"Ġrec":664,"Ġtw":665,"ian":666,"irst":667,"ark":668,"ors":669,"Ġwork":670,"ade":671,"ob":672,"Ġshe":673,"Ġour":674,"wn":675,"ink":676,"lic":677,"Ġ19":678,"ĠHe":679,"ish":680,"nder":681,"ause":682,"Ġhim":683,"ons":684,"Ġ[":685,"Ġro":686,"form":687,"ild":688,"ates":689,"vers":690,"Ġonly":691,"oll":692,"Ġspe":693,"ck":694,"ell":695,"amp":696,"Ġacc":697,"Ġbl":698,"ious":699,"urn":700,"ft":701,"ood":702,"Ġhow":703,"hed":704,"Ġ'":705,"Ġafter":706,"aw":707,"Ġatt":708,"ov":709,"ne":710,"Ġplay":711,"erv":712,"ict":713,"Ġcould":714,"itt":715,"Ġam":716,"Ġfirst":717,"Ġ6":718,"Ġact":719,"Ġ$":720,"ec":721,"hing":722,"ual":723,"ull":724,"Ġcomm":725,"oy":726,"old":727,"ces":728,"ater":729,"Ġfe":730,"Ġbet":731,"we":732,"iff":733,"Ġtwo":734,"ock":735,"Ġback":736,").":737,"ident":738,"Ġunder":739,"rough":740,"sel":741,"xt":742,"Ġmay":743,"round":744,"Ġpo":745,"ph":746,"iss":747,"Ġdes":748,"Ġmost":749,"Ġdid":750,"Ġadd":751,"ject":752,"Ġinc":753,"fore":754,"Ġpol":755,"ont":756,"Ġagain":757,"clud":758,"tern":759,"Ġknow":760,"Ġneed":761,"Ġcons":762,"Ġco":763,"Ġ.":764,"Ġwant":765,"Ġsee":766,"Ġ7":767,"ning":768,"iew":769,"ĠThis":770,"ced":771,"Ġeven":772,"Ġind":773,"ty":774,"ĠWe":775,"ath":776,"Ġthese":777,"Ġpr":778,"Ġuse":779,"Ġbecause":780,"Ġfl":781,"ng":782,"Ġnow":783,"ĠâĢĵ":784,"com":785,"ise":786,"Ġmake":787,"Ġthen":788,"ower":789,"Ġevery":790,"ĠUn":791,"Ġsec":792,"oss":793,"uch":794,"Ġem":795,"Ġ=":796,"ĠRe":797,"ied":798,"rit":799,"Ġinv":800,"lect":801,"Ġsupp":802,"ating":803,"Ġlook":804,"man":805,"pect":806,"Ġ8":807,"row":808,"Ġbu":809,"Ġwhere":810,"ific":811,"Ġyears":812,"ily":813,"Ġdiff":814,"Ġshould":815,"Ġrem":816,"Th":817,"In":818,"Ġev":819,"day":820,"'re":821,"rib":822,"Ġrel":823,"ss":824,"Ġdef":825,"Ġright":826,"Ġsy":827,"),":828,"les":829,"000":830,"hen":831,"Ġthrough":832,"ĠTr":833,"__":834,"Ġway":835,"Ġdon":836,"Ġ,":837,"Ġ10":838,"ased":839,"Ġass":840,"ublic":841,"Ġreg":842,"ĠAnd":843,"ix":844,"Ġvery":845,"Ġinclud":846,"other":847,"Ġimp":848,"oth":849,"Ġsub":850,"ĠâĢĶ":851,"Ġbeing":852,"arg":853,"ĠWh":854,"==":855,"ible":856,"Ġdoes":857,"ange":858,"ram":859,"Ġ9":860,"ert":861,"ps":862,"ited":863,"ational":864,"Ġbr":865,"Ġdown":866,"Ġmany":867,"aking":868,"Ġcall":869,"uring":870,"ities":871,"Ġph":872,"ics":873,"als":874,"Ġdec":875,"ative":876,"ener":877,"Ġbefore":878,"ility":879,"Ġwell":880,"Ġmuch":881,"erson":882,"Ġthose":883,"Ġsuch":884,"Ġke":885,"Ġend":886,"ĠBut":887,"ason":888,"ting":889,"Ġlong":890,"ef":891,"Ġthink":892,"ys":893,"Ġbel":894,"Ġsm":895,"its":896,"ax":897,"Ġown":898,"Ġprov":899,"Ġset":900,"ife":901,"ments":902,"ble":903,"ward":904,"Ġshow":905,"Ġpres":906,"ms":907,"omet":908,"Ġob":909,"Ġsay":910,"ĠSh":911,"ts":912,"ful":913,"Ġeff":914,"Ġgu":915,"Ġinst":916,"und":917,"ren":918,"cess":919,"Ġent":920,"ĠYou":921,"Ġgood":922,"Ġstart":923,"ince":924,"Ġmade":925,"tt":926,"stem":927,"olog":928,"up":929,"Ġ|":930,"ump":931,"Ġhel":932,"vern":933,"ular":934,"ually":935,"Ġac":936,"Ġmon":937,"Ġlast":938,"Ġ200":939,"10":940,"Ġstud":941,"ures":942,"ĠAr":943,"self":944,"ars":945,"meric":946,"ues":947,"cy":948,"Ġmin":949,"ollow":950,"Ġcol":951,"io":952,"Ġmod":953,"Ġcount":954,"ĠCom":955,"hes":956,"Ġfin":957,"air":958,"ier":959,"âĢĶ":960,"read":961,"ank":962,"atch":963,"ever":964,"Ġstr":965,"Ġpoint":966,"ork":967,"ĠNew":968,"Ġsur":969,"ool":970,"alk":971,"ement":972,"Ġused":973,"ract":974,"ween":975,"Ġsame":976,"oun":977,"ĠAl":978,"ci":979,"Ġdiffere":980,"Ġwhile":981,"--------":982,"Ġgame":983,"cept":984,"Ġsim":985,"...":986,"Ġinter":987,"ek":988,"Ġreport":989,"Ġprodu":990,"Ġstill":991,"led":992,"ah":993,"Ġhere":994,"Ġworld":995,"Ġthough":996,"Ġnum":997,"arch":998,"imes":999,"ale":1000,"ĠSe":1001,"ĠIf":1002,"//":1003,"ĠLe":1004,"Ġret":1005,"Ġref":1006,"Ġtrans":1007,"ner":1008,"ution":1009,"ters":1010,"Ġtake":1011,"ĠCl":1012,"Ġconf":1013,"way":1014,"ave":1015,"Ġgoing":1016,"Ġsl":1017,"ug":1018,"ĠAmeric":1019,"Ġspec":1020,"Ġhand":1021,"Ġbetween":1022,"ists":1023,"ĠDe":1024,"oot":1025,"It":1026,"Ġear":1027,"Ġagainst":1028,"Ġhigh":1029,"gan":1030,"az":1031,"ather":1032,"Ġexp":1033,"Ġop":1034,"Ġins":1035,"Ġgr":1036,"Ġhelp":1037,"Ġrequ":1038,"ets":1039,"ins":1040,"ĠPro":1041,"ism":1042,"Ġfound":1043,"land":1044,"ata":1045,"uss":1046,"ames":1047,"Ġperson":1048,"Ġgreat":1049,"pr":1050,"Ġsign":1051,"ĠAn":1052,"'ve":1053,"Ġsomet":1054,"Ġser":1055,"hip":1056,"Ġrun":1057,"Ġ:":1058,"Ġter":1059,"irect":1060,"Ġfollow":1061,"Ġdet":1062,"ices":1063,"Ġfind":1064,"12":1065,"Ġmem":1066,"Ġcr":1067,"ered":1068,"ex":1069,"Ġext":1070,"uth":1071,"ense":1072,"co":1073,"Ġteam":1074,"ving":1075,"ouse":1076,"ash":1077,"att":1078,"ved":1079,"Ġsystem":1080,"ĠAs":1081,"der":1082,"ives":1083,"min":1084,"Ġlead":1085,"ĠBl":1086,"cent":1087,"Ġaround":1088,"Ġgovern":1089,"Ġcur":1090,"velop":1091,"any":1092,"Ġcour":1093,"alth":1094,"ages":1095,"ize":1096,"Ġcar":1097,"ode":1098,"Ġlaw":1099,"Ġread":1100,"'m":1101,"con":1102,"Ġreal":1103,"Ġsupport":1104,"Ġ12":1105,"....":1106,"Ġreally":1107,"ness":1108,"Ġfact":1109,"Ġday":1110,"Ġboth":1111,"ying":1112,"Ġserv":1113,"ĠFor":1114,"Ġthree":1115,"Ġwom":1116,"Ġmed":1117,"ody":1118,"ĠThey":1119,"50":1120,"Ġexper":1121,"ton":1122,"Ġeach":1123,"akes":1124,"Ġche":1125,"Ġcre":1126,"ines":1127,"Ġrep":1128,"19":1129,"gg":1130,"illion":1131,"Ġgrou":1132,"ute":1133,"ik":1134,"We":1135,"get":1136,"ER":1137,"Ġmet":1138,"Ġsays":1139,"ox":1140,"Ġduring":1141,"ern":1142,"ized":1143,"ared":1144,"Ġfam":1145,"ically":1146,"Ġhapp":1147,"ĠIs":1148,"Ġchar":1149,"med":1150,"vent":1151,"Ġgener":1152,"ient":1153,"ple":1154,"iet":1155,"rent":1156,"11":1157,"ves":1158,"ption":1159,"Ġ20":1160,"formation":1161,"Ġcor":1162,"Ġoffic":1163,"ield":1164,"Ġtoo":1165,"ision":1166,"Ġinf":1167,"ĠZ":1168,"the":1169,"oad":1170,"Ġpublic":1171,"Ġprog":1172,"ric":1173,"**":1174,"Ġwar":1175,"Ġpower":1176,"view":1177,"Ġfew":1178,"Ġloc":1179,"Ġdifferent":1180,"Ġstate":1181,"Ġhead":1182,"'ll":1183,"Ġposs":1184,"Ġstat":1185,"ret":1186,"ants":1187,"Ġval":1188,"Ġiss":1189,"Ġcle":1190,"ivers":1191,"anc":1192,"Ġexpl":1193,"Ġanother":1194,"ĠQ":1195,"Ġav":1196,"thing":1197,"nce":1198,"Wh":1199,"Ġchild":1200,"Ġsince":1201,"ired":1202,"less":1203,"Ġlife":1204,"Ġdevelop":1205,"ittle":1206,"Ġdep":1207,"Ġpass":1208,"ãĥ":1209,"Ġturn":1210,"orn":1211,"This":1212,"bers":1213,"ross":1214,"ĠAd":1215,"Ġfr":1216,"Ġresp":1217,"Ġsecond":1218,"oh":1219,"Ġ/":1220,"Ġdisc":1221,"Ġ&":1222,"Ġsomething":1223,"Ġcomple":1224,"Ġed":1225,"Ġfil":1226,"Ġmonth":1227,"aj":1228,"uc":1229,"Ġgovernment":1230,"Ġwithout":1231,"Ġleg":1232,"Ġdist":1233,"Ġput":1234,"Ġquest":1235,"ann":1236,"Ġprot":1237,"20":1238,"Ġnever":1239,"ience":1240,"Ġlevel":1241,"Ġart":1242,"Ġthings":1243,"Ġmight":1244,"Ġeffect":1245,"Ġcontro":1246,"Ġcent":1247,"Ġ18":1248,"Ġallow":1249,"Ġbelie":1250,"chool":1251,"ott":1252,"Ġincre":1253,"Ġfeel":1254,"Ġresult":1255,"Ġlot":1256,"Ġfun":1257,"ote":1258,"Ġty":1259,"erest":1260,"Ġcontin":1261,"Ġusing":1262,"Ġbig":1263,"201":1264,"Ġask":1265,"Ġbest":1266,"Ġ)":1267,"IN":1268,"Ġopp":1269,"30":1270,"Ġnumber":1271,"iness":1272,"St":1273,"lease":1274,"Ġca":1275,"Ġmust":1276,"Ġdirect":1277,"Ġgl":1278,"Ġ<":1279,"Ġopen":1280,"Ġpost":1281,"Ġcome":1282,"Ġseem":1283,"ording":1284,"Ġweek":1285,"ately":1286,"ital":1287,"Ġel":1288,"riend":1289,"Ġfar":1290,"Ġtra":1291,"inal":1292,"Ġpri":1293,"ĠUS":1294,"Ġplace":1295,"Ġform":1296,"Ġtold":1297,"\":":1298,"ains":1299,"ature":1300,"ĠTrump":1301,"Ġstand":1302,"Ġ#":1303,"ider":1304,"ĠFr":1305,"Ġnext":1306,"Ġsoc":1307,"Ġpur":1308,"Ġlet":1309,"Ġlittle":1310,"Ġhum":1311,"Ġi":1312,"ron":1313,"15":1314,"Ġ15":1315,"Ġcommun":1316,"Ġmark":1317,"ĠThere":1318,"Ġwr":1319,"ĠThat":1320,"Ġinformation":1321,"ways":1322,"Ġbus":1323,"app":1324,"Ġinvest":1325,"me":1326,"Ġhard":1327,"ained":1328,"ead":1329,"Ġimport":1330,"Ġappro":1331,"Ġtest":1332,"Ġtri":1333,"Ġrest":1334,"osed":1335,"Ġfull":1336,"Ġcare":1337,"ĠSp":1338,"Ġcase":1339,"ON":1340,"Ġsk":1341,"Ġless":1342,"Ġ+":1343,"Ġpartic":1344,"ĠPl":1345,"ably":1346,"uck":1347,"ished":1348,"chn":1349,"be":1350,"Ġlist":1351,"ator":1352,"Ġtop":1353,"Ġadv":1354,"ĠBe":1355,"ruct":1356,"Ġdem":1357,"ration":1358,"ling":1359,"gy":1360,"reen":1361,"ger":1362,"Ġhome":1363,"Ġleft":1364,"Ġbetter":1365,"Ġdata":1366,"Ġ11":1367,"Ġattack":1368,"Ġproble":1369,"line":1370,"ards":1371,"Ġbeh":1372,"ral":1373,"ĠHow":1374,"ĠShe":1375,"arge":1376,"Ġ--":1377,"://":1378,"Ġbro":1379,"ĠPh":1380,"ats":1381,"Ġbuild":1382,"ww":1383,"ided":1384,"aim":1385,"ases":1386,"ency":1387,"Ġmain":1388,"ined":1389,"Ġincluding":1390,"Ġ{":1391,"Ġgot":1392,"Ġinterest":1393,"Ġkeep":1394,"ĠX":1395,"Ġeas":1396,"aining":1397,"Ġclass":1398,"âĢ¦":1399,"ĠNo":1400,"Ġvar":1401,"Ġsmall":1402,"ample":1403,"AT":1404,"Ġide":1405,"ĠSo":1406,"Ġrece":1407,"Ġpolit":1408,"Ġmov":1409,"Ġplan":1410,"Ġpercent":1411,"iving":1412,"Ġcamp":1413,"Ġpay":1414,"14":1415,"sc":1416,"ised":1417,"Ġunt":1418,"oney":1419,"ploy":1420,"====":1421,"Ġdidn":1422,"ĠInd":1423,"els":1424,"ertain":1425,"Ġpos":1426,"____":1427,"iver":1428,"Ġprocess":1429,"Ġprogram":1430,"ified":1431,"ĠRep":1432,"16":1433,"uro":1434,"ology":1435,"atter":1436,"ina":1437,"Ġname":1438,"ĠAll":1439,"Ġfour":1440,"Ġreturn":1441,"vious":1442,"bs":1443,"Ġcalled":1444,"Ġmove":1445,"ĠSc":1446,"ird":1447,"Ġgroup":1448,"Ġbre":1449,"Ġmen":1450,"Ġcap":1451,"ten":1452,"ee":1453,"Ġdri":1454,"leg":1455,"here":1456,"uthor":1457,"Ġpat":1458,"Ġcurrent":1459,"ides":1460,"Ġpop":1461,"to":1462,"ention":1463,"Ġalways":1464,"Ġmil":1465,"Ġwomen":1466,"Ġ16":1467,"Ġold":1468,"iven":1469,"raph":1470,"ĠOr":1471,"ror":1472,"ently":1473,"Ġnear":1474,"ĠEx":1475,"ream":1476,"sh":1477,"Ġ14":1478,"Ġfree":1479,"ission":1480,"stand":1481,"ĠCon":1482,"ality":1483,"used":1484,"13":1485,"Ġdesign":1486,"Ġchange":1487,"Ġchang":1488,"Ġbo":1489,"Ġvis":1490,"ember":1491,"Ġbook":1492,"ready":1493,"Ġkill":1494,"25":1495,"pped":1496,"Ġaway":1497,"Ġable":1498,"Ġcountry":1499,"Ġconst":1500,"arn":1501,"Ġorder":1502,"AR":1503,"ior":1504,"ium":1505,"orth":1506,"18":1507,"ailable":1508,"Ġsw":1509,"Ġmillion":1510,"Ġ13":1511,"atic":1512,"ted":1513,"ĠGo":1514,"Ġoper":1515,"eng":1516,"Ġthing":1517,"ajor":1518,"conom":1519,"ĠComm":1520,"Ġwhy":1521,"ured":1522,"ural":1523,"Ġschool":1524,"by":1525,"ĠMar":1526,"Ġaff":1527,"Ġdays":1528,"Ġann":1529,"ush":1530,"ane":1531,"If":1532,"eg":1533,"Ġprof":1534,"Ġhealth":1535,"outh":1536,"But":1537,"ional":1538,".,":1539,"Ġsol":1540,"Ġalready":1541,"Ġ30":1542,"Ġcharact":1543,"He":1544,"Ġfriend":1545,"ES":1546,"ians":1547,"icle":1548,"'d":1549,"ĠOn":1550,"Ġleast":1551,"Ġprom":1552,"Ġdr":1553,"Ġhist":1554,"ither":1555,"Ġest":1556,"iqu":1557,"17":1558,"son":1559,"Ġtell":1560,"Ġtalk":1561,"ohn":1562,"oint":1563,"lection":1564,"AN":1565,"Ġuntil":1566,"augh":1567,"Ġlater":1568,"Ġve":1569,"Ġview":1570,"ending":1571,"ived":1572,"Ġword":1573,"ware":1574,"Ġcost":1575,"Ġenough":1576,"Ġgive":1577,"ĠUnited":1578,"Ġtechn":1579,"arent":1580,"OR":1581,"Ġpar":1582,"ĠDr":1583,"Ġ2016":1584,"rist":1585,"ering":1586,"ĠÂ":1587,"Ġlarge":1588,"side":1589,"acy":1590,"ccess":1591,"Ġwin":1592,"Ġimportant":1593,"Ġ199":1594,"Ġdoesn":1595,"Ġ17":1596,"Ġbusiness":1597,"Ġclear":1598,"Ġrese":1599,"\",":1600,"ury":1601,"Ġequ":1602,"aster":1603,"alf":1604,"ĠAmerican":1605,"nect":1606,"Ġexpect":1607,"iversity":1608,"Ġocc":1609,"ĠFl":1610,"Ġkind":1611,"Ġmean":1612,"Ġpast":1613,"Ġdev":1614,"Ġbas":1615,"let":1616,"raft":1617,"Ġorgan":1618,"Ġdel":1619,"Ġperform":1620,"Ġstory":1621,"Ġseason":1622,"ĠCol":1623,"Ġclaim":1624,"Ġcame":1625,"Ġwithin":1626,"Ġline":1627,"Ġproject":1628,"ĠAt":1629,"Ġcontrol":1630,"ended":1631,"ĠSy":1632,"Ġair":1633,"ization":1634,"Ġ*":1635,"ley":1636,"Ġmoney":1637,"idd":1638,"You":1639,"for":1640,"Ġfamily":1641,"Ġmaking":1642,"Ġbit":1643,"Ġpolice":1644,"Ġhappen":1645,"Ġvers":1646,"ony":1647,"uff":1648,"ĠWhen":1649,"Ġsit":1650,"ideo":1651,"lf":1652,"ison":1653,"Ġsure":1654,"gin":1655,"Ġappear":1656,"Ġlight":1657,"Ġes":1658,"of":1659,"Ġwater":1660,"Ġtimes":1661,"not":1662,"Ġgrow":1663,"Ġcompany":1664,"ĠTe":1665,"ows":1666,"Ġmar":1667,"ource":1668,"iol":1669,"arm":1670,"br":1671,"Ġexample":1672,"Ġconc":1673,"Ġfore":1674,"ĠTo":1675,"pro":1676,"EN":1677,"ries":1678,"Ġ25":1679,"ĠCan":1680,"ney":1681,"Ġactually":1682,"Ġever":1683,"urity":1684,"aken":1685,"aps":1686,"Ġtax":1687,"Ġmajor":1688,"ama":1689,"Ġoften":1690,"eral":1691,"Ġhuman":1692,"Ġjob":1693,"ister":1694,"Ġavailable":1695,"ocr":1696,"enn":1697,"aid":1698,"ivid":1699,"Ġrecord":1700,"?\"":1701,"Ġsing":1702,"ĠAm":1703,"idence":1704,"Ġnews":1705,"ster":1706,"Ġeconom":1707,"Ġfollowing":1708,"ĠBr":1709,"ising":1710,"Ġhour":1711,"most":1712,"ument":1713,"Ġsex":1714,"Ġdesc":1715,"Ġbecome":1716,"ĠEd":1717,"Ġtook":1718,"Ġhaving":1719,"Ġproduct":1720,"ault":1721,"As":1722,"aring":1723,"Ġmeans":1724,"Ġhop":1725,"une":1726,"Ġcho":1727,"Ġcertain":1728,"Ġnon":1729,"Ġdeal":1730,"24":1731,"lement":1732,"oci":1733,"ene":1734,"Ġside":1735,"ĠPr":1736,"ĠMay":1737,"Ġreason":1738,"ued":1739,"ched":1740,"ulation":1741,"Ġelect":1742,"Ġofficial":1743,"Ġpossible":1744,"Ġhold":1745,"ands":1746,"ots":1747,"Ġcity":1748,"ories":1749,"Ġsever":1750,"Ġchildren":1751,"Ġonce":1752,"Ġactiv":1753,"ler":1754,"Ġnight":1755,"itions":1756,"ĠJohn":1757,"ape":1758,"play":1759,"Ġdone":1760,"Ġlim":1761,"Ġworking":1762,"ĠPres":1763,"orld":1764,"eb":1765,"ĠCo":1766,"Ġbody":1767,"ails":1768,"utes":1769,"ĠMr":1770,"Ġwhether":1771,"Ġauthor":1772,"rop":1773,"Ġproper":1774,"Ġseen":1775,");":1776,"Ġfac":1777,"ĠSu":1778,"Ġcond":1779,"iting":1780,"Ġcourse":1781,"Ġ}":1782,"----------------":1783,"aign":1784,"Ġevent":1785,"Ġeng":1786,"Ġpot":1787,"Ġintern":1788,"iam":1789,"Ġshort":1790,"empt":1791,"ãĤ":1792,"ĠGod":1793,"ilar":1794,"80":1795,"Ġorig":1796,"IS":1797,"ourn":1798,"ability":1799,"itive":1800,"Ġdam":1801,"Ġ100":1802,"Ġpress":1803,"Ġdoing":1804,"Ġprotect":1805,"ring":1806,"Ġthought":1807,"Ġquestion":1808,"rew":1809,"ĠWar":1810,"Ġseveral":1811,"ĠState":1812,"Ġgiven":1813,"Ġfund":1814,"ĠTw":1815,"Ġwent":1816,"ances":1817,"work":1818,"por":1819,"my":1820,"40":1821,"Ġarg":1822,"artment":1823,"ustom":1824,"Ġpolic":1825,"Ġmeet":1826,"Ġcreat":1827,"22":1828,"ĠStates":1829,"Ġgames":1830,"raw":1831,"uture":1832,"Ġunderstand":1833,"urs":1834,"ĠOb":1835,"lish":1836,"sy":1837,"Ġmakes":1838,"Ġwon":1839,"agon":1840,"Ġhtt":1841,"Ġlove":1842,"ential":1843,"Ġcomplete":1844,"par":1845,"ĠIm":1846,"AL":1847,"Ġaccount":1848,"Âł":1849,"ored":1850,"vert":1851,"Ġident":1852,"Ġ2015":1853,"Ġothers":1854,"ĠMin":1855,"iber":1856,"verage":1857,"There":1858,"itional":1859,"dd":1860,"Ġprob":1861,"Ġyoung":1862,"Ġalong":1863,"Ġaccording":1864,"Ġyet":1865,"Ġmembers":1866,"ĠWhat":1867,"oid":1868,"ĠMan":1869,"And":1870,"Ġamong":1871,"ai":1872,"Ġemploy":1873,"ĠRes":1874,"Ġ>":1875,"Ġinvol":1876,"Ġlow":1877,"af":1878,"ĠCar":1879,"Ġhig":1880,"ĠOne":1881,"ĠSec":1882,"ination":1883,"Ġlikely":1884,"Ġant":1885,"aged":1886,"ĠRuss":1887,"Ġben":1888,"Ġrele":1889,"For":1890,"back":1891,"ĠNot":1892,"Ġpresident":1893,"ball":1894,"Ġaccess":1895,"ividual":1896,"ĠDem":1897,"ĠEuro":1898,"60":1899,"Ġknown":1900,"irl":1901,"ĠGr":1902,"Ġearly":1903,"use":1904,"iety":1905,"âĢĵ":1906,"Ġfight":1907,"Ġsent":1908,"Ġtoday":1909,"Ġmarket":1910,"\".":1911,"Ġbased":1912,"Ġstrong":1913,"urther":1914,"Ġdeb":1915,"mber":1916,"Ġproblem":1917,"Ġdeath":1918,"Ġsocial":1919,"imate":1920,"AS":1921,"ortun":1922,"Ġcampaign":1923,"ery":1924,"Ch":1925,"Ġey":1926,"ially":1927,"Ġmus":1928,"wh":1929,"pos":1930,"Ġer":1931,"Ġsaf":1932,"Ġmonths":1933,"iron":1934,"Ġviol":1935,"Ġfive":1936,"Ġstre":1937,"Ġplayers":1938,"inc":1939,"ald":1940,"year":1941,"aun":1942,"Ġsuccess":1943,"Ġpresent":1944,"erence":1945,"Ġ2014":1946,"Ġsugg":1947,"Ġparticular":1948,"Ġtry":1949,"Ġsuggest":1950,"ĠChrist":1951,"ones":1952,"Ġpriv":1953,"23":1954,"Ġcrit":1955,"Ġland":1956,"Ġlocal":1957,"ify":1958,"29":1959,"Ġaut":1960,"ED":1961,"ĠGu":1962,"Ġmult":1963,"Ġpolitical":1964,"Ġasked":1965,"Ġformer":1966,"itter":1967,"ript":1968,"Ġclose":1969,"Ġpract":1970,"ĠYork":1971,"Ġgetting":1972,"Ġacross":1973,"Ġcomb":1974,"Ġbelieve":1975,"Ġz":1976,"Ġtoget":1977,"Ġtogether":1978,"ĠCent":1979,"irc":1980,"Ġindividual":1981,"ĠMc":1982,"27":1983,"isk":1984,"ĠEng":1985,"Ġface":1986,"Ġ24":1987,"Ġvalue":1988,"Ġarea":1989,"ev":1990,"Ġwrit":1991,"ĠPresident":1992,"Ġvot":1993,"Ġkey":1994,"Ġmom":1995,"put":1996,"Ġanything":1997,"Ġexperience":1998,"attle":1999,"Ġmind":2000,"aff":2001,"omm":2002,"Ġfuture":2003,"ged":2004,"Ġcut":2005,"Ġtot":2006,"itch":2007,"Ġvideo":2008,"Ġinvestig":2009,"Ġnet":2010,"ĠMy":2011,"rict":2012,"ien":2013,".)":2014,"Ġimpro":2015,"though":2016,"wards":2017,"Ġconnect":2018,"ĠMed":2019,"selves":2020,"ensive":2021,"mb":2022,"ober":2023,"ators":2024,"An":2025,"Ġ50":2026,"Ġredu":2027,"resent":2028,"Ġabove":2029,"Ġfre":2030,"ĠEurope":2031,"sw":2032,"Ġamount":2033,"ĠApp":2034,"Ġeither":2035,"Ġmilit":2036,"Ġanal":2037,"Ġfail":2038,"ĠEn":2039,"ales":2040,"Ġspecial":2041,"Ġblack":2042,"IT":2043,"cher":2044,"Ġlooking":2045,"Ġfire":2046,"yn":2047,"Ġalmost":2048,"oon":2049,"Ġstudy":2050,"Ġmiss":2051,"ches":2052,"rown":2053,"Ġtre":2054,"Ġcommunity":2055,"Ġmedia":2056,"Ġfood":2057,"Ġcomes":2058,"ĠUniversity":2059,"Ġsingle":2060,"What":2061,"uly":2062,"Ġhalf":2063,"ague":2064,"hod":2065,"ĠRepublic":2066,"Ġstarted":2067,"Ġquick":2068,"oto":2069,"book":2070,"Ġissue":2071,"itor":2072,"Ġelse":2073,"Ġconsider":2074,"26":2075,"rodu":2076,"Ġtaken":2077,"28":2078,"99":2079,"ĠWith":2080,"Ġtrue":2081,"Ġwa":2082,"Ġtrad":2083,"Ġago":2084,"Ġmess":2085,"ief":2086,"Ġadded":2087,"oke":2088,"Ġbad":2089,"Ġfav":2090,"33":2091,"Ġsimilar":2092,"ask":2093,"ĠDon":2094,"Ġcharacter":2095,"orts":2096,"ĠHouse":2097,"Ġreported":2098,"Ġtype":2099,"val":2100,"iod":2101,"ĠHowever":2102,"Ġtarg":2103,"Ġentire":2104,"pping":2105,"Ġhistory":2106,"Ġlive":2107,"ffic":2108,"........":2109,"ederal":2110,"Ġtrying":2111,"Ġdiscuss":2112,"ĠHar":2113,"aces":2114,"lished":2115,"Ġself":2116,"osp":2117,"rest":2118,"Ġroom":2119,"elt":2120,"Ġfall":2121,"olution":2122,"Ġet":2123,"Ġx":2124,"Ġisn":2125,"Ġidea":2126,"bo":2127,"Ġsound":2128,"ĠDep":2129,"Ġsomeone":2130,"cially":2131,"ully":2132,"Ġfoc":2133,"Ġobject":2134,"ift":2135,"aper":2136,"Ġplayer":2137,"Ġrather":2138,"Ġservice":2139,"ashing":2140,"ĠDo":2141,"ĠPart":2142,"rug":2143,"mon":2144,"ply":2145,"Ġmor":2146,"Ġnothing":2147,"Ġprovide":2148,"IC":2149,"ung":2150,"Ġparty":2151,"Ġexist":2152,"Ġmag":2153,"70":2154,"Ġrul":2155,"Ġhouse":2156,"Ġbehind":2157,"Ġhowever":2158,"ĠWorld":2159,"Ġsum":2160,"Ġapplic":2161,"Ġ;":2162,"Ġfunction":2163,"gr":2164,"ĠPol":2165,"Ġfront":2166,"200":2167,"Ġseries":2168,"Ġtem":2169,"Ġtyp":2170,"ills":2171,"Ġopt":2172,"Ġpoints":2173,"Ġbelow":2174,"itted":2175,"Ġspecific":2176,"Ġ2017":2177,"umb":2178,"Ġra":2179,"Ġprevious":2180,"Ġpret":2181,"reme":2182,"Ġcustom":2183,"Ġcourt":2184,"ĠMe":2185,"Ġrepl":2186,"Ġwhole":2187,"go":2188,"cer":2189,"Ġtreat":2190,"ĠAct":2191,"Ġprobably":2192,"Ġlearn":2193,"ender":2194,"ĠAss":2195,"Ġversion":2196,"now":2197,"Ġcheck":2198,"ĠCal":2199,"RE":2200,"minist":2201,"On":2202,"ources":2203,"Ġbenef":2204,"Ġdoc":2205,"Ġdeter":2206,"Ġenc":2207,"Ġsuper":2208,"Ġaddress":2209,"Ġvict":2210,"Ġ2013":2211,"Ġmeas":2212,"tr":2213,"Ġfield":2214,"When":2215,"Ġsignific":2216,"uge":2217,"Ġfeat":2218,"Ġcommon":2219,"load":2220,"Ġbegin":2221,"Ġbring":2222,"Ġaction":2223,"erman":2224,"Ġdescrib":2225,"Ġindust":2226,"Ġwanted":2227,"ried":2228,"ming":2229,"Ġattempt":2230,"45":2231,"fer":2232,"Ġdue":2233,"ression":2234,"##":2235,"Ġshall":2236,"Ġsix":2237,"oo":2238,"Ġstep":2239,"Ġpub":2240,"Ġhimself":2241,"Ġ23":2242,"Ġcop":2243,"Ġdest":2244,"Ġstop":2245,"AC":2246,"ibility":2247,"Ġlab":2248,"icult":2249,"Ġhours":2250,"Ġcreate":2251,"Ġfurther":2252,"ĠAmerica":2253,"ĠCity":2254,"Ġdou":2255,"head":2256,"ST":2257,"ĠNorth":2258,"cing":2259,"Ġnational":2260,"ule":2261,"ĠInst":2262,"Ġtaking":2263,"ĠQu":2264,"irt":2265,"Ġred":2266,"Ġresearch":2267,"viron":2268,"ĠGe":2269,"Ġbreak":2270,"ana":2271,"Ġspace":2272,"aterial":2273,"Ġrecent":2274,"ĠAb":2275,"Ġgeneral":2276,"Ġhit":2277,"Ġperiod":2278,"Ġeverything":2279,"ively":2280,"Ġphys":2281,"Ġsaying":2282,"anks":2283,"Ġcou":2284,"Ġcult":2285,"aced":2286,"eal":2287,"uation":2288,"Ġcoun":2289,"lu":2290,"Ġinclude":2291,"Ġposition":2292,"ĠAfter":2293,"ĠCanad":2294,"ĠEm":2295,"Ġimm":2296,"ĠRed":2297,"Ġpick":2298,"Ġcompl":2299,"Ġmatter":2300,"reg":2301,"ext":2302,"angu":2303,"isc":2304,"ole":2305,"aut":2306,"Ġcompet":2307,"eed":2308,"fect":2309,"Ġ21":2310,"ĠSen":2311,"ĠThese":2312,"asing":2313,"Ġcannot":2314,"Ġinit":2315,"Ġrelations":2316,"ached":2317,"Ġbar":2318,"Ġ40":2319,"ĠTH":2320,"Ġ2012":2321,"Ġvol":2322,"Ġground":2323,"Ġsecurity":2324,"Ġupd":2325,"ilt":2326,"35":2327,"Ġconcern":2328,"ĠJust":2329,"Ġwhite":2330,"Ġseems":2331,"ĠHer":2332,"pecially":2333,"ients":2334,"Ġannoun":2335,"Ġfig":2336,"ights":2337,"Ġstri":2338,"like":2339,"ids":2340,"Ġsus":2341,"Ġwatch":2342,"Ġâ":2343,"Ġwind":2344,"ĠCont":2345,"Ġitself":2346,"Ġmass":2347,"Al":2348,"yle":2349,"ique":2350,"ĠNational":2351,"Ġabs":2352,"Ġpack":2353,"Ġoutside":2354,"Ġanim":2355,"Ġpain":2356,"eter":2357,"Ġmanag":2358,"duct":2359,"ogn":2360,"Ġ]":2361,"ĠSept":2362,"sec":2363,"off":2364,"ĠJan":2365,"Ġfoot":2366,"ades":2367,"Ġthird":2368,"Ġmot":2369,"Ġevidence":2370,"inton":2371,"Ġthreat":2372,"apt":2373,"ples":2374,"cle":2375,"Ġlo":2376,"Ġdecl":2377,"Ġitem":2378,"medi":2379,"Ġrepresent":2380,"omb":2381,"amer":2382,"Ġsignificant":2383,"ograph":2384,"su":2385,"Ġcal":2386,"ires":2387,"0000":2388,"ID":2389,"AM":2390,"Ġsimply":2391,"Ġlonger":2392,"Ġfile":2393,"OT":2394,"che":2395,"So":2396,"ateg":2397,"org":2398,"ĠHis":2399,"Ġener":2400,"Ġdom":2401,"Ġupon":2402,"ili":2403,"\":\"":2404,"Ġthemselves":2405,"Ġcoming":2406,"Ġquite":2407,"Ġdifficult":2408,"ĠBar":2409,"ilities":2410,"rel":2411,"ends":2412,"cial":2413,"64":2414,"Ġwoman":2415,"rap":2416,"yr":2417,"Ġnecess":2418,"ips":2419,"Ġtext":2420,"Ġrequire":2421,"Ġmilitary":2422,"Ġreview":2423,"Ġrespons":2424,"75":2425,"Ġsubject":2426,"Ġinstead":2427,"Ġissues":2428,"Ġgen":2429,"\",\"":2430,"Ġminutes":2431,"Ġweap":2432,"ray":2433,"amed":2434,"time":2435,"bl":2436,"How":2437,"Ġcode":2438,"ĠSm":2439,"Ġhigher":2440,"ĠSte":2441,"ris":2442,"Ġpage":2443,"Ġstudents":2444,"ĠIntern":2445,"Ġmethod":2446,"ĠAug":2447,"ĠPer":2448,"ĠAg":2449,"Ġpolicy":2450,"ĠSw":2451,"Ġexec":2452,"Ġaccept":2453,"ume":2454,"ribut":2455,"Ġwords":2456,"Ġfinal":2457,"Ġchanges":2458,"ĠDemocr":2459,"Ġfriends":2460,"Ġrespect":2461,"Ġep":2462,"Ġcompan":2463,"ivil":2464,"Ġdamage":2465,"****":2466,"ogle":2467,"vironment":2468,"Ġneg":2469,"ental":2470,"Ġap":2471,"Ġtotal":2472,"ival":2473,"!\"":2474,"lim":2475,"Ġneeds":2476,"Ġagre":2477,"Ġdevelopment":2478,"Ġage":2479,"iple":2480,"21":2481,"Ġresults":2482,"ĠAf":2483,"Sh":2484,"Ġgun":2485,"ĠObama":2486,"roll":2487,"Ġ@":2488,"Ġrights":2489,"ĠBrit":2490,"Ġrunning":2491,"Ġwasn":2492,"Ġport":2493,"Ġrate":2494,"Ġpretty":2495,"Ġtarget":2496,"Ġsaw":2497,"Ġcirc":2498,"Ġworks":2499,"icro":2500,"alt":2501,"over":2502,"www":2503,"That":2504,"lier":2505,"Ġeveryone":2506,"ude":2507,"Ġpie":2508,"iddle":2509,"rael":2510,"Ġrad":2511,"Ġblock":2512,"Ġwalk":2513,"To":2514,"ãģ":2515,"nes":2516,"ĠAust":2517,"aul":2518,"rote":2519,"ĠSouth":2520,"ession":2521,"oph":2522,"Ġshows":2523,"Ġsite":2524,"Ġjo":2525,"Ġrisk":2526,"clus":2527,"lt":2528,"Ġinj":2529,"iding":2530,"ĠSpe":2531,"Ġchall":2532,"irm":2533,"Ġ22":2534,"itting":2535,"str":2536,"Ġhy":2537,"LE":2538,"key":2539,"Ġbegan":2540,"atur":2541,"ashington":2542,"lam":2543,"ĠDav":2544,"bit":2545,"Ġsize":2546,"ĠPar":2547,"38":2548,"ournal":2549,"face":2550,"Ġdecision":2551,"Ġlarg":2552,"Ġjud":2553,"rect":2554,"Ġcontinue":2555,"ĠOct":2556,"overed":2557,"ĠInt":2558,"========":2559,"Ġparent":2560,"ĠWill":2561,"Ġeasy":2562,"Ġdrug":2563,"anger":2564,"Ġsense":2565,"Ġdi":2566,"iday":2567,"Ġenergy":2568,"istic":2569,"Ġassoci":2570,"arter":2571,"obal":2572,"eks":2573,"ĠEl":2574,"urch":2575,"Ġgirl":2576,"oe":2577,"itle":2578,"Ġ28":2579,"ĠChe":2580,"Ġrequest":2581,"Ġsoon":2582,"Ġhost":2583,"ky":2584,"Ġstates":2585,"omes":2586,"Ġmaterial":2587,"lex":2588,"Ġmoment":2589,"Ġansw":2590,"onse":2591,"Ġespecially":2592,"Ġnorm":2593,"Ġservices":2594,"pite":2595,"ran":2596,"Ġrole":2597,"44":2598,"):":2599,"Ġcred":2600,"Cl":2601,"________":2602,"Ġmat":2603,"Ġlog":2604,"ĠClinton":2605,"OU":2606,"Ġoffice":2607,"Ġ26":2608,"Ġcharg":2609,"Ġtrack":2610,"ma":2611,"Ġheart":2612,"Ġball":2613,"Ġpersonal":2614,"Ġbuilding":2615,"na":2616,"set":2617,"body":2618,"ĠBlack":2619,"Ġincrease":2620,"itten":2621,"Ġneeded":2622,"36":2623,"32":2624,"=\"":2625,"Ġlost":2626,"Ġbecame":2627,"Ġgroups":2628,"ĠMus":2629,"Ġwrote":2630,"ĠPe":2631,"Ġprop":2632,"joy":2633,"é":2634,"ĠWhite":2635,"Ġdead":2636,".'":2637,"Ġhttp":2638,"Ġwebs":2639,"OS":2640,"Ġinside":2641,"Ġwrong":2642,"Ġstatement":2643,"Ġ...":2644,"yl":2645,"Ġfilm":2646,"Ġmusic":2647,"Ġshare":2648,"ification":2649,"Ġrelease":2650,"Ġforward":2651,"Ġstay":2652,"Ġcomput":2653,"itte":2654,"ser":2655,"Ġoriginal":2656,"Ġcard":2657,"Ġcand":2658,"Ġdiv":2659,"atural":2660,"Ġfavor":2661,"OM":2662,"Ġcases":2663,"uses":2664,"Ġsection":2665,"Ġleave":2666,"ging":2667,"oved":2668,"ĠWashington":2669,"39":2670,"ĠGl":2671,"Ġrequired":2672,"action":2673,"apan":2674,"oor":2675,"iter":2676,"ĠKing":2677,"Ġcountries":2678,"ĠGerman":2679,"lling":2680,"Ġ27":2681,"34":2682,"Ġquestions":2683,"Ġprim":2684,"Ġcell":2685,"Ġshoot":2686,"Ġanyone":2687,"ĠWest":2688,"Ġaffect":2689,"epend":2690,"Ġonline":2691,"ĠIsrael":2692,"ĠSeptember":2693,"Ġability":2694,"Ġcontent":2695,"ises":2696,"Ġreve":2697,"Ġlaun":2698,"Ġindic":2699,"Ġforce":2700,"cast":2701,"Ġsold":2702,"aving":2703,"fl":2704,"Ġsoft":2705,"Ġcompanies":2706,"ceed":2707,"Ġarticle":2708,"Ġaud":2709,"Ġrev":2710,"Ġeduc":2711,"Ġplaying":2712,"05":2713,"Ġheld":2714,"ctor":2715,"Ġreleased":2716,"Ġfederal":2717,"37":2718,"Ġadminist":2719,"Ġinterview":2720,"Ġinstall":2721,"Ġreceived":2722,"Ġsource":2723,"uk":2724,"Ph":2725,"Ġserious":2726,"Ġcreated":2727,"Ġcause":2728,"Ġimmedi":2729,"Ġdefin":2730,"uel":2731,"ĠDepartment":2732,"ctions":2733,"ĠCour":2734,"ĠNow":2735,"ze":2736,"ites":2737,"itution":2738,"Ġlate":2739,"Ġspeak":2740,"ners":2741,"Ġlegal":2742,"ari":2743,"ĠCor":2744,"Ġweeks":2745,"Ġmodel":2746,"Ġpred":2747,"Ġexact":2748,"BC":2749,"ĠBy":2750,"ING":2751,"osing":2752,"Ġtakes":2753,"Ġregard":2754,"Ġopportun":2755,"Ġprice":2756,"Ġ198":2757,"ĠApr":2758,"fully":2759,"Ġord":2760,"Ġproblems":2761,"ruction":2762,"ham":2763,"ĠCount":2764,"lege":2765,"Ġleaders":2766,"ET":2767,"lev":2768,"Ġdeep":2769,"ological":2770,"ese":2771,"haps":2772,"ĠSome":2773,"Ġpers":2774,"Ġcontract":2775,"Ġrelationship":2776,"sp":2777,"oud":2778,"Ġbase":2779,"48":2780,"mit":2781,"Ad":2782,"ancial":2783,"Ġconsum":2784,"Ġpotential":2785,"Ġlangu":2786,"rem":2787,"eth":2788,"Ġrelig":2789,"ressed":2790,"66":2791,"Ġlink":2792,"Ġlower":2793,"ayer":2794,"ĠJune":2795,"Ġfem":2796,"unt":2797,"erc":2798,"urd":2799,"Ġcontact":2800,"Ġill":2801,"Ġmother":2802,"Ġestab":2803,"htt":2804,"ĠMarch":2805,"ĠBro":2806,"ĠChina":2807,"Ġ29":2808,"Ġsqu":2809,"Ġprovided":2810,"Ġaverage":2811,"asons":2812,"Ġ2011":2813,"Ġexam":2814,"lin":2815,"55":2816,"ned":2817,"Ġperfect":2818,"Ġtou":2819,"alse":2820,"ux":2821,"Ġbuy":2822,"Ġshot":2823,"Ġcollect":2824,"Ġphot":2825,"Ġplayed":2826,"Ġsurpr":2827,"Ġofficials":2828,"Ġsimple":2829,"avy":2830,"Ġindustry":2831,"Ġhands":2832,"ground":2833,"Ġpull":2834,"Ġround":2835,"Ġuser":2836,"Ġrange":2837,"uary":2838,"Ġprivate":2839,"ops":2840,"ees":2841,"Ġways":2842,"ĠMich":2843,"Ġveh":2844,"Ġexcept":2845,"Ġterms":2846,"imum":2847,"pper":2848,"ION":2849,"ores":2850,"ĠDragon":2851,"oul":2852,"Ġden":2853,"Ġperformance":2854,"Ġbill":2855,"cil":2856,"47":2857,"Ġenvironment":2858,"Ġexc":2859,"add":2860,"Ġworth":2861,"Ġpict":2862,"Ġchance":2863,"Ġ2018":2864,"bor":2865,"Ġspeed":2866,"iction":2867,"Ġalleg":2868,"ĠJapan":2869,"atory":2870,"reet":2871,"Ġmatch":2872,"ĠII":2873,"Ġstru":2874,"order":2875,"Ġste":2876,"Ġliving":2877,"Ġstruct":2878,"ino":2879,"Ġsepar":2880,"hern":2881,"Ġresponse":2882,"Ġenjoy":2883,"Ġvia":2884,"AD":2885,"uments":2886,"acebook":2887,"Ġmember":2888,"ibr":2889,"izing":2890,"Ġtool":2891,"ĠMon":2892,"ĠWhile":2893,"hood":2894,"ĠAng":2895,"ĠDef":2896,"Ġoffer":2897,"Tr":2898,"aur":2899,"Ġturned":2900,"ĠJuly":2901,"down":2902,"anced":2903,"Ġrecently":2904,"ĠEar":2905,"Ġce":2906,"ĠStar":2907,"ĠCong":2908,"rought":2909,"Ġblood":2910,"Ġhope":2911,"Ġcomment":2912,"aint":2913,"Ġarri":2914,"iles":2915,"Ġparticip":2916,"ought":2917,"ription":2918,"08":2919,"49":2920,"Ġgave":2921,"Ġselect":2922,"Ġkilled":2923,"sych":2924,"Ġgoes":2925,"ij":2926,"Ġcoll":2927,"Ġimpact":2928,"atives":2929,"ĠSer":2930,"09":2931,"ĠAugust":2932,"Ġboy":2933,"de":2934,"ĠDes":2935,"Ġfelt":2936,"US":2937,"Ġexpected":2938,"Ġimage":2939,"ĠMark":2940,"ccording":2941,"oice":2942,"EC":2943,"ĠMag":2944,"ened":2945,"hold":2946,"ĠPost":2947,"Ġprevent":2948,"No":2949,"Ġinvolved":2950,"Ġeyes":2951,"Ġquickly":2952,"At":2953,"unk":2954,"Ġbehav":2955,"Ġur":2956,"Ġled":2957,"come":2958,"ey":2959,"Ġcandid":2960,"Ġearlier":2961,"Ġfocus":2962,"ety":2963,"Pro":2964,"ledge":2965,"ixed":2966,"illed":2967,"Ġpopular":2968,"AP":2969,"Ġsett":2970,"light":2971,"Ġvarious":2972,"inks":2973,"Ġlevels":2974,"Ġroad":2975,"ellig":2976,"ables":2977,"hel":2978,"ittee":2979,"ĠGener":2980,"ype":2981,"Ġheard":2982,"icles":2983,"Ġmis":2984,"Ġusers":2985,"ĠSan":2986,"Ġimprove":2987,"Ġfather":2988,"Ġsearch":2989,"They":2990,"vil":2991,"Ġprofess":2992,"Ġknew":2993,"Ġloss":2994,"Ġevents":2995,"65":2996,"Ġbillion":2997,"07":2998,"02":2999,"ĠNews":3000,"ĠAM":3001,"Ġcover":3002,"where":3003,"ension":3004,"Ġbott":3005,"Ġareas":3006,"ences":3007,"ope":3008,"ĠTwitter":3009,"ael":3010,"Ġgets":3011,"ĠGoogle":3012,"Ġsn":3013,"iant":3014,"Ġvote":3015,"Ġnearly":3016,"Ġincluded":3017,"Ġrecogn":3018,"zz":3019,"mm":3020,"aled":3021,"Ġhappened":3022,"04":3023,"Ġhot":3024,"Ġwhose":3025,"Ġcivil":3026,"Ġsuff":3027,"oes":3028,"itiz":3029,"ĠSyri":3030,"Ġrespond":3031,"Ġhon":3032,"Ġfeatures":3033,"Ġeconomic":3034,"ĠApril":3035,"rim":3036,"Ġtechnology":3037,"Ġoption":3038,"aging":3039,"Ġpurch":3040,"Re":3041,"Ġlat":3042,"chie":3043,"isl":3044,"Ġrecomm":3045,"uf":3046,"Ġtraining":3047,"Ġeffects":3048,"Ġfast":3049,"Ġ2010":3050,"Ġoccur":3051,"Ġwebsite":3052,"Ġemail":3053,"Ġsens":3054,"ech":3055,"Ġoil":3056,"Ġinflu":3057,"Ġcurrently":3058,"ĠSch":3059,"ĠAdd":3060,"Ġgoal":3061,"Ġscient":3062,"Ġconv":3063,"100":3064,"emy":3065,"Ġdecided":3066,"Ġtravel":3067,"Ġmention":3068,"LL":3069,"03":3070,"Ġelection":3071,"Ġphone":3072,"Ġlooks":3073,"Ġsituation":3074,"Ġcy":3075,"Ġhor":3076,"bed":3077,"ĠCourt":3078,"aily":3079,"aves":3080,"Ġquality":3081,"ĠComp":3082,"wise":3083,"Ġtable":3084,"Ġstaff":3085,"ĠWind":3086,"ett":3087,"Ġtried":3088,"idered":3089,"Ġaddition":3090,"Ġbox":3091,"Ġlack":3092,"arily":3093,"Ġwide":3094,"Ġmid":3095,"Ġboard":3096,"ysis":3097,"Ġanti":3098,"ha":3099,"Ġdig":3100,"ening":3101,"Ġdro":3102,"Con":3103,"68":3104,"Ġslow":3105,"based":3106,"sequ":3107,"Ġpath":3108,"Ex":3109,"aker":3110,"Ġworked":3111,"Ġpen":3112,"Ġengine":3113,"Ġlooked":3114,"ĠSuper":3115,"ĠServ":3116,"Ġvictim":3117,"Un":3118,"Ġproperty":3119,"Ġintrodu":3120,"Ġexecut":3121,"ĠPM":3122,"Le":3123,"Ġcolor":3124,"ĠMore":3125,"Ġ60":3126,"Ġnetwork":3127,"Ġdate":3128,"cul":3129,"idge":3130,"Ġextra":3131,"31":3132,"Ġsle":3133,"67":3134,"Ġwond":3135,"Ġreports":3136,"just":3137,"ĠAustral":3138,"Ġcapital":3139,"Ġens":3140,"Ġcommand":3141,"Ġallowed":3142,"Ġprep":3143,"Ġcapt":3144,"hib":3145,"Ġnumbers":3146,"chan":3147,"Ġfair":3148,"mp":3149,"oms":3150,"Ġreach":3151,"With":3152,"tain":3153,"Ġbroad":3154,"Ġcouple":3155,"ecause":3156,"lying":3157,"ĠFeb":3158,"Ġscreen":3159,"Ġlives":3160,"Ġprior":3161,"ĠCongress":3162,"Ar":3163,"Ġapproach":3164,"Ġemer":3165,"aries":3166,"ĠDis":3167,"serv":3168,"ĠNe":3169,"Ġbuilt":3170,"cies":3171,"Ġrepe":3172,"Ġrules":3173,"force":3174,"ĠPal":3175,"Ġfinancial":3176,"Ġconsidered":3177,"ĠChar":3178,"nces":3179,"ĠIS":3180,"Ġbrought":3181,"Ġbi":3182,"iers":3183,"ĠSim":3184,"OP":3185,"Ġproducts":3186,"Ġvisit":3187,"Ġdocument":3188,"Ġconduct":3189,"Ġcompletely":3190,"ining":3191,"ĠCalif":3192,"ibly":3193,"Ġwritten":3194,"ĠTV":3195,"ements":3196,"Ġdraw":3197,"One":3198,"Ġpublished":3199,"Ġsecret":3200,"rain":3201,"het":3202,"ĠFacebook":3203,"onday":3204,"ĠUp":3205,"Ġsexual":3206,"Ġthous":3207,"ĠPat":3208,"Ġess":3209,"Ġstandard":3210,"Ġarm":3211,"ges":3212,"ection":3213,"Ġfell":3214,"Ġforeign":3215,"ani":3216,"ĠFriday":3217,"Ġregular":3218,"inary":3219,"Ġincreased":3220,"Ġusually":3221,"Ġdemon":3222,"Ġdark":3223,"Ġadditional":3224,"rol":3225,"ĠOf":3226,"Ġproduction":3227,"!!":3228,"undred":3229,"Ġinternational":3230,"idents":3231,"ĠFree":3232,"roup":3233,"Ġrace":3234,"Ġmach":3235,"Ġhuge":3236,"All":3237,"lear":3238,"ovember":3239,"Ġtown":3240,"Ġattention":3241,"ĠOff":3242,"yond":3243,"ĠThen":3244,"field":3245,"Ġterror":3246,"raz":3247,"ĠBo":3248,"Ġmeeting":3249,"ĠPark":3250,"Ġarrest":3251,"Ġfear":3252,"Ġaw":3253,"ĠVal":3254,"oring":3255,"',":3256,"Ġextreme":3257,"arr":3258,"Ġworkers":3259,"After":3260,"Ġ31":3261,"net":3262,"ament":3263,"Ġdirectly":3264,"Ġpopulation":3265,"ube":3266,"ĠOctober":3267,"ĠIN":3268,"ĠJanuary":3269,"59":3270,"ĠDavid":3271,"Ġcross":3272,"cember":3273,"ĠFirst":3274,"Ġmessage":3275,"irit":3276,"Ġnation":3277,"Ġpoll":3278,"isions":3279,"Ġanswer":3280,"ny":3281,"isode":3282,"Ġcarry":3283,"ĠRussia":3284,"Ġhear":3285,"ength":3286,"roy":3287,"Ġnatural":3288,"inally":3289,"Ġdog":3290,"mitted":3291,"Ġtrade":3292,"Ġsubst":3293,"Ġmultiple":3294,"ĠAfric":3295,"Ġfans":3296,"Ġsort":3297,"Ġglobal":3298,"ication":3299,"ĠWed":3300,"ara":3301,"Ġachie":3302,"Ġlanguage":3303,"vey":3304,"Ġtal":3305,"Ġnecessary":3306,"Ġdetails":3307,"Ġsen":3308,"ĠSund":3309,"ĠReg":3310,"ĠRec":3311,"06":3312,"Ġsil":3313,"ressive":3314,"Ġmedical":3315,"unch":3316,"ornia":3317,"Ġund":3318,"fort":3319,"ocks":3320,"ĠMonday":3321,"uesday":3322,"craft":3323,"77":3324,"urt":3325,"Ġver":3326,"ĠHill":3327,"Ġreceive":3328,"Ġmorning":3329,"estern":3330,"Ġbank":3331,"Ġsat":3332,"irth":3333,"ĠHigh":3334,"Ġdevice":3335,"ĠTHE":3336,"ĠCenter":3337,"Ġsafe":3338,"Ġple":3339,"ĠCanada":3340,"Ġsystems":3341,"Ġassist":3342,"Ġsurv":3343,"Ġbattle":3344,"ĠSoc":3345,"vertis":3346,"She":3347,"Ġpaper":3348,"Ġgrowth":3349,"Ġcast":3350,"Sc":3351,"Ġplans":3352,"lled":3353,"Ġparts":3354,"Ġwall":3355,"Ġmovement":3356,"Ġpractice":3357,"imately":3358,"Ġdisplay":3359,"Ġsometimes":3360,"omp":3361,"ĠPaul":3362,"ĠYes":3363,"king":3364,"58":3365,"oly":3366,"Ġson":3367,"Ġavoid":3368,"okes":3369,"ĠJew":3370,"Ġtowards":3371,"asc":3372,"Ġ//":3373,"ĠKore":3374,"Ġtalking":3375,"Ġcorrect":3376,"Ġspent":3377,"icks":3378,"iable":3379,"eared":3380,"Ġterm":3381,"Ġwants":3382,"oming":3383,"Ġut":3384,"Ġdoub":3385,"Ġforces":3386,"Ġplease":3387,"69":3388,"ĠNovember":3389,"atform":3390,"ondon":3391,"Ġones":3392,"Ġimmediately":3393,"ĠRussian":3394,"ĠMet":3395,"Ġdeg":3396,"Ġparents":3397,"CH":3398,"ĠAmericans":3399,"aly":3400,"ĠMod":3401,"Ġshown":3402,"Ġconditions":3403,"Ġstuff":3404,"Ġreb":3405,"ĠYour":3406,"Ġincludes":3407,"nown":3408,"ĠSam":3409,"Ġexperien":3410,"mission":3411,"ĠEven":3412,"aught":3413,"Ġannounced":3414,"ĠRepublican":3415,"Ġdetermin":3416,"Ġdescribed":3417,"ĠCounty":3418,"()":3419,"Ġdoor":3420,"Ġchanged":3421,"Ġneigh":3422,"ĠHere":3423,"Ġclean":3424,"Ġpan":3425,"ĠDecember":3426,"ĠEuropean":3427,"iring":3428,"apter":3429,"Ġclub":3430,"ĠTuesday":3431,"Ġpaid":3432,"ĠNet":3433,"Ġattacks":3434,"Ġcharacters":3435,"Ġalone":3436,"Ġdirector":3437,"dom":3438,"Ġ35":3439,"Ġload":3440,"Ġrout":3441,"ĠCalifornia":3442,"Ġfinally":3443,"Ġrac":3444,"Ġcontr":3445,"Ġexactly":3446,"resh":3447,"pri":3448,"ĠIslam":3449,"Ġnature":3450,"Ġcareer":3451,"Ġlatest":3452,"Ġconvers":3453,"ĠSl":3454,"pose":3455,"cient":3456,"ĠInc":3457,"ivity":3458,"88":3459,"ĠAtt":3460,"ĠMor":3461,"nesday":3462,"Ġweight":3463,"ken":3464,"Ġnote":3465,"Ġteams":3466,"Ġ\\":3467,"airs":3468,"ĠGreen":3469,"Ġhundred":3470,"onent":3471,"Ġstreng":3472,"Ġconsist":3473,"icated":3474,"Ġregul":3475,"Ġlic":3476,"astic":3477,"Ġten":3478,"ursday":3479,"elligence":3480,"ously":3481,"ĠUK":3482,"BI":3483,"Ġcosts":3484,"Ġindepend":3485,"ĠAP":3486,"Ġnormal":3487,"Ġhom":3488,"Ġobvious":3489,"Ġswe":3490,"Ġstar":3491,"Ġready":3492,"acher":3493,"Ġimplement":3494,"gest":3495,"Ġsong":3496,"ĠGet":3497,"ĠLab":3498,"Ġinteresting":3499,"using":3500,"Ġgiving":3501,"ĠSunday":3502,"Ġetc":3503,"Ġmiddle":3504,"Ġremember":3505,"right":3506,"osition":3507,"utions":3508,"Ġmax":3509,"46":3510,"Ġyourself":3511,"Ġdemand":3512,"Ġtreatment":3513,"Ġdanger":3514,"ĠCons":3515,"Ġguy":3516,"ĠBritish":3517,"Ġphysical":3518,"Ġrelated":3519,"Ġremain":3520,"Ġcouldn":3521,"Ġrefer":3522,"Ġcitiz":3523,"box":3524,"ENT":3525,"board":3526,"Ġinn":3527,"IG":3528,"ero":3529,"ĠStreet":3530,"ospital":3531,"rench":3532,"chers":3533,"Ġstra":3534,"OL":3535,"ager":3536,"ĠAN":3537,"Ġeasily":3538,"IA":3539,"enge":3540,"iny":3541,"Ġclos":3542,"ocked":3543,"Ġuses":3544,"ĠCoun":3545,"Im":3546,"uild":3547,"??":3548,"more":3549,"Ġang":3550,"Ġwrite":3551,"olute":3552,"57":3553,"Ġleader":3554,"Ġreading":3555,"":3784,"Ġfigure":3785,"Ġdisapp":3786,"enty":3787,"Ġsoftware":3788,"Ġult":3789,"Ġofficers":3790,"New":3791,"Is":3792,"Ġremains":3793,"ĠIndia":3794,"Ġpsych":3795,"rief":3796,"Ġcat":3797,"esc":3798,"Ġobserv":3799,"Ġstage":3800,"ĠDark":3801,"Ġenter":3802,"change":3803,"Ġpassed":3804,"Ġdespite":3805,"ĠOut":3806,"Ġmovie":3807,"rs":3808,"Ġvoice":3809,"mine":3810,"ĠPlay":3811,"Ġtoward":3812,"ĠTer":3813,"Ġregion":3814,"Ġvalues":3815,"orters":3816,"Ġmount":3817,"Ġofficer":3818,"ĠOther":3819,"ban":3820,"Ġhous":3821,"wood":3822,"room":3823,"IV":3824,"ĠSun":3825,"see":3826,"ĠOver":3827,"rog":3828,"90":3829,"Ġlay":3830,"ĠTur":3831,"awn":3832,"Ġpressure":3833,"ĠSub":3834,"Ġbooks":3835,"edom":3836,"ĠSand":3837,"AA":3838,"ago":3839,"Ġreasons":3840,"ford":3841,"Ġactivity":3842,"UT":3843,"Now":3844,"ĠSenate":3845,"cell":3846,"night":3847,"Ġcalls":3848,"inter":3849,"Ġletter":3850,"ĠRob":3851,"ĠJe":3852,"Ġchoose":3853,"ĠLaw":3854,"Get":3855,"Be":3856,"Ġrob":3857,"Ġtypes":3858,"Ġplatform":3859,"Ġquarter":3860,"RA":3861,"ĠTime":3862,"Ġmaybe":3863,"ĠCr":3864,"95":3865,"pre":3866,"Ġmoving":3867,"Ġlif":3868,"Ġgold":3869,"Ġsom":3870,"Ġpatients":3871,"Ġtruth":3872,"ĠKe":3873,"urance":3874,"antly":3875,"mar":3876,"Ġcharge":3877,"ĠGreat":3878,"Ġcele":3879,"--------------------------------":3880,"Ġrock":3881,"roid":3882,"ancy":3883,"Ġcredit":3884,"aud":3885,"By":3886,"ĠEvery":3887,"Ġmoved":3888,"inger":3889,"ribution":3890,"Ġnames":3891,"Ġstraight":3892,"ĠHealth":3893,"ĠWell":3894,"Ġfeature":3895,"Ġrule":3896,"Ġsche":3897,"inated":3898,"ĠMichael":3899,"berg":3900,"41":3901,"iled":3902,"band":3903,"Ġclick":3904,"ĠAngel":3905,"onents":3906,"ÂŃ":3907,"ĠIraq":3908,"ĠSaturday":3909,"Ġaware":3910,"part":3911,"Ġpattern":3912,"OW":3913,"ĠLet":3914,"Ġgrad":3915,"igned":3916,"Ġassociated":3917,"Ġstyle":3918,"no":3919,"iation":3920,"aith":3921,"ilies":3922,"Ġstories":3923,"uration":3924,"Ġindividuals":3925,"ĠâĢ¦":3926,"miss":3927,"ĠAssoci":3928,"ishing":3929,"aby":3930,"Ġsummer":3931,"ĠBen":3932,"Ġ32":3933,"Ġarch":3934,"uty":3935,"ĠTexas":3936,"hol":3937,"Ġfully":3938,"Ġmill":3939,"Ġfollowed":3940,"ĠBill":3941,"ĠIndian":3942,"ĠSecret":3943,"ĠBel":3944,"ĠFebruary":3945,"Ġjobs":3946,"Ġseemed":3947,"ĠGovern":3948,"ipped":3949,"Ġreality":3950,"Ġlines":3951,"Ġpark":3952,"Ġmeasure":3953,"ĠOur":3954,"IM":3955,"Ġbrother":3956,"Ġgrowing":3957,"Ġban":3958,"Ġestim":3959,"Ġcry":3960,"ĠSchool":3961,"Ġmechan":3962,"ĠOF":3963,"ĠWindows":3964,"Ġrates":3965,"ĠOh":3966,"Ġpositive":3967,"Ġculture":3968,"istics":3969,"ica":3970,"Ġhar":3971,"ya":3972,"itely":3973,"ipp":3974,"Ġmap":3975,"encies":3976,"ĠWilliam":3977,"II":3978,"akers":3979,"56":3980,"ĠMart":3981,"ĠRem":3982,"Ġaltern":3983,"itude":3984,"Ġcoach":3985,"rowd":3986,"Don":3987,"Ġkids":3988,"Ġjournal":3989,"Ġcorpor":3990,"Ġfalse":3991,"Ġweb":3992,"Ġsleep":3993,"Ġcontain":3994,"Ġsto":3995,"Ġbed":3996,"iverse":3997,"ĠRich":3998,"ĠChinese":3999,"Ġpun":4000,"Ġmeant":4001,"known":4002,"Ġnotice":4003,"Ġfavorite":4004,"aven":4005,"Ġcondition":4006,"Ġpurpose":4007,"))":4008,"Ġorganization":4009,"Ġchalleng":4010,"Ġmanufact":4011,"Ġsusp":4012,"ĠAc":4013,"Ġcritic":4014,"unes":4015,"uclear":4016,"Ġmer":4017,"vention":4018,"Ġ80":4019,"Ġmist":4020,"ĠUs":4021,"ĠTor":4022,"http":4023,"olf":4024,"Ġlarger":4025,"Ġadvant":4026,"Ġresear":4027,"Ġactions":4028,"ml":4029,"Ġkept":4030,"Ġaim":4031,",'":4032,"col":4033,"Ġbenefits":4034,"ifying":4035,"Ġactual":4036,"ĠInternational":4037,"Ġvehicle":4038,"Ġchief":4039,"Ġefforts":4040,"ĠLeague":4041,"ĠMost":4042,"Ġwait":4043,"Ġadult":4044,"Ġoverall":4045,"Ġspeech":4046,"Ġhighly":4047,"Ġfemale":4048,"Ġerror":4049,"Ġeffective":4050,"54":4051,"Ġencour":4052,"well":4053,"Ġfailed":4054,"Ġconserv":4055,"Ġprograms":4056,"Ġtrou":4057,"Ġahead":4058,"500":4059,"vertisement":4060,"IP":4061,"ĠFound":4062,"pir":4063,"Ġ%":4064,"Ġcrime":4065,"ander":4066,"Ġlocation":4067,"ĠIran":4068,"Ġbehavior":4069,"azing":4070,"Ġrare":4071,"Ġemb":4072,"Ġcaused":4073,"Ġship":4074,"Ġactive":4075,"Ġcontribut":4076,"Ġgreen":4077,"Ġacqu":4078,"Ġreflect":4079,"venue":4080,"Ġfirm":4081,"Ġbirth":4082,"].":4083,"Ġclearly":4084,"Ġemot":4085,"Ġagency":4086,"riage":4087,"Ġmemory":4088,"98":4089,"SA":4090,"ĠSee":4091,"acing":4092,"CC":4093,"Ġbiggest":4094,"Ġrap":4095,"Ġbasic":4096,"Ġband":4097,"eat":4098,"Ġsuspect":4099,"ĠMac":4100,"Ġ90":4101,"mark":4102,"istan":4103,"Ġspread":4104,"ams":4105,"ki":4106,"asy":4107,"rav":4108,"ĠRober":4109,"Ġdemonstr":4110,"rated":4111,"Ġabsolute":4112,"Ġplaces":4113,"Ġimpl":4114,"ibrary":4115,"Ġcards":4116,"Ġdestroy":4117,"Ġvirt":4118,"vere":4119,"Ġappeared":4120,"yan":4121,"point":4122,"Ġbeg":4123,"Ġtemper":4124,"spe":4125,"anted":4126,"ears":4127,"ĠDirect":4128,"Ġlength":4129,"Ġblog":4130,"amb":4131,"Ġinteg":4132,"Ġresources":4133,"acc":4134,"iful":4135,"Ġspot":4136,"Ġforced":4137,"Ġthousands":4138,"ĠMinister":4139,"Ġqual":4140,"ĠFrench":4141,"atically":4142,"Ġgenerally":4143,"Ġdrink":4144,"Ġthus":4145,"IL":4146,"odes":4147,"Ġappropri":4148,"ĠRead":4149,"Ġwhom":4150,"Ġeye":4151,"Ġcollege":4152,"Ġ45":4153,"irection":4154,"Ġensure":4155,"Ġapparent":4156,"iders":4157,"Ġreligious":4158,"Ġminor":4159,"olic":4160,"Ġtro":4161,"ĠWhy":4162,"ribute":4163,"met":4164,"Ġprimary":4165,"Ġdeveloped":4166,"Ġpeace":4167,"Ġskin":4168,"ste":4169,"ava":4170,"Ġblue":4171,"Ġfamilies":4172,"Ġir":4173,"Ġapply":4174,"Ġinform":4175,"ĠSmith":4176,"CT":4177,"ii":4178,"Ġlimit":4179,"Ġresist":4180,"................":4181,"umn":4182,"Ġconflic":4183,"Ġtwe":4184,"udd":4185,"ĠTom":4186,"Ġliter":4187,"que":4188,"bon":4189,"Ġhair":4190,"Ġeventually":4191,"Ġpus":4192,"Ġhelped":4193,"Ġagg":4194,"orney":4195,"ĠApple":4196,"Ġfit":4197,"ĠSur":4198,"Ġprem":4199,"Ġsales":4200,"Ġseconds":4201,"Ġstrength":4202,"Ġfeeling":4203,"¿½":4204,"Ġtour":4205,"Ġknows":4206,"oom":4207,"Ġexerc":4208,"Ġsomew":4209,"�":4210,">>":4211,"Ġspokes":4212,"Ġideas":4213,"Ġregist":4214,"soft":4215,"ĠDel":4216,"ĠPC":4217,"Ġpropos":4218,"Ġlaunch":4219,"Ġbottom":4220,"TH":4221,"ĠPlease":4222,"vest":4223,"itz":4224,"ĠInter":4225,"Ġscript":4226,"Ġrat":4227,"arning":4228,"Ġil":4229,"ĠJer":4230,"ĠAre":4231,"Ġwhatever":4232,"oken":4233,"cience":4234,"Ġmode":4235,"Ġagree":4236,"Ġsources":4237,"Ġinitial":4238,"Ġrestrict":4239,"Ġwonder":4240,"usion":4241,"####":4242,"ĠSil":4243,"ville":4244,"Ġburn":4245,"tw":4246,"asion":4247,"Ġ£":4248,"Ġnor":4249,"uing":4250,"Ġreached":4251,"Ġsun":4252,"Ġcateg":4253,"igration":4254,"Ġcook":4255,"Ġpromot":4256,"Ġmale":4257,"Ġclimate":4258,"Ġfix":4259,"Ġalleged":4260,"UR":4261,"alled":4262,"Ġimages":4263,"Cont":4264,"ota":4265,"Ġschools":4266,"ios":4267,"Ġdrop":4268,"Ġstream":4269,"ĠMo":4270,"Ġpreviously":4271,"aling":4272,"Ġpet":4273,"Ġdouble":4274,"Ġ(@":4275,"annel":4276,"Ġdefault":4277,"ties":4278,"Ġrank":4279,"ĠDec":4280,"ĠCouncil":4281,"Ġweapon":4282,"Ġstock":4283,"Ġanaly":4284,"ĠStr":4285,"Ġpicture":4286,"ĠPolice":4287,"ference":4288,"Ġcentury":4289,"Ġcitizens":4290,"Ġonto":4291,"Ġexpand":4292,"Ġhero":4293,"ĠSol":4294,"Ġwild":4295,"Ġupdate":4296,"Ġcustomers":4297,"ront":4298,"def":4299,"Ġlik":4300,"Ġcriminal":4301,"ĠChristian":4302,"SP":4303,"76":4304,"Ġleaving":4305,"Ġotherwise":4306,"ĠDist":4307,"Ġbasis":4308,"52":4309,"53":4310,"icip":4311,"ĠBer":4312,"Ġrecommend":4313,"Ġfloor":4314,"Ġcrowd":4315,"oles":4316,"Ġ70":4317,"Ġcentral":4318,"ĠEv":4319,"Ġdream":4320,"Ġdownload":4321,"Ġconfir":4322,"ĠThom":4323,"Ġwindow":4324,"Ġhappens":4325,"Ġunit":4326,"Ġtend":4327,"Ġspl":4328,"Ġbecomes":4329,"Ġfighting":4330,"Ġpredict":4331,"ĠPress":4332,"ĠPower":4333,"Ġheavy":4334,"aked":4335,"Ġfan":4336,"orter":4337,"ategy":4338,"BA":4339,"izes":4340,"Ġspend":4341,"Here":4342,"Ġ2007":4343,"Ġadop":4344,"ĠHam":4345,"Ġfootball":4346,"ĠPort":4347,"oday":4348,"51":4349,"ampions":4350,"Ġtransfer":4351,"ht":4352,"Ġ38":4353,"term":4354,"acity":4355,"Ġbur":4356,"],":4357,"ternal":4358,"rig":4359,"but":4360,"Ġtherefore":4361,"ĠBecause":4362,"resp":4363,"rey":4364,"Ġmission":4365,"Some":4366,"Ġnoted":4367,"Ġassum":4368,"Ġdisease":4369,"Ġedit":4370,"Ġprogress":4371,"rd":4372,"ĠBrown":4373,"ocal":4374,"Ġadding":4375,"Ġraised":4376,"ĠAny":4377,"Ġtick":4378,"Ġseeing":4379,"ĠPeople":4380,"Ġagreement":4381,"Ġserver":4382,"Ġwat":4383,"Ġdebate":4384,"Ġsupposed":4385,"iling":4386,"Ġlargest":4387,"Ġsuccessful":4388,"ĠPri":4389,"ĠDemocratic":4390,"Ġjump":4391,"ĠSyria":4392,"Ġowners":4393,"Ġoffers":4394,"Ġshooting":4395,"Ġeffic":4396,"sey":4397,"Ġhaven":4398,"verse":4399,"tered":4400,"ĠLight":4401,"imal":4402,"ĠBig":4403,"Ġdefend":4404,"Ġbeat":4405,"Ġrecords":4406,"%)":4407,"Ġscen":4408,"Ġemployees":4409,"Ġdevices":4410,"hem":4411,"Ġcommer":4412,"ĠMex":4413,"Ġbenefit":4414,"ĠProf":4415,"Ġilleg":4416,"Ġsurface":4417,"ĠAlso":4418,"Ġharm":4419,"ingly":4420,"wide":4421,"ĠAlex":4422,"Ġshut":4423,"ĠCur":4424,"Ġlose":4425,"pm":4426,"Ġchallenge":4427,"semb":4428,"Ġstation":4429,"Ġintelligence":4430,"Ġaccur":4431,"ĠFlor":4432,"Ġrequires":4433,"ĠMal":4434,"bum":4435,"Ġhospital":4436,"Ġspirit":4437,"Ġoffered":4438,"Ġproduce":4439,"ĠCommun":4440,"Ġcreating":4441,"Ġcris":4442,"spect":4443,"Ġended":4444,"Ġdaily":4445,"Ġvoters":4446,"lands":4447,"ias":4448,"ih":4449,"ona":4450,"Ġsmart":4451,"ĠOffice":4452,"ĠLord":4453,"rial":4454,"ĠInternet":4455,"Ġcircum":4456,"Ġextremely":4457,"'.":4458,"Ġopinion":4459,"ĠMil":4460,"Ġgain":4461,"BS":4462,"ĠFin":4463,"yp":4464,"Ġuseful":4465,"Ġbudget":4466,"Ġcomfort":4467,"isf":4468,"Ġbackground":4469,"eline":4470,"Ġepisode":4471,"Ġenemy":4472,"Ġtrial":4473,"Ġestablish":4474,"date":4475,"ĠCap":4476,"Ġcontinues":4477,"Ġshowing":4478,"ĠUnion":4479,"with":4480,"Ġposted":4481,"ĠSystem":4482,"Ġeat":4483,"rian":4484,"Ġrise":4485,"ĠGermany":4486,"ils":4487,"Ġsigned":4488,"Ġvill":4489,"Ġgrand":4490,"mor":4491,"ĠEngland":4492,"Ġprojects":4493,"umber":4494,"Ġconference":4495,"za":4496,"Ġresponsible":4497,"ĠArab":4498,"Ġlearned":4499,"âĢĶâĢĶ":4500,"ipping":4501,"ĠGeorge":4502,"OC":4503,"Ġreturned":4504,"ĠAustralia":4505,"Ġbrief":4506,"Qu":4507,"Ġbrand":4508,"illing":4509,"abled":4510,"Ġhighest":4511,"Ġtrain":4512,"ĠCommission":4513,"while":4514,"Ġnom":4515,"ception":4516,"Ġmut":4517,"ĠBlue":4518,"Ġincident":4519,"vant":4520,"86":4521,"ĠID":4522,"Ġnuclear":4523,"74":4524,"ĠLike":4525,"ĠRE":4526,"ĠMicro":4527,"li":4528,"mail":4529,"Ġcharges":4530,"89":4531,"Ġadjust":4532,"ado":4533,"Ġearth":4534,"NA":4535,"Ġprices":4536,"PA":4537,"Ġdraft":4538,"Ġruns":4539,"Ġcandidate":4540,"enses":4541,"Ġmanagement":4542,"ĠPhil":4543,"ĠMiss":4544,"Ġteach":4545,"gram":4546,"Ġunderstanding":4547,"ait":4548,"icago":4549,"Add":4550,"ĠEp":4551,"secut":4552,"Ġseparate":4553,"Ġinstance":4554,"Ġeth":4555,"Ġunless":4556,"********":4557,"ĠFore":4558,"inate":4559,"Ġoperations":4560,"Sp":4561,"Ġfaith":4562,"gar":4563,"ĠChurch":4564,"ronic":4565,"Ġconfig":4566,"osure":4567,"Ġactivities":4568,"Ġtraditional":4569,"Ġ36":4570,"Ġdirection":4571,"Ġmachine":4572,"Ġsurround":4573,"Ġpush":4574,"unction":4575,"ĠEU":4576,"Ġeasier":4577,"Ġargument":4578,"GB":4579,"Ġmicro":4580,"Ġspending":4581,"izations":4582,"Ġtheory":4583,"adow":4584,"Ġcalling":4585,"ĠLast":4586,"Ġder":4587,"Ġinfluence":4588,"Ġcommit":4589,"Ġphoto":4590,"Ġunc":4591,"istry":4592,"gn":4593,"aste":4594,"acks":4595,"Ġdisp":4596,"ady":4597,"do":4598,"ĠGood":4599,"Ġ`":4600,"Ġwish":4601,"Ġrevealed":4602,"³³":4603,"lig":4604,"Ġenforce":4605,"ĠCommittee":4606,"Ġchem":4607,"Ġmiles":4608,"Ġinterested":4609,"Ġsolution":4610,"icy":4611,"inct":4612,"Ġ->":4613,"ĠDet":4614,"Ġremoved":4615,"Ġcompar":4616,"eah":4617,"Ġplant":4618,"ĠSince":4619,"Ġachieve":4620,"Ġadvantage":4621,"Ġslightly":4622,"bing":4623,"Ġplaced":4624,"under":4625,"2015":4626,"ĠMad":4627,"Ġtim":4628,"oses":4629,"Ġcru":4630,"ĠRock":4631,"Ġmostly":4632,"Ġnegative":4633,"Ġsetting":4634,"Ġproduced":4635,"Ġmur":4636,"Ġconnection":4637,"ĠMer":4638,"Ġdriver":4639,"Ġexecutive":4640,"Ġassault":4641,"Ġborn":4642,"ĠVer":4643,"tained":4644,"Ġstructure":4645,"Ġreduce":4646,"Ġdecades":4647,"Ġded":4648,"uke":4649,"ĠMany":4650,"idden":4651,"Ġleague":4652,"Se":4653,"Ġjoin":4654,"Ġdisco":4655,"Ġdie":4656,"cks":4657,"actions":4658,"Ġassess":4659,"agn":4660,"Ġgoals":4661,"ours":4662,"IR":4663,"Ġsenior":4664,"iller":4665,"mod":4666,"ipment":4667,"ocol":4668,"uy":4669,"ĠQue":4670,"Ġparties":4671,"irgin":4672,"Ġlearning":4673,"itable":4674,"Ġstreet":4675,"Ġcamera":4676,"App":4677,"Ġskills":4678,"bre":4679,"cious":4680,"Ġcelebr":4681,"ĠFranc":4682,"Ġexisting":4683,"Ġwilling":4684,"lor":4685,"Ġid":4686,"ĠSpace":4687,"Ġcritical":4688,"ĠLa":4689,"ortunately":4690,"Ġserve":4691,"Ġcold":4692,"Ġspecies":4693,"TS":4694,"Ġanimals":4695,"ĠBay":4696,"Ġolder":4697,"ĠUnder":4698,"estic":4699,"ĠTre":4700,"Ġteacher":4701,"Ġprefer":4702,"vis":4703,"Ġthread":4704,"ĠMatt":4705,"Ġmanager":4706,"ãĥ»":4707,"Ġprofessional":4708,"ĠVol":4709,"Ġnotes":4710,"These":4711,"ula":4712,"Ġfresh":4713,"ented":4714,"uzz":4715,"edy":4716,"clusion":4717,"ĠRel":4718,"Ġdoubt":4719,"EO":4720,"Ġopened":4721,"ĠBit":4722,"Advertisement":4723,"Ġguess":4724,"ĠUN":4725,"Ġsequ":4726,"Ġexplain":4727,"otten":4728,"Ġattract":4729,"aks":4730,"Ġstring":4731,"Ġcontext":4732,"ossible":4733,"ĠRepublicans":4734,"Ġsolid":4735,"Ġcities":4736,"Ġasking":4737,"Ġrandom":4738,"ups":4739,"uries":4740,"arant":4741,"dden":4742,"gl":4743,"ĠFlorida":4744,"Ġdepend":4745,"ĠScott":4746,"Ġ33":4747,"ĠiT":4748,"icon":4749,"Ġmentioned":4750,"Ġ2000":4751,"Ġclaimed":4752,"Ġdefinitely":4753,"ulf":4754,"Ġcore":4755,"Ġopening":4756,"ĠConst":4757,"which":4758,"ĠTra":4759,"AG":4760,"72":4761,"Ġbelieved":4762,"ada":4763,"Ġ48":4764,"ĠSecurity":4765,"yright":4766,"ĠPet":4767,"ĠLou":4768,"Ġholding":4769,"================":4770,"Ġice":4771,"Ġbrow":4772,"Ġauthorities":4773,"host":4774,"word":4775,"Ġscore":4776,"ĠDiv":4777,"Ġcells":4778,"Ġtransl":4779,"Ġneighbor":4780,"Ġremove":4781,"uct":4782,"Ġdistrict":4783,"ĠAccording":4784,"Ġworse":4785,"Ġconcerns":4786,"Ġpresidential":4787,"Ġpolicies":4788,"ĠHall":4789,"73":4790,"Ġhus":4791,"AY":4792,"Ġ2006":4793,"ĠJud":4794,"Ġindependent":4795,"ĠJustice":4796,"iliar":4797,"print":4798,"ighter":4799,"Ġprotection":4800,"zen":4801,"Ġsudden":4802,"house":4803,"ĠJes":4804,"PR":4805,"ĠInf":4806,"Ġbul":4807,"Ġ_":4808,"ĠService":4809,"ĠPR":4810,"Ġstrategy":4811,"ffect":4812,"Ġgirls":4813,"Ġmissing":4814,"oyal":4815,"ĠTeam":4816,"ulated":4817,"Ġdat":4818,"Ġpolitics":4819,"abor":4820,"According":4821,"Ġspell":4822,"Ġgraph":4823,"orthern":4824,"TC":4825,"Ab":4826,"Ġlabor":4827,"isher":4828,"Ġkick":4829,"ĠiTunes":4830,"Ġsteps":4831,"poses":4832,"Ġsmaller":4833,"En":4834,"bert":4835,"Ġroll":4836,"Ġresearchers":4837,"Ġclosed":4838,"Ġtransport":4839,"Ġlawy":4840,"________________":4841,"ĠChicago":4842,"Ġaspect":4843,"Ġnone":4844,"Ġmarriage":4845,"96":4846,"Ġelements":4847,"ĠFre":4848,"ĠSal":4849,"Ġdram":4850,"FC":4851,"top":4852,"equ":4853,"Ġhearing":4854,"Ġsupported":4855,"Ġtesting":4856,"cohol":4857,"Ġmassive":4858,"Ġstick":4859,"Ġguard":4860,"isco":4861,"phone":4862,"From":4863,"However":4864,"Ġborder":4865,"Ġcopy":4866,"ography":4867,"list":4868,"71":4869,"Ġowner":4870,"class":4871,"ruit":4872,"rate":4873,"ĠOnce":4874,"Ġdigital":4875,"Ġtask":4876,"ERS":4877,"Ġincred":4878,"tes":4879,"++":4880,"ĠFrance":4881,"Ġbreat":4882,"owl":4883,"Ġissued":4884,"ĠWestern":4885,"Ġdetect":4886,"Ġpartners":4887,"Ġshared":4888,"ĠCall":4889,"Ġcancer":4890,"ache":4891,"ribe":4892,"Ġexplained":4893,"Ġheat":4894,"{\"":4895,"Ġinvestment":4896,"ĠBook":4897,"Ġwood":4898,"Ġtools":4899,"ĠAlthough":4900,"Ġbelief":4901,"Ġcrisis":4902,"Ġge":4903,"ĠMP":4904,"Ġoperation":4905,"type":4906,"~~":4907,"ga":4908,"Ġcontains":4909,"anta":4910,"Ġexpress":4911,"ĠGroup":4912,"ĠJournal":4913,"ka":4914,"Ġamb":4915,"ĠUSA":4916,"Ġfinding":4917,"Ġfunding":4918,"how":4919,"Ġestablished":4920,"ideos":4921,"Ġdegree":4922,"Ġdangerous":4923,"anging":4924,"Ġfreedom":4925,"pport":4926,"outhern":4927,"Ġchurch":4928,"Ġcatch":4929,"ĠTwo":4930,"Ġpresence":4931,"ĠGuard":4932,"Up":4933,"Ġauthority":4934,"ĠProject":4935,"Ġbutton":4936,"Ġconsequ":4937,"Ġvalid":4938,"Ġweak":4939,"Ġstarts":4940,"Ġreference":4941,"ĠMem":4942,"\")":4943,"UN":4944,"orage":4945,"ĠOpen":4946,"Ġcollection":4947,"ym":4948,"gency":4949,"Ġbeautiful":4950,"ros":4951,"Ġtells":4952,"Ġwaiting":4953,"nel":4954,"Ġproviding":4955,"ĠDemocrats":4956,"Ġdaughter":4957,"Ġmaster":4958,"Ġpurposes":4959,"ĠJapanese":4960,"Ġequal":4961,"Ġturns":4962,"Ġdocuments":4963,"Ġwatching":4964,"Res":4965,"Ġran":4966,"2014":4967,"Ġreject":4968,"ĠKorea":4969,"Ġvictims":4970,"Level":4971,"erences":4972,"Ġwitness":4973,"Ġ34":4974,"Ġreform":4975,"coming":4976,"Ġoccup":4977,"Ġcaught":4978,"Ġtraffic":4979,"ading":4980,"Ġmodels":4981,"ario":4982,"Ġserved":4983,"Ġbatter":4984,"uate":4985,"ĠSecretary":4986,"Ġagreed":4987,"Ġtruly":4988,"ynam":4989,"ĠRet":4990,"Ġunits":4991,"ĠResearch":4992,"hand":4993,"azine":4994,"ĠMike":4995,"Ġvariety":4996,"otal":4997,"Ġamazing":4998,"Ġconfirmed":4999,"Ġentirely":5000,"Ġpurchase":5001,"Ġelement":5002,"Ġcash":5003,"Ġdetermine":5004,"De":5005,"Ġcars":5006,"ĠWall":5007,"âĸ":5008,"Ġviews":5009,"Ġdrugs":5010,"Ġdepartment":5011,"ĠStep":5012,"uit":5013,"Ġ39":5014,"asure":5015,"ĠClass":5016,"Ġcovered":5017,"ĠBank":5018,"Ġmere":5019,"uana":5020,"Ġmulti":5021,"Ġmix":5022,"Ġunlike":5023,"levision":5024,"Ġstopped":5025,"Ġsem":5026,"ĠGal":5027,"ules":5028,"Ġwel":5029,"ĠJohnson":5030,"la":5031,"Ġskill":5032,"Ġbecoming":5033,"rie":5034,"Ġappropriate":5035,"fe":5036,"ellow":5037,"ĠProt":5038,"ulate":5039,"ocation":5040,"Ġweekend":5041,"odies":5042,"Ġsites":5043,"Ġanimal":5044,"ĠTim":5045,"Ġscale":5046,"Ġcharged":5047,"Ġinstruct":5048,"illa":5049,"Ġmethods":5050,"Ġcert":5051,"Ġjudge":5052,"ĠHel":5053,"Ġdollars":5054,"Ġstanding":5055,"ĠSqu":5056,"Ġdebt":5057,"liam":5058,"Ġdriving":5059,"ĠSum":5060,"ĠEdition":5061,"Ġalbum":5062,"andon":5063,"IF":5064,"ĠUk":5065,"63":5066,"ader":5067,"Ġcommercial":5068,"esh":5069,"ĠGovernment":5070,"Ġdiscovered":5071,"Ġoutput":5072,"ĠHillary":5073,"ĠCarol":5074,"Ġ2005":5075,"Ġabuse":5076,"ancing":5077,"Ġswitch":5078,"Ġannual":5079,"Tw":5080,"Ġstated":5081,"agement":5082,"inner":5083,"Ġdemocr":5084,"Ġresidents":5085,"Ġallowing":5086,"Ġfactors":5087,"odd":5088,"Ġfuck":5089,"emies":5090,"Ġoccurred":5091,"oti":5092,"Ġnorth":5093,"ĠPublic":5094,"Ġinjury":5095,"Ġinsurance":5096,"CL":5097,"olly":5098,"ãĢ":5099,"Ġrepeated":5100,"Ġarms":5101,"anged":5102,"Ġconstruction":5103,"Ġfle":5104,"PU":5105,"icians":5106,"Ġforms":5107,"ĠMcC":5108,"antic":5109,"Ġmental":5110,"pire":5111,"Ġequipment":5112,"Ġfant":5113,"Ġdiscussion":5114,"Ġregarding":5115,"kin":5116,"arp":5117,"Ġchair":5118,"ogue":5119,"Ġproceed":5120,"ĠId":5121,"Our":5122,"Ġmurder":5123,"Man":5124,"Ġ49":5125,"asp":5126,"Ġsupply":5127,"Ġinput":5128,"Ġwealth":5129,"liament":5130,"Ġproced":5131,"orial":5132,"ĠStat":5133,"ĠNFL":5134,"hens":5135,"ĠInstitute":5136,"Ġputting":5137,"ournament":5138,"etic":5139,"Ġlocated":5140,"Ġkid":5141,"eria":5142,"run":5143,"Ġprinc":5144,"Ġ!":5145,"going":5146,"ĠBet":5147,"Ġclot":5148,"Ġtelling":5149,"Ġproposed":5150,"iot":5151,"orry":5152,"Ġfunds":5153,"gment":5154,"ĠLife":5155,"Ġbaby":5156,"ĠBack":5157,"Ġspoke":5158,"Image":5159,"Ġearn":5160,"ĠAT":5161,"gu":5162,"Ġexchange":5163,"ĠLin":5164,"oving":5165,"Ġpair":5166,"More":5167,"azon":5168,"Ġarrested":5169,"Ġkilling":5170,"can":5171,"ĠCard":5172,"yd":5173,"Ġidentified":5174,"Ġmobile":5175,"Ġthanks":5176,"onym":5177,"ĠForm":5178,"Ġhundreds":5179,"ĠChris":5180,"ĠCat":5181,"Ġtrend":5182,"hat":5183,"ĠAv":5184,"oman":5185,"Ġelectric":5186,"ĠWil":5187,"SE":5188,"Of":5189,"Ġrestaur":5190,"oted":5191,"Ġtrig":5192,"Ġnine":5193,"Ġbomb":5194,"Why":5195,"¯":5196,"Ġcoverage":5197,"Ġappeal":5198,"ĠRobert":5199,"ĠSup":5200,"Ġfinished":5201,"Ġflow":5202,"Ġdeliver":5203,"Ġcalcul":5204,"Ġphotos":5205,"Ġphil":5206,"Ġpieces":5207,"Ġappre":5208,"kes":5209,"Ġrough":5210,"Do":5211,"Ġpartner":5212,"Ġconcerned":5213,"Ġ37":5214,"ĠGen":5215,"Col":5216,"ctors":5217,"Ġ=>":5218,"state":5219,"Ġsuggested":5220,"ĠForce":5221,"CE":5222,"Ġherself":5223,"ĠPlan":5224,"works":5225,"ooth":5226,"rency":5227,"Ġcorner":5228,"Ġhusband":5229,"Ġinternet":5230,"ĠAut":5231,"ems":5232,"osen":5233,"ĠAtl":5234,"gen":5235,"Ġbalance":5236,"62":5237,"Ġsounds":5238,"text":5239,"Ġarr":5240,"oves":5241,"Ġmillions":5242,"Ġradio":5243,"Ġsatisf":5244,"ĠDam":5245,"Mr":5246,"Go":5247,"Spe":5248,"Ġcombat":5249,"rant":5250,"ĠGree":5251,"Ġfuel":5252,"Ġdistance":5253,"Ġtests":5254,"Ġdecre":5255,"ĠEr":5256,"Ġmanaged":5257,"DS":5258,"Ġtit":5259,"Ġmeasures":5260,"ĠLiber":5261,"Ġattend":5262,"ashed":5263,"ĠJose":5264,"ĠNight":5265,"dit":5266,"ĠNov":5267,"ĠEnd":5268,"outs":5269,"Ġgeneration":5270,"Ġadvoc":5271,"yth":5272,"Ġconversation":5273,"ĠSky":5274,"active":5275,"cel":5276,"rier":5277,"ĠFrank":5278,"Ġgender":5279,"Ġconcent":5280,"Ġcarried":5281,"anda":5282,"ĠVirgin":5283,"Ġarrived":5284,"icide":5285,"aded":5286,"Ġfailure":5287,"Ġminimum":5288,"lets":5289,"Ġworst":5290,"Ġkeeping":5291,"Ġintended":5292,"Ġillegal":5293,"Ġsubsc":5294,"Ġdetermined":5295,"Ġtrip":5296,"Yes":5297,"Ġraise":5298,"Ġ~":5299,"Ġfeels":5300,"Ġpackage":5301,"ĠJo":5302,"hi":5303,"2016":5304,"real":5305,"Ġfra":5306,"Ġsymb":5307,"Me":5308,"ucky":5309,"pret":5310,"ĠKh":5311,"ĠEdit":5312,"ĠWeb":5313,"emic":5314,"ĠColor":5315,"Ġjustice":5316,"Int":5317,"Ġfarm":5318,"cknow":5319,"\">":5320,"eless":5321,"Ġreduced":5322,"Ġ500":5323,"xx":5324,"ĠRad":5325,"ĠWood":5326,"Ġclin":5327,"Ġhyp":5328,"iler":5329,"ura":5330,"kins":5331,"85":5332,"61":5333,"ĠTheir":5334,"ĠMary":5335,"Ġsan":5336,"Ġnovel":5337,"ĠWho":5338,"Ġcapacity":5339,"Ġimpossible":5340,"Ġplays":5341,"Ġminister":5342,"ijuana":5343,"icate":5344,"ĠSet":5345,"Ġfram":5346,"Ġing":5347,"Ġcommunities":5348,"ĠFBI":5349,"ita":5350,"Ġbon":5351,"Ġstrateg":5352,"Ġinterests":5353,"lock":5354,"gers":5355,"mas":5356,"ĠAND":5357,"Ġconflict":5358,"Ġrequirements":5359,"Ġsac":5360,"Ġoperating":5361,"ini":5362,"related":5363,"Ġcommitted":5364,"Ġrelatively":5365,"Ġsouth":5366,"¯¯":5367,"Ġafford":5368,"Ġidentity":5369,"Ġdecisions":5370,"Ġaccused":5371,"place":5372,"Ġvictory":5373,"och":5374,"iat":5375,"Name":5376,"Com":5377,"tion":5378,"eds":5379,"Ġseek":5380,"Ġtight":5381,"ĠImages":5382,"Ġiniti":5383,"Ġhumans":5384,"Ġfamiliar":5385,"Ġaudience":5386,"Ġinternal":5387,"venture":5388,"Ġsides":5389,"ĠTO":5390,"Ġdim":5391,"Ġconclud":5392,"Ġappoint":5393,"Ġenforcement":5394,"ĠJim":5395,"ĠAssociation":5396,"Ġcircumst":5397,"ĠCanadian":5398,"Ġjoined":5399,"Ġdifferences":5400,"ĠLos":5401,"Ġprotest":5402,"Ġtwice":5403,"win":5404,"Ġglass":5405,"arsh":5406,"ĠArmy":5407,"Ġexpression":5408,"Ġdecide":5409,"Ġplanning":5410,"ania":5411,"Ġhandle":5412,"ĠMicrosoft":5413,"ĠNor":5414,"Ġmaximum":5415,"ĠRev":5416,"Ġsea":5417,"Ġeval":5418,"Ġhelps":5419,"ref":5420,"Ġbound":5421,"Ġmouth":5422,"Ġstandards":5423,"Ġclim":5424,"ĠCamp":5425,"ĠFox":5426,"cles":5427,"Ġarmy":5428,"ĠTechn":5429,"acking":5430,"xy":5431,"SS":5432,"Ġ42":5433,"Ġbug":5434,"ĠUkrain":5435,"ĠMax":5436,"ĠJones":5437,"ĠShow":5438,"lo":5439,"Ġplanet":5440,"Ġ75":5441,"Ġwinning":5442,"Ġfaster":5443,"Ġspect":5444,"Ġbroken":5445,"TR":5446,"Ġdefined":5447,"Ġhealthy":5448,"Ġcompetition":5449,"https":5450,"ĠIsland":5451,"ĠFe":5452,"Ġannounce":5453,"ĠCup":5454,"ĠInstead":5455,"Ġclient":5456,"Ġpossibly":5457,"section":5458,"ocket":5459,"look":5460,"Ġfinish":5461,"Ġcrew":5462,"Ġreserv":5463,"Ġeditor":5464,"Ġhate":5465,"Ġsale":5466,"Ġcontrovers":5467,"Ġpages":5468,"wing":5469,"Ġnumer":5470,"Ġopposition":5471,"Ġ2004":5472,"Ġrefuge":5473,"Ġflight":5474,"Ġapart":5475,"ĠLat":5476,"Americ":5477,"ĠAfrica":5478,"Ġapplications":5479,"ĠPalest":5480,"ĠBur":5481,"Ġgar":5482,"ĠSocial":5483,"Ġupgr":5484,"Ġshape":5485,"Ġspeaking":5486,"ansion":5487,"ao":5488,"ĠSn":5489,"Ġworry":5490,"ĠBritain":5491,"Please":5492,"roud":5493,"Ġhun":5494,"Ġintroduced":5495,"Ġdiet":5496,"Ind":5497,"ĠSecond":5498,"Ġfunctions":5499,"uts":5500,"ĠEach":5501,"ĠJeff":5502,"Ġstress":5503,"Ġaccounts":5504,"Ġguarant":5505,"ĠAnn":5506,"edia":5507,"Ġhonest":5508,"Ġtree":5509,"ĠAfrican":5510,"ĠBush":5511,"},":5512,"Ġsch":5513,"ĠOnly":5514,"Ġfif":5515,"igan":5516,"Ġexercise":5517,"ĠExp":5518,"Ġscientists":5519,"Ġlegislation":5520,"ĠWork":5521,"ĠSpr":5522,"ÃĤ":5523,"ĠHuman":5524,"Ġè":5525,"Ġsurvey":5526,"Ġrich":5527,"rip":5528,"Ġmaintain":5529,"Ġflo":5530,"Ġleadership":5531,"stream":5532,"ĠIslamic":5533,"Ġ01":5534,"ĠCollege":5535,"Ġmagic":5536,"ĠPrime":5537,"Ġfigures":5538,"2017":5539,"inder":5540,"xual":5541,"ĠDead":5542,"Ġabsolutely":5543,"Ġfourth":5544,"Ġpresented":5545,"respond":5546,"rible":5547,"Ġalcohol":5548,"ato":5549,"ĠDE":5550,"porary":5551,"Ġgrab":5552,"Ġvari":5553,"Ġquant":5554,"ĠPhoto":5555,"Ġplus":5556,"rick":5557,"arks":5558,"Ġalternative":5559,"Ġpil":5560,"Ġapprox":5561,"that":5562,"Ġobjects":5563,"ĠRo":5564,"ĠAndroid":5565,"Ġsignificantly":5566,"ĠRoad":5567,"kay":5568,"Read":5569,"avor":5570,"Ġacknow":5571,"ĠHD":5572,"ĠSing":5573,"Or":5574,"ĠMont":5575,"Ġuns":5576,"prof":5577,"Ġnegoti":5578,"ĠArch":5579,"iki":5580,"Ġtelevision":5581,"ĠJewish":5582,"Ġcommittee":5583,"Ġmotor":5584,"Ġappearance":5585,"Ġsitting":5586,"Ġstrike":5587,"ĠDown":5588,"comp":5589,"ĠHist":5590,"Ġfold":5591,"acement":5592,"ĠLouis":5593,"Ġbelong":5594,"ĠâĢ¢":5595,"Ġmort":5596,"Ġprepared":5597,"Ġ64":5598,"ĠMaster":5599,"Ġindeed":5600,"ĠDen":5601,"Ġrent":5602,"TA":5603,"ourney":5604,"arc":5605,"Su":5606,"97":5607,"Ġadvice":5608,"Ġchanging":5609,"Ġlisted":5610,"Ġlaunched":5611,"isation":5612,"ĠPeter":5613,"ishes":5614,"Ġlived":5615,"ĠMel":5616,"ĠSupreme":5617,"ĠFederal":5618,"Ġ);":5619,"ructure":5620,"Ġsets":5621,"Ġphilos":5622,"uous":5623,"ĠÂł":5624,"Ġapplied":5625,"ĠNOT":5626,"Ġhousing":5627,"ĠMount":5628,"Ġodd":5629,"Ġsust":5630,"DA":5631,"fficient":5632,"Ġ?":5633,"olved":5634,"Ġpowers":5635,"Ġthr":5636,"Ġremaining":5637,"ĠWater":5638,"LC":5639,"Ġcauses":5640,"ãģ®":5641,"Ġmanner":5642,"ads":5643,"Ġsuggests":5644,"Ġends":5645,"standing":5646,"fig":5647,"ĠDun":5648,"idth":5649,"Ġgay":5650,"Ġtermin":5651,"ĠAngeles":5652,"MS":5653,"Ġscientific":5654,"Ġcoal":5655,"apers":5656,"bar":5657,"ĠThomas":5658,"Ġsym":5659,"ĠRun":5660,"this":5661,"PC":5662,"igrants":5663,"Ġminute":5664,"ĠDistrict":5665,"cellent":5666,"Ġleaves":5667,"Ġcompleted":5668,"amin":5669,"Ġfocused":5670,"Ġmonitor":5671,"Ġvehicles":5672,"MA":5673,"ĠMass":5674,"ĠGrand":5675,"Ġaffected":5676,"itutional":5677,"Ġconstruct":5678,"Ġfollows":5679,"Ġton":5680,"reens":5681,"Ġhomes":5682,"ĠExt":5683,"ĠLevel":5684,"rast":5685,"ĠIr":5686,"Ġelim":5687,"Ġlargely":5688,"ĠJoe":5689,"Ġvotes":5690,"alls":5691,"Ġbusinesses":5692,"ĠFoundation":5693,"ĠCentral":5694,"Ġyards":5695,"Ġmaterials":5696,"ulner":5697,"Ġguide":5698,"Ġcloser":5699,"ums":5700,"Ġsports":5701,"eder":5702,"Just":5703,"Ġtaxes":5704,"84":5705,"ĠOld":5706,"Ġdecade":5707,"ola":5708,"Ġvir":5709,"Ġdropped":5710,"Ġdelay":5711,"itect":5712,"Ġsecure":5713,"stein":5714,"level":5715,"Ġtreated":5716,"Ġfiled":5717,"aine":5718,"Ġvan":5719,"Ġmir":5720,"Ġcolumn":5721,"icted":5722,"eper":5723,"Ġrot":5724,"Ġconsult":5725,"Ġentry":5726,"Ġmarijuana":5727,"ĠDou":5728,"Ġapparently":5729,"oking":5730,"clusive":5731,"Ġincreases":5732,"ano":5733,"Ġspecifically":5734,"Ġtele":5735,"ensions":5736,"Ġreligion":5737,"abilities":5738,"Ġframe":5739,"ĠNote":5740,"ĠLee":5741,"Ġhelping":5742,"Ġedge":5743,"oston":5744,"Ġorganizations":5745,"Ãĥ":5746,"ĠBoth":5747,"hips":5748,"Ġbigger":5749,"Ġboost":5750,"ĠStand":5751,"Ġrow":5752,"uls":5753,"abase":5754,"Ġrid":5755,"Let":5756,"aren":5757,"rave":5758,"Ġstret":5759,"PD":5760,"Ġvision":5761,"Ġwearing":5762,"Ġappreci":5763,"Ġaward":5764,"ĠUse":5765,"Ġfactor":5766,"war":5767,"ulations":5768,")(":5769,"Ġgod":5770,"Ġterrit":5771,"Ġparam":5772,"asts":5773,"87":5774,"Ġenemies":5775,"ĠGames":5776,"FF":5777,"Ġaccident":5778,"Well":5779,"ĠMartin":5780,"TER":5781,"Ġath":5782,"ĠHell":5783,"Ġforg":5784,"Ġveter":5785,"ĠMedic":5786,"free":5787,"Ġstars":5788,"Ġexpensive":5789,"Ġacad":5790,"rawn":5791,"ĠWhe":5792,"Ġlock":5793,"Ġformat":5794,"Ġsoldiers":5795,"sm":5796,"Ġagent":5797,"Ġresponsibility":5798,"ora":5799,"ĠScience":5800,"Ġrapid":5801,"Ġtough":5802,"ĠJesus":5803,"Ġbelieves":5804,"ML":5805,"Ġwear":5806,"lete":5807,"ÃĥÃĤ":5808,"ĠDri":5809,"Ġcommission":5810,"ĠBob":5811,"Oh":5812,"aped":5813,"Ġwarm":5814,"ÃĥÃĤÃĥÃĤ":5815,"Ġ2003":5816,"ortion":5817,"Ġhasn":5818,"uster":5819,"Ġunivers":5820,"ĠIll":5821,"Ġking":5822,"ologies":5823,"94":5824,"ĠTem":5825,"ĠMos":5826,"Ġpatient":5827,"ĠMexico":5828,"cean":5829,"ĠDeath":5830,"ĠSanders":5831,"you":5832,"ĠCast":5833,"ĠCompany":5834,"pty":5835,"Ġhappening":5836,"FP":5837,"ĠBattle":5838,"Ġbought":5839,"Am":5840,"Mod":5841,"Us":5842,"uters":5843,"ĠCre":5844,"ĠThose":5845,"Ġ44":5846,"iser":5847,"Ġsoul":5848,"ĠTop":5849,"ĠHarry":5850,"ĠAw":5851,"Ġseat":5852,"ffee":5853,"Ġrevolution":5854,"Ġ(\"":5855,"ĠDuring":5856,"ette":5857,"Ġring":5858,"Ġoffensive":5859,"Ġreturns":5860,"Ġvideos":5861,"Ġdiscl":5862,"Ġfamous":5863,"enced":5864,"ĠSign":5865,"ĠRiver":5866,"Ġ300":5867,"PM":5868,"ĠBus":5869,"ĠCH":5870,"Ġcandidates":5871,"arden":5872,"Ġpercentage":5873,"Ġvisual":5874,"Ġthank":5875,"Ġtrouble":5876,"nergy":5877,"Ġ2001":5878,"Ġprove":5879,"ashion":5880,"Ġenh":5881,"ĠLong":5882,"UM":5883,"Ġconnected":5884,"Ġpossibility":5885,"Over":5886,"Ġexpert":5887,"Ġlibrary":5888,"arts":5889,"ĠDirector":5890,"Ġfellow":5891,"92":5892,"irty":5893,"Ġdry":5894,"Ġsigns":5895,"ĠLove":5896,"Ġquiet":5897,"foot":5898,"Ġpure":5899,"ĠHun":5900,"Ġfilled":5901,"phas":5902,"ĠElect":5903,"endment":5904,"ĠExpl":5905,"Ġunable":5906,"ns":5907,"mo":5908,"Ġvast":5909,"obe":5910,"Ġidentify":5911,"apping":5912,"ĠCarolina":5913,"gress":5914,"Ġprote":5915,"Ġfish":5916,"Ġcircumstances":5917,"razy":5918,"ĠPhot":5919,"Ġbodies":5920,"ĠMur":5921,"Ġdeveloping":5922,"ĠAR":5923,"Ġexperienced":5924,"Ġsubstant":5925,"ĠBoard":5926,"esome":5927,"Ġdomestic":5928,"Ġcombined":5929,"ĠPut":5930,"Ġchemical":5931,"ĠChild":5932,"Ġpool":5933,"ĠCy":5934,"Ġegg":5935,"cons":5936,"sters":5937,"Ġhurt":5938,"Ġmarkets":5939,"Ġconservative":5940,"Ġsupporters":5941,"Ġagencies":5942,"idel":5943,"Ob":5944,"urb":5945,"Ġ43":5946,"ĠDefense":5947,"ye":5948,"ĠAp":5949,"dule":5950,"Ġtemperature":5951,"Ġconducted":5952,"ĠChief":5953,"Ġpulled":5954,"Ġfol":5955,"Last":5956,"onto":5957,"osis":5958,"VER":5959,"Des":5960,"ĠPan":5961,"First":5962,"Ġadvance":5963,"Ġlicense":5964,"rors":5965,"ĠJon":5966,"Ġimagine":5967,"Ġhell":5968,"Ġfixed":5969,"Ġincor":5970,"osite":5971,"ĠLog":5972,"icken":5973,"]:":5974,"Ġsurprise":5975,"hab":5976,"Ġcraft":5977,"olt":5978,"ĠJul":5979,"Ġdial":5980,"Ġrelevant":5981,"Ġentered":5982,"Ġleads":5983,"ĠAD":5984,"ĠClean":5985,"Ġpictures":5986,"essor":5987,"Ġalt":5988,"Ġpaying":5989,"Per":5990,"ĠMarket":5991,"Ġupdates":5992,"amily":5993,"ĠType":5994,"ĠHome":5995,"Ġ55":5996,"sembly":5997,"rome":5998,"83":5999,"Ġgreatest":6000,"Ġheight":6001,"Ġheav":6002,"aints":6003,"Ġlisten":6004,"aser":6005,"ĠSH":6006,"Ġcapable":6007,"acle":6008,"Ġperspect":6009,"inating":6010,"Ġoffering":6011,"rypt":6012,"ĠDevelop":6013,"abin":6014,"rc":6015,"Ġbright":6016,"alty":6017,"arrow":6018,"Ġsuppl":6019,"inding":6020,"acked":6021,"gypt":6022,"ĠAnother":6023,"pg":6024,"ĠVirginia":6025,"ĠLu":6026,"Ġplanned":6027,"Ġpit":6028,"Ġsweet":6029,"Type":6030,"ĠDi":6031,"Ġtypically":6032,"ĠFrancisco":6033,"Ġprospect":6034,"ĠDan":6035,"Ġteen":6036,"rees":6037,"Ġsched":6038,"Ġhol":6039,"Ġscr":6040,"Ġlots":6041,"life":6042,"Ġnewsp":6043,"Ġforget":6044,"ĠNone":6045,"ĠMiddle":6046,"ĠRyan":6047,"edd":6048,"Ġsevere":6049,"Ġsuit":6050,"ller":6051,"93":6052,"Ġcorrespond":6053,"Ġexplos":6054,"uations":6055,"Ġflag":6056,"game":6057,"rid":6058,"Ġprin":6059,"ĠData":6060,"Ġdeploy":6061,"ĠEnter":6062,"suit":6063,"ghan":6064,"ĠMen":6065,"Ġthoughts":6066,"Ġmatters":6067,"Ġadapt":6068,"ĠAri":6069,"Ġfill":6070,"Ġforth":6071,"Ġsam":6072,"Ġ41":6073,"Ġpayment":6074,"ĠHor":6075,"Ġspring":6076,"duc":6077,"Ġlosing":6078,"Ġbringing":6079,"FO":6080,"ala":6081,"Ġdistribution":6082,"hered":6083,"bour":6084,"ĠIsraeli":6085,"oma":6086,"Ġcombination":6087,"Ġplenty":6088,"VE":6089,"Can":6090,"ĠHaw":6091,"Ġperman":6092,"ĠSpecial":6093,"Ġtow":6094,"Ġseeking":6095,"Ġexamples":6096,"Ġclasses":6097,"cr":6098,"Ġbeer":6099,"Ġmoves":6100,"ĠIP":6101,"ĠKn":6102,"Ġpanel":6103,"Even":6104,"Ġproperly":6105,"Ġris":6106,"Ġplug":6107,"Ġestimated":6108,"Every":6109,"Ġdefensive":6110,"agraph":6111,"Ġpregn":6112,"Ġinstit":6113,"ĠVict":6114,"Ġvolume":6115,"Ġpositions":6116,"Ġlinks":6117,"ĠProgram":6118,"ĠWeek":6119,"agues":6120,"Ġtransform":6121,"ker":6122,"ĠCEO":6123,"Ġcas":6124,"Ġopponent":6125,"Ġtweet":6126,"ĠCode":6127,"Ġshop":6128,"Ġfly":6129,"Ġtalks":6130,"Ġbag":6131,"Phone":6132,"Ġaid":6133,"Ġplants":6134,"Ġ65":6135,"Ġattorney":6136,"arters":6137,"quest":6138,"ĠMagic":6139,"Ġbegins":6140,"Ġmyster":6141,"Ġenvironmental":6142,"Ġstorage":6143,"NN":6144,"Ġmarg":6145,"Ġske":6146,"Ġmetal":6147,"elly":6148,"Ġordered":6149,"Ġremained":6150,"Ġloved":6151,"Ġprompt":6152,"Ġupdated":6153,"Ġexperts":6154,"Ġwalking":6155,"Ġancient":6156,"Ġperformed":6157,"ATE":6158,"Ġneither":6159,"iency":6160,"Ġmanufacture":6161,"ĠPak":6162,"Ġselected":6163,"Ġmine":6164,"Ġultimately":6165,"Ġexplan":6166,"Ġlabel":6167,"ĠServices":6168,"ributed":6169,"Trump":6170,"Ġsyn":6171,"ĠUlt":6172,"SC":6173,"Ġmeat":6174,"Ġgiant":6175,"ĠWars":6176,"ĠON":6177,"Ġadm":6178,"Ġinterpret":6179,"Ġevening":6180,"Ġevil":6181,"ĠBoston":6182,"ĠWild":6183,"ĠÃ":6184,"ĠBitcoin":6185,"ĠAmazon":6186,"Dr":6187,"ĠInformation":6188,"Ġobviously":6189,"Ġadvanced":6190,"Photo":6191,"olar":6192,"Ġweather":6193,"Ġsymbol":6194,"Ġsole":6195,"Ġpotentially":6196,"oster":6197,"Ġoriginally":6198,"mun":6199,"300":6200,"aze":6201,"essions":6202,"Ġdeck":6203,"Ġstood":6204,"Ġyouth":6205,"ĠBern":6206,"Rep":6207,"ĠTest":6208,"Ġbasically":6209,"otic":6210,"Ġinvolve":6211,"olit":6212,"lyn":6213,"See":6214,"Ġaircraft":6215,"Ġconfirm":6216,"EW":6217,"Ġmessages":6218,"ĠRichard":6219,"Ġkit":6220,"Ġprohib":6221,"Ġvulner":6222,"isters":6223,"Ġexistence":6224,"Ġturning":6225,"ĠSP":6226,"Ġdesire":6227,"Ġflat":6228,"Ġment":6229,"season":6230,"anges":6231,"Ġneighborhood":6232,"ĠLake":6233,"ATION":6234,"Ġpointed":6235,"bur":6236,"Ġinnov":6237,"ucks":6238,"UL":6239,"Ġprofessor":6240,"Ġexpressed":6241,"AB":6242,"icious":6243,"Ġ2002":6244,"ĠDev":6245,"Ġsession":6246,"Ġbare":6247,"sen":6248,"Ġdiss":6249,"ĠCath":6250,"ĠPass":6251,"ĠPoint":6252,"Ġdoctor":6253,"orrow":6254,"ailed":6255,"ĠRub":6256,"ĠDC":6257,"ĠCharl":6258,"person":6259,"Ġwriter":6260,"ighters":6261,"ureau":6262,"Ġoblig":6263,"Ġrecorded":6264,"Ġbroke":6265,"Ġorders":6266,"ilty":6267,"Ġmotion":6268,"inity":6269,"law":6270,"adium":6271,"Ġimmigration":6272,"Ġcontrast":6273,"Ġbatt":6274,"Ġexcellent":6275,"Ġtechnical":6276,"ami":6277,"Ġtun":6278,"Ġcloud":6279,"ĠYear":6280,"geon":6281,"Ġcreation":6282,"Ġstrange":6283,"Ġauth":6284,"Ġfort":6285,"born":6286,"Ġextent":6287,"ĠToday":6288,"ĠClub":6289,"Ġrain":6290,"Ġsample":6291,"Ġaccepted":6292,"Ġtact":6293,"Ġfired":6294,"ĠSon":6295,"Ġstands":6296,"Ġboot":6297,"Ġ47":6298,"Ġstatements":6299,"Ġversions":6300,"Ġselling":6301,"ounded":6302,"Ġ1990":6303,"Ġweren":6304,"ĠWatch":6305,"Ġexperiment":6306,"Post":6307,"Ġretail":6308,"uled":6309,"Inst":6310,"unte":6311,"ãĥ¼":6312,"Ġdepart":6313,"Ġbond":6314,"ivery":6315,"ompl":6316,"Ġreaction":6317,"ĠSyrian":6318,"ĠPac":6319,"apped":6320,"aniel":6321,"DP":6322,"Ġresolution":6323,"Ġreact":6324,"Ġapproved":6325,"onom":6326,"mond":6327,"ĠOffic":6328,"---":6329,"Ġreplace":6330,"Ġtack":6331,"Ġsport":6332,"Ġchain":6333,"Ġemergency":6334,"rad":6335,"ĠPalestin":6336,"Ġ46":6337,"Ġautomatically":6338,"Ġroute":6339,"Ġpal":6340,"Ġbanks":6341,"ĠParis":6342,"ĠMedia":6343,"road":6344,"icing":6345,"ixt":6346,"isted":6347,"Ġgrew":6348,"Ġcoord":6349,"ĠWhere":6350,"omin":6351,"Ġsubs":6352,"��":6353,"Ġ±":6354,"Ġcorporate":6355,"Ġselection":6356,"noon":6357,"ĠReport":6358,"cs":6359,"cluding":6360,"orders":6361,"anche":6362,"ĠIts":6363,"Ġslowly":6364,"ĠEgypt":6365,"ĠAcc":6366,"Ġcolle":6367,"iques":6368,"EX":6369,"Ġattempts":6370,"url":6371,"ĠCross":6372,"Ġfindings":6373,"ĠSC":6374,"ĠOR":6375,"Ġindex":6376,"ensity":6377,"ĠWay":6378,"ĠLand":6379,"Ġshock":6380,"dis":6381,"Ġdynam":6382,"Ġcart":6383,"mosp":6384,"Since":6385,"iest":6386,"ĠBoy":6387,"Ġstorm":6388,"ĠContin":6389,"2013":6390,"hew":6391,"ilit":6392,"Ġessential":6393,"iquid":6394,"Other":6395,"ivered":6396,"Ġreasonable":6397,"Act":6398,"Ġsubsequ":6399,"ĠPack":6400,"ĠFort":6401,"Ġconsidering":6402,"Ġuniversity":6403,"log":6404,"Ġmarried":6405,"Ġillust":6406,"ĠTrue":6407,"£ı":6408,"Ġnumerous":6409,"rastructure":6410,"Ġseriously":6411,"Ġreferred":6412,"ua":6413,"Ġconsistent":6414,"onna":6415,"ĠReal":6416,"ruption":6417,"ciples":6418,"Ġfacts":6419,"91":6420,"otes":6421,"erg":6422,"Then":6423,"Ġaccompl":6424,"Note":6425,"Ġrevenue":6426,"Ġpassing":6427,"Ġmal":6428,"een":6429,"ĠYet":6430,"Ġgather":6431,"terday":6432,"ework":6433,"ĠAuthor":6434,"Pe":6435,"Ġoptim":6436,"Ġrub":6437,"Ġè£ı":6438,"Ġunknown":6439,"stone":6440,"Ġunion":6441,"olve":6442,"Ġopportunities":6443,"Ġbrowser":6444,"ĠWal":6445,"ĠCost":6446,"Ġreporting":6447,"sts":6448,"pet":6449,"Ġsand":6450,"Ġsuddenly":6451,"Ġsurprising":6452,"ĠVR":6453,"Ġsomewhat":6454,"ĠBas":6455,"ulture":6456,"izz":6457,"ĠCD":6458,"Ġchallenges":6459,"Ġsettings":6460,"Ġexperiences":6461,"ĠFull":6462,"Ġcann":6463,"Ġreceiving":6464,"EST":6465,"Ġjoint":6466,"Ġcultural":6467,"Ġast":6468,"82":6469,"astern":6470,"ceived":6471,"ĠCru":6472,"Ġbull":6473,"pired":6474,"amm":6475,"Ġfacing":6476,"power":6477,"Ġboss":6478,"ĠHol":6479,"Ġinstr":6480,"Ġincreasingly":6481,"Ġshift":6482,"Ġstreets":6483,"ĠWilliams":6484,"abb":6485,"Ġlie":6486,"Ġlaugh":6487,"ĠCa":6488,"PL":6489,"Ġadults":6490,"Ġcustomer":6491,"Ġobtained":6492,"Ġsupporting":6493,"html":6494,"fire":6495,"Ġdetailed":6496,"Ġpicked":6497,"ĠRight":6498,"lder":6499,"EE":6500,"stood":6501,"ĠKim":6502,"Ġwire":6503,"Ġsight":6504,"Ġdevelopers":6505,"Ġpersons":6506,"Ġsad":6507,"Ġcup":6508,"Ġwarning":6509,"Ġboys":6510,"long":6511,"Ġbird":6512,"fo":6513,"Ġwal":6514,"Ġobserved":6515,"Ġzone":6516,"iveness":6517,"Ġchannel":6518,"cript":6519,"Ġrefused":6520,"ĠAgain":6521,"Ġsuc":6522,"Ġspokesman":6523,"ĠRef":6524,"rite":6525,"ouston":6526,"ãĥ³":6527,"ĠSher":6528,"Ġacts":6529,"ĠName":6530,"Ġstruggle":6531,"arry":6532,"ometimes":6533,"Ġdiscrim":6534,"HT":6535,"Ġcategory":6536,"Ġrealize":6537,"Ġemployee":6538,"ĠAfghan":6539,"enger":6540,"Ġguns":6541,"ĠSteve":6542,"ĠMot":6543,"ĠOl":6544,"oked":6545,"Ġthick":6546,"Ġfairly":6547,"illy":6548,"Ġsurve":6549,"ĠMat":6550,"weight":6551,"âĶ":6552,"Ġtroops":6553,"Ġagents":6554,"Ġbattery":6555,"Ġmotiv":6556,"á":6557,"Sec":6558,"den":6559,"overy":6560,"LS":6561,"Ġflu":6562,"Ġconfident":6563,"ĠOper":6564,"Ġempty":6565,"Ġphen":6566,"Ġsector":6567,"Ġexcited":6568,"Ġremote":6569,"aph":6570,"oen":6571,"Ġdestroyed":6572,"Ġmoral":6573,"ĠHP":6574,"ĠRon":6575,"Ġdress":6576,"ĠBat":6577,"Ġlit":6578,"ĠMS":6579,"Ġaf":6580,"HL":6581,"rum":6582,"isms":6583,"Ġshouldn":6584,"Ġsympt":6585,"ĠToronto":6586,"hetic":6587,"Ġcarbon":6588,"Ġinstalled":6589,"Ġviolent":6590,"Ġsolar":6591,"ja":6592,"Ġpractices":6593,"Ġride":6594,"ĠPenn":6595,"Ġimproved":6596,"Ġaudio":6597,"Ġbehavi":6598,"ĠPS":6599,"Ġeating":6600,"Data":6601,"ĠReview":6602,"pass":6603,"claim":6604,"uated":6605,"angers":6606,"chen":6607,"Ġproperties":6608,"Ġanywhere":6609,"Another":6610,"Ġblow":6611,"ĠJackson":6612,"Ġproud":6613,"Ġplane":6614,"lines":6615,"Ġsquare":6616,"Ġproof":6617,"ansas":6618,"Ġtalked":6619,"makers":6620,"Ġsister":6621,"Ġholds":6622,"Ġresident":6623,"Ġ==":6624,"Ġresistance":6625,"Ġsplit":6626,"Ġprosecut":6627,"Ġconfidence":6628,"resents":6629,"Ġcuts":6630,"Ġexception":6631,"Ġzero":6632,"Getty":6633,"Ġcopyright":6634,"Ġtotally":6635,"ormal":6636,"ifications":6637,"ĠAustralian":6638,"Ġsick":6639,"Ġ150":6640,"Ġhousehold":6641,"Ġfees":6642,"Ġdrivers":6643,"ogen":6644,"ĠNY":6645,"Ġnecessarily":6646,"Ġregulations":6647,"earing":6648,"sl":6649,"Ġperspective":6650,"care":6651,"icial":6652,"His":6653,"Ġescape":6654,"Ġsurprised":6655,"ĠVan":6656,"urrent":6657,"Ġvac":6658,"81":6659,"ĠThus":6660,"Ġemphas":6661,"ĠChampions":6662,"ĠIce":6663,"Ġnarr":6664,"Ġheads":6665,"Ġcausing":6666,"bel":6667,"fortunately":6668,"ĠMa":6669,"Ġtargets":6670,"cipl":6671,"Ġafternoon":6672,"Ġadds":6673,"ĠMaybe":6674,"ĠFour":6675,"essed":6676,"plete":6677,"Ġusual":6678,"cho":6679,"ingu":6680,"Ġwithd":6681,"ĠEnergy":6682,"ĠEconom":6683,"OO":6684,"Ġarticles":6685,"Ġinjured":6686,"Ġmanage":6687,"Ġexplains":6688,"Ġdiagn":6689,"Rec":6690,"atures":6691,"Ġlinked":6692,"Ġdiscussed":6693,"Ġexplo":6694,"Ġoccasion":6695,"athan":6696,"Ġopposite":6697,"Ġfaces":6698,"Ġdenied":6699,"ĠKnight":6700,"Ġnut":6701,"Ġapproximately":6702,"Ġdisappoint":6703,"onymous":6704,"ĠBest":6705,"ĠLo":6706,"ĠHy":6707,"ĠAff":6708,"Ġvoting":6709,"anwhile":6710,"ĠIII":6711,"Ġinstitutions":6712,"agram":6713,"ĠDaily":6714,"Ġdrag":6715,"Ġnearby":6716,"Ġguilty":6717,"Ġconver":6718,"Pre":6719,"ship":6720,"Ġreward":6721,"Ġphilosoph":6722,"ĠSS":6723,"ugh":6724,"Ġapps":6725,"friend":6726,"Ġupper":6727,"Ġadvert":6728,"Ġsnow":6729,"Ġfrust":6730,"Ġourselves":6731,"Fr":6732,"ĠDie":6733,"ampion":6734,"Ġdismiss":6735,"Ġcere":6736,"Ġsignal":6737,"from":6738,"Ġ).":6739,"Ġ52":6740,"Ġcrimes":6741,"itors":6742,"estival":6743,"useum":6744,"Ġcouncil":6745,"ĠSaud":6746,"May":6747,"ĠGun":6748,"ician":6749,"ether":6750,"Ġsufficient":6751,"ĠHen":6752,"sole":6753,"Ġhistorical":6754,"ĠFar":6755,"ĠTurn":6756,"Ġpin":6757,"Ġsucceed":6758,"mat":6759,"lymp":6760,"Ġtradition":6761,"ĠOk":6762,"Ġcro":6763,"Ġdescription":6764,"alle":6765,"Ġsky":6766,"Te":6767,"Ġwidely":6768,"Ġwave":6769,"Ġdefinition":6770,"ĠJews":6771,"Ġcycle":6772,"Ġrefere":6773,"Ġbrings":6774,"usal":6775,"Ġalive":6776,"Ġfrequently":6777,"Ġintention":6778,"ĠControl":6779,"lv":6780,"ystem":6781,"Ġprivacy":6782,"gent":6783,"rence":6784,"ĠQuest":6785,"ĠChristmas":6786,"Ġrail":6787,"Ġcooper":6788,"Ġtested":6789,"ĠCapt":6790,"asks":6791,"Ġcomfortable":6792,"Ġdelivered":6793,"scape":6794,"Ġdepth":6795,"ĠGOP":6796,"Ġwrites":6797,"Ġassets":6798,"Ġsav":6799,"iments":6800,"Ġtransition":6801,"Ġartist":6802,"ĠLook":6803,"Ġlob":6804,"Ġcomponents":6805,"arity":6806,"Ġwalked":6807,"Ġroot":6808,"Ġparticipants":6809,"Ġnoticed":6810,"Ġresc":6811,"Ġnav":6812,"ĠAdminist":6813,"da":6814,"utral":6815,"plate":6816,"Ġimportance":6817,"Ġassert":6818,"iously":6819,"cription":6820,"Ġinjuries":6821,"ĠCheck":6822,"Ġregistered":6823,"Ġintent":6824,"Ġmissed":6825,"ographic":6826,"Ġsentence":6827,"ounter":6828,"Ġassistance":6829,"evin":6830,"Ġdatabase":6831,"Ġbuildings":6832,"Ġclassic":6833,"Ġthinks":6834,"ĠOhio":6835,"Pr":6836,"ugg":6837,"Ġfee":6838,"pan":6839,"Ġeffectively":6840,"Ġfacility":6841,"Ġbear":6842,"Ġchapter":6843,"Ġdogs":6844,"ĠColumb":6845,"Ġlatter":6846,"itial":6847,"Ġadmitted":6848,"TV":6849,"ĠGeorg":6850,"Ġposts":6851,"\\\\":6852,"Ġlawyer":6853,"Ġequival":6854,"Ġmand":6855,"Ġcontrolled":6856,"ĠWalk":6857,"ĠAndrew":6858,"Ġmenu":6859,"amental":6860,"Ġprotected":6861,"va":6862,"Ġadministr":6863,"oral":6864,"Ġrein":6865,"ĠSar":6866,"Ġamounts":6867,"Ġnative":6868,"ĠMoon":6869,"Ġrepresents":6870,"Ġabandon":6871,"Ġcarrying":6872,"Ġtank":6873,"mary":6874,"Ġdeclared":6875,"Tube":6876,"Ġhat":6877,"Ġpunish":6878,"ellect":6879,"mes":6880,"Ġuniverse":6881,"ĠRod":6882,"phy":6883,"Ġinfrastructure":6884,"Ġ51":6885,"Ġopposed":6886,"ownt":6887,"ca":6888,"ĠMake":6889,"Ġhardware":6890,"Ġcoffee":6891,"Rel":6892,"bal":6893,"world":6894,"ĠSaf":6895,"ĠSea":6896,"inals":6897,"Ġowned":6898,"Ġhall":6899,"ersion":6900,"Ġdescribe":6901,"ĠPot":6902,"Ġportion":6903,"Ġatmosp":6904,"Ġgovernments":6905,"Ġdepending":6906,"Ġoffense":6907,"Ġtrick":6908,"awa":6909,"ĠLine":6910,"ĠVis":6911,"ĠHard":6912,"ĠOrig":6913,"ĠClick":6914,"Ġdesk":6915,"ĠValley":6916,"ĠSov":6917,"Ġmovies":6918,"Ġremark":6919,"Ġmail":6920,"Ġconscious":6921,"Ġruling":6922,"ĠRights":6923,"Ġmedic":6924,"hent":6925,"ĠWomen":6926,"><":6927,"Ġreplaced":6928,"ĠPrem":6929,"ĠThanks":6930,"Ġrenew":6931,"ĠBall":6932,"iform":6933,"Ġshots":6934,"Comm":6935,"Ġarmed":6936,"Ġconstant":6937,"Ġtaste":6938,"Ġrealized":6939,"Ġbuff":6940,"Ġmo":6941,"Ġefficient":6942,"Most":6943,"oration":6944,"ifies":6945,"Ġcommunication":6946,"Ġflood":6947,"Ġconsequences":6948,"Ġanyway":6949,"igg":6950,"ĠGM":6951,"ĠThank":6952,"Ġiron":6953,"Ġevolution":6954,"ĠCop":6955,"twitter":6956,"Ġ95":6957,"Ġrelationships":6958,"adel":6959,"ĠYoung":6960,"Ġproposal":6961,"ayers":6962,"uilding":6963,"ĠHot":6964,"ORE":6965,"cos":6966,"Ġcollabor":6967,"PG":6968,"axy":6969,"Ġknowing":6970,"Ġsupports":6971,"owed":6972,"Ġcontrols":6973,"Ġmerely":6974,"umer":6975,"Ġathlet":6976,"Ġfashion":6977,"path":6978,"Ġgift":6979,"Ġera":6980,"AND":6981,"Ġkinds":6982,"ĠKorean":6983,"Ġlegit":6984,"ulous":6985,"Ġessentially":6986,"Ġtherap":6987,"nic":6988,"Ġsuffered":6989,"Ġhur":6990,"Ġpromise":6991,"Ġexcess":6992,"Ġoverw":6993,"Ġprime":6994,"ĠHouston":6995,"erry":6996,"ĠMs":6997,"RS":6998,"2012":6999,"Ġstores":7000,"ĠOlymp":7001,"Ġjourney":7002,"Although":7003,"Sub":7004,"ĠEduc":7005,"ĠChapter":7006,"Ġrequests":7007,"Ġconsumers":7008,"Ġtiny":7009,"Ġisol":7010,"ĠFair":7011,"ba":7012,"ĠYOU":7013,"Ġcrash":7014,"celer":7015,"Ġemotional":7016,"Ġgoods":7017,"Ġelected":7018,"Ġmoder":7019,"ĠLinux":7020,"Ġblocks":7021,"Ġisland":7022,"ĠSociety":7023,"Ġelections":7024,"Ġbroadcast":7025,"Ġcheap":7026,"Ġnations":7027,"Ġseasons":7028,"400":7029,"Ġwaste":7030,"ĠSat":7031,"Ġfields":7032,"employ":7033,"Ġprofile":7034,"Ġauthors":7035,"ALL":7036,"ĠGra":7037,"west":7038,"ĠTy":7039,"Ġdeaths":7040,"Ġvacc":7041,"Ġformed":7042,"Ġdu":7043,"Ġongoing":7044,"ĠMuslims":7045,"elf":7046,"igure":7047,"Ġassume":7048,"ĠUkraine":7049,"water":7050,"Ġcoast":7051,"Ġvoted":7052,"gor":7053,"ĠAS":7054,"ĠMichigan":7055,"aza":7056,"ĠArm":7057,"iro":7058,"Ġflex":7059,"asters":7060,"''":7061,"Ġwelcome":7062,"arl":7063,"Ġlocations":7064,"igation":7065,"ĠFil":7066,"Ġbuying":7067,"Ġarchitect":7068,"Ġharder":7069,"ĠCub":7070,"Ġinterface":7071,"Ġrestaurant":7072,"Ġdiscover":7073,"Ġexceed":7074,"Ġfavour":7075,"gery":7076,"Ġduty":7077,"Ġpitch":7078,"ador":7079,"ĠMach":7080,"boy":7081,"Ġresponded":7082,"Ġextended":7083,"hers":7084,"Many":7085,"raid":7086,"ifer":7087,"ĠIns":7088,"Ser":7089,"Ġmedium":7090,"she":7091,"ĠSports":7092,"Ġmagazine":7093,"utation":7094,"Ġlimits":7095,"ĠGall":7096,"Ġexternal":7097,"razil":7098,"Ġyounger":7099,"tle":7100,"Ġremind":7101,"ĠCON":7102,"Ġimmediate":7103,"Ġhidden":7104,"Ġvolunte":7105,"Ġsimpl":7106,"odcast":7107,"Ġphase":7108,"dr":7109,"Ġplot":7110,"Ġexposure":7111,"RI":7112,"ograp":7113,"vin":7114,"anish":7115,"ĠAcad":7116,"ĠEngine":7117,"Ġexpansion":7118,"ĠPay":7119,"Your":7120,"Ġpushed":7121,"ĠEll":7122,"ĠHead":7123,"Ġmarketing":7124,"ĠAC":7125,"ket":7126,"Ġhits":7127,"Ġgro":7128,"ĠAge":7129,"ĠScot":7130,"][":7131,"Ġstim":7132,"ĠiPhone":7133,"ĪĴ":7134,"Ġnarrow":7135,"ĠGetty":7136,"ĠTurkey":7137,"Ġperfectly":7138,"Ġenable":7139,"utch":7140,"Ġprecise":7141,"Ġregime":7142,"Ġshif":7143,"Ġcompens":7144,"gun":7145,"div":7146,"Ġchosen":7147,"ĠKen":7148,"Any":7149,"Ġtrees":7150,"Ġrecommended":7151,"ĠRen":7152,"uable":7153,"ĠHT":7154,"Follow":7155,"EG":7156,"ĠHand":7157,"ĠKenn":7158,"Ġarguments":7159,"Ġexists":7160,"Ġbike":7161,"ĠConserv":7162,"Ġbreaking":7163,"ĠGar":7164,"Ġcrazy":7165,"Ġvirtual":7166,"aylor":7167,"ixel":7168,"Ġ1980":7169,"Ġpermission":7170,"ĠSeries":7171,"Ġconsumer":7172,"Ġclosely":7173,"called":7174,"Ġ54":7175,"Ġhopes":7176,"Ġarray":7177,"ĠWin":7178,"ĠLabour":7179,"Ġspons":7180,"ĠIre":7181,"Ġpow":7182,"Ġreaders":7183,"Ġemployment":7184,"Ġcreature":7185,"Ġresulting":7186,"Ġaccurate":7187,"Ġmoments":7188,"Ġargued":7189,"Ġped":7190,"During":7191,"Ġ53":7192,"ĠTal":7193,"Ġsought":7194,"Ġsuffering":7195,"Ġicon":7196,"lee":7197,"Ġ($":7198,"alian":7199,"°":7200,"Ġpra":7201,"Ġbonus":7202,"(\"":7203,"ko":7204,"Ġacting":7205,"DE":7206,"fall":7207,"Ġcomparison":7208,"Ġsmooth":7209,"ĠNAS":7210,"upp":7211,"ĠJoseph":7212,"eping":7213,"ĠTake":7214,"ĠMid":7215,"Ġsending":7216,"fast":7217,"ĠFall":7218,"Ġdealing":7219,"user":7220,"ĠOrgan":7221,"Co":7222,"Ġattached":7223,"Ġsees":7224,"%.":7225,"Ġtypical":7226,"ART":7227,"Ġfinds":7228,"ĠAsia":7229,"umin":7230,"ĠCore":7231,"ĠEnt":7232,"inent":7233,"uce":7234,"ĠBlood":7235,"ĠNever":7236,"Ġemails":7237,"Ġhighlight":7238,"Ġconfront":7239,"atus":7240,"uted":7241,"Ġunus":7242,"Ġtopic":7243,"ĠAdam":7244,"Ġble":7245,"ati":7246,"Ġunderstood":7247,"Set":7248,"struct":7249,"TP":7250,"Ġmob":7251,"aa":7252,"ĠStart":7253,"pected":7254,"sell":7255,"Ġdedicated":7256,"ĠCA":7257,"uan":7258,"Ġsongs":7259,"escription":7260,"Ġtech":7261,"Ġrape":7262,"Ġaside":7263,"Ġgrant":7264,"Ġ56":7265,"sub":7266,"Ġargue":7267,"Ġcontaining":7268,"Ġschedule":7269,"Ġliberal":7270,"Ġpublicly":7271,"Ġheavily":7272,"ĠUt":7273,"iner":7274,"ĠSection":7275,"ĠCare":7276,"weet":7277,"ls":7278,"Dis":7279,"âĶĢ":7280,"ĠFollow":7281,"Back":7282,"ĠIT":7283,"Ġbes":7284,"ji":7285,"ĠHit":7286,"ested":7287,"Ġeverybody":7288,"ĠSwed":7289,"Ġfemin":7290,"Ġfacilities":7291,"Ġconven":7292,"Comp":7293,"ĠOS":7294,"core":7295,"Ġanx":7296,"Ġdivision":7297,"ĠCam":7298,"ĠStan":7299,"mates":7300,"Ġexplore":7301,"plom":7302,"Ġshares":7303,"pload":7304,"anes":7305,"Ġideal":7306,"eters":7307,"ĠBase":7308,"Ġplastic":7309,"Ġdistinct":7310,"ĠNetwork":7311,"ĠSeattle":7312,"Ġtrading":7313,"ensus":7314,"intend":7315,"Ġexhib":7316,"Ġinitially":7317,"ĠFood":7318,"Ġthousand":7319,"ĠBusiness":7320,"acter":7321,"Ġparagraph":7322,"Ġroughly":7323,"Ġwww":7324,"Ġcreative":7325,"ĠConf":7326,"Ġconsumption":7327,"Ġfilms":7328,"agan":7329,"Ġobtain":7330,"Ġtall":7331,"Ġtor":7332,"Ġacknowled":7333,"Ġgrown":7334,"alo":7335,"KE":7336,"Ġ400":7337,"enders":7338,"taining":7339,"UG":7340,"Ġsuicide":7341,"Ġwatched":7342,"ĠList":7343,"ali":7344,"rehens":7345,"Ġsurrounding":7346,"Ġpip":7347,"Ġflying":7348,"ĠJava":7349,"ordan":7350,"Ġserving":7351,"inations":7352,"post":7353,"Ġsho":7354,"Av":7355,"Ġjail":7356,"zy":7357,"Ġ1999":7358,"Ġ>":9609,"orous":9610,"Ġfirms":9611,"screen":9612,"una":9613,"Ġembarrass":9614,"ulse":9615,"Ġletting":9616,"Ġthrew":9617,"iley":9618,"Ġchannels":9619,"lan":9620,"ĠVegas":9621,"Ġsear":9622,"Ġfantastic":9623,"arre":9624,"uzzle":9625,"ĠDer":9626,"Those":9627,"Ġswing":9628,"Ġsheet":9629,"index":9630,"cover":9631,"ogan":9632,"Ġvariables":9633,"ĠTech":9634,"Ġspoken":9635,"achel":9636,"ĠDa":9637,"ĠMountain":9638,"Ġloaded":9639,"Ġfootage":9640,"version":9641,"Ġunl":9642,"ĠPhoenix":9643,"Ġthrowing":9644,"Ġfiring":9645,"Ġtracking":9646,"Ġwidth":9647,"Ġstruggling":9648,"rooms":9649,"otion":9650,"Ġmonthly":9651,"ĠServer":9652,"Ġeggs":9653,"open":9654,"MC":9655,"Ġ1993":9656,"Ġhired":9657,"Ġstayed":9658,"ĠAllen":9659,"Ġstro":9660,"Ġ98":9661,"step":9662,"ĠTurkish":9663,"Ġfabric":9664,"isting":9665,"ĠDom":9666,"Ġdates":9667,"Ġpron":9668,"Ġbasketball":9669,"Ġlucky":9670,"ĠArabia":9671,"Ġassumed":9672,"esty":9673,"Ġaffairs":9674,"Ġglad":9675,"ĠIndeed":9676,"ĠFA":9677,"ĠWord":9678,"Ġjoining":9679,"ifice":9680,"pread":9681,"irts":9682,"ĠSelect":9683,"Ġpopulations":9684,"aware":9685,"Ġnose":9686,"Ġcomplaints":9687,"start":9688,"Ġscoring":9689,"Thanks":9690,"Ġmining":9691,"Ġvisitors":9692,"SH":9693,"Ġdamaged":9694,"Ġcharacteristics":9695,"ĠPent":9696,"DC":9697,"Ġ83":9698,"ĠSix":9699,"rates":9700,"Ġflags":9701,"ĠBrew":9702,"dog":9703,"Mark":9704,"////":9705,"Ġexecution":9706,"Ġjoke":9707,"phones":9708,"Ġtestimony":9709,"Ġobst":9710,"QL":9711,"ĠCut":9712,"Ġstudied":9713,"ĠNintendo":9714,"icket":9715,"ĠNBC":9716,"Ġlad":9717,"ĠBra":9718,"ĠMoh":9719,"Ġkernel":9720,"Ġoverwhelming":9721,"Ġaged":9722,"Ġapplicable":9723,"ĠCond":9724,"Ġroads":9725,"ĠBlock":9726,"made":9727,"odge":9728,"Ġcommands":9729,"Ġoffices":9730,"veland":9731,"Ġtut":9732,"Ġreceiver":9733,"ĠFro":9734,"Ġshopping":9735,"ĠiP":9736,"ĠStre":9737,"ĠABC":9738,"Ġentertainment":9739,"ĠBow":9740,"orted":9741,"Mc":9742,"Ġreads":9743,"grad":9744,"ĠCollect":9745,"ĠâĪĴ":9746,"ĠCapital":9747,"ederation":9748,"Ġemployer":9749,"Ġinvolvement":9750,"Ġanxiety":9751,"alia":9752,"Ġroof":9753,"ĠAmong":9754,"ĠDemocrat":9755,"Ġstats":9756,"ĠVill":9757,"Ġconstitutional":9758,"Ġreferring":9759,"itty":9760,"Ġtackle":9761,"outube":9762,"Ġbacked":9763,"ĠHong":9764,"ĠBroad":9765,"Ġele":9766,"ĠOtt":9767,"Ġ1992":9768,"hour":9769,"achusetts":9770,"Cal":9771,"Ġdefeated":9772,"Ġ81":9773,"esp":9774,"Ġseemingly":9775,"was":9776,"ĠJenn":9777,"ĠKurd":9778,"Ġgene":9779,"Ġdiscount":9780,"Ret":9781,"ECT":9782,"();":9783,"Ġclubs":9784,"Ġsid":9785,"ĠMarsh":9786,"Check":9787,"Ġpp":9788,"ĠEag":9789,"idespread":9790,"Ġbeings":9791,"FT":9792,"Ġintroduction":9793,"ĠChange":9794,"ARD":9795,"Ġ110":9796,"adows":9797,"ierce":9798,"Ġmeal":9799,"author":9800,"ĠBang":9801,"lahoma":9802,"Ġranks":9803,"2011":9804,"????":9805,"max":9806,"Ġcollapse":9807,"Ġopens":9808,"Ġecho":9809,"Ġsoph":9810,"Ġracist":9811,"Ġenormous":9812,"Ġwaves":9813,"Ġtap":9814,"Ġcomprehensive":9815,".--":9816,"ĠRoy":9817,"Ġfarmers":9818,"Related":9819,"aired":9820,"rones":9821,"ĠCrim":9822,"Ġproportion":9823,"Ġdesigns":9824,"Ġnegotiations":9825,"Ġvirtually":9826,"ĠBatman":9827,"Ġwarn":9828,"Ġlegitimate":9829,"mate":9830,"Ġconvention":9831,",,":9832,"netic":9833,"ĠSD":9834,"Ġconsistently":9835,"Ġcompensation":9836,"Ġpunishment":9837,"Ġye":9838,"Ġtie":9839,"ĠBureau":9840,"irlf":9841,"ĠBu":9842,"ĠAren":9843,"ĠPhilipp":9844,"Ġknife":9845,"Ġmemories":9846,"ĠRoss":9847,"Ġangle":9848,"Ġ86":9849,"ĠThunder":9850,"Ġrend":9851,"ĠTour":9852,"Ġcounts":9853,"sung":9854,"ĠImp":9855,"Ġeducational":9856,"Ġaccessible":9857,"COM":9858,"Ġdrew":9859,"yer":9860,"Gl":9861,"amine":9862,"ORT":9863,"OB":9864,"IB":9865,"master":9866,"Ġtrials":9867,"ogy":9868,"har":9869,"ĠTrust":9870,"Ġpreferred":9871,"irlfriend":9872,"ĠNev":9873,"Ġbin":9874,"Ġcow":9875,"Page":9876,"Ġsignature":9877,"ĠBL":9878,"700":9879,"Ġretired":9880,"Ġbytes":9881,"Ġneighb":9882,"ĠLegend":9883,"Ġdevast":9884,"Ġsuspected":9885,"isons":9886,"ĠPokémon":9887,"scale":9888,"Ġcapabilities":9889,"Ġrevel":9890,"Ġcheese":9891,"dy":9892,"igrant":9893,"Ġfailing":9894,"bits":9895,"ĠHeroes":9896,"ĠGhost":9897,"ĠScient":9898,"Ġappointed":9899,"uri":9900,"Ġinstitution":9901,"Ġexpanded":9902,"greg":9903,"Ġmonitoring":9904,"Ġpodcast":9905,"Ġcoalition":9906,"Ġ96":9907,"Jo":9908,"Ġstolen":9909,"ĠSab":9910,"Ġstops":9911,"Ġholiday":9912,"Ġintr":9913,"Car":9914,"Black":9915,"ĠLGBT":9916,"Ġwarming":9917,"ĠAnderson":9918,"Ġ89":9919,"Ġproducer":9920,"Med":9921,"Ġaccuracy":9922,"ĠMarvel":9923,"izabeth":9924,"ĠPatrick":9925,"mony":9926,"Ġmini":9927,"acles":9928,"Ġovert":9929,"they":9930,"Ġmembership":9931,"ĠVen":9932,"Ġexch":9933,"Ġremoval":9934,"ĠDave":9935,"TY":9936,"mad":9937,"ĠFind":9938,"Ġadequ":9939,"Ġec":9940,"Ġteeth":9941,"Ġemotion":9942,"Ġperm":9943,"Ġsolely":9944,"db":9945,"Ġextraord":9946,"IGHT":9947,"cal":9948,"Ġguidelines":9949,"Ġdying":9950,"Ġsuspended":9951,"ĠPremier":9952,"ĠAnthony":9953,"elve":9954,"Ġdad":9955,"ĠEth":9956,"ĠFootball":9957,"Ġabandoned":9958,"Ġ<<":9959,"Ġmarch":9960,"Ġhorror":9961,"âĢ¦\"":9962,"Ġchildhood":9963,"Ġcampaigns":9964,"Ġlunch":9965,"ĠAlbert":9966,"block":9967,"âĸĪâĸĪ":9968,"ounding":9969,"Ġbone":9970,"organ":9971,"aders":9972,"ĠFlash":9973,"ĠDrive":9974,"Ġtonight":9975,"Ġwars":9976,"ĠFL":9977,"Ġformation":9978,"const":9979,"News":9980,"Ġcompe":9981,"orious":9982,"ĠStaff":9983,"Ġdiscussions":9984,"ĠProtection":9985,"ĠJam":9986,"Ġcriteria":9987,"Ġinstallation":9988,"Ġaccomplish":9989,"izza":9990,"Ġpublisher":9991,"Ġrescue":9992,"ĠTry":9993,"ULL":9994,"ĠSom":9995,"ĠHop":9996,"oret":9997,"ths":9998,"ordon":9999,"Ġpocket":10000,"ĠInv":10001,"Download":10002,"ĠCrime":10003,"Ġbene":10004,"ĠGuide":10005,"ĠAssembly":10006,"Ġparameters":10007,"IE":10008,"ĠAlexander":10009,"Ġconcert":10010,"ĠSche":10011,"Ġshoes":10012,"Ġvisiting":10013,"Ġrecall":10014,"Ġbub":10015,"Ġrural":10016,"Ġconcrete":10017,"ĠRos":10018,"Next":10019,"Russ":10020,"Ġloans":10021,"ĠShield":10022,"Ġtrem":10023,"hemat":10024,"kg":10025,"ĠHarris":10026,"isition":10027,"ĠMove":10028,"ĠFC":10029,"Ġfate":10030,"ĠCho":10031,"Ġtired":10032,"Ġprincipal":10033,"hist":10034,"iences":10035,"athy":10036,"Ġsevent":10037,"Ġmood":10038,"Ġstrategic":10039,"Ġdiseases":10040,"Ġforum":10041,"Ġtempor":10042,"Ġheadquarters":10043,"Par":10044,"ige":10045,"flix":10046,"Ġguitar":10047,"Ġ94":10048,"Only":10049,"Ġreleases":10050,"roph":10051,"================================":10052,"Ġ600":10053,"ĠContinue":10054,"igate":10055,"ĠCrit":10056,"system":10057,"Ġdisabled":10058,"Ġunexpected":10059,"ithub":10060,"Ġunclear":10061,"ĠEst":10062,"Ġcontrad":10063,"Ġstrategies":10064,"ventures":10065,"Ġpassage":10066,"AME":10067,"Ġimproving":10068,"Ġreveals":10069,"Ġdecrease":10070,"ova":10071,"Ġannoy":10072,"ĠShort":10073,"ĠLibrary":10074,"Ġcyber":10075,"nell":10076,"ĠHur":10077,"ĠCB":10078,"Ġphotograp":10079,"UI":10080,"Ġsed":10081,"Ge":10082,"Ġ87":10083,"Ġdiverse":10084,"Ġencouraged":10085,"Ġconspiracy":10086,"Ġbirds":10087,"Ġoperator":10088,"Ġhandful":10089,"Ġclassified":10090,"?)":10091,"Ġdramatic":10092,"Ġinvestigators":10093,"ito":10094,"Ġwidespread":10095,"ĠRoom":10096,"----------------------------------------------------------------":10097,"Ġcollective":10098,"Ġjournalist":10099,"String":10100,"Ġtemperatures":10101,"ila":10102,"Ġguid":10103,"Ġinspect":10104,"Ġmissile":10105,"ĠMayor":10106,"Ġmanual":10107,"Ġsimultane":10108,"Ġratings":10109,"Ġsuck":10110,"Ġ97":10111,"Ġuniversal":10112,"Ġpharm":10113,"Ġdisrupt":10114,"iano":10115,"AV":10116,"Ġft":10117,"Ġstatist":10118,"olds":10119,"ĠWalker":10120,"php":10121,"Ġundert":10122,"ĠLas":10123,"ishop":10124,"ntil":10125,"reshold":10126,"ĠWhether":10127,"Ms":10128,"Ġdeny":10129,"ĠCloud":10130,"Ġprovider":10131,"Ġsurviv":10132,"ĠUpdate":10133,"has":10134,"Ġmistakes":10135,"charge":10136,"pled":10137,"rity":10138,"Ġnode":10139,"ĠMassachusetts":10140,"ools":10141,"lication":10142,"Ġfails":10143,"emale":10144,"ori":10145,"backs":10146,"Ġshirt":10147,"Ġ''":10148,"ĠNAT":10149,"Ġwaters":10150,"elson":10151,"Ġease":10152,"Ġscar":10153,"Ġcontents":10154,"mind":10155,"Ġcontribution":10156,"Ġshr":10157,"Ġhanded":10158,"Ġstability":10159,"Ġtrave":10160,"Em":10161,"Ġmirror":10162,"123":10163,"Ġweigh":10164,"Ġfiction":10165,"ouver":10166,"istant":10167,"rition":10168,"ĠFed":10169,"Ġphysically":10170,"Ġstake":10171,"ĠArticle":10172,"ĠArc":10173,"ĠLewis":10174,"ĠMind":10175,"Ġdemonstrate":10176,"Ġprofits":10177,"vision":10178,"omic":10179,"olid":10180,"Ġbattles":10181,"Ġdrives":10182,"Ġeastern":10183,"ĠSony":10184,"!!!":10185,"aration":10186,"vard":10187,"ĠGL":10188,"portation":10189,"Ġ92":10190,"Ġlawmakers":10191,"Ġprotecting":10192,"ĠEPA":10193,"Ġyeah":10194,"Ġshame":10195,"olph":10196,"even":10197,"xit":10198,"Ġattach":10199,"Ġrepresenting":10200,"Ġobs":10201,"ĠUtah":10202,"iffs":10203,"ĠFreedom":10204,"ó":10205,"AK":10206,"Ġincidents":10207,"itage":10208,"Ġviewers":10209,"cd":10210,"Ġmouse":10211,"Ġclar":10212,"Ġaccordance":10213,"Ġbot":10214,"cor":10215,"ĠSummer":10216,"held":10217,"Ġinnocent":10218,"Ġinitiative":10219,"ols":10220,"________________________________":10221,"Ġspots":10222,"pace":10223,"Ġconventional":10224,"Ġcorporations":10225,"Ġblocked":10226,"HD":10227,"attered":10228,"Ġrefers":10229,"Ġbuck":10230,"ĠDigital":10231,"120":10232,"Ġtopics":10233,"TF":10234,"Äģ":10235,"brid":10236,"reement":10237,"Ġunderlying":10238,"ĠMember":10239,"Ġinvestigating":10240,"Ġpregnancy":10241,"Ġtouchdown":10242,"ĠBand":10243,"ĠCaller":10244,"Ġinstances":10245,"PP":10246,"wa":10247,"Good":10248,"Ġ1991":10249,"ĠCold":10250,"Ġfears":10251,"Ġremarks":10252,"ĨĴ":10253,"atal":10254,"Ġmit":10255,"Ġexperiments":10256,"ipt":10257,"Color":10258,"indu":10259,"Update":10260,"Ġ93":10261,"Ag":10262,"Ġå":10263,"ancouver":10264,"Both":10265,"Ġjudges":10266,"Object":10267,"Ġstere":10268,"umbn":10269,"Ġparticipation":10270,"ĠStars":10271,"ĠJere":10272,"Ġweekly":10273,"ĠBan":10274,"Ġconversations":10275,"ĠPitt":10276,"uz":10277,"ĠIndiana":10278,"ĠKick":10279,"Ġinfection":10280,"Ġheroes":10281,"Ġsettled":10282,"Ġstrip":10283,"Ġhal":10284,"Ġdump":10285,"ĠSci":10286,"Ġles":10287,"Ġreferences":10288,"ĠURL":10289,"ĠBridge":10290,"Ġwanting":10291,"Force":10292,"Ġexclus":10293,"Meanwhile":10294,"mn":10295,"Ġgentle":10296,"maker":10297,"senal":10298,"ĠGro":10299,"ouri":10300,"ĠRain":10301,"ĠAlliance":10302,"Ġlift":10303,"ela":10304,"SD":10305,"ĠCleveland":10306,"Ġranked":10307,"Ġstadium":10308,"Ġdeadly":10309,"ä¸":10310,"Ġriding":10311,"aria":10312,"ĠArmor":10313,"Ġdocumentation":10314,"ĠGreece":10315,"reek":10316,"Ġlens":10317,"ĠSa":10318,"Ġgross":10319,"ĠEmer":10320,"agers":10321,"ĠDub":10322,"ĠRh":10323,"ĠAMD":10324,"Ġarrival":10325,"Ġdesert":10326,"Ġsupplement":10327,"ĠResp":10328,"Ġknee":10329,"Ġmargin":10330,"font":10331,"ogg":10332,"2010":10333,"ĠPir":10334,"ĠProm":10335,"ivals":10336,"Ġintake":10337,"Ġdifferently":10338,"ugs":10339,"Ġbits":10340,"cluded":10341,"Ġsearching":10342,"ĠDu":10343,"umble":10344,"Ġfunctional":10345,"ĠBaltimore":10346,"ĠCould":10347,"Ġdesired":10348,"Ġcircuit":10349,"ĠLyn":10350,"ĠGO":10351,"ĠFalse":10352,"repre":10353,"':":10354,"alties":10355,"Ġminim":10356,"Ġdrove":10357,"ĠShould":10358,"Ġhip":10359,"Ġpros":10360,"Ġutility":10361,"ĠNature":10362,"ĠMode":10363,"President":10364,"opp":10365,"rat":10366,"formance":10367,"Ġconcentration":10368,"Ġfont":10369,"ĠBud":10370,"Ġamid":10371,"Ġrevers":10372,"ĠML":10373,"Bar":10374,"Ġinteraction":10375,"Ġjurisd":10376,"Ġspells":10377,"dep":10378,"fil":10379,"Ġcivilians":10380,"utter":10381,"ĠCooper":10382,"ĠBelow":10383,"Ġentrance":10384,"Ġconvert":10385,"Ġcontroversy":10386,"owered":10387,"Ġcontrary":10388,"Ġarc":10389,"ĠExecutive":10390,"ĠOfficer":10391,"Ġpackages":10392,"Ġprogressive":10393,"width":10394,"Ġreserved":10395,"vol":10396,"ĠSamsung":10397,"Ġprinted":10398,"Ġcenters":10399,"Ġintroduce":10400,"ĠKennedy":10401,"Ġodds":10402,"Ġsurely":10403,"Ġindependence":10404,"Ġpassengers":10405,"reprene":10406,"ĠBeh":10407,"Ġloves":10408,"ĠESPN":10409,"Ġfacilit":10410,"Ġidentical":10411,"Ġdoct":10412,"Ġpartnership":10413,"conf":10414,"ĠHide":10415,"Ġconfused":10416,"ĠCow":10417,"Men":10418,"Ġwrest":10419,"ĠIraqi":10420,"Ġholes":10421,"ĠStudies":10422,"Ġpregnant":10423,"hard":10424,"Ġsignals":10425,"IX":10426,"Ġpulling":10427,"Ġgraduate":10428,"Ġnominee":10429,"Date":10430,"Ġpermitted":10431,"ĠâĤ¬":10432,"ĠOklahoma":10433,"Start":10434,"Ġauthorized":10435,"Ġalarm":10436,"ĠCos":10437,"van":10438,"Ġgenerations":10439,"cular":10440,"Ġdragon":10441,"ĠSoftware":10442,"ĠEdward":10443,"Ġcontroller":10444,"Sen":10445,"gered":10446,"ĠVik":10447,"Ġapproached":10448,"Thank":10449,"Ġcance":10450,"Ġformula":10451,"ĠSmall":10452,"Ġweakness":10453,"Ġramp":10454,"itudes":10455,"jud":10456,"Ġbrilliant":10457,"Ġaccus":10458,"source":10459,"Ġ800":10460,"ĠEvil":10461,"Sw":10462,"Ġhomeless":10463,"week":10464,"iens":10465,"rics":10466,"ĠThird":10467,"TO":10468,"Ġorganic":10469,"Ġpresentation":10470,"agh":10471,"ĠDownload":10472,"vation":10473,"Ġassembly":10474,"orable":10475,"holders":10476,"ĠBernie":10477,"ĠHelp":10478,"Ġtong":10479,"ĠFight":10480,"Ġbeach":10481,"Book":10482,"ĠLic":10483,"Ġrush":10484,"ĠRound":10485,"oup":10486,"ĠMarx":10487,"Ġcalculated":10488,"ĠDevil":10489,"ĠSarah":10490,"Ġoccasionally":10491,"Ġbullet":10492,"Available":10493,"gate":10494,"Ġ91":10495,"Ġhosp":10496,"Ġpromises":10497,"ĠHIV":10498,"ĠStadium":10499,"ĠStock":10500,"ĠCorporation":10501,"gage":10502,"NG":10503,"ĠCredit":10504,"Ġsne":10505,"ibl":10506,"Ġaccum":10507,"such":10508,"Ġterrorists":10509,"Ġconsciousness":10510,"ĠZh":10511,"Ġdrama":10512,"oola":10513,"piration":10514,"Ġlabour":10515,"ĠNin":10516,"Ġutter":10517,"Ġdemocratic":10518,"Ġassass":10519,"ilation":10520,"Ġgest":10521,"Ġabroad":10522,"Ġmetab":10523,"Ġsorts":10524,"Ġflav":10525,"UB":10526,"Ġmg":10527,"ĠNothing":10528,"ĠOd":10529,"Ġmusical":10530,"2009":10531,"Ġdrops":10532,"ocated":10533,"ateral":10534,"000000":10535,"Ġgre":10536,"Ġequality":10537,"Ġburden":10538,"Ġvig":10539,"ĠLeader":10540,"------------":10541,"Ġceremony":10542,"Ġfighter":10543,"Ġactors":10544,"Ġæ":10545,"aman":10546,"Fi":10547,"Ġalign":10548,"puter":10549,"Ġelder":10550,"ĠNSA":10551,"Ġrepresentation":10552,"ĠOntario":10553,"ITH":10554,"usalem":10555,"Ġharassment":10556,"itzer":10557,"Ġsymp":10558,"Ġboxes":10559,"ĠDR":10560,"Ġmanifest":10561,"atre":10562,"Ġ^":10563,"Ġdies":10564,"leton":10565,"Ġmissions":10566,"ethe":10567,"Ġresolve":10568,"Ġfollowers":10569,"Ġasc":10570,"Ġkm":10571,"lord":10572,"ammed":10573,"Ġsilent":10574,"ĠAssociated":10575,"Ġtiming":10576,"Ġprisoners":10577,"ĠKings":10578,"ĠFive":10579,"Ġtower":10580,"Ġapproaches":10581,"Ġprecisely":10582,"Ġbureau":10583,"ĠMother":10584,"ĠIss":10585,"Ġkeyboard":10586,"itual":10587,"Ġfunded":10588,"Ġstaying":10589,"Ġpsychological":10590,"Ġmile":10591,"ĠLeon":10592,"ĠBarb":10593,"will":10594,"Ġwider":10595,"ĠAtlantic":10596,"Ġtill":10597,"ĠRome":10598,"rot":10599,"Ġaccompan":10600,"Ġflour":10601,"aco":10602,"World":10603,"ĠExpress":10604,"ĠYu":10605,"Cor":10606,"Ġpleased":10607,"party":10608,"Ġpointing":10609,"Ġinflation":10610,"Ġroy":10611,"Ġ),":10612,"ainer":10613,"Ġwedding":10614,"ormon":10615,"Ġrequiring":10616,"Ġqualified":10617,"Ġsegment":10618,"END":10619,"Ġsizes":10620,"eals":10621,"Ġcorrupt":10622,"assador":10623,"Ġceleb":10624,"Ġdreams":10625,"ĠMess":10626,"Ġchecking":10627,"ĠVersion":10628,"Ġpreparing":10629,"Ġactively":10630,"ĠDiff":10631,"Ġlux":10632,"ĠWinter":10633,"acteria":10634,"ĠNE":10635,"Ġdeputy":10636,"Ġtransgender":10637,"Ġsummary":10638,"Ġinher":10639,"eries":10640,"char":10641,"ĠYan":10642,"Ġknock":10643,"ĠPath":10644,"Ġlip":10645,"roller":10646,"Ġimpression":10647,"Ġcelebrate":10648,"Ġslide":10649,"Ġguests":10650,"Ġclip":10651,"FS":10652,"Ġsavings":10653,"Ġcaptain":10654,"Ġlegacy":10655,"ĠDenver":10656,"Ġwounded":10657,"taboola":10658,"ACT":10659,"Ġpursue":10660,"Ġoxy":10661,"Ġq":10662,"Ġsemi":10663,"ĠNeed":10664,"ĠAffairs":10665,"Ġobsc":10666,"Ġchecked":10667,"Ġdual":10668,"Code":10669,"ĠMD":10670,"lem":10671,"ulty":10672,"Ġ©":10673,"ĠElizabeth":10674,"Ġcenturies":10675,"arded":10676,"src":10677,"Ġevident":10678,"ennis":10679,"atin":10680,"Ġunemployment":10681,"ĠMario":10682,"Ġintim":10683,"Christ":10684,"Ġbiological":10685,"Ġsoldier":10686,"ĠAdded":10687,"Ġmath":10688,"ĠGil":10689,"Ġbias":10690,"Ġdating":10691,"ĠOcean":10692,"Ġmice":10693,"Mus":10694,"hire":10695,"ĠTes":10696,"Server":10697,"limited":10698,"Size":10699,"Ġmeters":10700,"Ġrocket":10701,"essee":10702,"Ġcertificate":10703,"ĠIranian":10704,"ASS":10705,"Ġgrid":10706,"Dec":10707,"Ġrolling":10708,"commun":10709,"ĠSweden":10710,"bury":10711,"Ġtissue":10712,"Ġracism":10713,"ĠLocal":10714,"Ġmystery":10715,"Ġexamine":10716,"Ġstem":10717,"Ġsits":10718,"Ġhoped":10719,"oting":10720,"Ġdialogue":10721,"Ġpersu":10722,"Watch":10723,"lay":10724,"MAN":10725,"Ġchronic":10726,"ĠPortland":10727,"market":10728,"ĠSEC":10729,"Ġparallel":10730,"Ġscandal":10731,"Ġcarries":10732,"Ġphenomenon":10733,"human":10734,"acker":10735,"ĠOx":10736,"Ġretirement":10737,"tainment":10738,"ovie":10739,"ĠGear":10740,"Ġduties":10741,"Ġdose":10742,"Ġscroll":10743,"MB":10744,"inf":10745,"Ġsauce":10746,"Ġlandscape":10747,"reddit":10748,"ĠChampionship":10749,"ĠReddit":10750,"alid":10751,"Ġcoin":10752,"Ġovers":10753,"Ġposting":10754,"about":10755,"Ġfel":10756,"andy":10757,"Ġbold":10758,"Ġfocusing":10759,"effect":10760,"GR":10761,"Ġdeemed":10762,"Ġrecommendations":10763,"Ġstepped":10764,"Ġvoter":10765,"ĠDeep":10766,"ĠInstagram":10767,"Ġmoderate":10768,"ĠMaryland":10769,"Ġrestricted":10770,"ĠMB":10771,"ĠChall":10772,"Ġtob":10773,"Ġcir":10774,"ĠOcc":10775,"ĠEver":10776,"Ġcollaps":10777,"INFO":10778,"=-":10779,"ĠPict":10780,"ĠAccount":10781,"nc":10782,"Ġought":10783,"Ġexport":10784,"Ġdrunk":10785,"('":10786,"Ġwise":10787,"ĠMort":10788,"necess":10789,"Ġancest":10790,"ĠIncre":10791,"Ġfrequent":10792,"mir":10793,"Ġinterpretation":10794,"Ġdependent":10795,"Ġcoins":10796,"ĠBol":10797,"Video":10798,"ĠJustin":10799,"Ġfatal":10800,"Ġcooking":10801,"Ġconfusion":10802,"ipher":10803,"Ġcustody":10804,"ĠMorgan":10805,"omach":10806,"ĠGovernor":10807,"Ġrestaurants":10808,"eling":10809,"Ġacknowledged":10810,"Ġther":10811,"Ġgenes":10812,"ching":10813,"Hey":10814,"Ġtactics":10815,"ĠMexican":10816,"Ġvend":10817,"Ġhes":10818,"quer":10819,"Ġnoting":10820,"ĠCameron":10821,"Ġtargeting":10822,"rock":10823,"Ġcredits":10824,"Ġemotions":10825,"Ġrepresentatives":10826,"news":10827,"Ġlegislative":10828,"Ġremoving":10829,"Ġtweeted":10830,"ĠCarter":10831,"ĠFixed":10832,"Ġforcing":10833,"Ġspeaker":10834,"Ġmales":10835,"ĠVietnam":10836,"lined":10837,"Ġconcepts":10838,"Ġvoices":10839,"oir":10840,"ĠTrib":10841,"Whe":10842,"ĠJerusalem":10843,"ĠSant":10844,"Ġcul":10845,"Ġlady":10846,"ĠHawai":10847,"Ġarts":10848,"ĠInn":10849,"ĠMachine":10850,"ĠEmperor":10851,"Ġslot":10852,"gly":10853,"ĠProcess":10854,"III":10855,"Ġathletes":10856,"ĠTemple":10857,"ĠRepresent":10858,"Ġpresc":10859,"Ġtons":10860,"Ġgolden":10861,"Ġpunch":10862,"ĠGR":10863,"iverpool":10864,"Ġenact":10865,"Ġlobby":10866,"Ġmos":10867,"Ġpicking":10868,"Ġlifetime":10869,"Ġcognitive":10870,"Each":10871,"zo":10872,"Ġdub":10873,"Ġconsists":10874,"oln":10875,"Ġfestival":10876,"amous":10877,"Ġintellig":10878,"words":10879,"ĠSmart":10880,"Ġdele":10881,"Ġlapt":10882,"Ġmagical":10883,"ĠSin":10884,"bus":10885,"urities":10886,"ighth":10887,"ĠRuby":10888,"ĠSure":10889,"olving":10890,"Ġjun":10891,"OST":10892,"Ġimposed":10893,"Ġastron":10894,"Ġcorrel":10895,"ĠNS":10896,"ĠKit":10897,"ĠFuture":10898,"burn":10899,"Ġimmune":10900,"ocus":10901,"Ġcourses":10902,"ĠString":10903,"Ġlean":10904,"Ġghost":10905,"Ġoutcomes":10906,"Ġexpense":10907,"Ġeveryday":10908,"Ġacceptable":10909,"Ah":10910,"Ġequipped":10911,"Ġorange":10912,"FR":10913,"ĠDutch":10914,"Though":10915,"ĠRank":10916,"QU":10917,"ĠRoberts":10918,"what":10919,"rend":10920,"Ġdisappear":10921,"Ġspawn":10922,"ĠLam":10923,"ois":10924,"Ġdeserve":10925,"Ġminimal":10926,"Ġnervous":10927,"ĠWould":10928,"Ġrook":10929,"ĠVancouver":10930,"Ġresign":10931,"shire":10932,"ĠWorks":10933,"ĠBuild":10934,"Ġaffordable":10935,"ĠGary":10936,"ĠArena":10937,"Ġhanging":10938,"Ġimplications":10939,"ĠSong":10940,"Ġmaintaining":10941,"Ġguards":10942,"CON":10943,"Ġderived":10944,"Ġexecuted":10945,"Ġtheories":10946,"Ġquoted":10947,"ĠAndre":10948,"oga":10949,"seless":10950,"info":10951,"ĠBelg":10952,"Ġtears":10953,"ĠSurv":10954,"Ġbirthday":10955,"igious":10956,"immer":10957,"Ġspectrum":10958,"Ġarchitecture":10959,"Ġrecruit":10960,"arma":10961,"Table":10962,"Ġmonsters":10963,"ĠGov":10964,"Ġdestination":10965,"Ġattractive":10966,"Ġfoss":10967,"ĠMoreover":10968,"Ġpresents":10969,"THE":10970,"Ġreply":10971,"pton":10972,"Ġcum":10973,"Ġdelight":10974,"Ġaffects":10975,"Ġdonations":10976,"ĠToy":10977,"ĠHim":10978,"MENT":10979,"Ġovercome":10980,"itched":10981,"ĠFantasy":10982,"ĠHat":10983,"ĠBeast":10984,"bott":10985,"Ġinvestigations":10986,"Run":10987,"Ġhunting":10988,"di":10989,"fund":10990,"Ġsessions":10991,"estyle":10992,"Ġportray":10993,"oids":10994,"Yeah":10995,"Ġcommunicate":10996,"Ġcomedy":10997,"ĠYang":10998,"Ġbelt":10999,"ĠMarine":11000,"Ġpredicted":11001,"Play":11002,"Ġimportantly":11003,"Ġremarkable":11004,"Ġeliminate":11005,"David":11006,"Ġbind":11007,"VID":11008,"Ġadvocates":11009,"ĠGaza":11010,"imp":11011,"DB":11012,"ĠNa":11013,"ĠSimilar":11014,"IES":11015,"Ġcharity":11016,"vas":11017,"math":11018,"Ġâĸ":11019,"oker":11020,"ndum":11021,"Ġcaps":11022,"ĠHal":11023,"2000":11024,"ean":11025,"Ġfleet":11026,"Ġrecre":11027,"Right":11028,"Ġsleeping":11029,"ijing":11030,"kind":11031,"Ġdesignated":11032,"ä":11033,"Ġanimation":11034,"kee":11035,"ĠIntrodu":11036,"Ġ/>":11037,"Ġdelayed":11038,"Ġtremend":11039,"Ġcurious":11040,"Use":11041,"Ġlect":11042,"dam":11043,"Ġinnovation":11044,"ĠPoints":11045,"Ġloading":11046,"Ġdispute":11047,"ctic":11048,"irds":11049,"ĠBY":11050,"Ġnurs":11051,"ĠValue":11052,"IONS":11053,"ĠHum":11054,"Ġtemplate":11055,"mers":11056,"Ġappearances":11057,"ĠEntertainment":11058,"Ġtranslation":11059,"Ġsake":11060,"Ġbeneath":11061,"Ġinhib":11062,"Ġeuro":11063,"abetes":11064,"Ġstudying":11065,"ĠMas":11066,"Ġperceived":11067,"Ġexamined":11068,"Ġeager":11069,"Ġcoaches":11070,"Ġimper":11071,"chi":11072,"Ġproduces":11073,"\").":11074,"ĠEveryone":11075,"Ġmunicip":11076,"Ġgirlfriend":11077,"Ġhire":11078,"ĠVice":11079,"Ġsuitable":11080,"opy":11081,"Ġinequ":11082,"ĠDuke":11083,"fish":11084,"first":11085,"ĠObs":11086,"Ġinterior":11087,"ĠBruce":11088,"ĠRy":11089,"Ġanalys":11090,"Ġconsiderable":11091,"Ġforecast":11092,"Ġfert":11093,"orship":11094,"ĠDrug":11095,"ĠALL":11096,":\"":11097,"thur":11098,"ĠMail":11099,"Ġballot":11100,"Ġinstantly":11101,"ĠChannel":11102,"Ġpicks":11103,"Ġ1989":11104,"Ġtent":11105,"oli":11106,"Ġcivilian":11107,"bling":11108,"ello":11109,"bu":11110,"Ġinch":11111,"Ġlogo":11112,"Ġcooperation":11113,"Ġwalks":11114,"Ġinvestments":11115,"Ġimprison":11116,"ĠFestival":11117,"ĠKy":11118,"Ġlegally":11119,"Ġgri":11120,"charg":11121,"Sl":11122,"Ġthreatening":11123,"duction":11124,"flow":11125,"Ġdismissed":11126,"ibraries":11127,"cap":11128,"ele":11129,"ĠMcG":11130,"ĠHarvard":11131,"ĠConservative":11132,"ĠCBS":11133,"png":11134,"Ġroots":11135,"ĠHaving":11136,"umbled":11137,"ĠFun":11138,"\\/":11139,"ĠSearch":11140,"plex":11141,"Ġdiscussing":11142,"Ġcontinu":11143,"ĠTai":11144,"ĠWik":11145,"Free":11146,"fit":11147,"Ġrefuse":11148,"Ġmanaging":11149,"Ġsynd":11150,"ipedia":11151,"walk":11152,"Ġprofessionals":11153,"Ġguidance":11154,"Ġuniversities":11155,"Ġassemb":11156,"untu":11157,"Finally":11158,"ASE":11159,"ĠAuto":11160,"ĠHad":11161,"Ġanniversary":11162,"LD":11163,"ĠDur":11164,"ĠUltimate":11165,"ihad":11166,"product":11167,"Ġtransit":11168,"Ġrestore":11169,"Ġexplaining":11170,"Ġasset":11171,"Ġtransferred":11172,"Ġburst":11173,"apolis":11174,"ĠMagazine":11175,"ĠCra":11176,"ĠBR":11177,"gged":11178,"ĠHE":11179,"Mich":11180,"bet":11181,"ĠLady":11182,"ylum":11183,"erves":11184,"Ġmeets":11185,"white":11186,"Log":11187,"Ġcorresponding":11188,"Ġinsisted":11189,"GG":11190,"Ġsurrounded":11191,"Ġtens":11192,"Ġlane":11193,"Ġcoinc":11194,"home":11195,"Ġexisted":11196,"ected":11197,"ĠDouble":11198,"lamm":11199,"Ġskept":11200,"exp":11201,"Ġperception":11202,"iev":11203,"ĠBeing":11204,"oft":11205,"Ġadopt":11206,".:":11207,"];":11208,"Windows":11209,"Ġsatellite":11210,"ASH":11211,"Ġinfant":11212,"description":11213,"ĠMeanwhile":11214,"cm":11215,"oca":11216,"ĠTreat":11217,"actor":11218,"Ġtobacco":11219,"ĠNorm":11220,"emption":11221,"Ġflesh":11222,"Ġje":11223,"oop":11224,"ĠHeaven":11225,"Ġbeating":11226,"anim":11227,"Ġgathering":11228,"Ġcultiv":11229,"GO":11230,"abe":11231,"ĠJonathan":11232,"ĠSafety":11233,"Ġbadly":11234,"prot":11235,"Ġchoosing":11236,"Ġcontacted":11237,"Ġquit":11238,"Ġdistur":11239,"Ġstir":11240,"Ġtoken":11241,"Det":11242,"ĠPa":11243,"Ġfunctionality":11244,"003":11245,"some":11246,"Ġlimitations":11247,"Ġmeth":11248,"build":11249,"config":11250,"NT":11251,"rell":11252,"blem":11253,"ĠMom":11254,"Ġveterans":11255,"ĠHu":11256,"Ġtrends":11257,"arer":11258,"ĠGiven":11259,"ĠCaption":11260,"may":11261,"AST":11262,"Ġwondering":11263,"ĠClark":11264,"normal":11265,"Ġseparated":11266,"Ġdesp":11267,"stic":11268,"brew":11269,"Ġrelating":11270,"ĠNik":11271,"ĠFarm":11272,"Ġenthusi":11273,"good":11274,"deb":11275,"Ġactivist":11276,"Ġmart":11277,"Ġexplosion":11278,"ĠEconomic":11279,"Link":11280,"Ġinsight":11281,"Ġconvenient":11282,"Ġcounterpart":11283,"support":11284,"ĠVirt":11285,"agen":11286,"ĠTennessee":11287,"ĠSimon":11288,"ĠAward":11289,"OCK":11290,"ĠFigure":11291,"Ġoverseas":11292,"Ġpride":11293,"ĠCas":11294,"note":11295,"mg":11296,"Current":11297,"Ġdisplays":11298,"content":11299,"Ġtraveling":11300,"Ġhospitals":11301,"ĠFinancial":11302,"ĠPast":11303,"Ġdefendant":11304,"Ġstreaming":11305,"mble":11306,"ĠBerlin":11307,"uki":11308,"Ġdistribut":11309,"Ġantib":11310,"Ġchocolate":11311,"ĠCastle":11312,"Ġinterrupt":11313,"ĠRow":11314,"Ġconversion":11315,"Ġbugs":11316,"ĠRather":11317,"liest":11318,"LY":11319,"ĠJean":11320,"common":11321,"akh":11322,"Ġ130":11323,"otton":11324,"ĠDean":11325,"Ġamendment":11326,"Ġgameplay":11327,"ĠWarren":11328,"oda":11329,"Ġhighlights":11330,"Ġirre":11331,"ĠNATO":11332,"Ġballs":11333,"Ġdemanding":11334,"URE":11335,"ĠLuke":11336,"Figure":11337,"stop":11338,"onia":11339,"zone":11340,"izers":11341,"ĠWR":11342,"Ġawarded":11343,"Ġregulatory":11344,"ĠHart":11345,"ĠSN":11346,"pling":11347,"Ġsour":11348,"ĠPixel":11349,"usive":11350,"Ġfet":11351,"ĠSent":11352,"Ġautomatic":11353,"Ġfer":11354,"vernment":11355,"ĠKhan":11356,"TON":11357,"father":11358,"Ġextraordinary":11359,"throp":11360,"ĠPython":11361,"ĠGPU":11362,"Ġsexually":11363,"Ġdesktop":11364,"itivity":11365,"ĠAntonio":11366,"Ġorient":11367,"Ġears":11368,"obby":11369,"ouses":11370,"vertisements":11371,"Ġmanufacturers":11372,"icient":11373,"minute":11374,"Ġconviction":11375,"Ġgarden":11376,"public":11377,"Ġsatisfied":11378,"fold":11379,"OK":11380,"Ġinhab":11381,"ĠThink":11382,"Ġprogramme":11383,"Ġstomach":11384,"Ġcoordin":11385,"Ġholy":11386,"Ġthreshold":11387,"Ġrhet":11388,"Ġserial":11389,"Ġemployers":11390,"ĠEverything":11391,"rah":11392,"Ġbother":11393,"Ġbrands":11394,"Value":11395,"ĠTed":11396,"ĠPlanet":11397,"Ġpink":11398,"ĠFurthermore":11399,"sa":11400,"PE":11401,"reck":11402,"ĠUSD":11403,"otte":11404,"Ġ&&":11405,"Ġlanded":11406,"gets":11407,"Ġproducers":11408,"Ġhealthcare":11409,"Ġdominant":11410,"Ġdestro":11411,"Ġamended":11412,"chron":11413,"Ġfits":11414,"ĠSyd":11415,"ĠAuthority":11416,"ATCH":11417,"Ġfights":11418,"ĠLLC":11419,"Ġ---":11420,"ĠCorp":11421,"Ġtoxic":11422,"specific":11423,"ĠCorn":11424,"ĠChel":11425,"Ġtelephone":11426,"ĠPant":11427,"Ġmysterious":11428,"aunch":11429,"odox":11430,"media":11431,"Ġwitnesses":11432,"agu":11433,"Ġquestioned":11434,"ĠBrexit":11435,"ĠRemember":11436,"enez":11437,"Ġendorse":11438,"iatric":11439,"ĠIdent":11440,"Ġridiculous":11441,"110":11442,"Ġprayer":11443,"Ġscientist":11444,"Ġ1950":11445,"ĠAqu":11446,"Ġunderground":11447,"ĠUFC":11448,"mare":11449,"ĠLater":11450,"wich":11451,"Ġsubscrib":11452,"Ġhosts":11453,"Ġerr":11454,"Ġgrants":11455,"antom":11456,"Ġsummon":11457,"early":11458,"ĠClear":11459,"ĠPrim":11460,"Ġsuspension":11461,"Ġguaranteed":11462,"apper":11463,"Ġrice":11464,"ĠSean":11465,"ĠShin":11466,"Ġreferendum":11467,"Ġfled":11468,"rust":11469,"Ġ360":11470,"tery":11471,"Ġshocked":11472,"BR":11473,"ĠOil":11474,"ĠAllah":11475,"Ġpartly":11476,"Ġignor":11477,"Ġtransmission":11478,"Ġhomosexual":11479,"iversal":11480,"Ġhopefully":11481,"ãĤ¤":11482,"Ġlesson":11483,"Leg":11484,"Ġ..":11485,"Yet":11486,"table":11487,"appropri":11488,"rett":11489,"Ġboards":11490,"Ġincorrect":11491,"Ġbacteria":11492,"aru":11493,"amac":11494,"Ġsnap":11495,".'\"":11496,"Ġparad":11497,"tem":11498,"heart":11499,"Ġavailability":11500,"Ġwisdom":11501,"Ġ(+":11502,"Ġpriest":11503,"ĠÂłĠÂł":11504,"Open":11505,"Ġspan":11506,"Ġparameter":11507,"Ġconvince":11508,"Ġ(%)":11509,"rac":11510,"Ġfo":11511,"Ġsafely":11512,"Ġconverted":11513,"ĠOlympic":11514,"Ġreserve":11515,"Ġhealing":11516,"ĠMine":11517,"Max":11518,"Ġinherent":11519,"ĠGraham":11520,"Ġintegrated":11521,"Dem":11522,"Ġpipeline":11523,"Ġapplying":11524,"Ġembed":11525,"ĠCharlie":11526,"Ġcave":11527,"2008":11528,"Ġconsensus":11529,"Ġrewards":11530,"Pal":11531,"ĠHTML":11532,"Ġpopularity":11533,"looking":11534,"ĠSword":11535,"ĠArts":11536,"')":11537,"Ġelectron":11538,"clusions":11539,"Ġintegrity":11540,"Ġexclusively":11541,"Ġgrace":11542,"Ġtorture":11543,"Ġburned":11544,"two":11545,"Ġ180":11546,"Produ":11547,"Ġentreprene":11548,"raphics":11549,"Ġgym":11550,"ricane":11551,"ĠTam":11552,"Ġadministrative":11553,"Ġmanufacturer":11554,"Ġvel":11555,"ĠNi":11556,"Ġisolated":11557,"ĠMedicine":11558,"Ġbackup":11559,"Ġpromoting":11560,"Ġcommander":11561,"Ġflee":11562,"ĠRussell":11563,"Ġforgotten":11564,"ĠMissouri":11565,"Ġresidence":11566,"mons":11567,"Ġresemb":11568,"Ġwand":11569,"Ġmeaningful":11570,"PT":11571,"Ġbol":11572,"Ġhelic":11573,"Ġwealthy":11574,"Ġrifle":11575,"strong":11576,"rowing":11577,"plan":11578,"asury":11579,"âĢ¦.":11580,"Ġexpanding":11581,"ĠHamilton":11582,"Ġreceives":11583,"SI":11584,"eatures":11585,"ĠAnim":11586,"REE":11587,"Put":11588,"Ġbriefly":11589,"rive":11590,"Ġstimul":11591,"Ġ``(":11592,"Ġ__":11593,"Ġchip":11594,"Ġhaz":11595,"Ġprize":11596,"ĠThings":11597,"ACE":11598,"ulin":11599,"dict":11600,"oku":11601,"Ġassociate":11602,"ockets":11603,"youtube":11604,"Story":11605,"ategory":11606,"Ġmild":11607,"ailing":11608,"ĠYe":11609,"Orig":11610,"ĠKa":11611,"orig":11612,"Ġpropaganda":11613,"Ġanonymous":11614,"Ġstruggled":11615,"Ġoutrage":11616,"ATED":11617,"ĠBeijing":11618,"rary":11619,"Ġleather":11620,"Ġworlds":11621,"Ġbroader":11622,"125":11623,"idal":11624,"ĠBetter":11625,"Ġtear":11626,"Ext":11627,"Ġproposals":11628,"Ġiter":11629,"ĠSquad":11630,"Ġvolunt":11631,"mi":11632,"Did":11633,"ĠPu":11634,"pin":11635,"Ġspeakers":11636,"Ġborders":11637,"Ġfigured":11638,"='":11639,"Ġsimultaneously":11640,"aeda":11641,"Ġcharging":11642,"Ġurged":11643,"Ġconj":11644,"256":11645,"ĠGordon":11646,"merce":11647,"Ġdocumentary":11648,"Share":11649,"itol":11650,"ONE":11651,"ĠGarden":11652,"hatt":11653,"ĠThompson":11654,"aneous":11655,"apore":11656,"Ġtanks":11657,"Ġlessons":11658,"track":11659,"Ġoutstanding":11660,"Ġvolunteers":11661,"Ġspray":11662,"Ġmanagers":11663,"large":11664,"Ġcamps":11665,"Ġartificial":11666,"ĠRu":11667,"Ġbags":11668,"thal":11669,"Ġcompatible":11670,"ĠBlade":11671,"Ġfed":11672,"Ġargues":11673,"FI":11674,"Ġunfair":11675,"Ġcorn":11676,"Ġoffset":11677,"Ġdirections":11678,"Ġdisappointed":11679,"ĠConvention":11680,"Ġviewing":11681,"ME":11682,"ocity":11683,"Ġtowns":11684,"Ġlayers":11685,"Ġrolled":11686,"Ġjumped":11687,"Ġattribute":11688,"Ġunnecess":11689,"incoln":11690,"Ġsuppose":11691,"ĠNether":11692,"cha":11693,"Ġburied":11694,"Ġsixth":11695,"Ben":11696,"ressing":11697,"OUR":11698,"Ġwound":11699,"Ġcycl":11700,"Ġmechanisms":11701,"Ġcongressional":11702,"ĠElement":11703,"Ġagreements":11704,"Ġdecor":11705,"Ġclosest":11706,"ĠMit":11707,"Google":11708,"}}":11709,"Ġmixture":11710,"Ġfluid":11711,"Sign":11712,"ĠScholar":11713,"Ġpist":11714,"asket":11715,"abling":11716,"Ġracing":11717,"hero":11718,"riel":11719,"assy":11720,"Ġcheaper":11721,"ben":11722,"Ġvertical":11723,"amacare":11724,"ĠReading":11725,"gments":11726,"Ġhelicop":11727,"Ġsacrifice":11728,"aya":11729,"paren":11730,"VA":11731,"ĠLes":11732,"ĠStudio":11733,"Ġviolations":11734,"ĠAnna":11735,"acer":11736,"é¾":11737,"ĠRat":11738,"ĠBeck":11739,"ĠDick":11740,"ĠACT":11741,"Ġcomposition":11742,"Ġtexture":11743,"ĠOwn":11744,"Ġsmartphone":11745,"ĠNA":11746,"Ġforb":11747,"import":11748,"Ġdefending":11749,"ilst":11750,"rer":11751,"Ġoh":11752,"ĠJeremy":11753,"Ġbanking":11754,"ceptions":11755,"Ġrespective":11756,"/.":11757,"Ġdrinks":11758,"ĠWi":11759,"Ġbands":11760,"ĠLiverpool":11761,"Ġgrip":11762,"ĠBuy":11763,"Ġopenly":11764,"Ġreviewed":11765,"pert":11766,"Ġverify":11767,"ĠCole":11768,"ĠWales":11769,"MO":11770,"Ġunpre":11771,"Ġshelter":11772,"ĠImperial":11773,"Ġgui":11774,"ĠDak":11775,"Ġsuggestions":11776,"Ġexplicitly":11777,"Ġslave":11778,"Ġblockchain":11779,"Ġcompeting":11780,"Ġpromising":11781,"SON":11782,"Ġsoccer":11783,"Ġconstitution":11784,"429":11785,"Ġdistract":11786,"ĠUser":11787,"esides":11788,"ĠMethod":11789,"ĠTokyo":11790,"Ġaccompanied":11791,"Client":11792,"sur":11793,"alog":11794,"Ġidentification":11795,"Ġinvasion":11796,"asma":11797,"Ġindustries":11798,"ppers":11799,"Ġsubtle":11800,"ĠUnit":11801,"natural":11802,"Ġsurvived":11803,"Ġflaw":11804,"ĺħ":11805,"ĠHoll":11806,"Ġdeficit":11807,"Ġtutorial":11808,"ĠChance":11809,"Ġarguing":11810,"Ġcontemporary":11811,"Ġintegration":11812,"forward":11813,"Ġtum":11814,"itis":11815,"Ġhiding":11816,"ĠDomin":11817,"ĠTan":11818,"ĠBuilding":11819,"ĠVin":11820,"Ġspokesperson":11821,"ĠNotes":11822,"Ġemerging":11823,"Ġpreparation":11824,"Ġprost":11825,"Ġsuspects":11826,"Ġautonom":11827,"Description":11828,"Ġdealt":11829,"ĠPear":11830,"Ġsteady":11831,"Ġdecreased":11832,"Ġsovere":11833,"ĠClin":11834,"Ġgradually":11835,"orses":11836,"ĠWAR":11837,"Serv":11838,"ãĤ¢":11839,"hr":11840,"Ġdirty":11841,"ĠBarn":11842,"ĠBC":11843,"Ġdil":11844,"Ġcalendar":11845,"Ġcompliance":11846,"Ġchamber":11847,"bb":11848,"Ġpassenger":11849,"ateful":11850,"ĠTitle":11851,"ĠSydney":11852,"ĠGot":11853,"Ġdarkness":11854,"Ġdefect":11855,"Ġpacked":11856,"assion":11857,"Ġgods":11858,"Ġharsh":11859,"ICK":11860,"leans":11861,"Ġalgorithm":11862,"Ġoxygen":11863,"Ġvisits":11864,"Ġblade":11865,"Ġkilomet":11866,"ĠKentucky":11867,"Ġkiller":11868,"Pack":11869,"enny":11870,"Ġdivine":11871,"Ġnomination":11872,"being":11873,"Ġengines":11874,"Ġcats":11875,"Ġbuffer":11876,"ĠPhill":11877,"Ġtraff":11878,"AGE":11879,"Ġtongue":11880,"Ġradiation":11881,"erer":11882,"mem":11883,"ĠExplicit":11884,"é¾į":11885,"Ġcouples":11886,"Ġphysics":11887,"ĠMcK":11888,"Ġpolitically":11889,"awks":11890,"ĠBloom":11891,"Ġworship":11892,"eger":11893,"uter":11894,"ĠFO":11895,"Ġmathemat":11896,"Ġsentenced":11897,"Ġdisk":11898,"ĠMarg":11899,"Ġ/*":11900,"PI":11901,"Ġoptional":11902,"Ġbabies":11903,"Ġseeds":11904,"ĠScottish":11905,"Ġthy":11906,"]]":11907,"ĠHitler":11908,"PH":11909,"ngth":11910,"Ġrecovered":11911,"inge":11912,"Ġpowder":11913,"Ġlips":11914,"Ġdesigner":11915,"Ġdisorders":11916,"Ġcourage":11917,"Ġchaos":11918,"\"},{\"":11919,"Ġcarrier":11920,"bably":11921,"High":11922,"ĠRT":11923,"esity":11924,"len":11925,"Ġroutes":11926,"uating":11927,"Fil":11928,"NOT":11929,"wall":11930,"sburgh":11931,"Ġengaging":11932,"ĠJavaScript":11933,"orer":11934,"lihood":11935,"Ġunions":11936,"ĠFederation":11937,"ĠTesla":11938,"Ġcompletion":11939,"ĠTa":11940,"Ġprivilege":11941,"ĠOrange":11942,"Ġneur":11943,"parency":11944,"Ġbones":11945,"Ġtitled":11946,"Ġprosecutors":11947,"ĠME":11948,"Ġengineer":11949,"ĠUniverse":11950,"ĠHig":11951,"nie":11952,"oard":11953,"Ġhearts":11954,"ĠGre":11955,"ussion":11956,"Ġministry":11957,"Ġpenet":11958,"ĠNut":11959,"ĠOw":11960,"ĠXP":11961,"instein":11962,"Ġbulk":11963,"System":11964,"icism":11965,"ĠMarketable":11966,"Ġpreval":11967,"Ġposter":11968,"Ġattending":11969,"urable":11970,"Ġlicensed":11971,"ĠGh":11972,"etry":11973,"ĠTradable":11974,"Ġblast":11975,"à¤":11976,"ĠTitan":11977,"elled":11978,"die":11979,"Have":11980,"ĠFlame":11981,"Ġprofound":11982,"Ġparticipating":11983,"Ġanime":11984,"ĠEss":11985,"Ġspecify":11986,"Ġregarded":11987,"ĠSpell":11988,"Ġsons":11989,"owned":11990,"Ġmerc":11991,"Ġexperimental":11992,"lando":11993,"hs":11994,"ĠDungeon":11995,"inos":11996,"Ġcomply":11997,"ĠSystems":11998,"arth":11999,"Ġseized":12000,"local":12001,"ĠGirls":12002,"udo":12003,"oned":12004,"ĠFle":12005,"Ġconstructed":12006,"Ġhosted":12007,"Ġscared":12008,"actic":12009,"ĠIslands":12010,"ĠMORE":12011,"Ġbless":12012,"Ġblocking":12013,"Ġchips":12014,"Ġevac":12015,"Ps":12016,"Ġcorporation":12017,"Ġox":12018,"Ġlighting":12019,"Ġneighbors":12020,"ĠUb":12021,"aro":12022,"Ġbeef":12023,"ĠUber":12024,"Facebook":12025,"armed":12026,"itate":12027,"ĠRating":12028,"ĠQuick":12029,"Ġoccupied":12030,"Ġaims":12031,"ĠAdditionally":12032,"ĠInterest":12033,"Ġdramatically":12034,"Ġheal":12035,"Ġpainting":12036,"Ġengineers":12037,"MM":12038,"ĠMust":12039,"Ġquantity":12040,"Paul":12041,"Ġearnings":12042,"ĠPosts":12043,"stra":12044,"ãĥ¼ãĥ":12045,"Ġstance":12046,"Ġdropping":12047,"script":12048,"Ġdressed":12049,"Make":12050,"Ġjustify":12051,"ĠLtd":12052,"Ġprompted":12053,"Ġscrut":12054,"Ġspeeds":12055,"ĠGiants":12056,"omer":12057,"ĠEditor":12058,"Ġdescribing":12059,"ĠLie":12060,"mented":12061,"Ġnowhere":12062,"ocaly":12063,"Ġinstruction":12064,"fortable":12065,"Ġentities":12066,"Ġcm":12067,"ĠNatural":12068,"Ġinquiry":12069,"Ġpressed":12070,"izont":12071,"forced":12072,"Ġraises":12073,"ĠNetflix":12074,"ĠSide":12075,"Ġouter":12076,"Ġamongst":12077,"ims":12078,"owski":12079,"Ġclimb":12080,"never":12081,"Ġcombine":12082,"ding":12083,"Ġcompr":12084,"Ġsignificance":12085,"Ġremembered":12086,"ĠNevada":12087,"ĠTel":12088,"ĠScar":12089,"ĠWarriors":12090,"ĠJane":12091,"Ġcoup":12092,"bas":12093,"Ġterminal":12094,",-":12095,"OH":12096,"Ġtension":12097,"Ġwings":12098,"ĠMyster":12099,"����":12100,"ĠUnlike":12101,"valid":12102,"vironments":12103,"ĠAli":12104,"Ġnaked":12105,"books":12106,"ĠMun":12107,"ĠGulf":12108,"Ġdensity":12109,"Ġdimin":12110,"Ġdesperate":12111,"Ġpresidency":12112,"Ġ1986":12113,"hy":12114,"IND":12115,"Ġunlock":12116,"imens":12117,"Ġhandled":12118,"ĠEb":12119,"Ġdisappeared":12120,"Ġgenre":12121,"Ġ1988":12122,"Ġdetermination":12123,"Stream":12124,"iko":12125,"apters":12126,"Ġacknowledge":12127,"Jan":12128,"Ġcapitalism":12129,"Pat":12130,"Ġ2020":12131,"Ġpainful":12132,"Ġcurve":12133,"Ġbombs":12134,"storm":12135,"ĠMetal":12136,"encer":12137,"ĠFig":12138,"ĠAaron":12139,"anches":12140,"Ġinspiration":12141,"Ġexhaust":12142,"tains":12143,"ashi":12144,"Ġdescript":12145,"Ġritual":12146,"ĠChelsea":12147,"Ġpromotion":12148,"ĠHung":12149,"ĠWard":12150,"iva":12151,"ĠET":12152,"Ġtoss":12153,"allow":12154,"ĠFrancis":12155,"Dep":12156,"Ġhappiness":12157,"ĠGlass":12158,"Ġbeta":12159,"Ġstrengthen":12160,"NE":12161,"oa":12162,"Ġbuttons":12163,"ĠMurray":12164,"Ġkicked":12165,"Quest":12166,"ĠTalk":12167,"ĠSeveral":12168,"ĠZero":12169,"Ġdrone":12170,"ulk":12171,"Ġcam":12172,"ĠMobile":12173,"Ġpreventing":12174,"Ġretro":12175,"ĠAx":12176,"Ġcruel":12177,"Ġfloat":12178,".),":12179,"Ġfiling":12180,"ĠGrant":12181,"ĠBor":12182,"Ġrib":12183,"Ġchampionship":12184,"ĠMerc":12185,"Ġstyles":12186,"Ġcake":12187,"Ġbuilds":12188,"ĠSelf":12189,"iox":12190,"Ġepic":12191,"oyd":12192,"Bel":12193,"ĠStew":12194,".(":12195,"ahu":12196,"ĠBeyond":12197,"Ġouts":12198,"Ġsolo":12199,"ĠTree":12200,"Ġpreserve":12201,"Ġtub":12202,"ARE":12203,"roc":12204,"ĠImpro":12205,"ĠWright":12206,"Ġbund":12207,"Ġtraged":12208,"Ġoccasional":12209,"bian":12210,"Second":12211,"rons":12212,"Ġinteractions":12213,"formed":12214,"sing":12215,"Ġowns":12216,"Ġhockey":12217,"General":12218,"Ġlogical":12219,"Ġexpend":12220,"Ġescal":12221,"ĠGriff":12222,"ĠCrown":12223,"ĠReserve":12224,"Ġstopping":12225,"Ġexcuse":12226,"second":12227,"Ġoperated":12228,"Ġreaches":12229,"ĠMalays":12230,"Ġpollution":12231,"ĠBrooklyn":12232,"Ġdelete":12233,"Ġhash":12234,"Block":12235,"aha":12236,"âĢ³":12237,"Ġshorter":12238,"piece":12239,">>>":13163,"ĠMormon":13164,"tor":13165,"Ġparticles":13166,"ĠBart":13167,"ryption":13168,"Ġadmin":13169,"Ġsquee":13170,"VIDIA":13171,"Ġcreator":13172,"iameter":13173,"icular":13174,"NBC":13175,"Ġgrabbed":13176,"Ġnodd":13177,"Ġrated":13178,"Ġrotation":13179,"Ġgrasp":13180,"Ġexcessive":13181,"ĠEC":13182,"ĠWhit":13183,"Ġinventory":13184,"aults":13185,"ĠFB":13186,"Ġecosystem":13187,"Ġbillions":13188,"Ġventure":13189,"named":13190,"Ġdefender":13191,"oute":13192,"Instead":13193,"irable":13194,"War":13195,"Ġassumption":13196,"Ġbite":13197,"Ġearthqu":13198,"tail":13199,"space":13200,"Ġgifts":13201,"boys":13202,"Ġinevitable":13203,"Ġstructural":13204,"Ġbeneficial":13205,"Ġcompelling":13206,"hole":13207,"ervation":13208,"Ġcoat":13209,"oj":13210,"incarn":13211,"ĠYears":13212,"Ġdetermining":13213,"Ġrhetoric":13214,"Ġboundaries":13215,"Ġwhites":13216,"Ant":13217,"addy":13218,")-":13219,"raham":13220,"etermin":13221,"Ġharvest":13222,"ĠConc":13223,"Ġlaptop":13224,"ĠMatch":13225,"Ġenjoying":13226,"cca":13227,"ollar":13228,"Ġtrips":13229,"Ġaddiction":13230,"ĠSak":13231,"Ġpowered":13232,"Ġcous":13233,"ĠRussians":13234,"iere":13235,"Ġretrie":13236,"quality":13237,"Ġdiffer":13238,"Ġkingdom":13239,"ĠLaur":13240,"ĠCapitol":13241,"Ġconclusions":13242,"ĠAltern":13243,"ĠNav":13244,"Ġtransparent":13245,"BER":13246,"Group":13247,"ĠComplete":13248,"Ġinfer":13249,"Ġintrig":13250,"Ġinsane":13251,"RO":13252,"ophob":13253,"isen":13254,"qual":13255,"Michael":13256,"Ġmuseum":13257,"ĠPope":13258,"Ġreset":13259,"rative":13260,"five":13261,"Ġaggreg":13262,"ittees":13263,"ository":13264,"Ġcarb":13265,"ĠRecord":13266,"Ġdecides":13267,"ĠFix":13268,"Ġexceptions":13269,"ĠCommissioner":13270,"uns":13271,"ĠEnvironmental":13272,"Ġlegendary":13273,"istence":13274,"Ġtunnel":13275,"km":13276,"Ġinsult":13277,"Ġtroll":13278,"Ġshake":13279,"Ġdetention":13280,"ques":13281,"ĠChrome":13282,"ĠFiles":13283,"Ġsubt":13284,"Ġprospects":13285,"Ġprol":13286,"render":13287,"proof":13288,"Ġperformances":13289,"Str":13290,"Ġhref":13291,"ername":13292,"Ġachievement":13293,"Ġfut":13294,"Full":13295,"ĠLeban":13296,"google":13297,"ãĥĪ":13298,"ampa":13299,"Maybe":13300,"Ġprojected":13301,"ĠEmb":13302,"Ġcolleg":13303,"Ġawards":13304,"ĠâĶ":13305,"Gold":13306,"ĠBlake":13307,"ĠRaj":13308,"ifting":13309,"Ġpending":13310,"Ġinstinct":13311,"Ġdevelopments":13312,"Connect":13313,"ĠMand":13314,"ĠWITH":13315,"ĠPhilippines":13316,"profile":13317,"Ġaltogether":13318,"ĠBund":13319,"ĠTD":13320,"oooo":13321,"amped":13322,"iph":13323,"Ġsteam":13324,"Ġoldest":13325,"Ġdetection":13326,"ulpt":13327,"Ġç":13328,"ĠWayne":13329,"2006":13330,"fa":13331,"Ġcircles":13332,"ĠFu":13333,"Ġdonors":13334,"appropriate":13335,"ĠDakota":13336,"jamin":13337,"Ġmotivated":13338,"Ġpurchases":13339,"ĠLouisiana":13340,"ĠSpl":13341,"Ġglobe":13342,"Ġ105":13343,"zip":13344,"call":13345,"Ġdepartments":13346,"Ġsustainable":13347,"105":13348,"ĠOP":13349,"ifiers":13350,"Ġprevented":13351,"Ġincomp":13352,"ĠCommander":13353,"Ġdominated":13354,"Ġ»":13355,"Ġinvested":13356,"Ġcomplexity":13357,"Ġincl":13358,"Ġensuring":13359,"Ġrealm":13360,"ync":13361,"ĠIndependent":13362,"rained":13363,"ĠJen":13364,"ĠFlight":13365,"Ġathe":13366,"Ġspeculation":13367,"ĠTE":13368,"ocate":13369,"tic":13370,"Ġplaint":13371,"herry":13372,"Ġtoy":13373,"Ġ111":13374,"Ġplates":13375,"status":13376,"ĠIsa":13377,"Ġdevoted":13378,"Cop":13379,"ĠES":13380,"255":13381,"urrency":13382,"Main":13383,"Ġslaves":13384,"Ġpepper":13385,"Ġquotes":13386,"Ġceiling":13387,"ĠFish":13388,"Ġtransformation":13389,"Ġfraction":13390,"Ġadvantages":13391,"Ġtoile":13392,"Ġstunning":13393,"Ġmoist":13394,"breaking":13395,"si":13396,"ĠLocation":13397,"ĠMedium":13398,"Ġtexts":13399,"Ġugly":13400,"Ġbio":13401,".âĢĶ":13402,"ĠBased":13403,"Ġtrains":13404,"ĠWing":13405,"ĠAncient":13406,"ĠRecords":13407,"ĠHope":13408,"Special":13409,"adesh":13410,"obi":13411,"[/":13412,"Ġtemporarily":13413,"Ver":13414,"hu":13415,"oser":13416,"Ġovernight":13417,"Ġmamm":13418,"ĠTreasury":13419,"ĠVenezuel":13420,"ĠMega":13421,"Ġtar":13422,"Ġexpects":13423,"black":13424,"orph":13425,"\\\\\\\\":13426,"Ġacceptance":13427,"Ġradar":13428,"sis":13429,"Ġjunior":13430,"Ġframes":13431,"Ġobservation":13432,"acies":13433,"Power":13434,"ĠAdvanced":13435,"Mag":13436,"ologically":13437,"ĠMechan":13438,"Ġsentences":13439,"Ġanalysts":13440,"aughters":13441,"forcement":13442,"Ġvague":13443,"Ġclause":13444,"Ġdirectors":13445,"Ġevaluate":13446,"Ġcabinet":13447,"Matt":13448,"ĠClassic":13449,"Ang":13450,"Ġcler":13451,"ĠBuck":13452,"Ġresearcher":13453,"Ġ160":13454,"Ġpoorly":13455,"Ġexperiencing":13456,"ĠPed":13457,"ĠManhattan":13458,"Ġfreed":13459,"Ġthemes":13460,"advant":13461,"Ġnin":13462,"Ġpraise":13463,"104":13464,"ĠLibya":13465,"best":13466,"Ġtrusted":13467,"Ġcease":13468,"Ġdign":13469,"Direct":13470,"Ġbombing":13471,"Ġmigration":13472,"ĠSciences":13473,"Ġmunicipal":13474,"ĠAverage":13475,"Ġglory":13476,"Ġrevealing":13477,"Ġarena":13478,"Ġuncertainty":13479,"Ġbattlefield":13480,"iao":13481,"God":13482,"Ġcinem":13483,"rape":13484,"elle":13485,"apons":13486,"Ġlisting":13487,"Ġwaited":13488,"Ġspotted":13489,"keley":13490,"ĠAudio":13491,"eor":13492,"arding":13493,"idding":13494,"igma":13495,"ĠNeg":13496,"Ġlone":13497,"Ġ----":13498,"exe":13499,"deg":13500,"Ġtransf":13501,"Ġwash":13502,"Ġslavery":13503,"Ġexploring":13504,"ĠWW":13505,"atson":13506,"Ġencl":13507,"lies":13508,"ĠCreek":13509,"Ġwooden":13510,"Manager":13511,"ĠBrand":13512,"ummy":13513,"ĠArthur":13514,"Ġbureaucr":13515,"Ġblend":13516,"arians":13517,"Further":13518,"Ġsupposedly":13519,"Ġwinds":13520,"Ġ1979":13521,"Ġgravity":13522,"Ġanalyses":13523,"ĠTravel":13524,"ĠVeter":13525,"Ġdumb":13526,"Ġalternate":13527,"gal":13528,"Ġconsumed":13529,"Ġeffectiveness":13530,".''":13531,"Ġpaths":13532,"onda":13533,"LA":13534,"ĠStrong":13535,"Ġenables":13536,"Ġescaped":13537,"Ġ\"\"":13538,"Ġ112":13539,"Ġ1983":13540,"Ġsmiled":13541,"Ġtendency":13542,"Fire":13543,"Ġpars":13544,"ĠRoc":13545,"Ġlake":13546,"Ġfitness":13547,"ĠAth":13548,"ĠHorn":13549,"Ġhier":13550,"Ġimpose":13551,"mother":13552,"Ġpension":13553,"icut":13554,"borne":13555,"iciary":13556,"._":13557,"ĠSU":13558,"Ġpolar":13559,"isy":13560,"engu":13561,"itialized":13562,"ATA":13563,"write":13564,"Ġexercises":13565,"ĠDiamond":13566,"otypes":13567,"Ġharmful":13568,"onz":13569,"Ġprinting":13570,"story":13571,"Ġexpertise":13572,"ĠGer":13573,"Ġtragedy":13574,"ĠFly":13575,"Ġdivid":13576,"ampire":13577,"stock":13578,"Mem":13579,"Ġreign":13580,"Ġunve":13581,"Ġamend":13582,"ĠProphet":13583,"Ġmutual":13584,"ĠFac":13585,"Ġreplacing":13586,"Har":13587,"ĠCircuit":13588,"Ġthroat":13589,"ĠShot":13590,"Ġbatteries":13591,"Ġtoll":13592,"Ġaddressing":13593,"ĠMedicaid":13594,"Ġpupp":13595,"ĠNar":13596,"olk":13597,"Ġequity":13598,"MR":13599,"ĠHispan":13600,"ĠLarge":13601,"mid":13602,"Dev":13603,"Ġexped":13604,"Ġdemo":13605,"ĠMarshall":13606,"ergus":13607,"Ġfiber":13608,"Ġdivorce":13609,"ĠCreate":13610,"Ġslower":13611,"ĠParker":13612,"ĠStudent":13613,"ĠTraining":13614,"Return":13615,"ĠTru":13616,"Ġcub":13617,"ĠReached":13618,"Ġpanic":13619,"Ġquarters":13620,"Ġrect":13621,"Ġtreating":13622,"Ġrats":13623,"ĠChristianity":13624,"oler":13625,"Ġsacred":13626,"Ġdeclare":13627,"ulative":13628,"eting":13629,"Ġdelivering":13630,"estone":13631,"Ġtel":13632,"ĠLarry":13633,"Ġmeta":13634,"accept":13635,"artz":13636,"ĠRoger":13637,"handed":13638,"Ġheader":13639,"Ġtrapped":13640,"ĠCentury":13641,"Ġknocked":13642,"ĠOxford":13643,"Ġsurvivors":13644,"bot":13645,"Ġdemonstration":13646,"Ġdirt":13647,"Ġassists":13648,"OME":13649,"ĠDraft":13650,"ortunate":13651,"folio":13652,"pered":13653,"usters":13654,"gt":13655,"ĠLock":13656,"Ġjudicial":13657,"verted":13658,"Ġsecured":13659,"outing":13660,"ĠBooks":13661,"Ġhosting":13662,"Ġlifted":13663,"length":13664,"Ġjer":13665,"Ġwheels":13666,"ĠRange":13667,"umbnails":13668,"Ġdiagnosis":13669,"tech":13670,"ĠStewart":13671,"ĠPract":13672,"Ġnationwide":13673,"Ġdear":13674,"Ġobligations":13675,"Ġgrows":13676,"Ġmandatory":13677,"Ġsuspicious":13678,"!'":13679,"Apr":13680,"Great":13681,"Ġmortgage":13682,"Ġprosecutor":13683,"Ġeditorial":13684,"ĠKr":13685,"Ġprocessed":13686,"ungle":13687,"Ġflexibility":13688,"Earlier":13689,"ĠCart":13690,"ĠSug":13691,"Ġfocuses":13692,"Ġstartup":13693,"Ġbreach":13694,"ĠTob":13695,"cycle":13696,"ãĢĮ":13697,"rose":13698,"Ġbizarre":13699,"ãĢį":13700,"Ġvegetables":13701,"$$":13702,"Ġretreat":13703,"oshi":13704,"ĠShop":13705,"ĠGround":13706,"ĠStop":13707,"ĠHawaii":13708,"ĠAy":13709,"Perhaps":13710,"ĠBeaut":13711,"uffer":13712,"enna":13713,"Ġproductivity":13714,"Fixed":13715,"control":13716,"Ġabsent":13717,"ĠCampaign":13718,"Green":13719,"Ġidentifying":13720,"Ġregret":13721,"Ġpromoted":13722,"ĠSeven":13723,"Ġeru":13724,"neath":13725,"aughed":13726,"ĠPin":13727,"ĠLiving":13728,"Cost":13729,"omatic":13730,"mega":13731,"ĠNig":13732,"ocy":13733,"Ġinbox":13734,"Ġempire":13735,"Ġhorizont":13736,"Ġbranches":13737,"Ġmetaph":13738,"Active":13739,"edi":13740,"ĠFilm":13741,"ĠSomething":13742,"Ġmods":13743,"incial":13744,"ĠOriginal":13745,"Gen":13746,"Ġspirits":13747,"Ġearning":13748,"Hist":13749,"Ġriders":13750,"Ġsacrific":13751,"MT":13752,"ĠVA":13753,"ĠSalt":13754,"Ġoccupation":13755,"ĠMi":13756,"Ġdisg":13757,"lict":13758,"Ġnit":13759,"Ġnodes":13760,"eem":13761,"ĠPier":13762,"Ġhatred":13763,"psy":13764,"ãĥī":13765,"Ġtheater":13766,"Ġsophisticated":13767,"Ġdefended":13768,"Ġbesides":13769,"Ġthoroughly":13770,"ĠMedicare":13771,"Ġblamed":13772,"arently":13773,"Ġcrying":13774,"FOR":13775,"priv":13776,"Ġsinging":13777,"ĠIl":13778,"Ġcute":13779,"oided":13780,"olitical":13781,"ĠNeuro":13782,"å¤":13783,"Ġdonation":13784,"ĠEagles":13785,"ĠGive":13786,"Tom":13787,"Ġsubstantially":13788,"ĠLicense":13789,"ĠJa":13790,"Ġgrey":13791,"ĠAnimal":13792,"ĠER":13793,"ĠUnd":13794,"Ġkeen":13795,"Ġconclude":13796,"ĠMississippi":13797,"Engine":13798,"ĠStudios":13799,"Press":13800,"overs":13801,"llers":13802,"Ġ350":13803,"ĠRangers":13804,"Ġrou":13805,"erto":13806,"Ep":13807,"issa":13808,"ivan":13809,"Ġseal":13810,"ĠRegist":13811,"display":13812,"Ġweaken":13813,"uum":13814,"ĠCommons":13815,"ĠSay":13816,"Ġcultures":13817,"Ġlaughed":13818,"Ġslip":13819,"Ġtreatments":13820,"izable":13821,"mart":13822,"ĠRice":13823,"Ġbeast":13824,"Ġobesity":13825,"ĠLaure":13826,"iga":13827,"Which":13828,"holder":13829,"Ġelderly":13830,"Ġpays":13831,"Ġcomplained":13832,"Ġcrop":13833,"Ġproc":13834,"Ġexplosive":13835,"ĠFan":13836,"ĠArsenal":13837,"Author":13838,"eful":13839,"Ġmeals":13840,"Ġ(-":13841,"idays":13842,"Ġimagination":13843,"Ġannually":13844,"Ġms":13845,"asures":13846,"Head":13847,"ikh":13848,"matic":13849,"Ġboyfriend":13850,"ĠComputer":13851,"Ġbump":13852,"Ġsurge":13853,"ĠCraig":13854,"ĠKirk":13855,"Del":13856,"mediate":13857,"Ġscenarios":13858,"ĠMut":13859,"ĠStream":13860,"Ġcompetitors":13861,"ÙĦ":13862,"ĠStanford":13863,"ĠResources":13864,"azed":13865,"bage":13866,"Ġorganis":13867,"ĠRelease":13868,"Ġseparately":13869,"Ġhabits":13870,"Ġmeasurements":13871,"ĠClose":13872,"Ġaccompany":13873,"Ġgly":13874,"Ġtang":13875,"ĠRou":13876,"Ġplugin":13877,"Ġconvey":13878,"ĠChallenge":13879,"oots":13880,"jan":13881,"Ġcurs":13882,"ĠRelations":13883,"keeper":13884,"Ġapproaching":13885,"ping":13886,"Speaking":13887,"Ġarrangement":13888,"ĠVI":13889,"arettes":13890,"Ġaffecting":13891,"Ġpermits":13892,"because":13893,"Ġuseless":13894,"ĠHus":13895,"!!!!":13896,"Ġdestroying":13897,"Unfortunately":13898,"Ġfascinating":13899,"Sem":13900,"Ġelectoral":13901,"Ġtransparency":13902,"ĠChaos":13903,"Ġvolunteer":13904,"Ġstatistical":13905,"Ġactivated":13906,"rox":13907,"Web":13908,"HE":13909,"ĠHampshire":13910,"isive":13911,"Map":13912,"Ġtrash":13913,"ĠLawrence":13914,"stick":13915,"Cr":13916,"Ġrings":13917,"EXT":13918,"Ġoperational":13919,"opes":13920,"Does":13921,"ĠEvans":13922,"Ġwitnessed":13923,"Port":13924,"Ġlaunching":13925,"econom":13926,"wear":13927,"ĠParticip":13928,"umm":13929,"cules":13930,"ĠRAM":13931,"ĠTun":13932,"Ġassured":13933,"Ġbinary":13934,"Ġbetray":13935,"Ġexploration":13936,"ĠFel":13937,"Ġadmission":13938,"itated":13939,"Sy":13940,"Ġavoided":13941,"ĠSimulator":13942,"Ġcelebrated":13943,"ĠElectric":13944,"¥ŀ":13945,"Ġcluster":13946,"itzerland":13947,"health":13948,"Line":13949,"ĠNash":13950,"aton":13951,"Ġspare":13952,"Ġenterprise":13953,"ĠDIS":13954,"cludes":13955,"Ġflights":13956,"Ġregards":13957,"ĠÃĹ":13958,"half":13959,"Ġtrucks":13960,"Ġcontacts":13961,"Ġuncons":13962,"ĠClimate":13963,"Ġimmense":13964,"NEW":13965,"occ":13966,"ective":13967,"Ġembod":13968,"Ġpatrol":13969,"Ġbeside":13970,"Ġviable":13971,"Ġcreep":13972,"Ġtriggered":13973,"verning":13974,"Ġcomparable":13975,"ql":13976,"Ġgaining":13977,"asses":13978,"Ġ();":13979,"ĠGrey":13980,"ĠMLS":13981,"sized":13982,"Ġprosper":13983,"\"?":13984,"Ġpolling":13985,"Ġshar":13986,"ĠRC":13987,"Ġfirearm":13988,"orient":13989,"Ġfence":13990,"Ġvariations":13991,"giving":13992,"ĠPi":13993,"ospel":13994,"Ġpledge":13995,"Ġcure":13996,"Ġspy":13997,"Ġviolated":13998,"Ġrushed":13999,"Ġstroke":14000,"ĠBlog":14001,"sels":14002,"ĠEc":14003,",''":14004,"Ġpale":14005,"ĠCollins":14006,"terror":14007,"ĠCanadians":14008,"Ġtune":14009,"Ġlaboratory":14010,"Ġnons":14011,"tarian":14012,"Ġdisability":14013,"ĠGam":14014,"Ġsinger":14015,"alg":14016,"ĠSenior":14017,"Ġtraded":14018,"ĠWarrior":14019,"Ġinfring":14020,"ĠFranklin":14021,"Ġstrain":14022,"ĠSwedish":14023,"Ġseventh":14024,"ĠBenn":14025,"ĠTell":14026,"Ġsyndrome":14027,"Ġwondered":14028,"iden":14029,"++++":14030,"igo":14031,"Ġpurple":14032,"Ġjournalism":14033,"Ġrebel":14034,"Ġfu":14035,"blog":14036,"Ġinvite":14037,"rencies":14038,"ĠContact":14039,"Israel":14040,"ĠContent":14041,"Ġcheer":14042,"Ġbedroom":14043,"ĠEngineering":14044,"ĠQueens":14045,"Ġdwell":14046,"ĠPlayStation":14047,"ĠDim":14048,"ĠColon":14049,"lr":14050,"Ġoperates":14051,"Ġmotivation":14052,"USA":14053,"astered":14054,"Core":14055,"ĠTruth":14056,"olo":14057,"OSE":14058,"ĠMemory":14059,"Ġpredec":14060,"Ġanarch":14061,"Ġ1920":14062,"ĠYam":14063,"è":14064,"bid":14065,"Ġgrateful":14066,"Ġexcitement":14067,"Ġtreasure":14068,"Ġlongest":14069,"ctive":14070,"Ġdeserves":14071,"Ġreserves":14072,"Ġcops":14073,"ĠOttawa":14074,"ĠEgyptian":14075,"anked":14076,"Ġartif":14077,"Ġhypothesis":14078,":/":14079,"Ġpurchasing":14080,"Ġlovely":14081,"HP":14082,"Ġdivide":14083,"Ġstrictly":14084,"Ġquestioning":14085,"Ġtaxpayers":14086,"ĠJoy":14087,"Ġrolls":14088,"ĠHeavy":14089,"Ġports":14090,"Ġmagnetic":14091,"Ġinflamm":14092,"Ġbrush":14093,"tics":14094,"âĪĴ":14095,"Ġbottles":14096,"ppy":14097,"Ġpadd":14098,"ãĤ¯":14099,"million":14100,"Ġdevastating":14101,"Ġcompiled":14102,"Ġmedication":14103,"Ġtwelve":14104,"ĠPerry":14105,"Space":14106,"imb":14107,"your":14108,"Ġleaked":14109,"ĠTar":14110,"Ġunity":14111,"Ġinfected":14112,"Ġtraveled":14113,"IDE":14114,"ĠMcDonald":14115,"txt":14116,"ĠPrinc":14117,"Ġinterven":14118,"ĠTaiwan":14119,"ĠPow":14120,"Ġbearing":14121,"ĠThread":14122,"Ġzones":14123,"izards":14124,"unks":14125,"Chapter":14126,"llor":14127,"Ġ·":14128,"Ġwounds":14129,"Ġdiscretion":14130,"Ġsucceeded":14131,"iking":14132,"Ġiconic":14133,"Call":14134,"Ġscreening":14135,"ĠMis":14136,"icts":14137,"Ġministers":14138,"Ġseparation":14139,"Player":14140,"Ġbip":14141,"Ġbeloved":14142,"Ġcounting":14143,"ĠEye":14144,"around":14145,"inging":14146,"Ġtablet":14147,"Ġoffence":14148,"inance":14149,"have":14150,"ĠInfo":14151,"ĠNinja":14152,"Ġprotective":14153,"ĠCass":14154,"Mac":14155,"ĠQuality":14156,"North":14157,"Ġic":14158,"ĠCuba":14159,"ĠChronicle":14160,"ĠProperty":14161,"Ġfastest":14162,"otos":14163,"ĠGerm":14164,"OWN":14165,"Ġboom":14166,"ĠStanley":14167,"erguson":14168,"Ġclever":14169,"Ġenters":14170,"mode":14171,"terior":14172,"ĠSens":14173,"Ġlinear":14174,"ARK":14175,"Ġcomparing":14176,"Ġpurely":14177,"Ġsafer":14178,"ĠPotter":14179,"Ġcups":14180,"RT":14181,"Ġgluc":14182,"Ġattributed":14183,"Ġdupl":14184,"ĠPap":14185,"Ġprecious":14186,"Ġpa":14187,"ictionary":14188,"ĠTig":14189,"ĠToo":14190,"olutions":14191,"stan":14192,"Ġrobots":14193,"Ġlobb":14194,"Ġstatute":14195,"Ġprevention":14196,"western":14197,"160":14198,"ĠActive":14199,"ĠMaria":14200,"hal":14201,"None":14202,"ellar":14203,"ĠKB":14204,"ĠPartners":14205,"ĠSingle":14206,"ĠFollowing":14207,"ango":14208,"acious":14209,"Ġthou":14210,"Ġkg":14211,"Ġinfluential":14212,"ĠFriends":14213,"Sur":14214,"ainted":14215,"Ġforums":14216,"Ġstarter":14217,"Ġcitizenship":14218,"ĠElection":14219,"onge":14220,"otation":14221,"osph":14222,";;;;":14223,"utical":14224,"pur":14225,"eren":14226,"Ġaccusations":14227,"bitious":14228,"abbit":14229,"ĠOrd":14230,"Posted":14231,"irk":14232,"Ġsensitivity":14233,"iche":14234,"ĠAmy":14235,"ĠFab":14236,"Ġsummit":14237,"Ġpedest":14238,"Ġrubber":14239,"Ġagricultural":14240,"Ġcancel":14241,"AE":14242,"Ġinaug":14243,"Ġcontam":14244,"Ġfirmly":14245,"iw":14246,"stage":14247,"ĠKan":14248,"Ġtier":14249,"Ġinvention":14250,"Ġtranslated":14251,"ĠRules":14252,"Box":14253,"Twitter":14254,"IDS":14255,"Ġpizza":14256,"Ġdebug":14257,"ĠDrop":14258,"vs":14259,"Ġhorses":14260,"big":14261,"Ġboring":14262,"Ġhood":14263,"ĠMcCain":14264,"atched":14265,"ĠBros":14266,"Ġskip":14267,"Ġessay":14268,"stat":14269,"ĠLegends":14270,"Ġammunition":14271,"auc":14272,"Ġshooter":14273,"Ġunh":14274,"Ġsupplied":14275,"Ġgeneric":14276,"ĠSK":14277,"iban":14278,"yrics":14279,"Ġ255":14280,"Ġclimbing":14281,"Former":14282,"Ġflip":14283,"Ġjumping":14284,"Ġfrustration":14285,"ĠTerry":14286,"Ġneighborhoods":14287,"Ġmedian":14288,"bean":14289,"Ġbrains":14290,"Following":14291,"Ġshaped":14292,"Ġdraws":14293,"Ġaltered":14294,"Jack":14295,"Ġrecipes":14296,"Ġskilled":14297,"wealth":14298,"achi":14299,"election":14300,"Ġbehaviors":14301,"deals":14302,"ĠUntil":14303,"Fe":14304,"Ġdeclaration":14305,"marks":14306,"ĠBetween":14307,"celona":14308,"Ġreson":14309,"Ġbubble":14310,"Among":14311,"Ġimperial":14312,"GS":14313,"Ġfeminist":14314,"2005":14315,"ĠKyle":14316,"Ġaccounting":14317,"ĠTele":14318,"ĠTyr":14319,"Ġconnecting":14320,"Ġrehab":14321,"ĠPred":14322,"sim":14323,"Ġmeantime":14324,"Ġphysician":14325,"MW":14326,"ĠCampbell":14327,"ĠBrandon":14328,"Ġcontributing":14329,"ĠRule":14330,"ĠWeight":14331,"ĠNap":14332,"Ġinteractive":14333,"Ġvag":14334,"Ġhelmet":14335,"ĠComb":14336,"four":14337,"Ġshipped":14338,"Ġcompleting":14339,"ĠPD":14340,"PDATE":14341,"Ġspreading":14342,"Ġscary":14343,"erving":14344,"ĠGas":14345,"Ġfrank":14346,"school":14347,"Ġromantic":14348,"Ġstabil":14349,"Rob":14350,"Ġaccurately":14351,"Ġacute":14352,"ĠHann":14353,"Ġsymbols":14354,"Ġcivilization":14355,"ĠAW":14356,"Ġlightning":14357,"Ġconsiders":14358,"Ġvenue":14359,"Ġ×":14360,"Ġoven":14361,"ĠSF":14362,"his":14363,"Ġnu":14364,"ĠLearn":14365,"Ġpeoples":14366,"Ġstd":14367,"Ġslee":14368,"Ġslic":14369,"ĠStatistics":14370,"Ġcorners":14371,"ĠBaker":14372,"Ġ:)":14373,"mentation":14374,"olver":14375,"Ġlaughing":14376,"ĠTodd":14377,"onde":14378,"ĠHills":14379,"Ġnuts":14380,"ĠWoman":14381,"plane":14382,"Ġliver":14383,"ĠInside":14384,"Sorry":14385,"Ġagrees":14386,"Ġfundament":14387,"ĠFisher":14388,"Ġauction":14389,"Ġthreads":14390,"glas":14391,"ĠBasic":14392,"ĠNat":14393,"Ġlacking":14394,"Ġcelebration":14395,"ju":14396,"Ġsilly":14397,"Euro":14398,"Ġtatt":14399,"ighty":14400,"controlled":14401,"Test":14402,"ĠSingh":14403,"Ġrage":14404,"Ġrhyth":14405,"offic":14406,"ĠPhantom":14407,"Ġheadlines":14408,"Ġresponding":14409,"ĠMorning":14410,"Ġvitamin":14411,"Ġboots":14412,"ĠSite":14413,"alin":14414,"pi":14415,"Ġviral":14416,"ĠUC":14417,"DER":14418,"ĠSex":14419,"Ġstocks":14420,"current":14421,"Ġchurches":14422,"ĠRare":14423,"ĠMurphy":14424,"Ġdenial":14425,"ĠGaming":14426,"Ġtoug":14427,"Ġnick":14428,"Ġmakers":14429,"ĠRonald":14430,"Ġgenerous":14431,"ĠDoc":14432,"ĠMorris":14433,"Ġtransformed":14434,"ĠNormal":14435,"Ġ104":14436,"ĠKickstarter":14437,"ĠUpon":14438,"Online":14439,"ĠIRS":14440,"Ġwrap":14441,"Ġloving":14442,"Ġarrives":14443,"ĠDue":14444,"Ġheter":14445,"ĠMade":14446,"Ġrental":14447,"Ġbelongs":14448,"Ġattorneys":14449,"Ġcrops":14450,"Ġmatched":14451,"ulum":14452,"oline":14453,"109":14454,"Ġdispar":14455,"Ġbuyers":14456,"ĠCambridge":14457,"Ġethics":14458,"roups":14459,"Ġjustified":14460,"Ġmarginal":14461,"Ġrespected":14462,"winning":14463,"Ġnodded":14464,"ĠSerge":14465,"ĠFormer":14466,"Craft":14467,"################":14468,"ĠWarner":14469,"Ġdash":14470,"ete":14471,"Ġentert":14472,"ĠEscape":14473,"outheast":14474,"Ġknees":14475,"ĠBomb":14476,"Ġrug":14477,"Pass":14478,"Ġattitudes":14479,"government":14480,"ĠPrior":14481,"Ġqualities":14482,"Ġnotification":14483,"ĠPhone":14484,"lie":14485,"Ġanticipated":14486,"ĠCombat":14487,"ĠBarry":14488,"Ġ1982":14489,"Users":14490,"oner":14491,"Ġcomputing":14492,"ĠConnecticut":14493,"Ġlesser":14494,"Ġpeers":14495,"ĠCu":14496,"Ġtechnically":14497,"Ġsubmission":14498,"ĠUniversal":14499,"Ġmanually":14500,"ourge":14501,"Ġrespondents":14502,"ĠBTC":14503,"ĠHost":14504,"Ġfare":14505,"ĠBird":14506,"Ġreceipt":14507,"also":14508,"Ġjack":14509,"Ġagriculture":14510,"Ġskull":14511,"Ġ!=":14512,"Ġpassive":14513,"ĠCI":14514,"Ġsocieties":14515,"Ġreminded":14516,"Ġinterference":14517,"Buy":14518,"Ġâľ":14519,"gon":14520,"Ġscrutiny":14521,"ĠWitch":14522,"Ġconducting":14523,"Ġãĥ":14524,"Ġexchanges":14525,"ĠMitchell":14526,"Ġinhabit":14527,"Ġtwist":14528,"BD":14529,"Ġwherever":14530,"groupon":14531,"Ġjokes":14532,"ĠBenjamin":14533,"ĠRandom":14534,"frame":14535,"ĠLions":14536,"Ġhighlighted":14537,"ĠArkansas":14538,"Ent":14539,"Ġpile":14540,"Ġprelim":14541,"gs":14542,"minded":14543,"Ġfelony":14544,"ĠGA":14545,"ĠLuck":14546,"Ġpractically":14547,"ĠBos":14548,"Ġactress":14549,"Dam":14550,"ĠBou":14551,"Ġvisa":14552,"Ġembedded":14553,"Ġhybrid":14554,"Ġearliest":14555,"Ġsooner":14556,"social":14557,"ĠHA":14558,"Ġsteep":14559,"Ġdisadvant":14560,"Ġexploit":14561,"ĠEgg":14562,"ĠUltra":14563,"Ġnecessity":14564,"Local":14565,"iege":14566,"Ġdated":14567,"Ġmasses":14568,"Ġsubscription":14569,"pless":14570,"Ġanonym":14571,"Ġpresumably":14572,"Blue":14573,"Their":14574,"asketball":14575,"ĠPhilip":14576,"Ġcomed":14577,"loaded":14578,"rane":14579,"Ġreflection":14580,"China":14581,"Ġextends":14582,"Ġforming":14583,"Ġunders":14584,"2001":14585,"Ġgrat":14586,"Ġconcentrations":14587,"Ġinsulin":14588,"Ġsecular":14589,"Ġwhilst":14590,"Ġwinners":14591,"Advertisements":14592,"Ġdeliberately":14593,"ĠWorking":14594,"Ġsink":14595,"etics":14596,"dale":14597,"Ġmandate":14598,"Ġgram":14599,"Ġvacation":14600,"Ġwarnings":14601,"ripp":14602,"ĠTHAT":14603,"Ġcommentary":14604,"Ġintu":14605,"Ġaest":14606,"Ġreasoning":14607,"Ġbreakdown":14608,"ĠZombie":14609,"Ġ-->":14610,"ĠPolitical":14611,"cott":14612,"Ġthrust":14613,"Ġtechnological":14614,"Ġdeciding":14615,"Ġtrafficking":14616,"Long":14617,"Welcome":14618,"prising":14619,"ĠCommunications":14620,"Ġendors":14621,"Ġswift":14622,"Ġmetabol":14623,"coins":14624,"resa":14625,"ĠHTTP":14626,"Ġenroll":14627,"ĠHappy":14628,"usr":14629,"intage":14630,"Ġ[\"":14631,"uably":14632,"ĠMaterial":14633,"Ġrepeal":14634,"Sept":14635,"kh":14636,"ĠModi":14637,"Ġunderneath":14638,"ĠIL":14639,"shore":14640,"Ġdiagnosed":14641,"aceutical":14642,"Ġshower":14643,"aux":14644,"ĠSwitch":14645,"ĠStrength":14646,"Ġjihad":14647,"national":14648,"Ġtrauma":14649,"ussy":14650,"oni":14651,"Ġconsolid":14652,"Ġcalories":14653,"ĠFlynn":14654,"agged":14655,"168":14656,"ĠPink":14657,"Ġfulfill":14658,"Ġchains":14659,"Ġnotably":14660,"ĠAV":14661,"Life":14662,"ĠChuck":14663,"mus":14664,"ĠUrban":14665,"ĠHend":14666,"Ġdeposit":14667,"ĠSad":14668,"Ġaffair":14669,"ORK":14670,"ieval":14671,"ĠFDA":14672,"Ġtrop":14673,"ĠOverall":14674,"Ġvirtue":14675,"Ġsatisfaction":14676,"aund":14677,"Ġlun":14678,"ĠSwitzerland":14679,"ĠOperation":14680,"process":14681,"Ġshook":14682,"Ġcounties":14683,"leased":14684,"ĠCharlotte":14685,"112":14686,"Ġtranscript":14687,"Ġredd":14688,"push":14689,"ĠHey":14690,"ĠAnalysis":14691,"[\"":14692,"Ġalternatives":14693,"ardless":14694,"Ġeleph":14695,"Ġprejud":14696,"ĠLeaf":14697,"Having":14698,"ĠHub":14699,"Ġexpressions":14700,"ĠVolume":14701,"Ġshocking":14702,"ĠReds":14703,"Ġreadily":14704,"Ġplanets":14705,"adata":14706,"Ġcollapsed":14707,"ĠMadrid":14708,"Ġirrit":14709,"ipper":14710,"ĠEnc":14711,"ĠWire":14712,"Ġbuzz":14713,"ĠGP":14714,"asha":14715,"Ġaccidentally":14716,"uru":14717,"Ġfrustrated":14718,"ĠSA":14719,"Ġhungry":14720,"ĠHuff":14721,"Ġlabels":14722,"anto":14723,"ĠEP":14724,"Ġbarriers":14725,")|":14726,"ĠBerkeley":14727,"ĠJets":14728,"Ġpairs":14729,"ĠLan":14730,"James":14731,"ĠBear":14732,"Ġhumor":14733,"ĠLiberty":14734,"Ġmagnitude":14735,"Ġaging":14736,"ĠMason":14737,"Ġfriendship":14738,"umbling":14739,"Ġemerge":14740,"Ġnewspapers":14741,"Ġambitious":14742,"ĠRichards":14743,"aternal":14744,"Ġ1981":14745,"Ġcookies":14746,"Ġsculpt":14747,"Ġpursuit":14748,"Location":14749,"Ġscripts":14750,"pc":14751,"Ġarrangements":14752,"Ġdiameter":14753,"Ġloses":14754,"amation":14755,"Ġliqu":14756,"ĠJake":14757,"arette":14758,"Ġunderstands":14759,"ĠZen":14760,"vm":14761,"Ġapprove":14762,"Ġwip":14763,"Ġultra":14764,"Ġintend":14765,"ĠDI":14766,"ascular":14767,"Ġstays":14768,"ĠKor":14769,"ĠKl":14770,"Ġinvesting":14771,"La":14772,"Ġbelieving":14773,"bad":14774,"mouth":14775,"Ġtaxpayer":14776,"ãĥĥ":14777,"ĠQuebec":14778,"Ġlap":14779,"ĠSwiss":14780,"drop":14781,"Ġdrain":14782,"iri":14783,"etc":14784,"ften":14785,"ĠNex":14786,"Ġstraw":14787,"Ġscreaming":14788,"Ġcounted":14789,"Ġdamaging":14790,"Ġambassador":14791,"century":14792,"Ġprox":14793,"Ġarrests":14794,"uv":14795,"ilateral":14796,"ĠCharg":14797,"Ġprescribed":14798,"Ġindependently":14799,"Ġfierce":14800,"ĠBaby":14801,"Ġbrave":14802,"Ġsuits":14803,"=>":14804,"Ġbaseline":14805,"ĠRate":14806,"Ġislands":14807,"Ġ((":14808,"green":14809,"ixels":14810,"Ġnamely":14811,"ĠVillage":14812,"than":14813,"amy":14814,"Version":14815,"gmail":14816,"entials":14817,"ĠSud":14818,"ĠMelbourne":14819,"Ġarriving":14820,"Ġquantum":14821,"eff":14822,"ropolitan":14823,"Tri":14824,"Ġfuneral":14825,"ĠIR":14826,"ÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤ":14827,"ĠCob":14828,"itably":14829,"Ġturb":14830,"Ġcombo":14831,"Review":14832,"Ġdeployment":14833,"uity":14834,"ĠBott":14835,"Ġinvisible":14836,"Ġrendering":14837,"Ġunlocked":14838,"Ġaqu":14839,"ĠVladimir":14840,"Ġpad":14841,"ĠBrain":14842,"ĠLegacy":14843,"dragon":14844,"ĠKurdish":14845,"Ġsounded":14846,"Ġdetained":14847,"ĠDM":14848,"gary":14849,"Ġdaughters":14850,"Ġdisturbing":14851,"uka":14852,"ĠParad":14853,"Ġtast":14854,"Ġunfortunate":14855,"Ġul":14856,"emin":14857,"Ġattendance":14858,"trl":14859,"Ġparks":14860,"ĠMemorial":14861,"ĠAlice":14862,"othy":14863,"guard":14864,"ĠDise":14865,"ĠShan":14866,"ĠForum":14867,"Rich":14868,"Ġshifted":14869,"uez":14870,"Ġlighter":14871,"ĠMagn":14872,"Ġcod":14873,"Sch":14874,"hammad":14875,"Pub":14876,"350":14877,"ĠPokemon":14878,"Ġprototype":14879,"Ġunre":14880,"Base":14881,"ĠStudents":14882,"ĠReply":14883,"ĠCommunist":14884,"Ġgau":14885,"ĠTyler":14886,"IZ":14887,"Ġparticipated":14888,"Ġsuprem":14889,"ĠDetails":14890,"Ġvessels":14891,"rod":14892,"Ġtribe":14893,"keep":14894,"Ġassumptions":14895,"Ġpound":14896,"Ġcrude":14897,"ĠAvailable":14898,"Ġswimming":14899,"Ġinclusion":14900,"Ġadvances":14901,"culation":14902,"Ġconservation":14903,"Ġoverd":14904,"ĠBuffalo":14905,"Article":14906,"edge":14907,"Ġawa":14908,"ĠMadison":14909,"Ġsidew":14910,"Ġcatast":14911,"ĠKrist":14912,"ucle":14913,"ĠHighway":14914,"ĠTerror":14915,"Ġactivation":14916,"Ġunconscious":14917,"ĠSatan":14918,"ĠSusan":14919,"illery":14920,"Ġarranged":14921,"iop":14922,"Ġrumors":14923,"urring":14924,"think":14925,"ĠKeith":14926,"ĠKind":14927,"Ġavoiding":14928,"byn":14929,"nut":14930,"ĠSpeaker":14931,"rus":14932,"names":14933,"Ġguilt":14934,"ĠOlympics":14935,"Ġsail":14936,"ĠMes":14937,"levant":14938,"ĠColumbus":14939,"aft":14940,"City":14941,"South":14942,"ĠHarvey":14943,"ĠPun":14944,"Several":14945,"Ġmentally":14946,"Ġimpress":14947,"mount":14948,"ĠUbuntu":14949,"âĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶ":14950,"ĠSuperman":14951,"ĠMPs":14952,"Ġintentions":14953,"ĠRacing":14954,"Ġlikelihood":14955,"Ġ240":14956,"Total":14957,"Ġtoys":14958,"ĠWatson":14959,"Ġurge":14960,"Lear":14961,"ĠPaper":14962,"Ġoccurring":14963,"ĠBeng":14964,"ĠCert":14965,"Ġstones":14966,"Tim":14967,"ĠTwin":14968,"zb":14969,"ĠDynam":14970,"Ġpolitician":14971,"kens":14972,"ĠEnterprise":14973,"UTERS":14974,"Ġabol":14975,"Ġrefresh":14976,"Ġarbitrary":14977,"pection":14978,"Ġtroubles":14979,"Ġ});":14980,"tv":14981,"Ġpilots":14982,"Ġdistribute":14983,"Ġaudit":14984,"Ġpause":14985,"original":14986,"Ġrivals":14987,"£":14988,"Fig":14989,"TL":14990,"abil":14991,"rying":14992,"Lin":14993,"ioned":14994,"lon":14995,"Ġfancy":14996,"Ġcrashed":14997,"Ġtract":14998,"Ġshed":14999,"Ġconsume":15000,"Based":15001,"download":15002,"init":15003,"Ġvoltage":15004,"Introdu":15005,"Ġcondemned":15006,"ĠFinance":15007,"respect":15008,"Ġexcluded":15009,"Ġestablishing":15010,"heric":15011,"Ġheritage":15012,"Ġspectacular":15013,"Ġunst":15014,"ĠSnowden":15015,"ĠLane":15016,"San":15017,"Ġprotections":15018,"struction":15019,"incinn":15020,"Ġmacro":15021,"Custom":15022,"iosity":15023,"Ġesp":15024,"Ġfunctioning":15025,"Ġmush":15026,"Ġpuzzle":15027,"Ġethical":15028,"Mal":15029,"Ġgoverning":15030,"ĠFerguson":15031,"Ġrestored":15032,"Ġstressed":15033,"ĠCounter":15034,"ĠKas":15035,"clip":15036,"ANS":15037,"Ġseiz":15038,"UK":15039,"byss":15040,"oldown":15041,"api":15042,"Ġpermanently":15043,"ounters":15044,"West":15045,"Through":15046,"Light":15047,"atoes":15048,"Ġneat":15049,"Ġcord":15050,"urer":15051,"Ġseverely":15052,"ĠAven":15053,"Ġinterrog":15054,"Ġtriple":15055,"Given":15056,"Number":15057,"Ġarise":15058,"Ġsher":15059,"plant":15060,"Ġflower":15061,"ĠCou":15062,"Ġate":15063,"Ġnewer":15064,"bul":15065,"Ġmeanwhile":15066,"ĠLair":15067,"Ġadjustment":15068,"ĠCopyright":15069,"Ġdivers":15070,"iological":15071,"Ġgamers":15072,"oat":15073,"Ġhistorically":15074,"Ġanalog":15075,"Ġlongtime":15076,"Ġprescription":15077,"ĠMist":15078,"ĠHyper":15079,"ĠMaine":15080,"ĠDeity":15081,"Ġmultipl":15082,"ĠReincarn":15083,"ĠHyd":15084,"ĠPic":15085,"Sil":15086,"rants":15087,"ĠCris":15088,".;":15089,"({":15090,"ependence":15091,"Ġrecy":15092,"ateur":15093,"Ġquad":15094,"Ġglob":15095,"Ġconced":15096,"team":15097,"Ġcapitalist":15098,"ĠLot":15099,"Ġroyal":15100,"ĠCyber":15101,"Ġblacks":15102,"metic":15103,"riv":15104,"ĠDanny":15105,"Ġspo":15106,"ĠRO":15107,"Ġanimated":15108,"rypted":15109,"ĠDeputy":15110,"Ġrendered":15111,"FE":15112,"Ġstreak":15113,"Ġclouds":15114,"ĠDoug":15115,"~~~~~~~~":15116,"Ġdiscour":15117,"ĠVeh":15118,"Ġpsychology":15119,"ĠJourney":15120,"Ġcrystal":15121,"ĠFrost":15122,"Ġsuspicion":15123,"Ġrelate":15124,"orus":15125,"ĠCrypt":15126,"ĠNVIDIA":15127,"comed":15128,"uting":15129,"incinnati":15130,"Ġvulnerability":15131,"ostic":15132,"Ġisolation":15133,"Ġcooling":15134,"ĠCoalition":15135,"Ġ119":15136,"Four":15137,"ĠDeal":15138,"Ġâī":15139,"semble":15140,"rament":15141,"ĠBarcelona":15142,"Ġ102":15143,"Ġcocaine":15144,"ocalypse":15145,"Feb":15146,"ogenic":15147,"Ġmutation":15148,"Ġcryptoc":15149,"ĠKel":15150,"ĠGit":15151,"ais":15152,"Ġsisters":15153,"ANK":15154,"Ġactivate":15155,"Ter":15156,"Ġdread":15157,"ylon":15158,"Ġpropri":15159,"Aust":15160,"ĠDefault":15161,"Ġoutdoor":15162,"Ġsheer":15163,"ceive":15164,"Ġgently":15165,"о":15166,"Program":15167,"ĠâĨĴ":15168,"Ġvegan":15169,"ĠCrus":15170,"Ġresponsibilities":15171,"ĠHR":15172,"OLD":15173,"Ġprevents":15174,"Ġstiff":15175,"ĠWere":15176,"Ġathletic":15177,"ĠScore":15178,"Ġ):":15179,"Ġcolumns":15180,"ĠLoc":15181,"available":15182,"ĠFram":15183,"ĠSessions":15184,"Ġcompanion":15185,"Ġpacks":15186,"140":15187,"ĠKnights":15188,"Ġfart":15189,"Ġstreams":15190,"Ġshore":15191,"Ġappeals":15192,"ĠPerformance":15193,"haul":15194,"ĠStra":15195,"ĠNag":15196,"103":15197,"ĠTransportation":15198,"BB":15199,"Ev":15200,"zan":15201,"Public":15202,"Ġtwin":15203,"ulsion":15204,"Mult":15205,"Ġelectro":15206,"Ġstatue":15207,"ationally":15208,"ĠNort":15209,"Ġinspection":15210,"/*":15211,"igue":15212,"Ġcompassion":15213,"ĠTales":15214,"ĠStein":15215,"ĠScreen":15216,"ĠBug":15217,"ĠLion":15218,"girl":15219,"Ġwithdrawal":15220,"Ġobjectives":15221,"Ġbloody":15222,"Ġpreliminary":15223,"Ġjacket":15224,"Ġdimensions":15225,"ĠCool":15226,"ĠOccup":15227,"Ġwreck":15228,"Ġdoubled":15229,"anking":15230,"Ġ1975":15231,"Ġglasses":15232,"ĠWang":15233,"prov":15234,"Path":15235,"connected":15236,"ĠMulti":15237,"ĠNorway":15238,"agonist":15239,"Ġfeared":15240,"Ġtouching":15241,"Ġarguably":15242,"¯¯¯¯¯¯¯¯":15243,"ĠNCAA":15244,"chem":15245,"Ġspat":15246,"ĠWWE":15247,"ĠCel":15248,"igger":15249,"Ġattacker":15250,"ĠJoin":15251,"object":15252,"etta":15253,"Ġeliminated":15254,"det":15255,"Ġdestruct":15256,"ĠLucas":15257,"ctuary":15258,"180":15259,"ĠBrady":15260,"ĠBlues":15261,"Bay":15262,"aukee":15263,"Ġtimeline":15264,"Ġdelegates":15265,"written":15266,"ufficient":15267,"Ġshapes":15268,"Copyright":15269,"ouble":15270,"service":15271,"Ġpione":15272,"Ġcolleges":15273,"Ġrows":15274,"Ġspite":15275,"Ġassessed":15276,"360":15277,"Ġlease":15278,"Ġconfidential":15279,"cker":15280,"ĠManning":15281,"ĠVoice":15282,"Ġsealed":15283,"Ġcalculate":15284,"NO":15285,"ĠAssistant":15286,"Ġteenager":15287,"ulent":15288,"atherine":15289,"Ġmock":15290,"Ġdiamond":15291,"Ġfest":15292,"Ġswitched":15293,"Ġresume":15294,"ĠPuerto":15295,"Ġlanes":15296,"iration":15297,"ĠSimilarly":15298,"Ġrod":15299,"ĠSel":15300,"ĠPalace":15301,"ĠLimited":15302,"eous":15303,"Ġvariant":15304,"Ġward":15305,"Ġ))":15306,"Show":15307,"OOK":15308,"Alex":15309,"ĠNep":15310,"bris":15311,"ĠWikipedia":15312,"Ġexceptional":15313,"Ġmanages":15314,"ĠDraw":15315,"Again":15316,"Ġcopper":15317,"utt":15318,"Ġexports":15319,"Ġportfolio":15320,"Ġelevated":15321,"Rated":15322,"ĠOtherwise":15323,"ĠTact":15324,"ĠShel":15325,"ĠTX":15326,"\"âĢĶ":15327,"Ġresur":15328,"ĠWa":15329,"venant":15330,"Ġmonetary":15331,"people":15332,"Email":15333,"Ġfifty":15334,"ĠSweet":15335,"ĠMalaysia":15336,"Ġconfusing":15337,"ĠRio":15338,"uda":15339,"utenant":15340,"\");":15341,"Ġpraised":15342,"Ġvolumes":15343,"turn":15344,"Ġmature":15345,"Ġnonprofit":15346,"Ġpassionate":15347,"ĠPrivate":15348,"Ġ103":15349,"Ġdescend":15350,"ç¥ŀ":15351,"uffy":15352,"headed":15353,"Whether":15354,"rien":15355,"zech":15356,"beit":15357,"Ġchrom":15358,"ĠMcM":15359,"Ġdancing":15360,"Ġeleg":15361,"ĠNoticed":15362,"115":15363,"Ġadvocacy":15364,"ENTS":15365,"ambling":15366,"ĠMinor":15367,"ĠFinn":15368,"Ġpriorities":15369,"Ġthereof":15370,"ĠStage":15371,"ĠRogers":15372,"Ġsubstitute":15373,"ĠJar":15374,"ĠJefferson":15375,"Ġlightly":15376,"102":15377,"ĠLisa":15378,"uits":15379,"ysical":15380,"Ġshifts":15381,"Ġdrones":15382,"Ġworkplace":15383,"Ġresid":15384,"ensed":15385,"ahn":15386,"Ġpreferences":15387,"server":15388,"Ġdebates":15389,"doc":15390,"ĠGods":15391,"Ġhelicopter":15392,"Ġhonour":15393,"Ġconsiderably":15394,"eded":15395,"ĠFemale":15396,"ĠAnne":15397,"Ġreun":15398,"ĠFace":15399,"ĠHallow":15400,"ĠBudget":15401,"Ġcondemn":15402,"Ġtender":15403,"Prof":15404,"ocratic":15405,"ĠTurner":15406,"ĠAgric":15407,"Ġ1976":15408,"Ġapt":15409,"disc":15410,"ĠFighter":15411,"ĠAur":15412,"Ġgarbage":15413,"input":15414,"ĠKarl":15415,"ĠOliver":15416,"ĠLanguage":15417,"kn":15418,"Non":15419,"ĠClar":15420,"Ġtraditions":15421,"Ġadvertisement":15422,"ĠSor":15423,"Ġarchive":15424,"Ġvillages":15425,"750":15426,"Ġimplementing":15427,"waukee":15428,"Ġdietary":15429,"Ġswitching":15430,"Republic":15431,"Ġvelocity":15432,"Ġcit":15433,"ĠAwards":15434,"Ġfinancing":15435,"Ġlasted":15436,")]":15437,"Ġreminder":15438,"Person":15439,"Ġprecision":15440,"Ġdesigners":15441,"ĠFried":15442,"ĠBorder":15443,"Ġtragic":15444,"Ġwield":15445,"Ġinitiatives":15446,"ĠTank":15447,"wer":15448,"Ġjoins":15449,"Ro":15450,"inery":15451,"Ġarrow":15452,"Ġgenerating":15453,"founder":15454,"Ġsearches":15455,"Ġrandomly":15456,"Access":15457,"Ġbatch":15458,"Ġposed":15459,"lat":15460,"Ġpursuing":15461,"asa":15462,"Ġtestified":15463,"forming":15464,"ĠShar":15465,"wiki":15466,"ĠEither":15467,"Sometimes":15468,"Ġsenators":15469,"ĠJohnny":15470,"ĠTaliban":15471,"ĠGPS":15472,"\":\"/":15473,"ãģ®å":15474,"Ġanalyzed":15475,"ĠRubio":15476,"ĠMovement":15477,"opard":15478,"iii":15479,"Stand":15480,"fight":15481,"Ġignoring":15482,"iang":15483,"ĠGN":15484,"soever":15485,"ĠSTAT":15486,"Ġrefusing":15487,"Ġsweat":15488,"Ġbay":15489,"PORT":15490,"irmed":15491,"aky":15492,"Ġdispro":15493,"Ġlabeled":15494,"Ġ108":15495,"Hello":15496,"Ġpleasant":15497,"aba":15498,"Ġtriumph":15499,"Ġaboard":15500,"Ġincom":15501,"ĠCrow":15502,"lett":15503,"Ġfolk":15504,"Ġchase":15505,"``":15506,"ĠBrus":15507,"Ġteens":15508,"cue":15509,"Ġterrain":15510,"hyd":15511,"ilight":15512,"ORY":15513,"Support":15514,"ews":15515,"lli":15516,"raints":15517,"ĠCand":15518,"Ġabused":15519,"achment":15520,"larg":15521,"Bas":15522,"ĠCancer":15523,"Ġ1978":15524,"Ġsupporter":15525,"access":15526,"ĠTermin":15527,"ĠTampa":15528,"ĠANY":15529,"Ġnewest":15530,"ĠCriminal":15531,"edu":15532,"Ġ1930":15533,"Ġadmits":15534,"Ġende":15535,"Ġfailures":15536,"urate":15537,"fulness":15538,"cycl":15539,"ĠSubject":15540,"Ġinfinite":15541,"three":15542,"WA":15543,"pit":15544,"ĠInstall":15545,"Rad":15546,"iliation":15547,"GM":15548,"Ġcontinent":15549,"Ġaccommodate":15550,"ĠClay":15551,"Ġpup":15552,"ĠFunction":15553,"Ġhammer":15554,"ĠAlberta":15555,"Ġrevised":15556,"Ġminorities":15557,"Ġmeasurement":15558,"Connell":15559,"Ġdisable":15560,"ĠMix":15561,"Incre":15562,"Ġfork":15563,"ĠRosen":15564,"Ġimplies":15565,"umblr":15566,"ANG":15567,"Ġproteins":15568,"Ġaggression":15569,"Ġfacilitate":15570,"SN":15571,"Ġillegally":15572,"uer":15573,"Ġacadem":15574,"Ġpuzz":15575,"ĠShift":15576,"pay":15577,"ollo":15578,"Ġaudiences":15579,"Build":15580,"Ġnoble":15581,"Ġsyntax":15582,"âĺħ":15583,"Ġbeam":15584,"ĠBed":15585,"ĠAld":15586,"Ġorigins":15587,"video":15588,"Ġ1977":15589,"ĠAssault":15590,"Ġgarage":15591,"Team":15592,"Ġverdict":15593,"Ġdwar":15594,"ĠVirtual":15595,"event":15596,"Keep":15597,"Ġsentiment":15598,"Ġwildlife":15599,"shirt":15600,"Ġburg":15601,"Ġrecommendation":15602,"represent":15603,"Ġgallery":15604,"owners":15605,"Ġscholar":15606,"Ġconvenience":15607,"ĠSwift":15608,"Ġconvinc":15609,"Cap":15610,"Ġwarfare":15611,"ĠVisual":15612,"Ġconstitute":15613,"Ġabort":15614,"ĠWeather":15615,"ĠLooking":15616,"ĠHem":15617,"Ġmartial":15618,"Ġincoming":15619,"etition":15620,"Ġtolerance":15621,"ĠCreated":15622,"Ġflows":15623,"ĠElder":15624,"Ġsouls":15625,"Ġfoul":15626,"ĠPain":15627,"ĠCAN":15628,"Ġ220":15629,"bc":15630,"hend":15631,"Ġgenius":15632,"Real":15633,"ĠWr":15634,"ometer":15635,"pad":15636,"Ġlimiting":15637,"ĠSi":15638,"ĠLore":15639,"ĠAdventures":15640,"Ġvaried":15641,"Disc":15642,"fin":15643,"ĠPersonal":15644,"Chris":15645,"Ġinvented":15646,"Ġdive":15647,"ĠRise":15648,"Ġoz":15649,"ĠComics":15650,"Ġexpose":15651,"ĠReb":15652,"letters":15653,"site":15654,"imated":15655,"Ġhacking":15656,"Ġeducated":15657,"ĠNobody":15658,"Ġdepri":15659,"Ġincentive":15660,"ãĤ·":15661,"Ġoversight":15662,"Ġtribes":15663,"ĠBelgium":15664,"Ġlicensing":15665,"ourt":15666,"Product":15667,"ahl":15668,"ĠGem":15669,"Ġspecialist":15670,"Ġcra":15671,"anners":15672,"ĠCorbyn":15673,"Ġ1973":15674,"READ":15675,"Ġsummar":15676,"Ġoverlook":15677,"ĠApplication":15678,"Ġinappropriate":15679,"Ġdownloaded":15680,"Que":15681,"ĠBears":15682,"Ġthumb":15683,"ĠCharacter":15684,"ĠReincarnated":15685,"ĠSid":15686,"Ġdemonstrates":15687,"sky":15688,"ĠBloomberg":15689,"ĠArray":15690,"ĠResults":15691,"ĠFourth":15692,"ĠEDT":15693,"ĠOscar":15694,"cend":15695,"Ġ106":15696,"ĠNULL":15697,"ĠHERE":15698,"match":15699,"ĠBrun":15700,"Ġglucose":15701,"ieg":15702,"egu":15703,"Ġcertified":15704,"Ġrelie":15705,"Ġhumanitarian":15706,"Ġprayers":15707,"King":15708,"Ġnan":15709,"hou":15710,"108":15711,"ulu":15712,"Ġrenewable":15713,"Ġdistinguish":15714,"Ġdense":15715,"ĠVent":15716,"ĠPackage":15717,"ĠBoss":15718,"Ġeditors":15719,"Ġmigr":15720,"Tra":15721,"ĠPeters":15722,"ĠArctic":15723,"2004":15724,"ĠCape":15725,"Ġlocally":15726,"Ġlasting":15727,"Ġhandy":15728,".).":15729,"Pan":15730,"ĠRES":15731,"Index":15732,"Ġtensions":15733,"Ġformerly":15734,"Ġideological":15735,"Ġsensors":15736,"Ġdealers":15737,"Ġdefines":15738,"Sk":15739,"Ġproceeds":15740,"Ġproxy":15741,"azines":15742,"ĠBash":15743,"ĠPad":15744,"ĠCraft":15745,"ealous":15746,"Ġsheets":15747,"ometry":15748,"June":15749,"clock":15750,"TT":15751,"ĠTheatre":15752,"ĠBuzz":15753,"Ġchapters":15754,"Ġmillenn":15755,"Ġdough":15756,"ĠCongressional":15757,"Ġimagined":15758,"avior":15759,"Ġclinic":15760,"Ġ1945":15761,"Ġholder":15762,"root":15763,"olester":15764,"Ġrestart":15765,"BN":15766,"ĠHamas":15767,"ĠJob":15768,"Ġorb":15769,"Ġram":15770,"Ġdisclose":15771,"Ġtranslate":15772,"Ġimmigrant":15773,"Ġannoying":15774,"Ġtreaty":15775,"anium":15776,"ĠTea":15777,"ĠLegion":15778,"Ġcrowds":15779,"ĠBec":15780,"ĠAer":15781,"ohyd":15782,"Bro":15783,"Looking":15784,"Ġlbs":15785,"Ġaggress":15786,"Ġseam":15787,"Ġintercept":15788,"ĠMI":15789,"mercial":15790,"activ":15791,"ĠCit":15792,"Ġdimension":15793,"Ġconsistency":15794,"Ġrushing":15795,"ĠDouglas":15796,"Ġtrim":15797,"Install":15798,"icker":15799,"Ġshy":15800,"106":15801,"Ġmentions":15802,"pelled":15803,"ĠTak":15804,"cost":15805,"Ġclassroom":15806,"Ġfortune":15807,"driven":15808,"Ġunle":15809,"ĠWheel":15810,"Ġinvestor":15811,"ĠMasters":15812,"kit":15813,"Ġassociations":15814,"ĠEvolution":15815,"oping":15816,"uscript":15817,"Ġprovincial":15818,"ĠWalter":15819,"avi":15820,"SO":15821,"Ġunlimited":15822,"English":15823,"ĠCards":15824,"ĠEbola":15825,"nered":15826,"Ġrevenge":15827,"Ġoutright":15828,"umper":15829,"Ġfitting":15830,"ĠSolid":15831,"Ġformally":15832,"Ġproblematic":15833,"Ġhazard":15834,"Ġencryption":15835,"Ġstraightforward":15836,"ĠAK":15837,"Ġpse":15838,"ĠOrb":15839,"ĠChamber":15840,"ĠMak":15841,"Contents":15842,"Ġloyalty":15843,"Ġlyrics":15844,"ĠSym":15845,"Ġwelcomed":15846,"Ġcooked":15847,"Ġmonop":15848,"Ġnurse":15849,"Ġmisleading":15850,"Ġeternal":15851,"Ġshifting":15852,"Ġ+=":15853,"Vis":15854,"Ġinstitutional":15855,"illary":15856,"Ġpant":15857,"VERT":15858,"ĠACC":15859,"ĠEnh":15860,"Ġincon":15861,"ĠREUTERS":15862,"Ġdonated":15863,"âĢ¦âĢ¦âĢ¦âĢ¦":15864,"Intern":15865,"Ġexhibit":15866,"Ġtire":15867,"ĠRic":15868,"ĠChampion":15869,"ĠMuhammad":15870,"NING":15871,"ĠSoccer":15872,"Ġmobility":15873,"Ġvarying":15874,"ĠMovie":15875,"Ġlord":15876,"oak":15877,"Field":15878,"Ġvector":15879,"usions":15880,"Ġscrap":15881,"Ġenabling":15882,"make":15883,"Tor":15884,".*":15885,"||":15886,"ĠWebsite":15887,"ĠNPC":15888,"Ġsocialist":15889,"ĠBilly":15890,"ĠAdditional":15891,"Ġcargo":15892,"Ġfarms":15893,"ĠSoon":15894,"ĠPrize":15895,"Ġmidnight":15896,"Ġ900":15897,"seen":15898,"ĠSpot":15899,"Ġsheep":15900,"Ġsponsored":15901,"ĠHi":15902,"ĠJump":15903,"Ġ1967":15904,"Microsoft":15905,"ĠAgent":15906,"Ġcharts":15907,"dir":15908,"Ġadjacent":15909,"Ġtricks":15910,"Ġmanga":15911,"Ġexagger":15912,"/>":15913,"football":15914,"ĠFCC":15915,"GC":15916,"ĠTier":15917,"andra":15918,"OUND":15919,"%),":15920,"Ġfruits":15921,"VC":15922,"ĠAA":15923,"Rober":15924,"Ġmidst":15925,"âĹ":15926,"anka":15927,"Ġlegislature":15928,"ĠNeil":15929,"Ġtourists":15930,"\"\"":15931,"ĠWarning":15932,"ĠNevertheless":15933,"ĠOfficial":15934,"ĠWhatever":15935,"Ġmold":15936,"Ġdrafted":15937,"Ġsubstances":15938,"Ġbreed":15939,"Ġtags":15940,"ĠTask":15941,"Ġverb":15942,"Ġmanufactured":15943,"comments":15944,"ĠPolish":15945,"Prov":15946,"Ġdetermines":15947,"Obama":15948,"kers":15949,"Ġutterly":15950,"Ġsect":15951,"sche":15952,"ĠGates":15953,"ĠChap":15954,"Ġaluminum":15955,"Ġzombie":15956,"ĠTouch":15957,"ĠUP":15958,"Ġsatisfy":15959,"Ġpredomin":15960,"ascript":15961,"Ġelaborate":15962,"Ġ1968":15963,"Ġmeasuring":15964,"ĠVari":15965,"anyahu":15966,"Ġsir":15967,"ulates":15968,"idges":15969,"ickets":15970,"ĠSpencer":15971,"TM":15972,"oubted":15973,"Ġprey":15974,"Ġinstalling":15975,"ĠCab":15976,"reed":15977,"reated":15978,"Supp":15979,"Ġwrist":15980,"ĠKerry":15981,"107":15982,"ĠKle":15983,"ĠRachel":15984,"Ġcotton":15985,"ĠARE":15986,"ĠEle":15987,"Control":15988,"Ġloads":15989,"ĠDod":15990,"anas":15991,"bone":15992,"Ġclassical":15993,"ĠRegional":15994,"ĠInteg":15995,"VM":15996,"Ġdesires":15997,"Ġautism":15998,"supported":15999,"ĠMessage":16000,"Ġcompact":16001,"writer":16002,"Ġ109":16003,"ĠHurricane":16004,"cision":16005,"Ġcycles":16006,"Ġdrill":16007,"Ġcolleague":16008,"Ġmaker":16009,"German":16010,"Ġmistaken":16011,"Sun":16012,"ĠGay":16013,"Ġwhatsoever":16014,"Ġsells":16015,"ĠAirl":16016,"liv":16017,"ĠOption":16018,"Ġsolved":16019,"Ġsectors":16020,"Ġhorizontal":16021,"Ġequation":16022,"ĠSkill":16023,"ĠBio":16024,"gement":16025,"ĠSnap":16026,"ĠLegal":16027,"Ġtrademark":16028,"Ġmakeup":16029,"Ġassembled":16030,"Ġsaves":16031,"ĠHalloween":16032,"ĠVermont":16033,"ĠFROM":16034,"Ġfarming":16035,"ĠPodcast":16036,"acceptable":16037,"ĠHigher":16038,"Ġasleep":16039,"ullivan":16040,"Ġreferen":16041,"ĠLev":16042,"Ġbullets":16043,"oko":16044,"HC":16045,"Ġstairs":16046,"Ġmaintains":16047,"ĠLower":16048,"ĠVi":16049,"Ġmarine":16050,"Ġacres":16051,"Ġcoordinator":16052,"ĠJoh":16053,"Ġcounterparts":16054,"ĠBrothers":16055,"Ġindict":16056,"bra":16057,"Ġchunk":16058,"Ġcents":16059,"Home":16060,"ĠMonth":16061,"Ġaccordingly":16062,"ifles":16063,"ĠGermans":16064,"ĠSyn":16065,"Hub":16066,"Ġeyeb":16067,"âĶĢâĶĢâĶĢâĶĢ":16068,"Ġranges":16069,"ĠHolland":16070,"ĠRobot":16071,"fc":16072,"Mike":16073,"Ġplasma":16074,"Ġswap":16075,"Ġathlete":16076,"ĠRams":16077,",'\"":16078,"Ġinfections":16079,"Ġcorrid":16080,"Ġvib":16081,"Ġpatches":16082,"Ġtraditionally":16083,"Ġrevelation":16084,"Ġsweep":16085,"Ġglance":16086,"Ġinex":16087,"2003":16088,"ĠRaw":16089,"working":16090,"osures":16091,"ĠDat":16092,"ĠLynch":16093,"Ġleverage":16094,"ĠReid":16095,"Ġcorrelation":16096,"iances":16097,"avascript":16098,"Ġrepository":16099,"retty":16100,"Ġ1972":16101,"240":16102,"Ġoun":16103,"pol":16104,"ĠReed":16105,"Ġtactical":16106,"isite":16107,"Apple":16108,"ĠQuinn":16109,"Ġraped":16110,"illo":16111,"Europe":16112,"Ġalgorithms":16113,"ĠRodrig":16114,"iu":16115,"Ġillum":16116,"Ġfame":16117,"Ġintroducing":16118,"Ġdelays":16119,"ĠRaiders":16120,"Ġwhistle":16121,"Ġnovels":16122,"ĠReally":16123,"Ġderiv":16124,"Ġpublications":16125,"ĠNeither":16126,"ĠCommerce":16127,"Ġaston":16128,"language":16129,"Notes":16130,"ĠRoth":16131,"ĠFear":16132,"Ġmate":16133,"Ġparade":16134,"ĠQB":16135,"Ġmaneu":16136,"ĠCincinnati":16137,"mitting":16138,"Ġwaist":16139,"ĠRew":16140,"Ġdiscont":16141,"а":16142,"Ġstaring":16143,"Ġalias":16144,"Ġsecurities":16145,"Ġtoilet":16146,"ĠJedi":16147,"Ġunlaw":16148,"vised":16149,"////////":16150,"](":16151,"ĠWeiss":16152,"Ġprest":16153,"ĠCompan":16154,"Ġmemo":16155,"ĠGrace":16156,"July":16157,"ĠElite":16158,"center":16159,"ĠStay":16160,"Ġgalaxy":16161,"Ġtooth":16162,"ĠSettings":16163,"Ġsubjected":16164,"ãĤ¦":16165,"Ġlineback":16166,"Ġretailers":16167,"ĠWant":16168,"Ġdangers":16169,"Air":16170,"Ġvoluntary":16171,"eway":16172,"Ġinterpreted":16173,"otine":16174,"ç":16175,"Ġpel":16176,"Service":16177,"ĠEventually":16178,"Ġcareers":16179,"Ġthreaten":16180,"Ġmemor":16181,"ĠBradley":16182,"ancies":16183,"sn":16184,"ĠUnknown":16185,"National":16186,"Ġshadows":16187,"ailand":16188,"ĠDash":16189,"Everyone":16190,"izzard":16191,"March":16192,"=(":16193,"Ġpulls":16194,"Ġstranger":16195,"Ġbackwards":16196,"ĠBernard":16197,"imensional":16198,"Ġchron":16199,"Ġtheoretical":16200,"ktop":16201,"Ġware":16202,"ĠInvestig":16203,"ĠIniti":16204,"ĠOperations":16205,"oven":16206,"ocide":16207,"*/":16208,"Ġflames":16209,"ĠCash":16210,"shit":16211,"Ġcab":16212,"ĠAnaly":16213,"ĠSeah":16214,"Ġdefining":16215,"Ġordering":16216,"Ġimmun":16217,"Ġpersistent":16218,"ACH":16219,"Russian":16220,"mans":16221,"Ġhind":16222,"Ġphotography":16223,"©":16224,"Ġhug":16225,"Ġ107":16226,"ĠHence":16227,"iots":16228,"udeau":16229,"Ġsubsidies":16230,"Ġroutinely":16231,"ĠDevice":16232,"itic":16233,"Ġdisgust":16234,"lander":16235,"Ġ1940":16236,"Ġassignment":16237,"ĠBesides":16238,"wick":16239,"ĠDust":16240,"usc":16241,"structed":16242,"111":16243,"develop":16244,"Ġfond":16245,"Ġintersection":16246,"Ġdignity":16247,"Ġcommissioner":16248,"Without":16249,"reach":16250,"Ġcartoon":16251,"Ġscales":16252,"ãĥŃ":16253,"FIG":16254,"Ġsurveys":16255,"ĠIndonesia":16256,"Ġartwork":16257,"Ġunch":16258,"Ġcycling":16259,"unct":16260,"auer":16261,"orate":16262,"ĠObviously":16263,"Ġcharacterized":16264,"feld":16265,"Ġaffirm":16266,"Ġinnings":16267,"Ġé":16268,"Ġaliens":16269,"Ġcloth":16270,"etooth":16271,"ĠCertain":16272,"§":16273,"Ġdigest":16274,"know":16275,"ĠXL":16276,"Ġpredictions":16277,"Ġdin":16278,"WAR":16279,"Ġaftermath":16280,"Example":16281,"ĠSuccess":16282,"ĠThr":16283,"IGN":16284,"Ġminer":16285,"Bus":16286,"Ġclarity":16287,"heimer":16288,"ĠOUT":16289,"ĠSend":16290,"ĠCircle":16291,"ĠDiet":16292,"Ġpronounced":16293,"Ġcreators":16294,"Ġearthquake":16295,"attery":16296,"geons":16297,"Ġod":16298,"Ġlaying":16299,"orp":16300,"Ult":16301,"project":16302,"Ġundermin":16303,"Ġsequel":16304,"Sam":16305,"ĠDarkness":16306,"Ġreception":16307,"bull":16308,"YS":16309,"ĠVir":16310,"Ġsequences":16311,"ĠCoin":16312,"Ġoutfit":16313,"ĠWait":16314,"119":16315,"Ġdelivers":16316,"......":16317,"Ġblown":16318,"ĠEsc":16319,"ĠMath":16320,"perm":16321,"ĠUl":16322,"Ġglim":16323,"Ġfacial":16324,"Ġgreenhouse":16325,"Ġtokens":16326,"/-":16327,"ĠAnnual":16328,"ĠONE":16329,"Ġteenage":16330,"ĠPhysical":16331,"ĠLang":16332,"ĠCelt":16333,"Ġsued":16334,"ividually":16335,"Ġpatience":16336,"chair":16337,"regular":16338,"Ġaug":16339,"inv":16340,"except":16341,"ĠLil":16342,"Ġnest":16343,"fd":16344,"sum":16345,"ĠChase":16346,"Russia":16347,"ĠJennifer":16348,"Ġoffseason":16349,"Overall":16350,"Fore":16351,"Ġriot":16352,"Aud":16353,"former":16354,"Ġdefenders":16355,"ĠCT":16356,"iotic":16357,"ribly":16358,"Ġautomated":16359,"Ġpenis":16360,"Ġinsist":16361,"Ġdiagram":16362,"ĠSQL":16363,"ĠGarc":16364,"Ġwitch":16365,"client":16366,"ierra":16367,"ambers":16368,"Ġrecount":16369,"far":16370,"Very":16371,"osterone":16372,"Ġappreciated":16373,"ĠPerfect":16374,"Section":16375,"Ġdoses":16376,"ocaust":16377,"Ġcostly":16378,"Ġgrams":16379,"ĠShi":16380,"Ġwrestling":16381,"Ġ1971":16382,"Ġtrophy":16383,"Ġnerve":16384,"ĠKaz":16385,"ĠExperience":16386,"Ġpledged":16387,"Ġplayback":16388,"Ġcreativity":16389,"bye":16390,"Ġattackers":16391,"Ġholders":16392,"ĠCoach":16393,"ĠPhD":16394,"Ġtransfers":16395,"Ġcolored":16396,"ĠHindu":16397,"Ġdrown":16398,"Ġlistened":16399,"ĠWA":16400,"iasm":16401,"PO":16402,"Ġappealing":16403,"Ġdisclosed":16404,"ĠChicken":16405,"agging":16406,"Ġpleaded":16407,"Ġnavigation":16408,"ĠReturns":16409,"Ġ[[":16410,"ROR":16411,"EA":16412,"Ġphotographer":16413,"ĠRider":16414,"ippers":16415,"Ġslice":16416,"Ġerect":16417,"Ġhed":16418,"issance":16419,"ĠVikings":16420,"urious":16421,"Ġappet":16422,"oubtedly":16423,"Child":16424,"Ġauthentic":16425,"oos":16426,"ĠMaking":16427,"Ġannouncing":16428,"Ġbod":16429,"Ġmeter":16430,"ĠNine":16431,"ĠRogue":16432,"Ġworkforce":16433,"Ġrenewed":16434,"Ġorganisations":16435,"acs":16436,"PLE":16437,"Short":16438,"Ġcompounds":16439,"ĠVisit":16440,"Ġenvelop":16441,"earth":16442,"Ġsupportive":16443,"ggle":16444,"ĠBrussels":16445,"ĠGuild":16446,"Create":16447,"REL":16448,"Ġaveraged":16449,"Ġ1969":16450,"riages":16451,"Ġlengthy":16452,"Ġforgot":16453,"Okay":16454,"ĠErd":16455,"Ġdealer":16456,"Ġrecession":16457,"DD":16458,"Ġdesperately":16459,"Ġhunger":16460,"Ġsticks":16461,"Ġmph":16462,"ĠFaith":16463,"Ġintentionally":16464,"Ġdemol":16465,"ueller":16466,"ĠSale":16467,"Ġdebris":16468,"spring":16469,"Ġleap":16470,">>>>":16471,"Ġcontainers":16472,"selling":16473,"ranean":16474,"attering":16475,"Ġcommented":16476,"ĠCM":16477,"onut":16478,"Ġwoods":16479,"especially":16480,"Ġorganize":16481,"ivic":16482,"ĠWoods":16483,"anga":16484,"squ":16485,"Ġmaj":16486,"amon":16487,"Ġaxis":16488,"Ġ1974":16489,"ĠDenmark":16490,"Ġwarrior":16491,"ĠPand":16492,"Ġoutlined":16493,"ĠBO":16494,"insula":16495,"zilla":16496,"ebook":16497,"Ġdare":16498,"Ġsearched":16499,"Ġnavigate":16500,"Sn":16501,"writing":16502,"Ġunited":16503,"Japan":16504,"ĠHebrew":16505,"Ġflame":16506,"Ġrelies":16507,"Ġcatching":16508,"ĠSho":16509,"Ġimprisonment":16510,"Ġpockets":16511,"Ġclosure":16512,"ĠFam":16513,"tim":16514,"adequ":16515,"Activity":16516,"Ġrecruiting":16517,"ĠWATCH":16518,"ĠArgentina":16519,"dest":16520,"Ġapologize":16521,"oro":16522,"Ġlacks":16523,"Ġtuned":16524,"ĠGriffin":16525,"Ġinfamous":16526,"Ġcelebrity":16527,"sson":16528,"Ġ----------------------------------------------------------------":16529,"ĠIsis":16530,"ĠDisplay":16531,"Ġcredibility":16532,"Ġeconomies":16533,"Ġheadline":16534,"ĠCowboys":16535,"Ġindef":16536,"Ġlately":16537,"Ġincentives":16538,"button":16539,"ĠMob":16540,"Aut":16541,"Ġresigned":16542,"ĠOm":16543,"camp":16544,"Ġprofiles":16545,"Ġschemes":16546,"olphins":16547,"ayed":16548,"Clinton":16549,"enh":16550,"ĠYahoo":16551,"Ġabst":16552,"Ġank":16553,"suits":16554,"Ġwished":16555,"ĠMarco":16556,"udden":16557,"Ġsphere":16558,"ĠBishop":16559,"Ġincorporated":16560,"ĠPlant":16561,"114":16562,"Ġhated":16563,"pic":16564,"Ġdonate":16565,"Ġlined":16566,"Ġbeans":16567,"Ġstealing":16568,"Ġcostume":16569,"Ġsheriff":16570,"Ġforty":16571,"Ġintact":16572,"Ġadapted":16573,"Ġtravelling":16574,"bart":16575,"Ġnicely":16576,"Ġdried":16577,"Ġscal":16578,"osity":16579,"NOTE":16580,"ĠBh":16581,"ĠBroncos":16582,"ĠIgn":16583,"Ġintimate":16584,"Ġchemistry":16585,"Ġoptimal":16586,"Deb":16587,"ĠGeneration":16588,"Ġ],":16589,"ichi":16590,"ĠWii":16591,"ĠYOUR":16592,"ventions":16593,"Write":16594,"Ġpopul":16595,"unning":16596,"ĠWor":16597,"Vol":16598,"Ġqueen":16599,"heads":16600,"KK":16601,"Ġanalyze":16602,"opic":16603,"earchers":16604,"Ġdot":16605,"legraph":16606,"astically":16607,"Ġupgrades":16608,"Ġcares":16609,"Ġextending":16610,"Ġfreeze":16611,"Ġinability":16612,"Ġorgans":16613,"Ġpretend":16614,"Ġoutlet":16615,"113":16616,"olan":16617,"ĠMall":16618,"uling":16619,"talk":16620,"Ġexpressing":16621,"ĠAlways":16622,"ĠBegin":16623,"files":16624,"Ġlicenses":16625,"%%":16626,"ĠMitt":16627,"Ġfilters":16628,"ĠMilwaukee":16629,"GN":16630,"Ġunfold":16631,"Mo":16632,"Ġnutrition":16633,"ppo":16634,"Bo":16635,"Ġfounding":16636,"Ġundermine":16637,"Ġeasiest":16638,"ĠCzech":16639,"ĠMack":16640,"Ġsexuality":16641,"ĠNixon":16642,"Win":16643,"ĠArn":16644,"ĠKin":16645,"ãĤ£":16646,"icer":16647,"Ġfortun":16648,"Ġsurfaces":16649,"aghd":16650,"Ġcarriers":16651,"ĠPART":16652,"ĠTib":16653,"Ġinterval":16654,"Ġfrustrating":16655,"ĠShip":16656,"ĠArmed":16657,"ffe":16658,"Ġboats":16659,"ĠAbraham":16660,"inis":16661,"Ġsuited":16662,"thread":16663,"iov":16664,"abul":16665,"ĠVenezuela":16666,"Ġtom":16667,"super":16668,"Ġcastle":16669,"although":16670,"ioxide":16671,"eches":16672,"Ġevolutionary":16673,"Ġnegotiate":16674,"Ġconfronted":16675,"Remember":16676,"Ġ170":16677,"Such":16678,"Ġ911":16679,"mult":16680,"ĠAbyss":16681,"urry":16682,"kees":16683,"spec":16684,"ĠBarbara":16685,"Ġbelonging":16686,"Ġvillain":16687,"istani":16688,"Ġaccountable":16689,"Ġportions":16690,"ĠDecl":16691,"Ur":16692,"ĠKate":16693,"gre":16694,"Ġmagazines":16695,"UCK":16696,"Ġregulate":16697,"omon":16698,"ĠAlmost":16699,"Ġoverview":16700,"Ġscram":16701,"Ġloot":16702,"ĠFitz":16703,"Ġcharacteristic":16704,"ĠSnake":16705,"say":16706,"ĠRico":16707,"Ġtrait":16708,"ĠJoined":16709,"aucus":16710,"Ġadaptation":16711,"ĠAirlines":16712,"Ġarchae":16713,"ĠIde":16714,"Ġbikes":16715,"Ġliterary":16716,"Ġinfluences":16717,"ĠUsed":16718,"Creat":16719,"Ġplea":16720,"ĠDefence":16721,"ĠAssass":16722,"Ġpond":16723,"ULT":16724,")\"":16725,"Ġevaluated":16726,"Ġobtaining":16727,"Ġdemographic":16728,"Ġvigil":16729,"aley":16730,"Ġspouse":16731,"ĠSeahawks":16732,"respons":16733,"ĠBelt":16734,"umatic":16735,"Ġrises":16736,"runner":16737,"ĠMichelle":16738,"Ġpotent":16739,"race":16740,"ĠPAC":16741,"Find":16742,"olesterol":16743,"ISS":16744,"ĠIntroduced":16745,"resses":16746,"ignment":16747,"Os":16748,"ĠTu":16749,"ĠDex":16750,"icides":16751,"Ġsparked":16752,"ĠLaura":16753,"ĠBryant":16754,"Ġsmiling":16755,"ĠNexus":16756,"Ġdefendants":16757,"ĠCatal":16758,"Ġdishes":16759,"shaped":16760,"Ġprolong":16761,"mt":16762,"($":16763,"ãĢĤ":16764,"Ġcalculations":16765,"ĠSame":16766,"Ġpiv":16767,"HH":16768,"Ġcancelled":16769,"Ġgrin":16770,"Ġterritories":16771,"istically":16772,"Come":16773,"ĠParent":16774,"Project":16775,"Ġneglig":16776,"ĠPrivacy":16777,"Ġammo":16778,"LECT":16779,"olutely":16780,"ĠEpic":16781,"Ġmisunder":16782,"wal":16783,"April":16784,"mos":16785,"pathy":16786,"ĠCarson":16787,"Ġalbums":16788,"ĠEasy":16789,"Ġpistol":16790,"<<":16791,"Ġ\\(":16792,"target":16793,"help":16794,"Ġinterpre":16795,"conscious":16796,"ĠHousing":16797,"ĠJoint":16798,"127":16799,"Ġbeers":16800,"science":16801,"ĠFirefox":16802,"effective":16803,"ĠCabin":16804,"ĠOkay":16805,"ĠApplic":16806,"Ġspacecraft":16807,"ĠSR":16808,"vet":16809,"ĠStrange":16810,"SB":16811,"Ġcorps":16812,"iberal":16813,"efficient":16814,"Ġprevalence":16815,"Ġeconomists":16816,"118":16817,"Thread":16818,"ordable":16819,"ODE":16820,"ĠCant":16821,"=-=-":16822,"ifiable":16823,"ĠAround":16824,"Ġpole":16825,"Ġwillingness":16826,"CLA":16827,"ĠKid":16828,"Ġcomplement":16829,"Ġscattered":16830,"Ġinmates":16831,"Ġbleeding":16832,"every":16833,"Ġqueue":16834,"ĠTrain":16835,"Ġhij":16836,"Ġmelee":16837,"pleted":16838,"Ġdigit":16839,"Ġgem":16840,"official":16841,"Ġlifting":16842,"е":16843,"Requ":16844,"itutes":16845,"Ġpackaging":16846,"ĠWorkers":16847,"hran":16848,"ĠLebanon":16849,"olesc":16850,"Ġpunished":16851,"ĠJuan":16852,"Ġjam":16853,"ĠDocument":16854,"Ġmapping":16855,"icates":16856,"Ġinevitably":16857,"Ġvanilla":16858,"ĠTon":16859,"Ġwatches":16860,"Ġleagues":16861,"Ġinitiated":16862,"degree":16863,"portion":16864,"Ġrecalls":16865,"Ġruin":16866,"Ġmelt":16867,"IAN":16868,"Ġhem":16869,"Exp":16870,"Ġbaking":16871,"ĠColomb":16872,"atible":16873,"Ġradius":16874,"plug":16875,"ĠIF":16876,"etically":16877,"Ġfict":16878,"HER":16879,"ĠTap":16880,"atinum":16881,"Ġink":16882,"Ġcoh":16883,"ĠWizard":16884,"both":16885,"tex":16886,"Ġspends":16887,"ĠCurrently":16888,"ĠPit":16889,"Ġneurons":16890,"ignt":16891,"Ġrall":16892,"Ġbuses":16893,"building":16894,"Ġadjustments":16895,"Ġcried":16896,"iblical":16897,"atted":16898,"ĠZion":16899,"ĠMatter":16900,"Ġmeditation":16901,"ĠDennis":16902,"Ġours":16903,"ĠTab":16904,"Ġrankings":16905,"ortal":16906,"Ġadvers":16907,"Ġsurrender":16908,"ĠGob":16909,"cium":16910,"omas":16911,"imeter":16912,"Ġmultiplayer":16913,"Ġheroin":16914,"Ġoptimistic":16915,"Ġindicator":16916,"ĠBrig":16917,"Ġgrocery":16918,"Ġapplicant":16919,"ĠRocket":16920,"vid":16921,"Exception":16922,"pent":16923,"Ġorganizing":16924,"Ġencounters":16925,"ĠTOD":16926,"Ġjewel":16927,"Save":16928,"ĠChristie":16929,"Ġheating":16930,"Ġlazy":16931,"ĠCP":16932,"Ġcousin":16933,"Config":16934,"Ġregener":16935,"Ġnearest":16936,"Ġachieving":16937,"ENS":16938,"throw":16939,"ĠRichmond":16940,"antle":16941,"2002":16942,"Ġanten":16943,"bird":16944,"133":16945,"Ġnarc":16946,"raint":16947,"unny":16948,"ĠHispanic":16949,"ournaments":16950,"Ġprophe":16951,"ĠThailand":16952,"ĠTi":16953,"Ġinjection":16954,"Ġinherit":16955,"ravis":16956,"Ġmedi":16957,"Ġwhoever":16958,"ĠDEBUG":16959,"GP":16960,"ĠHud":16961,"Card":16962,"prom":16963,"Ġpor":16964,"Ġoverhead":16965,"Law":16966,"Ġviolate":16967,"Ġheated":16968,"Ġdescriptions":16969,"Ġachievements":16970,"ĠBeer":16971,"ĠQuant":16972,"Was":16973,"Ġeighth":16974,"ĠIv":16975,"Ġspecialized":16976,"UPDATE":16977,"ĠDelta":16978,"Pop":16979,"Jul":16980,"ĠAsk":16981,"ophy":16982,"Ġnewsletters":16983,"ĠTool":16984,"Ġgard":16985,"ĠConfeder":16986,"ĠGMT":16987,"ĠAbbott":16988,"Ġimmunity":16989,"ĠVM":16990,"Islam":16991,"Ġimplicit":16992,"wd":16993,"Ġ1944":16994,"ravity":16995,"ometric":16996,"Ġsurviving":16997,"urai":16998,"ĠPrison":16999,"Ġrust":17000,"ĠSketch":17001,"Ġbees":17002,"ĠTheory":17003,"Ġmerit":17004,"Tex":17005,"chat":17006,"Ġmim":17007,"Ġpaste":17008,"ĠKoch":17009,"Ġignorance":17010,"ĠShoot":17011,"Ġbasement":17012,"United":17013,"ĠAdvis":17014,"height":17015,"Ġfoster":17016,"Ġdetain":17017,"information":17018,"Ġneural":17019,"';":17020,"Ġproves":17021,"allery":17022,"Ġinvitation":17023,"umbers":17024,"Ġcattle":17025,"Ġbicycle":17026,"zi":17027,"Ġconsultant":17028,"Ġapology":17029,"ĠTiger":17030,"Ġ123":17031,"999":17032,"Ġindividually":17033,"rt":17034,"igion":17035,"ĠBrazilian":17036,"Ġdisturb":17037,"Ġentrepreneurs":17038,"Ġforests":17039,"cerpt":17040,"plates":17041,"pher":17042,"clipse":17043,"Ġtwitter":17044,"Ġacids":17045,"ographical":17046,"hum":17047,"ĠBald":17048,"ifully":17049,"Ġcompiler":17050,"ĠDA":17051,"Ġdonor":17052,"asi":17053,"Ġtribal":17054,"lash":17055,"ĠConfig":17056,"Ġapplicants":17057,"Ġsalaries":17058,"135":17059,"Putin":17060,"ĠFocus":17061,"irs":17062,"Ġmisconduct":17063,"ĠHaz":17064,"Ġeaten":17065,"Mobile":17066,"Muslim":17067,"ĠMarcus":17068,"viol":17069,"Ġfavorable":17070,"Ġstub":17071,"adin":17072,"ĠHob":17073,"Ġfaithful":17074,"Ġelectronics":17075,"Ġvacuum":17076,"wait":17077,"backed":17078,"economic":17079,"dist":17080,"Ġtenure":17081,"Ġsincere":17082,"ĠTogether":17083,"ĠWave":17084,"Ġprogression":17085,"Ġdenying":17086,"Ġdistress":17087,"braska":17088,"third":17089,"Ġmixing":17090,"Ġcolonial":17091,"Ġprivately":17092,"Ġunrest":17093,"aternity":17094,"Ġpremises":17095,"anti":17096,"gregation":17097,"Ġlicence":17098,"ĠHind":17099,"ĠSamuel":17100,"Ġconvincing":17101,"ĠAce":17102,"ĠRust":17103,"ĠNetanyahu":17104,"Ġhandles":17105,"ĠPatch":17106,"oriented":17107,"aho":17108,"ĠGonz":17109,"Ġhackers":17110,"claimer":17111,"Ġcustoms":17112,"ĠGran":17113,"fighters":17114,"Ġluc":17115,"Ġmanuscript":17116,"arenthood":17117,"Ġdevil":17118,"Ġwarriors":17119,"Ġoffenders":17120,"William":17121,"Ġholidays":17122,"Ġnightmare":17123,"Ġlever":17124,"ifferent":17125,"Stat":17126,"Ġexhibition":17127,"puted":17128,"ĠPure":17129,"Ġalpha":17130,"Ġenthusiasm":17131,"ĠRepresentatives":17132,"EAR":17133,"ĠTyp":17134,"Ġwheat":17135,"ĠAlf":17136,"Ġcorrection":17137,"Ġevangel":17138,"ATT":17139,"Miss":17140,"Ġsoup":17141,"Ġimplied":17142,"param":17143,"Ġsexy":17144,"ĠLux":17145,"Ġrepublic":17146,"patch":17147,"ablish":17148,"Ġicons":17149,"Ġfathers":17150,"ĠGET":17151,"ĠCarib":17152,"Ġregulated":17153,"ĠCohen":17154,"ĠBobby":17155,"Ġner":17156,"Ġbent":17157,"ventory":17158,"ĠAlong":17159,"ĠEST":17160,"ĠWallace":17161,"Ġmurders":17162,"rise":17163,"kell":17164,"ĠCommonwealth":17165,"Ġnasty":17166,"eta":17167,"ĠMIT":17168,"Ġadministered":17169,"Ġgenuinely":17170,"Editor":17171,"nick":17172,"Ġhydro":17173,"********************************":17174,"ĠBle":17175,"Ġfines":17176,"Ġgorge":17177,"ausible":17178,"rh":17179,"Ġapple":17180,"mentioned":17181,"Ġrope":17182,"otyp":17183,"HR":17184,"Ġdisappointing":17185,"Ġcage":17186,"nik":17187,"Ġdoubts":17188,"ĠFREE":17189,"prints":17190,"ĠMUST":17191,"Ġvendors":17192,"ĠInqu":17193,"Ġliberals":17194,"Ġcontractor":17195,"Ġupside":17196,"children":17197,"Ġtricky":17198,"Ġregulators":17199,"charged":17200,"liter":17201,"Ġ***":17202,"Ġrebell":17203,"lang":17204,"Ġlocals":17205,"Ġphysicians":17206,"Ġhey":17207,"arse":17208,"tm":17209,"ĠLex":17210,"Ġbehavioral":17211,"successful":17212,"FX":17213,"Ġbrick":17214,"ovic":17215,"Ġconform":17216,"Ġreviewing":17217,"Ġinsights":17218,"Ġbiology":17219,"ĠRemove":17220,"ĠExtra":17221,"Ġcommitting":17222,"induced":17223,"ignty":17224,"igm":17225,"Ġatomic":17226,"Common":17227,"ĠEM":17228,"ĠPere":17229,"ĠItems":17230,"eh":17231,"Ġpreserved":17232,"ĠHood":17233,"Ġprisoner":17234,"Ġbankruptcy":17235,"Ġgren":17236,"ushes":17237,"Ġexploitation":17238,"Ġsignatures":17239,"Ġfinan":17240,"],\"":17241,"ĠMR":17242,"Ġmeg":17243,"remlin":17244,"Ġmusicians":17245,"Ġselecting":17246,"Ġexamining":17247,"INK":17248,"lated":17249,"Hi":17250,"Ġartic":17251,"Ġpets":17252,"Ġimpair":17253,"ĠMAN":17254,"Ġtablets":17255,"include":17256,"Range":17257,"Ġcaut":17258,"Ġlogs":17259,"Ġmounting":17260,"Ġunaware":17261,"Ġdynamics":17262,"ĠPalestine":17263,"ĠQuarter":17264,"ĠPurple":17265,"Ġma":17266,"ĠImport":17267,"Ġcollections":17268,"ciation":17269,"Ġsuccessor":17270,"Ġclone":17271,"Ġaiming":17272,"Ġpossessed":17273,"Ġsticking":17274,"Ġshaking":17275,"Ġlocate":17276,"ĠHockey":17277,"Turn":17278,"170":17279,"Ġfifteen":17280,"ĠHarrison":17281,"Ġcontinuously":17282,"ĠTC":17283,"ĠValent":17284,"ĠRescue":17285,"Ġbypass":17286,"amount":17287,"Ġmast":17288,"Ġprotects":17289,"Ġartistic":17290,"Ġsometime":17291,"Ġshoe":17292,"Ġshouted":17293,"ificant":17294,"etitive":17295,"ĠRegister":17296,"ĠJin":17297,"Ġconcentrated":17298,"lington":17299,"onies":17300,"Ġgenerator":17301,"yrim":17302,"ĠArmen":17303,"Ġclearing":17304,"ido":17305,"ĠTW":17306,"alph":17307,"Ġladies":17308,"Hard":17309,"Ġdialog":17310,"Ġinputs":17311,"æľ":17312,"Ġposes":17313,"Ġslots":17314,"ĠPremium":17315,"Ġleaks":17316,"Ġbosses":17317,"Ġ113":17318,"course":17319,"Acc":17320,"ĠNewton":17321,"ĠAustria":17322,"ĠMage":17323,"Ġteaches":17324,"abad":17325,"Ġwears":17326,"Ġcyl":17327,"Ġcurse":17328,"ĠSales":17329,"ĠWings":17330,"Ġpsy":17331,"Ġgaps":17332,"ĠIceland":17333,"ĠPinterest":17334,"Ġlandlord":17335,"Ġdefinitions":17336,"ĠKer":17337,"Ġsufficiently":17338,"ĠPence":17339,"ĠArchitect":17340,"Ġsurpass":17341,"Ġ114":17342,"Ġsuperhero":17343,"ĠDisease":17344,"Ġpriests":17345,"ĠCulture":17346,"Ġdefinitive":17347,"Ġsecretly":17348,"ĠDance":17349,"install":17350,"chief":17351,"ĠJessica":17352,"Would":17353,"Updated":17354,"Ġlocker":17355,"ĠKay":17356,"Ġmemorial":17357,"è¦":17358,"fat":17359,"Ġdisgu":17360,"Ġflavors":17361,"ĠBaseball":17362,"ĠResistance":17363,"Ġkicks":17364,"Ġenv":17365,"Ġteenagers":17366,"Dark":17367,"ĠCAR":17368,"Ġhalt":17369,"ĠLG":17370,"ĠGabriel":17371,"Ġfever":17372,"Ġsatur":17373,"Ġmall":17374,"Ġaffiliate":17375,"ĠSleep":17376,"ĠSpecific":17377,"ĠVel":17378,"Ġjar":17379,"ĠSacred":17380,"ĠEdwards":17381,"ĠACL":17382,"Ġretained":17383,"ĠGiant":17384,"Ġlimitation":17385,"inces":17386,"Ġrefusal":17387,"ĠTale":17388,"ĠButler":17389,"Ġaccidents":17390,"ĠCSS":17391,"Ġimported":17392,"ĠCopy":17393,"α":17394,"ERT":17395,"zel":17396,"Ġdivisions":17397,"hots":17398,"ĠAlb":17399,"ĠDS":17400,"Loader":17401,"Washington":17402,"atisf":17403,"ĠCreative":17404,"\\.":17405,"ĠAutom":17406,"redict":17407,"Ġreceptor":17408,"ĠCarlos":17409,"Method":17410,"oka":17411,"Ġmalicious":17412,"Ġstepping":17413,",[":17414,"ĠDad":17415,"Ġattraction":17416,"ĠEffects":17417,"ĠPirate":17418,"ĠCer":17419,"ĠIndustry":17420,"ĠRud":17421,"Ġcharter":17422,"Ġdining":17423,"Ġinsists":17424,"Ġconfigure":17425,"Ġ(#":17426,"ĠSimple":17427,"ĠScroll":17428,"UTC":17429,"175":17430,"ĠKon":17431,"Ġmarketplace":17432,"ĠãĤ":17433,"Ġrefres":17434,"Ġgates":17435,"erred":17436,"ĠPod":17437,"Ġbehave":17438,"Frank":17439,"node":17440,"Ġendorsed":17441,"hett":17442,"asive":17443,"ĠHomeland":17444,"Ġrides":17445,"ĠLeave":17446,"erness":17447,"Ġflooding":17448,"AFP":17449,"Ġrisen":17450,"Ġcontinually":17451,"Ġunanim":17452,"ĠContract":17453,"ĠPas":17454,"Ġguided":17455,"ĠChile":17456,"bd":17457,"Ġsucc":17458,"ptic":17459,"Ġcommittees":17460,"ĠLuther":17461,"ĠAnyone":17462,"Ġsab":17463,"124":17464,"Ġpixel":17465,"ĠBak":17466,"ĠTag":17467,"ĠBennett":17468,"Enter":17469,"small":17470,"ĠPresidential":17471,"Ġpul":17472,"Ġcontrace":17473,"archive":17474,"Ġcoastal":17475,"ĠKids":17476,"192":17477,"âĢ²":17478,"icky":17479,"INGTON":17480,"Ġwolf":17481,"ĠStalin":17482,"Tur":17483,"idget":17484,"amas":17485,"ĠUnless":17486,"Ġsponsor":17487,"Ġmorph":17488,"ĠChoose":17489,"Ġrunner":17490,"Ġunbel":17491,"Ġmud":17492,"ĠMana":17493,"Ġdubbed":17494,"Ġgodd":17495,"urers":17496,"window":17497,"Ġrelied":17498,"Ġcelebrating":17499,"osc":17500,"Ġ135":17501,"Ġlobbying":17502,"Ġincomplete":17503,"Ġrestriction":17504,"Ġincap":17505,"itus":17506,"Ġexpectation":17507,"ĠApollo":17508,"Ġintens":17509,"Ġsync":17510,"GH":17511,"Ġmanipulation":17512,"BY":17513,"Ġspear":17514,"Ġbreasts":17515,"Ġvolcan":17516,"ilia":17517,"Material":17518,"Ġformats":17519,"ĠBast":17520,"Ġparliamentary":17521,"Ġsnake":17522,"Ġservants":17523,"ĠTrudeau":17524,"ĠGrim":17525,"ĠArabic":17526,"ĠSCP":17527,"ĠBoys":17528,"station":17529,"Ġprospective":17530,"orde":17531,"initialized":17532,"Ġbored":17533,"ABLE":17534,"Ġaccessed":17535,"Ġtaxi":17536,"ĠShell":17537,"aiden":17538,"ursed":17539,"inates":17540,"ĠInsurance":17541,"ĠPete":17542,"September":17543,"650":17544,"Ġadventures":17545,"ĠCover":17546,"Ġtribute":17547,"Ġsketch":17548,"Ġempower":17549,"ĠØ":17550,"ĠGlenn":17551,"ĠDaw":17552,"=\\\"":17553,"ĠPolitics":17554,"Ġguides":17555,"Ġdioxide":17556,"ĠGore":17557,"ĠBright":17558,"ĠSierra":17559,"Ġvalued":17560,"cond":17561,"Ġpointer":17562,"Select":17563,"Ġrisky":17564,"Ġabsorb":17565,"images":17566,"Ġrefuses":17567,"Ġbonuses":17568,"___":17569,"Ġhilar":17570,"ĠFeatures":17571,"220":17572,"ĠCollector":17573,"Foot":17574,"Ġ1964":17575,"culus":17576,"Ġdawn":17577,"Ġworkout":17578,"ĠLO":17579,"Ġphilosophical":17580,"ĠSandy":17581,"ĠYouth":17582,"Ġliable":17583,"Af":17584,"blue":17585,"Ġoverturn":17586,"lessness":17587,"ĠTribune":17588,"ĠIng":17589,"Ġfactories":17590,"Ġcatches":17591,"Ġprone":17592,"Ġmatrix":17593,"Ġlogin":17594,"Ġinacc":17595,"Ġexert":17596,"sys":17597,"Ġneedle":17598,"ĠQur":17599,"Ġnotified":17600,"oulder":17601,"tx":17602,"Ġreminds":17603,"Ġpublishers":17604,"Ġnort":17605,"Ġgit":17606,"Ġflies":17607,"ĠEmily":17608,"Ġflowing":17609,"ĠAlien":17610,"ĠStrateg":17611,"Ġhardest":17612,"Ġmodification":17613,"API":17614,"ĠMY":17615,"Ġcrashes":17616,"stairs":17617,"number":17618,"Ġurging":17619,"channel":17620,"ĠFalcon":17621,"Ġinhabitants":17622,"Ġterrifying":17623,"Ġutilize":17624,"Ġbanner":17625,"Ġcigarettes":17626,"Ġsenses":17627,"ĠHolmes":17628,"Ġpractition":17629,"ĠPhillips":17630,"otto":17631,"Ġcompile":17632,"Model":17633,"ĠKo":17634,"Ġ[]":17635,"Americans":17636,"ĠTerms":17637,"Ġmedications":17638,"ĠAna":17639,"Ġfundamentally":17640,"ĠNotice":17641,"Ġweaker":17642,"Ġ0000":17643,"Ġgarlic":17644,"Ġoutbreak":17645,"Ġeconomist":17646,"ĠBirth":17647,"Ġobstacles":17648,"arcer":17649,"ĠOrthodox":17650,"Ġplacebo":17651,"ĠCrew":17652,"aspberry":17653,"ĠAngels":17654,"Ġdischarge":17655,"Ġdestructive":17656,"117":17657,"ĠRising":17658,"Ġdairy":17659,"late":17660,"Ġcollision":17661,"ĠTigers":17662,"eanor":17663,"ocumented":17664,"ĠInvalid":17665,"Ġdont":17666,"ĠLiter":17667,"ĠVa":17668,"Ġhydrogen":17669,"Ġvariants":17670,"ĠBrowns":17671,"Ġ1965":17672,"Ġindigenous":17673,"Ġtrades":17674,"Ġremainder":17675,"Ġswept":17676,"ĠImpact":17677,"Ġredist":17678,"Ġunint":17679,"graduate":17680,"ãĥķ":17681,"ĠWILL":17682,"ãģ®ç":17683,"ĠCritical":17684,"Ġfisher":17685,"Ġvicious":17686,"Ġreversed":17687,"Year":17688,"ĠSox":17689,"Ġshootings":17690,"Ġfilming":17691,"Ġtouchdowns":17692,"aires":17693,"mel":17694,"Ġgrandfather":17695,"Ġaffection":17696,"ingle":17697,"Ġoverly":17698,"Additional":17699,"Ġsupreme":17700,"ĠGrad":17701,"Ġsporting":17702,"Ġmercy":17703,"ĠBrooks":17704,"ounty":17705,"Ġperforms":17706,"Ġtightly":17707,"Ġdemons":17708,"Ġkillings":17709,"Ġfaction":17710,"ĠNova":17711,"auts":17712,"Ġundoubtedly":17713,"arin":17714,"Ġunderway":17715,"rak":17716,"Ġliv":17717,"ĠRegion":17718,"Ġbriefing":17719,"sers":17720,"cloud":17721,"ĠMik":17722,"usp":17723,"Ġprediction":17724,"azor":17725,"Ġportable":17726,"ĠGand":17727,"Ġpresenting":17728,"Ġ1080":17729,"»":17730,"ushi":17731,"ĠSpark":17732,"thereum":17733,"Ġjustification":17734,"ĠNy":17735,"Ġcontractors":17736,"mingham":17737,"ĠStyle":17738,"åħ":17739,"ĠChronicles":17740,"ĠPicture":17741,"Ġproving":17742,"Ġwives":17743,"sett":17744,"Ġmolecules":17745,"ĠFairy":17746,"Ġconsisting":17747,"Ġpier":17748,"alone":17749,"inition":17750,"Ġnucle":17751,"json":17752,"Ġgotta":17753,"Ġmobil":17754,"Ġverbal":17755,"arium":17756,"Ġmonument":17757,"ucked":17758,"Ġ256":17759,"Tech":17760,"minecraft":17761,"ĠTrack":17762,"Ġtile":17763,"Ġcompatibility":17764,"asis":17765,"Ġsadd":17766,"Ġinstructed":17767,"ĠMueller":17768,"Ġlethal":17769,"Ġhormone":17770,"Ġorche":17771,"else":17772,"Ġskelet":17773,"Ġentertaining":17774,"Ġminimize":17775,"again":17776,"Ġundergo":17777,"Ġconstraints":17778,"Ġcigarette":17779,"ĠIslamist":17780,"Ġtravels":17781,"ĠPanthers":17782,"lings":17783,"Care":17784,"Ġlawsuits":17785,"uras":17786,"Ġcryst":17787,"Ġlowered":17788,"Ġaerial":17789,"Ġcombinations":17790,"Ġhaun":17791,"Ġcha":17792,"Ġvine":17793,"Ġquantities":17794,"Ġlinking":17795,"bank":17796,"Ġsoy":17797,"Bill":17798,"ĠAngela":17799,"Ġrecipient":17800,"ĠProtest":17801,"Ġsocket":17802,"Ġsolidarity":17803,"ĠâĨ":17804,"mill":17805,"Ġvaries":17806,"ĠPakistani":17807,"Dragon":17808,"Ġune":17809,"Ġhorizon":17810,"³³³³³³³³":17811,"Ġprovinces":17812,"Ġfrankly":17813,"Ġenacted":17814,"notes":17815,"['":17816,"Ġ192":17817,"ocracy":17818,"Ġendorsement":17819,"Ġovertime":17820,"True":17821,"Lab":17822,"licted":17823,"ĠDNC":17824,"Ġbeats":17825,"ĠJamie":17826,"152":17827,"ĠINT":17828,"Contact":17829,"Ġaccounted":17830,"hash":17831,"ĠPackers":17832,"pires":17833,"Ġlesbian":17834,"Ġamendments":17835,"Ġhopeful":17836,"ĠFinland":17837,"Ġspotlight":17838,"Ġconfigured":17839,"Ġtroubled":17840,"Ġgaze":17841,"ĠCalgary":17842,"Ġreliability":17843,"Ġinsurg":17844,"swer":17845,"buy":17846,"ĠSkin":17847,"Ġpixels":17848,"Ġhandgun":17849,"Ġparas":17850,"Ġcategor":17851,"ĠEL":17852,"ĠRex":17853,"Indeed":17854,"Ġkinda":17855,"Ġconjunction":17856,"ĠBryan":17857,"ĠManufact":17858,"yang":17859,"Plus":17860,"SQL":17861,"ishment":17862,"Ġdominate":17863,"Ġnail":17864,"Ġoath":17865,"Ġerupt":17866,"ĠFine":17867,"itbart":17868,"ĠChip":17869,"ĠAbd":17870,"ĠNam":17871,"Ġbuyer":17872,"Ġdissent":17873,"Leaks":17874,"Contin":17875,"Ġrider":17876,"ĠSomeone":17877,"Ġillusion":17878,"cin":17879,"ĠBoeing":17880,"Ġinadequ":17881,"ovation":17882,"iants":17883,"Ġrebuild":17884,"450":17885,"ĠDestiny":17886,"SW":17887,"ĠTill":17888,"Hit":17889,"iaz":17890,"ĠBangl":17891,"achers":17892,"ĠReform":17893,"Ġsegments":17894,"Ġsystematic":17895,"dc":17896,"ĠConservatives":17897,"Ġportal":17898,"hor":17899,"ĠDragonbound":17900,"Ġdragged":17901,"omo":17902,"Ġthee":17903,"advert":17904,"ĠReports":17905,"ĠEt":17906,"Ġbarrels":17907,"August":17908,"Ġcomparisons":17909,"Ġhex":17910,"Ġanthrop":17911,"\"[":17912,"borough":17913,"abi":17914,"Ġpictured":17915,"playing":17916,"ĠAddress":17917,"ĠMirror":17918,"Smith":17919,"Ġtires":17920,"ĠNPR":17921,"AAAA":17922,"Ġclassification":17923,"ĠThan":17924,"ĠHarm":17925,"ĠRA":17926,"Ġrejection":17927,"mination":17928,"Ġranged":17929,"ĠFalls":17930,"DI":17931,"Host":17932,"ãĤ´":17933,"ĠExample":17934,"listed":17935,"thirds":17936,"Ġsafegu":17937,"brand":17938,"Ġprobable":17939,"Canada":17940,"ITION":17941,"ĠQaeda":17942,"Ġchick":17943,"Ġimports":17944,"hit":17945,"loc":17946,"WW":17947,"Ġblew":17948,"Ġanytime":17949,"Ġwholes":17950,"iked":17951,"Ġcalculation":17952,"create":17953,"ĠOri":17954,"Ġupgraded":17955,"Ġappar":17956,"utory":17957,"ĠMol":17958,"Brit":17959,"ĠJong":17960,"INAL":17961,"ĠStarting":17962,"Ġdice":17963,"urtle":17964,"Ġrelying":17965,"closure":17966,"Ġprofitable":17967,"Ġslaughter":17968,"ĠManual":17969,"caster":17970,"Ġ\"$":17971,"Ġfeather":17972,"ĠSimply":17973,"ieves":17974,"Ġdeterior":17975,"ĠPCI":17976,"Ġstamp":17977,"Ġflaws":17978,"Ġshade":17979,"hammer":17980,"Ġpassport":17981,"Ġconting":17982,"amel":17983,"Ġobservers":17984,"Ġneglect":17985,"ĠRB":17986,"ĠBrotherhood":17987,"Ġskeptical":17988,"family":17989,"usk":17990,"Ġemotionally":17991,"âĻ":17992,"ĠBeta":17993,"asonable":17994,"idity":17995,"ĠMul":17996,"Ġkicking":17997,"ĠCarm":17998,"ollah":17999,"VERTIS":18000,"ĠAthen":18001,"Ġladder":18002,"ĠBullet":18003,"å£":18004,"0001":18005,"ĠWildlife":18006,"ĠMask":18007,"ĠNan":18008,"Rev":18009,"Ġunacceptable":18010,"legal":18011,"Ġcrowded":18012,"agi":18013,"ĠCox":18014,"je":18015,"Ġmorality":18016,"Ġfuels":18017,"Ġcables":18018,"Ġmankind":18019,"ĠCaribbean":18020,"Ġanchor":18021,"Ġbyte":18022,"ĠOften":18023,"ĠOz":18024,"Ġcrafted":18025,"Ġhistorian":18026,"ĠWu":18027,"Ġtowers":18028,"ĠCitizens":18029,"Ġhelm":18030,"Ġcredentials":18031,"Ġsingular":18032,"ĠJesse":18033,"Ġtackles":18034,"Ġcontempt":18035,"Ġafore":18036,"ĠShadows":18037,"Ġnil":18038,"Ġurgent":18039,"apple":18040,"blood":18041,"Ġvon":18042,"Ġoffline":18043,"Ġbreathe":18044,"Ġjumps":18045,"Ġirrelevant":18046,"oxic":18047,"omal":18048,"important":18049,"Jim":18050,"Ġgloves":18051,"arming":18052,"depth":18053,"Ġtalents":18054,"ookie":18055,"ĠSB":18056,"Ġpalm":18057,"uffs":18058,"esta":18059,"IGH":18060,"Ġcanon":18061,"ĠVerizon":18062,"ĠPle":18063,"Ġcoupled":18064,"velt":18065,"Ġfundraising":18066,"ĠGetting":18067,"ĠDLC":18068,"Ġmathematical":18069,"ĠHS":18070,"ĠCardinals":18071,"telling":18072,"Ġsponsors":18073,"ĠÏ":18074,"ĠBulls":18075,"option":18076,"Ġpropose":18077,"Ġmemorable":18078,"Ġembraced":18079,"Ġdeclining":18080,"Health":18081,"eda":18082,"Ġ};":18083,"Ġspam":18084,"mile":18085,"Ġpitcher":18086,"ĠEight":18087,"Ġcaring":18088,"utic":18089,"role":18090,"Ġairline":18091,"ernandez":18092,"ĠAthlet":18093,"Ġcertification":18094,"uxe":18095,"riger":18096,"Ġempir":18097,"Ġsensation":18098,"Ġdism":18099,"Ġbolt":18100,"Ġevolve":18101,"House":18102,"Ġconsultation":18103,"ĠDuty":18104,"Ġtouches":18105,"ĠNathan":18106,"Ġfaint":18107,"had":18108,"\"(":18109,"ĠConsumer":18110,"ĠExtreme":18111,"Ġ127":18112,"ĠHerm":18113,"ĠSacrament":18114,"izoph":18115,"Ġanxious":18116,"ulously":18117,"Ġsocially":18118,"ĠUTC":18119,"Ġsolving":18120,"ĠLetter":18121,"History":18122,"educ":18123,"Price":18124,"));":18125,"Ġreload":18126,"amic":18127,"Ġpork":18128,"Ġdiscourse":18129,"Ġtournaments":18130,"airo":18131,"ĠKur":18132,"ĠCosta":18133,"Ġviolating":18134,"Ġinterfere":18135,"Ġrecreational":18136,"uffle":18137,"Ġspeeches":18138,"Ġneeding":18139,"Ġremembers":18140,"Ġcredited":18141,"nia":18142,"focused":18143,"amera":18144,"Ġbru":18145,"umbs":18146,"ĠCuban":18147,"Ġpreceding":18148,"Ġnonsense":18149,"acial":18150,"Ġsmartphones":18151,"ĠStories":18152,"Sports":18153,"ĠEmergency":18154,"ouncing":18155,"efined":18156,"Ġber":18157,"Ġconsulting":18158,"Ġmasters":18159,"heastern":18160,".\"[":18161,"ĠRunning":18162,"Ġsuscept":18163,"ĠFeng":18164,"America":18165,"prises":18166,"stitial":18167,"ĠWeekly":18168,"ĠGreater":18169,"modules":18170,"ifter":18171,"Graphics":18172,"uler":18173,"Ġwholly":18174,"Ġsuppress":18175,"Ġconcealed":18176,"Ġhappily":18177,"Ġaccepts":18178,"ĠEnjoy":18179,"Ġrivers":18180,"ĠExcept":18181,"225":18182,"ĠNHS":18183,"ĠMcConnell":18184,"Ġpussy":18185,"ferred":18186,"utable":18187,"Ġattain":18188,"Ġ>=":18189,"Ġdeposits":18190,"rophic":18191,"Ġnotorious":18192,"ĠShaw":18193,"ilitation":18194,"Ġepidemic":18195,"allic":18196,"Ġsmallest":18197,"ovich":18198,"Ġaccessories":18199,"perties":18200,"Ġsurplus":18201,"ĠMech":18202,"Ġambig":18203,"ĠImmigration":18204,"Ġchim":18205,"eval":18206,"Ġpracticing":18207,"ĠMystery":18208,"Ġdomains":18209,"ĠSilicon":18210,"apps":18211,"Ġkilometers":18212,"ea":18213,"ĠSmash":18214,"Ġwarranty":18215,"Ġnost":18216,"sil":18217,"rev":18218,"Jon":18219,"ĠDublin":18220,"Ġtastes":18221,"Ġbout":18222,"great":18223,"error":18224,"Ġswitches":18225,"ĠBapt":18226,"DO":18227,"oki":18228,"Ġsourced":18229,"produ":18230,"Ġattachment":18231,"ĠIssue":18232,"ĠQuestion":18233,"Join":18234,"Ġfitted":18235,"Ġunlawful":18236,"^^":18237,"erek":18238,"Ġauthentication":18239,"Ġstole":18240,"Ġaccountability":18241,"label":18242,"Search":18243,"Ġalbeit":18244,"atican":18245,"funded":18246,"ĠAdding":18247,"ĠIQ":18248,"Ġsubmar":18249,"lit":18250,"aque":18251,"ĠLearning":18252,"Ġinteger":18253,"Master":18254,"ĠChrom":18255,"Ġpremier":18256,"Op":18257,"ĠLiu":18258,"Ġblessed":18259,"ĠGlobe":18260,"ĠResponse":18261,"Ġlegitim":18262,"ĠMerkel":18263,"Ġdisposal":18264,"´":18265,"Ġgauge":18266,"peat":18267,"Ġinduced":18268,"Ġquestionable":18269,"arthy":18270,"ĠVit":18271,"ĠFeed":18272,"Until":18273,"Ut":18274,"worthy":18275,"RY":18276,"ĠHerald":18277,"ĠHammer":18278,"Ġmedal":18279,"ĠRivers":18280,"ĠHack":18281,"Ġclarify":18282,"Ġtracked":18283,"Ġautonomous":18284,"Ġtenant":18285,"ĠQatar":18286,"erie":18287,"Ġgrim":18288,"ĠMonitor":18289,"Ġresistant":18290,"ĠSpec":18291,"ĠWells":18292,"NAS":18293,"148":18294,"Ġminers":18295,"iotics":18296,"Ġmisses":18297,"116":18298,"gian":18299,"git":18300,"ĠEyes":18301,"pres":18302,"Ġgraduated":18303,"Ġangel":18304,"Ġsynchron":18305,"Ġefficiently":18306,"Ġtransmitted":18307,"Harry":18308,"Ġglobally":18309,"ENCE":18310,"ĠMontana":18311,"raged":18312,"ĠPrevention":18313,"Ġpiss":18314,"ĠLl":18315,"Ġshelf":18316,"ĠBJP":18317,"ĠTestament":18318,"ĠLate":18319,"iker":18320,"ĠHapp":18321,"ĠJulian":18322,"hall":18323,"Ġspont":18324,"Ġshutdown":18325,"Ġinconsistent":18326,"Ġsubscribers":18327,"Ġskeleton":18328,"ĠNebraska":18329,"Ġinspire":18330,"ĠVoid":18331,"Feed":18332,"Ġangles":18333,"ĠSprings":18334,"Ġbenchmark":18335,"Ġvaccines":18336,"izophren":18337,"sexual":18338,"uffed":18339,"Ġshine":18340,"ĠKath":18341,"Ġgesture":18342,"inea":18343,"Ġrip":18344,"Ġoppression":18345,"Ġconscience":18346,"bt":18347,"ĠLum":18348,"Ġincidence":18349,"ĠFa":18350,"wr":18351,"Ġmineral":18352,"ĠSpurs":18353,"alky":18354,"Ġthunder":18355,"Ġopio":18356,"Being":18357,"ĠPalm":18358,"Ġwasted":18359,"Ġlb":18360,"iaries":18361,"ĠInitiative":18362,"Ġcurric":18363,"Ġmarker":18364,"ĠMcL":18365,"Ġextensions":18366,"ĠPv":18367,"ĠArms":18368,"Ġofferings":18369,"Ġdefenses":18370,"Ġvendor":18371,"Ġcontradict":18372,"ĠColin":18373,"Ġreddit":18374,"Ġperipher":18375,"122":18376,"Ġsins":18377,"Edit":18378,"ICT":18379,"Soft":18380,"ĠShah":18381,"Ġadministrator":18382,"ĠTrip":18383,"Ġpornography":18384,"Ġtuition":18385,"inence":18386,"ĠProgress":18387,"Ġcatalog":18388,"Ġsuite":18389,"Ġhike":18390,"Ġreproductive":18391,"engine":18392,"Ġdrought":18393,"ĠNoah":18394,"Ġ230":18395,"Ġdude":18396,"Ġrelaxed":18397,"Ġpartition":18398,"Ġparticipant":18399,"Ġtelesc":18400,"Ġfeas":18401,"ĠFF":18402,"owner":18403,"Ġsweeping":18404,"Ġlenses":18405,"Ġmatchup":18406,"ĠRepl":18407,"ournals":18408,"Ġcredible":18409,"Ġgrandmother":18410,"Ġthermal":18411,"Ġsubscribing":18412,"Ġidentities":18413,"colm":18414,"UCT":18415,"Ġreluctant":18416,"users":18417,"ĠCort":18418,"Ġassisted":18419,"OSS":18420,"ATIONS":18421,"ISH":18422,"Ġpharmaceutical":18423,"icable":18424,"adian":18425,"ĠSonic":18426,"ĠFury":18427,"ĠMong":18428,"AH":18429,"ĠPsychology":18430,"Ġphosph":18431,"Ġtreats":18432,"ŃĶ":18433,"Ġsteadily":18434,"ĠHello":18435,"Ġrelates":18436,"Ġclue":18437,"Expl":18438,"auth":18439,"Ġrevision":18440,"Ġeld":18441,"osion":18442,"Ġbron":18443,"144":18444,"rikes":18445,"Ġmines":18446,"Ġblanket":18447,"ĠFail":18448,"eled":18449,"ĠImagine":18450,"ĠPlanned":18451,"aic":18452,"Request":18453,"Mad":18454,"ĠHorse":18455,"ĠEagle":18456,"Ġcapac":18457,"157":18458,"Ġling":18459,"ĠNice":18460,"ĠParenthood":18461,"minster":18462,"ogs":18463,"ensitive":18464,"Nothing":18465,"Ġcarn":18466,"Fin":18467,"ĠPE":18468,"Ġrifles":18469,"ĠLP":18470,"Sand":18471,"ĠguiActive":18472,"Ġtourist":18473,"CNN":18474,"Ġunveiled":18475,"Ġpredecessor":18476,"}{":18477,"uber":18478,"Ġoffshore":18479,"Ġoptical":18480,"ĠRot":18481,"ĠPearl":18482,"eton":18483,"Ġstared":18484,"Ġfarther":18485,"atility":18486,"contin":18487,"ĠGy":18488,"ĠFoster":18489,"ĠCoc":18490,"rients":18491,"Ġdesigning":18492,"ĠEconomy":18493,"ONG":18494,"Women":18495,"ĠNancy":18496,"erver":18497,"Ġmascul":18498,"Ġcasualties":18499,"Ġ225":18500,"ĠSullivan":18501,"ĠChoice":18502,"Ġaster":18503,"ws":18504,"Ġhotels":18505,"Ġconsiderations":18506,"Ġcouch":18507,"ĠStrip":18508,"ĠGn":18509,"Ġmanipulate":18510,"lied":18511,"Ġsynthetic":18512,"Ġassaulted":18513,"Ġoffenses":18514,"ĠDrake":18515,"Ġimpe":18516,"October":18517,"ĠHeritage":18518,"hl":18519,"ĠBlair":18520,"Unlike":18521,"Ġgrief":18522,"Ġ450":18523,"Ġopted":18524,"Ġresignation":18525,"ilo":18526,"Ġverse":18527,"ĠTomb":18528,"Ġupt":18529,"Ġaired":18530,"ĠHook":18531,"ĠMLB":18532,"Ġassumes":18533,"outed":18534,"ĠVers":18535,"Ġinferior":18536,"Ġbundle":18537,"ĠDNS":18538,"ographer":18539,"Ġmultip":18540,"ĠSouls":18541,"Ġillustrated":18542,"Ġtactic":18543,"Ġdressing":18544,"Ġduo":18545,"Conf":18546,"Ġrelent":18547,"Ġcant":18548,"Ġscarce":18549,"Ġcandy":18550,"ĠCF":18551,"Ġaffiliated":18552,"Ġsprint":18553,"ylan":18554,"ĠGarcia":18555,"Ġjunk":18556,"Print":18557,"exec":18558,"Crit":18559,"Ġportrait":18560,"iries":18561,"ĠOFF":18562,"Ġdisputes":18563,"WR":18564,"Love":18565,"ãģĦ":18566,"ĠReyn":18567,"Ġhipp":18568,"opath":18569,"Ġfloors":18570,"ĠFeel":18571,"Ġworries":18572,"Ġsettlements":18573,"ĠPos":18574,"Ġmosque":18575,"Ġfinals":18576,"Ġcrushed":18577,"ĠProbably":18578,"ĠBot":18579,"ĠMans":18580,"ĠPeriod":18581,"Ġsovereignty":18582,"Ġseller":18583,"Ġapost":18584,"Ġamateur":18585,"Ġdorm":18586,"Ġconsuming":18587,"Ġarmour":18588,"ĠRoose":18589,"Ġintensive":18590,"Ġeliminating":18591,"ĠSunni":18592,"ĠAleppo":18593,"jin":18594,"Ġadvise":18595,"pal":18596,"ĠHalo":18597,"Ġdescent":18598,"Ġsimpler":18599,"Ġbooth":18600,"STR":18601,"Later":18602,"ĠCave":18603,"===":18604,"Ġmol":18605,"Ġfist":18606,"Ġshotgun":18607,"supp":18608,"Ġrobbery":18609,"Effect":18610,"Ġobscure":18611,"ĠProfessional":18612,"Ġembassy":18613,"Ġmilitant":18614,"Ġincarcer":18615,"Ġgenerates":18616,"Ġlaunches":18617,"Ġadministrators":18618,"Ġshaft":18619,"Ġcircular":18620,"Ġfreshman":18621,"ĠWes":18622,"ĠJoel":18623,"ĠDrew":18624,"ĠDuncan":18625,"ĠApparently":18626,"sight":18627,"ĠInternal":18628,"ĠIndividual":18629,"ĠFE":18630,"Ġbore":18631,"ĠMt":18632,"Ġbroadly":18633,"ĠOptions":18634,"ountain":18635,"ipes":18636,"ĠVideos":18637,"204":18638,"Ġhills":18639,"Ġsimulation":18640,"Ġdisappointment":18641,"itan":18642,"ĠLaboratory":18643,"Ġupward":18644,"Ġboundary":18645,"Ġdarker":18646,"hart":18647,"Ġdominance":18648,"Cong":18649,"ĠOracle":18650,"ĠLords":18651,"Ġscholarship":18652,"ĠVincent":18653,"ede":18654,"ĠRah":18655,"Ġencourages":18656,"rov":18657,"Ġquo":18658,"Ġpremise":18659,"ĠCrisis":18660,"ĠHolocaust":18661,"Ġrhythm":18662,"Ġmetric":18663,"club":18664,"Ġtransported":18665,"Ġnod":18666,"ĠPist":18667,"Ġancestors":18668,"ĠFreder":18669,"thumbnails":18670,"ĠCE":18671,"OND":18672,"Phil":18673,"venge":18674,"ĠProducts":18675,"castle":18676,"Ġqualifying":18677,"ĠKaren":18678,"VERTISEMENT":18679,"Ġmighty":18680,"Ġexplanations":18681,"Ġfixing":18682,"Di":18683,"Ġdeclaring":18684,"Ġanonymity":18685,"Ġjuven":18686,"ĠNord":18687,"ĠDoom":18688,"ĠActually":18689,"Ok":18690,"phis":18691,"ĠDesert":18692,"Ġ116":18693,"IK":18694,"ĠFM":18695,"Ġincomes":18696,"VEL":18697,"okers":18698,"Ġpecul":18699,"Ġlightweight":18700,"gue":18701,"Ġaccent":18702,"Ġincrement":18703,"ĠChan":18704,"Ġcomplaining":18705,"ĠBaghd":18706,"Ġmidfielder":18707,"Ġoverhaul":18708,"Process":18709,"ĠHollow":18710,"ĠTitans":18711,"Small":18712,"manuel":18713,"ĠUnity":18714,"ĠEvents":18715,"Sty":18716,"Ġdisproportion":18717,"nesty":18718,"enes":18719,"ĠCod":18720,"Ġdemonstrations":18721,"ĠCrimson":18722,"ĠOH":18723,"Ġenrolled":18724,"Ġcel":18725,"ĠBrett":18726,"Ġaide":18727,"Ġheels":18728,"Ġbroadband":18729,"Ġmarking":18730,"Ġwizard":18731,"ĠNJ":18732,"ĠChiefs":18733,"Ġingredient":18734,"Ġdug":18735,"ĠShut":18736,"urchase":18737,"endor":18738,"Ġfarmer":18739,"ĠGoldman":18740,"129":18741,"155":18742,"Order":18743,"Ġlion":18744,"iably":18745,"Ġstain":18746,"array":18747,"ilitary":18748,"ĠFAQ":18749,"Ġexploded":18750,"ĠMcCarthy":18751,"ĠTweet":18752,"ĠGreens":18753,"eking":18754,"ln":18755,"ensen":18756,"Ġmotorcycle":18757,"Ġparticle":18758,"Ġcholesterol":18759,"Bron":18760,"Ġstair":18761,"Ġoxid":18762,"Ġdesirable":18763,"ibles":18764,"Ġtheor":18765,"forcing":18766,"Ġpromotional":18767,"ovo":18768,"boot":18769,"ĠBonus":18770,"rawling":18771,"Ġshortage":18772,"ĠPsy":18773,"Ġrecruited":18774,"Ġinfants":18775,"Ġtestosterone":18776,"Ġdeduct":18777,"Ġdistinctive":18778,"Ġfirmware":18779,"built":18780,"145":18781,"Ġexplored":18782,"Ġfactions":18783,"Ġvide":18784,"Ġtattoo":18785,"Ġfinancially":18786,"Ġfatigue":18787,"Ġproceeding":18788,"constitutional":18789,"Ġmiser":18790,"Ġchairs":18791,"gging":18792,"ipple":18793,"Ġdent":18794,"Ġdisreg":18795,"çĶ":18796,"stant":18797,"llo":18798,"bps":18799,"akening":18800,"Ġabnormal":18801,"ĠERA":18802,"士":18803,"ĠHBO":18804,"ĠMAR":18805,"Ġconcess":18806,"Ġservant":18807,"Ġaspir":18808,"lav":18809,"ĠPanel":18810,"amo":18811,"Ġprecip":18812,"Ġrecordings":18813,"Ġproceeded":18814,"Ġcolony":18815,"ĠTang":18816,"ablo":18817,"Ġstripped":18818,"Left":18819,"too":18820,"Ġpotatoes":18821,"Ġfinest":18822,"%).":18823,"Ġcrap":18824,"ĠZach":18825,"abases":18826,"ĠGoth":18827,"Ġbillionaire":18828,"wolf":18829,"Ġsanction":18830,"SK":18831,"Ġlogged":18832,"Po":18833,"eyed":18834,"unal":18835,"Ġcricket":18836,"Ġarmies":18837,"Ġuncovered":18838,"Cloud":18839,"ón":18840,"Ġrebounds":18841,"Ġmes":18842,"Oper":18843,"Pac":18844,"Ġnationally":18845,"Ġinserted":18846,"pict":18847,"Ġgovernance":18848,"и":18849,"Ġprivileges":18850,"GET":18851,"Ġfavorites":18852,"imity":18853,"Ġlover":18854,"them":18855,"empl":18856,"Ġgorgeous":18857,"Ann":18858,"Ġslipped":18859,"Ġveto":18860,"Bob":18861,"Ġslim":18862,"ucc":18863,"ĠFame":18864,"uddenly":18865,"Ġdenies":18866,"ĠMaur":18867,"Ġdistances":18868,"Ġwanna":18869,"tar":18870,"ĠSER":18871,"ĠâĪ":18872,"Ġlemon":18873,"athetic":18874,"Ġliteral":18875,"Ġdistinguished":18876,"Ġanswering":18877,"GI":18878,"Ġreligions":18879,"ĠPhilos":18880,"ĠLay":18881,"Ġcompos":18882,"irements":18883,"ĠKos":18884,"inez":18885,"rolling":18886,"Ġyoungest":18887,"andise":18888,"ĠBorn":18889,"Ġaltar":18890,"amina":18891,"ĠBoot":18892,"voc":18893,"Ġdigging":18894,"Ġpressures":18895,"Ġlen":18896,"264":18897,"Ġassassination":18898,"ĠBirmingham":18899,"ĠMyth":18900,"Ġsovereign":18901,"ĠArtist":18902,"ĠPhotograph":18903,"Ġdepicted":18904,"Ġdispens":18905,"orthy":18906,"Ġambul":18907,"integ":18908,"ĠCele":18909,"ĠTibet":18910,"Ġhierarchy":18911,"Ġcu":18912,"Ġpreseason":18913,"ĠPeterson":18914,"Ġcolours":18915,"Ġworrying":18916,"Ġbackers":18917,"ĠPalmer":18918,"Ġμ":18919,"Ġcontributor":18920,"Ġhearings":18921,"Ġurine":18922,"ĠÙ":18923,"ourgeois":18924,"Similar":18925,"ĠZimmer":18926,"something":18927,"ĠUSC":18928,"Ġstrengths":18929,"ĠFI":18930,"Ġlogging":18931,"Asked":18932,"ĠThai":18933,"inqu":18934,"ĠWalt":18935,"Ġcrews":18936,"itism":18937,"301":18938,"Ġsharply":18939,"umed":18940,"Ġredirect":18941,"rators":18942,"Inf":18943,"ĠWeapons":18944,"Ġteasp":18945,"1999":18946,"Live":18947,"ĠEspecially":18948,"ĠSter":18949,"ĠVeterans":18950,"Ġintro":18951,"otherapy":18952,"Ġmalware":18953,"Ġbreeding":18954,"Ġmolecular":18955,"ĠRoute":18956,"ĠComment":18957,"ochem":18958,"Ġain":18959,"Season":18960,"Ġlinebacker":18961,"Ä«":18962,"ĠEconomics":18963,"esar":18964,"ĠLives":18965,"ĠEmma":18966,"Ġkin":18967,"ĠTerrit":18968,"Ġplanted":18969,"oton":18970,"ĠButter":18971,"ĠSpons":18972,"PER":18973,"Ġdungeon":18974,"Ġsymbolic":18975,"Ġfilmed":18976,"Ġdiets":18977,"Ġconcludes":18978,"Ġcertainty":18979,"ĠFormat":18980,"Ġstrangers":18981,"format":18982,"ĠPhase":18983,"Ġcopied":18984,"Ġmetres":18985,"lda":18986,"ĠUsers":18987,"Ġdeliberate":18988,"Ġwashed":18989,"ĠLance":18990,"imation":18991,"Ġimproper":18992,"ĠGenesis":18993,"ickr":18994,"ĠKush":18995,"Ġrealise":18996,"Ġembarrassing":18997,"alking":18998,"bucks":18999,"Ġverified":19000,"Ġoutline":19001,"years":19002,"ĠIncome":19003,"202":19004,"Ġzombies":19005,"Final":19006,"ĠMillenn":19007,"Ġmodifications":19008,"ĠVision":19009,"ĠMoses":19010,"verb":19011,"iterranean":19012,"ĠJet":19013,"Ġnaval":19014,"ĠAgg":19015,"Ġurl":19016,"Ġvictories":19017,"Ġnonetheless":19018,"Ġinjust":19019,"ĠFact":19020,"çļ":19021,"Ġinsufficient":19022,"review":19023,"facebook":19024,"Ġnegotiating":19025,"Ġguarantees":19026,"imen":19027,"utenberg":19028,"Ġgambling":19029,"Ġcongr":19030,"Loading":19031,"Ġnevertheless":19032,"Ġpresidents":19033,"ĠIndustrial":19034,"Ġ118":19035,"Ġpoured":19036,"ĠTory":19037,"Ġ175":19038,"Ġ:=":19039,"Scott":19040,"angered":19041,"Tok":19042,"Ġorganizers":19043,"Mat":19044,"ĠGrowth":19045,"Ġadul":19046,"Ġensures":19047,"Ġ117":19048,"é¾įå":19049,"Ġmassacre":19050,"Ġgrades":19051,"before":19052,"ADVERTISEMENT":19053,"ĠSlow":19054,"ĠMMA":19055,"âĢĶ\"":19056,"ĠVatican":19057,"Qaeda":19058,"Ġowe":19059,"6666":19060,"ĠSorry":19061,"ĠGrass":19062,"Ġbackgrounds":19063,"Ġexhausted":19064,"Ġclan":19065,"Ġcompromised":19066,"ĠElf":19067,"ĠIsaac":19068,"enson":19069,"Invest":19070,"IFA":19071,"Ġinterrupted":19072,"ãĥīãĥ©":19073,"Ġtwisted":19074,"ĠDragons":19075,"Mode":19076,"ĠKremlin":19077,"Ġfertil":19078,"heres":19079,"phan":19080,"ĠNode":19081,"fed":19082,"ĠOrc":19083,"Ġunwilling":19084,"Cent":19085,"Ġpriorit":19086,"Ġgraduates":19087,"Ġsubjective":19088,"Ġissuing":19089,"ĠLt":19090,"Ġviewer":19091,"Ġwoke":19092,"Thus":19093,"brook":19094,"Ġdepressed":19095,"Ġbracket":19096,"ĠGor":19097,"ĠFighting":19098,"Ġstriker":19099,"Report":19100,"ĠPortugal":19101,"Ġneo":19102,"wed":19103,"199":19104,"Ġfleeing":19105,"shadow":19106,"identified":19107,"USE":19108,"Steam":19109,"Ġstretched":19110,"Ġrevelations":19111,"arted":19112,"ĠDw":19113,"Ġalignment":19114,"eston":19115,"ĠJared":19116,"Sep":19117,"Ġblogs":19118,"update":19119,"gom":19120,"risk":19121,"Ġclash":19122,"ĠHour":19123,"Ġruntime":19124,"Ġunwanted":19125,"Ġscam":19126,"Ġrack":19127,"Ġenlight":19128,"onest":19129,"ĠFerr":19130,"Ġconvictions":19131,"Ġpiano":19132,"Ġcirculation":19133,"ĠWelcome":19134,"Ġbacklash":19135,"ĠWade":19136,"Ġreceivers":19137,"otive":19138,"Jeff":19139,"Ġnetworking":19140,"ĠPrep":19141,"ĠExplorer":19142,"Ġlecture":19143,"Ġuploaded":19144,"ĠMeat":19145,"BLE":19146,"ĠNazis":19147,"ĠSynd":19148,"stud":19149,"roots":19150,"rians":19151,"Ġportrayed":19152,"Ġ??":19153,"ĠBuddha":19154,"sun":19155,"Robert":19156,"ĠComplex":19157,"Ġoversee":19158,"Ġstealth":19159,"Title":19160,"ĠJobs":19161,"ĠKum":19162,"Ġappreciation":19163,"ĠMOD":19164,"Ġbasics":19165,"Ġclips":19166,"Ġnursing":19167,"Ġproposition":19168,"Ġrealised":19169,"ĠNYC":19170,"Ġallocated":19171,"rium":19172,"aran":19173,"ĠProduction":19174,"ĠVote":19175,"Ġsmugg":19176,"Ġhunter":19177,"azer":19178,"ĠChanges":19179,"Ġfluct":19180,"yon":19181,"Array":19182,"Ġkits":19183,"Water":19184,"Ġuncommon":19185,"Ġresting":19186,"ells":19187,"would":19188,"Ġpursued":19189,"Ġassertion":19190,"ometown":19191,"ĠMosul":19192,"ĠPlatform":19193,"iolet":19194,"Ġshareholders":19195,"Ġtrails":19196,"Pay":19197,"ĠEnforcement":19198,"types":19199,"ĠAnonymous":19200,"Ġsatisfying":19201,"ilogy":19202,"Ġ('":19203,"wave":19204,"city":19205,"Steve":19206,"Ġconfrontation":19207,"ĠEld":19208,"Capt":19209,"ahan":19210,"htm":19211,"ĠCtrl":19212,"ONS":19213,"230":19214,"ifa":19215,"holding":19216,"Ġdelicate":19217,"Ġjaw":19218,"ĠGoing":19219,"orum":19220,"Sal":19221,"Ġdull":19222,"ĠBeth":19223,"Ġprisons":19224,"Ġego":19225,"ĠElsa":19226,"avorite":19227,"ĠGang":19228,"ĠNuclear":19229,"Ġspider":19230,"atsu":19231,"Ġsampling":19232,"Ġabsorbed":19233,"ĠPharm":19234,"ieth":19235,"Ġbucket":19236,"ĠRecomm":19237,"OF":19238,"ĠFactory":19239,"ANCE":19240,"Ġbacter":19241,"Has":19242,"ĠObserv":19243,"121":19244,"Ġpremiere":19245,"Develop":19246,"Ġcurrencies":19247,"Cast":19248,"Ġaccompanying":19249,"ĠNashville":19250,"Ġfatty":19251,"ĠBrend":19252,"Ġlocks":19253,"Ġcentered":19254,"ĠUT":19255,"aughs":19256,"orie":19257,"ĠAffordable":19258,"vance":19259,"DL":19260,"emet":19261,"Ġthrone":19262,"ĠBluetooth":19263,"Ġnaming":19264,"ifts":19265,"ADE":19266,"Ġcorrected":19267,"Ġpromptly":19268,"ĠSTR":19269,"Ġgenome":19270,"Ġcope":19271,"Ġvalley":19272,"Ġrounded":19273,"ĠKend":19274,"alion":19275,"pers":19276,"Ġtourism":19277,"Ġstark":19278,"vl":19279,"Ġblowing":19280,"ĠSchedule":19281,"std":19282,"Ġunhappy":19283,"Ġlitigation":19284,"cedes":19285,"Ġandroid":19286,"Ġintegral":19287,"erers":19288,"uded":19289,"tax":19290,"Ġreiter":19291,"ĠMotors":19292,"ociated":19293,"Ġwonders":19294,"ĠApost":19295,"ucking":19296,"ĠRoosevelt":19297,"fram":19298,"Ġyields":19299,"Ġconstitutes":19300,"awk":19301,"Interest":19302,"Ġinterim":19303,"Ġbreakthrough":19304,"ĠCher":19305,"Ġprosec":19306,"ĠDj":19307,"ĠMT":19308,"Resp":19309,"ĠPT":19310,"Ġsperm":19311,"edit":19312,"BT":19313,"Linux":19314,"country":19315,"league":19316,"Ġdick":19317,"Ġoct":19318,"Ġinserting":19319,"Ġscra":19320,"ĠBrewing":19321,"Ġ1966":19322,"Ġrunners":19323,"Ġplun":19324,"idy":19325,"ĠDian":19326,"Ġdysfunction":19327,"Ġexclusion":19328,"Ġdisgr":19329,"Ġincorporate":19330,"Ġreconc":19331,"Ġnominated":19332,"ĠArcher":19333,"draw":19334,"achelor":19335,"Ġwritings":19336,"Ġshallow":19337,"Ġhast":19338,"ĠBMW":19339,"ĠRS":19340,"Ġthigh":19341,"Ġ1963":19342,"Ġlamb":19343,"Ġfavored":19344,"agle":19345,"Ġcooler":19346,"ĠHours":19347,"ĠGU":19348,"ĠOrigin":19349,"Ġglimpse":19350,"--------------------":19351,"Lim":19352,"Ġcheek":19353,"Ġjealous":19354,"-'":19355,"Ġharness":19356,"ĠPoison":19357,"Ġdisabilities":19358,"neapolis":19359,"Ġoutlook":19360,"Ġnotify":19361,"ĠIndianapolis":19362,"Ġabrupt":19363,"nsic":19364,"Ġencrypted":19365,"Ġforfe":19366,"reath":19367,"Ġrabb":19368,"Ġfoundations":19369,"Ġcompliment":19370,"ĠInterview":19371,"ĠSwe":19372,"Ġadolesc":19373,"Ġmonitors":19374,"ĠSacramento":19375,"Ġtimely":19376,"Ġcontempl":19377,"Ġpositioned":19378,"Ġposters":19379,"phies":19380,"iovascular":19381,"void":19382,"ĠFifth":19383,"Ġinvestigative":19384,"OUN":19385,"Ġintegrate":19386,"ĠINC":19387,"isha":19388,"iblings":19389,"ĠRequest":19390,"ĠRodriguez":19391,"Ġslides":19392,"ĠDX":19393,"Ġfeminism":19394,"Ġdatas":19395,"Ġbend":19396,"irus":19397,"ĠNigeria":19398,"Fox":19399,"Change":19400,"Ġairplane":19401,"ĠLaden":19402,"Ġpublicity":19403,"ixty":19404,"Ġcommitments":19405,"Ġaggregate":19406,"Ġdisplaying":19407,"ĠArrow":19408,"Ġ122":19409,"Ġrespects":19410,"android":19411,"six":19412,"ĠSha":19413,"Ġrestoration":19414,")\\":19415,"WS":19416,"oys":19417,"Ġillustrate":19418,"without":19419,"126":19420,"ĠâĶĤ":19421,"Ġpickup":19422,"nels":19423,"Ġ....":19424,"food":19425,"ĠFen":19426,")?":19427,"Ġphenomena":19428,"Ġcompanions":19429,"ĠWrite":19430,"Ġspill":19431,"Ġbridges":19432,"ĠUpdated":19433,"ĠFo":19434,"Ġinsects":19435,"ASHINGTON":19436,"Ġscare":19437,"iltr":19438,"ĠZhang":19439,"Ġseverity":19440,"Ġindul":19441,"149":19442,"ĠCoffee":19443,"Ġnorms":19444,"Ġpulse":19445,"ĠFT":19446,"Ġhorrific":19447,"ĠDestroy":19448,"ĠJSON":19449,"Ġolive":19450,"Ġdiscusses":19451,"Rest":19452,"Elect":19453,"ĠWinn":19454,"ĠSurviv":19455,"ĠHait":19456,"Sure":19457,"oped":19458,"Ġrooted":19459,"ĠSke":19460,"ĠBronze":19461,"Ġlol":19462,"Default":19463,"Ġcommodity":19464,"redited":19465,"Ġlibertarian":19466,"Ġforbidden":19467,"Ġgran":19468,"à¨":19469,"Ġlag":19470,"enz":19471,"drive":19472,"Ġmathematics":19473,"Ġwires":19474,"Ġcritically":19475,"Ġcarbohyd":19476,"ĠChancellor":19477,"ĠEddie":19478,"Ġbanning":19479,"ĠFri":19480,"Ġcomplications":19481,"etric":19482,"ĠBangladesh":19483,"Ġbandwidth":19484,"Stop":19485,"ĠOriginally":19486,"Ġhalfway":19487,"ynasty":19488,"shine":19489,"Ġtales":19490,"rities":19491,"avier":19492,"Ġspinning":19493,"ĠWHO":19494,"Ġneighbourhood":19495,"bach":19496,"Ġcommerce":19497,"ĠSle":19498,"BU":19499,"Ġentrepreneur":19500,"Ġpeculiar":19501,"ĠComments":19502,"fre":19503,"320":19504,"ICS":19505,"Ġimagery":19506,"ĠCanon":19507,"ĠElectronic":19508,"short":19509,"((":19510,"Dig":19511,"Ġcommem":19512,"uced":19513,"Ġinclined":19514,"ĠSummon":19515,"Ġcliff":19516,"ĠMediterranean":19517,"Ġpoetry":19518,"Ġprosperity":19519,"ĠRece":19520,"Ġpills":19521,"member":19522,"Ġfinale":19523,"unc":19524,"ĠGig":19525,"ä½":19526,"Ġlod":19527,"Ġbackward":19528,"-+":19529,"ĠForward":19530,"Ġthri":19531,"sure":19532,"Ġsoap":19533,"ĠFX":19534,"RES":19535,"ĠSexual":19536,"oulos":19537,"Ġfoolish":19538,"Ġrighteous":19539,"Ġcoff":19540,"terrorism":19541,"ustain":19542,"oter":19543,"Ġabuses":19544,"next":19545,"Ġabusive":19546,"Ġthereafter":19547,"Ġprohibition":19548,"ĠSUP":19549,"Ġdip":19550,"Ġripped":19551,"Ġinherited":19552,"Ġbats":19553,"stru":19554,"GT":19555,"Ġflawed":19556,"phabet":19557,"Ġfog":19558,"doors":19559,"Ġimaging":19560,"Ġdigits":19561,"ĠHungary":19562,"Ġarrog":19563,"Ġteachings":19564,"Ġprotocols":19565,"ĠBanks":19566,"à¸":19567,"pound":19568,"ĠCurt":19569,".\")":19570,"./":19571,"Ġexemption":19572,"endix":19573,"ĠMull":19574,"Ġimproves":19575,"ĠGamer":19576,"dimensional":19577,"Icon":19578,"ĠMargaret":19579,"Status":19580,"dates":19581,"Ġintends":19582,"Ġdepict":19583,"Ġparked":19584,"Joe":19585,"ĠMarines":19586,"chnology":19587,"!).":19588,"Ġjudged":19589,"Ġweights":19590,"Ray":19591,"Ġapartments":19592,"hester":19593,"Ġreinforce":19594,"Ġoffender":19595,"occup":19596,"Ġsore":19597,"ept":19598,"ĠPHP":19599,"ĠBrow":19600,"Ġauthorization":19601,"ĠRisk":19602,"ĠDelaware":19603,"ĠQU":19604,"Ġnotifications":19605,"Ġsunlight":19606,"Ġexclude":19607,"dat":19608,"Ġmesh":19609,"ĠSudan":19610,"Ġbelonged":19611,"Ġsubway":19612,"Ġnoon":19613,"ĠInterior":19614,"olics":19615,"ĠLakers":19616,"Ġcoding":19617,"Disclaimer":19618,"Calif":19619,"Old":19620,"Ġdisl":19621,"?????":19622,"Ġconfirms":19623,"Ġrecruitment":19624,"Ġhomicide":19625,"Consider":19626,"ĠJeffrey":19627,"fty":19628,"};":19629,"Ġobjection":19630,"doing":19631,"ĠLeo":19632,"Want":19633,"Ġglow":19634,"ĠClarke":19635,"ĠNorman":19636,"Ġverification":19637,"Ġpacket":19638,"ĠFormula":19639,"Ġplag":19640,"esville":19641,"Ġshouting":19642,"Ġov":19643,"ĠREC":19644,"ĠBub":19645,"Ġninth":19646,"Ġenerg":19647,"Ġvalidity":19648,"Ġups":19649,"jack":19650,"Ġneighboring":19651,"ĠNec":19652,"eworks":19653,"ĠHab":19654,"arez":19655,"Ġspine":19656,"Ġeventual":19657,"ĠLeaders":19658,"ĠCarn":19659,"Ġprobation":19660,"Ġromance":19661,"msg":19662,"ĠMechanical":19663,"ERY":19664,"Rock":19665,"Ġpartisan":19666,"Node":19667,"assets":19668,"minent":19669,"Ġforeigners":19670,"Ġtestify":19671,"ĠUsually":19672,"lords":19673,"ĠGren":19674,"ĠPowell":19675,"BIL":19676,"Ġsr":19677,"Ġaddict":19678,"Ġshells":19679,"Ġsigh":19680,"ĠYale":19681,"ternity":19682,"Ġ750":19683,"EU":19684,"ĠRifle":19685,"Ġpatron":19686,"ema":19687,"ĠBannon":19688,"anity":19689,"Ġtropical":19690,"ĠVII":19691,"cross":19692,"Everything":19693,"ĠISO":19694,"Ġhumble":19695,"assing":19696,"ĠFIG":19697,"Ġupdating":19698,"yson":19699,"Ġcalcium":19700,"Ġcompetent":19701,"Ġsteering":19702,"Prot":19703,"ĠSY":19704,"ĠFinals":19705,"ĠRug":19706,"159":19707,"137":19708,"ĠGolf":19709,"Ġ126":19710,"Ġaccommodation":19711,"ĠHughes":19712,"Ġaesthetic":19713,"artisan":19714,"ĠTwilight":19715,"Ġprince":19716,"ĠAgriculture":19717,"ĠDisco":19718,"Ġprecedent":19719,"Ġtyping":19720,"authorized":19721,"Option":19722,"ĠAub":19723,"lishes":19724,"acht":19725,"mag":19726,"Peter":19727,"ĠUFO":19728,"monton":19729,"ĠLith":19730,"Ġarom":19731,"Ġsecuring":19732,"Ġconfined":19733,"private":19734,"Ġswords":19735,"Ġmarkers":19736,"Ġmetabolic":19737,"select":19738,"ĠCurse":19739,"ĠOt":19740,"gressive":19741,"Ġincumb":19742,"ĠSaga":19743,"Ġpriced":19744,"Ġclearance":19745,"Content":19746,"Ġdrilling":19747,"Ġnotices":19748,"Ġbourgeois":19749,"Ġvest":19750,"Ġcookie":19751,"ĠGuardians":19752,"rys":19753,"inyl":19754,"Ġ124":19755,"Ġplausible":19756,"ongh":19757,"ĠOdin":19758,"Ġconception":19759,"ĠYuk":19760,"ĠBaghdad":19761,"ĠFlag":19762,"Austral":19763,"ĠIBM":19764,"Ġinternationally":19765,"ĠWikiLeaks":19766,"IED":19767,"Ġcyn":19768,"Ġchooses":19769,"ĠPill":19770,"Ġcombining":19771,"Ġradi":19772,"ĠMohammed":19773,"defense":19774,"atching":19775,"Subject":19776,"iciency":19777,"Frame":19778,"Ġ{\"":19779,"Ġchess":19780,"Ġtimer":19781,"190":19782,"Ġtin":19783,"Ġordinance":19784,"emetery":19785,"Ġaccusing":19786,"Ġnoticeable":19787,"Ġcentres":19788,"Ġlid":19789,"ĠMills":19790,"imgur":19791,"Ġzoom":19792,"ergic":19793,"Ġcompression":19794,"prim":19795,"find":19796,"Ġsurg":19797,"Ġpand":19798,"ĠKee":19799,"ĠChad":19800,"cellence":19801,"oyle":19802,"Ġsocialism":19803,"ĠTravis":19804,"ĠMHz":19805,"Ġguild":19806,"ALLY":19807,"ĠSubscribe":19808,"ĠRelated":19809,"Ġoccurrence":19810,"itching":19811,"Ġfictional":19812,"Ġcrush":19813,"ĠEA":19814,"cod":19815,"mix":19816,"ĠTriple":19817,"Ġretrieve":19818,"Ġstimulus":19819,"Ġpsychiat":19820,"ĠDoor":19821,"Ġhomosexuality":19822,"Ġelementary":19823,"Ġcellular":19824,"idian":19825,"ĠLaun":19826,"Ġintriguing":19827,"Ġfoam":19828,"ĠBass":19829,"idi":19830,"itsu":19831,"Ġassure":19832,"Ġcongrat":19833,"Ġbusinessman":19834,"ĠBoost":19835,"close":19836,"Ġlied":19837,"Ġsciences":19838,"ĠOmega":19839,"ĠGraphics":19840,"Ġ<=":19841,"spoken":19842,"Ġconnectivity":19843,"Saturday":19844,"ĠAvengers":19845,"Ġtoggle":19846,"Ġankle":19847,"Ġnationalist":19848,"model":19849,"ĠPool":19850,"ophobia":19851,"Var":19852,"ĠMons":19853,"atories":19854,"Ġaggressively":19855,"Clear":19856,"Forge":19857,"acters":19858,"Ġhedge":19859,"Ġpipes":19860,"Ġblunt":19861,"Ġsq":19862,"Ġremotely":19863,"Wed":19864,"asers":19865,"Ġrefriger":19866,"Ġtiles":19867,"Ġrescued":19868,"Ġcomprised":19869,"insky":19870,"Ġmanif":19871,"avanaugh":19872,"Ġprolifer":19873,"Ġaligned":19874,"xml":19875,"Ġtriv":19876,"Ġcoordination":19877,"ĠPER":19878,"ĠQuote":19879,"134":19880,"bf":19881,"ĠSaw":19882,"Ġtermination":19883,"Ġ190":19884,"Ġadditions":19885,"Ġtrio":19886,"Ġprojections":19887,"Ġpositively":19888,"Ġinclusive":19889,"Ġmembr":19890,"1990":19891,"older":19892,"Ġpracticed":19893,"inkle":19894,"Arch":19895,"Ġstarters":19896,"arius":19897,"Ġintermediate":19898,"ĠBenef":19899,"ĠKiller":19900,"Ġinterventions":19901,"ĠKil":19902,"ĠFlying":19903,"Inv":19904,"Ġpremature":19905,"Ġpsychiatric":19906,"Ġindie":19907,"Ġcollar":19908,"ĠRainbow":19909,"afi":19910,"Ġdisruption":19911,"ĠFOX":19912,"casting":19913,"Ġmisdem":19914,"cro":19915,"Ġwipe":19916,"ardon":19917,"Ġbast":19918,"ĠTommy":19919,"ĠRepresentative":19920,"Ġbelly":19921,"ĠPO":19922,"ĠBreitbart":19923,"132":19924,"Ġmessaging":19925,"Should":19926,"References":19927,"ĠGRE":19928,"istical":19929,"LP":19930,"ĠCav":19931,"ĠCrazy":19932,"Ġintuitive":19933,"keeping":19934,"ĠMoss":19935,"Ġdiscontin":19936,"ĠModule":19937,"Ġunrelated":19938,"ĠPractice":19939,"ĠTransport":19940,"Ġstatistically":19941,"orns":19942,"Ġsized":19943,"pu":19944,"Ġcaf":19945,"ĠWorlds":19946,"ĠRodgers":19947,"ĠLun":19948,"ĠComic":19949,"living":19950,"Ġcared":19951,"Ġclimbed":19952,"){":19953,"Ġconsisted":19954,"Ġmedieval":19955,"folk":19956,"Ġhacked":19957,"Ġdire":19958,"ĠHermione":19959,"Ġtended":19960,"ceans":19961,"Daniel":19962,"went":19963,"Ġlegislators":19964,"Ġredes":19965,"games":19966,"Ġgn":19967,"amiliar":19968,"Ġ++":19969,"ggy":19970,"threat":19971,"Ġmagnet":19972,"Ġperceive":19973,"Ġzip":19974,"Ġindictment":19975,"Ġcritique":19976,"gard":19977,"ĠSafe":19978,"ĠCream":19979,"Ġadvent":19980,"oba":19981,"Ġvowed":19982,"ousands":19983,"Ġski":19984,"Ġabortions":19985,"uart":19986,"Ġstunned":19987,"Ġadvancing":19988,"Ġlacked":19989,"Ġ\\\"":19990,"Ġschizophren":19991,"Ġelegant":19992,"Ġconferences":19993,"Ġcanceled":19994,"ĠHudson":19995,"ĠHopefully":19996,"Ġtrump":19997,"Ġfrequencies":19998,"Ġmeteor":19999,"ĠJunior":20000,"ĠFleet":20001,"ĠMalcolm":20002,"ĠTools":20003,"Ġ........":20004,"Ġhobby":20005,"ĠEuropeans":20006,"Ġ1500":20007,"ĠInto":20008,"Ġsway":20009,"ĠAppro":20010,"ĠCompl":20011,"Community":20012,"Ġtide":20013,"ĠSummit":20014,"ä»":20015,"Ġintervals":20016,"ĠEther":20017,"Ġhabitat":20018,"ĠStevens":20019,"lishing":20020,"ĠDomain":20021,"Ġtriggers":20022,"Ġchasing":20023,"Ġcharm":20024,"ĠFlower":20025,"itored":20026,"Ġblessing":20027,"Ġtextures":20028,"Five":20029,"Ġliquor":20030,"RP":20031,"FIN":20032,"Ġ1962":20033,"CAR":20034,"Unknown":20035,"Ġresil":20036,"ĠLily":20037,"Ġabundance":20038,"Ġpredictable":20039,"rar":20040,"Ġbullshit":20041,"leen":20042,"chet":20043,"Mor":20044,"Much":20045,"ä¹":20046,"Ġemphasized":20047,"Ġcrust":20048,"Ġprimitive":20049,"Ġenjoyable":20050,"ĠPictures":20051,"Ġteammate":20052,"pler":20053,"ĠTol":20054,"ĠKane":20055,"Ġsummoned":20056,"thy":20057,"rama":20058,"ĠHonda":20059,"Ġrealizing":20060,"Ġquicker":20061,"Ġconcentrate":20062,"clear":20063,"Ġ210":20064,"ĠErdogan":20065,"aris":20066,"Ġresponds":20067,"ĠBI":20068,"Ġeligibility":20069,"Ġpushes":20070,"ĠIdaho":20071,"Ġaggrav":20072,"Ġruins":20073,"urations":20074,"Ġbans":20075,"Ġanat":20076,"share":20077,"Ġgrind":20078,"hin":20079,"umen":20080,"Ġutilities":20081,"ĠYankees":20082,"Ġdatabases":20083,"ĠDD":20084,"Ġdisplaced":20085,"Ġdependencies":20086,"Ġstimulation":20087,"hun":20088,"houses":20089,"ĠPretty":20090,"ĠRavens":20091,"ĠTODAY":20092,"Ġassociates":20093,"Ġtherape":20094,"cled":20095,"Ġdeer":20096,"Ġrepairs":20097,"rentice":20098,"Ġreceptors":20099,"Ġremed":20100,"ĠCe":20101,"Ġmarriages":20102,"Ġballots":20103,"ĠSoldier":20104,"Ġhilarious":20105,"opl":20106,"138":20107,"Ġinherently":20108,"Ġignorant":20109,"Ġbounce":20110,"ĠEaster":20111,"RELATED":20112,"ĠCurrency":20113,"EV":20114,"ãĥŀ":20115,"ĠLead":20116,"Ġdeceased":20117,"Brien":20118,"ĠMusk":20119,"JS":20120,"Ġmerge":20121,"hearted":20122,"creat":20123,"mitt":20124,"mund":20125,"ĠâĢĭ":20126,"ĠBag":20127,"Ġprojection":20128,"Ġjava":20129,"ĠStandards":20130,"ĠLeonard":20131,"Ġcoconut":20132,"ĠPopulation":20133,"Ġtraject":20134,"Ġimply":20135,"Ġcuriosity":20136,"ĠDB":20137,"ĠFresh":20138,"ĠPor":20139,"Ġheavier":20140,"neys":20141,"gomery":20142,"Ġdeserved":20143,"Ġphrases":20144,"ĠGC":20145,"Ġyeast":20146,"desc":20147,"Death":20148,"Ġreboot":20149,"Ġmetadata":20150,"ICAL":20151,"Ġrepay":20152,"ĠIndependence":20153,"Ġsuburban":20154,"icals":20155,"Ġatop":20156,"Ġallocation":20157,"generation":20158,"ĠGram":20159,"Ġmoisture":20160,"Ġpine":20161,"ĠLiberals":20162,"Ġaides":20163,"Ġunderest":20164,"ĠBerry":20165,"Ġceremon":20166,"370":20167,"astrous":20168,"ĠPirates":20169,"Ġtense":20170,"ĠIndustries":20171,"ĠAppeals":20172,"ĠNear":20173,"Ġè£ıç":20174,"Ġlovers":20175,"ĠCAP":20176,"ĠCraw":20177,"Ġgiants":20178,"Ġefficacy":20179,"Element":20180,"ĠBehavior":20181,"ĠToyota":20182,"Ġintest":20183,"Priv":20184,"AI":20185,"Ġmaneuver":20186,"Ġperfection":20187,"Ġbang":20188,"paper":20189,"rill":20190,"George":20191,"border":20192,"inters":20193,"ĠSeth":20194,"Ġclues":20195,"ĠLevi":20196,"ĠRevenue":20197,"147":20198,"Ġvapor":20199,"Ġfortunate":20200,"Ġthreatens":20201,"Ġvet":20202,"Ġdependency":20203,"ersed":20204,"article":20205,"ĠBlizzard":20206,"Ġchlor":20207,"Ġminus":20208,"ĠBills":20209,"Ġcryptocurrency":20210,"Ġmetabolism":20211,"tering":20212,"Ġpestic":20213,"steps":20214,"ĠTreasure":20215,"racted":20216,"ĠConstant":20217,"Ġtemp":20218,"139":20219,"ĠDetective":20220,"urally":20221,"Ġrecovering":20222,"Ġcortex":20223,"Ġ144":20224,"closed":20225,"Ġprejudice":20226,"aunted":20227,"Ġstorms":20228,"ĠNOW":20229,"Ġmachinery":20230,"Address":20231,"Ġcompelled":20232,"270":20233,"Ġdespair":20234,"bane":20235,"Ġvegetable":20236,"Ġbeds":20237,"Learn":20238,"Ġcolorful":20239,"Ġspike":20240,"Ġmargins":20241,"Ġsympathy":20242,"Ġworkshop":20243,"ĠCBC":20244,"Sat":20245,"Ġburns":20246,"ĠGender":20247,"Ġ129":20248,"ĠCable":20249,"Ġdebts":20250,"ĠTheresa":20251,"Ġreflecting":20252,"Ġairst":20253,"Ġrim":20254,"ramid":20255,"Ġweaknesses":20256,"Writ":20257,"oggle":20258,"ti":20259,"ĠCharge":20260,"Ġweighed":20261,"Ġ(.":20262,"Ġlaughter":20263,"Ġrouter":20264,"ĠDemocracy":20265,"Dear":20266,"Ġhasht":20267,"Ġdy":20268,"Ġhints":20269,"running":20270,"Ġfinishes":20271,"arus":20272,"Mass":20273,"result":20274,"ascus":20275,"Ġvintage":20276,"Ġconqu":20277,"Ġwildly":20278,"acist":20279,"Ġlingu":20280,"Ġprotagonist":20281,"strom":20282,"teenth":20283,"ĠSolo":20284,"mac":20285,"filled":20286,"Ġrenown":20287,"itives":20288,"Ġmotive":20289,"ĠAntar":20290,"ĠMann":20291,"ĠAdjust":20292,"Ġrockets":20293,"Ġtroubling":20294,"ei":20295,"Ġorganisms":20296,"assis":20297,"Christian":20298,"Ġ145":20299,"ĠHass":20300,"Ġswall":20301,"Ġwax":20302,"ĠSurvival":20303,"VS":20304,"ĠMurd":20305,"vd":20306,"standard":20307,"Ġdragons":20308,"Ġacceleration":20309,"rational":20310,"final":20311,"Ġpaired":20312,"ĠEthereum":20313,"Ġinterfaces":20314,"Ġresent":20315,"Ġartifacts":20316,"Å«":20317,"arel":20318,"Ġcompetitor":20319,"ĠNicholas":20320,"ĠSurface":20321,"cpp":20322,"ĠTot":20323,"Ġeconomically":20324,"Ġorganised":20325,"Ġenforced":20326,"inho":20327,"Ġvarieties":20328,"Ġabdom":20329,"ĠBailey":20330,"idav":20331,"ĠSalv":20332,"paid":20333,"Ġaltitude":20334,"essert":20335,"ĠGutenberg":20336,"area":20337,"opoulos":20338,"Ġprofessors":20339,"iggs":20340,"ĠFate":20341,"hey":20342,"Ġ3000":20343,"Dist":20344,"Ġtwins":20345,"cill":20346,"ĠMaps":20347,"Ġtraps":20348,"Ġweed":20349,"ĠKiss":20350,"Ġyoga":20351,"Ġrecipients":20352,"ĠWestminster":20353,"Ġpools":20354,"ĠWalmart":20355,"188":20356,"ĠSchools":20357,"attack":20358,"ĠARM":20359,"paragraph":20360,"Warning":20361,"jl":20362,"Ġselfish":20363,"anchez":20364,"ĠHeights":20365,"Fre":20366,"ĠSoph":20367,"Ġ--------------------------------":20368,"tml":20369,"333":20370,"Ġraids":20371,"Ġsatellites":20372,"KEY":20373,"Ġlasts":20374,"ÑĤ":20375,"Ins":20376,"ĠDame":20377,"Ġunpredict":20378,"///":20379,"ghai":20380,"Ġartillery":20381,"Ġcruise":20382,"Ġgel":20383,"ĠCabinet":20384,"Ġblows":20385,"ĠEsp":20386,"Ġproximity":20387,"othe":20388,"ĠSkills":20389,"ĠUpper":20390,"obo":20391,"ĠNDP":20392,"Ġenjoys":20393,"Ġrepeating":20394,"ĠConstruction":20395,"ĠQuestions":20396,"Hillary":20397,"Ġuint":20398,"Ġprocessors":20399,"ĠGibson":20400,"ĠMultiple":20401,"qa":20402,"ĠBom":20403,"ĠMiles":20404,"ventional":20405,"Ġhurts":20406,"skin":20407,"ĠAIDS":20408,"Ġadvisers":20409,"ĠRoot":20410,"Ġmethodology":20411,"ĠDale":20412,"Ġdeton":20413,"ĠKnowledge":20414,"sequently":20415,"Ġ121":20416,"Ġconnects":20417,"Cy":20418,"ĠDanger":20419,"Ġcontributors":20420,"ĠBent":20421,"Ġbrass":20422,"ĠGuns":20423,"into":20424,"ĠFortune":20425,"Ġbroker":20426,"balance":20427,"Ġlengths":20428,"Ġvic":20429,"Ġaveraging":20430,"Ġappropriately":20431,"ĠCamera":20432,"Ġsandwich":20433,"ĠCDC":20434,"Ġcoordinate":20435,"Ġnavig":20436,"Ġgoodness":20437,"laim":20438,"Ġbrake":20439,"Ġextremist":20440,"ĠWake":20441,"ĠMend":20442,"ĠTiny":20443,"ĠCOL":20444,"ĠRF":20445,"ĠDual":20446,"ĠWine":20447,"Case":20448,"Ġrefined":20449,"Ġlamp":20450,"Lead":20451,"Ġbapt":20452,"ĠCarb":20453,"ĠSadd":20454,"ĠMinneapolis":20455,"PDF":20456,"Early":20457,"ĠHidden":20458,"Its":20459,"ĠTIME":20460,"Ġpap":20461,"Ġcommissioned":20462,"ĠFew":20463,"ĠColts":20464,"ĠBren":20465,"Ġbothered":20466,"Ġlikewise":20467,"Exper":20468,"ĠSchw":20469,"cry":20470,"nn":20471,"ĠMitch":20472,"imon":20473,"MG":20474,"bm":20475,"UMP":20476,"rays":20477,"Ġregistry":20478,"Ġ270":20479,"achine":20480,"rella":20481,"anting":20482,"00000":20483,"Ġruined":20484,"spot":20485,"Ġta":20486,"Ġmaximize":20487,"Ġinconven":20488,"Dead":20489,"Human":20490,"Enabled":20491,"ĠMarie":20492,"Ġchill":20493,"ĠParadise":20494,"Ġstarring":20495,"ĠLatino":20496,"ĠProtocol":20497,"ĠEVER":20498,"Ġsuppliers":20499,"message":20500,"ĠBrock":20501,"Ġserum":20502,"âĸĪâĸĪâĸĪâĸĪ":20503,"Ġencomp":20504,"Ġambition":20505,"uese":20506,"Ġarrows":20507,"Andrew":20508,"Ġantenna":20509,"Ġ1961":20510,"ĠBark":20511,"Ġbool":20512,"ãĤª":20513,"ĠStorage":20514,"Ġrailway":20515,"Ġtougher":20516,"ĠCad":20517,"Ġwashing":20518,"Py":20519,"']":20520,"embed":20521,"ĠMemphis":20522,"ackle":20523,"Ġfamously":20524,"ĠFortunately":20525,"ovies":20526,"Ġmindset":20527,"Ġsneak":20528,"ĠDh":20529,"RAW":20530,"ĠSimpson":20531,"Ġlivest":20532,"Ġlandmark":20533,"Ġcement":20534,"Low":20535,"Ġthrilled":20536,"ĠCourse":20537,"inel":20538,"Ġchuck":20539,"idate":20540,"global":20541,"Ġwhit":20542,"Ġ�":20543,"adays":20544,"ski":20545,"ĠSV":20546,"Ġviruses":20547,"306":20548,"ĠRespons":20549,"Ġtheaters":20550,"ĠBranch":20551,"ĠGeneva":20552,"ĠMK":20553,"Ġunbeliev":20554,"Ġcommunist":20555,"Original":20556,"ĠReceived":20557,"ĠTransfer":20558,"ĠArg":20559,"Input":20560,"ĠStrategy":20561,"Ġpalace":20562,"thening":20563,"Dri":20564,"Ġsentencing":20565,"umbnail":20566,"Ġpins":20567,"recy":20568,"Ġsiblings":20569,"Getting":20570,"ĠBU":20571,"ĠNorthwest":20572,"Ġprolonged":20573,"ĠSakura":20574,"Comb":20575,"ĠBour":20576,"Ġinadequate":20577,"ĠKash":20578,"Ġusername":20579,"ĠImprove":20580,"Ġbattling":20581,"ĠMAC":20582,"Ġcurriculum":20583,"Ġsoda":20584,"ĠCannon":20585,"Ġsensible":20586,"spons":20587,"December":20588,"Ġwicked":20589,"ĠPengu":20590,"Ġdictators":20591,"ĠHearts":20592,"ogyn":20593,"Ġsimilarities":20594,"ĠStats":20595,"Ġhollow":20596,"itations":20597,"\":[":20598,"Ġhover":20599,"ĠListen":20600,"sch":20601,"Sund":20602,"Ġcad":20603,"ĠParks":20604,"Ġlur":20605,"Ġhype":20606,"ĠLem":20607,"NAME":20608,"isure":20609,"Friday":20610,"Ġshoots":20611,"Ġcloses":20612,"Ġdb":20613,"ĠRidge":20614,"ĠDifferent":20615,"Ġreplies":20616,"ĠBroadway":20617,"opers":20618,"Ġintoler":20619,"ĠZeus":20620,"akespe":20621,"Ġproprietary":20622,"Ġrequesting":20623,"Ġcontrollers":20624,"ĠMIN":20625,"imedia":20626,"becca":20627,"Ġexpans":20628,"Ġoils":20629,"Bot":20630,"ĠChand":20631,"Ġprinter":20632,"Ġtopped":20633,"ĠPOL":20634,"ĠEarlier":20635,"Social":20636,"avin":20637,"Ġdecreases":20638,"ĠSeb":20639,"Ġspecifications":20640,"ĠBlast":20641,"ĠKurt":20642,"Ġfreel":20643,"Brown":20644,"Ġdilig":20645,"roe":20646,"ĠProblem":20647,"ĠQuad":20648,"Ġdecentral":20649,"ĠVector":20650,"anut":20651,"Ġplugins":20652,"ĠGregory":20653,"Ġfucked":20654,"elines":20655,"ĠAmbassador":20656,"take":20657,"Ġcleans":20658,"ongyang":20659,"Anonymous":20660,"stro":20661,"\"}":20662,"aline":20663,"ĠOdd":20664,"ĠEug":20665,"216":20666,"Ġboil":20667,"ĠPowers":20668,"Ġnurses":20669,"Obviously":20670,"ĠTechnical":20671,"Ġexceeded":20672,"ORS":20673,"Ġextremists":20674,"Ġtraces":20675,"expl":20676,"Ġcomr":20677,"ĠSach":20678,")/":20679,"Ġmasks":20680,"Ġsci":20681,"Bon":20682,"Ġregression":20683,"wegian":20684,"Ġadvisor":20685,"itures":20686,"ĠVo":20687,"example":20688,"ĠInstruct":20689,"Ġsiege":20690,"Ġreductions":20691,"ptr":20692,"Ġstatutory":20693,"Ġremoves":20694,"Ġpuck":20695,"redits":20696,"Ġbee":20697,"Ġsalad":20698,"Ġpromotions":20699,"ĠJoshua":20700,"withstanding":20701,"ETH":20702,"ĠCha":20703,"imus":20704,"Ġexpenditure":20705,"aunting":20706,"Ġdelighted":20707,"Ġ155":20708,"beh":20709,"Ġcarpet":20710,"ĠSpart":20711,"Ġjungle":20712,"lists":20713,"Ġbullying":20714,"ĠNobel":20715,"ĠGlen":20716,"Ġreferenced":20717,"Ġintroduces":20718,"sein":20719,"Ġchopped":20720,"glass":20721,"ĠWrest":20722,"Ġneutrality":20723,"ĠâĻ":20724,"Ġinvestigator":20725,"Ġshelves":20726,"Ġunconstitutional":20727,"Ġreproduction":20728,"Ġmerchant":20729,"mia":20730,"Ġmetrics":20731,"Ġexplosives":20732,"ĠSonia":20733,"Ġbodily":20734,"Ġthickness":20735,"Ġpredominantly":20736,"ĠAbility":20737,"Ġmonitored":20738,"ICH":20739,"Ġ].":20740,"ĠMartinez":20741,"Ġvisibility":20742,"Ġqueries":20743,"Ġgenocide":20744,"ĠWarfare":20745,"Query":20746,"Ġstudios":20747,"Ġembry":20748,"Ġcorridor":20749,"Ġcleaned":20750,"complete":20751,"ĠMH":20752,"Ġenrollment":20753,"INGS":20754,"Ġimpacted":20755,"Ġdisastrous":20756,"ĠYun":20757,"ĠClaire":20758,"ĠBasically":20759,"yt":20760,"usterity":20761,"Ġindirectly":20762,"wik":20763,"Ġdod":20764,"ĠCarr":20765,"Ġamp":20766,"Ġprohibit":20767,"ĠInitial":20768,"ĠRd":20769,"iji":20770,"Ġeducate":20771,"corn":20772,"iott":20773,"ĠBeauty":20774,"Ġdetective":20775,"ĠConn":20776,"since":20777,"Ġstagger":20778,"Ġobese":20779,"Ġbree":20780,"ologic":20781,"isse":20782,"walker":20783,"Ġblades":20784,"Ġlawful":20785,"func":20786,"ĠBehind":20787,"Ġappetite":20788,"Ġ(*":20789,"Ġtennis":20790,"Ġoffspring":20791,"Ġjets":20792,"Ġstructured":20793,"Ġaforementioned":20794,"Nov":20795,"Ġscaling":20796,"fill":20797,"Ġstew":20798,"Ġcurb":20799,"ĠStephan":20800,"edIn":20801,"SF":20802,"obic":20803,"éŃĶ":20804,"oug":20805,"ĠMM":20806,"Ġgenetically":20807,"opez":20808,"136":20809,"Ġumb":20810,"ancers":20811,"Ġcohort":20812,"Ġmerchandise":20813,"Ġimposing":20814,"ĠLegislature":20815,"ĠArchive":20816,"ivia":20817,"ĠNaval":20818,"Ġoffences":20819,"Ġmiracle":20820,"Ġsnapped":20821,"Ġfoes":20822,"Ġextensively":20823,"ĠRaf":20824,"Ġcater":20825,"edience":20826,"Kit":20827,"ĠBin":20828,"Ġrecommends":20829,"ĠCities":20830,"Ġrigid":20831,"ĠREAD":20832,"ĠNoble":20833,"ĠTian":20834,"Ġcertificates":20835,"antis":20836,"oiler":20837,"ĠBuddhist":20838,"did":20839,"Ġsurveyed":20840,"Ġdownward":20841,"Ġprints":20842,"ĠMotion":20843,"ronics":20844,"ĠSans":20845,"ossibly":20846,"uctions":20847,"Ġcolonies":20848,"ĠDanish":20849,"unit":20850,"Ġspoil":20851,"Ġadvisory":20852,"berries":20853,"Plan":20854,"Ġspecification":20855,"ophers":20856,"ĠResource":20857,"Ġshirts":20858,"prisingly":20859,"communications":20860,"Ġtrivial":20861,"Ġmentioning":20862,"isexual":20863,"Ġsupplements":20864,"Ġsupervision":20865,"BP":20866,"vor":20867,"Ġwit":20868,"Ġcooldown":20869,"Ġplaintiff":20870,"ĠReviews":20871,"ĠSri":20872,"ĠMint":20873,"ĠSugar":20874,"Ġafterward":20875,"ĠPriest":20876,"ĠInvestment":20877,"ogene":20878,"ĠTaking":20879,"Ġstretching":20880,"Ġinflammation":20881,"ĠTehran":20882,"Ġlining":20883,"Ġfreezing":20884,"ĠEntity":20885,"Ġinspiring":20886,"special":20887,"price":20888,"Ġsue":20889,"ĠPorter":20890,"ounge":20891,"ETA":20892,"ĠDerek":20893,"ĠLuis":20894,"uo":20895,"ymph":20896,"Ġexterior":20897,"ihil":20898,"ĠAshley":20899,"inator":20900,"Ġnutrients":20901,"ĠThrones":20902,"Ġfinances":20903,"ĠInspect":20904,"Ġspecially":20905,"ĠRequired":20906,"ĠPTS":20907,"ĠViolence":20908,"ointed":20909,"shots":20910,"Ġexcerpt":20911,"coon":20912,"INS":20913,"ĠGri":20914,"Ġrecognised":20915,"Week":20916,"Young":20917,"Ġvom":20918,"isle":20919,"ĠCurry":20920,"ĠBuddh":20921,"Ġnotebook":20922,"Ġdurable":20923,"/?":20924,"ĠGad":20925,"ĠPupp":20926,"Ġforgive":20927,"park":20928,"Ġpersonalities":20929,"analysis":20930,"clamation":20931,"Ġelevator":20932,"Ġwarehouse":20933,"ĠRole":20934,"unn":20935,"Ġillustration":20936,"ĠScan":20937,"Ġatmospheric":20938,"Import":20939,"ANC":20940,"ricted":20941,"fu":20942,"010":20943,"Ġarche":20944,"Ġrewarded":20945,"akespeare":20946,"Ġinternally":20947,"ĠRBI":20948,"alker":20949,"Ġelephant":20950,"owitz":20951,"ĠPizza":20952,"Ġbipartisan":20953,"és":20954,"Ġslowed":20955,"ĠStark":20956,"Ġoverride":20957,"OUS":20958,"Ġ320":20959,"undreds":20960,"ĠDeck":20961,"ĠCensus":20962,"bee":20963,"146":20964,"otor":20965,"Ġip":20966,"Ġub":20967,"ocations":20968,"ĠButton":20969,"rice":20970,"Ġcripp":20971,"fff":20972,"Ġoriginated":20973,"Ġoverwhelmed":20974,"appa":20975,"Ġforemost":20976,"âĢij":20977,"ĠLEG":20978,"release":20979,"eatured":20980,"atches":20981,"Ġreps":20982,"Ġlending":20983,"ĠReference":20984,"ĠClient":20985,"165":20986,"venth":20987,"Complete":20988,"ĠPatrol":20989,"Ġsworn":20990,"cam":20991,"Ġshuttle":20992,"ĠRalph":20993,"Ġhometown":20994,"-,":20995,"onal":20996,"ĠBP":20997,"åı":20998,"Ġpersuade":20999,"ĠAlexand":21000,"Ġcombines":21001,"Ġvivid":21002,"ĠLag":21003,"Ġencoding":21004,"Ġsalvation":21005,"wen":21006,"ĠRecovery":21007,"iya":21008,"University":21009,"ĠBiden":21010,"Ġbudgets":21011,"ĠTexans":21012,"fits":21013,"Ġhonored":21014,"Ġpython":21015,"TD":21016,"###":21017,"clone":21018,"Ġblink":21019,"ĠLiquid":21020,"Ġunemployed":21021,"Ġclashes":21022,"ĠCounsel":21023,"Ġdirecting":21024,"Ġpunct":21025,"ĠFalcons":21026,"Ġshark":21027,"ĠDamascus":21028,"Ġjeans":21029,"Ġembark":21030,"Ġseize":21031,"Ġupwards":21032,"280":21033,"ĠEz":21034,"ĠAnything":21035,"Ġexotic":21036,"lower":21037,"ĠCreator":21038,"ĠUm":21039,"Ġsuburbs":21040,"berger":21041,"ĠWend":21042,"Ġmint":21043,"ĠXX":21044,"ĠDro":21045,"Ġsuffers":21046,"Ġherb":21047,"tree":21048,"Ġfragile":21049,"Ġflooded":21050,"ĠAlcohol":21051,"olean":21052,"nyder":21053,"ĠKO":21054,"Fram":21055,"Ġ136":21056,"Ġowed":21057,"ĠMelee":21058,"ĠHash":21059,"Ġwhisk":21060,"Ġsudo":21061,"rr":21062,"Quick":21063,"appro":21064,"Ġii":21065,"ĠExamples":21066,"hee":21067,"Ġpromotes":21068,"perature":21069,"kar":21070,"ĠHonor":21071,"Ġsodium":21072,"ĠLif":21073,"rosso":21074,"intendent":21075,"Ġcorrespondent":21076,"Found":21077,"secret":21078,"Ġidentifies":21079,"agne":21080,"Ġlou":21081,"ĠPP":21082,"Ġcoincidence":21083,"move":21084,"Ġmilitia":21085,"Ġinfiltr":21086,"ĠPrimary":21087,"Ġpitching":21088,"ĠIb":21089,"ĠGOOD":21090,"ãĤ¸":21091,"ĠWizards":21092,"iral":21093,"ĠVenus":21094,"RR":21095,"ĠâĢķ":21096,"ĠCasey":21097,"Ġsadly":21098,"Ġadmire":21099,"Ġembarrassed":21100,"cb":21101,"Mel":21102,"Ġtubes":21103,"Ġbeautifully":21104,"ĠQueensland":21105,"Below":21106,"rez":21107,"quet":21108,"pleasant":21109,"Ġ«":21110,"Camp":21111,"Ġdecisive":21112,"1998":21113,"ĠLamb":21114,"utton":21115,"hn":21116,"ĠJagu":21117,"aunder":21118,"ĠCord":21119,"Ġclerk":21120,"Ġcaffe":21121,"Ġwiped":21122,"Ġreim":21123,"ĠMountains":21124,"Ġimprisoned":21125,"Ġdevelops":21126,"ĠPra":21127,"Ġmodeling":21128,"Anyone":21129,"ancel":21130,"ĠSit":21131,"Ġshields":21132,"Ġlawn":21133,"Ġcardiovascular":21134,"Ġdemonstrating":21135,"Ġparse":21136,"ĠIsraelis":21137,"Ġeuros":21138,"143":21139,"Ġglorious":21140,"inski":21141,"ecd":21142,"Ġconditioning":21143,"Ġhelpless":21144,"Ġmicrosc":21145,"ĠHarbor":21146,"Ġstakes":21147,"Ġ260":21148,"Ġunequ":21149,"ĠFloyd":21150,"Ġdamp":21151,"Ġapparatus":21152,"ĠLaws":21153,"Ġcounters":21154,"Ġinduce":21155,"atable":21156,"ĠAhmed":21157,"Ġslam":21158,"November":21159,"Ġpersist":21160,"Ġimminent":21161,"án":21162,"Ġshred":21163,"Ġphases":21164,"ĠEdmonton":21165,"ĠArmstrong":21166,"ĠMeet":21167,"ĠKitty":21168,"ÑĢ":21169,"circ":21170,"ĠAdult":21171,"Ġarose":21172,"ĠXen":21173,"Dan":21174,"gow":21175,"Ġsuperf":21176,"ĠAdmir":21177,"Ġendure":21178,"Ġkeyword":21179,"yrus":21180,"Ġyarn":21181,"Ġpathway":21182,"ĠHopkins":21183,"midt":21184,"Ġcensorship":21185,"dependent":21186,"Ġinstructor":21187,"Sources":21188,"Ġtoe":21189,"Ġballoon":21190,"Nob":21191,"Ġswear":21192,"ĠCastro":21193,"Ġgloss":21194,"ĠKavanaugh":21195,"Ġremarkably":21196,"Photos":21197,"ĠNom":21198,"ĠSoutheast":21199,"yers":21200,"Ġvalidation":21201,"Ġcannon":21202,"ĠVictory":21203,"ĠPierre":21204,"Ġcautious":21205,"Audio":21206,"Ġfetch":21207,"ĠGift":21208,"ĠHyp":21209,"Ġremedy":21210,"ZE":21211,"Ġscent":21212,"Ġbeard":21213,"ĠRut":21214,"-\"":21215,"Ġpatents":21216,"Hy":21217,"Ġunjust":21218,"Ġpotato":21219,"Ġforthcoming":21220,"Ġchef":21221,"ĠRift":21222,"affe":21223,"ĠROM":21224,"ĠLaunch":21225,"Ġpads":21226,"ĠNeo":21227,"Ġonset":21228,"Ġsqueeze":21229,"safe":21230,"Ġprefix":21231,"ĠTM":21232,"ĠNearly":21233,"ĠClinical":21234,"ĠMental":21235,"otiation":21236,"ĠUnic":21237,"antry":21238,"ĠCir":21239,"Ġepit":21240,"æ":21241,"Ġextracted":21242,"versely":21243,"riad":21244,"Ġstrains":21245,"Ġtops":21246,"Ġpoem":21247,"ĠRandy":21248,"ĠMaple":21249,"THER":21250,"upiter":21251,"ĠSSD":21252,"ļé":21253,"Ġuncon":21254,"pering":21255,"Ġslept":21256,"iners":21257,"Ġunderwater":21258,"ĠEvidence":21259,"gone":21260,"205":21261,"Ġhistorians":21262,"Ġsynthesis":21263,"Ġfrog":21264,"basketball":21265,"Ġvibrant":21266,"Ġsubord":21267,"Ġ365":21268,"ĠDial":21269,"Ġcooperate":21270,"HAHA":21271,"Ġgreeted":21272,"158":21273,"Ġjazz":21274,"Ġintox":21275,"ĠWalking":21276,"Ġsupervisor":21277,"ĠFusion":21278,"ĠMercedes":21279,"send":21280,"Ham":21281,"sd":21282,"nl":21283,"Ġtours":21284,"ĠFIFA":21285,"Ġculp":21286,"gd":21287,"304":21288,"Ġpleas":21289,"Ġillustrates":21290,"ĠColombia":21291,"Ġhighlighting":21292,"ĠSummary":21293,"Ġexposing":21294,"ĠDru":21295,"Ġirony":21296,"ritional":21297,"ĠCarroll":21298,"ĠEllis":21299,"Pict":21300,"ĠRapt":21301,"Ġadapter":21302,"Ġunm":21303,"Ġcorpse":21304,"Ġcelebrities":21305,"Den":21306,"atum":21307,"ĠApocalypse":21308,"ĠWag":21309,"lining":21310,"Ġhormones":21311,"Rub":21312,"ĠXi":21313,"ĠVaults":21314,"208":21315,"alkyrie":21316,"inosaur":21317,"Ġfeeds":21318,"vity":21319,"Ġdefeating":21320,"Wait":21321,"Ġemphasize":21322,"ĠSteelers":21323,"yrinth":21324,"leys":21325,"ĠWhenever":21326,"Currently":21327,"ĠClock":21328,"Ġcollectively":21329,"anyon":21330,"ĠJP":21331,"Ġmentality":21332,"Ġdownloads":21333,"Ġsurroundings":21334,"ĠBarnes":21335,"Ġflagship":21336,"Ġindicators":21337,"Ġgrapp":21338,"January":21339,"ĠElemental":21340,"ĠAthena":21341,"ibal":21342,"Ġsights":21343,"Ġcapita":21344,"ĠTreaty":21345,"Ġvoiced":21346,"ĠGaz":21347,"lette":21348,"Ġya":21349,"Ġexpired":21350,"Legend":21351,"Hot":21352,"nature":21353,"Ġunstable":21354,"Ġ280":21355,"ú":21356,"Comment":21357,"ALE":21358,"Ġquests":21359,"Ġhandler":21360,"nis":21361,"Ġversatile":21362,"Ġconceal":21363,"engeance":21364,"ĠInteractive":21365,"Ġobsessed":21366,"ĠDogs":21367,"Ġcracked":21368,"Sound":21369,"sv":21370,"ĠDylan":21371,"roads":21372,"fx":21373,"ĠCatholics":21374,"ĠHag":21375,"Ġslammed":21376,"Ġglowing":21377,"sale":21378,"Ġtissues":21379,"ĠChi":21380,"nee":21381,"Ġcher":21382,"sic":21383,"urrection":21384,"Ġbacon":21385,"ulatory":21386,").\"":21387,"Ġirregular":21388,"FORM":21389,"assed":21390,"Ġintentional":21391,"Ġcompensate":21392,"ĠSpeaking":21393,"ĠSets":21394,"153":21395,"Ġconventions":21396,"bands":21397,"emade":21398,"Ġecc":21399,"ĠWinston":21400,"ĠAssassin":21401,"ĠBelgian":21402,"Ġdependence":21403,"Ġniche":21404,"Ġbark":21405,"ĠJazz":21406,"Ġdisadvantage":21407,"Ġgasoline":21408,"Ġ165":21409,"çļĦ":21410,"essa":21411,"module":21412,"angular":21413,"OY":21414,"ĠTreatment":21415,"itas":21416,"olation":21417,"ĠArnold":21418,"Ġfeud":21419,"ĠNest":21420,"Ġtheatre":21421,"ewater":21422,"Ġminors":21423,"olicy":21424,"ĠHaven":21425,"division":21426,"Ġtrunk":21427,"Far":21428,"ĠPull":21429,"Ġcapturing":21430,"Ġ1800":21431,"ĠTeen":21432,"Ġexempl":21433,"Ġclinics":21434,"ĠBurg":21435,"Ġsubstit":21436,"Ġpayload":21437,"ĠLav":21438,"ĠTroy":21439,"ĠWitness":21440,"Ġfragments":21441,"Ġpasswords":21442,"Ġgospel":21443,"ĠGin":21444,"Ġtenants":21445,"olith":21446,"Six":21447,"Previous":21448,"ĠAges":21449,"ĠDarwin":21450,"Ġblat":21451,"Ġempathy":21452,"smith":21453,"bag":21454,"ĠEcho":21455,"ĠCamb":21456,"ĠMadd":21457,"ĠBoo":21458,"Ġrede":21459,"ĠBurning":21460,"Ġsmoothly":21461,"ĠAdrian":21462,"ĠVampire":21463,"ĠMonsters":21464,"steam":21465,"Style":21466,"Ma":21467,"rea":21468,"ĠDwar":21469,"alyst":21470,"ursor":21471,"Ġelimination":21472,"Ġcrypto":21473,"cht":21474,"ĠEternal":21475,"âĢ¦]":21476,"ĠSorce":21477,"Ill":21478,"NER":21479,"Ġuh":21480,"Conclusion":21481,"wage":21482,"Ġrespir":21483,"Ġreminis":21484,"hetical":21485,"Ġgy":21486,"Ġutilized":21487,"icidal":21488,"Ġ1900":21489,"Ġhunters":21490,"ĠSwan":21491,"ĠReact":21492,"Ġvisitor":21493,"ĠThanksgiving":21494,"308":21495,"Posts":21496,"Ġhips":21497,"1997":21498,"omers":21499,"Ġknocking":21500,"ĠVehicle":21501,"Ġtil":21502,"Ġ138":21503,"Ġmi":21504,"ĠInvestigation":21505,"ĠKenya":21506,"Ġcasino":21507,"Ġmotives":21508,"Ġregain":21509,"rex":21510,"Ġweekends":21511,"Ġstabbed":21512,"boro":21513,"Ġexploited":21514,"ĠHAVE":21515,"ĠTelevision":21516,"cock":21517,"Ġpreparations":21518,"Ġendeav":21519,"ĠRemote":21520,"ĠMaker":21521,"ĠProdu":21522,"ĠEvan":21523,"Ġinformational":21524,"ĠLouisville":21525,"154":21526,"ĠDreams":21527,"Ġplots":21528,"ĠRunner":21529,"Ġhurting":21530,"Ġacademy":21531,"ĠMontgomery":21532,"nm":21533,"ĠLanc":21534,"ĠAlz":21535,"210":21536,"elong":21537,"Ġretailer":21538,"Ġarising":21539,"Ġrebellion":21540,"Ġblonde":21541,"played":21542,"Ġinstrumental":21543,"Cross":21544,"Ġretention":21545,"Ġtherapeutic":21546,"Ġseas":21547,"Ġinfantry":21548,"ĠClint":21549,"Ġprompting":21550,"Ġbitch":21551,"Ġstems":21552,"ĠKra":21553,"Ġthesis":21554,"ĠBog":21555,"rued":21556,"Ġkings":21557,"Ġclay":21558,"ificent":21559,"ĠYES":21560,"ĠThing":21561,"ĠCubs":21562,"veyard":21563,"elsh":21564,"inarily":21565,"ĠEy":21566,"ĠRolling":21567,"Ġevolving":21568,"India":21569,"Ġrecognizes":21570,"Ġgraduation":21571,"isers":21572,"Ġfertility":21573,"ĠMilan":21574,"Command":21575,"Ġboxing":21576,"Ġ1943":21577,"Ġgluten":21578,"ĠEmir":21579,"Ġidol":21580,"Ġconceived":21581,"ĠCreation":21582,"Merit":21583,"uddy":21584,"ussions":21585,"ĠLieutenant":21586,"ietal":21587,"Ġunchanged":21588,"ĠScale":21589,"ĠCrimea":21590,"balls":21591,"atorial":21592,"Ġdepths":21593,"Ġempirical":21594,"Ġtransm":21595,"Ġunsafe":21596,"missible":21597,"comfort":21598,"156":21599,"Ġmechanic":21600,"002":21601,"lins":21602,"Ġsmoked":21603,"Pos":21604,"Ġslowing":21605,"Ġlav":21606,"Texas":21607,"Ġcheating":21608,"ĠMetropolitan":21609,"ethyl":21610,"Ġdiscovering":21611,"asse":21612,"Ġpencil":21613,"ĠPyongyang":21614,"Ġcloset":21615,"ĠSheet":21616,"ĠEntry":21617,"oustic":21618,"Ġmyst":21619,"erate":21620,"ariat":21621,"Ġminerals":21622,"Ġmusician":21623,"ĠPul":21624,"ĠMaz":21625,"249":21626,"Ġpermissions":21627,"Ġiv":21628,"enary":21629,"ickers":21630,"ĠBing":21631,"hea":21632,"enable":21633,"Ġgriev":21634,"Ġasserted":21635,"ĠColonel":21636,"Ġaffidav":21637,"wo":21638,"Ġseated":21639,"ĠRide":21640,"Ġpaintings":21641,"ĠPix":21642,"Ġ137":21643,"ishi":21644,"umbai":21645,"gotten":21646,"ĠEarl":21647,"Ġinning":21648,"Ġcensus":21649,"Ġtravelled":21650,"ĠConsult":21651,"185":21652,"bind":21653,"Ġsimplicity":21654,"Ġoverlooked":21655,"ĠHelpful":21656,"Ġmonkey":21657,"Ġoverwhelmingly":21658,"Blood":21659,"ĠFlint":21660,"ĠJama":21661,"ĠPresent":21662,"ĠRage":21663,"ĠTA":21664,"ptive":21665,"Ġturnout":21666,"wald":21667,"ĠDolphins":21668,"ĠVPN":21669,"Ġonion":21670,"Ġcrafting":21671,"mma":21672,"ĠMercury":21673,"Ġarrange":21674,"Ġalerts":21675,"ĠOT":21676,"zbollah":21677,"Ġgases":21678,"ĠRichardson":21679,"sal":21680,"lar":21681,"Ġfrost":21682,"Ġlowering":21683,"Ġacclaim":21684,"Ġstartups":21685,"ĠGain":21686,"essment":21687,"Ġguardian":21688,"人":21689,"ĠPie":21690,"ĠLinks":21691,"Ġmerits":21692,"Ġawake":21693,"Ġparental":21694,"Ġexceeds":21695,"Ġidle":21696,"ĠPilot":21697,"ĠeBay":21698,"ĠAccept":21699,"ipeg":21700,"Cam":21701,"ĠKot":21702,"Ġtraders":21703,"olitics":21704,"unker":21705,"ĠPale":21706,"osi":21707,"anmar":21708,"Ġ1947":21709,"ĠFell":21710,"estial":21711,"itating":21712,"GF":21713,"ĠSr":21714,"ifted":21715,"Ġconnector":21716,"ĠBone":21717,"illes":21718,"260":21719,"hma":21720,"Ġoverlap":21721,"ĠGitHub":21722,"Ġcleaner":21723,"ĠBaptist":21724,"ĠWAS":21725,"Ġlungs":21726,"Ñģ":21727,"ĠBUT":21728,"Ġcite":21729,"Ġpitched":21730,"reatment":21731,"Ġtrophies":21732,"ĠNu":21733,"386":21734,"ĠPride":21735,"Ġattendees":21736,"[]":21737,"179":21738,"Ġspatial":21739,"Ġprizes":21740,"ĠReligion":21741,"Ġshowcase":21742,"ĠCategory":21743,"vidia":21744,"Target":21745,"Property":21746,"?,":21747,"Ġfusion":21748,"pie":21749,"ĠUCLA":21750,"Ġsoundtrack":21751,"Ġprincess":21752,"ĠCaval":21753,"should":21754,"Ġlimbs":21755,"Background":21756,"Ġlonely":21757,"Ġcores":21758,"ĠTail":21759,"sheet":21760,"Ġ132":21761,"Ra":21762,"ãĤ«":21763,"ĠBolt":21764,"Ġbooked":21765,"Ġadminister":21766,"Ġequals":21767,"wy":21768,"Ġobserving":21769,"ĠBaron":21770,"ĠAdobe":21771,"Ġvirgin":21772,"ĠSocialist":21773,"Move":21774,"ghazi":21775,"ĠLinda":21776,"212":21777,"Ġbrewing":21778,"Ġmerchants":21779,"burse":21780,"Ġdivor":21781,"Ġmetals":21782,"ĠNer":21783,"Ġsums":21784,"ĠEnemy":21785,"Ġenvision":21786,"Ġgranting":21787,"ĠHoney":21788,"ĠSkyrim":21789,"Ġsocio":21790,"graded":21791,"Ġselective":21792,"WASHINGTON":21793,"Ġ1948":21794,"ĠSirius":21795,"ĠGross":21796,"activity":21797,"ĠIvan":21798,"Ġfurious":21799,"BSD":21800,"ĠPrevious":21801,"Ġresponsive":21802,"Ġcharitable":21803,"Ġleaning":21804,"ĠPew":21805,"Ġviolates":21806,"\\\\\\\\\\\\\\\\":21807,"ĠComing":21808,"wire":21809,"Ġpoet":21810,"Ġresolutions":21811,"command":21812,"ĠPortuguese":21813,"Ġnickname":21814,"Ġdeaf":21815,"February":21816,"Ġrecognise":21817,"Ġentirety":21818,"Ġseasonal":21819,"placed":21820,"ĠTelegraph":21821,"Ġmicrophone":21822,"ouring":21823,"Ġgrains":21824,"Ġgoverned":21825,"Ġpostp":21826,"ĠWaters":21827,"inement":21828,"Ġundocumented":21829,"ĠComcast":21830,"Ġfox":21831,"Ġassaults":21832,"reon":21833,"many":21834,"ĠJenkins":21835,"ĠAnyway":21836,"Ġassessments":21837,"Ġdowns":21838,"ĠMouse":21839,"Ġsuperb":21840,"kt":21841,"ĠDow":21842,"Ġtaxation":21843,"401":21844,"Ġsmiles":21845,"Ġundertaken":21846,"Ġexh":21847,"Ġenthusiastic":21848,"Ġtwent":21849,"Ġgovernmental":21850,"Ġautonomy":21851,"ĠTechnologies":21852,"ĠChain":21853,"Ġprevalent":21854,"fb":21855,"Ġnicotine":21856,"ogram":21857,"job":21858,"Ġawaiting":21859,"ĠMenu":21860,"Ġdeputies":21861,"kov":21862,"ishops":21863,"Button":21864,"ĠShanghai":21865,"Ġdiesel":21866,"ĠDuck":21867,"Ryan":21868,"ĠPCs":21869,"NF":21870,"jury":21871,"ente":21872,"Ġinaccurate":21873,"eddy":21874,"Whatever":21875,"Ġshowc":21876,"ĠNad":21877,"odus":21878,"etr":21879,"Ġplaintiffs":21880,"ĠWOR":21881,"ĠAssange":21882,"Ġprivat":21883,"Ġpremiums":21884,"Ġtam":21885,"URL":21886,"Ġelites":21887,"ĠRanger":21888,"ottenham":21889,"ĠHoff":21890,"ĠAthens":21891,"Ġdefinite":21892,"Ġsighed":21893,"Ġevenly":21894,"211":21895,"ĠAmber":21896,"akia":21897,"Ġmailing":21898,"Ġcrashing":21899,"ĠConfederate":21900,"rugged":21901,"Wal":21902,"ĠDepths":21903,"Ġjuvenile":21904,"Ġreactor":21905,"Introduction":21906,"ĠDeluxe":21907,"1995":21908,"ĠSanchez":21909,"ĠMead":21910,"ivable":21911,":-":21912,"ĠPlanning":21913,"ĠTrap":21914,"quin":21915,"ĠProtect":21916,"vered":21917,"Information":21918,"Ġkidney":21919,"innamon":21920,"las":21921,"Ġpolicing":21922,"Ġtolerate":21923,"ĠQi":21924,"Ġbiased":21925,"Fort":21926,"ĠKi":21927,"save":21928,"Ġprivileged":21929,"Ġbeasts":21930,"ĠGlas":21931,"ĠCinem":21932,"Ġcomeback":21933,"Sunday":21934,"Ġextinction":21935,"hops":21936,"Ġtransmit":21937,"Ġdoubles":21938,"ĠFlat":21939,"167":21940,"Ġdisputed":21941,"Ġinjustice":21942,"foo":21943,"Vict":21944,"roleum":21945,"ĠJulie":21946,"Context":21947,"ĠRarity":21948,"issue":21949,"Component":21950,"Ġcounseling":21951,"anne":21952,"dark":21953,"Ġobjections":21954,"uilt":21955,"Ġgast":21956,"Ġplac":21957,"Ġunused":21958,"ãĥĩ":21959,"ĠTrial":21960,"ĠJas":21961,"hedral":21962,"obb":21963,"Ġtemporal":21964,"ĠPRO":21965,"ĠNW":21966,"ĠAnniversary":21967,"Large":21968,"Ġtherm":21969,"Ġdavid":21970,"Ġsystemic":21971,"ĠShir":21972,"mut":21973,"ĠNept":21974,"address":21975,"Ġscanning":21976,"Ġunderstandable":21977,"Ġcanvas":21978,"Cat":21979,"ĠZoo":21980,"Ġangels":21981,"LO":21982,"ĠStatement":21983,"ĠSig":21984,"ovable":21985,"ĠAway":21986,"sharing":21987,"ocrats":21988,"stated":21989,"Ġweighing":21990,"Nor":21991,"wild":21992,"Bey":21993,"Ġastonishing":21994,"ĠReynolds":21995,"Ġopener":21996,"Ġtrainer":21997,"Ġsurgical":21998,"pn":21999,"Ġadjusting":22000,"wheel":22001,"Ġfrown":22002,"ervative":22003,"Ġsuspend":22004,"Within":22005,"tein":22006,"Ġobstacle":22007,"Ġliberties":22008,"ymes":22009,"Ġuranium":22010,"ansom":22011,"anol":22012,"uba":22013,"ĠLoss":22014,"Ġarous":22015,"ĠHenderson":22016,"Wow":22017,"spl":22018,"cur":22019,"ĠÂŃ":22020,"Ġtheirs":22021,"Damage":22022,"Ġdownloading":22023,"Ġdiscern":22024,"ĠSto":22025,"ĠFla":22026,"Ġhath":22027,"ĠAj":22028,"Ġunpleasant":22029,"European":22030,"expensive":22031,"Ġscreenshot":22032,"ĠUV":22033,"Ġallied":22034,"ĠPersian":22035,"Ġmonopoly":22036,"Ġatom":22037,"ĠRedskins":22038,"\"><":22039,"Ġcancell":22040,"Ġcinema":22041,"131":22042,"fair":22043,"ĠAlfred":22044,"Ġduck":22045,"args":22046,"223":22047,"ĠISI":22048,"Ġsignaling":22049,"inar":22050,"Ġlaughs":22051,"Ġforwards":22052,"Ġreckless":22053,"Ġlisteners":22054,"ativity":22055,"Ġvastly":22056,"nant":22057,"Less":22058,"ĠHunting":22059,"ĠScientific":22060,"ITED":22061,"Ġknight":22062,"ĠHTC":22063,"usa":22064,"tmp":22065,"Ġrude":22066,"ĠLegendary":22067,"Ġarises":22068,"Bad":22069,"ĠClaim":22070,"peg":22071,"Ġrealities":22072,"Think":22073,"Ġ°":22074,"Ġrode":22075,"Ġstrive":22076,"Ġanecd":22077,"Ġshorts":22078,"Ġhypothes":22079,"Ġcoordinated":22080,"ĠGandhi":22081,"ĠFPS":22082,"RED":22083,"Ġsusceptible":22084,"Ġshrink":22085,"ĠChart":22086,"Help":22087,"Ġion":22088,"deep":22089,"ribes":22090,"ĠKai":22091,"ĠCustomer":22092,"Summary":22093,"Ġcough":22094,"wife":22095,"Ġlend":22096,"Ġpositioning":22097,"Ġlottery":22098,"ĠCanyon":22099,"Ġfade":22100,"Ġbronze":22101,"ĠKenny":22102,"Ġboasts":22103,"ĠEnhanced":22104,"record":22105,"Ġemergence":22106,"Ġakin":22107,"ĠBert":22108,"itous":22109,"âĸij":22110,"Ġstip":22111,"Ġexchanged":22112,"omore":22113,"alsh":22114,"Ġreservoir":22115,"Ġstandpoint":22116,"WM":22117,"Ġinitiate":22118,"Ġdecay":22119,"Ġbrewery":22120,"Ġterribly":22121,"Ġmortal":22122,"levard":22123,"Ġrevis":22124,"NI":22125,"elo":22126,"Ġconfess":22127,"ĠMSNBC":22128,"Ġsubmissions":22129,"Controller":22130,"Ġ202":22131,"ĠRuth":22132,"});":22133,"ĠAzure":22134,"Ġ.\"":22135,"206":22136,"ĠMarketing":22137,"Ġlaund":22138,"iencies":22139,"Ġrenowned":22140,"ĠTrou":22141,"ĠNGO":22142,"blems":22143,"Ġterrified":22144,"Ġwarns":22145,"Ġpert":22146,"Ġunsure":22147,"480":22148,"alez":22149,"ultz":22150,"ĠOutside":22151,"Ġstyl":22152,"ĠUnderground":22153,"Ġpanc":22154,"Ġdictionary":22155,"Ġfoe":22156,"riminal":22157,"ĠNorwegian":22158,"Ġjailed":22159,"Ġmaternal":22160,"ée":22161,"ĠLucy":22162,"cop":22163,"Cho":22164,"Ġunsigned":22165,"ĠZelda":22166,"ĠInsider":22167,"ĠContinued":22168,"Ġ133":22169,"ĠNaruto":22170,"ĠMajority":22171,"169":22172,"ĠWo":22173,"ãĤĵ":22174,"Ġpastor":22175,"Ġinformal":22176,"н":22177,"anthrop":22178,"join":22179,"ãģĹ":22180,"itational":22181,"NP":22182,"ĠWriting":22183,"fn":22184,"ĠBever":22185,"195":22186,"Ġyelling":22187,"Ġdrastically":22188,"Ġeject":22189,"Ġneut":22190,"Ġthrive":22191,"ĠFrequ":22192,"oux":22193,"Ġpossesses":22194,"ĠSenators":22195,"ĠDES":22196,"ĠShakespeare":22197,"ĠFranco":22198,"ĠLB":22199,"uchi":22200,"Ġincarn":22201,"Ġfounders":22202,"Function":22203,"Ġbrightness":22204,"ĠBT":22205,"Ġwhale":22206,"ĠTheater":22207,"mass":22208,"ĠDoll":22209,"Something":22210,"Ġechoed":22211,"ĠHex":22212,"crit":22213,"afia":22214,"Ġgoddess":22215,"Ġeleven":22216,"ĠPreview":22217,"ĠAurora":22218,"Ġ401":22219,"ulsive":22220,"ĠLogan":22221,"inburgh":22222,"ĠCenters":22223,"ĠONLY":22224,"ĠAid":22225,"Ġparadox":22226,"Ġhurd":22227,"ĠLC":22228,"Due":22229,"court":22230,"Ġoffended":22231,"Ġevaluating":22232,"ĠMatthews":22233,"Ġtomb":22234,"Ġpayroll":22235,"Ġextraction":22236,"ĠHands":22237,"ifi":22238,"Ġsupernatural":22239,"ĠCOMM":22240,"]=":22241,"dogs":22242,"Ġ512":22243,"ĠMeeting":22244,"Richard":22245,"ĠMaximum":22246,"Ġideals":22247,"Things":22248,"mand":22249,"ĠRegardless":22250,"Ġhumili":22251,"buffer":22252,"Little":22253,"ĠDani":22254,"ĠNak":22255,"Ġliberation":22256,"ĠAbe":22257,"ĠOL":22258,"Ġstuffed":22259,"aca":22260,"inda":22261,"raphic":22262,"Ġmosqu":22263,"Ġcampaigning":22264,"Ġoccupy":22265,"Squ":22266,"rina":22267,"ĠWel":22268,"ĠVS":22269,"Ġphysic":22270,"Ġpuls":22271,"rint":22272,"oaded":22273,"ETF":22274,"ĠArchives":22275,"Ġvenues":22276,"hner":22277,"ĠTurbo":22278,"Ġlust":22279,"Ġappealed":22280,"quez":22281,"ilib":22282,"ĠTimothy":22283,"Ġomn":22284,"dro":22285,"Ġobsession":22286,"ĠSavage":22287,"1996":22288,"Global":22289,"Jes":22290,"214":22291,"Ġsliding":22292,"Ġdisappro":22293,"ĠMagical":22294,"Ġvoluntarily":22295,"gb":22296,"aney":22297,"Ġprophet":22298,"ĠRein":22299,"ĠJulia":22300,"ĠWorth":22301,"aurus":22302,"Ġbounds":22303,"ieu":22304,")))":22305,"Ġcrore":22306,"ĠCitizen":22307,"Sky":22308,"Ġcolumnist":22309,"Ġseekers":22310,"ondo":22311,"ISA":22312,"ĠLength":22313,"Ġnostalg":22314,"Ġnewcom":22315,"Ġdetrim":22316,"entric":22317,"375":22318,"ĠGE":22319,"Ġautop":22320,"Ġacademics":22321,"AppData":22322,"ĠShen":22323,"Ġidiot":22324,"ĠTransit":22325,"Ġteaspoon":22326,"Wil":22327,"KO":22328,"ĠComedy":22329,">,":22330,"Ġpopulated":22331,"WD":22332,"Ġpigs":22333,"ĠOculus":22334,"Ġsympathetic":22335,"Ġmarathon":22336,"198":22337,"Ġseizure":22338,"sided":22339,"Ġdop":22340,"irtual":22341,"Land":22342,"ĠFloor":22343,"osaurs":22344,"...]":22345,"Ġlos":22346,"Ġsubsidiary":22347,"EY":22348,"ĠParts":22349,"ĠStef":22350,"ĠJudiciary":22351,"Ġ134":22352,"Ġmirrors":22353,"Ġket":22354,"times":22355,"Ġneurolog":22356,"Ġcav":22357,"ĠGuest":22358,"Ġtumor":22359,"scill":22360,"ĠLloyd":22361,"Est":22362,"Ġclearer":22363,"Ġstereotypes":22364,"Ġdur":22365,"nothing":22366,"Reddit":22367,"Ġnegotiated":22368,"------------------------":22369,"235":22370,"Ġflown":22371,"ĠSeoul":22372,"ĠResident":22373,"ĠSCH":22374,"Ġdisappearance":22375,"ĠVince":22376,"grown":22377,"Ġgrabs":22378,"ril":22379,"ĠInfinite":22380,"ĠTwenty":22381,"Ġpedestrian":22382,"Ġjersey":22383,"ĠFur":22384,"ĠInfinity":22385,"ĠElliott":22386,"Ġmentor":22387,"Ġmorally":22388,"Ġobey":22389,"secure":22390,"iffe":22391,"Ġantibiotics":22392,"angled":22393,"ĠFreeman":22394,"ĠIntroduction":22395,"Jun":22396,"Ġmarsh":22397,"icans":22398,"ĠEVENTS":22399,"ochond":22400,"Wall":22401,"iculty":22402,"Ġmisdemeanor":22403,"Ġly":22404,"Thomas":22405,"ĠResolution":22406,"Ġanimations":22407,"ĠDry":22408,"Ġintercourse":22409,"ĠNewcastle":22410,"ĠHog":22411,"ĠEquipment":22412,"177":22413,"Ġterritorial":22414,"Ġarchives":22415,"203":22416,"Filter":22417,"ĠMunich":22418,"Ġcommanded":22419,"ĠWand":22420,"Ġpitches":22421,"ĠCroat":22422,"Ġratios":22423,"ĠMits":22424,"Ġaccumulated":22425,"ĠSpecifically":22426,"Ġgentleman":22427,"acerb":22428,"Ġpenn":22429,"Ġaka":22430,"ĠFuk":22431,"Ġintervene":22432,"ĠRefuge":22433,"ĠAlzheimer":22434,"Ġsuccession":22435,"ohan":22436,"does":22437,"Lord":22438,"Ġseparat":22439,"Ġcorrespondence":22440,"Ġshiny":22441,"Prior":22442,"Ġsulf":22443,"Ġmiserable":22444,"Ġdedication":22445,"().":22446,"Ġspecialists":22447,"Ġdefects":22448,"ĠCult":22449,"ĠXia":22450,"Ġjeopard":22451,"ĠOre":22452,"Ability":22453,"Ġlear":22454,"Ġambitions":22455,"ĠBMI":22456,"ĠArabs":22457,"Ġ1942":22458,"Ġpreservation":22459,"ificate":22460,"Ġashamed":22461,"loss":22462,"ĠRestaur":22463,"Ġresemble":22464,"Ġenrich":22465,"ĠKN":22466,"ĠClan":22467,"float":22468,"Ġplayable":22469,"ITT":22470,"Ġharmony":22471,"arrison":22472,"ĠWeinstein":22473,"were":22474,"Ġpoisoning":22475,"ĠComput":22476,"ĠWordPress":22477,"major":22478,"ĠValve":22479,"Fan":22480,"ĠThrow":22481,"ĠRomans":22482,"ĠDepression":22483,"ados":22484,"Ġtortured":22485,"Ġbalancing":22486,"bottom":22487,"Ġacquiring":22488,"ĠMonte":22489,"ardi":22490,"Ġaura":22491,"Ġ##":22492,"ĠStanding":22493,"ĠAtlas":22494,"CF":22495,"Ġintrins":22496,"ĠBenghazi":22497,"Ġcamping":22498,"Ġtapped":22499,"blade":22500,"strous":22501,"ĠRabb":22502,"ĠWritten":22503,"tip":22504,"ĠNeigh":22505,"sterdam":22506,"ĠAllow":22507,"ĠHealing":22508,"ĠRhod":22509,"num":22510,"Ġcaffeine":22511,"ĠPercent":22512,"Ġboo":22513,"Ġapples":22514,"305":22515,"Ġwelcoming":22516,"Ġapplaud":22517,"Ġausterity":22518,"±":22519,"ĠReality":22520,"efe":22521,"å®":22522,"Ġsucks":22523,"Ġtabs":22524,"ĠPayPal":22525,"Ġbackpack":22526,"Ġgifted":22527,"abulary":22528,"ĠScout":22529,"irteen":22530,"Ġchin":22531,"Ġomitted":22532,"Ġnegatively":22533,"Ġaccessing":22534,"ĠEarn":22535,"Ġambulance":22536,"Ġheadphones":22537,"Ġ205":22538,"ĠRefresh":22539,"president":22540,"ĠKitchen":22541,"ĠEntered":22542,"ĠSnyder":22543,"005":22544,"omical":22545,"Ġborrowed":22546,"ĠNem":22547,"Ġaviation":22548,"Ġstall":22549,"rimination":22550,"Ġuniforms":22551,"itime":22552,"ĠSimmons":22553,"energy":22554,"ablished":22555,"yy":22556,"qualified":22557,"Ġrallies":22558,"ĠStuart":22559,"flight":22560,"Ġgangs":22561,"rag":22562,"Ġvault":22563,"lux":22564,"ĠCompar":22565,"Ġdesignation":22566,"209":22567,"ĠJos":22568,"dollar":22569,"zero":22570,"Ġwells":22571,"303":22572,"Ġconstituents":22573,"Ġheck":22574,"Ġcows":22575,"Ġcommanders":22576,"Ġdifferential":22577,"ĠCatherine":22578,"299":22579,"Ġvalve":22580,"Ġbrace":22581,"Ġperspectives":22582,"cert":22583,"fact":22584,"icularly":22585,"ĠMcN":22586,"planes":22587,"Ġintric":22588,"Ġpeas":22589,"ovan":22590,"Ġtossed":22591,"retch":22592,"ĠLopez":22593,"Ġunfamiliar":22594,"death":22595,"ĠApart":22596,"ĠChang":22597,"Ġrelieved":22598,"rophe":22599,"Ġairports":22600,"Ġfreak":22601,"util":22602,"Mill":22603,"ĠChin":22604,"ĠOwen":22605,"male":22606,"ĠBroken":22607,"ĠWinds":22608,"rob":22609,"rising":22610,"Ġfirefighters":22611,"Ġauthoritarian":22612,"Ġ148":22613,"Bitcoin":22614,"external":22615,"Ġbrowsers":22616,"ichever":22617,"orian":22618,"Ġunb":22619,"Ġpoke":22620,"ĠZot":22621,"Mid":22622,"ĠPopular":22623,"Ġcovert":22624,"Ġcontributes":22625,"Ġ650":22626,"Ġcontention":22627,"Gate":22628,"Ġconsoles":22629,"Ġchromos":22630,"ĠIX":22631,"Ġvisually":22632,"ĠEisen":22633,"Ġjewelry":22634,"Ġdelegation":22635,"Ġaccelerate":22636,"ĠRiley":22637,"Ġslope":22638,"Ġindoor":22639,"itially":22640,"Ġhugely":22641,"Ġtunnels":22642,"Ġfined":22643,"Ġdirective":22644,"Ġforehead":22645,"ustomed":22646,"Ġskate":22647,"Music":22648,"gas":22649,"Ġrecognizing":22650,"ambo":22651,"Ġoverweight":22652,"ĠGrade":22653,"ÙĬ":22654,"Ġsounding":22655,"Ġlocking":22656,"ĠREM":22657,"Store":22658,"Ġexcav":22659,"ĠLikewise":22660,"ĠLights":22661,"Ġelbow":22662,"ĠSupply":22663,"wic":22664,"Ġhandsome":22665,"1994":22666,"Coll":22667,"Ġadequately":22668,"ĠAssociate":22669,"Ġstrips":22670,"Ġcrackdown":22671,"Ġmarvel":22672,"ĠKun":22673,"Ġpassages":22674,"@@@@":22675,"ĠTall":22676,"Ġthoughtful":22677,"namese":22678,"Ġprostitution":22679,"business":22680,"Ġballistic":22681,"personal":22682,"cig":22683,"izational":22684,"Round":22685,"ĠÂłĠÂłĠÂłĠÂł":22686,"ĠColeman":22687,"Ġadmitting":22688,"ĠPlug":22689,"Ġbitcoins":22690,"ĠSuz":22691,"Ġfairness":22692,"Ġsupplier":22693,"Ġcatastrophic":22694,"ĠHelen":22695,"oqu":22696,"Marc":22697,"ĠArticles":22698,"gie":22699,"Ġendangered":22700,"Ġdestiny":22701,"ĠVolt":22702,"olia":22703,"axis":22704,"Ġcheat":22705,"Ġunified":22706,"ICO":22707,"quote":22708,"302":22709,"ĠSed":22710,"Ġsuppression":22711,"Ġanalyzing":22712,"Ġsquat":22713,"Ġfiguring":22714,"Ġcoordinates":22715,"Ġchunks":22716,"Ġ1946":22717,"Ġsubp":22718,"Ġwiki":22719,"ĠForbes":22720,"ĠJupiter":22721,"ĠErik":22722,"imer":22723,"ĠCommercial":22724,"\\)":22725,"Ġlegitimacy":22726,"Ġdental":22727,"ĠMean":22728,"Ġdeficits":22729,"550":22730,"Originally":22731,"ĠHorror":22732,"Ġcontamination":22733,"llah":22734,"Ġconfisc":22735,"ĠClare":22736,"TB":22737,"ĠFailed":22738,"aned":22739,"Ġruler":22740,"ĠController":22741,"Ġfeminists":22742,"Fix":22743,"gay":22744,"207":22745,"Ġrabbit":22746,"Third":22747,"owntown":22748,"Ġglue":22749,"Ġvolatile":22750,"Ġshining":22751,"Ġfoll":22752,"Ġimpaired":22753,"Ġsupers":22754,"æĪ":22755,"Ġclutch":22756,"ļéĨĴ":22757,"Ġprolet":22758,"Ġ(!":22759,"Ġyelled":22760,"ĠKiev":22761,"ĠErn":22762,"ĠShock":22763,"KB":22764,"Ġsituated":22765,"query":22766,"ĠNas":22767,"Ġannex":22768,"character":22769,"ĠHoliday":22770,"Ġautomation":22771,"ĠJill":22772,"ĠRemastered":22773,"Ġlinem":22774,"Ġwilderness":22775,"ĠHorizon":22776,"ĠGuinea":22777,"AZ":22778,"Ġmainland":22779,"Ġsecrecy":22780,"LEASE":22781,"Ġpunk":22782,"ĠProvince":22783,"(),":22784,"Speed":22785,"Ġhanding":22786,"ĠSebast":22787,"Sir":22788,"rase":22789,"Ġjournals":22790,"Ġcongest":22791,"ĠTut":22792,"irrel":22793,"Ġschizophrenia":22794,"Ġmisogyn":22795,"healthy":22796,"Iron":22797,"Ġreacted":22798,"-$":22799,"252":22800,"Ġplural":22801,"Ġplum":22802,"Ġbargain":22803,"Ġgrounded":22804,"finder":22805,"Ġdisse":22806,"ĠLaz":22807,"OOD":22808,"Ġatroc":22809,"Factory":22810,"Ġminions":22811,"Ġori":22812,"ĠBrave":22813,"ĠPRE":22814,"ĠMyanmar":22815,"ĠHod":22816,"Ġexpedition":22817,"Ġexplode":22818,"ĠCoord":22819,"Ġextr":22820,"ĠBrief":22821,"ĠADHD":22822,"Ġhardcore":22823,"feeding":22824,"Ġdile":22825,"ĠFruit":22826,"Ġvaccination":22827,"ĠMao":22828,"osphere":22829,"Ġcontests":22830,"-|":22831,"Ġfren":22832,"isphere":22833,"Rom":22834,"ĠSharp":22835,"ĠTrend":22836,"Ġdisconnect":22837,"âĢ¢âĢ¢":22838,"Ġpersecution":22839,"Earth":22840,"Ġhealthier":22841,"384":22842,"Ġcob":22843,"ĠTrinity":22844,"OWS":22845,"ANN":22846,"Ġspecialty":22847,"Ġgru":22848,"Ġcooperative":22849,"why":22850,"Starting":22851,"ĠIssues":22852,"stre":22853,"ensor":22854,"Ġ185":22855,"Adv":22856,"!?":22857,"ĠRevel":22858,"emia":22859,"ĠHulk":22860,"Ġcelebrations":22861,"ĠSou":22862,"raud":22863,"ĠKlein":22864,"Ġunreal":22865,"context":22866,"Ġpartnerships":22867,"Ġadopting":22868,"tical":22869,"Ġsplash":22870,"ĠHezbollah":22871,"category":22872,"cyclop":22873,"xton":22874,"ĠDot":22875,"urdy":22876,"tz":22877,"Ġenvelope":22878,"ĠNL":22879,"âķ":22880,"Ġwherein":22881,"Spec":22882,"184":22883,"Ġtelev":22884,"aliation":22885,"Ġmyths":22886,"å°":22887,"Ġrigorous":22888,"Ġcommunicating":22889,"Ġobserver":22890,"Ġrehe":22891,"ĠWash":22892,"Ġapologized":22893,"ĠTin":22894,"Ġexpenditures":22895,"workers":22896,"document":22897,"Ġhesitate":22898,"ĠLenin":22899,"Ġunpredictable":22900,"Ġrenewal":22901,"cler":22902,"okia":22903,"ĠCONT":22904,"Ġpostseason":22905,"Tokens":22906,"Ġexacerb":22907,"Ġbetting":22908,"Ġ147":22909,"Ġelevation":22910,"Wood":22911,"ĠSolomon":22912,"194":22913,"004":22914,"output":22915,"Ġredund":22916,"ĠMumbai":22917,"ĠpH":22918,"Ġreproduce":22919,"ĠDuration":22920,"MAX":22921,"Ġbog":22922,"CBS":22923,"ĠBalance":22924,"ĠSgt":22925,"ĠRecent":22926,"Ġcd":22927,"Ġpopped":22928,"Ġincompet":22929,"prop":22930,"ayan":22931,"guy":22932,"Pacific":22933,"Ġtyr":22934,"Ġ{{":22935,"ĠMystic":22936,"ĠDana":22937,"Ġmasturb":22938,"Ġgeometry":22939,"â":22940,"ĠCorrect":22941,"Ġtrajectory":22942,"Ġdistracted":22943,"Ġfoo":22944,"ĠWelsh":22945,"Luc":22946,"mith":22947,"Ġrugby":22948,"Ġrespiratory":22949,"Ġtriangle":22950,"Ġ215":22951,"Ġundergraduate":22952,"ĠSuperior":22953,"changing":22954,"_-":22955,"Ġrightly":22956,"Ġreferee":22957,"Ġlucrative":22958,"Ġunauthorized":22959,"Ġresembles":22960,"ĠGNU":22961,"ĠDerby":22962,"Ġpathways":22963,"ĠLed":22964,"Ġendurance":22965,"Ġstint":22966,"Ġcollector":22967,"Fast":22968,"Ġdots":22969,"Ġnationals":22970,"ĠSecurities":22971,"Ġwhip":22972,"Param":22973,"Ġlearns":22974,"Magic":22975,"Ġdetailing":22976,"moon":22977,"Ġbroadcasting":22978,"Ġbaked":22979,"265":22980,"holm":22981,"ĠSah":22982,"ĠHussein":22983,"ĠCourtesy":22984,"174":22985,"Ġ146":22986,"Ġgeographic":22987,"peace":22988,"Ġjudging":22989,"ĠStern":22990,"Bur":22991,"Ġstoryline":22992,"Gun":22993,"ĠStick":22994,"245":22995,"307":22996,"ãĤ´ãĥ³":22997,"ĠAdministrator":22998,"Ġburnt":22999,"Ġpave":23000,"choes":23001,"Exec":23002,"Ġcampuses":23003,"Result":23004,"Ġmutations":23005,"ĠCharter":23006,"Ġcaptures":23007,"Ġcompares":23008,"Ġbadge":23009,"Scient":23010,"Ġerad":23011,"iery":23012,"oi":23013,"ettes":23014,"ĠEstate":23015,"Ġstrap":23016,"Ġproudly":23017,"Ġfried":23018,"Ġwithdrawn":23019,"ĠVoy":23020,"phony":23021,"Items":23022,"ĠPierce":23023,"bard":23024,"Ġannotation":23025,"anton":23026,"illon":23027,"Impro":23028,"...)":23029,"Ġhappier":23030,"------":23031,"adjust":23032,"Ġstaffers":23033,"Ġactivism":23034,"Ġperf":23035,"Ġalright":23036,"Need":23037,"Ġcommence":23038,"Ġopioid":23039,"ĠAmanda":23040,"Es":23041,"ĠPars":23042,"ĠKaw":23043,"Works":23044,"248":23045,"Ġindo":23046,"tc":23047,"endant":23048,"ĠMoto":23049,"Ġlegalization":23050,"OTE":23051,"Ġtasked":23052,"Ġtsp":23053,"ĠACTIONS":23054,"166":23055,"Ġrefreshing":23056,"ĠNR":23057,"ĠPerez":23058,"Ġinfringement":23059,"SY":23060,"Listen":23061,"inning":23062,"ku":23063,"Ġrotate":23064,"program":23065,"arah":23066,"Design":23067,"Ġ(£":23068,"Ġstoring":23069,"Ġwarrants":23070,"Ġjudgement":23071,"ĠBrist":23072,"usually":23073,"photo":23074,"ĠRan":23075,"ĠPine":23076,"Ġoutrageous":23077,"ĠValentine":23078,"luence":23079,"ĠEverybody":23080,"Altern":23081,"Ġrelevance":23082,"Ġterminated":23083,"Ġdessert":23084,"Ġfulfilled":23085,"Ġprosecuted":23086,"ĠWords":23087,"Ġmigrant":23088,"Ġcultivation":23089,"ÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤ":23090,"idelity":23091,"ĠVern":23092,"ĠLogin":23093,"Ġmetaphor":23094,"ĠTip":23095,"Ġrecruits":23096,"ĠPig":23097,"ribing":23098,"Ġenthusiasts":23099,"exper":23100,"Ġfrightening":23101,"ĠHair":23102,"anson":23103,"strate":23104,"Ġhi":23105,"Height":23106,"Ġowning":23107,"none":23108,"Ġdislike":23109,"Ġknives":23110,"pherd":23111,"Ġloudly":23112,"ĠAPIs":23113,"Display":23114,"ĠLac":23115,"ĠUSS":23116,"abl":23117,"verages":23118,"Jew":23119,"Ġ172":23120,"ĠHistorical":23121,"atoon":23122,"ĠPhysics":23123,"intern":23124,"Ġwarmth":23125,"Ġtopp":23126,"DM":23127,"Ġgunman":23128,"Ġemperor":23129,"odi":23130,"ãĥ£":23131,"inatory":23132,"ĠRib":23133,"Ġ131":23134,"ĠSaturn":23135,"ĠShining":23136,"Ġwaking":23137,"Quotes":23138,"Ġcomedian":23139,"enberg":23140,"½":23141,"Ġbelievers":23142,"Ġpaperwork":23143,"custom":23144,"Ġlev":23145,"Ġlament":23146,"Ġpouring":23147,"222":23148,"political":23149,"ĠSupplement":23150,"maid":23151,"Ġcruelty":23152,"Ġtread":23153,"ysics":23154,"Aw":23155,"rites":23156,"Ġmodifier":23157,"ĠPosition":23158,"Adam":23159,"lb":23160,"ubs":23161,"Ġimperfect":23162,"Ġclusters":23163,"ĠEngineer":23164,"ĠCherry":23165,"Ġinauguration":23166,"ĠSau":23167,"Ġembodiment":23168,"ĠUncle":23169,"Ġoverr":23170,"Ġexplosions":23171,"cule":23172,"ĠPrinceton":23173,"ĠAndrea":23174,"Ġincorrectly":23175,"Ġearnest":23176,"Ġpilgr":23177,"ĠSprint":23178,"Ġsleeve":23179,"Ġhears":23180,"ĠAmazing":23181,"Ġbrowsing":23182,"agin":23183,"Ġhomeland":23184,"Ġhaw":23185,"Ġdiving":23186,"istered":23187,"178":23188,"Ġbargaining":23189,"ĠArcade":23190,"Ġdelegate":23191,"terson":23192,"................................................................":23193,"ĠJacksonville":23194,"275":23195,"Ġstagn":23196,"Ġadam":23197,"ĠSherman":23198,"CB":23199,"Ġsuburb":23200,"ĠFoods":23201,"Ġconverting":23202,"ĠArist":23203,"Ġchambers":23204,"love":23205,"Ġamino":23206,"ĠGan":23207,"Ġmadness":23208,"mc":23209,"ĠUSE":23210,"defined":23211,"Ġultr":23212,"indust":23213,"Ġwolves":23214,"lance":23215,"Additionally":23216,"Ġcracks":23217,"asia":23218,"ĠReason":23219,"ĠPump":23220,"Ġaccidental":23221,"ĠLaser":23222,"ĠRid":23223,"Ġinitialized":23224,"elli":23225,"Ġunnamed":23226,"Ġnoun":23227,"ĠPassed":23228,"Ġhostage":23229,"ĠEthiop":23230,"shirts":23231,"Ġunrel":23232,"ĠEmbassy":23233,"Ġ1941":23234,"Ġatoms":23235,"Ġpurported":23236,"164":23237,"ĠFi":23238,"Ġgallons":23239,"ĠMonica":23240,"Ġpg":23241,"enment":23242,"Ġsorted":23243,"ĠGospel":23244,"Ġheights":23245,"Ġtraced":23246,"Ġundergoing":23247,"Shell":23248,"Ġsacks":23249,"Ġproportions":23250,"Ġhalluc":23251,"Font":23252,"acet":23253,"Ġwarmer":23254,"ĠINTER":23255,"Ġgrabbing":23256,"Plug":23257,"Ġrealization":23258,"ĠBurke":23259,"Ġenchant":23260,"ATER":23261,"ĠSeed":23262,"Ġabundant":23263,"FM":23264,"Ġcivic":23265,"Vs":23266,"isi":23267,"Ġvow":23268,"Ġreper":23269,"ĠPartnership":23270,"Ġpenetration":23271,"Ġaxe":23272,"Ġshattered":23273,"ĠZombies":23274,"Ġvinyl":23275,"ĠAlert":23276,"eon":23277,"Ġobliged":23278,"ĠIllust":23279,"ĠPlaza":23280,"ĠFrontier":23281,"Ġdavidjl":23282,"ĠSerial":23283,"ĠHav":23284,"ĠNutrition":23285,"Bi":23286,"ĠâĸĪ":23287,"ĠJays":23288,"linux":23289,"Ġhurry":23290,"Ġvoy":23291,"Ġhopeless":23292,"ĠStealth":23293,"Ġãģ":23294,"essors":23295,"ttle":23296,"borg":23297,"ĠSafari":23298,"fell":23299,"Ġwary":23300,"due":23301,"ĠAbove":23302,"Ha":23303,"ELL":23304,"Ġnotor":23305,"ĠWon":23306,"Too":23307,"Ġoccupations":23308,"Ġpossessions":23309,"Ġinviting":23310,"Ġpredators":23311,"Ġaccelerated":23312,"Ġ157":23313,"uterte":23314,"ĠCube":23315,"east":23316,"account":23317,"Give":23318,"Ġtransplant":23319,"redients":23320,"idable":23321,"Ġscreenshots":23322,"ĠGund":23323,"ĠFS":23324,"Ġtravelers":23325,"Ġsensory":23326,"ĠFiat":23327,"ĠRockets":23328,"İĭ":23329,"_{":23330,"Friend":23331,"Ġcharming":23332,"ALS":23333,"Ġenjoyment":23334,"mph":23335,"Ġ5000":23336,"ĠREG":23337,"ÙĨ":23338,"bia":23339,"Ġcompilation":23340,"rost":23341,"ĠVP":23342,"ĠSchne":23343,"2019":23344,"Ġcopying":23345,"MORE":23346,"ĠFlore":23347,"falls":23348,"215":23349,"total":23350,"Ġdisciples":23351,"double":23352,"Ġexceeding":23353,"Ġsmashed":23354,"Ġconceptual":23355,"ĠRomania":23356,"ĠBrent":23357,"ĠICE":23358,"ĠTou":23359,"Ġgrap":23360,"Ġnails":23361,"189":23362,"ãĥĺ":23363,"Ġprocure":23364,"eur":23365,"Ġconfirming":23366,"ĠCec":23367,"awi":23368,"ĠEden":23369,"Ġng":23370,"Ġengineered":23371,"atics":23372,"Ġhooked":23373,"Ġdisgusting":23374,"ĠMurder":23375,"ãĤ¿":23376,"Library":23377,"Ġ168":23378,"Almost":23379,"hematic":23380,"Menu":23381,"ĠNotre":23382,"ĠJur":23383,"Ġkidnapped":23384,"Ġhacker":23385,"ĠJade":23386,"Ġcreepy":23387,"Ġdrawings":23388,"ĠSponsor":23389,"Ġcyclists":23390,"ĠGoblin":23391,"Ġoptimized":23392,"Ġstaged":23393,"ĠMcD":23394,"between":23395,"Age":23396,"eno":23397,"Sex":23398,"ĠWide":23399,"nings":23400,"avis":23401,"Ġincapable":23402,"ĠKob":23403,"Ġrewarding":23404,"ĠLone":23405,"olescent":23406,"Ġcontracted":23407,"Ġsticky":23408,"Jose":23409,"Ball":23410,"fest":23411,"ĠInput":23412,"ĠRecently":23413,"Ġtomat":23414,"square":23415,"Application":23416,"Ġnitrogen":23417,"Ġduplicate":23418,"ĠRecon":23419,"ĠDear":23420,"London":23421,"Ġintra":23422,"Ġdock":23423,"Ġoutreach":23424,"ĠMillion":23425,"Ġmammals":23426,"ampton":23427,"VAL":23428,"Ġsnaps":23429,"Ġdos":23430,"ĠWhole":23431,"ĠReady":23432,"Try":23433,"ĠWinnipeg":23434,"earance":23435,"Ġincurred":23436,"renched":23437,"ĠNSW":23438,"ilot":23439,"raine":23440,"Ġcube":23441,"got":23442,"Ġrunway":23443,"etermined":23444,"ĠHawks":23445,"Ġsurvivor":23446,"ĠWish":23447,"ĠDin":23448,"ĠDEF":23449,"ĠVault":23450,"187":23451,"Ġmushrooms":23452,"Ġcrisp":23453,"bey":23454,"ĠDiscovery":23455,"Ġdevelopmental":23456,"Ġparadigm":23457,"Ġchaotic":23458,"ĠTsu":23459,"Ġ333":23460,"bons":23461,"Ġbacterial":23462,"Ġcommits":23463,"Ġcosmic":23464,"Ġmega":23465,"ocative":23466,"ĠPaint":23467,"ophobic":23468,"Ġvain":23469,"Ġcarved":23470,"ĠThief":23471,"ĠGul":23472,"owship":23473,"Ġcites":23474,"ĠEdinburgh":23475,"Ġdiminished":23476,"Ġacknowledges":23477,"ĠKills":23478,"Ġmicrow":23479,"ĠHera":23480,"Ġseniors":23481,"Ġwhereby":23482,"Hop":23483,"atron":23484,"Ġunavailable":23485,"ĠNate":23486,"Ġ480":23487,"Ġslated":23488,"ĠRebecca":23489,"ĠBattery":23490,"Ġgrammar":23491,"Ġheadset":23492,"Ġcursor":23493,"Ġexcluding":23494,"anye":23495,"aundering":23496,"ebin":23497,"Ġfeasible":23498,"ĠPublishing":23499,"ĠLabs":23500,"ĠCliff":23501,"ĠFerrari":23502,"Ġpac":23503,"visible":23504,"marked":23505,"pell":23506,"Ġpolite":23507,"Ġstaggering":23508,"ĠGalactic":23509,"Ġsuperst":23510,"Ġparan":23511,"ĠOfficers":23512,"ãĢģ":23513,"Ġspecifics":23514,"ulus":23515,"239":23516,"ĠPaste":23517,"AMP":23518,"ĠPanama":23519,"ĠDelete":23520,"anguard":23521,"restrial":23522,"Ġheroic":23523,"ĠDy":23524,"اÙĦ":23525,"Ġincumbent":23526,"Ġcrunch":23527,"tro":23528,"Ġscoop":23529,"Ġblogger":23530,"Ġsellers":23531,"uren":23532,"Ġmedicines":23533,"ĠCaps":23534,"ĠAnimation":23535,"oxy":23536,"Ġoutward":23537,"Ġinquiries":23538,"229":23539,"Ġpsychologist":23540,"ĠSask":23541,"evil":23542,"Ġcontaminated":23543,"ãĤ¨":23544,"herence":23545,"Ġbranded":23546,"ĠAbdul":23547,"zh":23548,"Ġparagraphs":23549,"Ġmins":23550,"Ġcorrelated":23551,"erb":23552,"Ġimpart":23553,"Ġmilestone":23554,"ĠSolutions":23555,"otle":23556,"Ġundercover":23557,"Ġmarched":23558,"ĠChargers":23559,"fax":23560,"ĠSecrets":23561,"Ġruth":23562,"weather":23563,"Ġfeminine":23564,"Ġsham":23565,"Ġprestigious":23566,"iggins":23567,"Ġsung":23568,"history":23569,"ettle":23570,"ggie":23571,"Ġoutdated":23572,"oland":23573,"Ġperceptions":23574,"ĠSession":23575,"ĠDodgers":23576,"uj":23577,"ĠEND":23578,"Doc":23579,"Ġdeficiency":23580,"Grand":23581,"ĠJoker":23582,"Ġretrospect":23583,"Ġdiagnostic":23584,"Ġharmless":23585,"Ġrogue":23586,"ĠAval":23587,"Equ":23588,"Ġtransc":23589,"ĠRobertson":23590,"ĠDepending":23591,"ĠBurns":23592,"ivo":23593,"Ġhostility":23594,"Features":23595,"ĵĺ":23596,"Ġdiscomfort":23597,"ĠLCD":23598,"specified":23599,"ĠExpect":23600,"340":23601,"Ġimperative":23602,"ĠRegular":23603,"Chinese":23604,"Ġstatewide":23605,"Ġsymm":23606,"Ġloops":23607,"Ġautumn":23608,"Nick":23609,"Ġshaping":23610,"Ġquot":23611,"Ġcherry":23612,"ĠCrossref":23613,"è¦ļéĨĴ":23614,"Standard":23615,"heed":23616,"ĠDell":23617,"ĠVietnamese":23618,"Ġost":23619,"ĠValkyrie":23620,"OA":23621,"Assad":23622,"Ġrebound":23623,"ĠTraffic":23624,"places":23625,"æĺ":23626,"ĠBuc":23627,"172":23628,"Ġshelters":23629,"Ġinsisting":23630,"ĠCertainly":23631,"ĠKenneth":23632,"ĠTCP":23633,"Ġpenal":23634,"ĠReplay":23635,"heard":23636,"Ġdialect":23637,"iza":23638,"ĠFY":23639,"itcher":23640,"ĠDL":23641,"Ġspiral":23642,"Ġquarterbacks":23643,"Ġhull":23644,"Ġgoogle":23645,"Ġtodd":23646,"ĠSterling":23647,"ĠPlate":23648,"Ġspying":23649,"mbol":23650,"ĠRealm":23651,"ĠProced":23652,"ĠCrash":23653,"Ġterminate":23654,"Ġprotesting":23655,"Center":23656,"guided":23657,"Ġuncover":23658,"Ġboycott":23659,"Ġrealizes":23660,"sound":23661,"Ġpretending":23662,"ĠVas":23663,"1980":23664,"Ġframed":23665,"Ġ139":23666,"Ġdescended":23667,"Ġrehabilitation":23668,"Ġborrowing":23669,"ĠBuch":23670,"Ġblur":23671,"Ron":23672,"ĠFrozen":23673,"enza":23674,"Chief":23675,"ĠPoor":23676,"Ġtranslates":23677,"MIN":23678,"Ġ212":23679,"JECT":23680,"Ġerupted":23681,"Ġsuccesses":23682,"SEC":23683,"Ġplague":23684,"Ġgems":23685,"doms":23686,"Ġstretches":23687,"ĠSpy":23688,"Ġstorytelling":23689,"Credit":23690,"ĠPush":23691,"Ġtraction":23692,"Ġineffective":23693,"ĠLuna":23694,"Ġtapes":23695,"Ġanalytics":23696,"ercise":23697,"Ġprogrammes":23698,"ĠCarbon":23699,"Ġbehold":23700,"heavy":23701,"ĠConservation":23702,"ĠFIR":23703,"Ġsack":23704,"termin":23705,"ricks":23706,"Ġhoused":23707,"Ġunusually":23708,"Ice":23709,"Ġexecuting":23710,"ĠMoroc":23711,"eday":23712,"Ġeditions":23713,"Ġsmarter":23714,"ĠBA":23715,"Ġoutlaw":23716,"Ġvanished":23717,"iba":23718,"ALSE":23719,"ĠSilva":23720,"238":23721,"Could":23722,"Ġphilosopher":23723,"Ġevacuated":23724,"Secret":23725,"142":23726,"Ġvisas":23727,"ãĤ¬":23728,"ĠMalt":23729,"ĠClearly":23730,"ĠNiger":23731,"ĠCairo":23732,"ĠFist":23733,"380":23734,"ĠXML":23735,"auto":23736,"itant":23737,"Ġreinforced":23738,"Record":23739,"ĠSurvivor":23740,"GHz":23741,"Ġscrews":23742,"parents":23743,"Ġoceans":23744,"mares":23745,"Ġbrakes":23746,"vasive":23747,"Ġhello":23748,"ĠSIM":23749,"rimp":23750,"Ġore":23751,"ĠArmour":23752,"247":23753,"Ġterrific":23754,"Ġtones":23755,"141":23756,"ĠMinutes":23757,"Episode":23758,"Ġcurves":23759,"Ġinflammatory":23760,"Ġbatting":23761,"ĠBeautiful":23762,"Lay":23763,"Ġunpop":23764,"vable":23765,"Ġriots":23766,"ĠTactics":23767,"baugh":23768,"ĠCock":23769,"Ġorgasm":23770,"ĠSas":23771,"Ġconstructor":23772,"etz":23773,"Gov":23774,"Ġantagon":23775,"Ġtheat":23776,"Ġdeeds":23777,"hao":23778,"cuts":23779,"ĠMcCl":23780,"Ġum":23781,"ĠScientists":23782,"Ġgrassroots":23783,"yssey":23784,"\"]=>":23785,"Ġsurfaced":23786,"Ġshades":23787,"Ġneighbours":23788,"Ġadvertis":23789,"oya":23790,"Ġmerged":23791,"Upon":23792,"Ġgad":23793,"Ġanticipate":23794,"Anyway":23795,"Ġslogan":23796,"Ġdisrespect":23797,"Iran":23798,"ĠTB":23799,"acted":23800,"Ġsubpoen":23801,"mediately":23802,"OOOO":23803,"Ġwaiver":23804,"Ġvulnerabilities":23805,"ottesville":23806,"ĠHuffington":23807,"Josh":23808,"ĠDH":23809,"Monday":23810,"ĠEllen":23811,"Know":23812,"xon":23813,"items":23814,"228":23815,"Ġfills":23816,"ĠNike":23817,"Ġcumulative":23818,"andals":23819,"Ir":23820,"Ġì":23821,"Ġfriction":23822,"igator":23823,"Ġscans":23824,"ĠVienna":23825,"ldom":23826,"Ġperformers":23827,"Prim":23828,"Ġbidding":23829,"Mur":23830,"Ġleaned":23831,"ĠPrix":23832,"alks":23833,"Ġ[âĢ¦]":23834,"ĠTwitch":23835,"ĠDeveloper":23836,"ĠGir":23837,"Ġcallback":23838,"Abstract":23839,"Ġaccustomed":23840,"Ġfreedoms":23841,"ĠPG":23842,"uracy":23843,"Ġlump":23844,"isman":23845,",,,,":23846,"1992":23847,"ĠRED":23848,"Ġworm":23849,"Match":23850,"ĠPlatinum":23851,"IJ":23852,"ĠOwner":23853,"Trivia":23854,"compl":23855,"Ġnewborn":23856,"Ġfantas":23857,"Own":23858,"Ġ1959":23859,"Ġsympath":23860,"Ġubiqu":23861,"Ġoutputs":23862,"Ġallev":23863,"Ġprag":23864,"Kevin":23865,"Ġfavors":23866,"Ġburial":23867,"Ġnurt":23868,"solete":23869,"cache":23870,"Ġ156":23871,"Ġunlocks":23872,"techn":23873,"Making":23874,"Ġconquer":23875,"adic":23876,"æĸ":23877,"Ġelf":23878,"Ġelectorate":23879,"ĠKurds":23880,"ĠStack":23881,"ĠSamurai":23882,"Ġâĺħ":23883,"Ġ{}":23884,"ĠSaid":23885,"ĠFallout":23886,"Ġkindness":23887,"ĠCustoms":23888,"ĠBoulevard":23889,"Ġhelicopters":23890,"otics":23891,"ĠVeget":23892,"comment":23893,"Ġcriticised":23894,"Ġpolished":23895,"ĠRemix":23896,"ĠCultural":23897,"Ġrecons":23898,"Ġdoi":23899,"atem":23900,"Screen":23901,"Ġbarred":23902,"Comments":23903,"ĠGenerally":23904,"Ġslap":23905,"720":23906,"Vari":23907,"pine":23908,"Ġempt":23909,"Ġhats":23910,"ĠPlaying":23911,"lab":23912,"average":23913,"forms":23914,"ĠCotton":23915,"Ġcans":23916,"ĠDON":23917,"ĠSomalia":23918,"Crypt":23919,"ĠIncreases":23920,"Ever":23921,"modern":23922,"Ġsurgeon":23923,"3000":23924,"Ġrandomized":23925,"================================================================":23926,"Bern":23927,"impl":23928,"ĠCOR":23929,"Ġproclaim":23930,"thouse":23931,"Ġtoes":23932,"Ġample":23933,"Ġpreserving":23934,"Ġdisbel":23935,"grand":23936,"Besides":23937,"Ġsilk":23938,"ĠPattern":23939,"hm":23940,"Ġenterprises":23941,"Ġaffidavit":23942,"ĠAdvisory":23943,"Ġadvertised":23944,"ĠReligious":23945,"sections":23946,"psych":23947,"ĠFields":23948,"aways":23949,"Ġhashtag":23950,"ĠNightmare":23951,"Ġvampire":23952,"Ġforensic":23953,"rossover":23954,"nar":23955,"Ġnavy":23956,"Ġvacant":23957,"ĠDuel":23958,"Ġhallway":23959,"Ġfacebook":23960,"identally":23961,"ĠNRA":23962,"Ġmatt":23963,"Ġhurricane":23964,"ĠKirby":23965,"ĠPuzzle":23966,"Ġskirt":23967,"oust":23968,"dullah":23969,"Ġanalogy":23970,"inion":23971,"Ġtomatoes":23972,"ĠNV":23973,"ĠPeak":23974,"ĠMeyer":23975,"Ġappointments":23976,"Ġmasc":23977,"Ġalley":23978,"rehend":23979,"Ġcharities":23980,"Ġundo":23981,"Ġdestinations":23982,"ĠTesting":23983,"\">\"":24618,"cats":24619,"*.":24620,"Ġgestures":24621,"general":24622,"League":24623,"Ġpackets":24624,"ĠInspector":24625,"ĠBerg":24626,"Ġfraudulent":24627,"Ġcriticize":24628,"Fun":24629,"Ġblaming":24630,"ndra":24631,"Ġslash":24632,"ĠEston":24633,"Ġproposing":24634,"Ġwhales":24635,"Ġtherapist":24636,"Ġsubset":24637,"Ġleisure":24638,"ELD":24639,"ĠCVE":24640,"ĠActivity":24641,"Ġculmin":24642,"shop":24643,"ĠDAY":24644,"ischer":24645,"ĠAdmiral":24646,"ĠAttacks":24647,"Ġ1958":24648,"Ġmemoir":24649,"Ġfolded":24650,"Ġsexist":24651,"Ġ153":24652,"ĠLI":24653,"Ġreadings":24654,"Ġembarrassment":24655,"ĠEmployment":24656,"wart":24657,"chin":24658,"Ġcontinuation":24659,"lia":24660,"Recently":24661,"Ġduel":24662,"Ġevacuation":24663,"ĠKashmir":24664,"Ġdisposition":24665,"ĠRig":24666,"Ġbolts":24667,"Ġinsurers":24668,"467":24669,"Mex":24670,"Ġretaliation":24671,"Ġmisery":24672,"Ġunreasonable":24673,"raining":24674,"Imm":24675,"ĠPU":24676,"emer":24677,"Ġgenital":24678,"ãĤ³":24679,"ĠCandy":24680,"Ġonions":24681,"ĠPatt":24682,"liner":24683,"Ġconceded":24684,"Ġfa":24685,"Ġforc":24686,"ĠHernandez":24687,"ĠGeoff":24688,"debian":24689,"ĠTeams":24690,"Ġcries":24691,"Ġhomeowners":24692,"237":24693,"ABC":24694,"Ġstitch":24695,"Ġstatistic":24696,"Ġheaders":24697,"ĠBiology":24698,"Ġmotors":24699,"ĠGEN":24700,"ĠLip":24701,"Ġhates":24702,"Ġheel":24703,"Self":24704,"ipl":24705,"EDIT":24706,"orting":24707,"Ġannot":24708,"ĠSpeech":24709,"oldemort":24710,"ĠJavascript":24711,"ĠLeBron":24712,"Ġfootprint":24713,"Ġfn":24714,"Ġseizures":24715,"nas":24716,"hide":24717,"Ġ1954":24718,"ĠBee":24719,"ĠDeclaration":24720,"ĠKatie":24721,"Ġreservations":24722,"NR":24723,"female":24724,"Ġsaturated":24725,"Ġbiblical":24726,"Ġtrolls":24727,"Device":24728,"photos":24729,"Ġdrums":24730,"ãĥīãĥ©ãĤ´ãĥ³":24731,"Night":24732,"fighter":24733,"ĠHak":24734,"riber":24735,"Ġcush":24736,"Ġdisciplinary":24737,"baum":24738,"ĠGH":24739,"ĠSchmidt":24740,"ilibrium":24741,"Ġsixty":24742,"ĠKushner":24743,"rots":24744,"Ġpund":24745,"ĠRac":24746,"Ġsprings":24747,"Ġconve":24748,"Business":24749,"Fall":24750,"Ġqualifications":24751,"Ġverses":24752,"Ġnarciss":24753,"ĠKoh":24754,"ĠWow":24755,"ĠCharlottesville":24756,"edo":24757,"Ġinterrogation":24758,"ĠWool":24759,"365":24760,"Brian":24761,"Ġâľĵ":24762,"Ġalleges":24763,"onds":24764,"idation":24765,"ĠJackie":24766,"yu":24767,"Ġlakes":24768,"Ġworthwhile":24769,"Ġcrystals":24770,"ĠJuda":24771,"Ġcomprehend":24772,"Ġflush":24773,"Ġabsorption":24774,"ĠOC":24775,"Ġfrightened":24776,"ĠChocolate":24777,"Martin":24778,"Ġbuys":24779,"Ġbucks":24780,"Ġappell":24781,"ĠChampionships":24782,"Ġlistener":24783,"ĠDefensive":24784,"Ġcz":24785,"uds":24786,"ĠMate":24787,"Ġreplay":24788,"Ġdecorated":24789,"Ġsunk":24790,"ĠVIP":24791,"ĠAnk":24792,"Ġ195":24793,"aaaa":24794,"Nobody":24795,"ĠMilk":24796,"ĠGur":24797,"ĠMk":24798,"ĠSara":24799,"Ġseating":24800,"ĠWid":24801,"Track":24802,"Ġemploys":24803,"Ġgigantic":24804,"APP":24805,"ãĤ§":24806,"inventory":24807,"Ġtowel":24808,"atche":24809,"lasting":24810,"ĠTL":24811,"Ġlatency":24812,"Ġkne":24813,"Ber":24814,"meaning":24815,"Ġupheld":24816,"Ġplayground":24817,"Ġmant":24818,"Side":24819,"Ġstereo":24820,"Ġnorthwest":24821,"Ġexceptionally":24822,"Ġrays":24823,"Ġrecurring":24824,"Drive":24825,"Ġupright":24826,"Ġabduct":24827,"ĠMarathon":24828,"Ġgoodbye":24829,"Ġalphabet":24830,"hp":24831,"Ġcourtroom":24832,"rington":24833,"othing":24834,"Tag":24835,"Ġdiplomats":24836,"Ġbarbar":24837,"ĠAqua":24838,"183":24839,"3333":24840,"Ġmaturity":24841,"Ġinstability":24842,"ĠApache":24843,"Ġ===":24844,"Ġfasting":24845,"ĠGrid":24846,"ModLoader":24847,"Ġ152":24848,"Abs":24849,"ĠOperating":24850,"etti":24851,"Ġacquaint":24852,"Donnell":24853,"ĠKem":24854,"ĠForge":24855,"Ġarmored":24856,"Mil":24857,"Ġphilosophers":24858,"invest":24859,"Players":24860,"âĪ":24861,"Ġmyriad":24862,"Ġcomrades":24863,"Rot":24864,"Ġremembering":24865,"Ġcorresponds":24866,"Ġprogrammers":24867,"ĠLynn":24868,"Ġolig":24869,"Ġcoherent":24870,"ynchron":24871,"ĠChemical":24872,"Ġjugg":24873,"pair":24874,"posts":24875,"Eye":24876,"ĠInner":24877,"Ġsemester":24878,"ottest":24879,"ĠEmirates":24880,"ricanes":24881,"orously":24882,"mits":24883,"ĠWis":24884,"Ġdodge":24885,"location":24886,"Ġfaded":24887,"Amazon":24888,"ĠProceed":24889,"ĠINFO":24890,"journal":24891,"ĠTruck":24892,"Ten":24893,"Ġ217":24894,"Ġstatutes":24895,"mobile":24896,"ĠTypes":24897,"Recomm":24898,"buster":24899,"pex":24900,"Ġlegends":24901,"Ġheadache":24902,"faced":24903,"ĠWiFi":24904,"ifty":24905,"ĠHER":24906,"Ġcircuits":24907,"ERROR":24908,"226":24909,"olin":24910,"Ġcylinder":24911,"ospace":24912,"ikers":24913,"Prem":24914,"Quant":24915,"Ġconflicting":24916,"Ġslightest":24917,"Ġforged":24918,"ionage":24919,"Stephen":24920,"ĠKub":24921,"ĠOpportun":24922,"ĠHeal":24923,"Ġblo":24924,"Ġrulers":24925,"Ġhuh":24926,"Ġsubmarine":24927,"fy":24928,"asser":24929,"Ġallowance":24930,"ĠKasich":24931,"ĠTas":24932,"ĠAustralians":24933,"ForgeModLoader":24934,"ĠâĨij":24935,"ĠMatrix":24936,"amins":24937,"Ġ1200":24938,"ĠAcqu":24939,"236":24940,"Document":24941,"ĠBreaking":24942,"193":24943,"ĠSubst":24944,"ĠRoller":24945,"ĠProperties":24946,"ĠNI":24947,"tier":24948,"Ġcrushing":24949,"Ġadvocating":24950,"Furthermore":24951,"keepers":24952,"Ġsexism":24953,"xd":24954,"Ġcaller":24955,"ĠSense":24956,"chieve":24957,"ĠTF":24958,"Ġfueled":24959,"Ġreminiscent":24960,"Ġobsess":24961,"urst":24962,"Ġuphold":24963,"ĠFans":24964,"hetics":24965,"ĠâĹ":24966,"ĠBath":24967,"Ġbeverage":24968,"Ġoscill":24969,"254":24970,"Ġpoles":24971,"Ġgradual":24972,"Ġexting":24973,"ĠSuff":24974,"ĠSuddenly":24975,"Ġliking":24976,"Ġ1949":24977,"unciation":24978,"amination":24979,"ĠOmar":24980,"ĠLV":24981,"ĠConsequently":24982,"Ġsynthes":24983,"ĠGIF":24984,"Ġpains":24985,"Ġinteracting":24986,"uously":24987,"incre":24988,"Ġrumor":24989,"ĠScientology":24990,"197":24991,"ĠZig":24992,"Ġspelling":24993,"ĠASS":24994,"Ġextingu":24995,"mson":24996,"Ġgh":24997,"Ġremarked":24998,"ĠStrategic":24999,"ĠMON":25000,"å¥":25001,"gae":25002,"ĠWHAT":25003,"Eric":25004,"ĠCampus":25005,"Ġmethane":25006,"Ġimagin":25007,"JUST":25008,"ĠAlm":25009,"XT":25010,"iq":25011,"ĠRSS":25012,"Ġwrongdoing":25013,"atta":25014,"Ġbigot":25015,"Ġdemonstrators":25016,"ĠCalvin":25017,"ĠVilla":25018,"Ġmembrane":25019,"ĠAwesome":25020,"Ġbenefic":25021,"268":25022,"Ġmagnificent":25023,"ĠLots":25024,"Greg":25025,"ĠBoris":25026,"Ġdetainees":25027,"ĠHerman":25028,"Ġwhispered":25029,"Ġawe":25030,"Professor":25031,"funding":25032,"Ġphysiological":25033,"ĠDestruction":25034,"Ġlimb":25035,"Ġmanipulated":25036,"Ġbubbles":25037,"Ġpseud":25038,"Ġhydra":25039,"ĠBristol":25040,"Ġstellar":25041,"ĠExpansion":25042,"ĠKell":25043,"ĠInterestingly":25044,"Ġmans":25045,"Ġdragging":25046,"Ġecological":25047,"ĠFit":25048,"Ġgent":25049,"Ġbenefited":25050,"ĠHaiti":25051,"Ġpolyg":25052,"ãĥİ":25053,"Ġ2030":25054,"Ġprow":25055,"Ġreconstruction":25056,"Ġwast":25057,"Ġpsychic":25058,"ĠGreeks":25059,"Handler":25060,"162":25061,"ĠPulse":25062,"Ġsolicit":25063,"Ġsys":25064,"Ġinflux":25065,"ĠGentle":25066,"percent":25067,"Ġproliferation":25068,"Ġtaxable":25069,"Ġdisregard":25070,"Ġescaping":25071,"Ġginger":25072,"Ġwithstand":25073,"Ġdevastated":25074,"ĠDew":25075,"series":25076,"Ġinjected":25077,"elaide":25078,"Ġturnover":25079,"heat":25080,"ĻĤ":25081,"Happy":25082,"ĠSilent":25083,"ãĤŃ":25084,"ivism":25085,"Ġirrational":25086,"AMA":25087,"Ġreef":25088,"rub":25089,"Ġ162":25090,"Ġbankers":25091,"ĠEthics":25092,"vv":25093,"Ġcriticisms":25094,"Kn":25095,"186":25096,"Movie":25097,"ĠTories":25098,"Ġnood":25099,"Ġdistortion":25100,"False":25101,"odore":25102,"Ġtasty":25103,"Research":25104,"ĠUID":25105,"-)":25106,"Ġdivorced":25107,"ĠMU":25108,"ĠHayes":25109,"ĠIsn":25110,"iani":25111,"ĠHQ":25112,"Ġ\"#":25113,"ignant":25114,"Ġtraumatic":25115,"ĠLing":25116,"Hun":25117,"Ġsabot":25118,"online":25119,"random":25120,"Ġrenamed":25121,"rared":25122,"KA":25123,"dead":25124,"ét":25125,"ĠAssistance":25126,"Ġseaf":25127,"++++++++":25128,"Ġseldom":25129,"ĠWebb":25130,"Ġboolean":25131,"ulet":25132,"Ġrefrain":25133,"ĠDIY":25134,"rule":25135,"Ġshutting":25136,"Ġutilizing":25137,"loading":25138,"ĠParam":25139,"coal":25140,"ooter":25141,"Ġattracting":25142,"ĠDol":25143,"Ġhers":25144,"agnetic":25145,"ĠReach":25146,"imo":25147,"Ġdiscarded":25148,"ĠPip":25149,"015":25150,"ür":25151,"Ġmug":25152,"Imagine":25153,"COL":25154,"Ġcursed":25155,"ĠShows":25156,"ĠCurtis":25157,"ĠSachs":25158,"speaking":25159,"ĠVista":25160,"ĠFramework":25161,"ongo":25162,"Ġsubreddit":25163,"Ġcrus":25164,"ĠOval":25165,"Row":25166,"growing":25167,"Ġinstallment":25168,"Ġglac":25169,"ĠAdvance":25170,"ECK":25171,"ĠLGBTQ":25172,"LEY":25173,"Ġacet":25174,"Ġsuccessive":25175,"ĠNicole":25176,"Ġ1957":25177,"Quote":25178,"Ġcircumstance":25179,"ackets":25180,"Ġ142":25181,"ortium":25182,"Ġguessed":25183,"ĠFrame":25184,"Ġperpetrators":25185,"ĠAviation":25186,"ĠBench":25187,"Ġhandc":25188,"Ap":25189,"Ġ1956":25190,"259":25191,"rand":25192,"NetMessage":25193,"din":25194,"urtles":25195,"hig":25196,"ĠVIII":25197,"ffiti":25198,"ĠSwords":25199,"bial":25200,"Ġkidnapping":25201,"device":25202,"Ġbarn":25203,"ĠEli":25204,"aucas":25205,"Send":25206,"Constructed":25207,"Ġ½":25208,"Ġneedles":25209,"Ġadvertisements":25210,"Ġvou":25211,"Ġexhibited":25212,"ĠFortress":25213,"Ask":25214,"Berry":25215,"TYPE":25216,"Ġcancers":25217,"umping":25218,"ĠTerritory":25219,"Ġprud":25220,"Ġnas":25221,"Ġatheist":25222,"Ġbalances":25223,"ãģŁ":25224,"ĠShawn":25225,"&&":25226,"Ġlandsc":25227,"ĠRGB":25228,"Ġpetty":25229,"Ġexcellence":25230,"Ġtranslations":25231,"Ġparcel":25232,"ĠChev":25233,"East":25234,"ĠOutput":25235,"imi":25236,"Ġambient":25237,"ĠThreat":25238,"Ġvillains":25239,"Ġ550":25240,"ICA":25241,"Ġtaller":25242,"Ġleaking":25243,"cup":25244,"Ġpolish":25245,"Ġinfectious":25246,"ĠKC":25247,"Ġ@@":25248,"background":25249,"Ġbureaucracy":25250,"ĠSai":25251,"unless":25252,"itious":25253,"ĠSkype":25254,"Atl":25255,"IDENT":25256,"008":25257,"Ġhypocr":25258,"Ġpitchers":25259,"Ġguessing":25260,"ĠFINAL":25261,"Between":25262,"Ġvillagers":25263,"Ġ252":25264,"fashion":25265,"ĠTunis":25266,"Beh":25267,"ĠExc":25268,"ĠMID":25269,"288":25270,"ĠHaskell":25271,"196":25272,"ĠNOR":25273,"Ġspecs":25274,"Ġinvari":25275,"Ġglut":25276,"ĠCars":25277,"Ġimpulse":25278,"Ġhonors":25279,"gel":25280,"Ġjurisdictions":25281,"ĠBundle":25282,"ulas":25283,"California":25284,"ĠIncrease":25285,"Ġpear":25286,"Ġsingles":25287,"Ġcues":25288,"Ġunderwent":25289,"ĠWS":25290,"Ġexaggerated":25291,"Ġdubious":25292,"Ġflashing":25293,"LOG":25294,")].":25295,"Journal":25296,"tg":25297,"Van":25298,"ĠIstanbul":25299,"ĠInsp":25300,"ĠFranken":25301,"Draw":25302,"Ġsadness":25303,"Ġironic":25304,"ĠFry":25305,"xc":25306,"Ġ164":25307,"isch":25308,"Way":25309,"ĠProtestant":25310,"horn":25311,"Ġunaff":25312,"ĠViv":25313,"illas":25314,"ĠProductions":25315,"ĠHogan":25316,"Ġperimeter":25317,"ĠSisters":25318,"Ġspontaneous":25319,"Ġdownside":25320,"Ġdescendants":25321,"Ġorn":25322,"worm":25323,"Japanese":25324,"Ġ1955":25325,"Ġ151":25326,"ĠDoing":25327,"elsen":25328,"umbles":25329,"Ġradically":25330,"ĠDrum":25331,"ĠBach":25332,"Ġliabilities":25333,"ĠOB":25334,"ĠElementary":25335,"Ġmeme":25336,"ynes":25337,"Ġfingerprint":25338,"ĠGrab":25339,"Ġundertake":25340,"Members":25341,"ĠReader":25342,"ĠSims":25343,"god":25344,"Ġhypothetical":25345,"scient":25346,"ĠAJ":25347,"Ġcharism":25348,"Ġadmissions":25349,"ĠMissile":25350,"trade":25351,"Ġexercising":25352,"ĠBackground":25353,"Written":25354,"Ġvocals":25355,"whether":25356,"Ġvi":25357,"ĠWinner":25358,"Ġlitter":25359,"ĠShooting":25360,"STEM":25361,"ãĤ¡":25362,"ĠAFL":25363,"Ġvariability":25364,"Ġeats":25365,"ĠDPS":25366,"brow":25367,"Ġelephants":25368,"Ġstrat":25369,"ĠÅ":25370,"Ġsettlers":25371,"Matthew":25372,"Ġinadvert":25373,"HI":25374,"ĠIMF":25375,"ĠGoal":25376,"Ġnerves":25377,"Johnson":25378,"eye":25379,"ablishment":25380,"Thursday":25381,"BILITY":25382,"Had":25383,"amoto":25384,"hetamine":25385,"eps":25386,"Ġmitochond":25387,"Ġcompressed":25388,"ĠTrevor":25389,"ĠAnimals":25390,"Tool":25391,"Lock":25392,"Ġtweak":25393,"Ġpinch":25394,"Ġcancellation":25395,"Pot":25396,"Ġfocal":25397,"ĠAstron":25398,"173":25399,"ĠASC":25400,"ĠOTHER":25401,"umni":25402,"Ġdemise":25403,"dl":25404,"Ùħ":25405,"Semitism":25406,"Ġcracking":25407,"Ġcollaborative":25408,"Ġexplores":25409,"sql":25410,"Ġherbs":25411,"Ġconfigurations":25412,"mis":25413,"ĠResult":25414,"acey":25415,"ĠSmoke":25416,"Ġsanct":25417,"elia":25418,"Ġdegener":25419,"Ġdeepest":25420,"Ġscreamed":25421,"Ġnap":25422,"Software":25423,"ĠSTAR":25424,"EF":25425,"ĠXin":25426,"sponsored":25427,"manship":25428,"233":25429,"Ġprimaries":25430,"Ġfiltering":25431,"Ġassemble":25432,"mil":25433,"ĠMyers":25434,"bows":25435,"Ġpunched":25436,"Mic":25437,"Ġinnovations":25438,"Ġfunc":25439,"ando":25440,"Ġfracking":25441,"ĠVul":25442,"оÐ":25443,"oshop":25444,"ĠImmun":25445,"Ġsettling":25446,"Ġadolescents":25447,"Ġrebuilding":25448,"Ġtransforming":25449,"Ġparole":25450,"Ġharbor":25451,"Ġbooking":25452,"otional":25453,"ongevity":25454,"ĠYo":25455,"bug":25456,"Ġemerges":25457,"ĠMethods":25458,"ĠChu":25459,"Pres":25460,"ĠDungeons":25461,"Ġtrailing":25462,"ĠRum":25463,"ĠHugh":25464,"天":25465,"ĠEra":25466,"ĠBattles":25467,"Results":25468,"ĠTrading":25469,"Ġversa":25470,"css":25471,"axies":25472,"heet":25473,"Ġgreed":25474,"1989":25475,"Ġgardens":25476,"Ġcontingent":25477,"Park":25478,"ĠLeafs":25479,"hook":25480,"robe":25481,"Ġdiplomacy":25482,"ĠFuel":25483,"ĠInvasion":25484,"Ġupgrading":25485,"Male":25486,"Ġelic":25487,"Ġrelentless":25488,"ĠCovenant":25489,"apesh":25490,"ĠTrop":25491,"Ty":25492,"production":25493,"arty":25494,"Ġpunches":25495,"ako":25496,"cyclopedia":25497,"ĠRabbit":25498,"ĠHDMI":25499,"Ġ141":25500,"Ġfoil":25501,"ItemImage":25502,"ĠFG":25503,"Ġimplementations":25504,"ĠPom":25505,"ixtures":25506,"Ġawait":25507,"Ġ330":25508,"amus":25509,"Ġumbrella":25510,"Ġforesee":25511,"separ":25512,"Ġcircumcision":25513,"Ġperipheral":25514,"Say":25515,"ĠExpert":25516,"Inc":25517,"Ġwithdrew":25518,"ĠAnders":25519,"fried":25520,"Ġradioactive":25521,"ĠOpening":25522,"Ġboarding":25523,"ĠND":25524,"Ġoverthrow":25525,"Activ":25526,"WP":25527,"ĠActs":25528,"×Ļ":25529,"Ġmotions":25530,"vic":25531,"ĠMighty":25532,"ĠDefender":25533,"aer":25534,"Ġthankful":25535,"ĠKilling":25536,"ĠBris":25537,"moil":25538,"Ġpredicting":25539,"266":25540,"choice":25541,"Ġkillers":25542,"Ġincub":25543,"ĠChest":25544,"athering":25545,"Ġproclaimed":25546,"flower":25547,"ossom":25548,"umbledore":25549,"ĠCycling":25550,"ĠOccupy":25551,"AGES":25552,"Pen":25553,"ĠYug":25554,"Ġpackaged":25555,"Ġheightened":25556,"cot":25557,"stack":25558,"Cond":25559,"Ġstamps":25560,"mage":25561,"Ġpersuaded":25562,"Ġensl":25563,"ĠCardinal":25564,"Ġsolitary":25565,"Ġpossessing":25566,"ĠCork":25567,"Ġevid":25568,"ĠTay":25569,"Ġblues":25570,"Ġextremism":25571,"Ġlunar":25572,"Ġclown":25573,"Techn":25574,"Ġfestivals":25575,"ĠPvP":25576,"ĠLar":25577,"Ġconsequently":25578,"present":25579,"Ġsomeday":25580,"çİĭ":25581,"ĠMeteor":25582,"Ġtouring":25583,"culture":25584,"Ġbeaches":25585,"Ship":25586,"cause":25587,"ĠFlood":25588,"ãĥ¯":25589,"Ġpurity":25590,"those":25591,"Ġemission":25592,"bolt":25593,"Ġchord":25594,"ĠScripture":25595,"Lu":25596,"Ġ${":25597,"created":25598,"Others":25599,"258":25600,"Ġelemental":25601,"Ġannoyed":25602,"ĠAE":25603,"dan":25604,"ĠSag":25605,"Researchers":25606,"Ġfairy":25607,"âĢĵâĢĵ":25608,"============":25609,"Smart":25610,"GGGG":25611,"Ġskeletons":25612,"Ġpupils":25613,"linked":25614,"Ġurgency":25615,"enabled":25616,"ĠFuck":25617,"Ġcouncill":25618,"rab":25619,"UAL":25620,"TI":25621,"Ġlifes":25622,"Ġconfessed":25623,"Bug":25624,"Ġharmon":25625,"ĠCONFIG":25626,"ĠNeutral":25627,"Double":25628,"Ġstaple":25629,"ĠSHA":25630,"British":25631,"ĠSNP":25632,"ATOR":25633,"oco":25634,"Ġswinging":25635,"gex":25636,"oleon":25637,"plain":25638,"ĠMissing":25639,"ĠTrophy":25640,"vari":25641,"ranch":25642,"Ġ301":25643,"440":25644,"0000000000000000":25645,"Ġrestoring":25646,"Ġhaul":25647,"ucing":25648,"nerg":25649,"Ġfutures":25650,"Ġstrategist":25651,"question":25652,"Ġlateral":25653,"ĠBard":25654,"Ġsor":25655,"ĠRhodes":25656,"ĠDowntown":25657,"?????-":25658,"ĠLit":25659,"ĠBened":25660,"Ġcoil":25661,"street":25662,"ĠPortal":25663,"FILE":25664,"ĠGru":25665,"*,":25666,"231":25667,"neum":25668,"Ġsucked":25669,"Ġrapper":25670,"Ġtendencies":25671,"ĠLauren":25672,"cellaneous":25673,"267":25674,"Ġbrowse":25675,"Ġoverc":25676,"header":25677,"oise":25678,"Ġbeet":25679,"ĠGle":25680,"Stay":25681,"Ġmum":25682,"Ġtyped":25683,"Ġdiscounts":25684,"Talk":25685,"ĠOg":25686,"existing":25687,"ĠSell":25688,"uph":25689,"CI":25690,"ĠAustrian":25691,"ĠWarm":25692,"Ġdismissal":25693,"Ġaverages":25694,"camera":25695,"Ġallegiance":25696,"LAN":25697,"=\"#":25698,"Ġcommentators":25699,"ĠSetting":25700,"ĠMidwest":25701,"Ġpharmac":25702,"ĠEXP":25703,"Ġstainless":25704,"Chicago":25705,"Ġtan":25706,"244":25707,"Ġcountryside":25708,"ĠVac":25709,"295":25710,"Ġpinned":25711,"Ġcrises":25712,"Ġstandardized":25713,"Task":25714,"ĠJail":25715,"ĠDocker":25716,"colored":25717,"forth":25718,"\"},":25719,"Ġpatrons":25720,"Ġspice":25721,"Ġmourn":25722,"ĠMood":25723,"Ġlaundry":25724,"Ġequip":25725,"ĠMole":25726,"yll":25727,"ĠTHC":25728,"nation":25729,"ĠSherlock":25730,"Ġissu":25731,"ĠKre":25732,"ĠAmericas":25733,"ĠAAA":25734,"Ġsystematically":25735,"Ġcontra":25736,"ĠSally":25737,"Ġrationale":25738,"Ġcarriage":25739,"Ġpeaks":25740,"Ġcontradiction":25741,"ensation":25742,"ĠFailure":25743,"Ġprops":25744,"Ġnamespace":25745,"Ġcove":25746,"fields":25747,"ãĤĭ":25748,"Ġwool":25749,"ĠCatch":25750,"Ġpresumed":25751,"ĠDiana":25752,"ragon":25753,"igi":25754,"Ġhamm":25755,"Ġstunt":25756,"ĠGUI":25757,"ĠObservatory":25758,"ĠShore":25759,"Ġsmells":25760,"annah":25761,"Ġcockpit":25762,"ĠDuterte":25763,"850":25764,"Ġoppressed":25765,"breaker":25766,"ĠContribut":25767,"ĠPeru":25768,"ĠMonsanto":25769,"ĠAttempt":25770,"Ġcommanding":25771,"Ġfridge":25772,"ĠRin":25773,"ĠChess":25774,"uality":25775,"Ġol":25776,"Republican":25777,"ĠGlory":25778,"ĠWIN":25779,".......":25780,"agent":25781,"reading":25782,"Ġinh":25783,"Jones":25784,"Ġclicks":25785,"alan":25786,"Ġ[];":25787,"ĠMajesty":25788,"ĠCed":25789,"opus":25790,"atel":25791,"ê":25792,"ARC":25793,"ĠEcuador":25794,"ãĥł":25795,"ĠKuro":25796,"Ġrituals":25797,"Ġcaptive":25798,"Ġounce":25799,"Ġdisagreement":25800,"Ġslog":25801,"fuel":25802,"Pet":25803,"Mail":25804,"Ġexercised":25805,"Ġsolic":25806,"Ġrainfall":25807,"Ġdevotion":25808,"ĠAssessment":25809,"Ġrobotic":25810,"options":25811,"ĠRP":25812,"ĠFamilies":25813,"ĠFlames":25814,"Ġassignments":25815,"007":25816,"akedown":25817,"Ġvocabulary":25818,"Reilly":25819,"Ġcaval":25820,"gars":25821,"Ġsuppressed":25822,"ĠSET":25823,"ĠJohns":25824,"Ġwarp":25825,"broken":25826,"Ġstatues":25827,"Ġadvocated":25828,"Ġ275":25829,"Ġperil":25830,"omorph":25831,"ĠFemin":25832,"perfect":25833,"Ġhatch":25834,"Lib":25835,"512":25836,"Ġlifelong":25837,"313":25838,"Ġcheeks":25839,"Ġnumbered":25840,"ĠMug":25841,"Body":25842,"ravel":25843,"Weight":25844,"ĠJak":25845,"ĠHeath":25846,"Ġkissing":25847,"ĠJUST":25848,"Ġwaving":25849,"upload":25850,"Ġinsider":25851,"ĠProgressive":25852,"ĠFilter":25853,"tta":25854,"ĠBeam":25855,"Ġviolently":25856,"ipation":25857,"Ġskepticism":25858,"Ġ1918":25859,"ĠAnnie":25860,"ĠSI":25861,"Ġgenetics":25862,"Ġonboard":25863,"atl":25864,"ĠFriedman":25865,"ĠBri":25866,"ceptive":25867,"Ġpirate":25868,"ĠReporter":25869,"278":25870,"Ġmythology":25871,"Ġeclipse":25872,"Ġskins":25873,"Ġglyph":25874,"ingham":25875,"Files":25876,"Cour":25877,"women":25878,"Ġregimes":25879,"Ġphotographed":25880,"Kat":25881,"ĠMAX":25882,"Officials":25883,"Ġunexpectedly":25884,"Ġimpressions":25885,"Front":25886,";;;;;;;;":25887,"Ġsupremacy":25888,"Ġsang":25889,"Ġaggravated":25890,"Ġabruptly":25891,"ĠSector":25892,"Ġexcuses":25893,"Ġcosting":25894,"idepress":25895,"Stack":25896,"ĠRNA":25897,"obil":25898,"Ġghosts":25899,"ldon":25900,"atibility":25901,"Topics":25902,"Ġreimburse":25903,"ĠHM":25904,"ĠDeg":25905,"Ġthief":25906,"yet":25907,"ogenesis":25908,"leaning":25909,"ĠKol":25910,"ĠBasketball":25911,"Ġfi":25912,"ĠSeeing":25913,"Ġrecycling":25914,"Ġ[-":25915,"Congress":25916,"Ġlectures":25917,"Psy":25918,"Ġnep":25919,"Ġmaid":25920,"Ġoriented":25921,"AX":25922,"Ġrespectful":25923,"rene":25924,"flush":25925,"ĠUnloaded":25926,"request":25927,"grid":25928,"ĠAlternatively":25929,"ĠHugo":25930,"Ġdecree":25931,"ĠBuddhism":25932,"andum":25933,"Android":25934,"ĠCongo":25935,"ĠJoyce":25936,"Ġacknowledging":25937,"hesive":25938,"ĠTomorrow":25939,"ĠHiro":25940,"thren":25941,"ĠMaced":25942,"Ġhoax":25943,"ĠIncreased":25944,"ĠPradesh":25945,"Wild":25946,"______":25947,"161":25948,"Ġaunt":25949,"Ġdistributing":25950,"ĠTucker":25951,"ĠSSL":25952,"ĠWolves":25953,"Building":25954,"oult":25955,"ĠLuo":25956,"ĠYas":25957,"ĠSpir":25958,"ĠShape":25959,"ĠCambod":25960,"ĠIPv":25961,"Ġml":25962,"Ġextrad":25963,"390":25964,"ĠPenny":25965,"dream":25966,"Ġstationed":25967,"optional":25968,"eworthy":25969,".":26700,"ĠWorkshop":26701,"ĠRetail":26702,"ĠAvatar":26703,"625":26704,"Na":26705,"ĠVC":26706,"ĠSecure":26707,"MY":26708,"1988":26709,"ossip":26710,"Ġprostate":26711,"Ġunden":26712,"Ġgamer":26713,"ĠContents":26714,"ĠWarhammer":26715,"ĠSentinel":26716,"310":26717,"Ġsegregation":26718,"ĠFlex":26719,"ĠMAY":26720,"Ġdrills":26721,"ĠDrugs":26722,"Islamic":26723,"Ġspur":26724,"Ġcafe":26725,"Ġimaginary":26726,"Ġguiding":26727,"Ġswings":26728,"ĠTheme":26729,"oby":26730,"Ġnud":26731,"Ġbegging":26732,"Ġstrongh":26733,"Ġrejecting":26734,"Ġpedestrians":26735,"ĠProspect":26736,"Rare":26737,"sle":26738,"Ġconcessions":26739,"ĠConstitutional":26740,"Ġbeams":26741,"Ġfibers":26742,"poon":26743,"Ġinstincts":26744,"property":26745,"ĠBIG":26746,"Sanders":26747,"imates":26748,"Ġcoating":26749,"Ġcorpses":26750,"ĠTRUE":26751,"checked":26752,"Ġ166":26753,"Ash":26754,"ĠJS":26755,"ĠFiction":26756,"Ġcommunal":26757,"Ġenergetic":26758,"oooooooo":26759,"Ġnowadays":26760,"ILD":26761,"ibo":26762,"ĠSUV":26763,"Ren":26764,"Ġdwelling":26765,"Silver":26766,"Ġtally":26767,"ĠMoving":26768,"Ġcoward":26769,"Ġgenerals":26770,"Ġhorns":26771,"Ġcirculated":26772,"Ġrobbed":26773,"ĠUnlimited":26774,"Ġharassed":26775,"Ġinhibit":26776,"Ġcomposer":26777,"ĠSpotify":26778,"Ġspreads":26779,"364":26780,"Ġsuicidal":26781,"Ġnoises":26782,"ĠStur":26783,"Ġsaga":26784,"ĠKag":26785,"iso":26786,"Ġtheoretically":26787,"Money":26788,"Ġsimilarity":26789,"Ġsliced":26790,"utils":26791,"inges":26792,"\"-":26793,"Ġanth":26794,"Ġimped":26795,"Module":26796,"Throughout":26797,"Ġmenus":26798,"committee":26799,"andi":26800,"obj":26801,"inav":26802,"fired":26803,"ĠAbdullah":26804,"Ġundead":26805,"Ġfonts":26806,"Hold":26807,"ENG":26808,"Ġsustainability":26809,"Ġflick":26810,"Ġrazor":26811,"ĠFest":26812,"ĠCharacters":26813,"Ġwording":26814,"Ġpopulist":26815,"Ġcriticizing":26816,"Ġmuse":26817,"vine":26818,"Ġcardboard":26819,"Ġkindly":26820,"Ġfringe":26821,"ĠTheft":26822,"icultural":26823,"Ġgovernors":26824,"Ġ����":26825,"Ġ163":26826,"Ġtimeout":26827,"ĠAuth":26828,"Children":26829,"AU":26830,"Ġredemption":26831,"ĠAlger":26832,"Ġ1914":26833,"Ġwaved":26834,"Ġastronauts":26835,"ograms":26836,"Ġswamp":26837,"ĠFinnish":26838,"Ġcandle":26839,"Ġtonnes":26840,"utm":26841,"Ġray":26842,"Ġspun":26843,"Ġfearful":26844,"articles":26845,"Ġcaus":26846,"orically":26847,"ĠRequires":26848,"ĠGol":26849,"Ġpope":26850,"Ġinaugural":26851,"Ġgle":26852,"ADA":26853,"ĠISIL":26854,"ĠOffensive":26855,"Ġwatchdog":26856,"Ġbalcon":26857,"entity":26858,"ĠHoo":26859,"Ġgallon":26860,"ACC":26861,"Ġdoubling":26862,"Ġimplication":26863,"ĠSight":26864,"Ġdoctr":26865,"-------":26866,"Ġ\\\\":26867,"Ġmalt":26868,"Roll":26869,"Ġâī¥":26870,"Ġrecap":26871,"adding":26872,"uces":26873,"ĠBend":26874,"figure":26875,"Ġturkey":26876,"Ġsocietal":26877,"ĠTickets":26878,"Ġcommercially":26879,"Ġspicy":26880,"Ġ216":26881,"ĠRamp":26882,"Ġsuperiority":26883,"ï":26884,"ĠTracker":26885,"Carl":26886,"ĠCoy":26887,"ĠPatriot":26888,"Ġconsulted":26889,"Ġlistings":26890,"Ġslew":26891,"reenshot":26892,"ĠGone":26893,"Ġ[...]":26894,"309":26895,"Ġhottest":26896,"ر":26897,"Ġrocky":26898,"ĠDiaz":26899,"Ġmassage":26900,"Ġparaly":26901,"Ġpony":26902,"Az":26903,"Ġcartridge":26904,"ĠNZ":26905,"Ġsnack":26906,"ĠLamar":26907,"plement":26908,"ĠLeslie":26909,"Ġmater":26910,"Ġsnipp":26911,"246":26912,"Ġjointly":26913,"ĠBrisbane":26914,"ĠiPod":26915,"Ġpumping":26916,"Ġgoat":26917,"ĠSharon":26918,"ealing":26919,"Ġcoron":26920,"Ġanomal":26921,"rahim":26922,"ĠConnection":26923,"Ġsculpture":26924,"Ġscheduling":26925,"ĠDaddy":26926,"athing":26927,"Ġeyebrows":26928,"Ġcurved":26929,"Ġsentiments":26930,"Ġdrafting":26931,"Drop":26932,"([":26933,"Ġnominal":26934,"ĠLeadership":26935,"ĠGrow":26936,"Ġ176":26937,"Ġconstructive":26938,"ivation":26939,"Ġcorrupted":26940,"gerald":26941,"ĠCros":26942,"ĠChester":26943,"ĠLap":26944,"ãģª":26945,"OTH":26946,"DATA":26947,"Ġalmond":26948,"probably":26949,"Imp":26950,"Ġfeast":26951,"ĠWarcraft":26952,"Flor":26953,"Ġcheckpoint":26954,"Ġtranscription":26955,"Ġ204":26956,"Ġtweaks":26957,"Ġrelieve":26958,"Science":26959,"Ġperformer":26960,"Zone":26961,"Ġturmoil":26962,"igated":26963,"hibit":26964,"ĠCafe":26965,"themed":26966,"Ġfluor":26967,"bench":26968,"Ġdecom":26969,"ĠUnt":26970,"ĠBarrett":26971,"ĠFacts":26972,"Ġtasting":26973,"ĠPTSD":26974,"ĠSeal":26975,"ĠJudaism":26976,"ĠDynamic":26977,"ĠCors":26978,"Ve":26979,"ĠMing":26980,"ĠTransform":26981,"von":26982,"ĠDefenders":26983,"ĠTactical":26984,"ĠVon":26985,"ĠUnivers":26986,"Ġdistorted":26987,"ĠBreath":26988,"?'\"":26989,"Ġagon":26990,"ĠDeadly":26991,"Ġlan":26992,"ĠCycle":26993,"orned":26994,"Ġreliably":26995,"Ġglor":26996,"ĠMonkey":26997,"ãĥ¡":26998,"Ġadren":26999,"Ġmicrowave":27000,"ĠAlban":27001,"ircraft":27002,"digit":27003,"smart":27004,"ĠDread":27005,"¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯":27006,"{{":27007,"ĠRochester":27008,"Ġsimplified":27009,"Ġinflicted":27010,"Ġtakeover":27011,"Ġyourselves":27012,"aditional":27013,"Ġmuscular":27014,"KS":27015,"Ġingen":27016,"Tax":27017,"ĠFeature":27018,"277":27019,"Ġcruc":27020,"Ġcrate":27021,"Ġunidentified":27022,"Ġacclaimed":27023,"ĠManga":27024,"ĠFrances":27025,"ĠNepal":27026,"ĠGerald":27027,"ĠKuwait":27028,"Ġslain":27029,"ĠHeb":27030,"ĠGoku":27031,"ãģ®æ":27032,"286":27033,"Mrs":27034,"ĠCody":27035,"ĠSanctuary":27036,"016":27037,"Ġdismant":27038,"Ġdataset":27039,"ĠHond":27040,"buck":27041,"ĠPatterson":27042,"Ġpalette":27043,"ĠGD":27044,"icol":27045,"ĠLodge":27046,"Ġplanetary":27047,"akin":27048,"ĠRegistered":27049,"abwe":27050,"ĠPetersburg":27051,"Ġhailed":27052,"ĠPiece":27053,"Sche":27054,"ĠDOJ":27055,"Ġenumer":27056,"181":27057,"ĠObserver":27058,"ĠBold":27059,"founded":27060,"commerce":27061,"Ġexploits":27062,"ĠFinding":27063,"URN":27064,"ĠSne":27065,"ĠAcid":27066,"ayette":27067,"ĠValues":27068,"Ġdrastic":27069,"Ġarchitectural":27070,"Ġ\".":27071,"×ķ":27072,"umped":27073,"Ġwrapping":27074,"Ġwidow":27075,"ĠSlayer":27076,"lace":27077,"once":27078,"Germany":27079,"avoid":27080,"Ġtemples":27081,"PAR":27082,"ô":27083,"ĠLucifer":27084,"ĠFlickr":27085,"lov":27086,"forces":27087,"Ġscouting":27088,"Ġlouder":27089,"tesy":27090,"Ġbeforehand":27091,"Äĵ":27092,"ĠNeon":27093,"ĠWol":27094,"ĠTypically":27095,"ĠPolitico":27096,"-+-+":27097,"Ġbuilder":27098,"Ġderive":27099,"Kill":27100,"Ġpoker":27101,"Ġambiguous":27102,"Ġlifts":27103,"Ġcyt":27104,"Ġribs":27105,"oodle":27106,"ĠSounds":27107,"hair":27108,"ĠSyndrome":27109,"tf":27110,"Ġproportional":27111,"uid":27112,"Ġpertaining":27113,"ĠKindle":27114,"ĠNegro":27115,"Ġreiterated":27116,"ĠTonight":27117,"oths":27118,"ĠCornell":27119,"Ġowing":27120,"Ġ208":27121,"elfare":27122,"ocating":27123,"ĠBirds":27124,"Subscribe":27125,"Ġessays":27126,"Ġburdens":27127,"Ġillustrations":27128,"arious":27129,"ERAL":27130,"ĠCalcul":27131,"Ġxen":27132,"ĠLinkedIn":27133,"ĠJung":27134,"Ġredesign":27135,"Connor":27136,"296":27137,"Ġreversal":27138,"ĠAdelaide":27139,"ĠLL":27140,"Ġsinking":27141,"Ġgum":27142,"USH":27143,"capt":27144,"ĠGrimm":27145,"Ġfootsteps":27146,"ĠCBD":27147,"ispers":27148,"Ġprose":27149,"Wednesday":27150,"ĠMovies":27151,"edin":27152,"Ġoverturned":27153,"Ġcontentious":27154,"USB":27155,"~~~~~~~~~~~~~~~~":27156,"ĠCopper":27157,"Ġpointless":27158,"NV":27159,"values":27160,"olphin":27161,"dain":27162,"Ġdeposited":27163,"ĠGW":27164,"Ġpreceded":27165,"ĠCla":27166,"ĠGolem":27167,"ĠNim":27168,"Ġβ":27169,"ĠEngineers":27170,"middle":27171,"Ġflatt":27172,"operative":27173,"Ġcouncils":27174,"imbabwe":27175,"elin":27176,"Ġstressful":27177,"ĠLD":27178,"Ġresh":27179,"lake":27180,"Ġwheelchair":27181,"ĠAlternative":27182,"Ġoptimize":27183,"operation":27184,"Ġpeek":27185,"Ġoneself":27186,"igil":27187,"Ġtransitions":27188,"opathy":27189,"blank":27190,"Ġ169":27191,"171":27192,"________________________________________________________________":27193,"Ġlaundering":27194,"Enc":27195,"ĠDEC":27196,"Ġworkouts":27197,"Ġspikes":27198,"Ġdinosaurs":27199,"Ġdiscriminatory":27200,"Pool":27201,"Rather":27202,"385":27203,"RNA":27204,"testers":27205,"eto":27206,"ĠIdentity":27207,"Ġvein":27208,"ĠBurton":27209,"Ġarcade":27210,"420":27211,"Ultimately":27212,"ĠSadly":27213,"ð":27214,"pill":27215,"Ġcubic":27216,"ĠSpectrum":27217,"these":27218,"states":27219,"Ġunofficial":27220,"hawks":27221,"ĠEVERY":27222,"Ġrainbow":27223,"Ġincarceration":27224,"anding":27225,"Ġsyll":27226,"ĠEverton":27227,"Ġ179":27228,"ĠSerbia":27229,"Ġ189":27230,"meter":27231,"ĠMickey":27232,"Ġantiqu":27233,"Ġfactual":27234,"neck":27235,"ĠNare":27236,"norm":27237,"must":27238,"Ġhighways":27239,"Ġglam":27240,"Ġdividing":27241,"ĠSquadron":27242,"ĠMartha":27243,"Ġbirths":27244,"Cover":27245,"////////////////":27246,"ĠWong":27247,"Phot":27248,"ĠALS":27249,"rio":27250,"ĠNonetheless":27251,"ĠLemon":27252,"Ġ206":27253,"ĠEE":27254,"Ġderivative":27255,"ĠWWII":27256,"vote":27257,"Ġtherein":27258,"Ġseparating":27259,"446":27260,"sync":27261,"ĠStreets":27262,"Ġratt":27263,"Ġmunicipality":27264,"ĠShortly":27265,"Ġmonk":27266,"),\"":27267,"Ġscrub":27268,"Ġoperatives":27269,"Neither":27270,"Place":27271,"ĠLimit":27272,"Female":27273,"ĠActor":27274,"Character":27275,"Ġconstituted":27276,"357":27277,"Ġprotested":27278,"ĠStraw":27279,"ĠHeight":27280,"ilda":27281,"ĠTyph":27282,"Ġfloods":27283,"Ġcosmetic":27284,"WAY":27285,"perture":27286,"upon":27287,"tons":27288,"essing":27289,"ĠPocket":27290,"Ġrooft":27291,"ĠCaucas":27292,"Ġantidepress":27293,"Ġincompatible":27294,"ECD":27295,"Ġopera":27296,"ĠContest":27297,"Ġgenerators":27298,"lime":27299,"Defense":27300,"1987":27301,"forum":27302,"Ġsavage":27303,"ĠHungarian":27304,"nz":27305,"Ġmetallic":27306,"Ġexpelled":27307,"Ġresidency":27308,"Ġdresses":27309,"666":27310,"ĠClement":27311,"fires":27312,"Category":27313,"Ġgeek":27314,"alis":27315,"Ġcemetery":27316,"educated":27317,"Ġcrawl":27318,"ĠUnable":27319,"ĠTyson":27320,"akis":27321,"Ġpardon":27322,"ĠWra":27323,"Ġstrengthened":27324,"ĠFors":27325,"335":27326,"ĠHC":27327,"ĠMond":27328,"Ġvisuals":27329,"ĠBeatles":27330,"ettlement":27331,"Ġï":27332,"gro":27333,"Ġbash":27334,"Ġpoorest":27335,"Ġexcel":27336,"Ġaspirations":27337,"ĠMunicip":27338,"ensible":27339,"Ġceremonies":27340,"Ġintimidation":27341,"ĠCONTR":27342,"beck":27343,"ĠKap":27344,"asu":27345,"Ġtrademarks":27346,"ĠSew":27347,"ĠCompetition":27348,"network":27349,"ĠArri":27350,"ĠTet":27351,"Roaming":27352,"WC":27353,"Dat":27354,"Ġsob":27355,"Ġpairing":27356,"Ġoverdose":27357,"SAY":27358,"aber":27359,"Ġrevolt":27360,"ĠFah":27361,"acting":27362,"eq":27363,"estation":27364,"Fight":27365,"ĠMarks":27366,"273":27367,"Ġ178":27368,"Raw":27369,"ãģĭ":27370,"349":27371,"blocks":27372,"Ġverge":27373,"estine":27374,"ĠPodesta":27375,"Ġinvasive":27376,"Ġprofoundly":27377,"ĠAo":27378,"each":27379,"Ġlest":27380,"interpret":27381,"Ġshrinking":27382,"Ġerrone":27383,"Ġchees":27384,"lys":27385,"ĠIvy":27386,"ĠDirectory":27387,"Ġhinted":27388,"VICE":27389,"Ġcontacting":27390,"ĠGent":27391,"hei":27392,"Ġlabeling":27393,"Ġmercury":27394,"ĠLite":27395,"Ġexpires":27396,"Ġdestabil":27397,"ritis":27398,"cu":27399,"Ġfeathers":27400,"Ġsteer":27401,"Ġprogrammed":27402,"ĠVader":27403,"Going":27404,"ĠElim":27405,"Ġyo":27406,"ĠMiche":27407,"Ġ203":27408,"Ġsleeves":27409,"Ġbully":27410,"ĠHumans":27411,"368":27412,"Ġcompress":27413,"ĠBanner":27414,"ARS":27415,"Ġawhile":27416,"Ġcalib":27417,"Ġsponsorship":27418,"ĠDifficulty":27419,"ĠPapers":27420,"Ġidentifier":27421,"}.":27422,"Ġyog":27423,"ĠShia":27424,"Ġcleanup":27425,"Ġvibe":27426,"introdu":27427,"imming":27428,"Australia":27429,"Ġoutlines":27430,"ĠYoutube":27431,"train":27432,"ĠMakes":27433,"Ġdeported":27434,"Ġcentr":27435,"ĠDug":27436,"ĠBoulder":27437,"ĠBuffy":27438,"Ġinjunction":27439,"ĠHarley":27440,"ĠGroups":27441,"ĠDumbledore":27442,"ĠClara":27443,"Ġ\"-":27444,"Ġsacrificed":27445,"eph":27446,"Shadow":27447,"ibling":27448,"Ġfreelance":27449,"Ġevidently":27450,"phal":27451,"Ġretains":27452,"Mir":27453,"Ġfinite":27454,"dar":27455,"ĠCous":27456,"Ġrepaired":27457,"Ġperiodic":27458,"Ġchampionships":27459,"Ġasteroid":27460,"blind":27461,"Ġexpressly":27462,"ĠAstros":27463,"Ġscaled":27464,"Ġgeographical":27465,"ĠRapids":27466,"Enjoy":27467,"Ġelastic":27468,"ĠMohamed":27469,"Market":27470,"begin":27471,"Ġdiscovers":27472,"Ġtelecommunications":27473,"Ġscanner":27474,"Ġenlarge":27475,"Ġsharks":27476,"Ġpsychedel":27477,"ĠRouge":27478,"Ġsnapshot":27479,"isine":27480,"XP":27481,"Ġpesticides":27482,"ĠLSD":27483,"ĠDistribution":27484,"really":27485,"Ġdegradation":27486,"Ġdisguise":27487,"Ġbiom":27488,"ĠEXT":27489,"Ġequations":27490,"Ġhazards":27491,"ĠCompared":27492,")*":27493,"Ġvirtues":27494,"Ġelders":27495,"Ġenhancing":27496,"ĠAcross":27497,"eros":27498,"angling":27499,"Ġcombust":27500,"ucci":27501,"Ġconcussion":27502,"Ġcontraception":27503,"ĠKang":27504,"Ġexpresses":27505,"Ġaux":27506,"ĠPione":27507,"Ġexhibits":27508,"Debug":27509,"OTAL":27510,"ĠAlready":27511,"ĠWheeler":27512,"Ġexpands":27513,"?:":27514,"Ġreconciliation":27515,"Ġpirates":27516,"Ġpurse":27517,"Ġdiscourage":27518,"Ġspectacle":27519,"Rank":27520,"Ġwraps":27521,"ĠThought":27522,"Ġimpending":27523,"Opp":27524,"ĠAnglo":27525,"ĠEUR":27526,"Ġscrewed":27527,"retched":27528,"Ġencouragement":27529,"models":27530,"Ġconfuse":27531,"mmm":27532,"ĠVitamin":27533,"âĸijâĸij":27534,"Cru":27535,"Ġknights":27536,"Ġdiscard":27537,"Ġbishops":27538,"ĠWear":27539,"ĠGarrett":27540,"kan":27541,"ãĥŁ":27542,"Ġmasculine":27543,"capital":27544,"ĠAus":27545,"Ġfatally":27546,"thanks":27547,"ĠAU":27548,"ĠGut":27549,"1200":27550,"Ġ00000000":27551,"Ġsurrog":27552,"ĠBIOS":27553,"raits":27554,"ĠWatts":27555,"Ġresurrection":27556,"ĠElectoral":27557,"ĠTips":27558,"4000":27559,"Ġnutrient":27560,"Ġdepicting":27561,"Ġsprink":27562,"Ġmuff":27563,"ĠLIM":27564,"ĠSample":27565,"psc":27566,"ibi":27567,"generated":27568,"Ġspecimens":27569,"Ġdissatisf":27570,"Ġtailored":27571,"Ġholdings":27572,"ĠMonthly":27573,"ĠEat":27574,"poons":27575,"Ġnec":27576,"ĠCage":27577,"ĠLotus":27578,"ĠLantern":27579,"Ġfrontier":27580,"Ġpensions":27581,"Ġjoked":27582,"ĠHardy":27583,"=-=-=-=-":27584,"rade":27585,"UID":27586,"Ġrails":27587,"Ġemit":27588,"Ġslate":27589,"Ġsmug":27590,"Ġspit":27591,"ĠCalls":27592,"ĠJacobs":27593,"feat":27594,"ĠUE":27595,"Ġrestruct":27596,"Ġregeneration":27597,"Ġenergies":27598,"ĠConnor":27599,"OHN":27600,"ĠCheese":27601,"Ġger":27602,"Ġresurrect":27603,"management":27604,"NW":27605,"Ġpresently":27606,"ĠBruins":27607,"Member":27608,"ĠMang":27609,"idan":27610,"Ġboosting":27611,"wyn":27612,"+.":27613,"requisite":27614,"ĠNYPD":27615,"ĠMegan":27616,"ĠConditions":27617,"Ġpics":27618,"nesium":27619,"ĠRash":27620,"Ġ174":27621,"ĠDucks":27622,"Ġembro":27623,"zu":27624,"onian":27625,"religious":27626,"Ġcraz":27627,"ĠACA":27628,"ĠZucker":27629,"EMA":27630,"ĠPros":27631,"Weapon":27632,"ĠKnox":27633,"ĠArduino":27634,"Ġstove":27635,"Ġheavens":27636,"ĠPurchase":27637,"Ġherd":27638,"Ġfundraiser":27639,"Digital":27640,"5000":27641,"Ġproponents":27642,"/âĢĭ":27643,"Ġjelly":27644,"ĠVisa":27645,"Ġmonks":27646,"Ġadvancement":27647,"ĠWer":27648,"Ġ187":27649,"eus":27650,"ertility":27651,"Ġfetal":27652,"Ġ1936":27653,"Lo":27654,"Ġoutfits":27655,"Ġstaircase":27656,"bomb":27657,"Ġcustomized":27658,"clair":27659,"Tree":27660,"Ġmapped":27661,"ĠConsidering":27662,"ĠTorres":27663,"Ġmethyl":27664,"Ġapproximate":27665,"Ġdoom":27666,"ĠHansen":27667,"Ġcrossover":27668,"Ġstandalone":27669,"ä¼":27670,"Ġinvites":27671,"Ġgraveyard":27672,"Ġhp":27673,"DonaldTrump":27674,"Ġescort":27675,"Gar":27676,"Ġpredecessors":27677,"Ġhay":27678,"Ġenzyme":27679,"ĠStraight":27680,"visors":27681,"Ing":27682,"aneously":27683,"ĠApplied":27684,"Ġfec":27685,"ĠDurant":27686,"Ġoutspoken":27687,"orb":27688,"Ġzeal":27689,"Ġdisgrace":27690,"').":27691,"ĠCheng":27692,"289":27693,"ĠRena":27694,"ĠSuicide":27695,"294":27696,"Ġoutraged":27697,"ĠNewman":27698,"ĠNvidia":27699,"ĠAber":27700,"ĠBers":27701,"Ġrecreation":27702,"Window":27703,"ĠDP":27704,"xe":27705,"Ġpedoph":27706,"Ġfallout":27707,"amboo":27708,"Ġpresentations":27709,"ĠApps":27710,"Ġhtml":27711,"345":27712,"ĠXXX":27713,"Ġrubbing":27714,"ĠLeather":27715,"Ġhumidity":27716,"seys":27717,"established":27718,"ĠUnits":27719,"646":27720,"Ġrespectable":27721,"Auto":27722,"Ġthriving":27723,"ĠInnovation":27724,"angs":27725,"Extra":27726,"regulation":27727,"298":27728,"pick":27729,"Examples":27730,"ĠCJ":27731,"Attack":27732,"Ġdracon":27733,"LT":27734,"Ġsticker":27735,"rers":27736,"Ġsunny":27737,"Iss":27738,"regulated":27739,"dim":27740,"ĠAbstract":27741,"Ġhusbands":27742,"Office":27743,"omination":27744,"itars":27745,"ANGE":27746,"ascal":27747,"ĠKris":27748,"ĠInfantry":27749,"Ġmalf":27750,"ĠAthe":27751,"ĠRally":27752,"balanced":27753,"........................":27754,"OUP":27755,"Ġmolecule":27756,"metics":27757,"ĠSplit":27758,"ĠInstructions":27759,"ĠNights":27760,"cards":27761,"Ġtug":27762,"Ġcone":27763,"åŃ":27764,"Ġtx":27765,"ĠDiscussion":27766,"Ġcatastrophe":27767,"ppe":27768,"gio":27769,"Ġcommunism":27770,"Ġhalted":27771,"ĠGuant":27772,"clean":27773,"ĠSched":27774,"ĠKanye":27775,"Ġwander":27776,"ĠSeriously":27777,"Ġ188":27778,"ennial":27779,"follow":27780,"productive":27781,"ĠFlow":27782,"ĠSail":27783,"Ġcraw":27784,"Ġsimulations":27785,"oru":27786,"angles":27787,"ĠNolan":27788,"Ġmenstru":27789,"470":27790,"Ġ207":27791,"aja":27792,"Ġcasually":27793,"boarding":27794,"Ġ222":27795,"ovy":27796,"ĠNumbers":27797,"umat":27798,"OE":27799,"287":27800,"ĠClemson":27801,"Ġcerts":27802,"Ġslid":27803,"ĠTribe":27804,"Ġtoast":27805,"Ġfortunes":27806,"Ġfals":27807,"ĠCommittees":27808,"Ġgp":27809,"Ġfiery":27810,"ĠNets":27811,"ĠAnime":27812,"Package":27813,"ĠCompare":27814,"laughter":27815,"infect":27816,"Ġatrocities":27817,"Ġjustices":27818,"Ġinsults":27819,"ĠVernon":27820,"Ġshaken":27821,"Ġpersona":27822,"estamp":27823,"367":27824,"brain":27825,"Ġexperimenting":27826,"Ken":27827,"ĠElectronics":27828,"Ġ161":27829,"domain":27830,"Ġgraphical":27831,"bishop":27832,"Ġwhopping":27833,"ĠEvangel":27834,"Ġadvertisers":27835,"ĠSpear":27836,"Ġbids":27837,"Ġdestroys":27838,"utz":27839,"Ġundersc":27840,"ĠADD":27841,"Ġants":27842,"ĠCum":27843,"ipples":27844,"ĠFill":27845,"Ġglanced":27846,"Ġindicted":27847,"ĠEff":27848,"Ġmiscon":27849,"ĠDesktop":27850,"Ġabide":27851,"ãĥĢ":27852,"ĠIo":27853,"ĠCoul":27854,"Ġcapsule":27855,"ĠChrys":27856,"MON":27857,"Ġundes":27858,"ĠIRA":27859,"Ġcitation":27860,"Ġdictate":27861,"ĠNetworks":27862,"ĠConflict":27863,"ĠStuff":27864,"xa":27865,"isec":27866,"ĠChemistry":27867,"Ġquarterly":27868,"Williams":27869,"anan":27870,"Opt":27871,"ĠAlexandria":27872,"outheastern":27873,"ĠSpringfield":27874,"ĠBlacks":27875,"Ġgeography":27876,"242":27877,"Ġutmost":27878,"ĠExxon":27879,"abouts":27880,"EVA":27881,"ĠEnable":27882,"ĠBarr":27883,"Ġdisagreed":27884,"ĠCyprus":27885,"Ġdementia":27886,"Ġlabs":27887,"Ġubiquitous":27888,"ĠLOVE":27889,"Ġconsolidated":27890,"sr":27891,"Ġcreamy":27892,"ĠTimber":27893,"Regardless":27894,"ĠCertificate":27895,"Ġ\"...":27896,"ogenous":27897,"Captain":27898,"Ġinsulting":27899,"ĠSoros":27900,"ĠInstr":27901,"ĠBulgaria":27902,"better":27903,"Ġsucking":27904,"ĠDavidson":27905,"atz":27906,"Ġcollateral":27907,"gif":27908,"Ġplagued":27909,"ĠCancel":27910,"ĠGardner":27911,"RB":27912,"Ġsixteen":27913,"Remove":27914,"uristic":27915,"cook":27916,"Rod":27917,"Ġcomprising":27918,"fle":27919,")âĢĶ":27920,"ĠViking":27921,"growth":27922,"agonal":27923,"Ġsrf":27924,"afety":27925,"mot":27926,"Nearly":27927,"stown":27928,"ĠFactor":27929,"Ġautomobile":27930,"Ġprocedural":27931,"mask":27932,"ampires":27933,"Ġdisappears":27934,"jab":27935,"315":27936,"Ġ1951":27937,"needed":27938,"Ġdaring":27939,"leader":27940,"Ġpodium":27941,"Ġunhealthy":27942,"Ġmund":27943,"Ġpyramid":27944,"ocre":27945,"Ġkissed":27946,"Ġdreamed":27947,"ĠFantastic":27948,"ĠGly":27949,"åĬ":27950,"Ġgreatness":27951,"Ġspices":27952,"Ġmetropolitan":27953,"Ġcompuls":27954,"iets":27955,"1016":27956,"ĠSham":27957,"ĠPyr":27958,"flies":27959,"ĠMidnight":27960,"Ġswallowed":27961,"Ġgenres":27962,"ĠLucky":27963,"ĠRewards":27964,"Ġdispatch":27965,"ĠIPA":27966,"ĠApply":27967,"Ġaven":27968,"alities":27969,"312":27970,"things":27971,"Ġ().":27972,"Ġmates":27973,"ĠSz":27974,"ĠCOP":27975,"olate":27976,"OFF":27977,"Ġrecharge":27978,"caps":27979,"ĠYorker":27980,"icone":27981,"Ġgalaxies":27982,"ileaks":27983,"Dave":27984,"ĠPuzz":27985,"ĠCeltic":27986,"ĠAFC":27987,"276":27988,"ĠSons":27989,"Ġaffirmative":27990,"Hor":27991,"Ġtutorials":27992,"ĠCITY":27993,"ĠRosa":27994,"ĠExtension":27995,"Series":27996,"Ġfats":27997,"Ġrab":27998,"lis":27999,"Ġunic":28000,"Ġeve":28001,"ĠSpin":28002,"Ġadulthood":28003,"typ":28004,"Ġsectarian":28005,"Ġcheckout":28006,"ĠCycl":28007,"Single":28008,"Ġmartyr":28009,"Ġchilling":28010,"888":28011,"oufl":28012,"Ġ];":28013,"Ġcongestion":28014,"mk":28015,"ĠWhereas":28016,"Ġ1938":28017,"urrencies":28018,"erion":28019,"Ġboast":28020,"ĠPatients":28021,"Ġchap":28022,"ĠBD":28023,"realDonaldTrump":28024,"Ġexamines":28025,"hov":28026,"Ġstartling":28027,"ĠBabylon":28028,"wid":28029,"omew":28030,"brance":28031,"ĠOdyssey":28032,"wig":28033,"Ġtorch":28034,"ĠVox":28035,"ĠMoz":28036,"ĠTroll":28037,"ĠAns":28038,"Similarly":28039,"ĠFul":28040,"006":28041,"Unless":28042,"ĠAlone":28043,"stead":28044,"ĠPublisher":28045,"rights":28046,"tu":28047,"ĠDoesn":28048,"Ġprofessionally":28049,"Ġclo":28050,"icz":28051,"Ġsteals":28052,"Ġá":28053,"1986":28054,"Ġsturdy":28055,"ĠJohann":28056,"Ġmedals":28057,"Ġfilings":28058,"ĠFraser":28059,"done":28060,"Ġmultinational":28061,"Ġfeder":28062,"Ġworthless":28063,"Ġpest":28064,"Yesterday":28065,"ankind":28066,"Ġgays":28067,"Ġborne":28068,"ĠPOS":28069,"Picture":28070,"Ġpercentages":28071,"251":28072,"rame":28073,"Ġpotions":28074,"AMD":28075,"ĠLebanese":28076,"Ġrang":28077,"ĠLSU":28078,"ongs":28079,"Ġpeninsula":28080,"ĠClause":28081,"ALK":28082,"oha":28083,"ĠMacBook":28084,"Ġunanimous":28085,"Ġlenders":28086,"Ġhangs":28087,"Ġfranchises":28088,"orers":28089,"ĠUpdates":28090,"Ġisolate":28091,"andro":28092,"Soon":28093,"Ġdisruptive":28094,"ĠSurve":28095,"Ġstitches":28096,"ĠScorp":28097,"ĠDominion":28098,"Ġsupplying":28099,"Arg":28100,"Ġturret":28101,"ĠLuk":28102,"Ġbrackets":28103,"*)":28104,"ĠRevolutionary":28105,"ĠHonest":28106,"Ġnoticing":28107,"ĠShannon":28108,"Ġafforded":28109,"Ġtha":28110,"ĠJanet":28111,"!--":28112,"ĠNarendra":28113,"ĠPlot":28114,"Hol":28115,"sever":28116,"eenth":28117,"Ġobstruction":28118,"Ġ1024":28119,"staff":28120,"jas":28121,"orget":28122,"scenes":28123,"laughs":28124,"ĠFargo":28125,"crime":28126,"Ġorchestr":28127,"Ġdelet":28128,"iliary":28129,"rieved":28130,"Ġmilitar":28131,"ĠGreene":28132,"âĹı":28133,"ãģ¦":28134,"ĠGuards":28135,"Ġunleashed":28136,"ĠWeber":28137,"Ġadjustable":28138,"Ġcaliber":28139,"Ġmotivations":28140,"ĠÃł":28141,"mAh":28142,"ĠLanka":28143,"handle":28144,"Ġpent":28145,"ĠRav":28146,"ĠAngular":28147,"ĠKau":28148,"umbing":28149,"Ġphilanthrop":28150,"Ġdehyd":28151,"Ġtoxicity":28152,"eer":28153,"ĠYORK":28154,"witz":28155,"å¼":28156,"ĠIE":28157,"community":28158,"ĠAH":28159,"Ġretali":28160,"Ġmassively":28161,"ĠDaniels":28162,"ĠDEL":28163,"Ġcarcin":28164,"Url":28165,"Ġrouting":28166,"ĠNPCs":28167,"ĠRAF":28168,"ryce":28169,"Ġwaived":28170,"ĠGuatem":28171,"Everybody":28172,"Ġcovenant":28173,"Ġ173":28174,"Ġrelaxing":28175,"Ġquart":28176,"almost":28177,"Ġguarded":28178,"ĠSoldiers":28179,"ĠPLAY":28180,"Ġoutgoing":28181,"LAND":28182,"Ġrewrite":28183,"ĠMOV":28184,"ĠImper":28185,"ĠSolution":28186,"Ġphenomenal":28187,"Ġlongevity":28188,"Ġimpat":28189,"ĠNissan":28190,"irie":28191,"Ġodor":28192,"ĠZar":28193,"oks":28194,"Ġmilitias":28195,"ĠSPEC":28196,"Ġtolerated":28197,"arser":28198,"ĠBradford":28199,"+,":28200,"Ġsurreal":28201,"sf":28202,"Canadian":28203,"Ġresemblance":28204,"Ġcarbohydrate":28205,"VIEW":28206,"Ġaccessory":28207,"meal":28208,"largest":28209,"iegel":28210,"Someone":28211,"Ġtoughest":28212,"oso":28213,"Ġfunnel":28214,"Ġcondemnation":28215,"luent":28216,"Ġwired":28217,"ĠSunset":28218,"Jesus":28219,"ĠPST":28220,"ĠPages":28221,"ĠTycoon":28222,"ĠPF":28223,"Ġselections":28224,"Ġà¤":28225,"partisan":28226,"Ġhighs":28227,"ĠRune":28228,"Ġcrafts":28229,"lead":28230,"ĠParents":28231,"Ġreclaim":28232,"eker":28233,"ĠAllied":28234,"aeper":28235,"Ġlooming":28236,"Ġbeneficiaries":28237,"ĠHull":28238,"Students":28239,"Jewish":28240,"dj":28241,"Ġpact":28242,"template":28243,"ĠOfficials":28244,"ĠBaylor":28245,"Ġhemp":28246,"Ġyouths":28247,"ĠLevels":28248,"ĠXiao":28249,"ĠChes":28250,"Ġendeavor":28251,"ĠRemoved":28252,"Ġhippocamp":28253,"Hell":28254,"ãĤĬ":28255,"805":28256,"Ġdinosaur":28257,"ĠWrath":28258,"ĠIndonesian":28259,"Ġcalculator":28260,"ĠDictionary":28261,"Ġ420":28262,"ĠMAG":28263,"(_":28264,"!,":28265,"tarians":28266,"Ġrestricting":28267,"racuse":28268,"Ġweekday":28269,"OUNT":28270,"Ġshrugged":28271,"leground":28272,"Ġbald":28273,"ĠDoctors":28274,"Ġtouted":28275,"ĠMaxwell":28276,"Ġ214":28277,"Ġdiplomat":28278,"Ġrepression":28279,"Ġconstituency":28280,"vice":28281,"ranked":28282,"ĠNapoleon":28283,"gang":28284,"ĠForever":28285,"tun":28286,"Ġbulb":28287,"ĠPDT":28288,"ĠCisco":28289,"VEN":28290,"Ġresumed":28291,"Steven":28292,"ĠManitoba":28293,"Ġfabulous":28294,"ĠAgents":28295,"1984":28296,"Ġamusing":28297,"ĠMysteries":28298,"Ġorthodox":28299,"floor":28300,"Ġquestionnaire":28301,"Ġpenetrate":28302,"Ġfilmmakers":28303,"ĠUnc":28304,"Ġstamped":28305,"Ġthirteen":28306,"Ġoutfield":28307,"Ġforwarded":28308,"Ġappra":28309,"Ġaided":28310,"try":28311,"Ġunfocused":28312,"ĠLiz":28313,"ĠWendy":28314,"ĠScene":28315,"Charg":28316,"Ġrejects":28317,"Ġleftist":28318,"ĠProvidence":28319,"ĠBrid":28320,"regn":28321,"Ġprophecy":28322,"ĠLIVE":28323,"499":28324,"Ġforge":28325,"ĠFML":28326,"Ġintrinsic":28327,"ĠFrog":28328,"Ġwont":28329,"ĠHolt":28330,"Ġfamed":28331,"CLUS":28332,"aepernick":28333,"ĠHate":28334,"ĠCay":28335,"Ġregistering":28336,"ortality":28337,"ropy":28338,"ocalyptic":28339,"aan":28340,"nav":28341,"Ġfascist":28342,"IFIED":28343,"Ġimplicated":28344,"ĠResort":28345,"ĠChandler":28346,"ĠBrick":28347,"Pin":28348,"ysc":28349,"Usage":28350,"ĠHelm":28351,"usra":28352,"âĺħâĺħ":28353,"ĠAbbas":28354,"Ġunanimously":28355,"Ġkeeper":28356,"Ġaddicted":28357,"???":28358,"Ġhelmets":28359,"Ġantioxid":28360,"apsed":28361,"808":28362,"giene":28363,"Ġwaits":28364,"Ġminion":28365,"raved":28366,"ĠPorsche":28367,"Ġdreaming":28368,"Ġ171":28369,"ĠCain":28370,"Ġunfor":28371,"asso":28372,"ĠConfiguration":28373,"kun":28374,"hardt":28375,"Ġnested":28376,"ĠLDS":28377,"LES":28378,"Ġtying":28379,"enos":28380,"Ġcue":28381,"ĠMarqu":28382,"skirts":28383,"Ġclicked":28384,"Ġexpiration":28385,"ĠAccordingly":28386,"ĠWC":28387,"Ġblessings":28388,"Ġaddictive":28389,"ĠNarr":28390,"yx":28391,"ĠJaguars":28392,"Ġrents":28393,"ĠSiber":28394,"Ġtipped":28395,"ousse":28396,"ĠFitzgerald":28397,"Ġhierarch":28398,"outine":28399,"Ġwavelength":28400,">.":28401,"chid":28402,"ĠProcessing":28403,"/+":28404,"ranking":28405,"Easy":28406,"ĠConstruct":28407,"Ġtet":28408,"insured":28409,"HUD":28410,"Ġquoting":28411,"Ġcommunicated":28412,"inx":28413,"Ġinmate":28414,"Ġerected":28415,"ĠAbsolutely":28416,"ĠSurely":28417,"Ġunim":28418,"ĠThrone":28419,"heid":28420,"Ġclaws":28421,"Ġsuperstar":28422,"ĠLenn":28423,"ĠWhis":28424,"Uk":28425,"abol":28426,"Ġsket":28427,"ĠNiet":28428,"Ġperks":28429,"Ġaffinity":28430,"Ġopenings":28431,"phasis":28432,"Ġdiscriminate":28433,"Tip":28434,"vc":28435,"Ġgrinding":28436,"ĠJenny":28437,"Ġasthma":28438,"holes":28439,"ĠHomer":28440,"Ġregisters":28441,"ĠGlad":28442,"Ġcreations":28443,"Ġlithium":28444,"Ġapplause":28445,"until":28446,"Justice":28447,"ĠTurks":28448,"Ġscandals":28449,"Ġbake":28450,"tank":28451,"Mech":28452,"ĠMeans":28453,"ĠMaid":28454,"Republicans":28455,"isal":28456,"windows":28457,"ĠSantos":28458,"Ġvegetation":28459,"338":28460,"tri":28461,"Ġflux":28462,"insert":28463,"Ġclarified":28464,"Ġmortg":28465,"ĠChim":28466,"ĠTort":28467,"Ġdisclaim":28468,"metal":28469,"ĠAside":28470,"Ġinduction":28471,"Ġinfl":28472,"Ġatheists":28473,"amph":28474,"Ġether":28475,"ĠVital":28476,"ĠBuilt":28477,"Mind":28478,"Ġweaponry":28479,"SET":28480,"Ġ186":28481,"admin":28482,"gam":28483,"contract":28484,"afa":28485,"Ġderivatives":28486,"Ġsnacks":28487,"Ġchurn":28488,"Econom":28489,"Ġcapped":28490,"ĠUnderstanding":28491,"ĠHers":28492,"ĠIz":28493,"Ġduct":28494,"IENT":28495,"aughty":28496,"ĠâľĶ":28497,"ĠNP":28498,"Ġsailing":28499,"Initialized":28500,"Ġted":28501,"Ġreactors":28502,"ĠLomb":28503,"Ġchoke":28504,"ĠWorm":28505,"Ġadmiration":28506,"Ġswung":28507,"ensibly":28508,"Ġrash":28509,"ĠGoals":28510,"ĠImportant":28511,"Shot":28512,"ĠRas":28513,"Ġtrainers":28514,"ĠBun":28515,"Working":28516,"Ġharmed":28517,"ĠPandora":28518,"ĠLTE":28519,"Ġmushroom":28520,"ĠCHAR":28521,"ĠFee":28522,"ĠMoy":28523,"Born":28524,"oliberal":28525,"ĠMartial":28526,"Ġgentlemen":28527,"Ġlingering":28528,"Official":28529,"Ġgraffiti":28530,"ĠNames":28531,"Der":28532,"Ġquint":28533,"istrate":28534,"azeera":28535,"ĠNOTICE":28536,"ĠFlorence":28537,"Ġpayable":28538,"Ġdepicts":28539,"ĠSpecies":28540,"Heart":28541,"âĶĢâĶĢâĶĢâĶĢâĶĢâĶĢâĶĢâĶĢ":28542,"Ġenclosed":28543,"Increases":28544,"Daily":28545,"ĠLis":28546,"Ġenactment":28547,"ĠBacon":28548,"ĠSteele":28549,"demand":28550,"Ġ183":28551,"Ġmouths":28552,"Ġstranded":28553,"Ġenhancement":28554,"011":28555,"ĠWhats":28556,"Ġhealed":28557,"eny":28558,"ĠRab":28559,"Ġ340":28560,"ĠLabyrinth":28561,"roach":28562,"ĠYosh":28563,"ĠClippers":28564,"Ġconcerts":28565,"Internet":28566,"355":28567,"Ġstickers":28568,"Ġtermed":28569,"ĠAxe":28570,"Ġgrandparents":28571,"France":28572,"ĠClim":28573,"ĠUh":28574,"ulic":28575,"Ġthrill":28576,"centric":28577,"ĠOverview":28578,"ĠConduct":28579,"Ġsubstantive":28580,"Ġ182":28581,"mur":28582,"Ġstray":28583,"ĠCoff":28584,"Ġrepetitive":28585,"ĠForgotten":28586,"Ġqualification":28587,"ewitness":28588,"ĠZimbabwe":28589,"Ġsimulated":28590,"ĠJD":28591,"253":28592,"ĠWare":28593,"Ġunsc":28594,"Times":28595,"Ġsummons":28596,"Ġdisconnected":28597,"Ġ184":28598,"cius":28599,"ĠGujar":28600,"odka":28601,"Ġerase":28602,"ĠTobacco":28603,"elected":28604,"Ġuncont":28605,"ĠShepard":28606,"ĠLamp":28607,"Ġalerted":28608,"Ġoperative":28609,"arna":28610,"uint":28611,"Ġnegligence":28612,"acements":28613,"Ġsupra":28614,"Ġprevail":28615,"ĠShark":28616,"Ġbelts":28617,"ãģ«":28618,"Ġtighter":28619,"Engineers":28620,"Ġinactive":28621,"Ġexponent":28622,"ĠWillie":28623,"aples":28624,"Ġheir":28625,"ĠHits":28626,"iann":28627,"ĠSays":28628,"Ġcurrents":28629,"ĠBengal":28630,"Ġarist":28631,"Buffer":28632,"Ġbreeze":28633,"ĠWesley":28634,"Cola":28635,"Ġpronoun":28636,"Ġdeed":28637,"ĠKling":28638,"Ġoft":28639,"Ġinflict":28640,"Ġpunishing":28641,"Ġnm":28642,"iku":28643,"ODUCT":28644,"014":28645,"Ġsubsidy":28646,"ĠDEA":28647,"ĠHerbert":28648,"ĠJal":28649,"Bank":28650,"Ġdeferred":28651,"Ġshipment":28652,"Bott":28653,"Ġalle":28654,"bearing":28655,"HTML":28656,"Offline":28657,"Ġ213":28658,"Ġscrolling":28659,"Ġscanned":28660,"ĠLibyan":28661,"ĠTOP":28662,"chrom":28663,"dt":28664,"column":28665,"PsyNetMessage":28666,"Zero":28667,"Ġtorso":28668,"050":28669,"âķIJ":28670,"Ġimperson":28671,"ĠSchwartz":28672,"udic":28673,"Ġpissed":28674,"ĠSapp":28675,"257":28676,"ĠISPs":28677,"ogl":28678,"Ġsupervised":28679,"Ġadolescent":28680,"Ġattained":28681,"ĠDelivery":28682,"ĠBunny":28683,"Ġ1937":28684,"Ġminiature":28685,"Ġos":28686,"Ġ370":28687,"608":28688,"ĠMourinho":28689,"Ġinnate":28690,"Ġtempo":28691,"ĠNM":28692,"ĠFallen":28693,"009":28694,"Ġprovocative":28695,"Streamer":28696,"ĠBenedict":28697,"ĠBolshe":28698,"Ġturtle":28699,"ĠPCB":28700,"ĠEqual":28701,"Director":28702,"ĠRend":28703,"Ġfluids":28704,"Authorities":28705,"Ġcousins":28706,"requency":28707,"ĠNeighbor":28708,"sets":28709,"shared":28710,"Charles":28711,"password":28712,"Ġgears":28713,"Ġ211":28714,"ĠHardware":28715,"rika":28716,"Ġupstream":28717,"Hom":28718,"Ġdisproportionately":28719,"ivities":28720,"Ġundefined":28721,"Ġelectrons":28722,"Ġcommemor":28723,"Eventually":28724,"Ġ><":28725,"Ġirresponsible":28726,"218":28727,"ĠReleased":28728,"ĠOVER":28729,"ĠIGN":28730,"ĠBread":28731,"stellar":28732,"ĠSage":28733,"tted":28734,"damage":28735,"edition":28736,"ĠPrec":28737,"Ġlime":28738,"Ġconfinement":28739,"Ġcalorie":28740,"weapon":28741,"Ġdiffering":28742,"ĠSina":28743,"mys":28744,"amd":28745,"Ġintricate":28746,"kk":28747,"ĠPAT":28748,"ão":28749,"stones":28750,"links":28751,"Ġranch":28752,"Semitic":28753,"Ġdifferentiate":28754,"ĠSinger":28755,"occupied":28756,"Ġfortress":28757,"cmd":28758,"Ġinterception":28759,"ĠAnkara":28760,"Ġrept":28761,"ĠSolitaire":28762,"Ġremake":28763,"pred":28764,"Ġdared":28765,"autions":28766,"ĠBACK":28767,"Running":28768,"Ġdebugging":28769,"Ġgraphs":28770,"399":28771,"ĠNigel":28772,"Ġbun":28773,"Ġpillow":28774,"Ġprogressed":28775,"fashioned":28776,"Ġobedience":28777,"ERN":28778,"Ġrehears":28779,"Cell":28780,"tl":28781,"Sher":28782,"Ġherald":28783,"ĠPayment":28784,"ĠCory":28785,"ĠDept":28786,"Ġrepent":28787,"ĠWeak":28788,"uckland":28789,"Ġpleasing":28790,"Ġshortages":28791,"Ġjurors":28792,"ĠKab":28793,"qqa":28794,"Anti":28795,"Ġwow":28796,"ĠRCMP":28797,"Ġtsun":28798,"ĠSic":28799,"Ġcomprises":28800,"Ġspies":28801,"Ġprecinct":28802,"nu":28803,"Ġurges":28804,"Ġtimed":28805,"Ġstripes":28806,"ĠBoots":28807,"Ġyen":28808,"Advanced":28809,"Ġdiscrete":28810,"ĠArchangel":28811,"employment":28812,"Diff":28813,"Ġmonuments":28814,"Ġ209":28815,"worker":28816,"Ġ196":28817,"ĠIg":28818,"utterstock":28819,"TPS":28820,"Jac":28821,"Ġhomelessness":28822,"Ġcommentator":28823,"Ġracially":28824,"fing":28825,"seed":28826,"Ele":28827,"ellation":28828,"Ġethanol":28829,"Ġparish":28830,"ĠDong":28831,"ĠAwakening":28832,"Ġdeviation":28833,"ĠBearing":28834,"ĠTsuk":28835,"Ġrecess":28836,"Ġlymph":28837,"ĠCannabis":28838,"åľ":28839,"ĠNEWS":28840,"Ġdra":28841,"ĠStefan":28842,"ĠWrong":28843,"ĠSAM":28844,"Ġloosely":28845,"Ġinterpreter":28846,"ĠPlain":28847,"Government":28848,"Ġbigotry":28849,"Ġgrenades":28850,"avez":28851,"pictured":28852,"Ġmandated":28853,"ĠMonk":28854,"ĠPedro":28855,"Ġlava":28856,"274":28857,"Ġcynical":28858,"ĠScrolls":28859,"locks":28860,"Mp":28861,"Ġcongregation":28862,"ornings":28863,"phil":28864,"ĠIbid":28865,"Ġferv":28866,"Ġdisappearing":28867,"Ġarrogant":28868,"syn":28869,"ĠMaver":28870,"ĠSuit":28871,"241":28872,"Ġabbre":28873,"ackers":28874,"Pa":28875,"ĠYel":28876,"Whenever":28877,"Ġ235":28878,"ĠVine":28879,"ĠAnat":28880,"Ġextinct":28881,"LET":28882,"Ġexecutable":28883,"VERS":28884,"oxide":28885,"DNA":28886,"ĠPrel":28887,"Ġresentment":28888,"Ġcomprise":28889,"ĠAviv":28890,"Ġinterceptions":28891,"Ġprolific":28892,"INA":28893,"ĠErin":28894,"thought":28895,"219":28896,"ĠPsychiatry":28897,"unky":28898,"chemist":28899,"Ho":28900,"ĠMcCoy":28901,"Ġbricks":28902,"Los":28903,"rily":28904,"ĠUSSR":28905,"Ġrud":28906,"Ġlaud":28907,"ĠWise":28908,"ĠEmerald":28909,"Ġrevived":28910,"Ġdamned":28911,"ĠRepair":28912,"idem":28913,"ctica":28914,"Ġpatriarch":28915,"ĠNurs":28916,"meg":28917,"Ġcheapest":28918,"reements":28919,"empty":28920,"ĠCelebr":28921,"Ġdeprivation":28922,"chanted":28923,"ĠThumbnails":28924,"Energy":28925,"ĠEthan":28926,"ĠQing":28927,"Ġopposes":28928,"WIND":28929,"vik":28930,"ĠMau":28931,"ĠSUB":28932,"667":28933,"GRE":28934,"ĠVolunte":28935,"nton":28936,"Cook":28937,"åIJ":28938,"esque":28939,"Ġplummet":28940,"Ġsuing":28941,"Ġpronounce":28942,"Ġresisting":28943,"ĠFishing":28944,"ĠTrials":28945,"Ġyell":28946,"Ġ310":28947,"Ġinduct":28948,"Ġpersonalized":28949,"often":28950,"Reb":28951,"EMBER":28952,"Ġviewpoint":28953,"Ġexistential":28954,"())":28955,"remove":28956,"MENTS":28957,"lasses":28958,"Ġevapor":28959,"Ġaisle":28960,"meta":28961,"Ġreflective":28962,"Ġentitlement":28963,"Ġdevised":28964,"music":28965,"ascade":28966,"Ġwinding":28967,"offset":28968,"Ġaccessibility":28969,"kered":28970,"Better":28971,"ĠJohnston":28972,"thinking":28973,"Snow":28974,"ĠCroatia":28975,"ĠAtomic":28976,"271":28977,"348":28978,"Ġtextbook":28979,"ĠSixth":28980,"ĠاÙĦ":28981,"Ġslider":28982,"ĠBurger":28983,"bol":28984,"Sync":28985,"Ġgrandchildren":28986,"Ġcerv":28987,"+)":28988,"Ġeternity":28989,"Ġtweeting":28990,"Ġspeculative":28991,"Ġpivotal":28992,"ĠWP":28993,"ĠTER":28994,"ynamic":28995,"Ġupl":28996,"ĠCats":28997,"perhaps":28998,"Ġclassmates":28999,"Ġblatant":29000,"'-":29001,"Ġlakh":29002,"antine":29003,"ĠBorg":29004,"iom":29005,"/(":29006,"ĠAthletic":29007,"Ġsar":29008,"OTA":29009,"ĠHoffman":29010,"Nevertheless":29011,"Ġadorable":29012,"Ġspawned":29013,"Associated":29014,"ĠDomestic":29015,"Ġimplant":29016,"ĠLuxem":29017,"ĠKens":29018,"Ġpumps":29019,"ĠSAT":29020,"Attributes":29021,"509":29022,"avour":29023,"Ġcentralized":29024,"ĠTN":29025,"Ġfreshly":29026,"ĠAchieve":29027,"Ġoutsiders":29028,"herty":29029,"ĠRee":29030,"ĠTowers":29031,"ĠDart":29032,"akable":29033,"Ġmp":29034,"ĠHeavenly":29035,"Ġripe":29036,"ĠCaroline":29037,"ryan":29038,"Ġclassics":29039,"Ġretiring":29040,"Ġ228":29041,"Ġah":29042,"Ġdealings":29043,"Ġpunching":29044,"ĠChapman":29045,"Options":29046,"maxwell":29047,"volume":29048,"Ġstal":29049,"Ġexported":29050,"ĠQuite":29051,"Ġnumerical":29052,"Burn":29053,"Fact":29054,"ĠKeystone":29055,"Ġtrending":29056,"Ġaltering":29057,"ĠAfricans":29058,"478":29059,"ĠMN":29060,"ĠKnock":29061,"Ġtemptation":29062,"Ġprestige":29063,"Overview":29064,"ĠTraditional":29065,"ĠBahrain":29066,"Private":29067,"ĠHOU":29068,"Ġbarr":29069,"ĠTat":29070,"Cube":29071,"USD":29072,"ĠGrande":29073,"ĠGat":29074,"ĠFlo":29075,"Ġresides":29076,"Ġindec":29077,"volent":29078,"Ġperpetual":29079,"ubes":29080,"Ġworldview":29081,"ĠQuantum":29082,"Ġfiltered":29083,"Ġensu":29084,"orgetown":29085,"ERSON":29086,"ĠMild":29087,"379":29088,"OTT":29089,"Ã¥":29090,"Ġvitamins":29091,"Ġribbon":29092,"Ġsincerely":29093,"ĠHin":29094,"Ġeighteen":29095,"Ġcontradictory":29096,"Ġglaring":29097,"Ġexpectancy":29098,"Ġconspir":29099,"Ġmonstrous":29100,"Ġ380":29101,"reci":29102,"Ġhandic":29103,"Ġpumped":29104,"Ġindicative":29105,"Ġrapp":29106,"Ġavail":29107,"ĠLEGO":29108,"ĠMarijuana":29109,"1985":29110,"erton":29111,"Ġtwentieth":29112,"################################":29113,"ĠSwamp":29114,"Ġvaluation":29115,"Ġaffiliates":29116,"adjusted":29117,"ĠFacility":29118,"262":29119,"Ġenzymes":29120,"itudinal":29121,"Ġimprint":29122,"Site":29123,"Ġinstaller":29124,"ĠTRA":29125,"mology":29126,"linear":29127,"ĠCollective":29128,"igating":29129,"ĠToken":29130,"Ġspeculated":29131,"KN":29132,"ĠCly":29133,"ority":29134,"Ġdefer":29135,"Ġinspectors":29136,"approved":29137,"RM":29138,"ĠSuns":29139,"Ġinforming":29140,"ĠSyracuse":29141,"ibli":29142,"765":29143,"Ġglove":29144,"Ġauthorize":29145,"âĢ¦âĢ¦âĢ¦âĢ¦âĢ¦âĢ¦âĢ¦âĢ¦":29146,"ĠCruise":29147,"Ġcontracting":29148,"shell":29149,"IFE":29150,"ĠJewel":29151,"pract":29152,"ĠPhotoshop":29153,"ĠKnowing":29154,"harm":29155,"Ġattractions":29156,"adan":29157,"etus":29158,"018":29159,"wagen":29160,"Alt":29161,"Ġmultiply":29162,"Ġequilibrium":29163,":{":29164,"ĠFighters":29165,"ĠEdgar":29166,"Ġfourteen":29167,"Govern":29168,"Ġmisuse":29169,"Ġabusing":29170,"Ġancestry":29171,"ramer":29172,"644":29173,"Ġworms":29174,"Ġthicker":29175,"ĠCombine":29176,"Ġpeasants":29177,"Ġvind":29178,"Ġconquest":29179,"Ġmocked":29180,"Ġcinnamon":29181,"ĠCald":29182,"ĠGallup":29183,"Ġavoidance":29184,"Ġincarnation":29185,"ĠStrat":29186,"Ġtasted":29187,"enta":29188,"ĠNeal":29189,"pared":29190,"Ġterminology":29191,"jection":29192,"Scientists":29193,"ĠINS":29194,"ĠDee":29195,"Ġdirectories":29196,"Road":29197,"ĠShap":29198,"bright":29199,"ĠDirectors":29200,"ĠColumn":29201,"Ġbob":29202,"Ġpreferably":29203,"Ġglitch":29204,"furt":29205,"Ġeg":29206,"idis":29207,"CBC":29208,"Ġsurrendered":29209,"Ġtestament":29210,"336":29211,"uggest":29212,"ĠNil":29213,"another":29214,"Ġpathetic":29215,"ĠDonna":29216,"Ġ218":29217,"ĠAvery":29218,"Ġwhiskey":29219,"Ġfixture":29220,"ĠConquest":29221,"Ġbets":29222,"Occ":29223,"ĠLeicester":29224,"].\"":29225,"Ġ));":29226,"Ġflashes":29227,"456":29228,"Ġmasked":29229,"gebra":29230,"Ġcomputed":29231,"chel":29232,"auder":29233,"Ġdefeats":29234,"ĠLiberation":29235,"ĠOsama":29236,"ĠVive":29237,"Changes":29238,"Channel":29239,"Ġtariffs":29240,"Ġmage":29241,"ĠSax":29242,"Ġinadvertently":29243,"ĠCRE":29244,"ĠReaper":29245,"inky":29246,"grading":29247,"Ġstereotyp":29248,"Ġcurl":29249,"ĠFANT":29250,"Ġframeworks":29251,"Mom":29252,"ĠAnch":29253,"Ġflavour":29254,"carbon":29255,"Ġpermitting":29256,"letcher":29257,"ĠMozilla":29258,"ĠParking":29259,"ĠChamp":29260,"Scroll":29261,"Ġmurderer":29262,"Ġrested":29263,"Ġowes":29264,"ĠPoss":29265,"ADD":29266,"IFF":29267,"resolution":29268,"ĠMining":29269,"Ġcomparative":29270,"Dim":29271,"Ġneighbouring":29272,"ĠAST":29273,"ĠToxic":29274,"Ġbiases":29275,"Ġgunfire":29276,"urous":29277,"ĠMoment":29278,"1983":29279,"Ġpervasive":29280,"ttp":29281,"ĠNormally":29282,"rir":29283,"Sarah":29284,"ĠAlbany":29285,"Ġunsett":29286,"ĠSMS":29287,"ipers":29288,"layer":29289,"ĠWhites":29290,"uple":29291,"Ġturbo":29292,"ĠLeeds":29293,"Ġthats":29294,"ĠMiner":29295,"MER":29296,"ĠReign":29297,"Ġperme":29298,"ĠBlitz":29299,"Ġ1934":29300,"Ġintimidating":29301,"tube":29302,"Ġeccentric":29303,"abolic":29304,"boxes":29305,"ĠAssociates":29306,"votes":29307,"Ġsimulate":29308,"umbo":29309,"astery":29310,"Ġshipments":29311,"FFFF":29312,"anth":29313,"Ġseasoned":29314,"Ġexperimentation":29315,"âĸł":29316,"laws":29317,"Meet":29318,"iddles":29319,"antics":29320,"Rating":29321,"ISIS":29322,"hift":29323,"Ġfronts":29324,"buf":29325,"017":29326,"Ġunatt":29327,"ĠDil":29328,"leases":29329,"ĠGardens":29330,"777":29331,"touch":29332,"vell":29333,"458":29334,"Ġ=====":29335,"saving":29336,"Ġerosion":29337,"ĠQuin":29338,"Ġearns":29339,"Ġaccomplishment":29340,"ĠWei":29341,"Ġ<[":29342,"_____":29343,"Ġirrig":29344,"ĠTeddy":29345,"Ġconquered":29346,"ĠArmored":29347,"Ġasserts":29348,"Ġmanipulating":29349,"ré":29350,"Ġtranscripts":29351,"Gallery":29352,"Ġplotting":29353,"Neil":29354,"Ġbetrayal":29355,"loader":29356,"ĠSul":29357,"Ġdisplacement":29358,"Ġroyalty":29359,"ĠWI":29360,"heit":29361,"ĠDevices":29362,"allel":29363,"Ġmunicipalities":29364,"Ġcanal":29365,"Stars":29366,"ĠUAE":29367,"Ġ\"âĢ¦":29368,"ĠCU":29369,"above":29370,"Ġresonance":29371,"ĠguiActiveUn":29372,"added":29373,"ĠBraves":29374,"ĠIbn":29375,"Ġhereby":29376,"ĠBRE":29377,"Ġshareholder":29378,"ĠHir":29379,"ĠJi":29380,"Ġstrangely":29381,"Ġadmired":29382,"Ġplight":29383,"Ġbachelor":29384,"ĠPole":29385,"ciplinary":29386,"Tony":29387,"ĠArmenian":29388,"Ġunman":29389,"ĠZionist":29390,"Stage":29391,"iscover":29392,"Ġautomotive":29393,"Ġsidelines":29394,"Ġslick":29395,"ĠRenaissance":29396,"ĠFUN":29397,"Images":29398,"ĠHaj":29399,"Ġping":29400,"Ġshortcut":29401,"ĠBlvd":29402,"ĠLooks":29403,"Ġbursts":29404,"Ġclamp":29405,"Ġmish":29406,"Ġsorting":29407,"Ġpatriot":29408,"Ġcorrectness":29409,"ĠScandinav":29410,"ĠCavaliers":29411,"python":29412,"azar":29413,"Ġ375":29414,"ĠJaune":29415,"409":29416,"Ġdetrimental":29417,"Ġstabbing":29418,"Ġpoisoned":29419,"Ġfountain":29420,"ocent":29421,"orst":29422,"ĠMari":29423,"Ġrains":29424,"ĠOvers":29425,"ĠInstitution":29426,"udget":29427,"AMY":29428,"tale":29429,"ĠKR":29430,"ĠPrices":29431,"Ġheadaches":29432,"Ġlandsl":29433,"ĠAura":29434,"Bonus":29435,"ĠZhao":29436,"ĠHip":29437,"Ġhops":29438,"ĠKurdistan":29439,"Ġexploiting":29440,"ryn":29441,"Ġhypocrisy":29442,"opening":29443,"Ġgunshot":29444,"Ġwed":29445,"interstitial":29446,"Interstitial":29447,"Ġamen":29448,"Breaking":29449,"Ġmarketed":29450,"Wire":29451,"ĠCrowd":29452,"Continue":29453,"ĠKnown":29454,"ĠEffective":29455,"orean":29456,"izons":29457,"Joseph":29458,"Ġescalation":29459,"username":29460,"Ġcurtain":29461,"ATES":29462,"ĠPAR":29463,"ĠMiy":29464,"Ġcounterfe":29465,"lene":29466,"Ġcontenders":29467,"daily":29468,"ĠAsc":29469,"ĠPhillip":29470,"mostly":29471,"Ġfilename":29472,"hene":29473,"Ġresembling":29474,"Ġstaging":29475,"ĠChloe":29476,"Ġwiring":29477,"Hon":29478,"ĠRenew":29479,"ottage":29480,"ĠHybrid":29481,"much":29482,"Ġstrokes":29483,"Ġpolicymakers":29484,"APTER":29485,"ĠArkham":29486,"plot":29487,"Ġassistants":29488,"Ġdeport":29489,"ĠSega":29490,"Ġinfluenza":29491,"ĠCursed":29492,"ĠKobe":29493,"Ġskinny":29494,"Provider":29495,"ĠRip":29496,"Ġincremental":29497,"products":29498,"BF":29499,"Ġdome":29500,"ĠCredits":29501,"Ġlosers":29502,"ints":29503,"ĠBetty":29504,"ĠTalent":29505,"ĠDAM":29506,"Lv":29507,"Ess":29508,"Ġdens":29509,"temp":29510,"Judge":29511,"odic":29512,"Ġ'(":29513,"URES":29514,"etsk":29515,"VO":29516,"Ġretrieved":29517,"Ġarchitects":29518,"Ùĩ":29519,"Ġethic":29520,"ĠSecondary":29521,"stocks":29522,"adia":29523,"Ġ325":29524,"ĠOpinion":29525,"Ġsimultaneous":29526,"Ġdizz":29527,"ulp":29528,"Ġsmuggling":29529,"ippery":29530,"Random":29531,"facing":29532,"ĠDas":29533,"Ġstockp":29534,"Ġdisclosures":29535,"pointer":29536,"Ġcoral":29537,"ĠSelection":29538,"ĠPike":29539,"ivalent":29540,"Ġruthless":29541,"ĠRim":29542,"Ġensuing":29543,"ĠExperiment":29544,"Ġcongressman":29545,"Ġbeliever":29546,"Ġunspecified":29547,"ĠMord":29548,"Ġknowledgeable":29549,"ĠVERY":29550,"TX":29551,"Ġstraps":29552,"Ġturf":29553,"apeshifter":29554,"Ġmarital":29555,"Ġflock":29556,"ãģĨ":29557,"263":29558,"AMES":29559,"ĠOpposition":29560,"Ġtreasures":29561,"ĠGOD":29562,"Ġmodeled":29563,"ĠWORLD":29564,"Ġ([":29565,"ĠUsage":29566,"HF":29567,"Ġ$(":29568,"ussed":29569,"Ġpioneer":29570,"Eight":29571,"parse":29572,"bread":29573,"ritz":29574,"ĠMiranda":29575,"ĠKant":29576,"++)":29577,"oren":29578,"Ġprovoked":29579,"Ġbreeds":29580,"ĠIncludes":29581,"ĠPastebin":29582,"ĠFlip":29583,"Java":29584,"Ġbrink":29585,"Ġrumored":29586,"Ġunseen":29587,"Ġgarnered":29588,"ĠDefin":29589,"alted":29590,"Ġtattoos":29591,"Ġhesitation":29592,"isitions":29593,"ĠWeaver":29594,"ĠReporting":29595,"Ġtherapies":29596,"Ġconsultants":29597,"Ġresidual":29598,"ĠMali":29599,"ĠRoma":29600,"iago":29601,"ĠResidents":29602,"ubi":29603,"Ġremedies":29604,"Ġadaptive":29605,"ĠAlive":29606,"ĠBarcl":29607,"Ġwallets":29608,"crypt":29609,"etermination":29610,"ĠPelosi":29611,"Ġslipping":29612,"otonin":29613,"Ġalliances":29614,"patrick":29615,"iris":29616,"Ġorth":29617,"ĠPerkins":29618,"ĠDeV":29619,"ĠGets":29620,"Ġdrying":29621,"gee":29622,"forest":29623,"ĠForget":29624,"orem":29625,"339":29626,"Ġvaguely":29627,"ĠDion":29628,"ĠPorn":29629,"ĠHOW":29630,"Ġpneum":29631,"Ġrubble":29632,"ĠTaste":29633,"encia":29634,"ĠGel":29635,"Ġdst":29636,"Ġ245":29637,"ĠMorocco":29638,"inflamm":29639,"ĠTwins":29640,"Ġbots":29641,"daughter":29642,"ĠBalk":29643,"Ġbrethren":29644,"Ġlogos":29645,"Ġgobl":29646,"fps":29647,"Ġsubdivision":29648,"Ġpawn":29649,"Ġsqueezed":29650,"Ġmorale":29651,"ĠDW":29652,"'\"":29653,"Ġknot":29654,"ooky":29655,"Ġdivisive":29656,"Ġboosted":29657,"chy":29658,"ãĥIJ":29659,"ifact":29660,"Ġnewcomers":29661,"ĠWrestling":29662,"Ġscouts":29663,"wolves":29664,"Rat":29665,"Ġnineteenth":29666,"ĠOsborne":29667,"Stats":29668,"Ġempowered":29669,"Ġpsychopath":29670,"ĠOEM":29671,"uggage":29672,"ĠPK":29673,"ĠMohammad":29674,"Pak":29675,"Ġanarchists":29676,"ĠExtract":29677,"esthes":29678,"ĠStockholm":29679,"loo":29680,"ĠGraph":29681,"Ġdeploying":29682,"ĠStranger":29683,"ĠMold":29684,"Ġstaffer":29685,"Ġdiscounted":29686,"uckle":29687,"please":29688,"ĠLanding":29689,"ÃŃa":29690,"Ġ193":29691,"Ġante":29692,"Ġrepetition":29693,"Ġ+/-":29694,"Ġparody":29695,"Ġlively":29696,"AAA":29697,"ĠHorus":29698,"Ġpits":29699,"inders":29700,"LOC":29701,"ĠVenice":29702,"406":29703,"ĠDiscover":29704,"âĨ":29705,"ellectual":29706,"Ġpens":29707,"Ġeyel":29708,"iguous":29709,"Impl":29710,"Ġjoking":29711,"Ġinval":29712,"ĠBelfast":29713,"Ġcreditors":29714,"ĠSkywalker":29715,"ovsky":29716,"Ġceasefire":29717,"Ġseals":29718,"isoft":29719,")).":29720,"ĠFelix":29721,"ITS":29722,"Ġtresp":29723,"ĠBlockchain":29724,"eware":29725,"ĠSchwar":29726,"enne":29727,"mounted":29728,"ĠBeacon":29729,"lesh":29730,"Ġimmensely":29731,"Ġcheering":29732,"Employ":29733,"scene":29734,"ishly":29735,"atchewan":29736,"ĠNicolas":29737,"Ġdrained":29738,"ĠExit":29739,"ĠAzerb":29740,"jun":29741,"Ġfloated":29742,"uania":29743,"Deep":29744,"Ġsuperv":29745,"Ġmystical":29746,"ĠDollar":29747,"ĠApostle":29748,"ĠREL":29749,"ĠProvided":29750,"ĠBucks":29751,"ãĥ´":29752,"cutting":29753,"Ġenhancements":29754,"ĠPenguins":29755,"ĠIsaiah":29756,"Ġjerk":29757,"ĠWyn":29758,"Ġstalled":29759,"Ġcryptocurrencies":29760,"ĠRoland":29761,"single":29762,"Ġlumin":29763,"ĠFellow":29764,"ĠCapacity":29765,"ĠKazakh":29766,"WN":29767,"Ġfinanced":29768,"389":29769,"Ġtid":29770,"Ġcollusion":29771,"ĠMyr":29772,"îĢ":29773,"Senator":29774,"Ġpediatric":29775,"Ġneatly":29776,"Ġsandwiches":29777,"ĠArchitecture":29778,"Ġtucked":29779,"Ġbalcony":29780,"Ġearthquakes":29781,"quire":29782,"Future":29783,"Ġhefty":29784,"éĹ":29785,"Ġspecializes":29786,"Ġstresses":29787,"Ġsender":29788,"Ġmisunderstanding":29789,"Ġepile":29790,"Ġprovoke":29791,"ĠColors":29792,"Ġdismay":29793,"uko":29794,"[_":29795,"586":29796,"neutral":29797,"Ġdonating":29798,"ĠRandall":29799,"Multi":29800,"Ġconveniently":29801,"ĠSung":29802,"ĠCoca":29803,"Ġtents":29804,"ĠAcceler":29805,"Ġpartnered":29806,"272":29807,"irming":29808,"ĠBAS":29809,"sometimes":29810,"Ġobjected":29811,"ubric":29812,"posed":29813,"LCS":29814,"grass":29815,"Ġattributable":29816,"VIS":29817,"Israeli":29818,"Ġrepeats":29819,"ĠRM":29820,"vag":29821,"uta":29822,"inous":29823,"Ġinert":29824,"ĠMiguel":29825,"æŃ":29826,"ĠHawaiian":29827,"Board":29828,"Ġartific":29829,"ĠAzerbai":29830,"asio":29831,"ĠRent":29832,"AIN":29833,"Ġappliances":29834,"Ġnationality":29835,"Ġasshole":29836,"ĠNeb":29837,"Ġnotch":29838,"hani":29839,"ĠBride":29840,"Availability":29841,"Ġintercepted":29842,"Ġcontinental":29843,"Ġswelling":29844,"ĠPerspect":29845,"bies":29846,".<":29847,"ithmetic":29848,"ĠLara":29849,"Ġtempting":29850,"addr":29851,"Ġoverseeing":29852,"clad":29853,"ĠDV":29854,"ĠGingrich":29855,"Ġmun":29856,"ĠAppropri":29857,"Ġalterations":29858,"ĠPatreon":29859,"Ġhavoc":29860,"Ġdisciplines":29861,"Ġnotoriously":29862,"akuya":29863,"ieri":29864,"?).":29865,"ĠWent":29866,"Ġsilicon":29867,"Ġtremb":29868,"Container":29869,"Known":29870,"Ġmortar":29871,"este":29872,"icka":29873,"Arthur":29874,"ĠPreviously":29875,"ĠMarty":29876,"Ġsparse":29877,"gins":29878,"Ġinward":29879,"ĠParticipant":29880,"Copy":29881,"ĠMisc":29882,"Ġantibiotic":29883,"ĠRetro":29884,"Ġelusive":29885,"Ġassail":29886,"ĠBattalion":29887,"ĠBought":29888,"Ġdiminish":29889,"ĠEuropa":29890,"session":29891,"ĠDangerous":29892,"iesel":29893,"Ġdisbelief":29894,"Ġblasts":29895,"extreme":29896,"ĠBoyd":29897,"ĠProjects":29898,"ĠGuys":29899,"Ġundergone":29900,"Ġgrill":29901,"ĠDwight":29902,"Ġ197":29903,"USER":29904,"Ġfilesystem":29905,"Ġclocks":29906,"Taylor":29907,"Ġwrapper":29908,"Ġfolding":29909,"ousand":29910,"ĠPhilippine":29911,"ATIONAL":29912,"ĠPerth":29913,"Ġashes":29914,"Ġaccumulate":29915,"ĠGateway":29916,"Shop":29917,"orkshire":29918,"Han":29919,"ĠBarrel":29920,"ĠLeh":29921,"ĠXV":29922,"Ġwhim":29923,"Ġrepo":29924,"ĠCG":29925,"ĠMam":29926,"Ġincorporating":29927,"Ġbailout":29928,"Ġlinguistic":29929,"Ġdisinteg":29930,"CLE":29931,"Ġcinematic":29932,"ĠFiber":29933,"Syn":29934,"ilion":29935,"ĠCompos":29936,"chens":29937,"Ġneoc":29938,"Ġboiled":29939,"FINE":29940,"ono":29941,"uncle":29942,"iken":29943,"ĠBM":29944,"ι":29945,"Ġreceipts":29946,"Ġdisposed":29947,"ĠThirty":29948,"ĠRough":29949,"ĠABS":29950,"Ġnotwithstanding":29951,"ollen":29952,"#$":29953,"Ġunreliable":29954,"Ġbloom":29955,"Ġmediocre":29956,"Ġtram":29957,"ĠTasman":29958,"Ġshakes":29959,"Ġmanifesto":29960,"ĠMW":29961,"Ġsatisfactory":29962,"Ġshores":29963,"Ġcomputation":29964,"Ġassertions":29965,"ormons":29966,"arag":29967,"abit":29968,"Democrats":29969,"ĠLoot":29970,"ĠVolks":29971,"haired":29972,"Ġgravitational":29973,"Sing":29974,"ĠMiz":29975,"Ġthrottle":29976,"Ġtyranny":29977,"ĠViews":29978,"Ġrobber":29979,"ĠMinority":29980,"Ġshrine":29981,"scope":29982,"purpose":29983,"Ġnucleus":29984,"ourcing":29985,"ĠUSDA":29986,"ĠDHS":29987,"wra":29988,"ĠBowie":29989,"Scale":29990,"ĠBEL":29991,"xi":29992,"Iter":29993,"Ġ(),":29994,"wright":29995,"Ġsailors":29996,"oused":29997,"NASA":29998,"ĠProof":29999,"ĠMineral":30000,"token":30001,"ĠFD":30002,"Rew":30003,"Ġell":30004,"630":30005,"Ġchancellor":30006,"ĠGos":30007,"Ġamounted":30008,"ĠRecre":30009,"omez":30010,"ĠOptim":30011,"ĠOlive":30012,"Ġtracker":30013,"owler":30014,"ĠUnique":30015,"Root":30016,"Ġmaritime":30017,"ĠQuran":30018,"ĠAdapt":30019,"Ġecosystems":30020,"ĠRepeat":30021,"ĠSoy":30022,"ĠIMP":30023,"Ġgraduating":30024,"andem":30025,"Pur":30026,"ĠReset":30027,"ĠTrick":30028,"ĠPhilly":30029,"ĠTue":30030,"ĠMalaysian":30031,"Ġclimax":30032,"Ġbury":30033,"Ġconspic":30034,"ĠSouthampton":30035,"ĠFlowers":30036,"Ġescorted":30037,"ĠEducational":30038,"ĠIRC":30039,"Ġbrutally":30040,"eating":30041,"Ġpillar":30042,"ĠSang":30043,"ĠJude":30044,"arling":30045,"ĠAmnesty":30046,"Ġreminding":30047,"ĠAdministrative":30048,"hesda":30049,"Ġflashed":30050,"ĠPBS":30051,"perate":30052,"feature":30053,"Ġswipe":30054,"Ġgraves":30055,"oultry":30056,"261":30057,"breaks":30058,"ĠGuer":30059,"Ġshrimp":30060,"ĠVoting":30061,"quist":30062,"Ġanalytical":30063,"Ġtablespoons":30064,"ĠSOU":30065,"Ġresearched":30066,"Ġdisrupted":30067,"Ġjour":30068,"Ġreplica":30069,"Ġcartoons":30070,"bians":30071,"})":30072,"copy":30073,"Got":30074,"ouched":30075,"PUT":30076,"Ġswarm":30077,"notations":30078,"said":30079,"Ġrebuilt":30080,"Ġcollaborate":30081,"Ġraging":30082,"Ġnar":30083,"Ġdemographics":30084,"ĠDDR":30085,"Ġdistrust":30086,"ossier":30087,"ĠKro":30088,"Ġpumpkin":30089,"Ġregrets":30090,"Ġfatalities":30091,"ĠLens":30092,"ĠOle":30093,"pd":30094,"Ġpuppet":30095,"ĠOutlook":30096,"ĠStam":30097,"Ol":30098,"Fair":30099,"UU":30100,"Ġrewritten":30101,"ı":30102,"Ġfascinated":30103,"Ġvectors":30104,"Ġtribunal":30105,"uay":30106,"ĠMats":30107,"ĠCoins":30108,"[[":30109,"Ġ181":30110,"Ġrenders":30111,"ĠKaepernick":30112,"Ġespionage":30113,"Ġsumm":30114,"Ġditch":30115,"Account":30116,"Ġspreadsheet":30117,"Ġmutant":30118,"past":30119,"407":30120,"Ġdye":30121,"Ġinitiation":30122,"Ġ4000":30123,"Ġpunishable":30124,"Ġthinner":30125,"ĠKhal":30126,"Ġintermedi":30127,"Dun":30128,"ĠGotham":30129,"Ġeagerly":30130,"Ġvaginal":30131,"powers":30132,"VW":30133,"ĠWATCHED":30134,"Ġpredator":30135,"amsung":30136,"Ġdisparity":30137,"Ġ[*":30138,"Ġamph":30139,"Ġoutskirts":30140,"ĠSpirits":30141,"Ġskeletal":30142,"л":30143,"ĠRear":30144,"Ġissuance":30145,"ĠLogic":30146,"released":30147,"ZZ":30148,"ĠBound":30149,"Entry":30150,"Ġexits":30151,"isol":30152,"ĠFounder":30153,"Ġwre":30154,"ĠGreenland":30155,"ĠMMO":30156,"taker":30157,"INC":30158,"ãģ¾":30159,"Ġhourly":30160,"henko":30161,"Ġfantasies":30162,"Ġdisob":30163,"Ġdemolition":30164,"ãĥĭ":30165,"Ġenlisted":30166,"ratulations":30167,"Ġmisguided":30168,"Ġensured":30169,"Ġdiscouraged":30170,"mort":30171,"Ġflank":30172,"Ġcess":30173,"Ġreacts":30174,"ĠSere":30175,"sensitive":30176,"ĠSerpent":30177,"assad":30178,"Ġ247":30179,"Ġcalmly":30180,"busters":30181,"Ġbleed":30182,"ĠStro":30183,"Ġamusement":30184,"ĠAntarctica":30185,"Ġscept":30186,"ĠGaw":30187,"aq":30188,"asonic":30189,"Ġsprawling":30190,"native":30191,"aturated":30192,"ĠBattlefield":30193,"IVERS":30194,"EB":30195,"ĠGems":30196,"ĠNorthwestern":30197,"ĠFilms":30198,"ĠAutomatic":30199,"Ġapprehend":30200,"ãģ¨":30201,"ĠguiName":30202,"Ġbackend":30203,"Ġevidenced":30204,"geant":30205,"012":30206,"ĠSiege":30207,"ĠexternalTo":30208,"ĠunfocusedRange":30209,"ĠguiActiveUnfocused":30210,"ĠguiIcon":30211,"ĠexternalToEVA":30212,"ĠexternalToEVAOnly":30213,"Fri":30214,"chard":30215,"enaries":30216,"Ġchiefs":30217,"Ġcf":30218,"ĠHUD":30219,"Ġcorrobor":30220,"ĠdB":30221,"ĠTaken":30222,"ĠPatricia":30223,"rail":30224,"ĠCharm":30225,"ĠLibertarian":30226,"rieve":30227,"Personal":30228,"ĠOUR":30229,"geries":30230,"Ġdumping":30231,"Ġneurological":30232,"itimate":30233,"ĠClintons":30234,"rafted":30235,"ĠMolly":30236,"Ġterminals":30237,"register":30238,"Ġflare":30239,"Ġencoded":30240,"Ġautopsy":30241,"pel":30242,"machine":30243,"Ġexemptions":30244,"ĠRoyals":30245,"distance":30246,"Ġdrafts":30247,"Ġlame":30248,"ĠCunning":30249,"Ġspouses":30250,"ĠMarkets":30251,"ĠCarrier":30252,"Ġimplying":30253,"ĠYak":30254,"sid":30255,"Ġloser":30256,"Ġvigilant":30257,"Ġimpeachment":30258,"Ġaugmented":30259,"ĠEmployees":30260,"Ġunintended":30261,"ternally":30262,"ĠWatt":30263,"Ġrecognizable":30264,"essim":30265,"æĿ":30266,"Ġcoated":30267,"rha":30268,"Ġlieutenant":30269,"ĠLegislation":30270,"published":30271,"444":30272,"013":30273,"Ġideally":30274,"ĠPassword":30275,"Ġsimplify":30276,"ĠMeta":30277,"ĠMRI":30278,"Ġpleading":30279,"organized":30280,"handler":30281,"Ġunravel":30282,"correct":30283,"Ġicy":30284,"Ġparanoid":30285,"Ġpasser":30286,"Ġinspections":30287,"ofer":30288,"ĠHealthcare":30289,"283":30290,"ĠBrut":30291,"iola":30292,"forge":30293,"ĠMedieval":30294,"MSN":30295,"ievers":30296,"ĠProgramming":30297,"åī":30298,"Ġ223":30299,"mu":30300,"ĠCLE":30301,"uga":30302,"Ġshoppers":30303,"Ġinformative":30304,"ĠPlans":30305,"Ġsupplementation":30306,"ĠTests":30307,"tyard":30308,"ocytes":30309,"ĠVega":30310,"ĠGujarat":30311,"ermanent":30312,"Except":30313,"ĠLOT":30314,"alla":30315,"ĠCumm":30316,"ĠOsw":30317,"Ġvenom":30318,"ĠDebt":30319,"ĠDOWN":30320,"Ġreunion":30321,"Ġmuc":30322,"ĠRelief":30323,"Ġgeop":30324,"ĠðŁĺ":30325,"alogue":30326,"Anth":30327,"echo":30328,"Ġcorros":30329,"Ġreplication":30330,"ĠBlazing":30331,"ĠDaughter":30332,"Ġinflic":30333,"ĠLindsey":30334,"ÙĪ":30335,"284":30336,"Exit":30337,"Ġgloom":30338,"TAIN":30339,"Ġundermining":30340,"Ġadvising":30341,"hidden":30342,"Ġoverflow":30343,"Ġgor":30344,"urdue":30345,"Ġechoes":30346,"enhagen":30347,"Ġimpuls":30348,"drug":30349,"cash":30350,"Ġasync":30351,"Ġmirac":30352,"atts":30353,"punk":30354,"Ġpivot":30355,"ĠLegislative":30356,"Ġbloggers":30357,"ĠClaw":30358,"sburg":30359,"dyl":30360,"ĠRecommend":30361,"Ġverte":30362,"Ġprohibiting":30363,"ĠPanther":30364,"Jonathan":30365,"Ġomin":30366,"Ġhateful":30367,"281":30368,"ĠOrche":30369,"ĠMurdoch":30370,"downs":30371,"Ġasymm":30372,"GER":30373,"Always":30374,"Ġinforms":30375,"ĠWM":30376,"ĠPony":30377,"ĠAppendix":30378,"ĠArlington":30379,"Jam":30380,"Ġmedicinal":30381,"ĠSlam":30382,"ITIES":30383,"Ġreaff":30384,"ĠRi":30385,"FG":30386,"Spring":30387,"bool":30388,"Ġthighs":30389,"Ġmarkings":30390,"ĠRaqqa":30391,"ĠLak":30392,"poll":30393,"tsky":30394,"ĠMorty":30395,"ĠDefinition":30396,"Ġdebunk":30397,"endered":30398,"ĠLeone":30399,"avers":30400,"Ġmortgages":30401,"Apparently":30402,"Nic":30403,"haus":30404,"ĠThousands":30405,"auld":30406,"Ġmash":30407,"shoot":30408,"Ġdiarr":30409,"Ġconsciously":30410,"Hero":30411,"eas":30412,"ĠNaturally":30413,"ĠDestroyer":30414,"Ġdashboard":30415,"services":30416,"Rog":30417,"Ġmillennials":30418,"Ġinvade":30419,"-(":30420,"Ġcommissions":30421,"ĠAuckland":30422,"Ġbroadcasts":30423,"Ġfrontal":30424,"Ġcrank":30425,"ĠHistoric":30426,"Ġrumours":30427,"CTV":30428,"Ġsteril":30429,"Ġbooster":30430,"rocket":30431,"ãĤ¼":30432,"utsche":30433,"ĠPI":30434,"Ġ233":30435,"ĠProducer":30436,"ĠAnalytics":30437,"Ġinvaluable":30438,"Ġunintention":30439,"ĠCY":30440,"Ġscrutin":30441,"Ġgigg":30442,"Ġengulf":30443,"Ġproletariat":30444,"Ġhacks":30445,"ĠHew":30446,"arak":30447,"ĠSlime":30448,"ielding":30449,"agher":30450,"ĠElliot":30451,"Ġtelecom":30452,"Ġ219":30453,"ultan":30454,"ĠArbor":30455,"ĠScouts":30456,"Ban":30457,"Ġlifespan":30458,"Ġblasp":30459,"388":30460,"Ġjudiciary":30461,"ĠContinental":30462,"asking":30463,"McC":30464,"LED":30465,"Ġbaggage":30466,"ĠSorcerer":30467,"Ġremnants":30468,"ĠGriffith":30469,"etsu":30470,"ĠSubaru":30471,"ĠPersonality":30472,"designed":30473,"ushima":30474,"agnar":30475,"Ġrecoil":30476,"Ġpassions":30477,"\\\":":30478,"Ġtee":30479,"Ġabolition":30480,"ĠCreating":30481,"jac":30482,"Ġ194":30483,"019":30484,"Ġpillars":30485,"riched":30486,"/\"":30487,"tk":30488,"Ġlivelihood":30489,"Ġroasted":30490,"ahon":30491,"ĠHutch":30492,"assert":30493,"Ġdividend":30494,"Ġknit":30495,"Ġdaunting":30496,"Ġdisturbance":30497,"Ġshale":30498,"Ġcultivated":30499,"Ġrefrigerator":30500,"LB":30501,"ĠNET":30502,"Ġcommercials":30503,"Ġthinkers":30504,"455":30505,"Ġchop":30506,"Broad":30507,"Ġsuspicions":30508,"Ġtagged":30509,"lifting":30510,"Ġstylish":30511,"ĠShields":30512,"Shortly":30513,"Ġtails":30514,"Auth":30515,"STE":30516,"ĠGAME":30517,"Ġseism":30518,"ĠKis":30519,"ologne":30520,"Ġcowork":30521,"Ġforcibly":30522,"Ġthyroid":30523,"ĠPB":30524,"ANE":30525,"married":30526,"horse":30527,"Ġpolymer":30528,"ĠChal":30529,"odor":30530,"DEBUG":30531,"ĠContext":30532,"Ġbliss":30533,"Ġpinpoint":30534,"ĠMathemat":30535,"legram":30536,"ĠWeekend":30537,"Ġlabelled":30538,"Ġbart":30539,"itles":30540,"Ġestrogen":30541,"âĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶ":30542,"\"'":30543,"Ġvisibly":30544,"Ġoutsider":30545,"aida":30546,"Area":30547,"Ġdissemin":30548,"Ġdishonest":30549,"ĠClosed":30550,"ĠBulletin":30551,"ĠRamsey":30552,"sword":30553,"ĠXI":30554,"ourced":30555,"Same":30556,"346":30557,"ĠRepe":30558,"ĠKou":30559,"cake":30560,"emis":30561,"Cache":30562,"ĠMeaning":30563,"ĠEnlight":30564,"onomy":30565,"Ġmanifestation":30566,"sworth":30567,"Jay":30568,"Ġchore":30569,"ör":30570,"Dream":30571,"Ġsanctioned":30572,"Ġculturally":30573,"ĠAra":30574,"Nav":30575,"Ġtheological":30576,"Ġstrut":30577,"ĠVO":30578,"ĠHandbook":30579,"Ġconstructing":30580,"Ġ¶":30581,"ĠBenefits":30582,"ĠPsychological":30583,"sac":30584,"å¸":30585,"policy":30586,"ĠMatters":30587,"ĠReported":30588,"ĠByte":30589,"Ġvitro":30590,"ĠMaiden":30591,"Ġlam":30592,"ĠJennings":30593,"Ġgarment":30594,"ĠRutgers":30595,"ĠStafford":30596,"ĠWellington":30597,"Ġintermitt":30598,"Ġnpm":30599,"Ġordeal":30600,"Ġplugged":30601,"ooming":30602,"inished":30603,"framework":30604,"Ġtimber":30605,"Ġcass":30606,"Ġ850":30607,"iless":30608,"ĠRedux":30609,"768":30610,"Stre":30611,"Ġsurpassed":30612,"whel":30613,"Ġparallels":30614,"Ġveil":30615,"ĠGI":30616,"ĠREST":30617,"Ġreadiness":30618,"sort":30619,"Ġmodifying":30620,"ĠSlate":30621,"ruff":30622,"Ġmarble":30623,"Ġinfrared":30624,"Ġauditor":30625,"ĠFANTASY":30626,"ĠPoverty":30627,"ĠSPD":30628,"Ġ\"(":30629,"Ky":30630,"RAY":30631,"Ġexecutions":30632,"ĠBeverly":30633,"ĠMarxism":30634,"ĠBurst":30635,"ĠKali":30636,"estones":30637,"Clearly":30638,"Ell":30639,"ãģ§":30640,"ĠProceedings":30641,"Token":30642,"IFIC":30643,"ña":30644,"Central":30645,"ĠHaley":30646,"ĠDrama":30647,"Ġformations":30648,"ORN":30649,"Books":30650,"Ġdominating":30651,"ĠFlyers":30652,"ĠCompanion":30653,"Ġdisciplined":30654,"ĠYugoslav":30655,"ĠSpells":30656,"Ġvengeance":30657,"Ġlandlords":30658,"Len":30659,"ĠOgre":30660,"anoia":30661,"Ġpiercing":30662,"Ġcongreg":30663,"Ġscorer":30664,"obia":30665,"Ġnickel":30666,"ĠLearns":30667,"Ġrejo":30668,"Ġmasterpiece":30669,"Flash":30670,"Ġinhabited":30671,"ĠOpenGL":30672,"ĠDud":30673,"ĠICO":30674,"Ġarter":30675,"Ġplur":30676,"Ġmastery":30677,"Ġlongstanding":30678,"sted":30679,"Ġwines":30680,"Ġtelevised":30681,"ĠShrine":30682,"ĠBayern":30683,"Ġâĵĺ":30684,"Ġenclosure":30685,"john":30686,"Ġprophets":30687,"ĠResurrection":30688,"ĠOrders":30689,"Ġuneven":30690,"rals":30691,"Ġdwind":30692,"ĠLah":30693,"ĠSloven":30694,"378":30695,"Ġinsistence":30696,"affle":30697,"ĠClone":30698,"Ġhardship":30699,"ĠCongressman":30700,"Ġplead":30701,"Ġreviewers":30702,"Ġcured":30703,"Ġ1935":30704,"asley":30705,"fake":30706,"ĠThinking":30707,"ydia":30708,"PART":30709,"ĠDota":30710,"oit":30711,"Ġwhipped":30712,"Ġbouncing":30713,"ĠHispanics":30714,"comings":30715,"Ġcannabin":30716,"ĠChambers":30717,"ĠZack":30718,"Optional":30719,"Ġcoats":30720,"Ġprowess":30721,"ĠNorton":30722,"Ġplainly":30723,"Ġfreight":30724,"Ġinhibition":30725,"Ġclam":30726,"Ġ303":30727,"kef":30728,"aleigh":30729,"Luke":30730,"Ġpsycho":30731,"atorium":30732,"MED":30733,"Ġtreaties":30734,"Ġindisc":30735,"Ġdc":30736,"OPS":30737,"Ġresilient":30738,"ĠInterstate":30739,"Ġslack":30740,"Ġmundane":30741,"Ġestablishes":30742,"359":30743,"Ġstrained":30744,"Ġnond":30745,"Sus":30746,"Ġcaste":30747,"arate":30748,"ieving":30749,"Ġunfairly":30750,"Ġparser":30751,"onial":30752,"ursive":30753,"Via":30754,"ĠOtto":30755,"ĠAuthorities":30756,"stroke":30757,"KR":30758,"ĠMercy":30759,"Ġfurnished":30760,"Ġoutset":30761,"Ġmetic":30762,"1982":30763,"olithic":30764,"ĠTent":30765,"ogical":30766,"ĠAircraft":30767,"Ġhides":30768,"ĠBecame":30769,"Ġeducators":30770,"reaching":30771,"Ġvolatility":30772,"Ġtoddler":30773,"ĠNASCAR":30774,"ĠTwelve":30775,"ĠHighlights":30776,"Ġgrape":30777,"Ġsplits":30778,"Ġpeasant":30779,"Ġreneg":30780,"ĠMSI":30781,"Temp":30782,"stars":30783,"Ġtrek":30784,"ĠHyde":30785,"binding":30786,"Ġrealism":30787,"Ġoxide":30788,"ĠHos":30789,"Ġmounts":30790,"Ġbiting":30791,"Ġcollapsing":30792,"Ġpostal":30793,"Ġmuseums":30794,"Ġdetached":30795,"Ġrespecting":30796,"Ġmonopol":30797,"Ġworkflow":30798,"ĠCake":30799,"Template":30800,"ĠOrganisation":30801,"Ġpersistence":30802,"369":30803,"Coming":30804,"Brad":30805,"Ġredundant":30806,"ĠGTA":30807,"Ġbending":30808,"Ġrevoked":30809,"Ġoffending":30810,"Ġframing":30811,"Ġprintf":30812,"Commun":30813,"members":30814,"Outside":30815,"Ġconstrued":30816,"Ġcoded":30817,"FORE":30818,"Ġchast":30819,"Chat":30820,"Indian":30821,"ĠYard":30822,"?!\"":30823,"ĠPorts":30824,"ĠXavier":30825,"ĠRET":30826,"'.\"":30827,"ĠBoat":30828,"ivated":30829,"icht":30830,"umerable":30831,"Ds":30832,"ĠDunn":30833,"Ġcoffin":30834,"Ġsecurely":30835,"ĠRaptors":30836,"ĠBes":30837,"Installation":30838,"Ġinception":30839,"ĠHealthy":30840,"endants":30841,"Ġpsychologists":30842,"ĠSheikh":30843,"cultural":30844,"ĠBlackBerry":30845,"shift":30846,"Fred":30847,"oche":30848,"Ġcakes":30849,"ĠSEO":30850,"ĠGian":30851,"ĠAsians":30852,"ogging":30853,"element":30854,"Ġpundits":30855,"ĠVaugh":30856,"ĠGavin":30857,"Ġhitter":30858,"Ġdrowned":30859,"Ġchalk":30860,"ĠZika":30861,"Ġmeasles":30862,"802":30863,"âĢ¦..":30864,"ĠAWS":30865,"]\"":30866,"Ġdistort":30867,"ĠMast":30868,"Ġantibodies":30869,"ĠMash":30870,"Memory":30871,"ĠUganda":30872,"ĠProb":30873,"Ġvomiting":30874,"ĠTurns":30875,"Ġoccupying":30876,"Ġevasion":30877,"ĠTherapy":30878,"Ġpromo":30879,"Ġelectr":30880,"Ġblueprint":30881,"ĠDre":30882,"priced":30883,"ĠDepot":30884,"Ġalleviate":30885,"ĠSomali":30886,"marg":30887,"nine":30888,"Ġnostalgia":30889,"ĠShepherd":30890,"Ġcavalry":30891,"Ġtorped":30892,"ĠBloody":30893,"xb":30894,"Ġsank":30895,"Ġgoalt":30896,"reportprint":30897,"embedreportprint":30898,"cloneembedreportprint":30899,"ĠInitially":30900,"ĠFischer":30901,"Ġnoteworthy":30902,"cern":30903,"Ġinefficient":30904,"rawdownload":30905,"rawdownloadcloneembedreportprint":30906,"cation":30907,"ĠDynasty":30908,"lag":30909,"DES":30910,"Ġdistinctly":30911,"ĠEstonia":30912,"Ġopenness":30913,"Ġgossip":30914,"ruck":30915,"Width":30916,"ĠIbrahim":30917,"Ġpetroleum":30918,"Ġavatar":30919,"ĠHed":30920,"atha":30921,"ĠHogwarts":30922,"Ġcaves":30923,"678":30924,"Ġsafeguard":30925,"ĠMog":30926,"isson":30927,"ĠDurham":30928,"slaught":30929,"ĠGraduate":30930,"Ġsubconscious":30931,"ĠExcellent":30932,"ĠDum":30933,"-----":30934,"Ġpiles":30935,"ĠWORK":30936,"ĠGarn":30937,"ĠFol":30938,"ĠATM":30939,"Ġavoids":30940,"ĠTul":30941,"Ġbleak":30942,"ELY":30943,"ivist":30944,"lightly":30945,"Pers":30946,"ĠDob":30947,"ĠLS":30948,"Ġinsanity":30949,"ε":30950,"atalie":30951,"Enlarge":30952,"Ġtwists":30953,"Ġfaulty":30954,"Ġpiracy":30955,"Ġimpover":30956,"Ġrugged":30957,"ĠFashion":30958,"Ġsands":30959,"'?":30960,"swick":30961,"Ġnatives":30962,"Ġhen":30963,"ĠNoise":30964,"ãĥĹ":30965,"Ġgreens":30966,"Ġfreezer":30967,"Ġdynasty":30968,"ĠFathers":30969,"ĠNewark":30970,"Ġarchaeological":30971,"Ġot":30972,"obar":30973,"Ġblockade":30974,"Ġallerg":30975,"LV":30976,"Ġdebit":30977,"ĠRFC":30978,"ĠMilton":30979,"ĠPressure":30980,"Ġwillingly":30981,"Ġdisproportionate":30982,"Ġoppressive":30983,"Ġdiamonds":30984,"Ġbelongings":30985,"1970":30986,"Ġbells":30987,"Ġimperialism":30988,"Ġ227":30989,"Ġexploding":30990,"ĠEclipse":30991,"Ġ1919":30992,"Ġrant":30993,"Ġnominations":30994,"347":30995,"Ġpeacefully":30996,"rica":30997,"ĠFUCK":30998,"Ġvibration":30999,"malink":31000,"Ġropes":31001,"ĠIvanka":31002,"ĠBrewery":31003,"ĠBooker":31004,"ĠOwens":31005,"goers":31006,"Services":31007,"ĠSnape":31008,"Ġ191":31009,"395":31010,"Ġ299":31011,"justice":31012,"Ġbri":31013,"Ġdiscs":31014,"Ġprominently":31015,"Ġvulgar":31016,"Ġskipping":31017,"lves":31018,"Ġtsunami":31019,"374":31020,"ĠUrug":31021,"ĠEid":31022,"recated":31023,"phen":31024,"Ġfaults":31025,"ĠStarted":31026,"950":31027,"Ġpi":31028,"Ġdetector":31029,"Ġbastard":31030,"Ġvalidated":31031,"SpaceEngineers":31032,"OURCE":31033,"Ġ(~":31034,"Ġunsur":31035,"Ġaffirmed":31036,"Ġfascism":31037,"Ġresolving":31038,"ĠChavez":31039,"ĠCyn":31040,"Ġdetract":31041,"Lost":31042,"Ġrigged":31043,"Ġhomage":31044,"ĠBruno":31045,"555":31046,"eca":31047,"Ġpresses":31048,"Ġhumour":31049,"Ġspacing":31050,"Ġ'/":31051,"olkien":31052,"Coun":31053,"OPER":31054,"Tre":31055,"Son":31056,"ĠCambodia":31057,"ierre":31058,"mong":31059,"ozy":31060,"Ġliquidity":31061,"ĠSoviets":31062,"ĠFernando":31063,"Ġ229":31064,"Ġslug":31065,"ĠCatalan":31066,"electric":31067,"Ġscenery":31068,"ĠHearth":31069,"Ġconstrained":31070,"Ġgoalie":31071,"ĠGuidelines":31072,"ĠAmmo":31073,"ĠPearson":31074,"Ġtaxed":31075,"Ġfetus":31076,"Response":31077,"ĠAlexis":31078,"thia":31079,"Guy":31080,"Ġreconstruct":31081,"Ġextremes":31082,"Ġconcluding":31083,"ĠPeg":31084,"ooks":31085,"Ġdeductions":31086,"Rose":31087,"Ġgroundbreaking":31088,"ĠTarg":31089,"ãĥģ":31090,"ĠReve":31091,"resource":31092,"Ġmoons":31093,"Ġelectromagnetic":31094,"Ġamidst":31095,"ĠViktor":31096,"NESS":31097,"BACK":31098,"Ġcommute":31099,"ĠAnaheim":31100,"Ġfluctuations":31101,"640":31102,"Ġnoodles":31103,"ĠCopenhagen":31104,"ĠTide":31105,"ĠGrizz":31106,"ĠSEE":31107,"Ġpipelines":31108,"Ġscars":31109,"endo":31110,"agus":31111,"ĠETF":31112,"/#":31113,"ĠBecome":31114,"448":31115,"Ġvisc":31116,"ĠRecommended":31117,"Ġjumper":31118,"Ġcognition":31119,"Ġassassin":31120,"Ġwitnessing":31121,"ĠSetup":31122,"Ġlac":31123,"vim":31124,"ISM":31125,"pages":31126,"SSL":31127,"358":31128,"Ġadject":31129,"industrial":31130,"lore":31131,"chery":31132,"Ġglitter":31133,"Ġcalf":31134,"Florida":31135,"Ġspoilers":31136,"Ġsucceeds":31137,"Ġchanting":31138,"Ġslogans":31139,"ĠTracy":31140,"Visit":31141,"rology":31142,"Ġmornings":31143,"Ġlineage":31144,"Ġsip":31145,"Ġintensely":31146,"Ġflourish":31147,"ĠSleeping":31148,"ĠFem":31149,"orpor":31150,"ĠKlan":31151,"ĠDarth":31152,"hack":31153,"ĠNielsen":31154,"Ġtumors":31155,"Ġprocurement":31156,"ĠYorkshire":31157,"Ġraided":31158,"KY":31159,"Anna":31160,"Ġ//[":31161,"ĠDisorder":31162,"ĠMustang":31163,"ĠWen":31164,"ĠTrying":31165,"sq":31166,"Ġdeliveries":31167,"Ġshutter":31168,"Ġcerebral":31169,"Ġbipolar":31170,"ĠCN":31171,"lass":31172,"jet":31173,"Ġdebating":31174,">:":31175,"Ġeagle":31176,"grades":31177,"ĠDixon":31178,"UGC":31179,"MAS":31180,"ĠDraco":31181,"ĠMachines":31182,"affer":31183,"Ġeman":31184,"²":31185,"pron":31186,"ĠGym":31187,"Ġcomparatively":31188,"ĠTribunal":31189,"PRO":31190,"Ġlex":31191,"Ġfertile":31192,"Ġdepressing":31193,"Ġsuperficial":31194,"essential":31195,"ĠHunters":31196,"gp":31197,"Ġprominence":31198,"Liber":31199,"ĠAncest":31200,"otechnology":31201,"Ġmocking":31202,"ĠTraff":31203,"ĸļ":31204,"Medium":31205,"Iraq":31206,"Ġpsychiatrist":31207,"Quantity":31208,"ĠLect":31209,"Ġnoisy":31210,"520":31211,"GY":31212,"Ġslapped":31213,"ĠMTV":31214,"Ġpara":31215,"pull":31216,"Multiple":31217,"asher":31218,"Ġnour":31219,"ĠSeg":31220,"Spell":31221,"vous":31222,"ordial":31223,"Senior":31224,"ĠGoldberg":31225,"ĠPlasma":31226,"need":31227,"Ġmessenger":31228,"eret":31229,"Ġteamed":31230,"Ġliteracy":31231,"ĠLeah":31232,"ĠDoyle":31233,"Ġemitted":31234,"UX":31235,"Ġevade":31236,"Ġmaze":31237,"Ġwrongly":31238,"ĠLars":31239,"Ġstereotype":31240,"Ġpledges":31241,"Ġaroma":31242,"ĠMET":31243,"Ġacre":31244,"ĠOD":31245,"Ġff":31246,"Ġbreweries":31247,"ĠHilton":31248,"undle":31249,"ĠKak":31250,"ĠThankfully":31251,"ĠCanucks":31252,"inctions":31253,"ĠAppears":31254,"Ġcoer":31255,"Ġundermined":31256,"rovers":31257,"Andre":31258,"Ġblaze":31259,"umers":31260,"Ġfamine":31261,"amphetamine":31262,"ulkan":31263,"Amount":31264,"Ġdesperation":31265,"wikipedia":31266,"development":31267,"ĠCorinth":31268,"ussia":31269,"Jackson":31270,"LI":31271,"Native":31272,"Rs":31273,"Ohio":31274,"ĠKathleen":31275,"Fortunately":31276,"Ġattendant":31277,"ĠPreferred":31278,"ĠDidn":31279,"ĠVs":31280,"Mis":31281,"Ġrespondent":31282,"Ġboun":31283,"stable":31284,"Ġpaved":31285,"Ġunexpl":31286,"ĠCheney":31287,"LM":31288,"ĠCull":31289,"blown":31290,"Ġconfronting":31291,"ocese":31292,"serving":31293,"Wi":31294,"ĠLithuania":31295,"anni":31296,"Ġstalk":31297,"hd":31298,"Ġvener":31299,"APH":31300,"ynchronous":31301,"URR":31302,"umably":31303,"historic":31304,"Half":31305,"Hay":31306,"Ġresilience":31307,"spection":31308,"Ġabandoning":31309,"Obs":31310,"ĠDebbie":31311,"Ġgradient":31312,"ĠPlaint":31313,"ĠCanal":31314,"ARCH":31315,"Ġexpansive":31316,"Ġfung":31317,"Ġbounced":31318,"Und":31319,"Ġprecautions":31320,"Ġclarification":31321,"Ġdagger":31322,"Ġgrips":31323,"Ġµ":31324,"ĠRivera":31325,"ĠUndead":31326,"isites":31327,"ĠFIRST":31328,"ño":31329,"audi":31330,"Ġhostages":31331,"Ġcompliant":31332,"Ġalumni":31333,"Seven":31334,"Ġcybersecurity":31335,"either":31336,"Collect":31337,"Ġinvariably":31338,"ĠSoci":31339,"Ġlawmaker":31340,"Ġale":31341,"ĠPersonally":31342,"Nazi":31343,"Ġcustomization":31344,"ĠProc":31345,"ĠSaskatchewan":31346,"eaturing":31347,"Ġspared":31348,"Ġdiscontinued":31349,"Ġcomputational":31350,"ĠMotorola":31351,"Ġsupremacist":31352,"governmental":31353,"Ġparadise":31354,"ĠDowning":31355,"ĠNikon":31356,"Ġcatalyst":31357,"berra":31358,"Toronto":31359,"875":31360,"beta":31361,"ĠMacron":31362,"Ġunrealistic":31363,"vector":31364,"ĠVehicles":31365,"itiveness":31366,"ĠRV":31367,"ĠColbert":31368,"sin":31369,"oji":31370,"entin":31371,"ĠKrish":31372,"hello":31373,"ffield":31374,"oky":31375,"ĠTate":31376,"Ġmaple":31377,"Ġaids":31378,"chemical":31379,"334":31380,"nuts":31381,"ĠWarp":31382,"Ġxx":31383,"ĠRobb":31384,"umerous":31385,"_-_":31386,"ftime":31387,"ĠVW":31388,"Ġwinger":31389,"ĠDome":31390,"tools":31391,"ĠPV":31392,"ĠGeorgetown":31393,"Ġgeared":31394,"Ġjihadists":31395,"Ġcp":31396,"Ġsteroids":31397,"Mother":31398,"clerosis":31399,"ĠDRM":31400,"nesia":31401,"Ġlinger":31402,"Ġimmersive":31403,"ĠCOUN":31404,"Ġoutweigh":31405,"ensual":31406,"Band":31407,"Ġtransforms":31408,"matched":31409,"psons":31410,"ĠJudicial":31411,"factor":31412,"Ġreferral":31413,"Ġoddly":31414,"ĠWenger":31415,"Bring":31416,"ĠBows":31417,"602":31418,"ICLE":31419,"Ġlions":31420,"ĠAcademic":31421,"ĠThorn":31422,"ĠRaider":31423,"kefeller":31424,"Storage":31425,"Lower":31426,"ĠOrt":31427,"ĠEquality":31428,"ALT":31429,"ĠSOC":31430,"Types":31431,"Ġlyn":31432,"ĠAsset":31433,"coat":31434,"TPP":31435,"CVE":31436,"ĠPioneer":31437,"application":31438,"Modern":31439,"ĠHK":31440,"Environment":31441,"Alright":31442,"Rain":31443,"IPP":31444,"ĠShiite":31445,"Ġmound":31446,"ĠAbilities":31447,"condition":31448,"Staff":31449,"Ġcompetence":31450,"ĠMoor":31451,"ĠDiablo":31452,"Ġwithheld":31453,"Ġostensibly":31454,"ĠBrom":31455,"Ġmsg":31456,"Ġdenomin":31457,"ĠReferences":31458,"ĠFP":31459,"Ġplunged":31460,"Ġpamph":31461,"moving":31462,"central":31463,"Ġdownright":31464,"Ġfading":31465,"Tal":31466,"Typ":31467,"ĠThy":31468,"ukes":31469,"ithe":31470,"Ġove":31471,"Ġbattled":31472,"Ġseafood":31473,"Ġfigur":31474,"ĠRD":31475,"crop":31476,"Ġsquads":31477,"{\\":31478,"à¹":31479,"ĠEh":31480,"Ġinterviewing":31481,"ĠQin":31482,"Ġaspiring":31483,"PLIC":31484,"Ġclauses":31485,"ĠGast":31486,"ĠNir":31487,"Ġluggage":31488,"Ġhose":31489,"Ġsystemd":31490,"Ġdescending":31491,"ĠRevised":31492,"ĠRails":31493,"align":31494,"709":31495,"337":31496,"Ġfug":31497,"charging":31498,"tags":31499,"Ġuter":31500,"kish":31501,"WARNING":31502,"490":31503,"profits":31504,"Ġvoyage":31505,"Ġace":31506,"ĠVanguard":31507,"ĠTanks":31508,"ĠMuk":31509,"Ġ226":31510,"Safe":31511,"Armor":31512,"Ġvolcanic":31513,"Ġwomb":31514,"ĠMIL":31515,"Ġbeginner":31516,"ĠRecogn":31517,"ĠAAP":31518,"PLAY":31519,")!":31520,"Ġdetecting":31521,"cn":31522,"Ġbreaches":31523,"Basically":31524,"ĠPag":31525,"ĠMunicipal":31526,"ĠIndie":31527,"ĠLaf":31528,"ĠDisable":31529,"ĠOlson":31530,"Ġrestrained":31531,"Ġrulings":31532,"Ġhumane":31533,"events":31534,"ĠCinema":31535,"displayText":31536,"ĠHatch":31537,"actionDate":31538,"onnaissance":31539,"Ġassaulting":31540,"ĠLug":31541,"CHAT":31542,"Ġvigorous":31543,"ĠPerse":31544,"Ġintolerance":31545,"ĠSnapchat":31546,"ĠSharks":31547,"Ġdummy":31548,"ĠDiagn":31549,"ĠGuitar":31550,"imeters":31551,"403":31552,"REG":31553,"Ax":31554,"Ġseparates":31555,"ĠMahm":31556,"Ġtv":31557,"jah":31558,"OOL":31559,"Circ":31560,"ĠWindsor":31561,"ussian":31562,"Ġintuition":31563,"Ġdisdain":31564,"ĠDonovan":31565,"Ġ221":31566,"Emb":31567,"Ġcondemning":31568,"Ġgenerosity":31569,"zzy":31570,"Ġpanties":31571,"ĠPrevent":31572,"ActionCode":31573,"ANA":31574,"342":31575,"externalActionCode":31576,"Ġspecifying":31577,"Ġcrystall":31578,"Jere":31579,"Ġrupt":31580,"ĠApprentice":31581,"Ġprofiling":31582,"к":31583,"Strike":31584,"Ġsideline":31585,"Ġobligated":31586,"Ġoccult":31587,"Ġbureaucratic":31588,"antically":31589,"rupted":31590,"negative":31591,"ĠEthiopia":31592,"ĠCivic":31593,"Ġinsiders":31594,"eligible":31595,"ĠTVs":31596,"ĠBAR":31597,"ĠTI":31598,"iologist":31599,"ĠAIR":31600,"Ġsubstituted":31601,"Arab":31602,"ĠSaul":31603,"ĠYog":31604,"prem":31605,"Ġbuilders":31606,"Ġstationary":31607,"Ġdoubtful":31608,"Ġvigorously":31609,"Ġthrilling":31610,"Physical":31611,"ĠCarey":31612,"ĠHydra":31613,"geoning":31614,"ĠSly":31615,"yton":31616,"Ġborrowers":31617,"ĠParkinson":31618,"Ġë":31619,"ĠJamaica":31620,"Ġsatir":31621,"Ġinsurgents":31622,"ĠFirm":31623,"Ġisot":31624,"ĠKarn":31625,"ourning":31626,"akens":31627,"docs":31628,"little":31629,"ĠMonaco":31630,"CLASS":31631,"Turkey":31632,"Ly":31633,"ĠConan":31634,"assic":31635,"Ġstarred":31636,"ĠPacers":31637,"eties":31638,"Ġtipping":31639,"Moon":31640,"ĠRw":31641,"same":31642,"Ġcavity":31643,"Ġgoof":31644,"ĠZo":31645,"Shock":31646,"ummer":31647,"Ġemphasizes":31648,"Ġregrett":31649,"Ġnovelty":31650,"Ġenvy":31651,"ĠPassive":31652,"rw":31653,"505":31654,"Ġindifferent":31655,"ĠRica":31656,"ĠHimself":31657,"ĠFreddie":31658,"Ġadip":31659,"ä¸Ģ":31660,"Ġbreakout":31661,"Ġhurried":31662,"ĠHuang":31663,"ĠDisk":31664,"Ġroaming":31665,"?????-?????-":31666,"UV":31667,"ĠRicky":31668,"ĠSigma":31669,"Ġmarginalized":31670,"Ġedits":31671,"Ġ304":31672,"memory":31673,"Ġspecimen":31674,"293":31675,"ãģ¯":31676,"Ġvertically":31677,"Ġaudition":31678,"ĠHeck":31679,"Ġcaster":31680,"ĠHoldings":31681,"adal":31682,"ĠCron":31683,"ĠLiam":31684,"Ġdeflect":31685,"Pick":31686,"ĠDebug":31687,"REF":31688,"Ġversatility":31689,"othes":31690,"classified":31691,"ĠMahar":31692,"ĠHort":31693,"Counter":31694,"stasy":31695,"noticed":31696,"331":31697,"ĠShim":31698,"fuck":31699,"ĠBie":31700,"Ġairing":31701,"ĠProtein":31702,"ĠHolding":31703,"Ġspectators":31704,"iliated":31705,"ĠThatcher":31706,"nosis":31707,"ãĥ¼ãĥ³":31708,"Tele":31709,"Boston":31710,"ĠTempl":31711,"stay":31712,"Ġdeclarations":31713,"479":31714,"Volume":31715,"ĠDesigner":31716,"ĠOverwatch":31717,"idae":31718,"Ġonwards":31719,"Ġnets":31720,"ĠManila":31721,"particularly":31722,"Ġpolitic":31723,"oother":31724,"Ġportraits":31725,"Ġpavement":31726,"cffff":31727,"Ġsaints":31728,"Ġbeginners":31729,"ESPN":31730,"Ġshortcomings":31731,"âķIJâķIJ":31732,"Ġcomet":31733,"ĠOrganic":31734,"quel":31735,"Ġhospitalized":31736,"Break":31737,"Ġpeel":31738,"dylib":31739,"aspx":31740,"urances":31741,"ĠTIM":31742,"Pg":31743,"Ġreadable":31744,"ĠMalik":31745,"Ġmuzzle":31746,"Ġbenchmarks":31747,"dal":31748,"ĠVacc":31749,"ĠHicks":31750,"609":31751,"ĠBiblical":31752,"heng":31753,"Ġoverload":31754,"ĠCivilization":31755,"Ġimmoral":31756,"Ġfries":31757,"ãĤĴ":31758,"Ġreproduced":31759,"Ġformulation":31760,"jug":31761,"irez":31762,"gear":31763,"Ġcoached":31764,"MpServer":31765,"ĠSJ":31766,"ĠKw":31767,"Init":31768,"deal":31769,"ĠOro":31770,"ĠLoki":31771,"ĠSongs":31772,"Ġ232":31773,"ĠLouise":31774,"asionally":31775,"Ġuncond":31776,"ollywood":31777,"Ġprogressives":31778,"ĠEnough":31779,"ĠDoe":31780,"Ġwreckage":31781,"Ġbrushed":31782,"ĠBaseType":31783,"Ġzoning":31784,"ishable":31785,"hetically":31786,"ĠCaucus":31787,"ĠHue":31788,"Ġkarma":31789,"ĠSporting":31790,"Ġtrader":31791,"Ġseeming":31792,"ĠCapture":31793,"430":31794,"bish":31795,"Ġtunes":31796,"Ġindoors":31797,"ĠSphere":31798,"ĠDancing":31799,"TERN":31800,"Ġnob":31801,"ĠGST":31802,"maps":31803,"Ġpeppers":31804,"Fit":31805,"Ġoversees":31806,"ĠRabbi":31807,"ĠRuler":31808,"vertising":31809,"office":31810,"xxx":31811,"Ġraft":31812,"Changed":31813,"Ġtextbooks":31814,"Links":31815,"ĠOmn":31816,"ãĢij":31817,"Ġinconvenience":31818,"ĠDonetsk":31819,"=~":31820,"Ġimplicitly":31821,"Ġboosts":31822,"ĠBones":31823,"ĠBoom":31824,"Courtesy":31825,"Ġsensational":31826,"ANY":31827,"Ġgreedy":31828,"eden":31829,"Ġinexper":31830,"ĠLer":31831,"ĠVale":31832,"Ġtighten":31833,"ĠEAR":31834,"ĠNum":31835,"Ġancestor":31836,"Sent":31837,"ĠHorde":31838,"urgical":31839,"allah":31840,"Ġsap":31841,"amba":31842,"ĠSpread":31843,"twitch":31844,"Ġgrandson":31845,"Ġfracture":31846,"Ġmoderator":31847,"ĠSeventh":31848,"ĠReverse":31849,"Ġestimation":31850,"Choose":31851,"Ġparach":31852,"Ġbarric":31853,"ãĢIJ":31854,"Ġcompass":31855,"Ġallergic":31856,"âĢķ":31857,"OTHER":31858,"errilla":31859,"Ġwagon":31860,"Ġzinc":31861,"Ġrubbed":31862,"ĠFuller":31863,"ĠLuxembourg":31864,"ĠHoover":31865,"Ġliar":31866,"ĠEvening":31867,"ĠCobb":31868,"esteem":31869,"Ġselector":31870,"ĠBrawl":31871,"isance":31872,"ĠEk":31873,"Ġtroop":31874,"Ġguts":31875,"ĠAppeal":31876,"ĠTibetan":31877,"Ġroutines":31878,"ĠMent":31879,"Ġsummarized":31880,"steamapps":31881,"Ġtranqu":31882,"Ġ1929":31883,"oran":31884,"ĠAuthent":31885,"Ġgmaxwell":31886,"Ġapprehens":31887,"Ġpoems":31888,"Ġsausage":31889,"ĠWebster":31890,"urus":31891,"Ġthemed":31892,"Ġlounge":31893,"Ġcharger":31894,"Spoiler":31895,"Ġspilled":31896,"hog":31897,"ĠSunder":31898,"ĠAin":31899,"ĠAngry":31900,"Ġdisqual":31901,"ĠFrequency":31902,"ĠEthernet":31903,"Ġhelper":31904,"Percent":31905,"Ġhorrifying":31906,"Ġail":31907,"ĠAllan":31908,"EEE":31909,"ĠCrossing":31910,"449":31911,"Ġholog":31912,"ĠPuzzles":31913,"ĠGoes":31914,"erenn":31915,"604":31916,"ãģı":31917,"ĠRafael":31918,"Ġatten":31919,"ĠEmanuel":31920,"Ġupro":31921,"ĠSusp":31922,"Psych":31923,"ĠTrainer":31924,"ĠNES":31925,"ĠHunts":31926,"becue":31927,"Ġcounselor":31928,"Rule":31929,"Ġtoxins":31930,"Ġbanners":31931,"rifice":31932,"Ġgreeting":31933,"Ġfrenzy":31934,"Ġallocate":31935,"Ġ*)":31936,"expr":31937,"503":31938,"ĠChick":31939,"ĠTorn":31940,"Ġconsolidation":31941,"ĠFletcher":31942,"switch":31943,"frac":31944,"clips":31945,"ĠMcKin":31946,"ĠLunar":31947,"Month":31948,"ITCH":31949,"Ġscholarly":31950,"raped":31951,"398":31952,"Ġ1910":31953,"Ġegreg":31954,"Ġinsecure":31955,"Ġvictorious":31956,"cffffcc":31957,"Ġsingled":31958,"Ġelves":31959,"ĠWond":31960,"burst":31961,"Ġcamoufl":31962,"ĠBLACK":31963,"Ġconditioned":31964,"çī":31965,"answered":31966,"Ġcompulsory":31967,"ascist":31968,"Ġpodcasts":31969,"ĠFrankfurt":31970,"bnb":31971,"Ġneoliberal":31972,"ĠKeyboard":31973,"ĠBelle":31974,"warm":31975,"Ġtrusts":31976,"Ġinsured":31977,"ĠBucc":31978,"usable":31979,"607":31980,"ĠPlains":31981,"Ġ1890":31982,"Ġsabotage":31983,"Ġlodged":31984,"felt":31985,"Ġga":31986,"ĠNarc":31987,"ĠSalem":31988,"Ġseventy":31989,"ĠBlank":31990,"pocket":31991,"Ġwhisper":31992,"Ġmating":31993,"omics":31994,"ĠSalman":31995,"ĠKad":31996,"Ġangered":31997,"Ġcollisions":31998,"Ġextraordinarily":31999,"Ġcoercion":32000,"Ghost":32001,"birds":32002,"èĢ":32003,"kok":32004,"Ġpermissible":32005,"avorable":32006,"Ġpointers":32007,"Ġdissip":32008,"aci":32009,"Ġtheatrical":32010,"ĠCosmic":32011,"Ġforgetting":32012,"Ġfinalized":32013,"大":32014,"yout":32015,"library":32016,"Ġbooming":32017,"ĠBelieve":32018,"ĠTeacher":32019,"ĠLiv":32020,"ĠGOODMAN":32021,"ĠDominican":32022,"ORED":32023,"ĠParties":32024,"Ġprecipitation":32025,"ĠSlot":32026,"Roy":32027,"ĠCombined":32028,"Ġintegrating":32029,"Ġchrome":32030,"Ġintestinal":32031,"ĠRebell":32032,"Ġmatchups":32033,"Ġblockbuster":32034,"ĠLoren":32035,"ĠLevy":32036,"Ġpreaching":32037,"ĠSending":32038,"ĠPurpose":32039,"rax":32040,"fif":32041,"Ġauthoritative":32042,"ĠPET":32043,"astical":32044,"Ġdishon":32045,"Ġchatting":32046,"Ġ\"$:/":32047,"Connection":32048,"Ġrecreate":32049,"Ġdelinqu":32050,"Ġbroth":32051,"ĠDirty":32052,"ĠAdmin":32053,"zman":32054,"Ġscholarships":32055,"Ġ253":32056,"contact":32057,"alsa":32058,"767":32059,"creen":32060,"abbage":32061,"Ġ1915":32062,"Ġblended":32063,"Ġalarmed":32064,"Language":32065,"356":32066,"Ġblends":32067,"ĠChanged":32068,"Wolf":32069,"Ġhepat":32070,"Creating":32071,"Ġpersecut":32072,"Ġsweetness":32073,"arte":32074,"Ġforfeiture":32075,"ĠRoberto":32076,"impro":32077,"NFL":32078,"ĠMagnet":32079,"Detailed":32080,"Ġinsignificant":32081,"ĠPOLIT":32082,"ĠBBQ":32083,"ĠCPS":32084,"Ġseaw":32085,"aminer":32086,"mL":32087,"endif":32088,"finals":32089,"Ġ265":32090,"uish":32091,"Ġ})":32092,"ĠProblems":32093,"Ġemblem":32094,"Ġseriousness":32095,"Ġparsing":32096,"Ġsubstitution":32097,"Ġpressured":32098,"Ġrecycled":32099,"aleb":32100,"Ruby":32101,"Ġproficiency":32102,"Driver":32103,"ĠWester":32104,":'":32105,"AFTA":32106,"Ġmantle":32107,"ĠClayton":32108,"flag":32109,"Ġpractitioner":32110,"covered":32111,"ĠStruct":32112,"addafi":32113,"425":32114,"ĠTownship":32115,"ĠHydro":32116,"Louis":32117,"343":32118,"Ġcondo":32119,"ĠTao":32120,"Ġutilization":32121,"Ġnausea":32122,"ĠDems":32123,"ridges":32124,"pause":32125,"Ġformulas":32126,"Ġchallenger":32127,"376":32128,"Ġdefective":32129,"ĠRailway":32130,"ĠPubMed":32131,"Ġyogurt":32132,"lbs":32133,"ĠNorfolk":32134,"OPE":32135,"ĠMoody":32136,"Ġdistributor":32137,"Ġscrolls":32138,"Ġextracts":32139,"Stan":32140,"Ġviability":32141,"Ġexposes":32142,"Ġstarvation":32143,"ĠSteps":32144,"ĠDodd":32145,"few":32146,"STD":32147,"332":32148,"Ġclosures":32149,"Ġcomplementary":32150,"ĠSasha":32151,"umpy":32152,"Ġmonet":32153,"Ġarticulate":32154,"ĠDoct":32155,"killer":32156,"Ġscrim":32157,"Ġ264":32158,"Ġprostitutes":32159,"Ġsevered":32160,"Ġattachments":32161,"Ġcooled":32162,"Lev":32163,"ĠFalk":32164,"fail":32165,"Ġpoliceman":32166,"ĠDag":32167,"Ġprayed":32168,"ĠKernel":32169,"Ġclut":32170,"Ġcath":32171,"Ġanomaly":32172,"Storm":32173,"emaker":32174,"ĠBreakfast":32175,"uli":32176,"oire":32177,"JJ":32178,"hz":32179,"Operation":32180,"ĠSick":32181,"354":32182,"ĠGuatemala":32183,"Rate":32184,"Ġexposures":32185,"faces":32186,"ĠArchae":32187,"raf":32188,"ĠMia":32189,"Ġ2025":32190,"Ġopaque":32191,"Ġdisguised":32192,"ĠHeadquarters":32193,"Sah":32194,"Ġpots":32195,"978":32196,"ĠMalf":32197,"Ġfrowned":32198,"Ġpoisonous":32199,"ĠConvers":32200,"eeks":32201,"Ġcrab":32202,".\"\"":32203,"Ġtreason":32204,"Ġranc":32205,"Ġescalating":32206,"Ġwarr":32207,"Ġmobs":32208,"Ġlamps":32209,"ĠSunshine":32210,"ĠBrunswick":32211,"Phones":32212,"Ġspelled":32213,"ĠSkip":32214,"Ġ2050":32215,"Ġ1911":32216,"ĠPluto":32217,"ĠAmend":32218,"Ġmeats":32219,"387":32220,"Ġstomp":32221,"ĠZhou":32222,"ĠLeviathan":32223,"ĠHazard":32224,"adv":32225,"ĠOrwell":32226,"Ġaloud":32227,"Ġbumper":32228,"ĠAnarch":32229,"ubuntu":32230,"ĠSerious":32231,"fitting":32232,"ĠOptional":32233,"ĠCecil":32234,"REAM":32235,"Ġserotonin":32236,"Ġcultivate":32237,"agogue":32238,"}\\":32239,"Ġmosques":32240,"ĠSunny":32241,"Ġreactive":32242,"revolution":32243,"ĠLup":32244,"ĠFedora":32245,"Ġdefenseman":32246,"ĠVID":32247,"istine":32248,"Ġdrowning":32249,"ĠBroadcasting":32250,"Ġthriller":32251,"ĠScy":32252,"Ġaccelerating":32253,"Ġdirects":32254,"odied":32255,"bike":32256,"duration":32257,"Ġpainfully":32258,"Redd":32259,"Ġproductions":32260,"Ġgag":32261,"Ġwhist":32262,"Ġsock":32263,"Ġinfinitely":32264,"ĠConcern":32265,"ĠCitadel":32266,"Ġlieu":32267,"Ġcandles":32268,"ogeneous":32269,"arger":32270,"Ġheavenly":32271,"inflammatory":32272,"Performance":32273,"Cs":32274,"ructose":32275,"azaki":32276,"Ġpessim":32277,"Ġinference":32278,"Ġpowd":32279,"ĠZoe":32280,"Ġpaints":32281,"Ġdazz":32282,"pta":32283,"-----------":32284,"Ġinspir":32285,"ĠExperimental":32286,"ĠKnife":32287,"regor":32288,"bors":32289,"Ġshowers":32290,"romeda":32291,"Ġsaint":32292,"Ġbenign":32293,"ĠJiang":32294,"Ġenvisioned":32295,"Ġshroud":32296,"IFT":32297,"HO":32298,"Ġshuff":32299,"ĠICC":32300,"Ġsegreg":32301,"Ġrevisit":32302,"ighthouse":32303,"Li":32304,"Ġsubstrate":32305,"ĠSeas":32306,"ĠReward":32307,"ĠHep":32308,"ĠBrass":32309,"sbm":32310,"Ġeliminates":32311,"Ġstamina":32312,"ĠVAT":32313,"ĠLoan":32314,"Ġconstraint":32315,"Ġappropriated":32316,"Ġpes":32317,"ĠALE":32318,"ranging":32319,"Ġ404":32320,"392":32321,"Ġintellectuals":32322,"achu":32323,"Ġrestructuring":32324,"ĠLevin":32325,"Ġrunes":32326,"Ġdelightful":32327,"Ġcarbohydrates":32328,"ĠModels":32329,"ĠExpo":32330,"Ġtransporting":32331,"alloc":32332,"Ġringing":32333,"Samsung":32334,"Ġscarcely":32335,"ĠURLs":32336,"ĠMAS":32337,"Ġprototypes":32338,"Ġnarrator":32339,"ĠCPUs":32340,"cdn":32341,"ĠBarton":32342,"Ġdecidedly":32343,"ĠShu":32344,"ixir":32345,"ocious":32346,"ĠMyst":32347,"Nintendo":32348,"Ġreuse":32349,"Ġforgiven":32350,"Few":32351,"inical":32352,"nat":32353,"Ġseamless":32354,"ĠEva":32355,"ĠEVE":32356,"ĠJO":32357,"landers":32358,"Ġsofter":32359,"negie":32360,"Ġtransient":32361,"Ġorbital":32362,"Ġfulfil":32363,"ĠKom":32364,"Hopefully":32365,"Ġdynamically":32366,"ĠHunger":32367,"åĽ":32368,"ĠArmenia":32369,"elman":32370,"berto":32371,"Ġpige":32372,"ĠIDs":32373,"limit":32374,"Ġveins":32375,"Ġsoaring":32376,"packs":32377,"Golden":32378,"ĠCrab":32379,"istor":32380,"ĠRPM":32381,"Ġ$$":32382,"gression":32383,"Ġjihadist":32384,"Ġgamble":32385,"Ġcareg":32386,"Ġinflated":32387,"Face":32388,"ĠFirearms":32389,"ĠEmmanuel":32390,"âĿ":32391,"Ġshocks":32392,"grab":32393,"Ġsplend":32394,"ĠHPV":32395,"abortion":32396,"Above":32397,"Entity":32398,"players":32399,"Ġcommenced":32400,"ulence":32401,"Ġfulfillment":32402,"Ġembodiments":32403,"ĠWelfare":32404,"Ġhail":32405,"Ġ<@":32406,"tten":32407,"Ġcatcher":32408,"ĠJazeera":32409,"Ġvolcano":32410,"Ġstabilize":32411,"ĠHandler":32412,"Ġintensified":32413,"ĠAbrams":32414,"Ġhumiliation":32415,"paced":32416,"605":32417,"ĠCentOS":32418,"Specific":32419,"Ġheed":32420,"ĠCAM":32421,"ĠGalile":32422,"Die":32423,"Ġabolished":32424,"ĠThomson":32425,"ĠTeachers":32426,"ĠWass":32427,"jong":32428,"ĠISBN":32429,"ĠAllies":32430,"shake":32431,"å·":32432,"vict":32433,"Howard":32434,"Ġdeem":32435,"Ġexceedingly":32436,"ĠSmartstocks":32437,"ibe":32438,"Ġdoorway":32439,"Ġcompeted":32440,"igmat":32441,"Ġnationalists":32442,"Ġgroom":32443,"ĠKeen":32444,"Ġdisposable":32445,"decl":32446,"ĠTolkien":32447,"ĠScheme":32448,"Ġbiod":32449,"Ġavid":32450,"ĠElon":32451,"agar":32452,"ĠTSA":32453,"Roman":32454,"Ġartificially":32455,"Ġadvisors":32456,"XL":32457,"ĠInferno":32458,"366":32459,"Ġtedious":32460,"ĠPhotography":32461,"ĠCarrie":32462,"Ġtrope":32463,"ĠSandra":32464,"Ġdecimal":32465,"Queen":32466,"ĠGundam":32467,"ĠOM":32468,"otech":32469,"NBA":32470,"Ġ1932":32471,"Ġentrenched":32472,"ĠMarion":32473,"Ġfraternity":32474,"Labour":32475,"Henry":32476,"Ġlatitude":32477,"Either":32478,"Ġenhances":32479,"ĠPotential":32480,"Ġshines":32481,"idad":32482,"Ġbreadth":32483,"Ġcapacities":32484,"ĠðŁĻĤ":32485,"ĠBronx":32486,"Ġsexes":32487,"Ġdifferentiation":32488,"Ġheavyweight":32489,"ĠTaj":32490,"dra":32491,"Ġmigrate":32492,"Ġexhaustion":32493,"ĠRUN":32494,"elsius":32495,"ĠCuomo":32496,"Ġguitars":32497,"Ġclones":32498,"ĠSomew":32499,"ĠPry":32500,"-------------":32501,"Ġwarranted":32502,"cycles":32503,"Ġsalvage":32504,"Ġdisks":32505,"RANT":32506,"ĠNGOs":32507,"ĠMartian":32508,"\":[{\"":32509,"Ġaddicts":32510,"ojure":32511,"illet":32512,"Ġamazingly":32513,"artments":32514,"pixel":32515,"ĠGPUs":32516,"Layout":32517,"è£":32518,"ĠTamil":32519,"ĠBasil":32520,"Ġimpartial":32521,"ĠStructure":32522,"fork":32523,"bryce":32524,"Ġridge":32525,"ĠHamburg":32526,"rious":32527,"Ġblitz":32528,"cigarettes":32529,"Ġcanned":32530,"402":32531,"Ġironically":32532,"Ġcompassionate":32533,"ĠHawkins":32534,".#":32535,"ĠCathedral":32536,"Ġrallied":32537,"internal":32538,"Ġquota":32539,"stakes":32540,"TEXT":32541,"mom":32542,"Ġcompletes":32543,"Ġ238":32544,"Ġshrug":32545,"ãĥij":32546,"ĠNinth":32547,"Ġrevise":32548,"ĠProvider":32549,"Ġtreacher":32550,"Ġquasi":32551,"ĠPRES":32552,"Ġdeposition":32553,"Ġconfidentiality":32554,"issors":32555,"Ġimbalance":32556,"Ġspanning":32557,"Ġangular":32558,"ĠCul":32559,"communication":32560,"ĠNora":32561,"ĠGenius":32562,"opter":32563,"Ġsacked":32564,"Spot":32565,"Ġfinely":32566,"ĠCHR":32567,"282":32568,"waves":32569,"Palest":32570,"ĠRohing":32571,"NL":32572,"è¿":32573,"Ġshitty":32574,"ĠScalia":32575,"475":32576,"Progress":32577,"Ġreferencing":32578,"Ġclassrooms":32579,"abee":32580,"Ġsod":32581,"hesion":32582,"708":32583,"ĠZuckerberg":32584,"ĠFinish":32585,"ĠScotia":32586,"ĠSavior":32587,"ĠInstallation":32588,"antha":32589,"(-":32590,"Ġ302":32591,"ĠPunk":32592,"Ġcrater":32593,"youtu":32594,"Ġroast":32595,"Ġinfluencing":32596,"Ġdup":32597,"ĠJR":32598,"ĠGrav":32599,"Ġstature":32600,"Ġbathrooms":32601,"Aside":32602,"Wiki":32603,"mean":32604,"ĠZak":32605,"ĠOnes":32606,"ĠNath":32607,"Ġhypert":32608,"Ġcommencement":32609,"Civil":32610,"Ġmoderately":32611,"Ġdistributors":32612,"Ġbreastfeeding":32613,"Ġ980":32614,"ĠSik":32615,"ĠCig":32616,"ĠAMER":32617,"RIP":32618,"ĠCareer":32619,"usting":32620,"Ġmessed":32621,"Ġeh":32622,"ĠJensen":32623,"/$":32624,"Ġblackmail":32625,"Ġconversions":32626,"Ġscientifically":32627,"Ġmantra":32628,"paying":32629,"Ġivory":32630,"ĠCourts":32631,"OUGH":32632,"auntlet":32633,"Serial":32634,"Brow":32635,"ĠHundreds":32636,"323":32637,"Ġpee":32638,"Ġlinux":32639,"Ġsubmer":32640,"ĠPrincipal":32641,"485":32642,"ĠDSL":32643,"ĠCousins":32644,"Ġdoctrines":32645,"ĠAthletics":32646,"Ġ315":32647,"ĠKarma":32648,"Ġattent":32649,"urger":32650,"Ġprescribe":32651,"Ġencaps":32652,"ĠCame":32653,"Ġsecretive":32654,"ĠCrimes":32655,"dn":32656,"Clean":32657,"ĠEgyptians":32658,"ĠCarpenter":32659,"Ġll":32660,"Hum":32661,"ĠMilo":32662,"Ġcapitalists":32663,"Ġbriefed":32664,"Twe":32665,"ĠBasin":32666,"elvet":32667,"Mos":32668,"Ġplunge":32669,"ĠKaiser":32670,"ĠFuj":32671,"illin":32672,"Ġsafeguards":32673,"Ġoste":32674,"ĠOpportunity":32675,"ĠMafia":32676,"ĠCalling":32677,"apa":32678,"urban":32679,"brush":32680,"illard":32681,"cé":32682,"intelligence":32683,"ĠLob":32684,"ĠDruid":32685,"Ġsmoother":32686,"Ġfooting":32687,"Ġmotorists":32688,"arcity":32689,"Ġmasculinity":32690,"Ġmism":32691,"Ġabdominal":32692,"ĠTavern":32693,"ĠRoh":32694,"Ġescapes":32695,"signed":32696,"Anthony":32697,"Ġsacrificing":32698,"Ġintimacy":32699,"Ġanterior":32700,"ĠKod":32701,"Ġmotif":32702,"Ġgraz":32703,"Ġvisualization":32704,"Ġguitarist":32705,"ĠTrotsky":32706,"magic":32707,"Dar":32708,"ĠMori":32709,"Ġwards":32710,"Ġtoilets":32711,"lest":32712,"Ġteleport":32713,"ĠSundays":32714,"ĠPlat":32715,"ETS":32716,"ĠeSports":32717,"Patrick":32718,"ĠKatherine":32719,"enko":32720,"Ġhassle":32721,"ĠMick":32722,"ggles":32723,"Ġhob":32724,"aintain":32725,"Ġairborne":32726,"Ġspans":32727,"Ġchili":32728,"Ġaperture":32729,"Ġvolunteered":32730,"ĠIncident":32731,"ĠFres":32732,"ĠVeteran":32733,"aughtered":32734,"ingo":32735,"Ġuninsured":32736,"CLOSE":32737,"Ġfuse":32738,"Ġerotic":32739,"Ġadvertise":32740,"raising":32741,"Texture":32742,"Ġattends":32743,"ĠREAL":32744,"uddled":32745,"Ġsmoot":32746,"Ġ305":32747,"ĠWillis":32748,"Ġblond":32749,"Analysis":32750,"ĠVT":32751,"onica":32752,"Ġstronghold":32753,"RF":32754,"NM":32755,".>>":32756,"Ġprosperous":32757,"Ġboasted":32758,"292":32759,"ĠManufacturing":32760,"PRESS":32761,"gren":32762,"Ġpharmacy":32763,"ĠRockefeller":32764,"kai":32765,"Ġthumbs":32766,"ĠHut":32767,"Ġmotherboard":32768,"Ġguardians":32769,"ĠAlter":32770,"llular":32771,"Ġshack":32772,"Ġwisely":32773,"Ġbackbone":32774,"erva":32775,"Ġsuicides":32776,"ĠMcGregor":32777,"ijah":32778,"Emer":32779,"ĠBrav":32780,"Ġdesignate":32781,"POST":32782,"produced":32783,"Ġcleansing":32784,"irlwind":32785,"existent":32786,"ĠHumph":32787,"ĠPayne":32788,"Ġvested":32789,"Å¡":32790,"Ġstringent":32791,"iona":32792,"Ġunsub":32793,"Ġsummed":32794,"ĠHercules":32795,"subject":32796,"ĠRagnar":32797,"ĠNos":32798,"Ġcharacterization":32799,"Ġsavvy":32800,"ĠDawson":32801,"ĠCasino":32802,"Ġfri":32803,"ĠBarrier":32804,"Ġmisinformation":32805,"Ġinsulation":32806,"Ġcorridors":32807,"Ġairplanes":32808,"ĠNoct":32809,"ahi":32810,"Ġ1916":32811,"kb":32812,"armac":32813,"Ġshun":32814,"Ġschema":32815,"Ġhorrified":32816,"Ġ239":32817,"aunders":32818,"NB":32819,"iates":32820,"erity":32821,"ĠShard":32822,"Ġrarity":32823,"Ġgrouped":32824,"ĠGhana":32825,"against":32826,"ĠBiological":32827,"ĠAware":32828,"owell":32829,"ÏĦ":32830,"ĠBeau":32831,"shaw":32832,"Hack":32833,"ĠJulius":32834,"USS":32835,"olson":32836,"auna":32837,"cru":32838,"ĠMaurice":32839,"ĠIk":32840,"Ġsequencing":32841,"Ġradicals":32842,"Ġ(?,":32843,"virtual":32844,"Ġanyways":32845,"Ġreperc":32846,"Ġhandlers":32847,"Ġhesitant":32848,"éĥ":32849,"ĠMF":32850,"plementation":32851,"associated":32852,"Ġcampaigned":32853,"ĠYue":32854,"utations":32855,"ĠYoga":32856,"Ġsimmer":32857,"Ġrods":32858,"Ġmelody":32859,"Ġconvoy":32860,"videos":32861,"Ġscreened":32862,"Neg":32863,"ochemical":32864,"Ġ())":32865,"Ġultras":32866,"Ġantip":32867,"ĠIslanders":32868,"704":32869,"Ġfetish":32870,"Ġridiculously":32871,"ĠKart":32872,"Ġmitochondrial":32873,"Ġinterfering":32874,"Builder":32875,"Ġoverfl":32876,"Ġacne":32877,"ĠMud":32878,"ĠKerr":32879,"flex":32880,"ĠPostal":32881,"ĠBaltic":32882,"477":32883,"ĠPersons":32884,"ourage":32885,"HB":32886,"ĠMuse":32887,"ĠImmortal":32888,"ĠDriving":32889,"Ġpetitions":32890,"Ġsubscript":32891,"Ġsorce":32892,"ĠProcessor":32893,"uton":32894,"Sony":32895,"Ġphon":32896,"Ġraced":32897,"ĠAnthrop":32898,"Ġdaytime":32899,"ĠExercise":32900,"Adding":32901,"Ġengages":32902,"ĠQualcomm":32903,"Ġmiracles":32904,"Ġmemes":32905,"ĠDrink":32906,"ĠOrioles":32907,"Ġhairs":32908,"ĠPolar":32909,"athom":32910,"Ġslippery":32911,"ĠRemy":32912,"Ġcaramel":32913,"ĠYEAR":32914,"Ġalk":32915,"Ign":32916,"aution":32917,"ĠMerlin":32918,"ĠCran":32919,"Ġapologies":32920,"Ġ410":32921,"Ġouting":32922,"ĠMemories":32923,"appointed":32924,"Ġcountered":32925,"uld":32926,"posing":32927,"Ġfirewall":32928,"ĠWast":32929,"ĠWet":32930,"worked":32931,"seller":32932,"Ġrepealed":32933,"ereo":32934,"assuming":32935,"BLIC":32936,"mite":32937,"ĠCEOs":32938,"ĠChapel":32939,"elligent":32940,"________________________":32941,"Dog":32942,"Ġwart":32943,"Ġsubscriber":32944,"sports":32945,"Ġbegged":32946,"ĠMV":32947,"Ġsemif":32948,"ethical":32949,"Ġpreach":32950,"Ġrevital":32951,"Ġpunitive":32952,"Ġshortcuts":32953,"Ġinstituted":32954,"ĠWarsaw":32955,"Ġabdomen":32956,"ĠKING":32957,"Ġsuperintendent":32958,"Ġfry":32959,"ĠGeo":32960,"TOR":32961,"Ġcontradictions":32962,"aptic":32963,"Ġlandscapes":32964,"bugs":32965,"Ġclust":32966,"Ġvolley":32967,"cribed":32968,"Ġtandem":32969,"Ġrobes":32970,"WHAT":32971,"Ġpromoter":32972,"Ġeloqu":32973,"reviewed":32974,"ĠDK":32975,"ĠPlato":32976,"Ġfps":32977,"Tank":32978,"ĠDerrick":32979,"Ġprioritize":32980,"asper":32981,"ĠHonduras":32982,"ĠCompleted":32983,"nec":32984,"Ġmog":32985,"nir":32986,"ĠMayo":32987,"DEF":32988,"stall":32989,"inness":32990,"ĠVolkswagen":32991,"Ġprecaution":32992,"ĠMell":32993,"iak":32994,"istries":32995,"Ġ248":32996,"Ġoverlapping":32997,"Senate":32998,"ĠEnhance":32999,"resy":33000,"racial":33001,"ORTS":33002,"ĠMormons":33003,"Strong":33004,"ĠCoch":33005,"Mexico":33006,"ĠMaduro":33007,"Ġjars":33008,"Ġcane":33009,"Wik":33010,"olla":33011,"ifference":33012,"Ġphysicist":33013,"ĠMaggie":33014,"Ġ285":33015,"Ġdepiction":33016,"ĠMcLaren":33017,"Ju":33018,"Ġslows":33019,"Ġcommissioners":33020,"ĠWillow":33021,"ĠExplos":33022,"hovah":33023,"Ġtechnician":33024,"Ġhomicides":33025,"ĠFlav":33026,"ĠTruman":33027,"Ġ10000":33028,"uctor":33029,"Ġshader":33030,"Newsletter":33031,"457":33032,"Ġrever":33033,"Ġhardened":33034,"Ġwhereabouts":33035,"Ġredevelop":33036,"Ġcarbs":33037,"Ġtravers":33038,"Ġsquirrel":33039,"Ġfollower":33040,"Ġsings":33041,"508":33042,"Ġrabbits":33043,"emonium":33044,"Ġdocumenting":33045,"Ġmisunderstood":33046,")'":33047,"Rick":33048,"ggies":33049,"Ġpremie":33050,"Ġskating":33051,"Ġpassports":33052,"Ġfists":33053,"ageddon":33054,"Haw":33055,"ACP":33056,"080":33057,"ĠThoughts":33058,"ĠCarlson":33059,"Ġpriesthood":33060,"hua":33061,"Ġdungeons":33062,"ĠLoans":33063,"Ġantis":33064,"Ġfamiliarity":33065,"ĠSabb":33066,"opal":33067,"ĠInk":33068,"strike":33069,"Ġcram":33070,"Ġlegalized":33071,"Ġcuisine":33072,"Ġfibre":33073,"Travel":33074,"ĠMonument":33075,"ODY":33076,"ethy":33077,"Ġinterstate":33078,"ĠPUR":33079,"emporary":33080,"ĠArabian":33081,"developed":33082,"Ġsaddle":33083,"Ġgithub":33084,"ĠOffer":33085,"ĠISP":33086,"rolet":33087,"ĠSUPER":33088,"ĠDenis":33089,"Ġmultiplier":33090,"Ġstirred":33091,"Interestingly":33092,"Ġcustomary":33093,"Ġbilled":33094,"hex":33095,"Ġmultiplied":33096,"Ġflipping":33097,"ĠCrosby":33098,"Ġfundamentals":33099,"iae":33100,"ĠPlayed":33101,"ĠAtom":33102,"amazon":33103,"ĠFlam":33104,"eez":33105,"activated":33106,"Ġtablespoon":33107,"Ġliberalism":33108,"ĠPalin":33109,"ĠPatel":33110,"Num":33111,"ĠTAM":33112,"Ġsurn":33113,"ĠReloaded":33114,"Ġcoined":33115,"\"],":33116,"ĠClash":33117,"ĠAgu":33118,"Ġpragmatic":33119,"ĠActivate":33120,"Ġ802":33121,"Ġtrailers":33122,"Ġsilhou":33123,"Ġprobes":33124,"Ġcircus":33125,"ĠBain":33126,"ĠLindsay":33127,"ĠAbbey":33128,"Delivery":33129,"Ġconcession":33130,"Ġgastro":33131,"ĠSprite":33132,"ÄŁ":33133,"andel":33134,"Ġgimm":33135,"Ġautobi":33136,"ĠTurtle":33137,"Ġwonderfully":33138,"ĠHaram":33139,"ĠWorldwide":33140,"ĠHandle":33141,"Ġtheorists":33142,"Ġsleek":33143,"ĠZhu":33144,"ographically":33145,"EGA":33146,"ĠOwners":33147,"aths":33148,"ĠAntarctic":33149,"natal":33150,"=\"\"":33151,"flags":33152,"````":33153,"Ġsul":33154,"Kh":33155,"Ġpotassium":33156,"Ġlineman":33157,"Ġcereal":33158,"ĠSeasons":33159,"Ġ2022":33160,"Ġmathematic":33161,"Ġastronomers":33162,"professional":33163,"Ġfares":33164,"cknowled":33165,"Ġchi":33166,"Ġyoungsters":33167,"Ġmistakenly":33168,"Ġhemisphere":33169,"ĠDivinity":33170,"rone":33171,"Ġ\",":33172,"rings":33173,"Ġattracts":33174,"vana":33175,"å¹":33176,"CAP":33177,"Ġplaylist":33178,"Ġporch":33179,"ãģ£":33180,"Ġincorporates":33181,"Ġsoak":33182,"Ġasserting":33183,"ĠTerrorism":33184,"ĠPablo":33185,"Ja":33186,"cester":33187,"Ġfearing":33188,"ĠPrayer":33189,"Ġescalated":33190,"GW":33191,"Ġrobe":33192,"ĠBrighton":33193,"acists":33194,"ĠSymphony":33195,"ĠDwarf":33196,"ĠParade":33197,"ĠLego":33198,"Ġinexpl":33199,"Ġlords":33200,"leaf":33201,"RAG":33202,"liber":33203,"Ġcigars":33204,"ĠJehovah":33205,"606":33206,"WINDOWS":33207,"ĠLiberia":33208,"ebus":33209,"Heavy":33210,"Ġlubric":33211,"ĠRW":33212,"anguages":33213,"Ġnarrowed":33214,"computer":33215,"ĠEmber":33216,"Ġmurdering":33217,"Ġdownstream":33218,"ĠTuls":33219,"ĠTables":33220,"Topic":33221,"ĠAccuracy":33222,"=/":33223,"lost":33224,"ĠRei":33225,"Ġprogresses":33226,"bear":33227,"Ġestablishments":33228,"Justin":33229,"ĠPeach":33230,"ĠGomez":33231,"å¿":33232,"ĠTriangle":33233,"Ident":33234,"ĠHive":33235,"Resources":33236,"Ġmixes":33237,"ĠAssuming":33238,"Mu":33239,"Ġhypoc":33240,"Ġsane":33241,"ĠWan":33242,"idious":33243,"Success":33244,"Ġio":33245,"Angel":33246,"Ġdangerously":33247,"ĠCreature":33248,"WORK":33249,":[":33250,"ĠKatrina":33251,"Listener":33252,"Miller":33253,"ĠIdlib":33254,"hang":33255,"Ġcircumvent":33256,"href":33257,"Ġcelestial":33258,"ĠWeeks":33259,"ĠPug":33260,"ĠDalton":33261,"Ġsubpoena":33262,"uku":33263,"Ġpersisted":33264,"pei":33265,"olding":33266,"ĠDocuments":33267,"ĠHast":33268,"ĠCENT":33269,"Ġprimer":33270,"Ġsynonymous":33271,"Ġnib":33272,"ombs":33273,"Ġnotation":33274,"ĠDish":33275,"ĠAtmosp":33276,"Ġforbid":33277,"ĠANG":33278,"pattern":33279,"los":33280,"Ġprojectiles":33281,"brown":33282,".\",":33283,"ĠVenom":33284,"Ġfiercely":33285,"ublished":33286,"ĠUran":33287,"ĠNicarag":33288,"410":33289,"ĠCAL":33290,"OTOS":33291,"ĠMiracle":33292,"ĠEnchant":33293,"Ġguarding":33294,"append":33295,"Attach":33296,"Ġleveled":33297,"Ġcondoms":33298,"ihilation":33299,"649":33300,"Ġnightmares":33301,"ĠTHEY":33302,"ĠSTART":33303,"ĠKinn":33304,"Ġroommate":33305,"Ġhygiene":33306,"opping":33307,"Job":33308,"Ġlvl":33309,"ĠVER":33310,"ĠKeeping":33311,"abetic":33312,"Ġformatting":33313,"erala":33314,"Ġrevisions":33315,"Ġresurg":33316,"Tel":33317,"ĠGoodman":33318,"353":33319,"pod":33320,"Ġindisp":33321,"ĠTranslation":33322,"Ġgown":33323,"ĠMund":33324,"Ġcis":33325,"Ġbystand":33326,"collect":33327,"ĠPunjab":33328,"actively":33329,"ĠGamb":33330,"tell":33331,"Ġimporting":33332,"gencies":33333,"Ġlocom":33334,"ĠBrill":33335,"Holy":33336,"ĠBerger":33337,"Ġshowdown":33338,"Ġresponders":33339,"ILY":33340,"Ġtakedown":33341,"leted":33342,"Ġmattered":33343,"Ġpredictive":33344,"Ġoverlay":33345,"GPU":33346,"ĠVick":33347,"Ġconveyed":33348,"Tab":33349,"peer":33350,"Scan":33351,"Ġdefensively":33352,"vae":33353,"Ġapproving":33354,"Ġtiers":33355,"ĠVia":33356,"querade":33357,"ĠSaudis":33358,"Ġdemolished":33359,"ĠProphe":33360,"Ġmono":33361,"Ġhospitality":33362,"HAM":33363,"ĠAriel":33364,"MOD":33365,"ĠTorah":33366,"Ġblah":33367,"ĠBelarus":33368,"erential":33369,"ĠTuc":33370,"Ġbanker":33371,"397":33372,"Ġmosquit":33373,"ĠScientist":33374,"ĠMusical":33375,"Ġhust":33376,"Shift":33377,"Ġtorment":33378,"Ġstandoff":33379,"Educ":33380,"ĠFog":33381,"Ġamplifier":33382,"Shape":33383,"Instance":33384,"ĠCritics":33385,"Ġdaemon":33386,"Houston":33387,"Ġmattress":33388,"ĠIDF":33389,"Ġobscene":33390,"ĠAmer":33391,"hetti":33392,"Ġcompiling":33393,"352":33394,"verett":33395,"ĠReduction":33396,"istration":33397,"ĠBlessed":33398,"ĠBachelor":33399,"316":33400,"Ġprank":33401,"ĠVulcan":33402,"dding":33403,"Ġmourning":33404,"ĠQuint":33405,"ĠBlaster":33406,"testing":33407,"Ġsediment":33408,">>>":33409,"ĠEternity":33410,"ĠWHERE":33411,"ĠMaze":33412,"Ġreacting":33413,"ĠAlv":33414,"omsday":33415,"ĠCRA":33416,"Ġtranslator":33417,"Ġbogus":33418,"atu":33419,"Website":33420,"olls":33421,"Ġbaptism":33422,"Ġsibling":33423,"ĠAutumn":33424,"vez":33425,"ãģ®é":33426,"guards":33427,"Georg":33428,"assadors":33429,"ĠFreud":33430,"Ġcontinents":33431,"ĠRegistry":33432,"Bernie":33433,"ĸļ士":33434,"Ġtolerant":33435,"ĠUW":33436,"Ġhorribly":33437,"995":33438,"ĠMIDI":33439,"Ġimpatient":33440,"ocado":33441,"eri":33442,"ĠWorst":33443,"ĠNorris":33444,"ĠTalking":33445,"Ġdefends":33446,"ensable":33447,"Ġ2021":33448,"Ġanatomy":33449,"Lew":33450,"Ġdrawer":33451,"ĠCanberra":33452,"Ġpatriotic":33453,"é¾įåĸļ士":33454,"ĠAvg":33455,"ARM":33456,"Ġundisclosed":33457,"Ġfarewell":33458,"459":33459,"bable":33460,"ĠAllison":33461,"OLOG":33462,"Ġconco":33463,"tight":33464,"ĠACPI":33465,"ĠMines":33466,"lich":33467,"ĠâĶľ":33468,"represented":33469,"200000":33470,"Ġenthusiast":33471,"OTS":33472,"bil":33473,"ĠIngredients":33474,"Ġinventor":33475,"ĠMySQL":33476,"³³³":33477,"ĠABOUT":33478,"within":33479,"Ġmk":33480,"Bul":33481,"ĠFake":33482,"Ġdraconian":33483,"Wa":33484,"helm":33485,"ĠTerran":33486,"erville":33487,"Ġcommonplace":33488,"SIZE":33489,"Ġ\"<":33490,"replace":33491,"ographs":33492,"ĠSELECT":33493,"incible":33494,"ĠMostly":33495,"ĠSheffield":33496,"ĠIDE":33497,"uggle":33498,"Ġcitations":33499,"hurst":33500,"ĠUnix":33501,"Ġunleash":33502,"ĠPiper":33503,"ĠNano":33504,"Ġsuccumb":33505,"Ġreluctance":33506,"Ġ2500":33507,"ĠMerchant":33508,"Ġwiret":33509,"Ġcombos":33510,"ĠBirthday":33511,"Ġcharcoal":33512,"ĠUPS":33513,"ĠFairfax":33514,"Ġdriveway":33515,"ĠTek":33516,"ĠPitch":33517,"overe":33518,"Ġtechnicians":33519,"ĠActual":33520,"flation":33521,"ĠFiscal":33522,"ĠEmpty":33523,"anamo":33524,"Ġmagnesium":33525,"Ġslut":33526,"Ġgrowers":33527,"Investigators":33528,"():":33529,"ĠSatellite":33530,"ĠKeynes":33531,"missive":33532,"lane":33533,"Ġborough":33534,"344":33535,"ĠTEAM":33536,"ĠBethesda":33537,"CV":33538,"hower":33539,"ĠRAD":33540,"Ġchant":33541,"ĠRiy":33542,"Ġcompositions":33543,"Ġmildly":33544,"Ġmeddling":33545,"Ġagility":33546,"aneers":33547,"501":33548,"Ġsynth":33549,"linger":33550,"291":33551,"Ġexclaimed":33552,"Party":33553,"Ġcontamin":33554,"ĠManor":33555,"ĠRespond":33556,"Ġpraising":33557,"Ġmanners":33558,"fleet":33559,"Summer":33560,"ĠLynd":33561,"ĠDefinitely":33562,"grim":33563,"Ġbowling":33564,"stri":33565,"çĽ":33566,"ynt":33567,"Ġmandates":33568,"DIV":33569,"Ġreconcile":33570,"views":33571,"ĠDamon":33572,"vette":33573,"Flo":33574,"ĠGreatest":33575,"ilon":33576,"icia":33577,"Ġportrayal":33578,"Ġcushion":33579,"504":33580,"1979":33581,"ossal":33582,"Applic":33583,"scription":33584,"Ġmitigation":33585,"ATS":33586,"pac":33587,"Ġerased":33588,"Ġdeficiencies":33589,"ĠHollande":33590,"ĠXu":33591,"Ġbred":33592,"Ġpregnancies":33593,"femin":33594,"Ġemph":33595,"Ġplanners":33596,"Ġoutper":33597,"uttering":33598,"Ġperpetrator":33599,"Ġmotto":33600,"ĠEllison":33601,"ĠNEVER":33602,"Ġadmittedly":33603,"ARI":33604,"ĠAzerbaijan":33605,"Ġmillisec":33606,"Ġcombustion":33607,"ĠBottle":33608,"ĠLund":33609,"ĠPs":33610,"ĠDress":33611,"Ġfabricated":33612,"Ġbattered":33613,"Ġsidel":33614,"ĠNotting":33615,"Foreign":33616,"ĠJerome":33617,"020":33618,"ĠArbit":33619,"Ġknots":33620,"ĠRIGHT":33621,"Moving":33622,"ãģĻ":33623,"Ġsurgeries":33624,"Ġcourthouse":33625,"Ġmastered":33626,"Ġhovering":33627,"ĠBran":33628,"ĠAlison":33629,"Ġsafest":33630,"military":33631,"Ġbullied":33632,"Ġbarrage":33633,"Reader":33634,"ESE":33635,"ĠGeographic":33636,"Tools":33637,"314":33638,"ĠGeek":33639,"roth":33640,"glers":33641,"ĠFIN":33642,"Ïģ":33643,"ĠAston":33644,"altern":33645,"488":33646,"Ġveterin":33647,"Gamer":33648,"Ġintel":33649,"renches":33650,"Shield":33651,"Ġamnesty":33652,"ĠBhar":33653,"Ġpiled":33654,"Ġhonorable":33655,"ĠInstitutes":33656,"Ġsoaked":33657,"Ġcoma":33658,"ĠEFF":33659,"341":33660,"bytes":33661,"ĠGmail":33662,"lein":33663,"ĠCanadiens":33664,"material":33665,"Il":33666,"Ġinstructors":33667,"ĠKY":33668,"Ġconceive":33669,"ubb":33670,"ĠPossible":33671,"Ġeasing":33672,"ĠChristina":33673,"Ġcaric":33674,"ĠHDR":33675,"ROM":33676,"Ġshovel":33677,"delete":33678,"Ġpuff":33679,"ĠChanging":33680,"Ġseamlessly":33681,"Attribute":33682,"Ġacquisitions":33683,"akery":33684,"ĠEF":33685,"Ġautistic":33686,"ĠTakes":33687,"ĠPowder":33688,"ĠStir":33689,"510":33690,"ĠBubble":33691,"settings":33692,"ĠFowler":33693,"Ġmustard":33694,"Ġmoreover":33695,"Ġcopyrighted":33696,"ĠLEDs":33697,"1500":33698,"æī":33699,"ĠHIS":33700,"enf":33701,"Ġcustod":33702,"ĠHuck":33703,"Gi":33704,"Ġimg":33705,"Answer":33706,"Ct":33707,"jay":33708,"ĠInfrastructure":33709,"Ġfederally":33710,"Loc":33711,"Ġmicrobes":33712,"Ġoverrun":33713,"dds":33714,"otent":33715,"adiator":33716,">>>>>>>>":33717,"Ġtornado":33718,"Ġadjud":33719,"Ġintrigued":33720,"Ġsi":33721,"ĠRevelation":33722,"progress":33723,"Ġburglary":33724,"ĠSaiyan":33725,"ĠKathy":33726,"Ġserpent":33727,"ĠAndreas":33728,"Ġcompel":33729,"essler":33730,"ĠPlastic":33731,"ĠAdvent":33732,"ĠPositive":33733,"ĠQt":33734,"ĠHindus":33735,"registered":33736,"ularity":33737,"Ġrighteousness":33738,"Ġdemonic":33739,"uitive":33740,"ĠBDS":33741,"ĠGregg":33742,"cia":33743,"ĠCrusade":33744,"ĠSinai":33745,"WARE":33746,"+(":33747,"Ġmell":33748,"Ġderail":33749,"yards":33750,"Ast":33751,"Ġnoticeably":33752,"ĠOber":33753,"Ram":33754,"Ġunnoticed":33755,"Ġseq":33756,"avage":33757,"Ts":33758,"Ġ640":33759,"Ġconcede":33760,"Ġ])":33761,"Fill":33762,"Ġcaptivity":33763,"ĠImprovement":33764,"ĠCrusader":33765,"araoh":33766,"MAP":33767,"æĹ":33768,"Ġstride":33769,"always":33770,"Fly":33771,"Nit":33772,"Ġalgae":33773,"ĠCooking":33774,"ĠDoors":33775,"Malley":33776,"Ġpolicemen":33777,"ãģį":33778,"Ġastronaut":33779,"accessible":33780,"495":33781,"ĠRAW":33782,"cliffe":33783,"udicrous":33784,"Ġdepended":33785,"alach":33786,"Ġventures":33787,"rake":33788,"Ġtits":33789,"ĠHou":33790,"Ġcondom":33791,"ormonal":33792,"Ġindent":33793,"Ġuploading":33794,"Footnote":33795,"Important":33796,"Ġ271":33797,"Ġmindful":33798,"Ġcontends":33799,"Cra":33800,"Ġcalibr":33801,"ĠOECD":33802,"plugin":33803,"Fat":33804,"ĠISS":33805,"ĠDynamics":33806,"ansen":33807,"686":33808,"'),":33809,"Ġsprite":33810,"Ġhandheld":33811,"ĠHipp":33812,"=~=~":33813,"Trust":33814,"Ġsemantics":33815,"ĠBundes":33816,"ĠReno":33817,"ĠLiterature":33818,"sense":33819,"Gary":33820,"ĠAeg":33821,"ĠTrin":33822,"EEK":33823,"Ġcleric":33824,"ĠSSH":33825,"Ġchrist":33826,"Ġinvading":33827,"ibu":33828,"Ġenum":33829,"aura":33830,"Ġallege":33831,"ĠIncredible":33832,"BBC":33833,"Ġthru":33834,"Ġsailed":33835,"Ġemulate":33836,"Ġinsecurity":33837,"Ġcrou":33838,"Ġaccommodations":33839,"Ġincompetent":33840,"Ġslips":33841,"ĠEarthqu":33842,"sama":33843,"ILLE":33844,"ĠiPhones":33845,"asaki":33846,"Ġbye":33847,"Ġard":33848,"Ġextras":33849,"Ġslaughtered":33850,"Ġcrowdfunding":33851,"resso":33852,"Ġfilib":33853,"ĠERROR":33854,"ĠTLS":33855,"egg":33856,"ĠItal":33857,"Ġenlist":33858,"ĠCatalonia":33859,"ĠScots":33860,"Ġsergeant":33861,"Ġdissolve":33862,"NH":33863,"Ġstandings":33864,"rique":33865,"IQ":33866,"Ġbeneficiary":33867,"Ġaquarium":33868,"YouTube":33869,"ĠPowerShell":33870,"Ġbrightest":33871,"ĠWarrant":33872,"Sold":33873,"Writing":33874,"Ġbeginnings":33875,"ĠReserved":33876,"ĠLatinos":33877,"heading":33878,"Ġ440":33879,"Ġrooftop":33880,"ATING":33881,"Ġ390":33882,"VPN":33883,"Gs":33884,"kernel":33885,"turned":33886,"Ġpreferable":33887,"Ġturnovers":33888,"ĠHels":33889,"Sa":33890,"ĠShinji":33891,"veh":33892,"ĠMODULE":33893,"Viol":33894,"Ġexiting":33895,"Ġjab":33896,"ĠVanilla":33897,"Ġacron":33898,"ĠGap":33899,"bern":33900,"Ak":33901,"ĠMcGu":33902,"Ġendlessly":33903,"ĠFarage":33904,"ĠNoel":33905,"Va":33906,"MK":33907,"Ġbrute":33908,"ĠKru":33909,"ĠESV":33910,"ĠOlivia":33911,"âĢł":33912,"ĠKaf":33913,"Ġtrusting":33914,"Ġhots":33915,"324":33916,"Ġmalaria":33917,"Ġjson":33918,"Ġpounding":33919,"ortment":33920,"Country":33921,"Ġpostponed":33922,"Ġunequiv":33923,"?),":33924,"ĠRooney":33925,"udding":33926,"ĠLeap":33927,"urrence":33928,"shapeshifter":33929,"ĠHAS":33930,"osate":33931,"Ġcavern":33932,"Ġconservatism":33933,"ĠBAD":33934,"Ġmileage":33935,"Ġarresting":33936,"Vaults":33937,"Ġmixer":33938,"Democratic":33939,"ĠBenson":33940,"Ġauthored":33941,"8000":33942,"Ġproactive":33943,"ĠSpiritual":33944,"tre":33945,"Ġincarcerated":33946,"ĠSort":33947,"Ġpeaked":33948,"Ġwielding":33949,"reciation":33950,"×Ļ×":33951,"Patch":33952,"ĠEmmy":33953,"Ġexqu":33954,"tto":33955,"ĠRatio":33956,"ĠPicks":33957,"ĠGry":33958,"phant":33959,"Ġfret":33960,"Ġethn":33961,"Ġarchived":33962,"%-":33963,"cases":33964,"ĠBlaze":33965,"Ġimb":33966,"cv":33967,"yss":33968,"imony":33969,"Ġcountdown":33970,"Ġawakening":33971,"ĠTunisia":33972,"ĠRefer":33973,"ĠMJ":33974,"Ġunnatural":33975,"ĠCarnegie":33976,"izen":33977,"ĠNuggets":33978,"hess":33979,"Ġevils":33980,"647":33981,"Ġintroductory":33982,"loving":33983,"ĠMcMahon":33984,"Ġambiguity":33985,"Label":33986,"ĠAlmighty":33987,"Ġcoloring":33988,"ĠClaus":33989,"setting":33990,"NULL":33991,"ĠFavorite":33992,"ĠSIG":33993,">(":33994,"ĠShiva":33995,"ĠMayer":33996,"Ġstormed":33997,"ĠCoverage":33998,"weapons":33999,"igham":34000,"Ġunanswered":34001,"Ġleve":34002,"Ġcoy":34003,"cas":34004,"bags":34005,"asured":34006,"Seattle":34007,"ĠSantorum":34008,"serious":34009,"Ġcourageous":34010,"ĠSoup":34011,"Ġconfiscated":34012,"Ġ///":34013,"Ġunconventional":34014,"Ġmoms":34015,"ĠRohingya":34016,"ĠOrchestra":34017,"ĠPotion":34018,"Ġdiscredit":34019,"ĠFIL":34020,"fixed":34021,"ĠDeer":34022,"doi":34023,"ĠDimension":34024,"Ġbureaucrats":34025,"eteen":34026,"ĠactionGroup":34027,"ohm":34028,"Ġbumps":34029,"ĠUtility":34030,"Ġsubmarines":34031,"renheit":34032,"research":34033,"ĠShapiro":34034,"Ġsketches":34035,"Ġdeceptive":34036,"ĠVil":34037,"esame":34038,"ĠEssentially":34039,"Ġrampage":34040,"isky":34041,"Ġmuttered":34042,"thritis":34043,"Ġ236":34044,"fet":34045,"bars":34046,"Ġpupil":34047,"ĠThou":34048,"oS":34049,"song":34050,"Ġfractured":34051,"Ġrevert":34052,"picture":34053,"Ġcriterion":34054,"usher":34055,"Ġrepercussions":34056,"ĠVintage":34057,"ĠSuperintendent":34058,"Officers":34059,"Ġflagged":34060,"Ġblames":34061,"Ġinverse":34062,"ographers":34063,"Ġmakeshift":34064,"Ġdevoid":34065,"Ġfossils":34066,"ĠAristotle":34067,"ĠFunds":34068,"Ġdepleted":34069,"ĠFlu":34070,"ĠYuan":34071,"Ġwoes":34072,"Ġlipid":34073,"Ġsitu":34074,"requisites":34075,"Ġfurnish":34076,"ĠSamar":34077,"Ġshameful":34078,"Ġadversely":34079,"Ġadept":34080,"Ġremorse":34081,"Ġmurderous":34082,"uckles":34083,"ĠESL":34084,"Ġ314":34085,"sent":34086,"Ġredef":34087,"ĠCache":34088,"ĠPurs":34089,"igans":34090,"Ġ460":34091,"Ġprescriptions":34092,"Ġfres":34093,"Fuck":34094,"ocrates":34095,"Twenty":34096,"ĠWeird":34097,"ĠToggle":34098,"ĠCalled":34099,"itizens":34100,"Ġpoultry":34101,"Ġharvesting":34102,"ãĤ¦ãĤ¹":34103,"Bottom":34104,"Ġcautioned":34105,"tn":34106,"396":34107,"ĠNikki":34108,"Ġevaluations":34109,"Ġharassing":34110,"Ġbindings":34111,"ĠMonetary":34112,"Ġhitters":34113,"Ġadversary":34114,"unts":34115,"Ġsetback":34116,"Ġencrypt":34117,"ĠCait":34118,"Ġlows":34119,"enges":34120,"ĠNorn":34121,"Ġbulbs":34122,"Ġbottled":34123,"ĠVoyager":34124,"317":34125,"Ġspheres":34126,"politics":34127,"Ġsubtract":34128,"Ġsensations":34129,"Ġappalling":34130,"Ġ316":34131,"Ġenvironmentally":34132,"ĠSTEM":34133,"Ġpublishes":34134,"560":34135,"Ġdiligence":34136,"484":34137,"Ġadvises":34138,"Ġpetrol":34139,"Ġimagining":34140,"Ġpatrols":34141,"ĠInteger":34142,"ĠAshes":34143,"actus":34144,"ĠRadiant":34145,"ĠLT":34146,"itability":34147,"htaking":34148,"Setting":34149,"Ġnuanced":34150,"ĠReef":34151,"ĠDevelopers":34152,"Ni":34153,"pieces":34154,"990":34155,"License":34156,"Ġlowers":34157,"ĠOttoman":34158,"327":34159,"ooo":34160,"Ġquitting":34161,"markets":34162,"Behind":34163,"Ġbasin":34164,"Ġdocs":34165,"anie":34166,"flash":34167,"ctl":34168,"Ġcivilized":34169,"ĠFukushima":34170,"\"],\"":34171,"ĠKS":34172,"ĠHonestly":34173,"arat":34174,"Ġconstructs":34175,"ĠLans":34176,"ĠDire":34177,"ĠLIKE":34178,"ĠTrouble":34179,"Ġwithholding":34180,"ĠOblivion":34181,"Ġsanity":34182,"anya":34183,"Const":34184,"Ġgrocer":34185,"ĠCelsius":34186,"Ġrecounted":34187,"ĠWife":34188,"Border":34189,"atered":34190,"happy":34191,"Ġspoiler":34192,"Ġlogically":34193,"Hall":34194,"Ġsucceeding":34195,"Ġpolymorph":34196,"Ġaxes":34197,"ĠShotgun":34198,"ĠSlim":34199,"ĠPrinciples":34200,"ĠLeth":34201,"arta":34202,"Ġscor":34203,"Screenshot":34204,"Ġrelaxation":34205,"#$#$":34206,"Ġdeterrent":34207,"iddy":34208,"Ġpowerless":34209,"Ġlesbians":34210,"Ġchords":34211,"ĠEdited":34212,"selected":34213,"Ġseparatists":34214,"0002":34215,"Ġairspace":34216,"Ġturnaround":34217,"Ġcunning":34218,"PATH":34219,"Poly":34220,"Ġbombed":34221,"Ġtion":34222,"xs":34223,"Ġwithhold":34224,"Ġwaged":34225,"ĠLiberties":34226,"Flag":34227,"Ġcomforting":34228,"454":34229,"ĠIris":34230,"arers":34231,"Ġrag":34232,"Ġrelocated":34233,"ĠGuarant":34234,"Ġstrategically":34235,"Ġgamma":34236,"uberty":34237,"ĠLockheed":34238,"gres":34239,"Ġgrilled":34240,"ĠLowe":34241,"stats":34242,"ĠRocks":34243,"Ġsensing":34244,"Ġrenting":34245,"ĠGeological":34246,"اØ":34247,"otrop":34248,"Ġsew":34249,"Ġimproperly":34250,"486":34251,"Ġâĸł":34252,"Ġstarving":34253,"ĠBj":34254,"Discussion":34255,"328":34256,"ĠCombo":34257,"ĠFixes":34258,"NAT":34259,"Ġstriving":34260,"thora":34261,"Ġharvested":34262,"ĠPing":34263,"Ġplayful":34264,"Ġavenues":34265,"Ġoccupational":34266,"Ġwakes":34267,"ĠCourier":34268,"Ġdrummer":34269,"ĠBrowser":34270,"ĠHouth":34271,"itu":34272,"Ġapparel":34273,"paste":34274,"Ġhunted":34275,"ĠSecondly":34276,"lain":34277,"XY":34278,"ĠPIN":34279,"icons":34280,"Ġcocktails":34281,"Ġsizable":34282,"Ġhurdles":34283,"estinal":34284,"ĠRecreation":34285,"Ġeco":34286,"648":34287,"ĠDied":34288,"mint":34289,"Ġfingerprints":34290,"Ġdispose":34291,"ĠBosnia":34292,"tsy":34293,"2200":34294,"Ġinspected":34295,"ĠFou":34296,"Ġfuss":34297,"Ġambush":34298,"ĠRak":34299,"Ġmanifested":34300,"Prosecut":34301,"Ġsuffice":34302,"rences":34303,"Ġcompensated":34304,"ĠCyrus":34305,"Ġgenus":34306,"ĠWolverine":34307,"ĠTrends":34308,"Ġhikes":34309,"ĠSeen":34310,"Ġenrol":34311,"Cold":34312,"Ġpolitely":34313,"ĠSlav":34314,"ĠRupert":34315,"Ġeyewitness":34316,"ĠAlto":34317,"Ġuncomp":34318,"Ġposterior":34319,"Must":34320,"ĠHerz":34321,"Ġprogressively":34322,"Ġ234":34323,"Ġindifference":34324,"ĠCunningham":34325,"Ġacademia":34326,"Ġsewer":34327,"Ġastounding":34328,"ĠAES":34329,"rather":34330,"Ġeldest":34331,"Ġclimbs":34332,"ĠAdds":34333,"Ġoutcry":34334,"Ġcontag":34335,"ĠHouses":34336,"Ġpept":34337,"ĠMelania":34338,"interested":34339,"ĠUCH":34340,"ĠRoots":34341,"ĠHubbard":34342,"ĠTBD":34343,"ĠRomanian":34344,"filename":34345,"Stone":34346,"ĠImpl":34347,"Ġchromosome":34348,"Cle":34349,"dx":34350,"Ġscrambled":34351,"ĠPt":34352,"Ġ242":34353,"OPLE":34354,"Ġtremendously":34355,"Street":34356,"Ġcraving":34357,"Ġbundled":34358,"ĠRG":34359,"pipe":34360,"Ġinjuring":34361,"Ġarcane":34362,"Particip":34363,"ĠHeroic":34364,"sty":34365,"Ġtopping":34366,"ĠTempest":34367,"rentices":34368,"bh":34369,"Ġparanoia":34370,"ĠUnicode":34371,"Ġegregious":34372,"Ġ\\'":34373,"ĠOswald":34374,"Ġgravel":34375,"ĠSimpsons":34376,"Ġbland":34377,"ĠGuantanamo":34378,"Writer":34379,"liners":34380,"ĠDice":34381,"JC":34382,"Ġparity":34383,"Ġsided":34384,"Ġ237":34385,"ĠPyrrha":34386,"atters":34387,"dk":34388,"Fine":34389,"compan":34390,"Ġformulated":34391,"ĠIdol":34392,"ilers":34393,"hemoth":34394,"ĠFav":34395,"Ġintrusion":34396,"Ġcarrots":34397,"ĠLayer":34398,"ĠHacker":34399,"Ġ----------------":34400,"Ġmoderation":34401,"éģ":34402,"ococ":34403,"Ġcharacterize":34404,"ĠTeresa":34405,"Ġsocioeconomic":34406,"Ġperk":34407,"ĠParticipation":34408,"training":34409,"ĠPaulo":34410,"phys":34411,"Ġtrustworthy":34412,"Ġembodied":34413,"ĠMerch":34414,"currency":34415,"ĠPriority":34416,"Ġteasing":34417,"Ġabsorbing":34418,"Ġunfinished":34419,"ĠComparison":34420,"Ġdisple":34421,"writers":34422,"Ġprofessions":34423,"ĠPenguin":34424,"Ġangrily":34425,"ĠLINK":34426,"688":34427,"ĠCorrespond":34428,"Ġprevailed":34429,"Ġcartel":34430,"lp":34431,"asms":34432,"ĠRedemption":34433,"ĠIslamists":34434,"effects":34435,"dose":34436,"ĠLatter":34437,"ĠHalifax":34438,"Ġvas":34439,"ĠTopics":34440,"ĠNamed":34441,"advertising":34442,"zza":34443,"ICES":34444,"Ġretarded":34445,"achable":34446,"ĠPuppet":34447,"ĠItemLevel":34448,"Ġretract":34449,"Ġidentifiable":34450,"Aaron":34451,"ĠBuster":34452,"sol":34453,"helle":34454,"assemb":34455,"Hope":34456,"ranged":34457,"Ba":34458,"ĠPurch":34459,"éĢ":34460,"ĠSiri":34461,"Ġarrivals":34462,"Ġ1912":34463,"Ġshortened":34464,"Ġ312":34465,"Ġdiscrepancy":34466,"ĠTemperature":34467,"ĠWalton":34468,"Ġkinderg":34469,"polit":34470,"Ġremix":34471,"Ġconnectors":34472,"ãĥĺãĥ©":34473,"ĠKazakhstan":34474,"dominated":34475,"Ġsugars":34476,"imble":34477,"ĠPanic":34478,"ĠDemand":34479,"ĠColony":34480,"onen":34481,"ĠMER":34482,"775":34483,"uria":34484,"azaar":34485,"ĠDegree":34486,"Pri":34487,"Ġsunshine":34488,"Ġ251":34489,"Ġpsychedelic":34490,"Ġdigitally":34491,"ĠBraun":34492,"Ġshimmer":34493,"Ġshave":34494,"ĠTelesc":34495,"ĠAstral":34496,"ĠVenezuelan":34497,"ĠOG":34498,"Ġcrawling":34499,"Integ":34500,"ĠFeather":34501,"Ġunfolding":34502,"Ġappropriation":34503,"Ġè£ıè":34504,"ĠMobility":34505,"ĠNey":34506,"-.":34507,"bilt":34508,"LIN":34509,"ĠTube":34510,"ĠConversely":34511,"Ġkeyboards":34512,"ĠCao":34513,"Ġoverth":34514,"Ġlaure":34515,">>\\":34516,"ĠViper":34517,"acha":34518,"Offset":34519,"ĠRaleigh":34520,"ĠJae":34521,"Jordan":34522,"jp":34523,"Ġtotalitarian":34524,"Connector":34525,"Ġobserves":34526,"ĠSpartan":34527,"ĠImmediately":34528,"ĠScal":34529,"Cool":34530,"Ġtaps":34531,"Ġroar":34532,"Past":34533,"Ġchars":34534,"ĠBender":34535,"ĠSheldon":34536,"Ġpainter":34537,"Ġbeacon":34538,"ĠCreatures":34539,"Ġdownturn":34540,"Ġhinder":34541,"ĠAndromeda":34542,"ÃĽ":34543,"ccoli":34544,"ĠFitness":34545,"etrical":34546,"Ġutilizes":34547,"Ġsenate":34548,"Ġensemble":34549,"Ġcheers":34550,"TW":34551,"Ġaffluent":34552,"kil":34553,"rylic":34554,"ordering":34555,"Computer":34556,"Ġgruesome":34557,"ostics":34558,"ĠUbisoft":34559,"ĠKelley":34560,"Ġwrench":34561,"Ġbourgeoisie":34562,"IBLE":34563,"ĠPreston":34564,"worn":34565,"arist":34566,"reating":34567,"Ġstained":34568,"arine":34569,"Ġslime":34570,"ENN":34571,"Ġchests":34572,"Ġgroundwater":34573,"annot":34574,"ĠTray":34575,"ĠLocke":34576,"ĠCTR":34577,"Ġdudes":34578,"ĠExternal":34579,"ĠDecoder":34580,"Ġparamed":34581,"ĠMedline":34582,"809":34583,"ĠDinner":34584,"rupal":34585,"gz":34586,"ĠGum":34587,"ĠDemo":34588,"jee":34589,"Ġdh":34590,"berman":34591,"archs":34592,"Ġenqu":34593,"ĠEpstein":34594,"Ġdevastation":34595,"Ġfriendships":34596,"ĠArd":34597,"Ġ231":34598,"ĠRubin":34599,"ĠDistance":34600,"Ġspurred":34601,"Ġdossier":34602,"Ġoverlooking":34603,"\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\":34604,"Forest":34605,"ĠComes":34606,"\\\",":34607,"ĠIranians":34608,"Ġfixtures":34609,"Laughs":34610,"Ġcurry":34611,"ĠKingston":34612,"Ġsquash":34613,"Ġcatalogue":34614,"Ġabnormalities":34615,"Ġdigestive":34616,".........":34617,"Ġsubordinate":34618,"ogly":34619,"Ġ249":34620,"Middle":34621,"Ġmassac":34622,"Ġburgers":34623,"Ġdownstairs":34624,"Ġ1931":34625,"394":34626,"ĠVG":34627,"Ġlasers":34628,"ĠSikh":34629,"ĠAlexa":34630,"derived":34631,"Ġcyclist":34632,"ãģ®éŃĶ":34633,"oneliness":34634,"!!!!!!!!":34635,"Ġbuffs":34636,"legate":34637,"Ġraping":34638,"Ġrecommending":34639,"rored":34640,"Ġmulticultural":34641,"unique":34642,"Ġbusinessmen":34643,"Ġuneasy":34644,"ĠMAP":34645,"Ġdispersed":34646,"cipline":34647,"Jess":34648,"ĠKerala":34649,"å§":34650,"Ġabstraction":34651,"Surv":34652,"Uh":34653,"Ġprinters":34654,"ija":34655,"owder":34656,"Ġanalogous":34657,"ĠASP":34658,"afer":34659,"Ġunfolded":34660,"Ġleveling":34661,"Ġbreached":34662,"ĠHearing":34663,"Ġnat":34664,"Ġtranslating":34665,"critical":34666,"Ġantagonist":34667,"ĠYesterday":34668,"Ġfuzzy":34669,"wash":34670,"mere":34671,"Ġbewild":34672,"ĠMae":34673,"Virgin":34674,"phrase":34675,"Ġsignaled":34676,"ĠHIGH":34677,"Ġprotester":34678,"Ġgarner":34679,"unknown":34680,"Ġkay":34681,"Ġabducted":34682,"Ġstalking":34683,"amn":34684,"Ġdeserving":34685,"ĠRiv":34686,"ĠJorge":34687,"Ġscratching":34688,"ĠSaving":34689,"iping":34690,"Ġtease":34691,"Ġmissionary":34692,"ĠMorrow":34693,"TIME":34694,"Present":34695,"Ġchemotherapy":34696,"terness":34697,"ĠHomes":34698,"ĠPurdue":34699,"Ġstaunch":34700,"ĠWhitney":34701,"ĠTHERE":34702,"μ":34703,"iatus":34704,"ĠErnest":34705,"ĠDeploy":34706,"Ġcoveted":34707,"FML":34708,"ĠDialogue":34709,"Ġexited":34710,"fruit":34711,"Ġnerd":34712,"\":\"\",\"":34713,"Ġvivo":34714,"ruly":34715,"460":34716,"ĠAmen":34717,"rehensible":34718,"Ġâĺ":34719,"DIR":34720,"Ġadherence":34721,"Ġchew":34722,"ĠCoke":34723,"ĠSergei":34724,"digital":34725,"ĠNeck":34726,"gently":34727,"enthal":34728,"/)":34729,"Ġweary":34730,"Ġguise":34731,"ĠConcord":34732,"ĠOnion":34733,"atcher":34734,"Ġbinge":34735,"ĠDirective":34736,"Ġmanned":34737,"ansk":34738,"Ġillusions":34739,"Ġbillionaires":34740,"383":34741,"olyn":34742,"odynamic":34743,"ĠWheat":34744,"ĠAlic":34745,"Ġcoloured":34746,"ĠNAFTA":34747,"abo":34748,"Ġmacros":34749,"independent":34750,"sweet":34751,"Ġspac":34752,"ĠKabul":34753,"ĠÄ":34754,"eme":34755,"Ġdictated":34756,"Ġshouts":34757,"={":34758,"Ġripping":34759,"ĠShay":34760,"ĠCricket":34761,"directed":34762,"Ġanalysed":34763,"ĠWARRANT":34764,"agons":34765,"ĠBlazers":34766,"Ġcheered":34767,"Ġarithmetic":34768,"ĠTanz":34769,"373":34770,"ĠFlags":34771,"Ġ295":34772,"Ġwitches":34773,"ĠIncluded":34774,"ĠGained":34775,"ĠBlades":34776,"Gam":34777,"ĠSamantha":34778,"ĠAtlantis":34779,"ĠPratt":34780,"Ġspoiled":34781,"ĠIB":34782,"ĠRamirez":34783,"Probably":34784,"rero":34785,"ĠNg":34786,"ĠWarlock":34787,"tp":34788,"Ġoverhe":34789,"Ġadministrations":34790,"Ġtint":34791,"Ġregiment":34792,"Ġpistols":34793,"Ġblankets":34794,"Ġepist":34795,"Ġbowls":34796,"Ġhydraulic":34797,"Ġdean":34798,"Ġjung":34799,"Ġascend":34800,"705":34801,"ĠSantiago":34802,"î":34803,"Ġunavoid":34804,"ĠShaman":34805,"reb":34806,"Ġstemming":34807,"998":34808,"ĠMG":34809,"sticks":34810,"esthesia":34811,"ERO":34812,"Ġmorbid":34813,"ĠGrill":34814,"ĠPoe":34815,"anyl":34816,"Ġdeleting":34817,"ĠSurveillance":34818,"Ġdirectives":34819,"Ġiterations":34820,"ĠRox":34821,"ĠMilky":34822,"Father":34823,"Ġpatented":34824,"447":34825,"Ġprecursor":34826,"Ġmaiden":34827,"ĠPhen":34828,"ĠVegan":34829,"ĠPatent":34830,"Kelly":34831,"Redditor":34832,"Ġnods":34833,"Ġventilation":34834,"ĠSchwarz":34835,"Ġwizards":34836,"Ġominous":34837,"ĠHeads":34838,"ĠBG":34839,"Ġlumber":34840,"ĠSpiel":34841,"ĠisEnabled":34842,"Ġancestral":34843,"ĠShips":34844,"Ġwrestler":34845,"phi":34846,"Ġyuan":34847,"ĠRebellion":34848,"Ġiceberg":34849,"Ġmagically":34850,"Ġdiversion":34851,"arro":34852,"ythm":34853,"ĠRiders":34854,"ĠRobbie":34855,"ĠKara":34856,"ĠMaintenance":34857,"ĠHerb":34858,"Ġharms":34859,"packed":34860,"ĠFeinstein":34861,"Ġmarrying":34862,"Ġblending":34863,"ĠRates":34864,"Ġ1880":34865,"Ġwrink":34866,"ĠUnch":34867,"ĠTorch":34868,"described":34869,"Ġhumanoid":34870,"ilitating":34871,"ĠConv":34872,"ĠFeld":34873,"IGHTS":34874,"Ġwhistleblower":34875,"ortmund":34876,"etsy":34877,"arrett":34878,"ĠMono":34879,"ĠIke":34880,"ĠCNBC":34881,"ĠWAY":34882,"ĠMDMA":34883,"ĠIndividuals":34884,"Ġsupplemental":34885,"Ġpowerhouse":34886,"ĠStru":34887,"Focus":34888,"aphael":34889,"ĠColleg":34890,"atti":34891,"ZA":34892,"Ġperenn":34893,"ĠSignature":34894,"ĠRodney":34895,"Ġcubes":34896,"iddled":34897,"ĠDante":34898,"ĠINV":34899,"ilingual":34900,"ĠCth":34901,"Ġsofa":34902,"Ġintimidate":34903,"ĠRoe":34904,"ĠDiplom":34905,"ĠCountries":34906,"ayson":34907,"Ġextradition":34908,"Ġdisabling":34909,"ĠCardiff":34910,"Ġmemorandum":34911,"ĠTrace":34912,"Ġ???":34913,"sector":34914,"ĠRouhani":34915,"ĠYates":34916,"ĠFreeze":34917,"Ġbladder":34918,"Motor":34919,"ĠPromise":34920,"antasy":34921,"Ġforeseeable":34922,"ĠCologne":34923,"container":34924,"ĠTrees":34925,"ĠGors":34926,"ĠSinclair":34927,"Ġbarring":34928,"keye":34929,"Ġslashed":34930,"ĠStatistical":34931,"éĩ":34932,"Ġâĸº":34933,"Allows":34934,"Ġhumility":34935,"Ġdrilled":34936,"ĠFurn":34937,"443":34938,"Ġsewage":34939,"Ġhomepage":34940,"Ġcourtyard":34941,"Ġvile":34942,"Ġsubsidiaries":34943,"ajo":34944,"directory":34945,"Ġammon":34946,"Vers":34947,"charges":34948,"Ġ}}":34949,"ĠChains":34950,"Ġ246":34951,"nob":34952,"Ġpercept":34953,"Ġgrit":34954,"Ġfishermen":34955,"ĠIraqis":34956,"ĠDISTR":34957,"ĠFULL":34958,"ĠEvaluation":34959,"graph":34960,"atial":34961,"Ġcooperating":34962,"Ġmelan":34963,"Ġenlightened":34964,"Ġali":34965,"tailed":34966,"Ġsalute":34967,"Ġweakest":34968,"ĠBulldogs":34969,"UA":34970,"ĠAlloy":34971,"Ġsemen":34972,"ocene":34973,"ĠWilliamson":34974,"spr":34975,",âĢĶ":34976,"ĠGF":34977,"ittens":34978,"Beat":34979,"ĠJunk":34980,"iphate":34981,"ĠFarmers":34982,"ĠBitcoins":34983,"igers":34984,"dh":34985,"ĠLoyal":34986,"payer":34987,"Ġentertained":34988,"Ġpenned":34989,"Ġcoupon":34990,"Queue":34991,"Ġweakening":34992,"carry":34993,"Ġunderestimate":34994,"Ġshootout":34995,"Ġcharismatic":34996,"ĠProcedure":34997,"Ġprudent":34998,"inances":34999,"Ġriches":35000,"Ġcortical":35001,"Ġstrides":35002,"Ġdrib":35003,"ĠOilers":35004,"540":35005,"ĠPerform":35006,"ĠBangkok":35007,"Ġeuth":35008,"SER":35009,"Ġsimplistic":35010,"tops":35011,"campaign":35012,"Quality":35013,"Ġimpoverished":35014,"ĠEisenhower":35015,"Ġaugment":35016,"ĠHarden":35017,"Ġintervened":35018,"Ġlistens":35019,"ĠKok":35020,"Ġsage":35021,"Ġrubbish":35022,"ĠDed":35023,"Ġmull":35024,"pelling":35025,"Ġvideot":35026,"Production":35027,"DJ":35028,"miah":35029,"Ġadaptations":35030,"Ġmedically":35031,"Ġboarded":35032,"Ġarrogance":35033,"Ġscrapped":35034,"Ġoppress":35035,"FORMATION":35036,"Ġjunction":35037,"415":35038,"EEEE":35039,"Skill":35040,"Ġsubdu":35041,"ĠSuggest":35042,"ĠPett":35043,"Ġlett":35044,"ĠManip":35045,"ĠCaf":35046,"ĠCooperation":35047,"Ther":35048,"Ġregained":35049,"¶æ":35050,"reflect":35051,"Ġthugs":35052,"ĠShelby":35053,"Ġdictates":35054,"ĠWeiner":35055,"ĠHale":35056,"Ġbattleground":35057,"schild":35058,"Ġcondol":35059,"hunt":35060,"ositories":35061,"Ġaccuses":35062,"Filename":35063,"Ġshri":35064,"Ġmotivate":35065,"Ġreflections":35066,"Null":35067,"ĠLobby":35068,"¥µ":35069,"ĠSATA":35070,"ĠBackup":35071,"Ñĥ":35072,"nin":35073,"ĠCorrection":35074,"Ġjuicy":35075,"utra":35076,"ĠPric":35077,"Ġrestraining":35078,"ĠAirbnb":35079,"ĠArrest":35080,"Ġappropriations":35081,"Ġslopes":35082,"Ġmanslaughter":35083,"Ġworkings":35084,"ĠHuss":35085,"ĠFrey":35086,"Leave":35087,"ĠHarmony":35088,"ĠFeder":35089,"Ġ430":35090,"Ġtrench":35091,"Ġgladly":35092,"Ġbullpen":35093,"ĠGau":35094,"bones":35095,"Ġgroove":35096,"Ġpretext":35097,"ãħĭ":35098,"Ġtransmitter":35099,"ĠComponent":35100,"Ġunderage":35101,"ĠEmpires":35102,"Tile":35103,"Ġoy":35104,"ĠMarvin":35105,"ĠCAS":35106,"Ġbloss":35107,"Ġreplicated":35108,"ĠMariners":35109,"Marcus":35110,"ĠBlocks":35111,"Ġliberated":35112,"Ġbutterfly":35113,"Feel":35114,"Ġfermentation":35115,"Ġyoutube":35116,"Ġoffend":35117,"ĠTerm":35118,"resist":35119,"Ġcessation":35120,"Ġinsurgency":35121,"Ġbir":35122,"ĠRaise":35123,"595":35124,"Ġhypotheses":35125,"502":35126,"Ġplaque":35127,"ocrat":35128,"Ġjackets":35129,"ĠHuffPost":35130,"among":35131,"Ġconfer":35132,"487":35133,"ĠLilly":35134,"Ġadapting":35135,"ĠFay":35136,"Ġshoved":35137,"vec":35138,"Ġrefine":35139,"Ġgon":35140,"Ġgunmen":35141,"zai":35142,"ĠShuttle":35143,"ĠIzan":35144,"Ġ1913":35145,"Ġplethora":35146,"··":35147,"Ġ510":35148,"Ġpuberty":35149,"Ġ241":35150,"ĠWealth":35151,"ĠAlma":35152,"ĠMEM":35153,"ĠAdults":35154,"Cas":35155,"prison":35156,"Race":35157,"Ġwaterproof":35158,"Ġathleticism":35159,"Ġcapitalize":35160,"ĠJuice":35161,"Ġilluminated":35162,"ĠPascal":35163,"Ġirritation":35164,"ĠWitnesses":35165,"adle":35166,"ĠAstro":35167,"Ġfax":35168,"ĠElvis":35169,"Primary":35170,"ĠLich":35171,"ĠElves":35172,"Ġresiding":35173,"Ġstumble":35174,"319":35175,"ĠPKK":35176,"Ġadversaries":35177,"DOS":35178,"ĠRitual":35179,"Ġsmear":35180,"Ġarson":35181,"idental":35182,"Ġscant":35183,"Ġmonarchy":35184,"Ġhalftime":35185,"Ġresidue":35186,"Ġindign":35187,"ĠShaun":35188,"ĠElm":35189,"auri":35190,"Aff":35191,"WATCH":35192,"ĠLyon":35193,"helps":35194,"361":35195,"Ġlobbyist":35196,"Ġdiminishing":35197,"Ġoutbreaks":35198,"Ġgoats":35199,"favorite":35200,"ĠNah":35201,"sonian":35202,"ĠBooster":35203,"Ġsandbox":35204,"ĠFare":35205,"ĠMalta":35206,"ĠattRot":35207,"ĠMOR":35208,"lde":35209,"Ġnavigating":35210,"Touch":35211,"Ġuntrue":35212,"ĠDisaster":35213,"Ġludicrous":35214,"Password":35215,"ĠJFK":35216,"blogspot":35217,"416":35218,"ĠUNDER":35219,"ernal":35220,"Ġdelaying":35221,"TOP":35222,"Ġimplants":35223,"ĠAVG":35224,"ĠHuge":35225,"attr":35226,"Ġjournalistic":35227,"ĠPeyton":35228,"ĠIA":35229,"Rap":35230,"goal":35231,"ĠProgramme":35232,"Ġsmashing":35233,"wives":35234,"println":35235,"ĠPlague":35236,"inus":35237,"EEP":35238,"Ġcruiser":35239,"ĠParish":35240,"uminium":35241,"Ġoccupants":35242,"ĠJihad":35243,"mop":35244,"Ġpint":35245,"Ġhect":35246,"ĠMecca":35247,"director":35248,"ĠFunding":35249,"ĠMixed":35250,"Ġstag":35251,"Tier":35252,"Ġgust":35253,"Ġbrightly":35254,"orsi":35255,"Ġuphill":35256,"RD":35257,"Ġlesions":35258,"ĠBundy":35259,"livious":35260,"Ġbiologist":35261,"ĠFaculty":35262,"ĠAuthorization":35263,"Ġ244":35264,"Allow":35265,"ï¸":35266,"ĠGiul":35267,"Ġpertinent":35268,"otaur":35269,"esse":35270,"ĠRoof":35271,"Ġunmanned":35272,"351":35273,"ĠShak":35274,"ĠOrient":35275,"Ġendanger":35276,"Dir":35277,"Ġreplen":35278,"edient":35279,"Ġtailor":35280,"Ġgadgets":35281,"Ġaudible":35282,"âĺĨ":35283,"Nice":35284,"Ġbombard":35285,"ĠRape":35286,"Ġdefiance":35287,"ĠTWO":35288,"ĠFilipino":35289,"Ġunaffected":35290,"ervatives":35291,"Ġsoared":35292,"ĠBolton":35293,"Ġcompromising":35294,"ĠBrewers":35295,"RAL":35296,"ĠAHL":35297,"icycle":35298,"Ġvampires":35299,"Ġdipped":35300,"oyer":35301,"ĠXIII":35302,"Ġsideways":35303,"ĠWaste":35304,"ĠDiss":35305,"ĠâĶľâĶĢâĶĢ":35306,"$.":35307,"Ġhabitats":35308,"ĠBeef":35309,"truth":35310,"trained":35311,"split":35312,"Rus":35313,"Andy":35314,"ĠBram":35315,"REP":35316,"pid":35317,"è£ħ":35318,"ĠMutant":35319,"Anim":35320,"ĠMarina":35321,"Ġfutile":35322,"highest":35323,"frequency":35324,"Ġepilepsy":35325,"Ġcoping":35326,"Ġconcise":35327,"Ġtracing":35328,"ĠSUN":35329,"panel":35330,"ĠSophie":35331,"ĠCrowley":35332,"ĠAdolf":35333,"ĠShooter":35334,"Ġshaky":35335,"ĠIG":35336,"ĠLies":35337,"ĠBarber":35338,"pkg":35339,"Ġuptake":35340,"Ġpredatory":35341,"ULTS":35342,"/**":35343,"Ġintoxicated":35344,"ĠWestbrook":35345,"odder":35346,"hement":35347,"Ġbaseman":35348,"APD":35349,"storage":35350,"ĠFifty":35351,"editor":35352,"GEN":35353,"UTION":35354,"irting":35355,"Ġsewing":35356,"rift":35357,"Ġagony":35358,"ĠSands":35359,"Ġ254":35360,"Cash":35361,"Ġlodge":35362,"Ġpunt":35363,"Natural":35364,"ĠIdeas":35365,"Ġerroneous":35366,"ĠSensor":35367,"ĠHannity":35368,"Ġ1921":35369,"Ġmould":35370,"ĠGon":35371,"kaya":35372,"Ġanonymously":35373,"ĠKEY":35374,"Ġsimulator":35375,"Winter":35376,"Ġstreamed":35377,"507":35378,"?\",":35379,"Ġteased":35380,"Ġcoefficient":35381,"Ġwartime":35382,"ĠTHR":35383,"''.":35384,"ĠBanking":35385,"mpire":35386,"Ġfandom":35387,"Ġlia":35388,"Ga":35389,"Ġdownhill":35390,"Ġinterpreting":35391,"Individual":35392,"Norm":35393,"Ġjealousy":35394,"bitcoin":35395,"Ġpleasures":35396,"ĠToys":35397,"ĠChevrolet":35398,"ĠAdvisor":35399,"IZE":35400,"Ġreceptions":35401,"706":35402,"Cro":35403,"Ġ262":35404,"Ġcitrus":35405,"iru":35406,"Reviewer":35407,"jected":35408,"UES":35409,"anz":35410,"1981":35411,"ĠWorker":35412,"Ġcomplied":35413,"orescent":35414,"continental":35415,"Ton":35416,"ĠPrism":35417,"ĠSheep":35418,"Ġ288":35419,"nox":35420,"ĠVog":35421,"Ord":35422,"Ġrealms":35423,"tek":35424,"Ġirrigation":35425,"Ġbicycles":35426,"Ġelectronically":35427,"poly":35428,"tall":35429,"());":35430,"Ġaesthetics":35431,"ĠIntegrated":35432,"Explore":35433,"Ġdunk":35434,"476":35435,"pain":35436,"ĠJacques":35437,"ĠDmit":35438,"Frames":35439,"Ġreunited":35440,"Ġhumid":35441,"Dro":35442,"Political":35443,"Ġyouthful":35444,"Ġentails":35445,"Ġmosquito":35446,"363":35447,"species":35448,"Ġcoordinating":35449,"ĠMayhem":35450,"ĠMagnus":35451,"Mount":35452,"Improved":35453,"ĠSTATE":35454,"ATTLE":35455,"Ġflowed":35456,"Ġtackled":35457,"Ġfashioned":35458,"Ġreorgan":35459,"ivari":35460,"finger":35461,"Ġreluctantly":35462,"etting":35463,"ĠVand":35464,"young":35465,"ĠGarland":35466,"Ġpresumption":35467,"Ġamenities":35468,"ĠPleasant":35469,"onential":35470,"ĠOxy":35471,"Ġmorals":35472,"ĠYah":35473,"Ready":35474,"Simon":35475,"Enh":35476,"Demon":35477,"Ġclich":35478,"Monitor":35479,"ĠDU":35480,"Ġwelcomes":35481,"Ġstandout":35482,"Ġdreadful":35483,"Ġbananas":35484,"Ġballoons":35485,"hooting":35486,"basic":35487,"Ġsuffix":35488,"Ġduly":35489,"cano":35490,"Chain":35491,"atos":35492,"Ġgeopolitical":35493,"Ġ(&":35494,"ĠGemini":35495,"ÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤ":35496,"Ġacquitted":35497,"Luck":35498,"protect":35499,"1024":35500,"Ġscarcity":35501,"Ġmindfulness":35502,"ecided":35503,"DN":35504,"prime":35505,"ĠPresidents":35506,"ĠVIDEO":35507,"Ġ(âĪĴ":35508,"addock":35509,"NOR":35510,"ĠPru":35511,"pun":35512,"ĠLOL":35513,"))))":35514,"ĠLiqu":35515,"ĠSAS":35516,"Ġstyling":35517,"Ġpunishments":35518,"Ġnumb":35519,"Ġascertain":35520,"ĠRockies":35521,"flu":35522,"Thumbnail":35523,"Ġperpetrated":35524,"ĠSemi":35525,"Ġdisarm":35526,"ĠOlder":35527,"ĠException":35528,"Ġexponentially":35529,"ĠCommunities":35530,"Ġabolish":35531,"ĠPartner":35532,"ptoms":35533,"Ġ777":35534,"ĠFoley":35535,"ĠCases":35536,"Ġgrease":35537,"ĠRebirth":35538,"Ground":35539,"Ġ;)":35540,"ĠDoctrine":35541,"ikini":35542,"Ye":35543,"ĠBlossom":35544,"Ġpersists":35545,"bill":35546,"Ġinfusion":35547,"Ġbuddies":35548,"911":35549,"ĠPatient":35550,"Ġdemos":35551,"Ġacquaintance":35552,"ĠPaw":35553,"atari":35554,"Ġxml":35555,"Ġfascination":35556,"ĠServe":35557,"ÏĤ":35558,"branded":35559,"Ġaz":35560,"Returns":35561,"Ġovershadow":35562,"Ġroam":35563,"Ġspeedy":35564,"numbered":35565,"helial":35566,"Ġdisciple":35567,"Ġassurances":35568,"given":35569,"pecting":35570,"ĠNatalie":35571,"çĶ°":35572,"Ġmosquitoes":35573,"rotein":35574,"Ġnumeric":35575,"Ġindependents":35576,"Ġtransitional":35577,"Ġreactionary":35578,"ĠMechdragon":35579,"doctor":35580,"Ġshortest":35581,"Ġsequential":35582,"ĠBac":35583,"ĠAccounts":35584,"ãģĮ":35585,"achy":35586,"ractive":35587,"ĠRegiment":35588,"Ġbreathtaking":35589,"fficiency":35590,"ĠBates":35591,"Ġ311":35592,"Ġwardrobe":35593,"fts":35594,"ĠBerk":35595,"Simply":35596,"ĠRiverside":35597,"ivering":35598,"idential":35599,"lucent":35600,"Ġenriched":35601,"ĠConver":35602,"ĠGiving":35603,"ãĥĻ":35604,"Ġlegalize":35605,"ĠFTC":35606,"Ġfreaking":35607,"Mix":35608,"Ġterrestrial":35609,"esian":35610,"cients":35611,"Wing":35612,"LOAD":35613,"Ġledge":35614,"ĠViolent":35615,"ĠMetall":35616,"Ġ308":35617,"Ġsoutheastern":35618,"hetto":35619,"Meat":35620,"Ġslowdown":35621,"Ġretreated":35622,"Jeremy":35623,"endas":35624,"*****":35625,"eric":35626,"Ġreins":35627,"oppable":35628,"ĠHumanity":35629,"earances":35630,"rigan":35631,"Camera":35632,"Ġwaivers":35633,"soc":35634,"Ġalteration":35635,"transform":35636,"ĠCemetery":35637,"506":35638,"Ġindefinite":35639,"Ġstimulating":35640,"yg":35641,"603":35642,"ĠSop":35643,"Ġdescriptive":35644,"Phase":35645,"ĠEdmund":35646,"Ġpneumonia":35647,"ventus":35648,"Amb":35649,"Ġlaboratories":35650,"ĠExclusive":35651,"ugar":35652,"Were":35653,"Ġmalfunction":35654,"Ġhomosexuals":35655,"Ġ-------":35656,"uni":35657,"Ġturbines":35658,"ĠEquity":35659,"Du":35660,"Ġminded":35661,"ĠRH":35662,"ĠBlackhawks":35663,"Ġfeats":35664,"Ġ1700":35665,"repl":35666,"362":35667,"laden":35668,"Ġindispensable":35669,"lyss":35670,"tti":35671,"Ġreel":35672,"Ġdiverted":35673,"Ġlikeness":35674,"Ġsubscriptions":35675,"Ġfingert":35676,"Ġfilthy":35677,"destruct":35678,"draft":35679,"ĠBernardino":35680,"launch":35681,"Ġperplex":35682,"ĠSUM":35683,"carb":35684,"Ġsweater":35685,"ĠVenture":35686,"ĠJag":35687,"ĠCeleb":35688,"ĠVoters":35689,"Ġsteadfast":35690,"Ġathletics":35691,"ĠHanson":35692,"ĠDrac":35693,"Tracker":35694,"Ġcommend":35695,"ĠPresidency":35696,"ĠDID":35697,"informed":35698,"Ġwebpage":35699,"Pretty":35700,"Ġforcefully":35701,"ãĥĥãĤ¯":35702,"Ġrelocation":35703,"Ġsatire":35704,"âī":35705,"ĠSunderland":35706,"æĦ":35707,"Voice":35708,"????????":35709,"Ġinformant":35710,"Ġbowel":35711,"ĠUniform":35712,"Ġ...\"":35713,"Ġpurge":35714,"Ġpicnic":35715,"ĠUmb":35716,"ĠUPDATE":35717,"ĠSapphire":35718,"ĠStall":35719,"learn":35720,"Ġobjectively":35721,"Ġobliter":35722,"Ġloophole":35723,"Ġjourneys":35724,"Ġomission":35725,"Pros":35726,"ĠSidney":35727,"ploma":35728,"Ġsprayed":35729,"Ġguru":35730,"Ġtraitor":35731,"Ġtimet":35732,"Ġsnapping":35733,"ĠSevent":35734,"urnal":35735,"ĠUkip":35736,"Ġbowed":35737,"poral":35738,"liberal":35739,"Ros":35740,"Questions":35741,"iOS":35742,"Ġsummarize":35743,"STAT":35744,"Ġ1850":35745,"apest":35746,"Ġlender":35747,"ĠVariable":35748,"bringing":35749,"ĠLORD":35750,",)":35751,"Ġcollapses":35752,"xiety":35753,"ĠNed":35754,"YD":35755,"ĠScha":35756,"Ġantibody":35757,"Ġdisband":35758,"yre":35759,"illusion":35760,"Ġrover":35761,"shed":35762,"ĠHirosh":35763,"cci":35764,"Ġcalam":35765,"ĠMorton":35766,"Pinterest":35767,"Ġ1928":35768,"ĠEuras":35769,"ordes":35770,"Ġfences":35771,"ĠInventory":35772,"ĠValencia":35773,"ĠUd":35774,"ĠTiff":35775,"Ġsque":35776,"Ġquotation":35777,"Ġtroublesome":35778,"erker":35779,"QUEST":35780,"ĠKingdoms":35781,"south":35782,"Ġlevy":35783,"Prince":35784,"ĠSting":35785,"Ġnicknamed":35786,"Ġappe":35787,"Ġphotographic":35788,"Ġcorpus":35789,"reference":35790,"ĠTrog":35791,"Unt":35792,")=(":35793,"ĠLatvia":35794,"Ġactivating":35795,"Ġlicensee":35796,"Ġdisparities":35797,"ĠNewsletter":35798,"ãĥĥãĥĪ":35799,"Ġfreeing":35800,"ĠJeep":35801,"ĠPerception":35802,"insk":35803,"Ġsilicone":35804,"ĠHayden":35805,"Lean":35806,"ĠSuzuki":35807,"ibrarian":35808,"668":35809,"Ġspor":35810,"Ġcorrelations":35811,"aghetti":35812,"Ġtuber":35813,"ĠIPCC":35814,"ilus":35815,"ĠVu":35816,"Ġwealthiest":35817,"ĠCarbuncle":35818,"anza":35819,"Ġfooled":35820,"ĠZur":35821,"Ġdaddy":35822,"rano":35823,"ilian":35824,"Ġknockout":35825,"fman":35826,"required":35827,"ĠWikileaks":35828,"ĠDuffy":35829,"ONT":35830,"Ġinsol":35831,"ĠObjects":35832,"Ġbou":35833,"ĠNordic":35834,"ĠInsert":35835,"scan":35836,"Ġdancers":35837,"Ġidiots":35838,"majority":35839,"ĠNeville":35840,"ĠFreeBSD":35841,"Ġtart":35842,"panic":35843,"690":35844,"Ġcocoa":35845,"Ġsampled":35846,"Ġlookup":35847,"Indust":35848,"Ġinjections":35849,"genre":35850,"Ġau":35851,"Ġroadway":35852,"Ġgenitals":35853,"Kind":35854,"ĠExaminer":35855,"ĠYaz":35856,"Fresh":35857,"Ġparalysis":35858,"ĠAluminum":35859,"Ġreap":35860,"oké":35861,"Ġsloppy":35862,"ĠTunnel":35863,"posium":35864,"nery":35865,"enic":35866,"Ġherbal":35867,"ĠOuter":35868,"ĠBuilder":35869,"Ġincur":35870,"Ġideologies":35871,"Ġbackups":35872,"consuming":35873,"ĠDetect":35874,"deck":35875,"ĠKNOW":35876,"ĠGret":35877,"ĠMIC":35878,"Ġtoughness":35879,"ĠExhibit":35880,"Ġhive":35881,"Les":35882,"ĠSCHOOL":35883,"ĠAtari":35884,"alde":35885,"ĠNull":35886,"andestine":35887,"mouse":35888,"Ġbrigade":35889,"489":35890,"Ġrevol":35891,"ĠLawson":35892,"ĠWah":35893,"opoly":35894,"ebted":35895,"ĠSaunders":35896,"Ġ313":35897,"ĠWinc":35898,"Ġtaboo":35899,"ĠHelmet":35900,"Ġwedge":35901,"chip":35902,"ĠTina":35903,"bg":35904,"Ġinfuri":35905,"rn":35906,"Ġanomalies":35907,"ĠSync":35908,"ĠExam":35909,"ĠCommit":35910,"ĠDiary":35911,"ĠALSO":35912,"ĠDebor":35913,"omedical":35914,"Ġcomprehension":35915,"655":35916,"Ġempowering":35917,"Ġire":35918,"Ġjuices":35919,"ĠETH":35920,"ĠBoxing":35921,"=\"/":35922,"Ġfacilitated":35923,"poke":35924,"ĠParsons":35925,"ĠModer":35926,"travel":35927,"Ġcivilizations":35928,"Ġlibertarians":35929,"Ġrune":35930,"ĠClarks":35931,"athed":35932,"Ġcampaigners":35933,"ĠDispatch":35934,"ĠFahrenheit":35935,"ĠCapcom":35936,"----------":35937,"Ġlace":35938,"Ġdraining":35939,"Ġliner":35940,"ĠArtificial":35941,"én":35942,"task":35943,"]).":35944,"ĠGMO":35945,"ĠOperator":35946,"ordinary":35947,"ĠInfluence":35948,"ĠUps":35949,"Ġpotency":35950,"ussen":35951,"ospons":35952,"ĠSwim":35953,"ĠDeadline":35954,"Unity":35955,"Ġculinary":35956,"Ġenlightenment":35957,"Ġwearer":35958,"Ġmined":35959,"Ġply":35960,"Ġincest":35961,"ĠDVDs":35962,"Walk":35963,"BTC":35964,"Trade":35965,"Ġdeval":35966,"iband":35967,"ĠOversight":35968,"Palestinian":35969,"Ġdart":35970,"Ġmul":35971,"LR":35972,"Ġremovable":35973,"ĠRealms":35974,"ìĿ":35975,"Ġmiscar":35976,"ĠVulkan":35977,"685":35978,"ère":35979,"ĠSap":35980,"Ġmerging":35981,"ĠCarly":35982,"chester":35983,"Ġbrisk":35984,"Ġluxurious":35985,"ĠGenerator":35986,"Ġbitterness":35987,"Ġedible":35988,"Ġ243":35989,"TG":35990,"Ġrectangle":35991,"WithNo":35992,"below":35993,"Jenn":35994,"Ġdarkest":35995,"Ġhitch":35996,"Ġdosage":35997,"Ġscaven":35998,"ĠKeller":35999,"ĠIllustrated":36000,"Certainly":36001,"ĠMavericks":36002,"Marginal":36003,"Ġdiarrhea":36004,"Ġenormously":36005,"Ġ999":36006,"shr":36007,"quart":36008,"Ġadamant":36009,"ĠMew":36010,"Ġrenovation":36011,"Ġcervical":36012,"ĠPercentage":36013,"eners":36014,"ĠKimber":36015,"Ġfloats":36016,"Ġdex":36017,"ĠWitcher":36018,"ĠSwansea":36019,"dm":36020,"Ġsalty":36021,"yellow":36022,"Ġcape":36023,"ĠDrain":36024,"ĠPaula":36025,"ĠToledo":36026,"lesi":36027,"Magazine":36028,"ĠWick":36029,"ĠMn":36030,"ĠAck":36031,"ĠRiding":36032,"ASON":36033,"Ġhomophobic":36034,"ARP":36035,"Ġwandered":36036,"CPU":36037,"oodoo":36038,"ĠPipe":36039,"Ġtightening":36040,"ĠButt":36041,"318":36042,"Ġdeserted":36043,"Session":36044,"Ġfacilitating":36045,"Jump":36046,"Ġemergencies":36047,"OWER":36048,"Ġexhaustive":36049,"ĠAFTER":36050,"Ġheartbeat":36051,"ĠLabel":36052,"acky":36053,"ĠCertified":36054,"iltration":36055,"Ze":36056,"ĠUtt":36057,"Ġ1300":36058,"Ġpresume":36059,"ĠDisp":36060,"Ġsurged":36061,"Ġdolls":36062,"Columb":36063,"Ġchimpan":36064,"ĠRazor":36065,"Ġticks":36066,"Ġcouncillor":36067,"Ġpilgrimage":36068,"ĠRebels":36069,"ĠQC":36070,"ĠAuction":36071,"xia":36072,"ikk":36073,"bred":36074,"Ġinsertion":36075,"Ġcoarse":36076,"dB":36077,"SEE":36078,"ĠZap":36079,"ĠFoo":36080,"Ġcontempor":36081,"ĠQuarterly":36082,"otions":36083,"ĠAlchemist":36084,"ĠTrey":36085,"ĠDuo":36086,"Sweet":36087,"804":36088,"ĠGiov":36089,"Ġfunn":36090,"Nin":36091,"hoff":36092,"Ġramifications":36093,"Ġ1922":36094,"ĠExperts":36095,"azes":36096,"Ġgarments":36097,"arial":36098,"ĠNab":36099,"Ġ257":36100,"ĠVed":36101,"Ġhumorous":36102,"ĠPompe":36103,"Ġnylon":36104,"Ġlurking":36105,"ĠSergey":36106,"ĠMattis":36107,"Ġmisogyny":36108,"ĠComponents":36109,"ĠWatching":36110,"ĠFolk":36111,"ractical":36112,"Bush":36113,"Ġtaped":36114,"Ġgrouping":36115,"Ġbeads":36116,"Ġ2048":36117,"Ġcondu":36118,"querque":36119,"Reading":36120,"Ġgrievances":36121,"Ultra":36122,"Ġendpoint":36123,"Hig":36124,"ĠStatic":36125,"ĠScarborough":36126,"Lua":36127,"ĠMessi":36128,"aqu":36129,"ĠPsyNet":36130,"ĠRudd":36131,"Ġavenue":36132,"vp":36133,"Jer":36134,"Ġshady":36135,"ĠResist":36136,"ĠArtemis":36137,"Ġcareless":36138,"Ġbrokers":36139,"Ġtemperament":36140,"Ġ520":36141,"Tags":36142,"ĠTurning":36143,"Ġuttered":36144,"Ġpedd":36145,"Ġimprovised":36146,"Ġ:(":36147,"Ġtabl":36148,"Ġplains":36149,"1600":36150,"pressure":36151,"ĠEssence":36152,"margin":36153,"friends":36154,"ĠRestoration":36155,"Ġpollut":36156,"ĠPoker":36157,"ĠAugustine":36158,"ĠCIS":36159,"ĠSEAL":36160,"orama":36161,"Ġthwart":36162,"seek":36163,"Ġpagan":36164,"º":36165,"cpu":36166,"Ġgarn":36167,"Ġassortment":36168,"ĠILCS":36169,"tower":36170,"Recommended":36171,"Ġunborn":36172,"ĠRandomRedditor":36173,"ĠRandomRedditorWithNo":36174,"Ġparalyzed":36175,"Ġeruption":36176,"Ġintersect":36177,"ĠStoke":36178,"ĠSco":36179,"Bind":36180,"å¾":36181,"ĠPNG":36182,"ĠNegative":36183,"ĠNOAA":36184,"Leon":36185,"Ġalloy":36186,"ĠLama":36187,"ĠDiversity":36188,"575":36189,"Ġunderestimated":36190,"ĠScor":36191,"Ġmural":36192,"Ġbusted":36193,"soon":36194,"lif":36195,"Ġnonex":36196,"Ġallergy":36197,"ĠUnderworld":36198,"ĠRays":36199,"ĠBlasio":36200,"Ġhrs":36201,"ĠDir":36202,"Ġ327":36203,"byter":36204,"Ġreplacements":36205,"Ġactivates":36206,"rived":36207,"MH":36208,"Ġpans":36209,"ĠHI":36210,"Ġlongitudinal":36211,"Ġnuisance":36212,"aler":36213,"Ġswell":36214,"ĠSigned":36215,"sci":36216,"ĠIsles":36217,"ĠAGA":36218,"Ġdefiant":36219,"Ġsonic":36220,"ocon":36221,"KC":36222,"ĠAim":36223,"tie":36224,"ahah":36225,"ĠmL":36226,"DX":36227,"Ġbisc":36228,"ĠBillboard":36229,"ĠSYSTEM":36230,"NEY":36231,"gaard":36232,"Ġdistressed":36233,"formerly":36234,"Alan":36235,"Ġchefs":36236,"Ġoptics":36237,"ĠComet":36238,"ĠAMC":36239,"Ġredesigned":36240,"irmation":36241,"Ġsightings":36242,"382":36243,"311":36244,"ĠWB":36245,"Ġcontraction":36246,"ĠTOTAL":36247,"Dual":36248,"Ġstartled":36249,"Ġunderstandably":36250,"Ġsunglasses":36251,"ETHOD":36252,"Ġdocker":36253,"Ġsurfing":36254,"ĠHEL":36255,"ĠSlack":36256,"tones":36257,"Ġshalt":36258,"Visual":36259,"498":36260,"Department":36261,"cussion":36262,"Ġunrestricted":36263,"Ġtad":36264,"Ġrename":36265,"employed":36266,"Ġeducating":36267,"Ġgrinned":36268,"bedroom":36269,"ĠActivities":36270,"ĠVelvet":36271,"ĠSWAT":36272,"Ġshuffle":36273,"igor":36274,"Ġsaturation":36275,"Finding":36276,"cream":36277,"icter":36278,"Ġvodka":36279,"tracking":36280,"tec":36281,"Ġforeground":36282,"iesta":36283,"Ġvehement":36284,"ĠECB":36285,"ĠTie":36286,"Ey":36287,"Ġturtles":36288,"ĠRailroad":36289,"ĠKatz":36290,"ĠFrames":36291,"Ġmenace":36292,"ĠFellowship":36293,"ĠEssential":36294,"uggish":36295,"Ġdrip":36296,"chwitz":36297,"ĠKyoto":36298,"sb":36299,"ĠNina":36300,"Parameter":36301,"Ġalarms":36302,"ĠClaud":36303,"Ġpioneering":36304,"Ġchiefly":36305,"ĠScream":36306,"Collection":36307,"Ġthankfully":36308,"ĠRonaldo":36309,"åŃIJ":36310,"strip":36311,"ĠDisneyland":36312,"commercial":36313,"Seeing":36314,"Soul":36315,"Ġevacuate":36316,"Ġciv":36317,"ĠAshe":36318,"Ġdivides":36319,"ĠDagger":36320,"rehensive":36321,"Ġberries":36322,"ĠDF":36323,"Ġsushi":36324,"Ġplurality":36325,"WI":36326,"Ġdisadvantaged":36327,"Ġbattalion":36328,"obiles":36329,"451":36330,"Ġcling":36331,"Ġundeniable":36332,"ĠLounge":36333,"Ġhaunt":36334,"phe":36335,"Ġquantify":36336,"Ġdiffered":36337,"Ġ[*]":36338,"ĠViz":36339,"cum":36340,"slave":36341,"Ġvideog":36342,"Ġquar":36343,"Ġbundles":36344,"ĠAlonso":36345,"tackle":36346,"Ġneuronal":36347,"Ġlandslide":36348,"confirmed":36349,"ĠDepth":36350,"Ġrenewables":36351,"Bear":36352,"ĠMacedonia":36353,"Ġjerseys":36354,"Ġbunk":36355,"ĠSpawn":36356,"ĠControls":36357,"ĠBuchanan":36358,"Ġrobotics":36359,"Ġemphasizing":36360,"ĠTutorial":36361,"hyp":36362,"iston":36363,"Ġmonumental":36364,"æ°":36365,"ĠCarry":36366,"Ġtbsp":36367,"enance":36368,"Hill":36369,"arthed":36370,"Ġrotten":36371,"Dean":36372,"Ġtwisting":36373,"Ġgoodwill":36374,"Ġimmersion":36375,"Living":36376,"Ġbrushes":36377,"ĠCGI":36378,"ĠAtk":36379,"traditional":36380,"Ġphantom":36381,"ĠStamina":36382,"Ġexpansions":36383,"ĠMarin":36384,"Ġembarked":36385,"ĠEg":36386,"intestinal":36387,"ĠPEOPLE":36388,"ĠBooth":36389,"ĠAppalach":36390,"Ġrelegated":36391,"VT":36392,"MIT":36393,"Ġmuster":36394,"Ġwithdrawing":36395,"Ġmicroscope":36396,"ĠGathering":36397,"ĠCrescent":36398,"ĠArgentine":36399,"ĠDecre":36400,"ĠDominic":36401,"Ġbuds":36402,"antage":36403,"ĠIon":36404,"Ġwidened":36405,"ONSORED":36406,"ĠGloves":36407,"iannopoulos":36408,"razen":36409,"feel":36410,"Ġrepayment":36411,"Ġhindsight":36412,"ĠREALLY":36413,"ĠPistol":36414,"ĠBrah":36415,"Ġwatts":36416,"Ġsurvives":36417,"Ġflurry":36418,"issy":36419,"Alert":36420,"ĠUruguay":36421,"Phoenix":36422,"Slow":36423,"ĠGrave":36424,"ĠFir":36425,"Ġmanageable":36426,"Ġtariff":36427,"ĠUDP":36428,"ĠPistons":36429,"ĠNigerian":36430,"Ġstrikeouts":36431,"Ġcosmetics":36432,"whelming":36433,"fab":36434,"cape":36435,"proxy":36436,"Ġrethink":36437,"Ġovercoming":36438,"simple":36439,"Ġwoo":36440,"Ġdistracting":36441,"ĠStanton":36442,"ĠTulsa":36443,"ĠDock":36444,"659":36445,"Ġdiscord":36446,"ĠEmacs":36447,"ĠVes":36448,"ĠROB":36449,"Ġreassuring":36450,"Ġconsortium":36451,"Muslims":36452,"321":36453,"Ġprompts":36454,"sei":36455,"ĠHitch":36456,"imposed":36457,"ĠFool":36458,"Ġindiscrim":36459,"wrong":36460,"buquerque":36461,"Davis":36462,"!]":36463,"Ġtimeless":36464,"ĠNEED":36465,"Ġpesticide":36466,"Ġrallying":36467,"ĠCalder":36468,"Ġå¤":36469,"Ġxp":36470,"ĠUnle":36471,"ĠExport":36472,"luaj":36473,"Buff":36474,")[":36937,"Ġsqor":36938,"Saudi":36939,"Ġistg":36940,"Ġindulge":36941,"proc":36942,"Ġdisgusted":36943,"Ġcompounded":36944,"Ġnem":36945,"Ġschooling":36946,"ĠCure":36947,"processing":36948,"Sol":36949,"Ġproverb":36950,"itized":36951,"ĠAlvarez":36952,"Ġscarf":36953,"Ġrectangular":36954,"reve":36955,"Ġhormonal":36956,"ĠStress":36957,"itizen":36958,"Ġ425":36959,"girls":36960,"ĠNoir":36961,"ĠRapp":36962,"Ġmarches":36963,"church":36964,"ĠUses":36965,"Ġ405":36966,"ĠBerm":36967,"Ġordinances":36968,"ĠJudgment":36969,"Charges":36970,"ĠZin":36971,"Ġdusty":36972,"Ġstrawberries":36973,"Ġperce":36974,"ĠThur":36975,"ĠDeborah":36976,"netflix":36977,"ĠLambert":36978,"Ġamused":36979,"ĠGuang":36980,"YOU":36981,"RGB":36982,"ĠCCTV":36983,"Ġfiat":36984,"rang":36985,"Ġfederation":36986,"ĠMant":36987,"ĠBust":36988,"ĠMare":36989,"respective":36990,"ĠMigration":36991,"ĠBIT":36992,"590":36993,"Ġpatriotism":36994,"Ġoutlining":36995,"region":36996,"ĠJosé":36997,"Ġblasting":36998,"ĠEzra":36999,"Bs":37000,"Ġundermines":37001,"ĠSmooth":37002,"Ġclashed":37003,"radio":37004,"Ġtransitioning":37005,"ĠBuccaneers":37006,"ĠOwl":37007,"Ġplugs":37008,"Ġhiatus":37009,"ĠPinball":37010,"Ġmig":37011,"ĠNutr":37012,"ĠWolfe":37013,"Ġintegers":37014,"Ġorbits":37015,"ĠEdwin":37016,"ĠDirectX":37017,"bite":37018,"Ġblazing":37019,"vr":37020,"Edge":37021,"ĠPID":37022,"exit":37023,"ĠComed":37024,"ĠPathfinder":37025,"ĠGuid":37026,"ĠSigns":37027,"ĠZer":37028,"ĠAgenda":37029,"Ġreimbursement":37030,"Mesh":37031,"iPhone":37032,"ĠMarcos":37033,"ĠSites":37034,"hate":37035,"enburg":37036,"Ġsockets":37037,"pend":37038,"Batman":37039,"vir":37040,"ĠSHOW":37041,"Ġprovisional":37042,"conn":37043,"ĠDeaths":37044,"ATIVE":37045,"Profile":37046,"sym":37047,"JA":37048,"Ġninja":37049,"installed":37050,"idates":37051,"ebra":37052,"ĠOmaha":37053,"Ġseizing":37054,"ĠBeasts":37055,"Ġsalts":37056,"Mission":37057,"Generally":37058,"ĠTrilogy":37059,"heon":37060,"legates":37061,"Ġdime":37062,"Ġfaire":37063,"parable":37064,"Graph":37065,"Ġtotaling":37066,"Ġdiagrams":37067,"ĠYanuk":37068,"plet":37069,"ĠMeh":37070,"Ġmythical":37071,"ĠStephens":37072,"autical":37073,"ochemistry":37074,"Ġkilograms":37075,"Ġelbows":37076,"ancock":37077,"ĠBCE":37078,"ĠPrague":37079,"Ġimprov":37080,"ĠDevin":37081,"Ġ\"\\":37082,"paralle":37083,"Ġsupremacists":37084,"ĠBillion":37085,"Ġregimen":37086,"innacle":37087,"Ġrequisite":37088,"angan":37089,"ĠBurlington":37090,"ainment":37091,"ĠObjective":37092,"omsky":37093,"GV":37094,"Ġunilateral":37095,"Ġtc":37096,"Ġhires":37097,"mental":37098,"Ġinvoluntary":37099,"Ġtranspl":37100,"ĠASCII":37101,"¨":37102,"Events":37103,"Ġdoubted":37104,"ĠKaplan":37105,"ĠCourage":37106,"igon":37107,"ĠManaging":37108,"ĠTart":37109,"Ġfalsehood":37110,"ĠViolet":37111,"Ġairs":37112,"Ġfertilizer":37113,"Britain":37114,"Ġaquatic":37115,"ouf":37116,"Words":37117,"ĠHartford":37118,"Ġevenings":37119,"ĠVengeance":37120,"quite":37121,"Gall":37122,"ĠPret":37123,"Ġpdf":37124,"ĠLM":37125,"ĠSochi":37126,"ĠIntercept":37127,"920":37128,"Ġprofitability":37129,"ĠIdle":37130,"ĠMacDonald":37131,"ĠEstablishment":37132,"umsy":37133,"Ġgatherings":37134,"ĠNaj":37135,"Charlie":37136,"Ġascent":37137,"ĠProtector":37138,"Ġalgebra":37139,"Ġbios":37140,"forums":37141,"ELS":37142,"Introduced":37143,"Ġ335":37144,"Ġastronomy":37145,"Contribut":37146,"ĠPolic":37147,"Platform":37148,"Ġcontainment":37149,"wrap":37150,"Ġcoronary":37151,"ĠJelly":37152,"manager":37153,"Ġheartbreaking":37154,"cair":37155,"ĠChero":37156,"cgi":37157,"Medical":37158,"ĠAccountability":37159,"!!\"":37160,"ophile":37161,"Ġpsychotic":37162,"ĠRestrict":37163,"Ġequitable":37164,"issues":37165,"Ġ1905":37166,"ĠNek":37167,"cised":37168,"ĠTracking":37169,"Ġozone":37170,"Ġcooker":37171,"rosis":37172,"Ġreopen":37173,"Ġinfinity":37174,"ĠPharmaceutical":37175,"ensional":37176,"Attempt":37177,"ĠRory":37178,"Marco":37179,"Ġawaits":37180,"HOW":37181,"treated":37182,"Ġbolst":37183,"Ġrevered":37184,"Ġpods":37185,"oppers":37186,"0010":37187,"Ġamplitude":37188,"rican":37189,"SPONSORED":37190,"Ġtrousers":37191,"Ġhalves":37192,"ĠKaine":37193,"ĠCutler":37194,"ĠAUTH":37195,"Ġsplendid":37196,"Ġpreventive":37197,"ĠDudley":37198,"ifacts":37199,"uminati":37200,"ĠYin":37201,"Ġadmon":37202,"ĠVag":37203,"Ġinverted":37204,"Ġhastily":37205,"ĠHague":37206,"Lyn":37207,"Ġledger":37208,"Ġastronomical":37209,"getting":37210,"Ġcirca":37211,"ĠCic":37212,"ĠTennis":37213,"Limited":37214,"Ġdru":37215,"ĠBYU":37216,"Ġtravellers":37217,"Ġpane":37218,"ĠIntro":37219,"Ġpatiently":37220,"Ġaiding":37221,"Ġloos":37222,"ĠTough":37223,"Ġ293":37224,"Ġconsumes":37225,"SourceFile":37226,"Ġ\"\"\"":37227,"Ġbonding":37228,"Ġtilted":37229,"Ġmenstrual":37230,"ĠCelestial":37231,"ULAR":37232,"Plugin":37233,"Ġrisking":37234,"Naz":37235,"ĠRiyadh":37236,"Ġaccredited":37237,"Ġskirm":37238,"éĽ":37239,"Ġexaminer":37240,"Ġmessing":37241,"Ġnearing":37242,"ĠChern":37243,"ĠBeckham":37244,"Ġswapped":37245,"Ġgoose":37246,"Kay":37247,"Ġlofty":37248,"ĠWallet":37249,"Ġ['":37250,"Ġapocalypse":37251,"Ġbamboo":37252,"ĠSPACE":37253,"ĠElena":37254,"Ġ306":37255,"acons":37256,"Ġtightened":37257,"Ġadolescence":37258,"Ġrainy":37259,"Ġvandalism":37260,"ĠNewtown":37261,"Ġconject":37262,"cakes":37263,"Ġcheated":37264,"Ġmoderators":37265,"params":37266,"EFF":37267,"Ġdeceit":37268,"ĠSTL":37269,"ĠTanzania":37270,"ĠRI":37271,"Ġ1923":37272,"ĠExile":37273,"thel":37274,"Ġtheolog":37275,"Ġquirky":37276,"ĠIrvine":37277,"Ġneedy":37278,"oris":37279,"Um":37280,"Ka":37281,"Ġmailbox":37282,"322":37283,"Ġbos":37284,"ĠPetra":37285,"KING":37286,"Ġenlarged":37287,"Often":37288,"Ġbadass":37289,"Ġ343":37290,"ĠPlaces":37291,"ĠCAD":37292,"Ġpristine":37293,"Ġintervening":37294,"direction":37295,"Ġlaz":37296,"ĠDSM":37297,"Ġprojecting":37298,"ĠFunk":37299,"agog":37300,"payment":37301,"nov":37302,"Ġchatter":37303,"ARB":37304,"Ġexaminations":37305,"ĠHousehold":37306,"ĠGus":37307,"Ford":37308,"414":37309,"Boss":37310,"Ġmystic":37311,"Ġleaps":37312,"ĠBav":37313,"ulz":37314,"budget":37315,"Football":37316,"Ġsubsidized":37317,"Ġfirsthand":37318,"Ġcoincide":37319,"ocular":37320,"Conn":37321,"ĠCollabor":37322,"Ġfools":37323,"amura":37324,"ahar":37325,"rists":37326,"Ġswollen":37327,"Ġexpended":37328,"ĠPau":37329,"sup":37330,"Ġspar":37331,"Ġkeynote":37332,"suff":37333,"Ġunequal":37334,"Ġprogressing":37335,"strings":37336,"ĠGamergate":37337,"Disney":37338,"ĠEleven":37339,"omnia":37340,"Ġscripted":37341,"Ġearners":37342,"brother":37343,"ĠEnabled":37344,"æ³":37345,"Ġlarvae":37346,"ĠLOC":37347,"mess":37348,"Wilson":37349,"ĠTemplate":37350,"successfully":37351,"Ġparamount":37352,"Ġcamouflage":37353,"Ġbinds":37354,"ĠQuiet":37355,"ĠShutterstock":37356,"rush":37357,"Ġmascot":37358,"fortune":37359,"ĠColt":37360,"ĠBeyon":37361,"habi":37362,"Ġhairc":37363,"Ġ267":37364,"ĠDeus":37365,"Ġtwitch":37366,"Ġconcentrating":37367,"Ġnipples":37368,"cible":37369,"Ġgir":37370,"NZ":37371,"Math":37372,"nih":37373,"Required":37374,"Ġponder":37375,"ĠSAN":37376,"Ġweddings":37377,"Ġloneliness":37378,"NES":37379,"ĠMahjong":37380,"695":37381,"addle":37382,"ĠGarner":37383,"ĠCOUR":37384,"Bridge":37385,"Ġspree":37386,"ĠCaldwell":37387,"Ġbribery":37388,"Ġ��������":37389,"plugins":37390,"Ġracket":37391,"Ġchampagne":37392,"versible":37393,"Vote":37394,"Ġmodifiers":37395,"Mayor":37396,"680":37397,"Ġassemblies":37398,"ĠSultan":37399,"ĠNing":37400,"ĠLadies":37401,"Ġsulfur":37402,"Ġorbs":37403,"Ġ-----":37404,"_______":37405,"ĠJournalism":37406,"Ġesports":37407,"Ġlush":37408,"Ġhue":37409,"Ġspectral":37410,"Honest":37411,"ãĥı":37412,"Ġbushes":37413,"Ġreinforcement":37414,"Ġreopened":37415,"ĠWheels":37416,"ĠMorg":37417,"rieving":37418,"Ġauxiliary":37419,"ĠjQuery":37420,"ĠBAT":37421,"tesque":37422,"Ġvertex":37423,"pure":37424,"frey":37425,"ãĤº":37426,"dos":37427,"Ġtyph":37428,"Ġcull":37429,"Ġeq":37430,"Ġdecon":37431,"Ġtossing":37432,"Ġdisparate":37433,"ĠBrigham":37434,"printf":37435,"ledged":37436,"Ġsund":37437,"Ġcozy":37438,"Ġhepatitis":37439,"performing":37440,"Ġaval":37441,"ĠGG":37442,"future":37443,"Ġpetertodd":37444,"ĠKosovo":37445,"Ġmagnets":37446,"Already":37447,"ĠEdison":37448,"ĠCeres":37449,"ĠRAID":37450,"Ġbrilliance":37451,"576":37452,"Ġderives":37453,"Ġhypertension":37454,"ĠÎĶ":37455,"Ġlambda":37456,"Ġflair":37457,"Ġmissionaries":37458,"Ġrapes":37459,"ĠStarter":37460,"ĠMonths":37461,"Ġdefy":37462,"Ġseismic":37463,"ĠRaphael":37464,"Ġeurozone":37465,"656":37466,"zsche":37467,"Ġscratched":37468,"Ġbows":37469,"ĠLennon":37470,"ĠGaia":37471,"Ġdripping":37472,"facts":37473,"Ale":37474,"Ġfrogs":37475,"ĠBreast":37476,"ogeneity":37477,"ĠProsecutor":37478,"Ġamplified":37479,"ĠHodg":37480,"ĠFn":37481,"Thousands":37482,"ĠNIH":37483,"ĠMonitoring":37484,"FTWARE":37485,"ĠPriebus":37486,"ĠGrowing":37487,"hunter":37488,"Ġdiagnose":37489,"ĠMald":37490,"ĠLR":37491,"Ġcrowned":37492,"Ġbursting":37493,"Ġdissolution":37494,"javascript":37495,"Ġusefulness":37496,"ĠExecution":37497,":(":37498,"ĠIvory":37499,"aah":37500,"Ġpersecuted":37501,"violence":37502,"istas":37503,"ĠCrate":37504,"Ġimpulses":37505,"ĠSpani":37506,"edes":37507,"Handle":37508,"ĠZerg":37509,"thinkable":37510,"Lastly":37511,"Ġspontaneously":37512,"Ġinconvenient":37513,"Ġdismissing":37514,"Ġplotted":37515,"Ġeighty":37516,"Ġ737":37517,"rish":37518,"ĠThornton":37519,"atham":37520,"Ġsitcom":37521,"Ven":37522,"Recipe":37523,"tel":37524,"lund":37525,"Ġclears":37526,"ĠSasuke":37527,"Ġ258":37528,"Ġopting":37529,"Ġenraged":37530,"esthetic":37531,"ĠAe":37532,"uchs":37533,"Prep":37534,"Flow":37535,"Ġrunoff":37536,"ĠEating":37537,"ĠGiles":37538,"ĠActing":37539,"resources":37540,"ibaba":37541,"Ġrpm":37542,"Ġskewed":37543,"ĠBlanc":37544,"ĠSakuya":37545,"Ġhotter":37546,"Ġ1924":37547,"opian":37548,"cko":37549,"Ġcrumbling":37550,"Ġcaptains":37551,"ĠAppropriations":37552,"leaders":37553,"dropping":37554,"anuts":37555,"Ġreversing":37556,"ĠPose":37557,"ĠSek":37558,"Scot":37559,"ĠIdea":37560,"cise":37561,"ĠSlovenia":37562,"Ġ317":37563,"Doctor":37564,"Ġcrocod":37565,"aldi":37566,"Sea":37567,"ĠFarrell":37568,"Ġmercenaries":37569,"ĠRNC":37570,"ĠGuess":37571,"Ġpacing":37572,"Machine":37573,"StreamerBot":37574,"ĠCharity":37575,"Ġ298":37576,"Ġcannons":37577,"ĠToby":37578,"TPPStreamerBot":37579,"ĠPassion":37580,"cfg":37581,"Thom":37582,"Ġbadges":37583,"ĠBernstein":37584,".âĢĵ":37585,"ĠPOP":37586,"ĠConj":37587,"Ġinitialization":37588,"Ġbiodiversity":37589,"Dub":37590,"Ġfeudal":37591,"Ġdisclaimer":37592,"Ġcrow":37593,"Ġignition":37594,"arf":37595,"SHA":37596,"ĠkHz":37597,"hazard":37598,"ĠArtists":37599,"oeuv":37600,"679":37601,"ĠRudy":37602,"Nine":37603,"ĠRamadan":37604,"å½":37605,"itto":37606,"Ġadrenaline":37607,"Cert":37608,"Ġsmelled":37609,"Ġimpunity":37610,"Ġagendas":37611,"ĠReborn":37612,"ĠConcent":37613,"ĠSeems":37614,"Ġomega":37615,"ĠDustin":37616,"Ġbacker":37617,"ĠSauce":37618,"ĠBoyle":37619,"WIN":37620,"Ġspins":37621,"Ġpauses":37622,"upt":37623,"Ġshredded":37624,"Ġstrapped":37625,"ĠCorruption":37626,"Ġscratches":37627,"Ġni":37628,"Ġattire":37629,"ĠSAF":37630,"FactoryReloaded":37631,"ĠIPS":37632,"Ġ(%":37633,"Ġseminar":37634,"focus":37635,"civil":37636,"Ġ1860":37637,"intosh":37638,"Ġcontinual":37639,"Ġabbrevi":37640,"ĠSok":37641,"ocobo":37642,"XM":37643,"Ġfrantic":37644,"Ġunavoidable":37645,"Ġartery":37646,"Ġannotations":37647,"bath":37648,"Climate":37649,"Ġdors":37650,"ĠSlide":37651,"coord":37652,"ĠReload":37653,"ĠLDL":37654,"ĠLovecraft":37655,"Ġunimagin":37656,"Ġresembled":37657,"Ġbarracks":37658,"np":37659,"Ġsurrogate":37660,"Ġcategorized":37661,"ãĤ©":37662,"Ġvaccinated":37663,"Ġdrainage":37664,"Ġindist":37665,"ĠWhatsApp":37666,"Ġ1870":37667,"olerance":37668,"invoke":37669,"amorph":37670,"Ġreconnect":37671,"Ġemanc":37672,"Ġblindness":37673,"Ġ1280":37674,"internet":37675,"collar":37676,"Ġaltru":37677,"Ġabyss":37678,"ĠTRI":37679,"657":37680,"Ġinfused":37681,"HEAD":37682,"Ġforestry":37683,"ĠWoody":37684,"ĠCi":37685,"wi":37686,"sam":37687,"784":37688,"holiday":37689,"Ġmogul":37690,"ĠFees":37691,"ĠDEN":37692,"Internal":37693,"urbed":37694,"fusc":37695,"atom":37696,"ĠIllusion":37697,"Ġpolled":37698,"Ġflap":37699,"Ġcoax":37700,"LGBT":37701,"Analy":37702,"ĠSections":37703,"ĠCaliforn":37704,"emn":37705,"Ġhither":37706,"ĠNIGHT":37707,"Ġnailed":37708,"ĠPipeline":37709,"391":37710,"oof":37711,"ĠPrimal":37712,"verend":37713,"Ġslashing":37714,"Ġretri":37715,"aviour":37716,"Ġdeparting":37717,"gil":37718,"ISC":37719,"Ġmidway":37720,"Ġultrasound":37721,"Ġbehaving":37722,"ĠTara":37723,"classes":37724,"Virtual":37725,"ĠColonial":37726,"Ġstripping":37727,"Ġorchestrated":37728,"ĠGraves":37729,"452":37730,"ĠIronically":37731,"ĠWriters":37732,"Ġlends":37733,"ĠManz":37734,"Ġraven":37735,"Ġoxidative":37736,"Ġ266":37737,"ELF":37738,"actually":37739,"ascar":37740,"Draft":37741,"Ġfavourable":37742,"Ġhumiliating":37743,"Ġfidelity":37744,"ĠHof":37745,"ĠXuan":37746,"496":37747,"Ġlayered":37748,"atis":37749,"790":37750,"Ġpaycheck":37751,"iton":37752,"Kar":37753,"ĠVMware":37754,"ĠFarmer":37755,"Ġservic":37756,"glomer":37757,"Ġslump":37758,"ĠFabric":37759,"ĠDOC":37760,"esting":37761,"Ġreassure":37762,"Ġphyl":37763,"volt":37764,"itory":37765,"Rules":37766,"Ġoxidation":37767,"Ġprized":37768,"Ġmistress":37769,"ĠDjango":37770,"WARN":37771,"åij":37772,"Ġencode":37773,"ĠFeedback":37774,"Ġstupidity":37775,"Ian":37776,"ĠYugoslavia":37777,"ר":37778,"acl":37779,"UTE":37780,"1977":37781,"Ġqualifies":37782,"Ġpulses":37783,"pretty":37784,"Ġfroze":37785,"Ġss":37786,"Iterator":37787,"Ġurgently":37788,"Ġmailed":37789,"ĠCham":37790,"Ġsustaining":37791,"Ġbasil":37792,"Ġpuppies":37793,"ilant":37794,"ĠPLEASE":37795,"lap":37796,"aceous":37797,"Fear":37798,"ĠMastery":37799,"automatic":37800,"ĠTAG":37801,"Ġantim":37802,"agles":37803,"473":37804,"frames":37805,"Ġwhispers":37806,"ĠWhoever":37807,"Ġbravery":37808,"ĠUKIP":37809,"ractions":37810,"\"\"\"":37811,"Ġtame":37812,"Ġparted":37813,"everything":37814,"CONT":37815,"Ġindebted":37816,"Ġaddr":37817,"rek":37818,"IRED":37819,"Ġeminent":37820,"clinton":37821,"Ġousted":37822,"Ġreviewer":37823,"Ġmeltdown":37824,"Ġrearr":37825,"ĠYao":37826,"thereal":37827,"abyte":37828,"Ġstumbling":37829,"Ġbatches":37830,"Ġ259":37831,"Ġcontraceptive":37832,"Ġprostitute":37833,"ensis":37834,"Decl":37835,"ĠStrikes":37836,"Military":37837,"ĠOath":37838,"vacc":37839,"ppings":37840,"052":37841,"ĠpartName":37842,"amping":37843,"Reports":37844,"KI":37845,"CHR":37846,"Ġsubtly":37847,"swers":37848,"Blake":37849,"usual":37850,"Ġcontestants":37851,"Ġcartridges":37852,"ĠGREAT":37853,"Ġblush":37854,"ĠâĢº":37855,"472":37856,"Ġreasoned":37857,"ãĥ¤":37858,"paralleled":37859,"Ġdyn":37860,"agate":37861,"Ġnightly":37862,"åĨ":37863,"556":37864,"Ġsemantic":37865,"ĠAdvoc":37866,"Ġ!!":37867,"Ġdisagrees":37868,"ĠBW":37869,"Veh":37870,"Ġharming":37871,"Ġembraces":37872,"Ġstrives":37873,"Ġinland":37874,"ĠKard":37875,"Ġheats":37876,"ĠGinny":37877,"utan":37878,"ernaut":37879,"ylene":37880,"ĠElev":37881,"JD":37882,"Ġhars":37883,"ĠStarr":37884,"Ġskysc":37885,"Ġcollaborators":37886,"Usually":37887,"Ġrevolutions":37888,"ĠSTATS":37889,"Ġdismantle":37890,"Ġconfidently":37891,"Ġkinetic":37892,"Ali":37893,"Ġpercentile":37894,"Ġextracting":37895,"illian":37896,"estead":37897,"Ġphysicists":37898,"ĠMarshal":37899,"Ġfellowship":37900,"Ġdashed":37901,"ĠUR":37902,"ĠSioux":37903,"ĠCompact":37904,"amide":37905,"Python":37906,"ĠLeigh":37907,"ĠPharmac":37908,"istrates":37909,"herical":37910,"Ġfue":37911,"ĠEmin":37912,"Ġ({":37913,"ĠNeighborhood":37914,"Ġdisrupting":37915,"ĠDup":37916,"Ġgland":37917,"ĠSev":37918,"ĠMarian":37919,"argon":37920,"ĠDund":37921,"Ġ":46904,"ĠPhilips":46905,"ĠKafka":46906,"Ġupheaval":46907,"Ġsentimental":46908,"Ġsax":46909,"ĠAkira":46910,"serial":46911,"Matrix":46912,"Ġelecting":46913,"Ġcommenter":46914,"ĠNebula":46915,"plets":46916,"ĠNadu":46917,"ĠAdren":46918,"Ġenshr":46919,"ĠRAND":46920,"financial":46921,"ĠClyde":46922,"utherford":46923,"Ġsignage":46924,"Ġdeline":46925,"Ġphosphate":46926,"roversial":46927,"fascist":46928,"ĠVall":46929,"ĠBethlehem":46930,"Ġfors":46931,"Ġenglish":46932,"Solid":46933,"Nature":46934,"Ġva":46935,"ĠGuests":46936,"Ġtantal":46937,"Ġautoimmune":46938,";;;;;;;;;;;;":46939,"ĠTotally":46940,"ĠOv":46941,"Ġdefences":46942,"ĠCoconut":46943,"Ġtranquil":46944,"Ġploy":46945,"Ġflavours":46946,"ĠFlask":46947,"ãĤ¨ãĥ«":46948,"ĠWeston":46949,"ĠVolvo":46950,"870":46951,"Ġmicrophones":46952,"verbal":46953,"RPG":46954,"Ġiii":46955,";}":46956,"028":46957,"Ġheadlined":46958,"Ġprimed":46959,"Ġhoard":46960,"ĠShad":46961,"ĠENTER":46962,"Ġtriangular":46963,"Ġcapit":46964,"lik":46965,"ĠAncients":46966,"Ġlash":46967,"Ġconvol":46968,"Ġcolonel":46969,"enemy":46970,"Gra":46971,"Ġpubs":46972,"utters":46973,"Ġassigns":46974,"ĠPenet":46975,"ĠMonstrous":46976,"ĠBowen":46977,"ilver":46978,"Haunted":46979,"ĠDing":46980,"started":46981,"plin":46982,"Ġcontaminants":46983,"ĠDOE":46984,"ffen":46985,"ĠTechnician":46986,"Ry":46987,"Ġrobbers":46988,"Ġhotline":46989,"ĠGuardiola":46990,"ĠKaufman":46991,"rower":46992,"ĠDresden":46993,"ĠAlpine":46994,"Elf":46995,"Ġfmt":46996,"ĠSard":46997,"urses":46998,"gpu":46999,"Unix":47000,"Ġunequivocally":47001,"ĠCitizenship":47002,"quad":47003,"mire":47004,"ĠSweeney":47005,"Battery":47006,"615":47007,"Ġpancakes":47008,"Ġoats":47009,"Maps":47010,"ĠContrast":47011,"mbudsman":47012,"ĠEPS":47013,"Ġsubcommittee":47014,"Ġsourcing":47015,"Ġsizing":47016,"ĠBuffer":47017,"ĠMandatory":47018,"Ġmoderates":47019,"ĠPatterns":47020,"ĠChocobo":47021,"ĠZan":47022,"ĠSTATES":47023,"ĠJudging":47024,"ĠInher":47025,"*:":47026,"Ġbil":47027,"ĠYen":47028,"Ġexhilar":47029,"ollower":47030,"zers":47031,"Ġsnug":47032,"maximum":47033,"Ġdespicable":47034,"ĠPACK":47035,"ĠAnnex":47036,"Ġsarcastic":47037,"Ġlatex":47038,"Ġtamp":47039,"ĠSao":47040,"bah":47041,"ĠReverend":47042,"ĠChinatown":47043,"ĠAUT":47044,"documented":47045,"ĠGABA":47046,"ĠCanaan":47047,"ĠÙħ":47048,"Ġgoverns":47049,"prev":47050,"Esc":47051,"ĠEstimates":47052,"OSP":47053,"Ġendeavour":47054,"ĠClosing":47055,"ometime":47056,"everyone":47057,"Ġworsen":47058,"Ġscanners":47059,"Ġdeviations":47060,"ĠRobotics":47061,"ĠCompton":47062,"Ġsorcerer":47063,"Ġendogenous":47064,"Ġemulation":47065,"ĠPiercing":47066,"ĠAph":47067,"ĠSocket":47068,"Ġbould":47069,"ĠOU":47070,"ĠBorderlands":47071,"Ġ1863":47072,"Gordon":47073,"ĠWTO":47074,"Ġrestricts":47075,"Ġmosaic":47076,"Ġmelodies":47077,"çĦ":47078,"Tar":47079,"Ġdisson":47080,"ĠProvides":47081,"Ġ......":47082,"bek":47083,"FIX":47084,"Ġbroom":47085,"anship":47086,"Doctors":47087,"Ġnerds":47088,"ĠRegions":47089,"naissance":47090,"Ġmete":47091,"Ġcrept":47092,"plings":47093,"Ġgirlfriends":47094,"knit":47095,"igent":47096,"owe":47097,"Ġushered":47098,"ĠBaz":47099,"Mobil":47100,"434":47101,"ĠPresents":47102,"origin":47103,"Ġinsomnia":47104,"ĠAux":47105,"439":47106,"ĠChili":47107,"irsch":47108,"GAME":47109,"Ġgestation":47110,"algia":47111,"romising":47112,"$,":47113,"crow":47114,"ĠInspection":47115,"atomic":47116,"Relations":47117,"JOHN":47118,"roman":47119,"ĠClockwork":47120,"ĠBakr":47121,"mone":47122,"MET":47123,"Ġthirsty":47124,"Ġbc":47125,"Ġfaculties":47126,"Rum":47127,"Ġnuance":47128,"ĠDarius":47129,"pleting":47130,"fters":47131,"etchup":47132,"Registration":47133,"ĠKE":47134,"Rah":47135,"Ġpreferential":47136,"ĠLash":47137,"ĠHH":47138,"Valid":47139,"ĠNAV":47140,"Ġstarve":47141,"ĠGong":47142,"zynski":47143,"ĠActress":47144,"Ġwik":47145,"Ġunaccompanied":47146,"lvl":47147,"Bride":47148,"ADS":47149,"ĠCommando":47150,"ĠVaughn":47151,"Wallet":47152,"Ġhopping":47153,"ĠVie":47154,"Ġcaveats":47155,"Ġalas":47156,"ifled":47157,"abuse":47158,"661":47159,"Ġibn":47160,"Ġgul":47161,"Ġrobbing":47162,"til":47163,"ILA":47164,"Ġmitigating":47165,"Ġaptly":47166,"Ġtyrant":47167,"Ġmidday":47168,"ĠGilmore":47169,"ĠDecker":47170,"Ġ§§":47171,"partial":47172,"Exactly":47173,"Ġphenotype":47174,"Ġ[+]":47175,"ĠPlex":47176,"ĠIps":47177,"versions":47178,"Ġebook":47179,"Ġchic":47180,"gross":47181,"\":\"\"},{\"":47182,"ĠSurprisingly":47183,"Morgan":47184,"Ġresidues":47185,"ĠConfederation":47186,"infeld":47187,"Ġlyr":47188,"moderate":47189,"Ġperpendicular":47190,"VK":47191,"Ġsynchronized":47192,"Ġrefreshed":47193,"Ġadore":47194,"ĠTorment":47195,"olina":47196,"Ġ2600":47197,"ItemTracker":47198,"Ġpies":47199,"ĠFAT":47200,"ĠRHP":47201,"048":47202,"ĠRESP":47203,"ĠBJ":47204,"allows":47205,"Pand":47206,"Ġunwelcome":47207,"ĠVoc":47208,"ĠBastard":47209,"ĠOW":47210,"ĠLAR":47211,"ĠHealer":47212,"Environmental":47213,"ĠKenyan":47214,"ĠTrance":47215,"ĠPats":47216,"Ġaliases":47217,"ĠGarfield":47218,"Ġcampaigner":47219,"Ġadvancements":47220,"ĠOkinawa":47221,"ĠCoh":47222,"owsky":47223,"Ġstarved":47224,"Ġsizeable":47225,"Ġ:-)":47226,"ĠmRNA":47227,"Ġsuspensions":47228,"istar":47229,"Scotland":47230,"Prin":47231,"------------------------------------------------":47232,"Ġ502":47233,"Ġteaspoons":47234,"Ġ1050":47235,"Ġcoercive":47236,"ĠMasonic":47237,"edded":47238,"ĠPassenger":47239,"Ġlatt":47240,"Ġbraces":47241,"ĠSteal":47242,"ĠNYT":47243,"ĠKats":47244,"ĠCelest":47245,"aez":47246,"Tu":47247,"ĠCoulter":47248,"ðŁĺ":47249,"Flickr":47250,"ĠWilmington":47251,"iths":47252,"++;":47253,"Ġvending":47254,"Ġnegro":47255,"ĠPhi":47256,"ĠYellowstone":47257,"Callback":47258,"Ġshampoo":47259,"ĠShades":47260,"wat":47261,"Ġsuperhuman":47262,"Ġridiculed":47263,"Ġholiest":47264,"ombo":47265,"Ġinterns":47266,"Ġhone":47267,"ĠParagu":47268,"URI":47269,"Ġdangling":47270,"ãĤ»":47271,"sov":47272,"ictional":47273,"availability":47274,"Ġrevocation":47275,"Ġdow":47276,"inic":47277,"ĠTHEIR":47278,"Ġiso":47279,"Ġoutings":47280,"ĠLethal":47281,"Ġ)))":47282,"Ġinaccur":47283,"Ġoutlandish":47284,"Ġanus":47285,"letico":47286,"idon":47287,"lol":47288,"Ġunregulated":47289,"Ġsuccumbed":47290,"Ġcuff":47291,"ĠWasteland":47292,"letal":47293,"Ġsubstr":47294,"Ġcoffers":47295,"Ġautomakers":47296,"ovi":47297,"ĠXue":47298,"ĠDaytona":47299,"Ġjarring":47300,"Ġfumes":47301,"Ġdisbanded":47302,"zik":47303,"itton":47304,"Ġstrikingly":47305,"Ġspores":47306,"Adapter":47307,".):":47308,"ĠLyndon":47309,"ivalry":47310,"Ġorally":47311,"Ġtumultuous":47312,"Ġdispleasure":47313,"Ġcones":47314,"orrect":47315,"Ġappease":47316,"Ġderby":47317,"ĠTripoli":47318,"ĠAless":47319,"Ġpoked":47320,"ĠGuilty":47321,"vP":47322,"Enough":47323,"Ġoriginals":47324,"699":47325,"Ġrabbi":47326,"Ġproverbial":47327,"Ġpostpone":47328,"elope":47329,"ĠMisty":47330,"Ġstaffed":47331,"ĠUnemployment":47332,"reditary":47333,"Ġdiligent":47334,"recomm":47335,"measures":47336,"asin":47337,"825":47338,"Ġponds":47339,"Ġmmol":47340,"ĠSAR":47341,"ĠCARE":47342,"Ġ371":47343,"Ġclenched":47344,"ĠCorsair":47345,"Ġcaricature":47346,"zn":47347,"attach":47348,"ĠSchro":47349,"speak":47350,"painted":47351,"ĠSuc":47352,"ĠENT":47353,"Ġcellul":47354,"ĠPaid":47355,"diagn":47356,"WHERE":47357,"Ġtexted":47358,"Barn":47359,"Ġretracted":47360,"ĠReferred":47361,"Sav":47362,"Ġupkeep":47363,"Ġworkplaces":47364,"ĠTokens":47365,"Ġamplify":47366,"clinical":47367,"Ġmultic":47368,"mberg":47369,"Ġconvoluted":47370,"Region":47371,"565":47372,"ĠTopic":47373,"Ġsnail":47374,"Ġsaline":47375,"Ġinsurrection":47376,"ĠPetr":47377,"forts":47378,"BAT":47379,"ĠNavajo":47380,"Ġrudimentary":47381,"ĠLaksh":47382,"ONDON":47383,"Measure":47384,"Ġtransformer":47385,"ĠGoddard":47386,"Ġcoincides":47387,"irin":47388,"Rex":47389,"ĠBok":47390,"quit":47391,"Ġshotguns":47392,"Ġproletarian":47393,"Ġscorp":47394,"ĠAda":47395,"514":47396,"Ġslander":47397,"recorded":47398,"Ġembell":47399,"risome":47400,"Ġapologizing":47401,"ĠMulcair":47402,"ĠGibraltar":47403,"Cla":47404,"Ġallot":47405,"ĠAttention":47406,"Ġ433":47407,"leave":47408,"Ġwhine":47409,"ĠIssa":47410,"ĠFaust":47411,"ĠBarron":47412,"heny":47413,"Ġvictimized":47414,"Jews":47415,"Ġnurturing":47416,"ettel":47417,"Winged":47418,"ĠSubtle":47419,"Ġflavorful":47420,"ĠReps":47421,"enged":47422,"callback":47423,"Ġdirectional":47424,"Ġclasp":47425,"ĠDirections":47426,"planet":47427,"iculture":47428,"Helper":47429,"icion":47430,"acia":47431,"Ġç¥ŀ":47432,"Ġsurges":47433,"Ġcanoe":47434,"ĠPremiership":47435,"been":47436,"Ġdefied":47437,"ĠTrooper":47438,"Ġtripod":47439,"Ġgasp":47440,"ĠEuph":47441,"ĠAds":47442,"vernight":47443,"highly":47444,"Role":47445,"Ġentangled":47446,"ĠZeit":47447,"618":47448,"ĠRusty":47449,"Ġhavens":47450,"ĠVaughan":47451,"HAEL":47452,"ĠSERVICE":47453,"/,":47454,"Ġstricken":47455,"Ġdelusions":47456,"Ġbis":47457,"ĠHaf":47458,"Ġgratification":47459,"Ġenticing":47460,"UNCH":47461,"Adams":47462,"ĠOLED":47463,"ĠBeetle":47464,"Ġ1899":47465,"ĠSOFTWARE":47466,"ategor":47467,"VL":47468,"ĠTotem":47469,"ĠGators":47470,"ATURES":47471,"Ġimpedance":47472,"Registered":47473,"ĠCary":47474,"ĠAerial":47475,"onne":47476,"enium":47477,"Ġdred":47478,"ĠBeg":47479,"Ġconcurrently":47480,"Ġsuperpower":47481,"ĠXan":47482,"jew":47483,"imester":47484,"ĠDickinson":47485,"âĶģ":47486,"Fla":47487,"Ġpree":47488,"ĠRollins":47489,"©¶æ":47490,"Ġdenomination":47491,"ĠLana":47492,"516":47493,"Ġinciting":47494,"scribed":47495,"juries":47496,"ĠWonders":47497,"approximately":47498,"Ġsuspending":47499,"Ġmountainous":47500,"ĠLaugh":47501,"oidal":47502,"Ns":47503,"Detect":47504,")=":47505,"ĠLuthor":47506,"ĠSchwarzenegger":47507,"ĠMuller":47508,"ĠDevi":47509,"ecycle":47510,"Jar":47511,"613":47512,"ĠLongh":47513,"Bah":47514,"ĠSPORTS":47515,"nw":47516,"Ġrefinement":47517,"Ġwaterways":47518,"Ġdiner":47519,"Blade":47520,"683":47521,"Fac":47522,"Ġinitials":47523,"Ġrog":47524,"Ġparanormal":47525,"BUT":47526,"Ġ[(":47527,"ĠSwanson":47528,"ĠMesh":47529,"âĸ¬":47530,"Improve":47531,"ĠRadiation":47532,"ĠEsther":47533,"ĠEsk":47534,"ĠAly":47535,"iky":47536,"Ġirrad":47537,"ĠBuckingham":47538,"Ġrefill":47539,"Ġ._":47540,"Repe":47541,"CONCLUS":47542,"Ġdifferentiated":47543,"Ġchirop":47544,"ĠAtkins":47545,"Pattern":47546,"Ġexcise":47547,"Ġcabal":47548,"NSA":47549,"ĠSTA":47550,"ĠSIL":47551,"ĠParaly":47552,"Ġrye":47553,"ĠHowell":47554,"ĠCountdown":47555,"nesses":47556,"alysed":47557,"Ġresize":47558,"ãĤ½":47559,"Ġbudgetary":47560,"ĠStras":47561,"wang":47562,"Ġapiece":47563,"Ġprecincts":47564,"Ġpeach":47565,"Ġskyline":47566,"Ġ353":47567,"popular":47568,"Appearances":47569,"ĠMechanics":47570,"ĠDevOnline":47571,"Sullivan":47572,"Zen":47573,"Ġpu":47574,"opolis":47575,"544":47576,"Ġdeform":47577,"Ġcounteract":47578,"ĠLange":47579,"Ġ417":47580,"Console":47581,"774":47582,"Ġnodding":47583,"Ġpopulism":47584,"Ġhep":47585,"Ġcounselling":47586,"compliance":47587,"UFF":47588,"Ġundeniably":47589,"Ġrailing":47590,"ĠHorowitz":47591,"ĠSimone":47592,"ĠBungie":47593,"Ġak":47594,"ĠTalks":47595,"xff":47596,"flake":47597,"Crash":47598,"Ġsweaty":47599,"Ġbanquet":47600,"ĠOFFIC":47601,"Ġinventive":47602,"Ġastronomer":47603,"ĠStamford":47604,"ĠScare":47605,"ĠGREEN":47606,"olicited":47607,"Ġrusher":47608,"Ġcentrist":47609,"ighting":47610,"Ġsubclass":47611,"Ġdisav":47612,"Ġdefund":47613,"ĠNanto":47614,"ociate":47615,"mast":47616,"Ġpacif":47617,"Ġmend":47618,"eers":47619,"immigration":47620,"ESSION":47621,"Ġnumbering":47622,"Ġlaughable":47623,"ĠEnded":47624,"viation":47625,"emark":47626,"Pitt":47627,"Ġmeticulous":47628,"ĠLF":47629,"Ġcongratulated":47630,"ĠBirch":47631,"Ġswayed":47632,"Ġsemifinals":47633,"Ġhumankind":47634,"matter":47635,"ĠEquip":47636,"opausal":47637,"Said":47638,"ĠLayout":47639,"Ġvoicing":47640,"Ġthug":47641,"Ġpornographic":47642,"IPS":47643,"Ġmoaning":47644,"Ġgrievance":47645,"Ġconfessions":47646,"escal":47647,"TEXTURE":47648,"Authent":47649,"osaurus":47650,"Purchase":47651,"Ġrelegation":47652,"alter":47653,"Ġ³³":47654,"Ġriddled":47655,"Ġogre":47656,"ĠLowell":47657,"Occup":47658,"Eat":47659,"ĠHyder":47660,"ĠAdviser":47661,"Commerce":47662,"Hunt":47663,"ĠOrth":47664,"ĠCompetitive":47665,"ĠCLA":47666,"CDC":47667,"Ġsalads":47668,"Fle":47669,"Ġindustrialized":47670,"`,":47671,"ĠOWN":47672,"Ġbeck":47673,"ĠParticularly":47674,"oubt":47675,"ĠmM":47676,"ĠHussain":47677,"ĠChennai":47678,"Ġ920":47679,"Ġappointing":47680,"ĠCullen":47681,",,,,,,,,":47682,"Ġpores":47683,"verified":47684,"Ġbiochemical":47685,"emate":47686,"Ġcowardly":47687,"ĠHelsinki":47688,"ĠEthiopian":47689,"SOURCE":47690,"ERC":47691,"estro":47692,"Ġbiotech":47693,"ĠSour":47694,"Ġbrewer":47695,"Bloomberg":47696,"Ġintensify":47697,"Glass":47698,"anco":47699,"ĠFDR":47700,"greSQL":47701,"ĠFires":47702,"©¶æ¥µ":47703,"eco":47704,"1001":47705,"ĠHomeless":47706,"Ġinstantaneous":47707,"ĠHaste":47708,"igel":47709,"Diamond":47710,"Ġpaving":47711,"Ġlandfill":47712,"Ġdads":47713,"houn":47714,":]":47715,"Ġincendiary":47716,"ĠLivingston":47717,"ĠHilbert":47718,"ĠChecks":47719,"styles":47720,"inators":47721,"ĠClive":47722,"phrine":47723,"Ġchimpanzees":47724,"Ġpall":47725,"ĠJM":47726,"ĠAadhaar":47727,"ðĿ":47728,"Ġachievable":47729,"disabled":47730,"PET":47731,"OOOOOOOO":47732,"Mot":47733,"Ġintangible":47734,"Ġballet":47735,"ĠWebs":47736,"ĠEstimated":47737,"Effects":47738,"Ġbailed":47739,"Joshua":47740,"Ġturbulence":47741,"Ġoccupant":47742,"ĠDaylight":47743,"Ġ361":47744,"meet":47745,"Ġstatically":47746,"Ġonlook":47747,"Ġki":47748,"illegal":47749,"Ġvelvet":47750,"Ġdehydration":47751,"Ġacquies":47752,"ĠRez":47753,"akura":47754,"ĠUpton":47755,"atro":47756,"Ġincomprehensible":47757,"Ġbackdoor":47758,"ĠRhino":47759,"727":47760,"Ġmaths":47761,")+":47762,"Ġheresy":47763,"Ġdf":47764,"ĠRoche":47765,"ĠLydia":47766,"Ġpancreat":47767,"reply":47768,"arrell":47769,"Ġsolicitation":47770,"Ġcircadian":47771,"BIP":47772,"Ġforay":47773,"Ġcryptic":47774,"izu":47775,"imeo":47776,"ĠTomato":47777,"ĠHoms":47778,"examination":47779,"Ġquarry":47780,"ĠValiant":47781,"ĠJericho":47782,"ĠINCLUD":47783,"Ġ1840":47784,"519":47785,"Ġresists":47786,"Ġsnapshots":47787,"ĠSpur":47788,"ĠAntiqu":47789,"Login":47790,"Ġbestselling":47791,"Ġantic":47792,"ĠSutherland":47793,"ãĤ¢ãĥ«":47794,"Ġ~/":47795,"ĠParm":47796,"èĥ":47797,"Pages":47798,"intensity":47799,"Ġimmobil":47800,"Ġ1865":47801,"zzo":47802,"Ġnifty":47803,"Ġfentanyl":47804,"ĠPreservation":47805,"ophen":47806,"Ġdarts":47807,"ĠDinosaur":47808,"pointers":47809,"ĠRite":47810,"suggest":47811,"awareness":47812,"ĠSheridan":47813,"Ġstances":47814,"Ġsorcery":47815,"Ġperjury":47816,"ĠNikola":47817,"iever":47818,"Ġfiance":47819,"ĠJordanian":47820,"ĠBalloon":47821,"Ġnab":47822,"Ġkb":47823,"Ġhumanities":47824,"ĠTanaka":47825,"hillary":47826,"Ġconsultancy":47827,"ĠZub":47828,"Ġremission":47829,"Ġconfid":47830,"CHQ":47831,"ĠFug":47832,"Ġimprovis":47833,"Yep":47834,"/_":47835,"Ġunwillingness":47836,"Ġportfolios":47837,"055":47838,"ĠInstructor":47839,"aiman":47840,"Ġclaimants":47841,"Mbps":47842,"ĠBye":47843,"received":47844,"Tweet":47845,"Ġindemn":47846,"riz":47847,"amara":47848,"Nat":47849,"Ġevaluates":47850,"ĠLur":47851,"epad":47852,"FOX":47853,"ĠThro":47854,"Ġrusty":47855,"Ġbedrock":47856,"ĠOprah":47857,"JB":47858,"Ġmanipulative":47859,"Ġwillful":47860,"Ġrelapse":47861,"Ġextant":47862,"Theme":47863,"Sensor":47864,"ĠStability":47865,"govern":47866,"Ġpoppy":47867,"Ġknack":47868,"Ġinsulated":47869,"ĠTile":47870,"ĠExtrem":47871,"Ġuntold":47872,"Ġconverge":47873,"Ġrefuel":47874,"igroup":47875,"Ġdistortions":47876,"Ġravaged":47877,"Ġmechanically":47878,"ĠReilly":47879,"ĠNose":47880,"ĠIncarnation":47881,"ĠBecky":47882,"abbling":47883,"Ġtaco":47884,"Ġrake":47885,"Ġmelancholy":47886,"Ġillustrious":47887,"ĠDartmouth":47888,"Guide":47889,"ĠRazer":47890,"ĠBenz":47891,"Ultimate":47892,"ĠSurprise":47893,"Ġpageant":47894,"offer":47895,"Whoever":47896,"Ġwiser":47897,"Ġchemist":47898,"ĠHELL":47899,"ĠBulk":47900,"Ġplutonium":47901,"ĠCOVER":47902,"Ö¼":47903,"failed":47904,"Ġtirelessly":47905,"Ġinfertility":47906,"ĠTrident":47907,"ĠShowtime":47908,"ĠCiv":47909,"Vice":47910,"requires":47911,"ittance":47912,"Ġuncontrolled":47913,"interesting":47914,"561":47915,"Ġinnovate":47916,"ategic":47917,"Lie":47918,"ĠSelling":47919,"Ul":47920,"Ġsavior":47921,"ĠTosh":47922,"Ġswast":47923,"PASS":47924,"Ġrink":47925,"Ġcardio":47926,"ĠIro":47927,"udi":47928,"Ġvantage":47929,"Ġvans":47930,"ĠNiño":47931,"+=":47932,"Ġpropagate":47933,"":49029,"Ġleukemia":49030,"Ġeluc":49031,"Ġannouncer":49032,"ĠLithuan":49033,"ĠArmageddon":49034,"åĩ":49035,"Lenin":49036,"ĠRuk":49037,"Ġpepp":49038,"ĠRomantic":49039,"ĠPIT":49040,"ĠInterstellar":49041,"ĠAtkinson":49042,"Raid":49043,"Js":49044,"Goal":49045,"Course":49046,"Ġvanishing":49047,"esley":49048,"ĠRounds":49049,"Elsa":49050,"593":49051,"Ġredundancy":49052,"ĠSTAND":49053,"Ġprophetic":49054,"Ġhabitable":49055,"ryu":49056,"Ġfaintly":49057,"MODE":49058,"Ġflanked":49059,"IRC":49060,"Awesome":49061,"Ġspurious":49062,"ĠZah":49063,"ĠMSG":49064,"Ġshading":49065,"Ġmotivational":49066,"ĠSantana":49067,"ĠSPR":49068,"Ġexcruciating":49069,"omial":49070,"ĠMiko":49071,"ĠLeopard":49072,"Abyss":49073,"Ġ[|":49074,"dirty":49075,"Ġbaths":49076,"Ġdemoral":49077,"andre":49078,"PB":49079,"Ġunification":49080,"Ġsacrament":49081,"Ġ[&":49082,"Ġpriceless":49083,"Ġgelatin":49084,"Ġemanating":49085,"ĠAllaah":49086,"986":49087,"Ġoutburst":49088,"Ġeras":49089,"ĠXVI":49090,"ĠSPI":49091,"Ott":49092,"ĠLazarus":49093,"PLIED":49094,"Flying":49095,"blogs":49096,"Wisconsin":49097,"Raven":49098,"Ġrebate":49099,"Ġcreeps":49100,"ĠSpan":49101,"ĠPainter":49102,"ĠKira":49103,"ĠAmos":49104,"ĠCorvette":49105,"Consumer":49106,"ĠRecover":49107,"cki":49108,"Ġpesky":49109,"ĠInvention":49110,"Companies":49111,"Ġchallengers":49112,"ademic":49113,"ĠUkrainians":49114,"ĠNeurolog":49115,"ĠForsaken":49116,"Ġentrants":49117,"Ġembattled":49118,"Ġdefunct":49119,"ĠGlacier":49120,"Ġpoisons":49121,"ĠHorses":49122,"makes":49123,"ĠDirt":49124,"Ġ423":49125,"hhh":49126,"ĠTransformation":49127,"QUIRE":49128,"..................":49129,"Ġtraveller":49130,"ĠSexy":49131,"ĠKern":49132,"ipolar":49133,"Ġransomware":49134,"oooooooooooooooo":49135,"Ec":49136,"ruby":49137,"Professional":49138,"ĠOutbreak":49139,"argument":49140,"Grey":49141,"ĠFifa":49142,"ĠCHO":49143,"ĠFORM":49144,"ĠAmtrak":49145,"-[":49146,"Ġcradle":49147,"Ġantioxidants":49148,"ãģ®å®":49149,"736":49150,"ĠNASL":49151,"ĠContributions":49152,"Indiana":49153,"ĠSTEP":49154,"CSS":49155,"Ġsalient":49156,"Ġallocations":49157,"yrights":49158,"Ġmashed":49159,"ĠCutter":49160,"Sexual":49161,"Ġpounded":49162,"Ġfanbase":49163,"Ġcasc":49164,"ĠTransparency":49165,"Ġanalytic":49166,"ĠSummoner":49167,"×ŀ":49168,"ĠADC":49169,"detail":49170,"Ġvanquished":49171,"Ġcrabs":49172,"arie":49173,"Destroy":49174,"ĠSack":49175,"Ġtransistor":49176,"Alabama":49177,"ĠKoen":49178,"ĠFisheries":49179,"cone":49180,"Ġannexed":49181,"ĠMGM":49182,"esa":49183,"Ġfaked":49184,"ĠCongratulations":49185,"Ġhindered":49186,"Ġcorrectional":49187,"ĠITV":49188,"leeve":49189,"Ġinappropriately":49190,"licks":49191,"Ġtrespass":49192,"Ġpaws":49193,"Ġnegotiator":49194,"ĠChristensen":49195,"limits":49196,"ĠDianne":49197,"Ġelegance":49198,"ĠContracts":49199,"anke":49200,"Obj":49201,"Ġvigilance":49202,"Ġcastles":49203,"ĠNAD":49204,"ĠHolo":49205,"Ġemphatically":49206,"ĠTitus":49207,"ĠServing":49208,"ĠRichie":49209,"ĠPigs":49210,"568":49211,"Ġanimosity":49212,"ĠAttributes":49213,"ĠUriel":49214,"MQ":49215,"myra":49216,"ĠApplicant":49217,"Ġpsychiatrists":49218,"ĠVij":49219,"ĠAbby":49220,"agree":49221,"Push":49222,"ĠkWh":49223,"hiba":49224,"Ġincite":49225,"ĠWeasley":49226,"ĠTaxi":49227,"ministic":49228,"hyper":49229,"ĠFarn":49230,"Ġ601":49231,"ĠNationwide":49232,"Fake":49233,"952":49234,"Ġmaize":49235,"Ġinteracted":49236,"Ġtransitioned":49237,"Ġparasitic":49238,"Ġharmonic":49239,"Ġdecaying":49240,"Ġbaseless":49241,"nsics":49242,"Ġtranspired":49243,"Ġabundantly":49244,"ĠForensic":49245,"Ġtreadmill":49246,"ĠJav":49247,"aband":49248,"Ġsshd":49249,"Ġfrontman":49250,"ĠJakarta":49251,"oller":49252,"drops":49253,"ĠSERVICES":49254,"romptu":49255,"ophical":49256,"hospital":49257,"bledon":49258,"645":49259,"Ġmidrange":49260,"ĠEVENT":49261,"culated":49262,"rawled":49263,"Ġperched":49264,"Ġoverboard":49265,"ĠPeel":49266,"ĠPwr":49267,"ĠCarth":49268,"ĠCOMPLE":49269,"coe":49270,"shall":49271,"Ġdeterrence":49272,"METHOD":49273,"ĠAbsent":49274,"MEN":49275,"Ġsill":49276,"ĠLEVEL":49277,"York":49278,"Ġsinners":49279,"ĠOPEC":49280,"ĠNur":49281,"ĠDesigns":49282,"selection":49283,"Ġunworthy":49284,"CHA":49285,"Ġstrengthens":49286,"883":49287,"edly":49288,"Ġslicing":49289,"Ġmalnutrition":49290,"Ġfilmmaking":49291,"ĠPolk":49292,"urated":49293,"Ġ421":49294,"breakers":49295,"!'\"":49296,"Ġwetlands":49297,"ĠDiscrimination":49298,"Ġallowable":49299,"Ġsteered":49300,"ĠSicily":49301,"SAM":49302,"Ġmustache":49303,"Ġmids":49304,"Ġclipped":49305,"Ġcirculate":49306,"Ġbrittle":49307,"ĠBuildings":49308,"raised":49309,"ĠRoundup":49310,"Ġwealthier":49311,"Ġoverwrite":49312,"Ġoverpowered":49313,"ĠGerrard":49314,"sites":49315,"PDATED":49316,"Ġacutely":49317,"ĠGamble":49318,"Ġpim":49319,"ĠKus":49320,"Typically":49321,"Deploy":49322,"ĠMoroccan":49323,"potion":49324,"combe":49325,"Ġvigilante":49326,"Ġ363":49327,"Stew":49328,"ĠBagg":49329,"Ġresided":49330,"ĠSpo":49331,"Ġremnant":49332,"Ġemptiness":49333,"brainer":49334,"Ġoutpatient":49335,"priority":49336,"Ġleptin":49337,"ĠPayton":49338,"ĠGleaming":49339,"ĠShed":49340,"ĠPolo":49341,"ĠMormonism":49342,"restricted":49343,"arlane":49344,"wx":49345,"Ġcreatine":49346,"ĠAnon":49347,"ĠSTUD":49348,"ĠJUL":49349,"ĠTee":49350,"528":49351,"089":49352,"Ġhatched":49353,"Dispatch":49354,"ĠComposite":49355,"Ġ451":49356,"puff":49357,"ĠXCOM":49358,"ĠOrn":49359,"ĠTHANK":49360,"ENDED":49361,"ĠAsheville":49362,"ĠÃľ":49363,"Ġmango":49364,"ĠSlightly":49365,"worldly":49366,"ĠWander":49367,"ĠExpand":49368,"ĠChr":49369,"Mist":49370,"Ġorthodoxy":49371,"ĠUNESCO":49372,"regate":49373,"Elsewhere":49374,"kie":49375,"irled":49376,"Ġtopple":49377,"Ġadoptive":49378,"ĠLegs":49379,"dress":49380,"ĠSagan":49381,"bare":49382,"ĠGlou":49383,"Crunch":49384,"Ġhelpers":49385,"Ġchronically":49386,"ĠHuma":49387,"10000":49388,"Ġaccommodating":49389,"äºĶ":49390,"Ġwrinkles":49391,"Ġdodged":49392,"fourth":49393,"Ġprecon":49394,"Ġcompressor":49395,"ĠKare":49396,"Ġevict":49397,"ĠWarwick":49398,"imar":49399,"Ġmodernization":49400,"Ġbandwagon":49401,"Ġrefuted":49402,"Ġnetted":49403,"ĠNaples":49404,"ĠGenie":49405,"perors":49406,"Ġfielded":49407,"Ġdere":49408,"ĠParables":49409,"lees":49410,"Ġtrout":49411,"aspers":49412,"Ġnihil":49413,"Ġhappiest":49414,"Ġfloppy":49415,"ĠLoft":49416,"ĠHeard":49417,"Ġunison":49418,"Ġlug":49419,"ĠRedmond":49420,"classic":49421,"Supporters":49422,"SHIP":49423,"GMT":49424,"Ġfuelled":49425,"çIJ":49426,"Ġdd":49427,"ĠEminem":49428,"Ġ1897":49429,"NYSE":49430,"Ġsecretaries":49431,"ĠFIA":49432,"ĠCanaveral":49433,"Favorite":49434,"Ġpomp":49435,"Ġdetainee":49436,"ership":49437,"aimon":49438,"iour":49439,"ĠApex":49440,"Ġplantations":49441,"amia":49442,"acion":49443,"Rust":49444,"Ġtowed":49445,"ĠTruly":49446,"577":49447,"Ġsheltered":49448,"rider":49449,"Wo":49450,"Ġlair":49451,"ĠIntelligent":49452,"improve":49453,"matically":49454,"Ġetiquette":49455,"adra":49456,"allo":49457,"ĠJuno":49458,"anything":49459,"ĠStruggle":49460,"ĠPredict":49461,"ĠGrimes":49462,"ĠAMERICA":49463,"ctx":49464,"ĠSituation":49465,"WOOD":49466,"Ġsoluble":49467,"meier":49468,"Ġintolerable":49469,"angering":49470,"Ġuninterrupted":49471,"Ġtooltip":49472,"Ġinterrogated":49473,"Ġgunned":49474,"ĠSneak":49475,"æѦ":49476,"Ġtether":49477,"Ġcrumble":49478,"Lens":49479,"Ġclustered":49480,"ĠSyl":49481,"ĠHasan":49482,"Ġdystopian":49483,"wana":49484,"Ġjoystick":49485,"ĠThib":49486,"ammu":49487,"Tomorrow":49488,"546":49489,"Ġovercame":49490,"Ġminimized":49491,"ceptor":49492,"Runner":49493,"ENGTH":49494,"ĠBrenda":49495,"ĠAchievements":49496,"Ġtorches":49497,"Ġrapport":49498,"ĠInvestigator":49499,"ĠHandling":49500,"relation":49501,"grey":49502,"815":49503,"Ġkcal":49504,"ĠCommands":49505,"dq":49506,"Ġcurls":49507,"Ġbearer":49508,"Ġcynicism":49509,"itri":49510,"ĠUseful":49511,"Bee":49512,"DCS":49513,"Ġabras":49514,"Pract":49515,"BILITIES":49516,"712":49517,"Ġdebugger":49518,"Ġdebtor":49519,"ĠLia":49520,"ĠKers":49521,"Ġexacerbate":49522,"ĠStacy":49523,"ĠBland":49524,"ĠScenes":49525,"Ġbranching":49526,"âĸĪâĸĪâĸĪâĸĪâĸĪâĸĪâĸĪâĸĪ":49527,"apeake":49528,"Ġsalsa":49529,"Ġmishand":49530,"ĠKonami":49531,"ĠNib":49532,"Ġanecdote":49533,"Ġagreeable":49534,"Ïī":49535,"ĠNathaniel":49536,"ĠHeisman":49537,"ĠBeware":49538,"Ġ1886":49539,"spective":49540,"691":49541,"522":49542,"Ġinhibits":49543,"Ġhashing":49544,"Ġ1889":49545,"å°Ĩ":49546,"vich":49547,"Pure":49548,"Ġsolidly":49549,"Ġaspirin":49550,"imaru":49551,"Ġstreetcar":49552,"ĠUCS":49553,"ĠJudd":49554,"Ġflashbacks":49555,"pins":49556,"Ġ1440":49557,"ĠUNHCR":49558,"ĠSymptoms":49559,"TIT":49560,"538":49561,"Fra":49562,"%);":49563,"Ġooz":49564,"Ġcurfew":49565,"Ġcalmed":49566,"Ġparticipates":49567,"TeX":49568,"Ġnonsensical":49569,"Ġfullback":49570,"ĠDeL":49571,"monkey":49572,"hari":49573,"Ġmetabolites":49574,"Ġlooted":49575,"ĠALWAYS":49576,"ĠBCC":49577,"Lt":49578,"ochet":49579,"Bone":49580,"Ġvetoed":49581,"Ġgcc":49582,"ĠCLICK":49583,"Ġ1888":49584,"saf":49585,"Ġstiffness":49586,"Ġlowly":49587,"ĠGeh":49588,"verson":49589,"orset":49590,"Ġunforeseen":49591,"Ġanesthesia":49592,"ĠOptical":49593,"Ġreconstructed":49594,"ĠTup":49595,"shows":49596,"NEWS":49597,"ĠNewspaper":49598,"ĠASA":49599,"tera":49600,"Numbers":49601,"Ġinexplicable":49602,"×ij":49603,"Ġhardness":49604,"untarily":49605,"ĠAcer":49606,"gradient":49607,"ARDIS":49608,"Ġwoodland":49609,"Ġmetaphors":49610,"ĠWembley":49611,"ĠPavel":49612,"philis":49613,"Ġrewriting":49614,"Ġperceptual":49615,"Ġ1070":49616,"worms":49617,"ĠDowns":49618,"Ġunsurprisingly":49619,"Ġtagging":49620,"flame":49621,"Ġlitres":49622,"Ġbounces":49623,"ĠBabe":49624,"shut":49625,"Ġoverdoses":49626,"ĠSheila":49627,"ĠChau":49628,"ĠBless":49629,"Capture":49630,"ĠSignificant":49631,"ĠScion":49632,"Ġ389":49633,"ĠMcH":49634,"ĠTitanium":49635,"ĠMeal":49636,"ameda":49637,"agents":49638,"aggressive":49639,"Billy":49640,"763":49641,"ĠSaying":49642,"DERR":49643,"itone":49644,"Collins":49645,"Bound":49646,"Ġbolted":49647,"ĠDMCA":49648,"953":49649,"Ġuniqueness":49650,"Ġepigen":49651,"unci":49652,"antam":49653,"Ġreckoning":49654,"chairs":49655,"OGR":49656,"ĠSenegal":49657,"Ġ1862":49658,"relevant":49659,"Ġ¯":49660,"Ġpharmacies":49661,"ĠGeral":49662,"vier":49663,"Yan":49664,"ORPG":49665,"Ġrabid":49666,"bending":49667,"ĠUNITED":49668,"Ġ465":49669,"Assembly":49670,"Ġweep":49671,"Ġbehest":49672,"ĠMothers":49673,"ĠJace":49674,"hid":49675,"Ġwhirlwind":49676,"ĠUNIVERS":49677,"Ġutopian":49678,"Ġkidnap":49679,"Philipp":49680,"Kin":49681,"893":49682,"Ġlivestream":49683,"ĠMISS":49684,"Ġsubversive":49685,"ĠTechniques":49686,"ĠJUSTICE":49687,"ĠBASE":49688,"Ġ387":49689,"Ġassailants":49690,"ĠHardcore":49691,"Ġsprinkled":49692,"ĠPse":49693,"éļ":49694,"printed":49695,"ĠHau":49696,"ORGE":49697,"ĠTOUR":49698,"Ġlaced":49699,"Ġitch":49700,"Giving":49701,"Ġported":49702,"781":49703,"////////////////////////////////":49704,"breeding":49705,"Ġlogger":49706,"ĠHOL":49707,"innie":49708,"Firstly":49709,"Ġembryonic":49710,"Ġdelegated":49711,"pai":49712,"OIL":49713,"Ġcentrally":49714,"ĠRx":49715,"ĠScouting":49716,"Dutch":49717,"Ġhereditary":49718,"ĠCruiser":49719,"sat":49720,"529":49721,"ĠMarriott":49722,"othermal":49723,"Ġprohibitions":49724,"Earn":49725,"ĠStab":49726,"ĠColleges":49727,"ĠBelief":49728,"stretched":49729,"ĠLH":49730,"ĠEntityItem":49731,"CIA":49732,"Ġunrem":49733,"Ġlaureate":49734,"Ġdenominations":49735,"summary":49736,"hler":49737,"Spect":49738,"ĠKlaus":49739,"ĠBeans":49740,"Ġinsur":49741,"ĠPAX":49742,"Ġfielder":49743,"ĠVet":49744,"ĠSparrow":49745,"zie":49746,"ĠSQ":49747,"ĠMondays":49748,"ĠOffline":49749,"ĠLerner":49750,"ĠExtensions":49751,"Ireland":49752,"Ġpatronage":49753,"Ġcontrasted":49754,"ĠMania":49755,"hirt":49756,"Moscow":49757,"Ġcondemns":49758,"ĠAnge":49759,"Ġcomposing":49760,"ĠPepe":49761,"ĠPaddock":49762,"Ġheterogeneity":49763,"Ġideologically":49764,"Ġfishes":49765,"Ġcursing":49766,"ĠRutherford":49767,"ĠFloating":49768,"ĠAmelia":49769,"Tea":49770,"Synopsis":49771,"Ġstunts":49772,"Ġbead":49773,"Ġstocking":49774,"ĠMILL":49775,"obook":49776,"massive":49777,"\\<":49778,"Ġhump":49779,"ĠPreferences":49780,"EngineDebug":49781,"geist":49782,"ĠNieto":49783,"omever":49784,"ishy":49785,"evaluate":49786,"colonial":49787,"Alternative":49788,"ĠGoPro":49789,"ĠVortex":49790,"ĠNETWORK":49791,"ansky":49792,"Secure":49793,"ĠThrust":49794,"Snake":49795,"Ġparcels":49796,"Ġsamurai":49797,"Ġactresses":49798,"Nap":49799,"MF":49800,"iferation":49801,"Beer":49802,"523":49803,"ĠIly":49804,"ointment":49805,"Ping":49806,"Ġstriped":49807,"ĠMellon":49808,"ossession":49809,"Ġneutron":49810,"endium":49811,"Ġaph":49812,"ĠFlavoring":49813,"Ġ383":49814,"Ġresponsiveness":49815,"ĠJindal":49816,"ĠHitchcock":49817,"Denver":49818,"ĠDRAGON":49819,"smanship":49820,"ĠDupl":49821,"Ġsly":49822,"Ġwebcam":49823,"ĠTwain":49824,"ĠDarling":49825,"iliate":49826,"consumer":49827,"DIT":49828,"Ġnamesake":49829,"Ġunorthodox":49830,"Ġfuner":49831,"ĠPLoS":49832,"ĠCONTROL":49833,"ozyg":49834,"oglobin":49835,"FACE":49836,"ERG":49837,"ĠDia":49838,"ĠFiesta":49839,"cele":49840,"034":49841,"Ġenclave":49842,"âĸ¬âĸ¬":49843,"onement":49844,"alist":49845,"Mand":49846,"Ġhomegrown":49847,"ĠFancy":49848,"Ġconceptions":49849,"ĠContains":49850,"ureen":49851,"Ġreiterate":49852,"Ġmeager":49853,"Ġinstallments":49854,"Spawn":49855,"627":49856,"Ġphotoc":49857,"ĠCabrera":49858,"ĠRosenthal":49859,"ĠLansing":49860,"isner":49861,"Ġinvests":49862,"ĠUFOs":49863,"EXP":49864,"Hardware":49865,"Ġtragically":49866,"Ġconcedes":49867,"ieft":49868,"cham":49869,"borgh":49870,"ĠSchr":49871,"ĠMelanie":49872,"ĠHoy":49873,"Ġvisitation":49874,"Ġidiosyncr":49875,"Ġfractions":49876,"Ġforeskin":49877,"obos":49878,"Ġpoaching":49879,"ĠVIEW":49880,"Ġstimulates":49881,"ĠGork":49882,"canon":49883,"MIC":49884,"ĠNemesis":49885,"ĠIndra":49886,"ĠDMV":49887,"Ġ529":49888,"Ġinspecting":49889,"Ġgrandma":49890,"ĠWhedon":49891,"ĠShant":49892,"ĠPurg":49893,"ikan":49894,"ĠTeg":49895,"ĠCLR":49896,"zac":49897,"Victoria":49898,"ĠVerify":49899,"ionics":49900,"Ġpartying":49901,"ĠMou":49902,"colour":49903,"Ġtestimonies":49904,"lations":49905,"Ġpressuring":49906,"hiro":49907,"acers":49908,"Ġfid":49909,"angler":49910,"ĠCSI":49911,"Ġhereafter":49912,"Ġdissidents":49913,"reporting":49914,"iphany":49915,"chev":49916,"Ġsolitude":49917,"Ġlobe":49918,"Ġindis":49919,"Ġcredential":49920,"recent":49921,"adult":49922,"ĠNirvana":49923,"ĠFranchise":49924,"Layer":49925,"Hyp":49926,"ĠBerkshire":49927,"Ġwills":49928,"tif":49929,"Ġtotem":49930,"ĠJudah":49931,"repair":49932,"Instant":49933,"548":49934,"Ġembassies":49935,"Ġbottleneck":49936,"Ġbount":49937,"Ġtypew":49938,"ĠAlvin":49939,"jing":49940,"imilar":49941,"Rush":49942,"Ġbrim":49943,"ĠHELP":49944,"Aim":49945,"]'":49946,"Ġpassively":49947,"Ġbounded":49948,"ĠRated":49949,"Ġcriminality":49950,"Ġbiomark":49951,"Ġdispatcher":49952,"ĠTowards":49953,"Ġ+++":49954,"righteous":49955,"frog":49956,"ĠPanc":49957,"Carter":49958,"032":49959,"æ©Ł":49960,"Ġultraviolet":49961,"ĠLicensed":49962,"ĠTata":49963,"ĠBlessing":49964,"ĠGAM":49965,"Ġchemically":49966,"ĠSeaf":49967,"ĠRELE":49968,"ĠMercenary":49969,"capitalist":49970,"Ġformulations":49971,"Ġannihilation":49972,"ĠVerb":49973,"ĠArgon":49974,"Ġunloaded":49975,"Ġmorphed":49976,"Ġconquering":49977,"backer":49978,"IELD":49979,"Ġthefts":49980,"Ġfrontrunner":49981,"ĠRoyale":49982,"ĠFundamental":49983,"elight":49984,"Chip":49985,"necessary":49986,"ayn":49987,"ĠSlip":49988,"Ġ448":49989,"cerned":49990,"Pause":49991,"Ġshockingly":49992,"ĠABV":49993,"Ġcomposure":49994,"733":49995,"ĠMotorsport":49996,"ahime":49997,"Murray":49998,"Mach":49999,"Ġgrids":50000,"Ġdebian":50001,"Ġfurthermore":50002,"Ġdexterity":50003,"ĠCollections":50004,"oslov":50005,"ilage":50006,"bj":50007,"ĠMonteneg":50008,"ĠstrutConnector":50009,"Ġmassacres":50010,"Ġbriefs":50011,"fetched":50012,"uvian":50013,"olition":50014,"Failure":50015,"emonic":50016,"Ġflared":50017,"Ġclaimant":50018,"Ġcures":50019,"Ġgiveaways":50020,"ĠSubstance":50021,"alions":50022,"Ġcringe":50023,"ĠKul":50024,"Ġaristocracy":50025,"ĠUlster":50026,"olated":50027,"housing":50028,"ĠMIS":50029,"Ġglared":50030,"ĠWilhelm":50031,"needs":50032,"lambda":50033,"builders":50034,"ĠVIS":50035,"Ġradiator":50036,"ĠGhostbusters":50037,"Ġ436":50038,"actual":50039,"Ġherds":50040,"ça":50041,"watching":50042,"Ġcountering":50043,"Charge":50044,"Ġcharred":50045,"Ġwarheads":50046,"Ġiodine":50047,"ĠMacy":50048,"041":50049,"Ġdepartures":50050,"ĠSins":50051,"Ġdyed":50052,"ĠConcepts":50053,"gado":50054,"713":50055,"Ġquotations":50056,"Ġgist":50057,"ĠChristy":50058,"Ġantigen":50059,"ĠHemp":50060,"ĠDrawn":50061,"ĠBarg":50062,"ezvous":50063,"Ġpaternity":50064,"Ġardu":50065,"ĠAnchorage":50066,"ĠRik":50067,"Ġoverloaded":50068,"ĠUsername":50069,"ĠTammy":50070,"ĠNau":50071,"ĠCellular":50072,"Ġwaning":50073,"Ġrodent":50074,"ĠWorcester":50075,"ilts":50076,"ĠTad":50077,"Ġdwellings":50078,"Ġbullish":50079,"431":50080,"Ġretaliate":50081,"Ġmigraine":50082,"ĠChevron":50083,"CHECK":50084,"Ġdonkey":50085,"crim":50086,"SPA":50087,"ĠAnalog":50088,"Ġmarquee":50089,"ĠHaas":50090,"Bir":50091,"ĠGDDR":50092,"ĠDownloads":50093,"Ġwillpower":50094,"ĠForth":50095,"ĠRecorded":50096,"Ġimpossibility":50097,"ĠLogged":50098,"ĠFranks":50099,"ĠRatt":50100,"initions":50101,"Ġcleaners":50102,"Ġsorely":50103,"Ġflickering":50104,"ĠExamination":50105,"catching":50106,"alloween":50107,"Msg":50108,"Ġdunno":50109,"Fa":50110,"Ġdysph":50111,"crazy":50112,".''.":50113,"Ġmainline":50114,"Ġcs":50115,"Ġptr":50116,"ĠWally":50117,"igun":50118,"951":50119,"ĠBigfoot":50120,"fights":50121,"Ġretrieving":50122,"Jr":50123,"Ġduplication":50124,"ĠExplan":50125,"Ġrelational":50126,"Ġquaint":50127,"Ġbiscuits":50128,"Ġado":50129,"Ġshudder":50130,"Ġantidote":50131,"blooded":50132,"ksh":50133,"Ġsauces":50134,"Ġreinvest":50135,"Ġdispensary":50136,"ĠDiver":50137,"Ġ9000":50138,"student":50139,"Ġinsepar":50140,"escap":50141,"Ġtoddlers":50142,"ĠGPIO":50143,"ĠAssignment":50144,"headers":50145,"Ġlackluster":50146,"Ġaback":50147,"956":50148,"Ġtoolbar":50149,"745":50150,"Ġoust":50151,"Ġcontemplation":50152,"ĠPRESIDENT":50153,"Ġ458":50154,"======":50155,"Ġguaranteeing":50156,"ĠHeist":50157,"ĠCannes":50158,"Ļ½":50159,"Ġcollaborator":50160,"ĠAmp":50161,"Ġgou":50162,"ĠSHALL":50163,"stories":50164,"783":50165,"Ġmobilized":50166,"Ġbrood":50167,"ĠLU":50168,"ĠðŁij":50169,"Ġrefin":50170,"ĠAnthropology":50171,"vind":50172,"illi":50173,"Ġwarranties":50174,"ĠBabel":50175,"Ġswath":50176,"Ġcaches":50177,"Ġantagonists":50178,"artifacts":50179,"Ġhotly":50180,"ĠStarts":50181,"ĠGö":50182,"zag":50183,"!!!!!":50184,"Ġscourge":50185,"Ġconspiring":50186,"ruits":50187,"reverse":50188,"ĠSheen":50189,"ĠJesuit":50190,"ĠGiovanni":50191,"adies":50192,"Ġbuttocks":50193,"earcher":50194,"acan":50195,"Ġvolleyball":50196,"Ġshrouded":50197,"Ġscoreboard":50198,"bats":50199,"ĠIPM":50200,"Ġasses":50201,"Ġderegulation":50202,"ĠTelegram":50203,"ĠReboot":50204,"Ġ7000":50205,"ĠCanary":50206,"Ġkernels":50207,"ĠFrançois":50208,"ĠDuff":50209,"ĠPon":50210,"ĠLeica":50211,"ĠGarmin":50212,"Ġorphans":50213,"ĠClaudia":50214,"Ġcalendars":50215,"ĠLeilan":50216,"ento":50217,"Rocket":50218,"Ġbrunch":50219,"ĠHawking":50220,"ainers":50221,"Ġsensibilities":50222,"ĠkW":50223,"ĠKand":50224,"Ġreclaimed":50225,"Ġinterestingly":50226,"ש":50227,"romy":50228,"JM":50229,"ĠEnhancement":50230,"bush":50231,"Skip":50232,"Ġrappers":50233,"Ġgazing":50234,"pedia":50235,"athlon":50236,"Revolution":50237,"Ġsnipers":50238,"Ġreverted":50239,"Ġconglomerate":50240,"Terry":50241,"794":50242,"Ġharsher":50243,"Ġdesolate":50244,"ĠHitman":50245,"Commission":50246,"Ġ(/":50247,"âĢ¦.\"":50248,"Compar":50249,"Ġamplification":50250,"ominated":50251,"Ġregress":50252,"ĠCollider":50253,"Ġinformants":50254,"Ġgazed":50255,"<|endoftext|>":50256} \ No newline at end of file diff --git a/experiments/elasticdnn/gpt_neo/125m_ckpt/config.json b/experiments/elasticdnn/gpt_neo/125m_ckpt/config.json new file mode 100644 index 0000000000000000000000000000000000000000..06db8066850aa0bb609cc71c77ae27dacaddde8b --- /dev/null +++ b/experiments/elasticdnn/gpt_neo/125m_ckpt/config.json @@ -0,0 +1,52 @@ +{ + "activation_function": "gelu_new", + "architectures": [ + "GPTNeoForCausalLM" + ], + "attention_dropout": 0, + "attention_layers": [ + "global", + "local", + "global", + "local", + "global", + "local", + "global", + "local", + "global", + "local", + "global", + "local" + ], + "attention_types": [ + [ + [ + "global", + "local" + ], + 6 + ] + ], + "bos_token_id": 50256, + "embed_dropout": 0, + "eos_token_id": 50256, + "gradient_checkpointing": false, + "hidden_size": 768, + "initializer_range": 0.02, + "intermediate_size": null, + "layer_norm_epsilon": 1e-05, + "max_position_embeddings": 2048, + "model_type": "gpt_neo", + "num_heads": 12, + "num_layers": 12, + "resid_dropout": 0, + "summary_activation": null, + "summary_first_dropout": 0.1, + "summary_proj_to_labels": true, + "summary_type": "cls_index", + "summary_use_proj": true, + "transformers_version": "4.9.0.dev0", + "use_cache": true, + "vocab_size": 50257, + "window_size": 256 +} diff --git a/experiments/elasticdnn/gpt_neo/125m_ckpt/merges.txt b/experiments/elasticdnn/gpt_neo/125m_ckpt/merges.txt new file mode 100644 index 0000000000000000000000000000000000000000..226b0752cac7789c48f0cb3ec53eda48b7be36cc --- /dev/null +++ b/experiments/elasticdnn/gpt_neo/125m_ckpt/merges.txt @@ -0,0 +1,50001 @@ +#version: 0.2 +Ġ t +Ġ a +h e +i n +r e +o n +Ġt he +e r +Ġ s +a t +Ġ w +Ġ o +e n +Ġ c +i t +i s +a n +o r +e s +Ġ b +e d +Ġ f +in g +Ġ p +o u +Ġa n +a l +a r +Ġt o +Ġ m +Ġo f +Ġ in +Ġ d +Ġ h +Ġan d +i c +a s +l e +Ġt h +i on +o m +l l +en t +Ġ n +Ġ l +s t +Ġ re +v e +Ġ e +r o +l y +Ġb e +Ġ g +Ġ T +c t +Ġ S +i d +o t +Ġ I +u t +e t +Ġ A +Ġ is +Ġ on +i m +a m +o w +a y +a d +s e +Ġth at +Ġ C +i g +Ġf or +a c +Ġ y +v er +u r +Ġ u +l d +Ġs t +Ġ M +' s +Ġ he +Ġ it +at ion +it h +i r +c e +Ġy ou +i l +Ġ B +Ġw h +o l +Ġ P +Ġw ith +Ġ 1 +t er +c h +Ġa s +Ġw e +Ġ ( +n d +i ll +Ġ D +i f +Ġ 2 +a g +er s +k e +Ġ " +Ġ H +e m +Ġc on +Ġ W +Ġ R +he r +Ġw as +Ġ r +o d +Ġ F +u l +at e +Ġa t +r i +p p +o re +ĠT he +Ġs e +u s +Ġp ro +Ġh a +u m +Ġa re +Ġd e +a in +an d +Ġo r +ig h +es t +is t +a b +r om +Ġ N +t h +Ġc om +Ġ G +u n +o p +0 0 +Ġ L +Ġn ot +es s +Ġe x +Ġ v +re s +Ġ E +e w +it y +an t +Ġb y +e l +o s +or t +o c +q u +Ġf rom +Ġha ve +Ġs u +i ve +ou ld +Ġs h +Ġth is +n t +r a +p e +igh t +ar t +m ent +Ġa l +u st +en d +- - +al l +Ġ O +ac k +Ġc h +Ġ le +i es +re d +ar d +â Ģ +ou t +Ġ J +Ġa b +e ar +i v +al ly +ou r +o st +g h +p t +Ġp l +as t +Ġc an +a k +om e +u d +T he +Ġh is +Ġd o +Ġg o +Ġh as +g e +' t +Ġ U +r ou +Ġs a +Ġ j +Ġb ut +Ġw or +Ġa ll +e ct +Ġ k +am e +Ġw ill +o k +Ġw he +Ġthe y +id e +0 1 +f f +ic h +p l +t her +Ġt r +. . +Ġin t +i e +u re +ag e +Ġn e +i al +a p +in e +ic e +Ġm e +Ġo ut +an s +on e +on g +ion s +Ġwh o +Ġ K +Ġu p +Ġthe ir +Ġa d +Ġ 3 +Ġu s +at ed +ou s +Ġm ore +u e +o g +ĠS t +in d +i ke +Ġs o +im e +p er +. " +b er +i z +a ct +Ġon e +Ġsa id +Ġ - +a re +Ġyou r +c c +ĠT h +Ġc l +e p +a ke +ab le +i p +Ġcon t +Ġwh ich +i a +Ġ im +Ġab out +Ġwe re +ver y +u b +Ġh ad +Ġ en +Ġcom p +, " +ĠI n +Ġu n +Ġa g +i re +ac e +a u +ar y +Ġw ould +as s +r y +Ġ âĢ +c l +o ok +e re +s o +Ġ V +ig n +i b +Ġof f +Ġt e +v en +Ġ Y +i le +o se +it e +or m +Ġ2 01 +Ġre s +Ġm an +Ġp er +Ġo ther +or d +ul t +Ġbe en +Ġl ike +as e +an ce +k s +ay s +ow n +en ce +Ġd is +ct ion +Ġan y +Ġa pp +Ġs p +in t +res s +ation s +a il +Ġ 4 +ic al +Ġthe m +Ġhe r +ou nt +ĠC h +Ġa r +Ġ if +Ġthe re +Ġp e +Ġy ear +a v +Ġm y +Ġs ome +Ġwhe n +ou gh +ac h +Ġth an +r u +on d +ic k +Ġo ver +ve l +Ġ qu +Ċ Ċ +Ġs c +re at +re e +ĠI t +ou nd +p ort +Ġal so +Ġp art +f ter +Ġk n +Ġbe c +Ġt ime +en s +Ġ 5 +op le +Ġwh at +Ġn o +d u +m er +an g +Ġn ew +-- -- +Ġg et +or y +it ion +ing s +Ġj ust +Ġint o +Ġ 0 +ent s +o ve +t e +Ġpe ople +Ġp re +Ġit s +Ġre c +Ġt w +i an +ir st +ar k +or s +Ġwor k +ad e +o b +Ġs he +Ġo ur +w n +in k +l ic +Ġ1 9 +ĠH e +is h +nd er +au se +Ġh im +on s +Ġ [ +Ġ ro +f orm +i ld +at es +ver s +Ġon ly +o ll +Ġs pe +c k +e ll +am p +Ġa cc +Ġb l +i ous +ur n +f t +o od +Ġh ow +he d +Ġ ' +Ġa fter +a w +Ġat t +o v +n e +Ġpl ay +er v +ic t +Ġc ould +it t +Ġa m +Ġf irst +Ġ 6 +Ġa ct +Ġ $ +e c +h ing +u al +u ll +Ġcom m +o y +o ld +c es +at er +Ġf e +Ġbe t +w e +if f +Ġtw o +oc k +Ġb ack +) . +id ent +Ġu nder +rou gh +se l +x t +Ġm ay +rou nd +Ġp o +p h +is s +Ġd es +Ġm ost +Ġd id +Ġad d +j ect +Ġin c +f ore +Ġp ol +on t +Ġag ain +cl ud +ter n +Ġkn ow +Ġne ed +Ġcon s +Ġc o +Ġ . +Ġw ant +Ġse e +Ġ 7 +n ing +i ew +ĠTh is +c ed +Ġe ven +Ġin d +t y +ĠW e +at h +Ġthe se +Ġp r +Ġu se +Ġbec ause +Ġf l +n g +Ġn ow +ĠâĢ ĵ +c om +is e +Ġm ake +Ġthe n +ow er +Ġe very +ĠU n +Ġse c +os s +u ch +Ġe m +Ġ = +ĠR e +i ed +r it +Ġin v +le ct +Ġsu pp +at ing +Ġl ook +m an +pe ct +Ġ 8 +ro w +Ġb u +Ġwhe re +if ic +Ġyear s +i ly +Ġd iff +Ġsh ould +Ġre m +T h +I n +Ġe v +d ay +' re +ri b +Ġre l +s s +Ġde f +Ġr ight +Ġs y +) , +l es +00 0 +he n +Ġth rough +ĠT r +_ _ +Ġw ay +Ġd on +Ġ , +Ġ1 0 +as ed +Ġas s +ub lic +Ġre g +ĠA nd +i x +Ġ very +Ġin clud +ot her +Ġim p +ot h +Ġsu b +ĠâĢ Ķ +Ġbe ing +ar g +ĠW h += = +ib le +Ġdo es +an ge +r am +Ġ 9 +er t +p s +it ed +ation al +Ġb r +Ġd own +Ġman y +ak ing +Ġc all +ur ing +it ies +Ġp h +ic s +al s +Ġde c +at ive +en er +Ġbe fore +il ity +Ġwe ll +Ġm uch +ers on +Ġth ose +Ġsu ch +Ġ ke +Ġ end +ĠB ut +as on +t ing +Ġl ong +e f +Ġth ink +y s +Ġbe l +Ġs m +it s +a x +Ġo wn +Ġpro v +Ġs et +if e +ment s +b le +w ard +Ġsh ow +Ġp res +m s +om et +Ġo b +Ġs ay +ĠS h +t s +f ul +Ġe ff +Ġg u +Ġin st +u nd +re n +c ess +Ġ ent +ĠY ou +Ġgo od +Ġst art +in ce +Ġm ade +t t +st em +ol og +u p +Ġ | +um p +Ġhe l +ver n +ul ar +u ally +Ġa c +Ġm on +Ġl ast +Ġ2 00 +1 0 +Ġst ud +u res +ĠA r +sel f +ar s +mer ic +u es +c y +Ġm in +oll ow +Ġc ol +i o +Ġm od +Ġc ount +ĠC om +he s +Ġf in +a ir +i er +âĢ Ķ +re ad +an k +at ch +e ver +Ġst r +Ġpo int +or k +ĠN ew +Ġs ur +o ol +al k +em ent +Ġus ed +ra ct +we en +Ġs ame +ou n +ĠA l +c i +Ġdiff ere +Ġwh ile +---- ---- +Ġg ame +ce pt +Ġs im +.. . +Ġin ter +e k +Ġre port +Ġpro du +Ġst ill +l ed +a h +Ġhe re +Ġwor ld +Ġth ough +Ġn um +ar ch +im es +al e +ĠS e +ĠI f +/ / +ĠL e +Ġre t +Ġre f +Ġtr ans +n er +ut ion +ter s +Ġt ake +ĠC l +Ġcon f +w ay +a ve +Ġgo ing +Ġs l +u g +ĠA meric +Ġspe c +Ġh and +Ġbet ween +ist s +ĠD e +o ot +I t +Ġe ar +Ġagain st +Ġh igh +g an +a z +at her +Ġex p +Ġo p +Ġin s +Ġg r +Ġhel p +Ġre qu +et s +in s +ĠP ro +is m +Ġf ound +l and +at a +us s +am es +Ġp erson +Ġg reat +p r +Ġs ign +ĠA n +' ve +Ġs omet +Ġs er +h ip +Ġr un +Ġ : +Ġt er +ire ct +Ġf ollow +Ġd et +ic es +Ġf ind +1 2 +Ġm em +Ġc r +e red +e x +Ġex t +ut h +en se +c o +Ġte am +v ing +ou se +as h +at t +v ed +Ġsy stem +ĠA s +d er +iv es +m in +Ġle ad +ĠB l +c ent +Ġa round +Ġgo vern +Ġc ur +vel op +an y +Ġc our +al th +ag es +iz e +Ġc ar +od e +Ġl aw +Ġre ad +' m +c on +Ġre al +Ġsupp ort +Ġ1 2 +.. .. +Ġre ally +n ess +Ġf act +Ġd ay +Ġb oth +y ing +Ġs erv +ĠF or +Ġth ree +Ġw om +Ġm ed +od y +ĠThe y +5 0 +Ġex per +t on +Ġe ach +ak es +Ġc he +Ġc re +in es +Ġre p +1 9 +g g +ill ion +Ġg rou +ut e +i k +W e +g et +E R +Ġm et +Ġs ays +o x +Ġd uring +er n +iz ed +a red +Ġf am +ic ally +Ġha pp +ĠI s +Ġch ar +m ed +v ent +Ġg ener +i ent +p le +i et +re nt +1 1 +v es +pt ion +Ġ2 0 +form ation +Ġc or +Ġoff ic +ie ld +Ġto o +is ion +Ġin f +Ġ Z +t he +o ad +Ġp ublic +Ġpro g +r ic +* * +Ġw ar +Ġp ower +v iew +Ġf ew +Ġl oc +Ġdiffere nt +Ġst ate +Ġhe ad +' ll +Ġp oss +Ġst at +re t +ant s +Ġv al +Ġis s +Ġc le +i vers +an c +Ġex pl +Ġan other +Ġ Q +Ġa v +th ing +n ce +W h +Ġch ild +Ġs ince +i red +l ess +Ġl ife +Ġde velop +itt le +Ġde p +Ġp ass +ã ĥ +Ġt urn +or n +Th is +b ers +ro ss +ĠA d +Ġf r +Ġres p +Ġsec ond +o h +Ġ / +Ġdis c +Ġ & +Ġsomet hing +Ġcomp le +Ġ ed +Ġf il +Ġmon th +a j +u c +Ġgovern ment +Ġwith out +Ġle g +Ġd ist +Ġp ut +Ġqu est +an n +Ġpro t +2 0 +Ġne ver +i ence +Ġle vel +Ġar t +Ġth ings +Ġm ight +Ġeff ect +Ġcont ro +Ġc ent +Ġ1 8 +Ġall ow +Ġbel ie +ch ool +ot t +Ġinc re +Ġfe el +Ġres ult +Ġl ot +Ġf un +ot e +Ġt y +ere st +Ġcont in +Ġus ing +Ġb ig +2 01 +Ġas k +Ġb est +Ġ ) +I N +Ġo pp +3 0 +Ġnum ber +in ess +S t +le ase +Ġc a +Ġm ust +Ġd irect +Ġg l +Ġ < +Ġop en +Ġp ost +Ġcom e +Ġse em +ord ing +Ġwe ek +ate ly +it al +Ġe l +ri end +Ġf ar +Ġt ra +in al +Ġp ri +ĠU S +Ġpl ace +Ġfor m +Ġto ld +" : +ain s +at ure +ĠTr ump +Ġst and +Ġ # +id er +ĠF r +Ġne xt +Ġs oc +Ġp ur +Ġle t +Ġl ittle +Ġh um +Ġ i +r on +1 5 +Ġ1 5 +Ġcomm un +Ġm ark +ĠThe re +Ġw r +ĠTh at +Ġin formation +w ays +Ġb us +a pp +Ġinv est +m e +Ġh ard +ain ed +e ad +Ġim port +Ġapp ro +Ġt est +Ġt ri +Ġre st +os ed +Ġf ull +Ġc are +ĠS p +Ġc ase +O N +Ġs k +Ġl ess +Ġ + +Ġpart ic +ĠP l +ab ly +u ck +is hed +ch n +b e +Ġl ist +at or +Ġto p +Ġad v +ĠB e +ru ct +Ġd em +r ation +l ing +g y +re en +g er +Ġh ome +Ġle ft +Ġbet ter +Ġd ata +Ġ1 1 +Ġatt ack +Ġpro ble +l ine +ard s +Ġbe h +r al +ĠH ow +ĠS he +ar ge +Ġ -- +: // +Ġb ro +ĠP h +at s +Ġbu ild +w w +id ed +a im +as es +en cy +Ġm ain +in ed +Ġinclud ing +Ġ { +Ġg ot +Ġint erest +Ġke ep +Ġ X +Ġe as +ain ing +Ġcl ass +âĢ ¦ +ĠN o +Ġv ar +Ġsm all +amp le +A T +Ġ ide +ĠS o +Ġre ce +Ġpol it +Ġm ov +Ġpl an +Ġper cent +iv ing +Ġc amp +Ġp ay +1 4 +s c +is ed +Ġu nt +one y +pl oy +== == +Ġdid n +ĠI nd +el s +ert ain +Ġp os +__ __ +i ver +Ġpro cess +Ġprog ram +if ied +ĠR ep +1 6 +u ro +olog y +at ter +in a +Ġn ame +ĠA ll +Ġf our +Ġret urn +v ious +b s +Ġcall ed +Ġm ove +ĠS c +ir d +Ġgrou p +Ġb re +Ġm en +Ġc ap +t en +e e +Ġd ri +le g +he re +uth or +Ġp at +Ġcur rent +id es +Ġp op +t o +ent ion +Ġal ways +Ġm il +Ġwom en +Ġ1 6 +Ġo ld +iv en +ra ph +ĠO r +r or +ent ly +Ġn ear +ĠE x +re am +s h +Ġ1 4 +Ġf ree +iss ion +st and +ĠC on +al ity +us ed +1 3 +Ġdes ign +Ġch ange +Ġch ang +Ġb o +Ġv is +em ber +Ġb ook +read y +Ġk ill +2 5 +pp ed +Ġa way +Ġab le +Ġcount ry +Ġcon st +ar n +Ġor der +A R +i or +i um +or th +1 8 +ail able +Ġs w +Ġm illion +Ġ1 3 +at ic +t ed +ĠG o +Ġo per +en g +Ġth ing +aj or +con om +ĠCom m +Ġwh y +u red +ur al +Ġs chool +b y +ĠM ar +Ġa ff +Ġd ays +Ġan n +us h +an e +I f +e g +Ġpro f +Ġhe alth +ou th +B ut +ion al +. , +Ġs ol +Ġal ready +Ġ3 0 +Ġchar act +H e +Ġf riend +E S +i ans +ic le +' d +ĠO n +Ġle ast +Ġp rom +Ġd r +Ġh ist +it her +Ġ est +i qu +1 7 +s on +Ġte ll +Ġt alk +oh n +o int +le ction +A N +Ġunt il +au gh +Ġl ater +Ġ ve +Ġv iew +end ing +iv ed +Ġwor d +w are +Ġc ost +Ġen ough +Ġg ive +ĠUn ited +Ġte chn +are nt +O R +Ġp ar +ĠD r +Ġ201 6 +r ist +er ing +Ġ  +Ġl arge +s ide +ac y +cc ess +Ġw in +Ġimport ant +Ġ19 9 +Ġdoes n +Ġ1 7 +Ġbus iness +Ġcle ar +Ġre se +" , +ur y +Ġe qu +as ter +al f +ĠAmeric an +n ect +Ġex pect +ivers ity +Ġo cc +ĠF l +Ġk ind +Ġme an +Ġp ast +Ġde v +Ġb as +le t +ra ft +Ġor gan +Ġde l +Ġper form +Ġst ory +Ġse ason +ĠC ol +Ġcl aim +Ġc ame +Ġwith in +Ġl ine +Ġpro ject +ĠA t +Ġcontro l +end ed +ĠS y +Ġa ir +iz ation +Ġ * +le y +Ġm oney +id d +Y ou +f or +Ġfam ily +Ġm aking +Ġb it +Ġpol ice +Ġhapp en +Ġ vers +on y +u ff +ĠW hen +Ġs it +ide o +l f +is on +Ġsu re +g in +Ġapp ear +Ġl ight +Ġ es +o f +Ġw ater +Ġt imes +n ot +Ġg row +Ġcomp any +ĠT e +ow s +Ġm ar +our ce +i ol +ar m +b r +Ġex ample +Ġcon c +Ġf ore +ĠT o +p ro +E N +ri es +Ġ2 5 +ĠC an +ne y +Ġact ually +Ġe ver +ur ity +ak en +ap s +Ġt ax +Ġm ajor +am a +Ġof ten +er al +Ġhum an +Ġj ob +is ter +Ġav ailable +oc r +en n +a id +iv id +Ġrec ord +? " +Ġs ing +ĠA m +id ence +Ġnew s +st er +Ġe conom +Ġfollow ing +ĠB r +is ing +Ġh our +m ost +um ent +Ġse x +Ġdes c +Ġbec ome +ĠE d +Ġto ok +Ġha ving +Ġprodu ct +a ult +A s +ar ing +Ġme ans +Ġh op +un e +Ġch o +Ġc ertain +Ġn on +Ġde al +2 4 +le ment +oc i +en e +Ġs ide +ĠP r +ĠM ay +Ġre ason +u ed +c hed +ul ation +Ġe lect +Ġoffic ial +Ġposs ible +Ġh old +and s +ot s +Ġc ity +or ies +Ġse ver +Ġchild ren +Ġon ce +Ġact iv +l er +Ġn ight +it ions +ĠJ ohn +a pe +pl ay +Ġd one +Ġl im +Ġwork ing +ĠP res +or ld +e b +ĠC o +Ġb ody +ail s +ut es +ĠM r +Ġwhe ther +Ġa uthor +ro p +Ġpro per +Ġse en +) ; +Ġf ac +ĠS u +Ġcon d +it ing +Ġcour se +Ġ } +-------- -------- +a ign +Ġev ent +Ġen g +Ġp ot +Ġin tern +i am +Ġsh ort +em pt +ã Ĥ +ĠG od +il ar +8 0 +Ġor ig +I S +our n +ab ility +it ive +Ġd am +Ġ1 00 +Ġp ress +Ġdo ing +Ġprot ect +r ing +Ġthough t +Ġquest ion +re w +ĠW ar +Ġsever al +ĠSt ate +Ġg iven +Ġf und +ĠT w +Ġw ent +an ces +w ork +p or +m y +4 0 +Ġar g +art ment +ust om +Ġpol ic +Ġme et +Ġc reat +2 2 +ĠSt ates +Ġg ames +ra w +ut ure +Ġunder stand +ur s +ĠO b +l ish +s y +Ġm akes +Ġw on +ag on +Ġh tt +Ġl ove +ent ial +Ġcomple te +p ar +ĠI m +A L +Ġacc ount + ł +ore d +ver t +Ġ ident +Ġ201 5 +Ġother s +ĠM in +i ber +ver age +The re +ition al +d d +Ġpro b +Ġyou ng +Ġal ong +Ġacc ording +Ġy et +Ġmem bers +ĠWh at +o id +ĠM an +A nd +Ġam ong +a i +Ġem ploy +ĠR es +Ġ > +Ġinv ol +Ġl ow +a f +ĠC ar +Ġh ig +ĠO ne +ĠS ec +in ation +Ġlike ly +Ġan t +ag ed +ĠR uss +Ġb en +Ġre le +F or +b ack +ĠN ot +Ġpres ident +b all +Ġacc ess +ivid ual +ĠD em +ĠE uro +6 0 +Ġkn own +ir l +ĠG r +Ġear ly +u se +iet y +âĢ ĵ +Ġf ight +Ġs ent +Ġto day +Ġmark et +" . +Ġb ased +Ġstr ong +ur ther +Ġde b +m ber +Ġproble m +Ġde ath +Ġsoc ial +im ate +A S +ort un +Ġcamp aign +er y +C h +Ġe y +i ally +Ġm us +w h +p os +Ġ er +Ġsa f +Ġmonth s +ir on +Ġv iol +Ġf ive +Ġst re +Ġplay ers +in c +al d +y ear +a un +Ġsu ccess +Ġpres ent +ere nce +Ġ201 4 +Ġsu gg +Ġpartic ular +Ġtr y +Ġsugg est +ĠCh rist +on es +Ġpri v +2 3 +Ġc rit +Ġl and +Ġloc al +if y +2 9 +Ġa ut +E D +ĠG u +Ġm ult +Ġpolit ical +Ġask ed +Ġfor mer +it ter +ri pt +Ġcl ose +Ġp ract +ĠY ork +Ġget ting +Ġac ross +Ġcom b +Ġbelie ve +Ġ z +Ġto get +Ġtoget her +ĠC ent +ir c +Ġind ividual +ĠM c +2 7 +is k +ĠE ng +Ġf ace +Ġ2 4 +Ġval ue +Ġare a +e v +Ġw rit +ĠPres ident +Ġv ot +Ġke y +Ġm om +p ut +Ġany thing +Ġexper ience +att le +Ġm ind +a ff +om m +Ġf uture +g ed +Ġc ut +Ġto t +it ch +Ġv ideo +Ġinvest ig +Ġn et +ĠM y +r ict +i en +. ) +Ġimp ro +th ough +ward s +Ġcon nect +ĠM ed +sel ves +ens ive +m b +o ber +at ors +A n +Ġ5 0 +Ġre du +res ent +Ġab ove +Ġf re +ĠEuro pe +s w +Ġam ount +ĠA pp +Ġe ither +Ġmil it +Ġan al +Ġf ail +ĠE n +al es +Ġspec ial +Ġbl ack +I T +c her +Ġlook ing +Ġf ire +y n +Ġal most +o on +Ġstud y +Ġm iss +c hes +ro wn +Ġt re +Ġcommun ity +Ġmed ia +Ġf ood +Ġcom es +ĠUn iversity +Ġsing le +Wh at +u ly +Ġh alf +ag ue +h od +ĠRep ublic +Ġstart ed +Ġqu ick +ot o +b ook +Ġiss ue +it or +Ġel se +Ġcons ider +2 6 +ro du +Ġt aken +2 8 +9 9 +ĠW ith +Ġtr ue +Ġw a +Ġtr ad +Ġag o +Ġm ess +ie f +Ġadd ed +o ke +Ġb ad +Ġf av +3 3 +Ġsim ilar +as k +ĠD on +Ġcharact er +ort s +ĠH ouse +Ġreport ed +Ġty pe +v al +i od +ĠHow ever +Ġt arg +Ġent ire +pp ing +Ġhist ory +Ġl ive +ff ic +.... .... +ed eral +Ġtr ying +Ġdisc uss +ĠH ar +ac es +l ished +Ġse lf +os p +re st +Ġro om +el t +Ġf all +ol ution +Ġe t +Ġ x +Ġis n +Ġide a +b o +Ġs ound +ĠD ep +Ġsome one +ci ally +ull y +Ġf oc +Ġob ject +if t +ap er +Ġplay er +Ġr ather +Ġserv ice +as hing +ĠD o +ĠP art +ru g +m on +p ly +Ġm or +Ġnot hing +Ġprov ide +I C +un g +Ġpart y +Ġex ist +Ġm ag +7 0 +Ġr ul +Ġh ouse +Ġbeh ind +Ġhow ever +ĠW orld +Ġs um +Ġapp lic +Ġ ; +Ġfun ction +g r +ĠP ol +Ġfr ont +2 00 +Ġser ies +Ġt em +Ġty p +ill s +Ġo pt +Ġpoint s +Ġbel ow +itt ed +Ġspec ific +Ġ201 7 +um b +Ġr a +Ġpre vious +Ġpre t +re me +Ġc ustom +Ġcour t +ĠM e +Ġre pl +Ġwho le +g o +c er +Ġt reat +ĠA ct +Ġprob ably +Ġle arn +end er +ĠA ss +Ġvers ion +n ow +Ġche ck +ĠC al +R E +min ist +O n +our ces +Ġben ef +Ġd oc +Ġdet er +Ġen c +Ġsu per +Ġadd ress +Ġv ict +Ġ201 3 +Ġme as +t r +Ġf ield +W hen +Ġsign ific +u ge +Ġfe at +Ġcomm on +l oad +Ġbe gin +Ġbr ing +Ġa ction +er man +Ġdesc rib +Ġind ust +Ġwant ed +ri ed +m ing +Ġatt empt +4 5 +f er +Ġd ue +ress ion +# # +Ġsh all +Ġs ix +o o +Ġst ep +Ġp ub +Ġhim self +Ġ2 3 +Ġc op +Ġd est +Ġst op +A C +ib ility +Ġl ab +ic ult +Ġhour s +Ġcre ate +Ġf urther +ĠAmeric a +ĠC ity +Ġd ou +he ad +S T +ĠN orth +c ing +Ġn ational +u le +ĠIn st +Ġt aking +ĠQ u +ir t +Ġre d +Ġrese arch +v iron +ĠG e +Ġbre ak +an a +Ġsp ace +ater ial +Ġrec ent +ĠA b +Ġgener al +Ġh it +Ġper iod +Ġevery thing +ive ly +Ġph ys +Ġsay ing +an ks +Ġc ou +Ġc ult +ac ed +e al +u ation +Ġc oun +l u +Ġinclud e +Ġpos ition +ĠA fter +ĠCan ad +ĠE m +Ġim m +ĠR ed +Ġp ick +Ġcom pl +Ġm atter +re g +e xt +ang u +is c +o le +a ut +Ġcomp et +e ed +f ect +Ġ2 1 +ĠS en +ĠThe se +as ing +Ġcan not +Ġin it +Ġrel ations +ac hed +Ġb ar +Ġ4 0 +ĠT H +Ġ201 2 +Ġv ol +Ġg round +Ġsec urity +Ġup d +il t +3 5 +Ġconc ern +ĠJ ust +Ġwh ite +Ġseem s +ĠH er +pe cially +i ents +Ġann oun +Ġf ig +ight s +Ġst ri +l ike +id s +Ġs us +Ġw atch +Ġ â +Ġw ind +ĠC ont +Ġit self +Ġm ass +A l +y le +iqu e +ĠN ational +Ġab s +Ġp ack +Ġout side +Ġan im +Ġp ain +et er +Ġman ag +du ct +og n +Ġ ] +ĠSe pt +se c +o ff +ĠJ an +Ġf oot +ad es +Ġth ird +Ġm ot +Ġev idence +int on +Ġth reat +a pt +pl es +c le +Ġl o +Ġde cl +Ġit em +med i +Ġrep resent +om b +am er +Ġsignific ant +og raph +s u +Ġc al +i res +00 00 +I D +A M +Ġsim ply +Ġlong er +Ġf ile +O T +c he +S o +ate g +or g +ĠH is +Ġen er +Ġd om +Ġup on +il i +": " +Ġthem selves +Ġcom ing +Ġqu ite +Ġdiff icult +ĠB ar +il ities +re l +end s +c ial +6 4 +Ġwom an +ra p +y r +Ġne cess +ip s +Ġte xt +Ġrequ ire +Ġmilit ary +Ġre view +Ġresp ons +7 5 +Ġsub ject +Ġinst ead +Ġiss ues +Ġg en +" ," +Ġmin utes +Ġwe ap +r ay +am ed +t ime +b l +H ow +Ġc ode +ĠS m +Ġhig her +ĠSt e +r is +Ġp age +Ġstud ents +ĠIn tern +Ġmet hod +ĠA ug +ĠP er +ĠA g +Ġpolic y +ĠS w +Ġex ec +Ġac cept +um e +rib ut +Ġword s +Ġfin al +Ġchang es +ĠDem ocr +Ġfriend s +Ġres pect +Ġe p +Ġcomp an +iv il +Ġdam age +** ** +og le +viron ment +Ġne g +ent al +Ġa p +Ġtot al +iv al +! " +l im +Ġneed s +Ġag re +Ġdevelop ment +Ġa ge +ip le +2 1 +Ġresult s +ĠA f +S h +Ġg un +ĠOb ama +ro ll +Ġ @ +Ġright s +ĠB rit +Ġrun ning +Ġwas n +Ġp ort +Ġr ate +Ġpret ty +Ġtarg et +Ġsa w +Ġc irc +Ġwor ks +ic ro +al t +o ver +ww w +Th at +l ier +Ġevery one +ud e +Ġp ie +idd le +ra el +Ġr ad +Ġbl ock +Ġw alk +T o +ã ģ +n es +ĠA ust +a ul +ro te +ĠS outh +ess ion +op h +Ġshow s +Ġs ite +Ġj o +Ġr isk +cl us +l t +Ġin j +id ing +ĠS pe +Ġch all +ir m +Ġ2 2 +itt ing +st r +Ġh y +L E +ke y +Ġbe gan +at ur +ashing ton +l am +ĠD av +b it +Ġs ize +ĠP ar +3 8 +ourn al +f ace +Ġdec ision +Ġl arg +Ġj ud +re ct +Ġcontin ue +ĠO ct +ove red +ĠI nt +==== ==== +Ġp arent +ĠW ill +Ġeas y +Ġd rug +ang er +Ġs ense +Ġd i +id ay +Ġener gy +ist ic +Ġass oci +ar ter +ob al +e ks +ĠE l +ur ch +Ġg irl +o e +it le +Ġ2 8 +ĠC he +Ġrequ est +Ġso on +Ġh ost +k y +Ġst ates +om es +Ġm aterial +le x +Ġmom ent +Ġan sw +on se +Ġes pecially +Ġn orm +Ġserv ices +p ite +r an +Ġro le +4 4 +) : +Ġc red +C l +____ ____ +Ġm at +Ġl og +ĠCl inton +O U +Ġoff ice +Ġ2 6 +Ġch arg +Ġtr ack +m a +Ġhe art +Ġb all +Ġperson al +Ġbuild ing +n a +s et +b ody +ĠBl ack +Ġincre ase +itt en +Ġneed ed +3 6 +3 2 += " +Ġl ost +Ġbec ame +Ġgrou ps +ĠM us +Ġw rote +ĠP e +Ġpro p +j oy +à © +ĠWh ite +Ġde ad +. ' +Ġhtt p +Ġwe bs +O S +Ġins ide +Ġwr ong +Ġstat ement +Ġ ... +y l +Ġfil m +Ġmus ic +Ġsh are +ific ation +Ġre lease +Ġfor ward +Ġst ay +Ġcomp ut +it te +s er +Ġorig inal +Ġc ard +Ġc and +Ġd iv +at ural +Ġfav or +O M +Ġc ases +us es +Ġse ction +Ġle ave +g ing +ov ed +ĠW ashington +3 9 +ĠG l +Ġrequ ired +act ion +ap an +o or +it er +ĠK ing +Ġcount ries +ĠG erman +ll ing +Ġ2 7 +3 4 +Ġquest ions +Ġpr im +Ġc ell +Ġsh oot +Ġany one +ĠW est +Ġaff ect +ep end +Ġon line +ĠIs rael +ĠSept ember +Ġab ility +Ġcont ent +is es +Ġre ve +Ġl aun +Ġind ic +Ġfor ce +c ast +Ġso ld +av ing +f l +Ġso ft +Ġcompan ies +ce ed +Ġart icle +Ġa ud +Ġre v +Ġed uc +Ġplay ing +0 5 +Ġhe ld +ct or +Ġrele ased +Ġf ederal +3 7 +Ġad minist +Ġinter view +Ġinst all +Ġrece ived +Ġs ource +u k +P h +Ġser ious +Ġcre ated +Ġc ause +Ġim medi +Ġdef in +u el +ĠDep artment +ct ions +ĠC our +ĠN ow +z e +it es +it ution +Ġl ate +Ġspe ak +n ers +Ġleg al +ar i +ĠC or +Ġwe eks +Ġmod el +Ġp red +Ġex act +B C +ĠB y +IN G +os ing +Ġt akes +Ġreg ard +Ġopp ortun +Ġpr ice +Ġ19 8 +ĠA pr +f ully +Ġor d +Ġproble ms +ru ction +h am +ĠC ount +le ge +Ġlead ers +E T +le v +Ġde ep +olog ical +es e +h aps +ĠS ome +Ġp ers +Ġcont ract +Ġrelations hip +s p +ou d +Ġb ase +4 8 +m it +A d +anc ial +Ġcons um +Ġpot ential +Ġl angu +re m +et h +Ġrel ig +ress ed +6 6 +Ġl ink +Ġl ower +ay er +ĠJ une +Ġf em +un t +er c +ur d +Ġcont act +Ġ ill +Ġm other +Ġest ab +h tt +ĠM arch +ĠB ro +ĠCh ina +Ġ2 9 +Ġs qu +Ġprov ided +Ġa verage +as ons +Ġ201 1 +Ġex am +l in +5 5 +n ed +Ġper fect +Ġt ou +al se +u x +Ġbu y +Ġsh ot +Ġcol lect +Ġph ot +Ġplay ed +Ġsur pr +Ġofficial s +Ġsim ple +av y +Ġindust ry +Ġhand s +g round +Ġp ull +Ġr ound +Ġus er +Ġr ange +u ary +Ġpriv ate +op s +e es +Ġw ays +ĠM ich +Ġve h +Ġex cept +Ġter ms +im um +pp er +I ON +ore s +ĠDr agon +ou l +Ġd en +Ġperform ance +Ġb ill +c il +4 7 +Ġen vironment +Ġex c +ad d +Ġwor th +Ġp ict +Ġch ance +Ġ201 8 +b or +Ġspe ed +ict ion +Ġal leg +ĠJ apan +at ory +re et +Ġm atch +ĠI I +Ġst ru +ord er +Ġst e +Ġl iving +Ġst ruct +in o +Ġse par +her n +Ġresp onse +Ġen joy +Ġv ia +A D +um ents +ace book +Ġmem ber +ib r +iz ing +Ġto ol +ĠM on +ĠWh ile +h ood +ĠA ng +ĠD ef +Ġoff er +T r +a ur +Ġturn ed +ĠJ uly +d own +an ced +Ġrec ently +ĠE ar +Ġc e +ĠSt ar +ĠC ong +rough t +Ġbl ood +Ġhop e +Ġcom ment +ain t +Ġar ri +il es +Ġpartic ip +ough t +ri ption +0 8 +4 9 +Ġg ave +Ġse lect +Ġkill ed +sy ch +Ġgo es +i j +Ġc oll +Ġimp act +at ives +ĠS er +0 9 +ĠAug ust +Ġb oy +d e +ĠD es +Ġf elt +U S +Ġexpect ed +Ġim age +ĠM ark +cc ording +o ice +E C +ĠM ag +en ed +h old +ĠP ost +Ġpre vent +N o +Ġinvol ved +Ġey es +Ġquick ly +A t +un k +Ġbeh av +Ġ ur +Ġl ed +c ome +e y +Ġcand id +Ġear lier +Ġfoc us +et y +P ro +led ge +ix ed +ill ed +Ġpop ular +A P +Ġset t +l ight +Ġvar ious +in ks +Ġlevel s +Ġro ad +ell ig +ab les +he l +itte e +ĠG ener +y pe +Ġhe ard +ic les +Ġm is +Ġus ers +ĠS an +Ġimpro ve +Ġf ather +Ġse arch +The y +v il +Ġprof ess +Ġkn ew +Ġl oss +Ġev ents +6 5 +Ġb illion +0 7 +0 2 +ĠNew s +ĠA M +Ġco ver +w here +ens ion +Ġb ott +Ġare as +en ces +op e +ĠTw itter +a el +Ġget s +ĠGo ogle +Ġs n +i ant +Ġv ote +Ġnear ly +Ġinclud ed +Ġrec ogn +z z +m m +al ed +Ġhappen ed +0 4 +Ġh ot +Ġwho se +Ġc ivil +Ġsu ff +o es +it iz +ĠSy ri +Ġresp ond +Ġh on +Ġfeat ures +Ġeconom ic +ĠApr il +r im +Ġtechn ology +Ġo ption +ag ing +Ġpur ch +R e +Ġl at +ch ie +is l +Ġrec omm +u f +Ġtr aining +Ġeffect s +Ġf ast +Ġ201 0 +Ġocc ur +Ġwebs ite +Ġem ail +Ġs ens +e ch +Ġo il +Ġinf lu +Ġcurrent ly +ĠS ch +ĠAd d +Ġgo al +Ġsc ient +Ġcon v +1 00 +em y +Ġdec ided +Ġtra vel +Ġm ention +L L +0 3 +Ġe lection +Ġph one +Ġlook s +Ġsit uation +Ġc y +Ġh or +b ed +ĠCour t +a ily +av es +Ġqu ality +ĠCom p +w ise +Ġt able +Ġst aff +ĠW ind +et t +Ġtri ed +ide red +Ġadd ition +Ġb ox +Ġl ack +ar ily +Ġw ide +Ġm id +Ġbo ard +ys is +Ġant i +h a +Ġd ig +en ing +Ġd ro +C on +6 8 +Ġsl ow +b ased +se qu +Ġp ath +E x +ak er +Ġwork ed +Ġp en +Ġeng ine +Ġlook ed +ĠSu per +ĠS erv +Ġvict im +U n +Ġproper ty +Ġint rodu +Ġexec ut +ĠP M +L e +Ġcol or +ĠM ore +Ġ6 0 +Ġnet work +Ġd ate +c ul +id ge +Ġext ra +3 1 +Ġs le +6 7 +Ġw ond +Ġreport s +j ust +ĠAust ral +Ġcap ital +Ġen s +Ġcomm and +Ġallow ed +Ġpre p +Ġca pt +h ib +Ġnum bers +ch an +Ġf air +m p +om s +Ġre ach +W ith +t ain +Ġbro ad +Ġcou ple +ec ause +ly ing +ĠF eb +Ġsc reen +Ġl ives +Ġpri or +ĠCong ress +A r +Ġappro ach +Ġe mer +ar ies +ĠD is +s erv +ĠN e +Ġbu ilt +c ies +Ġre pe +Ġrul es +for ce +ĠP al +Ġfin ancial +Ġcons idered +ĠCh ar +n ces +ĠI S +Ġb rought +Ġb i +i ers +ĠS im +O P +Ġproduct s +Ġvis it +Ġdoc ument +Ġcon duct +Ġcomplete ly +in ing +ĠCal if +ib ly +Ġwr itten +ĠT V +em ents +Ġd raw +O ne +Ġpub lished +Ġsec ret +r ain +he t +ĠF acebook +ond ay +ĠU p +Ġsex ual +Ġth ous +ĠP at +Ġ ess +Ġstand ard +Ġar m +g es +ect ion +Ġf ell +Ġfore ign +an i +ĠFr iday +Ġreg ular +in ary +Ġincre ased +Ġus ually +Ġdem on +Ġd ark +Ġadd itional +ro l +ĠO f +Ġprodu ction +! ! +und red +Ġintern ational +id ents +ĠF ree +rou p +Ġr ace +Ġm ach +Ġh uge +A ll +le ar +ove mber +Ġto wn +Ġatt ention +ĠO ff +y ond +ĠThe n +f ield +Ġter ror +ra z +ĠB o +Ġmeet ing +ĠP ark +Ġar rest +Ġf ear +Ġa w +ĠV al +or ing +' , +Ġext reme +ar r +Ġwork ers +A fter +Ġ3 1 +n et +am ent +Ġdirect ly +Ġpop ulation +ub e +ĠOct ober +ĠI N +ĠJan uary +5 9 +ĠDav id +Ġc ross +ce mber +ĠF irst +Ġmess age +ir it +Ġn ation +Ġp oll +is ions +Ġansw er +n y +is ode +Ġcar ry +ĠRuss ia +Ġhe ar +eng th +ro y +Ġn atural +in ally +Ġdo g +m itted +Ġtr ade +Ġsub st +Ġmult iple +ĠAf ric +Ġf ans +Ġs ort +Ġgl obal +ic ation +ĠW ed +ar a +Ġa chie +Ġlangu age +ve y +Ġt al +Ġnecess ary +Ġdet ails +Ġs en +ĠS und +ĠRe g +ĠR ec +0 6 +Ġs il +ress ive +Ġmed ical +un ch +orn ia +Ġu nd +f ort +oc ks +ĠM onday +ues day +c raft +7 7 +ur t +Ġ ver +ĠH ill +Ġrece ive +Ġmor ning +es tern +Ġb ank +Ġs at +ir th +ĠH igh +Ġdev ice +ĠTH E +ĠCent er +Ġsaf e +Ġp le +ĠCanad a +Ġsystem s +Ġass ist +Ġsur v +Ġb attle +ĠS oc +vert is +S he +Ġp aper +Ġgrow th +Ġc ast +S c +Ġpl ans +ll ed +Ġpart s +Ġw all +Ġmove ment +Ġpract ice +im ately +Ġdis play +Ġsomet imes +om p +ĠP aul +ĠY es +k ing +5 8 +o ly +Ġs on +Ġav oid +ok es +ĠJ ew +Ġto wards +as c +Ġ // +ĠK ore +Ġtalk ing +Ġcor rect +Ġsp ent +ic ks +i able +e ared +Ġter m +Ġwant s +om ing +Ġ ut +Ġdou b +Ġfor ces +Ġp lease +6 9 +ĠN ovember +at form +ond on +Ġon es +Ġimmedi ately +ĠRuss ian +ĠM et +Ġde g +Ġparent s +C H +ĠAmeric ans +al y +ĠM od +Ġsh own +Ġcond itions +Ġst uff +Ġre b +ĠY our +Ġinclud es +n own +ĠS am +Ġexper ien +m ission +ĠE ven +augh t +Ġannoun ced +ĠRepublic an +Ġdeter min +Ġdescrib ed +ĠCount y +( ) +Ġdo or +Ġchang ed +Ġne igh +ĠH ere +Ġcle an +Ġp an +ĠDe cember +ĠEurope an +ir ing +ap ter +Ġcl ub +ĠT uesday +Ġp aid +ĠN et +Ġattack s +Ġcharact ers +Ġal one +Ġdirect or +d om +Ġ3 5 +Ġl oad +Ġr out +ĠCalif ornia +Ġfin ally +Ġr ac +Ġcont r +Ġexact ly +res h +p ri +ĠIs lam +Ġn ature +Ġcare er +Ġlat est +Ġcon vers +ĠS l +p ose +ci ent +ĠIn c +iv ity +8 8 +ĠA tt +ĠM or +nes day +Ġwe ight +k en +Ġnot e +Ġteam s +Ġ \ +air s +ĠG reen +Ġh undred +on ent +Ġstre ng +Ġcons ist +ic ated +Ġreg ul +Ġl ic +ast ic +Ġt en +urs day +ellig ence +ous ly +ĠU K +B I +Ġcost s +Ġind epend +ĠA P +Ġnorm al +Ġh om +Ġob vious +Ġs we +Ġst ar +Ġread y +ac her +Ġimp lement +g est +Ġs ong +ĠG et +ĠL ab +Ġinterest ing +us ing +Ġg iving +ĠSund ay +Ġet c +Ġm iddle +Ġrem ember +r ight +os ition +ut ions +Ġm ax +4 6 +Ġyour self +Ġdem and +Ġtreat ment +Ġd anger +ĠC ons +Ġgu y +ĠBrit ish +Ġphys ical +Ġrel ated +Ġrem ain +Ġcould n +Ġref er +Ġc itiz +b ox +EN T +bo ard +Ġin n +I G +er o +ĠSt reet +osp ital +ren ch +cher s +Ġst ra +O L +ag er +ĠA N +Ġeas ily +I A +en ge +in y +Ġcl os +ock ed +Ġus es +ĠC oun +I m +u ild +? ? +m ore +Ġan g +Ġwr ite +ol ute +5 7 +Ġlead er +Ġread ing +< / +Ġaut om +est s +4 3 +Ġleg isl +ĠG old +Ġdesign ed +ĠS T +ĠLe g +a res +Ġbe aut +ĠT ex +Ġappear s +Ġstru gg +ĠR om +Ġ 00 +Ġcho ice +Ġparticular ly +ĠF rom +op er +ĠL ondon +ann ed +Ġallow s +ob ile +Ġdiffere nce +âĢ ¢ +ĠV iew +ĠWed nesday +Ġal though +Ġrel ative +Ġapplic ation +ate ver +Ġare n +Ġmy self +Ġim ag +Ġdis e +Ġsoc iety +Ġfre qu +ĠEng lish +Ġpo or +ĠD ay +Ġwrit ing +Ġse ven +Ġstart ing +Ġb ud +Ġpr int +ĠTr ans +uf act +ĠSt ud +n ew +Ġcr im +Ġg ives +Ġco ol +a e +i ance +ĠGener al +Ġthink ing +Ġsa ve +Ġlim ited +ĠPart y +Ġmean ing +p en +ow ers +ĠJ ack +E M +Ġn ice +ru pt +Ġg as +Ġe ight +Ġfe et +Ġeff ort +Ġ ign +ic it +B l +co in +Ġop in +Ġbr ain +Wh ile +he st +ĠTh ursday +Ġwould n +augh ter +Ġtou ch +le ments +Ġstud ies +Ġcent er +c ont +or ge +Ġcomput er +Ġinvestig ation +P l +or ks +Ġ200 8 +Ġincre asing +Ġst ore +Ġcom ments +Ġb al +m en +Ġdo ll +Ġl iber +Ġw ife +Ġlaw s +atur day +it ness +Ġmod ern +ĠS k +Ġadminist ration +Ġopportun ity +Ġs al +Ġpower ful +M y +Ġclaim s +ĠEar th +ord s +Ġt itle +Ġes c +n ame +N ot +om en +Ġbe yond +Ġc amer +Ġse ll +it ute +ear ch +Ġapp l +im ent +4 2 +ĠAr t +Ġun f +Ġviol ence +ur g +ĠE ast +Ġcomp ared +Ġopt ions +Ġthrough out +Ġv s +ig r +. [ +ac hes +7 8 +Ġfil es +F L +E L +ar ian +ĠJ ames +ĠA ir +an ch +Ġdet ail +Ġpie ce +P S +Ġn amed +Ġeduc ation +Ġdri ve +Ġitem s +Ġstud ent +ic ed +: : +ic o +Ġth row +Ġsc ene +Ġcomple x +Ġ200 9 +Ġpre c +ĠB re +7 9 +Ġcon cept +Ġstat us +am ing +Ġd ied +Ġknow ledge +Ġbegin ning +O D +ru ary +Ġcertain ly +Ġgu ys +Ġsl ight +in n +ound s +Ġf ine +Ġf at +ic ations +Ġper haps +ĠA nt +Ġinc ome +Ġhtt ps +Ġmajor ity +port s +st on +Ġgreat er +Ġfe ed +ent ially +Ġsaf ety +Ġun ique +and om +Ġg one +Ġshow ed +Ġhist or +Ġcoun ter +i us +id a +Ġlead ing +i pe +Ġs end +ĠDon ald +er ve +Ġdef ense +ines e +Ġy es +ĠF ire +ĠMus lim +ra q +Ġcontin ued +os h +Ġprov ides +Ġpr ison +ĠP re +Ġhapp y +Ġeconom y +Ġtr ust +ag s +ĠG ame +Ġweap ons +um an +ĠC le +it ation +Ġanal ysis +ĠT imes +Ġsc ience +- > +Ġfig ure +Ġdis app +ent y +Ġsoft ware +Ġu lt +Ġoffic ers +N ew +I s +Ġrem ains +ĠInd ia +Ġp sych +ri ef +Ġc at +es c +Ġob serv +Ġst age +ĠD ark +Ġent er +ch ange +Ġpass ed +Ġdes pite +ĠO ut +Ġmov ie +r s +Ġv oice +m ine +ĠPl ay +Ġto ward +ĠT er +Ġreg ion +Ġval ues +or ters +Ġm ount +Ġoffic er +ĠO ther +b an +Ġh ous +w ood +ro om +I V +ĠS un +se e +ĠO ver +ro g +9 0 +Ġl ay +ĠT ur +a wn +Ġpress ure +ĠS ub +Ġbook s +ed om +ĠS and +A A +ag o +Ġre asons +f ord +Ġactiv ity +U T +N ow +ĠSen ate +ce ll +n ight +Ġcall s +in ter +Ġlet ter +ĠR ob +ĠJ e +Ġcho ose +ĠL aw +G et +B e +Ġro b +Ġtyp es +Ġpl atform +Ġqu arter +R A +ĠT ime +Ġmay be +ĠC r +9 5 +p re +Ġmov ing +Ġl if +Ġgo ld +Ġs om +Ġpat ients +Ġtr uth +ĠK e +ur ance +ant ly +m ar +Ġchar ge +ĠG reat +Ġce le +---------------- ---------------- +Ġro ck +ro id +an cy +Ġcred it +a ud +B y +ĠE very +Ġmov ed +ing er +rib ution +Ġn ames +Ġstra ight +ĠHe alth +ĠW ell +Ġfe ature +Ġr ule +Ġsc he +in ated +ĠMich ael +ber g +4 1 +il ed +b and +Ġcl ick +ĠAng el +on ents +Â Ń +ĠI raq +ĠS aturday +Ġa ware +p art +Ġpat tern +O W +ĠL et +Ġgr ad +ign ed +Ġassoci ated +Ġst yle +n o +i ation +a ith +il ies +Ġst ories +ur ation +Ġindividual s +ĠâĢ ¦ +m iss +ĠAss oci +ish ing +ab y +Ġsum mer +ĠB en +Ġ3 2 +Ġar ch +ut y +ĠTex as +h ol +Ġfull y +Ġm ill +Ġfollow ed +ĠB ill +ĠInd ian +ĠSec ret +ĠB el +ĠFeb ruary +Ġjob s +Ġseem ed +ĠGo vern +i pped +Ġreal ity +Ġl ines +Ġp ark +Ġmeas ure +ĠO ur +I M +Ġbro ther +Ġgrow ing +Ġb an +Ġest im +Ġc ry +ĠS chool +Ġme chan +ĠO F +ĠWind ows +Ġr ates +ĠO h +Ġpos itive +Ġcult ure +ist ics +ic a +Ġh ar +y a +ite ly +i pp +Ġm ap +en cies +ĠWill iam +I I +ak ers +5 6 +ĠM art +ĠR em +Ġal tern +it ude +Ġco ach +row d +D on +Ġk ids +Ġj ournal +Ġcor por +Ġf alse +Ġwe b +Ġsle ep +Ġcont ain +Ġst o +Ġb ed +iver se +ĠR ich +ĠCh inese +Ġp un +Ġme ant +k nown +Ġnot ice +Ġfavor ite +a ven +Ġcond ition +Ġpur pose +) ) +Ġorgan ization +Ġchall eng +Ġman ufact +Ġsus p +ĠA c +Ġcrit ic +un es +uc lear +Ġm er +vent ion +Ġ8 0 +Ġm ist +ĠU s +ĠT or +htt p +ol f +Ġlarg er +Ġadv ant +Ġrese ar +Ġact ions +m l +Ġke pt +Ġa im +, ' +c ol +Ġbenef its +if ying +Ġact ual +ĠIntern ational +Ġveh icle +Ġch ief +Ġeff orts +ĠLe ague +ĠM ost +Ġwa it +Ġad ult +Ġover all +Ġspe ech +Ġhigh ly +Ġfem ale +Ġer ror +Ġeffect ive +5 4 +Ġenc our +w ell +Ġfail ed +Ġcons erv +Ġprogram s +Ġt rou +Ġa head +5 00 +vertis ement +I P +ĠF ound +p ir +Ġ % +Ġcr ime +and er +Ġloc ation +ĠI ran +Ġbehav ior +az ing +Ġr are +Ġem b +Ġca used +Ġsh ip +Ġact ive +Ġcont ribut +Ġg reen +Ġac qu +Ġref lect +ven ue +Ġf irm +Ġb irth +] . +Ġclear ly +Ġem ot +Ġag ency +ri age +Ġmem ory +9 8 +S A +ĠSe e +ac ing +C C +Ġbig gest +Ġr ap +Ġbas ic +Ġb and +e at +Ġsus pect +ĠM ac +Ġ9 0 +m ark +ist an +Ġsp read +am s +k i +as y +ra v +ĠR ober +Ġdemon str +r ated +Ġabs olute +Ġpl aces +Ġim pl +ibr ary +Ġc ards +Ġdest roy +Ġv irt +ve re +Ġapp eared +y an +p oint +Ġbe g +Ġtem per +s pe +ant ed +ear s +ĠD irect +Ġl ength +Ġbl og +am b +Ġint eg +Ġres ources +ac c +if ul +Ġsp ot +Ġfor ced +Ġthous ands +ĠMin ister +Ġqu al +ĠF rench +at ically +Ġgener ally +Ġdr ink +Ġth us +I L +od es +Ġappro pri +ĠRe ad +Ġwh om +Ġey e +Ġcol lege +Ġ4 5 +ire ction +Ġens ure +Ġapp arent +id ers +Ġrelig ious +Ġmin or +ol ic +Ġt ro +ĠWh y +rib ute +m et +Ġprim ary +Ġdevelop ed +Ġpe ace +Ġsk in +st e +av a +Ġbl ue +Ġfam ilies +Ġ ir +Ġapp ly +Ġin form +ĠSm ith +C T +i i +Ġlim it +Ġres ist +........ ........ +um n +Ġconf lic +Ġtw e +ud d +ĠT om +Ġl iter +qu e +b on +Ġha ir +Ġevent ually +Ġp us +Ġhelp ed +Ġag g +or ney +ĠApp le +Ġf it +ĠS ur +Ġpre m +Ġs ales +Ġsecond s +Ġstreng th +Ġfeel ing +¿ ½ +Ġt our +Ġknow s +o om +Ġex erc +Ġsom ew +ï ¿½ +> > +Ġsp okes +Ġide as +Ġreg ist +so ft +ĠD el +ĠP C +Ġpro pos +Ġlaun ch +Ġbott om +T H +ĠP lease +v est +it z +ĠIn ter +Ġsc ript +Ġr at +ar ning +Ġ il +ĠJ er +ĠA re +Ġwh atever +ok en +ci ence +Ġmod e +Ġag ree +Ġs ources +Ġinit ial +Ġrest rict +Ġwond er +us ion +## ## +ĠS il +vil le +Ġb urn +t w +as ion +Ġ £ +Ġn or +u ing +Ġre ached +Ġs un +Ġc ateg +ig ration +Ġc ook +Ġprom ot +Ġm ale +Ġcl imate +Ġf ix +Ġalleg ed +U R +all ed +Ġim ages +C ont +ot a +Ġschool s +i os +Ġd rop +Ġst ream +ĠM o +Ġprevious ly +al ing +Ġp et +Ġdou ble +Ġ( @ +ann el +Ġdef ault +t ies +Ġr ank +ĠD ec +ĠCoun cil +Ġweap on +Ġst ock +Ġanal y +ĠSt r +Ġpict ure +ĠPol ice +f erence +Ġcent ury +Ġcitiz ens +Ġon to +Ġexp and +Ġhe ro +ĠS ol +Ġw ild +Ġupd ate +Ġcustom ers +r ont +d ef +Ġl ik +Ġcrim inal +ĠChrist ian +S P +7 6 +Ġle aving +Ġother wise +ĠD ist +Ġbas is +5 2 +5 3 +ic ip +ĠB er +Ġrecomm end +Ġfl oor +Ġc rowd +ol es +Ġ7 0 +Ġcent ral +ĠE v +Ġd ream +Ġdown load +Ġconf ir +ĠTh om +Ġwind ow +Ġhapp ens +Ġun it +Ġt end +Ġs pl +Ġbec omes +Ġfight ing +Ġpred ict +ĠP ress +ĠP ower +Ġhe avy +ak ed +Ġf an +or ter +ate gy +B A +iz es +Ġsp end +H ere +Ġ200 7 +Ġad op +ĠH am +Ġfoot ball +ĠP ort +od ay +5 1 +amp ions +Ġtrans fer +h t +Ġ3 8 +ter m +ac ity +Ġb ur +] , +tern al +r ig +b ut +Ġthere fore +ĠB ecause +res p +re y +Ġm ission +S ome +Ġnot ed +Ġass um +Ġdise ase +Ġed it +Ġprog ress +r d +ĠB rown +oc al +Ġadd ing +Ġra ised +ĠAn y +Ġt ick +Ġsee ing +ĠPe ople +Ġagre ement +Ġser ver +Ġw at +Ġdeb ate +Ġsupp osed +il ing +Ġlarg est +Ġsuccess ful +ĠP ri +ĠDemocr atic +Ġj ump +ĠSyri a +Ġown ers +Ġoff ers +Ġshoot ing +Ġeff ic +se y +Ġha ven +ver se +te red +ĠL ight +im al +ĠB ig +Ġdef end +Ġbe at +Ġrecord s +% ) +Ġsc en +Ġemploy ees +Ġdev ices +he m +Ġcom mer +ĠM ex +Ġbenef it +ĠPro f +Ġil leg +Ġsur face +ĠAl so +Ġh arm +ing ly +w ide +ĠA lex +Ġsh ut +ĠC ur +Ġl ose +p m +Ġchall enge +se mb +Ġst ation +Ġint elligence +Ġacc ur +ĠFl or +Ġrequ ires +ĠM al +b um +Ġh ospital +Ġsp irit +Ġoff ered +Ġprodu ce +ĠComm un +Ġcreat ing +Ġcr is +s pect +Ġend ed +Ġd aily +Ġvot ers +land s +i as +i h +on a +Ġsm art +ĠOff ice +ĠL ord +ri al +ĠIntern et +Ġcirc um +Ġextreme ly +' . +Ġopin ion +ĠM il +Ġg ain +B S +ĠF in +y p +Ġuse ful +Ġbud get +Ġcom fort +is f +Ġback ground +el ine +Ġep isode +Ġen emy +Ġtri al +Ġestab lish +d ate +ĠC ap +Ġcontin ues +Ġshow ing +ĠUn ion +w ith +Ġpost ed +ĠSy stem +Ġe at +ri an +Ġr ise +ĠGerman y +il s +Ġsign ed +Ġv ill +Ġgr and +m or +ĠEng land +Ġproject s +um ber +Ġconf erence +z a +Ġrespons ible +ĠAr ab +Ġlearn ed +âĢĶ âĢĶ +i pping +ĠGe orge +O C +Ġreturn ed +ĠAustral ia +Ġb rief +Q u +Ġbr and +ill ing +ab led +Ġhig hest +Ġtr ain +ĠComm ission +wh ile +Ġn om +cept ion +Ġm ut +ĠBl ue +Ġinc ident +v ant +8 6 +ĠI D +Ġn uclear +7 4 +ĠL ike +ĠR E +ĠM icro +l i +m ail +Ġcharg es +8 9 +Ġad just +ad o +Ġear th +N A +Ġpr ices +P A +Ġd raft +Ġrun s +Ġcandid ate +ens es +Ġmanag ement +ĠPh il +ĠM iss +Ġte ach +g ram +Ġunderstand ing +a it +ic ago +A dd +ĠE p +sec ut +Ġsepar ate +Ġinst ance +Ġe th +Ġun less +**** **** +ĠF ore +in ate +Ġoper ations +S p +Ġf aith +g ar +ĠCh urch +ron ic +Ġconf ig +os ure +Ġactiv ities +Ġtrad itional +Ġ3 6 +Ġd irection +Ġmach ine +Ġsur round +Ġp ush +un ction +ĠE U +Ġeas ier +Ġarg ument +G B +Ġm icro +Ġsp ending +iz ations +Ġthe ory +ad ow +Ġcall ing +ĠL ast +Ġd er +Ġinflu ence +Ġcomm it +Ġph oto +Ġun c +ist ry +g n +ast e +ack s +Ġdis p +ad y +d o +ĠG ood +Ġ ` +Ġw ish +Ġreve aled +Âł Âł +l ig +Ġen force +ĠComm ittee +Ġche m +Ġmil es +Ġinterest ed +Ġsol ution +ic y +in ct +Ġ- > +ĠD et +Ġrem oved +Ġcomp ar +e ah +Ġpl ant +ĠS ince +Ġachie ve +Ġadvant age +Ġslight ly +b ing +Ġpl aced +u nder +201 5 +ĠM ad +Ġt im +os es +Ġc ru +ĠR ock +Ġmost ly +Ġneg ative +Ġset ting +Ġprodu ced +Ġm ur +Ġconnect ion +ĠM er +Ġdri ver +Ġexecut ive +Ġass ault +Ġb orn +ĠV er +t ained +Ġstruct ure +Ġredu ce +Ġdec ades +Ġd ed +u ke +ĠM any +idd en +Ġle ague +S e +Ġjo in +Ġdis co +Ġd ie +c ks +act ions +Ġass ess +ag n +Ġgo als +our s +I R +Ġsen ior +ill er +m od +ip ment +oc ol +u y +ĠQ ue +Ġpart ies +ir gin +Ġle arning +it able +Ġstre et +Ġcamer a +A pp +Ġsk ills +b re +c ious +Ġcele br +ĠFr anc +Ġexist ing +Ġwill ing +l or +Ġ id +ĠSp ace +Ġcrit ical +ĠL a +ortun ately +Ġser ve +Ġc old +Ġspec ies +T S +Ġanim als +ĠB ay +Ġold er +ĠU nder +est ic +ĠT re +Ġte acher +Ġpre fer +v is +Ġth read +ĠM att +Ġmanag er +ãĥ » +Ġprofess ional +ĠV ol +Ġnot es +The se +ul a +Ġf resh +ent ed +u zz +ed y +clus ion +ĠR el +Ġdoub t +E O +Ġopen ed +ĠB it +Ad vertisement +Ġgu ess +ĠU N +Ġse qu +Ġexpl ain +ott en +Ġatt ract +ak s +Ġstr ing +Ġcont ext +oss ible +ĠRepublic ans +Ġsol id +Ġc ities +Ġask ing +Ġr andom +u ps +ur ies +ar ant +dd en +g l +ĠFlor ida +Ġdep end +ĠSc ott +Ġ3 3 +Ġi T +ic on +Ġmention ed +Ġ2 000 +Ġclaim ed +Ġdefin itely +ul f +Ġc ore +Ġopen ing +ĠCon st +wh ich +ĠT ra +A G +7 2 +Ġbelie ved +ad a +Ġ4 8 +ĠSec urity +yr ight +ĠP et +ĠL ou +Ġhold ing +======== ======== +Ġ ice +Ġb row +Ġauthor ities +h ost +w ord +Ġsc ore +ĠD iv +Ġcell s +Ġtrans l +Ġneigh bor +Ġrem ove +u ct +Ġdist rict +ĠA ccording +Ġwor se +Ġconcern s +Ġpresident ial +Ġpolic ies +ĠH all +7 3 +Ġh us +A Y +Ġ200 6 +ĠJ ud +Ġindepend ent +ĠJust ice +ili ar +pr int +igh ter +Ġprotect ion +z en +Ġsu dden +h ouse +ĠJ es +P R +ĠIn f +Ġb ul +Ġ _ +ĠServ ice +ĠP R +Ġstr ategy +ff ect +Ġgirl s +Ġmiss ing +oy al +ĠTe am +ul ated +Ġd at +Ġpolit ics +ab or +A ccording +Ġspe ll +Ġg raph +ort hern +T C +A b +Ġlab or +is her +Ġk ick +ĠiT unes +Ġstep s +pos es +Ġsmall er +E n +ber t +Ġro ll +Ġresear chers +Ġcl osed +Ġtrans port +Ġlaw y +________ ________ +ĠCh icago +Ġas pect +Ġn one +Ġmar riage +9 6 +Ġe lements +ĠF re +ĠS al +Ġd ram +F C +t op +e qu +Ġhe aring +Ġsupport ed +Ġtest ing +co hol +Ġmass ive +Ġst ick +Ġgu ard +is co +ph one +F rom +How ever +Ġb order +Ġcop y +ograph y +l ist +7 1 +Ġown er +cl ass +ru it +r ate +ĠO nce +Ġdig ital +Ġt ask +ER S +Ġinc red +t es ++ + +ĠFr ance +Ġb reat +ow l +Ġiss ued +ĠW estern +Ġdet ect +Ġpart ners +Ġsh ared +ĠC all +Ġcan cer +ac he +rib e +Ġexpl ained +Ġhe at +{ " +Ġinvest ment +ĠB ook +Ġw ood +Ġtool s +ĠAl though +Ġbelie f +Ġcris is +Ġg e +ĠM P +Ġoper ation +ty pe +~ ~ +g a +Ġcont ains +ant a +Ġexp ress +ĠG roup +ĠJ ournal +k a +Ġam b +ĠUS A +Ġfind ing +Ġfund ing +h ow +Ġestab lished +ide os +Ġdeg ree +Ġdanger ous +ang ing +Ġfre edom +pp ort +out hern +Ġch urch +Ġc atch +ĠTw o +Ġpres ence +ĠGu ard +U p +Ġauthor ity +ĠPro ject +Ġbut ton +Ġcon sequ +Ġval id +Ġwe ak +Ġstart s +Ġref erence +ĠM em +" ) +U N +or age +ĠO pen +Ġcol lection +y m +g ency +Ġbeaut iful +ro s +Ġtell s +Ġwa iting +n el +Ġprov iding +ĠDemocr ats +Ġd aughter +Ġm aster +Ġpur poses +ĠJapan ese +Ġequ al +Ġturn s +Ġdoc uments +Ġwatch ing +R es +Ġr an +201 4 +Ġre ject +ĠKore a +Ġvictim s +Le vel +ere nces +Ġw itness +Ġ3 4 +Ġre form +com ing +Ġocc up +Ġc aught +Ġtra ffic +ad ing +Ġmod els +ar io +Ġserv ed +Ġb atter +u ate +ĠSecret ary +Ġagre ed +Ġtr uly +yn am +ĠR et +Ġun its +ĠRes earch +h and +az ine +ĠM ike +Ġvar iety +ot al +Ġam azing +Ġconfir med +Ġentire ly +Ġpurch ase +Ġe lement +Ġc ash +Ġdeter mine +D e +Ġc ars +ĠW all +â ĸ +Ġview s +Ġdrug s +Ġdep artment +ĠSt ep +u it +Ġ3 9 +as ure +ĠCl ass +Ġc overed +ĠB ank +Ġme re +u ana +Ġmult i +Ġm ix +Ġun like +lev ision +Ġsto pped +Ġs em +ĠG al +ul es +Ġwe l +ĠJohn son +l a +Ġsk ill +Ġbec oming +ri e +Ġappropri ate +f e +ell ow +ĠPro t +ul ate +oc ation +Ġweek end +od ies +Ġsit es +Ġanim al +ĠT im +Ġsc ale +Ġcharg ed +Ġinst ruct +ill a +Ġmethod s +Ġc ert +Ġjud ge +ĠH el +Ġdoll ars +Ġstand ing +ĠS qu +Ġdeb t +l iam +Ġdri ving +ĠS um +ĠEd ition +Ġal bum +and on +I F +ĠU k +6 3 +ad er +Ġcommer cial +es h +ĠGovern ment +Ġdisc overed +Ġout put +ĠHill ary +ĠCar ol +Ġ200 5 +Ġab use +anc ing +Ġsw itch +Ġann ual +T w +Ġst ated +ag ement +in ner +Ġdem ocr +Ġres idents +Ġallow ing +Ġfact ors +od d +Ġf uck +em ies +Ġoccur red +ot i +Ġn orth +ĠP ublic +Ġinj ury +Ġins urance +C L +oll y +ã Ģ +Ġrepe ated +Ġar ms +ang ed +Ġconst ruction +Ġf le +P U +ic ians +Ġfor ms +ĠMc C +ant ic +Ġm ental +p ire +Ġequ ipment +Ġf ant +Ġdiscuss ion +Ġregard ing +k in +ar p +Ġch air +og ue +Ġpro ceed +ĠI d +O ur +Ġmur der +M an +Ġ4 9 +as p +Ġsupp ly +Ġin put +Ġwe alth +liam ent +Ġpro ced +or ial +ĠSt at +ĠN FL +hen s +ĠInst itute +Ġput ting +ourn ament +et ic +Ġloc ated +Ġk id +er ia +r un +Ġpr inc +Ġ ! +go ing +ĠB et +Ġcl ot +Ġtell ing +Ġprop osed +i ot +or ry +Ġfund s +g ment +ĠL ife +Ġb aby +ĠB ack +Ġsp oke +Im age +Ġear n +ĠA T +g u +Ġex change +ĠL in +ov ing +Ġp air +M ore +az on +Ġarrest ed +Ġkill ing +c an +ĠC ard +y d +Ġident ified +Ġm obile +Ġthan ks +ony m +ĠF orm +Ġhundred s +ĠCh ris +ĠC at +Ġtre nd +h at +ĠA v +om an +Ġelect ric +ĠW il +S E +O f +Ġrest aur +ot ed +Ġtr ig +Ġn ine +Ġb omb +Wh y + ¯ +Ġco verage +Ġapp eal +ĠRober t +ĠS up +Ġfin ished +Ġfl ow +Ġdel iver +Ġcal cul +Ġphot os +Ġph il +Ġpie ces +Ġapp re +k es +Ġr ough +D o +Ġpart ner +Ġconcern ed +Ġ3 7 +ĠG en +C ol +ct ors +Ġ= > +st ate +Ġsuggest ed +ĠFor ce +C E +Ġher self +ĠPl an +w orks +o oth +ren cy +Ġcor ner +Ġhus band +Ġintern et +ĠA ut +em s +os en +ĠAt l +g en +Ġbal ance +6 2 +Ġsound s +te xt +Ġar r +ov es +Ġmill ions +Ġrad io +Ġsat isf +ĠD am +M r +G o +S pe +Ġcomb at +r ant +ĠG ree +Ġf uel +Ġdist ance +Ġtest s +Ġdec re +ĠE r +Ġman aged +D S +Ġt it +Ġmeas ures +ĠL iber +Ġatt end +as hed +ĠJ ose +ĠN ight +d it +ĠN ov +ĠE nd +out s +Ġgener ation +Ġadv oc +y th +Ġconvers ation +ĠS ky +act ive +ce l +ri er +ĠFr ank +Ġg ender +Ġcon cent +Ġcar ried +and a +ĠV irgin +Ġarri ved +ic ide +ad ed +Ġfail ure +Ġmin imum +le ts +Ġwor st +Ġkeep ing +Ġint ended +Ġilleg al +Ġsub sc +Ġdetermin ed +Ġtri p +Y es +Ġra ise +Ġ ~ +Ġfeel s +Ġpack age +ĠJ o +h i +201 6 +re al +Ġf ra +Ġsy mb +M e +uck y +p ret +ĠK h +ĠEd it +ĠWe b +em ic +ĠCol or +Ġjust ice +I nt +Ġfar m +ck now +" > +el ess +Ġredu ced +Ġ5 00 +x x +ĠR ad +ĠW ood +Ġcl in +Ġhy p +il er +ur a +k ins +8 5 +6 1 +ĠThe ir +ĠM ary +Ġs an +Ġno vel +ĠWh o +Ġcap acity +Ġimp ossible +Ġpl ays +Ġmin ister +ij uana +ic ate +ĠS et +Ġf ram +Ġ ing +Ġcommun ities +ĠF BI +it a +Ġb on +Ġstr ateg +Ġinterest s +l ock +g ers +m as +ĠAN D +Ġconflic t +Ġrequire ments +Ġs ac +Ġoper ating +in i +rel ated +Ġcomm itted +Ġrelative ly +Ġs outh +¯ ¯ +Ġaff ord +Ġident ity +Ġdec isions +Ġacc used +pl ace +Ġvict ory +o ch +i at +N ame +C om +t ion +ed s +Ġsee k +Ġt ight +ĠIm ages +Ġinit i +Ġhum ans +Ġfam iliar +Ġaud ience +Ġintern al +vent ure +Ġs ides +ĠT O +Ġd im +Ġcon clud +Ġapp oint +Ġenforce ment +ĠJ im +ĠAssoci ation +Ġcircum st +ĠCanad ian +Ġjo ined +Ġdiffere nces +ĠL os +Ġprot est +Ġtw ice +w in +Ġgl ass +ars h +ĠAr my +Ġexp ression +Ġdec ide +Ġplan ning +an ia +Ġhand le +ĠMicro soft +ĠN or +Ġmax imum +ĠRe v +Ġse a +Ġev al +Ġhel ps +re f +Ġb ound +Ġm outh +Ġstand ards +Ġcl im +ĠC amp +ĠF ox +cl es +Ġar my +ĠTe chn +ack ing +x y +S S +Ġ4 2 +Ġbu g +ĠUk rain +ĠM ax +ĠJ ones +ĠSh ow +l o +Ġplan et +Ġ7 5 +Ġwin ning +Ġf aster +Ġspe ct +Ġbro ken +T R +Ġdef ined +Ġhealth y +Ġcompet ition +htt ps +ĠIs land +ĠF e +Ġannoun ce +ĠC up +ĠInst ead +Ġcl ient +Ġposs ibly +se ction +ock et +l ook +Ġfin ish +Ġcre w +Ġres erv +Ġed itor +Ġh ate +Ġs ale +Ġcontro vers +Ġp ages +w ing +Ġnum er +Ġopp osition +Ġ200 4 +Ġref uge +Ġfl ight +Ġap art +ĠL at +A meric +ĠAfric a +Ġapplic ations +ĠPal est +ĠB ur +Ġg ar +ĠSoc ial +Ġup gr +Ġsh ape +Ġspe aking +ans ion +a o +ĠS n +Ġwor ry +ĠBrit ain +P lease +rou d +Ġh un +Ġintrodu ced +Ġd iet +I nd +ĠSec ond +Ġfun ctions +ut s +ĠE ach +ĠJe ff +Ġst ress +Ġaccount s +Ġgu arant +ĠAn n +ed ia +Ġhon est +Ġt ree +ĠAfric an +ĠB ush +} , +Ġs ch +ĠOn ly +Ġf if +ig an +Ġexerc ise +ĠEx p +Ġscient ists +Ġlegisl ation +ĠW ork +ĠS pr +à Ĥ +ĠH uman +Ġ è +Ġsur vey +Ġr ich +ri p +Ġmain tain +Ġfl o +Ġleaders hip +st ream +ĠIslam ic +Ġ 01 +ĠCol lege +Ġmag ic +ĠPr ime +Ġfig ures +201 7 +ind er +x ual +ĠDe ad +Ġabsolute ly +Ġfour th +Ġpresent ed +resp ond +rib le +Ġal cohol +at o +ĠD E +por ary +Ġgr ab +Ġvar i +Ġqu ant +ĠPh oto +Ġpl us +r ick +ar ks +Ġaltern ative +Ġp il +Ġappro x +th at +Ġobject s +ĠR o +ĠAnd roid +Ġsignificant ly +ĠR oad +k ay +R ead +av or +Ġa cknow +ĠH D +ĠS ing +O r +ĠM ont +Ġun s +pro f +Ġneg oti +ĠAr ch +ik i +Ġte levision +ĠJew ish +Ġcomm ittee +Ġmot or +Ġappear ance +Ġs itting +Ġstri ke +ĠD own +com p +ĠH ist +Ġf old +ac ement +ĠLou is +Ġbel ong +ĠâĢ ¢ +Ġm ort +Ġprep ared +Ġ6 4 +ĠM aster +Ġind eed +ĠD en +Ġre nt +T A +our ney +ar c +S u +9 7 +Ġadv ice +Ġchang ing +Ġlist ed +Ġlaun ched +is ation +ĠP eter +is hes +Ġl ived +ĠM el +ĠSup reme +ĠF ederal +Ġ) ; +ruct ure +Ġset s +Ġphil os +u ous +Ġ ł +Ġappl ied +ĠN OT +Ġhous ing +ĠM ount +Ġo dd +Ġsu st +D A +ffic ient +Ġ ? +ol ved +Ġp owers +Ġth r +Ġrem aining +ĠW ater +L C +Ġca uses +ãģ ® +Ġman ner +ad s +Ġsuggest s +Ġend s +stand ing +f ig +ĠD un +id th +Ġg ay +Ġter min +ĠAngel es +M S +Ġscient ific +Ġco al +ap ers +b ar +ĠThom as +Ġsy m +ĠR un +th is +P C +igr ants +Ġmin ute +ĠDist rict +cell ent +Ġle aves +Ġcomple ted +am in +Ġfoc used +Ġmon itor +Ġveh icles +M A +ĠM ass +ĠGr and +Ġaffect ed +itution al +Ġconst ruct +Ġfollow s +Ġt on +re ens +Ġh omes +ĠE xt +ĠLe vel +r ast +ĠI r +Ġel im +Ġlarge ly +ĠJ oe +Ġvot es +all s +Ġbusiness es +ĠFound ation +ĠCent ral +Ġy ards +Ġmaterial s +ul ner +Ġgu ide +Ġclos er +um s +Ġsp orts +ed er +J ust +Ġtax es +8 4 +ĠO ld +Ġdec ade +ol a +Ġv ir +Ġdro pped +Ġdel ay +it ect +Ġsec ure +ste in +le vel +Ġtre ated +Ġfil ed +ain e +Ġv an +Ġm ir +Ġcol umn +ict ed +e per +Ġro t +Ġcons ult +Ġent ry +Ġmar ijuana +ĠD ou +Ġapparent ly +ok ing +clus ive +Ġincre ases +an o +Ġspecific ally +Ġte le +ens ions +Ġrelig ion +ab ilities +Ġfr ame +ĠN ote +ĠLe e +Ġhelp ing +Ġed ge +ost on +Ġorgan izations +à ĥ +ĠB oth +hip s +Ġbig ger +Ġbo ost +ĠSt and +Ġro w +ul s +ab ase +Ġr id +L et +are n +ra ve +Ġst ret +P D +Ġv ision +Ġwe aring +Ġappre ci +Ġa ward +ĠU se +Ġfact or +w ar +ul ations +) ( +Ġg od +Ġter rit +Ġpar am +ast s +8 7 +Ġen emies +ĠG ames +F F +Ġacc ident +W ell +ĠMart in +T ER +Ġat h +ĠHe ll +Ġfor g +Ġve ter +ĠMed ic +f ree +Ġst ars +Ġexp ensive +Ġac ad +ra wn +ĠW he +Ġl ock +Ġform at +Ġsold iers +s m +Ġag ent +Ġrespons ibility +or a +ĠS cience +Ġrap id +Ġt ough +ĠJes us +Ġbelie ves +M L +Ġwe ar +le te +Ãĥ ÃĤ +ĠD ri +Ġcomm ission +ĠB ob +O h +ap ed +Ġwar m +ÃĥÃĤ ÃĥÃĤ +Ġ200 3 +ort ion +Ġhas n +ust er +Ġun ivers +ĠI ll +Ġk ing +olog ies +9 4 +ĠT em +ĠM os +Ġpat ient +ĠMex ico +ce an +ĠDe ath +ĠSand ers +y ou +ĠC ast +ĠComp any +pt y +Ġhappen ing +F P +ĠB attle +Ġb ought +A m +M od +U s +ut ers +ĠC re +ĠTh ose +Ġ4 4 +is er +Ġs oul +ĠT op +ĠHar ry +ĠA w +Ġse at +ff ee +Ġrev olution +Ġ( " +ĠD uring +et te +Ġr ing +Ġoff ensive +Ġreturn s +Ġv ideos +Ġdis cl +Ġfam ous +en ced +ĠS ign +ĠR iver +Ġ3 00 +P M +ĠB us +ĠC H +Ġcandid ates +ard en +Ġpercent age +Ġvis ual +Ġthan k +Ġtrou ble +ner gy +Ġ200 1 +Ġpro ve +ash ion +Ġen h +ĠL ong +U M +Ġconnect ed +Ġposs ibility +O ver +Ġexper t +Ġl ibrary +art s +ĠDirect or +Ġfell ow +9 2 +ir ty +Ġd ry +Ġsign s +ĠL ove +Ġqu iet +f oot +Ġp ure +ĠH un +Ġf illed +ph as +ĠE lect +end ment +ĠEx pl +Ġun able +n s +m o +Ġv ast +ob e +Ġident ify +app ing +ĠCarol ina +g ress +Ġpro te +Ġf ish +Ġcircumst ances +raz y +ĠPh ot +Ġb odies +ĠM ur +Ġdevelop ing +ĠA R +Ġexperien ced +Ġsubst ant +ĠBo ard +es ome +Ġdom estic +Ġcomb ined +ĠP ut +Ġchem ical +ĠCh ild +Ġpo ol +ĠC y +Ġe gg +c ons +st ers +Ġh urt +Ġmark ets +Ġconserv ative +Ġsupp orters +Ġag encies +id el +O b +ur b +Ġ4 3 +ĠDef ense +y e +ĠA p +du le +Ġtemper ature +Ġconduct ed +ĠCh ief +Ġpull ed +Ġf ol +L ast +ont o +os is +V ER +D es +ĠP an +F irst +Ġadv ance +Ġlic ense +r ors +ĠJ on +Ġimag ine +Ġhe ll +Ġf ixed +Ġinc or +os ite +ĠL og +ick en +] : +Ġsurpr ise +h ab +Ġc raft +ol t +ĠJ ul +Ġd ial +Ġrele vant +Ġent ered +Ġlead s +ĠA D +ĠCle an +Ġpict ures +ess or +Ġal t +Ġpay ing +P er +ĠMark et +Ġupd ates +am ily +ĠT ype +ĠH ome +Ġ5 5 +semb ly +rom e +8 3 +Ġgreat est +Ġhe ight +Ġhe av +ain ts +Ġlist en +as er +ĠS H +Ġcap able +ac le +Ġpers pect +in ating +Ġoff ering +ry pt +ĠDe velop +ab in +r c +Ġbr ight +al ty +ar row +Ġsupp l +ind ing +ack ed +gy pt +ĠAn other +p g +ĠVirgin ia +ĠL u +Ġpl anned +Ġp it +Ġswe et +T ype +ĠD i +Ġtyp ically +ĠFranc isco +Ġpro spect +ĠD an +Ġte en +re es +Ġsc hed +Ġh ol +Ġsc r +Ġlot s +l ife +Ġnews p +Ġfor get +ĠN one +ĠM iddle +ĠR yan +ed d +Ġse vere +Ġsu it +ll er +9 3 +Ġcor respond +Ġexpl os +u ations +Ġfl ag +g ame +r id +Ġpr in +ĠD ata +Ġde ploy +ĠEn ter +su it +gh an +ĠM en +Ġthough ts +Ġmat ters +Ġad apt +ĠA ri +Ġf ill +Ġfor th +Ġs am +Ġ4 1 +Ġpay ment +ĠH or +Ġsp ring +du c +Ġl osing +Ġbring ing +F O +al a +Ġdist ribution +he red +b our +ĠIsrael i +om a +Ġcomb ination +Ġpl enty +V E +C an +ĠH aw +Ġper man +ĠSpe cial +Ġto w +Ġsee king +Ġexam ples +Ġclass es +c r +Ġbe er +Ġmov es +ĠI P +ĠK n +Ġpan el +E ven +Ġproper ly +Ġr is +Ġpl ug +Ġestim ated +E very +Ġdef ensive +ag raph +Ġpre gn +Ġinst it +ĠV ict +Ġvol ume +Ġpos itions +Ġl inks +ĠPro gram +ĠWe ek +ag ues +Ġtrans form +k er +ĠC EO +Ġc as +Ġopp onent +Ġtwe et +ĠC ode +Ġsh op +Ġf ly +Ġtal ks +Ġb ag +Ph one +Ġa id +Ġpl ants +Ġ6 5 +Ġatt orney +ar ters +qu est +ĠMag ic +Ġbeg ins +Ġmy ster +Ġenvironment al +Ġst orage +N N +Ġm arg +Ġs ke +Ġmet al +ell y +Ġord ered +Ġrem ained +Ġl oved +Ġprom pt +Ġupd ated +Ġexper ts +Ġwalk ing +Ġan cient +Ġperform ed +AT E +Ġne ither +i ency +Ġmanufact ure +ĠP ak +Ġselect ed +Ġm ine +Ġult imately +Ġexpl an +Ġlab el +ĠServ ices +ribut ed +Tr ump +Ġsy n +ĠU lt +S C +Ġme at +Ġg iant +ĠW ars +ĠO N +Ġad m +Ġinter pret +Ġeven ing +Ġev il +ĠB oston +ĠW ild +Ġ à +ĠBit coin +ĠAm azon +D r +ĠIn formation +Ġobvious ly +Ġadv anced +Ph oto +ol ar +Ġwe ather +Ġsymb ol +Ġso le +Ġpot entially +ost er +Ġorig inally +m un +3 00 +az e +ess ions +Ġde ck +Ġst ood +Ġyou th +ĠB ern +R ep +ĠT est +Ġbas ically +ot ic +Ġinvol ve +ol it +ly n +S ee +Ġair craft +Ġconf irm +E W +Ġmess ages +ĠRich ard +Ġk it +Ġpro hib +Ġv ulner +is ters +Ġexist ence +Ġturn ing +ĠS P +Ġdes ire +Ġfl at +Ġm ent +se ason +ang es +Ġneighbor hood +ĠL ake +AT ION +Ġpoint ed +b ur +Ġinn ov +uc ks +U L +Ġprofess or +Ġexp ressed +A B +ic ious +Ġ200 2 +ĠDe v +Ġs ession +Ġb are +s en +Ġdis s +ĠC ath +ĠP ass +ĠP oint +Ġdo ctor +or row +ail ed +ĠR ub +ĠD C +ĠChar l +p erson +Ġwrit er +igh ters +ure au +Ġob lig +Ġrecord ed +Ġbro ke +Ġord ers +il ty +Ġmot ion +in ity +l aw +ad ium +Ġimm igration +Ġcontr ast +Ġb att +Ġex cellent +Ġtechn ical +am i +Ġt un +Ġcl oud +ĠY ear +ge on +Ġcre ation +Ġstr ange +Ġa uth +Ġfor t +b orn +Ġext ent +ĠT oday +ĠCl ub +Ġr ain +Ġs ample +Ġaccept ed +Ġt act +Ġf ired +ĠS on +Ġstand s +Ġb oot +Ġ4 7 +Ġstat ements +Ġvers ions +Ġse lling +ound ed +Ġ199 0 +Ġwere n +ĠW atch +Ġexper iment +P ost +Ġret ail +ul ed +In st +un te +ãĥ ¼ +Ġdep art +Ġb ond +i very +om pl +Ġre action +ĠSyri an +ĠP ac +app ed +ani el +D P +Ġres olution +Ġre act +Ġappro ved +on om +m ond +ĠO ffic +-- - +Ġrepl ace +Ġt ack +Ġsp ort +Ġch ain +Ġemer gency +r ad +ĠPalest in +Ġ4 6 +Ġautom atically +Ġrout e +Ġp al +Ġb anks +ĠPar is +ĠMed ia +ro ad +ic ing +i xt +ist ed +Ġg rew +Ġco ord +ĠW here +om in +Ġsub s +� � +Ġ ± +Ġcorpor ate +Ġse lection +n oon +ĠRep ort +c s +clud ing +ord ers +anc he +ĠIt s +Ġslow ly +ĠE gypt +ĠA cc +Ġcol le +iqu es +E X +Ġattempt s +ur l +ĠC ross +Ġfind ings +ĠS C +ĠO R +Ġind ex +ens ity +ĠW ay +ĠL and +Ġsh ock +d is +Ġd ynam +Ġc art +m osp +S ince +i est +ĠB oy +Ġst orm +ĠCont in +201 3 +he w +il it +Ġess ential +iqu id +O ther +ive red +Ġreason able +A ct +Ġsub sequ +ĠP ack +ĠF ort +Ġconsider ing +Ġun iversity +l og +Ġmar ried +Ġill ust +ĠTr ue +£ ı +Ġnumer ous +rast ructure +Ġserious ly +Ġrefer red +u a +Ġconsist ent +on na +ĠRe al +ru ption +ci ples +Ġfact s +9 1 +ot es +er g +The n +Ġacc ompl +N ote +Ġre venue +Ġpass ing +Ġm al +e en +ĠY et +Ġg ather +ter day +ew ork +ĠA uthor +P e +Ġopt im +Ġr ub +Ġè £ı +Ġun known +st one +Ġun ion +ol ve +Ġopportun ities +Ġbrow ser +ĠW al +ĠC ost +Ġreport ing +st s +p et +Ġs and +Ġsudden ly +Ġsurpr ising +ĠV R +Ġsomew hat +ĠB as +ult ure +iz z +ĠC D +Ġchalleng es +Ġsett ings +Ġexperien ces +ĠF ull +Ġcan n +Ġrece iving +ES T +Ġj oint +Ġcult ural +Ġa st +8 2 +as tern +ce ived +ĠC ru +Ġb ull +p ired +am m +Ġfac ing +p ower +Ġb oss +ĠH ol +Ġinst r +Ġincreasing ly +Ġsh ift +Ġstre ets +ĠWilliam s +ab b +Ġl ie +Ġl augh +ĠC a +P L +Ġadult s +Ġcustom er +Ġob tained +Ġsupport ing +ht ml +f ire +Ġdetail ed +Ġpick ed +ĠR ight +ld er +E E +st ood +ĠK im +Ġw ire +Ġs ight +Ġdevelop ers +Ġpers ons +Ġs ad +Ġc up +Ġwar ning +Ġboy s +l ong +Ġb ird +f o +Ġw al +Ġobserv ed +Ġz one +iven ess +Ġch annel +c ript +Ġref used +ĠAg ain +Ġsu c +Ġspokes man +ĠRe f +r ite +ou ston +ãĥ ³ +ĠS her +Ġact s +ĠN ame +Ġstrugg le +ar ry +omet imes +Ġdisc rim +H T +Ġcateg ory +Ġreal ize +Ġemploy ee +ĠAf ghan +en ger +Ġgun s +ĠSte ve +ĠM ot +ĠO l +ok ed +Ġth ick +Ġfair ly +ill y +Ġsur ve +ĠM at +we ight +â Ķ +Ġtro ops +Ġag ents +Ġbatter y +Ġmot iv +à ¡ +S ec +d en +o very +L S +Ġfl u +Ġconf ident +ĠO per +Ġem pty +Ġp hen +Ġse ctor +Ġexc ited +Ġrem ote +ap h +o en +Ġdestroy ed +Ġmor al +ĠH P +ĠR on +Ġd ress +ĠB at +Ġl it +ĠM S +Ġa f +H L +r um +is ms +Ġshould n +Ġsym pt +ĠTor onto +het ic +Ġcar bon +Ġinstall ed +Ġviol ent +Ġsol ar +j a +Ġpract ices +Ġr ide +ĠP enn +Ġimpro ved +Ġaud io +Ġbehav i +ĠP S +Ġe ating +D ata +ĠRe view +p ass +cl aim +u ated +ang ers +c hen +Ġproper ties +Ġany where +An other +Ġbl ow +ĠJack son +Ġp roud +Ġplan e +l ines +Ġsqu are +Ġpro of +ans as +Ġtalk ed +m akers +Ġs ister +Ġhold s +Ġres ident +Ġ= = +Ġresist ance +Ġspl it +Ġpro secut +Ġconf idence +res ents +Ġcut s +Ġexcept ion +Ġz ero +Get ty +Ġcop yright +Ġtot ally +orm al +ific ations +ĠAustral ian +Ġs ick +Ġ1 50 +Ġhouse hold +Ġfe es +Ġdri vers +og en +ĠN Y +Ġnecess arily +Ġregul ations +ear ing +s l +Ġperspect ive +c are +ic ial +H is +Ġesc ape +Ġsurpr ised +ĠV an +ur rent +Ġv ac +8 1 +ĠTh us +Ġem phas +ĠCh ampions +ĠI ce +Ġn arr +Ġhead s +Ġca using +b el +f ortunately +ĠM a +Ġtarg ets +ci pl +Ġafter noon +Ġadd s +ĠMay be +ĠF our +ess ed +ple te +Ġus ual +ch o +ing u +Ġwith d +ĠE nergy +ĠE conom +O O +Ġart icles +Ġinj ured +Ġman age +Ġexpl ains +Ġdi agn +R ec +at ures +Ġlink ed +Ġdiscuss ed +Ġexpl o +Ġocc asion +ath an +Ġopp osite +Ġfac es +Ġden ied +ĠK night +Ġn ut +Ġapprox imately +Ġdisapp oint +onym ous +ĠB est +ĠL o +ĠH y +ĠA ff +Ġvot ing +an while +ĠII I +Ġinstit utions +ag ram +ĠD aily +Ġdr ag +Ġnear by +Ġgu ilty +Ġcon ver +P re +s hip +Ġre ward +Ġphilos oph +ĠS S +u gh +Ġapp s +f riend +Ġu pper +Ġad vert +Ġs now +Ġfr ust +Ġour selves +F r +ĠD ie +amp ion +Ġdis miss +Ġc ere +Ġsign al +f rom +Ġ ). +Ġ5 2 +Ġcr imes +it ors +est ival +use um +Ġcoun cil +ĠS aud +M ay +ĠG un +ic ian +et her +Ġsu fficient +ĠH en +so le +Ġhistor ical +ĠF ar +ĠT urn +Ġp in +Ġsuc ceed +m at +ly mp +Ġtrad ition +ĠO k +Ġc ro +Ġdesc ription +al le +Ġsk y +T e +Ġwide ly +Ġw ave +Ġdefin ition +ĠJew s +Ġcy cle +Ġref ere +Ġbr ings +us al +Ġal ive +Ġfrequ ently +Ġint ention +ĠCont rol +l v +y stem +Ġpriv acy +g ent +ren ce +ĠQu est +ĠChrist mas +Ġr ail +Ġco oper +Ġtest ed +ĠC apt +as ks +Ġcomfort able +Ġdel ivered +sc ape +Ġdep th +ĠG OP +Ġwrit es +Ġass ets +Ġsa v +im ents +Ġtrans ition +Ġart ist +ĠL ook +Ġl ob +Ġcomp onents +ar ity +Ġwalk ed +Ġro ot +Ġparticip ants +Ġnot iced +Ġres c +Ġn av +ĠAd minist +d a +ut ral +pl ate +Ġimport ance +Ġass ert +ious ly +c ription +Ġinj uries +ĠChe ck +Ġregist ered +Ġint ent +Ġmiss ed +ograph ic +Ġsent ence +oun ter +Ġassist ance +ev in +Ġdat abase +Ġbuild ings +Ġclass ic +Ġth inks +ĠOh io +P r +ug g +Ġfe e +p an +Ġeffect ively +Ġfac ility +Ġbe ar +Ġch apter +Ġdog s +ĠCol umb +Ġl atter +it ial +Ġad mitted +T V +ĠGe org +Ġpost s +\ \ +Ġlawy er +Ġequ ival +Ġm and +Ġcontro lled +ĠW alk +ĠAnd rew +Ġmen u +am ental +Ġprotect ed +v a +Ġadminist r +or al +Ġre in +ĠS ar +Ġamount s +Ġn ative +ĠM oon +Ġrep resents +Ġab andon +Ġcarry ing +Ġt ank +m ary +Ġdecl ared +T ube +Ġh at +Ġpun ish +el lect +m es +Ġun iverse +ĠR od +ph y +Ġinf rastructure +Ġ5 1 +Ġopp osed +ow nt +c a +ĠM ake +Ġhard ware +Ġco ffee +R el +b al +w orld +ĠS af +ĠSe a +in als +Ġown ed +Ġh all +ers ion +Ġdescrib e +ĠP ot +Ġport ion +Ġat mosp +Ġgovern ments +Ġdep ending +Ġoff ense +Ġtr ick +aw a +ĠL ine +ĠV is +ĠH ard +ĠOr ig +ĠCl ick +Ġdes k +ĠVal ley +ĠS ov +Ġmov ies +Ġrem ark +Ġm ail +Ġcons cious +Ġrul ing +ĠR ights +Ġmed ic +he nt +ĠW omen +> < +Ġrepl aced +ĠP rem +ĠTh anks +Ġre new +ĠB all +if orm +Ġsh ots +C omm +Ġar med +Ġconst ant +Ġt aste +Ġreal ized +Ġbu ff +Ġm o +Ġeffic ient +M ost +or ation +if ies +Ġcommun ication +Ġfl ood +Ġconsequ ences +Ġany way +ig g +ĠG M +ĠTh ank +Ġ iron +Ġev olution +ĠC op +tw itter +Ġ9 5 +Ġrelationship s +ad el +ĠYou ng +Ġpropos al +ay ers +uild ing +ĠH ot +OR E +c os +Ġcoll abor +P G +ax y +Ġknow ing +Ġsupport s +ow ed +Ġcontrol s +Ġmere ly +um er +Ġath let +Ġf ashion +p ath +Ġg ift +Ġer a +AN D +Ġkind s +ĠKore an +Ġleg it +ul ous +Ġess entially +Ġthe rap +n ic +Ġsuff ered +Ġh ur +Ġprom ise +Ġex cess +Ġover w +Ġpr ime +ĠH ouston +er ry +ĠM s +R S +201 2 +Ġst ores +ĠO lymp +Ġj ourney +Al though +S ub +ĠE duc +ĠCh apter +Ġrequest s +Ġconsum ers +Ġt iny +Ġis ol +ĠF air +b a +ĠY OU +Ġcr ash +ce ler +Ġemot ional +Ġgood s +Ġelect ed +Ġmod er +ĠLin ux +Ġbl ocks +Ġis land +ĠSoc iety +Ġelect ions +Ġbroad cast +Ġche ap +Ġn ations +Ġse asons +4 00 +Ġwas te +ĠS at +Ġfield s +em ploy +Ġprof ile +Ġauth ors +AL L +ĠG ra +w est +ĠT y +Ġdeath s +Ġv acc +Ġfor med +Ġd u +Ġon going +ĠMuslim s +el f +ig ure +Ġass ume +ĠUkrain e +w ater +Ġco ast +Ġvot ed +g or +ĠA S +ĠMich igan +az a +ĠAr m +i ro +Ġf lex +as ters +' ' +Ġwel come +ar l +Ġloc ations +ig ation +ĠF il +Ġbu ying +Ġarch itect +Ġhard er +ĠC ub +Ġinter face +Ġrestaur ant +Ġdisco ver +Ġex ceed +Ġfav our +ger y +Ġd uty +Ġp itch +ad or +ĠM ach +b oy +Ġrespond ed +Ġext ended +her s +M any +ra id +if er +ĠIn s +S er +Ġmed ium +s he +ĠS ports +Ġmag azine +ut ation +Ġlim its +ĠG all +Ġex ternal +raz il +Ġyoung er +t le +Ġrem ind +ĠC ON +Ġimmedi ate +Ġh idden +Ġvol unte +Ġsim pl +od cast +Ġph ase +d r +Ġpl ot +Ġexp osure +R I +og rap +v in +an ish +ĠAc ad +ĠEng ine +Ġexp ansion +ĠP ay +Y our +Ġpus hed +ĠE ll +ĠHe ad +Ġmarket ing +ĠA C +k et +Ġh its +Ġg ro +ĠA ge +ĠSc ot +] [ +Ġst im +Ġi Phone +Ī Ĵ +Ġn arrow +ĠGet ty +ĠTur key +Ġperfect ly +Ġen able +ut ch +Ġprec ise +Ġreg ime +Ġsh if +Ġcomp ens +g un +d iv +Ġch osen +ĠK en +An y +Ġtre es +Ġrecomm ended +ĠR en +u able +ĠH T +F ollow +E G +ĠH and +ĠK enn +Ġarg uments +Ġex ists +Ġb ike +ĠCons erv +Ġbre aking +ĠG ar +Ġc razy +Ġvirt ual +ay lor +ix el +Ġ19 80 +Ġper mission +ĠSer ies +Ġconsum er +Ġclose ly +c alled +Ġ5 4 +Ġhop es +Ġar ray +ĠW in +ĠLab our +Ġsp ons +ĠI re +Ġp ow +Ġread ers +Ġemploy ment +Ġcreat ure +Ġresult ing +Ġaccur ate +Ġmom ents +Ġarg ued +Ġp ed +D uring +Ġ5 3 +ĠT al +Ġs ought +Ġsuff ering +Ġ icon +le e +Ġ( $ +al ian + ° +Ġp ra +Ġbon us +( " +k o +Ġact ing +D E +f all +Ġcompar ison +Ġsm ooth +ĠN AS +u pp +ĠJose ph +ep ing +ĠT ake +ĠM id +Ġs ending +f ast +ĠF all +Ġdeal ing +us er +ĠOr gan +C o +Ġatt ached +Ġse es +% . +Ġtyp ical +AR T +Ġfind s +ĠAs ia +um in +ĠC ore +ĠE nt +in ent +u ce +ĠBl ood +ĠN ever +Ġem ails +Ġhigh light +Ġconf ront +at us +ut ed +Ġun us +Ġtop ic +ĠAd am +Ġb le +at i +Ġunder stood +S et +st ruct +T P +Ġm ob +a a +ĠSt art +pect ed +se ll +Ġded icated +ĠC A +u an +Ġsong s +esc ription +Ġte ch +Ġr ape +Ġas ide +Ġgr ant +Ġ5 6 +s ub +Ġarg ue +Ġcont aining +Ġsche dule +Ġliber al +Ġpublic ly +Ġheav ily +ĠU t +in er +ĠS ection +ĠC are +we et +l s +D is +âĶ Ģ +ĠF ollow +B ack +ĠI T +Ġb es +j i +ĠH it +est ed +Ġevery body +ĠSw ed +Ġfem in +Ġfac ilities +Ġcon ven +C omp +ĠO S +c ore +Ġan x +Ġdiv ision +ĠC am +ĠSt an +m ates +Ġexpl ore +pl om +Ġsh ares +pl oad +an es +Ġide al +et ers +ĠB ase +Ġpl astic +Ġdist inct +ĠNet work +ĠSe attle +Ġtrad ing +ens us +int end +Ġex hib +Ġinit ially +ĠF ood +Ġthous and +ĠBus iness +act er +Ġpar agraph +Ġrough ly +Ġw ww +Ġcreat ive +ĠCon f +Ġconsum ption +Ġfil ms +ag an +Ġob tain +Ġt all +Ġt or +Ġacknow led +Ġg rown +al o +K E +Ġ4 00 +end ers +t aining +U G +Ġsu icide +Ġwat ched +ĠL ist +al i +re hens +Ġsurround ing +Ġp ip +Ġf lying +ĠJ ava +ord an +Ġserv ing +in ations +p ost +Ġsh o +A v +Ġj ail +z y +Ġ199 9 +Ġ< / +Ġliter ally +ĠS ir +Ġexp osed +Ġl ies +st ar +Ġb at +Ġear ned +ĠD ig +Ġspec ified +ĠSe ason +Ġdeg rees +Don ald +Ġcent re +Ġsh aring +Ġwin ter +ĠC O +C he +Ġ Î +M P +Ġun w +Ġfew er +ĠM ir +Ġsomew here +ĠK ey +Ġattack ed +ĠK ir +Ġdom ain +Ġstrong er +Ġ9 9 +Ġpen alty +I d +Sc ript +Ġdecl ined +Ġne ck +Ġfra ud +Ġcur rency +Ġr ising +R C +âĢ¦ âĢ¦ +H z +Ġt ab +Ġtal ent +n am +ĠN BA +Ġvill age +Ġleg s +ĠN ext +E d +Ġac id +Ġhy d +8 00 +Ġinvol ving +ĠIm age +ĠBe fore +F l +Ġyes terday +S ource +Ġterror ist +Ġsu p +Ġsy nt +ĠSaud i +Ġw est +Ġr u +b urg +Ġvis ible +Ġstru ck +r ison +Ġaw esome +Ġd rawn +Ġansw ers +ĠG irl +ĠR am +Ġthreat s +Ġdef eat +os it +Ġv ent +atur ally +Americ an +end a +ĠH oly +Ġr um +% , +c ase +ĠHist ory +ĠYou Tube +Ġsit uations +ĠD NA +S te +Ġsa ved +It em +Ġrec ip +olog ist +Ġfac ed +Ġel ig +O nce +ĠL i +u h +Ġmist ake +ĠDiv ision +ĠB ell +Ġsympt oms + ® +Ġdom in +Ġfall ing +Ġend ing +as hes +Ġmat ches +ĠOn line +Ġexplan ation +D ef +red it +Ġany more +ĠT otal +ĠF OR +us hed +Ġlet ters +Ġris ks +ĠO K +Ġreported ly +: \ +Ġpl ate +Ġsubject s +Ġattempt ed +if ier +ian a +Ġunlike ly +ĠTh ough +um a +ĠIn vest +ĠPr in +ic an +ĠD ar +ĠColor ado +au g +Ġve get +a os +ri a +Ġshe l +Ġmark ed +Ġ( ) +Ġsp r +p o +ĠL ink +Ġdef e +ĠJ r +Ġthem e +Ġpass ion +ĠP en +Ġinf o +iz er +Ġsh it +ĠC ivil +ap se +c re +Ġpo ly +Ġcomp onent +ĠChar les +ĠIre land +ĠPro v +Ġdo ctors +Ġgr anted +Ġpain t +Ġhon or +Ġsm oke +Ġpay ments +Ġprim arily +ĠKing dom +r ich +ate ll +Ġde als +Ġsched uled +Ġfund amental +Ġprote in +Ġnewsp aper +Ġcl ients +yth on +ĠD ate +h us +Ġfeed back +Ġstret ch +Ġc ock +Ġhot el +ĠQue en +Ġsu gar +Ġj u +Ġmil k +Ġappro val +ĠL ive +Ġequival ent +ef ully +Ġins ert +z ona +Ġext ension +d ri +J ohn +Ġacc omp +S m +ĠF und +Ġconst antly +Ġ` ` +Ġgener ated +ĠA ction +ĠP sych +ĠT ri +Ġrecogn ize +Ġv ary +ph a +ĠR a +d f +et ch +ĠSov iet +Tw o +Ġpattern s +Ġprof ession +an ing +T ime +ĠL im +Ġcol ors +ĠA z +ĠT R +Ġinf ect +Ġphen omen +Ġshe ll +Al so +Ġput s +Ġdel ivery +Ġbro wn +Ġprocess ing +Ġlight s +ess age +ĠBro ok +ĠA ud +l ation +Ġindust rial +L ike +ĠB razil +rou s +ES S +ĠL uc +Ġsome how +Ġ8 5 +Ġpro port +Ġpolit icians +Ġindic ate +Ġh ole +Ġtechn iques +Ġcompet itive +Ġph r +Ġv o +ist ent +ĠD ream +Ġcamp us +Ġaspect s +Ġhelp ful +Ġsh ield +or se +Ġtrig ger +m al +Ġ5 8 +Ġt ort +Ġperson ally +Ġt ag +Ġkeep s +ĠV ideo +Ġben ch +Ġg ap +a ire +Ġe ast +Ġrec overy +per ial +Ġprof it +ĠM ic +Ġ5 7 +Ġcol on +Ġstrong ly +st yle +Ġalleg ations +h an +Ġrep orters +j o +r ine +arg et +and al +Ġ0 3 +Ġfl ash +tr ans +Ġstr ict +Ġpark ing +ĠPak istan +Ġl i +Ġwe ird +ĠE ric +Ġreg ions +ĠJ un +Ġint ellect +ĠW H +od ing +rib utes +up id +ĠT it +Ġf inger +or ia +Ġe lev +ĠF ield +Ġcon clusion +; ; +Ġfeel ings +Ġext ensive +Ġm ixed +Ġne uro +v y +Ġhar ass +ĠC irc +ou ch +Ġterrit ory +Ġsuccess fully +M ar +Ġing red +Ġoverw hel +Ġl ayer +V iew +Ġall ies +ill ance +ĠTh ree +Ġb unch +Ġnorm ally +Ġnet works +Ġsac r +ĠC IA +b les +Ġch ose +Ġopp onents +Ġregard less +Ġfr anch +Ġpre f +ĠP o +Ġbr idge +ann a +ĠSil ver +Ġw age +p age +ri or +Ġrad ical +ĠL ittle +Ġman ip +Ġsecret ary +Ġg ang +D R +F A +Ġdec ent +ĠSp irit +Ġun cle +ĠDevelop ment +Ġinvest ors +Ġwall s +Ġpub lish +Ġgener ate +iss ions +c ar +Ġprom ote +Ġcut ting +Ġche st +Ġdrink ing +Ġcollect ed +Ġ7 2 +Ġhop ing +Ġem br +gor ith +Ġwar ned +Ġinstruct ions +O G +ĠD id +ĠAg ency +Ġg ear +Ġcritic ism +ĠF urther +Ġut il +ann y +R ed +Ġcoun sel +ĠAs ian +Ġredu ction +p ool +Ġteach ing +Ġdeep ly +i y +Ġestim ates +Ġcho ices +Ġperman ent +in em +ke l +Ġf asc +p se +f ile +ĠL ow +ĠP erson +Ġt ournament +st al +Ġm el +U ST +ĠR ay +az i +V al +Ġcont ained +ĠH olly +Ġw ake +Ġreve al +Ġprocess es +ĠIS IS +Ġ0 9 +Ġbl ind +Ġste el +ĠB ad +Ġcare fully +app y +ro it +Ġg aming +Ġhous es +ĠC oll +Ġtr uck +er m +Ġsc ored +Ġocc as +ret urn +b ound +v ar +Ġsh arp +Ġaf raid +ĠE X +am ber +c ific +Ġsche me +N C +ĠPol it +Ġdecl ine +Ġ199 8 +Ġpus hing +Ġposs ession +Ġpriv ile +Ġteacher s +Ġy ield +H A +ĠDav is +it led +#### #### +Ġr ig +ĠD aniel +ac on +Ġh ide +ut en +Ġcolle agues +Ġprin ciples +Ġl oud +Ġs in +ĠDem on +Ġst one +Ġ0 2 +Ġt aught +Ġter rible +Ġst uck +ĠPol icy +te en +Ġimplement ation +ĠB BC +ĠAP I +Ġwhe el +all as +Ġch ampions +ol ars +play er +Ġrepeated ly +ĠSt ill +Ġlik es +ast y +es ter +ĠCath olic +R L +Ġb ath +Ġno ise +t itle +Ġn orthern +P art +Ġmag n +Ġf ab +ĠAs h +Ġdis pl +Ġtick et +Ġm urd +Ġalong side +ĠMus ic +Ġr iver +ĠSte el +ĠC L +ĠPl ayer +ĠM ult +ow ing +re p +s ize +Ġt ur +ĠGeorg ia +isc al +ra ction +Ġc able +Ġ5 9 +Ġw ins +Ġup coming +Ġsurv ive +Ġins pired +ĠEduc ation +Ġstat istics +ĠF oot +iam i +Ġy ellow +ĠP age +. - +ĠH as +Ġur ban +Ġa x +es sel +\ " +Ġquarter back +Ġreg ister +ĠLab or +Ġab ilities +ĠF amily +Ġvar iable +ĠPr ice +Ġcont em +Ġth in +ĠE qu +d ata +Ġg otten +Ġconst it +Ġas ks +Ġt ail +Ġexc iting +ĠE ffect +ĠSp anish +Ġencour age +ins on +ĠA h +Ġcommit ment +C S +Ġr ally +Ġ: : +Ġsubs id +Ġsp in +Ġcapt ured +201 8 +Ġinn oc +Ġalleged ly +ĠC ome +Ġart ists +ĠN umber +Ġelect ronic +Ġreg ional +ap es +Ġw ra +Ġmy th +pr ise +ĠM iller +ĠC reat +ĠEp isode +b ell +Ġdirect ed +Ġext ract +Ġs orry +Ġv ice +ag ger +ĠSu pport +Ġ6 6 +ĠI ron +Ġwonder ful +Ġg ra +N et +ion e +E ng +Ġsh ips +ik es +ĠK evin +it ar +Ġactiv ists +tr ue +ĠAri zona +ent h +ĠDes pite +ĠS E +Ġha bit +ern el +Ġin qu +Ġab ortion +Ġv oid +Ġexpl icit +Ġeng aged +Ġang ry +Ġr ating +Ġfr ag +b ro +ick ing +d ev +Ġwor ried +Ġob ser +Ġap artment +ĠG T +Ġest ate +ĠConst itution +em on +ĠS now +Ġcount y +Ġdis ag +ĠStep hen +Ġimm igrants +w ind +ĠN ations +Ġfol ks +O ut +Ġg all +Ġtarget ed +Ġst ead +ĠB on +ĠL ib +Ġinform ed +Ġ12 0 +ch ain +idel ines +or ough +Ġdri ven +Ġregular ly +Ġbas ket +Ġprinc iple +oc ument +Ġst un +ib ilities +ĠRom an +ĠAb out +Ġal ert +Ġdemocr acy +Ġrepresent ed +H S +c ers +p arent +Ar t +p ack +Ġdi plom +re ts +ĠN O +Ġcapt ure +ĠAd v +Ħ ¢ +Ġannounce ment +ĠL ear +Ġh ook +Ġpur s +ĠS uch +ĠC amer +Ġrefuge es +ĠV e +P ol +Ġrecogn ized +l ib +Ġhad n +A ss +Ġpil ot +us hing +Ġreturn ing +Ġtra il +ĠSt one +Ġrout ine +Ġcour ts +Ġdes per +Ġfriend ly +ĠIt aly +Ġpl ed +Ġbreat h +Ġstud io +N S +Ġimp ressive +ĠAfghan istan +Ġf ing +Ġd ownt +ink ing +ĠR og +i ary +col or +se x +ar on +Ġf ault +ĠN ick +D own +ĠR ose +ĠS outhern +X X +is odes +L ist +6 00 +Ġout come +er r +Ġelse where +Ġret ire +Ġp ounds +ĠGl obal +Pe ople +Ġcommun ications +Ġlo an +Ġrat io +ĠEm pire +Ġg onna +Ġinv ent +D F +Ġ19 70 +ĠComm on +p at +Ġprom ised +Ġd inner +ĠH om +Ġcreat es +Ġoper ate +ver ty +ĠJ ordan +et ime +Ġsust ain +R eg +Ġincred ible +im a +Ġwar rant +Ġm m +A tt +Ġlaw suit +Ġreview s +it ure +ĠS ource +l ights +ĠF ord +Ġ6 3 +g roup +st ore +Ġfeat ured +Ġfore ver +Ġpo verty +ĠP op +ĠC NN +az z +ab is +ach ing +Ġl aid +ĠSu pp +Ġfil ter +en a +ĠCommun ity +Ġcreat ures +u ction +ĠR oyal +Ġassoci ation +ĠCon nect +ĠBr ad +âĸ Ī +l ers +the re +ĠG i +Ġval uable +AC K +ĠT aylor +Ġl iquid +ĠAtt orney +ĠCar l +ĠF inal +ag a +ĠWil son +B ecause +ĠProf essor +ak a +Ġincred ibly +r ance +! ) +R ef +s k +Ġsol utions +Ġatmosp here +Ġbl ame +um es +ĠN ob +C A +um ps +r ical +ĠPut in +ĠD est +or ic +ĠP A +Ġrespect ively +w an +Ġfif th +â Ħ¢ +ĠC ry +Ġgovern or +res ident +Ġpurch ased +Ġh ack +Ġint ense +ob s +Ġorig in +Ġdef ine +Ġcare ful +** * +Ġshould er +Cl ick +Ġt ied +Ġdest ruction +ou red +Ġno body +Ġh o +ĠEx per +Ġt ip +" ; +Ġtechn ique +Ġj ur +ĠP ok +b ow +Ġleg end +Ġacc ord +Ġbus y +ĠInt el +Ġh ang +ak i +. ] +âĢĶâĢĶ âĢĶâĢĶ +Ġsur gery +Ġrep rodu +Ġun iform +Ġscen es +c ode +Ġ6 2 +l isher +ĠH ave +ph ia +Ġcry pt +Ġrec on +Ġsc ream +Ġadop ted +Ġsc ores +N e +ĠIt alian +in cluding +B O +Ġindic ated +Ġent ertain +G u +T ext +i el +Ġtw enty +Ġeng age +off s +ĠPac ific +Ġsm ile +Ġperson nel +Ġto ler +Ġdo ors +Ġt one +Ġmach ines +Ġent ering +ten ance +C O +ĠJer sey +Ġfore st +Ġhor se +Ġcompl aint +ĠSpr ing +y o +ĠPl us +ed ing +ĠRet urn +qu arters +ial s +c ow +Ġacad emic +Ġf ruit +Ġ199 6 +og ether +Ġw ine +Ġpur su +ĠSte ven +Ġlic ens +Wh o +Ġclot hes +re ction +Ġsqu ad +Ġst able +Ġr aw +z ens +St ar +ut ies +anc er +Ġke ys +ĠM u +Ġcompl icated +ig er +ĠTe xt +Ġabs or +Ġ6 8 +Ġfun ny +Ġrel ief +ĠL ew +ĠC ook +Ġch art +Ġdraw ing +G E +Ġmod ule +ĠB ull +I LL +Ġs alt +0000 0000 +il le +Ġres ource +aw ay +adel phia +ĠB ru +Ġ6 7 +Ġsome body +Ġparticip ate +Ġro se +we red +Ġmus cle +Ġcons ent +Ġcontin uing +ĠGuard ian +ĠOr der +reg on +Ġre ar +Ġprov ision +Ġlik ed +ri ent +Ġb ra +Tr ans +Ġmeet ings +Ġto x +Ġcon vent +Ġaut o +Ġrec ording +ĠSo ft +00 1 +ĠR oll +Ġprogram ming +Ġp ic +Ġprov ed +Ġst ab +ĠA st +Ġca ption +ul ating +ĠAtt ack +Ġnew ly +Ġ199 7 +f r +Ġdis cipl +ĠGree k +Ġed ition +ĠDo es +ĠB ox +if le +ack et +Ġpass es +Ġgu est +Ġac celer +it als +U D +Ġaut hent +ĠR est +ov al +t a +u ine +Ġarm or +ĠT own +Ġcomp at +Ġinc hes +Des pite +Ġass ign +he rent +Ġprep are +ĠM eg +oc key +Ġdep ends +Ġtrack s +w atch +Ġl ists +ĠN orthern +Ġal ter +re c +ĠE astern +Ġcond em +Ġevery where +? ' +Ġaff ili +Ġf ought +": {" +Ġm ac +it arian +Ġsc ope +ĠA L +aw s +ar ms +Ġqu e +Ġenjoy ed +nes ota +Ġagg ressive +ĠSt ory +ĠI V +Ġrec ipe +Ġrare ly +ĠMed ical +val ue +ang el +ay ing +omet hing +Ġsub section +Ġs outhern +Ġfrequ ency +re te +roll ed +ult s +ĠN ic +Ġbeh alf +Ġsequ ence +ab et +Ġcontrovers ial +Ġcomp rom +Ġwork er +Ġmain ly +Ġal gorith +ĠM ajor +or ce +g ender +Ġorgan ized +Ġf ake +Ġconclud ed +ĠE D +ĠEx ec +r age +Ġch ances +ber ry +ĠTr ad +Ġconfig uration +Ġwithd raw +Ġf ro +ud es +ĠBro ther +ĠB rian +Ġtri es +Ġsam ples +Ġb id +ĠGold en +Ġphot ograph +if est +ĠD O +ĠPar liament +******** ******** +R em +Ġcont est +Ġsign ing +p x +ĠZ eal +âĶĢ âĶĢ +E ar +Ġex it +Be fore +ĠCor por +n ull +mon th +Ġrac ial +ott ed +ĠV eg +ĠRe uters +Ġsw ord +ps on +ĠRom ney +a ed +Ġt rib +Ġin ner +Ġprot ocol +ĠB i +ĠM iami +ever al +p ress +Ġsh ipping +ĠAm endment +ĠHow ard +con nect +ĠD isc +ĠJ ac +iam ond +ĠThere fore +s es +ĠPrin cess +ĠUS B +ĠAn th +Ġsurve illance +Ġap olog +Ġ6 1 +ow a +Ġf ulf +j s +Ġl uck +ust ed +Ġ § +n i +Ġant icip +em an +Ġwin ner +Ġsil ver +ll a +ic ity +Ġunus ual +Ġcr ack +Ġt ies +e z +Ġpract ical +Ġprov ince +ĠPl ace +Ġprior ity +IC E +Ġdescrib es +Ġbr anch +F orm +ask a +miss ions +b i +Ġp orn +ĠTur k +Ġent hus +Ġf ighters +Ġ0 8 +ĠDet roit +Ġfound ation +av id +A re +Ġjud gment +cl ing +Ġsol ve +ĠDes ign +W here +hes is +ĠT ro +a fter +Ġne utral +ĠPalestin ian +ĠHolly wood +Ġadv is +ĠN on +y es +ol is +Ġrep utation +Ġsm ell +Ġb read +ĠB ul +ĠBe ach +Ġclaim ing +Ġgen etic +Ġtechn ologies +Ġupgr ade +row s +Ġdevelop er +ĠJ osh +ĠDis ney +erv ed +ip al +Ġun ex +Ġbare ly +t hen +ĠP ub +Ġill ness +et ary +ĠB al +Ġp atch +Ġbut t +Ġst upid +ĠD og +ĠD allas +f ront +ie ce +Ġprot ests +Ġch at +oen ix +Ġw ing +Ġpar liament +Ġ7 7 +ose xual +Ġre nder +pt ions +ĠCo ast +os a +ĠG reg +h op +ĠMan agement +Ġbit coin +Ġrec over +Ġincor por +or ne +ĠUs ing +Ġpre ced +Ġthreat ened +Ġspirit ual +ĠE vent +ĠF red +Ġadvert ising +Ġimprove ments +ĠC ustom +Ġer rors +Ġsens itive +ĠN avy +Ġcre am +L ook +Ġex clusive +Ġcomp rehens +Ġde leg +Ġcon ce +Ġrem em +Ġstruct ures +Ġst ored +N D +Ġ1 000 +U P +ĠB udd +A F +w oman +ĠAcad emy +ð Ł +se a +Ġtem porary +Ab out +es ters +Ġtick ets +Ġposs ess +in ch +o z +Ġl a +Ġcontract s +Ġun p +Ġc ig +ĠK at +ult ural +as m +Ġmount ain +ĠCapt ain +St ep +m aking +ĠSp ain +Ġequ ally +Ġl ands +at ers +Ġreject ed +er a +im m +ri x +C D +Ġtrans action +g ener +less ly +Ġ| | +Ġc os +ĠHen ry +Ġprov isions +Ġg ained +Ġdirect ory +Ġra ising +ĠS ep +ol en +ond er +Ġcon sole +in st +Ġb om +Ġunc ertain +1 50 +ock ing +Ġmeas ured +Ġpl ain +Ġse ats +Ġd ict +S L +af e +Ġest imate +iz on +at hered +Ġcontribut ed +Ġep isodes +omm od +G r +AN T +Ġ6 9 +G ener +Ġ2 50 +vious ly +rog en +Ġterror ism +Ġmove ments +ent le +oun ce +ĠS oul +Ġpre v +ĠT able +act s +ri ors +t ab +Ġsuff er +Ġn erv +Ġmain stream +ĠW olf +Ġfranch ise +b at +Ġdem ands +Ġag enda +Ġdo zen +Ġclin ical +iz ard +ĠO p +t d +Ġvis ited +ĠPer haps +Ġact or +Ġde lic +Ġcont ribute +Ġin ject +ĠE s +ac co +Ġlist ening +Ġcon gress +epend ent +Ġprem ium +Ġ7 6 +ĠIr ish +Ġass igned +ĠPh ys +Ġworld wide +Ġnarr ative +ot ype +m ont +b ase +ĠB owl +ĠAdminist ration +Ġrel ation +ĠE V +C P +Ġco vers +Ġ7 8 +Ġcert ific +Ġgr ass +Ġ0 4 +pir acy +ir a +Ġengine ering +ĠM ars +Ġun employ +ĠFore ign +st ract +Ġv en +Ġst eal +Ġrepl ied +Ġult imate +Ġtit les +d ated +Ġj oy +a us +Ġhy per +ak u +Ġoffic ially +ĠPro duct +Ġdifficult y +per or +Ġresult ed +rib ed +l ink +wh o +~~ ~~ +ĠSpe ed +ĠV iet +W ind +ĠBar ack +Ġrestrict ions +ĠSh are +Ġ199 5 +ition ally +Ġbeaut y +op t +Ġm aps +ĠC R +ĠN ation +ĠCru z +W ill +Ġelectric ity +Ġor g +Ġb urd +Ġviol ation +Ġus age +Ġper mit +ĠCh ron +ĠF ant +Ġn aturally +Ġ0 7 +Ġth rown +ĠAw oken +Ġal ien +ĠHer o +ĠK ent +ĠR ick +ri ke +Ġp ace +}, {" +G L +Ġpo ison +ĠT ower +Ġform al +al ysis +Ġgen uine +Ġk il +a ver +Ġproced ure +ĠPro p +intend o +ĠM ain +as ant +Ġtr ained +G ame +ĠL oad +ĠM A +Ġcru cial +Ġle ts +ĠF R +Ġch ampion +1 01 +ĠCon ference +Ġwrit ers +Ġconnect ions +Ġo kay +ir ms +ĠR and +Ġenc ounter +ĠB uff +Ġachie ved +Ġche cks +isc ons +Ġassist ant +Ġwhen ever +ĠA ccess +ĠU r +b in +Ġcl ock +is p +op her +Ġb orrow +Ġm ad +Ġperson ality +on ly +IS T +ab ama +Ġg ains +Ġcommon ly +Ġter r +Ġhyp ot +Ġre ly +Ġt iss +iscons in +Ġrid ic +f unction +ĠO regon +Ġun com +r ating +el and +ĠN C +Ġm oon +ann on +Ġvulner able +ut ive +³³ ³³ +ĠRad io +Ġw estern +se ct +ĠT ony +Ġocc urs +ĠO s +ĠH on +Ã Ń +Ġv essel +ĠScot land +Ġdiscrim ination +Ġsubsequ ent +st ring +Ġfant asy +ĠSh adow +Ġtest im +W E +it i +r as +Ġbo at +Ġmar ks +Ġord inary +Ġre n +Ġrepresent ative +Ġpet ition +Ġ7 3 +Ġad venture +Ġign ore +ĠPhil adelphia +ĠS av +V P +Ġfact ory +Ġt asks +Ġdep ression +z ed +................ ................ +ĠSt orm +Ġc ogn +Ġelig ible +Ġredu cing +v ia +Ġ0 5 +Ġstri king +Ġdoll ar +h o +O V +Ġinstr ument +Ġphilosoph y +ĠMo ore +ĠA venue +Ġrul ed +ĠFr ont +IN E +ĠM ah +Ġscen ario +ĠNAS A +Ġen orm +Ġdeb ut +Ġte a +T oday +Ġabs ence +S im +Ġh am +le ep +Ġt ables +ĠHe art +M I +K e +re qu +V D +m ap +Ġchair man +Ġp ump +Ġrapid ly +v i +Ġsubstant ial +E P +d es +ch ant +ili pp +ĠS anta +ri ers +anche ster +L oad +ĠC ase +Ġsa ving +Ġ7 4 +ĠA FP +er ning +oun ced +ĠMin nesota +ĠW as +Ġrec ru +Ġassess ment +ĠB ron +U E +Ġdynam ic +Ġf urn +ul ator +Ġprop ag +h igh +Ġacc ommod +Ġst ack +ĠS us +w rit +Ġre ven +ĠGod d +ĠZeal and +ab s +Ġbr ut +Ġper pet +h ot +Ġhard ly +ĠB urn +ãĤ ¹ +Ġst y +Ġtrans actions +Ġg ate +Ġsc reens +Ġsub mitted +Ġ1 01 +Ġlangu ages +ugh t +em en +Ġfall s +Ġc oc +Ĥ ¬ +Ġstri kes +p a +Ġdel iber +ĠI M +Ġrel ax +ann els +ĠSen ator +Ġext rem +Ġ} , +ĠDe b +Ġbe ll +Ġdis order +c ut +Ġi OS +Ġl ocked +Ġem issions +Ġshort ly +" ] +ĠJud ge +ĠS ometimes +Ġr ival +Ġd ust +Ġreach ing +F ile +¯¯ ¯¯ +ino is +ĠJ ason +Ġs atell +are t +Ġst ations +Ġag ric +ĠTechn ology +com es +ĠUn fortunately +ĠChild ren +Ġappl ies +ast ed +Ġan ger +ail ability +ĠDam age +Ġcomp are +ĠStand ard +Ġaim ed +ĠB a +angu age +Ġreg ulation +Ġj ury +Ġair port +Ġse ctions +ĠPr ince +em ed +Ġmedic ine +Ġh itting +Ġsp ark +ol ves +Ġad s +St ate +Ġfood s +Ġrepl acement +Ġch icken +Ġlow est +Ġmind s +Ġinvol ves +u i +Ġarr ang +Ġproced ures +ĠWh ich +ivers ary +Ġb ills +Ġimprove ment +Ġin ev +Ġexpect ations +Ġintellect ual +Ġsp aces +Ġmechan ism +2 50 +bre ak +ĠZ e +ĠT enn +ĠB alt +Ġbar rel +Ġstat ic +man n +Pol ice +Ġt ips +Ġhand ling +c us +od ed +il ton +ir y +Ġjournal ists +our se +Ġcom ic +Ġnom ine +IT Y +Ġvers us +Ġlo op +Ġsur f +ĠInd ust +ĠHun ter +Ġbelief s +is an +Ġset up +Ġbre w +im age +Ġcomput ers +f ol +} ," +ĠMed al +Ġtax p +Ġdisplay ed +Ġg rav +Ġf iscal +M on +ĠMos cow +ĠK ong +ĠCent re +Ġcamer as +ĠMr s +ĠH ay +Ġa ver +ĠK elly +p y +Ġrequire ment +Ġent itled +omb ie +Ġsh adow +ag ic +ĠA k +Ġel ite +Ġdiv ided +Ġhead ing +Ġcop ies +Ġloss es +Ġv it +k ed +ĠB ry +Ġan s +ĠSte am +Ġrep orter +he im +ĠIt em +Ġsuper ior +d on +ere nt +à ¶ +Ġtherap y +Ġpe ak +ĠMod el +Ġl ying +Ġg am +z er +r itten +Ġrespons es +Ġconsider ation +ĠB ible +Ġl oyal +Ġinst ant +Ġp m +ĠFore st +à ¼ +Ġext end +Ġconv icted +Ġfound er +Ġconv in +ĠO ak +che ck +Ġsch olars +p ed +Ġover se +T op +c ount +ĠAr k + · +Ġ0 6 +ĠL A +m d +ĠLat in +im ental +ĠC PU +Ġsubst ance +Ġminor ity +Ġmanufact uring +E r +ocol ate +Ġatt ended +ĠMan ager +r ations +Ġappreci ate +om y +GB T +id ency +B L +Ġguarant ee +pos ition +Ġo cean +clud e +Ġhead ed +Ġt ape +Ġlo ose +Ġlog ic +Ġpro ven +Ġsp ir +Ġad mit +is a +Ġinvestig ate +Ġ199 4 +sy lv +ĠL ost +c est +Ġ7 1 +Ġrequest ed +Ġwind ows +ĠPok é +ĠWith out +M et +Ġbehavi our +Ġread er +Ġh ung +ĠKe ep +Ġro les +Ġimplement ed +Ġbl ank +Ġserv es +ĠJ ay +Ġc ited +ĠF riend +prof it +ap on +Ġrep air +it em +arr ass +Ġcrit ics +ad i +ĠF ather +Ġsh out +Ġf ool +Ġ8 8 +Ġprodu cing +Ġl ib +Ġround s +Ġcirc le +Ġpre par +Ġsub mit +Ġn ic +mor row +ãĥ « +U nder +Ġv ital +ater n +Ġpass word +Ġpublic ation +Ġprom inent +Ġspeak s +Ġb ars +Ġde eper +ĠM ill +port ed +Ġw id +Ġbut ter +Ġsm oking +Ġindic ates +K ey +rop ri +ĠF ile +all ing +ast ing +ĠR us +Ġad j +Ġ7 9 +av al +Ġpres um +bur gh +on ic +Ġf ur +Ġpoll s +ik a +Ġsecond ary +Ġmon ster +ig s +ĠCur rent +E vent +Ġowners hip +end ar +Ġarri ve +ĠT ax +Ġn ull +ĠPri v +Ġth ro +Ġk iss +c at +Ġup set +ang le +it ches +ect or +olog ists +ĠGal axy +Ġcor ruption +Ġh int +ent er +ĠH ospital +Ġgreat ly +Ġbeg un +es y +Ġso il +ĠAnt on +Ġmain tenance +ãĥ © +Ġdo zens +Ġhuman ity +ĠAl abama +Ġr om +w orth +ap ing +sylv ania +l ah +Ġg athered +G A +Ġattack ing +f ound +ĠSqu are +Ġar bit +ict ions +ĠW isconsin +Ġd ance +ĠS aint +arch y +Ġbase ball +Ġcontribut ions +Ġliter ature +Ġex ha +per ty +t est +Ġb ab +Ġcontain er +let ter +Ġfall en +Ġwebs ites +Ġbott le +ĠS ac +Ġbre ast +ĠP L +Ġveter an +Ġinterview s +ĠA le +Ġb anned +eng ers +ĠRev olution +in th +Ġconc erning +IV E +Ġexp enses +ĠMatt hew +ĠColumb ia +d s +ist ance +Ġent ity +.. ." +Ġrel iable +Ġpar alle +ĠChrist ians +Ġopin ions +Ġin du +l ow +Ġcompet e +Ġth orough +Ġemploy ed +Ġestablish ment +ig en +ĠC ro +Ġlawy ers +ĠSt ation +T E +ĠL ind +ĠP ur +it ary +Ġeffic iency +âĢ IJ +ĠL y +Ġm ask +Ġdis aster +Ġag es +ER E +es is +ĠH old +Ġcas ual +b led +Ġen abled +ĠEn vironment +ĠInt elligence +i per +ĠM ap +ĠB E +Ġemer ged +is dom +Ġc abin +Ġregist ration +Ġfing ers +Ġro ster +Ġfram ework +ĠDo ctor +et ts +Ġtransport ation +Ġaware ness +H er +Ġattempt ing +O ff +ĠSt ore +ÃĥÃĤÃĥÃĤ ÃĥÃĤÃĥÃĤ +ĠK now +Ġdef ence +Ġsc an +ĠT en +ĠCh air +ĠP H +ĠAtl anta +Ġfuck ing +Ġans wered +b n +ĠK ar +Ġcateg ories +Ġr ational +Ġc ust +Ġrob ot +Ġcorrect ly +Ġg if +Ġgraph ics +m ic +Ġground s +ĠO pp +i ate +Ġdist ributed +Ġsan ctions +Ġchalleng ing +ut o +Ġingred ients +Ġinv ited +Ġfound ed +ĠRe qu +d ed +Ġb owl +Ġbrother s +ĠH a +I O +Ġw ages +im ore +oc ial +Ġse ed +ative ly +Ġaddress es +ĠI owa +ab eth +Ġatt itude +is d +ch ild +Ġm ole +Ġdisco very +y ard +B r +Ġ8 2 +Ġsuppl ies +ell ing +Ġdist ingu +C R +Ġre cept +Ġ vert +Ġsw im +b ec +d oor +ĠY eah +Ġg al +Ġinter act +ĠE SP +ĠC S +amp s +Ġconvin ced +Ġobject ive +Ġdis h +ĠPhot os +l ad +Ġdownt own +o il +in ction +Ġto morrow +ĠC OM +Ġsurv ival +sh ot +Ġsett lement +C ons +ĠX box +int erest +ĠS M +arg o +en ess +Ġeth nic +b ered +M in +ĠT ok +Ġinc ent +ĠComm and +Ġmain tained +Ġbreak s +br idge +at ar +ag g +ĠF inally +un icip +ĠO nt +le ft +Ġrecogn ition +Ġ* / +ĠP ers +Ġwe lf +Ġaddress ed +ĠK ansas +Ġvir us +Ġwhere as +Ġp apers +ram s +ĠMin istry +Ġple asure +Ġacqu ired +Ġd uration +j pg +Ġcal m +ĠN HL +Ġburn ing +Ġfold er +ick ed +ĠP y +ĠIll inois +Cl ass +ĠGodd ess +Ġperform ing +Ġwelf are +j ar +In ter +Ġl in +Ġenh ance +Ġnot ion +f are +yp es +ĠAre a +Ġcann abis +ĠDie go +f s +ĠM anchester +com m +in ite +Ġcover ing +ĠS ound +Ġ19 60 +Ġ8 4 +e lect +z ing +Ġcitiz en +Ġph ones +Ġr aid +Ġign ored +ĠOb ject +Ġu pload +c ard +Ġmod ified +Ġroom s +ia h +r ange +he ast +ach us +Ġsuggest ing +âĢ ĭ +gr ade +E l +Ġclot hing +Ġr h +ĠH an +un ity +en cing +ĠAust in +sec ution +t ra +d em +ĠQ ual +Ġhe aven +Ġst ages +Ġw edd +pl us +ific ial +ĠIm m +ĠH o +iet ies +Ġphr ase +Ġbr ill +act ory +Ġprov iders +Ġsil ence +Ġa er +ĠA I +ĠAd venture +Ġplatform s +Ġdemonstr ated +Ġinter f +ing ton +Ġr aces +Ġgr ade +ult ane +ĠTh rough +f alse +Ġb ow +ĠA B +Ġfl avor +Ġhistor ic +g ov +Ġcol our +Ġview ed +ĠEm ail +el come +Ġinter vention +Ġd iversity +Ġperiod s +Ġre verse +ĠV ery +Ġqu ote +ĠLe ft +th rough +Ġsc rew +Ġland ing +Ġp ill +Ġw et +Ġprot esters +Ġrepe at +av ed +er k +Ġsal ary +ĠPenn sylvania +St ill +Ġmay or +Ġkit chen +Ġfeat uring +ĠM useum +ĠT ournament +ĠF al +Ġser vers +U C +Ġany body +im g +ĠTr ade +ixt ure +the less +Ġfin ance +Ġcl osing +ĠPat ri +i ac +ab el +Ġ> > +or ous +Ġf irms +sc reen +un a +Ġemb arrass +ul se +Ġlet ting +Ġth rew +ile y +Ġch annels +l an +ĠVeg as +Ġse ar +Ġfant astic +ar re +uzz le +ĠD er +Th ose +Ġsw ing +Ġshe et +ind ex +co ver +og an +Ġvari ables +ĠTe ch +Ġsp oken +ac hel +ĠD a +ĠMount ain +Ġload ed +Ġfoot age +vers ion +Ġun l +ĠPh oenix +Ġthrow ing +Ġf iring +Ġtrack ing +Ġw idth +Ġstrugg ling +ro oms +ot ion +Ġmonth ly +ĠSer ver +Ġegg s +op en +M C +Ġ199 3 +Ġh ired +Ġstay ed +ĠAll en +Ġst ro +Ġ9 8 +st ep +ĠTurk ish +Ġfab ric +ist ing +ĠD om +Ġd ates +Ġpr on +Ġbasket ball +Ġl ucky +ĠArab ia +Ġassum ed +est y +Ġaff airs +Ġgl ad +ĠInd eed +ĠF A +ĠW ord +Ġjo ining +if ice +p read +ir ts +ĠSe lect +Ġpop ulations +aw are +Ġn ose +Ġcompl aints +st art +Ġsc oring +Th anks +Ġmin ing +Ġvisit ors +S H +Ġdam aged +Ġcharacter istics +ĠP ent +D C +Ġ8 3 +ĠS ix +r ates +Ġfl ags +ĠB rew +d og +M ark +// // +Ġexec ution +Ġj oke +ph ones +Ġtestim ony +Ġob st +Q L +ĠC ut +Ġstud ied +ĠN intendo +ick et +ĠN BC +Ġl ad +ĠB ra +ĠM oh +Ġk ernel +Ġoverwhel ming +Ġag ed +Ġapplic able +ĠC ond +Ġroad s +ĠBl ock +m ade +od ge +Ġcomm ands +Ġoff ices +vel and +Ġt ut +Ġrece iver +ĠF ro +Ġsho pping +Ġi P +ĠSt re +ĠA BC +Ġentertain ment +ĠB ow +ort ed +M c +Ġread s +gr ad +ĠCol lect +Ġâ ĪĴ +ĠCap ital +eder ation +Ġemploy er +Ġinvolve ment +Ġanx iety +al ia +Ġro of +ĠAm ong +ĠDemocr at +Ġstat s +ĠV ill +Ġconst itutional +Ġrefer ring +itt y +Ġtack le +out ube +Ġback ed +ĠH ong +ĠBro ad +Ġe le +ĠO tt +Ġ199 2 +h our +achus etts +C al +Ġdefe ated +Ġ8 1 +es p +Ġseem ingly +w as +ĠJ enn +ĠK urd +Ġg ene +Ġdisc ount +R et +EC T +( ); +Ġclub s +Ġs id +ĠM arsh +Che ck +Ġp p +ĠE ag +ides pread +Ġbe ings +F T +Ġintrodu ction +ĠCh ange +AR D +Ġ1 10 +ad ows +ier ce +Ġme al +a uthor +ĠB ang +lah oma +Ġr anks +201 1 +?? ?? +m ax +Ġcoll apse +Ġop ens +Ġe cho +Ġs oph +Ġrac ist +Ġenorm ous +Ġw aves +Ġt ap +Ġcomprehens ive +. -- +ĠR oy +Ġfarm ers +Rel ated +a ired +ron es +ĠC rim +Ġproport ion +Ġdesign s +Ġnegoti ations +Ġvirt ually +ĠBat man +Ġwar n +Ġlegit imate +m ate +Ġcon vention +, , +net ic +ĠS D +Ġconsist ently +Ġcompens ation +Ġpunish ment +Ġy e +Ġt ie +ĠB ureau +ir lf +ĠB u +ĠA ren +ĠPh ilipp +Ġkn ife +Ġmem ories +ĠR oss +Ġang le +Ġ8 6 +ĠTh under +Ġre nd +ĠT our +Ġcount s +s ung +ĠIm p +Ġeduc ational +Ġaccess ible +C OM +Ġd rew +y er +G l +am ine +OR T +O B +I B +m aster +Ġtri als +og y +h ar +ĠTr ust +Ġprefer red +irlf riend +ĠN ev +Ġb in +Ġc ow +P age +Ġsign ature +ĠB L +7 00 +Ġret ired +Ġby tes +Ġneigh b +ĠLeg end +Ġdev ast +Ġsuspect ed +is ons +ĠPoké mon +sc ale +Ġcap abilities +Ġre vel +Ġche ese +d y +igr ant +Ġfail ing +b its +ĠHer oes +ĠG host +ĠS cient +Ġappoint ed +ur i +Ġinst itution +Ġexpand ed +g reg +Ġmonitor ing +Ġp odcast +Ġcoal ition +Ġ9 6 +J o +Ġst olen +ĠS ab +Ġstop s +Ġhol iday +Ġint r +C ar +Bl ack +ĠL GBT +Ġwar ming +ĠAnd erson +Ġ8 9 +Ġprodu cer +M ed +Ġaccur acy +ĠMar vel +iz abeth +ĠPat rick +m ony +Ġmin i +ac les +Ġover t +the y +Ġmembers hip +ĠV en +Ġex ch +Ġrem oval +ĠD ave +T Y +m ad +ĠF ind +Ġad equ +Ġe c +Ġte eth +Ġemot ion +Ġper m +Ġsole ly +d b +Ġextra ord +IG HT +c al +Ġgu idelines +Ġd ying +Ġsusp ended +ĠPrem ier +ĠAnth ony +el ve +Ġd ad +ĠE th +ĠFoot ball +Ġabandon ed +Ġ< < +Ġm arch +Ġhor ror +âĢ¦ " +Ġchild hood +Ġcampaign s +Ġl unch +ĠAl bert +bl ock +âĸĪ âĸĪ +ound ing +Ġb one +or gan +ad ers +ĠFl ash +ĠDri ve +Ġton ight +Ġw ars +ĠF L +Ġform ation +con st +New s +Ġcom pe +or ious +ĠSt aff +Ġdiscuss ions +ĠProt ection +ĠJ am +Ġcrit eria +Ġinstall ation +Ġaccompl ish +iz za +Ġpub lisher +Ġresc ue +ĠT ry +U LL +ĠS om +ĠH op +ore t +th s +ord on +Ġp ocket +ĠIn v +Down load +ĠCr ime +Ġb ene +ĠGu ide +ĠAs sembly +Ġparam eters +I E +ĠAlex ander +Ġconc ert +ĠSc he +Ġsh oes +Ġvis iting +Ġrec all +Ġb ub +Ġr ural +Ġconc rete +ĠR os +N ext +R uss +Ġlo ans +ĠSh ield +Ġtre m +hem at +k g +ĠHar ris +is ition +ĠM ove +ĠF C +Ġf ate +ĠCh o +Ġt ired +Ġprinc ipal +h ist +ien ces +ath y +Ġse vent +Ġm ood +Ġstrateg ic +Ġdise ases +Ġfor um +Ġtem por +Ġhead quarters +P ar +ig e +fl ix +Ġgu itar +Ġ9 4 +On ly +Ġrele ases +ro ph +================ ================ +Ġ6 00 +ĠContin ue +ig ate +ĠC rit +sy stem +Ġdis abled +Ġunex pected +ith ub +Ġuncle ar +ĠE st +Ġcontr ad +Ġstrateg ies +vent ures +Ġpass age +AM E +Ġimpro ving +Ġreve als +Ġdecre ase +ov a +Ġann oy +ĠSh ort +ĠL ibrary +Ġcy ber +n ell +ĠH ur +ĠC B +Ġphot ograp +U I +Ġs ed +G e +Ġ8 7 +Ġd iverse +Ġencour aged +Ġcons piracy +Ġbird s +Ġoper ator +Ġhand ful +Ġclass ified +? ) +Ġdram atic +Ġinvestig ators +it o +Ġw idespread +ĠR oom +-------------------------------- -------------------------------- +Ġcollect ive +Ġjournal ist +St ring +Ġtemper atures +il a +Ġgu id +Ġins pect +Ġmiss ile +ĠMay or +Ġman ual +Ġsim ultane +Ġrat ings +Ġsu ck +Ġ9 7 +Ġunivers al +Ġph arm +Ġdis rupt +ian o +A V +Ġf t +Ġstat ist +old s +ĠWalk er +ph p +Ġunder t +ĠL as +ish op +nt il +res hold +ĠWhe ther +M s +Ġden y +ĠCl oud +Ġprov ider +Ġsurv iv +ĠUp date +h as +Ġmist akes +ch arge +pl ed +r ity +Ġn ode +ĠMass achusetts +ool s +lic ation +Ġf ails +em ale +or i +back s +Ġsh irt +Ġ' ' +ĠN AT +Ġwat ers +els on +Ġe ase +Ġsc ar +Ġcont ents +m ind +Ġcont ribution +Ġsh r +Ġhand ed +Ġst ability +Ġtra ve +E m +Ġmir ror +12 3 +Ġwe igh +Ġf iction +ou ver +ist ant +r ition +ĠF ed +Ġphys ically +Ġst ake +ĠArt icle +ĠAr c +ĠLew is +ĠM ind +Ġdemonstr ate +Ġprof its +v ision +om ic +ol id +Ġbatt les +Ġdri ves +Ġeas tern +ĠS ony +!! ! +ar ation +v ard +ĠG L +port ation +Ġ9 2 +Ġlaw makers +Ġprotect ing +ĠE PA +Ġy eah +Ġsh ame +ol ph +e ven +x it +Ġatt ach +Ġrepresent ing +Ġob s +ĠUt ah +iff s +ĠFre edom +à ³ +A K +Ġinc idents +it age +Ġview ers +c d +Ġm ouse +Ġcl ar +Ġaccord ance +Ġb ot +c or +ĠSum mer +he ld +Ġinnoc ent +Ġiniti ative +ol s +________________ ________________ +Ġsp ots +p ace +Ġconvent ional +Ġcorpor ations +Ġblock ed +H D +at tered +Ġref ers +Ġbu ck +ĠDig ital +12 0 +Ġtop ics +T F +Ä ģ +br id +re ement +Ġunder lying +ĠM ember +Ġinvestig ating +Ġpregn ancy +Ġtouch down +ĠB and +ĠCall er +Ġinst ances +P P +w a +G ood +Ġ199 1 +ĠC old +Ġfear s +Ġrem arks +Ĩ Ĵ +at al +Ġm it +Ġexper iments +i pt +Col or +ind u +Up date +Ġ9 3 +A g +Ġ å +anc ouver +B oth +Ġjud ges +Ob ject +Ġst ere +umb n +Ġparticip ation +ĠSt ars +ĠJ ere +Ġweek ly +ĠB an +Ġconvers ations +ĠP itt +u z +ĠIndian a +ĠK ick +Ġinf ection +Ġhero es +Ġsett led +Ġstri p +Ġh al +Ġd ump +ĠS ci +Ġl es +Ġref erences +ĠU RL +ĠBr idge +Ġwant ing +For ce +Ġex clus +Me anwhile +m n +Ġg entle +m aker +sen al +ĠG ro +ou ri +ĠR ain +ĠAll iance +Ġl ift +el a +S D +ĠCle veland +Ġrank ed +Ġst adium +Ġdead ly +ä ¸ +Ġr iding +ar ia +ĠAr mor +Ġdocument ation +ĠGree ce +ree k +Ġl ens +ĠS a +Ġg ross +ĠE mer +ag ers +ĠD ub +ĠR h +ĠAM D +Ġarri val +Ġdes ert +Ġsupp lement +ĠRes p +Ġkn ee +Ġmarg in +f ont +og g +201 0 +ĠP ir +ĠP rom +iv als +Ġint ake +Ġdifferent ly +ug s +Ġb its +clud ed +Ġsearch ing +ĠD u +um ble +Ġfunction al +ĠBalt imore +ĠC ould +Ġdes ired +Ġcirc uit +ĠL yn +ĠG O +ĠF alse +re pre +' : +alt ies +Ġmin im +Ġdro ve +ĠSh ould +Ġh ip +Ġpro s +Ġut ility +ĠN ature +ĠM ode +P resident +o pp +r at +form ance +Ġconcent ration +Ġf ont +ĠB ud +Ġam id +Ġre vers +ĠM L +B ar +Ġinter action +Ġjur isd +Ġspell s +d ep +f il +Ġcivil ians +ut ter +ĠCo oper +ĠBel ow +Ġent rance +Ġcon vert +Ġcontrovers y +ow ered +Ġcontr ary +Ġar c +ĠExec utive +ĠOffic er +Ġpack ages +Ġprog ressive +w idth +Ġreserv ed +v ol +ĠSam sung +Ġprint ed +Ġcent ers +Ġintrodu ce +ĠKenn edy +Ġodd s +Ġsure ly +Ġindepend ence +Ġpass engers +repre ne +ĠBe h +Ġl oves +ĠESP N +Ġfac ilit +Ġident ical +Ġdo ct +Ġpartners hip +con f +ĠH ide +Ġconf used +ĠC ow +M en +Ġw rest +ĠIraq i +Ġh oles +ĠStud ies +Ġpregn ant +h ard +Ġsign als +I X +Ġpull ing +Ġgrad uate +Ġnomine e +D ate +Ġper mitted +Ġâ Ĥ¬ +ĠOk lahoma +St art +Ġauthor ized +Ġal arm +ĠC os +v an +Ġgener ations +c ular +Ġdr agon +ĠSoft ware +ĠEd ward +Ġcontro ller +S en +ge red +ĠV ik +Ġappro ached +Th ank +Ġcan ce +Ġform ula +ĠSm all +Ġweak ness +Ġr amp +it udes +j ud +Ġbrill iant +Ġacc us +s ource +Ġ8 00 +ĠE vil +S w +Ġhom eless +we ek +i ens +r ics +ĠTh ird +T O +Ġorgan ic +Ġpresent ation +ag h +ĠDown load +v ation +Ġas sembly +or able +hold ers +ĠBern ie +ĠHel p +Ġt ong +ĠF ight +Ġbe ach +B ook +ĠL ic +Ġr ush +ĠR ound +ou p +ĠMar x +Ġcalcul ated +ĠDe vil +ĠSar ah +Ġoccasion ally +Ġbul let +Av ailable +g ate +Ġ9 1 +Ġh osp +Ġprom ises +ĠH IV +ĠSt adium +ĠSt ock +ĠCorpor ation +g age +N G +ĠC redit +Ġs ne +ib l +Ġacc um +s uch +Ġterror ists +Ġconscious ness +ĠZ h +Ġdram a +ool a +pir ation +Ġlab our +ĠN in +Ġut ter +Ġdemocr atic +Ġass ass +il ation +Ġg est +Ġab road +Ġmet ab +Ġs orts +Ġfl av +U B +Ġm g +ĠNot hing +ĠO d +Ġmus ical +200 9 +Ġdro ps +oc ated +ater al +0000 00 +Ġg re +Ġequ ality +Ġburd en +Ġv ig +ĠLe ader +-------- ---- +Ġcere mony +Ġf ighter +Ġact ors +Ġ æ +am an +F i +Ġal ign +put er +Ġe lder +ĠN SA +Ġrepresent ation +ĠOnt ario +IT H +usal em +Ġharass ment +itz er +Ġsy mp +Ġbox es +ĠD R +Ġman ifest +at re +Ġ ^ +Ġd ies +le ton +Ġmiss ions +et he +Ġres olve +Ġfollow ers +Ġas c +Ġk m +l ord +am med +Ġsil ent +ĠAssoci ated +Ġtim ing +Ġprison ers +ĠK ings +ĠF ive +Ġtow er +Ġappro aches +Ġprecise ly +Ġb ureau +ĠM other +ĠI ss +Ġkey board +it ual +Ġfund ed +Ġstay ing +Ġpsych ological +Ġm ile +ĠLe on +ĠBar b +w ill +Ġw ider +ĠAtl antic +Ġt ill +ĠR ome +ro t +Ġaccomp an +Ġfl our +ac o +W orld +ĠExp ress +ĠY u +C or +Ġple ased +part y +Ġpoint ing +Ġinf lation +Ġro y +Ġ ), +ain er +Ġwedd ing +orm on +Ġrequ iring +Ġqual ified +Ġse gment +EN D +Ġs izes +e als +Ġcor rupt +ass ador +Ġcele b +Ġdream s +ĠM ess +Ġcheck ing +ĠV ersion +Ġprep aring +Ġact ively +ĠD iff +Ġl ux +ĠW inter +act eria +ĠN E +Ġdep uty +Ġtrans gender +Ġsum mary +Ġin her +er ies +ch ar +ĠY an +Ġkn ock +ĠP ath +Ġl ip +roll er +Ġimp ression +Ġcelebr ate +Ġsl ide +Ġgu ests +Ġcl ip +F S +Ġsav ings +Ġcapt ain +Ġleg acy +ĠDen ver +Ġw ounded +tab oola +AC T +Ġpurs ue +Ġo xy +Ġ q +Ġsem i +ĠN eed +ĠAff airs +Ġob sc +Ġcheck ed +Ġd ual +C ode +ĠM D +le m +ult y +Ġ © +ĠEl izabeth +Ġcent uries +ard ed +s rc +Ġev ident +enn is +at in +Ġunemploy ment +ĠMar io +Ġint im +Ch rist +Ġbi ological +Ġsold ier +ĠAdd ed +Ġm ath +ĠG il +Ġbi as +Ġd ating +ĠO cean +Ġm ice +M us +h ire +ĠT es +Ser ver +lim ited +S ize +Ġmet ers +Ġrock et +es see +Ġcertific ate +ĠIran ian +AS S +Ġgr id +D ec +Ġro lling +com mun +ĠSwed en +b ury +Ġtiss ue +Ġrac ism +ĠL ocal +Ġmyster y +Ġexam ine +Ġst em +Ġs its +Ġhop ed +ot ing +Ġdial ogue +Ġpers u +W atch +l ay +M AN +Ġch ronic +ĠPort land +mark et +ĠS EC +Ġparalle l +Ġsc andal +Ġcar ries +Ġphenomen on +h uman +ack er +ĠO x +Ġretire ment +tain ment +ov ie +ĠG ear +Ġd uties +Ġdo se +Ġsc roll +M B +in f +Ġsa uce +Ġland scape +red dit +ĠChampions hip +ĠRed dit +al id +Ġco in +Ġover s +Ġpost ing +ab out +Ġf el +and y +Ġb old +Ġfocus ing +e ffect +G R +Ġde emed +Ġrecommend ations +Ġste pped +Ġvot er +ĠDe ep +ĠInst agram +Ġmoder ate +ĠMary land +Ġrestrict ed +ĠM B +ĠCh all +Ġto b +Ġc ir +ĠO cc +ĠE ver +Ġcoll aps +IN FO += - +ĠP ict +ĠAcc ount +n c +Ġo ught +Ġex port +Ġdr unk +( ' +Ġw ise +ĠM ort +ne cess +Ġan cest +ĠInc re +Ġfrequ ent +m ir +Ġinterpret ation +Ġdepend ent +Ġco ins +ĠB ol +V ideo +ĠJust in +Ġfat al +Ġcook ing +Ġconf usion +ip her +Ġcust ody +ĠMor gan +om ach +ĠGovern or +Ġrestaur ants +el ing +Ġacknowled ged +Ġthe r +Ġgen es +ch ing +He y +Ġtact ics +ĠMex ican +Ġv end +Ġhe s +qu er +Ġnot ing +ĠCamer on +Ġtarget ing +ro ck +Ġcred its +Ġemot ions +Ġrepresent atives +new s +Ġlegisl ative +Ġrem oving +Ġtweet ed +ĠCar ter +ĠF ixed +Ġfor cing +Ġspeak er +Ġm ales +ĠViet nam +l ined +Ġconcept s +Ġvo ices +o ir +ĠT rib +W he +ĠJer usalem +ĠS ant +Ġc ul +Ġl ady +ĠHaw ai +Ġar ts +ĠIn n +ĠMach ine +ĠEm peror +Ġsl ot +g ly +ĠPro cess +II I +Ġathlet es +ĠTem ple +ĠRep resent +Ġpres c +Ġt ons +Ġgold en +Ġp unch +ĠG R +iver pool +Ġen act +Ġlob by +Ġm os +Ġpick ing +Ġlif etime +Ġcogn itive +E ach +z o +Ġd ub +Ġcons ists +ol n +Ġf estival +am ous +Ġint ellig +w ords +ĠSm art +Ġde le +Ġl apt +Ġmag ical +ĠS in +b us +ur ities +igh th +ĠRub y +ĠS ure +ol ving +Ġj un +O ST +Ġimp osed +Ġast ron +Ġcor rel +ĠN S +ĠK it +ĠF uture +b urn +Ġimm une +oc us +Ġcour ses +ĠSt ring +Ġle an +Ġg host +Ġout comes +Ġexp ense +Ġevery day +Ġaccept able +A h +Ġequ ipped +Ġor ange +F R +ĠD utch +Th ough +ĠR ank +Q U +ĠRober ts +wh at +re nd +Ġdisapp ear +Ġsp awn +ĠL am +o is +Ġdes erve +Ġmin imal +Ġnerv ous +ĠW ould +Ġro ok +ĠV ancouver +Ġres ign +sh ire +ĠW orks +ĠB uild +Ġafford able +ĠG ary +ĠAren a +Ġh anging +Ġimpl ications +ĠS ong +Ġmain taining +Ġgu ards +C ON +Ġder ived +Ġexecut ed +Ġthe ories +Ġqu oted +ĠAnd re +og a +sel ess +in fo +ĠBel g +Ġt ears +ĠSur v +Ġbirth day +ig ious +im mer +Ġspect rum +Ġarchitect ure +Ġrec ruit +arm a +T able +Ġmon sters +ĠG ov +Ġdest ination +Ġattract ive +Ġf oss +ĠMore over +Ġpres ents +TH E +Ġrep ly +pt on +Ġc um +Ġdel ight +Ġaffect s +Ġdon ations +ĠT oy +ĠH im +M ENT +Ġover come +it ched +ĠFant asy +ĠH at +ĠBe ast +b ott +Ġinvestig ations +R un +Ġhun ting +d i +f und +Ġs essions +est yle +Ġport ray +oid s +Y eah +Ġcommun icate +Ġcom edy +ĠY ang +Ġbel t +ĠMar ine +Ġpredict ed +Pl ay +Ġimportant ly +Ġremark able +Ġelim inate +D avid +Ġb ind +V ID +Ġadvoc ates +ĠG aza +im p +D B +ĠN a +ĠSim ilar +I ES +Ġchar ity +v as +m ath +Ġâ ĸ +ok er +nd um +Ġcap s +ĠH al +2 000 +e an +Ġfle et +Ġrec re +R ight +Ġsleep ing +ij ing +k ind +Ġdesign ated +à ¤ +Ġanim ation +ke e +ĠInt rodu +Ġ/ > +Ġdelay ed +Ġtrem end +Ġcur ious +U se +Ġle ct +d am +Ġinnov ation +ĠPoint s +Ġload ing +Ġdisp ute +ct ic +ird s +ĠB Y +Ġn urs +ĠVal ue +ION S +ĠH um +Ġtem plate +m ers +Ġappear ances +ĠEnter tainment +Ġtransl ation +Ġsa ke +Ġbene ath +Ġin hib +Ġe uro +abet es +Ġstud ying +ĠM as +Ġper ceived +Ġexam ined +Ġe ager +Ġco aches +Ġim per +ch i +Ġprodu ces +" ). +ĠEvery one +Ġm unicip +Ġg irlfriend +Ġh ire +ĠV ice +Ġsu itable +op y +Ġin equ +ĠD uke +f ish +f irst +ĠO bs +Ġinter ior +ĠBru ce +ĠR y +Ġanal ys +Ġconsider able +Ġfore cast +Ġf ert +ors hip +ĠD rug +ĠA LL +: " +th ur +ĠM ail +Ġball ot +Ġinst antly +ĠCh annel +Ġp icks +Ġ198 9 +Ġt ent +ol i +Ġcivil ian +b ling +ell o +b u +Ġin ch +Ġlog o +Ġcooper ation +Ġwal ks +Ġinvest ments +Ġimp rison +ĠF estival +ĠK y +Ġleg ally +Ġg ri +ch arg +S l +Ġthreat ening +du ction +fl ow +Ġdismiss ed +ibr aries +c ap +e le +ĠMc G +ĠHar vard +ĠConserv ative +ĠC BS +p ng +Ġro ots +ĠH aving +umb led +ĠF un +\ / +ĠS earch +ple x +Ġdiscuss ing +Ġcontin u +ĠT ai +ĠW ik +F ree +f it +Ġref use +Ġmanag ing +Ġsy nd +ip edia +w alk +Ġprofession als +Ġguid ance +Ġunivers ities +Ġas semb +unt u +F inally +AS E +ĠAut o +ĠH ad +Ġann iversary +L D +ĠD ur +ĠUlt imate +ih ad +pro duct +Ġtrans it +Ġrest ore +Ġexpl aining +Ġass et +Ġtransfer red +Ġbur st +ap olis +ĠMag azine +ĠC ra +ĠB R +gg ed +ĠH E +M ich +b et +ĠL ady +yl um +erv es +Ġme ets +wh ite +L og +Ġcorrespond ing +Ġins isted +G G +Ġsurround ed +Ġt ens +Ġl ane +Ġco inc +h ome +Ġexist ed +ect ed +ĠDou ble +lam m +Ġske pt +ex p +Ġper ception +ie v +ĠBe ing +o ft +Ġadop t +. : +] ; +Wind ows +Ġsatell ite +AS H +Ġinf ant +d escription +ĠMe anwhile +c m +oc a +ĠT reat +act or +Ġtob acco +ĠN orm +em ption +Ġfl esh +Ġj e +o op +ĠHe aven +Ġbe ating +an im +Ġgather ing +Ġcult iv +G O +ab e +ĠJon athan +ĠSaf ety +Ġbad ly +pro t +Ġcho osing +Ġcontact ed +Ġqu it +Ġdist ur +Ġst ir +Ġto ken +D et +ĠP a +Ġfunction ality +00 3 +s ome +Ġlimit ations +Ġmet h +b uild +con fig +N T +re ll +ble m +ĠM om +Ġveter ans +ĠH u +Ġtrend s +are r +ĠG iven +ĠCa ption +m ay +AS T +Ġwond ering +ĠCl ark +n ormal +Ġsepar ated +Ġdes p +st ic +b rew +Ġrel ating +ĠN ik +ĠF arm +Ġenthus i +g ood +d eb +Ġactiv ist +Ġm art +Ġexplos ion +ĠEconom ic +L ink +Ġins ight +Ġconven ient +Ġcounter part +su pport +ĠV irt +ag en +ĠTenn essee +ĠSim on +ĠA ward +OC K +ĠF igure +Ġoverse as +Ġpr ide +ĠC as +n ote +m g +C urrent +Ġdispl ays +cont ent +Ġtravel ing +Ġhosp itals +ĠFin ancial +ĠP ast +Ġdefend ant +Ġstream ing +m ble +ĠBer lin +uk i +Ġdist ribut +Ġant ib +Ġch ocolate +ĠCast le +Ġinter rupt +ĠR ow +Ġconvers ion +Ġbug s +ĠR ather +li est +L Y +ĠJe an +com mon +ak h +Ġ1 30 +ot ton +ĠDe an +Ġam endment +Ġgame play +ĠWar ren +od a +Ġhigh lights +Ġir re +ĠNAT O +Ġball s +Ġdemand ing +U RE +ĠL uke +F igure +st op +on ia +z one +iz ers +ĠW R +Ġaward ed +Ġregul atory +ĠH art +ĠS N +pl ing +Ġs our +ĠP ixel +us ive +Ġf et +ĠS ent +Ġautom atic +Ġf er +vern ment +ĠKh an +T ON +f ather +Ġextraord inary +th rop +ĠP ython +ĠG PU +Ġsex ually +Ġdesk top +it ivity +ĠAnton io +Ġo rient +Ġe ars +ob by +ous es +vertis ements +Ġmanufacture rs +ic ient +min ute +Ġconv iction +Ġg arden +p ublic +Ġsatisf ied +f old +O K +Ġin hab +ĠTh ink +Ġprogram me +Ġst omach +Ġcoord in +Ġh oly +Ġth reshold +Ġr het +Ġser ial +Ġemploy ers +ĠEvery thing +ra h +Ġb other +Ġbr ands +Val ue +ĠT ed +ĠPlan et +Ġp ink +ĠFurther more +s a +P E +re ck +ĠUS D +ot te +Ġ& & +Ġland ed +g ets +Ġprodu cers +Ġhealth care +Ġdomin ant +Ġdest ro +Ġam ended +ch ron +Ġf its +ĠSy d +ĠAuthor ity +AT CH +Ġfight s +ĠL LC +Ġ-- - +ĠCor p +Ġtox ic +spe cific +ĠC orn +ĠChe l +Ġtele phone +ĠP ant +Ġmyster ious +aun ch +od ox +med ia +Ġwitness es +ag u +Ġquestion ed +ĠBre xit +ĠRem ember +ene z +Ġend orse +iat ric +ĠId ent +Ġridic ulous +1 10 +Ġpr ayer +Ġscient ist +Ġ19 50 +ĠA qu +Ġunder ground +ĠU FC +m are +ĠL ater +w ich +Ġsubsc rib +Ġhost s +Ġer r +Ġgr ants +ant om +Ġsum mon +ear ly +ĠC lear +ĠPr im +Ġsusp ension +Ġguarant eed +app er +Ġr ice +ĠSe an +ĠSh in +Ġrefere ndum +Ġfl ed +r ust +Ġ3 60 +ter y +Ġsh ocked +B R +ĠO il +ĠAll ah +Ġpart ly +Ġign or +Ġtrans mission +Ġhom osexual +ivers al +Ġhop efully +ãĤ ¤ +Ġless on +L eg +Ġ .. +Y et +t able +app ropri +re tt +Ġbo ards +Ġincor rect +Ġb acteria +ar u +am ac +Ġsn ap +.' " +Ġpar ad +t em +he art +Ġav ailability +Ġw isdom +Ġ( + +Ġpri est +ĠÂł ĠÂł +O pen +Ġsp an +Ġparam eter +Ġconv ince +Ġ( %) +r ac +Ġf o +Ġsafe ly +Ġconver ted +ĠOlymp ic +Ġres erve +Ġhe aling +ĠM ine +M ax +Ġin herent +ĠGra ham +Ġinteg rated +D em +Ġpip eline +Ġapp lying +Ġem bed +ĠCharl ie +Ġc ave +200 8 +Ġcons ensus +Ġre wards +P al +ĠHT ML +Ġpopular ity +look ing +ĠSw ord +ĠAr ts +' ) +Ġelect ron +clus ions +Ġinteg rity +Ġexclus ively +Ġgr ace +Ġtort ure +Ġburn ed +tw o +Ġ18 0 +P rodu +Ġent reprene +raph ics +Ġg ym +ric ane +ĠT am +Ġadministr ative +Ġmanufacture r +Ġ vel +ĠN i +Ġisol ated +ĠMedic ine +Ġback up +Ġpromot ing +Ġcommand er +Ġfle e +ĠRus sell +Ġforg otten +ĠMiss ouri +Ġres idence +m ons +Ġrese mb +Ġw and +Ġmeaning ful +P T +Ġb ol +Ġhe lic +Ġwealth y +Ġr ifle +str ong +row ing +pl an +as ury +âĢ¦ . +Ġexpand ing +ĠHam ilton +Ġrece ives +S I +eat ures +ĠAn im +RE E +P ut +Ġbrief ly +ri ve +Ġstim ul +Ġ`` ( +Ġ __ +Ġch ip +Ġha z +Ġpri ze +ĠTh ings +AC E +ul in +d ict +ok u +Ġassoci ate +ock ets +y outube +St ory +ateg ory +Ġm ild +ail ing +ĠY e +O rig +ĠK a +or ig +Ġpropag anda +Ġan onymous +Ġstrugg led +Ġout rage +AT ED +ĠBe ijing +r ary +Ġle ather +Ġworld s +Ġbroad er +12 5 +id al +ĠBet ter +Ġt ear +E xt +Ġpropos als +Ġit er +ĠSqu ad +Ġvol unt +m i +D id +ĠP u +p in +Ġspeak ers +Ġb orders +Ġfig ured += ' +Ġsimultane ously +aed a +Ġcharg ing +Ġur ged +Ġcon j +25 6 +ĠG ordon +mer ce +Ġdocument ary +Sh are +it ol +ON E +ĠG arden +h att +ĠThom pson +ane ous +ap ore +Ġt anks +Ġless ons +tr ack +Ġout standing +Ġvolunte ers +Ġsp ray +Ġmanag ers +l arge +Ġcamp s +Ġart ificial +ĠR u +Ġb ags +th al +Ġcompat ible +ĠBl ade +Ġf ed +Ġarg ues +F I +Ġunf air +Ġcor n +Ġoff set +Ġdirect ions +Ġdisappoint ed +ĠCon vention +Ġview ing +M E +oc ity +Ġtown s +Ġlay ers +Ġro lled +Ġjump ed +Ġatt ribute +Ġun necess +inc oln +Ġsupp ose +ĠNet her +ch a +Ġbur ied +Ġsix th +B en +ress ing +OU R +Ġw ound +Ġcy cl +Ġmechan isms +Ġcongress ional +ĠE lement +Ġagre ements +Ġdec or +Ġclos est +ĠM it +Go ogle +} } +Ġm ixture +Ġflu id +S ign +ĠSch olar +Ġp ist +ask et +ab ling +Ġrac ing +he ro +ri el +ass y +Ġche aper +b en +Ġvert ical +amac are +ĠRead ing +g ments +Ġhelic op +Ġsacr ifice +ay a +p aren +V A +ĠL es +ĠStud io +Ġviol ations +ĠAn na +ac er +é ¾ +ĠR at +ĠBe ck +ĠD ick +ĠA CT +Ġcomp osition +Ġtext ure +ĠO wn +Ġsmart phone +ĠN A +Ġfor b +im port +Ġdef ending +il st +re r +Ġo h +ĠJere my +Ġbank ing +cept ions +Ġrespect ive +/ . +Ġdr inks +ĠW i +Ġb ands +ĠL iverpool +Ġg rip +ĠB uy +Ġopen ly +Ġreview ed +per t +Ġver ify +ĠCo le +ĠW ales +M O +Ġun pre +Ġshel ter +ĠIm perial +Ġgu i +ĠD ak +Ġsuggest ions +Ġexplicit ly +Ġsl ave +Ġblock chain +Ġcompet ing +Ġprom ising +S ON +Ġsoc cer +Ġconst itution +4 29 +Ġdist ract +ĠU ser +es ides +ĠMet hod +ĠTok yo +Ġaccompan ied +Cl ient +s ur +al og +Ġident ification +Ġinv asion +as ma +Ġindust ries +pp ers +Ġsub tle +ĠUn it +n atural +Ġsurv ived +Ġfl aw +ĺ ħ +ĠH oll +Ġdef icit +Ġtut orial +ĠCh ance +Ġarg uing +Ġcontem porary +Ġinteg ration +for ward +Ġt um +it is +Ġh iding +ĠD omin +ĠT an +ĠB uilding +ĠV in +Ġspokes person +ĠNot es +Ġemer ging +Ġprepar ation +Ġpro st +Ġsuspect s +Ġaut onom +D escription +Ġdeal t +ĠP ear +Ġstead y +Ġdecre ased +Ġso vere +ĠCl in +Ġgrad ually +ors es +ĠW AR +S erv +ãĤ ¢ +h r +Ġd irty +ĠB arn +ĠB C +Ġd il +Ġcal endar +Ġcompl iance +Ġch amber +b b +Ġpass enger +ate ful +ĠT itle +ĠSyd ney +ĠG ot +Ġdark ness +Ġdef ect +Ġpack ed +ass ion +Ġgod s +Ġh arsh +IC K +le ans +Ġalgorith m +Ġoxy gen +Ġvis its +Ġbl ade +Ġkil omet +ĠKent ucky +Ġkill er +P ack +enn y +Ġdiv ine +Ġnom ination +be ing +Ġeng ines +Ġc ats +Ġbuff er +ĠPh ill +Ġtra ff +AG E +Ġtong ue +Ġrad iation +ere r +m em +ĠExpl icit +é¾ į +Ġcou ples +Ġphys ics +ĠMc K +Ġpolit ically +aw ks +ĠBl oom +Ġwor ship +e ger +ut er +ĠF O +Ġmat hemat +Ġsent enced +Ġdis k +ĠM arg +Ġ/ * +P I +Ġoption al +Ġbab ies +Ġse eds +ĠScott ish +Ġth y +] ] +ĠHit ler +P H +ng th +Ġrec overed +ing e +Ġpow der +Ġl ips +Ġdesign er +Ġdis orders +Ġcour age +Ġch aos +" },{" +Ġcar rier +b ably +H igh +ĠR T +es ity +l en +Ġrout es +u ating +F il +N OT +w all +s burgh +Ġeng aging +ĠJava Script +ore r +li hood +Ġun ions +ĠF ederation +ĠTes la +Ġcomple tion +ĠT a +Ġprivile ge +ĠOr ange +Ġne ur +paren cy +Ġb ones +Ġtit led +Ġprosecut ors +ĠM E +Ġengine er +ĠUn iverse +ĠH ig +n ie +o ard +Ġheart s +ĠG re +uss ion +Ġmin istry +Ġpen et +ĠN ut +ĠO w +ĠX P +in stein +Ġbul k +S ystem +ic ism +ĠMarket able +Ġpre val +Ġpost er +Ġatt ending +ur able +Ġlicens ed +ĠG h +et ry +ĠTrad able +Ġbl ast +à ¤ +ĠTit an +ell ed +d ie +H ave +ĠFl ame +Ġprof ound +Ġparticip ating +Ġan ime +ĠE ss +Ġspec ify +Ġregard ed +ĠSpe ll +Ġs ons +own ed +Ġm erc +Ġexper imental +land o +h s +ĠDun geon +in os +Ġcomp ly +ĠSystem s +ar th +Ġse ized +l ocal +ĠGirl s +ud o +on ed +ĠF le +Ġconstruct ed +Ġhost ed +Ġsc ared +act ic +ĠIs lands +ĠM ORE +Ġbl ess +Ġblock ing +Ġch ips +Ġev ac +P s +Ġcorpor ation +Ġo x +Ġlight ing +Ġneighb ors +ĠU b +ar o +Ġbe ef +ĠU ber +F acebook +ar med +it ate +ĠR ating +ĠQu ick +Ġoccup ied +Ġaim s +ĠAdd itionally +ĠInt erest +Ġdram atically +Ġhe al +Ġpain ting +Ġengine ers +M M +ĠM ust +Ġquant ity +P aul +Ġearn ings +ĠPost s +st ra +ãĥ¼ ãĥ +Ġst ance +Ġdro pping +sc ript +Ġd ressed +M ake +Ġjust ify +ĠL td +Ġprompt ed +Ġscr ut +Ġspeed s +ĠGi ants +om er +ĠEd itor +Ġdescrib ing +ĠL ie +ment ed +Ġnow here +oc aly +Ġinst ruction +fort able +Ġent ities +Ġc m +ĠN atural +Ġinqu iry +Ġpress ed +iz ont +for ced +Ġra ises +ĠNet flix +ĠS ide +Ġout er +Ġamong st +im s +ows ki +Ġclim b +ne ver +Ġcomb ine +d ing +Ġcomp r +Ġsignific ance +Ġremem bered +ĠNev ada +ĠT el +ĠSc ar +ĠWar riors +ĠJ ane +Ġcou p +b as +Ġtermin al +, - +O H +Ġt ension +Ġw ings +ĠMy ster +�� �� +ĠUn like +val id +viron ments +ĠAl i +Ġn aked +book s +ĠM un +ĠG ulf +Ġd ensity +Ġdim in +Ġdesper ate +Ġpres idency +Ġ198 6 +h y +IN D +Ġun lock +im ens +Ġhand led +ĠE b +Ġdisapp eared +Ġgen re +Ġ198 8 +Ġdetermin ation +St ream +ik o +ap ters +Ġacknow ledge +J an +Ġcapital ism +P at +Ġ20 20 +Ġpain ful +Ġcur ve +Ġbom bs +st orm +ĠMet al +en cer +ĠF ig +ĠA aron +anc hes +Ġins piration +Ġexha ust +t ains +ash i +Ġdesc ript +Ġr itual +ĠChel sea +Ġpromot ion +ĠH ung +ĠW ard +iv a +ĠE T +Ġto ss +all ow +ĠFranc is +D ep +Ġhapp iness +ĠGl ass +Ġbet a +Ġstreng then +N E +o a +Ġbutt ons +ĠMur ray +Ġkick ed +Qu est +ĠT alk +ĠS everal +ĠZ ero +Ġdr one +ul k +Ġc am +ĠM obile +Ġprevent ing +Ġret ro +ĠA x +Ġcru el +Ġflo at +. ), +Ġfil ing +ĠGr ant +ĠB or +Ġr ib +Ġchampions hip +ĠM erc +Ġsty les +Ġc ake +Ġbuild s +ĠS elf +io x +Ġep ic +oy d +B el +ĠSt ew +. ( +ah u +ĠBe yond +Ġout s +Ġsol o +ĠT ree +Ġpres erve +Ġt ub +AR E +ro c +ĠIm pro +ĠW right +Ġbu nd +Ġtr aged +Ġoccas ional +b ian +Sec ond +r ons +Ġinter actions +form ed +s ing +Ġown s +Ġh ockey +Gener al +Ġlog ical +Ġexp end +Ġesc al +ĠGr iff +ĠC rown +ĠRes erve +Ġsto pping +Ġexc use +sec ond +Ġoper ated +Ġre aches +ĠMal ays +Ġpoll ution +ĠBrook lyn +Ġde lete +Ġhas h +Bl ock +ah a +âĢ ³ +Ġsh orter +p iece +> >> +ĠM ormon +t or +Ġpartic les +ĠB art +ry ption +Ġad min +Ġsqu ee +VID IA +Ġcreat or +iam eter +ic ular +N BC +Ġgrab bed +Ġn odd +Ġr ated +Ġrot ation +Ġgr asp +Ġexcess ive +ĠE C +ĠWh it +Ġinvent ory +ault s +ĠF B +Ġe cosystem +Ġbill ions +Ġvent ure +n amed +Ġdef ender +out e +Inst ead +ir able +W ar +Ġassum ption +Ġb ite +Ġearth qu +t ail +sp ace +Ġgif ts +boy s +Ġinev itable +Ġstruct ural +Ġbenef icial +Ġcompe lling +h ole +erv ation +Ġco at +o j +inc arn +ĠY ears +Ġdetermin ing +Ġrhet oric +Ġbound aries +Ġwh ites +A nt +add y +) - +ra ham +eter min +Ġhar vest +ĠCon c +Ġlapt op +ĠM atch +Ġenjoy ing +cc a +oll ar +Ġtri ps +Ġadd iction +ĠS ak +Ġpow ered +Ġc ous +ĠRuss ians +ie re +Ġret rie +qu ality +Ġdiff er +Ġking dom +ĠL aur +ĠCap itol +Ġcon clusions +ĠAl tern +ĠN av +Ġtrans parent +B ER +G roup +ĠCom plete +Ġinf er +Ġint rig +Ġins ane +R O +oph ob +is en +qu al +Mich ael +Ġm useum +ĠP ope +Ġres et +r ative +f ive +Ġagg reg +itte es +osit ory +Ġcar b +ĠRec ord +Ġdec ides +ĠF ix +Ġexcept ions +ĠCommission er +un s +ĠEnvironment al +Ġlegend ary +ist ence +Ġtun nel +k m +Ġins ult +Ġt roll +Ġsh ake +Ġdet ention +qu es +ĠCh rome +ĠF iles +Ġsub t +Ġprospect s +Ġpro l +re nder +pro of +Ġperform ances +St r +Ġh ref +ern ame +Ġachieve ment +Ġf ut +F ull +ĠLe ban +go ogle +ãĥ Ī +amp a +May be +Ġproject ed +ĠE mb +Ġcol leg +Ġa wards +Ġâ Ķ +G old +ĠBl ake +ĠR aj +if ting +Ġp ending +Ġinst inct +Ġdevelop ments +Con nect +ĠM and +ĠW ITH +ĠPhilipp ines +prof ile +Ġalt ogether +ĠB und +ĠT D +oo oo +amp ed +ip h +Ġste am +Ġold est +Ġdet ection +ul pt +Ġ ç +ĠWay ne +200 6 +f a +Ġcir cles +ĠF u +Ġdon ors +appropri ate +ĠDak ota +j amin +Ġmotiv ated +Ġpurch ases +ĠLouis iana +ĠS pl +Ġgl obe +Ġ10 5 +z ip +c all +Ġdepart ments +Ġsustain able +10 5 +ĠO P +if iers +Ġprevent ed +Ġinc omp +ĠComm ander +Ġdom inated +Ġ » +Ġinvest ed +Ġcomplex ity +Ġin cl +Ġens uring +Ġreal m +yn c +ĠInd ependent +r ained +ĠJ en +ĠFl ight +Ġat he +Ġspec ulation +ĠT E +oc ate +t ic +Ġpl aint +her ry +Ġto y +Ġ1 11 +Ġpl ates +st atus +ĠIs a +Ġdev oted +C op +ĠE S +25 5 +ur rency +M ain +Ġsl aves +Ġpe pper +Ġqu otes +Ġce iling +ĠF ish +Ġtrans formation +Ġfra ction +Ġadvant ages +Ġto ile +Ġstun ning +Ġmo ist +bre aking +s i +ĠL ocation +ĠMed ium +Ġtext s +Ġu gly +Ġb io +. âĢĶ +ĠB ased +Ġtr ains +ĠW ing +ĠAn cient +ĠRec ords +ĠH ope +Spe cial +ades h +ob i +[ / +Ġtempor arily +V er +h u +os er +Ġover night +Ġm amm +ĠTre asury +ĠV enezuel +ĠMeg a +Ġt ar +Ġexpect s +bl ack +or ph +\\ \\ +Ġaccept ance +Ġrad ar +s is +Ġjun ior +Ġfram es +Ġobserv ation +ac ies +P ower +ĠAdv anced +M ag +olog ically +ĠMe chan +Ġsent ences +Ġanaly sts +augh ters +force ment +Ġv ague +Ġcl ause +Ġdirect ors +Ġeval uate +Ġcabin et +M att +ĠClass ic +A ng +Ġcl er +ĠB uck +Ġresear cher +Ġ16 0 +Ġpoor ly +Ġexperien cing +ĠP ed +ĠMan hattan +Ġfre ed +Ġthem es +ad vant +Ġn in +Ġpra ise +10 4 +ĠLib ya +b est +Ġtrust ed +Ġce ase +Ġd ign +D irect +Ġbomb ing +Ġm igration +ĠSci ences +Ġmunicip al +ĠA verage +Ġgl ory +Ġreve aling +Ġare na +Ġuncertain ty +Ġbattle field +ia o +G od +Ġc inem +ra pe +el le +ap ons +Ġlist ing +Ġwa ited +Ġsp otted +ke ley +ĠAud io +e or +ard ing +idd ing +ig ma +ĠN eg +Ġl one +Ġ ---- +ex e +d eg +Ġtrans f +Ġwas h +Ġsl avery +Ġexpl oring +ĠW W +ats on +Ġen cl +l ies +ĠC reek +Ġwood en +Man ager +ĠBr and +um my +ĠAr thur +Ġbureau cr +Ġbl end +ar ians +F urther +Ġsupposed ly +Ġwind s +Ġ19 79 +Ġgrav ity +Ġanalys es +ĠTra vel +ĠV eter +Ġd umb +Ġaltern ate +g al +Ġconsum ed +Ġeffect iveness +.' ' +Ġpath s +ond a +L A +ĠStr ong +Ġen ables +Ġesc aped +Ġ" " +Ġ1 12 +Ġ198 3 +Ġsm iled +Ġtend ency +F ire +Ġp ars +ĠR oc +Ġl ake +Ġf itness +ĠA th +ĠH orn +Ġh ier +Ġimp ose +m other +Ġp ension +ic ut +bor ne +ic iary +. _ +ĠS U +Ġpol ar +is y +eng u +itial ized +AT A +w rite +Ġexerc ises +ĠD iamond +ot ypes +Ġharm ful +on z +Ġprint ing +st ory +Ġexpert ise +ĠG er +Ġtraged y +ĠF ly +Ġd ivid +amp ire +st ock +M em +Ġre ign +Ġun ve +Ġam end +ĠProp het +Ġmut ual +ĠF ac +Ġrepl acing +H ar +ĠCirc uit +Ġthro at +ĠSh ot +Ġbatter ies +Ġto ll +Ġaddress ing +ĠMedic aid +Ġp upp +ĠN ar +ol k +Ġequ ity +M R +ĠHis pan +ĠL arge +m id +D ev +Ġexp ed +Ġdem o +ĠMarsh all +erg us +Ġf iber +Ġdiv orce +ĠCre ate +Ġsl ower +ĠPark er +ĠStud ent +ĠTr aining +Ret urn +ĠT ru +Ġc ub +ĠRe ached +Ġpan ic +Ġqu arters +Ġre ct +Ġtreat ing +Ġr ats +ĠChristian ity +ol er +Ġsac red +Ġdecl are +ul ative +et ing +Ġdeliver ing +est one +Ġt el +ĠL arry +Ġmet a +ac cept +art z +ĠRog er +hand ed +Ġhead er +Ġtra pped +ĠCent ury +Ġkn ocked +ĠOx ford +Ġsurviv ors +b ot +Ġdemon stration +Ġd irt +Ġass ists +OM E +ĠD raft +ortun ate +fol io +pe red +ust ers +g t +ĠL ock +Ġjud icial +ver ted +Ġsec ured +out ing +ĠBook s +Ġhost ing +Ġlif ted +l ength +Ġj er +Ġwhe els +ĠR ange +umbn ails +Ġdiagn osis +te ch +ĠStew art +ĠP ract +Ġnation wide +Ġde ar +Ġoblig ations +Ġgrow s +Ġmand atory +Ġsusp icious +! ' +A pr +G reat +Ġmort gage +Ġprosecut or +Ġeditor ial +ĠK r +Ġprocess ed +ung le +Ġflex ibility +Ear lier +ĠC art +ĠS ug +Ġfoc uses +Ġstart up +Ġbre ach +ĠT ob +cy cle +ãĢ Į +ro se +Ġb izarre +ãĢ į +Ġveget ables +$ $ +Ġret reat +osh i +ĠSh op +ĠG round +ĠSt op +ĠHawai i +ĠA y +Per haps +ĠBe aut +uff er +enn a +Ġproduct ivity +F ixed +cont rol +Ġabs ent +ĠCamp aign +G reen +Ġident ifying +Ġreg ret +Ġpromot ed +ĠSe ven +Ġer u +ne ath +aug hed +ĠP in +ĠL iving +C ost +om atic +me ga +ĠN ig +oc y +Ġin box +Ġem pire +Ġhor izont +Ġbr anches +Ġmet aph +Act ive +ed i +ĠFil m +ĠS omething +Ġmod s +inc ial +ĠOrig inal +G en +Ġspir its +Ġear ning +H ist +Ġr iders +Ġsacr ific +M T +ĠV A +ĠS alt +Ġoccup ation +ĠM i +Ġdis g +lic t +Ġn it +Ġn odes +e em +ĠP ier +Ġhat red +ps y +ãĥ ī +Ġthe ater +Ġsophistic ated +Ġdef ended +Ġbes ides +Ġthorough ly +ĠMedic are +Ġbl amed +arent ly +Ġcry ing +F OR +pri v +Ġsing ing +ĠI l +Ġc ute +o ided +olit ical +ĠNe uro +å ¤ +Ġdon ation +ĠEag les +ĠG ive +T om +Ġsubstant ially +ĠLic ense +ĠJ a +Ġg rey +ĠAn imal +ĠE R +ĠU nd +Ġke en +Ġconclud e +ĠMississ ippi +Eng ine +ĠStud ios +P ress +o vers +ll ers +Ġ3 50 +ĠR angers +Ġr ou +ert o +E p +iss a +iv an +Ġse al +ĠReg ist +dis play +Ġwe aken +u um +ĠComm ons +ĠS ay +Ġcult ures +Ġl aughed +Ġsl ip +Ġtreat ments +iz able +m art +ĠR ice +Ġbe ast +Ġob esity +ĠLa ure +ig a +Wh ich +hold er +Ġelder ly +Ġp ays +Ġcompl ained +Ġc rop +Ġpro c +Ġexplos ive +ĠF an +ĠAr senal +A uthor +ef ul +Ġme als +Ġ( - +id ays +Ġimag ination +Ġann ually +Ġm s +as ures +H ead +ik h +m atic +Ġboy friend +ĠCom puter +Ġb ump +Ġsur ge +ĠCra ig +ĠKir k +D el +medi ate +Ġscen arios +ĠM ut +ĠSt ream +Ġcompet itors +Ù Ħ +ĠStan ford +ĠRes ources +az ed +b age +Ġorgan is +ĠRe lease +Ġsepar ately +Ġha bits +Ġmeasure ments +ĠCl ose +Ġaccomp any +Ġg ly +Ġt ang +ĠR ou +Ġplug in +Ġcon vey +ĠChall enge +oot s +j an +Ġcur s +ĠRel ations +ke eper +Ġapproach ing +p ing +Spe aking +Ġarrang ement +ĠV I +are ttes +Ġaffect ing +Ġperm its +b ecause +Ġu seless +ĠH us +!! !! +Ġdestro ying +Un fortunately +Ġfasc inating +S em +Ġelect oral +Ġtrans parency +ĠCh aos +Ġvolunte er +Ġstatist ical +Ġactiv ated +ro x +We b +H E +ĠHamp shire +is ive +M ap +Ġtr ash +ĠLaw rence +st ick +C r +Ġr ings +EX T +Ġoper ational +op es +D oes +ĠEv ans +Ġwitness ed +P ort +Ġlaunch ing +ec onom +w ear +ĠPart icip +um m +cul es +ĠR AM +ĠT un +Ġass ured +Ġb inary +Ġbet ray +Ġexpl oration +ĠF el +Ġad mission +it ated +S y +Ġav oided +ĠSim ulator +Ġcelebr ated +ĠElect ric +¥ ŀ +Ġcl uster +itzer land +he alth +L ine +ĠN ash +at on +Ġsp are +Ġenter prise +ĠD IS +clud es +Ġfl ights +Ġreg ards +ĠÃ Ĺ +h alf +Ġtr ucks +Ġcontact s +Ġunc ons +ĠCl imate +Ġimm ense +N EW +oc c +ect ive +Ġemb od +Ġpat rol +Ġbes ide +Ġv iable +Ġcre ep +Ġtrig gered +ver ning +Ġcompar able +q l +Ġg aining +ass es +Ġ( ); +ĠG rey +ĠM LS +s ized +Ġpros per +" ? +Ġpoll ing +Ġsh ar +ĠR C +Ġfire arm +or ient +Ġf ence +Ġvari ations +g iving +ĠP i +osp el +Ġpled ge +Ġc ure +Ġsp y +Ġviol ated +Ġr ushed +Ġstro ke +ĠBl og +sel s +ĠE c +,' ' +Ġp ale +ĠColl ins +ter ror +ĠCanad ians +Ġt une +Ġlabor atory +Ġn ons +t arian +Ġdis ability +ĠG am +Ġsing er +al g +ĠSen ior +Ġtrad ed +ĠWar rior +Ġinf ring +ĠFrank lin +Ġstr ain +ĠSwed ish +Ġsevent h +ĠB enn +ĠT ell +Ġsynd rome +Ġwond ered +id en +++ ++ +ig o +Ġpur ple +Ġjournal ism +Ġreb el +Ġf u +bl og +Ġinv ite +ren cies +ĠCont act +Is rael +ĠCont ent +Ġche er +Ġbed room +ĠEngine ering +ĠQue ens +Ġd well +ĠPlay Station +ĠD im +ĠCol on +l r +Ġoper ates +Ġmotiv ation +US A +ast ered +C ore +ĠTr uth +ol o +OS E +ĠMem ory +Ġpred ec +Ġan arch +Ġ19 20 +ĠY am +à ¨ +b id +Ġgr ateful +Ġexc itement +Ġtre asure +Ġlong est +ct ive +Ġdes erves +Ġreserv es +Ġcop s +ĠOtt awa +ĠEgypt ian +ank ed +Ġart if +Ġhypot hesis +: / +Ġpurch asing +Ġlove ly +H P +Ġdiv ide +Ġstrict ly +Ġquestion ing +Ġtaxp ayers +ĠJ oy +Ġroll s +ĠHe avy +Ġp orts +Ġmag netic +Ġinf lamm +Ġbr ush +t ics +â ĪĴ +Ġbott les +pp y +Ġp add +ãĤ ¯ +m illion +Ġdevast ating +Ġcomp iled +Ġmed ication +Ġtw elve +ĠPer ry +Sp ace +im b +y our +Ġle aked +ĠT ar +Ġun ity +Ġinfect ed +Ġtravel ed +ID E +ĠMc Donald +t xt +ĠPr inc +Ġinter ven +ĠTai wan +ĠP ow +Ġbe aring +ĠTh read +Ġz ones +iz ards +un ks +Ch apter +ll or +Ġ · +Ġw ounds +Ġdisc retion +Ġsucceed ed +ik ing +Ġicon ic +C all +Ġscreen ing +ĠM is +ict s +Ġmin isters +Ġsepar ation +Pl ayer +Ġb ip +Ġbel oved +Ġcount ing +ĠE ye +ar ound +ing ing +Ġtable t +Ġoff ence +in ance +h ave +ĠInf o +ĠNin ja +Ġprotect ive +ĠC ass +M ac +ĠQual ity +N orth +Ġ ic +ĠCub a +ĠChron icle +ĠPro perty +Ġfast est +ot os +ĠG erm +OW N +Ġbo om +ĠStan ley +ergus on +Ġcle ver +Ġent ers +m ode +ter ior +ĠS ens +Ġlin ear +AR K +Ġcomp aring +Ġpure ly +Ġsaf er +ĠPot ter +Ġc ups +R T +Ġgl uc +Ġatt ributed +Ġdu pl +ĠP ap +Ġprec ious +Ġp a +iction ary +ĠT ig +ĠTo o +ol utions +st an +Ġrob ots +Ġlob b +Ġstat ute +Ġprevent ion +w estern +16 0 +ĠAct ive +ĠMar ia +h al +N one +ell ar +ĠK B +ĠPart ners +ĠSing le +ĠFollow ing +ang o +ac ious +Ġth ou +Ġk g +Ġinflu ential +ĠFriend s +S ur +ain ted +Ġfor ums +Ġst arter +Ġcitizens hip +ĠE lection +on ge +ot ation +os ph +;; ;; +ut ical +p ur +ere n +Ġaccus ations +bit ious +ab bit +ĠOr d +Post ed +ir k +Ġsens itivity +ic he +ĠAm y +ĠF ab +Ġsum mit +Ġped est +Ġrub ber +Ġagric ultural +Ġcan cel +A E +Ġin aug +Ġcont am +Ġfirm ly +i w +st age +ĠK an +Ġt ier +Ġinv ention +Ġtransl ated +ĠR ules +B ox +Tw itter +ID S +Ġp izza +Ġdeb ug +ĠD rop +v s +Ġh orses +b ig +Ġb oring +Ġh ood +ĠMcC ain +at ched +ĠBro s +Ġsk ip +Ġess ay +st at +ĠLeg ends +Ġam munition +au c +Ġshoot er +Ġun h +Ġsuppl ied +Ġgener ic +ĠS K +ib an +yr ics +Ġ25 5 +Ġclim bing +Form er +Ġfl ip +Ġjump ing +Ġfrust ration +ĠTer ry +Ġneighborhood s +Ġmed ian +be an +Ġbr ains +Follow ing +Ġsh aped +Ġdraw s +Ġal tered +J ack +Ġrecip es +Ġsk illed +we alth +ach i +e lection +Ġbehavi ors +de als +ĠU ntil +F e +Ġdecl aration +mar ks +ĠBet ween +cel ona +Ġres on +Ġbub ble +Am ong +Ġim perial +G S +Ġfemin ist +200 5 +ĠK yle +Ġaccount ing +ĠTe le +ĠT yr +Ġconnect ing +Ġre hab +ĠP red +s im +Ġmeant ime +Ġphys ician +M W +ĠCamp bell +ĠBr andon +Ġcontribut ing +ĠR ule +ĠWe ight +ĠN ap +Ġinter active +Ġv ag +Ġhel met +ĠCom b +f our +Ġsh ipped +Ġcomple ting +ĠP D +PD ATE +Ġspread ing +Ġsc ary +erv ing +ĠG as +Ġfr ank +s chool +Ġrom antic +Ġstab il +R ob +Ġaccur ately +Ġac ute +ĠH ann +Ġsymbol s +Ġcivil ization +ĠA W +Ġlight ning +Ġcons iders +Ġven ue +Ġ × +Ġo ven +ĠS F +h is +Ġn u +ĠLear n +Ġpe oples +Ġst d +Ġsle e +Ġs lic +ĠStat istics +Ġcor ners +ĠB aker +Ġ: ) +ment ation +ol ver +Ġlaugh ing +ĠT odd +ond e +ĠH ills +Ġn uts +ĠW oman +pl ane +Ġl iver +ĠIn side +S orry +Ġagre es +Ġfund ament +ĠF isher +Ġa uction +Ġthread s +gl as +ĠBas ic +ĠN at +Ġlack ing +Ġceleb ration +j u +Ġs illy +E uro +Ġt att +ight y +cont rolled +T est +ĠSing h +Ġr age +Ġrh yth +o ffic +ĠPh antom +Ġhead lines +Ġrespond ing +ĠMor ning +Ġvit amin +Ġboot s +ĠS ite +al in +p i +Ġvir al +ĠU C +D ER +ĠSe x +Ġst ocks +c urrent +Ġch urches +ĠR are +ĠMur phy +Ġden ial +ĠG aming +Ġtou g +Ġn ick +Ġm akers +ĠRon ald +Ġgener ous +ĠD oc +ĠMor ris +Ġtransform ed +ĠN ormal +Ġ10 4 +ĠKick starter +ĠUp on +On line +ĠI RS +Ġw rap +Ġl oving +Ġarri ves +ĠD ue +Ġhe ter +ĠM ade +Ġrent al +Ġbelong s +Ġatt orneys +Ġcro ps +Ġmat ched +ul um +ol ine +10 9 +Ġdis par +Ġbuy ers +ĠCam bridge +Ġeth ics +rou ps +Ġjust ified +Ġmarg inal +Ġrespect ed +win ning +Ġnodd ed +ĠSer ge +ĠForm er +C raft +######## ######## +ĠWar ner +Ġd ash +et e +Ġent ert +ĠE scape +out heast +Ġkn ees +ĠB omb +Ġr ug +P ass +Ġatt itudes +go vernment +ĠPri or +Ġqual ities +Ġnot ification +ĠPh one +l ie +Ġanticip ated +ĠCom bat +ĠBar ry +Ġ198 2 +Us ers +on er +Ġcomput ing +ĠConnect icut +Ġless er +Ġpe ers +ĠC u +Ġtechn ically +Ġsub mission +ĠUn iversal +Ġman ually +our ge +Ġrespond ents +ĠB TC +ĠH ost +Ġf are +ĠB ird +Ġrece ipt +al so +Ġj ack +Ġagric ulture +Ġsk ull +Ġ! = +Ġpass ive +ĠC I +Ġsoc ieties +Ġremind ed +Ġinter ference +B uy +Ġâ ľ +g on +Ġscrut iny +ĠW itch +Ġconduct ing +Ġ ãĥ +Ġexch anges +ĠMit chell +Ġinhab it +Ġtw ist +B D +Ġwhere ver +group on +Ġj okes +ĠBen jamin +ĠR andom +fr ame +ĠL ions +Ġhighlight ed +ĠArk ansas +E nt +Ġp ile +Ġpre lim +g s +mind ed +Ġfel ony +ĠG A +ĠL uck +Ġpract ically +ĠB os +Ġact ress +D am +ĠB ou +Ġvis a +Ġembed ded +Ġhy brid +Ġear liest +Ġsoon er +s ocial +ĠH A +Ġste ep +Ġdis advant +Ġexplo it +ĠE gg +ĠUlt ra +Ġnecess ity +L ocal +ie ge +Ġd ated +Ġmass es +Ġsubsc ription +pl ess +Ġan onym +Ġpresum ably +Bl ue +The ir +asket ball +ĠPhil ip +Ġcom ed +load ed +r ane +Ġref lection +Ch ina +Ġext ends +Ġform ing +Ġund ers +200 1 +Ġgr at +Ġconcent rations +Ġins ulin +Ġsec ular +Ġwh ilst +Ġwin ners +Ad vertisements +Ġdeliber ately +ĠWork ing +Ġs ink +et ics +d ale +Ġmand ate +Ġg ram +Ġvac ation +Ġwarn ings +ri pp +ĠTH AT +Ġcomment ary +Ġint u +Ġa est +Ġreason ing +Ġbreak down +ĠZ ombie +Ġ-- > +ĠPolit ical +c ott +Ġthr ust +Ġtechn ological +Ġdec iding +Ġtraff icking +L ong +W elcome +pr ising +ĠCommun ications +Ġend ors +Ġsw ift +Ġmetab ol +co ins +res a +ĠHT TP +Ġen roll +ĠH appy +us r +int age +Ġ[ " +u ably +ĠM aterial +Ġrepe al +Se pt +k h +ĠMod i +Ġunder neath +ĠI L +sh ore +Ġdiagn osed +ace utical +Ġsh ower +au x +ĠSw itch +ĠStre ngth +Ġj ihad +n ational +Ġtra uma +uss y +on i +Ġcons olid +Ġcal ories +ĠF lynn +ag ged +16 8 +ĠP ink +Ġfulf ill +Ġch ains +Ġnot ably +ĠA V +L ife +ĠCh uck +m us +ĠUr ban +ĠH end +Ġdep osit +ĠS ad +Ġaff air +OR K +ie val +ĠF DA +Ġt rop +ĠOver all +Ġvirt ue +Ġsatisf action +au nd +Ġl un +ĠSw itzerland +ĠOper ation +pro cess +Ġsh ook +Ġcount ies +le ased +ĠCharl otte +1 12 +Ġtrans cript +Ġre dd +p ush +ĠHe y +ĠAn alysis +[ " +Ġaltern atives +ard less +Ġele ph +Ġpre jud +ĠLe af +H aving +ĠH ub +Ġexpress ions +ĠVol ume +Ġshock ing +ĠRed s +Ġread ily +Ġplan ets +ad ata +Ġcollaps ed +ĠMad rid +Ġir rit +i pper +ĠEn c +ĠW ire +Ġbu zz +ĠG P +ash a +Ġaccident ally +ur u +Ġfrust rated +ĠS A +Ġhung ry +ĠH uff +Ġlab els +ant o +ĠE P +Ġbar riers +) | +ĠBer keley +ĠJ ets +Ġp airs +ĠL an +J ames +ĠB ear +Ġhum or +ĠLiber ty +Ġmagn itude +Ġag ing +ĠM ason +Ġfriends hip +umb ling +Ġemer ge +Ġnewsp apers +Ġam bitious +ĠRich ards +atern al +Ġ198 1 +Ġcook ies +Ġsc ulpt +Ġpur suit +L ocation +Ġscript s +p c +Ġarrang ements +Ġd iameter +Ġl oses +am ation +Ġl iqu +ĠJ ake +aret te +Ġunderstand s +ĠZ en +v m +Ġappro ve +Ġw ip +Ġult ra +Ġint end +ĠD I +asc ular +Ġst ays +ĠK or +ĠK l +Ġinvest ing +L a +Ġbelie ving +b ad +m outh +Ġtaxp ayer +ãĥ ĥ +ĠQue bec +Ġl ap +ĠSw iss +d rop +Ġdr ain +ir i +et c +ft en +ĠN ex +Ġst raw +Ġscream ing +Ġcount ed +Ġdam aging +Ġamb assador +cent ury +Ġpro x +Ġarrest s +u v +il ateral +ĠCh arg +Ġpresc ribed +Ġindepend ently +Ġf ierce +ĠB aby +Ġb rave +Ġsu its += > +Ġbas eline +ĠR ate +Ġis lands +Ġ( ( +g reen +ix els +Ġname ly +ĠVill age +th an +am y +V ersion +g mail +ential s +ĠS ud +ĠMel bourne +Ġarri ving +Ġquant um +e ff +rop olitan +T ri +Ġfun eral +ĠI R +ÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤ ÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤ +ĠC ob +it ably +Ġt urb +Ġcomb o +Re view +Ġdeploy ment +u ity +ĠB ott +Ġinv isible +Ġrender ing +Ġunl ocked +Ġa qu +ĠVlad imir +Ġp ad +ĠBr ain +ĠLeg acy +dr agon +ĠKurd ish +Ġsound ed +Ġdet ained +ĠD M +g ary +Ġd aughters +Ġdistur bing +uk a +ĠPar ad +Ġt ast +Ġunf ortunate +Ġu l +em in +Ġattend ance +tr l +Ġpar ks +ĠMem orial +ĠAl ice +oth y +gu ard +ĠD ise +ĠSh an +ĠFor um +R ich +Ġshif ted +ue z +Ġl ighter +ĠMag n +Ġc od +S ch +ham mad +P ub +3 50 +ĠP okemon +Ġprot otype +Ġun re +B ase +ĠStud ents +ĠRep ly +ĠCommun ist +Ġg au +ĠTy ler +I Z +Ġparticip ated +Ġsup rem +ĠDet ails +Ġvessel s +ro d +Ġt ribe +ke ep +Ġassum ptions +Ġp ound +Ġcr ude +ĠAv ailable +Ġswim ming +Ġin clusion +Ġadv ances +c ulation +Ġconserv ation +Ġover d +ĠBuff alo +Art icle +ed ge +Ġaw a +ĠMad ison +Ġsid ew +Ġcat ast +ĠK rist +uc le +ĠHigh way +ĠTer ror +Ġactiv ation +Ġuncons cious +ĠSat an +ĠSus an +ill ery +Ġarr anged +i op +Ġrum ors +ur ring +th ink +ĠKe ith +ĠK ind +Ġavoid ing +by n +n ut +ĠSpe aker +r us +n ames +Ġgu ilt +ĠOlymp ics +Ġsa il +ĠM es +lev ant +ĠColumb us +a ft +C ity +S outh +ĠHar vey +ĠP un +S everal +Ġment ally +Ġimp ress +m ount +ĠUb untu +âĢĶâĢĶâĢĶâĢĶ âĢĶâĢĶâĢĶâĢĶ +ĠSuper man +ĠMP s +Ġintent ions +ĠR acing +Ġlike lihood +Ġ2 40 +T otal +Ġto ys +ĠW atson +Ġur ge +L ear +ĠP aper +Ġoccur ring +ĠB eng +ĠC ert +Ġst ones +T im +ĠTw in +z b +ĠD ynam +Ġpolit ician +k ens +ĠEnter prise +UT ERS +Ġab ol +Ġref resh +Ġarbit rary +pe ction +Ġtrou bles +Ġ} ); +t v +Ġpil ots +Ġdist ribute +Ġaud it +Ġp ause +orig inal +Ġr ivals + £ +F ig +T L +ab il +ry ing +L in +ion ed +l on +Ġf ancy +Ġcr ashed +Ġt ract +Ġshe d +Ġcons ume +B ased +down load +in it +Ġvolt age +Int rodu +Ġcondem ned +ĠFin ance +res pect +Ġex cluded +Ġestablish ing +her ic +Ġher itage +Ġspect acular +Ġun st +ĠSnow den +ĠL ane +S an +Ġprotect ions +st ruction +inc inn +Ġmac ro +C ustom +ios ity +Ġes p +Ġfunction ing +Ġm ush +Ġp uzzle +Ġeth ical +M al +Ġgo verning +ĠF erguson +Ġrest ored +Ġst ressed +ĠCoun ter +ĠK as +cl ip +AN S +Ġse iz +U K +by ss +old own +ap i +Ġperman ently +oun ters +W est +Th rough +L ight +at oes +Ġne at +Ġc ord +ure r +Ġsevere ly +ĠA ven +Ġinter rog +Ġtri ple +G iven +N umber +Ġar ise +Ġs her +pl ant +Ġfl ower +ĠC ou +Ġat e +Ġnew er +b ul +Ġmean while +ĠL air +Ġadjust ment +ĠCop yright +Ġd ivers +i ological +Ġgam ers +o at +Ġhistor ically +Ġanal og +Ġlong time +Ġpres cription +ĠM ist +ĠHy per +ĠM aine +ĠDe ity +Ġmulti pl +ĠRe incarn +ĠH yd +ĠP ic +S il +r ants +ĠC ris +. ; +( { +epend ence +Ġrec y +ate ur +Ġqu ad +Ġgl ob +Ġcon ced +te am +Ġcapital ist +ĠL ot +Ġroy al +ĠCy ber +Ġblack s +met ic +ri v +ĠD anny +Ġsp o +ĠR O +Ġanim ated +rypt ed +ĠDep uty +Ġrend ered +F E +Ġstre ak +Ġcloud s +ĠDou g +~~~~ ~~~~ +Ġdisc our +ĠVe h +Ġpsych ology +ĠJ ourney +Ġcry stal +ĠFro st +Ġsuspic ion +Ġrel ate +or us +ĠC rypt +ĠN VIDIA +com ed +ut ing +incinn ati +Ġvulner ability +ost ic +Ġisol ation +Ġcool ing +ĠCoal ition +Ġ1 19 +F our +ĠDe al +Ġâ ī +se mble +ram ent +ĠBar celona +Ġ10 2 +Ġcoc aine +ocaly pse +F eb +ogen ic +Ġmut ation +Ġcrypt oc +ĠK el +ĠG it +a is +Ġs isters +AN K +Ġactiv ate +T er +Ġd read +yl on +Ġprop ri +A ust +ĠDef ault +Ġout door +Ġshe er +ce ive +Ġg ently +Ð ¾ +Pro gram +Ġâ ĨĴ +Ġve gan +ĠCr us +Ġrespons ibilities +ĠH R +OL D +Ġprev ents +Ġst iff +ĠW ere +Ġathlet ic +ĠSc ore +Ġ) : +Ġcolumn s +ĠL oc +av ailable +ĠF ram +ĠS essions +Ġcompan ion +Ġpack s +14 0 +ĠKn ights +Ġf art +Ġstream s +Ġsh ore +Ġapp eals +ĠPer formance +h aul +ĠSt ra +ĠN ag +10 3 +ĠTrans portation +B B +E v +z an +P ublic +Ġtw in +uls ion +M ult +Ġelect ro +Ġstat ue +ation ally +ĠN ort +Ġins pection +/ * +ig ue +Ġcomp assion +ĠT ales +ĠSte in +ĠSc reen +ĠB ug +ĠL ion +g irl +Ġwithdraw al +Ġobject ives +Ġblood y +Ġprelim inary +Ġj acket +Ġdim ensions +ĠC ool +ĠOcc up +Ġw reck +Ġdoub led +ank ing +Ġ19 75 +Ġglass es +ĠW ang +pro v +P ath +connect ed +ĠMult i +ĠNor way +agon ist +Ġfe ared +Ġtouch ing +Ġarg uably +¯¯¯¯ ¯¯¯¯ +ĠNC AA +che m +Ġsp at +ĠW WE +ĠC el +ig ger +Ġattack er +ĠJo in +ob ject +ett a +Ġelim inated +d et +Ġdest ruct +ĠLuc as +ct uary +18 0 +ĠBr ady +ĠBl ues +B ay +au kee +Ġtim eline +Ġdeleg ates +w ritten +uff icient +Ġsh apes +Cop yright +ou ble +serv ice +Ġp ione +Ġcolleg es +Ġrow s +Ġsp ite +Ġassess ed +3 60 +Ġle ase +Ġconfident ial +ck er +ĠMan ning +ĠV oice +Ġse aled +Ġcalcul ate +N O +ĠAss istant +Ġteen ager +ul ent +ather ine +Ġm ock +Ġd iamond +Ġf est +Ġsw itched +Ġres ume +ĠPu erto +Ġl anes +ir ation +ĠSimilar ly +Ġro d +ĠS el +ĠPal ace +ĠLim ited +e ous +Ġvar iant +Ġw ard +Ġ) ) +Sh ow +OO K +A lex +ĠN ep +br is +ĠWik ipedia +Ġexcept ional +Ġman ages +ĠD raw +Ag ain +Ġco pper +ut t +Ġex ports +Ġport folio +Ġelev ated +R ated +ĠOther wise +ĠT act +ĠShe l +ĠT X +" âĢĶ +Ġres ur +ĠW a +ven ant +Ġmon etary +pe ople +E mail +Ġfif ty +ĠS weet +ĠMalays ia +Ġconf using +ĠR io +ud a +uten ant +" ); +Ġpra ised +Ġvol umes +t urn +Ġm ature +Ġnon profit +Ġpassion ate +ĠPriv ate +Ġ10 3 +Ġdesc end +ç ¥ŀ +uff y +head ed +Whe ther +ri en +ze ch +be it +Ġch rom +ĠMc M +Ġd ancing +Ġe leg +ĠNot iced +11 5 +Ġadvoc acy +ENT S +amb ling +ĠMin or +ĠF inn +Ġprior ities +Ġthere of +ĠSt age +ĠRog ers +Ġsubst itute +ĠJ ar +ĠJeff erson +Ġlight ly +10 2 +ĠL isa +u its +ys ical +Ġshif ts +Ġd rones +Ġwork place +Ġres id +ens ed +ah n +Ġpref erences +ser ver +Ġdeb ates +d oc +ĠGod s +Ġhelicop ter +Ġhon our +Ġconsider ably +ed ed +ĠF emale +ĠAn ne +Ġre un +ĠF ace +ĠHall ow +ĠBud get +Ġcondem n +Ġt ender +Pro f +ocr atic +ĠTurn er +ĠAg ric +Ġ19 76 +Ġa pt +d isc +ĠF ighter +ĠA ur +Ġgar bage +in put +ĠK arl +ĠOl iver +ĠL anguage +k n +N on +ĠCl ar +Ġtrad itions +Ġad vertisement +ĠS or +Ġarch ive +Ġvill ages +7 50 +Ġimplement ing +w aukee +Ġdiet ary +Ġswitch ing +Rep ublic +Ġvel ocity +Ġc it +ĠA wards +Ġfin ancing +Ġlast ed +) ] +Ġrem inder +P erson +Ġprec ision +Ġdesign ers +ĠF ried +ĠB order +Ġtr agic +Ġw ield +Ġiniti atives +ĠT ank +w er +Ġjo ins +R o +in ery +Ġar row +Ġgener ating +found er +Ġsear ches +Ġrandom ly +A ccess +Ġb atch +Ġp osed +l at +Ġpursu ing +as a +Ġtest ified +form ing +ĠSh ar +w iki +ĠE ither +S ometimes +Ġsen ators +ĠJohn ny +ĠTal iban +ĠG PS +":" / +ãģ® å +Ġanaly zed +ĠRub io +ĠMove ment +op ard +ii i +St and +f ight +Ġign oring +i ang +ĠG N +so ever +ĠST AT +Ġref using +Ġswe at +Ġb ay +P ORT +ir med +ak y +Ġdis pro +Ġlabel ed +Ġ10 8 +H ello +Ġple asant +ab a +Ġtri umph +Ġab oard +Ġinc om +ĠC row +le tt +Ġfol k +Ġch ase +` ` +ĠBr us +Ġte ens +c ue +Ġter rain +h yd +il ight +OR Y +Su pport +ew s +ll i +rain ts +ĠC and +Ġab used +ach ment +l arg +B as +ĠC ancer +Ġ19 78 +Ġsupp orter +ac cess +ĠTer min +ĠT ampa +ĠAN Y +Ġnew est +ĠCrim inal +ed u +Ġ19 30 +Ġadm its +Ġend e +Ġfail ures +ur ate +ful ness +cy cl +ĠSub ject +Ġinf inite +th ree +W A +p it +ĠInst all +R ad +ili ation +G M +Ġcontin ent +Ġaccommod ate +ĠCl ay +Ġp up +ĠF unction +Ġham mer +ĠAlbert a +Ġrev ised +Ġminor ities +Ġmeasure ment +Con nell +Ġdis able +ĠM ix +In cre +Ġfor k +ĠR osen +Ġimpl ies +umb lr +AN G +Ġprote ins +Ġagg ression +Ġfacilit ate +S N +Ġilleg ally +u er +Ġacad em +Ġp uzz +ĠSh ift +p ay +oll o +Ġaud iences +B uild +Ġno ble +Ġsynt ax +â ĺħ +Ġbe am +ĠB ed +ĠA ld +Ġorig ins +v ideo +Ġ19 77 +ĠAss ault +Ġgar age +Te am +Ġver dict +Ġd war +ĠVirt ual +e vent +Ke ep +Ġsent iment +Ġwild life +sh irt +Ġb urg +Ġrecommend ation +rep resent +Ġgall ery +own ers +Ġsch olar +Ġconven ience +ĠSw ift +Ġconv inc +C ap +Ġwar fare +ĠVis ual +Ġconst itute +Ġab ort +ĠWe ather +ĠLook ing +ĠH em +Ġmart ial +Ġinc oming +et ition +Ġtoler ance +ĠCre ated +Ġfl ows +ĠE lder +Ġsoul s +Ġf oul +ĠP ain +ĠC AN +Ġ2 20 +b c +he nd +Ġgen ius +R eal +ĠW r +omet er +p ad +Ġlim iting +ĠS i +ĠL ore +ĠAd ventures +Ġvar ied +D isc +f in +ĠPerson al +Ch ris +Ġinv ented +Ġd ive +ĠR ise +Ġo z +ĠCom ics +Ġexp ose +ĠRe b +let ters +s ite +im ated +Ġh acking +Ġeduc ated +ĠNob ody +Ġdep ri +Ġincent ive +ãĤ · +Ġovers ight +Ġtrib es +ĠBelg ium +Ġlicens ing +our t +Produ ct +ah l +ĠG em +Ġspecial ist +Ġc ra +ann ers +ĠCor byn +Ġ19 73 +RE AD +Ġsum mar +Ġover look +ĠApp lication +Ġin appropriate +Ġdownload ed +Q ue +ĠB ears +Ġth umb +ĠChar acter +ĠReincarn ated +ĠS id +Ġdemonstr ates +s ky +ĠBloom berg +ĠAr ray +ĠRes ults +ĠFour th +ĠED T +ĠO scar +c end +Ġ10 6 +ĠN ULL +ĠH ERE +m atch +ĠBr un +Ġgluc ose +ie g +eg u +Ġcert ified +Ġrel ie +Ġhuman itarian +Ġpr ayers +K ing +Ġn an +h ou +10 8 +ul u +Ġrenew able +Ġdistingu ish +Ġd ense +ĠV ent +ĠPack age +ĠB oss +Ġedit ors +Ġm igr +T ra +ĠPet ers +ĠAr ctic +200 4 +ĠC ape +Ġloc ally +Ġlast ing +Ġhand y +. ). +P an +ĠR ES +Ind ex +Ġt ensions +Ġformer ly +Ġide ological +Ġsens ors +Ġdeal ers +Ġdef ines +S k +Ġproceed s +Ġpro xy +az ines +ĠB ash +ĠP ad +ĠC raft +eal ous +Ġshe ets +omet ry +J une +cl ock +T T +ĠThe atre +ĠB uzz +Ġch apters +Ġmill enn +Ġd ough +ĠCongress ional +Ġimag ined +av ior +Ġclin ic +Ġ19 45 +Ġhold er +ro ot +oles ter +Ġrest art +B N +ĠHam as +ĠJ ob +Ġor b +Ġr am +Ġdiscl ose +Ġtransl ate +Ġimm igrant +Ġannoy ing +Ġtreat y +an ium +ĠTe a +ĠLeg ion +Ġcrowd s +ĠB ec +ĠA er +oh yd +B ro +Look ing +Ġl bs +Ġagg ress +Ġse am +Ġinter cept +ĠM I +mer cial +act iv +ĠC it +Ġdim ension +Ġconsist ency +Ġr ushing +ĠDou glas +Ġtr im +Inst all +ick er +Ġsh y +10 6 +Ġment ions +pe lled +ĠT ak +c ost +Ġclass room +Ġfort une +dri ven +Ġun le +ĠWhe el +Ġinvest or +ĠM asters +k it +Ġassoci ations +ĠEv olution +op ing +us cript +Ġprov incial +ĠWal ter +av i +S O +Ġun limited +Eng lish +ĠC ards +ĠEb ola +ne red +Ġreven ge +Ġout right +um per +Ġf itting +ĠSol id +Ġform ally +Ġproblem atic +Ġhaz ard +Ġenc ryption +Ġstraight forward +ĠA K +Ġp se +ĠOr b +ĠCh amber +ĠM ak +Cont ents +Ġloyal ty +Ġl yrics +ĠSy m +Ġwel comed +Ġcook ed +Ġmon op +Ġn urse +Ġmis leading +Ġe ternal +Ġshif ting +Ġ+ = +V is +Ġinst itutional +ill ary +Ġp ant +VER T +ĠA CC +ĠEn h +Ġinc on +ĠRE UTERS +Ġdon ated +âĢ¦âĢ¦ âĢ¦âĢ¦ +In tern +Ġexhib it +Ġt ire +ĠR ic +ĠCh ampion +ĠMu hammad +N ING +ĠSoc cer +Ġmob ility +Ġvary ing +ĠM ovie +Ġl ord +o ak +F ield +Ġve ctor +us ions +Ġsc rap +Ġen abling +m ake +T or +. * +| | +ĠWe bsite +ĠN PC +Ġsocial ist +ĠBill y +ĠAdd itional +Ġc argo +Ġfar ms +ĠSo on +ĠPri ze +Ġmid night +Ġ9 00 +se en +ĠSp ot +Ġshe ep +Ġspons ored +ĠH i +ĠJ ump +Ġ19 67 +Micro soft +ĠAg ent +Ġch arts +d ir +Ġadj acent +Ġtr icks +Ġman ga +Ġex agger +/ > +foot ball +ĠF CC +G C +ĠT ier +and ra +OU ND +% ), +Ġfru its +V C +ĠA A +R ober +Ġmid st +â Ĺ +ank a +Ġlegisl ature +ĠNe il +Ġtour ists +" " +ĠWar ning +ĠNever theless +ĠOffic ial +ĠWh atever +Ġm old +Ġdraft ed +Ġsubst ances +Ġbre ed +Ġt ags +ĠT ask +Ġver b +Ġmanufact ured +com ments +ĠPol ish +Pro v +Ġdetermin es +Ob ama +k ers +Ġutter ly +Ġse ct +sc he +ĠG ates +ĠCh ap +Ġal uminum +Ġz ombie +ĠT ouch +ĠU P +Ġsatisf y +Ġpred omin +asc ript +Ġelabor ate +Ġ19 68 +Ġmeas uring +ĠV ari +any ahu +Ġs ir +ul ates +id ges +ick ets +ĠSp encer +T M +oub ted +Ġpre y +Ġinstall ing +ĠC ab +re ed +re ated +Su pp +Ġwr ist +ĠK erry +10 7 +ĠK le +ĠR achel +Ġc otton +ĠA RE +ĠE le +Cont rol +Ġload s +ĠD od +an as +b one +Ġclass ical +ĠReg ional +ĠInt eg +V M +Ġdes ires +Ġaut ism +support ed +ĠM essage +Ġcomp act +writ er +Ġ10 9 +ĠHur ricane +c ision +Ġcy cles +Ġdr ill +Ġcolle ague +Ġm aker +G erman +Ġmist aken +S un +ĠG ay +Ġwhat soever +Ġsell s +ĠA irl +l iv +ĠO ption +Ġsol ved +Ġse ctors +Ġhorizont al +Ġequ ation +ĠSk ill +ĠB io +g ement +ĠSn ap +ĠLeg al +Ġtradem ark +Ġmake up +Ġassemb led +Ġsa ves +ĠHallow een +ĠVer mont +ĠFR OM +Ġfar ming +ĠP odcast +accept able +ĠHig her +Ġas leep +ull ivan +Ġrefere n +ĠLe v +Ġbul lets +ok o +H C +Ġst airs +Ġmain tains +ĠL ower +ĠV i +Ġmar ine +Ġac res +Ġcoordin ator +ĠJ oh +Ġcounterpart s +ĠBrother s +Ġind ict +b ra +Ġch unk +Ġc ents +H ome +ĠMon th +Ġaccording ly +if les +ĠGerm ans +ĠSy n +H ub +Ġey eb +âĶĢâĶĢ âĶĢâĶĢ +Ġr anges +ĠHoll and +ĠRob ot +f c +M ike +Ġpl asma +Ġsw ap +Ġath lete +ĠR ams +,' " +Ġinfect ions +Ġcor rid +Ġv ib +Ġpat ches +Ġtradition ally +Ġrevel ation +Ġswe ep +Ġgl ance +Ġin ex +200 3 +ĠR aw +work ing +os ures +ĠD at +ĠLyn ch +Ġle verage +ĠRe id +Ġcorrel ation +ian ces +av ascript +Ġrep ository +ret ty +Ġ19 72 +24 0 +Ġo un +p ol +ĠRe ed +Ġtact ical +is ite +App le +ĠQu inn +Ġrap ed +ill o +Euro pe +Ġalgorith ms +ĠRod rig +i u +Ġill um +Ġf ame +Ġintrodu cing +Ġdel ays +ĠRaid ers +Ġwh istle +Ġnovel s +ĠRe ally +Ġder iv +Ġpublic ations +ĠNe ither +ĠCom merce +Ġa ston +l anguage +Not es +ĠR oth +ĠF ear +Ġm ate +Ġpar ade +ĠQ B +Ġman eu +ĠC incinnati +m itting +Ġwa ist +ĠR ew +Ġdisc ont +Ð ° +Ġst aring +Ġal ias +Ġsec urities +Ġtoile t +ĠJ edi +Ġun law +v ised +//// //// +] ( +ĠWe iss +Ġpre st +ĠComp an +Ġmem o +ĠGr ace +J uly +ĠEl ite +cent er +ĠSt ay +Ġgal axy +Ġto oth +ĠS ettings +Ġsubject ed +ãĤ ¦ +Ġline back +Ġretail ers +ĠW ant +Ġd angers +A ir +Ġvolunt ary +ew ay +Ġinterpret ed +ot ine +à § +Ġp el +Serv ice +ĠEvent ually +Ġcare ers +Ġthreat en +Ġmem or +ĠBrad ley +anc ies +s n +ĠUn known +N ational +Ġsh adows +ail and +ĠD ash +Every one +izz ard +M arch += ( +Ġpull s +Ġstr anger +Ġback wards +ĠBern ard +imens ional +Ġch ron +Ġtheoret ical +k top +Ġw are +ĠInvest ig +ĠIn iti +ĠOper ations +o ven +oc ide +* / +Ġfl ames +ĠC ash +sh it +Ġc ab +ĠAn aly +ĠSe ah +Ġdefin ing +Ġorder ing +Ġimm un +Ġpers istent +AC H +Russ ian +m ans +Ġh ind +Ġphot ography + © +Ġh ug +Ġ10 7 +ĠH ence +i ots +ude au +Ġsubsid ies +Ġroutine ly +ĠDev ice +it ic +Ġdisg ust +land er +Ġ19 40 +Ġassign ment +ĠB esides +w ick +ĠD ust +us c +struct ed +11 1 +de velop +Ġf ond +Ġinter section +Ġdign ity +Ġcommission er +With out +re ach +Ġcart oon +Ġsc ales +ãĥ Ń +F IG +Ġsurve ys +ĠIndones ia +Ġart work +Ġun ch +Ġcy cling +un ct +au er +or ate +ĠOb viously +Ġcharacter ized +fe ld +Ġaff irm +Ġinn ings +Ġ é +Ġal iens +Ġcl oth +et ooth +ĠC ertain + § +Ġdig est +k now +ĠX L +Ġpredict ions +Ġd in +W AR +Ġafter math +Ex ample +ĠSu ccess +ĠTh r +IG N +Ġmin er +B us +Ġcl arity +heim er +ĠO UT +ĠS end +ĠCirc le +ĠD iet +Ġpron ounced +Ġcreat ors +Ġearthqu ake +atter y +ge ons +Ġo d +Ġlay ing +or p +U lt +pro ject +Ġunder min +Ġsequ el +S am +ĠDark ness +Ġre ception +b ull +Y S +ĠV ir +Ġsequ ences +ĠCo in +Ġout fit +ĠW ait +1 19 +Ġdel ivers +.... .. +Ġbl own +ĠE sc +ĠM ath +per m +ĠU l +Ġgl im +Ġfac ial +Ġgreen house +Ġto kens +/ - +ĠAnn ual +ĠON E +Ġteen age +ĠPhys ical +ĠL ang +ĠC elt +Ġsu ed +ivid ually +Ġpat ience +ch air +reg ular +Ġa ug +in v +ex cept +ĠL il +Ġn est +f d +s um +ĠCh ase +Russ ia +ĠJenn ifer +Ġoff season +Over all +F ore +Ġr iot +A ud +form er +Ġdefend ers +ĠC T +iot ic +rib ly +Ġautom ated +Ġpen is +Ġins ist +Ġdi agram +ĠS QL +ĠG arc +Ġw itch +cl ient +ier ra +am bers +Ġrec ount +f ar +V ery +oster one +Ġappreci ated +ĠPer fect +S ection +Ġd oses +oca ust +Ġcost ly +Ġg rams +ĠSh i +Ġwrest ling +Ġ19 71 +Ġtro phy +Ġn erve +ĠK az +ĠExper ience +Ġpled ged +Ġplay back +Ġcreat ivity +by e +Ġattack ers +Ġhold ers +ĠCo ach +ĠPh D +Ġtransf ers +Ġcol ored +ĠH indu +Ġd rown +Ġlist ened +ĠW A +ias m +P O +Ġappeal ing +Ġdiscl osed +ĠCh icken +ag ging +Ġple aded +Ġnav igation +ĠReturn s +Ġ[ [ +R OR +E A +Ġphotograp her +ĠR ider +ipp ers +Ġsl ice +Ġe rect +Ġhe d +iss ance +ĠVik ings +ur ious +Ġapp et +oubted ly +Ch ild +Ġauthent ic +o os +ĠM aking +Ġannoun cing +Ġb od +Ġmet er +ĠN ine +ĠR ogue +Ġwork force +Ġrenew ed +Ġorganis ations +ac s +P LE +Sh ort +Ġcomp ounds +ĠVis it +Ġen velop +ear th +Ġsupport ive +gg le +ĠBrus sels +ĠGu ild +Cre ate +RE L +Ġaver aged +Ġ19 69 +ri ages +Ġlength y +Ġforg ot +O kay +ĠE rd +Ġdeal er +Ġrec ession +D D +Ġdesper ately +Ġhun ger +Ġst icks +Ġm ph +ĠF aith +Ġintention ally +Ġdem ol +ue ller +ĠS ale +Ġde bris +s pring +Ġle ap +>> >> +Ġcontain ers +se lling +rane an +atter ing +Ġcomment ed +ĠC M +on ut +Ġwood s +es pecially +Ġorgan ize +iv ic +ĠWood s +ang a +s qu +Ġm aj +am on +Ġax is +Ġ19 74 +ĠDen mark +Ġwar rior +ĠP and +Ġout lined +ĠB O +ins ula +z illa +eb ook +Ġd are +Ġsear ched +Ġnav igate +S n +writ ing +Ġun ited +J apan +ĠHe brew +Ġfl ame +Ġrel ies +Ġcatch ing +ĠSh o +Ġimprison ment +Ġp ockets +Ġclos ure +ĠF am +t im +ade qu +Act ivity +Ġrecru iting +ĠW ATCH +ĠArgent ina +d est +Ġapolog ize +or o +Ġlack s +Ġtun ed +ĠGriff in +Ġinf amous +Ġcelebr ity +ss on +Ġ ---------------------------------------------------------------- +ĠIs is +ĠDis play +Ġcred ibility +Ġeconom ies +Ġhead line +ĠCow boys +Ġind ef +Ġl ately +Ġincent ives +but ton +ĠM ob +A ut +Ġres igned +ĠO m +c amp +Ġprof iles +Ġsche mes +olph ins +ay ed +Cl inton +en h +ĠY ahoo +Ġab st +Ġan k +su its +Ġw ished +ĠMar co +udd en +Ġsp here +ĠB ishop +Ġincorpor ated +ĠPl ant +11 4 +Ġh ated +p ic +Ġdon ate +Ġl ined +Ġbe ans +Ġsteal ing +Ġcost ume +Ġsher iff +Ġfor ty +Ġint act +Ġadapt ed +Ġtrave lling +b art +Ġnice ly +Ġdri ed +Ġsc al +os ity +NOT E +ĠB h +ĠBron cos +ĠI gn +Ġint imate +Ġchem istry +Ġopt imal +D eb +ĠGener ation +Ġ] , +ich i +ĠW ii +ĠYOU R +vent ions +W rite +Ġpop ul +un ning +ĠW or +V ol +Ġqu een +head s +K K +Ġanaly ze +op ic +ear chers +Ġd ot +leg raph +ast ically +Ġupgr ades +Ġca res +Ġext ending +Ġfree ze +Ġin ability +Ġorg ans +Ġpret end +Ġout let +11 3 +ol an +ĠM all +ul ing +t alk +Ġexpress ing +ĠAl ways +ĠBe gin +f iles +Ġlic enses +% % +ĠM itt +Ġfil ters +ĠMil waukee +G N +Ġunf old +M o +Ġnut rition +pp o +B o +Ġfound ing +Ġunder mine +Ġeas iest +ĠC zech +ĠM ack +Ġsexual ity +ĠN ixon +W in +ĠAr n +ĠK in +ãĤ £ +ic er +Ġfort un +Ġsurf aces +agh d +Ġcar riers +ĠP ART +ĠT ib +Ġinter val +Ġfrust rating +ĠSh ip +ĠAr med +ff e +Ġbo ats +ĠAb raham +in is +Ġsu ited +th read +i ov +ab ul +ĠVenezuel a +Ġto m +su per +Ġcast le +alth ough +iox ide +ec hes +Ġevolution ary +Ġnegoti ate +Ġconfront ed +Rem ember +Ġ17 0 +S uch +Ġ9 11 +m ult +ĠA byss +ur ry +ke es +spe c +ĠBarb ara +Ġbelong ing +Ġvill ain +ist ani +Ġaccount able +Ġport ions +ĠDe cl +U r +ĠK ate +g re +Ġmag azines +UC K +Ġregul ate +om on +ĠAl most +Ġover view +Ġsc ram +Ġl oot +ĠF itz +Ġcharacter istic +ĠSn ake +s ay +ĠR ico +Ġtra it +ĠJo ined +au cus +Ġadapt ation +ĠAirl ines +Ġarch ae +ĠI de +Ġb ikes +Ġliter ary +Ġinflu ences +ĠUs ed +C reat +Ġple a +ĠDef ence +ĠAss ass +Ġp ond +UL T +) " +Ġeval uated +Ġob taining +Ġdem ographic +Ġvig il +ale y +Ġsp ouse +ĠSeah awks +resp ons +ĠB elt +um atic +Ġr ises +run ner +ĠMichel le +Ġpot ent +r ace +ĠP AC +F ind +olester ol +IS S +ĠIntrodu ced +ress es +ign ment +O s +ĠT u +ĠDe x +ic ides +Ġspark ed +ĠLaur a +ĠBry ant +Ġsm iling +ĠNex us +Ġdefend ants +ĠCat al +Ġdis hes +sh aped +Ġpro long +m t +( $ +ãĢ Ĥ +Ġcalcul ations +ĠS ame +Ġp iv +H H +Ġcance lled +Ġgr in +Ġterrit ories +ist ically +C ome +ĠP arent +Pro ject +Ġneg lig +ĠPriv acy +Ġam mo +LE CT +olute ly +ĠEp ic +Ġmis under +w al +Apr il +m os +path y +ĠC arson +Ġalbum s +ĠE asy +Ġpist ol +< < +Ġ\ ( +t arget +hel p +Ġinter pre +cons cious +ĠH ousing +ĠJ oint +12 7 +Ġbe ers +s cience +ĠFire fox +effect ive +ĠC abin +ĠO kay +ĠApp lic +Ġspace craft +ĠS R +ve t +ĠStr ange +S B +Ġcor ps +iber al +e fficient +Ġpreval ence +Ġeconom ists +11 8 +Th read +ord able +OD E +ĠC ant +=- =- +if iable +ĠA round +Ġpo le +Ġwilling ness +CL A +ĠK id +Ġcomple ment +Ġsc attered +Ġin mates +Ġble eding +e very +Ġque ue +ĠTr ain +Ġh ij +Ġme lee +ple ted +Ġdig it +Ġg em +offic ial +Ġlif ting +Ð µ +Re qu +it utes +Ġpack aging +ĠWork ers +h ran +ĠLeban on +ol esc +Ġpun ished +ĠJ uan +Ġj am +ĠD ocument +Ġm apping +ic ates +Ġinev itably +Ġvan illa +ĠT on +Ġwat ches +Ġle agues +Ġiniti ated +deg ree +port ion +Ġrec alls +Ġru in +Ġm elt +I AN +Ġhe m +Ex p +Ġb aking +ĠCol omb +at ible +Ġrad ius +pl ug +ĠI F +et ically +Ġf ict +H ER +ĠT ap +atin um +Ġin k +Ġco h +ĠW izard +b oth +te x +Ġsp ends +ĠCurrent ly +ĠP it +Ġneur ons +ig nt +Ġr all +Ġbus es +b uilding +Ġadjust ments +Ġc ried +ibl ical +att ed +ĠZ ion +ĠM atter +Ġmed itation +ĠD ennis +Ġour s +ĠT ab +Ġrank ings +ort al +Ġad vers +Ġsur render +ĠG ob +ci um +om as +im eter +Ġmulti player +Ġhero in +Ġoptim istic +Ġindic ator +ĠBr ig +Ġgro cery +Ġapplic ant +ĠRock et +v id +Ex ception +p ent +Ġorgan izing +Ġenc ounters +ĠT OD +Ġjew el +S ave +ĠChrist ie +Ġhe ating +Ġl azy +ĠC P +Ġcous in +Con fig +Ġreg ener +Ġne arest +Ġachie ving +EN S +th row +ĠRich mond +ant le +200 2 +Ġan ten +b ird +13 3 +Ġn arc +r aint +un ny +ĠHispan ic +ourn aments +Ġprop he +ĠTh ailand +ĠT i +Ġinject ion +Ġinher it +rav is +Ġmed i +Ġwho ever +ĠDE BUG +G P +ĠH ud +C ard +p rom +Ġp or +Ġover head +L aw +Ġviol ate +Ġhe ated +Ġdescript ions +Ġachieve ments +ĠBe er +ĠQu ant +W as +Ġe ighth +ĠI v +Ġspecial ized +U PDATE +ĠD elta +P op +J ul +ĠAs k +oph y +Ġnews letters +ĠT ool +Ġg ard +ĠConf eder +ĠGM T +ĠAb bott +Ġimm unity +ĠV M +Is lam +Ġimpl icit +w d +Ġ19 44 +rav ity +omet ric +Ġsurv iving +ur ai +ĠPr ison +Ġr ust +ĠSk etch +Ġbe es +ĠThe ory +Ġmer it +T ex +ch at +Ġm im +Ġpast e +ĠK och +Ġignor ance +ĠSh oot +Ġbas ement +Un ited +ĠAd vis +he ight +Ġf oster +Ġdet ain +in formation +Ġne ural +' ; +Ġprov es +all ery +Ġinv itation +um bers +Ġc attle +Ġbicy cle +z i +Ġconsult ant +Ġap ology +ĠT iger +Ġ12 3 +99 9 +Ġind ividually +r t +ig ion +ĠBrazil ian +Ġdist urb +Ġentreprene urs +Ġfore sts +cer pt +pl ates +p her +clip se +Ġtw itter +Ġac ids +ograph ical +h um +ĠB ald +if ully +Ġcomp iler +ĠD A +Ġdon or +as i +Ġtrib al +l ash +ĠCon fig +Ġapplic ants +Ġsal aries +13 5 +Put in +ĠF ocus +ir s +Ġmisc onduct +ĠH az +Ġeat en +M obile +Mus lim +ĠMar cus +v iol +Ġfavor able +Ġst ub +ad in +ĠH ob +Ġfaith ful +Ġelectron ics +Ġvac uum +w ait +back ed +econom ic +d ist +Ġten ure +Ġsince re +ĠT ogether +ĠW ave +Ġprog ression +Ġden ying +Ġdist ress +br aska +th ird +Ġmix ing +Ġcolon ial +Ġpriv ately +Ġun rest +atern ity +Ġprem ises +ant i +greg ation +Ġlic ence +ĠH ind +ĠSam uel +Ġconvinc ing +ĠA ce +ĠR ust +ĠNet anyahu +Ġhand les +ĠP atch +orient ed +ah o +ĠG onz +Ġhack ers +claim er +Ġcustom s +ĠGr an +f ighters +Ġl uc +Ġman uscript +aren thood +Ġdev il +Ġwar riors +Ġoff enders +Will iam +Ġhol idays +Ġnight mare +Ġle ver +iff erent +St at +Ġexhib ition +put ed +ĠP ure +Ġal pha +Ġenthus iasm +ĠRepresent atives +E AR +ĠT yp +Ġwhe at +ĠAl f +Ġcor rection +Ġev angel +AT T +M iss +Ġs oup +Ġimpl ied +par am +Ġsex y +ĠL ux +Ġrep ublic +p atch +ab lish +Ġic ons +Ġfather s +ĠG ET +ĠCar ib +Ġregul ated +ĠCo hen +ĠBob by +Ġn er +Ġb ent +vent ory +ĠAl ong +ĠE ST +ĠWall ace +Ġmurd ers +r ise +ke ll +ĠCommon wealth +Ġn asty +et a +ĠM IT +Ġadminist ered +Ġgenuine ly +Ed itor +n ick +Ġhyd ro +**************** **************** +ĠB le +Ġfin es +Ġg orge +aus ible +r h +Ġapp le +ment ioned +Ġro pe +ot yp +H R +Ġdisappoint ing +Ġc age +n ik +Ġdoub ts +ĠF REE +print s +ĠM UST +Ġvend ors +ĠIn qu +Ġliber als +Ġcontract or +Ġup side +child ren +Ġtrick y +Ġregul ators +charg ed +l iter +Ġ *** +Ġreb ell +l ang +Ġloc als +Ġphys icians +Ġhe y +ar se +t m +ĠLe x +Ġbehavior al +success ful +F X +Ġbr ick +ov ic +Ġcon form +Ġreview ing +Ġins ights +Ġbi ology +ĠRem ove +ĠExt ra +Ġcomm itting +indu ced +ignt y +ig m +Ġat omic +Comm on +ĠE M +ĠP ere +ĠIt ems +e h +Ġpres erved +ĠH ood +Ġprison er +Ġbankrupt cy +Ġg ren +us hes +Ġexplo itation +Ġsign atures +Ġfin an +] ," +ĠM R +Ġme g +rem lin +Ġmusic ians +Ġselect ing +Ġexam ining +IN K +l ated +H i +Ġart ic +Ġp ets +Ġimp air +ĠM AN +Ġtable ts +in clude +R ange +Ġca ut +Ġlog s +Ġmount ing +Ġun aware +Ġdynam ics +ĠPalest ine +ĠQu arter +ĠPur ple +Ġm a +ĠIm port +Ġcollect ions +ci ation +Ġsuccess or +Ġcl one +Ġaim ing +Ġposs essed +Ġstick ing +Ġsh aking +Ġloc ate +ĠH ockey +T urn +17 0 +Ġfif teen +ĠHar rison +Ġcontinu ously +ĠT C +ĠVal ent +ĠRes cue +Ġby pass +am ount +Ġm ast +Ġprotect s +Ġart istic +Ġsomet ime +Ġsh oe +Ġshout ed +ific ant +et itive +ĠReg ister +ĠJ in +Ġconcent rated +ling ton +on ies +Ġgener ator +yr im +ĠAr men +Ġclear ing +id o +ĠT W +al ph +Ġlad ies +H ard +Ġdial og +Ġinput s +æ ľ +Ġpos es +Ġsl ots +ĠPrem ium +Ġle aks +Ġboss es +Ġ11 3 +c ourse +A cc +ĠNew ton +ĠAust ria +ĠM age +Ġte aches +ab ad +Ġwe ars +Ġc yl +Ġcur se +ĠS ales +ĠW ings +Ġp sy +Ġg aps +ĠIce land +ĠP interest +Ġland lord +Ġdefin itions +ĠK er +Ġsufficient ly +ĠP ence +ĠArch itect +Ġsur pass +Ġ11 4 +Ġsuper hero +ĠDise ase +Ġpri ests +ĠC ulture +Ġdefin itive +Ġsecret ly +ĠD ance +inst all +ch ief +ĠJess ica +W ould +Up dated +Ġlock er +ĠK ay +Ġmem orial +è ¦ +f at +Ġdis gu +Ġflav ors +ĠBase ball +ĠRes istance +Ġk icks +Ġen v +Ġteen agers +D ark +ĠC AR +Ġh alt +ĠL G +ĠGab riel +Ġfe ver +Ġs atur +Ġm all +Ġaffili ate +ĠS leep +ĠSpe cific +ĠV el +Ġj ar +ĠSac red +ĠEd wards +ĠA CL +Ġret ained +ĠG iant +Ġlim itation +in ces +Ġref usal +ĠT ale +ĠBut ler +Ġacc idents +ĠC SS +Ġimport ed +ĠCop y +Î ± +ER T +z el +Ġdiv isions +h ots +ĠAl b +ĠD S +Load er +W ashington +at isf +ĠCreat ive +\ . +ĠAut om +red ict +Ġrecept or +ĠCarl os +Met hod +ok a +Ġmal icious +Ġste pping +, [ +ĠD ad +Ġatt raction +ĠEffect s +ĠPir ate +ĠC er +ĠIndust ry +ĠR ud +Ġchar ter +Ġd ining +Ġins ists +Ġconfig ure +Ġ( # +ĠSim ple +ĠSc roll +UT C +17 5 +ĠK on +Ġmarket place +Ġ ãĤ +Ġref res +Ġg ates +er red +ĠP od +Ġbeh ave +Fr ank +n ode +Ġendors ed +he tt +as ive +ĠHom eland +Ġr ides +ĠLe ave +er ness +Ġflood ing +A FP +Ġris en +Ġcontin ually +Ġun anim +ĠCont ract +ĠP as +Ġgu ided +ĠCh ile +b d +Ġsu cc +pt ic +Ġcomm ittees +ĠL uther +ĠAny one +Ġs ab +12 4 +Ġp ixel +ĠB ak +ĠT ag +ĠBenn ett +En ter +sm all +ĠPresident ial +Ġp ul +Ġcontr ace +arch ive +Ġcoast al +ĠK ids +19 2 +âĢ ² +ick y +ING TON +Ġw olf +ĠSt alin +T ur +id get +am as +ĠUn less +Ġspons or +Ġmor ph +ĠCho ose +Ġrun ner +Ġun bel +Ġm ud +ĠMan a +Ġdub bed +Ġg odd +ure rs +wind ow +Ġrel ied +Ġcelebr ating +os c +Ġ13 5 +Ġlobb ying +Ġincom plete +Ġrestrict ion +Ġinc ap +it us +Ġexpect ation +ĠAp ollo +Ġint ens +Ġsyn c +G H +Ġmanip ulation +B Y +Ġspe ar +Ġbre asts +Ġvol can +il ia +M aterial +Ġform ats +ĠB ast +Ġparliament ary +Ġsn ake +Ġserv ants +ĠTr udeau +ĠGr im +ĠArab ic +ĠSC P +ĠBoy s +st ation +Ġprospect ive +ord e +in itialized +Ġb ored +AB LE +Ġaccess ed +Ġtax i +ĠShe ll +aid en +urs ed +in ates +ĠIns urance +ĠPet e +Sept ember +6 50 +Ġad ventures +ĠCo ver +Ġt ribute +Ġsk etch +Ġem power +Ġ Ø +ĠGl enn +ĠD aw += \" +ĠPolit ics +Ġgu ides +Ġd ioxide +ĠG ore +ĠBr ight +ĠS ierra +Ġval ued +c ond +Ġpo inter +Se lect +Ġrisk y +Ġabsor b +im ages +Ġref uses +Ġbon uses +__ _ +Ġh ilar +ĠF eatures +2 20 +ĠCollect or +F oot +Ġ19 64 +cul us +Ġd awn +Ġwork out +ĠL O +Ġphilosoph ical +ĠSand y +ĠYou th +Ġl iable +A f +bl ue +Ġovert urn +less ness +ĠTrib une +ĠIn g +Ġfact ories +Ġcat ches +Ġpr one +Ġmat rix +Ġlog in +Ġin acc +Ġex ert +s ys +Ġneed le +ĠQ ur +Ġnot ified +ould er +t x +Ġremind s +Ġpublisher s +Ġn ort +Ġg it +Ġfl ies +ĠEm ily +Ġflow ing +ĠAl ien +ĠStr ateg +Ġhard est +Ġmod ification +AP I +ĠM Y +Ġcr ashes +st airs +n umber +Ġur ging +ch annel +ĠFal con +Ġinhabit ants +Ġterr ifying +Ġutil ize +Ġban ner +Ġcig arettes +Ġsens es +ĠHol mes +Ġpract ition +ĠPhill ips +ott o +Ġcomp ile +Mod el +ĠK o +Ġ[ ] +Americ ans +ĠTer ms +Ġmed ications +ĠAn a +Ġfundament ally +ĠNot ice +Ġwe aker +Ġ 0000 +Ġgar lic +Ġout break +Ġeconom ist +ĠB irth +Ġobst acles +ar cer +ĠOr thodox +Ġplace bo +ĠC rew +asp berry +ĠAng els +Ġdis charge +Ġdestruct ive +11 7 +ĠR ising +Ġd airy +l ate +Ġcoll ision +ĠTig ers +ean or +ocument ed +ĠIn valid +Ġd ont +ĠL iter +ĠV a +Ġhyd rogen +Ġvari ants +ĠBrown s +Ġ19 65 +Ġind igenous +Ġtrad es +Ġremain der +Ġswe pt +ĠImp act +Ġred ist +Ġun int +grad uate +ãĥ ķ +ĠW ILL +ãģ® ç +ĠCrit ical +Ġf isher +Ġv icious +Ġrevers ed +Y ear +ĠS ox +Ġshoot ings +Ġfil ming +Ġtouchdown s +ai res +m el +Ġgrand father +Ġaffect ion +ing le +Ġover ly +Add itional +Ġsup reme +ĠGr ad +Ġsport ing +Ġmer cy +ĠBrook s +ount y +Ġperform s +Ġtight ly +Ġdem ons +Ġkill ings +Ġfact ion +ĠNov a +aut s +Ġund oubtedly +ar in +Ġunder way +ra k +Ġl iv +ĠReg ion +Ġbrief ing +s ers +cl oud +ĠM ik +us p +Ġpred iction +az or +Ġport able +ĠG and +Ġpresent ing +Ġ10 80 + » +ush i +ĠSp ark +there um +Ġjust ification +ĠN y +Ġcontract ors +ming ham +ĠSt yle +å ħ +ĠChron icles +ĠPict ure +Ġprov ing +Ġw ives +set t +Ġmole cules +ĠFair y +Ġconsist ing +Ġp ier +al one +in ition +Ġn ucle +j son +Ġg otta +Ġmob il +Ġver bal +ar ium +Ġmon ument +uck ed +Ġ25 6 +T ech +mine craft +ĠTr ack +Ġt ile +Ġcompat ibility +as is +Ġs add +Ġinstruct ed +ĠM ueller +Ġle thal +Ġhorm one +Ġor che +el se +Ġske let +Ġentert aining +Ġminim ize +ag ain +Ġunder go +Ġconst raints +Ġcig arette +ĠIslam ist +Ġtravel s +ĠPant hers +l ings +C are +Ġlaw suits +ur as +Ġcry st +Ġlow ered +Ġaer ial +Ġcomb inations +Ġha un +Ġch a +Ġv ine +Ġquant ities +Ġlink ing +b ank +Ġso y +B ill +ĠAngel a +Ġrecip ient +ĠProt est +Ġs ocket +Ġsolid arity +Ġâ Ĩ +m ill +Ġvar ies +ĠPak istani +Dr agon +Ġun e +Ġhor izon +³³³³ ³³³³ +Ġprov inces +Ġfrank ly +Ġenact ed +not es +[ ' +Ġ19 2 +ocr acy +Ġendorse ment +Ġover time +Tr ue +L ab +lic ted +ĠD NC +Ġbe ats +ĠJam ie +15 2 +ĠIN T +Cont act +Ġaccount ed +h ash +ĠPack ers +p ires +Ġles bian +Ġamend ments +Ġhop eful +ĠFin land +Ġspot light +Ġconfig ured +Ġtrou bled +Ġg aze +ĠCal gary +Ġrel iability +Ġins urg +sw er +b uy +ĠSk in +Ġp ixels +Ġhand gun +Ġpar as +Ġcateg or +ĠE L +ĠRe x +Ind eed +Ġkind a +Ġconj unction +ĠBry an +ĠMan ufact +y ang +Pl us +S QL +ish ment +Ġdom inate +Ġn ail +Ġo ath +Ġeru pt +ĠF ine +it bart +ĠCh ip +ĠAb d +ĠN am +Ġbuy er +Ġdiss ent +Le aks +Cont in +Ġr ider +ĠSome one +Ġill usion +c in +ĠBoe ing +Ġin adequ +ov ation +i ants +Ġreb uild +4 50 +ĠDest iny +S W +ĠT ill +H it +ia z +ĠBang l +acher s +ĠRe form +Ġse gments +Ġsystem atic +d c +ĠConserv atives +Ġport al +h or +ĠDragon bound +Ġdrag ged +om o +Ġthe e +ad vert +ĠRep orts +ĠE t +Ġbarrel s +Aug ust +Ġcompar isons +Ġhe x +Ġan throp +" [ +bor ough +ab i +Ġpict ured +play ing +ĠAdd ress +ĠMir ror +Sm ith +Ġt ires +ĠN PR +AA AA +Ġclass ification +ĠTh an +ĠH arm +ĠR A +Ġreject ion +min ation +Ġr anged +ĠF alls +D I +H ost +ãĤ ´ +ĠEx ample +list ed +th irds +Ġsaf egu +br and +Ġprob able +Can ada +IT ION +ĠQ aeda +Ġch ick +Ġimport s +h it +l oc +W W +Ġble w +Ġany time +Ġwh oles +ik ed +Ġcal culation +cre ate +ĠO ri +Ġupgr aded +Ġapp ar +ut ory +ĠM ol +B rit +ĠJ ong +IN AL +ĠStart ing +Ġd ice +urt le +Ġre lying +cl osure +Ġprof itable +Ġsl aughter +ĠMan ual +c aster +Ġ" $ +Ġfe ather +ĠSim ply +ie ves +Ġdeter ior +ĠPC I +Ġst amp +Ġfl aws +Ġsh ade +ham mer +Ġpass port +Ġcont ing +am el +Ġobser vers +Ġneg lect +ĠR B +ĠBrother hood +Ġskept ical +f amily +us k +Ġemotion ally +â Ļ +ĠBet a +ason able +id ity +ĠM ul +Ġkick ing +ĠC arm +oll ah +VERT IS +ĠAt hen +Ġlad der +ĠBul let +å £ +00 01 +ĠWild life +ĠM ask +ĠN an +R ev +Ġun acceptable +leg al +Ġcrowd ed +ag i +ĠC ox +j e +Ġmor ality +Ġfu els +Ġc ables +Ġman kind +ĠCarib bean +Ġanch or +Ġby te +ĠO ften +ĠO z +Ġcraft ed +Ġhistor ian +ĠW u +Ġtow ers +ĠCitiz ens +Ġhel m +Ġcred entials +Ġsing ular +ĠJes se +Ġtack les +Ġcont empt +Ġa fore +ĠSh adows +Ġn il +Ġur gent +app le +bl ood +Ġv on +Ġoff line +Ġbreat he +Ġj umps +Ġirre levant +ox ic +om al +import ant +J im +Ġgl oves +arm ing +dep th +Ġtal ents +ook ie +ĠS B +Ġpal m +uff s +est a +IG H +Ġcan on +ĠVer izon +ĠP le +Ġcou pled +vel t +Ġfundra ising +ĠGet ting +ĠD LC +Ġmathemat ical +ĠH S +ĠCard inals +te lling +Ġspons ors +Ġ Ï +ĠBull s +op tion +Ġprop ose +Ġmem orable +Ġembr aced +Ġdecl ining +He alth +ed a +Ġ} ; +Ġsp am +m ile +Ġpit cher +ĠE ight +Ġcar ing +ut ic +ro le +Ġair line +ernand ez +ĠAth let +Ġcert ification +ux e +rig er +Ġem pir +Ġsens ation +Ġdis m +Ġb olt +Ġev olve +H ouse +Ġconsult ation +ĠD uty +Ġtou ches +ĠN athan +Ġf aint +h ad +" ( +ĠCons umer +ĠExt reme +Ġ12 7 +ĠHer m +ĠSac rament +iz oph +Ġanx ious +ul ously +Ġsoc ially +ĠU TC +Ġsol ving +ĠLet ter +Hist ory +ed uc +Pr ice +) ); +Ġrel oad +am ic +Ġp ork +Ġdisc ourse +Ġt ournaments +ai ro +ĠK ur +ĠCost a +Ġviol ating +Ġinterf ere +Ġrecre ational +uff le +Ġspe eches +Ġneed ing +Ġremem bers +Ġcred ited +n ia +f ocused +amer a +Ġb ru +um bs +ĠCub an +Ġpreced ing +Ġnons ense +ac ial +Ġsmart phones +ĠSt ories +S ports +ĠEmer gency +oun cing +ef ined +Ġb er +Ġconsult ing +Ġm asters +he astern +." [ +ĠRun ning +Ġsus cept +ĠF eng +Americ a +pr ises +st itial +ĠWeek ly +ĠGreat er +mod ules +if ter +G raphics +ul er +Ġwho lly +Ġsupp ress +Ġconce aled +Ġhapp ily +Ġaccept s +ĠEn joy +Ġr ivers +ĠEx cept +2 25 +ĠN HS +ĠMc Connell +Ġp ussy +fer red +ut able +Ġatt ain +Ġ> = +Ġdepos its +roph ic +Ġnot orious +ĠSh aw +il itation +Ġepid emic +all ic +Ġsmall est +ov ich +Ġaccess ories +per ties +Ġsur plus +ĠMe ch +Ġamb ig +ĠImm igration +Ġch im +ev al +Ġpract icing +ĠMyster y +Ġdom ains +ĠSil icon +app s +Ġkilomet ers +e a +ĠSm ash +Ġwarrant y +Ġn ost +s il +re v +J on +ĠDub lin +Ġtast es +Ġb out +g reat +er ror +Ġsw itches +ĠB apt +D O +ok i +Ġsour ced +pro du +Ġattach ment +ĠIss ue +ĠQuest ion +Jo in +Ġf itted +Ġunlaw ful +^ ^ +ere k +Ġauthent ication +Ġst ole +Ġaccount ability +l abel +S earch +Ġal beit +atic an +fund ed +ĠAdd ing +ĠI Q +Ġsub mar +l it +a que +ĠLear ning +Ġint eger +M aster +ĠCh rom +Ġprem ier +O p +ĠLi u +Ġbl essed +ĠGl obe +ĠResp onse +Ġlegit im +ĠMer kel +Ġdispos al + ´ +Ġgau ge +pe at +Ġindu ced +Ġquestion able +arth y +ĠV it +ĠF eed +U ntil +U t +worth y +R Y +ĠH erald +ĠHam mer +Ġmed al +ĠR ivers +ĠH ack +Ġclar ify +Ġtrack ed +Ġautonom ous +Ġten ant +ĠQ atar +er ie +Ġgr im +ĠMon itor +Ġresist ant +ĠSpe c +ĠWell s +N AS +14 8 +Ġmin ers +iot ics +Ġmiss es +11 6 +g ian +g it +ĠE yes +p res +Ġgrad uated +Ġang el +Ġsyn chron +Ġefficient ly +Ġtrans mitted +H arry +Ġglob ally +EN CE +ĠMont ana +r aged +ĠPre vention +Ġp iss +ĠL l +Ġshe lf +ĠB JP +ĠTest ament +ĠL ate +ik er +ĠH app +ĠJul ian +h all +Ġsp ont +Ġshut down +Ġincons istent +Ġsubscrib ers +Ġske leton +ĠNe braska +Ġins pire +ĠV oid +F eed +Ġang les +ĠSpr ings +Ġbench mark +Ġvacc ines +izoph ren +se xual +uff ed +Ġsh ine +ĠK ath +Ġgest ure +ine a +Ġr ip +Ġopp ression +Ġcons cience +b t +ĠL um +Ġinc idence +ĠF a +w r +Ġmin eral +ĠSp urs +alk y +Ġth under +Ġop io +Be ing +ĠPal m +Ġwas ted +Ġl b +i aries +ĠIniti ative +Ġcur ric +Ġmark er +ĠMc L +Ġext ensions +ĠP v +ĠAr ms +Ġoffer ings +Ġdef enses +Ġvend or +Ġcontrad ict +ĠCol in +Ġredd it +Ġper ipher +12 2 +Ġs ins +E dit +IC T +So ft +ĠSh ah +Ġadministr ator +ĠT rip +Ġporn ography +Ġtu ition +in ence +ĠPro gress +Ġcat alog +Ġsu ite +Ġh ike +Ġreprodu ctive +eng ine +Ġd rought +ĠNo ah +Ġ2 30 +Ġd ude +Ġrelax ed +Ġpart ition +Ġparticip ant +Ġtel esc +Ġfe as +ĠF F +own er +Ġswe eping +Ġl enses +Ġmatch up +ĠRe pl +ourn als +Ġcred ible +Ġgrand mother +Ġther mal +Ġsubscrib ing +Ġident ities +col m +U CT +Ġreluct ant +us ers +ĠC ort +Ġassist ed +OS S +ATION S +IS H +Ġpharm aceutical +ic able +ad ian +ĠSon ic +ĠF ury +ĠM ong +A H +ĠPsych ology +Ġph osph +Ġtreat s +Ń Ķ +Ġstead ily +ĠHell o +Ġrel ates +Ġcl ue +Ex pl +a uth +Ġrev ision +Ġe ld +os ion +Ġbr on +14 4 +ri kes +Ġmin es +Ġblank et +ĠF ail +el ed +ĠIm agine +ĠPl anned +a ic +Re quest +M ad +ĠHor se +ĠEag le +Ġcap ac +15 7 +Ġl ing +ĠN ice +ĠP arenthood +min ster +og s +ens itive +Not hing +Ġcar n +F in +ĠP E +Ġr ifles +ĠL P +S and +Ġgui Active +Ġtour ist +C NN +Ġunve iled +Ġpredec essor +} { +u ber +Ġoff shore +Ġopt ical +ĠR ot +ĠPear l +et on +Ġst ared +Ġfart her +at ility +cont in +ĠG y +ĠF oster +ĠC oc +ri ents +Ġdesign ing +ĠEconom y +ON G +W omen +ĠN ancy +er ver +Ġmas cul +Ġcasual ties +Ġ2 25 +ĠS ullivan +ĠCh oice +Ġa ster +w s +Ġhot els +Ġconsider ations +Ġcou ch +ĠSt rip +ĠG n +Ġmanip ulate +l ied +Ġsynt hetic +Ġassault ed +Ġoff enses +ĠDra ke +Ġim pe +Oct ober +ĠHer itage +h l +ĠBl air +Un like +Ġg rief +Ġ4 50 +Ġopt ed +Ġresign ation +il o +Ġver se +ĠT omb +Ġu pt +Ġa ired +ĠH ook +ĠML B +Ġassum es +out ed +ĠV ers +Ġinfer ior +Ġbund le +ĠD NS +ograp her +Ġmult ip +ĠSoul s +Ġillust rated +Ġtact ic +Ġdress ing +Ġdu o +Con f +Ġrel ent +Ġc ant +Ġscar ce +Ġcand y +ĠC F +Ġaffili ated +Ġspr int +yl an +ĠGarc ia +Ġj unk +Pr int +ex ec +C rit +Ġport rait +ir ies +ĠOF F +Ġdisp utes +W R +L ove +ãģ Ħ +ĠRe yn +Ġh ipp +op ath +Ġflo ors +ĠFe el +Ġwor ries +Ġsett lements +ĠP os +Ġmos que +Ġfin als +Ġcr ushed +ĠPro bably +ĠB ot +ĠM ans +ĠPer iod +Ġsovere ignty +Ġsell er +Ġap ost +Ġam ateur +Ġd orm +Ġconsum ing +Ġarm our +ĠRo ose +Ġint ensive +Ġelim inating +ĠSun ni +ĠAle ppo +j in +Ġadv ise +p al +ĠH alo +Ġdes cent +Ġsimpl er +Ġbo oth +ST R +L ater +ĠC ave +== = +Ġm ol +Ġf ist +Ġshot gun +su pp +Ġrob bery +E ffect +Ġobsc ure +ĠProf essional +Ġemb assy +Ġmilit ant +Ġinc arcer +Ġgener ates +Ġlaun ches +Ġadministr ators +Ġsh aft +Ġcirc ular +Ġfresh man +ĠW es +ĠJo el +ĠD rew +ĠDun can +ĠApp arently +s ight +ĠIntern al +ĠInd ividual +ĠF E +Ġb ore +ĠM t +Ġbroad ly +ĠO ptions +ount ain +ip es +ĠV ideos +20 4 +Ġh ills +Ġsim ulation +Ġdisappoint ment +it an +ĠLabor atory +Ġup ward +Ġbound ary +Ġdark er +h art +Ġdomin ance +C ong +ĠOr acle +ĠL ords +Ġscholars hip +ĠVin cent +ed e +ĠR ah +Ġencour ages +ro v +Ġqu o +Ġprem ise +ĠCris is +ĠHol ocaust +Ġrhyth m +Ġmet ric +cl ub +Ġtransport ed +Ġn od +ĠP ist +Ġancest ors +ĠFred er +th umbnails +ĠC E +ON D +Ph il +ven ge +ĠProduct s +cast le +Ġqual ifying +ĠK aren +VERTIS EMENT +Ġmight y +Ġexplan ations +Ġfix ing +D i +Ġdecl aring +Ġanonym ity +Ġju ven +ĠN ord +ĠDo om +ĠAct ually +O k +ph is +ĠDes ert +Ġ11 6 +I K +ĠF M +Ġinc omes +V EL +ok ers +Ġpe cul +Ġlight weight +g ue +Ġacc ent +Ġincre ment +ĠCh an +Ġcompl aining +ĠB aghd +Ġmidfield er +Ġover haul +Pro cess +ĠH ollow +ĠTit ans +Sm all +man uel +ĠUn ity +ĠEv ents +S ty +Ġdispro portion +n esty +en es +ĠC od +Ġdemonstr ations +ĠCrim son +ĠO H +Ġen rolled +Ġc el +ĠBre tt +Ġa ide +Ġhe els +Ġbroad band +Ġmark ing +Ġw izard +ĠN J +ĠChief s +Ġingred ient +Ġd ug +ĠSh ut +urch ase +end or +Ġfar mer +ĠGold man +12 9 +15 5 +Or der +Ġl ion +i ably +Ġst ain +ar ray +ilit ary +ĠFA Q +Ġexpl oded +ĠMcC arthy +ĠT weet +ĠG reens +ek ing +l n +ens en +Ġmotor cycle +Ġpartic le +Ġch olesterol +B ron +Ġst air +Ġox id +Ġdes irable +ib les +Ġthe or +for cing +Ġpromot ional +ov o +b oot +ĠBon us +raw ling +Ġshort age +ĠP sy +Ġrecru ited +Ġinf ants +Ġtest osterone +Ġded uct +Ġdistinct ive +Ġfirm ware +bu ilt +14 5 +Ġexpl ored +Ġfact ions +Ġv ide +Ġtatt oo +Ġfinan cially +Ġfat igue +Ġproceed ing +const itutional +Ġmis er +Ġch airs +gg ing +ipp le +Ġd ent +Ġdis reg +ç Ķ +st ant +ll o +b ps +aken ing +Ġab normal +ĠE RA +å£ « +ĠH BO +ĠM AR +Ġcon cess +Ġserv ant +Ġas pir +l av +ĠPan el +am o +Ġprec ip +Ġrecord ings +Ġproceed ed +Ġcol ony +ĠT ang +ab lo +Ġstri pped +Le ft +to o +Ġpot atoes +Ġfin est +% ). +Ġc rap +ĠZ ach +ab ases +ĠG oth +Ġbillion aire +w olf +Ġsan ction +S K +Ġlog ged +P o +ey ed +un al +Ġcr icket +Ġarm ies +Ġunc overed +Cl oud +ó n +Ġreb ounds +Ġm es +O per +P ac +Ġnation ally +Ġinsert ed +p ict +Ġgovern ance +Ð ¸ +Ġprivile ges +G ET +Ġfavor ites +im ity +Ġlo ver +the m +em pl +Ġgorge ous +An n +Ġsl ipped +Ġve to +B ob +Ġsl im +u cc +ĠF ame +udden ly +Ġden ies +ĠM aur +Ġdist ances +Ġw anna +t ar +ĠS ER +Ġâ Ī +Ġle mon +at hetic +Ġlit eral +Ġdistingu ished +Ġansw ering +G I +Ġrelig ions +ĠPhil os +ĠL ay +Ġcomp os +ire ments +ĠK os +ine z +roll ing +Ġyoung est +and ise +ĠB orn +Ġalt ar +am ina +ĠB oot +v oc +Ġdig ging +Ġpress ures +Ġl en +26 4 +Ġassass ination +ĠBir mingham +ĠMy th +Ġsovere ign +ĠArt ist +ĠPhot ograph +Ġdep icted +Ġdisp ens +orth y +Ġamb ul +int eg +ĠC ele +ĠTib et +Ġhier archy +Ġc u +Ġpre season +ĠPet erson +Ġcol ours +Ġworry ing +Ġback ers +ĠPal mer +ĠÎ ¼ +Ġcontribut or +Ġhear ings +Ġur ine +Ġ Ù +ourge ois +Sim ilar +ĠZ immer +s omething +ĠUS C +Ġstrength s +ĠF I +Ġlog ging +As ked +ĠTh ai +in qu +ĠW alt +Ġcrew s +it ism +3 01 +Ġshar ply +um ed +Ġred irect +r ators +In f +ĠWe apons +Ġte asp +19 99 +L ive +ĠEs pecially +ĠS ter +ĠVeter ans +Ġint ro +other apy +Ġmal ware +Ġbre eding +Ġmole cular +ĠR oute +ĠCom ment +oc hem +Ġa in +Se ason +Ġlineback er +Ä « +ĠEconom ics +es ar +ĠL ives +ĠEm ma +Ġk in +ĠTer rit +Ġpl anted +ot on +ĠBut ter +ĠSp ons +P ER +Ġdun geon +Ġsymb olic +Ġfil med +Ġdi ets +Ġconclud es +Ġcertain ty +ĠForm at +Ġstr angers +form at +ĠPh ase +Ġcop ied +Ġmet res +ld a +ĠUs ers +Ġdeliber ate +Ġwas hed +ĠL ance +im ation +Ġimpro per +ĠGen esis +ick r +ĠK ush +Ġreal ise +Ġembarrass ing +alk ing +b ucks +Ġver ified +Ġout line +year s +ĠIn come +20 2 +Ġz ombies +F inal +ĠMill enn +Ġmod ifications +ĠV ision +ĠM oses +ver b +iter ranean +ĠJ et +Ġnav al +ĠA gg +Ġur l +Ġvict ories +Ġnon etheless +Ġinj ust +ĠF act +ç ļ +Ġins ufficient +re view +face book +Ġnegoti ating +Ġguarant ees +im en +uten berg +Ġg ambling +Ġcon gr +Load ing +Ġnever theless +Ġpres idents +ĠIndust rial +Ġ11 8 +Ġp oured +ĠT ory +Ġ17 5 +Ġ: = +Sc ott +ange red +T ok +Ġorgan izers +M at +ĠG rowth +Ġad ul +Ġens ures +Ġ11 7 +é¾į å +Ġmass acre +Ġgr ades +be fore +AD VERTISEMENT +ĠSl ow +ĠM MA +âĢĶ " +ĠV atican +Q aeda +Ġo we +66 66 +ĠS orry +ĠGr ass +Ġbackground s +Ġexha usted +Ġcl an +Ġcomprom ised +ĠE lf +ĠIsa ac +ens on +In vest +IF A +Ġinterrupt ed +ãĥī ãĥ© +Ġtw isted +ĠDrag ons +M ode +ĠK remlin +Ġfert il +he res +ph an +ĠN ode +f ed +ĠOr c +Ġunw illing +C ent +Ġprior it +Ġgrad uates +Ġsubject ive +Ġiss uing +ĠL t +Ġview er +Ġw oke +Th us +bro ok +Ġdep ressed +Ġbr acket +ĠG or +ĠFight ing +Ġstri ker +Rep ort +ĠPortug al +Ġne o +w ed +19 9 +Ġflee ing +sh adow +ident ified +US E +Ste am +Ġstret ched +Ġrevel ations +art ed +ĠD w +Ġalign ment +est on +ĠJ ared +S ep +Ġblog s +up date +g om +r isk +Ġcl ash +ĠH our +Ġrun time +Ġunw anted +Ġsc am +Ġr ack +Ġen light +on est +ĠF err +Ġconv ictions +Ġp iano +Ġcirc ulation +ĠW elcome +Ġback lash +ĠW ade +Ġrece ivers +ot ive +J eff +Ġnetwork ing +ĠPre p +ĠExpl orer +Ġlect ure +Ġupload ed +ĠMe at +B LE +ĠNaz is +ĠSy nd +st ud +ro ots +ri ans +Ġportray ed +Ġ ?? +ĠBudd ha +s un +Rober t +ĠCom plex +Ġover see +Ġste alth +T itle +ĠJ obs +ĠK um +Ġappreci ation +ĠM OD +Ġbas ics +Ġcl ips +Ġnurs ing +Ġpropos ition +Ġreal ised +ĠNY C +Ġall ocated +ri um +ar an +ĠPro duction +ĠV ote +Ġsm ugg +Ġhun ter +az er +ĠCh anges +Ġfl uct +y on +Ar ray +Ġk its +W ater +Ġuncom mon +Ġrest ing +ell s +w ould +Ġpurs ued +Ġassert ion +omet own +ĠMos ul +ĠPl atform +io let +Ġshare holders +Ġtra ils +P ay +ĠEn forcement +ty pes +ĠAn onymous +Ġsatisf ying +il ogy +Ġ( ' +w ave +c ity +Ste ve +Ġconfront ation +ĠE ld +C apt +ah an +ht m +ĠC trl +ON S +2 30 +if a +hold ing +Ġdelic ate +Ġj aw +ĠGo ing +or um +S al +Ġd ull +ĠB eth +Ġpr isons +Ġe go +ĠEl sa +avor ite +ĠG ang +ĠN uclear +Ġsp ider +ats u +Ġsam pling +Ġabsor bed +ĠPh arm +iet h +Ġbuck et +ĠRec omm +O F +ĠF actory +AN CE +Ġb acter +H as +ĠObs erv +12 1 +Ġprem iere +De velop +Ġcur rencies +C ast +Ġaccompany ing +ĠNash ville +Ġfat ty +ĠBre nd +Ġloc ks +Ġcent ered +ĠU T +augh s +or ie +ĠAff ordable +v ance +D L +em et +Ġthr one +ĠBlu etooth +Ġn aming +if ts +AD E +Ġcorrect ed +Ġprompt ly +ĠST R +Ġgen ome +Ġcop e +Ġval ley +Ġround ed +ĠK end +al ion +p ers +Ġtour ism +Ġst ark +v l +Ġblow ing +ĠSche dule +st d +Ġunh appy +Ġlit igation +ced es +Ġand roid +Ġinteg ral +ere rs +ud ed +t ax +Ġre iter +ĠMot ors +oci ated +Ġwond ers +ĠAp ost +uck ing +ĠRoose velt +f ram +Ġyield s +Ġconstit utes +aw k +Int erest +Ġinter im +Ġbreak through +ĠC her +Ġpro sec +ĠD j +ĠM T +Res p +ĠP T +Ġs perm +ed it +B T +Lin ux +count ry +le ague +Ġd ick +Ġo ct +Ġinsert ing +Ġsc ra +ĠBrew ing +Ġ19 66 +Ġrun ners +Ġpl un +id y +ĠD ian +Ġdys function +Ġex clusion +Ġdis gr +Ġincorpor ate +Ġrecon c +Ġnom inated +ĠAr cher +d raw +achel or +Ġwrit ings +Ġshall ow +Ġh ast +ĠB MW +ĠR S +Ġth igh +Ġ19 63 +Ġl amb +Ġfav ored +ag le +Ġcool er +ĠH ours +ĠG U +ĠOrig in +Ġglim pse +---------------- ---- +L im +Ġche ek +Ġj ealous +- ' +Ġhar ness +ĠPo ison +Ġdis abilities +ne apolis +Ġout look +Ġnot ify +ĠIndian apolis +Ġab rupt +ns ic +Ġenc rypted +Ġfor fe +reat h +Ġr abb +Ġfound ations +Ġcompl iment +ĠInter view +ĠS we +Ġad olesc +Ġmon itors +ĠSacrament o +Ġtime ly +Ġcontem pl +Ġposition ed +Ġpost ers +ph ies +iov ascular +v oid +ĠFif th +Ġinvestig ative +OU N +Ġinteg rate +ĠIN C +ish a +ibl ings +ĠRe quest +ĠRodrig uez +Ġsl ides +ĠD X +Ġfemin ism +Ġdat as +Ġb end +ir us +ĠNig eria +F ox +Ch ange +Ġair plane +ĠLad en +Ġpublic ity +ixt y +Ġcommit ments +Ġaggreg ate +Ġdisplay ing +ĠAr row +Ġ12 2 +Ġrespect s +and roid +s ix +ĠSh a +Ġrest oration +) \ +W S +oy s +Ġillust rate +with out +12 6 +ĠâĶ Ĥ +Ġpick up +n els +Ġ .... +f ood +ĠF en +) ? +Ġphenomen a +Ġcompan ions +ĠW rite +Ġsp ill +Ġbr idges +ĠUp dated +ĠF o +Ġinsect s +ASH INGTON +Ġsc are +il tr +ĠZh ang +Ġsever ity +Ġind ul +14 9 +ĠCo ffee +Ġnorm s +Ġp ulse +ĠF T +Ġhorr ific +ĠDest roy +ĠJ SON +Ġo live +Ġdiscuss es +R est +E lect +ĠW inn +ĠSurv iv +ĠH ait +S ure +op ed +Ġro oted +ĠS ke +ĠBron ze +Ġl ol +Def ault +Ġcommod ity +red ited +Ġliber tarian +Ġforb idden +Ġgr an +à ¨ +Ġl ag +en z +dri ve +Ġmathemat ics +Ġw ires +Ġcrit ically +Ġcarb ohyd +ĠChance llor +ĠEd die +Ġban ning +ĠF ri +Ġcompl ications +et ric +ĠBangl adesh +Ġband width +St op +ĠOrig inally +Ġhalf way +yn asty +sh ine +Ġt ales +rit ies +av ier +Ġspin ning +ĠWH O +Ġneighbour hood +b ach +Ġcommer ce +ĠS le +B U +Ġentreprene ur +Ġpecul iar +ĠCom ments +f re +3 20 +IC S +Ġimag ery +ĠCan on +ĠElect ronic +sh ort +( ( +D ig +Ġcomm em +u ced +Ġincl ined +ĠSum mon +Ġcl iff +ĠMed iterranean +Ġpo etry +Ġprosper ity +ĠRe ce +Ġp ills +m ember +Ġfin ale +un c +ĠG ig +ä ½ +Ġl od +Ġback ward +- + +ĠFor ward +Ġth ri +s ure +Ġso ap +ĠF X +R ES +ĠSe xual +oul os +Ġfool ish +Ġright eous +Ġco ff +terror ism +ust ain +ot er +Ġab uses +ne xt +Ġab usive +Ġthere after +Ġprohib ition +ĠS UP +Ġd ip +Ġr ipped +Ġinher ited +Ġb ats +st ru +G T +Ġflaw ed +ph abet +Ġf og +do ors +Ġim aging +Ġdig its +ĠHung ary +Ġar rog +Ġteach ings +Ġprotocol s +ĠB anks +à ¸ +p ound +ĠC urt +." ) +. / +Ġex emption +end ix +ĠM ull +Ġimpro ves +ĠG amer +d imensional +I con +ĠMarg aret +St atus +d ates +Ġint ends +Ġdep ict +Ġpark ed +J oe +ĠMar ines +chn ology +! ). +Ġjud ged +Ġwe ights +R ay +Ġapart ments +he ster +Ġrein force +Ġoff ender +occ up +Ġs ore +e pt +ĠPH P +ĠB row +Ġauthor ization +ĠR isk +ĠDel aware +ĠQ U +Ġnot ifications +Ġsun light +Ġex clude +d at +Ġm esh +ĠSud an +Ġbelong ed +Ġsub way +Ġno on +ĠInter ior +ol ics +ĠL akers +Ġc oding +Dis claimer +Cal if +O ld +Ġdis l +???? ? +Ġconfir ms +Ġrecruit ment +Ġhom icide +Cons ider +ĠJeff rey +ft y +} ; +Ġobject ion +do ing +ĠLe o +W ant +Ġgl ow +ĠClar ke +ĠNorm an +Ġver ification +Ġpack et +ĠForm ula +Ġpl ag +es ville +Ġshout ing +Ġo v +ĠR EC +ĠB ub +Ġn inth +Ġener g +Ġvalid ity +Ġup s +j ack +Ġneighbor ing +ĠN ec +ew orks +ĠH ab +are z +Ġsp ine +Ġevent ual +ĠLe aders +ĠC arn +Ġprob ation +Ġrom ance +ms g +ĠMechan ical +ER Y +R ock +Ġpart isan +N ode +ass ets +min ent +Ġforeign ers +Ġtest ify +ĠUs ually +l ords +ĠG ren +ĠPow ell +BI L +Ġs r +Ġadd ict +Ġshell s +Ġs igh +ĠY ale +tern ity +Ġ7 50 +E U +ĠR ifle +Ġpat ron +em a +ĠB annon +an ity +Ġtrop ical +ĠV II +c ross +Every thing +ĠIS O +Ġhum ble +ass ing +ĠF IG +Ġupd ating +ys on +Ġcal cium +Ġcompet ent +Ġste ering +Pro t +ĠS Y +ĠFin als +ĠR ug +15 9 +13 7 +ĠG olf +Ġ12 6 +Ġaccommod ation +ĠHug hes +Ġaest hetic +art isan +ĠTw ilight +Ġpr ince +ĠAgric ulture +ĠDis co +Ġpreced ent +Ġtyp ing +author ized +O ption +ĠA ub +l ishes +ach t +m ag +P eter +ĠU FO +mont on +ĠL ith +Ġa rom +Ġsec uring +Ġconf ined +priv ate +Ġsw ords +Ġmark ers +Ġmetab olic +se lect +ĠCur se +ĠO t +g ressive +Ġinc umb +ĠS aga +Ġpr iced +Ġclear ance +Cont ent +Ġdr illing +Ġnot ices +Ġb ourgeois +Ġv est +Ġcook ie +ĠGuard ians +ry s +in yl +Ġ12 4 +Ġpl ausible +on gh +ĠOd in +Ġconcept ion +ĠY uk +ĠBaghd ad +ĠFl ag +Aust ral +ĠI BM +Ġintern ationally +ĠWiki Leaks +I ED +Ġc yn +Ġcho oses +ĠP ill +Ġcomb ining +Ġrad i +ĠMoh ammed +def ense +atch ing +Sub ject +ic iency +Fr ame +Ġ{ " +Ġche ss +Ġtim er +19 0 +Ġt in +Ġord inance +emet ery +Ġacc using +Ġnotice able +Ġcent res +Ġl id +ĠM ills +img ur +Ġz oom +erg ic +Ġcomp ression +pr im +f ind +Ġsur g +Ġp and +ĠK ee +ĠCh ad +cell ence +oy le +Ġsocial ism +ĠT ravis +ĠM Hz +Ġgu ild +ALL Y +ĠSub scribe +ĠRel ated +Ġoccur rence +itch ing +Ġfict ional +Ġcr ush +ĠE A +c od +m ix +ĠTri ple +Ġretrie ve +Ġstimul us +Ġpsych iat +ĠDo or +Ġhomosexual ity +Ġelement ary +Ġcell ular +id ian +ĠL aun +Ġintrig uing +Ġfo am +ĠB ass +id i +its u +Ġass ure +Ġcongr at +Ġbusiness man +ĠBo ost +cl ose +Ġl ied +Ġsc iences +ĠO mega +ĠG raphics +Ġ< = +sp oken +Ġconnect ivity +S aturday +ĠAven gers +Ġto ggle +Ġank le +Ġnational ist +mod el +ĠP ool +ophob ia +V ar +ĠM ons +ator ies +Ġaggress ively +C lear +For ge +act ers +Ġhed ge +Ġpip es +Ġbl unt +Ġs q +Ġremote ly +W ed +as ers +Ġref riger +Ġt iles +Ġresc ued +Ġcompr ised +ins ky +Ġman if +avan augh +Ġprol ifer +Ġal igned +x ml +Ġtri v +Ġcoord ination +ĠP ER +ĠQu ote +13 4 +b f +ĠS aw +Ġtermin ation +Ġ19 0 +Ġadd itions +Ġtri o +Ġproject ions +Ġpositive ly +Ġin clusive +Ġmem br +19 90 +old er +Ġpract iced +ink le +Ar ch +Ġstar ters +ari us +Ġinter mediate +ĠBen ef +ĠK iller +Ġinter ventions +ĠK il +ĠF lying +In v +Ġprem ature +Ġpsych iatric +Ġind ie +Ġcoll ar +ĠRain bow +af i +Ġdis ruption +ĠFO X +cast ing +Ġmis dem +c ro +Ġw ipe +ard on +Ġb ast +ĠTom my +ĠRepresent ative +Ġbell y +ĠP O +ĠBre itbart +13 2 +Ġmess aging +Sh ould +Ref erences +ĠG RE +ist ical +L P +ĠC av +ĠC razy +Ġintu itive +ke eping +ĠM oss +Ġdiscont in +ĠMod ule +Ġun related +ĠPract ice +ĠTrans port +Ġstatist ically +orn s +Ġs ized +p u +Ġca f +ĠWorld s +ĠRod gers +ĠL un +ĠCom ic +l iving +Ġc ared +Ġclim bed +) { +Ġconsist ed +Ġmed ieval +fol k +Ġh acked +Ġd ire +ĠHerm ione +Ġt ended +ce ans +D aniel +w ent +Ġlegisl ators +Ġred es +g ames +Ġg n +am iliar +Ġ+ + +gg y +th reat +Ġmag net +Ġper ceive +Ġz ip +Ġindict ment +Ġcrit ique +g ard +ĠSaf e +ĠC ream +Ġad vent +ob a +Ġv owed +ous ands +Ġsk i +Ġabort ions +u art +Ġstun ned +Ġadv ancing +Ġlack ed +Ġ\ " +Ġsch izophren +Ġeleg ant +Ġconf erences +Ġcance led +ĠHud son +ĠHop efully +Ġtr ump +Ġfrequ encies +Ġmet eor +ĠJun ior +ĠFle et +ĠMal colm +ĠT ools +Ġ ........ +Ġh obby +ĠEurope ans +Ġ15 00 +ĠInt o +Ġs way +ĠApp ro +ĠCom pl +Comm unity +Ġt ide +ĠSum mit +ä » +Ġinter vals +ĠE ther +Ġhabit at +ĠSteven s +lish ing +ĠDom ain +Ġtrig gers +Ġch asing +Ġchar m +ĠFl ower +it ored +Ġbless ing +Ġtext ures +F ive +Ġliqu or +R P +F IN +Ġ19 62 +C AR +Un known +Ġres il +ĠL ily +Ġabund ance +Ġpredict able +r ar +Ġbull shit +le en +che t +M or +M uch +ä ¹ +Ġemphas ized +Ġcr ust +Ġprim itive +Ġenjoy able +ĠPict ures +Ġteam mate +pl er +ĠT ol +ĠK ane +Ġsummon ed +th y +ram a +ĠH onda +Ġreal izing +Ġquick er +Ġconcent rate +cle ar +Ġ2 10 +ĠErd ogan +ar is +Ġrespond s +ĠB I +Ġelig ibility +Ġpus hes +ĠId aho +Ġagg rav +Ġru ins +ur ations +Ġb ans +Ġan at +sh are +Ġgr ind +h in +um en +Ġut ilities +ĠYan kees +Ġdat abases +ĠD D +Ġdispl aced +Ġdepend encies +Ġstim ulation +h un +h ouses +ĠP retty +ĠRaven s +ĠTOD AY +Ġassoci ates +Ġthe rape +cl ed +Ġde er +Ġrep airs +rent ice +Ġrecept ors +Ġrem ed +ĠC e +Ġmar riages +Ġball ots +ĠSold ier +Ġhilar ious +op l +13 8 +Ġinherent ly +Ġignor ant +Ġb ounce +ĠE aster +REL ATED +ĠCur rency +E V +ãĥ ŀ +ĠLe ad +Ġdece ased +B rien +ĠMus k +J S +Ġmer ge +heart ed +c reat +m itt +m und +ĠâĢ ĭ +ĠB ag +Ġproject ion +Ġj ava +ĠStand ards +ĠLeon ard +Ġcoc onut +ĠPop ulation +Ġtra ject +Ġimp ly +Ġcur iosity +ĠD B +ĠF resh +ĠP or +Ġheav ier +ne ys +gom ery +Ġdes erved +Ġphr ases +ĠG C +Ġye ast +d esc +De ath +Ġreb oot +Ġmet adata +IC AL +Ġrep ay +ĠInd ependence +Ġsubur ban +ical s +Ġat op +Ġall ocation +gener ation +ĠG ram +Ġmoist ure +Ġp ine +ĠLiber als +Ġa ides +Ġund erest +ĠBer ry +Ġcere mon +3 70 +ast rous +ĠPir ates +Ġt ense +ĠIndust ries +ĠApp eals +ĠN ear +Ġè£ı ç +Ġlo vers +ĠC AP +ĠC raw +Ġg iants +Ġeffic acy +E lement +ĠBeh avior +ĠToy ota +Ġint est +P riv +A I +Ġmaneu ver +Ġperfect ion +Ġb ang +p aper +r ill +Ge orge +b order +in ters +ĠS eth +Ġcl ues +ĠLe vi +ĠRe venue +14 7 +Ġv apor +Ġfortun ate +Ġthreat ens +Ġve t +Ġdepend ency +ers ed +art icle +ĠBl izzard +Ġch lor +Ġmin us +ĠB ills +Ġcryptoc urrency +Ġmetabol ism +ter ing +Ġp estic +step s +ĠTre asure +ract ed +ĠConst ant +Ġtem p +13 9 +ĠDet ective +ur ally +Ġrecover ing +Ġcort ex +Ġ14 4 +cl osed +Ġprejud ice +aun ted +Ġstorm s +ĠN OW +Ġmach inery +Add ress +Ġcompe lled +27 0 +Ġdesp air +b ane +Ġveget able +Ġbed s +Lear n +Ġcolor ful +Ġsp ike +Ġmarg ins +Ġsymp athy +Ġworks hop +ĠC BC +S at +Ġburn s +ĠG ender +Ġ12 9 +ĠC able +Ġdeb ts +ĠThe resa +Ġreflect ing +Ġa irst +Ġr im +ram id +Ġweakness es +W rit +ogg le +t i +ĠCh arge +Ġwe ighed +Ġ( . +Ġl aughter +Ġrou ter +ĠDemocr acy +D ear +Ġhas ht +Ġd y +Ġhint s +run ning +Ġfin ishes +ar us +M ass +res ult +asc us +Ġv intage +Ġcon qu +Ġwild ly +ac ist +Ġl ingu +Ġprot agonist +st rom +te enth +ĠSol o +m ac +f illed +Ġre nown +it ives +Ġmot ive +ĠAnt ar +ĠM ann +ĠAd just +Ġrock ets +Ġtrou bling +e i +Ġorgan isms +ass is +Christ ian +Ġ14 5 +ĠH ass +Ġsw all +Ġw ax +ĠSurv ival +V S +ĠM urd +v d +stand ard +Ġdrag ons +Ġacceler ation +r ational +f inal +Ġp aired +ĠE thereum +Ġinterf aces +Ġres ent +Ġartif acts +Å « +are l +Ġcompet itor +ĠNich olas +ĠSur face +c pp +ĠT ot +Ġeconom ically +Ġorgan ised +Ġen forced +in ho +Ġvar ieties +Ġab dom +ĠBa iley +id av +ĠSal v +p aid +Ġalt itude +ess ert +ĠG utenberg +are a +op oulos +Ġprofess ors +igg s +ĠF ate +he y +Ġ3 000 +D ist +Ġtw ins +c ill +ĠM aps +Ġtra ps +Ġwe ed +ĠK iss +Ġy oga +Ġrecip ients +ĠWest minster +Ġpool s +ĠWal mart +18 8 +ĠSchool s +att ack +ĠAR M +par agraph +W arning +j l +Ġself ish +anche z +ĠHe ights +F re +ĠS oph +Ġ -------------------------------- +t ml +33 3 +Ġraid s +Ġsatell ites +KE Y +Ġlast s +Ñ Ĥ +In s +ĠD ame +Ġunp redict +// / +gh ai +Ġart illery +Ġcru ise +Ġg el +ĠCabin et +Ġbl ows +ĠE sp +Ġprox imity +ot he +ĠSk ills +ĠU pper +ob o +ĠN DP +Ġenjoy s +Ġrepe ating +ĠConst ruction +ĠQuest ions +H illary +Ġu int +Ġprocess ors +ĠGib son +ĠMult iple +q a +ĠB om +ĠM iles +vent ional +Ġhur ts +s kin +ĠA IDS +Ġadvis ers +ĠR oot +Ġmethod ology +ĠD ale +Ġdet on +ĠKnow ledge +sequ ently +Ġ12 1 +Ġconnect s +C y +ĠD anger +Ġcontribut ors +ĠB ent +Ġbr ass +ĠGun s +int o +ĠFort une +Ġbro ker +bal ance +Ġlength s +Ġv ic +Ġaver aging +Ġappropri ately +ĠCamer a +Ġsand wich +ĠCD C +Ġcoord inate +Ġnav ig +Ġgood ness +l aim +Ġbra ke +Ġextrem ist +ĠW ake +ĠM end +ĠT iny +ĠC OL +ĠR F +ĠD ual +ĠW ine +C ase +Ġref ined +Ġl amp +L ead +Ġb apt +ĠCar b +ĠS add +ĠMin neapolis +PD F +Ear ly +ĠH idden +I ts +ĠT IME +Ġp ap +Ġcommission ed +ĠF ew +ĠCol ts +ĠB ren +Ġbot hered +Ġlike wise +Ex per +ĠSch w +c ry +n n +ĠM itch +im on +M G +b m +UM P +r ays +Ġregist ry +Ġ2 70 +ach ine +re lla +ant ing +00 000 +Ġru ined +sp ot +Ġt a +Ġmaxim ize +Ġincon ven +D ead +H uman +En abled +ĠMar ie +Ġch ill +ĠParad ise +Ġstar ring +ĠLat ino +ĠProt ocol +ĠE VER +Ġsuppl iers +m essage +ĠBro ck +Ġser um +âĸĪâĸĪ âĸĪâĸĪ +Ġen comp +Ġamb ition +ues e +Ġar rows +And rew +Ġanten na +Ġ19 61 +ĠB ark +Ġb ool +ãĤ ª +ĠSt orage +Ġrail way +Ġtoug her +ĠC ad +Ġwas hing +P y +' ] +em bed +ĠMem phis +ack le +Ġfam ously +ĠF ortunately +ov ies +Ġmind set +Ġsne ak +ĠD h +RA W +ĠSim pson +Ġliv est +Ġland mark +Ġc ement +L ow +Ġthr illed +ĠCour se +in el +Ġch uck +id ate +gl obal +Ġwh it +Ġ � +ad ays +s ki +ĠS V +Ġvir uses +30 6 +ĠResp ons +Ġthe aters +ĠBr anch +ĠGene va +ĠM K +Ġunbel iev +Ġcommun ist +Orig inal +ĠRe ceived +ĠTrans fer +ĠAr g +In put +ĠStr ategy +Ġpal ace +the ning +D ri +Ġsent encing +umbn ail +Ġp ins +re cy +Ġs iblings +Get ting +ĠB U +ĠNorth west +Ġprolong ed +ĠSak ura +C omb +ĠB our +Ġinadequ ate +ĠK ash +Ġus ername +ĠImpro ve +Ġbatt ling +ĠM AC +Ġcurric ulum +Ġs oda +ĠC annon +Ġsens ible +sp ons +De cember +Ġw icked +ĠP engu +Ġdict ators +ĠHe arts +og yn +Ġsimilar ities +ĠSt ats +Ġh ollow +it ations +": [ +Ġh over +ĠList en +s ch +S und +Ġc ad +ĠPar ks +Ġl ur +Ġhy pe +ĠL em +N AME +is ure +Fr iday +Ġshoot s +Ġclos es +Ġd b +ĠR idge +ĠDiff erent +Ġrepl ies +ĠBroad way +op ers +Ġint oler +ĠZe us +akes pe +Ġpropri etary +Ġrequest ing +Ġcontro llers +ĠM IN +im edia +be cca +Ġexp ans +Ġoil s +B ot +ĠCh and +Ġpr inter +Ġto pped +ĠP OL +ĠEar lier +S ocial +av in +Ġdecre ases +ĠSe b +Ġspecific ations +ĠBl ast +ĠK urt +Ġfre el +B rown +Ġdil ig +ro e +ĠPro blem +ĠQu ad +Ġdecent ral +ĠV ector +an ut +Ġplug ins +ĠGreg ory +Ġfuck ed +el ines +ĠAmb assador +t ake +Ġcle ans +ong yang +An onymous +st ro +" } +al ine +ĠO dd +ĠE ug +2 16 +Ġbo il +ĠP owers +Ġnurs es +Ob viously +ĠTechn ical +Ġexceed ed +OR S +Ġextrem ists +Ġtr aces +ex pl +Ġcom r +ĠS ach +) / +Ġm asks +Ġsc i +B on +Ġreg ression +we gian +Ġadvis or +it ures +ĠV o +ex ample +ĠInst ruct +Ġs iege +Ġredu ctions +pt r +Ġstat utory +Ġrem oves +Ġp uck +red its +Ġbe e +Ġsal ad +Ġpromot ions +ĠJosh ua +with standing +ET H +ĠCh a +im us +Ġexpend iture +aun ting +Ġdelight ed +Ġ15 5 +be h +Ġcar pet +ĠSp art +Ġj ungle +l ists +Ġbull ying +ĠNob el +ĠGl en +Ġreferen ced +Ġintrodu ces +se in +Ġcho pped +gl ass +ĠW rest +Ġneutral ity +Ġâ Ļ +Ġinvestig ator +Ġshel ves +Ġun constitutional +Ġreprodu ction +Ġmer chant +m ia +Ġmet rics +Ġexplos ives +ĠSon ia +Ġbod ily +Ġthick ness +Ġpredomin antly +ĠAb ility +Ġmon itored +IC H +Ġ] . +ĠMart inez +Ġvis ibility +Ġqu eries +Ġgen ocide +ĠWar fare +Qu ery +Ġstud ios +Ġemb ry +Ġcorrid or +Ġclean ed +com plete +ĠM H +Ġenroll ment +ING S +Ġimpact ed +Ġdis astrous +ĠY un +ĠCl aire +ĠBas ically +y t +uster ity +Ġindirect ly +w ik +Ġd od +ĠCar r +Ġam p +Ġprohib it +ĠIn itial +ĠR d +ij i +Ġeduc ate +c orn +i ott +ĠBeaut y +Ġdetect ive +ĠCon n +s ince +Ġst agger +Ġob ese +Ġb ree +olog ic +is se +walk er +Ġbl ades +Ġlaw ful +fun c +ĠBeh ind +Ġappet ite +Ġ( * +Ġt ennis +Ġoff spring +Ġj ets +Ġstruct ured +Ġafore mentioned +N ov +Ġsc aling +f ill +Ġst ew +Ġcur b +ĠStep han +ed In +S F +ob ic +é ŃĶ +ou g +ĠM M +Ġgen etically +ope z +13 6 +Ġu mb +anc ers +Ġcoh ort +Ġmerch andise +Ġimp osing +ĠLegisl ature +ĠArch ive +iv ia +ĠN aval +Ġoff ences +Ġmir acle +Ġsn apped +Ġf oes +Ġextensive ly +ĠR af +Ġc ater +ed ience +K it +ĠB in +Ġrecomm ends +ĠC ities +Ġrig id +ĠRE AD +ĠNob le +ĠT ian +Ġcertific ates +ant is +o iler +ĠBudd hist +d id +Ġsurvey ed +Ġdown ward +Ġprint s +ĠMot ion +ron ics +ĠS ans +oss ibly +u ctions +Ġcolon ies +ĠDan ish +un it +Ġsp oil +Ġadvis ory +ber ries +Pl an +Ġspecific ation +op hers +ĠRes ource +Ġsh irts +prising ly +commun ications +Ġtriv ial +Ġmention ing +ise xual +Ġsupp lements +Ġsuper vision +B P +v or +Ġw it +Ġco oldown +Ġplaint iff +ĠReview s +ĠS ri +ĠM int +ĠSug ar +Ġafter ward +ĠPri est +ĠInvest ment +og ene +ĠT aking +Ġstretch ing +Ġinflamm ation +ĠTe hran +Ġl ining +Ġfree zing +ĠEnt ity +Ġins piring +spe cial +pr ice +Ġsu e +ĠP orter +oun ge +ET A +ĠD erek +ĠLu is +u o +ym ph +Ġex terior +ih il +ĠAsh ley +in ator +Ġnut rients +ĠTh rones +Ġfin ances +ĠIn spect +Ġspe cially +ĠRequ ired +ĠP TS +ĠViol ence +oint ed +sh ots +Ġex cerpt +co on +IN S +ĠG ri +Ġrecogn ised +We ek +You ng +Ġv om +is le +ĠCur ry +ĠBudd h +Ġnot ebook +Ġd urable +/ ? +ĠG ad +ĠP upp +Ġforg ive +p ark +Ġpersonal ities +an alysis +cl amation +Ġelev ator +Ġware house +ĠR ole +un n +Ġillust ration +ĠSc an +Ġatmosp heric +Im port +AN C +rict ed +f u +01 0 +Ġar che +Ġreward ed +akespe are +Ġintern ally +ĠR BI +alk er +Ġeleph ant +ow itz +ĠP izza +Ġbip artisan +é s +Ġslow ed +ĠSt ark +Ġover ride +OU S +Ġ3 20 +undred s +ĠDe ck +ĠC ensus +be e +14 6 +ot or +Ġ ip +Ġu b +oc ations +ĠBut ton +r ice +Ġc ripp +ff f +Ġorig inated +Ġoverwhel med +app a +Ġfore most +âĢ ij +ĠL EG +re lease +eat ured +at ches +Ġre ps +Ġl ending +ĠRe ference +ĠCl ient +16 5 +vent h +Com plete +ĠPat rol +Ġsw orn +c am +Ġshut tle +ĠR alph +Ġh ometown +- , +on al +ĠB P +å ı +Ġpersu ade +ĠAlex and +Ġcomb ines +Ġv ivid +ĠL ag +Ġenc oding +Ġsal vation +w en +ĠRec overy +i ya +Un iversity +ĠB iden +Ġbud gets +ĠTex ans +f its +Ġhon ored +Ġp ython +T D +## # +cl one +Ġbl ink +ĠL iquid +Ġunemploy ed +Ġcl ashes +ĠCoun sel +Ġdirect ing +Ġpun ct +ĠFal cons +Ġsh ark +ĠDam ascus +Ġje ans +Ġemb ark +Ġse ize +Ġup wards +2 80 +ĠE z +ĠAny thing +Ġex otic +l ower +ĠCreat or +ĠU m +Ġsubur bs +ber ger +ĠW end +Ġm int +ĠX X +ĠD ro +Ġsuff ers +Ġher b +t ree +Ġfrag ile +Ġflood ed +ĠAl cohol +ole an +ny der +ĠK O +F ram +Ġ13 6 +Ġow ed +ĠMe lee +ĠH ash +Ġwh isk +Ġsu do +r r +Qu ick +app ro +Ġi i +ĠEx amples +he e +Ġpromot es +per ature +k ar +ĠHon or +Ġs odium +ĠL if +ros so +intend ent +Ġcorrespond ent +F ound +sec ret +Ġident ifies +ag ne +Ġl ou +ĠP P +Ġcoinc idence +m ove +Ġmilit ia +Ġinf iltr +ĠPrim ary +Ġpitch ing +ĠI b +ĠGO OD +ãĤ ¸ +ĠW izards +ir al +ĠVen us +R R +ĠâĢ ķ +ĠCase y +Ġsad ly +Ġadm ire +Ġembarrass ed +c b +M el +Ġtub es +Ġbeaut ifully +ĠQueens land +Bel ow +re z +qu et +ple asant +Ġ « +C amp +Ġdec isive +19 98 +ĠL amb +ut ton +h n +ĠJ agu +au nder +ĠC ord +Ġcl erk +Ġca ffe +Ġwip ed +Ġre im +ĠMount ains +Ġimprison ed +Ġdevelop s +ĠP ra +Ġmodel ing +Any one +ance l +ĠS it +Ġshield s +Ġl awn +Ġcard iovascular +Ġdemonstr ating +Ġpar se +ĠIsrael is +Ġeuro s +14 3 +Ġgl orious +ins ki +ec d +Ġcondition ing +Ġhel pless +Ġmicro sc +ĠHar bor +Ġst akes +Ġ2 60 +Ġun equ +ĠFl oyd +Ġd amp +Ġappar atus +ĠLaw s +Ġcoun ters +Ġindu ce +at able +ĠAh med +Ġsl am +N ovember +Ġpers ist +Ġim minent +á n +Ġsh red +Ġph ases +ĠEd monton +ĠArm strong +ĠMe et +ĠK itty +Ñ Ģ +c irc +ĠAd ult +Ġa rose +ĠX en +D an +g ow +Ġsuper f +ĠAd mir +Ġend ure +Ġkey word +yr us +Ġy arn +Ġpath way +ĠHop kins +mid t +Ġcens orship +d ependent +Ġinstruct or +S ources +Ġto e +Ġball oon +N ob +Ġsw ear +ĠCast ro +Ġgl oss +ĠK avanaugh +Ġremark ably +Ph otos +ĠN om +ĠS outheast +y ers +Ġvalid ation +Ġcann on +ĠVict ory +ĠPier re +Ġcaut ious +Aud io +Ġf etch +ĠG ift +ĠH yp +Ġrem edy +Z E +Ġsc ent +Ġbe ard +ĠR ut +- " +Ġpat ents +H y +Ġun just +Ġpot ato +Ġforth coming +Ġche f +ĠR ift +aff e +ĠR OM +ĠL aunch +Ġp ads +ĠNe o +Ġon set +Ġsquee ze +s afe +Ġpref ix +ĠT M +ĠN early +ĠClin ical +ĠM ental +ot iation +ĠUn ic +ant ry +ĠC ir +Ġep it +à ¦ +Ġextract ed +verse ly +ri ad +Ġstr ains +Ġto ps +Ġpo em +ĠRand y +ĠMap le +TH ER +up iter +ĠSS D +ļ é +Ġun con +per ing +Ġsle pt +in ers +Ġunder water +ĠEv idence +g one +20 5 +Ġhistor ians +Ġsynt hesis +Ġf rog +b asketball +Ġvibr ant +Ġsub ord +Ġ3 65 +ĠD ial +Ġcooper ate +HA HA +Ġgreet ed +15 8 +Ġj azz +Ġinto x +ĠWalk ing +Ġsuper visor +ĠF usion +ĠMer cedes +s end +H am +s d +n l +Ġtour s +ĠF IFA +Ġcul p +g d +30 4 +Ġple as +Ġillust rates +ĠColomb ia +Ġhighlight ing +ĠSum mary +Ġexp osing +ĠD ru +Ġir ony +r itional +ĠCar roll +ĠEll is +P ict +ĠR apt +Ġad apter +Ġun m +Ġcor pse +Ġceleb rities +D en +at um +ĠAp ocalypse +ĠW ag +lin ing +Ġhorm ones +R ub +ĠX i +ĠV aults +20 8 +alky rie +inos aur +Ġfeed s +v ity +Ġdefe ating +W ait +Ġemphas ize +ĠSteel ers +yr inth +le ys +ĠWhe never +Current ly +ĠCl ock +Ġcollect ively +any on +ĠJ P +Ġment ality +Ġdownload s +Ġsurround ings +ĠBarn es +Ġflags hip +Ġindic ators +Ġgra pp +Jan uary +ĠElement al +ĠAthen a +ib al +Ġs ights +Ġcap ita +ĠTreat y +Ġvo iced +ĠG az +let te +Ġy a +Ġexp ired +Leg end +H ot +n ature +Ġunst able +Ġ2 80 +à º +Com ment +AL E +Ġquest s +Ġhand ler +n is +Ġvers atile +Ġconce al +enge ance +ĠInter active +Ġobs essed +ĠDog s +Ġcr acked +S ound +s v +ĠD ylan +ro ads +f x +ĠCath olics +ĠH ag +Ġsl ammed +Ġgl owing +s ale +Ġtiss ues +ĠCh i +ne e +Ġc her +s ic +ur rection +Ġb acon +ul atory +) ." +Ġir regular +FOR M +ass ed +Ġintention al +Ġcompens ate +ĠSpe aking +ĠS ets +15 3 +Ġconvent ions +b ands +em ade +Ġe cc +ĠWin ston +ĠAssass in +ĠBelg ian +Ġdepend ence +Ġnic he +Ġb ark +ĠJ azz +Ġdisadvant age +Ġgas oline +Ġ16 5 +çļ Ħ +ess a +mod ule +ang ular +O Y +ĠTreat ment +it as +ol ation +ĠArn old +Ġfe ud +ĠN est +Ġthe atre +ew ater +Ġmin ors +olic y +ĠH aven +div ision +Ġtr unk +F ar +ĠP ull +Ġcapt uring +Ġ18 00 +ĠTe en +Ġex empl +Ġclin ics +ĠB urg +Ġsubst it +Ġpay load +ĠL av +ĠT roy +ĠW itness +Ġfrag ments +Ġpass words +Ġg ospel +ĠG in +Ġten ants +ol ith +S ix +Pre vious +ĠAg es +ĠDar win +Ġbl at +Ġem pathy +sm ith +b ag +ĠE cho +ĠC amb +ĠM add +ĠB oo +Ġred e +ĠBurn ing +Ġsmooth ly +ĠAd rian +ĠV ampire +ĠMon sters +ste am +Sty le +M a +re a +ĠD war +aly st +urs or +Ġelim ination +Ġcrypt o +ch t +ĠE ternal +âĢ¦ ] +ĠS orce +I ll +N ER +Ġu h +Con clusion +w age +Ġresp ir +Ġrem inis +het ical +Ġg y +Ġutil ized +ic idal +Ġ19 00 +Ġhun ters +ĠSw an +ĠRe act +Ġvis itor +ĠThanks giving +30 8 +Post s +Ġh ips +19 97 +om ers +Ġkn ocking +ĠVeh icle +Ġt il +Ġ13 8 +Ġm i +ĠInvest igation +ĠKen ya +Ġcas ino +Ġmot ives +Ġreg ain +re x +Ġweek ends +Ġstab bed +bor o +Ġexplo ited +ĠHA VE +ĠTe levision +c ock +Ġprepar ations +Ġende av +ĠRem ote +ĠM aker +ĠPro du +ĠEv an +Ġinform ational +ĠLouis ville +15 4 +ĠDream s +Ġpl ots +ĠRun ner +Ġhur ting +Ġacad emy +ĠMont gomery +n m +ĠL anc +ĠAl z +2 10 +el ong +Ġretail er +Ġar ising +Ġrebell ion +Ġbl onde +play ed +Ġinstrument al +C ross +Ġret ention +Ġtherape utic +Ġse as +Ġinfant ry +ĠCl int +Ġprompt ing +Ġbit ch +Ġst ems +ĠK ra +Ġthe sis +ĠB og +ru ed +Ġk ings +Ġcl ay +ific ent +ĠY ES +ĠTh ing +ĠCub s +vey ard +els h +in arily +ĠE y +ĠRoll ing +Ġev olving +Ind ia +Ġrecogn izes +Ġgrad uation +is ers +Ġfert ility +ĠMil an +Comm and +Ġbox ing +Ġ19 43 +Ġgl uten +ĠEm ir +Ġid ol +Ġcon ceived +ĠCre ation +Mer it +udd y +uss ions +ĠLie utenant +iet al +Ġunch anged +ĠSc ale +ĠCrime a +ball s +ator ial +Ġdepth s +Ġempir ical +Ġtrans m +Ġuns afe +miss ible +com fort +15 6 +Ġmechan ic +00 2 +l ins +Ġsm oked +P os +Ġslow ing +Ġl av +Tex as +Ġche ating +ĠMet ropolitan +eth yl +Ġdiscover ing +as se +Ġpen cil +ĠPy ongyang +Ġclos et +ĠShe et +ĠEnt ry +ou stic +Ġmy st +er ate +ari at +Ġminer als +Ġmusic ian +ĠP ul +ĠM az +24 9 +Ġper missions +Ġ iv +en ary +ick ers +ĠB ing +he a +en able +Ġgri ev +Ġassert ed +ĠColon el +Ġaff idav +w o +Ġse ated +ĠR ide +Ġpaint ings +ĠP ix +Ġ13 7 +ish i +umb ai +g otten +ĠEar l +Ġin ning +Ġc ensus +Ġtrave lled +ĠCons ult +18 5 +b ind +Ġsimpl icity +Ġoverlook ed +ĠHelp ful +Ġmon key +Ġoverwhelming ly +Bl ood +ĠFl int +ĠJ ama +ĠPres ent +ĠR age +ĠT A +pt ive +Ġturn out +w ald +ĠD olphins +ĠV PN +Ġon ion +Ġcraft ing +m ma +ĠMerc ury +Ġarr ange +Ġalert s +ĠO T +zb ollah +Ġg ases +ĠRichards on +s al +l ar +Ġfro st +Ġlower ing +Ġacc laim +Ġstart ups +ĠG ain +ess ment +Ġguard ian +äº º +ĠP ie +ĠL inks +Ġmer its +Ġaw ake +Ġparent al +Ġexceed s +Ġid le +ĠPil ot +Ġe Bay +ĠAc cept +ipe g +C am +ĠK ot +Ġtrad ers +olit ics +unk er +ĠP ale +os i +an mar +Ġ19 47 +ĠF ell +est ial +it ating +G F +ĠS r +if ted +Ġconnect or +ĠB one +ill es +2 60 +h ma +Ġoverl ap +ĠGit Hub +Ġclean er +ĠBapt ist +ĠW AS +Ġlung s +Ñ ģ +ĠB UT +Ġc ite +Ġpit ched +reat ment +Ġtro phies +ĠN u +38 6 +ĠPr ide +Ġattend ees +[ ] +17 9 +Ġspat ial +Ġpri zes +ĠRel igion +Ġshow case +ĠC ategory +vid ia +T arget +Pro perty +? , +Ġf usion +p ie +ĠU CLA +Ġsound track +Ġprin cess +ĠC aval +sh ould +Ġlim bs +Back ground +Ġlone ly +Ġc ores +ĠT ail +she et +Ġ13 2 +R a +ãĤ « +ĠB olt +Ġbook ed +Ġadmin ister +Ġequ als +w y +Ġobserv ing +ĠBar on +ĠAd obe +Ġv irgin +ĠSocial ist +M ove +gh azi +ĠLind a +2 12 +Ġbre wing +Ġmerch ants +bur se +Ġdiv or +Ġmet als +ĠN er +Ġsum s +ĠEn emy +Ġen vision +Ġgrant ing +ĠH oney +ĠSk yrim +Ġsoc io +gr aded +Ġselect ive +W ASHINGTON +Ġ19 48 +ĠSir ius +ĠG ross +act ivity +ĠI van +Ġfur ious +BS D +ĠPre vious +Ġrespons ive +Ġchar itable +Ġle aning +ĠP ew +Ġviol ates +\\\\ \\\\ +ĠCom ing +w ire +Ġpo et +Ġres olutions +comm and +ĠPortug uese +Ġnick name +Ġde af +Feb ruary +Ġrecogn ise +Ġentire ty +Ġseason al +pl aced +ĠTe legraph +Ġmicro phone +our ing +Ġgr ains +Ġgovern ed +Ġpost p +ĠW aters +in ement +Ġund ocumented +ĠCom cast +Ġf ox +Ġassault s +re on +man y +ĠJen kins +ĠAny way +Ġassess ments +Ġdown s +ĠM ouse +Ġsuper b +k t +ĠD ow +Ġtax ation +4 01 +Ġsm iles +Ġundert aken +Ġex h +Ġenthusi astic +Ġtw ent +Ġgovernment al +Ġautonom y +ĠTechn ologies +ĠCh ain +Ġpreval ent +f b +Ġnic otine +og ram +j ob +Ġawa iting +ĠMen u +Ġdep uties +k ov +ish ops +But ton +ĠShan ghai +Ġdies el +ĠD uck +R yan +ĠPC s +N F +j ury +ent e +Ġinacc urate +edd y +Wh atever +Ġshow c +ĠN ad +od us +et r +Ġplaint iffs +ĠW OR +ĠAss ange +Ġpriv at +Ġpremium s +Ġt am +UR L +Ġel ites +ĠR anger +otten ham +ĠH off +ĠAt hens +Ġdefin ite +Ġs ighed +Ġeven ly +2 11 +ĠAm ber +ak ia +Ġmail ing +Ġcr ashing +ĠConfeder ate +ru gged +W al +ĠDep ths +Ġjuven ile +Ġreact or +Introdu ction +ĠDel uxe +19 95 +ĠS anchez +ĠM ead +iv able +: - +ĠPlan ning +ĠT rap +qu in +ĠProt ect +ve red +In formation +Ġkid ney +inn amon +l as +Ġpolic ing +Ġtoler ate +ĠQ i +Ġbi ased +F ort +ĠK i +s ave +Ġprivile ged +Ġbe asts +ĠGl as +ĠC inem +Ġcome back +Sund ay +Ġext inction +h ops +Ġtrans mit +Ġdoub les +ĠFl at +16 7 +Ġdis puted +Ġinjust ice +f oo +V ict +role um +ĠJul ie +Con text +ĠR arity +iss ue +Comp onent +Ġcounsel ing +an ne +d ark +Ġobject ions +u ilt +Ġg ast +Ġpl ac +Ġun used +ãĥ ĩ +ĠT rial +ĠJ as +hed ral +ob b +Ġtempor al +ĠPR O +ĠN W +ĠAnn iversary +L arge +Ġther m +Ġd avid +Ġsystem ic +ĠSh ir +m ut +ĠNe pt +add ress +Ġscan ning +Ġunderstand able +Ġcan vas +C at +ĠZ oo +Ġang els +L O +ĠStat ement +ĠS ig +ov able +ĠA way +sh aring +ocr ats +st ated +Ġweigh ing +N or +w ild +B ey +Ġaston ishing +ĠReyn olds +Ġop ener +Ġtrain er +Ġsurg ical +p n +Ġadjust ing +whe el +Ġf rown +erv ative +Ġsusp end +With in +te in +Ġobst acle +Ġliber ties +ym es +Ġur anium +ans om +an ol +ub a +ĠL oss +Ġa rous +ĠHend erson +W ow +s pl +c ur +ĠÂ Ń +Ġtheir s +Dam age +Ġdownload ing +Ġdisc ern +ĠSt o +ĠFl a +Ġh ath +ĠA j +Ġun pleasant +Europe an +exp ensive +Ġscreens hot +ĠU V +Ġall ied +ĠPers ian +Ġmonop oly +Ġat om +ĠReds kins +"> < +Ġcan cell +Ġcinem a +13 1 +f air +ĠAlf red +Ġd uck +arg s +22 3 +ĠIS I +Ġsign aling +in ar +Ġlaugh s +Ġfor wards +Ġreck less +Ġlisten ers +at ivity +Ġvast ly +n ant +L ess +ĠHun ting +ĠScient ific +IT ED +Ġkn ight +ĠH TC +us a +t mp +Ġr ude +ĠLegend ary +Ġar ises +B ad +ĠCl aim +pe g +Ġreal ities +Th ink +Ġ ° +Ġro de +Ġstri ve +Ġan ecd +Ġshort s +Ġhypot hes +Ġcoord inated +ĠGand hi +ĠF PS +R ED +Ġsuscept ible +Ġshr ink +ĠCh art +Hel p +Ġ ion +de ep +rib es +ĠK ai +ĠCustom er +Sum mary +Ġc ough +w ife +Ġl end +Ġposition ing +Ġlot tery +ĠC anyon +Ġf ade +Ġbron ze +ĠKenn y +Ġbo asts +ĠEnh anced +rec ord +Ġemer gence +Ġa kin +ĠB ert +it ous +âĸ ij +Ġst ip +Ġexch anged +om ore +als h +Ġreserv oir +Ġstand point +W M +Ġiniti ate +Ġdec ay +Ġbrew ery +Ġter ribly +Ġmort al +lev ard +Ġrev is +N I +el o +Ġconf ess +ĠMS NBC +Ġsub missions +Cont roller +Ġ20 2 +ĠR uth +} ); +ĠAz ure +Ġ ." +20 6 +ĠMarket ing +Ġl aund +ien cies +Ġrenown ed +ĠT rou +ĠN GO +ble ms +Ġterr ified +Ġwar ns +Ġper t +Ġuns ure +4 80 +ale z +ult z +ĠOut side +Ġst yl +ĠUnder ground +Ġp anc +Ġd ictionary +Ġf oe +rim inal +ĠNor wegian +Ġj ailed +Ġm aternal +é e +ĠLu cy +c op +Ch o +Ġuns igned +ĠZe lda +ĠIns ider +ĠContin ued +Ġ13 3 +ĠNar uto +ĠMajor ity +16 9 +ĠW o +ãĤ ĵ +Ġpast or +Ġinform al +Ð ½ +an throp +jo in +ãģ Ĺ +it ational +N P +ĠWrit ing +f n +ĠB ever +19 5 +Ġy elling +Ġdr astically +Ġe ject +Ġne ut +Ġth rive +ĠFre qu +ou x +Ġpossess es +ĠSen ators +ĠD ES +ĠSh akespeare +ĠFran co +ĠL B +uch i +Ġinc arn +Ġfound ers +F unction +Ġbright ness +ĠB T +Ġwh ale +ĠThe ater +m ass +ĠD oll +S omething +Ġecho ed +ĠHe x +c rit +af ia +Ġgodd ess +Ġele ven +ĠPre view +ĠAur ora +Ġ4 01 +uls ive +ĠLog an +in burgh +ĠCent ers +ĠON LY +ĠA id +Ġparad ox +Ġh urd +ĠL C +D ue +c ourt +Ġoff ended +Ġeval uating +ĠMatthew s +Ġto mb +Ġpay roll +Ġextra ction +ĠH ands +if i +Ġsuper natural +ĠCOM M +] = +dog s +Ġ5 12 +ĠMe eting +Rich ard +ĠMax imum +Ġide als +Th ings +m and +ĠReg ardless +Ġhum ili +b uffer +L ittle +ĠD ani +ĠN ak +Ġliber ation +ĠA be +ĠO L +Ġstuff ed +ac a +ind a +raph ic +Ġmos qu +Ġcampaign ing +Ġoccup y +S qu +r ina +ĠW el +ĠV S +Ġphys ic +Ġp uls +r int +oad ed +ET F +ĠArch ives +Ġven ues +h ner +ĠTur bo +Ġl ust +Ġappeal ed +que z +il ib +ĠTim othy +Ġo mn +d ro +Ġobs ession +ĠSav age +19 96 +Gl obal +J es +2 14 +Ġsl iding +Ġdisapp ro +ĠMag ical +Ġvolunt arily +g b +ane y +Ġprop het +ĠRe in +ĠJul ia +ĠW orth +aur us +Ġb ounds +ie u +)) ) +Ġcro re +ĠCitiz en +S ky +Ġcolumn ist +Ġseek ers +ond o +IS A +ĠL ength +Ġnost alg +Ġnew com +Ġdet rim +ent ric +3 75 +ĠG E +Ġaut op +Ġacadem ics +App Data +ĠS hen +Ġid iot +ĠTrans it +Ġteasp oon +W il +K O +ĠCom edy +> , +Ġpop ulated +W D +Ġp igs +ĠO culus +Ġsymp athetic +Ġmar athon +19 8 +Ġseiz ure +s ided +Ġd op +irt ual +L and +ĠFl oor +osa urs +... ] +Ġl os +Ġsubsid iary +E Y +ĠPart s +ĠSt ef +ĠJud iciary +Ġ13 4 +Ġmir rors +Ġk et +t imes +Ġneuro log +Ġc av +ĠGu est +Ġtum or +sc ill +ĠLl oyd +E st +Ġcle arer +Ġstere otypes +Ġd ur +not hing +Red dit +Ġnegoti ated +---------------- -------- +23 5 +Ġfl own +ĠSe oul +ĠRes ident +ĠS CH +Ġdisappear ance +ĠV ince +g rown +Ġgrab s +r il +ĠInf inite +ĠTw enty +Ġpedest rian +Ġjer sey +ĠF ur +ĠInf inity +ĠEll iott +Ġment or +Ġmor ally +Ġob ey +sec ure +iff e +Ġantib iotics +ang led +ĠFre eman +ĠIntrodu ction +J un +Ġm arsh +ic ans +ĠEV ENTS +och ond +W all +icult y +Ġmisdem eanor +Ġl y +Th omas +ĠRes olution +Ġanim ations +ĠD ry +Ġinter course +ĠNew castle +ĠH og +ĠEqu ipment +17 7 +Ġterrit orial +Ġarch ives +20 3 +Fil ter +ĠMun ich +Ġcommand ed +ĠW and +Ġpit ches +ĠCro at +Ġrat ios +ĠM its +Ġaccum ulated +ĠSpecific ally +Ġgentle man +acer b +Ġp enn +Ġa ka +ĠF uk +Ġinterven e +ĠRef uge +ĠAlz heimer +Ġsuccess ion +oh an +d oes +L ord +Ġsepar at +Ġcorrespond ence +Ġsh iny +P rior +Ġs ulf +Ġmiser able +Ġded ication +( ). +Ġspecial ists +Ġdefect s +ĠC ult +ĠX ia +Ġje opard +ĠO re +Ab ility +Ġle ar +Ġamb itions +ĠB MI +ĠArab s +Ġ19 42 +Ġpres ervation +ific ate +Ġash amed +l oss +ĠRest aur +Ġrese mble +Ġen rich +ĠK N +ĠCl an +fl oat +Ġplay able +IT T +Ġharm ony +arr ison +ĠWe instein +w ere +Ġpoison ing +ĠCom put +ĠWord Press +m ajor +ĠVal ve +F an +ĠTh row +ĠRom ans +ĠDep ression +ad os +Ġtort ured +Ġbal ancing +bott om +Ġacqu iring +ĠMon te +ard i +Ġa ura +Ġ# # +ĠStand ing +ĠAtl as +C F +Ġintr ins +ĠBen ghazi +Ġcamp ing +Ġt apped +bl ade +st rous +ĠR abb +ĠW ritten +t ip +ĠNe igh +ster dam +ĠAll ow +ĠHe aling +ĠR hod +n um +Ġcaffe ine +ĠPer cent +Ġbo o +Ġapp les +30 5 +Ġwel coming +Ġappl aud +Ġa usterity + ± +ĠRe ality +ef e +å ® +Ġsu cks +Ġtab s +ĠPay Pal +Ġback pack +Ġgif ted +abul ary +ĠSc out +ir teen +Ġch in +Ġo mitted +Ġnegative ly +Ġaccess ing +ĠE arn +Ġambul ance +Ġhead phones +Ġ20 5 +ĠRef resh +p resident +ĠKit chen +ĠEnt ered +ĠS nyder +00 5 +om ical +Ġborrow ed +ĠN em +Ġav iation +Ġst all +rim ination +Ġuniform s +it ime +ĠSim mons +ener gy +ab lished +y y +qual ified +Ġrall ies +ĠSt uart +fl ight +Ġgang s +r ag +Ġv ault +lu x +ĠCom par +Ġdesign ation +20 9 +ĠJ os +d ollar +z ero +Ġwell s +30 3 +Ġconstitu ents +Ġhe ck +Ġc ows +Ġcommand ers +Ġdifferent ial +ĠC atherine +29 9 +Ġval ve +Ġbr ace +Ġperspect ives +c ert +f act +icular ly +ĠMc N +pl anes +Ġint ric +Ġpe as +ov an +Ġtoss ed +ret ch +ĠL opez +Ġunf amiliar +de ath +ĠA part +ĠCh ang +Ġrelie ved +rop he +Ġair ports +Ġfre ak +ut il +M ill +ĠCh in +ĠOw en +m ale +ĠBro ken +ĠWind s +ro b +r ising +Ġfire fighters +Ġauthor itarian +Ġ14 8 +Bit coin +ex ternal +Ġbrow sers +iche ver +or ian +Ġun b +Ġpo ke +ĠZ ot +M id +ĠPop ular +Ġco vert +Ġcont ributes +Ġ6 50 +Ġcont ention +G ate +Ġcons oles +Ġchrom os +ĠI X +Ġvis ually +ĠE isen +Ġjewel ry +Ġdeleg ation +Ġacceler ate +ĠR iley +Ġsl ope +Ġind oor +it ially +Ġhuge ly +Ġtun nels +Ġfin ed +Ġdirect ive +Ġfore head +ustom ed +Ġsk ate +Mus ic +g as +Ġrecogn izing +am bo +Ġover weight +ĠGr ade +Ù Ĭ +Ġsound ing +Ġlock ing +ĠR EM +St ore +Ġexc av +ĠLike wise +ĠL ights +Ġel bow +ĠSupp ly +w ic +Ġhands ome +19 94 +C oll +Ġadequ ately +ĠAssoci ate +Ġstri ps +Ġcrack down +Ġmar vel +ĠK un +Ġpass ages +@@ @@ +ĠT all +Ġthought ful +names e +Ġprost itution +bus iness +Ġball istic +person al +c ig +iz ational +R ound +ĠÂłĠÂł ĠÂłĠÂł +ĠCole man +Ġadm itting +ĠPl ug +Ġbit coins +ĠSu z +Ġfair ness +Ġsupp lier +Ġcatast rophic +ĠHel en +o qu +M arc +ĠArt icles +g ie +Ġend angered +Ġdest iny +ĠVol t +ol ia +ax is +Ġche at +Ġun ified +IC O +qu ote +30 2 +ĠS ed +Ġsupp ression +Ġanaly zing +Ġsqu at +Ġfig uring +Ġcoordin ates +Ġch unks +Ġ19 46 +Ġsub p +Ġw iki +ĠFor bes +ĠJ upiter +ĠE rik +im er +ĠCom mercial +\ ) +Ġlegitim acy +Ġd ental +ĠMe an +Ġdefic its +5 50 +Orig inally +ĠHor ror +Ġcontam ination +ll ah +Ġconf isc +ĠCl are +T B +ĠF ailed +an ed +Ġrul er +ĠCont roller +Ġfemin ists +F ix +g ay +20 7 +Ġr abbit +Th ird +ownt own +Ġgl ue +Ġvol atile +Ġsh ining +Ġf oll +Ġimp aired +Ġsup ers +æ Ī +Ġcl utch +ļé ĨĴ +Ġpro let +Ġ( ! +Ġy elled +ĠK iev +ĠEr n +ĠSh ock +K B +Ġsit uated +qu ery +ĠN as +Ġan nex +char acter +ĠHol iday +Ġautom ation +ĠJ ill +ĠRem astered +Ġl inem +Ġwild erness +ĠHor izon +ĠGu inea +A Z +Ġmain land +Ġsec recy +LE ASE +Ġp unk +ĠProv ince +( ), +Spe ed +Ġhand ing +ĠSeb ast +S ir +r ase +Ġj ournals +Ġcon gest +ĠT ut +ir rel +Ġschizophren ia +Ġmis ogyn +health y +I ron +Ġreact ed +- $ +25 2 +Ġpl ural +Ġpl um +Ġbarg ain +Ġground ed +f inder +Ġdis se +ĠL az +O OD +Ġat roc +F actory +Ġmin ions +Ġo ri +ĠB rave +ĠP RE +ĠMy anmar +ĠH od +Ġexped ition +Ġexpl ode +ĠCo ord +Ġext r +ĠB rief +ĠAD HD +Ġhard core +feed ing +Ġd ile +ĠF ruit +Ġvacc ination +ĠM ao +osp here +Ġcont ests +- | +Ġf ren +isp here +R om +ĠSh arp +ĠTre nd +Ġdis connect +âĢ¢ âĢ¢ +Ġper secution +Ear th +Ġhealth ier +38 4 +Ġc ob +ĠTr inity +OW S +AN N +Ġspecial ty +Ġg ru +Ġcooper ative +wh y +Start ing +ĠIss ues +st re +ens or +Ġ18 5 +Ad v +! ? +ĠRe vel +em ia +ĠH ulk +Ġcelebr ations +ĠS ou +ra ud +ĠKle in +Ġun real +con text +Ġpartners hips +Ġadop ting +t ical +Ġspl ash +ĠHe zbollah +c ategory +cycl op +xt on +ĠD ot +urd y +t z +Ġenvelop e +ĠN L +â ķ +Ġwhere in +Spe c +18 4 +Ġte lev +al iation +Ġmyth s +å ° +Ġrig orous +Ġcommun icating +Ġobser ver +Ġre he +ĠW ash +Ġapolog ized +ĠT in +Ġexpend itures +work ers +d ocument +Ġhes itate +ĠLen in +Ġunpredict able +Ġrenew al +cl er +ok ia +ĠCON T +Ġpost season +Tok ens +Ġex acerb +Ġbet ting +Ġ14 7 +Ġelev ation +W ood +ĠSol omon +19 4 +00 4 +out put +Ġredu nd +ĠM umbai +Ġp H +Ġreprodu ce +ĠD uration +MA X +Ġb og +C BS +ĠBal ance +ĠS gt +ĠRec ent +Ġc d +Ġpo pped +Ġincomp et +pro p +ay an +g uy +Pac ific +Ġty r +Ġ{ { +ĠMy stic +ĠD ana +Ġmast urb +Ġge ometry +à ¢ +ĠCor rect +Ġtraject ory +Ġdistract ed +Ġf oo +ĠW elsh +L uc +m ith +Ġrug by +Ġrespir atory +Ġtri angle +Ġ2 15 +Ġunder graduate +ĠSuper ior +ch anging +_ - +Ġright ly +Ġrefere e +Ġluc rative +Ġun authorized +Ġresemb les +ĠGN U +ĠDer by +Ġpath ways +ĠL ed +Ġend urance +Ġst int +Ġcollect or +F ast +Ġd ots +Ġnational s +ĠSec urities +Ġwh ip +Par am +Ġlearn s +M agic +Ġdetail ing +m oon +Ġbroadcast ing +Ġb aked +26 5 +hol m +ĠS ah +ĠHus sein +ĠCourt esy +17 4 +Ġ14 6 +Ġge ographic +pe ace +Ġjud ging +ĠS tern +B ur +Ġstory line +G un +ĠSt ick +24 5 +30 7 +ãĤ´ ãĥ³ +ĠAdminist rator +Ġbur nt +Ġp ave +ch oes +Ex ec +Ġcamp uses +Res ult +Ġmut ations +ĠCh arter +Ġcapt ures +Ġcomp ares +Ġbad ge +S cient +Ġer ad +ier y +o i +ett es +ĠE state +Ġst rap +Ġproud ly +Ġf ried +Ġwithd rawn +ĠV oy +ph ony +It ems +ĠP ierce +b ard +Ġann otation +ant on +ill on +Im pro +... ) +Ġhapp ier +---- -- +ad just +Ġstaff ers +Ġactiv ism +Ġper f +Ġal right +N eed +Ġcomm ence +Ġopio id +ĠAm anda +E s +ĠP ars +ĠK aw +W orks +24 8 +Ġind o +t c +end ant +ĠM oto +Ġlegal ization +OT E +Ġtask ed +Ġt sp +ĠACT IONS +16 6 +Ġrefres hing +ĠN R +ĠPere z +Ġinfring ement +S Y +List en +in ning +k u +Ġrot ate +pro gram +ar ah +Des ign +Ġ( £ +Ġst oring +Ġwar rants +Ġjud gement +ĠB rist +us ually +ph oto +ĠR an +ĠP ine +Ġoutrage ous +ĠValent ine +lu ence +ĠEvery body +Al tern +Ġrele vance +Ġtermin ated +Ġd essert +Ġfulf illed +Ġprosecut ed +ĠW ords +Ġm igrant +Ġcultiv ation +ÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤ ÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤ +idel ity +ĠV ern +ĠLog in +Ġmetaph or +ĠT ip +Ġrecru its +ĠP ig +rib ing +Ġenthusi asts +ex per +Ġfright ening +ĠH air +ans on +str ate +Ġh i +He ight +Ġown ing +n one +Ġdis like +Ġkn ives +pher d +Ġloud ly +ĠAP Is +Dis play +ĠL ac +ĠUS S +ab l +ver ages +J ew +Ġ17 2 +ĠHist orical +at oon +ĠPhys ics +in tern +Ġwarm th +Ġto pp +D M +Ġgun man +Ġem peror +od i +ãĥ £ +in atory +ĠR ib +Ġ13 1 +ĠSat urn +ĠSh ining +Ġw aking +Qu otes +Ġcomed ian +en berg + ½ +Ġbelie vers +Ġpaper work +c ustom +Ġle v +Ġl ament +Ġpour ing +22 2 +p olitical +ĠSupp lement +m aid +Ġcruel ty +Ġt read +ys ics +A w +rit es +Ġmod ifier +ĠP osition +Ad am +l b +ub s +Ġimper fect +Ġcl usters +ĠEngine er +ĠC herry +Ġinaug uration +ĠS au +Ġembod iment +ĠUn cle +Ġover r +Ġexplos ions +c ule +ĠPrinc eton +ĠAndre a +Ġincorrect ly +Ġearn est +Ġpil gr +ĠS print +Ġslee ve +Ġhe ars +ĠAm azing +Ġbrow sing +ag in +Ġhom eland +Ġha w +Ġd iving +ist ered +17 8 +Ġbarg aining +ĠArc ade +Ġdeleg ate +ters on +................................ ................................ +ĠJackson ville +27 5 +Ġst agn +Ġad am +ĠSher man +C B +Ġsub urb +ĠFood s +Ġconver ting +ĠAr ist +Ġch ambers +l ove +Ġam ino +ĠG an +Ġmad ness +m c +ĠUS E +def ined +Ġul tr +ind ust +Ġw olves +l ance +Add itionally +Ġcr acks +as ia +ĠRe ason +ĠP ump +Ġaccident al +ĠL aser +ĠR id +Ġinitial ized +ell i +Ġun named +Ġn oun +ĠPass ed +Ġhost age +ĠEth iop +sh irts +Ġun rel +ĠEmb assy +Ġ19 41 +Ġat oms +Ġpur ported +16 4 +ĠF i +Ġgall ons +ĠMon ica +Ġp g +en ment +Ġsort ed +ĠG ospel +Ġhe ights +Ġtr aced +Ġunder going +She ll +Ġs acks +Ġproport ions +Ġhall uc +F ont +ac et +Ġwar mer +ĠIN TER +Ġgrab bing +Pl ug +Ġreal ization +ĠBur ke +Ġen chant +AT ER +ĠSe ed +Ġabund ant +F M +Ġc ivic +V s +is i +Ġv ow +Ġre per +ĠPartners hip +Ġpenet ration +Ġax e +Ġsh attered +ĠZ ombies +Ġv inyl +ĠAl ert +e on +Ġoblig ed +ĠIll ust +ĠPl aza +ĠFront ier +Ġdavid jl +ĠSer ial +ĠH av +ĠNut rition +B i +Ġâĸ Ī +ĠJ ays +lin ux +Ġhur ry +Ġv oy +Ġhop eless +ĠSte alth +Ġ ãģ +ess ors +tt le +b org +ĠSaf ari +f ell +Ġw ary +d ue +ĠAb ove +H a +E LL +Ġnot or +ĠW on +T oo +Ġoccup ations +Ġposs essions +Ġinv iting +Ġpred ators +Ġacceler ated +Ġ15 7 +uter te +ĠC ube +e ast +acc ount +G ive +Ġtrans plant +red ients +id able +Ġscreens hots +ĠG und +ĠF S +Ġtravel ers +Ġsens ory +ĠF iat +ĠRock ets +İ ĭ +_ { +F riend +Ġchar ming +AL S +Ġenjoy ment +m ph +Ġ5 000 +ĠRE G +Ù Ĩ +b ia +Ġcomp ilation +ro st +ĠV P +ĠSch ne +201 9 +Ġcop ying +M ORE +ĠFl ore +f alls +2 15 +t otal +Ġdis ciples +d ouble +Ġexceed ing +Ġsm ashed +Ġconcept ual +ĠRom ania +ĠB rent +ĠI CE +ĠT ou +Ġg rap +Ġn ails +18 9 +ãĥ ĺ +Ġproc ure +e ur +Ġconfir ming +ĠC ec +aw i +ĠEd en +Ġn g +Ġengine ered +at ics +Ġhook ed +Ġdisgust ing +ĠMur der +ãĤ ¿ +L ibrary +Ġ16 8 +Al most +hem atic +Men u +ĠNot re +ĠJ ur +Ġkidn apped +Ġhack er +ĠJ ade +Ġcreep y +Ġdraw ings +ĠSpons or +Ġcycl ists +ĠGob lin +Ġoptim ized +Ġst aged +ĠMc D +bet ween +A ge +en o +S ex +ĠW ide +n ings +av is +Ġincap able +ĠK ob +Ġreward ing +ĠL one +oles cent +Ġcontract ed +Ġstick y +J ose +B all +f est +ĠIn put +ĠRec ently +Ġto mat +squ are +App lication +Ġnit rogen +Ġdupl icate +ĠRec on +ĠD ear +L ondon +Ġint ra +Ġd ock +Ġout reach +ĠM illion +Ġmamm als +am pton +V AL +Ġsn aps +Ġd os +ĠWh ole +ĠRead y +T ry +ĠWinn ipeg +ear ance +Ġinc urred +ren ched +ĠNS W +il ot +rain e +Ġc ube +g ot +Ġrun way +etermin ed +ĠHaw ks +Ġsurviv or +ĠW ish +ĠD in +ĠDE F +ĠV ault +18 7 +Ġmush rooms +Ġcris p +be y +ĠDisco very +Ġdevelopment al +Ġparad igm +Ġcha otic +ĠT su +Ġ3 33 +b ons +Ġbacter ial +Ġcomm its +Ġcos mic +Ġme ga +oc ative +ĠP aint +ophob ic +Ġv ain +Ġcar ved +ĠTh ief +ĠG ul +ows hip +Ġc ites +ĠEd inburgh +Ġdimin ished +Ġacknowled ges +ĠK ills +Ġmic row +ĠHer a +Ġsen iors +Ġwhere by +H op +at ron +Ġun available +ĠN ate +Ġ4 80 +Ġsl ated +ĠRe becca +ĠB attery +Ġgram mar +Ġhead set +Ġcurs or +Ġex cluding +any e +aunder ing +eb in +Ġfeas ible +ĠPub lishing +ĠLab s +ĠCl iff +ĠFerr ari +Ġp ac +vis ible +mark ed +pe ll +Ġpol ite +Ġstagger ing +ĠGal actic +Ġsuper st +Ġpar an +ĠOffic ers +ãĢ ģ +Ġspecific s +ul us +23 9 +ĠP aste +AM P +ĠPan ama +ĠDe lete +angu ard +rest rial +Ġhero ic +ĠD y +ا ÙĦ +Ġincumb ent +Ġcr unch +t ro +Ġsc oop +Ġblog ger +Ġsell ers +ure n +Ġmedic ines +ĠC aps +ĠAnim ation +ox y +Ġout ward +Ġinqu iries +22 9 +Ġpsych ologist +ĠS ask +ev il +Ġcontam inated +ãĤ ¨ +he rence +Ġbrand ed +ĠAbd ul +z h +Ġparagraph s +Ġmin s +Ġcor related +er b +Ġimp art +Ġmil estone +ĠSol utions +ot le +Ġunder cover +Ġmar ched +ĠCharg ers +f ax +ĠSec rets +Ġr uth +we ather +Ġfemin ine +Ġsh am +Ġprest igious +igg ins +Ġs ung +hist ory +ett le +gg ie +Ġout dated +ol and +Ġper ceptions +ĠS ession +ĠDod gers +u j +ĠE ND +D oc +Ġdefic iency +Gr and +ĠJ oker +Ġretro spect +Ġdiagn ostic +Ġharm less +Ġro gue +ĠA val +E qu +Ġtrans c +ĠRoberts on +ĠDep ending +ĠBurn s +iv o +Ġhost ility +F eatures +ĵ ĺ +Ġdis comfort +ĠL CD +spec ified +ĠEx pect +3 40 +Ġimper ative +ĠReg ular +Ch inese +Ġstate wide +Ġsy mm +Ġlo ops +Ġaut umn +N ick +Ġsh aping +Ġqu ot +Ġc herry +ĠCross ref +è¦ ļéĨĴ +Stand ard +he ed +ĠD ell +ĠViet namese +Ġo st +ĠV alkyrie +O A +Ass ad +Ġreb ound +ĠTra ffic +pl aces +æ ĺ +ĠB uc +17 2 +Ġshel ters +Ġins isting +ĠCertain ly +ĠKenn eth +ĠT CP +Ġpen al +ĠRe play +he ard +Ġdial ect +iz a +ĠF Y +it cher +ĠD L +Ġspir al +Ġquarterback s +Ġh ull +Ġgo ogle +Ġto dd +ĠSter ling +ĠPl ate +Ġsp ying +mb ol +ĠReal m +ĠPro ced +ĠCr ash +Ġtermin ate +Ġprotest ing +C enter +gu ided +Ġun cover +Ġboy cott +Ġreal izes +s ound +Ġpret ending +ĠV as +19 80 +Ġfram ed +Ġ13 9 +Ġdesc ended +Ġrehab ilitation +Ġborrow ing +ĠB uch +Ġbl ur +R on +ĠFro zen +en za +Ch ief +ĠP oor +Ġtransl ates +M IN +Ġ2 12 +J ECT +Ġerupt ed +Ġsuccess es +S EC +Ġpl ague +Ġg ems +d oms +Ġstret ches +ĠSp y +Ġstory telling +C redit +ĠP ush +Ġtra ction +Ġin effective +ĠL una +Ġt apes +Ġanaly tics +erc ise +Ġprogram mes +ĠCar bon +Ġbeh old +he avy +ĠConserv ation +ĠF IR +Ġs ack +ter min +ric ks +Ġhous ed +Ġunus ually +I ce +Ġexecut ing +ĠMor oc +ed ay +Ġed itions +Ġsm arter +ĠB A +Ġout law +Ġvan ished +ib a +AL SE +ĠSil va +23 8 +C ould +Ġphilos opher +Ġevac uated +Sec ret +14 2 +Ġvis as +ãĤ ¬ +ĠM alt +ĠClear ly +ĠN iger +ĠC airo +ĠF ist +3 80 +ĠX ML +aut o +it ant +Ġrein forced +Rec ord +ĠSurviv or +G Hz +Ġscrew s +parent s +Ġo ceans +ma res +Ġbra kes +vas ive +Ġhell o +ĠS IM +rim p +Ġo re +ĠArm our +24 7 +Ġterr ific +Ġt ones +14 1 +ĠMin utes +Ep isode +Ġcur ves +Ġinflamm atory +Ġbat ting +ĠBeaut iful +L ay +Ġunp op +v able +Ġr iots +ĠTact ics +b augh +ĠC ock +Ġorg asm +ĠS as +Ġconstruct or +et z +G ov +Ġant agon +Ġthe at +Ġde eds +ha o +c uts +ĠMc Cl +Ġu m +ĠScient ists +Ġgrass roots +ys sey +"] => +Ġsurf aced +Ġsh ades +Ġneighb ours +Ġad vertis +oy a +Ġmer ged +Up on +Ġg ad +Ġanticip ate +Any way +Ġsl ogan +Ġdis respect +I ran +ĠT B +act ed +Ġsubp oen +medi ately +OO OO +Ġwa iver +Ġvulner abilities +ott esville +ĠHuff ington +J osh +ĠD H +M onday +ĠEll en +K now +x on +it ems +22 8 +Ġf ills +ĠN ike +Ġcum ulative +and als +I r +Ġ ì +Ġfr iction +ig ator +Ġsc ans +ĠVi enna +ld om +Ġperform ers +P rim +Ġb idding +M ur +Ġlean ed +ĠPri x +al ks +Ġ[ âĢ¦] +ĠTw itch +ĠDevelop er +ĠG ir +Ġcall back +Ab stract +Ġacc ustomed +Ġfreed oms +ĠP G +ur acy +Ġl ump +is man +,, ,, +19 92 +ĠR ED +Ġwor m +M atch +ĠPl atinum +I J +ĠOwn er +Tri via +com pl +Ġnew born +Ġfant as +O wn +Ġ19 59 +Ġsymp ath +Ġub iqu +Ġoutput s +Ġal lev +Ġpr ag +K evin +Ġfav ors +Ġbur ial +Ġn urt +so lete +c ache +Ġ15 6 +Ġunl ocks +te chn +M aking +Ġcon quer +ad ic +æ ĸ +Ġel f +Ġelect orate +ĠKurd s +ĠSt ack +ĠSam urai +Ġâ ĺħ +Ġ{ } +ĠS aid +ĠFall out +Ġkind ness +ĠCustom s +ĠBou levard +Ġhelicop ters +ot ics +ĠVe get +com ment +Ġcritic ised +Ġpol ished +ĠRem ix +ĠC ultural +Ġrec ons +Ġdo i +at em +Sc reen +Ġbar red +Com ments +ĠGener ally +Ġsl ap +7 20 +V ari +p ine +Ġem pt +Ġh ats +ĠPlay ing +l ab +a verage +form s +ĠC otton +Ġcan s +ĠD ON +ĠSom alia +C rypt +ĠIncre ases +E ver +mod ern +Ġsur geon +3 000 +Ġrandom ized +================================ ================================ +B ern +im pl +ĠC OR +Ġpro claim +th ouse +Ġto es +Ġam ple +Ġpres erving +Ġdis bel +gr and +B esides +Ġsil k +ĠPat tern +h m +Ġenter prises +Ġaffidav it +ĠAdvis ory +Ġadvert ised +ĠRel igious +se ctions +psy ch +ĠField s +aw ays +Ġhasht ag +ĠNight mare +Ġv ampire +Ġfore nsic +rosso ver +n ar +Ġn avy +Ġvac ant +ĠD uel +Ġhall way +Ġface book +ident ally +ĠN RA +Ġm att +Ġhur ricane +ĠKir by +ĠP uzzle +Ġsk irt +ou st +du llah +Ġanal ogy +in ion +Ġtomat oes +ĠN V +ĠPe ak +ĠMe yer +Ġappoint ments +Ġm asc +Ġal ley +re hend +Ġchar ities +Ġund o +Ġdest inations +ĠTest ing +"> " +c ats +* . +Ġgest ures +gener al +Le ague +Ġpack ets +ĠInspect or +ĠBer g +Ġfraud ulent +Ġcritic ize +F un +Ġbl aming +nd ra +Ġsl ash +ĠE ston +Ġpropos ing +Ġwh ales +Ġtherap ist +Ġsub set +Ġle isure +EL D +ĠC VE +ĠAct ivity +Ġcul min +sh op +ĠD AY +is cher +ĠAdmir al +ĠAtt acks +Ġ19 58 +Ġmem oir +Ġfold ed +Ġsex ist +Ġ15 3 +ĠL I +Ġread ings +Ġembarrass ment +ĠEmploy ment +w art +ch in +Ġcontin uation +l ia +Rec ently +Ġd uel +Ġevac uation +ĠKash mir +Ġdis position +ĠR ig +Ġbol ts +Ġins urers +4 67 +M ex +Ġret aliation +Ġmis ery +Ġunre asonable +r aining +I mm +ĠP U +em er +Ġgen ital +ãĤ ³ +ĠC andy +Ġon ions +ĠP att +lin er +Ġconced ed +Ġf a +Ġfor c +ĠH ernandez +ĠGe off +deb ian +ĠTe ams +Ġc ries +Ġhome owners +23 7 +A BC +Ġst itch +Ġstat istic +Ġhead ers +ĠBi ology +Ġmot ors +ĠG EN +ĠL ip +Ġh ates +Ġhe el +S elf +i pl +ED IT +ort ing +Ġann ot +ĠSpe ech +old emort +ĠJ avascript +ĠLe Bron +Ġfoot print +Ġf n +Ġseiz ures +n as +h ide +Ġ19 54 +ĠBe e +ĠDecl aration +ĠKat ie +Ġreserv ations +N R +f emale +Ġsatur ated +Ġb iblical +Ġtroll s +Dev ice +ph otos +Ġdr ums +ãĥīãĥ© ãĤ´ãĥ³ +N ight +f ighter +ĠH ak +ri ber +Ġc ush +Ġdiscipl inary +ba um +ĠG H +ĠSch midt +ilib rium +Ġs ixty +ĠKush ner +ro ts +Ġp und +ĠR ac +Ġspr ings +Ġcon ve +Bus iness +F all +Ġqual ifications +Ġvers es +Ġnarc iss +ĠK oh +ĠW ow +ĠCharl ottesville +ed o +Ġinterrog ation +ĠW ool +36 5 +B rian +Ġâľ ĵ +Ġalleg es +ond s +id ation +ĠJack ie +y u +Ġl akes +Ġworth while +Ġcryst als +ĠJud a +Ġcomp rehend +Ġfl ush +Ġabsor ption +ĠO C +Ġfright ened +ĠCh ocolate +Mart in +Ġbu ys +Ġbu cks +Ġapp ell +ĠChampions hips +Ġlist ener +ĠDef ensive +Ġc z +ud s +ĠM ate +Ġre play +Ġdecor ated +Ġs unk +ĠV IP +ĠAn k +Ġ19 5 +aa aa +Nob ody +ĠMil k +ĠG ur +ĠM k +ĠS ara +Ġse ating +ĠW id +Tr ack +Ġemploy s +Ġgig antic +AP P +ãĤ § +in ventory +Ġtow el +at che +l asting +ĠT L +Ġlat ency +Ġkn e +B er +me aning +Ġup held +Ġplay ground +Ġm ant +S ide +Ġstere o +Ġnorth west +Ġexception ally +Ġr ays +Ġrec urring +D rive +Ġup right +Ġab duct +ĠMar athon +Ġgood bye +Ġal phabet +h p +Ġcourt room +ring ton +ot hing +T ag +Ġdiplom ats +Ġbar bar +ĠAqu a +18 3 +33 33 +Ġmat urity +Ġinst ability +ĠAp ache +Ġ= == +Ġfast ing +ĠGr id +Mod Loader +Ġ15 2 +A bs +ĠOper ating +ett i +Ġacqu aint +Don nell +ĠK em +ĠFor ge +Ġarm ored +M il +Ġphilos ophers +in vest +Pl ayers +â Ī +Ġmy riad +Ġcomr ades +R ot +Ġremember ing +Ġcorrespond s +Ġprogram mers +ĠLyn n +Ġo lig +Ġco herent +yn chron +ĠChem ical +Ġj ugg +p air +post s +E ye +ĠIn ner +Ġsem ester +ott est +ĠEmir ates +ric anes +or ously +m its +ĠW is +Ġd odge +l ocation +Ġf aded +Am azon +ĠPro ceed +ĠIN FO +j ournal +ĠTru ck +T en +Ġ2 17 +Ġstat utes +m obile +ĠT ypes +Rec omm +b uster +pe x +Ġleg ends +Ġhead ache +f aced +ĠWi Fi +if ty +ĠH ER +Ġcirc uits +ER ROR +22 6 +ol in +Ġcyl inder +osp ace +ik ers +P rem +Qu ant +Ġconflic ting +Ġslight est +Ġfor ged +ion age +Step hen +ĠK ub +ĠOpp ortun +ĠHe al +Ġbl o +Ġrul ers +Ġh uh +Ġsubmar ine +f y +ass er +Ġallow ance +ĠKas ich +ĠT as +ĠAustral ians +Forge ModLoader +ĠâĨ ij +ĠMat rix +am ins +Ġ12 00 +ĠAc qu +23 6 +D ocument +ĠBre aking +19 3 +ĠSub st +ĠRoll er +ĠPro perties +ĠN I +t ier +Ġcr ushing +Ġadvoc ating +Further more +keep ers +Ġsex ism +x d +Ġcall er +ĠS ense +chie ve +ĠT F +Ġfuel ed +Ġreminis cent +Ġobs ess +ur st +Ġup hold +ĠF ans +het ics +Ġâ Ĺ +ĠB ath +Ġbe verage +Ġo scill +25 4 +Ġpol es +Ġgrad ual +Ġex ting +ĠS uff +ĠS uddenly +Ġlik ing +Ġ19 49 +un ciation +am ination +ĠO mar +ĠL V +ĠCon sequently +Ġsynt hes +ĠG IF +Ġp ains +Ġinteract ing +u ously +inc re +Ġrum or +ĠScient ology +19 7 +ĠZ ig +Ġspe lling +ĠA SS +Ġexting u +ms on +Ġg h +Ġremark ed +ĠStrateg ic +ĠM ON +å ¥ +g ae +ĠWH AT +E ric +ĠCamp us +Ġmeth ane +Ġimag in +J UST +ĠAl m +X T +i q +ĠR SS +Ġwrong doing +att a +Ġbig ot +Ġdemonstr ators +ĠCal vin +ĠV illa +Ġmembr ane +ĠAw esome +Ġbenef ic +26 8 +Ġmagn ificent +ĠL ots +G reg +ĠBor is +Ġdetain ees +ĠH erman +Ġwhis pered +Ġa we +Prof essor +fund ing +Ġphys iological +ĠDest ruction +Ġlim b +Ġmanip ulated +Ġbub bles +Ġpse ud +Ġhyd ra +ĠBrist ol +Ġst ellar +ĠExp ansion +ĠK ell +ĠInterest ingly +Ġm ans +Ġdrag ging +Ġec ological +ĠF it +Ġg ent +Ġbenef ited +ĠHait i +Ġpoly g +ãĥ İ +Ġ20 30 +Ġpro w +Ġrecon struction +Ġwas t +Ġpsych ic +ĠGree ks +Hand ler +16 2 +ĠP ulse +Ġsol icit +Ġsy s +Ġinflu x +ĠG entle +per cent +Ġprolifer ation +Ġtax able +Ġdisreg ard +Ġesc aping +Ġg inger +Ġwith stand +Ġdevast ated +ĠD ew +ser ies +Ġinject ed +ela ide +Ġturn over +he at +Ļ Ĥ +H appy +ĠSil ent +ãĤ Ń +iv ism +Ġir rational +AM A +Ġre ef +r ub +Ġ16 2 +Ġbank ers +ĠEth ics +v v +Ġcritic isms +K n +18 6 +M ovie +ĠT ories +Ġno od +Ġdist ortion +F alse +od ore +Ġt asty +Res earch +ĠU ID +- ) +Ġdivor ced +ĠM U +ĠHay es +ĠIs n +ian i +ĠH Q +Ġ" # +ign ant +Ġtra umatic +ĠL ing +H un +Ġsab ot +on line +r andom +Ġren amed +ra red +K A +d ead +é t +ĠAss istance +Ġse af +++++ ++++ +Ġse ldom +ĠWeb b +Ġbo olean +u let +Ġref rain +ĠDI Y +ru le +Ġshut ting +Ġutil izing +load ing +ĠPar am +co al +oot er +Ġattract ing +ĠD ol +Ġher s +ag netic +ĠRe ach +im o +Ġdisc arded +ĠP ip +01 5 +ü r +Ġm ug +Im agine +C OL +Ġcurs ed +ĠSh ows +ĠCurt is +ĠSach s +spe aking +ĠV ista +ĠFram ework +ong o +Ġsub reddit +Ġcr us +ĠO val +R ow +g rowing +Ġinstall ment +Ġgl ac +ĠAdv ance +EC K +ĠLGBT Q +LE Y +Ġac et +Ġsuccess ive +ĠNic ole +Ġ19 57 +Qu ote +Ġcircumst ance +ack ets +Ġ14 2 +ort ium +Ġguess ed +ĠFr ame +Ġperpet rators +ĠAv iation +ĠBen ch +Ġhand c +A p +Ġ19 56 +25 9 +r and +Net Message +d in +urt les +h ig +ĠV III +ff iti +ĠSw ords +b ial +Ġkidn apping +dev ice +Ġb arn +ĠEl i +auc as +S end +Con structed +Ġ ½ +Ġneed les +Ġad vertisements +Ġv ou +Ġexhib ited +ĠFort ress +As k +B erry +TY PE +Ġcan cers +ump ing +ĠTerrit ory +Ġpr ud +Ġn as +Ġathe ist +Ġbal ances +ãģ Ł +ĠSh awn +& & +Ġland sc +ĠR GB +Ġpet ty +Ġex cellence +Ġtransl ations +Ġpar cel +ĠChe v +E ast +ĠOut put +im i +Ġamb ient +ĠTh reat +Ġvill ains +Ġ5 50 +IC A +Ġtall er +Ġle aking +c up +Ġpol ish +Ġinfect ious +ĠK C +Ġ@ @ +back ground +Ġbureaucr acy +ĠS ai +un less +it ious +ĠSky pe +At l +ID ENT +00 8 +Ġhyp ocr +Ġpit chers +Ġguess ing +ĠF INAL +Bet ween +Ġvill agers +Ġ25 2 +f ashion +ĠTun is +Be h +ĠEx c +ĠM ID +28 8 +ĠHas kell +19 6 +ĠN OR +Ġspec s +Ġinv ari +Ġgl ut +ĠC ars +Ġimp ulse +Ġhon ors +g el +Ġjurisd ictions +ĠBund le +ul as +Calif ornia +ĠIncre ase +Ġp ear +Ġsing les +Ġc ues +Ġunder went +ĠW S +Ġexagger ated +Ġdub ious +Ġfl ashing +L OG +) ]. +J ournal +t g +V an +ĠI stanbul +ĠIn sp +ĠFrank en +D raw +Ġsad ness +Ġiron ic +ĠF ry +x c +Ġ16 4 +is ch +W ay +ĠProtest ant +h orn +Ġun aff +ĠV iv +ill as +ĠProduct ions +ĠH ogan +Ġper imeter +ĠS isters +Ġspont aneous +Ġdown side +Ġdescend ants +Ġor n +w orm +Japan ese +Ġ19 55 +Ġ15 1 +ĠDo ing +els en +umb les +Ġrad ically +ĠDr um +ĠB ach +Ġli abilities +ĠO B +ĠElement ary +Ġmem e +yn es +Ġfinger print +ĠGr ab +Ġundert ake +Mem bers +ĠRead er +ĠSim s +g od +Ġhypot hetical +s cient +ĠA J +Ġchar ism +Ġad missions +ĠMiss ile +tr ade +Ġexerc ising +ĠBack ground +W ritten +Ġvoc als +whe ther +Ġv i +ĠW inner +Ġl itter +ĠSh ooting +ST EM +ãĤ ¡ +ĠA FL +Ġvari ability +Ġe ats +ĠD PS +b row +Ġeleph ants +Ġstr at +Ġ Å +Ġsett lers +Matt hew +Ġin advert +H I +ĠIM F +ĠGo al +Ġnerv es +John son +ey e +ablish ment +Th ursday +BIL ITY +H ad +am oto +het amine +ep s +Ġmit ochond +Ġcomp ressed +ĠTre vor +ĠAnim als +T ool +L ock +Ġtwe ak +Ġpin ch +Ġcancell ation +P ot +Ġfoc al +ĠAst ron +17 3 +ĠA SC +ĠO THER +umn i +Ġdem ise +d l +Ù ħ +Sem itism +Ġcr acking +Ġcollabor ative +Ġexpl ores +s ql +Ġher bs +Ġconfig urations +m is +ĠRes ult +ace y +ĠSm oke +Ġsan ct +el ia +Ġdeg ener +Ġdeep est +Ġscream ed +Ġn ap +Soft ware +ĠST AR +E F +ĠX in +spons ored +mans hip +23 3 +Ġprim aries +Ġfilter ing +Ġas semble +m il +ĠMy ers +b ows +Ġpun ched +M ic +Ġinnov ations +Ġfun c +and o +Ġfr acking +ĠV ul +о Ð +osh op +ĠIm mun +Ġsett ling +Ġadolesc ents +Ġreb uilding +Ġtransform ing +Ġpar ole +Ġhar bor +Ġbook ing +ot ional +onge vity +ĠY o +b ug +Ġemer ges +ĠMethod s +ĠCh u +P res +ĠDun geons +Ġtra iling +ĠR um +ĠH ugh +å¤ © +ĠE ra +ĠBatt les +Res ults +ĠTr ading +Ġvers a +c ss +ax ies +he et +Ġgre ed +19 89 +Ġgard ens +Ġconting ent +P ark +ĠLeaf s +h ook +ro be +Ġdiplom acy +ĠF uel +ĠInv asion +Ġupgr ading +M ale +Ġe lic +Ġrelent less +ĠCo venant +ap esh +ĠT rop +T y +pro duction +art y +Ġpun ches +ak o +cyclop edia +ĠR abbit +ĠHD MI +Ġ14 1 +Ġf oil +Item Image +ĠF G +Ġimplement ations +ĠP om +ixt ures +Ġaw ait +Ġ3 30 +am us +Ġumb rella +Ġfore see +se par +Ġcircum cision +Ġperipher al +S ay +ĠExper t +In c +Ġwithd rew +ĠAnd ers +f ried +Ġradio active +ĠOp ening +Ġboard ing +ĠN D +Ġover throw +Act iv +W P +ĠAct s +× Ļ +Ġmot ions +v ic +ĠM ighty +ĠDef ender +a er +Ġthank ful +ĠK illing +ĠBr is +mo il +Ġpredict ing +26 6 +ch oice +Ġkill ers +Ġinc ub +ĠChe st +ather ing +Ġpro claimed +fl ower +oss om +umbled ore +ĠCy cling +ĠOccup y +AG ES +P en +ĠY ug +Ġpack aged +Ġheight ened +c ot +st ack +C ond +Ġst amps +m age +Ġpersu aded +Ġens l +ĠCard inal +Ġsol itary +Ġpossess ing +ĠC ork +Ġev id +ĠT ay +Ġbl ues +Ġextrem ism +Ġlun ar +Ġcl own +Te chn +Ġfest ivals +ĠPv P +ĠL ar +Ġconsequ ently +p resent +Ġsom eday +ç İĭ +ĠMet eor +Ġtour ing +c ulture +Ġbe aches +S hip +c ause +ĠFl ood +ãĥ ¯ +Ġpur ity +th ose +Ġem ission +b olt +Ġch ord +ĠScript ure +L u +Ġ$ { +cre ated +Other s +25 8 +Ġelement al +Ġannoy ed +ĠA E +d an +ĠS ag +Res earchers +Ġfair y +âĢĵ âĢĵ +======== ==== +Sm art +GG GG +Ġskelet ons +Ġpup ils +link ed +Ġur gency +en abled +ĠF uck +Ġcoun cill +r ab +U AL +T I +Ġlif es +Ġconf essed +B ug +Ġharm on +ĠCON FIG +ĠNe utral +D ouble +Ġst aple +ĠSH A +Brit ish +ĠSN P +AT OR +oc o +Ġswing ing +ge x +ole on +pl ain +ĠMiss ing +ĠTro phy +v ari +ran ch +Ġ3 01 +4 40 +00000000 00000000 +Ġrest oring +Ġha ul +uc ing +ner g +Ġfut ures +Ġstrateg ist +quest ion +Ġlater al +ĠB ard +Ġs or +ĠRhod es +ĠD owntown +????? - +ĠL it +ĠB ened +Ġco il +st reet +ĠPort al +FI LE +ĠG ru +* , +23 1 +ne um +Ġsuck ed +Ġr apper +Ġtend encies +ĠLaure n +cell aneous +26 7 +Ġbrow se +Ġover c +head er +o ise +Ġbe et +ĠG le +St ay +Ġm um +Ġtyp ed +Ġdiscount s +T alk +ĠO g +ex isting +ĠS ell +u ph +C I +ĠAust rian +ĠW arm +Ġdismiss al +Ġaver ages +c amera +Ġalleg iance +L AN +=" # +Ġcomment ators +ĠSet ting +ĠMid west +Ġpharm ac +ĠEX P +Ġstain less +Ch icago +Ġt an +24 4 +Ġcountry side +ĠV ac +29 5 +Ġpin ned +Ġcr ises +Ġstandard ized +T ask +ĠJ ail +ĠD ocker +col ored +f orth +" }, +Ġpat rons +Ġsp ice +Ġm ourn +ĠM ood +Ġlaund ry +Ġequ ip +ĠM ole +y ll +ĠTH C +n ation +ĠSher lock +Ġiss u +ĠK re +ĠAmeric as +ĠA AA +Ġsystem atically +Ġcont ra +ĠS ally +Ġrational e +Ġcar riage +Ġpe aks +Ġcontrad iction +ens ation +ĠFail ure +Ġpro ps +Ġnames pace +Ġc ove +field s +ãĤ ĭ +Ġw ool +ĠC atch +Ġpresum ed +ĠD iana +r agon +ig i +Ġh amm +Ġst unt +ĠG UI +ĠObserv atory +ĠSh ore +Ġsmell s +ann ah +Ġcock pit +ĠD uterte +8 50 +Ġopp ressed +bre aker +ĠCont ribut +ĠPer u +ĠMons anto +ĠAtt empt +Ġcommand ing +Ġfr idge +ĠR in +ĠChe ss +ual ity +Ġo l +Republic an +ĠGl ory +ĠW IN +.... ... +ag ent +read ing +Ġin h +J ones +Ġcl icks +al an +Ġ[ ]; +ĠMaj esty +ĠC ed +op us +ate l +à ª +AR C +ĠEc uador +ãĥ ł +ĠK uro +Ġritual s +Ġcapt ive +Ġoun ce +Ġdisag reement +Ġsl og +f uel +P et +M ail +Ġexerc ised +Ġsol ic +Ġrain fall +Ġdev otion +ĠAss essment +Ġrob otic +opt ions +ĠR P +ĠFam ilies +ĠFl ames +Ġassign ments +00 7 +aked own +Ġvoc abulary +Re illy +Ġc aval +g ars +Ġsupp ressed +ĠS ET +ĠJohn s +Ġwar p +bro ken +Ġstat ues +Ġadvoc ated +Ġ2 75 +Ġper il +om orph +ĠF emin +per fect +Ġh atch +L ib +5 12 +Ġlif elong +3 13 +Ġche eks +Ġnum bered +ĠM ug +B ody +ra vel +We ight +ĠJ ak +ĠHe ath +Ġkiss ing +ĠJ UST +Ġw aving +u pload +Ġins ider +ĠPro gressive +ĠFil ter +tt a +ĠBe am +Ġviol ently +ip ation +Ġskept icism +Ġ19 18 +ĠAnn ie +ĠS I +Ġgen etics +Ġon board +at l +ĠFried man +ĠB ri +cept ive +Ġpir ate +ĠRep orter +27 8 +Ġmyth ology +Ġe clipse +Ġsk ins +Ġgly ph +ing ham +F iles +C our +w omen +Ġreg imes +Ġphotograp hed +K at +ĠMA X +Offic ials +Ġunexpected ly +Ġimpress ions +F ront +;;;; ;;;; +Ġsuprem acy +Ġs ang +Ġaggrav ated +Ġabrupt ly +ĠS ector +Ġexc uses +Ġcost ing +ide press +St ack +ĠR NA +ob il +Ġghost s +ld on +at ibility +Top ics +Ġreim burse +ĠH M +ĠDe g +Ġth ief +y et +ogen esis +le aning +ĠK ol +ĠB asketball +Ġf i +ĠSee ing +Ġrecy cling +Ġ[ - +Cong ress +Ġlect ures +P sy +Ġne p +Ġm aid +Ġori ented +A X +Ġrespect ful +re ne +fl ush +ĠUn loaded +re quest +gr id +ĠAltern atively +ĠHug o +Ġdec ree +ĠBuddh ism +and um +And roid +ĠCong o +ĠJoy ce +Ġacknowled ging +hes ive +ĠTom orrow +ĠH iro +th ren +ĠM aced +Ġho ax +ĠIncre ased +ĠPr adesh +W ild +____ __ +16 1 +Ġa unt +Ġdistribut ing +ĠT ucker +ĠSS L +ĠW olves +B uilding +ou lt +ĠLu o +ĠY as +ĠSp ir +ĠSh ape +ĠCamb od +ĠIP v +Ġm l +Ġext rad +39 0 +ĠPenn y +d ream +Ġstation ed +opt ional +ew orthy +. +ĠWorks hop +ĠRet ail +ĠAv atar +6 25 +N a +ĠV C +ĠSec ure +M Y +19 88 +oss ip +Ġpro state +Ġund en +Ġg amer +ĠCont ents +ĠWar hammer +ĠSent inel +3 10 +Ġse gregation +ĠF lex +ĠM AY +Ġdr ills +ĠDrug s +Islam ic +Ġsp ur +Ġca fe +Ġimag inary +Ġgu iding +Ġsw ings +ĠThe me +ob y +Ġn ud +Ġbe gging +Ġstr ongh +Ġreject ing +Ġpedest rians +ĠPro spect +R are +s le +Ġconcess ions +ĠConst itutional +Ġbe ams +Ġfib ers +p oon +Ġinstinct s +pro perty +ĠB IG +Sand ers +im ates +Ġco ating +Ġcorps es +ĠTR UE +check ed +Ġ16 6 +A sh +ĠJ S +ĠF iction +Ġcommun al +Ġener getic +oooo oooo +Ġnow adays +IL D +ib o +ĠSU V +R en +Ġdwell ing +Sil ver +Ġt ally +ĠM oving +Ġcow ard +Ġgener als +Ġhorn s +Ġcirc ulated +Ġrob bed +ĠUn limited +Ġharass ed +Ġinhib it +Ġcomp oser +ĠSpot ify +Ġspread s +3 64 +Ġsu icidal +Ġno ises +ĠSt ur +Ġs aga +ĠK ag +is o +Ġtheoret ically +M oney +Ġsimilar ity +Ġslic ed +ut ils +ing es +" - +Ġan th +Ġimp ed +Mod ule +Through out +Ġmen us +comm ittee +and i +ob j +in av +f ired +ĠAb dullah +Ġund ead +Ġfont s +H old +EN G +Ġsustain ability +Ġfl ick +Ġr azor +ĠF est +ĠChar acters +Ġword ing +Ġpopul ist +Ġcritic izing +Ġm use +v ine +Ġcard board +Ġkind ly +Ġfr inge +ĠThe ft +icult ural +Ġgovern ors +Ġ ���� +Ġ16 3 +Ġtime out +ĠA uth +Child ren +A U +Ġred emption +ĠAl ger +Ġ19 14 +Ġw aved +Ġastron auts +og rams +Ġsw amp +ĠFinn ish +Ġcand le +Ġton nes +ut m +Ġr ay +Ġsp un +Ġfear ful +art icles +Ġca us +or ically +ĠRequ ires +ĠG ol +Ġpop e +Ġinaug ural +Ġg le +AD A +ĠIS IL +ĠOff ensive +Ġwatch dog +Ġbal con +ent ity +ĠH oo +Ġgall on +AC C +Ġdoub ling +Ġimpl ication +ĠS ight +Ġdoct r +---- --- +Ġ\ \ +Ġm alt +R oll +Ġâī ¥ +Ġrec ap +add ing +u ces +ĠB end +fig ure +Ġtur key +Ġsoc ietal +ĠT ickets +Ġcommer cially +Ġsp icy +Ġ2 16 +ĠR amp +Ġsuperior ity +à ¯ +ĠTr acker +C arl +ĠC oy +ĠPatri ot +Ġconsult ed +Ġlist ings +Ġsle w +reens hot +ĠG one +Ġ[ ...] +30 9 +Ġh ottest +Ø ± +Ġrock y +ĠD iaz +Ġmass age +Ġpar aly +Ġp ony +A z +Ġcart ridge +ĠN Z +Ġsn ack +ĠLam ar +ple ment +ĠLes lie +Ġm ater +Ġsn ipp +24 6 +Ġjoint ly +ĠBris bane +ĠiP od +Ġpump ing +Ġgo at +ĠSh aron +eal ing +Ġcor on +Ġan omal +rah im +ĠConnect ion +Ġsculpt ure +Ġsched uling +ĠD addy +at hing +Ġeyeb rows +Ġcur ved +Ġsent iments +Ġdraft ing +D rop +( [ +Ġnom inal +ĠLeaders hip +ĠG row +Ġ17 6 +Ġconstruct ive +iv ation +Ġcorrupt ed +ger ald +ĠC ros +ĠChe ster +ĠL ap +ãģ ª +OT H +D ATA +Ġal mond +pro bably +I mp +Ġfe ast +ĠWar craft +F lor +Ġcheck point +Ġtrans cription +Ġ20 4 +Ġtwe aks +Ġrel ieve +S cience +Ġperform er +Z one +Ġtur moil +ig ated +hib it +ĠC afe +the med +Ġflu or +ben ch +Ġde com +ĠU nt +ĠBar rett +ĠF acts +Ġt asting +ĠPTS D +ĠSe al +ĠJuda ism +ĠDynam ic +ĠC ors +V e +ĠM ing +ĠTrans form +v on +ĠDef enders +ĠTact ical +ĠV on +ĠUn ivers +Ġdist orted +ĠB reath +?' " +Ġag on +ĠDead ly +Ġl an +ĠCy cle +orn ed +Ġrel iably +Ġgl or +ĠMon key +ãĥ ¡ +Ġad ren +Ġmicrow ave +ĠAl ban +irc raft +dig it +sm art +ĠD read +¯¯¯¯¯¯¯¯ ¯¯¯¯¯¯¯¯ +{ { +ĠRoc hester +Ġsimpl ified +Ġinf licted +Ġtake over +Ġyour selves +ad itional +Ġmus cular +K S +Ġing en +T ax +ĠFe ature +27 7 +Ġcru c +Ġcr ate +Ġun identified +Ġacclaim ed +ĠM anga +ĠFr ances +ĠNep al +ĠG erald +ĠKu wait +Ġsl ain +ĠHe b +ĠG oku +ãģ® æ +28 6 +M rs +ĠC ody +ĠSan ctuary +01 6 +Ġdism ant +Ġdatas et +ĠH ond +b uck +ĠPat terson +Ġpal ette +ĠG D +ic ol +ĠL odge +Ġplanet ary +ak in +ĠRegist ered +ab we +ĠPeters burg +Ġha iled +ĠP iece +S che +ĠDO J +Ġen umer +18 1 +ĠObs erver +ĠB old +f ounded +com merce +Ġexplo its +ĠF inding +UR N +ĠS ne +ĠAc id +ay ette +ĠVal ues +Ġdr astic +Ġarchitect ural +Ġ" . +× ķ +ump ed +Ġwra pping +Ġwid ow +ĠSl ayer +l ace +on ce +German y +av oid +Ġtem ples +P AR +à ´ +ĠLuc ifer +ĠFl ickr +l ov +for ces +Ġsc outing +Ġlou der +tes y +Ġbefore hand +Ä ĵ +ĠNe on +ĠW ol +ĠTyp ically +ĠPolit ico +-+ -+ +Ġbuild er +Ġder ive +K ill +Ġp oker +Ġambig uous +Ġlif ts +Ġcy t +Ġrib s +ood le +ĠS ounds +h air +ĠSynd rome +t f +Ġproport ional +u id +Ġper taining +ĠKind le +ĠNeg ro +Ġreiter ated +ĠTon ight +oth s +ĠCorn ell +Ġo wing +Ġ20 8 +elf are +oc ating +ĠB irds +Sub scribe +Ġess ays +Ġburd ens +Ġillust rations +ar ious +ER AL +ĠCal cul +Ġx en +ĠLink edIn +ĠJ ung +Ġredes ign +Con nor +29 6 +Ġrevers al +ĠAd elaide +ĠL L +Ġs inking +Ġg um +US H +c apt +ĠGr imm +Ġfoot steps +ĠCB D +isp ers +Ġpro se +Wed nesday +ĠM ovies +ed in +Ġoverturn ed +Ġcontent ious +US B +~~~~~~~~ ~~~~~~~~ +ĠCo pper +Ġpoint less +N V +val ues +olph in +d ain +Ġdepos ited +ĠG W +Ġpreced ed +ĠCl a +ĠGo lem +ĠN im +ĠÎ ² +ĠEngine ers +m iddle +Ġfl att +oper ative +Ġcouncil s +imb abwe +el in +Ġstress ful +ĠL D +Ġres h +l ake +Ġwheel chair +ĠAltern ative +Ġoptim ize +oper ation +Ġpe ek +Ġones elf +ig il +Ġtrans itions +op athy +bl ank +Ġ16 9 +17 1 +________________________________ ________________________________ +Ġl aundering +En c +ĠD EC +Ġwork outs +Ġsp ikes +Ġdin osaurs +Ġdiscrim inatory +P ool +R ather +38 5 +R NA +tes ters +et o +ĠIdent ity +Ġve in +ĠBur ton +Ġarc ade +4 20 +Ult imately +ĠSad ly +à ° +p ill +Ġcub ic +ĠSpect rum +the se +st ates +Ġun official +h awks +ĠEVER Y +Ġrain bow +Ġincarcer ation +and ing +Ġsy ll +ĠEver ton +Ġ17 9 +ĠSer bia +Ġ18 9 +m eter +ĠMic key +Ġant iqu +Ġfact ual +ne ck +ĠN are +n orm +m ust +Ġhigh ways +Ġgl am +Ġdivid ing +ĠSquad ron +ĠMar tha +Ġbirth s +C over +//////// //////// +ĠW ong +Ph ot +ĠA LS +ri o +ĠNon etheless +ĠL emon +Ġ20 6 +ĠE E +Ġderiv ative +ĠWW II +v ote +Ġthere in +Ġsepar ating +44 6 +sy nc +ĠStre ets +Ġr att +Ġmunicip ality +ĠShort ly +Ġmon k +) ," +Ġscr ub +Ġoper atives +Ne ither +Pl ace +ĠLim it +F emale +ĠAct or +Char acter +Ġconstit uted +35 7 +Ġprotest ed +ĠSt raw +ĠHe ight +ild a +ĠTy ph +Ġflood s +Ġcos metic +W AY +pert ure +up on +t ons +ess ing +ĠP ocket +Ġro oft +ĠC aucas +Ġant idepress +Ġincomp atible +EC D +Ġoper a +ĠCont est +Ġgener ators +l ime +Def ense +19 87 +for um +Ġsav age +ĠHung arian +n z +Ġmet allic +Ġex pelled +Ġres idency +Ġdress es +66 6 +ĠC lement +f ires +C ategory +Ġge ek +al is +Ġc emetery +educ ated +Ġc rawl +ĠUn able +ĠT yson +ak is +Ġp ardon +ĠW ra +Ġstrengthen ed +ĠF ors +33 5 +ĠH C +ĠM ond +Ġvisual s +ĠBeat les +ett lement +Ġ ï +g ro +Ġb ash +Ġpo orest +Ġex cel +Ġaspir ations +ĠM unicip +ens ible +Ġceremon ies +Ġintimid ation +ĠCON TR +be ck +ĠK ap +as u +Ġtradem arks +ĠS ew +ĠComp etition +net work +ĠAr ri +ĠT et +Ro aming +W C +D at +Ġso b +Ġpair ing +Ġoverd ose +SA Y +ab er +Ġrev olt +ĠF ah +act ing +e q +est ation +F ight +ĠMar ks +27 3 +Ġ17 8 +R aw +ãģ ĭ +34 9 +bl ocks +Ġver ge +est ine +ĠPod esta +Ġinv asive +Ġprofound ly +ĠA o +e ach +Ġl est +inter pret +Ġshr inking +Ġerr one +Ġche es +ly s +ĠI vy +ĠDirect ory +Ġhint ed +V ICE +Ġcontact ing +ĠG ent +he i +Ġlabel ing +Ġmerc ury +ĠL ite +Ġexp ires +Ġdest abil +rit is +c u +Ġfeather s +Ġste er +Ġprogram med +ĠV ader +Go ing +ĠE lim +Ġy o +ĠMic he +Ġ20 3 +Ġslee ves +Ġb ully +ĠHum ans +36 8 +Ġcomp ress +ĠBan ner +AR S +Ġa while +Ġcal ib +Ġspons orship +ĠDiff iculty +ĠP apers +Ġident ifier +} . +Ġy og +ĠSh ia +Ġclean up +Ġvib e +int rodu +im ming +Austral ia +Ġout lines +ĠY outube +tr ain +ĠM akes +Ġde ported +Ġcent r +ĠD ug +ĠB oulder +ĠBuff y +Ġinj unction +ĠHar ley +ĠG roups +ĠD umbledore +ĠCl ara +Ġ" - +Ġsacrific ed +ep h +Sh adow +ib ling +Ġfreel ance +Ġevident ly +ph al +Ġret ains +M ir +Ġfin ite +d ar +ĠC ous +Ġrep aired +Ġperiod ic +Ġchampions hips +Ġaster oid +bl ind +Ġexpress ly +ĠAst ros +Ġsc aled +Ġge ographical +ĠRap ids +En joy +Ġel astic +ĠMoh amed +Mark et +be gin +Ġdisco vers +Ġtele communications +Ġscan ner +Ġen large +Ġsh arks +Ġpsy chedel +ĠRou ge +Ġsnap shot +is ine +X P +Ġpestic ides +ĠL SD +ĠDist ribution +re ally +Ġde gradation +Ġdisgu ise +Ġbi om +ĠEX T +Ġequ ations +Ġhaz ards +ĠComp ared +) * +Ġvirt ues +Ġeld ers +Ġenh ancing +ĠAc ross +er os +ang ling +Ġcomb ust +ucc i +Ġconc ussion +Ġcontrace ption +ĠK ang +Ġexpress es +Ġa ux +ĠP ione +Ġexhib its +Deb ug +OT AL +ĠAl ready +ĠWheel er +Ġexp ands +? : +Ġreconc iliation +Ġpir ates +Ġpur se +Ġdiscour age +Ġspect acle +R ank +Ġwra ps +ĠTh ought +Ġimp ending +O pp +ĠAng lo +ĠE UR +Ġscrew ed +ret ched +Ġencour agement +mod els +Ġconf use +mm m +ĠVit amin +âĸij âĸij +C ru +Ġkn ights +Ġdisc ard +Ġb ishops +ĠW ear +ĠGar rett +k an +ãĥ Ł +Ġmascul ine +cap ital +ĠA us +Ġfat ally +th anks +ĠA U +ĠG ut +12 00 +Ġ 00000000 +Ġsur rog +ĠBI OS +ra its +ĠWat ts +Ġresur rection +ĠElect oral +ĠT ips +4 000 +Ġnut rient +Ġdepict ing +Ġspr ink +Ġm uff +ĠL IM +ĠS ample +ps c +ib i +gener ated +Ġspec imens +Ġdiss atisf +Ġtail ored +Ġhold ings +ĠMonth ly +ĠE at +po ons +Ġne c +ĠC age +ĠLot us +ĠLan tern +Ġfront ier +Ġp ensions +Ġj oked +ĠHard y +=-=- =-=- +r ade +U ID +Ġr ails +Ġem it +Ġsl ate +Ġsm ug +Ġsp it +ĠCall s +ĠJac obs +f eat +ĠU E +Ġrest ruct +Ġregener ation +Ġenerg ies +ĠCon nor +OH N +ĠChe ese +Ġg er +Ġresur rect +man agement +N W +Ġpres ently +ĠBru ins +M ember +ĠM ang +id an +Ġboost ing +w yn ++ . +requ isite +ĠNY PD +ĠMe gan +ĠCond itions +Ġp ics +nes ium +ĠR ash +Ġ17 4 +ĠD ucks +Ġemb ro +z u +on ian +rel igious +Ġc raz +ĠAC A +ĠZ ucker +EM A +ĠPro s +We apon +ĠKn ox +ĠAr duino +Ġst ove +Ġheaven s +ĠP urchase +Ġher d +Ġfundra iser +Dig ital +5 000 +Ġprop onents +/ âĢĭ +Ġj elly +ĠVis a +Ġmon ks +Ġadvance ment +ĠW er +Ġ18 7 +e us +ert ility +Ġfet al +Ġ19 36 +L o +Ġout fits +Ġstair case +b omb +Ġcustom ized +cl air +T ree +Ġm apped +ĠConsider ing +ĠTor res +Ġmeth yl +Ġapprox imate +Ġdo om +ĠHans en +Ġc rossover +Ġstand alone +ä ¼ +Ġinv ites +Ġgra veyard +Ġh p +Donald Trump +Ġesc ort +G ar +Ġpredec essors +Ġh ay +Ġen zyme +ĠStra ight +vis ors +I ng +ane ously +ĠApp lied +Ġf ec +ĠDur ant +Ġout spoken +or b +Ġz eal +Ġdisgr ace +' ). +ĠChe ng +28 9 +ĠRen a +ĠSu icide +29 4 +Ġout raged +ĠNew man +ĠN vidia +ĠA ber +ĠB ers +Ġrecre ation +Wind ow +ĠD P +x e +Ġped oph +Ġfall out +ambo o +Ġpresent ations +ĠApp s +Ġh tml +3 45 +ĠX XX +Ġrub bing +ĠLe ather +Ġhum idity +se ys +est ablished +ĠUn its +64 6 +Ġrespect able +A uto +Ġthri ving +ĠInn ovation +ang s +Ext ra +reg ulation +29 8 +p ick +Ex amples +ĠC J +Att ack +Ġdr acon +L T +Ġstick er +re rs +Ġsun ny +I ss +reg ulated +d im +ĠAb stract +Ġhus bands +Off ice +om ination +it ars +AN GE +asc al +ĠK ris +ĠInf antry +Ġm alf +ĠA the +ĠR ally +bal anced +................ ........ +OU P +Ġmole cule +met ics +ĠSpl it +ĠInstruct ions +ĠN ights +c ards +Ġt ug +Ġcon e +å Ń +Ġt x +ĠDisc ussion +Ġcatast rophe +pp e +g io +Ġcommun ism +Ġhal ted +ĠGu ant +cle an +ĠSc hed +ĠK anye +Ġw ander +ĠSer iously +Ġ18 8 +enn ial +f ollow +product ive +ĠFl ow +ĠS ail +Ġc raw +Ġsim ulations +or u +ang les +ĠN olan +Ġmen stru +4 70 +Ġ20 7 +aj a +Ġcas ually +board ing +Ġ2 22 +ov y +ĠN umbers +um at +O E +28 7 +ĠCle mson +Ġcert s +Ġsl id +ĠT ribe +Ġto ast +Ġfort unes +Ġf als +ĠComm ittees +Ġg p +Ġf iery +ĠN ets +ĠAn ime +Pack age +ĠComp are +l aughter +in fect +Ġatroc ities +Ġjust ices +Ġins ults +ĠVern on +Ġsh aken +Ġperson a +est amp +36 7 +br ain +Ġexperiment ing +K en +ĠElect ronics +Ġ16 1 +dom ain +Ġgraph ical +b ishop +Ġwho pping +ĠEv angel +Ġadvertis ers +ĠSpe ar +Ġb ids +Ġdestro ys +ut z +Ġunders c +ĠAD D +Ġan ts +ĠC um +ipp les +ĠF ill +Ġgl anced +Ġind icted +ĠE ff +Ġmis con +ĠDes ktop +Ġab ide +ãĥ Ģ +ĠI o +ĠC oul +Ġcaps ule +ĠCh rys +M ON +Ġund es +ĠI RA +Ġc itation +Ġdict ate +ĠNet works +ĠConf lict +ĠSt uff +x a +is ec +ĠChem istry +Ġquarter ly +William s +an an +O pt +ĠAlexand ria +out heastern +ĠSpring field +ĠBlack s +Ġge ography +24 2 +Ġut most +ĠEx xon +ab outs +E VA +ĠEn able +ĠBar r +Ġdisag reed +ĠCy prus +Ġdement ia +Ġlab s +Ġubiqu itous +ĠLO VE +Ġconsolid ated +s r +Ġcream y +ĠTim ber +Reg ardless +ĠCert ificate +Ġ" ... +ogen ous +Capt ain +Ġinsult ing +ĠSor os +ĠInst r +ĠBulgar ia +bet ter +Ġsuck ing +ĠDavid son +at z +Ġcoll ateral +g if +Ġplag ued +ĠC ancel +ĠGard ner +R B +Ġsix teen +Rem ove +ur istic +c ook +R od +Ġcompr ising +f le +) âĢĶ +ĠVik ing +g rowth +agon al +Ġsr f +af ety +m ot +N early +st own +ĠF actor +Ġautom obile +Ġproced ural +m ask +amp ires +Ġdisapp ears +j ab +3 15 +Ġ19 51 +ne eded +Ġd aring +le ader +Ġp odium +Ġun healthy +Ġm und +Ġpy ramid +oc re +Ġkiss ed +Ġdream ed +ĠFant astic +ĠG ly +å Ĭ +Ġgreat ness +Ġsp ices +Ġmet ropolitan +Ġcomp uls +i ets +101 6 +ĠSh am +ĠP yr +fl ies +ĠMid night +Ġswall owed +Ġgen res +ĠL ucky +ĠRew ards +Ġdisp atch +ĠI PA +ĠApp ly +Ġa ven +al ities +3 12 +th ings +Ġ( ). +Ġm ates +ĠS z +ĠC OP +ol ate +O FF +Ġre charge +c aps +ĠYork er +ic one +Ġgal axies +ile aks +D ave +ĠP uzz +ĠCelt ic +ĠA FC +27 6 +ĠS ons +Ġaffirm ative +H or +Ġtutorial s +ĠC ITY +ĠR osa +ĠExt ension +Ser ies +Ġf ats +Ġr ab +l is +Ġun ic +Ġe ve +ĠSp in +Ġadul thood +ty p +Ġsect arian +Ġcheck out +ĠCy cl +S ingle +Ġmart yr +Ġch illing +88 8 +ou fl +Ġ] ; +Ġcongest ion +m k +ĠWhere as +Ġ19 38 +ur rencies +er ion +Ġbo ast +ĠPat ients +Ġch ap +ĠB D +real DonaldTrump +Ġexam ines +h ov +Ġstart ling +ĠBab ylon +w id +om ew +br ance +ĠOd yssey +w ig +Ġtor ch +ĠV ox +ĠMo z +ĠT roll +ĠAn s +Similar ly +ĠF ul +00 6 +Un less +ĠAl one +st ead +ĠPub lisher +r ights +t u +ĠDoes n +Ġprofession ally +Ġcl o +ic z +Ġste als +Ġ á +19 86 +Ġst urdy +ĠJoh ann +Ġmed als +Ġfil ings +ĠFr aser +d one +Ġmult inational +Ġf eder +Ġworth less +Ġp est +Yes terday +ank ind +Ġg ays +Ġb orne +ĠP OS +Pict ure +Ġpercent ages +25 1 +r ame +Ġpot ions +AM D +ĠLeban ese +Ġr ang +ĠL SU +ong s +Ġpen insula +ĠCl ause +AL K +oh a +ĠMac Book +Ġunanim ous +Ġl enders +Ġhang s +Ġfranch ises +ore rs +ĠUp dates +Ġisol ate +and ro +S oon +Ġdisrupt ive +ĠSur ve +Ġst itches +ĠSc orp +ĠDomin ion +Ġsupp lying +Ar g +Ġtur ret +ĠL uk +Ġbr ackets +* ) +ĠRevolution ary +ĠHon est +Ġnot icing +ĠSh annon +Ġafford ed +Ġth a +ĠJan et +! -- +ĠNare ndra +ĠPl ot +H ol +se ver +e enth +Ġobst ruction +Ġ10 24 +st aff +j as +or get +sc enes +l aughs +ĠF argo +cr ime +Ġorche str +Ġde let +ili ary +rie ved +Ġmilit ar +ĠGreen e +âĹ ı +ãģ ¦ +ĠGu ards +Ġunle ashed +ĠWe ber +Ġadjust able +Ġcal iber +Ġmotiv ations +Ġà ł +m Ah +ĠL anka +hand le +Ġp ent +ĠR av +ĠAng ular +ĠK au +umb ing +Ġphil anthrop +Ġde hyd +Ġtox icity +e er +ĠY ORK +w itz +å ¼ +ĠI E +commun ity +ĠA H +Ġret ali +Ġmass ively +ĠDani els +ĠD EL +Ġcar cin +Ur l +Ġrout ing +ĠNPC s +ĠR AF +ry ce +Ġwa ived +ĠGu atem +Every body +Ġco venant +Ġ17 3 +Ġrelax ing +Ġqu art +al most +Ġguard ed +ĠSold iers +ĠPL AY +Ġout going +L AND +Ġre write +ĠM OV +ĠIm per +ĠS olution +Ġphenomen al +Ġl ongevity +Ġimp at +ĠN issan +ir ie +Ġod or +ĠZ ar +ok s +Ġmilit ias +ĠSP EC +Ġtoler ated +ars er +ĠBrad ford ++ , +Ġsur real +s f +Can adian +Ġresemb lance +Ġcarbohyd rate +VI EW +Ġaccess ory +me al +larg est +ieg el +Some one +Ġtoug hest +os o +Ġfun nel +Ġcondemn ation +lu ent +Ġw ired +ĠSun set +Jes us +ĠP ST +ĠP ages +ĠTy coon +ĠP F +Ġselect ions +Ġ ठ+part isan +Ġhigh s +ĠR une +Ġcraft s +le ad +ĠParent s +Ġre claim +ek er +ĠAll ied +ae per +Ġlo oming +Ġbenefic iaries +ĠH ull +Stud ents +Jew ish +d j +Ġp act +tem plate +ĠOffic ials +ĠBay lor +Ġhe mp +Ġyouth s +ĠLevel s +ĠX iao +ĠC hes +Ġende avor +ĠRem oved +Ġhipp ocamp +H ell +ãĤ Ĭ +80 5 +Ġd inosaur +ĠWr ath +ĠIndones ian +Ġcalcul ator +ĠD ictionary +Ġ4 20 +ĠM AG +( _ +! , +t arians +Ġrestrict ing +rac use +Ġweek day +OU NT +Ġsh rugged +leg round +Ġb ald +ĠDo ctors +Ġt outed +ĠMax well +Ġ2 14 +Ġdiplom at +Ġrep ression +Ġconstitu ency +v ice +r anked +ĠNap oleon +g ang +ĠFore ver +t un +Ġbul b +ĠPD T +ĠC isco +V EN +Ġres umed +Ste ven +ĠManit oba +Ġfab ulous +ĠAg ents +19 84 +Ġam using +ĠMyster ies +Ġor thodox +fl oor +Ġquestion naire +Ġpenet rate +Ġfilm makers +ĠUn c +Ġst amped +Ġth irteen +Ġout field +Ġforward ed +Ġapp ra +Ġa ided +t ry +Ġunf ocused +ĠL iz +ĠWend y +ĠSc ene +Ch arg +Ġreject s +Ġleft ist +ĠProv idence +ĠBr id +reg n +Ġprophe cy +ĠL IVE +4 99 +Ġfor ge +ĠF ML +Ġintrins ic +ĠF rog +Ġw ont +ĠH olt +Ġfam ed +CL US +aeper nick +ĠH ate +ĠC ay +Ġregister ing +ort ality +rop y +ocaly ptic +a an +n av +Ġfasc ist +IF IED +Ġimpl icated +ĠRes ort +ĠChand ler +ĠBr ick +P in +ys c +Us age +ĠHel m +us ra +âĺħ âĺħ +ĠAb bas +Ġunanim ously +Ġke eper +Ġadd icted +?? ? +Ġhelm ets +Ġant ioxid +aps ed +80 8 +gi ene +Ġwa its +Ġmin ion +ra ved +ĠP orsche +Ġdream ing +Ġ17 1 +ĠC ain +Ġun for +ass o +ĠConfig uration +k un +hard t +Ġn ested +ĠL DS +L ES +Ġt ying +en os +Ġc ue +ĠMar qu +sk irts +Ġclick ed +Ġexp iration +ĠAccording ly +ĠW C +Ġbless ings +Ġaddict ive +ĠN arr +y x +ĠJagu ars +Ġrent s +ĠS iber +Ġt ipped +ous se +ĠFitz gerald +Ġhier arch +out ine +Ġwa velength +> . +ch id +ĠProcess ing +/ + +r anking +E asy +ĠConst ruct +Ġt et +ins ured +H UD +Ġqu oting +Ġcommun icated +in x +Ġin mate +Ġerect ed +ĠAbs olutely +ĠSure ly +Ġun im +ĠThr one +he id +Ġcl aws +Ġsuper star +ĠL enn +ĠWh is +U k +ab ol +Ġsk et +ĠN iet +Ġper ks +Ġaff inity +Ġopen ings +phas is +Ġdiscrim inate +T ip +v c +Ġgr inding +ĠJenn y +Ġast hma +hol es +ĠHom er +Ġreg isters +ĠGl ad +Ġcre ations +Ġlith ium +Ġappl ause +unt il +Just ice +ĠTur ks +Ġsc andals +Ġb ake +t ank +M ech +ĠMe ans +ĠM aid +Republic ans +is al +wind ows +ĠSant os +Ġveget ation +33 8 +t ri +Ġfl ux +ins ert +Ġclar ified +Ġmort g +ĠCh im +ĠT ort +Ġdiscl aim +met al +ĠAs ide +Ġindu ction +Ġinf l +Ġathe ists +amp h +Ġe ther +ĠV ital +ĠBu ilt +M ind +Ġweapon ry +S ET +Ġ18 6 +ad min +g am +cont ract +af a +Ġderiv atives +Ġsn acks +Ġch urn +E conom +Ġca pped +ĠUnder standing +ĠH ers +ĠI z +Ġd uct +I ENT +augh ty +Ġâľ Ķ +ĠN P +Ġsa iling +In itialized +Ġt ed +Ġreact ors +ĠL omb +Ġcho ke +ĠW orm +Ġadm iration +Ġsw ung +ens ibly +Ġr ash +ĠGo als +ĠImport ant +Sh ot +ĠR as +Ġtrain ers +ĠB un +Work ing +Ġhar med +ĠPand ora +ĠL TE +Ġmush room +ĠCH AR +ĠF ee +ĠM oy +B orn +ol iberal +ĠMart ial +Ġgentle men +Ġling ering +Offic ial +Ġgra ffiti +ĠN ames +D er +Ġqu int +ist rate +aze era +ĠNOT ICE +ĠFlore nce +Ġpay able +Ġdep icts +ĠSpe cies +He art +âĶĢâĶĢâĶĢâĶĢ âĶĢâĶĢâĶĢâĶĢ +Ġencl osed +Incre ases +D aily +ĠL is +Ġenact ment +ĠB acon +ĠSt eele +dem and +Ġ18 3 +Ġmouth s +Ġstr anded +Ġenhance ment +01 1 +ĠWh ats +Ġhe aled +en y +ĠR ab +Ġ3 40 +ĠLab yrinth +ro ach +ĠY osh +ĠCl ippers +Ġconcert s +Intern et +35 5 +Ġstick ers +Ġter med +ĠAx e +Ġgrand parents +Fr ance +ĠCl im +ĠU h +ul ic +Ġthr ill +cent ric +ĠOver view +ĠCond uct +Ġsubstant ive +Ġ18 2 +m ur +Ġstr ay +ĠCo ff +Ġrep etitive +ĠFor gotten +Ġqual ification +ew itness +ĠZ imbabwe +Ġsim ulated +ĠJ D +25 3 +ĠW are +Ġun sc +T imes +Ġsum mons +Ġdis connected +Ġ18 4 +ci us +ĠGu jar +od ka +Ġer ase +ĠTob acco +elect ed +Ġun cont +ĠShe pard +ĠL amp +Ġalert ed +Ġoper ative +arn a +u int +Ġneglig ence +ac ements +Ġsup ra +Ġprev ail +ĠSh ark +Ġbel ts +ãģ « +Ġt ighter +Engine ers +Ġin active +Ġexp onent +ĠWill ie +a ples +Ġhe ir +ĠH its +ian n +ĠS ays +Ġcurrent s +ĠBeng al +Ġar ist +B uffer +Ġbree ze +ĠWes ley +Col a +Ġpron oun +Ġde ed +ĠK ling +Ġof t +Ġinf lict +Ġpun ishing +Ġn m +ik u +OD UCT +01 4 +Ġsubsid y +ĠDE A +ĠHer bert +ĠJ al +B ank +Ġdef erred +Ġship ment +B ott +Ġal le +b earing +HT ML +Off line +Ġ2 13 +Ġscroll ing +Ġsc anned +ĠLib yan +ĠT OP +ch rom +d t +col umn +Psy NetMessage +Z ero +Ġtor so +0 50 +âķ IJ +Ġimp erson +ĠSchw artz +ud ic +Ġpiss ed +ĠS app +25 7 +ĠIS Ps +og l +Ġsuper vised +Ġad olescent +Ġatt ained +ĠDel ivery +ĠB unny +Ġ19 37 +Ġmini ature +Ġo s +Ġ3 70 +60 8 +ĠMour inho +Ġinn ate +Ġtem po +ĠN M +ĠFall en +00 9 +Ġprov ocative +Stream er +ĠBened ict +ĠBol she +Ġt urtle +ĠPC B +ĠEqu al +Direct or +ĠR end +Ġflu ids +Author ities +Ġcous ins +requ ency +ĠNeigh bor +s ets +sh ared +Char les +pass word +Ġg ears +Ġ2 11 +ĠHard ware +ri ka +Ġup stream +H om +Ġdisproportion ately +iv ities +Ġund efined +Ġelect rons +Ġcommem or +Event ually +Ġ> < +Ġir responsible +2 18 +ĠRe leased +ĠO VER +ĠI GN +ĠB read +st ellar +ĠS age +tt ed +dam age +ed ition +ĠPre c +Ġl ime +Ġconf inement +Ġcal orie +we apon +Ġdiff ering +ĠS ina +m ys +am d +Ġintric ate +k k +ĠP AT +ã o +st ones +lin ks +Ġr anch +Sem itic +Ġdifferent iate +ĠS inger +occup ied +Ġfort ress +c md +Ġinter ception +ĠAnk ara +Ġre pt +ĠSol itaire +Ġrem ake +p red +Ġd ared +aut ions +ĠB ACK +Run ning +Ġdebug ging +Ġgraph s +3 99 +ĠNig el +Ġb un +Ġpill ow +Ġprog ressed +fashion ed +Ġob edience +ER N +Ġrehe ars +C ell +t l +S her +Ġher ald +ĠPay ment +ĠC ory +ĠDe pt +Ġrep ent +ĠWe ak +uck land +Ġple asing +Ġshort ages +Ġjur ors +ĠK ab +q qa +Ant i +Ġw ow +ĠRC MP +Ġt sun +ĠS ic +Ġcomp rises +Ġsp ies +Ġprec inct +n u +Ġur ges +Ġtim ed +Ġstrip es +ĠB oots +Ġy en +Adv anced +Ġdisc rete +ĠArch angel +employ ment +D iff +Ġmon uments +Ġ20 9 +work er +Ġ19 6 +ĠI g +utter stock +T PS +J ac +Ġhomeless ness +Ġcomment ator +Ġrac ially +f ing +se ed +E le +ell ation +Ġeth anol +Ġpar ish +ĠD ong +ĠAw akening +Ġdev iation +ĠB earing +ĠTsu k +Ġrec ess +Ġl ymph +ĠCann abis +å ľ +ĠNEW S +Ġd ra +ĠStef an +ĠWr ong +ĠS AM +Ġloose ly +Ġinterpre ter +ĠPl ain +Go vernment +Ġbigot ry +Ġgren ades +ave z +pict ured +Ġmand ated +ĠMon k +ĠPed ro +Ġl ava +27 4 +Ġcyn ical +ĠScroll s +l ocks +M p +Ġcon gregation +orn ings +ph il +ĠI bid +Ġf erv +Ġdisapp earing +Ġarrog ant +sy n +ĠMa ver +ĠSu it +24 1 +Ġab bre +ack ers +P a +ĠY el +Whe never +Ġ23 5 +ĠV ine +ĠAn at +Ġext inct +LE T +Ġexecut able +V ERS +ox ide +D NA +ĠP rel +Ġresent ment +Ġcompr ise +ĠAv iv +Ġinter ceptions +Ġprol ific +IN A +ĠEr in +though t +2 19 +ĠPsychiat ry +un ky +chem ist +H o +ĠMcC oy +Ġbr icks +L os +ri ly +ĠUS SR +Ġr ud +Ġl aud +ĠW ise +ĠEmer ald +Ġrev ived +Ġdam ned +ĠRep air +id em +ct ica +Ġpatri arch +ĠN urs +me g +Ġcheap est +re ements +empt y +ĠCele br +Ġdepri vation +ch anted +ĠTh umbnails +E nergy +ĠEth an +ĠQ ing +Ġopp oses +W IND +v ik +ĠM au +ĠS UB +66 7 +G RE +ĠVol unte +nt on +C ook +å IJ +es que +Ġplum met +Ġsu ing +Ġpron ounce +Ġresist ing +ĠF ishing +ĠTri als +Ġy ell +Ġ3 10 +Ġin duct +Ġpersonal ized +oft en +R eb +EM BER +Ġview point +Ġexist ential +() ) +rem ove +MENT S +l asses +Ġev apor +Ġa isle +met a +Ġreflect ive +Ġentit lement +Ġdev ised +mus ic +asc ade +Ġwind ing +off set +Ġaccess ibility +ke red +Bet ter +ĠJohn ston +th inking +S now +ĠCroat ia +ĠAt omic +27 1 +34 8 +Ġtext book +ĠSix th +Ġ اÙĦ +Ġsl ider +ĠBur ger +b ol +S ync +Ġgrand children +Ġc erv ++ ) +Ġe ternity +Ġtweet ing +Ġspec ulative +Ġpiv otal +ĠW P +ĠT ER +ynam ic +Ġu pl +ĠC ats +per haps +Ġclass mates +Ġblat ant +' - +Ġl akh +ant ine +ĠB org +i om +/ ( +ĠAthlet ic +Ġs ar +OT A +ĠHoff man +Never theless +Ġad orable +Ġspawn ed +Ass ociated +ĠDom estic +Ġimpl ant +ĠLux em +ĠK ens +Ġp umps +ĠS AT +Att ributes +50 9 +av our +Ġcentral ized +ĠT N +Ġfresh ly +ĠA chieve +Ġouts iders +her ty +ĠRe e +ĠT owers +ĠD art +ak able +Ġm p +ĠHeaven ly +Ġr ipe +ĠCarol ine +ry an +Ġclass ics +Ġret iring +Ġ2 28 +Ġa h +Ġdeal ings +Ġpunch ing +ĠChap man +O ptions +max well +vol ume +Ġst al +Ġex ported +ĠQu ite +Ġnumer ical +B urn +F act +ĠKey stone +Ġtrend ing +Ġalter ing +ĠAfric ans +47 8 +ĠM N +ĠKn ock +Ġtempt ation +Ġprest ige +Over view +ĠTrad itional +ĠBah rain +Priv ate +ĠH OU +Ġbar r +ĠT at +C ube +US D +ĠGrand e +ĠG at +ĠFl o +Ġres ides +Ġind ec +vol ent +Ġperpet ual +ub es +Ġworld view +ĠQuant um +Ġfil tered +Ġen su +orget own +ERS ON +ĠM ild +37 9 +OT T +à ¥ +Ġvit amins +Ġrib bon +Ġsincere ly +ĠH in +Ġeight een +Ġcontradict ory +Ġgl aring +Ġexpect ancy +Ġcons pir +Ġmon strous +Ġ3 80 +re ci +Ġhand ic +Ġpump ed +Ġindic ative +Ġr app +Ġav ail +ĠLEG O +ĠMar ijuana +19 85 +ert on +Ġtwent ieth +################ ################ +ĠSw amp +Ġval uation +Ġaffili ates +adjust ed +ĠFac ility +26 2 +Ġenz ymes +itud inal +Ġimp rint +S ite +Ġinstall er +ĠT RA +m ology +lin ear +ĠCollect ive +ig ating +ĠT oken +Ġspec ulated +K N +ĠC ly +or ity +Ġdef er +Ġinspect ors +appro ved +R M +ĠSun s +Ġinform ing +ĠSy racuse +ib li +7 65 +Ġgl ove +Ġauthor ize +âĢ¦âĢ¦âĢ¦âĢ¦ âĢ¦âĢ¦âĢ¦âĢ¦ +ĠCru ise +Ġcontract ing +she ll +IF E +ĠJew el +p ract +ĠPhot oshop +ĠKnow ing +h arm +Ġattract ions +ad an +et us +01 8 +w agen +Al t +Ġmultip ly +Ġequ ilibrium +: { +ĠF ighters +ĠEd gar +Ġfour teen +Go vern +Ġmis use +Ġab using +Ġancest ry +ram er +64 4 +Ġwor ms +Ġthick er +ĠComb ine +Ġpeas ants +Ġv ind +Ġcon quest +Ġm ocked +Ġc innamon +ĠC ald +ĠGall up +Ġavoid ance +Ġincarn ation +ĠStr at +Ġt asted +ent a +ĠN eal +p ared +Ġtermin ology +ject ion +Scient ists +ĠIN S +ĠDe e +Ġdirect ories +R oad +ĠSh ap +br ight +ĠDirect ors +ĠCol umn +Ġb ob +Ġprefer ably +Ġgl itch +f urt +Ġe g +id is +C BC +Ġsur rendered +Ġtest ament +33 6 +ug gest +ĠN il +an other +Ġpat hetic +ĠDon na +Ġ2 18 +ĠA very +Ġwhis key +Ġf ixture +ĠCon quest +Ġbet s +O cc +ĠLe icester +] ." +Ġ) ); +Ġfl ashes +45 6 +Ġmask ed +ge bra +Ġcomput ed +che l +aud er +Ġdefe ats +ĠLiber ation +ĠOs ama +ĠV ive +Ch anges +Ch annel +Ġtar iffs +Ġm age +ĠS ax +Ġinadvert ently +ĠC RE +ĠRe aper +ink y +gr ading +Ġstere otyp +Ġcur l +ĠF ANT +Ġfram eworks +M om +ĠAn ch +Ġflav our +car bon +Ġperm itting +let cher +ĠMo zilla +ĠPark ing +ĠCh amp +Sc roll +Ġmurd erer +Ġrest ed +Ġow es +ĠP oss +AD D +IF F +res olution +ĠMin ing +Ġcompar ative +D im +Ġneighbour ing +ĠA ST +ĠT oxic +Ġbi ases +Ġgun fire +ur ous +ĠMom ent +19 83 +Ġper vasive +tt p +ĠNorm ally +r ir +S arah +ĠAlb any +Ġun sett +ĠS MS +ip ers +l ayer +ĠWh ites +up le +Ġtur bo +ĠLe eds +Ġthat s +ĠMin er +M ER +ĠRe ign +Ġper me +ĠBl itz +Ġ19 34 +Ġintimid ating +t ube +Ġecc entric +ab olic +box es +ĠAssoci ates +v otes +Ġsim ulate +um bo +aster y +Ġship ments +FF FF +an th +Ġseason ed +Ġexperiment ation +âĸ ł +law s +Me et +idd les +ant ics +R ating +IS IS +h ift +Ġfront s +b uf +01 7 +Ġun att +ĠD il +le ases +ĠGard ens +77 7 +t ouch +ve ll +45 8 +Ġ= ==== +s aving +Ġer osion +ĠQu in +Ġearn s +Ġaccomplish ment +ĠWe i +Ġ< [ +____ _ +Ġir rig +ĠT eddy +Ġconqu ered +ĠArm ored +Ġassert s +Ġmanip ulating +r é +Ġtranscript s +G allery +Ġplot ting +Ne il +Ġbetray al +load er +ĠS ul +Ġdispl acement +Ġroy alty +ĠW I +he it +ĠDev ices +alle l +Ġmunicipal ities +Ġcan al +St ars +ĠU AE +Ġ" âĢ¦ +ĠC U +ab ove +Ġreson ance +ĠguiActive Un +add ed +ĠBra ves +ĠI bn +Ġhere by +ĠB RE +Ġshare holder +ĠH ir +ĠJ i +Ġstrange ly +Ġadm ired +Ġpl ight +Ġb achelor +ĠP ole +cipl inary +T ony +ĠArmen ian +Ġun man +ĠZion ist +St age +isco ver +Ġautom otive +Ġs idelines +Ġsl ick +ĠRena issance +ĠF UN +Im ages +ĠH aj +Ġp ing +Ġshort cut +ĠBl vd +ĠLook s +Ġbur sts +Ġcl amp +Ġm ish +Ġsort ing +Ġpatri ot +Ġcorrect ness +ĠScand inav +ĠCaval iers +p ython +az ar +Ġ3 75 +ĠJa une +40 9 +Ġdetrim ental +Ġstab bing +Ġpoison ed +Ġf ountain +oc ent +or st +ĠMar i +Ġr ains +ĠO vers +ĠInst itution +ud get +AM Y +t ale +ĠK R +ĠPr ices +Ġhead aches +Ġlands l +ĠA ura +Bon us +ĠZ hao +ĠH ip +Ġhop s +ĠKurd istan +Ġexplo iting +ry n +Ġhypocr isy +op ening +Ġgun shot +Ġw ed +inter stitial +Inter stitial +Ġam en +Bre aking +Ġmarket ed +W ire +ĠC rowd +Contin ue +ĠK nown +ĠEffect ive +ore an +iz ons +Jose ph +Ġescal ation +us ername +Ġcur tain +AT ES +ĠP AR +ĠM iy +Ġcounter fe +l ene +Ġcont enders +d aily +ĠAs c +ĠPhill ip +most ly +Ġfil ename +he ne +Ġresemb ling +Ġst aging +ĠCh loe +Ġw iring +H on +ĠRen ew +ott age +ĠHy brid +m uch +Ġstro kes +Ġpolicy makers +AP TER +ĠArk ham +pl ot +Ġassist ants +Ġde port +ĠSe ga +Ġinflu enza +ĠC ursed +ĠK obe +Ġskin ny +Prov ider +ĠR ip +Ġincrement al +product s +B F +Ġd ome +ĠC redits +Ġlos ers +int s +ĠBet ty +ĠTal ent +ĠD AM +L v +E ss +Ġd ens +tem p +J udge +od ic +Ġ' ( +UR ES +ets k +V O +Ġretrie ved +Ġarchitect s +Ù ĩ +Ġeth ic +ĠSecond ary +st ocks +ad ia +Ġ3 25 +ĠOp inion +Ġsimultane ous +Ġd izz +ul p +Ġsmugg ling +ipp ery +R andom +f acing +ĠD as +Ġstock p +Ġdiscl osures +po inter +Ġcor al +ĠSe lection +ĠP ike +ival ent +Ġruth less +ĠR im +Ġensu ing +ĠExper iment +Ġcongress man +Ġbelie ver +Ġun specified +ĠM ord +Ġknowledge able +ĠV ERY +T X +Ġstra ps +Ġtur f +apesh ifter +Ġmar ital +Ġfl ock +ãģ Ĩ +26 3 +AM ES +ĠOpp osition +Ġtre asures +ĠG OD +Ġmodel ed +ĠWOR LD +Ġ( [ +ĠUs age +H F +Ġ$ ( +uss ed +Ġpione er +E ight +par se +b read +rit z +ĠMir anda +ĠK ant +++ ) +ore n +Ġprov oked +Ġbre eds +ĠIn cludes +ĠPast ebin +ĠFl ip +J ava +Ġbr ink +Ġrum ored +Ġun seen +Ġgar nered +ĠDef in +al ted +Ġtatt oos +Ġhes itation +is itions +ĠWe aver +ĠReport ing +Ġtherap ies +Ġconsult ants +Ġresid ual +ĠMal i +ĠRom a +i ago +ĠRes idents +ub i +Ġremed ies +Ġadapt ive +ĠAl ive +ĠBar cl +Ġwal lets +c rypt +etermin ation +ĠPel osi +Ġsl ipping +oton in +Ġall iances +pat rick +ir is +Ġor th +ĠPer kins +ĠDe V +ĠG ets +Ġdry ing +ge e +fore st +ĠFor get +ore m +33 9 +Ġvague ly +ĠD ion +ĠP orn +ĠH OW +Ġp neum +Ġrub ble +ĠT aste +enc ia +ĠG el +Ġd st +Ġ24 5 +ĠMoroc co +inf lamm +ĠTw ins +Ġb ots +d aughter +ĠB alk +Ġbre thren +Ġlog os +Ġgo bl +f ps +Ġsub division +Ġp awn +Ġsquee zed +Ġmor ale +ĠD W +' " +Ġkn ot +ook y +Ġdiv isive +Ġboost ed +ch y +ãĥ IJ +if act +Ġnewcom ers +ĠWrest ling +Ġsc outs +w olves +R at +Ġnin eteenth +ĠOs borne +St ats +Ġem powered +Ġpsych opath +ĠO EM +ugg age +ĠP K +ĠMoh ammad +P ak +Ġanarch ists +ĠExt ract +est hes +ĠStock holm +l oo +ĠG raph +Ġdeploy ing +ĠStr anger +ĠM old +Ġstaff er +Ġdiscount ed +uck le +ple ase +ĠLand ing +ÃŃ a +Ġ19 3 +Ġan te +Ġrep etition +Ġ+ /- +Ġpar ody +Ġlive ly +AA A +ĠHor us +Ġp its +ind ers +L OC +ĠVen ice +40 6 +ĠDis cover +â Ĩ +ellect ual +Ġp ens +Ġey el +ig uous +Im pl +Ġj oking +Ġinv al +ĠBel fast +Ġcredit ors +ĠSky walker +ov sky +Ġcease fire +Ġse als +is oft +) ). +ĠFel ix +IT S +Ġt resp +ĠBlock chain +ew are +ĠSch war +en ne +mount ed +ĠBe acon +les h +Ġimmense ly +Ġche ering +Em ploy +sc ene +ish ly +atche wan +ĠNic olas +Ġdr ained +ĠEx it +ĠAz erb +j un +Ġflo ated +u ania +De ep +Ġsuper v +Ġmyst ical +ĠD ollar +ĠApost le +ĠR EL +ĠProv ided +ĠB ucks +ãĥ ´ +cut ting +Ġenhance ments +ĠPengu ins +ĠIsa iah +Ġj erk +ĠW yn +Ġst alled +Ġcryptoc urrencies +ĠR oland +sing le +Ġl umin +ĠF ellow +ĠCap acity +ĠKaz akh +W N +Ġfin anced +38 9 +Ġt id +Ġcoll usion +ĠMy r +î Ģ +Sen ator +Ġped iatric +Ġneat ly +Ġsandwic hes +ĠArchitect ure +Ġt ucked +Ġbalcon y +Ġearthqu akes +qu ire +F uture +Ġhe fty +é Ĺ +Ġspecial izes +Ġstress es +Ġs ender +Ġmisunder standing +Ġep ile +Ġprov oke +ĠCol ors +Ġdis may +uk o +[ _ +58 6 +ne utral +Ġdon ating +ĠRand all +Mult i +Ġconvenient ly +ĠS ung +ĠC oca +Ġt ents +ĠAc celer +Ġpart nered +27 2 +ir ming +ĠB AS +s ometimes +Ġobject ed +ub ric +p osed +LC S +gr ass +Ġattribut able +V IS +Israel i +Ġrepe ats +ĠR M +v ag +ut a +in ous +Ġin ert +ĠMig uel +æ Ń +ĠHawai ian +B oard +Ġart ific +ĠAzerb ai +as io +ĠR ent +A IN +Ġappl iances +Ġnational ity +Ġass hole +ĠN eb +Ġnot ch +h ani +ĠBr ide +Av ailability +Ġintercept ed +Ġcontin ental +Ġsw elling +ĠPers pect +b ies +. < +ith metic +ĠL ara +Ġtempt ing +add r +Ġoversee ing +cl ad +ĠD V +ĠGing rich +Ġm un +ĠApp ropri +Ġalter ations +ĠPat reon +Ġha voc +Ġdiscipl ines +Ġnotor iously +aku ya +ier i +? ). +ĠW ent +Ġsil icon +Ġtre mb +Cont ainer +K nown +Ġmort ar +est e +ick a +Ar thur +ĠPre viously +ĠMart y +Ġsp arse +g ins +Ġin ward +ĠParticip ant +C opy +ĠM isc +Ġantib iotic +ĠRet ro +Ġel usive +Ġass ail +ĠBatt alion +ĠB ought +Ġdimin ish +ĠEuro pa +s ession +ĠDanger ous +ies el +Ġdisbel ief +Ġbl asts +ext reme +ĠBoy d +ĠProject s +ĠGu ys +Ġunder gone +Ġgr ill +ĠDw ight +Ġ19 7 +US ER +Ġfiles ystem +Ġcl ocks +T aylor +Ġwra pper +Ġfold ing +ous and +ĠPhilipp ine +ATION AL +ĠPer th +Ġas hes +Ġaccum ulate +ĠGate way +Sh op +orks hire +H an +ĠBar rel +ĠLe h +ĠX V +Ġwh im +Ġrep o +ĠC G +ĠM am +Ġincorpor ating +Ġbail out +Ġlingu istic +Ġdis integ +C LE +Ġcinem atic +ĠF iber +S yn +il ion +ĠCom pos +c hens +Ġne oc +Ġbo iled +F INE +on o +un cle +ik en +ĠB M +Î ¹ +Ġreceipt s +Ġdisp osed +ĠTh irty +ĠR ough +ĠA BS +Ġnot withstanding +oll en +# $ +Ġunrel iable +Ġbl oom +Ġmedi ocre +Ġtr am +ĠTas man +Ġsh akes +Ġmanifest o +ĠM W +Ġsatisf actory +Ġsh ores +Ġcomput ation +Ġassert ions +orm ons +ar ag +ab it +Dem ocrats +ĠL oot +ĠVol ks +ha ired +Ġgrav itational +S ing +ĠM iz +Ġthro ttle +Ġtyr anny +ĠView s +Ġrob ber +ĠMinor ity +Ġsh rine +sc ope +pur pose +Ġnucle us +our cing +ĠUS DA +ĠD HS +w ra +ĠBow ie +Sc ale +ĠB EL +x i +I ter +Ġ( ), +w right +Ġsail ors +ous ed +NAS A +ĠPro of +ĠMin eral +t oken +ĠF D +R ew +Ġe ll +6 30 +Ġchance llor +ĠG os +Ġamount ed +ĠRec re +ome z +ĠOpt im +ĠOl ive +Ġtrack er +ow ler +ĠUn ique +R oot +Ġmar itime +ĠQur an +ĠAd apt +Ġecosystem s +ĠRe peat +ĠS oy +ĠI MP +Ġgrad uating +and em +P ur +ĠRes et +ĠTr ick +ĠPh illy +ĠT ue +ĠMalays ian +Ġclim ax +Ġb ury +Ġcons pic +ĠSouth ampton +ĠFl owers +Ġesc orted +ĠEduc ational +ĠI RC +Ġbrut ally +e ating +Ġpill ar +ĠS ang +ĠJ ude +ar ling +ĠAm nesty +Ġrem inding +ĠAdminist rative +hes da +Ġfl ashed +ĠP BS +per ate +fe ature +Ġsw ipe +Ġgra ves +oult ry +26 1 +bre aks +ĠGu er +Ġsh rimp +ĠV oting +qu ist +Ġanaly tical +Ġtables poons +ĠS OU +Ġresear ched +Ġdisrupt ed +Ġj our +Ġrepl ica +Ġcart oons +b ians +} ) +c opy +G ot +ou ched +P UT +Ġsw arm +not ations +s aid +Ġreb uilt +Ġcollabor ate +Ġr aging +Ġn ar +Ġdem ographics +ĠD DR +Ġdist rust +oss ier +ĠK ro +Ġpump kin +Ġreg rets +Ġfatal ities +ĠL ens +ĠO le +p d +Ġpupp et +ĠOut look +ĠSt am +O l +F air +U U +Ġre written +Ä ± +Ġfasc inated +Ġve ctors +Ġtrib unal +u ay +ĠM ats +ĠCo ins +[ [ +Ġ18 1 +Ġrend ers +ĠK aepernick +Ġesp ionage +Ġsum m +Ġd itch +Acc ount +Ġspread sheet +Ġmut ant +p ast +40 7 +Ġd ye +Ġinit iation +Ġ4 000 +Ġpunish able +Ġth inner +ĠKh al +Ġinter medi +D un +ĠGoth am +Ġeager ly +Ġvag inal +p owers +V W +ĠWATCH ED +Ġpred ator +ams ung +Ġdispar ity +Ġ[ * +Ġam ph +Ġout skirts +ĠSpir its +Ġskelet al +Ð » +ĠR ear +Ġissu ance +ĠLog ic +re leased +Z Z +ĠB ound +Ent ry +Ġex its +is ol +ĠFound er +Ġw re +ĠGreen land +ĠM MO +t aker +IN C +ãģ ¾ +Ġhour ly +hen ko +Ġfantas ies +Ġdis ob +Ġdemol ition +ãĥ ĭ +Ġen listed +rat ulations +Ġmis guided +Ġens ured +Ġdiscour aged +m ort +Ġfl ank +Ġc ess +Ġreact s +ĠS ere +s ensitive +ĠSer pent +ass ad +Ġ24 7 +Ġcalm ly +b usters +Ġble ed +ĠSt ro +Ġamuse ment +ĠAntar ctica +Ġs cept +ĠG aw +a q +ason ic +Ġsp rawling +n ative +atur ated +ĠBattle field +IV ERS +E B +ĠG ems +ĠNorth western +ĠFil ms +ĠAut omatic +Ġappre hend +ãģ ¨ +Ġgui Name +Ġback end +Ġevid enced +ge ant +01 2 +ĠS iege +Ġexternal To +Ġunfocused Range +ĠguiActiveUn focused +Ġgui Icon +ĠexternalTo EVA +ĠexternalToEVA Only +F ri +ch ard +en aries +Ġchief s +Ġc f +ĠH UD +Ġcorro bor +Ġd B +ĠT aken +ĠPat ricia +ra il +ĠCh arm +ĠLiber tarian +rie ve +Person al +ĠO UR +ger ies +Ġdump ing +Ġneurolog ical +it imate +ĠClint ons +raft ed +ĠM olly +Ġtermin als +reg ister +Ġfl are +Ġenc oded +Ġautop sy +p el +m achine +Ġexempt ions +ĠRoy als +d istance +Ġdraft s +Ġl ame +ĠC unning +Ġsp ouses +ĠMark ets +ĠCar rier +Ġimp lying +ĠY ak +s id +Ġl oser +Ġvigil ant +Ġimpe achment +Ġaug mented +ĠEmploy ees +Ġunint ended +tern ally +ĠW att +Ġrecogn izable +ess im +æ Ŀ +Ġco ated +r ha +Ġlie utenant +ĠLegisl ation +pub lished +44 4 +01 3 +Ġide ally +ĠPass word +Ġsimpl ify +ĠMet a +ĠM RI +Ġple ading +organ ized +hand ler +Ġun ravel +cor rect +Ġ icy +Ġparan oid +Ġpass er +Ġinspect ions +of er +ĠHealth care +28 3 +ĠBr ut +iol a +for ge +ĠMed ieval +MS N +ie vers +ĠProgram ming +å ī +Ġ2 23 +m u +ĠC LE +ug a +Ġsho ppers +Ġinform ative +ĠPl ans +Ġsupplement ation +ĠT ests +ty ard +ocy tes +ĠVeg a +ĠGujar at +erman ent +Ex cept +ĠL OT +all a +ĠC umm +ĠO sw +Ġven om +ĠDeb t +ĠD OWN +Ġreun ion +Ġm uc +ĠRel ief +Ġge op +ĠðŁ ĺ +al ogue +An th +ech o +Ġcor ros +Ġrepl ication +ĠBl azing +ĠD aughter +Ġinf lic +ĠLind sey +Ù Ī +28 4 +Ex it +Ġgl oom +TA IN +Ġundermin ing +Ġadv ising +h idden +Ġover flow +Ġg or +urd ue +Ġe choes +enh agen +Ġimp uls +d rug +c ash +Ġas ync +Ġmir ac +at ts +p unk +Ġpiv ot +ĠLegisl ative +Ġblog gers +ĠCl aw +s burg +d yl +ĠRecomm end +Ġver te +Ġprohib iting +ĠPant her +Jon athan +Ġo min +Ġhate ful +28 1 +ĠOr che +ĠMurd och +down s +Ġas ymm +G ER +Al ways +Ġinform s +ĠW M +ĠP ony +ĠApp endix +ĠAr lington +J am +Ġmedic inal +ĠS lam +IT IES +Ġre aff +ĠR i +F G +S pring +b ool +Ġthigh s +Ġmark ings +ĠRa qqa +ĠL ak +p oll +ts ky +ĠMort y +ĠDef inition +Ġdeb unk +end ered +ĠLe one +a vers +Ġmortg ages +App arently +N ic +ha us +ĠTh ousands +au ld +Ġm ash +sh oot +Ġdi arr +Ġconscious ly +H ero +e as +ĠN aturally +ĠDestroy er +Ġdash board +serv ices +R og +Ġmillenn ials +Ġinv ade +- ( +Ġcomm issions +ĠA uckland +Ġbroadcast s +Ġfront al +Ġcr ank +ĠHist oric +Ġrum ours +CT V +Ġster il +Ġboost er +rock et +ãĤ ¼ +ut sche +ĠP I +Ġ2 33 +ĠProdu cer +ĠAnaly tics +Ġinval uable +Ġunint ention +ĠC Y +Ġscrut in +Ġg igg +Ġeng ulf +Ġprolet ariat +Ġh acks +ĠH ew +ar ak +ĠSl ime +ield ing +ag her +ĠEll iot +Ġtele com +Ġ2 19 +ult an +ĠAr bor +ĠSc outs +B an +Ġlifes pan +Ġbl asp +38 8 +Ġjud iciary +ĠContin ental +ask ing +Mc C +L ED +Ġbag gage +ĠSorce rer +Ġrem nants +ĠGriff ith +ets u +ĠSub aru +ĠPerson ality +des igned +ush ima +agn ar +Ġrec oil +Ġpass ions +\ ": +Ġte e +Ġabol ition +ĠCreat ing +j ac +Ġ19 4 +01 9 +Ġpill ars +ric hed +/ " +t k +Ġlive lihood +Ġro asted +ah on +ĠH utch +ass ert +Ġdivid end +Ġkn it +Ġd aunting +Ġdisturb ance +Ġsh ale +Ġcultiv ated +Ġrefriger ator +L B +ĠN ET +Ġcommercial s +Ġthink ers +45 5 +Ġch op +B road +Ġsuspic ions +Ġtag ged +l ifting +Ġsty lish +ĠShield s +Short ly +Ġt ails +A uth +ST E +ĠG AME +Ġse ism +ĠK is +olog ne +Ġcow ork +Ġforc ibly +Ġthy roid +ĠP B +AN E +mar ried +h orse +Ġpoly mer +ĠCh al +od or +DE BUG +ĠCon text +Ġbl iss +Ġpin point +ĠMat hemat +leg ram +ĠWeek end +Ġlab elled +Ġb art +it les +Ġest rogen +âĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶ âĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶ +" ' +Ġvis ibly +Ġouts ider +aid a +Are a +Ġdisse min +Ġdish onest +ĠCl osed +ĠBullet in +ĠRam sey +sw ord +ĠX I +our ced +S ame +34 6 +ĠRe pe +ĠK ou +c ake +em is +C ache +ĠMe aning +ĠEn light +onom y +Ġmanifest ation +sw orth +J ay +Ġch ore +ö r +D ream +Ġsanction ed +Ġcult urally +ĠA ra +N av +Ġthe ological +Ġstr ut +ĠV O +ĠHand book +Ġconstruct ing +Ġ ¶ +ĠBenef its +ĠPsych ological +s ac +å ¸ +p olicy +ĠMat ters +ĠReport ed +ĠBy te +Ġvit ro +ĠM aiden +Ġl am +ĠJenn ings +Ġgar ment +ĠRut gers +ĠStaff ord +ĠWell ington +Ġinter mitt +Ġn pm +Ġord eal +Ġplug ged +o oming +in ished +fram ework +Ġtim ber +Ġc ass +Ġ8 50 +il ess +ĠRed ux +7 68 +St re +Ġsurpass ed +w hel +Ġparalle ls +Ġve il +ĠG I +ĠR EST +Ġread iness +s ort +Ġmod ifying +ĠSl ate +ru ff +Ġmar ble +Ġinf rared +Ġaud itor +ĠFANT ASY +ĠP overty +ĠS PD +Ġ" ( +K y +RA Y +Ġexecut ions +ĠBever ly +ĠMarx ism +ĠBur st +ĠK ali +est ones +Clear ly +E ll +ãģ § +ĠProceed ings +T oken +IF IC +ñ a +Cent ral +ĠH aley +ĠD rama +Ġform ations +OR N +Book s +Ġdom inating +ĠFly ers +ĠCompan ion +Ġdiscipl ined +ĠYug oslav +ĠSpell s +Ġv engeance +Ġland lords +L en +ĠO gre +ano ia +Ġpier cing +Ġcon greg +Ġscore r +ob ia +Ġnic kel +ĠLear ns +Ġre jo +Ġmaster piece +Fl ash +Ġinhab ited +ĠOpen GL +ĠD ud +ĠI CO +Ġar ter +Ġpl ur +Ġmaster y +Ġlong standing +st ed +Ġw ines +Ġtelev ised +ĠSh rine +ĠBay ern +Ġâ ĵĺ +Ġencl osure +j ohn +Ġprophe ts +ĠRes urrection +ĠOrd ers +Ġun even +r als +Ġd wind +ĠL ah +ĠSl oven +37 8 +Ġins istence +aff le +ĠCl one +Ġhard ship +ĠCongress man +Ġple ad +Ġreview ers +Ġc ured +Ġ19 35 +as ley +f ake +ĠTh inking +yd ia +P ART +ĠD ota +o it +Ġwh ipped +Ġb ouncing +ĠHispan ics +com ings +Ġcann abin +ĠCh ambers +ĠZ ack +Option al +Ġco ats +Ġprow ess +ĠNort on +Ġplain ly +Ġfre ight +Ġinhib ition +Ġcl am +Ġ30 3 +ke f +ale igh +L uke +Ġpsych o +ator ium +M ED +Ġtreat ies +Ġind isc +Ġd c +OP S +Ġresil ient +ĠInter state +Ġsl ack +Ġmund ane +Ġestab lishes +35 9 +Ġstr ained +Ġn ond +S us +Ġcast e +ar ate +ie ving +Ġunfair ly +Ġpars er +on ial +urs ive +V ia +ĠOtt o +ĠAuthor ities +stro ke +K R +ĠMer cy +Ġfurn ished +Ġout set +Ġmet ic +19 82 +olith ic +ĠT ent +og ical +ĠA ircraft +Ġh ides +ĠBec ame +Ġeduc ators +re aching +Ġvol atility +Ġtodd ler +ĠNAS CAR +ĠTw elve +ĠHigh lights +Ġgra pe +Ġspl its +Ġpe asant +Ġre neg +ĠMS I +Tem p +st ars +Ġtre k +ĠHy de +b inding +Ġreal ism +Ġox ide +ĠH os +Ġmount s +Ġbit ing +Ġcollaps ing +Ġpost al +Ġmuse ums +Ġdet ached +Ġrespect ing +Ġmonop ol +Ġwork flow +ĠC ake +Tem plate +ĠOrgan isation +Ġpers istence +36 9 +C oming +B rad +Ġredund ant +ĠG TA +Ġb ending +Ġrev oked +Ġoff ending +Ġfram ing +Ġprint f +Comm un +mem bers +Out side +Ġconst rued +Ġc oded +F ORE +Ġch ast +Ch at +Ind ian +ĠY ard +? !" +ĠP orts +ĠX avier +ĠR ET +' ." +ĠBo at +iv ated +ich t +umer able +D s +ĠDun n +Ġcoff in +Ġsecure ly +ĠRapt ors +ĠB es +Install ation +Ġin ception +ĠHealth y +end ants +Ġpsych ologists +ĠShe ikh +c ultural +ĠBlack Berry +sh ift +F red +oc he +Ġc akes +ĠS EO +ĠG ian +ĠAs ians +og ging +e lement +Ġpund its +ĠV augh +ĠG avin +Ġh itter +Ġdrown ed +Ġch alk +ĠZ ika +Ġmeas les +80 2 +âĢ¦ .. +ĠAW S +] " +Ġdist ort +ĠM ast +Ġantib odies +ĠM ash +Mem ory +ĠUg anda +ĠPro b +Ġvom iting +ĠTurn s +Ġoccup ying +Ġev asion +ĠTher apy +Ġprom o +Ġelect r +Ġblue print +ĠD re +pr iced +ĠDep ot +Ġallev iate +ĠSom ali +m arg +n ine +Ġnostalg ia +ĠShe pherd +Ġcaval ry +Ġtor ped +ĠBlood y +x b +Ġs ank +Ġgo alt +report print +embed reportprint +clone embedreportprint +ĠIn itially +ĠF ischer +Ġnot eworthy +c ern +Ġin efficient +raw download +rawdownload cloneembedreportprint +c ation +ĠD ynasty +l ag +D ES +Ġdistinct ly +ĠEston ia +Ġopen ness +Ġg ossip +ru ck +W idth +ĠIb rahim +Ġpet roleum +Ġav atar +ĠH ed +ath a +ĠHog warts +Ġc aves +67 8 +Ġsafegu ard +ĠM og +iss on +ĠDur ham +sl aught +ĠGrad uate +Ġsub conscious +ĠEx cellent +ĠD um +---- - +Ġp iles +ĠW ORK +ĠG arn +ĠF ol +ĠAT M +Ġavoid s +ĠT ul +Ġble ak +EL Y +iv ist +light ly +P ers +ĠD ob +ĠL S +Ġins anity +Î µ +atal ie +En large +Ġtw ists +Ġfault y +Ġpir acy +Ġimp over +Ġrug ged +ĠF ashion +Ġs ands +' ? +sw ick +Ġn atives +Ġhe n +ĠNo ise +ãĥ Ĺ +Ġg reens +Ġfree zer +Ġd ynasty +ĠFather s +ĠNew ark +Ġarchae ological +Ġo t +ob ar +Ġblock ade +Ġall erg +L V +Ġdeb it +ĠR FC +ĠMil ton +ĠPress ure +Ġwill ingly +Ġdisproportion ate +Ġopp ressive +Ġdiamond s +Ġbelong ings +19 70 +Ġbell s +Ġimperial ism +Ġ2 27 +Ġexpl oding +ĠE clipse +Ġ19 19 +Ġr ant +Ġnom inations +34 7 +Ġpeace fully +ric a +ĠF UCK +Ġvib ration +mal ink +Ġro pes +ĠIv anka +ĠBrew ery +ĠBook er +ĠOw ens +go ers +Serv ices +ĠSn ape +Ġ19 1 +39 5 +Ġ2 99 +just ice +Ġb ri +Ġdisc s +Ġprom inently +Ġvul gar +Ġsk ipping +l ves +Ġtsun ami +37 4 +ĠU rug +ĠE id +rec ated +p hen +Ġfault s +ĠStart ed +9 50 +Ġp i +Ġdetect or +Ġbast ard +Ġvalid ated +Space Engineers +OUR CE +Ġ( ~ +Ġuns ur +Ġaff irmed +Ġfasc ism +Ġres olving +ĠCh avez +ĠC yn +Ġdet ract +L ost +Ġrig ged +Ġhom age +ĠBrun o +55 5 +ec a +Ġpress es +Ġhum our +Ġsp acing +Ġ' / +olk ien +C oun +OP ER +T re +S on +ĠCambod ia +ier re +m ong +o zy +Ġliquid ity +ĠSov iets +ĠFernand o +Ġ2 29 +Ġsl ug +ĠCatal an +elect ric +Ġsc enery +ĠH earth +Ġconst rained +Ġgoal ie +ĠGu idelines +ĠAm mo +ĠPear son +Ġtax ed +Ġfet us +Resp onse +ĠAlex is +th ia +G uy +Ġrecon struct +Ġextrem es +Ġconclud ing +ĠP eg +ook s +Ġded uctions +R ose +Ġground breaking +ĠT arg +ãĥ ģ +ĠRe ve +res ource +Ġmo ons +Ġelectrom agnetic +Ġamid st +ĠVik tor +N ESS +B ACK +Ġcomm ute +ĠAna heim +Ġfluct uations +6 40 +Ġnood les +ĠCop enhagen +ĠT ide +ĠGri zz +ĠS EE +Ġpip elines +Ġsc ars +end o +ag us +ĠE TF +/ # +ĠBec ome +44 8 +Ġvis c +ĠRecomm ended +Ġj umper +Ġcogn ition +Ġassass in +Ġwitness ing +ĠSet up +Ġl ac +v im +IS M +p ages +SS L +35 8 +Ġad ject +indust rial +l ore +cher y +Ġgl itter +Ġc alf +Flor ida +Ġspoil ers +Ġsucceed s +Ġch anting +Ġslog ans +ĠTr acy +Vis it +rol ogy +Ġm ornings +Ġline age +Ġs ip +Ġintense ly +Ġflour ish +ĠSle eping +ĠF em +or por +ĠK lan +ĠDar th +h ack +ĠNi elsen +Ġtum ors +Ġprocure ment +ĠY orkshire +Ġra ided +K Y +An na +Ġ// [ +ĠDis order +ĠMust ang +ĠW en +ĠTry ing +s q +Ġdeliver ies +Ġshut ter +Ġcere bral +Ġbip olar +ĠC N +l ass +j et +Ġdeb ating +> : +Ġe agle +gr ades +ĠD ixon +UG C +M AS +ĠDr aco +ĠMach ines +aff er +Ġem an + ² +pr on +ĠG ym +Ġcompar atively +ĠTrib unal +PR O +Ġle x +Ġfert ile +Ġdep ressing +Ġsuperf icial +ess ential +ĠHun ters +g p +Ġprom inence +L iber +ĠAn cest +ote chnology +Ġm ocking +ĠTra ff +ĸ ļ +Med ium +I raq +Ġpsychiat rist +Quant ity +ĠL ect +Ġno isy +5 20 +G Y +Ġsl apped +ĠM TV +Ġpar a +p ull +Mult iple +as her +Ġn our +ĠSe g +Spe ll +v ous +ord ial +Sen ior +ĠGold berg +ĠPl asma +ne ed +Ġmess enger +ere t +Ġteam ed +Ġliter acy +ĠLe ah +ĠD oyle +Ġem itted +U X +Ġev ade +Ġm aze +Ġwrong ly +ĠL ars +Ġstere otype +Ġpled ges +Ġarom a +ĠM ET +Ġac re +ĠO D +Ġf f +Ġbrew eries +ĠH ilton +und le +ĠK ak +ĠThank fully +ĠCan ucks +in ctions +ĠApp ears +Ġco er +Ġundermin ed +ro vers +And re +Ġbl aze +um ers +Ġfam ine +amp hetamine +ulk an +Am ount +Ġdesper ation +wik ipedia +develop ment +ĠCor inth +uss ia +Jack son +L I +N ative +R s +Oh io +ĠKath leen +F ortunately +Ġattend ant +ĠPre ferred +ĠDid n +ĠV s +M is +Ġrespond ent +Ġb oun +st able +Ġp aved +Ġunex pl +ĠChe ney +L M +ĠC ull +bl own +Ġconfront ing +oc ese +serv ing +W i +ĠLith uania +ann i +Ġst alk +h d +Ġv ener +AP H +ynchron ous +UR R +um ably +hist oric +H alf +H ay +Ġresil ience +spe ction +Ġabandon ing +O bs +ĠDeb bie +Ġgrad ient +ĠPl aint +ĠCan al +AR CH +Ġexpans ive +Ġfun g +Ġb ounced +U nd +Ġprec autions +Ġclar ification +Ġd agger +Ġgri ps +Ġ µ +ĠRiver a +ĠUnd ead +is ites +ĠFIR ST +ñ o +aud i +Ġhost ages +Ġcompl iant +Ġal umni +Se ven +Ġcyber security +e ither +Col lect +Ġinvari ably +ĠS oci +Ġlaw maker +Ġa le +ĠPerson ally +N azi +Ġcustom ization +ĠPro c +ĠSask atchewan +eat uring +Ġsp ared +Ġdiscontin ued +Ġcomput ational +ĠMotor ola +Ġsuprem acist +government al +Ġparad ise +ĠDown ing +ĠNik on +Ġcat alyst +ber ra +Tor onto +8 75 +bet a +ĠMac ron +Ġunreal istic +ve ctor +ĠVeh icles +it iveness +ĠR V +ĠCol bert +s in +o ji +ent in +ĠKr ish +hell o +ff ield +ok y +ĠT ate +Ġmap le +Ġa ids +chem ical +33 4 +n uts +ĠWar p +Ġx x +ĠRob b +umer ous +_- _ +ft ime +ĠV W +Ġw inger +ĠD ome +t ools +ĠP V +ĠGe orgetown +Ġg eared +Ġjihad ists +Ġc p +Ġster oids +M other +cler osis +ĠDR M +nes ia +Ġl inger +Ġimm ersive +ĠC OUN +Ġoutwe igh +ens ual +B and +Ġtransform s +mat ched +ps ons +ĠJud icial +f actor +Ġrefer ral +Ġodd ly +ĠW enger +B ring +ĠB ows +60 2 +IC LE +Ġl ions +ĠAcad emic +ĠTh orn +ĠRa ider +kef eller +St orage +L ower +ĠOr t +ĠEqu ality +AL T +ĠS OC +T ypes +Ġl yn +ĠAss et +co at +TP P +C VE +ĠPione er +app lication +Mod ern +ĠH K +En vironment +Al right +R ain +IP P +ĠShi ite +Ġm ound +ĠAb ilities +cond ition +St aff +Ġcompet ence +ĠM oor +ĠDi ablo +Ġwith held +Ġost ensibly +ĠB rom +Ġms g +Ġden omin +ĠRef erences +ĠF P +Ġplun ged +Ġp amph +m oving +cent ral +Ġdown right +Ġf ading +T al +T yp +ĠTh y +uk es +it he +Ġo ve +Ġbatt led +Ġseaf ood +Ġfig ur +ĠR D +c rop +Ġsqu ads +{ \ +à ¹ +ĠE h +Ġinterview ing +ĠQ in +Ġas piring +PL IC +Ġcla uses +ĠG ast +ĠN ir +Ġl uggage +Ġh ose +Ġsystem d +Ġdesc ending +ĠRev ised +ĠR ails +al ign +70 9 +33 7 +Ġf ug +charg ing +t ags +Ġut er +k ish +WAR NING +49 0 +prof its +Ġvoy age +Ġa ce +ĠV anguard +ĠT anks +ĠM uk +Ġ2 26 +S afe +Ar mor +Ġvolcan ic +Ġwom b +ĠM IL +Ġbegin ner +ĠRec ogn +ĠA AP +PL AY +) ! +Ġdetect ing +c n +Ġbre aches +Bas ically +ĠP ag +ĠMunicip al +ĠInd ie +ĠL af +ĠDis able +ĠOl son +Ġrest rained +Ġrul ings +Ġhum ane +ev ents +ĠCinem a +display Text +ĠH atch +action Date +onna issance +Ġassault ing +ĠL ug +CH AT +Ġvig orous +ĠPer se +Ġintoler ance +ĠSnap chat +ĠSh arks +Ġd ummy +ĠDi agn +ĠGu itar +im eters +40 3 +RE G +A x +Ġsepar ates +ĠMah m +Ġt v +j ah +O OL +C irc +ĠWinds or +uss ian +Ġintu ition +Ġdis dain +ĠDon ovan +Ġ2 21 +E mb +Ġcondem ning +Ġgener osity +zz y +Ġpant ies +ĠPre vent +Action Code +AN A +34 2 +external ActionCode +Ġspec ifying +Ġcryst all +J ere +Ġru pt +ĠApp rentice +Ġprof iling +Ð º +St rike +Ġsid eline +Ġoblig ated +Ġocc ult +Ġbureaucr atic +ant ically +rupt ed +neg ative +ĠEthiop ia +ĠC ivic +Ġins iders +el igible +ĠTV s +ĠB AR +ĠT I +i ologist +ĠA IR +Ġsubstit uted +Ar ab +ĠS aul +ĠY og +p rem +Ġbuild ers +Ġstation ary +Ġdoubt ful +Ġvig orously +Ġthr illing +Ph ysical +ĠCare y +ĠHyd ra +geon ing +ĠS ly +y ton +Ġborrow ers +ĠPark inson +Ġ ë +ĠJama ica +Ġsat ir +Ġinsurg ents +ĠF irm +Ġis ot +ĠK arn +our ning +ak ens +doc s +l ittle +ĠMon aco +CL ASS +Tur key +L y +ĠCon an +ass ic +Ġstar red +ĠPac ers +et ies +Ġt ipping +M oon +ĠR w +s ame +Ġcav ity +Ġgo of +ĠZ o +Sh ock +um mer +Ġemphas izes +Ġreg rett +Ġnovel ty +Ġen vy +ĠPass ive +r w +50 5 +Ġind ifferent +ĠR ica +ĠHim self +ĠFred die +Ġad ip +ä¸ Ģ +Ġbreak out +Ġhur ried +ĠHu ang +ĠD isk +Ġro aming +?????- ?????- +U V +ĠRick y +ĠS igma +Ġmarginal ized +Ġed its +Ġ30 4 +mem ory +Ġspec imen +29 3 +ãģ ¯ +Ġvert ically +Ġaud ition +ĠHe ck +Ġc aster +ĠHold ings +ad al +ĠC ron +ĠL iam +Ġdef lect +P ick +ĠDeb ug +RE F +Ġvers atility +ot hes +class ified +ĠMah ar +ĠH ort +C ounter +st asy +not iced +33 1 +ĠSh im +f uck +ĠB ie +Ġair ing +ĠPro tein +ĠHold ing +Ġspect ators +ili ated +ĠThat cher +n osis +ãĥ¼ ãĥ³ +Te le +B oston +ĠTem pl +st ay +Ġdecl arations +47 9 +Vol ume +ĠDesign er +ĠOver watch +id ae +Ġon wards +Ġn ets +ĠMan ila +part icularly +Ġpolit ic +o other +Ġport raits +Ġpave ment +c ffff +Ġs aints +Ġbegin ners +ES PN +Ġshort comings +âķIJ âķIJ +Ġcom et +ĠOrgan ic +qu el +Ġhospital ized +Bre ak +Ġpe el +dyl ib +asp x +ur ances +ĠT IM +P g +Ġread able +ĠMal ik +Ġm uzzle +Ġbench marks +d al +ĠV acc +ĠH icks +60 9 +ĠB iblical +he ng +Ġover load +ĠCivil ization +Ġimm oral +Ġf ries +ãĤ Ĵ +Ġreprodu ced +Ġform ulation +j ug +ire z +g ear +Ġco ached +Mp Server +ĠS J +ĠK w +In it +d eal +ĠO ro +ĠL oki +ĠSong s +Ġ23 2 +ĠLou ise +asion ally +Ġunc ond +olly wood +Ġprogress ives +ĠEn ough +ĠDo e +Ġwreck age +Ġbr ushed +ĠBase Type +Ġz oning +ish able +het ically +ĠC aucus +ĠH ue +Ġk arma +ĠSport ing +Ġtrad er +Ġseem ing +ĠCapt ure +4 30 +b ish +Ġt unes +Ġindo ors +ĠSp here +ĠD ancing +TER N +Ġno b +ĠG ST +m aps +Ġpe ppers +F it +Ġoverse es +ĠRabb i +ĠR uler +vert ising +off ice +xx x +Ġra ft +Ch anged +Ġtext books +L inks +ĠO mn +ãĢ ij +Ġinconven ience +ĠDon etsk += ~ +Ġimplicit ly +Ġboost s +ĠB ones +ĠBo om +Cour tesy +Ġsens ational +AN Y +Ġgre edy +ed en +Ġinex per +ĠL er +ĠV ale +Ġtight en +ĠE AR +ĠN um +Ġancest or +S ent +ĠH orde +urg ical +all ah +Ġsa p +amb a +ĠSp read +tw itch +Ġgrand son +Ġfract ure +Ġmoder ator +ĠSe venth +ĠRe verse +Ġestim ation +Cho ose +Ġpar ach +Ġbar ric +ãĢ IJ +Ġcomp ass +Ġall ergic +âĢ ķ +OT HER +err illa +Ġw agon +Ġz inc +Ġrub bed +ĠFull er +ĠLuxem bourg +ĠHoo ver +Ġli ar +ĠEven ing +ĠCob b +est eem +Ġselect or +ĠB rawl +is ance +ĠE k +Ġtro op +Ġg uts +ĠApp eal +ĠTibet an +Ġrout ines +ĠM ent +Ġsummar ized +steam apps +Ġtr anqu +Ġ19 29 +or an +ĠAut hent +Ġg maxwell +Ġappre hens +Ġpo ems +Ġsa usage +ĠWeb ster +ur us +Ġthem ed +Ġl ounge +Ġcharg er +Sp oiler +Ġsp illed +h og +ĠSu nder +ĠA in +ĠAng ry +Ġdis qual +ĠFrequ ency +ĠEther net +Ġhel per +Per cent +Ġhorr ifying +Ġa il +ĠAll an +EE E +ĠCross ing +44 9 +Ġh olog +ĠPuzz les +ĠGo es +eren n +60 4 +ãģ ı +ĠRaf ael +Ġatt en +ĠE manuel +Ġup ro +ĠSus p +P sych +ĠTr ainer +ĠN ES +ĠHun ts +bec ue +Ġcounsel or +R ule +Ġtox ins +Ġb anners +r ifice +Ġgreet ing +Ġfren zy +Ġall ocate +Ġ* ) +ex pr +50 3 +ĠCh ick +ĠT orn +Ġconsolid ation +ĠF letcher +sw itch +fr ac +cl ips +ĠMcK in +ĠLun ar +Mon th +IT CH +Ġscholar ly +rap ed +39 8 +Ġ19 10 +Ġe greg +Ġin secure +Ġvict orious +cffff cc +Ġsing led +Ġel ves +ĠW ond +bur st +Ġcam oufl +ĠBL ACK +Ġcondition ed +ç ī +ans wered +Ġcompuls ory +asc ist +Ġpodcast s +ĠFrank furt +bn b +Ġne oliberal +ĠKey board +ĠBel le +w arm +Ġtrust s +Ġins ured +ĠBu cc +us able +60 7 +ĠPl ains +Ġ18 90 +Ġsabot age +Ġlod ged +f elt +Ġg a +ĠN arc +ĠSal em +Ġsevent y +ĠBl ank +p ocket +Ġwhis per +Ġm ating +om ics +ĠSal man +ĠK ad +Ġan gered +Ġcoll isions +Ġextraord inarily +Ġcoerc ion +G host +b irds +è Ģ +k ok +Ġper missible +avor able +Ġpo inters +Ġdiss ip +ac i +Ġtheat rical +ĠCos mic +Ġforget ting +Ġfinal ized +å¤ § +y out +l ibrary +Ġbo oming +ĠBel ieve +ĠTe acher +ĠL iv +ĠGOOD MAN +ĠDomin ican +OR ED +ĠPart ies +Ġprecip itation +ĠSl ot +R oy +ĠComb ined +Ġinteg rating +Ġch rome +Ġintest inal +ĠRe bell +Ġmatch ups +Ġblock buster +ĠLore n +ĠLe vy +Ġpre aching +ĠS ending +ĠPur pose +ra x +f if +Ġauthor itative +ĠP ET +ast ical +Ġdish on +Ġchat ting +Ġ"$ :/ +Connect ion +Ġrecre ate +Ġdel inqu +Ġbro th +ĠD irty +ĠAd min +z man +Ġscholars hips +Ġ25 3 +cont act +als a +7 67 +c reen +abb age +Ġ19 15 +Ġbl ended +Ġal armed +L anguage +35 6 +Ġbl ends +ĠCh anged +W olf +Ġhe pat +Creat ing +Ġper secut +Ġsweet ness +art e +Ġforfe iture +ĠRober to +im pro +N FL +ĠMag net +Det ailed +Ġinsign ificant +ĠPOL IT +ĠBB Q +ĠC PS +Ġse aw +amin er +m L +end if +f inals +Ġ26 5 +u ish +Ġ} ) +ĠPro blems +Ġem blem +Ġserious ness +Ġpars ing +Ġsubst itution +Ġpress ured +Ġrecy cled +ale b +Rub y +Ġprof iciency +Dri ver +ĠW ester +: ' +AF TA +Ġm antle +ĠClay ton +fl ag +Ġpractition er +c overed +ĠSt ruct +add afi +4 25 +ĠTown ship +ĠHyd ro +Lou is +34 3 +Ġcond o +ĠT ao +Ġutil ization +Ġnause a +ĠDem s +rid ges +p ause +Ġform ulas +Ġchall enger +37 6 +Ġdefect ive +ĠRail way +ĠPub Med +Ġyog urt +l bs +ĠNor folk +OP E +ĠMood y +Ġdistribut or +Ġscroll s +Ġextract s +St an +Ġv iability +Ġexp oses +Ġstar vation +ĠStep s +ĠD odd +f ew +ST D +33 2 +Ġclos ures +Ġcomplement ary +ĠS asha +ump y +Ġmon et +Ġartic ulate +ĠDo ct +k iller +Ġsc rim +Ġ2 64 +Ġprost itutes +Ġse vered +Ġattach ments +Ġcool ed +L ev +ĠF alk +f ail +Ġpolic eman +ĠD ag +Ġpray ed +ĠK ernel +Ġcl ut +Ġc ath +Ġan omaly +St orm +em aker +ĠBreak fast +ul i +o ire +J J +h z +Oper ation +ĠS ick +35 4 +ĠGuatem ala +R ate +Ġexp osures +f aces +ĠArch ae +ra f +ĠM ia +Ġ20 25 +Ġop aque +Ġdisgu ised +ĠHead quarters +S ah +Ġp ots +9 78 +ĠM alf +Ġfrown ed +Ġpoison ous +ĠCon vers +ee ks +Ġcr ab +." " +Ġtre ason +Ġr anc +Ġescal ating +Ġwar r +Ġmob s +Ġl amps +ĠSun shine +ĠBrun swick +Ph ones +Ġspe lled +ĠSk ip +Ġ20 50 +Ġ19 11 +ĠPl uto +ĠAm end +Ġme ats +38 7 +Ġst omp +ĠZh ou +ĠLevi athan +ĠHaz ard +ad v +ĠOr well +Ġal oud +Ġb umper +ĠAn arch +ub untu +ĠSer ious +f itting +ĠOption al +ĠCec il +RE AM +Ġser otonin +Ġcultiv ate +ag ogue +} \ +Ġmos ques +ĠSun ny +Ġre active +rev olution +ĠL up +ĠFed ora +Ġdefense man +ĠV ID +ist ine +Ġdrown ing +ĠBroad casting +Ġthr iller +ĠS cy +Ġacceler ating +Ġdirect s +od ied +b ike +d uration +Ġpain fully +R edd +Ġproduct ions +Ġg ag +Ġwh ist +Ġs ock +Ġinf initely +ĠConc ern +ĠCit adel +Ġlie u +Ġcand les +ogene ous +arg er +Ġheaven ly +inflamm atory +Per formance +C s +ruct ose +az aki +Ġp essim +Ġinf erence +Ġpow d +ĠZ oe +Ġpain ts +Ġd azz +pt a +-------- --- +Ġins pir +ĠExper imental +ĠKn ife +reg or +b ors +Ġshow ers +rom eda +Ġs aint +Ġben ign +ĠJ iang +Ġenvision ed +Ġsh roud +IF T +H O +Ġsh uff +ĠI CC +Ġse greg +Ġrevis it +ighth ouse +L i +Ġsub strate +ĠSe as +ĠRew ard +ĠH ep +ĠBr ass +s bm +Ġelim inates +Ġst amina +ĠV AT +ĠLo an +Ġconst raint +Ġappropri ated +Ġp es +ĠA LE +r anging +Ġ40 4 +39 2 +Ġintellectual s +ach u +Ġrestruct uring +ĠLe vin +Ġrun es +Ġdelight ful +Ġcarbohyd rates +ĠMod els +ĠExp o +Ġtransport ing +all oc +Ġring ing +S amsung +Ġscarce ly +ĠURL s +ĠM AS +Ġprot otypes +Ġnarr ator +ĠCPU s +cd n +ĠBart on +Ġdecided ly +ĠSh u +ix ir +oc ious +ĠMy st +N intendo +Ġre use +Ġforg iven +F ew +in ical +n at +Ġseam less +ĠEv a +ĠE VE +ĠJ O +land ers +Ġso fter +neg ie +Ġtrans ient +Ġorb ital +Ġfulf il +ĠK om +Hop efully +Ġdynam ically +ĠHun ger +å Ľ +ĠArmen ia +el man +ber to +Ġp ige +ĠID s +lim it +Ġve ins +Ġso aring +p acks +Gold en +ĠCr ab +ist or +ĠR PM +Ġ$ $ +g ression +Ġjihad ist +Ġgam ble +Ġcare g +Ġinf lated +F ace +ĠFire arms +ĠEm manuel +â Ŀ +Ġsh ocks +gr ab +Ġspl end +ĠHP V +ab ortion +Ab ove +Ent ity +play ers +Ġcomm enced +ul ence +Ġfulfill ment +Ġembod iments +ĠW elfare +Ġha il +Ġ< @ +tt en +Ġcat cher +ĠJ azeera +Ġvolcan o +Ġstabil ize +ĠHand ler +Ġintens ified +ĠAb rams +Ġhum iliation +p aced +60 5 +ĠCent OS +Spe cific +Ġhe ed +ĠC AM +ĠGal ile +D ie +Ġabol ished +ĠThom son +ĠTe achers +ĠW ass +j ong +ĠIS BN +ĠAll ies +sh ake +å · +v ict +How ard +Ġde em +Ġexceed ingly +ĠSmart stocks +ib e +Ġdoor way +Ġcompet ed +ig mat +Ġnational ists +Ġg room +ĠKe en +Ġdispos able +de cl +ĠT olkien +ĠSche me +Ġb iod +Ġav id +ĠEl on +ag ar +ĠT SA +R oman +Ġartific ially +Ġadvis ors +X L +ĠInf erno +36 6 +Ġted ious +ĠPhot ography +ĠCar rie +Ġtro pe +ĠSand ra +Ġdec imal +Que en +ĠGund am +ĠO M +ote ch +N BA +Ġ19 32 +Ġent renched +ĠMar ion +Ġfr aternity +Lab our +Hen ry +Ġlat itude +E ither +Ġenh ances +ĠPot ential +Ġsh ines +id ad +Ġbread th +Ġcapac ities +ĠðŁ ĻĤ +ĠBron x +Ġsex es +Ġdifferent iation +Ġheavy weight +ĠT aj +d ra +Ġmigr ate +Ġexhaust ion +ĠR UN +els ius +ĠCu omo +Ġgu itars +Ġcl ones +ĠSom ew +ĠP ry +------------ - +Ġwarr anted +cy cles +Ġsalv age +Ġdis ks +R ANT +ĠNGO s +ĠMart ian +":[ {" +Ġadd icts +oj ure +il let +Ġamazing ly +art ments +p ixel +ĠGPU s +Lay out +è £ +ĠTam il +ĠBas il +Ġimpart ial +ĠSt ructure +f ork +b ryce +Ġr idge +ĠHamb urg +ri ous +Ġbl itz +cig arettes +Ġcan ned +40 2 +Ġiron ically +Ġcompassion ate +ĠHaw kins +. # +ĠCat hedral +Ġrall ied +in ternal +Ġqu ota +st akes +T EXT +m om +Ġcomple tes +Ġ23 8 +Ġsh rug +ãĥ ij +ĠN inth +Ġrev ise +ĠProv ider +Ġtre acher +Ġqu asi +ĠPR ES +Ġdep osition +Ġconfidential ity +iss ors +Ġim balance +Ġspan ning +Ġang ular +ĠC ul +commun ication +ĠNor a +ĠGen ius +op ter +Ġs acked +Sp ot +Ġfine ly +ĠCH R +28 2 +w aves +Pal est +ĠRo hing +N L +è ¿ +Ġsh itty +ĠSc alia +4 75 +Pro gress +Ġreferen cing +Ġclass rooms +ab ee +Ġs od +hes ion +70 8 +ĠZucker berg +ĠFin ish +ĠScot ia +ĠSav ior +ĠInstall ation +an tha +( - +Ġ30 2 +ĠP unk +Ġcr ater +yout u +Ġro ast +Ġinflu encing +Ġd up +ĠJ R +ĠG rav +Ġstat ure +Ġbath rooms +A side +W iki +me an +ĠZ ak +ĠOn es +ĠN ath +Ġhyper t +Ġcommence ment +C ivil +Ġmoder ately +Ġdistribut ors +Ġbreast feeding +Ġ9 80 +ĠS ik +ĠC ig +ĠAM ER +R IP +ĠCare er +ust ing +Ġmess ed +Ġe h +ĠJ ensen +/ $ +Ġblack mail +Ġconvers ions +Ġscientific ally +Ġmant ra +p aying +Ġiv ory +ĠCour ts +OU GH +aunt let +Ser ial +B row +ĠH undreds +3 23 +Ġpe e +Ġlin ux +Ġsub mer +ĠPrinc ipal +48 5 +ĠD SL +ĠCous ins +Ġdoctr ines +ĠAthlet ics +Ġ3 15 +ĠK arma +Ġatt ent +ur ger +Ġpresc ribe +Ġenc aps +ĠC ame +Ġsecret ive +ĠCr imes +d n +C lean +ĠEgypt ians +ĠCar penter +Ġ ll +H um +ĠMil o +Ġcapital ists +Ġbrief ed +T we +ĠBas in +elve t +M os +Ġplun ge +ĠKa iser +ĠFu j +ill in +Ġsafegu ards +Ġo ste +ĠOpportun ity +ĠM afia +ĠCall ing +ap a +ur ban +br ush +ill ard +c é +int elligence +ĠL ob +ĠDru id +Ġsm oother +Ġfoot ing +Ġmotor ists +arc ity +Ġmascul inity +Ġm ism +Ġabdom inal +ĠTa vern +ĠR oh +Ġesc apes +s igned +Anth ony +Ġsacrific ing +Ġintim acy +Ġan terior +ĠK od +Ġmot if +Ġg raz +Ġvisual ization +Ġguitar ist +ĠTro tsky +m agic +D ar +ĠMor i +Ġw ards +Ġtoile ts +l est +Ġtele port +ĠSund ays +ĠPl at +ET S +Ġe Sports +Pat rick +ĠK atherine +en ko +Ġhas sle +ĠM ick +gg les +Ġh ob +aint ain +Ġair borne +Ġsp ans +Ġch ili +Ġa perture +Ġvolunte ered +ĠInc ident +ĠF res +ĠVeter an +augh tered +ing o +Ġun insured +CL OSE +Ġf use +Ġer otic +Ġadvert ise +ra ising +Text ure +Ġatt ends +ĠRE AL +udd led +Ġsm oot +Ġ30 5 +ĠWill is +Ġbl ond +An alysis +ĠV T +on ica +Ġstrongh old +R F +N M +. >> +Ġprosper ous +Ġbo asted +29 2 +ĠManufact uring +PR ESS +g ren +Ġpharm acy +ĠRoc kefeller +k ai +Ġth umbs +ĠH ut +Ġmother board +Ġguard ians +ĠAl ter +ll ular +Ġsh ack +Ġwise ly +Ġback bone +erv a +Ġsu icides +ĠMcG regor +ij ah +E mer +ĠB rav +Ġdesign ate +P OST +produ ced +Ġcleans ing +irl wind +ex istent +ĠHum ph +ĠPay ne +Ġv ested +Å ¡ +Ġstring ent +ion a +Ġuns ub +Ġsum med +ĠHer cules +sub ject +ĠR agnar +ĠN os +Ġcharacter ization +Ġsav vy +ĠDaw son +ĠCas ino +Ġf ri +ĠBar rier +Ġmis information +Ġins ulation +Ġcorrid ors +Ġair planes +ĠNo ct +ah i +Ġ19 16 +k b +arm ac +Ġsh un +Ġsche ma +Ġhorr ified +Ġ23 9 +aund ers +N B +i ates +er ity +ĠSh ard +Ġr arity +Ġgroup ed +ĠGh ana +again st +ĠBi ological +ĠA ware +ow ell +Ï Ħ +ĠBe au +sh aw +H ack +ĠJul ius +US S +ol son +aun a +c ru +ĠMaur ice +ĠI k +Ġsequ encing +Ġradical s +Ġ( ?, +v irtual +Ġany ways +Ġreper c +Ġhand lers +Ġhes itant +é ĥ +ĠM F +ple mentation +ass ociated +Ġcampaign ed +ĠY ue +ut ations +ĠY oga +Ġsim mer +Ġro ds +Ġmel ody +Ġconv oy +v ideos +Ġscreen ed +N eg +ochem ical +Ġ( )) +Ġultr as +Ġant ip +ĠIsland ers +70 4 +Ġfet ish +Ġridic ulously +ĠK art +Ġmitochond rial +Ġinterf ering +Build er +Ġover fl +Ġac ne +ĠM ud +ĠK err +f lex +ĠPost al +ĠBalt ic +47 7 +ĠPers ons +our age +H B +ĠM use +ĠImm ortal +ĠDri ving +Ġpet itions +Ġsubsc ript +Ġs orce +ĠProcess or +ut on +S ony +Ġph on +Ġr aced +ĠAnth rop +Ġday time +ĠEx ercise +Add ing +Ġeng ages +ĠQual comm +Ġmir acles +Ġmem es +ĠDr ink +ĠOri oles +Ġhair s +ĠPol ar +ath om +Ġsl ippery +ĠR emy +Ġcar amel +ĠY EAR +Ġal k +I gn +a ution +ĠMer lin +ĠC ran +Ġap ologies +Ġ4 10 +Ġout ing +ĠMem ories +app ointed +Ġcount ered +u ld +pos ing +Ġfire wall +ĠW ast +ĠW et +work ed +se ller +Ġrepe aled +ere o +ass uming +BL IC +m ite +ĠCEO s +ĠChap el +ellig ent +________________ ________ +D og +Ġw art +Ġsubsc riber +s ports +Ġbe gged +ĠM V +Ġsem if +eth ical +Ġpre ach +Ġrev ital +Ġpun itive +Ġshort cuts +Ġinstit uted +ĠWars aw +Ġabdom en +ĠK ING +Ġsuper intendent +Ġf ry +ĠGe o +T OR +Ġcontrad ictions +apt ic +Ġlandsc apes +b ugs +Ġcl ust +Ġvol ley +c ribed +Ġt andem +Ġrob es +WH AT +Ġpromot er +Ġel oqu +review ed +ĠD K +ĠPl ato +Ġf ps +T ank +ĠDer rick +Ġpriorit ize +as per +ĠHond uras +ĠCom pleted +ne c +Ġm og +n ir +ĠMay o +DE F +st all +in ness +ĠVolks wagen +Ġprec aution +ĠM ell +i ak +ist ries +Ġ24 8 +Ġoverl apping +Sen ate +ĠEnh ance +res y +rac ial +OR TS +ĠM ormons +Str ong +ĠCo ch +Mex ico +ĠMad uro +Ġj ars +Ġcan e +W ik +oll a +iff erence +Ġphysic ist +ĠMag gie +Ġ28 5 +Ġdep iction +ĠMcL aren +J u +Ġsl ows +Ġcommission ers +ĠWill ow +ĠExpl os +hov ah +Ġtechn ician +Ġhom icides +ĠFl av +ĠTr uman +Ġ100 00 +u ctor +Ġsh ader +News letter +45 7 +Ġre ver +Ġhard ened +Ġwhere abouts +Ġrede velop +Ġcar bs +Ġtra vers +Ġsqu irrel +Ġfoll ower +Ġs ings +50 8 +Ġrabb its +emon ium +Ġdocument ing +Ġmisunder stood +) ' +R ick +gg ies +Ġprem ie +Ġsk ating +Ġpass ports +Ġf ists +aged don +H aw +AC P +0 80 +ĠThough ts +ĠCarl son +Ġpriest hood +h ua +Ġdun geons +ĠLo ans +Ġant is +Ġfamiliar ity +ĠS abb +op al +ĠIn k +st rike +Ġc ram +Ġlegal ized +Ġcu isine +Ġfib re +Tra vel +ĠMon ument +OD Y +eth y +Ġinter state +ĠP UR +em porary +ĠArab ian +develop ed +Ġsadd le +Ġg ithub +ĠOff er +ĠIS P +ro let +ĠSUP ER +ĠDen is +Ġmultipl ier +Ġstir red +Interest ingly +Ġcustom ary +Ġbill ed +he x +Ġmultipl ied +Ġfl ipping +ĠCros by +Ġfundament als +ia e +ĠPlay ed +ĠAt om +am azon +ĠFl am +ee z +activ ated +Ġtables poon +Ġliberal ism +ĠPal in +ĠP atel +N um +ĠT AM +Ġs urn +ĠRel oaded +Ġco ined +" ], +ĠCl ash +ĠAg u +Ġprag matic +ĠActiv ate +Ġ8 02 +Ġtrail ers +Ġsil hou +Ġprob es +Ġcirc us +ĠB ain +ĠLind say +ĠAb bey +Del ivery +Ġconcess ion +Ġgast ro +ĠSpr ite +Ä Ł +and el +Ġg imm +Ġaut obi +ĠT urtle +Ġwonder fully +ĠHar am +ĠWorld wide +ĠHand le +Ġtheor ists +Ġsle ek +ĠZh u +ograph ically +EG A +ĠOwn ers +ath s +ĠAntar ctic +n atal +=" " +fl ags +`` `` +Ġs ul +K h +Ġpot assium +Ġlinem an +Ġcere al +ĠSe asons +Ġ20 22 +Ġmat hematic +Ġastron omers +prof essional +Ġf ares +cknow led +Ġch i +Ġyoung sters +Ġmistaken ly +Ġhem isphere +ĠDiv inity +r one +Ġ" , +r ings +Ġattract s +v ana +å ¹ +C AP +Ġplay list +Ġpor ch +ãģ £ +Ġincorpor ates +Ġso ak +Ġassert ing +ĠTerror ism +ĠP ablo +J a +ces ter +Ġfear ing +ĠPr ayer +Ġescal ated +G W +Ġro be +ĠBright on +ac ists +ĠSym phony +ĠDwar f +ĠPar ade +ĠLe go +Ġinex pl +Ġl ords +le af +RA G +l iber +Ġcig ars +ĠJe hovah +60 6 +WIND OWS +ĠLiber ia +eb us +He avy +Ġl ubric +ĠR W +angu ages +Ġnarrow ed +com puter +ĠE mber +Ġmurder ing +Ġdown stream +ĠT uls +ĠT ables +Top ic +ĠAcc uracy += / +l ost +ĠRe i +Ġprogress es +b ear +Ġestablish ments +Just in +ĠPe ach +ĠG omez +å ¿ +ĠTri angle +Id ent +ĠH ive +Res ources +Ġmix es +ĠAss uming +M u +Ġhyp oc +Ġs ane +ĠW an +id ious +Su ccess +Ġ io +Ang el +Ġdanger ously +ĠCreat ure +W ORK +: [ +ĠKat rina +List ener +M iller +ĠId lib +h ang +Ġcircum vent +h ref +Ġcel estial +ĠWe eks +ĠP ug +ĠDal ton +Ġsubpoen a +uk u +Ġpers isted +pe i +old ing +ĠDoc uments +ĠH ast +ĠC ENT +Ġprim er +Ġsyn onymous +Ġn ib +om bs +Ġnot ation +ĠD ish +ĠAt mosp +Ġforb id +ĠAN G +pat tern +l os +Ġproject iles +b rown +." , +ĠVen om +Ġfierce ly +ub lished +ĠU ran +ĠNic arag +4 10 +ĠC AL +OT OS +ĠMir acle +ĠEn chant +Ġguard ing +app end +Att ach +Ġlevel ed +Ġcond oms +ih ilation +64 9 +Ġnight mares +ĠTHE Y +ĠST ART +ĠK inn +Ġroomm ate +Ġhy giene +o pping +J ob +Ġl vl +ĠV ER +ĠKe eping +ab etic +Ġformat ting +eral a +Ġrev isions +Ġres urg +T el +ĠGood man +35 3 +p od +Ġind isp +ĠTrans lation +Ġg own +ĠM und +Ġc is +Ġby stand +col lect +ĠPun jab +act ively +ĠG amb +te ll +Ġimport ing +g encies +Ġloc om +ĠBr ill +H oly +ĠBer ger +Ġshow down +Ġrespond ers +IL Y +Ġt akedown +le ted +Ġmat tered +Ġpredict ive +Ġover lay +G PU +ĠV ick +Ġconvey ed +T ab +pe er +Sc an +Ġdefensive ly +v ae +Ġappro ving +Ġt iers +ĠV ia +quer ade +ĠSaud is +Ġdemol ished +ĠProp he +Ġmon o +Ġhospital ity +H AM +ĠAri el +M OD +ĠTor ah +Ġbl ah +ĠBel arus +erent ial +ĠT uc +Ġbank er +39 7 +Ġmosqu it +ĠScient ist +ĠMus ical +Ġh ust +Sh ift +Ġtor ment +Ġstand off +E duc +ĠF og +Ġampl ifier +Sh ape +Inst ance +ĠCrit ics +Ġda emon +H ouston +Ġmatt ress +ĠID F +Ġobsc ene +ĠA mer +hett i +Ġcomp iling +35 2 +vere tt +ĠRed uction +ist ration +ĠBl essed +ĠB achelor +3 16 +Ġpr ank +ĠVul can +dd ing +Ġm ourning +ĠQu int +ĠBl aster +test ing +Ġsed iment +>> > +ĠE ternity +ĠWH ERE +ĠM aze +Ġreact ing +ĠAl v +oms day +ĠC RA +Ġtransl ator +Ġbog us +at u +We bsite +oll s +Ġbapt ism +Ġs ibling +ĠAut umn +ve z +ãģ® é +gu ards +Ge org +assad ors +ĠFre ud +Ġcontin ents +ĠReg istry +Bern ie +ĸļ 士 +Ġtoler ant +ĠU W +Ġhor ribly +99 5 +ĠMID I +Ġimpat ient +oc ado +er i +ĠWor st +ĠNor ris +ĠTalk ing +Ġdef ends +ens able +Ġ20 21 +Ġanat omy +L ew +Ġdraw er +ĠCan berra +Ġpatri otic +é¾įå ĸļ士 +ĠAv g +AR M +Ġundis closed +Ġfare well +45 9 +b able +ĠAll ison +OL OG +Ġcon co +t ight +ĠAC PI +ĠM ines +l ich +ĠâĶ ľ +represent ed +200 000 +Ġenthusi ast +OT S +b il +ĠIng redients +Ġinvent or +ĠMy SQL +³³ Âł +ĠAB OUT +with in +Ġm k +B ul +ĠF ake +Ġdracon ian +W a +hel m +ĠTer ran +erv ille +Ġcommon place +SI ZE +Ġ" < +re place +ograph s +ĠSE LECT +inc ible +ĠMost ly +ĠShe ffield +ĠID E +ugg le +Ġcit ations +h urst +ĠUn ix +Ġunle ash +ĠP iper +ĠN ano +Ġsucc umb +Ġreluct ance +Ġ25 00 +ĠMer chant +Ġwire t +Ġcomb os +ĠBirth day +Ġchar coal +ĠU PS +ĠFair fax +Ġdrive way +ĠT ek +ĠP itch +ove re +Ġtechn icians +ĠAct ual +fl ation +ĠF iscal +ĠEm pty +an amo +Ġmag nesium +Ġsl ut +Ġgrow ers +Invest igators +( ): +ĠS atellite +ĠKe ynes +miss ive +l ane +Ġb orough +3 44 +ĠTE AM +ĠBet hesda +C V +h ower +ĠR AD +Ġch ant +ĠR iy +Ġcompos itions +Ġmild ly +Ġmedd ling +Ġag ility +ane ers +5 01 +Ġsyn th +ling er +29 1 +Ġex claimed +Part y +Ġcont amin +ĠMan or +ĠResp ond +Ġpra ising +Ġman ners +fle et +Sum mer +ĠLy nd +ĠDef initely +gr im +Ġbow ling +st ri +ç Ľ +y nt +Ġmand ates +D IV +Ġreconc ile +view s +ĠDam on +vet te +F lo +ĠGreat est +il on +ic ia +Ġportray al +Ġcush ion +50 4 +19 79 +oss al +App lic +sc ription +Ġmit igation +AT S +p ac +Ġer ased +Ġdefic iencies +ĠHolland e +ĠX u +Ġb red +Ġpregn ancies +f emin +Ġem ph +Ġpl anners +Ġout per +utter ing +Ġperpet rator +Ġm otto +ĠEll ison +ĠNE VER +Ġadmitted ly +AR I +ĠAzerbai jan +Ġmill isec +Ġcombust ion +ĠBott le +ĠL und +ĠP s +ĠD ress +Ġfabric ated +Ġbat tered +Ġs idel +ĠNot ting +Fore ign +ĠJer ome +0 20 +ĠAr bit +Ġkn ots +ĠR IGHT +M oving +ãģ Ļ +Ġsur geries +Ġcour thouse +Ġm astered +Ġhover ing +ĠBr an +ĠAl ison +Ġsaf est +m ilitary +Ġbull ied +Ġbar rage +Read er +ES E +ĠGe ographic +T ools +3 14 +ĠGe ek +ro th +gl ers +ĠF IN +Ï ģ +ĠA ston +al tern +48 8 +Ġveter in +G amer +Ġint el +ren ches +Sh ield +Ġam nesty +ĠB har +Ġp iled +Ġhonor able +ĠInst itutes +Ġso aked +Ġcom a +ĠE FF +34 1 +by tes +ĠG mail +le in +ĠCanad iens +m aterial +I l +Ġinstruct ors +ĠK Y +Ġconce ive +ub b +ĠP ossible +Ġeas ing +ĠChrist ina +Ġcar ic +ĠHD R +R OM +Ġsho vel +de lete +Ġp uff +ĠCh anging +Ġseam lessly +Att ribute +Ġacqu isitions +ak ery +ĠE F +Ġaut istic +ĠT akes +ĠPow der +ĠSt ir +5 10 +ĠBub ble +sett ings +ĠF owler +Ġmust ard +Ġmore over +Ġcopyright ed +ĠLED s +15 00 +æ ī +ĠH IS +en f +Ġcust od +ĠH uck +G i +Ġim g +An swer +C t +j ay +ĠInf rastructure +Ġfeder ally +L oc +Ġmicro bes +Ġover run +dd s +ot ent +adi ator +>>>> >>>> +Ġtorn ado +Ġadj ud +Ġintrig ued +Ġs i +ĠRevel ation +pro gress +Ġburgl ary +ĠSai yan +ĠK athy +Ġser pent +ĠAndre as +Ġcomp el +ess ler +ĠPl astic +ĠAd vent +ĠPos itive +ĠQ t +ĠHind us +reg istered +ular ity +Ġrighteous ness +Ġdemon ic +u itive +ĠB DS +ĠGre gg +c ia +ĠCrus ade +ĠSina i +W ARE ++ ( +Ġme ll +Ġder ail +y ards +A st +Ġnotice ably +ĠO ber +R am +Ġun noticed +Ġse q +av age +T s +Ġ6 40 +Ġconced e +Ġ] ) +F ill +Ġcapt ivity +ĠImprove ment +ĠCrus ader +ara oh +M AP +æ Ĺ +Ġstr ide +al ways +F ly +N it +Ġal gae +ĠCook ing +ĠDo ors +Mal ley +Ġpolic emen +ãģ į +Ġastron aut +access ible +49 5 +ĠR AW +cl iffe +udic rous +Ġdep ended +al ach +Ġvent ures +ra ke +Ġt its +ĠH ou +Ġcond om +ormon al +Ġind ent +Ġupload ing +Foot note +Import ant +Ġ27 1 +Ġmind ful +Ġcont ends +C ra +Ġcal ibr +ĠO ECD +plug in +F at +ĠIS S +ĠDynam ics +ans en +68 6 +' ), +Ġsp rite +Ġhand held +ĠH ipp +=~ =~ +Tr ust +Ġsem antics +ĠBund es +ĠRen o +ĠLiter ature +s ense +G ary +ĠA eg +ĠTr in +EE K +Ġcler ic +ĠSS H +Ġch rist +Ġinv ading +ib u +Ġen um +aur a +Ġal lege +ĠInc redible +B BC +Ġth ru +Ġsa iled +Ġem ulate +Ġin security +Ġc rou +Ġaccommod ations +Ġincompet ent +Ġsl ips +ĠEarth qu +s ama +IL LE +Ġi Phones +as aki +Ġby e +Ġar d +Ġext ras +Ġsl aughtered +Ġcrowd funding +res so +Ġfil ib +ĠER ROR +ĠT LS +e gg +ĠIt al +Ġen list +ĠCatal onia +ĠSc ots +Ġser geant +Ġdiss olve +N H +Ġstand ings +ri que +I Q +Ġbenef iciary +Ġaqu arium +You Tube +ĠPower Shell +Ġbright est +ĠWar rant +S old +Writ ing +Ġbegin nings +ĠRes erved +ĠLatin os +head ing +Ġ4 40 +Ġrooft op +AT ING +Ġ3 90 +VP N +G s +k ernel +turn ed +Ġprefer able +Ġturn overs +ĠH els +S a +ĠShin ji +ve h +ĠMOD ULE +V iol +Ġex iting +Ġj ab +ĠVan illa +Ġac ron +ĠG ap +ber n +A k +ĠMc Gu +Ġend lessly +ĠFar age +ĠNo el +V a +M K +Ġbr ute +ĠK ru +ĠES V +ĠOl ivia +âĢ ł +ĠK af +Ġtrust ing +Ġh ots +3 24 +Ġmal aria +Ġj son +Ġp ounding +ort ment +Count ry +Ġpostp oned +Ġunequ iv +? ), +ĠRo oney +udd ing +ĠLe ap +ur rence +sh apeshifter +ĠH AS +os ate +Ġca vern +Ġconserv atism +ĠB AD +Ġmile age +Ġarrest ing +V aults +Ġmix er +Dem ocratic +ĠB enson +Ġauth ored +8 000 +Ġpro active +ĠSpirit ual +t re +Ġincarcer ated +ĠS ort +Ġpe aked +Ġwield ing +re ciation +×Ļ × +P atch +ĠEm my +Ġex qu +tt o +ĠRat io +ĠP icks +ĠG ry +ph ant +Ġf ret +Ġeth n +Ġarch ived +% - +c ases +ĠBl aze +Ġim b +c v +y ss +im ony +Ġcount down +Ġaw akening +ĠTunis ia +ĠRe fer +ĠM J +Ġun natural +ĠCar negie +iz en +ĠN uggets +he ss +Ġev ils +64 7 +Ġintrodu ctory +l oving +ĠMcM ahon +Ġambig uity +L abel +ĠAlm ighty +Ġcolor ing +ĠCl aus +set ting +N ULL +ĠF avorite +ĠS IG +> ( +ĠSh iva +ĠMay er +Ġstorm ed +ĠCo verage +we apons +igh am +Ġun answered +Ġle ve +Ġc oy +c as +b ags +as ured +Se attle +ĠSant orum +ser ious +Ġcourage ous +ĠS oup +Ġconfisc ated +Ġ// / +Ġuncon ventional +Ġmom s +ĠRohing ya +ĠOrche stra +ĠPot ion +Ġdisc redit +ĠF IL +f ixed +ĠDe er +do i +ĠDim ension +Ġbureaucr ats +et een +Ġaction Group +oh m +Ġb umps +ĠUt ility +Ġsubmar ines +ren heit +re search +ĠShap iro +Ġsket ches +Ġde ceptive +ĠV il +es ame +ĠEss entially +Ġramp age +isk y +Ġmut tered +th ritis +Ġ23 6 +f et +b ars +Ġpup il +ĠTh ou +o S +s ong +Ġfract ured +Ġre vert +pict ure +Ġcrit erion +us her +Ġreperc ussions +ĠV intage +ĠSuper intendent +Offic ers +Ġflag ged +Ġbl ames +Ġin verse +ograp hers +Ġmakes hift +Ġdev oid +Ġfoss ils +ĠArist otle +ĠFund s +Ġde pleted +ĠFl u +ĠY uan +Ġw oes +Ġlip id +Ġsit u +requ isites +Ġfurn ish +ĠSam ar +Ġshame ful +Ġadverse ly +Ġad ept +Ġrem orse +Ġmurder ous +uck les +ĠE SL +Ġ3 14 +s ent +Ġred ef +ĠC ache +ĠP urs +ig ans +Ġ4 60 +Ġpres criptions +Ġf res +F uck +ocr ates +Tw enty +ĠWe ird +ĠT oggle +ĠC alled +itiz ens +Ġp oultry +Ġharvest ing +ãĤ¦ ãĤ¹ +Bott om +Ġcaution ed +t n +39 6 +ĠNik ki +Ġeval uations +Ġharass ing +Ġbind ings +ĠMon etary +Ġhit ters +Ġadvers ary +un ts +Ġset back +Ġenc rypt +ĠC ait +Ġl ows +eng es +ĠN orn +Ġbul bs +Ġbott led +ĠVoy ager +3 17 +Ġsp heres +p olitics +Ġsubt ract +Ġsens ations +Ġapp alling +Ġ3 16 +Ġenvironment ally +ĠST EM +Ġpub lishes +5 60 +Ġdilig ence +48 4 +Ġadv ises +Ġpet rol +Ġimag ining +Ġpatrol s +ĠInt eger +ĠAs hes +act us +ĠRad iant +ĠL T +it ability +ht aking +Set ting +Ġnu anced +ĠRe ef +ĠDevelop ers +N i +pie ces +99 0 +Lic ense +Ġlow ers +ĠOtt oman +3 27 +oo o +Ġqu itting +mark ets +Beh ind +Ġbas in +Ġdoc s +an ie +fl ash +ct l +Ġcivil ized +ĠFuk ushima +"] ," +ĠK S +ĠHonest ly +ar at +Ġconstruct s +ĠL ans +ĠD ire +ĠLI KE +ĠTrou ble +Ġwith holding +ĠOb livion +Ġsan ity +any a +Con st +Ġgro cer +ĠC elsius +Ġrecount ed +ĠW ife +B order +ate red +h appy +Ġspo iler +Ġlog ically +H all +Ġsucceed ing +Ġpoly morph +Ġax es +ĠShot gun +ĠS lim +ĠPrin ciples +ĠL eth +art a +Ġsc or +Sc reenshot +Ġrelax ation +#$ #$ +Ġdeter rent +idd y +Ġpower less +Ġles bians +Ġch ords +ĠEd ited +se lected +Ġseparat ists +000 2 +Ġair space +Ġturn around +Ġc unning +P ATH +P oly +Ġbomb ed +Ġt ion +x s +Ġwith hold +Ġw aged +ĠLiber ties +Fl ag +Ġcomfort ing +45 4 +ĠI ris +are rs +Ġr ag +Ġrel ocated +ĠGu arant +Ġstrateg ically +Ġgam ma +uber ty +ĠLock heed +g res +Ġgr illed +ĠLow e +st ats +ĠR ocks +Ġsens ing +Ġrent ing +ĠGe ological +ا Ø +ot rop +Ġse w +Ġimproper ly +48 6 +Ġâĸ ł +Ġstar ving +ĠB j +Disc ussion +3 28 +ĠCom bo +ĠFix es +N AT +Ġstri ving +th ora +Ġharvest ed +ĠP ing +Ġplay ful +Ġaven ues +Ġoccup ational +Ġw akes +ĠCou rier +Ġdrum mer +ĠBrow ser +ĠH outh +it u +Ġapp arel +p aste +Ġhun ted +ĠSecond ly +l ain +X Y +ĠP IN +ic ons +Ġcock tails +Ġs izable +Ġhurd les +est inal +ĠRecre ation +Ġe co +64 8 +ĠD ied +m int +Ġfinger prints +Ġdis pose +ĠBos nia +ts y +22 00 +Ġins pected +ĠF ou +Ġf uss +Ġamb ush +ĠR ak +Ġmanif ested +Pro secut +Ġsuff ice +ren ces +Ġcompens ated +ĠC yrus +Ġgen us +ĠWolver ine +ĠTrend s +Ġh ikes +ĠSe en +Ġen rol +C old +Ġpol itely +ĠSl av +ĠRu pert +Ġey ewitness +ĠAl to +Ġun comp +Ġposter ior +M ust +ĠHer z +Ġprogress ively +Ġ23 4 +Ġind ifference +ĠCunning ham +Ġacadem ia +Ġse wer +Ġast ounding +ĠA ES +r ather +Ġeld est +Ġclim bs +ĠAdd s +Ġout cry +Ġcont ag +ĠH ouses +Ġpe pt +ĠMel ania +interest ed +ĠU CH +ĠR oots +ĠHub bard +ĠT BD +ĠRoman ian +fil ename +St one +ĠIm pl +Ġchromos ome +C le +d x +Ġscram bled +ĠP t +Ġ24 2 +OP LE +Ġtremend ously +St reet +Ġcra ving +Ġbund led +ĠR G +p ipe +Ġinj uring +Ġarc ane +Part icip +ĠHero ic +st y +Ġto pping +ĠTemp est +rent ices +b h +Ġpar anoia +ĠUnic ode +Ġegreg ious +Ġ\ ' +ĠOsw ald +Ġgra vel +ĠSim psons +Ġbl and +ĠGuant anamo +Writ er +lin ers +ĠD ice +J C +Ġpar ity +Ġs ided +Ġ23 7 +ĠPyr rha +at ters +d k +F ine +comp an +Ġform ulated +ĠId ol +il ers +hem oth +ĠF av +Ġintr usion +Ġcar rots +ĠL ayer +ĠH acker +Ġ ---------------- +Ġmoder ation +é ģ +oc oc +Ġcharacter ize +ĠTe resa +Ġsocio economic +Ġper k +ĠParticip ation +tr aining +ĠPaul o +ph ys +Ġtrust worthy +Ġembod ied +ĠMer ch +c urrency +ĠPrior ity +Ġte asing +Ġabsor bing +Ġunf inished +ĠCompar ison +Ġdis ple +writ ers +Ġprofess ions +ĠPengu in +Ġang rily +ĠL INK +68 8 +ĠCor respond +Ġprev ailed +Ġcart el +l p +as ms +ĠRed emption +ĠIslam ists +effect s +d ose +ĠL atter +ĠHal ifax +Ġv as +ĠTop ics +ĠN amed +advert ising +zz a +IC ES +Ġret arded +ach able +ĠPupp et +ĠItem Level +Ġret ract +Ġident ifiable +A aron +ĠB uster +s ol +hel le +as semb +H ope +r anged +B a +ĠP urch +é Ģ +ĠSir i +Ġarri vals +Ġ19 12 +Ġshort ened +Ġ3 12 +Ġdiscrep ancy +ĠTem perature +ĠWal ton +Ġkind erg +p olit +Ġrem ix +Ġconnect ors +ãĥĺ ãĥ© +ĠKazakh stan +dom inated +Ġsu gars +im ble +ĠPan ic +ĠDem and +ĠCol ony +on en +ĠM ER +7 75 +ur ia +aza ar +ĠDeg ree +P ri +Ġsun shine +Ġ25 1 +Ġpsychedel ic +Ġdigit ally +ĠBra un +Ġsh immer +Ġsh ave +ĠTel esc +ĠAst ral +ĠVenezuel an +ĠO G +Ġc rawling +Int eg +ĠFe ather +Ġunfold ing +Ġappropri ation +Ġè£ı è +ĠMob ility +ĠN ey +- . +b ilt +L IN +ĠT ube +ĠCon versely +Ġkey boards +ĠC ao +Ġover th +Ġla ure +>> \ +ĠV iper +ach a +Off set +ĠR aleigh +ĠJ ae +J ordan +j p +Ġtotal itarian +Connect or +Ġobserv es +ĠSpart an +ĠIm mediately +ĠSc al +C ool +Ġt aps +Ġro ar +P ast +Ġch ars +ĠB ender +ĠShe ldon +Ġpain ter +Ġbe acon +ĠCreat ures +Ġdownt urn +Ġh inder +ĠAnd romeda +à Ľ +cc oli +ĠF itness +et rical +Ġutil izes +Ġsen ate +Ġen semble +Ġche ers +T W +Ġaff luent +k il +ry lic +ord ering +Com puter +Ġgru esome +ost ics +ĠUb isoft +ĠKel ley +Ġw rench +Ġbourgeois ie +IB LE +ĠPrest on +w orn +ar ist +reat ing +Ġst ained +ar ine +Ġsl ime +EN N +Ġche sts +Ġground water +ann ot +ĠTr ay +ĠLoc ke +ĠC TR +Ġd udes +ĠEx ternal +ĠDec oder +Ġpar amed +ĠMed line +80 9 +ĠD inner +rup al +g z +ĠG um +ĠDem o +j ee +Ġd h +ber man +arch s +Ġen qu +ĠEp stein +Ġdevast ation +Ġfriends hips +ĠAr d +Ġ23 1 +ĠRub in +ĠDist ance +Ġsp urred +Ġd ossier +Ġover looking +\\\\\\\\ \\\\\\\\ +Fore st +ĠCom es +\ ", +ĠIran ians +Ġf ixtures +L aughs +Ġcur ry +ĠKing ston +Ġsqu ash +Ġcat alogue +Ġabnormal ities +Ġdigest ive +.... ..... +Ġsubord inate +og ly +Ġ24 9 +M iddle +Ġmass ac +Ġburg ers +Ġdown stairs +Ġ19 31 +39 4 +ĠV G +Ġl asers +ĠS ikh +ĠAlex a +der ived +Ġcycl ist +ãģ® éŃĶ +onel iness +!!!! !!!! +Ġbuff s +leg ate +Ġrap ing +Ġrecomm ending +ro red +Ġmult icultural +un ique +Ġbusiness men +Ġune asy +ĠM AP +Ġdisp ersed +cipl ine +J ess +ĠK erala +å § +Ġabst raction +Sur v +U h +Ġprin ters +ij a +ow der +Ġanalog ous +ĠA SP +af er +Ġunfold ed +Ġlevel ing +Ġbre ached +ĠH earing +Ġn at +Ġtransl ating +crit ical +Ġant agonist +ĠYes terday +Ġfuzz y +w ash +m ere +Ġbe wild +ĠM ae +V irgin +ph rase +Ġsign aled +ĠH IGH +Ġprot ester +Ġgar ner +unk nown +Ġk ay +Ġabduct ed +Ġst alking +am n +Ġdes erving +ĠR iv +ĠJ orge +Ġscratch ing +ĠS aving +ip ing +Ġte ase +Ġmission ary +ĠMor row +T IME +P resent +Ġchem otherapy +tern ess +ĠH omes +ĠP urdue +Ġst aunch +ĠWhit ney +ĠTH ERE +Î ¼ +iat us +ĠErn est +ĠDe ploy +Ġcove ted +F ML +ĠDial ogue +Ġex ited +f ruit +Ġner d +":" "," +Ġv ivo +ru ly +4 60 +ĠAm en +rehens ible +Ġâ ĺ +D IR +Ġad herence +Ġche w +ĠCo ke +ĠSerge i +dig ital +ĠNe ck +g ently +enth al +/ ) +Ġwe ary +Ġgu ise +ĠConc ord +ĠOn ion +at cher +Ġb inge +ĠDirect ive +Ġman ned +ans k +Ġill usions +Ġbillion aires +38 3 +oly n +odynam ic +ĠWhe at +ĠA lic +Ġcol oured +ĠN AFTA +ab o +Ġmac ros +ind ependent +s weet +Ġsp ac +ĠK abul +Ġ Ä +em e +Ġdict ated +Ġsh outs += { +Ġr ipping +ĠSh ay +ĠCr icket +direct ed +Ġanalys ed +ĠWAR RANT +ag ons +ĠBlaz ers +Ġche ered +Ġar ithmetic +ĠTan z +37 3 +ĠFl ags +Ġ29 5 +Ġw itches +ĠIn cluded +ĠG ained +ĠBl ades +G am +ĠSam antha +ĠAtl antis +ĠPr att +Ġspo iled +ĠI B +ĠRam irez +Pro bably +re ro +ĠN g +ĠWar lock +t p +Ġover he +Ġadministr ations +Ġt int +Ġreg iment +Ġpist ols +Ġblank ets +Ġep ist +Ġbowl s +Ġhydra ulic +Ġde an +Ġj ung +Ġasc end +70 5 +ĠSant iago +à ® +Ġun avoid +ĠSh aman +re b +Ġstem ming +99 8 +ĠM G +st icks +esthes ia +ER O +Ġmor bid +ĠGr ill +ĠP oe +any l +Ġdele ting +ĠSurve illance +Ġdirect ives +Ġiter ations +ĠR ox +ĠMil ky +F ather +Ġpat ented +44 7 +Ġprec ursor +Ġm aiden +ĠP hen +ĠVe gan +ĠPat ent +K elly +Redd itor +Ġn ods +Ġvent ilation +ĠSchwar z +Ġw izards +Ġomin ous +ĠHe ads +ĠB G +Ġl umber +ĠSp iel +Ġis Enabled +Ġancest ral +ĠSh ips +Ġwrest ler +ph i +Ġy uan +ĠRebell ion +Ġice berg +Ġmag ically +Ġdivers ion +ar ro +yth m +ĠR iders +ĠRob bie +ĠK ara +ĠMain tenance +ĠHer b +Ġhar ms +p acked +ĠFe instein +Ġmarry ing +Ġbl ending +ĠR ates +Ġ18 80 +Ġwr ink +ĠUn ch +ĠTor ch +desc ribed +Ġhuman oid +ilit ating +ĠCon v +ĠFe ld +IGH TS +Ġwhistlebl ower +ort mund +ets y +arre tt +ĠMon o +ĠI ke +ĠC NBC +ĠW AY +ĠMD MA +ĠIndividual s +Ġsupplement al +Ġpower house +ĠSt ru +F ocus +aph ael +ĠCol leg +att i +Z A +Ġp erenn +ĠSign ature +ĠRod ney +Ġcub es +idd led +ĠD ante +ĠIN V +iling ual +ĠC th +Ġso fa +Ġintimid ate +ĠR oe +ĠDi plom +ĠCount ries +ays on +Ġextrad ition +Ġdis abling +ĠCard iff +Ġmemor andum +ĠTr ace +Ġ?? ? +se ctor +ĠRou hani +ĠY ates +ĠFree ze +Ġbl adder +M otor +ĠProm ise +ant asy +Ġforesee able +ĠC ologne +cont ainer +ĠTre es +ĠG ors +ĠSin clair +Ġbar ring +key e +Ġsl ashed +ĠStat istical +é ĩ +Ġâĸ º +All ows +Ġhum ility +Ġdr illed +ĠF urn +44 3 +Ġse wage +Ġhome page +Ġcour tyard +Ġv ile +Ġsubsid iaries +aj o +direct ory +Ġam mon +V ers +charg es +Ġ} } +ĠCh ains +Ġ24 6 +n ob +Ġper cept +Ġg rit +Ġfisher men +ĠIraq is +ĠDIS TR +ĠF ULL +ĠEval uation +g raph +at ial +Ġcooper ating +Ġmel an +Ġenlight ened +Ġal i +t ailed +Ġsal ute +Ġweak est +ĠBull dogs +U A +ĠAll oy +Ġsem en +oc ene +ĠWilliam son +s pr +, âĢĶ +ĠG F +itt ens +Be at +ĠJ unk +iph ate +ĠFarm ers +ĠBit coins +ig ers +d h +ĠL oyal +p ayer +Ġentert ained +Ġpenn ed +Ġcoup on +Que ue +Ġweaken ing +c arry +Ġunderest imate +Ġshoot out +Ġcharism atic +ĠProced ure +Ġprud ent +in ances +Ġric hes +Ġcort ical +Ġstr ides +Ġd rib +ĠOil ers +5 40 +ĠPer form +ĠBang kok +Ġe uth +S ER +Ġsimpl istic +t ops +camp aign +Q uality +Ġimpover ished +ĠEisen hower +Ġaug ment +ĠH arden +Ġinterven ed +Ġlist ens +ĠK ok +Ġs age +Ġrub bish +ĠD ed +Ġm ull +pe lling +Ġvide ot +Produ ction +D J +m iah +Ġadapt ations +Ġmed ically +Ġboard ed +Ġarrog ance +Ġscra pped +Ġopp ress +FORM ATION +Ġj unction +4 15 +EE EE +S kill +Ġsub du +ĠSug gest +ĠP ett +Ġle tt +ĠMan ip +ĠC af +ĠCooper ation +T her +Ġreg ained +¶ æ +ref lect +Ġth ugs +ĠShel by +Ġdict ates +ĠWe iner +ĠH ale +Ġbatt leground +s child +Ġcond ol +h unt +osit ories +Ġacc uses +Fil ename +Ġsh ri +Ġmotiv ate +Ġreflect ions +N ull +ĠL obby +¥ µ +ĠS ATA +ĠBack up +Ñ ĥ +n in +ĠCor rection +Ġju icy +ut ra +ĠP ric +Ġrest raining +ĠAir bnb +ĠAr rest +Ġappropri ations +Ġsl opes +Ġmans laughter +Ġwork ings +ĠH uss +ĠF rey +Le ave +ĠHarm ony +ĠF eder +Ġ4 30 +Ġt rench +Ġglad ly +Ġbull pen +ĠG au +b ones +Ġgro ove +Ġpre text +ã ħĭ +Ġtransm itter +ĠComp onent +Ġunder age +ĠEm pires +T ile +Ġo y +ĠMar vin +ĠC AS +Ġbl oss +Ġrepl icated +ĠMar iners +Marc us +ĠBl ocks +Ġliber ated +Ġbutter fly +Fe el +Ġfer mentation +Ġyou tube +Ġoff end +ĠTer m +res ist +Ġcess ation +Ġinsurg ency +Ġb ir +ĠRa ise +59 5 +Ġhypothes es +50 2 +Ġpl aque +ocr at +Ġjack ets +ĠHuff Post +am ong +Ġconf er +48 7 +ĠL illy +Ġadapt ing +ĠF ay +Ġsh oved +ve c +Ġref ine +Ġg on +Ġgun men +z ai +ĠShut tle +ĠI zan +Ġ19 13 +Ġple thora +· · +Ġ5 10 +Ġp uberty +Ġ24 1 +ĠWe alth +ĠAl ma +ĠM EM +ĠAd ults +C as +pr ison +R ace +Ġwater proof +Ġathlet icism +Ġcapital ize +ĠJu ice +Ġillum inated +ĠP ascal +Ġirrit ation +ĠWitness es +ad le +ĠAst ro +Ġf ax +ĠEl vis +Prim ary +ĠL ich +ĠEl ves +Ġres iding +Ġst umble +3 19 +ĠP KK +Ġadvers aries +D OS +ĠR itual +Ġsm ear +Ġar son +ident al +Ġsc ant +Ġmon archy +Ġhal ftime +Ġresid ue +Ġind ign +ĠSh aun +ĠEl m +aur i +A ff +W ATCH +ĠLy on +hel ps +36 1 +Ġlobby ist +Ġdimin ishing +Ġout breaks +Ġgo ats +f avorite +ĠN ah +son ian +ĠBo oster +Ġsand box +ĠF are +ĠMalt a +Ġatt Rot +ĠM OR +ld e +Ġnavig ating +T ouch +Ġunt rue +ĠDis aster +Ġl udicrous +Pass word +ĠJ FK +blog spot +4 16 +ĠUN DER +ern al +Ġdelay ing +T OP +Ġimpl ants +ĠAV G +ĠH uge +att r +Ġjournal istic +ĠPe yton +ĠI A +R ap +go al +ĠProgram me +Ġsm ashing +w ives +print ln +ĠPl ague +in us +EE P +Ġcru iser +ĠPar ish +umin ium +Ġoccup ants +ĠJ ihad +m op +Ġp int +Ġhe ct +ĠMe cca +direct or +ĠFund ing +ĠM ixed +Ġst ag +T ier +Ġg ust +Ġbright ly +ors i +Ġup hill +R D +Ġles ions +ĠBund y +liv ious +Ġbi ologist +ĠFac ulty +ĠAuthor ization +Ġ24 4 +All ow +ï ¸ +ĠGi ul +Ġpert inent +ot aur +es se +ĠRo of +Ġunman ned +35 1 +ĠSh ak +ĠO rient +Ġend anger +D ir +Ġrepl en +ed ient +Ġtail or +Ġgad gets +Ġaud ible +âĺ Ĩ +N ice +Ġbomb ard +ĠR ape +Ġdef iance +ĠTW O +ĠFilip ino +Ġunaff ected +erv atives +Ġso ared +ĠBol ton +Ġcomprom ising +ĠBrew ers +R AL +ĠA HL +icy cle +Ġv ampires +Ġdi pped +oy er +ĠX III +Ġsidew ays +ĠW aste +ĠD iss +ĠâĶľ âĶĢâĶĢ +$ . +Ġhabit ats +ĠBe ef +tr uth +tr ained +spl it +R us +And y +ĠB ram +RE P +p id +è£ ħ +ĠMut ant +An im +ĠMar ina +Ġfut ile +hig hest +f requency +Ġepile psy +Ġcop ing +Ġconc ise +Ġtr acing +ĠS UN +pan el +ĠSoph ie +ĠCrow ley +ĠAd olf +ĠShoot er +Ġsh aky +ĠI G +ĠL ies +ĠBar ber +p kg +Ġupt ake +Ġpred atory +UL TS +/ ** +Ġintox icated +ĠWest brook +od der +he ment +Ġbas eman +AP D +st orage +ĠFif ty +ed itor +G EN +UT ION +ir ting +Ġse wing +r ift +Ġag ony +ĠS ands +Ġ25 4 +C ash +Ġl odge +Ġp unt +N atural +ĠIde as +Ġerrone ous +ĠSens or +ĠHann ity +Ġ19 21 +Ġm ould +ĠG on +kay a +Ġanonym ously +ĠK EY +Ġsim ulator +W inter +Ġstream ed +50 7 +? ", +Ġte ased +Ġco efficient +Ġwart ime +ĠTH R +' '. +ĠBank ing +mp ire +Ġf andom +Ġl ia +G a +Ġdown hill +Ġinterpre ting +Ind ividual +N orm +Ġjealous y +bit coin +Ġple asures +ĠToy s +ĠChev rolet +ĠAd visor +IZ E +Ġrecept ions +70 6 +C ro +Ġ26 2 +Ġcit rus +ir u +Review er +ject ed +U ES +an z +19 81 +ĠWork er +Ġcompl ied +ores cent +contin ental +T on +ĠPr ism +ĠShe ep +Ġ28 8 +n ox +ĠV og +O rd +Ġreal ms +te k +Ġirrig ation +Ġbicy cles +Ġelectron ically +p oly +t all +() ); +Ġaest hetics +ĠInteg rated +Expl ore +Ġd unk +47 6 +p ain +ĠJac ques +ĠD mit +Fram es +Ġreun ited +Ġhum id +D ro +P olitical +Ġyouth ful +Ġent ails +Ġmosqu ito +36 3 +spe cies +Ġcoord inating +ĠMay hem +ĠMagn us +M ount +Impro ved +ĠST ATE +ATT LE +Ġflow ed +Ġtack led +Ġfashion ed +Ġre organ +iv ari +f inger +Ġreluct antly +et ting +ĠV and +you ng +ĠGar land +Ġpresum ption +Ġamen ities +ĠPle asant +on ential +ĠO xy +Ġmor als +ĠY ah +Read y +Sim on +En h +D emon +Ġcl ich +Mon itor +ĠD U +Ġwel comes +Ġstand out +Ġdread ful +Ġban anas +Ġball oons +h ooting +bas ic +Ġsuff ix +Ġd uly +can o +Ch ain +at os +Ġgeop olitical +Ġ( & +ĠGem ini +ÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤ ÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤ +Ġacqu itted +L uck +prot ect +10 24 +Ġsc arcity +Ġmind fulness +ec ided +D N +pr ime +ĠPres idents +ĠVID EO +Ġ( âĪĴ +add ock +N OR +ĠP ru +p un +ĠL OL +)) )) +ĠL iqu +ĠS AS +Ġsty ling +Ġpunish ments +Ġnum b +Ġasc ertain +ĠRock ies +f lu +Th umbnail +Ġperpet rated +ĠSem i +Ġdis arm +ĠOld er +ĠEx ception +Ġexponent ially +ĠCommun ities +Ġabol ish +ĠPart ner +pt oms +Ġ7 77 +ĠFo ley +ĠC ases +Ġgre ase +ĠReb irth +G round +Ġ; ) +ĠDoct rine +ik ini +Y e +ĠBl ossom +Ġpers ists +b ill +Ġinf usion +Ġbud dies +9 11 +ĠPat ient +Ġdem os +Ġacquaint ance +ĠP aw +at ari +Ġx ml +Ġfasc ination +ĠSer ve +Ï Ĥ +br anded +Ġa z +Return s +Ġover shadow +Ġro am +Ġspeed y +n umbered +hel ial +Ġdisc iple +Ġass urances +g iven +pect ing +ĠN atalie +çĶ ° +Ġmosquit oes +rote in +Ġnumer ic +Ġindepend ents +Ġtrans itional +Ġreaction ary +ĠMech dragon +do ctor +Ġshort est +Ġsequ ential +ĠB ac +ĠAccount s +ãģ Į +ach y +ract ive +ĠReg iment +Ġbreat htaking +ffic iency +ĠB ates +Ġ3 11 +Ġward robe +ft s +ĠBer k +Sim ply +ĠRivers ide +iver ing +ident ial +lu cent +Ġen riched +ĠCon ver +ĠG iving +ãĥ Ļ +Ġlegal ize +ĠF TC +Ġfre aking +M ix +Ġter restrial +es ian +ci ents +W ing +LO AD +Ġled ge +ĠViol ent +ĠMet all +Ġ30 8 +Ġs outheastern +hett o +M eat +Ġslow down +Ġret reated +Jere my +end as +**** * +er ic +Ġre ins +opp able +ĠHuman ity +ear ances +rig an +C amera +Ġwa ivers +s oc +Ġalter ation +trans form +ĠC emetery +50 6 +Ġindef inite +Ġstim ulating +y g +60 3 +ĠS op +Ġdescript ive +Ph ase +ĠEd mund +Ġpneum onia +vent us +A mb +Ġlabor atories +ĠEx clusive +ug ar +W ere +Ġmalf unction +Ġhomosexual s +Ġ---- --- +un i +Ġturb ines +ĠEqu ity +D u +Ġmind ed +ĠR H +ĠBlack hawks +Ġfe ats +Ġ17 00 +re pl +36 2 +lad en +Ġindisp ensable +ly ss +tt i +Ġre el +Ġdiver ted +Ġlik eness +Ġsubscript ions +Ġfing ert +Ġfil thy +dest ruct +d raft +ĠBernard ino +l aunch +Ġper plex +ĠS UM +car b +Ġswe ater +ĠVent ure +ĠJ ag +ĠCele b +ĠV oters +Ġstead fast +Ġathlet ics +ĠHans on +ĠDr ac +Tr acker +Ġcomm end +ĠPres idency +ĠD ID +in formed +Ġweb page +P retty +Ġforce fully +ãĥĥ ãĤ¯ +Ġrel ocation +Ġsat ire +â ī +ĠSunder land +æ Ħ +V oice +???? ???? +Ġinform ant +Ġbow el +ĠUn iform +Ġ ..." +Ġpur ge +Ġpic nic +ĠU mb +ĠU PDATE +ĠSapp hire +ĠSt all +le arn +Ġobject ively +Ġob liter +Ġlooph ole +Ġjour neys +Ġo mission +Pro s +ĠSid ney +pl oma +Ġspray ed +Ġg uru +Ġtra itor +Ġtim et +Ġsn apping +ĠSe vent +urn al +ĠUk ip +Ġb owed +por al +l iberal +R os +Quest ions +i OS +Ġsummar ize +ST AT +Ġ18 50 +ap est +Ġl ender +ĠVari able +br inging +ĠL ORD +, ) +Ġcollaps es +x iety +ĠN ed +Y D +ĠSch a +Ġantib ody +Ġdis band +y re +ill usion +Ġro ver +s hed +ĠHiro sh +cc i +Ġcal am +ĠMort on +P interest +Ġ19 28 +ĠE uras +ord es +Ġf ences +ĠIn ventory +ĠVal encia +ĠU d +ĠT iff +Ġsqu e +Ġqu otation +Ġtroubles ome +er ker +QU EST +ĠKing doms +s outh +Ġle vy +Pr ince +ĠSt ing +Ġnick named +Ġapp e +Ġphot ographic +Ġcorp us +re ference +ĠT rog +U nt +) =( +ĠLat via +Ġactiv ating +Ġlicense e +Ġdispar ities +ĠNews letter +ãĥĥ ãĥĪ +Ġfree ing +ĠJe ep +ĠPer ception +ins k +Ġsil icone +ĠHay den +Le an +ĠSuz uki +ibr arian +66 8 +Ġsp or +Ġcorrel ations +ag hetti +Ġtu ber +ĠIP CC +il us +ĠV u +Ġwealth iest +ĠCarb uncle +an za +Ġfool ed +ĠZ ur +Ġd addy +ran o +il ian +Ġknock out +f man +requ ired +ĠWik ileaks +ĠD uffy +ON T +Ġins ol +ĠObject s +Ġb ou +ĠNord ic +ĠIns ert +sc an +Ġd ancers +Ġid iots +major ity +ĠNev ille +ĠFree BSD +Ġt art +pan ic +69 0 +Ġcoc oa +Ġsam pled +Ġlook up +Ind ust +Ġinject ions +gen re +Ġa u +Ġroad way +Ġgen itals +K ind +ĠEx aminer +ĠY az +F resh +Ġpar alysis +ĠAl uminum +Ġre ap +ok é +Ġsl oppy +ĠTun nel +pos ium +ner y +en ic +Ġher bal +ĠOut er +ĠBuild er +Ġinc ur +Ġide ologies +Ġback ups +cons uming +ĠDet ect +de ck +ĠKN OW +ĠG ret +ĠM IC +Ġtough ness +ĠEx hibit +Ġh ive +L es +ĠSCH OOL +ĠAt ari +ald e +ĠN ull +and estine +m ouse +Ġbrig ade +48 9 +Ġrev ol +ĠLaw son +ĠW ah +op oly +eb ted +ĠS aunders +Ġ3 13 +ĠW inc +Ġtab oo +ĠHel met +Ġw edge +ch ip +ĠT ina +b g +Ġinf uri +r n +Ġanomal ies +ĠSy nc +ĠEx am +ĠComm it +ĠDi ary +ĠALS O +ĠDe bor +omed ical +Ġcomprehens ion +6 55 +Ġempower ing +Ġ ire +Ġju ices +ĠE TH +ĠBox ing +=" / +Ġfacilit ated +p oke +ĠPars ons +ĠMod er +tra vel +Ġcivil izations +Ġliber tarians +Ġrun e +ĠCl arks +at hed +Ġcampaign ers +ĠDis patch +ĠFah renheit +ĠCap com +-------- -- +Ġl ace +Ġdr aining +Ġl iner +ĠArt ificial +é n +t ask +] ). +ĠGM O +ĠOper ator +ord inary +ĠInf luence +ĠU ps +Ġpot ency +uss en +osp ons +ĠSw im +ĠDead line +Un ity +Ġcul inary +Ġenlight enment +Ġwe arer +Ġmin ed +Ġp ly +Ġinc est +ĠDVD s +W alk +B TC +Tr ade +Ġdev al +ib and +ĠOvers ight +Palest inian +Ġd art +Ġm ul +L R +Ġrem ovable +ĠReal ms +ì Ŀ +Ġmisc ar +ĠV ulkan +68 5 +è re +ĠS ap +Ġmer ging +ĠCar ly +che ster +Ġbr isk +Ġlux urious +ĠGener ator +Ġbit terness +Ġed ible +Ġ24 3 +T G +Ġrect angle +With No +bel ow +J enn +Ġdark est +Ġh itch +Ġdos age +Ġsc aven +ĠK eller +ĠIllust rated +Certain ly +ĠMaver icks +Marg inal +Ġdiarr hea +Ġenorm ously +Ġ9 99 +sh r +qu art +Ġadam ant +ĠM ew +Ġren ovation +Ġcerv ical +ĠPercent age +en ers +ĠKim ber +Ġflo ats +Ġde x +ĠW itcher +ĠSwan sea +d m +Ġsal ty +y ellow +Ġca pe +ĠDr ain +ĠPaul a +ĠTol edo +les i +Mag azine +ĠW ick +ĠM n +ĠA ck +ĠR iding +AS ON +Ġhom ophobic +AR P +Ġwand ered +C PU +ood oo +ĠP ipe +Ġtight ening +ĠBut t +3 18 +Ġdesert ed +S ession +Ġfacilit ating +J ump +Ġemer gencies +OW ER +Ġexhaust ive +ĠAF TER +Ġheart beat +ĠLab el +ack y +ĠCert ified +ilt ration +Z e +ĠU tt +Ġ13 00 +Ġpres ume +ĠDis p +Ġsur ged +Ġdoll s +Col umb +Ġchim pan +ĠR azor +Ġt icks +Ġcouncill or +Ġpilgr image +ĠReb els +ĠQ C +ĠA uction +x ia +ik k +b red +Ġinsert ion +Ġco arse +d B +SE E +ĠZ ap +ĠF oo +Ġcontem por +ĠQuarter ly +ot ions +ĠAl chemist +ĠT rey +ĠDu o +S weet +80 4 +ĠGi ov +Ġfun n +N in +h off +Ġram ifications +Ġ19 22 +ĠExper ts +az es +Ġgar ments +ar ial +ĠN ab +Ġ25 7 +ĠV ed +Ġhum orous +ĠPom pe +Ġn ylon +Ġlur king +ĠSerge y +ĠMatt is +Ġmisogyn y +ĠComp onents +ĠWatch ing +ĠF olk +ract ical +B ush +Ġt aped +Ġgroup ing +Ġbe ads +Ġ20 48 +Ġcon du +quer que +Read ing +Ġgriev ances +Ult ra +Ġend point +H ig +ĠSt atic +ĠScar borough +L ua +ĠMess i +a qu +ĠPsy Net +ĠR udd +Ġa venue +v p +J er +Ġsh ady +ĠRes ist +ĠArt emis +Ġcare less +Ġbro kers +Ġtemper ament +Ġ5 20 +T ags +ĠTurn ing +Ġut tered +Ġp edd +Ġimpro vised +Ġ: ( +Ġtab l +Ġpl ains +16 00 +press ure +ĠEss ence +marg in +friend s +ĠRest oration +Ġpoll ut +ĠPok er +ĠAugust ine +ĠC IS +ĠSE AL +or ama +Ġth wart +se ek +Ġp agan + º +cp u +Ġg arn +Ġass ortment +ĠI LCS +t ower +Recomm ended +Ġun born +ĠRandom Redditor +ĠRandomRedditor WithNo +Ġparaly zed +Ġeru ption +Ġinter sect +ĠSt oke +ĠS co +B ind +å ¾ +ĠP NG +ĠNeg ative +ĠNO AA +Le on +Ġall oy +ĠL ama +ĠD iversity +5 75 +Ġunderest imated +ĠSc or +Ġm ural +Ġb usted +so on +l if +Ġnone x +Ġall ergy +ĠUnder world +ĠR ays +ĠBl asio +Ġh rs +ĠD ir +Ġ3 27 +by ter +Ġrepl acements +Ġactiv ates +ri ved +M H +Ġp ans +ĠH I +Ġlong itudinal +Ġnu isance +al er +Ġsw ell +ĠS igned +s ci +ĠIs les +ĠA GA +Ġdef iant +Ġson ic +oc on +K C +ĠA im +t ie +ah ah +Ġm L +D X +Ġb isc +ĠBill board +ĠSY STEM +NE Y +ga ard +Ġdist ressed +former ly +Al an +Ġche fs +Ġopt ics +ĠC omet +ĠAM C +Ġredes igned +irm ation +Ġsight ings +38 2 +3 11 +ĠW B +Ġcont raction +ĠT OTAL +D ual +Ġstart led +Ġunderstand ably +Ġsung lasses +ETH OD +Ġd ocker +Ġsurf ing +ĠH EL +ĠSl ack +ton es +Ġsh alt +Vis ual +49 8 +Dep artment +c ussion +Ġunrest ricted +Ġt ad +Ġre name +employ ed +Ġeduc ating +Ġgrin ned +bed room +ĠActiv ities +ĠV elvet +ĠSW AT +Ġsh uffle +ig or +Ġsatur ation +F inding +c ream +ic ter +Ġv odka +tr acking +te c +Ġfore ground +iest a +Ġve hement +ĠEC B +ĠT ie +E y +Ġt urtles +ĠRail road +ĠKat z +ĠFram es +Ġmen ace +ĠFell owship +ĠEss ential +ugg ish +Ġdri p +ch witz +ĠKy oto +s b +ĠN ina +Param eter +Ġal arms +ĠCl aud +Ġpione ering +Ġchief ly +ĠSc ream +Col lection +Ġthank fully +ĠRonald o +åŃ IJ +st rip +ĠDisney land +com mercial +See ing +S oul +Ġevac uate +Ġc iv +ĠAs he +Ġdiv ides +ĠD agger +rehens ive +Ġber ries +ĠD F +Ġs ushi +Ġplur ality +W I +Ġdisadvant aged +Ġbatt alion +ob iles +45 1 +Ġcl ing +Ġunden iable +ĠL ounge +Ġha unt +p he +Ġquant ify +Ġdiff ered +Ġ[* ] +ĠV iz +c um +sl ave +Ġvide og +Ġqu ar +Ġbund les +ĠAl onso +t ackle +Ġneur onal +Ġlandsl ide +conf irmed +ĠDep th +Ġrenew ables +B ear +ĠMaced onia +Ġjer seys +Ġb unk +ĠSp awn +ĠControl s +ĠBuch anan +Ġrobot ics +Ġemphas izing +ĠTut orial +h yp +ist on +Ġmonument al +æ ° +ĠCar ry +Ġt bsp +en ance +H ill +art hed +Ġro tten +De an +Ġtw isting +Ġgood will +Ġimm ersion +L iving +Ġbr ushes +ĠC GI +ĠAt k +tr aditional +Ġph antom +ĠSt amina +Ġexpans ions +ĠMar in +Ġembark ed +ĠE g +int estinal +ĠPE OPLE +ĠBo oth +ĠApp alach +Ġreleg ated +V T +M IT +Ġmust er +Ġwithdraw ing +Ġmicrosc ope +ĠG athering +ĠC rescent +ĠArgent ine +ĠDec re +ĠDomin ic +Ġbud s +ant age +ĠI on +Ġwid ened +ONS ORED +ĠGl oves +iann opoulos +raz en +fe el +Ġrepay ment +Ġhind sight +ĠRE ALLY +ĠPist ol +ĠBra h +Ġwat ts +Ġsurv ives +Ġfl urry +iss y +Al ert +ĠUrug uay +Ph oenix +S low +ĠG rave +ĠF ir +Ġmanage able +Ġtar iff +ĠU DP +ĠPist ons +ĠNiger ian +Ġstrike outs +Ġcos metics +whel ming +f ab +c ape +pro xy +Ġre think +Ġover coming +sim ple +Ġw oo +Ġdistract ing +ĠSt anton +ĠTuls a +ĠD ock +65 9 +Ġdisc ord +ĠEm acs +ĠV es +ĠR OB +Ġreass uring +Ġcons ortium +Muslim s +3 21 +Ġprompt s +se i +ĠH itch +imp osed +ĠF ool +Ġindisc rim +wr ong +bu querque +D avis +! ] +Ġtim eless +ĠNE ED +Ġpestic ide +Ġrally ing +ĠCal der +Ġå ¤ +Ġx p +ĠUn le +ĠEx port +lu aj +B uff +) [ +Ġsq or +S audi +Ġis tg +Ġindul ge +pro c +Ġdisg usted +Ġcomp ounded +Ġn em +Ġschool ing +ĠC ure +process ing +S ol +Ġpro verb +it ized +ĠAlv arez +Ġscar f +Ġrect angular +re ve +Ġh ormonal +ĠSt ress +itiz en +Ġ4 25 +girl s +ĠNo ir +ĠR app +Ġmar ches +ch urch +ĠUs es +Ġ40 5 +ĠBer m +Ġord inances +ĠJud gment +Charg es +ĠZ in +Ġdust y +Ġstraw berries +Ġper ce +ĠTh ur +ĠDebor ah +net flix +ĠLam bert +Ġam used +ĠGu ang +Y OU +R GB +ĠC CTV +Ġf iat +r ang +Ġf ederation +ĠM ant +ĠB ust +ĠM are +respect ive +ĠM igration +ĠB IT +59 0 +Ġpatriot ism +Ġout lining +reg ion +ĠJos é +Ġbl asting +ĠEz ra +B s +Ġundermin es +ĠSm ooth +Ġcl ashed +rad io +Ġtransition ing +ĠBucc aneers +ĠOw l +Ġplug s +Ġh iatus +ĠPin ball +Ġm ig +ĠNut r +ĠWolf e +Ġinteg ers +Ġor bits +ĠEd win +ĠDirect X +b ite +Ġbl azing +v r +Ed ge +ĠP ID +ex it +ĠCom ed +ĠPath finder +ĠGu id +ĠSign s +ĠZ er +ĠAg enda +Ġreimburse ment +M esh +i Phone +ĠMar cos +ĠS ites +h ate +en burg +Ġs ockets +p end +Bat man +v ir +ĠSH OW +Ġprovision al +con n +ĠDeath s +AT IVE +Pro file +sy m +J A +Ġnin ja +inst alled +id ates +eb ra +ĠOm aha +Ġse izing +ĠBe asts +Ġsal ts +M ission +Gener ally +ĠTr ilogy +he on +leg ates +Ġd ime +Ġf aire +par able +G raph +Ġtotal ing +Ġdiagram s +ĠYan uk +ple t +ĠMe h +Ġmyth ical +ĠStep hens +aut ical +ochem istry +Ġkil ograms +Ġel bows +anc ock +ĠB CE +ĠPr ague +Ġimpro v +ĠDev in +Ġ" \ +par alle +Ġsuprem acists +ĠB illion +Ġreg imen +inn acle +Ġrequ isite +ang an +ĠBur lington +ain ment +ĠObject ive +oms ky +G V +Ġun ilateral +Ġt c +Ġh ires +ment al +Ġinvol untary +Ġtrans pl +ĠASC II + ¨ +Ev ents +Ġdoub ted +ĠKa plan +ĠCour age +ig on +ĠMan aging +ĠT art +Ġfalse hood +ĠV iolet +Ġair s +Ġfertil izer +Brit ain +Ġaqu atic +ou f +W ords +ĠHart ford +Ġeven ings +ĠV engeance +qu ite +G all +ĠP ret +Ġp df +ĠL M +ĠSo chi +ĠInter cept +9 20 +Ġprofit ability +ĠId le +ĠMac Donald +ĠEst ablishment +um sy +Ġgather ings +ĠN aj +Charl ie +Ġas cent +ĠProt ector +Ġal gebra +Ġbi os +for ums +EL S +Introdu ced +Ġ3 35 +Ġastron omy +Cont ribut +ĠPol ic +Pl atform +Ġcontain ment +w rap +Ġcoron ary +ĠJ elly +man ager +Ġheart breaking +c air +ĠChe ro +c gi +Med ical +ĠAccount ability +! !" +oph ile +Ġpsych otic +ĠRest rict +Ġequ itable +iss ues +Ġ19 05 +ĠN ek +c ised +ĠTr acking +Ġo zone +Ġcook er +ros is +Ġre open +Ġinf inity +ĠPharm aceutical +ens ional +Att empt +ĠR ory +Mar co +Ġawa its +H OW +t reated +Ġbol st +Ġreve red +Ġp ods +opp ers +00 10 +Ġampl itude +ric an +SP ONSORED +Ġtrou sers +Ġhal ves +ĠK aine +ĠCut ler +ĠA UTH +Ġsplend id +Ġprevent ive +ĠDud ley +if acts +umin ati +ĠY in +Ġad mon +ĠV ag +Ġin verted +Ġhast ily +ĠH ague +L yn +Ġled ger +Ġastron omical +get ting +Ġcirc a +ĠC ic +ĠTenn is +Lim ited +Ġd ru +ĠBY U +Ġtrave llers +Ġp ane +ĠInt ro +Ġpatient ly +Ġa iding +Ġlo os +ĠT ough +Ġ29 3 +Ġconsum es +Source File +Ġ"" " +Ġbond ing +Ġtil ted +Ġmenstru al +ĠCel estial +UL AR +Plug in +Ġrisk ing +N az +ĠRiy adh +Ġacc redited +Ġsk irm +é Ľ +Ġexam iner +Ġmess ing +Ġnear ing +ĠC hern +ĠBeck ham +Ġsw apped +Ġgo ose +K ay +Ġlo fty +ĠWal let +Ġ[ ' +Ġap ocalypse +Ġb amboo +ĠSP ACE +ĠEl ena +Ġ30 6 +ac ons +Ġtight ened +Ġadolesc ence +Ġrain y +Ġvandal ism +ĠNew town +Ġcon ject +c akes +Ġche ated +Ġmoder ators +par ams +E FF +Ġdece it +ĠST L +ĠTanz ania +ĠR I +Ġ19 23 +ĠEx ile +the l +Ġthe olog +Ġquir ky +ĠIr vine +Ġneed y +or is +U m +K a +Ġmail box +3 22 +Ġb os +ĠPet ra +K ING +Ġenlarg ed +O ften +Ġbad ass +Ġ3 43 +ĠPl aces +ĠC AD +Ġpr istine +Ġinterven ing +d irection +Ġl az +ĠD SM +Ġproject ing +ĠF unk +ag og +pay ment +n ov +Ġch atter +AR B +Ġexam inations +ĠHouse hold +ĠG us +F ord +4 14 +B oss +Ġmy stic +Ġle aps +ĠB av +ul z +b udget +Foot ball +Ġsubsid ized +Ġfirst hand +Ġcoinc ide +oc ular +Con n +ĠColl abor +Ġfool s +am ura +ah ar +r ists +Ġsw ollen +Ġexp ended +ĠP au +s up +Ġsp ar +Ġkey note +s uff +Ġunequ al +Ġprogress ing +str ings +ĠGamer gate +Dis ney +ĠEle ven +om nia +Ġscript ed +Ġear ners +bro ther +ĠEn abled +æ ³ +Ġlar vae +ĠL OC +m ess +Wil son +ĠTem plate +success fully +Ġparam ount +Ġcamoufl age +Ġbind s +ĠQu iet +ĠSh utterstock +r ush +Ġmasc ot +fort une +ĠCol t +ĠBe yon +hab i +Ġha irc +Ġ26 7 +ĠDe us +Ġtw itch +Ġconcent rating +Ġn ipples +c ible +Ġg ir +N Z +M ath +n ih +Requ ired +Ġp onder +ĠS AN +Ġwedd ings +Ġl oneliness +N ES +ĠMah jong +69 5 +add le +ĠGar ner +ĠC OUR +Br idge +Ġsp ree +ĠCald well +Ġbri bery +Ġ���� ���� +plug ins +Ġr acket +Ġchamp agne +vers ible +V ote +Ġmod ifiers +May or +6 80 +Ġassemb lies +ĠS ultan +ĠN ing +ĠLad ies +Ġsulf ur +Ġor bs +Ġ---- - +____ ___ +ĠJournal ism +Ġes ports +Ġl ush +Ġh ue +Ġspect ral +H onest +ãĥ ı +Ġbus hes +Ġrein forcement +Ġre opened +ĠWhe els +ĠM org +rie ving +Ġaux iliary +Ġj Query +ĠB AT +tes que +Ġver tex +p ure +f rey +ãĤ º +d os +Ġty ph +Ġc ull +Ġe q +Ġdec on +Ġtoss ing +Ġdispar ate +ĠBr igham +print f +led ged +Ġsu nd +Ġco zy +Ġhepat itis +per forming +Ġav al +ĠG G +f uture +Ġpet ertodd +ĠKos ovo +Ġmagn ets +Al ready +ĠEd ison +ĠCe res +ĠRA ID +Ġbrill iance +57 6 +Ġder ives +Ġhypert ension +ĠÎ Ķ +Ġlamb da +Ġfl air +Ġmission aries +Ġrap es +ĠSt arter +ĠMon ths +Ġdef y +Ġseism ic +ĠR aphael +Ġeuro zone +65 6 +z sche +Ġscr atched +Ġb ows +ĠLenn on +ĠGa ia +Ġdri pping +f acts +A le +Ġfrog s +ĠBre ast +ogene ity +ĠProsecut or +Ġampl ified +ĠHod g +ĠF n +Th ousands +ĠNI H +ĠMonitor ing +FT WARE +ĠPri ebus +ĠG rowing +hun ter +Ġdiagn ose +ĠM ald +ĠL R +Ġcrown ed +Ġburst ing +Ġdiss olution +j avascript +Ġuseful ness +ĠExec ution +: ( +ĠIv ory +a ah +Ġpersecut ed +viol ence +ist as +ĠCr ate +Ġimpuls es +ĠSp ani +ed es +Hand le +ĠZ erg +think able +Last ly +Ġspont aneously +Ġinconven ient +Ġdismiss ing +Ġpl otted +Ġeight y +Ġ7 37 +r ish +ĠThor nton +ath am +Ġsit com +V en +Rec ipe +t el +l und +Ġcle ars +ĠSas uke +Ġ25 8 +Ġopt ing +Ġen raged +est hetic +ĠA e +uch s +Pre p +Fl ow +Ġrun off +ĠE ating +ĠG iles +ĠAct ing +res ources +ib aba +Ġr pm +Ġske wed +ĠBl anc +ĠS akuya +Ġhot ter +Ġ19 24 +op ian +ck o +Ġcr umbling +Ġcapt ains +ĠAppropri ations +le aders +dro pping +an uts +Ġrevers ing +ĠP ose +ĠS ek +Sc ot +ĠIde a +c ise +ĠSloven ia +Ġ3 17 +Do ctor +Ġcro cod +ald i +Se a +ĠFar rell +Ġmerc enaries +ĠR NC +ĠGu ess +Ġp acing +M achine +Streamer Bot +ĠChar ity +Ġ29 8 +Ġcann ons +ĠTob y +TPP StreamerBot +ĠPass ion +cf g +Th om +Ġbad ges +ĠBern stein +. âĢĵ +ĠP OP +ĠCon j +Ġinitial ization +Ġbiod iversity +D ub +Ġfeud al +Ġdisclaim er +Ġc row +Ġign ition +ar f +S HA +Ġk Hz +h azard +ĠArt ists +oe uv +67 9 +ĠRud y +N ine +ĠRam adan +å ½ +itt o +Ġadren aline +C ert +Ġsmell ed +Ġimp unity +Ġag endas +ĠRe born +ĠCon cent +ĠSe ems +Ġo mega +ĠDust in +Ġback er +ĠSau ce +ĠBoy le +W IN +Ġsp ins +Ġpa uses +u pt +Ġshred ded +Ġstra pped +ĠCor ruption +Ġscr atches +Ġn i +Ġatt ire +ĠS AF +Factory Reloaded +ĠI PS +Ġ( % +Ġsem inar +f ocus +c ivil +Ġ18 60 +int osh +Ġcontin ual +Ġabbre vi +ĠS ok +oc obo +X M +Ġfr antic +Ġunavoid able +Ġar tery +Ġannot ations +b ath +Cl imate +Ġd ors +ĠSl ide +co ord +ĠRel oad +ĠL DL +ĠLove craft +Ġunim agin +Ġresemb led +Ġbarr acks +n p +Ġsurrog ate +Ġcategor ized +ãĤ © +Ġvacc inated +Ġdrain age +Ġind ist +ĠWhats App +Ġ18 70 +oler ance +inv oke +am orph +Ġrecon nect +Ġem anc +Ġblind ness +Ġ12 80 +intern et +c ollar +Ġalt ru +Ġab yss +ĠT RI +65 7 +Ġinf used +HE AD +Ġforest ry +ĠWood y +ĠC i +w i +s am +78 4 +hol iday +Ġmog ul +ĠF ees +ĠD EN +In ternal +ur bed +f usc +at om +ĠIll usion +Ġpoll ed +Ġfl ap +Ġco ax +L GBT +An aly +ĠSect ions +ĠCalif orn +em n +Ġh ither +ĠN IGHT +Ġn ailed +ĠPip eline +39 1 +o of +ĠPr imal +vere nd +Ġsl ashing +Ġret ri +avi our +Ġdepart ing +g il +IS C +Ġmid way +Ġultras ound +Ġbeh aving +ĠT ara +class es +V irtual +ĠColon ial +Ġstri pping +Ġorchestr ated +ĠGra ves +45 2 +ĠIron ically +ĠWrit ers +Ġl ends +ĠMan z +Ġra ven +Ġoxid ative +Ġ26 6 +EL F +act ually +asc ar +D raft +Ġfavour able +Ġhumili ating +Ġf idelity +ĠH of +ĠX uan +49 6 +Ġlay ered +at is +79 0 +Ġpay check +it on +K ar +ĠVM ware +ĠFar mer +Ġserv ic +gl omer +Ġsl ump +ĠFab ric +ĠD OC +est ing +Ġreass ure +Ġph yl +v olt +it ory +R ules +Ġoxid ation +Ġpri zed +Ġmist ress +ĠDj ango +WAR N +å ij +Ġenc ode +ĠFeed back +Ġstupid ity +I an +ĠYugoslav ia +× ¨ +ac l +UT E +19 77 +Ġqual ifies +Ġpuls es +pret ty +Ġfro ze +Ġs s +Iter ator +Ġur gently +Ġm ailed +ĠCh am +Ġsust aining +Ġbas il +Ġpupp ies +il ant +ĠP LEASE +l ap +ace ous +F ear +ĠMaster y +aut omatic +ĠT AG +Ġant im +ag les +47 3 +fram es +Ġwh ispers +ĠWho ever +Ġbra very +ĠUK IP +ract ions +"" " +Ġt ame +Ġpart ed +every thing +CON T +Ġind ebted +Ġadd r +re k +IR ED +Ġem inent +cl inton +Ġo usted +Ġreview er +Ġmelt down +Ġre arr +ĠY ao +the real +aby te +Ġst umbling +Ġbat ches +Ġ25 9 +Ġcontrace ptive +Ġprost itute +ens is +De cl +ĠSt rikes +M ilitary +ĠO ath +v acc +pp ings +05 2 +Ġpart Name +amp ing +Rep orts +K I +CH R +Ġsubt ly +sw ers +Bl ake +us ual +Ġcontest ants +Ġcart ridges +ĠGRE AT +Ġbl ush +ĠâĢ º +47 2 +Ġreason ed +ãĥ ¤ +paralle led +Ġd yn +ag ate +Ġnight ly +å Ĩ +55 6 +Ġsem antic +ĠAdv oc +Ġ !! +Ġdisag rees +ĠB W +V eh +Ġharm ing +Ġembr aces +Ġstri ves +Ġin land +ĠK ard +Ġhe ats +ĠGin ny +ut an +ern aut +yl ene +ĠE lev +J D +Ġh ars +ĠStar r +Ġsk ysc +Ġcollabor ators +Us ually +Ġrev olutions +ĠSTAT S +Ġdism antle +Ġconfident ly +Ġkin etic +Al i +Ġpercent ile +Ġextract ing +ill ian +est ead +Ġphysic ists +ĠMarsh al +Ġfell owship +Ġd ashed +ĠU R +ĠSi oux +ĠComp act +am ide +P ython +ĠLe igh +ĠPharm ac +ist rates +her ical +Ġf ue +ĠE min +Ġ( { +ĠNeighbor hood +Ġdisrupt ing +ĠD up +Ġg land +ĠSe v +ĠMar ian +arg on +ĠD und +Ġ< !-- +Ġstr and +Ġstadium s +z os +Ġpsych osis +ĠR ack +Ġbrilliant ly +ï¸ ı +Ġsubmer ged +ĠInst it +ĠCh ow +Ġc ages +ĠH ats +ĠU rs +Ġdil uted +us at +ien ne +ĠMembers hip +ĠBur k +Ġ ie +Ġarche type +D rug +ult on +ĠSp ock +ĠMcK ay +ĠDep end +F eatured +S oc +19 78 +ĠB ere +Ġrelent lessly +Ġcripp ling +Ġar thritis +çĶ Ł +ĠTrop ical +ĠBul g +ĠCher yl +Ġadm irable +Ġsub title +Over ride +Ġorig inating +ĠC CP +Ġsw ore +ĠSo le +ĠDis orders +3 29 +Ġprocess ion +Ġref urb +Ġimm ersed +requ ently +Ġskept ics +Ġcer amic +m itter +en stein +b elt +ĠT IT +b idden +Ġf ir +m ist +> ] +Ġwe ave +ĠParad ox +Ġentr usted +ĠBarcl ays +Ġnovel ist +og ie +80 6 +Ġnin ety +Ġdisag reements +@@@@ @@@@ +ĠAus chwitz +c ars +ĠL ET +t ub +arant ine +P OS +Ġback story +Ġcheer ful +ĠR ag +ek a +bi ased +Ġinexper ienced +ak ra +ĠW itt +t an +Ġrap ist +Ġplate au +ch al +ĠInqu is +exp ression +Ġc ipher +Ġsh aving +add en +re ly +( \ +ism a +ĠReg ulatory +CH AR +ily n +N VIDIA +G U +Ġmur m +la us +Christ opher +Ġcontract ual +ĠPro xy +ĠJa ime +ĠMethod ist +Ġstew ards +st a +per ia +Ġphys iology +Ġbump ed +Ġf ructose +Austral ian +ĠMet allic +ĠMas querade +ar b +Ġprom ul +Ġdown fall +Ġbut cher +Ġb our +ĠIN FORMATION +ĠB is +pect s +ad ena +Ġcontempl ating +ar oo +cent ered +ĠPe aks +Us ed +Ġmod em +Ġg enders +Ġ8 000 +37 1 +Ġm aternity +ĠR az +Ġrock ing +Ġhandgun s +ĠD ACA +Aut om +ĠN ile +Ġtum ult +ĠBenef it +ĠAppro ach +works hop +ĠLe aving +G er +inst ead +Ġvibr ations +Ġrep ositories +49 7 +ĠA unt +ĠJ ub +ĠExp edition +Al pha +Ġs ans +Ġoverd ue +Ġoverc rowd +Ġlegisl atures +Ġp aternal +ĠLeon ardo +Ġexp ressive +Ġdistract ions +Ġsil enced +tr ust +Ġb iking +Ġ5 60 +Ġpropri et +Ġimp osition +Ġcon glomer +Ġ= ================================================================ +ĠTe aching +ĠY ose +int ensive +T own +Ġtroll ing +ĠGr ac +ĠAS US +Y o +Ġspecial s +ĠNep h +ĠGod zilla +Dat abase +ĠHe gel +Ġ27 2 +19 76 +ĠGl oria +Ġdis emb +ĠInvestig ations +ĠB ane +ag ements +St range +Ġtre asury +ĠPl ays +Ġundes irable +Ġwid ening +Ġverb ally +Ġinf ancy +Ġcut ter +f ml +Ġ21 00 +prot otype +f ine +Ġdec riminal +Ġdysfunction al +Ġbes ie +ĠErn st +z eb +Ġnort heastern +Ġa ust +por ate +ĠMar lins +Ġsegreg ated +ew orld +ĠMa her +Ġtra verse +Ġmon astery +ur gy +G ear +s and +Com pl +ĠE MP +Ġpl ent +ĠMer cer +Ġ27 6 +TA BLE +Config uration +H undreds +Ġpr ic +Ġcollabor ating +ĠPar amount +ĠCumm ings +Ġ( < +Ġrecord er +Ġfl ats +Ġ4 16 +wh ose +Font Size +ĠOr bit +Y R +Ġwr ists +Ġb akery +) } +ĠB ounty +ĠLanc aster +Ġend ings +acc ording +ĠSal am +e asy +75 5 +ĠBur r +ĠBarn ett +onom ous +Un ion +Ġpreced ence +ĠScholars hip +ĠU X +Ġroll out +Ġbo on +al m +ĠCan ter +æ µ +Ġround ing +Ġcl ad +Ġv ap +ĠF eatured +is ations +Ġ5 40 +pol ice +Ġunsett ling +Ġdr ifting +ĠLum ia +ĠObama Care +ĠF avor +Hy per +ĠRoth schild +ĠMil iband +an aly +ĠJul iet +H u +Ġrec alling +a head +69 6 +Ġunf avorable +Ġd ances +O x +Ġleg ality +Ġ40 3 +rom ancer +Ġinqu ire +ĠM oves +\ "> +ĠVari ant +ĠMess iah +ĠL CS +ĠBah á +75 6 +Ġeyeb row +Ġ ¥ +ĠMc F +ĠFort y +M as +Ġpan icked +Ġtransform ations +q q +Ġrev olves +ring e +ĠA i +ax e +Ġon ward +ĠC FR +ĠB are +log in +Ġliqu ids +Ġde comp +second ary +il an +ĠCon vert +ami ya +Ġprosecut ing +Ġâī ¡ +ĠYork ers +ĠByr ne +sl ow +aw ei +J ean +Ġ26 9 +ĠSky dragon +Ġ é +ĠNicarag ua +ĠHuck abee +ĠHigh ly +Ġamph ib +ĠPast or +ĠL ets +Ġbl urred +Ġvisc eral +ĠC BO +Ġcollabor ated +z ig +Leg al +Ġapart heid +Ġbr id +Ġpres et +ĠD ET +ĠAM A +× Ķ +arch ing +auc uses +build er +Ġpo etic +Ġem ulator +ĠMole cular +Ġhon oring +ise um +Ġtract or +ĠCl uster +ĠCal m +ared evil +Ġsidew alks +Ġviol in +Ġgeneral ized +ĠAle c +Ġemb argo +Ġfast ball +ĠHT TPS +ĠL ack +ĠCh ill +ri ver +C hel +ĠSw arm +ĠLev ine +ro ying +L aunch +Ġkick er +Ġadd itive +ĠDe als +W idget +cont aining +Ġescal ate +ĠOP EN +Ġtwe aked +Ġst ash +Ġsp arks +ĠEs sex +ĠE cc +Ġconv ict +Ġblog ging +I ER +ĠH L +Ġmurd erers +75 9 +ĠH ib +Ġde pl +ĠJ ord +S ac +Ġdis sect +ĠHow e +os her +Ġcustom izable +ĠFran z +Ġat ro +Ä ĩ +Ġ000 4 +Ġout post +R oss +Ġglyph osate +ĠHast ings +ĠBE FORE +Ġsh ove +o pped +ĠSc ala +Ġam ulet +an ian +Ġexacerb ated +Ġe ater +47 1 +UM E +Ġpul p +izont al +ĠZ am +ĠAT I +imm une +aby tes +Ġunnecess arily +ĠC AT +ĠAx is +Ġvisual ize +à ī +ĠRad ical +f m +Doc uments +ĠFor rest +Ġcontext ual +ĠSy mbol +Ġtent ative +ĠDO ES +ĠGood s +Ġintermitt ent +} : +medi ated +Ġridic ule +Ġathe ism +Ġpath ogens +ĠM um +Ġre introdu +Ġ30 7 +i HUD +Ġflash light +Ġsw earing +Ġp engu +B u +Ġrot ated +ĠCr ane +Ġ() ); +Ġfashion able +Ġendors ing +46 3 +) [ +Ġingest ion +Ġcook s +Ġ9 50 +ot omy +ĠIm am +Ġk a +Ġte aser +ĠGhost s +ĠãĤ µ +19 69 +Ï ĥ +ub by +Ġconver ter +zan ne +end e +ĠPre par +ĠNic kel +ĠChim era +h im +ĠTyr ann +ĠSabb ath +ĠNich ols +Ġra pt +ih ar +Ġshe lling +Ġillum inate +Ġdent ist +ut or +ĠInteg ration +Ġwh ims +ĠLiter ary +Be aut +Ġp archment +ag ara +Br and +Ġder og +âĢ¦ ) +ĠNor se +Ġunw itting +Ġc uc +Ġborder line +Ġupset ting +Ġrec ourse +Ġd raped +ĠRad ar +Ġcold er +ĠPep si +im inary +], [ +65 8 +V i +ĠF rem +ĠP es +Ġveter inary +ĠT ED +ĠEp idem +n ova +k id +Ġdev out +o ct +j ad +M oh +ĠP AY +Ġge ometric +Ġ3 23 +Ġcircum ference +ich ick +19 75 +ĠY uri +ĠSh all +ĠH over +un in +S pr +Ġg raft +ĠHapp iness +Ġdisadvant ages +att acks +Ġhub s +ĠStar Craft +é ĸ +Ġgall eries +ĠKor ra +Ġgrocer ies +ĠGors uch +Ġrap ists +Ġfun gi +ĠTyph oon +V ector +ĠEm press +b attle +4 68 +Ġparas ite +ĠBom ber +S G +ex ist +ĠP f +Ġun se +Ġsurge ons +B irth +ĠUn sure +ĠPrint ed +ĠBehavior al +ĠA ster +Pak istan +Ġun ethical +Ġs v +ĠIo T +Ġlay outs +P ain +Ġconst ants +ĠL W +ĠB ake +Ġtow els +Ġdeterior ation +ĠBol ivia +Ġblind ed +ĠW arden +ĠMist ress +Ġon stage +Ġcl ans +ĠB EST +19 60 +Ġant ique +Ġrhet orical +ĠPer cy +ĠRw anda +, . +B ruce +Ġtra umat +ĠParliament ary +Ġfoot note +id ia +ĠLear ned +se eking +gen ic +Ġdim ensional +H ide +èĢ ħ +Ġintrig ue +in se +Ġle ases +Ġapp rentices +w ashing +Ġ19 26 +V ILLE +Ġsw oop +s cl +Ġbed rooms +on ics +ĠCr unch +comp atible +Ġincap ac +ĠYemen i +ash tra +z hou +d anger +Ġmanifest ations +ĠDem ons +AA F +Secret ary +ACT ED +L OD +Ġam y +ra per +eth nic +4 17 +Ġpos itives +Ġ27 3 +ĠRefuge es +Ġus b +ĠV ald +odd y +ĠMahm oud +As ia +Ġskull s +ĠEx odus +ĠComp et +ĠL IC +ĠM ansion +ĠA me +Ġconsolid ate +storm s +ont ent +99 6 +Ġcl en +Ġm ummy +fl at +75 8 +ĠV OL +oter ic +n en +ĠMin ute +S ov +Ġfin er +R h +ly cer +Ġreinforce ments +ĠJohann es +ĠGall agher +Ġgym n +S uddenly +Ġext ortion +k r +i ator +T a +Ġhippocamp us +N PR +ĠComput ing +Ġsquare ly +Ġmod elling +ĠFor ums +ĠL isp +ĠKrish na +Ġ3 24 +Ġr ushes +Ġens ued +Ġcre eping +on te +n ai +il ater +ĠHorn ets +Ġob livious +IN ST +55 9 +Ġjeopard y +Ġdistingu ishing +j ured +Ġbeg s +sim ilar +ph ot +5 30 +ĠPark way +Ġs inks +ĠHearth stone +ib ur +ĠBat on +Av oid +Ġd ancer +Ġmag istrate +ary n +Ġdisturb ances +ĠRom ero +Ġpar aph +Ġmis chief +âĸ ĵ +ĠSh aria +Ġur inary +r oute +iv as +f itted +Ġeject ed +ĠAl buquerque +Ġ4 70 +Ġirrit ated +ĠZ ip +ĠB iol +à į +Ġden ounce +Ġbin aries +ĠVer se +Ġopp os +ĠKend rick +ĠG PL +Ġsp ew +ĠEl ijah +ĠE as +Ġdr ifted +so far +Ġannoy ance +ĠB ET +47 4 +ĠSt rongh +it ates +ĠCogn itive +oph one +ĠIdent ification +ocr ine +connect ion +Ġbox er +ĠAS D +ĠAre as +Y ang +t ch +ull ah +Ġdece ive +Comb at +ep isode +cre te +W itness +Ġcondol ences +ht ar +Ġhe als +Ġbuck ets +ĠLA W +B lu +Ġsl ab +ĠOR DER +oc l +att on +ĠSteven son +ĠG inger +ĠFriend ly +ĠVander bilt +sp irit +ig l +ĠReg arding +ĠPR OG +Ġse aling +start ing +Ġcard inal +ĠV ec +ĠBe ir +Ġmillisec onds +we ak +per se +Ġster ile +ĠCont emporary +ĠPh ant +ĠCl o +Ġout p +Ġex iled +Ġ27 7 +Ġself ie +Ġman ic +Ġn ano +ter ms +Alex ander +Ġres olves +Ġmillenn ia +Ġexpl odes +Ġconst ellation +Ġadul tery +m otion +D OC +Ġbroad casters +Ġkinderg arten +ĠMay weather +ĠE co +ich o +Ġ28 7 +l aun +Ġm ute +Ġdisc reet +Ġpres chool +Ġpre empt +De lete +ĠFre ed +P i +H K +Ġblock er +ĠC umber +Ġw rought +d ating +Ġins urer +Ġquot as +Ġpre ached +Ġev iction +ĠReg ina +ĠP ens +Ġsevent een +ĠN ass +D ick +Ġfold s +Ġd otted +ĠA ad +Un iversal +Ġp izz +ĠG uru +Ġso ils +Ġno vice +ĠNe ander +Ġst ool +Ġdeton ated +ĠPik achu +ĠMass ive +IV ER +ĠAb del +Ġsubdu ed +Ġtall est +Ġprec arious +Ġa y +r ification +ĠOb j +c ale +Ġun question +cul osis +ad as +igr ated +D ays +Ġque ens +ĠGaz ette +ĠCol our +ĠBow man +ĠJ J +ï ve +Ġdomin ates +Stud ent +Ġm u +Ġback log +ĠElect ro +Tr uth +48 3 +Ġcond ensed +r ules +ĠCons piracy +Ġacron ym +hand led +ĠMat te +j ri +ĠImp ossible +l ude +cre ation +Ġwar med +ĠSl ave +Ġmis led +Ġfer ment +ĠK ah +ink i +ke leton +cy l +ĠKar in +Hun ter +Reg ister +ĠSur rey +Ġst ares +ĠW idth +ĠN ay +ĠSk i +Ġblack list +uck et +Ġexp ulsion +im et +Ġret weet +vant age +Fe ature +Ġtro opers +Ġhom ers +9 69 +Ġconting ency +ĠW TC +ĠBrew er +fore ign +W are +S olar +Ġund ue +RE C +ulner able +path ic +ĠBo ise +Ġ3 22 +Ġarous ed +ĠY ing +ä¸ į +uel ess +Ġp as +Ġmor p +Ġfl oral +Ex press +ud ging +k B +ĠGr anted +Ø ¯ +ĠMich a +ĠGoth ic +ĠSPEC IAL +ĠRic ardo +F ran +Ġadminister ing +6 20 +por a +Ġ ® +Ġcomprom ises +Ġb itten +Ac cept +Th irty +Ð ² +Ġmater ially +ĠTer r +ig matic +ch ains +Ġdo ve +stad t +Mar vel +FA ULT +Ġwind shield +Ġ3 36 +ad ier +Ġsw apping +Ġflaw less +ĠPred ator +ĠMiche le +Ġprop ulsion +ĠPsych ic +Ġassign ing +Ġfabric ation +Ġbar ley +l ust +Ġtow ering +Ġalter cation +ĠBent ley +Sp here +Ġtun a +ĠClass es +Fre edom +un er +L ady +v oice +Ġcool est +or r +Ġpal p +$ { +Ġhyster ia +ĠMet atron +p ants +Ġspawn ing +Exper ts +ĠInvest ors +ĠAn archy +Ġshr unk +ĠVict im +Ġ28 9 +Ġec stasy +ĠB inding +58 5 +ĠMel ody +57 8 +ot ally +ĠE tsy +lig a +Ġapplaud ed +Ġswe ating +Ġredist ributed +Ġpop corn +Ġsem inal +f ur +ĠNeuro science +R and +ĠO st +ĠMadd en +ĠIncre asing +ĠDaw kins +ĠSub way +Ġar sen +cons erv +B UR +Ġsp iked +ĠLy ft +ĠImper ium +ĠDrop box +Ġfav oured +Ġencomp asses +gh ost +Ġins pires +Ġbur geoning +ĠY oshi +ĠVert ical +ĠAud itor +Ġint ending +Ġfilib uster +Bl oom +f ac +ĠCav s +ign ing +Ġcowork ers +ĠBarb arian +rem ember +FL AG +Ġaudit ory +ason ry +Col lege +Ġmut ed +gem ony +ob in +ĠPsych o +9 68 +Ġlav ish +Ġhierarch ical +ĠDr one +ou k +Ġcripp led +ĠMax im +Sl ot +Ġqu iz +ĠV id +if ling +Ġarchae ologists +Ġabandon ment +d ial +le on +ĠF as +T ed +Ġr aspberry +Ġmaneu vers +Ġbehavi ours +Ġins ure +Ġrem od +Sw itch +h oe +Ġsp aced +Ġafford ability +ĠF ern +not ation +ĠBal anced +Ġoccup ies +en vironment +Ġneck lace +Ġsed an +F U +ĠBrav o +Ġab users +ĠAn ita +met adata +ĠG ithub +ait o +ĠF aster +ĠWass erman +ĠF lesh +Ġth orn +r arily +ĠMer ry +w ine +Ġpopul ace +ĠL ann +Ġrepair ing +Ġpsy che +Ġmod ulation +aw aru +âĢĭ âĢĭ +ari j +Ġdecor ations +Ġapolog ise +ĠG arg +app ly +Ġgive away +ĠFl an +ĠWy att +U ber +Ġauthor ised +ĠMor al +HAHA HAHA +activ ate +Ġtorped o +ĠF AR +Ġam assed +ĠA ram +ark in +ĠVict ims +st ab +Ġo m +ĠE CO +Ġopio ids +Ġpurpose ly +ĠV est +Ġer g +at an +ĠSur gery +Ġcorrect ing +ĠOrt iz +ĠBe et +Ġrev oke +Ġfre eway +ĠH iggins +F ail +ĠFar ms +ĠAT P +h ound +Ġp oking +ĠCommun ists +mon ster +iment ary +Ġunlock ing +Ġunf it +we ed +en ario +at ical +ĠEnlight enment +ĠN G +ĠComp ensation +de en +ĠWid ow +ĠCind y +ĠAfter wards +Ġ6 000 +ikh ail +ag ically +Ġrat ified +Ġcasual ty +H OME +p sey +f ee +Ġspark ling +Ġd é +Ġconcert ed +C atal +Ġcomp lying +ĠA res +ĠD ent +Sh ut +Ġsk im +ad minist +Ġhost ilities +ĠG ins +Ġ6 08 +Ġm uddy +ĠMc Int +ĠDec ay +5 25 +Ġconspic uous +ĠEx posure +Ġresc ind +Ġwear able +Ġ3 28 +our met +ah s +ĠRob ots +Ġe clips +inst ance +ĠRE PORT +ĠApp l +0 30 +ĠSk ies +01 00 +Ġfall acy +S ocket +ĠRece iver +Ġsol ves +ĠButter fly +ĠSho pping +ĠFI RE +65 4 +Med ic +Ġsing ers +ĠNeed less +'' '' +isher s +ĠD ive +58 8 +Ġselect ively +Ġcl umsy +88 9 +Ġpurch aser +ear ned +ard y +Ġbenef iting +eng lish +Ġyield ing +ĠP our +Ġspin ach +Ġdel ve +ĠC rom +6 10 +Ġexport ing +ĠMA KE +Ġ26 3 +Ġg rop +Ġenv oy +ĠInqu iry +ĠLu igi +d ry +ĠT uring +Thumbnail Image +ĠVar iety +Ġfac et +Ġfl uffy +Ġexcerpt s +Ġsh orth +ĠOl sen +CL UD +Ġrel iant +ĠUN C +T our +Ġbat hing +Comp any +Ġglobal ization +P red +ĠMalf oy +Ġh oc +j am +craft ed +ĠBond s +ĠKiss inger +Eng land +Ġorder ly +cat entry +Ġ26 1 +Ġexch anging +ĠInt ent +ĠAmend ments +D OM +Ġst out +³³³³³³³³ ³³³³³³³³ +ĠAir bus +Ġ27 8 +hy de +P oll +Item ThumbnailImage +Ġlooph oles +ĠPill ar +Ġexpl or +St retch +A part +Ġun married +Lim it +ĠTransform ers +Ġintellect ually +unct ure +18 00 +Ġd arn +B razil +Ġleft over +ber us +f red +Mine craft +3 26 +ĠForm s +Ġproof s +ĠDes igned +Ġindex es +ĠSupp ose +EM S +ĠL oving +ĠBon nie +im ating +OT US +Ġconduct or +Ġbehav ed +ĠF ren +Ġsy nerg +Ġmillenn ium +Ġcater ing +ĠL auder +W r +ĠY iannopoulos +ĠAT F +Ġensl aved +Ġawaken ed +D VD +ĠED ITION +ĠConc ert +ĠChall enger +ĠH aku +umer ic +Ġdep recated +ĠSH AR +4 12 +Ġdy stop +Ġtremb ling +Ġdread ed +ĠSp ac +p adding +Re pl +ĠG arrison +M ini +Ġun paralleled +am ar +URR ENT +w reck +c ertain +t al +ĠC LS +app ings +Ġsens ed +Ġf encing +ĠPas o +ĠDes k +Ġsc off +Ġcontem plate +ĠL iga +l iquid +75 7 +Ġapp rentice +ĠUCH IJ +5 70 +ĠTh ousand +ĠIll um +Ġchampion ed +ãĤ Į +Ġelect ors +Ġ3 98 +ĠH ancock +round ed +ĠJ OHN +Ġuns atisf +Ġqual ifier +ĠGad get +EN E +Ġdead liest +ĠPl ants +Ġ ions +Ġacc ents +Ġtwe aking +Ġsh aved +F REE +ĠCh aser +Again st +9 60 +Ġmeth amphetamine +Ġnormal ized +Ġ$ \ +ĠPre cision +ĠGu am +Ġch oked +ĠX II +ĠCast ing +Tor rent +Ġscal p +ĠJagu ar +w it +Ġsem ic +ix ie +ĠG ould +Ġconf ines +N usra +ĠL on +ĠJ ugg +y cle +ĠCod ec +E gypt +Ġrest rain +ĠAl iens +Ġch oking +ĠD unk +ĠBell a +ab c +Ġsl ang +Ġneuro trans +s av +Ġempower ment +â ĨĴ +Ġclim bers +ĠM im +ĠF ra +ros se +Cap ital +ĠCth ulhu +Inter face +Ġprof icient +ĠIN TO +Ġ3 18 +ront al +5 80 +ĠDes pair +K enn +Ġscrim mage +ĠCo at +as ions +Ġwall paper +ĠJ ol +Ġresurg ence +Ġant iv +ĠB alls +² ¾ +Ġbuff ers +Ġsub system +ĠSt ellar +ĠL ung +A IDS +Ġerad icate +Ġblat antly +Ġbehav es +ĠN un +Ġant ics +ex port +DE V +w b +Ġph p +ĠInteg rity +Ġexplore r +Ġrev olving +auth ored +g ans +Ġbas k +Ġas ynchronous +å į +TH ING +69 8 +G ene +ĠR acer +ĠN ico +iss ued +Ġser mon +p ossibly +Ġsize of +Ġentrepreneur ial +ox in +ĠMin erva +Ġpl atoon +n os +ri ks +A UT +ĠAval anche +ĠDes c +ij 士 +ĠP oc +Ġconf erred +Î » +Ġpat ched +F BI +66 2 +Ġfract ures +Ġdetect s +Ġded icate +Ġconstitu ent +Ġcos mos +W T +Ġswe ats +Ġspr ung +b ara +s olid +Ġuns us +Ġbul ky +ĠPhilipp e +ĠFen rir +Ġtherap ists +ore al +^^ ^^ +Ġtotal ed +Ġboo ze +ĠR PC +Prosecut ors +Ġdis eng +ĠSh ared +Ġmotor cycles +Ġinvent ions +Ġlett uce +ĠMer ge +ĠJ C +Ġspiritual ity +ĠWAR NING +Ġunl ucky +ĠT ess +Ġtong ues +ĠD UI +T umblr +Ġle ans +Ġinv aders +Ġcan opy +ĠHur ricanes +ĠB ret +ĠAP PLIC +id ine +ick le +Reg arding +Ġve ggies +Ġe jac +ju ven +F ish +D EM +ĠD ino +Th row +ĠCheck ing +be ard +( & +Ġj ails +Ġh r +trans fer +iv ating +Ġfle ets +ĠIm ag +ĠMc Donnell +Ġsnipp et +Is a +ĠCh att +ĠSt ain +ĠSet FontSize +ĠO y +ĠMathemat ics +49 4 +Ġelectro ly +ĠG ott +ĠBr as +B OOK +ĠF inger +d ump +Ġmut ants +Ġrent als +Ġinter tw +Ġc reek +ail a +Bro ther +ĠDisc ord +pe e +raw ler +Ġcar p +Ġ27 9 +ãĤ· ãĥ£ +rel ations +Ġcontr asts +Col umn +Ġrec onnaissance +Ġun know +Ġl ooting +Ġregul ates +Ġopt imum +ĠChero kee +ĠA ry +Lat est +Ġroad side +Ġd anced +ĠUnic orn +A cknowled +Ġuncont roll +ĠM US +at io +ch ance +ha ven +VAL UE +Ġfavour ites +Ġceremon ial +b inary +pe ed +wood s +EM P +Ġv ascular +Ġcontempl ated +Ġbar ren +ĠL IST +Y ellow +ospons ors +Ġwhisk y +ĠM amm +ĠDeV os +min imum +H ung +44 2 +P ic +ĠSnap dragon +77 6 +Ġcar ving +Ġund ecided +Ġadvantage ous +Ġpal ms +ĠA Q +Ġst arch +L oop +Ġpadd le +Ġfl aming +ĠHor izons +An imation +bo ost +Ġprob abilities +ĠM ish +Ġex odus +ĠEditor ial +Ġfung us +Ġdissent ing +ĠDel icious +rog ram +ĠD yn +d isk +t om +Ġfab rics +ĠC ove +ĠB ans +Ġsoft en +ĠCON S +Ġin eligible +Ġestim ating +ĠLex ington +pract ice +of i +Ġshe dding +ĠN ope +Ġbreat hed +ĠCorinth ians +y ne +ek i +B ull +Ġatt aching +reens hots +Ġanaly se +ĠK appa +Ġuns ustainable +Ġinter pol +ank y +he mer +Ġprot agonists +Ġform atted +ĠBry ce +ĠAch illes +ĠAb edin +sh ock +Ġb um +b os +qu a +ĠW arn +q t +ĠDi abetes +8 64 +ĠIn visible +Ġvan ish +Ġtrans mitting +Ġmur ky +ĠFe i +Ġawa ited +ĠJur assic +umm ies +Ġmen acing +g all +C ath +B uilt +ild o +ĠV otes +Ġon t +Ġmun itions +ĠFre em +ÃŃ n +Ġdec ency +lo pp +ie ved +ĠG ord +Ġun thinkable +ĠNews week +Ġ3 21 +He at +Ġpresent er +ji ang +Ġpl ank +ĠAval on +Ġben z +ĠR out +Ġslam ming +ĠD ai +ou ter +ĠCook ie +ĠAlic ia +ge y +Ġvan ity +Ġow l +á µ +t ested +ĠAw akens +Ġcan v +Ġblind ly +ĠRid ley +ĠEm ails +Requ ires +ĠSer bian +ograp hed +if rame +eter ia +Ġaltern ating +qu iet +Ġsoc iology +ĠUn lock +ĠCommun ism +Ġo ps +Ġatt ribution +Ġab duction +ĠAb ram +Ġsidel ined +ĠB OOK +Ġref ining +ĠFe eling +ĠOs lo +ĠPru itt +r ack +ang ible +Ġcaut iously +ĠM ARK +eed s +M ouse +ĠStep h +ĠP air +S ab +99 7 +ĠBa al +B ec +Ġcomm a +ĠP all +ĠG ael +Ġmisunder stand +ĠP esh +Order able +Ġdis mal +ĠSh iny +% " +Ġreal istically +Ġpat io +ĠG w +ĠVirt ue +Ġexhaust ing +wh atever +oph ys +y ip +4 18 +Ad just +ĠWa iting +ess on +ĠMaz da +ĠDo zens +Ġstream lined +Ġincompet ence +ĠM eth +Ġeth os +ON ES +Ġincent iv +Ġgr itty +ĠBut cher +Head er +Ġexp onential +à Ł +Ġcorrel ate +Ġcons ensual +s ounding +R ing +Orig in +Ġcon clusive +fe et +ac ly +ĠF ernandez +Buy able +Ġd ucks +aunt lets +Ġel ong +Ġ28 6 +Ġsim ul +G as +ĠK irst +Ġprot r +ĠRob o +ĠAo E +op ol +Ġpsych ologically +sp in +ilater ally +ĠCon rad +W ave +44 1 +ĠAd vertisement +ĠHarm on +ĠOri ental +is Special +Ġpresum ptive +Ġw il +ĠK ier +ne a +Ġp pm +Ġhar bour +ĠW ired +comp any +Ġcor oner +atur days +ĠP roud +ĠN EXT +ĠFl ake +val ued +ce iver +Ġfra ught +Ġc asing +Ġrun away +Ġg in +ĠLaure nt +ĠHar lem +ĠCur iosity +qu ished +Ġneuro science +ĠH ulu +Ġborrow er +Ġpetition er +ĠCo oldown +W ARD +Ġinv oking +conf idence +For ward +Ġst s +pop ulation +Delivery Date +Fil m +ĠC ov +quick Ship +quickShip Available +prim ary +isSpecial Orderable +inventory Quantity +channel Availability +BO X +ĠMulti player +ĠJen ner +77 8 +ĠM d +Ġ~ /. +M N +Ġchild ish +Ġantioxid ant +ĠChrom ebook +Ġ27 4 +Ġscreen play +Ġadvent urous +ĠRelations hip +respons ive +ming ton +Ġcorner stone +ĠF ey +F IR +Ġrook ies +ĠF eaturing +Ġorig inate +Ġelectro des +ant es +Ġscript ures +Ġgl ued +Ġdiscont ent +Ġaff licted +lay out +B rave +Ġm osa +ĠQuant ity +ĠH ik +w inner +H ours +Ġent ail +ĠCell s +olog ue +Ġv il +Ġpre acher +Ġdecor ative +d ifferent +Ġprejud ices +ĠSm oking +ĠNotting ham +so Type +Ġrhyth ms +ĠAl ph +bl ast +Ste el +ĠDaniel le +Ġstr ife +Ġrem atch +so DeliveryDate +ĠF ork +t rip +ol ulu +hes es +C G +ĠPOLIT ICO +ost a +ĠDr ift +é¾įå ¥ +é¾įå¥ ij士 +Ġvet ting +ĠJin ping +ĠRec ession +Min or +ĠF raud +enf ranch +Ġconven ed +ĠNA ACP +ĠMill ions +ĠFarm ing +ĠW oo +ĠFl are +rit o +imm igrant +Ġvac ancy +ĠHE AD +ĠV aj +eg al +ĠV igil +Stud y +Ġru ining +Ġr acks +Ġhe ater +ĠRand olph +ĠBr ush +ĠT ir +Ø ¨ +Ġc ov +% ] +Ġrecount s +ĠO PT +ĠM elt +Ġtr uce +Ġcas inos +Ġcrus ade +Ġcarn age +Ġstri pe +ĠK yl +Text ures +Ġ6 98 +Ġpro clamation +Ġgood ies +Ġ........ .. +pro claimed +P olit +Ġtop ical +Ġspecial ize +ĠA min +g m +Ġanch ored +Ġbear ings +s ample +ĠHigh land +ĠAut ism +Ġmerc enary +Ġinterview er +L ER +ĠSom ers +Ġembry o +ĠAss y +Ġ28 1 +ĠEd iting +ĠCh osen +6 60 +Ġp ci +ĠThunder bolt +BI LL +Ġchuck led +jri wal +h of +Ġearth ly +() { +ind ependence +Ġdisp ers +ĠV endor +ĠG areth +Ġp als +P enn +ĠSub mit +ic um +Th u +Ġcl andestine +Ġcann ibal +ĠCl erk +E Stream +gal itarian +âĻ ¥ +g ew +Ġhor rend +ĠL ov +ĠRe action +ocr in +Class ic +Ġecho ing +Ġdiscl osing +ĠIns ight +og un +ĠInc arn +upload s +pp erc +guy en +Ġ19 01 +ĠB ars +68 7 +Ġb ribes +ĠFres no +ur at +ĠRe ese +Ġintr usive +Ġgri pping +ĠBlue print +ĠR asm +un ia +man aged +ĠHeb do +Ġ3 45 +Ġdec oding +Ġpo ets +Ġj aws +ĠF IGHT +am eless +ĠMead ows +ĠHar baugh +Inter view +ĠH osp +ĠB RA +Ġdelet ion +m ob +W alker +ĠMoon light +ĠJ ed +ĠSoph ia +Ġus ur +Ġfortun ately +ĠPut ting +ĠF old +Ġsan itation +Ġpart isans +IS ON +B ow +ĠCON C +ĠRed uced +ĠS utton +Ġtouch screen +Ġembry os +âĢ¢âĢ¢ âĢ¢âĢ¢ +ĠK rug +com bat +ĠPet roleum +Ġam d +ĠCos mos +Ġpresc ribing +Ġconform ity +ours es +Ġplent iful +Ġdis illusion +ĠEc ology +itt al +Ġf anc +Ġassass inated +regn ancy +Ġperenn ial +ĠBul lets +Ġst ale +Ġc ached +ĠJud ith +ĠDise ases +All en +Ġl as +Ġsh ards +ĠSu arez +ĠFriend ship +inter face +ĠSupp orters +add ons +46 2 +ĠIm ran +ĠW im +Ġnew found +ĠM b +An imal +Ġd arling +and e +Ġrh y +ĠTw isted +pos al +yn ski +Var ious +× ľ +ĠK iw +uy omi +Ġwell being +ĠL au +an os +Ġunm ist +Ġmac OS +Ġrest room +ĠOl iv +ĠAir ways +Ġtimet able +9 80 +Ġrad ios +v oy +ias co +Ġcloud y +ĠDraw ing +Any thing +Sy ria +ĠH ert +st aking +Ġun checked +Ġb razen +ĠN RS +69 7 +onom ic +est ablish +Ġl eng +Ġdi agonal +ĠF ior +L air +ĠSt ard +Ġdef icient +jo ining +be am +Ġomn ip +Ġbl ender +Ġsun rise +Mo ore +ĠF ault +ĠCost ume +ĠM ub +Fl ags +an se +Ġpay out +ĠGovern ors +ĠD illon +ĠBan ana +N ar +Ġtra iled +Ġimperial ist +um ann +ats uki +4 35 +ĠRoad s +Ġsl ur +ĠIde ally +Ġt renches +C trl +Ġmir rored +ĠZ el +ĠC rest +Comp at +ĠRoll s +sc rib +ĠTra ils +omet ers +w inter +Ġimm ortality +il ated +Ġcontrad icts +un iversal +ill ions +ĠM ama +opt im +AT URE +Ġge o +et ter +ĠCar lo +4 24 +Ġcanon ical +ĠStrongh old +n ear +Ġperf ume +Ġorche stra +od iac +Ġup he +Ġreign ing +vers ive +Ġc aucuses +ĠD EM +Ġinsult ed +Ġ---- -- +ĠCr ush +Ġroot ing +ĠWra ith +Ġwh ore +Ġto fu +C md +ĠB ree +Ġ$ _ +Ġr ive +ĠAd vertising +Ġw att +ĠH O +Ġpersu asive +ĠParam eters +Ġobserv ational +ĠN CT +ĠMo j +ĠSal on +Ġtr unc +Ġexqu isite +ĠMar a +Ġpo op +ĠAN N +Ex c +ĠWonder ful +ĠT aco +Ġhome owner +ĠSmith sonian +orpor ated +mm mm +Ġlo af +ĠYam ato +ĠInd o +Ġcl inging +á s +Ġimm utable +h ub +Or ange +Ġfingert ips +ĠWood en +ĠK idd +ĠJ PM +ĠDam n +C ow +c odes +48 2 +Ġiniti ating +ĠEl k +ĠCut ting +Ġabsent ee +ĠV ance +ĠLil ith +G UI +Ġobsc ured +Ġdwar ves +ĠCh op +ĠB oko +Val ues +Ġmult imedia +Ġbrew ed +Reg ular +CRIP TION +ĠMort al +Ġa pex +Ġtravel er +Ġbo ils +Ġspray ing +Rep resent +ĠStars hip +4 28 +Ġdisappro val +Ġshadow y +Ġlament ed +ĠRe place +ĠFran ç +67 7 +d or +Ġunst oppable +Ġcoh orts +gy n +ĠClass ics +ĠAm ph +Ġsl uggish +ĠAdd iction +ĠPad res +Ġins cription +Ġin human +min us +ĠJere miah +at ars +Ter ror +ĠT os +ĠSh arma +ast a +c atch +Ġpl umbing +ĠTim bers +Sh ar +H al +ĠO sc +Ġcou pling +hum ans +Ġsp onge +Ġid ols +ĠSp a +ĠAdv ocate +ĠBe ats +lu a +Ġtick ing +Ġload er +ĠG ron +8 10 +Ġstim ulated +Ġside bar +ĠManufact urer +ore And +19 73 +Ġpra ises +ĠFl ores +dis able +ĠElect rical +ra ise +E th +Ġmigr ated +Ġlect urer +K ids +ĠCa vern +Ġk ettle +Ġgly c +ĠMand ela +ĠF ully +å§ « +FIN EST +Ġsquee zing +ĠRy der +amp oo +oreAnd Online +Inst oreAndOnline +Buyable InstoreAndOnline +Ġcommem orate +ĠRamp age +Aust in +ĠSh roud +ĠRu ins +9 15 +ĠK H +Ġwater front +ĠE SC +b aby +ĠC out +ĠEm blem +Ġequival ents +49 2 +Un ique +ĠNiet zsche +brow ser +Ġim itation +ĠWere wolf +ĠKir in +ac as +' ," +Ġà ¾ +Review ed +Ġc unt +Ġvo ic +ĠLen ovo +Ġbond ed +48 1 +Ġinhib itors +Ġendeav ors +ĠHav ana +ĠSt out +ĠJ olly +A ctor +*/ ( +Ġoccur rences +ĠT ens +Incre ased +ĠACT ION +Ġ ãĢĮ +ĠRank ings +ĠB reat +Ġ30 9 +D ou +Ġimpact ing +ĠDuc hess +pre fix +Q B +Ġsummon ing +Ġbest owed +ĠKe pler +ĠPOW ER +c ube +ĠK its +ĠG rip +Ġop ium +Ġrep utable +t oc +ich ael +ĠR ipple +Ġcaf é +ĠZ oom +ĠBur ma +Ġwa ive +Ġst alls +Ġdem eanor +inc erity +Ġfluor ide +ĠSH OULD +Par is +Ġlong ing +Ġpl at +Ġgross ly +Ġbull s +Ġshowc asing +ex pected +ĠG addafi +engine ering +Re peat +ĠK ut +Ġconce ivable +Ġtrim med +osc ope +ĠCand idate +ĠT ears +rol og +Lew is +S UP +Ġroad map +Ġsal iva +Ġtrump et +Jim my +Ġmirac ulous +Ġcolon ization +Ġam put +ĠGN OME +ate ch +D ifferent +ĠE LE +ĠGovern ments +ĠA head +ãħĭ ãħĭ +word press +L IB +ĠIn clude +ĠDor othy +0 45 +ĠColomb ian +Ġle ased +88 4 +Ġde grading +ĠDa isy +i ations +Ġbapt ized +Ġsurn ame +co x +Ġblink ed +ãĥ ¢ +Ġpoll en +Ġder mat +Ġre gex +ĠNich olson +ĠE ater +ç ľ +rad or +Ġnarrow er +Ġhur ricanes +Ġhalluc inations +r idden +ISS ION +ĠFire fly +Ġattain ment +Ġnom inate +Ġav ocado +ĠM eredith +Ġt s +Ġreve rence +Ġe uph +Ġcr ates +ĠT EXT +Ġ4 43 +Ġ3 19 +J SON +iqu ette +Ġshort stop +ic key +Ġpro pelled +Ġap i +ĠTh ieves +77 9 +Ġovers aw +Ġcol i +ĠNic ola +Ġover cl +ik awa +ĠC yr +Ġ38 4 +78 9 +ĠAll ows +10 27 +Det roit +TR Y +set up +ĠSocial ism +Sov iet +s usp +ĠAP R +ĠShut down +Ġal uminium +zb ek +ĠL over +GGGG GGGG +Ġdemocr acies +Ġ19 08 +ĠMer rill +ĠFranco is +gd ala +Ġtraff ickers +ĠT il +ĠGo at +Ġsp ed +ĠRes erv +Ġpro d +55 2 +Ġc ac +ĠUn iv +ĠSch we +Ġsw irling +ĠWild erness +ĠEgg s +Ġsadd ened +Ġarch aic +H yd +Ġexcess ively +B RE +Ġaer ospace +ĠVo ices +Cra ig +Ġign ited +In itially +ĠMc A +Ġhand set +Ġreform ing +Ġfrust rations +ĠDead pool +ĠBel ichick +ract or +ĠRagnar ok +ĠD rupal +ĠApp roximately +19 20 +ĠHub ble +arm or +ĠSar as +ĠJon as +Ġnostalg ic +Ġfeas ibility +Sah aran +Ġorb iting +Ġ9 70 +R u +Ġsh in +ĠInvestig ators +Ġinconsist encies +ĠP AN +B G +Ġgraz ing +Ġdetect ors +ĠStart up +ĠFun ny +ĠNa omi +Consider ing +Ġh og +ut f +ce mic +Ġfort ified +ĠFun ctions +Ġcod ec +nut rition +H at +" ! +micro soft +55 8 +ĠTh in +ĠA CE +Al ias +ĠO PS +p apers +P K +ãĢ İ +Ġimpro bable +N orthern +equ al +Ġlook out +Ġty res +ĠMod ified +ĠK op +Abs olutely +Ġbuild up +sil ver +Ġaud i +Ġgro tesque +ĠSab er +ĠPres byter +ON Y +Ġglac iers +ĠSho als +ĠK ass +ĠH RC +ĠNic ol +ĠL unch +ĠF oss +âĸ Ĵ +AD RA +ĠOne Plus +o ing +ground s +Ġincident al +Ġdatas ets +68 9 +ĠClarks on +Ġassemb ling +ĠCorrect ions +Ġdrink ers +Ġqual ifiers +Ġle ash +Ġunf ounded +ĠH undred +Ġkick off +T i +Ġrecon cil +ĠGr ants +ĠCompl iance +ĠDexter ity +Ġ19 06 +w arn +D allas +Max imum +n ard +av ia +be aut +ens itivity +tr ace +Ġpione ers +ĠF ract +ãĢ ı +Ġpre cept +Ġgloss y +ĠI EEE +Ac ross +Ġ6 80 +S leep +che on +Ġsatir ical +ĠMin otaur +ĠCla ude +Ġr é +ape go +Ġcar rot +ĠSem in +ino a +Ġz o +Ind ependent +Ġdiagn oses +ĠC ue +M AR +Ġrend ition +ĠK ik +Ġpath ology +Ġselect s +Link edIn +Ġass ay +ĠD res +Ġtext ual +post ed +IT AL +ĠM aul +N eal +Ġinter connected +Ġerr atic +ĠVir us +Ġ5 30 +Ġenvironmental ists +ĠP helps +Ġeng agements +ĠIN ST +Ġeconom ical +nox ious +Ġg earing +izz y +Ġfavor ably +ĠMcG ill +T erm +Ġh anged +Ġball park +ĠRe yes +Ġbe ware +ĠP sal +ĠMass acre +q i +Ġin accessible +acly sm +Ġfr ay +ill ac +Ġbitter ly +ĠCert ification +Mich igan +Ġir respective +al ore +Em pty +Ġendorse ments +Ġund et +f g +equ ipped +Ġmerc iless +ĠC ust +Ġimm ature +Ġvou cher +ĠBlack well +Ñ ı +h awk +dis ciplinary +ile e +ĠMak oto +ĠD ude +ãĥĩ ãĤ£ +Y ears +Ġin ver +Ġsh aman +ĠY ong +ip el +ell en +ĠCath y +br ids +Ġs arc +65 1 +N ear +Ġground work +Ġam az +Ġ4 15 +ĠHunting ton +hew s +ĠB ung +Ġarbit rarily +ĠW it +ĠAl berto +Ġdis qualified +best os +46 1 +Ġp c +Ġ28 4 +ro bat +Rob in +Ġh ugs +ĠTrans ition +ĠOcc asionally +Ġ3 26 +ĠWh ilst +ĠLe y +Ġspaces hip +cs v +Ġun successfully +ĠA u +le ck +ĠWing ed +ĠGrizz lies +. � +Ġne arer +ĠSorce ress +ĠInd igo +El se +8 40 +let es +Co ach +Ġup bringing +ĠK es +Ġseparat ist +Ġrac ists +Ġch ained +Ġabst inence +lear ning +Ġrein stated +Ġsymm etry +Ġremind ers +ĠChe vy +Ġm ont +Ġexempl ary +ĠT OR +Z X +Ġqual itative +ĠSt amp +ĠSav annah +ĠRoss i +Ġp aed +Ġdispens aries +ĠWall s +ĠCh ronic +Ġcompliment ary +ĠBeir ut +Ġ+ --- +igs list +Ġcrypt ographic +mas ters +ĠCap itals +Ġmax imal +Ġent ropy +Point s +Ġcombat ants +l ip +ĠGl ob +ĠB MC +ph ase +th ank +HT TP +Ġcomm uter +Ġ\( \ +.. / +ĠReg ener +ĠDO I +ĠActiv ision +Ġsl it +os al +RE M +Ġch ants +Y u +Ke ys +Bre xit +ĠFor ced +Ari zona +Ġsquad ron +IS O +ĠMal one +Ġ3 38 +Ġcontrast ing +Ġt idal +Ġlib el +Ġimpl anted +Ġupro ar +ĠC ater +Ġpropos itions +M anchester +ĠEuro s +it amin +G il +ĠEl ven +ĠSe ek +ĠB ai +Ġredevelop ment +ĠTown s +ĠL ub +! ", +al on +K rist +Ġmeas urable +Ġimagin able +Ġapost les +Y N +7 60 +Ġster oid +Ġspecific ity +ĠL ocated +ĠBeck er +ĠE du +ĠDiet ary +uts ch +ĠMar ilyn +Ġbl ister +ĠM EP +ĠK oz +ĠC MS +y ahoo +ĠCar ney +Ġbo asting +ĠC aleb +By te +read s +ad en +Pro blem +ĠWood ward +S we +S up +ĠK GB +Set up +Ġtac it +Ġret ribution +Ġd ues +ĠM ü +. ? +ä¸ Ń +p ots +Ġcame o +ĠP AL +educ ation +A my +like ly +g ling +Ġconstitution ally +ĠHam m +ĠSpe ak +Ġwid gets +br ate +Ġcra ppy +ĠI ter +Ġanticip ating +ĠB out +P ixel +ĠY ep +ĠLaur ie +Ġh ut +Ġbullet in +ĠSal vation +Ġch ats +ear able +Honest ly +AL TH +onse qu +c ult +isco very +ovy ch +Ġse lves +ĠSat oshi +S ounds +Ġconver gence +ĠRosen berg +19 74 +Ġnas al +Ġfull est +Ġfer ocious +x us +ist e +AM S +Ġlobb ied +Ġso othing +ĠGun n +t oday +0 24 +Ġinspir ational +ĠN BN +p b +g ewater +or ah +all owed +ĠCol iseum +Ġspecial izing +Ġinsane ly +ĠT ape +del ay +Ġt arn +ĠP ound +Ġmel anch +Ġdeploy ments +il and +Ġless en +Ġfur ry +ĠUE FA +Ġblood shed +ĠMe ier +ither ing +Ġhe irs +ĠJ aw +ax ter +ĠPublic ations +Ġal ters +int ention +ĠWinc hester +d etermination +ĠLif etime +th in +Mon ster +7 80 +Ġapprox imation +Ġsuper markets +ĠSecond s +or os +h uge +Ġb ribe +ĠLIM ITED +un ed +Ġmis interpret +ĠIn jury +Ġ3 67 +Ġthreshold s +ĠCarn ival +Ġgastro intestinal +Ġguid eline +Ġde ceived +f eatures +Ġpurported ly +ĠRon nie +ĠNew t +Ġsp acious +as us +Ġsuperhero es +ĠCyn thia +le gged +k amp +ch io +Ġth umbnail +ĠShir ley +ill ation +Ġshe ds +ĠZ y +E PA +Ġdam s +Ġy awn +n ah +ĠPe ggy +ĠE rie +ĠJu ventus +ĠF ountain +r x +don ald +al bum +ĠComp rehensive +Ġc aching +ĠU z +ulner ability +ĠPrinc iple +ĠJ ian +ing ers +cast s +ĠOs iris +ch art +t ile +ĠTiff any +ĠPatt on +ĠWh ip +Ġovers ized +J e +ĠCind erella +ĠB orders +ĠDa esh +M ah +Ġdog ma +Ġcommun ists +v u +Coun cil +Ġfresh water +Ġw ounding +Ġdeb acle +Ġyoung ster +Ġthread ed +ĠB ots +ĠSav ings +ãģ Ĥ +ol ing +oh o +Ġillum ination +M RI +Ġlo osen +tr ump +ag ency +ur ion +Ġmoment arily +ĠCh un +ĠBud apest +ĠAl ley +D isk +Ġaston ished +ĠCon quer +ĠAccount ing +h aving +ĠWe in +ĠAl right +Ġrev olver +Ġdel usion +Ġrelic s +Ġad herent +qu ant +Ġhand made +or io +Ġcomb ating +c oded +Ġquad ru +re th +N ik +ĠTrib al +ĠMyster ious +Ġin hal +ĠWin ning +ĠClass ification +ch anged +Ġun ab +Ġsc orn +icip ated +w l +ond uctor +Ġrein forcing +ĠChild hood +an ova +Ġadventure r +Ġdoctor al +ĠStrateg ies +Ġengulf ed +ĠEnc ounter +Ġl ashes +Crit ical +ric ular +ĠU TF +oci ation +check ing +ĠConsult ing +Run time +per iod +ĠAs gard +Ġdist illed +ĠPas adena +ĠD ying +ĠCOUN TY +Ġgran ite +Ġsm ack +Ġparach ute +ĠS UR +Virgin ia +ĠF urious +78 7 +ĠO kin +Ġcam el +ĠM bps +19 72 +ĠCh ao +ĠC yan +j oice +ef er +ĠW rap +ĠDeb ate +S eg +Ġfore arm +ĠIgn ore +Ġtim estamp +Ġprob ing +ĠNo on +ĠGra il +f en +Ġdorm ant +ĠFirst ly +ĠE ighth +ĠH UN +ĠDes ire +or as +Girl s +ĠDes mond +z ar +am ines +O AD +exec ute +Ġbo obs +ĠAT L +_ ( +Chel sea +Ġmasturb ation +ĠCo C +Ġdestroy er +ĠCh omsky +Ġsc atter +ĠAss ets +79 6 +ĠC argo +Ġrecept ive +ĠSc ope +Ġmarket ers +Ġlaun chers +Ġax le +ĠSE A +se q +ĠM off +f inding +ĠGib bs +Georg ia +extreme ly +N J +Ġlab orers +st als +Ġmed iation +ĠH edge +at own +Ġi od +des pite +v ill +J ane +ex istence +Ġcoinc ided +ĠUt ilities +ĠChe ap +Ġlog istical +Ġcul mination +ĠNic otine +p ak +F older +Ġrod ents +st uff +Ġlaw fully +Ġreper to +io ch +j j +Dial ogue +HH HH +lic tion +Look s +Ġ29 7 +Ġtur rets +ĠAb andon +Ġinc ess +ĠTraff ord +Ġcur led +Ġprefer ring +Ġprivat ization +Ġir resist +ĠP anda +ĠSh ake +ĠMc Gr +ãĥ Ħ +und ers +Ġdiscrim inated +Ġbart ender +I LE +Atl antic +Ġprop ensity +ĠW iz +ĠG im +con ference +Ġrein forces +G h +w agon +Ġe erie +F al +Ġhug ged +rac ist +R IC +F u +Ġf iller +ĠSt ub +Ġeng raved +ĠWrest le +Ġimagin ative +ĠPe er +ĠFact ors +an us +ĠDrac ula +mon itor +Ġrou ters +ib ia +ĠBoo lean +end ale +ĠSl aughter +ĠSh ack +R FC +ĠSpiel berg +S ax +ĠPH OTO +ĠCl over +ĠR ae +Dep ending +ĠMem or +ar am +Ġpier ced +Ġcur tains +v ale +ĠInqu isition +ĠP oke +Ġforecast ing +Ġcompl ains +S ense +ĠHer mes +isc overed +Ġb ible +ĠMor ph +Ġg erm +78 5 +D ON +Ġcon gen +Ġcr ane +ĠD PR +Ġrespect fully +R oom +ĠN aw +ĠDal ai +re ason +ĠAng us +Educ ation +ĠTitan ic +Ë ľ +Ġo val +un ited +Ġthird s +Ġmoist ur +ĠC PC +M iami +Ġtent acles +ĠPol aris +ex c +ex clusive +ĠPra irie +Ġcol ossal +ĠBl end +sur prisingly +ÃŃ s +Ġindo ctr +Ġbas al +ĠMP EG +und o +Spl it +Develop ment +Ġlan tern +19 71 +Ġprov ocation +Ġang uish +ĠB ind +ĠLe ia +duc ers +ipp y +conserv ancy +Ġinitial ize +ĠTw ice +ĠSu k +Ġpred ic +Ġdi ploma +Ġsoc iop +Ing redients +Ġhamm ered +ĠIr ma +Q aida +Ġglim ps +ĠB ian +Ġst acking +Ġf end +gov track +Ġun n +dem ocratic +ig ree +Ġ5 80 +Ġ29 4 +Ġstraw berry +ID ER +Ġcher ished +ĠH ots +Ġinfer red +Ġ8 08 +ĠS ocrates +O regon +ĠR oses +ĠFO IA +Ġins ensitive +Ġ40 8 +Recomm end +ĠSh ine +Ġpain staking +UG E +ĠHell er +ĠEnter prises +I OR +ad j +N RS +L G +Ġalien ated +Ġacknowled gement +ĠA UD +ĠRen eg +Ġvou chers +Ġ9 60 +Ġm oot +ĠDim ensions +Ġc abbage +B right +g at +ĠK lu +Ġlat ent +Ġz e +ĠM eng +Ġdis perse +Ġpand emonium +H Q +Ġvirt uous +ĠLoc ations +ee per +prov ided +Ġse ams +ĠW T +iz o +PR OV +Ġtit anium +Ġrecol lection +Ġcr an +Ġ7 80 +ĠN F +49 1 +64 2 +p acking +59 8 +text ure +Sp ider +fre edom +cipl ed +ĠTAM ADRA +âĻ ¦ +aut hent +ĠW ANT +r ified +Ġr ites +Ġuter us +k iss +Ġâī ¤ +Ġsk illet +Ġdis enfranch +ĠGa al +Comp an +Ġage ing +gu ide +B alt +Ġiter ator +Ġdiscretion ary +t ips +Ġprim ates +ĠTechn ique +ĠPay ments +az el +ĠR OCK +stant ial +0 60 +Ġd mg +ĠJack ets +ĠPlay off +Ġnurs ery +ĠSy mb +art on +Ġannex ation +Color ado +Ġco ils +ĠSh oes +âĦ¢ : +ĠRo z +COM PLE +ĠEve rest +ĠTri umph +J oy +G rid +à ¼ +process or +ĠPros per +ĠSever us +ĠSelect ed +r g +ĠTay yip +St ra +Ġski ing +Ġ? ) +Ġpe g +Tes la +Ġtime frame +Ġmaster mind +ĠN B +scient ific +ĠSh it +gener ic +IN TER +N UM +Ġst roll +ĠEn ix +ĠM MR +ĠE MS +m ovie +Ĥ ª +Ġminim izing +idd ling +Ġilleg itimate +Ġprot otyp +Ġpremature ly +Ġmanual s +obb ies +ĠCass idy +D EC +des ktop +Ġaer os +Ġscreen ings +Ġdeb ilitating +ĠGr ind +nature conservancy +Ġf ades +ter mination +assets adobe +F actor +Ġdefinitive ly +P oké +ap ult +ĠLaf ayette +C orn +ĠCor al +Ġstagn ant +T ue +Ġdissatisf action +G ender +Ġkid neys +ĠG ow +ĠDef eat +ĠAsh ton +Ġcart els +Ġfore closure +ĠExpl ore +stre ngth +ot in +Ġveterin arian +Ġf umble +Ġpar ap +ĠSt rait +r ils +Ġpr ick +ĠBerm uda +ĠAm munition +skin ned +Ġab ound +ĠB raz +Ġshar per +ĠAsc ension +Ġ9 78 +Ġpreview s +Ġcommun ion +ĠX Y +Ġph ony +Ġnewcom er +Ġ3 32 +." ," +Ġredist ribution +Prot ect +ĠSo f +K al +Ġlip stick +w orst +Ġtang led +Ġretrospect ive +int eger +Ġvolunte ering +Ġ19 07 +Ġ -------------------- +ic hen +Ġunve iling +Ġsen seless +Ġfisher ies +\ - +Ġh inges +Ġcalcul us +My th +Ġund efeated +Ġoptim izations +Ġdep ress +Ġbill board +ĠY ad +ĠPy ramid +Is n +I de +Ġleg ion +ĠK ramer +ent anyl +Ġpenet rating +ĠHaw th +ĠPR ODUCT +ĠGer ard +ĠP act +ĠIn cluding +ĠEl ias +ĠEl aine +vis ual +Ġhum ming +Ġcond esc +ĠF asc +ä¸ Ĭ +Ġe galitarian +Ġdev s +ĠD ahl +O ps +D H +ĠB ounce +id ated +ald o +Ġrepublic an +Ġh amb +ĠS ett +ograph ies +CH APTER +Ġtrans sexual +Ġsky rocket +ans wer +Ġmark up +Ø ª +Ġhero ine +Comp are +ĠT av +Be ast +Ġsuccess ors +Ġna ïve +ĠBuck ley +st ress +me at +Ġdownload able +Ġindex ed +Ġsc aff +ĠL ump +ĠHom o +Stud io +In sp +Ġr acked +far ious +ĠPet ty +Ex ternal +Ġ19 09 +W ars +com mit +put ers +Ġun ob +ĠEr r +ĠE G +ĠAl am +ĠSiber ia +ĠAtmosp heric +IS TER +ĠSatan ic +trans lation +ĠL oud +tra umatic +l ique +Ġreson ate +ĠWel ch +Ġspark ing +ĠT OM +t one +Ġout l +Ġhandc uffed +ĠSer ie +8 01 +Ġland marks +ĠRee ves +Ġsoft ened +Ġdazz ling +ĠW anted +month s +Mag ikarp +Ġunt reated +ĠBed ford +M i +ĠDynam o +O re +79 5 +Ġwrong ful +Ġl ured +Ġcort isol +Ġve x +d rawn +ile t +Download ha +ĠF action +Ġlab yrinth +Ġhij acked +w aters +er ick +Ġsuper iors +ĠRow ling +ĠGu inness +Ġt d +99 2 +Ġune arthed +Ġcentr if +Ġsham eless +P od +ĠF ib +Ġ icing +Ġpredict or +Ġ29 2 +fore station +con struct +C and +@ # +Ġag itated +Ġre pr +OV A +Ġkn itting +ĠLim a +Ġf odder +68 4 +ĠPerson a +k l +7 01 +Ġbreak up +á ¸ +Ġapp alled +Ġantidepress ants +ĠSus sex +Har ris +ĠTher mal +ee ee +U pload +Ġg ulf +Ġdoor step +ĠSh ank +L U +ĠM EN +ĠP ond +s orry +Ġmis fortune +n ance +Ġb ona +M ut +Ġde graded +ĠL OG +ĠN ess +an imal +Ġa version +und own +Ġsupplement ed +ĠC ups +Ġ50 4 +Ġdep rive +ĠSpark le +Å Ĥ +ĠMed itation +auth ors +ĠSab an +ĠN aked +air d +ĠMand arin +ĠScript ures +ĠPerson nel +ĠMahar ashtra +Ġ19 03 +ĠP ai +ĠMir age +omb at +Access ory +Ġfrag mented +T ogether +Ġbelie vable +ĠGl adiator +al igned +ĠSl ug +M AT +Ġconvert ible +ĠBour bon +amer on +ĠRe hab +nt ax +Ġpowd ered +pill ar +Ġsm oker +ĠMans on +ĠB F +5 11 +ĠGood ell +ĠD AR +m ud +g art +Ġob edient +ĠTrans mission +ĠDon ation +8 80 +Ġbother ing +Material s +ãĤ ± +dest roy +Ġfore going +Ġanarch ism +ĠK ry +ice ps +Ġl ittered +ĠSch iff +Ġanecd otal +un its +Ġf ian +ĠSt im +ĠS OME +ĠInv aders +Ġbehaviour al +ĠVent ures +Ġsub lime +Ġfru ition +ĠPen alty +Ġcorros ion +¶ ħ +Ġlik ened +Ġbesie ged +ween ey +ĠCre ep +Ġlinem en +mult i +ic ably +ud der +Ġvital ity +Ġshort fall +ĠP ants +ap ist +H idden +ĠDro ps +med ical +Ġpron unciation +ĠN RL +Ġinsight ful +J V +ĠBe ard +ĠCh ou +Ġchar ms +Ġb ins +Ġamb assadors +ĠS aturdays +Ġinhib itor +ĠFr anch +6 01 +', ' +ĠCon or +art ney +ĠX peria +g rave +be es +ĠProtest ants +Ġso aking +ĠM andal +Ġph ased +Ġ6 60 +Ġsc ams +Ġbuzz ing +ĠItal ians +ĠLoren zo +ĠJ A +Ġhes itated +Ġcl iffs +ĠG OT +ingu ishable +Ġk o +Ġinter ruption +Z ip +Lear ning +Ġundersc ores +ĠBl ink +K u +57 9 +ĠAut ob +I RE +Ġwater ing +Ġpast ry +8 20 +Ġvision ary +ĠTempl ar +awa ited +Ġpist on +Ġant id +current ly +Ġp ard +Ġw aging +Ġnob ility +ĠY us +Ġinject ing +f aith +ĠP ASS +å º +Ġret ake +ĠPR OC +Ġcat hedral +b ash +Ġwrest lers +Ġpartner ing +Ġn oses +Ġ3 58 +Trans form +am en +Ġb outs +ĠId eal +ĠConstant in +Ġse p +ĠMon arch +att en +ĠPe oples +mod ified +Ġmor atorium +Ġpen chant +Ġoffensive ly +Ġprox ies +ok ane +ĠTaiwan ese +ĠP oo +ĠH OME +us ional +Ġver bs +ĠO man +vis ory +Ġpersu asion +Ġmult it +Ġsc issors +G ay +ow ay +oph ysical +l us +gn u +Ġap ocalyptic +Ġabsurd ity +Ġplay book +Ġautobi ography +I UM +Ġsne aking +ĠSim ulation +pp s +ell ery +Plan et +Ġright fully +Ġn iece +ĠN EC +ĠIP O +ĠDis closure +lean or +ous y +ST ER +Ġ28 2 +Cru z +Ch all +64 3 +ĠSurv ive +ĠF atal +ĠAm id +ap o +We apons +D EN +7 70 +ĠGreen wald +Ġlin en +al os +Ġpollut ants +ĠPCI e +k at +Ġp aw +ĠK raft +C hem +ĠTermin ator +Ġre incarn +Ġ] [ +ĠSe eds +Ġsilhou ette +ĠSt ores +Ġgro oming +ĠD irection +ĠIs abel +ĠBr idges +ðŁ ij +E ED +ĠM orsi +Ġval ves +ĠRank ed +ĠPh arma +ĠOrgan izations +Ġpenet rated +ĠRod ham +ĠProt oss +Ġove rest +Ġex asper +ĠT J +Ġ 000000 +Ġtrick le +Ġbour bon +WH O +Ġw retched +Ġmicrosc opic +Ġcheck list +Ġad orned +R oyal +Ad minist +ĠRet irement +ĠHig hest +We ather +ile ge +Ġincre ments +ĠC osponsors +Ġmas se +ĠS inn +r f +Ġh ordes +as sembly +75 4 +ĠNat asha +ĠTY PE +ĠGEN ERAL +Ġarr anging +Ġ40 7 +l ator +Ġg lean +Ġdisc redited +Ġclin icians +UN E +Ġachie ves +ĠEm erson +com plex += [ +Ġprincip ally +Ġfra il +p icked +Ġthan king +Ġre cl +ĠL AST +Ġsupp ressing +il ic +Ġantidepress ant +ĠLis bon +Ġth or +Ġsp a +Ġking doms +ĠPear ce +em o +Ġpl ung +Ġdiv est +Ġ ******************************** +b is +osp els +ad r +Sp irit +hall a +P ink +end ez +Ġresurrect ed +esc ape +ĠRosen stein +Ġge ological +Ġnecess ities +Ġcarn iv +ĠE lys +ĠBar ney +Ġ29 6 +dig y +ST ON +D OWN +Ġmil estones +Ġk er +Ġdismant ling +Ġre prim +Ġcross ings +19 45 +Ġpatri archy +Ġblasp hemy +Ġ3 59 +met ry +ĠOb esity +ĠDiff erences +bl ocking +ãĥķ ãĤ¡ +ich ita +ĠSab ha +ph alt +ĠCol o +ual a +effic ients +ĠMed ina +con sole +55 7 +ĠHann ibal +ĠHab it +ĠF ever +Ġthen ce +Ġsyn agogue +Ġessential s +Ġw ink +ĠTr ader +ID A +ĠSp oiler +ĠIceland ic +ĠHay ward +Ġpe ac +Ġmal ice +Ġflash back +Ġth w +Ġlay offs +L iquid +Ġtro oper +Ġh inge +ĠRead ers +Ph ill +ĠB auer +Cre ated +Ġaud its +ac compan +Ġunsus pecting +ier a +6666 6666 +Ġbro ch +Ġapprehend ed +ĠM alk +cer ning +ĠCod ex +O VER +M arsh +ĠD eng +ĠExp ression +Ġdisrespect ful +Ġasc ending +t ests +ĠPlaint iff +ster y +ĠAl ibaba +din and +ĠDem psey +Applic ations +mor al +Ġthrough put +Ġquar rel +Ġm ills +Ġhe mor +ĠC ASE +terror ist +st im +ifest yle +ro zen +CE PT +Ar k +u ci +lect ic +Ġirrit ating +she ets +A y +Ġrede emed +Ġhorn y +ĠTe ach +ĠS ear +dem ocracy +4 65 +ĠRest ore +Ġstand by +ĠP is +iff in +Ġsleep y +Ġextr ater +Ġcompl iments +Fram eworks +Ġinstall s +Ġb anging +sur face +found land +Ġmetaph ysical +Ġ28 3 +oul s +dev ices +Ar gs +ĠSac rifice +ĠMcC orm +es on +Cons ervative +ĠM ikhail +see ing +is ively +ĠRo oms +ĠGener ic +Ġenthusi astically +Ġgri pped +Ġcomed ic +ĠElectric ity +Ġgu errilla +Ġdec oration +ĠPerspect ive +Ġconsult ations +Ġun amb +Ġplag iar +Ġmagic ian +Ġe rection +ĠTour ism +or ied +ro xy +11 00 +T am +Ī è +Î ³ +× ª +ĠPred ators +Nit rome +Ġtelesc opes +project s +Ġun protected +Ġst ocked +ĠEnt reprene +nex pected +Ġwast ewater +V ill +Ġint imately +Ġi Cloud +ĠConst able +Ġspo of +Ġne farious +Ġfin s +Ġcens or +ĠMod es +ĠEs per +ar bon +Ġinter sections +Ġlaud ed +Ġphys i +Ġgener ously +ĠThe Nitrome +ĠTheNitrome Fan +Ġar isen +ĠÙ Ī +Ġg lands +ĠPav ilion +ĠGu pta +Ġuniform ly +Ġr amps +ri et +ĠWH EN +ĠVan essa +Ġrout ed +Ġlim p +ĠC PI +p ter +int uitive +Ġv aping +Ġexperiment ed +ĠOlymp us +ĠAm on +Ġsight ing +Ġinfiltr ate +ĠGentle man +Ġsign ings +ĠMe ow +ĠNav igation +che cks +4 33 +Ġel apsed +ĠBulg arian +esp ie +ĠS OM +d uring +Ġsp ills +anc a +ĠPly mouth +M AL +Ġdomest ically +ĠWater gate +ĠF AM +k illed +ed ited +ĠYour self +Ġsynchron ization +ĠPract ices +ST EP +Ġgen omes +ĠQ R +not ice +Ġloc ating +z in +Ġ3 29 +al cohol +Ġk itten +V o +Ġr inse +Ġgrapp le +ĠSc rew +ĠD ul +A IR +Ġle asing +ĠCaf é +Ġro ses +ĠRes pect +Ġmis lead +Ġperfect ed +Ġnud ity +Ġnon partisan +ĠCons umption +Report ing +Ġnu ances +Ġdeduct ible +ĠSh ots +Ġ3 77 +Ġæ ľ +ano oga +Ben ef +ĠB am +ĠS amp +if ix +Ġgal van +ĠMed als +rad ius +Ġno bles +Ġe aves +igr ate +K T +ĠHar bour +u ers +Ġrisk ed +re q +Ġneuro t +get table +ain a +Rom ney +Ġunder pin +Ġlo ft +ĠSub committee +ĠMong ol +b iz +Ġmanif ests +ass isted +ĠG aga +Ġsy nergy +Ġreligious ly +ĠPre f +ĠG erry +T AG +ĠCho i +4 66 +beh ind +ĠO u +Gold Magikarp +Ġhemor rh +R iver +Ġtend on +Ġinj ure +ĠF iona +Ġp ag +Ġag itation +|| || +ur an +ĠE SA +Ġest eem +Ġdod ging +Ġ4 12 +r ss +Ġce ases +ex cluding +Ġint akes +Ġinsert s +Ġemb old +ĠO ral +up uncture +4 11 +ĠUn ified +ĠDe le +Ġfurn ace +ĠCoy otes +ĠBr ach +L abor +Ġhand shake +Ġbru ises +Gr ade +éĹ ĺ +ĠGram my +ile en +St ates +ĠScandinav ian +ĠKard ash +8 66 +Ġeffort lessly +ĠDI RECT +ĠTH EN +ĠMe i +ert ation +19 68 +Ġgro in +w itch +Requ irements +98 5 +Ġroof s +Ġest ates +ĠH F +Ġha ha +Ġdense ly +ĠO CT +Ġpl astics +Ġincident ally +ĠTr acks +ĠTax es +Ġch anted +Ġforce ful +ĠBie ber +ĠK ahn +K ent +ĠC ot +lic ts +F ed +Ġhide ous +ĠVer d +ĠSynd icate +ĠIl legal +J et +ĠD AV +re asonable +c rew +Ġfundamental ist +Ġtruth ful +ĠJ ing +Ġl il +Ġdown ed +Ġen chanted +ĠPolic ies +ĠMcM aster +ĠH are +ides how +Ġpar ams +en cers +gorith m +Ġallow ances +Ġturb ulent +Ġcomplex ities +ĠK T +Ġ3 37 +ĠGen etic +F UN +D oug +t ick +Ġg igs +ument hal +Ġpatriarch al +Ġcal c +, ... +Ġc out +ĠGu an +Ġpath ological +ĠR ivals +Ġunder rated +Ġflu orescent +ĠJ iu +arna ev +ĠQu an +Ġ4 29 +Ġ ਠ+M ario +Con struct +ĠC itation +ĠR acial +ĠR SA +ĠF idel +Ġ3 95 +Person ally +C ause +à » +rad ical +in en +Ġvehement ly +ĠPap a +Ġintern ship +Ġfl akes +ĠRe ck +Luck ily +B ra +20 20 +rav ings +R N +W onder +Ser iously +Ġre usable +Ġpoll uted +ĠP eng +le igh +ind le +Ġcircuit ry +ĠMad onna +ĠB ART +Res idents +att ribute +Phil adelphia +Cl ub +Ġplan ner +Ġfr antically +Ġfaith fully +ĠTerrit ories +ĠL AT +ĠAnders en +an u +ĠP ARK +ĠS ora +i age +ĠPlay offs +ĠG CC +4 27 +Ġab norm +ĠL ever +Ġdisob edience +As ync +ĠShe a +V ert +Ġsk irts +ĠSaw yer +x p +Ġwors ening +Ġsc apego +ĠAng le +oth al +Ġtro ve +ĠSt y +ĠN guyen +mar ine +ide on +Dep ths +Bl og +ĠIll uminati +Ġtract s +Ġorgan ise +Ġo str +F s +Ġlever aging +ĠD aredevil +as ar +Ġl ang +Ġex termin +urs ions +ĠRom o +ãĤ¤ ãĥĪ +Ġcont ended +Ġencounter ing +ĠTable t +ĠAltern ate +sk ill +Ġswe ets +Ġco hesive +cap acity +Ġrep ud +Ġl izard +ro o +Ġpilgr ims +ĠR uff +ĠInstr ument +ĠLog o +uit ous +E H +Ġsales man +Ġank les +L ed +ĠPat ty +ud os +Own er +Ġdiscrep ancies +k j +M U +Ġuncond itional +Dragon Magazine +i ard +O ak +ĠConvers ation +be er +ĠOs aka +D elta +us ky +Ġsecret ion +Ġpl aza +Ġm ing +Ġde pletion +ĠM ous +ĠI TS +ĠH imal +ĠFle ming +Ġcyt ok +ĠH ick +Ġbat ters +ĠInt ellectual +6 75 +é r +IS ION +ĠQu entin +ĠCh apters +ih adi +Ġco aster +WAY S +ĠL izard +ĠY or +and ering +S kin +ha ust +ab by +Ġportray ing +Ġwield ed +d ash +Ġprop onent +Ġr ipple +Ġgrap hene +Ġfly er +Ġrec urrent +Ġdev ils +Ġwater fall +æĺ ¯ +go o +Text Color +Ġtam pering +IV ES +TR UMP +ĠAb el +ĠS AL +ĠHend ricks +ĠLu cius +b ots +Ġ40 96 +IST ORY +Gu est +ĠN X +in ant +Ben z +ĠLoad ed +ĠCle ver +t reatment +Ġta vern +Ġ3 39 +ĠT NT +ific antly +Tem perature +F el +Ġunder world +ĠJud ges +Ġ< + +Ġst ump +Ġoccup ancy +Ġab er +ĠF inder +) ", +ĠN unes +res et +in et +ect omy +Ġwell ness +ĠP eb +quart ered +and an +Ġneg atives +ĠTh iel +ĠCl ip +ĠL TD +Ġbl ight +Ġreperto ire +K yle +Ġqu er +ĠC es +Ġha pl +98 9 +ĠTh ames +isc opal +Des k +ivari ate +ĠEx cellence +found ation +Ġâ ĩ +X i +Ġmyster iously +esty les +Ġper ish +ĠEng els +ĠDE AD +09 0 +}} } +ĠUn real +Ġrest less +ID ES +orth odox +ĠInter mediate +Ġdin ners +ĠTr out +ĠSe ym +ĠHall s +og ged +Ġtraged ies +Ġdid nt +67 6 +Ġail ments +Ġobserv able +ĠV ide +ad apt +ĠD usk +Ġprofessional ism +ĠPres cott +ĠInd ies +p ox +ĠMe hran +W ide +Ġend emic +ĠPar an +B ird +Ġped als +ĠI U +ĠAdam ant +ĠH urt +Ġcorrel ates +urd en +Ġspons oring +cl imate +ĠUnivers ities +ĠK not +enn es +ĠDam ian +ĠAx el +S port +Ġbar b +ĠS no +sh own +ste en +ud ence +Ġnon violent +Ġhom ophobia +Ġbiom ass +ĠDet ail +Ġsrf N +ĠT une +accompan ied +I ENCE +Al bert +ĠMong o +z x +ĠCer berus +or bit +c ens +Ġsl ay +SH ARE +H Y +Ġb rawl +ĠPro be +Ġnonex istent +ĠClare nce +ĠBlack burn +Ġport als +ĠR ita +ĠRem ain +ĠLe vant +Ġtrick ed +ĠF erry +aver ing +ĠStraw berry +ĠAn swers +Ġhorrend ous +ĠA man +Supp lement +ĠT oad +Ġpe eled +Ġman oeuv +ĠU zbek +mond s +ĠH ector +Ġ40 2 +pe es +fix es +Ġd j +Ġres umes +Ġaccount ant +Ġadvers ity +Ġham pered +ĠL arson +Ġd oping +part s +H ur +Ġbe arded +Ġy r +ĠPlug in +å¥ ³ +Ġ/ ** +rol ley +Ġwaters hed +ĠSub mission +if lower +AS C +Ġcho ir +Ġsculpt ures +m A +incre asing +ai i +Ġsne akers +Ġconfront s +ĠEle phant +ĠEl ixir +Ġrec al +ĠT TL +w idget +ĠW ax +ĠGr ayson +Ġha irst +Ġhumili ated +ĠWAR N +app iness +ĠT TC +F uel +Ġpol io +Ġcomplex es +Ġbab e +ĠX IV +P F +). [ +P arts +Ġ4 35 +M eg +ĠY ards +ĠAL P +Ġy ells +Ġprin ces +Ġbull ies +ĠCapital ism +ex empt +FA Q +ĠSp onge +ĠAl a +Ġpleas antly +Ġbu f +Ġden ote +Ġunp ublished +Ġkne eling +asc a +Ġl apse +al ien +99 4 +Ġrefere es +ĠLaw yers +S anta +Ġpuzz ling +ĠProm etheus +ĠPh araoh +ĠDel ay +Ġfacilit ates +ĠC ES +Ġjew els +Ġbook let +ond ing +Ġpolar ization +ĠMor an +ĠSal ad +ĠS OS +ĠAdv ice +PH OTOS +IC AN +iat ures +ex press +ĠWonder land +ĠC ODE +ĠCL ASS +9 75 +Ġg rep +ĠD iesel +ĠGl ac +! ?" +Ġr m +o ine +disc rimination +ĠN urse +m allow +Ġv ortex +ĠCons ortium +Ġlarge Download +stra ight +augh lin +G rad +Ġpublic ized +ĠW aves +ĠRed d +Ġfest ivities +ĠM ane +ar ov +Ġfleet ing +ĠDr unk +ug en +C ele +Ġchromos omes +ĠD OT +-+-+ -+-+ +Ġbus iest +ĠBe aver +Sy rian +ĠK yr +k as +ĠCross Ref +19 50 +76 01 +Ġrepe aling +ĠWin ners +ĠMac ro +ĠD OD +bl ance +S ort +64 1 +Ġmet re +ĠD irk +Ġgo ggles +Ġdraw backs +Ġcomplain ant +Ġauthor izing +Ġantit rust +oper ated +Ġm ah +Ġexagger ation +Am azing +ĠSer aph +Ġha ze +w ow +Ġextingu ished +Ġcan yon +ĠB osh +Ġv ents +Ġsc rape +Cor rect +4 26 +Ġav g +Dem and +ĠâĪ ¼ +Ġmicrobi ota +"} ]," +ĠSt ev +B io +ĠPlan es +Ġsuggest ive +Ġdec ipher +ĠRefuge e +ĠKe jriwal +ĠGreen peace +Ġdecl ass +ĠSound ers +Ġth o +Ġdec rypt +Ġbr ushing +ĠJane iro +ip op +S i +8 77 +ĠGeoff rey +Ġc pu +ĠHaz el +Ġview points +Ġcris py +ĠNot ification +Ġsold er +ĠMod est +ĠHem isphere +Ġcass ette +in cludes +Ġident ifiers +ĠC ALL +in cent +T odd +ĠSwe ep +Ġ3 34 +b oss +Ġsm ir +gin x +Ġtown ship +Ġg rieving +ĠMos que +Net flix +AS ED +ĠMillenn ials +oc om +19 67 +Ġbold ly +s leep +Ġes che +arij uana +Ġsw irl +ĠPen al +Ġneglig ent +ĠStephen son +K ER +ĠZ oro +ris is +Ġlocal ization +ĠSeym our +ĠAng lic +red itation +prot ection +ĠPa ige +Ġo mit +ĠR ousse +ĠT ub +Ġinv itations +t ty +Ġm oss +ph ysical +C redits +Ġan archy +Ġchild care +Ġl ull +ĠM ek +ĠL anguages +lat est +ĠSan ford +Ġus ability +Ġdiff use +ĠD ATA +Ġsp rites +ĠVeget a +ĠProm otion +ãĥ¼ ãĤ¯ +rict ing +z ee +Tur kish +ĠTD s +pro ven +57 1 +Ġsmug glers +707 10 +Ġreform ed +ĠLo is +Ġun fl +ĠWITH OUT +ĠReturn ing +ann ie +ĠTom as +Fr anc +ĠProf it +ĠSER V +ĠR umble +ik uman +es an +Ġt esters +Ġgad get +Ġbrace let +ĠF SA +comp onent +Ġparamed ics +Ġj an +ĠRem em +ĠSk inner +Ġl ov +ĠQu ake +rom a +Ġfl ask +Pr inc +Ġover power +Ġlod ging +ĠK KK +ret te +Ġabsor bs +w rote +Ġ ," +K ings +ĠH ail +ĠFall ing +xt ap +ĠHel ena +ire ns +L arry +Ġpamph let +ĠC PR +G ro +ĠHirosh ima +Ġhol istic +". [ +Ġdet achment +Ġas pire +Ġcompl icit +ĠGreen wood +Ġresp awn +ĠSt upid +ĠFin ished +f al +b ass +Ġab hor +Ġmock ery +ĠFe ast +VID EO +Ġcon sec +ĠHung ry +P ull +ĠH ust +it ance +? ãĢį +) -- +ĠPar allel +con v +4 69 +ha ar +w ant +P aper +m ins +ĠTor o +ĠTR UMP +ĠR ai +D W +ĠW icked +ĠL ep +Ġfun ky +Ġdetrim ent +ios is +ache v +Ġde grade +im ilation +Ġret ard +Ġfrag mentation +Ġcow boy +ĠY PG +ĠH AL +Parent s +ĠS ieg +ĠStra uss +ĠRub ber +× IJ +Fr ag +Ġp t +Ġoption ally +ĠZ IP +ĠTrans cript +ĠD well +88 2 +M erc +ĠM OT +ãĥ¯ ãĥ³ +Ġhun ts +Ġexec utes +In cludes +Ġacid ic +ĠRespons ibility +ĠD umb +we i +And erson +ĠJas per +ight on +abs olutely +Ad ult +Ġpl under +Mor ning +ĠT ours +ĠD ane +Î º +ĠT EST +ĠG ina +Ġcan ine +aw an +Ġsocial ists +ĠS oda +Ġimp etus +ĠSupplement ary +oli ath +ĠKinn ikuman +mitted ly +second s +Ġorganis ers +Ġdocument aries +Vari able +GRE EN +Ġres orts +Ġbr agging +Ġ3 68 +Art ist +w k +bl ers +Un common +ĠRet rieved +Ġhect ares +Ġtox in +r ank +Ġfaith s +ĠG raphic +Ġve c +ĠL IA +Af rican +Ġard ent +end iary +L ake +ĠD OS +cient ious +ĠOk awaru +ĠAll y +ĠTim eline +D ash +ĠI c +contin ue +Ġt idy +Ġinstinct ively +ĠP ossibly +ĠOut door +ĠWould n +Ġl ich +ĠBr ay +ĠA X +Ġà ī +Ġ+ # +\ ' +Direct ory +ab iding +Ġf eral +ic ative +but t +Ġper verse +S alt +Ġwar ped +Ġnin eteen +Ġcabin ets +Ġsrf Attach +ĠSl oan +Ġpower ing +reg ation +F light +se vere +Ġst ren +Ġc og +ap ache +Ġâ Ŀ +Ġcaf eteria +p aces +ĠGrim oire +uton ium +Ġr aining +Ġcir cling +Ġlineback ers +c redit +Ġrep atri +ĠCam den +lic ense +Ġly ric +Ġdescript or +Ġval leys +Ġre q +Ġback stage +ĠPro hibition +ĠK et +Op ening +S ym +æĸ ¹ +Ġserv ings +Ġoverse en +Ġaster oids +ĠMod s +ĠSpr inger +ĠCont ainer +è » +ĠM ens +Ġmult im +Ġfire fighter +pe c +Ġchlor ine +Ð ¼ +end i +Ġsp aring +Ġpolyg amy +ĠR N +ĠP ell +Ġt igers +Ġflash y +ĠMad ame +S word +Ġpref rontal +Ġpre requisite +uc a +Ġw ifi +Ġmiscon ception +Ġharsh ly +ĠStream ing +ot om +ĠGiul iani +foot ed +Ġtub ing +ind ividual +z ek +n uclear +m ol +Ġright ful +49 3 +Ġspecial ization +Ġpassion ately +ĠVel ocity +ĠAv ailability +T enn +Ġl atch +ĠSome body +Ġhel ium +cl aw +Ġdi pping +XX X +Ġinter personal +7 10 +Ġsub ter +Ġbi ologists +ĠLight ing +Ġopt ic +Ġden im +end on +ĠC orm +Ġ3 41 +ĠC oup +Ġfear less +Ġal ot +ĠCliff ord +ĠRun time +ĠProv ision +up dated +lene ck +Ġneur on +Ġgrad ing +ĠC t +sequ ence +in ia +con cept +Ġro aring +ri val +ĠCaucas ian +Ġmon og +key es +Ġappell ate +Ġlia ison +EStream Frame +ĠPl um +! . +Ġsp herical +Ġper ished +Ġbl ot +Ġben ches +Ġ4 11 +Ġpione ered +Ġhur led +Jenn ifer +ĠYose mite +Ch air +Ġreef s +Ġelect or +ĠAnt hem +65 2 +Ġun install +Ġimp ede +Ġbl inking +Ġgot o +Dec re +A ren +Ġstabil ization +ĠDis abled +ĠYanuk ovych +Ġoutlaw ed +ĠVent ura +ten ess +Ġplant ation +Ġy acht +ĠHu awei +Ġsol vent +Ġgr acious +Ġcur iously +Ġcapac itor +Ġc x +ĠRef lex +Ph ys +ĠC f +pt in +cons ervative +Ġinv ocation +c our +F N +ĠNew ly +H our +As ian +ĠLe ading +ĠAer ospace +An ne +Ġpre natal +Ġdeterior ating +H CR +ĠNorm andy +ol ini +ĠAm bro +9 10 +Ġset backs +ĠT RE +Ġs ig +ĠSc ourge +59 7 +79 8 +Game play +Ġm sec +M X +Ġprice y +ĠL LP +aker u +Ġover arching +ĠB ale +Ġworld ly +Cl ark +Ġscen ic +Ġdisl iked +ĠCont rolled +T ickets +ĠE W +ab ies +ĠPl enty +Non etheless +Ġart isan +Trans fer +ĠF amous +Ġinf ield +ble y +Ġunres olved +ĠML A +ãĤ Ĥ +Cor rection +Ġdemocr at +ĠMore no +ro cal +il ings +Ġsail or +Ġr ife +h ung +Ġtrop es +Ġsn atched +ĠL IN +ĠB ib +ES A +ĠPre v +ĠCam el +run time +Ġob noxious +4 37 +Ġsum mers +Ġunexpl ained +ĠWal ters +cal iber +Ġg ull +ĠEnd urance +ä½ ľ +Ġ3 47 +Ir ish +Ġaer obic +Ġcr amped +ĠHon olulu +à © +us erc +ec ast +AC Y +ĠQu ery +ãĤ¹ ãĥĪ +Bet a +Ġsuscept ibility +ĠSh iv +ĠLim baugh +Ġà ĸ +ĠN XT +ĠM uss +ĠBrit ons +ES CO +EG IN +Ġ% % +Ġsec ession +ĠPat ron +ĠLu a +n aires +ĠJPM organ +us b +ocy te +Ġcouncill ors +ĠLi ang +f arm +Ġnerv ously +Ġattract iveness +ĠK ov +j ump +Pl ot +Ġst ains +ĠStat ue +ĠApost les +he ter +ĠSUP PORT +Ġoverwhel m +Y ES +Ġ29 1 +d ensity +Ġtra pping +M it +Ġf ide +ĠPam ela +atl antic +Dam n +Ġp ts +OP A +Ġserv icing +Ġoverfl owing +ul o +ĠE rit +t icket +light ing +ĠH mm +ãĥ¼ ãĥ« +im oto +Ġchuck le +4 23 +ãģ ķ +sh ape +Ġque ues +Ġanch ors +ãĤ¼ ãĤ¦ãĤ¹ +F er +Ġaw oke +Ġ6 66 +h ands +Ġdiver gence +Ġ50 5 +T ips +Ġdep ot +Ġske w +ĠDel iver +op ot +Ġdiv ul +ĠE B +uns igned +ĠUn i +X box +Ġfor ks +Ġ7 02 +å ¯ +Ġpromot ers +ĠV apor +Ġlev ied +sl ot +Ġpig ment +Ġcyl inders +C RE +Ġsn atch +Ġperpet ually +Ġl icking +ĠFe et +ĠKra ken +ĠHold en +ĠCLS ID +m r +Ġproject or +Ġden otes +Ġchap el +ĠTor rent +b ler +R oute +ĠDef endant +ĠPublisher s +ĠM ales +ĠInn ov +ĠAg ility +rit er +ty mology +st ores +L ind +Ġf olly +ĠZur ich +B le +Ġnurt ure +Ġcoast line +uch in +D omin +Ġfri vol +ĠCons olid +res ults +M J +Ġphyl ogen +Ġha uled +ĠW iley +ĠJess ie +ĠPrep are +ĠE ps +Ġtreasure r +I AS +Ġcolon ists +Ġin und +ĠWW F +ĠCon verted +6 000 +out side +ĠApp earance +ĠRel ic +ĠM ister +s aw +Ġresult ant +Ġadject ive +ĠLaure l +ĠHind i +b da +Pe ace +Ġreb irth +Ġmembr anes +Ġforward ing +Ġcoll ided +ĠCar olyn +K ansas +5 99 +ĠSolid GoldMagikarp +Be ck +Ġstress ing +ĠGo o +ĠCooper ative +Ġf s +ĠAr chie +L iter +ĠK lopp +J erry +Ġfoot wear +War ren +Ġsc ree +h are +Under standing +P ed +Ġanth ology +ĠAnn ounce +M ega +Ġflu ent +Ġbond age +ĠDisc ount +il ial +C art +ĠNight mares +Sh am +ĠB oll +uss ie +H ttp +Atl anta +Ġun recogn +ĠB id +Ġunder grad +Ġforg iving +ĠGl over +AAAA AAAA +4 45 +V G +pa io +kill ers +Ġrespons ibly +Ġmobil ize +Ġeffect ed +ĠL umin +Ġk ale +Ġinfring ing +ann ounced +Ġf itt +b atch +ĠT ackle +ĠL ime +ĠAP P +uke mia +Ġrub y +Ġex oner +ĠCas ual +0 70 +Ġpel vic +Ġautom ate +ĠK ear +ĠCoast al +Ġcre ed +Ġbored om +ĠSt un +ri ott +Ĥ İ +Ġregener ate +Ġcomed ians +ĠOP ER +Sp ons +id ium +on is +L ocated +05 7 +Ġsusp ense +ĠD ating +C ass +Ġneoc ons +ĠShin zo +Ġaw oken +ch rist +ĠMess ages +att led +ĠSpr ay +ĠSp ice +C W +Ġshield ing +ĠG aul +Am id +Ġparam ilitary +Ġmult if +ĠTan ner +il k +Ġgodd amn +g ements +Ġbe friend +m obi +Ġ3 88 +fold er +acc a +Ġins in +g ap +N ev +fif th +Ġpsychiat ry +b anks +TH IS +Ġhar b +ac qu +Ġfac ade +ĠPower Point +80 3 +Ġbl uff +Sh ares +Ġfavor ing +El izabeth +Ãį Ãį +Ġr anger +77 2 +ĠAr che +h ak +ĠGen etics +ĠF EMA +Ġev olves +Ġest e +ĠP ets +ĠM é +ĠInterest ing +ĠCanter bury +ch apter +ĠStar fleet +Sp anish +Ġdraw back +ĠNor wich +9 70 +n orth +ag anda +Ġtransform ative +ram ids +bi ology +ad ay +Ġpropag ation +ĠGam ma +ĠDen ise +ĠCalcul ator +ent imes +ĠB ett +Ġapp endix +ĠHD D +AK ING +Ġst igmat +Ġhol ster +Ġord inarily +Ch ance +ĠCont rary +Ġad hesive +Ġgather s +6 12 +re au +ony ms +ew ays +Ġindu ces +Ġinterchange able +se m +Wh it +Ġtr ance +Ġincorpor ation +ĠExt ras +Fin ancial +Ġawkward ly +ĠStur geon +ĠH Y +Norm ally +ĠEnd ing +ĠAss ist +enc rypted +Ġsub jug +Ġn os +Ġfan atic +C ub +C U +?" . +Ġirre versible +å Ĥ +03 1 +ĠH AR +sp read +ul ia += $ +Sc ope +L ots +Ġlif estyles +ol on +Ġf eds +Ġcongrat ulate +web kit +Ġindist inguishable +ĠSw ing +Ġcommand ments +qu ila +ab ella +m ethyl +ann abin +Ġo vere +Ġlob ster +ĠQU EST +ĠCONT IN +bern atorial +:::: :::: +ĠTra ve +ĠSam oa +AN I +75 2 +Ð ´ +userc ontent +ĠMod erate +y eah +ĠK itt +Ġwe e +Ġstuff ing +ĠInter vention +ĠD ign +Ġware houses +ĠF iji +Ġpel lets +Ġtake away +ĠT ABLE +ĠClass ical +col lection +Ġland fall +ĠMus cle +Ġsett les +ĠAD V +Ġ3 44 +L aura +Ġf ared +ĠPart ial +4 36 +oss ibility +ĠD aly +ĠT arant +ĠFu ji +am l +c ence +55 1 +ĠProced ures +ĠO CD +ĠU D +t in +Q UI +ach o +4 38 +Ġgl itches +Ġenchant ment +Ġcalcul ates +IR O +ĠH ua +alys es +ĠL ift +um o +Ġle apt +Ġhypothes ized +ĠGust av +it ans +VERS ION +æ ł +Rog er +Ġr and +ĠAd apter +Ġ3 31 +ĠPet ition +k ies +M ars +Ġunder cut +ze es +ĠLy ons +ĠDH CP +Miss ing +Ġretire es +Ġins idious +el i +> ) +. ãĢį +Ġfinal ists +ĠA ure +Ġacc user +Ġwas tes +ĠY s +ĠL ori +Ġconstitu encies +Ġsupp er +Ġmay hem +or ange +Ġmis placed +Ġmanager ial +Ġex ce +ĠCL I +Ġprim al +ĠL ent +Cry stal +h over +ĠN TS +end um +Ġd w +ĠAl c +n ostic +Ġpres erves +ĠTs arnaev +Ġtri pled +rel ative +Arc ade +k illing +ĠW EEK +ĠH anna +D ust +Com pleted +ģ « +Ġappro ves +ĠSur f +ĠLuther an +ven ants +Ġrobber ies +we ights +soft ware +at ana +ug al +Ġgrav y +ĠC ance +OLOG Y +ly ak +Ton ight +Ġunve il +Ġ19 04 +ĠMin ion +ent ious +st ice +pack ages +ĠG EAR +Ġg ol +ĠHutch inson +ĠProf ession +ĠG UN +ĠDiff erence +ĠTsuk uyomi +ĠLes bian +6 70 +Ġfug itive +ĠPlan etary +-------------------------------- ------------------------ +Ġacc rued +Ġch icks +Ġsto pp +Ġblock ers +C od +Ġcomment ers +ĠSomew here +ĠPhot ographer +the me +Ġmay oral +w u +Ġanten nas +Ġrev amped +ĠSubject s +it é +im ura +Ġentr ances +liter ally +Ġten ets +ĠO MG +ĠMP H +ĠDon key +ĠOff ense +Ġ" + +Sn ap +ĠAF B +Ġan imate +ĠS od +His panic +Ġinconsist ency +D b +F Y +Ex port +Ġa pe +Ġpear l +ib el +ĠPAC s +Ġ{ \ +Ġact u +ĠHS BC +camp us +Ġpay off +Ġde ities +ĠN ato +ou ple +Ġcens ored +ĠCl ojure +Ġconf ounding +en i +Ġreck on +op he +Ġspot ting +Ġsign ifies +Ġprop el +Ġfest ive +S uggest +Ġpled ging +ĠB erman +Ġrebell ious +Ġovershadow ed +Ġinfiltr ated +j obs +67 2 +Ġscal able +Ġdomin ion +ĠNew foundland +ĠMead ow +Ġpart itions +AM I +Ġsupplement ary +str ument +Ġhair y +Ġperpet uate +Ġnuts hell +ĠPot ato +ĠHob bit +Ġcur ses +Flo at +Ġquiet er +Ġfuel ing +Ġcaps ules +ĠL ust +ĠH aunted +Exec utive +Ġchild birth +G re +Ġrad iant +å İ +Ġm alls +Ġin ept +ĠWarrant y +Ġspect ator +E h +t hens +Ġculmin ating +æ © +ary a +ãĤ ® +ilit arian +ĠOR IG +ĠSp ending +pt ives +ĠS iren +ĠRec ording +ay ne +Ġv im +Ġspr ang +T ang +ĠM FT +mor ning +ĠWe ed +m peg +cess ion +ĠCh ung +7 30 +w arning +56 2 +handed ly +P oor +P olitics +: # +Ġp ian +Ġfec es +ĠDocument ation +Ġban ished +Ġ3 99 +ĠAR C +Ġhe inous +J ake +ĠAm ir +way ne +v re +os henko +Ġnotebook s +Ġfound ational +Ġmarvel ous +ixt ape +Ġwithdraw als +Ġh orde +ĠD habi +is able +ĠK D +Ġcontag ious +ĠD ip +ĠAr rows +Ġpronoun s +Ġmorph ine +ĠB US +68 2 +Ġk osher +fin ished +ĠInstr uments +Ġf used +yd en +ĠSal mon +F ab +aff ected +K EN +C ENT +Dom ain +Ġpoke mon +ĠDr inking +G rowing +ĠInvestig ative +ĠA ether +em i +Ġtabl oid +Ġrep ro +ĠNot withstanding +ĠBers erker +Ġdram as +Ġclich é +Ġb ung +ĠU RI +ĠD os +0 44 +Ġpast ors +Ġl s +Ġac rylic +aun ts +Ed ward +Ġmajor ities +B ang +Ġfield ing +ĠRepl acement +ĠAl chemy +pp ard +ĠRome o +ĠSan ct +ĠLav rov +ib ble +Inst ruct +Ġimp ractical +ĠPlay boy +ce phal +Ġsw aps +Ġk an +ĠThe o +Ġillust rating +Ġdismant led +ĠTrans gender +ĠG uth +UG H +Ġtriumph ant +Ġencomp ass +Ġbook mark +udd in +j er +Ġpred icate +ES H +Ġwhen ce +ĠAB E +Ġnon profits +Se qu +Ġdi abetic +Ġp end +Ġheart felt +sh i +Ġinter acts +ĠTele com +Ġbombard ment +dep ending +ĠLow ry +ĠAd mission +ĠBl ooming +ust ration +ene gger +B rew +Ġmol ten +ĠNer d +P IN +âĸ Ģ +ave ment +Ġtou red +Ġco efficients +ĠTray von +ans son +Ġsand y +t old +fl ows +Ġpop ulous +ĠT inder +ĠBl iss +R achel +Min imum +Ġcontest ant +ĠRed uce +ĠMor se +ĠGrass ley +ĠClick er +Ġexp r +Ġs incerity +Ġmar qu +Ġelic it +ĠPro position +ĠDemon ic +Ġtac os +G reek +Ġpost war +Ġin sofar +ĠP ork +Ġ35 2 +doctor al +walk ing +Ġmid term +ĠSam my +sight ed +ĠTR ANS +ic i +AL D +ĠUS L +ĠF ISA +ĠAm pl +ĠAlex andra +ine lli +Tr ain +Ġsign ify +ĠVers us +Ġob fusc +Ġk h +Ġagg ro +ĠRen ault +Ġ3 48 +5 18 +ox icity +0 22 +ĠTw ist +Ġgoof y +D ynamic +Ġbrief ings +m ight +8 99 +Ġderog atory +T ro +Ġfor ging +ĠKor an +ĠMar ried +ĠBuc s +Ġpal ate +ĠCon version +m able +4 13 +Ġ( _ +Ġs iph +ĠN EO +col lege +Ġmarg inally +Ġfl irt +ĠTra ps +ĠP ace +é »Ĵ +Ġgoalt ender +Ġforb ids +Ġcler ks +ĠT ant +ĠRobb ins +ĠPrint ing +Ġpremie red +Ġmagn ification +ĠT G +ĠR ouse +ĠM ock +odynam ics +Ġpre clude +ism o +ĠPul itzer +Ġaval anche +ĠK odi +rib une +ĠL ena +Elect ric +Ġref inery +Ġend owed +Ġcounsel ors +Ġd olphin +ĠM ith +Ġarm oured +hib ited +Beg in +ĠP W +O il +ĠV or +ĠShar if +ĠFraz ier +est ate +Ġj ams +Pro xy +Ġband its +ĠPresbyter ian +ĠPrem iere +t iny +ĠCru el +Test ing +Ġhom er +ĠV ERS +ĠPro l +ĠDep osit +ĠCoff in +Ġsemin ars +Ġs ql +ĠDef endants +Altern atively +ĠR ats +ç « +ethy st +' > +Ġiss uer +58 9 +Ġch aired +ĠAccess ories +man ent +Ġmar row +ĠPrim ordial +C N +Ġlimit less +ĠCarn age +Ġund rafted +q v +IN ESS +on ew +Ġco hesion +98 7 +Ġne cks +Ġfootball er +ĠG ER +Ġdetect able +ĠSupport ing +ĠCS V +oc ally +k Hz +Ġund e +Ġsh one +Ġbud ding +tra k +Stand ing +ĠStar craft +ĠKem p +Ben ch +Ġthw arted +ĠGround s +ath i +L isa +Dial og +ĠS X +V ision +Ġingen ious +Ù IJ +Ġfost ering +ĠZ a +ĠIn gram +Ġ" @ +N aturally +6 16 +0 35 +ĠF AC +H mm +55 4 +Ġacceler ator +ĠV end +Ġsun screen +Ġtuber culosis +rav iolet +ĠFunction al +ĠEr rors +ed ar +19 66 +ĠSpect re +ĠRec ipes +88 5 +ĠM ankind +L iverpool +Ġ| -- +Ġsubst itutes +ĠX T +w ired +Ġinc o +ĠAf gh +E va +ic c +S ong +K night +Ġdilig ently +ĠBroad cast +A id +Ġaf ar +ĠH MS +aton in +ĠGr ateful +Ġfire place +ĠOm ni +e uro +ĠF RE +ĠSh ib +ĠDig est +t oggle +Ġheads ets +Ġdiff usion +ĠSqu irrel +ĠF N +Ġdark ened +out her +Ġsleep s +ĠX er +gun s +Ġset ups +Ġpars ed +Ġmamm oth +ĠCur ious +g ob +ĠFitz patrick +ĠEm il +im ov +........ ..... +ĠB enny +Second ly +Ġheart y +Ġcons on +st ained +Ġgal actic +cl ave +Ġplummet ed +Ġp ests +Ġsw at +Ġrefer rals +ĠLion el +h oly +Ġunder dog +ĠSl ater +ĠProv ide +ĠAm ar +ress or +å Į +ong a +Ġtim id +Ġp iety +ĠD ek +Ġsur ging +az o +Ġ6 10 +Ġdes ks +ĠSp okane +ĠAn field +Ġwars hips +ĠCob ra +Ġar ming +clus ively +ĠBad ge +ag ascar +ĠPR ESS +ĠMcK enzie +ĠFer dinand +burn ing +Af ee +Ġtyr ann +ĠI w +ĠBo one +100 7 +ĠRe pt +Ċ Âł +Ġcar avan +ĠD ill +ĠBundes liga +Ch uck +Ġheal er +ãĥ¼ãĥ Ĩ +ĠH obby +Ġneg ate +Ġcrit iques +section al +mop olitan +Ġd x +Ġouts ourcing +ĠC ipher +t ap +Sh arp +Ġup beat +Ġhang ar +Ġcru ising +ĠNi agara +Ġ3 42 +ill us +ĠS v +Ġsubt itles +Ġsqu ared +Ġbook store +Ġrevolution aries +ĠCarl ton +ab al +Ut ah +Ġdesp ise +ĠU M +cons ider +aid o +Ġc arts +ĠT urtles +Tr aining +Ġhonor ary + ¢ +Ġtri angles +4 22 +Ġreprint ed +Ġgrace ful +ĠMong olia +Ġdisrupt ions +ĠB oh +Ġ3 49 +Ġdr ains +Ġcons ulate +Ġb ends +Ġm afia +ur on +ĠF ulton +m isc +Ġren al +Ġin action +ck ing +Ġphot ons +Ġbru ised +ĠC odes +og i +Ġn ests +ĠLove ly +ĠLib re +ĠD aryl +Ġ# ## +S ys +. ," +Ġfree zes +est ablishment +and owski +Ġcum bers +ĠSt arg +ĠBom bs +Ġleg ions +Ġhand writing +Ġgr un +ĠC ah +sequ ent +Ġm oth +ĠMS M +Ins ert +F if +Ġmot el +Ġdex ter +ĠB ild +hearted ly +Ġpro pe +ĠText ure +ĠJ unction +ynt hesis +oc ard +ĠVer a +ĠBar th +Ġμ g +Ġl ashed +Ġ35 1 +ĠZ amb +ĠSt aples +ĠCort ex +ĠCork er +Ġcontinu um +ĠWR ITE +unt a +rid or +Ġde ems +0 33 +ĠG OLD +p as +Ġrep ressive +ãĥĨ ãĤ£ +Ġbaff led +Sc ar +Ġc rave +Ġ ______ +Ġentrepreneurs hip +ĠDirector ate +Ġ' [ +Ġv ines +Ġasc ended +ĠGR OUP +ĠGood bye +Ġdo gged +ãĥ´ ãĤ¡ +Man ufact +Ġunimagin able +ri ots +ier rez +Ġrel ativity +ĠCraft ing +ra ught +ud en +c ookie +Ġassass ins +Ġdissatisf ied +ac ci +Ġcondu it +Sp read +ĠR ican +n ice +izz le +Ġsc ares +ĠWH Y +ph ans +5 35 +Ġprot racted +ĠKrist en +5 36 +ĠSc rib +ĠNe h +Ġtwent ies +Ġpredic ament +Ġhandc uffs +Ġfruit ful +ĠU L +ĠLud wig +Ġatt est +ĠBre aker +Ġbi ologically +ĠDeal er +Ġrenov ations +f w +ess en +Al ice +ĠHen ri +Ġun ilaterally +ĠS idd +h ai +ĠSt retch +S ales +Ġcumbers ome +ĠJ avier +Ġtrend y +Ġrot ting +ĠChall enges +Ġscra ps +Ġfac ets +ĠVer onica +ĠVer ge +ĠS ana +Al ien +ĠR ih +Ġrad ial +ect ar +Ġ6 30 +cl i +Mar ie +Ġwild fire +ĠCat o +h ander +Ġwait ress +Ġch ops +ĠS ECTION +Ġblunt ly +ĠCat alog +n ian +stud y +Ġpat rolling +ĠT enth +nex us +ĠN ON +op sy +Ġsc athing +s ie +Ġdeterior ated +V B +Naz is +Ġdep ictions +Ġauthent icated +ĠCon ce +k rit +Ġpromul g +ĠL ONG +U FC +ĠVis itors +ĠRec all +Ġrehab ilit +ĠSL I +Ġglac ier +ĠB ite +Ġ50 3 +Ġvom it +Ġfer mented +ĠKh alid +Ġgrad ed +ĠMag icka +ĠIch igo +power ful +ic ators +75 3 +Ġsh rew +Ġ35 6 +Ġlegal izing +Ġall otted +ĠArch demon +ith ing +igg urat +V OL +Le od +Ġo ily +Ġindu cing +Ġamy gdala +Ġadm ins +ĠAcqu isition +C AN +Ġsche matic +Ġmo an +ĠCamer oon +Ġt ink +Ġmer ry +Ġbutter flies +ĠGo ff +Ġworks pace +ĠCor ona +Ġj avascript +ĠD olphin +ĠCant or +4 64 +to e +AP S +ĠAg ing +Ġpadd ed +ĠZ heng +ĠHe ld +Ġest ranged +Ġ7 70 +. } +ĠDun ham +Ġsm okes +Ġcap itals +und ai +Sh in +ĠFound ing +Ġent itle +Ġcenter piece +D iscover +Ġthere to +al ert +ĠN ou +ĠAnaly st +l c +F H +FI ELD +ĠP OV +gr ay +Ġar cs +ĠH OT +Ġr s +Ġoblig atory +ĠArchitect s +ĠS ven +ĠF EC +0 200 +Christ mas +ĠAlban ia +rat om +58 7 +Ġhard ships +Ġaut os +ĠCharg es +Ġap es +Ġ3 76 +wal let +Ġintox ication +Ġgobl in +Ġ5 70 +++++++++ ++++++++ +ĠYel p +ĠMag netic +ĠBr iggs +R ail +Ġspawn s +ĠW iggins +Ġshowc ased +Ġres orted +ub en +Ġwh ipping +Ġim itate +Ġdigest ion +ĠUS PS +ĠG est +Ġye a +ĠT ight +ind al +ic as +` . +C AST +'' ; +ĠF et +opath ic +In valid +Ġregrett ed +Ġbro ccoli +ĠSc ores +e ve +Ġpost ings +Ġaccum ulating +Ġneed less +elf th +Ġmay ors +Ġsc rib +Ġanecd otes +Ġbot ched +ĠRib bon +ĠConstant ine +i uses +ess es +Ġdev ise +Comp ared +Ġp udding +Ġg arg +Ġev oke +79 7 +Ġdet ox +9 09 +ĠPie ces +ĠMcC artney +Ġmet ast +ĠK rypt +P OR +Ġt ending +ĠMerch ants +Pro of +ĠV arg +ĠPort able +ãĥ¼ãĥĨ ãĤ£ +B rain +25 00 +Ġfol iage +Ø ¹ +Ġment ors +ĠA ires +Ġminimal ist +Ġing ested +ĠTro jan +ĠQ ian +inv olved +0 27 +Ġer oded +RA FT +Ġbl urry +M ob +Ġbuff et +ĠFn atic +ae a +KN OWN +ĠIn it +s afety +en um +ACT ION +ĠCrus her +ĠD ates +Ġ ................ +c alling +ak ov +Ġvent ured +Ġ5 55 +au ga +H art +ĠA ero +M AC +Ġthin ly +Ġar ra +ST ATE +ild e +ĠJac qu +ĠFem ales +Ġthe orem +Ġ3 46 +Ġsmart est +ĠPU BLIC +ĠK ron +ĠB its +ĠV essel +ĠTele phone +Ġdec ap +Ġadj unct +ĠS EN +mer ga +Ġred acted +Ġpre historic +Ġexplan atory +ĠRun s +ĠUtt ar +ĠM anny +ĠAUTH OR +ĠUnle ashed +ĠBow ling +be ans +79 3 +Ġunivers es +Ġsens it +ĠK ung +re peat +ctr l +Ġp aced +Ġfull er +Cl ock +Ġrec omb +ĠF aul +ĠB unker +Ġpool ed +Ġan a +ĠM outh +LL OW +hum ane +Ġbull do +ĠMicha els +f am +Ġwreck ed +Ġport rays +ĠWh ale +ĠH es +Ġguess es +ĠBrow se +ĠL APD +Ġconsequ ential +ĠInn ocent +ĠD RAG +Ġtrans gress +ĠO aks +Ġtri via +ĠRes on +ĠA DS +-- + +ĠT oll +Ġgrasp ing +ĠTHE M +ĠT ags +ĠCon clusion +Ġpract icable +Ġho op +Ġunintention ally +Ġign ite +ĠM ov +ur ized +le hem +Ter min +Ġcolour ful +ĠLin ear +ĠEll ie +G y +Ġman power +Ġj s +Ġem oji +ĠSHAR ES +_ . +0000 7 +Ġsophistic ation +Ġunders core +Ġpract ise +Ġbl ob +op ens +Uk raine +Ke eping +Y C +J R +ult imate +Cl aim +Ġautom obiles +99 3 +ste el +Ġpart ing +ĠL ank +... ? +Ġ38 5 +Ġremem brance +Ġe ased +Ġcov ari +ĠS ind +Effect ive +Ġdisse mination +ĠMo ose +ĠCl apper +br ates +App ly +Ġinv is +Ġwors ened +âĢĶ - +Ġlegisl ator +ĠL ol +ĠRow e +Ġdealers hip +um ar +id ences +Ġinvestig ates +Ġc ascade +Ġbid der +ĠB EN +Iron ically +Ġpres iding +Ġd ing +Ġcontrad icted +Ġshut s +ĠF IX +Ġ3 66 +Dist rict +Ġsin ful +ĠChar isma +o ops +Ġtot ality +Ġrest itution +ĠOpt imus +ĠD ah +Ġcl ueless +urn ed +Ġnut rit +Ġland owners +Ġfl ushed +Ġbroad en +m ie +Ġprint ln +Ġn ig +ĠCorp us +J en +Ġprot o +ĠWik imedia +ĠPal o +C OR +Ġstory lines +Ġevangel icals +ĠDar rell +Ġrot or +ĠH W +sk illed +ery l +Ġbe gg +ĠBl umenthal +Ġwe aving +Ġdown wards +ĠJack et +ĠANG EL +Te chnology +Ġes oteric +alde hyde +Ġfur iously +Ġforeign er +We ak +CH O +ĠH ound +Exper ience +ĠPlay station +ĠM IA +ĠU ng +cl oth +ag all +Ġcal ming +iz ens +St ruct +ĠW itches +ĠCeleb ration +Ġ........ ...... +pt roller +ĠTC U +Ġb unny +ãĥ į +ut orial +Ġup scale +ĠSt a +ĠCol ossus +Ġchlor ide +ĠZ ac +ĠRe asons +ĠBrook ings +ĠWH ITE +][ / +ĠL ose +9 05 +Ġunders ide +ern els +Ġv ape +do zen +upp et +ĠST OP +mat ical +ĠStat ements +hed dar +P AC +Custom er +Ġmem os +ĠP J +end ars +ĠLim its +l augh +Ġstabil ized +ĠALE C +Y A +Up grade +al am +Ġtechn o +Ġan ew +fore seen +Ġcolleg iate +ĠPy ro +ĠD ism +Ġfront line +Ġammon ia +I U +Qu ite +John ny +ass in +G OP +ĠSt yles +ĠSovere ign +acter ial +5 49 +ĠR IP +ĠL ists +Ġ3 64 +ĠRece p +s ocket +ĠByr d +ĠCand le +An cient +Ġappell ant +en forcement +ace a +ans ki +Ġold s +88 6 +Ġsl urs +Ġem pires +Ġbuck le +Ġalien ation +ĠAber deen +Ġunic orn +Ġoverr iding +ĠL X +pp a +Ġdesp ised +ĠB ugs +ĠB ST +S outhern +5 33 +Ġhall mark +ĠPost er +Ġstem med +Ġprincip als +ĠT ECH +ĠSand wich +It aly +Ġche esy +ĠSet TextColor +ĠProt ective +ĠC ohn +J O +apt op +Re ason +Lead er +ĠUnder stand +ĠFr idays +ĠContin uous +Ġcl ipping +ĠR ye +Ġber th +tim er +ann is +re act +Ġbuff alo +ĠPar as +Ġ6 55 +Ġpres ided +ĠSun rise +Ġve ts +Ġcl oves +ĠMcC ull +Stre ngth +G AN +Ġill iter +ĠPric ing +l é +Ġresist or +Ġbr un +ĠSuff olk +Ñ ĭ +ĠL iver +Re leased +Ġwhat s +8 60 +ĠMe asures +Ġden ouncing +ĠRy zen +Ġsou ven +Ġcareg ivers +ch ini +ĠScar lett +Ġt rough +Cong ratulations +Ġtax is +ĠTrad ition +j it +Ġtable top +Ġhither to +Ġdis information +off ensive +h ra +ĠDISTR ICT +Ġcompl icate +chen ko +ĠRecon struction +Ġpalp able +Ġa usp +Ġ4 28 +Ġshowc ases +ĠPublic ation +know ledge +inn on +4 19 +Ġretri eval +and ers +Ġref ute +Ġinqu ired +g ur +Ġneg ativity +Ġcons erve +Ġafter life +Ġpres upp +ĠGill espie +Ġm t +ĠD N +T ap +Ġper pend +ĠS my +does n +Ġsp illing +Ġhyp ers +K ate +® , +ke pt +ĠP owered +Ġj a +ĠK lux +ard e +ab an +Ġ4 44 +Ġflatt ened +ĠImprove ments +urg a +ĠK und +Ġins cribed +Ġfac ult +Ġunpre pared +ĠCons umers +Ġsatisf ies +Ġpul monary +Ġinf iltration +Ġex ternally +Ġcongrat ulations +ag han +Ġair liner +Ġfl ung +Ġfly ers +G D +Ġsnipp ets +Ġrec ursive +Ġmaster ing +L ex +Ġovert ly +v g +Ġluck ily +Ġenc ro +ĠLanc et +ĠAbyss al +function al +Ġs ow +Ġsqu id +Ġnar ration +Ġn aughty +ĠHon our +ĠSpart ans +Ġsh atter +ĠTac oma +ĠCal ories +ĠR aces +Sub mit +Ġpurpose fully +w av +ĠY ok +F est +ĠG err +Met ro +Ġit iner +f amous +Ġ" { +in line +was her +Iss ue +ĠCL IENT +oz o +Vers ions +7 25 +ĠGl ock +Ġshield ed +ĠPC R +ENC Y +ĠWe ld +ĠSim pl +Ġredirect ed +ĠK ham +Ġ( > +Ġlab ou +Ġdi apers +ss l +Ġcell ar +organ isms +ore sc +ĠBer ks +did n +Sh ipping +C hest +Ġund one +Ġmillion aire +Ġc ords +ĠYoung er +appropri ately +Ġsequ els +u ve +ant icipated +Ġle wd +ĠSh irt +ĠDmit ry +V eter +Ġsl aying +ĠY ar +Ġcompl ication +I owa +ĠEric a +ĠBL M +g irlfriend +b odied +6 26 +19 63 +Ġintermedi ary +Ġcons olation +M ask +ĠSi em +ow an +Beg inning +Ġfix me +Ġculmin ated +Ġcon duc +ĠVolunte er +Ġpos itional +Ġgre ets +ĠDefin itions +Ġthink er +Ġingen uity +Ġfresh men +ĠMom ents +Ġ35 7 +ate urs +ĠFed Ex +s g +69 4 +Ġdwind ling +ĠBO X +sel age +Ġt mp +Ġst en +ĠS ut +Ġneighbourhood s +Ġclass mate +f ledged +Ġleft ists +Ġclim ates +ATH ER +ĠScy the +ul iffe +Ġs ag +Ġho pped +ĠF t +ĠE ck +ĠC K +ĠDo omsday +k ids +Ġgas ped +Ġmon iker +ĠL od +ĠC FL +t ions +r ums +fol ios +Ġm d +Ġunc anny +Ġtrans ports +ĠLab rador +Ġrail ways +Ġappl iance +ĠCTR L +æ Ģ +Pop ulation +ĠConfeder acy +Ġunb earable +Ġdors al +ĠIn form +op ted +ĠK ILL +Mar x +Ġhypoc ritical +q us +ĠN umerous +ĠGeorg ian +ĠAmbro se +ĠL och +Ġgu bernatorial +ĠX eon +ĠSupp orts +ens er +ee ly +ĠAven ger +19 65 +Ar my +Ġju xtap +Ġcho pping +ĠSpl ash +ĠS ustainable +ĠFin ch +Ġ18 61 +ict ive +at meal +ĠG ohan +Ġlights aber +ĠG PA +ug u +ĠRE PL +vari able +Ġher pes +Ġdesert s +ac iously +Ġsitu ational +week ly +ob l +Ġtext ile +ĠCorn wall +Ġcontrace ptives +ĠA ke +] - +ä¹ ĭ +: , +ĠW em +ĠB ihar +Ġ' . +Ġbe re +Ġanal ogue +ĠCook ies +Ġtake off +Whe el +Ġmaj estic +Ġcomm uting +0 23 +ĠCor pse +ass ment +min i +Ġgor illa +ĠAl as +ere e +Ġacquaint ances +ĠAd vantage +Ġspirit ually +Ġey ed +pm wiki +ĠE nder +Ġtrans lucent +Ġnight time +ĠIM AGES +5 45 +ĠK amp +ĠFre ak +Ġ ig +Port land +4 32 +ĠM ata +Ġmar ines +Ġh ors +ater asu +ĠAtt ribution +Ġ-------- - +Ġk ins +ĠBEL OW +++ + +Ġre eling +ol ed +Ġcl utter +ĠRel ative +Ġ4 27 +B US +Ġa vert +ĠChe ong +ĠA ble +ĠPry or +Develop er +Ġen cyclopedia +ĠUSA F +ĠG arry +Sp ain +Bl ocks +Ġexp osition +ĠGamer Gate +W OR +Ġstockp ile +Ġclot hed +ĠT one +ĠR ue +t umblr +Ġtreacher ous +Ġf rying +Ñ Į +ĠS ph +Ġrest raints +Ġemb odies +ĠG es +S afety +Ġnegoti ators +min ing +ĠAppalach ian +L OS +ĠJenn a +Ġpass ers +ç ĭ +sn ap +Ġshort en +creat or +Ġinn umerable +uther land +67 4 +ĠW OM +ĠAs cend +ĠArm ory +ĠTrans action +K ick +Ġsuit case +day Name +Ġwaste ful +mar riage +ĠMcC abe +ite ch +ĠO ss +Cl osure +ĠTreasure r +Ġindec ent +ĠD ull +Ġresid ences +19 59 +ĠS ettlement +Ham ilton +Ġself ies +ĠRank ing +ĠBark ley +ĠB ore +ĠW CS +ĠMar itime +ĠH uh +ĠForest ry +Ġcultiv ating +ĠBall ard +Ġg arrison +ĠSD L +9 30 +Ġnas cent +Ġirresist ible +Ġaw fully +\/ \/ +Ġequ ate +Ġanthrop ology +ĠSylv ia +Ġintest ine +Ġinnoc uous +cess ive +ag ra +ĠMet roid +G rant +8 55 +ģ ĸ +Ġ" _ +ãĥĥ ãĥī +Ġappra isal +ĠFred dy +04 6 +Ġ40 6 +Ġ18 30 +Ġd ocking +St atic +Ġp ont +ĠVolt age +ĠSt ead +ĠMort gage +ĠJon ah +Y L +CLASS IFIED +Ġas bestos +nik ov +Ġcoll agen +ĠOrb ital +P ocket +7 99 +Ġhy brids +inc hes +Ġinv oice +und y +Ġinequ alities +T rend +w ashed +B ALL +Ġluc id +ĠComment ary +Ġw itty +Br andon +Ġbru ising +Ġ6 20 +es cent +box ing +P OL +Ġ3 78 +R ect +Ġlic ences +ĠMcG ee +p ressed +D anny +Ġj ammed +ord inate +Ġle th +Ġdistingu ishes +ĠYam aha +IL S +ĠH ume +ĠC ategories +Rober ts +Ch art +Ġbeet le +ĠGra veyard +Ġ($ ) +o ÄŁ +Ġtw ilight +are lla +á ½ +Ġbooth s +ĠH HS +ĠFeld man +Ġexcav ation +Ġphilosoph ies +at ography +ĠGar age +te chnology +Ġunfor gettable +Ġver ifying +Ġsubord inates +E ls +Ġne b +G aming +EN A +ĠAchieve ment +it ters +ĠG abe +Ġd umps +for cer +Ġpo ignant +ĠM BA +ĠHe idi +ime i +Ġm ages +Ġliber ate +Ġcircum cised +ĠMer maid +ĠMat th +t ogether +ĠW ichita +Ġstore front +ĠAd in +V II +Four th +Ġexplore rs +W ER +Not able +Bro ok +m ens +F aith +-------- - +ĠJ ou +¬ ¼ +Ġpine apple +Ġam alg +el n +ark able +ĠãĤµ ãĥ¼ãĥĨãĤ£ +ĠãĤµãĥ¼ãĥĨãĤ£ ãĥ¯ãĥ³ +Ġov arian +ĠE choes +Ġhairc ut +Ġp av +Ġch illed +anas ia +Ġsty led +Ġd ab +ni per +Ġminister ial +ĠD UP +T an +Ġsul ph +ĠD eter +ĠBo hem +od an +Ġeduc ator +â ĵĺ +sp ir +Ch icken +ĠE leanor +Ġqu i +Ġheav iest +Ġgrasp ed +U RA +Ġcro oked +Jess ica +pro blem +Ġpred etermined +Ġman iac +Ġbreath s +ĠLauder dale +Ġh obbies +y z +Cr ime +Ġcharism a +d L +Ġle aping +Ġk ittens +Ang elo +ĠJ ACK +ĠSu zanne +Ġhal ting +ENT ION +Ġswall owing +ĠEarthqu ake +Ġeight eenth +ĠN IC +ĠIN F +ĠCons cious +Ġparticular s +circ le +7 40 +Ġbene volent +Ġ7 47 +Ġ4 90 +Ġr undown +ĠVal erie +ĠB UR +Ġcivil isation +ĠS chn +W B +ot ide +intern ational +Ġj ohn +Ġ19 02 +Ġpe anuts +Ġflav ored +k us +Ġro ared +Ġcut off +é £ +Ġorn ament +Ġarchitect ures +Ġ3 69 +ol or +ĠWild e +ĠC RC +ĠAdjust ed +Ġprov oking +land ish +Ġrational ity +Ġjust ifies +Ġdisp el +Ġa meric +ĠPol es +Ø © +Ġen vis +ĠD oodle +ä½ ¿ +igs aw +auld ron +Techn ical +T een +up hem +ĠX iang +Ġdetract ors +ĠZ i +ĠJournal ists +Ġconduc ive +ĠVolunte ers +Ġs d +Know ing +Ġtrans missions +ĠPL AN +ĠL IB +Ġall uded +Ġob e +Ġd ope +ĠGold stein +Ġwavelength s +ĠDest ination +nd a +ug i +Ġattent ive +ĠLe an +ral tar +Ġman g +mb uds +ak ings +b ender +Ġacc ol +Ġcraw led +N OW +Min nesota +Ġflour ished +ĠZ up +ĠSuper visor +ĠOliv ier +Ex cellent +Ġwid en +D one +Ġw ig +Ġmiscon ceptions +Cor p +W an +Ġvener able +ĠNot ably +ĠKling on +an imate +Bo ost +ĠS AY +miss ing +ibli ography +mel on +Ġpay day +Ø ³ +bo le +Ġve iled +ĠAl phabet +It alian +Ġever lasting +ĠR IS +ĠC ree +rom pt +Ġh ating +Ġgrin ning +Ġge ographically +OS H +Ġwe eping +ĠÂłĠÂłĠÂłĠÂł ĠÂłĠÂłĠÂłĠÂł +Ġimpe cc +Let ter +Ġblo ated +PL A +ĠFe in +Ġper sever +Th under +Ġa ur +ĠR L +Ġpit falls +âĸ º +Ġpredomin ant +Ġ5 25 +7 18 +AP E +7 14 +Ġfarm land +ĠQ iao +Ġv iolet +ĠBah amas +Ġinflic ting +ĠE fficiency +Ġhome brew +Ġundert ook +Ġcur ly +ĠHard ing +man ia +59 6 +Ġtem pered +Ġhar rowing +ĠP ledge +ĠFranken stein +è ª +M otion +Ġpredict ably +ĠExpl osion +oc using +er d +col o +FF ER +Ġback field +ĠV IDE +ue bl +N arr +ĠArg ument +Ġgen omic +Ġbout ique +Ġbatt ed +ĠB inary +Ġg amb +ĠRh ythm +67 3 +Ġa float +ĠOlymp ia +Y ING +Ġend if +is in +Ġwin ters +Ġsc attering +I v +D istance +Ġtr u +ĠCom fort +Ġne xus +Ġair flow +ĠByz antine +p ayers +con i +ĠB etsy +D eal +ĠN ug +ĠContin ent +red ibly +Ġoptim izing +al beit +Ġec static +ĠPro to +ç · +iv ot +âĸ Ħ +em p +rou nder +Ġcl out +ĠI ST +66 3 +ĠDoll ars +ĠD AC +Ġsubsc ribed +Ġrehears al +Ġam ps +ĠSh ang +es m +Ġspr inkle +Ġassail ant +ĠO o +ĠCoin base +T act +Ġret ina +Ġn uns +R ON +att o +Ġj ug +ĠSV G +Ġb ikini +ĠFI LE +ĠFound ers +ep ort +ĠK P +Ġrest ores +ĠTh ick +Ġash ore +Ġappro vals +R ender +M AG +G raham +ĠCort ana +ãĥ³ ãĤ¸ +ss h +or ians +ars ity +ĠInsp ired +u pper +Ġsign alling +Ġreb uke +Ġfl ares +Ġdownt ime +Stud ies +Ġstagn ation +ĠSequ ence +Ġgr unt +Ġass ures +ĠPL A +59 2 +Ġintra ven +d epend +Sus an +ĠManz iel +Man ia +Cont ract +Ġsl ams +Ġcult ured +Ġcred itor +L IST +ĠH UM +ĠChatt anooga +serv ed +Ġclo aked +ĠF TP +p owder +ĠSt ella +uct ive +Ġcheap ly +ĠMU CH +ĠGalile o +Ġsu ites +spe ech +Ġdeliber ations +ĠCh ips +« ĺ +Bal ance +ĠWyn ne +ĠAk ron +Ass et +Ġhon oured +Ġed ged +Like wise +anim ous +ĠW age +ĠEz ek +ad vertisement +ĠRT X +ĠM AD +Ġmigr ating +ĠS QU +Ġ4 75 +Ed ited +Ġshorth and +ĠBas ics +Ġcro tch +ĠEV EN +Ġv m +effic iency +Ġcal ves +ĠF rie +ĠBrill iant +Ġstri kers +Ġrepent ance +Ġarter ies +r l +B ed +h ap +Ġcrypt ography +ĠSab res +Ġ4 14 +vi ks +ih ara +aps es +T alking +Ġintertw ined +Ġdoc ks +Ġalle le +ĠArt ifact +ĠH IM +t orn +ç ķ +Ġop acity +ĠE ly +os uke +Ġn ipple +Ġhand written +ĠV K +ĠChamber lain +ĠLa os +ig raph +g row +Ġtr illions +Ġdescend ant +ĠSail or +as uring +Ġce ilings +ĠWare house +f lying +ĠGl ow +Ġn ont +Ġmiscar riage +Ġrig s +Ġmin istries +Ġelabor ated +Ġdel usional +ĠHum ane +Ġ3 79 +n ets +Ġblack out +add ers +Ġn p +ĠT ire +ro sc +Ġsub div +Ġlink age +Ġchron ological +ĠHER O +Ġres ettlement +ĠVin yl +Ġpast oral +ĠMob il +ĠBar bar +Co oldown +ĠF ritz +c riminal +re pe +Ġbell ig +ĠBre ed +Ġ4 18 +Ġsem blance +ij k +Ġcur tail +Ġclin ch +cont ained +ĠProm pt +ast on +Ġw i +Ġpursu its +5 15 +ĠGl oss +Ġfl ips +Ġcoup ons +Ġcl oning +ĠLike ly +Rem oved +ĠQu artz +r ices +ĠSpe ars +Ġp ious +Ġdep reciation +ĠD are +oun ces +am az +O nt +Ġp innacle +d ocker +0 26 +ĠW yr +ĠPro per +Ë Ī +n il +By tes +Ġseek er +t rial +Ġunf olds +ĠMar se +Ġextravag ant +ĠSurviv ors +RED ACTED +ĠSpeed way +ĠCra igslist +sub mit +ĠGener ations +Ġup holding +Ġblood stream +ĠMiss ions +ĠL awn +Ġlim bo +ene i +H uh +ĠWild cats +pre p +ĠMark us +ĠFor bidden +rit ic +IN O +Ġexhib iting +requ ent +ch uk +Ġhabit ual +ĠComp atibility +Dr ag +RIP T +uj ah +GR OUND +Ġdelinqu ent +Ġburn er +Ġcontempor aries +Ġgimm ick +load s +Ġno zzle +p odcast +ĠW ak +ĠStat en +ĠK uh +ãģ ĵ +inter rupted +Ġinv incible +ĠBurn ett +cig arette +ĠPeb ble +ĠTem porary +ĠMar ino +58 2 +Ġwast eland +ident ly +T x +Ġr ite +ĠPan asonic +ĠM iddles +ĠHort on +ae us +Ġc uring +Ġm ats +Ġadj ourn +Ġfears ome +pe z +bo ats +Ġpro pell +Ġconflic ted +ĠAng er +Ġinsurg ent +K arl +Ġco ales +Ġsouth western +Ġdis su +ĠO vert +******** **** +Ġbox ed +ĠBr une +aa a +Ġgard ening +ĠEng el +tr acks +Ġpur ified +Ġplace holder +ĠL ikes +Ġd an +G ab +Ġe ct +ĠF aw +ĠEl iot +Ġ' , +otrop ic +ĠRu in +hed on +Ġca ul +Ġa ft +ĠCad illac +gh a +ass ian +ud eb +ĠT ick +Ġadjust s +AR GET +5 37 +isc he +ant y +ĠFried rich +ĠBl izz +ĠA OL +Camp aign +Ġmamm al +ĠVe il +ĠK ev +ĠMaur it +ĠDam ien +N ation +E astern +Ġ{ : +Ġ= ================================ +Ġstereotyp ical +Ġatt ic +ĠCy borg +requ ire +Ġaward ing +ĠPap ua +bt n +b ent +B oo +Ġ( = +ĠX ander +ĠSomers et +Ġcatch y +Ġcert ify +STR UCT +Ġit al +Ġt ides +ĠBr ands +G ray +comp etitive +Ġcur ator +ĠD G +omin ium +ĠGM Os +ci ating +ĠCarm en +ow ard +Balt imore +Ġr gb +C u +Ġwip es +spe ll +IT NESS +Ġsummar izes +ĠRe vis +Ġwhistlebl owers +ĠBre ach +Ġcro chet +k os +ews ki +Ġrep et +Ġcrim son +ĠKar achi +read able +dim ension +ĠI gor +ild ed +ĠZ ed +ĠKe ane +ĠCos metic +DE P +Ġretreat ing +ĠU A +ens ical +Ġd usk +ĠDick ens +Ġaren as +ĠPass age +level s +Ġcur v +P ope +Ġch ores +ĠEl ise +ĠComp ass +b ub +Ġmamm alian +ĠSans krit +ĠAN C +ĠCr ack +Q ual +L aun +amp unk +Ġlearn ers +Ġglam orous +Ġfur the +erm ott +c and +Gener ic +Ġnarr ated +Ġdisorder ly +ĠTrans actions +ĠDet ention +ĠR oku +Ä į +Ġunder statement +ĠS aur +ĠRodrig o +ĠAS AP +S in +Ġre joice +Method s +Ġelectro de +Ġworsh ipped +Ġid i +ĠPhys icians +Ġpop up +Ġde ft +ĠRem oval +ĠBu enos +ver bs +Ġfun k +ush a +rict ion +ore a +ĠBang alore +ĠKen obi +zz i +Ġnorm ative +Ġgobl ins +Ġcaf es +ĠUN CLASSIFIED +ĠF ired +S IGN +Ġs clerosis +ĠV oter +ĠSon ny +ĠExt end +ĠEV s +Ar senal +Ġp si +Ġwid est +ĠT us +Ġlo oms +Ġjust ifying +ĠGr anger +è ¯ +Ref er +58 3 +Ġflour ishing +ab re +Ġr ave +ĠCont ra +Ġ18 98 +Add s +Ġf ul +ĠCo oke +some one += # +67 1 +Ġy ak +Ġar te +ĠMis cellaneous +ĠDet ection +ĠCl ancy +â ģ +ass ies +Ġval iant +ĠFemin ist +cor ruption +V el +P ear +Ġsucc inct +Ġquick est +k w +Ġsp itting +ĠL ibraries +åħ ī +ant z +D ad +ĠSpec ifications +rup ulous +and r +RES ULTS +Ġsnow ball +Ġpred is +ĠB axter +ĠNurs ing +ĠCh aff +s we +Ġout age +Ġnest ing +Ġnotor iety +tr igger +on ite +j on +Ġf ou +ook ed +ĠCelebr ity +re ality +Ġfat ig +Ġhug ging +Ġbother s +ĠPan zer +ĠCh andra +fig ured +Ġvol ts +ĠCloud s +Ġfee ble +ĠCur ve +ĠAs us +78 6 +abs or +ĠV ICE +ĠH ess +Ġmanufact ures +Ġgri zz +ĠPower ful +ac id +Ġsub sections +ĠKrug man +ĠAl ps +is u +Ġsequ est +ĠUlt ron +ĠT inker +ĠGo ose +Ġmism atch +Att orney +Ġmorph ology +ĠSix ers +ut tered +ĠE LECT +gr an +Rus sell +ĠG SL +Ġfort night +Ġ. ) +Ġapost le +pr one +el ist +Unt itled +ĠIm plementation +ist ors +Ġtank er +Ġpl ush +Ġattend ants +ĠT ik +ĠGreen wich +ĠY on +ĠSP L +cell s +unt led +S olution +ĠQu é +Ġvac ated +Ġupt ick +ĠMer idian +æ ĥ +ĠDr ill +9 25 +58 4 +Ġrenov ated +ĠKub rick +zy k +Ġl ousy +pp el +ohyd rate +ĠI zzy +lesi astical +CC C +ĠAj ax +Ġad apters +ĠPetra eus +Ġaffirm ation +ĠST OR +le ms +ad oes +ĠConstantin ople +Ġp onies +Ġl ighthouse +Ġadherent s +ĠBre es +omorph ic +Fight ing +Ġpl aster +ĠP VC +ĠOb st +Ġdear ly +ĠTo oth +icks on +Ġsh aming +P lex +A gg +ĠâĢ¦ " +Ġsub reddits +Ġpige on +ĠResident ial +ĠPass ing +Ġl um +ĠP ension +Ġpessim istic +Ġ4 32 +z inski +c ade +0 75 +Ġapolog ised +iy ah +Put ting +Ġgloom y +ĠLy me +=-=-=-=- =-=-=-=- +ĠT ome +ĠPsych iatric +ĠH IT +c ms +ap olog +Ġbreak er +Ġdeep en +Ġtheor ist +ĠHigh lands +Ġb aker +Ġst aples +Ġinterf ered +ĠAb ortion +jo ined +ch u +Ġform ulate +Ġvacc inations +Ġban ter +phe us +Ġoutfield er +ĠM eter +Ġ# #### +Ġ18 95 +Ġnarrow ing +ĠST ORY +f p +ĠC ST +ign ore +Ġproclaim ing +ĠR U +ĠB ALL +yn a +65 3 +Ġpos it +P RE +59 4 +ĠRegist rar +ĠPil grim +ic io +Ġpre tt +Ġlif eless +Ġ__ _ +Ne igh +ĠCh urches +orn o +Ġor cs +Ġkind red +ĠAud it +Ġmillenn ial +ĠPers ia +g ravity +ĠDis ability +ĠD ARK +W s +od on +Ġgrand daughter +ĠBro oke +ĠA DA +ER A +Ġpick ups +ĠWil kinson +ĠSh ards +ĠN K +Ġexp el +ĠKis lyak +Ġj argon +Ġpolar ized +ian e +Pub lisher +Ġreb utt +Ġapprehens ion +ĠK essler +Ġpr ism +F UL +19 64 +ĠL oll +ä ¿ +le thal +Å Ł +Ġg hetto +Ġb oulder +ĠSlow ly +ĠOsc ars +ĠInst ruction +ĠUl tr +ĠM oe +N ich +ĠP ATH +( * +ĠRE LEASE +un ing +rou se +en eg +Ġre imb +ĠDet ected +Do S +Ġster ling +Ġaggreg ation +ĠLone ly +ĠAtt end +hig her +Ġairst rike +ks on +SE LECT +Ġdef lation +ĠHer rera +C ole +rit ch +Ġadvis able +F ax +Ġwork around +Ġp id +mort em +ers en +Ġtyp o +Ġal um +78 2 +ĠJam al +script s +Ġcapt ives +ĠPres ence +ĠLie berman +angel o +Ġalcohol ism +ass i +Ġrec ite +Ġgap ing +Ġbask ets +ĠG ou +Brow ser +ne au +Ġcorrect ive +und a +sc oring +ĠX D +Ġfil ament +Ġdeep ening +ĠStain less +Int eger +Ġbu ggy +Ġten ancy +ĠMub arak +Ġt uple +ĠD roid +ĠS itting +Ġforfe it +ĠRasm ussen +ixt ies +es i +ĠKim mel +Ġmetic ulously +Ġap opt +ĠS eller +08 8 +ec ake +hem atically +T N +Ġmind less +Ġdig s +ĠAcc ord +ons ense +em ing +br ace +Ġe Book +ĠDist ribut +ĠInvest ments +w t +] ), +beh avior +56 3 +Ġbl inding +ĠPro testers +top ia +Ġreb orn +ĠKel vin +ĠDo ver +ĠD airy +ĠOut s +Ġ[ / +Ï Ģ +b p +ĠVan ity +ĠRec ap +ĠHOU SE +ĠF ACE +Ġ4 22 +69 2 +ĠAnt ioch +cook ed +Ġcoll ide +Ġa pr +Ġsle eper +ĠJar vis +Ġalternative ly +ĠLe aves +ĠM aw +Ġantiqu ity +ĠAdin ida +Ġab user +Poké mon +Ġass orted +ĠRev ision +ĠP iano +ĠG ideon +O cean +Ġsal on +Ġbust ling +ogn itive +ĠRah man +Ġwa iter +Ġpres ets +ĠO sh +ĠG HC +oper ator +Ġrept iles +Ġ4 13 +ĠG arr +ĠCh ak +Ġhas hes +Ġfail ings +Ġfolk lore +Ġab l +ĠC ena +ĠMac Arthur +ĠCOUR T +Ġperipher y +app ers +Ġreck oned +ĠInf lu +ĠC ET +Ġ3 72 +ĠDefin itive +ass ault +4 21 +Ġreservoir s +Ġd ives +ĠCo il +DA Q +Ġvivid ly +ĠR J +ĠBel lev +Ġec lectic +ĠShow down +ĠK M +ip ed +reet ings +ĠAs uka +L iberal +ĠÏ Ħ +Ġbystand ers +ĠGood win +uk ong +S it +ĠT rem +Ġcrim inally +ĠCirc us +ch rome +88 7 +Ġnan op +ĠOb i +ĠL OW +o gh +ĠAuth ors +ob yl +Ur ban +Ġt i +ĠWe ir +t rap +ag y +Ġparent heses +Ġout numbered +Ġcounter productive +ĠTob ias +ub is +P arser +ST AR +Ġsyn aptic +ĠG ears +Ġh iber +Ġdebunk ed +Ġex alted +aw atts +H OU +Ch urch +ĠPix ie +ĠU ri +ĠForm ation +ĠPred iction +C EO +Ġthro tt +ĠBrit ann +ĠMad agascar +ë ĭ +Ġbill boards +ĠRPG s +ĠBe es +complete ly +F IL +Ġdoes nt +ĠGreen berg +re ys +Ġsl ing +Ġempt ied +ĠPix ar +ĠDh arma +l uck +ingu ished +Ġend ot +Ġbab ys +05 9 +che st +r ats +Ġr idden +Ġbeet les +Ġillum inating +Ġfict itious +ĠProv incial +Ġ7 68 +Ġshe pherd +ĠR ender +Ġ18 96 +C rew +Ġmold ed +ĠXia omi +ĠSp iral +Ġdel im +Ġorgan ising +Ġho ops +ĠBe i +z hen +Ġfuck in +Ġdec ad +Ġun biased +am my +sw ing +Ġsmugg led +Ġk ios +ĠP ERSON +ĠInquis itor +Ġsnow y +Ġscrap ing +ĠBurg ess +P tr +ag ame +R W +Ġdro id +ĠL ys +ĠCass andra +Jac ob +Ġ35 4 +Ġpast ure +Ġfr anc +ĠScot ch +ĠEnd s +ĠI GF +def inition +Ġhyster ical +ĠBrown e +77 1 +Ġmobil ization +æ ķ +iqu eness +Th or +Ġspear headed +Ġembro iled +Ġconject ure +jud icial +Ch oice +Ġpaper back +P ir +Ġrec overs +ĠSur ge +ĠSh ogun +ĠPed iatrics +ãģ ł +Ġsweep s +ĠLabor atories +ĠP acks +al us +add in +Ġhead lights +g ra +Ev idence +COL OR +Ad min +Ĭ ± +Ġconco ct +s ufficient +Ġun marked +Ġrich ness +Ġdiss ertation +Ġseason ing +Ġg ib +ĠM ages +un ctions +ĠN id +che at +ĠTM Z +c itizens +ĠCatholic ism +n b +Ġdisemb ark +ĠPROG RAM +a ques +Ty ler +Or g +ĠSl ay +ĠN ero +ĠTown send +IN TON +te le +Ġmes mer +9 01 +Ġfire ball +ev idence +aff iliated +ĠFrench man +ĠAugust a +0 21 +Ġs led +Ġre used +ĠImmun ity +Ġwrest le +assemb led +Mar ia +Ġgun shots +ĠBarb ie +Ġcannabin oids +ĠTo ast +ĠK inder +IR D +Ġre juven +Ġg ore +Ġrupt ure +Ġbre aching +ĠCart oon +Ġ4 55 +ĠPale o +6 14 +Ġspe ars +ĠAm es +ab us +Mad ison +GR OUP +Ġab orted +y ah +Ġfel on +Ġcaus ation +Ġprep aid +Ġp itted +op lan +ĠShel ley +ĠRus so +ĠP agan +Ġwill fully +ĠCan aver +und rum +ĠSal ary +ĠAr paio +read er +ĠR ational +ĠOver se +ĠCa uses +Ġ* . +Ġw ob +Ke ith +ĠCons ent +man ac +77 3 +6 23 +Ġfate ful +et imes +Ġspir ited +ĠD ys +Ġhe gemony +Ġboy cot +ĠEn rique +em outh +Ġtim elines +ĠSah ara +ĠRel ax +ĠQuin cy +ĠLess ons +ĠE QU +SE A +N K +ĠCost co +Incre ase +Ġmotiv ating +ĠCh ong +am aru +ĠDiv ide +Ġped igree +ĠTasman ia +ĠPrel ude +L as +9 40 +57 4 +Ġch au +ĠSp iegel +un ic +-- > +ĠPhil ips +ĠKaf ka +Ġuphe aval +Ġsent imental +Ġsa x +ĠAk ira +ser ial +Mat rix +Ġelect ing +Ġcomment er +ĠNeb ula +ple ts +ĠNad u +ĠAd ren +Ġen shr +ĠR AND +fin ancial +ĠCly de +uther ford +Ġsign age +Ġde line +Ġphosph ate +rovers ial +f ascist +ĠV all +ĠBeth lehem +Ġfor s +Ġeng lish +S olid +N ature +Ġv a +ĠGu ests +Ġtant al +Ġauto immune +;;;;;;;; ;;;; +ĠTot ally +ĠO v +Ġdef ences +ĠCoc onut +Ġtranqu il +Ġpl oy +Ġflav ours +ĠFl ask +ãĤ¨ ãĥ« +ĠWest on +ĠVol vo +8 70 +Ġmicro phones +ver bal +R PG +Ġi ii +; } +0 28 +Ġhead lined +Ġprim ed +Ġho ard +ĠSh ad +ĠEN TER +Ġtri angular +Ġcap it +l ik +ĠAn cients +Ġl ash +Ġconv ol +Ġcolon el +en emy +G ra +Ġpub s +ut ters +Ġassign s +ĠPen et +ĠMon strous +ĠBow en +il ver +H aunted +ĠD ing +start ed +pl in +Ġcontamin ants +ĠDO E +ff en +ĠTechn ician +R y +Ġrob bers +Ġhot line +ĠGuard iola +ĠKau fman +row er +ĠDres den +ĠAl pine +E lf +Ġf mt +ĠS ard +urs es +g pu +Un ix +Ġunequiv ocally +ĠCitizens hip +qu ad +m ire +ĠS weeney +B attery +6 15 +Ġpanc akes +Ġo ats +M aps +ĠCont rast +mbuds man +ĠE PS +Ġsub committee +Ġsour cing +Ġs izing +ĠBuff er +ĠMand atory +Ġmoder ates +ĠPattern s +ĠCh ocobo +ĠZ an +ĠSTAT ES +ĠJud ging +ĠIn her +* : +Ġb il +ĠY en +Ġexh ilar +oll ower +z ers +Ġsn ug +max imum +Ġdesp icable +ĠP ACK +ĠAn nex +Ġsarcast ic +Ġlate x +Ġt amp +ĠS ao +b ah +ĠRe verend +ĠChin atown +ĠA UT +d ocumented +ĠGA BA +ĠCan aan +ĠÙ ħ +Ġgovern s +pre v +E sc +ĠEst imates +OS P +Ġendeav our +ĠCl osing +omet ime +every one +Ġwor sen +Ġsc anners +Ġdev iations +ĠRobot ics +ĠCom pton +Ġsorce rer +Ġend ogenous +Ġem ulation +ĠPier cing +ĠA ph +ĠS ocket +Ġb ould +ĠO U +ĠBorder lands +Ġ18 63 +G ordon +ĠW TO +Ġrestrict s +Ġmosa ic +Ġmel odies +ç Ħ +T ar +Ġdis son +ĠProv ides +Ġ ...... +b ek +F IX +Ġbro om +ans hip +Do ctors +Ġner ds +ĠReg ions +na issance +Ġmet e +Ġcre pt +pl ings +Ġgirlfriend s +kn it +ig ent +ow e +Ġus hered +ĠB az +M obil +4 34 +ĠPres ents +orig in +Ġins omnia +ĠA ux +4 39 +ĠCh ili +irs ch +G AME +Ġgest ation +alg ia +rom ising +$ , +c row +ĠIn spection +at omic +Rel ations +J OHN +rom an +ĠClock work +ĠBak r +m one +M ET +Ġthirst y +Ġb c +Ġfacult ies +R um +Ġnu ance +ĠD arius +ple ting +fter s +etch up +Reg istration +ĠK E +R ah +Ġpref erential +ĠL ash +ĠH H +Val id +ĠN AV +Ġstar ve +ĠG ong +z ynski +ĠAct ress +Ġw ik +Ġun accompanied +lv l +Br ide +AD S +ĠCommand o +ĠVaugh n +Wal let +Ġho pping +ĠV ie +Ġcave ats +Ġal as +if led +ab use +66 1 +Ġib n +Ġg ul +Ġrob bing +t il +IL A +Ġmit igating +Ġapt ly +Ġty rant +Ġmid day +ĠGil more +ĠDe cker +Ġ§ § +part ial +Ex actly +Ġphen otype +Ġ[+ ] +ĠP lex +ĠI ps +vers ions +Ġe book +Ġch ic +g ross +":" "},{" +ĠSur prisingly +M organ +Ġresid ues +ĠConf ederation +in feld +Ġl yr +mod erate +Ġperpend icular +V K +Ġsynchron ized +Ġrefres hed +Ġad ore +ĠTor ment +ol ina +Ġ26 00 +Item Tracker +Ġp ies +ĠF AT +ĠR HP +0 48 +ĠRES P +ĠB J +all ows +P and +Ġunw elcome +ĠV oc +ĠBast ard +ĠO W +ĠL AR +ĠHeal er +Environment al +ĠKen yan +ĠTr ance +ĠP ats +Ġali ases +ĠGar field +Ġcampaign er +Ġadvance ments +ĠOkin awa +ĠC oh +ows ky +Ġstar ved +Ġsize able +Ġ: -) +Ġm RNA +Ġsusp ensions +ist ar +Scot land +Pr in +-------------------------------- ---------------- +Ġ50 2 +Ġteasp oons +Ġ10 50 +Ġcoerc ive +ĠMason ic +edd ed +ĠPass enger +Ġl att +Ġbr aces +ĠSt eal +ĠNY T +ĠK ats +ĠCel est +ae z +T u +ĠCoul ter +ðŁ ĺ +Fl ickr +ĠWil mington +ith s +++ ; +Ġv ending +Ġneg ro +ĠPh i +ĠYellow stone +Call back +Ġsh ampoo +ĠSh ades +w at +Ġsuper human +Ġridic uled +Ġhol iest +om bo +Ġintern s +Ġh one +ĠPar agu +UR I +Ġd angling +ãĤ » +so v +ict ional +av ailability +Ġrev ocation +Ġd ow +in ic +ĠTHE IR +Ġis o +Ġout ings +ĠLeth al +Ġ) )) +Ġinacc ur +Ġout landish +Ġan us +let ico +id on +l ol +Ġun regulated +Ġsuccumb ed +Ġc uff +ĠWast eland +let al +Ġsub str +Ġcoff ers +Ġautom akers +ov i +ĠX ue +ĠDayton a +Ġjar ring +Ġf umes +Ġdisband ed +z ik +itt on +Ġstriking ly +Ġsp ores +Ad apter +.) : +ĠLynd on +ival ry +Ġor ally +Ġtumult uous +Ġdisple asure +Ġcon es +or rect +Ġappe ase +Ġder by +ĠTrip oli +ĠAl ess +Ġp oked +ĠGu ilty +v P +En ough +Ġorig inals +6 99 +Ġrabb i +Ġproverb ial +Ġpostp one +el ope +ĠMist y +Ġstaff ed +ĠUn employment +redit ary +Ġdilig ent +re comm +me asures +as in +8 25 +Ġpond s +Ġmm ol +ĠS AR +ĠC ARE +Ġ3 71 +Ġclen ched +ĠCors air +Ġcaric ature +z n +att ach +ĠSch ro +spe ak +p ainted +ĠS uc +ĠE NT +Ġcell ul +ĠP aid +di agn +WH ERE +Ġtext ed +B arn +Ġret racted +ĠRe ferred +S av +Ġup keep +Ġwork places +ĠTok ens +Ġampl ify +cl inical +Ġmult ic +mber g +Ġconvol uted +Reg ion +5 65 +ĠTop ic +Ġsn ail +Ġsal ine +Ġins urrection +ĠPet r +f orts +B AT +ĠNav ajo +Ġrud imentary +ĠLak sh +OND ON +Me asure +Ġtransform er +ĠGodd ard +Ġcoinc ides +ir in +R ex +ĠB ok +qu it +Ġshotgun s +Ġprolet arian +Ġsc orp +ĠAd a +5 14 +Ġsl ander +record ed +Ġemb ell +ris ome +Ġapolog izing +ĠMul cair +ĠGib raltar +Cl a +Ġall ot +ĠAtt ention +Ġ4 33 +le ave +Ġwh ine +ĠIss a +ĠFa ust +ĠBar ron +hen y +Ġvictim ized +J ews +Ġnurt uring +ett el +W inged +ĠSub tle +Ġflavor ful +ĠRep s +eng ed +call back +Ġdirection al +Ġcl asp +ĠDirect ions +plan et +icult ure +Hel per +ic ion +ac ia +Ġç ¥ŀ +Ġsur ges +Ġcan oe +ĠPrem iership +be en +Ġdef ied +ĠTro oper +Ġtrip od +Ġgas p +ĠE uph +ĠAd s +vern ight +high ly +R ole +Ġent angled +ĠZe it +6 18 +ĠRust y +Ġhaven s +ĠVaugh an +HA EL +ĠSER VICE +/ , +Ġstr icken +Ġdel usions +Ġb is +ĠH af +Ġgrat ification +Ġent icing +UN CH +Ad ams +ĠOL ED +ĠBeet le +Ġ18 99 +ĠSO FTWARE +ateg or +V L +ĠTot em +ĠG ators +AT URES +Ġimped ance +Reg istered +ĠC ary +ĠAer ial +on ne +en ium +Ġd red +ĠBe g +Ġconcurrent ly +Ġsuper power +ĠX an +j ew +imes ter +ĠDick inson +âĶ ģ +F la +Ġp ree +ĠRoll ins +© ¶æ +Ġden omination +ĠL ana +5 16 +Ġinc iting +sc ribed +j uries +ĠWond ers +app roximately +Ġsusp ending +Ġmountain ous +ĠL augh +oid al +N s +Det ect +) = +ĠL uthor +ĠSchwarz enegger +ĠMull er +ĠDev i +ec ycle +J ar +6 13 +ĠL ongh +B ah +ĠSP ORTS +n w +Ġref inement +Ġwater ways +Ġd iner +Bl ade +68 3 +F ac +Ġinitial s +Ġro g +Ġparan ormal +B UT +Ġ[ ( +ĠSw anson +ĠM esh +âĸ ¬ +Impro ve +ĠRad iation +ĠEst her +ĠE sk +ĠA ly +ik y +Ġir rad +ĠBuck ingham +Ġref ill +Ġ. _ +Re pe +CON CLUS +Ġdifferent iated +Ġchi rop +ĠAt kins +Pat tern +Ġexc ise +Ġcab al +N SA +ĠST A +ĠS IL +ĠPar aly +Ġr ye +ĠHow ell +ĠCount down +ness es +alys ed +Ġres ize +ãĤ ½ +Ġbudget ary +ĠStr as +w ang +Ġap iece +Ġprecinct s +Ġpe ach +Ġsky line +Ġ35 3 +pop ular +App earances +ĠMechan ics +ĠDev Online +S ullivan +Z en +Ġp u +op olis +5 44 +Ġde form +Ġcounter act +ĠL ange +Ġ4 17 +Con sole +77 4 +Ġnodd ing +Ġpopul ism +Ġhe p +Ġcoun selling +compl iance +U FF +Ġunden iably +Ġrail ing +ĠHor owitz +ĠSim one +ĠBung ie +Ġa k +ĠTal ks +x ff +fl ake +Cr ash +Ġsweat y +Ġban quet +ĠOFF IC +Ġinvent ive +Ġastron omer +ĠStam ford +ĠSc are +ĠGRE EN +olic ited +Ġr usher +Ġcent rist +ight ing +Ġsub class +Ġdis av +Ġdef und +ĠN anto +oci ate +m ast +Ġpac if +Ġm end +e ers +imm igration +ESS ION +Ġnumber ing +Ġlaugh able +ĠEnd ed +v iation +em ark +P itt +Ġmetic ulous +ĠL F +Ġcongrat ulated +ĠBir ch +Ġsway ed +Ġsemif inals +Ġhum ankind +m atter +ĠEqu ip +opa usal +S aid +ĠLay out +Ġvo icing +Ġth ug +Ġporn ographic +I PS +Ġmo aning +Ġgriev ance +Ġconf essions +esc al +TEXT URE +Aut hent +os aurus +P urchase +Ġreleg ation +al ter +ĠÂł Âł +Ġr iddled +Ġo gre +ĠLow ell +Occ up +E at +ĠHy der +ĠAdvis er +Com merce +H unt +ĠOr th +ĠComp etitive +ĠCL A +CD C +Ġsal ads +F le +Ġindustrial ized +` , +ĠO WN +Ġbec k +ĠPart icularly +oub t +Ġm M +ĠHuss ain +ĠChen nai +Ġ9 20 +Ġappoint ing +ĠCull en +,,,, ,,,, +Ġp ores +ver ified +Ġbi ochemical +em ate +Ġcoward ly +ĠHels inki +ĠEthiop ian +S OURCE +ER C +est ro +Ġbi otech +ĠS our +Ġbrew er +Bloom berg +Ġintens ify +Gl ass +an co +ĠF DR +gre SQL +ĠF ires +©¶æ ¥µ +ec o +100 1 +ĠHom eless +Ġinstant aneous +ĠH aste +ig el +D iamond +Ġp aving +Ġland fill +Ġd ads +h oun +: ] +Ġinc endiary +ĠLiving ston +ĠHil bert +ĠChe cks +st yles +in ators +ĠCl ive +ph rine +Ġchimpan zees +Ġp all +ĠJ M +ĠAad haar +ð Ŀ +Ġachie vable +dis abled +P ET +OOOO OOOO +M ot +Ġint angible +Ġbal let +ĠWe bs +ĠEst imated +Effect s +Ġb ailed +Josh ua +Ġturb ulence +Ġoccup ant +ĠDay light +Ġ36 1 +me et +Ġstat ically +Ġon look +Ġk i +il legal +Ġvel vet +Ġdehyd ration +Ġacqu ies +ĠRe z +ak ura +ĠU pton +at ro +Ġincomp rehensible +Ġback door +ĠRh ino +7 27 +Ġmath s +) + +Ġhe resy +Ġd f +ĠRoc he +ĠL ydia +Ġpanc reat +re ply +arre ll +Ġsolicit ation +Ġcirc adian +BI P +Ġfor ay +Ġcrypt ic +iz u +ime o +ĠTom ato +ĠH oms +ex amination +Ġqu arry +ĠVal iant +ĠJer icho +ĠIN CLUD +Ġ18 40 +5 19 +Ġres ists +Ġsnap shots +ĠSp ur +ĠAnt iqu +Log in +Ġbest selling +Ġant ic +ĠS utherland +ãĤ¢ ãĥ« +Ġ~ / +ĠP arm +è ĥ +P ages +int ensity +Ġimm obil +Ġ18 65 +zz o +Ġn ifty +Ġf entanyl +ĠPres ervation +op hen +Ġd arts +ĠD inosaur +po inters +ĠR ite +s uggest +aware ness +ĠSher idan +Ġst ances +Ġsor cery +Ġper jury +ĠNik ola +ie ver +Ġf iance +ĠJordan ian +ĠBall oon +Ġn ab +Ġk b +Ġhuman ities +ĠTan aka +hill ary +Ġconsult ancy +ĠZ ub +Ġrem ission +Ġconf id +CH Q +ĠF ug +Ġimpro vis +Y ep +/ _ +Ġunwilling ness +Ġport folios +05 5 +ĠInstruct or +aim an +Ġclaim ants +M bps +ĠBy e +re ceived +T weet +Ġind emn +ri z +am ara +N at +Ġeval uates +ĠL ur +ep ad +FO X +ĠTh ro +Ġrust y +Ġbed rock +ĠOp rah +J B +Ġmanip ulative +Ġwill ful +Ġrel apse +Ġext ant +The me +S ensor +ĠSt ability +go vern +Ġpo ppy +Ġkn ack +Ġins ulated +ĠT ile +ĠExt rem +Ġunt old +Ġconver ge +Ġref uel +ig roup +Ġdistort ions +Ġrav aged +Ġmechan ically +ĠRe illy +ĠN ose +ĠIncarn ation +ĠBeck y +abb ling +Ġt aco +Ġr ake +Ġmelanch oly +Ġillust rious +ĠDart mouth +Gu ide +ĠR azer +ĠBen z +Ult imate +ĠSur prise +Ġpage ant +off er +Who ever +Ġw iser +Ġchem ist +ĠHE LL +ĠBul k +Ġpl utonium +ĠCO VER +Ö ¼ +f ailed +Ġtire lessly +Ġinf ertility +ĠTr ident +ĠShow time +ĠC iv +V ice +requ ires +itt ance +Ġun controlled +interest ing +56 1 +Ġinnov ate +ateg ic +L ie +ĠS elling +U l +Ġsav ior +ĠT osh +Ġsw ast +P ASS +Ġr ink +Ġcard io +ĠI ro +ud i +Ġv antage +Ġv ans +ĠNi ño ++ = +Ġpropag ate +< ? +Ġmethod ological +204 39 +Ġtrig lycer +Ġing rained +ĠAn notations +arr anted +6 17 +ĠS odium +ĠA AC +techn ical +mult ipl +Ġ3 73 +å ĭ +Ġdec isively +Ġboost ers +Ġdessert s +ĠGren ade +Ġtest ifying +ĠSc ully +ID s +Ġlock down +ĠSc her +ĠR é +ĠWhit man +ĠRams ay +rem ote +Ġh ikers +ĠHy undai +Ġcons cientious +Ġcler ics +ĠSiber ian +ut i +is bury +Ġrel ayed +Ġqu artz +ĠC BI +seek ers +ull a +Ġweld ing +ĠSh al +ble acher +T ai +ĠSam son +Ġt umble +ĠInvest or +Ġsub contract +ĠShin ra +ow icz +j andro +d ad +Ġtermin ating +ĠNe ural +ä» £ +Ġleak age +ĠMid lands +ĠCaucas us +í ķ +c it +ll an +iv ably +ĠAlb ion +Ġ4 57 +Ġregist rations +Ġcomr ade +Ġclip board +0 47 +Ġdiscour aging +ĠO ops +Ad apt +Ġem path +n v +ĠPR OT +ĠDon n +ĠP ax +ĠB ayer +t is +Squ are +Ġfoot prints +part icip +ĠChile an +B rend +ind ucing +M agn +Ġclub house +ĠMagn um +Ġenc amp +ĠEth nic +uch a +ere y +Ġw atered +ĠCal ais +Ġcomplex ion +Ġsect s +Ġren ters +Ġbr as +oÄŁ an +Time out +Man agement +Ġinf ographic +P okemon +Cl ar +Ġloc ality +Ġfl ora +as el +P ont +Ġpop ulate +ĠO ng +Ġsubs istence +Ġa uctions +ĠMcA uliffe +ĠL OOK +br inger +Ġtit an +Ġmanif old +ĠâĹ ı +Ġcalibr ated +Ġcal iphate +ĠSH E +ĠCommission ers +ce ivable +j c +W inner +5 24 +Ġcond one +Other wise +Ġp iling +Ġem body +ĠCrime an +ut ics +ĠEx hibition +Ġ4 26 +e ering +Ġv ying +ĠH UGE +* =- +Ġprin cipled +à ¦ +Ġquir ks +ĠEdit ors +put ing +G ES +ĠF TA +ठ¾ +add on +ĠH AM +ĠFrie za +W oman +. $ +Ġc rib +ĠHer od +Ġtim ers +ĠSp aces +ĠMac intosh +at aka +Ġgl ide +Ġsmell ing +ĠB AL +Ġun su +Ġcond os +Ġbicy cl +ĠRev ival +55 3 +Ġjugg ling +H ug +ĠKardash ian +ĠBalk ans +mult iple +Ġnutrit ious +oc ry +19 00 +Ġinteg rates +Ġad joining +ĠF older +roll ment +ven ient +Ġu ber +y i +Ġwh iff +ĠJu ven +ĠB orough +net te +Ġb ilingual +ĠSp arks +ph thal +man ufact +Ġt outing +ĠPH I +Ke efe +Rew ard +Ġinf all +ĠTem per +typ ically +ĠNik ol +Ġregular s +Ġpseud onym +Ġexhib itions +Ġbl aster +Ġ40 9 +w arming +Ġrever ber +Ġrecip rocal +Ġ6 70 +ip ient +b ett +ĠBe gins +Ġit ching +ĠPh ar +Ass uming +Ġem itting +ĠML G +Ġbirth place +Ġt aunt +ĠL uffy +ĠAm it +Ġcir cled +ĠN ost +enn ett +Ġde forestation +ĠHist orically +ĠEvery day +Ġovert ake +79 2 +Ġn un +ĠLuc ia +Ġaccompan ies +ĠSe eking +ĠTr ash +an ism +R ogue +Ġnorth western +ĠSupplement al +ĠNY U +ĠF RI +ĠSat isf +x es +5 17 +Ġreass ured +Ġspor adic +Ġ7 01 +Ġmed ial +Ġcannabin oid +Ġbarbar ic +Ġep is +ĠExplos ive +ĠD ough +Ġuns olved +Support ed +Ġacknowled gment +sp awn +Ġkit chens +Ġ- = +talk ing +ic ist +ĠPeg asus +ĠPS U +Ġphot on +ĠAuthent ication +R G +@# & +76 2 +ĠCl air +Ġdi aper +Ġbr ist +ĠProsecut ors +ĠJ em +6 28 +ĠEvery where +ĠJean ne +equ ality +ãĥ© ãĥ³ +object s +ĠPel icans +Ġ39 2 +Ġbl u +b ys +ĠA go +Ġinstruction al +Ġdiscrim inating +ĠTR AN +ĠCorn el +ag os +Ġty re +Ġas piration +ĠBrid gewater +": - +! ". +ĠEn s +ĠCoc o +P ie +Ġdet ach +ĠC ouch +Ġphys ique +ĠOccup ations +osc opic +en ough +B uzz +App earance +Y P +Ġrac er +Ġcompl icity +r pm +T oy +Ġinterrupt s +ĠCat alyst +Ġut ilitarian +imp act +Ġsp aghetti +Ġp orous +Ġeste emed +Ġinc iner +ĠI OC +7 48 +Ġesp resso +ĠSm ile +abil ia +6 35 +Ġmathematic ian +Ġ4 24 +ĠK L +ĠH IP +Ġover heard +ĠT ud +ĠT ec +Ġqu izz +Ġfl attering +Ġcon n +âĢ İ +Ġatt aches +ĠR OS +ĠAC S +Ġt cp +ĠSh ame +sk ip +res pected +ĠTrin idad +gr ain +Ġfooth old +ĠUnch arted +ĠJul io +z l +av ored +ĠAn xiety +er rors +ĠCent auri +its ch +D addy +Ġclutch ing +ĠIm plement +ĠGut ierrez +Ġ7 60 +Ġtele portation +end ra +Ġrevers ible +st ros +Ad venture +08 3 +Ġliber ating +Ġas phalt +ĠSp end +AR DS +im sy +PR ES +ĠEmer ging +Ġwild fires +Ġtechn ologically +Ġem its +ĠART ICLE +Ġirregular ities +Ġcher ish +çī Ī +Ġst ink +ĠR ost +Econom ic +Ġcough ing +ĠMcC ann +pro perties +ilant ro +Ġreneg oti +Trans lation +Ġin quest +ĠGra pe +oot ers +gu i +ĠSwords man +ace ae +h itting +Ġr c +Ġexert ed +ĠS AP +it ent +Ġperil ous +Ġobsc urity +Ġassass inate +Ġab original +Ġresc uing +ĠSh attered +lock ing +all ion +Ch anging +ĠHar rington +ĠB ord +ĠAfgh ans +Jam ie +aret z +ĠAugust us +Ġ38 6 +8 30 +Ġj og +ok ingly +Tr igger +ĠH OR +Stat istics +Ġviewers hip +Ġadd itives +h ur +Ġmaxim izing +ĠR ove +ĠLou ie +ĠBuck et +ĠCHR IST +ou sel +Ġstre aks +ir ted +Ġt ert +Ġcolonial ism +Ġbur ying +y k +Cond ition +ĠDPR K +By Id +75 1 +âĹ ¼ +Ġwor risome +Ġvoc ational +sl ice +Ġsa ils +ĠCorrection al +95 4 +Ġt ul +K id +l uster +Ġfam ilial +ĠSp it +ĠEp iscopal +Specific ally +ĠVol cano +run s +q s +Ġve tted +Ġcram med +t rop +here r +Thank fully +Ġper cussion +Ġor anges +Ġround up +Ġ4 99 +x ious +Char acters +ĠZion ism +ĠR ao +ÃĽ ÃĽ +W F +Ġunintention al +ONE Y +Gr ab +Com mercial +Ġglut amate +ĠMcK enna +ru ciating +ning ton +ih u +Ch an +ĠSw ap +Ġleaf lets +Ġfunction ally +er ous +F arm +Ġcal oric +ĠLiter ally +con cert +Ġshe nan +Ġrep aid +ey es +Ġbas hing +ĠG orge +Ġcollabor ations +Ġun account +itch ie +Ġteam work +pp elin +Ġpip ing +Ġmin ced +Ġd iam +ri eg +Ġmasc ara +Ġsuck er +ĠMo ons +App s +ĠPe ck +Ġper v +ĠFl oat +o ley +ĠN ish +im ize +Ġarom atic +u in +end ish +! / +ĠB icycle +ĠAS IC +ile ged +ĠQuad ro +ios yn +Ġlock out +ĠW ink +SP EC +Attempt s +Ġseed ed +red o +ias is +Ġsn ag +ãĥķ ãĤ© +ãĤ ¶ +Ġground ing +Ġrelie ver +Ġfrivol ous +ĠG ifts +ĠF aces +Es pecially +Ġmicrobi ome +im ag +ĠSch l +ĠP les +ĠBle ach +ĠIr win +ĠE aton +ĠDisc iple +Ġmultipl ication +Ġcoer ced +Ġ4 19 +st h +E vil +B omb +Ġex orc +Ġstag gered +L ESS +Ġinert ia +ĠED IT +Ġgo b +Tr aditional +Ġclass y +Lear y +ĠP AGE +yr s +Ġtrans porter +Ġmat ured +Ġhij ab +Ġbi ome +Where as +Ġex termination +ĠT ues +ĠT akeru +ĠAud rey +er ial +ĠAd en +aff les +Ġnarciss istic +ĠB aird +UT F +I re +ĠCon nie +Ch amp +Ġwhis pering +ĠH att +D K +Ġdis infect +Ġdeduct ed +Ġpart ake +Ġdown grade +ĠEs ports +ĠContin uing +Ġdemocr atically +icro bial +itt a +Ġlim estone +Ġexempt ed +ĠFren zy +H erm +7 28 +Ġfled gling +Met a +765 61 +69 3 +% : +w ake +5 26 +ĠDis cipline +Ġvirgin ity +ĠLeg ions +ĠFrank ie +int ent +Ġrest rooms +ĠRou ter +da q +Ġobjection able +âĨ ij +w ark +ĠRah ul +g ain +activ ation +abs olute +ĠAccess ed +Ġ24 00 +ogg les +Ġsecond ly +ĠDEF ENSE +Ġpost age +wra pper +sh arp +7 29 +Ġcommun icates +Ġadd on +ĠMil itia +H ong +Ġsl umped +ĠJP EG +ĠI car +ad ish +68 1 +Ġmaj esty +ĠWolf gang +ĠEl astic +u per +Ġv iz +Ġunconscious ly +ĠST D +ĠS ass +Ġflower ing +ĠHel ic +ĠDra per +ĠAm ateur +Ġman ure +Ġdis ingen +ĠLe i +br ing +9 49 +Ġinhib ited +Ġhead quartered +Ġen igmatic +�� � +Ġred ress +R H +Ġratt led +Ġd iction +l io +ĠT BA +ĠSN AP +C alling +Ġfasc ists +ĠD ove +iew icz +0 36 +Ġco asts +ĠR ect +Ġ) ] +L ot +6 29 +ĠS EM +ĠPeters en +ĠExpl ain +ĠBo ards +ĠBe zos +ĠJ ournals +Ġ20 24 +p arser +Ġmist rust +Ġgr ate +ĠL ocked +bo a +S aint +g aming +Ġvow el +in ately +bl ow +All ah +Ġun matched +Ġb ordering +ĠExp end +n r +Or acle +rou ch +Ġcont iguous +ac us +Ġdist raught +58 1 +Ġanat omical +O X +ap ixel +8 33 +ĠPL US +Ġres usc +Ġab iding +57 3 +Ġvac ancies +Em ily +Ġhyp othal +ĠWer ner +ĠWe e +ĠDJ s +5 13 +Ġwitch craft +Ġac upuncture +ent ary +benef it +Product s +ĠP SP +ĠMP G +ĠJ inn +ĠJ arrett +Ġ4 45 +ĠIm aging +ĠP yth +Fin ish +Ġte x +Ġjuven iles +Ġhero ism +Ġdoubt less +ĠA ki +ĠT end +ĠPatri arch +Ġbit ters +ĠTele communications +it atively +ag na +Ġr g +ĠS OLD +Ġcomp ulsion +ĠN asa +ĠKath ryn +Ġmillion aires +Ġintrins ically +Ġbolst ered +time out +fl o +Ġtut or +p our +Stat ement +Ġ{ * +ĠRud olph +ĠKimber ly +rog ens +adi q +] + +Ġindign ation +Ġfract uring +ĠRe leases +ĠGr ain +pro tein +L ago +Ġvac ations +Ġboot ed +ĠTH REE +ĠH G +oresc ence +Ġt f +Ġso ar +iosyn cr +Ġgl ances +ĠSp oon +ĠJ ury +ĠCow boy +Ġcreat ively +Hig her +Ġsolic itor +Ġhaw k +ac io +89 6 +Ġsuperf lu +Ġbombs hell +ct ure +Ġbroker age +Ġraid ing +Ġf rench +Ġang led +Trans action +ĠGen ocide +u pe +ĠHait ian +57 2 +! : +Ġunwitting ly +iter ator +sc roll +Ġtall ied +Ġbi omedical +ĠC ARD +Ġe uphem +Ġbrain storm +a quin +K o +Mic helle +ĠR unes +ĠBall istic +ud ers +Ġmod esty +ĠiP ads +ĠEzek iel +Y E +Ġstars hip +Ġpower fully +Ġper l +ĠSh ade +ĠQu art +ĠE EG +Ġfisher man +OS ED +ĠTyp ical +df x +Ġmes hes +Ġet ched +worth iness +Ġtopp led +Ġ3 96 +or ius +We iss +Ġmy sql +ĠVal halla +Ù Ĵ +le asing +Ġrec omp +rap nel +S el +04 3 +Ġder ailed +ĠGu ides +IR T +Ġde human +ĠBritt any +" )) +Ġex claim +Ġb alk +Ġ8 40 +CLA IM +int el +L AB +Ġpe gged +Ġast roph +sm oking +Ġrig ging +Ġfix ation +Ġcat apult +ins ide +ĠC ascade +ĠBolshe vik +G aza +Dep th +Ġloud spe +Ġalmond s +me yer +l eness +j en +f resh +Ġunbeat en +ĠSqu id +ĠPres umably +Tim er +B W +Ġro sters +Ġell ipt +ĠHar riet +dat abase +ĠMut ual +ĠComm odore +uk ed +kn ife +ĠCOMM UN +h ya +Ġmel ts +arch ives +Ġrat ification +Ġmultip lying +Ġinter oper +Ġasc ert +w ings +ver ting +ĠScorp ion +ay e +ĠPorts mouth +ĠM TA +n it +iaz ep +Ġqu arantine +Ġslides how +Ġcent imeters +Ġsyn opsis +Ġsp ate +th irst +Ġnom inating +ĠMel vin +Pre view +Ġthro b +Ġgener ational +ĠRad ius +rest ling +put able +aw ar +N ECT +Ġunlaw fully +ĠRevel ations +Wik ipedia +sur v +Ġeye ing +ij n +ĠF W +Ġbr unt +Ġinter stellar +Ġcl itor +ĠCroat ian +ĠCh ic +ev a +ĠDis app +ĠA kin +iner ies +d ust +Interest ed +Ġgen esis +ĠE ucl +ö n +p icking +Ġmut ated +Ġdisappro ve +ĠHD L +Ġ6 25 +Ì ¶ +c ancer +Ġsqu ats +Ġle vers +Disc uss += ] +D ex +ĠVIDE OS +A UD +Ġtrans act +ĠKin ect +ĠK uala +ĠC yp +7 47 +Ġsh attering +Ġarsen ic +ĠInt ake +ĠAngel o +ĠQu it +ĠK he +Ġ18 93 +M aker +0 29 +ĠPain ting +Dis able +9 16 +Ġanal ges +Ġtact ile +Ġprop hes +Ġd iced +ĠTravel s +ĠHe ader +ĠClub s +Ass istant +Ġinc rim +Ġd ips +Ġcruc ifix +ĠShan ahan +ĠInter pret +Ġ40 90 +al ogy +abb a +Ġsimul ac +hus band +S IM +Ġrecy cle +uc er +ed ged +Ġre naissance +ĠBomb ay +Cath olic +ĠL INE +ĠCl othing +re ports +Ġpl aus +Ġd ag +ĠM ace +Z I +Ġintr uder +ĠVeter inary +g ru +Ġsne aky +ĠS ie +ĠC innamon +P OSE +Ġcou rier +ĠC NS +Ġemanc ipation +s it +Ġplay through +ĠFac ilities +v irt +ĠG auntlet +Thom pson +Ġunbeliev ably +Param eters +Ġst itching +ign e +ĠTH ESE +Priv acy +Ġshenan igans +Ġvit ri +ĠVal id +59 1 +Ń · +ĠProt otype +ink a +SC P +ĠT id +è Ī +old ed +Ġindividual ity +Ġbark ing +Ġm ars +ĠW D +Ġ8 20 +Ġt ir +Ġsl apping +Ġdisgr untled +ĠAng ola +ri us +ĠTorn ado +ĠTh urs +Ġcapt cha +Ġang st +ĠP og +ĠAssass ins +ĠAd idas +Ġjoy ful +Ġwh ining +Emer gency +Ġphosph orus +Ġatt rition +oph on +ĠTimber wolves +ĠJ ah +ĠBr inging +ĠW ad +ĠEn sure +oh l +ĠX ie +omm el +c mp +Ġz ipper +Ġrel at +ĠCor ridor +m ilo +T ING +Av g +Ġcro pped +] } +Ġr aged +ĠLump ur +ĠGuer rero +our ke +N ut +Ġoff sets +og lu +dr m +Ġmort als +lat able +Ġdismiss ive +ä¸ ī +Ġthro ats +Ġchips et +ĠSpot light +Catal og +art ist +G b +Ġch illy +Ġst oked +Ġ3 74 +W ard +L atin +Ġf iasco +Ġble ach +Ġb rav +Enh anced +Ġin oc +ĠFior ina +_ > +Ġle ukemia +Ġel uc +Ġannoun cer +ĠLith uan +ĠArm ageddon +å ĩ +Len in +ĠR uk +Ġpe pp +ĠRom antic +ĠP IT +ĠInter stellar +ĠAt kinson +R aid +J s +Go al +C ourse +Ġvan ishing +es ley +ĠR ounds +Els a +59 3 +Ġredund ancy +ĠST AND +Ġprop hetic +Ġhabit able +ry u +Ġfaint ly +M ODE +Ġfl anked +IR C +Aw esome +Ġsp urious +ĠZ ah +ĠMS G +Ġsh ading +Ġmotiv ational +ĠSant ana +ĠS PR +Ġexc ruciating +om ial +ĠM iko +ĠLe opard +A byss +Ġ[ | +d irty +Ġbath s +Ġdem oral +and re +P B +Ġun ification +Ġsac rament +Ġ[ & +Ġpric eless +Ġgel atin +Ġeman ating +ĠAll aah +98 6 +Ġout burst +Ġer as +ĠX VI +ĠSP I +O tt +ĠLaz arus +PL IED +F lying +blog s +W isconsin +R aven +Ġreb ate +Ġcreep s +ĠSp an +ĠPain ter +ĠKir a +ĠAm os +ĠCor vette +Cons umer +ĠRec over +ck i +Ġpes ky +ĠIn vention +Compan ies +Ġchalleng ers +ad emic +ĠUkrain ians +ĠNeuro log +ĠFors aken +Ġent rants +Ġemb attled +Ġdef unct +ĠGlac ier +Ġpo isons +ĠH orses +m akes +ĠD irt +Ġ4 23 +hh h +ĠTrans formation +QUI RE +................ .. +Ġtrave ller +ĠSe xy +ĠK ern +ip olar +Ġransom ware +oooooooo oooooooo +E c +rub y +Prof essional +ĠOut break +arg ument +G rey +ĠFif a +ĠCH O +ĠFOR M +ĠAm trak +- [ +Ġcr adle +Ġantioxid ants +ãģ®å ® +7 36 +ĠNAS L +ĠContribut ions +Ind iana +ĠST EP +C SS +Ġsal ient +Ġall ocations +yr ights +Ġm ashed +ĠCut ter +Sex ual +Ġp ounded +Ġfan base +Ġc asc +ĠTrans parency +Ġanaly tic +ĠSummon er +× ŀ +ĠAD C +det ail +Ġvan quished +Ġcr abs +ar ie +Dest roy +ĠS ack +Ġtrans istor +Al abama +ĠK oen +ĠFisher ies +c one +Ġannex ed +ĠM GM +es a +Ġf aked +ĠCong ratulations +Ġhind ered +Ġcorrection al +ĠI TV +lee ve +Ġin appropriately +lic ks +Ġtresp ass +Ġp aws +Ġnegoti ator +ĠChrist ensen +lim its +ĠDian ne +Ġeleg ance +ĠContract s +an ke +Ob j +Ġvigil ance +Ġcast les +ĠN AD +ĠHol o +Ġemph atically +ĠTit us +ĠServ ing +ĠRich ie +ĠP igs +5 68 +Ġanim osity +ĠAtt ributes +ĠU riel +M Q +my ra +ĠApplic ant +Ġpsychiat rists +ĠV ij +ĠAb by +ag ree +P ush +Ġk Wh +hib a +Ġinc ite +ĠWe asley +ĠTax i +minist ic +hy per +ĠF arn +Ġ6 01 +ĠNation wide +F ake +95 2 +Ġma ize +Ġinteract ed +Ġtransition ed +Ġparas itic +Ġharm onic +Ġdec aying +Ġbas eless +ns ics +Ġtrans pired +Ġabund antly +ĠFore nsic +Ġtread mill +ĠJ av +ab and +Ġssh d +Ġfront man +ĠJak arta +oll er +dro ps +ĠSERV ICES +rompt u +oph ical +h ospital +bled on +6 45 +Ġmid range +ĠEV ENT +cul ated +raw led +Ġper ched +Ġover board +ĠPe el +ĠP wr +ĠCar th +ĠCOM PLE +co e +sh all +Ġdeter rence +M ETHOD +ĠAbs ent +M EN +Ġs ill +ĠLE VEL +Y ork +Ġsin ners +ĠOP EC +ĠN ur +ĠDesign s +se lection +Ġunw orthy +CH A +Ġstreng thens +88 3 +ed ly +Ġslic ing +Ġmal nutrition +Ġfilm making +ĠPol k +ur ated +Ġ4 21 +bre akers +!' " +Ġwet lands +ĠDisc rimination +Ġallow able +Ġste ered +ĠSic ily +S AM +Ġmust ache +Ġm ids +Ġcl ipped +Ġcirc ulate +Ġbr ittle +ĠBuild ings +ra ised +ĠRound up +Ġwealth ier +Ġoverw rite +Ġover powered +ĠGerr ard +s ites +PD ATED +Ġacute ly +ĠGam ble +Ġp im +ĠK us +Typ ically +De ploy +ĠMoroc can +p otion +com be +Ġvigil ante +Ġ36 3 +St ew +ĠB agg +Ġres ided +ĠSp o +Ġrem nant +Ġempt iness +br ainer +Ġout patient +pri ority +Ġle ptin +ĠPay ton +ĠGle aming +ĠS hed +ĠPol o +ĠMormon ism +rest ricted +arl ane +w x +Ġcreat ine +ĠAn on +ĠST UD +ĠJ UL +ĠT ee +5 28 +08 9 +Ġhat ched +Dis patch +ĠCompos ite +Ġ45 1 +p uff +ĠX COM +ĠOr n +ĠTH ANK +END ED +ĠAshe ville +Ġà ľ +Ġman go +ĠS lightly +world ly +ĠW ander +ĠExp and +ĠCh r +M ist +Ġorthodox y +ĠUN ESCO +reg ate +Else where +k ie +ir led +Ġtopp le +Ġadopt ive +ĠLeg s +d ress +ĠS agan +b are +ĠGl ou +Cr unch +Ġhelp ers +Ġchron ically +ĠH uma +1 0000 +Ġaccommod ating +äº Ķ +Ġwrink les +Ġdod ged +four th +Ġpre con +Ġcompress or +ĠK are +Ġev ict +ĠWar wick +im ar +Ġmodern ization +Ġband wagon +Ġref uted +Ġnet ted +ĠNa ples +ĠGen ie +per ors +Ġfield ed +Ġde re +ĠPar ables +le es +Ġtr out +asp ers +Ġn ihil +Ġhapp iest +Ġflo ppy +ĠLo ft +ĠHe ard +Ġun ison +Ġl ug +ĠRed mond +class ic +Supp orters +SH IP +G MT +Ġfue lled +ç IJ +Ġd d +ĠEmin em +Ġ18 97 +NY SE +Ġsecret aries +ĠF IA +ĠCanaver al +F avorite +Ġp omp +Ġdetain ee +ers hip +aim on +i our +ĠA pex +Ġplant ations +am ia +ac ion +R ust +Ġtow ed +ĠTru ly +5 77 +Ġshel tered +r ider +W o +Ġl air +ĠInt elligent +impro ve +m atically +Ġet iquette +ad ra +all o +ĠJun o +any thing +ĠStru ggle +ĠPred ict +ĠGr imes +ĠAMER ICA +ct x +ĠSit uation +W OOD +Ġsol uble +me ier +Ġintoler able +ang ering +Ġun interrupted +Ġtool tip +Ġinterrog ated +Ġgun ned +ĠSne ak +æŃ ¦ +Ġt ether +Ġcr umble +L ens +Ġclust ered +ĠSy l +ĠHas an +Ġdystop ian +w ana +Ġjoy stick +ĠTh ib +amm u +Tom orrow +5 46 +Ġoverc ame +Ġminim ized +cept or +Run ner +ENG TH +ĠBrend a +ĠAchieve ments +Ġtor ches +Ġrapp ort +ĠInvestig ator +ĠHand ling +rel ation +g rey +8 15 +Ġk cal +ĠComm ands +d q +Ġcur ls +Ġbe arer +Ġcyn icism +it ri +ĠUse ful +B ee +D CS +Ġab ras +P ract +BIL ITIES +7 12 +Ġdebug ger +Ġdebt or +ĠL ia +ĠK ers +Ġexacerb ate +ĠSt acy +ĠB land +ĠSc enes +Ġbranch ing +âĸĪâĸĪâĸĪâĸĪ âĸĪâĸĪâĸĪâĸĪ +ape ake +Ġs alsa +Ġmish and +ĠKon ami +ĠN ib +Ġanecd ote +Ġagree able +Ï ī +ĠNath aniel +ĠHe isman +ĠB eware +Ġ18 86 +spect ive +69 1 +5 22 +Ġinhib its +Ġhas hing +Ġ18 89 +å° Ĩ +v ich +P ure +Ġsolid ly +Ġaspir in +im aru +Ġstreet car +ĠU CS +ĠJ udd +Ġflash backs +p ins +Ġ14 40 +ĠUN HCR +ĠSym ptoms +T IT +5 38 +F ra +% ); +Ġo oz +Ġcur few +Ġcal med +Ġparticip ates +Te X +Ġnons ensical +Ġfull back +ĠDe L +mon key +h ari +Ġmetabol ites +Ġloot ed +ĠAL WAYS +ĠB CC +L t +oc het +B one +Ġveto ed +Ġg cc +ĠCL ICK +Ġ18 88 +s af +Ġstiff ness +Ġlow ly +ĠGe h +vers on +ors et +Ġun foreseen +Ġan esthesia +ĠOpt ical +Ġrecon structed +ĠT up +sh ows +NEW S +ĠNewsp aper +ĠA SA +ter a +N umbers +Ġinexpl icable +× ij +Ġhard ness +unt arily +ĠA cer +grad ient +ARD IS +Ġwood land +Ġmetaph ors +ĠWem bley +ĠPa vel +phil is +Ġre writing +Ġpercept ual +Ġ10 70 +worm s +ĠDown s +Ġunsur prisingly +Ġtag ging +fl ame +Ġlit res +Ġboun ces +ĠB abe +sh ut +Ġoverd oses +ĠShe ila +ĠCh au +ĠBl ess +Capt ure +ĠSign ificant +ĠSc ion +Ġ38 9 +ĠMc H +ĠTitan ium +ĠMe al +amed a +ag ents +agg ressive +B illy +76 3 +ĠS aying +DER R +it one +Coll ins +B ound +Ġbol ted +ĠDM CA +95 3 +Ġun iqueness +Ġep igen +un ci +ant am +Ġreck oning +ch airs +OG R +ĠSen egal +Ġ18 62 +re levant +Ġ ¯ +Ġpharm acies +ĠG eral +v ier +Y an +OR PG +Ġrab id +b ending +ĠUN ITED +Ġ4 65 +As sembly +Ġwe ep +Ġbe hest +ĠMother s +ĠJ ace +h id +Ġwh irlwind +ĠUN IVERS +Ġut opian +Ġkidn ap +Ph ilipp +K in +89 3 +Ġlivest ream +ĠM ISS +Ġsub versive +ĠTechn iques +ĠJUST ICE +ĠB ASE +Ġ38 7 +Ġassail ants +ĠHard core +Ġsprink led +ĠP se +é ļ +print ed +ĠH au +OR GE +ĠT OUR +Ġl aced +Ġit ch +G iving +Ġport ed +78 1 +//////////////// //////////////// +bre eding +Ġlog ger +ĠH OL +inn ie +First ly +Ġembry onic +Ġdeleg ated +p ai +O IL +Ġcentr ally +ĠR x +ĠSc outing +D utch +Ġhe reditary +ĠCru iser +s at +5 29 +ĠMar riott +other mal +Ġprohib itions +E arn +ĠSt ab +ĠColleg es +ĠBel ief +st retched +ĠL H +ĠEntity Item +C IA +Ġun rem +Ġlaure ate +Ġdenomin ations +sum mary +h ler +S pect +ĠK laus +ĠBe ans +Ġins ur +ĠPA X +Ġfield er +ĠV et +ĠSp arrow +z ie +ĠS Q +ĠMond ays +ĠOff line +ĠLer ner +ĠExt ensions +Ire land +Ġpatron age +Ġcontrast ed +ĠMan ia +h irt +Mos cow +Ġcondem ns +ĠAn ge +Ġcomp osing +ĠPe pe +ĠP addock +Ġheter ogeneity +Ġide ologically +Ġf ishes +Ġcur sing +ĠR utherford +ĠFlo ating +ĠAm elia +Te a +Syn opsis +Ġstun ts +Ġbe ad +Ġstock ing +ĠM ILL +ob ook +mass ive +\ < +Ġh ump +ĠPref erences +Engine Debug +ge ist +ĠNiet o +ome ver +ish y +eval uate +col onial +Altern ative +ĠGo Pro +ĠV ortex +ĠNET WORK +ans ky +Sec ure +ĠTh rust +Sn ake +Ġparcel s +Ġsam urai +Ġactress es +N ap +M F +ifer ation +Be er +5 23 +ĠI ly +oint ment +P ing +Ġstri ped +ĠMell on +oss ession +Ġneut ron +end ium +Ġa ph +ĠFlav oring +Ġ38 3 +Ġrespons iveness +ĠJ indal +ĠHitch cock +Den ver +ĠDRAG ON +sm anship +ĠDu pl +Ġs ly +Ġweb cam +ĠTw ain +ĠDar ling +ili ate +cons umer +D IT +Ġnames ake +Ġun orthodox +Ġfun er +ĠPL oS +ĠCONTR OL +ozy g +ogl obin +F ACE +ER G +ĠD ia +ĠF iesta +ce le +0 34 +Ġencl ave +âĸ¬ âĸ¬ +on ement +al ist +M and +Ġhome grown +ĠF ancy +Ġconcept ions +ĠCont ains +ure en +Ġreiter ate +Ġme ager +Ġinstall ments +Sp awn +6 27 +Ġphot oc +ĠCab rera +ĠRos enthal +ĠLans ing +is ner +Ġinvest s +ĠUFO s +EX P +Hard ware +Ġtr agically +Ġconced es +ie ft +ch am +bor gh +ĠSch r +ĠMel anie +ĠH oy +Ġvisit ation +Ġid iosyncr +Ġfract ions +Ġfore skin +ob os +Ġpo aching +ĠVI EW +Ġstimul ates +ĠG ork +can on +M IC +ĠNem esis +ĠInd ra +ĠDM V +Ġ5 29 +Ġinspect ing +Ġgrand ma +ĠW hedon +ĠSh ant +ĠP urg +ik an +ĠT eg +ĠCL R +z ac +Vict oria +ĠVer ify +ion ics +Ġpart ying +ĠM ou +col our +Ġtestim onies +l ations +Ġpress uring +hi ro +ac ers +Ġf id +ang ler +ĠCS I +Ġhere after +Ġdiss idents +report ing +iph any +che v +Ġsol itude +Ġl obe +Ġind is +Ġcred ential +re cent +ad ult +ĠNir vana +ĠFranch ise +L ayer +H yp +ĠBerks hire +Ġwill s +t if +Ġtot em +ĠJud ah +rep air +Inst ant +5 48 +Ġemb assies +Ġbott leneck +Ġb ount +Ġtyp ew +ĠAl vin +j ing +im ilar +R ush +Ġbr im +ĠHEL P +A im +] ' +Ġpass ively +Ġbound ed +ĠR ated +Ġcriminal ity +Ġbiom ark +Ġdisp atcher +ĠTow ards +Ġ+ ++ +right eous +f rog +ĠP anc +C arter +0 32 +æ© Ł +Ġult raviolet +ĠLic ensed +ĠT ata +ĠBl essing +ĠG AM +Ġchem ically +ĠSe af +ĠRE LE +ĠMerc enary +capital ist +Ġform ulations +Ġann ihilation +ĠVer b +ĠAr gon +Ġun loaded +Ġmorp hed +Ġconqu ering +back er +I ELD +Ġtheft s +Ġfront runner +ĠRoy ale +ĠFund amental +el ight +C hip +necess ary +ay n +ĠSl ip +Ġ4 48 +cern ed +P ause +Ġshock ingly +ĠAB V +Ġcomp osure +7 33 +ĠMotors port +ah ime +Mur ray +M ach +Ġgr ids +Ġdeb ian +Ġfurther more +Ġdexter ity +ĠCollect ions +os lov +il age +b j +ĠMont eneg +Ġstrut Connector +Ġmassac res +Ġbrief s +fet ched +uv ian +ol ition +Fail ure +emon ic +Ġfl ared +Ġclaim ant +Ġc ures +Ġgive aways +ĠSubst ance +al ions +Ġcr inge +ĠK ul +Ġarist ocracy +ĠUl ster +ol ated +h ousing +ĠM IS +Ġgl ared +ĠWil helm +ne eds +lam bda +build ers +ĠV IS +Ġradi ator +ĠGhost busters +Ġ4 36 +act ual +Ġher ds +ç a +watch ing +Ġcounter ing +Ch arge +Ġchar red +Ġwar heads +Ġiod ine +ĠM acy +04 1 +Ġdepart ures +ĠS ins +Ġdy ed +ĠConcept s +g ado +7 13 +Ġquot ations +Ġg ist +ĠChrist y +Ġant igen +ĠHem p +ĠD rawn +ĠB arg +ez vous +Ġp aternity +Ġar du +ĠAnch orage +ĠR ik +Ġover loaded +ĠUs ername +ĠTam my +ĠN au +ĠCell ular +Ġw aning +Ġrod ent +ĠWor cester +il ts +ĠT ad +Ġdwell ings +Ġbull ish +4 31 +Ġretali ate +Ġmig raine +ĠChev ron +CH ECK +Ġdon key +c rim +SP A +ĠAn alog +Ġmarqu ee +ĠHa as +B ir +ĠGD DR +ĠDownload s +Ġwill power +ĠFor th +ĠRecord ed +Ġimp ossibility +ĠLog ged +ĠFr anks +ĠR att +in itions +Ġclean ers +Ġsore ly +Ġflick ering +ĠEx amination +c atching +allow een +Ms g +Ġdun no +F a +Ġdys ph +c razy +.' '. +Ġmain line +Ġc s +Ġp tr +ĠW ally +ig un +95 1 +ĠBig foot +f ights +Ġretrie ving +J r +Ġdupl ication +ĠExpl an +Ġrel ational +Ġqu aint +Ġbisc uits +Ġad o +Ġsh udder +Ġantid ote +blood ed +ks h +Ġsa uces +Ġrein vest +Ġdispens ary +ĠD iver +Ġ9 000 +stud ent +Ġin separ +esc ap +Ġtodd lers +ĠGP IO +ĠAss ignment +head ers +Ġlack luster +Ġab ack +95 6 +Ġtool bar +7 45 +Ġo ust +Ġcontempl ation +ĠPRES IDENT +Ġ4 58 +==== == +Ġguarantee ing +ĠHe ist +ĠCann es +Ļ ½ +Ġcollabor ator +ĠAm p +Ġg ou +ĠSH ALL +st ories +78 3 +Ġmobil ized +Ġbro od +ĠL U +ĠðŁ ij +Ġref in +ĠAnthrop ology +v ind +ill i +Ġwarrant ies +ĠB abel +Ġsw ath +Ġc aches +Ġantagon ists +art ifacts +Ġhot ly +ĠSt arts +ĠG ö +z ag +!! !!! +Ġsc ourge +Ġcons piring +ru its +re verse +ĠShe en +ĠJes uit +ĠGiov anni +ad ies +Ġbutt ocks +ear cher +ac an +Ġvolley ball +Ġshroud ed +Ġscore board +b ats +ĠI PM +Ġass es +Ġde regulation +ĠTe legram +ĠReb oot +Ġ7 000 +ĠCan ary +Ġk ernels +ĠFranç ois +ĠD uff +ĠP on +ĠLe ica +ĠGar min +Ġor phans +ĠClaud ia +Ġcal endars +ĠLe ilan +ent o +R ocket +Ġbr unch +ĠHaw king +ain ers +Ġsens ibilities +Ġk W +ĠK and +Ġre claimed +Ġinteresting ly +× © +rom y +J M +ĠEnhance ment +b ush +Sk ip +Ġrapp ers +Ġg azing +p edia +ath lon +Rev olution +Ġsn ipers +Ġre verted +Ġconglomer ate +T erry +79 4 +Ġhars her +Ġdes olate +ĠHit man +Comm ission +Ġ( / +âĢ¦ ." +Com par +Ġampl ification +om inated +Ġreg ress +ĠColl ider +Ġinform ants +Ġg azed diff --git a/experiments/elasticdnn/gpt_neo/125m_ckpt/tokenizer_config.json b/experiments/elasticdnn/gpt_neo/125m_ckpt/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..8bc66235bd92562997dfe19abc4aa10abef2414d --- /dev/null +++ b/experiments/elasticdnn/gpt_neo/125m_ckpt/tokenizer_config.json @@ -0,0 +1 @@ +{"errors": "replace", "unk_token": {"content": "<|endoftext|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "bos_token": {"content": "<|endoftext|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "eos_token": {"content": "<|endoftext|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "add_prefix_space": false, "model_max_length": 2048, "special_tokens_map_file": null, "name_or_path": "gpt2"} \ No newline at end of file diff --git a/experiments/elasticdnn/gpt_neo/125m_ckpt/vocab.json b/experiments/elasticdnn/gpt_neo/125m_ckpt/vocab.json new file mode 100644 index 0000000000000000000000000000000000000000..b00361fece0387ca34b4b8b8539ed830d644dbeb --- /dev/null +++ b/experiments/elasticdnn/gpt_neo/125m_ckpt/vocab.json @@ -0,0 +1 @@ +{"!": 0, "\"": 1, "#": 2, "$": 3, "%": 4, "&": 5, "'": 6, "(": 7, ")": 8, "*": 9, "+": 10, ",": 11, "-": 12, ".": 13, "/": 14, "0": 15, "1": 16, "2": 17, "3": 18, "4": 19, "5": 20, "6": 21, "7": 22, "8": 23, "9": 24, ":": 25, ";": 26, "<": 27, "=": 28, ">": 29, "?": 30, "@": 31, "A": 32, "B": 33, "C": 34, "D": 35, "E": 36, "F": 37, "G": 38, "H": 39, "I": 40, "J": 41, "K": 42, "L": 43, "M": 44, "N": 45, "O": 46, "P": 47, "Q": 48, "R": 49, "S": 50, "T": 51, "U": 52, "V": 53, "W": 54, "X": 55, "Y": 56, "Z": 57, "[": 58, "\\": 59, "]": 60, "^": 61, "_": 62, "`": 63, "a": 64, "b": 65, "c": 66, "d": 67, "e": 68, "f": 69, "g": 70, "h": 71, "i": 72, "j": 73, "k": 74, "l": 75, "m": 76, "n": 77, "o": 78, "p": 79, "q": 80, "r": 81, "s": 82, "t": 83, "u": 84, "v": 85, "w": 86, "x": 87, "y": 88, "z": 89, "{": 90, "|": 91, "}": 92, "~": 93, "¡": 94, "¢": 95, "£": 96, "¤": 97, "¥": 98, "¦": 99, "§": 100, "¨": 101, "©": 102, "ª": 103, "«": 104, "¬": 105, "®": 106, "¯": 107, "°": 108, "±": 109, "²": 110, "³": 111, "´": 112, "µ": 113, "¶": 114, "·": 115, "¸": 116, "¹": 117, "º": 118, "»": 119, "¼": 120, "½": 121, "¾": 122, "¿": 123, "À": 124, "Á": 125, "Â": 126, "Ã": 127, "Ä": 128, "Å": 129, "Æ": 130, "Ç": 131, "È": 132, "É": 133, "Ê": 134, "Ë": 135, "Ì": 136, "Í": 137, "Î": 138, "Ï": 139, "Ð": 140, "Ñ": 141, "Ò": 142, "Ó": 143, "Ô": 144, "Õ": 145, "Ö": 146, "×": 147, "Ø": 148, "Ù": 149, "Ú": 150, "Û": 151, "Ü": 152, "Ý": 153, "Þ": 154, "ß": 155, "à": 156, "á": 157, "â": 158, "ã": 159, "ä": 160, "å": 161, "æ": 162, "ç": 163, "è": 164, "é": 165, "ê": 166, "ë": 167, "ì": 168, "í": 169, "î": 170, "ï": 171, "ð": 172, "ñ": 173, "ò": 174, "ó": 175, "ô": 176, "õ": 177, "ö": 178, "÷": 179, "ø": 180, "ù": 181, "ú": 182, "û": 183, "ü": 184, "ý": 185, "þ": 186, "ÿ": 187, "Ā": 188, "ā": 189, "Ă": 190, "ă": 191, "Ą": 192, "ą": 193, "Ć": 194, "ć": 195, "Ĉ": 196, "ĉ": 197, "Ċ": 198, "ċ": 199, "Č": 200, "č": 201, "Ď": 202, "ď": 203, "Đ": 204, "đ": 205, "Ē": 206, "ē": 207, "Ĕ": 208, "ĕ": 209, "Ė": 210, "ė": 211, "Ę": 212, "ę": 213, "Ě": 214, "ě": 215, "Ĝ": 216, "ĝ": 217, "Ğ": 218, "ğ": 219, "Ġ": 220, "ġ": 221, "Ģ": 222, "ģ": 223, "Ĥ": 224, "ĥ": 225, "Ħ": 226, "ħ": 227, "Ĩ": 228, "ĩ": 229, "Ī": 230, "ī": 231, "Ĭ": 232, "ĭ": 233, "Į": 234, "į": 235, "İ": 236, "ı": 237, "IJ": 238, "ij": 239, "Ĵ": 240, "ĵ": 241, "Ķ": 242, "ķ": 243, "ĸ": 244, "Ĺ": 245, "ĺ": 246, "Ļ": 247, "ļ": 248, "Ľ": 249, "ľ": 250, "Ŀ": 251, "ŀ": 252, "Ł": 253, "ł": 254, "Ń": 255, "Ġt": 256, "Ġa": 257, "he": 258, "in": 259, "re": 260, "on": 261, "Ġthe": 262, "er": 263, "Ġs": 264, "at": 265, "Ġw": 266, "Ġo": 267, "en": 268, "Ġc": 269, "it": 270, "is": 271, "an": 272, "or": 273, "es": 274, "Ġb": 275, "ed": 276, "Ġf": 277, "ing": 278, "Ġp": 279, "ou": 280, "Ġan": 281, "al": 282, "ar": 283, "Ġto": 284, "Ġm": 285, "Ġof": 286, "Ġin": 287, "Ġd": 288, "Ġh": 289, "Ġand": 290, "ic": 291, "as": 292, "le": 293, "Ġth": 294, "ion": 295, "om": 296, "ll": 297, "ent": 298, "Ġn": 299, "Ġl": 300, "st": 301, "Ġre": 302, "ve": 303, "Ġe": 304, "ro": 305, "ly": 306, "Ġbe": 307, "Ġg": 308, "ĠT": 309, "ct": 310, "ĠS": 311, "id": 312, "ot": 313, "ĠI": 314, "ut": 315, "et": 316, "ĠA": 317, "Ġis": 318, "Ġon": 319, "im": 320, "am": 321, "ow": 322, "ay": 323, "ad": 324, "se": 325, "Ġthat": 326, "ĠC": 327, "ig": 328, "Ġfor": 329, "ac": 330, "Ġy": 331, "ver": 332, "ur": 333, "Ġu": 334, "ld": 335, "Ġst": 336, "ĠM": 337, "'s": 338, "Ġhe": 339, "Ġit": 340, "ation": 341, "ith": 342, "ir": 343, "ce": 344, "Ġyou": 345, "il": 346, "ĠB": 347, "Ġwh": 348, "ol": 349, "ĠP": 350, "Ġwith": 351, "Ġ1": 352, "ter": 353, "ch": 354, "Ġas": 355, "Ġwe": 356, "Ġ(": 357, "nd": 358, "ill": 359, "ĠD": 360, "if": 361, "Ġ2": 362, "ag": 363, "ers": 364, "ke": 365, "Ġ\"": 366, "ĠH": 367, "em": 368, "Ġcon": 369, "ĠW": 370, "ĠR": 371, "her": 372, "Ġwas": 373, "Ġr": 374, "od": 375, "ĠF": 376, "ul": 377, "ate": 378, "Ġat": 379, "ri": 380, "pp": 381, "ore": 382, "ĠThe": 383, "Ġse": 384, "us": 385, "Ġpro": 386, "Ġha": 387, "um": 388, "Ġare": 389, "Ġde": 390, "ain": 391, "and": 392, "Ġor": 393, "igh": 394, "est": 395, "ist": 396, "ab": 397, "rom": 398, "ĠN": 399, "th": 400, "Ġcom": 401, "ĠG": 402, "un": 403, "op": 404, "00": 405, "ĠL": 406, "Ġnot": 407, "ess": 408, "Ġex": 409, "Ġv": 410, "res": 411, "ĠE": 412, "ew": 413, "ity": 414, "ant": 415, "Ġby": 416, "el": 417, "os": 418, "ort": 419, "oc": 420, "qu": 421, "Ġfrom": 422, "Ġhave": 423, "Ġsu": 424, "ive": 425, "ould": 426, "Ġsh": 427, "Ġthis": 428, "nt": 429, "ra": 430, "pe": 431, "ight": 432, "art": 433, "ment": 434, "Ġal": 435, "ust": 436, "end": 437, "--": 438, "all": 439, "ĠO": 440, "ack": 441, "Ġch": 442, "Ġle": 443, "ies": 444, "red": 445, "ard": 446, "âĢ": 447, "out": 448, "ĠJ": 449, "Ġab": 450, "ear": 451, "iv": 452, "ally": 453, "our": 454, "ost": 455, "gh": 456, "pt": 457, "Ġpl": 458, "ast": 459, "Ġcan": 460, "ak": 461, "ome": 462, "ud": 463, "The": 464, "Ġhis": 465, "Ġdo": 466, "Ġgo": 467, "Ġhas": 468, "ge": 469, "'t": 470, "ĠU": 471, "rou": 472, "Ġsa": 473, "Ġj": 474, "Ġbut": 475, "Ġwor": 476, "Ġall": 477, "ect": 478, "Ġk": 479, "ame": 480, "Ġwill": 481, "ok": 482, "Ġwhe": 483, "Ġthey": 484, "ide": 485, "01": 486, "ff": 487, "ich": 488, "pl": 489, "ther": 490, "Ġtr": 491, "..": 492, "Ġint": 493, "ie": 494, "ure": 495, "age": 496, "Ġne": 497, "ial": 498, "ap": 499, "ine": 500, "ice": 501, "Ġme": 502, "Ġout": 503, "ans": 504, "one": 505, "ong": 506, "ions": 507, "Ġwho": 508, "ĠK": 509, "Ġup": 510, "Ġtheir": 511, "Ġad": 512, "Ġ3": 513, "Ġus": 514, "ated": 515, "ous": 516, "Ġmore": 517, "ue": 518, "og": 519, "ĠSt": 520, "ind": 521, "ike": 522, "Ġso": 523, "ime": 524, "per": 525, ".\"": 526, "ber": 527, "iz": 528, "act": 529, "Ġone": 530, "Ġsaid": 531, "Ġ-": 532, "are": 533, "Ġyour": 534, "cc": 535, "ĠTh": 536, "Ġcl": 537, "ep": 538, "ake": 539, "able": 540, "ip": 541, "Ġcont": 542, "Ġwhich": 543, "ia": 544, "Ġim": 545, "Ġabout": 546, "Ġwere": 547, "very": 548, "ub": 549, "Ġhad": 550, "Ġen": 551, "Ġcomp": 552, ",\"": 553, "ĠIn": 554, "Ġun": 555, "Ġag": 556, "ire": 557, "ace": 558, "au": 559, "ary": 560, "Ġwould": 561, "ass": 562, "ry": 563, "ĠâĢ": 564, "cl": 565, "ook": 566, "ere": 567, "so": 568, "ĠV": 569, "ign": 570, "ib": 571, "Ġoff": 572, "Ġte": 573, "ven": 574, "ĠY": 575, "ile": 576, "ose": 577, "ite": 578, "orm": 579, "Ġ201": 580, "Ġres": 581, "Ġman": 582, "Ġper": 583, "Ġother": 584, "ord": 585, "ult": 586, "Ġbeen": 587, "Ġlike": 588, "ase": 589, "ance": 590, "ks": 591, "ays": 592, "own": 593, "ence": 594, "Ġdis": 595, "ction": 596, "Ġany": 597, "Ġapp": 598, "Ġsp": 599, "int": 600, "ress": 601, "ations": 602, "ail": 603, "Ġ4": 604, "ical": 605, "Ġthem": 606, "Ġher": 607, "ount": 608, "ĠCh": 609, "Ġar": 610, "Ġif": 611, "Ġthere": 612, "Ġpe": 613, "Ġyear": 614, "av": 615, "Ġmy": 616, "Ġsome": 617, "Ġwhen": 618, "ough": 619, "ach": 620, "Ġthan": 621, "ru": 622, "ond": 623, "ick": 624, "Ġover": 625, "vel": 626, "Ġqu": 627, "ĊĊ": 628, "Ġsc": 629, "reat": 630, "ree": 631, "ĠIt": 632, "ound": 633, "port": 634, "Ġalso": 635, "Ġpart": 636, "fter": 637, "Ġkn": 638, "Ġbec": 639, "Ġtime": 640, "ens": 641, "Ġ5": 642, "ople": 643, "Ġwhat": 644, "Ġno": 645, "du": 646, "mer": 647, "ang": 648, "Ġnew": 649, "----": 650, "Ġget": 651, "ory": 652, "ition": 653, "ings": 654, "Ġjust": 655, "Ġinto": 656, "Ġ0": 657, "ents": 658, "ove": 659, "te": 660, "Ġpeople": 661, "Ġpre": 662, "Ġits": 663, "Ġrec": 664, "Ġtw": 665, "ian": 666, "irst": 667, "ark": 668, "ors": 669, "Ġwork": 670, "ade": 671, "ob": 672, "Ġshe": 673, "Ġour": 674, "wn": 675, "ink": 676, "lic": 677, "Ġ19": 678, "ĠHe": 679, "ish": 680, "nder": 681, "ause": 682, "Ġhim": 683, "ons": 684, "Ġ[": 685, "Ġro": 686, "form": 687, "ild": 688, "ates": 689, "vers": 690, "Ġonly": 691, "oll": 692, "Ġspe": 693, "ck": 694, "ell": 695, "amp": 696, "Ġacc": 697, "Ġbl": 698, "ious": 699, "urn": 700, "ft": 701, "ood": 702, "Ġhow": 703, "hed": 704, "Ġ'": 705, "Ġafter": 706, "aw": 707, "Ġatt": 708, "ov": 709, "ne": 710, "Ġplay": 711, "erv": 712, "ict": 713, "Ġcould": 714, "itt": 715, "Ġam": 716, "Ġfirst": 717, "Ġ6": 718, "Ġact": 719, "Ġ$": 720, "ec": 721, "hing": 722, "ual": 723, "ull": 724, "Ġcomm": 725, "oy": 726, "old": 727, "ces": 728, "ater": 729, "Ġfe": 730, "Ġbet": 731, "we": 732, "iff": 733, "Ġtwo": 734, "ock": 735, "Ġback": 736, ").": 737, "ident": 738, "Ġunder": 739, "rough": 740, "sel": 741, "xt": 742, "Ġmay": 743, "round": 744, "Ġpo": 745, "ph": 746, "iss": 747, "Ġdes": 748, "Ġmost": 749, "Ġdid": 750, "Ġadd": 751, "ject": 752, "Ġinc": 753, "fore": 754, "Ġpol": 755, "ont": 756, "Ġagain": 757, "clud": 758, "tern": 759, "Ġknow": 760, "Ġneed": 761, "Ġcons": 762, "Ġco": 763, "Ġ.": 764, "Ġwant": 765, "Ġsee": 766, "Ġ7": 767, "ning": 768, "iew": 769, "ĠThis": 770, "ced": 771, "Ġeven": 772, "Ġind": 773, "ty": 774, "ĠWe": 775, "ath": 776, "Ġthese": 777, "Ġpr": 778, "Ġuse": 779, "Ġbecause": 780, "Ġfl": 781, "ng": 782, "Ġnow": 783, "ĠâĢĵ": 784, "com": 785, "ise": 786, "Ġmake": 787, "Ġthen": 788, "ower": 789, "Ġevery": 790, "ĠUn": 791, "Ġsec": 792, "oss": 793, "uch": 794, "Ġem": 795, "Ġ=": 796, "ĠRe": 797, "ied": 798, "rit": 799, "Ġinv": 800, "lect": 801, "Ġsupp": 802, "ating": 803, "Ġlook": 804, "man": 805, "pect": 806, "Ġ8": 807, "row": 808, "Ġbu": 809, "Ġwhere": 810, "ific": 811, "Ġyears": 812, "ily": 813, "Ġdiff": 814, "Ġshould": 815, "Ġrem": 816, "Th": 817, "In": 818, "Ġev": 819, "day": 820, "'re": 821, "rib": 822, "Ġrel": 823, "ss": 824, "Ġdef": 825, "Ġright": 826, "Ġsy": 827, "),": 828, "les": 829, "000": 830, "hen": 831, "Ġthrough": 832, "ĠTr": 833, "__": 834, "Ġway": 835, "Ġdon": 836, "Ġ,": 837, "Ġ10": 838, "ased": 839, "Ġass": 840, "ublic": 841, "Ġreg": 842, "ĠAnd": 843, "ix": 844, "Ġvery": 845, "Ġinclud": 846, "other": 847, "Ġimp": 848, "oth": 849, "Ġsub": 850, "ĠâĢĶ": 851, "Ġbeing": 852, "arg": 853, "ĠWh": 854, "==": 855, "ible": 856, "Ġdoes": 857, "ange": 858, "ram": 859, "Ġ9": 860, "ert": 861, "ps": 862, "ited": 863, "ational": 864, "Ġbr": 865, "Ġdown": 866, "Ġmany": 867, "aking": 868, "Ġcall": 869, "uring": 870, "ities": 871, "Ġph": 872, "ics": 873, "als": 874, "Ġdec": 875, "ative": 876, "ener": 877, "Ġbefore": 878, "ility": 879, "Ġwell": 880, "Ġmuch": 881, "erson": 882, "Ġthose": 883, "Ġsuch": 884, "Ġke": 885, "Ġend": 886, "ĠBut": 887, "ason": 888, "ting": 889, "Ġlong": 890, "ef": 891, "Ġthink": 892, "ys": 893, "Ġbel": 894, "Ġsm": 895, "its": 896, "ax": 897, "Ġown": 898, "Ġprov": 899, "Ġset": 900, "ife": 901, "ments": 902, "ble": 903, "ward": 904, "Ġshow": 905, "Ġpres": 906, "ms": 907, "omet": 908, "Ġob": 909, "Ġsay": 910, "ĠSh": 911, "ts": 912, "ful": 913, "Ġeff": 914, "Ġgu": 915, "Ġinst": 916, "und": 917, "ren": 918, "cess": 919, "Ġent": 920, "ĠYou": 921, "Ġgood": 922, "Ġstart": 923, "ince": 924, "Ġmade": 925, "tt": 926, "stem": 927, "olog": 928, "up": 929, "Ġ|": 930, "ump": 931, "Ġhel": 932, "vern": 933, "ular": 934, "ually": 935, "Ġac": 936, "Ġmon": 937, "Ġlast": 938, "Ġ200": 939, "10": 940, "Ġstud": 941, "ures": 942, "ĠAr": 943, "self": 944, "ars": 945, "meric": 946, "ues": 947, "cy": 948, "Ġmin": 949, "ollow": 950, "Ġcol": 951, "io": 952, "Ġmod": 953, "Ġcount": 954, "ĠCom": 955, "hes": 956, "Ġfin": 957, "air": 958, "ier": 959, "âĢĶ": 960, "read": 961, "ank": 962, "atch": 963, "ever": 964, "Ġstr": 965, "Ġpoint": 966, "ork": 967, "ĠNew": 968, "Ġsur": 969, "ool": 970, "alk": 971, "ement": 972, "Ġused": 973, "ract": 974, "ween": 975, "Ġsame": 976, "oun": 977, "ĠAl": 978, "ci": 979, "Ġdiffere": 980, "Ġwhile": 981, "--------": 982, "Ġgame": 983, "cept": 984, "Ġsim": 985, "...": 986, "Ġinter": 987, "ek": 988, "Ġreport": 989, "Ġprodu": 990, "Ġstill": 991, "led": 992, "ah": 993, "Ġhere": 994, "Ġworld": 995, "Ġthough": 996, "Ġnum": 997, "arch": 998, "imes": 999, "ale": 1000, "ĠSe": 1001, "ĠIf": 1002, "//": 1003, "ĠLe": 1004, "Ġret": 1005, "Ġref": 1006, "Ġtrans": 1007, "ner": 1008, "ution": 1009, "ters": 1010, "Ġtake": 1011, "ĠCl": 1012, "Ġconf": 1013, "way": 1014, "ave": 1015, "Ġgoing": 1016, "Ġsl": 1017, "ug": 1018, "ĠAmeric": 1019, "Ġspec": 1020, "Ġhand": 1021, "Ġbetween": 1022, "ists": 1023, "ĠDe": 1024, "oot": 1025, "It": 1026, "Ġear": 1027, "Ġagainst": 1028, "Ġhigh": 1029, "gan": 1030, "az": 1031, "ather": 1032, "Ġexp": 1033, "Ġop": 1034, "Ġins": 1035, "Ġgr": 1036, "Ġhelp": 1037, "Ġrequ": 1038, "ets": 1039, "ins": 1040, "ĠPro": 1041, "ism": 1042, "Ġfound": 1043, "land": 1044, "ata": 1045, "uss": 1046, "ames": 1047, "Ġperson": 1048, "Ġgreat": 1049, "pr": 1050, "Ġsign": 1051, "ĠAn": 1052, "'ve": 1053, "Ġsomet": 1054, "Ġser": 1055, "hip": 1056, "Ġrun": 1057, "Ġ:": 1058, "Ġter": 1059, "irect": 1060, "Ġfollow": 1061, "Ġdet": 1062, "ices": 1063, "Ġfind": 1064, "12": 1065, "Ġmem": 1066, "Ġcr": 1067, "ered": 1068, "ex": 1069, "Ġext": 1070, "uth": 1071, "ense": 1072, "co": 1073, "Ġteam": 1074, "ving": 1075, "ouse": 1076, "ash": 1077, "att": 1078, "ved": 1079, "Ġsystem": 1080, "ĠAs": 1081, "der": 1082, "ives": 1083, "min": 1084, "Ġlead": 1085, "ĠBl": 1086, "cent": 1087, "Ġaround": 1088, "Ġgovern": 1089, "Ġcur": 1090, "velop": 1091, "any": 1092, "Ġcour": 1093, "alth": 1094, "ages": 1095, "ize": 1096, "Ġcar": 1097, "ode": 1098, "Ġlaw": 1099, "Ġread": 1100, "'m": 1101, "con": 1102, "Ġreal": 1103, "Ġsupport": 1104, "Ġ12": 1105, "....": 1106, "Ġreally": 1107, "ness": 1108, "Ġfact": 1109, "Ġday": 1110, "Ġboth": 1111, "ying": 1112, "Ġserv": 1113, "ĠFor": 1114, "Ġthree": 1115, "Ġwom": 1116, "Ġmed": 1117, "ody": 1118, "ĠThey": 1119, "50": 1120, "Ġexper": 1121, "ton": 1122, "Ġeach": 1123, "akes": 1124, "Ġche": 1125, "Ġcre": 1126, "ines": 1127, "Ġrep": 1128, "19": 1129, "gg": 1130, "illion": 1131, "Ġgrou": 1132, "ute": 1133, "ik": 1134, "We": 1135, "get": 1136, "ER": 1137, "Ġmet": 1138, "Ġsays": 1139, "ox": 1140, "Ġduring": 1141, "ern": 1142, "ized": 1143, "ared": 1144, "Ġfam": 1145, "ically": 1146, "Ġhapp": 1147, "ĠIs": 1148, "Ġchar": 1149, "med": 1150, "vent": 1151, "Ġgener": 1152, "ient": 1153, "ple": 1154, "iet": 1155, "rent": 1156, "11": 1157, "ves": 1158, "ption": 1159, "Ġ20": 1160, "formation": 1161, "Ġcor": 1162, "Ġoffic": 1163, "ield": 1164, "Ġtoo": 1165, "ision": 1166, "Ġinf": 1167, "ĠZ": 1168, "the": 1169, "oad": 1170, "Ġpublic": 1171, "Ġprog": 1172, "ric": 1173, "**": 1174, "Ġwar": 1175, "Ġpower": 1176, "view": 1177, "Ġfew": 1178, "Ġloc": 1179, "Ġdifferent": 1180, "Ġstate": 1181, "Ġhead": 1182, "'ll": 1183, "Ġposs": 1184, "Ġstat": 1185, "ret": 1186, "ants": 1187, "Ġval": 1188, "Ġiss": 1189, "Ġcle": 1190, "ivers": 1191, "anc": 1192, "Ġexpl": 1193, "Ġanother": 1194, "ĠQ": 1195, "Ġav": 1196, "thing": 1197, "nce": 1198, "Wh": 1199, "Ġchild": 1200, "Ġsince": 1201, "ired": 1202, "less": 1203, "Ġlife": 1204, "Ġdevelop": 1205, "ittle": 1206, "Ġdep": 1207, "Ġpass": 1208, "ãĥ": 1209, "Ġturn": 1210, "orn": 1211, "This": 1212, "bers": 1213, "ross": 1214, "ĠAd": 1215, "Ġfr": 1216, "Ġresp": 1217, "Ġsecond": 1218, "oh": 1219, "Ġ/": 1220, "Ġdisc": 1221, "Ġ&": 1222, "Ġsomething": 1223, "Ġcomple": 1224, "Ġed": 1225, "Ġfil": 1226, "Ġmonth": 1227, "aj": 1228, "uc": 1229, "Ġgovernment": 1230, "Ġwithout": 1231, "Ġleg": 1232, "Ġdist": 1233, "Ġput": 1234, "Ġquest": 1235, "ann": 1236, "Ġprot": 1237, "20": 1238, "Ġnever": 1239, "ience": 1240, "Ġlevel": 1241, "Ġart": 1242, "Ġthings": 1243, "Ġmight": 1244, "Ġeffect": 1245, "Ġcontro": 1246, "Ġcent": 1247, "Ġ18": 1248, "Ġallow": 1249, "Ġbelie": 1250, "chool": 1251, "ott": 1252, "Ġincre": 1253, "Ġfeel": 1254, "Ġresult": 1255, "Ġlot": 1256, "Ġfun": 1257, "ote": 1258, "Ġty": 1259, "erest": 1260, "Ġcontin": 1261, "Ġusing": 1262, "Ġbig": 1263, "201": 1264, "Ġask": 1265, "Ġbest": 1266, "Ġ)": 1267, "IN": 1268, "Ġopp": 1269, "30": 1270, "Ġnumber": 1271, "iness": 1272, "St": 1273, "lease": 1274, "Ġca": 1275, "Ġmust": 1276, "Ġdirect": 1277, "Ġgl": 1278, "Ġ<": 1279, "Ġopen": 1280, "Ġpost": 1281, "Ġcome": 1282, "Ġseem": 1283, "ording": 1284, "Ġweek": 1285, "ately": 1286, "ital": 1287, "Ġel": 1288, "riend": 1289, "Ġfar": 1290, "Ġtra": 1291, "inal": 1292, "Ġpri": 1293, "ĠUS": 1294, "Ġplace": 1295, "Ġform": 1296, "Ġtold": 1297, "\":": 1298, "ains": 1299, "ature": 1300, "ĠTrump": 1301, "Ġstand": 1302, "Ġ#": 1303, "ider": 1304, "ĠFr": 1305, "Ġnext": 1306, "Ġsoc": 1307, "Ġpur": 1308, "Ġlet": 1309, "Ġlittle": 1310, "Ġhum": 1311, "Ġi": 1312, "ron": 1313, "15": 1314, "Ġ15": 1315, "Ġcommun": 1316, "Ġmark": 1317, "ĠThere": 1318, "Ġwr": 1319, "ĠThat": 1320, "Ġinformation": 1321, "ways": 1322, "Ġbus": 1323, "app": 1324, "Ġinvest": 1325, "me": 1326, "Ġhard": 1327, "ained": 1328, "ead": 1329, "Ġimport": 1330, "Ġappro": 1331, "Ġtest": 1332, "Ġtri": 1333, "Ġrest": 1334, "osed": 1335, "Ġfull": 1336, "Ġcare": 1337, "ĠSp": 1338, "Ġcase": 1339, "ON": 1340, "Ġsk": 1341, "Ġless": 1342, "Ġ+": 1343, "Ġpartic": 1344, "ĠPl": 1345, "ably": 1346, "uck": 1347, "ished": 1348, "chn": 1349, "be": 1350, "Ġlist": 1351, "ator": 1352, "Ġtop": 1353, "Ġadv": 1354, "ĠBe": 1355, "ruct": 1356, "Ġdem": 1357, "ration": 1358, "ling": 1359, "gy": 1360, "reen": 1361, "ger": 1362, "Ġhome": 1363, "Ġleft": 1364, "Ġbetter": 1365, "Ġdata": 1366, "Ġ11": 1367, "Ġattack": 1368, "Ġproble": 1369, "line": 1370, "ards": 1371, "Ġbeh": 1372, "ral": 1373, "ĠHow": 1374, "ĠShe": 1375, "arge": 1376, "Ġ--": 1377, "://": 1378, "Ġbro": 1379, "ĠPh": 1380, "ats": 1381, "Ġbuild": 1382, "ww": 1383, "ided": 1384, "aim": 1385, "ases": 1386, "ency": 1387, "Ġmain": 1388, "ined": 1389, "Ġincluding": 1390, "Ġ{": 1391, "Ġgot": 1392, "Ġinterest": 1393, "Ġkeep": 1394, "ĠX": 1395, "Ġeas": 1396, "aining": 1397, "Ġclass": 1398, "âĢ¦": 1399, "ĠNo": 1400, "Ġvar": 1401, "Ġsmall": 1402, "ample": 1403, "AT": 1404, "Ġide": 1405, "ĠSo": 1406, "Ġrece": 1407, "Ġpolit": 1408, "Ġmov": 1409, "Ġplan": 1410, "Ġpercent": 1411, "iving": 1412, "Ġcamp": 1413, "Ġpay": 1414, "14": 1415, "sc": 1416, "ised": 1417, "Ġunt": 1418, "oney": 1419, "ploy": 1420, "====": 1421, "Ġdidn": 1422, "ĠInd": 1423, "els": 1424, "ertain": 1425, "Ġpos": 1426, "____": 1427, "iver": 1428, "Ġprocess": 1429, "Ġprogram": 1430, "ified": 1431, "ĠRep": 1432, "16": 1433, "uro": 1434, "ology": 1435, "atter": 1436, "ina": 1437, "Ġname": 1438, "ĠAll": 1439, "Ġfour": 1440, "Ġreturn": 1441, "vious": 1442, "bs": 1443, "Ġcalled": 1444, "Ġmove": 1445, "ĠSc": 1446, "ird": 1447, "Ġgroup": 1448, "Ġbre": 1449, "Ġmen": 1450, "Ġcap": 1451, "ten": 1452, "ee": 1453, "Ġdri": 1454, "leg": 1455, "here": 1456, "uthor": 1457, "Ġpat": 1458, "Ġcurrent": 1459, "ides": 1460, "Ġpop": 1461, "to": 1462, "ention": 1463, "Ġalways": 1464, "Ġmil": 1465, "Ġwomen": 1466, "Ġ16": 1467, "Ġold": 1468, "iven": 1469, "raph": 1470, "ĠOr": 1471, "ror": 1472, "ently": 1473, "Ġnear": 1474, "ĠEx": 1475, "ream": 1476, "sh": 1477, "Ġ14": 1478, "Ġfree": 1479, "ission": 1480, "stand": 1481, "ĠCon": 1482, "ality": 1483, "used": 1484, "13": 1485, "Ġdesign": 1486, "Ġchange": 1487, "Ġchang": 1488, "Ġbo": 1489, "Ġvis": 1490, "ember": 1491, "Ġbook": 1492, "ready": 1493, "Ġkill": 1494, "25": 1495, "pped": 1496, "Ġaway": 1497, "Ġable": 1498, "Ġcountry": 1499, "Ġconst": 1500, "arn": 1501, "Ġorder": 1502, "AR": 1503, "ior": 1504, "ium": 1505, "orth": 1506, "18": 1507, "ailable": 1508, "Ġsw": 1509, "Ġmillion": 1510, "Ġ13": 1511, "atic": 1512, "ted": 1513, "ĠGo": 1514, "Ġoper": 1515, "eng": 1516, "Ġthing": 1517, "ajor": 1518, "conom": 1519, "ĠComm": 1520, "Ġwhy": 1521, "ured": 1522, "ural": 1523, "Ġschool": 1524, "by": 1525, "ĠMar": 1526, "Ġaff": 1527, "Ġdays": 1528, "Ġann": 1529, "ush": 1530, "ane": 1531, "If": 1532, "eg": 1533, "Ġprof": 1534, "Ġhealth": 1535, "outh": 1536, "But": 1537, "ional": 1538, ".,": 1539, "Ġsol": 1540, "Ġalready": 1541, "Ġ30": 1542, "Ġcharact": 1543, "He": 1544, "Ġfriend": 1545, "ES": 1546, "ians": 1547, "icle": 1548, "'d": 1549, "ĠOn": 1550, "Ġleast": 1551, "Ġprom": 1552, "Ġdr": 1553, "Ġhist": 1554, "ither": 1555, "Ġest": 1556, "iqu": 1557, "17": 1558, "son": 1559, "Ġtell": 1560, "Ġtalk": 1561, "ohn": 1562, "oint": 1563, "lection": 1564, "AN": 1565, "Ġuntil": 1566, "augh": 1567, "Ġlater": 1568, "Ġve": 1569, "Ġview": 1570, "ending": 1571, "ived": 1572, "Ġword": 1573, "ware": 1574, "Ġcost": 1575, "Ġenough": 1576, "Ġgive": 1577, "ĠUnited": 1578, "Ġtechn": 1579, "arent": 1580, "OR": 1581, "Ġpar": 1582, "ĠDr": 1583, "Ġ2016": 1584, "rist": 1585, "ering": 1586, "ĠÂ": 1587, "Ġlarge": 1588, "side": 1589, "acy": 1590, "ccess": 1591, "Ġwin": 1592, "Ġimportant": 1593, "Ġ199": 1594, "Ġdoesn": 1595, "Ġ17": 1596, "Ġbusiness": 1597, "Ġclear": 1598, "Ġrese": 1599, "\",": 1600, "ury": 1601, "Ġequ": 1602, "aster": 1603, "alf": 1604, "ĠAmerican": 1605, "nect": 1606, "Ġexpect": 1607, "iversity": 1608, "Ġocc": 1609, "ĠFl": 1610, "Ġkind": 1611, "Ġmean": 1612, "Ġpast": 1613, "Ġdev": 1614, "Ġbas": 1615, "let": 1616, "raft": 1617, "Ġorgan": 1618, "Ġdel": 1619, "Ġperform": 1620, "Ġstory": 1621, "Ġseason": 1622, "ĠCol": 1623, "Ġclaim": 1624, "Ġcame": 1625, "Ġwithin": 1626, "Ġline": 1627, "Ġproject": 1628, "ĠAt": 1629, "Ġcontrol": 1630, "ended": 1631, "ĠSy": 1632, "Ġair": 1633, "ization": 1634, "Ġ*": 1635, "ley": 1636, "Ġmoney": 1637, "idd": 1638, "You": 1639, "for": 1640, "Ġfamily": 1641, "Ġmaking": 1642, "Ġbit": 1643, "Ġpolice": 1644, "Ġhappen": 1645, "Ġvers": 1646, "ony": 1647, "uff": 1648, "ĠWhen": 1649, "Ġsit": 1650, "ideo": 1651, "lf": 1652, "ison": 1653, "Ġsure": 1654, "gin": 1655, "Ġappear": 1656, "Ġlight": 1657, "Ġes": 1658, "of": 1659, "Ġwater": 1660, "Ġtimes": 1661, "not": 1662, "Ġgrow": 1663, "Ġcompany": 1664, "ĠTe": 1665, "ows": 1666, "Ġmar": 1667, "ource": 1668, "iol": 1669, "arm": 1670, "br": 1671, "Ġexample": 1672, "Ġconc": 1673, "Ġfore": 1674, "ĠTo": 1675, "pro": 1676, "EN": 1677, "ries": 1678, "Ġ25": 1679, "ĠCan": 1680, "ney": 1681, "Ġactually": 1682, "Ġever": 1683, "urity": 1684, "aken": 1685, "aps": 1686, "Ġtax": 1687, "Ġmajor": 1688, "ama": 1689, "Ġoften": 1690, "eral": 1691, "Ġhuman": 1692, "Ġjob": 1693, "ister": 1694, "Ġavailable": 1695, "ocr": 1696, "enn": 1697, "aid": 1698, "ivid": 1699, "Ġrecord": 1700, "?\"": 1701, "Ġsing": 1702, "ĠAm": 1703, "idence": 1704, "Ġnews": 1705, "ster": 1706, "Ġeconom": 1707, "Ġfollowing": 1708, "ĠBr": 1709, "ising": 1710, "Ġhour": 1711, "most": 1712, "ument": 1713, "Ġsex": 1714, "Ġdesc": 1715, "Ġbecome": 1716, "ĠEd": 1717, "Ġtook": 1718, "Ġhaving": 1719, "Ġproduct": 1720, "ault": 1721, "As": 1722, "aring": 1723, "Ġmeans": 1724, "Ġhop": 1725, "une": 1726, "Ġcho": 1727, "Ġcertain": 1728, "Ġnon": 1729, "Ġdeal": 1730, "24": 1731, "lement": 1732, "oci": 1733, "ene": 1734, "Ġside": 1735, "ĠPr": 1736, "ĠMay": 1737, "Ġreason": 1738, "ued": 1739, "ched": 1740, "ulation": 1741, "Ġelect": 1742, "Ġofficial": 1743, "Ġpossible": 1744, "Ġhold": 1745, "ands": 1746, "ots": 1747, "Ġcity": 1748, "ories": 1749, "Ġsever": 1750, "Ġchildren": 1751, "Ġonce": 1752, "Ġactiv": 1753, "ler": 1754, "Ġnight": 1755, "itions": 1756, "ĠJohn": 1757, "ape": 1758, "play": 1759, "Ġdone": 1760, "Ġlim": 1761, "Ġworking": 1762, "ĠPres": 1763, "orld": 1764, "eb": 1765, "ĠCo": 1766, "Ġbody": 1767, "ails": 1768, "utes": 1769, "ĠMr": 1770, "Ġwhether": 1771, "Ġauthor": 1772, "rop": 1773, "Ġproper": 1774, "Ġseen": 1775, ");": 1776, "Ġfac": 1777, "ĠSu": 1778, "Ġcond": 1779, "iting": 1780, "Ġcourse": 1781, "Ġ}": 1782, "----------------": 1783, "aign": 1784, "Ġevent": 1785, "Ġeng": 1786, "Ġpot": 1787, "Ġintern": 1788, "iam": 1789, "Ġshort": 1790, "empt": 1791, "ãĤ": 1792, "ĠGod": 1793, "ilar": 1794, "80": 1795, "Ġorig": 1796, "IS": 1797, "ourn": 1798, "ability": 1799, "itive": 1800, "Ġdam": 1801, "Ġ100": 1802, "Ġpress": 1803, "Ġdoing": 1804, "Ġprotect": 1805, "ring": 1806, "Ġthought": 1807, "Ġquestion": 1808, "rew": 1809, "ĠWar": 1810, "Ġseveral": 1811, "ĠState": 1812, "Ġgiven": 1813, "Ġfund": 1814, "ĠTw": 1815, "Ġwent": 1816, "ances": 1817, "work": 1818, "por": 1819, "my": 1820, "40": 1821, "Ġarg": 1822, "artment": 1823, "ustom": 1824, "Ġpolic": 1825, "Ġmeet": 1826, "Ġcreat": 1827, "22": 1828, "ĠStates": 1829, "Ġgames": 1830, "raw": 1831, "uture": 1832, "Ġunderstand": 1833, "urs": 1834, "ĠOb": 1835, "lish": 1836, "sy": 1837, "Ġmakes": 1838, "Ġwon": 1839, "agon": 1840, "Ġhtt": 1841, "Ġlove": 1842, "ential": 1843, "Ġcomplete": 1844, "par": 1845, "ĠIm": 1846, "AL": 1847, "Ġaccount": 1848, "Âł": 1849, "ored": 1850, "vert": 1851, "Ġident": 1852, "Ġ2015": 1853, "Ġothers": 1854, "ĠMin": 1855, "iber": 1856, "verage": 1857, "There": 1858, "itional": 1859, "dd": 1860, "Ġprob": 1861, "Ġyoung": 1862, "Ġalong": 1863, "Ġaccording": 1864, "Ġyet": 1865, "Ġmembers": 1866, "ĠWhat": 1867, "oid": 1868, "ĠMan": 1869, "And": 1870, "Ġamong": 1871, "ai": 1872, "Ġemploy": 1873, "ĠRes": 1874, "Ġ>": 1875, "Ġinvol": 1876, "Ġlow": 1877, "af": 1878, "ĠCar": 1879, "Ġhig": 1880, "ĠOne": 1881, "ĠSec": 1882, "ination": 1883, "Ġlikely": 1884, "Ġant": 1885, "aged": 1886, "ĠRuss": 1887, "Ġben": 1888, "Ġrele": 1889, "For": 1890, "back": 1891, "ĠNot": 1892, "Ġpresident": 1893, "ball": 1894, "Ġaccess": 1895, "ividual": 1896, "ĠDem": 1897, "ĠEuro": 1898, "60": 1899, "Ġknown": 1900, "irl": 1901, "ĠGr": 1902, "Ġearly": 1903, "use": 1904, "iety": 1905, "âĢĵ": 1906, "Ġfight": 1907, "Ġsent": 1908, "Ġtoday": 1909, "Ġmarket": 1910, "\".": 1911, "Ġbased": 1912, "Ġstrong": 1913, "urther": 1914, "Ġdeb": 1915, "mber": 1916, "Ġproblem": 1917, "Ġdeath": 1918, "Ġsocial": 1919, "imate": 1920, "AS": 1921, "ortun": 1922, "Ġcampaign": 1923, "ery": 1924, "Ch": 1925, "Ġey": 1926, "ially": 1927, "Ġmus": 1928, "wh": 1929, "pos": 1930, "Ġer": 1931, "Ġsaf": 1932, "Ġmonths": 1933, "iron": 1934, "Ġviol": 1935, "Ġfive": 1936, "Ġstre": 1937, "Ġplayers": 1938, "inc": 1939, "ald": 1940, "year": 1941, "aun": 1942, "Ġsuccess": 1943, "Ġpresent": 1944, "erence": 1945, "Ġ2014": 1946, "Ġsugg": 1947, "Ġparticular": 1948, "Ġtry": 1949, "Ġsuggest": 1950, "ĠChrist": 1951, "ones": 1952, "Ġpriv": 1953, "23": 1954, "Ġcrit": 1955, "Ġland": 1956, "Ġlocal": 1957, "ify": 1958, "29": 1959, "Ġaut": 1960, "ED": 1961, "ĠGu": 1962, "Ġmult": 1963, "Ġpolitical": 1964, "Ġasked": 1965, "Ġformer": 1966, "itter": 1967, "ript": 1968, "Ġclose": 1969, "Ġpract": 1970, "ĠYork": 1971, "Ġgetting": 1972, "Ġacross": 1973, "Ġcomb": 1974, "Ġbelieve": 1975, "Ġz": 1976, "Ġtoget": 1977, "Ġtogether": 1978, "ĠCent": 1979, "irc": 1980, "Ġindividual": 1981, "ĠMc": 1982, "27": 1983, "isk": 1984, "ĠEng": 1985, "Ġface": 1986, "Ġ24": 1987, "Ġvalue": 1988, "Ġarea": 1989, "ev": 1990, "Ġwrit": 1991, "ĠPresident": 1992, "Ġvot": 1993, "Ġkey": 1994, "Ġmom": 1995, "put": 1996, "Ġanything": 1997, "Ġexperience": 1998, "attle": 1999, "Ġmind": 2000, "aff": 2001, "omm": 2002, "Ġfuture": 2003, "ged": 2004, "Ġcut": 2005, "Ġtot": 2006, "itch": 2007, "Ġvideo": 2008, "Ġinvestig": 2009, "Ġnet": 2010, "ĠMy": 2011, "rict": 2012, "ien": 2013, ".)": 2014, "Ġimpro": 2015, "though": 2016, "wards": 2017, "Ġconnect": 2018, "ĠMed": 2019, "selves": 2020, "ensive": 2021, "mb": 2022, "ober": 2023, "ators": 2024, "An": 2025, "Ġ50": 2026, "Ġredu": 2027, "resent": 2028, "Ġabove": 2029, "Ġfre": 2030, "ĠEurope": 2031, "sw": 2032, "Ġamount": 2033, "ĠApp": 2034, "Ġeither": 2035, "Ġmilit": 2036, "Ġanal": 2037, "Ġfail": 2038, "ĠEn": 2039, "ales": 2040, "Ġspecial": 2041, "Ġblack": 2042, "IT": 2043, "cher": 2044, "Ġlooking": 2045, "Ġfire": 2046, "yn": 2047, "Ġalmost": 2048, "oon": 2049, "Ġstudy": 2050, "Ġmiss": 2051, "ches": 2052, "rown": 2053, "Ġtre": 2054, "Ġcommunity": 2055, "Ġmedia": 2056, "Ġfood": 2057, "Ġcomes": 2058, "ĠUniversity": 2059, "Ġsingle": 2060, "What": 2061, "uly": 2062, "Ġhalf": 2063, "ague": 2064, "hod": 2065, "ĠRepublic": 2066, "Ġstarted": 2067, "Ġquick": 2068, "oto": 2069, "book": 2070, "Ġissue": 2071, "itor": 2072, "Ġelse": 2073, "Ġconsider": 2074, "26": 2075, "rodu": 2076, "Ġtaken": 2077, "28": 2078, "99": 2079, "ĠWith": 2080, "Ġtrue": 2081, "Ġwa": 2082, "Ġtrad": 2083, "Ġago": 2084, "Ġmess": 2085, "ief": 2086, "Ġadded": 2087, "oke": 2088, "Ġbad": 2089, "Ġfav": 2090, "33": 2091, "Ġsimilar": 2092, "ask": 2093, "ĠDon": 2094, "Ġcharacter": 2095, "orts": 2096, "ĠHouse": 2097, "Ġreported": 2098, "Ġtype": 2099, "val": 2100, "iod": 2101, "ĠHowever": 2102, "Ġtarg": 2103, "Ġentire": 2104, "pping": 2105, "Ġhistory": 2106, "Ġlive": 2107, "ffic": 2108, "........": 2109, "ederal": 2110, "Ġtrying": 2111, "Ġdiscuss": 2112, "ĠHar": 2113, "aces": 2114, "lished": 2115, "Ġself": 2116, "osp": 2117, "rest": 2118, "Ġroom": 2119, "elt": 2120, "Ġfall": 2121, "olution": 2122, "Ġet": 2123, "Ġx": 2124, "Ġisn": 2125, "Ġidea": 2126, "bo": 2127, "Ġsound": 2128, "ĠDep": 2129, "Ġsomeone": 2130, "cially": 2131, "ully": 2132, "Ġfoc": 2133, "Ġobject": 2134, "ift": 2135, "aper": 2136, "Ġplayer": 2137, "Ġrather": 2138, "Ġservice": 2139, "ashing": 2140, "ĠDo": 2141, "ĠPart": 2142, "rug": 2143, "mon": 2144, "ply": 2145, "Ġmor": 2146, "Ġnothing": 2147, "Ġprovide": 2148, "IC": 2149, "ung": 2150, "Ġparty": 2151, "Ġexist": 2152, "Ġmag": 2153, "70": 2154, "Ġrul": 2155, "Ġhouse": 2156, "Ġbehind": 2157, "Ġhowever": 2158, "ĠWorld": 2159, "Ġsum": 2160, "Ġapplic": 2161, "Ġ;": 2162, "Ġfunction": 2163, "gr": 2164, "ĠPol": 2165, "Ġfront": 2166, "200": 2167, "Ġseries": 2168, "Ġtem": 2169, "Ġtyp": 2170, "ills": 2171, "Ġopt": 2172, "Ġpoints": 2173, "Ġbelow": 2174, "itted": 2175, "Ġspecific": 2176, "Ġ2017": 2177, "umb": 2178, "Ġra": 2179, "Ġprevious": 2180, "Ġpret": 2181, "reme": 2182, "Ġcustom": 2183, "Ġcourt": 2184, "ĠMe": 2185, "Ġrepl": 2186, "Ġwhole": 2187, "go": 2188, "cer": 2189, "Ġtreat": 2190, "ĠAct": 2191, "Ġprobably": 2192, "Ġlearn": 2193, "ender": 2194, "ĠAss": 2195, "Ġversion": 2196, "now": 2197, "Ġcheck": 2198, "ĠCal": 2199, "RE": 2200, "minist": 2201, "On": 2202, "ources": 2203, "Ġbenef": 2204, "Ġdoc": 2205, "Ġdeter": 2206, "Ġenc": 2207, "Ġsuper": 2208, "Ġaddress": 2209, "Ġvict": 2210, "Ġ2013": 2211, "Ġmeas": 2212, "tr": 2213, "Ġfield": 2214, "When": 2215, "Ġsignific": 2216, "uge": 2217, "Ġfeat": 2218, "Ġcommon": 2219, "load": 2220, "Ġbegin": 2221, "Ġbring": 2222, "Ġaction": 2223, "erman": 2224, "Ġdescrib": 2225, "Ġindust": 2226, "Ġwanted": 2227, "ried": 2228, "ming": 2229, "Ġattempt": 2230, "45": 2231, "fer": 2232, "Ġdue": 2233, "ression": 2234, "##": 2235, "Ġshall": 2236, "Ġsix": 2237, "oo": 2238, "Ġstep": 2239, "Ġpub": 2240, "Ġhimself": 2241, "Ġ23": 2242, "Ġcop": 2243, "Ġdest": 2244, "Ġstop": 2245, "AC": 2246, "ibility": 2247, "Ġlab": 2248, "icult": 2249, "Ġhours": 2250, "Ġcreate": 2251, "Ġfurther": 2252, "ĠAmerica": 2253, "ĠCity": 2254, "Ġdou": 2255, "head": 2256, "ST": 2257, "ĠNorth": 2258, "cing": 2259, "Ġnational": 2260, "ule": 2261, "ĠInst": 2262, "Ġtaking": 2263, "ĠQu": 2264, "irt": 2265, "Ġred": 2266, "Ġresearch": 2267, "viron": 2268, "ĠGe": 2269, "Ġbreak": 2270, "ana": 2271, "Ġspace": 2272, "aterial": 2273, "Ġrecent": 2274, "ĠAb": 2275, "Ġgeneral": 2276, "Ġhit": 2277, "Ġperiod": 2278, "Ġeverything": 2279, "ively": 2280, "Ġphys": 2281, "Ġsaying": 2282, "anks": 2283, "Ġcou": 2284, "Ġcult": 2285, "aced": 2286, "eal": 2287, "uation": 2288, "Ġcoun": 2289, "lu": 2290, "Ġinclude": 2291, "Ġposition": 2292, "ĠAfter": 2293, "ĠCanad": 2294, "ĠEm": 2295, "Ġimm": 2296, "ĠRed": 2297, "Ġpick": 2298, "Ġcompl": 2299, "Ġmatter": 2300, "reg": 2301, "ext": 2302, "angu": 2303, "isc": 2304, "ole": 2305, "aut": 2306, "Ġcompet": 2307, "eed": 2308, "fect": 2309, "Ġ21": 2310, "ĠSen": 2311, "ĠThese": 2312, "asing": 2313, "Ġcannot": 2314, "Ġinit": 2315, "Ġrelations": 2316, "ached": 2317, "Ġbar": 2318, "Ġ40": 2319, "ĠTH": 2320, "Ġ2012": 2321, "Ġvol": 2322, "Ġground": 2323, "Ġsecurity": 2324, "Ġupd": 2325, "ilt": 2326, "35": 2327, "Ġconcern": 2328, "ĠJust": 2329, "Ġwhite": 2330, "Ġseems": 2331, "ĠHer": 2332, "pecially": 2333, "ients": 2334, "Ġannoun": 2335, "Ġfig": 2336, "ights": 2337, "Ġstri": 2338, "like": 2339, "ids": 2340, "Ġsus": 2341, "Ġwatch": 2342, "Ġâ": 2343, "Ġwind": 2344, "ĠCont": 2345, "Ġitself": 2346, "Ġmass": 2347, "Al": 2348, "yle": 2349, "ique": 2350, "ĠNational": 2351, "Ġabs": 2352, "Ġpack": 2353, "Ġoutside": 2354, "Ġanim": 2355, "Ġpain": 2356, "eter": 2357, "Ġmanag": 2358, "duct": 2359, "ogn": 2360, "Ġ]": 2361, "ĠSept": 2362, "sec": 2363, "off": 2364, "ĠJan": 2365, "Ġfoot": 2366, "ades": 2367, "Ġthird": 2368, "Ġmot": 2369, "Ġevidence": 2370, "inton": 2371, "Ġthreat": 2372, "apt": 2373, "ples": 2374, "cle": 2375, "Ġlo": 2376, "Ġdecl": 2377, "Ġitem": 2378, "medi": 2379, "Ġrepresent": 2380, "omb": 2381, "amer": 2382, "Ġsignificant": 2383, "ograph": 2384, "su": 2385, "Ġcal": 2386, "ires": 2387, "0000": 2388, "ID": 2389, "AM": 2390, "Ġsimply": 2391, "Ġlonger": 2392, "Ġfile": 2393, "OT": 2394, "che": 2395, "So": 2396, "ateg": 2397, "org": 2398, "ĠHis": 2399, "Ġener": 2400, "Ġdom": 2401, "Ġupon": 2402, "ili": 2403, "\":\"": 2404, "Ġthemselves": 2405, "Ġcoming": 2406, "Ġquite": 2407, "Ġdifficult": 2408, "ĠBar": 2409, "ilities": 2410, "rel": 2411, "ends": 2412, "cial": 2413, "64": 2414, "Ġwoman": 2415, "rap": 2416, "yr": 2417, "Ġnecess": 2418, "ips": 2419, "Ġtext": 2420, "Ġrequire": 2421, "Ġmilitary": 2422, "Ġreview": 2423, "Ġrespons": 2424, "75": 2425, "Ġsubject": 2426, "Ġinstead": 2427, "Ġissues": 2428, "Ġgen": 2429, "\",\"": 2430, "Ġminutes": 2431, "Ġweap": 2432, "ray": 2433, "amed": 2434, "time": 2435, "bl": 2436, "How": 2437, "Ġcode": 2438, "ĠSm": 2439, "Ġhigher": 2440, "ĠSte": 2441, "ris": 2442, "Ġpage": 2443, "Ġstudents": 2444, "ĠIntern": 2445, "Ġmethod": 2446, "ĠAug": 2447, "ĠPer": 2448, "ĠAg": 2449, "Ġpolicy": 2450, "ĠSw": 2451, "Ġexec": 2452, "Ġaccept": 2453, "ume": 2454, "ribut": 2455, "Ġwords": 2456, "Ġfinal": 2457, "Ġchanges": 2458, "ĠDemocr": 2459, "Ġfriends": 2460, "Ġrespect": 2461, "Ġep": 2462, "Ġcompan": 2463, "ivil": 2464, "Ġdamage": 2465, "****": 2466, "ogle": 2467, "vironment": 2468, "Ġneg": 2469, "ental": 2470, "Ġap": 2471, "Ġtotal": 2472, "ival": 2473, "!\"": 2474, "lim": 2475, "Ġneeds": 2476, "Ġagre": 2477, "Ġdevelopment": 2478, "Ġage": 2479, "iple": 2480, "21": 2481, "Ġresults": 2482, "ĠAf": 2483, "Sh": 2484, "Ġgun": 2485, "ĠObama": 2486, "roll": 2487, "Ġ@": 2488, "Ġrights": 2489, "ĠBrit": 2490, "Ġrunning": 2491, "Ġwasn": 2492, "Ġport": 2493, "Ġrate": 2494, "Ġpretty": 2495, "Ġtarget": 2496, "Ġsaw": 2497, "Ġcirc": 2498, "Ġworks": 2499, "icro": 2500, "alt": 2501, "over": 2502, "www": 2503, "That": 2504, "lier": 2505, "Ġeveryone": 2506, "ude": 2507, "Ġpie": 2508, "iddle": 2509, "rael": 2510, "Ġrad": 2511, "Ġblock": 2512, "Ġwalk": 2513, "To": 2514, "ãģ": 2515, "nes": 2516, "ĠAust": 2517, "aul": 2518, "rote": 2519, "ĠSouth": 2520, "ession": 2521, "oph": 2522, "Ġshows": 2523, "Ġsite": 2524, "Ġjo": 2525, "Ġrisk": 2526, "clus": 2527, "lt": 2528, "Ġinj": 2529, "iding": 2530, "ĠSpe": 2531, "Ġchall": 2532, "irm": 2533, "Ġ22": 2534, "itting": 2535, "str": 2536, "Ġhy": 2537, "LE": 2538, "key": 2539, "Ġbegan": 2540, "atur": 2541, "ashington": 2542, "lam": 2543, "ĠDav": 2544, "bit": 2545, "Ġsize": 2546, "ĠPar": 2547, "38": 2548, "ournal": 2549, "face": 2550, "Ġdecision": 2551, "Ġlarg": 2552, "Ġjud": 2553, "rect": 2554, "Ġcontinue": 2555, "ĠOct": 2556, "overed": 2557, "ĠInt": 2558, "========": 2559, "Ġparent": 2560, "ĠWill": 2561, "Ġeasy": 2562, "Ġdrug": 2563, "anger": 2564, "Ġsense": 2565, "Ġdi": 2566, "iday": 2567, "Ġenergy": 2568, "istic": 2569, "Ġassoci": 2570, "arter": 2571, "obal": 2572, "eks": 2573, "ĠEl": 2574, "urch": 2575, "Ġgirl": 2576, "oe": 2577, "itle": 2578, "Ġ28": 2579, "ĠChe": 2580, "Ġrequest": 2581, "Ġsoon": 2582, "Ġhost": 2583, "ky": 2584, "Ġstates": 2585, "omes": 2586, "Ġmaterial": 2587, "lex": 2588, "Ġmoment": 2589, "Ġansw": 2590, "onse": 2591, "Ġespecially": 2592, "Ġnorm": 2593, "Ġservices": 2594, "pite": 2595, "ran": 2596, "Ġrole": 2597, "44": 2598, "):": 2599, "Ġcred": 2600, "Cl": 2601, "________": 2602, "Ġmat": 2603, "Ġlog": 2604, "ĠClinton": 2605, "OU": 2606, "Ġoffice": 2607, "Ġ26": 2608, "Ġcharg": 2609, "Ġtrack": 2610, "ma": 2611, "Ġheart": 2612, "Ġball": 2613, "Ġpersonal": 2614, "Ġbuilding": 2615, "na": 2616, "set": 2617, "body": 2618, "ĠBlack": 2619, "Ġincrease": 2620, "itten": 2621, "Ġneeded": 2622, "36": 2623, "32": 2624, "=\"": 2625, "Ġlost": 2626, "Ġbecame": 2627, "Ġgroups": 2628, "ĠMus": 2629, "Ġwrote": 2630, "ĠPe": 2631, "Ġprop": 2632, "joy": 2633, "é": 2634, "ĠWhite": 2635, "Ġdead": 2636, ".'": 2637, "Ġhttp": 2638, "Ġwebs": 2639, "OS": 2640, "Ġinside": 2641, "Ġwrong": 2642, "Ġstatement": 2643, "Ġ...": 2644, "yl": 2645, "Ġfilm": 2646, "Ġmusic": 2647, "Ġshare": 2648, "ification": 2649, "Ġrelease": 2650, "Ġforward": 2651, "Ġstay": 2652, "Ġcomput": 2653, "itte": 2654, "ser": 2655, "Ġoriginal": 2656, "Ġcard": 2657, "Ġcand": 2658, "Ġdiv": 2659, "atural": 2660, "Ġfavor": 2661, "OM": 2662, "Ġcases": 2663, "uses": 2664, "Ġsection": 2665, "Ġleave": 2666, "ging": 2667, "oved": 2668, "ĠWashington": 2669, "39": 2670, "ĠGl": 2671, "Ġrequired": 2672, "action": 2673, "apan": 2674, "oor": 2675, "iter": 2676, "ĠKing": 2677, "Ġcountries": 2678, "ĠGerman": 2679, "lling": 2680, "Ġ27": 2681, "34": 2682, "Ġquestions": 2683, "Ġprim": 2684, "Ġcell": 2685, "Ġshoot": 2686, "Ġanyone": 2687, "ĠWest": 2688, "Ġaffect": 2689, "epend": 2690, "Ġonline": 2691, "ĠIsrael": 2692, "ĠSeptember": 2693, "Ġability": 2694, "Ġcontent": 2695, "ises": 2696, "Ġreve": 2697, "Ġlaun": 2698, "Ġindic": 2699, "Ġforce": 2700, "cast": 2701, "Ġsold": 2702, "aving": 2703, "fl": 2704, "Ġsoft": 2705, "Ġcompanies": 2706, "ceed": 2707, "Ġarticle": 2708, "Ġaud": 2709, "Ġrev": 2710, "Ġeduc": 2711, "Ġplaying": 2712, "05": 2713, "Ġheld": 2714, "ctor": 2715, "Ġreleased": 2716, "Ġfederal": 2717, "37": 2718, "Ġadminist": 2719, "Ġinterview": 2720, "Ġinstall": 2721, "Ġreceived": 2722, "Ġsource": 2723, "uk": 2724, "Ph": 2725, "Ġserious": 2726, "Ġcreated": 2727, "Ġcause": 2728, "Ġimmedi": 2729, "Ġdefin": 2730, "uel": 2731, "ĠDepartment": 2732, "ctions": 2733, "ĠCour": 2734, "ĠNow": 2735, "ze": 2736, "ites": 2737, "itution": 2738, "Ġlate": 2739, "Ġspeak": 2740, "ners": 2741, "Ġlegal": 2742, "ari": 2743, "ĠCor": 2744, "Ġweeks": 2745, "Ġmodel": 2746, "Ġpred": 2747, "Ġexact": 2748, "BC": 2749, "ĠBy": 2750, "ING": 2751, "osing": 2752, "Ġtakes": 2753, "Ġregard": 2754, "Ġopportun": 2755, "Ġprice": 2756, "Ġ198": 2757, "ĠApr": 2758, "fully": 2759, "Ġord": 2760, "Ġproblems": 2761, "ruction": 2762, "ham": 2763, "ĠCount": 2764, "lege": 2765, "Ġleaders": 2766, "ET": 2767, "lev": 2768, "Ġdeep": 2769, "ological": 2770, "ese": 2771, "haps": 2772, "ĠSome": 2773, "Ġpers": 2774, "Ġcontract": 2775, "Ġrelationship": 2776, "sp": 2777, "oud": 2778, "Ġbase": 2779, "48": 2780, "mit": 2781, "Ad": 2782, "ancial": 2783, "Ġconsum": 2784, "Ġpotential": 2785, "Ġlangu": 2786, "rem": 2787, "eth": 2788, "Ġrelig": 2789, "ressed": 2790, "66": 2791, "Ġlink": 2792, "Ġlower": 2793, "ayer": 2794, "ĠJune": 2795, "Ġfem": 2796, "unt": 2797, "erc": 2798, "urd": 2799, "Ġcontact": 2800, "Ġill": 2801, "Ġmother": 2802, "Ġestab": 2803, "htt": 2804, "ĠMarch": 2805, "ĠBro": 2806, "ĠChina": 2807, "Ġ29": 2808, "Ġsqu": 2809, "Ġprovided": 2810, "Ġaverage": 2811, "asons": 2812, "Ġ2011": 2813, "Ġexam": 2814, "lin": 2815, "55": 2816, "ned": 2817, "Ġperfect": 2818, "Ġtou": 2819, "alse": 2820, "ux": 2821, "Ġbuy": 2822, "Ġshot": 2823, "Ġcollect": 2824, "Ġphot": 2825, "Ġplayed": 2826, "Ġsurpr": 2827, "Ġofficials": 2828, "Ġsimple": 2829, "avy": 2830, "Ġindustry": 2831, "Ġhands": 2832, "ground": 2833, "Ġpull": 2834, "Ġround": 2835, "Ġuser": 2836, "Ġrange": 2837, "uary": 2838, "Ġprivate": 2839, "ops": 2840, "ees": 2841, "Ġways": 2842, "ĠMich": 2843, "Ġveh": 2844, "Ġexcept": 2845, "Ġterms": 2846, "imum": 2847, "pper": 2848, "ION": 2849, "ores": 2850, "ĠDragon": 2851, "oul": 2852, "Ġden": 2853, "Ġperformance": 2854, "Ġbill": 2855, "cil": 2856, "47": 2857, "Ġenvironment": 2858, "Ġexc": 2859, "add": 2860, "Ġworth": 2861, "Ġpict": 2862, "Ġchance": 2863, "Ġ2018": 2864, "bor": 2865, "Ġspeed": 2866, "iction": 2867, "Ġalleg": 2868, "ĠJapan": 2869, "atory": 2870, "reet": 2871, "Ġmatch": 2872, "ĠII": 2873, "Ġstru": 2874, "order": 2875, "Ġste": 2876, "Ġliving": 2877, "Ġstruct": 2878, "ino": 2879, "Ġsepar": 2880, "hern": 2881, "Ġresponse": 2882, "Ġenjoy": 2883, "Ġvia": 2884, "AD": 2885, "uments": 2886, "acebook": 2887, "Ġmember": 2888, "ibr": 2889, "izing": 2890, "Ġtool": 2891, "ĠMon": 2892, "ĠWhile": 2893, "hood": 2894, "ĠAng": 2895, "ĠDef": 2896, "Ġoffer": 2897, "Tr": 2898, "aur": 2899, "Ġturned": 2900, "ĠJuly": 2901, "down": 2902, "anced": 2903, "Ġrecently": 2904, "ĠEar": 2905, "Ġce": 2906, "ĠStar": 2907, "ĠCong": 2908, "rought": 2909, "Ġblood": 2910, "Ġhope": 2911, "Ġcomment": 2912, "aint": 2913, "Ġarri": 2914, "iles": 2915, "Ġparticip": 2916, "ought": 2917, "ription": 2918, "08": 2919, "49": 2920, "Ġgave": 2921, "Ġselect": 2922, "Ġkilled": 2923, "sych": 2924, "Ġgoes": 2925, "ij": 2926, "Ġcoll": 2927, "Ġimpact": 2928, "atives": 2929, "ĠSer": 2930, "09": 2931, "ĠAugust": 2932, "Ġboy": 2933, "de": 2934, "ĠDes": 2935, "Ġfelt": 2936, "US": 2937, "Ġexpected": 2938, "Ġimage": 2939, "ĠMark": 2940, "ccording": 2941, "oice": 2942, "EC": 2943, "ĠMag": 2944, "ened": 2945, "hold": 2946, "ĠPost": 2947, "Ġprevent": 2948, "No": 2949, "Ġinvolved": 2950, "Ġeyes": 2951, "Ġquickly": 2952, "At": 2953, "unk": 2954, "Ġbehav": 2955, "Ġur": 2956, "Ġled": 2957, "come": 2958, "ey": 2959, "Ġcandid": 2960, "Ġearlier": 2961, "Ġfocus": 2962, "ety": 2963, "Pro": 2964, "ledge": 2965, "ixed": 2966, "illed": 2967, "Ġpopular": 2968, "AP": 2969, "Ġsett": 2970, "light": 2971, "Ġvarious": 2972, "inks": 2973, "Ġlevels": 2974, "Ġroad": 2975, "ellig": 2976, "ables": 2977, "hel": 2978, "ittee": 2979, "ĠGener": 2980, "ype": 2981, "Ġheard": 2982, "icles": 2983, "Ġmis": 2984, "Ġusers": 2985, "ĠSan": 2986, "Ġimprove": 2987, "Ġfather": 2988, "Ġsearch": 2989, "They": 2990, "vil": 2991, "Ġprofess": 2992, "Ġknew": 2993, "Ġloss": 2994, "Ġevents": 2995, "65": 2996, "Ġbillion": 2997, "07": 2998, "02": 2999, "ĠNews": 3000, "ĠAM": 3001, "Ġcover": 3002, "where": 3003, "ension": 3004, "Ġbott": 3005, "Ġareas": 3006, "ences": 3007, "ope": 3008, "ĠTwitter": 3009, "ael": 3010, "Ġgets": 3011, "ĠGoogle": 3012, "Ġsn": 3013, "iant": 3014, "Ġvote": 3015, "Ġnearly": 3016, "Ġincluded": 3017, "Ġrecogn": 3018, "zz": 3019, "mm": 3020, "aled": 3021, "Ġhappened": 3022, "04": 3023, "Ġhot": 3024, "Ġwhose": 3025, "Ġcivil": 3026, "Ġsuff": 3027, "oes": 3028, "itiz": 3029, "ĠSyri": 3030, "Ġrespond": 3031, "Ġhon": 3032, "Ġfeatures": 3033, "Ġeconomic": 3034, "ĠApril": 3035, "rim": 3036, "Ġtechnology": 3037, "Ġoption": 3038, "aging": 3039, "Ġpurch": 3040, "Re": 3041, "Ġlat": 3042, "chie": 3043, "isl": 3044, "Ġrecomm": 3045, "uf": 3046, "Ġtraining": 3047, "Ġeffects": 3048, "Ġfast": 3049, "Ġ2010": 3050, "Ġoccur": 3051, "Ġwebsite": 3052, "Ġemail": 3053, "Ġsens": 3054, "ech": 3055, "Ġoil": 3056, "Ġinflu": 3057, "Ġcurrently": 3058, "ĠSch": 3059, "ĠAdd": 3060, "Ġgoal": 3061, "Ġscient": 3062, "Ġconv": 3063, "100": 3064, "emy": 3065, "Ġdecided": 3066, "Ġtravel": 3067, "Ġmention": 3068, "LL": 3069, "03": 3070, "Ġelection": 3071, "Ġphone": 3072, "Ġlooks": 3073, "Ġsituation": 3074, "Ġcy": 3075, "Ġhor": 3076, "bed": 3077, "ĠCourt": 3078, "aily": 3079, "aves": 3080, "Ġquality": 3081, "ĠComp": 3082, "wise": 3083, "Ġtable": 3084, "Ġstaff": 3085, "ĠWind": 3086, "ett": 3087, "Ġtried": 3088, "idered": 3089, "Ġaddition": 3090, "Ġbox": 3091, "Ġlack": 3092, "arily": 3093, "Ġwide": 3094, "Ġmid": 3095, "Ġboard": 3096, "ysis": 3097, "Ġanti": 3098, "ha": 3099, "Ġdig": 3100, "ening": 3101, "Ġdro": 3102, "Con": 3103, "68": 3104, "Ġslow": 3105, "based": 3106, "sequ": 3107, "Ġpath": 3108, "Ex": 3109, "aker": 3110, "Ġworked": 3111, "Ġpen": 3112, "Ġengine": 3113, "Ġlooked": 3114, "ĠSuper": 3115, "ĠServ": 3116, "Ġvictim": 3117, "Un": 3118, "Ġproperty": 3119, "Ġintrodu": 3120, "Ġexecut": 3121, "ĠPM": 3122, "Le": 3123, "Ġcolor": 3124, "ĠMore": 3125, "Ġ60": 3126, "Ġnetwork": 3127, "Ġdate": 3128, "cul": 3129, "idge": 3130, "Ġextra": 3131, "31": 3132, "Ġsle": 3133, "67": 3134, "Ġwond": 3135, "Ġreports": 3136, "just": 3137, "ĠAustral": 3138, "Ġcapital": 3139, "Ġens": 3140, "Ġcommand": 3141, "Ġallowed": 3142, "Ġprep": 3143, "Ġcapt": 3144, "hib": 3145, "Ġnumbers": 3146, "chan": 3147, "Ġfair": 3148, "mp": 3149, "oms": 3150, "Ġreach": 3151, "With": 3152, "tain": 3153, "Ġbroad": 3154, "Ġcouple": 3155, "ecause": 3156, "lying": 3157, "ĠFeb": 3158, "Ġscreen": 3159, "Ġlives": 3160, "Ġprior": 3161, "ĠCongress": 3162, "Ar": 3163, "Ġapproach": 3164, "Ġemer": 3165, "aries": 3166, "ĠDis": 3167, "serv": 3168, "ĠNe": 3169, "Ġbuilt": 3170, "cies": 3171, "Ġrepe": 3172, "Ġrules": 3173, "force": 3174, "ĠPal": 3175, "Ġfinancial": 3176, "Ġconsidered": 3177, "ĠChar": 3178, "nces": 3179, "ĠIS": 3180, "Ġbrought": 3181, "Ġbi": 3182, "iers": 3183, "ĠSim": 3184, "OP": 3185, "Ġproducts": 3186, "Ġvisit": 3187, "Ġdocument": 3188, "Ġconduct": 3189, "Ġcompletely": 3190, "ining": 3191, "ĠCalif": 3192, "ibly": 3193, "Ġwritten": 3194, "ĠTV": 3195, "ements": 3196, "Ġdraw": 3197, "One": 3198, "Ġpublished": 3199, "Ġsecret": 3200, "rain": 3201, "het": 3202, "ĠFacebook": 3203, "onday": 3204, "ĠUp": 3205, "Ġsexual": 3206, "Ġthous": 3207, "ĠPat": 3208, "Ġess": 3209, "Ġstandard": 3210, "Ġarm": 3211, "ges": 3212, "ection": 3213, "Ġfell": 3214, "Ġforeign": 3215, "ani": 3216, "ĠFriday": 3217, "Ġregular": 3218, "inary": 3219, "Ġincreased": 3220, "Ġusually": 3221, "Ġdemon": 3222, "Ġdark": 3223, "Ġadditional": 3224, "rol": 3225, "ĠOf": 3226, "Ġproduction": 3227, "!!": 3228, "undred": 3229, "Ġinternational": 3230, "idents": 3231, "ĠFree": 3232, "roup": 3233, "Ġrace": 3234, "Ġmach": 3235, "Ġhuge": 3236, "All": 3237, "lear": 3238, "ovember": 3239, "Ġtown": 3240, "Ġattention": 3241, "ĠOff": 3242, "yond": 3243, "ĠThen": 3244, "field": 3245, "Ġterror": 3246, "raz": 3247, "ĠBo": 3248, "Ġmeeting": 3249, "ĠPark": 3250, "Ġarrest": 3251, "Ġfear": 3252, "Ġaw": 3253, "ĠVal": 3254, "oring": 3255, "',": 3256, "Ġextreme": 3257, "arr": 3258, "Ġworkers": 3259, "After": 3260, "Ġ31": 3261, "net": 3262, "ament": 3263, "Ġdirectly": 3264, "Ġpopulation": 3265, "ube": 3266, "ĠOctober": 3267, "ĠIN": 3268, "ĠJanuary": 3269, "59": 3270, "ĠDavid": 3271, "Ġcross": 3272, "cember": 3273, "ĠFirst": 3274, "Ġmessage": 3275, "irit": 3276, "Ġnation": 3277, "Ġpoll": 3278, "isions": 3279, "Ġanswer": 3280, "ny": 3281, "isode": 3282, "Ġcarry": 3283, "ĠRussia": 3284, "Ġhear": 3285, "ength": 3286, "roy": 3287, "Ġnatural": 3288, "inally": 3289, "Ġdog": 3290, "mitted": 3291, "Ġtrade": 3292, "Ġsubst": 3293, "Ġmultiple": 3294, "ĠAfric": 3295, "Ġfans": 3296, "Ġsort": 3297, "Ġglobal": 3298, "ication": 3299, "ĠWed": 3300, "ara": 3301, "Ġachie": 3302, "Ġlanguage": 3303, "vey": 3304, "Ġtal": 3305, "Ġnecessary": 3306, "Ġdetails": 3307, "Ġsen": 3308, "ĠSund": 3309, "ĠReg": 3310, "ĠRec": 3311, "06": 3312, "Ġsil": 3313, "ressive": 3314, "Ġmedical": 3315, "unch": 3316, "ornia": 3317, "Ġund": 3318, "fort": 3319, "ocks": 3320, "ĠMonday": 3321, "uesday": 3322, "craft": 3323, "77": 3324, "urt": 3325, "Ġver": 3326, "ĠHill": 3327, "Ġreceive": 3328, "Ġmorning": 3329, "estern": 3330, "Ġbank": 3331, "Ġsat": 3332, "irth": 3333, "ĠHigh": 3334, "Ġdevice": 3335, "ĠTHE": 3336, "ĠCenter": 3337, "Ġsafe": 3338, "Ġple": 3339, "ĠCanada": 3340, "Ġsystems": 3341, "Ġassist": 3342, "Ġsurv": 3343, "Ġbattle": 3344, "ĠSoc": 3345, "vertis": 3346, "She": 3347, "Ġpaper": 3348, "Ġgrowth": 3349, "Ġcast": 3350, "Sc": 3351, "Ġplans": 3352, "lled": 3353, "Ġparts": 3354, "Ġwall": 3355, "Ġmovement": 3356, "Ġpractice": 3357, "imately": 3358, "Ġdisplay": 3359, "Ġsometimes": 3360, "omp": 3361, "ĠPaul": 3362, "ĠYes": 3363, "king": 3364, "58": 3365, "oly": 3366, "Ġson": 3367, "Ġavoid": 3368, "okes": 3369, "ĠJew": 3370, "Ġtowards": 3371, "asc": 3372, "Ġ//": 3373, "ĠKore": 3374, "Ġtalking": 3375, "Ġcorrect": 3376, "Ġspent": 3377, "icks": 3378, "iable": 3379, "eared": 3380, "Ġterm": 3381, "Ġwants": 3382, "oming": 3383, "Ġut": 3384, "Ġdoub": 3385, "Ġforces": 3386, "Ġplease": 3387, "69": 3388, "ĠNovember": 3389, "atform": 3390, "ondon": 3391, "Ġones": 3392, "Ġimmediately": 3393, "ĠRussian": 3394, "ĠMet": 3395, "Ġdeg": 3396, "Ġparents": 3397, "CH": 3398, "ĠAmericans": 3399, "aly": 3400, "ĠMod": 3401, "Ġshown": 3402, "Ġconditions": 3403, "Ġstuff": 3404, "Ġreb": 3405, "ĠYour": 3406, "Ġincludes": 3407, "nown": 3408, "ĠSam": 3409, "Ġexperien": 3410, "mission": 3411, "ĠEven": 3412, "aught": 3413, "Ġannounced": 3414, "ĠRepublican": 3415, "Ġdetermin": 3416, "Ġdescribed": 3417, "ĠCounty": 3418, "()": 3419, "Ġdoor": 3420, "Ġchanged": 3421, "Ġneigh": 3422, "ĠHere": 3423, "Ġclean": 3424, "Ġpan": 3425, "ĠDecember": 3426, "ĠEuropean": 3427, "iring": 3428, "apter": 3429, "Ġclub": 3430, "ĠTuesday": 3431, "Ġpaid": 3432, "ĠNet": 3433, "Ġattacks": 3434, "Ġcharacters": 3435, "Ġalone": 3436, "Ġdirector": 3437, "dom": 3438, "Ġ35": 3439, "Ġload": 3440, "Ġrout": 3441, "ĠCalifornia": 3442, "Ġfinally": 3443, "Ġrac": 3444, "Ġcontr": 3445, "Ġexactly": 3446, "resh": 3447, "pri": 3448, "ĠIslam": 3449, "Ġnature": 3450, "Ġcareer": 3451, "Ġlatest": 3452, "Ġconvers": 3453, "ĠSl": 3454, "pose": 3455, "cient": 3456, "ĠInc": 3457, "ivity": 3458, "88": 3459, "ĠAtt": 3460, "ĠMor": 3461, "nesday": 3462, "Ġweight": 3463, "ken": 3464, "Ġnote": 3465, "Ġteams": 3466, "Ġ\\": 3467, "airs": 3468, "ĠGreen": 3469, "Ġhundred": 3470, "onent": 3471, "Ġstreng": 3472, "Ġconsist": 3473, "icated": 3474, "Ġregul": 3475, "Ġlic": 3476, "astic": 3477, "Ġten": 3478, "ursday": 3479, "elligence": 3480, "ously": 3481, "ĠUK": 3482, "BI": 3483, "Ġcosts": 3484, "Ġindepend": 3485, "ĠAP": 3486, "Ġnormal": 3487, "Ġhom": 3488, "Ġobvious": 3489, "Ġswe": 3490, "Ġstar": 3491, "Ġready": 3492, "acher": 3493, "Ġimplement": 3494, "gest": 3495, "Ġsong": 3496, "ĠGet": 3497, "ĠLab": 3498, "Ġinteresting": 3499, "using": 3500, "Ġgiving": 3501, "ĠSunday": 3502, "Ġetc": 3503, "Ġmiddle": 3504, "Ġremember": 3505, "right": 3506, "osition": 3507, "utions": 3508, "Ġmax": 3509, "46": 3510, "Ġyourself": 3511, "Ġdemand": 3512, "Ġtreatment": 3513, "Ġdanger": 3514, "ĠCons": 3515, "Ġguy": 3516, "ĠBritish": 3517, "Ġphysical": 3518, "Ġrelated": 3519, "Ġremain": 3520, "Ġcouldn": 3521, "Ġrefer": 3522, "Ġcitiz": 3523, "box": 3524, "ENT": 3525, "board": 3526, "Ġinn": 3527, "IG": 3528, "ero": 3529, "ĠStreet": 3530, "ospital": 3531, "rench": 3532, "chers": 3533, "Ġstra": 3534, "OL": 3535, "ager": 3536, "ĠAN": 3537, "Ġeasily": 3538, "IA": 3539, "enge": 3540, "iny": 3541, "Ġclos": 3542, "ocked": 3543, "Ġuses": 3544, "ĠCoun": 3545, "Im": 3546, "uild": 3547, "??": 3548, "more": 3549, "Ġang": 3550, "Ġwrite": 3551, "olute": 3552, "57": 3553, "Ġleader": 3554, "Ġreading": 3555, "": 3784, "Ġfigure": 3785, "Ġdisapp": 3786, "enty": 3787, "Ġsoftware": 3788, "Ġult": 3789, "Ġofficers": 3790, "New": 3791, "Is": 3792, "Ġremains": 3793, "ĠIndia": 3794, "Ġpsych": 3795, "rief": 3796, "Ġcat": 3797, "esc": 3798, "Ġobserv": 3799, "Ġstage": 3800, "ĠDark": 3801, "Ġenter": 3802, "change": 3803, "Ġpassed": 3804, "Ġdespite": 3805, "ĠOut": 3806, "Ġmovie": 3807, "rs": 3808, "Ġvoice": 3809, "mine": 3810, "ĠPlay": 3811, "Ġtoward": 3812, "ĠTer": 3813, "Ġregion": 3814, "Ġvalues": 3815, "orters": 3816, "Ġmount": 3817, "Ġofficer": 3818, "ĠOther": 3819, "ban": 3820, "Ġhous": 3821, "wood": 3822, "room": 3823, "IV": 3824, "ĠSun": 3825, "see": 3826, "ĠOver": 3827, "rog": 3828, "90": 3829, "Ġlay": 3830, "ĠTur": 3831, "awn": 3832, "Ġpressure": 3833, "ĠSub": 3834, "Ġbooks": 3835, "edom": 3836, "ĠSand": 3837, "AA": 3838, "ago": 3839, "Ġreasons": 3840, "ford": 3841, "Ġactivity": 3842, "UT": 3843, "Now": 3844, "ĠSenate": 3845, "cell": 3846, "night": 3847, "Ġcalls": 3848, "inter": 3849, "Ġletter": 3850, "ĠRob": 3851, "ĠJe": 3852, "Ġchoose": 3853, "ĠLaw": 3854, "Get": 3855, "Be": 3856, "Ġrob": 3857, "Ġtypes": 3858, "Ġplatform": 3859, "Ġquarter": 3860, "RA": 3861, "ĠTime": 3862, "Ġmaybe": 3863, "ĠCr": 3864, "95": 3865, "pre": 3866, "Ġmoving": 3867, "Ġlif": 3868, "Ġgold": 3869, "Ġsom": 3870, "Ġpatients": 3871, "Ġtruth": 3872, "ĠKe": 3873, "urance": 3874, "antly": 3875, "mar": 3876, "Ġcharge": 3877, "ĠGreat": 3878, "Ġcele": 3879, "--------------------------------": 3880, "Ġrock": 3881, "roid": 3882, "ancy": 3883, "Ġcredit": 3884, "aud": 3885, "By": 3886, "ĠEvery": 3887, "Ġmoved": 3888, "inger": 3889, "ribution": 3890, "Ġnames": 3891, "Ġstraight": 3892, "ĠHealth": 3893, "ĠWell": 3894, "Ġfeature": 3895, "Ġrule": 3896, "Ġsche": 3897, "inated": 3898, "ĠMichael": 3899, "berg": 3900, "41": 3901, "iled": 3902, "band": 3903, "Ġclick": 3904, "ĠAngel": 3905, "onents": 3906, "ÂŃ": 3907, "ĠIraq": 3908, "ĠSaturday": 3909, "Ġaware": 3910, "part": 3911, "Ġpattern": 3912, "OW": 3913, "ĠLet": 3914, "Ġgrad": 3915, "igned": 3916, "Ġassociated": 3917, "Ġstyle": 3918, "no": 3919, "iation": 3920, "aith": 3921, "ilies": 3922, "Ġstories": 3923, "uration": 3924, "Ġindividuals": 3925, "ĠâĢ¦": 3926, "miss": 3927, "ĠAssoci": 3928, "ishing": 3929, "aby": 3930, "Ġsummer": 3931, "ĠBen": 3932, "Ġ32": 3933, "Ġarch": 3934, "uty": 3935, "ĠTexas": 3936, "hol": 3937, "Ġfully": 3938, "Ġmill": 3939, "Ġfollowed": 3940, "ĠBill": 3941, "ĠIndian": 3942, "ĠSecret": 3943, "ĠBel": 3944, "ĠFebruary": 3945, "Ġjobs": 3946, "Ġseemed": 3947, "ĠGovern": 3948, "ipped": 3949, "Ġreality": 3950, "Ġlines": 3951, "Ġpark": 3952, "Ġmeasure": 3953, "ĠOur": 3954, "IM": 3955, "Ġbrother": 3956, "Ġgrowing": 3957, "Ġban": 3958, "Ġestim": 3959, "Ġcry": 3960, "ĠSchool": 3961, "Ġmechan": 3962, "ĠOF": 3963, "ĠWindows": 3964, "Ġrates": 3965, "ĠOh": 3966, "Ġpositive": 3967, "Ġculture": 3968, "istics": 3969, "ica": 3970, "Ġhar": 3971, "ya": 3972, "itely": 3973, "ipp": 3974, "Ġmap": 3975, "encies": 3976, "ĠWilliam": 3977, "II": 3978, "akers": 3979, "56": 3980, "ĠMart": 3981, "ĠRem": 3982, "Ġaltern": 3983, "itude": 3984, "Ġcoach": 3985, "rowd": 3986, "Don": 3987, "Ġkids": 3988, "Ġjournal": 3989, "Ġcorpor": 3990, "Ġfalse": 3991, "Ġweb": 3992, "Ġsleep": 3993, "Ġcontain": 3994, "Ġsto": 3995, "Ġbed": 3996, "iverse": 3997, "ĠRich": 3998, "ĠChinese": 3999, "Ġpun": 4000, "Ġmeant": 4001, "known": 4002, "Ġnotice": 4003, "Ġfavorite": 4004, "aven": 4005, "Ġcondition": 4006, "Ġpurpose": 4007, "))": 4008, "Ġorganization": 4009, "Ġchalleng": 4010, "Ġmanufact": 4011, "Ġsusp": 4012, "ĠAc": 4013, "Ġcritic": 4014, "unes": 4015, "uclear": 4016, "Ġmer": 4017, "vention": 4018, "Ġ80": 4019, "Ġmist": 4020, "ĠUs": 4021, "ĠTor": 4022, "http": 4023, "olf": 4024, "Ġlarger": 4025, "Ġadvant": 4026, "Ġresear": 4027, "Ġactions": 4028, "ml": 4029, "Ġkept": 4030, "Ġaim": 4031, ",'": 4032, "col": 4033, "Ġbenefits": 4034, "ifying": 4035, "Ġactual": 4036, "ĠInternational": 4037, "Ġvehicle": 4038, "Ġchief": 4039, "Ġefforts": 4040, "ĠLeague": 4041, "ĠMost": 4042, "Ġwait": 4043, "Ġadult": 4044, "Ġoverall": 4045, "Ġspeech": 4046, "Ġhighly": 4047, "Ġfemale": 4048, "Ġerror": 4049, "Ġeffective": 4050, "54": 4051, "Ġencour": 4052, "well": 4053, "Ġfailed": 4054, "Ġconserv": 4055, "Ġprograms": 4056, "Ġtrou": 4057, "Ġahead": 4058, "500": 4059, "vertisement": 4060, "IP": 4061, "ĠFound": 4062, "pir": 4063, "Ġ%": 4064, "Ġcrime": 4065, "ander": 4066, "Ġlocation": 4067, "ĠIran": 4068, "Ġbehavior": 4069, "azing": 4070, "Ġrare": 4071, "Ġemb": 4072, "Ġcaused": 4073, "Ġship": 4074, "Ġactive": 4075, "Ġcontribut": 4076, "Ġgreen": 4077, "Ġacqu": 4078, "Ġreflect": 4079, "venue": 4080, "Ġfirm": 4081, "Ġbirth": 4082, "].": 4083, "Ġclearly": 4084, "Ġemot": 4085, "Ġagency": 4086, "riage": 4087, "Ġmemory": 4088, "98": 4089, "SA": 4090, "ĠSee": 4091, "acing": 4092, "CC": 4093, "Ġbiggest": 4094, "Ġrap": 4095, "Ġbasic": 4096, "Ġband": 4097, "eat": 4098, "Ġsuspect": 4099, "ĠMac": 4100, "Ġ90": 4101, "mark": 4102, "istan": 4103, "Ġspread": 4104, "ams": 4105, "ki": 4106, "asy": 4107, "rav": 4108, "ĠRober": 4109, "Ġdemonstr": 4110, "rated": 4111, "Ġabsolute": 4112, "Ġplaces": 4113, "Ġimpl": 4114, "ibrary": 4115, "Ġcards": 4116, "Ġdestroy": 4117, "Ġvirt": 4118, "vere": 4119, "Ġappeared": 4120, "yan": 4121, "point": 4122, "Ġbeg": 4123, "Ġtemper": 4124, "spe": 4125, "anted": 4126, "ears": 4127, "ĠDirect": 4128, "Ġlength": 4129, "Ġblog": 4130, "amb": 4131, "Ġinteg": 4132, "Ġresources": 4133, "acc": 4134, "iful": 4135, "Ġspot": 4136, "Ġforced": 4137, "Ġthousands": 4138, "ĠMinister": 4139, "Ġqual": 4140, "ĠFrench": 4141, "atically": 4142, "Ġgenerally": 4143, "Ġdrink": 4144, "Ġthus": 4145, "IL": 4146, "odes": 4147, "Ġappropri": 4148, "ĠRead": 4149, "Ġwhom": 4150, "Ġeye": 4151, "Ġcollege": 4152, "Ġ45": 4153, "irection": 4154, "Ġensure": 4155, "Ġapparent": 4156, "iders": 4157, "Ġreligious": 4158, "Ġminor": 4159, "olic": 4160, "Ġtro": 4161, "ĠWhy": 4162, "ribute": 4163, "met": 4164, "Ġprimary": 4165, "Ġdeveloped": 4166, "Ġpeace": 4167, "Ġskin": 4168, "ste": 4169, "ava": 4170, "Ġblue": 4171, "Ġfamilies": 4172, "Ġir": 4173, "Ġapply": 4174, "Ġinform": 4175, "ĠSmith": 4176, "CT": 4177, "ii": 4178, "Ġlimit": 4179, "Ġresist": 4180, "................": 4181, "umn": 4182, "Ġconflic": 4183, "Ġtwe": 4184, "udd": 4185, "ĠTom": 4186, "Ġliter": 4187, "que": 4188, "bon": 4189, "Ġhair": 4190, "Ġeventually": 4191, "Ġpus": 4192, "Ġhelped": 4193, "Ġagg": 4194, "orney": 4195, "ĠApple": 4196, "Ġfit": 4197, "ĠSur": 4198, "Ġprem": 4199, "Ġsales": 4200, "Ġseconds": 4201, "Ġstrength": 4202, "Ġfeeling": 4203, "¿½": 4204, "Ġtour": 4205, "Ġknows": 4206, "oom": 4207, "Ġexerc": 4208, "Ġsomew": 4209, "�": 4210, ">>": 4211, "Ġspokes": 4212, "Ġideas": 4213, "Ġregist": 4214, "soft": 4215, "ĠDel": 4216, "ĠPC": 4217, "Ġpropos": 4218, "Ġlaunch": 4219, "Ġbottom": 4220, "TH": 4221, "ĠPlease": 4222, "vest": 4223, "itz": 4224, "ĠInter": 4225, "Ġscript": 4226, "Ġrat": 4227, "arning": 4228, "Ġil": 4229, "ĠJer": 4230, "ĠAre": 4231, "Ġwhatever": 4232, "oken": 4233, "cience": 4234, "Ġmode": 4235, "Ġagree": 4236, "Ġsources": 4237, "Ġinitial": 4238, "Ġrestrict": 4239, "Ġwonder": 4240, "usion": 4241, "####": 4242, "ĠSil": 4243, "ville": 4244, "Ġburn": 4245, "tw": 4246, "asion": 4247, "Ġ£": 4248, "Ġnor": 4249, "uing": 4250, "Ġreached": 4251, "Ġsun": 4252, "Ġcateg": 4253, "igration": 4254, "Ġcook": 4255, "Ġpromot": 4256, "Ġmale": 4257, "Ġclimate": 4258, "Ġfix": 4259, "Ġalleged": 4260, "UR": 4261, "alled": 4262, "Ġimages": 4263, "Cont": 4264, "ota": 4265, "Ġschools": 4266, "ios": 4267, "Ġdrop": 4268, "Ġstream": 4269, "ĠMo": 4270, "Ġpreviously": 4271, "aling": 4272, "Ġpet": 4273, "Ġdouble": 4274, "Ġ(@": 4275, "annel": 4276, "Ġdefault": 4277, "ties": 4278, "Ġrank": 4279, "ĠDec": 4280, "ĠCouncil": 4281, "Ġweapon": 4282, "Ġstock": 4283, "Ġanaly": 4284, "ĠStr": 4285, "Ġpicture": 4286, "ĠPolice": 4287, "ference": 4288, "Ġcentury": 4289, "Ġcitizens": 4290, "Ġonto": 4291, "Ġexpand": 4292, "Ġhero": 4293, "ĠSol": 4294, "Ġwild": 4295, "Ġupdate": 4296, "Ġcustomers": 4297, "ront": 4298, "def": 4299, "Ġlik": 4300, "Ġcriminal": 4301, "ĠChristian": 4302, "SP": 4303, "76": 4304, "Ġleaving": 4305, "Ġotherwise": 4306, "ĠDist": 4307, "Ġbasis": 4308, "52": 4309, "53": 4310, "icip": 4311, "ĠBer": 4312, "Ġrecommend": 4313, "Ġfloor": 4314, "Ġcrowd": 4315, "oles": 4316, "Ġ70": 4317, "Ġcentral": 4318, "ĠEv": 4319, "Ġdream": 4320, "Ġdownload": 4321, "Ġconfir": 4322, "ĠThom": 4323, "Ġwindow": 4324, "Ġhappens": 4325, "Ġunit": 4326, "Ġtend": 4327, "Ġspl": 4328, "Ġbecomes": 4329, "Ġfighting": 4330, "Ġpredict": 4331, "ĠPress": 4332, "ĠPower": 4333, "Ġheavy": 4334, "aked": 4335, "Ġfan": 4336, "orter": 4337, "ategy": 4338, "BA": 4339, "izes": 4340, "Ġspend": 4341, "Here": 4342, "Ġ2007": 4343, "Ġadop": 4344, "ĠHam": 4345, "Ġfootball": 4346, "ĠPort": 4347, "oday": 4348, "51": 4349, "ampions": 4350, "Ġtransfer": 4351, "ht": 4352, "Ġ38": 4353, "term": 4354, "acity": 4355, "Ġbur": 4356, "],": 4357, "ternal": 4358, "rig": 4359, "but": 4360, "Ġtherefore": 4361, "ĠBecause": 4362, "resp": 4363, "rey": 4364, "Ġmission": 4365, "Some": 4366, "Ġnoted": 4367, "Ġassum": 4368, "Ġdisease": 4369, "Ġedit": 4370, "Ġprogress": 4371, "rd": 4372, "ĠBrown": 4373, "ocal": 4374, "Ġadding": 4375, "Ġraised": 4376, "ĠAny": 4377, "Ġtick": 4378, "Ġseeing": 4379, "ĠPeople": 4380, "Ġagreement": 4381, "Ġserver": 4382, "Ġwat": 4383, "Ġdebate": 4384, "Ġsupposed": 4385, "iling": 4386, "Ġlargest": 4387, "Ġsuccessful": 4388, "ĠPri": 4389, "ĠDemocratic": 4390, "Ġjump": 4391, "ĠSyria": 4392, "Ġowners": 4393, "Ġoffers": 4394, "Ġshooting": 4395, "Ġeffic": 4396, "sey": 4397, "Ġhaven": 4398, "verse": 4399, "tered": 4400, "ĠLight": 4401, "imal": 4402, "ĠBig": 4403, "Ġdefend": 4404, "Ġbeat": 4405, "Ġrecords": 4406, "%)": 4407, "Ġscen": 4408, "Ġemployees": 4409, "Ġdevices": 4410, "hem": 4411, "Ġcommer": 4412, "ĠMex": 4413, "Ġbenefit": 4414, "ĠProf": 4415, "Ġilleg": 4416, "Ġsurface": 4417, "ĠAlso": 4418, "Ġharm": 4419, "ingly": 4420, "wide": 4421, "ĠAlex": 4422, "Ġshut": 4423, "ĠCur": 4424, "Ġlose": 4425, "pm": 4426, "Ġchallenge": 4427, "semb": 4428, "Ġstation": 4429, "Ġintelligence": 4430, "Ġaccur": 4431, "ĠFlor": 4432, "Ġrequires": 4433, "ĠMal": 4434, "bum": 4435, "Ġhospital": 4436, "Ġspirit": 4437, "Ġoffered": 4438, "Ġproduce": 4439, "ĠCommun": 4440, "Ġcreating": 4441, "Ġcris": 4442, "spect": 4443, "Ġended": 4444, "Ġdaily": 4445, "Ġvoters": 4446, "lands": 4447, "ias": 4448, "ih": 4449, "ona": 4450, "Ġsmart": 4451, "ĠOffice": 4452, "ĠLord": 4453, "rial": 4454, "ĠInternet": 4455, "Ġcircum": 4456, "Ġextremely": 4457, "'.": 4458, "Ġopinion": 4459, "ĠMil": 4460, "Ġgain": 4461, "BS": 4462, "ĠFin": 4463, "yp": 4464, "Ġuseful": 4465, "Ġbudget": 4466, "Ġcomfort": 4467, "isf": 4468, "Ġbackground": 4469, "eline": 4470, "Ġepisode": 4471, "Ġenemy": 4472, "Ġtrial": 4473, "Ġestablish": 4474, "date": 4475, "ĠCap": 4476, "Ġcontinues": 4477, "Ġshowing": 4478, "ĠUnion": 4479, "with": 4480, "Ġposted": 4481, "ĠSystem": 4482, "Ġeat": 4483, "rian": 4484, "Ġrise": 4485, "ĠGermany": 4486, "ils": 4487, "Ġsigned": 4488, "Ġvill": 4489, "Ġgrand": 4490, "mor": 4491, "ĠEngland": 4492, "Ġprojects": 4493, "umber": 4494, "Ġconference": 4495, "za": 4496, "Ġresponsible": 4497, "ĠArab": 4498, "Ġlearned": 4499, "âĢĶâĢĶ": 4500, "ipping": 4501, "ĠGeorge": 4502, "OC": 4503, "Ġreturned": 4504, "ĠAustralia": 4505, "Ġbrief": 4506, "Qu": 4507, "Ġbrand": 4508, "illing": 4509, "abled": 4510, "Ġhighest": 4511, "Ġtrain": 4512, "ĠCommission": 4513, "while": 4514, "Ġnom": 4515, "ception": 4516, "Ġmut": 4517, "ĠBlue": 4518, "Ġincident": 4519, "vant": 4520, "86": 4521, "ĠID": 4522, "Ġnuclear": 4523, "74": 4524, "ĠLike": 4525, "ĠRE": 4526, "ĠMicro": 4527, "li": 4528, "mail": 4529, "Ġcharges": 4530, "89": 4531, "Ġadjust": 4532, "ado": 4533, "Ġearth": 4534, "NA": 4535, "Ġprices": 4536, "PA": 4537, "Ġdraft": 4538, "Ġruns": 4539, "Ġcandidate": 4540, "enses": 4541, "Ġmanagement": 4542, "ĠPhil": 4543, "ĠMiss": 4544, "Ġteach": 4545, "gram": 4546, "Ġunderstanding": 4547, "ait": 4548, "icago": 4549, "Add": 4550, "ĠEp": 4551, "secut": 4552, "Ġseparate": 4553, "Ġinstance": 4554, "Ġeth": 4555, "Ġunless": 4556, "********": 4557, "ĠFore": 4558, "inate": 4559, "Ġoperations": 4560, "Sp": 4561, "Ġfaith": 4562, "gar": 4563, "ĠChurch": 4564, "ronic": 4565, "Ġconfig": 4566, "osure": 4567, "Ġactivities": 4568, "Ġtraditional": 4569, "Ġ36": 4570, "Ġdirection": 4571, "Ġmachine": 4572, "Ġsurround": 4573, "Ġpush": 4574, "unction": 4575, "ĠEU": 4576, "Ġeasier": 4577, "Ġargument": 4578, "GB": 4579, "Ġmicro": 4580, "Ġspending": 4581, "izations": 4582, "Ġtheory": 4583, "adow": 4584, "Ġcalling": 4585, "ĠLast": 4586, "Ġder": 4587, "Ġinfluence": 4588, "Ġcommit": 4589, "Ġphoto": 4590, "Ġunc": 4591, "istry": 4592, "gn": 4593, "aste": 4594, "acks": 4595, "Ġdisp": 4596, "ady": 4597, "do": 4598, "ĠGood": 4599, "Ġ`": 4600, "Ġwish": 4601, "Ġrevealed": 4602, "³³": 4603, "lig": 4604, "Ġenforce": 4605, "ĠCommittee": 4606, "Ġchem": 4607, "Ġmiles": 4608, "Ġinterested": 4609, "Ġsolution": 4610, "icy": 4611, "inct": 4612, "Ġ->": 4613, "ĠDet": 4614, "Ġremoved": 4615, "Ġcompar": 4616, "eah": 4617, "Ġplant": 4618, "ĠSince": 4619, "Ġachieve": 4620, "Ġadvantage": 4621, "Ġslightly": 4622, "bing": 4623, "Ġplaced": 4624, "under": 4625, "2015": 4626, "ĠMad": 4627, "Ġtim": 4628, "oses": 4629, "Ġcru": 4630, "ĠRock": 4631, "Ġmostly": 4632, "Ġnegative": 4633, "Ġsetting": 4634, "Ġproduced": 4635, "Ġmur": 4636, "Ġconnection": 4637, "ĠMer": 4638, "Ġdriver": 4639, "Ġexecutive": 4640, "Ġassault": 4641, "Ġborn": 4642, "ĠVer": 4643, "tained": 4644, "Ġstructure": 4645, "Ġreduce": 4646, "Ġdecades": 4647, "Ġded": 4648, "uke": 4649, "ĠMany": 4650, "idden": 4651, "Ġleague": 4652, "Se": 4653, "Ġjoin": 4654, "Ġdisco": 4655, "Ġdie": 4656, "cks": 4657, "actions": 4658, "Ġassess": 4659, "agn": 4660, "Ġgoals": 4661, "ours": 4662, "IR": 4663, "Ġsenior": 4664, "iller": 4665, "mod": 4666, "ipment": 4667, "ocol": 4668, "uy": 4669, "ĠQue": 4670, "Ġparties": 4671, "irgin": 4672, "Ġlearning": 4673, "itable": 4674, "Ġstreet": 4675, "Ġcamera": 4676, "App": 4677, "Ġskills": 4678, "bre": 4679, "cious": 4680, "Ġcelebr": 4681, "ĠFranc": 4682, "Ġexisting": 4683, "Ġwilling": 4684, "lor": 4685, "Ġid": 4686, "ĠSpace": 4687, "Ġcritical": 4688, "ĠLa": 4689, "ortunately": 4690, "Ġserve": 4691, "Ġcold": 4692, "Ġspecies": 4693, "TS": 4694, "Ġanimals": 4695, "ĠBay": 4696, "Ġolder": 4697, "ĠUnder": 4698, "estic": 4699, "ĠTre": 4700, "Ġteacher": 4701, "Ġprefer": 4702, "vis": 4703, "Ġthread": 4704, "ĠMatt": 4705, "Ġmanager": 4706, "ãĥ»": 4707, "Ġprofessional": 4708, "ĠVol": 4709, "Ġnotes": 4710, "These": 4711, "ula": 4712, "Ġfresh": 4713, "ented": 4714, "uzz": 4715, "edy": 4716, "clusion": 4717, "ĠRel": 4718, "Ġdoubt": 4719, "EO": 4720, "Ġopened": 4721, "ĠBit": 4722, "Advertisement": 4723, "Ġguess": 4724, "ĠUN": 4725, "Ġsequ": 4726, "Ġexplain": 4727, "otten": 4728, "Ġattract": 4729, "aks": 4730, "Ġstring": 4731, "Ġcontext": 4732, "ossible": 4733, "ĠRepublicans": 4734, "Ġsolid": 4735, "Ġcities": 4736, "Ġasking": 4737, "Ġrandom": 4738, "ups": 4739, "uries": 4740, "arant": 4741, "dden": 4742, "gl": 4743, "ĠFlorida": 4744, "Ġdepend": 4745, "ĠScott": 4746, "Ġ33": 4747, "ĠiT": 4748, "icon": 4749, "Ġmentioned": 4750, "Ġ2000": 4751, "Ġclaimed": 4752, "Ġdefinitely": 4753, "ulf": 4754, "Ġcore": 4755, "Ġopening": 4756, "ĠConst": 4757, "which": 4758, "ĠTra": 4759, "AG": 4760, "72": 4761, "Ġbelieved": 4762, "ada": 4763, "Ġ48": 4764, "ĠSecurity": 4765, "yright": 4766, "ĠPet": 4767, "ĠLou": 4768, "Ġholding": 4769, "================": 4770, "Ġice": 4771, "Ġbrow": 4772, "Ġauthorities": 4773, "host": 4774, "word": 4775, "Ġscore": 4776, "ĠDiv": 4777, "Ġcells": 4778, "Ġtransl": 4779, "Ġneighbor": 4780, "Ġremove": 4781, "uct": 4782, "Ġdistrict": 4783, "ĠAccording": 4784, "Ġworse": 4785, "Ġconcerns": 4786, "Ġpresidential": 4787, "Ġpolicies": 4788, "ĠHall": 4789, "73": 4790, "Ġhus": 4791, "AY": 4792, "Ġ2006": 4793, "ĠJud": 4794, "Ġindependent": 4795, "ĠJustice": 4796, "iliar": 4797, "print": 4798, "ighter": 4799, "Ġprotection": 4800, "zen": 4801, "Ġsudden": 4802, "house": 4803, "ĠJes": 4804, "PR": 4805, "ĠInf": 4806, "Ġbul": 4807, "Ġ_": 4808, "ĠService": 4809, "ĠPR": 4810, "Ġstrategy": 4811, "ffect": 4812, "Ġgirls": 4813, "Ġmissing": 4814, "oyal": 4815, "ĠTeam": 4816, "ulated": 4817, "Ġdat": 4818, "Ġpolitics": 4819, "abor": 4820, "According": 4821, "Ġspell": 4822, "Ġgraph": 4823, "orthern": 4824, "TC": 4825, "Ab": 4826, "Ġlabor": 4827, "isher": 4828, "Ġkick": 4829, "ĠiTunes": 4830, "Ġsteps": 4831, "poses": 4832, "Ġsmaller": 4833, "En": 4834, "bert": 4835, "Ġroll": 4836, "Ġresearchers": 4837, "Ġclosed": 4838, "Ġtransport": 4839, "Ġlawy": 4840, "________________": 4841, "ĠChicago": 4842, "Ġaspect": 4843, "Ġnone": 4844, "Ġmarriage": 4845, "96": 4846, "Ġelements": 4847, "ĠFre": 4848, "ĠSal": 4849, "Ġdram": 4850, "FC": 4851, "top": 4852, "equ": 4853, "Ġhearing": 4854, "Ġsupported": 4855, "Ġtesting": 4856, "cohol": 4857, "Ġmassive": 4858, "Ġstick": 4859, "Ġguard": 4860, "isco": 4861, "phone": 4862, "From": 4863, "However": 4864, "Ġborder": 4865, "Ġcopy": 4866, "ography": 4867, "list": 4868, "71": 4869, "Ġowner": 4870, "class": 4871, "ruit": 4872, "rate": 4873, "ĠOnce": 4874, "Ġdigital": 4875, "Ġtask": 4876, "ERS": 4877, "Ġincred": 4878, "tes": 4879, "++": 4880, "ĠFrance": 4881, "Ġbreat": 4882, "owl": 4883, "Ġissued": 4884, "ĠWestern": 4885, "Ġdetect": 4886, "Ġpartners": 4887, "Ġshared": 4888, "ĠCall": 4889, "Ġcancer": 4890, "ache": 4891, "ribe": 4892, "Ġexplained": 4893, "Ġheat": 4894, "{\"": 4895, "Ġinvestment": 4896, "ĠBook": 4897, "Ġwood": 4898, "Ġtools": 4899, "ĠAlthough": 4900, "Ġbelief": 4901, "Ġcrisis": 4902, "Ġge": 4903, "ĠMP": 4904, "Ġoperation": 4905, "type": 4906, "~~": 4907, "ga": 4908, "Ġcontains": 4909, "anta": 4910, "Ġexpress": 4911, "ĠGroup": 4912, "ĠJournal": 4913, "ka": 4914, "Ġamb": 4915, "ĠUSA": 4916, "Ġfinding": 4917, "Ġfunding": 4918, "how": 4919, "Ġestablished": 4920, "ideos": 4921, "Ġdegree": 4922, "Ġdangerous": 4923, "anging": 4924, "Ġfreedom": 4925, "pport": 4926, "outhern": 4927, "Ġchurch": 4928, "Ġcatch": 4929, "ĠTwo": 4930, "Ġpresence": 4931, "ĠGuard": 4932, "Up": 4933, "Ġauthority": 4934, "ĠProject": 4935, "Ġbutton": 4936, "Ġconsequ": 4937, "Ġvalid": 4938, "Ġweak": 4939, "Ġstarts": 4940, "Ġreference": 4941, "ĠMem": 4942, "\")": 4943, "UN": 4944, "orage": 4945, "ĠOpen": 4946, "Ġcollection": 4947, "ym": 4948, "gency": 4949, "Ġbeautiful": 4950, "ros": 4951, "Ġtells": 4952, "Ġwaiting": 4953, "nel": 4954, "Ġproviding": 4955, "ĠDemocrats": 4956, "Ġdaughter": 4957, "Ġmaster": 4958, "Ġpurposes": 4959, "ĠJapanese": 4960, "Ġequal": 4961, "Ġturns": 4962, "Ġdocuments": 4963, "Ġwatching": 4964, "Res": 4965, "Ġran": 4966, "2014": 4967, "Ġreject": 4968, "ĠKorea": 4969, "Ġvictims": 4970, "Level": 4971, "erences": 4972, "Ġwitness": 4973, "Ġ34": 4974, "Ġreform": 4975, "coming": 4976, "Ġoccup": 4977, "Ġcaught": 4978, "Ġtraffic": 4979, "ading": 4980, "Ġmodels": 4981, "ario": 4982, "Ġserved": 4983, "Ġbatter": 4984, "uate": 4985, "ĠSecretary": 4986, "Ġagreed": 4987, "Ġtruly": 4988, "ynam": 4989, "ĠRet": 4990, "Ġunits": 4991, "ĠResearch": 4992, "hand": 4993, "azine": 4994, "ĠMike": 4995, "Ġvariety": 4996, "otal": 4997, "Ġamazing": 4998, "Ġconfirmed": 4999, "Ġentirely": 5000, "Ġpurchase": 5001, "Ġelement": 5002, "Ġcash": 5003, "Ġdetermine": 5004, "De": 5005, "Ġcars": 5006, "ĠWall": 5007, "âĸ": 5008, "Ġviews": 5009, "Ġdrugs": 5010, "Ġdepartment": 5011, "ĠStep": 5012, "uit": 5013, "Ġ39": 5014, "asure": 5015, "ĠClass": 5016, "Ġcovered": 5017, "ĠBank": 5018, "Ġmere": 5019, "uana": 5020, "Ġmulti": 5021, "Ġmix": 5022, "Ġunlike": 5023, "levision": 5024, "Ġstopped": 5025, "Ġsem": 5026, "ĠGal": 5027, "ules": 5028, "Ġwel": 5029, "ĠJohnson": 5030, "la": 5031, "Ġskill": 5032, "Ġbecoming": 5033, "rie": 5034, "Ġappropriate": 5035, "fe": 5036, "ellow": 5037, "ĠProt": 5038, "ulate": 5039, "ocation": 5040, "Ġweekend": 5041, "odies": 5042, "Ġsites": 5043, "Ġanimal": 5044, "ĠTim": 5045, "Ġscale": 5046, "Ġcharged": 5047, "Ġinstruct": 5048, "illa": 5049, "Ġmethods": 5050, "Ġcert": 5051, "Ġjudge": 5052, "ĠHel": 5053, "Ġdollars": 5054, "Ġstanding": 5055, "ĠSqu": 5056, "Ġdebt": 5057, "liam": 5058, "Ġdriving": 5059, "ĠSum": 5060, "ĠEdition": 5061, "Ġalbum": 5062, "andon": 5063, "IF": 5064, "ĠUk": 5065, "63": 5066, "ader": 5067, "Ġcommercial": 5068, "esh": 5069, "ĠGovernment": 5070, "Ġdiscovered": 5071, "Ġoutput": 5072, "ĠHillary": 5073, "ĠCarol": 5074, "Ġ2005": 5075, "Ġabuse": 5076, "ancing": 5077, "Ġswitch": 5078, "Ġannual": 5079, "Tw": 5080, "Ġstated": 5081, "agement": 5082, "inner": 5083, "Ġdemocr": 5084, "Ġresidents": 5085, "Ġallowing": 5086, "Ġfactors": 5087, "odd": 5088, "Ġfuck": 5089, "emies": 5090, "Ġoccurred": 5091, "oti": 5092, "Ġnorth": 5093, "ĠPublic": 5094, "Ġinjury": 5095, "Ġinsurance": 5096, "CL": 5097, "olly": 5098, "ãĢ": 5099, "Ġrepeated": 5100, "Ġarms": 5101, "anged": 5102, "Ġconstruction": 5103, "Ġfle": 5104, "PU": 5105, "icians": 5106, "Ġforms": 5107, "ĠMcC": 5108, "antic": 5109, "Ġmental": 5110, "pire": 5111, "Ġequipment": 5112, "Ġfant": 5113, "Ġdiscussion": 5114, "Ġregarding": 5115, "kin": 5116, "arp": 5117, "Ġchair": 5118, "ogue": 5119, "Ġproceed": 5120, "ĠId": 5121, "Our": 5122, "Ġmurder": 5123, "Man": 5124, "Ġ49": 5125, "asp": 5126, "Ġsupply": 5127, "Ġinput": 5128, "Ġwealth": 5129, "liament": 5130, "Ġproced": 5131, "orial": 5132, "ĠStat": 5133, "ĠNFL": 5134, "hens": 5135, "ĠInstitute": 5136, "Ġputting": 5137, "ournament": 5138, "etic": 5139, "Ġlocated": 5140, "Ġkid": 5141, "eria": 5142, "run": 5143, "Ġprinc": 5144, "Ġ!": 5145, "going": 5146, "ĠBet": 5147, "Ġclot": 5148, "Ġtelling": 5149, "Ġproposed": 5150, "iot": 5151, "orry": 5152, "Ġfunds": 5153, "gment": 5154, "ĠLife": 5155, "Ġbaby": 5156, "ĠBack": 5157, "Ġspoke": 5158, "Image": 5159, "Ġearn": 5160, "ĠAT": 5161, "gu": 5162, "Ġexchange": 5163, "ĠLin": 5164, "oving": 5165, "Ġpair": 5166, "More": 5167, "azon": 5168, "Ġarrested": 5169, "Ġkilling": 5170, "can": 5171, "ĠCard": 5172, "yd": 5173, "Ġidentified": 5174, "Ġmobile": 5175, "Ġthanks": 5176, "onym": 5177, "ĠForm": 5178, "Ġhundreds": 5179, "ĠChris": 5180, "ĠCat": 5181, "Ġtrend": 5182, "hat": 5183, "ĠAv": 5184, "oman": 5185, "Ġelectric": 5186, "ĠWil": 5187, "SE": 5188, "Of": 5189, "Ġrestaur": 5190, "oted": 5191, "Ġtrig": 5192, "Ġnine": 5193, "Ġbomb": 5194, "Why": 5195, "¯": 5196, "Ġcoverage": 5197, "Ġappeal": 5198, "ĠRobert": 5199, "ĠSup": 5200, "Ġfinished": 5201, "Ġflow": 5202, "Ġdeliver": 5203, "Ġcalcul": 5204, "Ġphotos": 5205, "Ġphil": 5206, "Ġpieces": 5207, "Ġappre": 5208, "kes": 5209, "Ġrough": 5210, "Do": 5211, "Ġpartner": 5212, "Ġconcerned": 5213, "Ġ37": 5214, "ĠGen": 5215, "Col": 5216, "ctors": 5217, "Ġ=>": 5218, "state": 5219, "Ġsuggested": 5220, "ĠForce": 5221, "CE": 5222, "Ġherself": 5223, "ĠPlan": 5224, "works": 5225, "ooth": 5226, "rency": 5227, "Ġcorner": 5228, "Ġhusband": 5229, "Ġinternet": 5230, "ĠAut": 5231, "ems": 5232, "osen": 5233, "ĠAtl": 5234, "gen": 5235, "Ġbalance": 5236, "62": 5237, "Ġsounds": 5238, "text": 5239, "Ġarr": 5240, "oves": 5241, "Ġmillions": 5242, "Ġradio": 5243, "Ġsatisf": 5244, "ĠDam": 5245, "Mr": 5246, "Go": 5247, "Spe": 5248, "Ġcombat": 5249, "rant": 5250, "ĠGree": 5251, "Ġfuel": 5252, "Ġdistance": 5253, "Ġtests": 5254, "Ġdecre": 5255, "ĠEr": 5256, "Ġmanaged": 5257, "DS": 5258, "Ġtit": 5259, "Ġmeasures": 5260, "ĠLiber": 5261, "Ġattend": 5262, "ashed": 5263, "ĠJose": 5264, "ĠNight": 5265, "dit": 5266, "ĠNov": 5267, "ĠEnd": 5268, "outs": 5269, "Ġgeneration": 5270, "Ġadvoc": 5271, "yth": 5272, "Ġconversation": 5273, "ĠSky": 5274, "active": 5275, "cel": 5276, "rier": 5277, "ĠFrank": 5278, "Ġgender": 5279, "Ġconcent": 5280, "Ġcarried": 5281, "anda": 5282, "ĠVirgin": 5283, "Ġarrived": 5284, "icide": 5285, "aded": 5286, "Ġfailure": 5287, "Ġminimum": 5288, "lets": 5289, "Ġworst": 5290, "Ġkeeping": 5291, "Ġintended": 5292, "Ġillegal": 5293, "Ġsubsc": 5294, "Ġdetermined": 5295, "Ġtrip": 5296, "Yes": 5297, "Ġraise": 5298, "Ġ~": 5299, "Ġfeels": 5300, "Ġpackage": 5301, "ĠJo": 5302, "hi": 5303, "2016": 5304, "real": 5305, "Ġfra": 5306, "Ġsymb": 5307, "Me": 5308, "ucky": 5309, "pret": 5310, "ĠKh": 5311, "ĠEdit": 5312, "ĠWeb": 5313, "emic": 5314, "ĠColor": 5315, "Ġjustice": 5316, "Int": 5317, "Ġfarm": 5318, "cknow": 5319, "\">": 5320, "eless": 5321, "Ġreduced": 5322, "Ġ500": 5323, "xx": 5324, "ĠRad": 5325, "ĠWood": 5326, "Ġclin": 5327, "Ġhyp": 5328, "iler": 5329, "ura": 5330, "kins": 5331, "85": 5332, "61": 5333, "ĠTheir": 5334, "ĠMary": 5335, "Ġsan": 5336, "Ġnovel": 5337, "ĠWho": 5338, "Ġcapacity": 5339, "Ġimpossible": 5340, "Ġplays": 5341, "Ġminister": 5342, "ijuana": 5343, "icate": 5344, "ĠSet": 5345, "Ġfram": 5346, "Ġing": 5347, "Ġcommunities": 5348, "ĠFBI": 5349, "ita": 5350, "Ġbon": 5351, "Ġstrateg": 5352, "Ġinterests": 5353, "lock": 5354, "gers": 5355, "mas": 5356, "ĠAND": 5357, "Ġconflict": 5358, "Ġrequirements": 5359, "Ġsac": 5360, "Ġoperating": 5361, "ini": 5362, "related": 5363, "Ġcommitted": 5364, "Ġrelatively": 5365, "Ġsouth": 5366, "¯¯": 5367, "Ġafford": 5368, "Ġidentity": 5369, "Ġdecisions": 5370, "Ġaccused": 5371, "place": 5372, "Ġvictory": 5373, "och": 5374, "iat": 5375, "Name": 5376, "Com": 5377, "tion": 5378, "eds": 5379, "Ġseek": 5380, "Ġtight": 5381, "ĠImages": 5382, "Ġiniti": 5383, "Ġhumans": 5384, "Ġfamiliar": 5385, "Ġaudience": 5386, "Ġinternal": 5387, "venture": 5388, "Ġsides": 5389, "ĠTO": 5390, "Ġdim": 5391, "Ġconclud": 5392, "Ġappoint": 5393, "Ġenforcement": 5394, "ĠJim": 5395, "ĠAssociation": 5396, "Ġcircumst": 5397, "ĠCanadian": 5398, "Ġjoined": 5399, "Ġdifferences": 5400, "ĠLos": 5401, "Ġprotest": 5402, "Ġtwice": 5403, "win": 5404, "Ġglass": 5405, "arsh": 5406, "ĠArmy": 5407, "Ġexpression": 5408, "Ġdecide": 5409, "Ġplanning": 5410, "ania": 5411, "Ġhandle": 5412, "ĠMicrosoft": 5413, "ĠNor": 5414, "Ġmaximum": 5415, "ĠRev": 5416, "Ġsea": 5417, "Ġeval": 5418, "Ġhelps": 5419, "ref": 5420, "Ġbound": 5421, "Ġmouth": 5422, "Ġstandards": 5423, "Ġclim": 5424, "ĠCamp": 5425, "ĠFox": 5426, "cles": 5427, "Ġarmy": 5428, "ĠTechn": 5429, "acking": 5430, "xy": 5431, "SS": 5432, "Ġ42": 5433, "Ġbug": 5434, "ĠUkrain": 5435, "ĠMax": 5436, "ĠJones": 5437, "ĠShow": 5438, "lo": 5439, "Ġplanet": 5440, "Ġ75": 5441, "Ġwinning": 5442, "Ġfaster": 5443, "Ġspect": 5444, "Ġbroken": 5445, "TR": 5446, "Ġdefined": 5447, "Ġhealthy": 5448, "Ġcompetition": 5449, "https": 5450, "ĠIsland": 5451, "ĠFe": 5452, "Ġannounce": 5453, "ĠCup": 5454, "ĠInstead": 5455, "Ġclient": 5456, "Ġpossibly": 5457, "section": 5458, "ocket": 5459, "look": 5460, "Ġfinish": 5461, "Ġcrew": 5462, "Ġreserv": 5463, "Ġeditor": 5464, "Ġhate": 5465, "Ġsale": 5466, "Ġcontrovers": 5467, "Ġpages": 5468, "wing": 5469, "Ġnumer": 5470, "Ġopposition": 5471, "Ġ2004": 5472, "Ġrefuge": 5473, "Ġflight": 5474, "Ġapart": 5475, "ĠLat": 5476, "Americ": 5477, "ĠAfrica": 5478, "Ġapplications": 5479, "ĠPalest": 5480, "ĠBur": 5481, "Ġgar": 5482, "ĠSocial": 5483, "Ġupgr": 5484, "Ġshape": 5485, "Ġspeaking": 5486, "ansion": 5487, "ao": 5488, "ĠSn": 5489, "Ġworry": 5490, "ĠBritain": 5491, "Please": 5492, "roud": 5493, "Ġhun": 5494, "Ġintroduced": 5495, "Ġdiet": 5496, "Ind": 5497, "ĠSecond": 5498, "Ġfunctions": 5499, "uts": 5500, "ĠEach": 5501, "ĠJeff": 5502, "Ġstress": 5503, "Ġaccounts": 5504, "Ġguarant": 5505, "ĠAnn": 5506, "edia": 5507, "Ġhonest": 5508, "Ġtree": 5509, "ĠAfrican": 5510, "ĠBush": 5511, "},": 5512, "Ġsch": 5513, "ĠOnly": 5514, "Ġfif": 5515, "igan": 5516, "Ġexercise": 5517, "ĠExp": 5518, "Ġscientists": 5519, "Ġlegislation": 5520, "ĠWork": 5521, "ĠSpr": 5522, "ÃĤ": 5523, "ĠHuman": 5524, "Ġè": 5525, "Ġsurvey": 5526, "Ġrich": 5527, "rip": 5528, "Ġmaintain": 5529, "Ġflo": 5530, "Ġleadership": 5531, "stream": 5532, "ĠIslamic": 5533, "Ġ01": 5534, "ĠCollege": 5535, "Ġmagic": 5536, "ĠPrime": 5537, "Ġfigures": 5538, "2017": 5539, "inder": 5540, "xual": 5541, "ĠDead": 5542, "Ġabsolutely": 5543, "Ġfourth": 5544, "Ġpresented": 5545, "respond": 5546, "rible": 5547, "Ġalcohol": 5548, "ato": 5549, "ĠDE": 5550, "porary": 5551, "Ġgrab": 5552, "Ġvari": 5553, "Ġquant": 5554, "ĠPhoto": 5555, "Ġplus": 5556, "rick": 5557, "arks": 5558, "Ġalternative": 5559, "Ġpil": 5560, "Ġapprox": 5561, "that": 5562, "Ġobjects": 5563, "ĠRo": 5564, "ĠAndroid": 5565, "Ġsignificantly": 5566, "ĠRoad": 5567, "kay": 5568, "Read": 5569, "avor": 5570, "Ġacknow": 5571, "ĠHD": 5572, "ĠSing": 5573, "Or": 5574, "ĠMont": 5575, "Ġuns": 5576, "prof": 5577, "Ġnegoti": 5578, "ĠArch": 5579, "iki": 5580, "Ġtelevision": 5581, "ĠJewish": 5582, "Ġcommittee": 5583, "Ġmotor": 5584, "Ġappearance": 5585, "Ġsitting": 5586, "Ġstrike": 5587, "ĠDown": 5588, "comp": 5589, "ĠHist": 5590, "Ġfold": 5591, "acement": 5592, "ĠLouis": 5593, "Ġbelong": 5594, "ĠâĢ¢": 5595, "Ġmort": 5596, "Ġprepared": 5597, "Ġ64": 5598, "ĠMaster": 5599, "Ġindeed": 5600, "ĠDen": 5601, "Ġrent": 5602, "TA": 5603, "ourney": 5604, "arc": 5605, "Su": 5606, "97": 5607, "Ġadvice": 5608, "Ġchanging": 5609, "Ġlisted": 5610, "Ġlaunched": 5611, "isation": 5612, "ĠPeter": 5613, "ishes": 5614, "Ġlived": 5615, "ĠMel": 5616, "ĠSupreme": 5617, "ĠFederal": 5618, "Ġ);": 5619, "ructure": 5620, "Ġsets": 5621, "Ġphilos": 5622, "uous": 5623, "ĠÂł": 5624, "Ġapplied": 5625, "ĠNOT": 5626, "Ġhousing": 5627, "ĠMount": 5628, "Ġodd": 5629, "Ġsust": 5630, "DA": 5631, "fficient": 5632, "Ġ?": 5633, "olved": 5634, "Ġpowers": 5635, "Ġthr": 5636, "Ġremaining": 5637, "ĠWater": 5638, "LC": 5639, "Ġcauses": 5640, "ãģ®": 5641, "Ġmanner": 5642, "ads": 5643, "Ġsuggests": 5644, "Ġends": 5645, "standing": 5646, "fig": 5647, "ĠDun": 5648, "idth": 5649, "Ġgay": 5650, "Ġtermin": 5651, "ĠAngeles": 5652, "MS": 5653, "Ġscientific": 5654, "Ġcoal": 5655, "apers": 5656, "bar": 5657, "ĠThomas": 5658, "Ġsym": 5659, "ĠRun": 5660, "this": 5661, "PC": 5662, "igrants": 5663, "Ġminute": 5664, "ĠDistrict": 5665, "cellent": 5666, "Ġleaves": 5667, "Ġcompleted": 5668, "amin": 5669, "Ġfocused": 5670, "Ġmonitor": 5671, "Ġvehicles": 5672, "MA": 5673, "ĠMass": 5674, "ĠGrand": 5675, "Ġaffected": 5676, "itutional": 5677, "Ġconstruct": 5678, "Ġfollows": 5679, "Ġton": 5680, "reens": 5681, "Ġhomes": 5682, "ĠExt": 5683, "ĠLevel": 5684, "rast": 5685, "ĠIr": 5686, "Ġelim": 5687, "Ġlargely": 5688, "ĠJoe": 5689, "Ġvotes": 5690, "alls": 5691, "Ġbusinesses": 5692, "ĠFoundation": 5693, "ĠCentral": 5694, "Ġyards": 5695, "Ġmaterials": 5696, "ulner": 5697, "Ġguide": 5698, "Ġcloser": 5699, "ums": 5700, "Ġsports": 5701, "eder": 5702, "Just": 5703, "Ġtaxes": 5704, "84": 5705, "ĠOld": 5706, "Ġdecade": 5707, "ola": 5708, "Ġvir": 5709, "Ġdropped": 5710, "Ġdelay": 5711, "itect": 5712, "Ġsecure": 5713, "stein": 5714, "level": 5715, "Ġtreated": 5716, "Ġfiled": 5717, "aine": 5718, "Ġvan": 5719, "Ġmir": 5720, "Ġcolumn": 5721, "icted": 5722, "eper": 5723, "Ġrot": 5724, "Ġconsult": 5725, "Ġentry": 5726, "Ġmarijuana": 5727, "ĠDou": 5728, "Ġapparently": 5729, "oking": 5730, "clusive": 5731, "Ġincreases": 5732, "ano": 5733, "Ġspecifically": 5734, "Ġtele": 5735, "ensions": 5736, "Ġreligion": 5737, "abilities": 5738, "Ġframe": 5739, "ĠNote": 5740, "ĠLee": 5741, "Ġhelping": 5742, "Ġedge": 5743, "oston": 5744, "Ġorganizations": 5745, "Ãĥ": 5746, "ĠBoth": 5747, "hips": 5748, "Ġbigger": 5749, "Ġboost": 5750, "ĠStand": 5751, "Ġrow": 5752, "uls": 5753, "abase": 5754, "Ġrid": 5755, "Let": 5756, "aren": 5757, "rave": 5758, "Ġstret": 5759, "PD": 5760, "Ġvision": 5761, "Ġwearing": 5762, "Ġappreci": 5763, "Ġaward": 5764, "ĠUse": 5765, "Ġfactor": 5766, "war": 5767, "ulations": 5768, ")(": 5769, "Ġgod": 5770, "Ġterrit": 5771, "Ġparam": 5772, "asts": 5773, "87": 5774, "Ġenemies": 5775, "ĠGames": 5776, "FF": 5777, "Ġaccident": 5778, "Well": 5779, "ĠMartin": 5780, "TER": 5781, "Ġath": 5782, "ĠHell": 5783, "Ġforg": 5784, "Ġveter": 5785, "ĠMedic": 5786, "free": 5787, "Ġstars": 5788, "Ġexpensive": 5789, "Ġacad": 5790, "rawn": 5791, "ĠWhe": 5792, "Ġlock": 5793, "Ġformat": 5794, "Ġsoldiers": 5795, "sm": 5796, "Ġagent": 5797, "Ġresponsibility": 5798, "ora": 5799, "ĠScience": 5800, "Ġrapid": 5801, "Ġtough": 5802, "ĠJesus": 5803, "Ġbelieves": 5804, "ML": 5805, "Ġwear": 5806, "lete": 5807, "ÃĥÃĤ": 5808, "ĠDri": 5809, "Ġcommission": 5810, "ĠBob": 5811, "Oh": 5812, "aped": 5813, "Ġwarm": 5814, "ÃĥÃĤÃĥÃĤ": 5815, "Ġ2003": 5816, "ortion": 5817, "Ġhasn": 5818, "uster": 5819, "Ġunivers": 5820, "ĠIll": 5821, "Ġking": 5822, "ologies": 5823, "94": 5824, "ĠTem": 5825, "ĠMos": 5826, "Ġpatient": 5827, "ĠMexico": 5828, "cean": 5829, "ĠDeath": 5830, "ĠSanders": 5831, "you": 5832, "ĠCast": 5833, "ĠCompany": 5834, "pty": 5835, "Ġhappening": 5836, "FP": 5837, "ĠBattle": 5838, "Ġbought": 5839, "Am": 5840, "Mod": 5841, "Us": 5842, "uters": 5843, "ĠCre": 5844, "ĠThose": 5845, "Ġ44": 5846, "iser": 5847, "Ġsoul": 5848, "ĠTop": 5849, "ĠHarry": 5850, "ĠAw": 5851, "Ġseat": 5852, "ffee": 5853, "Ġrevolution": 5854, "Ġ(\"": 5855, "ĠDuring": 5856, "ette": 5857, "Ġring": 5858, "Ġoffensive": 5859, "Ġreturns": 5860, "Ġvideos": 5861, "Ġdiscl": 5862, "Ġfamous": 5863, "enced": 5864, "ĠSign": 5865, "ĠRiver": 5866, "Ġ300": 5867, "PM": 5868, "ĠBus": 5869, "ĠCH": 5870, "Ġcandidates": 5871, "arden": 5872, "Ġpercentage": 5873, "Ġvisual": 5874, "Ġthank": 5875, "Ġtrouble": 5876, "nergy": 5877, "Ġ2001": 5878, "Ġprove": 5879, "ashion": 5880, "Ġenh": 5881, "ĠLong": 5882, "UM": 5883, "Ġconnected": 5884, "Ġpossibility": 5885, "Over": 5886, "Ġexpert": 5887, "Ġlibrary": 5888, "arts": 5889, "ĠDirector": 5890, "Ġfellow": 5891, "92": 5892, "irty": 5893, "Ġdry": 5894, "Ġsigns": 5895, "ĠLove": 5896, "Ġquiet": 5897, "foot": 5898, "Ġpure": 5899, "ĠHun": 5900, "Ġfilled": 5901, "phas": 5902, "ĠElect": 5903, "endment": 5904, "ĠExpl": 5905, "Ġunable": 5906, "ns": 5907, "mo": 5908, "Ġvast": 5909, "obe": 5910, "Ġidentify": 5911, "apping": 5912, "ĠCarolina": 5913, "gress": 5914, "Ġprote": 5915, "Ġfish": 5916, "Ġcircumstances": 5917, "razy": 5918, "ĠPhot": 5919, "Ġbodies": 5920, "ĠMur": 5921, "Ġdeveloping": 5922, "ĠAR": 5923, "Ġexperienced": 5924, "Ġsubstant": 5925, "ĠBoard": 5926, "esome": 5927, "Ġdomestic": 5928, "Ġcombined": 5929, "ĠPut": 5930, "Ġchemical": 5931, "ĠChild": 5932, "Ġpool": 5933, "ĠCy": 5934, "Ġegg": 5935, "cons": 5936, "sters": 5937, "Ġhurt": 5938, "Ġmarkets": 5939, "Ġconservative": 5940, "Ġsupporters": 5941, "Ġagencies": 5942, "idel": 5943, "Ob": 5944, "urb": 5945, "Ġ43": 5946, "ĠDefense": 5947, "ye": 5948, "ĠAp": 5949, "dule": 5950, "Ġtemperature": 5951, "Ġconducted": 5952, "ĠChief": 5953, "Ġpulled": 5954, "Ġfol": 5955, "Last": 5956, "onto": 5957, "osis": 5958, "VER": 5959, "Des": 5960, "ĠPan": 5961, "First": 5962, "Ġadvance": 5963, "Ġlicense": 5964, "rors": 5965, "ĠJon": 5966, "Ġimagine": 5967, "Ġhell": 5968, "Ġfixed": 5969, "Ġincor": 5970, "osite": 5971, "ĠLog": 5972, "icken": 5973, "]:": 5974, "Ġsurprise": 5975, "hab": 5976, "Ġcraft": 5977, "olt": 5978, "ĠJul": 5979, "Ġdial": 5980, "Ġrelevant": 5981, "Ġentered": 5982, "Ġleads": 5983, "ĠAD": 5984, "ĠClean": 5985, "Ġpictures": 5986, "essor": 5987, "Ġalt": 5988, "Ġpaying": 5989, "Per": 5990, "ĠMarket": 5991, "Ġupdates": 5992, "amily": 5993, "ĠType": 5994, "ĠHome": 5995, "Ġ55": 5996, "sembly": 5997, "rome": 5998, "83": 5999, "Ġgreatest": 6000, "Ġheight": 6001, "Ġheav": 6002, "aints": 6003, "Ġlisten": 6004, "aser": 6005, "ĠSH": 6006, "Ġcapable": 6007, "acle": 6008, "Ġperspect": 6009, "inating": 6010, "Ġoffering": 6011, "rypt": 6012, "ĠDevelop": 6013, "abin": 6014, "rc": 6015, "Ġbright": 6016, "alty": 6017, "arrow": 6018, "Ġsuppl": 6019, "inding": 6020, "acked": 6021, "gypt": 6022, "ĠAnother": 6023, "pg": 6024, "ĠVirginia": 6025, "ĠLu": 6026, "Ġplanned": 6027, "Ġpit": 6028, "Ġsweet": 6029, "Type": 6030, "ĠDi": 6031, "Ġtypically": 6032, "ĠFrancisco": 6033, "Ġprospect": 6034, "ĠDan": 6035, "Ġteen": 6036, "rees": 6037, "Ġsched": 6038, "Ġhol": 6039, "Ġscr": 6040, "Ġlots": 6041, "life": 6042, "Ġnewsp": 6043, "Ġforget": 6044, "ĠNone": 6045, "ĠMiddle": 6046, "ĠRyan": 6047, "edd": 6048, "Ġsevere": 6049, "Ġsuit": 6050, "ller": 6051, "93": 6052, "Ġcorrespond": 6053, "Ġexplos": 6054, "uations": 6055, "Ġflag": 6056, "game": 6057, "rid": 6058, "Ġprin": 6059, "ĠData": 6060, "Ġdeploy": 6061, "ĠEnter": 6062, "suit": 6063, "ghan": 6064, "ĠMen": 6065, "Ġthoughts": 6066, "Ġmatters": 6067, "Ġadapt": 6068, "ĠAri": 6069, "Ġfill": 6070, "Ġforth": 6071, "Ġsam": 6072, "Ġ41": 6073, "Ġpayment": 6074, "ĠHor": 6075, "Ġspring": 6076, "duc": 6077, "Ġlosing": 6078, "Ġbringing": 6079, "FO": 6080, "ala": 6081, "Ġdistribution": 6082, "hered": 6083, "bour": 6084, "ĠIsraeli": 6085, "oma": 6086, "Ġcombination": 6087, "Ġplenty": 6088, "VE": 6089, "Can": 6090, "ĠHaw": 6091, "Ġperman": 6092, "ĠSpecial": 6093, "Ġtow": 6094, "Ġseeking": 6095, "Ġexamples": 6096, "Ġclasses": 6097, "cr": 6098, "Ġbeer": 6099, "Ġmoves": 6100, "ĠIP": 6101, "ĠKn": 6102, "Ġpanel": 6103, "Even": 6104, "Ġproperly": 6105, "Ġris": 6106, "Ġplug": 6107, "Ġestimated": 6108, "Every": 6109, "Ġdefensive": 6110, "agraph": 6111, "Ġpregn": 6112, "Ġinstit": 6113, "ĠVict": 6114, "Ġvolume": 6115, "Ġpositions": 6116, "Ġlinks": 6117, "ĠProgram": 6118, "ĠWeek": 6119, "agues": 6120, "Ġtransform": 6121, "ker": 6122, "ĠCEO": 6123, "Ġcas": 6124, "Ġopponent": 6125, "Ġtweet": 6126, "ĠCode": 6127, "Ġshop": 6128, "Ġfly": 6129, "Ġtalks": 6130, "Ġbag": 6131, "Phone": 6132, "Ġaid": 6133, "Ġplants": 6134, "Ġ65": 6135, "Ġattorney": 6136, "arters": 6137, "quest": 6138, "ĠMagic": 6139, "Ġbegins": 6140, "Ġmyster": 6141, "Ġenvironmental": 6142, "Ġstorage": 6143, "NN": 6144, "Ġmarg": 6145, "Ġske": 6146, "Ġmetal": 6147, "elly": 6148, "Ġordered": 6149, "Ġremained": 6150, "Ġloved": 6151, "Ġprompt": 6152, "Ġupdated": 6153, "Ġexperts": 6154, "Ġwalking": 6155, "Ġancient": 6156, "Ġperformed": 6157, "ATE": 6158, "Ġneither": 6159, "iency": 6160, "Ġmanufacture": 6161, "ĠPak": 6162, "Ġselected": 6163, "Ġmine": 6164, "Ġultimately": 6165, "Ġexplan": 6166, "Ġlabel": 6167, "ĠServices": 6168, "ributed": 6169, "Trump": 6170, "Ġsyn": 6171, "ĠUlt": 6172, "SC": 6173, "Ġmeat": 6174, "Ġgiant": 6175, "ĠWars": 6176, "ĠON": 6177, "Ġadm": 6178, "Ġinterpret": 6179, "Ġevening": 6180, "Ġevil": 6181, "ĠBoston": 6182, "ĠWild": 6183, "ĠÃ": 6184, "ĠBitcoin": 6185, "ĠAmazon": 6186, "Dr": 6187, "ĠInformation": 6188, "Ġobviously": 6189, "Ġadvanced": 6190, "Photo": 6191, "olar": 6192, "Ġweather": 6193, "Ġsymbol": 6194, "Ġsole": 6195, "Ġpotentially": 6196, "oster": 6197, "Ġoriginally": 6198, "mun": 6199, "300": 6200, "aze": 6201, "essions": 6202, "Ġdeck": 6203, "Ġstood": 6204, "Ġyouth": 6205, "ĠBern": 6206, "Rep": 6207, "ĠTest": 6208, "Ġbasically": 6209, "otic": 6210, "Ġinvolve": 6211, "olit": 6212, "lyn": 6213, "See": 6214, "Ġaircraft": 6215, "Ġconfirm": 6216, "EW": 6217, "Ġmessages": 6218, "ĠRichard": 6219, "Ġkit": 6220, "Ġprohib": 6221, "Ġvulner": 6222, "isters": 6223, "Ġexistence": 6224, "Ġturning": 6225, "ĠSP": 6226, "Ġdesire": 6227, "Ġflat": 6228, "Ġment": 6229, "season": 6230, "anges": 6231, "Ġneighborhood": 6232, "ĠLake": 6233, "ATION": 6234, "Ġpointed": 6235, "bur": 6236, "Ġinnov": 6237, "ucks": 6238, "UL": 6239, "Ġprofessor": 6240, "Ġexpressed": 6241, "AB": 6242, "icious": 6243, "Ġ2002": 6244, "ĠDev": 6245, "Ġsession": 6246, "Ġbare": 6247, "sen": 6248, "Ġdiss": 6249, "ĠCath": 6250, "ĠPass": 6251, "ĠPoint": 6252, "Ġdoctor": 6253, "orrow": 6254, "ailed": 6255, "ĠRub": 6256, "ĠDC": 6257, "ĠCharl": 6258, "person": 6259, "Ġwriter": 6260, "ighters": 6261, "ureau": 6262, "Ġoblig": 6263, "Ġrecorded": 6264, "Ġbroke": 6265, "Ġorders": 6266, "ilty": 6267, "Ġmotion": 6268, "inity": 6269, "law": 6270, "adium": 6271, "Ġimmigration": 6272, "Ġcontrast": 6273, "Ġbatt": 6274, "Ġexcellent": 6275, "Ġtechnical": 6276, "ami": 6277, "Ġtun": 6278, "Ġcloud": 6279, "ĠYear": 6280, "geon": 6281, "Ġcreation": 6282, "Ġstrange": 6283, "Ġauth": 6284, "Ġfort": 6285, "born": 6286, "Ġextent": 6287, "ĠToday": 6288, "ĠClub": 6289, "Ġrain": 6290, "Ġsample": 6291, "Ġaccepted": 6292, "Ġtact": 6293, "Ġfired": 6294, "ĠSon": 6295, "Ġstands": 6296, "Ġboot": 6297, "Ġ47": 6298, "Ġstatements": 6299, "Ġversions": 6300, "Ġselling": 6301, "ounded": 6302, "Ġ1990": 6303, "Ġweren": 6304, "ĠWatch": 6305, "Ġexperiment": 6306, "Post": 6307, "Ġretail": 6308, "uled": 6309, "Inst": 6310, "unte": 6311, "ãĥ¼": 6312, "Ġdepart": 6313, "Ġbond": 6314, "ivery": 6315, "ompl": 6316, "Ġreaction": 6317, "ĠSyrian": 6318, "ĠPac": 6319, "apped": 6320, "aniel": 6321, "DP": 6322, "Ġresolution": 6323, "Ġreact": 6324, "Ġapproved": 6325, "onom": 6326, "mond": 6327, "ĠOffic": 6328, "---": 6329, "Ġreplace": 6330, "Ġtack": 6331, "Ġsport": 6332, "Ġchain": 6333, "Ġemergency": 6334, "rad": 6335, "ĠPalestin": 6336, "Ġ46": 6337, "Ġautomatically": 6338, "Ġroute": 6339, "Ġpal": 6340, "Ġbanks": 6341, "ĠParis": 6342, "ĠMedia": 6343, "road": 6344, "icing": 6345, "ixt": 6346, "isted": 6347, "Ġgrew": 6348, "Ġcoord": 6349, "ĠWhere": 6350, "omin": 6351, "Ġsubs": 6352, "��": 6353, "Ġ±": 6354, "Ġcorporate": 6355, "Ġselection": 6356, "noon": 6357, "ĠReport": 6358, "cs": 6359, "cluding": 6360, "orders": 6361, "anche": 6362, "ĠIts": 6363, "Ġslowly": 6364, "ĠEgypt": 6365, "ĠAcc": 6366, "Ġcolle": 6367, "iques": 6368, "EX": 6369, "Ġattempts": 6370, "url": 6371, "ĠCross": 6372, "Ġfindings": 6373, "ĠSC": 6374, "ĠOR": 6375, "Ġindex": 6376, "ensity": 6377, "ĠWay": 6378, "ĠLand": 6379, "Ġshock": 6380, "dis": 6381, "Ġdynam": 6382, "Ġcart": 6383, "mosp": 6384, "Since": 6385, "iest": 6386, "ĠBoy": 6387, "Ġstorm": 6388, "ĠContin": 6389, "2013": 6390, "hew": 6391, "ilit": 6392, "Ġessential": 6393, "iquid": 6394, "Other": 6395, "ivered": 6396, "Ġreasonable": 6397, "Act": 6398, "Ġsubsequ": 6399, "ĠPack": 6400, "ĠFort": 6401, "Ġconsidering": 6402, "Ġuniversity": 6403, "log": 6404, "Ġmarried": 6405, "Ġillust": 6406, "ĠTrue": 6407, "£ı": 6408, "Ġnumerous": 6409, "rastructure": 6410, "Ġseriously": 6411, "Ġreferred": 6412, "ua": 6413, "Ġconsistent": 6414, "onna": 6415, "ĠReal": 6416, "ruption": 6417, "ciples": 6418, "Ġfacts": 6419, "91": 6420, "otes": 6421, "erg": 6422, "Then": 6423, "Ġaccompl": 6424, "Note": 6425, "Ġrevenue": 6426, "Ġpassing": 6427, "Ġmal": 6428, "een": 6429, "ĠYet": 6430, "Ġgather": 6431, "terday": 6432, "ework": 6433, "ĠAuthor": 6434, "Pe": 6435, "Ġoptim": 6436, "Ġrub": 6437, "Ġè£ı": 6438, "Ġunknown": 6439, "stone": 6440, "Ġunion": 6441, "olve": 6442, "Ġopportunities": 6443, "Ġbrowser": 6444, "ĠWal": 6445, "ĠCost": 6446, "Ġreporting": 6447, "sts": 6448, "pet": 6449, "Ġsand": 6450, "Ġsuddenly": 6451, "Ġsurprising": 6452, "ĠVR": 6453, "Ġsomewhat": 6454, "ĠBas": 6455, "ulture": 6456, "izz": 6457, "ĠCD": 6458, "Ġchallenges": 6459, "Ġsettings": 6460, "Ġexperiences": 6461, "ĠFull": 6462, "Ġcann": 6463, "Ġreceiving": 6464, "EST": 6465, "Ġjoint": 6466, "Ġcultural": 6467, "Ġast": 6468, "82": 6469, "astern": 6470, "ceived": 6471, "ĠCru": 6472, "Ġbull": 6473, "pired": 6474, "amm": 6475, "Ġfacing": 6476, "power": 6477, "Ġboss": 6478, "ĠHol": 6479, "Ġinstr": 6480, "Ġincreasingly": 6481, "Ġshift": 6482, "Ġstreets": 6483, "ĠWilliams": 6484, "abb": 6485, "Ġlie": 6486, "Ġlaugh": 6487, "ĠCa": 6488, "PL": 6489, "Ġadults": 6490, "Ġcustomer": 6491, "Ġobtained": 6492, "Ġsupporting": 6493, "html": 6494, "fire": 6495, "Ġdetailed": 6496, "Ġpicked": 6497, "ĠRight": 6498, "lder": 6499, "EE": 6500, "stood": 6501, "ĠKim": 6502, "Ġwire": 6503, "Ġsight": 6504, "Ġdevelopers": 6505, "Ġpersons": 6506, "Ġsad": 6507, "Ġcup": 6508, "Ġwarning": 6509, "Ġboys": 6510, "long": 6511, "Ġbird": 6512, "fo": 6513, "Ġwal": 6514, "Ġobserved": 6515, "Ġzone": 6516, "iveness": 6517, "Ġchannel": 6518, "cript": 6519, "Ġrefused": 6520, "ĠAgain": 6521, "Ġsuc": 6522, "Ġspokesman": 6523, "ĠRef": 6524, "rite": 6525, "ouston": 6526, "ãĥ³": 6527, "ĠSher": 6528, "Ġacts": 6529, "ĠName": 6530, "Ġstruggle": 6531, "arry": 6532, "ometimes": 6533, "Ġdiscrim": 6534, "HT": 6535, "Ġcategory": 6536, "Ġrealize": 6537, "Ġemployee": 6538, "ĠAfghan": 6539, "enger": 6540, "Ġguns": 6541, "ĠSteve": 6542, "ĠMot": 6543, "ĠOl": 6544, "oked": 6545, "Ġthick": 6546, "Ġfairly": 6547, "illy": 6548, "Ġsurve": 6549, "ĠMat": 6550, "weight": 6551, "âĶ": 6552, "Ġtroops": 6553, "Ġagents": 6554, "Ġbattery": 6555, "Ġmotiv": 6556, "á": 6557, "Sec": 6558, "den": 6559, "overy": 6560, "LS": 6561, "Ġflu": 6562, "Ġconfident": 6563, "ĠOper": 6564, "Ġempty": 6565, "Ġphen": 6566, "Ġsector": 6567, "Ġexcited": 6568, "Ġremote": 6569, "aph": 6570, "oen": 6571, "Ġdestroyed": 6572, "Ġmoral": 6573, "ĠHP": 6574, "ĠRon": 6575, "Ġdress": 6576, "ĠBat": 6577, "Ġlit": 6578, "ĠMS": 6579, "Ġaf": 6580, "HL": 6581, "rum": 6582, "isms": 6583, "Ġshouldn": 6584, "Ġsympt": 6585, "ĠToronto": 6586, "hetic": 6587, "Ġcarbon": 6588, "Ġinstalled": 6589, "Ġviolent": 6590, "Ġsolar": 6591, "ja": 6592, "Ġpractices": 6593, "Ġride": 6594, "ĠPenn": 6595, "Ġimproved": 6596, "Ġaudio": 6597, "Ġbehavi": 6598, "ĠPS": 6599, "Ġeating": 6600, "Data": 6601, "ĠReview": 6602, "pass": 6603, "claim": 6604, "uated": 6605, "angers": 6606, "chen": 6607, "Ġproperties": 6608, "Ġanywhere": 6609, "Another": 6610, "Ġblow": 6611, "ĠJackson": 6612, "Ġproud": 6613, "Ġplane": 6614, "lines": 6615, "Ġsquare": 6616, "Ġproof": 6617, "ansas": 6618, "Ġtalked": 6619, "makers": 6620, "Ġsister": 6621, "Ġholds": 6622, "Ġresident": 6623, "Ġ==": 6624, "Ġresistance": 6625, "Ġsplit": 6626, "Ġprosecut": 6627, "Ġconfidence": 6628, "resents": 6629, "Ġcuts": 6630, "Ġexception": 6631, "Ġzero": 6632, "Getty": 6633, "Ġcopyright": 6634, "Ġtotally": 6635, "ormal": 6636, "ifications": 6637, "ĠAustralian": 6638, "Ġsick": 6639, "Ġ150": 6640, "Ġhousehold": 6641, "Ġfees": 6642, "Ġdrivers": 6643, "ogen": 6644, "ĠNY": 6645, "Ġnecessarily": 6646, "Ġregulations": 6647, "earing": 6648, "sl": 6649, "Ġperspective": 6650, "care": 6651, "icial": 6652, "His": 6653, "Ġescape": 6654, "Ġsurprised": 6655, "ĠVan": 6656, "urrent": 6657, "Ġvac": 6658, "81": 6659, "ĠThus": 6660, "Ġemphas": 6661, "ĠChampions": 6662, "ĠIce": 6663, "Ġnarr": 6664, "Ġheads": 6665, "Ġcausing": 6666, "bel": 6667, "fortunately": 6668, "ĠMa": 6669, "Ġtargets": 6670, "cipl": 6671, "Ġafternoon": 6672, "Ġadds": 6673, "ĠMaybe": 6674, "ĠFour": 6675, "essed": 6676, "plete": 6677, "Ġusual": 6678, "cho": 6679, "ingu": 6680, "Ġwithd": 6681, "ĠEnergy": 6682, "ĠEconom": 6683, "OO": 6684, "Ġarticles": 6685, "Ġinjured": 6686, "Ġmanage": 6687, "Ġexplains": 6688, "Ġdiagn": 6689, "Rec": 6690, "atures": 6691, "Ġlinked": 6692, "Ġdiscussed": 6693, "Ġexplo": 6694, "Ġoccasion": 6695, "athan": 6696, "Ġopposite": 6697, "Ġfaces": 6698, "Ġdenied": 6699, "ĠKnight": 6700, "Ġnut": 6701, "Ġapproximately": 6702, "Ġdisappoint": 6703, "onymous": 6704, "ĠBest": 6705, "ĠLo": 6706, "ĠHy": 6707, "ĠAff": 6708, "Ġvoting": 6709, "anwhile": 6710, "ĠIII": 6711, "Ġinstitutions": 6712, "agram": 6713, "ĠDaily": 6714, "Ġdrag": 6715, "Ġnearby": 6716, "Ġguilty": 6717, "Ġconver": 6718, "Pre": 6719, "ship": 6720, "Ġreward": 6721, "Ġphilosoph": 6722, "ĠSS": 6723, "ugh": 6724, "Ġapps": 6725, "friend": 6726, "Ġupper": 6727, "Ġadvert": 6728, "Ġsnow": 6729, "Ġfrust": 6730, "Ġourselves": 6731, "Fr": 6732, "ĠDie": 6733, "ampion": 6734, "Ġdismiss": 6735, "Ġcere": 6736, "Ġsignal": 6737, "from": 6738, "Ġ).": 6739, "Ġ52": 6740, "Ġcrimes": 6741, "itors": 6742, "estival": 6743, "useum": 6744, "Ġcouncil": 6745, "ĠSaud": 6746, "May": 6747, "ĠGun": 6748, "ician": 6749, "ether": 6750, "Ġsufficient": 6751, "ĠHen": 6752, "sole": 6753, "Ġhistorical": 6754, "ĠFar": 6755, "ĠTurn": 6756, "Ġpin": 6757, "Ġsucceed": 6758, "mat": 6759, "lymp": 6760, "Ġtradition": 6761, "ĠOk": 6762, "Ġcro": 6763, "Ġdescription": 6764, "alle": 6765, "Ġsky": 6766, "Te": 6767, "Ġwidely": 6768, "Ġwave": 6769, "Ġdefinition": 6770, "ĠJews": 6771, "Ġcycle": 6772, "Ġrefere": 6773, "Ġbrings": 6774, "usal": 6775, "Ġalive": 6776, "Ġfrequently": 6777, "Ġintention": 6778, "ĠControl": 6779, "lv": 6780, "ystem": 6781, "Ġprivacy": 6782, "gent": 6783, "rence": 6784, "ĠQuest": 6785, "ĠChristmas": 6786, "Ġrail": 6787, "Ġcooper": 6788, "Ġtested": 6789, "ĠCapt": 6790, "asks": 6791, "Ġcomfortable": 6792, "Ġdelivered": 6793, "scape": 6794, "Ġdepth": 6795, "ĠGOP": 6796, "Ġwrites": 6797, "Ġassets": 6798, "Ġsav": 6799, "iments": 6800, "Ġtransition": 6801, "Ġartist": 6802, "ĠLook": 6803, "Ġlob": 6804, "Ġcomponents": 6805, "arity": 6806, "Ġwalked": 6807, "Ġroot": 6808, "Ġparticipants": 6809, "Ġnoticed": 6810, "Ġresc": 6811, "Ġnav": 6812, "ĠAdminist": 6813, "da": 6814, "utral": 6815, "plate": 6816, "Ġimportance": 6817, "Ġassert": 6818, "iously": 6819, "cription": 6820, "Ġinjuries": 6821, "ĠCheck": 6822, "Ġregistered": 6823, "Ġintent": 6824, "Ġmissed": 6825, "ographic": 6826, "Ġsentence": 6827, "ounter": 6828, "Ġassistance": 6829, "evin": 6830, "Ġdatabase": 6831, "Ġbuildings": 6832, "Ġclassic": 6833, "Ġthinks": 6834, "ĠOhio": 6835, "Pr": 6836, "ugg": 6837, "Ġfee": 6838, "pan": 6839, "Ġeffectively": 6840, "Ġfacility": 6841, "Ġbear": 6842, "Ġchapter": 6843, "Ġdogs": 6844, "ĠColumb": 6845, "Ġlatter": 6846, "itial": 6847, "Ġadmitted": 6848, "TV": 6849, "ĠGeorg": 6850, "Ġposts": 6851, "\\\\": 6852, "Ġlawyer": 6853, "Ġequival": 6854, "Ġmand": 6855, "Ġcontrolled": 6856, "ĠWalk": 6857, "ĠAndrew": 6858, "Ġmenu": 6859, "amental": 6860, "Ġprotected": 6861, "va": 6862, "Ġadministr": 6863, "oral": 6864, "Ġrein": 6865, "ĠSar": 6866, "Ġamounts": 6867, "Ġnative": 6868, "ĠMoon": 6869, "Ġrepresents": 6870, "Ġabandon": 6871, "Ġcarrying": 6872, "Ġtank": 6873, "mary": 6874, "Ġdeclared": 6875, "Tube": 6876, "Ġhat": 6877, "Ġpunish": 6878, "ellect": 6879, "mes": 6880, "Ġuniverse": 6881, "ĠRod": 6882, "phy": 6883, "Ġinfrastructure": 6884, "Ġ51": 6885, "Ġopposed": 6886, "ownt": 6887, "ca": 6888, "ĠMake": 6889, "Ġhardware": 6890, "Ġcoffee": 6891, "Rel": 6892, "bal": 6893, "world": 6894, "ĠSaf": 6895, "ĠSea": 6896, "inals": 6897, "Ġowned": 6898, "Ġhall": 6899, "ersion": 6900, "Ġdescribe": 6901, "ĠPot": 6902, "Ġportion": 6903, "Ġatmosp": 6904, "Ġgovernments": 6905, "Ġdepending": 6906, "Ġoffense": 6907, "Ġtrick": 6908, "awa": 6909, "ĠLine": 6910, "ĠVis": 6911, "ĠHard": 6912, "ĠOrig": 6913, "ĠClick": 6914, "Ġdesk": 6915, "ĠValley": 6916, "ĠSov": 6917, "Ġmovies": 6918, "Ġremark": 6919, "Ġmail": 6920, "Ġconscious": 6921, "Ġruling": 6922, "ĠRights": 6923, "Ġmedic": 6924, "hent": 6925, "ĠWomen": 6926, "><": 6927, "Ġreplaced": 6928, "ĠPrem": 6929, "ĠThanks": 6930, "Ġrenew": 6931, "ĠBall": 6932, "iform": 6933, "Ġshots": 6934, "Comm": 6935, "Ġarmed": 6936, "Ġconstant": 6937, "Ġtaste": 6938, "Ġrealized": 6939, "Ġbuff": 6940, "Ġmo": 6941, "Ġefficient": 6942, "Most": 6943, "oration": 6944, "ifies": 6945, "Ġcommunication": 6946, "Ġflood": 6947, "Ġconsequences": 6948, "Ġanyway": 6949, "igg": 6950, "ĠGM": 6951, "ĠThank": 6952, "Ġiron": 6953, "Ġevolution": 6954, "ĠCop": 6955, "twitter": 6956, "Ġ95": 6957, "Ġrelationships": 6958, "adel": 6959, "ĠYoung": 6960, "Ġproposal": 6961, "ayers": 6962, "uilding": 6963, "ĠHot": 6964, "ORE": 6965, "cos": 6966, "Ġcollabor": 6967, "PG": 6968, "axy": 6969, "Ġknowing": 6970, "Ġsupports": 6971, "owed": 6972, "Ġcontrols": 6973, "Ġmerely": 6974, "umer": 6975, "Ġathlet": 6976, "Ġfashion": 6977, "path": 6978, "Ġgift": 6979, "Ġera": 6980, "AND": 6981, "Ġkinds": 6982, "ĠKorean": 6983, "Ġlegit": 6984, "ulous": 6985, "Ġessentially": 6986, "Ġtherap": 6987, "nic": 6988, "Ġsuffered": 6989, "Ġhur": 6990, "Ġpromise": 6991, "Ġexcess": 6992, "Ġoverw": 6993, "Ġprime": 6994, "ĠHouston": 6995, "erry": 6996, "ĠMs": 6997, "RS": 6998, "2012": 6999, "Ġstores": 7000, "ĠOlymp": 7001, "Ġjourney": 7002, "Although": 7003, "Sub": 7004, "ĠEduc": 7005, "ĠChapter": 7006, "Ġrequests": 7007, "Ġconsumers": 7008, "Ġtiny": 7009, "Ġisol": 7010, "ĠFair": 7011, "ba": 7012, "ĠYOU": 7013, "Ġcrash": 7014, "celer": 7015, "Ġemotional": 7016, "Ġgoods": 7017, "Ġelected": 7018, "Ġmoder": 7019, "ĠLinux": 7020, "Ġblocks": 7021, "Ġisland": 7022, "ĠSociety": 7023, "Ġelections": 7024, "Ġbroadcast": 7025, "Ġcheap": 7026, "Ġnations": 7027, "Ġseasons": 7028, "400": 7029, "Ġwaste": 7030, "ĠSat": 7031, "Ġfields": 7032, "employ": 7033, "Ġprofile": 7034, "Ġauthors": 7035, "ALL": 7036, "ĠGra": 7037, "west": 7038, "ĠTy": 7039, "Ġdeaths": 7040, "Ġvacc": 7041, "Ġformed": 7042, "Ġdu": 7043, "Ġongoing": 7044, "ĠMuslims": 7045, "elf": 7046, "igure": 7047, "Ġassume": 7048, "ĠUkraine": 7049, "water": 7050, "Ġcoast": 7051, "Ġvoted": 7052, "gor": 7053, "ĠAS": 7054, "ĠMichigan": 7055, "aza": 7056, "ĠArm": 7057, "iro": 7058, "Ġflex": 7059, "asters": 7060, "''": 7061, "Ġwelcome": 7062, "arl": 7063, "Ġlocations": 7064, "igation": 7065, "ĠFil": 7066, "Ġbuying": 7067, "Ġarchitect": 7068, "Ġharder": 7069, "ĠCub": 7070, "Ġinterface": 7071, "Ġrestaurant": 7072, "Ġdiscover": 7073, "Ġexceed": 7074, "Ġfavour": 7075, "gery": 7076, "Ġduty": 7077, "Ġpitch": 7078, "ador": 7079, "ĠMach": 7080, "boy": 7081, "Ġresponded": 7082, "Ġextended": 7083, "hers": 7084, "Many": 7085, "raid": 7086, "ifer": 7087, "ĠIns": 7088, "Ser": 7089, "Ġmedium": 7090, "she": 7091, "ĠSports": 7092, "Ġmagazine": 7093, "utation": 7094, "Ġlimits": 7095, "ĠGall": 7096, "Ġexternal": 7097, "razil": 7098, "Ġyounger": 7099, "tle": 7100, "Ġremind": 7101, "ĠCON": 7102, "Ġimmediate": 7103, "Ġhidden": 7104, "Ġvolunte": 7105, "Ġsimpl": 7106, "odcast": 7107, "Ġphase": 7108, "dr": 7109, "Ġplot": 7110, "Ġexposure": 7111, "RI": 7112, "ograp": 7113, "vin": 7114, "anish": 7115, "ĠAcad": 7116, "ĠEngine": 7117, "Ġexpansion": 7118, "ĠPay": 7119, "Your": 7120, "Ġpushed": 7121, "ĠEll": 7122, "ĠHead": 7123, "Ġmarketing": 7124, "ĠAC": 7125, "ket": 7126, "Ġhits": 7127, "Ġgro": 7128, "ĠAge": 7129, "ĠScot": 7130, "][": 7131, "Ġstim": 7132, "ĠiPhone": 7133, "ĪĴ": 7134, "Ġnarrow": 7135, "ĠGetty": 7136, "ĠTurkey": 7137, "Ġperfectly": 7138, "Ġenable": 7139, "utch": 7140, "Ġprecise": 7141, "Ġregime": 7142, "Ġshif": 7143, "Ġcompens": 7144, "gun": 7145, "div": 7146, "Ġchosen": 7147, "ĠKen": 7148, "Any": 7149, "Ġtrees": 7150, "Ġrecommended": 7151, "ĠRen": 7152, "uable": 7153, "ĠHT": 7154, "Follow": 7155, "EG": 7156, "ĠHand": 7157, "ĠKenn": 7158, "Ġarguments": 7159, "Ġexists": 7160, "Ġbike": 7161, "ĠConserv": 7162, "Ġbreaking": 7163, "ĠGar": 7164, "Ġcrazy": 7165, "Ġvirtual": 7166, "aylor": 7167, "ixel": 7168, "Ġ1980": 7169, "Ġpermission": 7170, "ĠSeries": 7171, "Ġconsumer": 7172, "Ġclosely": 7173, "called": 7174, "Ġ54": 7175, "Ġhopes": 7176, "Ġarray": 7177, "ĠWin": 7178, "ĠLabour": 7179, "Ġspons": 7180, "ĠIre": 7181, "Ġpow": 7182, "Ġreaders": 7183, "Ġemployment": 7184, "Ġcreature": 7185, "Ġresulting": 7186, "Ġaccurate": 7187, "Ġmoments": 7188, "Ġargued": 7189, "Ġped": 7190, "During": 7191, "Ġ53": 7192, "ĠTal": 7193, "Ġsought": 7194, "Ġsuffering": 7195, "Ġicon": 7196, "lee": 7197, "Ġ($": 7198, "alian": 7199, "°": 7200, "Ġpra": 7201, "Ġbonus": 7202, "(\"": 7203, "ko": 7204, "Ġacting": 7205, "DE": 7206, "fall": 7207, "Ġcomparison": 7208, "Ġsmooth": 7209, "ĠNAS": 7210, "upp": 7211, "ĠJoseph": 7212, "eping": 7213, "ĠTake": 7214, "ĠMid": 7215, "Ġsending": 7216, "fast": 7217, "ĠFall": 7218, "Ġdealing": 7219, "user": 7220, "ĠOrgan": 7221, "Co": 7222, "Ġattached": 7223, "Ġsees": 7224, "%.": 7225, "Ġtypical": 7226, "ART": 7227, "Ġfinds": 7228, "ĠAsia": 7229, "umin": 7230, "ĠCore": 7231, "ĠEnt": 7232, "inent": 7233, "uce": 7234, "ĠBlood": 7235, "ĠNever": 7236, "Ġemails": 7237, "Ġhighlight": 7238, "Ġconfront": 7239, "atus": 7240, "uted": 7241, "Ġunus": 7242, "Ġtopic": 7243, "ĠAdam": 7244, "Ġble": 7245, "ati": 7246, "Ġunderstood": 7247, "Set": 7248, "struct": 7249, "TP": 7250, "Ġmob": 7251, "aa": 7252, "ĠStart": 7253, "pected": 7254, "sell": 7255, "Ġdedicated": 7256, "ĠCA": 7257, "uan": 7258, "Ġsongs": 7259, "escription": 7260, "Ġtech": 7261, "Ġrape": 7262, "Ġaside": 7263, "Ġgrant": 7264, "Ġ56": 7265, "sub": 7266, "Ġargue": 7267, "Ġcontaining": 7268, "Ġschedule": 7269, "Ġliberal": 7270, "Ġpublicly": 7271, "Ġheavily": 7272, "ĠUt": 7273, "iner": 7274, "ĠSection": 7275, "ĠCare": 7276, "weet": 7277, "ls": 7278, "Dis": 7279, "âĶĢ": 7280, "ĠFollow": 7281, "Back": 7282, "ĠIT": 7283, "Ġbes": 7284, "ji": 7285, "ĠHit": 7286, "ested": 7287, "Ġeverybody": 7288, "ĠSwed": 7289, "Ġfemin": 7290, "Ġfacilities": 7291, "Ġconven": 7292, "Comp": 7293, "ĠOS": 7294, "core": 7295, "Ġanx": 7296, "Ġdivision": 7297, "ĠCam": 7298, "ĠStan": 7299, "mates": 7300, "Ġexplore": 7301, "plom": 7302, "Ġshares": 7303, "pload": 7304, "anes": 7305, "Ġideal": 7306, "eters": 7307, "ĠBase": 7308, "Ġplastic": 7309, "Ġdistinct": 7310, "ĠNetwork": 7311, "ĠSeattle": 7312, "Ġtrading": 7313, "ensus": 7314, "intend": 7315, "Ġexhib": 7316, "Ġinitially": 7317, "ĠFood": 7318, "Ġthousand": 7319, "ĠBusiness": 7320, "acter": 7321, "Ġparagraph": 7322, "Ġroughly": 7323, "Ġwww": 7324, "Ġcreative": 7325, "ĠConf": 7326, "Ġconsumption": 7327, "Ġfilms": 7328, "agan": 7329, "Ġobtain": 7330, "Ġtall": 7331, "Ġtor": 7332, "Ġacknowled": 7333, "Ġgrown": 7334, "alo": 7335, "KE": 7336, "Ġ400": 7337, "enders": 7338, "taining": 7339, "UG": 7340, "Ġsuicide": 7341, "Ġwatched": 7342, "ĠList": 7343, "ali": 7344, "rehens": 7345, "Ġsurrounding": 7346, "Ġpip": 7347, "Ġflying": 7348, "ĠJava": 7349, "ordan": 7350, "Ġserving": 7351, "inations": 7352, "post": 7353, "Ġsho": 7354, "Av": 7355, "Ġjail": 7356, "zy": 7357, "Ġ1999": 7358, "Ġ>": 9609, "orous": 9610, "Ġfirms": 9611, "screen": 9612, "una": 9613, "Ġembarrass": 9614, "ulse": 9615, "Ġletting": 9616, "Ġthrew": 9617, "iley": 9618, "Ġchannels": 9619, "lan": 9620, "ĠVegas": 9621, "Ġsear": 9622, "Ġfantastic": 9623, "arre": 9624, "uzzle": 9625, "ĠDer": 9626, "Those": 9627, "Ġswing": 9628, "Ġsheet": 9629, "index": 9630, "cover": 9631, "ogan": 9632, "Ġvariables": 9633, "ĠTech": 9634, "Ġspoken": 9635, "achel": 9636, "ĠDa": 9637, "ĠMountain": 9638, "Ġloaded": 9639, "Ġfootage": 9640, "version": 9641, "Ġunl": 9642, "ĠPhoenix": 9643, "Ġthrowing": 9644, "Ġfiring": 9645, "Ġtracking": 9646, "Ġwidth": 9647, "Ġstruggling": 9648, "rooms": 9649, "otion": 9650, "Ġmonthly": 9651, "ĠServer": 9652, "Ġeggs": 9653, "open": 9654, "MC": 9655, "Ġ1993": 9656, "Ġhired": 9657, "Ġstayed": 9658, "ĠAllen": 9659, "Ġstro": 9660, "Ġ98": 9661, "step": 9662, "ĠTurkish": 9663, "Ġfabric": 9664, "isting": 9665, "ĠDom": 9666, "Ġdates": 9667, "Ġpron": 9668, "Ġbasketball": 9669, "Ġlucky": 9670, "ĠArabia": 9671, "Ġassumed": 9672, "esty": 9673, "Ġaffairs": 9674, "Ġglad": 9675, "ĠIndeed": 9676, "ĠFA": 9677, "ĠWord": 9678, "Ġjoining": 9679, "ifice": 9680, "pread": 9681, "irts": 9682, "ĠSelect": 9683, "Ġpopulations": 9684, "aware": 9685, "Ġnose": 9686, "Ġcomplaints": 9687, "start": 9688, "Ġscoring": 9689, "Thanks": 9690, "Ġmining": 9691, "Ġvisitors": 9692, "SH": 9693, "Ġdamaged": 9694, "Ġcharacteristics": 9695, "ĠPent": 9696, "DC": 9697, "Ġ83": 9698, "ĠSix": 9699, "rates": 9700, "Ġflags": 9701, "ĠBrew": 9702, "dog": 9703, "Mark": 9704, "////": 9705, "Ġexecution": 9706, "Ġjoke": 9707, "phones": 9708, "Ġtestimony": 9709, "Ġobst": 9710, "QL": 9711, "ĠCut": 9712, "Ġstudied": 9713, "ĠNintendo": 9714, "icket": 9715, "ĠNBC": 9716, "Ġlad": 9717, "ĠBra": 9718, "ĠMoh": 9719, "Ġkernel": 9720, "Ġoverwhelming": 9721, "Ġaged": 9722, "Ġapplicable": 9723, "ĠCond": 9724, "Ġroads": 9725, "ĠBlock": 9726, "made": 9727, "odge": 9728, "Ġcommands": 9729, "Ġoffices": 9730, "veland": 9731, "Ġtut": 9732, "Ġreceiver": 9733, "ĠFro": 9734, "Ġshopping": 9735, "ĠiP": 9736, "ĠStre": 9737, "ĠABC": 9738, "Ġentertainment": 9739, "ĠBow": 9740, "orted": 9741, "Mc": 9742, "Ġreads": 9743, "grad": 9744, "ĠCollect": 9745, "ĠâĪĴ": 9746, "ĠCapital": 9747, "ederation": 9748, "Ġemployer": 9749, "Ġinvolvement": 9750, "Ġanxiety": 9751, "alia": 9752, "Ġroof": 9753, "ĠAmong": 9754, "ĠDemocrat": 9755, "Ġstats": 9756, "ĠVill": 9757, "Ġconstitutional": 9758, "Ġreferring": 9759, "itty": 9760, "Ġtackle": 9761, "outube": 9762, "Ġbacked": 9763, "ĠHong": 9764, "ĠBroad": 9765, "Ġele": 9766, "ĠOtt": 9767, "Ġ1992": 9768, "hour": 9769, "achusetts": 9770, "Cal": 9771, "Ġdefeated": 9772, "Ġ81": 9773, "esp": 9774, "Ġseemingly": 9775, "was": 9776, "ĠJenn": 9777, "ĠKurd": 9778, "Ġgene": 9779, "Ġdiscount": 9780, "Ret": 9781, "ECT": 9782, "();": 9783, "Ġclubs": 9784, "Ġsid": 9785, "ĠMarsh": 9786, "Check": 9787, "Ġpp": 9788, "ĠEag": 9789, "idespread": 9790, "Ġbeings": 9791, "FT": 9792, "Ġintroduction": 9793, "ĠChange": 9794, "ARD": 9795, "Ġ110": 9796, "adows": 9797, "ierce": 9798, "Ġmeal": 9799, "author": 9800, "ĠBang": 9801, "lahoma": 9802, "Ġranks": 9803, "2011": 9804, "????": 9805, "max": 9806, "Ġcollapse": 9807, "Ġopens": 9808, "Ġecho": 9809, "Ġsoph": 9810, "Ġracist": 9811, "Ġenormous": 9812, "Ġwaves": 9813, "Ġtap": 9814, "Ġcomprehensive": 9815, ".--": 9816, "ĠRoy": 9817, "Ġfarmers": 9818, "Related": 9819, "aired": 9820, "rones": 9821, "ĠCrim": 9822, "Ġproportion": 9823, "Ġdesigns": 9824, "Ġnegotiations": 9825, "Ġvirtually": 9826, "ĠBatman": 9827, "Ġwarn": 9828, "Ġlegitimate": 9829, "mate": 9830, "Ġconvention": 9831, ",,": 9832, "netic": 9833, "ĠSD": 9834, "Ġconsistently": 9835, "Ġcompensation": 9836, "Ġpunishment": 9837, "Ġye": 9838, "Ġtie": 9839, "ĠBureau": 9840, "irlf": 9841, "ĠBu": 9842, "ĠAren": 9843, "ĠPhilipp": 9844, "Ġknife": 9845, "Ġmemories": 9846, "ĠRoss": 9847, "Ġangle": 9848, "Ġ86": 9849, "ĠThunder": 9850, "Ġrend": 9851, "ĠTour": 9852, "Ġcounts": 9853, "sung": 9854, "ĠImp": 9855, "Ġeducational": 9856, "Ġaccessible": 9857, "COM": 9858, "Ġdrew": 9859, "yer": 9860, "Gl": 9861, "amine": 9862, "ORT": 9863, "OB": 9864, "IB": 9865, "master": 9866, "Ġtrials": 9867, "ogy": 9868, "har": 9869, "ĠTrust": 9870, "Ġpreferred": 9871, "irlfriend": 9872, "ĠNev": 9873, "Ġbin": 9874, "Ġcow": 9875, "Page": 9876, "Ġsignature": 9877, "ĠBL": 9878, "700": 9879, "Ġretired": 9880, "Ġbytes": 9881, "Ġneighb": 9882, "ĠLegend": 9883, "Ġdevast": 9884, "Ġsuspected": 9885, "isons": 9886, "ĠPokémon": 9887, "scale": 9888, "Ġcapabilities": 9889, "Ġrevel": 9890, "Ġcheese": 9891, "dy": 9892, "igrant": 9893, "Ġfailing": 9894, "bits": 9895, "ĠHeroes": 9896, "ĠGhost": 9897, "ĠScient": 9898, "Ġappointed": 9899, "uri": 9900, "Ġinstitution": 9901, "Ġexpanded": 9902, "greg": 9903, "Ġmonitoring": 9904, "Ġpodcast": 9905, "Ġcoalition": 9906, "Ġ96": 9907, "Jo": 9908, "Ġstolen": 9909, "ĠSab": 9910, "Ġstops": 9911, "Ġholiday": 9912, "Ġintr": 9913, "Car": 9914, "Black": 9915, "ĠLGBT": 9916, "Ġwarming": 9917, "ĠAnderson": 9918, "Ġ89": 9919, "Ġproducer": 9920, "Med": 9921, "Ġaccuracy": 9922, "ĠMarvel": 9923, "izabeth": 9924, "ĠPatrick": 9925, "mony": 9926, "Ġmini": 9927, "acles": 9928, "Ġovert": 9929, "they": 9930, "Ġmembership": 9931, "ĠVen": 9932, "Ġexch": 9933, "Ġremoval": 9934, "ĠDave": 9935, "TY": 9936, "mad": 9937, "ĠFind": 9938, "Ġadequ": 9939, "Ġec": 9940, "Ġteeth": 9941, "Ġemotion": 9942, "Ġperm": 9943, "Ġsolely": 9944, "db": 9945, "Ġextraord": 9946, "IGHT": 9947, "cal": 9948, "Ġguidelines": 9949, "Ġdying": 9950, "Ġsuspended": 9951, "ĠPremier": 9952, "ĠAnthony": 9953, "elve": 9954, "Ġdad": 9955, "ĠEth": 9956, "ĠFootball": 9957, "Ġabandoned": 9958, "Ġ<<": 9959, "Ġmarch": 9960, "Ġhorror": 9961, "âĢ¦\"": 9962, "Ġchildhood": 9963, "Ġcampaigns": 9964, "Ġlunch": 9965, "ĠAlbert": 9966, "block": 9967, "âĸĪâĸĪ": 9968, "ounding": 9969, "Ġbone": 9970, "organ": 9971, "aders": 9972, "ĠFlash": 9973, "ĠDrive": 9974, "Ġtonight": 9975, "Ġwars": 9976, "ĠFL": 9977, "Ġformation": 9978, "const": 9979, "News": 9980, "Ġcompe": 9981, "orious": 9982, "ĠStaff": 9983, "Ġdiscussions": 9984, "ĠProtection": 9985, "ĠJam": 9986, "Ġcriteria": 9987, "Ġinstallation": 9988, "Ġaccomplish": 9989, "izza": 9990, "Ġpublisher": 9991, "Ġrescue": 9992, "ĠTry": 9993, "ULL": 9994, "ĠSom": 9995, "ĠHop": 9996, "oret": 9997, "ths": 9998, "ordon": 9999, "Ġpocket": 10000, "ĠInv": 10001, "Download": 10002, "ĠCrime": 10003, "Ġbene": 10004, "ĠGuide": 10005, "ĠAssembly": 10006, "Ġparameters": 10007, "IE": 10008, "ĠAlexander": 10009, "Ġconcert": 10010, "ĠSche": 10011, "Ġshoes": 10012, "Ġvisiting": 10013, "Ġrecall": 10014, "Ġbub": 10015, "Ġrural": 10016, "Ġconcrete": 10017, "ĠRos": 10018, "Next": 10019, "Russ": 10020, "Ġloans": 10021, "ĠShield": 10022, "Ġtrem": 10023, "hemat": 10024, "kg": 10025, "ĠHarris": 10026, "isition": 10027, "ĠMove": 10028, "ĠFC": 10029, "Ġfate": 10030, "ĠCho": 10031, "Ġtired": 10032, "Ġprincipal": 10033, "hist": 10034, "iences": 10035, "athy": 10036, "Ġsevent": 10037, "Ġmood": 10038, "Ġstrategic": 10039, "Ġdiseases": 10040, "Ġforum": 10041, "Ġtempor": 10042, "Ġheadquarters": 10043, "Par": 10044, "ige": 10045, "flix": 10046, "Ġguitar": 10047, "Ġ94": 10048, "Only": 10049, "Ġreleases": 10050, "roph": 10051, "================================": 10052, "Ġ600": 10053, "ĠContinue": 10054, "igate": 10055, "ĠCrit": 10056, "system": 10057, "Ġdisabled": 10058, "Ġunexpected": 10059, "ithub": 10060, "Ġunclear": 10061, "ĠEst": 10062, "Ġcontrad": 10063, "Ġstrategies": 10064, "ventures": 10065, "Ġpassage": 10066, "AME": 10067, "Ġimproving": 10068, "Ġreveals": 10069, "Ġdecrease": 10070, "ova": 10071, "Ġannoy": 10072, "ĠShort": 10073, "ĠLibrary": 10074, "Ġcyber": 10075, "nell": 10076, "ĠHur": 10077, "ĠCB": 10078, "Ġphotograp": 10079, "UI": 10080, "Ġsed": 10081, "Ge": 10082, "Ġ87": 10083, "Ġdiverse": 10084, "Ġencouraged": 10085, "Ġconspiracy": 10086, "Ġbirds": 10087, "Ġoperator": 10088, "Ġhandful": 10089, "Ġclassified": 10090, "?)": 10091, "Ġdramatic": 10092, "Ġinvestigators": 10093, "ito": 10094, "Ġwidespread": 10095, "ĠRoom": 10096, "----------------------------------------------------------------": 10097, "Ġcollective": 10098, "Ġjournalist": 10099, "String": 10100, "Ġtemperatures": 10101, "ila": 10102, "Ġguid": 10103, "Ġinspect": 10104, "Ġmissile": 10105, "ĠMayor": 10106, "Ġmanual": 10107, "Ġsimultane": 10108, "Ġratings": 10109, "Ġsuck": 10110, "Ġ97": 10111, "Ġuniversal": 10112, "Ġpharm": 10113, "Ġdisrupt": 10114, "iano": 10115, "AV": 10116, "Ġft": 10117, "Ġstatist": 10118, "olds": 10119, "ĠWalker": 10120, "php": 10121, "Ġundert": 10122, "ĠLas": 10123, "ishop": 10124, "ntil": 10125, "reshold": 10126, "ĠWhether": 10127, "Ms": 10128, "Ġdeny": 10129, "ĠCloud": 10130, "Ġprovider": 10131, "Ġsurviv": 10132, "ĠUpdate": 10133, "has": 10134, "Ġmistakes": 10135, "charge": 10136, "pled": 10137, "rity": 10138, "Ġnode": 10139, "ĠMassachusetts": 10140, "ools": 10141, "lication": 10142, "Ġfails": 10143, "emale": 10144, "ori": 10145, "backs": 10146, "Ġshirt": 10147, "Ġ''": 10148, "ĠNAT": 10149, "Ġwaters": 10150, "elson": 10151, "Ġease": 10152, "Ġscar": 10153, "Ġcontents": 10154, "mind": 10155, "Ġcontribution": 10156, "Ġshr": 10157, "Ġhanded": 10158, "Ġstability": 10159, "Ġtrave": 10160, "Em": 10161, "Ġmirror": 10162, "123": 10163, "Ġweigh": 10164, "Ġfiction": 10165, "ouver": 10166, "istant": 10167, "rition": 10168, "ĠFed": 10169, "Ġphysically": 10170, "Ġstake": 10171, "ĠArticle": 10172, "ĠArc": 10173, "ĠLewis": 10174, "ĠMind": 10175, "Ġdemonstrate": 10176, "Ġprofits": 10177, "vision": 10178, "omic": 10179, "olid": 10180, "Ġbattles": 10181, "Ġdrives": 10182, "Ġeastern": 10183, "ĠSony": 10184, "!!!": 10185, "aration": 10186, "vard": 10187, "ĠGL": 10188, "portation": 10189, "Ġ92": 10190, "Ġlawmakers": 10191, "Ġprotecting": 10192, "ĠEPA": 10193, "Ġyeah": 10194, "Ġshame": 10195, "olph": 10196, "even": 10197, "xit": 10198, "Ġattach": 10199, "Ġrepresenting": 10200, "Ġobs": 10201, "ĠUtah": 10202, "iffs": 10203, "ĠFreedom": 10204, "ó": 10205, "AK": 10206, "Ġincidents": 10207, "itage": 10208, "Ġviewers": 10209, "cd": 10210, "Ġmouse": 10211, "Ġclar": 10212, "Ġaccordance": 10213, "Ġbot": 10214, "cor": 10215, "ĠSummer": 10216, "held": 10217, "Ġinnocent": 10218, "Ġinitiative": 10219, "ols": 10220, "________________________________": 10221, "Ġspots": 10222, "pace": 10223, "Ġconventional": 10224, "Ġcorporations": 10225, "Ġblocked": 10226, "HD": 10227, "attered": 10228, "Ġrefers": 10229, "Ġbuck": 10230, "ĠDigital": 10231, "120": 10232, "Ġtopics": 10233, "TF": 10234, "Äģ": 10235, "brid": 10236, "reement": 10237, "Ġunderlying": 10238, "ĠMember": 10239, "Ġinvestigating": 10240, "Ġpregnancy": 10241, "Ġtouchdown": 10242, "ĠBand": 10243, "ĠCaller": 10244, "Ġinstances": 10245, "PP": 10246, "wa": 10247, "Good": 10248, "Ġ1991": 10249, "ĠCold": 10250, "Ġfears": 10251, "Ġremarks": 10252, "ĨĴ": 10253, "atal": 10254, "Ġmit": 10255, "Ġexperiments": 10256, "ipt": 10257, "Color": 10258, "indu": 10259, "Update": 10260, "Ġ93": 10261, "Ag": 10262, "Ġå": 10263, "ancouver": 10264, "Both": 10265, "Ġjudges": 10266, "Object": 10267, "Ġstere": 10268, "umbn": 10269, "Ġparticipation": 10270, "ĠStars": 10271, "ĠJere": 10272, "Ġweekly": 10273, "ĠBan": 10274, "Ġconversations": 10275, "ĠPitt": 10276, "uz": 10277, "ĠIndiana": 10278, "ĠKick": 10279, "Ġinfection": 10280, "Ġheroes": 10281, "Ġsettled": 10282, "Ġstrip": 10283, "Ġhal": 10284, "Ġdump": 10285, "ĠSci": 10286, "Ġles": 10287, "Ġreferences": 10288, "ĠURL": 10289, "ĠBridge": 10290, "Ġwanting": 10291, "Force": 10292, "Ġexclus": 10293, "Meanwhile": 10294, "mn": 10295, "Ġgentle": 10296, "maker": 10297, "senal": 10298, "ĠGro": 10299, "ouri": 10300, "ĠRain": 10301, "ĠAlliance": 10302, "Ġlift": 10303, "ela": 10304, "SD": 10305, "ĠCleveland": 10306, "Ġranked": 10307, "Ġstadium": 10308, "Ġdeadly": 10309, "ä¸": 10310, "Ġriding": 10311, "aria": 10312, "ĠArmor": 10313, "Ġdocumentation": 10314, "ĠGreece": 10315, "reek": 10316, "Ġlens": 10317, "ĠSa": 10318, "Ġgross": 10319, "ĠEmer": 10320, "agers": 10321, "ĠDub": 10322, "ĠRh": 10323, "ĠAMD": 10324, "Ġarrival": 10325, "Ġdesert": 10326, "Ġsupplement": 10327, "ĠResp": 10328, "Ġknee": 10329, "Ġmargin": 10330, "font": 10331, "ogg": 10332, "2010": 10333, "ĠPir": 10334, "ĠProm": 10335, "ivals": 10336, "Ġintake": 10337, "Ġdifferently": 10338, "ugs": 10339, "Ġbits": 10340, "cluded": 10341, "Ġsearching": 10342, "ĠDu": 10343, "umble": 10344, "Ġfunctional": 10345, "ĠBaltimore": 10346, "ĠCould": 10347, "Ġdesired": 10348, "Ġcircuit": 10349, "ĠLyn": 10350, "ĠGO": 10351, "ĠFalse": 10352, "repre": 10353, "':": 10354, "alties": 10355, "Ġminim": 10356, "Ġdrove": 10357, "ĠShould": 10358, "Ġhip": 10359, "Ġpros": 10360, "Ġutility": 10361, "ĠNature": 10362, "ĠMode": 10363, "President": 10364, "opp": 10365, "rat": 10366, "formance": 10367, "Ġconcentration": 10368, "Ġfont": 10369, "ĠBud": 10370, "Ġamid": 10371, "Ġrevers": 10372, "ĠML": 10373, "Bar": 10374, "Ġinteraction": 10375, "Ġjurisd": 10376, "Ġspells": 10377, "dep": 10378, "fil": 10379, "Ġcivilians": 10380, "utter": 10381, "ĠCooper": 10382, "ĠBelow": 10383, "Ġentrance": 10384, "Ġconvert": 10385, "Ġcontroversy": 10386, "owered": 10387, "Ġcontrary": 10388, "Ġarc": 10389, "ĠExecutive": 10390, "ĠOfficer": 10391, "Ġpackages": 10392, "Ġprogressive": 10393, "width": 10394, "Ġreserved": 10395, "vol": 10396, "ĠSamsung": 10397, "Ġprinted": 10398, "Ġcenters": 10399, "Ġintroduce": 10400, "ĠKennedy": 10401, "Ġodds": 10402, "Ġsurely": 10403, "Ġindependence": 10404, "Ġpassengers": 10405, "reprene": 10406, "ĠBeh": 10407, "Ġloves": 10408, "ĠESPN": 10409, "Ġfacilit": 10410, "Ġidentical": 10411, "Ġdoct": 10412, "Ġpartnership": 10413, "conf": 10414, "ĠHide": 10415, "Ġconfused": 10416, "ĠCow": 10417, "Men": 10418, "Ġwrest": 10419, "ĠIraqi": 10420, "Ġholes": 10421, "ĠStudies": 10422, "Ġpregnant": 10423, "hard": 10424, "Ġsignals": 10425, "IX": 10426, "Ġpulling": 10427, "Ġgraduate": 10428, "Ġnominee": 10429, "Date": 10430, "Ġpermitted": 10431, "ĠâĤ¬": 10432, "ĠOklahoma": 10433, "Start": 10434, "Ġauthorized": 10435, "Ġalarm": 10436, "ĠCos": 10437, "van": 10438, "Ġgenerations": 10439, "cular": 10440, "Ġdragon": 10441, "ĠSoftware": 10442, "ĠEdward": 10443, "Ġcontroller": 10444, "Sen": 10445, "gered": 10446, "ĠVik": 10447, "Ġapproached": 10448, "Thank": 10449, "Ġcance": 10450, "Ġformula": 10451, "ĠSmall": 10452, "Ġweakness": 10453, "Ġramp": 10454, "itudes": 10455, "jud": 10456, "Ġbrilliant": 10457, "Ġaccus": 10458, "source": 10459, "Ġ800": 10460, "ĠEvil": 10461, "Sw": 10462, "Ġhomeless": 10463, "week": 10464, "iens": 10465, "rics": 10466, "ĠThird": 10467, "TO": 10468, "Ġorganic": 10469, "Ġpresentation": 10470, "agh": 10471, "ĠDownload": 10472, "vation": 10473, "Ġassembly": 10474, "orable": 10475, "holders": 10476, "ĠBernie": 10477, "ĠHelp": 10478, "Ġtong": 10479, "ĠFight": 10480, "Ġbeach": 10481, "Book": 10482, "ĠLic": 10483, "Ġrush": 10484, "ĠRound": 10485, "oup": 10486, "ĠMarx": 10487, "Ġcalculated": 10488, "ĠDevil": 10489, "ĠSarah": 10490, "Ġoccasionally": 10491, "Ġbullet": 10492, "Available": 10493, "gate": 10494, "Ġ91": 10495, "Ġhosp": 10496, "Ġpromises": 10497, "ĠHIV": 10498, "ĠStadium": 10499, "ĠStock": 10500, "ĠCorporation": 10501, "gage": 10502, "NG": 10503, "ĠCredit": 10504, "Ġsne": 10505, "ibl": 10506, "Ġaccum": 10507, "such": 10508, "Ġterrorists": 10509, "Ġconsciousness": 10510, "ĠZh": 10511, "Ġdrama": 10512, "oola": 10513, "piration": 10514, "Ġlabour": 10515, "ĠNin": 10516, "Ġutter": 10517, "Ġdemocratic": 10518, "Ġassass": 10519, "ilation": 10520, "Ġgest": 10521, "Ġabroad": 10522, "Ġmetab": 10523, "Ġsorts": 10524, "Ġflav": 10525, "UB": 10526, "Ġmg": 10527, "ĠNothing": 10528, "ĠOd": 10529, "Ġmusical": 10530, "2009": 10531, "Ġdrops": 10532, "ocated": 10533, "ateral": 10534, "000000": 10535, "Ġgre": 10536, "Ġequality": 10537, "Ġburden": 10538, "Ġvig": 10539, "ĠLeader": 10540, "------------": 10541, "Ġceremony": 10542, "Ġfighter": 10543, "Ġactors": 10544, "Ġæ": 10545, "aman": 10546, "Fi": 10547, "Ġalign": 10548, "puter": 10549, "Ġelder": 10550, "ĠNSA": 10551, "Ġrepresentation": 10552, "ĠOntario": 10553, "ITH": 10554, "usalem": 10555, "Ġharassment": 10556, "itzer": 10557, "Ġsymp": 10558, "Ġboxes": 10559, "ĠDR": 10560, "Ġmanifest": 10561, "atre": 10562, "Ġ^": 10563, "Ġdies": 10564, "leton": 10565, "Ġmissions": 10566, "ethe": 10567, "Ġresolve": 10568, "Ġfollowers": 10569, "Ġasc": 10570, "Ġkm": 10571, "lord": 10572, "ammed": 10573, "Ġsilent": 10574, "ĠAssociated": 10575, "Ġtiming": 10576, "Ġprisoners": 10577, "ĠKings": 10578, "ĠFive": 10579, "Ġtower": 10580, "Ġapproaches": 10581, "Ġprecisely": 10582, "Ġbureau": 10583, "ĠMother": 10584, "ĠIss": 10585, "Ġkeyboard": 10586, "itual": 10587, "Ġfunded": 10588, "Ġstaying": 10589, "Ġpsychological": 10590, "Ġmile": 10591, "ĠLeon": 10592, "ĠBarb": 10593, "will": 10594, "Ġwider": 10595, "ĠAtlantic": 10596, "Ġtill": 10597, "ĠRome": 10598, "rot": 10599, "Ġaccompan": 10600, "Ġflour": 10601, "aco": 10602, "World": 10603, "ĠExpress": 10604, "ĠYu": 10605, "Cor": 10606, "Ġpleased": 10607, "party": 10608, "Ġpointing": 10609, "Ġinflation": 10610, "Ġroy": 10611, "Ġ),": 10612, "ainer": 10613, "Ġwedding": 10614, "ormon": 10615, "Ġrequiring": 10616, "Ġqualified": 10617, "Ġsegment": 10618, "END": 10619, "Ġsizes": 10620, "eals": 10621, "Ġcorrupt": 10622, "assador": 10623, "Ġceleb": 10624, "Ġdreams": 10625, "ĠMess": 10626, "Ġchecking": 10627, "ĠVersion": 10628, "Ġpreparing": 10629, "Ġactively": 10630, "ĠDiff": 10631, "Ġlux": 10632, "ĠWinter": 10633, "acteria": 10634, "ĠNE": 10635, "Ġdeputy": 10636, "Ġtransgender": 10637, "Ġsummary": 10638, "Ġinher": 10639, "eries": 10640, "char": 10641, "ĠYan": 10642, "Ġknock": 10643, "ĠPath": 10644, "Ġlip": 10645, "roller": 10646, "Ġimpression": 10647, "Ġcelebrate": 10648, "Ġslide": 10649, "Ġguests": 10650, "Ġclip": 10651, "FS": 10652, "Ġsavings": 10653, "Ġcaptain": 10654, "Ġlegacy": 10655, "ĠDenver": 10656, "Ġwounded": 10657, "taboola": 10658, "ACT": 10659, "Ġpursue": 10660, "Ġoxy": 10661, "Ġq": 10662, "Ġsemi": 10663, "ĠNeed": 10664, "ĠAffairs": 10665, "Ġobsc": 10666, "Ġchecked": 10667, "Ġdual": 10668, "Code": 10669, "ĠMD": 10670, "lem": 10671, "ulty": 10672, "Ġ©": 10673, "ĠElizabeth": 10674, "Ġcenturies": 10675, "arded": 10676, "src": 10677, "Ġevident": 10678, "ennis": 10679, "atin": 10680, "Ġunemployment": 10681, "ĠMario": 10682, "Ġintim": 10683, "Christ": 10684, "Ġbiological": 10685, "Ġsoldier": 10686, "ĠAdded": 10687, "Ġmath": 10688, "ĠGil": 10689, "Ġbias": 10690, "Ġdating": 10691, "ĠOcean": 10692, "Ġmice": 10693, "Mus": 10694, "hire": 10695, "ĠTes": 10696, "Server": 10697, "limited": 10698, "Size": 10699, "Ġmeters": 10700, "Ġrocket": 10701, "essee": 10702, "Ġcertificate": 10703, "ĠIranian": 10704, "ASS": 10705, "Ġgrid": 10706, "Dec": 10707, "Ġrolling": 10708, "commun": 10709, "ĠSweden": 10710, "bury": 10711, "Ġtissue": 10712, "Ġracism": 10713, "ĠLocal": 10714, "Ġmystery": 10715, "Ġexamine": 10716, "Ġstem": 10717, "Ġsits": 10718, "Ġhoped": 10719, "oting": 10720, "Ġdialogue": 10721, "Ġpersu": 10722, "Watch": 10723, "lay": 10724, "MAN": 10725, "Ġchronic": 10726, "ĠPortland": 10727, "market": 10728, "ĠSEC": 10729, "Ġparallel": 10730, "Ġscandal": 10731, "Ġcarries": 10732, "Ġphenomenon": 10733, "human": 10734, "acker": 10735, "ĠOx": 10736, "Ġretirement": 10737, "tainment": 10738, "ovie": 10739, "ĠGear": 10740, "Ġduties": 10741, "Ġdose": 10742, "Ġscroll": 10743, "MB": 10744, "inf": 10745, "Ġsauce": 10746, "Ġlandscape": 10747, "reddit": 10748, "ĠChampionship": 10749, "ĠReddit": 10750, "alid": 10751, "Ġcoin": 10752, "Ġovers": 10753, "Ġposting": 10754, "about": 10755, "Ġfel": 10756, "andy": 10757, "Ġbold": 10758, "Ġfocusing": 10759, "effect": 10760, "GR": 10761, "Ġdeemed": 10762, "Ġrecommendations": 10763, "Ġstepped": 10764, "Ġvoter": 10765, "ĠDeep": 10766, "ĠInstagram": 10767, "Ġmoderate": 10768, "ĠMaryland": 10769, "Ġrestricted": 10770, "ĠMB": 10771, "ĠChall": 10772, "Ġtob": 10773, "Ġcir": 10774, "ĠOcc": 10775, "ĠEver": 10776, "Ġcollaps": 10777, "INFO": 10778, "=-": 10779, "ĠPict": 10780, "ĠAccount": 10781, "nc": 10782, "Ġought": 10783, "Ġexport": 10784, "Ġdrunk": 10785, "('": 10786, "Ġwise": 10787, "ĠMort": 10788, "necess": 10789, "Ġancest": 10790, "ĠIncre": 10791, "Ġfrequent": 10792, "mir": 10793, "Ġinterpretation": 10794, "Ġdependent": 10795, "Ġcoins": 10796, "ĠBol": 10797, "Video": 10798, "ĠJustin": 10799, "Ġfatal": 10800, "Ġcooking": 10801, "Ġconfusion": 10802, "ipher": 10803, "Ġcustody": 10804, "ĠMorgan": 10805, "omach": 10806, "ĠGovernor": 10807, "Ġrestaurants": 10808, "eling": 10809, "Ġacknowledged": 10810, "Ġther": 10811, "Ġgenes": 10812, "ching": 10813, "Hey": 10814, "Ġtactics": 10815, "ĠMexican": 10816, "Ġvend": 10817, "Ġhes": 10818, "quer": 10819, "Ġnoting": 10820, "ĠCameron": 10821, "Ġtargeting": 10822, "rock": 10823, "Ġcredits": 10824, "Ġemotions": 10825, "Ġrepresentatives": 10826, "news": 10827, "Ġlegislative": 10828, "Ġremoving": 10829, "Ġtweeted": 10830, "ĠCarter": 10831, "ĠFixed": 10832, "Ġforcing": 10833, "Ġspeaker": 10834, "Ġmales": 10835, "ĠVietnam": 10836, "lined": 10837, "Ġconcepts": 10838, "Ġvoices": 10839, "oir": 10840, "ĠTrib": 10841, "Whe": 10842, "ĠJerusalem": 10843, "ĠSant": 10844, "Ġcul": 10845, "Ġlady": 10846, "ĠHawai": 10847, "Ġarts": 10848, "ĠInn": 10849, "ĠMachine": 10850, "ĠEmperor": 10851, "Ġslot": 10852, "gly": 10853, "ĠProcess": 10854, "III": 10855, "Ġathletes": 10856, "ĠTemple": 10857, "ĠRepresent": 10858, "Ġpresc": 10859, "Ġtons": 10860, "Ġgolden": 10861, "Ġpunch": 10862, "ĠGR": 10863, "iverpool": 10864, "Ġenact": 10865, "Ġlobby": 10866, "Ġmos": 10867, "Ġpicking": 10868, "Ġlifetime": 10869, "Ġcognitive": 10870, "Each": 10871, "zo": 10872, "Ġdub": 10873, "Ġconsists": 10874, "oln": 10875, "Ġfestival": 10876, "amous": 10877, "Ġintellig": 10878, "words": 10879, "ĠSmart": 10880, "Ġdele": 10881, "Ġlapt": 10882, "Ġmagical": 10883, "ĠSin": 10884, "bus": 10885, "urities": 10886, "ighth": 10887, "ĠRuby": 10888, "ĠSure": 10889, "olving": 10890, "Ġjun": 10891, "OST": 10892, "Ġimposed": 10893, "Ġastron": 10894, "Ġcorrel": 10895, "ĠNS": 10896, "ĠKit": 10897, "ĠFuture": 10898, "burn": 10899, "Ġimmune": 10900, "ocus": 10901, "Ġcourses": 10902, "ĠString": 10903, "Ġlean": 10904, "Ġghost": 10905, "Ġoutcomes": 10906, "Ġexpense": 10907, "Ġeveryday": 10908, "Ġacceptable": 10909, "Ah": 10910, "Ġequipped": 10911, "Ġorange": 10912, "FR": 10913, "ĠDutch": 10914, "Though": 10915, "ĠRank": 10916, "QU": 10917, "ĠRoberts": 10918, "what": 10919, "rend": 10920, "Ġdisappear": 10921, "Ġspawn": 10922, "ĠLam": 10923, "ois": 10924, "Ġdeserve": 10925, "Ġminimal": 10926, "Ġnervous": 10927, "ĠWould": 10928, "Ġrook": 10929, "ĠVancouver": 10930, "Ġresign": 10931, "shire": 10932, "ĠWorks": 10933, "ĠBuild": 10934, "Ġaffordable": 10935, "ĠGary": 10936, "ĠArena": 10937, "Ġhanging": 10938, "Ġimplications": 10939, "ĠSong": 10940, "Ġmaintaining": 10941, "Ġguards": 10942, "CON": 10943, "Ġderived": 10944, "Ġexecuted": 10945, "Ġtheories": 10946, "Ġquoted": 10947, "ĠAndre": 10948, "oga": 10949, "seless": 10950, "info": 10951, "ĠBelg": 10952, "Ġtears": 10953, "ĠSurv": 10954, "Ġbirthday": 10955, "igious": 10956, "immer": 10957, "Ġspectrum": 10958, "Ġarchitecture": 10959, "Ġrecruit": 10960, "arma": 10961, "Table": 10962, "Ġmonsters": 10963, "ĠGov": 10964, "Ġdestination": 10965, "Ġattractive": 10966, "Ġfoss": 10967, "ĠMoreover": 10968, "Ġpresents": 10969, "THE": 10970, "Ġreply": 10971, "pton": 10972, "Ġcum": 10973, "Ġdelight": 10974, "Ġaffects": 10975, "Ġdonations": 10976, "ĠToy": 10977, "ĠHim": 10978, "MENT": 10979, "Ġovercome": 10980, "itched": 10981, "ĠFantasy": 10982, "ĠHat": 10983, "ĠBeast": 10984, "bott": 10985, "Ġinvestigations": 10986, "Run": 10987, "Ġhunting": 10988, "di": 10989, "fund": 10990, "Ġsessions": 10991, "estyle": 10992, "Ġportray": 10993, "oids": 10994, "Yeah": 10995, "Ġcommunicate": 10996, "Ġcomedy": 10997, "ĠYang": 10998, "Ġbelt": 10999, "ĠMarine": 11000, "Ġpredicted": 11001, "Play": 11002, "Ġimportantly": 11003, "Ġremarkable": 11004, "Ġeliminate": 11005, "David": 11006, "Ġbind": 11007, "VID": 11008, "Ġadvocates": 11009, "ĠGaza": 11010, "imp": 11011, "DB": 11012, "ĠNa": 11013, "ĠSimilar": 11014, "IES": 11015, "Ġcharity": 11016, "vas": 11017, "math": 11018, "Ġâĸ": 11019, "oker": 11020, "ndum": 11021, "Ġcaps": 11022, "ĠHal": 11023, "2000": 11024, "ean": 11025, "Ġfleet": 11026, "Ġrecre": 11027, "Right": 11028, "Ġsleeping": 11029, "ijing": 11030, "kind": 11031, "Ġdesignated": 11032, "ä": 11033, "Ġanimation": 11034, "kee": 11035, "ĠIntrodu": 11036, "Ġ/>": 11037, "Ġdelayed": 11038, "Ġtremend": 11039, "Ġcurious": 11040, "Use": 11041, "Ġlect": 11042, "dam": 11043, "Ġinnovation": 11044, "ĠPoints": 11045, "Ġloading": 11046, "Ġdispute": 11047, "ctic": 11048, "irds": 11049, "ĠBY": 11050, "Ġnurs": 11051, "ĠValue": 11052, "IONS": 11053, "ĠHum": 11054, "Ġtemplate": 11055, "mers": 11056, "Ġappearances": 11057, "ĠEntertainment": 11058, "Ġtranslation": 11059, "Ġsake": 11060, "Ġbeneath": 11061, "Ġinhib": 11062, "Ġeuro": 11063, "abetes": 11064, "Ġstudying": 11065, "ĠMas": 11066, "Ġperceived": 11067, "Ġexamined": 11068, "Ġeager": 11069, "Ġcoaches": 11070, "Ġimper": 11071, "chi": 11072, "Ġproduces": 11073, "\").": 11074, "ĠEveryone": 11075, "Ġmunicip": 11076, "Ġgirlfriend": 11077, "Ġhire": 11078, "ĠVice": 11079, "Ġsuitable": 11080, "opy": 11081, "Ġinequ": 11082, "ĠDuke": 11083, "fish": 11084, "first": 11085, "ĠObs": 11086, "Ġinterior": 11087, "ĠBruce": 11088, "ĠRy": 11089, "Ġanalys": 11090, "Ġconsiderable": 11091, "Ġforecast": 11092, "Ġfert": 11093, "orship": 11094, "ĠDrug": 11095, "ĠALL": 11096, ":\"": 11097, "thur": 11098, "ĠMail": 11099, "Ġballot": 11100, "Ġinstantly": 11101, "ĠChannel": 11102, "Ġpicks": 11103, "Ġ1989": 11104, "Ġtent": 11105, "oli": 11106, "Ġcivilian": 11107, "bling": 11108, "ello": 11109, "bu": 11110, "Ġinch": 11111, "Ġlogo": 11112, "Ġcooperation": 11113, "Ġwalks": 11114, "Ġinvestments": 11115, "Ġimprison": 11116, "ĠFestival": 11117, "ĠKy": 11118, "Ġlegally": 11119, "Ġgri": 11120, "charg": 11121, "Sl": 11122, "Ġthreatening": 11123, "duction": 11124, "flow": 11125, "Ġdismissed": 11126, "ibraries": 11127, "cap": 11128, "ele": 11129, "ĠMcG": 11130, "ĠHarvard": 11131, "ĠConservative": 11132, "ĠCBS": 11133, "png": 11134, "Ġroots": 11135, "ĠHaving": 11136, "umbled": 11137, "ĠFun": 11138, "\\/": 11139, "ĠSearch": 11140, "plex": 11141, "Ġdiscussing": 11142, "Ġcontinu": 11143, "ĠTai": 11144, "ĠWik": 11145, "Free": 11146, "fit": 11147, "Ġrefuse": 11148, "Ġmanaging": 11149, "Ġsynd": 11150, "ipedia": 11151, "walk": 11152, "Ġprofessionals": 11153, "Ġguidance": 11154, "Ġuniversities": 11155, "Ġassemb": 11156, "untu": 11157, "Finally": 11158, "ASE": 11159, "ĠAuto": 11160, "ĠHad": 11161, "Ġanniversary": 11162, "LD": 11163, "ĠDur": 11164, "ĠUltimate": 11165, "ihad": 11166, "product": 11167, "Ġtransit": 11168, "Ġrestore": 11169, "Ġexplaining": 11170, "Ġasset": 11171, "Ġtransferred": 11172, "Ġburst": 11173, "apolis": 11174, "ĠMagazine": 11175, "ĠCra": 11176, "ĠBR": 11177, "gged": 11178, "ĠHE": 11179, "Mich": 11180, "bet": 11181, "ĠLady": 11182, "ylum": 11183, "erves": 11184, "Ġmeets": 11185, "white": 11186, "Log": 11187, "Ġcorresponding": 11188, "Ġinsisted": 11189, "GG": 11190, "Ġsurrounded": 11191, "Ġtens": 11192, "Ġlane": 11193, "Ġcoinc": 11194, "home": 11195, "Ġexisted": 11196, "ected": 11197, "ĠDouble": 11198, "lamm": 11199, "Ġskept": 11200, "exp": 11201, "Ġperception": 11202, "iev": 11203, "ĠBeing": 11204, "oft": 11205, "Ġadopt": 11206, ".:": 11207, "];": 11208, "Windows": 11209, "Ġsatellite": 11210, "ASH": 11211, "Ġinfant": 11212, "description": 11213, "ĠMeanwhile": 11214, "cm": 11215, "oca": 11216, "ĠTreat": 11217, "actor": 11218, "Ġtobacco": 11219, "ĠNorm": 11220, "emption": 11221, "Ġflesh": 11222, "Ġje": 11223, "oop": 11224, "ĠHeaven": 11225, "Ġbeating": 11226, "anim": 11227, "Ġgathering": 11228, "Ġcultiv": 11229, "GO": 11230, "abe": 11231, "ĠJonathan": 11232, "ĠSafety": 11233, "Ġbadly": 11234, "prot": 11235, "Ġchoosing": 11236, "Ġcontacted": 11237, "Ġquit": 11238, "Ġdistur": 11239, "Ġstir": 11240, "Ġtoken": 11241, "Det": 11242, "ĠPa": 11243, "Ġfunctionality": 11244, "003": 11245, "some": 11246, "Ġlimitations": 11247, "Ġmeth": 11248, "build": 11249, "config": 11250, "NT": 11251, "rell": 11252, "blem": 11253, "ĠMom": 11254, "Ġveterans": 11255, "ĠHu": 11256, "Ġtrends": 11257, "arer": 11258, "ĠGiven": 11259, "ĠCaption": 11260, "may": 11261, "AST": 11262, "Ġwondering": 11263, "ĠClark": 11264, "normal": 11265, "Ġseparated": 11266, "Ġdesp": 11267, "stic": 11268, "brew": 11269, "Ġrelating": 11270, "ĠNik": 11271, "ĠFarm": 11272, "Ġenthusi": 11273, "good": 11274, "deb": 11275, "Ġactivist": 11276, "Ġmart": 11277, "Ġexplosion": 11278, "ĠEconomic": 11279, "Link": 11280, "Ġinsight": 11281, "Ġconvenient": 11282, "Ġcounterpart": 11283, "support": 11284, "ĠVirt": 11285, "agen": 11286, "ĠTennessee": 11287, "ĠSimon": 11288, "ĠAward": 11289, "OCK": 11290, "ĠFigure": 11291, "Ġoverseas": 11292, "Ġpride": 11293, "ĠCas": 11294, "note": 11295, "mg": 11296, "Current": 11297, "Ġdisplays": 11298, "content": 11299, "Ġtraveling": 11300, "Ġhospitals": 11301, "ĠFinancial": 11302, "ĠPast": 11303, "Ġdefendant": 11304, "Ġstreaming": 11305, "mble": 11306, "ĠBerlin": 11307, "uki": 11308, "Ġdistribut": 11309, "Ġantib": 11310, "Ġchocolate": 11311, "ĠCastle": 11312, "Ġinterrupt": 11313, "ĠRow": 11314, "Ġconversion": 11315, "Ġbugs": 11316, "ĠRather": 11317, "liest": 11318, "LY": 11319, "ĠJean": 11320, "common": 11321, "akh": 11322, "Ġ130": 11323, "otton": 11324, "ĠDean": 11325, "Ġamendment": 11326, "Ġgameplay": 11327, "ĠWarren": 11328, "oda": 11329, "Ġhighlights": 11330, "Ġirre": 11331, "ĠNATO": 11332, "Ġballs": 11333, "Ġdemanding": 11334, "URE": 11335, "ĠLuke": 11336, "Figure": 11337, "stop": 11338, "onia": 11339, "zone": 11340, "izers": 11341, "ĠWR": 11342, "Ġawarded": 11343, "Ġregulatory": 11344, "ĠHart": 11345, "ĠSN": 11346, "pling": 11347, "Ġsour": 11348, "ĠPixel": 11349, "usive": 11350, "Ġfet": 11351, "ĠSent": 11352, "Ġautomatic": 11353, "Ġfer": 11354, "vernment": 11355, "ĠKhan": 11356, "TON": 11357, "father": 11358, "Ġextraordinary": 11359, "throp": 11360, "ĠPython": 11361, "ĠGPU": 11362, "Ġsexually": 11363, "Ġdesktop": 11364, "itivity": 11365, "ĠAntonio": 11366, "Ġorient": 11367, "Ġears": 11368, "obby": 11369, "ouses": 11370, "vertisements": 11371, "Ġmanufacturers": 11372, "icient": 11373, "minute": 11374, "Ġconviction": 11375, "Ġgarden": 11376, "public": 11377, "Ġsatisfied": 11378, "fold": 11379, "OK": 11380, "Ġinhab": 11381, "ĠThink": 11382, "Ġprogramme": 11383, "Ġstomach": 11384, "Ġcoordin": 11385, "Ġholy": 11386, "Ġthreshold": 11387, "Ġrhet": 11388, "Ġserial": 11389, "Ġemployers": 11390, "ĠEverything": 11391, "rah": 11392, "Ġbother": 11393, "Ġbrands": 11394, "Value": 11395, "ĠTed": 11396, "ĠPlanet": 11397, "Ġpink": 11398, "ĠFurthermore": 11399, "sa": 11400, "PE": 11401, "reck": 11402, "ĠUSD": 11403, "otte": 11404, "Ġ&&": 11405, "Ġlanded": 11406, "gets": 11407, "Ġproducers": 11408, "Ġhealthcare": 11409, "Ġdominant": 11410, "Ġdestro": 11411, "Ġamended": 11412, "chron": 11413, "Ġfits": 11414, "ĠSyd": 11415, "ĠAuthority": 11416, "ATCH": 11417, "Ġfights": 11418, "ĠLLC": 11419, "Ġ---": 11420, "ĠCorp": 11421, "Ġtoxic": 11422, "specific": 11423, "ĠCorn": 11424, "ĠChel": 11425, "Ġtelephone": 11426, "ĠPant": 11427, "Ġmysterious": 11428, "aunch": 11429, "odox": 11430, "media": 11431, "Ġwitnesses": 11432, "agu": 11433, "Ġquestioned": 11434, "ĠBrexit": 11435, "ĠRemember": 11436, "enez": 11437, "Ġendorse": 11438, "iatric": 11439, "ĠIdent": 11440, "Ġridiculous": 11441, "110": 11442, "Ġprayer": 11443, "Ġscientist": 11444, "Ġ1950": 11445, "ĠAqu": 11446, "Ġunderground": 11447, "ĠUFC": 11448, "mare": 11449, "ĠLater": 11450, "wich": 11451, "Ġsubscrib": 11452, "Ġhosts": 11453, "Ġerr": 11454, "Ġgrants": 11455, "antom": 11456, "Ġsummon": 11457, "early": 11458, "ĠClear": 11459, "ĠPrim": 11460, "Ġsuspension": 11461, "Ġguaranteed": 11462, "apper": 11463, "Ġrice": 11464, "ĠSean": 11465, "ĠShin": 11466, "Ġreferendum": 11467, "Ġfled": 11468, "rust": 11469, "Ġ360": 11470, "tery": 11471, "Ġshocked": 11472, "BR": 11473, "ĠOil": 11474, "ĠAllah": 11475, "Ġpartly": 11476, "Ġignor": 11477, "Ġtransmission": 11478, "Ġhomosexual": 11479, "iversal": 11480, "Ġhopefully": 11481, "ãĤ¤": 11482, "Ġlesson": 11483, "Leg": 11484, "Ġ..": 11485, "Yet": 11486, "table": 11487, "appropri": 11488, "rett": 11489, "Ġboards": 11490, "Ġincorrect": 11491, "Ġbacteria": 11492, "aru": 11493, "amac": 11494, "Ġsnap": 11495, ".'\"": 11496, "Ġparad": 11497, "tem": 11498, "heart": 11499, "Ġavailability": 11500, "Ġwisdom": 11501, "Ġ(+": 11502, "Ġpriest": 11503, "ĠÂłĠÂł": 11504, "Open": 11505, "Ġspan": 11506, "Ġparameter": 11507, "Ġconvince": 11508, "Ġ(%)": 11509, "rac": 11510, "Ġfo": 11511, "Ġsafely": 11512, "Ġconverted": 11513, "ĠOlympic": 11514, "Ġreserve": 11515, "Ġhealing": 11516, "ĠMine": 11517, "Max": 11518, "Ġinherent": 11519, "ĠGraham": 11520, "Ġintegrated": 11521, "Dem": 11522, "Ġpipeline": 11523, "Ġapplying": 11524, "Ġembed": 11525, "ĠCharlie": 11526, "Ġcave": 11527, "2008": 11528, "Ġconsensus": 11529, "Ġrewards": 11530, "Pal": 11531, "ĠHTML": 11532, "Ġpopularity": 11533, "looking": 11534, "ĠSword": 11535, "ĠArts": 11536, "')": 11537, "Ġelectron": 11538, "clusions": 11539, "Ġintegrity": 11540, "Ġexclusively": 11541, "Ġgrace": 11542, "Ġtorture": 11543, "Ġburned": 11544, "two": 11545, "Ġ180": 11546, "Produ": 11547, "Ġentreprene": 11548, "raphics": 11549, "Ġgym": 11550, "ricane": 11551, "ĠTam": 11552, "Ġadministrative": 11553, "Ġmanufacturer": 11554, "Ġvel": 11555, "ĠNi": 11556, "Ġisolated": 11557, "ĠMedicine": 11558, "Ġbackup": 11559, "Ġpromoting": 11560, "Ġcommander": 11561, "Ġflee": 11562, "ĠRussell": 11563, "Ġforgotten": 11564, "ĠMissouri": 11565, "Ġresidence": 11566, "mons": 11567, "Ġresemb": 11568, "Ġwand": 11569, "Ġmeaningful": 11570, "PT": 11571, "Ġbol": 11572, "Ġhelic": 11573, "Ġwealthy": 11574, "Ġrifle": 11575, "strong": 11576, "rowing": 11577, "plan": 11578, "asury": 11579, "âĢ¦.": 11580, "Ġexpanding": 11581, "ĠHamilton": 11582, "Ġreceives": 11583, "SI": 11584, "eatures": 11585, "ĠAnim": 11586, "REE": 11587, "Put": 11588, "Ġbriefly": 11589, "rive": 11590, "Ġstimul": 11591, "Ġ``(": 11592, "Ġ__": 11593, "Ġchip": 11594, "Ġhaz": 11595, "Ġprize": 11596, "ĠThings": 11597, "ACE": 11598, "ulin": 11599, "dict": 11600, "oku": 11601, "Ġassociate": 11602, "ockets": 11603, "youtube": 11604, "Story": 11605, "ategory": 11606, "Ġmild": 11607, "ailing": 11608, "ĠYe": 11609, "Orig": 11610, "ĠKa": 11611, "orig": 11612, "Ġpropaganda": 11613, "Ġanonymous": 11614, "Ġstruggled": 11615, "Ġoutrage": 11616, "ATED": 11617, "ĠBeijing": 11618, "rary": 11619, "Ġleather": 11620, "Ġworlds": 11621, "Ġbroader": 11622, "125": 11623, "idal": 11624, "ĠBetter": 11625, "Ġtear": 11626, "Ext": 11627, "Ġproposals": 11628, "Ġiter": 11629, "ĠSquad": 11630, "Ġvolunt": 11631, "mi": 11632, "Did": 11633, "ĠPu": 11634, "pin": 11635, "Ġspeakers": 11636, "Ġborders": 11637, "Ġfigured": 11638, "='": 11639, "Ġsimultaneously": 11640, "aeda": 11641, "Ġcharging": 11642, "Ġurged": 11643, "Ġconj": 11644, "256": 11645, "ĠGordon": 11646, "merce": 11647, "Ġdocumentary": 11648, "Share": 11649, "itol": 11650, "ONE": 11651, "ĠGarden": 11652, "hatt": 11653, "ĠThompson": 11654, "aneous": 11655, "apore": 11656, "Ġtanks": 11657, "Ġlessons": 11658, "track": 11659, "Ġoutstanding": 11660, "Ġvolunteers": 11661, "Ġspray": 11662, "Ġmanagers": 11663, "large": 11664, "Ġcamps": 11665, "Ġartificial": 11666, "ĠRu": 11667, "Ġbags": 11668, "thal": 11669, "Ġcompatible": 11670, "ĠBlade": 11671, "Ġfed": 11672, "Ġargues": 11673, "FI": 11674, "Ġunfair": 11675, "Ġcorn": 11676, "Ġoffset": 11677, "Ġdirections": 11678, "Ġdisappointed": 11679, "ĠConvention": 11680, "Ġviewing": 11681, "ME": 11682, "ocity": 11683, "Ġtowns": 11684, "Ġlayers": 11685, "Ġrolled": 11686, "Ġjumped": 11687, "Ġattribute": 11688, "Ġunnecess": 11689, "incoln": 11690, "Ġsuppose": 11691, "ĠNether": 11692, "cha": 11693, "Ġburied": 11694, "Ġsixth": 11695, "Ben": 11696, "ressing": 11697, "OUR": 11698, "Ġwound": 11699, "Ġcycl": 11700, "Ġmechanisms": 11701, "Ġcongressional": 11702, "ĠElement": 11703, "Ġagreements": 11704, "Ġdecor": 11705, "Ġclosest": 11706, "ĠMit": 11707, "Google": 11708, "}}": 11709, "Ġmixture": 11710, "Ġfluid": 11711, "Sign": 11712, "ĠScholar": 11713, "Ġpist": 11714, "asket": 11715, "abling": 11716, "Ġracing": 11717, "hero": 11718, "riel": 11719, "assy": 11720, "Ġcheaper": 11721, "ben": 11722, "Ġvertical": 11723, "amacare": 11724, "ĠReading": 11725, "gments": 11726, "Ġhelicop": 11727, "Ġsacrifice": 11728, "aya": 11729, "paren": 11730, "VA": 11731, "ĠLes": 11732, "ĠStudio": 11733, "Ġviolations": 11734, "ĠAnna": 11735, "acer": 11736, "é¾": 11737, "ĠRat": 11738, "ĠBeck": 11739, "ĠDick": 11740, "ĠACT": 11741, "Ġcomposition": 11742, "Ġtexture": 11743, "ĠOwn": 11744, "Ġsmartphone": 11745, "ĠNA": 11746, "Ġforb": 11747, "import": 11748, "Ġdefending": 11749, "ilst": 11750, "rer": 11751, "Ġoh": 11752, "ĠJeremy": 11753, "Ġbanking": 11754, "ceptions": 11755, "Ġrespective": 11756, "/.": 11757, "Ġdrinks": 11758, "ĠWi": 11759, "Ġbands": 11760, "ĠLiverpool": 11761, "Ġgrip": 11762, "ĠBuy": 11763, "Ġopenly": 11764, "Ġreviewed": 11765, "pert": 11766, "Ġverify": 11767, "ĠCole": 11768, "ĠWales": 11769, "MO": 11770, "Ġunpre": 11771, "Ġshelter": 11772, "ĠImperial": 11773, "Ġgui": 11774, "ĠDak": 11775, "Ġsuggestions": 11776, "Ġexplicitly": 11777, "Ġslave": 11778, "Ġblockchain": 11779, "Ġcompeting": 11780, "Ġpromising": 11781, "SON": 11782, "Ġsoccer": 11783, "Ġconstitution": 11784, "429": 11785, "Ġdistract": 11786, "ĠUser": 11787, "esides": 11788, "ĠMethod": 11789, "ĠTokyo": 11790, "Ġaccompanied": 11791, "Client": 11792, "sur": 11793, "alog": 11794, "Ġidentification": 11795, "Ġinvasion": 11796, "asma": 11797, "Ġindustries": 11798, "ppers": 11799, "Ġsubtle": 11800, "ĠUnit": 11801, "natural": 11802, "Ġsurvived": 11803, "Ġflaw": 11804, "ĺħ": 11805, "ĠHoll": 11806, "Ġdeficit": 11807, "Ġtutorial": 11808, "ĠChance": 11809, "Ġarguing": 11810, "Ġcontemporary": 11811, "Ġintegration": 11812, "forward": 11813, "Ġtum": 11814, "itis": 11815, "Ġhiding": 11816, "ĠDomin": 11817, "ĠTan": 11818, "ĠBuilding": 11819, "ĠVin": 11820, "Ġspokesperson": 11821, "ĠNotes": 11822, "Ġemerging": 11823, "Ġpreparation": 11824, "Ġprost": 11825, "Ġsuspects": 11826, "Ġautonom": 11827, "Description": 11828, "Ġdealt": 11829, "ĠPear": 11830, "Ġsteady": 11831, "Ġdecreased": 11832, "Ġsovere": 11833, "ĠClin": 11834, "Ġgradually": 11835, "orses": 11836, "ĠWAR": 11837, "Serv": 11838, "ãĤ¢": 11839, "hr": 11840, "Ġdirty": 11841, "ĠBarn": 11842, "ĠBC": 11843, "Ġdil": 11844, "Ġcalendar": 11845, "Ġcompliance": 11846, "Ġchamber": 11847, "bb": 11848, "Ġpassenger": 11849, "ateful": 11850, "ĠTitle": 11851, "ĠSydney": 11852, "ĠGot": 11853, "Ġdarkness": 11854, "Ġdefect": 11855, "Ġpacked": 11856, "assion": 11857, "Ġgods": 11858, "Ġharsh": 11859, "ICK": 11860, "leans": 11861, "Ġalgorithm": 11862, "Ġoxygen": 11863, "Ġvisits": 11864, "Ġblade": 11865, "Ġkilomet": 11866, "ĠKentucky": 11867, "Ġkiller": 11868, "Pack": 11869, "enny": 11870, "Ġdivine": 11871, "Ġnomination": 11872, "being": 11873, "Ġengines": 11874, "Ġcats": 11875, "Ġbuffer": 11876, "ĠPhill": 11877, "Ġtraff": 11878, "AGE": 11879, "Ġtongue": 11880, "Ġradiation": 11881, "erer": 11882, "mem": 11883, "ĠExplicit": 11884, "é¾į": 11885, "Ġcouples": 11886, "Ġphysics": 11887, "ĠMcK": 11888, "Ġpolitically": 11889, "awks": 11890, "ĠBloom": 11891, "Ġworship": 11892, "eger": 11893, "uter": 11894, "ĠFO": 11895, "Ġmathemat": 11896, "Ġsentenced": 11897, "Ġdisk": 11898, "ĠMarg": 11899, "Ġ/*": 11900, "PI": 11901, "Ġoptional": 11902, "Ġbabies": 11903, "Ġseeds": 11904, "ĠScottish": 11905, "Ġthy": 11906, "]]": 11907, "ĠHitler": 11908, "PH": 11909, "ngth": 11910, "Ġrecovered": 11911, "inge": 11912, "Ġpowder": 11913, "Ġlips": 11914, "Ġdesigner": 11915, "Ġdisorders": 11916, "Ġcourage": 11917, "Ġchaos": 11918, "\"},{\"": 11919, "Ġcarrier": 11920, "bably": 11921, "High": 11922, "ĠRT": 11923, "esity": 11924, "len": 11925, "Ġroutes": 11926, "uating": 11927, "Fil": 11928, "NOT": 11929, "wall": 11930, "sburgh": 11931, "Ġengaging": 11932, "ĠJavaScript": 11933, "orer": 11934, "lihood": 11935, "Ġunions": 11936, "ĠFederation": 11937, "ĠTesla": 11938, "Ġcompletion": 11939, "ĠTa": 11940, "Ġprivilege": 11941, "ĠOrange": 11942, "Ġneur": 11943, "parency": 11944, "Ġbones": 11945, "Ġtitled": 11946, "Ġprosecutors": 11947, "ĠME": 11948, "Ġengineer": 11949, "ĠUniverse": 11950, "ĠHig": 11951, "nie": 11952, "oard": 11953, "Ġhearts": 11954, "ĠGre": 11955, "ussion": 11956, "Ġministry": 11957, "Ġpenet": 11958, "ĠNut": 11959, "ĠOw": 11960, "ĠXP": 11961, "instein": 11962, "Ġbulk": 11963, "System": 11964, "icism": 11965, "ĠMarketable": 11966, "Ġpreval": 11967, "Ġposter": 11968, "Ġattending": 11969, "urable": 11970, "Ġlicensed": 11971, "ĠGh": 11972, "etry": 11973, "ĠTradable": 11974, "Ġblast": 11975, "à¤": 11976, "ĠTitan": 11977, "elled": 11978, "die": 11979, "Have": 11980, "ĠFlame": 11981, "Ġprofound": 11982, "Ġparticipating": 11983, "Ġanime": 11984, "ĠEss": 11985, "Ġspecify": 11986, "Ġregarded": 11987, "ĠSpell": 11988, "Ġsons": 11989, "owned": 11990, "Ġmerc": 11991, "Ġexperimental": 11992, "lando": 11993, "hs": 11994, "ĠDungeon": 11995, "inos": 11996, "Ġcomply": 11997, "ĠSystems": 11998, "arth": 11999, "Ġseized": 12000, "local": 12001, "ĠGirls": 12002, "udo": 12003, "oned": 12004, "ĠFle": 12005, "Ġconstructed": 12006, "Ġhosted": 12007, "Ġscared": 12008, "actic": 12009, "ĠIslands": 12010, "ĠMORE": 12011, "Ġbless": 12012, "Ġblocking": 12013, "Ġchips": 12014, "Ġevac": 12015, "Ps": 12016, "Ġcorporation": 12017, "Ġox": 12018, "Ġlighting": 12019, "Ġneighbors": 12020, "ĠUb": 12021, "aro": 12022, "Ġbeef": 12023, "ĠUber": 12024, "Facebook": 12025, "armed": 12026, "itate": 12027, "ĠRating": 12028, "ĠQuick": 12029, "Ġoccupied": 12030, "Ġaims": 12031, "ĠAdditionally": 12032, "ĠInterest": 12033, "Ġdramatically": 12034, "Ġheal": 12035, "Ġpainting": 12036, "Ġengineers": 12037, "MM": 12038, "ĠMust": 12039, "Ġquantity": 12040, "Paul": 12041, "Ġearnings": 12042, "ĠPosts": 12043, "stra": 12044, "ãĥ¼ãĥ": 12045, "Ġstance": 12046, "Ġdropping": 12047, "script": 12048, "Ġdressed": 12049, "Make": 12050, "Ġjustify": 12051, "ĠLtd": 12052, "Ġprompted": 12053, "Ġscrut": 12054, "Ġspeeds": 12055, "ĠGiants": 12056, "omer": 12057, "ĠEditor": 12058, "Ġdescribing": 12059, "ĠLie": 12060, "mented": 12061, "Ġnowhere": 12062, "ocaly": 12063, "Ġinstruction": 12064, "fortable": 12065, "Ġentities": 12066, "Ġcm": 12067, "ĠNatural": 12068, "Ġinquiry": 12069, "Ġpressed": 12070, "izont": 12071, "forced": 12072, "Ġraises": 12073, "ĠNetflix": 12074, "ĠSide": 12075, "Ġouter": 12076, "Ġamongst": 12077, "ims": 12078, "owski": 12079, "Ġclimb": 12080, "never": 12081, "Ġcombine": 12082, "ding": 12083, "Ġcompr": 12084, "Ġsignificance": 12085, "Ġremembered": 12086, "ĠNevada": 12087, "ĠTel": 12088, "ĠScar": 12089, "ĠWarriors": 12090, "ĠJane": 12091, "Ġcoup": 12092, "bas": 12093, "Ġterminal": 12094, ",-": 12095, "OH": 12096, "Ġtension": 12097, "Ġwings": 12098, "ĠMyster": 12099, "����": 12100, "ĠUnlike": 12101, "valid": 12102, "vironments": 12103, "ĠAli": 12104, "Ġnaked": 12105, "books": 12106, "ĠMun": 12107, "ĠGulf": 12108, "Ġdensity": 12109, "Ġdimin": 12110, "Ġdesperate": 12111, "Ġpresidency": 12112, "Ġ1986": 12113, "hy": 12114, "IND": 12115, "Ġunlock": 12116, "imens": 12117, "Ġhandled": 12118, "ĠEb": 12119, "Ġdisappeared": 12120, "Ġgenre": 12121, "Ġ1988": 12122, "Ġdetermination": 12123, "Stream": 12124, "iko": 12125, "apters": 12126, "Ġacknowledge": 12127, "Jan": 12128, "Ġcapitalism": 12129, "Pat": 12130, "Ġ2020": 12131, "Ġpainful": 12132, "Ġcurve": 12133, "Ġbombs": 12134, "storm": 12135, "ĠMetal": 12136, "encer": 12137, "ĠFig": 12138, "ĠAaron": 12139, "anches": 12140, "Ġinspiration": 12141, "Ġexhaust": 12142, "tains": 12143, "ashi": 12144, "Ġdescript": 12145, "Ġritual": 12146, "ĠChelsea": 12147, "Ġpromotion": 12148, "ĠHung": 12149, "ĠWard": 12150, "iva": 12151, "ĠET": 12152, "Ġtoss": 12153, "allow": 12154, "ĠFrancis": 12155, "Dep": 12156, "Ġhappiness": 12157, "ĠGlass": 12158, "Ġbeta": 12159, "Ġstrengthen": 12160, "NE": 12161, "oa": 12162, "Ġbuttons": 12163, "ĠMurray": 12164, "Ġkicked": 12165, "Quest": 12166, "ĠTalk": 12167, "ĠSeveral": 12168, "ĠZero": 12169, "Ġdrone": 12170, "ulk": 12171, "Ġcam": 12172, "ĠMobile": 12173, "Ġpreventing": 12174, "Ġretro": 12175, "ĠAx": 12176, "Ġcruel": 12177, "Ġfloat": 12178, ".),": 12179, "Ġfiling": 12180, "ĠGrant": 12181, "ĠBor": 12182, "Ġrib": 12183, "Ġchampionship": 12184, "ĠMerc": 12185, "Ġstyles": 12186, "Ġcake": 12187, "Ġbuilds": 12188, "ĠSelf": 12189, "iox": 12190, "Ġepic": 12191, "oyd": 12192, "Bel": 12193, "ĠStew": 12194, ".(": 12195, "ahu": 12196, "ĠBeyond": 12197, "Ġouts": 12198, "Ġsolo": 12199, "ĠTree": 12200, "Ġpreserve": 12201, "Ġtub": 12202, "ARE": 12203, "roc": 12204, "ĠImpro": 12205, "ĠWright": 12206, "Ġbund": 12207, "Ġtraged": 12208, "Ġoccasional": 12209, "bian": 12210, "Second": 12211, "rons": 12212, "Ġinteractions": 12213, "formed": 12214, "sing": 12215, "Ġowns": 12216, "Ġhockey": 12217, "General": 12218, "Ġlogical": 12219, "Ġexpend": 12220, "Ġescal": 12221, "ĠGriff": 12222, "ĠCrown": 12223, "ĠReserve": 12224, "Ġstopping": 12225, "Ġexcuse": 12226, "second": 12227, "Ġoperated": 12228, "Ġreaches": 12229, "ĠMalays": 12230, "Ġpollution": 12231, "ĠBrooklyn": 12232, "Ġdelete": 12233, "Ġhash": 12234, "Block": 12235, "aha": 12236, "âĢ³": 12237, "Ġshorter": 12238, "piece": 12239, ">>>": 13163, "ĠMormon": 13164, "tor": 13165, "Ġparticles": 13166, "ĠBart": 13167, "ryption": 13168, "Ġadmin": 13169, "Ġsquee": 13170, "VIDIA": 13171, "Ġcreator": 13172, "iameter": 13173, "icular": 13174, "NBC": 13175, "Ġgrabbed": 13176, "Ġnodd": 13177, "Ġrated": 13178, "Ġrotation": 13179, "Ġgrasp": 13180, "Ġexcessive": 13181, "ĠEC": 13182, "ĠWhit": 13183, "Ġinventory": 13184, "aults": 13185, "ĠFB": 13186, "Ġecosystem": 13187, "Ġbillions": 13188, "Ġventure": 13189, "named": 13190, "Ġdefender": 13191, "oute": 13192, "Instead": 13193, "irable": 13194, "War": 13195, "Ġassumption": 13196, "Ġbite": 13197, "Ġearthqu": 13198, "tail": 13199, "space": 13200, "Ġgifts": 13201, "boys": 13202, "Ġinevitable": 13203, "Ġstructural": 13204, "Ġbeneficial": 13205, "Ġcompelling": 13206, "hole": 13207, "ervation": 13208, "Ġcoat": 13209, "oj": 13210, "incarn": 13211, "ĠYears": 13212, "Ġdetermining": 13213, "Ġrhetoric": 13214, "Ġboundaries": 13215, "Ġwhites": 13216, "Ant": 13217, "addy": 13218, ")-": 13219, "raham": 13220, "etermin": 13221, "Ġharvest": 13222, "ĠConc": 13223, "Ġlaptop": 13224, "ĠMatch": 13225, "Ġenjoying": 13226, "cca": 13227, "ollar": 13228, "Ġtrips": 13229, "Ġaddiction": 13230, "ĠSak": 13231, "Ġpowered": 13232, "Ġcous": 13233, "ĠRussians": 13234, "iere": 13235, "Ġretrie": 13236, "quality": 13237, "Ġdiffer": 13238, "Ġkingdom": 13239, "ĠLaur": 13240, "ĠCapitol": 13241, "Ġconclusions": 13242, "ĠAltern": 13243, "ĠNav": 13244, "Ġtransparent": 13245, "BER": 13246, "Group": 13247, "ĠComplete": 13248, "Ġinfer": 13249, "Ġintrig": 13250, "Ġinsane": 13251, "RO": 13252, "ophob": 13253, "isen": 13254, "qual": 13255, "Michael": 13256, "Ġmuseum": 13257, "ĠPope": 13258, "Ġreset": 13259, "rative": 13260, "five": 13261, "Ġaggreg": 13262, "ittees": 13263, "ository": 13264, "Ġcarb": 13265, "ĠRecord": 13266, "Ġdecides": 13267, "ĠFix": 13268, "Ġexceptions": 13269, "ĠCommissioner": 13270, "uns": 13271, "ĠEnvironmental": 13272, "Ġlegendary": 13273, "istence": 13274, "Ġtunnel": 13275, "km": 13276, "Ġinsult": 13277, "Ġtroll": 13278, "Ġshake": 13279, "Ġdetention": 13280, "ques": 13281, "ĠChrome": 13282, "ĠFiles": 13283, "Ġsubt": 13284, "Ġprospects": 13285, "Ġprol": 13286, "render": 13287, "proof": 13288, "Ġperformances": 13289, "Str": 13290, "Ġhref": 13291, "ername": 13292, "Ġachievement": 13293, "Ġfut": 13294, "Full": 13295, "ĠLeban": 13296, "google": 13297, "ãĥĪ": 13298, "ampa": 13299, "Maybe": 13300, "Ġprojected": 13301, "ĠEmb": 13302, "Ġcolleg": 13303, "Ġawards": 13304, "ĠâĶ": 13305, "Gold": 13306, "ĠBlake": 13307, "ĠRaj": 13308, "ifting": 13309, "Ġpending": 13310, "Ġinstinct": 13311, "Ġdevelopments": 13312, "Connect": 13313, "ĠMand": 13314, "ĠWITH": 13315, "ĠPhilippines": 13316, "profile": 13317, "Ġaltogether": 13318, "ĠBund": 13319, "ĠTD": 13320, "oooo": 13321, "amped": 13322, "iph": 13323, "Ġsteam": 13324, "Ġoldest": 13325, "Ġdetection": 13326, "ulpt": 13327, "Ġç": 13328, "ĠWayne": 13329, "2006": 13330, "fa": 13331, "Ġcircles": 13332, "ĠFu": 13333, "Ġdonors": 13334, "appropriate": 13335, "ĠDakota": 13336, "jamin": 13337, "Ġmotivated": 13338, "Ġpurchases": 13339, "ĠLouisiana": 13340, "ĠSpl": 13341, "Ġglobe": 13342, "Ġ105": 13343, "zip": 13344, "call": 13345, "Ġdepartments": 13346, "Ġsustainable": 13347, "105": 13348, "ĠOP": 13349, "ifiers": 13350, "Ġprevented": 13351, "Ġincomp": 13352, "ĠCommander": 13353, "Ġdominated": 13354, "Ġ»": 13355, "Ġinvested": 13356, "Ġcomplexity": 13357, "Ġincl": 13358, "Ġensuring": 13359, "Ġrealm": 13360, "ync": 13361, "ĠIndependent": 13362, "rained": 13363, "ĠJen": 13364, "ĠFlight": 13365, "Ġathe": 13366, "Ġspeculation": 13367, "ĠTE": 13368, "ocate": 13369, "tic": 13370, "Ġplaint": 13371, "herry": 13372, "Ġtoy": 13373, "Ġ111": 13374, "Ġplates": 13375, "status": 13376, "ĠIsa": 13377, "Ġdevoted": 13378, "Cop": 13379, "ĠES": 13380, "255": 13381, "urrency": 13382, "Main": 13383, "Ġslaves": 13384, "Ġpepper": 13385, "Ġquotes": 13386, "Ġceiling": 13387, "ĠFish": 13388, "Ġtransformation": 13389, "Ġfraction": 13390, "Ġadvantages": 13391, "Ġtoile": 13392, "Ġstunning": 13393, "Ġmoist": 13394, "breaking": 13395, "si": 13396, "ĠLocation": 13397, "ĠMedium": 13398, "Ġtexts": 13399, "Ġugly": 13400, "Ġbio": 13401, ".âĢĶ": 13402, "ĠBased": 13403, "Ġtrains": 13404, "ĠWing": 13405, "ĠAncient": 13406, "ĠRecords": 13407, "ĠHope": 13408, "Special": 13409, "adesh": 13410, "obi": 13411, "[/": 13412, "Ġtemporarily": 13413, "Ver": 13414, "hu": 13415, "oser": 13416, "Ġovernight": 13417, "Ġmamm": 13418, "ĠTreasury": 13419, "ĠVenezuel": 13420, "ĠMega": 13421, "Ġtar": 13422, "Ġexpects": 13423, "black": 13424, "orph": 13425, "\\\\\\\\": 13426, "Ġacceptance": 13427, "Ġradar": 13428, "sis": 13429, "Ġjunior": 13430, "Ġframes": 13431, "Ġobservation": 13432, "acies": 13433, "Power": 13434, "ĠAdvanced": 13435, "Mag": 13436, "ologically": 13437, "ĠMechan": 13438, "Ġsentences": 13439, "Ġanalysts": 13440, "aughters": 13441, "forcement": 13442, "Ġvague": 13443, "Ġclause": 13444, "Ġdirectors": 13445, "Ġevaluate": 13446, "Ġcabinet": 13447, "Matt": 13448, "ĠClassic": 13449, "Ang": 13450, "Ġcler": 13451, "ĠBuck": 13452, "Ġresearcher": 13453, "Ġ160": 13454, "Ġpoorly": 13455, "Ġexperiencing": 13456, "ĠPed": 13457, "ĠManhattan": 13458, "Ġfreed": 13459, "Ġthemes": 13460, "advant": 13461, "Ġnin": 13462, "Ġpraise": 13463, "104": 13464, "ĠLibya": 13465, "best": 13466, "Ġtrusted": 13467, "Ġcease": 13468, "Ġdign": 13469, "Direct": 13470, "Ġbombing": 13471, "Ġmigration": 13472, "ĠSciences": 13473, "Ġmunicipal": 13474, "ĠAverage": 13475, "Ġglory": 13476, "Ġrevealing": 13477, "Ġarena": 13478, "Ġuncertainty": 13479, "Ġbattlefield": 13480, "iao": 13481, "God": 13482, "Ġcinem": 13483, "rape": 13484, "elle": 13485, "apons": 13486, "Ġlisting": 13487, "Ġwaited": 13488, "Ġspotted": 13489, "keley": 13490, "ĠAudio": 13491, "eor": 13492, "arding": 13493, "idding": 13494, "igma": 13495, "ĠNeg": 13496, "Ġlone": 13497, "Ġ----": 13498, "exe": 13499, "deg": 13500, "Ġtransf": 13501, "Ġwash": 13502, "Ġslavery": 13503, "Ġexploring": 13504, "ĠWW": 13505, "atson": 13506, "Ġencl": 13507, "lies": 13508, "ĠCreek": 13509, "Ġwooden": 13510, "Manager": 13511, "ĠBrand": 13512, "ummy": 13513, "ĠArthur": 13514, "Ġbureaucr": 13515, "Ġblend": 13516, "arians": 13517, "Further": 13518, "Ġsupposedly": 13519, "Ġwinds": 13520, "Ġ1979": 13521, "Ġgravity": 13522, "Ġanalyses": 13523, "ĠTravel": 13524, "ĠVeter": 13525, "Ġdumb": 13526, "Ġalternate": 13527, "gal": 13528, "Ġconsumed": 13529, "Ġeffectiveness": 13530, ".''": 13531, "Ġpaths": 13532, "onda": 13533, "LA": 13534, "ĠStrong": 13535, "Ġenables": 13536, "Ġescaped": 13537, "Ġ\"\"": 13538, "Ġ112": 13539, "Ġ1983": 13540, "Ġsmiled": 13541, "Ġtendency": 13542, "Fire": 13543, "Ġpars": 13544, "ĠRoc": 13545, "Ġlake": 13546, "Ġfitness": 13547, "ĠAth": 13548, "ĠHorn": 13549, "Ġhier": 13550, "Ġimpose": 13551, "mother": 13552, "Ġpension": 13553, "icut": 13554, "borne": 13555, "iciary": 13556, "._": 13557, "ĠSU": 13558, "Ġpolar": 13559, "isy": 13560, "engu": 13561, "itialized": 13562, "ATA": 13563, "write": 13564, "Ġexercises": 13565, "ĠDiamond": 13566, "otypes": 13567, "Ġharmful": 13568, "onz": 13569, "Ġprinting": 13570, "story": 13571, "Ġexpertise": 13572, "ĠGer": 13573, "Ġtragedy": 13574, "ĠFly": 13575, "Ġdivid": 13576, "ampire": 13577, "stock": 13578, "Mem": 13579, "Ġreign": 13580, "Ġunve": 13581, "Ġamend": 13582, "ĠProphet": 13583, "Ġmutual": 13584, "ĠFac": 13585, "Ġreplacing": 13586, "Har": 13587, "ĠCircuit": 13588, "Ġthroat": 13589, "ĠShot": 13590, "Ġbatteries": 13591, "Ġtoll": 13592, "Ġaddressing": 13593, "ĠMedicaid": 13594, "Ġpupp": 13595, "ĠNar": 13596, "olk": 13597, "Ġequity": 13598, "MR": 13599, "ĠHispan": 13600, "ĠLarge": 13601, "mid": 13602, "Dev": 13603, "Ġexped": 13604, "Ġdemo": 13605, "ĠMarshall": 13606, "ergus": 13607, "Ġfiber": 13608, "Ġdivorce": 13609, "ĠCreate": 13610, "Ġslower": 13611, "ĠParker": 13612, "ĠStudent": 13613, "ĠTraining": 13614, "Return": 13615, "ĠTru": 13616, "Ġcub": 13617, "ĠReached": 13618, "Ġpanic": 13619, "Ġquarters": 13620, "Ġrect": 13621, "Ġtreating": 13622, "Ġrats": 13623, "ĠChristianity": 13624, "oler": 13625, "Ġsacred": 13626, "Ġdeclare": 13627, "ulative": 13628, "eting": 13629, "Ġdelivering": 13630, "estone": 13631, "Ġtel": 13632, "ĠLarry": 13633, "Ġmeta": 13634, "accept": 13635, "artz": 13636, "ĠRoger": 13637, "handed": 13638, "Ġheader": 13639, "Ġtrapped": 13640, "ĠCentury": 13641, "Ġknocked": 13642, "ĠOxford": 13643, "Ġsurvivors": 13644, "bot": 13645, "Ġdemonstration": 13646, "Ġdirt": 13647, "Ġassists": 13648, "OME": 13649, "ĠDraft": 13650, "ortunate": 13651, "folio": 13652, "pered": 13653, "usters": 13654, "gt": 13655, "ĠLock": 13656, "Ġjudicial": 13657, "verted": 13658, "Ġsecured": 13659, "outing": 13660, "ĠBooks": 13661, "Ġhosting": 13662, "Ġlifted": 13663, "length": 13664, "Ġjer": 13665, "Ġwheels": 13666, "ĠRange": 13667, "umbnails": 13668, "Ġdiagnosis": 13669, "tech": 13670, "ĠStewart": 13671, "ĠPract": 13672, "Ġnationwide": 13673, "Ġdear": 13674, "Ġobligations": 13675, "Ġgrows": 13676, "Ġmandatory": 13677, "Ġsuspicious": 13678, "!'": 13679, "Apr": 13680, "Great": 13681, "Ġmortgage": 13682, "Ġprosecutor": 13683, "Ġeditorial": 13684, "ĠKr": 13685, "Ġprocessed": 13686, "ungle": 13687, "Ġflexibility": 13688, "Earlier": 13689, "ĠCart": 13690, "ĠSug": 13691, "Ġfocuses": 13692, "Ġstartup": 13693, "Ġbreach": 13694, "ĠTob": 13695, "cycle": 13696, "ãĢĮ": 13697, "rose": 13698, "Ġbizarre": 13699, "ãĢį": 13700, "Ġvegetables": 13701, "$$": 13702, "Ġretreat": 13703, "oshi": 13704, "ĠShop": 13705, "ĠGround": 13706, "ĠStop": 13707, "ĠHawaii": 13708, "ĠAy": 13709, "Perhaps": 13710, "ĠBeaut": 13711, "uffer": 13712, "enna": 13713, "Ġproductivity": 13714, "Fixed": 13715, "control": 13716, "Ġabsent": 13717, "ĠCampaign": 13718, "Green": 13719, "Ġidentifying": 13720, "Ġregret": 13721, "Ġpromoted": 13722, "ĠSeven": 13723, "Ġeru": 13724, "neath": 13725, "aughed": 13726, "ĠPin": 13727, "ĠLiving": 13728, "Cost": 13729, "omatic": 13730, "mega": 13731, "ĠNig": 13732, "ocy": 13733, "Ġinbox": 13734, "Ġempire": 13735, "Ġhorizont": 13736, "Ġbranches": 13737, "Ġmetaph": 13738, "Active": 13739, "edi": 13740, "ĠFilm": 13741, "ĠSomething": 13742, "Ġmods": 13743, "incial": 13744, "ĠOriginal": 13745, "Gen": 13746, "Ġspirits": 13747, "Ġearning": 13748, "Hist": 13749, "Ġriders": 13750, "Ġsacrific": 13751, "MT": 13752, "ĠVA": 13753, "ĠSalt": 13754, "Ġoccupation": 13755, "ĠMi": 13756, "Ġdisg": 13757, "lict": 13758, "Ġnit": 13759, "Ġnodes": 13760, "eem": 13761, "ĠPier": 13762, "Ġhatred": 13763, "psy": 13764, "ãĥī": 13765, "Ġtheater": 13766, "Ġsophisticated": 13767, "Ġdefended": 13768, "Ġbesides": 13769, "Ġthoroughly": 13770, "ĠMedicare": 13771, "Ġblamed": 13772, "arently": 13773, "Ġcrying": 13774, "FOR": 13775, "priv": 13776, "Ġsinging": 13777, "ĠIl": 13778, "Ġcute": 13779, "oided": 13780, "olitical": 13781, "ĠNeuro": 13782, "å¤": 13783, "Ġdonation": 13784, "ĠEagles": 13785, "ĠGive": 13786, "Tom": 13787, "Ġsubstantially": 13788, "ĠLicense": 13789, "ĠJa": 13790, "Ġgrey": 13791, "ĠAnimal": 13792, "ĠER": 13793, "ĠUnd": 13794, "Ġkeen": 13795, "Ġconclude": 13796, "ĠMississippi": 13797, "Engine": 13798, "ĠStudios": 13799, "Press": 13800, "overs": 13801, "llers": 13802, "Ġ350": 13803, "ĠRangers": 13804, "Ġrou": 13805, "erto": 13806, "Ep": 13807, "issa": 13808, "ivan": 13809, "Ġseal": 13810, "ĠRegist": 13811, "display": 13812, "Ġweaken": 13813, "uum": 13814, "ĠCommons": 13815, "ĠSay": 13816, "Ġcultures": 13817, "Ġlaughed": 13818, "Ġslip": 13819, "Ġtreatments": 13820, "izable": 13821, "mart": 13822, "ĠRice": 13823, "Ġbeast": 13824, "Ġobesity": 13825, "ĠLaure": 13826, "iga": 13827, "Which": 13828, "holder": 13829, "Ġelderly": 13830, "Ġpays": 13831, "Ġcomplained": 13832, "Ġcrop": 13833, "Ġproc": 13834, "Ġexplosive": 13835, "ĠFan": 13836, "ĠArsenal": 13837, "Author": 13838, "eful": 13839, "Ġmeals": 13840, "Ġ(-": 13841, "idays": 13842, "Ġimagination": 13843, "Ġannually": 13844, "Ġms": 13845, "asures": 13846, "Head": 13847, "ikh": 13848, "matic": 13849, "Ġboyfriend": 13850, "ĠComputer": 13851, "Ġbump": 13852, "Ġsurge": 13853, "ĠCraig": 13854, "ĠKirk": 13855, "Del": 13856, "mediate": 13857, "Ġscenarios": 13858, "ĠMut": 13859, "ĠStream": 13860, "Ġcompetitors": 13861, "ÙĦ": 13862, "ĠStanford": 13863, "ĠResources": 13864, "azed": 13865, "bage": 13866, "Ġorganis": 13867, "ĠRelease": 13868, "Ġseparately": 13869, "Ġhabits": 13870, "Ġmeasurements": 13871, "ĠClose": 13872, "Ġaccompany": 13873, "Ġgly": 13874, "Ġtang": 13875, "ĠRou": 13876, "Ġplugin": 13877, "Ġconvey": 13878, "ĠChallenge": 13879, "oots": 13880, "jan": 13881, "Ġcurs": 13882, "ĠRelations": 13883, "keeper": 13884, "Ġapproaching": 13885, "ping": 13886, "Speaking": 13887, "Ġarrangement": 13888, "ĠVI": 13889, "arettes": 13890, "Ġaffecting": 13891, "Ġpermits": 13892, "because": 13893, "Ġuseless": 13894, "ĠHus": 13895, "!!!!": 13896, "Ġdestroying": 13897, "Unfortunately": 13898, "Ġfascinating": 13899, "Sem": 13900, "Ġelectoral": 13901, "Ġtransparency": 13902, "ĠChaos": 13903, "Ġvolunteer": 13904, "Ġstatistical": 13905, "Ġactivated": 13906, "rox": 13907, "Web": 13908, "HE": 13909, "ĠHampshire": 13910, "isive": 13911, "Map": 13912, "Ġtrash": 13913, "ĠLawrence": 13914, "stick": 13915, "Cr": 13916, "Ġrings": 13917, "EXT": 13918, "Ġoperational": 13919, "opes": 13920, "Does": 13921, "ĠEvans": 13922, "Ġwitnessed": 13923, "Port": 13924, "Ġlaunching": 13925, "econom": 13926, "wear": 13927, "ĠParticip": 13928, "umm": 13929, "cules": 13930, "ĠRAM": 13931, "ĠTun": 13932, "Ġassured": 13933, "Ġbinary": 13934, "Ġbetray": 13935, "Ġexploration": 13936, "ĠFel": 13937, "Ġadmission": 13938, "itated": 13939, "Sy": 13940, "Ġavoided": 13941, "ĠSimulator": 13942, "Ġcelebrated": 13943, "ĠElectric": 13944, "¥ŀ": 13945, "Ġcluster": 13946, "itzerland": 13947, "health": 13948, "Line": 13949, "ĠNash": 13950, "aton": 13951, "Ġspare": 13952, "Ġenterprise": 13953, "ĠDIS": 13954, "cludes": 13955, "Ġflights": 13956, "Ġregards": 13957, "ĠÃĹ": 13958, "half": 13959, "Ġtrucks": 13960, "Ġcontacts": 13961, "Ġuncons": 13962, "ĠClimate": 13963, "Ġimmense": 13964, "NEW": 13965, "occ": 13966, "ective": 13967, "Ġembod": 13968, "Ġpatrol": 13969, "Ġbeside": 13970, "Ġviable": 13971, "Ġcreep": 13972, "Ġtriggered": 13973, "verning": 13974, "Ġcomparable": 13975, "ql": 13976, "Ġgaining": 13977, "asses": 13978, "Ġ();": 13979, "ĠGrey": 13980, "ĠMLS": 13981, "sized": 13982, "Ġprosper": 13983, "\"?": 13984, "Ġpolling": 13985, "Ġshar": 13986, "ĠRC": 13987, "Ġfirearm": 13988, "orient": 13989, "Ġfence": 13990, "Ġvariations": 13991, "giving": 13992, "ĠPi": 13993, "ospel": 13994, "Ġpledge": 13995, "Ġcure": 13996, "Ġspy": 13997, "Ġviolated": 13998, "Ġrushed": 13999, "Ġstroke": 14000, "ĠBlog": 14001, "sels": 14002, "ĠEc": 14003, ",''": 14004, "Ġpale": 14005, "ĠCollins": 14006, "terror": 14007, "ĠCanadians": 14008, "Ġtune": 14009, "Ġlaboratory": 14010, "Ġnons": 14011, "tarian": 14012, "Ġdisability": 14013, "ĠGam": 14014, "Ġsinger": 14015, "alg": 14016, "ĠSenior": 14017, "Ġtraded": 14018, "ĠWarrior": 14019, "Ġinfring": 14020, "ĠFranklin": 14021, "Ġstrain": 14022, "ĠSwedish": 14023, "Ġseventh": 14024, "ĠBenn": 14025, "ĠTell": 14026, "Ġsyndrome": 14027, "Ġwondered": 14028, "iden": 14029, "++++": 14030, "igo": 14031, "Ġpurple": 14032, "Ġjournalism": 14033, "Ġrebel": 14034, "Ġfu": 14035, "blog": 14036, "Ġinvite": 14037, "rencies": 14038, "ĠContact": 14039, "Israel": 14040, "ĠContent": 14041, "Ġcheer": 14042, "Ġbedroom": 14043, "ĠEngineering": 14044, "ĠQueens": 14045, "Ġdwell": 14046, "ĠPlayStation": 14047, "ĠDim": 14048, "ĠColon": 14049, "lr": 14050, "Ġoperates": 14051, "Ġmotivation": 14052, "USA": 14053, "astered": 14054, "Core": 14055, "ĠTruth": 14056, "olo": 14057, "OSE": 14058, "ĠMemory": 14059, "Ġpredec": 14060, "Ġanarch": 14061, "Ġ1920": 14062, "ĠYam": 14063, "è": 14064, "bid": 14065, "Ġgrateful": 14066, "Ġexcitement": 14067, "Ġtreasure": 14068, "Ġlongest": 14069, "ctive": 14070, "Ġdeserves": 14071, "Ġreserves": 14072, "Ġcops": 14073, "ĠOttawa": 14074, "ĠEgyptian": 14075, "anked": 14076, "Ġartif": 14077, "Ġhypothesis": 14078, ":/": 14079, "Ġpurchasing": 14080, "Ġlovely": 14081, "HP": 14082, "Ġdivide": 14083, "Ġstrictly": 14084, "Ġquestioning": 14085, "Ġtaxpayers": 14086, "ĠJoy": 14087, "Ġrolls": 14088, "ĠHeavy": 14089, "Ġports": 14090, "Ġmagnetic": 14091, "Ġinflamm": 14092, "Ġbrush": 14093, "tics": 14094, "âĪĴ": 14095, "Ġbottles": 14096, "ppy": 14097, "Ġpadd": 14098, "ãĤ¯": 14099, "million": 14100, "Ġdevastating": 14101, "Ġcompiled": 14102, "Ġmedication": 14103, "Ġtwelve": 14104, "ĠPerry": 14105, "Space": 14106, "imb": 14107, "your": 14108, "Ġleaked": 14109, "ĠTar": 14110, "Ġunity": 14111, "Ġinfected": 14112, "Ġtraveled": 14113, "IDE": 14114, "ĠMcDonald": 14115, "txt": 14116, "ĠPrinc": 14117, "Ġinterven": 14118, "ĠTaiwan": 14119, "ĠPow": 14120, "Ġbearing": 14121, "ĠThread": 14122, "Ġzones": 14123, "izards": 14124, "unks": 14125, "Chapter": 14126, "llor": 14127, "Ġ·": 14128, "Ġwounds": 14129, "Ġdiscretion": 14130, "Ġsucceeded": 14131, "iking": 14132, "Ġiconic": 14133, "Call": 14134, "Ġscreening": 14135, "ĠMis": 14136, "icts": 14137, "Ġministers": 14138, "Ġseparation": 14139, "Player": 14140, "Ġbip": 14141, "Ġbeloved": 14142, "Ġcounting": 14143, "ĠEye": 14144, "around": 14145, "inging": 14146, "Ġtablet": 14147, "Ġoffence": 14148, "inance": 14149, "have": 14150, "ĠInfo": 14151, "ĠNinja": 14152, "Ġprotective": 14153, "ĠCass": 14154, "Mac": 14155, "ĠQuality": 14156, "North": 14157, "Ġic": 14158, "ĠCuba": 14159, "ĠChronicle": 14160, "ĠProperty": 14161, "Ġfastest": 14162, "otos": 14163, "ĠGerm": 14164, "OWN": 14165, "Ġboom": 14166, "ĠStanley": 14167, "erguson": 14168, "Ġclever": 14169, "Ġenters": 14170, "mode": 14171, "terior": 14172, "ĠSens": 14173, "Ġlinear": 14174, "ARK": 14175, "Ġcomparing": 14176, "Ġpurely": 14177, "Ġsafer": 14178, "ĠPotter": 14179, "Ġcups": 14180, "RT": 14181, "Ġgluc": 14182, "Ġattributed": 14183, "Ġdupl": 14184, "ĠPap": 14185, "Ġprecious": 14186, "Ġpa": 14187, "ictionary": 14188, "ĠTig": 14189, "ĠToo": 14190, "olutions": 14191, "stan": 14192, "Ġrobots": 14193, "Ġlobb": 14194, "Ġstatute": 14195, "Ġprevention": 14196, "western": 14197, "160": 14198, "ĠActive": 14199, "ĠMaria": 14200, "hal": 14201, "None": 14202, "ellar": 14203, "ĠKB": 14204, "ĠPartners": 14205, "ĠSingle": 14206, "ĠFollowing": 14207, "ango": 14208, "acious": 14209, "Ġthou": 14210, "Ġkg": 14211, "Ġinfluential": 14212, "ĠFriends": 14213, "Sur": 14214, "ainted": 14215, "Ġforums": 14216, "Ġstarter": 14217, "Ġcitizenship": 14218, "ĠElection": 14219, "onge": 14220, "otation": 14221, "osph": 14222, ";;;;": 14223, "utical": 14224, "pur": 14225, "eren": 14226, "Ġaccusations": 14227, "bitious": 14228, "abbit": 14229, "ĠOrd": 14230, "Posted": 14231, "irk": 14232, "Ġsensitivity": 14233, "iche": 14234, "ĠAmy": 14235, "ĠFab": 14236, "Ġsummit": 14237, "Ġpedest": 14238, "Ġrubber": 14239, "Ġagricultural": 14240, "Ġcancel": 14241, "AE": 14242, "Ġinaug": 14243, "Ġcontam": 14244, "Ġfirmly": 14245, "iw": 14246, "stage": 14247, "ĠKan": 14248, "Ġtier": 14249, "Ġinvention": 14250, "Ġtranslated": 14251, "ĠRules": 14252, "Box": 14253, "Twitter": 14254, "IDS": 14255, "Ġpizza": 14256, "Ġdebug": 14257, "ĠDrop": 14258, "vs": 14259, "Ġhorses": 14260, "big": 14261, "Ġboring": 14262, "Ġhood": 14263, "ĠMcCain": 14264, "atched": 14265, "ĠBros": 14266, "Ġskip": 14267, "Ġessay": 14268, "stat": 14269, "ĠLegends": 14270, "Ġammunition": 14271, "auc": 14272, "Ġshooter": 14273, "Ġunh": 14274, "Ġsupplied": 14275, "Ġgeneric": 14276, "ĠSK": 14277, "iban": 14278, "yrics": 14279, "Ġ255": 14280, "Ġclimbing": 14281, "Former": 14282, "Ġflip": 14283, "Ġjumping": 14284, "Ġfrustration": 14285, "ĠTerry": 14286, "Ġneighborhoods": 14287, "Ġmedian": 14288, "bean": 14289, "Ġbrains": 14290, "Following": 14291, "Ġshaped": 14292, "Ġdraws": 14293, "Ġaltered": 14294, "Jack": 14295, "Ġrecipes": 14296, "Ġskilled": 14297, "wealth": 14298, "achi": 14299, "election": 14300, "Ġbehaviors": 14301, "deals": 14302, "ĠUntil": 14303, "Fe": 14304, "Ġdeclaration": 14305, "marks": 14306, "ĠBetween": 14307, "celona": 14308, "Ġreson": 14309, "Ġbubble": 14310, "Among": 14311, "Ġimperial": 14312, "GS": 14313, "Ġfeminist": 14314, "2005": 14315, "ĠKyle": 14316, "Ġaccounting": 14317, "ĠTele": 14318, "ĠTyr": 14319, "Ġconnecting": 14320, "Ġrehab": 14321, "ĠPred": 14322, "sim": 14323, "Ġmeantime": 14324, "Ġphysician": 14325, "MW": 14326, "ĠCampbell": 14327, "ĠBrandon": 14328, "Ġcontributing": 14329, "ĠRule": 14330, "ĠWeight": 14331, "ĠNap": 14332, "Ġinteractive": 14333, "Ġvag": 14334, "Ġhelmet": 14335, "ĠComb": 14336, "four": 14337, "Ġshipped": 14338, "Ġcompleting": 14339, "ĠPD": 14340, "PDATE": 14341, "Ġspreading": 14342, "Ġscary": 14343, "erving": 14344, "ĠGas": 14345, "Ġfrank": 14346, "school": 14347, "Ġromantic": 14348, "Ġstabil": 14349, "Rob": 14350, "Ġaccurately": 14351, "Ġacute": 14352, "ĠHann": 14353, "Ġsymbols": 14354, "Ġcivilization": 14355, "ĠAW": 14356, "Ġlightning": 14357, "Ġconsiders": 14358, "Ġvenue": 14359, "Ġ×": 14360, "Ġoven": 14361, "ĠSF": 14362, "his": 14363, "Ġnu": 14364, "ĠLearn": 14365, "Ġpeoples": 14366, "Ġstd": 14367, "Ġslee": 14368, "Ġslic": 14369, "ĠStatistics": 14370, "Ġcorners": 14371, "ĠBaker": 14372, "Ġ:)": 14373, "mentation": 14374, "olver": 14375, "Ġlaughing": 14376, "ĠTodd": 14377, "onde": 14378, "ĠHills": 14379, "Ġnuts": 14380, "ĠWoman": 14381, "plane": 14382, "Ġliver": 14383, "ĠInside": 14384, "Sorry": 14385, "Ġagrees": 14386, "Ġfundament": 14387, "ĠFisher": 14388, "Ġauction": 14389, "Ġthreads": 14390, "glas": 14391, "ĠBasic": 14392, "ĠNat": 14393, "Ġlacking": 14394, "Ġcelebration": 14395, "ju": 14396, "Ġsilly": 14397, "Euro": 14398, "Ġtatt": 14399, "ighty": 14400, "controlled": 14401, "Test": 14402, "ĠSingh": 14403, "Ġrage": 14404, "Ġrhyth": 14405, "offic": 14406, "ĠPhantom": 14407, "Ġheadlines": 14408, "Ġresponding": 14409, "ĠMorning": 14410, "Ġvitamin": 14411, "Ġboots": 14412, "ĠSite": 14413, "alin": 14414, "pi": 14415, "Ġviral": 14416, "ĠUC": 14417, "DER": 14418, "ĠSex": 14419, "Ġstocks": 14420, "current": 14421, "Ġchurches": 14422, "ĠRare": 14423, "ĠMurphy": 14424, "Ġdenial": 14425, "ĠGaming": 14426, "Ġtoug": 14427, "Ġnick": 14428, "Ġmakers": 14429, "ĠRonald": 14430, "Ġgenerous": 14431, "ĠDoc": 14432, "ĠMorris": 14433, "Ġtransformed": 14434, "ĠNormal": 14435, "Ġ104": 14436, "ĠKickstarter": 14437, "ĠUpon": 14438, "Online": 14439, "ĠIRS": 14440, "Ġwrap": 14441, "Ġloving": 14442, "Ġarrives": 14443, "ĠDue": 14444, "Ġheter": 14445, "ĠMade": 14446, "Ġrental": 14447, "Ġbelongs": 14448, "Ġattorneys": 14449, "Ġcrops": 14450, "Ġmatched": 14451, "ulum": 14452, "oline": 14453, "109": 14454, "Ġdispar": 14455, "Ġbuyers": 14456, "ĠCambridge": 14457, "Ġethics": 14458, "roups": 14459, "Ġjustified": 14460, "Ġmarginal": 14461, "Ġrespected": 14462, "winning": 14463, "Ġnodded": 14464, "ĠSerge": 14465, "ĠFormer": 14466, "Craft": 14467, "################": 14468, "ĠWarner": 14469, "Ġdash": 14470, "ete": 14471, "Ġentert": 14472, "ĠEscape": 14473, "outheast": 14474, "Ġknees": 14475, "ĠBomb": 14476, "Ġrug": 14477, "Pass": 14478, "Ġattitudes": 14479, "government": 14480, "ĠPrior": 14481, "Ġqualities": 14482, "Ġnotification": 14483, "ĠPhone": 14484, "lie": 14485, "Ġanticipated": 14486, "ĠCombat": 14487, "ĠBarry": 14488, "Ġ1982": 14489, "Users": 14490, "oner": 14491, "Ġcomputing": 14492, "ĠConnecticut": 14493, "Ġlesser": 14494, "Ġpeers": 14495, "ĠCu": 14496, "Ġtechnically": 14497, "Ġsubmission": 14498, "ĠUniversal": 14499, "Ġmanually": 14500, "ourge": 14501, "Ġrespondents": 14502, "ĠBTC": 14503, "ĠHost": 14504, "Ġfare": 14505, "ĠBird": 14506, "Ġreceipt": 14507, "also": 14508, "Ġjack": 14509, "Ġagriculture": 14510, "Ġskull": 14511, "Ġ!=": 14512, "Ġpassive": 14513, "ĠCI": 14514, "Ġsocieties": 14515, "Ġreminded": 14516, "Ġinterference": 14517, "Buy": 14518, "Ġâľ": 14519, "gon": 14520, "Ġscrutiny": 14521, "ĠWitch": 14522, "Ġconducting": 14523, "Ġãĥ": 14524, "Ġexchanges": 14525, "ĠMitchell": 14526, "Ġinhabit": 14527, "Ġtwist": 14528, "BD": 14529, "Ġwherever": 14530, "groupon": 14531, "Ġjokes": 14532, "ĠBenjamin": 14533, "ĠRandom": 14534, "frame": 14535, "ĠLions": 14536, "Ġhighlighted": 14537, "ĠArkansas": 14538, "Ent": 14539, "Ġpile": 14540, "Ġprelim": 14541, "gs": 14542, "minded": 14543, "Ġfelony": 14544, "ĠGA": 14545, "ĠLuck": 14546, "Ġpractically": 14547, "ĠBos": 14548, "Ġactress": 14549, "Dam": 14550, "ĠBou": 14551, "Ġvisa": 14552, "Ġembedded": 14553, "Ġhybrid": 14554, "Ġearliest": 14555, "Ġsooner": 14556, "social": 14557, "ĠHA": 14558, "Ġsteep": 14559, "Ġdisadvant": 14560, "Ġexploit": 14561, "ĠEgg": 14562, "ĠUltra": 14563, "Ġnecessity": 14564, "Local": 14565, "iege": 14566, "Ġdated": 14567, "Ġmasses": 14568, "Ġsubscription": 14569, "pless": 14570, "Ġanonym": 14571, "Ġpresumably": 14572, "Blue": 14573, "Their": 14574, "asketball": 14575, "ĠPhilip": 14576, "Ġcomed": 14577, "loaded": 14578, "rane": 14579, "Ġreflection": 14580, "China": 14581, "Ġextends": 14582, "Ġforming": 14583, "Ġunders": 14584, "2001": 14585, "Ġgrat": 14586, "Ġconcentrations": 14587, "Ġinsulin": 14588, "Ġsecular": 14589, "Ġwhilst": 14590, "Ġwinners": 14591, "Advertisements": 14592, "Ġdeliberately": 14593, "ĠWorking": 14594, "Ġsink": 14595, "etics": 14596, "dale": 14597, "Ġmandate": 14598, "Ġgram": 14599, "Ġvacation": 14600, "Ġwarnings": 14601, "ripp": 14602, "ĠTHAT": 14603, "Ġcommentary": 14604, "Ġintu": 14605, "Ġaest": 14606, "Ġreasoning": 14607, "Ġbreakdown": 14608, "ĠZombie": 14609, "Ġ-->": 14610, "ĠPolitical": 14611, "cott": 14612, "Ġthrust": 14613, "Ġtechnological": 14614, "Ġdeciding": 14615, "Ġtrafficking": 14616, "Long": 14617, "Welcome": 14618, "prising": 14619, "ĠCommunications": 14620, "Ġendors": 14621, "Ġswift": 14622, "Ġmetabol": 14623, "coins": 14624, "resa": 14625, "ĠHTTP": 14626, "Ġenroll": 14627, "ĠHappy": 14628, "usr": 14629, "intage": 14630, "Ġ[\"": 14631, "uably": 14632, "ĠMaterial": 14633, "Ġrepeal": 14634, "Sept": 14635, "kh": 14636, "ĠModi": 14637, "Ġunderneath": 14638, "ĠIL": 14639, "shore": 14640, "Ġdiagnosed": 14641, "aceutical": 14642, "Ġshower": 14643, "aux": 14644, "ĠSwitch": 14645, "ĠStrength": 14646, "Ġjihad": 14647, "national": 14648, "Ġtrauma": 14649, "ussy": 14650, "oni": 14651, "Ġconsolid": 14652, "Ġcalories": 14653, "ĠFlynn": 14654, "agged": 14655, "168": 14656, "ĠPink": 14657, "Ġfulfill": 14658, "Ġchains": 14659, "Ġnotably": 14660, "ĠAV": 14661, "Life": 14662, "ĠChuck": 14663, "mus": 14664, "ĠUrban": 14665, "ĠHend": 14666, "Ġdeposit": 14667, "ĠSad": 14668, "Ġaffair": 14669, "ORK": 14670, "ieval": 14671, "ĠFDA": 14672, "Ġtrop": 14673, "ĠOverall": 14674, "Ġvirtue": 14675, "Ġsatisfaction": 14676, "aund": 14677, "Ġlun": 14678, "ĠSwitzerland": 14679, "ĠOperation": 14680, "process": 14681, "Ġshook": 14682, "Ġcounties": 14683, "leased": 14684, "ĠCharlotte": 14685, "112": 14686, "Ġtranscript": 14687, "Ġredd": 14688, "push": 14689, "ĠHey": 14690, "ĠAnalysis": 14691, "[\"": 14692, "Ġalternatives": 14693, "ardless": 14694, "Ġeleph": 14695, "Ġprejud": 14696, "ĠLeaf": 14697, "Having": 14698, "ĠHub": 14699, "Ġexpressions": 14700, "ĠVolume": 14701, "Ġshocking": 14702, "ĠReds": 14703, "Ġreadily": 14704, "Ġplanets": 14705, "adata": 14706, "Ġcollapsed": 14707, "ĠMadrid": 14708, "Ġirrit": 14709, "ipper": 14710, "ĠEnc": 14711, "ĠWire": 14712, "Ġbuzz": 14713, "ĠGP": 14714, "asha": 14715, "Ġaccidentally": 14716, "uru": 14717, "Ġfrustrated": 14718, "ĠSA": 14719, "Ġhungry": 14720, "ĠHuff": 14721, "Ġlabels": 14722, "anto": 14723, "ĠEP": 14724, "Ġbarriers": 14725, ")|": 14726, "ĠBerkeley": 14727, "ĠJets": 14728, "Ġpairs": 14729, "ĠLan": 14730, "James": 14731, "ĠBear": 14732, "Ġhumor": 14733, "ĠLiberty": 14734, "Ġmagnitude": 14735, "Ġaging": 14736, "ĠMason": 14737, "Ġfriendship": 14738, "umbling": 14739, "Ġemerge": 14740, "Ġnewspapers": 14741, "Ġambitious": 14742, "ĠRichards": 14743, "aternal": 14744, "Ġ1981": 14745, "Ġcookies": 14746, "Ġsculpt": 14747, "Ġpursuit": 14748, "Location": 14749, "Ġscripts": 14750, "pc": 14751, "Ġarrangements": 14752, "Ġdiameter": 14753, "Ġloses": 14754, "amation": 14755, "Ġliqu": 14756, "ĠJake": 14757, "arette": 14758, "Ġunderstands": 14759, "ĠZen": 14760, "vm": 14761, "Ġapprove": 14762, "Ġwip": 14763, "Ġultra": 14764, "Ġintend": 14765, "ĠDI": 14766, "ascular": 14767, "Ġstays": 14768, "ĠKor": 14769, "ĠKl": 14770, "Ġinvesting": 14771, "La": 14772, "Ġbelieving": 14773, "bad": 14774, "mouth": 14775, "Ġtaxpayer": 14776, "ãĥĥ": 14777, "ĠQuebec": 14778, "Ġlap": 14779, "ĠSwiss": 14780, "drop": 14781, "Ġdrain": 14782, "iri": 14783, "etc": 14784, "ften": 14785, "ĠNex": 14786, "Ġstraw": 14787, "Ġscreaming": 14788, "Ġcounted": 14789, "Ġdamaging": 14790, "Ġambassador": 14791, "century": 14792, "Ġprox": 14793, "Ġarrests": 14794, "uv": 14795, "ilateral": 14796, "ĠCharg": 14797, "Ġprescribed": 14798, "Ġindependently": 14799, "Ġfierce": 14800, "ĠBaby": 14801, "Ġbrave": 14802, "Ġsuits": 14803, "=>": 14804, "Ġbaseline": 14805, "ĠRate": 14806, "Ġislands": 14807, "Ġ((": 14808, "green": 14809, "ixels": 14810, "Ġnamely": 14811, "ĠVillage": 14812, "than": 14813, "amy": 14814, "Version": 14815, "gmail": 14816, "entials": 14817, "ĠSud": 14818, "ĠMelbourne": 14819, "Ġarriving": 14820, "Ġquantum": 14821, "eff": 14822, "ropolitan": 14823, "Tri": 14824, "Ġfuneral": 14825, "ĠIR": 14826, "ÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤ": 14827, "ĠCob": 14828, "itably": 14829, "Ġturb": 14830, "Ġcombo": 14831, "Review": 14832, "Ġdeployment": 14833, "uity": 14834, "ĠBott": 14835, "Ġinvisible": 14836, "Ġrendering": 14837, "Ġunlocked": 14838, "Ġaqu": 14839, "ĠVladimir": 14840, "Ġpad": 14841, "ĠBrain": 14842, "ĠLegacy": 14843, "dragon": 14844, "ĠKurdish": 14845, "Ġsounded": 14846, "Ġdetained": 14847, "ĠDM": 14848, "gary": 14849, "Ġdaughters": 14850, "Ġdisturbing": 14851, "uka": 14852, "ĠParad": 14853, "Ġtast": 14854, "Ġunfortunate": 14855, "Ġul": 14856, "emin": 14857, "Ġattendance": 14858, "trl": 14859, "Ġparks": 14860, "ĠMemorial": 14861, "ĠAlice": 14862, "othy": 14863, "guard": 14864, "ĠDise": 14865, "ĠShan": 14866, "ĠForum": 14867, "Rich": 14868, "Ġshifted": 14869, "uez": 14870, "Ġlighter": 14871, "ĠMagn": 14872, "Ġcod": 14873, "Sch": 14874, "hammad": 14875, "Pub": 14876, "350": 14877, "ĠPokemon": 14878, "Ġprototype": 14879, "Ġunre": 14880, "Base": 14881, "ĠStudents": 14882, "ĠReply": 14883, "ĠCommunist": 14884, "Ġgau": 14885, "ĠTyler": 14886, "IZ": 14887, "Ġparticipated": 14888, "Ġsuprem": 14889, "ĠDetails": 14890, "Ġvessels": 14891, "rod": 14892, "Ġtribe": 14893, "keep": 14894, "Ġassumptions": 14895, "Ġpound": 14896, "Ġcrude": 14897, "ĠAvailable": 14898, "Ġswimming": 14899, "Ġinclusion": 14900, "Ġadvances": 14901, "culation": 14902, "Ġconservation": 14903, "Ġoverd": 14904, "ĠBuffalo": 14905, "Article": 14906, "edge": 14907, "Ġawa": 14908, "ĠMadison": 14909, "Ġsidew": 14910, "Ġcatast": 14911, "ĠKrist": 14912, "ucle": 14913, "ĠHighway": 14914, "ĠTerror": 14915, "Ġactivation": 14916, "Ġunconscious": 14917, "ĠSatan": 14918, "ĠSusan": 14919, "illery": 14920, "Ġarranged": 14921, "iop": 14922, "Ġrumors": 14923, "urring": 14924, "think": 14925, "ĠKeith": 14926, "ĠKind": 14927, "Ġavoiding": 14928, "byn": 14929, "nut": 14930, "ĠSpeaker": 14931, "rus": 14932, "names": 14933, "Ġguilt": 14934, "ĠOlympics": 14935, "Ġsail": 14936, "ĠMes": 14937, "levant": 14938, "ĠColumbus": 14939, "aft": 14940, "City": 14941, "South": 14942, "ĠHarvey": 14943, "ĠPun": 14944, "Several": 14945, "Ġmentally": 14946, "Ġimpress": 14947, "mount": 14948, "ĠUbuntu": 14949, "âĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶ": 14950, "ĠSuperman": 14951, "ĠMPs": 14952, "Ġintentions": 14953, "ĠRacing": 14954, "Ġlikelihood": 14955, "Ġ240": 14956, "Total": 14957, "Ġtoys": 14958, "ĠWatson": 14959, "Ġurge": 14960, "Lear": 14961, "ĠPaper": 14962, "Ġoccurring": 14963, "ĠBeng": 14964, "ĠCert": 14965, "Ġstones": 14966, "Tim": 14967, "ĠTwin": 14968, "zb": 14969, "ĠDynam": 14970, "Ġpolitician": 14971, "kens": 14972, "ĠEnterprise": 14973, "UTERS": 14974, "Ġabol": 14975, "Ġrefresh": 14976, "Ġarbitrary": 14977, "pection": 14978, "Ġtroubles": 14979, "Ġ});": 14980, "tv": 14981, "Ġpilots": 14982, "Ġdistribute": 14983, "Ġaudit": 14984, "Ġpause": 14985, "original": 14986, "Ġrivals": 14987, "£": 14988, "Fig": 14989, "TL": 14990, "abil": 14991, "rying": 14992, "Lin": 14993, "ioned": 14994, "lon": 14995, "Ġfancy": 14996, "Ġcrashed": 14997, "Ġtract": 14998, "Ġshed": 14999, "Ġconsume": 15000, "Based": 15001, "download": 15002, "init": 15003, "Ġvoltage": 15004, "Introdu": 15005, "Ġcondemned": 15006, "ĠFinance": 15007, "respect": 15008, "Ġexcluded": 15009, "Ġestablishing": 15010, "heric": 15011, "Ġheritage": 15012, "Ġspectacular": 15013, "Ġunst": 15014, "ĠSnowden": 15015, "ĠLane": 15016, "San": 15017, "Ġprotections": 15018, "struction": 15019, "incinn": 15020, "Ġmacro": 15021, "Custom": 15022, "iosity": 15023, "Ġesp": 15024, "Ġfunctioning": 15025, "Ġmush": 15026, "Ġpuzzle": 15027, "Ġethical": 15028, "Mal": 15029, "Ġgoverning": 15030, "ĠFerguson": 15031, "Ġrestored": 15032, "Ġstressed": 15033, "ĠCounter": 15034, "ĠKas": 15035, "clip": 15036, "ANS": 15037, "Ġseiz": 15038, "UK": 15039, "byss": 15040, "oldown": 15041, "api": 15042, "Ġpermanently": 15043, "ounters": 15044, "West": 15045, "Through": 15046, "Light": 15047, "atoes": 15048, "Ġneat": 15049, "Ġcord": 15050, "urer": 15051, "Ġseverely": 15052, "ĠAven": 15053, "Ġinterrog": 15054, "Ġtriple": 15055, "Given": 15056, "Number": 15057, "Ġarise": 15058, "Ġsher": 15059, "plant": 15060, "Ġflower": 15061, "ĠCou": 15062, "Ġate": 15063, "Ġnewer": 15064, "bul": 15065, "Ġmeanwhile": 15066, "ĠLair": 15067, "Ġadjustment": 15068, "ĠCopyright": 15069, "Ġdivers": 15070, "iological": 15071, "Ġgamers": 15072, "oat": 15073, "Ġhistorically": 15074, "Ġanalog": 15075, "Ġlongtime": 15076, "Ġprescription": 15077, "ĠMist": 15078, "ĠHyper": 15079, "ĠMaine": 15080, "ĠDeity": 15081, "Ġmultipl": 15082, "ĠReincarn": 15083, "ĠHyd": 15084, "ĠPic": 15085, "Sil": 15086, "rants": 15087, "ĠCris": 15088, ".;": 15089, "({": 15090, "ependence": 15091, "Ġrecy": 15092, "ateur": 15093, "Ġquad": 15094, "Ġglob": 15095, "Ġconced": 15096, "team": 15097, "Ġcapitalist": 15098, "ĠLot": 15099, "Ġroyal": 15100, "ĠCyber": 15101, "Ġblacks": 15102, "metic": 15103, "riv": 15104, "ĠDanny": 15105, "Ġspo": 15106, "ĠRO": 15107, "Ġanimated": 15108, "rypted": 15109, "ĠDeputy": 15110, "Ġrendered": 15111, "FE": 15112, "Ġstreak": 15113, "Ġclouds": 15114, "ĠDoug": 15115, "~~~~~~~~": 15116, "Ġdiscour": 15117, "ĠVeh": 15118, "Ġpsychology": 15119, "ĠJourney": 15120, "Ġcrystal": 15121, "ĠFrost": 15122, "Ġsuspicion": 15123, "Ġrelate": 15124, "orus": 15125, "ĠCrypt": 15126, "ĠNVIDIA": 15127, "comed": 15128, "uting": 15129, "incinnati": 15130, "Ġvulnerability": 15131, "ostic": 15132, "Ġisolation": 15133, "Ġcooling": 15134, "ĠCoalition": 15135, "Ġ119": 15136, "Four": 15137, "ĠDeal": 15138, "Ġâī": 15139, "semble": 15140, "rament": 15141, "ĠBarcelona": 15142, "Ġ102": 15143, "Ġcocaine": 15144, "ocalypse": 15145, "Feb": 15146, "ogenic": 15147, "Ġmutation": 15148, "Ġcryptoc": 15149, "ĠKel": 15150, "ĠGit": 15151, "ais": 15152, "Ġsisters": 15153, "ANK": 15154, "Ġactivate": 15155, "Ter": 15156, "Ġdread": 15157, "ylon": 15158, "Ġpropri": 15159, "Aust": 15160, "ĠDefault": 15161, "Ġoutdoor": 15162, "Ġsheer": 15163, "ceive": 15164, "Ġgently": 15165, "о": 15166, "Program": 15167, "ĠâĨĴ": 15168, "Ġvegan": 15169, "ĠCrus": 15170, "Ġresponsibilities": 15171, "ĠHR": 15172, "OLD": 15173, "Ġprevents": 15174, "Ġstiff": 15175, "ĠWere": 15176, "Ġathletic": 15177, "ĠScore": 15178, "Ġ):": 15179, "Ġcolumns": 15180, "ĠLoc": 15181, "available": 15182, "ĠFram": 15183, "ĠSessions": 15184, "Ġcompanion": 15185, "Ġpacks": 15186, "140": 15187, "ĠKnights": 15188, "Ġfart": 15189, "Ġstreams": 15190, "Ġshore": 15191, "Ġappeals": 15192, "ĠPerformance": 15193, "haul": 15194, "ĠStra": 15195, "ĠNag": 15196, "103": 15197, "ĠTransportation": 15198, "BB": 15199, "Ev": 15200, "zan": 15201, "Public": 15202, "Ġtwin": 15203, "ulsion": 15204, "Mult": 15205, "Ġelectro": 15206, "Ġstatue": 15207, "ationally": 15208, "ĠNort": 15209, "Ġinspection": 15210, "/*": 15211, "igue": 15212, "Ġcompassion": 15213, "ĠTales": 15214, "ĠStein": 15215, "ĠScreen": 15216, "ĠBug": 15217, "ĠLion": 15218, "girl": 15219, "Ġwithdrawal": 15220, "Ġobjectives": 15221, "Ġbloody": 15222, "Ġpreliminary": 15223, "Ġjacket": 15224, "Ġdimensions": 15225, "ĠCool": 15226, "ĠOccup": 15227, "Ġwreck": 15228, "Ġdoubled": 15229, "anking": 15230, "Ġ1975": 15231, "Ġglasses": 15232, "ĠWang": 15233, "prov": 15234, "Path": 15235, "connected": 15236, "ĠMulti": 15237, "ĠNorway": 15238, "agonist": 15239, "Ġfeared": 15240, "Ġtouching": 15241, "Ġarguably": 15242, "¯¯¯¯¯¯¯¯": 15243, "ĠNCAA": 15244, "chem": 15245, "Ġspat": 15246, "ĠWWE": 15247, "ĠCel": 15248, "igger": 15249, "Ġattacker": 15250, "ĠJoin": 15251, "object": 15252, "etta": 15253, "Ġeliminated": 15254, "det": 15255, "Ġdestruct": 15256, "ĠLucas": 15257, "ctuary": 15258, "180": 15259, "ĠBrady": 15260, "ĠBlues": 15261, "Bay": 15262, "aukee": 15263, "Ġtimeline": 15264, "Ġdelegates": 15265, "written": 15266, "ufficient": 15267, "Ġshapes": 15268, "Copyright": 15269, "ouble": 15270, "service": 15271, "Ġpione": 15272, "Ġcolleges": 15273, "Ġrows": 15274, "Ġspite": 15275, "Ġassessed": 15276, "360": 15277, "Ġlease": 15278, "Ġconfidential": 15279, "cker": 15280, "ĠManning": 15281, "ĠVoice": 15282, "Ġsealed": 15283, "Ġcalculate": 15284, "NO": 15285, "ĠAssistant": 15286, "Ġteenager": 15287, "ulent": 15288, "atherine": 15289, "Ġmock": 15290, "Ġdiamond": 15291, "Ġfest": 15292, "Ġswitched": 15293, "Ġresume": 15294, "ĠPuerto": 15295, "Ġlanes": 15296, "iration": 15297, "ĠSimilarly": 15298, "Ġrod": 15299, "ĠSel": 15300, "ĠPalace": 15301, "ĠLimited": 15302, "eous": 15303, "Ġvariant": 15304, "Ġward": 15305, "Ġ))": 15306, "Show": 15307, "OOK": 15308, "Alex": 15309, "ĠNep": 15310, "bris": 15311, "ĠWikipedia": 15312, "Ġexceptional": 15313, "Ġmanages": 15314, "ĠDraw": 15315, "Again": 15316, "Ġcopper": 15317, "utt": 15318, "Ġexports": 15319, "Ġportfolio": 15320, "Ġelevated": 15321, "Rated": 15322, "ĠOtherwise": 15323, "ĠTact": 15324, "ĠShel": 15325, "ĠTX": 15326, "\"âĢĶ": 15327, "Ġresur": 15328, "ĠWa": 15329, "venant": 15330, "Ġmonetary": 15331, "people": 15332, "Email": 15333, "Ġfifty": 15334, "ĠSweet": 15335, "ĠMalaysia": 15336, "Ġconfusing": 15337, "ĠRio": 15338, "uda": 15339, "utenant": 15340, "\");": 15341, "Ġpraised": 15342, "Ġvolumes": 15343, "turn": 15344, "Ġmature": 15345, "Ġnonprofit": 15346, "Ġpassionate": 15347, "ĠPrivate": 15348, "Ġ103": 15349, "Ġdescend": 15350, "ç¥ŀ": 15351, "uffy": 15352, "headed": 15353, "Whether": 15354, "rien": 15355, "zech": 15356, "beit": 15357, "Ġchrom": 15358, "ĠMcM": 15359, "Ġdancing": 15360, "Ġeleg": 15361, "ĠNoticed": 15362, "115": 15363, "Ġadvocacy": 15364, "ENTS": 15365, "ambling": 15366, "ĠMinor": 15367, "ĠFinn": 15368, "Ġpriorities": 15369, "Ġthereof": 15370, "ĠStage": 15371, "ĠRogers": 15372, "Ġsubstitute": 15373, "ĠJar": 15374, "ĠJefferson": 15375, "Ġlightly": 15376, "102": 15377, "ĠLisa": 15378, "uits": 15379, "ysical": 15380, "Ġshifts": 15381, "Ġdrones": 15382, "Ġworkplace": 15383, "Ġresid": 15384, "ensed": 15385, "ahn": 15386, "Ġpreferences": 15387, "server": 15388, "Ġdebates": 15389, "doc": 15390, "ĠGods": 15391, "Ġhelicopter": 15392, "Ġhonour": 15393, "Ġconsiderably": 15394, "eded": 15395, "ĠFemale": 15396, "ĠAnne": 15397, "Ġreun": 15398, "ĠFace": 15399, "ĠHallow": 15400, "ĠBudget": 15401, "Ġcondemn": 15402, "Ġtender": 15403, "Prof": 15404, "ocratic": 15405, "ĠTurner": 15406, "ĠAgric": 15407, "Ġ1976": 15408, "Ġapt": 15409, "disc": 15410, "ĠFighter": 15411, "ĠAur": 15412, "Ġgarbage": 15413, "input": 15414, "ĠKarl": 15415, "ĠOliver": 15416, "ĠLanguage": 15417, "kn": 15418, "Non": 15419, "ĠClar": 15420, "Ġtraditions": 15421, "Ġadvertisement": 15422, "ĠSor": 15423, "Ġarchive": 15424, "Ġvillages": 15425, "750": 15426, "Ġimplementing": 15427, "waukee": 15428, "Ġdietary": 15429, "Ġswitching": 15430, "Republic": 15431, "Ġvelocity": 15432, "Ġcit": 15433, "ĠAwards": 15434, "Ġfinancing": 15435, "Ġlasted": 15436, ")]": 15437, "Ġreminder": 15438, "Person": 15439, "Ġprecision": 15440, "Ġdesigners": 15441, "ĠFried": 15442, "ĠBorder": 15443, "Ġtragic": 15444, "Ġwield": 15445, "Ġinitiatives": 15446, "ĠTank": 15447, "wer": 15448, "Ġjoins": 15449, "Ro": 15450, "inery": 15451, "Ġarrow": 15452, "Ġgenerating": 15453, "founder": 15454, "Ġsearches": 15455, "Ġrandomly": 15456, "Access": 15457, "Ġbatch": 15458, "Ġposed": 15459, "lat": 15460, "Ġpursuing": 15461, "asa": 15462, "Ġtestified": 15463, "forming": 15464, "ĠShar": 15465, "wiki": 15466, "ĠEither": 15467, "Sometimes": 15468, "Ġsenators": 15469, "ĠJohnny": 15470, "ĠTaliban": 15471, "ĠGPS": 15472, "\":\"/": 15473, "ãģ®å": 15474, "Ġanalyzed": 15475, "ĠRubio": 15476, "ĠMovement": 15477, "opard": 15478, "iii": 15479, "Stand": 15480, "fight": 15481, "Ġignoring": 15482, "iang": 15483, "ĠGN": 15484, "soever": 15485, "ĠSTAT": 15486, "Ġrefusing": 15487, "Ġsweat": 15488, "Ġbay": 15489, "PORT": 15490, "irmed": 15491, "aky": 15492, "Ġdispro": 15493, "Ġlabeled": 15494, "Ġ108": 15495, "Hello": 15496, "Ġpleasant": 15497, "aba": 15498, "Ġtriumph": 15499, "Ġaboard": 15500, "Ġincom": 15501, "ĠCrow": 15502, "lett": 15503, "Ġfolk": 15504, "Ġchase": 15505, "``": 15506, "ĠBrus": 15507, "Ġteens": 15508, "cue": 15509, "Ġterrain": 15510, "hyd": 15511, "ilight": 15512, "ORY": 15513, "Support": 15514, "ews": 15515, "lli": 15516, "raints": 15517, "ĠCand": 15518, "Ġabused": 15519, "achment": 15520, "larg": 15521, "Bas": 15522, "ĠCancer": 15523, "Ġ1978": 15524, "Ġsupporter": 15525, "access": 15526, "ĠTermin": 15527, "ĠTampa": 15528, "ĠANY": 15529, "Ġnewest": 15530, "ĠCriminal": 15531, "edu": 15532, "Ġ1930": 15533, "Ġadmits": 15534, "Ġende": 15535, "Ġfailures": 15536, "urate": 15537, "fulness": 15538, "cycl": 15539, "ĠSubject": 15540, "Ġinfinite": 15541, "three": 15542, "WA": 15543, "pit": 15544, "ĠInstall": 15545, "Rad": 15546, "iliation": 15547, "GM": 15548, "Ġcontinent": 15549, "Ġaccommodate": 15550, "ĠClay": 15551, "Ġpup": 15552, "ĠFunction": 15553, "Ġhammer": 15554, "ĠAlberta": 15555, "Ġrevised": 15556, "Ġminorities": 15557, "Ġmeasurement": 15558, "Connell": 15559, "Ġdisable": 15560, "ĠMix": 15561, "Incre": 15562, "Ġfork": 15563, "ĠRosen": 15564, "Ġimplies": 15565, "umblr": 15566, "ANG": 15567, "Ġproteins": 15568, "Ġaggression": 15569, "Ġfacilitate": 15570, "SN": 15571, "Ġillegally": 15572, "uer": 15573, "Ġacadem": 15574, "Ġpuzz": 15575, "ĠShift": 15576, "pay": 15577, "ollo": 15578, "Ġaudiences": 15579, "Build": 15580, "Ġnoble": 15581, "Ġsyntax": 15582, "âĺħ": 15583, "Ġbeam": 15584, "ĠBed": 15585, "ĠAld": 15586, "Ġorigins": 15587, "video": 15588, "Ġ1977": 15589, "ĠAssault": 15590, "Ġgarage": 15591, "Team": 15592, "Ġverdict": 15593, "Ġdwar": 15594, "ĠVirtual": 15595, "event": 15596, "Keep": 15597, "Ġsentiment": 15598, "Ġwildlife": 15599, "shirt": 15600, "Ġburg": 15601, "Ġrecommendation": 15602, "represent": 15603, "Ġgallery": 15604, "owners": 15605, "Ġscholar": 15606, "Ġconvenience": 15607, "ĠSwift": 15608, "Ġconvinc": 15609, "Cap": 15610, "Ġwarfare": 15611, "ĠVisual": 15612, "Ġconstitute": 15613, "Ġabort": 15614, "ĠWeather": 15615, "ĠLooking": 15616, "ĠHem": 15617, "Ġmartial": 15618, "Ġincoming": 15619, "etition": 15620, "Ġtolerance": 15621, "ĠCreated": 15622, "Ġflows": 15623, "ĠElder": 15624, "Ġsouls": 15625, "Ġfoul": 15626, "ĠPain": 15627, "ĠCAN": 15628, "Ġ220": 15629, "bc": 15630, "hend": 15631, "Ġgenius": 15632, "Real": 15633, "ĠWr": 15634, "ometer": 15635, "pad": 15636, "Ġlimiting": 15637, "ĠSi": 15638, "ĠLore": 15639, "ĠAdventures": 15640, "Ġvaried": 15641, "Disc": 15642, "fin": 15643, "ĠPersonal": 15644, "Chris": 15645, "Ġinvented": 15646, "Ġdive": 15647, "ĠRise": 15648, "Ġoz": 15649, "ĠComics": 15650, "Ġexpose": 15651, "ĠReb": 15652, "letters": 15653, "site": 15654, "imated": 15655, "Ġhacking": 15656, "Ġeducated": 15657, "ĠNobody": 15658, "Ġdepri": 15659, "Ġincentive": 15660, "ãĤ·": 15661, "Ġoversight": 15662, "Ġtribes": 15663, "ĠBelgium": 15664, "Ġlicensing": 15665, "ourt": 15666, "Product": 15667, "ahl": 15668, "ĠGem": 15669, "Ġspecialist": 15670, "Ġcra": 15671, "anners": 15672, "ĠCorbyn": 15673, "Ġ1973": 15674, "READ": 15675, "Ġsummar": 15676, "Ġoverlook": 15677, "ĠApplication": 15678, "Ġinappropriate": 15679, "Ġdownloaded": 15680, "Que": 15681, "ĠBears": 15682, "Ġthumb": 15683, "ĠCharacter": 15684, "ĠReincarnated": 15685, "ĠSid": 15686, "Ġdemonstrates": 15687, "sky": 15688, "ĠBloomberg": 15689, "ĠArray": 15690, "ĠResults": 15691, "ĠFourth": 15692, "ĠEDT": 15693, "ĠOscar": 15694, "cend": 15695, "Ġ106": 15696, "ĠNULL": 15697, "ĠHERE": 15698, "match": 15699, "ĠBrun": 15700, "Ġglucose": 15701, "ieg": 15702, "egu": 15703, "Ġcertified": 15704, "Ġrelie": 15705, "Ġhumanitarian": 15706, "Ġprayers": 15707, "King": 15708, "Ġnan": 15709, "hou": 15710, "108": 15711, "ulu": 15712, "Ġrenewable": 15713, "Ġdistinguish": 15714, "Ġdense": 15715, "ĠVent": 15716, "ĠPackage": 15717, "ĠBoss": 15718, "Ġeditors": 15719, "Ġmigr": 15720, "Tra": 15721, "ĠPeters": 15722, "ĠArctic": 15723, "2004": 15724, "ĠCape": 15725, "Ġlocally": 15726, "Ġlasting": 15727, "Ġhandy": 15728, ".).": 15729, "Pan": 15730, "ĠRES": 15731, "Index": 15732, "Ġtensions": 15733, "Ġformerly": 15734, "Ġideological": 15735, "Ġsensors": 15736, "Ġdealers": 15737, "Ġdefines": 15738, "Sk": 15739, "Ġproceeds": 15740, "Ġproxy": 15741, "azines": 15742, "ĠBash": 15743, "ĠPad": 15744, "ĠCraft": 15745, "ealous": 15746, "Ġsheets": 15747, "ometry": 15748, "June": 15749, "clock": 15750, "TT": 15751, "ĠTheatre": 15752, "ĠBuzz": 15753, "Ġchapters": 15754, "Ġmillenn": 15755, "Ġdough": 15756, "ĠCongressional": 15757, "Ġimagined": 15758, "avior": 15759, "Ġclinic": 15760, "Ġ1945": 15761, "Ġholder": 15762, "root": 15763, "olester": 15764, "Ġrestart": 15765, "BN": 15766, "ĠHamas": 15767, "ĠJob": 15768, "Ġorb": 15769, "Ġram": 15770, "Ġdisclose": 15771, "Ġtranslate": 15772, "Ġimmigrant": 15773, "Ġannoying": 15774, "Ġtreaty": 15775, "anium": 15776, "ĠTea": 15777, "ĠLegion": 15778, "Ġcrowds": 15779, "ĠBec": 15780, "ĠAer": 15781, "ohyd": 15782, "Bro": 15783, "Looking": 15784, "Ġlbs": 15785, "Ġaggress": 15786, "Ġseam": 15787, "Ġintercept": 15788, "ĠMI": 15789, "mercial": 15790, "activ": 15791, "ĠCit": 15792, "Ġdimension": 15793, "Ġconsistency": 15794, "Ġrushing": 15795, "ĠDouglas": 15796, "Ġtrim": 15797, "Install": 15798, "icker": 15799, "Ġshy": 15800, "106": 15801, "Ġmentions": 15802, "pelled": 15803, "ĠTak": 15804, "cost": 15805, "Ġclassroom": 15806, "Ġfortune": 15807, "driven": 15808, "Ġunle": 15809, "ĠWheel": 15810, "Ġinvestor": 15811, "ĠMasters": 15812, "kit": 15813, "Ġassociations": 15814, "ĠEvolution": 15815, "oping": 15816, "uscript": 15817, "Ġprovincial": 15818, "ĠWalter": 15819, "avi": 15820, "SO": 15821, "Ġunlimited": 15822, "English": 15823, "ĠCards": 15824, "ĠEbola": 15825, "nered": 15826, "Ġrevenge": 15827, "Ġoutright": 15828, "umper": 15829, "Ġfitting": 15830, "ĠSolid": 15831, "Ġformally": 15832, "Ġproblematic": 15833, "Ġhazard": 15834, "Ġencryption": 15835, "Ġstraightforward": 15836, "ĠAK": 15837, "Ġpse": 15838, "ĠOrb": 15839, "ĠChamber": 15840, "ĠMak": 15841, "Contents": 15842, "Ġloyalty": 15843, "Ġlyrics": 15844, "ĠSym": 15845, "Ġwelcomed": 15846, "Ġcooked": 15847, "Ġmonop": 15848, "Ġnurse": 15849, "Ġmisleading": 15850, "Ġeternal": 15851, "Ġshifting": 15852, "Ġ+=": 15853, "Vis": 15854, "Ġinstitutional": 15855, "illary": 15856, "Ġpant": 15857, "VERT": 15858, "ĠACC": 15859, "ĠEnh": 15860, "Ġincon": 15861, "ĠREUTERS": 15862, "Ġdonated": 15863, "âĢ¦âĢ¦âĢ¦âĢ¦": 15864, "Intern": 15865, "Ġexhibit": 15866, "Ġtire": 15867, "ĠRic": 15868, "ĠChampion": 15869, "ĠMuhammad": 15870, "NING": 15871, "ĠSoccer": 15872, "Ġmobility": 15873, "Ġvarying": 15874, "ĠMovie": 15875, "Ġlord": 15876, "oak": 15877, "Field": 15878, "Ġvector": 15879, "usions": 15880, "Ġscrap": 15881, "Ġenabling": 15882, "make": 15883, "Tor": 15884, ".*": 15885, "||": 15886, "ĠWebsite": 15887, "ĠNPC": 15888, "Ġsocialist": 15889, "ĠBilly": 15890, "ĠAdditional": 15891, "Ġcargo": 15892, "Ġfarms": 15893, "ĠSoon": 15894, "ĠPrize": 15895, "Ġmidnight": 15896, "Ġ900": 15897, "seen": 15898, "ĠSpot": 15899, "Ġsheep": 15900, "Ġsponsored": 15901, "ĠHi": 15902, "ĠJump": 15903, "Ġ1967": 15904, "Microsoft": 15905, "ĠAgent": 15906, "Ġcharts": 15907, "dir": 15908, "Ġadjacent": 15909, "Ġtricks": 15910, "Ġmanga": 15911, "Ġexagger": 15912, "/>": 15913, "football": 15914, "ĠFCC": 15915, "GC": 15916, "ĠTier": 15917, "andra": 15918, "OUND": 15919, "%),": 15920, "Ġfruits": 15921, "VC": 15922, "ĠAA": 15923, "Rober": 15924, "Ġmidst": 15925, "âĹ": 15926, "anka": 15927, "Ġlegislature": 15928, "ĠNeil": 15929, "Ġtourists": 15930, "\"\"": 15931, "ĠWarning": 15932, "ĠNevertheless": 15933, "ĠOfficial": 15934, "ĠWhatever": 15935, "Ġmold": 15936, "Ġdrafted": 15937, "Ġsubstances": 15938, "Ġbreed": 15939, "Ġtags": 15940, "ĠTask": 15941, "Ġverb": 15942, "Ġmanufactured": 15943, "comments": 15944, "ĠPolish": 15945, "Prov": 15946, "Ġdetermines": 15947, "Obama": 15948, "kers": 15949, "Ġutterly": 15950, "Ġsect": 15951, "sche": 15952, "ĠGates": 15953, "ĠChap": 15954, "Ġaluminum": 15955, "Ġzombie": 15956, "ĠTouch": 15957, "ĠUP": 15958, "Ġsatisfy": 15959, "Ġpredomin": 15960, "ascript": 15961, "Ġelaborate": 15962, "Ġ1968": 15963, "Ġmeasuring": 15964, "ĠVari": 15965, "anyahu": 15966, "Ġsir": 15967, "ulates": 15968, "idges": 15969, "ickets": 15970, "ĠSpencer": 15971, "TM": 15972, "oubted": 15973, "Ġprey": 15974, "Ġinstalling": 15975, "ĠCab": 15976, "reed": 15977, "reated": 15978, "Supp": 15979, "Ġwrist": 15980, "ĠKerry": 15981, "107": 15982, "ĠKle": 15983, "ĠRachel": 15984, "Ġcotton": 15985, "ĠARE": 15986, "ĠEle": 15987, "Control": 15988, "Ġloads": 15989, "ĠDod": 15990, "anas": 15991, "bone": 15992, "Ġclassical": 15993, "ĠRegional": 15994, "ĠInteg": 15995, "VM": 15996, "Ġdesires": 15997, "Ġautism": 15998, "supported": 15999, "ĠMessage": 16000, "Ġcompact": 16001, "writer": 16002, "Ġ109": 16003, "ĠHurricane": 16004, "cision": 16005, "Ġcycles": 16006, "Ġdrill": 16007, "Ġcolleague": 16008, "Ġmaker": 16009, "German": 16010, "Ġmistaken": 16011, "Sun": 16012, "ĠGay": 16013, "Ġwhatsoever": 16014, "Ġsells": 16015, "ĠAirl": 16016, "liv": 16017, "ĠOption": 16018, "Ġsolved": 16019, "Ġsectors": 16020, "Ġhorizontal": 16021, "Ġequation": 16022, "ĠSkill": 16023, "ĠBio": 16024, "gement": 16025, "ĠSnap": 16026, "ĠLegal": 16027, "Ġtrademark": 16028, "Ġmakeup": 16029, "Ġassembled": 16030, "Ġsaves": 16031, "ĠHalloween": 16032, "ĠVermont": 16033, "ĠFROM": 16034, "Ġfarming": 16035, "ĠPodcast": 16036, "acceptable": 16037, "ĠHigher": 16038, "Ġasleep": 16039, "ullivan": 16040, "Ġreferen": 16041, "ĠLev": 16042, "Ġbullets": 16043, "oko": 16044, "HC": 16045, "Ġstairs": 16046, "Ġmaintains": 16047, "ĠLower": 16048, "ĠVi": 16049, "Ġmarine": 16050, "Ġacres": 16051, "Ġcoordinator": 16052, "ĠJoh": 16053, "Ġcounterparts": 16054, "ĠBrothers": 16055, "Ġindict": 16056, "bra": 16057, "Ġchunk": 16058, "Ġcents": 16059, "Home": 16060, "ĠMonth": 16061, "Ġaccordingly": 16062, "ifles": 16063, "ĠGermans": 16064, "ĠSyn": 16065, "Hub": 16066, "Ġeyeb": 16067, "âĶĢâĶĢâĶĢâĶĢ": 16068, "Ġranges": 16069, "ĠHolland": 16070, "ĠRobot": 16071, "fc": 16072, "Mike": 16073, "Ġplasma": 16074, "Ġswap": 16075, "Ġathlete": 16076, "ĠRams": 16077, ",'\"": 16078, "Ġinfections": 16079, "Ġcorrid": 16080, "Ġvib": 16081, "Ġpatches": 16082, "Ġtraditionally": 16083, "Ġrevelation": 16084, "Ġsweep": 16085, "Ġglance": 16086, "Ġinex": 16087, "2003": 16088, "ĠRaw": 16089, "working": 16090, "osures": 16091, "ĠDat": 16092, "ĠLynch": 16093, "Ġleverage": 16094, "ĠReid": 16095, "Ġcorrelation": 16096, "iances": 16097, "avascript": 16098, "Ġrepository": 16099, "retty": 16100, "Ġ1972": 16101, "240": 16102, "Ġoun": 16103, "pol": 16104, "ĠReed": 16105, "Ġtactical": 16106, "isite": 16107, "Apple": 16108, "ĠQuinn": 16109, "Ġraped": 16110, "illo": 16111, "Europe": 16112, "Ġalgorithms": 16113, "ĠRodrig": 16114, "iu": 16115, "Ġillum": 16116, "Ġfame": 16117, "Ġintroducing": 16118, "Ġdelays": 16119, "ĠRaiders": 16120, "Ġwhistle": 16121, "Ġnovels": 16122, "ĠReally": 16123, "Ġderiv": 16124, "Ġpublications": 16125, "ĠNeither": 16126, "ĠCommerce": 16127, "Ġaston": 16128, "language": 16129, "Notes": 16130, "ĠRoth": 16131, "ĠFear": 16132, "Ġmate": 16133, "Ġparade": 16134, "ĠQB": 16135, "Ġmaneu": 16136, "ĠCincinnati": 16137, "mitting": 16138, "Ġwaist": 16139, "ĠRew": 16140, "Ġdiscont": 16141, "а": 16142, "Ġstaring": 16143, "Ġalias": 16144, "Ġsecurities": 16145, "Ġtoilet": 16146, "ĠJedi": 16147, "Ġunlaw": 16148, "vised": 16149, "////////": 16150, "](": 16151, "ĠWeiss": 16152, "Ġprest": 16153, "ĠCompan": 16154, "Ġmemo": 16155, "ĠGrace": 16156, "July": 16157, "ĠElite": 16158, "center": 16159, "ĠStay": 16160, "Ġgalaxy": 16161, "Ġtooth": 16162, "ĠSettings": 16163, "Ġsubjected": 16164, "ãĤ¦": 16165, "Ġlineback": 16166, "Ġretailers": 16167, "ĠWant": 16168, "Ġdangers": 16169, "Air": 16170, "Ġvoluntary": 16171, "eway": 16172, "Ġinterpreted": 16173, "otine": 16174, "ç": 16175, "Ġpel": 16176, "Service": 16177, "ĠEventually": 16178, "Ġcareers": 16179, "Ġthreaten": 16180, "Ġmemor": 16181, "ĠBradley": 16182, "ancies": 16183, "sn": 16184, "ĠUnknown": 16185, "National": 16186, "Ġshadows": 16187, "ailand": 16188, "ĠDash": 16189, "Everyone": 16190, "izzard": 16191, "March": 16192, "=(": 16193, "Ġpulls": 16194, "Ġstranger": 16195, "Ġbackwards": 16196, "ĠBernard": 16197, "imensional": 16198, "Ġchron": 16199, "Ġtheoretical": 16200, "ktop": 16201, "Ġware": 16202, "ĠInvestig": 16203, "ĠIniti": 16204, "ĠOperations": 16205, "oven": 16206, "ocide": 16207, "*/": 16208, "Ġflames": 16209, "ĠCash": 16210, "shit": 16211, "Ġcab": 16212, "ĠAnaly": 16213, "ĠSeah": 16214, "Ġdefining": 16215, "Ġordering": 16216, "Ġimmun": 16217, "Ġpersistent": 16218, "ACH": 16219, "Russian": 16220, "mans": 16221, "Ġhind": 16222, "Ġphotography": 16223, "©": 16224, "Ġhug": 16225, "Ġ107": 16226, "ĠHence": 16227, "iots": 16228, "udeau": 16229, "Ġsubsidies": 16230, "Ġroutinely": 16231, "ĠDevice": 16232, "itic": 16233, "Ġdisgust": 16234, "lander": 16235, "Ġ1940": 16236, "Ġassignment": 16237, "ĠBesides": 16238, "wick": 16239, "ĠDust": 16240, "usc": 16241, "structed": 16242, "111": 16243, "develop": 16244, "Ġfond": 16245, "Ġintersection": 16246, "Ġdignity": 16247, "Ġcommissioner": 16248, "Without": 16249, "reach": 16250, "Ġcartoon": 16251, "Ġscales": 16252, "ãĥŃ": 16253, "FIG": 16254, "Ġsurveys": 16255, "ĠIndonesia": 16256, "Ġartwork": 16257, "Ġunch": 16258, "Ġcycling": 16259, "unct": 16260, "auer": 16261, "orate": 16262, "ĠObviously": 16263, "Ġcharacterized": 16264, "feld": 16265, "Ġaffirm": 16266, "Ġinnings": 16267, "Ġé": 16268, "Ġaliens": 16269, "Ġcloth": 16270, "etooth": 16271, "ĠCertain": 16272, "§": 16273, "Ġdigest": 16274, "know": 16275, "ĠXL": 16276, "Ġpredictions": 16277, "Ġdin": 16278, "WAR": 16279, "Ġaftermath": 16280, "Example": 16281, "ĠSuccess": 16282, "ĠThr": 16283, "IGN": 16284, "Ġminer": 16285, "Bus": 16286, "Ġclarity": 16287, "heimer": 16288, "ĠOUT": 16289, "ĠSend": 16290, "ĠCircle": 16291, "ĠDiet": 16292, "Ġpronounced": 16293, "Ġcreators": 16294, "Ġearthquake": 16295, "attery": 16296, "geons": 16297, "Ġod": 16298, "Ġlaying": 16299, "orp": 16300, "Ult": 16301, "project": 16302, "Ġundermin": 16303, "Ġsequel": 16304, "Sam": 16305, "ĠDarkness": 16306, "Ġreception": 16307, "bull": 16308, "YS": 16309, "ĠVir": 16310, "Ġsequences": 16311, "ĠCoin": 16312, "Ġoutfit": 16313, "ĠWait": 16314, "119": 16315, "Ġdelivers": 16316, "......": 16317, "Ġblown": 16318, "ĠEsc": 16319, "ĠMath": 16320, "perm": 16321, "ĠUl": 16322, "Ġglim": 16323, "Ġfacial": 16324, "Ġgreenhouse": 16325, "Ġtokens": 16326, "/-": 16327, "ĠAnnual": 16328, "ĠONE": 16329, "Ġteenage": 16330, "ĠPhysical": 16331, "ĠLang": 16332, "ĠCelt": 16333, "Ġsued": 16334, "ividually": 16335, "Ġpatience": 16336, "chair": 16337, "regular": 16338, "Ġaug": 16339, "inv": 16340, "except": 16341, "ĠLil": 16342, "Ġnest": 16343, "fd": 16344, "sum": 16345, "ĠChase": 16346, "Russia": 16347, "ĠJennifer": 16348, "Ġoffseason": 16349, "Overall": 16350, "Fore": 16351, "Ġriot": 16352, "Aud": 16353, "former": 16354, "Ġdefenders": 16355, "ĠCT": 16356, "iotic": 16357, "ribly": 16358, "Ġautomated": 16359, "Ġpenis": 16360, "Ġinsist": 16361, "Ġdiagram": 16362, "ĠSQL": 16363, "ĠGarc": 16364, "Ġwitch": 16365, "client": 16366, "ierra": 16367, "ambers": 16368, "Ġrecount": 16369, "far": 16370, "Very": 16371, "osterone": 16372, "Ġappreciated": 16373, "ĠPerfect": 16374, "Section": 16375, "Ġdoses": 16376, "ocaust": 16377, "Ġcostly": 16378, "Ġgrams": 16379, "ĠShi": 16380, "Ġwrestling": 16381, "Ġ1971": 16382, "Ġtrophy": 16383, "Ġnerve": 16384, "ĠKaz": 16385, "ĠExperience": 16386, "Ġpledged": 16387, "Ġplayback": 16388, "Ġcreativity": 16389, "bye": 16390, "Ġattackers": 16391, "Ġholders": 16392, "ĠCoach": 16393, "ĠPhD": 16394, "Ġtransfers": 16395, "Ġcolored": 16396, "ĠHindu": 16397, "Ġdrown": 16398, "Ġlistened": 16399, "ĠWA": 16400, "iasm": 16401, "PO": 16402, "Ġappealing": 16403, "Ġdisclosed": 16404, "ĠChicken": 16405, "agging": 16406, "Ġpleaded": 16407, "Ġnavigation": 16408, "ĠReturns": 16409, "Ġ[[": 16410, "ROR": 16411, "EA": 16412, "Ġphotographer": 16413, "ĠRider": 16414, "ippers": 16415, "Ġslice": 16416, "Ġerect": 16417, "Ġhed": 16418, "issance": 16419, "ĠVikings": 16420, "urious": 16421, "Ġappet": 16422, "oubtedly": 16423, "Child": 16424, "Ġauthentic": 16425, "oos": 16426, "ĠMaking": 16427, "Ġannouncing": 16428, "Ġbod": 16429, "Ġmeter": 16430, "ĠNine": 16431, "ĠRogue": 16432, "Ġworkforce": 16433, "Ġrenewed": 16434, "Ġorganisations": 16435, "acs": 16436, "PLE": 16437, "Short": 16438, "Ġcompounds": 16439, "ĠVisit": 16440, "Ġenvelop": 16441, "earth": 16442, "Ġsupportive": 16443, "ggle": 16444, "ĠBrussels": 16445, "ĠGuild": 16446, "Create": 16447, "REL": 16448, "Ġaveraged": 16449, "Ġ1969": 16450, "riages": 16451, "Ġlengthy": 16452, "Ġforgot": 16453, "Okay": 16454, "ĠErd": 16455, "Ġdealer": 16456, "Ġrecession": 16457, "DD": 16458, "Ġdesperately": 16459, "Ġhunger": 16460, "Ġsticks": 16461, "Ġmph": 16462, "ĠFaith": 16463, "Ġintentionally": 16464, "Ġdemol": 16465, "ueller": 16466, "ĠSale": 16467, "Ġdebris": 16468, "spring": 16469, "Ġleap": 16470, ">>>>": 16471, "Ġcontainers": 16472, "selling": 16473, "ranean": 16474, "attering": 16475, "Ġcommented": 16476, "ĠCM": 16477, "onut": 16478, "Ġwoods": 16479, "especially": 16480, "Ġorganize": 16481, "ivic": 16482, "ĠWoods": 16483, "anga": 16484, "squ": 16485, "Ġmaj": 16486, "amon": 16487, "Ġaxis": 16488, "Ġ1974": 16489, "ĠDenmark": 16490, "Ġwarrior": 16491, "ĠPand": 16492, "Ġoutlined": 16493, "ĠBO": 16494, "insula": 16495, "zilla": 16496, "ebook": 16497, "Ġdare": 16498, "Ġsearched": 16499, "Ġnavigate": 16500, "Sn": 16501, "writing": 16502, "Ġunited": 16503, "Japan": 16504, "ĠHebrew": 16505, "Ġflame": 16506, "Ġrelies": 16507, "Ġcatching": 16508, "ĠSho": 16509, "Ġimprisonment": 16510, "Ġpockets": 16511, "Ġclosure": 16512, "ĠFam": 16513, "tim": 16514, "adequ": 16515, "Activity": 16516, "Ġrecruiting": 16517, "ĠWATCH": 16518, "ĠArgentina": 16519, "dest": 16520, "Ġapologize": 16521, "oro": 16522, "Ġlacks": 16523, "Ġtuned": 16524, "ĠGriffin": 16525, "Ġinfamous": 16526, "Ġcelebrity": 16527, "sson": 16528, "Ġ----------------------------------------------------------------": 16529, "ĠIsis": 16530, "ĠDisplay": 16531, "Ġcredibility": 16532, "Ġeconomies": 16533, "Ġheadline": 16534, "ĠCowboys": 16535, "Ġindef": 16536, "Ġlately": 16537, "Ġincentives": 16538, "button": 16539, "ĠMob": 16540, "Aut": 16541, "Ġresigned": 16542, "ĠOm": 16543, "camp": 16544, "Ġprofiles": 16545, "Ġschemes": 16546, "olphins": 16547, "ayed": 16548, "Clinton": 16549, "enh": 16550, "ĠYahoo": 16551, "Ġabst": 16552, "Ġank": 16553, "suits": 16554, "Ġwished": 16555, "ĠMarco": 16556, "udden": 16557, "Ġsphere": 16558, "ĠBishop": 16559, "Ġincorporated": 16560, "ĠPlant": 16561, "114": 16562, "Ġhated": 16563, "pic": 16564, "Ġdonate": 16565, "Ġlined": 16566, "Ġbeans": 16567, "Ġstealing": 16568, "Ġcostume": 16569, "Ġsheriff": 16570, "Ġforty": 16571, "Ġintact": 16572, "Ġadapted": 16573, "Ġtravelling": 16574, "bart": 16575, "Ġnicely": 16576, "Ġdried": 16577, "Ġscal": 16578, "osity": 16579, "NOTE": 16580, "ĠBh": 16581, "ĠBroncos": 16582, "ĠIgn": 16583, "Ġintimate": 16584, "Ġchemistry": 16585, "Ġoptimal": 16586, "Deb": 16587, "ĠGeneration": 16588, "Ġ],": 16589, "ichi": 16590, "ĠWii": 16591, "ĠYOUR": 16592, "ventions": 16593, "Write": 16594, "Ġpopul": 16595, "unning": 16596, "ĠWor": 16597, "Vol": 16598, "Ġqueen": 16599, "heads": 16600, "KK": 16601, "Ġanalyze": 16602, "opic": 16603, "earchers": 16604, "Ġdot": 16605, "legraph": 16606, "astically": 16607, "Ġupgrades": 16608, "Ġcares": 16609, "Ġextending": 16610, "Ġfreeze": 16611, "Ġinability": 16612, "Ġorgans": 16613, "Ġpretend": 16614, "Ġoutlet": 16615, "113": 16616, "olan": 16617, "ĠMall": 16618, "uling": 16619, "talk": 16620, "Ġexpressing": 16621, "ĠAlways": 16622, "ĠBegin": 16623, "files": 16624, "Ġlicenses": 16625, "%%": 16626, "ĠMitt": 16627, "Ġfilters": 16628, "ĠMilwaukee": 16629, "GN": 16630, "Ġunfold": 16631, "Mo": 16632, "Ġnutrition": 16633, "ppo": 16634, "Bo": 16635, "Ġfounding": 16636, "Ġundermine": 16637, "Ġeasiest": 16638, "ĠCzech": 16639, "ĠMack": 16640, "Ġsexuality": 16641, "ĠNixon": 16642, "Win": 16643, "ĠArn": 16644, "ĠKin": 16645, "ãĤ£": 16646, "icer": 16647, "Ġfortun": 16648, "Ġsurfaces": 16649, "aghd": 16650, "Ġcarriers": 16651, "ĠPART": 16652, "ĠTib": 16653, "Ġinterval": 16654, "Ġfrustrating": 16655, "ĠShip": 16656, "ĠArmed": 16657, "ffe": 16658, "Ġboats": 16659, "ĠAbraham": 16660, "inis": 16661, "Ġsuited": 16662, "thread": 16663, "iov": 16664, "abul": 16665, "ĠVenezuela": 16666, "Ġtom": 16667, "super": 16668, "Ġcastle": 16669, "although": 16670, "ioxide": 16671, "eches": 16672, "Ġevolutionary": 16673, "Ġnegotiate": 16674, "Ġconfronted": 16675, "Remember": 16676, "Ġ170": 16677, "Such": 16678, "Ġ911": 16679, "mult": 16680, "ĠAbyss": 16681, "urry": 16682, "kees": 16683, "spec": 16684, "ĠBarbara": 16685, "Ġbelonging": 16686, "Ġvillain": 16687, "istani": 16688, "Ġaccountable": 16689, "Ġportions": 16690, "ĠDecl": 16691, "Ur": 16692, "ĠKate": 16693, "gre": 16694, "Ġmagazines": 16695, "UCK": 16696, "Ġregulate": 16697, "omon": 16698, "ĠAlmost": 16699, "Ġoverview": 16700, "Ġscram": 16701, "Ġloot": 16702, "ĠFitz": 16703, "Ġcharacteristic": 16704, "ĠSnake": 16705, "say": 16706, "ĠRico": 16707, "Ġtrait": 16708, "ĠJoined": 16709, "aucus": 16710, "Ġadaptation": 16711, "ĠAirlines": 16712, "Ġarchae": 16713, "ĠIde": 16714, "Ġbikes": 16715, "Ġliterary": 16716, "Ġinfluences": 16717, "ĠUsed": 16718, "Creat": 16719, "Ġplea": 16720, "ĠDefence": 16721, "ĠAssass": 16722, "Ġpond": 16723, "ULT": 16724, ")\"": 16725, "Ġevaluated": 16726, "Ġobtaining": 16727, "Ġdemographic": 16728, "Ġvigil": 16729, "aley": 16730, "Ġspouse": 16731, "ĠSeahawks": 16732, "respons": 16733, "ĠBelt": 16734, "umatic": 16735, "Ġrises": 16736, "runner": 16737, "ĠMichelle": 16738, "Ġpotent": 16739, "race": 16740, "ĠPAC": 16741, "Find": 16742, "olesterol": 16743, "ISS": 16744, "ĠIntroduced": 16745, "resses": 16746, "ignment": 16747, "Os": 16748, "ĠTu": 16749, "ĠDex": 16750, "icides": 16751, "Ġsparked": 16752, "ĠLaura": 16753, "ĠBryant": 16754, "Ġsmiling": 16755, "ĠNexus": 16756, "Ġdefendants": 16757, "ĠCatal": 16758, "Ġdishes": 16759, "shaped": 16760, "Ġprolong": 16761, "mt": 16762, "($": 16763, "ãĢĤ": 16764, "Ġcalculations": 16765, "ĠSame": 16766, "Ġpiv": 16767, "HH": 16768, "Ġcancelled": 16769, "Ġgrin": 16770, "Ġterritories": 16771, "istically": 16772, "Come": 16773, "ĠParent": 16774, "Project": 16775, "Ġneglig": 16776, "ĠPrivacy": 16777, "Ġammo": 16778, "LECT": 16779, "olutely": 16780, "ĠEpic": 16781, "Ġmisunder": 16782, "wal": 16783, "April": 16784, "mos": 16785, "pathy": 16786, "ĠCarson": 16787, "Ġalbums": 16788, "ĠEasy": 16789, "Ġpistol": 16790, "<<": 16791, "Ġ\\(": 16792, "target": 16793, "help": 16794, "Ġinterpre": 16795, "conscious": 16796, "ĠHousing": 16797, "ĠJoint": 16798, "127": 16799, "Ġbeers": 16800, "science": 16801, "ĠFirefox": 16802, "effective": 16803, "ĠCabin": 16804, "ĠOkay": 16805, "ĠApplic": 16806, "Ġspacecraft": 16807, "ĠSR": 16808, "vet": 16809, "ĠStrange": 16810, "SB": 16811, "Ġcorps": 16812, "iberal": 16813, "efficient": 16814, "Ġprevalence": 16815, "Ġeconomists": 16816, "118": 16817, "Thread": 16818, "ordable": 16819, "ODE": 16820, "ĠCant": 16821, "=-=-": 16822, "ifiable": 16823, "ĠAround": 16824, "Ġpole": 16825, "Ġwillingness": 16826, "CLA": 16827, "ĠKid": 16828, "Ġcomplement": 16829, "Ġscattered": 16830, "Ġinmates": 16831, "Ġbleeding": 16832, "every": 16833, "Ġqueue": 16834, "ĠTrain": 16835, "Ġhij": 16836, "Ġmelee": 16837, "pleted": 16838, "Ġdigit": 16839, "Ġgem": 16840, "official": 16841, "Ġlifting": 16842, "е": 16843, "Requ": 16844, "itutes": 16845, "Ġpackaging": 16846, "ĠWorkers": 16847, "hran": 16848, "ĠLebanon": 16849, "olesc": 16850, "Ġpunished": 16851, "ĠJuan": 16852, "Ġjam": 16853, "ĠDocument": 16854, "Ġmapping": 16855, "icates": 16856, "Ġinevitably": 16857, "Ġvanilla": 16858, "ĠTon": 16859, "Ġwatches": 16860, "Ġleagues": 16861, "Ġinitiated": 16862, "degree": 16863, "portion": 16864, "Ġrecalls": 16865, "Ġruin": 16866, "Ġmelt": 16867, "IAN": 16868, "Ġhem": 16869, "Exp": 16870, "Ġbaking": 16871, "ĠColomb": 16872, "atible": 16873, "Ġradius": 16874, "plug": 16875, "ĠIF": 16876, "etically": 16877, "Ġfict": 16878, "HER": 16879, "ĠTap": 16880, "atinum": 16881, "Ġink": 16882, "Ġcoh": 16883, "ĠWizard": 16884, "both": 16885, "tex": 16886, "Ġspends": 16887, "ĠCurrently": 16888, "ĠPit": 16889, "Ġneurons": 16890, "ignt": 16891, "Ġrall": 16892, "Ġbuses": 16893, "building": 16894, "Ġadjustments": 16895, "Ġcried": 16896, "iblical": 16897, "atted": 16898, "ĠZion": 16899, "ĠMatter": 16900, "Ġmeditation": 16901, "ĠDennis": 16902, "Ġours": 16903, "ĠTab": 16904, "Ġrankings": 16905, "ortal": 16906, "Ġadvers": 16907, "Ġsurrender": 16908, "ĠGob": 16909, "cium": 16910, "omas": 16911, "imeter": 16912, "Ġmultiplayer": 16913, "Ġheroin": 16914, "Ġoptimistic": 16915, "Ġindicator": 16916, "ĠBrig": 16917, "Ġgrocery": 16918, "Ġapplicant": 16919, "ĠRocket": 16920, "vid": 16921, "Exception": 16922, "pent": 16923, "Ġorganizing": 16924, "Ġencounters": 16925, "ĠTOD": 16926, "Ġjewel": 16927, "Save": 16928, "ĠChristie": 16929, "Ġheating": 16930, "Ġlazy": 16931, "ĠCP": 16932, "Ġcousin": 16933, "Config": 16934, "Ġregener": 16935, "Ġnearest": 16936, "Ġachieving": 16937, "ENS": 16938, "throw": 16939, "ĠRichmond": 16940, "antle": 16941, "2002": 16942, "Ġanten": 16943, "bird": 16944, "133": 16945, "Ġnarc": 16946, "raint": 16947, "unny": 16948, "ĠHispanic": 16949, "ournaments": 16950, "Ġprophe": 16951, "ĠThailand": 16952, "ĠTi": 16953, "Ġinjection": 16954, "Ġinherit": 16955, "ravis": 16956, "Ġmedi": 16957, "Ġwhoever": 16958, "ĠDEBUG": 16959, "GP": 16960, "ĠHud": 16961, "Card": 16962, "prom": 16963, "Ġpor": 16964, "Ġoverhead": 16965, "Law": 16966, "Ġviolate": 16967, "Ġheated": 16968, "Ġdescriptions": 16969, "Ġachievements": 16970, "ĠBeer": 16971, "ĠQuant": 16972, "Was": 16973, "Ġeighth": 16974, "ĠIv": 16975, "Ġspecialized": 16976, "UPDATE": 16977, "ĠDelta": 16978, "Pop": 16979, "Jul": 16980, "ĠAsk": 16981, "ophy": 16982, "Ġnewsletters": 16983, "ĠTool": 16984, "Ġgard": 16985, "ĠConfeder": 16986, "ĠGMT": 16987, "ĠAbbott": 16988, "Ġimmunity": 16989, "ĠVM": 16990, "Islam": 16991, "Ġimplicit": 16992, "wd": 16993, "Ġ1944": 16994, "ravity": 16995, "ometric": 16996, "Ġsurviving": 16997, "urai": 16998, "ĠPrison": 16999, "Ġrust": 17000, "ĠSketch": 17001, "Ġbees": 17002, "ĠTheory": 17003, "Ġmerit": 17004, "Tex": 17005, "chat": 17006, "Ġmim": 17007, "Ġpaste": 17008, "ĠKoch": 17009, "Ġignorance": 17010, "ĠShoot": 17011, "Ġbasement": 17012, "United": 17013, "ĠAdvis": 17014, "height": 17015, "Ġfoster": 17016, "Ġdetain": 17017, "information": 17018, "Ġneural": 17019, "';": 17020, "Ġproves": 17021, "allery": 17022, "Ġinvitation": 17023, "umbers": 17024, "Ġcattle": 17025, "Ġbicycle": 17026, "zi": 17027, "Ġconsultant": 17028, "Ġapology": 17029, "ĠTiger": 17030, "Ġ123": 17031, "999": 17032, "Ġindividually": 17033, "rt": 17034, "igion": 17035, "ĠBrazilian": 17036, "Ġdisturb": 17037, "Ġentrepreneurs": 17038, "Ġforests": 17039, "cerpt": 17040, "plates": 17041, "pher": 17042, "clipse": 17043, "Ġtwitter": 17044, "Ġacids": 17045, "ographical": 17046, "hum": 17047, "ĠBald": 17048, "ifully": 17049, "Ġcompiler": 17050, "ĠDA": 17051, "Ġdonor": 17052, "asi": 17053, "Ġtribal": 17054, "lash": 17055, "ĠConfig": 17056, "Ġapplicants": 17057, "Ġsalaries": 17058, "135": 17059, "Putin": 17060, "ĠFocus": 17061, "irs": 17062, "Ġmisconduct": 17063, "ĠHaz": 17064, "Ġeaten": 17065, "Mobile": 17066, "Muslim": 17067, "ĠMarcus": 17068, "viol": 17069, "Ġfavorable": 17070, "Ġstub": 17071, "adin": 17072, "ĠHob": 17073, "Ġfaithful": 17074, "Ġelectronics": 17075, "Ġvacuum": 17076, "wait": 17077, "backed": 17078, "economic": 17079, "dist": 17080, "Ġtenure": 17081, "Ġsincere": 17082, "ĠTogether": 17083, "ĠWave": 17084, "Ġprogression": 17085, "Ġdenying": 17086, "Ġdistress": 17087, "braska": 17088, "third": 17089, "Ġmixing": 17090, "Ġcolonial": 17091, "Ġprivately": 17092, "Ġunrest": 17093, "aternity": 17094, "Ġpremises": 17095, "anti": 17096, "gregation": 17097, "Ġlicence": 17098, "ĠHind": 17099, "ĠSamuel": 17100, "Ġconvincing": 17101, "ĠAce": 17102, "ĠRust": 17103, "ĠNetanyahu": 17104, "Ġhandles": 17105, "ĠPatch": 17106, "oriented": 17107, "aho": 17108, "ĠGonz": 17109, "Ġhackers": 17110, "claimer": 17111, "Ġcustoms": 17112, "ĠGran": 17113, "fighters": 17114, "Ġluc": 17115, "Ġmanuscript": 17116, "arenthood": 17117, "Ġdevil": 17118, "Ġwarriors": 17119, "Ġoffenders": 17120, "William": 17121, "Ġholidays": 17122, "Ġnightmare": 17123, "Ġlever": 17124, "ifferent": 17125, "Stat": 17126, "Ġexhibition": 17127, "puted": 17128, "ĠPure": 17129, "Ġalpha": 17130, "Ġenthusiasm": 17131, "ĠRepresentatives": 17132, "EAR": 17133, "ĠTyp": 17134, "Ġwheat": 17135, "ĠAlf": 17136, "Ġcorrection": 17137, "Ġevangel": 17138, "ATT": 17139, "Miss": 17140, "Ġsoup": 17141, "Ġimplied": 17142, "param": 17143, "Ġsexy": 17144, "ĠLux": 17145, "Ġrepublic": 17146, "patch": 17147, "ablish": 17148, "Ġicons": 17149, "Ġfathers": 17150, "ĠGET": 17151, "ĠCarib": 17152, "Ġregulated": 17153, "ĠCohen": 17154, "ĠBobby": 17155, "Ġner": 17156, "Ġbent": 17157, "ventory": 17158, "ĠAlong": 17159, "ĠEST": 17160, "ĠWallace": 17161, "Ġmurders": 17162, "rise": 17163, "kell": 17164, "ĠCommonwealth": 17165, "Ġnasty": 17166, "eta": 17167, "ĠMIT": 17168, "Ġadministered": 17169, "Ġgenuinely": 17170, "Editor": 17171, "nick": 17172, "Ġhydro": 17173, "********************************": 17174, "ĠBle": 17175, "Ġfines": 17176, "Ġgorge": 17177, "ausible": 17178, "rh": 17179, "Ġapple": 17180, "mentioned": 17181, "Ġrope": 17182, "otyp": 17183, "HR": 17184, "Ġdisappointing": 17185, "Ġcage": 17186, "nik": 17187, "Ġdoubts": 17188, "ĠFREE": 17189, "prints": 17190, "ĠMUST": 17191, "Ġvendors": 17192, "ĠInqu": 17193, "Ġliberals": 17194, "Ġcontractor": 17195, "Ġupside": 17196, "children": 17197, "Ġtricky": 17198, "Ġregulators": 17199, "charged": 17200, "liter": 17201, "Ġ***": 17202, "Ġrebell": 17203, "lang": 17204, "Ġlocals": 17205, "Ġphysicians": 17206, "Ġhey": 17207, "arse": 17208, "tm": 17209, "ĠLex": 17210, "Ġbehavioral": 17211, "successful": 17212, "FX": 17213, "Ġbrick": 17214, "ovic": 17215, "Ġconform": 17216, "Ġreviewing": 17217, "Ġinsights": 17218, "Ġbiology": 17219, "ĠRemove": 17220, "ĠExtra": 17221, "Ġcommitting": 17222, "induced": 17223, "ignty": 17224, "igm": 17225, "Ġatomic": 17226, "Common": 17227, "ĠEM": 17228, "ĠPere": 17229, "ĠItems": 17230, "eh": 17231, "Ġpreserved": 17232, "ĠHood": 17233, "Ġprisoner": 17234, "Ġbankruptcy": 17235, "Ġgren": 17236, "ushes": 17237, "Ġexploitation": 17238, "Ġsignatures": 17239, "Ġfinan": 17240, "],\"": 17241, "ĠMR": 17242, "Ġmeg": 17243, "remlin": 17244, "Ġmusicians": 17245, "Ġselecting": 17246, "Ġexamining": 17247, "INK": 17248, "lated": 17249, "Hi": 17250, "Ġartic": 17251, "Ġpets": 17252, "Ġimpair": 17253, "ĠMAN": 17254, "Ġtablets": 17255, "include": 17256, "Range": 17257, "Ġcaut": 17258, "Ġlogs": 17259, "Ġmounting": 17260, "Ġunaware": 17261, "Ġdynamics": 17262, "ĠPalestine": 17263, "ĠQuarter": 17264, "ĠPurple": 17265, "Ġma": 17266, "ĠImport": 17267, "Ġcollections": 17268, "ciation": 17269, "Ġsuccessor": 17270, "Ġclone": 17271, "Ġaiming": 17272, "Ġpossessed": 17273, "Ġsticking": 17274, "Ġshaking": 17275, "Ġlocate": 17276, "ĠHockey": 17277, "Turn": 17278, "170": 17279, "Ġfifteen": 17280, "ĠHarrison": 17281, "Ġcontinuously": 17282, "ĠTC": 17283, "ĠValent": 17284, "ĠRescue": 17285, "Ġbypass": 17286, "amount": 17287, "Ġmast": 17288, "Ġprotects": 17289, "Ġartistic": 17290, "Ġsometime": 17291, "Ġshoe": 17292, "Ġshouted": 17293, "ificant": 17294, "etitive": 17295, "ĠRegister": 17296, "ĠJin": 17297, "Ġconcentrated": 17298, "lington": 17299, "onies": 17300, "Ġgenerator": 17301, "yrim": 17302, "ĠArmen": 17303, "Ġclearing": 17304, "ido": 17305, "ĠTW": 17306, "alph": 17307, "Ġladies": 17308, "Hard": 17309, "Ġdialog": 17310, "Ġinputs": 17311, "æľ": 17312, "Ġposes": 17313, "Ġslots": 17314, "ĠPremium": 17315, "Ġleaks": 17316, "Ġbosses": 17317, "Ġ113": 17318, "course": 17319, "Acc": 17320, "ĠNewton": 17321, "ĠAustria": 17322, "ĠMage": 17323, "Ġteaches": 17324, "abad": 17325, "Ġwears": 17326, "Ġcyl": 17327, "Ġcurse": 17328, "ĠSales": 17329, "ĠWings": 17330, "Ġpsy": 17331, "Ġgaps": 17332, "ĠIceland": 17333, "ĠPinterest": 17334, "Ġlandlord": 17335, "Ġdefinitions": 17336, "ĠKer": 17337, "Ġsufficiently": 17338, "ĠPence": 17339, "ĠArchitect": 17340, "Ġsurpass": 17341, "Ġ114": 17342, "Ġsuperhero": 17343, "ĠDisease": 17344, "Ġpriests": 17345, "ĠCulture": 17346, "Ġdefinitive": 17347, "Ġsecretly": 17348, "ĠDance": 17349, "install": 17350, "chief": 17351, "ĠJessica": 17352, "Would": 17353, "Updated": 17354, "Ġlocker": 17355, "ĠKay": 17356, "Ġmemorial": 17357, "è¦": 17358, "fat": 17359, "Ġdisgu": 17360, "Ġflavors": 17361, "ĠBaseball": 17362, "ĠResistance": 17363, "Ġkicks": 17364, "Ġenv": 17365, "Ġteenagers": 17366, "Dark": 17367, "ĠCAR": 17368, "Ġhalt": 17369, "ĠLG": 17370, "ĠGabriel": 17371, "Ġfever": 17372, "Ġsatur": 17373, "Ġmall": 17374, "Ġaffiliate": 17375, "ĠSleep": 17376, "ĠSpecific": 17377, "ĠVel": 17378, "Ġjar": 17379, "ĠSacred": 17380, "ĠEdwards": 17381, "ĠACL": 17382, "Ġretained": 17383, "ĠGiant": 17384, "Ġlimitation": 17385, "inces": 17386, "Ġrefusal": 17387, "ĠTale": 17388, "ĠButler": 17389, "Ġaccidents": 17390, "ĠCSS": 17391, "Ġimported": 17392, "ĠCopy": 17393, "α": 17394, "ERT": 17395, "zel": 17396, "Ġdivisions": 17397, "hots": 17398, "ĠAlb": 17399, "ĠDS": 17400, "Loader": 17401, "Washington": 17402, "atisf": 17403, "ĠCreative": 17404, "\\.": 17405, "ĠAutom": 17406, "redict": 17407, "Ġreceptor": 17408, "ĠCarlos": 17409, "Method": 17410, "oka": 17411, "Ġmalicious": 17412, "Ġstepping": 17413, ",[": 17414, "ĠDad": 17415, "Ġattraction": 17416, "ĠEffects": 17417, "ĠPirate": 17418, "ĠCer": 17419, "ĠIndustry": 17420, "ĠRud": 17421, "Ġcharter": 17422, "Ġdining": 17423, "Ġinsists": 17424, "Ġconfigure": 17425, "Ġ(#": 17426, "ĠSimple": 17427, "ĠScroll": 17428, "UTC": 17429, "175": 17430, "ĠKon": 17431, "Ġmarketplace": 17432, "ĠãĤ": 17433, "Ġrefres": 17434, "Ġgates": 17435, "erred": 17436, "ĠPod": 17437, "Ġbehave": 17438, "Frank": 17439, "node": 17440, "Ġendorsed": 17441, "hett": 17442, "asive": 17443, "ĠHomeland": 17444, "Ġrides": 17445, "ĠLeave": 17446, "erness": 17447, "Ġflooding": 17448, "AFP": 17449, "Ġrisen": 17450, "Ġcontinually": 17451, "Ġunanim": 17452, "ĠContract": 17453, "ĠPas": 17454, "Ġguided": 17455, "ĠChile": 17456, "bd": 17457, "Ġsucc": 17458, "ptic": 17459, "Ġcommittees": 17460, "ĠLuther": 17461, "ĠAnyone": 17462, "Ġsab": 17463, "124": 17464, "Ġpixel": 17465, "ĠBak": 17466, "ĠTag": 17467, "ĠBennett": 17468, "Enter": 17469, "small": 17470, "ĠPresidential": 17471, "Ġpul": 17472, "Ġcontrace": 17473, "archive": 17474, "Ġcoastal": 17475, "ĠKids": 17476, "192": 17477, "âĢ²": 17478, "icky": 17479, "INGTON": 17480, "Ġwolf": 17481, "ĠStalin": 17482, "Tur": 17483, "idget": 17484, "amas": 17485, "ĠUnless": 17486, "Ġsponsor": 17487, "Ġmorph": 17488, "ĠChoose": 17489, "Ġrunner": 17490, "Ġunbel": 17491, "Ġmud": 17492, "ĠMana": 17493, "Ġdubbed": 17494, "Ġgodd": 17495, "urers": 17496, "window": 17497, "Ġrelied": 17498, "Ġcelebrating": 17499, "osc": 17500, "Ġ135": 17501, "Ġlobbying": 17502, "Ġincomplete": 17503, "Ġrestriction": 17504, "Ġincap": 17505, "itus": 17506, "Ġexpectation": 17507, "ĠApollo": 17508, "Ġintens": 17509, "Ġsync": 17510, "GH": 17511, "Ġmanipulation": 17512, "BY": 17513, "Ġspear": 17514, "Ġbreasts": 17515, "Ġvolcan": 17516, "ilia": 17517, "Material": 17518, "Ġformats": 17519, "ĠBast": 17520, "Ġparliamentary": 17521, "Ġsnake": 17522, "Ġservants": 17523, "ĠTrudeau": 17524, "ĠGrim": 17525, "ĠArabic": 17526, "ĠSCP": 17527, "ĠBoys": 17528, "station": 17529, "Ġprospective": 17530, "orde": 17531, "initialized": 17532, "Ġbored": 17533, "ABLE": 17534, "Ġaccessed": 17535, "Ġtaxi": 17536, "ĠShell": 17537, "aiden": 17538, "ursed": 17539, "inates": 17540, "ĠInsurance": 17541, "ĠPete": 17542, "September": 17543, "650": 17544, "Ġadventures": 17545, "ĠCover": 17546, "Ġtribute": 17547, "Ġsketch": 17548, "Ġempower": 17549, "ĠØ": 17550, "ĠGlenn": 17551, "ĠDaw": 17552, "=\\\"": 17553, "ĠPolitics": 17554, "Ġguides": 17555, "Ġdioxide": 17556, "ĠGore": 17557, "ĠBright": 17558, "ĠSierra": 17559, "Ġvalued": 17560, "cond": 17561, "Ġpointer": 17562, "Select": 17563, "Ġrisky": 17564, "Ġabsorb": 17565, "images": 17566, "Ġrefuses": 17567, "Ġbonuses": 17568, "___": 17569, "Ġhilar": 17570, "ĠFeatures": 17571, "220": 17572, "ĠCollector": 17573, "Foot": 17574, "Ġ1964": 17575, "culus": 17576, "Ġdawn": 17577, "Ġworkout": 17578, "ĠLO": 17579, "Ġphilosophical": 17580, "ĠSandy": 17581, "ĠYouth": 17582, "Ġliable": 17583, "Af": 17584, "blue": 17585, "Ġoverturn": 17586, "lessness": 17587, "ĠTribune": 17588, "ĠIng": 17589, "Ġfactories": 17590, "Ġcatches": 17591, "Ġprone": 17592, "Ġmatrix": 17593, "Ġlogin": 17594, "Ġinacc": 17595, "Ġexert": 17596, "sys": 17597, "Ġneedle": 17598, "ĠQur": 17599, "Ġnotified": 17600, "oulder": 17601, "tx": 17602, "Ġreminds": 17603, "Ġpublishers": 17604, "Ġnort": 17605, "Ġgit": 17606, "Ġflies": 17607, "ĠEmily": 17608, "Ġflowing": 17609, "ĠAlien": 17610, "ĠStrateg": 17611, "Ġhardest": 17612, "Ġmodification": 17613, "API": 17614, "ĠMY": 17615, "Ġcrashes": 17616, "stairs": 17617, "number": 17618, "Ġurging": 17619, "channel": 17620, "ĠFalcon": 17621, "Ġinhabitants": 17622, "Ġterrifying": 17623, "Ġutilize": 17624, "Ġbanner": 17625, "Ġcigarettes": 17626, "Ġsenses": 17627, "ĠHolmes": 17628, "Ġpractition": 17629, "ĠPhillips": 17630, "otto": 17631, "Ġcompile": 17632, "Model": 17633, "ĠKo": 17634, "Ġ[]": 17635, "Americans": 17636, "ĠTerms": 17637, "Ġmedications": 17638, "ĠAna": 17639, "Ġfundamentally": 17640, "ĠNotice": 17641, "Ġweaker": 17642, "Ġ0000": 17643, "Ġgarlic": 17644, "Ġoutbreak": 17645, "Ġeconomist": 17646, "ĠBirth": 17647, "Ġobstacles": 17648, "arcer": 17649, "ĠOrthodox": 17650, "Ġplacebo": 17651, "ĠCrew": 17652, "aspberry": 17653, "ĠAngels": 17654, "Ġdischarge": 17655, "Ġdestructive": 17656, "117": 17657, "ĠRising": 17658, "Ġdairy": 17659, "late": 17660, "Ġcollision": 17661, "ĠTigers": 17662, "eanor": 17663, "ocumented": 17664, "ĠInvalid": 17665, "Ġdont": 17666, "ĠLiter": 17667, "ĠVa": 17668, "Ġhydrogen": 17669, "Ġvariants": 17670, "ĠBrowns": 17671, "Ġ1965": 17672, "Ġindigenous": 17673, "Ġtrades": 17674, "Ġremainder": 17675, "Ġswept": 17676, "ĠImpact": 17677, "Ġredist": 17678, "Ġunint": 17679, "graduate": 17680, "ãĥķ": 17681, "ĠWILL": 17682, "ãģ®ç": 17683, "ĠCritical": 17684, "Ġfisher": 17685, "Ġvicious": 17686, "Ġreversed": 17687, "Year": 17688, "ĠSox": 17689, "Ġshootings": 17690, "Ġfilming": 17691, "Ġtouchdowns": 17692, "aires": 17693, "mel": 17694, "Ġgrandfather": 17695, "Ġaffection": 17696, "ingle": 17697, "Ġoverly": 17698, "Additional": 17699, "Ġsupreme": 17700, "ĠGrad": 17701, "Ġsporting": 17702, "Ġmercy": 17703, "ĠBrooks": 17704, "ounty": 17705, "Ġperforms": 17706, "Ġtightly": 17707, "Ġdemons": 17708, "Ġkillings": 17709, "Ġfaction": 17710, "ĠNova": 17711, "auts": 17712, "Ġundoubtedly": 17713, "arin": 17714, "Ġunderway": 17715, "rak": 17716, "Ġliv": 17717, "ĠRegion": 17718, "Ġbriefing": 17719, "sers": 17720, "cloud": 17721, "ĠMik": 17722, "usp": 17723, "Ġprediction": 17724, "azor": 17725, "Ġportable": 17726, "ĠGand": 17727, "Ġpresenting": 17728, "Ġ1080": 17729, "»": 17730, "ushi": 17731, "ĠSpark": 17732, "thereum": 17733, "Ġjustification": 17734, "ĠNy": 17735, "Ġcontractors": 17736, "mingham": 17737, "ĠStyle": 17738, "åħ": 17739, "ĠChronicles": 17740, "ĠPicture": 17741, "Ġproving": 17742, "Ġwives": 17743, "sett": 17744, "Ġmolecules": 17745, "ĠFairy": 17746, "Ġconsisting": 17747, "Ġpier": 17748, "alone": 17749, "inition": 17750, "Ġnucle": 17751, "json": 17752, "Ġgotta": 17753, "Ġmobil": 17754, "Ġverbal": 17755, "arium": 17756, "Ġmonument": 17757, "ucked": 17758, "Ġ256": 17759, "Tech": 17760, "minecraft": 17761, "ĠTrack": 17762, "Ġtile": 17763, "Ġcompatibility": 17764, "asis": 17765, "Ġsadd": 17766, "Ġinstructed": 17767, "ĠMueller": 17768, "Ġlethal": 17769, "Ġhormone": 17770, "Ġorche": 17771, "else": 17772, "Ġskelet": 17773, "Ġentertaining": 17774, "Ġminimize": 17775, "again": 17776, "Ġundergo": 17777, "Ġconstraints": 17778, "Ġcigarette": 17779, "ĠIslamist": 17780, "Ġtravels": 17781, "ĠPanthers": 17782, "lings": 17783, "Care": 17784, "Ġlawsuits": 17785, "uras": 17786, "Ġcryst": 17787, "Ġlowered": 17788, "Ġaerial": 17789, "Ġcombinations": 17790, "Ġhaun": 17791, "Ġcha": 17792, "Ġvine": 17793, "Ġquantities": 17794, "Ġlinking": 17795, "bank": 17796, "Ġsoy": 17797, "Bill": 17798, "ĠAngela": 17799, "Ġrecipient": 17800, "ĠProtest": 17801, "Ġsocket": 17802, "Ġsolidarity": 17803, "ĠâĨ": 17804, "mill": 17805, "Ġvaries": 17806, "ĠPakistani": 17807, "Dragon": 17808, "Ġune": 17809, "Ġhorizon": 17810, "³³³³³³³³": 17811, "Ġprovinces": 17812, "Ġfrankly": 17813, "Ġenacted": 17814, "notes": 17815, "['": 17816, "Ġ192": 17817, "ocracy": 17818, "Ġendorsement": 17819, "Ġovertime": 17820, "True": 17821, "Lab": 17822, "licted": 17823, "ĠDNC": 17824, "Ġbeats": 17825, "ĠJamie": 17826, "152": 17827, "ĠINT": 17828, "Contact": 17829, "Ġaccounted": 17830, "hash": 17831, "ĠPackers": 17832, "pires": 17833, "Ġlesbian": 17834, "Ġamendments": 17835, "Ġhopeful": 17836, "ĠFinland": 17837, "Ġspotlight": 17838, "Ġconfigured": 17839, "Ġtroubled": 17840, "Ġgaze": 17841, "ĠCalgary": 17842, "Ġreliability": 17843, "Ġinsurg": 17844, "swer": 17845, "buy": 17846, "ĠSkin": 17847, "Ġpixels": 17848, "Ġhandgun": 17849, "Ġparas": 17850, "Ġcategor": 17851, "ĠEL": 17852, "ĠRex": 17853, "Indeed": 17854, "Ġkinda": 17855, "Ġconjunction": 17856, "ĠBryan": 17857, "ĠManufact": 17858, "yang": 17859, "Plus": 17860, "SQL": 17861, "ishment": 17862, "Ġdominate": 17863, "Ġnail": 17864, "Ġoath": 17865, "Ġerupt": 17866, "ĠFine": 17867, "itbart": 17868, "ĠChip": 17869, "ĠAbd": 17870, "ĠNam": 17871, "Ġbuyer": 17872, "Ġdissent": 17873, "Leaks": 17874, "Contin": 17875, "Ġrider": 17876, "ĠSomeone": 17877, "Ġillusion": 17878, "cin": 17879, "ĠBoeing": 17880, "Ġinadequ": 17881, "ovation": 17882, "iants": 17883, "Ġrebuild": 17884, "450": 17885, "ĠDestiny": 17886, "SW": 17887, "ĠTill": 17888, "Hit": 17889, "iaz": 17890, "ĠBangl": 17891, "achers": 17892, "ĠReform": 17893, "Ġsegments": 17894, "Ġsystematic": 17895, "dc": 17896, "ĠConservatives": 17897, "Ġportal": 17898, "hor": 17899, "ĠDragonbound": 17900, "Ġdragged": 17901, "omo": 17902, "Ġthee": 17903, "advert": 17904, "ĠReports": 17905, "ĠEt": 17906, "Ġbarrels": 17907, "August": 17908, "Ġcomparisons": 17909, "Ġhex": 17910, "Ġanthrop": 17911, "\"[": 17912, "borough": 17913, "abi": 17914, "Ġpictured": 17915, "playing": 17916, "ĠAddress": 17917, "ĠMirror": 17918, "Smith": 17919, "Ġtires": 17920, "ĠNPR": 17921, "AAAA": 17922, "Ġclassification": 17923, "ĠThan": 17924, "ĠHarm": 17925, "ĠRA": 17926, "Ġrejection": 17927, "mination": 17928, "Ġranged": 17929, "ĠFalls": 17930, "DI": 17931, "Host": 17932, "ãĤ´": 17933, "ĠExample": 17934, "listed": 17935, "thirds": 17936, "Ġsafegu": 17937, "brand": 17938, "Ġprobable": 17939, "Canada": 17940, "ITION": 17941, "ĠQaeda": 17942, "Ġchick": 17943, "Ġimports": 17944, "hit": 17945, "loc": 17946, "WW": 17947, "Ġblew": 17948, "Ġanytime": 17949, "Ġwholes": 17950, "iked": 17951, "Ġcalculation": 17952, "create": 17953, "ĠOri": 17954, "Ġupgraded": 17955, "Ġappar": 17956, "utory": 17957, "ĠMol": 17958, "Brit": 17959, "ĠJong": 17960, "INAL": 17961, "ĠStarting": 17962, "Ġdice": 17963, "urtle": 17964, "Ġrelying": 17965, "closure": 17966, "Ġprofitable": 17967, "Ġslaughter": 17968, "ĠManual": 17969, "caster": 17970, "Ġ\"$": 17971, "Ġfeather": 17972, "ĠSimply": 17973, "ieves": 17974, "Ġdeterior": 17975, "ĠPCI": 17976, "Ġstamp": 17977, "Ġflaws": 17978, "Ġshade": 17979, "hammer": 17980, "Ġpassport": 17981, "Ġconting": 17982, "amel": 17983, "Ġobservers": 17984, "Ġneglect": 17985, "ĠRB": 17986, "ĠBrotherhood": 17987, "Ġskeptical": 17988, "family": 17989, "usk": 17990, "Ġemotionally": 17991, "âĻ": 17992, "ĠBeta": 17993, "asonable": 17994, "idity": 17995, "ĠMul": 17996, "Ġkicking": 17997, "ĠCarm": 17998, "ollah": 17999, "VERTIS": 18000, "ĠAthen": 18001, "Ġladder": 18002, "ĠBullet": 18003, "å£": 18004, "0001": 18005, "ĠWildlife": 18006, "ĠMask": 18007, "ĠNan": 18008, "Rev": 18009, "Ġunacceptable": 18010, "legal": 18011, "Ġcrowded": 18012, "agi": 18013, "ĠCox": 18014, "je": 18015, "Ġmorality": 18016, "Ġfuels": 18017, "Ġcables": 18018, "Ġmankind": 18019, "ĠCaribbean": 18020, "Ġanchor": 18021, "Ġbyte": 18022, "ĠOften": 18023, "ĠOz": 18024, "Ġcrafted": 18025, "Ġhistorian": 18026, "ĠWu": 18027, "Ġtowers": 18028, "ĠCitizens": 18029, "Ġhelm": 18030, "Ġcredentials": 18031, "Ġsingular": 18032, "ĠJesse": 18033, "Ġtackles": 18034, "Ġcontempt": 18035, "Ġafore": 18036, "ĠShadows": 18037, "Ġnil": 18038, "Ġurgent": 18039, "apple": 18040, "blood": 18041, "Ġvon": 18042, "Ġoffline": 18043, "Ġbreathe": 18044, "Ġjumps": 18045, "Ġirrelevant": 18046, "oxic": 18047, "omal": 18048, "important": 18049, "Jim": 18050, "Ġgloves": 18051, "arming": 18052, "depth": 18053, "Ġtalents": 18054, "ookie": 18055, "ĠSB": 18056, "Ġpalm": 18057, "uffs": 18058, "esta": 18059, "IGH": 18060, "Ġcanon": 18061, "ĠVerizon": 18062, "ĠPle": 18063, "Ġcoupled": 18064, "velt": 18065, "Ġfundraising": 18066, "ĠGetting": 18067, "ĠDLC": 18068, "Ġmathematical": 18069, "ĠHS": 18070, "ĠCardinals": 18071, "telling": 18072, "Ġsponsors": 18073, "ĠÏ": 18074, "ĠBulls": 18075, "option": 18076, "Ġpropose": 18077, "Ġmemorable": 18078, "Ġembraced": 18079, "Ġdeclining": 18080, "Health": 18081, "eda": 18082, "Ġ};": 18083, "Ġspam": 18084, "mile": 18085, "Ġpitcher": 18086, "ĠEight": 18087, "Ġcaring": 18088, "utic": 18089, "role": 18090, "Ġairline": 18091, "ernandez": 18092, "ĠAthlet": 18093, "Ġcertification": 18094, "uxe": 18095, "riger": 18096, "Ġempir": 18097, "Ġsensation": 18098, "Ġdism": 18099, "Ġbolt": 18100, "Ġevolve": 18101, "House": 18102, "Ġconsultation": 18103, "ĠDuty": 18104, "Ġtouches": 18105, "ĠNathan": 18106, "Ġfaint": 18107, "had": 18108, "\"(": 18109, "ĠConsumer": 18110, "ĠExtreme": 18111, "Ġ127": 18112, "ĠHerm": 18113, "ĠSacrament": 18114, "izoph": 18115, "Ġanxious": 18116, "ulously": 18117, "Ġsocially": 18118, "ĠUTC": 18119, "Ġsolving": 18120, "ĠLetter": 18121, "History": 18122, "educ": 18123, "Price": 18124, "));": 18125, "Ġreload": 18126, "amic": 18127, "Ġpork": 18128, "Ġdiscourse": 18129, "Ġtournaments": 18130, "airo": 18131, "ĠKur": 18132, "ĠCosta": 18133, "Ġviolating": 18134, "Ġinterfere": 18135, "Ġrecreational": 18136, "uffle": 18137, "Ġspeeches": 18138, "Ġneeding": 18139, "Ġremembers": 18140, "Ġcredited": 18141, "nia": 18142, "focused": 18143, "amera": 18144, "Ġbru": 18145, "umbs": 18146, "ĠCuban": 18147, "Ġpreceding": 18148, "Ġnonsense": 18149, "acial": 18150, "Ġsmartphones": 18151, "ĠStories": 18152, "Sports": 18153, "ĠEmergency": 18154, "ouncing": 18155, "efined": 18156, "Ġber": 18157, "Ġconsulting": 18158, "Ġmasters": 18159, "heastern": 18160, ".\"[": 18161, "ĠRunning": 18162, "Ġsuscept": 18163, "ĠFeng": 18164, "America": 18165, "prises": 18166, "stitial": 18167, "ĠWeekly": 18168, "ĠGreater": 18169, "modules": 18170, "ifter": 18171, "Graphics": 18172, "uler": 18173, "Ġwholly": 18174, "Ġsuppress": 18175, "Ġconcealed": 18176, "Ġhappily": 18177, "Ġaccepts": 18178, "ĠEnjoy": 18179, "Ġrivers": 18180, "ĠExcept": 18181, "225": 18182, "ĠNHS": 18183, "ĠMcConnell": 18184, "Ġpussy": 18185, "ferred": 18186, "utable": 18187, "Ġattain": 18188, "Ġ>=": 18189, "Ġdeposits": 18190, "rophic": 18191, "Ġnotorious": 18192, "ĠShaw": 18193, "ilitation": 18194, "Ġepidemic": 18195, "allic": 18196, "Ġsmallest": 18197, "ovich": 18198, "Ġaccessories": 18199, "perties": 18200, "Ġsurplus": 18201, "ĠMech": 18202, "Ġambig": 18203, "ĠImmigration": 18204, "Ġchim": 18205, "eval": 18206, "Ġpracticing": 18207, "ĠMystery": 18208, "Ġdomains": 18209, "ĠSilicon": 18210, "apps": 18211, "Ġkilometers": 18212, "ea": 18213, "ĠSmash": 18214, "Ġwarranty": 18215, "Ġnost": 18216, "sil": 18217, "rev": 18218, "Jon": 18219, "ĠDublin": 18220, "Ġtastes": 18221, "Ġbout": 18222, "great": 18223, "error": 18224, "Ġswitches": 18225, "ĠBapt": 18226, "DO": 18227, "oki": 18228, "Ġsourced": 18229, "produ": 18230, "Ġattachment": 18231, "ĠIssue": 18232, "ĠQuestion": 18233, "Join": 18234, "Ġfitted": 18235, "Ġunlawful": 18236, "^^": 18237, "erek": 18238, "Ġauthentication": 18239, "Ġstole": 18240, "Ġaccountability": 18241, "label": 18242, "Search": 18243, "Ġalbeit": 18244, "atican": 18245, "funded": 18246, "ĠAdding": 18247, "ĠIQ": 18248, "Ġsubmar": 18249, "lit": 18250, "aque": 18251, "ĠLearning": 18252, "Ġinteger": 18253, "Master": 18254, "ĠChrom": 18255, "Ġpremier": 18256, "Op": 18257, "ĠLiu": 18258, "Ġblessed": 18259, "ĠGlobe": 18260, "ĠResponse": 18261, "Ġlegitim": 18262, "ĠMerkel": 18263, "Ġdisposal": 18264, "´": 18265, "Ġgauge": 18266, "peat": 18267, "Ġinduced": 18268, "Ġquestionable": 18269, "arthy": 18270, "ĠVit": 18271, "ĠFeed": 18272, "Until": 18273, "Ut": 18274, "worthy": 18275, "RY": 18276, "ĠHerald": 18277, "ĠHammer": 18278, "Ġmedal": 18279, "ĠRivers": 18280, "ĠHack": 18281, "Ġclarify": 18282, "Ġtracked": 18283, "Ġautonomous": 18284, "Ġtenant": 18285, "ĠQatar": 18286, "erie": 18287, "Ġgrim": 18288, "ĠMonitor": 18289, "Ġresistant": 18290, "ĠSpec": 18291, "ĠWells": 18292, "NAS": 18293, "148": 18294, "Ġminers": 18295, "iotics": 18296, "Ġmisses": 18297, "116": 18298, "gian": 18299, "git": 18300, "ĠEyes": 18301, "pres": 18302, "Ġgraduated": 18303, "Ġangel": 18304, "Ġsynchron": 18305, "Ġefficiently": 18306, "Ġtransmitted": 18307, "Harry": 18308, "Ġglobally": 18309, "ENCE": 18310, "ĠMontana": 18311, "raged": 18312, "ĠPrevention": 18313, "Ġpiss": 18314, "ĠLl": 18315, "Ġshelf": 18316, "ĠBJP": 18317, "ĠTestament": 18318, "ĠLate": 18319, "iker": 18320, "ĠHapp": 18321, "ĠJulian": 18322, "hall": 18323, "Ġspont": 18324, "Ġshutdown": 18325, "Ġinconsistent": 18326, "Ġsubscribers": 18327, "Ġskeleton": 18328, "ĠNebraska": 18329, "Ġinspire": 18330, "ĠVoid": 18331, "Feed": 18332, "Ġangles": 18333, "ĠSprings": 18334, "Ġbenchmark": 18335, "Ġvaccines": 18336, "izophren": 18337, "sexual": 18338, "uffed": 18339, "Ġshine": 18340, "ĠKath": 18341, "Ġgesture": 18342, "inea": 18343, "Ġrip": 18344, "Ġoppression": 18345, "Ġconscience": 18346, "bt": 18347, "ĠLum": 18348, "Ġincidence": 18349, "ĠFa": 18350, "wr": 18351, "Ġmineral": 18352, "ĠSpurs": 18353, "alky": 18354, "Ġthunder": 18355, "Ġopio": 18356, "Being": 18357, "ĠPalm": 18358, "Ġwasted": 18359, "Ġlb": 18360, "iaries": 18361, "ĠInitiative": 18362, "Ġcurric": 18363, "Ġmarker": 18364, "ĠMcL": 18365, "Ġextensions": 18366, "ĠPv": 18367, "ĠArms": 18368, "Ġofferings": 18369, "Ġdefenses": 18370, "Ġvendor": 18371, "Ġcontradict": 18372, "ĠColin": 18373, "Ġreddit": 18374, "Ġperipher": 18375, "122": 18376, "Ġsins": 18377, "Edit": 18378, "ICT": 18379, "Soft": 18380, "ĠShah": 18381, "Ġadministrator": 18382, "ĠTrip": 18383, "Ġpornography": 18384, "Ġtuition": 18385, "inence": 18386, "ĠProgress": 18387, "Ġcatalog": 18388, "Ġsuite": 18389, "Ġhike": 18390, "Ġreproductive": 18391, "engine": 18392, "Ġdrought": 18393, "ĠNoah": 18394, "Ġ230": 18395, "Ġdude": 18396, "Ġrelaxed": 18397, "Ġpartition": 18398, "Ġparticipant": 18399, "Ġtelesc": 18400, "Ġfeas": 18401, "ĠFF": 18402, "owner": 18403, "Ġsweeping": 18404, "Ġlenses": 18405, "Ġmatchup": 18406, "ĠRepl": 18407, "ournals": 18408, "Ġcredible": 18409, "Ġgrandmother": 18410, "Ġthermal": 18411, "Ġsubscribing": 18412, "Ġidentities": 18413, "colm": 18414, "UCT": 18415, "Ġreluctant": 18416, "users": 18417, "ĠCort": 18418, "Ġassisted": 18419, "OSS": 18420, "ATIONS": 18421, "ISH": 18422, "Ġpharmaceutical": 18423, "icable": 18424, "adian": 18425, "ĠSonic": 18426, "ĠFury": 18427, "ĠMong": 18428, "AH": 18429, "ĠPsychology": 18430, "Ġphosph": 18431, "Ġtreats": 18432, "ŃĶ": 18433, "Ġsteadily": 18434, "ĠHello": 18435, "Ġrelates": 18436, "Ġclue": 18437, "Expl": 18438, "auth": 18439, "Ġrevision": 18440, "Ġeld": 18441, "osion": 18442, "Ġbron": 18443, "144": 18444, "rikes": 18445, "Ġmines": 18446, "Ġblanket": 18447, "ĠFail": 18448, "eled": 18449, "ĠImagine": 18450, "ĠPlanned": 18451, "aic": 18452, "Request": 18453, "Mad": 18454, "ĠHorse": 18455, "ĠEagle": 18456, "Ġcapac": 18457, "157": 18458, "Ġling": 18459, "ĠNice": 18460, "ĠParenthood": 18461, "minster": 18462, "ogs": 18463, "ensitive": 18464, "Nothing": 18465, "Ġcarn": 18466, "Fin": 18467, "ĠPE": 18468, "Ġrifles": 18469, "ĠLP": 18470, "Sand": 18471, "ĠguiActive": 18472, "Ġtourist": 18473, "CNN": 18474, "Ġunveiled": 18475, "Ġpredecessor": 18476, "}{": 18477, "uber": 18478, "Ġoffshore": 18479, "Ġoptical": 18480, "ĠRot": 18481, "ĠPearl": 18482, "eton": 18483, "Ġstared": 18484, "Ġfarther": 18485, "atility": 18486, "contin": 18487, "ĠGy": 18488, "ĠFoster": 18489, "ĠCoc": 18490, "rients": 18491, "Ġdesigning": 18492, "ĠEconomy": 18493, "ONG": 18494, "Women": 18495, "ĠNancy": 18496, "erver": 18497, "Ġmascul": 18498, "Ġcasualties": 18499, "Ġ225": 18500, "ĠSullivan": 18501, "ĠChoice": 18502, "Ġaster": 18503, "ws": 18504, "Ġhotels": 18505, "Ġconsiderations": 18506, "Ġcouch": 18507, "ĠStrip": 18508, "ĠGn": 18509, "Ġmanipulate": 18510, "lied": 18511, "Ġsynthetic": 18512, "Ġassaulted": 18513, "Ġoffenses": 18514, "ĠDrake": 18515, "Ġimpe": 18516, "October": 18517, "ĠHeritage": 18518, "hl": 18519, "ĠBlair": 18520, "Unlike": 18521, "Ġgrief": 18522, "Ġ450": 18523, "Ġopted": 18524, "Ġresignation": 18525, "ilo": 18526, "Ġverse": 18527, "ĠTomb": 18528, "Ġupt": 18529, "Ġaired": 18530, "ĠHook": 18531, "ĠMLB": 18532, "Ġassumes": 18533, "outed": 18534, "ĠVers": 18535, "Ġinferior": 18536, "Ġbundle": 18537, "ĠDNS": 18538, "ographer": 18539, "Ġmultip": 18540, "ĠSouls": 18541, "Ġillustrated": 18542, "Ġtactic": 18543, "Ġdressing": 18544, "Ġduo": 18545, "Conf": 18546, "Ġrelent": 18547, "Ġcant": 18548, "Ġscarce": 18549, "Ġcandy": 18550, "ĠCF": 18551, "Ġaffiliated": 18552, "Ġsprint": 18553, "ylan": 18554, "ĠGarcia": 18555, "Ġjunk": 18556, "Print": 18557, "exec": 18558, "Crit": 18559, "Ġportrait": 18560, "iries": 18561, "ĠOFF": 18562, "Ġdisputes": 18563, "WR": 18564, "Love": 18565, "ãģĦ": 18566, "ĠReyn": 18567, "Ġhipp": 18568, "opath": 18569, "Ġfloors": 18570, "ĠFeel": 18571, "Ġworries": 18572, "Ġsettlements": 18573, "ĠPos": 18574, "Ġmosque": 18575, "Ġfinals": 18576, "Ġcrushed": 18577, "ĠProbably": 18578, "ĠBot": 18579, "ĠMans": 18580, "ĠPeriod": 18581, "Ġsovereignty": 18582, "Ġseller": 18583, "Ġapost": 18584, "Ġamateur": 18585, "Ġdorm": 18586, "Ġconsuming": 18587, "Ġarmour": 18588, "ĠRoose": 18589, "Ġintensive": 18590, "Ġeliminating": 18591, "ĠSunni": 18592, "ĠAleppo": 18593, "jin": 18594, "Ġadvise": 18595, "pal": 18596, "ĠHalo": 18597, "Ġdescent": 18598, "Ġsimpler": 18599, "Ġbooth": 18600, "STR": 18601, "Later": 18602, "ĠCave": 18603, "===": 18604, "Ġmol": 18605, "Ġfist": 18606, "Ġshotgun": 18607, "supp": 18608, "Ġrobbery": 18609, "Effect": 18610, "Ġobscure": 18611, "ĠProfessional": 18612, "Ġembassy": 18613, "Ġmilitant": 18614, "Ġincarcer": 18615, "Ġgenerates": 18616, "Ġlaunches": 18617, "Ġadministrators": 18618, "Ġshaft": 18619, "Ġcircular": 18620, "Ġfreshman": 18621, "ĠWes": 18622, "ĠJoel": 18623, "ĠDrew": 18624, "ĠDuncan": 18625, "ĠApparently": 18626, "sight": 18627, "ĠInternal": 18628, "ĠIndividual": 18629, "ĠFE": 18630, "Ġbore": 18631, "ĠMt": 18632, "Ġbroadly": 18633, "ĠOptions": 18634, "ountain": 18635, "ipes": 18636, "ĠVideos": 18637, "204": 18638, "Ġhills": 18639, "Ġsimulation": 18640, "Ġdisappointment": 18641, "itan": 18642, "ĠLaboratory": 18643, "Ġupward": 18644, "Ġboundary": 18645, "Ġdarker": 18646, "hart": 18647, "Ġdominance": 18648, "Cong": 18649, "ĠOracle": 18650, "ĠLords": 18651, "Ġscholarship": 18652, "ĠVincent": 18653, "ede": 18654, "ĠRah": 18655, "Ġencourages": 18656, "rov": 18657, "Ġquo": 18658, "Ġpremise": 18659, "ĠCrisis": 18660, "ĠHolocaust": 18661, "Ġrhythm": 18662, "Ġmetric": 18663, "club": 18664, "Ġtransported": 18665, "Ġnod": 18666, "ĠPist": 18667, "Ġancestors": 18668, "ĠFreder": 18669, "thumbnails": 18670, "ĠCE": 18671, "OND": 18672, "Phil": 18673, "venge": 18674, "ĠProducts": 18675, "castle": 18676, "Ġqualifying": 18677, "ĠKaren": 18678, "VERTISEMENT": 18679, "Ġmighty": 18680, "Ġexplanations": 18681, "Ġfixing": 18682, "Di": 18683, "Ġdeclaring": 18684, "Ġanonymity": 18685, "Ġjuven": 18686, "ĠNord": 18687, "ĠDoom": 18688, "ĠActually": 18689, "Ok": 18690, "phis": 18691, "ĠDesert": 18692, "Ġ116": 18693, "IK": 18694, "ĠFM": 18695, "Ġincomes": 18696, "VEL": 18697, "okers": 18698, "Ġpecul": 18699, "Ġlightweight": 18700, "gue": 18701, "Ġaccent": 18702, "Ġincrement": 18703, "ĠChan": 18704, "Ġcomplaining": 18705, "ĠBaghd": 18706, "Ġmidfielder": 18707, "Ġoverhaul": 18708, "Process": 18709, "ĠHollow": 18710, "ĠTitans": 18711, "Small": 18712, "manuel": 18713, "ĠUnity": 18714, "ĠEvents": 18715, "Sty": 18716, "Ġdisproportion": 18717, "nesty": 18718, "enes": 18719, "ĠCod": 18720, "Ġdemonstrations": 18721, "ĠCrimson": 18722, "ĠOH": 18723, "Ġenrolled": 18724, "Ġcel": 18725, "ĠBrett": 18726, "Ġaide": 18727, "Ġheels": 18728, "Ġbroadband": 18729, "Ġmarking": 18730, "Ġwizard": 18731, "ĠNJ": 18732, "ĠChiefs": 18733, "Ġingredient": 18734, "Ġdug": 18735, "ĠShut": 18736, "urchase": 18737, "endor": 18738, "Ġfarmer": 18739, "ĠGoldman": 18740, "129": 18741, "155": 18742, "Order": 18743, "Ġlion": 18744, "iably": 18745, "Ġstain": 18746, "array": 18747, "ilitary": 18748, "ĠFAQ": 18749, "Ġexploded": 18750, "ĠMcCarthy": 18751, "ĠTweet": 18752, "ĠGreens": 18753, "eking": 18754, "ln": 18755, "ensen": 18756, "Ġmotorcycle": 18757, "Ġparticle": 18758, "Ġcholesterol": 18759, "Bron": 18760, "Ġstair": 18761, "Ġoxid": 18762, "Ġdesirable": 18763, "ibles": 18764, "Ġtheor": 18765, "forcing": 18766, "Ġpromotional": 18767, "ovo": 18768, "boot": 18769, "ĠBonus": 18770, "rawling": 18771, "Ġshortage": 18772, "ĠPsy": 18773, "Ġrecruited": 18774, "Ġinfants": 18775, "Ġtestosterone": 18776, "Ġdeduct": 18777, "Ġdistinctive": 18778, "Ġfirmware": 18779, "built": 18780, "145": 18781, "Ġexplored": 18782, "Ġfactions": 18783, "Ġvide": 18784, "Ġtattoo": 18785, "Ġfinancially": 18786, "Ġfatigue": 18787, "Ġproceeding": 18788, "constitutional": 18789, "Ġmiser": 18790, "Ġchairs": 18791, "gging": 18792, "ipple": 18793, "Ġdent": 18794, "Ġdisreg": 18795, "çĶ": 18796, "stant": 18797, "llo": 18798, "bps": 18799, "akening": 18800, "Ġabnormal": 18801, "ĠERA": 18802, "士": 18803, "ĠHBO": 18804, "ĠMAR": 18805, "Ġconcess": 18806, "Ġservant": 18807, "Ġaspir": 18808, "lav": 18809, "ĠPanel": 18810, "amo": 18811, "Ġprecip": 18812, "Ġrecordings": 18813, "Ġproceeded": 18814, "Ġcolony": 18815, "ĠTang": 18816, "ablo": 18817, "Ġstripped": 18818, "Left": 18819, "too": 18820, "Ġpotatoes": 18821, "Ġfinest": 18822, "%).": 18823, "Ġcrap": 18824, "ĠZach": 18825, "abases": 18826, "ĠGoth": 18827, "Ġbillionaire": 18828, "wolf": 18829, "Ġsanction": 18830, "SK": 18831, "Ġlogged": 18832, "Po": 18833, "eyed": 18834, "unal": 18835, "Ġcricket": 18836, "Ġarmies": 18837, "Ġuncovered": 18838, "Cloud": 18839, "ón": 18840, "Ġrebounds": 18841, "Ġmes": 18842, "Oper": 18843, "Pac": 18844, "Ġnationally": 18845, "Ġinserted": 18846, "pict": 18847, "Ġgovernance": 18848, "и": 18849, "Ġprivileges": 18850, "GET": 18851, "Ġfavorites": 18852, "imity": 18853, "Ġlover": 18854, "them": 18855, "empl": 18856, "Ġgorgeous": 18857, "Ann": 18858, "Ġslipped": 18859, "Ġveto": 18860, "Bob": 18861, "Ġslim": 18862, "ucc": 18863, "ĠFame": 18864, "uddenly": 18865, "Ġdenies": 18866, "ĠMaur": 18867, "Ġdistances": 18868, "Ġwanna": 18869, "tar": 18870, "ĠSER": 18871, "ĠâĪ": 18872, "Ġlemon": 18873, "athetic": 18874, "Ġliteral": 18875, "Ġdistinguished": 18876, "Ġanswering": 18877, "GI": 18878, "Ġreligions": 18879, "ĠPhilos": 18880, "ĠLay": 18881, "Ġcompos": 18882, "irements": 18883, "ĠKos": 18884, "inez": 18885, "rolling": 18886, "Ġyoungest": 18887, "andise": 18888, "ĠBorn": 18889, "Ġaltar": 18890, "amina": 18891, "ĠBoot": 18892, "voc": 18893, "Ġdigging": 18894, "Ġpressures": 18895, "Ġlen": 18896, "264": 18897, "Ġassassination": 18898, "ĠBirmingham": 18899, "ĠMyth": 18900, "Ġsovereign": 18901, "ĠArtist": 18902, "ĠPhotograph": 18903, "Ġdepicted": 18904, "Ġdispens": 18905, "orthy": 18906, "Ġambul": 18907, "integ": 18908, "ĠCele": 18909, "ĠTibet": 18910, "Ġhierarchy": 18911, "Ġcu": 18912, "Ġpreseason": 18913, "ĠPeterson": 18914, "Ġcolours": 18915, "Ġworrying": 18916, "Ġbackers": 18917, "ĠPalmer": 18918, "Ġμ": 18919, "Ġcontributor": 18920, "Ġhearings": 18921, "Ġurine": 18922, "ĠÙ": 18923, "ourgeois": 18924, "Similar": 18925, "ĠZimmer": 18926, "something": 18927, "ĠUSC": 18928, "Ġstrengths": 18929, "ĠFI": 18930, "Ġlogging": 18931, "Asked": 18932, "ĠThai": 18933, "inqu": 18934, "ĠWalt": 18935, "Ġcrews": 18936, "itism": 18937, "301": 18938, "Ġsharply": 18939, "umed": 18940, "Ġredirect": 18941, "rators": 18942, "Inf": 18943, "ĠWeapons": 18944, "Ġteasp": 18945, "1999": 18946, "Live": 18947, "ĠEspecially": 18948, "ĠSter": 18949, "ĠVeterans": 18950, "Ġintro": 18951, "otherapy": 18952, "Ġmalware": 18953, "Ġbreeding": 18954, "Ġmolecular": 18955, "ĠRoute": 18956, "ĠComment": 18957, "ochem": 18958, "Ġain": 18959, "Season": 18960, "Ġlinebacker": 18961, "Ä«": 18962, "ĠEconomics": 18963, "esar": 18964, "ĠLives": 18965, "ĠEmma": 18966, "Ġkin": 18967, "ĠTerrit": 18968, "Ġplanted": 18969, "oton": 18970, "ĠButter": 18971, "ĠSpons": 18972, "PER": 18973, "Ġdungeon": 18974, "Ġsymbolic": 18975, "Ġfilmed": 18976, "Ġdiets": 18977, "Ġconcludes": 18978, "Ġcertainty": 18979, "ĠFormat": 18980, "Ġstrangers": 18981, "format": 18982, "ĠPhase": 18983, "Ġcopied": 18984, "Ġmetres": 18985, "lda": 18986, "ĠUsers": 18987, "Ġdeliberate": 18988, "Ġwashed": 18989, "ĠLance": 18990, "imation": 18991, "Ġimproper": 18992, "ĠGenesis": 18993, "ickr": 18994, "ĠKush": 18995, "Ġrealise": 18996, "Ġembarrassing": 18997, "alking": 18998, "bucks": 18999, "Ġverified": 19000, "Ġoutline": 19001, "years": 19002, "ĠIncome": 19003, "202": 19004, "Ġzombies": 19005, "Final": 19006, "ĠMillenn": 19007, "Ġmodifications": 19008, "ĠVision": 19009, "ĠMoses": 19010, "verb": 19011, "iterranean": 19012, "ĠJet": 19013, "Ġnaval": 19014, "ĠAgg": 19015, "Ġurl": 19016, "Ġvictories": 19017, "Ġnonetheless": 19018, "Ġinjust": 19019, "ĠFact": 19020, "çļ": 19021, "Ġinsufficient": 19022, "review": 19023, "facebook": 19024, "Ġnegotiating": 19025, "Ġguarantees": 19026, "imen": 19027, "utenberg": 19028, "Ġgambling": 19029, "Ġcongr": 19030, "Loading": 19031, "Ġnevertheless": 19032, "Ġpresidents": 19033, "ĠIndustrial": 19034, "Ġ118": 19035, "Ġpoured": 19036, "ĠTory": 19037, "Ġ175": 19038, "Ġ:=": 19039, "Scott": 19040, "angered": 19041, "Tok": 19042, "Ġorganizers": 19043, "Mat": 19044, "ĠGrowth": 19045, "Ġadul": 19046, "Ġensures": 19047, "Ġ117": 19048, "é¾įå": 19049, "Ġmassacre": 19050, "Ġgrades": 19051, "before": 19052, "ADVERTISEMENT": 19053, "ĠSlow": 19054, "ĠMMA": 19055, "âĢĶ\"": 19056, "ĠVatican": 19057, "Qaeda": 19058, "Ġowe": 19059, "6666": 19060, "ĠSorry": 19061, "ĠGrass": 19062, "Ġbackgrounds": 19063, "Ġexhausted": 19064, "Ġclan": 19065, "Ġcompromised": 19066, "ĠElf": 19067, "ĠIsaac": 19068, "enson": 19069, "Invest": 19070, "IFA": 19071, "Ġinterrupted": 19072, "ãĥīãĥ©": 19073, "Ġtwisted": 19074, "ĠDragons": 19075, "Mode": 19076, "ĠKremlin": 19077, "Ġfertil": 19078, "heres": 19079, "phan": 19080, "ĠNode": 19081, "fed": 19082, "ĠOrc": 19083, "Ġunwilling": 19084, "Cent": 19085, "Ġpriorit": 19086, "Ġgraduates": 19087, "Ġsubjective": 19088, "Ġissuing": 19089, "ĠLt": 19090, "Ġviewer": 19091, "Ġwoke": 19092, "Thus": 19093, "brook": 19094, "Ġdepressed": 19095, "Ġbracket": 19096, "ĠGor": 19097, "ĠFighting": 19098, "Ġstriker": 19099, "Report": 19100, "ĠPortugal": 19101, "Ġneo": 19102, "wed": 19103, "199": 19104, "Ġfleeing": 19105, "shadow": 19106, "identified": 19107, "USE": 19108, "Steam": 19109, "Ġstretched": 19110, "Ġrevelations": 19111, "arted": 19112, "ĠDw": 19113, "Ġalignment": 19114, "eston": 19115, "ĠJared": 19116, "Sep": 19117, "Ġblogs": 19118, "update": 19119, "gom": 19120, "risk": 19121, "Ġclash": 19122, "ĠHour": 19123, "Ġruntime": 19124, "Ġunwanted": 19125, "Ġscam": 19126, "Ġrack": 19127, "Ġenlight": 19128, "onest": 19129, "ĠFerr": 19130, "Ġconvictions": 19131, "Ġpiano": 19132, "Ġcirculation": 19133, "ĠWelcome": 19134, "Ġbacklash": 19135, "ĠWade": 19136, "Ġreceivers": 19137, "otive": 19138, "Jeff": 19139, "Ġnetworking": 19140, "ĠPrep": 19141, "ĠExplorer": 19142, "Ġlecture": 19143, "Ġuploaded": 19144, "ĠMeat": 19145, "BLE": 19146, "ĠNazis": 19147, "ĠSynd": 19148, "stud": 19149, "roots": 19150, "rians": 19151, "Ġportrayed": 19152, "Ġ??": 19153, "ĠBuddha": 19154, "sun": 19155, "Robert": 19156, "ĠComplex": 19157, "Ġoversee": 19158, "Ġstealth": 19159, "Title": 19160, "ĠJobs": 19161, "ĠKum": 19162, "Ġappreciation": 19163, "ĠMOD": 19164, "Ġbasics": 19165, "Ġclips": 19166, "Ġnursing": 19167, "Ġproposition": 19168, "Ġrealised": 19169, "ĠNYC": 19170, "Ġallocated": 19171, "rium": 19172, "aran": 19173, "ĠProduction": 19174, "ĠVote": 19175, "Ġsmugg": 19176, "Ġhunter": 19177, "azer": 19178, "ĠChanges": 19179, "Ġfluct": 19180, "yon": 19181, "Array": 19182, "Ġkits": 19183, "Water": 19184, "Ġuncommon": 19185, "Ġresting": 19186, "ells": 19187, "would": 19188, "Ġpursued": 19189, "Ġassertion": 19190, "ometown": 19191, "ĠMosul": 19192, "ĠPlatform": 19193, "iolet": 19194, "Ġshareholders": 19195, "Ġtrails": 19196, "Pay": 19197, "ĠEnforcement": 19198, "types": 19199, "ĠAnonymous": 19200, "Ġsatisfying": 19201, "ilogy": 19202, "Ġ('": 19203, "wave": 19204, "city": 19205, "Steve": 19206, "Ġconfrontation": 19207, "ĠEld": 19208, "Capt": 19209, "ahan": 19210, "htm": 19211, "ĠCtrl": 19212, "ONS": 19213, "230": 19214, "ifa": 19215, "holding": 19216, "Ġdelicate": 19217, "Ġjaw": 19218, "ĠGoing": 19219, "orum": 19220, "Sal": 19221, "Ġdull": 19222, "ĠBeth": 19223, "Ġprisons": 19224, "Ġego": 19225, "ĠElsa": 19226, "avorite": 19227, "ĠGang": 19228, "ĠNuclear": 19229, "Ġspider": 19230, "atsu": 19231, "Ġsampling": 19232, "Ġabsorbed": 19233, "ĠPharm": 19234, "ieth": 19235, "Ġbucket": 19236, "ĠRecomm": 19237, "OF": 19238, "ĠFactory": 19239, "ANCE": 19240, "Ġbacter": 19241, "Has": 19242, "ĠObserv": 19243, "121": 19244, "Ġpremiere": 19245, "Develop": 19246, "Ġcurrencies": 19247, "Cast": 19248, "Ġaccompanying": 19249, "ĠNashville": 19250, "Ġfatty": 19251, "ĠBrend": 19252, "Ġlocks": 19253, "Ġcentered": 19254, "ĠUT": 19255, "aughs": 19256, "orie": 19257, "ĠAffordable": 19258, "vance": 19259, "DL": 19260, "emet": 19261, "Ġthrone": 19262, "ĠBluetooth": 19263, "Ġnaming": 19264, "ifts": 19265, "ADE": 19266, "Ġcorrected": 19267, "Ġpromptly": 19268, "ĠSTR": 19269, "Ġgenome": 19270, "Ġcope": 19271, "Ġvalley": 19272, "Ġrounded": 19273, "ĠKend": 19274, "alion": 19275, "pers": 19276, "Ġtourism": 19277, "Ġstark": 19278, "vl": 19279, "Ġblowing": 19280, "ĠSchedule": 19281, "std": 19282, "Ġunhappy": 19283, "Ġlitigation": 19284, "cedes": 19285, "Ġandroid": 19286, "Ġintegral": 19287, "erers": 19288, "uded": 19289, "tax": 19290, "Ġreiter": 19291, "ĠMotors": 19292, "ociated": 19293, "Ġwonders": 19294, "ĠApost": 19295, "ucking": 19296, "ĠRoosevelt": 19297, "fram": 19298, "Ġyields": 19299, "Ġconstitutes": 19300, "awk": 19301, "Interest": 19302, "Ġinterim": 19303, "Ġbreakthrough": 19304, "ĠCher": 19305, "Ġprosec": 19306, "ĠDj": 19307, "ĠMT": 19308, "Resp": 19309, "ĠPT": 19310, "Ġsperm": 19311, "edit": 19312, "BT": 19313, "Linux": 19314, "country": 19315, "league": 19316, "Ġdick": 19317, "Ġoct": 19318, "Ġinserting": 19319, "Ġscra": 19320, "ĠBrewing": 19321, "Ġ1966": 19322, "Ġrunners": 19323, "Ġplun": 19324, "idy": 19325, "ĠDian": 19326, "Ġdysfunction": 19327, "Ġexclusion": 19328, "Ġdisgr": 19329, "Ġincorporate": 19330, "Ġreconc": 19331, "Ġnominated": 19332, "ĠArcher": 19333, "draw": 19334, "achelor": 19335, "Ġwritings": 19336, "Ġshallow": 19337, "Ġhast": 19338, "ĠBMW": 19339, "ĠRS": 19340, "Ġthigh": 19341, "Ġ1963": 19342, "Ġlamb": 19343, "Ġfavored": 19344, "agle": 19345, "Ġcooler": 19346, "ĠHours": 19347, "ĠGU": 19348, "ĠOrigin": 19349, "Ġglimpse": 19350, "--------------------": 19351, "Lim": 19352, "Ġcheek": 19353, "Ġjealous": 19354, "-'": 19355, "Ġharness": 19356, "ĠPoison": 19357, "Ġdisabilities": 19358, "neapolis": 19359, "Ġoutlook": 19360, "Ġnotify": 19361, "ĠIndianapolis": 19362, "Ġabrupt": 19363, "nsic": 19364, "Ġencrypted": 19365, "Ġforfe": 19366, "reath": 19367, "Ġrabb": 19368, "Ġfoundations": 19369, "Ġcompliment": 19370, "ĠInterview": 19371, "ĠSwe": 19372, "Ġadolesc": 19373, "Ġmonitors": 19374, "ĠSacramento": 19375, "Ġtimely": 19376, "Ġcontempl": 19377, "Ġpositioned": 19378, "Ġposters": 19379, "phies": 19380, "iovascular": 19381, "void": 19382, "ĠFifth": 19383, "Ġinvestigative": 19384, "OUN": 19385, "Ġintegrate": 19386, "ĠINC": 19387, "isha": 19388, "iblings": 19389, "ĠRequest": 19390, "ĠRodriguez": 19391, "Ġslides": 19392, "ĠDX": 19393, "Ġfeminism": 19394, "Ġdatas": 19395, "Ġbend": 19396, "irus": 19397, "ĠNigeria": 19398, "Fox": 19399, "Change": 19400, "Ġairplane": 19401, "ĠLaden": 19402, "Ġpublicity": 19403, "ixty": 19404, "Ġcommitments": 19405, "Ġaggregate": 19406, "Ġdisplaying": 19407, "ĠArrow": 19408, "Ġ122": 19409, "Ġrespects": 19410, "android": 19411, "six": 19412, "ĠSha": 19413, "Ġrestoration": 19414, ")\\": 19415, "WS": 19416, "oys": 19417, "Ġillustrate": 19418, "without": 19419, "126": 19420, "ĠâĶĤ": 19421, "Ġpickup": 19422, "nels": 19423, "Ġ....": 19424, "food": 19425, "ĠFen": 19426, ")?": 19427, "Ġphenomena": 19428, "Ġcompanions": 19429, "ĠWrite": 19430, "Ġspill": 19431, "Ġbridges": 19432, "ĠUpdated": 19433, "ĠFo": 19434, "Ġinsects": 19435, "ASHINGTON": 19436, "Ġscare": 19437, "iltr": 19438, "ĠZhang": 19439, "Ġseverity": 19440, "Ġindul": 19441, "149": 19442, "ĠCoffee": 19443, "Ġnorms": 19444, "Ġpulse": 19445, "ĠFT": 19446, "Ġhorrific": 19447, "ĠDestroy": 19448, "ĠJSON": 19449, "Ġolive": 19450, "Ġdiscusses": 19451, "Rest": 19452, "Elect": 19453, "ĠWinn": 19454, "ĠSurviv": 19455, "ĠHait": 19456, "Sure": 19457, "oped": 19458, "Ġrooted": 19459, "ĠSke": 19460, "ĠBronze": 19461, "Ġlol": 19462, "Default": 19463, "Ġcommodity": 19464, "redited": 19465, "Ġlibertarian": 19466, "Ġforbidden": 19467, "Ġgran": 19468, "à¨": 19469, "Ġlag": 19470, "enz": 19471, "drive": 19472, "Ġmathematics": 19473, "Ġwires": 19474, "Ġcritically": 19475, "Ġcarbohyd": 19476, "ĠChancellor": 19477, "ĠEddie": 19478, "Ġbanning": 19479, "ĠFri": 19480, "Ġcomplications": 19481, "etric": 19482, "ĠBangladesh": 19483, "Ġbandwidth": 19484, "Stop": 19485, "ĠOriginally": 19486, "Ġhalfway": 19487, "ynasty": 19488, "shine": 19489, "Ġtales": 19490, "rities": 19491, "avier": 19492, "Ġspinning": 19493, "ĠWHO": 19494, "Ġneighbourhood": 19495, "bach": 19496, "Ġcommerce": 19497, "ĠSle": 19498, "BU": 19499, "Ġentrepreneur": 19500, "Ġpeculiar": 19501, "ĠComments": 19502, "fre": 19503, "320": 19504, "ICS": 19505, "Ġimagery": 19506, "ĠCanon": 19507, "ĠElectronic": 19508, "short": 19509, "((": 19510, "Dig": 19511, "Ġcommem": 19512, "uced": 19513, "Ġinclined": 19514, "ĠSummon": 19515, "Ġcliff": 19516, "ĠMediterranean": 19517, "Ġpoetry": 19518, "Ġprosperity": 19519, "ĠRece": 19520, "Ġpills": 19521, "member": 19522, "Ġfinale": 19523, "unc": 19524, "ĠGig": 19525, "ä½": 19526, "Ġlod": 19527, "Ġbackward": 19528, "-+": 19529, "ĠForward": 19530, "Ġthri": 19531, "sure": 19532, "Ġsoap": 19533, "ĠFX": 19534, "RES": 19535, "ĠSexual": 19536, "oulos": 19537, "Ġfoolish": 19538, "Ġrighteous": 19539, "Ġcoff": 19540, "terrorism": 19541, "ustain": 19542, "oter": 19543, "Ġabuses": 19544, "next": 19545, "Ġabusive": 19546, "Ġthereafter": 19547, "Ġprohibition": 19548, "ĠSUP": 19549, "Ġdip": 19550, "Ġripped": 19551, "Ġinherited": 19552, "Ġbats": 19553, "stru": 19554, "GT": 19555, "Ġflawed": 19556, "phabet": 19557, "Ġfog": 19558, "doors": 19559, "Ġimaging": 19560, "Ġdigits": 19561, "ĠHungary": 19562, "Ġarrog": 19563, "Ġteachings": 19564, "Ġprotocols": 19565, "ĠBanks": 19566, "à¸": 19567, "pound": 19568, "ĠCurt": 19569, ".\")": 19570, "./": 19571, "Ġexemption": 19572, "endix": 19573, "ĠMull": 19574, "Ġimproves": 19575, "ĠGamer": 19576, "dimensional": 19577, "Icon": 19578, "ĠMargaret": 19579, "Status": 19580, "dates": 19581, "Ġintends": 19582, "Ġdepict": 19583, "Ġparked": 19584, "Joe": 19585, "ĠMarines": 19586, "chnology": 19587, "!).": 19588, "Ġjudged": 19589, "Ġweights": 19590, "Ray": 19591, "Ġapartments": 19592, "hester": 19593, "Ġreinforce": 19594, "Ġoffender": 19595, "occup": 19596, "Ġsore": 19597, "ept": 19598, "ĠPHP": 19599, "ĠBrow": 19600, "Ġauthorization": 19601, "ĠRisk": 19602, "ĠDelaware": 19603, "ĠQU": 19604, "Ġnotifications": 19605, "Ġsunlight": 19606, "Ġexclude": 19607, "dat": 19608, "Ġmesh": 19609, "ĠSudan": 19610, "Ġbelonged": 19611, "Ġsubway": 19612, "Ġnoon": 19613, "ĠInterior": 19614, "olics": 19615, "ĠLakers": 19616, "Ġcoding": 19617, "Disclaimer": 19618, "Calif": 19619, "Old": 19620, "Ġdisl": 19621, "?????": 19622, "Ġconfirms": 19623, "Ġrecruitment": 19624, "Ġhomicide": 19625, "Consider": 19626, "ĠJeffrey": 19627, "fty": 19628, "};": 19629, "Ġobjection": 19630, "doing": 19631, "ĠLeo": 19632, "Want": 19633, "Ġglow": 19634, "ĠClarke": 19635, "ĠNorman": 19636, "Ġverification": 19637, "Ġpacket": 19638, "ĠFormula": 19639, "Ġplag": 19640, "esville": 19641, "Ġshouting": 19642, "Ġov": 19643, "ĠREC": 19644, "ĠBub": 19645, "Ġninth": 19646, "Ġenerg": 19647, "Ġvalidity": 19648, "Ġups": 19649, "jack": 19650, "Ġneighboring": 19651, "ĠNec": 19652, "eworks": 19653, "ĠHab": 19654, "arez": 19655, "Ġspine": 19656, "Ġeventual": 19657, "ĠLeaders": 19658, "ĠCarn": 19659, "Ġprobation": 19660, "Ġromance": 19661, "msg": 19662, "ĠMechanical": 19663, "ERY": 19664, "Rock": 19665, "Ġpartisan": 19666, "Node": 19667, "assets": 19668, "minent": 19669, "Ġforeigners": 19670, "Ġtestify": 19671, "ĠUsually": 19672, "lords": 19673, "ĠGren": 19674, "ĠPowell": 19675, "BIL": 19676, "Ġsr": 19677, "Ġaddict": 19678, "Ġshells": 19679, "Ġsigh": 19680, "ĠYale": 19681, "ternity": 19682, "Ġ750": 19683, "EU": 19684, "ĠRifle": 19685, "Ġpatron": 19686, "ema": 19687, "ĠBannon": 19688, "anity": 19689, "Ġtropical": 19690, "ĠVII": 19691, "cross": 19692, "Everything": 19693, "ĠISO": 19694, "Ġhumble": 19695, "assing": 19696, "ĠFIG": 19697, "Ġupdating": 19698, "yson": 19699, "Ġcalcium": 19700, "Ġcompetent": 19701, "Ġsteering": 19702, "Prot": 19703, "ĠSY": 19704, "ĠFinals": 19705, "ĠRug": 19706, "159": 19707, "137": 19708, "ĠGolf": 19709, "Ġ126": 19710, "Ġaccommodation": 19711, "ĠHughes": 19712, "Ġaesthetic": 19713, "artisan": 19714, "ĠTwilight": 19715, "Ġprince": 19716, "ĠAgriculture": 19717, "ĠDisco": 19718, "Ġprecedent": 19719, "Ġtyping": 19720, "authorized": 19721, "Option": 19722, "ĠAub": 19723, "lishes": 19724, "acht": 19725, "mag": 19726, "Peter": 19727, "ĠUFO": 19728, "monton": 19729, "ĠLith": 19730, "Ġarom": 19731, "Ġsecuring": 19732, "Ġconfined": 19733, "private": 19734, "Ġswords": 19735, "Ġmarkers": 19736, "Ġmetabolic": 19737, "select": 19738, "ĠCurse": 19739, "ĠOt": 19740, "gressive": 19741, "Ġincumb": 19742, "ĠSaga": 19743, "Ġpriced": 19744, "Ġclearance": 19745, "Content": 19746, "Ġdrilling": 19747, "Ġnotices": 19748, "Ġbourgeois": 19749, "Ġvest": 19750, "Ġcookie": 19751, "ĠGuardians": 19752, "rys": 19753, "inyl": 19754, "Ġ124": 19755, "Ġplausible": 19756, "ongh": 19757, "ĠOdin": 19758, "Ġconception": 19759, "ĠYuk": 19760, "ĠBaghdad": 19761, "ĠFlag": 19762, "Austral": 19763, "ĠIBM": 19764, "Ġinternationally": 19765, "ĠWikiLeaks": 19766, "IED": 19767, "Ġcyn": 19768, "Ġchooses": 19769, "ĠPill": 19770, "Ġcombining": 19771, "Ġradi": 19772, "ĠMohammed": 19773, "defense": 19774, "atching": 19775, "Subject": 19776, "iciency": 19777, "Frame": 19778, "Ġ{\"": 19779, "Ġchess": 19780, "Ġtimer": 19781, "190": 19782, "Ġtin": 19783, "Ġordinance": 19784, "emetery": 19785, "Ġaccusing": 19786, "Ġnoticeable": 19787, "Ġcentres": 19788, "Ġlid": 19789, "ĠMills": 19790, "imgur": 19791, "Ġzoom": 19792, "ergic": 19793, "Ġcompression": 19794, "prim": 19795, "find": 19796, "Ġsurg": 19797, "Ġpand": 19798, "ĠKee": 19799, "ĠChad": 19800, "cellence": 19801, "oyle": 19802, "Ġsocialism": 19803, "ĠTravis": 19804, "ĠMHz": 19805, "Ġguild": 19806, "ALLY": 19807, "ĠSubscribe": 19808, "ĠRelated": 19809, "Ġoccurrence": 19810, "itching": 19811, "Ġfictional": 19812, "Ġcrush": 19813, "ĠEA": 19814, "cod": 19815, "mix": 19816, "ĠTriple": 19817, "Ġretrieve": 19818, "Ġstimulus": 19819, "Ġpsychiat": 19820, "ĠDoor": 19821, "Ġhomosexuality": 19822, "Ġelementary": 19823, "Ġcellular": 19824, "idian": 19825, "ĠLaun": 19826, "Ġintriguing": 19827, "Ġfoam": 19828, "ĠBass": 19829, "idi": 19830, "itsu": 19831, "Ġassure": 19832, "Ġcongrat": 19833, "Ġbusinessman": 19834, "ĠBoost": 19835, "close": 19836, "Ġlied": 19837, "Ġsciences": 19838, "ĠOmega": 19839, "ĠGraphics": 19840, "Ġ<=": 19841, "spoken": 19842, "Ġconnectivity": 19843, "Saturday": 19844, "ĠAvengers": 19845, "Ġtoggle": 19846, "Ġankle": 19847, "Ġnationalist": 19848, "model": 19849, "ĠPool": 19850, "ophobia": 19851, "Var": 19852, "ĠMons": 19853, "atories": 19854, "Ġaggressively": 19855, "Clear": 19856, "Forge": 19857, "acters": 19858, "Ġhedge": 19859, "Ġpipes": 19860, "Ġblunt": 19861, "Ġsq": 19862, "Ġremotely": 19863, "Wed": 19864, "asers": 19865, "Ġrefriger": 19866, "Ġtiles": 19867, "Ġrescued": 19868, "Ġcomprised": 19869, "insky": 19870, "Ġmanif": 19871, "avanaugh": 19872, "Ġprolifer": 19873, "Ġaligned": 19874, "xml": 19875, "Ġtriv": 19876, "Ġcoordination": 19877, "ĠPER": 19878, "ĠQuote": 19879, "134": 19880, "bf": 19881, "ĠSaw": 19882, "Ġtermination": 19883, "Ġ190": 19884, "Ġadditions": 19885, "Ġtrio": 19886, "Ġprojections": 19887, "Ġpositively": 19888, "Ġinclusive": 19889, "Ġmembr": 19890, "1990": 19891, "older": 19892, "Ġpracticed": 19893, "inkle": 19894, "Arch": 19895, "Ġstarters": 19896, "arius": 19897, "Ġintermediate": 19898, "ĠBenef": 19899, "ĠKiller": 19900, "Ġinterventions": 19901, "ĠKil": 19902, "ĠFlying": 19903, "Inv": 19904, "Ġpremature": 19905, "Ġpsychiatric": 19906, "Ġindie": 19907, "Ġcollar": 19908, "ĠRainbow": 19909, "afi": 19910, "Ġdisruption": 19911, "ĠFOX": 19912, "casting": 19913, "Ġmisdem": 19914, "cro": 19915, "Ġwipe": 19916, "ardon": 19917, "Ġbast": 19918, "ĠTommy": 19919, "ĠRepresentative": 19920, "Ġbelly": 19921, "ĠPO": 19922, "ĠBreitbart": 19923, "132": 19924, "Ġmessaging": 19925, "Should": 19926, "References": 19927, "ĠGRE": 19928, "istical": 19929, "LP": 19930, "ĠCav": 19931, "ĠCrazy": 19932, "Ġintuitive": 19933, "keeping": 19934, "ĠMoss": 19935, "Ġdiscontin": 19936, "ĠModule": 19937, "Ġunrelated": 19938, "ĠPractice": 19939, "ĠTransport": 19940, "Ġstatistically": 19941, "orns": 19942, "Ġsized": 19943, "pu": 19944, "Ġcaf": 19945, "ĠWorlds": 19946, "ĠRodgers": 19947, "ĠLun": 19948, "ĠComic": 19949, "living": 19950, "Ġcared": 19951, "Ġclimbed": 19952, "){": 19953, "Ġconsisted": 19954, "Ġmedieval": 19955, "folk": 19956, "Ġhacked": 19957, "Ġdire": 19958, "ĠHermione": 19959, "Ġtended": 19960, "ceans": 19961, "Daniel": 19962, "went": 19963, "Ġlegislators": 19964, "Ġredes": 19965, "games": 19966, "Ġgn": 19967, "amiliar": 19968, "Ġ++": 19969, "ggy": 19970, "threat": 19971, "Ġmagnet": 19972, "Ġperceive": 19973, "Ġzip": 19974, "Ġindictment": 19975, "Ġcritique": 19976, "gard": 19977, "ĠSafe": 19978, "ĠCream": 19979, "Ġadvent": 19980, "oba": 19981, "Ġvowed": 19982, "ousands": 19983, "Ġski": 19984, "Ġabortions": 19985, "uart": 19986, "Ġstunned": 19987, "Ġadvancing": 19988, "Ġlacked": 19989, "Ġ\\\"": 19990, "Ġschizophren": 19991, "Ġelegant": 19992, "Ġconferences": 19993, "Ġcanceled": 19994, "ĠHudson": 19995, "ĠHopefully": 19996, "Ġtrump": 19997, "Ġfrequencies": 19998, "Ġmeteor": 19999, "ĠJunior": 20000, "ĠFleet": 20001, "ĠMalcolm": 20002, "ĠTools": 20003, "Ġ........": 20004, "Ġhobby": 20005, "ĠEuropeans": 20006, "Ġ1500": 20007, "ĠInto": 20008, "Ġsway": 20009, "ĠAppro": 20010, "ĠCompl": 20011, "Community": 20012, "Ġtide": 20013, "ĠSummit": 20014, "ä»": 20015, "Ġintervals": 20016, "ĠEther": 20017, "Ġhabitat": 20018, "ĠStevens": 20019, "lishing": 20020, "ĠDomain": 20021, "Ġtriggers": 20022, "Ġchasing": 20023, "Ġcharm": 20024, "ĠFlower": 20025, "itored": 20026, "Ġblessing": 20027, "Ġtextures": 20028, "Five": 20029, "Ġliquor": 20030, "RP": 20031, "FIN": 20032, "Ġ1962": 20033, "CAR": 20034, "Unknown": 20035, "Ġresil": 20036, "ĠLily": 20037, "Ġabundance": 20038, "Ġpredictable": 20039, "rar": 20040, "Ġbullshit": 20041, "leen": 20042, "chet": 20043, "Mor": 20044, "Much": 20045, "ä¹": 20046, "Ġemphasized": 20047, "Ġcrust": 20048, "Ġprimitive": 20049, "Ġenjoyable": 20050, "ĠPictures": 20051, "Ġteammate": 20052, "pler": 20053, "ĠTol": 20054, "ĠKane": 20055, "Ġsummoned": 20056, "thy": 20057, "rama": 20058, "ĠHonda": 20059, "Ġrealizing": 20060, "Ġquicker": 20061, "Ġconcentrate": 20062, "clear": 20063, "Ġ210": 20064, "ĠErdogan": 20065, "aris": 20066, "Ġresponds": 20067, "ĠBI": 20068, "Ġeligibility": 20069, "Ġpushes": 20070, "ĠIdaho": 20071, "Ġaggrav": 20072, "Ġruins": 20073, "urations": 20074, "Ġbans": 20075, "Ġanat": 20076, "share": 20077, "Ġgrind": 20078, "hin": 20079, "umen": 20080, "Ġutilities": 20081, "ĠYankees": 20082, "Ġdatabases": 20083, "ĠDD": 20084, "Ġdisplaced": 20085, "Ġdependencies": 20086, "Ġstimulation": 20087, "hun": 20088, "houses": 20089, "ĠPretty": 20090, "ĠRavens": 20091, "ĠTODAY": 20092, "Ġassociates": 20093, "Ġtherape": 20094, "cled": 20095, "Ġdeer": 20096, "Ġrepairs": 20097, "rentice": 20098, "Ġreceptors": 20099, "Ġremed": 20100, "ĠCe": 20101, "Ġmarriages": 20102, "Ġballots": 20103, "ĠSoldier": 20104, "Ġhilarious": 20105, "opl": 20106, "138": 20107, "Ġinherently": 20108, "Ġignorant": 20109, "Ġbounce": 20110, "ĠEaster": 20111, "RELATED": 20112, "ĠCurrency": 20113, "EV": 20114, "ãĥŀ": 20115, "ĠLead": 20116, "Ġdeceased": 20117, "Brien": 20118, "ĠMusk": 20119, "JS": 20120, "Ġmerge": 20121, "hearted": 20122, "creat": 20123, "mitt": 20124, "mund": 20125, "ĠâĢĭ": 20126, "ĠBag": 20127, "Ġprojection": 20128, "Ġjava": 20129, "ĠStandards": 20130, "ĠLeonard": 20131, "Ġcoconut": 20132, "ĠPopulation": 20133, "Ġtraject": 20134, "Ġimply": 20135, "Ġcuriosity": 20136, "ĠDB": 20137, "ĠFresh": 20138, "ĠPor": 20139, "Ġheavier": 20140, "neys": 20141, "gomery": 20142, "Ġdeserved": 20143, "Ġphrases": 20144, "ĠGC": 20145, "Ġyeast": 20146, "desc": 20147, "Death": 20148, "Ġreboot": 20149, "Ġmetadata": 20150, "ICAL": 20151, "Ġrepay": 20152, "ĠIndependence": 20153, "Ġsuburban": 20154, "icals": 20155, "Ġatop": 20156, "Ġallocation": 20157, "generation": 20158, "ĠGram": 20159, "Ġmoisture": 20160, "Ġpine": 20161, "ĠLiberals": 20162, "Ġaides": 20163, "Ġunderest": 20164, "ĠBerry": 20165, "Ġceremon": 20166, "370": 20167, "astrous": 20168, "ĠPirates": 20169, "Ġtense": 20170, "ĠIndustries": 20171, "ĠAppeals": 20172, "ĠNear": 20173, "Ġè£ıç": 20174, "Ġlovers": 20175, "ĠCAP": 20176, "ĠCraw": 20177, "Ġgiants": 20178, "Ġefficacy": 20179, "Element": 20180, "ĠBehavior": 20181, "ĠToyota": 20182, "Ġintest": 20183, "Priv": 20184, "AI": 20185, "Ġmaneuver": 20186, "Ġperfection": 20187, "Ġbang": 20188, "paper": 20189, "rill": 20190, "George": 20191, "border": 20192, "inters": 20193, "ĠSeth": 20194, "Ġclues": 20195, "ĠLevi": 20196, "ĠRevenue": 20197, "147": 20198, "Ġvapor": 20199, "Ġfortunate": 20200, "Ġthreatens": 20201, "Ġvet": 20202, "Ġdependency": 20203, "ersed": 20204, "article": 20205, "ĠBlizzard": 20206, "Ġchlor": 20207, "Ġminus": 20208, "ĠBills": 20209, "Ġcryptocurrency": 20210, "Ġmetabolism": 20211, "tering": 20212, "Ġpestic": 20213, "steps": 20214, "ĠTreasure": 20215, "racted": 20216, "ĠConstant": 20217, "Ġtemp": 20218, "139": 20219, "ĠDetective": 20220, "urally": 20221, "Ġrecovering": 20222, "Ġcortex": 20223, "Ġ144": 20224, "closed": 20225, "Ġprejudice": 20226, "aunted": 20227, "Ġstorms": 20228, "ĠNOW": 20229, "Ġmachinery": 20230, "Address": 20231, "Ġcompelled": 20232, "270": 20233, "Ġdespair": 20234, "bane": 20235, "Ġvegetable": 20236, "Ġbeds": 20237, "Learn": 20238, "Ġcolorful": 20239, "Ġspike": 20240, "Ġmargins": 20241, "Ġsympathy": 20242, "Ġworkshop": 20243, "ĠCBC": 20244, "Sat": 20245, "Ġburns": 20246, "ĠGender": 20247, "Ġ129": 20248, "ĠCable": 20249, "Ġdebts": 20250, "ĠTheresa": 20251, "Ġreflecting": 20252, "Ġairst": 20253, "Ġrim": 20254, "ramid": 20255, "Ġweaknesses": 20256, "Writ": 20257, "oggle": 20258, "ti": 20259, "ĠCharge": 20260, "Ġweighed": 20261, "Ġ(.": 20262, "Ġlaughter": 20263, "Ġrouter": 20264, "ĠDemocracy": 20265, "Dear": 20266, "Ġhasht": 20267, "Ġdy": 20268, "Ġhints": 20269, "running": 20270, "Ġfinishes": 20271, "arus": 20272, "Mass": 20273, "result": 20274, "ascus": 20275, "Ġvintage": 20276, "Ġconqu": 20277, "Ġwildly": 20278, "acist": 20279, "Ġlingu": 20280, "Ġprotagonist": 20281, "strom": 20282, "teenth": 20283, "ĠSolo": 20284, "mac": 20285, "filled": 20286, "Ġrenown": 20287, "itives": 20288, "Ġmotive": 20289, "ĠAntar": 20290, "ĠMann": 20291, "ĠAdjust": 20292, "Ġrockets": 20293, "Ġtroubling": 20294, "ei": 20295, "Ġorganisms": 20296, "assis": 20297, "Christian": 20298, "Ġ145": 20299, "ĠHass": 20300, "Ġswall": 20301, "Ġwax": 20302, "ĠSurvival": 20303, "VS": 20304, "ĠMurd": 20305, "vd": 20306, "standard": 20307, "Ġdragons": 20308, "Ġacceleration": 20309, "rational": 20310, "final": 20311, "Ġpaired": 20312, "ĠEthereum": 20313, "Ġinterfaces": 20314, "Ġresent": 20315, "Ġartifacts": 20316, "Å«": 20317, "arel": 20318, "Ġcompetitor": 20319, "ĠNicholas": 20320, "ĠSurface": 20321, "cpp": 20322, "ĠTot": 20323, "Ġeconomically": 20324, "Ġorganised": 20325, "Ġenforced": 20326, "inho": 20327, "Ġvarieties": 20328, "Ġabdom": 20329, "ĠBailey": 20330, "idav": 20331, "ĠSalv": 20332, "paid": 20333, "Ġaltitude": 20334, "essert": 20335, "ĠGutenberg": 20336, "area": 20337, "opoulos": 20338, "Ġprofessors": 20339, "iggs": 20340, "ĠFate": 20341, "hey": 20342, "Ġ3000": 20343, "Dist": 20344, "Ġtwins": 20345, "cill": 20346, "ĠMaps": 20347, "Ġtraps": 20348, "Ġweed": 20349, "ĠKiss": 20350, "Ġyoga": 20351, "Ġrecipients": 20352, "ĠWestminster": 20353, "Ġpools": 20354, "ĠWalmart": 20355, "188": 20356, "ĠSchools": 20357, "attack": 20358, "ĠARM": 20359, "paragraph": 20360, "Warning": 20361, "jl": 20362, "Ġselfish": 20363, "anchez": 20364, "ĠHeights": 20365, "Fre": 20366, "ĠSoph": 20367, "Ġ--------------------------------": 20368, "tml": 20369, "333": 20370, "Ġraids": 20371, "Ġsatellites": 20372, "KEY": 20373, "Ġlasts": 20374, "ÑĤ": 20375, "Ins": 20376, "ĠDame": 20377, "Ġunpredict": 20378, "///": 20379, "ghai": 20380, "Ġartillery": 20381, "Ġcruise": 20382, "Ġgel": 20383, "ĠCabinet": 20384, "Ġblows": 20385, "ĠEsp": 20386, "Ġproximity": 20387, "othe": 20388, "ĠSkills": 20389, "ĠUpper": 20390, "obo": 20391, "ĠNDP": 20392, "Ġenjoys": 20393, "Ġrepeating": 20394, "ĠConstruction": 20395, "ĠQuestions": 20396, "Hillary": 20397, "Ġuint": 20398, "Ġprocessors": 20399, "ĠGibson": 20400, "ĠMultiple": 20401, "qa": 20402, "ĠBom": 20403, "ĠMiles": 20404, "ventional": 20405, "Ġhurts": 20406, "skin": 20407, "ĠAIDS": 20408, "Ġadvisers": 20409, "ĠRoot": 20410, "Ġmethodology": 20411, "ĠDale": 20412, "Ġdeton": 20413, "ĠKnowledge": 20414, "sequently": 20415, "Ġ121": 20416, "Ġconnects": 20417, "Cy": 20418, "ĠDanger": 20419, "Ġcontributors": 20420, "ĠBent": 20421, "Ġbrass": 20422, "ĠGuns": 20423, "into": 20424, "ĠFortune": 20425, "Ġbroker": 20426, "balance": 20427, "Ġlengths": 20428, "Ġvic": 20429, "Ġaveraging": 20430, "Ġappropriately": 20431, "ĠCamera": 20432, "Ġsandwich": 20433, "ĠCDC": 20434, "Ġcoordinate": 20435, "Ġnavig": 20436, "Ġgoodness": 20437, "laim": 20438, "Ġbrake": 20439, "Ġextremist": 20440, "ĠWake": 20441, "ĠMend": 20442, "ĠTiny": 20443, "ĠCOL": 20444, "ĠRF": 20445, "ĠDual": 20446, "ĠWine": 20447, "Case": 20448, "Ġrefined": 20449, "Ġlamp": 20450, "Lead": 20451, "Ġbapt": 20452, "ĠCarb": 20453, "ĠSadd": 20454, "ĠMinneapolis": 20455, "PDF": 20456, "Early": 20457, "ĠHidden": 20458, "Its": 20459, "ĠTIME": 20460, "Ġpap": 20461, "Ġcommissioned": 20462, "ĠFew": 20463, "ĠColts": 20464, "ĠBren": 20465, "Ġbothered": 20466, "Ġlikewise": 20467, "Exper": 20468, "ĠSchw": 20469, "cry": 20470, "nn": 20471, "ĠMitch": 20472, "imon": 20473, "MG": 20474, "bm": 20475, "UMP": 20476, "rays": 20477, "Ġregistry": 20478, "Ġ270": 20479, "achine": 20480, "rella": 20481, "anting": 20482, "00000": 20483, "Ġruined": 20484, "spot": 20485, "Ġta": 20486, "Ġmaximize": 20487, "Ġinconven": 20488, "Dead": 20489, "Human": 20490, "Enabled": 20491, "ĠMarie": 20492, "Ġchill": 20493, "ĠParadise": 20494, "Ġstarring": 20495, "ĠLatino": 20496, "ĠProtocol": 20497, "ĠEVER": 20498, "Ġsuppliers": 20499, "message": 20500, "ĠBrock": 20501, "Ġserum": 20502, "âĸĪâĸĪâĸĪâĸĪ": 20503, "Ġencomp": 20504, "Ġambition": 20505, "uese": 20506, "Ġarrows": 20507, "Andrew": 20508, "Ġantenna": 20509, "Ġ1961": 20510, "ĠBark": 20511, "Ġbool": 20512, "ãĤª": 20513, "ĠStorage": 20514, "Ġrailway": 20515, "Ġtougher": 20516, "ĠCad": 20517, "Ġwashing": 20518, "Py": 20519, "']": 20520, "embed": 20521, "ĠMemphis": 20522, "ackle": 20523, "Ġfamously": 20524, "ĠFortunately": 20525, "ovies": 20526, "Ġmindset": 20527, "Ġsneak": 20528, "ĠDh": 20529, "RAW": 20530, "ĠSimpson": 20531, "Ġlivest": 20532, "Ġlandmark": 20533, "Ġcement": 20534, "Low": 20535, "Ġthrilled": 20536, "ĠCourse": 20537, "inel": 20538, "Ġchuck": 20539, "idate": 20540, "global": 20541, "Ġwhit": 20542, "Ġ�": 20543, "adays": 20544, "ski": 20545, "ĠSV": 20546, "Ġviruses": 20547, "306": 20548, "ĠRespons": 20549, "Ġtheaters": 20550, "ĠBranch": 20551, "ĠGeneva": 20552, "ĠMK": 20553, "Ġunbeliev": 20554, "Ġcommunist": 20555, "Original": 20556, "ĠReceived": 20557, "ĠTransfer": 20558, "ĠArg": 20559, "Input": 20560, "ĠStrategy": 20561, "Ġpalace": 20562, "thening": 20563, "Dri": 20564, "Ġsentencing": 20565, "umbnail": 20566, "Ġpins": 20567, "recy": 20568, "Ġsiblings": 20569, "Getting": 20570, "ĠBU": 20571, "ĠNorthwest": 20572, "Ġprolonged": 20573, "ĠSakura": 20574, "Comb": 20575, "ĠBour": 20576, "Ġinadequate": 20577, "ĠKash": 20578, "Ġusername": 20579, "ĠImprove": 20580, "Ġbattling": 20581, "ĠMAC": 20582, "Ġcurriculum": 20583, "Ġsoda": 20584, "ĠCannon": 20585, "Ġsensible": 20586, "spons": 20587, "December": 20588, "Ġwicked": 20589, "ĠPengu": 20590, "Ġdictators": 20591, "ĠHearts": 20592, "ogyn": 20593, "Ġsimilarities": 20594, "ĠStats": 20595, "Ġhollow": 20596, "itations": 20597, "\":[": 20598, "Ġhover": 20599, "ĠListen": 20600, "sch": 20601, "Sund": 20602, "Ġcad": 20603, "ĠParks": 20604, "Ġlur": 20605, "Ġhype": 20606, "ĠLem": 20607, "NAME": 20608, "isure": 20609, "Friday": 20610, "Ġshoots": 20611, "Ġcloses": 20612, "Ġdb": 20613, "ĠRidge": 20614, "ĠDifferent": 20615, "Ġreplies": 20616, "ĠBroadway": 20617, "opers": 20618, "Ġintoler": 20619, "ĠZeus": 20620, "akespe": 20621, "Ġproprietary": 20622, "Ġrequesting": 20623, "Ġcontrollers": 20624, "ĠMIN": 20625, "imedia": 20626, "becca": 20627, "Ġexpans": 20628, "Ġoils": 20629, "Bot": 20630, "ĠChand": 20631, "Ġprinter": 20632, "Ġtopped": 20633, "ĠPOL": 20634, "ĠEarlier": 20635, "Social": 20636, "avin": 20637, "Ġdecreases": 20638, "ĠSeb": 20639, "Ġspecifications": 20640, "ĠBlast": 20641, "ĠKurt": 20642, "Ġfreel": 20643, "Brown": 20644, "Ġdilig": 20645, "roe": 20646, "ĠProblem": 20647, "ĠQuad": 20648, "Ġdecentral": 20649, "ĠVector": 20650, "anut": 20651, "Ġplugins": 20652, "ĠGregory": 20653, "Ġfucked": 20654, "elines": 20655, "ĠAmbassador": 20656, "take": 20657, "Ġcleans": 20658, "ongyang": 20659, "Anonymous": 20660, "stro": 20661, "\"}": 20662, "aline": 20663, "ĠOdd": 20664, "ĠEug": 20665, "216": 20666, "Ġboil": 20667, "ĠPowers": 20668, "Ġnurses": 20669, "Obviously": 20670, "ĠTechnical": 20671, "Ġexceeded": 20672, "ORS": 20673, "Ġextremists": 20674, "Ġtraces": 20675, "expl": 20676, "Ġcomr": 20677, "ĠSach": 20678, ")/": 20679, "Ġmasks": 20680, "Ġsci": 20681, "Bon": 20682, "Ġregression": 20683, "wegian": 20684, "Ġadvisor": 20685, "itures": 20686, "ĠVo": 20687, "example": 20688, "ĠInstruct": 20689, "Ġsiege": 20690, "Ġreductions": 20691, "ptr": 20692, "Ġstatutory": 20693, "Ġremoves": 20694, "Ġpuck": 20695, "redits": 20696, "Ġbee": 20697, "Ġsalad": 20698, "Ġpromotions": 20699, "ĠJoshua": 20700, "withstanding": 20701, "ETH": 20702, "ĠCha": 20703, "imus": 20704, "Ġexpenditure": 20705, "aunting": 20706, "Ġdelighted": 20707, "Ġ155": 20708, "beh": 20709, "Ġcarpet": 20710, "ĠSpart": 20711, "Ġjungle": 20712, "lists": 20713, "Ġbullying": 20714, "ĠNobel": 20715, "ĠGlen": 20716, "Ġreferenced": 20717, "Ġintroduces": 20718, "sein": 20719, "Ġchopped": 20720, "glass": 20721, "ĠWrest": 20722, "Ġneutrality": 20723, "ĠâĻ": 20724, "Ġinvestigator": 20725, "Ġshelves": 20726, "Ġunconstitutional": 20727, "Ġreproduction": 20728, "Ġmerchant": 20729, "mia": 20730, "Ġmetrics": 20731, "Ġexplosives": 20732, "ĠSonia": 20733, "Ġbodily": 20734, "Ġthickness": 20735, "Ġpredominantly": 20736, "ĠAbility": 20737, "Ġmonitored": 20738, "ICH": 20739, "Ġ].": 20740, "ĠMartinez": 20741, "Ġvisibility": 20742, "Ġqueries": 20743, "Ġgenocide": 20744, "ĠWarfare": 20745, "Query": 20746, "Ġstudios": 20747, "Ġembry": 20748, "Ġcorridor": 20749, "Ġcleaned": 20750, "complete": 20751, "ĠMH": 20752, "Ġenrollment": 20753, "INGS": 20754, "Ġimpacted": 20755, "Ġdisastrous": 20756, "ĠYun": 20757, "ĠClaire": 20758, "ĠBasically": 20759, "yt": 20760, "usterity": 20761, "Ġindirectly": 20762, "wik": 20763, "Ġdod": 20764, "ĠCarr": 20765, "Ġamp": 20766, "Ġprohibit": 20767, "ĠInitial": 20768, "ĠRd": 20769, "iji": 20770, "Ġeducate": 20771, "corn": 20772, "iott": 20773, "ĠBeauty": 20774, "Ġdetective": 20775, "ĠConn": 20776, "since": 20777, "Ġstagger": 20778, "Ġobese": 20779, "Ġbree": 20780, "ologic": 20781, "isse": 20782, "walker": 20783, "Ġblades": 20784, "Ġlawful": 20785, "func": 20786, "ĠBehind": 20787, "Ġappetite": 20788, "Ġ(*": 20789, "Ġtennis": 20790, "Ġoffspring": 20791, "Ġjets": 20792, "Ġstructured": 20793, "Ġaforementioned": 20794, "Nov": 20795, "Ġscaling": 20796, "fill": 20797, "Ġstew": 20798, "Ġcurb": 20799, "ĠStephan": 20800, "edIn": 20801, "SF": 20802, "obic": 20803, "éŃĶ": 20804, "oug": 20805, "ĠMM": 20806, "Ġgenetically": 20807, "opez": 20808, "136": 20809, "Ġumb": 20810, "ancers": 20811, "Ġcohort": 20812, "Ġmerchandise": 20813, "Ġimposing": 20814, "ĠLegislature": 20815, "ĠArchive": 20816, "ivia": 20817, "ĠNaval": 20818, "Ġoffences": 20819, "Ġmiracle": 20820, "Ġsnapped": 20821, "Ġfoes": 20822, "Ġextensively": 20823, "ĠRaf": 20824, "Ġcater": 20825, "edience": 20826, "Kit": 20827, "ĠBin": 20828, "Ġrecommends": 20829, "ĠCities": 20830, "Ġrigid": 20831, "ĠREAD": 20832, "ĠNoble": 20833, "ĠTian": 20834, "Ġcertificates": 20835, "antis": 20836, "oiler": 20837, "ĠBuddhist": 20838, "did": 20839, "Ġsurveyed": 20840, "Ġdownward": 20841, "Ġprints": 20842, "ĠMotion": 20843, "ronics": 20844, "ĠSans": 20845, "ossibly": 20846, "uctions": 20847, "Ġcolonies": 20848, "ĠDanish": 20849, "unit": 20850, "Ġspoil": 20851, "Ġadvisory": 20852, "berries": 20853, "Plan": 20854, "Ġspecification": 20855, "ophers": 20856, "ĠResource": 20857, "Ġshirts": 20858, "prisingly": 20859, "communications": 20860, "Ġtrivial": 20861, "Ġmentioning": 20862, "isexual": 20863, "Ġsupplements": 20864, "Ġsupervision": 20865, "BP": 20866, "vor": 20867, "Ġwit": 20868, "Ġcooldown": 20869, "Ġplaintiff": 20870, "ĠReviews": 20871, "ĠSri": 20872, "ĠMint": 20873, "ĠSugar": 20874, "Ġafterward": 20875, "ĠPriest": 20876, "ĠInvestment": 20877, "ogene": 20878, "ĠTaking": 20879, "Ġstretching": 20880, "Ġinflammation": 20881, "ĠTehran": 20882, "Ġlining": 20883, "Ġfreezing": 20884, "ĠEntity": 20885, "Ġinspiring": 20886, "special": 20887, "price": 20888, "Ġsue": 20889, "ĠPorter": 20890, "ounge": 20891, "ETA": 20892, "ĠDerek": 20893, "ĠLuis": 20894, "uo": 20895, "ymph": 20896, "Ġexterior": 20897, "ihil": 20898, "ĠAshley": 20899, "inator": 20900, "Ġnutrients": 20901, "ĠThrones": 20902, "Ġfinances": 20903, "ĠInspect": 20904, "Ġspecially": 20905, "ĠRequired": 20906, "ĠPTS": 20907, "ĠViolence": 20908, "ointed": 20909, "shots": 20910, "Ġexcerpt": 20911, "coon": 20912, "INS": 20913, "ĠGri": 20914, "Ġrecognised": 20915, "Week": 20916, "Young": 20917, "Ġvom": 20918, "isle": 20919, "ĠCurry": 20920, "ĠBuddh": 20921, "Ġnotebook": 20922, "Ġdurable": 20923, "/?": 20924, "ĠGad": 20925, "ĠPupp": 20926, "Ġforgive": 20927, "park": 20928, "Ġpersonalities": 20929, "analysis": 20930, "clamation": 20931, "Ġelevator": 20932, "Ġwarehouse": 20933, "ĠRole": 20934, "unn": 20935, "Ġillustration": 20936, "ĠScan": 20937, "Ġatmospheric": 20938, "Import": 20939, "ANC": 20940, "ricted": 20941, "fu": 20942, "010": 20943, "Ġarche": 20944, "Ġrewarded": 20945, "akespeare": 20946, "Ġinternally": 20947, "ĠRBI": 20948, "alker": 20949, "Ġelephant": 20950, "owitz": 20951, "ĠPizza": 20952, "Ġbipartisan": 20953, "és": 20954, "Ġslowed": 20955, "ĠStark": 20956, "Ġoverride": 20957, "OUS": 20958, "Ġ320": 20959, "undreds": 20960, "ĠDeck": 20961, "ĠCensus": 20962, "bee": 20963, "146": 20964, "otor": 20965, "Ġip": 20966, "Ġub": 20967, "ocations": 20968, "ĠButton": 20969, "rice": 20970, "Ġcripp": 20971, "fff": 20972, "Ġoriginated": 20973, "Ġoverwhelmed": 20974, "appa": 20975, "Ġforemost": 20976, "âĢij": 20977, "ĠLEG": 20978, "release": 20979, "eatured": 20980, "atches": 20981, "Ġreps": 20982, "Ġlending": 20983, "ĠReference": 20984, "ĠClient": 20985, "165": 20986, "venth": 20987, "Complete": 20988, "ĠPatrol": 20989, "Ġsworn": 20990, "cam": 20991, "Ġshuttle": 20992, "ĠRalph": 20993, "Ġhometown": 20994, "-,": 20995, "onal": 20996, "ĠBP": 20997, "åı": 20998, "Ġpersuade": 20999, "ĠAlexand": 21000, "Ġcombines": 21001, "Ġvivid": 21002, "ĠLag": 21003, "Ġencoding": 21004, "Ġsalvation": 21005, "wen": 21006, "ĠRecovery": 21007, "iya": 21008, "University": 21009, "ĠBiden": 21010, "Ġbudgets": 21011, "ĠTexans": 21012, "fits": 21013, "Ġhonored": 21014, "Ġpython": 21015, "TD": 21016, "###": 21017, "clone": 21018, "Ġblink": 21019, "ĠLiquid": 21020, "Ġunemployed": 21021, "Ġclashes": 21022, "ĠCounsel": 21023, "Ġdirecting": 21024, "Ġpunct": 21025, "ĠFalcons": 21026, "Ġshark": 21027, "ĠDamascus": 21028, "Ġjeans": 21029, "Ġembark": 21030, "Ġseize": 21031, "Ġupwards": 21032, "280": 21033, "ĠEz": 21034, "ĠAnything": 21035, "Ġexotic": 21036, "lower": 21037, "ĠCreator": 21038, "ĠUm": 21039, "Ġsuburbs": 21040, "berger": 21041, "ĠWend": 21042, "Ġmint": 21043, "ĠXX": 21044, "ĠDro": 21045, "Ġsuffers": 21046, "Ġherb": 21047, "tree": 21048, "Ġfragile": 21049, "Ġflooded": 21050, "ĠAlcohol": 21051, "olean": 21052, "nyder": 21053, "ĠKO": 21054, "Fram": 21055, "Ġ136": 21056, "Ġowed": 21057, "ĠMelee": 21058, "ĠHash": 21059, "Ġwhisk": 21060, "Ġsudo": 21061, "rr": 21062, "Quick": 21063, "appro": 21064, "Ġii": 21065, "ĠExamples": 21066, "hee": 21067, "Ġpromotes": 21068, "perature": 21069, "kar": 21070, "ĠHonor": 21071, "Ġsodium": 21072, "ĠLif": 21073, "rosso": 21074, "intendent": 21075, "Ġcorrespondent": 21076, "Found": 21077, "secret": 21078, "Ġidentifies": 21079, "agne": 21080, "Ġlou": 21081, "ĠPP": 21082, "Ġcoincidence": 21083, "move": 21084, "Ġmilitia": 21085, "Ġinfiltr": 21086, "ĠPrimary": 21087, "Ġpitching": 21088, "ĠIb": 21089, "ĠGOOD": 21090, "ãĤ¸": 21091, "ĠWizards": 21092, "iral": 21093, "ĠVenus": 21094, "RR": 21095, "ĠâĢķ": 21096, "ĠCasey": 21097, "Ġsadly": 21098, "Ġadmire": 21099, "Ġembarrassed": 21100, "cb": 21101, "Mel": 21102, "Ġtubes": 21103, "Ġbeautifully": 21104, "ĠQueensland": 21105, "Below": 21106, "rez": 21107, "quet": 21108, "pleasant": 21109, "Ġ«": 21110, "Camp": 21111, "Ġdecisive": 21112, "1998": 21113, "ĠLamb": 21114, "utton": 21115, "hn": 21116, "ĠJagu": 21117, "aunder": 21118, "ĠCord": 21119, "Ġclerk": 21120, "Ġcaffe": 21121, "Ġwiped": 21122, "Ġreim": 21123, "ĠMountains": 21124, "Ġimprisoned": 21125, "Ġdevelops": 21126, "ĠPra": 21127, "Ġmodeling": 21128, "Anyone": 21129, "ancel": 21130, "ĠSit": 21131, "Ġshields": 21132, "Ġlawn": 21133, "Ġcardiovascular": 21134, "Ġdemonstrating": 21135, "Ġparse": 21136, "ĠIsraelis": 21137, "Ġeuros": 21138, "143": 21139, "Ġglorious": 21140, "inski": 21141, "ecd": 21142, "Ġconditioning": 21143, "Ġhelpless": 21144, "Ġmicrosc": 21145, "ĠHarbor": 21146, "Ġstakes": 21147, "Ġ260": 21148, "Ġunequ": 21149, "ĠFloyd": 21150, "Ġdamp": 21151, "Ġapparatus": 21152, "ĠLaws": 21153, "Ġcounters": 21154, "Ġinduce": 21155, "atable": 21156, "ĠAhmed": 21157, "Ġslam": 21158, "November": 21159, "Ġpersist": 21160, "Ġimminent": 21161, "án": 21162, "Ġshred": 21163, "Ġphases": 21164, "ĠEdmonton": 21165, "ĠArmstrong": 21166, "ĠMeet": 21167, "ĠKitty": 21168, "ÑĢ": 21169, "circ": 21170, "ĠAdult": 21171, "Ġarose": 21172, "ĠXen": 21173, "Dan": 21174, "gow": 21175, "Ġsuperf": 21176, "ĠAdmir": 21177, "Ġendure": 21178, "Ġkeyword": 21179, "yrus": 21180, "Ġyarn": 21181, "Ġpathway": 21182, "ĠHopkins": 21183, "midt": 21184, "Ġcensorship": 21185, "dependent": 21186, "Ġinstructor": 21187, "Sources": 21188, "Ġtoe": 21189, "Ġballoon": 21190, "Nob": 21191, "Ġswear": 21192, "ĠCastro": 21193, "Ġgloss": 21194, "ĠKavanaugh": 21195, "Ġremarkably": 21196, "Photos": 21197, "ĠNom": 21198, "ĠSoutheast": 21199, "yers": 21200, "Ġvalidation": 21201, "Ġcannon": 21202, "ĠVictory": 21203, "ĠPierre": 21204, "Ġcautious": 21205, "Audio": 21206, "Ġfetch": 21207, "ĠGift": 21208, "ĠHyp": 21209, "Ġremedy": 21210, "ZE": 21211, "Ġscent": 21212, "Ġbeard": 21213, "ĠRut": 21214, "-\"": 21215, "Ġpatents": 21216, "Hy": 21217, "Ġunjust": 21218, "Ġpotato": 21219, "Ġforthcoming": 21220, "Ġchef": 21221, "ĠRift": 21222, "affe": 21223, "ĠROM": 21224, "ĠLaunch": 21225, "Ġpads": 21226, "ĠNeo": 21227, "Ġonset": 21228, "Ġsqueeze": 21229, "safe": 21230, "Ġprefix": 21231, "ĠTM": 21232, "ĠNearly": 21233, "ĠClinical": 21234, "ĠMental": 21235, "otiation": 21236, "ĠUnic": 21237, "antry": 21238, "ĠCir": 21239, "Ġepit": 21240, "æ": 21241, "Ġextracted": 21242, "versely": 21243, "riad": 21244, "Ġstrains": 21245, "Ġtops": 21246, "Ġpoem": 21247, "ĠRandy": 21248, "ĠMaple": 21249, "THER": 21250, "upiter": 21251, "ĠSSD": 21252, "ļé": 21253, "Ġuncon": 21254, "pering": 21255, "Ġslept": 21256, "iners": 21257, "Ġunderwater": 21258, "ĠEvidence": 21259, "gone": 21260, "205": 21261, "Ġhistorians": 21262, "Ġsynthesis": 21263, "Ġfrog": 21264, "basketball": 21265, "Ġvibrant": 21266, "Ġsubord": 21267, "Ġ365": 21268, "ĠDial": 21269, "Ġcooperate": 21270, "HAHA": 21271, "Ġgreeted": 21272, "158": 21273, "Ġjazz": 21274, "Ġintox": 21275, "ĠWalking": 21276, "Ġsupervisor": 21277, "ĠFusion": 21278, "ĠMercedes": 21279, "send": 21280, "Ham": 21281, "sd": 21282, "nl": 21283, "Ġtours": 21284, "ĠFIFA": 21285, "Ġculp": 21286, "gd": 21287, "304": 21288, "Ġpleas": 21289, "Ġillustrates": 21290, "ĠColombia": 21291, "Ġhighlighting": 21292, "ĠSummary": 21293, "Ġexposing": 21294, "ĠDru": 21295, "Ġirony": 21296, "ritional": 21297, "ĠCarroll": 21298, "ĠEllis": 21299, "Pict": 21300, "ĠRapt": 21301, "Ġadapter": 21302, "Ġunm": 21303, "Ġcorpse": 21304, "Ġcelebrities": 21305, "Den": 21306, "atum": 21307, "ĠApocalypse": 21308, "ĠWag": 21309, "lining": 21310, "Ġhormones": 21311, "Rub": 21312, "ĠXi": 21313, "ĠVaults": 21314, "208": 21315, "alkyrie": 21316, "inosaur": 21317, "Ġfeeds": 21318, "vity": 21319, "Ġdefeating": 21320, "Wait": 21321, "Ġemphasize": 21322, "ĠSteelers": 21323, "yrinth": 21324, "leys": 21325, "ĠWhenever": 21326, "Currently": 21327, "ĠClock": 21328, "Ġcollectively": 21329, "anyon": 21330, "ĠJP": 21331, "Ġmentality": 21332, "Ġdownloads": 21333, "Ġsurroundings": 21334, "ĠBarnes": 21335, "Ġflagship": 21336, "Ġindicators": 21337, "Ġgrapp": 21338, "January": 21339, "ĠElemental": 21340, "ĠAthena": 21341, "ibal": 21342, "Ġsights": 21343, "Ġcapita": 21344, "ĠTreaty": 21345, "Ġvoiced": 21346, "ĠGaz": 21347, "lette": 21348, "Ġya": 21349, "Ġexpired": 21350, "Legend": 21351, "Hot": 21352, "nature": 21353, "Ġunstable": 21354, "Ġ280": 21355, "ú": 21356, "Comment": 21357, "ALE": 21358, "Ġquests": 21359, "Ġhandler": 21360, "nis": 21361, "Ġversatile": 21362, "Ġconceal": 21363, "engeance": 21364, "ĠInteractive": 21365, "Ġobsessed": 21366, "ĠDogs": 21367, "Ġcracked": 21368, "Sound": 21369, "sv": 21370, "ĠDylan": 21371, "roads": 21372, "fx": 21373, "ĠCatholics": 21374, "ĠHag": 21375, "Ġslammed": 21376, "Ġglowing": 21377, "sale": 21378, "Ġtissues": 21379, "ĠChi": 21380, "nee": 21381, "Ġcher": 21382, "sic": 21383, "urrection": 21384, "Ġbacon": 21385, "ulatory": 21386, ").\"": 21387, "Ġirregular": 21388, "FORM": 21389, "assed": 21390, "Ġintentional": 21391, "Ġcompensate": 21392, "ĠSpeaking": 21393, "ĠSets": 21394, "153": 21395, "Ġconventions": 21396, "bands": 21397, "emade": 21398, "Ġecc": 21399, "ĠWinston": 21400, "ĠAssassin": 21401, "ĠBelgian": 21402, "Ġdependence": 21403, "Ġniche": 21404, "Ġbark": 21405, "ĠJazz": 21406, "Ġdisadvantage": 21407, "Ġgasoline": 21408, "Ġ165": 21409, "çļĦ": 21410, "essa": 21411, "module": 21412, "angular": 21413, "OY": 21414, "ĠTreatment": 21415, "itas": 21416, "olation": 21417, "ĠArnold": 21418, "Ġfeud": 21419, "ĠNest": 21420, "Ġtheatre": 21421, "ewater": 21422, "Ġminors": 21423, "olicy": 21424, "ĠHaven": 21425, "division": 21426, "Ġtrunk": 21427, "Far": 21428, "ĠPull": 21429, "Ġcapturing": 21430, "Ġ1800": 21431, "ĠTeen": 21432, "Ġexempl": 21433, "Ġclinics": 21434, "ĠBurg": 21435, "Ġsubstit": 21436, "Ġpayload": 21437, "ĠLav": 21438, "ĠTroy": 21439, "ĠWitness": 21440, "Ġfragments": 21441, "Ġpasswords": 21442, "Ġgospel": 21443, "ĠGin": 21444, "Ġtenants": 21445, "olith": 21446, "Six": 21447, "Previous": 21448, "ĠAges": 21449, "ĠDarwin": 21450, "Ġblat": 21451, "Ġempathy": 21452, "smith": 21453, "bag": 21454, "ĠEcho": 21455, "ĠCamb": 21456, "ĠMadd": 21457, "ĠBoo": 21458, "Ġrede": 21459, "ĠBurning": 21460, "Ġsmoothly": 21461, "ĠAdrian": 21462, "ĠVampire": 21463, "ĠMonsters": 21464, "steam": 21465, "Style": 21466, "Ma": 21467, "rea": 21468, "ĠDwar": 21469, "alyst": 21470, "ursor": 21471, "Ġelimination": 21472, "Ġcrypto": 21473, "cht": 21474, "ĠEternal": 21475, "âĢ¦]": 21476, "ĠSorce": 21477, "Ill": 21478, "NER": 21479, "Ġuh": 21480, "Conclusion": 21481, "wage": 21482, "Ġrespir": 21483, "Ġreminis": 21484, "hetical": 21485, "Ġgy": 21486, "Ġutilized": 21487, "icidal": 21488, "Ġ1900": 21489, "Ġhunters": 21490, "ĠSwan": 21491, "ĠReact": 21492, "Ġvisitor": 21493, "ĠThanksgiving": 21494, "308": 21495, "Posts": 21496, "Ġhips": 21497, "1997": 21498, "omers": 21499, "Ġknocking": 21500, "ĠVehicle": 21501, "Ġtil": 21502, "Ġ138": 21503, "Ġmi": 21504, "ĠInvestigation": 21505, "ĠKenya": 21506, "Ġcasino": 21507, "Ġmotives": 21508, "Ġregain": 21509, "rex": 21510, "Ġweekends": 21511, "Ġstabbed": 21512, "boro": 21513, "Ġexploited": 21514, "ĠHAVE": 21515, "ĠTelevision": 21516, "cock": 21517, "Ġpreparations": 21518, "Ġendeav": 21519, "ĠRemote": 21520, "ĠMaker": 21521, "ĠProdu": 21522, "ĠEvan": 21523, "Ġinformational": 21524, "ĠLouisville": 21525, "154": 21526, "ĠDreams": 21527, "Ġplots": 21528, "ĠRunner": 21529, "Ġhurting": 21530, "Ġacademy": 21531, "ĠMontgomery": 21532, "nm": 21533, "ĠLanc": 21534, "ĠAlz": 21535, "210": 21536, "elong": 21537, "Ġretailer": 21538, "Ġarising": 21539, "Ġrebellion": 21540, "Ġblonde": 21541, "played": 21542, "Ġinstrumental": 21543, "Cross": 21544, "Ġretention": 21545, "Ġtherapeutic": 21546, "Ġseas": 21547, "Ġinfantry": 21548, "ĠClint": 21549, "Ġprompting": 21550, "Ġbitch": 21551, "Ġstems": 21552, "ĠKra": 21553, "Ġthesis": 21554, "ĠBog": 21555, "rued": 21556, "Ġkings": 21557, "Ġclay": 21558, "ificent": 21559, "ĠYES": 21560, "ĠThing": 21561, "ĠCubs": 21562, "veyard": 21563, "elsh": 21564, "inarily": 21565, "ĠEy": 21566, "ĠRolling": 21567, "Ġevolving": 21568, "India": 21569, "Ġrecognizes": 21570, "Ġgraduation": 21571, "isers": 21572, "Ġfertility": 21573, "ĠMilan": 21574, "Command": 21575, "Ġboxing": 21576, "Ġ1943": 21577, "Ġgluten": 21578, "ĠEmir": 21579, "Ġidol": 21580, "Ġconceived": 21581, "ĠCreation": 21582, "Merit": 21583, "uddy": 21584, "ussions": 21585, "ĠLieutenant": 21586, "ietal": 21587, "Ġunchanged": 21588, "ĠScale": 21589, "ĠCrimea": 21590, "balls": 21591, "atorial": 21592, "Ġdepths": 21593, "Ġempirical": 21594, "Ġtransm": 21595, "Ġunsafe": 21596, "missible": 21597, "comfort": 21598, "156": 21599, "Ġmechanic": 21600, "002": 21601, "lins": 21602, "Ġsmoked": 21603, "Pos": 21604, "Ġslowing": 21605, "Ġlav": 21606, "Texas": 21607, "Ġcheating": 21608, "ĠMetropolitan": 21609, "ethyl": 21610, "Ġdiscovering": 21611, "asse": 21612, "Ġpencil": 21613, "ĠPyongyang": 21614, "Ġcloset": 21615, "ĠSheet": 21616, "ĠEntry": 21617, "oustic": 21618, "Ġmyst": 21619, "erate": 21620, "ariat": 21621, "Ġminerals": 21622, "Ġmusician": 21623, "ĠPul": 21624, "ĠMaz": 21625, "249": 21626, "Ġpermissions": 21627, "Ġiv": 21628, "enary": 21629, "ickers": 21630, "ĠBing": 21631, "hea": 21632, "enable": 21633, "Ġgriev": 21634, "Ġasserted": 21635, "ĠColonel": 21636, "Ġaffidav": 21637, "wo": 21638, "Ġseated": 21639, "ĠRide": 21640, "Ġpaintings": 21641, "ĠPix": 21642, "Ġ137": 21643, "ishi": 21644, "umbai": 21645, "gotten": 21646, "ĠEarl": 21647, "Ġinning": 21648, "Ġcensus": 21649, "Ġtravelled": 21650, "ĠConsult": 21651, "185": 21652, "bind": 21653, "Ġsimplicity": 21654, "Ġoverlooked": 21655, "ĠHelpful": 21656, "Ġmonkey": 21657, "Ġoverwhelmingly": 21658, "Blood": 21659, "ĠFlint": 21660, "ĠJama": 21661, "ĠPresent": 21662, "ĠRage": 21663, "ĠTA": 21664, "ptive": 21665, "Ġturnout": 21666, "wald": 21667, "ĠDolphins": 21668, "ĠVPN": 21669, "Ġonion": 21670, "Ġcrafting": 21671, "mma": 21672, "ĠMercury": 21673, "Ġarrange": 21674, "Ġalerts": 21675, "ĠOT": 21676, "zbollah": 21677, "Ġgases": 21678, "ĠRichardson": 21679, "sal": 21680, "lar": 21681, "Ġfrost": 21682, "Ġlowering": 21683, "Ġacclaim": 21684, "Ġstartups": 21685, "ĠGain": 21686, "essment": 21687, "Ġguardian": 21688, "人": 21689, "ĠPie": 21690, "ĠLinks": 21691, "Ġmerits": 21692, "Ġawake": 21693, "Ġparental": 21694, "Ġexceeds": 21695, "Ġidle": 21696, "ĠPilot": 21697, "ĠeBay": 21698, "ĠAccept": 21699, "ipeg": 21700, "Cam": 21701, "ĠKot": 21702, "Ġtraders": 21703, "olitics": 21704, "unker": 21705, "ĠPale": 21706, "osi": 21707, "anmar": 21708, "Ġ1947": 21709, "ĠFell": 21710, "estial": 21711, "itating": 21712, "GF": 21713, "ĠSr": 21714, "ifted": 21715, "Ġconnector": 21716, "ĠBone": 21717, "illes": 21718, "260": 21719, "hma": 21720, "Ġoverlap": 21721, "ĠGitHub": 21722, "Ġcleaner": 21723, "ĠBaptist": 21724, "ĠWAS": 21725, "Ġlungs": 21726, "Ñģ": 21727, "ĠBUT": 21728, "Ġcite": 21729, "Ġpitched": 21730, "reatment": 21731, "Ġtrophies": 21732, "ĠNu": 21733, "386": 21734, "ĠPride": 21735, "Ġattendees": 21736, "[]": 21737, "179": 21738, "Ġspatial": 21739, "Ġprizes": 21740, "ĠReligion": 21741, "Ġshowcase": 21742, "ĠCategory": 21743, "vidia": 21744, "Target": 21745, "Property": 21746, "?,": 21747, "Ġfusion": 21748, "pie": 21749, "ĠUCLA": 21750, "Ġsoundtrack": 21751, "Ġprincess": 21752, "ĠCaval": 21753, "should": 21754, "Ġlimbs": 21755, "Background": 21756, "Ġlonely": 21757, "Ġcores": 21758, "ĠTail": 21759, "sheet": 21760, "Ġ132": 21761, "Ra": 21762, "ãĤ«": 21763, "ĠBolt": 21764, "Ġbooked": 21765, "Ġadminister": 21766, "Ġequals": 21767, "wy": 21768, "Ġobserving": 21769, "ĠBaron": 21770, "ĠAdobe": 21771, "Ġvirgin": 21772, "ĠSocialist": 21773, "Move": 21774, "ghazi": 21775, "ĠLinda": 21776, "212": 21777, "Ġbrewing": 21778, "Ġmerchants": 21779, "burse": 21780, "Ġdivor": 21781, "Ġmetals": 21782, "ĠNer": 21783, "Ġsums": 21784, "ĠEnemy": 21785, "Ġenvision": 21786, "Ġgranting": 21787, "ĠHoney": 21788, "ĠSkyrim": 21789, "Ġsocio": 21790, "graded": 21791, "Ġselective": 21792, "WASHINGTON": 21793, "Ġ1948": 21794, "ĠSirius": 21795, "ĠGross": 21796, "activity": 21797, "ĠIvan": 21798, "Ġfurious": 21799, "BSD": 21800, "ĠPrevious": 21801, "Ġresponsive": 21802, "Ġcharitable": 21803, "Ġleaning": 21804, "ĠPew": 21805, "Ġviolates": 21806, "\\\\\\\\\\\\\\\\": 21807, "ĠComing": 21808, "wire": 21809, "Ġpoet": 21810, "Ġresolutions": 21811, "command": 21812, "ĠPortuguese": 21813, "Ġnickname": 21814, "Ġdeaf": 21815, "February": 21816, "Ġrecognise": 21817, "Ġentirety": 21818, "Ġseasonal": 21819, "placed": 21820, "ĠTelegraph": 21821, "Ġmicrophone": 21822, "ouring": 21823, "Ġgrains": 21824, "Ġgoverned": 21825, "Ġpostp": 21826, "ĠWaters": 21827, "inement": 21828, "Ġundocumented": 21829, "ĠComcast": 21830, "Ġfox": 21831, "Ġassaults": 21832, "reon": 21833, "many": 21834, "ĠJenkins": 21835, "ĠAnyway": 21836, "Ġassessments": 21837, "Ġdowns": 21838, "ĠMouse": 21839, "Ġsuperb": 21840, "kt": 21841, "ĠDow": 21842, "Ġtaxation": 21843, "401": 21844, "Ġsmiles": 21845, "Ġundertaken": 21846, "Ġexh": 21847, "Ġenthusiastic": 21848, "Ġtwent": 21849, "Ġgovernmental": 21850, "Ġautonomy": 21851, "ĠTechnologies": 21852, "ĠChain": 21853, "Ġprevalent": 21854, "fb": 21855, "Ġnicotine": 21856, "ogram": 21857, "job": 21858, "Ġawaiting": 21859, "ĠMenu": 21860, "Ġdeputies": 21861, "kov": 21862, "ishops": 21863, "Button": 21864, "ĠShanghai": 21865, "Ġdiesel": 21866, "ĠDuck": 21867, "Ryan": 21868, "ĠPCs": 21869, "NF": 21870, "jury": 21871, "ente": 21872, "Ġinaccurate": 21873, "eddy": 21874, "Whatever": 21875, "Ġshowc": 21876, "ĠNad": 21877, "odus": 21878, "etr": 21879, "Ġplaintiffs": 21880, "ĠWOR": 21881, "ĠAssange": 21882, "Ġprivat": 21883, "Ġpremiums": 21884, "Ġtam": 21885, "URL": 21886, "Ġelites": 21887, "ĠRanger": 21888, "ottenham": 21889, "ĠHoff": 21890, "ĠAthens": 21891, "Ġdefinite": 21892, "Ġsighed": 21893, "Ġevenly": 21894, "211": 21895, "ĠAmber": 21896, "akia": 21897, "Ġmailing": 21898, "Ġcrashing": 21899, "ĠConfederate": 21900, "rugged": 21901, "Wal": 21902, "ĠDepths": 21903, "Ġjuvenile": 21904, "Ġreactor": 21905, "Introduction": 21906, "ĠDeluxe": 21907, "1995": 21908, "ĠSanchez": 21909, "ĠMead": 21910, "ivable": 21911, ":-": 21912, "ĠPlanning": 21913, "ĠTrap": 21914, "quin": 21915, "ĠProtect": 21916, "vered": 21917, "Information": 21918, "Ġkidney": 21919, "innamon": 21920, "las": 21921, "Ġpolicing": 21922, "Ġtolerate": 21923, "ĠQi": 21924, "Ġbiased": 21925, "Fort": 21926, "ĠKi": 21927, "save": 21928, "Ġprivileged": 21929, "Ġbeasts": 21930, "ĠGlas": 21931, "ĠCinem": 21932, "Ġcomeback": 21933, "Sunday": 21934, "Ġextinction": 21935, "hops": 21936, "Ġtransmit": 21937, "Ġdoubles": 21938, "ĠFlat": 21939, "167": 21940, "Ġdisputed": 21941, "Ġinjustice": 21942, "foo": 21943, "Vict": 21944, "roleum": 21945, "ĠJulie": 21946, "Context": 21947, "ĠRarity": 21948, "issue": 21949, "Component": 21950, "Ġcounseling": 21951, "anne": 21952, "dark": 21953, "Ġobjections": 21954, "uilt": 21955, "Ġgast": 21956, "Ġplac": 21957, "Ġunused": 21958, "ãĥĩ": 21959, "ĠTrial": 21960, "ĠJas": 21961, "hedral": 21962, "obb": 21963, "Ġtemporal": 21964, "ĠPRO": 21965, "ĠNW": 21966, "ĠAnniversary": 21967, "Large": 21968, "Ġtherm": 21969, "Ġdavid": 21970, "Ġsystemic": 21971, "ĠShir": 21972, "mut": 21973, "ĠNept": 21974, "address": 21975, "Ġscanning": 21976, "Ġunderstandable": 21977, "Ġcanvas": 21978, "Cat": 21979, "ĠZoo": 21980, "Ġangels": 21981, "LO": 21982, "ĠStatement": 21983, "ĠSig": 21984, "ovable": 21985, "ĠAway": 21986, "sharing": 21987, "ocrats": 21988, "stated": 21989, "Ġweighing": 21990, "Nor": 21991, "wild": 21992, "Bey": 21993, "Ġastonishing": 21994, "ĠReynolds": 21995, "Ġopener": 21996, "Ġtrainer": 21997, "Ġsurgical": 21998, "pn": 21999, "Ġadjusting": 22000, "wheel": 22001, "Ġfrown": 22002, "ervative": 22003, "Ġsuspend": 22004, "Within": 22005, "tein": 22006, "Ġobstacle": 22007, "Ġliberties": 22008, "ymes": 22009, "Ġuranium": 22010, "ansom": 22011, "anol": 22012, "uba": 22013, "ĠLoss": 22014, "Ġarous": 22015, "ĠHenderson": 22016, "Wow": 22017, "spl": 22018, "cur": 22019, "ĠÂŃ": 22020, "Ġtheirs": 22021, "Damage": 22022, "Ġdownloading": 22023, "Ġdiscern": 22024, "ĠSto": 22025, "ĠFla": 22026, "Ġhath": 22027, "ĠAj": 22028, "Ġunpleasant": 22029, "European": 22030, "expensive": 22031, "Ġscreenshot": 22032, "ĠUV": 22033, "Ġallied": 22034, "ĠPersian": 22035, "Ġmonopoly": 22036, "Ġatom": 22037, "ĠRedskins": 22038, "\"><": 22039, "Ġcancell": 22040, "Ġcinema": 22041, "131": 22042, "fair": 22043, "ĠAlfred": 22044, "Ġduck": 22045, "args": 22046, "223": 22047, "ĠISI": 22048, "Ġsignaling": 22049, "inar": 22050, "Ġlaughs": 22051, "Ġforwards": 22052, "Ġreckless": 22053, "Ġlisteners": 22054, "ativity": 22055, "Ġvastly": 22056, "nant": 22057, "Less": 22058, "ĠHunting": 22059, "ĠScientific": 22060, "ITED": 22061, "Ġknight": 22062, "ĠHTC": 22063, "usa": 22064, "tmp": 22065, "Ġrude": 22066, "ĠLegendary": 22067, "Ġarises": 22068, "Bad": 22069, "ĠClaim": 22070, "peg": 22071, "Ġrealities": 22072, "Think": 22073, "Ġ°": 22074, "Ġrode": 22075, "Ġstrive": 22076, "Ġanecd": 22077, "Ġshorts": 22078, "Ġhypothes": 22079, "Ġcoordinated": 22080, "ĠGandhi": 22081, "ĠFPS": 22082, "RED": 22083, "Ġsusceptible": 22084, "Ġshrink": 22085, "ĠChart": 22086, "Help": 22087, "Ġion": 22088, "deep": 22089, "ribes": 22090, "ĠKai": 22091, "ĠCustomer": 22092, "Summary": 22093, "Ġcough": 22094, "wife": 22095, "Ġlend": 22096, "Ġpositioning": 22097, "Ġlottery": 22098, "ĠCanyon": 22099, "Ġfade": 22100, "Ġbronze": 22101, "ĠKenny": 22102, "Ġboasts": 22103, "ĠEnhanced": 22104, "record": 22105, "Ġemergence": 22106, "Ġakin": 22107, "ĠBert": 22108, "itous": 22109, "âĸij": 22110, "Ġstip": 22111, "Ġexchanged": 22112, "omore": 22113, "alsh": 22114, "Ġreservoir": 22115, "Ġstandpoint": 22116, "WM": 22117, "Ġinitiate": 22118, "Ġdecay": 22119, "Ġbrewery": 22120, "Ġterribly": 22121, "Ġmortal": 22122, "levard": 22123, "Ġrevis": 22124, "NI": 22125, "elo": 22126, "Ġconfess": 22127, "ĠMSNBC": 22128, "Ġsubmissions": 22129, "Controller": 22130, "Ġ202": 22131, "ĠRuth": 22132, "});": 22133, "ĠAzure": 22134, "Ġ.\"": 22135, "206": 22136, "ĠMarketing": 22137, "Ġlaund": 22138, "iencies": 22139, "Ġrenowned": 22140, "ĠTrou": 22141, "ĠNGO": 22142, "blems": 22143, "Ġterrified": 22144, "Ġwarns": 22145, "Ġpert": 22146, "Ġunsure": 22147, "480": 22148, "alez": 22149, "ultz": 22150, "ĠOutside": 22151, "Ġstyl": 22152, "ĠUnderground": 22153, "Ġpanc": 22154, "Ġdictionary": 22155, "Ġfoe": 22156, "riminal": 22157, "ĠNorwegian": 22158, "Ġjailed": 22159, "Ġmaternal": 22160, "ée": 22161, "ĠLucy": 22162, "cop": 22163, "Cho": 22164, "Ġunsigned": 22165, "ĠZelda": 22166, "ĠInsider": 22167, "ĠContinued": 22168, "Ġ133": 22169, "ĠNaruto": 22170, "ĠMajority": 22171, "169": 22172, "ĠWo": 22173, "ãĤĵ": 22174, "Ġpastor": 22175, "Ġinformal": 22176, "н": 22177, "anthrop": 22178, "join": 22179, "ãģĹ": 22180, "itational": 22181, "NP": 22182, "ĠWriting": 22183, "fn": 22184, "ĠBever": 22185, "195": 22186, "Ġyelling": 22187, "Ġdrastically": 22188, "Ġeject": 22189, "Ġneut": 22190, "Ġthrive": 22191, "ĠFrequ": 22192, "oux": 22193, "Ġpossesses": 22194, "ĠSenators": 22195, "ĠDES": 22196, "ĠShakespeare": 22197, "ĠFranco": 22198, "ĠLB": 22199, "uchi": 22200, "Ġincarn": 22201, "Ġfounders": 22202, "Function": 22203, "Ġbrightness": 22204, "ĠBT": 22205, "Ġwhale": 22206, "ĠTheater": 22207, "mass": 22208, "ĠDoll": 22209, "Something": 22210, "Ġechoed": 22211, "ĠHex": 22212, "crit": 22213, "afia": 22214, "Ġgoddess": 22215, "Ġeleven": 22216, "ĠPreview": 22217, "ĠAurora": 22218, "Ġ401": 22219, "ulsive": 22220, "ĠLogan": 22221, "inburgh": 22222, "ĠCenters": 22223, "ĠONLY": 22224, "ĠAid": 22225, "Ġparadox": 22226, "Ġhurd": 22227, "ĠLC": 22228, "Due": 22229, "court": 22230, "Ġoffended": 22231, "Ġevaluating": 22232, "ĠMatthews": 22233, "Ġtomb": 22234, "Ġpayroll": 22235, "Ġextraction": 22236, "ĠHands": 22237, "ifi": 22238, "Ġsupernatural": 22239, "ĠCOMM": 22240, "]=": 22241, "dogs": 22242, "Ġ512": 22243, "ĠMeeting": 22244, "Richard": 22245, "ĠMaximum": 22246, "Ġideals": 22247, "Things": 22248, "mand": 22249, "ĠRegardless": 22250, "Ġhumili": 22251, "buffer": 22252, "Little": 22253, "ĠDani": 22254, "ĠNak": 22255, "Ġliberation": 22256, "ĠAbe": 22257, "ĠOL": 22258, "Ġstuffed": 22259, "aca": 22260, "inda": 22261, "raphic": 22262, "Ġmosqu": 22263, "Ġcampaigning": 22264, "Ġoccupy": 22265, "Squ": 22266, "rina": 22267, "ĠWel": 22268, "ĠVS": 22269, "Ġphysic": 22270, "Ġpuls": 22271, "rint": 22272, "oaded": 22273, "ETF": 22274, "ĠArchives": 22275, "Ġvenues": 22276, "hner": 22277, "ĠTurbo": 22278, "Ġlust": 22279, "Ġappealed": 22280, "quez": 22281, "ilib": 22282, "ĠTimothy": 22283, "Ġomn": 22284, "dro": 22285, "Ġobsession": 22286, "ĠSavage": 22287, "1996": 22288, "Global": 22289, "Jes": 22290, "214": 22291, "Ġsliding": 22292, "Ġdisappro": 22293, "ĠMagical": 22294, "Ġvoluntarily": 22295, "gb": 22296, "aney": 22297, "Ġprophet": 22298, "ĠRein": 22299, "ĠJulia": 22300, "ĠWorth": 22301, "aurus": 22302, "Ġbounds": 22303, "ieu": 22304, ")))": 22305, "Ġcrore": 22306, "ĠCitizen": 22307, "Sky": 22308, "Ġcolumnist": 22309, "Ġseekers": 22310, "ondo": 22311, "ISA": 22312, "ĠLength": 22313, "Ġnostalg": 22314, "Ġnewcom": 22315, "Ġdetrim": 22316, "entric": 22317, "375": 22318, "ĠGE": 22319, "Ġautop": 22320, "Ġacademics": 22321, "AppData": 22322, "ĠShen": 22323, "Ġidiot": 22324, "ĠTransit": 22325, "Ġteaspoon": 22326, "Wil": 22327, "KO": 22328, "ĠComedy": 22329, ">,": 22330, "Ġpopulated": 22331, "WD": 22332, "Ġpigs": 22333, "ĠOculus": 22334, "Ġsympathetic": 22335, "Ġmarathon": 22336, "198": 22337, "Ġseizure": 22338, "sided": 22339, "Ġdop": 22340, "irtual": 22341, "Land": 22342, "ĠFloor": 22343, "osaurs": 22344, "...]": 22345, "Ġlos": 22346, "Ġsubsidiary": 22347, "EY": 22348, "ĠParts": 22349, "ĠStef": 22350, "ĠJudiciary": 22351, "Ġ134": 22352, "Ġmirrors": 22353, "Ġket": 22354, "times": 22355, "Ġneurolog": 22356, "Ġcav": 22357, "ĠGuest": 22358, "Ġtumor": 22359, "scill": 22360, "ĠLloyd": 22361, "Est": 22362, "Ġclearer": 22363, "Ġstereotypes": 22364, "Ġdur": 22365, "nothing": 22366, "Reddit": 22367, "Ġnegotiated": 22368, "------------------------": 22369, "235": 22370, "Ġflown": 22371, "ĠSeoul": 22372, "ĠResident": 22373, "ĠSCH": 22374, "Ġdisappearance": 22375, "ĠVince": 22376, "grown": 22377, "Ġgrabs": 22378, "ril": 22379, "ĠInfinite": 22380, "ĠTwenty": 22381, "Ġpedestrian": 22382, "Ġjersey": 22383, "ĠFur": 22384, "ĠInfinity": 22385, "ĠElliott": 22386, "Ġmentor": 22387, "Ġmorally": 22388, "Ġobey": 22389, "secure": 22390, "iffe": 22391, "Ġantibiotics": 22392, "angled": 22393, "ĠFreeman": 22394, "ĠIntroduction": 22395, "Jun": 22396, "Ġmarsh": 22397, "icans": 22398, "ĠEVENTS": 22399, "ochond": 22400, "Wall": 22401, "iculty": 22402, "Ġmisdemeanor": 22403, "Ġly": 22404, "Thomas": 22405, "ĠResolution": 22406, "Ġanimations": 22407, "ĠDry": 22408, "Ġintercourse": 22409, "ĠNewcastle": 22410, "ĠHog": 22411, "ĠEquipment": 22412, "177": 22413, "Ġterritorial": 22414, "Ġarchives": 22415, "203": 22416, "Filter": 22417, "ĠMunich": 22418, "Ġcommanded": 22419, "ĠWand": 22420, "Ġpitches": 22421, "ĠCroat": 22422, "Ġratios": 22423, "ĠMits": 22424, "Ġaccumulated": 22425, "ĠSpecifically": 22426, "Ġgentleman": 22427, "acerb": 22428, "Ġpenn": 22429, "Ġaka": 22430, "ĠFuk": 22431, "Ġintervene": 22432, "ĠRefuge": 22433, "ĠAlzheimer": 22434, "Ġsuccession": 22435, "ohan": 22436, "does": 22437, "Lord": 22438, "Ġseparat": 22439, "Ġcorrespondence": 22440, "Ġshiny": 22441, "Prior": 22442, "Ġsulf": 22443, "Ġmiserable": 22444, "Ġdedication": 22445, "().": 22446, "Ġspecialists": 22447, "Ġdefects": 22448, "ĠCult": 22449, "ĠXia": 22450, "Ġjeopard": 22451, "ĠOre": 22452, "Ability": 22453, "Ġlear": 22454, "Ġambitions": 22455, "ĠBMI": 22456, "ĠArabs": 22457, "Ġ1942": 22458, "Ġpreservation": 22459, "ificate": 22460, "Ġashamed": 22461, "loss": 22462, "ĠRestaur": 22463, "Ġresemble": 22464, "Ġenrich": 22465, "ĠKN": 22466, "ĠClan": 22467, "float": 22468, "Ġplayable": 22469, "ITT": 22470, "Ġharmony": 22471, "arrison": 22472, "ĠWeinstein": 22473, "were": 22474, "Ġpoisoning": 22475, "ĠComput": 22476, "ĠWordPress": 22477, "major": 22478, "ĠValve": 22479, "Fan": 22480, "ĠThrow": 22481, "ĠRomans": 22482, "ĠDepression": 22483, "ados": 22484, "Ġtortured": 22485, "Ġbalancing": 22486, "bottom": 22487, "Ġacquiring": 22488, "ĠMonte": 22489, "ardi": 22490, "Ġaura": 22491, "Ġ##": 22492, "ĠStanding": 22493, "ĠAtlas": 22494, "CF": 22495, "Ġintrins": 22496, "ĠBenghazi": 22497, "Ġcamping": 22498, "Ġtapped": 22499, "blade": 22500, "strous": 22501, "ĠRabb": 22502, "ĠWritten": 22503, "tip": 22504, "ĠNeigh": 22505, "sterdam": 22506, "ĠAllow": 22507, "ĠHealing": 22508, "ĠRhod": 22509, "num": 22510, "Ġcaffeine": 22511, "ĠPercent": 22512, "Ġboo": 22513, "Ġapples": 22514, "305": 22515, "Ġwelcoming": 22516, "Ġapplaud": 22517, "Ġausterity": 22518, "±": 22519, "ĠReality": 22520, "efe": 22521, "å®": 22522, "Ġsucks": 22523, "Ġtabs": 22524, "ĠPayPal": 22525, "Ġbackpack": 22526, "Ġgifted": 22527, "abulary": 22528, "ĠScout": 22529, "irteen": 22530, "Ġchin": 22531, "Ġomitted": 22532, "Ġnegatively": 22533, "Ġaccessing": 22534, "ĠEarn": 22535, "Ġambulance": 22536, "Ġheadphones": 22537, "Ġ205": 22538, "ĠRefresh": 22539, "president": 22540, "ĠKitchen": 22541, "ĠEntered": 22542, "ĠSnyder": 22543, "005": 22544, "omical": 22545, "Ġborrowed": 22546, "ĠNem": 22547, "Ġaviation": 22548, "Ġstall": 22549, "rimination": 22550, "Ġuniforms": 22551, "itime": 22552, "ĠSimmons": 22553, "energy": 22554, "ablished": 22555, "yy": 22556, "qualified": 22557, "Ġrallies": 22558, "ĠStuart": 22559, "flight": 22560, "Ġgangs": 22561, "rag": 22562, "Ġvault": 22563, "lux": 22564, "ĠCompar": 22565, "Ġdesignation": 22566, "209": 22567, "ĠJos": 22568, "dollar": 22569, "zero": 22570, "Ġwells": 22571, "303": 22572, "Ġconstituents": 22573, "Ġheck": 22574, "Ġcows": 22575, "Ġcommanders": 22576, "Ġdifferential": 22577, "ĠCatherine": 22578, "299": 22579, "Ġvalve": 22580, "Ġbrace": 22581, "Ġperspectives": 22582, "cert": 22583, "fact": 22584, "icularly": 22585, "ĠMcN": 22586, "planes": 22587, "Ġintric": 22588, "Ġpeas": 22589, "ovan": 22590, "Ġtossed": 22591, "retch": 22592, "ĠLopez": 22593, "Ġunfamiliar": 22594, "death": 22595, "ĠApart": 22596, "ĠChang": 22597, "Ġrelieved": 22598, "rophe": 22599, "Ġairports": 22600, "Ġfreak": 22601, "util": 22602, "Mill": 22603, "ĠChin": 22604, "ĠOwen": 22605, "male": 22606, "ĠBroken": 22607, "ĠWinds": 22608, "rob": 22609, "rising": 22610, "Ġfirefighters": 22611, "Ġauthoritarian": 22612, "Ġ148": 22613, "Bitcoin": 22614, "external": 22615, "Ġbrowsers": 22616, "ichever": 22617, "orian": 22618, "Ġunb": 22619, "Ġpoke": 22620, "ĠZot": 22621, "Mid": 22622, "ĠPopular": 22623, "Ġcovert": 22624, "Ġcontributes": 22625, "Ġ650": 22626, "Ġcontention": 22627, "Gate": 22628, "Ġconsoles": 22629, "Ġchromos": 22630, "ĠIX": 22631, "Ġvisually": 22632, "ĠEisen": 22633, "Ġjewelry": 22634, "Ġdelegation": 22635, "Ġaccelerate": 22636, "ĠRiley": 22637, "Ġslope": 22638, "Ġindoor": 22639, "itially": 22640, "Ġhugely": 22641, "Ġtunnels": 22642, "Ġfined": 22643, "Ġdirective": 22644, "Ġforehead": 22645, "ustomed": 22646, "Ġskate": 22647, "Music": 22648, "gas": 22649, "Ġrecognizing": 22650, "ambo": 22651, "Ġoverweight": 22652, "ĠGrade": 22653, "ÙĬ": 22654, "Ġsounding": 22655, "Ġlocking": 22656, "ĠREM": 22657, "Store": 22658, "Ġexcav": 22659, "ĠLikewise": 22660, "ĠLights": 22661, "Ġelbow": 22662, "ĠSupply": 22663, "wic": 22664, "Ġhandsome": 22665, "1994": 22666, "Coll": 22667, "Ġadequately": 22668, "ĠAssociate": 22669, "Ġstrips": 22670, "Ġcrackdown": 22671, "Ġmarvel": 22672, "ĠKun": 22673, "Ġpassages": 22674, "@@@@": 22675, "ĠTall": 22676, "Ġthoughtful": 22677, "namese": 22678, "Ġprostitution": 22679, "business": 22680, "Ġballistic": 22681, "personal": 22682, "cig": 22683, "izational": 22684, "Round": 22685, "ĠÂłĠÂłĠÂłĠÂł": 22686, "ĠColeman": 22687, "Ġadmitting": 22688, "ĠPlug": 22689, "Ġbitcoins": 22690, "ĠSuz": 22691, "Ġfairness": 22692, "Ġsupplier": 22693, "Ġcatastrophic": 22694, "ĠHelen": 22695, "oqu": 22696, "Marc": 22697, "ĠArticles": 22698, "gie": 22699, "Ġendangered": 22700, "Ġdestiny": 22701, "ĠVolt": 22702, "olia": 22703, "axis": 22704, "Ġcheat": 22705, "Ġunified": 22706, "ICO": 22707, "quote": 22708, "302": 22709, "ĠSed": 22710, "Ġsuppression": 22711, "Ġanalyzing": 22712, "Ġsquat": 22713, "Ġfiguring": 22714, "Ġcoordinates": 22715, "Ġchunks": 22716, "Ġ1946": 22717, "Ġsubp": 22718, "Ġwiki": 22719, "ĠForbes": 22720, "ĠJupiter": 22721, "ĠErik": 22722, "imer": 22723, "ĠCommercial": 22724, "\\)": 22725, "Ġlegitimacy": 22726, "Ġdental": 22727, "ĠMean": 22728, "Ġdeficits": 22729, "550": 22730, "Originally": 22731, "ĠHorror": 22732, "Ġcontamination": 22733, "llah": 22734, "Ġconfisc": 22735, "ĠClare": 22736, "TB": 22737, "ĠFailed": 22738, "aned": 22739, "Ġruler": 22740, "ĠController": 22741, "Ġfeminists": 22742, "Fix": 22743, "gay": 22744, "207": 22745, "Ġrabbit": 22746, "Third": 22747, "owntown": 22748, "Ġglue": 22749, "Ġvolatile": 22750, "Ġshining": 22751, "Ġfoll": 22752, "Ġimpaired": 22753, "Ġsupers": 22754, "æĪ": 22755, "Ġclutch": 22756, "ļéĨĴ": 22757, "Ġprolet": 22758, "Ġ(!": 22759, "Ġyelled": 22760, "ĠKiev": 22761, "ĠErn": 22762, "ĠShock": 22763, "KB": 22764, "Ġsituated": 22765, "query": 22766, "ĠNas": 22767, "Ġannex": 22768, "character": 22769, "ĠHoliday": 22770, "Ġautomation": 22771, "ĠJill": 22772, "ĠRemastered": 22773, "Ġlinem": 22774, "Ġwilderness": 22775, "ĠHorizon": 22776, "ĠGuinea": 22777, "AZ": 22778, "Ġmainland": 22779, "Ġsecrecy": 22780, "LEASE": 22781, "Ġpunk": 22782, "ĠProvince": 22783, "(),": 22784, "Speed": 22785, "Ġhanding": 22786, "ĠSebast": 22787, "Sir": 22788, "rase": 22789, "Ġjournals": 22790, "Ġcongest": 22791, "ĠTut": 22792, "irrel": 22793, "Ġschizophrenia": 22794, "Ġmisogyn": 22795, "healthy": 22796, "Iron": 22797, "Ġreacted": 22798, "-$": 22799, "252": 22800, "Ġplural": 22801, "Ġplum": 22802, "Ġbargain": 22803, "Ġgrounded": 22804, "finder": 22805, "Ġdisse": 22806, "ĠLaz": 22807, "OOD": 22808, "Ġatroc": 22809, "Factory": 22810, "Ġminions": 22811, "Ġori": 22812, "ĠBrave": 22813, "ĠPRE": 22814, "ĠMyanmar": 22815, "ĠHod": 22816, "Ġexpedition": 22817, "Ġexplode": 22818, "ĠCoord": 22819, "Ġextr": 22820, "ĠBrief": 22821, "ĠADHD": 22822, "Ġhardcore": 22823, "feeding": 22824, "Ġdile": 22825, "ĠFruit": 22826, "Ġvaccination": 22827, "ĠMao": 22828, "osphere": 22829, "Ġcontests": 22830, "-|": 22831, "Ġfren": 22832, "isphere": 22833, "Rom": 22834, "ĠSharp": 22835, "ĠTrend": 22836, "Ġdisconnect": 22837, "âĢ¢âĢ¢": 22838, "Ġpersecution": 22839, "Earth": 22840, "Ġhealthier": 22841, "384": 22842, "Ġcob": 22843, "ĠTrinity": 22844, "OWS": 22845, "ANN": 22846, "Ġspecialty": 22847, "Ġgru": 22848, "Ġcooperative": 22849, "why": 22850, "Starting": 22851, "ĠIssues": 22852, "stre": 22853, "ensor": 22854, "Ġ185": 22855, "Adv": 22856, "!?": 22857, "ĠRevel": 22858, "emia": 22859, "ĠHulk": 22860, "Ġcelebrations": 22861, "ĠSou": 22862, "raud": 22863, "ĠKlein": 22864, "Ġunreal": 22865, "context": 22866, "Ġpartnerships": 22867, "Ġadopting": 22868, "tical": 22869, "Ġsplash": 22870, "ĠHezbollah": 22871, "category": 22872, "cyclop": 22873, "xton": 22874, "ĠDot": 22875, "urdy": 22876, "tz": 22877, "Ġenvelope": 22878, "ĠNL": 22879, "âķ": 22880, "Ġwherein": 22881, "Spec": 22882, "184": 22883, "Ġtelev": 22884, "aliation": 22885, "Ġmyths": 22886, "å°": 22887, "Ġrigorous": 22888, "Ġcommunicating": 22889, "Ġobserver": 22890, "Ġrehe": 22891, "ĠWash": 22892, "Ġapologized": 22893, "ĠTin": 22894, "Ġexpenditures": 22895, "workers": 22896, "document": 22897, "Ġhesitate": 22898, "ĠLenin": 22899, "Ġunpredictable": 22900, "Ġrenewal": 22901, "cler": 22902, "okia": 22903, "ĠCONT": 22904, "Ġpostseason": 22905, "Tokens": 22906, "Ġexacerb": 22907, "Ġbetting": 22908, "Ġ147": 22909, "Ġelevation": 22910, "Wood": 22911, "ĠSolomon": 22912, "194": 22913, "004": 22914, "output": 22915, "Ġredund": 22916, "ĠMumbai": 22917, "ĠpH": 22918, "Ġreproduce": 22919, "ĠDuration": 22920, "MAX": 22921, "Ġbog": 22922, "CBS": 22923, "ĠBalance": 22924, "ĠSgt": 22925, "ĠRecent": 22926, "Ġcd": 22927, "Ġpopped": 22928, "Ġincompet": 22929, "prop": 22930, "ayan": 22931, "guy": 22932, "Pacific": 22933, "Ġtyr": 22934, "Ġ{{": 22935, "ĠMystic": 22936, "ĠDana": 22937, "Ġmasturb": 22938, "Ġgeometry": 22939, "â": 22940, "ĠCorrect": 22941, "Ġtrajectory": 22942, "Ġdistracted": 22943, "Ġfoo": 22944, "ĠWelsh": 22945, "Luc": 22946, "mith": 22947, "Ġrugby": 22948, "Ġrespiratory": 22949, "Ġtriangle": 22950, "Ġ215": 22951, "Ġundergraduate": 22952, "ĠSuperior": 22953, "changing": 22954, "_-": 22955, "Ġrightly": 22956, "Ġreferee": 22957, "Ġlucrative": 22958, "Ġunauthorized": 22959, "Ġresembles": 22960, "ĠGNU": 22961, "ĠDerby": 22962, "Ġpathways": 22963, "ĠLed": 22964, "Ġendurance": 22965, "Ġstint": 22966, "Ġcollector": 22967, "Fast": 22968, "Ġdots": 22969, "Ġnationals": 22970, "ĠSecurities": 22971, "Ġwhip": 22972, "Param": 22973, "Ġlearns": 22974, "Magic": 22975, "Ġdetailing": 22976, "moon": 22977, "Ġbroadcasting": 22978, "Ġbaked": 22979, "265": 22980, "holm": 22981, "ĠSah": 22982, "ĠHussein": 22983, "ĠCourtesy": 22984, "174": 22985, "Ġ146": 22986, "Ġgeographic": 22987, "peace": 22988, "Ġjudging": 22989, "ĠStern": 22990, "Bur": 22991, "Ġstoryline": 22992, "Gun": 22993, "ĠStick": 22994, "245": 22995, "307": 22996, "ãĤ´ãĥ³": 22997, "ĠAdministrator": 22998, "Ġburnt": 22999, "Ġpave": 23000, "choes": 23001, "Exec": 23002, "Ġcampuses": 23003, "Result": 23004, "Ġmutations": 23005, "ĠCharter": 23006, "Ġcaptures": 23007, "Ġcompares": 23008, "Ġbadge": 23009, "Scient": 23010, "Ġerad": 23011, "iery": 23012, "oi": 23013, "ettes": 23014, "ĠEstate": 23015, "Ġstrap": 23016, "Ġproudly": 23017, "Ġfried": 23018, "Ġwithdrawn": 23019, "ĠVoy": 23020, "phony": 23021, "Items": 23022, "ĠPierce": 23023, "bard": 23024, "Ġannotation": 23025, "anton": 23026, "illon": 23027, "Impro": 23028, "...)": 23029, "Ġhappier": 23030, "------": 23031, "adjust": 23032, "Ġstaffers": 23033, "Ġactivism": 23034, "Ġperf": 23035, "Ġalright": 23036, "Need": 23037, "Ġcommence": 23038, "Ġopioid": 23039, "ĠAmanda": 23040, "Es": 23041, "ĠPars": 23042, "ĠKaw": 23043, "Works": 23044, "248": 23045, "Ġindo": 23046, "tc": 23047, "endant": 23048, "ĠMoto": 23049, "Ġlegalization": 23050, "OTE": 23051, "Ġtasked": 23052, "Ġtsp": 23053, "ĠACTIONS": 23054, "166": 23055, "Ġrefreshing": 23056, "ĠNR": 23057, "ĠPerez": 23058, "Ġinfringement": 23059, "SY": 23060, "Listen": 23061, "inning": 23062, "ku": 23063, "Ġrotate": 23064, "program": 23065, "arah": 23066, "Design": 23067, "Ġ(£": 23068, "Ġstoring": 23069, "Ġwarrants": 23070, "Ġjudgement": 23071, "ĠBrist": 23072, "usually": 23073, "photo": 23074, "ĠRan": 23075, "ĠPine": 23076, "Ġoutrageous": 23077, "ĠValentine": 23078, "luence": 23079, "ĠEverybody": 23080, "Altern": 23081, "Ġrelevance": 23082, "Ġterminated": 23083, "Ġdessert": 23084, "Ġfulfilled": 23085, "Ġprosecuted": 23086, "ĠWords": 23087, "Ġmigrant": 23088, "Ġcultivation": 23089, "ÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤ": 23090, "idelity": 23091, "ĠVern": 23092, "ĠLogin": 23093, "Ġmetaphor": 23094, "ĠTip": 23095, "Ġrecruits": 23096, "ĠPig": 23097, "ribing": 23098, "Ġenthusiasts": 23099, "exper": 23100, "Ġfrightening": 23101, "ĠHair": 23102, "anson": 23103, "strate": 23104, "Ġhi": 23105, "Height": 23106, "Ġowning": 23107, "none": 23108, "Ġdislike": 23109, "Ġknives": 23110, "pherd": 23111, "Ġloudly": 23112, "ĠAPIs": 23113, "Display": 23114, "ĠLac": 23115, "ĠUSS": 23116, "abl": 23117, "verages": 23118, "Jew": 23119, "Ġ172": 23120, "ĠHistorical": 23121, "atoon": 23122, "ĠPhysics": 23123, "intern": 23124, "Ġwarmth": 23125, "Ġtopp": 23126, "DM": 23127, "Ġgunman": 23128, "Ġemperor": 23129, "odi": 23130, "ãĥ£": 23131, "inatory": 23132, "ĠRib": 23133, "Ġ131": 23134, "ĠSaturn": 23135, "ĠShining": 23136, "Ġwaking": 23137, "Quotes": 23138, "Ġcomedian": 23139, "enberg": 23140, "½": 23141, "Ġbelievers": 23142, "Ġpaperwork": 23143, "custom": 23144, "Ġlev": 23145, "Ġlament": 23146, "Ġpouring": 23147, "222": 23148, "political": 23149, "ĠSupplement": 23150, "maid": 23151, "Ġcruelty": 23152, "Ġtread": 23153, "ysics": 23154, "Aw": 23155, "rites": 23156, "Ġmodifier": 23157, "ĠPosition": 23158, "Adam": 23159, "lb": 23160, "ubs": 23161, "Ġimperfect": 23162, "Ġclusters": 23163, "ĠEngineer": 23164, "ĠCherry": 23165, "Ġinauguration": 23166, "ĠSau": 23167, "Ġembodiment": 23168, "ĠUncle": 23169, "Ġoverr": 23170, "Ġexplosions": 23171, "cule": 23172, "ĠPrinceton": 23173, "ĠAndrea": 23174, "Ġincorrectly": 23175, "Ġearnest": 23176, "Ġpilgr": 23177, "ĠSprint": 23178, "Ġsleeve": 23179, "Ġhears": 23180, "ĠAmazing": 23181, "Ġbrowsing": 23182, "agin": 23183, "Ġhomeland": 23184, "Ġhaw": 23185, "Ġdiving": 23186, "istered": 23187, "178": 23188, "Ġbargaining": 23189, "ĠArcade": 23190, "Ġdelegate": 23191, "terson": 23192, "................................................................": 23193, "ĠJacksonville": 23194, "275": 23195, "Ġstagn": 23196, "Ġadam": 23197, "ĠSherman": 23198, "CB": 23199, "Ġsuburb": 23200, "ĠFoods": 23201, "Ġconverting": 23202, "ĠArist": 23203, "Ġchambers": 23204, "love": 23205, "Ġamino": 23206, "ĠGan": 23207, "Ġmadness": 23208, "mc": 23209, "ĠUSE": 23210, "defined": 23211, "Ġultr": 23212, "indust": 23213, "Ġwolves": 23214, "lance": 23215, "Additionally": 23216, "Ġcracks": 23217, "asia": 23218, "ĠReason": 23219, "ĠPump": 23220, "Ġaccidental": 23221, "ĠLaser": 23222, "ĠRid": 23223, "Ġinitialized": 23224, "elli": 23225, "Ġunnamed": 23226, "Ġnoun": 23227, "ĠPassed": 23228, "Ġhostage": 23229, "ĠEthiop": 23230, "shirts": 23231, "Ġunrel": 23232, "ĠEmbassy": 23233, "Ġ1941": 23234, "Ġatoms": 23235, "Ġpurported": 23236, "164": 23237, "ĠFi": 23238, "Ġgallons": 23239, "ĠMonica": 23240, "Ġpg": 23241, "enment": 23242, "Ġsorted": 23243, "ĠGospel": 23244, "Ġheights": 23245, "Ġtraced": 23246, "Ġundergoing": 23247, "Shell": 23248, "Ġsacks": 23249, "Ġproportions": 23250, "Ġhalluc": 23251, "Font": 23252, "acet": 23253, "Ġwarmer": 23254, "ĠINTER": 23255, "Ġgrabbing": 23256, "Plug": 23257, "Ġrealization": 23258, "ĠBurke": 23259, "Ġenchant": 23260, "ATER": 23261, "ĠSeed": 23262, "Ġabundant": 23263, "FM": 23264, "Ġcivic": 23265, "Vs": 23266, "isi": 23267, "Ġvow": 23268, "Ġreper": 23269, "ĠPartnership": 23270, "Ġpenetration": 23271, "Ġaxe": 23272, "Ġshattered": 23273, "ĠZombies": 23274, "Ġvinyl": 23275, "ĠAlert": 23276, "eon": 23277, "Ġobliged": 23278, "ĠIllust": 23279, "ĠPlaza": 23280, "ĠFrontier": 23281, "Ġdavidjl": 23282, "ĠSerial": 23283, "ĠHav": 23284, "ĠNutrition": 23285, "Bi": 23286, "ĠâĸĪ": 23287, "ĠJays": 23288, "linux": 23289, "Ġhurry": 23290, "Ġvoy": 23291, "Ġhopeless": 23292, "ĠStealth": 23293, "Ġãģ": 23294, "essors": 23295, "ttle": 23296, "borg": 23297, "ĠSafari": 23298, "fell": 23299, "Ġwary": 23300, "due": 23301, "ĠAbove": 23302, "Ha": 23303, "ELL": 23304, "Ġnotor": 23305, "ĠWon": 23306, "Too": 23307, "Ġoccupations": 23308, "Ġpossessions": 23309, "Ġinviting": 23310, "Ġpredators": 23311, "Ġaccelerated": 23312, "Ġ157": 23313, "uterte": 23314, "ĠCube": 23315, "east": 23316, "account": 23317, "Give": 23318, "Ġtransplant": 23319, "redients": 23320, "idable": 23321, "Ġscreenshots": 23322, "ĠGund": 23323, "ĠFS": 23324, "Ġtravelers": 23325, "Ġsensory": 23326, "ĠFiat": 23327, "ĠRockets": 23328, "İĭ": 23329, "_{": 23330, "Friend": 23331, "Ġcharming": 23332, "ALS": 23333, "Ġenjoyment": 23334, "mph": 23335, "Ġ5000": 23336, "ĠREG": 23337, "ÙĨ": 23338, "bia": 23339, "Ġcompilation": 23340, "rost": 23341, "ĠVP": 23342, "ĠSchne": 23343, "2019": 23344, "Ġcopying": 23345, "MORE": 23346, "ĠFlore": 23347, "falls": 23348, "215": 23349, "total": 23350, "Ġdisciples": 23351, "double": 23352, "Ġexceeding": 23353, "Ġsmashed": 23354, "Ġconceptual": 23355, "ĠRomania": 23356, "ĠBrent": 23357, "ĠICE": 23358, "ĠTou": 23359, "Ġgrap": 23360, "Ġnails": 23361, "189": 23362, "ãĥĺ": 23363, "Ġprocure": 23364, "eur": 23365, "Ġconfirming": 23366, "ĠCec": 23367, "awi": 23368, "ĠEden": 23369, "Ġng": 23370, "Ġengineered": 23371, "atics": 23372, "Ġhooked": 23373, "Ġdisgusting": 23374, "ĠMurder": 23375, "ãĤ¿": 23376, "Library": 23377, "Ġ168": 23378, "Almost": 23379, "hematic": 23380, "Menu": 23381, "ĠNotre": 23382, "ĠJur": 23383, "Ġkidnapped": 23384, "Ġhacker": 23385, "ĠJade": 23386, "Ġcreepy": 23387, "Ġdrawings": 23388, "ĠSponsor": 23389, "Ġcyclists": 23390, "ĠGoblin": 23391, "Ġoptimized": 23392, "Ġstaged": 23393, "ĠMcD": 23394, "between": 23395, "Age": 23396, "eno": 23397, "Sex": 23398, "ĠWide": 23399, "nings": 23400, "avis": 23401, "Ġincapable": 23402, "ĠKob": 23403, "Ġrewarding": 23404, "ĠLone": 23405, "olescent": 23406, "Ġcontracted": 23407, "Ġsticky": 23408, "Jose": 23409, "Ball": 23410, "fest": 23411, "ĠInput": 23412, "ĠRecently": 23413, "Ġtomat": 23414, "square": 23415, "Application": 23416, "Ġnitrogen": 23417, "Ġduplicate": 23418, "ĠRecon": 23419, "ĠDear": 23420, "London": 23421, "Ġintra": 23422, "Ġdock": 23423, "Ġoutreach": 23424, "ĠMillion": 23425, "Ġmammals": 23426, "ampton": 23427, "VAL": 23428, "Ġsnaps": 23429, "Ġdos": 23430, "ĠWhole": 23431, "ĠReady": 23432, "Try": 23433, "ĠWinnipeg": 23434, "earance": 23435, "Ġincurred": 23436, "renched": 23437, "ĠNSW": 23438, "ilot": 23439, "raine": 23440, "Ġcube": 23441, "got": 23442, "Ġrunway": 23443, "etermined": 23444, "ĠHawks": 23445, "Ġsurvivor": 23446, "ĠWish": 23447, "ĠDin": 23448, "ĠDEF": 23449, "ĠVault": 23450, "187": 23451, "Ġmushrooms": 23452, "Ġcrisp": 23453, "bey": 23454, "ĠDiscovery": 23455, "Ġdevelopmental": 23456, "Ġparadigm": 23457, "Ġchaotic": 23458, "ĠTsu": 23459, "Ġ333": 23460, "bons": 23461, "Ġbacterial": 23462, "Ġcommits": 23463, "Ġcosmic": 23464, "Ġmega": 23465, "ocative": 23466, "ĠPaint": 23467, "ophobic": 23468, "Ġvain": 23469, "Ġcarved": 23470, "ĠThief": 23471, "ĠGul": 23472, "owship": 23473, "Ġcites": 23474, "ĠEdinburgh": 23475, "Ġdiminished": 23476, "Ġacknowledges": 23477, "ĠKills": 23478, "Ġmicrow": 23479, "ĠHera": 23480, "Ġseniors": 23481, "Ġwhereby": 23482, "Hop": 23483, "atron": 23484, "Ġunavailable": 23485, "ĠNate": 23486, "Ġ480": 23487, "Ġslated": 23488, "ĠRebecca": 23489, "ĠBattery": 23490, "Ġgrammar": 23491, "Ġheadset": 23492, "Ġcursor": 23493, "Ġexcluding": 23494, "anye": 23495, "aundering": 23496, "ebin": 23497, "Ġfeasible": 23498, "ĠPublishing": 23499, "ĠLabs": 23500, "ĠCliff": 23501, "ĠFerrari": 23502, "Ġpac": 23503, "visible": 23504, "marked": 23505, "pell": 23506, "Ġpolite": 23507, "Ġstaggering": 23508, "ĠGalactic": 23509, "Ġsuperst": 23510, "Ġparan": 23511, "ĠOfficers": 23512, "ãĢģ": 23513, "Ġspecifics": 23514, "ulus": 23515, "239": 23516, "ĠPaste": 23517, "AMP": 23518, "ĠPanama": 23519, "ĠDelete": 23520, "anguard": 23521, "restrial": 23522, "Ġheroic": 23523, "ĠDy": 23524, "اÙĦ": 23525, "Ġincumbent": 23526, "Ġcrunch": 23527, "tro": 23528, "Ġscoop": 23529, "Ġblogger": 23530, "Ġsellers": 23531, "uren": 23532, "Ġmedicines": 23533, "ĠCaps": 23534, "ĠAnimation": 23535, "oxy": 23536, "Ġoutward": 23537, "Ġinquiries": 23538, "229": 23539, "Ġpsychologist": 23540, "ĠSask": 23541, "evil": 23542, "Ġcontaminated": 23543, "ãĤ¨": 23544, "herence": 23545, "Ġbranded": 23546, "ĠAbdul": 23547, "zh": 23548, "Ġparagraphs": 23549, "Ġmins": 23550, "Ġcorrelated": 23551, "erb": 23552, "Ġimpart": 23553, "Ġmilestone": 23554, "ĠSolutions": 23555, "otle": 23556, "Ġundercover": 23557, "Ġmarched": 23558, "ĠChargers": 23559, "fax": 23560, "ĠSecrets": 23561, "Ġruth": 23562, "weather": 23563, "Ġfeminine": 23564, "Ġsham": 23565, "Ġprestigious": 23566, "iggins": 23567, "Ġsung": 23568, "history": 23569, "ettle": 23570, "ggie": 23571, "Ġoutdated": 23572, "oland": 23573, "Ġperceptions": 23574, "ĠSession": 23575, "ĠDodgers": 23576, "uj": 23577, "ĠEND": 23578, "Doc": 23579, "Ġdeficiency": 23580, "Grand": 23581, "ĠJoker": 23582, "Ġretrospect": 23583, "Ġdiagnostic": 23584, "Ġharmless": 23585, "Ġrogue": 23586, "ĠAval": 23587, "Equ": 23588, "Ġtransc": 23589, "ĠRobertson": 23590, "ĠDepending": 23591, "ĠBurns": 23592, "ivo": 23593, "Ġhostility": 23594, "Features": 23595, "ĵĺ": 23596, "Ġdiscomfort": 23597, "ĠLCD": 23598, "specified": 23599, "ĠExpect": 23600, "340": 23601, "Ġimperative": 23602, "ĠRegular": 23603, "Chinese": 23604, "Ġstatewide": 23605, "Ġsymm": 23606, "Ġloops": 23607, "Ġautumn": 23608, "Nick": 23609, "Ġshaping": 23610, "Ġquot": 23611, "Ġcherry": 23612, "ĠCrossref": 23613, "è¦ļéĨĴ": 23614, "Standard": 23615, "heed": 23616, "ĠDell": 23617, "ĠVietnamese": 23618, "Ġost": 23619, "ĠValkyrie": 23620, "OA": 23621, "Assad": 23622, "Ġrebound": 23623, "ĠTraffic": 23624, "places": 23625, "æĺ": 23626, "ĠBuc": 23627, "172": 23628, "Ġshelters": 23629, "Ġinsisting": 23630, "ĠCertainly": 23631, "ĠKenneth": 23632, "ĠTCP": 23633, "Ġpenal": 23634, "ĠReplay": 23635, "heard": 23636, "Ġdialect": 23637, "iza": 23638, "ĠFY": 23639, "itcher": 23640, "ĠDL": 23641, "Ġspiral": 23642, "Ġquarterbacks": 23643, "Ġhull": 23644, "Ġgoogle": 23645, "Ġtodd": 23646, "ĠSterling": 23647, "ĠPlate": 23648, "Ġspying": 23649, "mbol": 23650, "ĠRealm": 23651, "ĠProced": 23652, "ĠCrash": 23653, "Ġterminate": 23654, "Ġprotesting": 23655, "Center": 23656, "guided": 23657, "Ġuncover": 23658, "Ġboycott": 23659, "Ġrealizes": 23660, "sound": 23661, "Ġpretending": 23662, "ĠVas": 23663, "1980": 23664, "Ġframed": 23665, "Ġ139": 23666, "Ġdescended": 23667, "Ġrehabilitation": 23668, "Ġborrowing": 23669, "ĠBuch": 23670, "Ġblur": 23671, "Ron": 23672, "ĠFrozen": 23673, "enza": 23674, "Chief": 23675, "ĠPoor": 23676, "Ġtranslates": 23677, "MIN": 23678, "Ġ212": 23679, "JECT": 23680, "Ġerupted": 23681, "Ġsuccesses": 23682, "SEC": 23683, "Ġplague": 23684, "Ġgems": 23685, "doms": 23686, "Ġstretches": 23687, "ĠSpy": 23688, "Ġstorytelling": 23689, "Credit": 23690, "ĠPush": 23691, "Ġtraction": 23692, "Ġineffective": 23693, "ĠLuna": 23694, "Ġtapes": 23695, "Ġanalytics": 23696, "ercise": 23697, "Ġprogrammes": 23698, "ĠCarbon": 23699, "Ġbehold": 23700, "heavy": 23701, "ĠConservation": 23702, "ĠFIR": 23703, "Ġsack": 23704, "termin": 23705, "ricks": 23706, "Ġhoused": 23707, "Ġunusually": 23708, "Ice": 23709, "Ġexecuting": 23710, "ĠMoroc": 23711, "eday": 23712, "Ġeditions": 23713, "Ġsmarter": 23714, "ĠBA": 23715, "Ġoutlaw": 23716, "Ġvanished": 23717, "iba": 23718, "ALSE": 23719, "ĠSilva": 23720, "238": 23721, "Could": 23722, "Ġphilosopher": 23723, "Ġevacuated": 23724, "Secret": 23725, "142": 23726, "Ġvisas": 23727, "ãĤ¬": 23728, "ĠMalt": 23729, "ĠClearly": 23730, "ĠNiger": 23731, "ĠCairo": 23732, "ĠFist": 23733, "380": 23734, "ĠXML": 23735, "auto": 23736, "itant": 23737, "Ġreinforced": 23738, "Record": 23739, "ĠSurvivor": 23740, "GHz": 23741, "Ġscrews": 23742, "parents": 23743, "Ġoceans": 23744, "mares": 23745, "Ġbrakes": 23746, "vasive": 23747, "Ġhello": 23748, "ĠSIM": 23749, "rimp": 23750, "Ġore": 23751, "ĠArmour": 23752, "247": 23753, "Ġterrific": 23754, "Ġtones": 23755, "141": 23756, "ĠMinutes": 23757, "Episode": 23758, "Ġcurves": 23759, "Ġinflammatory": 23760, "Ġbatting": 23761, "ĠBeautiful": 23762, "Lay": 23763, "Ġunpop": 23764, "vable": 23765, "Ġriots": 23766, "ĠTactics": 23767, "baugh": 23768, "ĠCock": 23769, "Ġorgasm": 23770, "ĠSas": 23771, "Ġconstructor": 23772, "etz": 23773, "Gov": 23774, "Ġantagon": 23775, "Ġtheat": 23776, "Ġdeeds": 23777, "hao": 23778, "cuts": 23779, "ĠMcCl": 23780, "Ġum": 23781, "ĠScientists": 23782, "Ġgrassroots": 23783, "yssey": 23784, "\"]=>": 23785, "Ġsurfaced": 23786, "Ġshades": 23787, "Ġneighbours": 23788, "Ġadvertis": 23789, "oya": 23790, "Ġmerged": 23791, "Upon": 23792, "Ġgad": 23793, "Ġanticipate": 23794, "Anyway": 23795, "Ġslogan": 23796, "Ġdisrespect": 23797, "Iran": 23798, "ĠTB": 23799, "acted": 23800, "Ġsubpoen": 23801, "mediately": 23802, "OOOO": 23803, "Ġwaiver": 23804, "Ġvulnerabilities": 23805, "ottesville": 23806, "ĠHuffington": 23807, "Josh": 23808, "ĠDH": 23809, "Monday": 23810, "ĠEllen": 23811, "Know": 23812, "xon": 23813, "items": 23814, "228": 23815, "Ġfills": 23816, "ĠNike": 23817, "Ġcumulative": 23818, "andals": 23819, "Ir": 23820, "Ġì": 23821, "Ġfriction": 23822, "igator": 23823, "Ġscans": 23824, "ĠVienna": 23825, "ldom": 23826, "Ġperformers": 23827, "Prim": 23828, "Ġbidding": 23829, "Mur": 23830, "Ġleaned": 23831, "ĠPrix": 23832, "alks": 23833, "Ġ[âĢ¦]": 23834, "ĠTwitch": 23835, "ĠDeveloper": 23836, "ĠGir": 23837, "Ġcallback": 23838, "Abstract": 23839, "Ġaccustomed": 23840, "Ġfreedoms": 23841, "ĠPG": 23842, "uracy": 23843, "Ġlump": 23844, "isman": 23845, ",,,,": 23846, "1992": 23847, "ĠRED": 23848, "Ġworm": 23849, "Match": 23850, "ĠPlatinum": 23851, "IJ": 23852, "ĠOwner": 23853, "Trivia": 23854, "compl": 23855, "Ġnewborn": 23856, "Ġfantas": 23857, "Own": 23858, "Ġ1959": 23859, "Ġsympath": 23860, "Ġubiqu": 23861, "Ġoutputs": 23862, "Ġallev": 23863, "Ġprag": 23864, "Kevin": 23865, "Ġfavors": 23866, "Ġburial": 23867, "Ġnurt": 23868, "solete": 23869, "cache": 23870, "Ġ156": 23871, "Ġunlocks": 23872, "techn": 23873, "Making": 23874, "Ġconquer": 23875, "adic": 23876, "æĸ": 23877, "Ġelf": 23878, "Ġelectorate": 23879, "ĠKurds": 23880, "ĠStack": 23881, "ĠSamurai": 23882, "Ġâĺħ": 23883, "Ġ{}": 23884, "ĠSaid": 23885, "ĠFallout": 23886, "Ġkindness": 23887, "ĠCustoms": 23888, "ĠBoulevard": 23889, "Ġhelicopters": 23890, "otics": 23891, "ĠVeget": 23892, "comment": 23893, "Ġcriticised": 23894, "Ġpolished": 23895, "ĠRemix": 23896, "ĠCultural": 23897, "Ġrecons": 23898, "Ġdoi": 23899, "atem": 23900, "Screen": 23901, "Ġbarred": 23902, "Comments": 23903, "ĠGenerally": 23904, "Ġslap": 23905, "720": 23906, "Vari": 23907, "pine": 23908, "Ġempt": 23909, "Ġhats": 23910, "ĠPlaying": 23911, "lab": 23912, "average": 23913, "forms": 23914, "ĠCotton": 23915, "Ġcans": 23916, "ĠDON": 23917, "ĠSomalia": 23918, "Crypt": 23919, "ĠIncreases": 23920, "Ever": 23921, "modern": 23922, "Ġsurgeon": 23923, "3000": 23924, "Ġrandomized": 23925, "================================================================": 23926, "Bern": 23927, "impl": 23928, "ĠCOR": 23929, "Ġproclaim": 23930, "thouse": 23931, "Ġtoes": 23932, "Ġample": 23933, "Ġpreserving": 23934, "Ġdisbel": 23935, "grand": 23936, "Besides": 23937, "Ġsilk": 23938, "ĠPattern": 23939, "hm": 23940, "Ġenterprises": 23941, "Ġaffidavit": 23942, "ĠAdvisory": 23943, "Ġadvertised": 23944, "ĠReligious": 23945, "sections": 23946, "psych": 23947, "ĠFields": 23948, "aways": 23949, "Ġhashtag": 23950, "ĠNightmare": 23951, "Ġvampire": 23952, "Ġforensic": 23953, "rossover": 23954, "nar": 23955, "Ġnavy": 23956, "Ġvacant": 23957, "ĠDuel": 23958, "Ġhallway": 23959, "Ġfacebook": 23960, "identally": 23961, "ĠNRA": 23962, "Ġmatt": 23963, "Ġhurricane": 23964, "ĠKirby": 23965, "ĠPuzzle": 23966, "Ġskirt": 23967, "oust": 23968, "dullah": 23969, "Ġanalogy": 23970, "inion": 23971, "Ġtomatoes": 23972, "ĠNV": 23973, "ĠPeak": 23974, "ĠMeyer": 23975, "Ġappointments": 23976, "Ġmasc": 23977, "Ġalley": 23978, "rehend": 23979, "Ġcharities": 23980, "Ġundo": 23981, "Ġdestinations": 23982, "ĠTesting": 23983, "\">\"": 24618, "cats": 24619, "*.": 24620, "Ġgestures": 24621, "general": 24622, "League": 24623, "Ġpackets": 24624, "ĠInspector": 24625, "ĠBerg": 24626, "Ġfraudulent": 24627, "Ġcriticize": 24628, "Fun": 24629, "Ġblaming": 24630, "ndra": 24631, "Ġslash": 24632, "ĠEston": 24633, "Ġproposing": 24634, "Ġwhales": 24635, "Ġtherapist": 24636, "Ġsubset": 24637, "Ġleisure": 24638, "ELD": 24639, "ĠCVE": 24640, "ĠActivity": 24641, "Ġculmin": 24642, "shop": 24643, "ĠDAY": 24644, "ischer": 24645, "ĠAdmiral": 24646, "ĠAttacks": 24647, "Ġ1958": 24648, "Ġmemoir": 24649, "Ġfolded": 24650, "Ġsexist": 24651, "Ġ153": 24652, "ĠLI": 24653, "Ġreadings": 24654, "Ġembarrassment": 24655, "ĠEmployment": 24656, "wart": 24657, "chin": 24658, "Ġcontinuation": 24659, "lia": 24660, "Recently": 24661, "Ġduel": 24662, "Ġevacuation": 24663, "ĠKashmir": 24664, "Ġdisposition": 24665, "ĠRig": 24666, "Ġbolts": 24667, "Ġinsurers": 24668, "467": 24669, "Mex": 24670, "Ġretaliation": 24671, "Ġmisery": 24672, "Ġunreasonable": 24673, "raining": 24674, "Imm": 24675, "ĠPU": 24676, "emer": 24677, "Ġgenital": 24678, "ãĤ³": 24679, "ĠCandy": 24680, "Ġonions": 24681, "ĠPatt": 24682, "liner": 24683, "Ġconceded": 24684, "Ġfa": 24685, "Ġforc": 24686, "ĠHernandez": 24687, "ĠGeoff": 24688, "debian": 24689, "ĠTeams": 24690, "Ġcries": 24691, "Ġhomeowners": 24692, "237": 24693, "ABC": 24694, "Ġstitch": 24695, "Ġstatistic": 24696, "Ġheaders": 24697, "ĠBiology": 24698, "Ġmotors": 24699, "ĠGEN": 24700, "ĠLip": 24701, "Ġhates": 24702, "Ġheel": 24703, "Self": 24704, "ipl": 24705, "EDIT": 24706, "orting": 24707, "Ġannot": 24708, "ĠSpeech": 24709, "oldemort": 24710, "ĠJavascript": 24711, "ĠLeBron": 24712, "Ġfootprint": 24713, "Ġfn": 24714, "Ġseizures": 24715, "nas": 24716, "hide": 24717, "Ġ1954": 24718, "ĠBee": 24719, "ĠDeclaration": 24720, "ĠKatie": 24721, "Ġreservations": 24722, "NR": 24723, "female": 24724, "Ġsaturated": 24725, "Ġbiblical": 24726, "Ġtrolls": 24727, "Device": 24728, "photos": 24729, "Ġdrums": 24730, "ãĥīãĥ©ãĤ´ãĥ³": 24731, "Night": 24732, "fighter": 24733, "ĠHak": 24734, "riber": 24735, "Ġcush": 24736, "Ġdisciplinary": 24737, "baum": 24738, "ĠGH": 24739, "ĠSchmidt": 24740, "ilibrium": 24741, "Ġsixty": 24742, "ĠKushner": 24743, "rots": 24744, "Ġpund": 24745, "ĠRac": 24746, "Ġsprings": 24747, "Ġconve": 24748, "Business": 24749, "Fall": 24750, "Ġqualifications": 24751, "Ġverses": 24752, "Ġnarciss": 24753, "ĠKoh": 24754, "ĠWow": 24755, "ĠCharlottesville": 24756, "edo": 24757, "Ġinterrogation": 24758, "ĠWool": 24759, "365": 24760, "Brian": 24761, "Ġâľĵ": 24762, "Ġalleges": 24763, "onds": 24764, "idation": 24765, "ĠJackie": 24766, "yu": 24767, "Ġlakes": 24768, "Ġworthwhile": 24769, "Ġcrystals": 24770, "ĠJuda": 24771, "Ġcomprehend": 24772, "Ġflush": 24773, "Ġabsorption": 24774, "ĠOC": 24775, "Ġfrightened": 24776, "ĠChocolate": 24777, "Martin": 24778, "Ġbuys": 24779, "Ġbucks": 24780, "Ġappell": 24781, "ĠChampionships": 24782, "Ġlistener": 24783, "ĠDefensive": 24784, "Ġcz": 24785, "uds": 24786, "ĠMate": 24787, "Ġreplay": 24788, "Ġdecorated": 24789, "Ġsunk": 24790, "ĠVIP": 24791, "ĠAnk": 24792, "Ġ195": 24793, "aaaa": 24794, "Nobody": 24795, "ĠMilk": 24796, "ĠGur": 24797, "ĠMk": 24798, "ĠSara": 24799, "Ġseating": 24800, "ĠWid": 24801, "Track": 24802, "Ġemploys": 24803, "Ġgigantic": 24804, "APP": 24805, "ãĤ§": 24806, "inventory": 24807, "Ġtowel": 24808, "atche": 24809, "lasting": 24810, "ĠTL": 24811, "Ġlatency": 24812, "Ġkne": 24813, "Ber": 24814, "meaning": 24815, "Ġupheld": 24816, "Ġplayground": 24817, "Ġmant": 24818, "Side": 24819, "Ġstereo": 24820, "Ġnorthwest": 24821, "Ġexceptionally": 24822, "Ġrays": 24823, "Ġrecurring": 24824, "Drive": 24825, "Ġupright": 24826, "Ġabduct": 24827, "ĠMarathon": 24828, "Ġgoodbye": 24829, "Ġalphabet": 24830, "hp": 24831, "Ġcourtroom": 24832, "rington": 24833, "othing": 24834, "Tag": 24835, "Ġdiplomats": 24836, "Ġbarbar": 24837, "ĠAqua": 24838, "183": 24839, "3333": 24840, "Ġmaturity": 24841, "Ġinstability": 24842, "ĠApache": 24843, "Ġ===": 24844, "Ġfasting": 24845, "ĠGrid": 24846, "ModLoader": 24847, "Ġ152": 24848, "Abs": 24849, "ĠOperating": 24850, "etti": 24851, "Ġacquaint": 24852, "Donnell": 24853, "ĠKem": 24854, "ĠForge": 24855, "Ġarmored": 24856, "Mil": 24857, "Ġphilosophers": 24858, "invest": 24859, "Players": 24860, "âĪ": 24861, "Ġmyriad": 24862, "Ġcomrades": 24863, "Rot": 24864, "Ġremembering": 24865, "Ġcorresponds": 24866, "Ġprogrammers": 24867, "ĠLynn": 24868, "Ġolig": 24869, "Ġcoherent": 24870, "ynchron": 24871, "ĠChemical": 24872, "Ġjugg": 24873, "pair": 24874, "posts": 24875, "Eye": 24876, "ĠInner": 24877, "Ġsemester": 24878, "ottest": 24879, "ĠEmirates": 24880, "ricanes": 24881, "orously": 24882, "mits": 24883, "ĠWis": 24884, "Ġdodge": 24885, "location": 24886, "Ġfaded": 24887, "Amazon": 24888, "ĠProceed": 24889, "ĠINFO": 24890, "journal": 24891, "ĠTruck": 24892, "Ten": 24893, "Ġ217": 24894, "Ġstatutes": 24895, "mobile": 24896, "ĠTypes": 24897, "Recomm": 24898, "buster": 24899, "pex": 24900, "Ġlegends": 24901, "Ġheadache": 24902, "faced": 24903, "ĠWiFi": 24904, "ifty": 24905, "ĠHER": 24906, "Ġcircuits": 24907, "ERROR": 24908, "226": 24909, "olin": 24910, "Ġcylinder": 24911, "ospace": 24912, "ikers": 24913, "Prem": 24914, "Quant": 24915, "Ġconflicting": 24916, "Ġslightest": 24917, "Ġforged": 24918, "ionage": 24919, "Stephen": 24920, "ĠKub": 24921, "ĠOpportun": 24922, "ĠHeal": 24923, "Ġblo": 24924, "Ġrulers": 24925, "Ġhuh": 24926, "Ġsubmarine": 24927, "fy": 24928, "asser": 24929, "Ġallowance": 24930, "ĠKasich": 24931, "ĠTas": 24932, "ĠAustralians": 24933, "ForgeModLoader": 24934, "ĠâĨij": 24935, "ĠMatrix": 24936, "amins": 24937, "Ġ1200": 24938, "ĠAcqu": 24939, "236": 24940, "Document": 24941, "ĠBreaking": 24942, "193": 24943, "ĠSubst": 24944, "ĠRoller": 24945, "ĠProperties": 24946, "ĠNI": 24947, "tier": 24948, "Ġcrushing": 24949, "Ġadvocating": 24950, "Furthermore": 24951, "keepers": 24952, "Ġsexism": 24953, "xd": 24954, "Ġcaller": 24955, "ĠSense": 24956, "chieve": 24957, "ĠTF": 24958, "Ġfueled": 24959, "Ġreminiscent": 24960, "Ġobsess": 24961, "urst": 24962, "Ġuphold": 24963, "ĠFans": 24964, "hetics": 24965, "ĠâĹ": 24966, "ĠBath": 24967, "Ġbeverage": 24968, "Ġoscill": 24969, "254": 24970, "Ġpoles": 24971, "Ġgradual": 24972, "Ġexting": 24973, "ĠSuff": 24974, "ĠSuddenly": 24975, "Ġliking": 24976, "Ġ1949": 24977, "unciation": 24978, "amination": 24979, "ĠOmar": 24980, "ĠLV": 24981, "ĠConsequently": 24982, "Ġsynthes": 24983, "ĠGIF": 24984, "Ġpains": 24985, "Ġinteracting": 24986, "uously": 24987, "incre": 24988, "Ġrumor": 24989, "ĠScientology": 24990, "197": 24991, "ĠZig": 24992, "Ġspelling": 24993, "ĠASS": 24994, "Ġextingu": 24995, "mson": 24996, "Ġgh": 24997, "Ġremarked": 24998, "ĠStrategic": 24999, "ĠMON": 25000, "å¥": 25001, "gae": 25002, "ĠWHAT": 25003, "Eric": 25004, "ĠCampus": 25005, "Ġmethane": 25006, "Ġimagin": 25007, "JUST": 25008, "ĠAlm": 25009, "XT": 25010, "iq": 25011, "ĠRSS": 25012, "Ġwrongdoing": 25013, "atta": 25014, "Ġbigot": 25015, "Ġdemonstrators": 25016, "ĠCalvin": 25017, "ĠVilla": 25018, "Ġmembrane": 25019, "ĠAwesome": 25020, "Ġbenefic": 25021, "268": 25022, "Ġmagnificent": 25023, "ĠLots": 25024, "Greg": 25025, "ĠBoris": 25026, "Ġdetainees": 25027, "ĠHerman": 25028, "Ġwhispered": 25029, "Ġawe": 25030, "Professor": 25031, "funding": 25032, "Ġphysiological": 25033, "ĠDestruction": 25034, "Ġlimb": 25035, "Ġmanipulated": 25036, "Ġbubbles": 25037, "Ġpseud": 25038, "Ġhydra": 25039, "ĠBristol": 25040, "Ġstellar": 25041, "ĠExpansion": 25042, "ĠKell": 25043, "ĠInterestingly": 25044, "Ġmans": 25045, "Ġdragging": 25046, "Ġecological": 25047, "ĠFit": 25048, "Ġgent": 25049, "Ġbenefited": 25050, "ĠHaiti": 25051, "Ġpolyg": 25052, "ãĥİ": 25053, "Ġ2030": 25054, "Ġprow": 25055, "Ġreconstruction": 25056, "Ġwast": 25057, "Ġpsychic": 25058, "ĠGreeks": 25059, "Handler": 25060, "162": 25061, "ĠPulse": 25062, "Ġsolicit": 25063, "Ġsys": 25064, "Ġinflux": 25065, "ĠGentle": 25066, "percent": 25067, "Ġproliferation": 25068, "Ġtaxable": 25069, "Ġdisregard": 25070, "Ġescaping": 25071, "Ġginger": 25072, "Ġwithstand": 25073, "Ġdevastated": 25074, "ĠDew": 25075, "series": 25076, "Ġinjected": 25077, "elaide": 25078, "Ġturnover": 25079, "heat": 25080, "ĻĤ": 25081, "Happy": 25082, "ĠSilent": 25083, "ãĤŃ": 25084, "ivism": 25085, "Ġirrational": 25086, "AMA": 25087, "Ġreef": 25088, "rub": 25089, "Ġ162": 25090, "Ġbankers": 25091, "ĠEthics": 25092, "vv": 25093, "Ġcriticisms": 25094, "Kn": 25095, "186": 25096, "Movie": 25097, "ĠTories": 25098, "Ġnood": 25099, "Ġdistortion": 25100, "False": 25101, "odore": 25102, "Ġtasty": 25103, "Research": 25104, "ĠUID": 25105, "-)": 25106, "Ġdivorced": 25107, "ĠMU": 25108, "ĠHayes": 25109, "ĠIsn": 25110, "iani": 25111, "ĠHQ": 25112, "Ġ\"#": 25113, "ignant": 25114, "Ġtraumatic": 25115, "ĠLing": 25116, "Hun": 25117, "Ġsabot": 25118, "online": 25119, "random": 25120, "Ġrenamed": 25121, "rared": 25122, "KA": 25123, "dead": 25124, "ét": 25125, "ĠAssistance": 25126, "Ġseaf": 25127, "++++++++": 25128, "Ġseldom": 25129, "ĠWebb": 25130, "Ġboolean": 25131, "ulet": 25132, "Ġrefrain": 25133, "ĠDIY": 25134, "rule": 25135, "Ġshutting": 25136, "Ġutilizing": 25137, "loading": 25138, "ĠParam": 25139, "coal": 25140, "ooter": 25141, "Ġattracting": 25142, "ĠDol": 25143, "Ġhers": 25144, "agnetic": 25145, "ĠReach": 25146, "imo": 25147, "Ġdiscarded": 25148, "ĠPip": 25149, "015": 25150, "ür": 25151, "Ġmug": 25152, "Imagine": 25153, "COL": 25154, "Ġcursed": 25155, "ĠShows": 25156, "ĠCurtis": 25157, "ĠSachs": 25158, "speaking": 25159, "ĠVista": 25160, "ĠFramework": 25161, "ongo": 25162, "Ġsubreddit": 25163, "Ġcrus": 25164, "ĠOval": 25165, "Row": 25166, "growing": 25167, "Ġinstallment": 25168, "Ġglac": 25169, "ĠAdvance": 25170, "ECK": 25171, "ĠLGBTQ": 25172, "LEY": 25173, "Ġacet": 25174, "Ġsuccessive": 25175, "ĠNicole": 25176, "Ġ1957": 25177, "Quote": 25178, "Ġcircumstance": 25179, "ackets": 25180, "Ġ142": 25181, "ortium": 25182, "Ġguessed": 25183, "ĠFrame": 25184, "Ġperpetrators": 25185, "ĠAviation": 25186, "ĠBench": 25187, "Ġhandc": 25188, "Ap": 25189, "Ġ1956": 25190, "259": 25191, "rand": 25192, "NetMessage": 25193, "din": 25194, "urtles": 25195, "hig": 25196, "ĠVIII": 25197, "ffiti": 25198, "ĠSwords": 25199, "bial": 25200, "Ġkidnapping": 25201, "device": 25202, "Ġbarn": 25203, "ĠEli": 25204, "aucas": 25205, "Send": 25206, "Constructed": 25207, "Ġ½": 25208, "Ġneedles": 25209, "Ġadvertisements": 25210, "Ġvou": 25211, "Ġexhibited": 25212, "ĠFortress": 25213, "Ask": 25214, "Berry": 25215, "TYPE": 25216, "Ġcancers": 25217, "umping": 25218, "ĠTerritory": 25219, "Ġprud": 25220, "Ġnas": 25221, "Ġatheist": 25222, "Ġbalances": 25223, "ãģŁ": 25224, "ĠShawn": 25225, "&&": 25226, "Ġlandsc": 25227, "ĠRGB": 25228, "Ġpetty": 25229, "Ġexcellence": 25230, "Ġtranslations": 25231, "Ġparcel": 25232, "ĠChev": 25233, "East": 25234, "ĠOutput": 25235, "imi": 25236, "Ġambient": 25237, "ĠThreat": 25238, "Ġvillains": 25239, "Ġ550": 25240, "ICA": 25241, "Ġtaller": 25242, "Ġleaking": 25243, "cup": 25244, "Ġpolish": 25245, "Ġinfectious": 25246, "ĠKC": 25247, "Ġ@@": 25248, "background": 25249, "Ġbureaucracy": 25250, "ĠSai": 25251, "unless": 25252, "itious": 25253, "ĠSkype": 25254, "Atl": 25255, "IDENT": 25256, "008": 25257, "Ġhypocr": 25258, "Ġpitchers": 25259, "Ġguessing": 25260, "ĠFINAL": 25261, "Between": 25262, "Ġvillagers": 25263, "Ġ252": 25264, "fashion": 25265, "ĠTunis": 25266, "Beh": 25267, "ĠExc": 25268, "ĠMID": 25269, "288": 25270, "ĠHaskell": 25271, "196": 25272, "ĠNOR": 25273, "Ġspecs": 25274, "Ġinvari": 25275, "Ġglut": 25276, "ĠCars": 25277, "Ġimpulse": 25278, "Ġhonors": 25279, "gel": 25280, "Ġjurisdictions": 25281, "ĠBundle": 25282, "ulas": 25283, "California": 25284, "ĠIncrease": 25285, "Ġpear": 25286, "Ġsingles": 25287, "Ġcues": 25288, "Ġunderwent": 25289, "ĠWS": 25290, "Ġexaggerated": 25291, "Ġdubious": 25292, "Ġflashing": 25293, "LOG": 25294, ")].": 25295, "Journal": 25296, "tg": 25297, "Van": 25298, "ĠIstanbul": 25299, "ĠInsp": 25300, "ĠFranken": 25301, "Draw": 25302, "Ġsadness": 25303, "Ġironic": 25304, "ĠFry": 25305, "xc": 25306, "Ġ164": 25307, "isch": 25308, "Way": 25309, "ĠProtestant": 25310, "horn": 25311, "Ġunaff": 25312, "ĠViv": 25313, "illas": 25314, "ĠProductions": 25315, "ĠHogan": 25316, "Ġperimeter": 25317, "ĠSisters": 25318, "Ġspontaneous": 25319, "Ġdownside": 25320, "Ġdescendants": 25321, "Ġorn": 25322, "worm": 25323, "Japanese": 25324, "Ġ1955": 25325, "Ġ151": 25326, "ĠDoing": 25327, "elsen": 25328, "umbles": 25329, "Ġradically": 25330, "ĠDrum": 25331, "ĠBach": 25332, "Ġliabilities": 25333, "ĠOB": 25334, "ĠElementary": 25335, "Ġmeme": 25336, "ynes": 25337, "Ġfingerprint": 25338, "ĠGrab": 25339, "Ġundertake": 25340, "Members": 25341, "ĠReader": 25342, "ĠSims": 25343, "god": 25344, "Ġhypothetical": 25345, "scient": 25346, "ĠAJ": 25347, "Ġcharism": 25348, "Ġadmissions": 25349, "ĠMissile": 25350, "trade": 25351, "Ġexercising": 25352, "ĠBackground": 25353, "Written": 25354, "Ġvocals": 25355, "whether": 25356, "Ġvi": 25357, "ĠWinner": 25358, "Ġlitter": 25359, "ĠShooting": 25360, "STEM": 25361, "ãĤ¡": 25362, "ĠAFL": 25363, "Ġvariability": 25364, "Ġeats": 25365, "ĠDPS": 25366, "brow": 25367, "Ġelephants": 25368, "Ġstrat": 25369, "ĠÅ": 25370, "Ġsettlers": 25371, "Matthew": 25372, "Ġinadvert": 25373, "HI": 25374, "ĠIMF": 25375, "ĠGoal": 25376, "Ġnerves": 25377, "Johnson": 25378, "eye": 25379, "ablishment": 25380, "Thursday": 25381, "BILITY": 25382, "Had": 25383, "amoto": 25384, "hetamine": 25385, "eps": 25386, "Ġmitochond": 25387, "Ġcompressed": 25388, "ĠTrevor": 25389, "ĠAnimals": 25390, "Tool": 25391, "Lock": 25392, "Ġtweak": 25393, "Ġpinch": 25394, "Ġcancellation": 25395, "Pot": 25396, "Ġfocal": 25397, "ĠAstron": 25398, "173": 25399, "ĠASC": 25400, "ĠOTHER": 25401, "umni": 25402, "Ġdemise": 25403, "dl": 25404, "Ùħ": 25405, "Semitism": 25406, "Ġcracking": 25407, "Ġcollaborative": 25408, "Ġexplores": 25409, "sql": 25410, "Ġherbs": 25411, "Ġconfigurations": 25412, "mis": 25413, "ĠResult": 25414, "acey": 25415, "ĠSmoke": 25416, "Ġsanct": 25417, "elia": 25418, "Ġdegener": 25419, "Ġdeepest": 25420, "Ġscreamed": 25421, "Ġnap": 25422, "Software": 25423, "ĠSTAR": 25424, "EF": 25425, "ĠXin": 25426, "sponsored": 25427, "manship": 25428, "233": 25429, "Ġprimaries": 25430, "Ġfiltering": 25431, "Ġassemble": 25432, "mil": 25433, "ĠMyers": 25434, "bows": 25435, "Ġpunched": 25436, "Mic": 25437, "Ġinnovations": 25438, "Ġfunc": 25439, "ando": 25440, "Ġfracking": 25441, "ĠVul": 25442, "оÐ": 25443, "oshop": 25444, "ĠImmun": 25445, "Ġsettling": 25446, "Ġadolescents": 25447, "Ġrebuilding": 25448, "Ġtransforming": 25449, "Ġparole": 25450, "Ġharbor": 25451, "Ġbooking": 25452, "otional": 25453, "ongevity": 25454, "ĠYo": 25455, "bug": 25456, "Ġemerges": 25457, "ĠMethods": 25458, "ĠChu": 25459, "Pres": 25460, "ĠDungeons": 25461, "Ġtrailing": 25462, "ĠRum": 25463, "ĠHugh": 25464, "天": 25465, "ĠEra": 25466, "ĠBattles": 25467, "Results": 25468, "ĠTrading": 25469, "Ġversa": 25470, "css": 25471, "axies": 25472, "heet": 25473, "Ġgreed": 25474, "1989": 25475, "Ġgardens": 25476, "Ġcontingent": 25477, "Park": 25478, "ĠLeafs": 25479, "hook": 25480, "robe": 25481, "Ġdiplomacy": 25482, "ĠFuel": 25483, "ĠInvasion": 25484, "Ġupgrading": 25485, "Male": 25486, "Ġelic": 25487, "Ġrelentless": 25488, "ĠCovenant": 25489, "apesh": 25490, "ĠTrop": 25491, "Ty": 25492, "production": 25493, "arty": 25494, "Ġpunches": 25495, "ako": 25496, "cyclopedia": 25497, "ĠRabbit": 25498, "ĠHDMI": 25499, "Ġ141": 25500, "Ġfoil": 25501, "ItemImage": 25502, "ĠFG": 25503, "Ġimplementations": 25504, "ĠPom": 25505, "ixtures": 25506, "Ġawait": 25507, "Ġ330": 25508, "amus": 25509, "Ġumbrella": 25510, "Ġforesee": 25511, "separ": 25512, "Ġcircumcision": 25513, "Ġperipheral": 25514, "Say": 25515, "ĠExpert": 25516, "Inc": 25517, "Ġwithdrew": 25518, "ĠAnders": 25519, "fried": 25520, "Ġradioactive": 25521, "ĠOpening": 25522, "Ġboarding": 25523, "ĠND": 25524, "Ġoverthrow": 25525, "Activ": 25526, "WP": 25527, "ĠActs": 25528, "×Ļ": 25529, "Ġmotions": 25530, "vic": 25531, "ĠMighty": 25532, "ĠDefender": 25533, "aer": 25534, "Ġthankful": 25535, "ĠKilling": 25536, "ĠBris": 25537, "moil": 25538, "Ġpredicting": 25539, "266": 25540, "choice": 25541, "Ġkillers": 25542, "Ġincub": 25543, "ĠChest": 25544, "athering": 25545, "Ġproclaimed": 25546, "flower": 25547, "ossom": 25548, "umbledore": 25549, "ĠCycling": 25550, "ĠOccupy": 25551, "AGES": 25552, "Pen": 25553, "ĠYug": 25554, "Ġpackaged": 25555, "Ġheightened": 25556, "cot": 25557, "stack": 25558, "Cond": 25559, "Ġstamps": 25560, "mage": 25561, "Ġpersuaded": 25562, "Ġensl": 25563, "ĠCardinal": 25564, "Ġsolitary": 25565, "Ġpossessing": 25566, "ĠCork": 25567, "Ġevid": 25568, "ĠTay": 25569, "Ġblues": 25570, "Ġextremism": 25571, "Ġlunar": 25572, "Ġclown": 25573, "Techn": 25574, "Ġfestivals": 25575, "ĠPvP": 25576, "ĠLar": 25577, "Ġconsequently": 25578, "present": 25579, "Ġsomeday": 25580, "çİĭ": 25581, "ĠMeteor": 25582, "Ġtouring": 25583, "culture": 25584, "Ġbeaches": 25585, "Ship": 25586, "cause": 25587, "ĠFlood": 25588, "ãĥ¯": 25589, "Ġpurity": 25590, "those": 25591, "Ġemission": 25592, "bolt": 25593, "Ġchord": 25594, "ĠScripture": 25595, "Lu": 25596, "Ġ${": 25597, "created": 25598, "Others": 25599, "258": 25600, "Ġelemental": 25601, "Ġannoyed": 25602, "ĠAE": 25603, "dan": 25604, "ĠSag": 25605, "Researchers": 25606, "Ġfairy": 25607, "âĢĵâĢĵ": 25608, "============": 25609, "Smart": 25610, "GGGG": 25611, "Ġskeletons": 25612, "Ġpupils": 25613, "linked": 25614, "Ġurgency": 25615, "enabled": 25616, "ĠFuck": 25617, "Ġcouncill": 25618, "rab": 25619, "UAL": 25620, "TI": 25621, "Ġlifes": 25622, "Ġconfessed": 25623, "Bug": 25624, "Ġharmon": 25625, "ĠCONFIG": 25626, "ĠNeutral": 25627, "Double": 25628, "Ġstaple": 25629, "ĠSHA": 25630, "British": 25631, "ĠSNP": 25632, "ATOR": 25633, "oco": 25634, "Ġswinging": 25635, "gex": 25636, "oleon": 25637, "plain": 25638, "ĠMissing": 25639, "ĠTrophy": 25640, "vari": 25641, "ranch": 25642, "Ġ301": 25643, "440": 25644, "0000000000000000": 25645, "Ġrestoring": 25646, "Ġhaul": 25647, "ucing": 25648, "nerg": 25649, "Ġfutures": 25650, "Ġstrategist": 25651, "question": 25652, "Ġlateral": 25653, "ĠBard": 25654, "Ġsor": 25655, "ĠRhodes": 25656, "ĠDowntown": 25657, "?????-": 25658, "ĠLit": 25659, "ĠBened": 25660, "Ġcoil": 25661, "street": 25662, "ĠPortal": 25663, "FILE": 25664, "ĠGru": 25665, "*,": 25666, "231": 25667, "neum": 25668, "Ġsucked": 25669, "Ġrapper": 25670, "Ġtendencies": 25671, "ĠLauren": 25672, "cellaneous": 25673, "267": 25674, "Ġbrowse": 25675, "Ġoverc": 25676, "header": 25677, "oise": 25678, "Ġbeet": 25679, "ĠGle": 25680, "Stay": 25681, "Ġmum": 25682, "Ġtyped": 25683, "Ġdiscounts": 25684, "Talk": 25685, "ĠOg": 25686, "existing": 25687, "ĠSell": 25688, "uph": 25689, "CI": 25690, "ĠAustrian": 25691, "ĠWarm": 25692, "Ġdismissal": 25693, "Ġaverages": 25694, "camera": 25695, "Ġallegiance": 25696, "LAN": 25697, "=\"#": 25698, "Ġcommentators": 25699, "ĠSetting": 25700, "ĠMidwest": 25701, "Ġpharmac": 25702, "ĠEXP": 25703, "Ġstainless": 25704, "Chicago": 25705, "Ġtan": 25706, "244": 25707, "Ġcountryside": 25708, "ĠVac": 25709, "295": 25710, "Ġpinned": 25711, "Ġcrises": 25712, "Ġstandardized": 25713, "Task": 25714, "ĠJail": 25715, "ĠDocker": 25716, "colored": 25717, "forth": 25718, "\"},": 25719, "Ġpatrons": 25720, "Ġspice": 25721, "Ġmourn": 25722, "ĠMood": 25723, "Ġlaundry": 25724, "Ġequip": 25725, "ĠMole": 25726, "yll": 25727, "ĠTHC": 25728, "nation": 25729, "ĠSherlock": 25730, "Ġissu": 25731, "ĠKre": 25732, "ĠAmericas": 25733, "ĠAAA": 25734, "Ġsystematically": 25735, "Ġcontra": 25736, "ĠSally": 25737, "Ġrationale": 25738, "Ġcarriage": 25739, "Ġpeaks": 25740, "Ġcontradiction": 25741, "ensation": 25742, "ĠFailure": 25743, "Ġprops": 25744, "Ġnamespace": 25745, "Ġcove": 25746, "fields": 25747, "ãĤĭ": 25748, "Ġwool": 25749, "ĠCatch": 25750, "Ġpresumed": 25751, "ĠDiana": 25752, "ragon": 25753, "igi": 25754, "Ġhamm": 25755, "Ġstunt": 25756, "ĠGUI": 25757, "ĠObservatory": 25758, "ĠShore": 25759, "Ġsmells": 25760, "annah": 25761, "Ġcockpit": 25762, "ĠDuterte": 25763, "850": 25764, "Ġoppressed": 25765, "breaker": 25766, "ĠContribut": 25767, "ĠPeru": 25768, "ĠMonsanto": 25769, "ĠAttempt": 25770, "Ġcommanding": 25771, "Ġfridge": 25772, "ĠRin": 25773, "ĠChess": 25774, "uality": 25775, "Ġol": 25776, "Republican": 25777, "ĠGlory": 25778, "ĠWIN": 25779, ".......": 25780, "agent": 25781, "reading": 25782, "Ġinh": 25783, "Jones": 25784, "Ġclicks": 25785, "alan": 25786, "Ġ[];": 25787, "ĠMajesty": 25788, "ĠCed": 25789, "opus": 25790, "atel": 25791, "ê": 25792, "ARC": 25793, "ĠEcuador": 25794, "ãĥł": 25795, "ĠKuro": 25796, "Ġrituals": 25797, "Ġcaptive": 25798, "Ġounce": 25799, "Ġdisagreement": 25800, "Ġslog": 25801, "fuel": 25802, "Pet": 25803, "Mail": 25804, "Ġexercised": 25805, "Ġsolic": 25806, "Ġrainfall": 25807, "Ġdevotion": 25808, "ĠAssessment": 25809, "Ġrobotic": 25810, "options": 25811, "ĠRP": 25812, "ĠFamilies": 25813, "ĠFlames": 25814, "Ġassignments": 25815, "007": 25816, "akedown": 25817, "Ġvocabulary": 25818, "Reilly": 25819, "Ġcaval": 25820, "gars": 25821, "Ġsuppressed": 25822, "ĠSET": 25823, "ĠJohns": 25824, "Ġwarp": 25825, "broken": 25826, "Ġstatues": 25827, "Ġadvocated": 25828, "Ġ275": 25829, "Ġperil": 25830, "omorph": 25831, "ĠFemin": 25832, "perfect": 25833, "Ġhatch": 25834, "Lib": 25835, "512": 25836, "Ġlifelong": 25837, "313": 25838, "Ġcheeks": 25839, "Ġnumbered": 25840, "ĠMug": 25841, "Body": 25842, "ravel": 25843, "Weight": 25844, "ĠJak": 25845, "ĠHeath": 25846, "Ġkissing": 25847, "ĠJUST": 25848, "Ġwaving": 25849, "upload": 25850, "Ġinsider": 25851, "ĠProgressive": 25852, "ĠFilter": 25853, "tta": 25854, "ĠBeam": 25855, "Ġviolently": 25856, "ipation": 25857, "Ġskepticism": 25858, "Ġ1918": 25859, "ĠAnnie": 25860, "ĠSI": 25861, "Ġgenetics": 25862, "Ġonboard": 25863, "atl": 25864, "ĠFriedman": 25865, "ĠBri": 25866, "ceptive": 25867, "Ġpirate": 25868, "ĠReporter": 25869, "278": 25870, "Ġmythology": 25871, "Ġeclipse": 25872, "Ġskins": 25873, "Ġglyph": 25874, "ingham": 25875, "Files": 25876, "Cour": 25877, "women": 25878, "Ġregimes": 25879, "Ġphotographed": 25880, "Kat": 25881, "ĠMAX": 25882, "Officials": 25883, "Ġunexpectedly": 25884, "Ġimpressions": 25885, "Front": 25886, ";;;;;;;;": 25887, "Ġsupremacy": 25888, "Ġsang": 25889, "Ġaggravated": 25890, "Ġabruptly": 25891, "ĠSector": 25892, "Ġexcuses": 25893, "Ġcosting": 25894, "idepress": 25895, "Stack": 25896, "ĠRNA": 25897, "obil": 25898, "Ġghosts": 25899, "ldon": 25900, "atibility": 25901, "Topics": 25902, "Ġreimburse": 25903, "ĠHM": 25904, "ĠDeg": 25905, "Ġthief": 25906, "yet": 25907, "ogenesis": 25908, "leaning": 25909, "ĠKol": 25910, "ĠBasketball": 25911, "Ġfi": 25912, "ĠSeeing": 25913, "Ġrecycling": 25914, "Ġ[-": 25915, "Congress": 25916, "Ġlectures": 25917, "Psy": 25918, "Ġnep": 25919, "Ġmaid": 25920, "Ġoriented": 25921, "AX": 25922, "Ġrespectful": 25923, "rene": 25924, "flush": 25925, "ĠUnloaded": 25926, "request": 25927, "grid": 25928, "ĠAlternatively": 25929, "ĠHugo": 25930, "Ġdecree": 25931, "ĠBuddhism": 25932, "andum": 25933, "Android": 25934, "ĠCongo": 25935, "ĠJoyce": 25936, "Ġacknowledging": 25937, "hesive": 25938, "ĠTomorrow": 25939, "ĠHiro": 25940, "thren": 25941, "ĠMaced": 25942, "Ġhoax": 25943, "ĠIncreased": 25944, "ĠPradesh": 25945, "Wild": 25946, "______": 25947, "161": 25948, "Ġaunt": 25949, "Ġdistributing": 25950, "ĠTucker": 25951, "ĠSSL": 25952, "ĠWolves": 25953, "Building": 25954, "oult": 25955, "ĠLuo": 25956, "ĠYas": 25957, "ĠSpir": 25958, "ĠShape": 25959, "ĠCambod": 25960, "ĠIPv": 25961, "Ġml": 25962, "Ġextrad": 25963, "390": 25964, "ĠPenny": 25965, "dream": 25966, "Ġstationed": 25967, "optional": 25968, "eworthy": 25969, ".": 26700, "ĠWorkshop": 26701, "ĠRetail": 26702, "ĠAvatar": 26703, "625": 26704, "Na": 26705, "ĠVC": 26706, "ĠSecure": 26707, "MY": 26708, "1988": 26709, "ossip": 26710, "Ġprostate": 26711, "Ġunden": 26712, "Ġgamer": 26713, "ĠContents": 26714, "ĠWarhammer": 26715, "ĠSentinel": 26716, "310": 26717, "Ġsegregation": 26718, "ĠFlex": 26719, "ĠMAY": 26720, "Ġdrills": 26721, "ĠDrugs": 26722, "Islamic": 26723, "Ġspur": 26724, "Ġcafe": 26725, "Ġimaginary": 26726, "Ġguiding": 26727, "Ġswings": 26728, "ĠTheme": 26729, "oby": 26730, "Ġnud": 26731, "Ġbegging": 26732, "Ġstrongh": 26733, "Ġrejecting": 26734, "Ġpedestrians": 26735, "ĠProspect": 26736, "Rare": 26737, "sle": 26738, "Ġconcessions": 26739, "ĠConstitutional": 26740, "Ġbeams": 26741, "Ġfibers": 26742, "poon": 26743, "Ġinstincts": 26744, "property": 26745, "ĠBIG": 26746, "Sanders": 26747, "imates": 26748, "Ġcoating": 26749, "Ġcorpses": 26750, "ĠTRUE": 26751, "checked": 26752, "Ġ166": 26753, "Ash": 26754, "ĠJS": 26755, "ĠFiction": 26756, "Ġcommunal": 26757, "Ġenergetic": 26758, "oooooooo": 26759, "Ġnowadays": 26760, "ILD": 26761, "ibo": 26762, "ĠSUV": 26763, "Ren": 26764, "Ġdwelling": 26765, "Silver": 26766, "Ġtally": 26767, "ĠMoving": 26768, "Ġcoward": 26769, "Ġgenerals": 26770, "Ġhorns": 26771, "Ġcirculated": 26772, "Ġrobbed": 26773, "ĠUnlimited": 26774, "Ġharassed": 26775, "Ġinhibit": 26776, "Ġcomposer": 26777, "ĠSpotify": 26778, "Ġspreads": 26779, "364": 26780, "Ġsuicidal": 26781, "Ġnoises": 26782, "ĠStur": 26783, "Ġsaga": 26784, "ĠKag": 26785, "iso": 26786, "Ġtheoretically": 26787, "Money": 26788, "Ġsimilarity": 26789, "Ġsliced": 26790, "utils": 26791, "inges": 26792, "\"-": 26793, "Ġanth": 26794, "Ġimped": 26795, "Module": 26796, "Throughout": 26797, "Ġmenus": 26798, "committee": 26799, "andi": 26800, "obj": 26801, "inav": 26802, "fired": 26803, "ĠAbdullah": 26804, "Ġundead": 26805, "Ġfonts": 26806, "Hold": 26807, "ENG": 26808, "Ġsustainability": 26809, "Ġflick": 26810, "Ġrazor": 26811, "ĠFest": 26812, "ĠCharacters": 26813, "Ġwording": 26814, "Ġpopulist": 26815, "Ġcriticizing": 26816, "Ġmuse": 26817, "vine": 26818, "Ġcardboard": 26819, "Ġkindly": 26820, "Ġfringe": 26821, "ĠTheft": 26822, "icultural": 26823, "Ġgovernors": 26824, "Ġ����": 26825, "Ġ163": 26826, "Ġtimeout": 26827, "ĠAuth": 26828, "Children": 26829, "AU": 26830, "Ġredemption": 26831, "ĠAlger": 26832, "Ġ1914": 26833, "Ġwaved": 26834, "Ġastronauts": 26835, "ograms": 26836, "Ġswamp": 26837, "ĠFinnish": 26838, "Ġcandle": 26839, "Ġtonnes": 26840, "utm": 26841, "Ġray": 26842, "Ġspun": 26843, "Ġfearful": 26844, "articles": 26845, "Ġcaus": 26846, "orically": 26847, "ĠRequires": 26848, "ĠGol": 26849, "Ġpope": 26850, "Ġinaugural": 26851, "Ġgle": 26852, "ADA": 26853, "ĠISIL": 26854, "ĠOffensive": 26855, "Ġwatchdog": 26856, "Ġbalcon": 26857, "entity": 26858, "ĠHoo": 26859, "Ġgallon": 26860, "ACC": 26861, "Ġdoubling": 26862, "Ġimplication": 26863, "ĠSight": 26864, "Ġdoctr": 26865, "-------": 26866, "Ġ\\\\": 26867, "Ġmalt": 26868, "Roll": 26869, "Ġâī¥": 26870, "Ġrecap": 26871, "adding": 26872, "uces": 26873, "ĠBend": 26874, "figure": 26875, "Ġturkey": 26876, "Ġsocietal": 26877, "ĠTickets": 26878, "Ġcommercially": 26879, "Ġspicy": 26880, "Ġ216": 26881, "ĠRamp": 26882, "Ġsuperiority": 26883, "ï": 26884, "ĠTracker": 26885, "Carl": 26886, "ĠCoy": 26887, "ĠPatriot": 26888, "Ġconsulted": 26889, "Ġlistings": 26890, "Ġslew": 26891, "reenshot": 26892, "ĠGone": 26893, "Ġ[...]": 26894, "309": 26895, "Ġhottest": 26896, "ر": 26897, "Ġrocky": 26898, "ĠDiaz": 26899, "Ġmassage": 26900, "Ġparaly": 26901, "Ġpony": 26902, "Az": 26903, "Ġcartridge": 26904, "ĠNZ": 26905, "Ġsnack": 26906, "ĠLamar": 26907, "plement": 26908, "ĠLeslie": 26909, "Ġmater": 26910, "Ġsnipp": 26911, "246": 26912, "Ġjointly": 26913, "ĠBrisbane": 26914, "ĠiPod": 26915, "Ġpumping": 26916, "Ġgoat": 26917, "ĠSharon": 26918, "ealing": 26919, "Ġcoron": 26920, "Ġanomal": 26921, "rahim": 26922, "ĠConnection": 26923, "Ġsculpture": 26924, "Ġscheduling": 26925, "ĠDaddy": 26926, "athing": 26927, "Ġeyebrows": 26928, "Ġcurved": 26929, "Ġsentiments": 26930, "Ġdrafting": 26931, "Drop": 26932, "([": 26933, "Ġnominal": 26934, "ĠLeadership": 26935, "ĠGrow": 26936, "Ġ176": 26937, "Ġconstructive": 26938, "ivation": 26939, "Ġcorrupted": 26940, "gerald": 26941, "ĠCros": 26942, "ĠChester": 26943, "ĠLap": 26944, "ãģª": 26945, "OTH": 26946, "DATA": 26947, "Ġalmond": 26948, "probably": 26949, "Imp": 26950, "Ġfeast": 26951, "ĠWarcraft": 26952, "Flor": 26953, "Ġcheckpoint": 26954, "Ġtranscription": 26955, "Ġ204": 26956, "Ġtweaks": 26957, "Ġrelieve": 26958, "Science": 26959, "Ġperformer": 26960, "Zone": 26961, "Ġturmoil": 26962, "igated": 26963, "hibit": 26964, "ĠCafe": 26965, "themed": 26966, "Ġfluor": 26967, "bench": 26968, "Ġdecom": 26969, "ĠUnt": 26970, "ĠBarrett": 26971, "ĠFacts": 26972, "Ġtasting": 26973, "ĠPTSD": 26974, "ĠSeal": 26975, "ĠJudaism": 26976, "ĠDynamic": 26977, "ĠCors": 26978, "Ve": 26979, "ĠMing": 26980, "ĠTransform": 26981, "von": 26982, "ĠDefenders": 26983, "ĠTactical": 26984, "ĠVon": 26985, "ĠUnivers": 26986, "Ġdistorted": 26987, "ĠBreath": 26988, "?'\"": 26989, "Ġagon": 26990, "ĠDeadly": 26991, "Ġlan": 26992, "ĠCycle": 26993, "orned": 26994, "Ġreliably": 26995, "Ġglor": 26996, "ĠMonkey": 26997, "ãĥ¡": 26998, "Ġadren": 26999, "Ġmicrowave": 27000, "ĠAlban": 27001, "ircraft": 27002, "digit": 27003, "smart": 27004, "ĠDread": 27005, "¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯": 27006, "{{": 27007, "ĠRochester": 27008, "Ġsimplified": 27009, "Ġinflicted": 27010, "Ġtakeover": 27011, "Ġyourselves": 27012, "aditional": 27013, "Ġmuscular": 27014, "KS": 27015, "Ġingen": 27016, "Tax": 27017, "ĠFeature": 27018, "277": 27019, "Ġcruc": 27020, "Ġcrate": 27021, "Ġunidentified": 27022, "Ġacclaimed": 27023, "ĠManga": 27024, "ĠFrances": 27025, "ĠNepal": 27026, "ĠGerald": 27027, "ĠKuwait": 27028, "Ġslain": 27029, "ĠHeb": 27030, "ĠGoku": 27031, "ãģ®æ": 27032, "286": 27033, "Mrs": 27034, "ĠCody": 27035, "ĠSanctuary": 27036, "016": 27037, "Ġdismant": 27038, "Ġdataset": 27039, "ĠHond": 27040, "buck": 27041, "ĠPatterson": 27042, "Ġpalette": 27043, "ĠGD": 27044, "icol": 27045, "ĠLodge": 27046, "Ġplanetary": 27047, "akin": 27048, "ĠRegistered": 27049, "abwe": 27050, "ĠPetersburg": 27051, "Ġhailed": 27052, "ĠPiece": 27053, "Sche": 27054, "ĠDOJ": 27055, "Ġenumer": 27056, "181": 27057, "ĠObserver": 27058, "ĠBold": 27059, "founded": 27060, "commerce": 27061, "Ġexploits": 27062, "ĠFinding": 27063, "URN": 27064, "ĠSne": 27065, "ĠAcid": 27066, "ayette": 27067, "ĠValues": 27068, "Ġdrastic": 27069, "Ġarchitectural": 27070, "Ġ\".": 27071, "×ķ": 27072, "umped": 27073, "Ġwrapping": 27074, "Ġwidow": 27075, "ĠSlayer": 27076, "lace": 27077, "once": 27078, "Germany": 27079, "avoid": 27080, "Ġtemples": 27081, "PAR": 27082, "ô": 27083, "ĠLucifer": 27084, "ĠFlickr": 27085, "lov": 27086, "forces": 27087, "Ġscouting": 27088, "Ġlouder": 27089, "tesy": 27090, "Ġbeforehand": 27091, "Äĵ": 27092, "ĠNeon": 27093, "ĠWol": 27094, "ĠTypically": 27095, "ĠPolitico": 27096, "-+-+": 27097, "Ġbuilder": 27098, "Ġderive": 27099, "Kill": 27100, "Ġpoker": 27101, "Ġambiguous": 27102, "Ġlifts": 27103, "Ġcyt": 27104, "Ġribs": 27105, "oodle": 27106, "ĠSounds": 27107, "hair": 27108, "ĠSyndrome": 27109, "tf": 27110, "Ġproportional": 27111, "uid": 27112, "Ġpertaining": 27113, "ĠKindle": 27114, "ĠNegro": 27115, "Ġreiterated": 27116, "ĠTonight": 27117, "oths": 27118, "ĠCornell": 27119, "Ġowing": 27120, "Ġ208": 27121, "elfare": 27122, "ocating": 27123, "ĠBirds": 27124, "Subscribe": 27125, "Ġessays": 27126, "Ġburdens": 27127, "Ġillustrations": 27128, "arious": 27129, "ERAL": 27130, "ĠCalcul": 27131, "Ġxen": 27132, "ĠLinkedIn": 27133, "ĠJung": 27134, "Ġredesign": 27135, "Connor": 27136, "296": 27137, "Ġreversal": 27138, "ĠAdelaide": 27139, "ĠLL": 27140, "Ġsinking": 27141, "Ġgum": 27142, "USH": 27143, "capt": 27144, "ĠGrimm": 27145, "Ġfootsteps": 27146, "ĠCBD": 27147, "ispers": 27148, "Ġprose": 27149, "Wednesday": 27150, "ĠMovies": 27151, "edin": 27152, "Ġoverturned": 27153, "Ġcontentious": 27154, "USB": 27155, "~~~~~~~~~~~~~~~~": 27156, "ĠCopper": 27157, "Ġpointless": 27158, "NV": 27159, "values": 27160, "olphin": 27161, "dain": 27162, "Ġdeposited": 27163, "ĠGW": 27164, "Ġpreceded": 27165, "ĠCla": 27166, "ĠGolem": 27167, "ĠNim": 27168, "Ġβ": 27169, "ĠEngineers": 27170, "middle": 27171, "Ġflatt": 27172, "operative": 27173, "Ġcouncils": 27174, "imbabwe": 27175, "elin": 27176, "Ġstressful": 27177, "ĠLD": 27178, "Ġresh": 27179, "lake": 27180, "Ġwheelchair": 27181, "ĠAlternative": 27182, "Ġoptimize": 27183, "operation": 27184, "Ġpeek": 27185, "Ġoneself": 27186, "igil": 27187, "Ġtransitions": 27188, "opathy": 27189, "blank": 27190, "Ġ169": 27191, "171": 27192, "________________________________________________________________": 27193, "Ġlaundering": 27194, "Enc": 27195, "ĠDEC": 27196, "Ġworkouts": 27197, "Ġspikes": 27198, "Ġdinosaurs": 27199, "Ġdiscriminatory": 27200, "Pool": 27201, "Rather": 27202, "385": 27203, "RNA": 27204, "testers": 27205, "eto": 27206, "ĠIdentity": 27207, "Ġvein": 27208, "ĠBurton": 27209, "Ġarcade": 27210, "420": 27211, "Ultimately": 27212, "ĠSadly": 27213, "ð": 27214, "pill": 27215, "Ġcubic": 27216, "ĠSpectrum": 27217, "these": 27218, "states": 27219, "Ġunofficial": 27220, "hawks": 27221, "ĠEVERY": 27222, "Ġrainbow": 27223, "Ġincarceration": 27224, "anding": 27225, "Ġsyll": 27226, "ĠEverton": 27227, "Ġ179": 27228, "ĠSerbia": 27229, "Ġ189": 27230, "meter": 27231, "ĠMickey": 27232, "Ġantiqu": 27233, "Ġfactual": 27234, "neck": 27235, "ĠNare": 27236, "norm": 27237, "must": 27238, "Ġhighways": 27239, "Ġglam": 27240, "Ġdividing": 27241, "ĠSquadron": 27242, "ĠMartha": 27243, "Ġbirths": 27244, "Cover": 27245, "////////////////": 27246, "ĠWong": 27247, "Phot": 27248, "ĠALS": 27249, "rio": 27250, "ĠNonetheless": 27251, "ĠLemon": 27252, "Ġ206": 27253, "ĠEE": 27254, "Ġderivative": 27255, "ĠWWII": 27256, "vote": 27257, "Ġtherein": 27258, "Ġseparating": 27259, "446": 27260, "sync": 27261, "ĠStreets": 27262, "Ġratt": 27263, "Ġmunicipality": 27264, "ĠShortly": 27265, "Ġmonk": 27266, "),\"": 27267, "Ġscrub": 27268, "Ġoperatives": 27269, "Neither": 27270, "Place": 27271, "ĠLimit": 27272, "Female": 27273, "ĠActor": 27274, "Character": 27275, "Ġconstituted": 27276, "357": 27277, "Ġprotested": 27278, "ĠStraw": 27279, "ĠHeight": 27280, "ilda": 27281, "ĠTyph": 27282, "Ġfloods": 27283, "Ġcosmetic": 27284, "WAY": 27285, "perture": 27286, "upon": 27287, "tons": 27288, "essing": 27289, "ĠPocket": 27290, "Ġrooft": 27291, "ĠCaucas": 27292, "Ġantidepress": 27293, "Ġincompatible": 27294, "ECD": 27295, "Ġopera": 27296, "ĠContest": 27297, "Ġgenerators": 27298, "lime": 27299, "Defense": 27300, "1987": 27301, "forum": 27302, "Ġsavage": 27303, "ĠHungarian": 27304, "nz": 27305, "Ġmetallic": 27306, "Ġexpelled": 27307, "Ġresidency": 27308, "Ġdresses": 27309, "666": 27310, "ĠClement": 27311, "fires": 27312, "Category": 27313, "Ġgeek": 27314, "alis": 27315, "Ġcemetery": 27316, "educated": 27317, "Ġcrawl": 27318, "ĠUnable": 27319, "ĠTyson": 27320, "akis": 27321, "Ġpardon": 27322, "ĠWra": 27323, "Ġstrengthened": 27324, "ĠFors": 27325, "335": 27326, "ĠHC": 27327, "ĠMond": 27328, "Ġvisuals": 27329, "ĠBeatles": 27330, "ettlement": 27331, "Ġï": 27332, "gro": 27333, "Ġbash": 27334, "Ġpoorest": 27335, "Ġexcel": 27336, "Ġaspirations": 27337, "ĠMunicip": 27338, "ensible": 27339, "Ġceremonies": 27340, "Ġintimidation": 27341, "ĠCONTR": 27342, "beck": 27343, "ĠKap": 27344, "asu": 27345, "Ġtrademarks": 27346, "ĠSew": 27347, "ĠCompetition": 27348, "network": 27349, "ĠArri": 27350, "ĠTet": 27351, "Roaming": 27352, "WC": 27353, "Dat": 27354, "Ġsob": 27355, "Ġpairing": 27356, "Ġoverdose": 27357, "SAY": 27358, "aber": 27359, "Ġrevolt": 27360, "ĠFah": 27361, "acting": 27362, "eq": 27363, "estation": 27364, "Fight": 27365, "ĠMarks": 27366, "273": 27367, "Ġ178": 27368, "Raw": 27369, "ãģĭ": 27370, "349": 27371, "blocks": 27372, "Ġverge": 27373, "estine": 27374, "ĠPodesta": 27375, "Ġinvasive": 27376, "Ġprofoundly": 27377, "ĠAo": 27378, "each": 27379, "Ġlest": 27380, "interpret": 27381, "Ġshrinking": 27382, "Ġerrone": 27383, "Ġchees": 27384, "lys": 27385, "ĠIvy": 27386, "ĠDirectory": 27387, "Ġhinted": 27388, "VICE": 27389, "Ġcontacting": 27390, "ĠGent": 27391, "hei": 27392, "Ġlabeling": 27393, "Ġmercury": 27394, "ĠLite": 27395, "Ġexpires": 27396, "Ġdestabil": 27397, "ritis": 27398, "cu": 27399, "Ġfeathers": 27400, "Ġsteer": 27401, "Ġprogrammed": 27402, "ĠVader": 27403, "Going": 27404, "ĠElim": 27405, "Ġyo": 27406, "ĠMiche": 27407, "Ġ203": 27408, "Ġsleeves": 27409, "Ġbully": 27410, "ĠHumans": 27411, "368": 27412, "Ġcompress": 27413, "ĠBanner": 27414, "ARS": 27415, "Ġawhile": 27416, "Ġcalib": 27417, "Ġsponsorship": 27418, "ĠDifficulty": 27419, "ĠPapers": 27420, "Ġidentifier": 27421, "}.": 27422, "Ġyog": 27423, "ĠShia": 27424, "Ġcleanup": 27425, "Ġvibe": 27426, "introdu": 27427, "imming": 27428, "Australia": 27429, "Ġoutlines": 27430, "ĠYoutube": 27431, "train": 27432, "ĠMakes": 27433, "Ġdeported": 27434, "Ġcentr": 27435, "ĠDug": 27436, "ĠBoulder": 27437, "ĠBuffy": 27438, "Ġinjunction": 27439, "ĠHarley": 27440, "ĠGroups": 27441, "ĠDumbledore": 27442, "ĠClara": 27443, "Ġ\"-": 27444, "Ġsacrificed": 27445, "eph": 27446, "Shadow": 27447, "ibling": 27448, "Ġfreelance": 27449, "Ġevidently": 27450, "phal": 27451, "Ġretains": 27452, "Mir": 27453, "Ġfinite": 27454, "dar": 27455, "ĠCous": 27456, "Ġrepaired": 27457, "Ġperiodic": 27458, "Ġchampionships": 27459, "Ġasteroid": 27460, "blind": 27461, "Ġexpressly": 27462, "ĠAstros": 27463, "Ġscaled": 27464, "Ġgeographical": 27465, "ĠRapids": 27466, "Enjoy": 27467, "Ġelastic": 27468, "ĠMohamed": 27469, "Market": 27470, "begin": 27471, "Ġdiscovers": 27472, "Ġtelecommunications": 27473, "Ġscanner": 27474, "Ġenlarge": 27475, "Ġsharks": 27476, "Ġpsychedel": 27477, "ĠRouge": 27478, "Ġsnapshot": 27479, "isine": 27480, "XP": 27481, "Ġpesticides": 27482, "ĠLSD": 27483, "ĠDistribution": 27484, "really": 27485, "Ġdegradation": 27486, "Ġdisguise": 27487, "Ġbiom": 27488, "ĠEXT": 27489, "Ġequations": 27490, "Ġhazards": 27491, "ĠCompared": 27492, ")*": 27493, "Ġvirtues": 27494, "Ġelders": 27495, "Ġenhancing": 27496, "ĠAcross": 27497, "eros": 27498, "angling": 27499, "Ġcombust": 27500, "ucci": 27501, "Ġconcussion": 27502, "Ġcontraception": 27503, "ĠKang": 27504, "Ġexpresses": 27505, "Ġaux": 27506, "ĠPione": 27507, "Ġexhibits": 27508, "Debug": 27509, "OTAL": 27510, "ĠAlready": 27511, "ĠWheeler": 27512, "Ġexpands": 27513, "?:": 27514, "Ġreconciliation": 27515, "Ġpirates": 27516, "Ġpurse": 27517, "Ġdiscourage": 27518, "Ġspectacle": 27519, "Rank": 27520, "Ġwraps": 27521, "ĠThought": 27522, "Ġimpending": 27523, "Opp": 27524, "ĠAnglo": 27525, "ĠEUR": 27526, "Ġscrewed": 27527, "retched": 27528, "Ġencouragement": 27529, "models": 27530, "Ġconfuse": 27531, "mmm": 27532, "ĠVitamin": 27533, "âĸijâĸij": 27534, "Cru": 27535, "Ġknights": 27536, "Ġdiscard": 27537, "Ġbishops": 27538, "ĠWear": 27539, "ĠGarrett": 27540, "kan": 27541, "ãĥŁ": 27542, "Ġmasculine": 27543, "capital": 27544, "ĠAus": 27545, "Ġfatally": 27546, "thanks": 27547, "ĠAU": 27548, "ĠGut": 27549, "1200": 27550, "Ġ00000000": 27551, "Ġsurrog": 27552, "ĠBIOS": 27553, "raits": 27554, "ĠWatts": 27555, "Ġresurrection": 27556, "ĠElectoral": 27557, "ĠTips": 27558, "4000": 27559, "Ġnutrient": 27560, "Ġdepicting": 27561, "Ġsprink": 27562, "Ġmuff": 27563, "ĠLIM": 27564, "ĠSample": 27565, "psc": 27566, "ibi": 27567, "generated": 27568, "Ġspecimens": 27569, "Ġdissatisf": 27570, "Ġtailored": 27571, "Ġholdings": 27572, "ĠMonthly": 27573, "ĠEat": 27574, "poons": 27575, "Ġnec": 27576, "ĠCage": 27577, "ĠLotus": 27578, "ĠLantern": 27579, "Ġfrontier": 27580, "Ġpensions": 27581, "Ġjoked": 27582, "ĠHardy": 27583, "=-=-=-=-": 27584, "rade": 27585, "UID": 27586, "Ġrails": 27587, "Ġemit": 27588, "Ġslate": 27589, "Ġsmug": 27590, "Ġspit": 27591, "ĠCalls": 27592, "ĠJacobs": 27593, "feat": 27594, "ĠUE": 27595, "Ġrestruct": 27596, "Ġregeneration": 27597, "Ġenergies": 27598, "ĠConnor": 27599, "OHN": 27600, "ĠCheese": 27601, "Ġger": 27602, "Ġresurrect": 27603, "management": 27604, "NW": 27605, "Ġpresently": 27606, "ĠBruins": 27607, "Member": 27608, "ĠMang": 27609, "idan": 27610, "Ġboosting": 27611, "wyn": 27612, "+.": 27613, "requisite": 27614, "ĠNYPD": 27615, "ĠMegan": 27616, "ĠConditions": 27617, "Ġpics": 27618, "nesium": 27619, "ĠRash": 27620, "Ġ174": 27621, "ĠDucks": 27622, "Ġembro": 27623, "zu": 27624, "onian": 27625, "religious": 27626, "Ġcraz": 27627, "ĠACA": 27628, "ĠZucker": 27629, "EMA": 27630, "ĠPros": 27631, "Weapon": 27632, "ĠKnox": 27633, "ĠArduino": 27634, "Ġstove": 27635, "Ġheavens": 27636, "ĠPurchase": 27637, "Ġherd": 27638, "Ġfundraiser": 27639, "Digital": 27640, "5000": 27641, "Ġproponents": 27642, "/âĢĭ": 27643, "Ġjelly": 27644, "ĠVisa": 27645, "Ġmonks": 27646, "Ġadvancement": 27647, "ĠWer": 27648, "Ġ187": 27649, "eus": 27650, "ertility": 27651, "Ġfetal": 27652, "Ġ1936": 27653, "Lo": 27654, "Ġoutfits": 27655, "Ġstaircase": 27656, "bomb": 27657, "Ġcustomized": 27658, "clair": 27659, "Tree": 27660, "Ġmapped": 27661, "ĠConsidering": 27662, "ĠTorres": 27663, "Ġmethyl": 27664, "Ġapproximate": 27665, "Ġdoom": 27666, "ĠHansen": 27667, "Ġcrossover": 27668, "Ġstandalone": 27669, "ä¼": 27670, "Ġinvites": 27671, "Ġgraveyard": 27672, "Ġhp": 27673, "DonaldTrump": 27674, "Ġescort": 27675, "Gar": 27676, "Ġpredecessors": 27677, "Ġhay": 27678, "Ġenzyme": 27679, "ĠStraight": 27680, "visors": 27681, "Ing": 27682, "aneously": 27683, "ĠApplied": 27684, "Ġfec": 27685, "ĠDurant": 27686, "Ġoutspoken": 27687, "orb": 27688, "Ġzeal": 27689, "Ġdisgrace": 27690, "').": 27691, "ĠCheng": 27692, "289": 27693, "ĠRena": 27694, "ĠSuicide": 27695, "294": 27696, "Ġoutraged": 27697, "ĠNewman": 27698, "ĠNvidia": 27699, "ĠAber": 27700, "ĠBers": 27701, "Ġrecreation": 27702, "Window": 27703, "ĠDP": 27704, "xe": 27705, "Ġpedoph": 27706, "Ġfallout": 27707, "amboo": 27708, "Ġpresentations": 27709, "ĠApps": 27710, "Ġhtml": 27711, "345": 27712, "ĠXXX": 27713, "Ġrubbing": 27714, "ĠLeather": 27715, "Ġhumidity": 27716, "seys": 27717, "established": 27718, "ĠUnits": 27719, "646": 27720, "Ġrespectable": 27721, "Auto": 27722, "Ġthriving": 27723, "ĠInnovation": 27724, "angs": 27725, "Extra": 27726, "regulation": 27727, "298": 27728, "pick": 27729, "Examples": 27730, "ĠCJ": 27731, "Attack": 27732, "Ġdracon": 27733, "LT": 27734, "Ġsticker": 27735, "rers": 27736, "Ġsunny": 27737, "Iss": 27738, "regulated": 27739, "dim": 27740, "ĠAbstract": 27741, "Ġhusbands": 27742, "Office": 27743, "omination": 27744, "itars": 27745, "ANGE": 27746, "ascal": 27747, "ĠKris": 27748, "ĠInfantry": 27749, "Ġmalf": 27750, "ĠAthe": 27751, "ĠRally": 27752, "balanced": 27753, "........................": 27754, "OUP": 27755, "Ġmolecule": 27756, "metics": 27757, "ĠSplit": 27758, "ĠInstructions": 27759, "ĠNights": 27760, "cards": 27761, "Ġtug": 27762, "Ġcone": 27763, "åŃ": 27764, "Ġtx": 27765, "ĠDiscussion": 27766, "Ġcatastrophe": 27767, "ppe": 27768, "gio": 27769, "Ġcommunism": 27770, "Ġhalted": 27771, "ĠGuant": 27772, "clean": 27773, "ĠSched": 27774, "ĠKanye": 27775, "Ġwander": 27776, "ĠSeriously": 27777, "Ġ188": 27778, "ennial": 27779, "follow": 27780, "productive": 27781, "ĠFlow": 27782, "ĠSail": 27783, "Ġcraw": 27784, "Ġsimulations": 27785, "oru": 27786, "angles": 27787, "ĠNolan": 27788, "Ġmenstru": 27789, "470": 27790, "Ġ207": 27791, "aja": 27792, "Ġcasually": 27793, "boarding": 27794, "Ġ222": 27795, "ovy": 27796, "ĠNumbers": 27797, "umat": 27798, "OE": 27799, "287": 27800, "ĠClemson": 27801, "Ġcerts": 27802, "Ġslid": 27803, "ĠTribe": 27804, "Ġtoast": 27805, "Ġfortunes": 27806, "Ġfals": 27807, "ĠCommittees": 27808, "Ġgp": 27809, "Ġfiery": 27810, "ĠNets": 27811, "ĠAnime": 27812, "Package": 27813, "ĠCompare": 27814, "laughter": 27815, "infect": 27816, "Ġatrocities": 27817, "Ġjustices": 27818, "Ġinsults": 27819, "ĠVernon": 27820, "Ġshaken": 27821, "Ġpersona": 27822, "estamp": 27823, "367": 27824, "brain": 27825, "Ġexperimenting": 27826, "Ken": 27827, "ĠElectronics": 27828, "Ġ161": 27829, "domain": 27830, "Ġgraphical": 27831, "bishop": 27832, "Ġwhopping": 27833, "ĠEvangel": 27834, "Ġadvertisers": 27835, "ĠSpear": 27836, "Ġbids": 27837, "Ġdestroys": 27838, "utz": 27839, "Ġundersc": 27840, "ĠADD": 27841, "Ġants": 27842, "ĠCum": 27843, "ipples": 27844, "ĠFill": 27845, "Ġglanced": 27846, "Ġindicted": 27847, "ĠEff": 27848, "Ġmiscon": 27849, "ĠDesktop": 27850, "Ġabide": 27851, "ãĥĢ": 27852, "ĠIo": 27853, "ĠCoul": 27854, "Ġcapsule": 27855, "ĠChrys": 27856, "MON": 27857, "Ġundes": 27858, "ĠIRA": 27859, "Ġcitation": 27860, "Ġdictate": 27861, "ĠNetworks": 27862, "ĠConflict": 27863, "ĠStuff": 27864, "xa": 27865, "isec": 27866, "ĠChemistry": 27867, "Ġquarterly": 27868, "Williams": 27869, "anan": 27870, "Opt": 27871, "ĠAlexandria": 27872, "outheastern": 27873, "ĠSpringfield": 27874, "ĠBlacks": 27875, "Ġgeography": 27876, "242": 27877, "Ġutmost": 27878, "ĠExxon": 27879, "abouts": 27880, "EVA": 27881, "ĠEnable": 27882, "ĠBarr": 27883, "Ġdisagreed": 27884, "ĠCyprus": 27885, "Ġdementia": 27886, "Ġlabs": 27887, "Ġubiquitous": 27888, "ĠLOVE": 27889, "Ġconsolidated": 27890, "sr": 27891, "Ġcreamy": 27892, "ĠTimber": 27893, "Regardless": 27894, "ĠCertificate": 27895, "Ġ\"...": 27896, "ogenous": 27897, "Captain": 27898, "Ġinsulting": 27899, "ĠSoros": 27900, "ĠInstr": 27901, "ĠBulgaria": 27902, "better": 27903, "Ġsucking": 27904, "ĠDavidson": 27905, "atz": 27906, "Ġcollateral": 27907, "gif": 27908, "Ġplagued": 27909, "ĠCancel": 27910, "ĠGardner": 27911, "RB": 27912, "Ġsixteen": 27913, "Remove": 27914, "uristic": 27915, "cook": 27916, "Rod": 27917, "Ġcomprising": 27918, "fle": 27919, ")âĢĶ": 27920, "ĠViking": 27921, "growth": 27922, "agonal": 27923, "Ġsrf": 27924, "afety": 27925, "mot": 27926, "Nearly": 27927, "stown": 27928, "ĠFactor": 27929, "Ġautomobile": 27930, "Ġprocedural": 27931, "mask": 27932, "ampires": 27933, "Ġdisappears": 27934, "jab": 27935, "315": 27936, "Ġ1951": 27937, "needed": 27938, "Ġdaring": 27939, "leader": 27940, "Ġpodium": 27941, "Ġunhealthy": 27942, "Ġmund": 27943, "Ġpyramid": 27944, "ocre": 27945, "Ġkissed": 27946, "Ġdreamed": 27947, "ĠFantastic": 27948, "ĠGly": 27949, "åĬ": 27950, "Ġgreatness": 27951, "Ġspices": 27952, "Ġmetropolitan": 27953, "Ġcompuls": 27954, "iets": 27955, "1016": 27956, "ĠSham": 27957, "ĠPyr": 27958, "flies": 27959, "ĠMidnight": 27960, "Ġswallowed": 27961, "Ġgenres": 27962, "ĠLucky": 27963, "ĠRewards": 27964, "Ġdispatch": 27965, "ĠIPA": 27966, "ĠApply": 27967, "Ġaven": 27968, "alities": 27969, "312": 27970, "things": 27971, "Ġ().": 27972, "Ġmates": 27973, "ĠSz": 27974, "ĠCOP": 27975, "olate": 27976, "OFF": 27977, "Ġrecharge": 27978, "caps": 27979, "ĠYorker": 27980, "icone": 27981, "Ġgalaxies": 27982, "ileaks": 27983, "Dave": 27984, "ĠPuzz": 27985, "ĠCeltic": 27986, "ĠAFC": 27987, "276": 27988, "ĠSons": 27989, "Ġaffirmative": 27990, "Hor": 27991, "Ġtutorials": 27992, "ĠCITY": 27993, "ĠRosa": 27994, "ĠExtension": 27995, "Series": 27996, "Ġfats": 27997, "Ġrab": 27998, "lis": 27999, "Ġunic": 28000, "Ġeve": 28001, "ĠSpin": 28002, "Ġadulthood": 28003, "typ": 28004, "Ġsectarian": 28005, "Ġcheckout": 28006, "ĠCycl": 28007, "Single": 28008, "Ġmartyr": 28009, "Ġchilling": 28010, "888": 28011, "oufl": 28012, "Ġ];": 28013, "Ġcongestion": 28014, "mk": 28015, "ĠWhereas": 28016, "Ġ1938": 28017, "urrencies": 28018, "erion": 28019, "Ġboast": 28020, "ĠPatients": 28021, "Ġchap": 28022, "ĠBD": 28023, "realDonaldTrump": 28024, "Ġexamines": 28025, "hov": 28026, "Ġstartling": 28027, "ĠBabylon": 28028, "wid": 28029, "omew": 28030, "brance": 28031, "ĠOdyssey": 28032, "wig": 28033, "Ġtorch": 28034, "ĠVox": 28035, "ĠMoz": 28036, "ĠTroll": 28037, "ĠAns": 28038, "Similarly": 28039, "ĠFul": 28040, "006": 28041, "Unless": 28042, "ĠAlone": 28043, "stead": 28044, "ĠPublisher": 28045, "rights": 28046, "tu": 28047, "ĠDoesn": 28048, "Ġprofessionally": 28049, "Ġclo": 28050, "icz": 28051, "Ġsteals": 28052, "Ġá": 28053, "1986": 28054, "Ġsturdy": 28055, "ĠJohann": 28056, "Ġmedals": 28057, "Ġfilings": 28058, "ĠFraser": 28059, "done": 28060, "Ġmultinational": 28061, "Ġfeder": 28062, "Ġworthless": 28063, "Ġpest": 28064, "Yesterday": 28065, "ankind": 28066, "Ġgays": 28067, "Ġborne": 28068, "ĠPOS": 28069, "Picture": 28070, "Ġpercentages": 28071, "251": 28072, "rame": 28073, "Ġpotions": 28074, "AMD": 28075, "ĠLebanese": 28076, "Ġrang": 28077, "ĠLSU": 28078, "ongs": 28079, "Ġpeninsula": 28080, "ĠClause": 28081, "ALK": 28082, "oha": 28083, "ĠMacBook": 28084, "Ġunanimous": 28085, "Ġlenders": 28086, "Ġhangs": 28087, "Ġfranchises": 28088, "orers": 28089, "ĠUpdates": 28090, "Ġisolate": 28091, "andro": 28092, "Soon": 28093, "Ġdisruptive": 28094, "ĠSurve": 28095, "Ġstitches": 28096, "ĠScorp": 28097, "ĠDominion": 28098, "Ġsupplying": 28099, "Arg": 28100, "Ġturret": 28101, "ĠLuk": 28102, "Ġbrackets": 28103, "*)": 28104, "ĠRevolutionary": 28105, "ĠHonest": 28106, "Ġnoticing": 28107, "ĠShannon": 28108, "Ġafforded": 28109, "Ġtha": 28110, "ĠJanet": 28111, "!--": 28112, "ĠNarendra": 28113, "ĠPlot": 28114, "Hol": 28115, "sever": 28116, "eenth": 28117, "Ġobstruction": 28118, "Ġ1024": 28119, "staff": 28120, "jas": 28121, "orget": 28122, "scenes": 28123, "laughs": 28124, "ĠFargo": 28125, "crime": 28126, "Ġorchestr": 28127, "Ġdelet": 28128, "iliary": 28129, "rieved": 28130, "Ġmilitar": 28131, "ĠGreene": 28132, "âĹı": 28133, "ãģ¦": 28134, "ĠGuards": 28135, "Ġunleashed": 28136, "ĠWeber": 28137, "Ġadjustable": 28138, "Ġcaliber": 28139, "Ġmotivations": 28140, "ĠÃł": 28141, "mAh": 28142, "ĠLanka": 28143, "handle": 28144, "Ġpent": 28145, "ĠRav": 28146, "ĠAngular": 28147, "ĠKau": 28148, "umbing": 28149, "Ġphilanthrop": 28150, "Ġdehyd": 28151, "Ġtoxicity": 28152, "eer": 28153, "ĠYORK": 28154, "witz": 28155, "å¼": 28156, "ĠIE": 28157, "community": 28158, "ĠAH": 28159, "Ġretali": 28160, "Ġmassively": 28161, "ĠDaniels": 28162, "ĠDEL": 28163, "Ġcarcin": 28164, "Url": 28165, "Ġrouting": 28166, "ĠNPCs": 28167, "ĠRAF": 28168, "ryce": 28169, "Ġwaived": 28170, "ĠGuatem": 28171, "Everybody": 28172, "Ġcovenant": 28173, "Ġ173": 28174, "Ġrelaxing": 28175, "Ġquart": 28176, "almost": 28177, "Ġguarded": 28178, "ĠSoldiers": 28179, "ĠPLAY": 28180, "Ġoutgoing": 28181, "LAND": 28182, "Ġrewrite": 28183, "ĠMOV": 28184, "ĠImper": 28185, "ĠSolution": 28186, "Ġphenomenal": 28187, "Ġlongevity": 28188, "Ġimpat": 28189, "ĠNissan": 28190, "irie": 28191, "Ġodor": 28192, "ĠZar": 28193, "oks": 28194, "Ġmilitias": 28195, "ĠSPEC": 28196, "Ġtolerated": 28197, "arser": 28198, "ĠBradford": 28199, "+,": 28200, "Ġsurreal": 28201, "sf": 28202, "Canadian": 28203, "Ġresemblance": 28204, "Ġcarbohydrate": 28205, "VIEW": 28206, "Ġaccessory": 28207, "meal": 28208, "largest": 28209, "iegel": 28210, "Someone": 28211, "Ġtoughest": 28212, "oso": 28213, "Ġfunnel": 28214, "Ġcondemnation": 28215, "luent": 28216, "Ġwired": 28217, "ĠSunset": 28218, "Jesus": 28219, "ĠPST": 28220, "ĠPages": 28221, "ĠTycoon": 28222, "ĠPF": 28223, "Ġselections": 28224, "Ġà¤": 28225, "partisan": 28226, "Ġhighs": 28227, "ĠRune": 28228, "Ġcrafts": 28229, "lead": 28230, "ĠParents": 28231, "Ġreclaim": 28232, "eker": 28233, "ĠAllied": 28234, "aeper": 28235, "Ġlooming": 28236, "Ġbeneficiaries": 28237, "ĠHull": 28238, "Students": 28239, "Jewish": 28240, "dj": 28241, "Ġpact": 28242, "template": 28243, "ĠOfficials": 28244, "ĠBaylor": 28245, "Ġhemp": 28246, "Ġyouths": 28247, "ĠLevels": 28248, "ĠXiao": 28249, "ĠChes": 28250, "Ġendeavor": 28251, "ĠRemoved": 28252, "Ġhippocamp": 28253, "Hell": 28254, "ãĤĬ": 28255, "805": 28256, "Ġdinosaur": 28257, "ĠWrath": 28258, "ĠIndonesian": 28259, "Ġcalculator": 28260, "ĠDictionary": 28261, "Ġ420": 28262, "ĠMAG": 28263, "(_": 28264, "!,": 28265, "tarians": 28266, "Ġrestricting": 28267, "racuse": 28268, "Ġweekday": 28269, "OUNT": 28270, "Ġshrugged": 28271, "leground": 28272, "Ġbald": 28273, "ĠDoctors": 28274, "Ġtouted": 28275, "ĠMaxwell": 28276, "Ġ214": 28277, "Ġdiplomat": 28278, "Ġrepression": 28279, "Ġconstituency": 28280, "vice": 28281, "ranked": 28282, "ĠNapoleon": 28283, "gang": 28284, "ĠForever": 28285, "tun": 28286, "Ġbulb": 28287, "ĠPDT": 28288, "ĠCisco": 28289, "VEN": 28290, "Ġresumed": 28291, "Steven": 28292, "ĠManitoba": 28293, "Ġfabulous": 28294, "ĠAgents": 28295, "1984": 28296, "Ġamusing": 28297, "ĠMysteries": 28298, "Ġorthodox": 28299, "floor": 28300, "Ġquestionnaire": 28301, "Ġpenetrate": 28302, "Ġfilmmakers": 28303, "ĠUnc": 28304, "Ġstamped": 28305, "Ġthirteen": 28306, "Ġoutfield": 28307, "Ġforwarded": 28308, "Ġappra": 28309, "Ġaided": 28310, "try": 28311, "Ġunfocused": 28312, "ĠLiz": 28313, "ĠWendy": 28314, "ĠScene": 28315, "Charg": 28316, "Ġrejects": 28317, "Ġleftist": 28318, "ĠProvidence": 28319, "ĠBrid": 28320, "regn": 28321, "Ġprophecy": 28322, "ĠLIVE": 28323, "499": 28324, "Ġforge": 28325, "ĠFML": 28326, "Ġintrinsic": 28327, "ĠFrog": 28328, "Ġwont": 28329, "ĠHolt": 28330, "Ġfamed": 28331, "CLUS": 28332, "aepernick": 28333, "ĠHate": 28334, "ĠCay": 28335, "Ġregistering": 28336, "ortality": 28337, "ropy": 28338, "ocalyptic": 28339, "aan": 28340, "nav": 28341, "Ġfascist": 28342, "IFIED": 28343, "Ġimplicated": 28344, "ĠResort": 28345, "ĠChandler": 28346, "ĠBrick": 28347, "Pin": 28348, "ysc": 28349, "Usage": 28350, "ĠHelm": 28351, "usra": 28352, "âĺħâĺħ": 28353, "ĠAbbas": 28354, "Ġunanimously": 28355, "Ġkeeper": 28356, "Ġaddicted": 28357, "???": 28358, "Ġhelmets": 28359, "Ġantioxid": 28360, "apsed": 28361, "808": 28362, "giene": 28363, "Ġwaits": 28364, "Ġminion": 28365, "raved": 28366, "ĠPorsche": 28367, "Ġdreaming": 28368, "Ġ171": 28369, "ĠCain": 28370, "Ġunfor": 28371, "asso": 28372, "ĠConfiguration": 28373, "kun": 28374, "hardt": 28375, "Ġnested": 28376, "ĠLDS": 28377, "LES": 28378, "Ġtying": 28379, "enos": 28380, "Ġcue": 28381, "ĠMarqu": 28382, "skirts": 28383, "Ġclicked": 28384, "Ġexpiration": 28385, "ĠAccordingly": 28386, "ĠWC": 28387, "Ġblessings": 28388, "Ġaddictive": 28389, "ĠNarr": 28390, "yx": 28391, "ĠJaguars": 28392, "Ġrents": 28393, "ĠSiber": 28394, "Ġtipped": 28395, "ousse": 28396, "ĠFitzgerald": 28397, "Ġhierarch": 28398, "outine": 28399, "Ġwavelength": 28400, ">.": 28401, "chid": 28402, "ĠProcessing": 28403, "/+": 28404, "ranking": 28405, "Easy": 28406, "ĠConstruct": 28407, "Ġtet": 28408, "insured": 28409, "HUD": 28410, "Ġquoting": 28411, "Ġcommunicated": 28412, "inx": 28413, "Ġinmate": 28414, "Ġerected": 28415, "ĠAbsolutely": 28416, "ĠSurely": 28417, "Ġunim": 28418, "ĠThrone": 28419, "heid": 28420, "Ġclaws": 28421, "Ġsuperstar": 28422, "ĠLenn": 28423, "ĠWhis": 28424, "Uk": 28425, "abol": 28426, "Ġsket": 28427, "ĠNiet": 28428, "Ġperks": 28429, "Ġaffinity": 28430, "Ġopenings": 28431, "phasis": 28432, "Ġdiscriminate": 28433, "Tip": 28434, "vc": 28435, "Ġgrinding": 28436, "ĠJenny": 28437, "Ġasthma": 28438, "holes": 28439, "ĠHomer": 28440, "Ġregisters": 28441, "ĠGlad": 28442, "Ġcreations": 28443, "Ġlithium": 28444, "Ġapplause": 28445, "until": 28446, "Justice": 28447, "ĠTurks": 28448, "Ġscandals": 28449, "Ġbake": 28450, "tank": 28451, "Mech": 28452, "ĠMeans": 28453, "ĠMaid": 28454, "Republicans": 28455, "isal": 28456, "windows": 28457, "ĠSantos": 28458, "Ġvegetation": 28459, "338": 28460, "tri": 28461, "Ġflux": 28462, "insert": 28463, "Ġclarified": 28464, "Ġmortg": 28465, "ĠChim": 28466, "ĠTort": 28467, "Ġdisclaim": 28468, "metal": 28469, "ĠAside": 28470, "Ġinduction": 28471, "Ġinfl": 28472, "Ġatheists": 28473, "amph": 28474, "Ġether": 28475, "ĠVital": 28476, "ĠBuilt": 28477, "Mind": 28478, "Ġweaponry": 28479, "SET": 28480, "Ġ186": 28481, "admin": 28482, "gam": 28483, "contract": 28484, "afa": 28485, "Ġderivatives": 28486, "Ġsnacks": 28487, "Ġchurn": 28488, "Econom": 28489, "Ġcapped": 28490, "ĠUnderstanding": 28491, "ĠHers": 28492, "ĠIz": 28493, "Ġduct": 28494, "IENT": 28495, "aughty": 28496, "ĠâľĶ": 28497, "ĠNP": 28498, "Ġsailing": 28499, "Initialized": 28500, "Ġted": 28501, "Ġreactors": 28502, "ĠLomb": 28503, "Ġchoke": 28504, "ĠWorm": 28505, "Ġadmiration": 28506, "Ġswung": 28507, "ensibly": 28508, "Ġrash": 28509, "ĠGoals": 28510, "ĠImportant": 28511, "Shot": 28512, "ĠRas": 28513, "Ġtrainers": 28514, "ĠBun": 28515, "Working": 28516, "Ġharmed": 28517, "ĠPandora": 28518, "ĠLTE": 28519, "Ġmushroom": 28520, "ĠCHAR": 28521, "ĠFee": 28522, "ĠMoy": 28523, "Born": 28524, "oliberal": 28525, "ĠMartial": 28526, "Ġgentlemen": 28527, "Ġlingering": 28528, "Official": 28529, "Ġgraffiti": 28530, "ĠNames": 28531, "Der": 28532, "Ġquint": 28533, "istrate": 28534, "azeera": 28535, "ĠNOTICE": 28536, "ĠFlorence": 28537, "Ġpayable": 28538, "Ġdepicts": 28539, "ĠSpecies": 28540, "Heart": 28541, "âĶĢâĶĢâĶĢâĶĢâĶĢâĶĢâĶĢâĶĢ": 28542, "Ġenclosed": 28543, "Increases": 28544, "Daily": 28545, "ĠLis": 28546, "Ġenactment": 28547, "ĠBacon": 28548, "ĠSteele": 28549, "demand": 28550, "Ġ183": 28551, "Ġmouths": 28552, "Ġstranded": 28553, "Ġenhancement": 28554, "011": 28555, "ĠWhats": 28556, "Ġhealed": 28557, "eny": 28558, "ĠRab": 28559, "Ġ340": 28560, "ĠLabyrinth": 28561, "roach": 28562, "ĠYosh": 28563, "ĠClippers": 28564, "Ġconcerts": 28565, "Internet": 28566, "355": 28567, "Ġstickers": 28568, "Ġtermed": 28569, "ĠAxe": 28570, "Ġgrandparents": 28571, "France": 28572, "ĠClim": 28573, "ĠUh": 28574, "ulic": 28575, "Ġthrill": 28576, "centric": 28577, "ĠOverview": 28578, "ĠConduct": 28579, "Ġsubstantive": 28580, "Ġ182": 28581, "mur": 28582, "Ġstray": 28583, "ĠCoff": 28584, "Ġrepetitive": 28585, "ĠForgotten": 28586, "Ġqualification": 28587, "ewitness": 28588, "ĠZimbabwe": 28589, "Ġsimulated": 28590, "ĠJD": 28591, "253": 28592, "ĠWare": 28593, "Ġunsc": 28594, "Times": 28595, "Ġsummons": 28596, "Ġdisconnected": 28597, "Ġ184": 28598, "cius": 28599, "ĠGujar": 28600, "odka": 28601, "Ġerase": 28602, "ĠTobacco": 28603, "elected": 28604, "Ġuncont": 28605, "ĠShepard": 28606, "ĠLamp": 28607, "Ġalerted": 28608, "Ġoperative": 28609, "arna": 28610, "uint": 28611, "Ġnegligence": 28612, "acements": 28613, "Ġsupra": 28614, "Ġprevail": 28615, "ĠShark": 28616, "Ġbelts": 28617, "ãģ«": 28618, "Ġtighter": 28619, "Engineers": 28620, "Ġinactive": 28621, "Ġexponent": 28622, "ĠWillie": 28623, "aples": 28624, "Ġheir": 28625, "ĠHits": 28626, "iann": 28627, "ĠSays": 28628, "Ġcurrents": 28629, "ĠBengal": 28630, "Ġarist": 28631, "Buffer": 28632, "Ġbreeze": 28633, "ĠWesley": 28634, "Cola": 28635, "Ġpronoun": 28636, "Ġdeed": 28637, "ĠKling": 28638, "Ġoft": 28639, "Ġinflict": 28640, "Ġpunishing": 28641, "Ġnm": 28642, "iku": 28643, "ODUCT": 28644, "014": 28645, "Ġsubsidy": 28646, "ĠDEA": 28647, "ĠHerbert": 28648, "ĠJal": 28649, "Bank": 28650, "Ġdeferred": 28651, "Ġshipment": 28652, "Bott": 28653, "Ġalle": 28654, "bearing": 28655, "HTML": 28656, "Offline": 28657, "Ġ213": 28658, "Ġscrolling": 28659, "Ġscanned": 28660, "ĠLibyan": 28661, "ĠTOP": 28662, "chrom": 28663, "dt": 28664, "column": 28665, "PsyNetMessage": 28666, "Zero": 28667, "Ġtorso": 28668, "050": 28669, "âķIJ": 28670, "Ġimperson": 28671, "ĠSchwartz": 28672, "udic": 28673, "Ġpissed": 28674, "ĠSapp": 28675, "257": 28676, "ĠISPs": 28677, "ogl": 28678, "Ġsupervised": 28679, "Ġadolescent": 28680, "Ġattained": 28681, "ĠDelivery": 28682, "ĠBunny": 28683, "Ġ1937": 28684, "Ġminiature": 28685, "Ġos": 28686, "Ġ370": 28687, "608": 28688, "ĠMourinho": 28689, "Ġinnate": 28690, "Ġtempo": 28691, "ĠNM": 28692, "ĠFallen": 28693, "009": 28694, "Ġprovocative": 28695, "Streamer": 28696, "ĠBenedict": 28697, "ĠBolshe": 28698, "Ġturtle": 28699, "ĠPCB": 28700, "ĠEqual": 28701, "Director": 28702, "ĠRend": 28703, "Ġfluids": 28704, "Authorities": 28705, "Ġcousins": 28706, "requency": 28707, "ĠNeighbor": 28708, "sets": 28709, "shared": 28710, "Charles": 28711, "password": 28712, "Ġgears": 28713, "Ġ211": 28714, "ĠHardware": 28715, "rika": 28716, "Ġupstream": 28717, "Hom": 28718, "Ġdisproportionately": 28719, "ivities": 28720, "Ġundefined": 28721, "Ġelectrons": 28722, "Ġcommemor": 28723, "Eventually": 28724, "Ġ><": 28725, "Ġirresponsible": 28726, "218": 28727, "ĠReleased": 28728, "ĠOVER": 28729, "ĠIGN": 28730, "ĠBread": 28731, "stellar": 28732, "ĠSage": 28733, "tted": 28734, "damage": 28735, "edition": 28736, "ĠPrec": 28737, "Ġlime": 28738, "Ġconfinement": 28739, "Ġcalorie": 28740, "weapon": 28741, "Ġdiffering": 28742, "ĠSina": 28743, "mys": 28744, "amd": 28745, "Ġintricate": 28746, "kk": 28747, "ĠPAT": 28748, "ão": 28749, "stones": 28750, "links": 28751, "Ġranch": 28752, "Semitic": 28753, "Ġdifferentiate": 28754, "ĠSinger": 28755, "occupied": 28756, "Ġfortress": 28757, "cmd": 28758, "Ġinterception": 28759, "ĠAnkara": 28760, "Ġrept": 28761, "ĠSolitaire": 28762, "Ġremake": 28763, "pred": 28764, "Ġdared": 28765, "autions": 28766, "ĠBACK": 28767, "Running": 28768, "Ġdebugging": 28769, "Ġgraphs": 28770, "399": 28771, "ĠNigel": 28772, "Ġbun": 28773, "Ġpillow": 28774, "Ġprogressed": 28775, "fashioned": 28776, "Ġobedience": 28777, "ERN": 28778, "Ġrehears": 28779, "Cell": 28780, "tl": 28781, "Sher": 28782, "Ġherald": 28783, "ĠPayment": 28784, "ĠCory": 28785, "ĠDept": 28786, "Ġrepent": 28787, "ĠWeak": 28788, "uckland": 28789, "Ġpleasing": 28790, "Ġshortages": 28791, "Ġjurors": 28792, "ĠKab": 28793, "qqa": 28794, "Anti": 28795, "Ġwow": 28796, "ĠRCMP": 28797, "Ġtsun": 28798, "ĠSic": 28799, "Ġcomprises": 28800, "Ġspies": 28801, "Ġprecinct": 28802, "nu": 28803, "Ġurges": 28804, "Ġtimed": 28805, "Ġstripes": 28806, "ĠBoots": 28807, "Ġyen": 28808, "Advanced": 28809, "Ġdiscrete": 28810, "ĠArchangel": 28811, "employment": 28812, "Diff": 28813, "Ġmonuments": 28814, "Ġ209": 28815, "worker": 28816, "Ġ196": 28817, "ĠIg": 28818, "utterstock": 28819, "TPS": 28820, "Jac": 28821, "Ġhomelessness": 28822, "Ġcommentator": 28823, "Ġracially": 28824, "fing": 28825, "seed": 28826, "Ele": 28827, "ellation": 28828, "Ġethanol": 28829, "Ġparish": 28830, "ĠDong": 28831, "ĠAwakening": 28832, "Ġdeviation": 28833, "ĠBearing": 28834, "ĠTsuk": 28835, "Ġrecess": 28836, "Ġlymph": 28837, "ĠCannabis": 28838, "åľ": 28839, "ĠNEWS": 28840, "Ġdra": 28841, "ĠStefan": 28842, "ĠWrong": 28843, "ĠSAM": 28844, "Ġloosely": 28845, "Ġinterpreter": 28846, "ĠPlain": 28847, "Government": 28848, "Ġbigotry": 28849, "Ġgrenades": 28850, "avez": 28851, "pictured": 28852, "Ġmandated": 28853, "ĠMonk": 28854, "ĠPedro": 28855, "Ġlava": 28856, "274": 28857, "Ġcynical": 28858, "ĠScrolls": 28859, "locks": 28860, "Mp": 28861, "Ġcongregation": 28862, "ornings": 28863, "phil": 28864, "ĠIbid": 28865, "Ġferv": 28866, "Ġdisappearing": 28867, "Ġarrogant": 28868, "syn": 28869, "ĠMaver": 28870, "ĠSuit": 28871, "241": 28872, "Ġabbre": 28873, "ackers": 28874, "Pa": 28875, "ĠYel": 28876, "Whenever": 28877, "Ġ235": 28878, "ĠVine": 28879, "ĠAnat": 28880, "Ġextinct": 28881, "LET": 28882, "Ġexecutable": 28883, "VERS": 28884, "oxide": 28885, "DNA": 28886, "ĠPrel": 28887, "Ġresentment": 28888, "Ġcomprise": 28889, "ĠAviv": 28890, "Ġinterceptions": 28891, "Ġprolific": 28892, "INA": 28893, "ĠErin": 28894, "thought": 28895, "219": 28896, "ĠPsychiatry": 28897, "unky": 28898, "chemist": 28899, "Ho": 28900, "ĠMcCoy": 28901, "Ġbricks": 28902, "Los": 28903, "rily": 28904, "ĠUSSR": 28905, "Ġrud": 28906, "Ġlaud": 28907, "ĠWise": 28908, "ĠEmerald": 28909, "Ġrevived": 28910, "Ġdamned": 28911, "ĠRepair": 28912, "idem": 28913, "ctica": 28914, "Ġpatriarch": 28915, "ĠNurs": 28916, "meg": 28917, "Ġcheapest": 28918, "reements": 28919, "empty": 28920, "ĠCelebr": 28921, "Ġdeprivation": 28922, "chanted": 28923, "ĠThumbnails": 28924, "Energy": 28925, "ĠEthan": 28926, "ĠQing": 28927, "Ġopposes": 28928, "WIND": 28929, "vik": 28930, "ĠMau": 28931, "ĠSUB": 28932, "667": 28933, "GRE": 28934, "ĠVolunte": 28935, "nton": 28936, "Cook": 28937, "åIJ": 28938, "esque": 28939, "Ġplummet": 28940, "Ġsuing": 28941, "Ġpronounce": 28942, "Ġresisting": 28943, "ĠFishing": 28944, "ĠTrials": 28945, "Ġyell": 28946, "Ġ310": 28947, "Ġinduct": 28948, "Ġpersonalized": 28949, "often": 28950, "Reb": 28951, "EMBER": 28952, "Ġviewpoint": 28953, "Ġexistential": 28954, "())": 28955, "remove": 28956, "MENTS": 28957, "lasses": 28958, "Ġevapor": 28959, "Ġaisle": 28960, "meta": 28961, "Ġreflective": 28962, "Ġentitlement": 28963, "Ġdevised": 28964, "music": 28965, "ascade": 28966, "Ġwinding": 28967, "offset": 28968, "Ġaccessibility": 28969, "kered": 28970, "Better": 28971, "ĠJohnston": 28972, "thinking": 28973, "Snow": 28974, "ĠCroatia": 28975, "ĠAtomic": 28976, "271": 28977, "348": 28978, "Ġtextbook": 28979, "ĠSixth": 28980, "ĠاÙĦ": 28981, "Ġslider": 28982, "ĠBurger": 28983, "bol": 28984, "Sync": 28985, "Ġgrandchildren": 28986, "Ġcerv": 28987, "+)": 28988, "Ġeternity": 28989, "Ġtweeting": 28990, "Ġspeculative": 28991, "Ġpivotal": 28992, "ĠWP": 28993, "ĠTER": 28994, "ynamic": 28995, "Ġupl": 28996, "ĠCats": 28997, "perhaps": 28998, "Ġclassmates": 28999, "Ġblatant": 29000, "'-": 29001, "Ġlakh": 29002, "antine": 29003, "ĠBorg": 29004, "iom": 29005, "/(": 29006, "ĠAthletic": 29007, "Ġsar": 29008, "OTA": 29009, "ĠHoffman": 29010, "Nevertheless": 29011, "Ġadorable": 29012, "Ġspawned": 29013, "Associated": 29014, "ĠDomestic": 29015, "Ġimplant": 29016, "ĠLuxem": 29017, "ĠKens": 29018, "Ġpumps": 29019, "ĠSAT": 29020, "Attributes": 29021, "509": 29022, "avour": 29023, "Ġcentralized": 29024, "ĠTN": 29025, "Ġfreshly": 29026, "ĠAchieve": 29027, "Ġoutsiders": 29028, "herty": 29029, "ĠRee": 29030, "ĠTowers": 29031, "ĠDart": 29032, "akable": 29033, "Ġmp": 29034, "ĠHeavenly": 29035, "Ġripe": 29036, "ĠCaroline": 29037, "ryan": 29038, "Ġclassics": 29039, "Ġretiring": 29040, "Ġ228": 29041, "Ġah": 29042, "Ġdealings": 29043, "Ġpunching": 29044, "ĠChapman": 29045, "Options": 29046, "maxwell": 29047, "volume": 29048, "Ġstal": 29049, "Ġexported": 29050, "ĠQuite": 29051, "Ġnumerical": 29052, "Burn": 29053, "Fact": 29054, "ĠKeystone": 29055, "Ġtrending": 29056, "Ġaltering": 29057, "ĠAfricans": 29058, "478": 29059, "ĠMN": 29060, "ĠKnock": 29061, "Ġtemptation": 29062, "Ġprestige": 29063, "Overview": 29064, "ĠTraditional": 29065, "ĠBahrain": 29066, "Private": 29067, "ĠHOU": 29068, "Ġbarr": 29069, "ĠTat": 29070, "Cube": 29071, "USD": 29072, "ĠGrande": 29073, "ĠGat": 29074, "ĠFlo": 29075, "Ġresides": 29076, "Ġindec": 29077, "volent": 29078, "Ġperpetual": 29079, "ubes": 29080, "Ġworldview": 29081, "ĠQuantum": 29082, "Ġfiltered": 29083, "Ġensu": 29084, "orgetown": 29085, "ERSON": 29086, "ĠMild": 29087, "379": 29088, "OTT": 29089, "Ã¥": 29090, "Ġvitamins": 29091, "Ġribbon": 29092, "Ġsincerely": 29093, "ĠHin": 29094, "Ġeighteen": 29095, "Ġcontradictory": 29096, "Ġglaring": 29097, "Ġexpectancy": 29098, "Ġconspir": 29099, "Ġmonstrous": 29100, "Ġ380": 29101, "reci": 29102, "Ġhandic": 29103, "Ġpumped": 29104, "Ġindicative": 29105, "Ġrapp": 29106, "Ġavail": 29107, "ĠLEGO": 29108, "ĠMarijuana": 29109, "1985": 29110, "erton": 29111, "Ġtwentieth": 29112, "################################": 29113, "ĠSwamp": 29114, "Ġvaluation": 29115, "Ġaffiliates": 29116, "adjusted": 29117, "ĠFacility": 29118, "262": 29119, "Ġenzymes": 29120, "itudinal": 29121, "Ġimprint": 29122, "Site": 29123, "Ġinstaller": 29124, "ĠTRA": 29125, "mology": 29126, "linear": 29127, "ĠCollective": 29128, "igating": 29129, "ĠToken": 29130, "Ġspeculated": 29131, "KN": 29132, "ĠCly": 29133, "ority": 29134, "Ġdefer": 29135, "Ġinspectors": 29136, "approved": 29137, "RM": 29138, "ĠSuns": 29139, "Ġinforming": 29140, "ĠSyracuse": 29141, "ibli": 29142, "765": 29143, "Ġglove": 29144, "Ġauthorize": 29145, "âĢ¦âĢ¦âĢ¦âĢ¦âĢ¦âĢ¦âĢ¦âĢ¦": 29146, "ĠCruise": 29147, "Ġcontracting": 29148, "shell": 29149, "IFE": 29150, "ĠJewel": 29151, "pract": 29152, "ĠPhotoshop": 29153, "ĠKnowing": 29154, "harm": 29155, "Ġattractions": 29156, "adan": 29157, "etus": 29158, "018": 29159, "wagen": 29160, "Alt": 29161, "Ġmultiply": 29162, "Ġequilibrium": 29163, ":{": 29164, "ĠFighters": 29165, "ĠEdgar": 29166, "Ġfourteen": 29167, "Govern": 29168, "Ġmisuse": 29169, "Ġabusing": 29170, "Ġancestry": 29171, "ramer": 29172, "644": 29173, "Ġworms": 29174, "Ġthicker": 29175, "ĠCombine": 29176, "Ġpeasants": 29177, "Ġvind": 29178, "Ġconquest": 29179, "Ġmocked": 29180, "Ġcinnamon": 29181, "ĠCald": 29182, "ĠGallup": 29183, "Ġavoidance": 29184, "Ġincarnation": 29185, "ĠStrat": 29186, "Ġtasted": 29187, "enta": 29188, "ĠNeal": 29189, "pared": 29190, "Ġterminology": 29191, "jection": 29192, "Scientists": 29193, "ĠINS": 29194, "ĠDee": 29195, "Ġdirectories": 29196, "Road": 29197, "ĠShap": 29198, "bright": 29199, "ĠDirectors": 29200, "ĠColumn": 29201, "Ġbob": 29202, "Ġpreferably": 29203, "Ġglitch": 29204, "furt": 29205, "Ġeg": 29206, "idis": 29207, "CBC": 29208, "Ġsurrendered": 29209, "Ġtestament": 29210, "336": 29211, "uggest": 29212, "ĠNil": 29213, "another": 29214, "Ġpathetic": 29215, "ĠDonna": 29216, "Ġ218": 29217, "ĠAvery": 29218, "Ġwhiskey": 29219, "Ġfixture": 29220, "ĠConquest": 29221, "Ġbets": 29222, "Occ": 29223, "ĠLeicester": 29224, "].\"": 29225, "Ġ));": 29226, "Ġflashes": 29227, "456": 29228, "Ġmasked": 29229, "gebra": 29230, "Ġcomputed": 29231, "chel": 29232, "auder": 29233, "Ġdefeats": 29234, "ĠLiberation": 29235, "ĠOsama": 29236, "ĠVive": 29237, "Changes": 29238, "Channel": 29239, "Ġtariffs": 29240, "Ġmage": 29241, "ĠSax": 29242, "Ġinadvertently": 29243, "ĠCRE": 29244, "ĠReaper": 29245, "inky": 29246, "grading": 29247, "Ġstereotyp": 29248, "Ġcurl": 29249, "ĠFANT": 29250, "Ġframeworks": 29251, "Mom": 29252, "ĠAnch": 29253, "Ġflavour": 29254, "carbon": 29255, "Ġpermitting": 29256, "letcher": 29257, "ĠMozilla": 29258, "ĠParking": 29259, "ĠChamp": 29260, "Scroll": 29261, "Ġmurderer": 29262, "Ġrested": 29263, "Ġowes": 29264, "ĠPoss": 29265, "ADD": 29266, "IFF": 29267, "resolution": 29268, "ĠMining": 29269, "Ġcomparative": 29270, "Dim": 29271, "Ġneighbouring": 29272, "ĠAST": 29273, "ĠToxic": 29274, "Ġbiases": 29275, "Ġgunfire": 29276, "urous": 29277, "ĠMoment": 29278, "1983": 29279, "Ġpervasive": 29280, "ttp": 29281, "ĠNormally": 29282, "rir": 29283, "Sarah": 29284, "ĠAlbany": 29285, "Ġunsett": 29286, "ĠSMS": 29287, "ipers": 29288, "layer": 29289, "ĠWhites": 29290, "uple": 29291, "Ġturbo": 29292, "ĠLeeds": 29293, "Ġthats": 29294, "ĠMiner": 29295, "MER": 29296, "ĠReign": 29297, "Ġperme": 29298, "ĠBlitz": 29299, "Ġ1934": 29300, "Ġintimidating": 29301, "tube": 29302, "Ġeccentric": 29303, "abolic": 29304, "boxes": 29305, "ĠAssociates": 29306, "votes": 29307, "Ġsimulate": 29308, "umbo": 29309, "astery": 29310, "Ġshipments": 29311, "FFFF": 29312, "anth": 29313, "Ġseasoned": 29314, "Ġexperimentation": 29315, "âĸł": 29316, "laws": 29317, "Meet": 29318, "iddles": 29319, "antics": 29320, "Rating": 29321, "ISIS": 29322, "hift": 29323, "Ġfronts": 29324, "buf": 29325, "017": 29326, "Ġunatt": 29327, "ĠDil": 29328, "leases": 29329, "ĠGardens": 29330, "777": 29331, "touch": 29332, "vell": 29333, "458": 29334, "Ġ=====": 29335, "saving": 29336, "Ġerosion": 29337, "ĠQuin": 29338, "Ġearns": 29339, "Ġaccomplishment": 29340, "ĠWei": 29341, "Ġ<[": 29342, "_____": 29343, "Ġirrig": 29344, "ĠTeddy": 29345, "Ġconquered": 29346, "ĠArmored": 29347, "Ġasserts": 29348, "Ġmanipulating": 29349, "ré": 29350, "Ġtranscripts": 29351, "Gallery": 29352, "Ġplotting": 29353, "Neil": 29354, "Ġbetrayal": 29355, "loader": 29356, "ĠSul": 29357, "Ġdisplacement": 29358, "Ġroyalty": 29359, "ĠWI": 29360, "heit": 29361, "ĠDevices": 29362, "allel": 29363, "Ġmunicipalities": 29364, "Ġcanal": 29365, "Stars": 29366, "ĠUAE": 29367, "Ġ\"âĢ¦": 29368, "ĠCU": 29369, "above": 29370, "Ġresonance": 29371, "ĠguiActiveUn": 29372, "added": 29373, "ĠBraves": 29374, "ĠIbn": 29375, "Ġhereby": 29376, "ĠBRE": 29377, "Ġshareholder": 29378, "ĠHir": 29379, "ĠJi": 29380, "Ġstrangely": 29381, "Ġadmired": 29382, "Ġplight": 29383, "Ġbachelor": 29384, "ĠPole": 29385, "ciplinary": 29386, "Tony": 29387, "ĠArmenian": 29388, "Ġunman": 29389, "ĠZionist": 29390, "Stage": 29391, "iscover": 29392, "Ġautomotive": 29393, "Ġsidelines": 29394, "Ġslick": 29395, "ĠRenaissance": 29396, "ĠFUN": 29397, "Images": 29398, "ĠHaj": 29399, "Ġping": 29400, "Ġshortcut": 29401, "ĠBlvd": 29402, "ĠLooks": 29403, "Ġbursts": 29404, "Ġclamp": 29405, "Ġmish": 29406, "Ġsorting": 29407, "Ġpatriot": 29408, "Ġcorrectness": 29409, "ĠScandinav": 29410, "ĠCavaliers": 29411, "python": 29412, "azar": 29413, "Ġ375": 29414, "ĠJaune": 29415, "409": 29416, "Ġdetrimental": 29417, "Ġstabbing": 29418, "Ġpoisoned": 29419, "Ġfountain": 29420, "ocent": 29421, "orst": 29422, "ĠMari": 29423, "Ġrains": 29424, "ĠOvers": 29425, "ĠInstitution": 29426, "udget": 29427, "AMY": 29428, "tale": 29429, "ĠKR": 29430, "ĠPrices": 29431, "Ġheadaches": 29432, "Ġlandsl": 29433, "ĠAura": 29434, "Bonus": 29435, "ĠZhao": 29436, "ĠHip": 29437, "Ġhops": 29438, "ĠKurdistan": 29439, "Ġexploiting": 29440, "ryn": 29441, "Ġhypocrisy": 29442, "opening": 29443, "Ġgunshot": 29444, "Ġwed": 29445, "interstitial": 29446, "Interstitial": 29447, "Ġamen": 29448, "Breaking": 29449, "Ġmarketed": 29450, "Wire": 29451, "ĠCrowd": 29452, "Continue": 29453, "ĠKnown": 29454, "ĠEffective": 29455, "orean": 29456, "izons": 29457, "Joseph": 29458, "Ġescalation": 29459, "username": 29460, "Ġcurtain": 29461, "ATES": 29462, "ĠPAR": 29463, "ĠMiy": 29464, "Ġcounterfe": 29465, "lene": 29466, "Ġcontenders": 29467, "daily": 29468, "ĠAsc": 29469, "ĠPhillip": 29470, "mostly": 29471, "Ġfilename": 29472, "hene": 29473, "Ġresembling": 29474, "Ġstaging": 29475, "ĠChloe": 29476, "Ġwiring": 29477, "Hon": 29478, "ĠRenew": 29479, "ottage": 29480, "ĠHybrid": 29481, "much": 29482, "Ġstrokes": 29483, "Ġpolicymakers": 29484, "APTER": 29485, "ĠArkham": 29486, "plot": 29487, "Ġassistants": 29488, "Ġdeport": 29489, "ĠSega": 29490, "Ġinfluenza": 29491, "ĠCursed": 29492, "ĠKobe": 29493, "Ġskinny": 29494, "Provider": 29495, "ĠRip": 29496, "Ġincremental": 29497, "products": 29498, "BF": 29499, "Ġdome": 29500, "ĠCredits": 29501, "Ġlosers": 29502, "ints": 29503, "ĠBetty": 29504, "ĠTalent": 29505, "ĠDAM": 29506, "Lv": 29507, "Ess": 29508, "Ġdens": 29509, "temp": 29510, "Judge": 29511, "odic": 29512, "Ġ'(": 29513, "URES": 29514, "etsk": 29515, "VO": 29516, "Ġretrieved": 29517, "Ġarchitects": 29518, "Ùĩ": 29519, "Ġethic": 29520, "ĠSecondary": 29521, "stocks": 29522, "adia": 29523, "Ġ325": 29524, "ĠOpinion": 29525, "Ġsimultaneous": 29526, "Ġdizz": 29527, "ulp": 29528, "Ġsmuggling": 29529, "ippery": 29530, "Random": 29531, "facing": 29532, "ĠDas": 29533, "Ġstockp": 29534, "Ġdisclosures": 29535, "pointer": 29536, "Ġcoral": 29537, "ĠSelection": 29538, "ĠPike": 29539, "ivalent": 29540, "Ġruthless": 29541, "ĠRim": 29542, "Ġensuing": 29543, "ĠExperiment": 29544, "Ġcongressman": 29545, "Ġbeliever": 29546, "Ġunspecified": 29547, "ĠMord": 29548, "Ġknowledgeable": 29549, "ĠVERY": 29550, "TX": 29551, "Ġstraps": 29552, "Ġturf": 29553, "apeshifter": 29554, "Ġmarital": 29555, "Ġflock": 29556, "ãģĨ": 29557, "263": 29558, "AMES": 29559, "ĠOpposition": 29560, "Ġtreasures": 29561, "ĠGOD": 29562, "Ġmodeled": 29563, "ĠWORLD": 29564, "Ġ([": 29565, "ĠUsage": 29566, "HF": 29567, "Ġ$(": 29568, "ussed": 29569, "Ġpioneer": 29570, "Eight": 29571, "parse": 29572, "bread": 29573, "ritz": 29574, "ĠMiranda": 29575, "ĠKant": 29576, "++)": 29577, "oren": 29578, "Ġprovoked": 29579, "Ġbreeds": 29580, "ĠIncludes": 29581, "ĠPastebin": 29582, "ĠFlip": 29583, "Java": 29584, "Ġbrink": 29585, "Ġrumored": 29586, "Ġunseen": 29587, "Ġgarnered": 29588, "ĠDefin": 29589, "alted": 29590, "Ġtattoos": 29591, "Ġhesitation": 29592, "isitions": 29593, "ĠWeaver": 29594, "ĠReporting": 29595, "Ġtherapies": 29596, "Ġconsultants": 29597, "Ġresidual": 29598, "ĠMali": 29599, "ĠRoma": 29600, "iago": 29601, "ĠResidents": 29602, "ubi": 29603, "Ġremedies": 29604, "Ġadaptive": 29605, "ĠAlive": 29606, "ĠBarcl": 29607, "Ġwallets": 29608, "crypt": 29609, "etermination": 29610, "ĠPelosi": 29611, "Ġslipping": 29612, "otonin": 29613, "Ġalliances": 29614, "patrick": 29615, "iris": 29616, "Ġorth": 29617, "ĠPerkins": 29618, "ĠDeV": 29619, "ĠGets": 29620, "Ġdrying": 29621, "gee": 29622, "forest": 29623, "ĠForget": 29624, "orem": 29625, "339": 29626, "Ġvaguely": 29627, "ĠDion": 29628, "ĠPorn": 29629, "ĠHOW": 29630, "Ġpneum": 29631, "Ġrubble": 29632, "ĠTaste": 29633, "encia": 29634, "ĠGel": 29635, "Ġdst": 29636, "Ġ245": 29637, "ĠMorocco": 29638, "inflamm": 29639, "ĠTwins": 29640, "Ġbots": 29641, "daughter": 29642, "ĠBalk": 29643, "Ġbrethren": 29644, "Ġlogos": 29645, "Ġgobl": 29646, "fps": 29647, "Ġsubdivision": 29648, "Ġpawn": 29649, "Ġsqueezed": 29650, "Ġmorale": 29651, "ĠDW": 29652, "'\"": 29653, "Ġknot": 29654, "ooky": 29655, "Ġdivisive": 29656, "Ġboosted": 29657, "chy": 29658, "ãĥIJ": 29659, "ifact": 29660, "Ġnewcomers": 29661, "ĠWrestling": 29662, "Ġscouts": 29663, "wolves": 29664, "Rat": 29665, "Ġnineteenth": 29666, "ĠOsborne": 29667, "Stats": 29668, "Ġempowered": 29669, "Ġpsychopath": 29670, "ĠOEM": 29671, "uggage": 29672, "ĠPK": 29673, "ĠMohammad": 29674, "Pak": 29675, "Ġanarchists": 29676, "ĠExtract": 29677, "esthes": 29678, "ĠStockholm": 29679, "loo": 29680, "ĠGraph": 29681, "Ġdeploying": 29682, "ĠStranger": 29683, "ĠMold": 29684, "Ġstaffer": 29685, "Ġdiscounted": 29686, "uckle": 29687, "please": 29688, "ĠLanding": 29689, "ÃŃa": 29690, "Ġ193": 29691, "Ġante": 29692, "Ġrepetition": 29693, "Ġ+/-": 29694, "Ġparody": 29695, "Ġlively": 29696, "AAA": 29697, "ĠHorus": 29698, "Ġpits": 29699, "inders": 29700, "LOC": 29701, "ĠVenice": 29702, "406": 29703, "ĠDiscover": 29704, "âĨ": 29705, "ellectual": 29706, "Ġpens": 29707, "Ġeyel": 29708, "iguous": 29709, "Impl": 29710, "Ġjoking": 29711, "Ġinval": 29712, "ĠBelfast": 29713, "Ġcreditors": 29714, "ĠSkywalker": 29715, "ovsky": 29716, "Ġceasefire": 29717, "Ġseals": 29718, "isoft": 29719, ")).": 29720, "ĠFelix": 29721, "ITS": 29722, "Ġtresp": 29723, "ĠBlockchain": 29724, "eware": 29725, "ĠSchwar": 29726, "enne": 29727, "mounted": 29728, "ĠBeacon": 29729, "lesh": 29730, "Ġimmensely": 29731, "Ġcheering": 29732, "Employ": 29733, "scene": 29734, "ishly": 29735, "atchewan": 29736, "ĠNicolas": 29737, "Ġdrained": 29738, "ĠExit": 29739, "ĠAzerb": 29740, "jun": 29741, "Ġfloated": 29742, "uania": 29743, "Deep": 29744, "Ġsuperv": 29745, "Ġmystical": 29746, "ĠDollar": 29747, "ĠApostle": 29748, "ĠREL": 29749, "ĠProvided": 29750, "ĠBucks": 29751, "ãĥ´": 29752, "cutting": 29753, "Ġenhancements": 29754, "ĠPenguins": 29755, "ĠIsaiah": 29756, "Ġjerk": 29757, "ĠWyn": 29758, "Ġstalled": 29759, "Ġcryptocurrencies": 29760, "ĠRoland": 29761, "single": 29762, "Ġlumin": 29763, "ĠFellow": 29764, "ĠCapacity": 29765, "ĠKazakh": 29766, "WN": 29767, "Ġfinanced": 29768, "389": 29769, "Ġtid": 29770, "Ġcollusion": 29771, "ĠMyr": 29772, "îĢ": 29773, "Senator": 29774, "Ġpediatric": 29775, "Ġneatly": 29776, "Ġsandwiches": 29777, "ĠArchitecture": 29778, "Ġtucked": 29779, "Ġbalcony": 29780, "Ġearthquakes": 29781, "quire": 29782, "Future": 29783, "Ġhefty": 29784, "éĹ": 29785, "Ġspecializes": 29786, "Ġstresses": 29787, "Ġsender": 29788, "Ġmisunderstanding": 29789, "Ġepile": 29790, "Ġprovoke": 29791, "ĠColors": 29792, "Ġdismay": 29793, "uko": 29794, "[_": 29795, "586": 29796, "neutral": 29797, "Ġdonating": 29798, "ĠRandall": 29799, "Multi": 29800, "Ġconveniently": 29801, "ĠSung": 29802, "ĠCoca": 29803, "Ġtents": 29804, "ĠAcceler": 29805, "Ġpartnered": 29806, "272": 29807, "irming": 29808, "ĠBAS": 29809, "sometimes": 29810, "Ġobjected": 29811, "ubric": 29812, "posed": 29813, "LCS": 29814, "grass": 29815, "Ġattributable": 29816, "VIS": 29817, "Israeli": 29818, "Ġrepeats": 29819, "ĠRM": 29820, "vag": 29821, "uta": 29822, "inous": 29823, "Ġinert": 29824, "ĠMiguel": 29825, "æŃ": 29826, "ĠHawaiian": 29827, "Board": 29828, "Ġartific": 29829, "ĠAzerbai": 29830, "asio": 29831, "ĠRent": 29832, "AIN": 29833, "Ġappliances": 29834, "Ġnationality": 29835, "Ġasshole": 29836, "ĠNeb": 29837, "Ġnotch": 29838, "hani": 29839, "ĠBride": 29840, "Availability": 29841, "Ġintercepted": 29842, "Ġcontinental": 29843, "Ġswelling": 29844, "ĠPerspect": 29845, "bies": 29846, ".<": 29847, "ithmetic": 29848, "ĠLara": 29849, "Ġtempting": 29850, "addr": 29851, "Ġoverseeing": 29852, "clad": 29853, "ĠDV": 29854, "ĠGingrich": 29855, "Ġmun": 29856, "ĠAppropri": 29857, "Ġalterations": 29858, "ĠPatreon": 29859, "Ġhavoc": 29860, "Ġdisciplines": 29861, "Ġnotoriously": 29862, "akuya": 29863, "ieri": 29864, "?).": 29865, "ĠWent": 29866, "Ġsilicon": 29867, "Ġtremb": 29868, "Container": 29869, "Known": 29870, "Ġmortar": 29871, "este": 29872, "icka": 29873, "Arthur": 29874, "ĠPreviously": 29875, "ĠMarty": 29876, "Ġsparse": 29877, "gins": 29878, "Ġinward": 29879, "ĠParticipant": 29880, "Copy": 29881, "ĠMisc": 29882, "Ġantibiotic": 29883, "ĠRetro": 29884, "Ġelusive": 29885, "Ġassail": 29886, "ĠBattalion": 29887, "ĠBought": 29888, "Ġdiminish": 29889, "ĠEuropa": 29890, "session": 29891, "ĠDangerous": 29892, "iesel": 29893, "Ġdisbelief": 29894, "Ġblasts": 29895, "extreme": 29896, "ĠBoyd": 29897, "ĠProjects": 29898, "ĠGuys": 29899, "Ġundergone": 29900, "Ġgrill": 29901, "ĠDwight": 29902, "Ġ197": 29903, "USER": 29904, "Ġfilesystem": 29905, "Ġclocks": 29906, "Taylor": 29907, "Ġwrapper": 29908, "Ġfolding": 29909, "ousand": 29910, "ĠPhilippine": 29911, "ATIONAL": 29912, "ĠPerth": 29913, "Ġashes": 29914, "Ġaccumulate": 29915, "ĠGateway": 29916, "Shop": 29917, "orkshire": 29918, "Han": 29919, "ĠBarrel": 29920, "ĠLeh": 29921, "ĠXV": 29922, "Ġwhim": 29923, "Ġrepo": 29924, "ĠCG": 29925, "ĠMam": 29926, "Ġincorporating": 29927, "Ġbailout": 29928, "Ġlinguistic": 29929, "Ġdisinteg": 29930, "CLE": 29931, "Ġcinematic": 29932, "ĠFiber": 29933, "Syn": 29934, "ilion": 29935, "ĠCompos": 29936, "chens": 29937, "Ġneoc": 29938, "Ġboiled": 29939, "FINE": 29940, "ono": 29941, "uncle": 29942, "iken": 29943, "ĠBM": 29944, "ι": 29945, "Ġreceipts": 29946, "Ġdisposed": 29947, "ĠThirty": 29948, "ĠRough": 29949, "ĠABS": 29950, "Ġnotwithstanding": 29951, "ollen": 29952, "#$": 29953, "Ġunreliable": 29954, "Ġbloom": 29955, "Ġmediocre": 29956, "Ġtram": 29957, "ĠTasman": 29958, "Ġshakes": 29959, "Ġmanifesto": 29960, "ĠMW": 29961, "Ġsatisfactory": 29962, "Ġshores": 29963, "Ġcomputation": 29964, "Ġassertions": 29965, "ormons": 29966, "arag": 29967, "abit": 29968, "Democrats": 29969, "ĠLoot": 29970, "ĠVolks": 29971, "haired": 29972, "Ġgravitational": 29973, "Sing": 29974, "ĠMiz": 29975, "Ġthrottle": 29976, "Ġtyranny": 29977, "ĠViews": 29978, "Ġrobber": 29979, "ĠMinority": 29980, "Ġshrine": 29981, "scope": 29982, "purpose": 29983, "Ġnucleus": 29984, "ourcing": 29985, "ĠUSDA": 29986, "ĠDHS": 29987, "wra": 29988, "ĠBowie": 29989, "Scale": 29990, "ĠBEL": 29991, "xi": 29992, "Iter": 29993, "Ġ(),": 29994, "wright": 29995, "Ġsailors": 29996, "oused": 29997, "NASA": 29998, "ĠProof": 29999, "ĠMineral": 30000, "token": 30001, "ĠFD": 30002, "Rew": 30003, "Ġell": 30004, "630": 30005, "Ġchancellor": 30006, "ĠGos": 30007, "Ġamounted": 30008, "ĠRecre": 30009, "omez": 30010, "ĠOptim": 30011, "ĠOlive": 30012, "Ġtracker": 30013, "owler": 30014, "ĠUnique": 30015, "Root": 30016, "Ġmaritime": 30017, "ĠQuran": 30018, "ĠAdapt": 30019, "Ġecosystems": 30020, "ĠRepeat": 30021, "ĠSoy": 30022, "ĠIMP": 30023, "Ġgraduating": 30024, "andem": 30025, "Pur": 30026, "ĠReset": 30027, "ĠTrick": 30028, "ĠPhilly": 30029, "ĠTue": 30030, "ĠMalaysian": 30031, "Ġclimax": 30032, "Ġbury": 30033, "Ġconspic": 30034, "ĠSouthampton": 30035, "ĠFlowers": 30036, "Ġescorted": 30037, "ĠEducational": 30038, "ĠIRC": 30039, "Ġbrutally": 30040, "eating": 30041, "Ġpillar": 30042, "ĠSang": 30043, "ĠJude": 30044, "arling": 30045, "ĠAmnesty": 30046, "Ġreminding": 30047, "ĠAdministrative": 30048, "hesda": 30049, "Ġflashed": 30050, "ĠPBS": 30051, "perate": 30052, "feature": 30053, "Ġswipe": 30054, "Ġgraves": 30055, "oultry": 30056, "261": 30057, "breaks": 30058, "ĠGuer": 30059, "Ġshrimp": 30060, "ĠVoting": 30061, "quist": 30062, "Ġanalytical": 30063, "Ġtablespoons": 30064, "ĠSOU": 30065, "Ġresearched": 30066, "Ġdisrupted": 30067, "Ġjour": 30068, "Ġreplica": 30069, "Ġcartoons": 30070, "bians": 30071, "})": 30072, "copy": 30073, "Got": 30074, "ouched": 30075, "PUT": 30076, "Ġswarm": 30077, "notations": 30078, "said": 30079, "Ġrebuilt": 30080, "Ġcollaborate": 30081, "Ġraging": 30082, "Ġnar": 30083, "Ġdemographics": 30084, "ĠDDR": 30085, "Ġdistrust": 30086, "ossier": 30087, "ĠKro": 30088, "Ġpumpkin": 30089, "Ġregrets": 30090, "Ġfatalities": 30091, "ĠLens": 30092, "ĠOle": 30093, "pd": 30094, "Ġpuppet": 30095, "ĠOutlook": 30096, "ĠStam": 30097, "Ol": 30098, "Fair": 30099, "UU": 30100, "Ġrewritten": 30101, "ı": 30102, "Ġfascinated": 30103, "Ġvectors": 30104, "Ġtribunal": 30105, "uay": 30106, "ĠMats": 30107, "ĠCoins": 30108, "[[": 30109, "Ġ181": 30110, "Ġrenders": 30111, "ĠKaepernick": 30112, "Ġespionage": 30113, "Ġsumm": 30114, "Ġditch": 30115, "Account": 30116, "Ġspreadsheet": 30117, "Ġmutant": 30118, "past": 30119, "407": 30120, "Ġdye": 30121, "Ġinitiation": 30122, "Ġ4000": 30123, "Ġpunishable": 30124, "Ġthinner": 30125, "ĠKhal": 30126, "Ġintermedi": 30127, "Dun": 30128, "ĠGotham": 30129, "Ġeagerly": 30130, "Ġvaginal": 30131, "powers": 30132, "VW": 30133, "ĠWATCHED": 30134, "Ġpredator": 30135, "amsung": 30136, "Ġdisparity": 30137, "Ġ[*": 30138, "Ġamph": 30139, "Ġoutskirts": 30140, "ĠSpirits": 30141, "Ġskeletal": 30142, "л": 30143, "ĠRear": 30144, "Ġissuance": 30145, "ĠLogic": 30146, "released": 30147, "ZZ": 30148, "ĠBound": 30149, "Entry": 30150, "Ġexits": 30151, "isol": 30152, "ĠFounder": 30153, "Ġwre": 30154, "ĠGreenland": 30155, "ĠMMO": 30156, "taker": 30157, "INC": 30158, "ãģ¾": 30159, "Ġhourly": 30160, "henko": 30161, "Ġfantasies": 30162, "Ġdisob": 30163, "Ġdemolition": 30164, "ãĥĭ": 30165, "Ġenlisted": 30166, "ratulations": 30167, "Ġmisguided": 30168, "Ġensured": 30169, "Ġdiscouraged": 30170, "mort": 30171, "Ġflank": 30172, "Ġcess": 30173, "Ġreacts": 30174, "ĠSere": 30175, "sensitive": 30176, "ĠSerpent": 30177, "assad": 30178, "Ġ247": 30179, "Ġcalmly": 30180, "busters": 30181, "Ġbleed": 30182, "ĠStro": 30183, "Ġamusement": 30184, "ĠAntarctica": 30185, "Ġscept": 30186, "ĠGaw": 30187, "aq": 30188, "asonic": 30189, "Ġsprawling": 30190, "native": 30191, "aturated": 30192, "ĠBattlefield": 30193, "IVERS": 30194, "EB": 30195, "ĠGems": 30196, "ĠNorthwestern": 30197, "ĠFilms": 30198, "ĠAutomatic": 30199, "Ġapprehend": 30200, "ãģ¨": 30201, "ĠguiName": 30202, "Ġbackend": 30203, "Ġevidenced": 30204, "geant": 30205, "012": 30206, "ĠSiege": 30207, "ĠexternalTo": 30208, "ĠunfocusedRange": 30209, "ĠguiActiveUnfocused": 30210, "ĠguiIcon": 30211, "ĠexternalToEVA": 30212, "ĠexternalToEVAOnly": 30213, "Fri": 30214, "chard": 30215, "enaries": 30216, "Ġchiefs": 30217, "Ġcf": 30218, "ĠHUD": 30219, "Ġcorrobor": 30220, "ĠdB": 30221, "ĠTaken": 30222, "ĠPatricia": 30223, "rail": 30224, "ĠCharm": 30225, "ĠLibertarian": 30226, "rieve": 30227, "Personal": 30228, "ĠOUR": 30229, "geries": 30230, "Ġdumping": 30231, "Ġneurological": 30232, "itimate": 30233, "ĠClintons": 30234, "rafted": 30235, "ĠMolly": 30236, "Ġterminals": 30237, "register": 30238, "Ġflare": 30239, "Ġencoded": 30240, "Ġautopsy": 30241, "pel": 30242, "machine": 30243, "Ġexemptions": 30244, "ĠRoyals": 30245, "distance": 30246, "Ġdrafts": 30247, "Ġlame": 30248, "ĠCunning": 30249, "Ġspouses": 30250, "ĠMarkets": 30251, "ĠCarrier": 30252, "Ġimplying": 30253, "ĠYak": 30254, "sid": 30255, "Ġloser": 30256, "Ġvigilant": 30257, "Ġimpeachment": 30258, "Ġaugmented": 30259, "ĠEmployees": 30260, "Ġunintended": 30261, "ternally": 30262, "ĠWatt": 30263, "Ġrecognizable": 30264, "essim": 30265, "æĿ": 30266, "Ġcoated": 30267, "rha": 30268, "Ġlieutenant": 30269, "ĠLegislation": 30270, "published": 30271, "444": 30272, "013": 30273, "Ġideally": 30274, "ĠPassword": 30275, "Ġsimplify": 30276, "ĠMeta": 30277, "ĠMRI": 30278, "Ġpleading": 30279, "organized": 30280, "handler": 30281, "Ġunravel": 30282, "correct": 30283, "Ġicy": 30284, "Ġparanoid": 30285, "Ġpasser": 30286, "Ġinspections": 30287, "ofer": 30288, "ĠHealthcare": 30289, "283": 30290, "ĠBrut": 30291, "iola": 30292, "forge": 30293, "ĠMedieval": 30294, "MSN": 30295, "ievers": 30296, "ĠProgramming": 30297, "åī": 30298, "Ġ223": 30299, "mu": 30300, "ĠCLE": 30301, "uga": 30302, "Ġshoppers": 30303, "Ġinformative": 30304, "ĠPlans": 30305, "Ġsupplementation": 30306, "ĠTests": 30307, "tyard": 30308, "ocytes": 30309, "ĠVega": 30310, "ĠGujarat": 30311, "ermanent": 30312, "Except": 30313, "ĠLOT": 30314, "alla": 30315, "ĠCumm": 30316, "ĠOsw": 30317, "Ġvenom": 30318, "ĠDebt": 30319, "ĠDOWN": 30320, "Ġreunion": 30321, "Ġmuc": 30322, "ĠRelief": 30323, "Ġgeop": 30324, "ĠðŁĺ": 30325, "alogue": 30326, "Anth": 30327, "echo": 30328, "Ġcorros": 30329, "Ġreplication": 30330, "ĠBlazing": 30331, "ĠDaughter": 30332, "Ġinflic": 30333, "ĠLindsey": 30334, "ÙĪ": 30335, "284": 30336, "Exit": 30337, "Ġgloom": 30338, "TAIN": 30339, "Ġundermining": 30340, "Ġadvising": 30341, "hidden": 30342, "Ġoverflow": 30343, "Ġgor": 30344, "urdue": 30345, "Ġechoes": 30346, "enhagen": 30347, "Ġimpuls": 30348, "drug": 30349, "cash": 30350, "Ġasync": 30351, "Ġmirac": 30352, "atts": 30353, "punk": 30354, "Ġpivot": 30355, "ĠLegislative": 30356, "Ġbloggers": 30357, "ĠClaw": 30358, "sburg": 30359, "dyl": 30360, "ĠRecommend": 30361, "Ġverte": 30362, "Ġprohibiting": 30363, "ĠPanther": 30364, "Jonathan": 30365, "Ġomin": 30366, "Ġhateful": 30367, "281": 30368, "ĠOrche": 30369, "ĠMurdoch": 30370, "downs": 30371, "Ġasymm": 30372, "GER": 30373, "Always": 30374, "Ġinforms": 30375, "ĠWM": 30376, "ĠPony": 30377, "ĠAppendix": 30378, "ĠArlington": 30379, "Jam": 30380, "Ġmedicinal": 30381, "ĠSlam": 30382, "ITIES": 30383, "Ġreaff": 30384, "ĠRi": 30385, "FG": 30386, "Spring": 30387, "bool": 30388, "Ġthighs": 30389, "Ġmarkings": 30390, "ĠRaqqa": 30391, "ĠLak": 30392, "poll": 30393, "tsky": 30394, "ĠMorty": 30395, "ĠDefinition": 30396, "Ġdebunk": 30397, "endered": 30398, "ĠLeone": 30399, "avers": 30400, "Ġmortgages": 30401, "Apparently": 30402, "Nic": 30403, "haus": 30404, "ĠThousands": 30405, "auld": 30406, "Ġmash": 30407, "shoot": 30408, "Ġdiarr": 30409, "Ġconsciously": 30410, "Hero": 30411, "eas": 30412, "ĠNaturally": 30413, "ĠDestroyer": 30414, "Ġdashboard": 30415, "services": 30416, "Rog": 30417, "Ġmillennials": 30418, "Ġinvade": 30419, "-(": 30420, "Ġcommissions": 30421, "ĠAuckland": 30422, "Ġbroadcasts": 30423, "Ġfrontal": 30424, "Ġcrank": 30425, "ĠHistoric": 30426, "Ġrumours": 30427, "CTV": 30428, "Ġsteril": 30429, "Ġbooster": 30430, "rocket": 30431, "ãĤ¼": 30432, "utsche": 30433, "ĠPI": 30434, "Ġ233": 30435, "ĠProducer": 30436, "ĠAnalytics": 30437, "Ġinvaluable": 30438, "Ġunintention": 30439, "ĠCY": 30440, "Ġscrutin": 30441, "Ġgigg": 30442, "Ġengulf": 30443, "Ġproletariat": 30444, "Ġhacks": 30445, "ĠHew": 30446, "arak": 30447, "ĠSlime": 30448, "ielding": 30449, "agher": 30450, "ĠElliot": 30451, "Ġtelecom": 30452, "Ġ219": 30453, "ultan": 30454, "ĠArbor": 30455, "ĠScouts": 30456, "Ban": 30457, "Ġlifespan": 30458, "Ġblasp": 30459, "388": 30460, "Ġjudiciary": 30461, "ĠContinental": 30462, "asking": 30463, "McC": 30464, "LED": 30465, "Ġbaggage": 30466, "ĠSorcerer": 30467, "Ġremnants": 30468, "ĠGriffith": 30469, "etsu": 30470, "ĠSubaru": 30471, "ĠPersonality": 30472, "designed": 30473, "ushima": 30474, "agnar": 30475, "Ġrecoil": 30476, "Ġpassions": 30477, "\\\":": 30478, "Ġtee": 30479, "Ġabolition": 30480, "ĠCreating": 30481, "jac": 30482, "Ġ194": 30483, "019": 30484, "Ġpillars": 30485, "riched": 30486, "/\"": 30487, "tk": 30488, "Ġlivelihood": 30489, "Ġroasted": 30490, "ahon": 30491, "ĠHutch": 30492, "assert": 30493, "Ġdividend": 30494, "Ġknit": 30495, "Ġdaunting": 30496, "Ġdisturbance": 30497, "Ġshale": 30498, "Ġcultivated": 30499, "Ġrefrigerator": 30500, "LB": 30501, "ĠNET": 30502, "Ġcommercials": 30503, "Ġthinkers": 30504, "455": 30505, "Ġchop": 30506, "Broad": 30507, "Ġsuspicions": 30508, "Ġtagged": 30509, "lifting": 30510, "Ġstylish": 30511, "ĠShields": 30512, "Shortly": 30513, "Ġtails": 30514, "Auth": 30515, "STE": 30516, "ĠGAME": 30517, "Ġseism": 30518, "ĠKis": 30519, "ologne": 30520, "Ġcowork": 30521, "Ġforcibly": 30522, "Ġthyroid": 30523, "ĠPB": 30524, "ANE": 30525, "married": 30526, "horse": 30527, "Ġpolymer": 30528, "ĠChal": 30529, "odor": 30530, "DEBUG": 30531, "ĠContext": 30532, "Ġbliss": 30533, "Ġpinpoint": 30534, "ĠMathemat": 30535, "legram": 30536, "ĠWeekend": 30537, "Ġlabelled": 30538, "Ġbart": 30539, "itles": 30540, "Ġestrogen": 30541, "âĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶ": 30542, "\"'": 30543, "Ġvisibly": 30544, "Ġoutsider": 30545, "aida": 30546, "Area": 30547, "Ġdissemin": 30548, "Ġdishonest": 30549, "ĠClosed": 30550, "ĠBulletin": 30551, "ĠRamsey": 30552, "sword": 30553, "ĠXI": 30554, "ourced": 30555, "Same": 30556, "346": 30557, "ĠRepe": 30558, "ĠKou": 30559, "cake": 30560, "emis": 30561, "Cache": 30562, "ĠMeaning": 30563, "ĠEnlight": 30564, "onomy": 30565, "Ġmanifestation": 30566, "sworth": 30567, "Jay": 30568, "Ġchore": 30569, "ör": 30570, "Dream": 30571, "Ġsanctioned": 30572, "Ġculturally": 30573, "ĠAra": 30574, "Nav": 30575, "Ġtheological": 30576, "Ġstrut": 30577, "ĠVO": 30578, "ĠHandbook": 30579, "Ġconstructing": 30580, "Ġ¶": 30581, "ĠBenefits": 30582, "ĠPsychological": 30583, "sac": 30584, "å¸": 30585, "policy": 30586, "ĠMatters": 30587, "ĠReported": 30588, "ĠByte": 30589, "Ġvitro": 30590, "ĠMaiden": 30591, "Ġlam": 30592, "ĠJennings": 30593, "Ġgarment": 30594, "ĠRutgers": 30595, "ĠStafford": 30596, "ĠWellington": 30597, "Ġintermitt": 30598, "Ġnpm": 30599, "Ġordeal": 30600, "Ġplugged": 30601, "ooming": 30602, "inished": 30603, "framework": 30604, "Ġtimber": 30605, "Ġcass": 30606, "Ġ850": 30607, "iless": 30608, "ĠRedux": 30609, "768": 30610, "Stre": 30611, "Ġsurpassed": 30612, "whel": 30613, "Ġparallels": 30614, "Ġveil": 30615, "ĠGI": 30616, "ĠREST": 30617, "Ġreadiness": 30618, "sort": 30619, "Ġmodifying": 30620, "ĠSlate": 30621, "ruff": 30622, "Ġmarble": 30623, "Ġinfrared": 30624, "Ġauditor": 30625, "ĠFANTASY": 30626, "ĠPoverty": 30627, "ĠSPD": 30628, "Ġ\"(": 30629, "Ky": 30630, "RAY": 30631, "Ġexecutions": 30632, "ĠBeverly": 30633, "ĠMarxism": 30634, "ĠBurst": 30635, "ĠKali": 30636, "estones": 30637, "Clearly": 30638, "Ell": 30639, "ãģ§": 30640, "ĠProceedings": 30641, "Token": 30642, "IFIC": 30643, "ña": 30644, "Central": 30645, "ĠHaley": 30646, "ĠDrama": 30647, "Ġformations": 30648, "ORN": 30649, "Books": 30650, "Ġdominating": 30651, "ĠFlyers": 30652, "ĠCompanion": 30653, "Ġdisciplined": 30654, "ĠYugoslav": 30655, "ĠSpells": 30656, "Ġvengeance": 30657, "Ġlandlords": 30658, "Len": 30659, "ĠOgre": 30660, "anoia": 30661, "Ġpiercing": 30662, "Ġcongreg": 30663, "Ġscorer": 30664, "obia": 30665, "Ġnickel": 30666, "ĠLearns": 30667, "Ġrejo": 30668, "Ġmasterpiece": 30669, "Flash": 30670, "Ġinhabited": 30671, "ĠOpenGL": 30672, "ĠDud": 30673, "ĠICO": 30674, "Ġarter": 30675, "Ġplur": 30676, "Ġmastery": 30677, "Ġlongstanding": 30678, "sted": 30679, "Ġwines": 30680, "Ġtelevised": 30681, "ĠShrine": 30682, "ĠBayern": 30683, "Ġâĵĺ": 30684, "Ġenclosure": 30685, "john": 30686, "Ġprophets": 30687, "ĠResurrection": 30688, "ĠOrders": 30689, "Ġuneven": 30690, "rals": 30691, "Ġdwind": 30692, "ĠLah": 30693, "ĠSloven": 30694, "378": 30695, "Ġinsistence": 30696, "affle": 30697, "ĠClone": 30698, "Ġhardship": 30699, "ĠCongressman": 30700, "Ġplead": 30701, "Ġreviewers": 30702, "Ġcured": 30703, "Ġ1935": 30704, "asley": 30705, "fake": 30706, "ĠThinking": 30707, "ydia": 30708, "PART": 30709, "ĠDota": 30710, "oit": 30711, "Ġwhipped": 30712, "Ġbouncing": 30713, "ĠHispanics": 30714, "comings": 30715, "Ġcannabin": 30716, "ĠChambers": 30717, "ĠZack": 30718, "Optional": 30719, "Ġcoats": 30720, "Ġprowess": 30721, "ĠNorton": 30722, "Ġplainly": 30723, "Ġfreight": 30724, "Ġinhibition": 30725, "Ġclam": 30726, "Ġ303": 30727, "kef": 30728, "aleigh": 30729, "Luke": 30730, "Ġpsycho": 30731, "atorium": 30732, "MED": 30733, "Ġtreaties": 30734, "Ġindisc": 30735, "Ġdc": 30736, "OPS": 30737, "Ġresilient": 30738, "ĠInterstate": 30739, "Ġslack": 30740, "Ġmundane": 30741, "Ġestablishes": 30742, "359": 30743, "Ġstrained": 30744, "Ġnond": 30745, "Sus": 30746, "Ġcaste": 30747, "arate": 30748, "ieving": 30749, "Ġunfairly": 30750, "Ġparser": 30751, "onial": 30752, "ursive": 30753, "Via": 30754, "ĠOtto": 30755, "ĠAuthorities": 30756, "stroke": 30757, "KR": 30758, "ĠMercy": 30759, "Ġfurnished": 30760, "Ġoutset": 30761, "Ġmetic": 30762, "1982": 30763, "olithic": 30764, "ĠTent": 30765, "ogical": 30766, "ĠAircraft": 30767, "Ġhides": 30768, "ĠBecame": 30769, "Ġeducators": 30770, "reaching": 30771, "Ġvolatility": 30772, "Ġtoddler": 30773, "ĠNASCAR": 30774, "ĠTwelve": 30775, "ĠHighlights": 30776, "Ġgrape": 30777, "Ġsplits": 30778, "Ġpeasant": 30779, "Ġreneg": 30780, "ĠMSI": 30781, "Temp": 30782, "stars": 30783, "Ġtrek": 30784, "ĠHyde": 30785, "binding": 30786, "Ġrealism": 30787, "Ġoxide": 30788, "ĠHos": 30789, "Ġmounts": 30790, "Ġbiting": 30791, "Ġcollapsing": 30792, "Ġpostal": 30793, "Ġmuseums": 30794, "Ġdetached": 30795, "Ġrespecting": 30796, "Ġmonopol": 30797, "Ġworkflow": 30798, "ĠCake": 30799, "Template": 30800, "ĠOrganisation": 30801, "Ġpersistence": 30802, "369": 30803, "Coming": 30804, "Brad": 30805, "Ġredundant": 30806, "ĠGTA": 30807, "Ġbending": 30808, "Ġrevoked": 30809, "Ġoffending": 30810, "Ġframing": 30811, "Ġprintf": 30812, "Commun": 30813, "members": 30814, "Outside": 30815, "Ġconstrued": 30816, "Ġcoded": 30817, "FORE": 30818, "Ġchast": 30819, "Chat": 30820, "Indian": 30821, "ĠYard": 30822, "?!\"": 30823, "ĠPorts": 30824, "ĠXavier": 30825, "ĠRET": 30826, "'.\"": 30827, "ĠBoat": 30828, "ivated": 30829, "icht": 30830, "umerable": 30831, "Ds": 30832, "ĠDunn": 30833, "Ġcoffin": 30834, "Ġsecurely": 30835, "ĠRaptors": 30836, "ĠBes": 30837, "Installation": 30838, "Ġinception": 30839, "ĠHealthy": 30840, "endants": 30841, "Ġpsychologists": 30842, "ĠSheikh": 30843, "cultural": 30844, "ĠBlackBerry": 30845, "shift": 30846, "Fred": 30847, "oche": 30848, "Ġcakes": 30849, "ĠSEO": 30850, "ĠGian": 30851, "ĠAsians": 30852, "ogging": 30853, "element": 30854, "Ġpundits": 30855, "ĠVaugh": 30856, "ĠGavin": 30857, "Ġhitter": 30858, "Ġdrowned": 30859, "Ġchalk": 30860, "ĠZika": 30861, "Ġmeasles": 30862, "802": 30863, "âĢ¦..": 30864, "ĠAWS": 30865, "]\"": 30866, "Ġdistort": 30867, "ĠMast": 30868, "Ġantibodies": 30869, "ĠMash": 30870, "Memory": 30871, "ĠUganda": 30872, "ĠProb": 30873, "Ġvomiting": 30874, "ĠTurns": 30875, "Ġoccupying": 30876, "Ġevasion": 30877, "ĠTherapy": 30878, "Ġpromo": 30879, "Ġelectr": 30880, "Ġblueprint": 30881, "ĠDre": 30882, "priced": 30883, "ĠDepot": 30884, "Ġalleviate": 30885, "ĠSomali": 30886, "marg": 30887, "nine": 30888, "Ġnostalgia": 30889, "ĠShepherd": 30890, "Ġcavalry": 30891, "Ġtorped": 30892, "ĠBloody": 30893, "xb": 30894, "Ġsank": 30895, "Ġgoalt": 30896, "reportprint": 30897, "embedreportprint": 30898, "cloneembedreportprint": 30899, "ĠInitially": 30900, "ĠFischer": 30901, "Ġnoteworthy": 30902, "cern": 30903, "Ġinefficient": 30904, "rawdownload": 30905, "rawdownloadcloneembedreportprint": 30906, "cation": 30907, "ĠDynasty": 30908, "lag": 30909, "DES": 30910, "Ġdistinctly": 30911, "ĠEstonia": 30912, "Ġopenness": 30913, "Ġgossip": 30914, "ruck": 30915, "Width": 30916, "ĠIbrahim": 30917, "Ġpetroleum": 30918, "Ġavatar": 30919, "ĠHed": 30920, "atha": 30921, "ĠHogwarts": 30922, "Ġcaves": 30923, "678": 30924, "Ġsafeguard": 30925, "ĠMog": 30926, "isson": 30927, "ĠDurham": 30928, "slaught": 30929, "ĠGraduate": 30930, "Ġsubconscious": 30931, "ĠExcellent": 30932, "ĠDum": 30933, "-----": 30934, "Ġpiles": 30935, "ĠWORK": 30936, "ĠGarn": 30937, "ĠFol": 30938, "ĠATM": 30939, "Ġavoids": 30940, "ĠTul": 30941, "Ġbleak": 30942, "ELY": 30943, "ivist": 30944, "lightly": 30945, "Pers": 30946, "ĠDob": 30947, "ĠLS": 30948, "Ġinsanity": 30949, "ε": 30950, "atalie": 30951, "Enlarge": 30952, "Ġtwists": 30953, "Ġfaulty": 30954, "Ġpiracy": 30955, "Ġimpover": 30956, "Ġrugged": 30957, "ĠFashion": 30958, "Ġsands": 30959, "'?": 30960, "swick": 30961, "Ġnatives": 30962, "Ġhen": 30963, "ĠNoise": 30964, "ãĥĹ": 30965, "Ġgreens": 30966, "Ġfreezer": 30967, "Ġdynasty": 30968, "ĠFathers": 30969, "ĠNewark": 30970, "Ġarchaeological": 30971, "Ġot": 30972, "obar": 30973, "Ġblockade": 30974, "Ġallerg": 30975, "LV": 30976, "Ġdebit": 30977, "ĠRFC": 30978, "ĠMilton": 30979, "ĠPressure": 30980, "Ġwillingly": 30981, "Ġdisproportionate": 30982, "Ġoppressive": 30983, "Ġdiamonds": 30984, "Ġbelongings": 30985, "1970": 30986, "Ġbells": 30987, "Ġimperialism": 30988, "Ġ227": 30989, "Ġexploding": 30990, "ĠEclipse": 30991, "Ġ1919": 30992, "Ġrant": 30993, "Ġnominations": 30994, "347": 30995, "Ġpeacefully": 30996, "rica": 30997, "ĠFUCK": 30998, "Ġvibration": 30999, "malink": 31000, "Ġropes": 31001, "ĠIvanka": 31002, "ĠBrewery": 31003, "ĠBooker": 31004, "ĠOwens": 31005, "goers": 31006, "Services": 31007, "ĠSnape": 31008, "Ġ191": 31009, "395": 31010, "Ġ299": 31011, "justice": 31012, "Ġbri": 31013, "Ġdiscs": 31014, "Ġprominently": 31015, "Ġvulgar": 31016, "Ġskipping": 31017, "lves": 31018, "Ġtsunami": 31019, "374": 31020, "ĠUrug": 31021, "ĠEid": 31022, "recated": 31023, "phen": 31024, "Ġfaults": 31025, "ĠStarted": 31026, "950": 31027, "Ġpi": 31028, "Ġdetector": 31029, "Ġbastard": 31030, "Ġvalidated": 31031, "SpaceEngineers": 31032, "OURCE": 31033, "Ġ(~": 31034, "Ġunsur": 31035, "Ġaffirmed": 31036, "Ġfascism": 31037, "Ġresolving": 31038, "ĠChavez": 31039, "ĠCyn": 31040, "Ġdetract": 31041, "Lost": 31042, "Ġrigged": 31043, "Ġhomage": 31044, "ĠBruno": 31045, "555": 31046, "eca": 31047, "Ġpresses": 31048, "Ġhumour": 31049, "Ġspacing": 31050, "Ġ'/": 31051, "olkien": 31052, "Coun": 31053, "OPER": 31054, "Tre": 31055, "Son": 31056, "ĠCambodia": 31057, "ierre": 31058, "mong": 31059, "ozy": 31060, "Ġliquidity": 31061, "ĠSoviets": 31062, "ĠFernando": 31063, "Ġ229": 31064, "Ġslug": 31065, "ĠCatalan": 31066, "electric": 31067, "Ġscenery": 31068, "ĠHearth": 31069, "Ġconstrained": 31070, "Ġgoalie": 31071, "ĠGuidelines": 31072, "ĠAmmo": 31073, "ĠPearson": 31074, "Ġtaxed": 31075, "Ġfetus": 31076, "Response": 31077, "ĠAlexis": 31078, "thia": 31079, "Guy": 31080, "Ġreconstruct": 31081, "Ġextremes": 31082, "Ġconcluding": 31083, "ĠPeg": 31084, "ooks": 31085, "Ġdeductions": 31086, "Rose": 31087, "Ġgroundbreaking": 31088, "ĠTarg": 31089, "ãĥģ": 31090, "ĠReve": 31091, "resource": 31092, "Ġmoons": 31093, "Ġelectromagnetic": 31094, "Ġamidst": 31095, "ĠViktor": 31096, "NESS": 31097, "BACK": 31098, "Ġcommute": 31099, "ĠAnaheim": 31100, "Ġfluctuations": 31101, "640": 31102, "Ġnoodles": 31103, "ĠCopenhagen": 31104, "ĠTide": 31105, "ĠGrizz": 31106, "ĠSEE": 31107, "Ġpipelines": 31108, "Ġscars": 31109, "endo": 31110, "agus": 31111, "ĠETF": 31112, "/#": 31113, "ĠBecome": 31114, "448": 31115, "Ġvisc": 31116, "ĠRecommended": 31117, "Ġjumper": 31118, "Ġcognition": 31119, "Ġassassin": 31120, "Ġwitnessing": 31121, "ĠSetup": 31122, "Ġlac": 31123, "vim": 31124, "ISM": 31125, "pages": 31126, "SSL": 31127, "358": 31128, "Ġadject": 31129, "industrial": 31130, "lore": 31131, "chery": 31132, "Ġglitter": 31133, "Ġcalf": 31134, "Florida": 31135, "Ġspoilers": 31136, "Ġsucceeds": 31137, "Ġchanting": 31138, "Ġslogans": 31139, "ĠTracy": 31140, "Visit": 31141, "rology": 31142, "Ġmornings": 31143, "Ġlineage": 31144, "Ġsip": 31145, "Ġintensely": 31146, "Ġflourish": 31147, "ĠSleeping": 31148, "ĠFem": 31149, "orpor": 31150, "ĠKlan": 31151, "ĠDarth": 31152, "hack": 31153, "ĠNielsen": 31154, "Ġtumors": 31155, "Ġprocurement": 31156, "ĠYorkshire": 31157, "Ġraided": 31158, "KY": 31159, "Anna": 31160, "Ġ//[": 31161, "ĠDisorder": 31162, "ĠMustang": 31163, "ĠWen": 31164, "ĠTrying": 31165, "sq": 31166, "Ġdeliveries": 31167, "Ġshutter": 31168, "Ġcerebral": 31169, "Ġbipolar": 31170, "ĠCN": 31171, "lass": 31172, "jet": 31173, "Ġdebating": 31174, ">:": 31175, "Ġeagle": 31176, "grades": 31177, "ĠDixon": 31178, "UGC": 31179, "MAS": 31180, "ĠDraco": 31181, "ĠMachines": 31182, "affer": 31183, "Ġeman": 31184, "²": 31185, "pron": 31186, "ĠGym": 31187, "Ġcomparatively": 31188, "ĠTribunal": 31189, "PRO": 31190, "Ġlex": 31191, "Ġfertile": 31192, "Ġdepressing": 31193, "Ġsuperficial": 31194, "essential": 31195, "ĠHunters": 31196, "gp": 31197, "Ġprominence": 31198, "Liber": 31199, "ĠAncest": 31200, "otechnology": 31201, "Ġmocking": 31202, "ĠTraff": 31203, "ĸļ": 31204, "Medium": 31205, "Iraq": 31206, "Ġpsychiatrist": 31207, "Quantity": 31208, "ĠLect": 31209, "Ġnoisy": 31210, "520": 31211, "GY": 31212, "Ġslapped": 31213, "ĠMTV": 31214, "Ġpara": 31215, "pull": 31216, "Multiple": 31217, "asher": 31218, "Ġnour": 31219, "ĠSeg": 31220, "Spell": 31221, "vous": 31222, "ordial": 31223, "Senior": 31224, "ĠGoldberg": 31225, "ĠPlasma": 31226, "need": 31227, "Ġmessenger": 31228, "eret": 31229, "Ġteamed": 31230, "Ġliteracy": 31231, "ĠLeah": 31232, "ĠDoyle": 31233, "Ġemitted": 31234, "UX": 31235, "Ġevade": 31236, "Ġmaze": 31237, "Ġwrongly": 31238, "ĠLars": 31239, "Ġstereotype": 31240, "Ġpledges": 31241, "Ġaroma": 31242, "ĠMET": 31243, "Ġacre": 31244, "ĠOD": 31245, "Ġff": 31246, "Ġbreweries": 31247, "ĠHilton": 31248, "undle": 31249, "ĠKak": 31250, "ĠThankfully": 31251, "ĠCanucks": 31252, "inctions": 31253, "ĠAppears": 31254, "Ġcoer": 31255, "Ġundermined": 31256, "rovers": 31257, "Andre": 31258, "Ġblaze": 31259, "umers": 31260, "Ġfamine": 31261, "amphetamine": 31262, "ulkan": 31263, "Amount": 31264, "Ġdesperation": 31265, "wikipedia": 31266, "development": 31267, "ĠCorinth": 31268, "ussia": 31269, "Jackson": 31270, "LI": 31271, "Native": 31272, "Rs": 31273, "Ohio": 31274, "ĠKathleen": 31275, "Fortunately": 31276, "Ġattendant": 31277, "ĠPreferred": 31278, "ĠDidn": 31279, "ĠVs": 31280, "Mis": 31281, "Ġrespondent": 31282, "Ġboun": 31283, "stable": 31284, "Ġpaved": 31285, "Ġunexpl": 31286, "ĠCheney": 31287, "LM": 31288, "ĠCull": 31289, "blown": 31290, "Ġconfronting": 31291, "ocese": 31292, "serving": 31293, "Wi": 31294, "ĠLithuania": 31295, "anni": 31296, "Ġstalk": 31297, "hd": 31298, "Ġvener": 31299, "APH": 31300, "ynchronous": 31301, "URR": 31302, "umably": 31303, "historic": 31304, "Half": 31305, "Hay": 31306, "Ġresilience": 31307, "spection": 31308, "Ġabandoning": 31309, "Obs": 31310, "ĠDebbie": 31311, "Ġgradient": 31312, "ĠPlaint": 31313, "ĠCanal": 31314, "ARCH": 31315, "Ġexpansive": 31316, "Ġfung": 31317, "Ġbounced": 31318, "Und": 31319, "Ġprecautions": 31320, "Ġclarification": 31321, "Ġdagger": 31322, "Ġgrips": 31323, "Ġµ": 31324, "ĠRivera": 31325, "ĠUndead": 31326, "isites": 31327, "ĠFIRST": 31328, "ño": 31329, "audi": 31330, "Ġhostages": 31331, "Ġcompliant": 31332, "Ġalumni": 31333, "Seven": 31334, "Ġcybersecurity": 31335, "either": 31336, "Collect": 31337, "Ġinvariably": 31338, "ĠSoci": 31339, "Ġlawmaker": 31340, "Ġale": 31341, "ĠPersonally": 31342, "Nazi": 31343, "Ġcustomization": 31344, "ĠProc": 31345, "ĠSaskatchewan": 31346, "eaturing": 31347, "Ġspared": 31348, "Ġdiscontinued": 31349, "Ġcomputational": 31350, "ĠMotorola": 31351, "Ġsupremacist": 31352, "governmental": 31353, "Ġparadise": 31354, "ĠDowning": 31355, "ĠNikon": 31356, "Ġcatalyst": 31357, "berra": 31358, "Toronto": 31359, "875": 31360, "beta": 31361, "ĠMacron": 31362, "Ġunrealistic": 31363, "vector": 31364, "ĠVehicles": 31365, "itiveness": 31366, "ĠRV": 31367, "ĠColbert": 31368, "sin": 31369, "oji": 31370, "entin": 31371, "ĠKrish": 31372, "hello": 31373, "ffield": 31374, "oky": 31375, "ĠTate": 31376, "Ġmaple": 31377, "Ġaids": 31378, "chemical": 31379, "334": 31380, "nuts": 31381, "ĠWarp": 31382, "Ġxx": 31383, "ĠRobb": 31384, "umerous": 31385, "_-_": 31386, "ftime": 31387, "ĠVW": 31388, "Ġwinger": 31389, "ĠDome": 31390, "tools": 31391, "ĠPV": 31392, "ĠGeorgetown": 31393, "Ġgeared": 31394, "Ġjihadists": 31395, "Ġcp": 31396, "Ġsteroids": 31397, "Mother": 31398, "clerosis": 31399, "ĠDRM": 31400, "nesia": 31401, "Ġlinger": 31402, "Ġimmersive": 31403, "ĠCOUN": 31404, "Ġoutweigh": 31405, "ensual": 31406, "Band": 31407, "Ġtransforms": 31408, "matched": 31409, "psons": 31410, "ĠJudicial": 31411, "factor": 31412, "Ġreferral": 31413, "Ġoddly": 31414, "ĠWenger": 31415, "Bring": 31416, "ĠBows": 31417, "602": 31418, "ICLE": 31419, "Ġlions": 31420, "ĠAcademic": 31421, "ĠThorn": 31422, "ĠRaider": 31423, "kefeller": 31424, "Storage": 31425, "Lower": 31426, "ĠOrt": 31427, "ĠEquality": 31428, "ALT": 31429, "ĠSOC": 31430, "Types": 31431, "Ġlyn": 31432, "ĠAsset": 31433, "coat": 31434, "TPP": 31435, "CVE": 31436, "ĠPioneer": 31437, "application": 31438, "Modern": 31439, "ĠHK": 31440, "Environment": 31441, "Alright": 31442, "Rain": 31443, "IPP": 31444, "ĠShiite": 31445, "Ġmound": 31446, "ĠAbilities": 31447, "condition": 31448, "Staff": 31449, "Ġcompetence": 31450, "ĠMoor": 31451, "ĠDiablo": 31452, "Ġwithheld": 31453, "Ġostensibly": 31454, "ĠBrom": 31455, "Ġmsg": 31456, "Ġdenomin": 31457, "ĠReferences": 31458, "ĠFP": 31459, "Ġplunged": 31460, "Ġpamph": 31461, "moving": 31462, "central": 31463, "Ġdownright": 31464, "Ġfading": 31465, "Tal": 31466, "Typ": 31467, "ĠThy": 31468, "ukes": 31469, "ithe": 31470, "Ġove": 31471, "Ġbattled": 31472, "Ġseafood": 31473, "Ġfigur": 31474, "ĠRD": 31475, "crop": 31476, "Ġsquads": 31477, "{\\": 31478, "à¹": 31479, "ĠEh": 31480, "Ġinterviewing": 31481, "ĠQin": 31482, "Ġaspiring": 31483, "PLIC": 31484, "Ġclauses": 31485, "ĠGast": 31486, "ĠNir": 31487, "Ġluggage": 31488, "Ġhose": 31489, "Ġsystemd": 31490, "Ġdescending": 31491, "ĠRevised": 31492, "ĠRails": 31493, "align": 31494, "709": 31495, "337": 31496, "Ġfug": 31497, "charging": 31498, "tags": 31499, "Ġuter": 31500, "kish": 31501, "WARNING": 31502, "490": 31503, "profits": 31504, "Ġvoyage": 31505, "Ġace": 31506, "ĠVanguard": 31507, "ĠTanks": 31508, "ĠMuk": 31509, "Ġ226": 31510, "Safe": 31511, "Armor": 31512, "Ġvolcanic": 31513, "Ġwomb": 31514, "ĠMIL": 31515, "Ġbeginner": 31516, "ĠRecogn": 31517, "ĠAAP": 31518, "PLAY": 31519, ")!": 31520, "Ġdetecting": 31521, "cn": 31522, "Ġbreaches": 31523, "Basically": 31524, "ĠPag": 31525, "ĠMunicipal": 31526, "ĠIndie": 31527, "ĠLaf": 31528, "ĠDisable": 31529, "ĠOlson": 31530, "Ġrestrained": 31531, "Ġrulings": 31532, "Ġhumane": 31533, "events": 31534, "ĠCinema": 31535, "displayText": 31536, "ĠHatch": 31537, "actionDate": 31538, "onnaissance": 31539, "Ġassaulting": 31540, "ĠLug": 31541, "CHAT": 31542, "Ġvigorous": 31543, "ĠPerse": 31544, "Ġintolerance": 31545, "ĠSnapchat": 31546, "ĠSharks": 31547, "Ġdummy": 31548, "ĠDiagn": 31549, "ĠGuitar": 31550, "imeters": 31551, "403": 31552, "REG": 31553, "Ax": 31554, "Ġseparates": 31555, "ĠMahm": 31556, "Ġtv": 31557, "jah": 31558, "OOL": 31559, "Circ": 31560, "ĠWindsor": 31561, "ussian": 31562, "Ġintuition": 31563, "Ġdisdain": 31564, "ĠDonovan": 31565, "Ġ221": 31566, "Emb": 31567, "Ġcondemning": 31568, "Ġgenerosity": 31569, "zzy": 31570, "Ġpanties": 31571, "ĠPrevent": 31572, "ActionCode": 31573, "ANA": 31574, "342": 31575, "externalActionCode": 31576, "Ġspecifying": 31577, "Ġcrystall": 31578, "Jere": 31579, "Ġrupt": 31580, "ĠApprentice": 31581, "Ġprofiling": 31582, "к": 31583, "Strike": 31584, "Ġsideline": 31585, "Ġobligated": 31586, "Ġoccult": 31587, "Ġbureaucratic": 31588, "antically": 31589, "rupted": 31590, "negative": 31591, "ĠEthiopia": 31592, "ĠCivic": 31593, "Ġinsiders": 31594, "eligible": 31595, "ĠTVs": 31596, "ĠBAR": 31597, "ĠTI": 31598, "iologist": 31599, "ĠAIR": 31600, "Ġsubstituted": 31601, "Arab": 31602, "ĠSaul": 31603, "ĠYog": 31604, "prem": 31605, "Ġbuilders": 31606, "Ġstationary": 31607, "Ġdoubtful": 31608, "Ġvigorously": 31609, "Ġthrilling": 31610, "Physical": 31611, "ĠCarey": 31612, "ĠHydra": 31613, "geoning": 31614, "ĠSly": 31615, "yton": 31616, "Ġborrowers": 31617, "ĠParkinson": 31618, "Ġë": 31619, "ĠJamaica": 31620, "Ġsatir": 31621, "Ġinsurgents": 31622, "ĠFirm": 31623, "Ġisot": 31624, "ĠKarn": 31625, "ourning": 31626, "akens": 31627, "docs": 31628, "little": 31629, "ĠMonaco": 31630, "CLASS": 31631, "Turkey": 31632, "Ly": 31633, "ĠConan": 31634, "assic": 31635, "Ġstarred": 31636, "ĠPacers": 31637, "eties": 31638, "Ġtipping": 31639, "Moon": 31640, "ĠRw": 31641, "same": 31642, "Ġcavity": 31643, "Ġgoof": 31644, "ĠZo": 31645, "Shock": 31646, "ummer": 31647, "Ġemphasizes": 31648, "Ġregrett": 31649, "Ġnovelty": 31650, "Ġenvy": 31651, "ĠPassive": 31652, "rw": 31653, "505": 31654, "Ġindifferent": 31655, "ĠRica": 31656, "ĠHimself": 31657, "ĠFreddie": 31658, "Ġadip": 31659, "ä¸Ģ": 31660, "Ġbreakout": 31661, "Ġhurried": 31662, "ĠHuang": 31663, "ĠDisk": 31664, "Ġroaming": 31665, "?????-?????-": 31666, "UV": 31667, "ĠRicky": 31668, "ĠSigma": 31669, "Ġmarginalized": 31670, "Ġedits": 31671, "Ġ304": 31672, "memory": 31673, "Ġspecimen": 31674, "293": 31675, "ãģ¯": 31676, "Ġvertically": 31677, "Ġaudition": 31678, "ĠHeck": 31679, "Ġcaster": 31680, "ĠHoldings": 31681, "adal": 31682, "ĠCron": 31683, "ĠLiam": 31684, "Ġdeflect": 31685, "Pick": 31686, "ĠDebug": 31687, "REF": 31688, "Ġversatility": 31689, "othes": 31690, "classified": 31691, "ĠMahar": 31692, "ĠHort": 31693, "Counter": 31694, "stasy": 31695, "noticed": 31696, "331": 31697, "ĠShim": 31698, "fuck": 31699, "ĠBie": 31700, "Ġairing": 31701, "ĠProtein": 31702, "ĠHolding": 31703, "Ġspectators": 31704, "iliated": 31705, "ĠThatcher": 31706, "nosis": 31707, "ãĥ¼ãĥ³": 31708, "Tele": 31709, "Boston": 31710, "ĠTempl": 31711, "stay": 31712, "Ġdeclarations": 31713, "479": 31714, "Volume": 31715, "ĠDesigner": 31716, "ĠOverwatch": 31717, "idae": 31718, "Ġonwards": 31719, "Ġnets": 31720, "ĠManila": 31721, "particularly": 31722, "Ġpolitic": 31723, "oother": 31724, "Ġportraits": 31725, "Ġpavement": 31726, "cffff": 31727, "Ġsaints": 31728, "Ġbeginners": 31729, "ESPN": 31730, "Ġshortcomings": 31731, "âķIJâķIJ": 31732, "Ġcomet": 31733, "ĠOrganic": 31734, "quel": 31735, "Ġhospitalized": 31736, "Break": 31737, "Ġpeel": 31738, "dylib": 31739, "aspx": 31740, "urances": 31741, "ĠTIM": 31742, "Pg": 31743, "Ġreadable": 31744, "ĠMalik": 31745, "Ġmuzzle": 31746, "Ġbenchmarks": 31747, "dal": 31748, "ĠVacc": 31749, "ĠHicks": 31750, "609": 31751, "ĠBiblical": 31752, "heng": 31753, "Ġoverload": 31754, "ĠCivilization": 31755, "Ġimmoral": 31756, "Ġfries": 31757, "ãĤĴ": 31758, "Ġreproduced": 31759, "Ġformulation": 31760, "jug": 31761, "irez": 31762, "gear": 31763, "Ġcoached": 31764, "MpServer": 31765, "ĠSJ": 31766, "ĠKw": 31767, "Init": 31768, "deal": 31769, "ĠOro": 31770, "ĠLoki": 31771, "ĠSongs": 31772, "Ġ232": 31773, "ĠLouise": 31774, "asionally": 31775, "Ġuncond": 31776, "ollywood": 31777, "Ġprogressives": 31778, "ĠEnough": 31779, "ĠDoe": 31780, "Ġwreckage": 31781, "Ġbrushed": 31782, "ĠBaseType": 31783, "Ġzoning": 31784, "ishable": 31785, "hetically": 31786, "ĠCaucus": 31787, "ĠHue": 31788, "Ġkarma": 31789, "ĠSporting": 31790, "Ġtrader": 31791, "Ġseeming": 31792, "ĠCapture": 31793, "430": 31794, "bish": 31795, "Ġtunes": 31796, "Ġindoors": 31797, "ĠSphere": 31798, "ĠDancing": 31799, "TERN": 31800, "Ġnob": 31801, "ĠGST": 31802, "maps": 31803, "Ġpeppers": 31804, "Fit": 31805, "Ġoversees": 31806, "ĠRabbi": 31807, "ĠRuler": 31808, "vertising": 31809, "office": 31810, "xxx": 31811, "Ġraft": 31812, "Changed": 31813, "Ġtextbooks": 31814, "Links": 31815, "ĠOmn": 31816, "ãĢij": 31817, "Ġinconvenience": 31818, "ĠDonetsk": 31819, "=~": 31820, "Ġimplicitly": 31821, "Ġboosts": 31822, "ĠBones": 31823, "ĠBoom": 31824, "Courtesy": 31825, "Ġsensational": 31826, "ANY": 31827, "Ġgreedy": 31828, "eden": 31829, "Ġinexper": 31830, "ĠLer": 31831, "ĠVale": 31832, "Ġtighten": 31833, "ĠEAR": 31834, "ĠNum": 31835, "Ġancestor": 31836, "Sent": 31837, "ĠHorde": 31838, "urgical": 31839, "allah": 31840, "Ġsap": 31841, "amba": 31842, "ĠSpread": 31843, "twitch": 31844, "Ġgrandson": 31845, "Ġfracture": 31846, "Ġmoderator": 31847, "ĠSeventh": 31848, "ĠReverse": 31849, "Ġestimation": 31850, "Choose": 31851, "Ġparach": 31852, "Ġbarric": 31853, "ãĢIJ": 31854, "Ġcompass": 31855, "Ġallergic": 31856, "âĢķ": 31857, "OTHER": 31858, "errilla": 31859, "Ġwagon": 31860, "Ġzinc": 31861, "Ġrubbed": 31862, "ĠFuller": 31863, "ĠLuxembourg": 31864, "ĠHoover": 31865, "Ġliar": 31866, "ĠEvening": 31867, "ĠCobb": 31868, "esteem": 31869, "Ġselector": 31870, "ĠBrawl": 31871, "isance": 31872, "ĠEk": 31873, "Ġtroop": 31874, "Ġguts": 31875, "ĠAppeal": 31876, "ĠTibetan": 31877, "Ġroutines": 31878, "ĠMent": 31879, "Ġsummarized": 31880, "steamapps": 31881, "Ġtranqu": 31882, "Ġ1929": 31883, "oran": 31884, "ĠAuthent": 31885, "Ġgmaxwell": 31886, "Ġapprehens": 31887, "Ġpoems": 31888, "Ġsausage": 31889, "ĠWebster": 31890, "urus": 31891, "Ġthemed": 31892, "Ġlounge": 31893, "Ġcharger": 31894, "Spoiler": 31895, "Ġspilled": 31896, "hog": 31897, "ĠSunder": 31898, "ĠAin": 31899, "ĠAngry": 31900, "Ġdisqual": 31901, "ĠFrequency": 31902, "ĠEthernet": 31903, "Ġhelper": 31904, "Percent": 31905, "Ġhorrifying": 31906, "Ġail": 31907, "ĠAllan": 31908, "EEE": 31909, "ĠCrossing": 31910, "449": 31911, "Ġholog": 31912, "ĠPuzzles": 31913, "ĠGoes": 31914, "erenn": 31915, "604": 31916, "ãģı": 31917, "ĠRafael": 31918, "Ġatten": 31919, "ĠEmanuel": 31920, "Ġupro": 31921, "ĠSusp": 31922, "Psych": 31923, "ĠTrainer": 31924, "ĠNES": 31925, "ĠHunts": 31926, "becue": 31927, "Ġcounselor": 31928, "Rule": 31929, "Ġtoxins": 31930, "Ġbanners": 31931, "rifice": 31932, "Ġgreeting": 31933, "Ġfrenzy": 31934, "Ġallocate": 31935, "Ġ*)": 31936, "expr": 31937, "503": 31938, "ĠChick": 31939, "ĠTorn": 31940, "Ġconsolidation": 31941, "ĠFletcher": 31942, "switch": 31943, "frac": 31944, "clips": 31945, "ĠMcKin": 31946, "ĠLunar": 31947, "Month": 31948, "ITCH": 31949, "Ġscholarly": 31950, "raped": 31951, "398": 31952, "Ġ1910": 31953, "Ġegreg": 31954, "Ġinsecure": 31955, "Ġvictorious": 31956, "cffffcc": 31957, "Ġsingled": 31958, "Ġelves": 31959, "ĠWond": 31960, "burst": 31961, "Ġcamoufl": 31962, "ĠBLACK": 31963, "Ġconditioned": 31964, "çī": 31965, "answered": 31966, "Ġcompulsory": 31967, "ascist": 31968, "Ġpodcasts": 31969, "ĠFrankfurt": 31970, "bnb": 31971, "Ġneoliberal": 31972, "ĠKeyboard": 31973, "ĠBelle": 31974, "warm": 31975, "Ġtrusts": 31976, "Ġinsured": 31977, "ĠBucc": 31978, "usable": 31979, "607": 31980, "ĠPlains": 31981, "Ġ1890": 31982, "Ġsabotage": 31983, "Ġlodged": 31984, "felt": 31985, "Ġga": 31986, "ĠNarc": 31987, "ĠSalem": 31988, "Ġseventy": 31989, "ĠBlank": 31990, "pocket": 31991, "Ġwhisper": 31992, "Ġmating": 31993, "omics": 31994, "ĠSalman": 31995, "ĠKad": 31996, "Ġangered": 31997, "Ġcollisions": 31998, "Ġextraordinarily": 31999, "Ġcoercion": 32000, "Ghost": 32001, "birds": 32002, "èĢ": 32003, "kok": 32004, "Ġpermissible": 32005, "avorable": 32006, "Ġpointers": 32007, "Ġdissip": 32008, "aci": 32009, "Ġtheatrical": 32010, "ĠCosmic": 32011, "Ġforgetting": 32012, "Ġfinalized": 32013, "大": 32014, "yout": 32015, "library": 32016, "Ġbooming": 32017, "ĠBelieve": 32018, "ĠTeacher": 32019, "ĠLiv": 32020, "ĠGOODMAN": 32021, "ĠDominican": 32022, "ORED": 32023, "ĠParties": 32024, "Ġprecipitation": 32025, "ĠSlot": 32026, "Roy": 32027, "ĠCombined": 32028, "Ġintegrating": 32029, "Ġchrome": 32030, "Ġintestinal": 32031, "ĠRebell": 32032, "Ġmatchups": 32033, "Ġblockbuster": 32034, "ĠLoren": 32035, "ĠLevy": 32036, "Ġpreaching": 32037, "ĠSending": 32038, "ĠPurpose": 32039, "rax": 32040, "fif": 32041, "Ġauthoritative": 32042, "ĠPET": 32043, "astical": 32044, "Ġdishon": 32045, "Ġchatting": 32046, "Ġ\"$:/": 32047, "Connection": 32048, "Ġrecreate": 32049, "Ġdelinqu": 32050, "Ġbroth": 32051, "ĠDirty": 32052, "ĠAdmin": 32053, "zman": 32054, "Ġscholarships": 32055, "Ġ253": 32056, "contact": 32057, "alsa": 32058, "767": 32059, "creen": 32060, "abbage": 32061, "Ġ1915": 32062, "Ġblended": 32063, "Ġalarmed": 32064, "Language": 32065, "356": 32066, "Ġblends": 32067, "ĠChanged": 32068, "Wolf": 32069, "Ġhepat": 32070, "Creating": 32071, "Ġpersecut": 32072, "Ġsweetness": 32073, "arte": 32074, "Ġforfeiture": 32075, "ĠRoberto": 32076, "impro": 32077, "NFL": 32078, "ĠMagnet": 32079, "Detailed": 32080, "Ġinsignificant": 32081, "ĠPOLIT": 32082, "ĠBBQ": 32083, "ĠCPS": 32084, "Ġseaw": 32085, "aminer": 32086, "mL": 32087, "endif": 32088, "finals": 32089, "Ġ265": 32090, "uish": 32091, "Ġ})": 32092, "ĠProblems": 32093, "Ġemblem": 32094, "Ġseriousness": 32095, "Ġparsing": 32096, "Ġsubstitution": 32097, "Ġpressured": 32098, "Ġrecycled": 32099, "aleb": 32100, "Ruby": 32101, "Ġproficiency": 32102, "Driver": 32103, "ĠWester": 32104, ":'": 32105, "AFTA": 32106, "Ġmantle": 32107, "ĠClayton": 32108, "flag": 32109, "Ġpractitioner": 32110, "covered": 32111, "ĠStruct": 32112, "addafi": 32113, "425": 32114, "ĠTownship": 32115, "ĠHydro": 32116, "Louis": 32117, "343": 32118, "Ġcondo": 32119, "ĠTao": 32120, "Ġutilization": 32121, "Ġnausea": 32122, "ĠDems": 32123, "ridges": 32124, "pause": 32125, "Ġformulas": 32126, "Ġchallenger": 32127, "376": 32128, "Ġdefective": 32129, "ĠRailway": 32130, "ĠPubMed": 32131, "Ġyogurt": 32132, "lbs": 32133, "ĠNorfolk": 32134, "OPE": 32135, "ĠMoody": 32136, "Ġdistributor": 32137, "Ġscrolls": 32138, "Ġextracts": 32139, "Stan": 32140, "Ġviability": 32141, "Ġexposes": 32142, "Ġstarvation": 32143, "ĠSteps": 32144, "ĠDodd": 32145, "few": 32146, "STD": 32147, "332": 32148, "Ġclosures": 32149, "Ġcomplementary": 32150, "ĠSasha": 32151, "umpy": 32152, "Ġmonet": 32153, "Ġarticulate": 32154, "ĠDoct": 32155, "killer": 32156, "Ġscrim": 32157, "Ġ264": 32158, "Ġprostitutes": 32159, "Ġsevered": 32160, "Ġattachments": 32161, "Ġcooled": 32162, "Lev": 32163, "ĠFalk": 32164, "fail": 32165, "Ġpoliceman": 32166, "ĠDag": 32167, "Ġprayed": 32168, "ĠKernel": 32169, "Ġclut": 32170, "Ġcath": 32171, "Ġanomaly": 32172, "Storm": 32173, "emaker": 32174, "ĠBreakfast": 32175, "uli": 32176, "oire": 32177, "JJ": 32178, "hz": 32179, "Operation": 32180, "ĠSick": 32181, "354": 32182, "ĠGuatemala": 32183, "Rate": 32184, "Ġexposures": 32185, "faces": 32186, "ĠArchae": 32187, "raf": 32188, "ĠMia": 32189, "Ġ2025": 32190, "Ġopaque": 32191, "Ġdisguised": 32192, "ĠHeadquarters": 32193, "Sah": 32194, "Ġpots": 32195, "978": 32196, "ĠMalf": 32197, "Ġfrowned": 32198, "Ġpoisonous": 32199, "ĠConvers": 32200, "eeks": 32201, "Ġcrab": 32202, ".\"\"": 32203, "Ġtreason": 32204, "Ġranc": 32205, "Ġescalating": 32206, "Ġwarr": 32207, "Ġmobs": 32208, "Ġlamps": 32209, "ĠSunshine": 32210, "ĠBrunswick": 32211, "Phones": 32212, "Ġspelled": 32213, "ĠSkip": 32214, "Ġ2050": 32215, "Ġ1911": 32216, "ĠPluto": 32217, "ĠAmend": 32218, "Ġmeats": 32219, "387": 32220, "Ġstomp": 32221, "ĠZhou": 32222, "ĠLeviathan": 32223, "ĠHazard": 32224, "adv": 32225, "ĠOrwell": 32226, "Ġaloud": 32227, "Ġbumper": 32228, "ĠAnarch": 32229, "ubuntu": 32230, "ĠSerious": 32231, "fitting": 32232, "ĠOptional": 32233, "ĠCecil": 32234, "REAM": 32235, "Ġserotonin": 32236, "Ġcultivate": 32237, "agogue": 32238, "}\\": 32239, "Ġmosques": 32240, "ĠSunny": 32241, "Ġreactive": 32242, "revolution": 32243, "ĠLup": 32244, "ĠFedora": 32245, "Ġdefenseman": 32246, "ĠVID": 32247, "istine": 32248, "Ġdrowning": 32249, "ĠBroadcasting": 32250, "Ġthriller": 32251, "ĠScy": 32252, "Ġaccelerating": 32253, "Ġdirects": 32254, "odied": 32255, "bike": 32256, "duration": 32257, "Ġpainfully": 32258, "Redd": 32259, "Ġproductions": 32260, "Ġgag": 32261, "Ġwhist": 32262, "Ġsock": 32263, "Ġinfinitely": 32264, "ĠConcern": 32265, "ĠCitadel": 32266, "Ġlieu": 32267, "Ġcandles": 32268, "ogeneous": 32269, "arger": 32270, "Ġheavenly": 32271, "inflammatory": 32272, "Performance": 32273, "Cs": 32274, "ructose": 32275, "azaki": 32276, "Ġpessim": 32277, "Ġinference": 32278, "Ġpowd": 32279, "ĠZoe": 32280, "Ġpaints": 32281, "Ġdazz": 32282, "pta": 32283, "-----------": 32284, "Ġinspir": 32285, "ĠExperimental": 32286, "ĠKnife": 32287, "regor": 32288, "bors": 32289, "Ġshowers": 32290, "romeda": 32291, "Ġsaint": 32292, "Ġbenign": 32293, "ĠJiang": 32294, "Ġenvisioned": 32295, "Ġshroud": 32296, "IFT": 32297, "HO": 32298, "Ġshuff": 32299, "ĠICC": 32300, "Ġsegreg": 32301, "Ġrevisit": 32302, "ighthouse": 32303, "Li": 32304, "Ġsubstrate": 32305, "ĠSeas": 32306, "ĠReward": 32307, "ĠHep": 32308, "ĠBrass": 32309, "sbm": 32310, "Ġeliminates": 32311, "Ġstamina": 32312, "ĠVAT": 32313, "ĠLoan": 32314, "Ġconstraint": 32315, "Ġappropriated": 32316, "Ġpes": 32317, "ĠALE": 32318, "ranging": 32319, "Ġ404": 32320, "392": 32321, "Ġintellectuals": 32322, "achu": 32323, "Ġrestructuring": 32324, "ĠLevin": 32325, "Ġrunes": 32326, "Ġdelightful": 32327, "Ġcarbohydrates": 32328, "ĠModels": 32329, "ĠExpo": 32330, "Ġtransporting": 32331, "alloc": 32332, "Ġringing": 32333, "Samsung": 32334, "Ġscarcely": 32335, "ĠURLs": 32336, "ĠMAS": 32337, "Ġprototypes": 32338, "Ġnarrator": 32339, "ĠCPUs": 32340, "cdn": 32341, "ĠBarton": 32342, "Ġdecidedly": 32343, "ĠShu": 32344, "ixir": 32345, "ocious": 32346, "ĠMyst": 32347, "Nintendo": 32348, "Ġreuse": 32349, "Ġforgiven": 32350, "Few": 32351, "inical": 32352, "nat": 32353, "Ġseamless": 32354, "ĠEva": 32355, "ĠEVE": 32356, "ĠJO": 32357, "landers": 32358, "Ġsofter": 32359, "negie": 32360, "Ġtransient": 32361, "Ġorbital": 32362, "Ġfulfil": 32363, "ĠKom": 32364, "Hopefully": 32365, "Ġdynamically": 32366, "ĠHunger": 32367, "åĽ": 32368, "ĠArmenia": 32369, "elman": 32370, "berto": 32371, "Ġpige": 32372, "ĠIDs": 32373, "limit": 32374, "Ġveins": 32375, "Ġsoaring": 32376, "packs": 32377, "Golden": 32378, "ĠCrab": 32379, "istor": 32380, "ĠRPM": 32381, "Ġ$$": 32382, "gression": 32383, "Ġjihadist": 32384, "Ġgamble": 32385, "Ġcareg": 32386, "Ġinflated": 32387, "Face": 32388, "ĠFirearms": 32389, "ĠEmmanuel": 32390, "âĿ": 32391, "Ġshocks": 32392, "grab": 32393, "Ġsplend": 32394, "ĠHPV": 32395, "abortion": 32396, "Above": 32397, "Entity": 32398, "players": 32399, "Ġcommenced": 32400, "ulence": 32401, "Ġfulfillment": 32402, "Ġembodiments": 32403, "ĠWelfare": 32404, "Ġhail": 32405, "Ġ<@": 32406, "tten": 32407, "Ġcatcher": 32408, "ĠJazeera": 32409, "Ġvolcano": 32410, "Ġstabilize": 32411, "ĠHandler": 32412, "Ġintensified": 32413, "ĠAbrams": 32414, "Ġhumiliation": 32415, "paced": 32416, "605": 32417, "ĠCentOS": 32418, "Specific": 32419, "Ġheed": 32420, "ĠCAM": 32421, "ĠGalile": 32422, "Die": 32423, "Ġabolished": 32424, "ĠThomson": 32425, "ĠTeachers": 32426, "ĠWass": 32427, "jong": 32428, "ĠISBN": 32429, "ĠAllies": 32430, "shake": 32431, "å·": 32432, "vict": 32433, "Howard": 32434, "Ġdeem": 32435, "Ġexceedingly": 32436, "ĠSmartstocks": 32437, "ibe": 32438, "Ġdoorway": 32439, "Ġcompeted": 32440, "igmat": 32441, "Ġnationalists": 32442, "Ġgroom": 32443, "ĠKeen": 32444, "Ġdisposable": 32445, "decl": 32446, "ĠTolkien": 32447, "ĠScheme": 32448, "Ġbiod": 32449, "Ġavid": 32450, "ĠElon": 32451, "agar": 32452, "ĠTSA": 32453, "Roman": 32454, "Ġartificially": 32455, "Ġadvisors": 32456, "XL": 32457, "ĠInferno": 32458, "366": 32459, "Ġtedious": 32460, "ĠPhotography": 32461, "ĠCarrie": 32462, "Ġtrope": 32463, "ĠSandra": 32464, "Ġdecimal": 32465, "Queen": 32466, "ĠGundam": 32467, "ĠOM": 32468, "otech": 32469, "NBA": 32470, "Ġ1932": 32471, "Ġentrenched": 32472, "ĠMarion": 32473, "Ġfraternity": 32474, "Labour": 32475, "Henry": 32476, "Ġlatitude": 32477, "Either": 32478, "Ġenhances": 32479, "ĠPotential": 32480, "Ġshines": 32481, "idad": 32482, "Ġbreadth": 32483, "Ġcapacities": 32484, "ĠðŁĻĤ": 32485, "ĠBronx": 32486, "Ġsexes": 32487, "Ġdifferentiation": 32488, "Ġheavyweight": 32489, "ĠTaj": 32490, "dra": 32491, "Ġmigrate": 32492, "Ġexhaustion": 32493, "ĠRUN": 32494, "elsius": 32495, "ĠCuomo": 32496, "Ġguitars": 32497, "Ġclones": 32498, "ĠSomew": 32499, "ĠPry": 32500, "-------------": 32501, "Ġwarranted": 32502, "cycles": 32503, "Ġsalvage": 32504, "Ġdisks": 32505, "RANT": 32506, "ĠNGOs": 32507, "ĠMartian": 32508, "\":[{\"": 32509, "Ġaddicts": 32510, "ojure": 32511, "illet": 32512, "Ġamazingly": 32513, "artments": 32514, "pixel": 32515, "ĠGPUs": 32516, "Layout": 32517, "è£": 32518, "ĠTamil": 32519, "ĠBasil": 32520, "Ġimpartial": 32521, "ĠStructure": 32522, "fork": 32523, "bryce": 32524, "Ġridge": 32525, "ĠHamburg": 32526, "rious": 32527, "Ġblitz": 32528, "cigarettes": 32529, "Ġcanned": 32530, "402": 32531, "Ġironically": 32532, "Ġcompassionate": 32533, "ĠHawkins": 32534, ".#": 32535, "ĠCathedral": 32536, "Ġrallied": 32537, "internal": 32538, "Ġquota": 32539, "stakes": 32540, "TEXT": 32541, "mom": 32542, "Ġcompletes": 32543, "Ġ238": 32544, "Ġshrug": 32545, "ãĥij": 32546, "ĠNinth": 32547, "Ġrevise": 32548, "ĠProvider": 32549, "Ġtreacher": 32550, "Ġquasi": 32551, "ĠPRES": 32552, "Ġdeposition": 32553, "Ġconfidentiality": 32554, "issors": 32555, "Ġimbalance": 32556, "Ġspanning": 32557, "Ġangular": 32558, "ĠCul": 32559, "communication": 32560, "ĠNora": 32561, "ĠGenius": 32562, "opter": 32563, "Ġsacked": 32564, "Spot": 32565, "Ġfinely": 32566, "ĠCHR": 32567, "282": 32568, "waves": 32569, "Palest": 32570, "ĠRohing": 32571, "NL": 32572, "è¿": 32573, "Ġshitty": 32574, "ĠScalia": 32575, "475": 32576, "Progress": 32577, "Ġreferencing": 32578, "Ġclassrooms": 32579, "abee": 32580, "Ġsod": 32581, "hesion": 32582, "708": 32583, "ĠZuckerberg": 32584, "ĠFinish": 32585, "ĠScotia": 32586, "ĠSavior": 32587, "ĠInstallation": 32588, "antha": 32589, "(-": 32590, "Ġ302": 32591, "ĠPunk": 32592, "Ġcrater": 32593, "youtu": 32594, "Ġroast": 32595, "Ġinfluencing": 32596, "Ġdup": 32597, "ĠJR": 32598, "ĠGrav": 32599, "Ġstature": 32600, "Ġbathrooms": 32601, "Aside": 32602, "Wiki": 32603, "mean": 32604, "ĠZak": 32605, "ĠOnes": 32606, "ĠNath": 32607, "Ġhypert": 32608, "Ġcommencement": 32609, "Civil": 32610, "Ġmoderately": 32611, "Ġdistributors": 32612, "Ġbreastfeeding": 32613, "Ġ980": 32614, "ĠSik": 32615, "ĠCig": 32616, "ĠAMER": 32617, "RIP": 32618, "ĠCareer": 32619, "usting": 32620, "Ġmessed": 32621, "Ġeh": 32622, "ĠJensen": 32623, "/$": 32624, "Ġblackmail": 32625, "Ġconversions": 32626, "Ġscientifically": 32627, "Ġmantra": 32628, "paying": 32629, "Ġivory": 32630, "ĠCourts": 32631, "OUGH": 32632, "auntlet": 32633, "Serial": 32634, "Brow": 32635, "ĠHundreds": 32636, "323": 32637, "Ġpee": 32638, "Ġlinux": 32639, "Ġsubmer": 32640, "ĠPrincipal": 32641, "485": 32642, "ĠDSL": 32643, "ĠCousins": 32644, "Ġdoctrines": 32645, "ĠAthletics": 32646, "Ġ315": 32647, "ĠKarma": 32648, "Ġattent": 32649, "urger": 32650, "Ġprescribe": 32651, "Ġencaps": 32652, "ĠCame": 32653, "Ġsecretive": 32654, "ĠCrimes": 32655, "dn": 32656, "Clean": 32657, "ĠEgyptians": 32658, "ĠCarpenter": 32659, "Ġll": 32660, "Hum": 32661, "ĠMilo": 32662, "Ġcapitalists": 32663, "Ġbriefed": 32664, "Twe": 32665, "ĠBasin": 32666, "elvet": 32667, "Mos": 32668, "Ġplunge": 32669, "ĠKaiser": 32670, "ĠFuj": 32671, "illin": 32672, "Ġsafeguards": 32673, "Ġoste": 32674, "ĠOpportunity": 32675, "ĠMafia": 32676, "ĠCalling": 32677, "apa": 32678, "urban": 32679, "brush": 32680, "illard": 32681, "cé": 32682, "intelligence": 32683, "ĠLob": 32684, "ĠDruid": 32685, "Ġsmoother": 32686, "Ġfooting": 32687, "Ġmotorists": 32688, "arcity": 32689, "Ġmasculinity": 32690, "Ġmism": 32691, "Ġabdominal": 32692, "ĠTavern": 32693, "ĠRoh": 32694, "Ġescapes": 32695, "signed": 32696, "Anthony": 32697, "Ġsacrificing": 32698, "Ġintimacy": 32699, "Ġanterior": 32700, "ĠKod": 32701, "Ġmotif": 32702, "Ġgraz": 32703, "Ġvisualization": 32704, "Ġguitarist": 32705, "ĠTrotsky": 32706, "magic": 32707, "Dar": 32708, "ĠMori": 32709, "Ġwards": 32710, "Ġtoilets": 32711, "lest": 32712, "Ġteleport": 32713, "ĠSundays": 32714, "ĠPlat": 32715, "ETS": 32716, "ĠeSports": 32717, "Patrick": 32718, "ĠKatherine": 32719, "enko": 32720, "Ġhassle": 32721, "ĠMick": 32722, "ggles": 32723, "Ġhob": 32724, "aintain": 32725, "Ġairborne": 32726, "Ġspans": 32727, "Ġchili": 32728, "Ġaperture": 32729, "Ġvolunteered": 32730, "ĠIncident": 32731, "ĠFres": 32732, "ĠVeteran": 32733, "aughtered": 32734, "ingo": 32735, "Ġuninsured": 32736, "CLOSE": 32737, "Ġfuse": 32738, "Ġerotic": 32739, "Ġadvertise": 32740, "raising": 32741, "Texture": 32742, "Ġattends": 32743, "ĠREAL": 32744, "uddled": 32745, "Ġsmoot": 32746, "Ġ305": 32747, "ĠWillis": 32748, "Ġblond": 32749, "Analysis": 32750, "ĠVT": 32751, "onica": 32752, "Ġstronghold": 32753, "RF": 32754, "NM": 32755, ".>>": 32756, "Ġprosperous": 32757, "Ġboasted": 32758, "292": 32759, "ĠManufacturing": 32760, "PRESS": 32761, "gren": 32762, "Ġpharmacy": 32763, "ĠRockefeller": 32764, "kai": 32765, "Ġthumbs": 32766, "ĠHut": 32767, "Ġmotherboard": 32768, "Ġguardians": 32769, "ĠAlter": 32770, "llular": 32771, "Ġshack": 32772, "Ġwisely": 32773, "Ġbackbone": 32774, "erva": 32775, "Ġsuicides": 32776, "ĠMcGregor": 32777, "ijah": 32778, "Emer": 32779, "ĠBrav": 32780, "Ġdesignate": 32781, "POST": 32782, "produced": 32783, "Ġcleansing": 32784, "irlwind": 32785, "existent": 32786, "ĠHumph": 32787, "ĠPayne": 32788, "Ġvested": 32789, "Å¡": 32790, "Ġstringent": 32791, "iona": 32792, "Ġunsub": 32793, "Ġsummed": 32794, "ĠHercules": 32795, "subject": 32796, "ĠRagnar": 32797, "ĠNos": 32798, "Ġcharacterization": 32799, "Ġsavvy": 32800, "ĠDawson": 32801, "ĠCasino": 32802, "Ġfri": 32803, "ĠBarrier": 32804, "Ġmisinformation": 32805, "Ġinsulation": 32806, "Ġcorridors": 32807, "Ġairplanes": 32808, "ĠNoct": 32809, "ahi": 32810, "Ġ1916": 32811, "kb": 32812, "armac": 32813, "Ġshun": 32814, "Ġschema": 32815, "Ġhorrified": 32816, "Ġ239": 32817, "aunders": 32818, "NB": 32819, "iates": 32820, "erity": 32821, "ĠShard": 32822, "Ġrarity": 32823, "Ġgrouped": 32824, "ĠGhana": 32825, "against": 32826, "ĠBiological": 32827, "ĠAware": 32828, "owell": 32829, "ÏĦ": 32830, "ĠBeau": 32831, "shaw": 32832, "Hack": 32833, "ĠJulius": 32834, "USS": 32835, "olson": 32836, "auna": 32837, "cru": 32838, "ĠMaurice": 32839, "ĠIk": 32840, "Ġsequencing": 32841, "Ġradicals": 32842, "Ġ(?,": 32843, "virtual": 32844, "Ġanyways": 32845, "Ġreperc": 32846, "Ġhandlers": 32847, "Ġhesitant": 32848, "éĥ": 32849, "ĠMF": 32850, "plementation": 32851, "associated": 32852, "Ġcampaigned": 32853, "ĠYue": 32854, "utations": 32855, "ĠYoga": 32856, "Ġsimmer": 32857, "Ġrods": 32858, "Ġmelody": 32859, "Ġconvoy": 32860, "videos": 32861, "Ġscreened": 32862, "Neg": 32863, "ochemical": 32864, "Ġ())": 32865, "Ġultras": 32866, "Ġantip": 32867, "ĠIslanders": 32868, "704": 32869, "Ġfetish": 32870, "Ġridiculously": 32871, "ĠKart": 32872, "Ġmitochondrial": 32873, "Ġinterfering": 32874, "Builder": 32875, "Ġoverfl": 32876, "Ġacne": 32877, "ĠMud": 32878, "ĠKerr": 32879, "flex": 32880, "ĠPostal": 32881, "ĠBaltic": 32882, "477": 32883, "ĠPersons": 32884, "ourage": 32885, "HB": 32886, "ĠMuse": 32887, "ĠImmortal": 32888, "ĠDriving": 32889, "Ġpetitions": 32890, "Ġsubscript": 32891, "Ġsorce": 32892, "ĠProcessor": 32893, "uton": 32894, "Sony": 32895, "Ġphon": 32896, "Ġraced": 32897, "ĠAnthrop": 32898, "Ġdaytime": 32899, "ĠExercise": 32900, "Adding": 32901, "Ġengages": 32902, "ĠQualcomm": 32903, "Ġmiracles": 32904, "Ġmemes": 32905, "ĠDrink": 32906, "ĠOrioles": 32907, "Ġhairs": 32908, "ĠPolar": 32909, "athom": 32910, "Ġslippery": 32911, "ĠRemy": 32912, "Ġcaramel": 32913, "ĠYEAR": 32914, "Ġalk": 32915, "Ign": 32916, "aution": 32917, "ĠMerlin": 32918, "ĠCran": 32919, "Ġapologies": 32920, "Ġ410": 32921, "Ġouting": 32922, "ĠMemories": 32923, "appointed": 32924, "Ġcountered": 32925, "uld": 32926, "posing": 32927, "Ġfirewall": 32928, "ĠWast": 32929, "ĠWet": 32930, "worked": 32931, "seller": 32932, "Ġrepealed": 32933, "ereo": 32934, "assuming": 32935, "BLIC": 32936, "mite": 32937, "ĠCEOs": 32938, "ĠChapel": 32939, "elligent": 32940, "________________________": 32941, "Dog": 32942, "Ġwart": 32943, "Ġsubscriber": 32944, "sports": 32945, "Ġbegged": 32946, "ĠMV": 32947, "Ġsemif": 32948, "ethical": 32949, "Ġpreach": 32950, "Ġrevital": 32951, "Ġpunitive": 32952, "Ġshortcuts": 32953, "Ġinstituted": 32954, "ĠWarsaw": 32955, "Ġabdomen": 32956, "ĠKING": 32957, "Ġsuperintendent": 32958, "Ġfry": 32959, "ĠGeo": 32960, "TOR": 32961, "Ġcontradictions": 32962, "aptic": 32963, "Ġlandscapes": 32964, "bugs": 32965, "Ġclust": 32966, "Ġvolley": 32967, "cribed": 32968, "Ġtandem": 32969, "Ġrobes": 32970, "WHAT": 32971, "Ġpromoter": 32972, "Ġeloqu": 32973, "reviewed": 32974, "ĠDK": 32975, "ĠPlato": 32976, "Ġfps": 32977, "Tank": 32978, "ĠDerrick": 32979, "Ġprioritize": 32980, "asper": 32981, "ĠHonduras": 32982, "ĠCompleted": 32983, "nec": 32984, "Ġmog": 32985, "nir": 32986, "ĠMayo": 32987, "DEF": 32988, "stall": 32989, "inness": 32990, "ĠVolkswagen": 32991, "Ġprecaution": 32992, "ĠMell": 32993, "iak": 32994, "istries": 32995, "Ġ248": 32996, "Ġoverlapping": 32997, "Senate": 32998, "ĠEnhance": 32999, "resy": 33000, "racial": 33001, "ORTS": 33002, "ĠMormons": 33003, "Strong": 33004, "ĠCoch": 33005, "Mexico": 33006, "ĠMaduro": 33007, "Ġjars": 33008, "Ġcane": 33009, "Wik": 33010, "olla": 33011, "ifference": 33012, "Ġphysicist": 33013, "ĠMaggie": 33014, "Ġ285": 33015, "Ġdepiction": 33016, "ĠMcLaren": 33017, "Ju": 33018, "Ġslows": 33019, "Ġcommissioners": 33020, "ĠWillow": 33021, "ĠExplos": 33022, "hovah": 33023, "Ġtechnician": 33024, "Ġhomicides": 33025, "ĠFlav": 33026, "ĠTruman": 33027, "Ġ10000": 33028, "uctor": 33029, "Ġshader": 33030, "Newsletter": 33031, "457": 33032, "Ġrever": 33033, "Ġhardened": 33034, "Ġwhereabouts": 33035, "Ġredevelop": 33036, "Ġcarbs": 33037, "Ġtravers": 33038, "Ġsquirrel": 33039, "Ġfollower": 33040, "Ġsings": 33041, "508": 33042, "Ġrabbits": 33043, "emonium": 33044, "Ġdocumenting": 33045, "Ġmisunderstood": 33046, ")'": 33047, "Rick": 33048, "ggies": 33049, "Ġpremie": 33050, "Ġskating": 33051, "Ġpassports": 33052, "Ġfists": 33053, "ageddon": 33054, "Haw": 33055, "ACP": 33056, "080": 33057, "ĠThoughts": 33058, "ĠCarlson": 33059, "Ġpriesthood": 33060, "hua": 33061, "Ġdungeons": 33062, "ĠLoans": 33063, "Ġantis": 33064, "Ġfamiliarity": 33065, "ĠSabb": 33066, "opal": 33067, "ĠInk": 33068, "strike": 33069, "Ġcram": 33070, "Ġlegalized": 33071, "Ġcuisine": 33072, "Ġfibre": 33073, "Travel": 33074, "ĠMonument": 33075, "ODY": 33076, "ethy": 33077, "Ġinterstate": 33078, "ĠPUR": 33079, "emporary": 33080, "ĠArabian": 33081, "developed": 33082, "Ġsaddle": 33083, "Ġgithub": 33084, "ĠOffer": 33085, "ĠISP": 33086, "rolet": 33087, "ĠSUPER": 33088, "ĠDenis": 33089, "Ġmultiplier": 33090, "Ġstirred": 33091, "Interestingly": 33092, "Ġcustomary": 33093, "Ġbilled": 33094, "hex": 33095, "Ġmultiplied": 33096, "Ġflipping": 33097, "ĠCrosby": 33098, "Ġfundamentals": 33099, "iae": 33100, "ĠPlayed": 33101, "ĠAtom": 33102, "amazon": 33103, "ĠFlam": 33104, "eez": 33105, "activated": 33106, "Ġtablespoon": 33107, "Ġliberalism": 33108, "ĠPalin": 33109, "ĠPatel": 33110, "Num": 33111, "ĠTAM": 33112, "Ġsurn": 33113, "ĠReloaded": 33114, "Ġcoined": 33115, "\"],": 33116, "ĠClash": 33117, "ĠAgu": 33118, "Ġpragmatic": 33119, "ĠActivate": 33120, "Ġ802": 33121, "Ġtrailers": 33122, "Ġsilhou": 33123, "Ġprobes": 33124, "Ġcircus": 33125, "ĠBain": 33126, "ĠLindsay": 33127, "ĠAbbey": 33128, "Delivery": 33129, "Ġconcession": 33130, "Ġgastro": 33131, "ĠSprite": 33132, "ÄŁ": 33133, "andel": 33134, "Ġgimm": 33135, "Ġautobi": 33136, "ĠTurtle": 33137, "Ġwonderfully": 33138, "ĠHaram": 33139, "ĠWorldwide": 33140, "ĠHandle": 33141, "Ġtheorists": 33142, "Ġsleek": 33143, "ĠZhu": 33144, "ographically": 33145, "EGA": 33146, "ĠOwners": 33147, "aths": 33148, "ĠAntarctic": 33149, "natal": 33150, "=\"\"": 33151, "flags": 33152, "````": 33153, "Ġsul": 33154, "Kh": 33155, "Ġpotassium": 33156, "Ġlineman": 33157, "Ġcereal": 33158, "ĠSeasons": 33159, "Ġ2022": 33160, "Ġmathematic": 33161, "Ġastronomers": 33162, "professional": 33163, "Ġfares": 33164, "cknowled": 33165, "Ġchi": 33166, "Ġyoungsters": 33167, "Ġmistakenly": 33168, "Ġhemisphere": 33169, "ĠDivinity": 33170, "rone": 33171, "Ġ\",": 33172, "rings": 33173, "Ġattracts": 33174, "vana": 33175, "å¹": 33176, "CAP": 33177, "Ġplaylist": 33178, "Ġporch": 33179, "ãģ£": 33180, "Ġincorporates": 33181, "Ġsoak": 33182, "Ġasserting": 33183, "ĠTerrorism": 33184, "ĠPablo": 33185, "Ja": 33186, "cester": 33187, "Ġfearing": 33188, "ĠPrayer": 33189, "Ġescalated": 33190, "GW": 33191, "Ġrobe": 33192, "ĠBrighton": 33193, "acists": 33194, "ĠSymphony": 33195, "ĠDwarf": 33196, "ĠParade": 33197, "ĠLego": 33198, "Ġinexpl": 33199, "Ġlords": 33200, "leaf": 33201, "RAG": 33202, "liber": 33203, "Ġcigars": 33204, "ĠJehovah": 33205, "606": 33206, "WINDOWS": 33207, "ĠLiberia": 33208, "ebus": 33209, "Heavy": 33210, "Ġlubric": 33211, "ĠRW": 33212, "anguages": 33213, "Ġnarrowed": 33214, "computer": 33215, "ĠEmber": 33216, "Ġmurdering": 33217, "Ġdownstream": 33218, "ĠTuls": 33219, "ĠTables": 33220, "Topic": 33221, "ĠAccuracy": 33222, "=/": 33223, "lost": 33224, "ĠRei": 33225, "Ġprogresses": 33226, "bear": 33227, "Ġestablishments": 33228, "Justin": 33229, "ĠPeach": 33230, "ĠGomez": 33231, "å¿": 33232, "ĠTriangle": 33233, "Ident": 33234, "ĠHive": 33235, "Resources": 33236, "Ġmixes": 33237, "ĠAssuming": 33238, "Mu": 33239, "Ġhypoc": 33240, "Ġsane": 33241, "ĠWan": 33242, "idious": 33243, "Success": 33244, "Ġio": 33245, "Angel": 33246, "Ġdangerously": 33247, "ĠCreature": 33248, "WORK": 33249, ":[": 33250, "ĠKatrina": 33251, "Listener": 33252, "Miller": 33253, "ĠIdlib": 33254, "hang": 33255, "Ġcircumvent": 33256, "href": 33257, "Ġcelestial": 33258, "ĠWeeks": 33259, "ĠPug": 33260, "ĠDalton": 33261, "Ġsubpoena": 33262, "uku": 33263, "Ġpersisted": 33264, "pei": 33265, "olding": 33266, "ĠDocuments": 33267, "ĠHast": 33268, "ĠCENT": 33269, "Ġprimer": 33270, "Ġsynonymous": 33271, "Ġnib": 33272, "ombs": 33273, "Ġnotation": 33274, "ĠDish": 33275, "ĠAtmosp": 33276, "Ġforbid": 33277, "ĠANG": 33278, "pattern": 33279, "los": 33280, "Ġprojectiles": 33281, "brown": 33282, ".\",": 33283, "ĠVenom": 33284, "Ġfiercely": 33285, "ublished": 33286, "ĠUran": 33287, "ĠNicarag": 33288, "410": 33289, "ĠCAL": 33290, "OTOS": 33291, "ĠMiracle": 33292, "ĠEnchant": 33293, "Ġguarding": 33294, "append": 33295, "Attach": 33296, "Ġleveled": 33297, "Ġcondoms": 33298, "ihilation": 33299, "649": 33300, "Ġnightmares": 33301, "ĠTHEY": 33302, "ĠSTART": 33303, "ĠKinn": 33304, "Ġroommate": 33305, "Ġhygiene": 33306, "opping": 33307, "Job": 33308, "Ġlvl": 33309, "ĠVER": 33310, "ĠKeeping": 33311, "abetic": 33312, "Ġformatting": 33313, "erala": 33314, "Ġrevisions": 33315, "Ġresurg": 33316, "Tel": 33317, "ĠGoodman": 33318, "353": 33319, "pod": 33320, "Ġindisp": 33321, "ĠTranslation": 33322, "Ġgown": 33323, "ĠMund": 33324, "Ġcis": 33325, "Ġbystand": 33326, "collect": 33327, "ĠPunjab": 33328, "actively": 33329, "ĠGamb": 33330, "tell": 33331, "Ġimporting": 33332, "gencies": 33333, "Ġlocom": 33334, "ĠBrill": 33335, "Holy": 33336, "ĠBerger": 33337, "Ġshowdown": 33338, "Ġresponders": 33339, "ILY": 33340, "Ġtakedown": 33341, "leted": 33342, "Ġmattered": 33343, "Ġpredictive": 33344, "Ġoverlay": 33345, "GPU": 33346, "ĠVick": 33347, "Ġconveyed": 33348, "Tab": 33349, "peer": 33350, "Scan": 33351, "Ġdefensively": 33352, "vae": 33353, "Ġapproving": 33354, "Ġtiers": 33355, "ĠVia": 33356, "querade": 33357, "ĠSaudis": 33358, "Ġdemolished": 33359, "ĠProphe": 33360, "Ġmono": 33361, "Ġhospitality": 33362, "HAM": 33363, "ĠAriel": 33364, "MOD": 33365, "ĠTorah": 33366, "Ġblah": 33367, "ĠBelarus": 33368, "erential": 33369, "ĠTuc": 33370, "Ġbanker": 33371, "397": 33372, "Ġmosquit": 33373, "ĠScientist": 33374, "ĠMusical": 33375, "Ġhust": 33376, "Shift": 33377, "Ġtorment": 33378, "Ġstandoff": 33379, "Educ": 33380, "ĠFog": 33381, "Ġamplifier": 33382, "Shape": 33383, "Instance": 33384, "ĠCritics": 33385, "Ġdaemon": 33386, "Houston": 33387, "Ġmattress": 33388, "ĠIDF": 33389, "Ġobscene": 33390, "ĠAmer": 33391, "hetti": 33392, "Ġcompiling": 33393, "352": 33394, "verett": 33395, "ĠReduction": 33396, "istration": 33397, "ĠBlessed": 33398, "ĠBachelor": 33399, "316": 33400, "Ġprank": 33401, "ĠVulcan": 33402, "dding": 33403, "Ġmourning": 33404, "ĠQuint": 33405, "ĠBlaster": 33406, "testing": 33407, "Ġsediment": 33408, ">>>": 33409, "ĠEternity": 33410, "ĠWHERE": 33411, "ĠMaze": 33412, "Ġreacting": 33413, "ĠAlv": 33414, "omsday": 33415, "ĠCRA": 33416, "Ġtranslator": 33417, "Ġbogus": 33418, "atu": 33419, "Website": 33420, "olls": 33421, "Ġbaptism": 33422, "Ġsibling": 33423, "ĠAutumn": 33424, "vez": 33425, "ãģ®é": 33426, "guards": 33427, "Georg": 33428, "assadors": 33429, "ĠFreud": 33430, "Ġcontinents": 33431, "ĠRegistry": 33432, "Bernie": 33433, "ĸļ士": 33434, "Ġtolerant": 33435, "ĠUW": 33436, "Ġhorribly": 33437, "995": 33438, "ĠMIDI": 33439, "Ġimpatient": 33440, "ocado": 33441, "eri": 33442, "ĠWorst": 33443, "ĠNorris": 33444, "ĠTalking": 33445, "Ġdefends": 33446, "ensable": 33447, "Ġ2021": 33448, "Ġanatomy": 33449, "Lew": 33450, "Ġdrawer": 33451, "ĠCanberra": 33452, "Ġpatriotic": 33453, "é¾įåĸļ士": 33454, "ĠAvg": 33455, "ARM": 33456, "Ġundisclosed": 33457, "Ġfarewell": 33458, "459": 33459, "bable": 33460, "ĠAllison": 33461, "OLOG": 33462, "Ġconco": 33463, "tight": 33464, "ĠACPI": 33465, "ĠMines": 33466, "lich": 33467, "ĠâĶľ": 33468, "represented": 33469, "200000": 33470, "Ġenthusiast": 33471, "OTS": 33472, "bil": 33473, "ĠIngredients": 33474, "Ġinventor": 33475, "ĠMySQL": 33476, "³³³": 33477, "ĠABOUT": 33478, "within": 33479, "Ġmk": 33480, "Bul": 33481, "ĠFake": 33482, "Ġdraconian": 33483, "Wa": 33484, "helm": 33485, "ĠTerran": 33486, "erville": 33487, "Ġcommonplace": 33488, "SIZE": 33489, "Ġ\"<": 33490, "replace": 33491, "ographs": 33492, "ĠSELECT": 33493, "incible": 33494, "ĠMostly": 33495, "ĠSheffield": 33496, "ĠIDE": 33497, "uggle": 33498, "Ġcitations": 33499, "hurst": 33500, "ĠUnix": 33501, "Ġunleash": 33502, "ĠPiper": 33503, "ĠNano": 33504, "Ġsuccumb": 33505, "Ġreluctance": 33506, "Ġ2500": 33507, "ĠMerchant": 33508, "Ġwiret": 33509, "Ġcombos": 33510, "ĠBirthday": 33511, "Ġcharcoal": 33512, "ĠUPS": 33513, "ĠFairfax": 33514, "Ġdriveway": 33515, "ĠTek": 33516, "ĠPitch": 33517, "overe": 33518, "Ġtechnicians": 33519, "ĠActual": 33520, "flation": 33521, "ĠFiscal": 33522, "ĠEmpty": 33523, "anamo": 33524, "Ġmagnesium": 33525, "Ġslut": 33526, "Ġgrowers": 33527, "Investigators": 33528, "():": 33529, "ĠSatellite": 33530, "ĠKeynes": 33531, "missive": 33532, "lane": 33533, "Ġborough": 33534, "344": 33535, "ĠTEAM": 33536, "ĠBethesda": 33537, "CV": 33538, "hower": 33539, "ĠRAD": 33540, "Ġchant": 33541, "ĠRiy": 33542, "Ġcompositions": 33543, "Ġmildly": 33544, "Ġmeddling": 33545, "Ġagility": 33546, "aneers": 33547, "501": 33548, "Ġsynth": 33549, "linger": 33550, "291": 33551, "Ġexclaimed": 33552, "Party": 33553, "Ġcontamin": 33554, "ĠManor": 33555, "ĠRespond": 33556, "Ġpraising": 33557, "Ġmanners": 33558, "fleet": 33559, "Summer": 33560, "ĠLynd": 33561, "ĠDefinitely": 33562, "grim": 33563, "Ġbowling": 33564, "stri": 33565, "çĽ": 33566, "ynt": 33567, "Ġmandates": 33568, "DIV": 33569, "Ġreconcile": 33570, "views": 33571, "ĠDamon": 33572, "vette": 33573, "Flo": 33574, "ĠGreatest": 33575, "ilon": 33576, "icia": 33577, "Ġportrayal": 33578, "Ġcushion": 33579, "504": 33580, "1979": 33581, "ossal": 33582, "Applic": 33583, "scription": 33584, "Ġmitigation": 33585, "ATS": 33586, "pac": 33587, "Ġerased": 33588, "Ġdeficiencies": 33589, "ĠHollande": 33590, "ĠXu": 33591, "Ġbred": 33592, "Ġpregnancies": 33593, "femin": 33594, "Ġemph": 33595, "Ġplanners": 33596, "Ġoutper": 33597, "uttering": 33598, "Ġperpetrator": 33599, "Ġmotto": 33600, "ĠEllison": 33601, "ĠNEVER": 33602, "Ġadmittedly": 33603, "ARI": 33604, "ĠAzerbaijan": 33605, "Ġmillisec": 33606, "Ġcombustion": 33607, "ĠBottle": 33608, "ĠLund": 33609, "ĠPs": 33610, "ĠDress": 33611, "Ġfabricated": 33612, "Ġbattered": 33613, "Ġsidel": 33614, "ĠNotting": 33615, "Foreign": 33616, "ĠJerome": 33617, "020": 33618, "ĠArbit": 33619, "Ġknots": 33620, "ĠRIGHT": 33621, "Moving": 33622, "ãģĻ": 33623, "Ġsurgeries": 33624, "Ġcourthouse": 33625, "Ġmastered": 33626, "Ġhovering": 33627, "ĠBran": 33628, "ĠAlison": 33629, "Ġsafest": 33630, "military": 33631, "Ġbullied": 33632, "Ġbarrage": 33633, "Reader": 33634, "ESE": 33635, "ĠGeographic": 33636, "Tools": 33637, "314": 33638, "ĠGeek": 33639, "roth": 33640, "glers": 33641, "ĠFIN": 33642, "Ïģ": 33643, "ĠAston": 33644, "altern": 33645, "488": 33646, "Ġveterin": 33647, "Gamer": 33648, "Ġintel": 33649, "renches": 33650, "Shield": 33651, "Ġamnesty": 33652, "ĠBhar": 33653, "Ġpiled": 33654, "Ġhonorable": 33655, "ĠInstitutes": 33656, "Ġsoaked": 33657, "Ġcoma": 33658, "ĠEFF": 33659, "341": 33660, "bytes": 33661, "ĠGmail": 33662, "lein": 33663, "ĠCanadiens": 33664, "material": 33665, "Il": 33666, "Ġinstructors": 33667, "ĠKY": 33668, "Ġconceive": 33669, "ubb": 33670, "ĠPossible": 33671, "Ġeasing": 33672, "ĠChristina": 33673, "Ġcaric": 33674, "ĠHDR": 33675, "ROM": 33676, "Ġshovel": 33677, "delete": 33678, "Ġpuff": 33679, "ĠChanging": 33680, "Ġseamlessly": 33681, "Attribute": 33682, "Ġacquisitions": 33683, "akery": 33684, "ĠEF": 33685, "Ġautistic": 33686, "ĠTakes": 33687, "ĠPowder": 33688, "ĠStir": 33689, "510": 33690, "ĠBubble": 33691, "settings": 33692, "ĠFowler": 33693, "Ġmustard": 33694, "Ġmoreover": 33695, "Ġcopyrighted": 33696, "ĠLEDs": 33697, "1500": 33698, "æī": 33699, "ĠHIS": 33700, "enf": 33701, "Ġcustod": 33702, "ĠHuck": 33703, "Gi": 33704, "Ġimg": 33705, "Answer": 33706, "Ct": 33707, "jay": 33708, "ĠInfrastructure": 33709, "Ġfederally": 33710, "Loc": 33711, "Ġmicrobes": 33712, "Ġoverrun": 33713, "dds": 33714, "otent": 33715, "adiator": 33716, ">>>>>>>>": 33717, "Ġtornado": 33718, "Ġadjud": 33719, "Ġintrigued": 33720, "Ġsi": 33721, "ĠRevelation": 33722, "progress": 33723, "Ġburglary": 33724, "ĠSaiyan": 33725, "ĠKathy": 33726, "Ġserpent": 33727, "ĠAndreas": 33728, "Ġcompel": 33729, "essler": 33730, "ĠPlastic": 33731, "ĠAdvent": 33732, "ĠPositive": 33733, "ĠQt": 33734, "ĠHindus": 33735, "registered": 33736, "ularity": 33737, "Ġrighteousness": 33738, "Ġdemonic": 33739, "uitive": 33740, "ĠBDS": 33741, "ĠGregg": 33742, "cia": 33743, "ĠCrusade": 33744, "ĠSinai": 33745, "WARE": 33746, "+(": 33747, "Ġmell": 33748, "Ġderail": 33749, "yards": 33750, "Ast": 33751, "Ġnoticeably": 33752, "ĠOber": 33753, "Ram": 33754, "Ġunnoticed": 33755, "Ġseq": 33756, "avage": 33757, "Ts": 33758, "Ġ640": 33759, "Ġconcede": 33760, "Ġ])": 33761, "Fill": 33762, "Ġcaptivity": 33763, "ĠImprovement": 33764, "ĠCrusader": 33765, "araoh": 33766, "MAP": 33767, "æĹ": 33768, "Ġstride": 33769, "always": 33770, "Fly": 33771, "Nit": 33772, "Ġalgae": 33773, "ĠCooking": 33774, "ĠDoors": 33775, "Malley": 33776, "Ġpolicemen": 33777, "ãģį": 33778, "Ġastronaut": 33779, "accessible": 33780, "495": 33781, "ĠRAW": 33782, "cliffe": 33783, "udicrous": 33784, "Ġdepended": 33785, "alach": 33786, "Ġventures": 33787, "rake": 33788, "Ġtits": 33789, "ĠHou": 33790, "Ġcondom": 33791, "ormonal": 33792, "Ġindent": 33793, "Ġuploading": 33794, "Footnote": 33795, "Important": 33796, "Ġ271": 33797, "Ġmindful": 33798, "Ġcontends": 33799, "Cra": 33800, "Ġcalibr": 33801, "ĠOECD": 33802, "plugin": 33803, "Fat": 33804, "ĠISS": 33805, "ĠDynamics": 33806, "ansen": 33807, "686": 33808, "'),": 33809, "Ġsprite": 33810, "Ġhandheld": 33811, "ĠHipp": 33812, "=~=~": 33813, "Trust": 33814, "Ġsemantics": 33815, "ĠBundes": 33816, "ĠReno": 33817, "ĠLiterature": 33818, "sense": 33819, "Gary": 33820, "ĠAeg": 33821, "ĠTrin": 33822, "EEK": 33823, "Ġcleric": 33824, "ĠSSH": 33825, "Ġchrist": 33826, "Ġinvading": 33827, "ibu": 33828, "Ġenum": 33829, "aura": 33830, "Ġallege": 33831, "ĠIncredible": 33832, "BBC": 33833, "Ġthru": 33834, "Ġsailed": 33835, "Ġemulate": 33836, "Ġinsecurity": 33837, "Ġcrou": 33838, "Ġaccommodations": 33839, "Ġincompetent": 33840, "Ġslips": 33841, "ĠEarthqu": 33842, "sama": 33843, "ILLE": 33844, "ĠiPhones": 33845, "asaki": 33846, "Ġbye": 33847, "Ġard": 33848, "Ġextras": 33849, "Ġslaughtered": 33850, "Ġcrowdfunding": 33851, "resso": 33852, "Ġfilib": 33853, "ĠERROR": 33854, "ĠTLS": 33855, "egg": 33856, "ĠItal": 33857, "Ġenlist": 33858, "ĠCatalonia": 33859, "ĠScots": 33860, "Ġsergeant": 33861, "Ġdissolve": 33862, "NH": 33863, "Ġstandings": 33864, "rique": 33865, "IQ": 33866, "Ġbeneficiary": 33867, "Ġaquarium": 33868, "YouTube": 33869, "ĠPowerShell": 33870, "Ġbrightest": 33871, "ĠWarrant": 33872, "Sold": 33873, "Writing": 33874, "Ġbeginnings": 33875, "ĠReserved": 33876, "ĠLatinos": 33877, "heading": 33878, "Ġ440": 33879, "Ġrooftop": 33880, "ATING": 33881, "Ġ390": 33882, "VPN": 33883, "Gs": 33884, "kernel": 33885, "turned": 33886, "Ġpreferable": 33887, "Ġturnovers": 33888, "ĠHels": 33889, "Sa": 33890, "ĠShinji": 33891, "veh": 33892, "ĠMODULE": 33893, "Viol": 33894, "Ġexiting": 33895, "Ġjab": 33896, "ĠVanilla": 33897, "Ġacron": 33898, "ĠGap": 33899, "bern": 33900, "Ak": 33901, "ĠMcGu": 33902, "Ġendlessly": 33903, "ĠFarage": 33904, "ĠNoel": 33905, "Va": 33906, "MK": 33907, "Ġbrute": 33908, "ĠKru": 33909, "ĠESV": 33910, "ĠOlivia": 33911, "âĢł": 33912, "ĠKaf": 33913, "Ġtrusting": 33914, "Ġhots": 33915, "324": 33916, "Ġmalaria": 33917, "Ġjson": 33918, "Ġpounding": 33919, "ortment": 33920, "Country": 33921, "Ġpostponed": 33922, "Ġunequiv": 33923, "?),": 33924, "ĠRooney": 33925, "udding": 33926, "ĠLeap": 33927, "urrence": 33928, "shapeshifter": 33929, "ĠHAS": 33930, "osate": 33931, "Ġcavern": 33932, "Ġconservatism": 33933, "ĠBAD": 33934, "Ġmileage": 33935, "Ġarresting": 33936, "Vaults": 33937, "Ġmixer": 33938, "Democratic": 33939, "ĠBenson": 33940, "Ġauthored": 33941, "8000": 33942, "Ġproactive": 33943, "ĠSpiritual": 33944, "tre": 33945, "Ġincarcerated": 33946, "ĠSort": 33947, "Ġpeaked": 33948, "Ġwielding": 33949, "reciation": 33950, "×Ļ×": 33951, "Patch": 33952, "ĠEmmy": 33953, "Ġexqu": 33954, "tto": 33955, "ĠRatio": 33956, "ĠPicks": 33957, "ĠGry": 33958, "phant": 33959, "Ġfret": 33960, "Ġethn": 33961, "Ġarchived": 33962, "%-": 33963, "cases": 33964, "ĠBlaze": 33965, "Ġimb": 33966, "cv": 33967, "yss": 33968, "imony": 33969, "Ġcountdown": 33970, "Ġawakening": 33971, "ĠTunisia": 33972, "ĠRefer": 33973, "ĠMJ": 33974, "Ġunnatural": 33975, "ĠCarnegie": 33976, "izen": 33977, "ĠNuggets": 33978, "hess": 33979, "Ġevils": 33980, "647": 33981, "Ġintroductory": 33982, "loving": 33983, "ĠMcMahon": 33984, "Ġambiguity": 33985, "Label": 33986, "ĠAlmighty": 33987, "Ġcoloring": 33988, "ĠClaus": 33989, "setting": 33990, "NULL": 33991, "ĠFavorite": 33992, "ĠSIG": 33993, ">(": 33994, "ĠShiva": 33995, "ĠMayer": 33996, "Ġstormed": 33997, "ĠCoverage": 33998, "weapons": 33999, "igham": 34000, "Ġunanswered": 34001, "Ġleve": 34002, "Ġcoy": 34003, "cas": 34004, "bags": 34005, "asured": 34006, "Seattle": 34007, "ĠSantorum": 34008, "serious": 34009, "Ġcourageous": 34010, "ĠSoup": 34011, "Ġconfiscated": 34012, "Ġ///": 34013, "Ġunconventional": 34014, "Ġmoms": 34015, "ĠRohingya": 34016, "ĠOrchestra": 34017, "ĠPotion": 34018, "Ġdiscredit": 34019, "ĠFIL": 34020, "fixed": 34021, "ĠDeer": 34022, "doi": 34023, "ĠDimension": 34024, "Ġbureaucrats": 34025, "eteen": 34026, "ĠactionGroup": 34027, "ohm": 34028, "Ġbumps": 34029, "ĠUtility": 34030, "Ġsubmarines": 34031, "renheit": 34032, "research": 34033, "ĠShapiro": 34034, "Ġsketches": 34035, "Ġdeceptive": 34036, "ĠVil": 34037, "esame": 34038, "ĠEssentially": 34039, "Ġrampage": 34040, "isky": 34041, "Ġmuttered": 34042, "thritis": 34043, "Ġ236": 34044, "fet": 34045, "bars": 34046, "Ġpupil": 34047, "ĠThou": 34048, "oS": 34049, "song": 34050, "Ġfractured": 34051, "Ġrevert": 34052, "picture": 34053, "Ġcriterion": 34054, "usher": 34055, "Ġrepercussions": 34056, "ĠVintage": 34057, "ĠSuperintendent": 34058, "Officers": 34059, "Ġflagged": 34060, "Ġblames": 34061, "Ġinverse": 34062, "ographers": 34063, "Ġmakeshift": 34064, "Ġdevoid": 34065, "Ġfossils": 34066, "ĠAristotle": 34067, "ĠFunds": 34068, "Ġdepleted": 34069, "ĠFlu": 34070, "ĠYuan": 34071, "Ġwoes": 34072, "Ġlipid": 34073, "Ġsitu": 34074, "requisites": 34075, "Ġfurnish": 34076, "ĠSamar": 34077, "Ġshameful": 34078, "Ġadversely": 34079, "Ġadept": 34080, "Ġremorse": 34081, "Ġmurderous": 34082, "uckles": 34083, "ĠESL": 34084, "Ġ314": 34085, "sent": 34086, "Ġredef": 34087, "ĠCache": 34088, "ĠPurs": 34089, "igans": 34090, "Ġ460": 34091, "Ġprescriptions": 34092, "Ġfres": 34093, "Fuck": 34094, "ocrates": 34095, "Twenty": 34096, "ĠWeird": 34097, "ĠToggle": 34098, "ĠCalled": 34099, "itizens": 34100, "Ġpoultry": 34101, "Ġharvesting": 34102, "ãĤ¦ãĤ¹": 34103, "Bottom": 34104, "Ġcautioned": 34105, "tn": 34106, "396": 34107, "ĠNikki": 34108, "Ġevaluations": 34109, "Ġharassing": 34110, "Ġbindings": 34111, "ĠMonetary": 34112, "Ġhitters": 34113, "Ġadversary": 34114, "unts": 34115, "Ġsetback": 34116, "Ġencrypt": 34117, "ĠCait": 34118, "Ġlows": 34119, "enges": 34120, "ĠNorn": 34121, "Ġbulbs": 34122, "Ġbottled": 34123, "ĠVoyager": 34124, "317": 34125, "Ġspheres": 34126, "politics": 34127, "Ġsubtract": 34128, "Ġsensations": 34129, "Ġappalling": 34130, "Ġ316": 34131, "Ġenvironmentally": 34132, "ĠSTEM": 34133, "Ġpublishes": 34134, "560": 34135, "Ġdiligence": 34136, "484": 34137, "Ġadvises": 34138, "Ġpetrol": 34139, "Ġimagining": 34140, "Ġpatrols": 34141, "ĠInteger": 34142, "ĠAshes": 34143, "actus": 34144, "ĠRadiant": 34145, "ĠLT": 34146, "itability": 34147, "htaking": 34148, "Setting": 34149, "Ġnuanced": 34150, "ĠReef": 34151, "ĠDevelopers": 34152, "Ni": 34153, "pieces": 34154, "990": 34155, "License": 34156, "Ġlowers": 34157, "ĠOttoman": 34158, "327": 34159, "ooo": 34160, "Ġquitting": 34161, "markets": 34162, "Behind": 34163, "Ġbasin": 34164, "Ġdocs": 34165, "anie": 34166, "flash": 34167, "ctl": 34168, "Ġcivilized": 34169, "ĠFukushima": 34170, "\"],\"": 34171, "ĠKS": 34172, "ĠHonestly": 34173, "arat": 34174, "Ġconstructs": 34175, "ĠLans": 34176, "ĠDire": 34177, "ĠLIKE": 34178, "ĠTrouble": 34179, "Ġwithholding": 34180, "ĠOblivion": 34181, "Ġsanity": 34182, "anya": 34183, "Const": 34184, "Ġgrocer": 34185, "ĠCelsius": 34186, "Ġrecounted": 34187, "ĠWife": 34188, "Border": 34189, "atered": 34190, "happy": 34191, "Ġspoiler": 34192, "Ġlogically": 34193, "Hall": 34194, "Ġsucceeding": 34195, "Ġpolymorph": 34196, "Ġaxes": 34197, "ĠShotgun": 34198, "ĠSlim": 34199, "ĠPrinciples": 34200, "ĠLeth": 34201, "arta": 34202, "Ġscor": 34203, "Screenshot": 34204, "Ġrelaxation": 34205, "#$#$": 34206, "Ġdeterrent": 34207, "iddy": 34208, "Ġpowerless": 34209, "Ġlesbians": 34210, "Ġchords": 34211, "ĠEdited": 34212, "selected": 34213, "Ġseparatists": 34214, "0002": 34215, "Ġairspace": 34216, "Ġturnaround": 34217, "Ġcunning": 34218, "PATH": 34219, "Poly": 34220, "Ġbombed": 34221, "Ġtion": 34222, "xs": 34223, "Ġwithhold": 34224, "Ġwaged": 34225, "ĠLiberties": 34226, "Flag": 34227, "Ġcomforting": 34228, "454": 34229, "ĠIris": 34230, "arers": 34231, "Ġrag": 34232, "Ġrelocated": 34233, "ĠGuarant": 34234, "Ġstrategically": 34235, "Ġgamma": 34236, "uberty": 34237, "ĠLockheed": 34238, "gres": 34239, "Ġgrilled": 34240, "ĠLowe": 34241, "stats": 34242, "ĠRocks": 34243, "Ġsensing": 34244, "Ġrenting": 34245, "ĠGeological": 34246, "اØ": 34247, "otrop": 34248, "Ġsew": 34249, "Ġimproperly": 34250, "486": 34251, "Ġâĸł": 34252, "Ġstarving": 34253, "ĠBj": 34254, "Discussion": 34255, "328": 34256, "ĠCombo": 34257, "ĠFixes": 34258, "NAT": 34259, "Ġstriving": 34260, "thora": 34261, "Ġharvested": 34262, "ĠPing": 34263, "Ġplayful": 34264, "Ġavenues": 34265, "Ġoccupational": 34266, "Ġwakes": 34267, "ĠCourier": 34268, "Ġdrummer": 34269, "ĠBrowser": 34270, "ĠHouth": 34271, "itu": 34272, "Ġapparel": 34273, "paste": 34274, "Ġhunted": 34275, "ĠSecondly": 34276, "lain": 34277, "XY": 34278, "ĠPIN": 34279, "icons": 34280, "Ġcocktails": 34281, "Ġsizable": 34282, "Ġhurdles": 34283, "estinal": 34284, "ĠRecreation": 34285, "Ġeco": 34286, "648": 34287, "ĠDied": 34288, "mint": 34289, "Ġfingerprints": 34290, "Ġdispose": 34291, "ĠBosnia": 34292, "tsy": 34293, "2200": 34294, "Ġinspected": 34295, "ĠFou": 34296, "Ġfuss": 34297, "Ġambush": 34298, "ĠRak": 34299, "Ġmanifested": 34300, "Prosecut": 34301, "Ġsuffice": 34302, "rences": 34303, "Ġcompensated": 34304, "ĠCyrus": 34305, "Ġgenus": 34306, "ĠWolverine": 34307, "ĠTrends": 34308, "Ġhikes": 34309, "ĠSeen": 34310, "Ġenrol": 34311, "Cold": 34312, "Ġpolitely": 34313, "ĠSlav": 34314, "ĠRupert": 34315, "Ġeyewitness": 34316, "ĠAlto": 34317, "Ġuncomp": 34318, "Ġposterior": 34319, "Must": 34320, "ĠHerz": 34321, "Ġprogressively": 34322, "Ġ234": 34323, "Ġindifference": 34324, "ĠCunningham": 34325, "Ġacademia": 34326, "Ġsewer": 34327, "Ġastounding": 34328, "ĠAES": 34329, "rather": 34330, "Ġeldest": 34331, "Ġclimbs": 34332, "ĠAdds": 34333, "Ġoutcry": 34334, "Ġcontag": 34335, "ĠHouses": 34336, "Ġpept": 34337, "ĠMelania": 34338, "interested": 34339, "ĠUCH": 34340, "ĠRoots": 34341, "ĠHubbard": 34342, "ĠTBD": 34343, "ĠRomanian": 34344, "filename": 34345, "Stone": 34346, "ĠImpl": 34347, "Ġchromosome": 34348, "Cle": 34349, "dx": 34350, "Ġscrambled": 34351, "ĠPt": 34352, "Ġ242": 34353, "OPLE": 34354, "Ġtremendously": 34355, "Street": 34356, "Ġcraving": 34357, "Ġbundled": 34358, "ĠRG": 34359, "pipe": 34360, "Ġinjuring": 34361, "Ġarcane": 34362, "Particip": 34363, "ĠHeroic": 34364, "sty": 34365, "Ġtopping": 34366, "ĠTempest": 34367, "rentices": 34368, "bh": 34369, "Ġparanoia": 34370, "ĠUnicode": 34371, "Ġegregious": 34372, "Ġ\\'": 34373, "ĠOswald": 34374, "Ġgravel": 34375, "ĠSimpsons": 34376, "Ġbland": 34377, "ĠGuantanamo": 34378, "Writer": 34379, "liners": 34380, "ĠDice": 34381, "JC": 34382, "Ġparity": 34383, "Ġsided": 34384, "Ġ237": 34385, "ĠPyrrha": 34386, "atters": 34387, "dk": 34388, "Fine": 34389, "compan": 34390, "Ġformulated": 34391, "ĠIdol": 34392, "ilers": 34393, "hemoth": 34394, "ĠFav": 34395, "Ġintrusion": 34396, "Ġcarrots": 34397, "ĠLayer": 34398, "ĠHacker": 34399, "Ġ----------------": 34400, "Ġmoderation": 34401, "éģ": 34402, "ococ": 34403, "Ġcharacterize": 34404, "ĠTeresa": 34405, "Ġsocioeconomic": 34406, "Ġperk": 34407, "ĠParticipation": 34408, "training": 34409, "ĠPaulo": 34410, "phys": 34411, "Ġtrustworthy": 34412, "Ġembodied": 34413, "ĠMerch": 34414, "currency": 34415, "ĠPriority": 34416, "Ġteasing": 34417, "Ġabsorbing": 34418, "Ġunfinished": 34419, "ĠComparison": 34420, "Ġdisple": 34421, "writers": 34422, "Ġprofessions": 34423, "ĠPenguin": 34424, "Ġangrily": 34425, "ĠLINK": 34426, "688": 34427, "ĠCorrespond": 34428, "Ġprevailed": 34429, "Ġcartel": 34430, "lp": 34431, "asms": 34432, "ĠRedemption": 34433, "ĠIslamists": 34434, "effects": 34435, "dose": 34436, "ĠLatter": 34437, "ĠHalifax": 34438, "Ġvas": 34439, "ĠTopics": 34440, "ĠNamed": 34441, "advertising": 34442, "zza": 34443, "ICES": 34444, "Ġretarded": 34445, "achable": 34446, "ĠPuppet": 34447, "ĠItemLevel": 34448, "Ġretract": 34449, "Ġidentifiable": 34450, "Aaron": 34451, "ĠBuster": 34452, "sol": 34453, "helle": 34454, "assemb": 34455, "Hope": 34456, "ranged": 34457, "Ba": 34458, "ĠPurch": 34459, "éĢ": 34460, "ĠSiri": 34461, "Ġarrivals": 34462, "Ġ1912": 34463, "Ġshortened": 34464, "Ġ312": 34465, "Ġdiscrepancy": 34466, "ĠTemperature": 34467, "ĠWalton": 34468, "Ġkinderg": 34469, "polit": 34470, "Ġremix": 34471, "Ġconnectors": 34472, "ãĥĺãĥ©": 34473, "ĠKazakhstan": 34474, "dominated": 34475, "Ġsugars": 34476, "imble": 34477, "ĠPanic": 34478, "ĠDemand": 34479, "ĠColony": 34480, "onen": 34481, "ĠMER": 34482, "775": 34483, "uria": 34484, "azaar": 34485, "ĠDegree": 34486, "Pri": 34487, "Ġsunshine": 34488, "Ġ251": 34489, "Ġpsychedelic": 34490, "Ġdigitally": 34491, "ĠBraun": 34492, "Ġshimmer": 34493, "Ġshave": 34494, "ĠTelesc": 34495, "ĠAstral": 34496, "ĠVenezuelan": 34497, "ĠOG": 34498, "Ġcrawling": 34499, "Integ": 34500, "ĠFeather": 34501, "Ġunfolding": 34502, "Ġappropriation": 34503, "Ġè£ıè": 34504, "ĠMobility": 34505, "ĠNey": 34506, "-.": 34507, "bilt": 34508, "LIN": 34509, "ĠTube": 34510, "ĠConversely": 34511, "Ġkeyboards": 34512, "ĠCao": 34513, "Ġoverth": 34514, "Ġlaure": 34515, ">>\\": 34516, "ĠViper": 34517, "acha": 34518, "Offset": 34519, "ĠRaleigh": 34520, "ĠJae": 34521, "Jordan": 34522, "jp": 34523, "Ġtotalitarian": 34524, "Connector": 34525, "Ġobserves": 34526, "ĠSpartan": 34527, "ĠImmediately": 34528, "ĠScal": 34529, "Cool": 34530, "Ġtaps": 34531, "Ġroar": 34532, "Past": 34533, "Ġchars": 34534, "ĠBender": 34535, "ĠSheldon": 34536, "Ġpainter": 34537, "Ġbeacon": 34538, "ĠCreatures": 34539, "Ġdownturn": 34540, "Ġhinder": 34541, "ĠAndromeda": 34542, "ÃĽ": 34543, "ccoli": 34544, "ĠFitness": 34545, "etrical": 34546, "Ġutilizes": 34547, "Ġsenate": 34548, "Ġensemble": 34549, "Ġcheers": 34550, "TW": 34551, "Ġaffluent": 34552, "kil": 34553, "rylic": 34554, "ordering": 34555, "Computer": 34556, "Ġgruesome": 34557, "ostics": 34558, "ĠUbisoft": 34559, "ĠKelley": 34560, "Ġwrench": 34561, "Ġbourgeoisie": 34562, "IBLE": 34563, "ĠPreston": 34564, "worn": 34565, "arist": 34566, "reating": 34567, "Ġstained": 34568, "arine": 34569, "Ġslime": 34570, "ENN": 34571, "Ġchests": 34572, "Ġgroundwater": 34573, "annot": 34574, "ĠTray": 34575, "ĠLocke": 34576, "ĠCTR": 34577, "Ġdudes": 34578, "ĠExternal": 34579, "ĠDecoder": 34580, "Ġparamed": 34581, "ĠMedline": 34582, "809": 34583, "ĠDinner": 34584, "rupal": 34585, "gz": 34586, "ĠGum": 34587, "ĠDemo": 34588, "jee": 34589, "Ġdh": 34590, "berman": 34591, "archs": 34592, "Ġenqu": 34593, "ĠEpstein": 34594, "Ġdevastation": 34595, "Ġfriendships": 34596, "ĠArd": 34597, "Ġ231": 34598, "ĠRubin": 34599, "ĠDistance": 34600, "Ġspurred": 34601, "Ġdossier": 34602, "Ġoverlooking": 34603, "\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\": 34604, "Forest": 34605, "ĠComes": 34606, "\\\",": 34607, "ĠIranians": 34608, "Ġfixtures": 34609, "Laughs": 34610, "Ġcurry": 34611, "ĠKingston": 34612, "Ġsquash": 34613, "Ġcatalogue": 34614, "Ġabnormalities": 34615, "Ġdigestive": 34616, ".........": 34617, "Ġsubordinate": 34618, "ogly": 34619, "Ġ249": 34620, "Middle": 34621, "Ġmassac": 34622, "Ġburgers": 34623, "Ġdownstairs": 34624, "Ġ1931": 34625, "394": 34626, "ĠVG": 34627, "Ġlasers": 34628, "ĠSikh": 34629, "ĠAlexa": 34630, "derived": 34631, "Ġcyclist": 34632, "ãģ®éŃĶ": 34633, "oneliness": 34634, "!!!!!!!!": 34635, "Ġbuffs": 34636, "legate": 34637, "Ġraping": 34638, "Ġrecommending": 34639, "rored": 34640, "Ġmulticultural": 34641, "unique": 34642, "Ġbusinessmen": 34643, "Ġuneasy": 34644, "ĠMAP": 34645, "Ġdispersed": 34646, "cipline": 34647, "Jess": 34648, "ĠKerala": 34649, "å§": 34650, "Ġabstraction": 34651, "Surv": 34652, "Uh": 34653, "Ġprinters": 34654, "ija": 34655, "owder": 34656, "Ġanalogous": 34657, "ĠASP": 34658, "afer": 34659, "Ġunfolded": 34660, "Ġleveling": 34661, "Ġbreached": 34662, "ĠHearing": 34663, "Ġnat": 34664, "Ġtranslating": 34665, "critical": 34666, "Ġantagonist": 34667, "ĠYesterday": 34668, "Ġfuzzy": 34669, "wash": 34670, "mere": 34671, "Ġbewild": 34672, "ĠMae": 34673, "Virgin": 34674, "phrase": 34675, "Ġsignaled": 34676, "ĠHIGH": 34677, "Ġprotester": 34678, "Ġgarner": 34679, "unknown": 34680, "Ġkay": 34681, "Ġabducted": 34682, "Ġstalking": 34683, "amn": 34684, "Ġdeserving": 34685, "ĠRiv": 34686, "ĠJorge": 34687, "Ġscratching": 34688, "ĠSaving": 34689, "iping": 34690, "Ġtease": 34691, "Ġmissionary": 34692, "ĠMorrow": 34693, "TIME": 34694, "Present": 34695, "Ġchemotherapy": 34696, "terness": 34697, "ĠHomes": 34698, "ĠPurdue": 34699, "Ġstaunch": 34700, "ĠWhitney": 34701, "ĠTHERE": 34702, "μ": 34703, "iatus": 34704, "ĠErnest": 34705, "ĠDeploy": 34706, "Ġcoveted": 34707, "FML": 34708, "ĠDialogue": 34709, "Ġexited": 34710, "fruit": 34711, "Ġnerd": 34712, "\":\"\",\"": 34713, "Ġvivo": 34714, "ruly": 34715, "460": 34716, "ĠAmen": 34717, "rehensible": 34718, "Ġâĺ": 34719, "DIR": 34720, "Ġadherence": 34721, "Ġchew": 34722, "ĠCoke": 34723, "ĠSergei": 34724, "digital": 34725, "ĠNeck": 34726, "gently": 34727, "enthal": 34728, "/)": 34729, "Ġweary": 34730, "Ġguise": 34731, "ĠConcord": 34732, "ĠOnion": 34733, "atcher": 34734, "Ġbinge": 34735, "ĠDirective": 34736, "Ġmanned": 34737, "ansk": 34738, "Ġillusions": 34739, "Ġbillionaires": 34740, "383": 34741, "olyn": 34742, "odynamic": 34743, "ĠWheat": 34744, "ĠAlic": 34745, "Ġcoloured": 34746, "ĠNAFTA": 34747, "abo": 34748, "Ġmacros": 34749, "independent": 34750, "sweet": 34751, "Ġspac": 34752, "ĠKabul": 34753, "ĠÄ": 34754, "eme": 34755, "Ġdictated": 34756, "Ġshouts": 34757, "={": 34758, "Ġripping": 34759, "ĠShay": 34760, "ĠCricket": 34761, "directed": 34762, "Ġanalysed": 34763, "ĠWARRANT": 34764, "agons": 34765, "ĠBlazers": 34766, "Ġcheered": 34767, "Ġarithmetic": 34768, "ĠTanz": 34769, "373": 34770, "ĠFlags": 34771, "Ġ295": 34772, "Ġwitches": 34773, "ĠIncluded": 34774, "ĠGained": 34775, "ĠBlades": 34776, "Gam": 34777, "ĠSamantha": 34778, "ĠAtlantis": 34779, "ĠPratt": 34780, "Ġspoiled": 34781, "ĠIB": 34782, "ĠRamirez": 34783, "Probably": 34784, "rero": 34785, "ĠNg": 34786, "ĠWarlock": 34787, "tp": 34788, "Ġoverhe": 34789, "Ġadministrations": 34790, "Ġtint": 34791, "Ġregiment": 34792, "Ġpistols": 34793, "Ġblankets": 34794, "Ġepist": 34795, "Ġbowls": 34796, "Ġhydraulic": 34797, "Ġdean": 34798, "Ġjung": 34799, "Ġascend": 34800, "705": 34801, "ĠSantiago": 34802, "î": 34803, "Ġunavoid": 34804, "ĠShaman": 34805, "reb": 34806, "Ġstemming": 34807, "998": 34808, "ĠMG": 34809, "sticks": 34810, "esthesia": 34811, "ERO": 34812, "Ġmorbid": 34813, "ĠGrill": 34814, "ĠPoe": 34815, "anyl": 34816, "Ġdeleting": 34817, "ĠSurveillance": 34818, "Ġdirectives": 34819, "Ġiterations": 34820, "ĠRox": 34821, "ĠMilky": 34822, "Father": 34823, "Ġpatented": 34824, "447": 34825, "Ġprecursor": 34826, "Ġmaiden": 34827, "ĠPhen": 34828, "ĠVegan": 34829, "ĠPatent": 34830, "Kelly": 34831, "Redditor": 34832, "Ġnods": 34833, "Ġventilation": 34834, "ĠSchwarz": 34835, "Ġwizards": 34836, "Ġominous": 34837, "ĠHeads": 34838, "ĠBG": 34839, "Ġlumber": 34840, "ĠSpiel": 34841, "ĠisEnabled": 34842, "Ġancestral": 34843, "ĠShips": 34844, "Ġwrestler": 34845, "phi": 34846, "Ġyuan": 34847, "ĠRebellion": 34848, "Ġiceberg": 34849, "Ġmagically": 34850, "Ġdiversion": 34851, "arro": 34852, "ythm": 34853, "ĠRiders": 34854, "ĠRobbie": 34855, "ĠKara": 34856, "ĠMaintenance": 34857, "ĠHerb": 34858, "Ġharms": 34859, "packed": 34860, "ĠFeinstein": 34861, "Ġmarrying": 34862, "Ġblending": 34863, "ĠRates": 34864, "Ġ1880": 34865, "Ġwrink": 34866, "ĠUnch": 34867, "ĠTorch": 34868, "described": 34869, "Ġhumanoid": 34870, "ilitating": 34871, "ĠConv": 34872, "ĠFeld": 34873, "IGHTS": 34874, "Ġwhistleblower": 34875, "ortmund": 34876, "etsy": 34877, "arrett": 34878, "ĠMono": 34879, "ĠIke": 34880, "ĠCNBC": 34881, "ĠWAY": 34882, "ĠMDMA": 34883, "ĠIndividuals": 34884, "Ġsupplemental": 34885, "Ġpowerhouse": 34886, "ĠStru": 34887, "Focus": 34888, "aphael": 34889, "ĠColleg": 34890, "atti": 34891, "ZA": 34892, "Ġperenn": 34893, "ĠSignature": 34894, "ĠRodney": 34895, "Ġcubes": 34896, "iddled": 34897, "ĠDante": 34898, "ĠINV": 34899, "ilingual": 34900, "ĠCth": 34901, "Ġsofa": 34902, "Ġintimidate": 34903, "ĠRoe": 34904, "ĠDiplom": 34905, "ĠCountries": 34906, "ayson": 34907, "Ġextradition": 34908, "Ġdisabling": 34909, "ĠCardiff": 34910, "Ġmemorandum": 34911, "ĠTrace": 34912, "Ġ???": 34913, "sector": 34914, "ĠRouhani": 34915, "ĠYates": 34916, "ĠFreeze": 34917, "Ġbladder": 34918, "Motor": 34919, "ĠPromise": 34920, "antasy": 34921, "Ġforeseeable": 34922, "ĠCologne": 34923, "container": 34924, "ĠTrees": 34925, "ĠGors": 34926, "ĠSinclair": 34927, "Ġbarring": 34928, "keye": 34929, "Ġslashed": 34930, "ĠStatistical": 34931, "éĩ": 34932, "Ġâĸº": 34933, "Allows": 34934, "Ġhumility": 34935, "Ġdrilled": 34936, "ĠFurn": 34937, "443": 34938, "Ġsewage": 34939, "Ġhomepage": 34940, "Ġcourtyard": 34941, "Ġvile": 34942, "Ġsubsidiaries": 34943, "ajo": 34944, "directory": 34945, "Ġammon": 34946, "Vers": 34947, "charges": 34948, "Ġ}}": 34949, "ĠChains": 34950, "Ġ246": 34951, "nob": 34952, "Ġpercept": 34953, "Ġgrit": 34954, "Ġfishermen": 34955, "ĠIraqis": 34956, "ĠDISTR": 34957, "ĠFULL": 34958, "ĠEvaluation": 34959, "graph": 34960, "atial": 34961, "Ġcooperating": 34962, "Ġmelan": 34963, "Ġenlightened": 34964, "Ġali": 34965, "tailed": 34966, "Ġsalute": 34967, "Ġweakest": 34968, "ĠBulldogs": 34969, "UA": 34970, "ĠAlloy": 34971, "Ġsemen": 34972, "ocene": 34973, "ĠWilliamson": 34974, "spr": 34975, ",âĢĶ": 34976, "ĠGF": 34977, "ittens": 34978, "Beat": 34979, "ĠJunk": 34980, "iphate": 34981, "ĠFarmers": 34982, "ĠBitcoins": 34983, "igers": 34984, "dh": 34985, "ĠLoyal": 34986, "payer": 34987, "Ġentertained": 34988, "Ġpenned": 34989, "Ġcoupon": 34990, "Queue": 34991, "Ġweakening": 34992, "carry": 34993, "Ġunderestimate": 34994, "Ġshootout": 34995, "Ġcharismatic": 34996, "ĠProcedure": 34997, "Ġprudent": 34998, "inances": 34999, "Ġriches": 35000, "Ġcortical": 35001, "Ġstrides": 35002, "Ġdrib": 35003, "ĠOilers": 35004, "540": 35005, "ĠPerform": 35006, "ĠBangkok": 35007, "Ġeuth": 35008, "SER": 35009, "Ġsimplistic": 35010, "tops": 35011, "campaign": 35012, "Quality": 35013, "Ġimpoverished": 35014, "ĠEisenhower": 35015, "Ġaugment": 35016, "ĠHarden": 35017, "Ġintervened": 35018, "Ġlistens": 35019, "ĠKok": 35020, "Ġsage": 35021, "Ġrubbish": 35022, "ĠDed": 35023, "Ġmull": 35024, "pelling": 35025, "Ġvideot": 35026, "Production": 35027, "DJ": 35028, "miah": 35029, "Ġadaptations": 35030, "Ġmedically": 35031, "Ġboarded": 35032, "Ġarrogance": 35033, "Ġscrapped": 35034, "Ġoppress": 35035, "FORMATION": 35036, "Ġjunction": 35037, "415": 35038, "EEEE": 35039, "Skill": 35040, "Ġsubdu": 35041, "ĠSuggest": 35042, "ĠPett": 35043, "Ġlett": 35044, "ĠManip": 35045, "ĠCaf": 35046, "ĠCooperation": 35047, "Ther": 35048, "Ġregained": 35049, "¶æ": 35050, "reflect": 35051, "Ġthugs": 35052, "ĠShelby": 35053, "Ġdictates": 35054, "ĠWeiner": 35055, "ĠHale": 35056, "Ġbattleground": 35057, "schild": 35058, "Ġcondol": 35059, "hunt": 35060, "ositories": 35061, "Ġaccuses": 35062, "Filename": 35063, "Ġshri": 35064, "Ġmotivate": 35065, "Ġreflections": 35066, "Null": 35067, "ĠLobby": 35068, "¥µ": 35069, "ĠSATA": 35070, "ĠBackup": 35071, "Ñĥ": 35072, "nin": 35073, "ĠCorrection": 35074, "Ġjuicy": 35075, "utra": 35076, "ĠPric": 35077, "Ġrestraining": 35078, "ĠAirbnb": 35079, "ĠArrest": 35080, "Ġappropriations": 35081, "Ġslopes": 35082, "Ġmanslaughter": 35083, "Ġworkings": 35084, "ĠHuss": 35085, "ĠFrey": 35086, "Leave": 35087, "ĠHarmony": 35088, "ĠFeder": 35089, "Ġ430": 35090, "Ġtrench": 35091, "Ġgladly": 35092, "Ġbullpen": 35093, "ĠGau": 35094, "bones": 35095, "Ġgroove": 35096, "Ġpretext": 35097, "ãħĭ": 35098, "Ġtransmitter": 35099, "ĠComponent": 35100, "Ġunderage": 35101, "ĠEmpires": 35102, "Tile": 35103, "Ġoy": 35104, "ĠMarvin": 35105, "ĠCAS": 35106, "Ġbloss": 35107, "Ġreplicated": 35108, "ĠMariners": 35109, "Marcus": 35110, "ĠBlocks": 35111, "Ġliberated": 35112, "Ġbutterfly": 35113, "Feel": 35114, "Ġfermentation": 35115, "Ġyoutube": 35116, "Ġoffend": 35117, "ĠTerm": 35118, "resist": 35119, "Ġcessation": 35120, "Ġinsurgency": 35121, "Ġbir": 35122, "ĠRaise": 35123, "595": 35124, "Ġhypotheses": 35125, "502": 35126, "Ġplaque": 35127, "ocrat": 35128, "Ġjackets": 35129, "ĠHuffPost": 35130, "among": 35131, "Ġconfer": 35132, "487": 35133, "ĠLilly": 35134, "Ġadapting": 35135, "ĠFay": 35136, "Ġshoved": 35137, "vec": 35138, "Ġrefine": 35139, "Ġgon": 35140, "Ġgunmen": 35141, "zai": 35142, "ĠShuttle": 35143, "ĠIzan": 35144, "Ġ1913": 35145, "Ġplethora": 35146, "··": 35147, "Ġ510": 35148, "Ġpuberty": 35149, "Ġ241": 35150, "ĠWealth": 35151, "ĠAlma": 35152, "ĠMEM": 35153, "ĠAdults": 35154, "Cas": 35155, "prison": 35156, "Race": 35157, "Ġwaterproof": 35158, "Ġathleticism": 35159, "Ġcapitalize": 35160, "ĠJuice": 35161, "Ġilluminated": 35162, "ĠPascal": 35163, "Ġirritation": 35164, "ĠWitnesses": 35165, "adle": 35166, "ĠAstro": 35167, "Ġfax": 35168, "ĠElvis": 35169, "Primary": 35170, "ĠLich": 35171, "ĠElves": 35172, "Ġresiding": 35173, "Ġstumble": 35174, "319": 35175, "ĠPKK": 35176, "Ġadversaries": 35177, "DOS": 35178, "ĠRitual": 35179, "Ġsmear": 35180, "Ġarson": 35181, "idental": 35182, "Ġscant": 35183, "Ġmonarchy": 35184, "Ġhalftime": 35185, "Ġresidue": 35186, "Ġindign": 35187, "ĠShaun": 35188, "ĠElm": 35189, "auri": 35190, "Aff": 35191, "WATCH": 35192, "ĠLyon": 35193, "helps": 35194, "361": 35195, "Ġlobbyist": 35196, "Ġdiminishing": 35197, "Ġoutbreaks": 35198, "Ġgoats": 35199, "favorite": 35200, "ĠNah": 35201, "sonian": 35202, "ĠBooster": 35203, "Ġsandbox": 35204, "ĠFare": 35205, "ĠMalta": 35206, "ĠattRot": 35207, "ĠMOR": 35208, "lde": 35209, "Ġnavigating": 35210, "Touch": 35211, "Ġuntrue": 35212, "ĠDisaster": 35213, "Ġludicrous": 35214, "Password": 35215, "ĠJFK": 35216, "blogspot": 35217, "416": 35218, "ĠUNDER": 35219, "ernal": 35220, "Ġdelaying": 35221, "TOP": 35222, "Ġimplants": 35223, "ĠAVG": 35224, "ĠHuge": 35225, "attr": 35226, "Ġjournalistic": 35227, "ĠPeyton": 35228, "ĠIA": 35229, "Rap": 35230, "goal": 35231, "ĠProgramme": 35232, "Ġsmashing": 35233, "wives": 35234, "println": 35235, "ĠPlague": 35236, "inus": 35237, "EEP": 35238, "Ġcruiser": 35239, "ĠParish": 35240, "uminium": 35241, "Ġoccupants": 35242, "ĠJihad": 35243, "mop": 35244, "Ġpint": 35245, "Ġhect": 35246, "ĠMecca": 35247, "director": 35248, "ĠFunding": 35249, "ĠMixed": 35250, "Ġstag": 35251, "Tier": 35252, "Ġgust": 35253, "Ġbrightly": 35254, "orsi": 35255, "Ġuphill": 35256, "RD": 35257, "Ġlesions": 35258, "ĠBundy": 35259, "livious": 35260, "Ġbiologist": 35261, "ĠFaculty": 35262, "ĠAuthorization": 35263, "Ġ244": 35264, "Allow": 35265, "ï¸": 35266, "ĠGiul": 35267, "Ġpertinent": 35268, "otaur": 35269, "esse": 35270, "ĠRoof": 35271, "Ġunmanned": 35272, "351": 35273, "ĠShak": 35274, "ĠOrient": 35275, "Ġendanger": 35276, "Dir": 35277, "Ġreplen": 35278, "edient": 35279, "Ġtailor": 35280, "Ġgadgets": 35281, "Ġaudible": 35282, "âĺĨ": 35283, "Nice": 35284, "Ġbombard": 35285, "ĠRape": 35286, "Ġdefiance": 35287, "ĠTWO": 35288, "ĠFilipino": 35289, "Ġunaffected": 35290, "ervatives": 35291, "Ġsoared": 35292, "ĠBolton": 35293, "Ġcompromising": 35294, "ĠBrewers": 35295, "RAL": 35296, "ĠAHL": 35297, "icycle": 35298, "Ġvampires": 35299, "Ġdipped": 35300, "oyer": 35301, "ĠXIII": 35302, "Ġsideways": 35303, "ĠWaste": 35304, "ĠDiss": 35305, "ĠâĶľâĶĢâĶĢ": 35306, "$.": 35307, "Ġhabitats": 35308, "ĠBeef": 35309, "truth": 35310, "trained": 35311, "split": 35312, "Rus": 35313, "Andy": 35314, "ĠBram": 35315, "REP": 35316, "pid": 35317, "è£ħ": 35318, "ĠMutant": 35319, "Anim": 35320, "ĠMarina": 35321, "Ġfutile": 35322, "highest": 35323, "frequency": 35324, "Ġepilepsy": 35325, "Ġcoping": 35326, "Ġconcise": 35327, "Ġtracing": 35328, "ĠSUN": 35329, "panel": 35330, "ĠSophie": 35331, "ĠCrowley": 35332, "ĠAdolf": 35333, "ĠShooter": 35334, "Ġshaky": 35335, "ĠIG": 35336, "ĠLies": 35337, "ĠBarber": 35338, "pkg": 35339, "Ġuptake": 35340, "Ġpredatory": 35341, "ULTS": 35342, "/**": 35343, "Ġintoxicated": 35344, "ĠWestbrook": 35345, "odder": 35346, "hement": 35347, "Ġbaseman": 35348, "APD": 35349, "storage": 35350, "ĠFifty": 35351, "editor": 35352, "GEN": 35353, "UTION": 35354, "irting": 35355, "Ġsewing": 35356, "rift": 35357, "Ġagony": 35358, "ĠSands": 35359, "Ġ254": 35360, "Cash": 35361, "Ġlodge": 35362, "Ġpunt": 35363, "Natural": 35364, "ĠIdeas": 35365, "Ġerroneous": 35366, "ĠSensor": 35367, "ĠHannity": 35368, "Ġ1921": 35369, "Ġmould": 35370, "ĠGon": 35371, "kaya": 35372, "Ġanonymously": 35373, "ĠKEY": 35374, "Ġsimulator": 35375, "Winter": 35376, "Ġstreamed": 35377, "507": 35378, "?\",": 35379, "Ġteased": 35380, "Ġcoefficient": 35381, "Ġwartime": 35382, "ĠTHR": 35383, "''.": 35384, "ĠBanking": 35385, "mpire": 35386, "Ġfandom": 35387, "Ġlia": 35388, "Ga": 35389, "Ġdownhill": 35390, "Ġinterpreting": 35391, "Individual": 35392, "Norm": 35393, "Ġjealousy": 35394, "bitcoin": 35395, "Ġpleasures": 35396, "ĠToys": 35397, "ĠChevrolet": 35398, "ĠAdvisor": 35399, "IZE": 35400, "Ġreceptions": 35401, "706": 35402, "Cro": 35403, "Ġ262": 35404, "Ġcitrus": 35405, "iru": 35406, "Reviewer": 35407, "jected": 35408, "UES": 35409, "anz": 35410, "1981": 35411, "ĠWorker": 35412, "Ġcomplied": 35413, "orescent": 35414, "continental": 35415, "Ton": 35416, "ĠPrism": 35417, "ĠSheep": 35418, "Ġ288": 35419, "nox": 35420, "ĠVog": 35421, "Ord": 35422, "Ġrealms": 35423, "tek": 35424, "Ġirrigation": 35425, "Ġbicycles": 35426, "Ġelectronically": 35427, "poly": 35428, "tall": 35429, "());": 35430, "Ġaesthetics": 35431, "ĠIntegrated": 35432, "Explore": 35433, "Ġdunk": 35434, "476": 35435, "pain": 35436, "ĠJacques": 35437, "ĠDmit": 35438, "Frames": 35439, "Ġreunited": 35440, "Ġhumid": 35441, "Dro": 35442, "Political": 35443, "Ġyouthful": 35444, "Ġentails": 35445, "Ġmosquito": 35446, "363": 35447, "species": 35448, "Ġcoordinating": 35449, "ĠMayhem": 35450, "ĠMagnus": 35451, "Mount": 35452, "Improved": 35453, "ĠSTATE": 35454, "ATTLE": 35455, "Ġflowed": 35456, "Ġtackled": 35457, "Ġfashioned": 35458, "Ġreorgan": 35459, "ivari": 35460, "finger": 35461, "Ġreluctantly": 35462, "etting": 35463, "ĠVand": 35464, "young": 35465, "ĠGarland": 35466, "Ġpresumption": 35467, "Ġamenities": 35468, "ĠPleasant": 35469, "onential": 35470, "ĠOxy": 35471, "Ġmorals": 35472, "ĠYah": 35473, "Ready": 35474, "Simon": 35475, "Enh": 35476, "Demon": 35477, "Ġclich": 35478, "Monitor": 35479, "ĠDU": 35480, "Ġwelcomes": 35481, "Ġstandout": 35482, "Ġdreadful": 35483, "Ġbananas": 35484, "Ġballoons": 35485, "hooting": 35486, "basic": 35487, "Ġsuffix": 35488, "Ġduly": 35489, "cano": 35490, "Chain": 35491, "atos": 35492, "Ġgeopolitical": 35493, "Ġ(&": 35494, "ĠGemini": 35495, "ÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤ": 35496, "Ġacquitted": 35497, "Luck": 35498, "protect": 35499, "1024": 35500, "Ġscarcity": 35501, "Ġmindfulness": 35502, "ecided": 35503, "DN": 35504, "prime": 35505, "ĠPresidents": 35506, "ĠVIDEO": 35507, "Ġ(âĪĴ": 35508, "addock": 35509, "NOR": 35510, "ĠPru": 35511, "pun": 35512, "ĠLOL": 35513, "))))": 35514, "ĠLiqu": 35515, "ĠSAS": 35516, "Ġstyling": 35517, "Ġpunishments": 35518, "Ġnumb": 35519, "Ġascertain": 35520, "ĠRockies": 35521, "flu": 35522, "Thumbnail": 35523, "Ġperpetrated": 35524, "ĠSemi": 35525, "Ġdisarm": 35526, "ĠOlder": 35527, "ĠException": 35528, "Ġexponentially": 35529, "ĠCommunities": 35530, "Ġabolish": 35531, "ĠPartner": 35532, "ptoms": 35533, "Ġ777": 35534, "ĠFoley": 35535, "ĠCases": 35536, "Ġgrease": 35537, "ĠRebirth": 35538, "Ground": 35539, "Ġ;)": 35540, "ĠDoctrine": 35541, "ikini": 35542, "Ye": 35543, "ĠBlossom": 35544, "Ġpersists": 35545, "bill": 35546, "Ġinfusion": 35547, "Ġbuddies": 35548, "911": 35549, "ĠPatient": 35550, "Ġdemos": 35551, "Ġacquaintance": 35552, "ĠPaw": 35553, "atari": 35554, "Ġxml": 35555, "Ġfascination": 35556, "ĠServe": 35557, "ÏĤ": 35558, "branded": 35559, "Ġaz": 35560, "Returns": 35561, "Ġovershadow": 35562, "Ġroam": 35563, "Ġspeedy": 35564, "numbered": 35565, "helial": 35566, "Ġdisciple": 35567, "Ġassurances": 35568, "given": 35569, "pecting": 35570, "ĠNatalie": 35571, "çĶ°": 35572, "Ġmosquitoes": 35573, "rotein": 35574, "Ġnumeric": 35575, "Ġindependents": 35576, "Ġtransitional": 35577, "Ġreactionary": 35578, "ĠMechdragon": 35579, "doctor": 35580, "Ġshortest": 35581, "Ġsequential": 35582, "ĠBac": 35583, "ĠAccounts": 35584, "ãģĮ": 35585, "achy": 35586, "ractive": 35587, "ĠRegiment": 35588, "Ġbreathtaking": 35589, "fficiency": 35590, "ĠBates": 35591, "Ġ311": 35592, "Ġwardrobe": 35593, "fts": 35594, "ĠBerk": 35595, "Simply": 35596, "ĠRiverside": 35597, "ivering": 35598, "idential": 35599, "lucent": 35600, "Ġenriched": 35601, "ĠConver": 35602, "ĠGiving": 35603, "ãĥĻ": 35604, "Ġlegalize": 35605, "ĠFTC": 35606, "Ġfreaking": 35607, "Mix": 35608, "Ġterrestrial": 35609, "esian": 35610, "cients": 35611, "Wing": 35612, "LOAD": 35613, "Ġledge": 35614, "ĠViolent": 35615, "ĠMetall": 35616, "Ġ308": 35617, "Ġsoutheastern": 35618, "hetto": 35619, "Meat": 35620, "Ġslowdown": 35621, "Ġretreated": 35622, "Jeremy": 35623, "endas": 35624, "*****": 35625, "eric": 35626, "Ġreins": 35627, "oppable": 35628, "ĠHumanity": 35629, "earances": 35630, "rigan": 35631, "Camera": 35632, "Ġwaivers": 35633, "soc": 35634, "Ġalteration": 35635, "transform": 35636, "ĠCemetery": 35637, "506": 35638, "Ġindefinite": 35639, "Ġstimulating": 35640, "yg": 35641, "603": 35642, "ĠSop": 35643, "Ġdescriptive": 35644, "Phase": 35645, "ĠEdmund": 35646, "Ġpneumonia": 35647, "ventus": 35648, "Amb": 35649, "Ġlaboratories": 35650, "ĠExclusive": 35651, "ugar": 35652, "Were": 35653, "Ġmalfunction": 35654, "Ġhomosexuals": 35655, "Ġ-------": 35656, "uni": 35657, "Ġturbines": 35658, "ĠEquity": 35659, "Du": 35660, "Ġminded": 35661, "ĠRH": 35662, "ĠBlackhawks": 35663, "Ġfeats": 35664, "Ġ1700": 35665, "repl": 35666, "362": 35667, "laden": 35668, "Ġindispensable": 35669, "lyss": 35670, "tti": 35671, "Ġreel": 35672, "Ġdiverted": 35673, "Ġlikeness": 35674, "Ġsubscriptions": 35675, "Ġfingert": 35676, "Ġfilthy": 35677, "destruct": 35678, "draft": 35679, "ĠBernardino": 35680, "launch": 35681, "Ġperplex": 35682, "ĠSUM": 35683, "carb": 35684, "Ġsweater": 35685, "ĠVenture": 35686, "ĠJag": 35687, "ĠCeleb": 35688, "ĠVoters": 35689, "Ġsteadfast": 35690, "Ġathletics": 35691, "ĠHanson": 35692, "ĠDrac": 35693, "Tracker": 35694, "Ġcommend": 35695, "ĠPresidency": 35696, "ĠDID": 35697, "informed": 35698, "Ġwebpage": 35699, "Pretty": 35700, "Ġforcefully": 35701, "ãĥĥãĤ¯": 35702, "Ġrelocation": 35703, "Ġsatire": 35704, "âī": 35705, "ĠSunderland": 35706, "æĦ": 35707, "Voice": 35708, "????????": 35709, "Ġinformant": 35710, "Ġbowel": 35711, "ĠUniform": 35712, "Ġ...\"": 35713, "Ġpurge": 35714, "Ġpicnic": 35715, "ĠUmb": 35716, "ĠUPDATE": 35717, "ĠSapphire": 35718, "ĠStall": 35719, "learn": 35720, "Ġobjectively": 35721, "Ġobliter": 35722, "Ġloophole": 35723, "Ġjourneys": 35724, "Ġomission": 35725, "Pros": 35726, "ĠSidney": 35727, "ploma": 35728, "Ġsprayed": 35729, "Ġguru": 35730, "Ġtraitor": 35731, "Ġtimet": 35732, "Ġsnapping": 35733, "ĠSevent": 35734, "urnal": 35735, "ĠUkip": 35736, "Ġbowed": 35737, "poral": 35738, "liberal": 35739, "Ros": 35740, "Questions": 35741, "iOS": 35742, "Ġsummarize": 35743, "STAT": 35744, "Ġ1850": 35745, "apest": 35746, "Ġlender": 35747, "ĠVariable": 35748, "bringing": 35749, "ĠLORD": 35750, ",)": 35751, "Ġcollapses": 35752, "xiety": 35753, "ĠNed": 35754, "YD": 35755, "ĠScha": 35756, "Ġantibody": 35757, "Ġdisband": 35758, "yre": 35759, "illusion": 35760, "Ġrover": 35761, "shed": 35762, "ĠHirosh": 35763, "cci": 35764, "Ġcalam": 35765, "ĠMorton": 35766, "Pinterest": 35767, "Ġ1928": 35768, "ĠEuras": 35769, "ordes": 35770, "Ġfences": 35771, "ĠInventory": 35772, "ĠValencia": 35773, "ĠUd": 35774, "ĠTiff": 35775, "Ġsque": 35776, "Ġquotation": 35777, "Ġtroublesome": 35778, "erker": 35779, "QUEST": 35780, "ĠKingdoms": 35781, "south": 35782, "Ġlevy": 35783, "Prince": 35784, "ĠSting": 35785, "Ġnicknamed": 35786, "Ġappe": 35787, "Ġphotographic": 35788, "Ġcorpus": 35789, "reference": 35790, "ĠTrog": 35791, "Unt": 35792, ")=(": 35793, "ĠLatvia": 35794, "Ġactivating": 35795, "Ġlicensee": 35796, "Ġdisparities": 35797, "ĠNewsletter": 35798, "ãĥĥãĥĪ": 35799, "Ġfreeing": 35800, "ĠJeep": 35801, "ĠPerception": 35802, "insk": 35803, "Ġsilicone": 35804, "ĠHayden": 35805, "Lean": 35806, "ĠSuzuki": 35807, "ibrarian": 35808, "668": 35809, "Ġspor": 35810, "Ġcorrelations": 35811, "aghetti": 35812, "Ġtuber": 35813, "ĠIPCC": 35814, "ilus": 35815, "ĠVu": 35816, "Ġwealthiest": 35817, "ĠCarbuncle": 35818, "anza": 35819, "Ġfooled": 35820, "ĠZur": 35821, "Ġdaddy": 35822, "rano": 35823, "ilian": 35824, "Ġknockout": 35825, "fman": 35826, "required": 35827, "ĠWikileaks": 35828, "ĠDuffy": 35829, "ONT": 35830, "Ġinsol": 35831, "ĠObjects": 35832, "Ġbou": 35833, "ĠNordic": 35834, "ĠInsert": 35835, "scan": 35836, "Ġdancers": 35837, "Ġidiots": 35838, "majority": 35839, "ĠNeville": 35840, "ĠFreeBSD": 35841, "Ġtart": 35842, "panic": 35843, "690": 35844, "Ġcocoa": 35845, "Ġsampled": 35846, "Ġlookup": 35847, "Indust": 35848, "Ġinjections": 35849, "genre": 35850, "Ġau": 35851, "Ġroadway": 35852, "Ġgenitals": 35853, "Kind": 35854, "ĠExaminer": 35855, "ĠYaz": 35856, "Fresh": 35857, "Ġparalysis": 35858, "ĠAluminum": 35859, "Ġreap": 35860, "oké": 35861, "Ġsloppy": 35862, "ĠTunnel": 35863, "posium": 35864, "nery": 35865, "enic": 35866, "Ġherbal": 35867, "ĠOuter": 35868, "ĠBuilder": 35869, "Ġincur": 35870, "Ġideologies": 35871, "Ġbackups": 35872, "consuming": 35873, "ĠDetect": 35874, "deck": 35875, "ĠKNOW": 35876, "ĠGret": 35877, "ĠMIC": 35878, "Ġtoughness": 35879, "ĠExhibit": 35880, "Ġhive": 35881, "Les": 35882, "ĠSCHOOL": 35883, "ĠAtari": 35884, "alde": 35885, "ĠNull": 35886, "andestine": 35887, "mouse": 35888, "Ġbrigade": 35889, "489": 35890, "Ġrevol": 35891, "ĠLawson": 35892, "ĠWah": 35893, "opoly": 35894, "ebted": 35895, "ĠSaunders": 35896, "Ġ313": 35897, "ĠWinc": 35898, "Ġtaboo": 35899, "ĠHelmet": 35900, "Ġwedge": 35901, "chip": 35902, "ĠTina": 35903, "bg": 35904, "Ġinfuri": 35905, "rn": 35906, "Ġanomalies": 35907, "ĠSync": 35908, "ĠExam": 35909, "ĠCommit": 35910, "ĠDiary": 35911, "ĠALSO": 35912, "ĠDebor": 35913, "omedical": 35914, "Ġcomprehension": 35915, "655": 35916, "Ġempowering": 35917, "Ġire": 35918, "Ġjuices": 35919, "ĠETH": 35920, "ĠBoxing": 35921, "=\"/": 35922, "Ġfacilitated": 35923, "poke": 35924, "ĠParsons": 35925, "ĠModer": 35926, "travel": 35927, "Ġcivilizations": 35928, "Ġlibertarians": 35929, "Ġrune": 35930, "ĠClarks": 35931, "athed": 35932, "Ġcampaigners": 35933, "ĠDispatch": 35934, "ĠFahrenheit": 35935, "ĠCapcom": 35936, "----------": 35937, "Ġlace": 35938, "Ġdraining": 35939, "Ġliner": 35940, "ĠArtificial": 35941, "én": 35942, "task": 35943, "]).": 35944, "ĠGMO": 35945, "ĠOperator": 35946, "ordinary": 35947, "ĠInfluence": 35948, "ĠUps": 35949, "Ġpotency": 35950, "ussen": 35951, "ospons": 35952, "ĠSwim": 35953, "ĠDeadline": 35954, "Unity": 35955, "Ġculinary": 35956, "Ġenlightenment": 35957, "Ġwearer": 35958, "Ġmined": 35959, "Ġply": 35960, "Ġincest": 35961, "ĠDVDs": 35962, "Walk": 35963, "BTC": 35964, "Trade": 35965, "Ġdeval": 35966, "iband": 35967, "ĠOversight": 35968, "Palestinian": 35969, "Ġdart": 35970, "Ġmul": 35971, "LR": 35972, "Ġremovable": 35973, "ĠRealms": 35974, "ìĿ": 35975, "Ġmiscar": 35976, "ĠVulkan": 35977, "685": 35978, "ère": 35979, "ĠSap": 35980, "Ġmerging": 35981, "ĠCarly": 35982, "chester": 35983, "Ġbrisk": 35984, "Ġluxurious": 35985, "ĠGenerator": 35986, "Ġbitterness": 35987, "Ġedible": 35988, "Ġ243": 35989, "TG": 35990, "Ġrectangle": 35991, "WithNo": 35992, "below": 35993, "Jenn": 35994, "Ġdarkest": 35995, "Ġhitch": 35996, "Ġdosage": 35997, "Ġscaven": 35998, "ĠKeller": 35999, "ĠIllustrated": 36000, "Certainly": 36001, "ĠMavericks": 36002, "Marginal": 36003, "Ġdiarrhea": 36004, "Ġenormously": 36005, "Ġ999": 36006, "shr": 36007, "quart": 36008, "Ġadamant": 36009, "ĠMew": 36010, "Ġrenovation": 36011, "Ġcervical": 36012, "ĠPercentage": 36013, "eners": 36014, "ĠKimber": 36015, "Ġfloats": 36016, "Ġdex": 36017, "ĠWitcher": 36018, "ĠSwansea": 36019, "dm": 36020, "Ġsalty": 36021, "yellow": 36022, "Ġcape": 36023, "ĠDrain": 36024, "ĠPaula": 36025, "ĠToledo": 36026, "lesi": 36027, "Magazine": 36028, "ĠWick": 36029, "ĠMn": 36030, "ĠAck": 36031, "ĠRiding": 36032, "ASON": 36033, "Ġhomophobic": 36034, "ARP": 36035, "Ġwandered": 36036, "CPU": 36037, "oodoo": 36038, "ĠPipe": 36039, "Ġtightening": 36040, "ĠButt": 36041, "318": 36042, "Ġdeserted": 36043, "Session": 36044, "Ġfacilitating": 36045, "Jump": 36046, "Ġemergencies": 36047, "OWER": 36048, "Ġexhaustive": 36049, "ĠAFTER": 36050, "Ġheartbeat": 36051, "ĠLabel": 36052, "acky": 36053, "ĠCertified": 36054, "iltration": 36055, "Ze": 36056, "ĠUtt": 36057, "Ġ1300": 36058, "Ġpresume": 36059, "ĠDisp": 36060, "Ġsurged": 36061, "Ġdolls": 36062, "Columb": 36063, "Ġchimpan": 36064, "ĠRazor": 36065, "Ġticks": 36066, "Ġcouncillor": 36067, "Ġpilgrimage": 36068, "ĠRebels": 36069, "ĠQC": 36070, "ĠAuction": 36071, "xia": 36072, "ikk": 36073, "bred": 36074, "Ġinsertion": 36075, "Ġcoarse": 36076, "dB": 36077, "SEE": 36078, "ĠZap": 36079, "ĠFoo": 36080, "Ġcontempor": 36081, "ĠQuarterly": 36082, "otions": 36083, "ĠAlchemist": 36084, "ĠTrey": 36085, "ĠDuo": 36086, "Sweet": 36087, "804": 36088, "ĠGiov": 36089, "Ġfunn": 36090, "Nin": 36091, "hoff": 36092, "Ġramifications": 36093, "Ġ1922": 36094, "ĠExperts": 36095, "azes": 36096, "Ġgarments": 36097, "arial": 36098, "ĠNab": 36099, "Ġ257": 36100, "ĠVed": 36101, "Ġhumorous": 36102, "ĠPompe": 36103, "Ġnylon": 36104, "Ġlurking": 36105, "ĠSergey": 36106, "ĠMattis": 36107, "Ġmisogyny": 36108, "ĠComponents": 36109, "ĠWatching": 36110, "ĠFolk": 36111, "ractical": 36112, "Bush": 36113, "Ġtaped": 36114, "Ġgrouping": 36115, "Ġbeads": 36116, "Ġ2048": 36117, "Ġcondu": 36118, "querque": 36119, "Reading": 36120, "Ġgrievances": 36121, "Ultra": 36122, "Ġendpoint": 36123, "Hig": 36124, "ĠStatic": 36125, "ĠScarborough": 36126, "Lua": 36127, "ĠMessi": 36128, "aqu": 36129, "ĠPsyNet": 36130, "ĠRudd": 36131, "Ġavenue": 36132, "vp": 36133, "Jer": 36134, "Ġshady": 36135, "ĠResist": 36136, "ĠArtemis": 36137, "Ġcareless": 36138, "Ġbrokers": 36139, "Ġtemperament": 36140, "Ġ520": 36141, "Tags": 36142, "ĠTurning": 36143, "Ġuttered": 36144, "Ġpedd": 36145, "Ġimprovised": 36146, "Ġ:(": 36147, "Ġtabl": 36148, "Ġplains": 36149, "1600": 36150, "pressure": 36151, "ĠEssence": 36152, "margin": 36153, "friends": 36154, "ĠRestoration": 36155, "Ġpollut": 36156, "ĠPoker": 36157, "ĠAugustine": 36158, "ĠCIS": 36159, "ĠSEAL": 36160, "orama": 36161, "Ġthwart": 36162, "seek": 36163, "Ġpagan": 36164, "º": 36165, "cpu": 36166, "Ġgarn": 36167, "Ġassortment": 36168, "ĠILCS": 36169, "tower": 36170, "Recommended": 36171, "Ġunborn": 36172, "ĠRandomRedditor": 36173, "ĠRandomRedditorWithNo": 36174, "Ġparalyzed": 36175, "Ġeruption": 36176, "Ġintersect": 36177, "ĠStoke": 36178, "ĠSco": 36179, "Bind": 36180, "å¾": 36181, "ĠPNG": 36182, "ĠNegative": 36183, "ĠNOAA": 36184, "Leon": 36185, "Ġalloy": 36186, "ĠLama": 36187, "ĠDiversity": 36188, "575": 36189, "Ġunderestimated": 36190, "ĠScor": 36191, "Ġmural": 36192, "Ġbusted": 36193, "soon": 36194, "lif": 36195, "Ġnonex": 36196, "Ġallergy": 36197, "ĠUnderworld": 36198, "ĠRays": 36199, "ĠBlasio": 36200, "Ġhrs": 36201, "ĠDir": 36202, "Ġ327": 36203, "byter": 36204, "Ġreplacements": 36205, "Ġactivates": 36206, "rived": 36207, "MH": 36208, "Ġpans": 36209, "ĠHI": 36210, "Ġlongitudinal": 36211, "Ġnuisance": 36212, "aler": 36213, "Ġswell": 36214, "ĠSigned": 36215, "sci": 36216, "ĠIsles": 36217, "ĠAGA": 36218, "Ġdefiant": 36219, "Ġsonic": 36220, "ocon": 36221, "KC": 36222, "ĠAim": 36223, "tie": 36224, "ahah": 36225, "ĠmL": 36226, "DX": 36227, "Ġbisc": 36228, "ĠBillboard": 36229, "ĠSYSTEM": 36230, "NEY": 36231, "gaard": 36232, "Ġdistressed": 36233, "formerly": 36234, "Alan": 36235, "Ġchefs": 36236, "Ġoptics": 36237, "ĠComet": 36238, "ĠAMC": 36239, "Ġredesigned": 36240, "irmation": 36241, "Ġsightings": 36242, "382": 36243, "311": 36244, "ĠWB": 36245, "Ġcontraction": 36246, "ĠTOTAL": 36247, "Dual": 36248, "Ġstartled": 36249, "Ġunderstandably": 36250, "Ġsunglasses": 36251, "ETHOD": 36252, "Ġdocker": 36253, "Ġsurfing": 36254, "ĠHEL": 36255, "ĠSlack": 36256, "tones": 36257, "Ġshalt": 36258, "Visual": 36259, "498": 36260, "Department": 36261, "cussion": 36262, "Ġunrestricted": 36263, "Ġtad": 36264, "Ġrename": 36265, "employed": 36266, "Ġeducating": 36267, "Ġgrinned": 36268, "bedroom": 36269, "ĠActivities": 36270, "ĠVelvet": 36271, "ĠSWAT": 36272, "Ġshuffle": 36273, "igor": 36274, "Ġsaturation": 36275, "Finding": 36276, "cream": 36277, "icter": 36278, "Ġvodka": 36279, "tracking": 36280, "tec": 36281, "Ġforeground": 36282, "iesta": 36283, "Ġvehement": 36284, "ĠECB": 36285, "ĠTie": 36286, "Ey": 36287, "Ġturtles": 36288, "ĠRailroad": 36289, "ĠKatz": 36290, "ĠFrames": 36291, "Ġmenace": 36292, "ĠFellowship": 36293, "ĠEssential": 36294, "uggish": 36295, "Ġdrip": 36296, "chwitz": 36297, "ĠKyoto": 36298, "sb": 36299, "ĠNina": 36300, "Parameter": 36301, "Ġalarms": 36302, "ĠClaud": 36303, "Ġpioneering": 36304, "Ġchiefly": 36305, "ĠScream": 36306, "Collection": 36307, "Ġthankfully": 36308, "ĠRonaldo": 36309, "åŃIJ": 36310, "strip": 36311, "ĠDisneyland": 36312, "commercial": 36313, "Seeing": 36314, "Soul": 36315, "Ġevacuate": 36316, "Ġciv": 36317, "ĠAshe": 36318, "Ġdivides": 36319, "ĠDagger": 36320, "rehensive": 36321, "Ġberries": 36322, "ĠDF": 36323, "Ġsushi": 36324, "Ġplurality": 36325, "WI": 36326, "Ġdisadvantaged": 36327, "Ġbattalion": 36328, "obiles": 36329, "451": 36330, "Ġcling": 36331, "Ġundeniable": 36332, "ĠLounge": 36333, "Ġhaunt": 36334, "phe": 36335, "Ġquantify": 36336, "Ġdiffered": 36337, "Ġ[*]": 36338, "ĠViz": 36339, "cum": 36340, "slave": 36341, "Ġvideog": 36342, "Ġquar": 36343, "Ġbundles": 36344, "ĠAlonso": 36345, "tackle": 36346, "Ġneuronal": 36347, "Ġlandslide": 36348, "confirmed": 36349, "ĠDepth": 36350, "Ġrenewables": 36351, "Bear": 36352, "ĠMacedonia": 36353, "Ġjerseys": 36354, "Ġbunk": 36355, "ĠSpawn": 36356, "ĠControls": 36357, "ĠBuchanan": 36358, "Ġrobotics": 36359, "Ġemphasizing": 36360, "ĠTutorial": 36361, "hyp": 36362, "iston": 36363, "Ġmonumental": 36364, "æ°": 36365, "ĠCarry": 36366, "Ġtbsp": 36367, "enance": 36368, "Hill": 36369, "arthed": 36370, "Ġrotten": 36371, "Dean": 36372, "Ġtwisting": 36373, "Ġgoodwill": 36374, "Ġimmersion": 36375, "Living": 36376, "Ġbrushes": 36377, "ĠCGI": 36378, "ĠAtk": 36379, "traditional": 36380, "Ġphantom": 36381, "ĠStamina": 36382, "Ġexpansions": 36383, "ĠMarin": 36384, "Ġembarked": 36385, "ĠEg": 36386, "intestinal": 36387, "ĠPEOPLE": 36388, "ĠBooth": 36389, "ĠAppalach": 36390, "Ġrelegated": 36391, "VT": 36392, "MIT": 36393, "Ġmuster": 36394, "Ġwithdrawing": 36395, "Ġmicroscope": 36396, "ĠGathering": 36397, "ĠCrescent": 36398, "ĠArgentine": 36399, "ĠDecre": 36400, "ĠDominic": 36401, "Ġbuds": 36402, "antage": 36403, "ĠIon": 36404, "Ġwidened": 36405, "ONSORED": 36406, "ĠGloves": 36407, "iannopoulos": 36408, "razen": 36409, "feel": 36410, "Ġrepayment": 36411, "Ġhindsight": 36412, "ĠREALLY": 36413, "ĠPistol": 36414, "ĠBrah": 36415, "Ġwatts": 36416, "Ġsurvives": 36417, "Ġflurry": 36418, "issy": 36419, "Alert": 36420, "ĠUruguay": 36421, "Phoenix": 36422, "Slow": 36423, "ĠGrave": 36424, "ĠFir": 36425, "Ġmanageable": 36426, "Ġtariff": 36427, "ĠUDP": 36428, "ĠPistons": 36429, "ĠNigerian": 36430, "Ġstrikeouts": 36431, "Ġcosmetics": 36432, "whelming": 36433, "fab": 36434, "cape": 36435, "proxy": 36436, "Ġrethink": 36437, "Ġovercoming": 36438, "simple": 36439, "Ġwoo": 36440, "Ġdistracting": 36441, "ĠStanton": 36442, "ĠTulsa": 36443, "ĠDock": 36444, "659": 36445, "Ġdiscord": 36446, "ĠEmacs": 36447, "ĠVes": 36448, "ĠROB": 36449, "Ġreassuring": 36450, "Ġconsortium": 36451, "Muslims": 36452, "321": 36453, "Ġprompts": 36454, "sei": 36455, "ĠHitch": 36456, "imposed": 36457, "ĠFool": 36458, "Ġindiscrim": 36459, "wrong": 36460, "buquerque": 36461, "Davis": 36462, "!]": 36463, "Ġtimeless": 36464, "ĠNEED": 36465, "Ġpesticide": 36466, "Ġrallying": 36467, "ĠCalder": 36468, "Ġå¤": 36469, "Ġxp": 36470, "ĠUnle": 36471, "ĠExport": 36472, "luaj": 36473, "Buff": 36474, ")[": 36937, "Ġsqor": 36938, "Saudi": 36939, "Ġistg": 36940, "Ġindulge": 36941, "proc": 36942, "Ġdisgusted": 36943, "Ġcompounded": 36944, "Ġnem": 36945, "Ġschooling": 36946, "ĠCure": 36947, "processing": 36948, "Sol": 36949, "Ġproverb": 36950, "itized": 36951, "ĠAlvarez": 36952, "Ġscarf": 36953, "Ġrectangular": 36954, "reve": 36955, "Ġhormonal": 36956, "ĠStress": 36957, "itizen": 36958, "Ġ425": 36959, "girls": 36960, "ĠNoir": 36961, "ĠRapp": 36962, "Ġmarches": 36963, "church": 36964, "ĠUses": 36965, "Ġ405": 36966, "ĠBerm": 36967, "Ġordinances": 36968, "ĠJudgment": 36969, "Charges": 36970, "ĠZin": 36971, "Ġdusty": 36972, "Ġstrawberries": 36973, "Ġperce": 36974, "ĠThur": 36975, "ĠDeborah": 36976, "netflix": 36977, "ĠLambert": 36978, "Ġamused": 36979, "ĠGuang": 36980, "YOU": 36981, "RGB": 36982, "ĠCCTV": 36983, "Ġfiat": 36984, "rang": 36985, "Ġfederation": 36986, "ĠMant": 36987, "ĠBust": 36988, "ĠMare": 36989, "respective": 36990, "ĠMigration": 36991, "ĠBIT": 36992, "590": 36993, "Ġpatriotism": 36994, "Ġoutlining": 36995, "region": 36996, "ĠJosé": 36997, "Ġblasting": 36998, "ĠEzra": 36999, "Bs": 37000, "Ġundermines": 37001, "ĠSmooth": 37002, "Ġclashed": 37003, "radio": 37004, "Ġtransitioning": 37005, "ĠBuccaneers": 37006, "ĠOwl": 37007, "Ġplugs": 37008, "Ġhiatus": 37009, "ĠPinball": 37010, "Ġmig": 37011, "ĠNutr": 37012, "ĠWolfe": 37013, "Ġintegers": 37014, "Ġorbits": 37015, "ĠEdwin": 37016, "ĠDirectX": 37017, "bite": 37018, "Ġblazing": 37019, "vr": 37020, "Edge": 37021, "ĠPID": 37022, "exit": 37023, "ĠComed": 37024, "ĠPathfinder": 37025, "ĠGuid": 37026, "ĠSigns": 37027, "ĠZer": 37028, "ĠAgenda": 37029, "Ġreimbursement": 37030, "Mesh": 37031, "iPhone": 37032, "ĠMarcos": 37033, "ĠSites": 37034, "hate": 37035, "enburg": 37036, "Ġsockets": 37037, "pend": 37038, "Batman": 37039, "vir": 37040, "ĠSHOW": 37041, "Ġprovisional": 37042, "conn": 37043, "ĠDeaths": 37044, "ATIVE": 37045, "Profile": 37046, "sym": 37047, "JA": 37048, "Ġninja": 37049, "installed": 37050, "idates": 37051, "ebra": 37052, "ĠOmaha": 37053, "Ġseizing": 37054, "ĠBeasts": 37055, "Ġsalts": 37056, "Mission": 37057, "Generally": 37058, "ĠTrilogy": 37059, "heon": 37060, "legates": 37061, "Ġdime": 37062, "Ġfaire": 37063, "parable": 37064, "Graph": 37065, "Ġtotaling": 37066, "Ġdiagrams": 37067, "ĠYanuk": 37068, "plet": 37069, "ĠMeh": 37070, "Ġmythical": 37071, "ĠStephens": 37072, "autical": 37073, "ochemistry": 37074, "Ġkilograms": 37075, "Ġelbows": 37076, "ancock": 37077, "ĠBCE": 37078, "ĠPrague": 37079, "Ġimprov": 37080, "ĠDevin": 37081, "Ġ\"\\": 37082, "paralle": 37083, "Ġsupremacists": 37084, "ĠBillion": 37085, "Ġregimen": 37086, "innacle": 37087, "Ġrequisite": 37088, "angan": 37089, "ĠBurlington": 37090, "ainment": 37091, "ĠObjective": 37092, "omsky": 37093, "GV": 37094, "Ġunilateral": 37095, "Ġtc": 37096, "Ġhires": 37097, "mental": 37098, "Ġinvoluntary": 37099, "Ġtranspl": 37100, "ĠASCII": 37101, "¨": 37102, "Events": 37103, "Ġdoubted": 37104, "ĠKaplan": 37105, "ĠCourage": 37106, "igon": 37107, "ĠManaging": 37108, "ĠTart": 37109, "Ġfalsehood": 37110, "ĠViolet": 37111, "Ġairs": 37112, "Ġfertilizer": 37113, "Britain": 37114, "Ġaquatic": 37115, "ouf": 37116, "Words": 37117, "ĠHartford": 37118, "Ġevenings": 37119, "ĠVengeance": 37120, "quite": 37121, "Gall": 37122, "ĠPret": 37123, "Ġpdf": 37124, "ĠLM": 37125, "ĠSochi": 37126, "ĠIntercept": 37127, "920": 37128, "Ġprofitability": 37129, "ĠIdle": 37130, "ĠMacDonald": 37131, "ĠEstablishment": 37132, "umsy": 37133, "Ġgatherings": 37134, "ĠNaj": 37135, "Charlie": 37136, "Ġascent": 37137, "ĠProtector": 37138, "Ġalgebra": 37139, "Ġbios": 37140, "forums": 37141, "ELS": 37142, "Introduced": 37143, "Ġ335": 37144, "Ġastronomy": 37145, "Contribut": 37146, "ĠPolic": 37147, "Platform": 37148, "Ġcontainment": 37149, "wrap": 37150, "Ġcoronary": 37151, "ĠJelly": 37152, "manager": 37153, "Ġheartbreaking": 37154, "cair": 37155, "ĠChero": 37156, "cgi": 37157, "Medical": 37158, "ĠAccountability": 37159, "!!\"": 37160, "ophile": 37161, "Ġpsychotic": 37162, "ĠRestrict": 37163, "Ġequitable": 37164, "issues": 37165, "Ġ1905": 37166, "ĠNek": 37167, "cised": 37168, "ĠTracking": 37169, "Ġozone": 37170, "Ġcooker": 37171, "rosis": 37172, "Ġreopen": 37173, "Ġinfinity": 37174, "ĠPharmaceutical": 37175, "ensional": 37176, "Attempt": 37177, "ĠRory": 37178, "Marco": 37179, "Ġawaits": 37180, "HOW": 37181, "treated": 37182, "Ġbolst": 37183, "Ġrevered": 37184, "Ġpods": 37185, "oppers": 37186, "0010": 37187, "Ġamplitude": 37188, "rican": 37189, "SPONSORED": 37190, "Ġtrousers": 37191, "Ġhalves": 37192, "ĠKaine": 37193, "ĠCutler": 37194, "ĠAUTH": 37195, "Ġsplendid": 37196, "Ġpreventive": 37197, "ĠDudley": 37198, "ifacts": 37199, "uminati": 37200, "ĠYin": 37201, "Ġadmon": 37202, "ĠVag": 37203, "Ġinverted": 37204, "Ġhastily": 37205, "ĠHague": 37206, "Lyn": 37207, "Ġledger": 37208, "Ġastronomical": 37209, "getting": 37210, "Ġcirca": 37211, "ĠCic": 37212, "ĠTennis": 37213, "Limited": 37214, "Ġdru": 37215, "ĠBYU": 37216, "Ġtravellers": 37217, "Ġpane": 37218, "ĠIntro": 37219, "Ġpatiently": 37220, "Ġaiding": 37221, "Ġloos": 37222, "ĠTough": 37223, "Ġ293": 37224, "Ġconsumes": 37225, "SourceFile": 37226, "Ġ\"\"\"": 37227, "Ġbonding": 37228, "Ġtilted": 37229, "Ġmenstrual": 37230, "ĠCelestial": 37231, "ULAR": 37232, "Plugin": 37233, "Ġrisking": 37234, "Naz": 37235, "ĠRiyadh": 37236, "Ġaccredited": 37237, "Ġskirm": 37238, "éĽ": 37239, "Ġexaminer": 37240, "Ġmessing": 37241, "Ġnearing": 37242, "ĠChern": 37243, "ĠBeckham": 37244, "Ġswapped": 37245, "Ġgoose": 37246, "Kay": 37247, "Ġlofty": 37248, "ĠWallet": 37249, "Ġ['": 37250, "Ġapocalypse": 37251, "Ġbamboo": 37252, "ĠSPACE": 37253, "ĠElena": 37254, "Ġ306": 37255, "acons": 37256, "Ġtightened": 37257, "Ġadolescence": 37258, "Ġrainy": 37259, "Ġvandalism": 37260, "ĠNewtown": 37261, "Ġconject": 37262, "cakes": 37263, "Ġcheated": 37264, "Ġmoderators": 37265, "params": 37266, "EFF": 37267, "Ġdeceit": 37268, "ĠSTL": 37269, "ĠTanzania": 37270, "ĠRI": 37271, "Ġ1923": 37272, "ĠExile": 37273, "thel": 37274, "Ġtheolog": 37275, "Ġquirky": 37276, "ĠIrvine": 37277, "Ġneedy": 37278, "oris": 37279, "Um": 37280, "Ka": 37281, "Ġmailbox": 37282, "322": 37283, "Ġbos": 37284, "ĠPetra": 37285, "KING": 37286, "Ġenlarged": 37287, "Often": 37288, "Ġbadass": 37289, "Ġ343": 37290, "ĠPlaces": 37291, "ĠCAD": 37292, "Ġpristine": 37293, "Ġintervening": 37294, "direction": 37295, "Ġlaz": 37296, "ĠDSM": 37297, "Ġprojecting": 37298, "ĠFunk": 37299, "agog": 37300, "payment": 37301, "nov": 37302, "Ġchatter": 37303, "ARB": 37304, "Ġexaminations": 37305, "ĠHousehold": 37306, "ĠGus": 37307, "Ford": 37308, "414": 37309, "Boss": 37310, "Ġmystic": 37311, "Ġleaps": 37312, "ĠBav": 37313, "ulz": 37314, "budget": 37315, "Football": 37316, "Ġsubsidized": 37317, "Ġfirsthand": 37318, "Ġcoincide": 37319, "ocular": 37320, "Conn": 37321, "ĠCollabor": 37322, "Ġfools": 37323, "amura": 37324, "ahar": 37325, "rists": 37326, "Ġswollen": 37327, "Ġexpended": 37328, "ĠPau": 37329, "sup": 37330, "Ġspar": 37331, "Ġkeynote": 37332, "suff": 37333, "Ġunequal": 37334, "Ġprogressing": 37335, "strings": 37336, "ĠGamergate": 37337, "Disney": 37338, "ĠEleven": 37339, "omnia": 37340, "Ġscripted": 37341, "Ġearners": 37342, "brother": 37343, "ĠEnabled": 37344, "æ³": 37345, "Ġlarvae": 37346, "ĠLOC": 37347, "mess": 37348, "Wilson": 37349, "ĠTemplate": 37350, "successfully": 37351, "Ġparamount": 37352, "Ġcamouflage": 37353, "Ġbinds": 37354, "ĠQuiet": 37355, "ĠShutterstock": 37356, "rush": 37357, "Ġmascot": 37358, "fortune": 37359, "ĠColt": 37360, "ĠBeyon": 37361, "habi": 37362, "Ġhairc": 37363, "Ġ267": 37364, "ĠDeus": 37365, "Ġtwitch": 37366, "Ġconcentrating": 37367, "Ġnipples": 37368, "cible": 37369, "Ġgir": 37370, "NZ": 37371, "Math": 37372, "nih": 37373, "Required": 37374, "Ġponder": 37375, "ĠSAN": 37376, "Ġweddings": 37377, "Ġloneliness": 37378, "NES": 37379, "ĠMahjong": 37380, "695": 37381, "addle": 37382, "ĠGarner": 37383, "ĠCOUR": 37384, "Bridge": 37385, "Ġspree": 37386, "ĠCaldwell": 37387, "Ġbribery": 37388, "Ġ��������": 37389, "plugins": 37390, "Ġracket": 37391, "Ġchampagne": 37392, "versible": 37393, "Vote": 37394, "Ġmodifiers": 37395, "Mayor": 37396, "680": 37397, "Ġassemblies": 37398, "ĠSultan": 37399, "ĠNing": 37400, "ĠLadies": 37401, "Ġsulfur": 37402, "Ġorbs": 37403, "Ġ-----": 37404, "_______": 37405, "ĠJournalism": 37406, "Ġesports": 37407, "Ġlush": 37408, "Ġhue": 37409, "Ġspectral": 37410, "Honest": 37411, "ãĥı": 37412, "Ġbushes": 37413, "Ġreinforcement": 37414, "Ġreopened": 37415, "ĠWheels": 37416, "ĠMorg": 37417, "rieving": 37418, "Ġauxiliary": 37419, "ĠjQuery": 37420, "ĠBAT": 37421, "tesque": 37422, "Ġvertex": 37423, "pure": 37424, "frey": 37425, "ãĤº": 37426, "dos": 37427, "Ġtyph": 37428, "Ġcull": 37429, "Ġeq": 37430, "Ġdecon": 37431, "Ġtossing": 37432, "Ġdisparate": 37433, "ĠBrigham": 37434, "printf": 37435, "ledged": 37436, "Ġsund": 37437, "Ġcozy": 37438, "Ġhepatitis": 37439, "performing": 37440, "Ġaval": 37441, "ĠGG": 37442, "future": 37443, "Ġpetertodd": 37444, "ĠKosovo": 37445, "Ġmagnets": 37446, "Already": 37447, "ĠEdison": 37448, "ĠCeres": 37449, "ĠRAID": 37450, "Ġbrilliance": 37451, "576": 37452, "Ġderives": 37453, "Ġhypertension": 37454, "ĠÎĶ": 37455, "Ġlambda": 37456, "Ġflair": 37457, "Ġmissionaries": 37458, "Ġrapes": 37459, "ĠStarter": 37460, "ĠMonths": 37461, "Ġdefy": 37462, "Ġseismic": 37463, "ĠRaphael": 37464, "Ġeurozone": 37465, "656": 37466, "zsche": 37467, "Ġscratched": 37468, "Ġbows": 37469, "ĠLennon": 37470, "ĠGaia": 37471, "Ġdripping": 37472, "facts": 37473, "Ale": 37474, "Ġfrogs": 37475, "ĠBreast": 37476, "ogeneity": 37477, "ĠProsecutor": 37478, "Ġamplified": 37479, "ĠHodg": 37480, "ĠFn": 37481, "Thousands": 37482, "ĠNIH": 37483, "ĠMonitoring": 37484, "FTWARE": 37485, "ĠPriebus": 37486, "ĠGrowing": 37487, "hunter": 37488, "Ġdiagnose": 37489, "ĠMald": 37490, "ĠLR": 37491, "Ġcrowned": 37492, "Ġbursting": 37493, "Ġdissolution": 37494, "javascript": 37495, "Ġusefulness": 37496, "ĠExecution": 37497, ":(": 37498, "ĠIvory": 37499, "aah": 37500, "Ġpersecuted": 37501, "violence": 37502, "istas": 37503, "ĠCrate": 37504, "Ġimpulses": 37505, "ĠSpani": 37506, "edes": 37507, "Handle": 37508, "ĠZerg": 37509, "thinkable": 37510, "Lastly": 37511, "Ġspontaneously": 37512, "Ġinconvenient": 37513, "Ġdismissing": 37514, "Ġplotted": 37515, "Ġeighty": 37516, "Ġ737": 37517, "rish": 37518, "ĠThornton": 37519, "atham": 37520, "Ġsitcom": 37521, "Ven": 37522, "Recipe": 37523, "tel": 37524, "lund": 37525, "Ġclears": 37526, "ĠSasuke": 37527, "Ġ258": 37528, "Ġopting": 37529, "Ġenraged": 37530, "esthetic": 37531, "ĠAe": 37532, "uchs": 37533, "Prep": 37534, "Flow": 37535, "Ġrunoff": 37536, "ĠEating": 37537, "ĠGiles": 37538, "ĠActing": 37539, "resources": 37540, "ibaba": 37541, "Ġrpm": 37542, "Ġskewed": 37543, "ĠBlanc": 37544, "ĠSakuya": 37545, "Ġhotter": 37546, "Ġ1924": 37547, "opian": 37548, "cko": 37549, "Ġcrumbling": 37550, "Ġcaptains": 37551, "ĠAppropriations": 37552, "leaders": 37553, "dropping": 37554, "anuts": 37555, "Ġreversing": 37556, "ĠPose": 37557, "ĠSek": 37558, "Scot": 37559, "ĠIdea": 37560, "cise": 37561, "ĠSlovenia": 37562, "Ġ317": 37563, "Doctor": 37564, "Ġcrocod": 37565, "aldi": 37566, "Sea": 37567, "ĠFarrell": 37568, "Ġmercenaries": 37569, "ĠRNC": 37570, "ĠGuess": 37571, "Ġpacing": 37572, "Machine": 37573, "StreamerBot": 37574, "ĠCharity": 37575, "Ġ298": 37576, "Ġcannons": 37577, "ĠToby": 37578, "TPPStreamerBot": 37579, "ĠPassion": 37580, "cfg": 37581, "Thom": 37582, "Ġbadges": 37583, "ĠBernstein": 37584, ".âĢĵ": 37585, "ĠPOP": 37586, "ĠConj": 37587, "Ġinitialization": 37588, "Ġbiodiversity": 37589, "Dub": 37590, "Ġfeudal": 37591, "Ġdisclaimer": 37592, "Ġcrow": 37593, "Ġignition": 37594, "arf": 37595, "SHA": 37596, "ĠkHz": 37597, "hazard": 37598, "ĠArtists": 37599, "oeuv": 37600, "679": 37601, "ĠRudy": 37602, "Nine": 37603, "ĠRamadan": 37604, "å½": 37605, "itto": 37606, "Ġadrenaline": 37607, "Cert": 37608, "Ġsmelled": 37609, "Ġimpunity": 37610, "Ġagendas": 37611, "ĠReborn": 37612, "ĠConcent": 37613, "ĠSeems": 37614, "Ġomega": 37615, "ĠDustin": 37616, "Ġbacker": 37617, "ĠSauce": 37618, "ĠBoyle": 37619, "WIN": 37620, "Ġspins": 37621, "Ġpauses": 37622, "upt": 37623, "Ġshredded": 37624, "Ġstrapped": 37625, "ĠCorruption": 37626, "Ġscratches": 37627, "Ġni": 37628, "Ġattire": 37629, "ĠSAF": 37630, "FactoryReloaded": 37631, "ĠIPS": 37632, "Ġ(%": 37633, "Ġseminar": 37634, "focus": 37635, "civil": 37636, "Ġ1860": 37637, "intosh": 37638, "Ġcontinual": 37639, "Ġabbrevi": 37640, "ĠSok": 37641, "ocobo": 37642, "XM": 37643, "Ġfrantic": 37644, "Ġunavoidable": 37645, "Ġartery": 37646, "Ġannotations": 37647, "bath": 37648, "Climate": 37649, "Ġdors": 37650, "ĠSlide": 37651, "coord": 37652, "ĠReload": 37653, "ĠLDL": 37654, "ĠLovecraft": 37655, "Ġunimagin": 37656, "Ġresembled": 37657, "Ġbarracks": 37658, "np": 37659, "Ġsurrogate": 37660, "Ġcategorized": 37661, "ãĤ©": 37662, "Ġvaccinated": 37663, "Ġdrainage": 37664, "Ġindist": 37665, "ĠWhatsApp": 37666, "Ġ1870": 37667, "olerance": 37668, "invoke": 37669, "amorph": 37670, "Ġreconnect": 37671, "Ġemanc": 37672, "Ġblindness": 37673, "Ġ1280": 37674, "internet": 37675, "collar": 37676, "Ġaltru": 37677, "Ġabyss": 37678, "ĠTRI": 37679, "657": 37680, "Ġinfused": 37681, "HEAD": 37682, "Ġforestry": 37683, "ĠWoody": 37684, "ĠCi": 37685, "wi": 37686, "sam": 37687, "784": 37688, "holiday": 37689, "Ġmogul": 37690, "ĠFees": 37691, "ĠDEN": 37692, "Internal": 37693, "urbed": 37694, "fusc": 37695, "atom": 37696, "ĠIllusion": 37697, "Ġpolled": 37698, "Ġflap": 37699, "Ġcoax": 37700, "LGBT": 37701, "Analy": 37702, "ĠSections": 37703, "ĠCaliforn": 37704, "emn": 37705, "Ġhither": 37706, "ĠNIGHT": 37707, "Ġnailed": 37708, "ĠPipeline": 37709, "391": 37710, "oof": 37711, "ĠPrimal": 37712, "verend": 37713, "Ġslashing": 37714, "Ġretri": 37715, "aviour": 37716, "Ġdeparting": 37717, "gil": 37718, "ISC": 37719, "Ġmidway": 37720, "Ġultrasound": 37721, "Ġbehaving": 37722, "ĠTara": 37723, "classes": 37724, "Virtual": 37725, "ĠColonial": 37726, "Ġstripping": 37727, "Ġorchestrated": 37728, "ĠGraves": 37729, "452": 37730, "ĠIronically": 37731, "ĠWriters": 37732, "Ġlends": 37733, "ĠManz": 37734, "Ġraven": 37735, "Ġoxidative": 37736, "Ġ266": 37737, "ELF": 37738, "actually": 37739, "ascar": 37740, "Draft": 37741, "Ġfavourable": 37742, "Ġhumiliating": 37743, "Ġfidelity": 37744, "ĠHof": 37745, "ĠXuan": 37746, "496": 37747, "Ġlayered": 37748, "atis": 37749, "790": 37750, "Ġpaycheck": 37751, "iton": 37752, "Kar": 37753, "ĠVMware": 37754, "ĠFarmer": 37755, "Ġservic": 37756, "glomer": 37757, "Ġslump": 37758, "ĠFabric": 37759, "ĠDOC": 37760, "esting": 37761, "Ġreassure": 37762, "Ġphyl": 37763, "volt": 37764, "itory": 37765, "Rules": 37766, "Ġoxidation": 37767, "Ġprized": 37768, "Ġmistress": 37769, "ĠDjango": 37770, "WARN": 37771, "åij": 37772, "Ġencode": 37773, "ĠFeedback": 37774, "Ġstupidity": 37775, "Ian": 37776, "ĠYugoslavia": 37777, "ר": 37778, "acl": 37779, "UTE": 37780, "1977": 37781, "Ġqualifies": 37782, "Ġpulses": 37783, "pretty": 37784, "Ġfroze": 37785, "Ġss": 37786, "Iterator": 37787, "Ġurgently": 37788, "Ġmailed": 37789, "ĠCham": 37790, "Ġsustaining": 37791, "Ġbasil": 37792, "Ġpuppies": 37793, "ilant": 37794, "ĠPLEASE": 37795, "lap": 37796, "aceous": 37797, "Fear": 37798, "ĠMastery": 37799, "automatic": 37800, "ĠTAG": 37801, "Ġantim": 37802, "agles": 37803, "473": 37804, "frames": 37805, "Ġwhispers": 37806, "ĠWhoever": 37807, "Ġbravery": 37808, "ĠUKIP": 37809, "ractions": 37810, "\"\"\"": 37811, "Ġtame": 37812, "Ġparted": 37813, "everything": 37814, "CONT": 37815, "Ġindebted": 37816, "Ġaddr": 37817, "rek": 37818, "IRED": 37819, "Ġeminent": 37820, "clinton": 37821, "Ġousted": 37822, "Ġreviewer": 37823, "Ġmeltdown": 37824, "Ġrearr": 37825, "ĠYao": 37826, "thereal": 37827, "abyte": 37828, "Ġstumbling": 37829, "Ġbatches": 37830, "Ġ259": 37831, "Ġcontraceptive": 37832, "Ġprostitute": 37833, "ensis": 37834, "Decl": 37835, "ĠStrikes": 37836, "Military": 37837, "ĠOath": 37838, "vacc": 37839, "ppings": 37840, "052": 37841, "ĠpartName": 37842, "amping": 37843, "Reports": 37844, "KI": 37845, "CHR": 37846, "Ġsubtly": 37847, "swers": 37848, "Blake": 37849, "usual": 37850, "Ġcontestants": 37851, "Ġcartridges": 37852, "ĠGREAT": 37853, "Ġblush": 37854, "ĠâĢº": 37855, "472": 37856, "Ġreasoned": 37857, "ãĥ¤": 37858, "paralleled": 37859, "Ġdyn": 37860, "agate": 37861, "Ġnightly": 37862, "åĨ": 37863, "556": 37864, "Ġsemantic": 37865, "ĠAdvoc": 37866, "Ġ!!": 37867, "Ġdisagrees": 37868, "ĠBW": 37869, "Veh": 37870, "Ġharming": 37871, "Ġembraces": 37872, "Ġstrives": 37873, "Ġinland": 37874, "ĠKard": 37875, "Ġheats": 37876, "ĠGinny": 37877, "utan": 37878, "ernaut": 37879, "ylene": 37880, "ĠElev": 37881, "JD": 37882, "Ġhars": 37883, "ĠStarr": 37884, "Ġskysc": 37885, "Ġcollaborators": 37886, "Usually": 37887, "Ġrevolutions": 37888, "ĠSTATS": 37889, "Ġdismantle": 37890, "Ġconfidently": 37891, "Ġkinetic": 37892, "Ali": 37893, "Ġpercentile": 37894, "Ġextracting": 37895, "illian": 37896, "estead": 37897, "Ġphysicists": 37898, "ĠMarshal": 37899, "Ġfellowship": 37900, "Ġdashed": 37901, "ĠUR": 37902, "ĠSioux": 37903, "ĠCompact": 37904, "amide": 37905, "Python": 37906, "ĠLeigh": 37907, "ĠPharmac": 37908, "istrates": 37909, "herical": 37910, "Ġfue": 37911, "ĠEmin": 37912, "Ġ({": 37913, "ĠNeighborhood": 37914, "Ġdisrupting": 37915, "ĠDup": 37916, "Ġgland": 37917, "ĠSev": 37918, "ĠMarian": 37919, "argon": 37920, "ĠDund": 37921, "Ġ": 46904, "ĠPhilips": 46905, "ĠKafka": 46906, "Ġupheaval": 46907, "Ġsentimental": 46908, "Ġsax": 46909, "ĠAkira": 46910, "serial": 46911, "Matrix": 46912, "Ġelecting": 46913, "Ġcommenter": 46914, "ĠNebula": 46915, "plets": 46916, "ĠNadu": 46917, "ĠAdren": 46918, "Ġenshr": 46919, "ĠRAND": 46920, "financial": 46921, "ĠClyde": 46922, "utherford": 46923, "Ġsignage": 46924, "Ġdeline": 46925, "Ġphosphate": 46926, "roversial": 46927, "fascist": 46928, "ĠVall": 46929, "ĠBethlehem": 46930, "Ġfors": 46931, "Ġenglish": 46932, "Solid": 46933, "Nature": 46934, "Ġva": 46935, "ĠGuests": 46936, "Ġtantal": 46937, "Ġautoimmune": 46938, ";;;;;;;;;;;;": 46939, "ĠTotally": 46940, "ĠOv": 46941, "Ġdefences": 46942, "ĠCoconut": 46943, "Ġtranquil": 46944, "Ġploy": 46945, "Ġflavours": 46946, "ĠFlask": 46947, "ãĤ¨ãĥ«": 46948, "ĠWeston": 46949, "ĠVolvo": 46950, "870": 46951, "Ġmicrophones": 46952, "verbal": 46953, "RPG": 46954, "Ġiii": 46955, ";}": 46956, "028": 46957, "Ġheadlined": 46958, "Ġprimed": 46959, "Ġhoard": 46960, "ĠShad": 46961, "ĠENTER": 46962, "Ġtriangular": 46963, "Ġcapit": 46964, "lik": 46965, "ĠAncients": 46966, "Ġlash": 46967, "Ġconvol": 46968, "Ġcolonel": 46969, "enemy": 46970, "Gra": 46971, "Ġpubs": 46972, "utters": 46973, "Ġassigns": 46974, "ĠPenet": 46975, "ĠMonstrous": 46976, "ĠBowen": 46977, "ilver": 46978, "Haunted": 46979, "ĠDing": 46980, "started": 46981, "plin": 46982, "Ġcontaminants": 46983, "ĠDOE": 46984, "ffen": 46985, "ĠTechnician": 46986, "Ry": 46987, "Ġrobbers": 46988, "Ġhotline": 46989, "ĠGuardiola": 46990, "ĠKaufman": 46991, "rower": 46992, "ĠDresden": 46993, "ĠAlpine": 46994, "Elf": 46995, "Ġfmt": 46996, "ĠSard": 46997, "urses": 46998, "gpu": 46999, "Unix": 47000, "Ġunequivocally": 47001, "ĠCitizenship": 47002, "quad": 47003, "mire": 47004, "ĠSweeney": 47005, "Battery": 47006, "615": 47007, "Ġpancakes": 47008, "Ġoats": 47009, "Maps": 47010, "ĠContrast": 47011, "mbudsman": 47012, "ĠEPS": 47013, "Ġsubcommittee": 47014, "Ġsourcing": 47015, "Ġsizing": 47016, "ĠBuffer": 47017, "ĠMandatory": 47018, "Ġmoderates": 47019, "ĠPatterns": 47020, "ĠChocobo": 47021, "ĠZan": 47022, "ĠSTATES": 47023, "ĠJudging": 47024, "ĠInher": 47025, "*:": 47026, "Ġbil": 47027, "ĠYen": 47028, "Ġexhilar": 47029, "ollower": 47030, "zers": 47031, "Ġsnug": 47032, "maximum": 47033, "Ġdespicable": 47034, "ĠPACK": 47035, "ĠAnnex": 47036, "Ġsarcastic": 47037, "Ġlatex": 47038, "Ġtamp": 47039, "ĠSao": 47040, "bah": 47041, "ĠReverend": 47042, "ĠChinatown": 47043, "ĠAUT": 47044, "documented": 47045, "ĠGABA": 47046, "ĠCanaan": 47047, "ĠÙħ": 47048, "Ġgoverns": 47049, "prev": 47050, "Esc": 47051, "ĠEstimates": 47052, "OSP": 47053, "Ġendeavour": 47054, "ĠClosing": 47055, "ometime": 47056, "everyone": 47057, "Ġworsen": 47058, "Ġscanners": 47059, "Ġdeviations": 47060, "ĠRobotics": 47061, "ĠCompton": 47062, "Ġsorcerer": 47063, "Ġendogenous": 47064, "Ġemulation": 47065, "ĠPiercing": 47066, "ĠAph": 47067, "ĠSocket": 47068, "Ġbould": 47069, "ĠOU": 47070, "ĠBorderlands": 47071, "Ġ1863": 47072, "Gordon": 47073, "ĠWTO": 47074, "Ġrestricts": 47075, "Ġmosaic": 47076, "Ġmelodies": 47077, "çĦ": 47078, "Tar": 47079, "Ġdisson": 47080, "ĠProvides": 47081, "Ġ......": 47082, "bek": 47083, "FIX": 47084, "Ġbroom": 47085, "anship": 47086, "Doctors": 47087, "Ġnerds": 47088, "ĠRegions": 47089, "naissance": 47090, "Ġmete": 47091, "Ġcrept": 47092, "plings": 47093, "Ġgirlfriends": 47094, "knit": 47095, "igent": 47096, "owe": 47097, "Ġushered": 47098, "ĠBaz": 47099, "Mobil": 47100, "434": 47101, "ĠPresents": 47102, "origin": 47103, "Ġinsomnia": 47104, "ĠAux": 47105, "439": 47106, "ĠChili": 47107, "irsch": 47108, "GAME": 47109, "Ġgestation": 47110, "algia": 47111, "romising": 47112, "$,": 47113, "crow": 47114, "ĠInspection": 47115, "atomic": 47116, "Relations": 47117, "JOHN": 47118, "roman": 47119, "ĠClockwork": 47120, "ĠBakr": 47121, "mone": 47122, "MET": 47123, "Ġthirsty": 47124, "Ġbc": 47125, "Ġfaculties": 47126, "Rum": 47127, "Ġnuance": 47128, "ĠDarius": 47129, "pleting": 47130, "fters": 47131, "etchup": 47132, "Registration": 47133, "ĠKE": 47134, "Rah": 47135, "Ġpreferential": 47136, "ĠLash": 47137, "ĠHH": 47138, "Valid": 47139, "ĠNAV": 47140, "Ġstarve": 47141, "ĠGong": 47142, "zynski": 47143, "ĠActress": 47144, "Ġwik": 47145, "Ġunaccompanied": 47146, "lvl": 47147, "Bride": 47148, "ADS": 47149, "ĠCommando": 47150, "ĠVaughn": 47151, "Wallet": 47152, "Ġhopping": 47153, "ĠVie": 47154, "Ġcaveats": 47155, "Ġalas": 47156, "ifled": 47157, "abuse": 47158, "661": 47159, "Ġibn": 47160, "Ġgul": 47161, "Ġrobbing": 47162, "til": 47163, "ILA": 47164, "Ġmitigating": 47165, "Ġaptly": 47166, "Ġtyrant": 47167, "Ġmidday": 47168, "ĠGilmore": 47169, "ĠDecker": 47170, "Ġ§§": 47171, "partial": 47172, "Exactly": 47173, "Ġphenotype": 47174, "Ġ[+]": 47175, "ĠPlex": 47176, "ĠIps": 47177, "versions": 47178, "Ġebook": 47179, "Ġchic": 47180, "gross": 47181, "\":\"\"},{\"": 47182, "ĠSurprisingly": 47183, "Morgan": 47184, "Ġresidues": 47185, "ĠConfederation": 47186, "infeld": 47187, "Ġlyr": 47188, "moderate": 47189, "Ġperpendicular": 47190, "VK": 47191, "Ġsynchronized": 47192, "Ġrefreshed": 47193, "Ġadore": 47194, "ĠTorment": 47195, "olina": 47196, "Ġ2600": 47197, "ItemTracker": 47198, "Ġpies": 47199, "ĠFAT": 47200, "ĠRHP": 47201, "048": 47202, "ĠRESP": 47203, "ĠBJ": 47204, "allows": 47205, "Pand": 47206, "Ġunwelcome": 47207, "ĠVoc": 47208, "ĠBastard": 47209, "ĠOW": 47210, "ĠLAR": 47211, "ĠHealer": 47212, "Environmental": 47213, "ĠKenyan": 47214, "ĠTrance": 47215, "ĠPats": 47216, "Ġaliases": 47217, "ĠGarfield": 47218, "Ġcampaigner": 47219, "Ġadvancements": 47220, "ĠOkinawa": 47221, "ĠCoh": 47222, "owsky": 47223, "Ġstarved": 47224, "Ġsizeable": 47225, "Ġ:-)": 47226, "ĠmRNA": 47227, "Ġsuspensions": 47228, "istar": 47229, "Scotland": 47230, "Prin": 47231, "------------------------------------------------": 47232, "Ġ502": 47233, "Ġteaspoons": 47234, "Ġ1050": 47235, "Ġcoercive": 47236, "ĠMasonic": 47237, "edded": 47238, "ĠPassenger": 47239, "Ġlatt": 47240, "Ġbraces": 47241, "ĠSteal": 47242, "ĠNYT": 47243, "ĠKats": 47244, "ĠCelest": 47245, "aez": 47246, "Tu": 47247, "ĠCoulter": 47248, "ðŁĺ": 47249, "Flickr": 47250, "ĠWilmington": 47251, "iths": 47252, "++;": 47253, "Ġvending": 47254, "Ġnegro": 47255, "ĠPhi": 47256, "ĠYellowstone": 47257, "Callback": 47258, "Ġshampoo": 47259, "ĠShades": 47260, "wat": 47261, "Ġsuperhuman": 47262, "Ġridiculed": 47263, "Ġholiest": 47264, "ombo": 47265, "Ġinterns": 47266, "Ġhone": 47267, "ĠParagu": 47268, "URI": 47269, "Ġdangling": 47270, "ãĤ»": 47271, "sov": 47272, "ictional": 47273, "availability": 47274, "Ġrevocation": 47275, "Ġdow": 47276, "inic": 47277, "ĠTHEIR": 47278, "Ġiso": 47279, "Ġoutings": 47280, "ĠLethal": 47281, "Ġ)))": 47282, "Ġinaccur": 47283, "Ġoutlandish": 47284, "Ġanus": 47285, "letico": 47286, "idon": 47287, "lol": 47288, "Ġunregulated": 47289, "Ġsuccumbed": 47290, "Ġcuff": 47291, "ĠWasteland": 47292, "letal": 47293, "Ġsubstr": 47294, "Ġcoffers": 47295, "Ġautomakers": 47296, "ovi": 47297, "ĠXue": 47298, "ĠDaytona": 47299, "Ġjarring": 47300, "Ġfumes": 47301, "Ġdisbanded": 47302, "zik": 47303, "itton": 47304, "Ġstrikingly": 47305, "Ġspores": 47306, "Adapter": 47307, ".):": 47308, "ĠLyndon": 47309, "ivalry": 47310, "Ġorally": 47311, "Ġtumultuous": 47312, "Ġdispleasure": 47313, "Ġcones": 47314, "orrect": 47315, "Ġappease": 47316, "Ġderby": 47317, "ĠTripoli": 47318, "ĠAless": 47319, "Ġpoked": 47320, "ĠGuilty": 47321, "vP": 47322, "Enough": 47323, "Ġoriginals": 47324, "699": 47325, "Ġrabbi": 47326, "Ġproverbial": 47327, "Ġpostpone": 47328, "elope": 47329, "ĠMisty": 47330, "Ġstaffed": 47331, "ĠUnemployment": 47332, "reditary": 47333, "Ġdiligent": 47334, "recomm": 47335, "measures": 47336, "asin": 47337, "825": 47338, "Ġponds": 47339, "Ġmmol": 47340, "ĠSAR": 47341, "ĠCARE": 47342, "Ġ371": 47343, "Ġclenched": 47344, "ĠCorsair": 47345, "Ġcaricature": 47346, "zn": 47347, "attach": 47348, "ĠSchro": 47349, "speak": 47350, "painted": 47351, "ĠSuc": 47352, "ĠENT": 47353, "Ġcellul": 47354, "ĠPaid": 47355, "diagn": 47356, "WHERE": 47357, "Ġtexted": 47358, "Barn": 47359, "Ġretracted": 47360, "ĠReferred": 47361, "Sav": 47362, "Ġupkeep": 47363, "Ġworkplaces": 47364, "ĠTokens": 47365, "Ġamplify": 47366, "clinical": 47367, "Ġmultic": 47368, "mberg": 47369, "Ġconvoluted": 47370, "Region": 47371, "565": 47372, "ĠTopic": 47373, "Ġsnail": 47374, "Ġsaline": 47375, "Ġinsurrection": 47376, "ĠPetr": 47377, "forts": 47378, "BAT": 47379, "ĠNavajo": 47380, "Ġrudimentary": 47381, "ĠLaksh": 47382, "ONDON": 47383, "Measure": 47384, "Ġtransformer": 47385, "ĠGoddard": 47386, "Ġcoincides": 47387, "irin": 47388, "Rex": 47389, "ĠBok": 47390, "quit": 47391, "Ġshotguns": 47392, "Ġproletarian": 47393, "Ġscorp": 47394, "ĠAda": 47395, "514": 47396, "Ġslander": 47397, "recorded": 47398, "Ġembell": 47399, "risome": 47400, "Ġapologizing": 47401, "ĠMulcair": 47402, "ĠGibraltar": 47403, "Cla": 47404, "Ġallot": 47405, "ĠAttention": 47406, "Ġ433": 47407, "leave": 47408, "Ġwhine": 47409, "ĠIssa": 47410, "ĠFaust": 47411, "ĠBarron": 47412, "heny": 47413, "Ġvictimized": 47414, "Jews": 47415, "Ġnurturing": 47416, "ettel": 47417, "Winged": 47418, "ĠSubtle": 47419, "Ġflavorful": 47420, "ĠReps": 47421, "enged": 47422, "callback": 47423, "Ġdirectional": 47424, "Ġclasp": 47425, "ĠDirections": 47426, "planet": 47427, "iculture": 47428, "Helper": 47429, "icion": 47430, "acia": 47431, "Ġç¥ŀ": 47432, "Ġsurges": 47433, "Ġcanoe": 47434, "ĠPremiership": 47435, "been": 47436, "Ġdefied": 47437, "ĠTrooper": 47438, "Ġtripod": 47439, "Ġgasp": 47440, "ĠEuph": 47441, "ĠAds": 47442, "vernight": 47443, "highly": 47444, "Role": 47445, "Ġentangled": 47446, "ĠZeit": 47447, "618": 47448, "ĠRusty": 47449, "Ġhavens": 47450, "ĠVaughan": 47451, "HAEL": 47452, "ĠSERVICE": 47453, "/,": 47454, "Ġstricken": 47455, "Ġdelusions": 47456, "Ġbis": 47457, "ĠHaf": 47458, "Ġgratification": 47459, "Ġenticing": 47460, "UNCH": 47461, "Adams": 47462, "ĠOLED": 47463, "ĠBeetle": 47464, "Ġ1899": 47465, "ĠSOFTWARE": 47466, "ategor": 47467, "VL": 47468, "ĠTotem": 47469, "ĠGators": 47470, "ATURES": 47471, "Ġimpedance": 47472, "Registered": 47473, "ĠCary": 47474, "ĠAerial": 47475, "onne": 47476, "enium": 47477, "Ġdred": 47478, "ĠBeg": 47479, "Ġconcurrently": 47480, "Ġsuperpower": 47481, "ĠXan": 47482, "jew": 47483, "imester": 47484, "ĠDickinson": 47485, "âĶģ": 47486, "Fla": 47487, "Ġpree": 47488, "ĠRollins": 47489, "©¶æ": 47490, "Ġdenomination": 47491, "ĠLana": 47492, "516": 47493, "Ġinciting": 47494, "scribed": 47495, "juries": 47496, "ĠWonders": 47497, "approximately": 47498, "Ġsuspending": 47499, "Ġmountainous": 47500, "ĠLaugh": 47501, "oidal": 47502, "Ns": 47503, "Detect": 47504, ")=": 47505, "ĠLuthor": 47506, "ĠSchwarzenegger": 47507, "ĠMuller": 47508, "ĠDevi": 47509, "ecycle": 47510, "Jar": 47511, "613": 47512, "ĠLongh": 47513, "Bah": 47514, "ĠSPORTS": 47515, "nw": 47516, "Ġrefinement": 47517, "Ġwaterways": 47518, "Ġdiner": 47519, "Blade": 47520, "683": 47521, "Fac": 47522, "Ġinitials": 47523, "Ġrog": 47524, "Ġparanormal": 47525, "BUT": 47526, "Ġ[(": 47527, "ĠSwanson": 47528, "ĠMesh": 47529, "âĸ¬": 47530, "Improve": 47531, "ĠRadiation": 47532, "ĠEsther": 47533, "ĠEsk": 47534, "ĠAly": 47535, "iky": 47536, "Ġirrad": 47537, "ĠBuckingham": 47538, "Ġrefill": 47539, "Ġ._": 47540, "Repe": 47541, "CONCLUS": 47542, "Ġdifferentiated": 47543, "Ġchirop": 47544, "ĠAtkins": 47545, "Pattern": 47546, "Ġexcise": 47547, "Ġcabal": 47548, "NSA": 47549, "ĠSTA": 47550, "ĠSIL": 47551, "ĠParaly": 47552, "Ġrye": 47553, "ĠHowell": 47554, "ĠCountdown": 47555, "nesses": 47556, "alysed": 47557, "Ġresize": 47558, "ãĤ½": 47559, "Ġbudgetary": 47560, "ĠStras": 47561, "wang": 47562, "Ġapiece": 47563, "Ġprecincts": 47564, "Ġpeach": 47565, "Ġskyline": 47566, "Ġ353": 47567, "popular": 47568, "Appearances": 47569, "ĠMechanics": 47570, "ĠDevOnline": 47571, "Sullivan": 47572, "Zen": 47573, "Ġpu": 47574, "opolis": 47575, "544": 47576, "Ġdeform": 47577, "Ġcounteract": 47578, "ĠLange": 47579, "Ġ417": 47580, "Console": 47581, "774": 47582, "Ġnodding": 47583, "Ġpopulism": 47584, "Ġhep": 47585, "Ġcounselling": 47586, "compliance": 47587, "UFF": 47588, "Ġundeniably": 47589, "Ġrailing": 47590, "ĠHorowitz": 47591, "ĠSimone": 47592, "ĠBungie": 47593, "Ġak": 47594, "ĠTalks": 47595, "xff": 47596, "flake": 47597, "Crash": 47598, "Ġsweaty": 47599, "Ġbanquet": 47600, "ĠOFFIC": 47601, "Ġinventive": 47602, "Ġastronomer": 47603, "ĠStamford": 47604, "ĠScare": 47605, "ĠGREEN": 47606, "olicited": 47607, "Ġrusher": 47608, "Ġcentrist": 47609, "ighting": 47610, "Ġsubclass": 47611, "Ġdisav": 47612, "Ġdefund": 47613, "ĠNanto": 47614, "ociate": 47615, "mast": 47616, "Ġpacif": 47617, "Ġmend": 47618, "eers": 47619, "immigration": 47620, "ESSION": 47621, "Ġnumbering": 47622, "Ġlaughable": 47623, "ĠEnded": 47624, "viation": 47625, "emark": 47626, "Pitt": 47627, "Ġmeticulous": 47628, "ĠLF": 47629, "Ġcongratulated": 47630, "ĠBirch": 47631, "Ġswayed": 47632, "Ġsemifinals": 47633, "Ġhumankind": 47634, "matter": 47635, "ĠEquip": 47636, "opausal": 47637, "Said": 47638, "ĠLayout": 47639, "Ġvoicing": 47640, "Ġthug": 47641, "Ġpornographic": 47642, "IPS": 47643, "Ġmoaning": 47644, "Ġgrievance": 47645, "Ġconfessions": 47646, "escal": 47647, "TEXTURE": 47648, "Authent": 47649, "osaurus": 47650, "Purchase": 47651, "Ġrelegation": 47652, "alter": 47653, "Ġ³³": 47654, "Ġriddled": 47655, "Ġogre": 47656, "ĠLowell": 47657, "Occup": 47658, "Eat": 47659, "ĠHyder": 47660, "ĠAdviser": 47661, "Commerce": 47662, "Hunt": 47663, "ĠOrth": 47664, "ĠCompetitive": 47665, "ĠCLA": 47666, "CDC": 47667, "Ġsalads": 47668, "Fle": 47669, "Ġindustrialized": 47670, "`,": 47671, "ĠOWN": 47672, "Ġbeck": 47673, "ĠParticularly": 47674, "oubt": 47675, "ĠmM": 47676, "ĠHussain": 47677, "ĠChennai": 47678, "Ġ920": 47679, "Ġappointing": 47680, "ĠCullen": 47681, ",,,,,,,,": 47682, "Ġpores": 47683, "verified": 47684, "Ġbiochemical": 47685, "emate": 47686, "Ġcowardly": 47687, "ĠHelsinki": 47688, "ĠEthiopian": 47689, "SOURCE": 47690, "ERC": 47691, "estro": 47692, "Ġbiotech": 47693, "ĠSour": 47694, "Ġbrewer": 47695, "Bloomberg": 47696, "Ġintensify": 47697, "Glass": 47698, "anco": 47699, "ĠFDR": 47700, "greSQL": 47701, "ĠFires": 47702, "©¶æ¥µ": 47703, "eco": 47704, "1001": 47705, "ĠHomeless": 47706, "Ġinstantaneous": 47707, "ĠHaste": 47708, "igel": 47709, "Diamond": 47710, "Ġpaving": 47711, "Ġlandfill": 47712, "Ġdads": 47713, "houn": 47714, ":]": 47715, "Ġincendiary": 47716, "ĠLivingston": 47717, "ĠHilbert": 47718, "ĠChecks": 47719, "styles": 47720, "inators": 47721, "ĠClive": 47722, "phrine": 47723, "Ġchimpanzees": 47724, "Ġpall": 47725, "ĠJM": 47726, "ĠAadhaar": 47727, "ðĿ": 47728, "Ġachievable": 47729, "disabled": 47730, "PET": 47731, "OOOOOOOO": 47732, "Mot": 47733, "Ġintangible": 47734, "Ġballet": 47735, "ĠWebs": 47736, "ĠEstimated": 47737, "Effects": 47738, "Ġbailed": 47739, "Joshua": 47740, "Ġturbulence": 47741, "Ġoccupant": 47742, "ĠDaylight": 47743, "Ġ361": 47744, "meet": 47745, "Ġstatically": 47746, "Ġonlook": 47747, "Ġki": 47748, "illegal": 47749, "Ġvelvet": 47750, "Ġdehydration": 47751, "Ġacquies": 47752, "ĠRez": 47753, "akura": 47754, "ĠUpton": 47755, "atro": 47756, "Ġincomprehensible": 47757, "Ġbackdoor": 47758, "ĠRhino": 47759, "727": 47760, "Ġmaths": 47761, ")+": 47762, "Ġheresy": 47763, "Ġdf": 47764, "ĠRoche": 47765, "ĠLydia": 47766, "Ġpancreat": 47767, "reply": 47768, "arrell": 47769, "Ġsolicitation": 47770, "Ġcircadian": 47771, "BIP": 47772, "Ġforay": 47773, "Ġcryptic": 47774, "izu": 47775, "imeo": 47776, "ĠTomato": 47777, "ĠHoms": 47778, "examination": 47779, "Ġquarry": 47780, "ĠValiant": 47781, "ĠJericho": 47782, "ĠINCLUD": 47783, "Ġ1840": 47784, "519": 47785, "Ġresists": 47786, "Ġsnapshots": 47787, "ĠSpur": 47788, "ĠAntiqu": 47789, "Login": 47790, "Ġbestselling": 47791, "Ġantic": 47792, "ĠSutherland": 47793, "ãĤ¢ãĥ«": 47794, "Ġ~/": 47795, "ĠParm": 47796, "èĥ": 47797, "Pages": 47798, "intensity": 47799, "Ġimmobil": 47800, "Ġ1865": 47801, "zzo": 47802, "Ġnifty": 47803, "Ġfentanyl": 47804, "ĠPreservation": 47805, "ophen": 47806, "Ġdarts": 47807, "ĠDinosaur": 47808, "pointers": 47809, "ĠRite": 47810, "suggest": 47811, "awareness": 47812, "ĠSheridan": 47813, "Ġstances": 47814, "Ġsorcery": 47815, "Ġperjury": 47816, "ĠNikola": 47817, "iever": 47818, "Ġfiance": 47819, "ĠJordanian": 47820, "ĠBalloon": 47821, "Ġnab": 47822, "Ġkb": 47823, "Ġhumanities": 47824, "ĠTanaka": 47825, "hillary": 47826, "Ġconsultancy": 47827, "ĠZub": 47828, "Ġremission": 47829, "Ġconfid": 47830, "CHQ": 47831, "ĠFug": 47832, "Ġimprovis": 47833, "Yep": 47834, "/_": 47835, "Ġunwillingness": 47836, "Ġportfolios": 47837, "055": 47838, "ĠInstructor": 47839, "aiman": 47840, "Ġclaimants": 47841, "Mbps": 47842, "ĠBye": 47843, "received": 47844, "Tweet": 47845, "Ġindemn": 47846, "riz": 47847, "amara": 47848, "Nat": 47849, "Ġevaluates": 47850, "ĠLur": 47851, "epad": 47852, "FOX": 47853, "ĠThro": 47854, "Ġrusty": 47855, "Ġbedrock": 47856, "ĠOprah": 47857, "JB": 47858, "Ġmanipulative": 47859, "Ġwillful": 47860, "Ġrelapse": 47861, "Ġextant": 47862, "Theme": 47863, "Sensor": 47864, "ĠStability": 47865, "govern": 47866, "Ġpoppy": 47867, "Ġknack": 47868, "Ġinsulated": 47869, "ĠTile": 47870, "ĠExtrem": 47871, "Ġuntold": 47872, "Ġconverge": 47873, "Ġrefuel": 47874, "igroup": 47875, "Ġdistortions": 47876, "Ġravaged": 47877, "Ġmechanically": 47878, "ĠReilly": 47879, "ĠNose": 47880, "ĠIncarnation": 47881, "ĠBecky": 47882, "abbling": 47883, "Ġtaco": 47884, "Ġrake": 47885, "Ġmelancholy": 47886, "Ġillustrious": 47887, "ĠDartmouth": 47888, "Guide": 47889, "ĠRazer": 47890, "ĠBenz": 47891, "Ultimate": 47892, "ĠSurprise": 47893, "Ġpageant": 47894, "offer": 47895, "Whoever": 47896, "Ġwiser": 47897, "Ġchemist": 47898, "ĠHELL": 47899, "ĠBulk": 47900, "Ġplutonium": 47901, "ĠCOVER": 47902, "Ö¼": 47903, "failed": 47904, "Ġtirelessly": 47905, "Ġinfertility": 47906, "ĠTrident": 47907, "ĠShowtime": 47908, "ĠCiv": 47909, "Vice": 47910, "requires": 47911, "ittance": 47912, "Ġuncontrolled": 47913, "interesting": 47914, "561": 47915, "Ġinnovate": 47916, "ategic": 47917, "Lie": 47918, "ĠSelling": 47919, "Ul": 47920, "Ġsavior": 47921, "ĠTosh": 47922, "Ġswast": 47923, "PASS": 47924, "Ġrink": 47925, "Ġcardio": 47926, "ĠIro": 47927, "udi": 47928, "Ġvantage": 47929, "Ġvans": 47930, "ĠNiño": 47931, "+=": 47932, "Ġpropagate": 47933, "": 49029, "Ġleukemia": 49030, "Ġeluc": 49031, "Ġannouncer": 49032, "ĠLithuan": 49033, "ĠArmageddon": 49034, "åĩ": 49035, "Lenin": 49036, "ĠRuk": 49037, "Ġpepp": 49038, "ĠRomantic": 49039, "ĠPIT": 49040, "ĠInterstellar": 49041, "ĠAtkinson": 49042, "Raid": 49043, "Js": 49044, "Goal": 49045, "Course": 49046, "Ġvanishing": 49047, "esley": 49048, "ĠRounds": 49049, "Elsa": 49050, "593": 49051, "Ġredundancy": 49052, "ĠSTAND": 49053, "Ġprophetic": 49054, "Ġhabitable": 49055, "ryu": 49056, "Ġfaintly": 49057, "MODE": 49058, "Ġflanked": 49059, "IRC": 49060, "Awesome": 49061, "Ġspurious": 49062, "ĠZah": 49063, "ĠMSG": 49064, "Ġshading": 49065, "Ġmotivational": 49066, "ĠSantana": 49067, "ĠSPR": 49068, "Ġexcruciating": 49069, "omial": 49070, "ĠMiko": 49071, "ĠLeopard": 49072, "Abyss": 49073, "Ġ[|": 49074, "dirty": 49075, "Ġbaths": 49076, "Ġdemoral": 49077, "andre": 49078, "PB": 49079, "Ġunification": 49080, "Ġsacrament": 49081, "Ġ[&": 49082, "Ġpriceless": 49083, "Ġgelatin": 49084, "Ġemanating": 49085, "ĠAllaah": 49086, "986": 49087, "Ġoutburst": 49088, "Ġeras": 49089, "ĠXVI": 49090, "ĠSPI": 49091, "Ott": 49092, "ĠLazarus": 49093, "PLIED": 49094, "Flying": 49095, "blogs": 49096, "Wisconsin": 49097, "Raven": 49098, "Ġrebate": 49099, "Ġcreeps": 49100, "ĠSpan": 49101, "ĠPainter": 49102, "ĠKira": 49103, "ĠAmos": 49104, "ĠCorvette": 49105, "Consumer": 49106, "ĠRecover": 49107, "cki": 49108, "Ġpesky": 49109, "ĠInvention": 49110, "Companies": 49111, "Ġchallengers": 49112, "ademic": 49113, "ĠUkrainians": 49114, "ĠNeurolog": 49115, "ĠForsaken": 49116, "Ġentrants": 49117, "Ġembattled": 49118, "Ġdefunct": 49119, "ĠGlacier": 49120, "Ġpoisons": 49121, "ĠHorses": 49122, "makes": 49123, "ĠDirt": 49124, "Ġ423": 49125, "hhh": 49126, "ĠTransformation": 49127, "QUIRE": 49128, "..................": 49129, "Ġtraveller": 49130, "ĠSexy": 49131, "ĠKern": 49132, "ipolar": 49133, "Ġransomware": 49134, "oooooooooooooooo": 49135, "Ec": 49136, "ruby": 49137, "Professional": 49138, "ĠOutbreak": 49139, "argument": 49140, "Grey": 49141, "ĠFifa": 49142, "ĠCHO": 49143, "ĠFORM": 49144, "ĠAmtrak": 49145, "-[": 49146, "Ġcradle": 49147, "Ġantioxidants": 49148, "ãģ®å®": 49149, "736": 49150, "ĠNASL": 49151, "ĠContributions": 49152, "Indiana": 49153, "ĠSTEP": 49154, "CSS": 49155, "Ġsalient": 49156, "Ġallocations": 49157, "yrights": 49158, "Ġmashed": 49159, "ĠCutter": 49160, "Sexual": 49161, "Ġpounded": 49162, "Ġfanbase": 49163, "Ġcasc": 49164, "ĠTransparency": 49165, "Ġanalytic": 49166, "ĠSummoner": 49167, "×ŀ": 49168, "ĠADC": 49169, "detail": 49170, "Ġvanquished": 49171, "Ġcrabs": 49172, "arie": 49173, "Destroy": 49174, "ĠSack": 49175, "Ġtransistor": 49176, "Alabama": 49177, "ĠKoen": 49178, "ĠFisheries": 49179, "cone": 49180, "Ġannexed": 49181, "ĠMGM": 49182, "esa": 49183, "Ġfaked": 49184, "ĠCongratulations": 49185, "Ġhindered": 49186, "Ġcorrectional": 49187, "ĠITV": 49188, "leeve": 49189, "Ġinappropriately": 49190, "licks": 49191, "Ġtrespass": 49192, "Ġpaws": 49193, "Ġnegotiator": 49194, "ĠChristensen": 49195, "limits": 49196, "ĠDianne": 49197, "Ġelegance": 49198, "ĠContracts": 49199, "anke": 49200, "Obj": 49201, "Ġvigilance": 49202, "Ġcastles": 49203, "ĠNAD": 49204, "ĠHolo": 49205, "Ġemphatically": 49206, "ĠTitus": 49207, "ĠServing": 49208, "ĠRichie": 49209, "ĠPigs": 49210, "568": 49211, "Ġanimosity": 49212, "ĠAttributes": 49213, "ĠUriel": 49214, "MQ": 49215, "myra": 49216, "ĠApplicant": 49217, "Ġpsychiatrists": 49218, "ĠVij": 49219, "ĠAbby": 49220, "agree": 49221, "Push": 49222, "ĠkWh": 49223, "hiba": 49224, "Ġincite": 49225, "ĠWeasley": 49226, "ĠTaxi": 49227, "ministic": 49228, "hyper": 49229, "ĠFarn": 49230, "Ġ601": 49231, "ĠNationwide": 49232, "Fake": 49233, "952": 49234, "Ġmaize": 49235, "Ġinteracted": 49236, "Ġtransitioned": 49237, "Ġparasitic": 49238, "Ġharmonic": 49239, "Ġdecaying": 49240, "Ġbaseless": 49241, "nsics": 49242, "Ġtranspired": 49243, "Ġabundantly": 49244, "ĠForensic": 49245, "Ġtreadmill": 49246, "ĠJav": 49247, "aband": 49248, "Ġsshd": 49249, "Ġfrontman": 49250, "ĠJakarta": 49251, "oller": 49252, "drops": 49253, "ĠSERVICES": 49254, "romptu": 49255, "ophical": 49256, "hospital": 49257, "bledon": 49258, "645": 49259, "Ġmidrange": 49260, "ĠEVENT": 49261, "culated": 49262, "rawled": 49263, "Ġperched": 49264, "Ġoverboard": 49265, "ĠPeel": 49266, "ĠPwr": 49267, "ĠCarth": 49268, "ĠCOMPLE": 49269, "coe": 49270, "shall": 49271, "Ġdeterrence": 49272, "METHOD": 49273, "ĠAbsent": 49274, "MEN": 49275, "Ġsill": 49276, "ĠLEVEL": 49277, "York": 49278, "Ġsinners": 49279, "ĠOPEC": 49280, "ĠNur": 49281, "ĠDesigns": 49282, "selection": 49283, "Ġunworthy": 49284, "CHA": 49285, "Ġstrengthens": 49286, "883": 49287, "edly": 49288, "Ġslicing": 49289, "Ġmalnutrition": 49290, "Ġfilmmaking": 49291, "ĠPolk": 49292, "urated": 49293, "Ġ421": 49294, "breakers": 49295, "!'\"": 49296, "Ġwetlands": 49297, "ĠDiscrimination": 49298, "Ġallowable": 49299, "Ġsteered": 49300, "ĠSicily": 49301, "SAM": 49302, "Ġmustache": 49303, "Ġmids": 49304, "Ġclipped": 49305, "Ġcirculate": 49306, "Ġbrittle": 49307, "ĠBuildings": 49308, "raised": 49309, "ĠRoundup": 49310, "Ġwealthier": 49311, "Ġoverwrite": 49312, "Ġoverpowered": 49313, "ĠGerrard": 49314, "sites": 49315, "PDATED": 49316, "Ġacutely": 49317, "ĠGamble": 49318, "Ġpim": 49319, "ĠKus": 49320, "Typically": 49321, "Deploy": 49322, "ĠMoroccan": 49323, "potion": 49324, "combe": 49325, "Ġvigilante": 49326, "Ġ363": 49327, "Stew": 49328, "ĠBagg": 49329, "Ġresided": 49330, "ĠSpo": 49331, "Ġremnant": 49332, "Ġemptiness": 49333, "brainer": 49334, "Ġoutpatient": 49335, "priority": 49336, "Ġleptin": 49337, "ĠPayton": 49338, "ĠGleaming": 49339, "ĠShed": 49340, "ĠPolo": 49341, "ĠMormonism": 49342, "restricted": 49343, "arlane": 49344, "wx": 49345, "Ġcreatine": 49346, "ĠAnon": 49347, "ĠSTUD": 49348, "ĠJUL": 49349, "ĠTee": 49350, "528": 49351, "089": 49352, "Ġhatched": 49353, "Dispatch": 49354, "ĠComposite": 49355, "Ġ451": 49356, "puff": 49357, "ĠXCOM": 49358, "ĠOrn": 49359, "ĠTHANK": 49360, "ENDED": 49361, "ĠAsheville": 49362, "ĠÃľ": 49363, "Ġmango": 49364, "ĠSlightly": 49365, "worldly": 49366, "ĠWander": 49367, "ĠExpand": 49368, "ĠChr": 49369, "Mist": 49370, "Ġorthodoxy": 49371, "ĠUNESCO": 49372, "regate": 49373, "Elsewhere": 49374, "kie": 49375, "irled": 49376, "Ġtopple": 49377, "Ġadoptive": 49378, "ĠLegs": 49379, "dress": 49380, "ĠSagan": 49381, "bare": 49382, "ĠGlou": 49383, "Crunch": 49384, "Ġhelpers": 49385, "Ġchronically": 49386, "ĠHuma": 49387, "10000": 49388, "Ġaccommodating": 49389, "äºĶ": 49390, "Ġwrinkles": 49391, "Ġdodged": 49392, "fourth": 49393, "Ġprecon": 49394, "Ġcompressor": 49395, "ĠKare": 49396, "Ġevict": 49397, "ĠWarwick": 49398, "imar": 49399, "Ġmodernization": 49400, "Ġbandwagon": 49401, "Ġrefuted": 49402, "Ġnetted": 49403, "ĠNaples": 49404, "ĠGenie": 49405, "perors": 49406, "Ġfielded": 49407, "Ġdere": 49408, "ĠParables": 49409, "lees": 49410, "Ġtrout": 49411, "aspers": 49412, "Ġnihil": 49413, "Ġhappiest": 49414, "Ġfloppy": 49415, "ĠLoft": 49416, "ĠHeard": 49417, "Ġunison": 49418, "Ġlug": 49419, "ĠRedmond": 49420, "classic": 49421, "Supporters": 49422, "SHIP": 49423, "GMT": 49424, "Ġfuelled": 49425, "çIJ": 49426, "Ġdd": 49427, "ĠEminem": 49428, "Ġ1897": 49429, "NYSE": 49430, "Ġsecretaries": 49431, "ĠFIA": 49432, "ĠCanaveral": 49433, "Favorite": 49434, "Ġpomp": 49435, "Ġdetainee": 49436, "ership": 49437, "aimon": 49438, "iour": 49439, "ĠApex": 49440, "Ġplantations": 49441, "amia": 49442, "acion": 49443, "Rust": 49444, "Ġtowed": 49445, "ĠTruly": 49446, "577": 49447, "Ġsheltered": 49448, "rider": 49449, "Wo": 49450, "Ġlair": 49451, "ĠIntelligent": 49452, "improve": 49453, "matically": 49454, "Ġetiquette": 49455, "adra": 49456, "allo": 49457, "ĠJuno": 49458, "anything": 49459, "ĠStruggle": 49460, "ĠPredict": 49461, "ĠGrimes": 49462, "ĠAMERICA": 49463, "ctx": 49464, "ĠSituation": 49465, "WOOD": 49466, "Ġsoluble": 49467, "meier": 49468, "Ġintolerable": 49469, "angering": 49470, "Ġuninterrupted": 49471, "Ġtooltip": 49472, "Ġinterrogated": 49473, "Ġgunned": 49474, "ĠSneak": 49475, "æѦ": 49476, "Ġtether": 49477, "Ġcrumble": 49478, "Lens": 49479, "Ġclustered": 49480, "ĠSyl": 49481, "ĠHasan": 49482, "Ġdystopian": 49483, "wana": 49484, "Ġjoystick": 49485, "ĠThib": 49486, "ammu": 49487, "Tomorrow": 49488, "546": 49489, "Ġovercame": 49490, "Ġminimized": 49491, "ceptor": 49492, "Runner": 49493, "ENGTH": 49494, "ĠBrenda": 49495, "ĠAchievements": 49496, "Ġtorches": 49497, "Ġrapport": 49498, "ĠInvestigator": 49499, "ĠHandling": 49500, "relation": 49501, "grey": 49502, "815": 49503, "Ġkcal": 49504, "ĠCommands": 49505, "dq": 49506, "Ġcurls": 49507, "Ġbearer": 49508, "Ġcynicism": 49509, "itri": 49510, "ĠUseful": 49511, "Bee": 49512, "DCS": 49513, "Ġabras": 49514, "Pract": 49515, "BILITIES": 49516, "712": 49517, "Ġdebugger": 49518, "Ġdebtor": 49519, "ĠLia": 49520, "ĠKers": 49521, "Ġexacerbate": 49522, "ĠStacy": 49523, "ĠBland": 49524, "ĠScenes": 49525, "Ġbranching": 49526, "âĸĪâĸĪâĸĪâĸĪâĸĪâĸĪâĸĪâĸĪ": 49527, "apeake": 49528, "Ġsalsa": 49529, "Ġmishand": 49530, "ĠKonami": 49531, "ĠNib": 49532, "Ġanecdote": 49533, "Ġagreeable": 49534, "Ïī": 49535, "ĠNathaniel": 49536, "ĠHeisman": 49537, "ĠBeware": 49538, "Ġ1886": 49539, "spective": 49540, "691": 49541, "522": 49542, "Ġinhibits": 49543, "Ġhashing": 49544, "Ġ1889": 49545, "å°Ĩ": 49546, "vich": 49547, "Pure": 49548, "Ġsolidly": 49549, "Ġaspirin": 49550, "imaru": 49551, "Ġstreetcar": 49552, "ĠUCS": 49553, "ĠJudd": 49554, "Ġflashbacks": 49555, "pins": 49556, "Ġ1440": 49557, "ĠUNHCR": 49558, "ĠSymptoms": 49559, "TIT": 49560, "538": 49561, "Fra": 49562, "%);": 49563, "Ġooz": 49564, "Ġcurfew": 49565, "Ġcalmed": 49566, "Ġparticipates": 49567, "TeX": 49568, "Ġnonsensical": 49569, "Ġfullback": 49570, "ĠDeL": 49571, "monkey": 49572, "hari": 49573, "Ġmetabolites": 49574, "Ġlooted": 49575, "ĠALWAYS": 49576, "ĠBCC": 49577, "Lt": 49578, "ochet": 49579, "Bone": 49580, "Ġvetoed": 49581, "Ġgcc": 49582, "ĠCLICK": 49583, "Ġ1888": 49584, "saf": 49585, "Ġstiffness": 49586, "Ġlowly": 49587, "ĠGeh": 49588, "verson": 49589, "orset": 49590, "Ġunforeseen": 49591, "Ġanesthesia": 49592, "ĠOptical": 49593, "Ġreconstructed": 49594, "ĠTup": 49595, "shows": 49596, "NEWS": 49597, "ĠNewspaper": 49598, "ĠASA": 49599, "tera": 49600, "Numbers": 49601, "Ġinexplicable": 49602, "×ij": 49603, "Ġhardness": 49604, "untarily": 49605, "ĠAcer": 49606, "gradient": 49607, "ARDIS": 49608, "Ġwoodland": 49609, "Ġmetaphors": 49610, "ĠWembley": 49611, "ĠPavel": 49612, "philis": 49613, "Ġrewriting": 49614, "Ġperceptual": 49615, "Ġ1070": 49616, "worms": 49617, "ĠDowns": 49618, "Ġunsurprisingly": 49619, "Ġtagging": 49620, "flame": 49621, "Ġlitres": 49622, "Ġbounces": 49623, "ĠBabe": 49624, "shut": 49625, "Ġoverdoses": 49626, "ĠSheila": 49627, "ĠChau": 49628, "ĠBless": 49629, "Capture": 49630, "ĠSignificant": 49631, "ĠScion": 49632, "Ġ389": 49633, "ĠMcH": 49634, "ĠTitanium": 49635, "ĠMeal": 49636, "ameda": 49637, "agents": 49638, "aggressive": 49639, "Billy": 49640, "763": 49641, "ĠSaying": 49642, "DERR": 49643, "itone": 49644, "Collins": 49645, "Bound": 49646, "Ġbolted": 49647, "ĠDMCA": 49648, "953": 49649, "Ġuniqueness": 49650, "Ġepigen": 49651, "unci": 49652, "antam": 49653, "Ġreckoning": 49654, "chairs": 49655, "OGR": 49656, "ĠSenegal": 49657, "Ġ1862": 49658, "relevant": 49659, "Ġ¯": 49660, "Ġpharmacies": 49661, "ĠGeral": 49662, "vier": 49663, "Yan": 49664, "ORPG": 49665, "Ġrabid": 49666, "bending": 49667, "ĠUNITED": 49668, "Ġ465": 49669, "Assembly": 49670, "Ġweep": 49671, "Ġbehest": 49672, "ĠMothers": 49673, "ĠJace": 49674, "hid": 49675, "Ġwhirlwind": 49676, "ĠUNIVERS": 49677, "Ġutopian": 49678, "Ġkidnap": 49679, "Philipp": 49680, "Kin": 49681, "893": 49682, "Ġlivestream": 49683, "ĠMISS": 49684, "Ġsubversive": 49685, "ĠTechniques": 49686, "ĠJUSTICE": 49687, "ĠBASE": 49688, "Ġ387": 49689, "Ġassailants": 49690, "ĠHardcore": 49691, "Ġsprinkled": 49692, "ĠPse": 49693, "éļ": 49694, "printed": 49695, "ĠHau": 49696, "ORGE": 49697, "ĠTOUR": 49698, "Ġlaced": 49699, "Ġitch": 49700, "Giving": 49701, "Ġported": 49702, "781": 49703, "////////////////////////////////": 49704, "breeding": 49705, "Ġlogger": 49706, "ĠHOL": 49707, "innie": 49708, "Firstly": 49709, "Ġembryonic": 49710, "Ġdelegated": 49711, "pai": 49712, "OIL": 49713, "Ġcentrally": 49714, "ĠRx": 49715, "ĠScouting": 49716, "Dutch": 49717, "Ġhereditary": 49718, "ĠCruiser": 49719, "sat": 49720, "529": 49721, "ĠMarriott": 49722, "othermal": 49723, "Ġprohibitions": 49724, "Earn": 49725, "ĠStab": 49726, "ĠColleges": 49727, "ĠBelief": 49728, "stretched": 49729, "ĠLH": 49730, "ĠEntityItem": 49731, "CIA": 49732, "Ġunrem": 49733, "Ġlaureate": 49734, "Ġdenominations": 49735, "summary": 49736, "hler": 49737, "Spect": 49738, "ĠKlaus": 49739, "ĠBeans": 49740, "Ġinsur": 49741, "ĠPAX": 49742, "Ġfielder": 49743, "ĠVet": 49744, "ĠSparrow": 49745, "zie": 49746, "ĠSQ": 49747, "ĠMondays": 49748, "ĠOffline": 49749, "ĠLerner": 49750, "ĠExtensions": 49751, "Ireland": 49752, "Ġpatronage": 49753, "Ġcontrasted": 49754, "ĠMania": 49755, "hirt": 49756, "Moscow": 49757, "Ġcondemns": 49758, "ĠAnge": 49759, "Ġcomposing": 49760, "ĠPepe": 49761, "ĠPaddock": 49762, "Ġheterogeneity": 49763, "Ġideologically": 49764, "Ġfishes": 49765, "Ġcursing": 49766, "ĠRutherford": 49767, "ĠFloating": 49768, "ĠAmelia": 49769, "Tea": 49770, "Synopsis": 49771, "Ġstunts": 49772, "Ġbead": 49773, "Ġstocking": 49774, "ĠMILL": 49775, "obook": 49776, "massive": 49777, "\\<": 49778, "Ġhump": 49779, "ĠPreferences": 49780, "EngineDebug": 49781, "geist": 49782, "ĠNieto": 49783, "omever": 49784, "ishy": 49785, "evaluate": 49786, "colonial": 49787, "Alternative": 49788, "ĠGoPro": 49789, "ĠVortex": 49790, "ĠNETWORK": 49791, "ansky": 49792, "Secure": 49793, "ĠThrust": 49794, "Snake": 49795, "Ġparcels": 49796, "Ġsamurai": 49797, "Ġactresses": 49798, "Nap": 49799, "MF": 49800, "iferation": 49801, "Beer": 49802, "523": 49803, "ĠIly": 49804, "ointment": 49805, "Ping": 49806, "Ġstriped": 49807, "ĠMellon": 49808, "ossession": 49809, "Ġneutron": 49810, "endium": 49811, "Ġaph": 49812, "ĠFlavoring": 49813, "Ġ383": 49814, "Ġresponsiveness": 49815, "ĠJindal": 49816, "ĠHitchcock": 49817, "Denver": 49818, "ĠDRAGON": 49819, "smanship": 49820, "ĠDupl": 49821, "Ġsly": 49822, "Ġwebcam": 49823, "ĠTwain": 49824, "ĠDarling": 49825, "iliate": 49826, "consumer": 49827, "DIT": 49828, "Ġnamesake": 49829, "Ġunorthodox": 49830, "Ġfuner": 49831, "ĠPLoS": 49832, "ĠCONTROL": 49833, "ozyg": 49834, "oglobin": 49835, "FACE": 49836, "ERG": 49837, "ĠDia": 49838, "ĠFiesta": 49839, "cele": 49840, "034": 49841, "Ġenclave": 49842, "âĸ¬âĸ¬": 49843, "onement": 49844, "alist": 49845, "Mand": 49846, "Ġhomegrown": 49847, "ĠFancy": 49848, "Ġconceptions": 49849, "ĠContains": 49850, "ureen": 49851, "Ġreiterate": 49852, "Ġmeager": 49853, "Ġinstallments": 49854, "Spawn": 49855, "627": 49856, "Ġphotoc": 49857, "ĠCabrera": 49858, "ĠRosenthal": 49859, "ĠLansing": 49860, "isner": 49861, "Ġinvests": 49862, "ĠUFOs": 49863, "EXP": 49864, "Hardware": 49865, "Ġtragically": 49866, "Ġconcedes": 49867, "ieft": 49868, "cham": 49869, "borgh": 49870, "ĠSchr": 49871, "ĠMelanie": 49872, "ĠHoy": 49873, "Ġvisitation": 49874, "Ġidiosyncr": 49875, "Ġfractions": 49876, "Ġforeskin": 49877, "obos": 49878, "Ġpoaching": 49879, "ĠVIEW": 49880, "Ġstimulates": 49881, "ĠGork": 49882, "canon": 49883, "MIC": 49884, "ĠNemesis": 49885, "ĠIndra": 49886, "ĠDMV": 49887, "Ġ529": 49888, "Ġinspecting": 49889, "Ġgrandma": 49890, "ĠWhedon": 49891, "ĠShant": 49892, "ĠPurg": 49893, "ikan": 49894, "ĠTeg": 49895, "ĠCLR": 49896, "zac": 49897, "Victoria": 49898, "ĠVerify": 49899, "ionics": 49900, "Ġpartying": 49901, "ĠMou": 49902, "colour": 49903, "Ġtestimonies": 49904, "lations": 49905, "Ġpressuring": 49906, "hiro": 49907, "acers": 49908, "Ġfid": 49909, "angler": 49910, "ĠCSI": 49911, "Ġhereafter": 49912, "Ġdissidents": 49913, "reporting": 49914, "iphany": 49915, "chev": 49916, "Ġsolitude": 49917, "Ġlobe": 49918, "Ġindis": 49919, "Ġcredential": 49920, "recent": 49921, "adult": 49922, "ĠNirvana": 49923, "ĠFranchise": 49924, "Layer": 49925, "Hyp": 49926, "ĠBerkshire": 49927, "Ġwills": 49928, "tif": 49929, "Ġtotem": 49930, "ĠJudah": 49931, "repair": 49932, "Instant": 49933, "548": 49934, "Ġembassies": 49935, "Ġbottleneck": 49936, "Ġbount": 49937, "Ġtypew": 49938, "ĠAlvin": 49939, "jing": 49940, "imilar": 49941, "Rush": 49942, "Ġbrim": 49943, "ĠHELP": 49944, "Aim": 49945, "]'": 49946, "Ġpassively": 49947, "Ġbounded": 49948, "ĠRated": 49949, "Ġcriminality": 49950, "Ġbiomark": 49951, "Ġdispatcher": 49952, "ĠTowards": 49953, "Ġ+++": 49954, "righteous": 49955, "frog": 49956, "ĠPanc": 49957, "Carter": 49958, "032": 49959, "æ©Ł": 49960, "Ġultraviolet": 49961, "ĠLicensed": 49962, "ĠTata": 49963, "ĠBlessing": 49964, "ĠGAM": 49965, "Ġchemically": 49966, "ĠSeaf": 49967, "ĠRELE": 49968, "ĠMercenary": 49969, "capitalist": 49970, "Ġformulations": 49971, "Ġannihilation": 49972, "ĠVerb": 49973, "ĠArgon": 49974, "Ġunloaded": 49975, "Ġmorphed": 49976, "Ġconquering": 49977, "backer": 49978, "IELD": 49979, "Ġthefts": 49980, "Ġfrontrunner": 49981, "ĠRoyale": 49982, "ĠFundamental": 49983, "elight": 49984, "Chip": 49985, "necessary": 49986, "ayn": 49987, "ĠSlip": 49988, "Ġ448": 49989, "cerned": 49990, "Pause": 49991, "Ġshockingly": 49992, "ĠABV": 49993, "Ġcomposure": 49994, "733": 49995, "ĠMotorsport": 49996, "ahime": 49997, "Murray": 49998, "Mach": 49999, "Ġgrids": 50000, "Ġdebian": 50001, "Ġfurthermore": 50002, "Ġdexterity": 50003, "ĠCollections": 50004, "oslov": 50005, "ilage": 50006, "bj": 50007, "ĠMonteneg": 50008, "ĠstrutConnector": 50009, "Ġmassacres": 50010, "Ġbriefs": 50011, "fetched": 50012, "uvian": 50013, "olition": 50014, "Failure": 50015, "emonic": 50016, "Ġflared": 50017, "Ġclaimant": 50018, "Ġcures": 50019, "Ġgiveaways": 50020, "ĠSubstance": 50021, "alions": 50022, "Ġcringe": 50023, "ĠKul": 50024, "Ġaristocracy": 50025, "ĠUlster": 50026, "olated": 50027, "housing": 50028, "ĠMIS": 50029, "Ġglared": 50030, "ĠWilhelm": 50031, "needs": 50032, "lambda": 50033, "builders": 50034, "ĠVIS": 50035, "Ġradiator": 50036, "ĠGhostbusters": 50037, "Ġ436": 50038, "actual": 50039, "Ġherds": 50040, "ça": 50041, "watching": 50042, "Ġcountering": 50043, "Charge": 50044, "Ġcharred": 50045, "Ġwarheads": 50046, "Ġiodine": 50047, "ĠMacy": 50048, "041": 50049, "Ġdepartures": 50050, "ĠSins": 50051, "Ġdyed": 50052, "ĠConcepts": 50053, "gado": 50054, "713": 50055, "Ġquotations": 50056, "Ġgist": 50057, "ĠChristy": 50058, "Ġantigen": 50059, "ĠHemp": 50060, "ĠDrawn": 50061, "ĠBarg": 50062, "ezvous": 50063, "Ġpaternity": 50064, "Ġardu": 50065, "ĠAnchorage": 50066, "ĠRik": 50067, "Ġoverloaded": 50068, "ĠUsername": 50069, "ĠTammy": 50070, "ĠNau": 50071, "ĠCellular": 50072, "Ġwaning": 50073, "Ġrodent": 50074, "ĠWorcester": 50075, "ilts": 50076, "ĠTad": 50077, "Ġdwellings": 50078, "Ġbullish": 50079, "431": 50080, "Ġretaliate": 50081, "Ġmigraine": 50082, "ĠChevron": 50083, "CHECK": 50084, "Ġdonkey": 50085, "crim": 50086, "SPA": 50087, "ĠAnalog": 50088, "Ġmarquee": 50089, "ĠHaas": 50090, "Bir": 50091, "ĠGDDR": 50092, "ĠDownloads": 50093, "Ġwillpower": 50094, "ĠForth": 50095, "ĠRecorded": 50096, "Ġimpossibility": 50097, "ĠLogged": 50098, "ĠFranks": 50099, "ĠRatt": 50100, "initions": 50101, "Ġcleaners": 50102, "Ġsorely": 50103, "Ġflickering": 50104, "ĠExamination": 50105, "catching": 50106, "alloween": 50107, "Msg": 50108, "Ġdunno": 50109, "Fa": 50110, "Ġdysph": 50111, "crazy": 50112, ".''.": 50113, "Ġmainline": 50114, "Ġcs": 50115, "Ġptr": 50116, "ĠWally": 50117, "igun": 50118, "951": 50119, "ĠBigfoot": 50120, "fights": 50121, "Ġretrieving": 50122, "Jr": 50123, "Ġduplication": 50124, "ĠExplan": 50125, "Ġrelational": 50126, "Ġquaint": 50127, "Ġbiscuits": 50128, "Ġado": 50129, "Ġshudder": 50130, "Ġantidote": 50131, "blooded": 50132, "ksh": 50133, "Ġsauces": 50134, "Ġreinvest": 50135, "Ġdispensary": 50136, "ĠDiver": 50137, "Ġ9000": 50138, "student": 50139, "Ġinsepar": 50140, "escap": 50141, "Ġtoddlers": 50142, "ĠGPIO": 50143, "ĠAssignment": 50144, "headers": 50145, "Ġlackluster": 50146, "Ġaback": 50147, "956": 50148, "Ġtoolbar": 50149, "745": 50150, "Ġoust": 50151, "Ġcontemplation": 50152, "ĠPRESIDENT": 50153, "Ġ458": 50154, "======": 50155, "Ġguaranteeing": 50156, "ĠHeist": 50157, "ĠCannes": 50158, "Ļ½": 50159, "Ġcollaborator": 50160, "ĠAmp": 50161, "Ġgou": 50162, "ĠSHALL": 50163, "stories": 50164, "783": 50165, "Ġmobilized": 50166, "Ġbrood": 50167, "ĠLU": 50168, "ĠðŁij": 50169, "Ġrefin": 50170, "ĠAnthropology": 50171, "vind": 50172, "illi": 50173, "Ġwarranties": 50174, "ĠBabel": 50175, "Ġswath": 50176, "Ġcaches": 50177, "Ġantagonists": 50178, "artifacts": 50179, "Ġhotly": 50180, "ĠStarts": 50181, "ĠGö": 50182, "zag": 50183, "!!!!!": 50184, "Ġscourge": 50185, "Ġconspiring": 50186, "ruits": 50187, "reverse": 50188, "ĠSheen": 50189, "ĠJesuit": 50190, "ĠGiovanni": 50191, "adies": 50192, "Ġbuttocks": 50193, "earcher": 50194, "acan": 50195, "Ġvolleyball": 50196, "Ġshrouded": 50197, "Ġscoreboard": 50198, "bats": 50199, "ĠIPM": 50200, "Ġasses": 50201, "Ġderegulation": 50202, "ĠTelegram": 50203, "ĠReboot": 50204, "Ġ7000": 50205, "ĠCanary": 50206, "Ġkernels": 50207, "ĠFrançois": 50208, "ĠDuff": 50209, "ĠPon": 50210, "ĠLeica": 50211, "ĠGarmin": 50212, "Ġorphans": 50213, "ĠClaudia": 50214, "Ġcalendars": 50215, "ĠLeilan": 50216, "ento": 50217, "Rocket": 50218, "Ġbrunch": 50219, "ĠHawking": 50220, "ainers": 50221, "Ġsensibilities": 50222, "ĠkW": 50223, "ĠKand": 50224, "Ġreclaimed": 50225, "Ġinterestingly": 50226, "ש": 50227, "romy": 50228, "JM": 50229, "ĠEnhancement": 50230, "bush": 50231, "Skip": 50232, "Ġrappers": 50233, "Ġgazing": 50234, "pedia": 50235, "athlon": 50236, "Revolution": 50237, "Ġsnipers": 50238, "Ġreverted": 50239, "Ġconglomerate": 50240, "Terry": 50241, "794": 50242, "Ġharsher": 50243, "Ġdesolate": 50244, "ĠHitman": 50245, "Commission": 50246, "Ġ(/": 50247, "âĢ¦.\"": 50248, "Compar": 50249, "Ġamplification": 50250, "ominated": 50251, "Ġregress": 50252, "ĠCollider": 50253, "Ġinformants": 50254, "Ġgazed": 50255, "<|endoftext|>": 50256} \ No newline at end of file diff --git a/experiments/elasticdnn/vilt/offline/fm_lora/vqa/vqa.py b/experiments/elasticdnn/vilt/offline/fm_lora/vqa/vqa.py new file mode 100644 index 0000000000000000000000000000000000000000..bb65342ee7c76cf9673be82065bebdfef713dc20 --- /dev/null +++ b/experiments/elasticdnn/vilt/offline/fm_lora/vqa/vqa.py @@ -0,0 +1,116 @@ +import torch +from methods.elasticdnn.api.model import ElasticDNN_OfflineVQAFMModel, ElasticDNN_OfflineVQAMDModel +from methods.elasticdnn.api.algs.fm_lora import ElasticDNN_FMLoRAAlg +from methods.elasticdnn.model.base import ElasticDNNUtil +from methods.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from methods.elasticdnn.pipeline.offline.fm_lora.vilt import FMLoRA_Vilt_Util +from methods.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from methods.elasticdnn.pipeline.offline.fm_to_md.vit import FM_to_MD_ViT_Util +from methods.elasticdnn.model.vit import ElasticViTUtil +from utils.dl.common.model import LayerActivation, get_module, get_parameter, set_module +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +from utils.common.log import logger +import torch.nn.functional as F +import sys + + +class ElasticDNN_Vilt_OfflineVQAFMModel(ElasticDNN_OfflineVQAFMModel): + def generate_md_by_reducing_width(self, reducing_width_ratio, samples: torch.Tensor): + # return FM_to_MD_ViT_Util().init_md_from_fm_by_reducing_width_with_perf_test(self.models_dict['main'], + # reducing_width_ratio, samples) + raise NotImplementedError + + def get_feature_hook(self) -> LayerActivation: + return LayerActivation(get_module(self.models_dict['main'], 'classifier.3'), True, self.device) + + def get_elastic_dnn_util(self) -> ElasticDNNUtil: + raise NotImplementedError + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + self.to_train_mode() + # print(x['input_ids'].size(), x['pixel_values'].size(), ) + o = self.infer(x).logits + + # print(o.size(), y.size(), o, y) + + return F.binary_cross_entropy_with_logits(o, y) * y.shape[1] + + def get_lora_util(self) -> FMLoRA_Util: + return FMLoRA_Vilt_Util() + + def get_task_head_params(self): + head = get_module(self.models_dict['main'], 'classifier') + params_name = {k for k, v in head.named_parameters()} + logger.info(f'task head params: {params_name}') + return list(head.parameters()) + + +class ElasticDNN_Vilt_OfflineVQAMDModel(ElasticDNN_OfflineVQAMDModel): + def get_feature_hook(self) -> LayerActivation: + return LayerActivation(get_module(self.models_dict['main'], 'classifier.3'), True, self.device) + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + self.to_train_mode() + o = self.infer(x) + return nn.functional.binary_cross_entropy_with_logits(o, y) * y.shape[1] + + +if __name__ == '__main__': + from utils.dl.common.env import set_random_seed + set_random_seed(1) + + scenario = build_scenario( + source_datasets_name=['VQAv2_split1'], + target_datasets_order=['VQAv2_split1_c'] * 1, # TODO + da_mode='close_set', + data_dirs={ + 'VQAv2_split1': '/data/zql/datasets/vqav2', + 'VQAv2_split1_c': '/data/zql/datasets/vqav2' + }, + ) + + # 2. init model + device = 'cuda' + from dnns.vilt import vilt_b_32 + model = vilt_b_32(num_classes=scenario.num_classes) + + fm_models_dict_path = save_models_dict_for_init({ + 'main': model + }, __file__, 'fm_vilt') + + fm_model = ElasticDNN_Vilt_OfflineVQAFMModel('fm', fm_models_dict_path, device) + + # 3. init alg + models = { + 'fm': fm_model + } + fm_lora_alg = ElasticDNN_FMLoRAAlg(models, get_res_save_dir(__file__, sys.argv[0])) + + + sample_dataset = list(scenario.get_offline_datasets().values())[0]['train'] + sample = sample_dataset[0][0] + + for k, v in sample.items(): + sample[k] = v.unsqueeze(0) + + # 4. run alg + from utils.dl.common.lr_scheduler import get_linear_schedule_with_warmup + fm_lora_alg.run(scenario, hyps={ + 'launch_tbboard': False, + + 'samples_size': sample, + + 'ab_r': 8, + 'train_batch_size': 64, + 'val_batch_size': 512, + 'num_workers': 16, + 'optimizer': 'AdamW', + 'optimizer_args': {'lr': 1e-4, 'betas': [0.9, 0.999]}, + 'scheduler': 'LambdaLR', + 'scheduler_args': {'lr_lambda': get_linear_schedule_with_warmup(10000, 310000)}, + 'num_iters': 320000, + 'val_freq': 400, + + # 'fm_lora_ckpt_path': 'experiments/elasticdnn/vit_b_16/offline/fm_lora/cls/results/cls.py/20230607/999995-234355-TokenClsial/models/fm_best.pt' + }) \ No newline at end of file diff --git a/experiments/elasticdnn/vilt/offline/fm_to_md/vqa/vqa_w_fbs_index.py b/experiments/elasticdnn/vilt/offline/fm_to_md/vqa/vqa_w_fbs_index.py new file mode 100644 index 0000000000000000000000000000000000000000..0c20fd8fca1bebeba4fa346a7bf3b06c2d17dfcf --- /dev/null +++ b/experiments/elasticdnn/vilt/offline/fm_to_md/vqa/vqa_w_fbs_index.py @@ -0,0 +1,192 @@ +import torch +import sys +from torch import nn +from methods.elasticdnn.api.model import ElasticDNN_OfflineVQAFMModel, ElasticDNN_OfflineVQAMDModel +from methods.elasticdnn.api.algs.md_pretraining_wo_fbs import ElasticDNN_MDPretrainingWoFBSAlg +from methods.elasticdnn.model.base import ElasticDNNUtil +from methods.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from methods.elasticdnn.pipeline.offline.fm_to_md.vilt import FM_to_MD_Vilt_Util +from methods.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from methods.elasticdnn.pipeline.offline.fm_lora.vilt import FMLoRA_Vilt_Util +from methods.elasticdnn.model.vilt import ElasticViltUtil +from utils.dl.common.model import LayerActivation2, get_module, get_parameter +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +from utils.dl.common.loss import CrossEntropyLossSoft +import torch.nn.functional as F +from utils.common.log import logger + + +class ElasticDNN_Vilt_OfflineVQAFMModel(ElasticDNN_OfflineVQAFMModel): + def generate_md_by_reducing_width(self, reducing_width_ratio, samples: torch.Tensor): # TODO: + return FM_to_MD_Vilt_Util().init_md_from_fm_by_reducing_width_with_perf_test(self.models_dict['main'], + reducing_width_ratio, samples) + # raise NotImplementedError + + def get_feature_hook(self) -> LayerActivation2: + return LayerActivation2(get_module(self.models_dict['main'], 'classifier.3')) + + def get_elastic_dnn_util(self) -> ElasticDNNUtil: # TODO: + return ElasticViltUtil() + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + self.to_train_mode() + # print(x['input_ids'].size(), x['pixel_values'].size(), ) + o = self.infer(x).logits + + # print(o.size(), y.size(), o, y) + + return F.binary_cross_entropy_with_logits(o, y) * y.shape[1] + + def get_lora_util(self) -> FMLoRA_Util: + return FMLoRA_Vilt_Util() + + def get_task_head_params(self): + head = get_module(self.models_dict['main'], 'classifier') + params_name = {k for k, v in head.named_parameters()} + logger.info(f'task head params: {params_name}') + return list(head.parameters()) + + +class ElasticDNN_Vilt_OfflineVQAMDModel(ElasticDNN_OfflineVQAMDModel): + # def __init__(self, name: str, models_dict_path: str, device: str): + # super().__init__(name, models_dict_path, device) + + # self.distill_criterion = CrossEntropyLossSoft() + + def get_feature_hook(self) -> LayerActivation2: + return LayerActivation2(get_module(self.models_dict['main'], 'classifier.3')) + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + self.to_train_mode() + # print(x['input_ids'].size(), x['pixel_values'].size(), ) + o = self.infer(x).logits + + # print(o.size(), y.size(), o, y) + + return F.binary_cross_entropy_with_logits(o, y) * y.shape[1] + + def get_distill_loss(self, student_output, teacher_output): + # print(student_output, teacher_output) + return F.mse_loss(student_output, teacher_output.detach()) + + def get_matched_param_of_fm(self, self_param_name, fm: nn.Module): # TODO: + if any([k in self_param_name for k in ['fbs', 'ab', 'embeddings']]): + return None + + p = get_parameter(self.models_dict['main'], self_param_name) + if p.dim() == 0: + return None + + elif p.dim() == 1 and 'layernorm' in self_param_name.lower() and 'weight' in self_param_name: + # if self_param_name.startswith('norm'): + # return None + return get_parameter(fm, self_param_name) + + # 1. xx.query.weight -> xx.query.fc.weight and xx.query.ab.0/1 + if ('query' in self_param_name or 'key' in self_param_name or \ + 'value' in self_param_name) and ('weight' in self_param_name): + ss = self_param_name.split('.') + # raise NotImplementedError() # TODO: + fm_qkv_name = '.'.join(ss[0: -1]) + '.fc' + fm_qkv = get_module(fm, fm_qkv_name) + + fm_abs_name = '.'.join(ss[0: -1]) + '.ab' + fm_abs = get_module(fm, fm_abs_name) + + return torch.cat([ + fm_qkv.weight.data, # task-agnositc params + fm_abs[1].weight @ fm_abs[0].weight + ], dim=0) + + elif ('query' in self_param_name or 'key' in self_param_name or \ + 'value' in self_param_name) and ('bias' in self_param_name): + ss = self_param_name.split('.') + + fm_qkv_name = '.'.join(ss[0: -1]) + '.fc.bias' + return get_parameter(fm, fm_qkv_name) + + elif 'intermediate.dense' in self_param_name: + fm_param_name = self_param_name.replace('.linear', '') + return get_parameter(fm, fm_param_name) + + else: + return get_parameter(fm, self_param_name) + + + +if __name__ == '__main__': + from utils.dl.common.env import set_random_seed + set_random_seed(1) + + # 3. init scenario + scenario = build_scenario( + source_datasets_name=['VQAv2_split1'], + target_datasets_order=['VQAv2_split1_c'] * 1, # TODO + da_mode='close_set', + data_dirs={ + 'VQAv2_split1': '/data/zql/datasets/vqav2', + 'VQAv2_split1_c': '/data/zql/datasets/vqav2' + }, + ) + + # 1. init model + # from dnns.deeplabv3.head import modify_forward_head + # modify_forward_head() # TODO: bring a bug + # from dnns.vit import vit_b_16 + fm_models_dict_path = 'experiments/elasticdnn/vilt/offline/fm_to_md/vqa/results/vqa_wo_fbs.py/20230730/999994-204649-trial/models/fm_best.pt' + fm_models = torch.load(fm_models_dict_path) + fm_models_dict_path = save_models_dict_for_init(fm_models, __file__, 'fm_vilt_vqa_lora') + md_models_dict_path = save_models_dict_for_init( + torch.load('experiments/elasticdnn/vilt/offline/fm_to_md/vqa/results/vqa_wo_fbs.py/20230730/999994-204649-trial/models/md_best.pt'), + __file__, 'md_vilt_vqa_raw_pretrained') + device = 'cuda' + + fm_model = ElasticDNN_Vilt_OfflineVQAFMModel('fm', fm_models_dict_path, device) + md_model = ElasticDNN_Vilt_OfflineVQAMDModel('md', md_models_dict_path, device) + + # 2. init alg + models = { + 'fm': fm_model, + 'md': md_model + } + from methods.elasticdnn.api.algs.md_pretraining_index_v2_train_index_and_md import ElasticDNN_MDPretrainingIndexAlg + fm_to_md_alg = ElasticDNN_MDPretrainingIndexAlg(models, get_res_save_dir(__file__, sys.argv[1])) + + sample_dataset = list(scenario.get_offline_datasets().values())[0]['train'] + sample = sample_dataset[0][0] + + from utils.dl.common.lr_scheduler import get_linear_schedule_with_warmup + fm_to_md_alg.run(scenario, hyps={ + 'launch_tbboard': False, + + 'samples_size': sample, + + 'FBS_r': 16, + 'FBS_ignore_layers': [], + + 'train_batch_size': 64, + 'val_batch_size': 256, + 'num_workers': 16, + 'optimizer': 'AdamW', + 'optimizer_args': {'lr': 1e-4, 'betas': [0.9, 0.999], 'weight_decay': 0.01}, + 'scheduler': 'LambdaLR', + 'scheduler_args': {'lr_lambda': get_linear_schedule_with_warmup(10000, 70000)}, + + 'indexes_optimizer_args': {'lr': 3e-3, 'momentum': 0.9, 'weight_decay': 5e-4}, + + 'num_iters': 80000, + 'val_freq': 400, + + 'max_sparsity': 0.9, + 'min_sparsity': 0.0, + 'l1_reg_loss_weight': 1e-9, + 'index_loss_weight': 1e-4, + 'val_num_sparsities': 4, + + 'bn_cal_num_iters': 0, + + 'index_init': 'zero', + 'index_guided_linear_comb_split_size': 512 + }) + \ No newline at end of file diff --git a/experiments/elasticdnn/vilt/offline/fm_to_md/vqa/vqa_wo_fbs.py b/experiments/elasticdnn/vilt/offline/fm_to_md/vqa/vqa_wo_fbs.py new file mode 100644 index 0000000000000000000000000000000000000000..749d25e46a4f177337d547cb73c91482e2d45703 --- /dev/null +++ b/experiments/elasticdnn/vilt/offline/fm_to_md/vqa/vqa_wo_fbs.py @@ -0,0 +1,179 @@ +import torch +from methods.elasticdnn.api.model import ElasticDNN_OfflineVQAFMModel, ElasticDNN_OfflineVQAMDModel +from methods.elasticdnn.api.algs.fm_lora import ElasticDNN_FMLoRAAlg +from methods.elasticdnn.model.base import ElasticDNNUtil +from methods.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from methods.elasticdnn.pipeline.offline.fm_lora.vilt import FMLoRA_Vilt_Util +from methods.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from methods.elasticdnn.pipeline.offline.fm_to_md.vilt import FM_to_MD_Vilt_Util +from methods.elasticdnn.model.vit import ElasticViTUtil +from utils.dl.common.model import LayerActivation, get_module, get_parameter, set_module +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +from utils.common.log import logger +import torch.nn.functional as F +import sys +from torch import nn +from methods.elasticdnn.api.algs.md_pretraining_wo_fbs import ElasticDNN_MDPretrainingWoFBSAlg + + + +class ElasticDNN_Vilt_OfflineVQAFMModel(ElasticDNN_OfflineVQAFMModel): + def generate_md_by_reducing_width(self, reducing_width_ratio, samples: torch.Tensor): + return FM_to_MD_Vilt_Util().init_md_from_fm_by_reducing_width_with_perf_test(self.models_dict['main'], + reducing_width_ratio, samples) + # raise NotImplementedError + + def get_feature_hook(self) -> LayerActivation: + return LayerActivation(get_module(self.models_dict['main'], 'classifier.3'), True, self.device) + + def get_elastic_dnn_util(self) -> ElasticDNNUtil: + raise NotImplementedError + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + self.to_train_mode() + # print(x['input_ids'].size(), x['pixel_values'].size(), ) + o = self.infer(x).logits + + # print(o.size(), y.size(), o, y) + + return F.binary_cross_entropy_with_logits(o, y) * y.shape[1] + + def get_lora_util(self) -> FMLoRA_Util: + return FMLoRA_Vilt_Util() + + def get_task_head_params(self): + head = get_module(self.models_dict['main'], 'classifier') + params_name = {k for k, v in head.named_parameters()} + logger.info(f'task head params: {params_name}') + return list(head.parameters()) + + +class ElasticDNN_Vilt_OfflineVQAMDModel(ElasticDNN_OfflineVQAMDModel): + def get_feature_hook(self) -> LayerActivation: + return LayerActivation(get_module(self.models_dict['main'], 'classifier.3'), True, self.device) + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + self.to_train_mode() + # print(x['input_ids'].size(), x['pixel_values'].size(), ) + o = self.infer(x).logits + + # print(o.size(), y.size(), o, y) + + return F.binary_cross_entropy_with_logits(o, y) * y.shape[1] + + def get_distill_loss(self, student_output, teacher_output): + # print(student_output, teacher_output) + return F.mse_loss(student_output, teacher_output.detach()) + + def get_trained_params(self): + return self.models_dict['main'].parameters() + + def get_matched_param_of_fm(self, self_param_name, fm: nn.Module): # TODO: + if any([k in self_param_name for k in ['fbs', 'ab', 'embeddings']]): + return None + + p = get_parameter(self.models_dict['main'], self_param_name) + if p.dim() == 0: + return None + + elif p.dim() == 1 and 'LayerNorm' in self_param_name and 'weight' in self_param_name: + # if self_param_name.startswith('norm'): + # return None + return get_parameter(fm, self_param_name) + + # 1. xx.query.weight -> xx.query.fc.weight and xx.query.ab.0/1 + if ('query' in self_param_name or 'key' in self_param_name or \ + 'value' in self_param_name) and ('weight' in self_param_name): + ss = self_param_name.split('.') + # raise NotImplementedError() # TODO: + fm_qkv_name = '.'.join(ss[0: -1]) + '.fc' + fm_qkv = get_module(fm, fm_qkv_name) + + fm_abs_name = '.'.join(ss[0: -1]) + '.ab' + fm_abs = get_module(fm, fm_abs_name) + + return torch.cat([ + fm_qkv.weight.data, # task-agnositc params + fm_abs[1].weight @ fm_abs[0].weight + ], dim=0) + + elif ('query' in self_param_name or 'key' in self_param_name or \ + 'value' in self_param_name) and ('bias' in self_param_name): + ss = self_param_name.split('.') + + fm_qkv_name = '.'.join(ss[0: -1]) + '.fc.bias' + return get_parameter(fm, fm_qkv_name) + + elif 'intermediate.dense' in self_param_name: + fm_param_name = self_param_name.replace('.linear', '') + return get_parameter(fm, fm_param_name) + + else: + return get_parameter(fm, self_param_name) + + +if __name__ == '__main__': + from utils.dl.common.env import set_random_seed + set_random_seed(1) + + scenario = build_scenario( + source_datasets_name=['VQAv2_split1'], + target_datasets_order=['VQAv2_split1_c'] * 1, # TODO + da_mode='close_set', + data_dirs={ + 'VQAv2_split1': '/data/zql/datasets/vqav2', + 'VQAv2_split1_c': '/data/zql/datasets/vqav2' + }, + ) + + # 1. init model + fm_models_dict_path = 'experiments/elasticdnn/vilt/offline/fm_lora/vqa/results/vqa.py/20230730/999971-190508-trial/models/fm_best.pt' + fm_models = torch.load(fm_models_dict_path) + fm_models_dict_path = save_models_dict_for_init(fm_models, __file__, 'fm_vilt_vqa_lora') + md_models_dict_path = save_models_dict_for_init({ + 'main': -1 + }, __file__, 'md_vilt_none') + device = 'cuda' + + fm_model = ElasticDNN_Vilt_OfflineVQAFMModel('fm', fm_models_dict_path, device) + md_model = ElasticDNN_Vilt_OfflineVQAMDModel('md', md_models_dict_path, device) + + # 2. init alg + models = { + 'fm': fm_model, + 'md': md_model + } + import sys + + # from experiments.elasticdnn.clip.offline.fm_to_md.cls.md_pretraining_wo_fbs import ElasticDNN_MDPretrainingWoFBSAlg + # from methods.elasticdnn.api.algs.md_pretraining_wo_fbs_clip_debug import ElasticDNN_MDPretrainingWoFBSAlg + fm_to_md_alg = ElasticDNN_MDPretrainingWoFBSAlg(models, get_res_save_dir(__file__, sys.argv[1])) + + # sample_dataset = list(scenario.get_offline_datasets().values())[0]['train'] + + sample_dataset = list(scenario.get_offline_datasets().values())[0]['train'] + sample = sample_dataset[0][0] + + for k, v in sample.items(): + sample[k] = v.unsqueeze(0) + + from utils.dl.common.lr_scheduler import get_linear_schedule_with_warmup + fm_to_md_alg.run(scenario, hyps={ + 'launch_tbboard': False, + + 'samples_size': sample, + 'generate_md_width_ratio': 4, + + 'train_batch_size': 64, + 'val_batch_size': 256, + 'num_workers': 16, + 'optimizer': 'AdamW', + 'optimizer_args': {'lr': 1e-4, 'betas': [0.9, 0.999], 'weight_decay': 0.01}, + 'scheduler': 'LambdaLR', + 'scheduler_args': {'lr_lambda': get_linear_schedule_with_warmup(10000, 70000)}, + 'num_iters': 80000, + 'val_freq': 1000, + 'distill_loss_weight': 1. + }) + \ No newline at end of file diff --git a/experiments/elasticdnn/vilt/online/vqa/model.py b/experiments/elasticdnn/vilt/online/vqa/model.py new file mode 100644 index 0000000000000000000000000000000000000000..4ea4d74a42b1ba988233c7282c7b932f2e25ce75 --- /dev/null +++ b/experiments/elasticdnn/vilt/online/vqa/model.py @@ -0,0 +1,332 @@ +from typing import List +from data.dataloader import build_dataloader +# from methods.elasticdnn.api.online_model import ElasticDNN_OnlineModel +from methods.elasticdnn.api.online_model_v2 import ElasticDNN_OnlineModel + +import torch +import sys +from torch import nn +from methods.elasticdnn.api.model import ElasticDNN_OfflineSegFMModel, ElasticDNN_OfflineSegMDModel +from methods.elasticdnn.api.algs.md_pretraining_wo_fbs import ElasticDNN_MDPretrainingWoFBSAlg +from methods.elasticdnn.model.base import ElasticDNNUtil +from methods.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from methods.elasticdnn.pipeline.offline.fm_to_md.vit import FM_to_MD_ViT_Util +from methods.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from methods.elasticdnn.pipeline.offline.fm_lora.vit import FMLoRA_ViT_Util +from methods.elasticdnn.model.vilt import ElasticViltUtil +from utils.common.file import ensure_dir +from utils.dl.common.model import LayerActivation, get_module, get_parameter, set_module +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +from utils.dl.common.loss import CrossEntropyLossSoft +import torch.nn.functional as F +from utils.dl.common.env import create_tbwriter +import os +from utils.common.log import logger +from utils.common.data_record import write_json +# from methods.shot.shot import OnlineShotModel +from methods.ewc.ewc_elasticfm import OnlineEWCModel +import tqdm +# from methods.feat_align.mmd import mmd_rbf +from copy import deepcopy + + +class ElasticDNN_VQAOnlineModel(ElasticDNN_OnlineModel): + @torch.no_grad() + def sd_feedback_to_md(self, after_da_sd, unpruned_indexes_of_layers): + self.models_dict['sd'] = after_da_sd + self.before_da_md = deepcopy(self.models_dict['md']) + + logger.info('\n\nsurrogate DNN feedback to master DNN...\n\n') + # one-to-one + + cur_unpruned_indexes = None + cur_unpruned_indexes_name = None + + for p_name, p in self.models_dict['sd'].named_parameters(): + matched_md_param = self.get_md_matched_param_of_sd_param(p_name) + logger.debug(f'if feedback: {p_name}') + if matched_md_param is None: + continue + logger.debug(f'start feedback: {p_name}, {p.size()} -> {matched_md_param.size()}') + # average + # setattr(matched_md_module, matched_md_param_name, (matched_md_param + p) / 2.) + + if p_name in unpruned_indexes_of_layers.keys(): + cur_unpruned_indexes = unpruned_indexes_of_layers[p_name] + cur_unpruned_indexes_name = p_name + + if p.size() != matched_md_param.size(): + logger.debug(f'cur unpruned indexes: {cur_unpruned_indexes_name}, {cur_unpruned_indexes.size()}') + + if p.dim() == 1: # norm + new_p = deepcopy(matched_md_param) + new_p[cur_unpruned_indexes] = p + elif p.dim() == 2: # linear + if p.size(0) < matched_md_param.size(0): # output pruned + new_p = deepcopy(matched_md_param) + new_p[cur_unpruned_indexes] = p + else: # input pruned + new_p = deepcopy(matched_md_param) + new_p[:, cur_unpruned_indexes] = p + p = new_p + + assert p.size() == matched_md_param.size(), f'{p.size()}, {matched_md_param.size()}' + + # if 'head' in p_name: + if False: + continue + # if False: + # self.last_trained_cls_indexes + assert hasattr(self, 'last_trained_cls_indexes') + print(self.last_trained_cls_indexes) + + diff = self._compute_diff(matched_md_param, p) + # matched_md_param[self.last_trained_cls_indexes].copy_(p[self.last_trained_cls_indexes.to(self.device)]) + matched_md_param.copy_(p) + logger.debug(f'SPECIFIC FOR CL HEAD | end feedback: {p_name}, diff: {diff:.6f}') + else: + diff = self._compute_diff(matched_md_param, (matched_md_param + p) / 2.) + matched_md_param.copy_((matched_md_param + p) / 2.) + logger.debug(f'end feedback: {p_name}, diff: {diff:.6f}') + + def add_cls_in_head(self, num_cls): # NOTE: + head: nn.Linear = get_module(self.models_dict['md'], 'head') + + new_head = nn.Linear(head.in_features, head.out_features + num_cls, head.bias is not None, device=self.device) + + # nn.init.zeros_(new_head.weight.data) + # nn.init.zeros_(new_head.bias.data) + + new_head.weight.data[0: head.out_features] = deepcopy(head.weight.data) + new_head.bias.data[0: head.out_features] = deepcopy(head.bias.data) + set_module(self.models_dict['md'], 'head', new_head) + set_module(self.models_dict['fm'], 'head', new_head) + + def get_accuracy(self, test_loader, *args, **kwargs): + acc = 0 + sample_num = 0 + + from methods.elasticdnn.api.model import VQAScore + vqa_score = VQAScore() + + self.to_eval_mode() + + with torch.no_grad(): + pbar = tqdm.tqdm(enumerate(test_loader), total=len(test_loader), dynamic_ncols=True, leave=False) + for batch_index, (x, y) in pbar: + for k, v in x.items(): + if isinstance(v, torch.Tensor): + x[k] = v.to(self.device) + y = y.to(self.device) + output = self.infer(x).logits + + vqa_score.update(output, y) + + pbar.set_description(f'cur_batch_total: {len(y)}, cur_batch_acc: {vqa_score.compute():.4f}') + + return float(vqa_score.compute()) + + def get_elastic_dnn_util(self) -> ElasticDNNUtil: + return ElasticViltUtil() + + def get_fm_matched_param_of_md_param(self, md_param_name): + # only between qkv.weight, norm.weight/bias + self_param_name = md_param_name + fm = self.models_dict['fm'] + if any([k in self_param_name for k in ['fbs', 'ab', 'embeddings']]): + return None + + p = get_parameter(self.models_dict['md'], self_param_name) + if p.dim() == 0: + return None + elif p.dim() == 1 and ('LayerNorm' in self_param_name or 'layernorm' in self_param_name) and 'weight' in self_param_name: + return get_parameter(fm, self_param_name) + + # 1. xx.qkv.to_qkv.yy to xx.qkv.qkv.aa and xx.qkv.abs.zz + if ('query' in self_param_name or 'key' in self_param_name or \ + 'value' in self_param_name) and ('weight' in self_param_name): + + ss = self_param_name.split('.') + + fm_qkv_name = '.'.join(ss[0: -1]) + '.fc' + fm_qkv = get_module(fm, fm_qkv_name) + + fm_abs_name = '.'.join(ss[0: -1]) + '.ab' + fm_abs = get_module(fm, fm_abs_name) + + # NOTE: unrecoverable operation! multiply LoRA parameters to allow it being updated in update_fm_param() + # TODO: if fm will be used for inference, _mul_lora_weight will not be applied! + if not hasattr(fm_abs, '_mul_lora_weight'): + logger.debug(f'set _mul_lora_weight in {fm_abs_name}') + setattr(fm_abs, '_mul_lora_weight', + nn.Parameter(fm_abs[1].weight @ fm_abs[0].weight)) + + return torch.cat([ + fm_qkv.weight.data, # task-agnositc params + fm_abs._mul_lora_weight.data # task-specific params (LoRA) + ], dim=0) + + # elif 'to_qkv.bias' in self_param_name: + # ss = self_param_name.split('.') + + # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv.bias' + # return get_parameter(fm, fm_qkv_name) + + elif 'dense' in self_param_name and 'weight' in self_param_name: + fm_param_name = self_param_name.replace('.linear', '') + return get_parameter(fm, fm_param_name) + + # elif 'mlp.fc2' in self_param_name and 'weight' in self_param_name: + # fm_param_name = self_param_name + # return get_parameter(fm, fm_param_name) + + else: + # return get_parameter(fm, self_param_name) + return None + + def update_fm_param(self, md_param_name, cal_new_fm_param_by_md_param): + if not ('query' in md_param_name or 'key' in md_param_name or 'value' in md_param_name): + matched_fm_param_ref = self.get_fm_matched_param_of_md_param(md_param_name) + matched_fm_param_ref.copy_(cal_new_fm_param_by_md_param) + else: + new_fm_attn_weight, new_fm_lora_weight = torch.chunk(cal_new_fm_param_by_md_param, 2, 0) + + ss = md_param_name.split('.') + fm = self.models_dict['fm'] + + # update task-agnostic parameters + fm_qkv_name = '.'.join(ss[0: -1]) + '.fc' + fm_qkv = get_module(fm, fm_qkv_name) + fm_qkv.weight.data.copy_(new_fm_attn_weight) + + # update task-specific parameters + fm_abs_name = '.'.join(ss[0: -1]) + '.ab' + fm_abs = get_module(fm, fm_abs_name) + fm_abs._mul_lora_weight.data.copy_(new_fm_lora_weight) # TODO: this will not be applied in inference! + + def get_md_matched_param_of_fm_param(self, fm_param_name): + return super().get_md_matched_param_of_fm_param(fm_param_name) + + def get_md_matched_param_of_sd_param(self, sd_param_name): + # raise NotImplementedError + + # only between qkv.weight, norm.weight/bias + self_param_name = sd_param_name + md = self.models_dict['md'] + if any([k in self_param_name for k in ['fbs', 'ab', 'embeddings']]): + return None + + p = get_parameter(self.models_dict['sd'], self_param_name) + if p.dim() == 0: + return None + elif p.dim() == 1 and ('LayerNorm' in self_param_name or 'layernorm' in self_param_name) and 'weight' in self_param_name: + return get_parameter(md, self_param_name) + + # 1. xx.qkv.to_qkv.yy to xx.qkv.qkv.aa and xx.qkv.abs.zz + if ('query' in self_param_name or 'key' in self_param_name or \ + 'value' in self_param_name) and ('weight' in self_param_name): + + + return get_parameter(md, self_param_name) # NOTE: no fbs in qkv! + + # elif 'to_qkv.bias' in self_param_name: + # ss = self_param_name.split('.') + + # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv.bias' + # return get_parameter(fm, fm_qkv_name) + + elif 'intermediate.dense.0.weight' in self_param_name: + fm_param_name = '.'.join(self_param_name.split('.')[0: -2]) + '.linear.weight' + return get_parameter(md, fm_param_name) + + elif 'output.dense' in self_param_name and 'weight' in self_param_name: + fm_param_name = self_param_name + return get_parameter(md, fm_param_name) + + else: + # return get_parameter(fm, self_param_name) + return None + + def get_task_head_params(self): + head = get_module(self.models_dict['sd'], 'classifier') + return list(head.parameters()) + + + + +from typing import List, Tuple +from data.dataloader import build_dataloader +# from methods.elasticdnn.api.online_model import ElasticDNN_OnlineModel +from methods.elasticdnn.api.online_model_v2 import ElasticDNN_OnlineModel + +import torch +import sys +from torch import nn +from methods.elasticdnn.api.model import ElasticDNN_OfflineSegFMModel, ElasticDNN_OfflineSegMDModel +from methods.elasticdnn.api.algs.md_pretraining_wo_fbs import ElasticDNN_MDPretrainingWoFBSAlg +from methods.elasticdnn.model.base import ElasticDNNUtil +from methods.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from methods.elasticdnn.pipeline.offline.fm_to_md.vit import FM_to_MD_ViT_Util +from methods.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from methods.elasticdnn.pipeline.offline.fm_lora.vit import FMLoRA_ViT_Util +from methods.elasticdnn.model.vit import ElasticViTUtil +from utils.common.file import ensure_dir +from utils.dl.common.model import LayerActivation, LayerActivation2, get_module, get_parameter, set_module +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +from utils.dl.common.loss import CrossEntropyLossSoft +import torch.nn.functional as F +from utils.dl.common.env import create_tbwriter +import os +from utils.common.log import logger +from utils.common.data_record import write_json +# from methods.shot.shot import OnlineShotModel +from methods.feat_align.main import OnlineFeatAlignModel +import tqdm +from methods.feat_align.mmd import mmd_rbf +from copy import deepcopy + + +class VQAOnlineFeatAlignModel(OnlineFeatAlignModel): + def get_trained_params(self): + qkv_and_norm_params = [p for n, p in self.models_dict['main'].named_parameters() if 'query' in n or 'key' in n or 'value' in n or 'dense' in n or 'LayerNorm' in n] + return qkv_and_norm_params + + def get_feature_hook(self): + return LayerActivation(get_module(self.models_dict['main'], 'classifier.3'), False, self.device) + + def forward_to_get_task_loss(self, x, y): + self.to_train_mode() + o = self.infer(x) + return F.binary_cross_entropy_with_logits(o, y) * y.shape[1] + + def get_mmd_loss(self, f1, f2): + return mmd_rbf(f1, f2) + + def infer(self, x, *args, **kwargs): + return self.models_dict['main'](**x).logits + + def get_accuracy(self, test_loader, *args, **kwargs): + acc = 0 + sample_num = 0 + + from methods.elasticdnn.api.model import VQAScore + vqa_score = VQAScore() + + self.to_eval_mode() + + with torch.no_grad(): + pbar = tqdm.tqdm(enumerate(test_loader), total=len(test_loader), dynamic_ncols=True, leave=False) + for batch_index, (x, y) in pbar: + for k, v in x.items(): + if isinstance(v, torch.Tensor): + x[k] = v.to(self.device) + y = y.to(self.device) + output = self.infer(x) + + vqa_score.update(output, y) + + pbar.set_description(f'cur_batch_total: {len(y)}, cur_batch_acc: {vqa_score.compute():.4f}') + + return float(vqa_score.compute()) \ No newline at end of file diff --git a/experiments/elasticdnn/vilt/online/vqa/vqa.py b/experiments/elasticdnn/vilt/online/vqa/vqa.py new file mode 100644 index 0000000000000000000000000000000000000000..9dd126627c930d3ff9c106135c17e22511300813 --- /dev/null +++ b/experiments/elasticdnn/vilt/online/vqa/vqa.py @@ -0,0 +1,96 @@ +from typing import List +from data.dataloader import build_dataloader +# from methods.elasticdnn.api.online_model import ElasticDNN_OnlineModel +from methods.elasticdnn.api.online_model_v2 import ElasticDNN_OnlineModel + +import torch +import sys +from torch import nn +from methods.elasticdnn.api.model import ElasticDNN_OfflineSegFMModel, ElasticDNN_OfflineSegMDModel +from methods.elasticdnn.api.algs.md_pretraining_wo_fbs import ElasticDNN_MDPretrainingWoFBSAlg +from methods.elasticdnn.model.base import ElasticDNNUtil +from methods.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from methods.elasticdnn.pipeline.offline.fm_to_md.vit import FM_to_MD_ViT_Util +from methods.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from methods.elasticdnn.pipeline.offline.fm_lora.vit import FMLoRA_ViT_Util +from methods.elasticdnn.model.vit import ElasticViTUtil +from utils.common.file import ensure_dir +from utils.dl.common.model import LayerActivation, get_module, get_parameter +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +from utils.dl.common.loss import CrossEntropyLossSoft +import torch.nn.functional as F +from utils.dl.common.env import create_tbwriter +import os +from utils.common.log import logger +from utils.common.data_record import write_json +# from methods.shot.shot import OnlineShotModel +from methods.feat_align.main import FeatAlignAlg +import tqdm +from methods.feat_align.mmd import mmd_rbf +from experiments.utils.elasticfm_da import init_online_model, elasticfm_da + + +os.environ['TOKENIZERS_PARALLELISM'] = 'false' + +device = 'cuda' +app_name = 'vqa' +sd_sparsity = 0.8 + +settings = { + 'involve_fm': True +} + +target_datasets = ['VQAv2_split1_c_gaussian_noise', 'VQAv2_split1_c_shot_noise', 'VQAv2_split1_c_impulse_noise', 'VQAv2_split1_c_defocus_blur', 'VQAv2_split1_c_glass_blur', 'VQAv2_split1_c_motion_blur', 'VQAv2_split1_c_zoom_blur', 'VQAv2_split1_c_snow', 'VQAv2_split1_c_frost', 'VQAv2_split1_c_fog', 'VQAv2_split1_c_brightness', 'VQAv2_split1_c_contrast', 'VQAv2_split1_c_elastic_transform', 'VQAv2_split1_c_pixelate', 'VQAv2_split1_c_jpeg_compression', 'VQAv2_split1_c_speckle_noise', 'VQAv2_split1_c_gaussian_blur', 'VQAv2_split1_c_spatter', 'VQAv2_split1_c_saturate'] * 2 +target_datasets = target_datasets[0: 30] +assert len(target_datasets) == 30 + +scenario = build_scenario( + source_datasets_name=['VQAv2_split1'], + target_datasets_order=target_datasets, + da_mode='close_set', + data_dirs={ + k: '/data/zql/datasets/vqav2' for k in ['VQAv2_split1', 'VQAv2_split1_c_gaussian_noise', 'VQAv2_split1_c_shot_noise', 'VQAv2_split1_c_impulse_noise', 'VQAv2_split1_c_defocus_blur', 'VQAv2_split1_c_glass_blur', 'VQAv2_split1_c_motion_blur', 'VQAv2_split1_c_zoom_blur', 'VQAv2_split1_c_snow', 'VQAv2_split1_c_frost', 'VQAv2_split1_c_fog', 'VQAv2_split1_c_brightness', 'VQAv2_split1_c_contrast', 'VQAv2_split1_c_elastic_transform', 'VQAv2_split1_c_pixelate', 'VQAv2_split1_c_jpeg_compression', 'VQAv2_split1_c_speckle_noise', 'VQAv2_split1_c_gaussian_blur', 'VQAv2_split1_c_spatter', 'VQAv2_split1_c_saturate'] + }, +) + +from experiments.elasticdnn.vilt.online.vqa.model import ElasticDNN_VQAOnlineModel +elasticfm_model = ElasticDNN_VQAOnlineModel('vqa', init_online_model( + 'experiments/elasticdnn/vilt/offline/fm_to_md/vqa/results/vqa_w_fbs_index.py/20230731/999999-095720-trial/models/fm_best.pt', + 'experiments/elasticdnn/vilt/offline/fm_to_md/vqa/results/vqa_w_fbs_index.py/20230731/999999-095720-trial/models/md_best.pt', + 'vqa', __file__ +), device, { + 'md_to_fm_alpha': 0.2, + 'fm_to_md_alpha': 0.2 +}) + +da_alg = FeatAlignAlg +from experiments.elasticdnn.vilt.online.vqa.model import VQAOnlineFeatAlignModel +da_model = VQAOnlineFeatAlignModel +da_alg_hyp = { + 'train_batch_size': 64, + 'val_batch_size': 256, + 'num_workers': 0, + 'optimizer': 'AdamW', + 'optimizer_args': {'lr': 1e-6, 'betas': [0.9, 0.999], 'weight_decay': 0.01}, + 'scheduler': '', + 'scheduler_args': {}, + 'num_iters': 100, + 'val_freq': 20, + 'feat_align_loss_weight': 1.0, + 'sd_sparsity': 0.7 +} + + +elasticfm_da( + [app_name], + [scenario], + [elasticfm_model], + [da_alg], + [da_alg_hyp], + [da_model], + device, + settings, + __file__, + sys.argv[1] +) diff --git a/experiments/elasticdnn/vilt/online/vqa_cl/model.py b/experiments/elasticdnn/vilt/online/vqa_cl/model.py new file mode 100644 index 0000000000000000000000000000000000000000..074bc84cfadaadce7bee305b55da58ee2bd7b188 --- /dev/null +++ b/experiments/elasticdnn/vilt/online/vqa_cl/model.py @@ -0,0 +1,373 @@ +from typing import List +from data.dataloader import build_dataloader +# from methods.elasticdnn.api.online_model import ElasticDNN_OnlineModel +from methods.elasticdnn.api.online_model_v2 import ElasticDNN_OnlineModel + +import torch +import sys +from torch import nn +from methods.elasticdnn.api.model import ElasticDNN_OfflineSegFMModel, ElasticDNN_OfflineSegMDModel +from methods.elasticdnn.api.algs.md_pretraining_wo_fbs import ElasticDNN_MDPretrainingWoFBSAlg +from methods.elasticdnn.model.base import ElasticDNNUtil +from methods.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from methods.elasticdnn.pipeline.offline.fm_to_md.vit import FM_to_MD_ViT_Util +from methods.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from methods.elasticdnn.pipeline.offline.fm_lora.vit import FMLoRA_ViT_Util +from methods.elasticdnn.model.vilt import ElasticViltUtil +from utils.common.file import ensure_dir +from utils.dl.common.model import LayerActivation, get_module, get_parameter, set_module +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +from utils.dl.common.loss import CrossEntropyLossSoft +import torch.nn.functional as F +from utils.dl.common.env import create_tbwriter +import os +from utils.common.log import logger +from utils.common.data_record import write_json +# from methods.shot.shot import OnlineShotModel +from methods.feat_align.main import OnlineFeatAlignModel, FeatAlignAlg +import tqdm +from methods.feat_align.mmd import mmd_rbf + +class ElasticDNN_VQAOnlineModel(ElasticDNN_OnlineModel): + + # @torch.no_grad() + # def sd_feedback_to_md(self, after_da_sd, unpruned_indexes_of_layers): + # self.models_dict['sd'] = after_da_sd + # self.before_da_md = deepcopy(self.models_dict['md']) + + # logger.info('\n\nsurrogate DNN feedback to master DNN...\n\n') + # # one-to-one + + # cur_unpruned_indexes = None + # cur_unpruned_indexes_name = None + + # for p_name, p in self.models_dict['sd'].named_parameters(): + # matched_md_param = self.get_md_matched_param_of_sd_param(p_name) + # logger.debug(f'if feedback: {p_name}') + # if matched_md_param is None: + # continue + # logger.debug(f'start feedback: {p_name}, {p.size()} -> {matched_md_param.size()}') + # # average + # # setattr(matched_md_module, matched_md_param_name, (matched_md_param + p) / 2.) + + # if p_name in unpruned_indexes_of_layers.keys(): + # cur_unpruned_indexes = unpruned_indexes_of_layers[p_name] + # cur_unpruned_indexes_name = p_name + + # if p.size() != matched_md_param.size(): + # logger.debug(f'cur unpruned indexes: {cur_unpruned_indexes_name}, {cur_unpruned_indexes.size()}') + + # if p.dim() == 1: # norm + # new_p = deepcopy(matched_md_param) + # new_p[cur_unpruned_indexes] = p + # elif p.dim() == 2: # linear + # if p.size(0) < matched_md_param.size(0): # output pruned + # new_p = deepcopy(matched_md_param) + # new_p[cur_unpruned_indexes] = p + # else: # input pruned + # new_p = deepcopy(matched_md_param) + # new_p[:, cur_unpruned_indexes] = p + # p = new_p + + # assert p.size() == matched_md_param.size(), f'{p.size()}, {matched_md_param.size()}' + + # if 'classifier' in p_name: + # continue + # # if False: + # # self.last_trained_cls_indexes + # assert hasattr(self, 'last_trained_cls_indexes') + # print(self.last_trained_cls_indexes) + + # diff = self._compute_diff(matched_md_param, p) + # # matched_md_param[self.last_trained_cls_indexes].copy_(p[self.last_trained_cls_indexes.to(self.device)]) + # matched_md_param.copy_(p) + # logger.debug(f'SPECIFIC FOR CL HEAD | end feedback: {p_name}, diff: {diff:.6f}') + # else: + # diff = self._compute_diff(matched_md_param, (matched_md_param + p) / 2.) + # matched_md_param.copy_((matched_md_param + p) / 2.) + # logger.debug(f'end feedback: {p_name}, diff: {diff:.6f}') + + def add_cls_in_head(self, num_cls): + head: nn.Linear = get_module(self.models_dict['md'], 'classifier.3') + + new_head = nn.Linear(head.in_features, head.out_features + num_cls, head.bias is not None, device=self.device) + + # nn.init.zeros_(new_head.weight.data) + # nn.init.zeros_(new_head.bias.data) + + new_head.weight.data[0: head.out_features] = deepcopy(head.weight.data) + new_head.bias.data[0: head.out_features] = deepcopy(head.bias.data) + set_module(self.models_dict['md'], 'classifier.3', new_head) + set_module(self.models_dict['fm'], 'classifier.3', new_head) + + + def get_accuracy(self, test_loader, *args, **kwargs): + _d = test_loader.dataset + from data import build_dataloader, split_dataset + if _d.__class__.__name__ == '_SplitDataset' and _d.underlying_dataset.__class__.__name__ == 'MergedDataset': # necessary for CL + print('\neval on merged datasets') + + merged_full_dataset = _d.underlying_dataset.datasets + ratio = len(_d.keys) / len(_d.underlying_dataset) + + if int(len(_d) * ratio) == 0: + ratio = 1. + # print(ratio) + # bs = + # test_loaders = [build_dataloader(split_dataset(d, min(max(test_loader.batch_size, int(len(d) * ratio)), len(d)))[0], # TODO: this might be overlapped with train dataset + # min(test_loader.batch_size, int(len(d) * ratio)), + # test_loader.num_workers, False, None) for d in merged_full_dataset] + + test_loaders = [] + for d in merged_full_dataset: + if len(d) == 0: + continue + + n = int(len(d) * ratio) + if n == 0: + n = len(d) + sub_dataset = split_dataset(d, min(max(test_loader.batch_size, n), len(d)))[0] + loader = build_dataloader(sub_dataset, min(test_loader.batch_size, n), test_loader.num_workers, False, None) + test_loaders += [loader] + + accs = [self.get_accuracy(loader) for loader in test_loaders] + print(accs) + return sum(accs) / len(accs) + + acc = 0 + sample_num = 0 + + from methods.elasticdnn.api.model import VQAScore + vqa_score = VQAScore() + + self.to_eval_mode() + + with torch.no_grad(): + pbar = tqdm.tqdm(enumerate(test_loader), total=len(test_loader), dynamic_ncols=True, leave=False) + for batch_index, (x, y) in pbar: + for k, v in x.items(): + if isinstance(v, torch.Tensor): + x[k] = v.to(self.device) + y = y.to(self.device) + output = self.infer(x).logits + + vqa_score.update(output, y) + + pbar.set_description(f'cur_batch_total: {len(y)}, cur_batch_acc: {vqa_score.compute():.4f}') + + return float(vqa_score.compute()) + + def get_elastic_dnn_util(self) -> ElasticDNNUtil: + return ElasticViltUtil() + + def get_fm_matched_param_of_md_param(self, md_param_name): + # only between qkv.weight, norm.weight/bias + self_param_name = md_param_name + fm = self.models_dict['fm'] + if any([k in self_param_name for k in ['fbs', 'ab', 'embeddings']]): + return None + + p = get_parameter(self.models_dict['md'], self_param_name) + if p.dim() == 0: + return None + elif p.dim() == 1 and ('LayerNorm' in self_param_name or 'layernorm' in self_param_name) and 'weight' in self_param_name: + return get_parameter(fm, self_param_name) + + # 1. xx.qkv.to_qkv.yy to xx.qkv.qkv.aa and xx.qkv.abs.zz + if ('query' in self_param_name or 'key' in self_param_name or \ + 'value' in self_param_name) and ('weight' in self_param_name): + + ss = self_param_name.split('.') + + fm_qkv_name = '.'.join(ss[0: -1]) + '.fc' + fm_qkv = get_module(fm, fm_qkv_name) + + fm_abs_name = '.'.join(ss[0: -1]) + '.ab' + fm_abs = get_module(fm, fm_abs_name) + + # NOTE: unrecoverable operation! multiply LoRA parameters to allow it being updated in update_fm_param() + # TODO: if fm will be used for inference, _mul_lora_weight will not be applied! + if not hasattr(fm_abs, '_mul_lora_weight'): + logger.debug(f'set _mul_lora_weight in {fm_abs_name}') + setattr(fm_abs, '_mul_lora_weight', + nn.Parameter(fm_abs[1].weight @ fm_abs[0].weight)) + + return torch.cat([ + fm_qkv.weight.data, # task-agnositc params + fm_abs._mul_lora_weight.data # task-specific params (LoRA) + ], dim=0) + + # elif 'to_qkv.bias' in self_param_name: + # ss = self_param_name.split('.') + + # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv.bias' + # return get_parameter(fm, fm_qkv_name) + + elif 'dense' in self_param_name and 'weight' in self_param_name: + fm_param_name = self_param_name.replace('.linear', '') + return get_parameter(fm, fm_param_name) + + # elif 'mlp.fc2' in self_param_name and 'weight' in self_param_name: + # fm_param_name = self_param_name + # return get_parameter(fm, fm_param_name) + + else: + # return get_parameter(fm, self_param_name) + return None + + def update_fm_param(self, md_param_name, cal_new_fm_param_by_md_param): + if not ('query' in md_param_name or 'key' in md_param_name or 'value' in md_param_name): + matched_fm_param_ref = self.get_fm_matched_param_of_md_param(md_param_name) + matched_fm_param_ref.copy_(cal_new_fm_param_by_md_param) + else: + new_fm_attn_weight, new_fm_lora_weight = torch.chunk(cal_new_fm_param_by_md_param, 2, 0) + + ss = md_param_name.split('.') + fm = self.models_dict['fm'] + + # update task-agnostic parameters + fm_qkv_name = '.'.join(ss[0: -1]) + '.fc' + fm_qkv = get_module(fm, fm_qkv_name) + fm_qkv.weight.data.copy_(new_fm_attn_weight) + + # update task-specific parameters + fm_abs_name = '.'.join(ss[0: -1]) + '.ab' + fm_abs = get_module(fm, fm_abs_name) + fm_abs._mul_lora_weight.data.copy_(new_fm_lora_weight) # TODO: this will not be applied in inference! + + def get_md_matched_param_of_fm_param(self, fm_param_name): + return super().get_md_matched_param_of_fm_param(fm_param_name) + + def get_md_matched_param_of_sd_param(self, sd_param_name): + # raise NotImplementedError + + # only between qkv.weight, norm.weight/bias + self_param_name = sd_param_name + md = self.models_dict['md'] + if any([k in self_param_name for k in ['fbs', 'ab', 'embeddings']]): + return None + + p = get_parameter(self.models_dict['sd'], self_param_name) + if p.dim() == 0: + return None + elif p.dim() == 1 and ('LayerNorm' in self_param_name and 'layernorm' in self_param_name) and 'weight' in self_param_name: + return get_parameter(md, self_param_name) + + if 'classifier' in self_param_name: + return get_parameter(md, self_param_name) + + # 1. xx.qkv.to_qkv.yy to xx.qkv.qkv.aa and xx.qkv.abs.zz + if ('query' in self_param_name or 'key' in self_param_name or \ + 'value' in self_param_name) and ('weight' in self_param_name): + + + return get_parameter(md, self_param_name) # NOTE: no fbs in qkv! + + # elif 'to_qkv.bias' in self_param_name: + # ss = self_param_name.split('.') + + # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv.bias' + # return get_parameter(fm, fm_qkv_name) + + elif 'intermediate.dense.0.weight' in self_param_name: + fm_param_name = '.'.join(self_param_name.split('.')[0: -2]) + '.linear.weight' + return get_parameter(md, fm_param_name) + + elif 'output.dense' in self_param_name and 'weight' in self_param_name: + fm_param_name = self_param_name + return get_parameter(md, fm_param_name) + + else: + # return get_parameter(fm, self_param_name) + return None + + def get_task_head_params(self): + head = get_module(self.models_dict['sd'], 'classifier') + return list(head.parameters()) + + +from methods.gem.gem_el_vilt import OnlineGEMModel +import tqdm +from methods.feat_align.mmd import mmd_rbf +from copy import deepcopy + + +class VQAOnlineGEMModel(OnlineGEMModel): + def get_trained_params(self): + qkv_and_norm_params = [p for n, p in self.models_dict['main'].named_parameters()] + return qkv_and_norm_params + + def forward_to_get_task_loss(self, x, y): + self.to_train_mode() + o = self.infer(x).logits + return F.binary_cross_entropy_with_logits(o, y) * y.shape[1] + + def add_cls_in_head(self, num_cls): + return + + head: nn.Linear = get_module(self.models_dict['main'], 'head') + + new_head = nn.Linear(head.in_features, head.out_features + num_cls, head.bias is not None, device=self.device) + new_head.weight.data[0: head.out_features] = deepcopy(head.weight.data) + new_head.bias.data[0: head.out_features] = deepcopy(head.bias.data) + set_module(self.models_dict['main'], 'head', new_head) + + def infer(self, x, *args, **kwargs): + return self.models_dict['main'](**x) + + def get_accuracy(self, test_loader, *args, **kwargs): + _d = test_loader.dataset + from data import build_dataloader, split_dataset + if _d.__class__.__name__ == '_SplitDataset' and _d.underlying_dataset.__class__.__name__ == 'MergedDataset': # necessary for CL + print('\neval on merged datasets') + + merged_full_dataset = _d.underlying_dataset.datasets + ratio = len(_d.keys) / len(_d.underlying_dataset) + + if int(len(_d) * ratio) == 0: + ratio = 1. + # print(ratio) + # bs = + # test_loaders = [build_dataloader(split_dataset(d, min(max(test_loader.batch_size, int(len(d) * ratio)), len(d)))[0], # TODO: this might be overlapped with train dataset + # min(test_loader.batch_size, int(len(d) * ratio)), + # test_loader.num_workers, False, None) for d in merged_full_dataset] + + test_loaders = [] + for d in merged_full_dataset: + if len(d) == 0: + continue + + n = int(len(d) * ratio) + if n == 0: + n = len(d) + sub_dataset = split_dataset(d, min(max(test_loader.batch_size, n), len(d)))[0] + loader = build_dataloader(sub_dataset, min(test_loader.batch_size, n), test_loader.num_workers, False, None) + test_loaders += [loader] + + accs = [self.get_accuracy(loader) for loader in test_loaders] + print(accs) + return sum(accs) / len(accs) + + acc = 0 + sample_num = 0 + + from methods.elasticdnn.api.model import VQAScore + vqa_score = VQAScore() + + self.to_eval_mode() + + with torch.no_grad(): + pbar = tqdm.tqdm(enumerate(test_loader), total=len(test_loader), dynamic_ncols=True, leave=False) + for batch_index, (x, y) in pbar: + for k, v in x.items(): + if isinstance(v, torch.Tensor): + x[k] = v.to(self.device) + y = y.to(self.device) + output = self.infer(x).logits + + vqa_score.update(output, y) + + pbar.set_description(f'cur_batch_total: {len(y)}, cur_batch_acc: {vqa_score.compute():.4f}') + + return float(vqa_score.compute()) \ No newline at end of file diff --git a/experiments/elasticdnn/vilt/online/vqa_cl/vqa_cl.py b/experiments/elasticdnn/vilt/online/vqa_cl/vqa_cl.py new file mode 100644 index 0000000000000000000000000000000000000000..fa0cd783331e24243312a0e85fb93acfb673145a --- /dev/null +++ b/experiments/elasticdnn/vilt/online/vqa_cl/vqa_cl.py @@ -0,0 +1,111 @@ +from typing import List +from data.dataloader import build_dataloader +# from methods.elasticdnn.api.online_model import ElasticDNN_OnlineModel +from methods.elasticdnn.api.online_model_v2 import ElasticDNN_OnlineModel + +import torch +import sys +from torch import nn +from methods.elasticdnn.api.model import ElasticDNN_OfflineSegFMModel, ElasticDNN_OfflineSegMDModel +from methods.elasticdnn.api.algs.md_pretraining_wo_fbs import ElasticDNN_MDPretrainingWoFBSAlg +from methods.elasticdnn.model.base import ElasticDNNUtil +from methods.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from methods.elasticdnn.pipeline.offline.fm_to_md.vit import FM_to_MD_ViT_Util +from methods.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from methods.elasticdnn.pipeline.offline.fm_lora.vit import FMLoRA_ViT_Util +from methods.elasticdnn.model.vit import ElasticViTUtil +from utils.common.file import ensure_dir +from utils.dl.common.model import LayerActivation, get_module, get_parameter +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +from utils.dl.common.loss import CrossEntropyLossSoft +import torch.nn.functional as F +from utils.dl.common.env import create_tbwriter +import os +from utils.common.log import logger +from utils.common.data_record import write_json +# from methods.shot.shot import OnlineShotModel +from methods.gem.gem_el_vilt import GEMAlg +import tqdm +from methods.feat_align.mmd import mmd_rbf +from experiments.utils.elasticfm_cl import init_online_model, elasticfm_cl +from data import build_cl_scenario, build_scenario + +device = 'cuda' +app_name = 'vqa' +sd_sparsity = 0.8 + +settings = { + 'involve_fm': True +} + +target_datasets = ['VQAv2_split1_c_gaussian_noise', 'VQAv2_split1_c_shot_noise', 'VQAv2_split1_c_impulse_noise', 'VQAv2_split1_c_defocus_blur', 'VQAv2_split1_c_glass_blur', 'VQAv2_split1_c_motion_blur', 'VQAv2_split1_c_zoom_blur', 'VQAv2_split1_c_snow', 'VQAv2_split1_c_frost', 'VQAv2_split1_c_fog', 'VQAv2_split1_c_brightness', 'VQAv2_split1_c_contrast', 'VQAv2_split1_c_elastic_transform', 'VQAv2_split1_c_pixelate', 'VQAv2_split1_c_jpeg_compression', 'VQAv2_split1_c_speckle_noise', 'VQAv2_split1_c_gaussian_blur', 'VQAv2_split1_c_spatter', 'VQAv2_split1_c_saturate'] * 2 +target_datasets = target_datasets[0: 30] +assert len(target_datasets) == 30 + +scenario = build_scenario( + source_datasets_name=['VQAv2_split1'], + target_datasets_order=target_datasets, + da_mode='close_set', + data_dirs={ + k: '/data/zql/datasets/vqav2vv' for k in ['VQAv2_split1', 'VQAv2_split1_c_gaussian_noise', 'VQAv2_split1_c_shot_noise', 'VQAv2_split1_c_impulse_noise', 'VQAv2_split1_c_defocus_blur', 'VQAv2_split1_c_glass_blur', 'VQAv2_split1_c_motion_blur', 'VQAv2_split1_c_zoom_blur', 'VQAv2_split1_c_snow', 'VQAv2_split1_c_frost', 'VQAv2_split1_c_fog', 'VQAv2_split1_c_brightness', 'VQAv2_split1_c_contrast', 'VQAv2_split1_c_elastic_transform', 'VQAv2_split1_c_pixelate', 'VQAv2_split1_c_jpeg_compression', 'VQAv2_split1_c_speckle_noise', 'VQAv2_split1_c_gaussian_blur', 'VQAv2_split1_c_spatter', 'VQAv2_split1_c_saturate'] + }, +) +scenario = build_cl_scenario( + da_scenario=scenario, + target_datasets_name=['VQAv2_split2'], + num_classes_per_task=20, + max_num_tasks=30, + data_dirs={ + 'VQAv2_split2': '/data/zql/datasets/vqav2vv' # NOTE: trick, see vqav2.py (the implementation of VQAv2 dataset) + }, + sanity_check=True +) + +from experiments.elasticdnn.vilt.online.vqa_cl.model import ElasticDNN_VQAOnlineModel +elasticfm_model = ElasticDNN_VQAOnlineModel('cls', init_online_model( + # 'experiments/elasticdnn/vit_b_16/offline/fm_to_md/results/cls_md_index.py/20230529/star_999997-154037-only_prune_mlp/models/fm_best.pt', + # 'experiments/elasticdnn/vit_b_16/offline/fm_to_md/results/cls_md_index.py/20230529/star_999997-154037-only_prune_mlp/models/md_best.pt', + 'experiments/elasticdnn/vilt/offline/fm_to_md/vqa/results/vqa_w_fbs_index.py/20230731/999999-095720-trial/models/fm_best.pt', + 'experiments/elasticdnn/vilt/offline/fm_to_md/vqa/results/vqa_w_fbs_index.py/20230731/999999-095720-trial/models/md_best.pt', + 'cls', __file__ +), device, { + 'md_to_fm_alpha': 0.1, + 'fm_to_md_alpha': 0.001 +}) + +da_alg = GEMAlg +from experiments.elasticdnn.vilt.online.vqa_cl.model import VQAOnlineGEMModel +da_model = VQAOnlineGEMModel +da_alg_hyp = { + 'train_batch_size': 64, + 'val_batch_size': 256, + 'num_workers': 0, + 'optimizer': 'AdamW', + 'optimizer_args': {'lr': 3e-3, 'betas': [0.9, 0.999], 'weight_decay': 0.0}, + 'scheduler': '', + 'scheduler_args': {}, + 'num_iters': 100 * 8, + 'val_freq': 20 * 8, + # 'num_iters': 1, + # 'val_freq': 1, + 'n_memories': 64, + 'n_inputs': 3 * 224 * 224, + 'margin': 0.5, + 'num_my_iters': 1, + 'sd_sparsity': sd_sparsity, +} + + +elasticfm_cl( + [app_name], + [scenario], + [elasticfm_model], + [da_alg], + [da_alg_hyp], + [da_model], + device, + settings, + __file__, + sys.argv[1] +) diff --git a/experiments/elasticdnn/vilt/test.py b/experiments/elasticdnn/vilt/test.py new file mode 100644 index 0000000000000000000000000000000000000000..6b1f8ef6672640552747a386cc4339f7d2b8c18f --- /dev/null +++ b/experiments/elasticdnn/vilt/test.py @@ -0,0 +1,5 @@ +from data import get_dataset + + +d = get_dataset('VQAv2_split1', '/data/zql/datasets/vqav2', 'train') +print(d[0]) diff --git a/experiments/elasticdnn/vit_b_16/offline/fm_lora/cls/cls.py b/experiments/elasticdnn/vit_b_16/offline/fm_lora/cls/cls.py new file mode 100644 index 0000000000000000000000000000000000000000..f8c71c91346454e86af13e2b2187fc3a29175391 --- /dev/null +++ b/experiments/elasticdnn/vit_b_16/offline/fm_lora/cls/cls.py @@ -0,0 +1,95 @@ +import torch +from methods.elasticdnn.api.model import ElasticDNN_OfflineClsFMModel, ElasticDNN_OfflineClsMDModel +from methods.elasticdnn.api.algs.fm_lora import ElasticDNN_FMLoRAAlg +from methods.elasticdnn.model.base import ElasticDNNUtil +from methods.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from methods.elasticdnn.pipeline.offline.fm_lora.vit import FMLoRA_ViT_Util +from methods.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from methods.elasticdnn.pipeline.offline.fm_to_md.vit import FM_to_MD_ViT_Util +from methods.elasticdnn.model.vit import ElasticViTUtil +from utils.dl.common.model import LayerActivation, get_module +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +import torch.nn.functional as F + + +class ElasticDNN_ViT_OfflineClsFMModel(ElasticDNN_OfflineClsFMModel): + def generate_md_by_reducing_width(self, reducing_width_ratio, samples: torch.Tensor): + return FM_to_MD_ViT_Util().init_md_from_fm_by_reducing_width_with_perf_test(self.models_dict['main'], + reducing_width_ratio, samples) + + def get_feature_hook(self) -> LayerActivation: + return LayerActivation(get_module(self.models_dict['main'], 'head'), True, self.device) + + def get_elastic_dnn_util(self) -> ElasticDNNUtil: + return ElasticViTUtil() + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + #print(self.infer(x)) + #print(self.models_dict['main'])) + #print(F.cross_entropy(self.infer(x), y)) + return F.cross_entropy(self.infer(x), y) + + def get_lora_util(self) -> FMLoRA_Util: + return FMLoRA_ViT_Util() + + def get_task_head_params(self): + head = get_module(self.models_dict['main'], 'head') + return list(head.parameters()) + + +class ElasticDNN_ViT_OfflineClsMDModel(ElasticDNN_OfflineClsMDModel): + def get_feature_hook(self) -> LayerActivation: + return LayerActivation(get_module(self.models_dict['main'], 'head'), True, self.device) + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + + return F.cross_entropy(self.infer(x), y) + + +if __name__ == '__main__': + # 1. init scenario + scenario = build_scenario( + source_datasets_name=['GTA5Cls', 'SuperviselyPersonCls'], + target_datasets_order=['CityscapesCls', 'BaiduPersonCls'] * 15, + da_mode='close_set', + data_dirs={ + 'GTA5Cls': '/data/zql/datasets/gta5_for_cls_task', + 'SuperviselyPersonCls': '/data/zql/datasets/supervisely_person_for_cls_task', + 'CityscapesCls': '/data/zql/datasets/cityscapes_for_cls_task', + 'BaiduPersonCls': '/data/zql/datasets/baidu_person_for_cls_task' + }, + ) + + # 2. init model + from dnns.vit import vit_b_16 + fm_models_dict_path = save_models_dict_for_init({ + 'main': vit_b_16(pretrained=True, num_classes=scenario.num_classes) + }, __file__, 'fm_vit_b_16_pretrained') + device = 'cuda' + fm_model = ElasticDNN_ViT_OfflineClsFMModel('fm', fm_models_dict_path, device) + + # 3. init alg + models = { + 'fm': fm_model + } + import sys + fm_lora_alg = ElasticDNN_FMLoRAAlg(models, get_res_save_dir(__file__, tag=sys.argv[0])) + + # 4. run alg + from utils.dl.common.lr_scheduler import get_linear_schedule_with_warmup + fm_lora_alg.run(scenario, hyps={ + 'launch_tbboard': False, + + 'samples_size': (1, 3, 224, 224), + 'ab_r': 8, + 'train_batch_size': 256, + 'val_batch_size': 512, + 'num_workers': 16, + 'optimizer': 'Adam', + 'optimizer_args': {'lr': 5e-4, 'betas': [0.9, 0.999]}, + 'scheduler': 'LambdaLR', + 'scheduler_args': {'lr_lambda': get_linear_schedule_with_warmup(10000, 70000)}, + 'num_iters': 80000, + 'val_freq': 4000 + }) \ No newline at end of file diff --git a/experiments/elasticdnn/vit_b_16/offline/fm_lora/det/det.py b/experiments/elasticdnn/vit_b_16/offline/fm_lora/det/det.py new file mode 100644 index 0000000000000000000000000000000000000000..bcc40dc8e56e915d22971f05035134ea344c0c73 --- /dev/null +++ b/experiments/elasticdnn/vit_b_16/offline/fm_lora/det/det.py @@ -0,0 +1,121 @@ +import torch +from methods.elasticdnn.api.model import ElasticDNN_OfflineDetFMModel, ElasticDNN_OfflineDetMDModel +from methods.elasticdnn.api.algs.fm_lora import ElasticDNN_FMLoRAAlg +from methods.elasticdnn.model.base import ElasticDNNUtil +from methods.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from methods.elasticdnn.pipeline.offline.fm_lora.vit import FMLoRA_ViT_Util +from methods.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from methods.elasticdnn.pipeline.offline.fm_to_md.vit import FM_to_MD_ViT_Util +from methods.elasticdnn.model.vit import ElasticViTUtil +from utils.dl.common.model import LayerActivation, get_module, get_parameter, set_module +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +from utils.common.log import logger +import torch.nn.functional as F +import sys + + +class ElasticDNN_ViT_OfflineDetFMModel(ElasticDNN_OfflineDetFMModel): + def generate_md_by_reducing_width(self, reducing_width_ratio, samples: torch.Tensor): + return FM_to_MD_ViT_Util().init_md_from_fm_by_reducing_width_with_perf_test(self.models_dict['main'], + reducing_width_ratio, samples) + + def get_feature_hook(self) -> LayerActivation: + return LayerActivation(get_module(self.models_dict['main'], 'head'), True, self.device) + + def get_elastic_dnn_util(self) -> ElasticDNNUtil: + return ElasticViTUtil() + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + # return F.cross_entropy(self.infer(x), y) + self.to_train_mode() + return self.models_dict['main'](x, y)['total_loss'] + + def get_lora_util(self) -> FMLoRA_Util: + return FMLoRA_ViT_Util() + + def get_task_head_params(self): + head = get_module(self.models_dict['main'], 'head') + params_name = {k for k, v in head.named_parameters()} + logger.info(f'task head params: {params_name}') + return list(head.parameters()) + + +class ElasticDNN_ViT_OfflineDetMDModel(ElasticDNN_OfflineDetMDModel): + def get_feature_hook(self) -> LayerActivation: + return LayerActivation(get_module(self.models_dict['main'], 'head'), True, self.device) + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + self.to_train_mode() + return self.models_dict['main'](x, y)['total_loss'] + + +if __name__ == '__main__': + from utils.dl.common.env import set_random_seed + set_random_seed(1) + # 1. init scenario + # scenario = build_scenario( + # source_datasets_name=['WI_Mask'], + # target_datasets_order=['MakeML_Mask'] * 10, + # da_mode='close_set', + # data_dirs={ + # 'COCO2017': '/data/zql/datasets/coco2017', + # 'WI_Mask': '/data/zql/datasets/face_mask/WI/Medical mask/Medical mask/Medical Mask/images', + # 'VOC2012': '/data/datasets/VOCdevkit/VOC2012/JPEGImages', + # 'MakeML_Mask': '/data/zql/datasets/face_mask/make_ml/images' + # }, + # ) + scenario = build_scenario( + source_datasets_name=['GTA5Det', 'SuperviselyPersonDet'], + target_datasets_order=['CityscapesDet', 'BaiduPersonDet'] * 15, + da_mode='close_set', + data_dirs={ + 'GTA5Det': '/data/zql/datasets/GTA-ls-copy/GTA5/', + 'SuperviselyPersonDet': '/data/zql/datasets/supervisely_person_full_20230635/Supervisely Person Dataset', + 'CityscapesDet': '', # not ready yet + 'BaiduPersonDet': '' # not ready yet + }, + ) + + # 2. init model + device = 'cuda' + from dnns.vit import vit_b_16 + det_model = vit_b_16(pretrained=True, num_classes=scenario.num_classes) + from dnns.yolov3.vit_yolov3 import make_vit_yolov3 + det_model = make_vit_yolov3(det_model, torch.rand((1, 3, 224, 224)), 16, 768, scenario.num_classes, use_bigger_fpns=0, + use_multi_layer_feature=False, init_head=True) + + fm_models_dict_path = save_models_dict_for_init({ + 'main': det_model + }, __file__, 'fm_vit_b_16_pretrained_with_det_head') + + fm_model = ElasticDNN_ViT_OfflineDetFMModel('fm', fm_models_dict_path, device, scenario.num_classes) + + # 3. init alg + models = { + 'fm': fm_model + } + fm_lora_alg = ElasticDNN_FMLoRAAlg(models, get_res_save_dir(__file__, sys.argv[1])) + + from PIL import ImageFile + ImageFile.LOAD_TRUNCATED_IMAGES = True + + # 4. run alg + from utils.dl.common.lr_scheduler import get_linear_schedule_with_warmup + fm_lora_alg.run(scenario, hyps={ + 'launch_tbboard': False, + + 'samples_size': (1, 3, 224, 224), + 'ab_r': 8, + 'train_batch_size': 32, + 'val_batch_size': 16, + 'num_workers': 16, + 'optimizer': 'AdamW', + 'optimizer_args': {'lr': 1e-4, 'betas': [0.9, 0.999]}, + 'scheduler': 'LambdaLR', + 'scheduler_args': {'lr_lambda': get_linear_schedule_with_warmup(10000, 310000)}, + 'num_iters': 320000, + 'val_freq': 400, + + 'fm_lora_ckpt_path': 'experiments/elasticdnn/vit_b_16/offline/fm_lora/cls/results/cls.py/20230607/999995-234355-trial/models/fm_best.pt' + }) \ No newline at end of file diff --git a/experiments/elasticdnn/vit_b_16/offline/fm_lora/seg/seg.py b/experiments/elasticdnn/vit_b_16/offline/fm_lora/seg/seg.py new file mode 100644 index 0000000000000000000000000000000000000000..b5f352f52471cc786dc7194450c653680f57773f --- /dev/null +++ b/experiments/elasticdnn/vit_b_16/offline/fm_lora/seg/seg.py @@ -0,0 +1,103 @@ +import torch +from methods.elasticdnn.api.model import ElasticDNN_OfflineSegFMModel, ElasticDNN_OfflineSegMDModel +from methods.elasticdnn.api.algs.fm_lora import ElasticDNN_FMLoRAAlg +from methods.elasticdnn.model.base import ElasticDNNUtil +from methods.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from methods.elasticdnn.pipeline.offline.fm_lora.vit import FMLoRA_ViT_Util +from methods.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from methods.elasticdnn.pipeline.offline.fm_to_md.vit import FM_to_MD_ViT_Util +from methods.elasticdnn.model.vit import ElasticViTUtil +from utils.dl.common.model import LayerActivation, get_module, get_parameter, set_module +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +import torch.nn.functional as F + + +class ElasticDNN_ViT_OfflineSegFMModel(ElasticDNN_OfflineSegFMModel): + def generate_md_by_reducing_width(self, reducing_width_ratio, samples: torch.Tensor): + return FM_to_MD_ViT_Util().init_md_from_fm_by_reducing_width_with_perf_test(self.models_dict['main'], + reducing_width_ratio, samples) + + def get_feature_hook(self) -> LayerActivation: + return LayerActivation(get_module(self.models_dict['main'], 'head'), True, self.device) + + def get_elastic_dnn_util(self) -> ElasticDNNUtil: + return ElasticViTUtil() + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + return F.cross_entropy(self.infer(x), y) + + def get_lora_util(self) -> FMLoRA_Util: + return FMLoRA_ViT_Util() + + def get_task_head_params(self): + head = get_module(self.models_dict['main'], 'head') + return list(head.parameters()) + + +class ElasticDNN_ViT_OfflineSegMDModel(ElasticDNN_OfflineSegMDModel): + def get_feature_hook(self) -> LayerActivation: + return LayerActivation(get_module(self.models_dict['main'], 'head'), True, self.device) + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + return F.cross_entropy(self.infer(x), y) + + +if __name__ == '__main__': + # 1. init scenario + scenario = build_scenario( + source_datasets_name=['GTA5', 'SuperviselyPerson'], + target_datasets_order=['Cityscapes', 'BaiduPerson'] * 10, + da_mode='close_set', + data_dirs={ + 'GTA5': '/data/zql/datasets/GTA-ls-copy/GTA5', + 'SuperviselyPerson': '/data/zql/datasets/supervisely_person/Supervisely Person Dataset', + 'Cityscapes': '/data/zql/datasets/cityscape/', + 'BaiduPerson': '/data/zql/datasets/baidu_person/clean_images/' + }, + ) + + # 2. init model + device = 'cuda' + from dnns.vit import vit_b_16 + seg_model = vit_b_16(pretrained=True, num_classes=scenario.num_classes) + from dnns.deeplabv3.head import DecoderLinear + head = DecoderLinear(scenario.num_classes, 16, 768, (224, 224)).to(device) + + from types import MethodType + from timm.models.vision_transformer import VisionTransformer + def forward_head(self, x, pre_logits: bool = False): + return self.head(x) + VisionTransformer.forward_head = MethodType(forward_head, seg_model) + fm_models_dict_path = save_models_dict_for_init({ + 'main': seg_model + }, __file__, 'fm_vit_b_16_pretrained_with_seg_head') + + fm_model = ElasticDNN_ViT_OfflineSegFMModel('fm', fm_models_dict_path, device, scenario.num_classes) + + # 3. init alg + models = { + 'fm': fm_model + } + fm_lora_alg = ElasticDNN_FMLoRAAlg(models, get_res_save_dir(__file__)) + + from PIL import ImageFile + ImageFile.LOAD_TRUNCATED_IMAGES = True + + # 4. run alg + from utils.dl.common.lr_scheduler import get_linear_schedule_with_warmup + fm_lora_alg.run(scenario, hyps={ + 'launch_tbboard': False, + + 'samples_size': (1, 3, 224, 224), + 'ab_r': 8, + 'train_batch_size': 16, + 'val_batch_size': 256, + 'num_workers': 16, + 'optimizer': 'Adam', + 'optimizer_args': {'lr': 5e-4, 'betas': [0.9, 0.999]}, + 'scheduler': 'LambdaLR', + 'scheduler_args': {'lr_lambda': get_linear_schedule_with_warmup(10000, 70000)}, + 'num_iters': 80000, + 'val_freq': 4000 + }) \ No newline at end of file diff --git a/experiments/elasticdnn/vit_b_16/offline/fm_to_md/cls/cls_md_index.py b/experiments/elasticdnn/vit_b_16/offline/fm_to_md/cls/cls_md_index.py new file mode 100644 index 0000000000000000000000000000000000000000..5e94e900262772466583f2f08adb91dfd38ff46d --- /dev/null +++ b/experiments/elasticdnn/vit_b_16/offline/fm_to_md/cls/cls_md_index.py @@ -0,0 +1,243 @@ +import torch +import sys +from torch import nn +from dnns.vit import make_softmax_prunable +from methods.elasticdnn.api.model import ElasticDNN_OfflineClsFMModel, ElasticDNN_OfflineClsMDModel +# from methods.elasticdnn.api.algs.md_pretraining_w_fbs import ElasticDNN_MDPretrainingWFBSAlg +from methods.elasticdnn.model.base import ElasticDNNUtil +from methods.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from methods.elasticdnn.pipeline.offline.fm_to_md.vit import FM_to_MD_ViT_Util +from methods.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from methods.elasticdnn.pipeline.offline.fm_lora.vit import FMLoRA_ViT_Util +from methods.elasticdnn.model.vit import ElasticViTUtil +from utils.dl.common.model import LayerActivation, get_module, get_parameter +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +from utils.dl.common.loss import CrossEntropyLossSoft +import torch.nn.functional as F + + +class ElasticDNN_ViT_OfflineClsFMModel(ElasticDNN_OfflineClsFMModel): + def generate_md_by_reducing_width(self, reducing_width_ratio, samples: torch.Tensor): + return FM_to_MD_ViT_Util().init_md_from_fm_by_reducing_width_with_perf_test(self.models_dict['main'], + reducing_width_ratio, samples).to(self.device) + + def get_feature_hook(self) -> LayerActivation: + return LayerActivation(get_module(self.models_dict['main'], 'head'), True, self.device) + + def get_elastic_dnn_util(self) -> ElasticDNNUtil: + return ElasticViTUtil() + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + return F.cross_entropy(self.infer(x), y) + + def get_lora_util(self) -> FMLoRA_Util: + return FMLoRA_ViT_Util() + + def get_task_head_params(self): + head = get_module(self.models_dict['main'], 'head') + return list(head.parameters()) + + +class ElasticDNN_ViT_OfflineClsMDModel(ElasticDNN_OfflineClsMDModel): + def __init__(self, name: str, models_dict_path: str, device: str): + super().__init__(name, models_dict_path, device) + + self.distill_criterion = CrossEntropyLossSoft() + + def get_feature_hook(self) -> LayerActivation: + return LayerActivation(get_module(self.models_dict['main'], 'head'), True, self.device) + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + return F.cross_entropy(self.infer(x), y) + + def get_distill_loss(self, student_output, teacher_output): + return self.distill_criterion(student_output, teacher_output) + + def get_matched_param_of_fm(self, self_param_name, fm: nn.Module): + # only between qkv.weight, norm.weight/bias + if any([k in self_param_name for k in ['fbs', 'cls_token', 'pos_embed']]): + return None + + p = get_parameter(self.models_dict['main'], self_param_name) + if p.dim() == 0: + return None + elif p.dim() == 1 and 'norm' in self_param_name and 'weight' in self_param_name: + return get_parameter(fm, self_param_name) + + # 1. xx.qkv.to_qkv.yy to xx.qkv.qkv.aa and xx.qkv.abs.zz + # if 'qkv.weight' in self_param_name: + # ss = self_param_name.split('.') + + # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv' + # fm_qkv = get_module(fm, fm_qkv_name) + + # fm_abs_name = '.'.join(ss[0: -2]) + '.abs' + # fm_abs = get_module(fm, fm_abs_name) + + # return torch.cat([ + # fm_qkv.weight.data, # task-agnositc params + # torch.cat([(_abs[0].weight.T @ _abs[1].weight.T).T for _abs in fm_abs], dim=0) # task-specific params (LoRA) + # ], dim=0) + + # # elif 'to_qkv.bias' in self_param_name: + # # ss = self_param_name.split('.') + + # # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv.bias' + # # return get_parameter(fm, fm_qkv_name) + + # elif 'mlp.fc1' in self_param_name and 'weight' in self_param_name: + # fm_param_name = self_param_name.replace('.linear', '') + # return get_parameter(fm, fm_param_name) + + # elif 'mlp.fc2' in self_param_name and 'weight' in self_param_name: + # fm_param_name = self_param_name + # return get_parameter(fm, fm_param_name) + + # else: + # # return get_parameter(fm, self_param_name) + # return None + + if 'qkv.weight' in self_param_name: + ss = self_param_name.split('.') + + fm_qkv_name = '.'.join(ss[0: -1]) + '.qkv' + fm_qkv = get_module(fm, fm_qkv_name) + + fm_abs_name = '.'.join(ss[0: -1]) + '.abs' + fm_abs = get_module(fm, fm_abs_name) + + return torch.cat([ + fm_qkv.weight.data, # task-agnositc params + torch.cat([(_abs[0].weight.T @ _abs[1].weight.T).T for _abs in fm_abs], dim=0) # task-specific params (LoRA) + ], dim=0) + + # elif 'to_qkv.bias' in self_param_name: + # ss = self_param_name.split('.') + + # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv.bias' + # return get_parameter(fm, fm_qkv_name) + + elif 'mlp.fc1' in self_param_name and 'weight' in self_param_name: + fm_param_name = self_param_name.replace('.linear', '') + return get_parameter(fm, fm_param_name) + + elif 'mlp.fc2' in self_param_name and 'weight' in self_param_name: + fm_param_name = self_param_name + res = get_parameter(fm, fm_param_name) + # print('mlp fc2 debug', fm_param_name, res is None) + return res + + else: + # return get_parameter(fm, self_param_name) + return None + + # def get_matched_param_of_fm(self, self_param_name, fm: nn.Module): + # if any([k in self_param_name for k in ['fbs', 'cls_token', 'pos_embed']]): + # return None + + # # 1. xx.qkv.to_qkv.yy to xx.qkv.qkv.aa and xx.qkv.abs.zz + # if 'to_qkv.weight' in self_param_name: + # ss = self_param_name.split('.') + + # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv' + # fm_qkv = get_module(fm, fm_qkv_name) + + # fm_abs_name = '.'.join(ss[0: -2]) + '.abs' + # fm_abs = get_module(fm, fm_abs_name) + + # return torch.cat([ + # fm_qkv.weight.data, # task-agnositc params + # torch.cat([(_abs[0].weight.T @ _abs[1].weight.T).T for _abs in fm_abs], dim=0) # task-specific params (LoRA) + # ], dim=0) + + # elif 'to_qkv.bias' in self_param_name: + # ss = self_param_name.split('.') + + # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv.bias' + # return get_parameter(fm, fm_qkv_name) + + # elif 'mlp.fc1' in self_param_name: + # fm_param_name = self_param_name.replace('.linear', '') + # return get_parameter(fm, fm_param_name) + + # else: + # return get_parameter(fm, self_param_name) + + +if __name__ == '__main__': + from utils.dl.common.env import set_random_seed + set_random_seed(1) + + # 1. init model + from dnns.vit import vit_b_16 + fm_models_dict_path = 'experiments/elasticdnn/vit_b_16/offline/fm_to_md/cls/results/cls_md_wo_fbs.py/20230611/999998-142441-new_md_structure/models/fm_best.pt' + fm_models_dict_path = save_models_dict_for_init(torch.load(fm_models_dict_path), __file__, 'fm_vit_b_16_cls_lora') + pretrained_md_models_dict_path = 'experiments/elasticdnn/vit_b_16/offline/fm_to_md/cls/results/cls_md_wo_fbs.py/20230611/999998-142441-new_md_structure/models/md_best.pt' + md_models_dict = torch.load(pretrained_md_models_dict_path) + md_models_dict_path = save_models_dict_for_init(md_models_dict, __file__, 'md_vit_b_16_cls_pretrained_wo_fbs') + device = 'cuda' + + fm_model = ElasticDNN_ViT_OfflineClsFMModel('fm', fm_models_dict_path, device) + md_model = ElasticDNN_ViT_OfflineClsMDModel('md', md_models_dict_path, device) + + # 2. init alg + models = { + 'fm': fm_model, + 'md': md_model + } + from methods.elasticdnn.api.algs.md_pretraining_index_v2_train_index_and_md import ElasticDNN_MDPretrainingIndexAlg + + fm_to_md_alg = ElasticDNN_MDPretrainingIndexAlg(models, get_res_save_dir(__file__, sys.argv[0])) + + # 3. init scenario + scenario = build_scenario( + source_datasets_name=['GTA5Cls', 'SuperviselyPersonCls'], + target_datasets_order=['CityscapesCls', 'BaiduPersonCls'] * 15, + da_mode='close_set', + data_dirs={ + 'GTA5Cls': '/data/zql/datasets/gta5_for_cls_task', + 'SuperviselyPersonCls': '/data/zql/datasets/supervisely_person_for_cls_task', + 'CityscapesCls': '/data/zql/datasets/cityscapes_for_cls_task', + 'BaiduPersonCls': '/data/zql/datasets/baidu_person_for_cls_task' + }, + ) + + from utils.dl.common.lr_scheduler import get_linear_schedule_with_warmup + fm_to_md_alg.run(scenario, hyps={ + 'launch_tbboard': False, + + 'samples_size': (1, 3, 224, 224), + + 'FBS_r': 16, + 'FBS_ignore_layers': [], + + 'train_batch_size': 128, + 'val_batch_size': 512, + 'num_workers': 16, + 'optimizer': 'AdamW', + # 'optimizer_args': {'lr': 1e-4, 'betas': [0.9, 0.999], 'weight_decay': 0.01}, + 'indexes_optimizer_args': {'lr': 3e-3, 'betas': [0.9, 0.999], 'weight_decay': 0.1}, + # 'scheduler': 'StepLR', + # 'scheduler_args': {'step_size': 20000, 'gamma': 0.1}, + # 'optimizer': 'AdamW', + # 'optimizer_args': {'lr': 1e-4, 'betas': [0.9, 0.999], 'weight_decay': 0.01}, + 'optimizer_args': {'lr': 1e-5, 'betas': [0.9, 0.999], 'weight_decay': 0.01}, + + 'scheduler': 'LambdaLR', + 'scheduler_args': {'lr_lambda': get_linear_schedule_with_warmup(10000, 70000)}, + + 'max_sparsity': 0.9, + 'min_sparsity': 0.0, + 'num_iters': 60000, + 'val_freq': 500, + 'index_loss_weight': 1e-4, + 'l1_reg_loss_weight': 1e-9, + 'val_num_sparsities': 4, + + 'bn_cal_num_iters': 0, + + 'index_init': 'zero', + 'index_guided_linear_comb_split_size': 512 + }) + \ No newline at end of file diff --git a/experiments/elasticdnn/vit_b_16/offline/fm_to_md/cls/cls_md_w_fbs.py b/experiments/elasticdnn/vit_b_16/offline/fm_to_md/cls/cls_md_w_fbs.py new file mode 100644 index 0000000000000000000000000000000000000000..1cf9f8ef702e04a1e17ca4613b28df1ba8c064d0 --- /dev/null +++ b/experiments/elasticdnn/vit_b_16/offline/fm_to_md/cls/cls_md_w_fbs.py @@ -0,0 +1,196 @@ +import torch +import sys +from torch import nn +from dnns.vit import make_softmax_prunable +from methods.elasticdnn.api.model import ElasticDNN_OfflineClsFMModel, ElasticDNN_OfflineClsMDModel +from methods.elasticdnn.api.algs.md_pretraining_w_fbs import ElasticDNN_MDPretrainingWFBSAlg +from methods.elasticdnn.model.base import ElasticDNNUtil +from methods.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from methods.elasticdnn.pipeline.offline.fm_to_md.vit import FM_to_MD_ViT_Util +from methods.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from methods.elasticdnn.pipeline.offline.fm_lora.vit import FMLoRA_ViT_Util +from methods.elasticdnn.model.vit import ElasticViTUtil +from utils.dl.common.model import LayerActivation, get_module, get_parameter +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +from utils.dl.common.loss import CrossEntropyLossSoft +import torch.nn.functional as F + + +class ElasticDNN_ViT_OfflineClsFMModel(ElasticDNN_OfflineClsFMModel): + def generate_md_by_reducing_width(self, reducing_width_ratio, samples: torch.Tensor): + return FM_to_MD_ViT_Util().init_md_from_fm_by_reducing_width_with_perf_test(self.models_dict['main'], + reducing_width_ratio, samples).to(self.device) + + def get_feature_hook(self) -> LayerActivation: + return LayerActivation(get_module(self.models_dict['main'], 'head'), True, self.device) + + def get_elastic_dnn_util(self) -> ElasticDNNUtil: + return ElasticViTUtil() + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + return F.cross_entropy(self.infer(x), y) + + def get_lora_util(self) -> FMLoRA_Util: + return FMLoRA_ViT_Util() + + def get_task_head_params(self): + head = get_module(self.models_dict['main'], 'head') + return list(head.parameters()) + + +class ElasticDNN_ViT_OfflineClsMDModel(ElasticDNN_OfflineClsMDModel): + def __init__(self, name: str, models_dict_path: str, device: str): + super().__init__(name, models_dict_path, device) + + self.distill_criterion = CrossEntropyLossSoft() + + def get_feature_hook(self) -> LayerActivation: + return LayerActivation(get_module(self.models_dict['main'], 'head'), True, self.device) + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + return F.cross_entropy(self.infer(x), y) + + def get_distill_loss(self, student_output, teacher_output): + return self.distill_criterion(student_output, teacher_output) + + def get_matched_param_of_fm(self, self_param_name, fm: nn.Module): + # only between qkv.weight, norm.weight/bias + if any([k in self_param_name for k in ['fbs', 'cls_token', 'pos_embed']]): + return None + + p = get_parameter(self.models_dict['main'], self_param_name) + if p.dim() == 0: + return None + elif p.dim() == 1 and 'norm' in self_param_name: + return get_parameter(fm, self_param_name) + + # 1. xx.qkv.to_qkv.yy to xx.qkv.qkv.aa and xx.qkv.abs.zz + if 'to_qkv.weight' in self_param_name: + ss = self_param_name.split('.') + + fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv' + fm_qkv = get_module(fm, fm_qkv_name) + + fm_abs_name = '.'.join(ss[0: -2]) + '.abs' + fm_abs = get_module(fm, fm_abs_name) + + return torch.cat([ + fm_qkv.weight.data, # task-agnositc params + torch.cat([(_abs[0].weight.T @ _abs[1].weight.T).T for _abs in fm_abs], dim=0) # task-specific params (LoRA) + ], dim=0) + + # elif 'to_qkv.bias' in self_param_name: + # ss = self_param_name.split('.') + + # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv.bias' + # return get_parameter(fm, fm_qkv_name) + + elif 'mlp.fc1' in self_param_name and 'weight' in self_param_name: + fm_param_name = self_param_name.replace('.linear', '') + return get_parameter(fm, fm_param_name) + + elif 'mlp.fc2' in self_param_name and 'weight' in self_param_name: + fm_param_name = self_param_name + return get_parameter(fm, fm_param_name) + + else: + # return get_parameter(fm, self_param_name) + return None + + # def get_matched_param_of_fm(self, self_param_name, fm: nn.Module): + # if any([k in self_param_name for k in ['fbs', 'cls_token', 'pos_embed']]): + # return None + + # # 1. xx.qkv.to_qkv.yy to xx.qkv.qkv.aa and xx.qkv.abs.zz + # if 'to_qkv.weight' in self_param_name: + # ss = self_param_name.split('.') + + # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv' + # fm_qkv = get_module(fm, fm_qkv_name) + + # fm_abs_name = '.'.join(ss[0: -2]) + '.abs' + # fm_abs = get_module(fm, fm_abs_name) + + # return torch.cat([ + # fm_qkv.weight.data, # task-agnositc params + # torch.cat([(_abs[0].weight.T @ _abs[1].weight.T).T for _abs in fm_abs], dim=0) # task-specific params (LoRA) + # ], dim=0) + + # elif 'to_qkv.bias' in self_param_name: + # ss = self_param_name.split('.') + + # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv.bias' + # return get_parameter(fm, fm_qkv_name) + + # elif 'mlp.fc1' in self_param_name: + # fm_param_name = self_param_name.replace('.linear', '') + # return get_parameter(fm, fm_param_name) + + # else: + # return get_parameter(fm, self_param_name) + + +if __name__ == '__main__': + from utils.dl.common.env import set_random_seed + set_random_seed(1) + + # 1. init model + from dnns.vit import vit_b_16 + fm_models_dict_path = 'experiments/elasticdnn/vit_b_16/offline/fm_to_md/cls/results/cls_md_wo_fbs.py/20230611/999998-142441-new_md_structure/models/fm_best.pt' + fm_models_dict_path = save_models_dict_for_init(torch.load(fm_models_dict_path), __file__, 'fm_vit_b_16_cls_lora') + pretrained_md_models_dict_path = 'experiments/elasticdnn/vit_b_16/offline/fm_to_md/cls/results/cls_md_wo_fbs.py/20230611/999998-142441-new_md_structure/models/md_best.pt' + md_models_dict = torch.load(pretrained_md_models_dict_path) + md_models_dict_path = save_models_dict_for_init(md_models_dict, __file__, 'md_vit_b_16_cls_pretrained_wo_fbs') + device = 'cuda' + + fm_model = ElasticDNN_ViT_OfflineClsFMModel('fm', fm_models_dict_path, device) + md_model = ElasticDNN_ViT_OfflineClsMDModel('md', md_models_dict_path, device) + + # 2. init alg + models = { + 'fm': fm_model, + 'md': md_model + } + fm_to_md_alg = ElasticDNN_MDPretrainingWFBSAlg(models, get_res_save_dir(__file__, sys.argv[1])) + + # 3. init scenario + scenario = build_scenario( + source_datasets_name=['GTA5Cls', 'SuperviselyPersonCls'], + target_datasets_order=['CityscapesCls', 'BaiduPersonCls'] * 15, + da_mode='close_set', + data_dirs={ + 'GTA5Cls': '/data/zql/datasets/gta5_for_cls_task', + 'SuperviselyPersonCls': '/data/zql/datasets/supervisely_person_for_cls_task', + 'CityscapesCls': '/data/zql/datasets/cityscapes_for_cls_task', + 'BaiduPersonCls': '/data/zql/datasets/baidu_person_for_cls_task' + }, + ) + + from utils.dl.common.lr_scheduler import get_linear_schedule_with_warmup + fm_to_md_alg.run(scenario, hyps={ + 'launch_tbboard': False, + + 'samples_size': (1, 3, 224, 224), + 'generate_md_width_ratio': 4, + + 'FBS_r': 16, + 'FBS_ignore_layers': [], + + 'train_batch_size': 128, + 'val_batch_size': 512, + 'num_workers': 16, + 'optimizer': 'AdamW', + 'optimizer_args': {'lr': 1e-4, 'betas': [0.9, 0.999], 'weight_decay': 0.01}, + 'scheduler': 'LambdaLR', + 'scheduler_args': {'lr_lambda': get_linear_schedule_with_warmup(10000, 70000)}, + 'num_iters': 80000, + 'val_freq': 4000, + 'max_sparsity': 0.8, + 'min_sparsity': 0.0, + 'l1_reg_loss_weight': 1e-9, + 'val_num_sparsities': 4, + + 'bn_cal_num_iters': 0 + }) + \ No newline at end of file diff --git a/experiments/elasticdnn/vit_b_16/offline/fm_to_md/cls/cls_md_wo_fbs.py b/experiments/elasticdnn/vit_b_16/offline/fm_to_md/cls/cls_md_wo_fbs.py new file mode 100644 index 0000000000000000000000000000000000000000..d75fcf7826b60b29302b8eff90ca3c60b69c601f --- /dev/null +++ b/experiments/elasticdnn/vit_b_16/offline/fm_to_md/cls/cls_md_wo_fbs.py @@ -0,0 +1,273 @@ +import torch +import sys +from torch import nn +from dnns.vit import make_softmax_prunable +from methods.elasticdnn.api.model import ElasticDNN_OfflineClsFMModel, ElasticDNN_OfflineClsMDModel +from methods.elasticdnn.api.algs.md_pretraining_wo_fbs import ElasticDNN_MDPretrainingWoFBSAlg +from methods.elasticdnn.model.base import ElasticDNNUtil +from methods.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from methods.elasticdnn.pipeline.offline.fm_to_md.vit import FM_to_MD_ViT_Util +from methods.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from methods.elasticdnn.pipeline.offline.fm_lora.vit import FMLoRA_ViT_Util +from methods.elasticdnn.model.vit import ElasticViTUtil +from utils.dl.common.model import LayerActivation, get_module, get_parameter +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +from utils.dl.common.loss import CrossEntropyLossSoft +import torch.nn.functional as F + + +# class ElasticDNN_ViT_OfflineClsFMModel(ElasticDNN_OfflineClsFMModel): +# def generate_md_by_reducing_width(self, reducing_width_ratio, samples: torch.Tensor): +# return FM_to_MD_ViT_Util().init_md_from_fm_by_reducing_width_with_perf_test(self.models_dict['main'], +# reducing_width_ratio, samples).to(self.device) + +# def get_feature_hook(self) -> LayerActivation: +# return LayerActivation(get_module(self.models_dict['main'], 'head'), True, self.device) + +# def get_elastic_dnn_util(self) -> ElasticDNNUtil: +# return ElasticViTUtil() + +# def forward_to_get_task_loss(self, x, y, *args, **kwargs): +# return F.cross_entropy(self.infer(x), y) + +# def get_lora_util(self) -> FMLoRA_Util: +# return FMLoRA_ViT_Util() + +# def get_task_head_params(self): +# head = get_module(self.models_dict['main'], 'head') +# return list(head.parameters()) + + +# class ElasticDNN_ViT_OfflineClsMDModel(ElasticDNN_OfflineClsMDModel): +# def __init__(self, name: str, models_dict_path: str, device: str): +# super().__init__(name, models_dict_path, device) + +# self.distill_criterion = CrossEntropyLossSoft() + +# def get_feature_hook(self) -> LayerActivation: +# return LayerActivation(get_module(self.models_dict['main'], 'head'), True, self.device) + +# def forward_to_get_task_loss(self, x, y, *args, **kwargs): +# return F.cross_entropy(self.infer(x), y) + +# def get_distill_loss(self, student_output, teacher_output): +# return self.distill_criterion(student_output, teacher_output) + +# def get_matched_param_of_fm(self, self_param_name, fm: nn.Module): +# # only between qkv.weight, norm.weight/bias +# if any([k in self_param_name for k in ['fbs', 'cls_token', 'pos_embed']]): +# return None + +# p = get_parameter(self.models_dict['main'], self_param_name) +# if p.dim() == 0: +# return None +# elif p.dim() == 1 and 'norm' in self_param_name: +# return get_parameter(fm, self_param_name) + +# # 1. xx.qkv.to_qkv.yy to xx.qkv.qkv.aa and xx.qkv.abs.zz +# if 'to_qkv.weight' in self_param_name: +# ss = self_param_name.split('.') + +# fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv' +# fm_qkv = get_module(fm, fm_qkv_name) + +# fm_abs_name = '.'.join(ss[0: -2]) + '.abs' +# fm_abs = get_module(fm, fm_abs_name) + +# return torch.cat([ +# fm_qkv.weight.data, # task-agnositc params +# torch.cat([(_abs[0].weight.T @ _abs[1].weight.T).T for _abs in fm_abs], dim=0) # task-specific params (LoRA) +# ], dim=0) + +# # elif 'to_qkv.bias' in self_param_name: +# # ss = self_param_name.split('.') + +# # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv.bias' +# # return get_parameter(fm, fm_qkv_name) + +# # elif 'mlp.fc1' in self_param_name: +# # fm_param_name = self_param_name.replace('.linear', '') +# # return get_parameter(fm, fm_param_name) + +# else: +# # return get_parameter(fm, self_param_name) +# return None + +# # def get_matched_param_of_fm(self, self_param_name, fm: nn.Module): +# # if any([k in self_param_name for k in ['fbs', 'cls_token', 'pos_embed']]): +# # return None + +# # # 1. xx.qkv.to_qkv.yy to xx.qkv.qkv.aa and xx.qkv.abs.zz +# # if 'to_qkv.weight' in self_param_name: +# # ss = self_param_name.split('.') + +# # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv' +# # fm_qkv = get_module(fm, fm_qkv_name) + +# # fm_abs_name = '.'.join(ss[0: -2]) + '.abs' +# # fm_abs = get_module(fm, fm_abs_name) + +# # return torch.cat([ +# # fm_qkv.weight.data, # task-agnositc params +# # torch.cat([(_abs[0].weight.T @ _abs[1].weight.T).T for _abs in fm_abs], dim=0) # task-specific params (LoRA) +# # ], dim=0) + +# # elif 'to_qkv.bias' in self_param_name: +# # ss = self_param_name.split('.') + +# # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv.bias' +# # return get_parameter(fm, fm_qkv_name) + +# # elif 'mlp.fc1' in self_param_name: +# # fm_param_name = self_param_name.replace('.linear', '') +# # return get_parameter(fm, fm_param_name) + +# # else: +# # return get_parameter(fm, self_param_name) + + + +class ElasticDNN_ViT_OfflineClsFMModel(ElasticDNN_OfflineClsFMModel): + def generate_md_by_reducing_width(self, reducing_width_ratio, samples: torch.Tensor): + return FM_to_MD_ViT_Util().init_md_from_fm_by_reducing_width_with_perf_test(self.models_dict['main'], + reducing_width_ratio, samples).to(self.device) + + def get_feature_hook(self) -> LayerActivation: + return LayerActivation(get_module(self.models_dict['main'], 'head'), True, self.device) + + def get_elastic_dnn_util(self) -> ElasticDNNUtil: + return ElasticViTUtil() + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + return F.cross_entropy(self.infer(x), y) + + def get_lora_util(self) -> FMLoRA_Util: + return FMLoRA_ViT_Util() + + def get_task_head_params(self): + head = get_module(self.models_dict['main'], 'head') + return list(head.parameters()) + + +class ElasticDNN_ViT_OfflineClsMDModel(ElasticDNN_OfflineClsMDModel): + def __init__(self, name: str, models_dict_path: str, device: str): + super().__init__(name, models_dict_path, device) + + self.distill_criterion = CrossEntropyLossSoft() + + def get_feature_hook(self) -> LayerActivation: + return LayerActivation(get_module(self.models_dict['main'], 'head'), True, self.device) + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + return F.cross_entropy(self.infer(x), y) + + def get_distill_loss(self, student_output, teacher_output): + return self.distill_criterion(student_output, teacher_output) + + def get_matched_param_of_fm(self, self_param_name, fm: nn.Module): + # only between qkv.weight, norm.weight/bias + if any([k in self_param_name for k in ['fbs', 'cls_token', 'pos_embed']]): + return None + + p = get_parameter(self.models_dict['main'], self_param_name) + if p.dim() == 0: + return None + elif p.dim() == 1 and 'norm' in self_param_name and 'weight' in self_param_name: + return get_parameter(fm, self_param_name) + + # 1. xx.qkv.to_qkv.yy to xx.qkv.qkv.aa and xx.qkv.abs.zz + if 'qkv.weight' in self_param_name: + ss = self_param_name.split('.') + + fm_qkv_name = '.'.join(ss[0: -1]) + '.qkv' + fm_qkv = get_module(fm, fm_qkv_name) + + fm_abs_name = '.'.join(ss[0: -1]) + '.abs' + fm_abs = get_module(fm, fm_abs_name) + + return torch.cat([ + fm_qkv.weight.data, # task-agnositc params + torch.cat([(_abs[0].weight.T @ _abs[1].weight.T).T for _abs in fm_abs], dim=0) # task-specific params (LoRA) + ], dim=0) + + # elif 'to_qkv.bias' in self_param_name: + # ss = self_param_name.split('.') + + # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv.bias' + # return get_parameter(fm, fm_qkv_name) + + elif 'mlp.fc1' in self_param_name and 'weight' in self_param_name: + fm_param_name = self_param_name.replace('.linear', '') + return get_parameter(fm, fm_param_name) + + elif 'mlp.fc2' in self_param_name and 'weight' in self_param_name: + fm_param_name = self_param_name + return get_parameter(fm, fm_param_name) + + else: + # return get_parameter(fm, self_param_name) + return None + + + + +if __name__ == '__main__': + from utils.dl.common.env import set_random_seed + set_random_seed(1) + + # 1. init model + from dnns.vit import vit_b_16 + fm_models_dict_path = 'experiments/elasticdnn/vit_b_16/offline/fm_lora/cls/results/cls.py/20230607/999995-234355-trial/models/fm_best.pt' + fm_models_dict = torch.load(fm_models_dict_path) + fm_models_dict['main'] = make_softmax_prunable(fm_models_dict['main']) + fm_models_dict_path = save_models_dict_for_init(fm_models_dict, __file__, 'fm_vit_b_16_cls_lora') + md_models_dict_path = save_models_dict_for_init({ + 'main': -1 + }, __file__, 'md_vit_b_16_none') + device = 'cuda' + + fm_model = ElasticDNN_ViT_OfflineClsFMModel('fm', fm_models_dict_path, device) + md_model = ElasticDNN_ViT_OfflineClsMDModel('md', md_models_dict_path, device) + + # 2. init alg + models = { + 'fm': fm_model, + 'md': md_model + } + fm_to_md_alg = ElasticDNN_MDPretrainingWoFBSAlg(models, get_res_save_dir(__file__, sys.argv[0])) + + # 3. init scenario + scenario = build_scenario( + source_datasets_name=['GTA5Cls', 'SuperviselyPersonCls'], + target_datasets_order=['CityscapesCls', 'BaiduPersonCls'] * 15, + da_mode='close_set', + data_dirs={ + 'GTA5Cls': '/data/zql/datasets/gta5_for_cls_task', + 'SuperviselyPersonCls': '/data/zql/datasets/supervisely_person_for_cls_task', + 'CityscapesCls': '/data/zql/datasets/cityscapes_for_cls_task', + 'BaiduPersonCls': '/data/zql/datasets/baidu_person_for_cls_task' + }, + ) + + from utils.dl.common.lr_scheduler import get_linear_schedule_with_warmup + fm_to_md_alg.run(scenario, hyps={ + 'launch_tbboard': False, + + 'samples_size': (1, 3, 224, 224), + 'generate_md_width_ratio': 4, + + 'train_batch_size': 128, + 'val_batch_size': 512, + 'num_workers': 16, + 'optimizer': 'AdamW', + 'optimizer_args': {'lr': 1e-4, 'betas': [0.9, 0.999], 'weight_decay': 0.01}, + 'scheduler': 'LambdaLR', + 'scheduler_args': {'lr_lambda': get_linear_schedule_with_warmup(10000, 70000)}, + 'num_iters': 80000, + 'val_freq': 4000, + 'distill_loss_weight': 1.0 + }) + + # TODO: + # 1. train MD before inserting FBS? \ No newline at end of file diff --git a/experiments/elasticdnn/vit_b_16/offline/fm_to_md/det/det_md_w_fbs_index.py b/experiments/elasticdnn/vit_b_16/offline/fm_to_md/det/det_md_w_fbs_index.py new file mode 100644 index 0000000000000000000000000000000000000000..8e4036791324603c52dfd09b3a6b59ec08717a4f --- /dev/null +++ b/experiments/elasticdnn/vit_b_16/offline/fm_to_md/det/det_md_w_fbs_index.py @@ -0,0 +1,177 @@ +import torch +import sys +from torch import nn +from methods.elasticdnn.api.model import ElasticDNN_OfflineDetFMModel, ElasticDNN_OfflineDetMDModel +from methods.elasticdnn.api.algs.md_pretraining_wo_fbs import ElasticDNN_MDPretrainingWoFBSAlg +from methods.elasticdnn.model.base import ElasticDNNUtil +from methods.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from methods.elasticdnn.pipeline.offline.fm_to_md.vit import FM_to_MD_ViT_Util +from methods.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from methods.elasticdnn.pipeline.offline.fm_lora.vit import FMLoRA_ViT_Util +from methods.elasticdnn.model.vit import ElasticViTUtil +from utils.dl.common.model import LayerActivation2, get_module, get_parameter +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +from utils.dl.common.loss import CrossEntropyLossSoft +import torch.nn.functional as F +from utils.common.log import logger + + +class ElasticDNN_ViT_OfflineDetFMModel(ElasticDNN_OfflineDetFMModel): + def generate_md_by_reducing_width(self, reducing_width_ratio, samples: torch.Tensor): + return FM_to_MD_ViT_Util().init_md_from_fm_by_reducing_width_with_perf_test(self.models_dict['main'], + reducing_width_ratio, samples) + + def get_feature_hook(self) -> LayerActivation2: + return LayerActivation2(get_module(self.models_dict['main'], 'blocks.11')) + + def get_elastic_dnn_util(self) -> ElasticDNNUtil: + return ElasticViTUtil() + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + # return F.cross_entropy(self.infer(x), y) + self.to_train_mode() + return self.models_dict['main'](x, y)['total_loss'] + + def get_lora_util(self) -> FMLoRA_Util: + return FMLoRA_ViT_Util() + + def get_task_head_params(self): + head = get_module(self.models_dict['main'], 'head') + params_name = {k for k, v in head.named_parameters()} + logger.info(f'task head params: {params_name}') + return list(head.parameters()) + + +class ElasticDNN_ViT_OfflineDetMDModel(ElasticDNN_OfflineDetMDModel): + def get_feature_hook(self) -> LayerActivation2: + return LayerActivation2(get_module(self.models_dict['main'], 'blocks.11')) + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + self.to_train_mode() + return self.models_dict['main'](x, y)['total_loss'] + + def get_distill_loss(self, student_output, teacher_output): + # print(student_output, teacher_output) + return F.mse_loss(student_output, teacher_output) + + def get_matched_param_of_fm(self, self_param_name, fm: nn.Module): + if any([k in self_param_name for k in ['fbs', 'cls_token', 'pos_embed']]): + return None + + p = get_parameter(self.models_dict['main'], self_param_name) + if p.dim() == 0: + return None + elif p.dim() == 1 and 'norm' in self_param_name and 'weight' in self_param_name: + if self_param_name.startswith('norm'): + return None + return get_parameter(fm, self_param_name) + + # 1. xx.qkv.to_qkv.yy to xx.qkv.qkv.aa and xx.qkv.abs.zz + if 'qkv.weight' in self_param_name: + ss = self_param_name.split('.') + + fm_qkv_name = '.'.join(ss[0: -1]) + '.qkv' + fm_qkv = get_module(fm, fm_qkv_name) + + fm_abs_name = '.'.join(ss[0: -1]) + '.abs' + fm_abs = get_module(fm, fm_abs_name) + + return torch.cat([ + fm_qkv.weight.data, # task-agnositc params + torch.cat([(_abs[0].weight.T @ _abs[1].weight.T).T for _abs in fm_abs], dim=0) # task-specific params (LoRA) + ], dim=0) + + # elif 'to_qkv.bias' in self_param_name: + # ss = self_param_name.split('.') + + # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv.bias' + # return get_parameter(fm, fm_qkv_name) + + elif 'mlp.fc1' in self_param_name and 'weight' in self_param_name: + fm_param_name = self_param_name.replace('.linear', '') + return get_parameter(fm, fm_param_name) + + elif 'mlp.fc2' in self_param_name and 'weight' in self_param_name: + fm_param_name = self_param_name + return get_parameter(fm, fm_param_name) + + else: + # return get_parameter(fm, self_param_name) + return None + + +if __name__ == '__main__': + from utils.dl.common.env import set_random_seed + set_random_seed(1) + + # 3. init scenario + scenario = build_scenario( + source_datasets_name=['GTA5Det', 'SuperviselyPersonDet'], + target_datasets_order=['CityscapesDet', 'BaiduPersonDet'] * 10, + da_mode='close_set', + data_dirs={ + 'GTA5Det': '/data/zql/datasets/GTA-ls-copy/GTA5', + 'SuperviselyPersonDet': '/data/zql/datasets/supervisely_person_full_20230635/Supervisely Person Dataset', + 'CityscapesDet': '/data/zql/datasets/cityscape/', + 'BaiduPersonDet': '/data/zql/datasets/baidu_person/clean_images/' + }, + ) + + # 1. init model + # from dnns.deeplabv3.head import modify_forward_head + # modify_forward_head() # TODO: bring a bug + # from dnns.vit import vit_b_16 + fm_models_dict_path = 'experiments/elasticdnn/vit_b_16/offline/fm_to_md/det/results/det_md_wo_fbs.py/20230703/999985-130543/models/fm_best.pt' + fm_models = torch.load(fm_models_dict_path) + fm_models_dict_path = save_models_dict_for_init(fm_models, __file__, 'fm_vit_b_16_det_lora') + md_models_dict_path = save_models_dict_for_init( + torch.load('experiments/elasticdnn/vit_b_16/offline/fm_to_md/det/results/det_md_wo_fbs.py/20230703/999985-130543/models/md_best.pt'), + __file__, 'md_vit_b_16_det_raw_pretrained') + device = 'cuda' + + fm_model = ElasticDNN_ViT_OfflineDetFMModel('fm', fm_models_dict_path, device, scenario.num_classes) + md_model = ElasticDNN_ViT_OfflineDetMDModel('md', md_models_dict_path, device, scenario.num_classes) + + # 2. init alg + models = { + 'fm': fm_model, + 'md': md_model + } + from methods.elasticdnn.api.algs.md_pretraining_index_v2_train_index_and_md import ElasticDNN_MDPretrainingIndexAlg + fm_to_md_alg = ElasticDNN_MDPretrainingIndexAlg(models, get_res_save_dir(__file__, sys.argv[1])) + + from utils.dl.common.lr_scheduler import get_linear_schedule_with_warmup + fm_to_md_alg.run(scenario, hyps={ + 'launch_tbboard': False, + + 'samples_size': (1, 3, 224, 224), + + 'FBS_r': 16, + 'FBS_ignore_layers': [], + + 'train_batch_size': 32, + 'val_batch_size': 32, + 'num_workers': 16, + 'optimizer': 'AdamW', + 'optimizer_args': {'lr': 1e-4, 'betas': [0.9, 0.999]}, + 'scheduler': 'LambdaLR', + 'scheduler_args': {'lr_lambda': get_linear_schedule_with_warmup(10000, 70000)}, + + 'indexes_optimizer_args': {'lr': 3e-3, 'momentum': 0.9, 'weight_decay': 5e-4}, + + 'num_iters': 80000, + 'val_freq': 400, + + 'max_sparsity': 0.9, + 'min_sparsity': 0.0, + 'l1_reg_loss_weight': 1e-9, + 'index_loss_weight': 1e-4, + 'val_num_sparsities': 4, + + 'bn_cal_num_iters': 400, + + 'index_init': 'zero', + 'index_guided_linear_comb_split_size': 512 + }) + \ No newline at end of file diff --git a/experiments/elasticdnn/vit_b_16/offline/fm_to_md/det/det_md_wo_fbs.py b/experiments/elasticdnn/vit_b_16/offline/fm_to_md/det/det_md_wo_fbs.py new file mode 100644 index 0000000000000000000000000000000000000000..bbe70f09181bfa75bf3ba12ca0fc4a3b6ac76c58 --- /dev/null +++ b/experiments/elasticdnn/vit_b_16/offline/fm_to_md/det/det_md_wo_fbs.py @@ -0,0 +1,148 @@ +import torch +import sys +from torch import nn +from methods.elasticdnn.api.model import ElasticDNN_OfflineDetFMModel, ElasticDNN_OfflineDetMDModel +from methods.elasticdnn.api.algs.md_pretraining_wo_fbs import ElasticDNN_MDPretrainingWoFBSAlg +from methods.elasticdnn.model.base import ElasticDNNUtil +from methods.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from methods.elasticdnn.pipeline.offline.fm_to_md.vit import FM_to_MD_ViT_Util +from methods.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from methods.elasticdnn.pipeline.offline.fm_lora.vit import FMLoRA_ViT_Util +from methods.elasticdnn.model.vit import ElasticViTUtil +from utils.dl.common.model import LayerActivation2, get_module, get_parameter +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +from utils.dl.common.loss import CrossEntropyLossSoft +import torch.nn.functional as F +from utils.common.log import logger + + +class ElasticDNN_ViT_OfflineDetFMModel(ElasticDNN_OfflineDetFMModel): + def generate_md_by_reducing_width(self, reducing_width_ratio, samples: torch.Tensor): + return FM_to_MD_ViT_Util().init_md_from_fm_by_reducing_width_with_perf_test(self.models_dict['main'], + reducing_width_ratio, samples) + + def get_feature_hook(self) -> LayerActivation2: + return LayerActivation2(get_module(self.models_dict['main'], 'blocks.11')) + + def get_elastic_dnn_util(self) -> ElasticDNNUtil: + return ElasticViTUtil() + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + # return F.cross_entropy(self.infer(x), y) + self.to_train_mode() + return self.models_dict['main'](x, y)['total_loss'] + + def get_lora_util(self) -> FMLoRA_Util: + return FMLoRA_ViT_Util() + + def get_task_head_params(self): + head = get_module(self.models_dict['main'], 'head') + params_name = {k for k, v in head.named_parameters()} + logger.info(f'task head params: {params_name}') + return list(head.parameters()) + + +class ElasticDNN_ViT_OfflineDetMDModel(ElasticDNN_OfflineDetMDModel): + def get_feature_hook(self) -> LayerActivation2: + return LayerActivation2(get_module(self.models_dict['main'], 'blocks.11')) + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + self.to_train_mode() + return self.models_dict['main'](x, y)['total_loss'] + + def get_distill_loss(self, student_output, teacher_output): + # print(student_output, teacher_output) + return F.mse_loss(student_output, teacher_output) + + def get_matched_param_of_fm(self, self_param_name, fm: nn.Module): + if any([k in self_param_name for k in ['fbs', 'cls_token', 'pos_embed']]): + return None + + # 1. xx.qkv.to_qkv.yy to xx.qkv.qkv.aa and xx.qkv.abs.zz + if 'to_qkv.weight' in self_param_name: + ss = self_param_name.split('.') + + fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv' + fm_qkv = get_module(fm, fm_qkv_name) + + fm_abs_name = '.'.join(ss[0: -2]) + '.abs' + fm_abs = get_module(fm, fm_abs_name) + + return torch.cat([ + fm_qkv.weight.data, # task-agnositc params + torch.cat([(_abs[0].weight.T @ _abs[1].weight.T).T for _abs in fm_abs], dim=0) # task-specific params (LoRA) + ], dim=0) + + elif 'to_qkv.bias' in self_param_name: + ss = self_param_name.split('.') + + fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv.bias' + return get_parameter(fm, fm_qkv_name) + + elif 'mlp.fc1' in self_param_name: + fm_param_name = self_param_name.replace('.linear', '') + return get_parameter(fm, fm_param_name) + + else: + return get_parameter(fm, self_param_name) + + +if __name__ == '__main__': + from utils.dl.common.env import set_random_seed + set_random_seed(1) + + # 3. init scenario + scenario = build_scenario( + source_datasets_name=['GTA5Det', 'SuperviselyPersonDet'], + target_datasets_order=['CityscapesDet', 'BaiduPersonDet'] * 10, + da_mode='close_set', + data_dirs={ + 'GTA5Det': '/data/zql/datasets/GTA-ls-copy/GTA5', + 'SuperviselyPersonDet': '/data/zql/datasets/supervisely_person_full_20230635/Supervisely Person Dataset', + 'CityscapesDet': '/data/zql/datasets/cityscape/', + 'BaiduPersonDet': '/data/zql/datasets/baidu_person/clean_images/' + }, + ) + + # 1. init model + # from dnns.deeplabv3.head import modify_forward_head + # modify_forward_head() # TODO: bring a bug + from dnns.vit import vit_b_16 + fm_models_dict_path = 'experiments/elasticdnn/vit_b_16/offline/fm_lora/det/results/det.py/20230627/star_final_999983-140249-3_class_det_trial_lr1e-4_use_cls_lora_ckpt/models/fm_best.pt' + fm_models = torch.load(fm_models_dict_path) + fm_models_dict_path = save_models_dict_for_init(fm_models, __file__, 'fm_vit_b_16_det_lora') + md_models_dict_path = save_models_dict_for_init({ + 'main': -1 + }, __file__, 'md_vit_b_16_none') + device = 'cuda' + + fm_model = ElasticDNN_ViT_OfflineDetFMModel('fm', fm_models_dict_path, device, scenario.num_classes) + md_model = ElasticDNN_ViT_OfflineDetMDModel('md', md_models_dict_path, device, scenario.num_classes) + + # 2. init alg + models = { + 'fm': fm_model, + 'md': md_model + } + fm_to_md_alg = ElasticDNN_MDPretrainingWoFBSAlg(models, get_res_save_dir(__file__, None)) + + from utils.dl.common.lr_scheduler import get_linear_schedule_with_warmup + fm_to_md_alg.run(scenario, hyps={ + 'launch_tbboard': False, + + 'samples_size': (1, 3, 224, 224), + 'generate_md_width_ratio': 4, + + 'train_batch_size': 32, + 'val_batch_size': 32, + 'num_workers': 16, + 'optimizer': 'AdamW', + 'optimizer_args': {'lr': 1e-4, 'betas': [0.9, 0.999]}, + 'scheduler': 'LambdaLR', + 'scheduler_args': {'lr_lambda': get_linear_schedule_with_warmup(10000, 70000)}, + 'num_iters': 80000, + 'val_freq': 4000, + 'distill_loss_weight': 1e-4 + }) + \ No newline at end of file diff --git a/experiments/elasticdnn/vit_b_16/offline/fm_to_md/seg/seg_md_index.py b/experiments/elasticdnn/vit_b_16/offline/fm_to_md/seg/seg_md_index.py new file mode 100644 index 0000000000000000000000000000000000000000..f7b46d329e55586b4db6b4d5dba190cc137f6e43 --- /dev/null +++ b/experiments/elasticdnn/vit_b_16/offline/fm_to_md/seg/seg_md_index.py @@ -0,0 +1,199 @@ +import torch +import sys +from torch import nn +from methods.elasticdnn.api.model import ElasticDNN_OfflineSegFMModel, ElasticDNN_OfflineSegMDModel +from methods.elasticdnn.model.base import ElasticDNNUtil +from methods.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from methods.elasticdnn.pipeline.offline.fm_to_md.vit import FM_to_MD_ViT_Util +from methods.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from methods.elasticdnn.pipeline.offline.fm_lora.vit import FMLoRA_ViT_Util +from methods.elasticdnn.model.vit import ElasticViTUtil +from utils.dl.common.model import LayerActivation, get_module, get_parameter +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +from utils.dl.common.loss import CrossEntropyLossSoft +import torch.nn.functional as F + + +class ElasticDNN_ViT_OfflineSegFMModel(ElasticDNN_OfflineSegFMModel): + def generate_md_by_reducing_width(self, reducing_width_ratio, samples: torch.Tensor): + return FM_to_MD_ViT_Util().init_md_from_fm_by_reducing_width_with_perf_test(self.models_dict['main'], + reducing_width_ratio, samples).to(self.device) + + def get_feature_hook(self) -> LayerActivation: + return LayerActivation(get_module(self.models_dict['main'], 'head'), True, self.device) + + def get_elastic_dnn_util(self) -> ElasticDNNUtil: + return ElasticViTUtil() + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + return F.cross_entropy(self.infer(x), y) + + def get_lora_util(self) -> FMLoRA_Util: + return FMLoRA_ViT_Util() + + def get_task_head_params(self): + head = get_module(self.models_dict['main'], 'norm.1.head') + return list(head.parameters()) + + +class ElasticDNN_ViT_OfflineSegMDModel(ElasticDNN_OfflineSegMDModel): + + def get_feature_hook(self) -> LayerActivation: + return LayerActivation(get_module(self.models_dict['main'], 'head'), True, self.device) + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + return F.cross_entropy(self.infer(x), y) + + def get_distill_loss(self, student_output, teacher_output): + return F.mse_loss(student_output, teacher_output) + + def get_matched_param_of_fm(self, self_param_name, fm: nn.Module): + # only between qkv.weight, norm.weight/bias + if any([k in self_param_name for k in ['fbs', 'cls_token', 'pos_embed']]): + return None + + p = get_parameter(self.models_dict['main'], self_param_name) + if p.dim() == 0: + return None + elif p.dim() == 1 and 'norm' in self_param_name and 'weight' in self_param_name: + if self_param_name.startswith('norm'): + self_param_name = self_param_name.replace('norm', 'norm.0') + return get_parameter(fm, self_param_name) + + # 1. xx.qkv.to_qkv.yy to xx.qkv.qkv.aa and xx.qkv.abs.zz + if 'qkv.weight' in self_param_name: + ss = self_param_name.split('.') + + fm_qkv_name = '.'.join(ss[0: -1]) + '.qkv' + fm_qkv = get_module(fm, fm_qkv_name) + + fm_abs_name = '.'.join(ss[0: -1]) + '.abs' + fm_abs = get_module(fm, fm_abs_name) + + return torch.cat([ + fm_qkv.weight.data, # task-agnositc params + torch.cat([(_abs[0].weight.T @ _abs[1].weight.T).T for _abs in fm_abs], dim=0) # task-specific params (LoRA) + ], dim=0) + + # elif 'to_qkv.bias' in self_param_name: + # ss = self_param_name.split('.') + + # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv.bias' + # return get_parameter(fm, fm_qkv_name) + + elif 'mlp.fc1' in self_param_name and 'weight' in self_param_name: + fm_param_name = self_param_name.replace('.linear', '') + return get_parameter(fm, fm_param_name) + + elif 'mlp.fc2' in self_param_name and 'weight' in self_param_name: + fm_param_name = self_param_name + return get_parameter(fm, fm_param_name) + + else: + # return get_parameter(fm, self_param_name) + return None + + +if __name__ == '__main__': + from utils.dl.common.env import set_random_seed + set_random_seed(1) + + # 3. init scenario + scenario = build_scenario( + source_datasets_name=['GTA5', 'SuperviselyPerson'], + target_datasets_order=['Cityscapes', 'BaiduPerson'] * 10, + da_mode='close_set', + data_dirs={ + 'GTA5': '/data/zql/datasets/GTA-ls-copy/GTA5', + 'SuperviselyPerson': '/data/zql/datasets/supervisely_person/Supervisely Person Dataset', + 'Cityscapes': '/data/zql/datasets/cityscape/', + 'BaiduPerson': '/data/zql/datasets/baidu_person/clean_images/' + }, + ) + + # 1. init model + # from dnns.deeplabv3.head import modify_forward_head + # modify_forward_head() # TODO: bring a bug + from dnns.vit import vit_b_16 + fm_models_dict_path = 'experiments/elasticdnn/vit_b_16/offline/fm_to_md/seg/results/seg_md_w_fbs.py/20230611/999999-183508-new_md_structure/models/fm_best.pt' + fm_models = torch.load(fm_models_dict_path) + from utils.dl.common.model import set_module + set_module( + fm_models['main'], + 'norm', + nn.Sequential( + get_module(fm_models['main'], 'norm'), + get_module(fm_models['main'], 'head') + ) + ) + set_module(fm_models['main'], 'head', nn.Identity()) + fm_models['main'].forward = fm_models['main'].forward_features + fm_models_dict_path = save_models_dict_for_init(fm_models, __file__, 'fm_vit_b_16_seg_lora') + + pretrained_md_models_dict_path = 'experiments/elasticdnn/vit_b_16/offline/fm_to_md/seg/results/seg_md_wo_fbs.py/20230611/999998-141814/models/md_best.pt' + md_models_dict_path = save_models_dict_for_init(torch.load(pretrained_md_models_dict_path), __file__, 'md_vit_b_16_seg_pretrained_w_fbs') + device = 'cuda' + + fm_model = ElasticDNN_ViT_OfflineSegFMModel('fm', fm_models_dict_path, device, scenario.num_classes) + md_model = ElasticDNN_ViT_OfflineSegMDModel('md', md_models_dict_path, device, scenario.num_classes) + + # 2. init alg + models = { + 'fm': fm_model, + 'md': md_model + } + from methods.elasticdnn.api.algs.md_pretraining_index_v2_train_index_and_md import ElasticDNN_MDPretrainingIndexAlg + fm_to_md_alg = ElasticDNN_MDPretrainingIndexAlg(models, get_res_save_dir(__file__, sys.argv[1])) + + + + from utils.dl.common.lr_scheduler import get_linear_schedule_with_warmup + fm_to_md_alg.run(scenario, hyps={ + # 'launch_tbboard': False, + + # 'samples_size': (1, 3, 224, 224), + # 'generate_md_width_ratio': 2, + + # 'train_batch_size': 16, + # 'val_batch_size': 128, + # 'num_workers': 16, + # 'optimizer': 'AdamW', + # 'optimizer_args': {'lr': 5e-4, 'betas': [0.9, 0.999], 'weight_decay': 0.01}, + # 'scheduler': 'LambdaLR', + # 'scheduler_args': {'lr_lambda': get_linear_schedule_with_warmup(10000, 70000)}, + # 'num_iters': 80000, + # 'val_freq': 4000, + # 'distill_loss_weight': 1.0 + + 'launch_tbboard': False, + + 'samples_size': (1, 3, 224, 224), + + 'FBS_r': 16, + 'FBS_ignore_layers': [], + + 'train_batch_size': 16, + 'val_batch_size': 128, + 'num_workers': 16, + 'optimizer': 'AdamW', + 'optimizer_args': {'lr': 1e-3 * 16 / 256, 'betas': [0.9, 0.999], 'weight_decay': 5e-4}, + 'scheduler': 'LambdaLR', + 'scheduler_args': {'lr_lambda': get_linear_schedule_with_warmup(10000, 70000)}, + 'indexes_optimizer_args': {'lr': 3e-3, 'momentum': 0.9, 'weight_decay': 5e-4}, + # 'scheduler': 'StepLR', + # 'scheduler_args': {'step_size': 40000, 'gamma': 0.1}, + 'num_iters': 60000, + 'val_freq': 500, + 'max_sparsity': 0.9, + 'min_sparsity': 0.0, + 'l1_reg_loss_weight': 1e-9, + 'index_loss_weight': 1e-4, + 'val_num_sparsities': 4, + + 'bn_cal_num_iters': 0, + + 'index_init': 'zero', + 'index_guided_linear_comb_split_size': 512 + }) + \ No newline at end of file diff --git a/experiments/elasticdnn/vit_b_16/offline/fm_to_md/seg/seg_md_w_fbs.py b/experiments/elasticdnn/vit_b_16/offline/fm_to_md/seg/seg_md_w_fbs.py new file mode 100644 index 0000000000000000000000000000000000000000..9fee076af2b806fc7951ea302ab06cedec5d0653 --- /dev/null +++ b/experiments/elasticdnn/vit_b_16/offline/fm_to_md/seg/seg_md_w_fbs.py @@ -0,0 +1,178 @@ +import torch +import sys +from torch import nn +from methods.elasticdnn.api.model import ElasticDNN_OfflineSegFMModel, ElasticDNN_OfflineSegMDModel +from methods.elasticdnn.api.algs.md_pretraining_w_fbs import ElasticDNN_MDPretrainingWFBSAlg +from methods.elasticdnn.model.base import ElasticDNNUtil +from methods.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from methods.elasticdnn.pipeline.offline.fm_to_md.vit import FM_to_MD_ViT_Util +from methods.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from methods.elasticdnn.pipeline.offline.fm_lora.vit import FMLoRA_ViT_Util +from methods.elasticdnn.model.vit import ElasticViTUtil +from utils.dl.common.model import LayerActivation, get_module, get_parameter +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +from utils.dl.common.loss import CrossEntropyLossSoft +import torch.nn.functional as F + + +class ElasticDNN_ViT_OfflineSegFMModel(ElasticDNN_OfflineSegFMModel): + def generate_md_by_reducing_width(self, reducing_width_ratio, samples: torch.Tensor): + return FM_to_MD_ViT_Util().init_md_from_fm_by_reducing_width_with_perf_test(self.models_dict['main'], + reducing_width_ratio, samples).to(self.device) + + def get_feature_hook(self) -> LayerActivation: + return LayerActivation(get_module(self.models_dict['main'], 'head'), True, self.device) + + def get_elastic_dnn_util(self) -> ElasticDNNUtil: + return ElasticViTUtil() + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + return F.cross_entropy(self.infer(x), y) + + def get_lora_util(self) -> FMLoRA_Util: + return FMLoRA_ViT_Util() + + def get_task_head_params(self): + head = get_module(self.models_dict['main'], 'norm.1.head') + return list(head.parameters()) + + +class ElasticDNN_ViT_OfflineSegMDModel(ElasticDNN_OfflineSegMDModel): + + def get_feature_hook(self) -> LayerActivation: + return LayerActivation(get_module(self.models_dict['main'], 'head'), True, self.device) + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + return F.cross_entropy(self.infer(x), y) + + def get_distill_loss(self, student_output, teacher_output): + return F.mse_loss(student_output, teacher_output) + + def get_matched_param_of_fm(self, self_param_name, fm: nn.Module): + # only between qkv.weight, norm.weight/bias + if any([k in self_param_name for k in ['fbs', 'cls_token', 'pos_embed']]): + return None + + p = get_parameter(self.models_dict['main'], self_param_name) + if p.dim() == 0: + return None + elif p.dim() == 1 and 'norm' in self_param_name: + return get_parameter(fm, self_param_name) + + # 1. xx.qkv.to_qkv.yy to xx.qkv.qkv.aa and xx.qkv.abs.zz + if 'to_qkv.weight' in self_param_name: + ss = self_param_name.split('.') + + fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv' + fm_qkv = get_module(fm, fm_qkv_name) + + fm_abs_name = '.'.join(ss[0: -2]) + '.abs' + fm_abs = get_module(fm, fm_abs_name) + + return torch.cat([ + fm_qkv.weight.data, # task-agnositc params + torch.cat([(_abs[0].weight.T @ _abs[1].weight.T).T for _abs in fm_abs], dim=0) # task-specific params (LoRA) + ], dim=0) + + # elif 'to_qkv.bias' in self_param_name: + # ss = self_param_name.split('.') + + # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv.bias' + # return get_parameter(fm, fm_qkv_name) + + elif 'mlp.fc1' in self_param_name and 'weight' in self_param_name: + fm_param_name = self_param_name.replace('.linear', '') + return get_parameter(fm, fm_param_name) + + elif 'mlp.fc2' in self_param_name and 'weight' in self_param_name: + fm_param_name = self_param_name + return get_parameter(fm, fm_param_name) + + else: + # return get_parameter(fm, self_param_name) + return None + + +if __name__ == '__main__': + from utils.dl.common.env import set_random_seed + set_random_seed(1) + + # 3. init scenario + scenario = build_scenario( + source_datasets_name=['GTA5', 'SuperviselyPerson'], + target_datasets_order=['Cityscapes', 'BaiduPerson'] * 10, + da_mode='close_set', + data_dirs={ + 'GTA5': '/data/zql/datasets/GTA-ls-copy/GTA5', + 'SuperviselyPerson': '/data/zql/datasets/supervisely_person/Supervisely Person Dataset', + 'Cityscapes': '/data/zql/datasets/cityscape/', + 'BaiduPerson': '/data/zql/datasets/baidu_person/clean_images/' + }, + ) + + # 1. init model + # from dnns.deeplabv3.head import modify_forward_head + # modify_forward_head() # TODO: bring a bug + from dnns.vit import vit_b_16 + fm_models_dict_path = 'experiments/elasticdnn/vit_b_16/offline/fm_to_md/seg/results/seg_md_wo_fbs.py/20230611/999998-141814/models/fm_best.pt' + fm_models_dict_path = save_models_dict_for_init(torch.load(fm_models_dict_path), __file__, 'fm_vit_b_16_seg_lora') + pretrained_md_models_dict_path = 'experiments/elasticdnn/vit_b_16/offline/fm_to_md/seg/results/seg_md_wo_fbs.py/20230611/999998-141814/models/md_best.pt' + md_models_dict_path = save_models_dict_for_init(torch.load(pretrained_md_models_dict_path), __file__, 'md_vit_b_16_seg_pretrained_wo_fbs') + device = 'cuda' + + fm_model = ElasticDNN_ViT_OfflineSegFMModel('fm', fm_models_dict_path, device, scenario.num_classes) + md_model = ElasticDNN_ViT_OfflineSegMDModel('md', md_models_dict_path, device, scenario.num_classes) + + # 2. init alg + models = { + 'fm': fm_model, + 'md': md_model + } + fm_to_md_alg = ElasticDNN_MDPretrainingWFBSAlg(models, get_res_save_dir(__file__, sys.argv[1])) + + + + from utils.dl.common.lr_scheduler import get_linear_schedule_with_warmup + fm_to_md_alg.run(scenario, hyps={ + # 'launch_tbboard': False, + + # 'samples_size': (1, 3, 224, 224), + # 'generate_md_width_ratio': 2, + + # 'train_batch_size': 16, + # 'val_batch_size': 128, + # 'num_workers': 16, + # 'optimizer': 'AdamW', + # 'optimizer_args': {'lr': 5e-4, 'betas': [0.9, 0.999], 'weight_decay': 0.01}, + # 'scheduler': 'LambdaLR', + # 'scheduler_args': {'lr_lambda': get_linear_schedule_with_warmup(10000, 70000)}, + # 'num_iters': 80000, + # 'val_freq': 4000, + # 'distill_loss_weight': 1.0 + + 'launch_tbboard': False, + + 'samples_size': (1, 3, 224, 224), + 'generate_md_width_ratio': 2, + + 'FBS_r': 16, + 'FBS_ignore_layers': [], + + 'train_batch_size': 16, + 'val_batch_size': 128, + 'num_workers': 16, + 'optimizer': 'AdamW', + 'optimizer_args': {'lr': 1e-3 * 16 / 256, 'betas': [0.9, 0.999], 'weight_decay': 5e-4}, + 'scheduler': 'LambdaLR', + 'scheduler_args': {'lr_lambda': get_linear_schedule_with_warmup(10000, 70000)}, + 'num_iters': 80000, + 'val_freq': 4000, + 'max_sparsity': 0.8, + 'min_sparsity': 0.0, + 'l1_reg_loss_weight': 1e-9, + 'val_num_sparsities': 4, + + 'bn_cal_num_iters': 0 + }) + \ No newline at end of file diff --git a/experiments/elasticdnn/vit_b_16/offline/fm_to_md/seg/seg_md_wo_fbs.py b/experiments/elasticdnn/vit_b_16/offline/fm_to_md/seg/seg_md_wo_fbs.py new file mode 100644 index 0000000000000000000000000000000000000000..492e0e8d7968f85cc5804968c7a4ed4bd76259f0 --- /dev/null +++ b/experiments/elasticdnn/vit_b_16/offline/fm_to_md/seg/seg_md_wo_fbs.py @@ -0,0 +1,154 @@ +import torch +import sys +from torch import nn +from methods.elasticdnn.api.model import ElasticDNN_OfflineSegFMModel, ElasticDNN_OfflineSegMDModel +from methods.elasticdnn.api.algs.md_pretraining_wo_fbs import ElasticDNN_MDPretrainingWoFBSAlg +from methods.elasticdnn.model.base import ElasticDNNUtil +from methods.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from methods.elasticdnn.pipeline.offline.fm_to_md.vit import FM_to_MD_ViT_Util +from methods.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from methods.elasticdnn.pipeline.offline.fm_lora.vit import FMLoRA_ViT_Util +from methods.elasticdnn.model.vit import ElasticViTUtil +from utils.dl.common.model import LayerActivation, get_module, get_parameter +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +from utils.dl.common.loss import CrossEntropyLossSoft +import torch.nn.functional as F + + +class ElasticDNN_ViT_OfflineSegFMModel(ElasticDNN_OfflineSegFMModel): + def generate_md_by_reducing_width(self, reducing_width_ratio, samples: torch.Tensor): + return FM_to_MD_ViT_Util().init_md_from_fm_by_reducing_width_with_perf_test(self.models_dict['main'], + reducing_width_ratio, samples).to(self.device) + + def get_feature_hook(self) -> LayerActivation: + return LayerActivation(get_module(self.models_dict['main'], 'head'), True, self.device) + + def get_elastic_dnn_util(self) -> ElasticDNNUtil: + return ElasticViTUtil() + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + return F.cross_entropy(self.infer(x), y) + + def get_lora_util(self) -> FMLoRA_Util: + return FMLoRA_ViT_Util() + + def get_task_head_params(self): + head = get_module(self.models_dict['main'], 'norm.1.head') + return list(head.parameters()) + + +class ElasticDNN_ViT_OfflineSegMDModel(ElasticDNN_OfflineSegMDModel): + + def get_feature_hook(self) -> LayerActivation: + return LayerActivation(get_module(self.models_dict['main'], 'head'), True, self.device) + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + return F.cross_entropy(self.infer(x), y) + + def get_distill_loss(self, student_output, teacher_output): + return F.mse_loss(student_output, teacher_output) + + def get_matched_param_of_fm(self, self_param_name, fm: nn.Module): + if any([k in self_param_name for k in ['fbs', 'cls_token', 'pos_embed']]): + return None + + # 1. xx.qkv.to_qkv.yy to xx.qkv.qkv.aa and xx.qkv.abs.zz + if 'to_qkv.weight' in self_param_name: + ss = self_param_name.split('.') + + fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv' + fm_qkv = get_module(fm, fm_qkv_name) + + fm_abs_name = '.'.join(ss[0: -2]) + '.abs' + fm_abs = get_module(fm, fm_abs_name) + + return torch.cat([ + fm_qkv.weight.data, # task-agnositc params + torch.cat([(_abs[0].weight.T @ _abs[1].weight.T).T for _abs in fm_abs], dim=0) # task-specific params (LoRA) + ], dim=0) + + elif 'to_qkv.bias' in self_param_name: + ss = self_param_name.split('.') + + fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv.bias' + return get_parameter(fm, fm_qkv_name) + + elif 'mlp.fc1' in self_param_name: + fm_param_name = self_param_name.replace('.linear', '') + return get_parameter(fm, fm_param_name) + + else: + return get_parameter(fm, self_param_name) + + +if __name__ == '__main__': + from utils.dl.common.env import set_random_seed + set_random_seed(1) + + # 3. init scenario + scenario = build_scenario( + source_datasets_name=['GTA5', 'SuperviselyPerson'], + target_datasets_order=['Cityscapes', 'BaiduPerson'] * 10, + da_mode='close_set', + data_dirs={ + 'GTA5': '/data/zql/datasets/GTA-ls-copy/GTA5', + 'SuperviselyPerson': '/data/zql/datasets/supervisely_person/Supervisely Person Dataset', + 'Cityscapes': '/data/zql/datasets/cityscape/', + 'BaiduPerson': '/data/zql/datasets/baidu_person/clean_images/' + }, + ) + + # 1. init model + # from dnns.deeplabv3.head import modify_forward_head + # modify_forward_head() # TODO: bring a bug + from dnns.vit import vit_b_16 + fm_models_dict_path = 'experiments/elasticdnn/vit_b_16/offline/fm_lora/seg/results/seg.py/20230521/999979-163448/models/fm_best.pt' + fm_models = torch.load(fm_models_dict_path) + from utils.dl.common.model import set_module + set_module( + fm_models['main'], + 'norm', + nn.Sequential( + get_module(fm_models['main'], 'norm'), + get_module(fm_models['main'], 'head') + ) + ) + set_module(fm_models['main'], 'head', nn.Identity()) + fm_models['main'].forward = fm_models['main'].forward_features + + fm_models_dict_path = save_models_dict_for_init(fm_models, __file__, 'fm_vit_b_16_seg_lora') + md_models_dict_path = save_models_dict_for_init({ + 'main': -1 + }, __file__, 'md_vit_b_16_none') + device = 'cuda' + + fm_model = ElasticDNN_ViT_OfflineSegFMModel('fm', fm_models_dict_path, device, scenario.num_classes) + md_model = ElasticDNN_ViT_OfflineSegMDModel('md', md_models_dict_path, device, scenario.num_classes) + + # 2. init alg + models = { + 'fm': fm_model, + 'md': md_model + } + fm_to_md_alg = ElasticDNN_MDPretrainingWoFBSAlg(models, get_res_save_dir(__file__, None)) + + from utils.dl.common.lr_scheduler import get_linear_schedule_with_warmup + fm_to_md_alg.run(scenario, hyps={ + 'launch_tbboard': False, + + 'samples_size': (1, 3, 224, 224), + 'generate_md_width_ratio': 4, + + 'train_batch_size': 16, + 'val_batch_size': 128, + 'num_workers': 16, + 'optimizer': 'AdamW', + 'optimizer_args': {'lr': 5e-4, 'betas': [0.9, 0.999], 'weight_decay': 0.01}, + 'scheduler': 'LambdaLR', + 'scheduler_args': {'lr_lambda': get_linear_schedule_with_warmup(10000, 70000)}, + 'num_iters': 80000, + 'val_freq': 4000, + 'distill_loss_weight': 1.0 + }) + \ No newline at end of file diff --git a/experiments/elasticdnn/vit_b_16/online_new/cls/cls.py b/experiments/elasticdnn/vit_b_16/online_new/cls/cls.py new file mode 100644 index 0000000000000000000000000000000000000000..4b7d1c6bd58a6c1b618e195e9bf88163390a3d93 --- /dev/null +++ b/experiments/elasticdnn/vit_b_16/online_new/cls/cls.py @@ -0,0 +1,109 @@ +from typing import List +from data.dataloader import build_dataloader +# from methods.elasticdnn.api.online_model import ElasticDNN_OnlineModel +from methods.elasticdnn.api.online_model_v2 import ElasticDNN_OnlineModel + +import torch +import sys +from torch import nn +from methods.elasticdnn.api.model import ElasticDNN_OfflineSegFMModel, ElasticDNN_OfflineSegMDModel +from methods.elasticdnn.api.algs.md_pretraining_wo_fbs import ElasticDNN_MDPretrainingWoFBSAlg +from methods.elasticdnn.model.base import ElasticDNNUtil +from methods.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from methods.elasticdnn.pipeline.offline.fm_to_md.vit import FM_to_MD_ViT_Util +from methods.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from methods.elasticdnn.pipeline.offline.fm_lora.vit import FMLoRA_ViT_Util +from methods.elasticdnn.model.vit import ElasticViTUtil +from utils.common.file import ensure_dir +from utils.dl.common.model import LayerActivation, get_module, get_parameter +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +from utils.dl.common.loss import CrossEntropyLossSoft +import torch.nn.functional as F +from utils.dl.common.env import create_tbwriter +import os +from utils.common.log import logger +from utils.common.data_record import write_json +# from methods.shot.shot import OnlineShotModel +from methods.feat_align.main import OnlineFeatAlignModel, FeatAlignAlg +import tqdm +from methods.feat_align.mmd import mmd_rbf +from experiments.utils.elasticfm_da import init_online_model, elasticfm_da + +device = 'cuda' +app_name = 'cls' +sd_sparsity = 0.8 + +settings = { + 'involve_fm': True +} + +scenario = build_scenario( + source_datasets_name=['GTA5Cls', 'SuperviselyPersonCls'], + target_datasets_order=['CityscapesCls', 'BaiduPersonCls'] * 15, + da_mode='close_set', + data_dirs={ + 'GTA5Cls': '/data/zql/datasets/gta5_for_cls_task', + 'SuperviselyPersonCls': '/data/zql/datasets/supervisely_person_for_cls_task', + 'CityscapesCls': '/data/zql/datasets/cityscapes_for_cls_task', + 'BaiduPersonCls': '/data/zql/datasets/baiduperson_for_cls_task' + }, +) + +from experiments.elasticdnn.vit_b_16.online_new.cls.model import ElasticDNN_ClsOnlineModel +elasticfm_model = ElasticDNN_ClsOnlineModel('cls', init_online_model( + # 'experiments/elasticdnn/vit_b_16/offline/fm_to_md/results/cls_md_index.py/20230529/star_999997-154037-only_prune_mlp/models/fm_best.pt', + # 'experiments/elasticdnn/vit_b_16/offline/fm_to_md/results/cls_md_index.py/20230529/star_999997-154037-only_prune_mlp/models/md_best.pt', + 'experiments/elasticdnn/vit_b_16/offline/fm_to_md/cls/results/cls_md_index.py/20230617/999992-101343-lr1e-5_index_bug_fixed/models/fm_best.pt', + 'experiments/elasticdnn/vit_b_16/offline/fm_to_md/cls/results/cls_md_index.py/20230617/999992-101343-lr1e-5_index_bug_fixed/models/md_best.pt', + 'cls', __file__ +), device, { + 'md_to_fm_alpha': 1, + 'fm_to_md_alpha': 1 +}) + +da_alg = FeatAlignAlg +from experiments.elasticdnn.vit_b_16.online_new.cls.model import ClsOnlineFeatAlignModel +da_model = ClsOnlineFeatAlignModel +da_alg_hyp = { + 'CityscapesCls': { + 'train_batch_size': 64, + 'val_batch_size': 512, + 'num_workers': 8, + 'optimizer': 'AdamW', + 'optimizer_args': {'lr': 4e-6/2, 'betas': [0.9, 0.999], 'weight_decay': 0.01}, + 'scheduler': '', + 'scheduler_args': {}, + 'num_iters': 100, + 'val_freq': 20, + 'sd_sparsity': sd_sparsity, + 'feat_align_loss_weight': 3.0 + }, + 'BaiduPersonCls': { + 'train_batch_size': 64, + 'val_batch_size': 512, + 'num_workers': 8, + 'optimizer': 'SGD', + 'optimizer_args': {'lr': 1e-4/2, 'momentum': 0.9}, + 'scheduler': '', + 'scheduler_args': {}, + 'num_iters': 100, + 'val_freq': 20, + 'sd_sparsity': sd_sparsity, + 'feat_align_loss_weight': 0.3 + } +} + + +elasticfm_da( + [app_name], + [scenario], + [elasticfm_model], + [da_alg], + [da_alg_hyp], + [da_model], + device, + settings, + __file__, + sys.argv[0] +) diff --git a/experiments/elasticdnn/vit_b_16/online_new/cls/model.py b/experiments/elasticdnn/vit_b_16/online_new/cls/model.py new file mode 100644 index 0000000000000000000000000000000000000000..faeced5ee707eac5df124adac061c3b7c5cb9278 --- /dev/null +++ b/experiments/elasticdnn/vit_b_16/online_new/cls/model.py @@ -0,0 +1,212 @@ +from typing import List +from data.dataloader import build_dataloader +# from methods.elasticdnn.api.online_model import ElasticDNN_OnlineModel +from methods.elasticdnn.api.online_model_v2 import ElasticDNN_OnlineModel + +import torch +import sys +from torch import nn +from methods.elasticdnn.api.model import ElasticDNN_OfflineSegFMModel, ElasticDNN_OfflineSegMDModel +from methods.elasticdnn.api.algs.md_pretraining_wo_fbs import ElasticDNN_MDPretrainingWoFBSAlg +from methods.elasticdnn.model.base import ElasticDNNUtil +from methods.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from methods.elasticdnn.pipeline.offline.fm_to_md.vit import FM_to_MD_ViT_Util +from methods.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from methods.elasticdnn.pipeline.offline.fm_lora.vit import FMLoRA_ViT_Util +from methods.elasticdnn.model.vit import ElasticViTUtil +from utils.common.file import ensure_dir +from utils.dl.common.model import LayerActivation, get_module, get_parameter +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +from utils.dl.common.loss import CrossEntropyLossSoft +import torch.nn.functional as F +from utils.dl.common.env import create_tbwriter +import os +from utils.common.log import logger +from utils.common.data_record import write_json +# from methods.shot.shot import OnlineShotModel +from methods.feat_align.main import OnlineFeatAlignModel, FeatAlignAlg +import tqdm +from methods.feat_align.mmd import mmd_rbf + +class ElasticDNN_ClsOnlineModel(ElasticDNN_OnlineModel): + def get_accuracy(self, test_loader, *args, **kwargs): + acc = 0 + sample_num = 0 + + self.to_eval_mode() + + with torch.no_grad(): + pbar = tqdm.tqdm(enumerate(test_loader), total=len(test_loader), dynamic_ncols=True, leave=False) + for batch_index, (x, y) in pbar: + x, y = x.to(self.device), y.to(self.device) + output = self.infer(x) + pred = F.softmax(output, dim=1).argmax(dim=1) + correct = torch.eq(pred, y).sum().item() + acc += correct + sample_num += len(y) + + pbar.set_description(f'cur_batch_total: {len(y)}, cur_batch_correct: {correct}, ' + f'cur_batch_acc: {(correct / len(y)):.4f}') + + acc /= sample_num + return acc + + def get_elastic_dnn_util(self) -> ElasticDNNUtil: + return ElasticViTUtil() + + def get_fm_matched_param_of_md_param(self, md_param_name): + # only between qkv.weight, norm.weight/bias + self_param_name = md_param_name + fm = self.models_dict['fm'] + if any([k in self_param_name for k in ['fbs', 'cls_token', 'pos_embed']]): + return None + + p = get_parameter(self.models_dict['md'], self_param_name) + if p.dim() == 0: + return None + elif p.dim() == 1 and 'norm' in self_param_name and 'weight' in self_param_name: + return get_parameter(fm, self_param_name) + + # 1. xx.qkv.to_qkv.yy to xx.qkv.qkv.aa and xx.qkv.abs.zz + if 'qkv.weight' in self_param_name: + ss = self_param_name.split('.') + + fm_qkv_name = '.'.join(ss[0: -1]) + '.qkv' + fm_qkv = get_module(fm, fm_qkv_name) + + fm_abs_name = '.'.join(ss[0: -1]) + '.abs' + fm_abs = get_module(fm, fm_abs_name) + + # NOTE: unrecoverable operation! multiply LoRA parameters to allow it being updated in update_fm_param() + # TODO: if fm will be used for inference, _mul_lora_weight will not be applied! + if not hasattr(fm_abs, '_mul_lora_weight'): + logger.debug(f'set _mul_lora_weight in {fm_abs_name}') + setattr(fm_abs, '_mul_lora_weight', + nn.Parameter(torch.cat([(_abs[0].weight.T @ _abs[1].weight.T).T for _abs in fm_abs], dim=0))) + + return torch.cat([ + fm_qkv.weight.data, # task-agnositc params + fm_abs._mul_lora_weight.data # task-specific params (LoRA) + ], dim=0) + + # elif 'to_qkv.bias' in self_param_name: + # ss = self_param_name.split('.') + + # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv.bias' + # return get_parameter(fm, fm_qkv_name) + + elif 'mlp.fc1' in self_param_name and 'weight' in self_param_name: + fm_param_name = self_param_name.replace('.linear', '') + return get_parameter(fm, fm_param_name) + + elif 'mlp.fc2' in self_param_name and 'weight' in self_param_name: + fm_param_name = self_param_name + return get_parameter(fm, fm_param_name) + + else: + # return get_parameter(fm, self_param_name) + return None + + def update_fm_param(self, md_param_name, cal_new_fm_param_by_md_param): + if not 'qkv.weight' in md_param_name: + matched_fm_param_ref = self.get_fm_matched_param_of_md_param(md_param_name) + matched_fm_param_ref.copy_(cal_new_fm_param_by_md_param) + else: + new_fm_attn_weight, new_fm_lora_weight = torch.chunk(cal_new_fm_param_by_md_param, 2, 0) + + ss = md_param_name.split('.') + fm = self.models_dict['fm'] + + # update task-agnostic parameters + fm_qkv_name = '.'.join(ss[0: -1]) + '.qkv' + fm_qkv = get_module(fm, fm_qkv_name) + fm_qkv.weight.data.copy_(new_fm_attn_weight) + + # update task-specific parameters + fm_abs_name = '.'.join(ss[0: -1]) + '.abs' + fm_abs = get_module(fm, fm_abs_name) + fm_abs._mul_lora_weight.data.copy_(new_fm_lora_weight) # TODO: this will not be applied in inference! + + def get_md_matched_param_of_fm_param(self, fm_param_name): + return super().get_md_matched_param_of_fm_param(fm_param_name) + + def get_md_matched_param_of_sd_param(self, sd_param_name): + # only between qkv.weight, norm.weight/bias + self_param_name = sd_param_name + md = self.models_dict['md'] + if any([k in self_param_name for k in ['fbs', 'cls_token', 'pos_embed']]): + return None + + p = get_parameter(self.models_dict['sd'], self_param_name) + if p.dim() == 0: + return None + elif p.dim() == 1 and 'norm' in self_param_name and 'weight' in self_param_name: + return get_parameter(md, self_param_name) + + # 1. xx.qkv.to_qkv.yy to xx.qkv.qkv.aa and xx.qkv.abs.zz + if 'qkv.weight' in self_param_name: + return get_parameter(md, self_param_name) + + # elif 'to_qkv.bias' in self_param_name: + # ss = self_param_name.split('.') + + # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv.bias' + # return get_parameter(fm, fm_qkv_name) + + elif 'mlp.fc1.0.weight' in self_param_name: + fm_param_name = '.'.join(self_param_name.split('.')[0: -2]) + '.linear.weight' + return get_parameter(md, fm_param_name) + + elif 'mlp.fc2' in self_param_name and 'weight' in self_param_name: + fm_param_name = self_param_name + return get_parameter(md, fm_param_name) + + else: + # return get_parameter(fm, self_param_name) + return None + + def get_task_head_params(self): + head = get_module(self.models_dict['sd'], 'head') + return list(head.parameters()) + + + +class ClsOnlineFeatAlignModel(OnlineFeatAlignModel): + def get_trained_params(self): # TODO: elastic fm only train a part of params + qkv_and_norm_params = [p for n, p in self.models_dict['main'].named_parameters() if 'qkv.weight' in n or 'norm' in n or 'mlp' in n] + return qkv_and_norm_params + + def get_feature_hook(self): + return LayerActivation(get_module(self.models_dict['main'], 'head'), False, self.device) + + def forward_to_get_task_loss(self, x, y): + return F.cross_entropy(self.infer(x), y) + + def get_mmd_loss(self, f1, f2): + return mmd_rbf(f1, f2) + + def infer(self, x, *args, **kwargs): + return self.models_dict['main'](x) + + def get_accuracy(self, test_loader, *args, **kwargs): + acc = 0 + sample_num = 0 + + self.to_eval_mode() + + with torch.no_grad(): + pbar = tqdm.tqdm(enumerate(test_loader), total=len(test_loader), dynamic_ncols=True, leave=False) + for batch_index, (x, y) in pbar: + x, y = x.to(self.device), y.to(self.device) + output = self.infer(x) + pred = F.softmax(output, dim=1).argmax(dim=1) + correct = torch.eq(pred, y).sum().item() + acc += correct + sample_num += len(y) + + pbar.set_description(f'cur_batch_total: {len(y)}, cur_batch_correct: {correct}, ' + f'cur_batch_acc: {(correct / len(y)):.4f}') + + acc /= sample_num + return acc \ No newline at end of file diff --git a/experiments/elasticdnn/vit_b_16/online_new/cls_cl/cls_cl.py b/experiments/elasticdnn/vit_b_16/online_new/cls_cl/cls_cl.py new file mode 100644 index 0000000000000000000000000000000000000000..9713b28af09777244a8e27a628feebedb9e38745 --- /dev/null +++ b/experiments/elasticdnn/vit_b_16/online_new/cls_cl/cls_cl.py @@ -0,0 +1,106 @@ +from typing import List +from data.dataloader import build_dataloader +# from methods.elasticdnn.api.online_model import ElasticDNN_OnlineModel +from methods.elasticdnn.api.online_model_v2 import ElasticDNN_OnlineModel + +import torch +import sys +from torch import nn +from methods.elasticdnn.api.model import ElasticDNN_OfflineSegFMModel, ElasticDNN_OfflineSegMDModel +from methods.elasticdnn.api.algs.md_pretraining_wo_fbs import ElasticDNN_MDPretrainingWoFBSAlg +from methods.elasticdnn.model.base import ElasticDNNUtil +from methods.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from methods.elasticdnn.pipeline.offline.fm_to_md.vit import FM_to_MD_ViT_Util +from methods.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from methods.elasticdnn.pipeline.offline.fm_lora.vit import FMLoRA_ViT_Util +from methods.elasticdnn.model.vit import ElasticViTUtil +from utils.common.file import ensure_dir +from utils.dl.common.model import LayerActivation, get_module, get_parameter +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +from utils.dl.common.loss import CrossEntropyLossSoft +import torch.nn.functional as F +from utils.dl.common.env import create_tbwriter +import os +from utils.common.log import logger +from utils.common.data_record import write_json +# from methods.shot.shot import OnlineShotModel +from methods.ewc.ewc_elasticfm import EWCAlg +import tqdm +from methods.feat_align.mmd import mmd_rbf +from experiments.utils.elasticfm_cl import init_online_model, elasticfm_cl +from data import build_cl_scenario, build_scenario + +device = 'cuda' +app_name = 'cls' +sd_sparsity = 0.8 + +settings = { + 'involve_fm': True +} + +scenario = build_scenario( + source_datasets_name=['GTA5Cls', 'SuperviselyPersonCls'], + target_datasets_order=['CityscapesCls', 'BaiduPersonCls'] * 15, + da_mode='close_set', + data_dirs={ + 'GTA5Cls': '/data/zql/datasets/gta5_for_cls_task', + 'SuperviselyPersonCls': '/data/zql/datasets/supervisely_person_for_cls_task', + 'CityscapesCls': '/data/zql/datasets/cityscapes_for_cls_task', + 'BaiduPersonCls': '/data/zql/datasets/baiduperson_for_cls_task' + }, +) +scenario = build_cl_scenario( + da_scenario=scenario, + target_datasets_name=['COCOCls'], + num_classes_per_task=2, + max_num_tasks=30, + data_dirs={ + 'GTA5Cls': '/data/zql/datasets/gta5_for_cls_task', + 'SuperviselyPersonCls': '/data/zql/datasets/supervisely_person_for_cls_task', + 'COCOCls': '/data/zql/datasets/coco2017_for_cls_task' + }, +) + +from experiments.elasticdnn.vit_b_16.online_new.cls_cl.model import ElasticDNN_ClsOnlineModel +elasticfm_model = ElasticDNN_ClsOnlineModel('cls', init_online_model( + # 'experiments/elasticdnn/vit_b_16/offline/fm_to_md/results/cls_md_index.py/20230529/star_999997-154037-only_prune_mlp/models/fm_best.pt', + # 'experiments/elasticdnn/vit_b_16/offline/fm_to_md/results/cls_md_index.py/20230529/star_999997-154037-only_prune_mlp/models/md_best.pt', + 'experiments/elasticdnn/vit_b_16/offline/fm_to_md/cls/results/cls_md_index.py/20230617/999992-101343-lr1e-5_index_bug_fixed/models/fm_best.pt', + 'experiments/elasticdnn/vit_b_16/offline/fm_to_md/cls/results/cls_md_index.py/20230617/999992-101343-lr1e-5_index_bug_fixed/models/md_best.pt', + 'cls', __file__ +), device, { + 'md_to_fm_alpha': 1, + 'fm_to_md_alpha': 1 +}) + +da_alg = EWCAlg +from experiments.elasticdnn.vit_b_16.online_new.cls_cl.model import ClsOnlineEWCModel +da_model = ClsOnlineEWCModel +da_alg_hyp = { + 'train_batch_size': 64, + 'val_batch_size': 512, + 'num_workers': 8, + 'optimizer': 'AdamW', + 'optimizer_args': {'lr': 1e-4, 'betas': [0.9, 0.999], 'weight_decay': 0.01}, + 'scheduler': '', + 'scheduler_args': {}, + 'num_iters': 100, + 'val_freq': 20, + 'reg_weight': 100000000., + 'sd_sparsity': sd_sparsity, +} + + +elasticfm_cl( + [app_name], + [scenario], + [elasticfm_model], + [da_alg], + [da_alg_hyp], + [da_model], + device, + settings, + __file__, + sys.argv[1] +) diff --git a/experiments/elasticdnn/vit_b_16/online_new/cls_cl/gem/cls_cl.py b/experiments/elasticdnn/vit_b_16/online_new/cls_cl/gem/cls_cl.py new file mode 100644 index 0000000000000000000000000000000000000000..5e4b6d4653f0d259c231b5a556eb7e70fc907406 --- /dev/null +++ b/experiments/elasticdnn/vit_b_16/online_new/cls_cl/gem/cls_cl.py @@ -0,0 +1,109 @@ +from typing import List +from data.dataloader import build_dataloader +# from methods.elasticdnn.api.online_model import ElasticDNN_OnlineModel +from methods.elasticdnn.api.online_model_v2 import ElasticDNN_OnlineModel + +import torch +import sys +from torch import nn +from methods.elasticdnn.api.model import ElasticDNN_OfflineSegFMModel, ElasticDNN_OfflineSegMDModel +from methods.elasticdnn.api.algs.md_pretraining_wo_fbs import ElasticDNN_MDPretrainingWoFBSAlg +from methods.elasticdnn.model.base import ElasticDNNUtil +from methods.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from methods.elasticdnn.pipeline.offline.fm_to_md.vit import FM_to_MD_ViT_Util +from methods.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from methods.elasticdnn.pipeline.offline.fm_lora.vit import FMLoRA_ViT_Util +from methods.elasticdnn.model.vit import ElasticViTUtil +from utils.common.file import ensure_dir +from utils.dl.common.model import LayerActivation, get_module, get_parameter +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +from utils.dl.common.loss import CrossEntropyLossSoft +import torch.nn.functional as F +from utils.dl.common.env import create_tbwriter +import os +from utils.common.log import logger +from utils.common.data_record import write_json +# from methods.shot.shot import OnlineShotModel +from methods.gem.gem_el import GEMAlg +import tqdm +from methods.feat_align.mmd import mmd_rbf +from experiments.utils.elasticfm_cl import init_online_model, elasticfm_cl +from data import build_cl_scenario, build_scenario + +device = 'cuda' +app_name = 'cls' +sd_sparsity = 0.8 + +settings = { + 'involve_fm': True +} + +scenario = build_scenario( + source_datasets_name=['GTA5Cls', 'SuperviselyPersonCls'], + target_datasets_order=['CityscapesCls', 'BaiduPersonCls'] * 15, + da_mode='close_set', + data_dirs={ + 'GTA5Cls': '/data/zql/datasets/gta5_for_cls_task', + 'SuperviselyPersonCls': '/data/zql/datasets/supervisely_person_for_cls_task', + 'CityscapesCls': '/data/zql/datasets/cityscapes_for_cls_task', + 'BaiduPersonCls': '/data/zql/datasets/baiduperson_for_cls_task' + }, +) +scenario = build_cl_scenario( + da_scenario=scenario, + target_datasets_name=['COCOCls'], + num_classes_per_task=2, + max_num_tasks=30, + data_dirs={ + 'GTA5Cls': '/data/zql/datasets/gta5_for_cls_task', + 'SuperviselyPersonCls': '/data/zql/datasets/supervisely_person_for_cls_task', + 'COCOCls': '/data/zql/datasets/coco2017_for_cls_task' + }, +) + +from experiments.elasticdnn.vit_b_16.online_new.cls_cl.gem.model import ElasticDNN_ClsOnlineModel +elasticfm_model = ElasticDNN_ClsOnlineModel('cls', init_online_model( + # 'experiments/elasticdnn/vit_b_16/offline/fm_to_md/results/cls_md_index.py/20230529/star_999997-154037-only_prune_mlp/models/fm_best.pt', + # 'experiments/elasticdnn/vit_b_16/offline/fm_to_md/results/cls_md_index.py/20230529/star_999997-154037-only_prune_mlp/models/md_best.pt', + 'experiments/elasticdnn/vit_b_16/offline/fm_to_md/cls/results/cls_md_index.py/20230617/999992-101343-lr1e-5_index_bug_fixed/models/fm_best.pt', + 'experiments/elasticdnn/vit_b_16/offline/fm_to_md/cls/results/cls_md_index.py/20230617/999992-101343-lr1e-5_index_bug_fixed/models/md_best.pt', + 'cls', __file__ +), device, { + 'md_to_fm_alpha': 1, + 'fm_to_md_alpha': 1 +}) + +da_alg = GEMAlg +from experiments.elasticdnn.vit_b_16.online_new.cls_cl.gem.model import ClsOnlineGEMModel +da_model = ClsOnlineGEMModel +da_alg_hyp = { + 'train_batch_size': 64, + 'val_batch_size': 512, + 'num_workers': 8, + 'optimizer': 'AdamW', + 'optimizer_args': {'lr': 1e-4, 'betas': [0.9, 0.999], 'weight_decay': 0.01}, + 'scheduler': '', + 'scheduler_args': {}, + 'num_iters': 100, + 'val_freq': 20, + 'n_memories': 64, + 'n_inputs': 3 * 224 * 224, + 'margin': 0.5, + 'num_my_iters': 3, + 'sd_sparsity': sd_sparsity, +} + + +elasticfm_cl( + [app_name], + [scenario], + [elasticfm_model], + [da_alg], + [da_alg_hyp], + [da_model], + device, + settings, + __file__, + sys.argv[1] +) diff --git a/experiments/elasticdnn/vit_b_16/online_new/cls_cl/gem/model.py b/experiments/elasticdnn/vit_b_16/online_new/cls_cl/gem/model.py new file mode 100644 index 0000000000000000000000000000000000000000..b8ac96bb338cd8e73143ef9951a956a96ee5b514 --- /dev/null +++ b/experiments/elasticdnn/vit_b_16/online_new/cls_cl/gem/model.py @@ -0,0 +1,356 @@ +from typing import List +from data.dataloader import build_dataloader +# from methods.elasticdnn.api.online_model import ElasticDNN_OnlineModel +from methods.elasticdnn.api.online_model_v2 import ElasticDNN_OnlineModel + +import torch +import sys +from torch import nn +from methods.elasticdnn.api.model import ElasticDNN_OfflineSegFMModel, ElasticDNN_OfflineSegMDModel +from methods.elasticdnn.api.algs.md_pretraining_wo_fbs import ElasticDNN_MDPretrainingWoFBSAlg +from methods.elasticdnn.model.base import ElasticDNNUtil +from methods.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from methods.elasticdnn.pipeline.offline.fm_to_md.vit import FM_to_MD_ViT_Util +from methods.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from methods.elasticdnn.pipeline.offline.fm_lora.vit import FMLoRA_ViT_Util +from methods.elasticdnn.model.vit import ElasticViTUtil +from utils.common.file import ensure_dir +from utils.dl.common.model import LayerActivation, get_module, get_parameter, set_module +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +from utils.dl.common.loss import CrossEntropyLossSoft +import torch.nn.functional as F +from utils.dl.common.env import create_tbwriter +import os +from utils.common.log import logger +from utils.common.data_record import write_json +# from methods.shot.shot import OnlineShotModel +from methods.ewc.ewc_elasticfm import OnlineEWCModel +import tqdm +# from methods.feat_align.mmd import mmd_rbf +from copy import deepcopy + + +class ElasticDNN_ClsOnlineModel(ElasticDNN_OnlineModel): + @torch.no_grad() + def sd_feedback_to_md(self, after_da_sd, unpruned_indexes_of_layers): + self.models_dict['sd'] = after_da_sd + self.before_da_md = deepcopy(self.models_dict['md']) + + logger.info('\n\nsurrogate DNN feedback to master DNN...\n\n') + # one-to-one + + cur_unpruned_indexes = None + cur_unpruned_indexes_name = None + + for p_name, p in self.models_dict['sd'].named_parameters(): + matched_md_param = self.get_md_matched_param_of_sd_param(p_name) + logger.debug(f'if feedback: {p_name}') + if matched_md_param is None: + continue + logger.debug(f'start feedback: {p_name}, {p.size()} -> {matched_md_param.size()}') + # average + # setattr(matched_md_module, matched_md_param_name, (matched_md_param + p) / 2.) + + if p_name in unpruned_indexes_of_layers.keys(): + cur_unpruned_indexes = unpruned_indexes_of_layers[p_name] + cur_unpruned_indexes_name = p_name + + if p.size() != matched_md_param.size(): + logger.debug(f'cur unpruned indexes: {cur_unpruned_indexes_name}, {cur_unpruned_indexes.size()}') + + if p.dim() == 1: # norm + new_p = deepcopy(matched_md_param) + new_p[cur_unpruned_indexes] = p + elif p.dim() == 2: # linear + if p.size(0) < matched_md_param.size(0): # output pruned + new_p = deepcopy(matched_md_param) + new_p[cur_unpruned_indexes] = p + else: # input pruned + new_p = deepcopy(matched_md_param) + new_p[:, cur_unpruned_indexes] = p + p = new_p + + assert p.size() == matched_md_param.size(), f'{p.size()}, {matched_md_param.size()}' + + if 'head' in p_name: + # if False: + # self.last_trained_cls_indexes + assert hasattr(self, 'last_trained_cls_indexes') + print(self.last_trained_cls_indexes) + + diff = self._compute_diff(matched_md_param, p) + # matched_md_param[self.last_trained_cls_indexes].copy_(p[self.last_trained_cls_indexes.to(self.device)]) + matched_md_param.copy_(p) + logger.debug(f'SPECIFIC FOR CL HEAD | end feedback: {p_name}, diff: {diff:.6f}') + else: + diff = self._compute_diff(matched_md_param, (matched_md_param + p) / 2.) + matched_md_param.copy_((matched_md_param + p) / 2.) + logger.debug(f'end feedback: {p_name}, diff: {diff:.6f}') + + def add_cls_in_head(self, num_cls): + head: nn.Linear = get_module(self.models_dict['md'], 'head') + + new_head = nn.Linear(head.in_features, head.out_features + num_cls, head.bias is not None, device=self.device) + + nn.init.zeros_(new_head.weight.data) + nn.init.zeros_(new_head.bias.data) + + new_head.weight.data[0: head.out_features] = deepcopy(head.weight.data) + new_head.bias.data[0: head.out_features] = deepcopy(head.bias.data) + set_module(self.models_dict['md'], 'head', new_head) + set_module(self.models_dict['fm'], 'head', new_head) + + def get_accuracy(self, test_loader, *args, **kwargs): + acc = 0 + sample_num = 0 + + self.to_eval_mode() + + with torch.no_grad(): + pbar = tqdm.tqdm(enumerate(test_loader), total=len(test_loader), dynamic_ncols=True, leave=False) + for batch_index, (x, y) in pbar: + x, y = x.to(self.device), y.to(self.device) + output = self.infer(x) + pred = F.softmax(output, dim=1).argmax(dim=1) + correct = torch.eq(pred, y).sum().item() + acc += correct + sample_num += len(y) + + pbar.set_description(f'cur_batch_total: {len(y)}, cur_batch_correct: {correct}, ' + f'cur_batch_acc: {(correct / len(y)):.4f}') + + acc /= sample_num + return acc + + def get_elastic_dnn_util(self) -> ElasticDNNUtil: + return ElasticViTUtil() + + def get_fm_matched_param_of_md_param(self, md_param_name): + # only between qkv.weight, norm.weight/bias + self_param_name = md_param_name + fm = self.models_dict['fm'] + if any([k in self_param_name for k in ['fbs', 'cls_token', 'pos_embed']]): + return None + + p = get_parameter(self.models_dict['md'], self_param_name) + if p.dim() == 0: + return None + elif p.dim() == 1 and 'norm' in self_param_name and 'weight' in self_param_name: + return get_parameter(fm, self_param_name) + + # if 'head' in self_param_name: + # return get_parameter(fm, self_param_name) + + # 1. xx.qkv.to_qkv.yy to xx.qkv.qkv.aa and xx.qkv.abs.zz + if 'qkv.weight' in self_param_name: + ss = self_param_name.split('.') + + fm_qkv_name = '.'.join(ss[0: -1]) + '.qkv' + fm_qkv = get_module(fm, fm_qkv_name) + + fm_abs_name = '.'.join(ss[0: -1]) + '.abs' + fm_abs = get_module(fm, fm_abs_name) + + # NOTE: unrecoverable operation! multiply LoRA parameters to allow it being updated in update_fm_param() + # TODO: if fm will be used for inference, _mul_lora_weight will not be applied! + if not hasattr(fm_abs, '_mul_lora_weight'): + logger.debug(f'set _mul_lora_weight in {fm_abs_name}') + setattr(fm_abs, '_mul_lora_weight', + nn.Parameter(torch.cat([(_abs[0].weight.T @ _abs[1].weight.T).T for _abs in fm_abs], dim=0))) + + return torch.cat([ + fm_qkv.weight.data, # task-agnositc params + fm_abs._mul_lora_weight.data # task-specific params (LoRA) + ], dim=0) + + # elif 'to_qkv.bias' in self_param_name: + # ss = self_param_name.split('.') + + # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv.bias' + # return get_parameter(fm, fm_qkv_name) + + elif 'mlp.fc1' in self_param_name and 'weight' in self_param_name: + fm_param_name = self_param_name.replace('.linear', '') + return get_parameter(fm, fm_param_name) + + elif 'mlp.fc2' in self_param_name and 'weight' in self_param_name: + fm_param_name = self_param_name + return get_parameter(fm, fm_param_name) + + else: + # return get_parameter(fm, self_param_name) + return None + + def update_fm_param(self, md_param_name, cal_new_fm_param_by_md_param): + if not 'qkv.weight' in md_param_name: + matched_fm_param_ref = self.get_fm_matched_param_of_md_param(md_param_name) + matched_fm_param_ref.copy_(cal_new_fm_param_by_md_param) + else: + new_fm_attn_weight, new_fm_lora_weight = torch.chunk(cal_new_fm_param_by_md_param, 2, 0) + + ss = md_param_name.split('.') + fm = self.models_dict['fm'] + + # update task-agnostic parameters + fm_qkv_name = '.'.join(ss[0: -1]) + '.qkv' + fm_qkv = get_module(fm, fm_qkv_name) + fm_qkv.weight.data.copy_(new_fm_attn_weight) + + # update task-specific parameters + fm_abs_name = '.'.join(ss[0: -1]) + '.abs' + fm_abs = get_module(fm, fm_abs_name) + fm_abs._mul_lora_weight.data.copy_(new_fm_lora_weight) # TODO: this will not be applied in inference! + + def get_md_matched_param_of_fm_param(self, fm_param_name): + return super().get_md_matched_param_of_fm_param(fm_param_name) + + def get_md_matched_param_of_sd_param(self, sd_param_name): + # only between qkv.weight, norm.weight/bias + self_param_name = sd_param_name + md = self.models_dict['md'] + if any([k in self_param_name for k in ['fbs', 'cls_token', 'pos_embed']]): + return None + + p = get_parameter(self.models_dict['sd'], self_param_name) + if p.dim() == 0: + return None + elif p.dim() == 1 and 'norm' in self_param_name and 'weight' in self_param_name: + return get_parameter(md, self_param_name) + + if 'head' in self_param_name: + return get_parameter(md, self_param_name) + + # 1. xx.qkv.to_qkv.yy to xx.qkv.qkv.aa and xx.qkv.abs.zz + if 'qkv.weight' in self_param_name: + return get_parameter(md, self_param_name) + + # elif 'to_qkv.bias' in self_param_name: + # ss = self_param_name.split('.') + + # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv.bias' + # return get_parameter(fm, fm_qkv_name) + + elif 'mlp.fc1.0.weight' in self_param_name: + fm_param_name = '.'.join(self_param_name.split('.')[0: -2]) + '.linear.weight' + return get_parameter(md, fm_param_name) + + elif 'mlp.fc2' in self_param_name and 'weight' in self_param_name: + fm_param_name = self_param_name + return get_parameter(md, fm_param_name) + + else: + # return get_parameter(fm, self_param_name) + return None + + def get_task_head_params(self): + head = get_module(self.models_dict['sd'], 'head') + return list(head.parameters()) + + + + +from typing import List, Tuple +from data.dataloader import build_dataloader +# from methods.elasticdnn.api.online_model import ElasticDNN_OnlineModel +from methods.elasticdnn.api.online_model_v2 import ElasticDNN_OnlineModel + +import torch +import sys +from torch import nn +from methods.elasticdnn.api.model import ElasticDNN_OfflineSegFMModel, ElasticDNN_OfflineSegMDModel +from methods.elasticdnn.api.algs.md_pretraining_wo_fbs import ElasticDNN_MDPretrainingWoFBSAlg +from methods.elasticdnn.model.base import ElasticDNNUtil +from methods.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from methods.elasticdnn.pipeline.offline.fm_to_md.vit import FM_to_MD_ViT_Util +from methods.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from methods.elasticdnn.pipeline.offline.fm_lora.vit import FMLoRA_ViT_Util +from methods.elasticdnn.model.vit import ElasticViTUtil +from utils.common.file import ensure_dir +from utils.dl.common.model import LayerActivation, LayerActivation2, get_module, get_parameter, set_module +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +from utils.dl.common.loss import CrossEntropyLossSoft +import torch.nn.functional as F +from utils.dl.common.env import create_tbwriter +import os +from utils.common.log import logger +from utils.common.data_record import write_json +# from methods.shot.shot import OnlineShotModel +from methods.gem.gem_el import OnlineGEMModel +import tqdm +from methods.feat_align.mmd import mmd_rbf +from copy import deepcopy + + +class ClsOnlineGEMModel(OnlineGEMModel): + def get_trained_params(self): + qkv_and_norm_params = [p for n, p in self.models_dict['main'].named_parameters() if 'qkv.weight' in n or 'norm' in n or 'mlp' in n or 'head' in n] + return qkv_and_norm_params + + def forward_to_get_task_loss(self, x, y): + return F.cross_entropy(self.infer(x), y) + + def add_cls_in_head(self, num_cls): + return + + head: nn.Linear = get_module(self.models_dict['main'], 'head') + + new_head = nn.Linear(head.in_features, head.out_features + num_cls, head.bias is not None, device=self.device) + new_head.weight.data[0: head.out_features] = deepcopy(head.weight.data) + new_head.bias.data[0: head.out_features] = deepcopy(head.bias.data) + set_module(self.models_dict['main'], 'head', new_head) + + def infer(self, x, *args, **kwargs): + return self.models_dict['main'](x) + + def get_accuracy(self, test_loader, *args, **kwargs): + _d = test_loader.dataset + from data import build_dataloader, split_dataset + if _d.__class__.__name__ == '_SplitDataset' and _d.underlying_dataset.__class__.__name__ == 'MergedDataset': # necessary for CL + print('\neval on merged datasets') + + merged_full_dataset = _d.underlying_dataset.datasets + ratio = len(_d.keys) / len(_d.underlying_dataset) + + if int(len(_d) * ratio) == 0: + ratio = 1. + # print(ratio) + # bs = + # test_loaders = [build_dataloader(split_dataset(d, min(max(test_loader.batch_size, int(len(d) * ratio)), len(d)))[0], # TODO: this might be overlapped with train dataset + # min(test_loader.batch_size, int(len(d) * ratio)), + # test_loader.num_workers, False, None) for d in merged_full_dataset] + + test_loaders = [] + for d in merged_full_dataset: + n = int(len(d) * ratio) + if n == 0: + n = len(d) + sub_dataset = split_dataset(d, min(max(test_loader.batch_size, n), len(d)))[0] + loader = build_dataloader(sub_dataset, min(test_loader.batch_size, n), test_loader.num_workers, False, None) + test_loaders += [loader] + + accs = [self.get_accuracy(loader) for loader in test_loaders] + print(accs) + return sum(accs) / len(accs) + + acc = 0 + sample_num = 0 + + self.to_eval_mode() + + with torch.no_grad(): + pbar = tqdm.tqdm(enumerate(test_loader), total=len(test_loader), dynamic_ncols=True, leave=False) + for batch_index, (x, y) in pbar: + x, y = x.to(self.device), y.to(self.device) + output = self.infer(x) + pred = F.softmax(output, dim=1).argmax(dim=1) + correct = torch.eq(pred, y).sum().item() + acc += correct + sample_num += len(y) + + pbar.set_description(f'cur_batch_total: {len(y)}, cur_batch_correct: {correct}, ' + f'cur_batch_acc: {(correct / len(y)):.4f}') + + acc /= sample_num + return acc \ No newline at end of file diff --git a/experiments/elasticdnn/vit_b_16/online_new/cls_cl/gem2/cls_cl.py b/experiments/elasticdnn/vit_b_16/online_new/cls_cl/gem2/cls_cl.py new file mode 100644 index 0000000000000000000000000000000000000000..6c6d002dbf1764dab26b09214118def5e0e86807 --- /dev/null +++ b/experiments/elasticdnn/vit_b_16/online_new/cls_cl/gem2/cls_cl.py @@ -0,0 +1,109 @@ +from typing import List +from data.dataloader import build_dataloader +# from methods.elasticdnn.api.online_model import ElasticDNN_OnlineModel +from methods.elasticdnn.api.online_model_v2 import ElasticDNN_OnlineModel + +import torch +import sys +from torch import nn +from methods.elasticdnn.api.model import ElasticDNN_OfflineSegFMModel, ElasticDNN_OfflineSegMDModel +from methods.elasticdnn.api.algs.md_pretraining_wo_fbs import ElasticDNN_MDPretrainingWoFBSAlg +from methods.elasticdnn.model.base import ElasticDNNUtil +from methods.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from methods.elasticdnn.pipeline.offline.fm_to_md.vit import FM_to_MD_ViT_Util +from methods.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from methods.elasticdnn.pipeline.offline.fm_lora.vit import FMLoRA_ViT_Util +from methods.elasticdnn.model.vit import ElasticViTUtil +from utils.common.file import ensure_dir +from utils.dl.common.model import LayerActivation, get_module, get_parameter +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +from utils.dl.common.loss import CrossEntropyLossSoft +import torch.nn.functional as F +from utils.dl.common.env import create_tbwriter +import os +from utils.common.log import logger +from utils.common.data_record import write_json +# from methods.shot.shot import OnlineShotModel +from methods.gem.gem_el import GEMAlg +import tqdm +from methods.feat_align.mmd import mmd_rbf +from experiments.utils.elasticfm_cl import init_online_model, elasticfm_cl +from data import build_cl_scenario, build_scenario + +device = 'cuda' +app_name = 'cls' +sd_sparsity = 0.8 + +settings = { + 'involve_fm': True +} + +scenario = build_scenario( + source_datasets_name=['GTA5Cls', 'SuperviselyPersonCls'], + target_datasets_order=['CityscapesCls', 'BaiduPersonCls'] * 15, + da_mode='close_set', + data_dirs={ + 'GTA5Cls': '/data/zql/datasets/gta5_for_cls_task', + 'SuperviselyPersonCls': '/data/zql/datasets/supervisely_person_for_cls_task', + 'CityscapesCls': '/data/zql/datasets/cityscapes_for_cls_task', + 'BaiduPersonCls': '/data/zql/datasets/baiduperson_for_cls_task' + }, +) +scenario = build_cl_scenario( + da_scenario=scenario, + target_datasets_name=['COCOCls'] * 4, + num_classes_per_task=10, + max_num_tasks=30, + data_dirs={ + 'GTA5Cls': '/data/zql/datasets/gta5_for_cls_task', + 'SuperviselyPersonCls': '/data/zql/datasets/supervisely_person_for_cls_task', + 'COCOCls': '/data/zql/datasets/coco2017_for_cls_task' + }, +) + +from experiments.elasticdnn.vit_b_16.online_new.cls_cl.gem2.model import ElasticDNN_ClsOnlineModel +elasticfm_model = ElasticDNN_ClsOnlineModel('cls', init_online_model( + # 'experiments/elasticdnn/vit_b_16/offline/fm_to_md/results/cls_md_index.py/20230529/star_999997-154037-only_prune_mlp/models/fm_best.pt', + # 'experiments/elasticdnn/vit_b_16/offline/fm_to_md/results/cls_md_index.py/20230529/star_999997-154037-only_prune_mlp/models/md_best.pt', + 'experiments/elasticdnn/vit_b_16/offline/fm_to_md/cls/results/cls_md_index.py/20230617/999992-101343-lr1e-5_index_bug_fixed/models/fm_best.pt', + 'experiments/elasticdnn/vit_b_16/offline/fm_to_md/cls/results/cls_md_index.py/20230617/999992-101343-lr1e-5_index_bug_fixed/models/md_best.pt', + 'cls', __file__ +), device, { + 'md_to_fm_alpha': 0.1, + 'fm_to_md_alpha': 0.1 +}) + +da_alg = GEMAlg +from experiments.elasticdnn.vit_b_16.online_new.cls_cl.gem2.model import ClsOnlineGEMModel +da_model = ClsOnlineGEMModel +da_alg_hyp = { + 'train_batch_size': 64, + 'val_batch_size': 512, + 'num_workers': 8, + 'optimizer': 'AdamW', + 'optimizer_args': {'lr': 5e-4, 'betas': [0.9, 0.999], 'weight_decay': 0.01}, + 'scheduler': '', + 'scheduler_args': {}, + 'num_iters': 100, + 'val_freq': 20, + 'n_memories': 64, + 'n_inputs': 3 * 224 * 224, + 'margin': 0.5, + 'num_my_iters': 0, + 'sd_sparsity': sd_sparsity, +} + + +elasticfm_cl( + [app_name], + [scenario], + [elasticfm_model], + [da_alg], + [da_alg_hyp], + [da_model], + device, + settings, + __file__, + sys.argv[1] +) diff --git a/experiments/elasticdnn/vit_b_16/online_new/cls_cl/gem2/model.py b/experiments/elasticdnn/vit_b_16/online_new/cls_cl/gem2/model.py new file mode 100644 index 0000000000000000000000000000000000000000..3ff7df29283c74b2279d2779b84e9c844dcba4dd --- /dev/null +++ b/experiments/elasticdnn/vit_b_16/online_new/cls_cl/gem2/model.py @@ -0,0 +1,357 @@ +from typing import List +from data.dataloader import build_dataloader +# from methods.elasticdnn.api.online_model import ElasticDNN_OnlineModel +from methods.elasticdnn.api.online_model_v2 import ElasticDNN_OnlineModel + +import torch +import sys +from torch import nn +from methods.elasticdnn.api.model import ElasticDNN_OfflineSegFMModel, ElasticDNN_OfflineSegMDModel +from methods.elasticdnn.api.algs.md_pretraining_wo_fbs import ElasticDNN_MDPretrainingWoFBSAlg +from methods.elasticdnn.model.base import ElasticDNNUtil +from methods.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from methods.elasticdnn.pipeline.offline.fm_to_md.vit import FM_to_MD_ViT_Util +from methods.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from methods.elasticdnn.pipeline.offline.fm_lora.vit import FMLoRA_ViT_Util +from methods.elasticdnn.model.vit import ElasticViTUtil +from utils.common.file import ensure_dir +from utils.dl.common.model import LayerActivation, get_module, get_parameter, set_module +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +from utils.dl.common.loss import CrossEntropyLossSoft +import torch.nn.functional as F +from utils.dl.common.env import create_tbwriter +import os +from utils.common.log import logger +from utils.common.data_record import write_json +# from methods.shot.shot import OnlineShotModel +from methods.ewc.ewc_elasticfm import OnlineEWCModel +import tqdm +# from methods.feat_align.mmd import mmd_rbf +from copy import deepcopy + + +class ElasticDNN_ClsOnlineModel(ElasticDNN_OnlineModel): + @torch.no_grad() + def sd_feedback_to_md(self, after_da_sd, unpruned_indexes_of_layers): + self.models_dict['sd'] = after_da_sd + self.before_da_md = deepcopy(self.models_dict['md']) + + logger.info('\n\nsurrogate DNN feedback to master DNN...\n\n') + # one-to-one + + cur_unpruned_indexes = None + cur_unpruned_indexes_name = None + + for p_name, p in self.models_dict['sd'].named_parameters(): + matched_md_param = self.get_md_matched_param_of_sd_param(p_name) + logger.debug(f'if feedback: {p_name}') + if matched_md_param is None: + continue + logger.debug(f'start feedback: {p_name}, {p.size()} -> {matched_md_param.size()}') + # average + # setattr(matched_md_module, matched_md_param_name, (matched_md_param + p) / 2.) + + if p_name in unpruned_indexes_of_layers.keys(): + cur_unpruned_indexes = unpruned_indexes_of_layers[p_name] + cur_unpruned_indexes_name = p_name + + if p.size() != matched_md_param.size(): + logger.debug(f'cur unpruned indexes: {cur_unpruned_indexes_name}, {cur_unpruned_indexes.size()}') + + if p.dim() == 1: # norm + new_p = deepcopy(matched_md_param) + new_p[cur_unpruned_indexes] = p + elif p.dim() == 2: # linear + if p.size(0) < matched_md_param.size(0): # output pruned + new_p = deepcopy(matched_md_param) + new_p[cur_unpruned_indexes] = p + else: # input pruned + new_p = deepcopy(matched_md_param) + new_p[:, cur_unpruned_indexes] = p + p = new_p + + assert p.size() == matched_md_param.size(), f'{p.size()}, {matched_md_param.size()}' + + if 'head' in p_name: + continue + # if False: + # self.last_trained_cls_indexes + assert hasattr(self, 'last_trained_cls_indexes') + print(self.last_trained_cls_indexes) + + diff = self._compute_diff(matched_md_param, p) + # matched_md_param[self.last_trained_cls_indexes].copy_(p[self.last_trained_cls_indexes.to(self.device)]) + matched_md_param.copy_(p) + logger.debug(f'SPECIFIC FOR CL HEAD | end feedback: {p_name}, diff: {diff:.6f}') + else: + diff = self._compute_diff(matched_md_param, (matched_md_param + p) / 2.) + matched_md_param.copy_((matched_md_param + p) / 2.) + logger.debug(f'end feedback: {p_name}, diff: {diff:.6f}') + + def add_cls_in_head(self, num_cls): + head: nn.Linear = get_module(self.models_dict['md'], 'head') + + new_head = nn.Linear(head.in_features, head.out_features + num_cls, head.bias is not None, device=self.device) + + # nn.init.zeros_(new_head.weight.data) + # nn.init.zeros_(new_head.bias.data) + + new_head.weight.data[0: head.out_features] = deepcopy(head.weight.data) + new_head.bias.data[0: head.out_features] = deepcopy(head.bias.data) + set_module(self.models_dict['md'], 'head', new_head) + set_module(self.models_dict['fm'], 'head', new_head) + + def get_accuracy(self, test_loader, *args, **kwargs): + acc = 0 + sample_num = 0 + + self.to_eval_mode() + + with torch.no_grad(): + pbar = tqdm.tqdm(enumerate(test_loader), total=len(test_loader), dynamic_ncols=True, leave=False) + for batch_index, (x, y) in pbar: + x, y = x.to(self.device), y.to(self.device) + output = self.infer(x) + pred = F.softmax(output, dim=1).argmax(dim=1) + correct = torch.eq(pred, y).sum().item() + acc += correct + sample_num += len(y) + + pbar.set_description(f'cur_batch_total: {len(y)}, cur_batch_correct: {correct}, ' + f'cur_batch_acc: {(correct / len(y)):.4f}') + + acc /= sample_num + return acc + + def get_elastic_dnn_util(self) -> ElasticDNNUtil: + return ElasticViTUtil() + + def get_fm_matched_param_of_md_param(self, md_param_name): + # only between qkv.weight, norm.weight/bias + self_param_name = md_param_name + fm = self.models_dict['fm'] + if any([k in self_param_name for k in ['fbs', 'cls_token', 'pos_embed']]): + return None + + p = get_parameter(self.models_dict['md'], self_param_name) + if p.dim() == 0: + return None + elif p.dim() == 1 and 'norm' in self_param_name and 'weight' in self_param_name: + return get_parameter(fm, self_param_name) + + # if 'head' in self_param_name: + # return get_parameter(fm, self_param_name) + + # 1. xx.qkv.to_qkv.yy to xx.qkv.qkv.aa and xx.qkv.abs.zz + if 'qkv.weight' in self_param_name: + ss = self_param_name.split('.') + + fm_qkv_name = '.'.join(ss[0: -1]) + '.qkv' + fm_qkv = get_module(fm, fm_qkv_name) + + fm_abs_name = '.'.join(ss[0: -1]) + '.abs' + fm_abs = get_module(fm, fm_abs_name) + + # NOTE: unrecoverable operation! multiply LoRA parameters to allow it being updated in update_fm_param() + # TODO: if fm will be used for inference, _mul_lora_weight will not be applied! + if not hasattr(fm_abs, '_mul_lora_weight'): + logger.debug(f'set _mul_lora_weight in {fm_abs_name}') + setattr(fm_abs, '_mul_lora_weight', + nn.Parameter(torch.cat([(_abs[0].weight.T @ _abs[1].weight.T).T for _abs in fm_abs], dim=0))) + + return torch.cat([ + fm_qkv.weight.data, # task-agnositc params + fm_abs._mul_lora_weight.data # task-specific params (LoRA) + ], dim=0) + + # elif 'to_qkv.bias' in self_param_name: + # ss = self_param_name.split('.') + + # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv.bias' + # return get_parameter(fm, fm_qkv_name) + + elif 'mlp.fc1' in self_param_name and 'weight' in self_param_name: + fm_param_name = self_param_name.replace('.linear', '') + return get_parameter(fm, fm_param_name) + + elif 'mlp.fc2' in self_param_name and 'weight' in self_param_name: + fm_param_name = self_param_name + return get_parameter(fm, fm_param_name) + + else: + # return get_parameter(fm, self_param_name) + return None + + def update_fm_param(self, md_param_name, cal_new_fm_param_by_md_param): + if not 'qkv.weight' in md_param_name: + matched_fm_param_ref = self.get_fm_matched_param_of_md_param(md_param_name) + matched_fm_param_ref.copy_(cal_new_fm_param_by_md_param) + else: + new_fm_attn_weight, new_fm_lora_weight = torch.chunk(cal_new_fm_param_by_md_param, 2, 0) + + ss = md_param_name.split('.') + fm = self.models_dict['fm'] + + # update task-agnostic parameters + fm_qkv_name = '.'.join(ss[0: -1]) + '.qkv' + fm_qkv = get_module(fm, fm_qkv_name) + fm_qkv.weight.data.copy_(new_fm_attn_weight) + + # update task-specific parameters + fm_abs_name = '.'.join(ss[0: -1]) + '.abs' + fm_abs = get_module(fm, fm_abs_name) + fm_abs._mul_lora_weight.data.copy_(new_fm_lora_weight) # TODO: this will not be applied in inference! + + def get_md_matched_param_of_fm_param(self, fm_param_name): + return super().get_md_matched_param_of_fm_param(fm_param_name) + + def get_md_matched_param_of_sd_param(self, sd_param_name): + # only between qkv.weight, norm.weight/bias + self_param_name = sd_param_name + md = self.models_dict['md'] + if any([k in self_param_name for k in ['fbs', 'cls_token', 'pos_embed']]): + return None + + p = get_parameter(self.models_dict['sd'], self_param_name) + if p.dim() == 0: + return None + elif p.dim() == 1 and 'norm' in self_param_name and 'weight' in self_param_name: + return get_parameter(md, self_param_name) + + if 'head' in self_param_name: + return get_parameter(md, self_param_name) + + # 1. xx.qkv.to_qkv.yy to xx.qkv.qkv.aa and xx.qkv.abs.zz + if 'qkv.weight' in self_param_name: + return get_parameter(md, self_param_name) + + # elif 'to_qkv.bias' in self_param_name: + # ss = self_param_name.split('.') + + # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv.bias' + # return get_parameter(fm, fm_qkv_name) + + elif 'mlp.fc1.0.weight' in self_param_name: + fm_param_name = '.'.join(self_param_name.split('.')[0: -2]) + '.linear.weight' + return get_parameter(md, fm_param_name) + + elif 'mlp.fc2' in self_param_name and 'weight' in self_param_name: + fm_param_name = self_param_name + return get_parameter(md, fm_param_name) + + else: + # return get_parameter(fm, self_param_name) + return None + + def get_task_head_params(self): + head = get_module(self.models_dict['sd'], 'head') + return list(head.parameters()) + + + + +from typing import List, Tuple +from data.dataloader import build_dataloader +# from methods.elasticdnn.api.online_model import ElasticDNN_OnlineModel +from methods.elasticdnn.api.online_model_v2 import ElasticDNN_OnlineModel + +import torch +import sys +from torch import nn +from methods.elasticdnn.api.model import ElasticDNN_OfflineSegFMModel, ElasticDNN_OfflineSegMDModel +from methods.elasticdnn.api.algs.md_pretraining_wo_fbs import ElasticDNN_MDPretrainingWoFBSAlg +from methods.elasticdnn.model.base import ElasticDNNUtil +from methods.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from methods.elasticdnn.pipeline.offline.fm_to_md.vit import FM_to_MD_ViT_Util +from methods.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from methods.elasticdnn.pipeline.offline.fm_lora.vit import FMLoRA_ViT_Util +from methods.elasticdnn.model.vit import ElasticViTUtil +from utils.common.file import ensure_dir +from utils.dl.common.model import LayerActivation, LayerActivation2, get_module, get_parameter, set_module +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +from utils.dl.common.loss import CrossEntropyLossSoft +import torch.nn.functional as F +from utils.dl.common.env import create_tbwriter +import os +from utils.common.log import logger +from utils.common.data_record import write_json +# from methods.shot.shot import OnlineShotModel +from methods.gem.gem_el import OnlineGEMModel +import tqdm +from methods.feat_align.mmd import mmd_rbf +from copy import deepcopy + + +class ClsOnlineGEMModel(OnlineGEMModel): + def get_trained_params(self): + qkv_and_norm_params = [p for n, p in self.models_dict['main'].named_parameters() if 'qkv.weight' in n or 'norm' in n or 'mlp' in n] + return qkv_and_norm_params + + def forward_to_get_task_loss(self, x, y): + return F.cross_entropy(self.infer(x), y) + + def add_cls_in_head(self, num_cls): + return + + head: nn.Linear = get_module(self.models_dict['main'], 'head') + + new_head = nn.Linear(head.in_features, head.out_features + num_cls, head.bias is not None, device=self.device) + new_head.weight.data[0: head.out_features] = deepcopy(head.weight.data) + new_head.bias.data[0: head.out_features] = deepcopy(head.bias.data) + set_module(self.models_dict['main'], 'head', new_head) + + def infer(self, x, *args, **kwargs): + return self.models_dict['main'](x) + + def get_accuracy(self, test_loader, *args, **kwargs): + _d = test_loader.dataset + from data import build_dataloader, split_dataset + if _d.__class__.__name__ == '_SplitDataset' and _d.underlying_dataset.__class__.__name__ == 'MergedDataset': # necessary for CL + print('\neval on merged datasets') + + merged_full_dataset = _d.underlying_dataset.datasets + ratio = len(_d.keys) / len(_d.underlying_dataset) + + if int(len(_d) * ratio) == 0: + ratio = 1. + # print(ratio) + # bs = + # test_loaders = [build_dataloader(split_dataset(d, min(max(test_loader.batch_size, int(len(d) * ratio)), len(d)))[0], # TODO: this might be overlapped with train dataset + # min(test_loader.batch_size, int(len(d) * ratio)), + # test_loader.num_workers, False, None) for d in merged_full_dataset] + + test_loaders = [] + for d in merged_full_dataset: + n = int(len(d) * ratio) + if n == 0: + n = len(d) + sub_dataset = split_dataset(d, min(max(test_loader.batch_size, n), len(d)))[0] + loader = build_dataloader(sub_dataset, min(test_loader.batch_size, n), test_loader.num_workers, False, None) + test_loaders += [loader] + + accs = [self.get_accuracy(loader) for loader in test_loaders] + print(accs) + return sum(accs) / len(accs) + + acc = 0 + sample_num = 0 + + self.to_eval_mode() + + with torch.no_grad(): + pbar = tqdm.tqdm(enumerate(test_loader), total=len(test_loader), dynamic_ncols=True, leave=False) + for batch_index, (x, y) in pbar: + x, y = x.to(self.device), y.to(self.device) + output = self.infer(x) + pred = F.softmax(output, dim=1).argmax(dim=1) + correct = torch.eq(pred, y).sum().item() + acc += correct + sample_num += len(y) + + pbar.set_description(f'cur_batch_total: {len(y)}, cur_batch_correct: {correct}, ' + f'cur_batch_acc: {(correct / len(y)):.4f}') + + acc /= sample_num + return acc \ No newline at end of file diff --git a/experiments/elasticdnn/vit_b_16/online_new/cls_cl/model.py b/experiments/elasticdnn/vit_b_16/online_new/cls_cl/model.py new file mode 100644 index 0000000000000000000000000000000000000000..6200e8dbacab9d41660be934fbfd9f57e36f7777 --- /dev/null +++ b/experiments/elasticdnn/vit_b_16/online_new/cls_cl/model.py @@ -0,0 +1,333 @@ +from typing import List +from data.dataloader import build_dataloader +# from methods.elasticdnn.api.online_model import ElasticDNN_OnlineModel +from methods.elasticdnn.api.online_model_v2 import ElasticDNN_OnlineModel + +import torch +import sys +from torch import nn +from methods.elasticdnn.api.model import ElasticDNN_OfflineSegFMModel, ElasticDNN_OfflineSegMDModel +from methods.elasticdnn.api.algs.md_pretraining_wo_fbs import ElasticDNN_MDPretrainingWoFBSAlg +from methods.elasticdnn.model.base import ElasticDNNUtil +from methods.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from methods.elasticdnn.pipeline.offline.fm_to_md.vit import FM_to_MD_ViT_Util +from methods.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from methods.elasticdnn.pipeline.offline.fm_lora.vit import FMLoRA_ViT_Util +from methods.elasticdnn.model.vit import ElasticViTUtil +from utils.common.file import ensure_dir +from utils.dl.common.model import LayerActivation, get_module, get_parameter, set_module +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +from utils.dl.common.loss import CrossEntropyLossSoft +import torch.nn.functional as F +from utils.dl.common.env import create_tbwriter +import os +from utils.common.log import logger +from utils.common.data_record import write_json +# from methods.shot.shot import OnlineShotModel +from methods.ewc.ewc_elasticfm import OnlineEWCModel +import tqdm +# from methods.feat_align.mmd import mmd_rbf +from copy import deepcopy + + +class ElasticDNN_ClsOnlineModel(ElasticDNN_OnlineModel): + @torch.no_grad() + def sd_feedback_to_md(self, after_da_sd, unpruned_indexes_of_layers): + self.models_dict['sd'] = after_da_sd + self.before_da_md = deepcopy(self.models_dict['md']) + + logger.info('\n\nsurrogate DNN feedback to master DNN...\n\n') + # one-to-one + + cur_unpruned_indexes = None + cur_unpruned_indexes_name = None + + for p_name, p in self.models_dict['sd'].named_parameters(): + matched_md_param = self.get_md_matched_param_of_sd_param(p_name) + logger.debug(f'if feedback: {p_name}') + if matched_md_param is None: + continue + logger.debug(f'start feedback: {p_name}, {p.size()} -> {matched_md_param.size()}') + # average + # setattr(matched_md_module, matched_md_param_name, (matched_md_param + p) / 2.) + + if p_name in unpruned_indexes_of_layers.keys(): + cur_unpruned_indexes = unpruned_indexes_of_layers[p_name] + cur_unpruned_indexes_name = p_name + + if p.size() != matched_md_param.size(): + logger.debug(f'cur unpruned indexes: {cur_unpruned_indexes_name}, {cur_unpruned_indexes.size()}') + + if p.dim() == 1: # norm + new_p = deepcopy(matched_md_param) + new_p[cur_unpruned_indexes] = p + elif p.dim() == 2: # linear + if p.size(0) < matched_md_param.size(0): # output pruned + new_p = deepcopy(matched_md_param) + new_p[cur_unpruned_indexes] = p + else: # input pruned + new_p = deepcopy(matched_md_param) + new_p[:, cur_unpruned_indexes] = p + p = new_p + + assert p.size() == matched_md_param.size(), f'{p.size()}, {matched_md_param.size()}' + + if 'head' in p_name: + # if False: + # self.last_trained_cls_indexes + assert hasattr(self, 'last_trained_cls_indexes') + print(self.last_trained_cls_indexes) + + diff = self._compute_diff(matched_md_param, p) + # matched_md_param[self.last_trained_cls_indexes].copy_(p[self.last_trained_cls_indexes.to(self.device)]) + matched_md_param.copy_(p) + logger.debug(f'SPECIFIC FOR CL HEAD | end feedback: {p_name}, diff: {diff:.6f}') + else: + diff = self._compute_diff(matched_md_param, (matched_md_param + p) / 2.) + matched_md_param.copy_((matched_md_param + p) / 2.) + logger.debug(f'end feedback: {p_name}, diff: {diff:.6f}') + + def add_cls_in_head(self, num_cls): + head: nn.Linear = get_module(self.models_dict['md'], 'head') + + new_head = nn.Linear(head.in_features, head.out_features + num_cls, head.bias is not None, device=self.device) + + nn.init.zeros_(new_head.weight.data) + nn.init.zeros_(new_head.bias.data) + + new_head.weight.data[0: head.out_features] = deepcopy(head.weight.data) + new_head.bias.data[0: head.out_features] = deepcopy(head.bias.data) + set_module(self.models_dict['md'], 'head', new_head) + set_module(self.models_dict['fm'], 'head', new_head) + + def get_accuracy(self, test_loader, *args, **kwargs): + acc = 0 + sample_num = 0 + + self.to_eval_mode() + + with torch.no_grad(): + pbar = tqdm.tqdm(enumerate(test_loader), total=len(test_loader), dynamic_ncols=True, leave=False) + for batch_index, (x, y) in pbar: + x, y = x.to(self.device), y.to(self.device) + output = self.infer(x) + pred = F.softmax(output, dim=1).argmax(dim=1) + correct = torch.eq(pred, y).sum().item() + acc += correct + sample_num += len(y) + + pbar.set_description(f'cur_batch_total: {len(y)}, cur_batch_correct: {correct}, ' + f'cur_batch_acc: {(correct / len(y)):.4f}') + + acc /= sample_num + return acc + + def get_elastic_dnn_util(self) -> ElasticDNNUtil: + return ElasticViTUtil() + + def get_fm_matched_param_of_md_param(self, md_param_name): + # only between qkv.weight, norm.weight/bias + self_param_name = md_param_name + fm = self.models_dict['fm'] + if any([k in self_param_name for k in ['fbs', 'cls_token', 'pos_embed']]): + return None + + p = get_parameter(self.models_dict['md'], self_param_name) + if p.dim() == 0: + return None + elif p.dim() == 1 and 'norm' in self_param_name and 'weight' in self_param_name: + return get_parameter(fm, self_param_name) + + # if 'head' in self_param_name: + # return get_parameter(fm, self_param_name) + + # 1. xx.qkv.to_qkv.yy to xx.qkv.qkv.aa and xx.qkv.abs.zz + if 'qkv.weight' in self_param_name: + ss = self_param_name.split('.') + + fm_qkv_name = '.'.join(ss[0: -1]) + '.qkv' + fm_qkv = get_module(fm, fm_qkv_name) + + fm_abs_name = '.'.join(ss[0: -1]) + '.abs' + fm_abs = get_module(fm, fm_abs_name) + + # NOTE: unrecoverable operation! multiply LoRA parameters to allow it being updated in update_fm_param() + # TODO: if fm will be used for inference, _mul_lora_weight will not be applied! + if not hasattr(fm_abs, '_mul_lora_weight'): + logger.debug(f'set _mul_lora_weight in {fm_abs_name}') + setattr(fm_abs, '_mul_lora_weight', + nn.Parameter(torch.cat([(_abs[0].weight.T @ _abs[1].weight.T).T for _abs in fm_abs], dim=0))) + + return torch.cat([ + fm_qkv.weight.data, # task-agnositc params + fm_abs._mul_lora_weight.data # task-specific params (LoRA) + ], dim=0) + + # elif 'to_qkv.bias' in self_param_name: + # ss = self_param_name.split('.') + + # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv.bias' + # return get_parameter(fm, fm_qkv_name) + + elif 'mlp.fc1' in self_param_name and 'weight' in self_param_name: + fm_param_name = self_param_name.replace('.linear', '') + return get_parameter(fm, fm_param_name) + + elif 'mlp.fc2' in self_param_name and 'weight' in self_param_name: + fm_param_name = self_param_name + return get_parameter(fm, fm_param_name) + + else: + # return get_parameter(fm, self_param_name) + return None + + def update_fm_param(self, md_param_name, cal_new_fm_param_by_md_param): + if not 'qkv.weight' in md_param_name: + matched_fm_param_ref = self.get_fm_matched_param_of_md_param(md_param_name) + matched_fm_param_ref.copy_(cal_new_fm_param_by_md_param) + else: + new_fm_attn_weight, new_fm_lora_weight = torch.chunk(cal_new_fm_param_by_md_param, 2, 0) + + ss = md_param_name.split('.') + fm = self.models_dict['fm'] + + # update task-agnostic parameters + fm_qkv_name = '.'.join(ss[0: -1]) + '.qkv' + fm_qkv = get_module(fm, fm_qkv_name) + fm_qkv.weight.data.copy_(new_fm_attn_weight) + + # update task-specific parameters + fm_abs_name = '.'.join(ss[0: -1]) + '.abs' + fm_abs = get_module(fm, fm_abs_name) + fm_abs._mul_lora_weight.data.copy_(new_fm_lora_weight) # TODO: this will not be applied in inference! + + def get_md_matched_param_of_fm_param(self, fm_param_name): + return super().get_md_matched_param_of_fm_param(fm_param_name) + + def get_md_matched_param_of_sd_param(self, sd_param_name): + # only between qkv.weight, norm.weight/bias + self_param_name = sd_param_name + md = self.models_dict['md'] + if any([k in self_param_name for k in ['fbs', 'cls_token', 'pos_embed']]): + return None + + p = get_parameter(self.models_dict['sd'], self_param_name) + if p.dim() == 0: + return None + elif p.dim() == 1 and 'norm' in self_param_name and 'weight' in self_param_name: + return get_parameter(md, self_param_name) + + if 'head' in self_param_name: + return get_parameter(md, self_param_name) + + # 1. xx.qkv.to_qkv.yy to xx.qkv.qkv.aa and xx.qkv.abs.zz + if 'qkv.weight' in self_param_name: + return get_parameter(md, self_param_name) + + # elif 'to_qkv.bias' in self_param_name: + # ss = self_param_name.split('.') + + # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv.bias' + # return get_parameter(fm, fm_qkv_name) + + elif 'mlp.fc1.0.weight' in self_param_name: + fm_param_name = '.'.join(self_param_name.split('.')[0: -2]) + '.linear.weight' + return get_parameter(md, fm_param_name) + + elif 'mlp.fc2' in self_param_name and 'weight' in self_param_name: + fm_param_name = self_param_name + return get_parameter(md, fm_param_name) + + else: + # return get_parameter(fm, self_param_name) + return None + + def get_task_head_params(self): + head = get_module(self.models_dict['sd'], 'head') + return list(head.parameters()) + + + +class ClsOnlineEWCModel(OnlineEWCModel): + def get_trained_params(self): + qkv_and_norm_params = [p for n, p in self.models_dict['main'].named_parameters() if 'head' in n] + return qkv_and_norm_params + + def forward_to_get_task_loss(self, x, y, offset, cur_num_cls): + return F.cross_entropy(self.infer(x)[:, 20: offset + cur_num_cls], y - 20) # TODO: debug + + def add_cls_in_head(self, num_cls): + # head: nn.Linear = get_module(self.models_dict['main'], 'head') + + # new_head = nn.Linear(head.in_features, head.out_features + num_cls, head.bias is not None, device=self.device) + # new_head.weight.data[0: head.out_features] = deepcopy(head.weight.data) + # new_head.bias.data[0: head.out_features] = deepcopy(head.bias.data) + # set_module(self.models_dict['main'], 'head', new_head) + pass + + def infer(self, x, *args, **kwargs): + return self.models_dict['main'](x) + + def get_accuracy(self, test_loader, *args, **kwargs): + _d = test_loader.dataset + from data import build_dataloader, split_dataset + if _d.__class__.__name__ == '_SplitDataset' and _d.underlying_dataset.__class__.__name__ == 'MergedDataset': # necessary for CL + print('\neval on merged datasets') + + merged_full_dataset = _d.underlying_dataset.datasets + ratio = len(_d.keys) / len(_d.underlying_dataset) + + if int(len(_d) * ratio) == 0: + ratio = 1. + # print(ratio) + # bs = + # test_loaders = [build_dataloader(split_dataset(d, min(max(test_loader.batch_size, int(len(d) * ratio)), len(d)))[0], # TODO: this might be overlapped with train dataset + # min(test_loader.batch_size, int(len(d) * ratio)), + # test_loader.num_workers, False, None) for d in merged_full_dataset] + + test_loaders = [] + for d in merged_full_dataset: + n = int(len(d) * ratio) + if n == 0: + n = len(d) + sub_dataset = split_dataset(d, min(max(test_loader.batch_size, n), len(d)))[0] + loader = build_dataloader(sub_dataset, min(test_loader.batch_size, n), test_loader.num_workers, False, None) + test_loaders += [loader] + + accs = [self.get_accuracy(loader) for loader in test_loaders] + print(accs) + return sum(accs) / len(accs) + + acc = 0 + sample_num = 0 + + self.to_eval_mode() + + cur_class_offset_end = self.cur_class_offset + source_class_offset = self.source_class_offset + + with torch.no_grad(): + pbar = tqdm.tqdm(enumerate(test_loader), total=len(test_loader), dynamic_ncols=True, leave=False) + for batch_index, (x, y) in pbar: + x, y = x.to(self.device), y.to(self.device) + + output = self.infer(x) + + output = output[:, source_class_offset: cur_class_offset_end] + if batch_index == 0: + print(source_class_offset, cur_class_offset_end) + print(output[0:3]) + print(self.models_dict['main'].head.weight[source_class_offset: ]) + y = y - source_class_offset + + pred = F.softmax(output, dim=1).argmax(dim=1) + correct = torch.eq(pred, y).sum().item() + acc += correct + sample_num += len(y) + + pbar.set_description(f'cur_batch_total: {len(y)}, cur_batch_correct: {correct}, ' + f'cur_batch_acc: {(correct / len(y)):.4f}') + + acc /= sample_num + return acc \ No newline at end of file diff --git a/experiments/elasticdnn/vit_b_16/online_new/det/det.py b/experiments/elasticdnn/vit_b_16/online_new/det/det.py new file mode 100644 index 0000000000000000000000000000000000000000..e51364f41b94b95a93e8c321f50f50268d02cba1 --- /dev/null +++ b/experiments/elasticdnn/vit_b_16/online_new/det/det.py @@ -0,0 +1,104 @@ +from typing import List +from data.dataloader import build_dataloader +# from methods.elasticdnn.api.online_model import ElasticDNN_OnlineModel +from methods.elasticdnn.api.online_model_v2 import ElasticDNN_OnlineModel + +import torch +import sys +from torch import nn +from methods.elasticdnn.api.model import ElasticDNN_OfflineSegFMModel, ElasticDNN_OfflineSegMDModel +from methods.elasticdnn.api.algs.md_pretraining_wo_fbs import ElasticDNN_MDPretrainingWoFBSAlg +from methods.elasticdnn.model.base import ElasticDNNUtil +from methods.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from methods.elasticdnn.pipeline.offline.fm_to_md.vit import FM_to_MD_ViT_Util +from methods.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from methods.elasticdnn.pipeline.offline.fm_lora.vit import FMLoRA_ViT_Util +from methods.elasticdnn.model.vit import ElasticViTUtil +from utils.common.file import ensure_dir +from utils.dl.common.model import LayerActivation, get_module, get_parameter +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +from utils.dl.common.loss import CrossEntropyLossSoft +import torch.nn.functional as F +from utils.dl.common.env import create_tbwriter +import os +from utils.common.log import logger +from utils.common.data_record import write_json +# from methods.shot.shot import OnlineShotModel +from methods.feat_align.main import OnlineFeatAlignModel, FeatAlignAlg +import tqdm +from methods.feat_align.mmd import mmd_rbf +from experiments.utils.elasticfm_da import init_online_model, elasticfm_da + +device = 'cuda' +app_name = 'det' +sd_sparsity = 0.6 + +settings = { + 'involve_fm': True +} + +scenario = build_scenario( + source_datasets_name=['GTA5Det', 'SuperviselyPersonDet'], + target_datasets_order=['CityscapesDet', 'BaiduPersonDet'] * 15, + da_mode='close_set', + data_dirs={ + 'GTA5Det': '/data/zql/datasets/GTA-ls-copy/GTA5', + 'SuperviselyPersonDet': '/data/zql/datasets/supervisely_person_full_20230635/Supervisely Person Dataset', + 'CityscapesDet': '/data/zql/datasets/cityscape/', + 'BaiduPersonDet': '/data/zql/datasets/baidu_person/clean_images/' + }, +) + +from experiments.elasticdnn.vit_b_16.online_new.det.model import ElasticDNN_DetOnlineModel +elasticfm_model = ElasticDNN_DetOnlineModel('cls', init_online_model( + 'experiments/elasticdnn/vit_b_16/offline/fm_to_md/det/results/det_md_w_fbs_index.py/20230703/999994-214617-trial_in_card1/models/fm_best.pt', + 'experiments/elasticdnn/vit_b_16/offline/fm_to_md/det/results/det_md_w_fbs_index.py/20230703/999994-214617-trial_in_card1/models/md_best.pt', + 'det', __file__ +), device, { + 'md_to_fm_alpha': 1.0, + 'fm_to_md_alpha': 1.0 +}, scenario.num_classes) + +da_alg = FeatAlignAlg +from experiments.elasticdnn.vit_b_16.online_new.det.model import DetOnlineFeatAlignModel +da_model = DetOnlineFeatAlignModel +da_alg_hyp = {'CityscapesDet': { + 'train_batch_size': 8, + 'val_batch_size': 32, + 'num_workers': 16, + 'optimizer': 'AdamW', + 'optimizer_args': {'lr': 1e-6, 'betas': [0.9, 0.999], 'weight_decay': 0.01}, + 'scheduler': '', + 'scheduler_args': {}, + 'num_iters': 100, + 'val_freq': 20, + 'sd_sparsity': 0.6, + 'feat_align_loss_weight': 0.3 +}, 'BaiduPersonDet': { + 'train_batch_size': 8, + 'val_batch_size': 32, + 'num_workers': 16, + 'optimizer': 'AdamW', + 'optimizer_args': {'lr': 1e-6, 'betas': [0.9, 0.999], 'weight_decay': 0.01}, + 'scheduler': '', + 'scheduler_args': {}, + 'num_iters': 100, + 'val_freq': 20, + 'sd_sparsity': 0.6, + 'feat_align_loss_weight': 0.3 +}} + + +elasticfm_da( + [app_name], + [scenario], + [elasticfm_model], + [da_alg], + [da_alg_hyp], + [da_model], + device, + settings, + __file__, + sys.argv[1] +) diff --git a/experiments/elasticdnn/vit_b_16/online_new/det/model.py b/experiments/elasticdnn/vit_b_16/online_new/det/model.py new file mode 100644 index 0000000000000000000000000000000000000000..5efe00c6fe94efc719f94971bd25da0d30d097c2 --- /dev/null +++ b/experiments/elasticdnn/vit_b_16/online_new/det/model.py @@ -0,0 +1,257 @@ +from typing import List +from data.dataloader import build_dataloader +# from methods.elasticdnn.api.online_model import ElasticDNN_OnlineModel +from methods.elasticdnn.api.online_model_v2 import ElasticDNN_OnlineModel + +import torch +import sys +from torch import nn +from methods.elasticdnn.api.model import ElasticDNN_OfflineSegFMModel, ElasticDNN_OfflineSegMDModel +from methods.elasticdnn.api.algs.md_pretraining_wo_fbs import ElasticDNN_MDPretrainingWoFBSAlg +from methods.elasticdnn.model.base import ElasticDNNUtil +from methods.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from methods.elasticdnn.pipeline.offline.fm_to_md.vit import FM_to_MD_ViT_Util +from methods.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from methods.elasticdnn.pipeline.offline.fm_lora.vit import FMLoRA_ViT_Util +from methods.elasticdnn.model.vit import ElasticViTUtil +from utils.common.file import ensure_dir +from utils.dl.common.model import LayerActivation, get_module, get_parameter +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +from utils.dl.common.loss import CrossEntropyLossSoft +import torch.nn.functional as F +from utils.dl.common.env import create_tbwriter +import os +from utils.common.log import logger +from utils.common.data_record import write_json +# from methods.shot.shot import OnlineShotModel +from methods.feat_align.main import OnlineFeatAlignModel, FeatAlignAlg +import tqdm +from methods.feat_align.mmd import mmd_rbf + + +class ElasticDNN_DetOnlineModel(ElasticDNN_OnlineModel): + def __init__(self, name: str, models_dict_path: str, device: str, ab_options: dict, num_classes: int): + super().__init__(name, models_dict_path, device, ab_options) + self.num_classes = num_classes + + def get_accuracy(self, test_loader, *args, **kwargs): + # print('DeeplabV3: start test acc') + _d = test_loader.dataset + from data import build_dataloader + if _d.__class__.__name__ == 'MergedDataset': + # print('\neval on merged datasets') + datasets = _d.datasets + test_loaders = [build_dataloader(d, test_loader.batch_size, test_loader.num_workers, False, None) for d in datasets] + accs = [self.get_accuracy(loader) for loader in test_loaders] + # print(accs) + return sum(accs) / len(accs) + + # print('dataset len', len(test_loader.dataset)) + + model = self.models_dict['main'] + device = self.device + model.eval() + + # print('# classes', model.num_classes) + + model = model.to(device) + from dnns.yolov3.coco_evaluator import COCOEvaluator + from utils.common.others import HiddenPrints + with torch.no_grad(): + with HiddenPrints(): + evaluator = COCOEvaluator( + dataloader=test_loader, + img_size=(224, 224), + confthre=0.01, + nmsthre=0.65, + num_classes=self.num_classes, + testdev=False + ) + res = evaluator.evaluate(model, False, False) + map50 = res[1] + # print('eval info', res[-1]) + return map50 + + def get_elastic_dnn_util(self) -> ElasticDNNUtil: + return ElasticViTUtil() + + def get_fm_matched_param_of_md_param(self, md_param_name): + # only between qkv.weight, norm.weight/bias + self_param_name = md_param_name + fm = self.models_dict['fm'] + if any([k in self_param_name for k in ['fbs', 'cls_token', 'pos_embed']]): + return None + + p = get_parameter(self.models_dict['md'], self_param_name) + if p.dim() == 0: + return None + elif p.dim() == 1 and 'norm' in self_param_name and 'weight' in self_param_name: + if self_param_name.startswith('norm'): + return None + return get_parameter(fm, self_param_name) + + # 1. xx.qkv.to_qkv.yy to xx.qkv.qkv.aa and xx.qkv.abs.zz + if 'qkv.weight' in self_param_name: + ss = self_param_name.split('.') + + fm_qkv_name = '.'.join(ss[0: -1]) + '.qkv' + fm_qkv = get_module(fm, fm_qkv_name) + + fm_abs_name = '.'.join(ss[0: -1]) + '.abs' + fm_abs = get_module(fm, fm_abs_name) + + # NOTE: unrecoverable operation! multiply LoRA parameters to allow it being updated in update_fm_param() + # TODO: if fm will be used for inference, _mul_lora_weight will not be applied! + if not hasattr(fm_abs, '_mul_lora_weight'): + logger.debug(f'set _mul_lora_weight in {fm_abs_name}') + setattr(fm_abs, '_mul_lora_weight', + nn.Parameter(torch.cat([(_abs[0].weight.T @ _abs[1].weight.T).T for _abs in fm_abs], dim=0))) + + return torch.cat([ + fm_qkv.weight.data, # task-agnositc params + fm_abs._mul_lora_weight.data # task-specific params (LoRA) + ], dim=0) + + # elif 'to_qkv.bias' in self_param_name: + # ss = self_param_name.split('.') + + # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv.bias' + # return get_parameter(fm, fm_qkv_name) + + elif 'mlp.fc1' in self_param_name and 'weight' in self_param_name: + fm_param_name = self_param_name.replace('.linear', '') + return get_parameter(fm, fm_param_name) + + elif 'mlp.fc2' in self_param_name and 'weight' in self_param_name: + fm_param_name = self_param_name + return get_parameter(fm, fm_param_name) + + else: + # return get_parameter(fm, self_param_name) + return None + + def update_fm_param(self, md_param_name, cal_new_fm_param_by_md_param): + if not 'qkv.weight' in md_param_name: + matched_fm_param_ref = self.get_fm_matched_param_of_md_param(md_param_name) + matched_fm_param_ref.copy_(cal_new_fm_param_by_md_param) + else: + new_fm_attn_weight, new_fm_lora_weight = torch.chunk(cal_new_fm_param_by_md_param, 2, 0) + + ss = md_param_name.split('.') + fm = self.models_dict['fm'] + + # update task-agnostic parameters + fm_qkv_name = '.'.join(ss[0: -1]) + '.qkv' + fm_qkv = get_module(fm, fm_qkv_name) + fm_qkv.weight.data.copy_(new_fm_attn_weight) + + # update task-specific parameters + fm_abs_name = '.'.join(ss[0: -1]) + '.abs' + fm_abs = get_module(fm, fm_abs_name) + fm_abs._mul_lora_weight.data.copy_(new_fm_lora_weight) # TODO: this will not be applied in inference! + + def get_md_matched_param_of_fm_param(self, fm_param_name): + return super().get_md_matched_param_of_fm_param(fm_param_name) + + def get_md_matched_param_of_sd_param(self, sd_param_name): + # only between qkv.weight, norm.weight/bias + self_param_name = sd_param_name + md = self.models_dict['md'] + if any([k in self_param_name for k in ['fbs', 'cls_token', 'pos_embed']]): + return None + + p = get_parameter(self.models_dict['sd'], self_param_name) + if p.dim() == 0: + return None + elif p.dim() == 1 and 'norm' in self_param_name and 'weight' in self_param_name: + return get_parameter(md, self_param_name) + + # 1. xx.qkv.to_qkv.yy to xx.qkv.qkv.aa and xx.qkv.abs.zz + if 'qkv.weight' in self_param_name: + return get_parameter(md, self_param_name) + + # elif 'to_qkv.bias' in self_param_name: + # ss = self_param_name.split('.') + + # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv.bias' + # return get_parameter(fm, fm_qkv_name) + + elif 'mlp.fc1.0.weight' in self_param_name: + fm_param_name = '.'.join(self_param_name.split('.')[0: -2]) + '.linear.weight' + return get_parameter(md, fm_param_name) + + elif 'mlp.fc2' in self_param_name and 'weight' in self_param_name: + fm_param_name = self_param_name + return get_parameter(md, fm_param_name) + + else: + # return get_parameter(fm, self_param_name) + return None + + def get_task_head_params(self): + head = get_module(self.models_dict['sd'], 'head') + return list(head.parameters()) + + +class DetOnlineFeatAlignModel(OnlineFeatAlignModel): + def __init__(self, name: str, models_dict_path: str, device: str, num_classes): + super().__init__(name, models_dict_path, device) + self.num_classes = num_classes + + def get_feature_hook(self): + return LayerActivation(get_module(self.models_dict['main'], 'blocks.11.drop_path2'), False, self.device) + + def forward_to_get_task_loss(self, x, y): + self.to_train_mode() + return self.models_dict['main'](x, y)['total_loss'] + + def get_mmd_loss(self, f1, f2): + return mmd_rbf(f1.flatten(1), f2.flatten(1)) + + def infer(self, x, *args, **kwargs): + if len(args) > 0: + return self.models_dict['main'](x, *args) # forward(x, label) + return self.models_dict['main'](x) + + def get_trained_params(self): + qkv_and_norm_params = [p for n, p in self.models_dict['main'].named_parameters() if 'qkv.weight' in n or 'norm' in n or 'mlp' in n or 'head' in n] + return qkv_and_norm_params + + def get_accuracy(self, test_loader, *args, **kwargs): + # print('DeeplabV3: start test acc') + _d = test_loader.dataset + from data import build_dataloader + if _d.__class__.__name__ == 'MergedDataset': + print('\neval on merged datasets') + datasets = _d.datasets + test_loaders = [build_dataloader(d, test_loader.batch_size, test_loader.num_workers, False, None) for d in datasets] + accs = [self.get_accuracy(loader) for loader in test_loaders] + # print(accs) + return sum(accs) / len(accs) + + # print('dataset len', len(test_loader.dataset)) + + model = self.models_dict['main'] + device = self.device + model.eval() + + # print('# classes', model.num_classes) + + model = model.to(device) + from dnns.yolov3.coco_evaluator import COCOEvaluator + from utils.common.others import HiddenPrints + with torch.no_grad(): + with HiddenPrints(): + evaluator = COCOEvaluator( + dataloader=test_loader, + img_size=(224, 224), + confthre=0.01, + nmsthre=0.65, + num_classes=self.num_classes, + testdev=False + ) + res = evaluator.evaluate(model, False, False) + map50 = res[1] + # print('eval info', res[-1]) + return map50 \ No newline at end of file diff --git a/experiments/elasticdnn/vit_b_16/online_new/seg/model.py b/experiments/elasticdnn/vit_b_16/online_new/seg/model.py new file mode 100644 index 0000000000000000000000000000000000000000..18cc80b3284cb4198f91cdaecb885589c5137fb4 --- /dev/null +++ b/experiments/elasticdnn/vit_b_16/online_new/seg/model.py @@ -0,0 +1,225 @@ +from typing import List +from data.dataloader import build_dataloader +# from methods.elasticdnn.api.online_model import ElasticDNN_OnlineModel +from methods.elasticdnn.api.online_model_v2 import ElasticDNN_OnlineModel + +import torch +import sys +from torch import nn +from methods.elasticdnn.api.model import ElasticDNN_OfflineSegFMModel, ElasticDNN_OfflineSegMDModel +from methods.elasticdnn.api.algs.md_pretraining_wo_fbs import ElasticDNN_MDPretrainingWoFBSAlg +from methods.elasticdnn.model.base import ElasticDNNUtil +from methods.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from methods.elasticdnn.pipeline.offline.fm_to_md.vit import FM_to_MD_ViT_Util +from methods.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from methods.elasticdnn.pipeline.offline.fm_lora.vit import FMLoRA_ViT_Util +from methods.elasticdnn.model.vit import ElasticViTUtil +from utils.common.file import ensure_dir +from utils.dl.common.model import LayerActivation, get_module, get_parameter +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +from utils.dl.common.loss import CrossEntropyLossSoft +import torch.nn.functional as F +from utils.dl.common.env import create_tbwriter +import os +from utils.common.log import logger +from utils.common.data_record import write_json +# from methods.shot.shot import OnlineShotModel +from methods.feat_align.main import OnlineFeatAlignModel, FeatAlignAlg +import tqdm +from methods.feat_align.mmd import mmd_rbf + + +class ElasticDNN_SegOnlineModel(ElasticDNN_OnlineModel): + def __init__(self, name: str, models_dict_path: str, device: str, ab_options: dict, num_classes: int): + super().__init__(name, models_dict_path, device, ab_options) + self.num_classes = num_classes + + def get_accuracy(self, test_loader, *args, **kwargs): + device = self.device + self.to_eval_mode() + from methods.elasticdnn.api.model import StreamSegMetrics + metrics = StreamSegMetrics(self.num_classes) + metrics.reset() + import tqdm + pbar = tqdm.tqdm(enumerate(test_loader), total=len(test_loader), leave=False, dynamic_ncols=True) + with torch.no_grad(): + for batch_index, (x, y) in pbar: + x, y = x.to(device, dtype=x.dtype, non_blocking=True, copy=False), \ + y.to(device, dtype=y.dtype, non_blocking=True, copy=False) + output = self.infer(x) + pred = output.detach().max(dim=1)[1].cpu().numpy() + metrics.update((y + 0).cpu().numpy(), pred) + + res = metrics.get_results() + pbar.set_description(f'cur batch mIoU: {res["Mean Acc"]:.4f}') + + res = metrics.get_results() + return res['Mean Acc'] + + def get_elastic_dnn_util(self) -> ElasticDNNUtil: + return ElasticViTUtil() + + def get_fm_matched_param_of_md_param(self, md_param_name): + # only between qkv.weight, norm.weight/bias + self_param_name = md_param_name + fm = self.models_dict['fm'] + if any([k in self_param_name for k in ['fbs', 'cls_token', 'pos_embed']]): + return None + + p = get_parameter(self.models_dict['md'], self_param_name) + if p.dim() == 0: + return None + elif p.dim() == 1 and 'norm' in self_param_name and 'weight' in self_param_name: + if self_param_name.startswith('norm'): + return None + return get_parameter(fm, self_param_name) + + # 1. xx.qkv.to_qkv.yy to xx.qkv.qkv.aa and xx.qkv.abs.zz + if 'qkv.weight' in self_param_name: + ss = self_param_name.split('.') + + fm_qkv_name = '.'.join(ss[0: -1]) + '.qkv' + fm_qkv = get_module(fm, fm_qkv_name) + + fm_abs_name = '.'.join(ss[0: -1]) + '.abs' + fm_abs = get_module(fm, fm_abs_name) + + # NOTE: unrecoverable operation! multiply LoRA parameters to allow it being updated in update_fm_param() + # TODO: if fm will be used for inference, _mul_lora_weight will not be applied! + if not hasattr(fm_abs, '_mul_lora_weight'): + logger.debug(f'set _mul_lora_weight in {fm_abs_name}') + setattr(fm_abs, '_mul_lora_weight', + nn.Parameter(torch.cat([(_abs[0].weight.T @ _abs[1].weight.T).T for _abs in fm_abs], dim=0))) + + return torch.cat([ + fm_qkv.weight.data, # task-agnositc params + fm_abs._mul_lora_weight.data # task-specific params (LoRA) + ], dim=0) + + # elif 'to_qkv.bias' in self_param_name: + # ss = self_param_name.split('.') + + # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv.bias' + # return get_parameter(fm, fm_qkv_name) + + elif 'mlp.fc1' in self_param_name and 'weight' in self_param_name: + fm_param_name = self_param_name.replace('.linear', '') + return get_parameter(fm, fm_param_name) + + elif 'mlp.fc2' in self_param_name and 'weight' in self_param_name: + fm_param_name = self_param_name + return get_parameter(fm, fm_param_name) + + else: + # return get_parameter(fm, self_param_name) + return None + + def update_fm_param(self, md_param_name, cal_new_fm_param_by_md_param): + if not 'qkv.weight' in md_param_name: + matched_fm_param_ref = self.get_fm_matched_param_of_md_param(md_param_name) + matched_fm_param_ref.copy_(cal_new_fm_param_by_md_param) + else: + new_fm_attn_weight, new_fm_lora_weight = torch.chunk(cal_new_fm_param_by_md_param, 2, 0) + + ss = md_param_name.split('.') + fm = self.models_dict['fm'] + + # update task-agnostic parameters + fm_qkv_name = '.'.join(ss[0: -1]) + '.qkv' + fm_qkv = get_module(fm, fm_qkv_name) + fm_qkv.weight.data.copy_(new_fm_attn_weight) + + # update task-specific parameters + fm_abs_name = '.'.join(ss[0: -1]) + '.abs' + fm_abs = get_module(fm, fm_abs_name) + fm_abs._mul_lora_weight.data.copy_(new_fm_lora_weight) # TODO: this will not be applied in inference! + + def get_md_matched_param_of_fm_param(self, fm_param_name): + return super().get_md_matched_param_of_fm_param(fm_param_name) + + def get_md_matched_param_of_sd_param(self, sd_param_name): + # only between qkv.weight, norm.weight/bias + self_param_name = sd_param_name + md = self.models_dict['md'] + if any([k in self_param_name for k in ['fbs', 'cls_token', 'pos_embed']]): + return None + + p = get_parameter(self.models_dict['sd'], self_param_name) + if p.dim() == 0: + return None + elif p.dim() == 1 and 'norm' in self_param_name and 'weight' in self_param_name: + return get_parameter(md, self_param_name) + + # 1. xx.qkv.to_qkv.yy to xx.qkv.qkv.aa and xx.qkv.abs.zz + if 'qkv.weight' in self_param_name: + return get_parameter(md, self_param_name) + + # elif 'to_qkv.bias' in self_param_name: + # ss = self_param_name.split('.') + + # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv.bias' + # return get_parameter(fm, fm_qkv_name) + + elif 'mlp.fc1.0.weight' in self_param_name: + fm_param_name = '.'.join(self_param_name.split('.')[0: -2]) + '.linear.weight' + return get_parameter(md, fm_param_name) + + elif 'mlp.fc2' in self_param_name and 'weight' in self_param_name: + fm_param_name = self_param_name + return get_parameter(md, fm_param_name) + + else: + # return get_parameter(fm, self_param_name) + return None + + def get_task_head_params(self): + head = get_module(self.models_dict['sd'], 'head') + return list(head.parameters()) + + +class SegOnlineFeatAlignModel(OnlineFeatAlignModel): + def __init__(self, name: str, models_dict_path: str, device: str, num_classes): + super().__init__(name, models_dict_path, device) + self.num_classes = num_classes + + def get_feature_hook(self): + return LayerActivation(get_module(self.models_dict['main'], 'norm.1.head'), False, self.device) + + def forward_to_get_task_loss(self, x, y): + return F.cross_entropy(self.infer(x), y) + + def get_mmd_loss(self, f1, f2): + return mmd_rbf(f1.flatten(1), f2.flatten(1)) + + def infer(self, x, *args, **kwargs): + return self.models_dict['main'](x) + + def get_trained_params(self): + qkv_and_norm_params = [p for n, p in self.models_dict['main'].named_parameters() if 'qkv.weight' in n or 'norm' in n or 'mlp' in n] + return qkv_and_norm_params + + def infer(self, x, *args, **kwargs): + return self.models_dict['main'](x) + + def get_accuracy(self, test_loader, *args, **kwargs): + device = self.device + self.to_eval_mode() + from methods.elasticdnn.api.model import StreamSegMetrics + metrics = StreamSegMetrics(self.num_classes) + metrics.reset() + import tqdm + pbar = tqdm.tqdm(enumerate(test_loader), total=len(test_loader), leave=False, dynamic_ncols=True) + with torch.no_grad(): + for batch_index, (x, y) in pbar: + x, y = x.to(device, dtype=x.dtype, non_blocking=True, copy=False), \ + y.to(device, dtype=y.dtype, non_blocking=True, copy=False) + output = self.infer(x) + pred = output.detach().max(dim=1)[1].cpu().numpy() + metrics.update((y + 0).cpu().numpy(), pred) + + res = metrics.get_results() + pbar.set_description(f'cur batch mIoU: {res["Mean Acc"]:.4f}') + + res = metrics.get_results() + return res['Mean Acc'] \ No newline at end of file diff --git a/experiments/elasticdnn/vit_b_16/online_new/seg/seg.py b/experiments/elasticdnn/vit_b_16/online_new/seg/seg.py new file mode 100644 index 0000000000000000000000000000000000000000..7cf8197259678e0b6c80df3f88d56dc23fc657ef --- /dev/null +++ b/experiments/elasticdnn/vit_b_16/online_new/seg/seg.py @@ -0,0 +1,106 @@ +from typing import List +from data.dataloader import build_dataloader +# from methods.elasticdnn.api.online_model import ElasticDNN_OnlineModel +from methods.elasticdnn.api.online_model_v2 import ElasticDNN_OnlineModel + +import torch +import sys +from torch import nn +from methods.elasticdnn.api.model import ElasticDNN_OfflineSegFMModel, ElasticDNN_OfflineSegMDModel +from methods.elasticdnn.api.algs.md_pretraining_wo_fbs import ElasticDNN_MDPretrainingWoFBSAlg +from methods.elasticdnn.model.base import ElasticDNNUtil +from methods.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from methods.elasticdnn.pipeline.offline.fm_to_md.vit import FM_to_MD_ViT_Util +from methods.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from methods.elasticdnn.pipeline.offline.fm_lora.vit import FMLoRA_ViT_Util +from methods.elasticdnn.model.vit import ElasticViTUtil +from utils.common.file import ensure_dir +from utils.dl.common.model import LayerActivation, get_module, get_parameter +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +from utils.dl.common.loss import CrossEntropyLossSoft +import torch.nn.functional as F +from utils.dl.common.env import create_tbwriter +import os +from utils.common.log import logger +from utils.common.data_record import write_json +# from methods.shot.shot import OnlineShotModel +from methods.feat_align.main import OnlineFeatAlignModel, FeatAlignAlg +import tqdm +from methods.feat_align.mmd import mmd_rbf +from experiments.utils.elasticfm_da import init_online_model, elasticfm_da + +device = 'cuda' +app_name = 'seg' +sd_sparsity = 0.8 + +settings = { + 'involve_fm': True +} + +scenario = build_scenario( + source_datasets_name=['GTA5', 'SuperviselyPerson'], + target_datasets_order=['Cityscapes', 'BaiduPerson'] * 15, + da_mode='close_set', + data_dirs={ + 'GTA5': '/data/zql/datasets/GTA-ls-copy/GTA5', + 'SuperviselyPerson': '/data/zql/datasets/supervisely_person/Supervisely Person Dataset', + 'Cityscapes': '/data/zql/datasets/cityscape/', + 'BaiduPerson': '/data/zql/datasets/baidu_person/clean_images/' + }, +) + +from experiments.elasticdnn.vit_b_16.online_new.seg.model import ElasticDNN_SegOnlineModel +elasticfm_model = ElasticDNN_SegOnlineModel('cls', init_online_model( + # 'experiments/elasticdnn/vit_b_16/offline/fm_to_md/results/cls_md_index.py/20230529/star_999997-154037-only_prune_mlp/models/fm_best.pt', + # 'experiments/elasticdnn/vit_b_16/offline/fm_to_md/results/cls_md_index.py/20230529/star_999997-154037-only_prune_mlp/models/md_best.pt', + 'experiments/elasticdnn/vit_b_16/offline/fm_to_md/seg/results/seg_md_index.py/20230613/999995-093643-new_md_structure/models/fm_best.pt', + 'experiments/elasticdnn/vit_b_16/offline/fm_to_md/seg/results/seg_md_index.py/20230613/999995-093643-new_md_structure/models/md_best.pt', + 'seg', __file__ +), device, { + 'md_to_fm_alpha': 0.1, + 'fm_to_md_alpha': 0.1 +}, scenario.num_classes) + +da_alg = FeatAlignAlg +from experiments.elasticdnn.vit_b_16.online_new.seg.model import SegOnlineFeatAlignModel +da_model = SegOnlineFeatAlignModel +da_alg_hyp = {'Cityscapes': { + 'train_batch_size': 16, + 'val_batch_size': 128, + 'num_workers': 16, + 'optimizer': 'AdamW', + 'optimizer_args': {'lr': 3e-6, 'betas': [0.9, 0.999], 'weight_decay': 0.01}, + 'scheduler': '', + 'scheduler_args': {}, + 'num_iters': 100, + 'val_freq': 20, + 'sd_sparsity': 0.8, + 'feat_align_loss_weight': 0.3 +}, 'BaiduPerson': { + 'train_batch_size': 16, + 'val_batch_size': 128, + 'num_workers': 16, + 'optimizer': 'AdamW', + 'optimizer_args': {'lr': 1e-7, 'betas': [0.9, 0.999], 'weight_decay': 0.01}, + 'scheduler': '', + 'scheduler_args': {}, + 'num_iters': 100, + 'val_freq': 20, + 'sd_sparsity': 0.8, + 'feat_align_loss_weight': 0.3 +}} + + +elasticfm_da( + [app_name], + [scenario], + [elasticfm_model], + [da_alg], + [da_alg_hyp], + [da_model], + device, + settings, + __file__, + sys.argv[1] +) diff --git a/experiments/utils/__pycache__/elasticfm_cl.cpython-38.pyc b/experiments/utils/__pycache__/elasticfm_cl.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..08704f54e60d081df80a688a16c48c738be992b6 Binary files /dev/null and b/experiments/utils/__pycache__/elasticfm_cl.cpython-38.pyc differ diff --git a/experiments/utils/baseline_cl.py b/experiments/utils/baseline_cl.py new file mode 100644 index 0000000000000000000000000000000000000000..8a0d567099d9e596a149aff8b97eaabcd2b37c3e --- /dev/null +++ b/experiments/utils/baseline_cl.py @@ -0,0 +1,97 @@ +import sys +from utils.dl.common.env import set_random_seed +set_random_seed(1) + +from typing import List +from data.dataloader import build_dataloader +from data import CLScenario, build_cl_scenario +from methods.elasticdnn.api.online_model_v2 import ElasticDNN_OnlineModel + +import torch +import sys +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from utils.dl.common.env import create_tbwriter +import os +from utils.common.log import logger +from utils.common.data_record import write_json +# from methods.shot.shot import OnlineShotModel +from methods.base.alg import BaseAlg +from methods.base.model import BaseModel + + +def baseline_cl(app_name: str, + scenario: CLScenario, + cl_alg: BaseAlg, + cl_alg_hyp: dict, + cl_model: BaseModel, + device, + __entry_file__, + tag=None): + + # involve_fm = settings['involve_fm'] + + # task_name = app_name + # online_model = elasticfm_model + + log_dir = get_res_save_dir(__entry_file__, tag=tag) + tb_writer = create_tbwriter(os.path.join(log_dir, 'tb_log'), False) + res = [] + global_avg_after_acc = 0. + global_iter = 0 + + for task_index, _ in enumerate(scenario.target_tasks_order): + + cur_target_task_name = scenario.target_tasks_order[scenario.cur_task_index] + # if cur_target_domain_name in da_alg_hyp: + # da_alg_hyp = da_alg_hyp[cur_target_domain_name] + # logger.info(f'use dataset-specific hyps') + + # tmp_sd_path = os.path.join(log_dir, 'tmp_sd_model.pt') + # torch.save({'main': sd}, tmp_sd_path) + + # if task_name != 'cls': + # da_model_args = [f'{task_name}/{domain_index}', + # tmp_sd_path, + # device, + # scenario.num_classes] + # else: + # da_model_args = [f'{task_name}/{domain_index}', + # tmp_sd_path, + # device] + # cur_da_model = da_model(*da_model_args) + + cl_metrics, after_cl_model = cl_alg( + {'main': cl_model}, + os.path.join(log_dir, f'{app_name}/{task_index}') + ).run(scenario, cl_alg_hyp) + # os.remove(tmp_sd_path) + + if task_index > 0: + import shutil + shutil.rmtree(os.path.join(log_dir, f'{app_name}/{task_index}/backup_codes')) + + accs = cl_metrics['accs'] + before_acc = accs[0]['acc'] + after_acc = accs[-1]['acc'] + + tb_writer.add_scalars(f'accs/{app_name}', dict(before=before_acc, after=after_acc), task_index) + tb_writer.add_scalar(f'times/{app_name}', cl_metrics['time'], task_index) + + for _acc in accs: + tb_writer.add_scalar('total_acc', _acc['acc'], _acc['iter'] + global_iter) + global_iter += _acc['iter'] + 1 + + scenario.next_task() + + logger.info(f"app: {app_name}, task {task_index}, acc: {before_acc:.4f} -> " + f"{after_acc:.4f} ({cl_metrics['time']:.2f}s)") + + global_avg_after_acc += after_acc + cur_res = cl_metrics + res += [cur_res] + write_json(os.path.join(log_dir, 'res.json'), res, backup=False) + + global_avg_after_acc /= (task_index + 1) + logger.info(f'-----> final metric: {global_avg_after_acc:.4f}') + write_json(os.path.join(log_dir, f'res_{global_avg_after_acc:.4f}.json'), res, backup=False) + \ No newline at end of file diff --git a/experiments/utils/baseline_da.py b/experiments/utils/baseline_da.py new file mode 100644 index 0000000000000000000000000000000000000000..b463cdac990177fd9236a90a2c9d53cd7670b19f --- /dev/null +++ b/experiments/utils/baseline_da.py @@ -0,0 +1,114 @@ +import sys +from utils.dl.common.env import set_random_seed +set_random_seed(1) + +from typing import List +from data.dataloader import build_dataloader +from data import Scenario +from methods.elasticdnn.api.online_model_v2 import ElasticDNN_OnlineModel + +import torch +import sys +from torch import nn +from methods.elasticdnn.api.model import ElasticDNN_OfflineSegFMModel, ElasticDNN_OfflineSegMDModel +from methods.elasticdnn.api.algs.md_pretraining_wo_fbs import ElasticDNN_MDPretrainingWoFBSAlg +from methods.elasticdnn.model.base import ElasticDNNUtil +from methods.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from methods.elasticdnn.pipeline.offline.fm_to_md.vit import FM_to_MD_ViT_Util +from methods.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from methods.elasticdnn.pipeline.offline.fm_lora.vit import FMLoRA_ViT_Util +from methods.elasticdnn.model.vit import ElasticViTUtil +from utils.common.file import ensure_dir +from utils.dl.common.model import LayerActivation, get_module, get_parameter +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +from utils.dl.common.loss import CrossEntropyLossSoft +import torch.nn.functional as F +from utils.dl.common.env import create_tbwriter +import os +import shutil +from utils.common.log import logger +from utils.common.data_record import write_json +# from methods.shot.shot import OnlineShotModel +from methods.feat_align.main import OnlineFeatAlignModel, FeatAlignAlg +import tqdm +from methods.feat_align.mmd import mmd_rbf +from methods.base.alg import BaseAlg +from methods.base.model import BaseModel + + +def baseline_da(app_name: str, + scenario: Scenario, + da_alg: BaseAlg, + da_alg_hyp: dict, + da_model: BaseModel, + device, + __entry_file__, + tag=None): + + # involve_fm = settings['involve_fm'] + + task_name = app_name + # online_model = elasticfm_model + + log_dir = get_res_save_dir(__entry_file__, tag=tag) + tb_writer = create_tbwriter(os.path.join(log_dir, 'tb_log'), False) + res = [] + global_avg_after_acc = 0. + global_iter = 0 + + for domain_index, _ in enumerate(scenario.target_domains_order): + + cur_target_domain_name = scenario.target_domains_order[scenario.cur_domain_index] + if cur_target_domain_name in da_alg_hyp: + da_alg_hyp = da_alg_hyp[cur_target_domain_name] + logger.info(f'use dataset-specific hyps') + + # tmp_sd_path = os.path.join(log_dir, 'tmp_sd_model.pt') + # torch.save({'main': sd}, tmp_sd_path) + + # if task_name != 'cls': + # da_model_args = [f'{task_name}/{domain_index}', + # tmp_sd_path, + # device, + # scenario.num_classes] + # else: + # da_model_args = [f'{task_name}/{domain_index}', + # tmp_sd_path, + # device] + # cur_da_model = da_model(*da_model_args) + + da_metrics, after_da_model = da_alg( + {'main': da_model}, + os.path.join(log_dir, f'{task_name}/{domain_index}') + ).run(scenario, da_alg_hyp) + # os.remove(tmp_sd_path) + + if domain_index > 0: + shutil.rmtree(os.path.join(log_dir, f'{task_name}/{domain_index}/backup_codes')) + + accs = da_metrics['accs'] + before_acc = accs[0]['acc'] + after_acc = accs[-1]['acc'] + + tb_writer.add_scalars(f'accs/{task_name}', dict(before=before_acc, after=after_acc), domain_index) + tb_writer.add_scalar(f'times/{task_name}', da_metrics['time'], domain_index) + + for _acc in accs: + tb_writer.add_scalar('total_acc', _acc['acc'], _acc['iter'] + global_iter) + global_iter += _acc['iter'] + 1 + + scenario.next_domain() + + logger.info(f"task: {task_name}, domain {domain_index}, acc: {before_acc:.4f} -> " + f"{after_acc:.4f} ({da_metrics['time']:.2f}s)") + + global_avg_after_acc += after_acc + cur_res = da_metrics + res += [cur_res] + write_json(os.path.join(log_dir, 'res.json'), res, backup=False) + + global_avg_after_acc /= (domain_index + 1) + logger.info(f'-----> final metric: {global_avg_after_acc:.4f}') + write_json(os.path.join(log_dir, f'res_{global_avg_after_acc:.4f}.json'), res, backup=False) + \ No newline at end of file diff --git a/experiments/utils/baseline_da_train_compress.py b/experiments/utils/baseline_da_train_compress.py new file mode 100644 index 0000000000000000000000000000000000000000..28b2092dc54d1e16329e2cec7cf9a113fa082deb --- /dev/null +++ b/experiments/utils/baseline_da_train_compress.py @@ -0,0 +1,198 @@ +import sys +from utils.dl.common.env import set_random_seed +set_random_seed(1) + +from typing import List +from data.dataloader import build_dataloader +from data import Scenario +from methods.elasticdnn.api.online_model_v2 import ElasticDNN_OnlineModel + +import torch +import sys +from torch import nn +from methods.elasticdnn.api.model import ElasticDNN_OfflineSegFMModel, ElasticDNN_OfflineSegMDModel +from methods.elasticdnn.api.algs.md_pretraining_wo_fbs import ElasticDNN_MDPretrainingWoFBSAlg +from methods.elasticdnn.model.base import ElasticDNNUtil +from methods.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from methods.elasticdnn.pipeline.offline.fm_to_md.vit import FM_to_MD_ViT_Util +from methods.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from methods.elasticdnn.pipeline.offline.fm_lora.vit import FMLoRA_ViT_Util +from methods.elasticdnn.model.vit import ElasticViTUtil +from utils.common.file import ensure_dir +from utils.dl.common.model import LayerActivation, get_module, get_parameter +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +from utils.dl.common.loss import CrossEntropyLossSoft +import torch.nn.functional as F +from utils.dl.common.env import create_tbwriter +import os +import shutil +from utils.common.log import logger +from utils.common.data_record import write_json +# from methods.shot.shot import OnlineShotModel +from methods.feat_align.main import OnlineFeatAlignModel, FeatAlignAlg +import tqdm +from methods.feat_align.mmd import mmd_rbf +from methods.base.alg import BaseAlg +from methods.base.model import BaseModel +from copy import deepcopy +import time + + +def baseline_da(app_name: str, + scenario: Scenario, + da_alg: BaseAlg, + da_alg_hyp: dict, + da_model: BaseModel, + device, + __entry_file__, + tag=None): + + # involve_fm = settings['involve_fm'] + + task_name = app_name + # online_model = elasticfm_model + + log_dir = get_res_save_dir(__entry_file__, tag=tag) + tb_writer = create_tbwriter(os.path.join(log_dir, 'tb_log'), False) + res = [] + global_avg_after_acc = 0. + global_iter = 0 + + for domain_index, _ in enumerate(scenario.target_domains_order): + + cur_target_domain_name = scenario.target_domains_order[scenario.cur_domain_index] + if cur_target_domain_name in da_alg_hyp: + da_alg_hyp = da_alg_hyp[cur_target_domain_name] + logger.info(f'use dataset-specific da_alg_hyp') + + da_metrics, after_da_model = da_alg( + {'main': da_model}, + os.path.join(log_dir, f'{task_name}/{domain_index}') + ).run(scenario, da_alg_hyp) + # os.remove(tmp_sd_path) + + # 前面在当前域上训练，在这里压缩调优？ + # print(da_model.models_dict['main']) + # 进行压缩 + reducing_width_ratio = 8 + samples = torch.rand(1, 3, 224, 224).to(device) + + trained_fm_model = deepcopy(da_model.models_dict['main']) + fm_da_model = deepcopy(da_model) # 保存大模型 + lora_util = FMLoRA_ViT_Util() + lora_absorbed_fm_model = lora_util.absorb_lora_and_recover_net_structure(trained_fm_model, samples) + compressed_fm_model = FM_to_MD_ViT_Util().init_md_from_fm_by_reducing_width_with_perf_test(lora_absorbed_fm_model, reducing_width_ratio, samples) + da_model.models_dict['main'] = compressed_fm_model + + # 进行调优？之前那个da_metrics是FM的结果吧，调优也能得到一个精度结果换成这个？ + + datasets_for_training = scenario.get_online_cur_domain_datasets_for_training() + train_dataset = datasets_for_training[cur_target_domain_name]['train'] + val_dataset = datasets_for_training[cur_target_domain_name]['val'] + datasets_for_inference = scenario.get_online_cur_domain_datasets_for_inference() + test_dataset = datasets_for_inference + + train_loader = iter(build_dataloader(train_dataset, da_alg_hyp['train_batch_size'], da_alg_hyp['num_workers'], True, None)) + test_loader = build_dataloader(test_dataset, da_alg_hyp['val_batch_size'], da_alg_hyp['num_workers'], False, False) + + for p in compressed_fm_model.parameters(): + p.requires_grad = True + da_model.to_train_mode() + + # 'distill_optimizer': 'AdamW', + # 'distill_optimizer_args': {'lr': 1e-4, 'betas': [0.9, 0.999], 'weight_decay': 0.01}, + optimizer = torch.optim.__dict__['AdamW']([ + {'params': da_model.models_dict['main'].parameters(), **{'lr': 1e-4, 'betas': [0.9, 0.999], 'weight_decay': 0.01}} + ]) + if da_alg_hyp['scheduler'] != '': + scheduler = torch.optim.lr_scheduler.__dict__[da_alg_hyp['scheduler']](optimizer, **da_alg_hyp['scheduler_args']) + else: + scheduler = None + + pbar = tqdm.tqdm(range(da_alg_hyp['num_iters']), dynamic_ncols=True) + + accs = [] + total_train_time = 0. + cur_acc = 0. + for iter_index in pbar: + cur_start_time = time.time() + da_model.to_train_mode() + fm_da_model.to_eval_mode() + + x, y = next(train_loader) + if isinstance(x, dict): + for k, v in x.items(): + if isinstance(v, torch.Tensor): + x[k] = v.to(device) + y = y.to(device) + else: + x, y = x.to(device), y.to(device) + + with torch.no_grad(): + fm_output = fm_da_model.infer(x) + md_output = da_model.infer(x) + + distill_criterion = CrossEntropyLossSoft() + total_loss = distill_criterion(md_output, fm_output) + + optimizer.zero_grad() + total_loss.backward() + optimizer.step() + if scheduler is not None: + scheduler.step() + total_train_time += time.time() - cur_start_time + + if (iter_index + 1) % da_alg_hyp['val_freq'] == 0: + from data import split_dataset + cur_md = da_model.models_dict['main'] + md_for_test = deepcopy(da_model.models_dict['main']) + da_model.models_dict['main'] = md_for_test + cur_test_batch_dataset = split_dataset(test_dataset, da_alg_hyp['val_batch_size'], iter_index + 1)[0] + cur_test_batch_dataloader = build_dataloader(cur_test_batch_dataset, da_alg_hyp['train_batch_size'], da_alg_hyp['num_workers'], False, False) + da_model.to_eval_mode() + cur_acc = da_model.get_accuracy(cur_test_batch_dataloader) + accs += [{ + 'iter': iter_index + 1, + 'acc': cur_acc + }] + pbar.set_description(f'loss: {total_loss:.6f}, cur_acc: {cur_acc:.4f}') + + time_usage = total_train_time + da_metrics = { + 'accs': accs, + 'time': time_usage + } + da_model = fm_da_model # 恢复大模型 + + # 蒸馏结束 + + + if domain_index > 0: + shutil.rmtree(os.path.join(log_dir, f'{task_name}/{domain_index}/backup_codes')) + + accs = da_metrics['accs'] + before_acc = accs[0]['acc'] + after_acc = accs[-1]['acc'] + + tb_writer.add_scalars(f'accs/{task_name}', dict(before=before_acc, after=after_acc), domain_index) + tb_writer.add_scalar(f'times/{task_name}', da_metrics['time'], domain_index) + + for _acc in accs: + tb_writer.add_scalar('total_acc', _acc['acc'], _acc['iter'] + global_iter) + global_iter += _acc['iter'] + 1 + + scenario.next_domain() + + logger.info(f"task: {task_name}, domain {domain_index}, acc: {before_acc:.4f} -> " + f"{after_acc:.4f} ({da_metrics['time']:.2f}s)") + + global_avg_after_acc += after_acc + cur_res = da_metrics + res += [cur_res] + write_json(os.path.join(log_dir, 'res.json'), res, backup=False) + + global_avg_after_acc /= (domain_index + 1) + logger.info(f'-----> final metric: {global_avg_after_acc:.4f}') + write_json(os.path.join(log_dir, f'res_{global_avg_after_acc:.4f}.json'), res, backup=False) + \ No newline at end of file diff --git a/experiments/utils/elasticfm_cl.py b/experiments/utils/elasticfm_cl.py new file mode 100644 index 0000000000000000000000000000000000000000..45f833c23df8996da14ef484171f9f03e9726866 --- /dev/null +++ b/experiments/utils/elasticfm_cl.py @@ -0,0 +1,175 @@ +import sys +from utils.dl.common.env import set_random_seed +set_random_seed(1) + +from typing import List +from data.dataloader import build_dataloader +from data import Scenario, CLScenario +from methods.elasticdnn.api.online_model_v2 import ElasticDNN_OnlineModel + +import torch +import sys +from torch import nn +from methods.elasticdnn.api.model import ElasticDNN_OfflineSegFMModel, ElasticDNN_OfflineSegMDModel +from methods.elasticdnn.api.algs.md_pretraining_wo_fbs import ElasticDNN_MDPretrainingWoFBSAlg +from methods.elasticdnn.model.base import ElasticDNNUtil +from methods.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from methods.elasticdnn.pipeline.offline.fm_to_md.vit import FM_to_MD_ViT_Util +from methods.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from methods.elasticdnn.pipeline.offline.fm_lora.vit import FMLoRA_ViT_Util +from methods.elasticdnn.model.vit import ElasticViTUtil +from utils.common.file import ensure_dir +from utils.dl.common.model import LayerActivation, get_module, get_parameter +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +from utils.dl.common.loss import CrossEntropyLossSoft +import torch.nn.functional as F +from utils.dl.common.env import create_tbwriter +import os +from utils.common.log import logger +from utils.common.data_record import write_json +# from methods.shot.shot import OnlineShotModel +from methods.feat_align.main import OnlineFeatAlignModel, FeatAlignAlg +import tqdm +from methods.feat_align.mmd import mmd_rbf +from methods.base.alg import BaseAlg +from methods.base.model import BaseModel +import shutil + + +def elasticfm_cl(apps_name: List[str], + scenarios: List[CLScenario], + elasticfm_models: List[ElasticDNN_OnlineModel], + da_algs: List[BaseAlg], + da_alg_hyps: List[dict], + da_models: List[BaseModel], + device, + settings, + __entry_file__, + tag=None): + + assert len(apps_name) == 1 + + involve_fm = settings['involve_fm'] + + tasks_name = apps_name + online_models = elasticfm_models + + log_dir = get_res_save_dir(__entry_file__, tag=tag) + tb_writer = create_tbwriter(os.path.join(log_dir, 'tb_log'), False) + res = [] + global_avg_after_acc = 0. + global_iter = 0 + + last_da_model_attrs = {} + + for domain_index, _ in enumerate(scenarios[0].target_tasks_order): + avg_before_acc, avg_after_acc = 0., 0. + cur_res = {} + + for task_name, online_model, scenario, da_alg, da_model, da_alg_hyp in zip(tasks_name, online_models, scenarios, da_algs, da_models, da_alg_hyps): + + cur_target_domain_name = scenario.target_tasks_order[scenario.cur_task_index] + if cur_target_domain_name in da_alg_hyp: + da_alg_hyp = da_alg_hyp[cur_target_domain_name] + logger.info(f'use dataset-specific hyps') + + if domain_index == 0: # add last layer + logger.info(f'add num of classes in the last layer') + online_model.add_cls_in_head(scenario.num_tasks_to_be_learn) + + online_model.set_sd_sparsity(da_alg_hyp['sd_sparsity']) + sd, unpruned_indexes_of_layers = online_model.generate_sd_by_target_samples(scenario.get_online_cur_task_samples_for_training(da_alg_hyp['train_batch_size'])) + + tmp_sd_path = os.path.join(log_dir, 'tmp_sd_model.pt') + torch.save({'main': sd}, tmp_sd_path) + + if 'cls' not in task_name and 'pos' not in task_name and 'vqa' not in task_name: + da_model_args = [f'{task_name}/{domain_index}', + tmp_sd_path, + device, + scenario.num_classes] + else: + da_model_args = [f'{task_name}/{domain_index}', + tmp_sd_path, + device] + + da_model_ent = da_model(*da_model_args) + for k, v in last_da_model_attrs.items(): + logger.info(f'set attr of last model: {k}') + setattr(da_model_ent, k, v) + + da_metrics, after_da_model = da_alg( + {'main': da_model_ent}, + os.path.join(log_dir, f'{task_name}/{domain_index}') + ).run(scenario, {_k: _v for _k, _v in da_alg_hyp.items() if _k != 'sd_sparsity'}) + + last_da_model_attrs = {k: getattr(after_da_model['main'], k) for k in dir(after_da_model['main']) if str(k).startswith('_el_data_')} + logger.info(f'collect attrs in last model: {last_da_model_attrs.keys()}') + + os.remove(tmp_sd_path) + if domain_index > 0: + shutil.rmtree(os.path.join(log_dir, f'{task_name}/{domain_index}/backup_codes')) + + + online_model.last_trained_cls_indexes = torch.LongTensor([i for i in range(scenario.get_cur_class_offset(), + scenario.get_cur_class_offset() + scenario.get_cur_num_class())]) + + online_model.sd_feedback_to_md(after_da_model['main'].models_dict['main'], unpruned_indexes_of_layers) + online_model.md_feedback_to_self_fm() + + accs = da_metrics['accs'] + before_acc = accs[0]['acc'] + after_acc = accs[-1]['acc'] + + avg_before_acc += before_acc + avg_after_acc += after_acc + + for _acc in accs: + tb_writer.add_scalar(f'total_acc', _acc['acc'], _acc['iter'] + global_iter) # TODO: bug here + global_iter += _acc['iter'] + 1 + + tb_writer.add_scalars(f'accs/{task_name}', dict(before=before_acc, after=after_acc), domain_index) + tb_writer.add_scalar(f'times/{task_name}', da_metrics['time'], domain_index) + + scenario.next_task() + + logger.info(f"task: {task_name}, domain {domain_index}, acc: {before_acc:.4f} -> " + f"{after_acc:.4f} ({da_metrics['time']:.2f}s)") + cur_res[task_name] = da_metrics + + if involve_fm: + for online_model in online_models: + online_model.aggregate_fms_to_self_fm([m.models_dict['fm'] for m in online_models]) + for online_model in online_models: + online_model.fm_feedback_to_md() + + avg_before_acc /= len(tasks_name) + avg_after_acc /= len(tasks_name) + tb_writer.add_scalars(f'accs/apps_avg', dict(before=avg_before_acc, after=avg_after_acc), domain_index) + logger.info(f"--> domain {domain_index}, avg_acc: {avg_before_acc:.4f} -> " + f"{avg_after_acc:.4f}") + res += [cur_res] + + global_avg_after_acc += avg_after_acc + + write_json(os.path.join(log_dir, 'res.json'), res, backup=False) + + global_avg_after_acc /= (domain_index + 1) + logger.info(f'-----> final metric: {global_avg_after_acc:.4f}') + write_json(os.path.join(log_dir, f'res_{global_avg_after_acc:.4f}.json'), res, backup=False) + + + +def init_online_model(fm_models_dict_path, md_models_dict_path, task_name, __entry_file__): + fm_models = torch.load(fm_models_dict_path) + md_models = torch.load(md_models_dict_path) + + online_models_dict_path = save_models_dict_for_init({ + 'fm': fm_models['main'], + 'md': md_models['main'], + 'sd': None, + 'indexes': md_models['indexes'], + 'bn_stats': md_models['bn_stats'] + }, __entry_file__, task_name) + return online_models_dict_path diff --git a/experiments/utils/elasticfm_da.py b/experiments/utils/elasticfm_da.py new file mode 100644 index 0000000000000000000000000000000000000000..6e97675ecdcf63c2360e75651e484b66dd2b3230 --- /dev/null +++ b/experiments/utils/elasticfm_da.py @@ -0,0 +1,154 @@ +import sys +from utils.dl.common.env import set_random_seed +set_random_seed(1) + +from typing import List +from data.dataloader import build_dataloader +from data import Scenario +from methods.elasticdnn.api.online_model_v2 import ElasticDNN_OnlineModel + +import torch +import sys +from torch import nn +from methods.elasticdnn.api.model import ElasticDNN_OfflineSegFMModel, ElasticDNN_OfflineSegMDModel +from methods.elasticdnn.api.algs.md_pretraining_wo_fbs import ElasticDNN_MDPretrainingWoFBSAlg +from methods.elasticdnn.model.base import ElasticDNNUtil +from methods.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from methods.elasticdnn.pipeline.offline.fm_to_md.vit import FM_to_MD_ViT_Util +from methods.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from methods.elasticdnn.pipeline.offline.fm_lora.vit import FMLoRA_ViT_Util +from methods.elasticdnn.model.vit import ElasticViTUtil +from utils.common.file import ensure_dir +from utils.dl.common.model import LayerActivation, get_module, get_parameter +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +from utils.dl.common.loss import CrossEntropyLossSoft +import torch.nn.functional as F +from utils.dl.common.env import create_tbwriter +import os +from utils.common.log import logger +from utils.common.data_record import write_json +# from methods.shot.shot import OnlineShotModel +from methods.feat_align.main import OnlineFeatAlignModel, FeatAlignAlg +import tqdm +from methods.feat_align.mmd import mmd_rbf +from methods.base.alg import BaseAlg +import shutil +from methods.base.model import BaseModel + + +def elasticfm_da(apps_name: List[str], + scenarios: List[Scenario], + elasticfm_models: List[ElasticDNN_OnlineModel], + da_algs: List[BaseAlg], + da_alg_hyps: List[dict], + da_models: List[BaseModel], + device, + settings, + __entry_file__, + tag=None): + + involve_fm = settings['involve_fm'] + + tasks_name = apps_name + online_models = elasticfm_models + + log_dir = get_res_save_dir(__entry_file__, tag=tag) + tb_writer = create_tbwriter(os.path.join(log_dir, 'tb_log'), False) + res = [] + global_avg_after_acc = 0. + global_iter = 0 + + for domain_index, _ in enumerate(scenarios[0].target_domains_order): + avg_before_acc, avg_after_acc = 0., 0. + cur_res = {} + + for task_name, online_model, scenario, da_alg, da_model, da_alg_hyp in zip(tasks_name, online_models, scenarios, da_algs, da_models, da_alg_hyps): + + cur_target_domain_name = scenario.target_domains_order[scenario.cur_domain_index] + if cur_target_domain_name in da_alg_hyp: + da_alg_hyp = da_alg_hyp[cur_target_domain_name] + logger.info(f'use dataset-specific hyps') + + online_model.set_sd_sparsity(da_alg_hyp['sd_sparsity']) + sd, unpruned_indexes_of_layers = online_model.generate_sd_by_target_samples(scenario.get_online_cur_domain_samples_for_training(da_alg_hyp['train_batch_size'])) + + tmp_sd_path = os.path.join(log_dir, 'tmp_sd_model.pt') + torch.save({'main': sd}, tmp_sd_path) + + if 'cls' not in task_name and 'pos' not in task_name and 'vqa' not in task_name: + da_model_args = [f'{task_name}/{domain_index}', + tmp_sd_path, + device, + scenario.num_classes] + else: + da_model_args = [f'{task_name}/{domain_index}', + tmp_sd_path, + device] + da_metrics, after_da_model = da_alg( + {'main': da_model(*da_model_args)}, + os.path.join(log_dir, f'{task_name}/{domain_index}') + ).run(scenario, {_k: _v for _k, _v in da_alg_hyp.items() if _k != 'sd_sparsity'}) + os.remove(tmp_sd_path) + + if domain_index > 0: + shutil.rmtree(os.path.join(log_dir, f'{task_name}/{domain_index}/backup_codes')) + + online_model.sd_feedback_to_md(after_da_model['main'].models_dict['main'], unpruned_indexes_of_layers) + online_model.md_feedback_to_self_fm() + + accs = da_metrics['accs'] + before_acc = accs[0]['acc'] + after_acc = accs[-1]['acc'] + + avg_before_acc += before_acc + avg_after_acc += after_acc + + for _acc in accs: + tb_writer.add_scalar(f'total_acc', _acc['acc'], _acc['iter'] + global_iter) # TODO: bug here + global_iter += _acc['iter'] + 1 + + tb_writer.add_scalars(f'accs/{task_name}', dict(before=before_acc, after=after_acc), domain_index) + tb_writer.add_scalar(f'times/{task_name}', da_metrics['time'], domain_index) + + scenario.next_domain() + + logger.info(f"task: {task_name}, domain {domain_index}, acc: {before_acc:.4f} -> " + f"{after_acc:.4f} ({da_metrics['time']:.2f}s)") + cur_res[task_name] = da_metrics + + if involve_fm: + for online_model in online_models: + online_model.aggregate_fms_to_self_fm([m.models_dict['fm'] for m in online_models]) + for online_model in online_models: + online_model.fm_feedback_to_md() + + avg_before_acc /= len(tasks_name) + avg_after_acc /= len(tasks_name) + tb_writer.add_scalars(f'accs/apps_avg', dict(before=avg_before_acc, after=avg_after_acc), domain_index) + logger.info(f"--> domain {domain_index}, avg_acc: {avg_before_acc:.4f} -> " + f"{avg_after_acc:.4f}") + res += [cur_res] + + global_avg_after_acc += avg_after_acc + + write_json(os.path.join(log_dir, 'res.json'), res, backup=False) + + global_avg_after_acc /= (domain_index + 1) + logger.info(f'-----> final metric: {global_avg_after_acc:.4f}') + write_json(os.path.join(log_dir, f'res_{global_avg_after_acc:.4f}.json'), res, backup=False) + + + +def init_online_model(fm_models_dict_path, md_models_dict_path, task_name, __entry_file__): + fm_models = torch.load(fm_models_dict_path) + md_models = torch.load(md_models_dict_path) + + online_models_dict_path = save_models_dict_for_init({ + 'fm': fm_models['main'], + 'md': md_models['main'], + 'sd': None, + 'indexes': md_models['indexes'], + 'bn_stats': md_models['bn_stats'] + }, __entry_file__, task_name) + return online_models_dict_path diff --git a/gen_lora.png b/gen_lora.png new file mode 100644 index 0000000000000000000000000000000000000000..e99726d21ebe7dd1d9a8087996a2b72fdec56008 Binary files /dev/null and b/gen_lora.png differ diff --git a/gen_md_w_fbs_index.png b/gen_md_w_fbs_index.png new file mode 100644 index 0000000000000000000000000000000000000000..570ab54b5b54eb408d458f0bb945705f78262fec Binary files /dev/null and b/gen_md_w_fbs_index.png differ diff --git a/gen_md_wo_fbs.png b/gen_md_wo_fbs.png new file mode 100644 index 0000000000000000000000000000000000000000..8da3dd9ffc2a97ceaae6d1d24d956bc9aa9a9412 Binary files /dev/null and b/gen_md_wo_fbs.png differ diff --git a/gen_online.png b/gen_online.png new file mode 100644 index 0000000000000000000000000000000000000000..9d44a7527ad86856f1267a2ce802e8906b6ba2d8 Binary files /dev/null and b/gen_online.png differ diff --git a/index.html b/index.html index b0c4b3666032a737f3903db53e6a8a9272483e28..57b4cb9395a095927d75a1dad55a72ba6ef1689b 100644 --- a/index.html +++ b/index.html @@ -1,19 +1,625 @@ - + - - - - My static Space - - - -
-

Welcome to your static Space!

-

You can modify this app directly by editing index.html in the Files and versions tab.

-

- Also don't forget to check the - Spaces documentation. -

-
- - + + + + + EdgeTA + + + + +
+

EdgeTA: Retraining Multiple Foundation Models
for Evolving Data at Edge

+ +

Table of Contents

+ + +

1. Introduction

+

Foundation models (FMs) such as large language models are the driving force of the next generation artificial + intelligence systems. The trend of deploying FMs at edge challenges their scaling potential when encountering + massive new input data with compressed model sizes and constrained device resources. The prior art sheds light on + learning new tasks and domains (data feature shifts) based on deployed networks. However, such learning approaches + exacerbate the existing limitations: (i) predetermined network architectures lower model accuracy, and (ii) fixed + model sizes hinder resource allocation optimization at a finer granularity.

+

In this paper, we propose EdgeTA, a lightweight, neuron-grained scaling solution to unlock FMs' scaling potency + in edge intelligence systems. EdgeTA achieves high accuracy and low overheads in model retraining by adaptively + transforming a FM into a compact model that retains the most important neurons to the current input data. At + run-time, EdgeTA determines optimal model sizes and assigned resources for multiple applications to maximize their + overall accuracy. We implement EdgeTA in prevalent FMs of natural language processing, computer vision and + multimodal applications and compare it against state-of-the-art techniques. Evaluation results show that our + approach improves accuracy by 21.88% while reducing memory footprint and energy consumptions by 27.14% and 65.65%, + and further achieves 15.96% overall accuracy improvement via neuron-grained resource scheduling.

+ +

2. Code and Installation

+

The code is released in https://huggingface.co/spaces/EdgeTA/EdgeTA/tree/main. You can use the "git clone" command to clone this repository:

+ git clone https://huggingface.co/spaces/EdgeTA/EdgeTA +

The directory structure is organized as below:

+
    +
  • data: it contains datasets implementation
    • +
  • dnns: it contains models implementation
    • +
  • experiments: it contains the files to launch the experiments in the submitted paper
    • +
  • methods: it contains EdgeTA implementation
    • +
  • new_impl: it applies EdgeTA in several SOTA FMs: CLIP, SAM, and GLIP
    • +
  • utils: it contains several our implemented tool packages
    • +
+ +

2.1 Requirements

+
    +
  • Linux and Windows
    • +
  • Python 3.8+
    • +
  • CUDA 10.2+
    • +
+ +

2.2 Preparing Environment

+

First, create a conda virtual environment and activate it:

+ + conda create -n EdgeTA python=3.8
+ conda activate EdgeTA + +

Second, install torch and torchvision according to the offical + site.

+ +

Get the installation command according to the selection in the official site, and copy them to the terminal.

+

Finally, install the required dependencies via pip:

+ + pip install -r requirements.txt + + +

3. Running Example 1: Supporting a Hugging Face FM Vision Transformer

+ +

3.1 Settings

+

Models. We use a semantic segmentation model based on Vision Transformer from Hugging Face as an example + to explain how to connect a Hugging Face FM to the EdgeTA.

+

Datasets. We use datasets GTA5 + and SuperviselyPerson as the source domain, and datasets Cityscapes + and BaiduPerson as the target domain.

+ +

3.2 Offline Elastic Proxy Construction

+

Run the following command sequentially to pre-train the knowledge base and index:

+ + python experiments/elasticdnn/vit_b_16/offline/fm_lora/cls/cls.py
+ python experiments/elasticdnn/vit_b_16/offline/fm_to_md/cls_md_wo_fbs.py
+ python experiments/elasticdnn/vit_b_16/offline/fm_to_md/cls_md_index.py
+ +

Note that the file path of the model checkpoint in last two files should be modified manually.

+

Run the following command to open TensorBoard and watch the metrics (e.g. losses and accuracy) during the + training process:

+ + tensorboard --logdir <the file path of tensorboard logs outputed in the terminal> + +

Here are three TensorBoard screenshots when three commands above are running:

+ + + + +

3.3 Online Evolving Input Data Adaptation

+

Run the following command to evaluate EdgeTA over evolving data:

+ + python experiments/elasticdnn/vit_b_16/online_new/cls/cls.py + +

You can also launch TensorBoard to watch the retraining accuracy and time during the retraining process. Here is + a screenshot:

+ + +

(Optional) 3.4 Tuning the hyperparameters

+

Most of hyperparameters are common and easy to understand (e.g. batch size, learning rate, and optimizer + arguments, etc). We introduce some unique hyperparameters in EdgeTA below.

+

For python experiments/elasticdnn/vit_b_16/offline/fm_lora/cls/cls.py:

+
    +
  • ab_r: the value of r in LoRA.
    • +
+

For python experiments/elasticdnn/vit_b_16/offline/fm_to_md/cls_md_wo_fbs.py:

+
    +
  • sample_size: the size of an input sample. For typical image workloads, the size is (1, 3, 224, 224). + For language workloads, you can directly pass in a tokenized sample directory instead of a size.
    • +
  • generate_md_width_ratio: the ratio of the original FM's width to knowledge base's width. We recommend + it is 4 or 8, which means that the knowledge base has 1/4 or 1/8 model size of the original FM.
    • +
  • distill_loss_weight: it controls the strength of distilling the original FM's feature to the knowledge + base's feature using feature-based knowledge distillation. This helps improve the accuracy of the knowledge + base.
    • +
+

For python experiments/elasticdnn/vit_b_16/offline/fm_to_md/cls_md_index.py:

+
    +
  • FBS_r: the value of r in FBS module. We recommend it is 16.
    • +
  • indexes_optimizer_args: the arguments of the optimizer used in training the neuron index between the + knowledge base and the FM.
    • +
  • min_sparisty and max_sparsity: in each training iteration, the knowledge base is set to a random + sparsity and then trained (refer to dynamic neural networks). min_sparisty and max_sparsity determine that the + maximal model size and the minimal model size of the generated proxy model. For example, if min_sparisty = 0 and + max_sparsity = 0.9, the maximal model size of the proxy model is the same to the knowledge base, and the minimal + model size of the proxy model is 10% of the knowledge base.
    • +
  • bn_cal_num_iters: BN statstics is unstable during the training of dynamic neural networks (refer to + S-Net (ICLR'19)). Therefore, before testing the accuracy of the knowledge base, its BN statstics should be + calibrated using several iterations of inference on the test dataset (if the model has any BN layers).
    • +
  • index_init: how the value of neuron index is initialized. We recommend it is 'zero'.
    • +
+ + + + +

4. Running Example 2: Supporting a Hugging Face FM CLIP

+ +

4.1 Settings

+

Models. We use a image classification model based on CLIP from Hugging Face as an example + to explain how to connect a Hugging Face FM to the EdgeTA.

+

Datasets. We use datasets GTA5 + and SuperviselyPerson as the source domain, and datasets Cityscapes + and BaiduPerson as the target domain. We convert these semantic segmentation datasets into image classification datasets by cropping and saving the images in the segmentation bounding boxes.

+ +

4.2 Offline Elastic Proxy Construction

+

Run the following command sequentially to pre-train the knowledge base and index:

+ + python new_impl/cv/clip/cls.py
+ python new_impl/cv/clip/cls_md_wo_fbs.py
+ python new_impl/cv/clip/cls_md_index.py
+ +

Note that the file path of the model checkpoint in last two files should be modified manually.

+

Run the following command to open TensorBoard and watch the metrics (e.g. losses and accuracy) during the + training process:

+ + tensorboard --logdir <the file path of tensorboard logs outputed in the terminal> + +

Here are three TensorBoard screenshots when three commands above are running:

+ + +

4.3 Online Evolving Input Data Adaptation

+

Run the following command to evaluate EdgeTA over evolving data:

+ + python new_impl/cv/clip/cls_online.py + +

You can also launch TensorBoard to watch the retraining accuracy and time during the retraining process. Here is + a screenshot:

+ + + + + +

5. Running Example 3: Supporting a user-specified FM SAM (Segment Anything)

+ +

5.1 Settings

+

Models. We use the SOTA segmentation foundation model SAM. In this example, we support SAM using our designed standard FM API to explain how to connect a user-specified FM to the EdgeTA.

+

Datasets. We use datasets GTA5 + and SuperviselyPerson as the source domain, and datasets Cityscapes + and BaiduPerson as the target domain.

+ +

5.2 Offline Elastic Proxy Construction

+

Run the following command sequentially to pre-train the knowledge base and index:

+ + python new_impl/cv/sam/seg.py
+ python new_impl/cv/sam/seg_md_wo_fbs.py
+ python new_impl/cv/sam/seg_md_index.py
+ +

Note that the file path of the model checkpoint in last two files should be modified manually.

+

Run the following command to open TensorBoard and watch the metrics (e.g. losses and accuracy) during the + training process:

+ + tensorboard --logdir <the file path of tensorboard logs outputed in the terminal> + +

Here are three TensorBoard screenshots when three commands above are running:

+ + +

5.3 Online Evolving Input Data Adaptation

+

Run the following command to evaluate EdgeTA over evolving data:

+ + python new_impl/cv/seg/seg_online.py + +

You can also launch TensorBoard to watch the retraining accuracy and time during the retraining process. Here is + a screenshot:

+ + + + + +

6. Running Example 4: Supporting a user-specified FM GLIP

+

6.1 Settings

+ +

Models. GLIP is a language image pretrained model used to learn object level, language aware, and semantically rich visual representations. GLIP combines object detection and phase grounding for pre training, enabling object detection of images based on prompts. In this example, we support GLIP using our designed standard FM API to explain how to connect a user-specified FM to the EdgeTA.

+

Because there is no GLIP model code in the transformers library, you need to download the code, the weight and the config of the GLIP model from github. Then you should place them under the path "new_impl/cv/glip/object_detection/pretrained_ model" and setup the code. In addition, you should also modify the code about GLIP to make the GLIP model (GeneralizedVLRCNN) outputs the token_logits and the dot_product_logits when it's in eval mode.

+

Datasets. In the example, we will use datasets COCO2017 as the source domain dataset, and Cityscapes and GTA5 as the target domain datasets.

+

6.2 Offline Elastic Proxy Construction

+

Run the following command sequentially to pre-train the knowledge base and index:

+ + python new_impl/cv/glip/object_detection/det_lora.py
+ python new_impl/cv/glip/object_detection/det_md_wo_fbs.py
+ python new_impl/cv/glip/object_detection/det_md_w_fbs_index.py
+ +

Note that the file path of the model checkpoint in last two files should be modified manually.

+

Run the following command to open TensorBoard and watch the metrics (e.g. losses and accuracy) during the training process:

+ + tensorboard --logdir <the file path of tensorboard logs outputed in the terminal> + +

Here are three TensorBoard screenshots when three commands above are running:

+ + + +

6.3 Online Evolving Input Data Adaptation

+

Run the following command to evaluate EdgeTA over evolving data:

+ + python new_impl/cv/glip/object_detection/det_online.py + +

You can launch TensorBoard to watch the retraining mAP@50 score and time during the retraining process. Here is a screenshot:

+ + +

7. Running Example 5: Supporting GPT-Neo

+

7.1 Settings

+

Models

+

GPT-Neo is an open-source text AI model launched by German company Eleuther Artificial Intelligence in late March 2021 to compensate for the lack of open-source GPT-3 models. In this example, we support GPT-Neo using our designed standard FM API to explain how to connect a user-specified FM to the EdgeTA.

+

Datasets

+

In the example, we will use datasets No_robots as the source domain dataset. Medicine-tasks and law-tasks as the target domain datasets. They are all conversational datasets.

+

7.2 Offline Elastic Proxy Construction

+

Run the following command sequentially to pre-train the knowledge base and index:

+ + python new_impl/nlp/gpt-neo/text_generation/gen_lora.py
+ python new_impl/nlp/gpt-neo/text_generation/gen_md_wo_fbs.py
+ python new_impl/nlp/gpt-neo/text_generation/gen_md_w_fbs_index.py
+ +

Note that the file path of the model checkpoint in last two files should be modified manually.

+

Run the following command to open TensorBoard and watch the metrics (e.g. losses and accuracy) during the training process:

+ + tensorboard --logdir <the file path of tensorboard logs outputed in the terminal> + +

Here are three TensorBoard screenshots when three commands above are running:

+ + + +

7.3 Online Evolving Input Data Adaptation

+

Run the following command to evaluate EdgeTA over evolving data:

+ + python new_impl/nlp/gpt-neo/text_generation/gen_online.py + +

You can launch TensorBoard to watch the retraining mAP50 score and time during the retraining process. Here is a screenshot:

+ +

8. Running Example 6: Supporting Roberta

+

8.1 Settings

+

Models

+

We used the base version of the Roberta model (an improved version of Bert) to demonstrate how to connect a Hugging Face FM to the EdgeTA.

+

Datasets

+

We use the dataset named HL5Domains which includes five datasets called ApexAD2600Progressive, CanonG3, CreativeLabsNomadJukeboxZenXtra40GB, NikonCoolpix4300 and Nokia6610. Among them, ApexAD2600Progressive, CanonG3 and CreativeLabsNomadJukeboxZenXtra40GB are used as the source domains, and the datasets NikonCoolpix4300 and Nokia6610 are used as the target domains. They are all from amazon.com.

+

8.2 Offline Elastic Proxy Construction

+

Run the following command sequentially to pre-train the knowledge base and index:

+ + python new_impl/nlp/roberta/sentiment-classification/cls_lora.py
+ python new_impl/nlp/roberta/sentiment-classification/cls_md_wo_fbs.py
+ python new_impl/nlp/roberta/sentiment-classification/cls_md_w_fbs_index.py
+ +

Note that the file path of the model checkpoint in last two files should be modified manually.

+

Run the following command to open TensorBoard and watch the metrics (e.g. losses and accuracy) during the training process:

+ + tensorboard --logdir <the file path of tensorboard logs outputed in the terminal> + +

Here are three TensorBoard screenshots when three commands above are running:

+ + + +

8.3 Online Evolving Input Data Adaptation

+

Run the following command to evaluate EdgeTA over evolving data:

+ + python new_impl/nlp/roberta/sentiment-classification/cls_online.py + +

You can launch TensorBoard to watch the retraining mAP50 score and time during the retraining process. Here is a screenshot:

+ > + +

9. Implementation (Development API Documentation)

+

EdgeTA is implemented in Python with 8k LOCs and it is currently targeted for transformers running on commodity + edge devices and Linux environment. Its scaling and retraining of transformers are implemented based on timm 0.9.1 + and transformers 4.30.2. Its scheduler is built upon the optimization problem solver in scikit-opt 0.6.6 and + resource management systems in Docker 10.03.6 and K3s 1.18.12.

+

Figure below illustrates the three steps of running a FM using EdgeTA. To facilitate the integration of a model, + EdgeTA decouples the integration of a model (step 1) from its offline construction of knowledge base and neuron + index (step 2) and online scaling and retraining of FM (step 3). This system design allows users only need to + implement the FM API at step 1 to integrate a model. Specifically, EdgeTA supports two types of models.

+ +

Hugging Face FMs. We implement EdgeTA to support FM APIs in the Hugging Face AI community. Using the + AutoModel as example, EdgeTA calls function AutoModel.from_pretrained() to initialize a FM and calls function + AutoModel.forward() to perform a forward operation. EdgeTA allows users to run a Hugging Face's FM using 30 about + LOCs.

+

User-specified FMs. EdgeTA designs a standard FM API (colored by green in the figure) to unify user + specified FM implementations. This API mainly defines: (i) how the FM performs an inference using the given + sample; (ii) how the accuracy of the FM is measured using the given test dataset; (iii) how to manipulate (e.g. + compress/update/remove) a specific layer in the FM. For each FM, this API can be implemented using about 200 LOCs. +

+ +

9.1 Supporting a Hugging Face FM

+ +

Supporting a Hugging Face Model is a simplification of supporting a user-specified model, because Hugging Face + FMs have many consistent implementation style so repetitive implementation work can be saved. The user can only + implement the following several simple functions:

+ +
    + +
  • + def get_feature_hook(self) +
      +
    • Get the PyTorch hook attached before the layer that extracts the key features.
      • +
    • + Output: A PyTorch hook. +
      • +
    +
    • + +
  • + def get_task_head_params(self) +
      +
    • Get the model parameters of the task head of the FM.
      • +
    • + Output: The model parameters of the task head of the FM. +
      • +
    +
    • + +
  • + def get_qkv_proj_ff1_ff2_layer_names(self) +
      +
    • Get a list of names, and each element in the list is a list that contains the names of Q/K/V layer, QKV + projection layer, Feed Forward Layer 1, Feed Forward Layer 2. For example, for Hugging Face's BERT, this + function should return [['bert.encoder.layer.0.attention.self.query', + 'bert.encoder.layer.0.attention.self.key', 'bert.encoder.layer.0.attention.self.value', + 'bert.encoder.layer.0.attention.output.dense', 'bert.encoder.layer.0.intermediate.dense', + 'bert.encoder.layer.0.output.dense'], ['bert.encoder.layer.1.attention.self.query', + 'bert.encoder.layer.1.attention.self.key', 'bert.encoder.layer.1.attention.self.value', + 'bert.encoder.layer.1.attention.output.dense', 'bert.encoder.layer.1.intermediate.dense', + 'bert.encoder.layer.1.output.dense'], ...]
      • +
    • + Output: A list of names. +
      • +
    +
    • + +
  • + def get_accuracy(self, test_loader) +
      +
    • Measure the accuracy of the FM using the given test data loader.
      • +
    • + Inputs: +
        +
      • test_loader: A given test dataloader.
        • +
      +
      • +
    • + Output: The measured accuracy. +
      • +
    +
    • + +
+ +

9.2 Supporting a user-specified FM

+

The user should implement the following functions in the standard FM API.

+
    + +
  • + def forward(self, x, *args, **kwargs) +
      +
    • Let the FM perform a forward inference operation using the given sample x.
      • +
    • + Inputs: +
        +
      • x: A given sample.
        • +
      • *args and **kwargs: Possible additional arguments used in the inference.
        • +
      +
      • +
    • + Output: The inference results. +
      • +
    +
    • + +
  • + def get_accuracy(self, test_loader) +
      +
    • Measure the accuracy of the FM using the given test data loader.
      • +
    • + Inputs: +
        +
      • test_loader: A given test dataloader.
        • +
      +
      • +
    • + Output: The measured accuracy. +
      • +
    +
    • + +
  • + def forward_to_get_task_loss(self, x, y *args, **kwargs) +
      +
    • Let the FM perform a forward operation using the given sample x, and calculate and return the task loss. +
      • +
    • + Inputs: +
        +
      • x: A given sample.
        • +
      • y: The corresponding label of x.
        • +
      • *args and **kwargs: Possible additional arguments used in the inference.
        • +
      +
      • +
    • + Output: The calculated task loss. +
      • +
    +
    • + +
  • + def get_feature_hook(self) +
      +
    • Get the PyTorch hook attached before the layer that extracts the key features.
      • +
    • + Output: A PyTorch hook. +
      • +
    +
    • + +
  • + def get_task_head_params(self) +
      +
    • Get the model parameters of the task head of the FM.
      • +
    • + Output: The model parameters of the task head of the FM. +
      • +
    +
    • + +
  • + def add_lora_ab_to_fm(self, ab_r: int, samples: torch.Tensor) +
      +
    • Add a LoRA matrix to each attention layer in the FM. The user should check if the FM's output is changed + before and after the LoRA is added into the FM.
      • +
    • + Inputs: +
        +
      • ab_r: the factor r in LoRA.
        • +
      • samples: A given sample for sanity check.
        • +
      +
      • +
    • + Output: A PyTorch hook. +
      • +
    +
    • + +
  • + def fuse_lora_and_recover_net_structure(self, samples: torch.Tensor) +
      +
    • Fuse the added LoRA matrix into the corresponding attention layer in the FM, and recover the network + structure to the original. This is invoked after the LoRA fine tuning.
      • +
    • + Inputs: +
        +
      • samples: A given sample for sanity check.
        • +
      +
      • +
    • + Output: A PyTorch hook. +
      • +
    +
    • + +
  • + def is_q_or_k_v_linear(self, layer_name: nn.Module) +
      +
    • Check if the given layer is a Q/K/V Linear in the FM.
      • +
    • + Inputs: +
        +
      • layer_name: The name of a layer in the FM.
        • +
      +
      • +
    • + Output: Return True if the given layer is a Q/K/V Linear in the FM. +
      • +
    +
    • + +
  • + def is_feed_forward(self, layer_name: nn.Module) +
      +
    • Check if the given layer is a feed forward layer in the FM.
      • +
    • + Inputs: +
        +
      • layer_name: The name of a layer in the FM.
        • +
      +
      • +
    • + Output: Return True if the given layer is a feed forward layer in the FM. +
      • +
    +
    • + +
  • + def prune_an_attention_layer(self, attention_layer_name, sparsity: float, samples: torch.Tensor) +
      +
    • Pruning an attention layer.
      • +
    • + Inputs: +
        +
      • attention_layer_name: The name of target attention layer.
        • +
      • sparsity: The pruning strength.
        • +
      • samples: A given sample.
        • +
      +
      • +
    +
    • + +
  • + def prune_an_feed_forward_layer(self, feed_forward_layer_name, sparsity: float, samples: torch.Tensor) +
      +
    • Pruning an feed forward layer.
      • +
    • + Inputs: +
        +
      • feed_forward_layer_name: The name of target feed forward layer.
        • +
      • sparsity: The pruning strength.
        • +
      • samples: A given sample.
        • +
      +
      • +
    +
    • + +
+ +

10. Experimental evaluation in ICDE 2024 submission

+

10.1 Basic settings

+

Testbeds. We evaluate EdgeTA on four heterogeneous edge devices: NVIDIA Jetson TX2 (8GB memory), NVIDIA Xavier NX (16GB memory), NVIDIA AGX Xavier (32GB memory), and NVIDIA AGX Orin (32GB memory).

+

Baselines. We compare EdgeTA with 13 adaptation methods, including 5 supervised continual learning methods and 8 unsupervised domain adaptation methods.

+

Workloads. We evaluate EdgeTA on three representative FMs: ViT-B/16 (CV), BERT_base (NLP), and ViLT (multimodal). ViT-B/16 is added three different application heads respectively to perform image classification, object detection, and semantic segmentation application. BERT_base is added two different application heads respectively to perform sentence classification and pos-of-tagging classification application. ViLT performs visual question answering application. Finally, GPT-Neo is evaluated in the discussion of PEFT techniques. We evaluate EdgeTA on 11 different datasets: GTA5, SuperviselyPerson, MSCOCO2017, Cityscapes, BaiduPerson, HL5Domains, Liu3Domains, Ding9Domains, SemEval14, 20Newsgroups, and VQAv2. More details refer to the table below.

+ + + +

10.2 Additional details

+

Online adaptation. For evolving domain shifts, EdgeTA uses the naive feature alignment (the most classical method for unsupervised domain adaptation) to retrain the proxy model. For evolving new tasks, EdgeTA uses the normal supervised learning to retrain the proxy model.

+

Applicability of baseline adaptation methods. Some baseline adaptation methods are inapplicable in some applications, so Figure 6, Table II, and Table III do not report their metrics. Specifically:

+
    +
  • SHOT and ConDA rely on pseudo labels, and their label generation algorithm can only generate the pseudo label with one dimension. However, the label in semantic segmentation application has three dimensions, the label in object detection application has bounding box information, the label in pos-of-tagging application has two dimensions, so SHOT and ConDA are inapplicable for these applications.
    • +
  • BUFR calculates the KL divergence between the model's outputs in the source domain and the target domain, and the KL divergence only applies in the classification output. So BUFR is also inapplicable in semantic segmentation, object detection, and pos-of-tagging application.
    • +
  • ACE uses a image generator so it is inapplicable in text classification, pos-of-tagging and visual question answering application.
    • +
+ +

10.3 Additional experiment results (applying EdgeTA in SOTA FMs: CLIP, SAM, GLIP, GPT-Neo and Roberta)

+

Besides the evaluated FMs in the submitted paper, there are some SOTA FMs. Some are SOTA CV FMs, such as CLIP for image classification, SAM for semantic segmentation, and GLIP for object detection, and the others are SOTA NLP FMs, such as GPT-Neo for text generation, and Roberta for sentiment classification. These SOTA FMs and the currently tested FMs have similar network architectures and sizes, that is, CLIP, SAM, and GLIP comprise a ViT and a GPT-2 (similar to BERT), as well as GPT Neo and Roberts including Transformer structure (the main structure used by BERT). We therefore expect EdgeTA still works well for these FMs.

+

To prove that, we ran a new set of experiments on CLIP, SAM, GLIP, GPT-Neo and Roberta to compare EdgeTA and CUA on NVIDIA Xavier NX. The results are demonstrated below. EdgeTA improves the accuracy by 12.71%, 7.36%, 6.41%, 10.81% and 10.38% for five FMs, respectively, which proves the applicability of EdgeTA to various FMs.

+ +
+ + + \ No newline at end of file diff --git a/methods/base/__pycache__/alg.cpython-38.pyc b/methods/base/__pycache__/alg.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8f94385f615f50933267ecdea4f8584da443d794 Binary files /dev/null and b/methods/base/__pycache__/alg.cpython-38.pyc differ diff --git a/methods/base/__pycache__/model.cpython-38.pyc b/methods/base/__pycache__/model.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..61535036cf658c4007d607e7a1b2417cec714a33 Binary files /dev/null and b/methods/base/__pycache__/model.cpython-38.pyc differ diff --git a/methods/base/alg.py b/methods/base/alg.py new file mode 100644 index 0000000000000000000000000000000000000000..ce3e6a47e2af7ae6f578192dab2a6d7943dd0e3a --- /dev/null +++ b/methods/base/alg.py @@ -0,0 +1,54 @@ +from typing import Dict, Any +from torch import nn +from data.datasets.ab_dataset import ABDataset +from abc import ABC, abstractmethod +from utils.common.log import logger +import json +import os + +from utils.common.others import backup_key_codes +from .model import BaseModel +from data import Scenario +from schema import Schema +from utils.common.data_record import write_json + + +class BaseAlg(ABC): + + def __init__(self, models: Dict[str, BaseModel], res_save_dir): + self.models = models + self.res_save_dir = res_save_dir + self.get_required_models_schema().validate(models) + + os.makedirs(res_save_dir) + logger.info(f'[alg] init alg: {self.__class__.__name__}, res saved in {res_save_dir}') + + @abstractmethod + def get_required_models_schema(self) -> Schema: + raise NotImplementedError + + @abstractmethod + def get_required_hyp_schema(self) -> Schema: + raise NotImplementedError + + @abstractmethod + def run(self, + scenario: Scenario, + hyps: Dict) -> Dict[str, Any]: + """ + return metrics + """ + + self.get_required_hyp_schema().validate(hyps) + + try: + write_json(os.path.join(self.res_save_dir, 'hyps.json'), hyps, ensure_obj_serializable=True) + except: + with open(os.path.join(self.res_save_dir, 'hyps.txt'), 'w') as f: + f.write(str(hyps)) + + write_json(os.path.join(self.res_save_dir, 'scenario.json'), scenario.to_json()) + + logger.info(f'[alg] alg {self.__class__.__name__} start running') + + backup_key_codes(os.path.join(self.res_save_dir, 'backup_codes')) \ No newline at end of file diff --git a/methods/base/model.py b/methods/base/model.py new file mode 100644 index 0000000000000000000000000000000000000000..200616054944746ca3fa1b4ee47c3000e171cef6 --- /dev/null +++ b/methods/base/model.py @@ -0,0 +1,74 @@ +import torch +from abc import ABC, abstractmethod +from utils.common.file import ensure_dir +from utils.common.log import logger +import time +from typing import List + + +class BaseModel(ABC): + def __init__(self, + name: str, + models_dict_path: str, + device: str): + + self.name = name + self.models_dict_path = models_dict_path + self.models_dict = torch.load(models_dict_path, map_location=device) + + self.device = device + + assert set(self.get_required_model_components()) <= set(list(self.models_dict.keys())) + + self.to(device) + + logger.info(f'[model] init model: {dict(name=name, components=self.get_required_model_components())}') + logger.debug(self.models_dict) + + @abstractmethod + def get_required_model_components(self) -> List[str]: + pass + + @abstractmethod + def get_accuracy(self, test_loader, *args, **kwargs): + pass + + @abstractmethod + def infer(self, x, *args, **kwargs): + pass + + def save_model(self, p: str): + logger.debug(f'[model] save model: {self.name}') + ensure_dir(p) + torch.save(self.models_dict, p) + + def load_model(self, p: str): + logger.debug(f'[model] load model: {self.name}, from {p}') + self.models_dict = torch.load(p, map_location=self.device) + + def to(self, device): + logger.debug(f'[model] to device: {device}') + for k, v in self.models_dict.items(): + try: + self.models_dict[k] = v.to(device) + except Exception as e: + pass + + def to_eval_mode(self, verbose=False): + if verbose: + logger.info(f'[model] to eval mode') + for k, v in self.models_dict.items(): + try: + self.models_dict[k].eval() + except Exception as e: + pass + + def to_train_mode(self, verbose=False): + if verbose: + logger.info(f'[model] to train mode') + for k, v in self.models_dict.items(): + try: + self.models_dict[k].train() + except Exception as e: + pass + \ No newline at end of file diff --git a/methods/elasticdnn/api/__pycache__/model.cpython-38.pyc b/methods/elasticdnn/api/__pycache__/model.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f7e97fe969e32a17fb47cbd9eae79be640a476e5 Binary files /dev/null and b/methods/elasticdnn/api/__pycache__/model.cpython-38.pyc differ diff --git a/methods/elasticdnn/api/__pycache__/online_model_v2.cpython-38.pyc b/methods/elasticdnn/api/__pycache__/online_model_v2.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..79f07313e19625f637fa3bd8ad89e9c3e7913652 Binary files /dev/null and b/methods/elasticdnn/api/__pycache__/online_model_v2.cpython-38.pyc differ diff --git a/methods/elasticdnn/api/algs/__pycache__/fm_lora.cpython-38.pyc b/methods/elasticdnn/api/algs/__pycache__/fm_lora.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..baea9ac55054d6cb6af8ecaa38203fba5d542e73 Binary files /dev/null and b/methods/elasticdnn/api/algs/__pycache__/fm_lora.cpython-38.pyc differ diff --git a/methods/elasticdnn/api/algs/__pycache__/md_pretraining_index_v2_train_index_and_md.cpython-38.pyc b/methods/elasticdnn/api/algs/__pycache__/md_pretraining_index_v2_train_index_and_md.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f9162c241550e72144688a6ceb21d93e0e1b8ae7 Binary files /dev/null and b/methods/elasticdnn/api/algs/__pycache__/md_pretraining_index_v2_train_index_and_md.cpython-38.pyc differ diff --git a/methods/elasticdnn/api/algs/__pycache__/md_pretraining_wo_fbs.cpython-38.pyc b/methods/elasticdnn/api/algs/__pycache__/md_pretraining_wo_fbs.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..145567875e5fcb886fdc488ca1a56c109ac91eb9 Binary files /dev/null and b/methods/elasticdnn/api/algs/__pycache__/md_pretraining_wo_fbs.cpython-38.pyc differ diff --git a/methods/elasticdnn/api/algs/__pycache__/md_pretraining_wo_fbs_gpt_neo.cpython-38.pyc b/methods/elasticdnn/api/algs/__pycache__/md_pretraining_wo_fbs_gpt_neo.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ba0e6b4069bcf4876f0b94400d60434bde930a9d Binary files /dev/null and b/methods/elasticdnn/api/algs/__pycache__/md_pretraining_wo_fbs_gpt_neo.cpython-38.pyc differ diff --git a/methods/elasticdnn/api/algs/fm_lora.py b/methods/elasticdnn/api/algs/fm_lora.py new file mode 100644 index 0000000000000000000000000000000000000000..bfa69081ce678f997a709c5e1a18a5d4e8cc0c7a --- /dev/null +++ b/methods/elasticdnn/api/algs/fm_lora.py @@ -0,0 +1,154 @@ +from typing import Any, Dict +from schema import Schema +from data import Scenario, MergedDataset +from methods.base.alg import BaseAlg +from data import build_dataloader +from ..model import ElasticDNN_OfflineFMModel +from ...model.base import ElasticDNNUtil +import torch.optim +import tqdm +from torch import nn +from torchvision.transforms import Compose +from utils.dl.common.env import create_tbwriter +import os +import random +import numpy as np +from copy import deepcopy +from utils.dl.common.model import get_module +from utils.common.log import logger + + +class ElasticDNN_FMLoRAAlg(BaseAlg): + def get_required_models_schema(self) -> Schema: + return Schema({ + 'fm': ElasticDNN_OfflineFMModel + }) + + def get_required_hyp_schema(self) -> Schema: + from schema import Optional + return Schema({ + 'launch_tbboard': bool, + + 'samples_size': object, + 'ab_r': int, + + 'train_batch_size': int, + 'val_batch_size': int, + 'num_workers': int, + 'optimizer': str, + 'optimizer_args': dict, + 'scheduler': str, + 'scheduler_args': dict, + 'num_iters': int, + 'val_freq': int, + + Optional('fm_lora_ckpt_path'): str, + Optional('transform'): Compose, + }) + + def run(self, scenario: Scenario, hyps: Dict, collate_fn=None) -> Dict[str, Any]: + super().run(scenario, hyps) + + assert isinstance(self.models['fm'], ElasticDNN_OfflineFMModel) # for auto completion + + # 1. add LoRA + lora_util = self.models['fm'].get_lora_util() + device = self.models['fm'].device + + sample = hyps['samples_size'] + if isinstance(sample, (tuple, list)) and isinstance(sample[0], int): + sample = torch.rand(hyps['samples_size']).to(device) + lora_util.add_lora_ab_to_fm(self.models['fm'].models_dict['main'], hyps['ab_r'], sample) + + if 'fm_lora_ckpt_path' in hyps.keys() and hyps['fm_lora_ckpt_path'] != '' and hyps['fm_lora_ckpt_path'] is not None: + _ckpt = torch.load(hyps['fm_lora_ckpt_path'])['main'] + + new_state_dict = deepcopy(self.models['fm'].models_dict['main'].state_dict()) + + for n, p in _ckpt.named_parameters(): + if 'qkv.abs' not in n: + continue + + new_state_dict[n] = p + logger.info(f'use {n} from ckpt') + + self.models['fm'].models_dict['main'].load_state_dict(new_state_dict) + + + # 2. train (knowledge distillation, index relationship) + if 'transform' in hyps.keys(): + offline_datasets = scenario.get_offline_datasets(transform=hyps['transform']) + else: + offline_datasets = scenario.get_offline_datasets() + train_dataset = MergedDataset([d['train'] for d in offline_datasets.values()]) + + # debug + # from data.visualize import visualize_classes_in_object_detection + # d = offline_datasets['GTA5Det']['val'] + # class_to_idx_map = {c: d.idx_map[i] for i, c in enumerate(d.classes)} + # print(class_to_idx_map) + # visualize_classes_in_object_detection(d, class_to_idx_map, + # {}, os.path.join(self.res_save_dir, 'debug.png')) + # exit() + + val_dataset = MergedDataset([d['val'] for d in offline_datasets.values()]) + train_loader = iter(build_dataloader(train_dataset, hyps['train_batch_size'], hyps['num_workers'], + True, None, collate_fn=collate_fn)) + + # if hyps['use_train_loader_for_val']: + # val_loader = build_dataloader(train_dataset, hyps['val_batch_size'], hyps['num_workers'], + # False, False) + # logger.warn('use train loader for val!!!') + # else: + val_loader = build_dataloader(val_dataset, hyps['val_batch_size'], hyps['num_workers'], + False, False, collate_fn=collate_fn) + + lora_params = lora_util.train_only_lora(self.models['fm'].models_dict['main']) + head_params = self.models['fm'].get_task_head_params() + + num_lora_params = sum([np.prod(p.size()) for p in lora_params]) + total_params = sum([np.prod(p.size()) for p in self.models['fm'].models_dict['main'].parameters()]) + logger.info(f'num lora params: {num_lora_params}, total params: {total_params}, ratio: {num_lora_params / total_params}') + + optimizer = torch.optim.__dict__[hyps['optimizer']](lora_params + head_params, **hyps['optimizer_args']) + scheduler = torch.optim.lr_scheduler.__dict__[hyps['scheduler']](optimizer, **hyps['scheduler_args']) + + fbs_tb_writer = create_tbwriter(os.path.join(self.res_save_dir, 'tb_log'), launch_tbboard=hyps['launch_tbboard']) + pbar = tqdm.tqdm(range(hyps['num_iters']), dynamic_ncols=True) + + best_val_acc = 0 + val_acc = 0 + + for iter_index in pbar: + self.models['fm'].to_train_mode() + + x, y = next(train_loader) + + if isinstance(x, dict): + for k, v in x.items(): + if isinstance(v, torch.Tensor): + x[k] = v.to(device) + y = y.to(device) + else: + x, y = x.to(device), y.to(device) + task_loss = self.models['fm'].forward_to_get_task_loss(x, y) + optimizer.zero_grad() + task_loss.backward() + optimizer.step() + scheduler.step() + + if (iter_index + 1) % hyps['val_freq'] == 0: + # logger.warn('use train loader for val!!!') + + self.models['fm'].to_eval_mode() + val_acc = self.models['fm'].get_accuracy(val_loader) + + self.models['fm'].save_model(os.path.join(self.res_save_dir, 'models/fm_last.pt')) + if val_acc > best_val_acc: + best_val_acc = val_acc + self.models['fm'].save_model(os.path.join(self.res_save_dir, 'models/fm_best.pt')) + + fbs_tb_writer.add_scalar(f'losses/task_loss', task_loss, iter_index) + fbs_tb_writer.add_scalar(f'accs/val_acc', val_acc, iter_index) + fbs_tb_writer.add_scalar(f'lr', optimizer.param_groups[0]['lr'], iter_index) + pbar.set_description(f'loss: {task_loss:.6f}, val_acc: {val_acc:.4f}') diff --git a/methods/elasticdnn/api/algs/md_pretraining_bk.py b/methods/elasticdnn/api/algs/md_pretraining_bk.py new file mode 100644 index 0000000000000000000000000000000000000000..f8903b7569f54f2dcd364359001de925a3ab9287 --- /dev/null +++ b/methods/elasticdnn/api/algs/md_pretraining_bk.py @@ -0,0 +1,292 @@ +from typing import Any, Dict +from schema import Schema, Or +import schema +from data import Scenario, MergedDataset +from methods.base.alg import BaseAlg +from data import build_dataloader +from ..model import ElasticDNN_OfflineFMModel, ElasticDNN_OfflineMDModel +from ...model.base import ElasticDNNUtil +import torch.optim +import tqdm +import torch.nn.functional as F +from torch import nn +from utils.dl.common.env import create_tbwriter +import os +import random +import numpy as np +from copy import deepcopy +from utils.dl.common.model import LayerActivation, get_module +from utils.common.log import logger + + +class ElasticDNN_MDPretrainingAlg(BaseAlg): + """ + TODO: fine-tuned FM -> init MD -> trained MD -> construct indexes (only between similar weights) and fine-tune + """ + def get_required_models_schema(self) -> Schema: + return Schema({ + 'fm': ElasticDNN_OfflineFMModel, + 'md': ElasticDNN_OfflineMDModel + }) + + def get_required_hyp_schema(self) -> Schema: + return Schema({ + 'launch_tbboard': bool, + + 'samples_size': (int, int, int, int), + 'generate_md_width_ratio': int, + + 'FBS_r': int, + 'FBS_ignore_layers': [str], + + 'train_batch_size': int, + 'val_batch_size': int, + 'num_workers': int, + 'optimizer': str, + 'md_optimizer_args': dict, + 'indexes_optimizer_args': dict, + 'scheduler': str, + 'scheduler_args': dict, + 'num_iters': int, + 'val_freq': int, + 'max_sparsity': float, + 'min_sparsity': float, + 'distill_loss_weight': float, + 'index_loss_weight': float, + 'val_num_sparsities': int, + + 'bn_cal_num_iters': int, + + 'index_guided_linear_comb_split_size': Or(int, None) + }) + + def upsample_2d_tensor(self, p: torch.Tensor, target_len: int): + assert p.dim() == 2 # regard 2d weight as (batch_size, 1d_vector_dim) + return F.upsample(p.unsqueeze(1).unsqueeze(3), + size=(target_len, 1), + mode='bilinear').squeeze(3).squeeze(1) + + def two_params_diff_fast(self, trained_p: torch.Tensor, ref_p: torch.Tensor, + index: torch.Tensor, + split_size: int): + + assert trained_p.dim() == ref_p.dim() + assert index.size(0) == trained_p.size(0) and index.size(1) == ref_p.size(0) + + # print(trained_p.size(), ref_p.size(), index.size()) + + ref_p = ref_p.detach() + if trained_p.dim() > 1: + trained_p = trained_p.flatten(1) + ref_p = ref_p.flatten(1) + + # the weight size of master DNN and foundation model may be totally different + + # MD -> FM: upsample first + # FM -> MD: downsample first + if trained_p.size(1) < ref_p.size(1): + trained_p = self.upsample_2d_tensor(trained_p, ref_p.size(1)) + + index = index.unsqueeze(-1) + # linear_combed_ref_p = (ref_p.unsqueeze(0) * index).sum(1) + # else: + + # print(trained_p.size(), ref_p.size(), index.size()) + + if split_size is None: + # old version: huge memory consumption, not recommended (although this is fastest) + # print('old version') + linear_combed_ref_p = (ref_p.unsqueeze(0) * index).sum(1) + + else: + # new version + linear_combed_ref_p = 0 + + cur_split_size = split_size + while index.size(1) % cur_split_size != 0: + cur_split_size -= 1 + # print(cur_split_size) + + for i in range(0, index.size(1), cur_split_size): + # if not isinstance(linear_combed_ref_p, int): + # print(linear_combed_ref_p.size(), ref_p.unsqueeze(0)[:, i: i + cur_split_size].size(), index[:, i: i + cur_split_size].size()) + linear_combed_ref_p += ref_p.unsqueeze(0)[:, i: i + cur_split_size] * index[:, i: i + cur_split_size] + linear_combed_ref_p = linear_combed_ref_p.sum(1) + + diff = (linear_combed_ref_p - trained_p).norm(2) ** 2 + return diff + + def get_index_loss(self, fm, md, indexes, match_fn, split_size): + res = 0. + + for name, p in md.named_parameters(): + if p.dim() == 0: + continue + + raw_p = match_fn(name, fm) + if raw_p is None: + continue + + index = indexes[name] + + # print(name) + res += self.two_params_diff_fast(p, raw_p, index, split_size) + return res + + def bn_cal(self, model: nn.Module, train_loader, num_iters, device): + has_bn = False + for n, m in model.named_modules(): + if isinstance(m, nn.BatchNorm2d): + has_bn = True + break + + if not has_bn: + return {} + + def bn_calibration_init(m): + """ calculating post-statistics of batch normalization """ + if getattr(m, 'track_running_stats', False): + # reset all values for post-statistics + m.reset_running_stats() + # set bn in training mode to update post-statistics + m.training = True + + with torch.no_grad(): + model.eval() + model.apply(bn_calibration_init) + for _ in range(num_iters): + x, _ = next(train_loader) + model(x.to(device)) + model.eval() + + bn_stats = {} + for n, m in model.named_modules(): + if isinstance(m, nn.BatchNorm2d): + bn_stats[n] = m + return bn_stats + + def run(self, scenario: Scenario, hyps: Dict) -> Dict[str, Any]: + super().run(scenario, hyps) + + # sanity check + # a= torch.tensor([[1, 2, 3], [1, 2, 4]]) + # index = torch.tensor([[1, 2, 3], + # [1, 2, 4]]) + # b = torch.tensor([[1, 2, 3], [1, 2, 4], [2, 3, 4]]) + # print(self.two_params_diff_fast(a, b, index, hyps['index_guided_linear_comb_split_size'])) + + assert isinstance(self.models['md'], ElasticDNN_OfflineMDModel) # for auto completion + assert isinstance(self.models['fm'], ElasticDNN_OfflineFMModel) # for auto completion + + # 1. add FBS + device = self.models['md'].device + + if self.models['md'].models_dict['main'] == -1: + logger.info(f'init master DNN by reducing width of an adapted foundation model (already tuned by LoRA)...') + + before_fm_model = deepcopy(self.models['fm'].models_dict['main']) + lora_util = self.models['fm'].get_lora_util() + lora_absorbed_fm_model = lora_util.absorb_lora_and_recover_net_structure(self.models['fm'].models_dict['main'], + torch.rand(hyps['samples_size']).to(device)) + self.models['fm'].models_dict['main'] = lora_absorbed_fm_model + master_dnn = self.models['fm'].generate_md_by_reducing_width(hyps['generate_md_width_ratio'], + torch.rand(hyps['samples_size']).to(device)) + self.models['fm'].models_dict['main'] = before_fm_model + + elastic_dnn_util = self.models['fm'].get_elastic_dnn_util() + master_dnn = elastic_dnn_util.convert_raw_dnn_to_master_dnn_with_perf_test(master_dnn, + hyps['FBS_r'], hyps['FBS_ignore_layers']) + self.models['md'].models_dict['main'] = master_dnn + self.models['md'].to(device) + + # 2. train (knowledge distillation, index relationship) + offline_datasets = scenario.get_offline_datasets() + train_dataset = MergedDataset([d['train'] for d in offline_datasets.values()]) + val_dataset = MergedDataset([d['val'] for d in offline_datasets.values()]) + train_loader = iter(build_dataloader(train_dataset, hyps['train_batch_size'], hyps['num_workers'], + True, None)) + val_loader = build_dataloader(val_dataset, hyps['val_batch_size'], hyps['num_workers'], + False, False) + + # 2.1 train whole master DNN (knowledge distillation) + for p in master_dnn.parameters(): + p.requires_grad = True + self.models['md'].to_train_mode() + + optimizer = torch.optim.__dict__[hyps['optimizer']]([ + {'params': self.models['md'].models_dict['main'].parameters(), **hyps['md_optimizer_args']} + ]) + scheduler = torch.optim.lr_scheduler.__dict__[hyps['scheduler']](optimizer, **hyps['scheduler_args']) + tb_writer = create_tbwriter(os.path.join(self.res_save_dir, 'tb_log'), launch_tbboard=hyps['launch_tbboard']) + pbar = tqdm.tqdm(range(hyps['num_iters']), dynamic_ncols=True) + best_avg_val_acc = 0. + + md_output_hook = None + + for iter_index in pbar: + self.models['md'].to_train_mode() + self.models['fm'].to_eval_mode() + + rand_sparsity = random.random() * (hyps['max_sparsity'] - hyps['min_sparsity']) + hyps['min_sparsity'] + elastic_dnn_util.set_master_dnn_sparsity(self.models['md'].models_dict['main'], rand_sparsity) + + x, y = next(train_loader) + x, y = x.to(device), y.to(device) + + with torch.no_grad(): + fm_output = self.models['fm'].infer(x) + + if md_output_hook is None: + md_output_hook = LayerActivation(self.models['md'].models_dict['main'], False, device) + task_loss = self.models['md'].forward_to_get_task_loss(x, y) + md_output = md_output_hook.output + distill_loss = hyps['distill_loss_weight'] * self.models['md'].get_distill_loss(md_output, fm_output) + total_loss = task_loss + distill_loss + + optimizer.zero_grad() + total_loss.backward() + optimizer.step() + scheduler.step() + + if (iter_index + 1) % hyps['val_freq'] == 0: + + elastic_dnn_util.clear_cached_channel_attention_in_master_dnn(self.models['md'].models_dict['main']) + md_output_hook.remove() + md_output_hook = None + + cur_md = self.models['md'].models_dict['main'] + md_for_test = deepcopy(self.models['md'].models_dict['main']) + val_accs = {} + avg_val_acc = 0. + bn_stats = {} + + for val_sparsity in np.linspace(hyps['min_sparsity'], hyps['max_sparsity'], num=hyps['val_num_sparsities']): + elastic_dnn_util.set_master_dnn_sparsity(md_for_test, val_sparsity) + bn_stats[f'{val_sparsity:.4f}'] = self.bn_cal(md_for_test, train_loader, hyps['bn_cal_num_iters'], device) + self.models['md'].models_dict['main'] = md_for_test + self.models['md'].to_eval_mode() + val_acc = self.models['md'].get_accuracy(val_loader) + + val_accs[f'{val_sparsity:.4f}'] = val_acc + avg_val_acc += val_acc + + avg_val_acc /= hyps['val_num_sparsities'] + + self.models['md'].models_dict['main'] = cur_md + self.models['md'].models_dict['bn_stats'] = bn_stats + + self.models['md'].save_model(os.path.join(self.res_save_dir, 'models/md_last.pt')) + self.models['fm'].save_model(os.path.join(self.res_save_dir, 'models/fm_last.pt')) + + if avg_val_acc > best_avg_val_acc: + best_avg_val_acc = avg_val_acc + self.models['md'].save_model(os.path.join(self.res_save_dir, 'models/md_best.pt')) + self.models['fm'].save_model(os.path.join(self.res_save_dir, 'models/fm_best.pt')) + + tb_writer.add_scalars(f'losses', dict(task=task_loss, distill=distill_loss, total=total_loss), iter_index) + pbar.set_description(f'loss: {total_loss:.6f}') + if (iter_index + 1) >= hyps['val_freq']: + tb_writer.add_scalars(f'accs/val_accs', val_accs, iter_index) + tb_writer.add_scalar(f'accs/avg_val_acc', avg_val_acc, iter_index) + pbar.set_description(f'loss: {total_loss:.6f}, avg_val_acc: {avg_val_acc:.4f}') + \ No newline at end of file diff --git a/methods/elasticdnn/api/algs/md_pretraining_index.py b/methods/elasticdnn/api/algs/md_pretraining_index.py new file mode 100644 index 0000000000000000000000000000000000000000..a9f71de66b096d8c341a1fd97214803e3f1a137b --- /dev/null +++ b/methods/elasticdnn/api/algs/md_pretraining_index.py @@ -0,0 +1,322 @@ +from typing import Any, Dict +from schema import Schema, Or +import schema +from data import Scenario, MergedDataset +from methods.base.alg import BaseAlg +from data import build_dataloader +from ..model import ElasticDNN_OfflineFMModel, ElasticDNN_OfflineMDModel +from ...model.base import ElasticDNNUtil +import torch.optim +import tqdm +import torch.nn.functional as F +from torch import nn +from utils.dl.common.env import create_tbwriter +import os +import random +import numpy as np +from copy import deepcopy +from utils.dl.common.model import LayerActivation, get_module +from utils.common.log import logger +import matplotlib.pyplot as plt + + +class ElasticDNN_MDPretrainingIndexAlg(BaseAlg): + """ + construct indexes between a filter/row of MD and all filters/rows of FM in the same layer + too huge indexes (~1GB), train so slow, hard to optimize + """ + def get_required_models_schema(self) -> Schema: + return Schema({ + 'fm': ElasticDNN_OfflineFMModel, + 'md': ElasticDNN_OfflineMDModel + }) + + def get_required_hyp_schema(self) -> Schema: + return Schema({ + 'launch_tbboard': bool, + + 'samples_size': (int, int, int, int), + + 'train_batch_size': int, + 'val_batch_size': int, + 'num_workers': int, + 'optimizer': str, + 'indexes_optimizer_args': dict, + 'scheduler': str, + 'scheduler_args': dict, + 'num_iters': int, + 'val_freq': int, + 'index_loss_l1_weight': float, + 'val_num_sparsities': int, + + 'bn_cal_num_iters': int, + + 'index_guided_linear_comb_split_size': Or(int, None) + }) + + def upsample_2d_tensor(self, p: torch.Tensor, target_len: int): + assert p.dim() == 2 # regard 2d weight as (batch_size, 1d_vector_dim) + return F.upsample(p.unsqueeze(1).unsqueeze(3), + size=(target_len, 1), + mode='bilinear').squeeze(3).squeeze(1) + + def two_params_diff_fast(self, trained_p: torch.Tensor, ref_p: torch.Tensor, + index: torch.Tensor, + split_size: int): + + assert trained_p.dim() == ref_p.dim() + assert index.size(0) == trained_p.size(0) and index.size(1) == ref_p.size(0) + + # print(trained_p.size(), ref_p.size(), index.size()) + + ref_p = ref_p.detach() + if trained_p.dim() > 1: + trained_p = trained_p.flatten(1) + ref_p = ref_p.flatten(1) + + # the weight size of master DNN and foundation model may be totally different + + # MD -> FM: upsample first + # FM -> MD: downsample first + if trained_p.size(1) < ref_p.size(1): + trained_p = self.upsample_2d_tensor(trained_p, ref_p.size(1)) + + index = index.unsqueeze(-1) + # linear_combed_ref_p = (ref_p.unsqueeze(0) * index).sum(1) + # else: + + # print(trained_p.size(), ref_p.size(), index.size()) + + if split_size is None: + # old version: huge memory consumption, not recommended (although this is fastest) + # print('old version') + linear_combed_ref_p = (ref_p.unsqueeze(0) * index).sum(1) + + else: + # new version + linear_combed_ref_p = 0 + + cur_split_size = split_size + while index.size(1) % cur_split_size != 0: + cur_split_size -= 1 + # print(cur_split_size) + + for i in range(0, index.size(1), cur_split_size): + # if not isinstance(linear_combed_ref_p, int): + # print(linear_combed_ref_p.size(), ref_p.unsqueeze(0)[:, i: i + cur_split_size].size(), index[:, i: i + cur_split_size].size()) + linear_combed_ref_p += ref_p.unsqueeze(0)[:, i: i + cur_split_size] * index[:, i: i + cur_split_size] + linear_combed_ref_p = linear_combed_ref_p.sum(1) + + diff = (linear_combed_ref_p - trained_p).norm(2) ** 2 + return diff + + def get_index_loss(self, fm, md, indexes, match_fn, split_size): + res = 0. + + for name, p in md.named_parameters(): + if name not in indexes.keys(): + continue + # if p.dim() == 0: + # continue + + raw_p = match_fn(name, fm) + # if raw_p is None: + # continue + + index = indexes[name] + + # print(name) + res += self.two_params_diff_fast(p, raw_p, index, split_size) + return res + + def bn_cal(self, model: nn.Module, train_loader, num_iters, device): + has_bn = False + for n, m in model.named_modules(): + if isinstance(m, nn.BatchNorm2d): + has_bn = True + break + + if not has_bn: + return {} + + def bn_calibration_init(m): + """ calculating post-statistics of batch normalization """ + if getattr(m, 'track_running_stats', False): + # reset all values for post-statistics + m.reset_running_stats() + # set bn in training mode to update post-statistics + m.training = True + + with torch.no_grad(): + model.eval() + model.apply(bn_calibration_init) + for _ in range(num_iters): + x, _ = next(train_loader) + model(x.to(device)) + model.eval() + + bn_stats = {} + for n, m in model.named_modules(): + if isinstance(m, nn.BatchNorm2d): + bn_stats[n] = m + return bn_stats + + def run(self, scenario: Scenario, hyps: Dict) -> Dict[str, Any]: + super().run(scenario, hyps) + + # sanity check + # a= torch.tensor([[1, 2, 3], [1, 2, 4]]) + # index = torch.tensor([[1, 2, 3], + # [1, 2, 4]]) + # b = torch.tensor([[1, 2, 3], [1, 2, 4], [2, 3, 4]]) + # print(self.two_params_diff_fast(a, b, index, hyps['index_guided_linear_comb_split_size'])) + + assert isinstance(self.models['md'], ElasticDNN_OfflineMDModel) # for auto completion + assert isinstance(self.models['fm'], ElasticDNN_OfflineFMModel) # for auto completion + + # 1. add FBS + device = self.models['md'].device + + # logger.info(f'init master DNN by reducing width of an adapted foundation model (already tuned by LoRA)...') + + # before_fm_model = deepcopy(self.models['fm'].models_dict['main']) + # lora_util = self.models['fm'].get_lora_util() + # lora_absorbed_fm_model = lora_util.absorb_lora_and_recover_net_structure(self.models['fm'].models_dict['main'], + # torch.rand(hyps['samples_size']).to(device)) + # self.models['fm'].models_dict['main'] = lora_absorbed_fm_model + # master_dnn = self.models['fm'].generate_md_by_reducing_width(hyps['generate_md_width_ratio'], + # torch.rand(hyps['samples_size']).to(device)) + # self.models['fm'].models_dict['main'] = before_fm_model + + elastic_dnn_util = self.models['fm'].get_elastic_dnn_util() + # master_dnn = elastic_dnn_util.convert_raw_dnn_to_master_dnn_with_perf_test(master_dnn, + # hyps['FBS_r'], hyps['FBS_ignore_layers']) + # self.models['md'].models_dict['main'] = master_dnn + # self.models['md'].to(device) + # master_dnn = self.models['md'].models_dict['main'] + + # 2. train (knowledge distillation, index relationship) + offline_datasets = scenario.get_offline_datasets() + train_dataset = MergedDataset([d['train'] for d in offline_datasets.values()]) + val_dataset = MergedDataset([d['val'] for d in offline_datasets.values()]) + train_loader = iter(build_dataloader(train_dataset, hyps['train_batch_size'], hyps['num_workers'], + True, None)) + val_loader = build_dataloader(val_dataset, hyps['val_batch_size'], hyps['num_workers'], + False, False) + + # 2.1 train only FBS (skipped because current md cannot do proper inference) + + # 2.2 train whole master DNN (knowledge distillation, index relationship) + # for p in master_dnn.parameters(): + # p.requires_grad = True + # self.models['md'].to_train_mode() + + indexes = {} + for name, p in self.models['md'].models_dict['main'].named_parameters(): + if p.dim() > 1: + matched_p_in_fm = self.models['md'].get_matched_param_of_fm(name, self.models['fm'].models_dict['main']) + if matched_p_in_fm is None: + continue + indexes[name] = torch.zeros((p.size(0), matched_p_in_fm.size(0))).to(device) + indexes[name].requires_grad = True + + logger.info(f'construct index in layer {name}') + + tmp_indexes_file_path = os.path.join(self.res_save_dir, 'tmp-indexes.pt') + torch.save(indexes, tmp_indexes_file_path) + logger.info(f'generate indexes ({(os.path.getsize(tmp_indexes_file_path) / 1024**2):.3f}MB)') + os.remove(tmp_indexes_file_path) + + optimizer = torch.optim.__dict__[hyps['optimizer']]([ + # {'params': self.models['md'].models_dict['main'].parameters(), **hyps['optimizer_args']}, + {'params': [v for v in indexes.values()], **hyps['indexes_optimizer_args']} + ]) + scheduler = torch.optim.lr_scheduler.__dict__[hyps['scheduler']](optimizer, **hyps['scheduler_args']) + tb_writer = create_tbwriter(os.path.join(self.res_save_dir, 'tb_log'), launch_tbboard=hyps['launch_tbboard']) + pbar = tqdm.tqdm(range(hyps['num_iters']), dynamic_ncols=True) + best_avg_val_acc = 0. + + for p in self.models['md'].models_dict['main'].parameters(): + p.requires_grad = False + for p in self.models['fm'].models_dict['main'].parameters(): + p.requires_grad = False + + for iter_index in pbar: + self.models['md'].to_eval_mode() + self.models['fm'].to_eval_mode() + + # rand_sparsity = random.random() * (hyps['max_sparsity'] - hyps['min_sparsity']) + hyps['min_sparsity'] + # elastic_dnn_util.set_master_dnn_sparsity(self.models['md'].models_dict['main'], rand_sparsity) + + # x, y = next(train_loader) + # x, y = x.to(device), y.to(device) + + # task_loss = self.models['md'].forward_to_get_task_loss(x, y) + # l1_reg_loss = hyps['l1_reg_loss_weight'] * elastic_dnn_util.get_accu_l1_reg_of_raw_channel_attention_in_master_dnn(master_dnn) + index_loss = self.get_index_loss(self.models['fm'].models_dict['main'], + self.models['md'].models_dict['main'], + indexes, + self.models['md'].get_matched_param_of_fm, + hyps['index_guided_linear_comb_split_size']) + index_l1_loss = hyps['index_loss_l1_weight'] * torch.FloatTensor([v.abs().sum() for v in indexes.values()]).sum() + + total_loss = index_loss + index_l1_loss + + optimizer.zero_grad() + total_loss.backward() + optimizer.step() + scheduler.step() + + if (iter_index + 1) % (10) == 0: + # visualize indexes histgoram + # do not use add_histogram because indexes is huge + os.makedirs(os.path.join(self.res_save_dir, 'index_hist'), exist_ok=True) + with torch.no_grad(): + for p_name, index in indexes.items(): + index_hist = index.view(-1).histc(bins=20).detach().cpu().numpy() + plt.bar(list(range(20)), index_hist) + plt.savefig(os.path.join(self.res_save_dir, f'index_hist/{p_name}.png')) + plt.clf() + + if (iter_index + 1) % hyps['val_freq'] == 0: + + elastic_dnn_util.clear_cached_channel_attention_in_master_dnn(self.models['md'].models_dict['main']) + + cur_md = self.models['md'].models_dict['main'] + md_for_test = deepcopy(self.models['md'].models_dict['main']) + val_accs = {} + avg_val_acc = 0. + bn_stats = {} + + for val_sparsity in [0.0, 0.2, 0.4, 0.8]: + elastic_dnn_util.set_master_dnn_sparsity(md_for_test, val_sparsity) + bn_stats[f'{val_sparsity:.4f}'] = self.bn_cal(md_for_test, train_loader, hyps['bn_cal_num_iters'], device) + self.models['md'].models_dict['main'] = md_for_test + self.models['md'].to_eval_mode() + val_acc = self.models['md'].get_accuracy(val_loader) + + val_accs[f'{val_sparsity:.4f}'] = val_acc + avg_val_acc += val_acc + avg_val_acc /= hyps['val_num_sparsities'] + + self.models['md'].models_dict['main'] = cur_md + self.models['md'].models_dict['indexes'] = indexes + self.models['md'].models_dict['bn_stats'] = bn_stats + self.models['fm'].models_dict['indexes'] = indexes + + self.models['md'].save_model(os.path.join(self.res_save_dir, 'models/md_last.pt')) + self.models['fm'].save_model(os.path.join(self.res_save_dir, 'models/fm_last.pt')) + + if avg_val_acc > best_avg_val_acc: + best_avg_val_acc = avg_val_acc + self.models['md'].save_model(os.path.join(self.res_save_dir, 'models/md_best.pt')) + self.models['fm'].save_model(os.path.join(self.res_save_dir, 'models/fm_best.pt')) + + tb_writer.add_scalars(f'losses', dict(index=index_loss, index_l1=index_l1_loss, total=total_loss), iter_index) + pbar.set_description(f'loss: {total_loss:.6f}, index_loss: {index_loss:.6f}, index_l1_loss: {index_l1_loss:.6f}') + if (iter_index + 1) >= hyps['val_freq']: + tb_writer.add_scalars(f'accs/val_accs', val_accs, iter_index) + tb_writer.add_scalar(f'accs/avg_val_acc', avg_val_acc, iter_index) + pbar.set_description(f'loss: {total_loss:.6f}, index_loss: {index_loss:.6f}, index_l1_loss: {index_l1_loss:.6f}, ' + f'avg_val_acc: {avg_val_acc:.4f}') + \ No newline at end of file diff --git a/methods/elasticdnn/api/algs/md_pretraining_index_bk_too_slow.py b/methods/elasticdnn/api/algs/md_pretraining_index_bk_too_slow.py new file mode 100644 index 0000000000000000000000000000000000000000..8bf1f57d5a0ff71f862a2c7d0aadddfeb41da84d --- /dev/null +++ b/methods/elasticdnn/api/algs/md_pretraining_index_bk_too_slow.py @@ -0,0 +1,427 @@ +from typing import Any, Dict +from schema import Schema, Or +import schema +from data import Scenario, MergedDataset +from methods.base.alg import BaseAlg +from data import build_dataloader +from ..model import ElasticDNN_OfflineFMModel, ElasticDNN_OfflineMDModel +from ...model.base import ElasticDNNUtil +import torch.optim +import tqdm +import torch.nn.functional as F +from torch import nn +from utils.dl.common.env import create_tbwriter +import os +import random +import numpy as np +from copy import deepcopy +from utils.dl.common.model import LayerActivation, get_module +from utils.common.log import logger + + +class ElasticDNN_MDPretrainingIndexAlg(BaseAlg): + """ + TODO: fine-tuned FM -> init MD -> trained MD -> construct indexes (only between similar weights) and fine-tune + """ + def get_required_models_schema(self) -> Schema: + return Schema({ + 'fm': ElasticDNN_OfflineFMModel, + 'md': ElasticDNN_OfflineMDModel + }) + + def get_required_hyp_schema(self) -> Schema: + return Schema({ + 'launch_tbboard': bool, + + 'samples_size': (int, int, int, int), + + 'train_batch_size': int, + 'val_batch_size': int, + 'num_workers': int, + 'optimizer': str, + 'optimizer_args': dict, + 'index_optimizer_args': dict, + 'scheduler': str, + 'scheduler_args': dict, + 'num_iters': int, + 'val_freq': int, + 'max_sparsity': float, + 'min_sparsity': float, + 'l1_reg_loss_weight': float, + 'index_loss_weight': float, + 'val_num_sparsities': int, + + 'bn_cal_num_iters': int, + + 'index_1_to_k': int + }) + + def bn_cal(self, model: nn.Module, train_loader, num_iters, device): + has_bn = False + for n, m in model.named_modules(): + if isinstance(m, nn.BatchNorm2d): + has_bn = True + break + + if not has_bn: + return {} + + def bn_calibration_init(m): + """ calculating post-statistics of batch normalization """ + if getattr(m, 'track_running_stats', False): + # reset all values for post-statistics + m.reset_running_stats() + # set bn in training mode to update post-statistics + m.training = True + + with torch.no_grad(): + model.eval() + model.apply(bn_calibration_init) + for _ in range(num_iters): + x, _ = next(train_loader) + model(x.to(device)) + model.eval() + + bn_stats = {} + for n, m in model.named_modules(): + if isinstance(m, nn.BatchNorm2d): + bn_stats[n] = m + return bn_stats + + def upsample_2d_tensor(self, p: torch.Tensor, target_len: int): + assert p.dim() == 2 # regard 2d weight as (batch_size, 1d_vector_dim) + return F.upsample(p.unsqueeze(1).unsqueeze(3), + size=(target_len, 1), + mode='bilinear').squeeze(3).squeeze(1) + + def two_params_diff_fast(self, trained_p: torch.Tensor, ref_p: torch.Tensor, + index: torch.Tensor, + split_size: int): + + assert trained_p.dim() == ref_p.dim() + assert index.size(0) == trained_p.size(0) and index.size(1) == ref_p.size(0) + + # print(trained_p.size(), ref_p.size(), index.size()) + + ref_p = ref_p.detach() + if trained_p.dim() > 1: + trained_p = trained_p.flatten(1) + ref_p = ref_p.flatten(1) + + # the weight size of master DNN and foundation model may be totally different + + # MD -> FM: upsample first + # FM -> MD: downsample first + if trained_p.size(1) < ref_p.size(1): + trained_p = self.upsample_2d_tensor(trained_p, ref_p.size(1)) + + index = index.unsqueeze(-1) + # linear_combed_ref_p = (ref_p.unsqueeze(0) * index).sum(1) + # else: + + # print(trained_p.size(), ref_p.size(), index.size()) + + if split_size is None: + # old version: huge memory consumption, not recommended (although this is fastest) + # print('old version') + linear_combed_ref_p = (ref_p.unsqueeze(0) * index).sum(1) + + else: + # new version + linear_combed_ref_p = 0 + + cur_split_size = split_size + while index.size(1) % cur_split_size != 0: + cur_split_size -= 1 + # print(cur_split_size) + + for i in range(0, index.size(1), cur_split_size): + # if not isinstance(linear_combed_ref_p, int): + # print(linear_combed_ref_p.size(), ref_p.unsqueeze(0)[:, i: i + cur_split_size].size(), index[:, i: i + cur_split_size].size()) + linear_combed_ref_p += ref_p.unsqueeze(0)[:, i: i + cur_split_size] * index[:, i: i + cur_split_size] + linear_combed_ref_p = linear_combed_ref_p.sum(1) + + diff = (linear_combed_ref_p - trained_p).norm(2) ** 2 + return diff + + def two_params_partial_diff(self, trained_p: torch.Tensor, ref_p: torch.Tensor, + index): + + assert trained_p.dim() == ref_p.dim() + # assert index.size(0) == trained_p.size(0) and index.size(1) == ref_p.size(0) + + # print(trained_p.size(), ref_p.size(), index.size()) + + ref_p = ref_p.detach() + if trained_p.dim() > 1: + trained_p = trained_p.flatten(1) + ref_p = ref_p.flatten(1) + + # the weight size of master DNN and foundation model may be totally different + + # MD -> FM: upsample first + # FM -> MD: downsample first + if trained_p.size(1) < ref_p.size(1): + trained_p = self.upsample_2d_tensor(trained_p, ref_p.size(1)) + + elif trained_p.dim() == 1: + trained_p = trained_p.unsqueeze(1) + ref_p = ref_p.unsqueeze(1) + + # index = index.unsqueeze(-1) + # linear_combed_ref_p = (ref_p.unsqueeze(0) * index).sum(1) + # else: + + # res = 0. + # # slow version + # for g1i, (g1, (_index, selected_g2is)) in enumerate(zip(trained_p, index)): + # comb_ref_p = (ref_p[torch.tensor(selected_g2is)] * _index.unsqueeze(1)).sum(0) + # res += ((comb_ref_p - g1) ** 2).sum() + # return res + # ------------- + + # (train_p.size(0), 2, ref_p.size(1)) + # if trained_p.dim() == 2: + + # NOTE: fast version? + selected_ref_p = torch.stack([ref_p[torch.tensor(selected_g2is)] for _, selected_g2is in index]) + # (train_p.size(0), 2) + indexes = torch.stack([_index for _index, _ in index]) + + # print(trained_p.size(), ref_p.size(), selected_ref_p.size(), indexes.size()) + + # (train_p.size(),) + # should be (train_p.size(0), ref_p.size(1)) + linear_combed_ref_p = (selected_ref_p * indexes.unsqueeze(-1)).sum(1) + # ------------- + + # else: + # selected_ref_p = torch.stack([ref_p[torch.tensor(selected_g2is)] for _, selected_g2is in index]) + # # (train_p.size(0), 2) + # indexes = torch.stack([_index for _index, _ in index]) + + # # print(trained_p.size(), ref_p.size(), selected_ref_p.size(), indexes.size()) + + # # (train_p.size(),) + # # should be (train_p.size(0), ref_p.size(1)) + # linear_combed_ref_p = (selected_ref_p * indexes).sum(1) + + diff = (linear_combed_ref_p - trained_p).norm(2) ** 2 + return diff + + # print(trained_p.size(), ref_p.size(), index.size()) + + # if split_size is None: + # # old version: huge memory consumption, not recommended (although this is fastest) + # # print('old version') + # linear_combed_ref_p = (ref_p.unsqueeze(0) * index).sum(1) + + # else: + # # new version + # linear_combed_ref_p = 0 + + # cur_split_size = split_size + # while index.size(1) % cur_split_size != 0: + # cur_split_size -= 1 + # # print(cur_split_size) + + # for i in range(0, index.size(1), cur_split_size): + # # if not isinstance(linear_combed_ref_p, int): + # # print(linear_combed_ref_p.size(), ref_p.unsqueeze(0)[:, i: i + cur_split_size].size(), index[:, i: i + cur_split_size].size()) + # linear_combed_ref_p += ref_p.unsqueeze(0)[:, i: i + cur_split_size] * index[:, i: i + cur_split_size] + # linear_combed_ref_p = linear_combed_ref_p.sum(1) + + # diff = (linear_combed_ref_p - trained_p).norm(2) ** 2 + # return diff + + def get_index_loss(self, fm, md, indexes, match_fn): + res = 0. + + for name, p in md.named_parameters(): + if p.dim() == 0: + continue + + raw_p = match_fn(name, fm) + if raw_p is None: + continue + + index = indexes[name] + + # print(name) + res += self.two_params_partial_diff(p, raw_p, index) + return res + + def run(self, scenario: Scenario, hyps: Dict) -> Dict[str, Any]: + super().run(scenario, hyps) + + assert isinstance(self.models['md'], ElasticDNN_OfflineMDModel) # for auto completion + assert isinstance(self.models['fm'], ElasticDNN_OfflineFMModel) # for auto completion + + # 1. add FBS + device = self.models['md'].device + + # 2. train (knowledge distillation, index relationship) + offline_datasets = scenario.get_offline_datasets() + train_dataset = MergedDataset([d['train'] for d in offline_datasets.values()]) + val_dataset = MergedDataset([d['val'] for d in offline_datasets.values()]) + train_loader = iter(build_dataloader(train_dataset, hyps['train_batch_size'], hyps['num_workers'], + True, None)) + val_loader = build_dataloader(val_dataset, hyps['val_batch_size'], hyps['num_workers'], + False, False) + + # logger.info(f'init master DNN acc w/ FBS (before constructing indexes): {self.models["md"].get_accuracy(val_loader):.4f}') + + master_dnn = self.models['md'].models_dict['main'] + elastic_dnn_util = self.models['fm'].get_elastic_dnn_util() + + with torch.no_grad(): + indexes = {} + trained_indexes = [] + for name, p in self.models['md'].models_dict['main'].named_parameters(): + if p.dim() > 0: + matched_p_in_fm = self.models['md'].get_matched_param_of_fm(name, self.models['fm'].models_dict['main']) + if matched_p_in_fm is None: + continue + + if p.dim() > 1: + p = p.flatten(1) + matched_p_in_fm = matched_p_in_fm.flatten(1) + + if p.size(1) < matched_p_in_fm.size(1): + p = self.upsample_2d_tensor(p, matched_p_in_fm.size(1)) + + indexes[name] = [] + for g1i, g1 in enumerate(p): + selected_g2is = torch.randperm(matched_p_in_fm.size(0))[0: hyps['index_1_to_k']] + index = torch.FloatTensor([1. / hyps['index_1_to_k']] * hyps['index_1_to_k']).to(device) + indexes[name] += [(index, list(selected_g2is))] + trained_indexes += [index] + # for selected_g2i in selected_g2is: + # indexes[name][g1i][selected_g2i] = torch.FloatTensor([1. / hyps['index_1_to_k']]).to(device) + # indexes[name][g1i][selected_g2i].requires_grad = True + # indexes[name][g1i][selected_g2i] = 1. + # trained_indexes += [indexes[name][g1i][selected_g2i]] + + # print(p.size(), selected_g2is) + + # NOTE: only constructing indexes between similar [weight row/filter] pair + # for g1i, g1 in tqdm.tqdm(enumerate(p), dynamic_ncols=True, leave=False, total=len(p)): + # indexes[name][g1i] = {} + + # # similarities = [] + # # for g2i, g2 in enumerate(matched_p_in_fm): + # # similarity = ((g1 - g2) ** 2).sum() + # # similarities += [similarity] + # if p.dim() == 1: + # similarities = ((g1 - matched_p_in_fm) ** 2) + # else: + # similarities = ((g1.unsqueeze(0) - matched_p_in_fm) ** 2).sum(1) + # assert similarities.size(0) == matched_p_in_fm.size(0) + + # most_similar_g2is = similarities.argsort(descending=True) + # accu_similarities = similarities.sort(descending=True)[0].cumsum(0) + + # for ai, accu_sim in enumerate(accu_similarities): + # if accu_sim > accu_similarities[-1] * hyps['index_construct_r']: + # break + + # # selected fm weight rows for constructing indexes + # selected_g2is = most_similar_g2is[0: ai] + # for selected_g2i in selected_g2is: + # indexes[name][g1i][selected_g2i] = torch.FloatTensor([1. / ai]).to(device) + # indexes[name][g1i][selected_g2i].requires_grad = True + # trained_indexes += [indexes[name][g1i][selected_g2i]] + + # num_indexes += 1 + + # index_percent = num_indexes / (matched_p_in_fm.size(0) * p.size(0)) * 100. + # logger.info(f'layer {name}: constructing {index_percent:.3f}% of indexes') + + tmp_indexes_file_path = os.path.join(self.res_save_dir, 'tmp-indexes.pt') + torch.save(indexes, tmp_indexes_file_path) + logger.info(f'# indexes: {len(trained_indexes)}; generate indexes ({(os.path.getsize(tmp_indexes_file_path) / 1024**2):.3f}MB)') + os.remove(tmp_indexes_file_path) + + # 2.1 train whole master DNN (knowledge distillation) + for p in master_dnn.parameters(): + p.requires_grad = True + self.models['md'].to_train_mode() + for p in trained_indexes: + p.requires_grad = True + for p in self.models['fm'].models_dict['main'].parameters(): + p.requires_grad = False + + optimizer = torch.optim.__dict__[hyps['optimizer']]([ + {'params': self.models['md'].models_dict['main'].parameters(), **hyps['optimizer_args']}, + # {'params': trained_indexes, **hyps['index_optimizer_args']} + ]) + scheduler = torch.optim.lr_scheduler.__dict__[hyps['scheduler']](optimizer, **hyps['scheduler_args']) + tb_writer = create_tbwriter(os.path.join(self.res_save_dir, 'tb_log'), launch_tbboard=hyps['launch_tbboard']) + pbar = tqdm.tqdm(range(hyps['num_iters']), dynamic_ncols=True) + best_avg_val_acc = 0. + + for iter_index in pbar: + self.models['md'].to_train_mode() + self.models['fm'].to_eval_mode() + + rand_sparsity = random.random() * (hyps['max_sparsity'] - hyps['min_sparsity']) + hyps['min_sparsity'] + elastic_dnn_util.set_master_dnn_sparsity(self.models['md'].models_dict['main'], rand_sparsity) + + x, y = next(train_loader) + x, y = x.to(device), y.to(device) + + task_loss = self.models['md'].forward_to_get_task_loss(x, y) + l1_reg_loss = hyps['l1_reg_loss_weight'] * elastic_dnn_util.get_accu_l1_reg_of_raw_channel_attention_in_master_dnn(master_dnn) + index_loss = hyps['index_loss_weight'] * self.get_index_loss(self.models['fm'].models_dict['main'], + self.models['md'].models_dict['main'], + indexes, + self.models['md'].get_matched_param_of_fm) + total_loss = task_loss + l1_reg_loss + index_loss + + optimizer.zero_grad() + total_loss.backward() + optimizer.step() + scheduler.step() + + if (iter_index + 1) % hyps['val_freq'] == 0: + + elastic_dnn_util.clear_cached_channel_attention_in_master_dnn(self.models['md'].models_dict['main']) + + cur_md = self.models['md'].models_dict['main'] + md_for_test = deepcopy(self.models['md'].models_dict['main']) + val_accs = {} + avg_val_acc = 0. + bn_stats = {} + + for val_sparsity in np.linspace(hyps['min_sparsity'], hyps['max_sparsity'], num=hyps['val_num_sparsities']): + elastic_dnn_util.set_master_dnn_sparsity(md_for_test, val_sparsity) + bn_stats[f'{val_sparsity:.4f}'] = self.bn_cal(md_for_test, train_loader, hyps['bn_cal_num_iters'], device) + self.models['md'].models_dict['main'] = md_for_test + self.models['md'].to_eval_mode() + val_acc = self.models['md'].get_accuracy(val_loader) + + val_accs[f'{val_sparsity:.4f}'] = val_acc + avg_val_acc += val_acc + + avg_val_acc /= hyps['val_num_sparsities'] + + self.models['md'].models_dict['main'] = cur_md + self.models['md'].models_dict['bn_stats'] = bn_stats + + self.models['md'].save_model(os.path.join(self.res_save_dir, 'models/md_last.pt')) + self.models['fm'].save_model(os.path.join(self.res_save_dir, 'models/fm_last.pt')) + + if avg_val_acc > best_avg_val_acc: + best_avg_val_acc = avg_val_acc + self.models['md'].save_model(os.path.join(self.res_save_dir, 'models/md_best.pt')) + self.models['fm'].save_model(os.path.join(self.res_save_dir, 'models/fm_best.pt')) + + tb_writer.add_scalars(f'losses', dict(task=task_loss, l1_reg=l1_reg_loss, index=index_loss, total=total_loss), iter_index) + pbar.set_description(f'loss: {total_loss:.6f}, task_loss: {task_loss:.6f}, index_loss: {index_loss:.6f}, ' + f'l1_loss: {l1_reg_loss:.6f}') + if (iter_index + 1) >= hyps['val_freq']: + tb_writer.add_scalars(f'accs/val_accs', val_accs, iter_index) + tb_writer.add_scalar(f'accs/avg_val_acc', avg_val_acc, iter_index) + pbar.set_description(f'loss: {total_loss:.6f}, task_loss: {task_loss:.6f}, index_loss: {index_loss:.6f}, ' + f'l1_loss: {l1_reg_loss:.6f}, avg_val_acc: {avg_val_acc:.4f}') + \ No newline at end of file diff --git a/methods/elasticdnn/api/algs/md_pretraining_index_v1_train_index_and_md.py b/methods/elasticdnn/api/algs/md_pretraining_index_v1_train_index_and_md.py new file mode 100644 index 0000000000000000000000000000000000000000..3eddb2d243b14d3898c6212bcba1b4f94cd045a8 --- /dev/null +++ b/methods/elasticdnn/api/algs/md_pretraining_index_v1_train_index_and_md.py @@ -0,0 +1,501 @@ +from typing import Any, Dict +from schema import Schema, Or +import schema +from data import Scenario, MergedDataset +from methods.base.alg import BaseAlg +from data import build_dataloader +from ..model import ElasticDNN_OfflineFMModel, ElasticDNN_OfflineMDModel +from ...model.base import ElasticDNNUtil +import torch.optim +import tqdm +import torch.nn.functional as F +from torch import nn +from utils.dl.common.env import create_tbwriter +import os +import random +import numpy as np +from copy import deepcopy +from utils.dl.common.model import LayerActivation, get_module +from utils.common.log import logger + + +class WeightAffine(nn.Module): + def __init__(self, a, b, r=16): + super(WeightAffine, self).__init__() + + self.a = a + self.b = b + + self.w1 = nn.Parameter(torch.rand((b // r, a))) + self.w2 = nn.Parameter(torch.rand((b, b // r))) + + self.a_to_b = True + + nn.init.zeros_(self.w1.data) + nn.init.zeros_(self.w2.data) + + def forward1(self, x): + return F.linear(F.linear(x, self.w1), self.w2) + + def forward2(self, x): + return F.linear(F.linear(x, self.w2.T), self.w1.T) + + def forward(self, x): + return self.forward1(x) if self.a_to_b else self.forward2(x) + + +class ElasticDNN_MDPretrainingIndexAlg(BaseAlg): + """ + construct indexes between a filter/row of MD and all filters/rows of FM in the same layer + too huge indexes (~1GB), train so slow, hard to optimize + """ + def get_required_models_schema(self) -> Schema: + return Schema({ + 'fm': ElasticDNN_OfflineFMModel, + 'md': ElasticDNN_OfflineMDModel + }) + + def get_required_hyp_schema(self) -> Schema: + return Schema({ + 'launch_tbboard': bool, + + 'samples_size': (int, int, int, int), + + 'train_batch_size': int, + 'val_batch_size': int, + 'num_workers': int, + 'optimizer': str, + 'optimizer_args': dict, + 'indexes_optimizer_args': dict, + 'scheduler': str, + 'scheduler_args': dict, + 'num_iters': int, + 'val_freq': int, + 'max_sparsity': float, + 'min_sparsity': float, + # 'l1_reg_loss_weight': float, + 'index_loss_weight': float, + 'val_num_sparsities': int, + + 'bn_cal_num_iters': int, + + 'index_init': str, + 'index_guided_linear_comb_split_size': Or(int, None) + }) + + # def upsample_2d_tensor(self, p: torch.Tensor, target_len: int, weight_affine: WeightAffine): + # assert p.dim() == 2 # regard 2d weight as (batch_size, 1d_vector_dim) + # # return F.upsample(p.unsqueeze(1).unsqueeze(3), + # # size=(target_len, 1), + # # mode='bilinear').squeeze(3).squeeze(1) + # weight_affine.a_to_b = True + # return weight_affine(p) + + # def downsample_2d_tensor(self, p: torch.Tensor, target_len: int, weight_affine: WeightAffine): + # assert p.dim() == 2 # regard 2d weight as (batch_size, 1d_vector_dim) + # # return F.interpolate(p.unsqueeze(1).unsqueeze(3), + # # size=(target_len, 1), + # # mode='bilinear').squeeze(3).squeeze(1) + # weight_affine.a_to_b = False + # return weight_affine(p) + + def two_params_diff_fast_no_sample(self, trained_p: torch.Tensor, ref_p: torch.Tensor, + index: torch.Tensor, + split_size: int): + + assert trained_p.dim() == ref_p.dim() + + if trained_p.dim() > 1 and index.size(0) == trained_p.size(1) and index.size(1) == ref_p.size(1): + assert trained_p.dim() == 2 + trained_p = trained_p.T + ref_p = ref_p.T + + assert index.size(0) == trained_p.size(0) and index.size(1) == ref_p.size(0) + + # print(trained_p.size(), ref_p.size(), index.size()) + + ref_p = ref_p.detach() + if trained_p.dim() > 1: + trained_p = trained_p.flatten(1) + ref_p = ref_p.flatten(1) + + # the weight size of master DNN and foundation model may be totally different + + # MD -> FM: upsample first + # FM -> MD: downsample first + # if trained_p.size(1) < ref_p.size(1): + # trained_p = self.upsample_2d_tensor(trained_p, ref_p.size(1), weight_affine) + + index = index.unsqueeze(-1) + # linear_combed_ref_p = (ref_p.unsqueeze(0) * index).sum(1) + # else: + + # print(trained_p.size(), ref_p.size(), index.size()) + + if split_size is None: + # old version: huge memory consumption, not recommended (although this is fastest) + # print('old version') + linear_combed_ref_p = (ref_p.unsqueeze(0) * index).sum(1) + + else: + # new version + linear_combed_ref_p = 0 + + cur_split_size = split_size + while index.size(1) % cur_split_size != 0: + cur_split_size -= 1 + # print(cur_split_size) + + for i in range(0, index.size(1), cur_split_size): + # if not isinstance(linear_combed_ref_p, int): + # print(linear_combed_ref_p.size(), ref_p.unsqueeze(0)[:, i: i + cur_split_size].size(), index[:, i: i + cur_split_size].size()) + linear_combed_ref_p += ref_p.unsqueeze(0)[:, i: i + cur_split_size] * index[:, i: i + cur_split_size] + linear_combed_ref_p = linear_combed_ref_p.sum(1) + + diff = (linear_combed_ref_p - trained_p).norm(2) ** 2 + return diff + + # def two_params_diff_fast_including_upsample(self, trained_p: torch.Tensor, ref_p: torch.Tensor, + # index: torch.Tensor, + # split_size: int, weight_affine: WeightAffine): + + # assert trained_p.dim() == ref_p.dim() + # assert index.size(0) == trained_p.size(0) and index.size(1) == ref_p.size(0) + + # # print(trained_p.size(), ref_p.size(), index.size()) + + # ref_p = ref_p.detach() + # if trained_p.dim() > 1: + # trained_p = trained_p.flatten(1) + # ref_p = ref_p.flatten(1) + + # # the weight size of master DNN and foundation model may be totally different + + # # MD -> FM: upsample first + # # FM -> MD: downsample first + # if trained_p.size(1) < ref_p.size(1): + # trained_p = self.upsample_2d_tensor(trained_p, ref_p.size(1), weight_affine) + + # index = index.unsqueeze(-1) + # # linear_combed_ref_p = (ref_p.unsqueeze(0) * index).sum(1) + # # else: + + # # print(trained_p.size(), ref_p.size(), index.size()) + + # if split_size is None: + # # old version: huge memory consumption, not recommended (although this is fastest) + # # print('old version') + # linear_combed_ref_p = (ref_p.unsqueeze(0) * index).sum(1) + + # else: + # # new version + # linear_combed_ref_p = 0 + + # cur_split_size = split_size + # while index.size(1) % cur_split_size != 0: + # cur_split_size -= 1 + # # print(cur_split_size) + + # for i in range(0, index.size(1), cur_split_size): + # # if not isinstance(linear_combed_ref_p, int): + # # print(linear_combed_ref_p.size(), ref_p.unsqueeze(0)[:, i: i + cur_split_size].size(), index[:, i: i + cur_split_size].size()) + # linear_combed_ref_p += ref_p.unsqueeze(0)[:, i: i + cur_split_size] * index[:, i: i + cur_split_size] + # linear_combed_ref_p = linear_combed_ref_p.sum(1) + + # diff = (linear_combed_ref_p - trained_p).norm(2) ** 2 + # return diff + + # def two_params_diff_fast_including_downsample(self, trained_p: torch.Tensor, ref_p: torch.Tensor, + # index: torch.Tensor, + # split_size: int, weight_affine: WeightAffine): + + # assert trained_p.dim() == ref_p.dim() + # assert index.size(0) == trained_p.size(0) and index.size(1) == ref_p.size(0) + + # # print(trained_p.size(), ref_p.size(), index.size()) + + # ref_p = ref_p.detach() + # if trained_p.dim() > 1: + # trained_p = trained_p.flatten(1) + # ref_p = ref_p.flatten(1) + + # # the weight size of master DNN and foundation model may be totally different + + # # MD -> FM: upsample first + # # FM -> MD: downsample first + # if trained_p.size(1) < ref_p.size(1): + # # trained_p = self.upsample_2d_tensor(trained_p, ref_p.size(1)) + # ref_p = self.downsample_2d_tensor(ref_p, trained_p.size(1), weight_affine) + + # index = index.unsqueeze(-1) + # # linear_combed_ref_p = (ref_p.unsqueeze(0) * index).sum(1) + # # else: + + # # print(trained_p.size(), ref_p.size(), index.size()) + + # if split_size is None: + # # old version: huge memory consumption, not recommended (although this is fastest) + # # print('old version') + # linear_combed_ref_p = (ref_p.unsqueeze(0) * index).sum(1) + + # else: + # # new version + # linear_combed_ref_p = 0 + + # cur_split_size = split_size + # while index.size(1) % cur_split_size != 0: + # cur_split_size -= 1 + # # print(cur_split_size) + + # for i in range(0, index.size(1), cur_split_size): + # # if not isinstance(linear_combed_ref_p, int): + # # print(linear_combed_ref_p.size(), ref_p.unsqueeze(0)[:, i: i + cur_split_size].size(), index[:, i: i + cur_split_size].size()) + # linear_combed_ref_p += ref_p.unsqueeze(0)[:, i: i + cur_split_size] * index[:, i: i + cur_split_size] + # linear_combed_ref_p = linear_combed_ref_p.sum(1) + + # diff = (linear_combed_ref_p - trained_p).norm(2) ** 2 + # return diff + + def get_index_loss(self, fm, md, indexes, match_fn, split_size): + res = 0. + + for name, p in md.named_parameters(): + if name not in indexes.keys(): + continue + # if p.dim() == 0: + # continue + + raw_p = match_fn(name, fm) + # if raw_p is None: + # continue + + index = indexes[name] + + # print(name) + # res += (self.two_params_diff_fast_including_upsample(p, raw_p, index, split_size, weight_affines[name]) + \ + # self.two_params_diff_fast_including_downsample(p, raw_p, index, split_size, weight_affines[name])) / 2. + + res += self.two_params_diff_fast_no_sample(p, raw_p, index, split_size) + + return res + + def bn_cal(self, model: nn.Module, train_loader, num_iters, device): + has_bn = False + for n, m in model.named_modules(): + if isinstance(m, nn.BatchNorm2d): + has_bn = True + break + + if not has_bn: + return {} + + def bn_calibration_init(m): + """ calculating post-statistics of batch normalization """ + if getattr(m, 'track_running_stats', False): + # reset all values for post-statistics + m.reset_running_stats() + # set bn in training mode to update post-statistics + m.training = True + + with torch.no_grad(): + model.eval() + model.apply(bn_calibration_init) + for _ in range(num_iters): + x, _ = next(train_loader) + model(x.to(device)) + model.eval() + + bn_stats = {} + for n, m in model.named_modules(): + if isinstance(m, nn.BatchNorm2d): + bn_stats[n] = m + return bn_stats + + def run(self, scenario: Scenario, hyps: Dict) -> Dict[str, Any]: + super().run(scenario, hyps) + + # sanity check + # a= torch.tensor([[1, 2, 3], [1, 2, 4]]) + # index = torch.tensor([[1, 2, 3], + # [1, 2, 4]]) + # b = torch.tensor([[1, 2, 3], [1, 2, 4], [2, 3, 4]]) + # print(self.two_params_diff_fast(a, b, index, hyps['index_guided_linear_comb_split_size'])) + + assert isinstance(self.models['md'], ElasticDNN_OfflineMDModel) # for auto completion + assert isinstance(self.models['fm'], ElasticDNN_OfflineFMModel) # for auto completion + + # 1. add FBS + device = self.models['md'].device + + # logger.info(f'init master DNN by reducing width of an adapted foundation model (already tuned by LoRA)...') + + # before_fm_model = deepcopy(self.models['fm'].models_dict['main']) + # lora_util = self.models['fm'].get_lora_util() + # lora_absorbed_fm_model = lora_util.absorb_lora_and_recover_net_structure(self.models['fm'].models_dict['main'], + # torch.rand(hyps['samples_size']).to(device)) + # self.models['fm'].models_dict['main'] = lora_absorbed_fm_model + # master_dnn = self.models['fm'].generate_md_by_reducing_width(hyps['generate_md_width_ratio'], + # torch.rand(hyps['samples_size']).to(device)) + # self.models['fm'].models_dict['main'] = before_fm_model + + elastic_dnn_util = self.models['fm'].get_elastic_dnn_util() + # master_dnn = elastic_dnn_util.convert_raw_dnn_to_master_dnn_with_perf_test(master_dnn, + # hyps['FBS_r'], hyps['FBS_ignore_layers']) + # self.models['md'].models_dict['main'] = master_dnn + # self.models['md'].to(device) + master_dnn = self.models['md'].models_dict['main'] + + # 2. train (knowledge distillation, index relationship) + offline_datasets = scenario.get_offline_datasets() + train_dataset = MergedDataset([d['train'] for d in offline_datasets.values()]) + val_dataset = MergedDataset([d['val'] for d in offline_datasets.values()]) + train_loader = iter(build_dataloader(train_dataset, hyps['train_batch_size'], hyps['num_workers'], + True, None)) + val_loader = build_dataloader(val_dataset, hyps['val_batch_size'], hyps['num_workers'], + False, False) + + # 2.1 train only FBS (skipped because current md cannot do proper inference) + + # 2.2 train whole master DNN (knowledge distillation, index relationship) + for p in master_dnn.parameters(): + p.requires_grad = True + self.models['md'].to_train_mode() + + _norm = lambda a: (a.T / a.T.sum(0)).T + + indexes = {} + # weight_affines = {} + # weight_affines_trained_p = [] + + for name, p in self.models['md'].models_dict['main'].named_parameters(): + # if p.dim() > 1: + # print(name) + + logger.info(f'try: layer {name}, {p.size()}') + + matched_p_in_fm = self.models['md'].get_matched_param_of_fm(name, self.models['fm'].models_dict['main']) + if matched_p_in_fm is None: + continue + + + + if p.dim() == 1: + assert p.size(0) == matched_p_in_fm.size(0), f'{p.size()}, {matched_p_in_fm.size()}' + + if hyps['index_init'] == 'rand_norm': + indexes[name] = _norm(torch.rand((p.size(0), matched_p_in_fm.size(0))).to(device)) + elif hyps['index_init'] == 'zero': + indexes[name] = torch.zeros((p.size(0), matched_p_in_fm.size(0))).to(device) + elif hyps['index_init'] == 'randn_norm': + indexes[name] = _norm(torch.randn((p.size(0), matched_p_in_fm.size(0))).to(device)) + logger.info(f'construct index {indexes[name].size()} in layer {name} | dim 0') + + elif p.dim() == 2: + assert p.size(0) == matched_p_in_fm.size(0) or p.size(1) == matched_p_in_fm.size(1), f'{p.size()}, {matched_p_in_fm.size()}' + + if p.size(0) == matched_p_in_fm.size(0): + if hyps['index_init'] == 'rand_norm': + indexes[name] = _norm(torch.rand((p.size(1), matched_p_in_fm.size(1))).to(device)) + elif hyps['index_init'] == 'zero': + indexes[name] = torch.zeros((p.size(1), matched_p_in_fm.size(1))).to(device) + elif hyps['index_init'] == 'randn_norm': + indexes[name] = _norm(torch.randn((p.size(1), matched_p_in_fm.size(1))).to(device)) + logger.info(f'construct index {indexes[name].size()} in layer {name} | dim 1') + + elif p.size(1) == matched_p_in_fm.size(1): + if hyps['index_init'] == 'rand_norm': + indexes[name] = _norm(torch.rand((p.size(0), matched_p_in_fm.size(0))).to(device)) + elif hyps['index_init'] == 'zero': + indexes[name] = torch.zeros((p.size(0), matched_p_in_fm.size(0))).to(device) + elif hyps['index_init'] == 'randn_norm': + indexes[name] = _norm(torch.randn((p.size(0), matched_p_in_fm.size(0))).to(device)) + logger.info(f'construct index {indexes[name].size()} in layer {name} | dim 0') + else: + raise NotImplementedError + + + indexes[name].requires_grad = True + + + + tmp_indexes_file_path = os.path.join(self.res_save_dir, 'tmp-indexes.pt') + torch.save([indexes], tmp_indexes_file_path) + logger.info(f'generate indexes ({(os.path.getsize(tmp_indexes_file_path) / 1024**2):.3f}MB)') + os.remove(tmp_indexes_file_path) + + optimizer = torch.optim.__dict__[hyps['optimizer']]([ + {'params': self.models['md'].models_dict['main'].parameters(), **hyps['optimizer_args']}, + {'params': [v for v in indexes.values()], **hyps['indexes_optimizer_args']} + ]) + scheduler = torch.optim.lr_scheduler.__dict__[hyps['scheduler']](optimizer, **hyps['scheduler_args']) + tb_writer = create_tbwriter(os.path.join(self.res_save_dir, 'tb_log'), launch_tbboard=hyps['launch_tbboard']) + pbar = tqdm.tqdm(range(hyps['num_iters']), dynamic_ncols=True) + best_avg_val_acc = 0. + + for iter_index in pbar: + self.models['md'].to_train_mode() + self.models['fm'].to_eval_mode() + + rand_sparsity = random.random() * (hyps['max_sparsity'] - hyps['min_sparsity']) + hyps['min_sparsity'] + elastic_dnn_util.set_master_dnn_sparsity(self.models['md'].models_dict['main'], rand_sparsity) + + x, y = next(train_loader) + x, y = x.to(device), y.to(device) + + task_loss = self.models['md'].forward_to_get_task_loss(x, y) + # l1_reg_loss = hyps['l1_reg_loss_weight'] * elastic_dnn_util.get_accu_l1_reg_of_raw_channel_attention_in_master_dnn(master_dnn) + index_loss = hyps['index_loss_weight'] * self.get_index_loss(self.models['fm'].models_dict['main'], + self.models['md'].models_dict['main'], + indexes, + self.models['md'].get_matched_param_of_fm, + hyps['index_guided_linear_comb_split_size']) + total_loss = task_loss + index_loss + + optimizer.zero_grad() + total_loss.backward() + optimizer.step() + scheduler.step() + + if (iter_index + 1) % hyps['val_freq'] == 0: + + elastic_dnn_util.clear_cached_channel_attention_in_master_dnn(self.models['md'].models_dict['main']) + + cur_md = self.models['md'].models_dict['main'] + md_for_test = deepcopy(self.models['md'].models_dict['main']) + val_accs = {} + avg_val_acc = 0. + bn_stats = {} + + for val_sparsity in np.linspace(hyps['min_sparsity'], hyps['max_sparsity'], num=hyps['val_num_sparsities']): + elastic_dnn_util.set_master_dnn_sparsity(md_for_test, val_sparsity) + bn_stats[f'{val_sparsity:.4f}'] = self.bn_cal(md_for_test, train_loader, hyps['bn_cal_num_iters'], device) + self.models['md'].models_dict['main'] = md_for_test + self.models['md'].to_eval_mode() + val_acc = self.models['md'].get_accuracy(val_loader) + + val_accs[f'{val_sparsity:.4f}'] = val_acc + avg_val_acc += val_acc + + avg_val_acc /= hyps['val_num_sparsities'] + + self.models['md'].models_dict['main'] = cur_md + self.models['md'].models_dict['indexes'] = indexes + self.models['md'].models_dict['bn_stats'] = bn_stats + self.models['fm'].models_dict['indexes'] = indexes + + self.models['md'].save_model(os.path.join(self.res_save_dir, 'models/md_last.pt')) + self.models['fm'].save_model(os.path.join(self.res_save_dir, 'models/fm_last.pt')) + + if avg_val_acc > best_avg_val_acc: + best_avg_val_acc = avg_val_acc + self.models['md'].save_model(os.path.join(self.res_save_dir, 'models/md_best.pt')) + self.models['fm'].save_model(os.path.join(self.res_save_dir, 'models/fm_best.pt')) + + tb_writer.add_scalars(f'losses', dict(task=task_loss, index=index_loss, total=total_loss), iter_index) + pbar.set_description(f'loss: {total_loss:.6f}, task_loss: {task_loss:.6f}, index_loss: {index_loss:.6f}') + if (iter_index + 1) >= hyps['val_freq']: + tb_writer.add_scalars(f'accs/val_accs', val_accs, iter_index) + tb_writer.add_scalar(f'accs/avg_val_acc', avg_val_acc, iter_index) + pbar.set_description(f'loss: {total_loss:.6f}, task_loss: {task_loss:.6f}, index_loss: {index_loss:.6f}, ' + f'avg_val_acc: {avg_val_acc:.4f}') + \ No newline at end of file diff --git a/methods/elasticdnn/api/algs/md_pretraining_index_v2_train_index_and_md.py b/methods/elasticdnn/api/algs/md_pretraining_index_v2_train_index_and_md.py new file mode 100644 index 0000000000000000000000000000000000000000..edf7afeefb368b75c810cef28d7b4849d0622b5d --- /dev/null +++ b/methods/elasticdnn/api/algs/md_pretraining_index_v2_train_index_and_md.py @@ -0,0 +1,536 @@ +from typing import Any, Dict +from schema import Schema, Or +import schema +from data import Scenario, MergedDataset +from methods.base.alg import BaseAlg +from data import build_dataloader +from ..model import ElasticDNN_OfflineFMModel, ElasticDNN_OfflineMDModel +from ...model.base import ElasticDNNUtil +import torch.optim +import tqdm +import torch.nn.functional as F +from torch import nn +from utils.dl.common.env import create_tbwriter +import os +import random +import numpy as np +from copy import deepcopy +from utils.dl.common.model import LayerActivation, get_module +from utils.common.log import logger +from torchvision.transforms import Compose + +# class WeightAffine(nn.Module): +# def __init__(self, a, b, r=16): +# super(WeightAffine, self).__init__() + +# self.a = a +# self.b = b + +# self.w1 = nn.Parameter(torch.rand((b // r, a))) +# self.w2 = nn.Parameter(torch.rand((b, b // r))) + +# self.a_to_b = True + +# nn.init.zeros_(self.w1.data) +# nn.init.zeros_(self.w2.data) + +# def forward1(self, x): +# return F.linear(F.linear(x, self.w1), self.w2) + +# def forward2(self, x): +# return F.linear(F.linear(x, self.w2.T), self.w1.T) + +# def forward(self, x): +# return self.forward1(x) if self.a_to_b else self.forward2(x) + + +class ElasticDNN_MDPretrainingIndexAlg(BaseAlg): + """ + construct indexes between a filter/row of MD and all filters/rows of FM in the same layer + too huge indexes (~1GB), train so slow, hard to optimize + """ + def get_required_models_schema(self) -> Schema: + return Schema({ + 'fm': ElasticDNN_OfflineFMModel, + 'md': ElasticDNN_OfflineMDModel + }) + + def get_required_hyp_schema(self) -> Schema: + from schema import Optional + return Schema({ + 'launch_tbboard': bool, + + 'samples_size': object, + + 'FBS_r': int, + 'FBS_ignore_layers': [str], + + 'train_batch_size': int, + 'val_batch_size': int, + 'num_workers': int, + 'optimizer': str, + 'optimizer_args': dict, + 'indexes_optimizer_args': dict, + 'scheduler': str, + 'scheduler_args': dict, + 'num_iters': int, + 'val_freq': int, + 'max_sparsity': float, + 'min_sparsity': float, + 'l1_reg_loss_weight': float, + 'index_loss_weight': float, + 'val_num_sparsities': int, + + 'bn_cal_num_iters': int, + + 'index_init': str, + 'index_guided_linear_comb_split_size': Or(int, None), + Optional('transform'): Compose, + }) + + # def upsample_2d_tensor(self, p: torch.Tensor, target_len: int, weight_affine: WeightAffine): + # assert p.dim() == 2 # regard 2d weight as (batch_size, 1d_vector_dim) + # # return F.upsample(p.unsqueeze(1).unsqueeze(3), + # # size=(target_len, 1), + # # mode='bilinear').squeeze(3).squeeze(1) + # weight_affine.a_to_b = True + # return weight_affine(p) + + # def downsample_2d_tensor(self, p: torch.Tensor, target_len: int, weight_affine: WeightAffine): + # assert p.dim() == 2 # regard 2d weight as (batch_size, 1d_vector_dim) + # # return F.interpolate(p.unsqueeze(1).unsqueeze(3), + # # size=(target_len, 1), + # # mode='bilinear').squeeze(3).squeeze(1) + # weight_affine.a_to_b = False + # return weight_affine(p) + + def two_params_diff_fast_no_sample(self, trained_p: torch.Tensor, ref_p: torch.Tensor, + index: torch.Tensor, + split_size: int): + + assert trained_p.dim() == ref_p.dim() + + if trained_p.dim() > 1 and index.size(0) == trained_p.size(1) and index.size(1) == ref_p.size(1): + assert trained_p.dim() == 2 + trained_p = trained_p.T + ref_p = ref_p.T + + assert index.size(0) == trained_p.size(0) and index.size(1) == ref_p.size(0) + + # print(trained_p.size(), ref_p.size(), index.size()) + + ref_p = ref_p.detach() + if trained_p.dim() > 1: + trained_p = trained_p.flatten(1) + ref_p = ref_p.flatten(1) + + # the weight size of master DNN and foundation model may be totally different + + # MD -> FM: upsample first + # FM -> MD: downsample first + # if trained_p.size(1) < ref_p.size(1): + # trained_p = self.upsample_2d_tensor(trained_p, ref_p.size(1), weight_affine) + + index = index.unsqueeze(-1) + # linear_combed_ref_p = (ref_p.unsqueeze(0) * index).sum(1) + # else: + + # print(trained_p.size(), ref_p.size(), index.size()) + + if split_size is None: + # old version: huge memory consumption, not recommended (although this is fastest) + # print('old version') + linear_combed_ref_p = (ref_p.unsqueeze(0) * index).sum(1) + + else: + # new version + linear_combed_ref_p = 0 + + cur_split_size = split_size + while index.size(1) % cur_split_size != 0: + cur_split_size -= 1 + # print(cur_split_size) + + for i in range(0, index.size(1), cur_split_size): + # if not isinstance(linear_combed_ref_p, int): + # print(linear_combed_ref_p.size(), ref_p.unsqueeze(0)[:, i: i + cur_split_size].size(), index[:, i: i + cur_split_size].size()) + linear_combed_ref_p += ref_p.unsqueeze(0)[:, i: i + cur_split_size] * index[:, i: i + cur_split_size] + linear_combed_ref_p = linear_combed_ref_p.sum(1) + + diff = (linear_combed_ref_p - trained_p).norm(2) ** 2 + return diff + + # def two_params_diff_fast_including_upsample(self, trained_p: torch.Tensor, ref_p: torch.Tensor, + # index: torch.Tensor, + # split_size: int, weight_affine: WeightAffine): + + # assert trained_p.dim() == ref_p.dim() + # assert index.size(0) == trained_p.size(0) and index.size(1) == ref_p.size(0) + + # # print(trained_p.size(), ref_p.size(), index.size()) + + # ref_p = ref_p.detach() + # if trained_p.dim() > 1: + # trained_p = trained_p.flatten(1) + # ref_p = ref_p.flatten(1) + + # # the weight size of master DNN and foundation model may be totally different + + # # MD -> FM: upsample first + # # FM -> MD: downsample first + # if trained_p.size(1) < ref_p.size(1): + # trained_p = self.upsample_2d_tensor(trained_p, ref_p.size(1), weight_affine) + + # index = index.unsqueeze(-1) + # # linear_combed_ref_p = (ref_p.unsqueeze(0) * index).sum(1) + # # else: + + # # print(trained_p.size(), ref_p.size(), index.size()) + + # if split_size is None: + # # old version: huge memory consumption, not recommended (although this is fastest) + # # print('old version') + # linear_combed_ref_p = (ref_p.unsqueeze(0) * index).sum(1) + + # else: + # # new version + # linear_combed_ref_p = 0 + + # cur_split_size = split_size + # while index.size(1) % cur_split_size != 0: + # cur_split_size -= 1 + # # print(cur_split_size) + + # for i in range(0, index.size(1), cur_split_size): + # # if not isinstance(linear_combed_ref_p, int): + # # print(linear_combed_ref_p.size(), ref_p.unsqueeze(0)[:, i: i + cur_split_size].size(), index[:, i: i + cur_split_size].size()) + # linear_combed_ref_p += ref_p.unsqueeze(0)[:, i: i + cur_split_size] * index[:, i: i + cur_split_size] + # linear_combed_ref_p = linear_combed_ref_p.sum(1) + + # diff = (linear_combed_ref_p - trained_p).norm(2) ** 2 + # return diff + + # def two_params_diff_fast_including_downsample(self, trained_p: torch.Tensor, ref_p: torch.Tensor, + # index: torch.Tensor, + # split_size: int, weight_affine: WeightAffine): + + # assert trained_p.dim() == ref_p.dim() + # assert index.size(0) == trained_p.size(0) and index.size(1) == ref_p.size(0) + + # # print(trained_p.size(), ref_p.size(), index.size()) + + # ref_p = ref_p.detach() + # if trained_p.dim() > 1: + # trained_p = trained_p.flatten(1) + # ref_p = ref_p.flatten(1) + + # # the weight size of master DNN and foundation model may be totally different + + # # MD -> FM: upsample first + # # FM -> MD: downsample first + # if trained_p.size(1) < ref_p.size(1): + # # trained_p = self.upsample_2d_tensor(trained_p, ref_p.size(1)) + # ref_p = self.downsample_2d_tensor(ref_p, trained_p.size(1), weight_affine) + + # index = index.unsqueeze(-1) + # # linear_combed_ref_p = (ref_p.unsqueeze(0) * index).sum(1) + # # else: + + # # print(trained_p.size(), ref_p.size(), index.size()) + + # if split_size is None: + # # old version: huge memory consumption, not recommended (although this is fastest) + # # print('old version') + # linear_combed_ref_p = (ref_p.unsqueeze(0) * index).sum(1) + + # else: + # # new version + # linear_combed_ref_p = 0 + + # cur_split_size = split_size + # while index.size(1) % cur_split_size != 0: + # cur_split_size -= 1 + # # print(cur_split_size) + + # for i in range(0, index.size(1), cur_split_size): + # # if not isinstance(linear_combed_ref_p, int): + # # print(linear_combed_ref_p.size(), ref_p.unsqueeze(0)[:, i: i + cur_split_size].size(), index[:, i: i + cur_split_size].size()) + # linear_combed_ref_p += ref_p.unsqueeze(0)[:, i: i + cur_split_size] * index[:, i: i + cur_split_size] + # linear_combed_ref_p = linear_combed_ref_p.sum(1) + + # diff = (linear_combed_ref_p - trained_p).norm(2) ** 2 + # return diff + + def get_index_loss(self, fm, md, indexes, match_fn, split_size): + res = 0. + + for name, p in md.named_parameters(): + if name not in indexes.keys(): + continue + # if p.dim() == 0: + # continue + + raw_p = match_fn(name, fm) + # if raw_p is None: + # continue + + index = indexes[name] + + # print(name) + # res += (self.two_params_diff_fast_including_upsample(p, raw_p, index, split_size, weight_affines[name]) + \ + # self.two_params_diff_fast_including_downsample(p, raw_p, index, split_size, weight_affines[name])) / 2. + + res += self.two_params_diff_fast_no_sample(p, raw_p, index, split_size) + + return res + + def bn_cal(self, model: nn.Module, train_loader, num_iters, device): + has_bn = False + for n, m in model.named_modules(): + if isinstance(m, nn.BatchNorm2d): + has_bn = True + break + + if not has_bn: + return {} + + def bn_calibration_init(m): + """ calculating post-statistics of batch normalization """ + if getattr(m, 'track_running_stats', False): + # reset all values for post-statistics + m.reset_running_stats() + # set bn in training mode to update post-statistics + m.training = True + + with torch.no_grad(): + model.eval() + model.apply(bn_calibration_init) + for _ in range(num_iters): + x, _ = next(train_loader) + model(x.to(device)) + model.eval() + + bn_stats = {} + for n, m in model.named_modules(): + if isinstance(m, nn.BatchNorm2d): + bn_stats[n] = m + return bn_stats + + def run(self, scenario: Scenario, hyps: Dict, collate_fn=None) -> Dict[str, Any]: + super().run(scenario, hyps) + + # sanity check + # a= torch.tensor([[1, 2, 3], [1, 2, 4]]) + # index = torch.tensor([[1, 2, 3], + # [1, 2, 4]]) + # b = torch.tensor([[1, 2, 3], [1, 2, 4], [2, 3, 4]]) + # print(self.two_params_diff_fast(a, b, index, hyps['index_guided_linear_comb_split_size'])) + + assert isinstance(self.models['md'], ElasticDNN_OfflineMDModel) # for auto completion + assert isinstance(self.models['fm'], ElasticDNN_OfflineFMModel) # for auto completion + + # 1. add FBS + device = self.models['md'].device + + # logger.info(f'init master DNN by reducing width of an adapted foundation model (already tuned by LoRA)...') + + # before_fm_model = deepcopy(self.models['fm'].models_dict['main']) + # lora_util = self.models['fm'].get_lora_util() + # lora_absorbed_fm_model = lora_util.absorb_lora_and_recover_net_structure(self.models['fm'].models_dict['main'], + # torch.rand(hyps['samples_size']).to(device)) + # self.models['fm'].models_dict['main'] = lora_absorbed_fm_model + # master_dnn = self.models['fm'].generate_md_by_reducing_width(hyps['generate_md_width_ratio'], + # torch.rand(hyps['samples_size']).to(device)) + # self.models['fm'].models_dict['main'] = before_fm_model + + # 2. train (knowledge distillation, index relationship) + if 'transform' in hyps.keys(): + offline_datasets = scenario.get_offline_datasets(transform=hyps['transform']) + else: + offline_datasets = scenario.get_offline_datasets() + train_dataset = MergedDataset([d['train'] for d in offline_datasets.values()]) + val_dataset = MergedDataset([d['val'] for d in offline_datasets.values()]) + train_loader = iter(build_dataloader(train_dataset, hyps['train_batch_size'], hyps['num_workers'], + True, None, collate_fn=collate_fn)) + val_loader = build_dataloader(val_dataset, hyps['val_batch_size'], hyps['num_workers'], + False, False, collate_fn=collate_fn) + + logger.info(f'master DNN acc before inserting FBS: {self.models["md"].get_accuracy(val_loader):.4f}') + + elastic_dnn_util = self.models['fm'].get_elastic_dnn_util() + + + master_dnn = self.models['md'].models_dict['main'] + elastic_dnn_util = self.models['fm'].get_elastic_dnn_util() + master_dnn = elastic_dnn_util.convert_raw_dnn_to_master_dnn_with_perf_test(master_dnn, hyps['FBS_r'], hyps['FBS_ignore_layers']).to(device) + self.models['md'].models_dict['main'] = master_dnn + # master_dnn = elastic_dnn_util.convert_raw_dnn_to_master_dnn_with_perf_test(master_dnn, + # hyps['FBS_r'], hyps['FBS_ignore_layers']) + # self.models['md'].models_dict['main'] = master_dnn + # self.models['md'].to(device) + # master_dnn = self.models['md'].models_dict['main'] + + + + + + # 2.1 train only FBS (skipped because current md cannot do proper inference) + + # 2.2 train whole master DNN (knowledge distillation, index relationship) + for p in master_dnn.parameters(): + p.requires_grad = True + self.models['md'].to_train_mode() + + _norm = lambda a: (a.T / a.T.sum(0)).T + + indexes = {} + # weight_affines = {} + # weight_affines_trained_p = [] + + for name, p in self.models['md'].models_dict['main'].named_parameters(): + # if p.dim() > 1: + # print(name) + + logger.debug(f'try: layer {name}, {p.size()}') + + matched_p_in_fm = self.models['md'].get_matched_param_of_fm(name, self.models['fm'].models_dict['main']) + if matched_p_in_fm is None: + logger.debug(f'layer {name} no matched fm param') + continue + logger.debug(f'layer {name} matched fm param: {matched_p_in_fm.size()}') + + + if p.dim() == 1: + # assert p.size(0) == matched_p_in_fm.size(0), f'{p.size()}, {matched_p_in_fm.size()}' + + if hyps['index_init'] == 'rand_norm': + indexes[name] = _norm(torch.rand((p.size(0), matched_p_in_fm.size(0))).to(device)) + elif hyps['index_init'] == 'zero': + indexes[name] = torch.zeros((p.size(0), matched_p_in_fm.size(0))).to(device) + elif hyps['index_init'] == 'randn_norm': + indexes[name] = _norm(torch.randn((p.size(0), matched_p_in_fm.size(0))).to(device)) + else: + raise NotImplementedError + logger.info(f'construct index {indexes[name].size()} in layer {name} | dim 0') + + elif p.dim() == 2: + assert p.size(0) == matched_p_in_fm.size(0) or p.size(1) == matched_p_in_fm.size(1), f'{p.size()}, {matched_p_in_fm.size()}' + + if p.size(0) == matched_p_in_fm.size(0): + if hyps['index_init'] == 'rand_norm': + indexes[name] = _norm(torch.rand((p.size(1), matched_p_in_fm.size(1))).to(device)) + elif hyps['index_init'] == 'zero': + indexes[name] = torch.zeros((p.size(1), matched_p_in_fm.size(1))).to(device) + elif hyps['index_init'] == 'randn_norm': + indexes[name] = _norm(torch.randn((p.size(1), matched_p_in_fm.size(1))).to(device)) + else: + raise NotImplementedError + logger.info(f'construct index {indexes[name].size()} in layer {name} | dim 1') + + elif p.size(1) == matched_p_in_fm.size(1): + if hyps['index_init'] == 'rand_norm': + indexes[name] = _norm(torch.rand((p.size(0), matched_p_in_fm.size(0))).to(device)) + elif hyps['index_init'] == 'zero': + indexes[name] = torch.zeros((p.size(0), matched_p_in_fm.size(0))).to(device) + elif hyps['index_init'] == 'randn_norm': + indexes[name] = _norm(torch.randn((p.size(0), matched_p_in_fm.size(0))).to(device)) + else: + raise NotImplementedError + logger.info(f'construct index {indexes[name].size()} in layer {name} | dim 0') + else: + print('a') + continue + # raise NotImplementedError + + + indexes[name].requires_grad = True + + + + tmp_indexes_file_path = os.path.join(self.res_save_dir, 'tmp-indexes.pt') + torch.save([indexes], tmp_indexes_file_path) + logger.info(f'generate indexes ({(os.path.getsize(tmp_indexes_file_path) / 1024**2):.3f}MB)') + os.remove(tmp_indexes_file_path) + + optimizer = torch.optim.__dict__[hyps['optimizer']]([ + {'params': self.models['md'].models_dict['main'].parameters(), **hyps['optimizer_args']}, + {'params': [v for v in indexes.values()], **hyps['indexes_optimizer_args']} + ]) + scheduler = torch.optim.lr_scheduler.__dict__[hyps['scheduler']](optimizer, **hyps['scheduler_args']) + tb_writer = create_tbwriter(os.path.join(self.res_save_dir, 'tb_log'), launch_tbboard=hyps['launch_tbboard']) + pbar = tqdm.tqdm(range(hyps['num_iters']), dynamic_ncols=True) + best_avg_val_acc = 0. + + for iter_index in pbar: + self.models['md'].to_train_mode() + self.models['fm'].to_eval_mode() + + rand_sparsity = random.random() * (hyps['max_sparsity'] - hyps['min_sparsity']) + hyps['min_sparsity'] + elastic_dnn_util.set_master_dnn_sparsity(self.models['md'].models_dict['main'], rand_sparsity) + + x, y = next(train_loader) + if isinstance(x, dict): + for k, v in x.items(): + if isinstance(v, torch.Tensor): + x[k] = v.to(device) + y = y.to(device) + else: + x, y = x.to(device), y.to(device) + + task_loss = self.models['md'].forward_to_get_task_loss(x, y) + l1_reg_loss = hyps['l1_reg_loss_weight'] * elastic_dnn_util.get_accu_l1_reg_of_raw_channel_attention_in_master_dnn(master_dnn) + index_loss = hyps['index_loss_weight'] * self.get_index_loss(self.models['fm'].models_dict['main'], + self.models['md'].models_dict['main'], + indexes, + self.models['md'].get_matched_param_of_fm, + hyps['index_guided_linear_comb_split_size']) + total_loss = task_loss + index_loss + l1_reg_loss + + optimizer.zero_grad() + total_loss.backward() + optimizer.step() + scheduler.step() + + if (iter_index + 1) % hyps['val_freq'] == 0: + + elastic_dnn_util.clear_cached_channel_attention_in_master_dnn(self.models['md'].models_dict['main']) + + cur_md = self.models['md'].models_dict['main'] + md_for_test = deepcopy(self.models['md'].models_dict['main']) + val_accs = {} + avg_val_acc = 0. + bn_stats = {} + + for val_sparsity in np.linspace(hyps['min_sparsity'], hyps['max_sparsity'], num=hyps['val_num_sparsities']): + elastic_dnn_util.set_master_dnn_sparsity(md_for_test, val_sparsity) + bn_stats[f'{val_sparsity:.4f}'] = self.bn_cal(md_for_test, train_loader, hyps['bn_cal_num_iters'], device) + self.models['md'].models_dict['main'] = md_for_test + self.models['md'].to_eval_mode() + val_acc = self.models['md'].get_accuracy(val_loader, collate_fn=collate_fn) + + val_accs[f'{val_sparsity:.4f}'] = val_acc + avg_val_acc += val_acc + + avg_val_acc /= hyps['val_num_sparsities'] + + self.models['md'].models_dict['main'] = cur_md + self.models['md'].models_dict['indexes'] = indexes + self.models['md'].models_dict['bn_stats'] = bn_stats + self.models['fm'].models_dict['indexes'] = indexes + + self.models['md'].save_model(os.path.join(self.res_save_dir, 'models/md_last.pt')) + self.models['fm'].save_model(os.path.join(self.res_save_dir, 'models/fm_last.pt')) + + if avg_val_acc > best_avg_val_acc: + best_avg_val_acc = avg_val_acc + self.models['md'].save_model(os.path.join(self.res_save_dir, 'models/md_best.pt')) + self.models['fm'].save_model(os.path.join(self.res_save_dir, 'models/fm_best.pt')) + + tb_writer.add_scalars(f'losses', dict(task=task_loss, index=index_loss, l1=l1_reg_loss, total=total_loss), iter_index) + pbar.set_description(f'loss: {total_loss:.6f}, task_loss: {task_loss:.6f}, index_loss: {index_loss:.6f}, l1_loss: {l1_reg_loss:.6f}') + if (iter_index + 1) >= hyps['val_freq']: + tb_writer.add_scalars(f'accs/val_accs', val_accs, iter_index) + tb_writer.add_scalar(f'accs/avg_val_acc', avg_val_acc, iter_index) + pbar.set_description(f'loss: {total_loss:.6f}, task_loss: {task_loss:.6f}, index_loss: {index_loss:.6f}, l1_loss: {l1_reg_loss:.6f}, ' + f'avg_val_acc: {avg_val_acc:.4f}') + \ No newline at end of file diff --git a/methods/elasticdnn/api/algs/md_pretraining_v1.py b/methods/elasticdnn/api/algs/md_pretraining_v1.py new file mode 100644 index 0000000000000000000000000000000000000000..1b62f0bf6798669a7fc874930055aa2342d31307 --- /dev/null +++ b/methods/elasticdnn/api/algs/md_pretraining_v1.py @@ -0,0 +1,316 @@ +from typing import Any, Dict +from schema import Schema, Or +import schema +from data import Scenario, MergedDataset +from methods.base.alg import BaseAlg +from data import build_dataloader +from ..model import ElasticDNN_OfflineFMModel, ElasticDNN_OfflineMDModel +from ...model.base import ElasticDNNUtil +import torch.optim +import tqdm +import torch.nn.functional as F +from torch import nn +from utils.dl.common.env import create_tbwriter +import os +import random +import numpy as np +from copy import deepcopy +from utils.dl.common.model import LayerActivation, get_module +from utils.common.log import logger + + +class ElasticDNN_MDPretrainingAlg(BaseAlg): + """ + construct indexes between a filter/row of MD and all filters/rows of FM in the same layer + too huge indexes (~1GB), train so slow, hard to optimize + """ + def get_required_models_schema(self) -> Schema: + return Schema({ + 'fm': ElasticDNN_OfflineFMModel, + 'md': ElasticDNN_OfflineMDModel + }) + + def get_required_hyp_schema(self) -> Schema: + return Schema({ + 'launch_tbboard': bool, + + 'samples_size': (int, int, int, int), + 'generate_md_width_ratio': int, + + 'FBS_r': int, + 'FBS_ignore_layers': [str], + + 'train_batch_size': int, + 'val_batch_size': int, + 'num_workers': int, + 'optimizer': str, + 'md_optimizer_args': dict, + 'indexes_optimizer_args': dict, + 'scheduler': str, + 'scheduler_args': dict, + 'num_iters': int, + 'val_freq': int, + 'max_sparsity': float, + 'min_sparsity': float, + 'distill_loss_weight': float, + 'index_loss_weight': float, + 'val_num_sparsities': int, + + 'bn_cal_num_iters': int, + + 'index_guided_linear_comb_split_size': Or(int, None) + }) + + def upsample_2d_tensor(self, p: torch.Tensor, target_len: int): + assert p.dim() == 2 # regard 2d weight as (batch_size, 1d_vector_dim) + return F.upsample(p.unsqueeze(1).unsqueeze(3), + size=(target_len, 1), + mode='bilinear').squeeze(3).squeeze(1) + + def two_params_diff_fast(self, trained_p: torch.Tensor, ref_p: torch.Tensor, + index: torch.Tensor, + split_size: int): + + assert trained_p.dim() == ref_p.dim() + assert index.size(0) == trained_p.size(0) and index.size(1) == ref_p.size(0) + + # print(trained_p.size(), ref_p.size(), index.size()) + + ref_p = ref_p.detach() + if trained_p.dim() > 1: + trained_p = trained_p.flatten(1) + ref_p = ref_p.flatten(1) + + # the weight size of master DNN and foundation model may be totally different + + # MD -> FM: upsample first + # FM -> MD: downsample first + if trained_p.size(1) < ref_p.size(1): + trained_p = self.upsample_2d_tensor(trained_p, ref_p.size(1)) + + index = index.unsqueeze(-1) + # linear_combed_ref_p = (ref_p.unsqueeze(0) * index).sum(1) + # else: + + # print(trained_p.size(), ref_p.size(), index.size()) + + if split_size is None: + # old version: huge memory consumption, not recommended (although this is fastest) + # print('old version') + linear_combed_ref_p = (ref_p.unsqueeze(0) * index).sum(1) + + else: + # new version + linear_combed_ref_p = 0 + + cur_split_size = split_size + while index.size(1) % cur_split_size != 0: + cur_split_size -= 1 + # print(cur_split_size) + + for i in range(0, index.size(1), cur_split_size): + # if not isinstance(linear_combed_ref_p, int): + # print(linear_combed_ref_p.size(), ref_p.unsqueeze(0)[:, i: i + cur_split_size].size(), index[:, i: i + cur_split_size].size()) + linear_combed_ref_p += ref_p.unsqueeze(0)[:, i: i + cur_split_size] * index[:, i: i + cur_split_size] + linear_combed_ref_p = linear_combed_ref_p.sum(1) + + diff = (linear_combed_ref_p - trained_p).norm(2) ** 2 + return diff + + def get_index_loss(self, fm, md, indexes, match_fn, split_size): + res = 0. + + for name, p in md.named_parameters(): + if p.dim() == 0: + continue + + raw_p = match_fn(name, fm) + if raw_p is None: + continue + + index = indexes[name] + + # print(name) + res += self.two_params_diff_fast(p, raw_p, index, split_size) + return res + + def bn_cal(self, model: nn.Module, train_loader, num_iters, device): + has_bn = False + for n, m in model.named_modules(): + if isinstance(m, nn.BatchNorm2d): + has_bn = True + break + + if not has_bn: + return {} + + def bn_calibration_init(m): + """ calculating post-statistics of batch normalization """ + if getattr(m, 'track_running_stats', False): + # reset all values for post-statistics + m.reset_running_stats() + # set bn in training mode to update post-statistics + m.training = True + + with torch.no_grad(): + model.eval() + model.apply(bn_calibration_init) + for _ in range(num_iters): + x, _ = next(train_loader) + model(x.to(device)) + model.eval() + + bn_stats = {} + for n, m in model.named_modules(): + if isinstance(m, nn.BatchNorm2d): + bn_stats[n] = m + return bn_stats + + def run(self, scenario: Scenario, hyps: Dict) -> Dict[str, Any]: + super().run(scenario, hyps) + + # sanity check + # a= torch.tensor([[1, 2, 3], [1, 2, 4]]) + # index = torch.tensor([[1, 2, 3], + # [1, 2, 4]]) + # b = torch.tensor([[1, 2, 3], [1, 2, 4], [2, 3, 4]]) + # print(self.two_params_diff_fast(a, b, index, hyps['index_guided_linear_comb_split_size'])) + + assert isinstance(self.models['md'], ElasticDNN_OfflineMDModel) # for auto completion + assert isinstance(self.models['fm'], ElasticDNN_OfflineFMModel) # for auto completion + + # 1. add FBS + device = self.models['md'].device + + logger.info(f'init master DNN by reducing width of an adapted foundation model (already tuned by LoRA)...') + + before_fm_model = deepcopy(self.models['fm'].models_dict['main']) + lora_util = self.models['fm'].get_lora_util() + lora_absorbed_fm_model = lora_util.absorb_lora_and_recover_net_structure(self.models['fm'].models_dict['main'], + torch.rand(hyps['samples_size']).to(device)) + self.models['fm'].models_dict['main'] = lora_absorbed_fm_model + master_dnn = self.models['fm'].generate_md_by_reducing_width(hyps['generate_md_width_ratio'], + torch.rand(hyps['samples_size']).to(device)) + self.models['fm'].models_dict['main'] = before_fm_model + + elastic_dnn_util = self.models['fm'].get_elastic_dnn_util() + master_dnn = elastic_dnn_util.convert_raw_dnn_to_master_dnn_with_perf_test(master_dnn, + hyps['FBS_r'], hyps['FBS_ignore_layers']) + self.models['md'].models_dict['main'] = master_dnn + self.models['md'].to(device) + + # 2. train (knowledge distillation, index relationship) + offline_datasets = scenario.get_offline_datasets() + train_dataset = MergedDataset([d['train'] for d in offline_datasets.values()]) + val_dataset = MergedDataset([d['val'] for d in offline_datasets.values()]) + train_loader = iter(build_dataloader(train_dataset, hyps['train_batch_size'], hyps['num_workers'], + True, None)) + val_loader = build_dataloader(val_dataset, hyps['val_batch_size'], hyps['num_workers'], + False, False) + + # 2.1 train only FBS (skipped because current md cannot do proper inference) + + # 2.2 train whole master DNN (knowledge distillation, index relationship) + for p in master_dnn.parameters(): + p.requires_grad = True + self.models['md'].to_train_mode() + + indexes = {} + for name, p in self.models['md'].models_dict['main'].named_parameters(): + if p.dim() > 0: + matched_p_in_fm = self.models['md'].get_matched_param_of_fm(name, self.models['fm'].models_dict['main']) + if matched_p_in_fm is None: + continue + indexes[name] = torch.zeros((p.size(0), matched_p_in_fm.size(0))).to(device) + indexes[name].requires_grad = True + + tmp_indexes_file_path = os.path.join(self.res_save_dir, 'tmp-indexes.pt') + torch.save(indexes, tmp_indexes_file_path) + logger.info(f'generate indexes ({(os.path.getsize(tmp_indexes_file_path) / 1024**2):.3f}MB)') + os.remove(tmp_indexes_file_path) + + optimizer = torch.optim.__dict__[hyps['optimizer']]([ + {'params': self.models['md'].models_dict['main'].parameters(), **hyps['md_optimizer_args']}, + {'params': [v for v in indexes.values()], **hyps['indexes_optimizer_args']} + ]) + scheduler = torch.optim.lr_scheduler.__dict__[hyps['scheduler']](optimizer, **hyps['scheduler_args']) + tb_writer = create_tbwriter(os.path.join(self.res_save_dir, 'tb_log'), launch_tbboard=hyps['launch_tbboard']) + pbar = tqdm.tqdm(range(hyps['num_iters']), dynamic_ncols=True) + best_avg_val_acc = 0. + + md_output_hook = None + + for iter_index in pbar: + self.models['md'].to_train_mode() + self.models['fm'].to_eval_mode() + + rand_sparsity = random.random() * (hyps['max_sparsity'] - hyps['min_sparsity']) + hyps['min_sparsity'] + elastic_dnn_util.set_master_dnn_sparsity(self.models['md'].models_dict['main'], rand_sparsity) + + x, y = next(train_loader) + x, y = x.to(device), y.to(device) + + with torch.no_grad(): + fm_output = self.models['fm'].infer(x) + + if md_output_hook is None: + md_output_hook = LayerActivation(self.models['md'].models_dict['main'], False, device) + task_loss = self.models['md'].forward_to_get_task_loss(x, y) + md_output = md_output_hook.output + distill_loss = hyps['distill_loss_weight'] * self.models['md'].get_distill_loss(md_output, fm_output) + index_loss = hyps['index_loss_weight'] * self.get_index_loss(self.models['fm'].models_dict['main'], + self.models['md'].models_dict['main'], + indexes, + self.models['md'].get_matched_param_of_fm, + hyps['index_guided_linear_comb_split_size']) + total_loss = task_loss + distill_loss + index_loss + + optimizer.zero_grad() + total_loss.backward() + optimizer.step() + scheduler.step() + + if (iter_index + 1) % hyps['val_freq'] == 0: + + elastic_dnn_util.clear_cached_channel_attention_in_master_dnn(self.models['md'].models_dict['main']) + md_output_hook.remove() + md_output_hook = None + + cur_md = self.models['md'].models_dict['main'] + md_for_test = deepcopy(self.models['md'].models_dict['main']) + val_accs = {} + avg_val_acc = 0. + bn_stats = {} + + for val_sparsity in np.linspace(hyps['min_sparsity'], hyps['max_sparsity'], num=hyps['val_num_sparsities']): + elastic_dnn_util.set_master_dnn_sparsity(md_for_test, val_sparsity) + bn_stats[f'{val_sparsity:.4f}'] = self.bn_cal(md_for_test, train_loader, hyps['bn_cal_num_iters'], device) + self.models['md'].models_dict['main'] = md_for_test + self.models['md'].to_eval_mode() + val_acc = self.models['md'].get_accuracy(val_loader) + + val_accs[f'{val_sparsity:.4f}'] = val_acc + avg_val_acc += val_acc + + avg_val_acc /= hyps['val_num_sparsities'] + + self.models['md'].models_dict['main'] = cur_md + self.models['md'].models_dict['indexes'] = indexes + self.models['md'].models_dict['bn_stats'] = bn_stats + self.models['fm'].models_dict['indexes'] = indexes + + self.models['md'].save_model(os.path.join(self.res_save_dir, 'models/md_last.pt')) + self.models['fm'].save_model(os.path.join(self.res_save_dir, 'models/fm_last.pt')) + + if avg_val_acc > best_avg_val_acc: + best_avg_val_acc = avg_val_acc + self.models['md'].save_model(os.path.join(self.res_save_dir, 'models/md_best.pt')) + self.models['fm'].save_model(os.path.join(self.res_save_dir, 'models/fm_best.pt')) + + tb_writer.add_scalars(f'losses', dict(task=task_loss, distill=distill_loss, index=index_loss, total=total_loss), iter_index) + pbar.set_description(f'loss: {total_loss:.6f}') + if (iter_index + 1) >= hyps['val_freq']: + tb_writer.add_scalars(f'accs/val_accs', val_accs, iter_index) + tb_writer.add_scalar(f'accs/avg_val_acc', avg_val_acc, iter_index) + pbar.set_description(f'loss: {total_loss:.6f}, avg_val_acc: {avg_val_acc:.4f}') + \ No newline at end of file diff --git a/methods/elasticdnn/api/algs/md_pretraining_w_fbs.py b/methods/elasticdnn/api/algs/md_pretraining_w_fbs.py new file mode 100644 index 0000000000000000000000000000000000000000..53242f1638acd9becf4f56107a712ebbe4ae4ccf --- /dev/null +++ b/methods/elasticdnn/api/algs/md_pretraining_w_fbs.py @@ -0,0 +1,192 @@ +from typing import Any, Dict +from schema import Schema, Or +import schema +from data import Scenario, MergedDataset +from methods.base.alg import BaseAlg +from data import build_dataloader +from ..model import ElasticDNN_OfflineFMModel, ElasticDNN_OfflineMDModel +from ...model.base import ElasticDNNUtil +import torch.optim +import tqdm +import torch.nn.functional as F +from torch import nn +from utils.dl.common.env import create_tbwriter +import os +import random +import numpy as np +from copy import deepcopy +from utils.dl.common.model import LayerActivation, get_module +from utils.common.log import logger + + +class ElasticDNN_MDPretrainingWFBSAlg(BaseAlg): + """ + TODO: fine-tuned FM -> init MD -> trained MD -> construct indexes (only between similar weights) and fine-tune + """ + def get_required_models_schema(self) -> Schema: + return Schema({ + 'fm': ElasticDNN_OfflineFMModel, + 'md': ElasticDNN_OfflineMDModel + }) + + def get_required_hyp_schema(self) -> Schema: + return Schema({ + 'launch_tbboard': bool, + + 'samples_size': (int, int, int, int), + 'generate_md_width_ratio': int, + + 'FBS_r': int, + 'FBS_ignore_layers': [str], + + 'train_batch_size': int, + 'val_batch_size': int, + 'num_workers': int, + 'optimizer': str, + 'optimizer_args': dict, + 'scheduler': str, + 'scheduler_args': dict, + 'num_iters': int, + 'val_freq': int, + 'max_sparsity': float, + 'min_sparsity': float, + 'l1_reg_loss_weight': float, + 'val_num_sparsities': int, + + 'bn_cal_num_iters': int + }) + + def bn_cal(self, model: nn.Module, train_loader, num_iters, device): + has_bn = False + for n, m in model.named_modules(): + if isinstance(m, nn.BatchNorm2d): + has_bn = True + break + + if not has_bn: + return {} + + def bn_calibration_init(m): + """ calculating post-statistics of batch normalization """ + if getattr(m, 'track_running_stats', False): + # reset all values for post-statistics + m.reset_running_stats() + # set bn in training mode to update post-statistics + m.training = True + + with torch.no_grad(): + model.eval() + model.apply(bn_calibration_init) + for _ in range(num_iters): + x, _ = next(train_loader) + model(x.to(device)) + model.eval() + + bn_stats = {} + for n, m in model.named_modules(): + if isinstance(m, nn.BatchNorm2d): + bn_stats[n] = m + return bn_stats + + def run(self, scenario: Scenario, hyps: Dict) -> Dict[str, Any]: + super().run(scenario, hyps) + + assert isinstance(self.models['md'], ElasticDNN_OfflineMDModel) # for auto completion + assert isinstance(self.models['fm'], ElasticDNN_OfflineFMModel) # for auto completion + + # 1. add FBS + device = self.models['md'].device + + # 2. train (knowledge distillation, index relationship) + offline_datasets = scenario.get_offline_datasets() + train_dataset = MergedDataset([d['train'] for d in offline_datasets.values()]) + val_dataset = MergedDataset([d['val'] for d in offline_datasets.values()]) + train_loader = iter(build_dataloader(train_dataset, hyps['train_batch_size'], hyps['num_workers'], + True, None)) + val_loader = build_dataloader(val_dataset, hyps['val_batch_size'], hyps['num_workers'], + False, False) + + logger.info(f'master DNN acc before inserting FBS: {self.models["md"].get_accuracy(val_loader):.4f}') + + master_dnn = self.models['md'].models_dict['main'] + elastic_dnn_util = self.models['fm'].get_elastic_dnn_util() + master_dnn = elastic_dnn_util.convert_raw_dnn_to_master_dnn_with_perf_test(master_dnn, hyps['FBS_r'], hyps['FBS_ignore_layers']).to(device) + self.models['md'].models_dict['main'] = master_dnn + + # 2.1 train whole master DNN (knowledge distillation) + for p in master_dnn.parameters(): + p.requires_grad = True + self.models['md'].to_train_mode() + + optimizer = torch.optim.__dict__[hyps['optimizer']]([ + {'params': self.models['md'].models_dict['main'].parameters(), **hyps['optimizer_args']} + ]) + scheduler = torch.optim.lr_scheduler.__dict__[hyps['scheduler']](optimizer, **hyps['scheduler_args']) + tb_writer = create_tbwriter(os.path.join(self.res_save_dir, 'tb_log'), launch_tbboard=hyps['launch_tbboard']) + pbar = tqdm.tqdm(range(hyps['num_iters']), dynamic_ncols=True) + best_avg_val_acc = 0. + + for iter_index in pbar: + self.models['md'].to_train_mode() + self.models['fm'].to_eval_mode() + + rand_sparsity = random.random() * (hyps['max_sparsity'] - hyps['min_sparsity']) + hyps['min_sparsity'] + elastic_dnn_util.set_master_dnn_sparsity(self.models['md'].models_dict['main'], rand_sparsity) + + x, y = next(train_loader) + x, y = x.to(device), y.to(device) + + task_loss = self.models['md'].forward_to_get_task_loss(x, y) + l1_reg_loss = hyps['l1_reg_loss_weight'] * elastic_dnn_util.get_accu_l1_reg_of_raw_channel_attention_in_master_dnn(master_dnn) + total_loss = task_loss + l1_reg_loss + + optimizer.zero_grad() + total_loss.backward() + optimizer.step() + scheduler.step() + + if (iter_index + 1) % hyps['val_freq'] == 0: + + elastic_dnn_util.clear_cached_channel_attention_in_master_dnn(self.models['md'].models_dict['main']) + + cur_md = self.models['md'].models_dict['main'] + md_for_test = deepcopy(self.models['md'].models_dict['main']) + val_accs = {} + avg_val_acc = 0. + bn_stats = {} + + for val_sparsity in np.linspace(hyps['min_sparsity'], hyps['max_sparsity'], num=hyps['val_num_sparsities']): + elastic_dnn_util.set_master_dnn_sparsity(md_for_test, val_sparsity) + bn_stats[f'{val_sparsity:.4f}'] = self.bn_cal(md_for_test, train_loader, hyps['bn_cal_num_iters'], device) + + # generate seperate surrogate DNN + test_sd = elastic_dnn_util.extract_surrogate_dnn_via_samples_with_perf_test(md_for_test, x) + + self.models['md'].models_dict['main'] = test_sd + self.models['md'].to_eval_mode() + val_acc = self.models['md'].get_accuracy(val_loader) + + val_accs[f'{val_sparsity:.4f}'] = val_acc + avg_val_acc += val_acc + + avg_val_acc /= hyps['val_num_sparsities'] + + self.models['md'].models_dict['main'] = cur_md + self.models['md'].models_dict['bn_stats'] = bn_stats + + self.models['md'].save_model(os.path.join(self.res_save_dir, 'models/md_last.pt')) + self.models['fm'].save_model(os.path.join(self.res_save_dir, 'models/fm_last.pt')) + + if avg_val_acc > best_avg_val_acc: + best_avg_val_acc = avg_val_acc + self.models['md'].save_model(os.path.join(self.res_save_dir, 'models/md_best.pt')) + self.models['fm'].save_model(os.path.join(self.res_save_dir, 'models/fm_best.pt')) + + tb_writer.add_scalars(f'losses', dict(task=task_loss, l1_reg=l1_reg_loss, total=total_loss), iter_index) + pbar.set_description(f'loss: {total_loss:.6f}') + if (iter_index + 1) >= hyps['val_freq']: + tb_writer.add_scalars(f'accs/val_accs', val_accs, iter_index) + tb_writer.add_scalar(f'accs/avg_val_acc', avg_val_acc, iter_index) + pbar.set_description(f'loss: {total_loss:.6f}, task_loss: {task_loss:.6f}, ' + f'l1_loss: {l1_reg_loss:.6f}, avg_val_acc: {avg_val_acc:.4f}') + \ No newline at end of file diff --git a/methods/elasticdnn/api/algs/md_pretraining_wo_fbs.py b/methods/elasticdnn/api/algs/md_pretraining_wo_fbs.py new file mode 100644 index 0000000000000000000000000000000000000000..28034221fc981d30b61b721f364759c9e617d8df --- /dev/null +++ b/methods/elasticdnn/api/algs/md_pretraining_wo_fbs.py @@ -0,0 +1,387 @@ +# from typing import Any, Dict +# from schema import Schema, Or +# import schema +# from data import Scenario, MergedDataset +# from methods.base.alg import BaseAlg +# from data import build_dataloader +# from ..model import ElasticDNN_OfflineFMModel, ElasticDNN_OfflineMDModel +# from ...model.base import ElasticDNNUtil +# import torch.optim +# import tqdm +# import torch.nn.functional as F +# from torch import nn +# from utils.dl.common.env import create_tbwriter +# import os +# import random +# import numpy as np +# from copy import deepcopy +# from utils.dl.common.model import LayerActivation2, get_module +# from utils.common.log import logger + + +# class ElasticDNN_MDPretrainingWoFBSAlg(BaseAlg): +# """ +# TODO: fine-tuned FM -> init MD -> trained MD -> construct indexes (only between similar weights) and fine-tune +# """ +# def get_required_models_schema(self) -> Schema: +# return Schema({ +# 'fm': ElasticDNN_OfflineFMModel, +# 'md': ElasticDNN_OfflineMDModel +# }) + +# def get_required_hyp_schema(self) -> Schema: +# return Schema({ +# 'launch_tbboard': bool, + +# 'samples_size': object, +# 'generate_md_width_ratio': int, + +# 'train_batch_size': int, +# 'val_batch_size': int, +# 'num_workers': int, +# 'optimizer': str, +# 'optimizer_args': dict, +# 'scheduler': str, +# 'scheduler_args': dict, +# 'num_iters': int, +# 'val_freq': int, +# 'distill_loss_weight': float +# }) + +# def run(self, scenario: Scenario, hyps: Dict) -> Dict[str, Any]: +# super().run(scenario, hyps) + +# assert isinstance(self.models['md'], ElasticDNN_OfflineMDModel) # for auto completion +# assert isinstance(self.models['fm'], ElasticDNN_OfflineFMModel) # for auto completion + +# # 1. add FBS +# device = self.models['md'].device + +# if self.models['md'].models_dict['main'] == -1: +# logger.info(f'init master DNN by reducing width of an adapted foundation model (already tuned by LoRA)...') + +# before_fm_model = deepcopy(self.models['fm'].models_dict['main']) +# lora_util = self.models['fm'].get_lora_util() + +# sample = hyps['samples_size'] +# if isinstance(sample, (tuple, list)) and isinstance(sample[0], int): +# sample = torch.rand(hyps['samples_size']).to(device) + +# lora_absorbed_fm_model = lora_util.absorb_lora_and_recover_net_structure(self.models['fm'].models_dict['main'], +# sample) +# self.models['fm'].models_dict['main'] = lora_absorbed_fm_model +# master_dnn = self.models['fm'].generate_md_by_reducing_width(hyps['generate_md_width_ratio'], +# sample) +# self.models['fm'].models_dict['main'] = before_fm_model + +# self.models['md'].models_dict['main'] = master_dnn +# self.models['md'].to(device) + + + +# # 2. train (knowledge distillation, index relationship) +# offline_datasets = scenario.get_offline_datasets() +# train_dataset = MergedDataset([d['train'] for d in offline_datasets.values()]) +# val_dataset = MergedDataset([d['val'] for d in offline_datasets.values()]) +# train_loader = iter(build_dataloader(train_dataset, hyps['train_batch_size'], hyps['num_workers'], +# True, None)) +# val_loader = build_dataloader(val_dataset, hyps['val_batch_size'], hyps['num_workers'], +# False, False) + +# # val_acc = self.models['md'].get_accuracy(val_loader) +# # print(val_acc) +# # exit() + +# # 2.1 train whole master DNN (knowledge distillation) +# self.models['md'].to_train_mode() +# for p in master_dnn.parameters(): +# p.requires_grad = True + +# if hasattr(self.models['md'], 'get_trained_params'): +# trained_p = self.models['md'].get_trained_params() +# logger.info(f'use custom trained parameters!!') +# else: +# trained_p = self.models['md'].models_dict['main'].parameters() +# for p in trained_p: +# p.requires_grad = True +# optimizer = torch.optim.__dict__[hyps['optimizer']]([ +# {'params': trained_p, **hyps['optimizer_args']} +# ]) +# scheduler = torch.optim.lr_scheduler.__dict__[hyps['scheduler']](optimizer, **hyps['scheduler_args']) +# tb_writer = create_tbwriter(os.path.join(self.res_save_dir, 'tb_log'), launch_tbboard=hyps['launch_tbboard']) +# pbar = tqdm.tqdm(range(hyps['num_iters']), dynamic_ncols=True) +# best_avg_val_acc = 0. + +# md_output_hook = None + +# for iter_index in pbar: +# self.models['md'].to_train_mode() +# self.models['fm'].to_eval_mode() + +# # rand_sparsity = random.random() * (hyps['max_sparsity'] - hyps['min_sparsity']) + hyps['min_sparsity'] +# # elastic_dnn_util.set_master_dnn_sparsity(self.models['md'].models_dict['main'], rand_sparsity) +# if md_output_hook is None: +# md_output_hook = self.models['md'].get_feature_hook() +# fm_output_hook = self.models['fm'].get_feature_hook() + +# x, y = next(train_loader) +# if isinstance(x, dict): +# for k, v in x.items(): +# if isinstance(v, torch.Tensor): +# x[k] = v.to(device) +# y = y.to(device) +# else: +# x, y = x.to(device), y.to(device) + +# with torch.no_grad(): +# fm_output = self.models['fm'].infer(x) +# task_loss = self.models['md'].forward_to_get_task_loss(x, y) + +# if isinstance(md_output_hook, (tuple, list)): +# distill_loss = 0. +# for h1, h2 in zip(md_output_hook, fm_output_hook): +# md_output = h1.output +# fm_output = h2.output +# distill_loss += hyps['distill_loss_weight'] * self.models['md'].get_distill_loss(md_output, fm_output) +# else: +# md_output = md_output_hook.output +# fm_output = fm_output_hook.output +# distill_loss = hyps['distill_loss_weight'] * self.models['md'].get_distill_loss(md_output, fm_output) + +# total_loss = task_loss + distill_loss + +# optimizer.zero_grad() +# total_loss.backward() + +# # for n, p in self.models['md'].models_dict['main'].named_parameters(): +# # if p.grad is not None: +# # print(n) +# # exit() + +# optimizer.step() +# scheduler.step() + +# if (iter_index + 1) % hyps['val_freq'] == 0: + +# # elastic_dnn_util.clear_cached_channel_attention_in_master_dnn(self.models['md'].models_dict['main']) +# if isinstance(md_output_hook, (tuple, list)): +# [h.remove() for h in md_output_hook] +# [h.remove() for h in fm_output_hook] +# else: +# md_output_hook.remove() +# fm_output_hook.remove() + +# md_output_hook = None +# fm_output_hook = None + +# cur_md = self.models['md'].models_dict['main'] +# md_for_test = deepcopy(self.models['md'].models_dict['main']) +# val_acc = 0. + +# self.models['md'].models_dict['main'] = md_for_test +# self.models['md'].to_eval_mode() +# val_acc = self.models['md'].get_accuracy(val_loader) + +# self.models['md'].models_dict['main'] = cur_md + +# self.models['md'].save_model(os.path.join(self.res_save_dir, 'models/md_last.pt')) +# self.models['fm'].save_model(os.path.join(self.res_save_dir, 'models/fm_last.pt')) + +# if val_acc > best_avg_val_acc: +# best_avg_val_acc = val_acc +# self.models['md'].save_model(os.path.join(self.res_save_dir, 'models/md_best.pt')) +# self.models['fm'].save_model(os.path.join(self.res_save_dir, 'models/fm_best.pt')) + +# tb_writer.add_scalars(f'losses', dict(task=task_loss, distill=distill_loss, total=total_loss), iter_index) +# pbar.set_description(f'loss: {total_loss:.6f}') +# if (iter_index + 1) >= hyps['val_freq']: +# tb_writer.add_scalar(f'accs/val_acc', val_acc, iter_index) +# pbar.set_description(f'loss: {total_loss:.6f}, val_acc: {val_acc:.4f}') + +# code below is commented on 0716 17:49, because of a bug that the loss cannot be gradient decented +# (bug confirmed, why? I dont know :) + + + + + + + + +from typing import Any, Dict +from schema import Schema, Or +import schema +from data import Scenario, MergedDataset +from methods.base.alg import BaseAlg +from data import build_dataloader +from ..model import ElasticDNN_OfflineFMModel, ElasticDNN_OfflineMDModel +from ...model.base import ElasticDNNUtil +import torch.optim +import tqdm +import torch.nn.functional as F +from torch import nn +from utils.dl.common.env import create_tbwriter +import os +import random +import numpy as np +from copy import deepcopy +from utils.dl.common.model import LayerActivation2, get_module +from utils.common.log import logger +from torchvision.transforms import Compose + +class ElasticDNN_MDPretrainingWoFBSAlg(BaseAlg): + """ + TODO: fine-tuned FM -> init MD -> trained MD -> construct indexes (only between similar weights) and fine-tune + """ + def get_required_models_schema(self) -> Schema: + return Schema({ + 'fm': ElasticDNN_OfflineFMModel, + 'md': ElasticDNN_OfflineMDModel + }) + + def get_required_hyp_schema(self) -> Schema: + from schema import Optional + return Schema({ + 'launch_tbboard': bool, + + 'samples_size': any, + 'generate_md_width_ratio': int, + + 'train_batch_size': int, + 'val_batch_size': int, + 'num_workers': int, + 'optimizer': str, + 'optimizer_args': dict, + 'scheduler': str, + 'scheduler_args': dict, + 'num_iters': int, + 'val_freq': int, + 'distill_loss_weight': float, + + Optional('transform'): Compose, + }) + + def run(self, scenario: Scenario, hyps: Dict, collate_fn=None) -> Dict[str, Any]: + super().run(scenario, hyps) + + assert isinstance(self.models['md'], ElasticDNN_OfflineMDModel) # for auto completion + assert isinstance(self.models['fm'], ElasticDNN_OfflineFMModel) # for auto completion + + # 1. add FBS + device = self.models['md'].device + + if self.models['md'].models_dict['main'] == -1: + logger.info(f'init master DNN by reducing width of an adapted foundation model (already tuned by LoRA)...') + + before_fm_model = deepcopy(self.models['fm'].models_dict['main']) + lora_util = self.models['fm'].get_lora_util() + + sample = hyps['samples_size'] + if isinstance(sample, (tuple, list)) and isinstance(sample[0], int): + sample = torch.rand(hyps['samples_size']).to(device) + + lora_absorbed_fm_model = lora_util.absorb_lora_and_recover_net_structure(self.models['fm'].models_dict['main'], + sample) + self.models['fm'].models_dict['main'] = lora_absorbed_fm_model + master_dnn = self.models['fm'].generate_md_by_reducing_width(hyps['generate_md_width_ratio'], + sample) + self.models['fm'].models_dict['main'] = before_fm_model + + self.models['md'].models_dict['main'] = master_dnn + self.models['md'].to(device) + + # 2. train (knowledge distillation, index relationship) + if 'transform' in hyps.keys(): + offline_datasets = scenario.get_offline_datasets(transform=hyps['transform']) + else: + offline_datasets = scenario.get_offline_datasets() + train_dataset = MergedDataset([d['train'] for d in offline_datasets.values()]) + val_dataset = MergedDataset([d['val'] for d in offline_datasets.values()]) + train_loader = iter(build_dataloader(train_dataset, hyps['train_batch_size'], hyps['num_workers'], + True, None, collate_fn=collate_fn)) + val_loader = build_dataloader(val_dataset, hyps['val_batch_size'], hyps['num_workers'], + False, False, collate_fn=collate_fn) + + # logger.info(f'FM acc: {self.models["fm"].get_accuracy(val_loader):.4f}') + + # 2.1 train whole master DNN (knowledge distillation) + for p in master_dnn.parameters(): + p.requires_grad = True + self.models['md'].to_train_mode() + + optimizer = torch.optim.__dict__[hyps['optimizer']]([ + {'params': self.models['md'].models_dict['main'].parameters(), **hyps['optimizer_args']} + ]) + scheduler = torch.optim.lr_scheduler.__dict__[hyps['scheduler']](optimizer, **hyps['scheduler_args']) + tb_writer = create_tbwriter(os.path.join(self.res_save_dir, 'tb_log'), launch_tbboard=hyps['launch_tbboard']) + pbar = tqdm.tqdm(range(hyps['num_iters']), dynamic_ncols=True) + best_avg_val_acc = 0. + + md_output_hook = None + + for iter_index in pbar: + self.models['md'].to_train_mode() + self.models['fm'].to_eval_mode() + + # rand_sparsity = random.random() * (hyps['max_sparsity'] - hyps['min_sparsity']) + hyps['min_sparsity'] + # elastic_dnn_util.set_master_dnn_sparsity(self.models['md'].models_dict['main'], rand_sparsity) + if md_output_hook is None: + md_output_hook = self.models['md'].get_feature_hook() + fm_output_hook = self.models['fm'].get_feature_hook() + + x, y = next(train_loader) + if isinstance(x, dict): + for k, v in x.items(): + if isinstance(v, torch.Tensor): + x[k] = v.to(device) + y = y.to(device) + else: + x, y = x.to(device), y.to(device) + + with torch.no_grad(): + fm_output = self.models['fm'].infer(x) + task_loss = self.models['md'].forward_to_get_task_loss(x, y) + + md_output = md_output_hook.output + fm_output = fm_output_hook.output + + distill_loss = hyps['distill_loss_weight'] * self.models['md'].get_distill_loss(md_output, fm_output) + total_loss = task_loss + distill_loss + + optimizer.zero_grad() + total_loss.backward() + optimizer.step() + scheduler.step() + + if (iter_index + 1) % hyps['val_freq'] == 0: + + # elastic_dnn_util.clear_cached_channel_attention_in_master_dnn(self.models['md'].models_dict['main']) + md_output_hook.remove() + md_output_hook = None + fm_output_hook.remove() + fm_output_hook = None + + cur_md = self.models['md'].models_dict['main'] + md_for_test = deepcopy(self.models['md'].models_dict['main']) + val_acc = 0. + + self.models['md'].models_dict['main'] = md_for_test + self.models['md'].to_eval_mode() + val_acc = self.models['md'].get_accuracy(val_loader) + + self.models['md'].models_dict['main'] = cur_md + + self.models['md'].save_model(os.path.join(self.res_save_dir, 'models/md_last.pt')) + self.models['fm'].save_model(os.path.join(self.res_save_dir, 'models/fm_last.pt')) + + if val_acc > best_avg_val_acc: + best_avg_val_acc = val_acc + self.models['md'].save_model(os.path.join(self.res_save_dir, 'models/md_best.pt')) + self.models['fm'].save_model(os.path.join(self.res_save_dir, 'models/fm_best.pt')) + + tb_writer.add_scalars(f'losses', dict(task=task_loss, distill=distill_loss, total=total_loss), iter_index) + pbar.set_description(f'loss: {total_loss:.6f}') + if (iter_index + 1) >= hyps['val_freq']: + tb_writer.add_scalar(f'accs/val_acc', val_acc, iter_index) + pbar.set_description(f'loss: {total_loss:.6f}, val_acc: {val_acc:.4f}') + \ No newline at end of file diff --git a/methods/elasticdnn/api/algs/md_pretraining_wo_fbs_clip_debug.py b/methods/elasticdnn/api/algs/md_pretraining_wo_fbs_clip_debug.py new file mode 100644 index 0000000000000000000000000000000000000000..a1b0f3b54fa9bc00d99dcb5de64b929c90d904da --- /dev/null +++ b/methods/elasticdnn/api/algs/md_pretraining_wo_fbs_clip_debug.py @@ -0,0 +1,412 @@ +# from typing import Any, Dict +# from schema import Schema, Or +# import schema +# from data import Scenario, MergedDataset +# from methods.base.alg import BaseAlg +# from data import build_dataloader +# from ..model import ElasticDNN_OfflineFMModel, ElasticDNN_OfflineMDModel +# from ...model.base import ElasticDNNUtil +# import torch.optim +# import tqdm +# import torch.nn.functional as F +# from torch import nn +# from utils.dl.common.env import create_tbwriter +# import os +# import random +# import numpy as np +# from copy import deepcopy +# from utils.dl.common.model import LayerActivation2, get_module +# from utils.common.log import logger + + +# class ElasticDNN_MDPretrainingWoFBSAlg(BaseAlg): +# """ +# TODO: fine-tuned FM -> init MD -> trained MD -> construct indexes (only between similar weights) and fine-tune +# """ +# def get_required_models_schema(self) -> Schema: +# return Schema({ +# 'fm': ElasticDNN_OfflineFMModel, +# 'md': ElasticDNN_OfflineMDModel +# }) + +# def get_required_hyp_schema(self) -> Schema: +# return Schema({ +# 'launch_tbboard': bool, + +# 'samples_size': object, +# 'generate_md_width_ratio': int, + +# 'train_batch_size': int, +# 'val_batch_size': int, +# 'num_workers': int, +# 'optimizer': str, +# 'optimizer_args': dict, +# 'scheduler': str, +# 'scheduler_args': dict, +# 'num_iters': int, +# 'val_freq': int, +# 'distill_loss_weight': float +# }) + +# def run(self, scenario: Scenario, hyps: Dict) -> Dict[str, Any]: +# super().run(scenario, hyps) + +# assert isinstance(self.models['md'], ElasticDNN_OfflineMDModel) # for auto completion +# assert isinstance(self.models['fm'], ElasticDNN_OfflineFMModel) # for auto completion + +# # 1. add FBS +# device = self.models['md'].device + +# if self.models['md'].models_dict['main'] == -1: +# logger.info(f'init master DNN by reducing width of an adapted foundation model (already tuned by LoRA)...') + +# before_fm_model = deepcopy(self.models['fm'].models_dict['main']) +# lora_util = self.models['fm'].get_lora_util() + +# sample = hyps['samples_size'] +# if isinstance(sample, (tuple, list)) and isinstance(sample[0], int): +# sample = torch.rand(hyps['samples_size']).to(device) + +# lora_absorbed_fm_model = lora_util.absorb_lora_and_recover_net_structure(self.models['fm'].models_dict['main'], +# sample) +# self.models['fm'].models_dict['main'] = lora_absorbed_fm_model +# master_dnn = self.models['fm'].generate_md_by_reducing_width(hyps['generate_md_width_ratio'], +# sample) +# self.models['fm'].models_dict['main'] = before_fm_model + +# self.models['md'].models_dict['main'] = master_dnn +# self.models['md'].to(device) + + + +# # 2. train (knowledge distillation, index relationship) +# offline_datasets = scenario.get_offline_datasets() +# train_dataset = MergedDataset([d['train'] for d in offline_datasets.values()]) +# val_dataset = MergedDataset([d['val'] for d in offline_datasets.values()]) +# train_loader = iter(build_dataloader(train_dataset, hyps['train_batch_size'], hyps['num_workers'], +# True, None)) +# val_loader = build_dataloader(val_dataset, hyps['val_batch_size'], hyps['num_workers'], +# False, False) + +# # val_acc = self.models['md'].get_accuracy(val_loader) +# # print(val_acc) +# # exit() + +# # 2.1 train whole master DNN (knowledge distillation) +# self.models['md'].to_train_mode() +# for p in master_dnn.parameters(): +# p.requires_grad = True + +# if hasattr(self.models['md'], 'get_trained_params'): +# trained_p = self.models['md'].get_trained_params() +# logger.info(f'use custom trained parameters!!') +# else: +# trained_p = self.models['md'].models_dict['main'].parameters() +# for p in trained_p: +# p.requires_grad = True +# optimizer = torch.optim.__dict__[hyps['optimizer']]([ +# {'params': trained_p, **hyps['optimizer_args']} +# ]) +# scheduler = torch.optim.lr_scheduler.__dict__[hyps['scheduler']](optimizer, **hyps['scheduler_args']) +# tb_writer = create_tbwriter(os.path.join(self.res_save_dir, 'tb_log'), launch_tbboard=hyps['launch_tbboard']) +# pbar = tqdm.tqdm(range(hyps['num_iters']), dynamic_ncols=True) +# best_avg_val_acc = 0. + +# md_output_hook = None + +# for iter_index in pbar: +# self.models['md'].to_train_mode() +# self.models['fm'].to_eval_mode() + +# # rand_sparsity = random.random() * (hyps['max_sparsity'] - hyps['min_sparsity']) + hyps['min_sparsity'] +# # elastic_dnn_util.set_master_dnn_sparsity(self.models['md'].models_dict['main'], rand_sparsity) +# if md_output_hook is None: +# md_output_hook = self.models['md'].get_feature_hook() +# fm_output_hook = self.models['fm'].get_feature_hook() + +# x, y = next(train_loader) +# if isinstance(x, dict): +# for k, v in x.items(): +# if isinstance(v, torch.Tensor): +# x[k] = v.to(device) +# y = y.to(device) +# else: +# x, y = x.to(device), y.to(device) + +# with torch.no_grad(): +# fm_output = self.models['fm'].infer(x) +# task_loss = self.models['md'].forward_to_get_task_loss(x, y) + +# if isinstance(md_output_hook, (tuple, list)): +# distill_loss = 0. +# for h1, h2 in zip(md_output_hook, fm_output_hook): +# md_output = h1.output +# fm_output = h2.output +# distill_loss += hyps['distill_loss_weight'] * self.models['md'].get_distill_loss(md_output, fm_output) +# else: +# md_output = md_output_hook.output +# fm_output = fm_output_hook.output +# distill_loss = hyps['distill_loss_weight'] * self.models['md'].get_distill_loss(md_output, fm_output) + +# total_loss = task_loss + distill_loss + +# optimizer.zero_grad() +# total_loss.backward() + +# # for n, p in self.models['md'].models_dict['main'].named_parameters(): +# # if p.grad is not None: +# # print(n) +# # exit() + +# optimizer.step() +# scheduler.step() + +# if (iter_index + 1) % hyps['val_freq'] == 0: + +# # elastic_dnn_util.clear_cached_channel_attention_in_master_dnn(self.models['md'].models_dict['main']) +# if isinstance(md_output_hook, (tuple, list)): +# [h.remove() for h in md_output_hook] +# [h.remove() for h in fm_output_hook] +# else: +# md_output_hook.remove() +# fm_output_hook.remove() + +# md_output_hook = None +# fm_output_hook = None + +# cur_md = self.models['md'].models_dict['main'] +# md_for_test = deepcopy(self.models['md'].models_dict['main']) +# val_acc = 0. + +# self.models['md'].models_dict['main'] = md_for_test +# self.models['md'].to_eval_mode() +# val_acc = self.models['md'].get_accuracy(val_loader) + +# self.models['md'].models_dict['main'] = cur_md + +# self.models['md'].save_model(os.path.join(self.res_save_dir, 'models/md_last.pt')) +# self.models['fm'].save_model(os.path.join(self.res_save_dir, 'models/fm_last.pt')) + +# if val_acc > best_avg_val_acc: +# best_avg_val_acc = val_acc +# self.models['md'].save_model(os.path.join(self.res_save_dir, 'models/md_best.pt')) +# self.models['fm'].save_model(os.path.join(self.res_save_dir, 'models/fm_best.pt')) + +# tb_writer.add_scalars(f'losses', dict(task=task_loss, distill=distill_loss, total=total_loss), iter_index) +# pbar.set_description(f'loss: {total_loss:.6f}') +# if (iter_index + 1) >= hyps['val_freq']: +# tb_writer.add_scalar(f'accs/val_acc', val_acc, iter_index) +# pbar.set_description(f'loss: {total_loss:.6f}, val_acc: {val_acc:.4f}') + +# code below is commented on 0716 17:49, because of a bug that the loss cannot be gradient decented +# (bug confirmed, why? I dont know :) + + + + + + + + +from typing import Any, Dict +from schema import Schema, Or +import schema +from data import Scenario, MergedDataset +from methods.base.alg import BaseAlg +from data import build_dataloader +from ..model import ElasticDNN_OfflineFMModel, ElasticDNN_OfflineMDModel +from ...model.base import ElasticDNNUtil +import torch.optim +import tqdm +import torch.nn.functional as F +from torch import nn +from utils.dl.common.env import create_tbwriter +import os +import random +import numpy as np +from copy import deepcopy +from utils.dl.common.model import LayerActivation2, get_module +from utils.common.log import logger +from data import split_dataset + + +class ElasticDNN_MDPretrainingWoFBSAlg(BaseAlg): + """ + TODO: fine-tuned FM -> init MD -> trained MD -> construct indexes (only between similar weights) and fine-tune + """ + def get_required_models_schema(self) -> Schema: + return Schema({ + 'fm': ElasticDNN_OfflineFMModel, + 'md': ElasticDNN_OfflineMDModel + }) + + def get_required_hyp_schema(self) -> Schema: + return Schema({ + 'launch_tbboard': bool, + + 'samples_size': any, + 'generate_md_width_ratio': int, + + 'train_batch_size': int, + 'val_batch_size': int, + 'num_workers': int, + 'optimizer': str, + 'optimizer_args': dict, + 'scheduler': str, + 'scheduler_args': dict, + 'num_iters': int, + 'val_freq': int, + 'distill_loss_weight': float + }) + + def run(self, scenario: Scenario, hyps: Dict) -> Dict[str, Any]: + super().run(scenario, hyps) + + assert isinstance(self.models['md'], ElasticDNN_OfflineMDModel) # for auto completion + assert isinstance(self.models['fm'], ElasticDNN_OfflineFMModel) # for auto completion + + # 1. add FBS + device = self.models['md'].device + + if self.models['md'].models_dict['main'] == -1: + logger.info(f'init master DNN by reducing width of an adapted foundation model (already tuned by LoRA)...') + + before_fm_model = deepcopy(self.models['fm'].models_dict['main']) + lora_util = self.models['fm'].get_lora_util() + + sample = hyps['samples_size'] + if isinstance(sample, (tuple, list)) and isinstance(sample[0], int): + sample = torch.rand(hyps['samples_size']).to(device) + + lora_absorbed_fm_model = lora_util.absorb_lora_and_recover_net_structure(self.models['fm'].models_dict['main'], + sample) + self.models['fm'].models_dict['main'] = lora_absorbed_fm_model + master_dnn = self.models['fm'].generate_md_by_reducing_width(hyps['generate_md_width_ratio'], + sample) + self.models['fm'].models_dict['main'] = before_fm_model + + self.models['md'].models_dict['main'] = master_dnn + self.models['md'].to(device) + + # 2. train (knowledge distillation, index relationship) + offline_datasets = scenario.get_offline_datasets() + train_dataset = MergedDataset([d['train'] for d in offline_datasets.values()]) + val_dataset = MergedDataset([d['val'] for d in offline_datasets.values()]) + train_loader = iter(build_dataloader(train_dataset, hyps['train_batch_size'], hyps['num_workers'], + True, None)) + val_loader = build_dataloader(val_dataset, hyps['val_batch_size'], hyps['num_workers'], + False, False) + + # logger.info(f'FM acc: {self.models["fm"].get_accuracy(val_loader):.4f}') + + # 2.1 train whole master DNN (knowledge distillation) + + # self.models['md'].models_dict['main'] = master_dnn = nn.DataParallel(master_dnn.to(device), device_ids=['cuda:0'], output_device=['cuda:0']) + + for p in master_dnn.model.parameters(): + p.requires_grad = True + for p in master_dnn.model.text_model.parameters(): + p.requires_grad = False + self.models['md'].to_train_mode() + + optimizer = torch.optim.__dict__[hyps['optimizer']]([ + {'params': [p for n, p in self.models['md'].models_dict['main'].named_parameters() if 'text_model' not in n], **hyps['optimizer_args']} + ]) + scheduler = torch.optim.lr_scheduler.__dict__[hyps['scheduler']](optimizer, **hyps['scheduler_args']) + tb_writer = create_tbwriter(os.path.join(self.res_save_dir, 'tb_log'), launch_tbboard=hyps['launch_tbboard']) + pbar = tqdm.tqdm(range(hyps['num_iters']), dynamic_ncols=True) + best_avg_val_acc = 0. + + md_output_hook = None + + train_ratio = 0.005 + + for iter_index in pbar: + + if iter_index % 500 == 0: + train_ratio += 0.005 + + train_dataset = MergedDataset([d['train'] for d in offline_datasets.values()]) + train_dataset = split_dataset(train_dataset, max(hyps['train_batch_size'], int(train_ratio * len(train_dataset))))[0] + train_loader = iter(build_dataloader(train_dataset, hyps['train_batch_size'], hyps['num_workers'], True, None)) + + self.models['md'].to_train_mode() + # self.models['fm'].to_eval_mode() + + # rand_sparsity = random.random() * (hyps['max_sparsity'] - hyps['min_sparsity']) + hyps['min_sparsity'] + # elastic_dnn_util.set_master_dnn_sparsity(self.models['md'].models_dict['main'], rand_sparsity) + if md_output_hook is None: + md_output_hook = self.models['md'].get_feature_hook() + fm_output_hook = self.models['fm'].get_feature_hook() + + x, y = next(train_loader) + if isinstance(x, dict): + for k, v in x.items(): + if isinstance(v, torch.Tensor): + x[k] = v.to(device) + y = y.to(device) + else: + x, y = x.to(device), y.to(device) + + # with torch.no_grad(): + # fm_output = self.models['fm'].infer(x) + # task_loss = self.models['md'].forward_to_get_task_loss(x, y) + task_loss = 0. + with torch.no_grad(): + fm_output = self.models['fm'].forward_to_get_task_loss(x, y) + md_output = self.models['md'].forward_to_get_task_loss(x, y) + + if iter_index % 10 == 0: + print(fm_output, md_output) + # md_output = md_output_hook.output + # fm_output = fm_output_hook.output + + distill_loss = hyps['distill_loss_weight'] * self.models['md'].get_distill_loss(md_output, fm_output) + # distill_loss = 0. + + # logit_diff_loss = -((md_output - md_output.mean(dim=1).unsqueeze(1)) ** 2).sum() * 0.01 + logit_diff_loss = 0. + + total_loss = task_loss + distill_loss + logit_diff_loss + + optimizer.zero_grad() + total_loss.backward() + optimizer.step() + scheduler.step() + + if (iter_index + 1) % hyps['val_freq'] == 0: + + # elastic_dnn_util.clear_cached_channel_attention_in_master_dnn(self.models['md'].models_dict['main']) + md_output_hook.remove() + md_output_hook = None + fm_output_hook.remove() + fm_output_hook = None + + cur_md = self.models['md'].models_dict['main'] + md_for_test = deepcopy(self.models['md'].models_dict['main']) + val_acc = 0. + + self.models['md'].models_dict['main'] = md_for_test + self.models['md'].to_eval_mode() + val_acc = self.models['md'].get_accuracy(val_loader) + + self.models['md'].models_dict['main'] = cur_md + + self.models['md'].save_model(os.path.join(self.res_save_dir, 'models/md_last.pt')) + self.models['fm'].save_model(os.path.join(self.res_save_dir, 'models/fm_last.pt')) + + if val_acc > best_avg_val_acc: + best_avg_val_acc = val_acc + self.models['md'].save_model(os.path.join(self.res_save_dir, 'models/md_best.pt')) + self.models['fm'].save_model(os.path.join(self.res_save_dir, 'models/fm_best.pt')) + + tb_writer.add_scalars(f'losses', dict(task=task_loss, distill=distill_loss, logit_diff_loss=logit_diff_loss, total=total_loss), iter_index) + # tb_writer.add_scalar(f'lr', optimizer.param_groups[0]['lr'], iter_index) + tb_writer.add_scalar(f'lr', optimizer.param_groups[0]['lr'], iter_index) + tb_writer.add_scalar(f'train_data_size', len(train_dataset), iter_index) + + pbar.set_description(f'loss: {total_loss:.6f}, logit_diff_loss: {logit_diff_loss:.6f}') + if (iter_index + 1) >= hyps['val_freq']: + tb_writer.add_scalar(f'accs/val_acc', val_acc, iter_index) + pbar.set_description(f'loss: {total_loss:.6f}, logit_diff_loss: {logit_diff_loss:.6f}, val_acc: {val_acc:.4f}') + \ No newline at end of file diff --git a/methods/elasticdnn/api/algs/md_pretraining_wo_fbs_gpt_neo.py b/methods/elasticdnn/api/algs/md_pretraining_wo_fbs_gpt_neo.py new file mode 100644 index 0000000000000000000000000000000000000000..5bfb54461ca1c3793b0e02059752864d06fd1d14 --- /dev/null +++ b/methods/elasticdnn/api/algs/md_pretraining_wo_fbs_gpt_neo.py @@ -0,0 +1,404 @@ +# from typing import Any, Dict +# from schema import Schema, Or +# import schema +# from data import Scenario, MergedDataset +# from methods.base.alg import BaseAlg +# from data import build_dataloader +# from ..model import ElasticDNN_OfflineFMModel, ElasticDNN_OfflineMDModel +# from ...model.base import ElasticDNNUtil +# import torch.optim +# import tqdm +# import torch.nn.functional as F +# from torch import nn +# from utils.dl.common.env import create_tbwriter +# import os +# import random +# import numpy as np +# from copy import deepcopy +# from utils.dl.common.model import LayerActivation2, get_module +# from utils.common.log import logger + + +# class ElasticDNN_MDPretrainingWoFBSAlg(BaseAlg): +# """ +# TODO: fine-tuned FM -> init MD -> trained MD -> construct indexes (only between similar weights) and fine-tune +# """ +# def get_required_models_schema(self) -> Schema: +# return Schema({ +# 'fm': ElasticDNN_OfflineFMModel, +# 'md': ElasticDNN_OfflineMDModel +# }) + +# def get_required_hyp_schema(self) -> Schema: +# return Schema({ +# 'launch_tbboard': bool, + +# 'samples_size': object, +# 'generate_md_width_ratio': int, + +# 'train_batch_size': int, +# 'val_batch_size': int, +# 'num_workers': int, +# 'optimizer': str, +# 'optimizer_args': dict, +# 'scheduler': str, +# 'scheduler_args': dict, +# 'num_iters': int, +# 'val_freq': int, +# 'distill_loss_weight': float +# }) + +# def run(self, scenario: Scenario, hyps: Dict) -> Dict[str, Any]: +# super().run(scenario, hyps) + +# assert isinstance(self.models['md'], ElasticDNN_OfflineMDModel) # for auto completion +# assert isinstance(self.models['fm'], ElasticDNN_OfflineFMModel) # for auto completion + +# # 1. add FBS +# device = self.models['md'].device + +# if self.models['md'].models_dict['main'] == -1: +# logger.info(f'init master DNN by reducing width of an adapted foundation model (already tuned by LoRA)...') + +# before_fm_model = deepcopy(self.models['fm'].models_dict['main']) +# lora_util = self.models['fm'].get_lora_util() + +# sample = hyps['samples_size'] +# if isinstance(sample, (tuple, list)) and isinstance(sample[0], int): +# sample = torch.rand(hyps['samples_size']).to(device) + +# lora_absorbed_fm_model = lora_util.absorb_lora_and_recover_net_structure(self.models['fm'].models_dict['main'], +# sample) +# self.models['fm'].models_dict['main'] = lora_absorbed_fm_model +# master_dnn = self.models['fm'].generate_md_by_reducing_width(hyps['generate_md_width_ratio'], +# sample) +# self.models['fm'].models_dict['main'] = before_fm_model + +# self.models['md'].models_dict['main'] = master_dnn +# self.models['md'].to(device) + + + +# # 2. train (knowledge distillation, index relationship) +# offline_datasets = scenario.get_offline_datasets() +# train_dataset = MergedDataset([d['train'] for d in offline_datasets.values()]) +# val_dataset = MergedDataset([d['val'] for d in offline_datasets.values()]) +# train_loader = iter(build_dataloader(train_dataset, hyps['train_batch_size'], hyps['num_workers'], +# True, None)) +# val_loader = build_dataloader(val_dataset, hyps['val_batch_size'], hyps['num_workers'], +# False, False) + +# # val_acc = self.models['md'].get_accuracy(val_loader) +# # print(val_acc) +# # exit() + +# # 2.1 train whole master DNN (knowledge distillation) +# self.models['md'].to_train_mode() +# for p in master_dnn.parameters(): +# p.requires_grad = True + +# if hasattr(self.models['md'], 'get_trained_params'): +# trained_p = self.models['md'].get_trained_params() +# logger.info(f'use custom trained parameters!!') +# else: +# trained_p = self.models['md'].models_dict['main'].parameters() +# for p in trained_p: +# p.requires_grad = True +# optimizer = torch.optim.__dict__[hyps['optimizer']]([ +# {'params': trained_p, **hyps['optimizer_args']} +# ]) +# scheduler = torch.optim.lr_scheduler.__dict__[hyps['scheduler']](optimizer, **hyps['scheduler_args']) +# tb_writer = create_tbwriter(os.path.join(self.res_save_dir, 'tb_log'), launch_tbboard=hyps['launch_tbboard']) +# pbar = tqdm.tqdm(range(hyps['num_iters']), dynamic_ncols=True) +# best_avg_val_acc = 0. + +# md_output_hook = None + +# for iter_index in pbar: +# self.models['md'].to_train_mode() +# self.models['fm'].to_eval_mode() + +# # rand_sparsity = random.random() * (hyps['max_sparsity'] - hyps['min_sparsity']) + hyps['min_sparsity'] +# # elastic_dnn_util.set_master_dnn_sparsity(self.models['md'].models_dict['main'], rand_sparsity) +# if md_output_hook is None: +# md_output_hook = self.models['md'].get_feature_hook() +# fm_output_hook = self.models['fm'].get_feature_hook() + +# x, y = next(train_loader) +# if isinstance(x, dict): +# for k, v in x.items(): +# if isinstance(v, torch.Tensor): +# x[k] = v.to(device) +# y = y.to(device) +# else: +# x, y = x.to(device), y.to(device) + +# with torch.no_grad(): +# fm_output = self.models['fm'].infer(x) +# task_loss = self.models['md'].forward_to_get_task_loss(x, y) + +# if isinstance(md_output_hook, (tuple, list)): +# distill_loss = 0. +# for h1, h2 in zip(md_output_hook, fm_output_hook): +# md_output = h1.output +# fm_output = h2.output +# distill_loss += hyps['distill_loss_weight'] * self.models['md'].get_distill_loss(md_output, fm_output) +# else: +# md_output = md_output_hook.output +# fm_output = fm_output_hook.output +# distill_loss = hyps['distill_loss_weight'] * self.models['md'].get_distill_loss(md_output, fm_output) + +# total_loss = task_loss + distill_loss + +# optimizer.zero_grad() +# total_loss.backward() + +# # for n, p in self.models['md'].models_dict['main'].named_parameters(): +# # if p.grad is not None: +# # print(n) +# # exit() + +# optimizer.step() +# scheduler.step() + +# if (iter_index + 1) % hyps['val_freq'] == 0: + +# # elastic_dnn_util.clear_cached_channel_attention_in_master_dnn(self.models['md'].models_dict['main']) +# if isinstance(md_output_hook, (tuple, list)): +# [h.remove() for h in md_output_hook] +# [h.remove() for h in fm_output_hook] +# else: +# md_output_hook.remove() +# fm_output_hook.remove() + +# md_output_hook = None +# fm_output_hook = None + +# cur_md = self.models['md'].models_dict['main'] +# md_for_test = deepcopy(self.models['md'].models_dict['main']) +# val_acc = 0. + +# self.models['md'].models_dict['main'] = md_for_test +# self.models['md'].to_eval_mode() +# val_acc = self.models['md'].get_accuracy(val_loader) + +# self.models['md'].models_dict['main'] = cur_md + +# self.models['md'].save_model(os.path.join(self.res_save_dir, 'models/md_last.pt')) +# self.models['fm'].save_model(os.path.join(self.res_save_dir, 'models/fm_last.pt')) + +# if val_acc > best_avg_val_acc: +# best_avg_val_acc = val_acc +# self.models['md'].save_model(os.path.join(self.res_save_dir, 'models/md_best.pt')) +# self.models['fm'].save_model(os.path.join(self.res_save_dir, 'models/fm_best.pt')) + +# tb_writer.add_scalars(f'losses', dict(task=task_loss, distill=distill_loss, total=total_loss), iter_index) +# pbar.set_description(f'loss: {total_loss:.6f}') +# if (iter_index + 1) >= hyps['val_freq']: +# tb_writer.add_scalar(f'accs/val_acc', val_acc, iter_index) +# pbar.set_description(f'loss: {total_loss:.6f}, val_acc: {val_acc:.4f}') + +# code below is commented on 0716 17:49, because of a bug that the loss cannot be gradient decented +# (bug confirmed, why? I dont know :) + + + + + + + + +from typing import Any, Dict +from schema import Schema, Or +import schema +from data import Scenario, MergedDataset +from methods.base.alg import BaseAlg +from data import build_dataloader +from ..model import ElasticDNN_OfflineFMModel, ElasticDNN_OfflineMDModel +from ...model.base import ElasticDNNUtil +import torch.optim +import tqdm +import torch.nn.functional as F +from torch import nn +from utils.dl.common.env import create_tbwriter +import os +import random +import numpy as np +from copy import deepcopy +from utils.dl.common.model import LayerActivation2, get_module +from utils.common.log import logger +from torchvision.transforms import Compose + +class ElasticDNN_MDPretrainingWoFBSAlg(BaseAlg): + """ + TODO: fine-tuned FM -> init MD -> trained MD -> construct indexes (only between similar weights) and fine-tune + """ + def get_required_models_schema(self) -> Schema: + return Schema({ + 'fm': ElasticDNN_OfflineFMModel, + 'md': ElasticDNN_OfflineMDModel + }) + + def get_required_hyp_schema(self) -> Schema: + from schema import Optional + return Schema({ + 'launch_tbboard': bool, + + 'samples_size': any, + 'generate_md_width_ratio': int, + # 'generate_md_layers': list, # <= 24 + + + 'train_batch_size': int, + 'val_batch_size': int, + 'num_workers': int, + 'optimizer': str, + 'optimizer_args': dict, + 'scheduler': str, + 'scheduler_args': dict, + 'num_iters': int, + 'val_freq': int, + 'distill_loss_weight': float, + + Optional('transform'): Compose, + }) + + def run(self, scenario: Scenario, hyps: Dict, collate_fn=None) -> Dict[str, Any]: + super().run(scenario, hyps) + + assert isinstance(self.models['md'], ElasticDNN_OfflineMDModel) # for auto completion + assert isinstance(self.models['fm'], ElasticDNN_OfflineFMModel) # for auto completion + + # 1. add FBS + device = self.models['md'].device + + if self.models['md'].models_dict['main'] == -1: + logger.info(f'init master DNN by reducing width of an adapted foundation model (already tuned by LoRA)...') + + before_fm_model = deepcopy(self.models['fm'].models_dict['main']) + lora_util = self.models['fm'].get_lora_util() + + sample = hyps['samples_size'] + if isinstance(sample, (tuple, list)) and isinstance(sample[0], int): + sample = torch.rand(hyps['samples_size']).to(device) + + lora_absorbed_fm_model = lora_util.absorb_lora_and_recover_net_structure(self.models['fm'].models_dict['main'], + sample) + self.models['fm'].models_dict['main'] = lora_absorbed_fm_model + master_dnn = self.models['fm'].generate_md_by_reducing_width(hyps['generate_md_width_ratio'], + sample) + + # master_dnn = self.models['fm'].generate_md_by_reducing_layers(hyps['generate_md_layers'], + # sample) + + self.models['fm'].models_dict['main'] = before_fm_model + + self.models['md'].models_dict['main'] = master_dnn + self.models['md'].to(device) + + # 2. train (knowledge distillation, index relationship) + if 'transform' in hyps.keys(): + offline_datasets = scenario.get_offline_datasets(transform=hyps['transform']) + else: + offline_datasets = scenario.get_offline_datasets() + train_dataset = MergedDataset([d['train'] for d in offline_datasets.values()]) + val_dataset = MergedDataset([d['val'] for d in offline_datasets.values()]) + train_loader = iter(build_dataloader(train_dataset, hyps['train_batch_size'], hyps['num_workers'], + True, None, collate_fn=collate_fn)) + val_loader = build_dataloader(val_dataset, hyps['val_batch_size'], hyps['num_workers'], + False, False, collate_fn=collate_fn) + + # logger.info(f'FM acc: {self.models["fm"].get_accuracy(val_loader):.4f}') + + # 2.1 train whole master DNN (knowledge distillation) + for p in master_dnn.parameters(): + p.requires_grad = True + self.models['md'].to_train_mode() + + optimizer = torch.optim.__dict__[hyps['optimizer']]([ + {'params': self.models['md'].models_dict['main'].parameters(), **hyps['optimizer_args']} + ]) + scheduler = torch.optim.lr_scheduler.__dict__[hyps['scheduler']](optimizer, **hyps['scheduler_args']) + tb_writer = create_tbwriter(os.path.join(self.res_save_dir, 'tb_log'), launch_tbboard=hyps['launch_tbboard']) + pbar = tqdm.tqdm(range(hyps['num_iters']), dynamic_ncols=True) + best_avg_val_acc = 0. + + md_output_hook = None + + for iter_index in pbar: + self.models['md'].to_train_mode() + self.models['fm'].to_eval_mode() + + # rand_sparsity = random.random() * (hyps['max_sparsity'] - hyps['min_sparsity']) + hyps['min_sparsity'] + # elastic_dnn_util.set_master_dnn_sparsity(self.models['md'].models_dict['main'], rand_sparsity) + + x, y = next(train_loader) + + if len(x) == 0: + continue + + if md_output_hook is None: + md_output_hook = self.models['md'].get_feature_hooks() + fm_output_hook = self.models['fm'].get_feature_hooks() + + + + if isinstance(x, dict): + for k, v in x.items(): + if isinstance(v, torch.Tensor): + x[k] = v.to(device) + y = y.to(device) + else: + x, y = x.to(device), y.to(device) + + with torch.no_grad(): + fm_output = self.models['fm'].infer(x) + task_loss = self.models['md'].forward_to_get_task_loss(x, y) + + # md_output = md_output_hook.output + # fm_output = fm_output_hook.output + + distill_loss = self.models['md'].get_distill_loss(md_output_hook, fm_output_hook, x['attention_mask']) + total_loss = (1 - hyps['distill_loss_weight']) * task_loss + hyps['distill_loss_weight'] * distill_loss + + optimizer.zero_grad() + total_loss.backward() + optimizer.step() + scheduler.step() + + if (iter_index + 1) % hyps['val_freq'] == 0: + + # elastic_dnn_util.clear_cached_channel_attention_in_master_dnn(self.models['md'].models_dict['main']) + md_output_hook['head'].remove() + for hidden in md_output_hook['hiddens']: + hidden.remove() + md_output_hook = None + fm_output_hook['head'].remove() + for hidden in fm_output_hook['hiddens']: + hidden.remove() + fm_output_hook = None + + cur_md = self.models['md'].models_dict['main'] + md_for_test = deepcopy(self.models['md'].models_dict['main']) + val_acc = 0. + + self.models['md'].models_dict['main'] = md_for_test + self.models['md'].to_eval_mode() + val_acc = self.models['md'].get_accuracy(val_loader) + + self.models['md'].models_dict['main'] = cur_md + + self.models['md'].save_model(os.path.join(self.res_save_dir, 'models/md_last.pt')) + # self.models['fm'].save_model(os.path.join(self.res_save_dir, 'models/fm_last.pt')) + + if val_acc > best_avg_val_acc: + best_avg_val_acc = val_acc + self.models['md'].save_model(os.path.join(self.res_save_dir, 'models/md_best.pt')) + # self.models['fm'].save_model(os.path.join(self.res_save_dir, 'models/fm_best.pt')) + + tb_writer.add_scalars(f'losses', dict(task=task_loss, distill=distill_loss, total=total_loss), iter_index) + pbar.set_description(f'loss: {total_loss:.6f}') + if (iter_index + 1) >= hyps['val_freq']: + tb_writer.add_scalar(f'accs/val_acc', val_acc, iter_index) + pbar.set_description(f'loss: {total_loss:.6f}, val_acc: {val_acc:.4f}') + \ No newline at end of file diff --git a/methods/elasticdnn/api/model.py b/methods/elasticdnn/api/model.py new file mode 100644 index 0000000000000000000000000000000000000000..c88b1693c317fe4000798b38000e17da13475b52 --- /dev/null +++ b/methods/elasticdnn/api/model.py @@ -0,0 +1,822 @@ +from typing import List +import torch +from methods.base.model import BaseModel +import tqdm +from torch import nn +import torch.nn.functional as F +from abc import abstractmethod +from methods.elasticdnn.model.base import ElasticDNNUtil +from methods.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from methods.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util + +from utils.dl.common.model import LayerActivation + + +class ElasticDNN_OfflineFMModel(BaseModel): + def get_required_model_components(self) -> List[str]: + return ['main'] + + @abstractmethod + def generate_md_by_reducing_width(self, reducing_width_ratio, samples: torch.Tensor): + pass + + @abstractmethod + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + raise NotImplementedError + + @abstractmethod + def get_feature_hook(self) -> LayerActivation: + pass + + @abstractmethod + def get_elastic_dnn_util(self) -> ElasticDNNUtil: + pass + + @abstractmethod + def get_lora_util(self) -> FMLoRA_Util: + pass + + @abstractmethod + def get_task_head_params(self): + pass + + +class ElasticDNN_OfflineClsFMModel(ElasticDNN_OfflineFMModel): + def get_accuracy(self, test_loader, *args, **kwargs): + acc = 0 + sample_num = 0 + + self.to_eval_mode() + + with torch.no_grad(): + pbar = tqdm.tqdm(enumerate(test_loader), total=len(test_loader), dynamic_ncols=True, leave=False) + for batch_index, (x, y) in pbar: + x, y = x.to(self.device), y.to(self.device) + output = self.infer(x) + pred = F.softmax(output.logits, dim=1).argmax(dim=1) + #correct = torch.eq(torch.argmax(output.logits,dim = 1), y).sum().item() + correct = torch.eq(pred, y).sum().item() + acc += correct + sample_num += len(y) + + pbar.set_description(f'cur_batch_total: {len(y)}, cur_batch_correct: {correct}, ' + f'cur_batch_acc: {(correct / len(y)):.4f}') + + acc /= sample_num + return acc + + def infer(self, x, *args, **kwargs): + return self.models_dict['main'](x) + + +import numpy as np +class StreamSegMetrics: + """ + Stream Metrics for Semantic Segmentation Task + """ + def __init__(self, n_classes): + self.n_classes = n_classes + self.confusion_matrix = np.zeros((n_classes, n_classes)) + + def update(self, label_trues, label_preds): + for lt, lp in zip(label_trues, label_preds): + self.confusion_matrix += self._fast_hist( lt.flatten(), lp.flatten() ) + + @staticmethod + def to_str(results): + string = "\n" + for k, v in results.items(): + if k!="Class IoU": + string += "%s: %f\n"%(k, v) + + return string + + def _fast_hist(self, label_true, label_pred): + mask = (label_true >= 0) & (label_true < self.n_classes) + hist = np.bincount( + self.n_classes * label_true[mask].astype(int) + label_pred[mask], + minlength=self.n_classes ** 2, + ).reshape(self.n_classes, self.n_classes) + return hist + + def get_results(self): + """Returns accuracy score evaluation result. + - overall accuracy + - mean accuracy + - mean IU + - fwavacc + """ + hist = self.confusion_matrix + acc = np.diag(hist).sum() / hist.sum() + acc_cls = np.diag(hist) / hist.sum(axis=1) + acc_cls = np.nanmean(acc_cls) + iu = np.diag(hist) / (hist.sum(axis=1) + hist.sum(axis=0) - np.diag(hist)) + mean_iu = np.nanmean(iu) + freq = hist.sum(axis=1) / hist.sum() + fwavacc = (freq[freq > 0] * iu[freq > 0]).sum() + cls_iu = dict(zip(range(self.n_classes), iu)) + + return { + "Overall Acc": acc, + "Mean Acc": acc_cls, + "FreqW Acc": fwavacc, + "Mean IoU": mean_iu, + "Class IoU": cls_iu, + } + + def reset(self): + self.confusion_matrix = np.zeros((self.n_classes, self.n_classes)) + + +class ElasticDNN_OfflineSegFMModel(ElasticDNN_OfflineFMModel): + def __init__(self, name: str, models_dict_path: str, device: str, num_classes): + super().__init__(name, models_dict_path, device) + self.num_classes = num_classes + + def get_accuracy(self, test_loader, *args, **kwargs): + device = self.device + self.to_eval_mode() + metrics = StreamSegMetrics(self.num_classes) + metrics.reset() + import tqdm + pbar = tqdm.tqdm(enumerate(test_loader), total=len(test_loader), leave=False, dynamic_ncols=True) + with torch.no_grad(): + for batch_index, (x, y) in pbar: + x, y = x.to(device, dtype=x.dtype, non_blocking=True, copy=False), \ + y.to(device, dtype=y.dtype, non_blocking=True, copy=False) + output = self.infer(x) + pred = output.detach().max(dim=1)[1].cpu().numpy() + metrics.update((y + 0).cpu().numpy(), pred) + + res = metrics.get_results() + pbar.set_description(f'cur batch mIoU: {res["Mean IoU"]:.4f}') + + res = metrics.get_results() + return res['Mean IoU'] + + def infer(self, x, *args, **kwargs): + return self.models_dict['main'](x) + + +class ElasticDNN_OfflineDetFMModel(ElasticDNN_OfflineFMModel): + def __init__(self, name: str, models_dict_path: str, device: str, num_classes): + super().__init__(name, models_dict_path, device) + self.num_classes = num_classes + + def get_accuracy(self, test_loader, *args, **kwargs): + # print('DeeplabV3: start test acc') + _d = test_loader.dataset + from data import build_dataloader + if _d.__class__.__name__ == 'MergedDataset': + # print('\neval on merged datasets') + datasets = _d.datasets + test_loaders = [build_dataloader(d, test_loader.batch_size, test_loader.num_workers, False, None) for d in datasets] + accs = [self.get_accuracy(loader) for loader in test_loaders] + # print(accs) + return sum(accs) / len(accs) + + # print('dataset len', len(test_loader.dataset)) + + model = self.models_dict['main'] + device = self.device + model.eval() + + # print('# classes', model.num_classes) + + model = model.to(device) + from dnns.yolov3.coco_evaluator import COCOEvaluator + from utils.common.others import HiddenPrints + with torch.no_grad(): + with HiddenPrints(): + evaluator = COCOEvaluator( + dataloader=test_loader, + img_size=(224, 224), + confthre=0.01, + nmsthre=0.65, + num_classes=self.num_classes, + testdev=False + ) + res = evaluator.evaluate(model, False, False) + map50 = res[1] + # print('eval info', res[-1]) + return map50 + + def infer(self, x, *args, **kwargs): + if len(args) > 0: + print(args, len(args)) + return self.models_dict['main'](x, *args) # forward(x, label) + return self.models_dict['main'](x) + + +class ElasticDNN_OfflineSenClsFMModel(ElasticDNN_OfflineFMModel): + def get_accuracy(self, test_loader, *args, **kwargs): + acc = 0 + sample_num = 0 + + self.to_eval_mode() + + with torch.no_grad(): + pbar = tqdm.tqdm(enumerate(test_loader), total=len(test_loader), dynamic_ncols=True, leave=False) + for batch_index, (x, y) in pbar: + for k, v in x.items(): + if isinstance(v, torch.Tensor): + x[k] = v.to(self.device) + y = y.to(self.device) + output = self.infer(x) + pred = F.softmax(output, dim=1).argmax(dim=1) + correct = torch.eq(pred, y).sum().item() + acc += correct + sample_num += len(y) + + pbar.set_description(f'cur_batch_total: {len(y)}, cur_batch_correct: {correct}, ' + f'cur_batch_acc: {(correct / len(y)):.4f}') + + acc /= sample_num + return acc + + def infer(self, x, *args, **kwargs): + return self.models_dict['main'].forward(**x) + + +from accelerate.utils.operations import pad_across_processes + + +class ElasticDNN_OfflineTrFMModel(ElasticDNN_OfflineFMModel): + + def get_accuracy(self, test_loader, *args, **kwargs): + # TODO: BLEU + from sacrebleu import corpus_bleu + + acc = 0 + num_batches = 0 + + self.to_eval_mode() + + from data.datasets.sentiment_classification.global_bert_tokenizer import get_tokenizer + tokenizer = get_tokenizer() + + def _decode(o): + # https://github.com/huggingface/transformers/blob/main/examples/research_projects/seq2seq-distillation/finetune.py#L133 + o = tokenizer.batch_decode(o, skip_special_tokens=True) + return [oi.strip() for oi in o] + + with torch.no_grad(): + pbar = tqdm.tqdm(enumerate(test_loader), total=len(test_loader), dynamic_ncols=True, leave=False) + for batch_index, (x, y) in pbar: + for k, v in x.items(): + if isinstance(v, torch.Tensor): + x[k] = v.to(self.device) + label = y.to(self.device) + + # generated_tokens = self.infer(x, generate=True) + + generated_tokens = self.infer(x).logits.argmax(-1) + + # pad tokens + generated_tokens = pad_across_processes( + generated_tokens, dim=1, pad_index=tokenizer.pad_token_id + ) + # pad label + label = pad_across_processes( + label, dim=1, pad_index=tokenizer.pad_token_id + ) + label = label.cpu().numpy() + label = np.where(label != -100, label, tokenizer.pad_token_id) + + decoded_output = _decode(generated_tokens) + decoded_y = _decode(y) + + decoded_y = [decoded_y] + + if batch_index == 0: + print(decoded_y, decoded_output) + + bleu = corpus_bleu(decoded_output, decoded_y).score + pbar.set_description(f'cur_batch_bleu: {bleu:.4f}') + + acc += bleu + num_batches += 1 + + acc /= num_batches + return acc + + def infer(self, x, *args, **kwargs): + if 'token_type_ids' in x.keys(): + del x['token_type_ids'] + + if 'generate' in kwargs: + return self.models_dict['main'].generate( + x['input_ids'], + attention_mask=x["attention_mask"], + max_length=512 + ) + + return self.models_dict['main'](**x) + + +from nltk.metrics import accuracy as nltk_acc + +class ElasticDNN_OfflineTokenClsFMModel(ElasticDNN_OfflineFMModel): + def get_accuracy(self, test_loader, *args, **kwargs): + acc = 0 + sample_num = 0 + + self.to_eval_mode() + + with torch.no_grad(): + pbar = tqdm.tqdm(enumerate(test_loader), total=len(test_loader), dynamic_ncols=True, leave=False) + for batch_index, (x, y) in pbar: + for k, v in x.items(): + if isinstance(v, torch.Tensor): + x[k] = v.to(self.device) + # print(x) + y = y.to(self.device) + output = self.infer(x) + + # torch.Size([16, 512, 43]) torch.Size([16, 512]) + + for oi, yi, xi in zip(output, y, x['input_ids']): + # oi: 512, 43; yi: 512 + seq_len = xi.nonzero().size(0) + + # print(output.size(), y.size()) + + pred = F.softmax(oi, dim=-1).argmax(dim=-1) + correct = torch.eq(pred[1: seq_len], yi[1: seq_len]).sum().item() + + # print(output.size(), y.size()) + + acc += correct + sample_num += seq_len + + pbar.set_description(f'seq_len: {seq_len}, cur_seq_acc: {(correct / seq_len):.4f}') + + acc /= sample_num + return acc + + def infer(self, x, *args, **kwargs): + return self.models_dict['main'](**x) + + +class ElasticDNN_OfflineMMClsFMModel(ElasticDNN_OfflineFMModel): + # def __init__(self, name: str, models_dict_path: str, device: str, class_to_label_idx_map): + # super().__init__(name, models_dict_path, device) + # self.class_to_label_idx_map = class_to_label_idx_map + + def get_accuracy(self, test_loader, *args, **kwargs): + acc = 0 + sample_num = 0 + + self.to_eval_mode() + + batch_size = 1 + + with torch.no_grad(): + pbar = tqdm.tqdm(enumerate(test_loader), total=len(test_loader), dynamic_ncols=True, leave=False) + for batch_index, (x, y) in pbar: + if batch_index * batch_size > 2000: + break + + for k, v in x.items(): + if isinstance(v, torch.Tensor): + x[k] = v.to(self.device) + y = y.to(self.device) + + # print(x) + + raw_texts = x['texts'][:] + x['texts'] = list(set(x['texts'])) + + # print(x['texts']) + + batch_size = len(y) + + x['for_training'] = False + + output = self.infer(x) + + output = output.logits_per_image + + # print(output.size()) + # exit() + + # y = torch.arange(len(y), device=self.device) + y = torch.LongTensor([x['texts'].index(rt) for rt in raw_texts]).to(self.device) + # print(y) + + # exit() + + # print(output.size(), y.size()) + + pred = F.softmax(output, dim=1).argmax(dim=1) + correct = torch.eq(pred, y).sum().item() + acc += correct + sample_num += len(y) + pbar.set_description(f'cur_batch_total: {len(y)}, cur_batch_correct: {correct}, ' + f'cur_batch_acc: {(correct / len(y)):.4f}') + + acc /= sample_num + return acc + + def infer(self, x, *args, **kwargs): + x['for_training'] = self.models_dict['main'].training + return self.models_dict['main'](**x) + + + +class VQAScore: + def __init__(self): + # self.add_state("score", default=torch.tensor(0.0), dist_reduce_fx="sum") + # self.add_state("total", default=torch.tensor(0.0), dist_reduce_fx="sum") + self.score = torch.tensor(0.0) + self.total = torch.tensor(0.0) + + def update(self, logits, target): + logits, target = ( + logits.detach().float().to(self.score.device), + target.detach().float().to(self.score.device), + ) + logits = torch.max(logits, 1)[1] + one_hots = torch.zeros(*target.size()).to(target) + one_hots.scatter_(1, logits.view(-1, 1), 1) + scores = one_hots * target + + self.score += scores.sum() + self.total += len(logits) + + def compute(self): + return self.score / self.total + + +class ElasticDNN_OfflineVQAFMModel(ElasticDNN_OfflineFMModel): + # def __init__(self, name: str, models_dict_path: str, device: str, class_to_label_idx_map): + # super().__init__(name, models_dict_path, device) + # self.class_to_label_idx_map = class_to_label_idx_map + + def get_accuracy(self, test_loader, *args, **kwargs): + acc = 0 + sample_num = 0 + + vqa_score = VQAScore() + + self.to_eval_mode() + + with torch.no_grad(): + pbar = tqdm.tqdm(enumerate(test_loader), total=len(test_loader), dynamic_ncols=True, leave=False) + for batch_index, (x, y) in pbar: + for k, v in x.items(): + if isinstance(v, torch.Tensor): + x[k] = v.to(self.device) + y = y.to(self.device) + output = self.infer(x).logits + + vqa_score.update(output, y) + + pbar.set_description(f'cur_batch_total: {len(y)}, cur_batch_acc: {vqa_score.compute():.4f}') + + return vqa_score.compute() + + def infer(self, x, *args, **kwargs): + return self.models_dict['main'](**x) + + +class ElasticDNN_OfflineMDModel(BaseModel): + def get_required_model_components(self) -> List[str]: + return ['main'] + + @abstractmethod + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + raise NotImplementedError + + @abstractmethod + def get_feature_hook(self) -> LayerActivation: + pass + + @abstractmethod + def get_distill_loss(self, student_output, teacher_output): + pass + + @abstractmethod + def get_matched_param_of_fm(self, self_param_name, fm: nn.Module): + pass + + +class ElasticDNN_OfflineClsMDModel(ElasticDNN_OfflineMDModel): + def get_accuracy(self, test_loader, *args, **kwargs): + acc = 0 + sample_num = 0 + + self.to_eval_mode() + + with torch.no_grad(): + pbar = tqdm.tqdm(enumerate(test_loader), total=len(test_loader), dynamic_ncols=True, leave=False) + for batch_index, (x, y) in pbar: + x, y = x.to(self.device), y.to(self.device) + output = self.infer(x) + pred = F.softmax(output, dim=1).argmax(dim=1) + correct = torch.eq(pred, y).sum().item() + acc += correct + sample_num += len(y) + + pbar.set_description(f'cur_batch_total: {len(y)}, cur_batch_correct: {correct}, ' + f'cur_batch_acc: {(correct / len(y)):.4f}') + + acc /= sample_num + return acc + + def infer(self, x, *args, **kwargs): + return self.models_dict['main'](x) + + +class ElasticDNN_OfflineSegMDModel(ElasticDNN_OfflineMDModel): + def __init__(self, name: str, models_dict_path: str, device: str, num_classes): + super().__init__(name, models_dict_path, device) + self.num_classes = num_classes + + def get_accuracy(self, test_loader, *args, **kwargs): + device = self.device + self.to_eval_mode() + metrics = StreamSegMetrics(self.num_classes) + metrics.reset() + import tqdm + pbar = tqdm.tqdm(enumerate(test_loader), total=len(test_loader), leave=False, dynamic_ncols=True) + with torch.no_grad(): + for batch_index, (x, y) in pbar: + x, y = x.to(device, dtype=x.dtype, non_blocking=True, copy=False), \ + y.to(device, dtype=y.dtype, non_blocking=True, copy=False) + output = self.infer(x) + pred = output.detach().max(dim=1)[1].cpu().numpy() + metrics.update((y + 0).cpu().numpy(), pred) + + res = metrics.get_results() + pbar.set_description(f'cur batch mIoU: {res["Mean IoU"]:.4f}') + + res = metrics.get_results() + return res['Mean IoU'] + + def infer(self, x, *args, **kwargs): + return self.models_dict['main'](x) + + +class ElasticDNN_OfflineDetMDModel(ElasticDNN_OfflineMDModel): + def __init__(self, name: str, models_dict_path: str, device: str, num_classes): + super().__init__(name, models_dict_path, device) + self.num_classes = num_classes + + def get_accuracy(self, test_loader, *args, **kwargs): + # print('DeeplabV3: start test acc') + _d = test_loader.dataset + from data import build_dataloader + if _d.__class__.__name__ == 'MergedDataset': + # print('\neval on merged datasets') + datasets = _d.datasets + test_loaders = [build_dataloader(d, test_loader.batch_size, test_loader.num_workers, False, None) for d in datasets] + accs = [self.get_accuracy(loader) for loader in test_loaders] + # print(accs) + return sum(accs) / len(accs) + + # print('dataset len', len(test_loader.dataset)) + + model = self.models_dict['main'] + device = self.device + model.eval() + + # print('# classes', model.num_classes) + + model = model.to(device) + from dnns.yolov3.coco_evaluator import COCOEvaluator + from utils.common.others import HiddenPrints + with torch.no_grad(): + with HiddenPrints(): + evaluator = COCOEvaluator( + dataloader=test_loader, + img_size=(224, 224), + confthre=0.01, + nmsthre=0.65, + num_classes=self.num_classes, + testdev=False + ) + res = evaluator.evaluate(model, False, False) + map50 = res[1] + # print('eval info', res[-1]) + return map50 + + def infer(self, x, *args, **kwargs): + if len(args) > 0: + return self.models_dict['main'](x, *args) # forward(x, label) + return self.models_dict['main'](x) + + +class ElasticDNN_OfflineSenClsMDModel(ElasticDNN_OfflineMDModel): + def get_accuracy(self, test_loader, *args, **kwargs): + acc = 0 + sample_num = 0 + + self.to_eval_mode() + + with torch.no_grad(): + pbar = tqdm.tqdm(enumerate(test_loader), total=len(test_loader), dynamic_ncols=True, leave=False) + for batch_index, (x, y) in pbar: + for k, v in x.items(): + if isinstance(v, torch.Tensor): + x[k] = v.to(self.device) + y = y.to(self.device) + output = self.infer(x) + pred = F.softmax(output, dim=1).argmax(dim=1) + + # print(pred, y) + + correct = torch.eq(pred, y).sum().item() + acc += correct + sample_num += len(y) + + pbar.set_description(f'cur_batch_total: {len(y)}, cur_batch_correct: {correct}, ' + f'cur_batch_acc: {(correct / len(y)):.4f}') + + acc /= sample_num + return acc + + def infer(self, x, *args, **kwargs): + return self.models_dict['main'](**x) + + +class ElasticDNN_OfflineTrMDModel(ElasticDNN_OfflineMDModel): + + def get_accuracy(self, test_loader, *args, **kwargs): + # TODO: BLEU + from sacrebleu import corpus_bleu + + acc = 0 + num_batches = 0 + + self.to_eval_mode() + + from data.datasets.sentiment_classification.global_bert_tokenizer import get_tokenizer + tokenizer = get_tokenizer() + + def _decode(o): + # https://github.com/huggingface/transformers/blob/main/examples/research_projects/seq2seq-distillation/finetune.py#L133 + o = tokenizer.batch_decode(o, skip_special_tokens=True, clean_up_tokenization_spaces=True) + return [oi.strip().replace(' ', '') for oi in o] + + with torch.no_grad(): + pbar = tqdm.tqdm(enumerate(test_loader), total=len(test_loader), dynamic_ncols=True, leave=False) + for batch_index, (x, y) in pbar: + for k, v in x.items(): + if isinstance(v, torch.Tensor): + x[k] = v.to(self.device) + y = y.to(self.device) + + output = self.infer(x) + decoded_output = _decode(output.argmax(-1)) + decoded_y = _decode(y) + + decoded_y = [decoded_y] + + # print(x, decoded_y, decoded_output, output.argmax(-1)) + + bleu = corpus_bleu(decoded_output, decoded_y).score + pbar.set_description(f'cur_batch_bleu: {bleu:.4f}') + + acc += bleu + num_batches += 1 + + acc /= num_batches + return acc + + def infer(self, x, *args, **kwargs): + return self.models_dict['main'](**x) + + +class ElasticDNN_OfflineTokenClsMDModel(ElasticDNN_OfflineMDModel): + def get_accuracy(self, test_loader, *args, **kwargs): + acc = 0 + sample_num = 0 + + self.to_eval_mode() + + with torch.no_grad(): + pbar = tqdm.tqdm(enumerate(test_loader), total=len(test_loader), dynamic_ncols=True, leave=False) + for batch_index, (x, y) in pbar: + for k, v in x.items(): + if isinstance(v, torch.Tensor): + x[k] = v.to(self.device) + # print(x) + y = y.to(self.device) + output = self.infer(x) + + # torch.Size([16, 512, 43]) torch.Size([16, 512]) + + for oi, yi, xi in zip(output, y, x['input_ids']): + # oi: 512, 43; yi: 512 + seq_len = xi.nonzero().size(0) + + # print(output.size(), y.size()) + + pred = F.softmax(oi, dim=-1).argmax(dim=-1) + correct = torch.eq(pred[1: seq_len], yi[1: seq_len]).sum().item() + + # print(output.size(), y.size()) + + acc += correct + sample_num += seq_len + + pbar.set_description(f'seq_len: {seq_len}, cur_seq_acc: {(correct / seq_len):.4f}') + + acc /= sample_num + return acc + + def infer(self, x, *args, **kwargs): + return self.models_dict['main'](**x) + + +class ElasticDNN_OfflineMMClsMDModel(ElasticDNN_OfflineMDModel): + def get_accuracy(self, test_loader, *args, **kwargs): + acc = 0 + sample_num = 0 + + self.to_eval_mode() + + batch_size = 1 + + with torch.no_grad(): + pbar = tqdm.tqdm(enumerate(test_loader), total=len(test_loader), dynamic_ncols=True, leave=False) + for batch_index, (x, y) in pbar: + if batch_index * batch_size > 2000: + break + + for k, v in x.items(): + if isinstance(v, torch.Tensor): + x[k] = v.to(self.device) + y = y.to(self.device) + + # print(x) + + raw_texts = x['texts'][:] + x['texts'] = list(set(x['texts'])) + + # print(x['texts']) + + batch_size = len(y) + + x['for_training'] = False + + output = self.infer(x) + + output = output.logits_per_image + + # print(output.size()) + # exit() + + # y = torch.arange(len(y), device=self.device) + y = torch.LongTensor([x['texts'].index(rt) for rt in raw_texts]).to(self.device) + # print(y) + + # exit() + + # print(output.size(), y.size()) + + pred = F.softmax(output, dim=1).argmax(dim=1) + correct = torch.eq(pred, y).sum().item() + acc += correct + sample_num += len(y) + pbar.set_description(f'cur_batch_total: {len(y)}, cur_batch_correct: {correct}, ' + f'cur_batch_acc: {(correct / len(y)):.4f}') + + acc /= sample_num + return acc + + def infer(self, x, *args, **kwargs): + return self.models_dict['main'](**x) + + +class ElasticDNN_OfflineVQAMDModel(ElasticDNN_OfflineMDModel): + # def __init__(self, name: str, models_dict_path: str, device: str, class_to_label_idx_map): + # super().__init__(name, models_dict_path, device) + # self.class_to_label_idx_map = class_to_label_idx_map + + def get_accuracy(self, test_loader, *args, **kwargs): + acc = 0 + sample_num = 0 + + vqa_score = VQAScore() + + self.to_eval_mode() + + with torch.no_grad(): + pbar = tqdm.tqdm(enumerate(test_loader), total=len(test_loader), dynamic_ncols=True, leave=False) + for batch_index, (x, y) in pbar: + for k, v in x.items(): + if isinstance(v, torch.Tensor): + x[k] = v.to(self.device) + y = y.to(self.device) + output = self.infer(x).logits + + vqa_score.update(output, y) + + pbar.set_description(f'cur_batch_total: {len(y)}, cur_batch_acc: {vqa_score.compute():.4f}') + + return vqa_score.compute() + + def infer(self, x, *args, **kwargs): + return self.models_dict['main'](**x) \ No newline at end of file diff --git a/methods/elasticdnn/api/online_model.py b/methods/elasticdnn/api/online_model.py new file mode 100644 index 0000000000000000000000000000000000000000..19426bbce451dae33b270a17ef4e6ca706fe6b01 --- /dev/null +++ b/methods/elasticdnn/api/online_model.py @@ -0,0 +1,246 @@ +from copy import deepcopy +from typing import List +import torch +from methods.base.model import BaseModel +import tqdm +from torch import nn +import torch.nn.functional as F +from abc import abstractmethod +from methods.elasticdnn.model.base import ElasticDNNUtil +from methods.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from methods.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from utils.common.log import logger +from utils.dl.common.model import LayerActivation, get_parameter + + +class ElasticDNN_OnlineModel(BaseModel): + def __init__(self, name: str, models_dict_path: str, device: str, ab_options: dict): + super().__init__(name, models_dict_path, device) + + assert [k in ab_options.keys() for k in ['md_to_fm_alpha', 'fm_to_md_alpha']] + self.ab_options = ab_options + + def get_required_model_components(self) -> List[str]: + return ['fm', 'md', 'sd', 'indexes', 'bn_stats'] + + @torch.no_grad() + def generate_sd_by_target_samples(self, target_samples: torch.Tensor): + elastic_dnn_util = self.get_elastic_dnn_util() + sd, unpruned_indexes_of_layers = elastic_dnn_util.extract_surrogate_dnn_via_samples_with_perf_test(self.models_dict['md'], target_samples.to(self.device), True) + logger.debug(f'generate sd: \n{sd}') + return sd, unpruned_indexes_of_layers + + @torch.no_grad() + def _compute_diff(self, old, new): + return (new - old).norm(1) / old.norm(1) + + @torch.no_grad() + def sd_feedback_to_md(self, after_da_sd, unpruned_indexes_of_layers): + self.models_dict['sd'] = after_da_sd + self.before_da_md = deepcopy(self.models_dict['md']) + + logger.info('\n\nsurrogate DNN feedback to master DNN...\n\n') + # one-to-one + + cur_unpruned_indexes = None + cur_unpruned_indexes_name = None + + for p_name, p in self.models_dict['sd'].named_parameters(): + matched_md_param = self.get_md_matched_param_of_sd_param(p_name) + logger.debug(f'if feedback: {p_name}') + if matched_md_param is None: + continue + logger.debug(f'start feedback: {p_name}, {p.size()} -> {matched_md_param.size()}') + # average + # setattr(matched_md_module, matched_md_param_name, (matched_md_param + p) / 2.) + + if p_name in unpruned_indexes_of_layers.keys(): + cur_unpruned_indexes = unpruned_indexes_of_layers[p_name] + cur_unpruned_indexes_name = p_name + + if p.size() != matched_md_param.size(): + logger.debug(f'cur unpruned indexes: {cur_unpruned_indexes_name}, {cur_unpruned_indexes.size()}') + + if p.dim() == 1: # norm + new_p = deepcopy(matched_md_param) + new_p[cur_unpruned_indexes] = p + elif p.dim() == 2: # linear + if p.size(0) < matched_md_param.size(0): # output pruned + new_p = deepcopy(matched_md_param) + new_p[cur_unpruned_indexes] = p + else: # input pruned + new_p = deepcopy(matched_md_param) + new_p[:, cur_unpruned_indexes] = p + p = new_p + + assert p.size() == matched_md_param.size(), f'{p.size()}, {matched_md_param.size()}' + + diff = self._compute_diff(matched_md_param, (matched_md_param + p) / 2.) + matched_md_param.copy_((matched_md_param + p) / 2.) + logger.debug(f'end feedback: {p_name}, diff: {diff:.6f}') + + def infer(self, x, *args, **kwargs): + return self.models_dict['sd'](x) + + def set_sd_sparsity(self, sparsity: float): + elastic_dnn_util = self.get_elastic_dnn_util() + elastic_dnn_util.clear_cached_channel_attention_in_master_dnn(self.models_dict['md']) + elastic_dnn_util.set_master_dnn_sparsity(self.models_dict['md'], sparsity) + + @torch.no_grad() + def md_feedback_to_self_fm(self): + logger.info('\n\nmaster DNN feedback to self foundation model...\n\n') + # one-to-many + + def upsample_2d_tensor(p: torch.Tensor, target_len: int): + assert p.dim() == 2 # regard 2d weight as (batch_size, 1d_vector_dim) + return F.upsample(p.unsqueeze(1).unsqueeze(3), + size=(target_len, 1), + mode='bilinear').squeeze(3).squeeze(1) + + for (p_name, p), before_p in zip(self.models_dict['md'].named_parameters(), self.before_da_md.parameters()): + matched_fm_param = self.get_fm_matched_param_of_md_param(p_name) + logger.debug(f'if feedback: {p_name}') + if matched_fm_param is None: + continue + + index = self.models_dict['indexes'][p_name] + logger.debug(f'start feedback: {p_name}, {p.size()} -> {matched_fm_param.size()}, index: {index.size()}') + + p_update = p - before_p + if p.dim() == 2: + p_update = upsample_2d_tensor(p_update, matched_fm_param.size(1)) + + p_update = p_update.unsqueeze(1) + index = index.unsqueeze(-1) + + # fast + # agg_p_update = (p_update * index).sum(0) + + # balanced agg + agg_p_update = 0 + + cur_split_size = 64 + while index.size(0) % cur_split_size != 0: + cur_split_size -= 1 + + for i in range(0, index.size(0), cur_split_size): + agg_p_update += p_update[i: i + cur_split_size] * index[i: i + cur_split_size] + agg_p_update = agg_p_update.sum(0) + + + else: + agg_p_update = (p_update.unsqueeze(1) * index).sum(0) + + new_fm_param = matched_fm_param + agg_p_update * self.ab_options['md_to_fm_alpha'] + + diff = self._compute_diff(matched_fm_param, new_fm_param) + + # NOTE: matched_fm_param may not be reference, may be a deepcopy!! + # and only here matched_fm_param needs to be updated, so another method dedicated for updating is necessary here + # matched_fm_param.copy_(new_fm_param) + self.update_fm_param(p_name, new_fm_param) + + logger.debug(f'end feedback: {p_name}, diff: {diff:.6f} (md_to_fm_alpha={self.ab_options["md_to_fm_alpha"]:.4f})') + + @abstractmethod + @torch.no_grad() + def update_fm_param(self, md_param_name, cal_new_fm_param_by_md_param): + """ + you should get the reference of fm_param and update it + """ + raise NotImplementedError + + @torch.no_grad() + def aggregate_fms_to_self_fm(self, fms: List[nn.Module]): + # average task-agnositc parameters + logger.info('\n\naggregate foundation models to self foundation model...\n\n') + for p_name, self_p in self.models_dict['fm'].named_parameters(): + logger.debug(f'if aggregate {p_name}') + if 'abs' in p_name or p_name.startswith('norm') or p_name.startswith('head'): + logger.debug(f'{p_name} belongs to LoRA parameters/task-specific head, i.e. task-specific parameters, skip') + continue + all_p = [get_parameter(fm, p_name) for fm in fms] + if any([_p is None for _p in all_p]): + continue + + avg_p = sum(all_p) / len(all_p) + # [_p.copy_(avg_p) for _p in all_p] + + diff = self._compute_diff(self_p, avg_p) + logger.debug(f'aggregate {p_name}, diff {diff:.6f}') + + self_p.copy_(avg_p) + + @torch.no_grad() + def fm_feedback_to_md(self): + logger.info('\n\nself foundation model feedback to master DNN...\n\n') + # one-to-many + + def downsample_2d_tensor(p: torch.Tensor, target_len: int): + assert p.dim() == 2 # regard 2d weight as (batch_size, 1d_vector_dim) + # return F.upsample(p.unsqueeze(1).unsqueeze(3), + # size=(target_len, 1), + # mode='bilinear').squeeze(3).squeeze(1) + return F.interpolate(p.unsqueeze(1).unsqueeze(3), size=(target_len, 1), mode='bilinear').squeeze(3).squeeze(1) + + + for p_name, p in self.models_dict['md'].named_parameters(): + matched_fm_param = self.get_fm_matched_param_of_md_param(p_name) + logger.debug(f'if feedback: {p_name}') + if matched_fm_param is None: + continue + + index = self.models_dict['indexes'][p_name] + logger.debug(f'start feedback: {p_name}, {p.size()} -> {matched_fm_param.size()}, index: {index.size()}') + + if p.dim() == 2: + matched_fm_param = downsample_2d_tensor(matched_fm_param, p.size(1)) + + matched_fm_param = matched_fm_param.unsqueeze(0) + index = index.unsqueeze(-1) + + # fast + # agg_p_update = (p_update * index).sum(0) + + # balanced agg + agg_fm_param = 0 + + cur_split_size = 64 + while index.size(1) % cur_split_size != 0: + cur_split_size -= 1 + + for i in range(0, index.size(1), cur_split_size): + agg_fm_param += matched_fm_param[:, i: i + cur_split_size] * index[:, i: i + cur_split_size] + agg_fm_param = agg_fm_param.sum(1) + # agg_fm_param = downsample_2d_tensor(agg_fm_param, p.size(1)) + + else: + agg_fm_param = (matched_fm_param.unsqueeze(0) * index).sum(1) + + + + diff = self._compute_diff(p, agg_fm_param) + p.copy_(agg_fm_param * self.ab_options['fm_to_md_alpha'] + (1. - self.ab_options['fm_to_md_alpha']) * p) + + logger.debug(f'end feedback: {p_name}, diff: {diff:.6f} (fm_to_md_alpha: {self.ab_options["fm_to_md_alpha"]:.4f})') + + @abstractmethod + def get_elastic_dnn_util(self) -> ElasticDNNUtil: + pass + + @abstractmethod + def get_task_head_params(self): + pass + + @abstractmethod + def get_md_matched_param_of_sd_param(self, sd_param_name): + pass + + @abstractmethod + def get_fm_matched_param_of_md_param(self, md_param_name): + pass + + @abstractmethod + def get_md_matched_param_of_fm_param(self, fm_param_name): + pass \ No newline at end of file diff --git a/methods/elasticdnn/api/online_model_v2.py b/methods/elasticdnn/api/online_model_v2.py new file mode 100644 index 0000000000000000000000000000000000000000..ad2ab7920049ad3d6dcf1144f33dc6e876915037 --- /dev/null +++ b/methods/elasticdnn/api/online_model_v2.py @@ -0,0 +1,269 @@ +from copy import deepcopy +from typing import List +import torch +from methods.base.model import BaseModel +import tqdm +from torch import nn +import torch.nn.functional as F +from abc import abstractmethod +from methods.elasticdnn.model.base import ElasticDNNUtil +from methods.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from methods.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from utils.common.log import logger +from utils.dl.common.model import LayerActivation, get_parameter + + +class ElasticDNN_OnlineModel(BaseModel): + def __init__(self, name: str, models_dict_path: str, device: str, ab_options: dict): + super().__init__(name, models_dict_path, device) + + assert [k in ab_options.keys() for k in ['md_to_fm_alpha', 'fm_to_md_alpha']] + self.ab_options = ab_options + + def get_required_model_components(self) -> List[str]: + return ['fm', 'md', 'sd', 'indexes', 'bn_stats'] + + @torch.no_grad() + def generate_sd_by_target_samples(self, target_samples: torch.Tensor): + elastic_dnn_util = self.get_elastic_dnn_util() + + if isinstance(target_samples, dict): + for k, v in target_samples.items(): + if isinstance(v, torch.Tensor): + target_samples[k] = v.to(self.device) + else: + target_samples = target_samples.to(self.device) + + sd, unpruned_indexes_of_layers = elastic_dnn_util.extract_surrogate_dnn_via_samples_with_perf_test(self.models_dict['md'], target_samples, True) + logger.debug(f'generate sd: \n{sd}') + return sd, unpruned_indexes_of_layers + + @torch.no_grad() + def _compute_diff(self, old, new): + return (new - old).norm(1) / old.norm(1) + + @torch.no_grad() + def sd_feedback_to_md(self, after_da_sd, unpruned_indexes_of_layers): + self.models_dict['sd'] = after_da_sd + self.before_da_md = deepcopy(self.models_dict['md']) + + logger.info('\n\nsurrogate DNN feedback to master DNN...\n\n') + # one-to-one + + cur_unpruned_indexes = None + cur_unpruned_indexes_name = None + + for p_name, p in self.models_dict['sd'].named_parameters(): + matched_md_param = self.get_md_matched_param_of_sd_param(p_name) + logger.debug(f'if feedback: {p_name}') + if matched_md_param is None: + continue + logger.debug(f'start feedback: {p_name}, {p.size()} -> {matched_md_param.size()}') + # average + # setattr(matched_md_module, matched_md_param_name, (matched_md_param + p) / 2.) + + if p_name in unpruned_indexes_of_layers.keys(): + cur_unpruned_indexes = unpruned_indexes_of_layers[p_name] + cur_unpruned_indexes_name = p_name + + if p.size() != matched_md_param.size(): + logger.debug(f'cur unpruned indexes: {cur_unpruned_indexes_name}, {cur_unpruned_indexes.size()}') + + if p.dim() == 1: # norm + new_p = deepcopy(matched_md_param) + new_p[cur_unpruned_indexes] = p + elif p.dim() == 2: # linear + if p.size(0) < matched_md_param.size(0): # output pruned + new_p = deepcopy(matched_md_param) + new_p[cur_unpruned_indexes] = p + else: # input pruned + new_p = deepcopy(matched_md_param) + new_p[:, cur_unpruned_indexes] = p + p = new_p + + assert p.size() == matched_md_param.size(), f'{p.size()}, {matched_md_param.size()}' + + diff = self._compute_diff(matched_md_param, (matched_md_param + p) / 2.) + matched_md_param.copy_((matched_md_param + p) / 2.) + logger.debug(f'end feedback: {p_name}, diff: {diff:.6f}') + + def infer(self, x, *args, **kwargs): + return self.models_dict['sd'](x) + + def set_sd_sparsity(self, sparsity: float): + elastic_dnn_util = self.get_elastic_dnn_util() + elastic_dnn_util.clear_cached_channel_attention_in_master_dnn(self.models_dict['md']) + elastic_dnn_util.set_master_dnn_sparsity(self.models_dict['md'], sparsity) + + @torch.no_grad() + def md_feedback_to_self_fm(self): + logger.info('\n\nmaster DNN feedback to self foundation model...\n\n') + # one-to-many + + # def upsample_2d_tensor(p: torch.Tensor, target_len: int): + # assert p.dim() == 2 # regard 2d weight as (batch_size, 1d_vector_dim) + # return F.upsample(p.unsqueeze(1).unsqueeze(3), + # size=(target_len, 1), + # mode='bilinear').squeeze(3).squeeze(1) + + for (p_name, p), before_p in zip(self.models_dict['md'].named_parameters(), self.before_da_md.parameters()): + matched_fm_param = self.get_fm_matched_param_of_md_param(p_name) + logger.debug(f'if feedback: {p_name}') + if matched_fm_param is None: + continue + # print(self.models_dict['indexes'].keys()) + index = self.models_dict['indexes'][p_name] + logger.debug(f'start feedback: {p_name}, {p.size()} -> {matched_fm_param.size()}, index: {index.size()}') + + p_update = p - before_p + + t = False + if p.dim() > 1 and index.size(0) == p.size(1) and index.size(1) == matched_fm_param.size(1): + assert p.dim() == 2 + p_update = p_update.T + matched_fm_param = matched_fm_param.T + t = True + logger.debug(f'transpose paramters') + + + if p.dim() == 2: + # p_update = upsample_2d_tensor(p_update, matched_fm_param.size(1)) + + p_update = p_update.unsqueeze(1) + index = index.unsqueeze(-1) + + # fast + # agg_p_update = (p_update * index).sum(0) + + # balanced agg + agg_p_update = 0 + + cur_split_size = 64 + while index.size(0) % cur_split_size != 0: + cur_split_size -= 1 + + for i in range(0, index.size(0), cur_split_size): + agg_p_update += p_update[i: i + cur_split_size] * index[i: i + cur_split_size] + agg_p_update = agg_p_update.sum(0) + + + else: + agg_p_update = (p_update.unsqueeze(1) * index).sum(0) + + new_fm_param = matched_fm_param + agg_p_update * self.ab_options['md_to_fm_alpha'] + + diff = self._compute_diff(matched_fm_param, new_fm_param) + + # NOTE: matched_fm_param may not be reference, may be a deepcopy!! + # and only here matched_fm_param needs to be updated, so another method dedicated for updating is necessary here + # matched_fm_param.copy_(new_fm_param) + self.update_fm_param(p_name, new_fm_param.T if t else new_fm_param) + + logger.debug(f'end feedback: {p_name}, diff: {diff:.6f} (md_to_fm_alpha={self.ab_options["md_to_fm_alpha"]:.4f})') + + @abstractmethod + @torch.no_grad() + def update_fm_param(self, md_param_name, cal_new_fm_param_by_md_param): + """ + you should get the reference of fm_param and update it + """ + raise NotImplementedError + + @torch.no_grad() + def aggregate_fms_to_self_fm(self, fms: List[nn.Module]): + # average task-agnositc parameters + logger.info('\n\naggregate foundation models to self foundation model...\n\n') + for p_name, self_p in self.models_dict['fm'].named_parameters(): + logger.debug(f'if aggregate {p_name}') + if 'abs' in p_name or p_name.startswith('norm') or p_name.startswith('head'): + logger.debug(f'{p_name} belongs to LoRA parameters/task-specific head, i.e. task-specific parameters, skip') + continue + all_p = [get_parameter(fm, p_name) for fm in fms] + if any([_p is None for _p in all_p]): + continue + + avg_p = sum(all_p) / len(all_p) + # [_p.copy_(avg_p) for _p in all_p] + + diff = self._compute_diff(self_p, avg_p) + logger.debug(f'aggregate {p_name}, diff {diff:.6f}') + + self_p.copy_(avg_p) + + @torch.no_grad() + def fm_feedback_to_md(self): + logger.info('\n\nself foundation model feedback to master DNN...\n\n') + # one-to-many + + # def downsample_2d_tensor(p: torch.Tensor, target_len: int): + # assert p.dim() == 2 # regard 2d weight as (batch_size, 1d_vector_dim) + # # return F.upsample(p.unsqueeze(1).unsqueeze(3), + # # size=(target_len, 1), + # # mode='bilinear').squeeze(3).squeeze(1) + # return F.interpolate(p.unsqueeze(1).unsqueeze(3), size=(target_len, 1), mode='bilinear').squeeze(3).squeeze(1) + + + for p_name, p in self.models_dict['md'].named_parameters(): + matched_fm_param = self.get_fm_matched_param_of_md_param(p_name) + logger.debug(f'if feedback: {p_name}') + if matched_fm_param is None: + continue + + index = self.models_dict['indexes'][p_name] + logger.debug(f'start feedback: {p_name}, {p.size()} -> {matched_fm_param.size()}, index: {index.size()}') + + if p.dim() > 1 and index.size(0) == p.size(1) and index.size(1) == matched_fm_param.size(1): + assert p.dim() == 2 + p = p.T + matched_fm_param = matched_fm_param.T + + if p.dim() == 2: + # matched_fm_param = downsample_2d_tensor(matched_fm_param, p.size(1)) + + matched_fm_param = matched_fm_param.unsqueeze(0) + index = index.unsqueeze(-1) + + # fast + # agg_p_update = (p_update * index).sum(0) + + # balanced agg + agg_fm_param = 0 + + cur_split_size = 64 + while index.size(1) % cur_split_size != 0: + cur_split_size -= 1 + + for i in range(0, index.size(1), cur_split_size): + agg_fm_param += matched_fm_param[:, i: i + cur_split_size] * index[:, i: i + cur_split_size] + agg_fm_param = agg_fm_param.sum(1) + # agg_fm_param = downsample_2d_tensor(agg_fm_param, p.size(1)) + + else: + agg_fm_param = (matched_fm_param.unsqueeze(0) * index).sum(1) + + + + diff = self._compute_diff(p, agg_fm_param) + p.copy_(agg_fm_param * self.ab_options['fm_to_md_alpha'] + (1. - self.ab_options['fm_to_md_alpha']) * p) + + logger.debug(f'end feedback: {p_name}, diff: {diff:.6f} (fm_to_md_alpha: {self.ab_options["fm_to_md_alpha"]:.4f})') + + @abstractmethod + def get_elastic_dnn_util(self) -> ElasticDNNUtil: + pass + + @abstractmethod + def get_task_head_params(self): + pass + + @abstractmethod + def get_md_matched_param_of_sd_param(self, sd_param_name): + pass + + @abstractmethod + def get_fm_matched_param_of_md_param(self, md_param_name): + pass + + @abstractmethod + def get_md_matched_param_of_fm_param(self, fm_param_name): + pass \ No newline at end of file diff --git a/methods/elasticdnn/hugging_face/internal_adapter.py b/methods/elasticdnn/hugging_face/internal_adapter.py new file mode 100644 index 0000000000000000000000000000000000000000..0fb95dfb42fe3099306771d14803e67e4df79de2 --- /dev/null +++ b/methods/elasticdnn/hugging_face/internal_adapter.py @@ -0,0 +1,641 @@ +from ..api.model import ElasticDNN_OfflineFMModel, ElasticDNN_OfflineMDModel +from .user_impl import HuggingFaceModelAPI + +from typing import List +from data.dataloader import build_dataloader +# from methods.elasticdnn.api.online_model import ElasticDNN_OnlineModel +from methods.elasticdnn.api.online_model_v2 import ElasticDNN_OnlineModel + +import torch +import sys +from torch import nn +from methods.elasticdnn.api.model import ElasticDNN_OfflineSegFMModel, ElasticDNN_OfflineSegMDModel +from methods.elasticdnn.api.algs.md_pretraining_wo_fbs import ElasticDNN_MDPretrainingWoFBSAlg +from methods.elasticdnn.model.base import ElasticDNNUtil +from methods.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from methods.elasticdnn.pipeline.offline.fm_to_md.vit import FM_to_MD_ViT_Util +from methods.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from methods.elasticdnn.pipeline.offline.fm_lora.vit import FMLoRA_ViT_Util +from methods.elasticdnn.model.bert import ElasticBertUtil +from utils.common.file import ensure_dir +from utils.dl.common.model import LayerActivation, get_module, get_parameter, get_super_module +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +from utils.dl.common.loss import CrossEntropyLossSoft +import torch.nn.functional as F +from utils.dl.common.env import create_tbwriter +import os +from utils.common.log import logger +from utils.common.data_record import write_json +# from methods.shot.shot import OnlineShotModel +from methods.feat_align.main import OnlineFeatAlignModel, FeatAlignAlg +import tqdm +from methods.feat_align.mmd import mmd_rbf +from copy import deepcopy +from typing import Optional, Union +import torch +from torch import nn +from einops import rearrange, repeat +from einops.layers.torch import Rearrange +import tqdm +from methods.elasticdnn.model.vit import Linear_WrappedWithFBS + +from utils.dl.common.model import get_model_device, get_model_size, set_module, get_module +import torch +from abc import abstractmethod + + +class ElasticDNN_OfflineFMModel_for_HuggingFaceFM(ElasticDNN_OfflineFMModel): + def set_hugging_face_api(self, hugging_face_api: HuggingFaceModelAPI): + self.hugging_face_api = hugging_face_api + + def get_accuracy(self, test_loader, *args, **kwargs): + return self.hugging_face_api.get_accuracy(self.models_dict['main'], test_loader, self.device, *args, **kwargs) + + def infer(self, x, *args, **kwargs): + return self.hugging_face_api.infer(self.models_dict['main'], x, *args, **kwargs) + + def get_required_model_components(self) -> List[str]: + return ['main'] + + def generate_md_by_reducing_width(self, reducing_width_ratio, samples: torch.Tensor): + res = FM_to_MD_HuggingFaceFM_Util() + res.set_hugging_face_api(self.hugging_face_api) + return res.init_md_from_fm_by_reducing_width_with_perf_test(self.models_dict['main'], reducing_width_ratio, samples) + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + return self.hugging_face_api.forward_to_get_task_loss(self.models_dict['main'], x, y) + + def get_feature_hook(self) -> LayerActivation: + return self.hugging_face_api.get_feature_hook(self.models_dict['main'], self.device) + + def get_elastic_dnn_util(self) -> ElasticDNNUtil: + res = ElasticHuggingFaceFMUtil() + res.set_hugging_face_api(self.hugging_face_api) + return res + + def get_lora_util(self) -> FMLoRA_Util: + res = FMLoRA_HuggingFaceFM_Util() + res.set_hugging_face_api(self.hugging_face_api) + return res + + def get_task_head_params(self): + return self.hugging_face_api.get_task_head_params(self.models_dict['main']) + + +class ElasticDNN_OfflineMDModel_for_HuggingFaceFM(ElasticDNN_OfflineMDModel): + def set_hugging_face_api(self, hugging_face_api: HuggingFaceModelAPI): + self.hugging_face_api = hugging_face_api + + def get_required_model_components(self) -> List[str]: + return ['main'] + + def get_accuracy(self, test_loader, *args, **kwargs): + return self.hugging_face_api.get_accuracy(self.models_dict['main'], test_loader, self.device, *args, **kwargs) + + def infer(self, x, *args, **kwargs): + return self.hugging_face_api.infer(self.models_dict['main'], x, *args, **kwargs) + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + return self.hugging_face_api.forward_to_get_task_loss(self.models_dict['main'], x, y) + + def get_feature_hook(self) -> LayerActivation: + return self.hugging_face_api.get_feature_hook(self.models_dict['main'], self.device) + + def get_distill_loss(self, student_output, teacher_output): + return CrossEntropyLossSoft()(student_output, teacher_output) + + def get_matched_param_of_fm(self, self_param_name, fm: nn.Module): + if any([k in self_param_name for k in ['fbs', 'cls_token', 'pos_embed']]): + return None + + p = get_parameter(self.models_dict['main'], self_param_name) + if p.dim() == 0: + return None + elif p.dim() == 1: + return None + + layers_name = self.hugging_face_api.get_qkv_proj_ff1_ff2_layer_names() + if len(layers_name[0]) == 4: + + + qkv_names = [layer[0] for layer in layers_name] + qkv_proj_names = [layer[1] for layer in layers_name] + ff1_names = [layer[-2] for layer in layers_name] + ff2_names = [layer[-1] for layer in layers_name] + + qkv_weight_names = [n + '.weight' for n in qkv_names] + + if self_param_name in qkv_weight_names: + ss = self_param_name.split('.') + + fm_qkv_name = '.'.join(ss[0: -1]) + '.qkv' + fm_qkv = get_module(fm, fm_qkv_name) + + fm_abs_name = '.'.join(ss[0: -1]) + '.abs' + fm_abs = get_module(fm, fm_abs_name) + + # print(fm_qkv_name, fm_abs_name, fm) + + return torch.cat([ + fm_qkv.weight.data, # task-agnositc params + torch.cat([(_abs[0].weight.T @ _abs[1].weight.T).T for _abs in fm_abs], dim=0) # task-specific params (LoRA) + ], dim=0) + else: + q_names = [layer[0] for layer in layers_name] + k_names = [layer[1] for layer in layers_name] + v_names = [layer[2] for layer in layers_name] + qkv_proj_names = [layer[3] for layer in layers_name] + ff1_names = [layer[-2] for layer in layers_name] + ff2_names = [layer[-1] for layer in layers_name] + + qkv_weight_names = [n + '.weight' for n in q_names + k_names + v_names] + + if self_param_name in qkv_weight_names: + + ss = self_param_name.split('.') + # raise NotImplementedError() # TODO: + fm_qkv_name = '.'.join(ss[0: -1]) + '.qkv' + fm_qkv = get_module(fm, fm_qkv_name) + + fm_abs_name = '.'.join(ss[0: -1]) + '.ab' + fm_abs = get_module(fm, fm_abs_name) + + # print(fm_qkv_name, fm_abs_name, fm) + + return torch.cat([ + fm_qkv.weight.data, # task-agnositc params + fm_abs[1].weight @ fm_abs[0].weight + ], dim=0) + + # elif 'to_qkv.bias' in self_param_name: + # ss = self_param_name.split('.') + + # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv.bias' + # return get_parameter(fm, fm_qkv_name) + + ff1_weight_names = [n + '.linear.weight' for n in ff1_names] + ff2_weight_names = [n + '.weight' for n in ff2_names] + + if self_param_name in ff1_weight_names: + fm_param_name = self_param_name.replace('.linear', '') + return get_parameter(fm, fm_param_name) + + if self_param_name in ff2_weight_names: + fm_param_name = self_param_name + return get_parameter(fm, fm_param_name) + + return None + + +class ElasticHuggingFaceFMUtil(ElasticDNNUtil): + def set_hugging_face_api(self, hugging_face_api: HuggingFaceModelAPI): + self.hugging_face_api = hugging_face_api + + def convert_raw_dnn_to_master_dnn(self, raw_dnn: nn.Module, r: float, ignore_layers=[]): + assert len(ignore_layers) == 0, 'not supported yet' + + raw_vit = deepcopy(raw_dnn) + + # set_module(module, 'patch_embed.proj', ProjConv_WrappedWithFBS(module.patch_embed.proj, r)) + layers = self.hugging_face_api.get_qkv_proj_ff1_ff2_layer_names() + ff1_names = [layer[-2] for layer in layers] + + for name, module in raw_vit.named_modules(): + # if name.endswith('attn'): + # set_module(module, 'qkv', ToQKV_WrappedWithFBS(module.qkv, r)) + if name in ff1_names: + # set_module(get_super_module(module, name), name.split('.')[-1], Linear_WrappedWithFBS(module, r)) + set_module(raw_vit, name, Linear_WrappedWithFBS(module, r)) + + return raw_vit + + def set_master_dnn_sparsity(self, master_dnn: nn.Module, sparsity: float): + return super().set_master_dnn_sparsity(master_dnn, sparsity) + + def select_most_rep_sample(self, master_dnn: nn.Module, samples: torch.Tensor): + # print(samples) + # return samples[0].unsqueeze(0) + res = {k: v[0: 1] for k, v in samples.items()} + return res + + def extract_surrogate_dnn_via_samples(self, master_dnn: nn.Module, samples: torch.Tensor, return_detail=False): + sample = self.select_most_rep_sample(master_dnn, samples) + # assert sample.dim() == 4 and sample.size(0) == 1 + + # print('before') + master_dnn.eval() + self.clear_cached_channel_attention_in_master_dnn(master_dnn) + with torch.no_grad(): + master_dnn_output = master_dnn(**sample) + + # print('after') + + boosted_vit = deepcopy(master_dnn) + + def get_unpruned_indexes_from_channel_attn(channel_attn: torch.Tensor, k): + assert channel_attn.size(0) == 1, 'use A representative sample to generate channel attentions' + + res = channel_attn[0].nonzero(as_tuple=True)[0] # should be one-dim + return res + + unpruned_indexes_of_layers = {} + + layers_name = self.hugging_face_api.get_qkv_proj_ff1_ff2_layer_names() + ff1_names = [layer[-2] for layer in layers] + ff2_names = [layer[-1] for layer in layers] + + for ff1_name, ff2_name in zip(ff1_names, ff2_names): + ff_0 = get_module(boosted_vit, ff1_name) + # ff_0_unpruned_indexes = get_unpruned_indexes_from_channel_attn(ff_0.cached_channel_attention, k) + ff_0_pruned_indexes = ff_0.k_takes_all.cached_i[0].sort()[0] + ff_0_unpruned_indexes = torch.LongTensor([ii for ii in range(ff_0.cached_channel_attention.size(1)) if ii not in ff_0_pruned_indexes]) + new_ff_0 = nn.Linear(ff_0.linear.in_features, ff_0_unpruned_indexes.size(0), ff_0.linear.bias is not None) + new_ff_0.weight.data.copy_(ff_0.linear.weight.data[ff_0_unpruned_indexes]) + if ff_0.linear.bias is not None: + new_ff_0.bias.data.copy_(ff_0.linear.bias.data[ff_0_unpruned_indexes]) + # set_module(get_super_module(ff_0, ff1_name), ff1_name.split('.')[-1], + # nn.Sequential(new_ff_0, StaticFBS(ff_0.cached_channel_attention[:, ff_0_unpruned_indexes]))) + set_module(boosted_vit, ff1_name, + nn.Sequential(new_ff_0, StaticFBS(ff_0.cached_channel_attention[:, ff_0_unpruned_indexes]))) + + ff_1 = get_module(boosted_vit, ff2_name) + new_ff_1 = nn.Linear(ff_0_unpruned_indexes.size(0), ff_1.out_features, ff_1.bias is not None) + new_ff_1.weight.data.copy_(ff_1.weight.data[:, ff_0_unpruned_indexes]) + if ff_1.bias is not None: + new_ff_1.bias.data.copy_(ff_1.bias.data) + # set_module(get_super_module(ff_1), ff2_name.split('.')[-1], new_ff_1) + set_module(boosted_vit, ff2_name, new_ff_1) + + unpruned_indexes_of_layers[f'{ff1_name}.0.weight'] = ff_0_unpruned_indexes + + surrogate_dnn = boosted_vit + surrogate_dnn.eval() + surrogate_dnn = surrogate_dnn.to(get_model_device(master_dnn)) + # logger.debug(surrogate_dnn) + with torch.no_grad(): + surrogate_dnn_output = surrogate_dnn(**sample) + + output_diff = ((surrogate_dnn_output - master_dnn_output) ** 2).sum() + # assert output_diff < 1e-4, output_diff + logger.info(f'output diff of master and surrogate DNN: {output_diff}') + logger.debug(f'example output of master/surrogate: {master_dnn_output.sum(0)[0: 10]}, {surrogate_dnn_output.sum(0)[0: 10]}') + # logger.info(f'\nonly prune mlp!!!!\n') + # logger.info(f'\nonly prune mlp!!!!\n') + + if return_detail: + return boosted_vit, unpruned_indexes_of_layers + + return boosted_vit + + def extract_surrogate_dnn_via_samples_with_perf_test(self, master_dnn: nn.Module, samples: torch.Tensor, return_detail=False): + master_dnn_size = get_model_size(master_dnn, True) + master_dnn_latency = self._get_model_latency(master_dnn, samples, 50, + get_model_device(master_dnn), 50, False) + + res = self.extract_surrogate_dnn_via_samples(master_dnn, samples, return_detail) + if not return_detail: + surrogate_dnn = res + else: + surrogate_dnn, unpruned_indexes_of_layers = res + surrogate_dnn_size = get_model_size(surrogate_dnn, True) + surrogate_dnn_latency = self._get_model_latency(master_dnn, samples, 50, + get_model_device(master_dnn), 50, False) + + logger.info(f'master DNN ({master_dnn_size:.3f}MB, {master_dnn_latency:.4f}s/sample) -> ' + f'surrogate DNN ({surrogate_dnn_size:.3f}MB, {surrogate_dnn_latency:.4f}s/sample)\n' + f'(model size: ↓ {(master_dnn_size / surrogate_dnn_size):.2f}x, ' + f'latency: ↓ {(master_dnn_latency / surrogate_dnn_latency):.2f}x)') + + return res + + def _get_model_latency(self, model: torch.nn.Module, model_input_size, sample_num: int, + device: str, warmup_sample_num: int, return_detail=False): + import time + + if isinstance(model_input_size, tuple): + dummy_input = torch.rand(model_input_size).to(device) + else: + dummy_input = model_input_size + + model = model.to(device) + model.eval() + + # warm up + with torch.no_grad(): + for _ in range(warmup_sample_num): + model(**dummy_input) + + infer_time_list = [] + + if device == 'cuda' or 'cuda' in str(device): + with torch.no_grad(): + for _ in range(sample_num): + s, e = torch.cuda.Event(enable_timing=True), torch.cuda.Event(enable_timing=True) + s.record() + model(**dummy_input) + e.record() + torch.cuda.synchronize() + cur_model_infer_time = s.elapsed_time(e) / 1000. + infer_time_list += [cur_model_infer_time] + + else: + with torch.no_grad(): + for _ in range(sample_num): + start = time.time() + model(**dummy_input) + cur_model_infer_time = time.time() - start + infer_time_list += [cur_model_infer_time] + + avg_infer_time = sum(infer_time_list) / sample_num + + if return_detail: + return avg_infer_time, infer_time_list + return avg_infer_time + + +class FMLoRA_HuggingFaceFM_Util(FMLoRA_Util): + def set_hugging_face_api(self, hugging_face_api: HuggingFaceModelAPI): + self.hugging_face_api = hugging_face_api + + @torch.no_grad() + def add_lora_ab_to_fm(self, fm: nn.Module, ab_r: int, samples: dict): + fm.eval() + + if isinstance(samples, dict): + o1 = fm(**samples) + else: + o1 = fm(samples) + + layers_name = self.hugging_face_api.get_qkv_proj_ff1_ff2_layer_names() + if len(layers_name[0]) == 4: + qkv_names = [layer[0] for layer in layers_name] + + from ..pipeline.offline.fm_lora.vit import ToQKV_WrappedWithLoRA + for name, module in fm.named_modules(): + if name in qkv_names: + set_module(fm, name, ToQKV_WrappedWithLoRA(module, ab_r)) + else: + qkv_names = [layer[0] for layer in layers_name] + [layer[1] for layer in layers_name] + [layer[2] for layer in layers_name] + + from ..pipeline.offline.fm_lora.bert import ToQKV_WrappedWithLoRA + for name, module in fm.named_modules(): + if name in qkv_names: + set_module(fm, name, ToQKV_WrappedWithLoRA(module, ab_r)) + + + + if isinstance(samples, dict): + o2 = fm(**samples) + else: + o2 = fm(samples) + + if isinstance(o1, tuple): + o1 = o1[-1] + o2 = o2[-1] + output_diff = ((o1 - o2) ** 2).sum() + assert output_diff < 1e-5 + + return fm + + @torch.no_grad() + def absorb_lora_and_recover_net_structure(self, fm: nn.Module, samples: dict): + fm.eval() + # print('absorb lora before') + if isinstance(samples, dict): + o1 = fm(**samples) + else: + o1 = fm(samples) + + from ..pipeline.offline.fm_lora.vit import ToQKV_WrappedWithLoRA as ToQKV_WrappedWithLoRA1 + from ..pipeline.offline.fm_lora.bert import ToQKV_WrappedWithLoRA as ToQKV_WrappedWithLoRA2 + + for name, module in fm.named_modules(): + if isinstance(module, ToQKV_WrappedWithLoRA1): + + qkv = module.qkv + fm_abs = module.abs + + fm_abs_weight = torch.cat([_abs[1].weight @ _abs[0].weight for _abs in fm_abs], dim=0) + qkv.weight.add_(fm_abs_weight) + + set_module(fm, name, qkv) + + elif isinstance(module, ToQKV_WrappedWithLoRA2): + + fc = module.fc + ab = module.ab + + fc.weight.add_(ab[1].weight @ ab[0].weight) + + set_module(fm, name, fc) + + # print('absorb lora after') + if isinstance(samples, dict): + o2 = fm(**samples) + else: + o2 = fm(samples) + + if isinstance(o1, tuple): + o1 = o1[-1] + o2 = o2[-1] + output_diff = ((o1 - o2) ** 2).sum() + assert output_diff < 1e-6, output_diff + + return fm + + +class FM_to_MD_HuggingFaceFM_Util(FM_to_MD_Util): + def set_hugging_face_api(self, hugging_face_api: HuggingFaceModelAPI): + self.hugging_face_api = hugging_face_api + + def init_md_from_fm_by_reducing_width(self, fm: nn.Module, reducing_width_ratio: int) -> nn.Module: + fm_vit = deepcopy(fm) + + # for block in fm_vit.bert.encoder.layer: + # set_module(block, 'attention.self', BertSelfAttentionPrunable.init_from_exist_self_attn(block.attention.self)) + + def _f(n): + return int(n // reducing_width_ratio) + + # def _rand_indexes(n): + # return torch.randperm(n)[0: int(n // reducing_width_ratio)] + + def l1_max_indexes(p: torch.Tensor, dim=0): + assert dim in [0, 1] + assert p.dim() in [1, 2, 4] + + if dim == 1: + p = p.T + + p_norm = p.abs().contiguous().view(p.size(0), -1).sum(dim=1) + n = p.size(0) + return p_norm.argsort(descending=True)[0: int(n // reducing_width_ratio)].sort()[0] + + layers_name = self.hugging_face_api.get_qkv_proj_ff1_ff2_layer_names() + if len(layers_name[0]) == 6: + q_names = [layer[0] for layer in layers_name] + k_names = [layer[1] for layer in layers_name] + v_names = [layer[2] for layer in layers_name] + qkv_proj_names = [layer[3] for layer in layers_name] + ff1_names = [layer[-2] for layer in layers_name] + ff2_names = [layer[-1] for layer in layers_name] + + for q_name, k_name, v_name, qkv_proj_name, ff1_name, ff2_name in zip(q_names, k_names, v_names, qkv_proj_names, ff1_names, ff2_names): + for k in [q_name, k_name, v_name]: + qkv = get_module(fm_vit, k) + + new_qkv = nn.Linear(qkv.in_features, _f(qkv.out_features), + qkv.bias is not None, qkv.weight.device) + indexes = l1_max_indexes(qkv.weight.data, 0) + + new_qkv.weight.data.copy_(qkv.weight.data[indexes]) + if qkv.bias is not None: + new_qkv.bias.data.copy_(qkv.bias.data[indexes]) + set_module(fm_vit, k, new_qkv) + + proj = get_module(fm_vit, qkv_proj_name) + new_proj = nn.Linear(_f(proj.in_features), proj.out_features, + proj.bias is not None, proj.weight.device) + new_proj.weight.data.copy_(proj.weight.data[:, l1_max_indexes(proj.weight.data, 1)]) + if proj.bias is not None: + new_proj.bias.data.copy_(proj.bias.data) + set_module(fm_vit, qkv_proj_name, new_proj) + + fc1 = get_module(fm_vit, ff1_name) + new_fc1 = nn.Linear(fc1.in_features, _f(fc1.out_features), + fc1.bias is not None, fc1.weight.device) + indexes = l1_max_indexes(fc1.weight.data, 0) + new_fc1.weight.data.copy_(fc1.weight.data[indexes]) + if fc1.bias is not None: + new_fc1.bias.data.copy_(fc1.bias.data[indexes]) + set_module(fm_vit, ff1_name, new_fc1) + + fc2 = get_module(fm_vit, ff2_name) + new_fc2 = nn.Linear(_f(fc2.in_features), fc2.out_features, + fc2.bias is not None, fc2.weight.device) + new_fc2.weight.data.copy_(fc2.weight.data[:, l1_max_indexes(fc2.weight.data, 1)]) + if fc2.bias is not None: + new_fc2.bias.data.copy_(fc2.bias.data) + set_module(fm_vit, ff2_name, new_fc2) + + if len(layers_name[0]) == 4: + qkv_names = [layer[0] for layer in layers_name] + qkv_proj_names = [layer[1] for layer in layers_name] + ff1_names = [layer[-2] for layer in layers_name] + ff2_names = [layer[-1] for layer in layers_name] + + for qkv_name, qkv_proj_name, ff1_name, ff2_name in zip(qkv_names, qkv_proj_names, ff1_names, ff2_names): + qkv = get_module(fm_vit, qkv_name) + new_qkv = nn.Linear(qkv.in_features, _f(qkv.out_features), + qkv.bias is not None, qkv.weight.device) + indexes = l1_max_indexes(qkv.weight.data, 0) + + new_qkv.weight.data.copy_(qkv.weight.data[indexes]) + if qkv.bias is not None: + new_qkv.bias.data.copy_(qkv.bias.data[indexes]) + set_module(fm_vit, qkv_name, new_qkv) + + proj = get_module(fm_vit, qkv_proj_name) + new_proj = nn.Linear(_f(proj.in_features), proj.out_features, + proj.bias is not None, proj.weight.device) + new_proj.weight.data.copy_(proj.weight.data[:, l1_max_indexes(proj.weight.data, 1)]) + if proj.bias is not None: + new_proj.bias.data.copy_(proj.bias.data) + set_module(fm_vit, qkv_proj_name, new_proj) + + fc1 = get_module(fm_vit, ff1_name) + new_fc1 = nn.Linear(fc1.in_features, _f(fc1.out_features), + fc1.bias is not None, fc1.weight.device) + indexes = l1_max_indexes(fc1.weight.data, 0) + new_fc1.weight.data.copy_(fc1.weight.data[indexes]) + if fc1.bias is not None: + new_fc1.bias.data.copy_(fc1.bias.data[indexes]) + set_module(fm_vit, ff1_name, new_fc1) + + fc2 = get_module(fm_vit, ff2_name) + new_fc2 = nn.Linear(_f(fc2.in_features), fc2.out_features, + fc2.bias is not None, fc2.weight.device) + new_fc2.weight.data.copy_(fc2.weight.data[:, l1_max_indexes(fc2.weight.data, 1)]) + if fc2.bias is not None: + new_fc2.bias.data.copy_(fc2.bias.data) + set_module(fm_vit, ff2_name, new_fc2) + + return fm_vit + + def init_md_from_fm_by_reducing_width_with_perf_test(self, fm: nn.Module, reducing_width_ratio: int, + samples: torch.Tensor) -> nn.Module: + fm_size = get_model_size(fm, True) + fm_latency = self._get_model_latency(fm, samples, 20, + get_model_device(fm), 20, False) + + master_dnn = self.init_md_from_fm_by_reducing_width(fm, reducing_width_ratio) + master_dnn_size = get_model_size(master_dnn, True) + logger.debug(f'inited master DNN: {master_dnn}') + master_dnn_latency = self._get_model_latency(master_dnn, samples, 20, + get_model_device(master_dnn), 20, False) + + logger.info(f'init master DNN (w/o FBS yet) by reducing foundation model\'s width (by {reducing_width_ratio:d}x)') + logger.info(f'foundation model ({fm_size:.3f}MB, {fm_latency:.4f}s/sample) -> ' + f'master DNN ({master_dnn_size:.3f}MB, {master_dnn_latency:.4f}s/sample)\n' + f'(model size: ↓ {(fm_size / master_dnn_size):.2f}x, ' + f'latency: ↓ {(fm_latency / master_dnn_latency):.2f}x)') + + return master_dnn + + def _get_model_latency(self, model: torch.nn.Module, model_input_size, sample_num: int, + device: str, warmup_sample_num: int, return_detail=False): + import time + + if isinstance(model_input_size, tuple): + dummy_input = torch.rand(model_input_size).to(device) + else: + dummy_input = model_input_size + + model = model.to(device) + model.eval() + + # warm up + with torch.no_grad(): + for _ in range(warmup_sample_num): + if isinstance(dummy_input, dict): + model(**dummy_input) + else: + model(dummy_input) + + infer_time_list = [] + + if device == 'cuda' or 'cuda' in str(device): + with torch.no_grad(): + for _ in range(sample_num): + s, e = torch.cuda.Event(enable_timing=True), torch.cuda.Event(enable_timing=True) + s.record() + + if isinstance(dummy_input, dict): + model(**dummy_input) + else: + model(dummy_input) + + e.record() + torch.cuda.synchronize() + cur_model_infer_time = s.elapsed_time(e) / 1000. + infer_time_list += [cur_model_infer_time] + + else: + with torch.no_grad(): + for _ in range(sample_num): + start = time.time() + + if isinstance(dummy_input, dict): + model(**dummy_input) + else: + model(dummy_input) + + cur_model_infer_time = time.time() - start + infer_time_list += [cur_model_infer_time] + + avg_infer_time = sum(infer_time_list) / sample_num + + if return_detail: + return avg_infer_time, infer_time_list + return avg_infer_time \ No newline at end of file diff --git a/methods/elasticdnn/hugging_face/user_impl.py b/methods/elasticdnn/hugging_face/user_impl.py new file mode 100644 index 0000000000000000000000000000000000000000..f4d1129c05cc100d0edccdbcf0ee1ab52d932df0 --- /dev/null +++ b/methods/elasticdnn/hugging_face/user_impl.py @@ -0,0 +1,38 @@ +from typing import List +import torch +from methods.base.model import BaseModel +import tqdm +from torch import nn +import torch.nn.functional as F +from abc import abstractmethod +from methods.elasticdnn.model.base import ElasticDNNUtil +from methods.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from methods.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util + +from utils.dl.common.model import LayerActivation + + +class HuggingFaceModelAPI: + @abstractmethod + def get_feature_hook(self, fm: nn.Module) -> LayerActivation: + pass + + @abstractmethod + def get_task_head_params(self, fm: nn.Module): + pass + + @abstractmethod + def get_qkv_proj_ff1_ff2_layer_names(self): + pass + + @abstractmethod + def get_accuracy(self, fm: nn.Module, test_loader, device, *args, **kwargs): + pass + + @abstractmethod + def infer(self, fm: nn.Module, x, *args, **kwargs): + pass + + @abstractmethod + def forward_to_get_task_loss(self, fm: nn.Module, x, y, *args, **kwargs): + pass \ No newline at end of file diff --git a/methods/elasticdnn/model/README.md b/methods/elasticdnn/model/README.md new file mode 100644 index 0000000000000000000000000000000000000000..1e979bdba16531dc6decf4fbd618c6d98f347010 --- /dev/null +++ b/methods/elasticdnn/model/README.md @@ -0,0 +1,8 @@ +## 20230527 + +only prune to_v (so should divide to_qkv to to_qkv and to_v, and only prune to_v) + +attn (1, 12, 197, 197) * pruned_v (1, 12, 197, int(64\*s)) = out1 (1, 12, 197, int(64\*s)) +out1 rearange to out2 (1, 197, int(768\*s)) + +out2 (1, 197, int(768\*s)) * pruned mlp_fc1 (int(768\*s), int(3072\*s)) \ No newline at end of file diff --git a/methods/elasticdnn/model/__init__.py b/methods/elasticdnn/model/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..90c653cedcc240c909fe3872390bf325d09a4de7 --- /dev/null +++ b/methods/elasticdnn/model/__init__.py @@ -0,0 +1,4 @@ +from .base import ElasticDNNUtil + +from .cnn import ElasticCNNUtil +from .vit import ElasticViTUtil diff --git a/methods/elasticdnn/model/__pycache__/__init__.cpython-38.pyc b/methods/elasticdnn/model/__pycache__/__init__.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ee3d3537fd488a0101129e372255c3351bff8aaf Binary files /dev/null and b/methods/elasticdnn/model/__pycache__/__init__.cpython-38.pyc differ diff --git a/methods/elasticdnn/model/__pycache__/base.cpython-38.pyc b/methods/elasticdnn/model/__pycache__/base.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..40a28be656772bc8e2a8991aab3520239d370363 Binary files /dev/null and b/methods/elasticdnn/model/__pycache__/base.cpython-38.pyc differ diff --git a/methods/elasticdnn/model/__pycache__/cnn.cpython-38.pyc b/methods/elasticdnn/model/__pycache__/cnn.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..960550955965407389fd98767ed61b6935f51c67 Binary files /dev/null and b/methods/elasticdnn/model/__pycache__/cnn.cpython-38.pyc differ diff --git a/methods/elasticdnn/model/__pycache__/vit.cpython-38.pyc b/methods/elasticdnn/model/__pycache__/vit.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ea08905b0fe45b5f775656d8ada340fac7b1c893 Binary files /dev/null and b/methods/elasticdnn/model/__pycache__/vit.cpython-38.pyc differ diff --git a/methods/elasticdnn/model/base.py b/methods/elasticdnn/model/base.py new file mode 100644 index 0000000000000000000000000000000000000000..86376da11c937273fc3b16b4c01f2d03c11d2033 --- /dev/null +++ b/methods/elasticdnn/model/base.py @@ -0,0 +1,139 @@ +import torch +from torch import nn +from abc import ABC, abstractmethod + +from utils.dl.common.model import get_model_device, get_model_latency, get_model_size +from utils.common.log import logger + + +class KTakesAll(nn.Module): + # k means sparsity (the larger k is, the smaller model is) + def __init__(self, k): + super(KTakesAll, self).__init__() + self.k = k + self.cached_i = None + + def forward(self, g: torch.Tensor): + # k = int(g.size(1) * self.k) + # i = (-g).topk(k, 1)[1] + # t = g.scatter(1, i, 0) + + k = int(g.size(-1) * self.k) + i = (-g).topk(k, -1)[1] + self.cached_i = i + t = g.scatter(-1, i, 0) + + return t + + +class Abs(nn.Module): + def __init__(self): + super(Abs, self).__init__() + + def forward(self, x): + return x.abs() + + +class Layer_WrappedWithFBS(nn.Module): + def __init__(self): + super(Layer_WrappedWithFBS, self).__init__() + + init_sparsity = 0.5 + self.k_takes_all = KTakesAll(init_sparsity) + + self.cached_raw_channel_attention = None + self.cached_channel_attention = None + self.use_cached_channel_attention = False + + +class ElasticDNNUtil(ABC): + @abstractmethod + def convert_raw_dnn_to_master_dnn(self, raw_dnn: nn.Module, r: float, ignore_layers=[]): + raise NotImplementedError + + def convert_raw_dnn_to_master_dnn_with_perf_test(self, raw_dnn: nn.Module, r: float, ignore_layers=[]): + raw_dnn_size = get_model_size(raw_dnn, True) + master_dnn = self.convert_raw_dnn_to_master_dnn(raw_dnn, r, ignore_layers) + master_dnn_size = get_model_size(master_dnn, True) + + logger.info(f'master DNN w/o FBS ({raw_dnn_size:.3f}MB) -> master DNN w/ FBS ({master_dnn_size:.3f}MB) ' + f'(↑ {(((master_dnn_size - raw_dnn_size) / raw_dnn_size) * 100.):.2f}%)') + return master_dnn + + def set_master_dnn_inference_via_cached_channel_attention(self, master_dnn: nn.Module): + for name, module in master_dnn.named_modules(): + if isinstance(module, Layer_WrappedWithFBS): + assert module.cached_channel_attention is not None + module.use_cached_channel_attention = True + + def set_master_dnn_dynamic_inference(self, master_dnn: nn.Module): + for name, module in master_dnn.named_modules(): + if isinstance(module, Layer_WrappedWithFBS): + module.cached_channel_attention = None + module.use_cached_channel_attention = False + + def train_only_fbs_of_master_dnn(self, master_dnn: nn.Module): + fbs_params = [] + for n, p in master_dnn.named_parameters(): + if '.fbs' in n: + fbs_params += [p] + p.requires_grad = True + else: + p.requires_grad = False + return fbs_params + + def get_accu_l1_reg_of_raw_channel_attention_in_master_dnn(self, master_dnn: nn.Module): + res = 0. + for name, module in master_dnn.named_modules(): + if isinstance(module, Layer_WrappedWithFBS): + res += module.cached_raw_channel_attention.norm(1) + return res + + def get_raw_channel_attention_in_master_dnn(self, master_dnn: nn.Module): + res = {} + for name, module in master_dnn.named_modules(): + if isinstance(module, Layer_WrappedWithFBS): + res[name] = module.cached_raw_channel_attention + return res + + def set_master_dnn_sparsity(self, master_dnn: nn.Module, sparsity: float): + assert 0 <= sparsity <= 1., sparsity + for name, module in master_dnn.named_modules(): + if isinstance(module, KTakesAll): + module.k = sparsity + logger.debug(f'set master DNN sparsity to {sparsity}') + + def clear_cached_channel_attention_in_master_dnn(self, master_dnn: nn.Module): + for name, module in master_dnn.named_modules(): + if isinstance(module, Layer_WrappedWithFBS): + module.cached_raw_channel_attention = None + module.cached_channel_attention = None + + @abstractmethod + def select_most_rep_sample(self, master_dnn: nn.Module, samples: torch.Tensor): + raise NotImplementedError + + @abstractmethod + def extract_surrogate_dnn_via_samples(self, master_dnn: nn.Module, samples: torch.Tensor, return_detail=False): + raise NotImplementedError + + def extract_surrogate_dnn_via_samples_with_perf_test(self, master_dnn: nn.Module, samples: torch.Tensor, return_detail=False): + master_dnn_size = get_model_size(master_dnn, True) + master_dnn_latency = get_model_latency(master_dnn, (1, *list(samples.size())[1:]), 50, + get_model_device(master_dnn), 50, False) + + res = self.extract_surrogate_dnn_via_samples(master_dnn, samples, return_detail) + if not return_detail: + surrogate_dnn = res + else: + surrogate_dnn, unpruned_indexes_of_layers = res + surrogate_dnn_size = get_model_size(surrogate_dnn, True) + surrogate_dnn_latency = get_model_latency(surrogate_dnn, (1, *list(samples.size())[1:]), 50, + get_model_device(surrogate_dnn), 50, False) + + logger.info(f'master DNN ({master_dnn_size:.3f}MB, {master_dnn_latency:.4f}s/sample) -> ' + f'surrogate DNN ({surrogate_dnn_size:.3f}MB, {surrogate_dnn_latency:.4f}s/sample)\n' + f'(model size: ↓ {(master_dnn_size / surrogate_dnn_size):.2f}x, ' + f'latency: ↓ {(master_dnn_latency / surrogate_dnn_latency):.2f}x)') + + return res diff --git a/methods/elasticdnn/model/bert.py b/methods/elasticdnn/model/bert.py new file mode 100644 index 0000000000000000000000000000000000000000..e8f87a6f4943b8193b7d2a3be7223e107a8e22e1 --- /dev/null +++ b/methods/elasticdnn/model/bert.py @@ -0,0 +1,359 @@ +from copy import deepcopy +from typing import Optional, Union +import torch +from torch import nn +from einops import rearrange, repeat +from einops.layers.torch import Rearrange +import tqdm + +from utils.dl.common.model import get_model_device, get_model_size, set_module, get_module +from .base import Abs, KTakesAll, ElasticDNNUtil, Layer_WrappedWithFBS +from utils.common.log import logger + + +class SqueezeLast(nn.Module): + def __init__(self): + super(SqueezeLast, self).__init__() + + def forward(self, x): + return x.squeeze(-1) + + +class ProjConv_WrappedWithFBS(Layer_WrappedWithFBS): + def __init__(self, proj: nn.Conv2d, r): + super(ProjConv_WrappedWithFBS, self).__init__() + + self.proj = proj + + # for conv: (B, C_in, H, W) -> (B, C_in) -> (B, C_out) + # for mlp in ViT: (B, #patches, D: dim of patches embedding) -> (B, D) -> (B, C_out) + self.fbs = nn.Sequential( + Abs(), + nn.AdaptiveAvgPool1d(1), + SqueezeLast(), + nn.Linear(proj.in_channels, proj.out_channels // r), + nn.ReLU(), + nn.Linear(proj.out_channels // r, proj.out_channels), + nn.ReLU() + ) + + nn.init.constant_(self.fbs[6].bias, 1.) + nn.init.kaiming_normal_(self.fbs[6].weight) + + + def forward(self, x): + if self.use_cached_channel_attention and self.cached_channel_attention is not None: + channel_attention = self.cached_channel_attention + else: + self.cached_raw_channel_attention = self.fbs(x) + self.cached_channel_attention = self.k_takes_all(self.cached_raw_channel_attention) + + channel_attention = self.cached_channel_attention + + raw_res = self.proj(x) + + return channel_attention.unsqueeze(1) * raw_res # TODO: + + +class Linear_WrappedWithFBS(Layer_WrappedWithFBS): + def __init__(self, linear: nn.Linear, r): + super(Linear_WrappedWithFBS, self).__init__() + + self.linear = linear + + # for conv: (B, C_in, H, W) -> (B, C_in) -> (B, C_out) + # for mlp in ViT: (B, #patches, D: dim of patches embedding) -> (B, D) -> (B, C_out) + self.fbs = nn.Sequential( + Rearrange('b n d -> b d n'), + Abs(), + nn.AdaptiveAvgPool1d(1), + SqueezeLast(), + nn.Linear(linear.in_features, linear.out_features // r), + nn.ReLU(), + nn.Linear(linear.out_features // r, linear.out_features), + nn.ReLU() + ) + + nn.init.constant_(self.fbs[6].bias, 1.) + nn.init.kaiming_normal_(self.fbs[6].weight) + + + def forward(self, x): + if self.use_cached_channel_attention and self.cached_channel_attention is not None: + channel_attention = self.cached_channel_attention + else: + self.cached_raw_channel_attention = self.fbs(x) + self.cached_channel_attention = self.k_takes_all(self.cached_raw_channel_attention) + + channel_attention = self.cached_channel_attention + + raw_res = self.linear(x) + + return channel_attention.unsqueeze(1) * raw_res + + +class ToQKV_WrappedWithFBS(Layer_WrappedWithFBS): + """ + This regards to_q/to_k/to_v as a whole (in fact it consists of multiple heads) and prunes it. + It seems different channels of different heads are pruned according to the input. + This is different from "removing some head" or "removing the same channels in each head". + """ + def __init__(self, to_qkv: nn.Linear, r): + super(ToQKV_WrappedWithFBS, self).__init__() + + # self.to_qkv = to_qkv + + self.to_qk = nn.Linear(to_qkv.in_features, to_qkv.out_features // 3 * 2, bias=to_qkv.bias is not None) + self.to_v = nn.Linear(to_qkv.in_features, to_qkv.out_features // 3, bias=to_qkv.bias is not None) + self.to_qk.weight.data.copy_(to_qkv.weight.data[0: to_qkv.out_features // 3 * 2]) + if to_qkv.bias is not None: + self.to_qk.bias.data.copy_(to_qkv.bias.data[0: to_qkv.out_features // 3 * 2]) + self.to_v.weight.data.copy_(to_qkv.weight.data[to_qkv.out_features // 3 * 2: ]) + if to_qkv.bias is not None: + self.to_v.bias.data.copy_(to_qkv.bias.data[to_qkv.out_features // 3 * 2: ]) + + self.fbs = nn.Sequential( + Rearrange('b n d -> b d n'), + Abs(), + nn.AdaptiveAvgPool1d(1), + SqueezeLast(), + nn.Linear(to_qkv.in_features, to_qkv.out_features // 3 // r), + nn.ReLU(), + # nn.Linear(to_qkv.out_features // 3 // r, to_qkv.out_features // 3), + nn.Linear(to_qkv.out_features // 3 // r, self.to_v.out_features), + nn.ReLU() + ) + + nn.init.constant_(self.fbs[6].bias, 1.) + nn.init.kaiming_normal_(self.fbs[6].weight) + + def forward(self, x): + if self.use_cached_channel_attention and self.cached_channel_attention is not None: + channel_attention = self.cached_channel_attention + else: + self.cached_raw_channel_attention = self.fbs(x) + + # print() + # for attn in self.cached_raw_channel_attention.chunk(3, dim=1)[0: 1]: + # print(self.cached_raw_channel_attention.size(), attn.size()) + # print(self.k_takes_all.k) + # print(attn[0].nonzero(as_tuple=True)[0].size(), attn[0]) + + self.cached_channel_attention = self.k_takes_all(self.cached_raw_channel_attention) + + + # for attn in self.cached_channel_attention.chunk(3, dim=1)[0: 1]: + # print(self.cached_channel_attention.size(), attn.size()) + # print(self.k_takes_all.k) + # print(attn[0].nonzero(as_tuple=True)[0].size(), attn[0]) + # print() + + channel_attention = self.cached_channel_attention + + qk = self.to_qk(x) + v = channel_attention.unsqueeze(1) * self.to_v(x) + return torch.cat([qk, v], dim=-1) + + # qkv = raw_res.chunk(3, dim = -1) + + # raw_v = qkv[2] + # print('raw_k, raw_v', qkv[0].sum((0, 1))[0: 10], qkv[0].sum((0, 1)).nonzero(as_tuple=True)[0].size(), + # qkv[1].sum((0, 1))[0: 10], qkv[1].sum((0, 1)).nonzero(as_tuple=True)[0].size(),) + # print('raw_v', raw_v.size(), raw_v.sum((0, 1))[0: 10], raw_v.sum((0, 1)).nonzero(as_tuple=True)[0].size()) + + # qkv_attn = channel_attention.chunk(3, dim=-1) + # print('attn', [attn[0][0: 10] for attn in qkv_attn]) + # print(channel_attention.unsqueeze(1).size(), raw_res.size()) + # print('fbs', channel_attention.size(), raw_res.size()) + # return channel_attention.unsqueeze(1) * raw_res + + +class StaticFBS(nn.Module): + def __init__(self, static_channel_attention): + super(StaticFBS, self).__init__() + assert static_channel_attention.dim() == 2 and static_channel_attention.size(0) == 1 + self.static_channel_attention = nn.Parameter(static_channel_attention, requires_grad=False) # (1, dim) + + def forward(self, x): + # print('staticfbs', x, self.static_channel_attention.unsqueeze(1)) + return x * self.static_channel_attention.unsqueeze(1) + + +class ElasticBertUtil(ElasticDNNUtil): + def convert_raw_dnn_to_master_dnn(self, raw_dnn: nn.Module, r: float, ignore_layers=[]): + assert len(ignore_layers) == 0, 'not supported yet' + + raw_vit = deepcopy(raw_dnn) + + # set_module(module, 'patch_embed.proj', ProjConv_WrappedWithFBS(module.patch_embed.proj, r)) + + for name, module in raw_vit.named_modules(): + # if name.endswith('attn'): + # set_module(module, 'qkv', ToQKV_WrappedWithFBS(module.qkv, r)) + if name.endswith('intermediate'): + set_module(module, 'dense', Linear_WrappedWithFBS(module.dense, r)) + + return raw_vit + + def set_master_dnn_sparsity(self, master_dnn: nn.Module, sparsity: float): + # for name, module in master_dnn.named_modules(): + # if not name.endswith('attn'): + # continue + + # q_features = module.qkv.to_qk.out_features // 2 + + # if (q_features - int(q_features * sparsity)) % module.num_heads != 0: + # # tune sparsity to ensure #unpruned channel % num_heads == 0 + # # so that the pruning seems to reduce the dim_head of each head + # tuned_sparsity = 1. - int((q_features - int(q_features * sparsity)) / module.num_heads) * module.num_heads / q_features + # logger.debug(f'tune sparsity from {sparsity:.2f} to {tuned_sparsity}') + # sparsity = tuned_sparsity + # break + + return super().set_master_dnn_sparsity(master_dnn, sparsity) + + def select_most_rep_sample(self, master_dnn: nn.Module, samples: torch.Tensor): + # print(samples) + # return samples[0].unsqueeze(0) + res = {k: v[0: 1] for k, v in samples.items()} + return res + + def extract_surrogate_dnn_via_samples(self, master_dnn: nn.Module, samples: torch.Tensor, return_detail=False): + sample = self.select_most_rep_sample(master_dnn, samples) + # assert sample.dim() == 4 and sample.size(0) == 1 + + # print('before') + master_dnn.eval() + self.clear_cached_channel_attention_in_master_dnn(master_dnn) + with torch.no_grad(): + master_dnn_output = master_dnn(**sample) + + # print('after') + + boosted_vit = deepcopy(master_dnn) + + def get_unpruned_indexes_from_channel_attn(channel_attn: torch.Tensor, k): + assert channel_attn.size(0) == 1, 'use A representative sample to generate channel attentions' + + # print('attn_in_unpruned', channel_attn[0][0: 10]) + + res = channel_attn[0].nonzero(as_tuple=True)[0] # should be one-dim + + # res = channel_attn[0].argsort(descending=True)[0: -int(channel_attn.size(1) * k)].sort()[0] + + # g = channel_attn + # k = g.size(1) - int(g.size(1) * k) + # res = g.topk(k, 1)[1][0].sort()[0] + + return res + + unpruned_indexes_of_layers = {} + + # for attn, ff in boosted_vit.transformer.layers: + # for block_i, block in enumerate(boosted_vit.blocks): + for block_i, block in enumerate(boosted_vit.bert.encoder.layer): + # attn = block.attn + # ff = block.mlp + + ff_0 = get_module(block, f'intermediate.dense') + # ff_0_unpruned_indexes = get_unpruned_indexes_from_channel_attn(ff_0.cached_channel_attention, k) + ff_0_pruned_indexes = ff_0.k_takes_all.cached_i[0].sort()[0] + ff_0_unpruned_indexes = torch.LongTensor([ii for ii in range(ff_0.cached_channel_attention.size(1)) if ii not in ff_0_pruned_indexes]) + new_ff_0 = nn.Linear(ff_0.linear.in_features, ff_0_unpruned_indexes.size(0), ff_0.linear.bias is not None) + new_ff_0.weight.data.copy_(ff_0.linear.weight.data[ff_0_unpruned_indexes]) + if ff_0.linear.bias is not None: + new_ff_0.bias.data.copy_(ff_0.linear.bias.data[ff_0_unpruned_indexes]) + set_module(block, 'intermediate.dense', nn.Sequential(new_ff_0, StaticFBS(ff_0.cached_channel_attention[:, ff_0_unpruned_indexes]))) + + ff_1 = get_module(block, f'output.dense') + new_ff_1 = nn.Linear(ff_0_unpruned_indexes.size(0), ff_1.out_features, ff_1.bias is not None) + new_ff_1.weight.data.copy_(ff_1.weight.data[:, ff_0_unpruned_indexes]) + if ff_1.bias is not None: + new_ff_1.bias.data.copy_(ff_1.bias.data) + set_module(block, 'output.dense', new_ff_1) + + unpruned_indexes_of_layers[f'bert.encoder.layer.{block_i}.intermediate.dense.0.weight'] = ff_0_unpruned_indexes + + surrogate_dnn = boosted_vit + surrogate_dnn.eval() + surrogate_dnn = surrogate_dnn.to(get_model_device(master_dnn)) + # logger.debug(surrogate_dnn) + with torch.no_grad(): + surrogate_dnn_output = surrogate_dnn(**sample) + + output_diff = ((surrogate_dnn_output - master_dnn_output) ** 2).sum() + # assert output_diff < 1e-4, output_diff + logger.info(f'output diff of master and surrogate DNN: {output_diff}') + logger.debug(f'example output of master/surrogate: {master_dnn_output.sum(0)[0: 10]}, {surrogate_dnn_output.sum(0)[0: 10]}') + # logger.info(f'\nonly prune mlp!!!!\n') + # logger.info(f'\nonly prune mlp!!!!\n') + + if return_detail: + return boosted_vit, unpruned_indexes_of_layers + + return boosted_vit + + def extract_surrogate_dnn_via_samples_with_perf_test(self, master_dnn: nn.Module, samples: torch.Tensor, return_detail=False): + master_dnn_size = get_model_size(master_dnn, True) + master_dnn_latency = self._get_model_latency(master_dnn, samples, 50, + get_model_device(master_dnn), 50, False) + + res = self.extract_surrogate_dnn_via_samples(master_dnn, samples, return_detail) + if not return_detail: + surrogate_dnn = res + else: + surrogate_dnn, unpruned_indexes_of_layers = res + surrogate_dnn_size = get_model_size(surrogate_dnn, True) + surrogate_dnn_latency = self._get_model_latency(master_dnn, samples, 50, + get_model_device(master_dnn), 50, False) + + logger.info(f'master DNN ({master_dnn_size:.3f}MB, {master_dnn_latency:.4f}s/sample) -> ' + f'surrogate DNN ({surrogate_dnn_size:.3f}MB, {surrogate_dnn_latency:.4f}s/sample)\n' + f'(model size: ↓ {(master_dnn_size / surrogate_dnn_size):.2f}x, ' + f'latency: ↓ {(master_dnn_latency / surrogate_dnn_latency):.2f}x)') + + return res + + def _get_model_latency(self, model: torch.nn.Module, model_input_size, sample_num: int, + device: str, warmup_sample_num: int, return_detail=False): + import time + + if isinstance(model_input_size, tuple): + dummy_input = torch.rand(model_input_size).to(device) + else: + dummy_input = model_input_size + + model = model.to(device) + model.eval() + + # warm up + with torch.no_grad(): + for _ in range(warmup_sample_num): + model(**dummy_input) + + infer_time_list = [] + + if device == 'cuda' or 'cuda' in str(device): + with torch.no_grad(): + for _ in range(sample_num): + s, e = torch.cuda.Event(enable_timing=True), torch.cuda.Event(enable_timing=True) + s.record() + model(**dummy_input) + e.record() + torch.cuda.synchronize() + cur_model_infer_time = s.elapsed_time(e) / 1000. + infer_time_list += [cur_model_infer_time] + + else: + with torch.no_grad(): + for _ in range(sample_num): + start = time.time() + model(**dummy_input) + cur_model_infer_time = time.time() - start + infer_time_list += [cur_model_infer_time] + + avg_infer_time = sum(infer_time_list) / sample_num + + if return_detail: + return avg_infer_time, infer_time_list + return avg_infer_time \ No newline at end of file diff --git a/methods/elasticdnn/model/clip.py b/methods/elasticdnn/model/clip.py new file mode 100644 index 0000000000000000000000000000000000000000..d7ff70af97a345fe2e7228d239c2ce2dda5ab1c0 --- /dev/null +++ b/methods/elasticdnn/model/clip.py @@ -0,0 +1,316 @@ +from copy import deepcopy +from typing import Optional, Union +import torch +from torch import nn +from einops import rearrange, repeat +from einops.layers.torch import Rearrange +import tqdm + +from utils.dl.common.model import get_model_device, get_model_size, set_module, get_module +from .base import Abs, KTakesAll, ElasticDNNUtil, Layer_WrappedWithFBS +from utils.common.log import logger + + +class SqueezeLast(nn.Module): + def __init__(self): + super(SqueezeLast, self).__init__() + + def forward(self, x): + return x.squeeze(-1) + + +class ProjConv_WrappedWithFBS(Layer_WrappedWithFBS): + def __init__(self, proj: nn.Conv2d, r): + super(ProjConv_WrappedWithFBS, self).__init__() + + self.proj = proj + + # for conv: (B, C_in, H, W) -> (B, C_in) -> (B, C_out) + # for mlp in ViT: (B, #patches, D: dim of patches embedding) -> (B, D) -> (B, C_out) + self.fbs = nn.Sequential( + Abs(), + nn.AdaptiveAvgPool1d(1), + SqueezeLast(), + nn.Linear(proj.in_channels, proj.out_channels // r), + nn.ReLU(), + nn.Linear(proj.out_channels // r, proj.out_channels), + nn.ReLU() + ) + + nn.init.constant_(self.fbs[6].bias, 1.) + nn.init.kaiming_normal_(self.fbs[6].weight) + + + def forward(self, x): + if self.use_cached_channel_attention and self.cached_channel_attention is not None: + channel_attention = self.cached_channel_attention + else: + self.cached_raw_channel_attention = self.fbs(x) + self.cached_channel_attention = self.k_takes_all(self.cached_raw_channel_attention) + + channel_attention = self.cached_channel_attention + + raw_res = self.proj(x) + + return channel_attention.unsqueeze(1) * raw_res # TODO: + + +class Linear_WrappedWithFBS(Layer_WrappedWithFBS): + def __init__(self, linear: nn.Linear, r): + super(Linear_WrappedWithFBS, self).__init__() + + self.linear = linear + + # for conv: (B, C_in, H, W) -> (B, C_in) -> (B, C_out) + # for mlp in ViT: (B, #patches, D: dim of patches embedding) -> (B, D) -> (B, C_out) + self.fbs = nn.Sequential( + Rearrange('b n d -> b d n'), + Abs(), + nn.AdaptiveAvgPool1d(1), + SqueezeLast(), + nn.Linear(linear.in_features, linear.out_features // r), + nn.ReLU(), + nn.Linear(linear.out_features // r, linear.out_features), + nn.ReLU() + ) + + nn.init.constant_(self.fbs[6].bias, 1.) + nn.init.kaiming_normal_(self.fbs[6].weight) + + + def forward(self, x): + if self.use_cached_channel_attention and self.cached_channel_attention is not None: + channel_attention = self.cached_channel_attention + else: + self.cached_raw_channel_attention = self.fbs(x) + self.cached_channel_attention = self.k_takes_all(self.cached_raw_channel_attention) + + channel_attention = self.cached_channel_attention + + raw_res = self.linear(x) + + return channel_attention.unsqueeze(1) * raw_res + + +class ToQKV_WrappedWithFBS(Layer_WrappedWithFBS): + """ + This regards to_q/to_k/to_v as a whole (in fact it consists of multiple heads) and prunes it. + It seems different channels of different heads are pruned according to the input. + This is different from "removing some head" or "removing the same channels in each head". + """ + def __init__(self, to_qkv: nn.Linear, r): + super(ToQKV_WrappedWithFBS, self).__init__() + + # self.to_qkv = to_qkv + + self.to_qk = nn.Linear(to_qkv.in_features, to_qkv.out_features // 3 * 2, bias=to_qkv.bias is not None) + self.to_v = nn.Linear(to_qkv.in_features, to_qkv.out_features // 3, bias=to_qkv.bias is not None) + self.to_qk.weight.data.copy_(to_qkv.weight.data[0: to_qkv.out_features // 3 * 2]) + if to_qkv.bias is not None: + self.to_qk.bias.data.copy_(to_qkv.bias.data[0: to_qkv.out_features // 3 * 2]) + self.to_v.weight.data.copy_(to_qkv.weight.data[to_qkv.out_features // 3 * 2: ]) + if to_qkv.bias is not None: + self.to_v.bias.data.copy_(to_qkv.bias.data[to_qkv.out_features // 3 * 2: ]) + + self.fbs = nn.Sequential( + Rearrange('b n d -> b d n'), + Abs(), + nn.AdaptiveAvgPool1d(1), + SqueezeLast(), + nn.Linear(to_qkv.in_features, to_qkv.out_features // 3 // r), + nn.ReLU(), + # nn.Linear(to_qkv.out_features // 3 // r, to_qkv.out_features // 3), + nn.Linear(to_qkv.out_features // 3 // r, self.to_v.out_features), + nn.ReLU() + ) + + nn.init.constant_(self.fbs[6].bias, 1.) + nn.init.kaiming_normal_(self.fbs[6].weight) + + def forward(self, x): + if self.use_cached_channel_attention and self.cached_channel_attention is not None: + channel_attention = self.cached_channel_attention + else: + self.cached_raw_channel_attention = self.fbs(x) + + # print() + # for attn in self.cached_raw_channel_attention.chunk(3, dim=1)[0: 1]: + # print(self.cached_raw_channel_attention.size(), attn.size()) + # print(self.k_takes_all.k) + # print(attn[0].nonzero(as_tuple=True)[0].size(), attn[0]) + + self.cached_channel_attention = self.k_takes_all(self.cached_raw_channel_attention) + + + # for attn in self.cached_channel_attention.chunk(3, dim=1)[0: 1]: + # print(self.cached_channel_attention.size(), attn.size()) + # print(self.k_takes_all.k) + # print(attn[0].nonzero(as_tuple=True)[0].size(), attn[0]) + # print() + + channel_attention = self.cached_channel_attention + + qk = self.to_qk(x) + v = channel_attention.unsqueeze(1) * self.to_v(x) + return torch.cat([qk, v], dim=-1) + + # qkv = raw_res.chunk(3, dim = -1) + + # raw_v = qkv[2] + # print('raw_k, raw_v', qkv[0].sum((0, 1))[0: 10], qkv[0].sum((0, 1)).nonzero(as_tuple=True)[0].size(), + # qkv[1].sum((0, 1))[0: 10], qkv[1].sum((0, 1)).nonzero(as_tuple=True)[0].size(),) + # print('raw_v', raw_v.size(), raw_v.sum((0, 1))[0: 10], raw_v.sum((0, 1)).nonzero(as_tuple=True)[0].size()) + + # qkv_attn = channel_attention.chunk(3, dim=-1) + # print('attn', [attn[0][0: 10] for attn in qkv_attn]) + # print(channel_attention.unsqueeze(1).size(), raw_res.size()) + # print('fbs', channel_attention.size(), raw_res.size()) + # return channel_attention.unsqueeze(1) * raw_res + + +class StaticFBS(nn.Module): + def __init__(self, static_channel_attention): + super(StaticFBS, self).__init__() + assert static_channel_attention.dim() == 2 and static_channel_attention.size(0) == 1 + self.static_channel_attention = nn.Parameter(static_channel_attention, requires_grad=False) # (1, dim) + + def forward(self, x): + # print('staticfbs', x, self.static_channel_attention.unsqueeze(1)) + return x * self.static_channel_attention.unsqueeze(1) + + +class ElasticCLIPUtil(ElasticDNNUtil): + def convert_raw_dnn_to_master_dnn(self, raw_dnn: nn.Module, r: float, ignore_layers=[]): + assert len(ignore_layers) == 0, 'not supported yet' + + raw_vit = deepcopy(raw_dnn) + + # set_module(module, 'patch_embed.proj', ProjConv_WrappedWithFBS(module.patch_embed.proj, r)) + + for name, module in raw_vit.named_modules(): + # if name.endswith('attn'): + # set_module(module, 'qkv', ToQKV_WrappedWithFBS(module.qkv, r)) + if 'vision_model' in name and name.endswith('mlp'): + set_module(module, 'fc1', Linear_WrappedWithFBS(module.fc1, r)) + + return raw_vit + + def set_master_dnn_sparsity(self, master_dnn: nn.Module, sparsity: float): + # for name, module in master_dnn.named_modules(): + # if not name.endswith('attn'): + # continue + + # q_features = module.qkv.to_qk.out_features // 2 + + # if (q_features - int(q_features * sparsity)) % module.num_heads != 0: + # # tune sparsity to ensure #unpruned channel % num_heads == 0 + # # so that the pruning seems to reduce the dim_head of each head + # tuned_sparsity = 1. - int((q_features - int(q_features * sparsity)) / module.num_heads) * module.num_heads / q_features + # logger.debug(f'tune sparsity from {sparsity:.2f} to {tuned_sparsity}') + # sparsity = tuned_sparsity + # break + + return super().set_master_dnn_sparsity(master_dnn, sparsity) + + def select_most_rep_sample(self, master_dnn: nn.Module, samples: torch.Tensor): + return samples[0].unsqueeze(0) + + def extract_surrogate_dnn_via_samples(self, master_dnn: nn.Module, samples: torch.Tensor, return_detail=False): + sample = self.select_most_rep_sample(master_dnn, samples) + assert sample.dim() == 4 and sample.size(0) == 1 + + # print('before') + master_dnn.eval() + self.clear_cached_channel_attention_in_master_dnn(master_dnn) + with torch.no_grad(): + master_dnn_output = master_dnn(sample) + + # print('after') + + boosted_vit = deepcopy(master_dnn) + + def get_unpruned_indexes_from_channel_attn(channel_attn: torch.Tensor, k): + assert channel_attn.size(0) == 1, 'use A representative sample to generate channel attentions' + + # print('attn_in_unpruned', channel_attn[0][0: 10]) + + res = channel_attn[0].nonzero(as_tuple=True)[0] # should be one-dim + + # res = channel_attn[0].argsort(descending=True)[0: -int(channel_attn.size(1) * k)].sort()[0] + + # g = channel_attn + # k = g.size(1) - int(g.size(1) * k) + # res = g.topk(k, 1)[1][0].sort()[0] + + return res + + unpruned_indexes_of_layers = {} + + # for attn, ff in boosted_vit.transformer.layers: + # for block_i, block in enumerate(boosted_vit.blocks): + # for block_i, block in enumerate(boosted_vit.model.text_model.encoder.layers): + # # attn = block.attn + # # ff = block.mlp + + # ff_0 = get_module(block, f'mlp.fc1') + # # ff_0_unpruned_indexes = get_unpruned_indexes_from_channel_attn(ff_0.cached_channel_attention, k) + # ff_0_pruned_indexes = ff_0.k_takes_all.cached_i[0].sort()[0] + # ff_0_unpruned_indexes = torch.LongTensor([ii for ii in range(ff_0.cached_channel_attention.size(1)) if ii not in ff_0_pruned_indexes]) + # new_ff_0 = nn.Linear(ff_0.linear.in_features, ff_0_unpruned_indexes.size(0), ff_0.linear.bias is not None) + # new_ff_0.weight.data.copy_(ff_0.linear.weight.data[ff_0_unpruned_indexes]) + # if ff_0.linear.bias is not None: + # new_ff_0.bias.data.copy_(ff_0.linear.bias.data[ff_0_unpruned_indexes]) + # set_module(block, 'mlp.fc1', nn.Sequential(new_ff_0, StaticFBS(ff_0.cached_channel_attention[:, ff_0_unpruned_indexes]))) + + # ff_1 = get_module(block, f'mlp.fc2') + # new_ff_1 = nn.Linear(ff_0_unpruned_indexes.size(0), ff_1.out_features, ff_1.bias is not None) + # new_ff_1.weight.data.copy_(ff_1.weight.data[:, ff_0_unpruned_indexes]) + # if ff_1.bias is not None: + # new_ff_1.bias.data.copy_(ff_1.bias.data) + # set_module(block, 'mlp.fc2', new_ff_1) + + # unpruned_indexes_of_layers[f'boosted_vit.model.text_model.encoder.layers.{block_i}.mlp.fc1.weight'] = ff_0_unpruned_indexes + + + for block_i, block in enumerate(boosted_vit.model.vision_model.encoder.layers): + # attn = block.attn + # ff = block.mlp + + ff_0 = get_module(block, f'mlp.fc1') + # ff_0_unpruned_indexes = get_unpruned_indexes_from_channel_attn(ff_0.cached_channel_attention, k) + ff_0_pruned_indexes = ff_0.k_takes_all.cached_i[0].sort()[0] + ff_0_unpruned_indexes = torch.LongTensor([ii for ii in range(ff_0.cached_channel_attention.size(1)) if ii not in ff_0_pruned_indexes]) + new_ff_0 = nn.Linear(ff_0.linear.in_features, ff_0_unpruned_indexes.size(0), ff_0.linear.bias is not None) + new_ff_0.weight.data.copy_(ff_0.linear.weight.data[ff_0_unpruned_indexes]) + if ff_0.linear.bias is not None: + new_ff_0.bias.data.copy_(ff_0.linear.bias.data[ff_0_unpruned_indexes]) + set_module(block, 'mlp.fc1', nn.Sequential(new_ff_0, StaticFBS(ff_0.cached_channel_attention[:, ff_0_unpruned_indexes]))) + + ff_1 = get_module(block, f'mlp.fc2') + new_ff_1 = nn.Linear(ff_0_unpruned_indexes.size(0), ff_1.out_features, ff_1.bias is not None) + new_ff_1.weight.data.copy_(ff_1.weight.data[:, ff_0_unpruned_indexes]) + if ff_1.bias is not None: + new_ff_1.bias.data.copy_(ff_1.bias.data) + set_module(block, 'mlp.fc2', new_ff_1) + + unpruned_indexes_of_layers[f'boosted_vit.model.vision_model.encoder.layers.{block_i}.mlp.fc1.weight'] = ff_0_unpruned_indexes + + surrogate_dnn = boosted_vit + surrogate_dnn.eval() + surrogate_dnn = surrogate_dnn.to(get_model_device(master_dnn)) + # logger.debug(surrogate_dnn) + with torch.no_grad(): + surrogate_dnn_output = surrogate_dnn(sample) + + output_diff = ((surrogate_dnn_output - master_dnn_output) ** 2).sum() + # assert output_diff < 1e-4, output_diff + logger.info(f'output diff of master and surrogate DNN: {output_diff}') + logger.debug(f'example output of master/surrogate: {master_dnn_output.sum(0)[0: 10]}, {surrogate_dnn_output.sum(0)[0: 10]}') + # logger.info(f'\nonly prune mlp!!!!\n') + # logger.info(f'\nonly prune mlp!!!!\n') + + if return_detail: + return boosted_vit, unpruned_indexes_of_layers + + return boosted_vit + \ No newline at end of file diff --git a/methods/elasticdnn/model/cnn.py b/methods/elasticdnn/model/cnn.py new file mode 100644 index 0000000000000000000000000000000000000000..c8b7fd4339921e08e715f6993d700bd8265b866f --- /dev/null +++ b/methods/elasticdnn/model/cnn.py @@ -0,0 +1,147 @@ +from typing import Optional +import torch +from copy import deepcopy +from torch import nn +from utils.common.others import get_cur_time_str +from utils.dl.common.model import get_model_device, get_model_latency, get_model_size, get_module, get_super_module, set_module +from utils.common.log import logger +from utils.third_party.nni_new.compression.pytorch.speedup import ModelSpeedup +import os + +from .base import Abs, KTakesAll, Layer_WrappedWithFBS, ElasticDNNUtil + + +class Conv2d_WrappedWithFBS(Layer_WrappedWithFBS): + def __init__(self, raw_conv2d: nn.Conv2d, raw_bn: nn.BatchNorm2d, r): + super(Conv2d_WrappedWithFBS, self).__init__() + + self.fbs = nn.Sequential( + Abs(), + nn.AdaptiveAvgPool2d(1), + nn.Flatten(), + nn.Linear(raw_conv2d.in_channels, raw_conv2d.out_channels // r), + nn.ReLU(), + nn.Linear(raw_conv2d.out_channels // r, raw_conv2d.out_channels), + nn.ReLU() + ) + + self.raw_conv2d = raw_conv2d + self.raw_bn = raw_bn # remember clear the original BNs in the network + + nn.init.constant_(self.fbs[5].bias, 1.) + nn.init.kaiming_normal_(self.fbs[5].weight) + + def forward(self, x): + raw_x = self.raw_bn(self.raw_conv2d(x)) + + if self.use_cached_channel_attention and self.cached_channel_attention is not None: + channel_attention = self.cached_channel_attention + else: + self.cached_raw_channel_attention = self.fbs(x) + self.cached_channel_attention = self.k_takes_all(self.cached_raw_channel_attention) + + channel_attention = self.cached_channel_attention + + return raw_x * channel_attention.unsqueeze(2).unsqueeze(3) + + +class StaticFBS(nn.Module): + def __init__(self, channel_attention: torch.Tensor): + super(StaticFBS, self).__init__() + assert channel_attention.dim() == 1 + self.channel_attention = nn.Parameter(channel_attention.unsqueeze(0).unsqueeze(2).unsqueeze(3), requires_grad=False) + + def forward(self, x): + return x * self.channel_attention + + def __str__(self) -> str: + return f'StaticFBS({len(self.channel_attention.size(1))})' + + +class ElasticCNNUtil(ElasticDNNUtil): + def convert_raw_dnn_to_master_dnn(self, raw_dnn: nn.Module, r: float, ignore_layers=[]): + model = deepcopy(raw_dnn) + + # clear original BNs + num_original_bns = 0 + last_conv_name = None + conv_bn_map = {} + for name, module in model.named_modules(): + if isinstance(module, nn.Conv2d): + last_conv_name = name + if isinstance(module, nn.BatchNorm2d) and (ignore_layers is not None and last_conv_name not in ignore_layers): + num_original_bns += 1 + conv_bn_map[last_conv_name] = name + + num_conv = 0 + for name, module in model.named_modules(): + if isinstance(module, nn.Conv2d) and (ignore_layers is not None and name not in ignore_layers): + set_module(model, name, Conv2d_WrappedWithFBS(module, get_module(model, conv_bn_map[name]), r)) + num_conv += 1 + + assert num_conv == num_original_bns + + for bn_layer in conv_bn_map.values(): + set_module(model, bn_layer, nn.Identity()) + + return model + + def select_most_rep_sample(self, master_dnn: nn.Module, samples: torch.Tensor): + return samples[0].unsqueeze(0) + + def extract_surrogate_dnn_via_samples(self, master_dnn: nn.Module, samples: torch.Tensor): + sample = self.select_most_rep_sample(master_dnn, samples) + assert sample.dim() == 4 and sample.size(0) == 1 + + master_dnn.eval() + with torch.no_grad(): + master_dnn_output = master_dnn(sample) + + pruning_info = {} + pruning_masks = {} + + for layer_name, layer in master_dnn.named_modules(): + if not isinstance(layer, Conv2d_WrappedWithFBS): + continue + + cur_pruning_mask = {'weight': torch.zeros_like(layer.raw_conv2d.weight.data)} + if layer.raw_conv2d.bias is not None: + cur_pruning_mask['bias'] = torch.zeros_like(layer.raw_conv2d.bias.data) + + w = get_module(master_dnn, layer_name).cached_channel_attention.squeeze(0) + unpruned_filters_index = w.nonzero(as_tuple=True)[0] + pruning_info[layer_name] = w + + cur_pruning_mask['weight'][unpruned_filters_index, ...] = 1. + if layer.raw_conv2d.bias is not None: + cur_pruning_mask['bias'][unpruned_filters_index, ...] = 1. + pruning_masks[layer_name + '.0'] = cur_pruning_mask + + surrogate_dnn = deepcopy(master_dnn) + for name, layer in surrogate_dnn.named_modules(): + if not isinstance(layer, Conv2d_WrappedWithFBS): + continue + set_module(surrogate_dnn, name, nn.Sequential(layer.raw_conv2d, layer.raw_bn, nn.Identity())) + + # fixed_pruning_masks = fix_mask_conflict(pruning_masks, fbs_model, sample.size(), None, True, True, True) + tmp_mask_path = f'tmp_mask_{get_cur_time_str()}_{os.getpid()}.pth' + torch.save(pruning_masks, tmp_mask_path) + surrogate_dnn.eval() + model_speedup = ModelSpeedup(surrogate_dnn, sample, tmp_mask_path, sample.device) + model_speedup.speedup_model() + os.remove(tmp_mask_path) + + # add feature boosting module + for layer_name, feature_boosting_w in pruning_info.items(): + feature_boosting_w = feature_boosting_w[feature_boosting_w.nonzero(as_tuple=True)[0]] + set_module(surrogate_dnn, layer_name + '.2', StaticFBS(feature_boosting_w)) + + surrogate_dnn.eval() + with torch.no_grad(): + surrogate_dnn_output = surrogate_dnn(sample) + output_diff = ((surrogate_dnn_output - master_dnn_output) ** 2).sum() + assert output_diff < 1e-4, output_diff + logger.info(f'output diff of master and surrogate DNN: {output_diff}') + + return surrogate_dnn + \ No newline at end of file diff --git a/methods/elasticdnn/model/test.py b/methods/elasticdnn/model/test.py new file mode 100644 index 0000000000000000000000000000000000000000..be961a0200479a9c98a2776dd678d87ebdb47295 --- /dev/null +++ b/methods/elasticdnn/model/test.py @@ -0,0 +1,138 @@ +import torch +from torch import nn + +from methods.elasticdnn.model.base import ElasticDNNUtil + + +def test(raw_dnn: nn.Module, ignore_layers, elastic_dnn_util: ElasticDNNUtil, input_sample: torch.Tensor, sparsity): + + # raw_dnn.eval() + # with torch.no_grad(): + # raw_dnn(input_sample) + + master_dnn = elastic_dnn_util.convert_raw_dnn_to_master_dnn_with_perf_test(raw_dnn, 16, ignore_layers) + # print(master_dnn) + # exit() + + elastic_dnn_util.set_master_dnn_sparsity(master_dnn, sparsity) + + # master_dnn.eval() + # with torch.no_grad(): + # master_dnn(input_sample) + + surrogate_dnn = elastic_dnn_util.extract_surrogate_dnn_via_samples_with_perf_test(master_dnn, input_sample) + + +if __name__ == '__main__': + from utils.dl.common.env import set_random_seed + set_random_seed(1) + + # from torchvision.models import resnet50 + # from methods.elasticdnn.model.cnn import ElasticCNNUtil + # raw_cnn = resnet50() + # prunable_layers = [] + # for i in range(1, 5): + # for j in range([3, 4, 6, 3][i - 1]): + # prunable_layers += [f'layer{i}.{j}.conv1', f'layer{i}.{j}.conv2'] + # ignore_layers = [layer for layer, m in raw_cnn.named_modules() if isinstance(m, nn.Conv2d) and layer not in prunable_layers] + # test(raw_cnn, ignore_layers, ElasticCNNUtil(), torch.rand(1, 3, 224, 224)) + ignore_layers = [] + from methods.elasticdnn.model.vit import ElasticViTUtil + # raw_vit = torch.load('tmp-master-dnn.pt') + raw_vit = torch.load('') + test(raw_vit, ignore_layers, ElasticViTUtil(), torch.rand(16, 3, 224, 224).cuda(), 0.9) + exit() + + + from dnns.vit import vit_b_16 + # from methods.elasticdnn.model.vit_new import ElasticViTUtil + from methods.elasticdnn.model.vit import ElasticViTUtil + # raw_vit = vit_b_16() + + for s in [0.8, 0.9, 0.95]: + raw_vit = vit_b_16().cuda() + + ignore_layers = [] + test(raw_vit, ignore_layers, ElasticViTUtil(), torch.rand(16, 3, 224, 224).cuda(), s) + + # for s in [0, 0.2, 0.4, 0.6, 0.8]: + # pretrained_md_models_dict_path = 'experiments/elasticdnn/vit_b_16/offline/fm_to_md/results/20230518/999999-164524-wo_FBS_trial_dsnet_lr/models/md_best.pt' + # raw_vit = torch.load(pretrained_md_models_dict_path)['main'].cuda() + + # ignore_layers = [] + # test(raw_vit, ignore_layers, ElasticViTUtil(), torch.rand(16, 3, 224, 224).cuda(), s) + # exit() + + + # weight = torch.rand((10, 5)) + # bias = torch.rand(10) + # x = torch.rand((1, 3, 5)) + + # t = torch.randperm(5) + # pruned, unpruned = t[0: 3], t[3: ] + + # mask = torch.ones_like(x) + # mask[:, :, pruned] = 0 + + # print(x, x * mask, (x * mask).sum((0, 1))) + + # import torch.nn.functional as F + # o1 = F.linear(x * mask, weight, bias) + # # print(o1) + + + # o2 = F.linear(x[:, :, unpruned], weight[:, unpruned], bias) + # # print(o2) + + # print(o1.size(), o2.size(), ((o1 - o2) ** 2).sum()) + + + + + # weight = torch.rand((130, 5)) + # bias = torch.rand(130) + # x = torch.rand((1, 3, 5)) + + # t = torch.randperm(5) + # pruned, unpruned = t[0: 3], t[3: ] + + # mask = torch.ones_like(x) + # mask[:, :, pruned] = 0 + + # print(x, x * mask, (x * mask).sum((0, 1))) + + # import torch.nn.functional as F + # o1 = F.linear(x * mask, weight, bias) + # # print(o1) + + + # o2 = F.linear(x[:, :, unpruned], weight[:, unpruned], bias) + # # print(o2) + + # print(o1.size(), o2.size(), ((o1 - o2) ** 2).sum()) + + + + + + # weight = torch.rand((1768, 768)) + # bias = torch.rand(1768) + # x = torch.rand([1, 197, 768]) + + # t = torch.randperm(768) + # unpruned, pruned = t[0: 144], t[144: ] + # unpruned = unpruned.sort()[0] + # pruned = pruned.sort()[0] + + # mask = torch.ones_like(x) + # mask[:, :, pruned] = 0 + + # print(x.sum((0, 1)).size(), (x * mask).sum((0, 1))[0: 10], x[:, :, unpruned].sum((0, 1))[0: 10]) + + # import torch.nn.functional as F + # o1 = F.linear(x * mask, weight, bias) + # o2 = F.linear(x[:, :, unpruned], weight[:, unpruned], bias) + # print(o1.sum((0, 1))[0: 10], o2.sum((0, 1))[0: 10], o1.size(), o2.size(), ((o1 - o2).abs()).sum(), ((o1 - o2) ** 2).sum()) + # unpruned_indexes = torch.randperm(5)[0: 2] + # o2 = F.linear(x[:, unpruned_indexes], weight[:, unpruned_indexes]) + # print(o2) \ No newline at end of file diff --git a/methods/elasticdnn/model/vilt.py b/methods/elasticdnn/model/vilt.py new file mode 100644 index 0000000000000000000000000000000000000000..f1facec6488bd3be8cabdd10c5bac9290d272167 --- /dev/null +++ b/methods/elasticdnn/model/vilt.py @@ -0,0 +1,359 @@ +from copy import deepcopy +from typing import Optional, Union +import torch +from torch import nn +from einops import rearrange, repeat +from einops.layers.torch import Rearrange +import tqdm + +from utils.dl.common.model import get_model_device, get_model_size, set_module, get_module +from .base import Abs, KTakesAll, ElasticDNNUtil, Layer_WrappedWithFBS +from utils.common.log import logger + + +class SqueezeLast(nn.Module): + def __init__(self): + super(SqueezeLast, self).__init__() + + def forward(self, x): + return x.squeeze(-1) + + +class ProjConv_WrappedWithFBS(Layer_WrappedWithFBS): + def __init__(self, proj: nn.Conv2d, r): + super(ProjConv_WrappedWithFBS, self).__init__() + + self.proj = proj + + # for conv: (B, C_in, H, W) -> (B, C_in) -> (B, C_out) + # for mlp in ViT: (B, #patches, D: dim of patches embedding) -> (B, D) -> (B, C_out) + self.fbs = nn.Sequential( + Abs(), + nn.AdaptiveAvgPool1d(1), + SqueezeLast(), + nn.Linear(proj.in_channels, proj.out_channels // r), + nn.ReLU(), + nn.Linear(proj.out_channels // r, proj.out_channels), + nn.ReLU() + ) + + nn.init.constant_(self.fbs[6].bias, 1.) + nn.init.kaiming_normal_(self.fbs[6].weight) + + + def forward(self, x): + if self.use_cached_channel_attention and self.cached_channel_attention is not None: + channel_attention = self.cached_channel_attention + else: + self.cached_raw_channel_attention = self.fbs(x) + self.cached_channel_attention = self.k_takes_all(self.cached_raw_channel_attention) + + channel_attention = self.cached_channel_attention + + raw_res = self.proj(x) + + return channel_attention.unsqueeze(1) * raw_res # TODO: + + +class Linear_WrappedWithFBS(Layer_WrappedWithFBS): + def __init__(self, linear: nn.Linear, r): + super(Linear_WrappedWithFBS, self).__init__() + + self.linear = linear + + # for conv: (B, C_in, H, W) -> (B, C_in) -> (B, C_out) + # for mlp in ViT: (B, #patches, D: dim of patches embedding) -> (B, D) -> (B, C_out) + self.fbs = nn.Sequential( + Rearrange('b n d -> b d n'), + Abs(), + nn.AdaptiveAvgPool1d(1), + SqueezeLast(), + nn.Linear(linear.in_features, linear.out_features // r), + nn.ReLU(), + nn.Linear(linear.out_features // r, linear.out_features), + nn.ReLU() + ) + + nn.init.constant_(self.fbs[6].bias, 1.) + nn.init.kaiming_normal_(self.fbs[6].weight) + + + def forward(self, x): + if self.use_cached_channel_attention and self.cached_channel_attention is not None: + channel_attention = self.cached_channel_attention + else: + self.cached_raw_channel_attention = self.fbs(x) + self.cached_channel_attention = self.k_takes_all(self.cached_raw_channel_attention) + + channel_attention = self.cached_channel_attention + + raw_res = self.linear(x) + + return channel_attention.unsqueeze(1) * raw_res + + +class ToQKV_WrappedWithFBS(Layer_WrappedWithFBS): + """ + This regards to_q/to_k/to_v as a whole (in fact it consists of multiple heads) and prunes it. + It seems different channels of different heads are pruned according to the input. + This is different from "removing some head" or "removing the same channels in each head". + """ + def __init__(self, to_qkv: nn.Linear, r): + super(ToQKV_WrappedWithFBS, self).__init__() + + # self.to_qkv = to_qkv + + self.to_qk = nn.Linear(to_qkv.in_features, to_qkv.out_features // 3 * 2, bias=to_qkv.bias is not None) + self.to_v = nn.Linear(to_qkv.in_features, to_qkv.out_features // 3, bias=to_qkv.bias is not None) + self.to_qk.weight.data.copy_(to_qkv.weight.data[0: to_qkv.out_features // 3 * 2]) + if to_qkv.bias is not None: + self.to_qk.bias.data.copy_(to_qkv.bias.data[0: to_qkv.out_features // 3 * 2]) + self.to_v.weight.data.copy_(to_qkv.weight.data[to_qkv.out_features // 3 * 2: ]) + if to_qkv.bias is not None: + self.to_v.bias.data.copy_(to_qkv.bias.data[to_qkv.out_features // 3 * 2: ]) + + self.fbs = nn.Sequential( + Rearrange('b n d -> b d n'), + Abs(), + nn.AdaptiveAvgPool1d(1), + SqueezeLast(), + nn.Linear(to_qkv.in_features, to_qkv.out_features // 3 // r), + nn.ReLU(), + # nn.Linear(to_qkv.out_features // 3 // r, to_qkv.out_features // 3), + nn.Linear(to_qkv.out_features // 3 // r, self.to_v.out_features), + nn.ReLU() + ) + + nn.init.constant_(self.fbs[6].bias, 1.) + nn.init.kaiming_normal_(self.fbs[6].weight) + + def forward(self, x): + if self.use_cached_channel_attention and self.cached_channel_attention is not None: + channel_attention = self.cached_channel_attention + else: + self.cached_raw_channel_attention = self.fbs(x) + + # print() + # for attn in self.cached_raw_channel_attention.chunk(3, dim=1)[0: 1]: + # print(self.cached_raw_channel_attention.size(), attn.size()) + # print(self.k_takes_all.k) + # print(attn[0].nonzero(as_tuple=True)[0].size(), attn[0]) + + self.cached_channel_attention = self.k_takes_all(self.cached_raw_channel_attention) + + + # for attn in self.cached_channel_attention.chunk(3, dim=1)[0: 1]: + # print(self.cached_channel_attention.size(), attn.size()) + # print(self.k_takes_all.k) + # print(attn[0].nonzero(as_tuple=True)[0].size(), attn[0]) + # print() + + channel_attention = self.cached_channel_attention + + qk = self.to_qk(x) + v = channel_attention.unsqueeze(1) * self.to_v(x) + return torch.cat([qk, v], dim=-1) + + # qkv = raw_res.chunk(3, dim = -1) + + # raw_v = qkv[2] + # print('raw_k, raw_v', qkv[0].sum((0, 1))[0: 10], qkv[0].sum((0, 1)).nonzero(as_tuple=True)[0].size(), + # qkv[1].sum((0, 1))[0: 10], qkv[1].sum((0, 1)).nonzero(as_tuple=True)[0].size(),) + # print('raw_v', raw_v.size(), raw_v.sum((0, 1))[0: 10], raw_v.sum((0, 1)).nonzero(as_tuple=True)[0].size()) + + # qkv_attn = channel_attention.chunk(3, dim=-1) + # print('attn', [attn[0][0: 10] for attn in qkv_attn]) + # print(channel_attention.unsqueeze(1).size(), raw_res.size()) + # print('fbs', channel_attention.size(), raw_res.size()) + # return channel_attention.unsqueeze(1) * raw_res + + +class StaticFBS(nn.Module): + def __init__(self, static_channel_attention): + super(StaticFBS, self).__init__() + assert static_channel_attention.dim() == 2 and static_channel_attention.size(0) == 1 + self.static_channel_attention = nn.Parameter(static_channel_attention, requires_grad=False) # (1, dim) + + def forward(self, x): + # print('staticfbs', x, self.static_channel_attention.unsqueeze(1)) + return x * self.static_channel_attention.unsqueeze(1) + + +class ElasticViltUtil(ElasticDNNUtil): + def convert_raw_dnn_to_master_dnn(self, raw_dnn: nn.Module, r: float, ignore_layers=[]): + assert len(ignore_layers) == 0, 'not supported yet' + + raw_vit = deepcopy(raw_dnn) + + # set_module(module, 'patch_embed.proj', ProjConv_WrappedWithFBS(module.patch_embed.proj, r)) + + for name, module in raw_vit.named_modules(): + # if name.endswith('attn'): + # set_module(module, 'qkv', ToQKV_WrappedWithFBS(module.qkv, r)) + if name.endswith('intermediate'): + set_module(module, 'dense', Linear_WrappedWithFBS(module.dense, r)) + + return raw_vit + + def set_master_dnn_sparsity(self, master_dnn: nn.Module, sparsity: float): + # for name, module in master_dnn.named_modules(): + # if not name.endswith('attn'): + # continue + + # q_features = module.qkv.to_qk.out_features // 2 + + # if (q_features - int(q_features * sparsity)) % module.num_heads != 0: + # # tune sparsity to ensure #unpruned channel % num_heads == 0 + # # so that the pruning seems to reduce the dim_head of each head + # tuned_sparsity = 1. - int((q_features - int(q_features * sparsity)) / module.num_heads) * module.num_heads / q_features + # logger.debug(f'tune sparsity from {sparsity:.2f} to {tuned_sparsity}') + # sparsity = tuned_sparsity + # break + + return super().set_master_dnn_sparsity(master_dnn, sparsity) + + def select_most_rep_sample(self, master_dnn: nn.Module, samples: torch.Tensor): + # print(samples) + # return samples[0].unsqueeze(0) + res = {k: v[0: 1] for k, v in samples.items()} + return res + + def extract_surrogate_dnn_via_samples(self, master_dnn: nn.Module, samples: torch.Tensor, return_detail=False): + sample = self.select_most_rep_sample(master_dnn, samples) + # assert sample.dim() == 4 and sample.size(0) == 1 + + # print('before') + master_dnn.eval() + self.clear_cached_channel_attention_in_master_dnn(master_dnn) + with torch.no_grad(): + master_dnn_output = master_dnn(**sample) + + # print('after') + + boosted_vit = deepcopy(master_dnn) + + def get_unpruned_indexes_from_channel_attn(channel_attn: torch.Tensor, k): + assert channel_attn.size(0) == 1, 'use A representative sample to generate channel attentions' + + # print('attn_in_unpruned', channel_attn[0][0: 10]) + + res = channel_attn[0].nonzero(as_tuple=True)[0] # should be one-dim + + # res = channel_attn[0].argsort(descending=True)[0: -int(channel_attn.size(1) * k)].sort()[0] + + # g = channel_attn + # k = g.size(1) - int(g.size(1) * k) + # res = g.topk(k, 1)[1][0].sort()[0] + + return res + + unpruned_indexes_of_layers = {} + + # for attn, ff in boosted_vit.transformer.layers: + # for block_i, block in enumerate(boosted_vit.blocks): + for block_i, block in enumerate(boosted_vit.vilt.encoder.layer): + # attn = block.attn + # ff = block.mlp + + ff_0 = get_module(block, f'intermediate.dense') + # ff_0_unpruned_indexes = get_unpruned_indexes_from_channel_attn(ff_0.cached_channel_attention, k) + ff_0_pruned_indexes = ff_0.k_takes_all.cached_i[0].sort()[0] + ff_0_unpruned_indexes = torch.LongTensor([ii for ii in range(ff_0.cached_channel_attention.size(1)) if ii not in ff_0_pruned_indexes]) + new_ff_0 = nn.Linear(ff_0.linear.in_features, ff_0_unpruned_indexes.size(0), ff_0.linear.bias is not None) + new_ff_0.weight.data.copy_(ff_0.linear.weight.data[ff_0_unpruned_indexes]) + if ff_0.linear.bias is not None: + new_ff_0.bias.data.copy_(ff_0.linear.bias.data[ff_0_unpruned_indexes]) + set_module(block, 'intermediate.dense', nn.Sequential(new_ff_0, StaticFBS(ff_0.cached_channel_attention[:, ff_0_unpruned_indexes]))) + + ff_1 = get_module(block, f'output.dense') + new_ff_1 = nn.Linear(ff_0_unpruned_indexes.size(0), ff_1.out_features, ff_1.bias is not None) + new_ff_1.weight.data.copy_(ff_1.weight.data[:, ff_0_unpruned_indexes]) + if ff_1.bias is not None: + new_ff_1.bias.data.copy_(ff_1.bias.data) + set_module(block, 'output.dense', new_ff_1) + + unpruned_indexes_of_layers[f'vilt.encoder.layer.{block_i}.intermediate.dense.0.weight'] = ff_0_unpruned_indexes + + surrogate_dnn = boosted_vit + surrogate_dnn.eval() + surrogate_dnn = surrogate_dnn.to(get_model_device(master_dnn)) + # logger.debug(surrogate_dnn) + with torch.no_grad(): + surrogate_dnn_output = surrogate_dnn(**sample) + + output_diff = ((surrogate_dnn_output.logits - master_dnn_output.logits) ** 2).sum() + # assert output_diff < 1e-4, output_diff + logger.info(f'output diff of master and surrogate DNN: {output_diff}') + # logger.debug(f'example output of master/surrogate: {master_dnn_output.sum(0)[0: 10]}, {surrogate_dnn_output.sum(0)[0: 10]}') + # logger.info(f'\nonly prune mlp!!!!\n') + # logger.info(f'\nonly prune mlp!!!!\n') + + if return_detail: + return boosted_vit, unpruned_indexes_of_layers + + return boosted_vit + + def extract_surrogate_dnn_via_samples_with_perf_test(self, master_dnn: nn.Module, samples: torch.Tensor, return_detail=False): + master_dnn_size = get_model_size(master_dnn, True) + master_dnn_latency = self._get_model_latency(master_dnn, samples, 50, + get_model_device(master_dnn), 50, False) + + res = self.extract_surrogate_dnn_via_samples(master_dnn, samples, return_detail) + if not return_detail: + surrogate_dnn = res + else: + surrogate_dnn, unpruned_indexes_of_layers = res + surrogate_dnn_size = get_model_size(surrogate_dnn, True) + surrogate_dnn_latency = self._get_model_latency(master_dnn, samples, 50, + get_model_device(master_dnn), 50, False) + + logger.info(f'master DNN ({master_dnn_size:.3f}MB, {master_dnn_latency:.4f}s/sample) -> ' + f'surrogate DNN ({surrogate_dnn_size:.3f}MB, {surrogate_dnn_latency:.4f}s/sample)\n' + f'(model size: ↓ {(master_dnn_size / surrogate_dnn_size):.2f}x, ' + f'latency: ↓ {(master_dnn_latency / surrogate_dnn_latency):.2f}x)') + + return res + + def _get_model_latency(self, model: torch.nn.Module, model_input_size, sample_num: int, + device: str, warmup_sample_num: int, return_detail=False): + import time + + if isinstance(model_input_size, tuple): + dummy_input = torch.rand(model_input_size).to(device) + else: + dummy_input = model_input_size + + model = model.to(device) + model.eval() + + # warm up + with torch.no_grad(): + for _ in range(warmup_sample_num): + model(**dummy_input) + + infer_time_list = [] + + if device == 'cuda' or 'cuda' in str(device): + with torch.no_grad(): + for _ in range(sample_num): + s, e = torch.cuda.Event(enable_timing=True), torch.cuda.Event(enable_timing=True) + s.record() + model(**dummy_input) + e.record() + torch.cuda.synchronize() + cur_model_infer_time = s.elapsed_time(e) / 1000. + infer_time_list += [cur_model_infer_time] + + else: + with torch.no_grad(): + for _ in range(sample_num): + start = time.time() + model(**dummy_input) + cur_model_infer_time = time.time() - start + infer_time_list += [cur_model_infer_time] + + avg_infer_time = sum(infer_time_list) / sample_num + + if return_detail: + return avg_infer_time, infer_time_list + return avg_infer_time \ No newline at end of file diff --git a/methods/elasticdnn/model/vit.py b/methods/elasticdnn/model/vit.py new file mode 100644 index 0000000000000000000000000000000000000000..98392df55d15335306a2dbae4fa148b3151e1c02 --- /dev/null +++ b/methods/elasticdnn/model/vit.py @@ -0,0 +1,343 @@ +from copy import deepcopy +from typing import Optional, Union +import torch +from torch import nn +from einops import rearrange, repeat +from einops.layers.torch import Rearrange +import tqdm + +from utils.dl.common.model import get_model_device, get_model_size, set_module +from .base import Abs, KTakesAll, ElasticDNNUtil, Layer_WrappedWithFBS +from utils.common.log import logger + + +class SqueezeLast(nn.Module): + def __init__(self): + super(SqueezeLast, self).__init__() + + def forward(self, x): + return x.squeeze(-1) + + +class ProjConv_WrappedWithFBS(Layer_WrappedWithFBS): + def __init__(self, proj: nn.Conv2d, r): + super(ProjConv_WrappedWithFBS, self).__init__() + + self.proj = proj + + # for conv: (B, C_in, H, W) -> (B, C_in) -> (B, C_out) + # for mlp in ViT: (B, #patches, D: dim of patches embedding) -> (B, D) -> (B, C_out) + self.fbs = nn.Sequential( + Abs(), + nn.AdaptiveAvgPool1d(1), + SqueezeLast(), + nn.Linear(proj.in_channels, proj.out_channels // r), + nn.ReLU(), + nn.Linear(proj.out_channels // r, proj.out_channels), + nn.ReLU() + ) + + nn.init.constant_(self.fbs[6].bias, 1.) + nn.init.kaiming_normal_(self.fbs[6].weight) + + + def forward(self, x): + if self.use_cached_channel_attention and self.cached_channel_attention is not None: + channel_attention = self.cached_channel_attention + else: + self.cached_raw_channel_attention = self.fbs(x) + self.cached_channel_attention = self.k_takes_all(self.cached_raw_channel_attention) + + channel_attention = self.cached_channel_attention + + raw_res = self.proj(x) + + return channel_attention.unsqueeze(1) * raw_res # TODO: + + +class Linear_WrappedWithFBS(Layer_WrappedWithFBS): + def __init__(self, linear: nn.Linear, r): + super(Linear_WrappedWithFBS, self).__init__() + + self.linear = linear + + # for conv: (B, C_in, H, W) -> (B, C_in) -> (B, C_out) + # for mlp in ViT: (B, #patches, D: dim of patches embedding) -> (B, D) -> (B, C_out) + self.fbs = nn.Sequential( + Rearrange('b n d -> b d n'), + Abs(), + nn.AdaptiveAvgPool1d(1), + SqueezeLast(), + nn.Linear(linear.in_features, linear.out_features // r), + nn.ReLU(), + nn.Linear(linear.out_features // r, linear.out_features), + nn.ReLU() + ) + + nn.init.constant_(self.fbs[6].bias, 1.) + nn.init.kaiming_normal_(self.fbs[6].weight) + + + def forward(self, x): + if self.use_cached_channel_attention and self.cached_channel_attention is not None: + channel_attention = self.cached_channel_attention + else: + self.cached_raw_channel_attention = self.fbs(x) + self.cached_channel_attention = self.k_takes_all(self.cached_raw_channel_attention) + + channel_attention = self.cached_channel_attention + + raw_res = self.linear(x) + + return channel_attention.unsqueeze(1) * raw_res + + +class ToQKV_WrappedWithFBS(Layer_WrappedWithFBS): + """ + This regards to_q/to_k/to_v as a whole (in fact it consists of multiple heads) and prunes it. + It seems different channels of different heads are pruned according to the input. + This is different from "removing some head" or "removing the same channels in each head". + """ + def __init__(self, to_qkv: nn.Linear, r): + super(ToQKV_WrappedWithFBS, self).__init__() + + # self.to_qkv = to_qkv + + self.to_qk = nn.Linear(to_qkv.in_features, to_qkv.out_features // 3 * 2, bias=to_qkv.bias is not None) + self.to_v = nn.Linear(to_qkv.in_features, to_qkv.out_features // 3, bias=to_qkv.bias is not None) + self.to_qk.weight.data.copy_(to_qkv.weight.data[0: to_qkv.out_features // 3 * 2]) + if to_qkv.bias is not None: + self.to_qk.bias.data.copy_(to_qkv.bias.data[0: to_qkv.out_features // 3 * 2]) + self.to_v.weight.data.copy_(to_qkv.weight.data[to_qkv.out_features // 3 * 2: ]) + if to_qkv.bias is not None: + self.to_v.bias.data.copy_(to_qkv.bias.data[to_qkv.out_features // 3 * 2: ]) + + self.fbs = nn.Sequential( + Rearrange('b n d -> b d n'), + Abs(), + nn.AdaptiveAvgPool1d(1), + SqueezeLast(), + nn.Linear(to_qkv.in_features, to_qkv.out_features // 3 // r), + nn.ReLU(), + # nn.Linear(to_qkv.out_features // 3 // r, to_qkv.out_features // 3), + nn.Linear(to_qkv.out_features // 3 // r, self.to_v.out_features), + nn.ReLU() + ) + + nn.init.constant_(self.fbs[6].bias, 1.) + nn.init.kaiming_normal_(self.fbs[6].weight) + + def forward(self, x): + if self.use_cached_channel_attention and self.cached_channel_attention is not None: + channel_attention = self.cached_channel_attention + else: + self.cached_raw_channel_attention = self.fbs(x) + + # print() + # for attn in self.cached_raw_channel_attention.chunk(3, dim=1)[0: 1]: + # print(self.cached_raw_channel_attention.size(), attn.size()) + # print(self.k_takes_all.k) + # print(attn[0].nonzero(as_tuple=True)[0].size(), attn[0]) + + self.cached_channel_attention = self.k_takes_all(self.cached_raw_channel_attention) + + + # for attn in self.cached_channel_attention.chunk(3, dim=1)[0: 1]: + # print(self.cached_channel_attention.size(), attn.size()) + # print(self.k_takes_all.k) + # print(attn[0].nonzero(as_tuple=True)[0].size(), attn[0]) + # print() + + channel_attention = self.cached_channel_attention + + qk = self.to_qk(x) + v = channel_attention.unsqueeze(1) * self.to_v(x) + return torch.cat([qk, v], dim=-1) + + # qkv = raw_res.chunk(3, dim = -1) + + # raw_v = qkv[2] + # print('raw_k, raw_v', qkv[0].sum((0, 1))[0: 10], qkv[0].sum((0, 1)).nonzero(as_tuple=True)[0].size(), + # qkv[1].sum((0, 1))[0: 10], qkv[1].sum((0, 1)).nonzero(as_tuple=True)[0].size(),) + # print('raw_v', raw_v.size(), raw_v.sum((0, 1))[0: 10], raw_v.sum((0, 1)).nonzero(as_tuple=True)[0].size()) + + # qkv_attn = channel_attention.chunk(3, dim=-1) + # print('attn', [attn[0][0: 10] for attn in qkv_attn]) + # print(channel_attention.unsqueeze(1).size(), raw_res.size()) + # print('fbs', channel_attention.size(), raw_res.size()) + # return channel_attention.unsqueeze(1) * raw_res + + +class StaticFBS(nn.Module): + def __init__(self, static_channel_attention): + super(StaticFBS, self).__init__() + assert static_channel_attention.dim() == 2 and static_channel_attention.size(0) == 1 + self.static_channel_attention = nn.Parameter(static_channel_attention, requires_grad=False) # (1, dim) + + def forward(self, x): + # print('staticfbs', x, self.static_channel_attention.unsqueeze(1)) + return x * self.static_channel_attention.unsqueeze(1) + + +class ElasticViTUtil(ElasticDNNUtil): + def convert_raw_dnn_to_master_dnn(self, raw_dnn: nn.Module, r: float, ignore_layers=[]): + assert len(ignore_layers) == 0, 'not supported yet' + + raw_vit = deepcopy(raw_dnn) + + # set_module(module, 'patch_embed.proj', ProjConv_WrappedWithFBS(module.patch_embed.proj, r)) + + for name, module in raw_vit.named_modules(): + # if name.endswith('attn'): + # set_module(module, 'qkv', ToQKV_WrappedWithFBS(module.qkv, r)) + if name.endswith('mlp'): + set_module(module, 'fc1', Linear_WrappedWithFBS(module.fc1, r)) + + return raw_vit + + def set_master_dnn_sparsity(self, master_dnn: nn.Module, sparsity: float): + # for name, module in master_dnn.named_modules(): + # if not name.endswith('attn'): + # continue + + # q_features = module.qkv.to_qk.out_features // 2 + + # if (q_features - int(q_features * sparsity)) % module.num_heads != 0: + # # tune sparsity to ensure #unpruned channel % num_heads == 0 + # # so that the pruning seems to reduce the dim_head of each head + # tuned_sparsity = 1. - int((q_features - int(q_features * sparsity)) / module.num_heads) * module.num_heads / q_features + # logger.debug(f'tune sparsity from {sparsity:.2f} to {tuned_sparsity}') + # sparsity = tuned_sparsity + # break + + return super().set_master_dnn_sparsity(master_dnn, sparsity) + + def select_most_rep_sample(self, master_dnn: nn.Module, samples: torch.Tensor): + return samples[0].unsqueeze(0) + + def extract_surrogate_dnn_via_samples(self, master_dnn: nn.Module, samples: torch.Tensor, return_detail=False): + sample = self.select_most_rep_sample(master_dnn, samples) + assert sample.dim() == 4 and sample.size(0) == 1 + + # print('before') + master_dnn.eval() + self.clear_cached_channel_attention_in_master_dnn(master_dnn) + with torch.no_grad(): + master_dnn_output = master_dnn(sample) + + # print('after') + + boosted_vit = deepcopy(master_dnn) + + def get_unpruned_indexes_from_channel_attn(channel_attn: torch.Tensor, k): + assert channel_attn.size(0) == 1, 'use A representative sample to generate channel attentions' + + # print('attn_in_unpruned', channel_attn[0][0: 10]) + + res = channel_attn[0].nonzero(as_tuple=True)[0] # should be one-dim + + # res = channel_attn[0].argsort(descending=True)[0: -int(channel_attn.size(1) * k)].sort()[0] + + # g = channel_attn + # k = g.size(1) - int(g.size(1) * k) + # res = g.topk(k, 1)[1][0].sort()[0] + + return res + + unpruned_indexes_of_layers = {} + + # for attn, ff in boosted_vit.transformer.layers: + for block_i, block in enumerate(boosted_vit.blocks): + attn = block.attn + ff = block.mlp + + # prune to_qkv + # to_qkv = attn.qkv + # # cached_i = to_qkv.k_takes_all.cached_i + # k = to_qkv.k_takes_all.k + # # to_q_unpruned_indexes = get_unpruned_indexes_from_channel_attn( + # # to_qkv.cached_channel_attention[:, 0: to_qkv.cached_channel_attention.size(1) // 3], k + # # ) + # # to_q_unpruned_indexes_w_offset = to_q_unpruned_indexes + # # to_k_unpruned_indexes = get_unpruned_indexes_from_channel_attn( + # # to_qkv.cached_channel_attention[:, to_qkv.cached_channel_attention.size(1) // 3: to_qkv.cached_channel_attention.size(1) // 3 * 2], k + # # ) + # # to_k_unpruned_indexes_w_offset = to_k_unpruned_indexes + to_qkv.cached_channel_attention.size(1) // 3 + # # to_v_unpruned_indexes = get_unpruned_indexes_from_channel_attn( + # # to_qkv.cached_channel_attention, k + # # ) + # to_v_pruned_indexes = to_qkv.k_takes_all.cached_i[0].sort()[0] + # # print(to_v_pruned_indexes.size(), to_qkv.cached_channel_attention.size()) + # to_v_unpruned_indexes = torch.LongTensor([ii for ii in range(to_qkv.cached_channel_attention.size(1)) if ii not in to_v_pruned_indexes]) + # # print(to_v_unpruned_indexes.size()) + # # exit() + # # to_q_unpruned_indexes = to_qkv.k_takes_all.cached_i[0] + # # to_q_unpruned_indexes_w_offset = to_q_unpruned_indexes + # # to_k_unpruned_indexes = to_qkv.k_takes_all.cached_i[1] + # # to_k_unpruned_indexes_w_offset = to_k_unpruned_indexes + to_qkv.cached_channel_attention.size(1) // 3 + # # to_v_unpruned_indexes = to_qkv.k_takes_all.cached_i[2] + # # to_v_unpruned_indexes_w_offset = to_v_unpruned_indexes + to_qkv.cached_channel_attention.size(1) // 3 * 2 + # # assert to_q_unpruned_indexes_w_offset.size(0) == to_k_unpruned_indexes_w_offset.size(0) == to_v_unpruned_indexes_w_offset.size(0), \ + # # f'{to_q_unpruned_indexes_w_offset.size(0)}, {to_k_unpruned_indexes_w_offset.size(0)}, {to_v_unpruned_indexes_w_offset.size(0)}' + # # print('unpruned indexes', to_q_unpruned_indexes[0: 10], to_k_unpruned_indexes[0: 10], to_v_unpruned_indexes[0: 10]) + # # exit() + # # print(to_q_unpruned_indexes_w_offset, to_k_unpruned_indexes_w_offset, to_v_unpruned_indexes_w_offset, to_v_unpruned_indexes) + # # to_qkv_unpruned_indexes = torch.cat([to_q_unpruned_indexes_w_offset, to_k_unpruned_indexes_w_offset, to_v_unpruned_indexes_w_offset]) + # new_to_qkv = nn.Linear(to_qkv.to_v.in_features, to_qkv.to_v.out_features * 2 + to_v_unpruned_indexes.size(0), to_qkv.to_v.bias is not None) + # new_to_qkv.weight.data.copy_(torch.cat([to_qkv.to_qk.weight.data, to_qkv.to_v.weight.data[to_v_unpruned_indexes]])) + # if to_qkv.to_qk.bias is not None: + # new_to_qkv.bias.data.copy_(torch.cat([to_qkv.to_qk.bias.data, to_qkv.to_v.bias.data[to_v_unpruned_indexes]])) + # set_module(attn, 'qkv', nn.Sequential(new_to_qkv, StaticFBS(torch.cat([ + # torch.ones_like(to_qkv.cached_channel_attention), + # torch.ones_like(to_qkv.cached_channel_attention), + # to_qkv.cached_channel_attention[:, to_v_unpruned_indexes] + # ], dim=1)))) + + # # prune to_out + # # print('to_v_unpruned_indexes', to_v_unpruned_indexes) + # to_out = attn.proj + # new_to_out = nn.Linear(to_v_unpruned_indexes.size(0), to_out.out_features, to_out.bias is not None) + # new_to_out.weight.data.copy_(to_out.weight.data[:, to_v_unpruned_indexes]) + # if to_out.bias is not None: + # new_to_out.bias.data.copy_(to_out.bias.data) + # # print('to_out copy bias') + # set_module(attn, 'proj', new_to_out) + + ff_0 = ff.fc1 + # ff_0_unpruned_indexes = get_unpruned_indexes_from_channel_attn(ff_0.cached_channel_attention, k) + ff_0_pruned_indexes = ff_0.k_takes_all.cached_i[0].sort()[0] + ff_0_unpruned_indexes = torch.LongTensor([ii for ii in range(ff_0.cached_channel_attention.size(1)) if ii not in ff_0_pruned_indexes]) + new_ff_0 = nn.Linear(ff_0.linear.in_features, ff_0_unpruned_indexes.size(0), ff_0.linear.bias is not None) + new_ff_0.weight.data.copy_(ff_0.linear.weight.data[ff_0_unpruned_indexes]) + if ff_0.linear.bias is not None: + new_ff_0.bias.data.copy_(ff_0.linear.bias.data[ff_0_unpruned_indexes]) + set_module(ff, 'fc1', nn.Sequential(new_ff_0, StaticFBS(ff_0.cached_channel_attention[:, ff_0_unpruned_indexes]))) + + ff_1 = ff.fc2 + new_ff_1 = nn.Linear(ff_0_unpruned_indexes.size(0), ff_1.out_features, ff_1.bias is not None) + new_ff_1.weight.data.copy_(ff_1.weight.data[:, ff_0_unpruned_indexes]) + if ff_1.bias is not None: + new_ff_1.bias.data.copy_(ff_1.bias.data) + set_module(ff, 'fc2', new_ff_1) + + unpruned_indexes_of_layers[f'blocks.{block_i}.mlp.fc1.0.weight'] = ff_0_unpruned_indexes + + surrogate_dnn = boosted_vit + surrogate_dnn.eval() + surrogate_dnn = surrogate_dnn.to(get_model_device(master_dnn)) + # logger.debug(surrogate_dnn) + with torch.no_grad(): + surrogate_dnn_output = surrogate_dnn(sample) + + output_diff = ((surrogate_dnn_output - master_dnn_output) ** 2).sum() + # assert output_diff < 1e-4, output_diff + logger.info(f'output diff of master and surrogate DNN: {output_diff}') + logger.debug(f'example output of master/surrogate: {master_dnn_output.sum(0)[0: 10]}, {surrogate_dnn_output.sum(0)[0: 10]}') + # logger.info(f'\nonly prune mlp!!!!\n') + # logger.info(f'\nonly prune mlp!!!!\n') + + if return_detail: + return boosted_vit, unpruned_indexes_of_layers + + return boosted_vit + \ No newline at end of file diff --git a/methods/elasticdnn/model/vit_new.py b/methods/elasticdnn/model/vit_new.py new file mode 100644 index 0000000000000000000000000000000000000000..3ff7bdfc3bcb9bb3e2c7c96b0385d821dfa6380e --- /dev/null +++ b/methods/elasticdnn/model/vit_new.py @@ -0,0 +1,341 @@ +from copy import deepcopy +from typing import Optional, Union +import torch +from torch import nn +from einops import rearrange, repeat +from einops.layers.torch import Rearrange +import tqdm + +from utils.dl.common.model import LayerActivation, get_model_device, get_model_size, set_module +from .base import Abs, KTakesAll, ElasticDNNUtil, Layer_WrappedWithFBS +from utils.common.log import logger + + +class SqueezeLast(nn.Module): + def __init__(self): + super(SqueezeLast, self).__init__() + + def forward(self, x): + return x.squeeze(-1) + + +class ProjConv_WrappedWithFBS(Layer_WrappedWithFBS): + def __init__(self, raw_conv2d: nn.Conv2d, r): + super(ProjConv_WrappedWithFBS, self).__init__() + + self.fbs = nn.Sequential( + Abs(), + nn.AdaptiveAvgPool2d(1), + nn.Flatten(), + nn.Linear(raw_conv2d.in_channels, raw_conv2d.out_channels // r), + nn.ReLU(), + nn.Linear(raw_conv2d.out_channels // r, raw_conv2d.out_channels), + nn.ReLU() + ) + + self.raw_conv2d = raw_conv2d + # self.raw_bn = raw_bn # remember clear the original BNs in the network + + nn.init.constant_(self.fbs[5].bias, 1.) + nn.init.kaiming_normal_(self.fbs[5].weight) + + def forward(self, x): + raw_x = self.raw_conv2d(x) + + if self.use_cached_channel_attention and self.cached_channel_attention is not None: + channel_attention = self.cached_channel_attention + else: + self.cached_raw_channel_attention = self.fbs(x) + self.cached_channel_attention = self.k_takes_all(self.cached_raw_channel_attention) + + channel_attention = self.cached_channel_attention + + return raw_x * channel_attention.unsqueeze(2).unsqueeze(3) + + +class Linear_WrappedWithFBS(Layer_WrappedWithFBS): + def __init__(self, linear: nn.Linear, r): + super(Linear_WrappedWithFBS, self).__init__() + + self.linear = linear + + # for conv: (B, C_in, H, W) -> (B, C_in) -> (B, C_out) + # for mlp in ViT: (B, #patches, D: dim of patches embedding) -> (B, D) -> (B, C_out) + self.fbs = nn.Sequential( + Rearrange('b n d -> b d n'), + Abs(), + nn.AdaptiveAvgPool1d(1), + SqueezeLast(), + nn.Linear(linear.in_features, linear.out_features // r), + nn.ReLU(), + nn.Linear(linear.out_features // r, linear.out_features), + nn.ReLU() + ) + + nn.init.constant_(self.fbs[6].bias, 1.) + nn.init.kaiming_normal_(self.fbs[6].weight) + + + def forward(self, x): + if self.use_cached_channel_attention and self.cached_channel_attention is not None: + channel_attention = self.cached_channel_attention + else: + self.cached_raw_channel_attention = self.fbs(x) + self.cached_channel_attention = self.k_takes_all(self.cached_raw_channel_attention) + + channel_attention = self.cached_channel_attention + + raw_res = self.linear(x) + + return channel_attention.unsqueeze(1) * raw_res + + +# class ToQKV_WrappedWithFBS(Layer_WrappedWithFBS): +# """ +# This regards to_q/to_k/to_v as a whole (in fact it consists of multiple heads) and prunes it. +# It seems different channels of different heads are pruned according to the input. +# This is different from "removing some head" or "removing the same channels in each head". +# """ +# def __init__(self, to_qkv: nn.Linear, r): +# super(ToQKV_WrappedWithFBS, self).__init__() + +# # self.to_qkv = to_qkv + +# self.to_qk = nn.Linear(to_qkv.in_features, to_qkv.out_features // 3 * 2, bias=to_qkv.bias is not None) +# self.to_v = nn.Linear(to_qkv.in_features, to_qkv.out_features // 3, bias=to_qkv.bias is not None) +# self.to_qk.weight.data.copy_(to_qkv.weight.data[0: to_qkv.out_features // 3 * 2]) +# if to_qkv.bias is not None: +# self.to_qk.bias.data.copy_(to_qkv.bias.data[0: to_qkv.out_features // 3 * 2]) +# self.to_v.weight.data.copy_(to_qkv.weight.data[to_qkv.out_features // 3 * 2: ]) +# if to_qkv.bias is not None: +# self.to_v.bias.data.copy_(to_qkv.bias.data[to_qkv.out_features // 3 * 2: ]) + +# self.fbs = nn.Sequential( +# Rearrange('b n d -> b d n'), +# Abs(), +# nn.AdaptiveAvgPool1d(1), +# SqueezeLast(), +# nn.Linear(to_qkv.in_features, to_qkv.out_features // 3 // r), +# nn.ReLU(), +# # nn.Linear(to_qkv.out_features // 3 // r, to_qkv.out_features // 3), +# nn.Linear(to_qkv.out_features // 3 // r, self.to_v.out_features), +# nn.ReLU() +# ) + +# nn.init.constant_(self.fbs[6].bias, 1.) +# nn.init.kaiming_normal_(self.fbs[6].weight) + +# def forward(self, x): +# if self.use_cached_channel_attention and self.cached_channel_attention is not None: +# channel_attention = self.cached_channel_attention +# else: +# self.cached_raw_channel_attention = self.fbs(x) + +# # print() +# # for attn in self.cached_raw_channel_attention.chunk(3, dim=1)[0: 1]: +# # print(self.cached_raw_channel_attention.size(), attn.size()) +# # print(self.k_takes_all.k) +# # print(attn[0].nonzero(as_tuple=True)[0].size(), attn[0]) + +# self.cached_channel_attention = self.k_takes_all(self.cached_raw_channel_attention) + + +# # for attn in self.cached_channel_attention.chunk(3, dim=1)[0: 1]: +# # print(self.cached_channel_attention.size(), attn.size()) +# # print(self.k_takes_all.k) +# # print(attn[0].nonzero(as_tuple=True)[0].size(), attn[0]) +# # print() + +# channel_attention = self.cached_channel_attention + +# qk = self.to_qk(x) +# v = channel_attention.unsqueeze(1) * self.to_v(x) +# return torch.cat([qk, v], dim=-1) + + # qkv = raw_res.chunk(3, dim = -1) + + # raw_v = qkv[2] + # print('raw_k, raw_v', qkv[0].sum((0, 1))[0: 10], qkv[0].sum((0, 1)).nonzero(as_tuple=True)[0].size(), + # qkv[1].sum((0, 1))[0: 10], qkv[1].sum((0, 1)).nonzero(as_tuple=True)[0].size(),) + # print('raw_v', raw_v.size(), raw_v.sum((0, 1))[0: 10], raw_v.sum((0, 1)).nonzero(as_tuple=True)[0].size()) + + # qkv_attn = channel_attention.chunk(3, dim=-1) + # print('attn', [attn[0][0: 10] for attn in qkv_attn]) + # print(channel_attention.unsqueeze(1).size(), raw_res.size()) + # print('fbs', channel_attention.size(), raw_res.size()) + # return channel_attention.unsqueeze(1) * raw_res + + +class LinearStaticFBS(nn.Module): + def __init__(self, static_channel_attention): + super(LinearStaticFBS, self).__init__() + assert static_channel_attention.dim() == 2 and static_channel_attention.size(0) == 1 + self.static_channel_attention = nn.Parameter(static_channel_attention, requires_grad=False) # (1, dim) + + def forward(self, x): + # print('staticfbs', x, self.static_channel_attention.unsqueeze(1)) + return x * self.static_channel_attention.unsqueeze(1) + +from .cnn import StaticFBS as ConvStaticFBS + + +class ElasticViTUtil(ElasticDNNUtil): + def convert_raw_dnn_to_master_dnn(self, raw_dnn: nn.Module, r: float, ignore_layers=[]): + assert len(ignore_layers) == 0, 'not supported yet' + + raw_vit = deepcopy(raw_dnn) + + set_module(raw_vit, 'patch_embed.proj', ProjConv_WrappedWithFBS(raw_vit.patch_embed.proj, r)) + + for name, module in raw_vit.named_modules(): + if name.endswith('mlp'): + set_module(module, 'fc1', Linear_WrappedWithFBS(module.fc1, r)) + + return raw_vit + + # def set_master_dnn_sparsity(self, master_dnn: nn.Module, sparsity: float): + # for name, module in master_dnn.named_modules(): + # if not name.endswith('attn'): + # continue + + # q_features = module.qkv.to_qk.out_features // 2 + + # if (q_features - int(q_features * sparsity)) % module.num_heads != 0: + # # tune sparsity to ensure #unpruned channel % num_heads == 0 + # # so that the pruning seems to reduce the dim_head of each head + # tuned_sparsity = 1. - int((q_features - int(q_features * sparsity)) / module.num_heads) * module.num_heads / q_features + # logger.debug(f'tune sparsity from {sparsity:.2f} to {tuned_sparsity}') + # sparsity = tuned_sparsity + # break + + # return super().set_master_dnn_sparsity(master_dnn, sparsity) + + def select_most_rep_sample(self, master_dnn: nn.Module, samples: torch.Tensor): + return samples[0].unsqueeze(0) + + def extract_surrogate_dnn_via_samples(self, master_dnn: nn.Module, samples: torch.Tensor): + sample = self.select_most_rep_sample(master_dnn, samples) + assert sample.dim() == 4 and sample.size(0) == 1 + + + print('WARN: for debug, modify cls_token and pos_embed') + master_dnn.pos_embed.data = torch.zeros_like(master_dnn.pos_embed.data) + + print('before') + master_dnn.eval() + self.clear_cached_channel_attention_in_master_dnn(master_dnn) + + # debug: add hooks + + hooks = { + 'blocks_input': LayerActivation(master_dnn.blocks, True, 'cuda') + } + + with torch.no_grad(): + master_dnn_output = master_dnn(sample) + + for k, v in hooks.items(): + print(f'{k}: {v.input.size()}') + + print('after') + + boosted_vit = master_dnn + + def get_unpruned_indexes_from_channel_attn(channel_attn: torch.Tensor, k): + assert channel_attn.size(0) == 1, 'use A representative sample to generate channel attentions' + res = channel_attn[0].nonzero(as_tuple=True)[0] + return res + + proj = boosted_vit.patch_embed.proj + proj_unpruned_indexes = get_unpruned_indexes_from_channel_attn( + proj.cached_channel_attention, proj.k_takes_all.k) + + # 1.1 prune proj itself + proj_conv = proj.raw_conv2d + new_proj = nn.Conv2d(proj_conv.in_channels, proj_unpruned_indexes.size(0), proj_conv.kernel_size, proj_conv.stride, proj_conv.padding, + proj_conv.dilation, proj_conv.groups, proj_conv.bias is not None, proj_conv.padding_mode, proj_conv.weight.device) + new_proj.weight.data.copy_(proj_conv.weight.data[proj_unpruned_indexes]) + if new_proj.bias is not None: + new_proj.bias.data.copy_(proj_conv.bias.data[proj_unpruned_indexes]) + set_module(boosted_vit.patch_embed, 'proj', nn.Sequential(new_proj, ConvStaticFBS(proj.cached_channel_attention[0][proj_unpruned_indexes]))) + + # print(boosted_vit.pos_embed.size(), boosted_vit.cls_token.size()) + boosted_vit.pos_embed.data = boosted_vit.pos_embed.data[:, :, proj_unpruned_indexes] + boosted_vit.cls_token.data = boosted_vit.cls_token.data[:, :, proj_unpruned_indexes] + + def reduce_linear_output(raw_linear: nn.Linear, layer_name, unpruned_indexes: torch.Tensor): + new_linear = nn.Linear(raw_linear.in_features, unpruned_indexes.size(0), raw_linear.bias is not None) + new_linear.weight.data.copy_(raw_linear.weight.data[unpruned_indexes]) + if raw_linear.bias is not None: + new_linear.bias.data.copy_(raw_linear.bias.data[unpruned_indexes]) + set_module(boosted_vit, layer_name, new_linear) + + def reduce_linear_input(raw_linear: nn.Linear, layer_name, unpruned_indexes: torch.Tensor): + new_linear = nn.Linear(unpruned_indexes.size(0), raw_linear.out_features, raw_linear.bias is not None) + new_linear.weight.data.copy_(raw_linear.weight.data[:, unpruned_indexes]) + if raw_linear.bias is not None: + new_linear.bias.data.copy_(raw_linear.bias.data) + set_module(boosted_vit, layer_name, new_linear) + + def reduce_norm(raw_norm: nn.LayerNorm, layer_name, unpruned_indexes: torch.Tensor): + new_norm = nn.LayerNorm(unpruned_indexes.size(0), raw_norm.eps, raw_norm.elementwise_affine) + new_norm.weight.data.copy_(raw_norm.weight.data[unpruned_indexes]) + new_norm.bias.data.copy_(raw_norm.bias.data[unpruned_indexes]) + set_module(boosted_vit, layer_name, new_norm) + + # 1.2 prune blocks.x.norm1/to_qkv/proj/fc1/fc2 + for block_i, block in enumerate(boosted_vit.blocks): + attn = block.attn + ff = block.mlp + + reduce_norm(block.norm1, f'blocks.{block_i}.norm1', proj_unpruned_indexes) + reduce_linear_input(attn.qkv, f'blocks.{block_i}.attn.qkv', proj_unpruned_indexes) + reduce_linear_output(attn.proj, f'blocks.{block_i}.attn.proj', proj_unpruned_indexes) + reduce_norm(block.norm2, f'blocks.{block_i}.norm2', proj_unpruned_indexes) + reduce_linear_input(ff.fc1.linear, f'blocks.{block_i}.mlp.fc1.linear', proj_unpruned_indexes) + reduce_linear_output(ff.fc2, f'blocks.{block_i}.mlp.fc2', proj_unpruned_indexes) + + # 1.3 prune norm, head + reduce_norm(boosted_vit.norm, f'norm', proj_unpruned_indexes) + reduce_linear_input(boosted_vit.head, f'head', proj_unpruned_indexes) + + # 2. prune blocks.x.fc1 + for block_i, block in enumerate(boosted_vit.blocks): + attn = block.attn + ff = block.mlp + + fc1 = ff.fc1 + fc1_unpruned_indexes = get_unpruned_indexes_from_channel_attn(fc1.cached_channel_attention, fc1.k_takes_all.k) + fc1_linear = fc1.linear + new_linear = nn.Linear(fc1_linear.in_features, fc1_unpruned_indexes.size(0), fc1_linear.bias is not None) + new_linear.weight.data.copy_(fc1_linear.weight.data[fc1_unpruned_indexes]) + if fc1_linear.bias is not None: + new_linear.bias.data.copy_(fc1_linear.bias.data[fc1_unpruned_indexes]) + set_module(boosted_vit, f'blocks.{block_i}.mlp.fc1', nn.Sequential(new_linear, LinearStaticFBS(fc1.cached_channel_attention[:, fc1_unpruned_indexes]))) + + reduce_linear_input(ff.fc2, f'blocks.{block_i}.mlp.fc2', fc1_unpruned_indexes) + + + + surrogate_dnn = boosted_vit + surrogate_dnn.eval() + surrogate_dnn = surrogate_dnn.to(get_model_device(master_dnn)) + print(surrogate_dnn) + + + hooks = { + 'blocks_input': LayerActivation(surrogate_dnn.blocks, True, 'cuda') + } + + with torch.no_grad(): + surrogate_dnn_output = surrogate_dnn(sample) + + for k, v in hooks.items(): + print(f'{k}: {v.input.size()}') + + output_diff = ((surrogate_dnn_output - master_dnn_output) ** 2).sum() + assert output_diff < 1e-4, output_diff + logger.info(f'output diff of master and surrogate DNN: {output_diff}') + + return boosted_vit + \ No newline at end of file diff --git a/methods/elasticdnn/pipeline/offline/fm_lora/__pycache__/base.cpython-38.pyc b/methods/elasticdnn/pipeline/offline/fm_lora/__pycache__/base.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..96c44c813b60205480ba90165c71c4ccd57a8660 Binary files /dev/null and b/methods/elasticdnn/pipeline/offline/fm_lora/__pycache__/base.cpython-38.pyc differ diff --git a/methods/elasticdnn/pipeline/offline/fm_lora/__pycache__/vit.cpython-38.pyc b/methods/elasticdnn/pipeline/offline/fm_lora/__pycache__/vit.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d87e54da59b3db2d1698d80350971a4ee6a37f55 Binary files /dev/null and b/methods/elasticdnn/pipeline/offline/fm_lora/__pycache__/vit.cpython-38.pyc differ diff --git a/methods/elasticdnn/pipeline/offline/fm_lora/base.py b/methods/elasticdnn/pipeline/offline/fm_lora/base.py new file mode 100644 index 0000000000000000000000000000000000000000..6dbbd9ebdfc784f8d195b19d982dfee2db5ea571 --- /dev/null +++ b/methods/elasticdnn/pipeline/offline/fm_lora/base.py @@ -0,0 +1,36 @@ +import torch +from torch import nn +from abc import ABC, abstractmethod + +from utils.dl.common.model import get_model_device, get_model_latency, get_model_size +from utils.common.log import logger + + +class LoRA(nn.Linear): + pass + + +class FMLoRA_Util(ABC): + @abstractmethod + def add_lora_ab_to_fm(self, fm: nn.Module, ab_r: int, samples: torch.Tensor): + """ + only applying LoRA to attention weights. + """ + pass + + def train_only_lora(self, fm: nn.Module): + res = [] + for n, m in fm.named_modules(): + if isinstance(m, LoRA): + for p in m.parameters(): + p.requires_grad = True + res += [p] + else: + for p in m.parameters(): + p.requires_grad = False + return res + + @abstractmethod + def absorb_lora_and_recover_net_structure(self, fm: nn.Module): + pass + \ No newline at end of file diff --git a/methods/elasticdnn/pipeline/offline/fm_lora/bert.py b/methods/elasticdnn/pipeline/offline/fm_lora/bert.py new file mode 100644 index 0000000000000000000000000000000000000000..a54d802afce5f2edeaf741d78a9c504232adcd8c --- /dev/null +++ b/methods/elasticdnn/pipeline/offline/fm_lora/bert.py @@ -0,0 +1,81 @@ +import torch +from torch import nn +from abc import ABC, abstractmethod + +from utils.dl.common.model import get_model_device, get_model_latency, get_model_size, set_module +from utils.common.log import logger +from .base import FMLoRA_Util, LoRA + + +class ToQKV_WrappedWithLoRA(nn.Module): + def __init__(self, fc: nn.Linear, ab_r: int): + super(ToQKV_WrappedWithLoRA, self).__init__() + + self.fc = fc + self.ab = self.create_ab_as_linear(fc.weight.data, ab_r) + + def create_ab_as_linear(self, fc_weight: torch.Tensor, ab_r: int): + res = nn.Sequential( + LoRA(fc_weight.size(1), fc_weight.size(0) // ab_r, bias=False), + LoRA(fc_weight.size(0) // ab_r, fc_weight.size(0), bias=False) + ).to(fc_weight.device) + nn.init.kaiming_uniform_(res[0].weight, a=5 ** 0.5) + nn.init.zeros_(res[1].weight) + return res + + def forward(self, x): + x1 = self.fc(x) + x2 = self.ab(x) + return x1 + x2 + + +class FMLoRA_Bert_Util(FMLoRA_Util): + + @torch.no_grad() + def add_lora_ab_to_fm(self, fm: nn.Module, ab_r: int, samples: dict): + fm.eval() + + o1 = fm(**samples) + + for name, module in fm.named_modules(): + if name.endswith(('query', 'key', 'value')): + set_module(fm, name, ToQKV_WrappedWithLoRA(module, ab_r)) + + o2 = fm(**samples) + + if isinstance(o1, tuple): + o1 = o1[-1] + o2 = o2[-1] + output_diff = ((o1 - o2) ** 2).sum() + assert output_diff < 1e-5 + + return fm + + @torch.no_grad() + def absorb_lora_and_recover_net_structure(self, fm: nn.Module, samples: dict): + fm.eval() + # print('absorb lora before') + o1 = fm(**samples) + + for name, module in fm.named_modules(): + if not isinstance(module, ToQKV_WrappedWithLoRA): + continue + + fc = module.fc + ab = module.ab + + fc.weight.add_(ab[1].weight @ ab[0].weight) + + set_module(fm, name, fc) + + # print('absorb lora after') + o2 = fm(**samples) + + if isinstance(o1, tuple): + o1 = o1[-1] + o2 = o2[-1] + output_diff = ((o1 - o2) ** 2).sum() + assert output_diff < 1e-6, output_diff + + return fm + \ No newline at end of file diff --git a/methods/elasticdnn/pipeline/offline/fm_lora/clip.py b/methods/elasticdnn/pipeline/offline/fm_lora/clip.py new file mode 100644 index 0000000000000000000000000000000000000000..35e38a30187a91371ddfe2535ab83b3cc04da0da --- /dev/null +++ b/methods/elasticdnn/pipeline/offline/fm_lora/clip.py @@ -0,0 +1,86 @@ +import torch +from torch import nn +from abc import ABC, abstractmethod + +from utils.dl.common.model import get_model_device, get_model_latency, get_model_size, set_module +from utils.common.log import logger +from .base import FMLoRA_Util, LoRA + + +class ToQKV_WrappedWithLoRA(nn.Module): + def __init__(self, fc: nn.Linear, ab_r: int): + super(ToQKV_WrappedWithLoRA, self).__init__() + + self.fc = fc + self.ab = self.create_ab_as_linear(fc.weight.data, ab_r) + + def create_ab_as_linear(self, fc_weight: torch.Tensor, ab_r: int): + res = nn.Sequential( + LoRA(fc_weight.size(1), fc_weight.size(0) // ab_r, bias=False), + LoRA(fc_weight.size(0) // ab_r, fc_weight.size(0), bias=False) + ).to(fc_weight.device) + nn.init.kaiming_uniform_(res[0].weight, a=5 ** 0.5) + nn.init.zeros_(res[1].weight) + return res + + def forward(self, x): + x1 = self.fc(x) + x2 = self.ab(x) + return x1 + x2 + + +class FMLoRA_CLIP_Util(FMLoRA_Util): + + @torch.no_grad() + def add_lora_ab_to_fm(self, fm: nn.Module, ab_r: int, samples: dict): + fm.eval() + + for k, v in samples.items(): + if isinstance(v, torch.Tensor): + samples[k] = v.to(get_model_device(fm)) + print(k) + + o1 = fm(**samples) + + for name, module in fm.named_modules(): + if name.endswith(('k_proj', 'q_proj', 'v_proj')): + set_module(fm, name, ToQKV_WrappedWithLoRA(module, ab_r)) + + o2 = fm(**samples) + + output_diff = ((o1.logits_per_image - o2.logits_per_image) ** 2).sum() + ((o1.logits_per_text - o2.logits_per_text) ** 2).sum() + assert output_diff < 1e-5 + + return fm + + @torch.no_grad() + def absorb_lora_and_recover_net_structure(self, fm: nn.Module, samples: dict): + fm.eval() + # print('absorb lora before') + + for k, v in samples.items(): + if isinstance(v, torch.Tensor): + samples[k] = v.to(get_model_device(fm)) + print(k) + + o1 = fm(**samples) + + for name, module in fm.named_modules(): + if not isinstance(module, ToQKV_WrappedWithLoRA): + continue + + fc = module.fc + ab = module.ab + + fc.weight.add_(ab[1].weight @ ab[0].weight) + + set_module(fm, name, fc) + + # print('absorb lora after') + o2 = fm(**samples) + + output_diff = ((o1.logits_per_image - o2.logits_per_image) ** 2).sum() + ((o1.logits_per_text - o2.logits_per_text) ** 2).sum() + assert output_diff < 1e-6, output_diff + + return fm + \ No newline at end of file diff --git a/methods/elasticdnn/pipeline/offline/fm_lora/vilt.py b/methods/elasticdnn/pipeline/offline/fm_lora/vilt.py new file mode 100644 index 0000000000000000000000000000000000000000..244e503be595d01b38dcbb12833535dfdcf3a811 --- /dev/null +++ b/methods/elasticdnn/pipeline/offline/fm_lora/vilt.py @@ -0,0 +1,91 @@ +import torch +from torch import nn +from abc import ABC, abstractmethod + +from utils.dl.common.model import get_model_device, get_model_latency, get_model_size, set_module +from utils.common.log import logger +from .base import FMLoRA_Util, LoRA + + +class ToQKV_WrappedWithLoRA(nn.Module): + def __init__(self, fc: nn.Linear, ab_r: int): + super(ToQKV_WrappedWithLoRA, self).__init__() + + self.fc = fc + self.ab = self.create_ab_as_linear(fc.weight.data, ab_r) + + def create_ab_as_linear(self, fc_weight: torch.Tensor, ab_r: int): + res = nn.Sequential( + LoRA(fc_weight.size(1), fc_weight.size(0) // ab_r, bias=False), + LoRA(fc_weight.size(0) // ab_r, fc_weight.size(0), bias=False) + ).to(fc_weight.device) + nn.init.kaiming_uniform_(res[0].weight, a=5 ** 0.5) + nn.init.zeros_(res[1].weight) + return res + + def forward(self, x): + x1 = self.fc(x) + x2 = self.ab(x) + return x1 + x2 + + +class FMLoRA_Vilt_Util(FMLoRA_Util): + + @torch.no_grad() + def add_lora_ab_to_fm(self, fm: nn.Module, ab_r: int, samples: dict): + fm.eval() + + # print(samples) + for k, v in samples.items(): + if isinstance(v, torch.Tensor): + samples[k] = v.to(get_model_device(fm)) + + o1 = fm(**samples) + + for name, module in fm.named_modules(): + if name.endswith(('query', 'key', 'value')): + set_module(fm, name, ToQKV_WrappedWithLoRA(module, ab_r)) + + o2 = fm(**samples) + + if isinstance(o1, tuple): + o1 = o1[-1] + o2 = o2[-1] + output_diff = ((o1.logits - o2.logits) ** 2).sum() + assert output_diff < 1e-5 + + return fm + + @torch.no_grad() + def absorb_lora_and_recover_net_structure(self, fm: nn.Module, samples: dict): + fm.eval() + # print('absorb lora before') + + for k, v in samples.items(): + if isinstance(v, torch.Tensor): + samples[k] = v.to(get_model_device(fm)) + + o1 = fm(**samples) + + for name, module in fm.named_modules(): + if not isinstance(module, ToQKV_WrappedWithLoRA): + continue + + fc = module.fc + ab = module.ab + + fc.weight.add_(ab[1].weight @ ab[0].weight) + + set_module(fm, name, fc) + + # print('absorb lora after') + o2 = fm(**samples) + + if isinstance(o1, tuple): + o1 = o1[-1] + o2 = o2[-1] + output_diff = ((o1.logits - o2.logits) ** 2).sum() + assert output_diff < 1e-6, output_diff + + return fm + \ No newline at end of file diff --git a/methods/elasticdnn/pipeline/offline/fm_lora/vit.py b/methods/elasticdnn/pipeline/offline/fm_lora/vit.py new file mode 100644 index 0000000000000000000000000000000000000000..0363e2ea8fb72c20b856bfd544ae0b265d41555e --- /dev/null +++ b/methods/elasticdnn/pipeline/offline/fm_lora/vit.py @@ -0,0 +1,79 @@ +import torch +from torch import nn +from abc import ABC, abstractmethod + +from utils.dl.common.model import get_model_device, get_model_latency, get_model_size, set_module +from utils.common.log import logger +from .base import FMLoRA_Util, LoRA + + +class ToQKV_WrappedWithLoRA(nn.Module): + def __init__(self, qkv: nn.Linear, ab_r: int): + super(ToQKV_WrappedWithLoRA, self).__init__() + + self.qkv = qkv + self.abs = nn.ModuleList([self.create_ab_as_linear(w, ab_r) for w in qkv.weight.data.chunk(3, dim=0)]) + + def create_ab_as_linear(self, fc_weight: torch.Tensor, ab_r: int): + res = nn.Sequential( + LoRA(fc_weight.size(1), fc_weight.size(0) // ab_r, bias=False), + LoRA(fc_weight.size(0) // ab_r, fc_weight.size(0), bias=False) + ).to(fc_weight.device) + nn.init.kaiming_uniform_(res[0].weight, a=5 ** 0.5) + nn.init.zeros_(res[1].weight) + return res + + def forward(self, x): + x1 = self.qkv(x) + x2 = torch.cat([ab(x) for ab in self.abs], dim=-1) + return x1 + x2 + + +class FMLoRA_ViT_Util(FMLoRA_Util): + + @torch.no_grad() + def add_lora_ab_to_fm(self, fm: nn.Module, ab_r: int, samples: torch.Tensor): + fm.eval() + o1 = fm(samples) + + for name, module in fm.named_modules(): + if not name.endswith('.qkv'): + continue + set_module(fm, name, ToQKV_WrappedWithLoRA(module, ab_r)) + + o2 = fm(samples) + + output_diff = ((o1 - o2) ** 2).sum() + assert output_diff < 1e-5 + + return fm + + @torch.no_grad() + def absorb_lora_and_recover_net_structure(self, fm: nn.Module, samples: torch.Tensor): + fm.eval() + # print('absorb lora before') + o1 = fm(samples) + + for name, module in fm.named_modules(): + if not isinstance(module, ToQKV_WrappedWithLoRA): + continue + + qkv = module.qkv + fm_abs = module.abs + + fm_abs_weight = torch.cat([_abs[1].weight @ _abs[0].weight for _abs in fm_abs], dim=0) + qkv.weight.add_(fm_abs_weight) + + set_module(fm, name, qkv) + + # print('absorb lora after') + o2 = fm(samples) + + output_diff = ((o1 - o2) ** 2).sum() + # print(o1) + # print(o2) + assert output_diff < 1e-5, output_diff + + return fm + + diff --git a/methods/elasticdnn/pipeline/offline/fm_to_md/__pycache__/base.cpython-38.pyc b/methods/elasticdnn/pipeline/offline/fm_to_md/__pycache__/base.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..50e7b2b262ae73d585a2a52421d491d9bc4654d7 Binary files /dev/null and b/methods/elasticdnn/pipeline/offline/fm_to_md/__pycache__/base.cpython-38.pyc differ diff --git a/methods/elasticdnn/pipeline/offline/fm_to_md/__pycache__/vit.cpython-38.pyc b/methods/elasticdnn/pipeline/offline/fm_to_md/__pycache__/vit.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4e187710e2b5f2adaee7c88b591dcc72434c5c5d Binary files /dev/null and b/methods/elasticdnn/pipeline/offline/fm_to_md/__pycache__/vit.cpython-38.pyc differ diff --git a/methods/elasticdnn/pipeline/offline/fm_to_md/base.py b/methods/elasticdnn/pipeline/offline/fm_to_md/base.py new file mode 100644 index 0000000000000000000000000000000000000000..fab5b61e491e54fcaca38571e5549f99aa79001b --- /dev/null +++ b/methods/elasticdnn/pipeline/offline/fm_to_md/base.py @@ -0,0 +1,48 @@ +import torch +from torch import nn +from abc import ABC, abstractmethod + +from utils.dl.common.model import get_model_device, get_model_latency, get_model_size +from utils.common.log import logger + + +class FM_to_MD_Util(ABC): + """ + Foundation Model (FM) to Master DNN (MD), where MD is a narrower FM (with smaller width but the same depth). + + MD is pre-trained by knowledge distillation; + Moreover, we construct the index relationship between FM and MD in this process, + enabling the lightweight knowledge feedback from MD to FM. + + NOTE: 索引建立在master DNN权重通道和LoRA的AB之间 + """ + + @abstractmethod + def init_md_from_fm_by_reducing_width(self, fm: nn.Module, reducing_width_ratio: int) -> nn.Module: + raise NotImplementedError + + def init_md_from_fm_by_reducing_width_with_perf_test(self, fm: nn.Module, reducing_width_ratio: int, + samples: torch.Tensor) -> nn.Module: + fm_size = get_model_size(fm, True) + fm_latency = get_model_latency(fm, (1, *list(samples.size())[1:]), 20, + get_model_device(fm), 20, False) + + master_dnn = self.init_md_from_fm_by_reducing_width(fm, reducing_width_ratio) + master_dnn_size = get_model_size(master_dnn, True) + logger.debug(f'inited master DNN: {master_dnn}') + # from utils.dl.common.model import get_module + # print('after generating') + # get_module(fm, 'head').debug() + # get_module(master_dnn, 'head').debug() + # print('test master latency') + master_dnn_latency = get_model_latency(master_dnn, (1, *list(samples.size())[1:]), 20, + get_model_device(master_dnn), 20, False) + + logger.info(f'init master DNN (w/o FBS yet) by reducing foundation model\'s width (by {reducing_width_ratio:d}x)') + logger.info(f'foundation model ({fm_size:.3f}MB, {fm_latency:.4f}s/sample) -> ' + f'master DNN ({master_dnn_size:.3f}MB, {master_dnn_latency:.4f}s/sample)\n' + f'(model size: ↓ {(fm_size / master_dnn_size):.2f}x, ' + f'latency: ↓ {(fm_latency / master_dnn_latency):.2f}x)') + + return master_dnn + \ No newline at end of file diff --git a/methods/elasticdnn/pipeline/offline/fm_to_md/bert.py b/methods/elasticdnn/pipeline/offline/fm_to_md/bert.py new file mode 100644 index 0000000000000000000000000000000000000000..b978953bc3092c0aec1d074b26bbbea38377744c --- /dev/null +++ b/methods/elasticdnn/pipeline/offline/fm_to_md/bert.py @@ -0,0 +1,279 @@ + +import torch +from torch import nn +from copy import deepcopy + +from .base import FM_to_MD_Util +from utils.common.log import logger +from utils.dl.common.model import set_module, get_module, get_super_module +from utils.dl.common.model import get_model_device, get_model_latency, get_model_size +from utils.common.log import logger + +from transformers.models.bert.modeling_bert import BertSelfAttention +from transformers import BertConfig + +from typing import Optional, Tuple +import math + +class BertSelfAttentionPrunable(BertSelfAttention): + def __init__(self): + config = BertConfig.from_pretrained('bert-base-multilingual-cased') + super(BertSelfAttentionPrunable, self).__init__(config) + + def transpose_for_scores(self, x): + new_x_shape = x.size()[:-1] + (self.num_attention_heads, -1) + x = x.view(new_x_shape) + return x.permute(0, 2, 1, 3) + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.FloatTensor] = None, + head_mask: Optional[torch.FloatTensor] = None, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + encoder_attention_mask: Optional[torch.FloatTensor] = None, + past_key_value: Optional[Tuple[Tuple[torch.FloatTensor]]] = None, + output_attentions: Optional[bool] = False, + ) -> Tuple[torch.Tensor]: + mixed_query_layer = self.query(hidden_states) + + # If this is instantiated as a cross-attention module, the keys + # and values come from an encoder; the attention mask needs to be + # such that the encoder's padding tokens are not attended to. + is_cross_attention = encoder_hidden_states is not None + + if is_cross_attention and past_key_value is not None: + # reuse k,v, cross_attentions + key_layer = past_key_value[0] + value_layer = past_key_value[1] + attention_mask = encoder_attention_mask + elif is_cross_attention: + key_layer = self.transpose_for_scores(self.key(encoder_hidden_states)) + value_layer = self.transpose_for_scores(self.value(encoder_hidden_states)) + attention_mask = encoder_attention_mask + elif past_key_value is not None: + key_layer = self.transpose_for_scores(self.key(hidden_states)) + value_layer = self.transpose_for_scores(self.value(hidden_states)) + key_layer = torch.cat([past_key_value[0], key_layer], dim=2) + value_layer = torch.cat([past_key_value[1], value_layer], dim=2) + else: + key_layer = self.transpose_for_scores(self.key(hidden_states)) + value_layer = self.transpose_for_scores(self.value(hidden_states)) + + query_layer = self.transpose_for_scores(mixed_query_layer) + + use_cache = past_key_value is not None + if self.is_decoder: + # if cross_attention save Tuple(torch.Tensor, torch.Tensor) of all cross attention key/value_states. + # Further calls to cross_attention layer can then reuse all cross-attention + # key/value_states (first "if" case) + # if uni-directional self-attention (decoder) save Tuple(torch.Tensor, torch.Tensor) of + # all previous decoder key/value_states. Further calls to uni-directional self-attention + # can concat previous decoder key/value_states to current projected key/value_states (third "elif" case) + # if encoder bi-directional self-attention `past_key_value` is always `None` + past_key_value = (key_layer, value_layer) + + # Take the dot product between "query" and "key" to get the raw attention scores. + attention_scores = torch.matmul(query_layer, key_layer.transpose(-1, -2)) + + if self.position_embedding_type == "relative_key" or self.position_embedding_type == "relative_key_query": + query_length, key_length = query_layer.shape[2], key_layer.shape[2] + if use_cache: + position_ids_l = torch.tensor(key_length - 1, dtype=torch.long, device=hidden_states.device).view( + -1, 1 + ) + else: + position_ids_l = torch.arange(query_length, dtype=torch.long, device=hidden_states.device).view(-1, 1) + position_ids_r = torch.arange(key_length, dtype=torch.long, device=hidden_states.device).view(1, -1) + distance = position_ids_l - position_ids_r + + positional_embedding = self.distance_embedding(distance + self.max_position_embeddings - 1) + positional_embedding = positional_embedding.to(dtype=query_layer.dtype) # fp16 compatibility + + if self.position_embedding_type == "relative_key": + relative_position_scores = torch.einsum("bhld,lrd->bhlr", query_layer, positional_embedding) + attention_scores = attention_scores + relative_position_scores + elif self.position_embedding_type == "relative_key_query": + relative_position_scores_query = torch.einsum("bhld,lrd->bhlr", query_layer, positional_embedding) + relative_position_scores_key = torch.einsum("bhrd,lrd->bhlr", key_layer, positional_embedding) + attention_scores = attention_scores + relative_position_scores_query + relative_position_scores_key + + attention_scores = attention_scores / math.sqrt(self.attention_head_size) + if attention_mask is not None: + # Apply the attention mask is (precomputed for all layers in BertModel forward() function) + attention_scores = attention_scores + attention_mask + + # Normalize the attention scores to probabilities. + attention_probs = nn.functional.softmax(attention_scores, dim=-1) + + # This is actually dropping out entire tokens to attend to, which might + # seem a bit unusual, but is taken from the original Transformer paper. + attention_probs = self.dropout(attention_probs) + + # Mask heads if we want to + if head_mask is not None: + attention_probs = attention_probs * head_mask + + context_layer = torch.matmul(attention_probs, value_layer) + + context_layer = context_layer.permute(0, 2, 1, 3).contiguous() + new_context_layer_shape = context_layer.size()[:-2] + (self.query.out_features,) # NOTE: modified + context_layer = context_layer.view(new_context_layer_shape) + + outputs = (context_layer, attention_probs) if output_attentions else (context_layer,) + + if self.is_decoder: + outputs = outputs + (past_key_value,) + return outputs + + @staticmethod + def init_from_exist_self_attn(attn: BertSelfAttention): + # print(attn) + + res = BertSelfAttentionPrunable() + + for attr in dir(attn): + # if str(attr) in ['transpose_for_scores'] or str(attr).startswith('_'): + # continue + # if isinstance(getattr(attn, attr), nn.Module): + # print(attr) + + if isinstance(getattr(attn, attr), nn.Module): + try: + # print(attr, 'ok') + setattr(res, attr, getattr(attn, attr)) + + except Exception as e: + print(attr, str(e)) + + + + return res + +class FM_to_MD_Bert_Util(FM_to_MD_Util): + def init_md_from_fm_by_reducing_width(self, fm: nn.Module, reducing_width_ratio: int) -> nn.Module: + fm_vit = deepcopy(fm) + + for block in fm_vit.bert.encoder.layer: + set_module(block, 'attention.self', BertSelfAttentionPrunable.init_from_exist_self_attn(block.attention.self)) + + def _f(n): + return int(n // reducing_width_ratio) + + # def _rand_indexes(n): + # return torch.randperm(n)[0: int(n // reducing_width_ratio)] + + def l1_max_indexes(p: torch.Tensor, dim=0): + assert dim in [0, 1] + assert p.dim() in [1, 2, 4] + + if dim == 1: + p = p.T + + p_norm = p.abs().contiguous().view(p.size(0), -1).sum(dim=1) + n = p.size(0) + return p_norm.argsort(descending=True)[0: int(n // reducing_width_ratio)].sort()[0] + + for block_i, block in enumerate(fm_vit.bert.encoder.layer): + for k in ['query', 'key', 'value']: + qkv = get_module(block, f'attention.self.{k}') + + new_qkv = nn.Linear(qkv.in_features, _f(qkv.out_features), + qkv.bias is not None, qkv.weight.device) + indexes = l1_max_indexes(qkv.weight.data, 0) + + new_qkv.weight.data.copy_(qkv.weight.data[indexes]) + if qkv.bias is not None: + new_qkv.bias.data.copy_(qkv.bias.data[indexes]) + set_module(block, f'attention.self.{k}', new_qkv) + + proj = get_module(block, f'attention.output.dense') + new_proj = nn.Linear(_f(proj.in_features), proj.out_features, + proj.bias is not None, proj.weight.device) + new_proj.weight.data.copy_(proj.weight.data[:, l1_max_indexes(proj.weight.data, 1)]) + if proj.bias is not None: + new_proj.bias.data.copy_(proj.bias.data) + set_module(block, f'attention.output.dense', new_proj) + + fc1 = get_module(block, f'intermediate.dense') + new_fc1 = nn.Linear(fc1.in_features, _f(fc1.out_features), + fc1.bias is not None, fc1.weight.device) + indexes = l1_max_indexes(fc1.weight.data, 0) + new_fc1.weight.data.copy_(fc1.weight.data[indexes]) + if fc1.bias is not None: + new_fc1.bias.data.copy_(fc1.bias.data[indexes]) + set_module(block, f'intermediate.dense', new_fc1) + + fc2 = get_module(block, f'output.dense') + new_fc2 = nn.Linear(_f(fc2.in_features), fc2.out_features, + fc2.bias is not None, fc2.weight.device) + new_fc2.weight.data.copy_(fc2.weight.data[:, l1_max_indexes(fc2.weight.data, 1)]) + if fc2.bias is not None: + new_fc2.bias.data.copy_(fc2.bias.data) + set_module(block, f'output.dense', new_fc2) + + return fm_vit + + def init_md_from_fm_by_reducing_width_with_perf_test(self, fm: nn.Module, reducing_width_ratio: int, + samples: torch.Tensor) -> nn.Module: + fm_size = get_model_size(fm, True) + fm_latency = self._get_model_latency(fm, samples, 20, + get_model_device(fm), 20, False) + + master_dnn = self.init_md_from_fm_by_reducing_width(fm, reducing_width_ratio) + master_dnn_size = get_model_size(master_dnn, True) + logger.debug(f'inited master DNN: {master_dnn}') + master_dnn_latency = self._get_model_latency(master_dnn, samples, 20, + get_model_device(master_dnn), 20, False) + + logger.info(f'init master DNN (w/o FBS yet) by reducing foundation model\'s width (by {reducing_width_ratio:d}x)') + logger.info(f'foundation model ({fm_size:.3f}MB, {fm_latency:.4f}s/sample) -> ' + f'master DNN ({master_dnn_size:.3f}MB, {master_dnn_latency:.4f}s/sample)\n' + f'(model size: ↓ {(fm_size / master_dnn_size):.2f}x, ' + f'latency: ↓ {(fm_latency / master_dnn_latency):.2f}x)') + + return master_dnn + + def _get_model_latency(self, model: torch.nn.Module, model_input_size, sample_num: int, + device: str, warmup_sample_num: int, return_detail=False): + import time + + if isinstance(model_input_size, tuple): + dummy_input = torch.rand(model_input_size).to(device) + else: + dummy_input = model_input_size + + model = model.to(device) + model.eval() + + # warm up + with torch.no_grad(): + for _ in range(warmup_sample_num): + model(**dummy_input) + + infer_time_list = [] + + if device == 'cuda' or 'cuda' in str(device): + with torch.no_grad(): + for _ in range(sample_num): + s, e = torch.cuda.Event(enable_timing=True), torch.cuda.Event(enable_timing=True) + s.record() + model(**dummy_input) + e.record() + torch.cuda.synchronize() + cur_model_infer_time = s.elapsed_time(e) / 1000. + infer_time_list += [cur_model_infer_time] + + else: + with torch.no_grad(): + for _ in range(sample_num): + start = time.time() + model(**dummy_input) + cur_model_infer_time = time.time() - start + infer_time_list += [cur_model_infer_time] + + avg_infer_time = sum(infer_time_list) / sample_num + + if return_detail: + return avg_infer_time, infer_time_list + return avg_infer_time \ No newline at end of file diff --git a/methods/elasticdnn/pipeline/offline/fm_to_md/clip.py b/methods/elasticdnn/pipeline/offline/fm_to_md/clip.py new file mode 100644 index 0000000000000000000000000000000000000000..9f35d59e15566def102e68b41d6a8172457f241d --- /dev/null +++ b/methods/elasticdnn/pipeline/offline/fm_to_md/clip.py @@ -0,0 +1,672 @@ + +import torch +from torch import nn +from copy import deepcopy + +from .base import FM_to_MD_Util +from utils.common.log import logger +from utils.dl.common.model import set_module, get_module, get_super_module +from utils.dl.common.model import get_model_device, get_model_latency, get_model_size +from utils.common.log import logger +from typing import Optional, Tuple + +from transformers.models.clip.modeling_clip import CLIPAttention +from transformers import CLIPVisionConfig + + +class CLIPAttentionPrunable(CLIPAttention): + """Multi-headed attention from 'Attention Is All You Need' paper""" + + + def __init__(self): + config = CLIPVisionConfig.from_pretrained('openai/clip-vit-base-patch16') + super(CLIPAttentionPrunable, self).__init__(config) + + + # def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int): + # # print(tensor.size(), self.num_heads, self.head_dim, bsz) # torch.Size([1, 197, 192]) 8 64 1 + # # head_dim should be modified + + # # 'b n (h d) -> b h n d', h = self.num_heads + + # if seq_len == -1: + # seq_len = tensor.size(1) + + # # print(tensor.size(), bsz, seq_len, self.num_heads, -1) + # return tensor.view(bsz, seq_len, self.num_heads, -1).transpose(1, 2).contiguous() + + # def forward( + # self, + # hidden_states: torch.Tensor, + # attention_mask: Optional[torch.Tensor] = None, + # causal_attention_mask: Optional[torch.Tensor] = None, + # output_attentions: Optional[bool] = False, + # ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: + # """Input shape: Batch x Time x Channel""" + + # bsz, tgt_len, embed_dim = hidden_states.size() + + # # get query proj + # query_states = self.q_proj(hidden_states) * self.scale + # key_states = self._shape(self.k_proj(hidden_states), -1, bsz) + # value_states = self._shape(self.v_proj(hidden_states), -1, bsz) + + # proj_shape = (-1, tgt_len, self.head_dim) + # # print(proj_shape, key_states.size()) + # query_states = self._shape(query_states, tgt_len, bsz).view(*proj_shape) + # key_states = key_states.view(*proj_shape) + # value_states = value_states.view(*proj_shape) + + # src_len = key_states.size(1) + # attn_weights = torch.bmm(query_states, key_states.transpose(1, 2)) + + # # if attn_weights.size() != (bsz * self.num_heads, tgt_len, src_len): + # # raise ValueError( + # # f"Attention weights should be of size {(bsz * self.num_heads, tgt_len, src_len)}, but is" + # # f" {attn_weights.size()}" + # # ) + + # # apply the causal_attention_mask first + # if causal_attention_mask is not None: + # if causal_attention_mask.size() != (bsz, 1, tgt_len, src_len): + # raise ValueError( + # f"Attention mask should be of size {(bsz, 1, tgt_len, src_len)}, but is" + # f" {causal_attention_mask.size()}" + # ) + # attn_weights = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + causal_attention_mask + # attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len) + + # if attention_mask is not None: + # if attention_mask.size() != (bsz, 1, tgt_len, src_len): + # raise ValueError( + # f"Attention mask should be of size {(bsz, 1, tgt_len, src_len)}, but is {attention_mask.size()}" + # ) + # attn_weights = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + attention_mask + # attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len) + + # attn_weights = nn.functional.softmax(attn_weights, dim=-1) + + # if output_attentions: + # # this operation is a bit akward, but it's required to + # # make sure that attn_weights keeps its gradient. + # # In order to do so, attn_weights have to reshaped + # # twice and have to be reused in the following + # attn_weights_reshaped = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + # attn_weights = attn_weights_reshaped.view(bsz * self.num_heads, tgt_len, src_len) + # else: + # attn_weights_reshaped = None + + # attn_probs = nn.functional.dropout(attn_weights, p=self.dropout, training=self.training) + + # attn_output = torch.bmm(attn_probs, value_states) + + # # if attn_output.size() != (bsz * self.num_heads, tgt_len, self.head_dim): + # # raise ValueError( + # # f"`attn_output` should be of size {(bsz, self.num_heads, tgt_len, self.head_dim)}, but is" + # # f" {attn_output.size()}" + # # ) + + # attn_output = attn_output.view(bsz, self.num_heads, tgt_len, -1) + # attn_output = attn_output.transpose(1, 2) + # attn_output = attn_output.reshape(bsz, tgt_len, -1) + + # attn_output = self.out_proj(attn_output) + + # return attn_output, attn_weights_reshaped + + + def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int): + return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous() + + def _shape_dynamic_head_dim(self, tensor: torch.Tensor, seq_len: int, bsz: int): + return tensor.view(bsz, seq_len, self.num_heads, -1).transpose(1, 2).contiguous() + + def _shape_dynamic_num_head(self, tensor: torch.Tensor, seq_len: int, bsz: int): + return tensor.view(bsz, seq_len, -1, self.head_dim).transpose(1, 2).contiguous() + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + causal_attention_mask: Optional[torch.Tensor] = None, + output_attentions: Optional[bool] = False, + ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: + """Input shape: Batch x Time x Channel""" + + bsz, tgt_len, embed_dim = hidden_states.size() + + # logger.info(f'hidden state size: {hidden_states.size()}') # (64, 197, 768) + + # get query proj + query_states = self.q_proj(hidden_states) * self.scale + key_states = self._shape_dynamic_head_dim(self.k_proj(hidden_states), tgt_len, bsz) + value_states = self._shape_dynamic_head_dim(self.v_proj(hidden_states), tgt_len, bsz) + + # (64, 197, 768), numhead: 12, head_dim: 64, seq_len: 197 + # logger.info(f'key states: {self.k_proj(hidden_states).size()}, bsz: {bsz}, num_heads: {self.num_heads}, head_dim: {self.head_dim}, ' + # f'seq_len: {self.k_proj(hidden_states).numel() / bsz / self.num_heads / self.head_dim}') + # (64, 197, 768), numhead: 12, head_dim: 64, seq_len: 197 + # logger.info(f'value states: {self.v_proj(hidden_states).size()}, bsz: {bsz}, num_heads: {self.num_heads}, head_dim: {self.head_dim}, ' + # f'seq_len: {self.v_proj(hidden_states).numel() / bsz / self.num_heads / self.head_dim}') + + proj_shape = (bsz * self.num_heads, tgt_len, -1) + query_states = self._shape_dynamic_head_dim(query_states, tgt_len, bsz).view(*proj_shape) + + # (64, 12, 197, 64), -1 means 197 + # logger.info(f'query states: {self._shape(query_states, tgt_len, bsz).size()}, ' + # f'-1 in proj_shape: {self._shape(query_states, tgt_len, bsz).numel() / bsz / self.num_heads / self.head_dim}') + + key_states = key_states.view(*proj_shape) + value_states = value_states.view(*proj_shape) + + src_len = key_states.size(1) + attn_weights = torch.bmm(query_states, key_states.transpose(1, 2)) + + if attn_weights.size() != (bsz * self.num_heads, tgt_len, src_len): + raise ValueError( + f"Attention weights should be of size {(bsz * self.num_heads, tgt_len, src_len)}, but is" + f" {attn_weights.size()}" + ) + + # apply the causal_attention_mask first + if causal_attention_mask is not None: + if causal_attention_mask.size() != (bsz, 1, tgt_len, src_len): + raise ValueError( + f"Attention mask should be of size {(bsz, 1, tgt_len, src_len)}, but is" + f" {causal_attention_mask.size()}" + ) + attn_weights = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + causal_attention_mask + attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len) + + if attention_mask is not None: + if attention_mask.size() != (bsz, 1, tgt_len, src_len): + raise ValueError( + f"Attention mask should be of size {(bsz, 1, tgt_len, src_len)}, but is {attention_mask.size()}" + ) + attn_weights = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + attention_mask + attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len) + + attn_weights = nn.functional.softmax(attn_weights, dim=-1) + + if output_attentions: + # this operation is a bit akward, but it's required to + # make sure that attn_weights keeps its gradient. + # In order to do so, attn_weights have to reshaped + # twice and have to be reused in the following + attn_weights_reshaped = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + attn_weights = attn_weights_reshaped.view(bsz * self.num_heads, tgt_len, src_len) + else: + attn_weights_reshaped = None + + attn_probs = nn.functional.dropout(attn_weights, p=self.dropout, training=self.training) + + attn_output = torch.bmm(attn_probs, value_states) + + # if attn_output.size() != (bsz * self.num_heads, tgt_len, self.head_dim): + # raise ValueError( + # f"`attn_output` should be of size {(bsz, self.num_heads, tgt_len, self.head_dim)}, but is" + # f" {attn_output.size()}" + # ) + # print(attn_output.size(), bsz, tgt_len, embed_dim) + attn_output = attn_output.view(bsz, self.num_heads, tgt_len, -1) + attn_output = attn_output.transpose(1, 2) + attn_output = attn_output.reshape(bsz, tgt_len, -1) + + attn_output = self.out_proj(attn_output) + + return attn_output, attn_weights_reshaped + + # reduce num_head + # def forward( + # self, + # hidden_states: torch.Tensor, + # attention_mask: Optional[torch.Tensor] = None, + # causal_attention_mask: Optional[torch.Tensor] = None, + # output_attentions: Optional[bool] = False, + # ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: + # """Input shape: Batch x Time x Channel""" + + # bsz, tgt_len, embed_dim = hidden_states.size() + + # # logger.info(f'hidden state size: {hidden_states.size()}') # (64, 197, 768) + + # # get query proj + # query_states = self.q_proj(hidden_states) * self.scale + # key_states = self._shape_dynamic_num_head(self.k_proj(hidden_states), tgt_len, bsz) + # value_states = self._shape_dynamic_num_head(self.v_proj(hidden_states), tgt_len, bsz) + + # # (64, 197, 768), numhead: 12, head_dim: 64, seq_len: 197 + # # logger.info(f'key states: {self.k_proj(hidden_states).size()}, bsz: {bsz}, num_heads: {self.num_heads}, head_dim: {self.head_dim}, ' + # # f'seq_len: {self.k_proj(hidden_states).numel() / bsz / self.num_heads / self.head_dim}') + # # (64, 197, 768), numhead: 12, head_dim: 64, seq_len: 197 + # # logger.info(f'value states: {self.v_proj(hidden_states).size()}, bsz: {bsz}, num_heads: {self.num_heads}, head_dim: {self.head_dim}, ' + # # f'seq_len: {self.v_proj(hidden_states).numel() / bsz / self.num_heads / self.head_dim}') + + # proj_shape = (-1, tgt_len, self.head_dim) + # query_states = self._shape_dynamic_head_dim(query_states, tgt_len, bsz).view(*proj_shape) + + # # (64, 12, 197, 64), -1 means 197 + # # logger.info(f'query states: {self._shape(query_states, tgt_len, bsz).size()}, ' + # # f'-1 in proj_shape: {self._shape(query_states, tgt_len, bsz).numel() / bsz / self.num_heads / self.head_dim}') + + # key_states = key_states.view(*proj_shape) + # value_states = value_states.view(*proj_shape) + + # src_len = key_states.size(1) + # attn_weights = torch.bmm(query_states, key_states.transpose(1, 2)) + + # # if attn_weights.size() != (bsz * self.num_heads, tgt_len, src_len): + # # raise ValueError( + # # f"Attention weights should be of size {(bsz * self.num_heads, tgt_len, src_len)}, but is" + # # f" {attn_weights.size()}" + # # ) + + # # apply the causal_attention_mask first + # if causal_attention_mask is not None: + # if causal_attention_mask.size() != (bsz, 1, tgt_len, src_len): + # raise ValueError( + # f"Attention mask should be of size {(bsz, 1, tgt_len, src_len)}, but is" + # f" {causal_attention_mask.size()}" + # ) + # attn_weights = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + causal_attention_mask + # attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len) + + # if attention_mask is not None: + # if attention_mask.size() != (bsz, 1, tgt_len, src_len): + # raise ValueError( + # f"Attention mask should be of size {(bsz, 1, tgt_len, src_len)}, but is {attention_mask.size()}" + # ) + # attn_weights = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + attention_mask + # attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len) + + # attn_weights = nn.functional.softmax(attn_weights, dim=-1) + + # if output_attentions: + # # this operation is a bit akward, but it's required to + # # make sure that attn_weights keeps its gradient. + # # In order to do so, attn_weights have to reshaped + # # twice and have to be reused in the following + # attn_weights_reshaped = attn_weights.view(bsz, -1, tgt_len, src_len) + # attn_weights = attn_weights_reshaped.view(-1, tgt_len, src_len) + # else: + # attn_weights_reshaped = None + + # attn_probs = nn.functional.dropout(attn_weights, p=self.dropout, training=self.training) + + # attn_output = torch.bmm(attn_probs, value_states) + + # # if attn_output.size() != (bsz * self.num_heads, tgt_len, self.head_dim): + # # raise ValueError( + # # f"`attn_output` should be of size {(bsz, self.num_heads, tgt_len, self.head_dim)}, but is" + # # f" {attn_output.size()}" + # # ) + # # print(attn_output.size(), bsz, tgt_len, embed_dim) + # attn_output = attn_output.view(bsz, -1, tgt_len, self.head_dim) + # attn_output = attn_output.transpose(1, 2) + # attn_output = attn_output.reshape(bsz, tgt_len, -1) + + # attn_output = self.out_proj(attn_output) + + # return attn_output, attn_weights_reshaped + + + @staticmethod + def init_from_exist_self_attn(attn: CLIPAttention): + # print(attn) + + res = CLIPAttentionPrunable() + + for attr in dir(attn): + # if str(attr) in ['transpose_for_scores'] or str(attr).startswith('_'): + # continue + # if isinstance(getattr(attn, attr), nn.Module): + # print(attr) + + if isinstance(getattr(attn, attr), nn.Module): + try: + # print(attr, 'ok') + setattr(res, attr, getattr(attn, attr)) + + except Exception as e: + print(attr, str(e)) + + + + return res + + +from einops import rearrange, repeat +from einops.layers.torch import Rearrange + +class PrunableAttention(nn.Module): + """ + https://github.com/lucidrains/vit-pytorch + """ + def __init__(self, dim, heads = 8, dim_head = 64, dropout = 0., qkv_bias = False): + super().__init__() + self.inner_dim = inner_dim = dim_head * heads + project_out = not (heads == 1 and dim_head == dim) + + self.num_heads = heads + self.scale = dim_head ** -0.5 + + self.attend = nn.Softmax(dim = -1) + self.dropout = nn.Dropout(dropout) + + self.qkv = nn.Linear(dim, inner_dim * 3, bias = qkv_bias) + + # self.proj = nn.Sequential( + # nn.Linear(inner_dim, dim), + # nn.Dropout(dropout) + # ) if project_out else nn.Identity() + + self.proj = nn.Linear(inner_dim, dim) if project_out else nn.Identity() + self.proj_dropout = nn.Dropout(dropout) + + def forward(self, hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + causal_attention_mask: Optional[torch.Tensor] = None, + output_attentions: Optional[bool] = False,): + + x = hidden_states + assert attention_mask is None + assert causal_attention_mask is None + assert not output_attentions + # qkv = self.qkv(x).chunk(3, dim = -1) + raw_qkv = self.qkv(x) + + self.inner_dim = (raw_qkv.size(-1) - self.proj.in_features) // 2 + qkv = raw_qkv[:, :, 0: self.inner_dim], raw_qkv[:, :, self.inner_dim: self.inner_dim * 2], raw_qkv[:, :, self.inner_dim * 2:] + + # print('v', qkv[0].size(), qkv[0].sum((0, 1))[0: 10], qkv[0].sum((0, 1)).nonzero(as_tuple=True)[0].size()) + + # raw_v = qkv[2] + # print('after_fbs_q, after_fbs_k', qkv[0].sum((0, 1))[0: 10], qkv[0].sum((0, 1)).nonzero(as_tuple=True)[0].size(), + # qkv[1].sum((0, 1))[0: 10], qkv[1].sum((0, 1)).nonzero(as_tuple=True)[0].size(),) + # print('after_fbs_v', raw_v.size(), raw_v.sum((0, 1))[0: 10], raw_v.sum((0, 1)).nonzero(as_tuple=True)[0].size()) + # print('q, before rearrage', qkv[0].size()) + q, k, v = qkv + # print('raw qkv size', q.size(), k.size(), v.size()) + # exit() + q, k, v = map(lambda t: rearrange(t, 'b n (h d) -> b h n d', h = self.num_heads), qkv) + # print('raw qkv size', q.size(), k.size(), v.size()) + + dots = torch.matmul(q, k.transpose(-1, -2)) * self.scale + + # print('q, k, dots, after rearrage', q.size(), k.transpose(-1, -2).size(), dots.size()) + + attn = self.attend(dots) + # attn = dots + attn = self.dropout(attn) + + # print(attn) + # print('attn', attn.size(), attn.sum((0, 1))[0: 10], attn.sum((0, 1)).nonzero(as_tuple=True)[0].size()) + # print('attn', attn.size(), attn.sum((0, 1))[0: 10], attn.sum((0, 1)).nonzero(as_tuple=True)[0].size()) + # print('v2', v.size()) + out = torch.matmul(attn, v) + # print('out1', out.size()) + # NOTE: just for trial debug + # out = v + + # print('out before rerange', out.size()) + + # print(v.size(), v) + # exit() + + out = rearrange(out, 'b h n d -> b n (h d)') + + # print('out', out.size(), out.sum((0, 1))[0: 10], out.sum((0, 1)).nonzero(as_tuple=True)[0].size()) + # exit() + + res = self.proj_dropout(self.proj(out)) + + # res = self.proj_dropout( + # F.linear(self.proj.weight.T, out.T, self.proj.bias) + # ) + # print(self.proj, self.proj_dropout) + # print('res', res.size(), res.sum((0, 1))[0: 10], res.sum((0, 1)).nonzero(as_tuple=True)[0].size()) + + return res, None + + +class FM_to_MD_CLIP_Util(FM_to_MD_Util): + def init_md_from_fm_by_reducing_width(self, fm: nn.Module, reducing_width_ratio: int) -> nn.Module: + fm_vit = deepcopy(fm) + + + # for block in fm_vit.model.text_model.encoder.layers: + # set_module(block, 'self_attn', CLIPAttentionPrunable.init_from_exist_self_attn(block.self_attn)) + + debug_input = torch.rand((1, 3, 32, 32)).cuda() + fm.eval() + o1 = fm.model.vision_model(debug_input).pooler_output + for block in fm_vit.model.vision_model.encoder.layers: + # set_module(block, 'self_attn', CLIPAttentionPrunable.init_from_exist_self_attn(block.self_attn)) + + attn: CLIPAttention = block.self_attn + # from dnns.vit import PrunableAttention + new_attn = PrunableAttention( + dim=768, + heads=12, + dim_head=64, + dropout=0, + qkv_bias=True + ) + new_attn.qkv.weight.data.copy_(torch.cat([ + attn.q_proj.weight, + attn.k_proj.weight, + attn.v_proj.weight + ], dim=0)) + new_attn.qkv.bias.data.copy_(torch.cat([ + attn.q_proj.bias, + attn.k_proj.bias, + attn.v_proj.bias + ], dim=0)) + new_attn.proj.weight.data.copy_(attn.out_proj.weight) + new_attn.proj.bias.data.copy_(attn.out_proj.bias) + set_module(block, 'self_attn', new_attn) + o2 = fm.model.vision_model(debug_input).pooler_output + + # NOTE: bug is here!!! + # although the diff is ZERO, but the logic of CLIPAttentionPrunable is incorrect!!!! + diff = ((o1 - o2) ** 2).sum() + print('diff before/after adding CLIPAttentionPrunable', diff) + assert diff < 1e-4 + + # print('\n\nDEBUG: WITHOUT ADDING CLIPAttentionPrunable\n\n') + + # exit() + + # return fm + def _f(n): + return int(n // reducing_width_ratio) + + # def _rand_indexes(n): + # return torch.randperm(n)[0: int(n // reducing_width_ratio)] + + def l1_max_indexes(p: torch.Tensor, dim=0): + assert dim in [0, 1] + assert p.dim() in [1, 2, 4] + + if dim == 1: + p = p.T + + p_norm = p.abs().contiguous().view(p.size(0), -1).sum(dim=1) + n = p.size(0) + res = p_norm.argsort(descending=True)[0: int(n // reducing_width_ratio)].sort()[0] + # print(res) + return res + + # first_attn = True + + # for block_i, block in enumerate(fm_vit.model.text_model.encoder.layers): + # for k in ['k_proj', 'q_proj', 'v_proj']: + # qkv = get_module(block, f'self_attn.{k}') + + # new_qkv = nn.Linear(qkv.in_features, _f(qkv.out_features), + # qkv.bias is not None, qkv.weight.device) + # indexes = l1_max_indexes(qkv.weight.data, 0) + + # new_qkv.weight.data.copy_(qkv.weight.data[indexes]) + # if qkv.bias is not None: + # new_qkv.bias.data.copy_(qkv.bias.data[indexes]) + # set_module(block, f'self_attn.{k}', new_qkv) + + # proj = block.self_attn.out_proj + # new_proj = nn.Linear(_f(proj.in_features), proj.out_features, + # proj.bias is not None, proj.weight.device) + # new_proj.weight.data.copy_(proj.weight.data[:, l1_max_indexes(proj.weight.data, 1)]) + # if proj.bias is not None: + # new_proj.bias.data.copy_(proj.bias.data) + # set_module(block, f'self_attn.out_proj', new_proj) + + # fc1 = block.mlp.fc1 + # new_fc1 = nn.Linear(fc1.in_features, _f(fc1.out_features), + # fc1.bias is not None, fc1.weight.device) + # indexes = l1_max_indexes(fc1.weight.data, 0) + # new_fc1.weight.data.copy_(fc1.weight.data[indexes]) + # if fc1.bias is not None: + # new_fc1.bias.data.copy_(fc1.bias.data[indexes]) + # set_module(block, f'mlp.fc1', new_fc1) + + # fc2 = block.mlp.fc2 + # new_fc2 = nn.Linear(_f(fc2.in_features), fc2.out_features, + # fc2.bias is not None, fc2.weight.device) + # new_fc2.weight.data.copy_(fc2.weight.data[:, l1_max_indexes(fc2.weight.data, 1)]) + # if fc2.bias is not None: + # new_fc2.bias.data.copy_(fc2.bias.data) + # set_module(block, f'mlp.fc2', new_fc2) + + + for block_i, block in enumerate(fm_vit.model.vision_model.encoder.layers): + # for k in ['k_proj', 'q_proj', 'v_proj']: + # qkv = get_module(block, f'self_attn.{k}') + + # new_qkv = nn.Linear(qkv.in_features, _f(qkv.out_features), + # qkv.bias is not None, qkv.weight.device) + # indexes = l1_max_indexes(qkv.weight.data, 0) + + # new_qkv.weight.data.copy_(qkv.weight.data[indexes]) + # if qkv.bias is not None: + # new_qkv.bias.data.copy_(qkv.bias.data[indexes]) + # set_module(block, f'self_attn.{k}', new_qkv) + + # proj = block.self_attn.out_proj + # new_proj = nn.Linear(_f(proj.in_features), proj.out_features, + # proj.bias is not None, proj.weight.device) + # new_proj.weight.data.copy_(proj.weight.data[:, l1_max_indexes(proj.weight.data, 1)]) + # if proj.bias is not None: + # new_proj.bias.data.copy_(proj.bias.data) + # set_module(block, f'self_attn.out_proj', new_proj) + + + # ------------------ + + qkv = block.self_attn.qkv + new_qkv = nn.Linear(qkv.in_features, _f(qkv.out_features), + qkv.bias is not None, qkv.weight.device) + indexes = l1_max_indexes(qkv.weight.data, 0) + + new_qkv.weight.data.copy_(qkv.weight.data[indexes]) + if qkv.bias is not None: + new_qkv.bias.data.copy_(qkv.bias.data[indexes]) + set_module(block, f'self_attn.qkv', new_qkv) + proj = block.self_attn.proj + new_proj = nn.Linear(_f(proj.in_features), proj.out_features, + proj.bias is not None, proj.weight.device) + new_proj.weight.data.copy_(proj.weight.data[:, l1_max_indexes(proj.weight.data, 1)]) + if proj.bias is not None: + new_proj.bias.data.copy_(proj.bias.data) + set_module(block, f'self_attn.proj', new_proj) + + # -------------------- + + fc1 = block.mlp.fc1 + new_fc1 = nn.Linear(fc1.in_features, _f(fc1.out_features), + fc1.bias is not None, fc1.weight.device) + indexes = l1_max_indexes(fc1.weight.data, 0) + new_fc1.weight.data.copy_(fc1.weight.data[indexes]) + if fc1.bias is not None: + new_fc1.bias.data.copy_(fc1.bias.data[indexes]) + set_module(block, f'mlp.fc1', new_fc1) + + fc2 = block.mlp.fc2 + new_fc2 = nn.Linear(_f(fc2.in_features), fc2.out_features, + fc2.bias is not None, fc2.weight.device) + new_fc2.weight.data.copy_(fc2.weight.data[:, l1_max_indexes(fc2.weight.data, 1)]) + if fc2.bias is not None: + new_fc2.bias.data.copy_(fc2.bias.data) + set_module(block, f'mlp.fc2', new_fc2) + + + return fm_vit + + + def init_md_from_fm_by_reducing_width_with_perf_test(self, fm: nn.Module, reducing_width_ratio: int, + samples: torch.Tensor) -> nn.Module: + fm_size = get_model_size(fm, True) + fm_latency = self._get_model_latency(fm, samples, 20, + get_model_device(fm), 20, False) + + master_dnn = self.init_md_from_fm_by_reducing_width(fm, reducing_width_ratio) + master_dnn_size = get_model_size(master_dnn, True) + logger.debug(f'inited master DNN: {master_dnn}') + # from utils.dl.common.model import get_module + # print('after generating') + # get_module(fm, 'head').debug() + # get_module(master_dnn, 'head').debug() + # print('test master latency') + master_dnn_latency = self._get_model_latency(master_dnn, samples, 20, + get_model_device(master_dnn), 20, False) + + logger.info(f'init master DNN (w/o FBS yet) by reducing foundation model\'s width (by {reducing_width_ratio:d}x)') + logger.info(f'foundation model ({fm_size:.3f}MB, {fm_latency:.4f}s/sample) -> ' + f'master DNN ({master_dnn_size:.3f}MB, {master_dnn_latency:.4f}s/sample)\n' + f'(model size: ↓ {(fm_size / master_dnn_size):.2f}x, ' + f'latency: ↓ {(fm_latency / master_dnn_latency):.2f}x)') + + return master_dnn + + def _get_model_latency(self, model: torch.nn.Module, model_input_size, sample_num: int, + device: str, warmup_sample_num: int, return_detail=False): + import time + + if isinstance(model_input_size, tuple): + dummy_input = torch.rand(model_input_size).to(device) + else: + dummy_input = model_input_size + + model = model.to(device) + model.eval() + + # warm up + with torch.no_grad(): + for _ in range(warmup_sample_num): + model(**dummy_input) + + infer_time_list = [] + + if device == 'cuda' or 'cuda' in str(device): + with torch.no_grad(): + for _ in range(sample_num): + s, e = torch.cuda.Event(enable_timing=True), torch.cuda.Event(enable_timing=True) + s.record() + model(**dummy_input) + e.record() + torch.cuda.synchronize() + cur_model_infer_time = s.elapsed_time(e) / 1000. + infer_time_list += [cur_model_infer_time] + + else: + with torch.no_grad(): + for _ in range(sample_num): + start = time.time() + model(**dummy_input) + cur_model_infer_time = time.time() - start + infer_time_list += [cur_model_infer_time] + + avg_infer_time = sum(infer_time_list) / sample_num + + if return_detail: + return avg_infer_time, infer_time_list + return avg_infer_time \ No newline at end of file diff --git a/methods/elasticdnn/pipeline/offline/fm_to_md/test.py b/methods/elasticdnn/pipeline/offline/fm_to_md/test.py new file mode 100644 index 0000000000000000000000000000000000000000..df3925ab7042fcf2094e2cb415506528a0469e55 --- /dev/null +++ b/methods/elasticdnn/pipeline/offline/fm_to_md/test.py @@ -0,0 +1,28 @@ +from methods.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +import torch + + +def test(fm, fm_to_md_util: FM_to_MD_Util, samples: torch.Tensor): + master_dnn = fm_to_md_util.init_md_from_fm_by_reducing_width_with_perf_test(fm, 4, samples) + torch.save(master_dnn, 'tmp-master-dnn.pt') + +if __name__ == '__main__': + from utils.dl.common.env import set_random_seed + set_random_seed(1) + + # from torchvision.models import resnet50 + # from methods.elasticdnn.model.cnn import ElasticCNNUtil + # raw_cnn = resnet50() + # prunable_layers = [] + # for i in range(1, 5): + # for j in range([3, 4, 6, 3][i - 1]): + # prunable_layers += [f'layer{i}.{j}.conv1', f'layer{i}.{j}.conv2'] + # ignore_layers = [layer for layer, m in raw_cnn.named_modules() if isinstance(m, nn.Conv2d) and layer not in prunable_layers] + # test(raw_cnn, ignore_layers, ElasticCNNUtil(), torch.rand(2, 3, 224, 224)) + + + from dnns.vit import vit_b_16 + from methods.elasticdnn.pipeline.offline.fm_to_md.vit import FM_to_MD_ViT_Util + raw_vit = vit_b_16() + test(raw_vit.cuda(), FM_to_MD_ViT_Util(), torch.rand(2, 3, 224, 224).cuda()) + \ No newline at end of file diff --git a/methods/elasticdnn/pipeline/offline/fm_to_md/vilt.py b/methods/elasticdnn/pipeline/offline/fm_to_md/vilt.py new file mode 100644 index 0000000000000000000000000000000000000000..80829ed7440ee83860e979a94e7d2a9cf8f79b92 --- /dev/null +++ b/methods/elasticdnn/pipeline/offline/fm_to_md/vilt.py @@ -0,0 +1,216 @@ + +import torch +from torch import nn +from copy import deepcopy + +from .base import FM_to_MD_Util +from utils.common.log import logger +from utils.dl.common.model import set_module, get_module, get_super_module +from utils.dl.common.model import get_model_device, get_model_latency, get_model_size +from utils.common.log import logger + +from transformers.models.vilt.modeling_vilt import ViltSelfAttention +from transformers import ViltConfig + +from typing import Optional, Tuple +import math + + +class ViltSelfAttentionPrunable(ViltSelfAttention): + def __init__(self): + config = ViltConfig.from_pretrained('dandelin/vilt-b32-mlm-itm') + super(ViltSelfAttentionPrunable, self).__init__(config) + + def transpose_for_scores(self, x): + new_x_shape = x.size()[:-1] + (self.num_attention_heads, -1) + x = x.view(*new_x_shape) + return x.permute(0, 2, 1, 3) + + def forward(self, hidden_states, attention_mask=None, head_mask=None, output_attentions=False): + mixed_query_layer = self.query(hidden_states) + + key_layer = self.transpose_for_scores(self.key(hidden_states)) + value_layer = self.transpose_for_scores(self.value(hidden_states)) + query_layer = self.transpose_for_scores(mixed_query_layer) + + # Take the dot product between "query" and "key" to get the raw attention scores. + attention_scores = torch.matmul(query_layer, key_layer.transpose(-1, -2)) + attention_scores = attention_scores / math.sqrt(self.attention_head_size) + if attention_mask is not None: + # Apply the attention mask is (precomputed for all layers in BertModel forward() function) + attention_scores = attention_scores + attention_mask + + # Normalize the attention scores to probabilities. + attention_probs = nn.Softmax(dim=-1)(attention_scores) + + # This is actually dropping out entire tokens to attend to, which might + # seem a bit unusual, but is taken from the original Transformer paper. + attention_probs = self.dropout(attention_probs) + + # Mask heads if we want to + if head_mask is not None: + attention_probs = attention_probs * head_mask + + context_layer = torch.matmul(attention_probs, value_layer) + + context_layer = context_layer.permute(0, 2, 1, 3).contiguous() + # new_context_layer_shape = context_layer.size()[:-2] + (self.all_head_size,) + new_context_layer_shape = context_layer.size()[:-2] + (-1,) + context_layer = context_layer.view(*new_context_layer_shape) + + outputs = (context_layer, attention_probs) if output_attentions else (context_layer,) + + return outputs + + @staticmethod + def init_from_exist_self_attn(attn: ViltSelfAttention): + # print(attn) + + res = ViltSelfAttentionPrunable() + + for attr in dir(attn): + # if str(attr) in ['transpose_for_scores'] or str(attr).startswith('_'): + # continue + # if isinstance(getattr(attn, attr), nn.Module): + # print(attr) + + if isinstance(getattr(attn, attr), nn.Module): + try: + # print(attr, 'ok') + setattr(res, attr, getattr(attn, attr)) + + except Exception as e: + print(attr, str(e)) + + + + return res + + +class FM_to_MD_Vilt_Util(FM_to_MD_Util): + def init_md_from_fm_by_reducing_width(self, fm: nn.Module, reducing_width_ratio: int) -> nn.Module: + fm_vit = deepcopy(fm) + + for block in fm_vit.vilt.encoder.layer: + set_module(block, 'attention.attention', ViltSelfAttentionPrunable.init_from_exist_self_attn(block.attention.attention)) + + def _f(n): + return int(n // reducing_width_ratio) + + # def _rand_indexes(n): + # return torch.randperm(n)[0: int(n // reducing_width_ratio)] + + def l1_max_indexes(p: torch.Tensor, dim=0): + assert dim in [0, 1] + assert p.dim() in [1, 2, 4] + + if dim == 1: + p = p.T + + p_norm = p.abs().contiguous().view(p.size(0), -1).sum(dim=1) + n = p.size(0) + return p_norm.argsort(descending=True)[0: int(n // reducing_width_ratio)].sort()[0] + + for block_i, block in enumerate(fm_vit.vilt.encoder.layer): + for k in ['query', 'key', 'value']: + qkv = get_module(block, f'attention.attention.{k}') + + new_qkv = nn.Linear(qkv.in_features, _f(qkv.out_features), + qkv.bias is not None, qkv.weight.device) + indexes = l1_max_indexes(qkv.weight.data, 0) + + new_qkv.weight.data.copy_(qkv.weight.data[indexes]) + if qkv.bias is not None: + new_qkv.bias.data.copy_(qkv.bias.data[indexes]) + set_module(block, f'attention.attention.{k}', new_qkv) + + proj = get_module(block, f'attention.output.dense') + new_proj = nn.Linear(_f(proj.in_features), proj.out_features, + proj.bias is not None, proj.weight.device) + new_proj.weight.data.copy_(proj.weight.data[:, l1_max_indexes(proj.weight.data, 1)]) + if proj.bias is not None: + new_proj.bias.data.copy_(proj.bias.data) + set_module(block, f'attention.output.dense', new_proj) + + fc1 = get_module(block, f'intermediate.dense') + new_fc1 = nn.Linear(fc1.in_features, _f(fc1.out_features), + fc1.bias is not None, fc1.weight.device) + indexes = l1_max_indexes(fc1.weight.data, 0) + new_fc1.weight.data.copy_(fc1.weight.data[indexes]) + if fc1.bias is not None: + new_fc1.bias.data.copy_(fc1.bias.data[indexes]) + set_module(block, f'intermediate.dense', new_fc1) + + fc2 = get_module(block, f'output.dense') + new_fc2 = nn.Linear(_f(fc2.in_features), fc2.out_features, + fc2.bias is not None, fc2.weight.device) + new_fc2.weight.data.copy_(fc2.weight.data[:, l1_max_indexes(fc2.weight.data, 1)]) + if fc2.bias is not None: + new_fc2.bias.data.copy_(fc2.bias.data) + set_module(block, f'output.dense', new_fc2) + + return fm_vit + + def init_md_from_fm_by_reducing_width_with_perf_test(self, fm: nn.Module, reducing_width_ratio: int, + samples: torch.Tensor) -> nn.Module: + fm_size = get_model_size(fm, True) + fm_latency = self._get_model_latency(fm, samples, 20, + get_model_device(fm), 20, False) + + master_dnn = self.init_md_from_fm_by_reducing_width(fm, reducing_width_ratio) + master_dnn_size = get_model_size(master_dnn, True) + logger.debug(f'inited master DNN: {master_dnn}') + master_dnn_latency = self._get_model_latency(master_dnn, samples, 20, + get_model_device(master_dnn), 20, False) + + logger.info(f'init master DNN (w/o FBS yet) by reducing foundation model\'s width (by {reducing_width_ratio:d}x)') + logger.info(f'foundation model ({fm_size:.3f}MB, {fm_latency:.4f}s/sample) -> ' + f'master DNN ({master_dnn_size:.3f}MB, {master_dnn_latency:.4f}s/sample)\n' + f'(model size: ↓ {(fm_size / master_dnn_size):.2f}x, ' + f'latency: ↓ {(fm_latency / master_dnn_latency):.2f}x)') + + return master_dnn + + def _get_model_latency(self, model: torch.nn.Module, model_input_size, sample_num: int, + device: str, warmup_sample_num: int, return_detail=False): + import time + + if isinstance(model_input_size, tuple): + dummy_input = torch.rand(model_input_size).to(device) + else: + dummy_input = model_input_size + + model = model.to(device) + model.eval() + + # warm up + with torch.no_grad(): + for _ in range(warmup_sample_num): + model(**dummy_input) + + infer_time_list = [] + + if device == 'cuda' or 'cuda' in str(device): + with torch.no_grad(): + for _ in range(sample_num): + s, e = torch.cuda.Event(enable_timing=True), torch.cuda.Event(enable_timing=True) + s.record() + model(**dummy_input) + e.record() + torch.cuda.synchronize() + cur_model_infer_time = s.elapsed_time(e) / 1000. + infer_time_list += [cur_model_infer_time] + + else: + with torch.no_grad(): + for _ in range(sample_num): + start = time.time() + model(**dummy_input) + cur_model_infer_time = time.time() - start + infer_time_list += [cur_model_infer_time] + + avg_infer_time = sum(infer_time_list) / sample_num + + if return_detail: + return avg_infer_time, infer_time_list + return avg_infer_time \ No newline at end of file diff --git a/methods/elasticdnn/pipeline/offline/fm_to_md/vit.py b/methods/elasticdnn/pipeline/offline/fm_to_md/vit.py new file mode 100644 index 0000000000000000000000000000000000000000..4bbc255d3468e0b7141a995144fc1c830a7b69ad --- /dev/null +++ b/methods/elasticdnn/pipeline/offline/fm_to_md/vit.py @@ -0,0 +1,130 @@ + +import torch +from torch import nn +from copy import deepcopy + +from .base import FM_to_MD_Util +from utils.common.log import logger +from utils.dl.common.model import set_module, get_module, get_super_module + + +class FM_to_MD_ViT_Util(FM_to_MD_Util): + def init_md_from_fm_by_reducing_width(self, fm: nn.Module, reducing_width_ratio: int) -> nn.Module: + fm_vit = deepcopy(fm) + + def _f(n): + return int(n // reducing_width_ratio) + + # def _rand_indexes(n): + # return torch.randperm(n)[0: int(n // reducing_width_ratio)] + + def l1_max_indexes(p: torch.Tensor, dim=0): + assert dim in [0, 1] + assert p.dim() in [1, 2, 4] + + if dim == 1: + p = p.T + + p_norm = p.abs().contiguous().view(p.size(0), -1).sum(dim=1) + n = p.size(0) + return p_norm.argsort(descending=True)[0: int(n // reducing_width_ratio)].sort()[0] + + # first_attn = True + + for block_i, block in enumerate(fm_vit.blocks): + qkv = block.attn.qkv + + new_qkv = nn.Linear(qkv.in_features, _f(qkv.out_features), + qkv.bias is not None, qkv.weight.device) + indexes = l1_max_indexes(qkv.weight.data, 0) + + new_qkv.weight.data.copy_(qkv.weight.data[indexes]) + if qkv.bias is not None: + new_qkv.bias.data.copy_(qkv.bias.data[indexes]) + set_module(fm_vit, f'blocks.{block_i}.attn.qkv', new_qkv) + + proj = block.attn.proj + new_proj = nn.Linear(_f(proj.in_features), proj.out_features, + proj.bias is not None, proj.weight.device) + new_proj.weight.data.copy_(proj.weight.data[:, l1_max_indexes(proj.weight.data, 1)]) + if proj.bias is not None: + new_proj.bias.data.copy_(proj.bias.data) + set_module(fm_vit, f'blocks.{block_i}.attn.proj', new_proj) + + fc1 = block.mlp.fc1 + new_fc1 = nn.Linear(fc1.in_features, _f(fc1.out_features), + fc1.bias is not None, fc1.weight.device) + indexes = l1_max_indexes(fc1.weight.data, 0) + new_fc1.weight.data.copy_(fc1.weight.data[indexes]) + if fc1.bias is not None: + new_fc1.bias.data.copy_(fc1.bias.data[indexes]) + set_module(fm_vit, f'blocks.{block_i}.mlp.fc1', new_fc1) + + fc2 = block.mlp.fc2 + new_fc2 = nn.Linear(_f(fc2.in_features), fc2.out_features, + fc2.bias is not None, fc2.weight.device) + new_fc2.weight.data.copy_(fc2.weight.data[:, l1_max_indexes(fc2.weight.data, 1)]) + if fc2.bias is not None: + new_fc2.bias.data.copy_(fc2.bias.data) + set_module(fm_vit, f'blocks.{block_i}.mlp.fc2', new_fc2) + # reduce dim_embedding + # if name.endswith('patch_embed.proj'): + # continue + + # new_layer = nn.Conv2d(module.in_channels, _f(module.out_channels), module.kernel_size, module.stride, + # module.padding, module.dilation, module.groups, module.bias is not None, module.padding_mode, + # module.weight.device) + + # rand_indexes = l1_max_indexes(module.weight.data) + # new_layer.weight.data.copy_(module.weight.data[rand_indexes]) + # if new_layer.bias is not None: + # new_layer.bias.data.copy_(module.bias.data[rand_indexes]) + + # fm_vit.cls_token.data = fm_vit.cls_token.data[:, :, rand_indexes] + # fm_vit.pos_embed.data = fm_vit.pos_embed.data[:, :, rand_indexes] + + # elif isinstance(module, nn.Linear): + + # if 'head' in name: + # continue + + # new_layer = nn.Linear(_f(module.in_features), module.out_features, + # module.bias is not None, module.weight.device) + # new_layer.weight.data.copy_(module.weight.data[:, l1_max_indexes(module.weight.data, 1)]) + # if new_layer.bias is not None: + # new_layer.bias.data.copy_(module.bias.data) + # else: + # first_attn = False + # if first_attn: + # first_attn = False + # new_layer = nn.Linear(module.in_features, _f(module.out_features), + # module.bias is not None, module.weight.device) + + # rand_indexes = l1_max_indexes(module.weight.data) + # new_layer.weight.data.copy_(module.weight.data[rand_indexes]) + # if new_layer.bias is not None: + # new_layer.bias.data.copy_(module.bias.data[rand_indexes]) + # else: + # new_layer = nn.Linear(_f(module.in_features), _f(module.out_features), + # module.bias is not None, module.weight.device) + + # rand_indexes = l1_max_indexes(module.weight.data) + # new_layer.weight.data.copy_(module.weight.data[rand_indexes][:, l1_max_indexes(module.weight.data, 1)]) + # if new_layer.bias is not None: + # new_layer.bias.data.copy_(module.bias.data[rand_indexes]) + + # elif isinstance(module, nn.LayerNorm) and ('block' in name or name == 'norm' or name == 'norm.0'): + # new_layer = nn.LayerNorm(_f(module.normalized_shape[0]), eps=module.eps, device=module.weight.device) + # rand_indexes = l1_max_indexes(module.weight.data) + # new_layer.weight.data.copy_(module.weight.data[rand_indexes]) + # new_layer.bias.data.copy_(module.bias.data[rand_indexes]) + + # else: + # continue + + # original_layer_str = str(module) + # set_module(fm_vit, name, new_layer) + # logger.debug(f'set_module, {name}, {new_layer}') + # logger.debug(f'slim {name} from {original_layer_str} to {new_layer}') + + return fm_vit \ No newline at end of file diff --git a/methods/elasticdnn/source-code-line-num.csv b/methods/elasticdnn/source-code-line-num.csv new file mode 100644 index 0000000000000000000000000000000000000000..13aa6b900cdf5c6db049587f57afce39de4d3d4a --- /dev/null +++ b/methods/elasticdnn/source-code-line-num.csv @@ -0,0 +1,34 @@ +file_path,line_num,file_size +/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/methods/elasticdnn/model/test.py,138,4455 +/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/methods/elasticdnn/model/base.py,139,5797 +/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/methods/elasticdnn/model/vit.py,343,16707 +/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/methods/elasticdnn/model/vit_new.py,341,15609 +/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/methods/elasticdnn/model/clip.py,316,14598 +/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/methods/elasticdnn/model/vilt.py,359,15983 +/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/methods/elasticdnn/model/bert.py,359,15967 +/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/methods/elasticdnn/model/__init__.py,4,98 +/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/methods/elasticdnn/model/cnn.py,147,6260 +/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/methods/elasticdnn/api/algs/fm_lora.py,146,6086 +/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/methods/elasticdnn/api/algs/md_pretraining_index_v2_train_index_and_md.py,529,24650 +/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/methods/elasticdnn/api/algs/md_pretraining_wo_fbs.py,381,17388 +/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/methods/elasticdnn/api/algs/md_pretraining_index_bk_too_slow.py,427,19855 +/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/methods/elasticdnn/api/algs/md_pretraining_index_v1_train_index_and_md.py,501,23379 +/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/methods/elasticdnn/api/algs/md_pretraining_wo_fbs_clip_debug.py,412,19146 +/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/methods/elasticdnn/api/algs/md_pretraining_bk.py,292,13039 +/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/methods/elasticdnn/api/algs/md_pretraining_index.py,322,15329 +/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/methods/elasticdnn/api/algs/md_pretraining_v1.py,316,14640 +/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/methods/elasticdnn/api/algs/md_pretraining_w_fbs.py,192,8653 +/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/methods/elasticdnn/api/online_model_v2.py,269,11807 +/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/methods/elasticdnn/api/model.py,821,29849 +/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/methods/elasticdnn/api/online_model.py,246,10834 +/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/methods/elasticdnn/pipeline/offline/fm_lora/base.py,36,932 +/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/methods/elasticdnn/pipeline/offline/fm_lora/vit.py,79,2484 +/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/methods/elasticdnn/pipeline/offline/fm_lora/clip.py,86,2780 +/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/methods/elasticdnn/pipeline/offline/fm_lora/vilt.py,91,2768 +/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/methods/elasticdnn/pipeline/offline/fm_lora/bert.py,81,2422 +/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/methods/elasticdnn/pipeline/offline/fm_to_md/test.py,28,1180 +/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/methods/elasticdnn/pipeline/offline/fm_to_md/base.py,48,2308 +/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/methods/elasticdnn/pipeline/offline/fm_to_md/vit.py,130,6544 +/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/methods/elasticdnn/pipeline/offline/fm_to_md/clip.py,672,30837 +/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/methods/elasticdnn/pipeline/offline/fm_to_md/vilt.py,216,9424 +/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/methods/elasticdnn/pipeline/offline/fm_to_md/bert.py,279,13316 diff --git a/methods/feat_align/__pycache__/main.cpython-38.pyc b/methods/feat_align/__pycache__/main.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a49b5a7cbc3341b81379d0fc4b8b65d4220599e1 Binary files /dev/null and b/methods/feat_align/__pycache__/main.cpython-38.pyc differ diff --git a/methods/feat_align/__pycache__/mmd.cpython-38.pyc b/methods/feat_align/__pycache__/mmd.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ca610377d4a2b998c834059df0b3dfeeec46b275 Binary files /dev/null and b/methods/feat_align/__pycache__/mmd.cpython-38.pyc differ diff --git a/methods/feat_align/main.py b/methods/feat_align/main.py new file mode 100644 index 0000000000000000000000000000000000000000..8d0a6a69f214e0a5eb157aff8d36e0f6ba9f2ff7 --- /dev/null +++ b/methods/feat_align/main.py @@ -0,0 +1,188 @@ +from typing import Any, Dict, List +from schema import Schema +from data import Scenario, MergedDataset +from methods.base.alg import BaseAlg +from methods.base.model import BaseModel +from data import build_dataloader +import torch.optim +import tqdm +import os +import time +from abc import abstractmethod +import matplotlib.pyplot as plt + + +class OnlineFeatAlignModel(BaseModel): + def get_required_model_components(self) -> List[str]: + return ['main'] + + @abstractmethod + def get_feature_hook(self): + pass + + @abstractmethod + def forward_to_get_task_loss(self, x, y): + pass + + @abstractmethod + def get_trained_params(self): + pass + + @abstractmethod + def get_mmd_loss(self, f1, f2): + pass + + +class FeatAlignAlg(BaseAlg): + def get_required_models_schema(self) -> Schema: + return Schema({ + 'main': OnlineFeatAlignModel + }) + + def get_required_hyp_schema(self) -> Schema: + return Schema({ + 'train_batch_size': int, + 'val_batch_size': int, + 'num_workers': int, + 'optimizer': str, + 'optimizer_args': dict, + 'scheduler': str, + 'scheduler_args': dict, + 'num_iters': int, + 'val_freq': int, + 'feat_align_loss_weight': float + }) + + def run(self, scenario: Scenario, hyps: Dict) -> Dict[str, Any]: + super().run(scenario, hyps) + + assert isinstance(self.models['main'], OnlineFeatAlignModel) # for auto completion + + cur_domain_name = scenario.target_domains_order[scenario.cur_domain_index] + datasets_for_training = scenario.get_online_cur_domain_datasets_for_training() + train_dataset = datasets_for_training[cur_domain_name]['train'] + val_dataset = datasets_for_training[cur_domain_name]['val'] + datasets_for_inference = scenario.get_online_cur_domain_datasets_for_inference() + test_dataset = datasets_for_inference + + train_loader = iter(build_dataloader(train_dataset, hyps['train_batch_size'], hyps['num_workers'], + True, None)) + test_loader = build_dataloader(test_dataset, hyps['val_batch_size'], hyps['num_workers'], + False, False) + + source_datasets = [d['train'] for n, d in datasets_for_training.items() if n != cur_domain_name] + source_dataset = MergedDataset(source_datasets) + source_train_loader = iter(build_dataloader(source_dataset, hyps['train_batch_size'], hyps['num_workers'], + True, None)) + + # 1. generate surrogate DNN + # for n, m in self.models['main'].models_dict['md'].named_modules(): + # if isinstance(m, nn.Linear): + # m.reset_parameters() + # from utils.dl.common.model import set_module + + # for n, m in self.models['main'].models_dict['md'].named_modules(): + # if m.__class__.__name__ == 'KTakesAll': + # set_module(self.models['main'].models_dict['md'], n, KTakesAll(0.5)) + # self.models['main'].set_sd_sparsity(hyps['sd_sparsity']) + device = self.models['main'].device + # surrogate_dnn = self.models['main'].generate_sd_by_target_samples(next(train_loader)[0].to(device)) + # self.models['sd'] = surrogate_dnn + + # 2. train surrogate DNN + # TODO: train only a part of filters + trained_params = self.models['main'].get_trained_params() + optimizer = torch.optim.__dict__[hyps['optimizer']](trained_params, **hyps['optimizer_args']) + if hyps['scheduler'] != '': + scheduler = torch.optim.lr_scheduler.__dict__[hyps['scheduler']](optimizer, **hyps['scheduler_args']) + else: + scheduler = None + + pbar = tqdm.tqdm(range(hyps['num_iters']), dynamic_ncols=True, desc='da...') + task_losses, mmd_losses = [], [] + accs = [] + + total_train_time = 0. + + feature_hook = self.models['main'].get_feature_hook() + + for iter_index in pbar: + + if iter_index % hyps['val_freq'] == 0: + from data import split_dataset + cur_test_batch_dataset = split_dataset(test_dataset, hyps['val_batch_size'], iter_index)[0] + cur_test_batch_dataloader = build_dataloader(cur_test_batch_dataset, hyps['train_batch_size'], hyps['num_workers'], False, False) + cur_acc = self.models['main'].get_accuracy(cur_test_batch_dataloader) + accs += [{ + 'iter': iter_index, + 'acc': cur_acc + }] + + cur_start_time = time.time() + + self.models['main'].to_train_mode() + + x, _ = next(train_loader) + + if isinstance(x, dict): + for k, v in x.items(): + if isinstance(v, torch.Tensor): + x[k] = v.to(device) + else: + x = x.to(device) + + source_x, source_y = next(source_train_loader) + + if isinstance(source_x, dict): + for k, v in source_x.items(): + if isinstance(v, torch.Tensor): + source_x[k] = v.to(device) + source_y = source_y.to(device) + else: + source_x, source_y = source_x.to(device), source_y.to(device) + + task_loss = self.models['main'].forward_to_get_task_loss(source_x, source_y) + source_features = feature_hook.input + + self.models['main'].infer(x) + target_features = feature_hook.input + + mmd_loss = hyps['feat_align_loss_weight'] * self.models['main'].get_mmd_loss(source_features, target_features) + + loss = task_loss + mmd_loss + + optimizer.zero_grad() + loss.backward() + optimizer.step() + if scheduler is not None: + scheduler.step() + + pbar.set_description(f'da... | cur_acc: {cur_acc:.4f}, task_loss: {task_loss:.6f}, mmd_loss: {mmd_loss:.6f}') + task_losses += [float(task_loss.cpu().item())] + mmd_losses += [float(mmd_loss.cpu().item())] + + total_train_time += time.time() - cur_start_time + + feature_hook.remove() + + time_usage = total_train_time + + plt.plot(task_losses, label='task') + plt.plot(mmd_losses, label='mmd') + plt.xlabel('iteration') + plt.ylabel('loss') + plt.savefig(os.path.join(self.res_save_dir, 'loss.png')) + plt.clf() + + cur_test_batch_dataset = split_dataset(test_dataset, hyps['train_batch_size'], iter_index + 1)[0] + cur_test_batch_dataloader = build_dataloader(cur_test_batch_dataset, len(cur_test_batch_dataset), hyps['num_workers'], False, False) + cur_acc = self.models['main'].get_accuracy(cur_test_batch_dataloader) + accs += [{ + 'iter': iter_index + 1, + 'acc': cur_acc + }] + + return { + 'accs': accs, + 'time': time_usage + }, self.models \ No newline at end of file diff --git a/methods/feat_align/main_partial.py b/methods/feat_align/main_partial.py new file mode 100644 index 0000000000000000000000000000000000000000..853ad67ade7eecf12ff89da616e75eeaca14477c --- /dev/null +++ b/methods/feat_align/main_partial.py @@ -0,0 +1,325 @@ +from typing import Any, Dict, List +from schema import Schema +from data import Scenario, MergedDataset +from methods.base.alg import BaseAlg +from methods.base.model import BaseModel +from data import build_dataloader +import torch.optim +import tqdm +import os +import time +from abc import abstractmethod +import matplotlib.pyplot as plt +from copy import deepcopy +from torch import nn + +import torch.optim + + +def tent_as_detector(online_model, x, num_iters=1, lr=1e-4, l1_wd=0., strategy='ours'): + model = online_model.models_dict['main'] + before_model = deepcopy(model) + + # from methods.tent import tent + + optimizer = torch.optim.SGD( + model.parameters(), lr=lr, weight_decay=l1_wd) + + # from .tent import configure_model, forward_and_adapt + # configure_model(model) + output = online_model.infer(x) + entropy = online_model.get_output_entropy(output).mean() + + entropy.backward() + # for _ in range(num_iters): + # forward_and_adapt(x, model, optimizer) + + # entropy_loss = model. + + filters_sen_info = {} + + last_conv_name = None + for (name, m1), m2 in zip(model.named_modules(), before_model.modules()): + if isinstance(m1, nn.Linear): + last_conv_name = name + + if not isinstance(m1, nn.LayerNorm): + continue + + with torch.no_grad(): + features_weight_diff = ((m1.weight.data - m2.weight.data).abs()) + features_bias_diff = ((m1.bias.data - m2.bias.data).abs()) + + features_diff = features_weight_diff + features_bias_diff + + features_diff_order = features_diff.argsort(descending=False) + + if strategy == 'ours': + untrained_filters_index = features_diff_order[: int(len(features_diff) * 0.8)] + elif strategy == 'random': + untrained_filters_index = torch.randperm(len(features_diff))[: int(len(features_diff) * 0.8)] + elif strategy == 'inversed_ours': + untrained_filters_index = features_diff_order.flip(0)[: int(len(features_diff) * 0.8)] + elif strategy == 'none': + untrained_filters_index = None + + filters_sen_info[name] = dict(untrained_filters_index=untrained_filters_index, conv_name=last_conv_name) + + return filters_sen_info + + +class SGDF(torch.optim.SGD): + + @torch.no_grad() + def step(self, p_names, conv_filters_sen_info, filters_sen_info, closure=None): + """Performs a single optimization step. + + Arguments: + closure (callable, optional): A closure that reevaluates the model + and returns the loss. + """ + loss = None + if closure is not None: + with torch.enable_grad(): + loss = closure() + + for group in self.param_groups: + weight_decay = group['weight_decay'] + momentum = group['momentum'] + dampening = group['dampening'] + nesterov = group['nesterov'] + + # assert len([i for i in model.named_parameters()]) == len([j for j in group['params']]) + + for name, p in zip(p_names, group['params']): + if p.grad is None: + continue + + layer_name = '.'.join(name.split('.')[0:-1]) + if layer_name in filters_sen_info.keys(): + untrained_filters_index = filters_sen_info[layer_name]['untrained_filters_index'] + elif layer_name in conv_filters_sen_info.keys(): + untrained_filters_index = conv_filters_sen_info[layer_name]['untrained_filters_index'] + else: + untrained_filters_index = [] + + d_p = p.grad + if weight_decay != 0: + d_p = d_p.add(p, alpha=weight_decay) + if momentum != 0: + param_state = self.state[p] + if 'momentum_buffer' not in param_state: + buf = param_state['momentum_buffer'] = torch.clone(d_p).detach() + else: + buf = param_state['momentum_buffer'] + buf.mul_(momentum).add_(d_p, alpha=1 - dampening) + if nesterov: + d_p = d_p.add(buf, alpha=momentum) + else: + d_p = buf + + try: + d_p[untrained_filters_index] = 0. + p.add_(d_p, alpha=-group['lr']) + except Exception as e: + print('SGDF error', name) + + return loss + + +class OnlineFeatAlignModel(BaseModel): + def get_required_model_components(self) -> List[str]: + return ['main'] + + @abstractmethod + def get_feature_hook(self): + pass + + @abstractmethod + def forward_to_get_task_loss(self, x, y): + pass + + @abstractmethod + def get_trained_params(self): + pass + + @abstractmethod + def get_mmd_loss(self, f1, f2): + pass + + @abstractmethod + def get_output_entropy(self, output): + pass + + +class FeatAlignAlg(BaseAlg): + def get_required_models_schema(self) -> Schema: + return Schema({ + 'main': OnlineFeatAlignModel + }) + + def get_required_hyp_schema(self) -> Schema: + return Schema({ + 'train_batch_size': int, + 'val_batch_size': int, + 'num_workers': int, + 'optimizer': str, + 'optimizer_args': dict, + 'scheduler': str, + 'scheduler_args': dict, + 'num_iters': int, + 'val_freq': int, + 'feat_align_loss_weight': float, + 'trained_neuron_selection_strategy': str + }) + + def run(self, scenario: Scenario, hyps: Dict) -> Dict[str, Any]: + super().run(scenario, hyps) + + assert isinstance(self.models['main'], OnlineFeatAlignModel) # for auto completion + + cur_domain_name = scenario.target_domains_order[scenario.cur_domain_index] + datasets_for_training = scenario.get_online_cur_domain_datasets_for_training() + train_dataset = datasets_for_training[cur_domain_name]['train'] + val_dataset = datasets_for_training[cur_domain_name]['val'] + datasets_for_inference = scenario.get_online_cur_domain_datasets_for_inference() + test_dataset = datasets_for_inference + + train_loader = iter(build_dataloader(train_dataset, hyps['train_batch_size'], hyps['num_workers'], + True, None)) + test_loader = build_dataloader(test_dataset, hyps['val_batch_size'], hyps['num_workers'], + False, False) + + source_datasets = [d['train'] for n, d in datasets_for_training.items() if n != cur_domain_name] + source_dataset = MergedDataset(source_datasets) + source_train_loader = iter(build_dataloader(source_dataset, hyps['train_batch_size'], hyps['num_workers'], + True, None)) + + # 1. generate surrogate DNN + # for n, m in self.models['main'].models_dict['md'].named_modules(): + # if isinstance(m, nn.Linear): + # m.reset_parameters() + # from utils.dl.common.model import set_module + + # for n, m in self.models['main'].models_dict['md'].named_modules(): + # if m.__class__.__name__ == 'KTakesAll': + # set_module(self.models['main'].models_dict['md'], n, KTakesAll(0.5)) + # self.models['main'].set_sd_sparsity(hyps['sd_sparsity']) + device = self.models['main'].device + # surrogate_dnn = self.models['main'].generate_sd_by_target_samples(next(train_loader)[0].to(device)) + # self.models['sd'] = surrogate_dnn + + # 2. train surrogate DNN + # TODO: train only a part of filters + trained_params, p_name = self.models['main'].get_trained_params() + + # optimizer = torch.optim.__dict__[hyps['optimizer']](trained_params, **hyps['optimizer_args']) + + optimizer = SGDF(trained_params, **hyps['optimizer_args']) + + if hyps['scheduler'] != '': + scheduler = torch.optim.lr_scheduler.__dict__[hyps['scheduler']](optimizer, **hyps['scheduler_args']) + else: + scheduler = None + + pbar = tqdm.tqdm(range(hyps['num_iters']), dynamic_ncols=True, desc='da...') + task_losses, mmd_losses = [], [] + accs = [] + + + x, _ = next(train_loader) + if isinstance(x, dict): + for k, v in x.items(): + if isinstance(v, torch.Tensor): + x[k] = v.to(device) + else: + x = x.to(device) + filters_sen_info = tent_as_detector(self.models['main'], x, strategy=hyps['trained_neuron_selection_strategy']) + conv_filters_sen_info = {v['conv_name']: v for _, v in filters_sen_info.items()} + + + total_train_time = 0. + + feature_hook = self.models['main'].get_feature_hook() + + for iter_index in pbar: + + if iter_index % hyps['val_freq'] == 0: + from data import split_dataset + cur_test_batch_dataset = split_dataset(test_dataset, hyps['val_batch_size'], iter_index)[0] + cur_test_batch_dataloader = build_dataloader(cur_test_batch_dataset, hyps['train_batch_size'], hyps['num_workers'], False, False) + cur_acc = self.models['main'].get_accuracy(cur_test_batch_dataloader) + accs += [{ + 'iter': iter_index, + 'acc': cur_acc + }] + + cur_start_time = time.time() + + self.models['main'].to_train_mode() + + x, _ = next(train_loader) + + if isinstance(x, dict): + for k, v in x.items(): + if isinstance(v, torch.Tensor): + x[k] = v.to(device) + else: + x = x.to(device) + + source_x, source_y = next(source_train_loader) + + if isinstance(source_x, dict): + for k, v in source_x.items(): + if isinstance(v, torch.Tensor): + source_x[k] = v.to(device) + source_y = source_y.to(device) + else: + source_x, source_y = source_x.to(device), source_y.to(device) + + task_loss = self.models['main'].forward_to_get_task_loss(source_x, source_y) + source_features = feature_hook.input + + self.models['main'].infer(x) + target_features = feature_hook.input + + mmd_loss = hyps['feat_align_loss_weight'] * self.models['main'].get_mmd_loss(source_features, target_features) + + loss = task_loss + mmd_loss + + optimizer.zero_grad() + loss.backward() + # optimizer.step() + optimizer.step(p_name, conv_filters_sen_info, filters_sen_info) + if scheduler is not None: + scheduler.step() + + pbar.set_description(f'da... | cur_acc: {cur_acc:.4f}, task_loss: {task_loss:.6f}, mmd_loss: {mmd_loss:.6f}') + task_losses += [float(task_loss.cpu().item())] + mmd_losses += [float(mmd_loss.cpu().item())] + + total_train_time += time.time() - cur_start_time + + feature_hook.remove() + + time_usage = total_train_time + + plt.plot(task_losses, label='task') + plt.plot(mmd_losses, label='mmd') + plt.xlabel('iteration') + plt.ylabel('loss') + plt.savefig(os.path.join(self.res_save_dir, 'loss.png')) + plt.clf() + + cur_test_batch_dataset = split_dataset(test_dataset, hyps['train_batch_size'], iter_index + 1)[0] + cur_test_batch_dataloader = build_dataloader(cur_test_batch_dataset, len(cur_test_batch_dataset), hyps['num_workers'], False, False) + cur_acc = self.models['main'].get_accuracy(cur_test_batch_dataloader) + accs += [{ + 'iter': iter_index + 1, + 'acc': cur_acc + }] + + return { + 'accs': accs, + 'time': time_usage + }, self.models \ No newline at end of file diff --git a/methods/feat_align/mmd.py b/methods/feat_align/mmd.py new file mode 100644 index 0000000000000000000000000000000000000000..647e5600e17cc1d62d2376b28346f5c6e0abc098 --- /dev/null +++ b/methods/feat_align/mmd.py @@ -0,0 +1,62 @@ +import torch +import random + +# 参考 +# 最大化均值差异MMD与Numpy/Tensorflow/Pytorch各类代码实现 - 小蔡叔叔开方舟的文章 - 知乎 +# https://zhuanlan.zhihu.com/p/461656480 + +def guassian_kernel(source, target, kernel_mul=2.0, kernel_num=5, fix_sigma=None): + ''' + 将源域数据和目标域数据转化为核矩阵，即上文中的K + Params: + source: 源域数据（n * len(x)) + target: 目标域数据（m * len(y)) + kernel_mul: + kernel_num: 取不同高斯核的数量 + fix_sigma: 不同高斯核的sigma值 + Return: + sum(kernel_val): 多个核矩阵之和 + ''' + n_samples = int(source.size()[0])+int(target.size()[0])# 求矩阵的行数，一般source和target的尺度是一样的，这样便于计算 + total = torch.cat([source, target], dim=0) #将source,target按列方向合并 + #将total复制（n+m）份 + total0 = total.unsqueeze(0).expand(int(total.size(0)), int(total.size(0)), int(total.size(1))) + #将total的每一行都复制成（n+m）行，即每个数据都扩展成（n+m）份 + total1 = total.unsqueeze(1).expand(int(total.size(0)), int(total.size(0)), int(total.size(1))) + #求任意两个数据之间的和，得到的矩阵中坐标（i,j）代表total中第i行数据和第j行数据之间的l2 distance(i==j时为0） + L2_distance = ((total0-total1)**2).sum(2) + #调整高斯核函数的sigma值 + if fix_sigma: + bandwidth = fix_sigma + else: + bandwidth = torch.sum(L2_distance.data) / (n_samples**2-n_samples) + #以fix_sigma为中值，以kernel_mul为倍数取kernel_num个bandwidth值（比如fix_sigma为1时，得到[0.25,0.5,1,2,4] + bandwidth /= kernel_mul ** (kernel_num // 2) + bandwidth_list = [bandwidth * (kernel_mul**i) for i in range(kernel_num)] + #高斯核函数的数学表达式 + kernel_val = [torch.exp(-L2_distance / bandwidth_temp) for bandwidth_temp in bandwidth_list] + #得到最终的核矩阵 + return sum(kernel_val)#/len(kernel_val) + +def mmd_rbf(source, target, kernel_mul=2.0, kernel_num=5, fix_sigma=None): + ''' + 计算源域数据和目标域数据的MMD距离 + Params: + source: 源域数据（n * len(x)) + target: 目标域数据（m * len(y)) + kernel_mul: + kernel_num: 取不同高斯核的数量 + fix_sigma: 不同高斯核的sigma值 + Return: + loss: MMD loss + ''' + batch_size = int(source.size()[0])#一般默认为源域和目标域的batchsize相同 + kernels = guassian_kernel(source, target, + kernel_mul=kernel_mul, kernel_num=kernel_num, fix_sigma=fix_sigma) + #根据式（3）将核矩阵分成4部分 + XX = kernels[:batch_size, :batch_size] + YY = kernels[batch_size:, batch_size:] + XY = kernels[:batch_size, batch_size:] + YX = kernels[batch_size:, :batch_size] + loss = torch.mean(XX + YY - XY -YX) + return loss#因为一般都是n==m，所以L矩阵一般不加入计算 \ No newline at end of file diff --git a/methods/method_utils/data.py b/methods/method_utils/data.py new file mode 100644 index 0000000000000000000000000000000000000000..e429e9a2a27c2607bbb8dcdad4d5b1613bd6f656 --- /dev/null +++ b/methods/method_utils/data.py @@ -0,0 +1,21 @@ +import torch +from torch import nn + + +def to(x, device): + if isinstance(x, dict): + for k, v in x.items(): + if isinstance(v, torch.Tensor): + x[k] = v.to(device) + else: + x = x.to(device) + return x + + +def get_cur_acc(testset, hyps, model, shuffle, iter_index): + from data import split_dataset, build_dataloader + cur_test_batch_dataset = split_dataset(testset, hyps['val_batch_size'], iter_index)[0] + cur_test_batch_dataloader = build_dataloader(cur_test_batch_dataset, hyps['train_batch_size'], hyps['num_workers'], False, shuffle) + cur_acc = model.get_accuracy(cur_test_batch_dataloader) + return cur_acc + \ No newline at end of file diff --git a/new_impl/cv/base/__pycache__/alg.cpython-38.pyc b/new_impl/cv/base/__pycache__/alg.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e9610f3c58ee413645d0954d0c65d34fcbcdac2b Binary files /dev/null and b/new_impl/cv/base/__pycache__/alg.cpython-38.pyc differ diff --git a/new_impl/cv/base/__pycache__/model.cpython-38.pyc b/new_impl/cv/base/__pycache__/model.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0d1a4d6b2d533aa1dcd8bf6c9bc3ac98847061ce Binary files /dev/null and b/new_impl/cv/base/__pycache__/model.cpython-38.pyc differ diff --git a/new_impl/cv/base/alg.py b/new_impl/cv/base/alg.py new file mode 100644 index 0000000000000000000000000000000000000000..ce3e6a47e2af7ae6f578192dab2a6d7943dd0e3a --- /dev/null +++ b/new_impl/cv/base/alg.py @@ -0,0 +1,54 @@ +from typing import Dict, Any +from torch import nn +from data.datasets.ab_dataset import ABDataset +from abc import ABC, abstractmethod +from utils.common.log import logger +import json +import os + +from utils.common.others import backup_key_codes +from .model import BaseModel +from data import Scenario +from schema import Schema +from utils.common.data_record import write_json + + +class BaseAlg(ABC): + + def __init__(self, models: Dict[str, BaseModel], res_save_dir): + self.models = models + self.res_save_dir = res_save_dir + self.get_required_models_schema().validate(models) + + os.makedirs(res_save_dir) + logger.info(f'[alg] init alg: {self.__class__.__name__}, res saved in {res_save_dir}') + + @abstractmethod + def get_required_models_schema(self) -> Schema: + raise NotImplementedError + + @abstractmethod + def get_required_hyp_schema(self) -> Schema: + raise NotImplementedError + + @abstractmethod + def run(self, + scenario: Scenario, + hyps: Dict) -> Dict[str, Any]: + """ + return metrics + """ + + self.get_required_hyp_schema().validate(hyps) + + try: + write_json(os.path.join(self.res_save_dir, 'hyps.json'), hyps, ensure_obj_serializable=True) + except: + with open(os.path.join(self.res_save_dir, 'hyps.txt'), 'w') as f: + f.write(str(hyps)) + + write_json(os.path.join(self.res_save_dir, 'scenario.json'), scenario.to_json()) + + logger.info(f'[alg] alg {self.__class__.__name__} start running') + + backup_key_codes(os.path.join(self.res_save_dir, 'backup_codes')) \ No newline at end of file diff --git a/new_impl/cv/base/model.py b/new_impl/cv/base/model.py new file mode 100644 index 0000000000000000000000000000000000000000..c0c7997e6fe3dc5f0c51377f365fbf993f5f060c --- /dev/null +++ b/new_impl/cv/base/model.py @@ -0,0 +1,73 @@ +import torch +from abc import ABC, abstractmethod +from utils.common.file import ensure_dir +from utils.common.log import logger +import time +from typing import List + + +class BaseModel(ABC): + def __init__(self, + name: str, + models_dict_path: str, + device: str): + + self.name = name + self.models_dict_path = models_dict_path + self.models_dict = torch.load(models_dict_path, map_location=device) + self.device = device + + assert set(self.get_required_model_components()) <= set(list(self.models_dict.keys())) + + self.to(device) + + logger.info(f'[model] init model: {dict(name=name, components=self.get_required_model_components())}') + logger.debug(self.models_dict) + + @abstractmethod + def get_required_model_components(self) -> List[str]: + pass + + @abstractmethod + def get_accuracy(self, test_loader, *args, **kwargs): + pass + + @abstractmethod + def infer(self, x, *args, **kwargs): + pass + + def save_model(self, p: str): + logger.debug(f'[model] save model: {self.name}') + ensure_dir(p) + torch.save(self.models_dict, p) + + def load_model(self, p: str): + logger.debug(f'[model] load model: {self.name}, from {p}') + self.models_dict = torch.load(p, map_location=self.device) + + def to(self, device): + logger.debug(f'[model] to device: {device}') + for k, v in self.models_dict.items(): + try: + self.models_dict[k] = v.to(device) + except Exception as e: + pass + + def to_eval_mode(self, verbose=False): + if verbose: + logger.info(f'[model] to eval mode') + for k, v in self.models_dict.items(): + try: + self.models_dict[k].eval() + except Exception as e: + pass + + def to_train_mode(self, verbose=False): + if verbose: + logger.info(f'[model] to train mode') + for k, v in self.models_dict.items(): + try: + self.models_dict[k].train() + except Exception as e: + pass + \ No newline at end of file diff --git a/new_impl/cv/beit/beit.py b/new_impl/cv/beit/beit.py new file mode 100644 index 0000000000000000000000000000000000000000..a4d21233351d5ad50bff077bddd0aa9b7e9db4f4 --- /dev/null +++ b/new_impl/cv/beit/beit.py @@ -0,0 +1,1141 @@ +from transformers import BlipForQuestionAnswering, BlipConfig,BlipModel +import torch +from torch import nn +from abc import ABC, abstractmethod +from copy import deepcopy +from typing import Optional, Union +from einops import rearrange, repeat +from einops.layers.torch import Rearrange +import tqdm + +from utils.dl.common.model import get_model_device, get_model_latency, get_model_size, set_module +from utils.dl.common.model import set_module, get_module, get_super_module +from utils.common.log import logger +from new_impl.cv.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util, LoRA +from transformers.models.blip.modeling_blip import BlipAttention +#from transformers.models.blip.modeling_blip_text import BlipTextSelfAttention,BlipTextAttention,BlipTextSelfOutput +from transformers.models.beit.modeling_beit import BeitSelfAttention,BeitConfig +from new_impl.cv.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from new_impl.cv.elasticdnn.model.base import Abs, KTakesAll, ElasticDNNUtil, Layer_WrappedWithFBS + +from typing import Optional, Tuple +import math + +# def blip(num_classes): +# model = BlipForQuestionAnswering.from_pretrained('new_impl/mm/Vis_bert/QuestionAnswering/VisBert_pretrained') +# # linear = model.text_decoder.cls.predictions.decoder +# # new_linear = nn.Linear(linear.in_features,30524,bias = True) +# # set_module(model,'text_decoder.cls.predictions.decoder',new_linear) +# return model +# def blip(num_classes): +# model = BlipForQuestionAnswering.from_pretrained('new_impl/mm/Vis_bert/QuestionAnswering/VisBert_pretrained') +# linear = model.text_decoder.cls.predictions.decoder +# new_linear = nn.Linear(linear.in_features,num_classes,bias = True) +# set_module(model,'text_decoder.cls.predictions.decoder',new_linear) +# return model +# class dinat(nn.Module): +# def __init__(self,num_classes): +# super(dinat,self).__init__() +# self.dinat = DinatModel.from_pretrained('shi-labs/dinat-mini-in1k-224') +# self.classifier = nn.Linear(768,num_classes) + +# def forward(self,**sample): +# output = self.dinat(**sample)[-1]#output the last hidden +# output = self.classifier(output[1]) +# return output + + +class ToQKV_WrappedWithLoRA(nn.Module): + def __init__(self, fc: nn.Linear, ab_r: int): + super(ToQKV_WrappedWithLoRA, self).__init__() + + self.fc = fc + self.ab = self.create_ab_as_linear(fc.weight.data, ab_r) + + def create_ab_as_linear(self, fc_weight: torch.Tensor, ab_r: int): + res = nn.Sequential( + LoRA(fc_weight.size(1), fc_weight.size(0) // ab_r, bias=False), + LoRA(fc_weight.size(0) // ab_r, fc_weight.size(0), bias=False) + ).to(fc_weight.device) + nn.init.kaiming_uniform_(res[0].weight, a=5 ** 0.5) + nn.init.zeros_(res[1].weight) + return res + + def forward(self, x): + x1 = self.fc(x) + x2 = self.ab(x) + return x1 + x2 + + +class FMLoRA_beit_Util(FMLoRA_Util): + + @torch.no_grad() + def add_lora_ab_to_fm(self, fm: nn.Module, ab_r: int, samples: torch.Tensor): + fm.eval() + + # print(samples) + # for k, v in samples.items(): + # if isinstance(v, torch.Tensor): + # samples[k] = v.to(get_model_device(fm)) + + #o1 = fm.generate(**samples) + o1 = fm(samples) + for name, module in fm.named_modules(): + if name.endswith(('query', 'key', 'value')): + set_module(fm, name, ToQKV_WrappedWithLoRA(module, ab_r)) + elif name.endswith('.qkv'): + set_module(fm, name, ToQKV_WrappedWithLoRA(module, ab_r)) + + + #o2 = fm.generate(**samples) + o2 = fm(samples) + if isinstance(o1, tuple): + o1 = o1[-1] + o2 = o2[-1] + output_diff = ((o1.logits - o2.logits) ** 2).sum() + assert output_diff < 1e-5 + return fm + + @torch.no_grad() + def absorb_lora_and_recover_net_structure(self, fm: nn.Module, samples: torch.Tensor): + fm.eval() + # print('absorb lora before') + + # for k, v in samples.items(): + # if isinstance(v, torch.Tensor): + # samples[k] = v.to(get_model_device(fm)) + + o1 = fm(samples) + + for name, module in fm.named_modules(): + if not isinstance(module, ToQKV_WrappedWithLoRA): + continue + + fc = module.fc + ab = module.ab + + fc.weight.add_(ab[1].weight @ ab[0].weight) + + set_module(fm, name, fc) + + # print('absorb lora after') + o2 = fm(samples) + + if isinstance(o1, tuple): + o1 = o1[-1] + o2 = o2[-1] + output_diff = ((o1.logits - o2.logits) ** 2).sum() + assert output_diff < 1e-6, output_diff + + return fm + + +####Here start with Fbs + +# class blipTextAttentionPrunable(BlipTextSelfAttention): +# def __init__(self,is_cross_attention): +# config = BlipConfig.from_pretrained('new_impl/mm/Vis_bert/QuestionAnswering/VisBert_pretrained') +# super(blipTextAttentionPrunable,self).__init__(config.text_config,is_cross_attention) + +# def save_attn_gradients(self, attn_gradients): +# self.attn_gradients = attn_gradients + +# def get_attn_gradients(self): +# return self.attn_gradients + +# def save_attention_map(self, attention_map): +# self.attention_map = attention_map + +# def get_attention_map(self): +# return self.attention_map + +# def transpose_for_scores(self, x): +# new_x_shape = x.size()[:-1] + (self.num_attention_heads, -1) +# x = x.view(*new_x_shape) +# return x.permute(0, 2, 1, 3) + +# def forward( +# self, +# hidden_states: torch.Tensor, +# attention_mask: Optional[torch.FloatTensor] = None, +# head_mask: Optional[torch.FloatTensor] = None, +# encoder_hidden_states: Optional[torch.FloatTensor] = None, +# encoder_attention_mask: Optional[torch.FloatTensor] = None, +# past_key_value: Optional[Tuple[Tuple[torch.FloatTensor]]] = None, +# output_attentions: Optional[bool] = False, +# ) -> Tuple[torch.Tensor]: +# mixed_query_layer = self.query(hidden_states) + +# # If this is instantiated as a cross-attention module, the keys +# # and values come from an encoder; the attention mask needs to be +# # such that the encoder's padding tokens are not attended to. +# is_cross_attention = encoder_hidden_states is not None + +# if is_cross_attention: +# key_layer = self.transpose_for_scores(self.key(encoder_hidden_states)) +# value_layer = self.transpose_for_scores(self.value(encoder_hidden_states)) +# attention_mask = encoder_attention_mask +# elif past_key_value is not None: +# key_layer = self.transpose_for_scores(self.key(hidden_states)) +# value_layer = self.transpose_for_scores(self.value(hidden_states)) +# key_layer = torch.cat([past_key_value[0], key_layer], dim=2) +# value_layer = torch.cat([past_key_value[1], value_layer], dim=2) +# else: +# key_layer = self.transpose_for_scores(self.key(hidden_states)) +# value_layer = self.transpose_for_scores(self.value(hidden_states)) + +# query_layer = self.transpose_for_scores(mixed_query_layer) + +# past_key_value = (key_layer, value_layer) + +# # Take the dot product between "query" and "key" to get the raw attention scores. +# attention_scores = torch.matmul(query_layer, key_layer.transpose(-1, -2)) + +# if self.position_embedding_type == "relative_key" or self.position_embedding_type == "relative_key_query": +# seq_length = hidden_states.size()[1] +# position_ids_l = torch.arange(seq_length, dtype=torch.long, device=hidden_states.device).view(-1, 1) +# position_ids_r = torch.arange(seq_length, dtype=torch.long, device=hidden_states.device).view(1, -1) +# distance = position_ids_l - position_ids_r +# positional_embedding = self.distance_embedding(distance + self.max_position_embeddings - 1) +# positional_embedding = positional_embedding.to(dtype=query_layer.dtype) # fp16 compatibility + +# if self.position_embedding_type == "relative_key": +# relative_position_scores = torch.einsum("bhld,lrd->bhlr", query_layer, positional_embedding) +# attention_scores = attention_scores + relative_position_scores +# elif self.position_embedding_type == "relative_key_query": +# relative_position_scores_query = torch.einsum("bhld,lrd->bhlr", query_layer, positional_embedding) +# relative_position_scores_key = torch.einsum("bhrd,lrd->bhlr", key_layer, positional_embedding) +# attention_scores = attention_scores + relative_position_scores_query + relative_position_scores_key + +# attention_scores = attention_scores / math.sqrt(self.attention_head_size) +# if attention_mask is not None: +# # Apply the attention mask is (precomputed for all layers in BlipTextModel forward() function) +# attention_scores = attention_scores + attention_mask.to(attention_scores.device) + +# # Normalize the attention scores to probabilities. +# attention_probs = nn.Softmax(dim=-1)(attention_scores) + +# # This is actually dropping out entire tokens to attend to, which might +# # seem a bit unusual, but is taken from the original Transformer paper. +# attention_probs_dropped = self.dropout(attention_probs) + +# # Mask heads if we want to +# if head_mask is not None: +# attention_probs_dropped = attention_probs_dropped * head_mask + +# context_layer = torch.matmul(attention_probs_dropped, value_layer) + +# context_layer = context_layer.permute(0, 2, 1, 3).contiguous() +# new_context_layer_shape = context_layer.size()[:-2] + (-1,) +# context_layer = context_layer.view(*new_context_layer_shape) + +# outputs = (context_layer, attention_probs) if output_attentions else (context_layer,) + +# outputs = outputs + (past_key_value,) +# return outputs +# @staticmethod +# def init_from_exist_self_attn(attn: BlipTextSelfAttention,is_cross_attention): +# # print(attn) + +# res = blipTextAttentionPrunable(is_cross_attention) + +# for attr in dir(attn): +# # if str(attr) in ['transpose_for_scores'] or str(attr).startswith('_'): +# # continue +# # if isinstance(getattr(attn, attr), nn.Module): +# # print(attr) + +# if isinstance(getattr(attn, attr), nn.Module): +# try: +# # print(attr, 'ok') +# setattr(res, attr, getattr(attn, attr)) + +# except Exception as e: +# print(attr, str(e)) + + + +# return res + + + +# class blipSelfTextAttentionPrunable(BlipTextAttention): +# def __init__(self, config, is_cross_attention=False): +# self.self = blipTextAttentionPrunable(config, is_cross_attention) +# self.output = BlipTextSelfOutput(config) +# self.pruned_heads = set() +# super(blipSelfTextAttentionPrunable,self).__init__(config) + +# def prune_heads(self, heads): +# if len(heads) == 0: +# return +# heads, index = find_pruneable_heads_and_indices( +# heads, self.self.num_attention_heads, self.self.attention_head_size, self.pruned_heads +# ) + +# # Prune linear layers +# self.self.query = prune_linear_layer(self.self.query, index) +# self.self.key = prune_linear_layer(self.self.key, index) +# self.self.value = prune_linear_layer(self.self.value, index) +# self.output.dense = prune_linear_layer(self.output.dense, index, dim=1) + +# # Update hyper params and store pruned heads +# self.self.num_attention_heads = self.self.num_attention_heads - len(heads) +# self.self.all_head_size = self.self.attention_head_size * self.self.num_attention_heads +# self.pruned_heads = self.pruned_heads.union(heads) + +# def forward( +# self, +# hidden_states: torch.Tensor, +# attention_mask: Optional[torch.FloatTensor] = None, +# head_mask: Optional[torch.FloatTensor] = None, +# encoder_hidden_states: Optional[torch.FloatTensor] = None, +# encoder_attention_mask: Optional[torch.FloatTensor] = None, +# past_key_value: Optional[Tuple[Tuple[torch.FloatTensor]]] = None, +# output_attentions: Optional[bool] = False, +# ) -> Tuple[torch.Tensor]: +# self_outputs = self.self( +# hidden_states, +# attention_mask, +# head_mask, +# encoder_hidden_states, +# encoder_attention_mask, +# past_key_value, +# output_attentions, +# ) +# attention_output = self.output(self_outputs[0], hidden_states) +# outputs = (attention_output,) + self_outputs[1:] # add attentions if we output them +# return outputs +# @staticmethod +# def init_from_exist_self_attn(attn: BlipTextAttention,config,is_cross_attention): +# # print(attn) + +# res = blipTextAttentionPrunable(config,is_cross_attention) + +# for attr in dir(attn): +# # if str(attr) in ['transpose_for_scores'] or str(attr).startswith('_'): +# # continue +# # if isinstance(getattr(attn, attr), nn.Module): +# # print(attr) + +# if isinstance(getattr(attn, attr), nn.Module): +# try: +# # print(attr, 'ok') +# setattr(res, attr, getattr(attn, attr)) + +# except Exception as e: +# print(attr, str(e)) + + + +# return res + + + + + + +# class blipSelfAttentionPrunable(BlipAttention): +# def __init__(self): +# config = BlipConfig.from_pretrained('new_impl/mm/Vis_bert/QuestionAnswering/VisBert_pretrained') +# super(blipSelfAttentionPrunable, self).__init__(config.vision_config) + +# def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int): +# return tensor.view(bsz, seq_len, self.num_heads, -1).transpose(1, 2).contiguous() + +# def forward( +# self, +# hidden_states: torch.Tensor, +# head_mask: Optional[torch.Tensor] = None, +# output_attentions: Optional[bool] = False, +# ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: +# """Input shape: Batch x Time x Channel""" + +# bsz, tgt_len, embed_dim = hidden_states.size() + +# mixed_qkv = ( +# self.qkv(hidden_states) +# .reshape(bsz, tgt_len, 3, self.num_heads, -1) +# .permute(2, 0, 3, 1, 4) +# ) +# query_states, key_states, value_states = mixed_qkv[0], mixed_qkv[1], mixed_qkv[2] + +# # Take the dot product between "query" and "key" to get the raw attention scores. +# attention_scores = torch.matmul(query_states, key_states.transpose(-1, -2)) + +# attention_scores = attention_scores * self.scale + +# # Normalize the attention scores to probabilities. +# attention_probs = nn.functional.softmax(attention_scores, dim=-1) + +# # This is actually dropping out entire tokens to attend to, which might +# # seem a bit unusual, but is taken from the original Transformer paper. +# attention_probs = self.dropout(attention_probs) + +# # Mask heads if we want to +# if head_mask is not None: +# attention_probs = attention_probs * head_mask + +# context_layer = torch.matmul(attention_probs, value_states).permute(0, 2, 1, 3) + +# new_context_layer_shape = context_layer.size()[:-2] + (-1,) +# context_layer = context_layer.reshape(new_context_layer_shape) + +# output = self.projection(context_layer) + +# outputs = (output, attention_probs) if output_attentions else (output, None) + +# return outputs + +# @staticmethod +# def init_from_exist_self_attn(attn: BlipAttention): +# # print(attn) + +# res = blipSelfAttentionPrunable() + +# for attr in dir(attn): +# # if str(attr) in ['transpose_for_scores'] or str(attr).startswith('_'): +# # continue +# # if isinstance(getattr(attn, attr), nn.Module): +# # print(attr) + +# if isinstance(getattr(attn, attr), nn.Module): +# try: +# # print(attr, 'ok') +# setattr(res, attr, getattr(attn, attr)) + +# except Exception as e: +# print(attr, str(e)) + + + +# return res +class BeitSelfAttentionPrunable(BeitSelfAttention): + def __init__(self, config: BeitConfig, window_size: Optional[tuple] = None) -> None: + config = BeitConfig.from_pretrained('new_impl/cv/beit/beit_model') + super(BeitSelfAttentionPrunable, self).__init__(config) + + def transpose_for_scores(self, x): + new_x_shape = x.size()[:-1] + (self.num_attention_heads, -1) + x = x.view(*new_x_shape) + return x.permute(0, 2, 1, 3) + + def forward( + self, + hidden_states: torch.Tensor, + head_mask: Optional[torch.Tensor] = None, + output_attentions: bool = False, + relative_position_bias: Optional["BeitRelativePositionBias"] = None, + ) -> Union[Tuple[torch.Tensor], Tuple[torch.Tensor, torch.Tensor]]: + mixed_query_layer = self.query(hidden_states) + + key_layer = self.transpose_for_scores(self.key(hidden_states)) + value_layer = self.transpose_for_scores(self.value(hidden_states)) + query_layer = self.transpose_for_scores(mixed_query_layer) + + # Take the dot product between "query" and "key" to get the raw attention scores. + attention_scores = torch.matmul(query_layer, key_layer.transpose(-1, -2)) + + attention_scores = attention_scores / math.sqrt(self.attention_head_size) + + # Add relative position bias if present. + if self.relative_position_bias is not None: + attention_scores = attention_scores + self.relative_position_bias().unsqueeze(0) + + # Add shared relative position bias if provided. + if relative_position_bias is not None: + attention_scores = attention_scores + relative_position_bias + + # Normalize the attention scores to probabilities. + attention_probs = nn.functional.softmax(attention_scores, dim=-1) + + # This is actually dropping out entire tokens to attend to, which might + # seem a bit unusual, but is taken from the original Transformer paper. + attention_probs = self.dropout(attention_probs) + + # Mask heads if we want to + if head_mask is not None: + attention_probs = attention_probs * head_mask + + context_layer = torch.matmul(attention_probs, value_layer) + + context_layer = context_layer.permute(0, 2, 1, 3).contiguous() + new_context_layer_shape = context_layer.size()[:-2] + (-1,) + context_layer = context_layer.view(*new_context_layer_shape) + + outputs = (context_layer, attention_probs) if output_attentions else (context_layer,) + + return outputs + + @staticmethod + def init_from_exist_self_attn(attn: BeitSelfAttention,config): + # print(attn) + + res = BeitSelfAttentionPrunable(config) + + for attr in dir(attn): + # if str(attr) in ['transpose_for_scores'] or str(attr).startswith('_'): + # continue + # if isinstance(getattr(attn, attr), nn.Module): + # print(attr) + + if isinstance(getattr(attn, attr), nn.Module): + try: + # print(attr, 'ok') + setattr(res, attr, getattr(attn, attr)) + + except Exception as e: + print(attr, str(e)) + return res + +class FM_to_MD_beit_Util(FM_to_MD_Util): + def init_md_from_fm_by_reducing_width(self, fm: nn.Module, reducing_width_ratio: int) -> nn.Module: + fm_vis = deepcopy(fm) + config = BeitConfig.from_pretrained('new_impl/cv/beit/beit_model') + + for block_i,block in enumerate(fm_vis.beit.encoder.layer): + set_module(block, 'attention.attention', BeitSelfAttentionPrunable.init_from_exist_self_attn(block.attention.attention,config)) + + def _f(n): + return int(n // reducing_width_ratio) + + # def _rand_indexes(n): + # return torch.randperm(n)[0: int(n // reducing_width_ratio)] + + def l1_max_indexes(p: torch.Tensor, dim=0): + assert dim in [0, 1] + assert p.dim() in [1, 2, 4] + + if dim == 1: + p = p.T + + p_norm = p.abs().contiguous().view(p.size(0), -1).sum(dim=1) + n = p.size(0) + return p_norm.argsort(descending=True)[0: int(n // reducing_width_ratio)].sort()[0] + + for block_i, block in enumerate(fm_vis.beit.encoder.layer): + for k in ['query', 'key', 'value']: + qkv = get_module(block, f'attention.attention.{k}') + + new_qkv = nn.Linear(qkv.in_features, _f(qkv.out_features), + qkv.bias is not None, qkv.weight.device) + indexes = l1_max_indexes(qkv.weight.data, 0) + + new_qkv.weight.data.copy_(qkv.weight.data[indexes]) + if qkv.bias is not None: + new_qkv.bias.data.copy_(qkv.bias.data[indexes]) + set_module(block, f'attention.attention.{k}', new_qkv) + + proj = get_module(block, f'attention.output.dense') + new_proj = nn.Linear(_f(proj.in_features), proj.out_features, + proj.bias is not None, proj.weight.device) + new_proj.weight.data.copy_(proj.weight.data[:, l1_max_indexes(proj.weight.data, 1)]) + if proj.bias is not None: + new_proj.bias.data.copy_(proj.bias.data) + set_module(block, f'attention.output.dense', new_proj) + + fc1 = get_module(block, f'intermediate.dense') + new_fc1 = nn.Linear(fc1.in_features, _f(fc1.out_features), + fc1.bias is not None, fc1.weight.device) + indexes = l1_max_indexes(fc1.weight.data, 0) + new_fc1.weight.data.copy_(fc1.weight.data[indexes]) + if fc1.bias is not None: + new_fc1.bias.data.copy_(fc1.bias.data[indexes]) + set_module(block, f'intermediate.dense', new_fc1) + + fc2 = get_module(block, f'output.dense') + new_fc2 = nn.Linear(_f(fc2.in_features), fc2.out_features, + fc2.bias is not None, fc2.weight.device) + new_fc2.weight.data.copy_(fc2.weight.data[:, l1_max_indexes(fc2.weight.data, 1)]) + if fc2.bias is not None: + new_fc2.bias.data.copy_(fc2.bias.data) + set_module(block, f'output.dense', new_fc2) + + + # for block_i, block in enumerate(fm_vis.text_decoder.bert.encoder.layer): + # for k in ['query', 'key', 'value']: + # qkv = get_module(block, f'crossattention.self.{k}') + + # new_qkv = nn.Linear(qkv.in_features, _f(qkv.out_features), + # qkv.bias is not None, qkv.weight.device) + # indexes = l1_max_indexes(qkv.weight.data, 0) + + # new_qkv.weight.data.copy_(qkv.weight.data[indexes]) + # if qkv.bias is not None: + # new_qkv.bias.data.copy_(qkv.bias.data[indexes]) + # set_module(block, f'crossattention.self.{k}', new_qkv) + + # proj = get_module(block, f'crossattention.output.dense') + # new_proj = nn.Linear(_f(proj.in_features), proj.out_features, + # proj.bias is not None, proj.weight.device) + # new_proj.weight.data.copy_(proj.weight.data[:, l1_max_indexes(proj.weight.data, 1)]) + # if proj.bias is not None: + # new_proj.bias.data.copy_(proj.bias.data) + # set_module(block, f'crossattention.output.dense', new_proj) + + + # for block_i, block in enumerate(fm_vis.text_encoder.encoder.layer): + # for k in ['query', 'key', 'value']: + # qkv = get_module(block, f'attention.self.{k}') + + # new_qkv = nn.Linear(qkv.in_features, _f(qkv.out_features), + # qkv.bias is not None, qkv.weight.device) + # indexes = l1_max_indexes(qkv.weight.data, 0) + + # new_qkv.weight.data.copy_(qkv.weight.data[indexes]) + # if qkv.bias is not None: + # new_qkv.bias.data.copy_(qkv.bias.data[indexes]) + # set_module(block, f'attention.self.{k}', new_qkv) + + # proj = get_module(block, f'attention.output.dense') + # new_proj = nn.Linear(_f(proj.in_features), proj.out_features, + # proj.bias is not None, proj.weight.device) + # new_proj.weight.data.copy_(proj.weight.data[:, l1_max_indexes(proj.weight.data, 1)]) + # if proj.bias is not None: + # new_proj.bias.data.copy_(proj.bias.data) + # set_module(block, f'attention.output.dense', new_proj) + + # fc1 = get_module(block, f'intermediate.dense') + # new_fc1 = nn.Linear(fc1.in_features, _f(fc1.out_features), + # fc1.bias is not None, fc1.weight.device) + # indexes = l1_max_indexes(fc1.weight.data, 0) + # new_fc1.weight.data.copy_(fc1.weight.data[indexes]) + # if fc1.bias is not None: + # new_fc1.bias.data.copy_(fc1.bias.data[indexes]) + # set_module(block, f'intermediate.dense', new_fc1) + + # fc2 = get_module(block, f'output.dense') + # new_fc2 = nn.Linear(_f(fc2.in_features), fc2.out_features, + # fc2.bias is not None, fc2.weight.device) + # new_fc2.weight.data.copy_(fc2.weight.data[:, l1_max_indexes(fc2.weight.data, 1)]) + # if fc2.bias is not None: + # new_fc2.bias.data.copy_(fc2.bias.data) + # set_module(block, f'output.dense', new_fc2) + + + # for block_i, block in enumerate(fm_vis.text_encoder.encoder.layer): + # for k in ['query', 'key', 'value']: + # qkv = get_module(block, f'crossattention.self.{k}') + + # new_qkv = nn.Linear(qkv.in_features, _f(qkv.out_features), + # qkv.bias is not None, qkv.weight.device) + # indexes = l1_max_indexes(qkv.weight.data, 0) + + # new_qkv.weight.data.copy_(qkv.weight.data[indexes]) + # if qkv.bias is not None: + # new_qkv.bias.data.copy_(qkv.bias.data[indexes]) + # set_module(block, f'crossattention.self.{k}', new_qkv) + + # proj = get_module(block, f'crossattention.output.dense') + # new_proj = nn.Linear(_f(proj.in_features), proj.out_features, + # proj.bias is not None, proj.weight.device) + # new_proj.weight.data.copy_(proj.weight.data[:, l1_max_indexes(proj.weight.data, 1)]) + # if proj.bias is not None: + # new_proj.bias.data.copy_(proj.bias.data) + # set_module(block, f'crossattention.output.dense', new_proj) + + + + # for block_i, block in enumerate(fm_vis.vision_model.encoder.layers): + # qkv = block.self_attn.qkv + + # new_qkv = nn.Linear(qkv.in_features, _f(qkv.out_features), + # qkv.bias is not None, qkv.weight.device) + # indexes = l1_max_indexes(qkv.weight.data, 0) + + # new_qkv.weight.data.copy_(qkv.weight.data[indexes]) + # if qkv.bias is not None: + # new_qkv.bias.data.copy_(qkv.bias.data[indexes]) + # set_module(fm_vis, f'vision_model.encoder.layers.{block_i}.self_attn.qkv', new_qkv) + + # proj = block.self_attn.projection + # new_proj = nn.Linear(_f(proj.in_features), proj.out_features, + # proj.bias is not None, proj.weight.device) + # new_proj.weight.data.copy_(proj.weight.data[:, l1_max_indexes(proj.weight.data, 1)]) + # if proj.bias is not None: + # new_proj.bias.data.copy_(proj.bias.data) + # set_module(fm_vis, f'vision_model.encoder.layers.{block_i}.self_attn.projection', new_proj) + + # fc1 = block.mlp.fc1 + # new_fc1 = nn.Linear(fc1.in_features, _f(fc1.out_features), + # fc1.bias is not None, fc1.weight.device) + # indexes = l1_max_indexes(fc1.weight.data, 0) + # new_fc1.weight.data.copy_(fc1.weight.data[indexes]) + # if fc1.bias is not None: + # new_fc1.bias.data.copy_(fc1.bias.data[indexes]) + # set_module(fm_vis, f'vision_model.encoder.layers.{block_i}.mlp.fc1', new_fc1) + + # fc2 = block.mlp.fc2 + # new_fc2 = nn.Linear(_f(fc2.in_features), fc2.out_features, + # fc2.bias is not None, fc2.weight.device) + # new_fc2.weight.data.copy_(fc2.weight.data[:, l1_max_indexes(fc2.weight.data, 1)]) + # if fc2.bias is not None: + # new_fc2.bias.data.copy_(fc2.bias.data) + # set_module(fm_vis, f'vision_model.encoder.layers.{block_i}.mlp.fc2', new_fc2) + + return fm_vis + + def init_md_from_fm_by_reducing_width_with_perf_test(self, fm: nn.Module, reducing_width_ratio: int, + samples: torch.Tensor) -> nn.Module: + fm_size = get_model_size(fm, True) + fm_latency = self._get_model_latency(fm, samples, 20, + get_model_device(fm), 20, False) + + master_dnn = self.init_md_from_fm_by_reducing_width(fm, reducing_width_ratio) + master_dnn_size = get_model_size(master_dnn, True) + logger.debug(f'inited master DNN: {master_dnn}') + master_dnn_latency = self._get_model_latency(master_dnn, samples, 20, + get_model_device(master_dnn), 20, False) + + logger.info(f'init master DNN (w/o FBS yet) by reducing foundation model\'s width (by {reducing_width_ratio:d}x)') + logger.info(f'foundation model ({fm_size:.3f}MB, {fm_latency:.4f}s/sample) -> ' + f'master DNN ({master_dnn_size:.3f}MB, {master_dnn_latency:.4f}s/sample)\n' + f'(model size: ↓ {(fm_size / master_dnn_size):.2f}x, ' + f'latency: ↓ {(fm_latency / master_dnn_latency):.2f}x)') + + return master_dnn + + def _get_model_latency(self, model: torch.nn.Module, model_input_size, sample_num: int, + device: str, warmup_sample_num: int, return_detail=False): + import time + + if isinstance(model_input_size, tuple): + dummy_input = torch.rand(model_input_size).to(device) + else: + dummy_input = model_input_size + + model = model.to(device) + model.eval() + + # warm up + with torch.no_grad(): + for _ in range(warmup_sample_num): + model(dummy_input) + + infer_time_list = [] + + if device == 'cuda' or 'cuda' in str(device): + with torch.no_grad(): + for _ in range(sample_num): + s, e = torch.cuda.Event(enable_timing=True), torch.cuda.Event(enable_timing=True) + s.record() + model(dummy_input) + e.record() + torch.cuda.synchronize() + cur_model_infer_time = s.elapsed_time(e) / 1000. + infer_time_list += [cur_model_infer_time] + + else: + with torch.no_grad(): + for _ in range(sample_num): + start = time.time() + model(**dummy_input) + cur_model_infer_time = time.time() - start + infer_time_list += [cur_model_infer_time] + + avg_infer_time = sum(infer_time_list) / sample_num + + if return_detail: + return avg_infer_time, infer_time_list + return avg_infer_time + + +####Here starts with index + +class SqueezeLast(nn.Module): + def __init__(self): + super(SqueezeLast, self).__init__() + + def forward(self, x): + return x.squeeze(-1) + + +class ProjConv_WrappedWithFBS(Layer_WrappedWithFBS): + def __init__(self, proj: nn.Conv2d, r): + super(ProjConv_WrappedWithFBS, self).__init__() + + self.proj = proj + + # for conv: (B, C_in, H, W) -> (B, C_in) -> (B, C_out) + # for mlp in ViT: (B, #patches, D: dim of patches embedding) -> (B, D) -> (B, C_out) + self.fbs = nn.Sequential( + Abs(), + nn.AdaptiveAvgPool1d(1), + SqueezeLast(), + nn.Linear(proj.in_channels, proj.out_channels // r), + nn.ReLU(), + nn.Linear(proj.out_channels // r, proj.out_channels), + nn.ReLU() + ) + + nn.init.constant_(self.fbs[6].bias, 1.) + nn.init.kaiming_normal_(self.fbs[6].weight) + + + def forward(self, x): + if self.use_cached_channel_attention and self.cached_channel_attention is not None: + channel_attention = self.cached_channel_attention + else: + self.cached_raw_channel_attention = self.fbs(x) + self.cached_channel_attention = self.k_takes_all(self.cached_raw_channel_attention) + + channel_attention = self.cached_channel_attention + + raw_res = self.proj(x) + + return channel_attention.unsqueeze(1) * raw_res # TODO: + + +class Linear_WrappedWithFBS(Layer_WrappedWithFBS): + def __init__(self, linear: nn.Linear, r): + super(Linear_WrappedWithFBS, self).__init__() + + self.linear = linear + + # for conv: (B, C_in, H, W) -> (B, C_in) -> (B, C_out) + # for mlp in ViT: (B, #patches, D: dim of patches embedding) -> (B, D) -> (B, C_out) + self.fbs = nn.Sequential( + Rearrange('b n d -> b d n'), + Abs(), + nn.AdaptiveAvgPool1d(1), + SqueezeLast(), + nn.Linear(linear.in_features, linear.out_features // r), + nn.ReLU(), + nn.Linear(linear.out_features // r, linear.out_features), + nn.ReLU() + ) + + nn.init.constant_(self.fbs[6].bias, 1.) + nn.init.kaiming_normal_(self.fbs[6].weight) + + + def forward(self, x): + if self.use_cached_channel_attention and self.cached_channel_attention is not None: + channel_attention = self.cached_channel_attention + else: + self.cached_raw_channel_attention = self.fbs(x) + self.cached_channel_attention = self.k_takes_all(self.cached_raw_channel_attention) + + channel_attention = self.cached_channel_attention + + raw_res = self.linear(x) + + return channel_attention.unsqueeze(1) * raw_res + + +class ToQKV_WrappedWithFBS(Layer_WrappedWithFBS): + """ + This regards to_q/to_k/to_v as a whole (in fact it consists of multiple heads) and prunes it. + It seems different channels of different heads are pruned according to the input. + This is different from "removing some head" or "removing the same channels in each head". + """ + def __init__(self, to_qkv: nn.Linear, r): + super(ToQKV_WrappedWithFBS, self).__init__() + + # self.to_qkv = to_qkv + + self.to_qk = nn.Linear(to_qkv.in_features, to_qkv.out_features // 3 * 2, bias=to_qkv.bias is not None) + self.to_v = nn.Linear(to_qkv.in_features, to_qkv.out_features // 3, bias=to_qkv.bias is not None) + self.to_qk.weight.data.copy_(to_qkv.weight.data[0: to_qkv.out_features // 3 * 2]) + if to_qkv.bias is not None: + self.to_qk.bias.data.copy_(to_qkv.bias.data[0: to_qkv.out_features // 3 * 2]) + self.to_v.weight.data.copy_(to_qkv.weight.data[to_qkv.out_features // 3 * 2: ]) + if to_qkv.bias is not None: + self.to_v.bias.data.copy_(to_qkv.bias.data[to_qkv.out_features // 3 * 2: ]) + + self.fbs = nn.Sequential( + Rearrange('b n d -> b d n'), + Abs(), + nn.AdaptiveAvgPool1d(1), + SqueezeLast(), + nn.Linear(to_qkv.in_features, to_qkv.out_features // 3 // r), + nn.ReLU(), + # nn.Linear(to_qkv.out_features // 3 // r, to_qkv.out_features // 3), + nn.Linear(to_qkv.out_features // 3 // r, self.to_v.out_features), + nn.ReLU() + ) + + nn.init.constant_(self.fbs[6].bias, 1.) + nn.init.kaiming_normal_(self.fbs[6].weight) + + def forward(self, x): + if self.use_cached_channel_attention and self.cached_channel_attention is not None: + channel_attention = self.cached_channel_attention + else: + self.cached_raw_channel_attention = self.fbs(x) + + # print() + # for attn in self.cached_raw_channel_attention.chunk(3, dim=1)[0: 1]: + # print(self.cached_raw_channel_attention.size(), attn.size()) + # print(self.k_takes_all.k) + # print(attn[0].nonzero(as_tuple=True)[0].size(), attn[0]) + + self.cached_channel_attention = self.k_takes_all(self.cached_raw_channel_attention) + + + # for attn in self.cached_channel_attention.chunk(3, dim=1)[0: 1]: + # print(self.cached_channel_attention.size(), attn.size()) + # print(self.k_takes_all.k) + # print(attn[0].nonzero(as_tuple=True)[0].size(), attn[0]) + # print() + + channel_attention = self.cached_channel_attention + + qk = self.to_qk(x) + v = channel_attention.unsqueeze(1) * self.to_v(x) + return torch.cat([qk, v], dim=-1) + + # qkv = raw_res.chunk(3, dim = -1) + + # raw_v = qkv[2] + # print('raw_k, raw_v', qkv[0].sum((0, 1))[0: 10], qkv[0].sum((0, 1)).nonzero(as_tuple=True)[0].size(), + # qkv[1].sum((0, 1))[0: 10], qkv[1].sum((0, 1)).nonzero(as_tuple=True)[0].size(),) + # print('raw_v', raw_v.size(), raw_v.sum((0, 1))[0: 10], raw_v.sum((0, 1)).nonzero(as_tuple=True)[0].size()) + + # qkv_attn = channel_attention.chunk(3, dim=-1) + # print('attn', [attn[0][0: 10] for attn in qkv_attn]) + # print(channel_attention.unsqueeze(1).size(), raw_res.size()) + # print('fbs', channel_attention.size(), raw_res.size()) + # return channel_attention.unsqueeze(1) * raw_res + + +class StaticFBS(nn.Module): + def __init__(self, static_channel_attention): + super(StaticFBS, self).__init__() + assert static_channel_attention.dim() == 2 and static_channel_attention.size(0) == 1 + self.static_channel_attention = nn.Parameter(static_channel_attention, requires_grad=False) # (1, dim) + + def forward(self, x): + # print('staticfbs', x, self.static_channel_attention.unsqueeze(1)) + return x * self.static_channel_attention.unsqueeze(1) + + +class ElasticbeitUtil(ElasticDNNUtil): + def convert_raw_dnn_to_master_dnn(self, raw_dnn: nn.Module, r: float, ignore_layers=[]): + assert len(ignore_layers) == 0, 'not supported yet' + + raw_vit = deepcopy(raw_dnn) + + # set_module(module, 'patch_embed.proj', ProjConv_WrappedWithFBS(module.patch_embed.proj, r)) + + for name, module in raw_vit.named_modules(): + # if name.endswith('attn'): + # set_module(module, 'qkv', ToQKV_WrappedWithFBS(module.qkv, r)) + if name.endswith('intermediate'): + set_module(module, 'dense', Linear_WrappedWithFBS(module.dense, r)) + elif name.endswith('mlp'): + set_module(module, 'fc1', Linear_WrappedWithFBS(module.fc1, r)) + + return raw_vit + + def set_master_dnn_sparsity(self, master_dnn: nn.Module, sparsity: float): + # for name, module in master_dnn.named_modules(): + # if not name.endswith('attn'): + # continue + + # q_features = module.qkv.to_qk.out_features // 2 + + # if (q_features - int(q_features * sparsity)) % module.num_heads != 0: + # # tune sparsity to ensure #unpruned channel % num_heads == 0 + # # so that the pruning seems to reduce the dim_head of each head + # tuned_sparsity = 1. - int((q_features - int(q_features * sparsity)) / module.num_heads) * module.num_heads / q_features + # logger.debug(f'tune sparsity from {sparsity:.2f} to {tuned_sparsity}') + # sparsity = tuned_sparsity + # break + + return super().set_master_dnn_sparsity(master_dnn, sparsity) + + def select_most_rep_sample(self, master_dnn: nn.Module, samples: torch.Tensor): + # print(samples) + return samples[0].unsqueeze(0) + # res = {k: v[0: 1] for k, v in samples.items()} + # return res + + def extract_surrogate_dnn_via_samples(self, master_dnn: nn.Module, samples: torch.Tensor, return_detail=False):#产生小模型的步骤 + sample = self.select_most_rep_sample(master_dnn, samples) + # assert sample.dim() == 4 and sample.size(0) == 1 + + # print('before') + master_dnn.eval() + self.clear_cached_channel_attention_in_master_dnn(master_dnn) + with torch.no_grad(): + master_dnn_output = master_dnn(sample) + + # print('after') + + boosted_vit = deepcopy(master_dnn) + + def get_unpruned_indexes_from_channel_attn(channel_attn: torch.Tensor, k): + assert channel_attn.size(0) == 1, 'use A representative sample to generate channel attentions' + + # print('attn_in_unpruned', channel_attn[0][0: 10]) + + res = channel_attn[0].nonzero(as_tuple=True)[0] # should be one-dim + + # res = channel_attn[0].argsort(descending=True)[0: -int(channel_attn.size(1) * k)].sort()[0] + + # g = channel_attn + # k = g.size(1) - int(g.size(1) * k) + # res = g.topk(k, 1)[1][0].sort()[0] + + return res + + unpruned_indexes_of_layers = {} + + # for attn, ff in boosted_vit.transformer.layers: + # for block_i, block in enumerate(boosted_vit.blocks): + for block_i, block in enumerate(boosted_vit.beit.encoder.layer): + # attn = block.attn + # ff = block.mlp + + ff_0 = get_module(block, f'intermediate.dense') + # ff_0_unpruned_indexes = get_unpruned_indexes_from_channel_attn(ff_0.cached_channel_attention, k) + ff_0_pruned_indexes = ff_0.k_takes_all.cached_i[0].sort()[0] + ff_0_unpruned_indexes = torch.LongTensor([ii for ii in range(ff_0.cached_channel_attention.size(1)) if ii not in ff_0_pruned_indexes]) + new_ff_0 = nn.Linear(ff_0.linear.in_features, ff_0_unpruned_indexes.size(0), ff_0.linear.bias is not None) + new_ff_0.weight.data.copy_(ff_0.linear.weight.data[ff_0_unpruned_indexes]) + if ff_0.linear.bias is not None: + new_ff_0.bias.data.copy_(ff_0.linear.bias.data[ff_0_unpruned_indexes]) + set_module(block, 'intermediate.dense', nn.Sequential(new_ff_0, StaticFBS(ff_0.cached_channel_attention[:, ff_0_unpruned_indexes]))) + + ff_1 = get_module(block, f'output.dense') + new_ff_1 = nn.Linear(ff_0_unpruned_indexes.size(0), ff_1.out_features, ff_1.bias is not None) + new_ff_1.weight.data.copy_(ff_1.weight.data[:, ff_0_unpruned_indexes]) + if ff_1.bias is not None: + new_ff_1.bias.data.copy_(ff_1.bias.data) + set_module(block, 'output.dense', new_ff_1) + + unpruned_indexes_of_layers[f'beit.encoder.layer.{block_i}.intermediate.dense.0.weight'] = ff_0_unpruned_indexes + # for block_i,block in enumerate(boosted_vit.vision_model.encoder.layers): + + # attn = block.self_attn + # ff = block.mlp + # ff_0 = ff.fc1 + # # ff_0_unpruned_indexes = get_unpruned_indexes_from_channel_attn(ff_0.cached_channel_attention, k) + # ff_0_pruned_indexes = ff_0.k_takes_all.cached_i[0].sort()[0] + # ff_0_unpruned_indexes = torch.LongTensor([ii for ii in range(ff_0.cached_channel_attention.size(1)) if ii not in ff_0_pruned_indexes]) + # new_ff_0 = nn.Linear(ff_0.linear.in_features, ff_0_unpruned_indexes.size(0), ff_0.linear.bias is not None) + # new_ff_0.weight.data.copy_(ff_0.linear.weight.data[ff_0_unpruned_indexes]) + # if ff_0.linear.bias is not None: + # new_ff_0.bias.data.copy_(ff_0.linear.bias.data[ff_0_unpruned_indexes]) + # set_module(ff, 'fc1', nn.Sequential(new_ff_0, StaticFBS(ff_0.cached_channel_attention[:, ff_0_unpruned_indexes]))) + + # ff_1 = ff.fc2 + # new_ff_1 = nn.Linear(ff_0_unpruned_indexes.size(0), ff_1.out_features, ff_1.bias is not None) + # new_ff_1.weight.data.copy_(ff_1.weight.data[:, ff_0_unpruned_indexes]) + # if ff_1.bias is not None: + # new_ff_1.bias.data.copy_(ff_1.bias.data) + # set_module(ff, 'fc2', new_ff_1) + + # unpruned_indexes_of_layers[f'vision_model.encoder.layers.{block_i}.mlp.fc1.0.weight'] = ff_0_unpruned_indexes + + + # for block_i, block in enumerate(boosted_vit.text_decoder.bert.encoder.layer): + # # attn = block.attn + # # ff = block.mlp + + # ff_0 = get_module(block, f'intermediate.dense') + # # ff_0_unpruned_indexes = get_unpruned_indexes_from_channel_attn(ff_0.cached_channel_attention, k) + # ff_0_pruned_indexes = ff_0.k_takes_all.cached_i[0].sort()[0] + # ff_0_unpruned_indexes = torch.LongTensor([ii for ii in range(ff_0.cached_channel_attention.size(1)) if ii not in ff_0_pruned_indexes]) + # new_ff_0 = nn.Linear(ff_0.linear.in_features, ff_0_unpruned_indexes.size(0), ff_0.linear.bias is not None) + # new_ff_0.weight.data.copy_(ff_0.linear.weight.data[ff_0_unpruned_indexes]) + # if ff_0.linear.bias is not None: + # new_ff_0.bias.data.copy_(ff_0.linear.bias.data[ff_0_unpruned_indexes]) + # set_module(block, 'intermediate.dense', nn.Sequential(new_ff_0, StaticFBS(ff_0.cached_channel_attention[:, ff_0_unpruned_indexes]))) + + # ff_1 = get_module(block, f'output.dense') + # new_ff_1 = nn.Linear(ff_0_unpruned_indexes.size(0), ff_1.out_features, ff_1.bias is not None) + # new_ff_1.weight.data.copy_(ff_1.weight.data[:, ff_0_unpruned_indexes]) + # if ff_1.bias is not None: + # new_ff_1.bias.data.copy_(ff_1.bias.data) + # set_module(block, 'output.dense', new_ff_1) + + # unpruned_indexes_of_layers[f'text_decoder.bert.encoder.layer.{block_i}.intermediate.dense.0.weight'] = ff_0_unpruned_indexes + surrogate_dnn = boosted_vit + surrogate_dnn.eval() + surrogate_dnn = surrogate_dnn.to(get_model_device(master_dnn)) + # logger.debug(surrogate_dnn) + with torch.no_grad(): + surrogate_dnn_output = surrogate_dnn(sample) + + output_diff = ((surrogate_dnn_output.logits - master_dnn_output.logits) ** 2).sum() + # assert output_diff < 1e-4, output_diff + logger.info(f'output diff of master and surrogate DNN: {output_diff}') + # logger.debug(f'example output of master/surrogate: {master_dnn_output.sum(0)[0: 10]}, {surrogate_dnn_output.sum(0)[0: 10]}') + # logger.info(f'\nonly prune mlp!!!!\n') + # logger.info(f'\nonly prune mlp!!!!\n') + + if return_detail: + return boosted_vit, unpruned_indexes_of_layers + + return boosted_vit + + def extract_surrogate_dnn_via_samples_with_perf_test(self, master_dnn: nn.Module, samples: torch.Tensor, return_detail=False): + master_dnn_size = get_model_size(master_dnn, True) + master_dnn_latency = self._get_model_latency(master_dnn, samples, 50, + get_model_device(master_dnn), 50, False) + + res = self.extract_surrogate_dnn_via_samples(master_dnn, samples, return_detail) + if not return_detail: + surrogate_dnn = res + else: + surrogate_dnn, unpruned_indexes_of_layers = res + surrogate_dnn_size = get_model_size(surrogate_dnn, True) + surrogate_dnn_latency = self._get_model_latency(master_dnn, samples, 50, + get_model_device(master_dnn), 50, False) + + logger.info(f'master DNN ({master_dnn_size:.3f}MB, {master_dnn_latency:.4f}s/sample) -> ' + f'surrogate DNN ({surrogate_dnn_size:.3f}MB, {surrogate_dnn_latency:.4f}s/sample)\n' + f'(model size: ↓ {(master_dnn_size / surrogate_dnn_size):.2f}x, ' + f'latency: ↓ {(master_dnn_latency / surrogate_dnn_latency):.2f}x)') + + return res + + def _get_model_latency(self, model: torch.nn.Module, model_input_size, sample_num: int, + device: str, warmup_sample_num: int, return_detail=False): + import time + + if isinstance(model_input_size, tuple): + dummy_input = torch.rand(model_input_size).to(device) + else: + dummy_input = model_input_size + + model = model.to(device) + model.eval() + + # warm up + with torch.no_grad(): + for _ in range(warmup_sample_num): + model(dummy_input) + + infer_time_list = [] + + if device == 'cuda' or 'cuda' in str(device): + with torch.no_grad(): + for _ in range(sample_num): + s, e = torch.cuda.Event(enable_timing=True), torch.cuda.Event(enable_timing=True) + s.record() + model(dummy_input) + e.record() + torch.cuda.synchronize() + cur_model_infer_time = s.elapsed_time(e) / 1000. + infer_time_list += [cur_model_infer_time] + + else: + with torch.no_grad(): + for _ in range(sample_num): + start = time.time() + model(**dummy_input) + cur_model_infer_time = time.time() - start + infer_time_list += [cur_model_infer_time] + + avg_infer_time = sum(infer_time_list) / sample_num + + if return_detail: + return avg_infer_time, infer_time_list + return avg_infer_time + + + +#####Here starts with online diff --git a/new_impl/cv/beit/beit_cls.py b/new_impl/cv/beit/beit_cls.py new file mode 100644 index 0000000000000000000000000000000000000000..243be5fa32ae184633c2ddbebcff84af73069f64 --- /dev/null +++ b/new_impl/cv/beit/beit_cls.py @@ -0,0 +1,92 @@ +import torch +from new_impl.cv.elasticdnn.api.model import ElasticDNN_OfflineClsFMModel +#from methods.elasticdnn.api.algs.fm_lora import ElasticDNN_FMLoRAAlg +from new_impl.cv.elasticdnn.api.algs.fm_lora import ElasticDNN_FMLoRAAlg +from methods.elasticdnn.model.base import ElasticDNNUtil +from new_impl.cv.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from beit import FMLoRA_beit_Util +from new_impl.cv.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util + +from new_impl.cv.elasticdnn.model.vit import ElasticViTUtil +from data import build_scenario +import torch.nn.functional as F +from utils.dl.common.model import LayerActivation, get_module +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +# from transformers import CvtForImageClassification +# model = CvtForImageClassification.from_pretrained("/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/new_impl/cv/cvt_model",num_labels=20,ignore_mismatched_sizes=True).to('cuda') + +class ElasticDNN_beit_OfflineClsFMModel(ElasticDNN_OfflineClsFMModel): + def generate_md_by_reducing_width(self, reducing_width_ratio, samples: torch.Tensor): + # return FM_to_MD_ViT_Util().init_md_from_fm_by_reducing_width_with_perf_test(self.models_dict['main'], + # reducing_width_ratio, samples) + raise NotImplementedError + def get_feature_hook(self) -> LayerActivation: + return LayerActivation(get_module(self.models_dict['main'], 'classifier'), True, self.device) + + def get_elastic_dnn_util(self) -> ElasticDNNUtil: + raise NotImplementedError + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + #x1 = torch.rand(1,3,224,224).to('cuda:1') + o1 = self.infer(x) + # o2 = self.infer(x1) + # print(o1.logits) + # print(o2.logits) + #print(self.models_dict['main']) + #print(o1.logits.shape) + #print(F.cross_entropy(self.infer(x).logits, y) ) + #formatted_values = [[round(value, 4) for value in row] for row in o1.logits.tolist()] + #return F.cross_entropy(torch.tensor(formatted_values).to('cuda'), y) + return F.cross_entropy(o1.logits, y) #这个是适用于hugging face模型的计算形式，因为它输出的是一个实例化的类，结果封装在类的属性里，你得去给它调出来。 + + def get_lora_util(self) -> FMLoRA_Util: + return FMLoRA_beit_Util() + + def get_task_head_params(self): + head = get_module(self.models_dict['main'], 'classifier') + return list(head.parameters()) +if __name__ == '__main__': + scenario = build_scenario( + source_datasets_name=['GTA5Cls', 'SuperviselyPersonCls'], + target_datasets_order=['CityscapesCls', 'BaiduPersonCls'] * 15, + da_mode='close_set', + data_dirs={ + 'GTA5Cls': '/data/zql/datasets/gta5_for_cls_task', + 'SuperviselyPersonCls': '/data/zql/datasets/supervisely_person_for_cls_task', + 'CityscapesCls': '/data/zql/datasets/cityscapes_for_cls_task', + 'BaiduPersonCls': '/data/zql/datasets/baidu_person_for_cls_task' + }, + ) + + from transformers import BeitForImageClassification + fm_models_dict_path = save_models_dict_for_init({ + 'main':BeitForImageClassification.from_pretrained('new_impl/cv/beit/beit_model',num_labels=scenario.num_classes,ignore_mismatched_sizes=True) + },__file__,'dinat_pretrained') + torch.cuda.set_device(1) + device = 'cuda' + #print(CvtForImageClassification.from_pretrained("/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/new_impl/cv/cvt_model",num_labels=scenario.num_classes,ignore_mismatched_sizes=True)) + fm_model = ElasticDNN_beit_OfflineClsFMModel('fm', fm_models_dict_path, device) + #fm_model = CvtForImageClassification.from_pretrained("/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/new_impl/cv/cvt_model",num_labels=scenario.num_classes,ignore_mismatched_sizes=True).to(device) + models = { + 'fm':fm_model + } + import sys + fm_lora_alg = ElasticDNN_FMLoRAAlg(models, get_res_save_dir(__file__, tag=sys.argv[0])) + + from utils.dl.common.lr_scheduler import get_linear_schedule_with_warmup + fm_lora_alg.run(scenario, hyps={ + 'launch_tbboard': False, + + 'samples_size': (1, 3, 224, 224), + 'ab_r': 3,#hugging face中的模型封装得特别严实，自注意力层里面，qkv是分开的，注意这个对应的层数不要设置太高 + 'train_batch_size': 256, + 'val_batch_size': 512, + 'num_workers': 16, + 'optimizer': 'Adam', + 'optimizer_args': {'lr': 1e-3, 'betas': [0.9, 0.999]},#不同的模型，注意调调学习率啊 + 'scheduler': 'LambdaLR', + 'scheduler_args': {'lr_lambda': get_linear_schedule_with_warmup(10000, 70000)}, + 'num_iters': 8000, + 'val_freq': 400 + } +) diff --git a/new_impl/cv/beit/beit_fbs.py b/new_impl/cv/beit/beit_fbs.py new file mode 100644 index 0000000000000000000000000000000000000000..b0b5dc459a56119588f1d8d1a5d17647e54f1d60 --- /dev/null +++ b/new_impl/cv/beit/beit_fbs.py @@ -0,0 +1,274 @@ +import torch +import sys +from torch import nn +from new_impl.cv.dnns.vit import make_softmax_prunable +from new_impl.cv.elasticdnn.api.model import ElasticDNN_OfflineClsFMModel, ElasticDNN_OfflineClsMDModel +from new_impl.cv.elasticdnn.api.algs.md_pretraining_wo_fbs import ElasticDNN_MDPretrainingWoFBSAlg +from new_impl.cv.elasticdnn.model.base import ElasticDNNUtil +from new_impl.cv.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from beit import FM_to_MD_beit_Util +from new_impl.cv.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from beit import FMLoRA_beit_Util +from new_impl.cv.elasticdnn.model.vit import ElasticViTUtil +from utils.dl.common.model import LayerActivation, get_module, get_parameter +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +from utils.dl.common.loss import CrossEntropyLossSoft +import torch.nn.functional as F + + +# class ElasticDNN_ViT_OfflineClsFMModel(ElasticDNN_OfflineClsFMModel): +# def generate_md_by_reducing_width(self, reducing_width_ratio, samples: torch.Tensor): +# return FM_to_MD_ViT_Util().init_md_from_fm_by_reducing_width_with_perf_test(self.models_dict['main'], +# reducing_width_ratio, samples).to(self.device) + +# def get_feature_hook(self) -> LayerActivation: +# return LayerActivation(get_module(self.models_dict['main'], 'head'), True, self.device) + +# def get_elastic_dnn_util(self) -> ElasticDNNUtil: +# return ElasticViTUtil() + +# def forward_to_get_task_loss(self, x, y, *args, **kwargs): +# return F.cross_entropy(self.infer(x), y) + +# def get_lora_util(self) -> FMLoRA_Util: +# return FMLoRA_ViT_Util() + +# def get_task_head_params(self): +# head = get_module(self.models_dict['main'], 'head') +# return list(head.parameters()) + + +# class ElasticDNN_ViT_OfflineClsMDModel(ElasticDNN_OfflineClsMDModel): +# def __init__(self, name: str, models_dict_path: str, device: str): +# super().__init__(name, models_dict_path, device) + +# self.distill_criterion = CrossEntropyLossSoft() + +# def get_feature_hook(self) -> LayerActivation: +# return LayerActivation(get_module(self.models_dict['main'], 'head'), True, self.device) + +# def forward_to_get_task_loss(self, x, y, *args, **kwargs): +# return F.cross_entropy(self.infer(x), y) + +# def get_distill_loss(self, student_output, teacher_output): +# return self.distill_criterion(student_output, teacher_output) + +# def get_matched_param_of_fm(self, self_param_name, fm: nn.Module): +# # only between qkv.weight, norm.weight/bias +# if any([k in self_param_name for k in ['fbs', 'cls_token', 'pos_embed']]): +# return None + +# p = get_parameter(self.models_dict['main'], self_param_name) +# if p.dim() == 0: +# return None +# elif p.dim() == 1 and 'norm' in self_param_name: +# return get_parameter(fm, self_param_name) + +# # 1. xx.qkv.to_qkv.yy to xx.qkv.qkv.aa and xx.qkv.abs.zz +# if 'to_qkv.weight' in self_param_name: +# ss = self_param_name.split('.') + +# fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv' +# fm_qkv = get_module(fm, fm_qkv_name) + +# fm_abs_name = '.'.join(ss[0: -2]) + '.abs' +# fm_abs = get_module(fm, fm_abs_name) + +# return torch.cat([ +# fm_qkv.weight.data, # task-agnositc params +# torch.cat([(_abs[0].weight.T @ _abs[1].weight.T).T for _abs in fm_abs], dim=0) # task-specific params (LoRA) +# ], dim=0) + +# # elif 'to_qkv.bias' in self_param_name: +# # ss = self_param_name.split('.') + +# # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv.bias' +# # return get_parameter(fm, fm_qkv_name) + +# # elif 'mlp.fc1' in self_param_name: +# # fm_param_name = self_param_name.replace('.linear', '') +# # return get_parameter(fm, fm_param_name) + +# else: +# # return get_parameter(fm, self_param_name) +# return None + +# # def get_matched_param_of_fm(self, self_param_name, fm: nn.Module): +# # if any([k in self_param_name for k in ['fbs', 'cls_token', 'pos_embed']]): +# # return None + +# # # 1. xx.qkv.to_qkv.yy to xx.qkv.qkv.aa and xx.qkv.abs.zz +# # if 'to_qkv.weight' in self_param_name: +# # ss = self_param_name.split('.') + +# # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv' +# # fm_qkv = get_module(fm, fm_qkv_name) + +# # fm_abs_name = '.'.join(ss[0: -2]) + '.abs' +# # fm_abs = get_module(fm, fm_abs_name) + +# # return torch.cat([ +# # fm_qkv.weight.data, # task-agnositc params +# # torch.cat([(_abs[0].weight.T @ _abs[1].weight.T).T for _abs in fm_abs], dim=0) # task-specific params (LoRA) +# # ], dim=0) + +# # elif 'to_qkv.bias' in self_param_name: +# # ss = self_param_name.split('.') + +# # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv.bias' +# # return get_parameter(fm, fm_qkv_name) + +# # elif 'mlp.fc1' in self_param_name: +# # fm_param_name = self_param_name.replace('.linear', '') +# # return get_parameter(fm, fm_param_name) + +# # else: +# # return get_parameter(fm, self_param_name) + + + +class ElasticDNN_beit_OfflineClsFMModel(ElasticDNN_OfflineClsFMModel): + def generate_md_by_reducing_width(self, reducing_width_ratio, samples: torch.Tensor): + return FM_to_MD_beit_Util().init_md_from_fm_by_reducing_width_with_perf_test(self.models_dict['main'], + reducing_width_ratio, samples).to(self.device) + + def get_feature_hook(self) -> LayerActivation: + return LayerActivation(get_module(self.models_dict['main'], 'classifier'), True, self.device) + + def get_elastic_dnn_util(self) -> ElasticDNNUtil: + return ElasticbeitUtil() + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + return F.cross_entropy(self.infer(x).logits, y) + + def get_lora_util(self) -> FMLoRA_Util: + return FMLoRA_beit_Util() + + def get_task_head_params(self): + head = get_module(self.models_dict['main'], 'classifier') + return list(head.parameters()) + + +class ElasticDNN_beit_OfflineClsMDModel(ElasticDNN_OfflineClsMDModel): + def __init__(self, name: str, models_dict_path: str, device: str): + super().__init__(name, models_dict_path, device) + + self.distill_criterion = CrossEntropyLossSoft() + + def get_feature_hook(self) -> LayerActivation: + return LayerActivation(get_module(self.models_dict['main'], 'classifier'), True, self.device) + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + return F.cross_entropy(self.infer(x).logits, y) + + def get_distill_loss(self, student_output, teacher_output): + return self.distill_criterion(student_output, teacher_output) + def get_matched_param_of_fm(self, self_param_name, fm: nn.Module): + # only between qkv.weight, norm.weight/bias + if any([k in self_param_name for k in ['fbs', 'cls_token', 'pos_embed']]): + return None + + p = get_parameter(self.models_dict['main'], self_param_name) + if p.dim() == 0: + return None + elif p.dim() == 1 and 'norm' in self_param_name and 'weight' in self_param_name: + return get_parameter(fm, self_param_name) + + # 1. xx.qkv.to_qkv.yy to xx.qkv.qkv.aa and xx.qkv.abs.zz + if ('attention.attention.query' in self_param_name or 'attention.attention.key' in self_param_name or \ + 'attention.attention.value' in self_param_name) and ('weight' in self_param_name): + ss = self_param_name.split('.') + + fm_qkv_name = '.'.join(ss[0: -1]) + '.qkv' + fm_qkv = get_module(fm, fm_qkv_name) + + fm_abs_name = '.'.join(ss[0: -1]) + '.abs' + fm_abs = get_module(fm, fm_abs_name) + + return torch.cat([ + fm_qkv.weight.data, # task-agnositc params + torch.cat([(_abs[0].weight.T @ _abs[1].weight.T).T for _abs in fm_abs], dim=0) # task-specific params (LoRA) + ], dim=0) + + # elif 'to_qkv.bias' in self_param_name: + # ss = self_param_name.split('.') + + # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv.bias' + # return get_parameter(fm, fm_qkv_name) + elif ('attention.attention.query' in self_param_name or 'attention.attention.key' in self_param_name or \ + 'attention.attention.value' in self_param_name) and ('bias' in self_param_name): + ss = self_param_name.split('.') + + fm_qkv_name = '.'.join(ss[0: -1]) + '.qkv.bias' + return get_parameter(fm, fm_qkv_name) + + elif 'intermediate.dense' in self_param_name and 'weight' in self_param_name: + fm_param_name = self_param_name.replace('.linear', '') + return get_parameter(fm, fm_param_name) + + else: + return get_parameter(fm, self_param_name) + #return None + + +if __name__ == '__main__': + from utils.dl.common.env import set_random_seed + set_random_seed(1) + + # 1. init model + #from dnns.vit import vit_b_16 + fm_models_dict_path = 'new_impl/cv/beit/results/beit_cls.py/20231028/999998-155807-/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/new_impl/cv/beit/beit_cls.py/models/fm_best.pt' + fm_models_dict = torch.load(fm_models_dict_path) + fm_models_dict['main'] = make_softmax_prunable(fm_models_dict['main']) + fm_models_dict_path = save_models_dict_for_init(fm_models_dict, __file__, 'fm_beit_cls_lora') + md_models_dict_path = save_models_dict_for_init({ + 'main': -1 + }, __file__, 'md_beit_cls_lora') + torch.cuda.set_device(1) + device = 'cuda' + + fm_model = ElasticDNN_beit_OfflineClsFMModel('fm', fm_models_dict_path, device) + md_model = ElasticDNN_beit_OfflineClsMDModel('md', md_models_dict_path, device) + + # 2. init alg + models = { + 'fm': fm_model, + 'md': md_model + } + fm_to_md_alg = ElasticDNN_MDPretrainingWoFBSAlg(models, get_res_save_dir(__file__, sys.argv[0])) + + # 3. init scenario + scenario = build_scenario( + source_datasets_name=['GTA5Cls', 'SuperviselyPersonCls'], + target_datasets_order=['CityscapesCls', 'BaiduPersonCls'] * 15, + da_mode='close_set', + data_dirs={ + 'GTA5Cls': '/data/zql/datasets/gta5_for_cls_task', + 'SuperviselyPersonCls': '/data/zql/datasets/supervisely_person_for_cls_task', + 'CityscapesCls': '/data/zql/datasets/cityscapes_for_cls_task', + 'BaiduPersonCls': '/data/zql/datasets/baidu_person_for_cls_task' + }, + ) + + from utils.dl.common.lr_scheduler import get_linear_schedule_with_warmup + fm_to_md_alg.run(scenario, hyps={ + 'launch_tbboard': False, + + 'samples_size': (1, 3, 224, 224), + 'generate_md_width_ratio': 4, + + 'train_batch_size': 128, + 'val_batch_size': 512, + 'num_workers': 16, + 'optimizer': 'AdamW', + 'optimizer_args': {'lr': 1e-4, 'betas': [0.9, 0.999], 'weight_decay': 0.01}, + 'scheduler': 'LambdaLR', + 'scheduler_args': {'lr_lambda': get_linear_schedule_with_warmup(10000, 70000)}, + 'num_iters': 80000, + 'val_freq': 400, + 'distill_loss_weight': 1.0 + }) + + # TODO: + # 1. train MD before inserting FBS? \ No newline at end of file diff --git a/new_impl/cv/beit/beit_index.py b/new_impl/cv/beit/beit_index.py new file mode 100644 index 0000000000000000000000000000000000000000..51036e616e80c96829f1db1032c6fc18732568d3 --- /dev/null +++ b/new_impl/cv/beit/beit_index.py @@ -0,0 +1,275 @@ +import torch +import sys +from torch import nn +from dnns.vit import make_softmax_prunable +from new_impl.cv.elasticdnn.api.model import ElasticDNN_OfflineClsFMModel, ElasticDNN_OfflineClsMDModel +# from methods.elasticdnn.api.algs.md_pretraining_w_fbs import ElasticDNN_MDPretrainingWFBSAlg +from new_impl.cv.elasticdnn.model.base import ElasticDNNUtil +from new_impl.cv.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from beit import FM_to_MD_beit_Util +from new_impl.cv.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from beit import FMLoRA_beit_Util +from beit import ElasticbeitUtil +from utils.dl.common.model import LayerActivation, get_module, get_parameter +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +from utils.dl.common.loss import CrossEntropyLossSoft +import torch.nn.functional as F + + +class ElasticDNN_beit_OfflineClsFMModel(ElasticDNN_OfflineClsFMModel): + def generate_md_by_reducing_width(self, reducing_width_ratio, samples: torch.Tensor): + return FM_to_MD_beit_Util().init_md_from_fm_by_reducing_width_with_perf_test(self.models_dict['main'], + reducing_width_ratio, samples).to(self.device) + + def get_feature_hook(self) -> LayerActivation: + return LayerActivation(get_module(self.models_dict['main'], 'classifier'), True, self.device) + + def get_elastic_dnn_util(self) -> ElasticDNNUtil: + return ElasticbeitUtil() + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + return F.cross_entropy(self.infer(x).logits, y) + + def get_lora_util(self) -> FMLoRA_Util: + return FMLoRA_beit_Util() + + def get_task_head_params(self): + head = get_module(self.models_dict['main'], 'classifier') + return list(head.parameters()) + + +class ElasticDNN_beit_OfflineClsMDModel(ElasticDNN_OfflineClsMDModel): + def __init__(self, name: str, models_dict_path: str, device: str): + super().__init__(name, models_dict_path, device) + + self.distill_criterion = CrossEntropyLossSoft() + + def get_feature_hook(self) -> LayerActivation: + return LayerActivation(get_module(self.models_dict['main'], 'classifier'), True, self.device) + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + return F.cross_entropy(self.infer(x).logits, y) + + def get_distill_loss(self, student_output, teacher_output): + return self.distill_criterion(student_output, teacher_output) + + def get_matched_param_of_fm(self, self_param_name, fm: nn.Module): + # only between qkv.weight, norm.weight/bias + if any([k in self_param_name for k in ['fbs', 'cls_token', 'pos_embed']]): + return None + + p = get_parameter(self.models_dict['main'], self_param_name) + if p.dim() == 0: + return None + elif p.dim() == 1 and 'layernorm' in self_param_name and 'weight' in self_param_name: + return get_parameter(fm, self_param_name) + + # 1. xx.qkv.to_qkv.yy to xx.qkv.qkv.aa and xx.qkv.abs.zz + # if 'qkv.weight' in self_param_name: + # ss = self_param_name.split('.') + + # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv' + # fm_qkv = get_module(fm, fm_qkv_name) + + # fm_abs_name = '.'.join(ss[0: -2]) + '.abs' + # fm_abs = get_module(fm, fm_abs_name) + + # return torch.cat([ + # fm_qkv.weight.data, # task-agnositc params + # torch.cat([(_abs[0].weight.T @ _abs[1].weight.T).T for _abs in fm_abs], dim=0) # task-specific params (LoRA) + # ], dim=0) + + # # elif 'to_qkv.bias' in self_param_name: + # # ss = self_param_name.split('.') + + # # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv.bias' + # # return get_parameter(fm, fm_qkv_name) + + # elif 'mlp.fc1' in self_param_name and 'weight' in self_param_name: + # fm_param_name = self_param_name.replace('.linear', '') + # return get_parameter(fm, fm_param_name) + + # elif 'mlp.fc2' in self_param_name and 'weight' in self_param_name: + # fm_param_name = self_param_name + # return get_parameter(fm, fm_param_name) + + # else: + # # return get_parameter(fm, self_param_name) + # return None + + # if 'qkv.weight' in self_param_name: + # ss = self_param_name.split('.') + + # fm_qkv_name = '.'.join(ss[0: -1]) + '.qkv' + # fm_qkv = get_module(fm, fm_qkv_name) + + # fm_abs_name = '.'.join(ss[0: -1]) + '.abs' + # fm_abs = get_module(fm, fm_abs_name) + + # return torch.cat([ + # fm_qkv.weight.data, # task-agnositc params + # torch.cat([(_abs[0].weight.T @ _abs[1].weight.T).T for _abs in fm_abs], dim=0) # task-specific params (LoRA) + # ], dim=0) + + # # elif 'to_qkv.bias' in self_param_name: + # # ss = self_param_name.split('.') + + # # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv.bias' + # # return get_parameter(fm, fm_qkv_name) + + # elif 'mlp.fc1' in self_param_name and 'weight' in self_param_name: + # fm_param_name = self_param_name.replace('.linear', '') + # return get_parameter(fm, fm_param_name) + + # elif 'mlp.fc2' in self_param_name and 'weight' in self_param_name: + # fm_param_name = self_param_name + # res = get_parameter(fm, fm_param_name) + # # print('mlp fc2 debug', fm_param_name, res is None) + # return res + + # else: + # # return get_parameter(fm, self_param_name) + # return None + + # def get_matched_param_of_fm(self, self_param_name, fm: nn.Module): + # if any([k in self_param_name for k in ['fbs', 'cls_token', 'pos_embed']]): + # return None + + # # 1. xx.qkv.to_qkv.yy to xx.qkv.qkv.aa and xx.qkv.abs.zz + # if 'to_qkv.weight' in self_param_name: + # ss = self_param_name.split('.') + + # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv' + # fm_qkv = get_module(fm, fm_qkv_name) + + # fm_abs_name = '.'.join(ss[0: -2]) + '.abs' + # fm_abs = get_module(fm, fm_abs_name) + + # return torch.cat([ + # fm_qkv.weight.data, # task-agnositc params + # torch.cat([(_abs[0].weight.T @ _abs[1].weight.T).T for _abs in fm_abs], dim=0) # task-specific params (LoRA) + # ], dim=0) + + # elif 'to_qkv.bias' in self_param_name: + # ss = self_param_name.split('.') + + # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv.bias' + # return get_parameter(fm, fm_qkv_name) + + # elif 'mlp.fc1' in self_param_name: + # fm_param_name = self_param_name.replace('.linear', '') + # return get_parameter(fm, fm_param_name) + + # else: + # return get_parameter(fm, self_param_name) + if ('attention.attention.query' in self_param_name or 'attention.attention.key' in self_param_name or \ + 'attention.attention.value' in self_param_name) and ('weight' in self_param_name): + ss = self_param_name.split('.') + + fm_qkv_name = '.'.join(ss[0: -1]) + '.fc' + fm_qkv = get_module(fm, fm_qkv_name) + + fm_abs_name = '.'.join(ss[0: -1]) + '.ab' + fm_abs = get_module(fm, fm_abs_name) + + return torch.cat([ + fm_qkv.weight.data, # task-agnositc params + fm_abs[1].weight @ fm_abs[0].weight + ], dim=0) + + elif ('attention.attention.query' in self_param_name or 'attention.attention.key' in self_param_name or \ + 'attention.attention.value' in self_param_name) and ('bias' in self_param_name): + ss = self_param_name.split('.') + + fm_qkv_name = '.'.join(ss[0: -1]) + '.fc.bias' + return get_parameter(fm, fm_qkv_name) + + elif 'intermediate.dense' in self_param_name and 'weight' in self_param_name: + fm_param_name = self_param_name.replace('.linear', '') + return get_parameter(fm, fm_param_name) + elif 'output.dense' in self_param_name and 'weight' in self_param_name: + fm_param_name = self_param_name.replace('.linear', '') + return get_parameter(fm, fm_param_name) + else: + #return get_parameter(fm, self_param_name) + return None + + +if __name__ == '__main__': + from utils.dl.common.env import set_random_seed + set_random_seed(1) + + # 1. init model + from dnns.vit import vit_b_16 + fm_models_dict_path = 'new_impl/cv/beit/results/beit_fbs.py/20231028/999996-170849-/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/new_impl/cv/beit/beit_fbs.py/models/fm_best.pt' + fm_models_dict_path = save_models_dict_for_init(torch.load(fm_models_dict_path), __file__, 'fm_beit_cls_lora') + pretrained_md_models_dict_path = 'new_impl/cv/beit/results/beit_fbs.py/20231028/999996-170849-/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/new_impl/cv/beit/beit_fbs.py/models/md_best.pt' + md_models_dict = torch.load(pretrained_md_models_dict_path) + md_models_dict_path = save_models_dict_for_init(md_models_dict, __file__, 'md_beit_cls_pretrained_wo_fbs') + torch.cuda.set_device(1) + device = 'cuda' + + fm_model = ElasticDNN_beit_OfflineClsFMModel('fm', fm_models_dict_path, device) + md_model = ElasticDNN_beit_OfflineClsMDModel('md', md_models_dict_path, device) + + # 2. init alg + models = { + 'fm': fm_model, + 'md': md_model + } + from new_impl.cv.elasticdnn.api.algs.md_pretraining_index_v2_train_index_and_md import ElasticDNN_MDPretrainingIndexAlg + + fm_to_md_alg = ElasticDNN_MDPretrainingIndexAlg(models, get_res_save_dir(__file__, sys.argv[0])) + + # 3. init scenario + scenario = build_scenario( + source_datasets_name=['GTA5Cls', 'SuperviselyPersonCls'], + target_datasets_order=['CityscapesCls', 'BaiduPersonCls'] * 15, + da_mode='close_set', + data_dirs={ + 'GTA5Cls': '/data/zql/datasets/gta5_for_cls_task', + 'SuperviselyPersonCls': '/data/zql/datasets/supervisely_person_for_cls_task', + 'CityscapesCls': '/data/zql/datasets/cityscapes_for_cls_task', + 'BaiduPersonCls': '/data/zql/datasets/baidu_person_for_cls_task' + }, + ) + + from utils.dl.common.lr_scheduler import get_linear_schedule_with_warmup + fm_to_md_alg.run(scenario, hyps={ + 'launch_tbboard': False, + + 'samples_size': (1, 3, 224, 224), + + 'FBS_r': 16, + 'FBS_ignore_layers': [], + + 'train_batch_size': 128, + 'val_batch_size': 512, + 'num_workers': 16, + 'optimizer': 'AdamW', + # 'optimizer_args': {'lr': 1e-4, 'betas': [0.9, 0.999], 'weight_decay': 0.01}, + 'indexes_optimizer_args': {'lr': 3e-3, 'betas': [0.9, 0.999], 'weight_decay': 0.1}, + # 'scheduler': 'StepLR', + # 'scheduler_args': {'step_size': 20000, 'gamma': 0.1}, + # 'optimizer': 'AdamW', + # 'optimizer_args': {'lr': 1e-4, 'betas': [0.9, 0.999], 'weight_decay': 0.01}, + 'optimizer_args': {'lr': 1e-3, 'betas': [0.9, 0.999], 'weight_decay': 0.01},#注意学习率的调整，不同的模型不一样。 + + 'scheduler': 'LambdaLR', + 'scheduler_args': {'lr_lambda': get_linear_schedule_with_warmup(10000, 70000)}, + + 'max_sparsity': 0.9, + 'min_sparsity': 0.0, + 'num_iters': 60000, + 'val_freq': 100, + 'index_loss_weight': 1e-4, + 'l1_reg_loss_weight': 1e-9, + 'val_num_sparsities': 4, + + 'bn_cal_num_iters': 0,#有bn层注意需要加上这个 + + 'index_init': 'zero', + 'index_guided_linear_comb_split_size': 512 + }) + \ No newline at end of file diff --git a/new_impl/cv/beit/beit_model/config.json b/new_impl/cv/beit/beit_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..271de77c82061fc217266b1ff34a4840de794c93 --- /dev/null +++ b/new_impl/cv/beit/beit_model/config.json @@ -0,0 +1,28 @@ +{ + "architectures": [ + "BeitForMaskedImageModeling" + ], + "attention_probs_dropout_prob": 0.0, + "drop_path_rate": 0.1, + "hidden_act": "gelu", + "hidden_dropout_prob": 0.0, + "hidden_size": 768, + "image_size": 224, + "initializer_range": 0.02, + "intermediate_size": 3072, + "layer_norm_eps": 1e-12, + "layer_scale_init_value": 0.1, + "model_type": "beit", + "num_attention_heads": 12, + "num_channels": 3, + "num_hidden_layers": 12, + "patch_size": 16, + "torch_dtype": "float32", + "transformers_version": "4.11.0.dev0", + "use_absolute_position_embeddings": false, + "use_mask_token": true, + "use_mean_pooling": true, + "use_relative_position_bias": false, + "use_shared_relative_position_bias": true, + "vocab_size": 8192 +} diff --git a/new_impl/cv/beit/beit_online.py b/new_impl/cv/beit/beit_online.py new file mode 100644 index 0000000000000000000000000000000000000000..6c736d95ba1c2389c2b2839a02fcdcfdb18c39ca --- /dev/null +++ b/new_impl/cv/beit/beit_online.py @@ -0,0 +1,347 @@ +from typing import List +from data.dataloader import build_dataloader +# from methods.elasticdnn.api.online_model import ElasticDNN_OnlineModel +from new_impl.cv.elasticdnn.api.online_model_v2 import ElasticDNN_OnlineModel + +import torch +import sys +from torch import nn +from new_impl.cv.elasticdnn.api.model import ElasticDNN_OfflineSegFMModel, ElasticDNN_OfflineSegMDModel +from new_impl.cv.elasticdnn.api.algs.md_pretraining_wo_fbs import ElasticDNN_MDPretrainingWoFBSAlg +from new_impl.cv.elasticdnn.model.base import ElasticDNNUtil +from new_impl.cv.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from new_impl.cv.elasticdnn.pipeline.offline.fm_to_md.vit import FM_to_MD_ViT_Util +from new_impl.cv.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from new_impl.cv.elasticdnn.pipeline.offline.fm_lora.vit import FMLoRA_ViT_Util +from beit import ElasticbeitUtil +from utils.common.file import ensure_dir +from utils.dl.common.model import LayerActivation, get_module, get_parameter +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +from utils.dl.common.loss import CrossEntropyLossSoft +import torch.nn.functional as F +from utils.dl.common.env import create_tbwriter +import os +from utils.common.log import logger +from utils.common.data_record import write_json +# from methods.shot.shot import OnlineShotModel +from new_impl.cv.feat_align.main import OnlineFeatAlignModel, FeatAlignAlg +import tqdm +from new_impl.cv.feat_align.mmd import mmd_rbf +from new_impl.cv.utils.elasticfm_da import init_online_model, elasticfm_da +torch.cuda.set_device(1) +device = 'cuda' +app_name = 'cls' +sd_sparsity = 0.8 + +settings = { + 'involve_fm': True +} + +scenario = build_scenario( + source_datasets_name=['GTA5Cls', 'SuperviselyPersonCls'], + target_datasets_order=['CityscapesCls', 'BaiduPersonCls'] * 15, + da_mode='close_set', + data_dirs={ + 'GTA5Cls': '/data/zql/datasets/gta5_for_cls_task', + 'SuperviselyPersonCls': '/data/zql/datasets/supervisely_person_for_cls_task', + 'CityscapesCls': '/data/zql/datasets/cityscapes_for_cls_task', + 'BaiduPersonCls': '/data/zql/datasets/baiduperson_for_cls_task' + }, +) + + +class ElasticDNN_ClsOnlineModel(ElasticDNN_OnlineModel): + def get_accuracy(self, test_loader, *args, **kwargs): + acc = 0 + sample_num = 0 + + self.to_eval_mode() + + with torch.no_grad(): + pbar = tqdm.tqdm(enumerate(test_loader), total=len(test_loader), dynamic_ncols=True, leave=False) + for batch_index, (x, y) in pbar: + x, y = x.to(self.device), y.to(self.device) + output = self.infer(x) + pred = F.softmax(output, dim=1).argmax(dim=1) + correct = torch.eq(pred, y).sum().item() + acc += correct + sample_num += len(y) + + pbar.set_description(f'cur_batch_total: {len(y)}, cur_batch_correct: {correct}, ' + f'cur_batch_acc: {(correct / len(y)):.4f}') + + acc /= sample_num + return acc + + def get_elastic_dnn_util(self) -> ElasticDNNUtil: + return ElasticbeitUtil() + + def get_fm_matched_param_of_md_param(self, md_param_name): + # only between qkv.weight, norm.weight/bias + self_param_name = md_param_name + fm = self.models_dict['fm'] + # if any([k in self_param_name for k in ['fbs', 'cls_token', 'pos_embed']]): + # return None + + # p = get_parameter(self.models_dict['md'], self_param_name) + # if p.dim() == 0: + # return None + # elif p.dim() == 1 and 'norm' in self_param_name and 'weight' in self_param_name: + # return get_parameter(fm, self_param_name) + + if any([k in self_param_name for k in ['fbs', 'cls_token', 'pos_embed']]): + return None + + p = get_parameter(self.models_dict['md'], self_param_name) + if p.dim() == 0: + return None + elif p.dim() == 1 and 'layernorm' in self_param_name and 'weight' in self_param_name: + return get_parameter(fm, self_param_name) + # 1. xx.qkv.to_qkv.yy to xx.qkv.qkv.aa and xx.qkv.abs.zz + # if 'qkv.weight' in self_param_name: + # ss = self_param_name.split('.') + + # fm_qkv_name = '.'.join(ss[0: -1]) + '.qkv' + # fm_qkv = get_module(fm, fm_qkv_name) + + # fm_abs_name = '.'.join(ss[0: -1]) + '.abs' + # fm_abs = get_module(fm, fm_abs_name) + + # # NOTE: unrecoverable operation! multiply LoRA parameters to allow it being updated in update_fm_param() + # # TODO: if fm will be used for inference, _mul_lora_weight will not be applied! + # if not hasattr(fm_abs, '_mul_lora_weight'): + # logger.debug(f'set _mul_lora_weight in {fm_abs_name}') + # setattr(fm_abs, '_mul_lora_weight', + # nn.Parameter(torch.cat([(_abs[0].weight.T @ _abs[1].weight.T).T for _abs in fm_abs], dim=0))) + + # return torch.cat([ + # fm_qkv.weight.data, # task-agnositc params + # fm_abs._mul_lora_weight.data # task-specific params (LoRA) + # ], dim=0) + + # # elif 'to_qkv.bias' in self_param_name: + # # ss = self_param_name.split('.') + + # # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv.bias' + # # return get_parameter(fm, fm_qkv_name) + + # elif 'mlp.fc1' in self_param_name and 'weight' in self_param_name: + # fm_param_name = self_param_name.replace('.linear', '') + # return get_parameter(fm, fm_param_name) + + # elif 'mlp.fc2' in self_param_name and 'weight' in self_param_name: + # fm_param_name = self_param_name + # return get_parameter(fm, fm_param_name) + + # else: + # # return get_parameter(fm, self_param_name) + # return None + if ('attention.attention.query' in self_param_name or 'attention.attention.key' in self_param_name or \ + 'attention.attention.value' in self_param_name) and ('weight' in self_param_name): + ss = self_param_name.split('.') + + fm_qkv_name = '.'.join(ss[0: -1]) + '.fc' + fm_qkv = get_module(fm, fm_qkv_name) + + fm_abs_name = '.'.join(ss[0: -1]) + '.ab' + fm_abs = get_module(fm, fm_abs_name) + + if not hasattr(fm_abs, '_mul_lora_weight'): + logger.debug(f'set _mul_lora_weight in {fm_abs_name}') + setattr(fm_abs, '_mul_lora_weight', + nn.Parameter(fm_abs[1].weight @ fm_abs[0].weight)) + + return torch.cat([ + fm_qkv.weight.data, # task-agnositc params + fm_abs._mul_lora_weight.data # task-specific params (LoRA) + ], dim=0) + + elif ('attention.attention.query' in self_param_name or 'attention.attention.key' in self_param_name or \ + 'attention.attention.value' in self_param_name) and ('bias' in self_param_name): + ss = self_param_name.split('.') + + fm_qkv_name = '.'.join(ss[0: -1]) + '.fc.bias' + return get_parameter(fm, fm_qkv_name) + + elif 'intermediate.dense' in self_param_name and 'weight' in self_param_name: + fm_param_name = self_param_name.replace('.linear', '') + return get_parameter(fm, fm_param_name) + elif 'output.dense' in self_param_name and 'weight' in self_param_name: + fm_param_name = self_param_name.replace('.linear', '') + return get_parameter(fm, fm_param_name) + else: + #return get_parameter(fm, self_param_name) + return None + + + def update_fm_param(self, md_param_name, cal_new_fm_param_by_md_param): + if not ('attention.attention.query.weight' in md_param_name or 'attention.attention.key.weight' in md_param_name or 'attention.attention.value.weight' in md_param_name): + matched_fm_param_ref = self.get_fm_matched_param_of_md_param(md_param_name) + matched_fm_param_ref.copy_(cal_new_fm_param_by_md_param) + else: + new_fm_attn_weight, new_fm_lora_weight = torch.chunk(cal_new_fm_param_by_md_param, 2, 0) + ss = md_param_name.split('.') + fm = self.models_dict['fm'] + # update task-agnostic parameters + fm_qkv_name = '.'.join(ss[0: -1]) + '.fc' + fm_qkv = get_module(fm, fm_qkv_name) + fm_qkv.weight.data.copy_(new_fm_attn_weight) + + # update task-specific parameters + fm_abs_name = '.'.join(ss[0: -1]) + '.ab' + fm_abs = get_module(fm, fm_abs_name) + fm_abs._mul_lora_weight.data.copy_(new_fm_lora_weight) # TODO: this will not be applied in inference! + + def get_md_matched_param_of_fm_param(self, fm_param_name): + return super().get_md_matched_param_of_fm_param(fm_param_name) + + def get_md_matched_param_of_sd_param(self, sd_param_name): + # raise NotImplementedError + + # only between qkv.weight, norm.weight/bias + self_param_name = sd_param_name + md = self.models_dict['md'] + if any([k in self_param_name for k in ['fbs', 'ab', 'embeddings']]): + return None + + p = get_parameter(self.models_dict['sd'], self_param_name) + if p.dim() == 0: + return None + elif p.dim() == 1 and ('LayerNorm' in self_param_name or 'layernorm' in self_param_name) and 'weight' in self_param_name: + return get_parameter(md, self_param_name) + + # 1. xx.qkv.to_qkv.yy to xx.qkv.qkv.aa and xx.qkv.abs.zz + if ('query' in self_param_name or 'key' in self_param_name or \ + 'value' in self_param_name) and ('weight' in self_param_name): + + + return get_parameter(md, self_param_name) # NOTE: no fbs in qkv! + + # elif 'to_qkv.bias' in self_param_name: + # ss = self_param_name.split('.') + + # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv.bias' + # return get_parameter(fm, fm_qkv_name) + + elif 'intermediate.dense.0.weight' in self_param_name: + fm_param_name = '.'.join(self_param_name.split('.')[0: -2]) + '.linear.weight' + return get_parameter(md, fm_param_name) + + elif 'output.dense' in self_param_name and 'weight' in self_param_name: + fm_param_name = self_param_name + return get_parameter(md, fm_param_name) + + else: + # return get_parameter(fm, self_param_name) + return None + + def get_task_head_params(self): + head = get_module(self.models_dict['sd'], 'classifier') + return list(head.parameters()) + + + +class ClsOnlineFeatAlignModel(OnlineFeatAlignModel): + def get_trained_params(self): # TODO: elastic fm only train a part of params + #qkv_and_norm_params = [p for n, p in self.models_dict['main'].named_parameters() if 'attention.attention.projection_query' in n or 'attention.attention.projection_key' in n or 'attention.attention.projection_value' in n or 'intermediate.dense' in n or 'output.dense' in n] + qkv_and_norm_params = [p for n, p in self.models_dict['main'].named_parameters()] + return qkv_and_norm_params + + def get_feature_hook(self): + return LayerActivation(get_module(self.models_dict['main'], 'classifier'), False, self.device) + + def forward_to_get_task_loss(self, x, y): + return F.cross_entropy(self.infer(x), y) + + def get_mmd_loss(self, f1, f2): + return mmd_rbf(f1, f2) + + def infer(self, x, *args, **kwargs): + return self.models_dict['main'](x).logits + + def get_accuracy(self, test_loader, *args, **kwargs): + acc = 0 + sample_num = 0 + + self.to_eval_mode() + + with torch.no_grad(): + pbar = tqdm.tqdm(enumerate(test_loader), total=len(test_loader), dynamic_ncols=True, leave=False) + for batch_index, (x, y) in pbar: + x, y = x.to(self.device), y.to(self.device) + output = self.infer(x) + pred = F.softmax(output, dim=1).argmax(dim=1) + correct = torch.eq(pred, y).sum().item() + acc += correct + sample_num += len(y) + + pbar.set_description(f'cur_batch_total: {len(y)}, cur_batch_correct: {correct}, ' + f'cur_batch_acc: {(correct / len(y)):.4f}') + + acc /= sample_num + return acc + + + + +#from new_impl.cv.model import ElasticDNN_ClsOnlineModel +elasticfm_model = ElasticDNN_ClsOnlineModel('cls', init_online_model( + # 'experiments/elasticdnn/vit_b_16/offline/fm_to_md/results/cls_md_index.py/20230529/star_999997-154037-only_prune_mlp/models/fm_best.pt', + # 'experiments/elasticdnn/vit_b_16/offline/fm_to_md/results/cls_md_index.py/20230529/star_999997-154037-only_prune_mlp/models/md_best.pt', + #'experiments/elasticdnn/vit_b_16/offline/fm_to_md/cls/results/cls_md_index.py/20230617/999992-101343-lr1e-5_index_bug_fixed/models/fm_best.pt', + #'experiments/elasticdnn/vit_b_16/offline/fm_to_md/cls/results/cls_md_index.py/20230617/999992-101343-lr1e-5_index_bug_fixed/models/md_best.pt', + 'new_impl/cv/beit/results/beit_index.py/20231028/999999-180237-/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/new_impl/cv/beit/beit_index.py/models/fm_best.pt', + 'new_impl/cv/beit/results/beit_index.py/20231028/999999-180237-/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/new_impl/cv/beit/beit_index.py/models/md_best.pt', + 'cls', __file__ +), device, { + 'md_to_fm_alpha': 1, + 'fm_to_md_alpha': 1 +}) + +da_alg = FeatAlignAlg +#from new_impl.cv.model import ClsOnlineFeatAlignModel +da_model = ClsOnlineFeatAlignModel +da_alg_hyp = { + 'CityscapesCls': { + 'train_batch_size': 64, + 'val_batch_size': 512, + 'num_workers': 8, + 'optimizer': 'AdamW', + 'optimizer_args': {'lr': 4e-3/2, 'betas': [0.9, 0.999], 'weight_decay': 0.01},#针对于cvt的online的学习率 + #'optimizer_args': {'lr': 1e-4/2, 'momentum': 0.9}, + 'scheduler': '', + 'scheduler_args': {}, + 'num_iters': 100, + 'val_freq': 20, + 'sd_sparsity':sd_sparsity , + 'feat_align_loss_weight': 3.0 + }, + 'BaiduPersonCls': { + 'train_batch_size': 64, + 'val_batch_size': 512, + 'num_workers': 8, + 'optimizer': 'SGD', + 'optimizer_args': {'lr': 1e-3/2, 'momentum': 0.9}, + 'scheduler': '', + 'scheduler_args': {}, + 'num_iters': 100, + 'val_freq': 20, + 'sd_sparsity': sd_sparsity, + 'feat_align_loss_weight': 0.3 + } +} + + +elasticfm_da( + [app_name], + [scenario], + [elasticfm_model], + [da_alg], + [da_alg_hyp], + [da_model], + device, + settings, + __file__, + sys.argv[0] +) diff --git a/new_impl/cv/beit/tmp.py b/new_impl/cv/beit/tmp.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/new_impl/cv/clip/clip.py b/new_impl/cv/clip/clip.py new file mode 100644 index 0000000000000000000000000000000000000000..82ae6c2ddad9ba1b93454b41859ef308f93b74a5 --- /dev/null +++ b/new_impl/cv/clip/clip.py @@ -0,0 +1,1174 @@ +from transformers import BlipForQuestionAnswering, BlipConfig,BlipModel,CLIPVisionModel,CLIPConfig +import torch +from torch import nn +from abc import ABC, abstractmethod +from copy import deepcopy +from typing import Optional, Union +from einops import rearrange, repeat +from einops.layers.torch import Rearrange +import tqdm + +from utils.dl.common.model import get_model_device, get_model_latency, get_model_size, set_module +from utils.dl.common.model import set_module, get_module, get_super_module +from utils.common.log import logger +from new_impl.cv.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util, LoRA +from transformers.models.blip.modeling_blip import BlipAttention +#from transformers.models.blip.modeling_blip_text import BlipTextSelfAttention,BlipTextAttention,BlipTextSelfOutput +from transformers.models.beit.modeling_beit import BeitSelfAttention,BeitConfig +from transformers.models.clip.modeling_clip import CLIPAttention +from new_impl.cv.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from new_impl.cv.elasticdnn.model.base import Abs, KTakesAll, ElasticDNNUtil, Layer_WrappedWithFBS + +from typing import Optional, Tuple +import math + +# def blip(num_classes): +# model = BlipForQuestionAnswering.from_pretrained('new_impl/mm/Vis_bert/QuestionAnswering/VisBert_pretrained') +# # linear = model.text_decoder.cls.predictions.decoder +# # new_linear = nn.Linear(linear.in_features,30524,bias = True) +# # set_module(model,'text_decoder.cls.predictions.decoder',new_linear) +# return model +# def blip(num_classes): +# model = BlipForQuestionAnswering.from_pretrained('new_impl/mm/Vis_bert/QuestionAnswering/VisBert_pretrained') +# linear = model.text_decoder.cls.predictions.decoder +# new_linear = nn.Linear(linear.in_features,num_classes,bias = True) +# set_module(model,'text_decoder.cls.predictions.decoder',new_linear) +# return model +class clip(nn.Module): + def __init__(self,num_classes): + super(clip,self).__init__() + self.clip = CLIPVisionModel.from_pretrained('new_impl/cv/clip/pretrained_model') + self.classifier = nn.Linear(768,num_classes) + + def forward(self,sample): + output = self.clip(sample)[-1]#output the last hidden + output = self.classifier(output) + return output + + +class ToQKV_WrappedWithLoRA(nn.Module): + def __init__(self, fc: nn.Linear, ab_r: int): + super(ToQKV_WrappedWithLoRA, self).__init__() + + self.fc = fc + self.ab = self.create_ab_as_linear(fc.weight.data, ab_r) + + def create_ab_as_linear(self, fc_weight: torch.Tensor, ab_r: int): + res = nn.Sequential( + LoRA(fc_weight.size(1), fc_weight.size(0) // ab_r, bias=False), + LoRA(fc_weight.size(0) // ab_r, fc_weight.size(0), bias=False) + ).to(fc_weight.device) + nn.init.kaiming_uniform_(res[0].weight, a=5 ** 0.5) + nn.init.zeros_(res[1].weight) + return res + + def forward(self, x): + x1 = self.fc(x) + x2 = self.ab(x) + return x1 + x2 + + +class FMLoRA_clip_Util(FMLoRA_Util): + + @torch.no_grad() + def add_lora_ab_to_fm(self, fm: nn.Module, ab_r: int, samples: torch.Tensor): + fm.eval() + + # print(samples) + # for k, v in samples.items(): + # if isinstance(v, torch.Tensor): + # samples[k] = v.to(get_model_device(fm)) + + #o1 = fm.generate(**samples) + o1 = fm(samples) + for name, module in fm.named_modules(): + if name.endswith(('query', 'key', 'value')): + set_module(fm, name, ToQKV_WrappedWithLoRA(module, ab_r)) + elif name.endswith('.qkv'): + set_module(fm, name, ToQKV_WrappedWithLoRA(module, ab_r)) + elif name.endswith(('k_proj','q_proj','v_proj')): + set_module(fm, name, ToQKV_WrappedWithLoRA(module, ab_r)) + + + #o2 = fm.generate(**samples) + o2 = fm(samples) + if isinstance(o1, tuple): + o1 = o1[-1] + o2 = o2[-1] + output_diff = ((o1 - o2) ** 2).sum() + assert output_diff < 1e-5 + return fm + + @torch.no_grad() + def absorb_lora_and_recover_net_structure(self, fm: nn.Module, samples: torch.Tensor): + fm.eval() + # print('absorb lora before') + + # for k, v in samples.items(): + # if isinstance(v, torch.Tensor): + # samples[k] = v.to(get_model_device(fm)) + + o1 = fm(samples) + + for name, module in fm.named_modules(): + if not isinstance(module, ToQKV_WrappedWithLoRA): + continue + + fc = module.fc + ab = module.ab + + fc.weight.add_(ab[1].weight @ ab[0].weight) + + set_module(fm, name, fc) + + # print('absorb lora after') + o2 = fm(samples) + + if isinstance(o1, tuple): + o1 = o1[-1] + o2 = o2[-1] + output_diff = ((o1 - o2) ** 2).sum() + assert output_diff < 1e-6, output_diff + + return fm + + +####Here start with Fbs + +# class blipTextAttentionPrunable(BlipTextSelfAttention): +# def __init__(self,is_cross_attention): +# config = BlipConfig.from_pretrained('new_impl/mm/Vis_bert/QuestionAnswering/VisBert_pretrained') +# super(blipTextAttentionPrunable,self).__init__(config.text_config,is_cross_attention) + +# def save_attn_gradients(self, attn_gradients): +# self.attn_gradients = attn_gradients + +# def get_attn_gradients(self): +# return self.attn_gradients + +# def save_attention_map(self, attention_map): +# self.attention_map = attention_map + +# def get_attention_map(self): +# return self.attention_map + +# def transpose_for_scores(self, x): +# new_x_shape = x.size()[:-1] + (self.num_attention_heads, -1) +# x = x.view(*new_x_shape) +# return x.permute(0, 2, 1, 3) + +# def forward( +# self, +# hidden_states: torch.Tensor, +# attention_mask: Optional[torch.FloatTensor] = None, +# head_mask: Optional[torch.FloatTensor] = None, +# encoder_hidden_states: Optional[torch.FloatTensor] = None, +# encoder_attention_mask: Optional[torch.FloatTensor] = None, +# past_key_value: Optional[Tuple[Tuple[torch.FloatTensor]]] = None, +# output_attentions: Optional[bool] = False, +# ) -> Tuple[torch.Tensor]: +# mixed_query_layer = self.query(hidden_states) + +# # If this is instantiated as a cross-attention module, the keys +# # and values come from an encoder; the attention mask needs to be +# # such that the encoder's padding tokens are not attended to. +# is_cross_attention = encoder_hidden_states is not None + +# if is_cross_attention: +# key_layer = self.transpose_for_scores(self.key(encoder_hidden_states)) +# value_layer = self.transpose_for_scores(self.value(encoder_hidden_states)) +# attention_mask = encoder_attention_mask +# elif past_key_value is not None: +# key_layer = self.transpose_for_scores(self.key(hidden_states)) +# value_layer = self.transpose_for_scores(self.value(hidden_states)) +# key_layer = torch.cat([past_key_value[0], key_layer], dim=2) +# value_layer = torch.cat([past_key_value[1], value_layer], dim=2) +# else: +# key_layer = self.transpose_for_scores(self.key(hidden_states)) +# value_layer = self.transpose_for_scores(self.value(hidden_states)) + +# query_layer = self.transpose_for_scores(mixed_query_layer) + +# past_key_value = (key_layer, value_layer) + +# # Take the dot product between "query" and "key" to get the raw attention scores. +# attention_scores = torch.matmul(query_layer, key_layer.transpose(-1, -2)) + +# if self.position_embedding_type == "relative_key" or self.position_embedding_type == "relative_key_query": +# seq_length = hidden_states.size()[1] +# position_ids_l = torch.arange(seq_length, dtype=torch.long, device=hidden_states.device).view(-1, 1) +# position_ids_r = torch.arange(seq_length, dtype=torch.long, device=hidden_states.device).view(1, -1) +# distance = position_ids_l - position_ids_r +# positional_embedding = self.distance_embedding(distance + self.max_position_embeddings - 1) +# positional_embedding = positional_embedding.to(dtype=query_layer.dtype) # fp16 compatibility + +# if self.position_embedding_type == "relative_key": +# relative_position_scores = torch.einsum("bhld,lrd->bhlr", query_layer, positional_embedding) +# attention_scores = attention_scores + relative_position_scores +# elif self.position_embedding_type == "relative_key_query": +# relative_position_scores_query = torch.einsum("bhld,lrd->bhlr", query_layer, positional_embedding) +# relative_position_scores_key = torch.einsum("bhrd,lrd->bhlr", key_layer, positional_embedding) +# attention_scores = attention_scores + relative_position_scores_query + relative_position_scores_key + +# attention_scores = attention_scores / math.sqrt(self.attention_head_size) +# if attention_mask is not None: +# # Apply the attention mask is (precomputed for all layers in BlipTextModel forward() function) +# attention_scores = attention_scores + attention_mask.to(attention_scores.device) + +# # Normalize the attention scores to probabilities. +# attention_probs = nn.Softmax(dim=-1)(attention_scores) + +# # This is actually dropping out entire tokens to attend to, which might +# # seem a bit unusual, but is taken from the original Transformer paper. +# attention_probs_dropped = self.dropout(attention_probs) + +# # Mask heads if we want to +# if head_mask is not None: +# attention_probs_dropped = attention_probs_dropped * head_mask + +# context_layer = torch.matmul(attention_probs_dropped, value_layer) + +# context_layer = context_layer.permute(0, 2, 1, 3).contiguous() +# new_context_layer_shape = context_layer.size()[:-2] + (-1,) +# context_layer = context_layer.view(*new_context_layer_shape) + +# outputs = (context_layer, attention_probs) if output_attentions else (context_layer,) + +# outputs = outputs + (past_key_value,) +# return outputs +# @staticmethod +# def init_from_exist_self_attn(attn: BlipTextSelfAttention,is_cross_attention): +# # print(attn) + +# res = blipTextAttentionPrunable(is_cross_attention) + +# for attr in dir(attn): +# # if str(attr) in ['transpose_for_scores'] or str(attr).startswith('_'): +# # continue +# # if isinstance(getattr(attn, attr), nn.Module): +# # print(attr) + +# if isinstance(getattr(attn, attr), nn.Module): +# try: +# # print(attr, 'ok') +# setattr(res, attr, getattr(attn, attr)) + +# except Exception as e: +# print(attr, str(e)) + + + +# return res + + + +# class blipSelfTextAttentionPrunable(BlipTextAttention): +# def __init__(self, config, is_cross_attention=False): +# self.self = blipTextAttentionPrunable(config, is_cross_attention) +# self.output = BlipTextSelfOutput(config) +# self.pruned_heads = set() +# super(blipSelfTextAttentionPrunable,self).__init__(config) + +# def prune_heads(self, heads): +# if len(heads) == 0: +# return +# heads, index = find_pruneable_heads_and_indices( +# heads, self.self.num_attention_heads, self.self.attention_head_size, self.pruned_heads +# ) + +# # Prune linear layers +# self.self.query = prune_linear_layer(self.self.query, index) +# self.self.key = prune_linear_layer(self.self.key, index) +# self.self.value = prune_linear_layer(self.self.value, index) +# self.output.dense = prune_linear_layer(self.output.dense, index, dim=1) + +# # Update hyper params and store pruned heads +# self.self.num_attention_heads = self.self.num_attention_heads - len(heads) +# self.self.all_head_size = self.self.attention_head_size * self.self.num_attention_heads +# self.pruned_heads = self.pruned_heads.union(heads) + +# def forward( +# self, +# hidden_states: torch.Tensor, +# attention_mask: Optional[torch.FloatTensor] = None, +# head_mask: Optional[torch.FloatTensor] = None, +# encoder_hidden_states: Optional[torch.FloatTensor] = None, +# encoder_attention_mask: Optional[torch.FloatTensor] = None, +# past_key_value: Optional[Tuple[Tuple[torch.FloatTensor]]] = None, +# output_attentions: Optional[bool] = False, +# ) -> Tuple[torch.Tensor]: +# self_outputs = self.self( +# hidden_states, +# attention_mask, +# head_mask, +# encoder_hidden_states, +# encoder_attention_mask, +# past_key_value, +# output_attentions, +# ) +# attention_output = self.output(self_outputs[0], hidden_states) +# outputs = (attention_output,) + self_outputs[1:] # add attentions if we output them +# return outputs +# @staticmethod +# def init_from_exist_self_attn(attn: BlipTextAttention,config,is_cross_attention): +# # print(attn) + +# res = blipTextAttentionPrunable(config,is_cross_attention) + +# for attr in dir(attn): +# # if str(attr) in ['transpose_for_scores'] or str(attr).startswith('_'): +# # continue +# # if isinstance(getattr(attn, attr), nn.Module): +# # print(attr) + +# if isinstance(getattr(attn, attr), nn.Module): +# try: +# # print(attr, 'ok') +# setattr(res, attr, getattr(attn, attr)) + +# except Exception as e: +# print(attr, str(e)) + + + +# return res + + + + + + +# class blipSelfAttentionPrunable(BlipAttention): +# def __init__(self): +# config = BlipConfig.from_pretrained('new_impl/mm/Vis_bert/QuestionAnswering/VisBert_pretrained') +# super(blipSelfAttentionPrunable, self).__init__(config.vision_config) + +# def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int): +# return tensor.view(bsz, seq_len, self.num_heads, -1).transpose(1, 2).contiguous() + +# def forward( +# self, +# hidden_states: torch.Tensor, +# head_mask: Optional[torch.Tensor] = None, +# output_attentions: Optional[bool] = False, +# ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: +# """Input shape: Batch x Time x Channel""" + +# bsz, tgt_len, embed_dim = hidden_states.size() + +# mixed_qkv = ( +# self.qkv(hidden_states) +# .reshape(bsz, tgt_len, 3, self.num_heads, -1) +# .permute(2, 0, 3, 1, 4) +# ) +# query_states, key_states, value_states = mixed_qkv[0], mixed_qkv[1], mixed_qkv[2] + +# # Take the dot product between "query" and "key" to get the raw attention scores. +# attention_scores = torch.matmul(query_states, key_states.transpose(-1, -2)) + +# attention_scores = attention_scores * self.scale + +# # Normalize the attention scores to probabilities. +# attention_probs = nn.functional.softmax(attention_scores, dim=-1) + +# # This is actually dropping out entire tokens to attend to, which might +# # seem a bit unusual, but is taken from the original Transformer paper. +# attention_probs = self.dropout(attention_probs) + +# # Mask heads if we want to +# if head_mask is not None: +# attention_probs = attention_probs * head_mask + +# context_layer = torch.matmul(attention_probs, value_states).permute(0, 2, 1, 3) + +# new_context_layer_shape = context_layer.size()[:-2] + (-1,) +# context_layer = context_layer.reshape(new_context_layer_shape) + +# output = self.projection(context_layer) + +# outputs = (output, attention_probs) if output_attentions else (output, None) + +# return outputs + +# @staticmethod +# def init_from_exist_self_attn(attn: BlipAttention): +# # print(attn) + +# res = blipSelfAttentionPrunable() + +# for attr in dir(attn): +# # if str(attr) in ['transpose_for_scores'] or str(attr).startswith('_'): +# # continue +# # if isinstance(getattr(attn, attr), nn.Module): +# # print(attr) + +# if isinstance(getattr(attn, attr), nn.Module): +# try: +# # print(attr, 'ok') +# setattr(res, attr, getattr(attn, attr)) + +# except Exception as e: +# print(attr, str(e)) + + + +# return res +class CLIPAttentionPrunable(CLIPAttention): + def __init__(self, config: CLIPConfig, ratio:int) -> None: + config = CLIPConfig.from_pretrained('new_impl/cv/clip/pretrained_model') + super(CLIPAttentionPrunable, self).__init__(config.vision_config) + self.head_dim = self.head_dim // ratio + + def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int): + return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous() + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + causal_attention_mask: Optional[torch.Tensor] = None, + output_attentions: Optional[bool] = False, + ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: + """Input shape: Batch x Time x Channel""" + + bsz, tgt_len, embed_dim = hidden_states.size() + + # get query proj + query_states = self.q_proj(hidden_states) * self.scale + key_states = self._shape(self.k_proj(hidden_states), -1, bsz) + value_states = self._shape(self.v_proj(hidden_states), -1, bsz) + + proj_shape = (bsz * self.num_heads, -1, self.head_dim) + query_states = self._shape(query_states, tgt_len, bsz).view(*proj_shape) + key_states = key_states.view(*proj_shape) + value_states = value_states.view(*proj_shape) + + src_len = key_states.size(1) + attn_weights = torch.bmm(query_states, key_states.transpose(1, 2)) + + if attn_weights.size() != (bsz * self.num_heads, tgt_len, src_len): + raise ValueError( + f"Attention weights should be of size {(bsz * self.num_heads, tgt_len, src_len)}, but is" + f" {attn_weights.size()}" + ) + + # apply the causal_attention_mask first + if causal_attention_mask is not None: + if causal_attention_mask.size() != (bsz, 1, tgt_len, src_len): + raise ValueError( + f"Attention mask should be of size {(bsz, 1, tgt_len, src_len)}, but is" + f" {causal_attention_mask.size()}" + ) + attn_weights = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + causal_attention_mask + attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len) + + if attention_mask is not None: + if attention_mask.size() != (bsz, 1, tgt_len, src_len): + raise ValueError( + f"Attention mask should be of size {(bsz, 1, tgt_len, src_len)}, but is {attention_mask.size()}" + ) + attn_weights = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + attention_mask + attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len) + + attn_weights = nn.functional.softmax(attn_weights, dim=-1) + + if output_attentions: + # this operation is a bit akward, but it's required to + # make sure that attn_weights keeps its gradient. + # In order to do so, attn_weights have to reshaped + # twice and have to be reused in the following + attn_weights_reshaped = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + attn_weights = attn_weights_reshaped.view(bsz * self.num_heads, tgt_len, src_len) + else: + attn_weights_reshaped = None + + attn_probs = nn.functional.dropout(attn_weights, p=self.dropout, training=self.training) + + attn_output = torch.bmm(attn_probs, value_states) + + if attn_output.size() != (bsz * self.num_heads, tgt_len, self.head_dim): + raise ValueError( + f"`attn_output` should be of size {(bsz, self.num_heads, tgt_len, self.head_dim)}, but is" + f" {attn_output.size()}" + ) + + attn_output = attn_output.view(bsz, self.num_heads, tgt_len, self.head_dim) + attn_output = attn_output.transpose(1, 2) + attn_output = attn_output.reshape(bsz, tgt_len, -1) + + attn_output = self.out_proj(attn_output) + + return attn_output, attn_weights_reshaped + + @staticmethod + def init_from_exist_self_attn(attn: CLIPAttention,config,ratio:int): + # print(attn) + + res = CLIPAttentionPrunable(config,ratio) + + for attr in dir(attn): + # if str(attr) in ['transpose_for_scores'] or str(attr).startswith('_'): + # continue + # if isinstance(getattr(attn, attr), nn.Module): + # print(attr) + + if isinstance(getattr(attn, attr), nn.Module): + try: + # print(attr, 'ok') + setattr(res, attr, getattr(attn, attr)) + + except Exception as e: + print(attr, str(e)) + return res + +class FM_to_MD_clip_Util(FM_to_MD_Util): + def init_md_from_fm_by_reducing_width(self, fm: nn.Module, reducing_width_ratio: int) -> nn.Module: + fm_vis = deepcopy(fm) + config = CLIPConfig.from_pretrained('new_impl/cv/clip/pretrained_model') + + for block_i,block in enumerate(fm_vis.clip.vision_model.encoder.layers): + set_module(block, 'self_attn', CLIPAttentionPrunable.init_from_exist_self_attn(block.self_attn,config.vision_config,reducing_width_ratio)) + + def _f(n): + return int(n // reducing_width_ratio) + + # def _rand_indexes(n): + # return torch.randperm(n)[0: int(n // reducing_width_ratio)] + + def l1_max_indexes(p: torch.Tensor, dim=0): + assert dim in [0, 1] + assert p.dim() in [1, 2, 4] + + if dim == 1: + p = p.T + + p_norm = p.abs().contiguous().view(p.size(0), -1).sum(dim=1) + n = p.size(0) + return p_norm.argsort(descending=True)[0: int(n // reducing_width_ratio)].sort()[0] + + for block_i, block in enumerate(fm_vis.clip.vision_model.encoder.layers): + for k in ['k_proj', 'q_proj', 'v_proj']: + qkv = get_module(block, f'self_attn.{k}') + + new_qkv = nn.Linear(qkv.in_features, _f(qkv.out_features), + qkv.bias is not None, qkv.weight.device) + indexes = l1_max_indexes(qkv.weight.data, 0) + + new_qkv.weight.data.copy_(qkv.weight.data[indexes]) + if qkv.bias is not None: + new_qkv.bias.data.copy_(qkv.bias.data[indexes]) + set_module(block, f'self_attn.{k}', new_qkv) + + proj = get_module(block, f'self_attn.out_proj') + new_proj = nn.Linear(_f(proj.in_features), proj.out_features, + proj.bias is not None, proj.weight.device) + new_proj.weight.data.copy_(proj.weight.data[:, l1_max_indexes(proj.weight.data, 1)]) + if proj.bias is not None: + new_proj.bias.data.copy_(proj.bias.data) + set_module(block, f'self_attn.out_proj', new_proj) + + fc1 = get_module(block, f'mlp.fc1') + new_fc1 = nn.Linear(fc1.in_features, _f(fc1.out_features), + fc1.bias is not None, fc1.weight.device) + indexes = l1_max_indexes(fc1.weight.data, 0) + new_fc1.weight.data.copy_(fc1.weight.data[indexes]) + if fc1.bias is not None: + new_fc1.bias.data.copy_(fc1.bias.data[indexes]) + set_module(block, f'mlp.fc1', new_fc1) + + fc2 = get_module(block, f'mlp.fc2') + new_fc2 = nn.Linear(_f(fc2.in_features), fc2.out_features, + fc2.bias is not None, fc2.weight.device) + new_fc2.weight.data.copy_(fc2.weight.data[:, l1_max_indexes(fc2.weight.data, 1)]) + if fc2.bias is not None: + new_fc2.bias.data.copy_(fc2.bias.data) + set_module(block, f'mlp.fc2', new_fc2) + + + # for block_i, block in enumerate(fm_vis.text_decoder.bert.encoder.layer): + # for k in ['query', 'key', 'value']: + # qkv = get_module(block, f'crossattention.self.{k}') + + # new_qkv = nn.Linear(qkv.in_features, _f(qkv.out_features), + # qkv.bias is not None, qkv.weight.device) + # indexes = l1_max_indexes(qkv.weight.data, 0) + + # new_qkv.weight.data.copy_(qkv.weight.data[indexes]) + # if qkv.bias is not None: + # new_qkv.bias.data.copy_(qkv.bias.data[indexes]) + # set_module(block, f'crossattention.self.{k}', new_qkv) + + # proj = get_module(block, f'crossattention.output.dense') + # new_proj = nn.Linear(_f(proj.in_features), proj.out_features, + # proj.bias is not None, proj.weight.device) + # new_proj.weight.data.copy_(proj.weight.data[:, l1_max_indexes(proj.weight.data, 1)]) + # if proj.bias is not None: + # new_proj.bias.data.copy_(proj.bias.data) + # set_module(block, f'crossattention.output.dense', new_proj) + + + # for block_i, block in enumerate(fm_vis.text_encoder.encoder.layer): + # for k in ['query', 'key', 'value']: + # qkv = get_module(block, f'attention.self.{k}') + + # new_qkv = nn.Linear(qkv.in_features, _f(qkv.out_features), + # qkv.bias is not None, qkv.weight.device) + # indexes = l1_max_indexes(qkv.weight.data, 0) + + # new_qkv.weight.data.copy_(qkv.weight.data[indexes]) + # if qkv.bias is not None: + # new_qkv.bias.data.copy_(qkv.bias.data[indexes]) + # set_module(block, f'attention.self.{k}', new_qkv) + + # proj = get_module(block, f'attention.output.dense') + # new_proj = nn.Linear(_f(proj.in_features), proj.out_features, + # proj.bias is not None, proj.weight.device) + # new_proj.weight.data.copy_(proj.weight.data[:, l1_max_indexes(proj.weight.data, 1)]) + # if proj.bias is not None: + # new_proj.bias.data.copy_(proj.bias.data) + # set_module(block, f'attention.output.dense', new_proj) + + # fc1 = get_module(block, f'intermediate.dense') + # new_fc1 = nn.Linear(fc1.in_features, _f(fc1.out_features), + # fc1.bias is not None, fc1.weight.device) + # indexes = l1_max_indexes(fc1.weight.data, 0) + # new_fc1.weight.data.copy_(fc1.weight.data[indexes]) + # if fc1.bias is not None: + # new_fc1.bias.data.copy_(fc1.bias.data[indexes]) + # set_module(block, f'intermediate.dense', new_fc1) + + # fc2 = get_module(block, f'output.dense') + # new_fc2 = nn.Linear(_f(fc2.in_features), fc2.out_features, + # fc2.bias is not None, fc2.weight.device) + # new_fc2.weight.data.copy_(fc2.weight.data[:, l1_max_indexes(fc2.weight.data, 1)]) + # if fc2.bias is not None: + # new_fc2.bias.data.copy_(fc2.bias.data) + # set_module(block, f'output.dense', new_fc2) + + + # for block_i, block in enumerate(fm_vis.text_encoder.encoder.layer): + # for k in ['query', 'key', 'value']: + # qkv = get_module(block, f'crossattention.self.{k}') + + # new_qkv = nn.Linear(qkv.in_features, _f(qkv.out_features), + # qkv.bias is not None, qkv.weight.device) + # indexes = l1_max_indexes(qkv.weight.data, 0) + + # new_qkv.weight.data.copy_(qkv.weight.data[indexes]) + # if qkv.bias is not None: + # new_qkv.bias.data.copy_(qkv.bias.data[indexes]) + # set_module(block, f'crossattention.self.{k}', new_qkv) + + # proj = get_module(block, f'crossattention.output.dense') + # new_proj = nn.Linear(_f(proj.in_features), proj.out_features, + # proj.bias is not None, proj.weight.device) + # new_proj.weight.data.copy_(proj.weight.data[:, l1_max_indexes(proj.weight.data, 1)]) + # if proj.bias is not None: + # new_proj.bias.data.copy_(proj.bias.data) + # set_module(block, f'crossattention.output.dense', new_proj) + + + + # for block_i, block in enumerate(fm_vis.vision_model.encoder.layers): + # qkv = block.self_attn.qkv + + # new_qkv = nn.Linear(qkv.in_features, _f(qkv.out_features), + # qkv.bias is not None, qkv.weight.device) + # indexes = l1_max_indexes(qkv.weight.data, 0) + + # new_qkv.weight.data.copy_(qkv.weight.data[indexes]) + # if qkv.bias is not None: + # new_qkv.bias.data.copy_(qkv.bias.data[indexes]) + # set_module(fm_vis, f'vision_model.encoder.layers.{block_i}.self_attn.qkv', new_qkv) + + # proj = block.self_attn.projection + # new_proj = nn.Linear(_f(proj.in_features), proj.out_features, + # proj.bias is not None, proj.weight.device) + # new_proj.weight.data.copy_(proj.weight.data[:, l1_max_indexes(proj.weight.data, 1)]) + # if proj.bias is not None: + # new_proj.bias.data.copy_(proj.bias.data) + # set_module(fm_vis, f'vision_model.encoder.layers.{block_i}.self_attn.projection', new_proj) + + # fc1 = block.mlp.fc1 + # new_fc1 = nn.Linear(fc1.in_features, _f(fc1.out_features), + # fc1.bias is not None, fc1.weight.device) + # indexes = l1_max_indexes(fc1.weight.data, 0) + # new_fc1.weight.data.copy_(fc1.weight.data[indexes]) + # if fc1.bias is not None: + # new_fc1.bias.data.copy_(fc1.bias.data[indexes]) + # set_module(fm_vis, f'vision_model.encoder.layers.{block_i}.mlp.fc1', new_fc1) + + # fc2 = block.mlp.fc2 + # new_fc2 = nn.Linear(_f(fc2.in_features), fc2.out_features, + # fc2.bias is not None, fc2.weight.device) + # new_fc2.weight.data.copy_(fc2.weight.data[:, l1_max_indexes(fc2.weight.data, 1)]) + # if fc2.bias is not None: + # new_fc2.bias.data.copy_(fc2.bias.data) + # set_module(fm_vis, f'vision_model.encoder.layers.{block_i}.mlp.fc2', new_fc2) + + return fm_vis + + def init_md_from_fm_by_reducing_width_with_perf_test(self, fm: nn.Module, reducing_width_ratio: int, + samples: torch.Tensor) -> nn.Module: + fm_size = get_model_size(fm, True) + fm_latency = self._get_model_latency(fm, samples, 20, + get_model_device(fm), 20, False) + + master_dnn = self.init_md_from_fm_by_reducing_width(fm, reducing_width_ratio) + master_dnn_size = get_model_size(master_dnn, True) + logger.debug(f'inited master DNN: {master_dnn}') + master_dnn_latency = self._get_model_latency(master_dnn, samples, 20, + get_model_device(master_dnn), 20, False) + + logger.info(f'init master DNN (w/o FBS yet) by reducing foundation model\'s width (by {reducing_width_ratio:d}x)') + logger.info(f'foundation model ({fm_size:.3f}MB, {fm_latency:.4f}s/sample) -> ' + f'master DNN ({master_dnn_size:.3f}MB, {master_dnn_latency:.4f}s/sample)\n' + f'(model size: ↓ {(fm_size / master_dnn_size):.2f}x, ' + f'latency: ↓ {(fm_latency / master_dnn_latency):.2f}x)') + + return master_dnn + + def _get_model_latency(self, model: torch.nn.Module, model_input_size, sample_num: int, + device: str, warmup_sample_num: int, return_detail=False): + import time + + if isinstance(model_input_size, tuple): + dummy_input = torch.rand(model_input_size).to(device) + else: + dummy_input = model_input_size + + model = model.to(device) + model.eval() + + # warm up + with torch.no_grad(): + for _ in range(warmup_sample_num): + model(dummy_input) + + infer_time_list = [] + + if device == 'cuda' or 'cuda' in str(device): + with torch.no_grad(): + for _ in range(sample_num): + s, e = torch.cuda.Event(enable_timing=True), torch.cuda.Event(enable_timing=True) + s.record() + model(dummy_input) + e.record() + torch.cuda.synchronize() + cur_model_infer_time = s.elapsed_time(e) / 1000. + infer_time_list += [cur_model_infer_time] + + else: + with torch.no_grad(): + for _ in range(sample_num): + start = time.time() + model(**dummy_input) + cur_model_infer_time = time.time() - start + infer_time_list += [cur_model_infer_time] + + avg_infer_time = sum(infer_time_list) / sample_num + + if return_detail: + return avg_infer_time, infer_time_list + return avg_infer_time + + +####Here starts with index + +class SqueezeLast(nn.Module): + def __init__(self): + super(SqueezeLast, self).__init__() + + def forward(self, x): + return x.squeeze(-1) + + +class ProjConv_WrappedWithFBS(Layer_WrappedWithFBS): + def __init__(self, proj: nn.Conv2d, r): + super(ProjConv_WrappedWithFBS, self).__init__() + + self.proj = proj + + # for conv: (B, C_in, H, W) -> (B, C_in) -> (B, C_out) + # for mlp in ViT: (B, #patches, D: dim of patches embedding) -> (B, D) -> (B, C_out) + self.fbs = nn.Sequential( + Abs(), + nn.AdaptiveAvgPool1d(1), + SqueezeLast(), + nn.Linear(proj.in_channels, proj.out_channels // r), + nn.ReLU(), + nn.Linear(proj.out_channels // r, proj.out_channels), + nn.ReLU() + ) + + nn.init.constant_(self.fbs[6].bias, 1.) + nn.init.kaiming_normal_(self.fbs[6].weight) + + + def forward(self, x): + if self.use_cached_channel_attention and self.cached_channel_attention is not None: + channel_attention = self.cached_channel_attention + else: + self.cached_raw_channel_attention = self.fbs(x) + self.cached_channel_attention = self.k_takes_all(self.cached_raw_channel_attention) + + channel_attention = self.cached_channel_attention + + raw_res = self.proj(x) + + return channel_attention.unsqueeze(1) * raw_res # TODO: + + +class Linear_WrappedWithFBS(Layer_WrappedWithFBS): + def __init__(self, linear: nn.Linear, r): + super(Linear_WrappedWithFBS, self).__init__() + + self.linear = linear + + # for conv: (B, C_in, H, W) -> (B, C_in) -> (B, C_out) + # for mlp in ViT: (B, #patches, D: dim of patches embedding) -> (B, D) -> (B, C_out) + self.fbs = nn.Sequential( + Rearrange('b n d -> b d n'), + Abs(), + nn.AdaptiveAvgPool1d(1), + SqueezeLast(), + nn.Linear(linear.in_features, linear.out_features // r), + nn.ReLU(), + nn.Linear(linear.out_features // r, linear.out_features), + nn.ReLU() + ) + + nn.init.constant_(self.fbs[6].bias, 1.) + nn.init.kaiming_normal_(self.fbs[6].weight) + + + def forward(self, x): + if self.use_cached_channel_attention and self.cached_channel_attention is not None: + channel_attention = self.cached_channel_attention + else: + self.cached_raw_channel_attention = self.fbs(x) + self.cached_channel_attention = self.k_takes_all(self.cached_raw_channel_attention) + + channel_attention = self.cached_channel_attention + + raw_res = self.linear(x) + + return channel_attention.unsqueeze(1) * raw_res + + +class ToQKV_WrappedWithFBS(Layer_WrappedWithFBS): + """ + This regards to_q/to_k/to_v as a whole (in fact it consists of multiple heads) and prunes it. + It seems different channels of different heads are pruned according to the input. + This is different from "removing some head" or "removing the same channels in each head". + """ + def __init__(self, to_qkv: nn.Linear, r): + super(ToQKV_WrappedWithFBS, self).__init__() + + # self.to_qkv = to_qkv + + self.to_qk = nn.Linear(to_qkv.in_features, to_qkv.out_features // 3 * 2, bias=to_qkv.bias is not None) + self.to_v = nn.Linear(to_qkv.in_features, to_qkv.out_features // 3, bias=to_qkv.bias is not None) + self.to_qk.weight.data.copy_(to_qkv.weight.data[0: to_qkv.out_features // 3 * 2]) + if to_qkv.bias is not None: + self.to_qk.bias.data.copy_(to_qkv.bias.data[0: to_qkv.out_features // 3 * 2]) + self.to_v.weight.data.copy_(to_qkv.weight.data[to_qkv.out_features // 3 * 2: ]) + if to_qkv.bias is not None: + self.to_v.bias.data.copy_(to_qkv.bias.data[to_qkv.out_features // 3 * 2: ]) + + self.fbs = nn.Sequential( + Rearrange('b n d -> b d n'), + Abs(), + nn.AdaptiveAvgPool1d(1), + SqueezeLast(), + nn.Linear(to_qkv.in_features, to_qkv.out_features // 3 // r), + nn.ReLU(), + # nn.Linear(to_qkv.out_features // 3 // r, to_qkv.out_features // 3), + nn.Linear(to_qkv.out_features // 3 // r, self.to_v.out_features), + nn.ReLU() + ) + + nn.init.constant_(self.fbs[6].bias, 1.) + nn.init.kaiming_normal_(self.fbs[6].weight) + + def forward(self, x): + if self.use_cached_channel_attention and self.cached_channel_attention is not None: + channel_attention = self.cached_channel_attention + else: + self.cached_raw_channel_attention = self.fbs(x) + + # print() + # for attn in self.cached_raw_channel_attention.chunk(3, dim=1)[0: 1]: + # print(self.cached_raw_channel_attention.size(), attn.size()) + # print(self.k_takes_all.k) + # print(attn[0].nonzero(as_tuple=True)[0].size(), attn[0]) + + self.cached_channel_attention = self.k_takes_all(self.cached_raw_channel_attention) + + + # for attn in self.cached_channel_attention.chunk(3, dim=1)[0: 1]: + # print(self.cached_channel_attention.size(), attn.size()) + # print(self.k_takes_all.k) + # print(attn[0].nonzero(as_tuple=True)[0].size(), attn[0]) + # print() + + channel_attention = self.cached_channel_attention + + qk = self.to_qk(x) + v = channel_attention.unsqueeze(1) * self.to_v(x) + return torch.cat([qk, v], dim=-1) + + # qkv = raw_res.chunk(3, dim = -1) + + # raw_v = qkv[2] + # print('raw_k, raw_v', qkv[0].sum((0, 1))[0: 10], qkv[0].sum((0, 1)).nonzero(as_tuple=True)[0].size(), + # qkv[1].sum((0, 1))[0: 10], qkv[1].sum((0, 1)).nonzero(as_tuple=True)[0].size(),) + # print('raw_v', raw_v.size(), raw_v.sum((0, 1))[0: 10], raw_v.sum((0, 1)).nonzero(as_tuple=True)[0].size()) + + # qkv_attn = channel_attention.chunk(3, dim=-1) + # print('attn', [attn[0][0: 10] for attn in qkv_attn]) + # print(channel_attention.unsqueeze(1).size(), raw_res.size()) + # print('fbs', channel_attention.size(), raw_res.size()) + # return channel_attention.unsqueeze(1) * raw_res + + +class StaticFBS(nn.Module): + def __init__(self, static_channel_attention): + super(StaticFBS, self).__init__() + assert static_channel_attention.dim() == 2 and static_channel_attention.size(0) == 1 + self.static_channel_attention = nn.Parameter(static_channel_attention, requires_grad=False) # (1, dim) + + def forward(self, x): + # print('staticfbs', x, self.static_channel_attention.unsqueeze(1)) + return x * self.static_channel_attention.unsqueeze(1) + + +class ElasticclipUtil(ElasticDNNUtil): + def convert_raw_dnn_to_master_dnn(self, raw_dnn: nn.Module, r: float, ignore_layers=[]): + assert len(ignore_layers) == 0, 'not supported yet' + + raw_vit = deepcopy(raw_dnn) + + # set_module(module, 'patch_embed.proj', ProjConv_WrappedWithFBS(module.patch_embed.proj, r)) + + for name, module in raw_vit.named_modules(): + # if name.endswith('attn'): + # set_module(module, 'qkv', ToQKV_WrappedWithFBS(module.qkv, r)) + if name.endswith('intermediate'): + set_module(module, 'dense', Linear_WrappedWithFBS(module.dense, r)) + elif name.endswith('mlp'): + set_module(module, 'fc1', Linear_WrappedWithFBS(module.fc1, r)) + + return raw_vit + + def set_master_dnn_sparsity(self, master_dnn: nn.Module, sparsity: float): + # for name, module in master_dnn.named_modules(): + # if not name.endswith('attn'): + # continue + + # q_features = module.qkv.to_qk.out_features // 2 + + # if (q_features - int(q_features * sparsity)) % module.num_heads != 0: + # # tune sparsity to ensure #unpruned channel % num_heads == 0 + # # so that the pruning seems to reduce the dim_head of each head + # tuned_sparsity = 1. - int((q_features - int(q_features * sparsity)) / module.num_heads) * module.num_heads / q_features + # logger.debug(f'tune sparsity from {sparsity:.2f} to {tuned_sparsity}') + # sparsity = tuned_sparsity + # break + + return super().set_master_dnn_sparsity(master_dnn, sparsity) + + def select_most_rep_sample(self, master_dnn: nn.Module, samples: torch.Tensor): + # print(samples) + return samples[0].unsqueeze(0) + # res = {k: v[0: 1] for k, v in samples.items()} + # return res + + def extract_surrogate_dnn_via_samples(self, master_dnn: nn.Module, samples: torch.Tensor, return_detail=False):#产生小模型的步骤 + sample = self.select_most_rep_sample(master_dnn, samples) + # assert sample.dim() == 4 and sample.size(0) == 1 + + # print('before') + master_dnn.eval() + self.clear_cached_channel_attention_in_master_dnn(master_dnn) + with torch.no_grad(): + master_dnn_output = master_dnn(sample) + + # print('after') + + boosted_vit = deepcopy(master_dnn) + + def get_unpruned_indexes_from_channel_attn(channel_attn: torch.Tensor, k): + assert channel_attn.size(0) == 1, 'use A representative sample to generate channel attentions' + + # print('attn_in_unpruned', channel_attn[0][0: 10]) + + res = channel_attn[0].nonzero(as_tuple=True)[0] # should be one-dim + + # res = channel_attn[0].argsort(descending=True)[0: -int(channel_attn.size(1) * k)].sort()[0] + + # g = channel_attn + # k = g.size(1) - int(g.size(1) * k) + # res = g.topk(k, 1)[1][0].sort()[0] + + return res + + unpruned_indexes_of_layers = {} + + # for attn, ff in boosted_vit.transformer.layers: + # for block_i, block in enumerate(boosted_vit.blocks): + for block_i, block in enumerate(boosted_vit.clip.vision_model.encoder.layers): + # attn = block.attn + # ff = block.mlp + + ff_0 = get_module(block, f'mlp.fc1') + # ff_0_unpruned_indexes = get_unpruned_indexes_from_channel_attn(ff_0.cached_channel_attention, k) + ff_0_pruned_indexes = ff_0.k_takes_all.cached_i[0].sort()[0] + ff_0_unpruned_indexes = torch.LongTensor([ii for ii in range(ff_0.cached_channel_attention.size(1)) if ii not in ff_0_pruned_indexes]) + new_ff_0 = nn.Linear(ff_0.linear.in_features, ff_0_unpruned_indexes.size(0), ff_0.linear.bias is not None) + new_ff_0.weight.data.copy_(ff_0.linear.weight.data[ff_0_unpruned_indexes]) + if ff_0.linear.bias is not None: + new_ff_0.bias.data.copy_(ff_0.linear.bias.data[ff_0_unpruned_indexes]) + set_module(block, 'mlp.fc1', nn.Sequential(new_ff_0, StaticFBS(ff_0.cached_channel_attention[:, ff_0_unpruned_indexes]))) + + ff_1 = get_module(block, f'mlp.fc2') + new_ff_1 = nn.Linear(ff_0_unpruned_indexes.size(0), ff_1.out_features, ff_1.bias is not None) + new_ff_1.weight.data.copy_(ff_1.weight.data[:, ff_0_unpruned_indexes]) + if ff_1.bias is not None: + new_ff_1.bias.data.copy_(ff_1.bias.data) + set_module(block, 'mlp.fc2', new_ff_1) + + unpruned_indexes_of_layers[f'clip.vision_model.encoder.layers.{block_i}.mlp.fc1.0.weight'] = ff_0_unpruned_indexes + # for block_i,block in enumerate(boosted_vit.vision_model.encoder.layers): + + # attn = block.self_attn + # ff = block.mlp + # ff_0 = ff.fc1 + # # ff_0_unpruned_indexes = get_unpruned_indexes_from_channel_attn(ff_0.cached_channel_attention, k) + # ff_0_pruned_indexes = ff_0.k_takes_all.cached_i[0].sort()[0] + # ff_0_unpruned_indexes = torch.LongTensor([ii for ii in range(ff_0.cached_channel_attention.size(1)) if ii not in ff_0_pruned_indexes]) + # new_ff_0 = nn.Linear(ff_0.linear.in_features, ff_0_unpruned_indexes.size(0), ff_0.linear.bias is not None) + # new_ff_0.weight.data.copy_(ff_0.linear.weight.data[ff_0_unpruned_indexes]) + # if ff_0.linear.bias is not None: + # new_ff_0.bias.data.copy_(ff_0.linear.bias.data[ff_0_unpruned_indexes]) + # set_module(ff, 'fc1', nn.Sequential(new_ff_0, StaticFBS(ff_0.cached_channel_attention[:, ff_0_unpruned_indexes]))) + + # ff_1 = ff.fc2 + # new_ff_1 = nn.Linear(ff_0_unpruned_indexes.size(0), ff_1.out_features, ff_1.bias is not None) + # new_ff_1.weight.data.copy_(ff_1.weight.data[:, ff_0_unpruned_indexes]) + # if ff_1.bias is not None: + # new_ff_1.bias.data.copy_(ff_1.bias.data) + # set_module(ff, 'fc2', new_ff_1) + + # unpruned_indexes_of_layers[f'vision_model.encoder.layers.{block_i}.mlp.fc1.0.weight'] = ff_0_unpruned_indexes + + + # for block_i, block in enumerate(boosted_vit.text_decoder.bert.encoder.layer): + # # attn = block.attn + # # ff = block.mlp + + # ff_0 = get_module(block, f'intermediate.dense') + # # ff_0_unpruned_indexes = get_unpruned_indexes_from_channel_attn(ff_0.cached_channel_attention, k) + # ff_0_pruned_indexes = ff_0.k_takes_all.cached_i[0].sort()[0] + # ff_0_unpruned_indexes = torch.LongTensor([ii for ii in range(ff_0.cached_channel_attention.size(1)) if ii not in ff_0_pruned_indexes]) + # new_ff_0 = nn.Linear(ff_0.linear.in_features, ff_0_unpruned_indexes.size(0), ff_0.linear.bias is not None) + # new_ff_0.weight.data.copy_(ff_0.linear.weight.data[ff_0_unpruned_indexes]) + # if ff_0.linear.bias is not None: + # new_ff_0.bias.data.copy_(ff_0.linear.bias.data[ff_0_unpruned_indexes]) + # set_module(block, 'intermediate.dense', nn.Sequential(new_ff_0, StaticFBS(ff_0.cached_channel_attention[:, ff_0_unpruned_indexes]))) + + # ff_1 = get_module(block, f'output.dense') + # new_ff_1 = nn.Linear(ff_0_unpruned_indexes.size(0), ff_1.out_features, ff_1.bias is not None) + # new_ff_1.weight.data.copy_(ff_1.weight.data[:, ff_0_unpruned_indexes]) + # if ff_1.bias is not None: + # new_ff_1.bias.data.copy_(ff_1.bias.data) + # set_module(block, 'output.dense', new_ff_1) + + # unpruned_indexes_of_layers[f'text_decoder.bert.encoder.layer.{block_i}.intermediate.dense.0.weight'] = ff_0_unpruned_indexes + surrogate_dnn = boosted_vit + surrogate_dnn.eval() + surrogate_dnn = surrogate_dnn.to(get_model_device(master_dnn)) + # logger.debug(surrogate_dnn) + with torch.no_grad(): + surrogate_dnn_output = surrogate_dnn(sample) + + output_diff = ((surrogate_dnn_output - master_dnn_output) ** 2).sum() + # assert output_diff < 1e-4, output_diff + logger.info(f'output diff of master and surrogate DNN: {output_diff}') + # logger.debug(f'example output of master/surrogate: {master_dnn_output.sum(0)[0: 10]}, {surrogate_dnn_output.sum(0)[0: 10]}') + # logger.info(f'\nonly prune mlp!!!!\n') + # logger.info(f'\nonly prune mlp!!!!\n') + + if return_detail: + return boosted_vit, unpruned_indexes_of_layers + + return boosted_vit + + def extract_surrogate_dnn_via_samples_with_perf_test(self, master_dnn: nn.Module, samples: torch.Tensor, return_detail=False): + master_dnn_size = get_model_size(master_dnn, True) + master_dnn_latency = self._get_model_latency(master_dnn, samples, 50, + get_model_device(master_dnn), 50, False) + + res = self.extract_surrogate_dnn_via_samples(master_dnn, samples, return_detail) + if not return_detail: + surrogate_dnn = res + else: + surrogate_dnn, unpruned_indexes_of_layers = res + surrogate_dnn_size = get_model_size(surrogate_dnn, True) + surrogate_dnn_latency = self._get_model_latency(master_dnn, samples, 50, + get_model_device(master_dnn), 50, False) + + logger.info(f'master DNN ({master_dnn_size:.3f}MB, {master_dnn_latency:.4f}s/sample) -> ' + f'surrogate DNN ({surrogate_dnn_size:.3f}MB, {surrogate_dnn_latency:.4f}s/sample)\n' + f'(model size: ↓ {(master_dnn_size / surrogate_dnn_size):.2f}x, ' + f'latency: ↓ {(master_dnn_latency / surrogate_dnn_latency):.2f}x)') + + return res + + def _get_model_latency(self, model: torch.nn.Module, model_input_size, sample_num: int, + device: str, warmup_sample_num: int, return_detail=False): + import time + + if isinstance(model_input_size, tuple): + dummy_input = torch.rand(model_input_size).to(device) + else: + dummy_input = model_input_size + + model = model.to(device) + model.eval() + + # warm up + with torch.no_grad(): + for _ in range(warmup_sample_num): + model(dummy_input) + + infer_time_list = [] + + if device == 'cuda' or 'cuda' in str(device): + with torch.no_grad(): + for _ in range(sample_num): + s, e = torch.cuda.Event(enable_timing=True), torch.cuda.Event(enable_timing=True) + s.record() + model(dummy_input) + e.record() + torch.cuda.synchronize() + cur_model_infer_time = s.elapsed_time(e) / 1000. + infer_time_list += [cur_model_infer_time] + + else: + with torch.no_grad(): + for _ in range(sample_num): + start = time.time() + model(**dummy_input) + cur_model_infer_time = time.time() - start + infer_time_list += [cur_model_infer_time] + + avg_infer_time = sum(infer_time_list) / sample_num + + if return_detail: + return avg_infer_time, infer_time_list + return avg_infer_time + + + +#####Here starts with online diff --git a/new_impl/cv/clip/cls.py b/new_impl/cv/clip/cls.py new file mode 100644 index 0000000000000000000000000000000000000000..9820ac9c40653d9df4de924dcac8804d0189c3c9 --- /dev/null +++ b/new_impl/cv/clip/cls.py @@ -0,0 +1,93 @@ +import torch +from new_impl.cv.elasticdnn.api.model import ElasticDNN_OfflineClsFMModel +#from methods.elasticdnn.api.algs.fm_lora import ElasticDNN_FMLoRAAlg +from new_impl.cv.elasticdnn.api.algs.fm_lora import ElasticDNN_FMLoRAAlg +from methods.elasticdnn.model.base import ElasticDNNUtil +from new_impl.cv.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from clip import FMLoRA_clip_Util +from new_impl.cv.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util + +from new_impl.cv.elasticdnn.model.vit import ElasticViTUtil +from data import build_scenario +import torch.nn.functional as F +from utils.dl.common.model import LayerActivation, get_module +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +# from transformers import CvtForImageClassification +# model = CvtForImageClassification.from_pretrained("/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/new_impl/cv/cvt_model",num_labels=20,ignore_mismatched_sizes=True).to('cuda') + +class ElasticDNN_beit_OfflineClsFMModel(ElasticDNN_OfflineClsFMModel): + def generate_md_by_reducing_width(self, reducing_width_ratio, samples: torch.Tensor): + # return FM_to_MD_ViT_Util().init_md_from_fm_by_reducing_width_with_perf_test(self.models_dict['main'], + # reducing_width_ratio, samples) + raise NotImplementedError + def get_feature_hook(self) -> LayerActivation: + return LayerActivation(get_module(self.models_dict['main'], 'classifier'), True, self.device) + + def get_elastic_dnn_util(self) -> ElasticDNNUtil: + raise NotImplementedError + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + #x1 = torch.rand(1,3,224,224).to('cuda:1') + o1 = self.infer(x) + # o2 = self.infer(x1) + # print(o1.logits) + # print(o2.logits) + #print(self.models_dict['main']) + #print(o1.logits.shape) + #print(F.cross_entropy(self.infer(x).logits, y) ) + #formatted_values = [[round(value, 4) for value in row] for row in o1.logits.tolist()] + #return F.cross_entropy(torch.tensor(formatted_values).to('cuda'), y) + return F.cross_entropy(o1, y) #这个是适用于hugging face模型的计算形式，因为它输出的是一个实例化的类，结果封装在类的属性里，你得去给它调出来。 + + def get_lora_util(self) -> FMLoRA_Util: + return FMLoRA_clip_Util() + + def get_task_head_params(self): + head = get_module(self.models_dict['main'], 'classifier') + return list(head.parameters()) +if __name__ == '__main__': + scenario = build_scenario( + source_datasets_name=['GTA5Cls', 'SuperviselyPersonCls'], + target_datasets_order=['CityscapesCls', 'BaiduPersonCls'] * 15, + da_mode='close_set', + data_dirs={ + 'GTA5Cls': '/data/zql/datasets/gta5_for_cls_task', + 'SuperviselyPersonCls': '/data/zql/datasets/supervisely_person_for_cls_task', + 'CityscapesCls': '/data/zql/datasets/cityscapes_for_cls_task', + 'BaiduPersonCls': '/data/zql/datasets/baidu_person_for_cls_task' + }, + ) + + #from transformers import BeitForImageClassification + from clip import clip + fm_models_dict_path = save_models_dict_for_init({ + 'main':clip(scenario.num_classes) + },__file__,'clip_pretrained') + torch.cuda.set_device(1) + device = 'cuda' + #print(CvtForImageClassification.from_pretrained("/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/new_impl/cv/cvt_model",num_labels=scenario.num_classes,ignore_mismatched_sizes=True)) + fm_model = ElasticDNN_beit_OfflineClsFMModel('fm', fm_models_dict_path, device) + #fm_model = CvtForImageClassification.from_pretrained("/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/new_impl/cv/cvt_model",num_labels=scenario.num_classes,ignore_mismatched_sizes=True).to(device) + models = { + 'fm':fm_model + } + import sys + fm_lora_alg = ElasticDNN_FMLoRAAlg(models, get_res_save_dir(__file__, tag=sys.argv[0])) + + from utils.dl.common.lr_scheduler import get_linear_schedule_with_warmup + fm_lora_alg.run(scenario, hyps={ + 'launch_tbboard': False, + + 'samples_size': (1, 3, 224, 224), + 'ab_r': 3,#hugging face中的模型封装得特别严实，自注意力层里面，qkv是分开的，注意这个对应的倍数不要设置太高 + 'train_batch_size': 256, + 'val_batch_size': 512, + 'num_workers': 16, + 'optimizer': 'Adam', + 'optimizer_args': {'lr': 1e-4, 'betas': [0.9, 0.999]},#不同的模型，注意调调学习率啊 + 'scheduler': 'LambdaLR', + 'scheduler_args': {'lr_lambda': get_linear_schedule_with_warmup(10000, 70000)}, + 'num_iters': 8000, + 'val_freq': 200 + } +) diff --git a/new_impl/cv/clip/cls_baseline.py b/new_impl/cv/clip/cls_baseline.py new file mode 100644 index 0000000000000000000000000000000000000000..b8593e73ac9a40e8cee398858c0e3b8ac07704bc --- /dev/null +++ b/new_impl/cv/clip/cls_baseline.py @@ -0,0 +1,132 @@ +from typing import List +from data.dataloader import build_dataloader +# from methods.elasticdnn.api.online_model import ElasticDNN_OnlineModel +from new_impl.cv.elasticdnn.api.online_model_v2 import ElasticDNN_OnlineModel + +import torch +import sys +from torch import nn +from new_impl.cv.elasticdnn.api.model import ElasticDNN_OfflineSegFMModel, ElasticDNN_OfflineSegMDModel +from new_impl.cv.elasticdnn.api.algs.md_pretraining_wo_fbs import ElasticDNN_MDPretrainingWoFBSAlg +from new_impl.cv.elasticdnn.model.base import ElasticDNNUtil +from new_impl.cv.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from clip import FM_to_MD_clip_Util +from new_impl.cv.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from clip import FMLoRA_clip_Util +from clip import ElasticclipUtil +from utils.common.file import ensure_dir +from utils.dl.common.model import LayerActivation, get_module, get_parameter +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +from utils.dl.common.loss import CrossEntropyLossSoft +import torch.nn.functional as F +from utils.dl.common.env import create_tbwriter +import os +from utils.common.log import logger +from utils.common.data_record import write_json +# from methods.shot.shot import OnlineShotModel +from new_impl.cv.feat_align.main import OnlineFeatAlignModel, FeatAlignAlg +import tqdm +from new_impl.cv.feat_align.mmd import mmd_rbf +from new_impl.cv.utils.baseline_da import baseline_da + +device = 'cuda' +app_name = 'cls' + +scenario = build_scenario( + source_datasets_name=['GTA5Cls', 'SuperviselyPersonCls'], + target_datasets_order=['CityscapesCls', 'BaiduPersonCls'] * 15, + da_mode='close_set', + data_dirs={ + 'GTA5Cls': '/data/zql/datasets/gta5_for_cls_task', + 'SuperviselyPersonCls': '/data/zql/datasets/supervisely_person_for_cls_task', + 'CityscapesCls': '/data/zql/datasets/cityscapes_for_cls_task', + 'BaiduPersonCls': '/data/zql/datasets/baiduperson_for_cls_task' + }, +) +class ClsOnlineFeatAlignModel(OnlineFeatAlignModel): + def get_trained_params(self): # TODO: elastic fm only train a part of params + #qkv_and_norm_params = [p for n, p in self.models_dict['main'].named_parameters() if 'attention.attention.projection_query' in n or 'attention.attention.projection_key' in n or 'attention.attention.projection_value' in n or 'intermediate.dense' in n or 'output.dense' in n] + qkv_and_norm_params = [p for n, p in self.models_dict['main'].named_parameters()] + return qkv_and_norm_params + + def get_feature_hook(self): + return LayerActivation(get_module(self.models_dict['main'], 'classifier'), False, self.device) + + def forward_to_get_task_loss(self, x, y): + return F.cross_entropy(self.infer(x), y) + + def get_mmd_loss(self, f1, f2): + return mmd_rbf(f1, f2) + + def infer(self, x, *args, **kwargs): + return self.models_dict['main'](x) + + def get_accuracy(self, test_loader, *args, **kwargs): + acc = 0 + sample_num = 0 + + self.to_eval_mode() + + with torch.no_grad(): + pbar = tqdm.tqdm(enumerate(test_loader), total=len(test_loader), dynamic_ncols=True, leave=False) + for batch_index, (x, y) in pbar: + x, y = x.to(self.device), y.to(self.device) + output = self.infer(x) + pred = F.softmax(output, dim=1).argmax(dim=1) + correct = torch.eq(pred, y).sum().item() + acc += correct + sample_num += len(y) + + pbar.set_description(f'cur_batch_total: {len(y)}, cur_batch_correct: {correct}, ' + f'cur_batch_acc: {(correct / len(y)):.4f}') + + acc /= sample_num + return acc + + +da_alg = FeatAlignAlg +#from experiments.cua.vit_b_16.online.cls.model import ClsOnlineFeatAlignModel +da_model = ClsOnlineFeatAlignModel( + app_name, + 'new_impl/cv/clip/results/cls_md_wo_fbs.py/20231115/999998-195939-/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/new_impl/cv/clip/cls_md_wo_fbs.py/models/md_best.pt', + device +) +da_alg_hyp = { + 'CityscapesCls': { + 'train_batch_size': 64, + 'val_batch_size': 512, + 'num_workers': 8, + 'optimizer': 'AdamW', + 'optimizer_args': {'lr': 4e-8/2, 'betas': [0.9, 0.999], 'weight_decay': 0.01}, + 'scheduler': '', + 'scheduler_args': {}, + 'num_iters': 100, + 'val_freq': 20, + 'feat_align_loss_weight': 3.0 + }, + 'BaiduPersonCls': { + 'train_batch_size': 64, + 'val_batch_size': 512, + 'num_workers': 8, + 'optimizer': 'SGD', + 'optimizer_args': {'lr': 1e-10, 'momentum': 0.9}, + 'scheduler': '', + 'scheduler_args': {}, + 'num_iters': 100, + 'val_freq': 20, + 'feat_align_loss_weight': 0.2 + } +} + + +baseline_da( + app_name, + scenario, + da_alg, + da_alg_hyp, + da_model, + device, + __file__, + sys.argv[0] +) diff --git a/new_impl/cv/clip/cls_md_index.py b/new_impl/cv/clip/cls_md_index.py new file mode 100644 index 0000000000000000000000000000000000000000..97779bd13e2b5788976ea7cd9dc9ed0941f462b2 --- /dev/null +++ b/new_impl/cv/clip/cls_md_index.py @@ -0,0 +1,275 @@ +import torch +import sys +from torch import nn +from dnns.vit import make_softmax_prunable +from new_impl.cv.elasticdnn.api.model import ElasticDNN_OfflineClsFMModel, ElasticDNN_OfflineClsMDModel +# from methods.elasticdnn.api.algs.md_pretraining_w_fbs import ElasticDNN_MDPretrainingWFBSAlg +from new_impl.cv.elasticdnn.model.base import ElasticDNNUtil +from new_impl.cv.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from clip import FM_to_MD_clip_Util +from new_impl.cv.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from clip import FMLoRA_clip_Util +from clip import ElasticclipUtil +from utils.dl.common.model import LayerActivation, get_module, get_parameter +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +from utils.dl.common.loss import CrossEntropyLossSoft +import torch.nn.functional as F + + +class ElasticDNN_clip_OfflineClsFMModel(ElasticDNN_OfflineClsFMModel): + def generate_md_by_reducing_width(self, reducing_width_ratio, samples: torch.Tensor): + return FM_to_MD_clip_Util().init_md_from_fm_by_reducing_width_with_perf_test(self.models_dict['main'], + reducing_width_ratio, samples).to(self.device) + + def get_feature_hook(self) -> LayerActivation: + return LayerActivation(get_module(self.models_dict['main'], 'classifier'), True, self.device) + + def get_elastic_dnn_util(self) -> ElasticDNNUtil: + return ElasticclipUtil() + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + return F.cross_entropy(self.infer(x), y) + + def get_lora_util(self) -> FMLoRA_Util: + return FMLoRA_clip_Util() + + def get_task_head_params(self): + head = get_module(self.models_dict['main'], 'classifier') + return list(head.parameters()) + + +class ElasticDNN_clip_OfflineClsMDModel(ElasticDNN_OfflineClsMDModel): + def __init__(self, name: str, models_dict_path: str, device: str): + super().__init__(name, models_dict_path, device) + + self.distill_criterion = CrossEntropyLossSoft() + + def get_feature_hook(self) -> LayerActivation: + return LayerActivation(get_module(self.models_dict['main'], 'classifier'), True, self.device) + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + return F.cross_entropy(self.infer(x), y) + + def get_distill_loss(self, student_output, teacher_output): + return self.distill_criterion(student_output, teacher_output) + + def get_matched_param_of_fm(self, self_param_name, fm: nn.Module): + # only between qkv.weight, norm.weight/bias + if any([k in self_param_name for k in ['fbs', 'cls_token', 'pos_embed']]): + return None + + p = get_parameter(self.models_dict['main'], self_param_name) + if p.dim() == 0: + return None + elif p.dim() == 1 and 'layernorm' in self_param_name and 'weight' in self_param_name: + return get_parameter(fm, self_param_name) + + # 1. xx.qkv.to_qkv.yy to xx.qkv.qkv.aa and xx.qkv.abs.zz + # if 'qkv.weight' in self_param_name: + # ss = self_param_name.split('.') + + # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv' + # fm_qkv = get_module(fm, fm_qkv_name) + + # fm_abs_name = '.'.join(ss[0: -2]) + '.abs' + # fm_abs = get_module(fm, fm_abs_name) + + # return torch.cat([ + # fm_qkv.weight.data, # task-agnositc params + # torch.cat([(_abs[0].weight.T @ _abs[1].weight.T).T for _abs in fm_abs], dim=0) # task-specific params (LoRA) + # ], dim=0) + + # # elif 'to_qkv.bias' in self_param_name: + # # ss = self_param_name.split('.') + + # # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv.bias' + # # return get_parameter(fm, fm_qkv_name) + + # elif 'mlp.fc1' in self_param_name and 'weight' in self_param_name: + # fm_param_name = self_param_name.replace('.linear', '') + # return get_parameter(fm, fm_param_name) + + # elif 'mlp.fc2' in self_param_name and 'weight' in self_param_name: + # fm_param_name = self_param_name + # return get_parameter(fm, fm_param_name) + + # else: + # # return get_parameter(fm, self_param_name) + # return None + + # if 'qkv.weight' in self_param_name: + # ss = self_param_name.split('.') + + # fm_qkv_name = '.'.join(ss[0: -1]) + '.qkv' + # fm_qkv = get_module(fm, fm_qkv_name) + + # fm_abs_name = '.'.join(ss[0: -1]) + '.abs' + # fm_abs = get_module(fm, fm_abs_name) + + # return torch.cat([ + # fm_qkv.weight.data, # task-agnositc params + # torch.cat([(_abs[0].weight.T @ _abs[1].weight.T).T for _abs in fm_abs], dim=0) # task-specific params (LoRA) + # ], dim=0) + + # # elif 'to_qkv.bias' in self_param_name: + # # ss = self_param_name.split('.') + + # # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv.bias' + # # return get_parameter(fm, fm_qkv_name) + + # elif 'mlp.fc1' in self_param_name and 'weight' in self_param_name: + # fm_param_name = self_param_name.replace('.linear', '') + # return get_parameter(fm, fm_param_name) + + # elif 'mlp.fc2' in self_param_name and 'weight' in self_param_name: + # fm_param_name = self_param_name + # res = get_parameter(fm, fm_param_name) + # # print('mlp fc2 debug', fm_param_name, res is None) + # return res + + # else: + # # return get_parameter(fm, self_param_name) + # return None + + # def get_matched_param_of_fm(self, self_param_name, fm: nn.Module): + # if any([k in self_param_name for k in ['fbs', 'cls_token', 'pos_embed']]): + # return None + + # # 1. xx.qkv.to_qkv.yy to xx.qkv.qkv.aa and xx.qkv.abs.zz + # if 'to_qkv.weight' in self_param_name: + # ss = self_param_name.split('.') + + # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv' + # fm_qkv = get_module(fm, fm_qkv_name) + + # fm_abs_name = '.'.join(ss[0: -2]) + '.abs' + # fm_abs = get_module(fm, fm_abs_name) + + # return torch.cat([ + # fm_qkv.weight.data, # task-agnositc params + # torch.cat([(_abs[0].weight.T @ _abs[1].weight.T).T for _abs in fm_abs], dim=0) # task-specific params (LoRA) + # ], dim=0) + + # elif 'to_qkv.bias' in self_param_name: + # ss = self_param_name.split('.') + + # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv.bias' + # return get_parameter(fm, fm_qkv_name) + + # elif 'mlp.fc1' in self_param_name: + # fm_param_name = self_param_name.replace('.linear', '') + # return get_parameter(fm, fm_param_name) + + # else: + # return get_parameter(fm, self_param_name) + if ('self_attn.q_proj' in self_param_name or 'self_attn.k_proj' in self_param_name or \ + 'self_attn.v_proj' in self_param_name) and ('weight' in self_param_name): + ss = self_param_name.split('.') + + fm_qkv_name = '.'.join(ss[0: -1]) + '.fc' + fm_qkv = get_module(fm, fm_qkv_name) + + fm_abs_name = '.'.join(ss[0: -1]) + '.ab' + fm_abs = get_module(fm, fm_abs_name) + + return torch.cat([ + fm_qkv.weight.data, # task-agnositc params + fm_abs[1].weight @ fm_abs[0].weight + ], dim=0) + + elif ('self_attn.q_proj' in self_param_name or 'self_attn.k_proj' in self_param_name or \ + 'self_attn.v_proj' in self_param_name) and ('bias' in self_param_name): + ss = self_param_name.split('.') + + fm_qkv_name = '.'.join(ss[0: -1]) + '.fc.bias' + return get_parameter(fm, fm_qkv_name) + + elif 'mlp.fc1' in self_param_name and 'weight' in self_param_name: + fm_param_name = self_param_name.replace('.linear', '') + return get_parameter(fm, fm_param_name) + elif 'mlp.fc2' in self_param_name and 'weight' in self_param_name: + fm_param_name = self_param_name.replace('.linear', '') + return get_parameter(fm, fm_param_name) + else: + #return get_parameter(fm, self_param_name) + return None + + +if __name__ == '__main__': + from utils.dl.common.env import set_random_seed + set_random_seed(1) + + # 1. init model + from dnns.vit import vit_b_16 + fm_models_dict_path = 'new_impl/cv/clip/results/cls_md_wo_fbs.py/20231113/999994-175831-/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/new_impl/cv/clip/cls_md_wo_fbs.py/models/fm_best.pt' + fm_models_dict_path = save_models_dict_for_init(torch.load(fm_models_dict_path), __file__, 'fm_clip_cls_lora') + pretrained_md_models_dict_path = 'new_impl/cv/clip/results/cls_md_wo_fbs.py/20231113/999994-175831-/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/new_impl/cv/clip/cls_md_wo_fbs.py/models/md_best.pt' + md_models_dict = torch.load(pretrained_md_models_dict_path) + md_models_dict_path = save_models_dict_for_init(md_models_dict, __file__, 'md_clip_cls_pretrained_wo_fbs') + torch.cuda.set_device(1) + device = 'cuda' + + fm_model = ElasticDNN_clip_OfflineClsFMModel('fm', fm_models_dict_path, device) + md_model = ElasticDNN_clip_OfflineClsMDModel('md', md_models_dict_path, device) + + # 2. init alg + models = { + 'fm': fm_model, + 'md': md_model + } + from new_impl.cv.elasticdnn.api.algs.md_pretraining_index_v2_train_index_and_md import ElasticDNN_MDPretrainingIndexAlg + + fm_to_md_alg = ElasticDNN_MDPretrainingIndexAlg(models, get_res_save_dir(__file__, sys.argv[0])) + + # 3. init scenario + scenario = build_scenario( + source_datasets_name=['GTA5Cls', 'SuperviselyPersonCls'], + target_datasets_order=['CityscapesCls', 'BaiduPersonCls'] * 15, + da_mode='close_set', + data_dirs={ + 'GTA5Cls': '/data/zql/datasets/gta5_for_cls_task', + 'SuperviselyPersonCls': '/data/zql/datasets/supervisely_person_for_cls_task', + 'CityscapesCls': '/data/zql/datasets/cityscapes_for_cls_task', + 'BaiduPersonCls': '/data/zql/datasets/baidu_person_for_cls_task' + }, + ) + + from utils.dl.common.lr_scheduler import get_linear_schedule_with_warmup + fm_to_md_alg.run(scenario, hyps={ + 'launch_tbboard': False, + + 'samples_size': (1, 3, 224, 224), + + 'FBS_r': 16, + 'FBS_ignore_layers': [], + + 'train_batch_size': 128, + 'val_batch_size': 512, + 'num_workers': 16, + 'optimizer': 'AdamW', + # 'optimizer_args': {'lr': 1e-4, 'betas': [0.9, 0.999], 'weight_decay': 0.01}, + 'indexes_optimizer_args': {'lr': 3e-4, 'betas': [0.9, 0.999], 'weight_decay': 0.1}, + # 'scheduler': 'StepLR', + # 'scheduler_args': {'step_size': 20000, 'gamma': 0.1}, + # 'optimizer': 'AdamW', + # 'optimizer_args': {'lr': 1e-4, 'betas': [0.9, 0.999], 'weight_decay': 0.01}, + 'optimizer_args': {'lr': 1e-3, 'betas': [0.9, 0.999], 'weight_decay': 0.01},#注意学习率的调整，不同的模型不一样。 + + 'scheduler': 'LambdaLR', + 'scheduler_args': {'lr_lambda': get_linear_schedule_with_warmup(10000, 70000)}, + + 'max_sparsity': 0.9, + 'min_sparsity': 0.3, + 'num_iters': 60000, + 'val_freq': 100, + 'index_loss_weight': 1e-4, + 'l1_reg_loss_weight': 1e-9, + 'val_num_sparsities': 4, + + 'bn_cal_num_iters': 0,#有bn层注意需要加上这个 + + 'index_init': 'zero', + 'index_guided_linear_comb_split_size': 512 + }) + \ No newline at end of file diff --git a/new_impl/cv/clip/cls_md_wo_fbs.py b/new_impl/cv/clip/cls_md_wo_fbs.py new file mode 100644 index 0000000000000000000000000000000000000000..00298d9f548c1fe8ed369675dd515b0d1e5390dd --- /dev/null +++ b/new_impl/cv/clip/cls_md_wo_fbs.py @@ -0,0 +1,274 @@ +import torch +import sys +from torch import nn +#from new_impl.cv.dnns.vit import make_softmax_prunable +from new_impl.cv.elasticdnn.api.model import ElasticDNN_OfflineClsFMModel, ElasticDNN_OfflineClsMDModel +from new_impl.cv.elasticdnn.api.algs.md_pretraining_wo_fbs import ElasticDNN_MDPretrainingWoFBSAlg +from new_impl.cv.elasticdnn.model.base import ElasticDNNUtil +from new_impl.cv.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from clip import FM_to_MD_clip_Util +from new_impl.cv.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from clip import FMLoRA_clip_Util +from new_impl.cv.elasticdnn.model.vit import ElasticViTUtil +from utils.dl.common.model import LayerActivation, get_module, get_parameter +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +from utils.dl.common.loss import CrossEntropyLossSoft +import torch.nn.functional as F + + +# class ElasticDNN_ViT_OfflineClsFMModel(ElasticDNN_OfflineClsFMModel): +# def generate_md_by_reducing_width(self, reducing_width_ratio, samples: torch.Tensor): +# return FM_to_MD_ViT_Util().init_md_from_fm_by_reducing_width_with_perf_test(self.models_dict['main'], +# reducing_width_ratio, samples).to(self.device) + +# def get_feature_hook(self) -> LayerActivation: +# return LayerActivation(get_module(self.models_dict['main'], 'head'), True, self.device) + +# def get_elastic_dnn_util(self) -> ElasticDNNUtil: +# return ElasticViTUtil() + +# def forward_to_get_task_loss(self, x, y, *args, **kwargs): +# return F.cross_entropy(self.infer(x), y) + +# def get_lora_util(self) -> FMLoRA_Util: +# return FMLoRA_ViT_Util() + +# def get_task_head_params(self): +# head = get_module(self.models_dict['main'], 'head') +# return list(head.parameters()) + + +# class ElasticDNN_ViT_OfflineClsMDModel(ElasticDNN_OfflineClsMDModel): +# def __init__(self, name: str, models_dict_path: str, device: str): +# super().__init__(name, models_dict_path, device) + +# self.distill_criterion = CrossEntropyLossSoft() + +# def get_feature_hook(self) -> LayerActivation: +# return LayerActivation(get_module(self.models_dict['main'], 'head'), True, self.device) + +# def forward_to_get_task_loss(self, x, y, *args, **kwargs): +# return F.cross_entropy(self.infer(x), y) + +# def get_distill_loss(self, student_output, teacher_output): +# return self.distill_criterion(student_output, teacher_output) + +# def get_matched_param_of_fm(self, self_param_name, fm: nn.Module): +# # only between qkv.weight, norm.weight/bias +# if any([k in self_param_name for k in ['fbs', 'cls_token', 'pos_embed']]): +# return None + +# p = get_parameter(self.models_dict['main'], self_param_name) +# if p.dim() == 0: +# return None +# elif p.dim() == 1 and 'norm' in self_param_name: +# return get_parameter(fm, self_param_name) + +# # 1. xx.qkv.to_qkv.yy to xx.qkv.qkv.aa and xx.qkv.abs.zz +# if 'to_qkv.weight' in self_param_name: +# ss = self_param_name.split('.') + +# fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv' +# fm_qkv = get_module(fm, fm_qkv_name) + +# fm_abs_name = '.'.join(ss[0: -2]) + '.abs' +# fm_abs = get_module(fm, fm_abs_name) + +# return torch.cat([ +# fm_qkv.weight.data, # task-agnositc params +# torch.cat([(_abs[0].weight.T @ _abs[1].weight.T).T for _abs in fm_abs], dim=0) # task-specific params (LoRA) +# ], dim=0) + +# # elif 'to_qkv.bias' in self_param_name: +# # ss = self_param_name.split('.') + +# # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv.bias' +# # return get_parameter(fm, fm_qkv_name) + +# # elif 'mlp.fc1' in self_param_name: +# # fm_param_name = self_param_name.replace('.linear', '') +# # return get_parameter(fm, fm_param_name) + +# else: +# # return get_parameter(fm, self_param_name) +# return None + +# # def get_matched_param_of_fm(self, self_param_name, fm: nn.Module): +# # if any([k in self_param_name for k in ['fbs', 'cls_token', 'pos_embed']]): +# # return None + +# # # 1. xx.qkv.to_qkv.yy to xx.qkv.qkv.aa and xx.qkv.abs.zz +# # if 'to_qkv.weight' in self_param_name: +# # ss = self_param_name.split('.') + +# # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv' +# # fm_qkv = get_module(fm, fm_qkv_name) + +# # fm_abs_name = '.'.join(ss[0: -2]) + '.abs' +# # fm_abs = get_module(fm, fm_abs_name) + +# # return torch.cat([ +# # fm_qkv.weight.data, # task-agnositc params +# # torch.cat([(_abs[0].weight.T @ _abs[1].weight.T).T for _abs in fm_abs], dim=0) # task-specific params (LoRA) +# # ], dim=0) + +# # elif 'to_qkv.bias' in self_param_name: +# # ss = self_param_name.split('.') + +# # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv.bias' +# # return get_parameter(fm, fm_qkv_name) + +# # elif 'mlp.fc1' in self_param_name: +# # fm_param_name = self_param_name.replace('.linear', '') +# # return get_parameter(fm, fm_param_name) + +# # else: +# # return get_parameter(fm, self_param_name) + + + +class ElasticDNN_beit_OfflineClsFMModel(ElasticDNN_OfflineClsFMModel): + def generate_md_by_reducing_width(self, reducing_width_ratio, samples: torch.Tensor): + return FM_to_MD_clip_Util().init_md_from_fm_by_reducing_width_with_perf_test(self.models_dict['main'], + reducing_width_ratio, samples).to(self.device) + + def get_feature_hook(self) -> LayerActivation: + return LayerActivation(get_module(self.models_dict['main'], 'classifier'), True, self.device) + + def get_elastic_dnn_util(self) -> ElasticDNNUtil: + return ElasticclipUtil() + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + return F.cross_entropy(self.infer(x), y) + + def get_lora_util(self) -> FMLoRA_Util: + return FMLoRA_clip_Util() + + def get_task_head_params(self): + head = get_module(self.models_dict['main'], 'classifier') + return list(head.parameters()) + + +class ElasticDNN_beit_OfflineClsMDModel(ElasticDNN_OfflineClsMDModel): + def __init__(self, name: str, models_dict_path: str, device: str): + super().__init__(name, models_dict_path, device) + + self.distill_criterion = CrossEntropyLossSoft() + + def get_feature_hook(self) -> LayerActivation: + return LayerActivation(get_module(self.models_dict['main'], 'classifier'), True, self.device) + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + return F.cross_entropy(self.infer(x), y) + + def get_distill_loss(self, student_output, teacher_output): + return self.distill_criterion(student_output, teacher_output) + def get_matched_param_of_fm(self, self_param_name, fm: nn.Module): + # only between qkv.weight, norm.weight/bias + if any([k in self_param_name for k in ['fbs', 'cls_token', 'pos_embed']]): + return None + + p = get_parameter(self.models_dict['main'], self_param_name) + if p.dim() == 0: + return None + elif p.dim() == 1 and 'norm' in self_param_name and 'weight' in self_param_name: + return get_parameter(fm, self_param_name) + + # 1. xx.qkv.to_qkv.yy to xx.qkv.qkv.aa and xx.qkv.abs.zz + if ('attention.attention.query' in self_param_name or 'attention.attention.key' in self_param_name or \ + 'attention.attention.value' in self_param_name) and ('weight' in self_param_name): + ss = self_param_name.split('.') + + fm_qkv_name = '.'.join(ss[0: -1]) + '.qkv' + fm_qkv = get_module(fm, fm_qkv_name) + + fm_abs_name = '.'.join(ss[0: -1]) + '.abs' + fm_abs = get_module(fm, fm_abs_name) + + return torch.cat([ + fm_qkv.weight.data, # task-agnositc params + torch.cat([(_abs[0].weight.T @ _abs[1].weight.T).T for _abs in fm_abs], dim=0) # task-specific params (LoRA) + ], dim=0) + + # elif 'to_qkv.bias' in self_param_name: + # ss = self_param_name.split('.') + + # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv.bias' + # return get_parameter(fm, fm_qkv_name) + elif ('attention.attention.query' in self_param_name or 'attention.attention.key' in self_param_name or \ + 'attention.attention.value' in self_param_name) and ('bias' in self_param_name): + ss = self_param_name.split('.') + + fm_qkv_name = '.'.join(ss[0: -1]) + '.qkv.bias' + return get_parameter(fm, fm_qkv_name) + + elif 'intermediate.dense' in self_param_name and 'weight' in self_param_name: + fm_param_name = self_param_name.replace('.linear', '') + return get_parameter(fm, fm_param_name) + + else: + return get_parameter(fm, self_param_name) + #return None + + +if __name__ == '__main__': + from utils.dl.common.env import set_random_seed + set_random_seed(1) + + # 1. init model + #from dnns.vit import vit_b_16 + fm_models_dict_path = 'new_impl/cv/clip/results/cls.py/20231113/999999-164026-/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/new_impl/cv/clip/cls.py/models/fm_best.pt' + fm_models_dict = torch.load(fm_models_dict_path) + #fm_models_dict['main'] = make_softmax_prunable(fm_models_dict['main']) + fm_models_dict_path = save_models_dict_for_init(fm_models_dict, __file__, 'fm_clip_cls_lora') + md_models_dict_path = save_models_dict_for_init({ + 'main': -1 + }, __file__, 'md_clip_cls_lora') + torch.cuda.set_device(1) + device = 'cuda' + + fm_model = ElasticDNN_beit_OfflineClsFMModel('fm', fm_models_dict_path, device) + md_model = ElasticDNN_beit_OfflineClsMDModel('md', md_models_dict_path, device) + + # 2. init alg + models = { + 'fm': fm_model, + 'md': md_model + } + fm_to_md_alg = ElasticDNN_MDPretrainingWoFBSAlg(models, get_res_save_dir(__file__, sys.argv[0])) + + # 3. init scenario + scenario = build_scenario( + source_datasets_name=['GTA5Cls', 'SuperviselyPersonCls'], + target_datasets_order=['CityscapesCls', 'BaiduPersonCls'] * 15, + da_mode='close_set', + data_dirs={ + 'GTA5Cls': '/data/zql/datasets/gta5_for_cls_task', + 'SuperviselyPersonCls': '/data/zql/datasets/supervisely_person_for_cls_task', + 'CityscapesCls': '/data/zql/datasets/cityscapes_for_cls_task', + 'BaiduPersonCls': '/data/zql/datasets/baidu_person_for_cls_task' + }, + ) + + from utils.dl.common.lr_scheduler import get_linear_schedule_with_warmup + fm_to_md_alg.run(scenario, hyps={ + 'launch_tbboard': False, + + 'samples_size': (1, 3, 224, 224), + 'generate_md_width_ratio': 8, + + 'train_batch_size': 256, + 'val_batch_size': 512, + 'num_workers': 16, + 'optimizer': 'AdamW', + 'optimizer_args': {'lr': 1e-4, 'betas': [0.9, 0.999], 'weight_decay': 0.01}, + 'scheduler': 'LambdaLR', + 'scheduler_args': {'lr_lambda': get_linear_schedule_with_warmup(10000, 70000)}, + 'num_iters': 80000, + 'val_freq': 400, + 'distill_loss_weight': 1.0 + }) + + # TODO: + # 1. train MD before inserting FBS? \ No newline at end of file diff --git a/new_impl/cv/clip/cls_online.py b/new_impl/cv/clip/cls_online.py new file mode 100644 index 0000000000000000000000000000000000000000..69f6644ed9148b104d7e0dd0409f301ad69251a8 --- /dev/null +++ b/new_impl/cv/clip/cls_online.py @@ -0,0 +1,349 @@ +from typing import List +from data.dataloader import build_dataloader +# from methods.elasticdnn.api.online_model import ElasticDNN_OnlineModel +from new_impl.cv.elasticdnn.api.online_model_v2 import ElasticDNN_OnlineModel + +import torch +import sys +from torch import nn +from new_impl.cv.elasticdnn.api.model import ElasticDNN_OfflineSegFMModel, ElasticDNN_OfflineSegMDModel +from new_impl.cv.elasticdnn.api.algs.md_pretraining_wo_fbs import ElasticDNN_MDPretrainingWoFBSAlg +from new_impl.cv.elasticdnn.model.base import ElasticDNNUtil +from new_impl.cv.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from new_impl.cv.elasticdnn.pipeline.offline.fm_to_md.vit import FM_to_MD_ViT_Util +from new_impl.cv.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from new_impl.cv.elasticdnn.pipeline.offline.fm_lora.vit import FMLoRA_ViT_Util +from clip import ElasticclipUtil +from utils.common.file import ensure_dir +from utils.dl.common.model import LayerActivation, get_module, get_parameter +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +from utils.dl.common.loss import CrossEntropyLossSoft +import torch.nn.functional as F +from utils.dl.common.env import create_tbwriter +import os +from utils.common.log import logger +from utils.common.data_record import write_json +# from methods.shot.shot import OnlineShotModel +from new_impl.cv.feat_align.main import OnlineFeatAlignModel, FeatAlignAlg +import tqdm +from new_impl.cv.feat_align.mmd import mmd_rbf +from new_impl.cv.utils.elasticfm_da import init_online_model, elasticfm_da +torch.cuda.set_device(1) +device = 'cuda' +app_name = 'cls' +sd_sparsity = 0.8 + +settings = { + 'involve_fm': True +} + +scenario = build_scenario( + source_datasets_name=['GTA5Cls', 'SuperviselyPersonCls'], + target_datasets_order=['CityscapesCls', 'BaiduPersonCls'] * 15, + da_mode='close_set', + data_dirs={ + 'GTA5Cls': '/data/zql/datasets/gta5_for_cls_task', + 'SuperviselyPersonCls': '/data/zql/datasets/supervisely_person_for_cls_task', + 'CityscapesCls': '/data/zql/datasets/cityscapes_for_cls_task', + 'BaiduPersonCls': '/data/zql/datasets/baiduperson_for_cls_task' + }, +) + + +class ElasticDNN_ClsOnlineModel(ElasticDNN_OnlineModel): + def get_accuracy(self, test_loader, *args, **kwargs): + acc = 0 + sample_num = 0 + + self.to_eval_mode() + + with torch.no_grad(): + pbar = tqdm.tqdm(enumerate(test_loader), total=len(test_loader), dynamic_ncols=True, leave=False) + for batch_index, (x, y) in pbar: + x, y = x.to(self.device), y.to(self.device) + output = self.infer(x) + pred = F.softmax(output, dim=1).argmax(dim=1) + correct = torch.eq(pred, y).sum().item() + acc += correct + sample_num += len(y) + + pbar.set_description(f'cur_batch_total: {len(y)}, cur_batch_correct: {correct}, ' + f'cur_batch_acc: {(correct / len(y)):.4f}') + + acc /= sample_num + return acc + + def get_elastic_dnn_util(self) -> ElasticDNNUtil: + return ElasticclipUtil() + + def get_fm_matched_param_of_md_param(self, md_param_name): + # only between qkv.weight, norm.weight/bias + self_param_name = md_param_name + fm = self.models_dict['fm'] + # if any([k in self_param_name for k in ['fbs', 'cls_token', 'pos_embed']]): + # return None + + # p = get_parameter(self.models_dict['md'], self_param_name) + # if p.dim() == 0: + # return None + # elif p.dim() == 1 and 'norm' in self_param_name and 'weight' in self_param_name: + # return get_parameter(fm, self_param_name) + + if any([k in self_param_name for k in ['fbs', 'cls_token', 'pos_embed']]): + return None + + p = get_parameter(self.models_dict['md'], self_param_name) + if p.dim() == 0: + return None + elif p.dim() == 1 and 'layernorm' in self_param_name and 'weight' in self_param_name: + return get_parameter(fm, self_param_name) + # 1. xx.qkv.to_qkv.yy to xx.qkv.qkv.aa and xx.qkv.abs.zz + # if 'qkv.weight' in self_param_name: + # ss = self_param_name.split('.') + + # fm_qkv_name = '.'.join(ss[0: -1]) + '.qkv' + # fm_qkv = get_module(fm, fm_qkv_name) + + # fm_abs_name = '.'.join(ss[0: -1]) + '.abs' + # fm_abs = get_module(fm, fm_abs_name) + + # # NOTE: unrecoverable operation! multiply LoRA parameters to allow it being updated in update_fm_param() + # # TODO: if fm will be used for inference, _mul_lora_weight will not be applied! + # if not hasattr(fm_abs, '_mul_lora_weight'): + # logger.debug(f'set _mul_lora_weight in {fm_abs_name}') + # setattr(fm_abs, '_mul_lora_weight', + # nn.Parameter(torch.cat([(_abs[0].weight.T @ _abs[1].weight.T).T for _abs in fm_abs], dim=0))) + + # return torch.cat([ + # fm_qkv.weight.data, # task-agnositc params + # fm_abs._mul_lora_weight.data # task-specific params (LoRA) + # ], dim=0) + + # # elif 'to_qkv.bias' in self_param_name: + # # ss = self_param_name.split('.') + + # # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv.bias' + # # return get_parameter(fm, fm_qkv_name) + + # elif 'mlp.fc1' in self_param_name and 'weight' in self_param_name: + # fm_param_name = self_param_name.replace('.linear', '') + # return get_parameter(fm, fm_param_name) + + # elif 'mlp.fc2' in self_param_name and 'weight' in self_param_name: + # fm_param_name = self_param_name + # return get_parameter(fm, fm_param_name) + + # else: + # # return get_parameter(fm, self_param_name) + # return None + if ('self_attn.q_proj' in self_param_name or 'self_attn.k_proj' in self_param_name or \ + 'self_attn.v_proj' in self_param_name) and ('weight' in self_param_name): + ss = self_param_name.split('.') + + fm_qkv_name = '.'.join(ss[0: -1]) + '.fc' + fm_qkv = get_module(fm, fm_qkv_name) + + fm_abs_name = '.'.join(ss[0: -1]) + '.ab' + fm_abs = get_module(fm, fm_abs_name) + + if not hasattr(fm_abs, '_mul_lora_weight'): + logger.debug(f'set _mul_lora_weight in {fm_abs_name}') + setattr(fm_abs, '_mul_lora_weight', + nn.Parameter(fm_abs[1].weight @ fm_abs[0].weight)) + + return torch.cat([ + fm_qkv.weight.data, # task-agnositc params + fm_abs._mul_lora_weight.data # task-specific params (LoRA) + ], dim=0) + + elif ('self_attn.q_proj' in self_param_name or 'self_attn.k_proj' in self_param_name or \ + 'self_attn.v_proj' in self_param_name) and ('bias' in self_param_name): + ss = self_param_name.split('.') + + fm_qkv_name = '.'.join(ss[0: -1]) + '.fc.bias' + return get_parameter(fm, fm_qkv_name) + + elif 'mlp.fc1' in self_param_name and 'weight' in self_param_name: + fm_param_name = self_param_name.replace('.linear', '') + return get_parameter(fm, fm_param_name) + elif 'mlp.fc2' in self_param_name and 'weight' in self_param_name: + fm_param_name = self_param_name.replace('.linear', '') + return get_parameter(fm, fm_param_name) + else: + #return get_parameter(fm, self_param_name) + return None + + + def update_fm_param(self, md_param_name, cal_new_fm_param_by_md_param): + if not ('self_attn.q_proj.weight' in md_param_name or 'self_attn.k_proj.weight' in md_param_name or 'self_attn.v_proj.weight' in md_param_name): + matched_fm_param_ref = self.get_fm_matched_param_of_md_param(md_param_name) + matched_fm_param_ref.copy_(cal_new_fm_param_by_md_param) + else: + new_fm_attn_weight, new_fm_lora_weight = torch.chunk(cal_new_fm_param_by_md_param, 2, 0) + ss = md_param_name.split('.') + fm = self.models_dict['fm'] + # update task-agnostic parameters + fm_qkv_name = '.'.join(ss[0: -1]) + '.fc' + fm_qkv = get_module(fm, fm_qkv_name) + fm_qkv.weight.data.copy_(new_fm_attn_weight) + + # update task-specific parameters + fm_abs_name = '.'.join(ss[0: -1]) + '.ab' + fm_abs = get_module(fm, fm_abs_name) + fm_abs._mul_lora_weight.data.copy_(new_fm_lora_weight) # TODO: this will not be applied in inference! + + def get_md_matched_param_of_fm_param(self, fm_param_name): + return super().get_md_matched_param_of_fm_param(fm_param_name) + + def get_md_matched_param_of_sd_param(self, sd_param_name): + # raise NotImplementedError + + # only between qkv.weight, norm.weight/bias + self_param_name = sd_param_name + md = self.models_dict['md'] + if any([k in self_param_name for k in ['fbs', 'ab', 'embeddings']]): + return None + + p = get_parameter(self.models_dict['sd'], self_param_name) + if p.dim() == 0: + return None + elif p.dim() == 1 and ('LayerNorm' in self_param_name or 'layernorm' in self_param_name) and 'weight' in self_param_name: + return get_parameter(md, self_param_name) + + # 1. xx.qkv.to_qkv.yy to xx.qkv.qkv.aa and xx.qkv.abs.zz + if ('q_proj' in self_param_name or 'k_proj' in self_param_name or \ + 'v_proj' in self_param_name) and ('weight' in self_param_name): + + + return get_parameter(md, self_param_name) # NOTE: no fbs in qkv! + + # elif 'to_qkv.bias' in self_param_name: + # ss = self_param_name.split('.') + + # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv.bias' + # return get_parameter(fm, fm_qkv_name) + + elif 'mlp.fc1.0.weight' in self_param_name: + fm_param_name = '.'.join(self_param_name.split('.')[0: -2]) + '.linear.weight' + return get_parameter(md, fm_param_name) + + elif 'mlp.fc2' in self_param_name and 'weight' in self_param_name: + fm_param_name = self_param_name + return get_parameter(md, fm_param_name) + + else: + # return get_parameter(fm, self_param_name) + return None + + def get_task_head_params(self): + head = get_module(self.models_dict['sd'], 'classifier') + return list(head.parameters()) + + + +class ClsOnlineFeatAlignModel(OnlineFeatAlignModel): + def get_trained_params(self): # TODO: elastic fm only train a part of params + #qkv_and_norm_params = [p for n, p in self.models_dict['main'].named_parameters() if 'attention.attention.projection_query' in n or 'attention.attention.projection_key' in n or 'attention.attention.projection_value' in n or 'intermediate.dense' in n or 'output.dense' in n] + qkv_and_norm_params = [p for n, p in self.models_dict['main'].named_parameters()] + return qkv_and_norm_params + + def get_feature_hook(self): + return LayerActivation(get_module(self.models_dict['main'], 'classifier'), False, self.device) + + def forward_to_get_task_loss(self, x, y): + return F.cross_entropy(self.infer(x), y) + + def get_mmd_loss(self, f1, f2): + return mmd_rbf(f1, f2) + + def infer(self, x, *args, **kwargs): + return self.models_dict['main'](x) + + def get_accuracy(self, test_loader, *args, **kwargs): + acc = 0 + sample_num = 0 + + self.to_eval_mode() + + with torch.no_grad(): + pbar = tqdm.tqdm(enumerate(test_loader), total=len(test_loader), dynamic_ncols=True, leave=False) + for batch_index, (x, y) in pbar: + x, y = x.to(self.device), y.to(self.device) + output = self.infer(x) + pred = F.softmax(output, dim=1).argmax(dim=1) + correct = torch.eq(pred, y).sum().item() + acc += correct + sample_num += len(y) + + pbar.set_description(f'cur_batch_total: {len(y)}, cur_batch_correct: {correct}, ' + f'cur_batch_acc: {(correct / len(y)):.4f}') + + acc /= sample_num + return acc + + + + +#from new_impl.cv.model import ElasticDNN_ClsOnlineModel +elasticfm_model = ElasticDNN_ClsOnlineModel('cls', init_online_model( + # 'experiments/elasticdnn/vit_b_16/offline/fm_to_md/results/cls_md_index.py/20230529/star_999997-154037-only_prune_mlp/models/fm_best.pt', + # 'experiments/elasticdnn/vit_b_16/offline/fm_to_md/results/cls_md_index.py/20230529/star_999997-154037-only_prune_mlp/models/md_best.pt', + #'experiments/elasticdnn/vit_b_16/offline/fm_to_md/cls/results/cls_md_index.py/20230617/999992-101343-lr1e-5_index_bug_fixed/models/fm_best.pt', + #'experiments/elasticdnn/vit_b_16/offline/fm_to_md/cls/results/cls_md_index.py/20230617/999992-101343-lr1e-5_index_bug_fixed/models/md_best.pt', + 'new_impl/cv/clip/results/cls_md_index.py/20231114/999997-112438-/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/new_impl/cv/clip/cls_md_index.py/models/fm_best.pt', + 'new_impl/cv/clip/results/cls_md_index.py/20231114/999997-112438-/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/new_impl/cv/clip/cls_md_index.py/models/md_best.pt', + 'cls', __file__ +), device, { + 'md_to_fm_alpha': 1, + 'fm_to_md_alpha': 0.01 +}) + +da_alg = FeatAlignAlg +from utils.dl.common.lr_scheduler import get_linear_schedule_with_warmup +#from new_impl.cv.model import ClsOnlineFeatAlignModel +da_model = ClsOnlineFeatAlignModel +da_alg_hyp = { + 'CityscapesCls': { + 'train_batch_size': 64, + 'val_batch_size': 512, + 'num_workers': 8, + 'optimizer': 'AdamW', + #'optimizer': 'SGD', + 'optimizer_args': {'lr': 4e-6/2, 'betas': [0.9, 0.999], 'weight_decay': 0.01},#针对于cvt的online的学习率 + #'optimizer_args': {'lr': 1e-3, 'momentum': 0.9}, + 'scheduler': '', + 'scheduler_args': {}, + 'num_iters': 100, + 'val_freq': 20, + 'sd_sparsity':0.8, + 'feat_align_loss_weight': 3.0 + }, + 'BaiduPersonCls': { + 'train_batch_size': 64, + 'val_batch_size': 512, + 'num_workers': 8, + 'optimizer': 'SGD', + 'optimizer_args': {'lr': 1e-4/2, 'momentum': 0.9}, + 'scheduler': '', + 'scheduler_args': {}, + 'num_iters': 100, + 'val_freq': 20, + 'sd_sparsity': sd_sparsity, + 'feat_align_loss_weight': 0.3 + } +} + + +elasticfm_da( + [app_name], + [scenario], + [elasticfm_model], + [da_alg], + [da_alg_hyp], + [da_model], + device, + settings, + __file__, + sys.argv[0] +) diff --git a/new_impl/cv/clip/pretrained_model/config.json b/new_impl/cv/clip/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..a2a88b96561196777ca173b15309ea859f4d2ce0 --- /dev/null +++ b/new_impl/cv/clip/pretrained_model/config.json @@ -0,0 +1,157 @@ +{ + "_name_or_path": "openai/clip-vit-base-patch32", + "architectures": [ + "CLIPModel" + ], + "initializer_factor": 1.0, + "logit_scale_init_value": 2.6592, + "model_type": "clip", + "projection_dim": 512, + "text_config": { + "_name_or_path": "", + "add_cross_attention": false, + "architectures": null, + "attention_dropout": 0.0, + "bad_words_ids": null, + "bos_token_id": 0, + "chunk_size_feed_forward": 0, + "cross_attention_hidden_size": null, + "decoder_start_token_id": null, + "diversity_penalty": 0.0, + "do_sample": false, + "dropout": 0.0, + "early_stopping": false, + "encoder_no_repeat_ngram_size": 0, + "eos_token_id": 2, + "finetuning_task": null, + "forced_bos_token_id": null, + "forced_eos_token_id": null, + "hidden_act": "quick_gelu", + "hidden_size": 512, + "id2label": { + "0": "LABEL_0", + "1": "LABEL_1" + }, + "initializer_factor": 1.0, + "initializer_range": 0.02, + "intermediate_size": 2048, + "is_decoder": false, + "is_encoder_decoder": false, + "label2id": { + "LABEL_0": 0, + "LABEL_1": 1 + }, + "layer_norm_eps": 1e-05, + "length_penalty": 1.0, + "max_length": 20, + "max_position_embeddings": 77, + "min_length": 0, + "model_type": "clip_text_model", + "no_repeat_ngram_size": 0, + "num_attention_heads": 8, + "num_beam_groups": 1, + "num_beams": 1, + "num_hidden_layers": 12, + "num_return_sequences": 1, + "output_attentions": false, + "output_hidden_states": false, + "output_scores": false, + "pad_token_id": 1, + "prefix": null, + "projection_dim": 512, + "problem_type": null, + "pruned_heads": {}, + "remove_invalid_values": false, + "repetition_penalty": 1.0, + "return_dict": true, + "return_dict_in_generate": false, + "sep_token_id": null, + "task_specific_params": null, + "temperature": 1.0, + "tie_encoder_decoder": false, + "tie_word_embeddings": true, + "tokenizer_class": null, + "top_k": 50, + "top_p": 1.0, + "torch_dtype": null, + "torchscript": false, + "transformers_version": "4.16.0.dev0", + "use_bfloat16": false, + "vocab_size": 49408 + }, + "text_config_dict": null, + "transformers_version": null, + "vision_config": { + "_name_or_path": "", + "add_cross_attention": false, + "architectures": null, + "attention_dropout": 0.0, + "bad_words_ids": null, + "bos_token_id": null, + "chunk_size_feed_forward": 0, + "cross_attention_hidden_size": null, + "decoder_start_token_id": null, + "diversity_penalty": 0.0, + "do_sample": false, + "dropout": 0.0, + "early_stopping": false, + "encoder_no_repeat_ngram_size": 0, + "eos_token_id": null, + "finetuning_task": null, + "forced_bos_token_id": null, + "forced_eos_token_id": null, + "hidden_act": "quick_gelu", + "hidden_size": 768, + "id2label": { + "0": "LABEL_0", + "1": "LABEL_1" + }, + "image_size": 224, + "initializer_factor": 1.0, + "initializer_range": 0.02, + "intermediate_size": 3072, + "is_decoder": false, + "is_encoder_decoder": false, + "label2id": { + "LABEL_0": 0, + "LABEL_1": 1 + }, + "layer_norm_eps": 1e-05, + "length_penalty": 1.0, + "max_length": 20, + "min_length": 0, + "model_type": "clip_vision_model", + "no_repeat_ngram_size": 0, + "num_attention_heads": 12, + "num_beam_groups": 1, + "num_beams": 1, + "num_hidden_layers": 12, + "num_return_sequences": 1, + "output_attentions": false, + "output_hidden_states": false, + "output_scores": false, + "pad_token_id": null, + "patch_size": 32, + "prefix": null, + "projection_dim" : 512, + "problem_type": null, + "pruned_heads": {}, + "remove_invalid_values": false, + "repetition_penalty": 1.0, + "return_dict": true, + "return_dict_in_generate": false, + "sep_token_id": null, + "task_specific_params": null, + "temperature": 1.0, + "tie_encoder_decoder": false, + "tie_word_embeddings": true, + "tokenizer_class": null, + "top_k": 50, + "top_p": 1.0, + "torch_dtype": null, + "torchscript": false, + "transformers_version": "4.16.0.dev0", + "use_bfloat16": false + }, + "vision_config_dict": null +} diff --git a/new_impl/cv/cls.py b/new_impl/cv/cls.py new file mode 100644 index 0000000000000000000000000000000000000000..143969bd663c9d8e43879d01cea1e29d3f17edbf --- /dev/null +++ b/new_impl/cv/cls.py @@ -0,0 +1,92 @@ +import torch +from new_impl.cv.elasticdnn.api.model import ElasticDNN_OfflineClsFMModel +#from methods.elasticdnn.api.algs.fm_lora import ElasticDNN_FMLoRAAlg +from new_impl.cv.elasticdnn.api.algs.fm_lora import ElasticDNN_FMLoRAAlg +from methods.elasticdnn.model.base import ElasticDNNUtil +from new_impl.cv.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from new_impl.cv.elasticdnn.pipeline.offline.fm_lora.vit import FMLoRA_ViT_Util +from new_impl.cv.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from new_impl.cv.elasticdnn.pipeline.offline.fm_to_md.vit import FM_to_MD_ViT_Util +from new_impl.cv.elasticdnn.model.vit import ElasticViTUtil +from data import build_scenario +import torch.nn.functional as F +from utils.dl.common.model import LayerActivation, get_module +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +# from transformers import CvtForImageClassification +# model = CvtForImageClassification.from_pretrained("/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/new_impl/cv/cvt_model",num_labels=20,ignore_mismatched_sizes=True).to('cuda') + +class ElasticDNN_ViT_OfflineClsFMModel(ElasticDNN_OfflineClsFMModel): + def generate_md_by_reducing_width(self, reducing_width_ratio, samples: torch.Tensor): + return FM_to_MD_ViT_Util().init_md_from_fm_by_reducing_width_with_perf_test(self.models_dict['main'], + reducing_width_ratio, samples) + + def get_feature_hook(self) -> LayerActivation: + return LayerActivation(get_module(self.models_dict['main'], 'classifier'), True, self.device) + + def get_elastic_dnn_util(self) -> ElasticDNNUtil: + return ElasticViTUtil() + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + #x1 = torch.rand(1,3,224,224).to('cuda:1') + o1 = self.infer(x) + # o2 = self.infer(x1) + # print(o1.logits) + # print(o2.logits) + #print(self.models_dict['main']) + #print(o1.logits.shape) + #print(F.cross_entropy(self.infer(x).logits, y) ) + #formatted_values = [[round(value, 4) for value in row] for row in o1.logits.tolist()] + #return F.cross_entropy(torch.tensor(formatted_values).to('cuda'), y) + return F.cross_entropy(o1.logits, y) #这个是适用于hugging face模型的计算形式，因为它输出的是一个实例化的类，结果封装在类的属性里，你得去给它调出来。 + + def get_lora_util(self) -> FMLoRA_Util: + return FMLoRA_ViT_Util() + + def get_task_head_params(self): + head = get_module(self.models_dict['main'], 'classifier') + return list(head.parameters()) +if __name__ == '__main__': + scenario = build_scenario( + source_datasets_name=['GTA5Cls', 'SuperviselyPersonCls'], + target_datasets_order=['CityscapesCls', 'BaiduPersonCls'] * 15, + da_mode='close_set', + data_dirs={ + 'GTA5Cls': '/data/zql/datasets/gta5_for_cls_task', + 'SuperviselyPersonCls': '/data/zql/datasets/supervisely_person_for_cls_task', + 'CityscapesCls': '/data/zql/datasets/cityscapes_for_cls_task', + 'BaiduPersonCls': '/data/zql/datasets/baidu_person_for_cls_task' + }, + ) + + from transformers import CvtForImageClassification + fm_models_dict_path = save_models_dict_for_init({ + 'main':CvtForImageClassification.from_pretrained("/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/new_impl/cv/cvt_model",num_labels=scenario.num_classes,ignore_mismatched_sizes=True) + },__file__,'cvt_pretrained') + torch.cuda.set_device(1) + device = 'cuda' + #print(CvtForImageClassification.from_pretrained("/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/new_impl/cv/cvt_model",num_labels=scenario.num_classes,ignore_mismatched_sizes=True)) + fm_model = ElasticDNN_ViT_OfflineClsFMModel('fm', fm_models_dict_path, device) + #fm_model = CvtForImageClassification.from_pretrained("/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/new_impl/cv/cvt_model",num_labels=scenario.num_classes,ignore_mismatched_sizes=True).to(device) + models = { + 'fm':fm_model + } + import sys + fm_lora_alg = ElasticDNN_FMLoRAAlg(models, get_res_save_dir(__file__, tag=sys.argv[0])) + + from utils.dl.common.lr_scheduler import get_linear_schedule_with_warmup + fm_lora_alg.run(scenario, hyps={ + 'launch_tbboard': False, + + 'samples_size': (1, 3, 224, 224), + 'ab_r': 3,#hugging face中的模型封装得特别严实，自注意力层里面，qkv是分开的，注意这个对应的层数不要设置太高 + 'train_batch_size': 16, + 'val_batch_size': 32, + 'num_workers': 16, + 'optimizer': 'Adam', + 'optimizer_args': {'lr': 1e-2, 'betas': [0.9, 0.999]},#不同的模型，注意调调学习率啊 + 'scheduler': 'LambdaLR', + 'scheduler_args': {'lr_lambda': get_linear_schedule_with_warmup(10000, 70000)}, + 'num_iters': 8000, + 'val_freq': 400 + } +) diff --git a/new_impl/cv/cls_md_index.py b/new_impl/cv/cls_md_index.py new file mode 100644 index 0000000000000000000000000000000000000000..502d578c84a244059c78f902744e74cd6c1d8344 --- /dev/null +++ b/new_impl/cv/cls_md_index.py @@ -0,0 +1,275 @@ +import torch +import sys +from torch import nn +from dnns.vit import make_softmax_prunable +from new_impl.cv.elasticdnn.api.model import ElasticDNN_OfflineClsFMModel, ElasticDNN_OfflineClsMDModel +# from methods.elasticdnn.api.algs.md_pretraining_w_fbs import ElasticDNN_MDPretrainingWFBSAlg +from new_impl.cv.elasticdnn.model.base import ElasticDNNUtil +from new_impl.cv.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from new_impl.cv.elasticdnn.pipeline.offline.fm_to_md.vit import FM_to_MD_ViT_Util +from new_impl.cv.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from new_impl.cv.elasticdnn.pipeline.offline.fm_lora.vit import FMLoRA_ViT_Util +from new_impl.cv.elasticdnn.model.vit import ElasticViTUtil +from utils.dl.common.model import LayerActivation, get_module, get_parameter +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +from utils.dl.common.loss import CrossEntropyLossSoft +import torch.nn.functional as F + + +class ElasticDNN_ViT_OfflineClsFMModel(ElasticDNN_OfflineClsFMModel): + def generate_md_by_reducing_width(self, reducing_width_ratio, samples: torch.Tensor): + return FM_to_MD_ViT_Util().init_md_from_fm_by_reducing_width_with_perf_test(self.models_dict['main'], + reducing_width_ratio, samples).to(self.device) + + def get_feature_hook(self) -> LayerActivation: + return LayerActivation(get_module(self.models_dict['main'], 'classifier'), True, self.device) + + def get_elastic_dnn_util(self) -> ElasticDNNUtil: + return ElasticViTUtil() + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + return F.cross_entropy(self.infer(x).logits, y) + + def get_lora_util(self) -> FMLoRA_Util: + return FMLoRA_ViT_Util() + + def get_task_head_params(self): + head = get_module(self.models_dict['main'], 'classifier') + return list(head.parameters()) + + +class ElasticDNN_ViT_OfflineClsMDModel(ElasticDNN_OfflineClsMDModel): + def __init__(self, name: str, models_dict_path: str, device: str): + super().__init__(name, models_dict_path, device) + + self.distill_criterion = CrossEntropyLossSoft() + + def get_feature_hook(self) -> LayerActivation: + return LayerActivation(get_module(self.models_dict['main'], 'classifier'), True, self.device) + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + return F.cross_entropy(self.infer(x).logits, y) + + def get_distill_loss(self, student_output, teacher_output): + return self.distill_criterion(student_output, teacher_output) + + def get_matched_param_of_fm(self, self_param_name, fm: nn.Module): + # only between qkv.weight, norm.weight/bias + if any([k in self_param_name for k in ['fbs', 'cls_token', 'pos_embed']]): + return None + + p = get_parameter(self.models_dict['main'], self_param_name) + if p.dim() == 0: + return None + elif p.dim() == 1 and 'layernorm' in self_param_name and 'weight' in self_param_name: + return get_parameter(fm, self_param_name) + + # 1. xx.qkv.to_qkv.yy to xx.qkv.qkv.aa and xx.qkv.abs.zz + # if 'qkv.weight' in self_param_name: + # ss = self_param_name.split('.') + + # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv' + # fm_qkv = get_module(fm, fm_qkv_name) + + # fm_abs_name = '.'.join(ss[0: -2]) + '.abs' + # fm_abs = get_module(fm, fm_abs_name) + + # return torch.cat([ + # fm_qkv.weight.data, # task-agnositc params + # torch.cat([(_abs[0].weight.T @ _abs[1].weight.T).T for _abs in fm_abs], dim=0) # task-specific params (LoRA) + # ], dim=0) + + # # elif 'to_qkv.bias' in self_param_name: + # # ss = self_param_name.split('.') + + # # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv.bias' + # # return get_parameter(fm, fm_qkv_name) + + # elif 'mlp.fc1' in self_param_name and 'weight' in self_param_name: + # fm_param_name = self_param_name.replace('.linear', '') + # return get_parameter(fm, fm_param_name) + + # elif 'mlp.fc2' in self_param_name and 'weight' in self_param_name: + # fm_param_name = self_param_name + # return get_parameter(fm, fm_param_name) + + # else: + # # return get_parameter(fm, self_param_name) + # return None + + # if 'qkv.weight' in self_param_name: + # ss = self_param_name.split('.') + + # fm_qkv_name = '.'.join(ss[0: -1]) + '.qkv' + # fm_qkv = get_module(fm, fm_qkv_name) + + # fm_abs_name = '.'.join(ss[0: -1]) + '.abs' + # fm_abs = get_module(fm, fm_abs_name) + + # return torch.cat([ + # fm_qkv.weight.data, # task-agnositc params + # torch.cat([(_abs[0].weight.T @ _abs[1].weight.T).T for _abs in fm_abs], dim=0) # task-specific params (LoRA) + # ], dim=0) + + # # elif 'to_qkv.bias' in self_param_name: + # # ss = self_param_name.split('.') + + # # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv.bias' + # # return get_parameter(fm, fm_qkv_name) + + # elif 'mlp.fc1' in self_param_name and 'weight' in self_param_name: + # fm_param_name = self_param_name.replace('.linear', '') + # return get_parameter(fm, fm_param_name) + + # elif 'mlp.fc2' in self_param_name and 'weight' in self_param_name: + # fm_param_name = self_param_name + # res = get_parameter(fm, fm_param_name) + # # print('mlp fc2 debug', fm_param_name, res is None) + # return res + + # else: + # # return get_parameter(fm, self_param_name) + # return None + + # def get_matched_param_of_fm(self, self_param_name, fm: nn.Module): + # if any([k in self_param_name for k in ['fbs', 'cls_token', 'pos_embed']]): + # return None + + # # 1. xx.qkv.to_qkv.yy to xx.qkv.qkv.aa and xx.qkv.abs.zz + # if 'to_qkv.weight' in self_param_name: + # ss = self_param_name.split('.') + + # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv' + # fm_qkv = get_module(fm, fm_qkv_name) + + # fm_abs_name = '.'.join(ss[0: -2]) + '.abs' + # fm_abs = get_module(fm, fm_abs_name) + + # return torch.cat([ + # fm_qkv.weight.data, # task-agnositc params + # torch.cat([(_abs[0].weight.T @ _abs[1].weight.T).T for _abs in fm_abs], dim=0) # task-specific params (LoRA) + # ], dim=0) + + # elif 'to_qkv.bias' in self_param_name: + # ss = self_param_name.split('.') + + # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv.bias' + # return get_parameter(fm, fm_qkv_name) + + # elif 'mlp.fc1' in self_param_name: + # fm_param_name = self_param_name.replace('.linear', '') + # return get_parameter(fm, fm_param_name) + + # else: + # return get_parameter(fm, self_param_name) + if ('attention.attention.projection_query' in self_param_name or 'attention.attention.projection_key' in self_param_name or \ + 'attention.attention.projection_value' in self_param_name) and ('weight' in self_param_name): + ss = self_param_name.split('.') + + fm_qkv_name = '.'.join(ss[0: -1]) + '.qkv' + fm_qkv = get_module(fm, fm_qkv_name) + + fm_abs_name = '.'.join(ss[0: -1]) + '.abs' + fm_abs = get_module(fm, fm_abs_name) + + return torch.cat([ + fm_qkv.weight.data, # task-agnositc params + torch.cat([(_abs[0].weight.T @ _abs[1].weight.T).T for _abs in fm_abs], dim=0) # task-specific params (LoRA) + ], dim=0) + + elif ('attention.attention.projection_query' in self_param_name or 'attention.attention.projection_key' in self_param_name or \ + 'attention.attention.projection_value' in self_param_name) and ('bias' in self_param_name): + ss = self_param_name.split('.') + + fm_qkv_name = '.'.join(ss[0: -1]) + '.qkv.bias' + return get_parameter(fm, fm_qkv_name) + + elif 'intermediate.dense' in self_param_name and 'weight' in self_param_name: + fm_param_name = self_param_name.replace('.linear', '') + return get_parameter(fm, fm_param_name) + elif 'output.dense' in self_param_name and 'weight' in self_param_name: + fm_param_name = self_param_name.replace('.linear', '') + return get_parameter(fm, fm_param_name) + else: + #return get_parameter(fm, self_param_name) + return None + + +if __name__ == '__main__': + from utils.dl.common.env import set_random_seed + set_random_seed(1) + + # 1. init model + from dnns.vit import vit_b_16 + fm_models_dict_path = 'new_impl/cv/results/cvt_md_wo_fbs.py/20231019/999998-135139-/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/new_impl/cv/cvt_md_wo_fbs.py/models/fm_best.pt' + fm_models_dict_path = save_models_dict_for_init(torch.load(fm_models_dict_path), __file__, 'fm_cvt_cls_lora') + pretrained_md_models_dict_path = 'new_impl/cv/results/cvt_md_wo_fbs.py/20231019/999998-135139-/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/new_impl/cv/cvt_md_wo_fbs.py/models/md_best.pt' + md_models_dict = torch.load(pretrained_md_models_dict_path) + md_models_dict_path = save_models_dict_for_init(md_models_dict, __file__, 'md_cvt_cls_pretrained_wo_fbs') + torch.cuda.set_device(1) + device = 'cuda' + + fm_model = ElasticDNN_ViT_OfflineClsFMModel('fm', fm_models_dict_path, device) + md_model = ElasticDNN_ViT_OfflineClsMDModel('md', md_models_dict_path, device) + + # 2. init alg + models = { + 'fm': fm_model, + 'md': md_model + } + from new_impl.cv.elasticdnn.api.algs.md_pretraining_index_v2_train_index_and_md import ElasticDNN_MDPretrainingIndexAlg + + fm_to_md_alg = ElasticDNN_MDPretrainingIndexAlg(models, get_res_save_dir(__file__, sys.argv[0])) + + # 3. init scenario + scenario = build_scenario( + source_datasets_name=['GTA5Cls', 'SuperviselyPersonCls'], + target_datasets_order=['CityscapesCls', 'BaiduPersonCls'] * 15, + da_mode='close_set', + data_dirs={ + 'GTA5Cls': '/data/zql/datasets/gta5_for_cls_task', + 'SuperviselyPersonCls': '/data/zql/datasets/supervisely_person_for_cls_task', + 'CityscapesCls': '/data/zql/datasets/cityscapes_for_cls_task', + 'BaiduPersonCls': '/data/zql/datasets/baidu_person_for_cls_task' + }, + ) + + from utils.dl.common.lr_scheduler import get_linear_schedule_with_warmup + fm_to_md_alg.run(scenario, hyps={ + 'launch_tbboard': False, + + 'samples_size': (1, 3, 224, 224), + + 'FBS_r': 16, + 'FBS_ignore_layers': [], + + 'train_batch_size': 128, + 'val_batch_size': 512, + 'num_workers': 16, + 'optimizer': 'AdamW', + # 'optimizer_args': {'lr': 1e-4, 'betas': [0.9, 0.999], 'weight_decay': 0.01}, + 'indexes_optimizer_args': {'lr': 3e-3, 'betas': [0.9, 0.999], 'weight_decay': 0.1}, + # 'scheduler': 'StepLR', + # 'scheduler_args': {'step_size': 20000, 'gamma': 0.1}, + # 'optimizer': 'AdamW', + # 'optimizer_args': {'lr': 1e-4, 'betas': [0.9, 0.999], 'weight_decay': 0.01}, + 'optimizer_args': {'lr': 1e-3, 'betas': [0.9, 0.999], 'weight_decay': 0.01},#注意学习率的调整，不同的模型不一样。 + + 'scheduler': 'LambdaLR', + 'scheduler_args': {'lr_lambda': get_linear_schedule_with_warmup(10000, 70000)}, + + 'max_sparsity': 0.9, + 'min_sparsity': 0.0, + 'num_iters': 60000, + 'val_freq': 1000, + 'index_loss_weight': 1e-4, + 'l1_reg_loss_weight': 1e-9, + 'val_num_sparsities': 4, + + 'bn_cal_num_iters': 800,#有bn层注意需要加上这个 + + 'index_init': 'zero', + 'index_guided_linear_comb_split_size': 512 + }) + \ No newline at end of file diff --git a/new_impl/cv/cls_md_wo_fbs.py b/new_impl/cv/cls_md_wo_fbs.py new file mode 100644 index 0000000000000000000000000000000000000000..e9f4f59a8929eedd23fc330de89f9087eee5efef --- /dev/null +++ b/new_impl/cv/cls_md_wo_fbs.py @@ -0,0 +1,274 @@ +import torch +import sys +from torch import nn +from new_impl.cv.dnns.vit import make_softmax_prunable +from new_impl.cv.elasticdnn.api.model import ElasticDNN_OfflineClsFMModel, ElasticDNN_OfflineClsMDModel +from new_impl.cv.elasticdnn.api.algs.md_pretraining_wo_fbs import ElasticDNN_MDPretrainingWoFBSAlg +from new_impl.cv.elasticdnn.model.base import ElasticDNNUtil +from new_impl.cv.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from new_impl.cv.elasticdnn.pipeline.offline.fm_to_md.vit import FM_to_MD_ViT_Util +from new_impl.cv.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from new_impl.cv.elasticdnn.pipeline.offline.fm_lora.vit import FMLoRA_ViT_Util +from new_impl.cv.elasticdnn.model.vit import ElasticViTUtil +from utils.dl.common.model import LayerActivation, get_module, get_parameter +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +from utils.dl.common.loss import CrossEntropyLossSoft +import torch.nn.functional as F + + +# class ElasticDNN_ViT_OfflineClsFMModel(ElasticDNN_OfflineClsFMModel): +# def generate_md_by_reducing_width(self, reducing_width_ratio, samples: torch.Tensor): +# return FM_to_MD_ViT_Util().init_md_from_fm_by_reducing_width_with_perf_test(self.models_dict['main'], +# reducing_width_ratio, samples).to(self.device) + +# def get_feature_hook(self) -> LayerActivation: +# return LayerActivation(get_module(self.models_dict['main'], 'head'), True, self.device) + +# def get_elastic_dnn_util(self) -> ElasticDNNUtil: +# return ElasticViTUtil() + +# def forward_to_get_task_loss(self, x, y, *args, **kwargs): +# return F.cross_entropy(self.infer(x), y) + +# def get_lora_util(self) -> FMLoRA_Util: +# return FMLoRA_ViT_Util() + +# def get_task_head_params(self): +# head = get_module(self.models_dict['main'], 'head') +# return list(head.parameters()) + + +# class ElasticDNN_ViT_OfflineClsMDModel(ElasticDNN_OfflineClsMDModel): +# def __init__(self, name: str, models_dict_path: str, device: str): +# super().__init__(name, models_dict_path, device) + +# self.distill_criterion = CrossEntropyLossSoft() + +# def get_feature_hook(self) -> LayerActivation: +# return LayerActivation(get_module(self.models_dict['main'], 'head'), True, self.device) + +# def forward_to_get_task_loss(self, x, y, *args, **kwargs): +# return F.cross_entropy(self.infer(x), y) + +# def get_distill_loss(self, student_output, teacher_output): +# return self.distill_criterion(student_output, teacher_output) + +# def get_matched_param_of_fm(self, self_param_name, fm: nn.Module): +# # only between qkv.weight, norm.weight/bias +# if any([k in self_param_name for k in ['fbs', 'cls_token', 'pos_embed']]): +# return None + +# p = get_parameter(self.models_dict['main'], self_param_name) +# if p.dim() == 0: +# return None +# elif p.dim() == 1 and 'norm' in self_param_name: +# return get_parameter(fm, self_param_name) + +# # 1. xx.qkv.to_qkv.yy to xx.qkv.qkv.aa and xx.qkv.abs.zz +# if 'to_qkv.weight' in self_param_name: +# ss = self_param_name.split('.') + +# fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv' +# fm_qkv = get_module(fm, fm_qkv_name) + +# fm_abs_name = '.'.join(ss[0: -2]) + '.abs' +# fm_abs = get_module(fm, fm_abs_name) + +# return torch.cat([ +# fm_qkv.weight.data, # task-agnositc params +# torch.cat([(_abs[0].weight.T @ _abs[1].weight.T).T for _abs in fm_abs], dim=0) # task-specific params (LoRA) +# ], dim=0) + +# # elif 'to_qkv.bias' in self_param_name: +# # ss = self_param_name.split('.') + +# # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv.bias' +# # return get_parameter(fm, fm_qkv_name) + +# # elif 'mlp.fc1' in self_param_name: +# # fm_param_name = self_param_name.replace('.linear', '') +# # return get_parameter(fm, fm_param_name) + +# else: +# # return get_parameter(fm, self_param_name) +# return None + +# # def get_matched_param_of_fm(self, self_param_name, fm: nn.Module): +# # if any([k in self_param_name for k in ['fbs', 'cls_token', 'pos_embed']]): +# # return None + +# # # 1. xx.qkv.to_qkv.yy to xx.qkv.qkv.aa and xx.qkv.abs.zz +# # if 'to_qkv.weight' in self_param_name: +# # ss = self_param_name.split('.') + +# # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv' +# # fm_qkv = get_module(fm, fm_qkv_name) + +# # fm_abs_name = '.'.join(ss[0: -2]) + '.abs' +# # fm_abs = get_module(fm, fm_abs_name) + +# # return torch.cat([ +# # fm_qkv.weight.data, # task-agnositc params +# # torch.cat([(_abs[0].weight.T @ _abs[1].weight.T).T for _abs in fm_abs], dim=0) # task-specific params (LoRA) +# # ], dim=0) + +# # elif 'to_qkv.bias' in self_param_name: +# # ss = self_param_name.split('.') + +# # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv.bias' +# # return get_parameter(fm, fm_qkv_name) + +# # elif 'mlp.fc1' in self_param_name: +# # fm_param_name = self_param_name.replace('.linear', '') +# # return get_parameter(fm, fm_param_name) + +# # else: +# # return get_parameter(fm, self_param_name) + + + +class ElasticDNN_ViT_OfflineClsFMModel(ElasticDNN_OfflineClsFMModel): + def generate_md_by_reducing_width(self, reducing_width_ratio, samples: torch.Tensor): + return FM_to_MD_ViT_Util().init_md_from_fm_by_reducing_width_with_perf_test(self.models_dict['main'], + reducing_width_ratio, samples).to(self.device) + + def get_feature_hook(self) -> LayerActivation: + return LayerActivation(get_module(self.models_dict['main'], 'classifier'), True, self.device) + + def get_elastic_dnn_util(self) -> ElasticDNNUtil: + return ElasticViTUtil() + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + return F.cross_entropy(self.infer(x).logits, y) + + def get_lora_util(self) -> FMLoRA_Util: + return FMLoRA_ViT_Util() + + def get_task_head_params(self): + head = get_module(self.models_dict['main'], 'classifier') + return list(head.parameters()) + + +class ElasticDNN_ViT_OfflineClsMDModel(ElasticDNN_OfflineClsMDModel): + def __init__(self, name: str, models_dict_path: str, device: str): + super().__init__(name, models_dict_path, device) + + self.distill_criterion = CrossEntropyLossSoft() + + def get_feature_hook(self) -> LayerActivation: + return LayerActivation(get_module(self.models_dict['main'], 'classifier'), True, self.device) + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + return F.cross_entropy(self.infer(x).logits, y) + + def get_distill_loss(self, student_output, teacher_output): + return self.distill_criterion(student_output, teacher_output) + + def get_matched_param_of_fm(self, self_param_name, fm: nn.Module): + # only between qkv.weight, norm.weight/bias + if any([k in self_param_name for k in ['fbs', 'cls_token', 'pos_embed']]): + return None + + p = get_parameter(self.models_dict['main'], self_param_name) + if p.dim() == 0: + return None + elif p.dim() == 1 and 'norm' in self_param_name and 'weight' in self_param_name: + return get_parameter(fm, self_param_name) + + # 1. xx.qkv.to_qkv.yy to xx.qkv.qkv.aa and xx.qkv.abs.zz + if 'qkv.weight' in self_param_name: + ss = self_param_name.split('.') + + fm_qkv_name = '.'.join(ss[0: -1]) + '.qkv' + fm_qkv = get_module(fm, fm_qkv_name) + + fm_abs_name = '.'.join(ss[0: -1]) + '.abs' + fm_abs = get_module(fm, fm_abs_name) + + return torch.cat([ + fm_qkv.weight.data, # task-agnositc params + torch.cat([(_abs[0].weight.T @ _abs[1].weight.T).T for _abs in fm_abs], dim=0) # task-specific params (LoRA) + ], dim=0) + + # elif 'to_qkv.bias' in self_param_name: + # ss = self_param_name.split('.') + + # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv.bias' + # return get_parameter(fm, fm_qkv_name) + + elif 'mlp.fc1' in self_param_name and 'weight' in self_param_name: + fm_param_name = self_param_name.replace('.linear', '') + return get_parameter(fm, fm_param_name) + + elif 'mlp.fc2' in self_param_name and 'weight' in self_param_name: + fm_param_name = self_param_name + return get_parameter(fm, fm_param_name) + + else: + # return get_parameter(fm, self_param_name) + return None + + + + +if __name__ == '__main__': + from utils.dl.common.env import set_random_seed + set_random_seed(1) + + # 1. init model + from dnns.vit import vit_b_16 + fm_models_dict_path = 'new_impl/cv/results/cvt_cls.py/20231019/999994-133914-/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/new_impl/cv/cvt_cls.py/models/fm_best.pt' + fm_models_dict = torch.load(fm_models_dict_path) + fm_models_dict['main'] = make_softmax_prunable(fm_models_dict['main']) + fm_models_dict_path = save_models_dict_for_init(fm_models_dict, __file__, 'fm_cvt_cls_lora') + md_models_dict_path = save_models_dict_for_init({ + 'main': -1 + }, __file__, 'md_cvt_cls_lora') + torch.cuda.set_device(1) + device = 'cuda' + + fm_model = ElasticDNN_ViT_OfflineClsFMModel('fm', fm_models_dict_path, device) + md_model = ElasticDNN_ViT_OfflineClsMDModel('md', md_models_dict_path, device) + + # 2. init alg + models = { + 'fm': fm_model, + 'md': md_model + } + fm_to_md_alg = ElasticDNN_MDPretrainingWoFBSAlg(models, get_res_save_dir(__file__, sys.argv[0])) + + # 3. init scenario + scenario = build_scenario( + source_datasets_name=['GTA5Cls', 'SuperviselyPersonCls'], + target_datasets_order=['CityscapesCls', 'BaiduPersonCls'] * 15, + da_mode='close_set', + data_dirs={ + 'GTA5Cls': '/data/zql/datasets/gta5_for_cls_task', + 'SuperviselyPersonCls': '/data/zql/datasets/supervisely_person_for_cls_task', + 'CityscapesCls': '/data/zql/datasets/cityscapes_for_cls_task', + 'BaiduPersonCls': '/data/zql/datasets/baidu_person_for_cls_task' + }, + ) + + from utils.dl.common.lr_scheduler import get_linear_schedule_with_warmup + fm_to_md_alg.run(scenario, hyps={ + 'launch_tbboard': False, + + 'samples_size': (1, 3, 224, 224), + 'generate_md_width_ratio': 4, + + 'train_batch_size': 128, + 'val_batch_size': 512, + 'num_workers': 16, + 'optimizer': 'AdamW', + 'optimizer_args': {'lr': 1e-4, 'betas': [0.9, 0.999], 'weight_decay': 0.01}, + 'scheduler': 'LambdaLR', + 'scheduler_args': {'lr_lambda': get_linear_schedule_with_warmup(10000, 70000)}, + 'num_iters': 80000, + 'val_freq': 400, + 'distill_loss_weight': 1.0 + }) + + # TODO: + # 1. train MD before inserting FBS? \ No newline at end of file diff --git a/new_impl/cv/cls_online.py b/new_impl/cv/cls_online.py new file mode 100644 index 0000000000000000000000000000000000000000..e12dc28fa5553fbe1972459436c6427a0a9317e4 --- /dev/null +++ b/new_impl/cv/cls_online.py @@ -0,0 +1,112 @@ +from typing import List +from data.dataloader import build_dataloader +# from methods.elasticdnn.api.online_model import ElasticDNN_OnlineModel +from new_impl.cv.elasticdnn.api.online_model_v2 import ElasticDNN_OnlineModel + +import torch +import sys +from torch import nn +from new_impl.cv.elasticdnn.api.model import ElasticDNN_OfflineSegFMModel, ElasticDNN_OfflineSegMDModel +from new_impl.cv.elasticdnn.api.algs.md_pretraining_wo_fbs import ElasticDNN_MDPretrainingWoFBSAlg +from new_impl.cv.elasticdnn.model.base import ElasticDNNUtil +from new_impl.cv.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from new_impl.cv.elasticdnn.pipeline.offline.fm_to_md.vit import FM_to_MD_ViT_Util +from new_impl.cv.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from new_impl.cv.elasticdnn.pipeline.offline.fm_lora.vit import FMLoRA_ViT_Util +from new_impl.cv.elasticdnn.model.vit import ElasticViTUtil +from utils.common.file import ensure_dir +from utils.dl.common.model import LayerActivation, get_module, get_parameter +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +from utils.dl.common.loss import CrossEntropyLossSoft +import torch.nn.functional as F +from utils.dl.common.env import create_tbwriter +import os +from utils.common.log import logger +from utils.common.data_record import write_json +# from methods.shot.shot import OnlineShotModel +from new_impl.cv.feat_align.main import OnlineFeatAlignModel, FeatAlignAlg +import tqdm +from new_impl.cv.feat_align.mmd import mmd_rbf +from new_impl.cv.utils.elasticfm_da import init_online_model, elasticfm_da +torch.cuda.set_device(1) +device = 'cuda' +app_name = 'cls' +sd_sparsity = 0.8 + +settings = { + 'involve_fm': True +} + +scenario = build_scenario( + source_datasets_name=['GTA5Cls', 'SuperviselyPersonCls'], + target_datasets_order=['CityscapesCls', 'BaiduPersonCls'] * 15, + da_mode='close_set', + data_dirs={ + 'GTA5Cls': '/data/zql/datasets/gta5_for_cls_task', + 'SuperviselyPersonCls': '/data/zql/datasets/supervisely_person_for_cls_task', + 'CityscapesCls': '/data/zql/datasets/cityscapes_for_cls_task', + 'BaiduPersonCls': '/data/zql/datasets/baiduperson_for_cls_task' + }, +) + +from new_impl.cv.model import ElasticDNN_ClsOnlineModel +elasticfm_model = ElasticDNN_ClsOnlineModel('cls', init_online_model( + # 'experiments/elasticdnn/vit_b_16/offline/fm_to_md/results/cls_md_index.py/20230529/star_999997-154037-only_prune_mlp/models/fm_best.pt', + # 'experiments/elasticdnn/vit_b_16/offline/fm_to_md/results/cls_md_index.py/20230529/star_999997-154037-only_prune_mlp/models/md_best.pt', + #'experiments/elasticdnn/vit_b_16/offline/fm_to_md/cls/results/cls_md_index.py/20230617/999992-101343-lr1e-5_index_bug_fixed/models/fm_best.pt', + #'experiments/elasticdnn/vit_b_16/offline/fm_to_md/cls/results/cls_md_index.py/20230617/999992-101343-lr1e-5_index_bug_fixed/models/md_best.pt', + 'new_impl/cv/results/cvt_md_index.py/20231019/999999-141144-/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/new_impl/cv/cvt_md_index.py/models/fm_best.pt', + 'new_impl/cv/results/cvt_md_index.py/20231019/999999-141144-/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/new_impl/cv/cvt_md_index.py/models/md_best.pt', + 'cls', __file__ +), device, { + 'md_to_fm_alpha': 1, + 'fm_to_md_alpha': 1 +}) + +da_alg = FeatAlignAlg +from new_impl.cv.model import ClsOnlineFeatAlignModel +da_model = ClsOnlineFeatAlignModel +da_alg_hyp = { + 'CityscapesCls': { + 'train_batch_size': 64, + 'val_batch_size': 512, + 'num_workers': 8, + 'optimizer': 'AdamW', + 'optimizer_args': {'lr': 4e-3/2, 'betas': [0.9, 0.999], 'weight_decay': 0.01},#针对于cvt的online的学习率 + #'optimizer_args': {'lr': 1e-4/2, 'momentum': 0.9}, + 'scheduler': '', + 'scheduler_args': {}, + 'num_iters': 100, + 'val_freq': 20, + 'sd_sparsity': sd_sparsity, + 'feat_align_loss_weight': 3.0 + }, + 'BaiduPersonCls': { + 'train_batch_size': 64, + 'val_batch_size': 512, + 'num_workers': 8, + 'optimizer': 'SGD', + 'optimizer_args': {'lr': 1e-3/2, 'momentum': 0.9}, + 'scheduler': '', + 'scheduler_args': {}, + 'num_iters': 100, + 'val_freq': 20, + 'sd_sparsity': sd_sparsity, + 'feat_align_loss_weight': 0.3 + } +} + + +elasticfm_da( + [app_name], + [scenario], + [elasticfm_model], + [da_alg], + [da_alg_hyp], + [da_model], + device, + settings, + __file__, + sys.argv[0] +) diff --git a/new_impl/cv/cvt_model/config.json b/new_impl/cv/cvt_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..d4199177f1f437ea2d7e9ec39a76471c09c6eac3 --- /dev/null +++ b/new_impl/cv/cvt_model/config.json @@ -0,0 +1,2111 @@ +{ + "architectures": [ + "CvtForImageClassification" + ], + "attention_drop_rate": [ + 0.0, + 0.0, + 0.0 + ], + "cls_token": [ + false, + false, + true + ], + "depth": [ + 1, + 2, + 10 + ], + "drop_path_rate": [ + 0.0, + 0.0, + 0.1 + ], + "drop_rate": [ + 0.0, + 0.0, + 0.0 + ], + "embed_dim": [ + 64, + 192, + 384 + ], + "id2label": { + "0": "tench, Tinca tinca", + "1": "goldfish, Carassius auratus", + "2": "great white shark, white shark, man-eater, man-eating shark, Carcharodon carcharias", + "3": "tiger shark, Galeocerdo cuvieri", + "4": "hammerhead, hammerhead shark", + "5": "electric ray, crampfish, numbfish, torpedo", + "6": "stingray", + "7": "cock", + "8": "hen", + "9": "ostrich, Struthio camelus", + "10": "brambling, Fringilla montifringilla", + "11": "goldfinch, Carduelis carduelis", + "12": "house finch, linnet, Carpodacus mexicanus", + "13": "junco, snowbird", + "14": "indigo bunting, indigo finch, indigo bird, Passerina cyanea", + "15": "robin, American robin, Turdus migratorius", + "16": "bulbul", + "17": "jay", + "18": "magpie", + "19": "chickadee", + "20": "water ouzel, dipper", + "21": "kite", + "22": "bald eagle, American eagle, Haliaeetus leucocephalus", + "23": "vulture", + "24": "great grey owl, great gray owl, Strix nebulosa", + "25": "European fire salamander, Salamandra salamandra", + "26": "common newt, Triturus vulgaris", + "27": "eft", + "28": "spotted salamander, Ambystoma maculatum", + "29": "axolotl, mud puppy, Ambystoma mexicanum", + "30": "bullfrog, Rana catesbeiana", + "31": "tree frog, tree-frog", + "32": "tailed frog, bell toad, ribbed toad, tailed toad, Ascaphus trui", + "33": "loggerhead, loggerhead turtle, Caretta caretta", + "34": "leatherback turtle, leatherback, leathery turtle, Dermochelys coriacea", + "35": "mud turtle", + "36": "terrapin", + "37": "box turtle, box tortoise", + "38": "banded gecko", + "39": "common iguana, iguana, Iguana iguana", + "40": "American chameleon, anole, Anolis carolinensis", + "41": "whiptail, whiptail lizard", + "42": "agama", + "43": "frilled lizard, Chlamydosaurus kingi", + "44": "alligator lizard", + "45": "Gila monster, Heloderma suspectum", + "46": "green lizard, Lacerta viridis", + "47": "African chameleon, Chamaeleo chamaeleon", + "48": "Komodo dragon, Komodo lizard, dragon lizard, giant lizard, Varanus komodoensis", + "49": "African crocodile, Nile crocodile, Crocodylus niloticus", + "50": "American alligator, Alligator mississipiensis", + "51": "triceratops", + "52": "thunder snake, worm snake, Carphophis amoenus", + "53": "ringneck snake, ring-necked snake, ring snake", + "54": "hognose snake, puff adder, sand viper", + "55": "green snake, grass snake", + "56": "king snake, kingsnake", + "57": "garter snake, grass snake", + "58": "water snake", + "59": "vine snake", + "60": "night snake, Hypsiglena torquata", + "61": "boa constrictor, Constrictor constrictor", + "62": "rock python, rock snake, Python sebae", + "63": "Indian cobra, Naja naja", + "64": "green mamba", + "65": "sea snake", + "66": "horned viper, cerastes, sand viper, horned asp, Cerastes cornutus", + "67": "diamondback, diamondback rattlesnake, Crotalus adamanteus", + "68": "sidewinder, horned rattlesnake, Crotalus cerastes", + "69": "trilobite", + "70": "harvestman, daddy longlegs, Phalangium opilio", + "71": "scorpion", + "72": "black and gold garden spider, Argiope aurantia", + "73": "barn spider, Araneus cavaticus", + "74": "garden spider, Aranea diademata", + "75": "black widow, Latrodectus mactans", + "76": "tarantula", + "77": "wolf spider, hunting spider", + "78": "tick", + "79": "centipede", + "80": "black grouse", + "81": "ptarmigan", + "82": "ruffed grouse, partridge, Bonasa umbellus", + "83": "prairie chicken, prairie grouse, prairie fowl", + "84": "peacock", + "85": "quail", + "86": "partridge", + "87": "African grey, African gray, Psittacus erithacus", + "88": "macaw", + "89": "sulphur-crested cockatoo, Kakatoe galerita, Cacatua galerita", + "90": "lorikeet", + "91": "coucal", + "92": "bee eater", + "93": "hornbill", + "94": "hummingbird", + "95": "jacamar", + "96": "toucan", + "97": "drake", + "98": "red-breasted merganser, Mergus serrator", + "99": "goose", + "100": "black swan, Cygnus atratus", + "101": "tusker", + "102": "echidna, spiny anteater, anteater", + "103": "platypus, duckbill, duckbilled platypus, duck-billed platypus, Ornithorhynchus anatinus", + "104": "wallaby, brush kangaroo", + "105": "koala, koala bear, kangaroo bear, native bear, Phascolarctos cinereus", + "106": "wombat", + "107": "jellyfish", + "108": "sea anemone, anemone", + "109": "brain coral", + "110": "flatworm, platyhelminth", + "111": "nematode, nematode worm, roundworm", + "112": "conch", + "113": "snail", + "114": "slug", + "115": "sea slug, nudibranch", + "116": "chiton, coat-of-mail shell, sea cradle, polyplacophore", + "117": "chambered nautilus, pearly nautilus, nautilus", + "118": "Dungeness crab, Cancer magister", + "119": "rock crab, Cancer irroratus", + "120": "fiddler crab", + "121": "king crab, Alaska crab, Alaskan king crab, Alaska king crab, Paralithodes camtschatica", + "122": "American lobster, Northern lobster, Maine lobster, Homarus americanus", + "123": "spiny lobster, langouste, rock lobster, crawfish, crayfish, sea crawfish", + "124": "crayfish, crawfish, crawdad, crawdaddy", + "125": "hermit crab", + "126": "isopod", + "127": "white stork, Ciconia ciconia", + "128": "black stork, Ciconia nigra", + "129": "spoonbill", + "130": "flamingo", + "131": "little blue heron, Egretta caerulea", + "132": "American egret, great white heron, Egretta albus", + "133": "bittern", + "134": "crane", + "135": "limpkin, Aramus pictus", + "136": "European gallinule, Porphyrio porphyrio", + "137": "American coot, marsh hen, mud hen, water hen, Fulica americana", + "138": "bustard", + "139": "ruddy turnstone, Arenaria interpres", + "140": "red-backed sandpiper, dunlin, Erolia alpina", + "141": "redshank, Tringa totanus", + "142": "dowitcher", + "143": "oystercatcher, oyster catcher", + "144": "pelican", + "145": "king penguin, Aptenodytes patagonica", + "146": "albatross, mollymawk", + "147": "grey whale, gray whale, devilfish, Eschrichtius gibbosus, Eschrichtius robustus", + "148": "killer whale, killer, orca, grampus, sea wolf, Orcinus orca", + "149": "dugong, Dugong dugon", + "150": "sea lion", + "151": "Chihuahua", + "152": "Japanese spaniel", + "153": "Maltese dog, Maltese terrier, Maltese", + "154": "Pekinese, Pekingese, Peke", + "155": "Shih-Tzu", + "156": "Blenheim spaniel", + "157": "papillon", + "158": "toy terrier", + "159": "Rhodesian ridgeback", + "160": "Afghan hound, Afghan", + "161": "basset, basset hound", + "162": "beagle", + "163": "bloodhound, sleuthhound", + "164": "bluetick", + "165": "black-and-tan coonhound", + "166": "Walker hound, Walker foxhound", + "167": "English foxhound", + "168": "redbone", + "169": "borzoi, Russian wolfhound", + "170": "Irish wolfhound", + "171": "Italian greyhound", + "172": "whippet", + "173": "Ibizan hound, Ibizan Podenco", + "174": "Norwegian elkhound, elkhound", + "175": "otterhound, otter hound", + "176": "Saluki, gazelle hound", + "177": "Scottish deerhound, deerhound", + "178": "Weimaraner", + "179": "Staffordshire bullterrier, Staffordshire bull terrier", + "180": "American Staffordshire terrier, Staffordshire terrier, American pit bull terrier, pit bull terrier", + "181": "Bedlington terrier", + "182": "Border terrier", + "183": "Kerry blue terrier", + "184": "Irish terrier", + "185": "Norfolk terrier", + "186": "Norwich terrier", + "187": "Yorkshire terrier", + "188": "wire-haired fox terrier", + "189": "Lakeland terrier", + "190": "Sealyham terrier, Sealyham", + "191": "Airedale, Airedale terrier", + "192": "cairn, cairn terrier", + "193": "Australian terrier", + "194": "Dandie Dinmont, Dandie Dinmont terrier", + "195": "Boston bull, Boston terrier", + "196": "miniature schnauzer", + "197": "giant schnauzer", + "198": "standard schnauzer", + "199": "Scotch terrier, Scottish terrier, Scottie", + "200": "Tibetan terrier, chrysanthemum dog", + "201": "silky terrier, Sydney silky", + "202": "soft-coated wheaten terrier", + "203": "West Highland white terrier", + "204": "Lhasa, Lhasa apso", + "205": "flat-coated retriever", + "206": "curly-coated retriever", + "207": "golden retriever", + "208": "Labrador retriever", + "209": "Chesapeake Bay retriever", + "210": "German short-haired pointer", + "211": "vizsla, Hungarian pointer", + "212": "English setter", + "213": "Irish setter, red setter", + "214": "Gordon setter", + "215": "Brittany spaniel", + "216": "clumber, clumber spaniel", + "217": "English springer, English springer spaniel", + "218": "Welsh springer spaniel", + "219": "cocker spaniel, English cocker spaniel, cocker", + "220": "Sussex spaniel", + "221": "Irish water spaniel", + "222": "kuvasz", + "223": "schipperke", + "224": "groenendael", + "225": "malinois", + "226": "briard", + "227": "kelpie", + "228": "komondor", + "229": "Old English sheepdog, bobtail", + "230": "Shetland sheepdog, Shetland sheep dog, Shetland", + "231": "collie", + "232": "Border collie", + "233": "Bouvier des Flandres, Bouviers des Flandres", + "234": "Rottweiler", + "235": "German shepherd, German shepherd dog, German police dog, alsatian", + "236": "Doberman, Doberman pinscher", + "237": "miniature pinscher", + "238": "Greater Swiss Mountain dog", + "239": "Bernese mountain dog", + "240": "Appenzeller", + "241": "EntleBucher", + "242": "boxer", + "243": "bull mastiff", + "244": "Tibetan mastiff", + "245": "French bulldog", + "246": "Great Dane", + "247": "Saint Bernard, St Bernard", + "248": "Eskimo dog, husky", + "249": "malamute, malemute, Alaskan malamute", + "250": "Siberian husky", + "251": "dalmatian, coach dog, carriage dog", + "252": "affenpinscher, monkey pinscher, monkey dog", + "253": "basenji", + "254": "pug, pug-dog", + "255": "Leonberg", + "256": "Newfoundland, Newfoundland dog", + "257": "Great Pyrenees", + "258": "Samoyed, Samoyede", + "259": "Pomeranian", + "260": "chow, chow chow", + "261": "keeshond", + "262": "Brabancon griffon", + "263": "Pembroke, Pembroke Welsh corgi", + "264": "Cardigan, Cardigan Welsh corgi", + "265": "toy poodle", + "266": "miniature poodle", + "267": "standard poodle", + "268": "Mexican hairless", + "269": "timber wolf, grey wolf, gray wolf, Canis lupus", + "270": "white wolf, Arctic wolf, Canis lupus tundrarum", + "271": "red wolf, maned wolf, Canis rufus, Canis niger", + "272": "coyote, prairie wolf, brush wolf, Canis latrans", + "273": "dingo, warrigal, warragal, Canis dingo", + "274": "dhole, Cuon alpinus", + "275": "African hunting dog, hyena dog, Cape hunting dog, Lycaon pictus", + "276": "hyena, hyaena", + "277": "red fox, Vulpes vulpes", + "278": "kit fox, Vulpes macrotis", + "279": "Arctic fox, white fox, Alopex lagopus", + "280": "grey fox, gray fox, Urocyon cinereoargenteus", + "281": "tabby, tabby cat", + "282": "tiger cat", + "283": "Persian cat", + "284": "Siamese cat, Siamese", + "285": "Egyptian cat", + "286": "cougar, puma, catamount, mountain lion, painter, panther, Felis concolor", + "287": "lynx, catamount", + "288": "leopard, Panthera pardus", + "289": "snow leopard, ounce, Panthera uncia", + "290": "jaguar, panther, Panthera onca, Felis onca", + "291": "lion, king of beasts, Panthera leo", + "292": "tiger, Panthera tigris", + "293": "cheetah, chetah, Acinonyx jubatus", + "294": "brown bear, bruin, Ursus arctos", + "295": "American black bear, black bear, Ursus americanus, Euarctos americanus", + "296": "ice bear, polar bear, Ursus Maritimus, Thalarctos maritimus", + "297": "sloth bear, Melursus ursinus, Ursus ursinus", + "298": "mongoose", + "299": "meerkat, mierkat", + "300": "tiger beetle", + "301": "ladybug, ladybeetle, lady beetle, ladybird, ladybird beetle", + "302": "ground beetle, carabid beetle", + "303": "long-horned beetle, longicorn, longicorn beetle", + "304": "leaf beetle, chrysomelid", + "305": "dung beetle", + "306": "rhinoceros beetle", + "307": "weevil", + "308": "fly", + "309": "bee", + "310": "ant, emmet, pismire", + "311": "grasshopper, hopper", + "312": "cricket", + "313": "walking stick, walkingstick, stick insect", + "314": "cockroach, roach", + "315": "mantis, mantid", + "316": "cicada, cicala", + "317": "leafhopper", + "318": "lacewing, lacewing fly", + "319": "dragonfly, darning needle, devil's darning needle, sewing needle, snake feeder, snake doctor, mosquito hawk, skeeter hawk", + "320": "damselfly", + "321": "admiral", + "322": "ringlet, ringlet butterfly", + "323": "monarch, monarch butterfly, milkweed butterfly, Danaus plexippus", + "324": "cabbage butterfly", + "325": "sulphur butterfly, sulfur butterfly", + "326": "lycaenid, lycaenid butterfly", + "327": "starfish, sea star", + "328": "sea urchin", + "329": "sea cucumber, holothurian", + "330": "wood rabbit, cottontail, cottontail rabbit", + "331": "hare", + "332": "Angora, Angora rabbit", + "333": "hamster", + "334": "porcupine, hedgehog", + "335": "fox squirrel, eastern fox squirrel, Sciurus niger", + "336": "marmot", + "337": "beaver", + "338": "guinea pig, Cavia cobaya", + "339": "sorrel", + "340": "zebra", + "341": "hog, pig, grunter, squealer, Sus scrofa", + "342": "wild boar, boar, Sus scrofa", + "343": "warthog", + "344": "hippopotamus, hippo, river horse, Hippopotamus amphibius", + "345": "ox", + "346": "water buffalo, water ox, Asiatic buffalo, Bubalus bubalis", + "347": "bison", + "348": "ram, tup", + "349": "bighorn, bighorn sheep, cimarron, Rocky Mountain bighorn, Rocky Mountain sheep, Ovis canadensis", + "350": "ibex, Capra ibex", + "351": "hartebeest", + "352": "impala, Aepyceros melampus", + "353": "gazelle", + "354": "Arabian camel, dromedary, Camelus dromedarius", + "355": "llama", + "356": "weasel", + "357": "mink", + "358": "polecat, fitch, foulmart, foumart, Mustela putorius", + "359": "black-footed ferret, ferret, Mustela nigripes", + "360": "otter", + "361": "skunk, polecat, wood pussy", + "362": "badger", + "363": "armadillo", + "364": "three-toed sloth, ai, Bradypus tridactylus", + "365": "orangutan, orang, orangutang, Pongo pygmaeus", + "366": "gorilla, Gorilla gorilla", + "367": "chimpanzee, chimp, Pan troglodytes", + "368": "gibbon, Hylobates lar", + "369": "siamang, Hylobates syndactylus, Symphalangus syndactylus", + "370": "guenon, guenon monkey", + "371": "patas, hussar monkey, Erythrocebus patas", + "372": "baboon", + "373": "macaque", + "374": "langur", + "375": "colobus, colobus monkey", + "376": "proboscis monkey, Nasalis larvatus", + "377": "marmoset", + "378": "capuchin, ringtail, Cebus capucinus", + "379": "howler monkey, howler", + "380": "titi, titi monkey", + "381": "spider monkey, Ateles geoffroyi", + "382": "squirrel monkey, Saimiri sciureus", + "383": "Madagascar cat, ring-tailed lemur, Lemur catta", + "384": "indri, indris, Indri indri, Indri brevicaudatus", + "385": "Indian elephant, Elephas maximus", + "386": "African elephant, Loxodonta africana", + "387": "lesser panda, red panda, panda, bear cat, cat bear, Ailurus fulgens", + "388": "giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca", + "389": "barracouta, snoek", + "390": "eel", + "391": "coho, cohoe, coho salmon, blue jack, silver salmon, Oncorhynchus kisutch", + "392": "rock beauty, Holocanthus tricolor", + "393": "anemone fish", + "394": "sturgeon", + "395": "gar, garfish, garpike, billfish, Lepisosteus osseus", + "396": "lionfish", + "397": "puffer, pufferfish, blowfish, globefish", + "398": "abacus", + "399": "abaya", + "400": "academic gown, academic robe, judge's robe", + "401": "accordion, piano accordion, squeeze box", + "402": "acoustic guitar", + "403": "aircraft carrier, carrier, flattop, attack aircraft carrier", + "404": "airliner", + "405": "airship, dirigible", + "406": "altar", + "407": "ambulance", + "408": "amphibian, amphibious vehicle", + "409": "analog clock", + "410": "apiary, bee house", + "411": "apron", + "412": "ashcan, trash can, garbage can, wastebin, ash bin, ash-bin, ashbin, dustbin, trash barrel, trash bin", + "413": "assault rifle, assault gun", + "414": "backpack, back pack, knapsack, packsack, rucksack, haversack", + "415": "bakery, bakeshop, bakehouse", + "416": "balance beam, beam", + "417": "balloon", + "418": "ballpoint, ballpoint pen, ballpen, Biro", + "419": "Band Aid", + "420": "banjo", + "421": "bannister, banister, balustrade, balusters, handrail", + "422": "barbell", + "423": "barber chair", + "424": "barbershop", + "425": "barn", + "426": "barometer", + "427": "barrel, cask", + "428": "barrow, garden cart, lawn cart, wheelbarrow", + "429": "baseball", + "430": "basketball", + "431": "bassinet", + "432": "bassoon", + "433": "bathing cap, swimming cap", + "434": "bath towel", + "435": "bathtub, bathing tub, bath, tub", + "436": "beach wagon, station wagon, wagon, estate car, beach waggon, station waggon, waggon", + "437": "beacon, lighthouse, beacon light, pharos", + "438": "beaker", + "439": "bearskin, busby, shako", + "440": "beer bottle", + "441": "beer glass", + "442": "bell cote, bell cot", + "443": "bib", + "444": "bicycle-built-for-two, tandem bicycle, tandem", + "445": "bikini, two-piece", + "446": "binder, ring-binder", + "447": "binoculars, field glasses, opera glasses", + "448": "birdhouse", + "449": "boathouse", + "450": "bobsled, bobsleigh, bob", + "451": "bolo tie, bolo, bola tie, bola", + "452": "bonnet, poke bonnet", + "453": "bookcase", + "454": "bookshop, bookstore, bookstall", + "455": "bottlecap", + "456": "bow", + "457": "bow tie, bow-tie, bowtie", + "458": "brass, memorial tablet, plaque", + "459": "brassiere, bra, bandeau", + "460": "breakwater, groin, groyne, mole, bulwark, seawall, jetty", + "461": "breastplate, aegis, egis", + "462": "broom", + "463": "bucket, pail", + "464": "buckle", + "465": "bulletproof vest", + "466": "bullet train, bullet", + "467": "butcher shop, meat market", + "468": "cab, hack, taxi, taxicab", + "469": "caldron, cauldron", + "470": "candle, taper, wax light", + "471": "cannon", + "472": "canoe", + "473": "can opener, tin opener", + "474": "cardigan", + "475": "car mirror", + "476": "carousel, carrousel, merry-go-round, roundabout, whirligig", + "477": "carpenter's kit, tool kit", + "478": "carton", + "479": "car wheel", + "480": "cash machine, cash dispenser, automated teller machine, automatic teller machine, automated teller, automatic teller, ATM", + "481": "cassette", + "482": "cassette player", + "483": "castle", + "484": "catamaran", + "485": "CD player", + "486": "cello, violoncello", + "487": "cellular telephone, cellular phone, cellphone, cell, mobile phone", + "488": "chain", + "489": "chainlink fence", + "490": "chain mail, ring mail, mail, chain armor, chain armour, ring armor, ring armour", + "491": "chain saw, chainsaw", + "492": "chest", + "493": "chiffonier, commode", + "494": "chime, bell, gong", + "495": "china cabinet, china closet", + "496": "Christmas stocking", + "497": "church, church building", + "498": "cinema, movie theater, movie theatre, movie house, picture palace", + "499": "cleaver, meat cleaver, chopper", + "500": "cliff dwelling", + "501": "cloak", + "502": "clog, geta, patten, sabot", + "503": "cocktail shaker", + "504": "coffee mug", + "505": "coffeepot", + "506": "coil, spiral, volute, whorl, helix", + "507": "combination lock", + "508": "computer keyboard, keypad", + "509": "confectionery, confectionary, candy store", + "510": "container ship, containership, container vessel", + "511": "convertible", + "512": "corkscrew, bottle screw", + "513": "cornet, horn, trumpet, trump", + "514": "cowboy boot", + "515": "cowboy hat, ten-gallon hat", + "516": "cradle", + "517": "crane", + "518": "crash helmet", + "519": "crate", + "520": "crib, cot", + "521": "Crock Pot", + "522": "croquet ball", + "523": "crutch", + "524": "cuirass", + "525": "dam, dike, dyke", + "526": "desk", + "527": "desktop computer", + "528": "dial telephone, dial phone", + "529": "diaper, nappy, napkin", + "530": "digital clock", + "531": "digital watch", + "532": "dining table, board", + "533": "dishrag, dishcloth", + "534": "dishwasher, dish washer, dishwashing machine", + "535": "disk brake, disc brake", + "536": "dock, dockage, docking facility", + "537": "dogsled, dog sled, dog sleigh", + "538": "dome", + "539": "doormat, welcome mat", + "540": "drilling platform, offshore rig", + "541": "drum, membranophone, tympan", + "542": "drumstick", + "543": "dumbbell", + "544": "Dutch oven", + "545": "electric fan, blower", + "546": "electric guitar", + "547": "electric locomotive", + "548": "entertainment center", + "549": "envelope", + "550": "espresso maker", + "551": "face powder", + "552": "feather boa, boa", + "553": "file, file cabinet, filing cabinet", + "554": "fireboat", + "555": "fire engine, fire truck", + "556": "fire screen, fireguard", + "557": "flagpole, flagstaff", + "558": "flute, transverse flute", + "559": "folding chair", + "560": "football helmet", + "561": "forklift", + "562": "fountain", + "563": "fountain pen", + "564": "four-poster", + "565": "freight car", + "566": "French horn, horn", + "567": "frying pan, frypan, skillet", + "568": "fur coat", + "569": "garbage truck, dustcart", + "570": "gasmask, respirator, gas helmet", + "571": "gas pump, gasoline pump, petrol pump, island dispenser", + "572": "goblet", + "573": "go-kart", + "574": "golf ball", + "575": "golfcart, golf cart", + "576": "gondola", + "577": "gong, tam-tam", + "578": "gown", + "579": "grand piano, grand", + "580": "greenhouse, nursery, glasshouse", + "581": "grille, radiator grille", + "582": "grocery store, grocery, food market, market", + "583": "guillotine", + "584": "hair slide", + "585": "hair spray", + "586": "half track", + "587": "hammer", + "588": "hamper", + "589": "hand blower, blow dryer, blow drier, hair dryer, hair drier", + "590": "hand-held computer, hand-held microcomputer", + "591": "handkerchief, hankie, hanky, hankey", + "592": "hard disc, hard disk, fixed disk", + "593": "harmonica, mouth organ, harp, mouth harp", + "594": "harp", + "595": "harvester, reaper", + "596": "hatchet", + "597": "holster", + "598": "home theater, home theatre", + "599": "honeycomb", + "600": "hook, claw", + "601": "hoopskirt, crinoline", + "602": "horizontal bar, high bar", + "603": "horse cart, horse-cart", + "604": "hourglass", + "605": "iPod", + "606": "iron, smoothing iron", + "607": "jack-o'-lantern", + "608": "jean, blue jean, denim", + "609": "jeep, landrover", + "610": "jersey, T-shirt, tee shirt", + "611": "jigsaw puzzle", + "612": "jinrikisha, ricksha, rickshaw", + "613": "joystick", + "614": "kimono", + "615": "knee pad", + "616": "knot", + "617": "lab coat, laboratory coat", + "618": "ladle", + "619": "lampshade, lamp shade", + "620": "laptop, laptop computer", + "621": "lawn mower, mower", + "622": "lens cap, lens cover", + "623": "letter opener, paper knife, paperknife", + "624": "library", + "625": "lifeboat", + "626": "lighter, light, igniter, ignitor", + "627": "limousine, limo", + "628": "liner, ocean liner", + "629": "lipstick, lip rouge", + "630": "Loafer", + "631": "lotion", + "632": "loudspeaker, speaker, speaker unit, loudspeaker system, speaker system", + "633": "loupe, jeweler's loupe", + "634": "lumbermill, sawmill", + "635": "magnetic compass", + "636": "mailbag, postbag", + "637": "mailbox, letter box", + "638": "maillot", + "639": "maillot, tank suit", + "640": "manhole cover", + "641": "maraca", + "642": "marimba, xylophone", + "643": "mask", + "644": "matchstick", + "645": "maypole", + "646": "maze, labyrinth", + "647": "measuring cup", + "648": "medicine chest, medicine cabinet", + "649": "megalith, megalithic structure", + "650": "microphone, mike", + "651": "microwave, microwave oven", + "652": "military uniform", + "653": "milk can", + "654": "minibus", + "655": "miniskirt, mini", + "656": "minivan", + "657": "missile", + "658": "mitten", + "659": "mixing bowl", + "660": "mobile home, manufactured home", + "661": "Model T", + "662": "modem", + "663": "monastery", + "664": "monitor", + "665": "moped", + "666": "mortar", + "667": "mortarboard", + "668": "mosque", + "669": "mosquito net", + "670": "motor scooter, scooter", + "671": "mountain bike, all-terrain bike, off-roader", + "672": "mountain tent", + "673": "mouse, computer mouse", + "674": "mousetrap", + "675": "moving van", + "676": "muzzle", + "677": "nail", + "678": "neck brace", + "679": "necklace", + "680": "nipple", + "681": "notebook, notebook computer", + "682": "obelisk", + "683": "oboe, hautboy, hautbois", + "684": "ocarina, sweet potato", + "685": "odometer, hodometer, mileometer, milometer", + "686": "oil filter", + "687": "organ, pipe organ", + "688": "oscilloscope, scope, cathode-ray oscilloscope, CRO", + "689": "overskirt", + "690": "oxcart", + "691": "oxygen mask", + "692": "packet", + "693": "paddle, boat paddle", + "694": "paddlewheel, paddle wheel", + "695": "padlock", + "696": "paintbrush", + "697": "pajama, pyjama, pj's, jammies", + "698": "palace", + "699": "panpipe, pandean pipe, syrinx", + "700": "paper towel", + "701": "parachute, chute", + "702": "parallel bars, bars", + "703": "park bench", + "704": "parking meter", + "705": "passenger car, coach, carriage", + "706": "patio, terrace", + "707": "pay-phone, pay-station", + "708": "pedestal, plinth, footstall", + "709": "pencil box, pencil case", + "710": "pencil sharpener", + "711": "perfume, essence", + "712": "Petri dish", + "713": "photocopier", + "714": "pick, plectrum, plectron", + "715": "pickelhaube", + "716": "picket fence, paling", + "717": "pickup, pickup truck", + "718": "pier", + "719": "piggy bank, penny bank", + "720": "pill bottle", + "721": "pillow", + "722": "ping-pong ball", + "723": "pinwheel", + "724": "pirate, pirate ship", + "725": "pitcher, ewer", + "726": "plane, carpenter's plane, woodworking plane", + "727": "planetarium", + "728": "plastic bag", + "729": "plate rack", + "730": "plow, plough", + "731": "plunger, plumber's helper", + "732": "Polaroid camera, Polaroid Land camera", + "733": "pole", + "734": "police van, police wagon, paddy wagon, patrol wagon, wagon, black Maria", + "735": "poncho", + "736": "pool table, billiard table, snooker table", + "737": "pop bottle, soda bottle", + "738": "pot, flowerpot", + "739": "potter's wheel", + "740": "power drill", + "741": "prayer rug, prayer mat", + "742": "printer", + "743": "prison, prison house", + "744": "projectile, missile", + "745": "projector", + "746": "puck, hockey puck", + "747": "punching bag, punch bag, punching ball, punchball", + "748": "purse", + "749": "quill, quill pen", + "750": "quilt, comforter, comfort, puff", + "751": "racer, race car, racing car", + "752": "racket, racquet", + "753": "radiator", + "754": "radio, wireless", + "755": "radio telescope, radio reflector", + "756": "rain barrel", + "757": "recreational vehicle, RV, R.V.", + "758": "reel", + "759": "reflex camera", + "760": "refrigerator, icebox", + "761": "remote control, remote", + "762": "restaurant, eating house, eating place, eatery", + "763": "revolver, six-gun, six-shooter", + "764": "rifle", + "765": "rocking chair, rocker", + "766": "rotisserie", + "767": "rubber eraser, rubber, pencil eraser", + "768": "rugby ball", + "769": "rule, ruler", + "770": "running shoe", + "771": "safe", + "772": "safety pin", + "773": "saltshaker, salt shaker", + "774": "sandal", + "775": "sarong", + "776": "sax, saxophone", + "777": "scabbard", + "778": "scale, weighing machine", + "779": "school bus", + "780": "schooner", + "781": "scoreboard", + "782": "screen, CRT screen", + "783": "screw", + "784": "screwdriver", + "785": "seat belt, seatbelt", + "786": "sewing machine", + "787": "shield, buckler", + "788": "shoe shop, shoe-shop, shoe store", + "789": "shoji", + "790": "shopping basket", + "791": "shopping cart", + "792": "shovel", + "793": "shower cap", + "794": "shower curtain", + "795": "ski", + "796": "ski mask", + "797": "sleeping bag", + "798": "slide rule, slipstick", + "799": "sliding door", + "800": "slot, one-armed bandit", + "801": "snorkel", + "802": "snowmobile", + "803": "snowplow, snowplough", + "804": "soap dispenser", + "805": "soccer ball", + "806": "sock", + "807": "solar dish, solar collector, solar furnace", + "808": "sombrero", + "809": "soup bowl", + "810": "space bar", + "811": "space heater", + "812": "space shuttle", + "813": "spatula", + "814": "speedboat", + "815": "spider web, spider's web", + "816": "spindle", + "817": "sports car, sport car", + "818": "spotlight, spot", + "819": "stage", + "820": "steam locomotive", + "821": "steel arch bridge", + "822": "steel drum", + "823": "stethoscope", + "824": "stole", + "825": "stone wall", + "826": "stopwatch, stop watch", + "827": "stove", + "828": "strainer", + "829": "streetcar, tram, tramcar, trolley, trolley car", + "830": "stretcher", + "831": "studio couch, day bed", + "832": "stupa, tope", + "833": "submarine, pigboat, sub, U-boat", + "834": "suit, suit of clothes", + "835": "sundial", + "836": "sunglass", + "837": "sunglasses, dark glasses, shades", + "838": "sunscreen, sunblock, sun blocker", + "839": "suspension bridge", + "840": "swab, swob, mop", + "841": "sweatshirt", + "842": "swimming trunks, bathing trunks", + "843": "swing", + "844": "switch, electric switch, electrical switch", + "845": "syringe", + "846": "table lamp", + "847": "tank, army tank, armored combat vehicle, armoured combat vehicle", + "848": "tape player", + "849": "teapot", + "850": "teddy, teddy bear", + "851": "television, television system", + "852": "tennis ball", + "853": "thatch, thatched roof", + "854": "theater curtain, theatre curtain", + "855": "thimble", + "856": "thresher, thrasher, threshing machine", + "857": "throne", + "858": "tile roof", + "859": "toaster", + "860": "tobacco shop, tobacconist shop, tobacconist", + "861": "toilet seat", + "862": "torch", + "863": "totem pole", + "864": "tow truck, tow car, wrecker", + "865": "toyshop", + "866": "tractor", + "867": "trailer truck, tractor trailer, trucking rig, rig, articulated lorry, semi", + "868": "tray", + "869": "trench coat", + "870": "tricycle, trike, velocipede", + "871": "trimaran", + "872": "tripod", + "873": "triumphal arch", + "874": "trolleybus, trolley coach, trackless trolley", + "875": "trombone", + "876": "tub, vat", + "877": "turnstile", + "878": "typewriter keyboard", + "879": "umbrella", + "880": "unicycle, monocycle", + "881": "upright, upright piano", + "882": "vacuum, vacuum cleaner", + "883": "vase", + "884": "vault", + "885": "velvet", + "886": "vending machine", + "887": "vestment", + "888": "viaduct", + "889": "violin, fiddle", + "890": "volleyball", + "891": "waffle iron", + "892": "wall clock", + "893": "wallet, billfold, notecase, pocketbook", + "894": "wardrobe, closet, press", + "895": "warplane, military plane", + "896": "washbasin, handbasin, washbowl, lavabo, wash-hand basin", + "897": "washer, automatic washer, washing machine", + "898": "water bottle", + "899": "water jug", + "900": "water tower", + "901": "whiskey jug", + "902": "whistle", + "903": "wig", + "904": "window screen", + "905": "window shade", + "906": "Windsor tie", + "907": "wine bottle", + "908": "wing", + "909": "wok", + "910": "wooden spoon", + "911": "wool, woolen, woollen", + "912": "worm fence, snake fence, snake-rail fence, Virginia fence", + "913": "wreck", + "914": "yawl", + "915": "yurt", + "916": "web site, website, internet site, site", + "917": "comic book", + "918": "crossword puzzle, crossword", + "919": "street sign", + "920": "traffic light, traffic signal, stoplight", + "921": "book jacket, dust cover, dust jacket, dust wrapper", + "922": "menu", + "923": "plate", + "924": "guacamole", + "925": "consomme", + "926": "hot pot, hotpot", + "927": "trifle", + "928": "ice cream, icecream", + "929": "ice lolly, lolly, lollipop, popsicle", + "930": "French loaf", + "931": "bagel, beigel", + "932": "pretzel", + "933": "cheeseburger", + "934": "hotdog, hot dog, red hot", + "935": "mashed potato", + "936": "head cabbage", + "937": "broccoli", + "938": "cauliflower", + "939": "zucchini, courgette", + "940": "spaghetti squash", + "941": "acorn squash", + "942": "butternut squash", + "943": "cucumber, cuke", + "944": "artichoke, globe artichoke", + "945": "bell pepper", + "946": "cardoon", + "947": "mushroom", + "948": "Granny Smith", + "949": "strawberry", + "950": "orange", + "951": "lemon", + "952": "fig", + "953": "pineapple, ananas", + "954": "banana", + "955": "jackfruit, jak, jack", + "956": "custard apple", + "957": "pomegranate", + "958": "hay", + "959": "carbonara", + "960": "chocolate sauce, chocolate syrup", + "961": "dough", + "962": "meat loaf, meatloaf", + "963": "pizza, pizza pie", + "964": "potpie", + "965": "burrito", + "966": "red wine", + "967": "espresso", + "968": "cup", + "969": "eggnog", + "970": "alp", + "971": "bubble", + "972": "cliff, drop, drop-off", + "973": "coral reef", + "974": "geyser", + "975": "lakeside, lakeshore", + "976": "promontory, headland, head, foreland", + "977": "sandbar, sand bar", + "978": "seashore, coast, seacoast, sea-coast", + "979": "valley, vale", + "980": "volcano", + "981": "ballplayer, baseball player", + "982": "groom, bridegroom", + "983": "scuba diver", + "984": "rapeseed", + "985": "daisy", + "986": "yellow lady's slipper, yellow lady-slipper, Cypripedium calceolus, Cypripedium parviflorum", + "987": "corn", + "988": "acorn", + "989": "hip, rose hip, rosehip", + "990": "buckeye, horse chestnut, conker", + "991": "coral fungus", + "992": "agaric", + "993": "gyromitra", + "994": "stinkhorn, carrion fungus", + "995": "earthstar", + "996": "hen-of-the-woods, hen of the woods, Polyporus frondosus, Grifola frondosa", + "997": "bolete", + "998": "ear, spike, capitulum", + "999": "toilet tissue, toilet paper, bathroom tissue" + }, + "image_size": 224, + "initializer_range": 0.02, + "kernel_qkv": [ + 3, + 3, + 3 + ], + "label2id": { + "Afghan hound, Afghan": 160, + "African chameleon, Chamaeleo chamaeleon": 47, + "African crocodile, Nile crocodile, Crocodylus niloticus": 49, + "African elephant, Loxodonta africana": 386, + "African grey, African gray, Psittacus erithacus": 87, + "African hunting dog, hyena dog, Cape hunting dog, Lycaon pictus": 275, + "Airedale, Airedale terrier": 191, + "American Staffordshire terrier, Staffordshire terrier, American pit bull terrier, pit bull terrier": 180, + "American alligator, Alligator mississipiensis": 50, + "American black bear, black bear, Ursus americanus, Euarctos americanus": 295, + "American chameleon, anole, Anolis carolinensis": 40, + "American coot, marsh hen, mud hen, water hen, Fulica americana": 137, + "American egret, great white heron, Egretta albus": 132, + "American lobster, Northern lobster, Maine lobster, Homarus americanus": 122, + "Angora, Angora rabbit": 332, + "Appenzeller": 240, + "Arabian camel, dromedary, Camelus dromedarius": 354, + "Arctic fox, white fox, Alopex lagopus": 279, + "Australian terrier": 193, + "Band Aid": 419, + "Bedlington terrier": 181, + "Bernese mountain dog": 239, + "Blenheim spaniel": 156, + "Border collie": 232, + "Border terrier": 182, + "Boston bull, Boston terrier": 195, + "Bouvier des Flandres, Bouviers des Flandres": 233, + "Brabancon griffon": 262, + "Brittany spaniel": 215, + "CD player": 485, + "Cardigan, Cardigan Welsh corgi": 264, + "Chesapeake Bay retriever": 209, + "Chihuahua": 151, + "Christmas stocking": 496, + "Crock Pot": 521, + "Dandie Dinmont, Dandie Dinmont terrier": 194, + "Doberman, Doberman pinscher": 236, + "Dungeness crab, Cancer magister": 118, + "Dutch oven": 544, + "Egyptian cat": 285, + "English foxhound": 167, + "English setter": 212, + "English springer, English springer spaniel": 217, + "EntleBucher": 241, + "Eskimo dog, husky": 248, + "European fire salamander, Salamandra salamandra": 25, + "European gallinule, Porphyrio porphyrio": 136, + "French bulldog": 245, + "French horn, horn": 566, + "French loaf": 930, + "German shepherd, German shepherd dog, German police dog, alsatian": 235, + "German short-haired pointer": 210, + "Gila monster, Heloderma suspectum": 45, + "Gordon setter": 214, + "Granny Smith": 948, + "Great Dane": 246, + "Great Pyrenees": 257, + "Greater Swiss Mountain dog": 238, + "Ibizan hound, Ibizan Podenco": 173, + "Indian cobra, Naja naja": 63, + "Indian elephant, Elephas maximus": 385, + "Irish setter, red setter": 213, + "Irish terrier": 184, + "Irish water spaniel": 221, + "Irish wolfhound": 170, + "Italian greyhound": 171, + "Japanese spaniel": 152, + "Kerry blue terrier": 183, + "Komodo dragon, Komodo lizard, dragon lizard, giant lizard, Varanus komodoensis": 48, + "Labrador retriever": 208, + "Lakeland terrier": 189, + "Leonberg": 255, + "Lhasa, Lhasa apso": 204, + "Loafer": 630, + "Madagascar cat, ring-tailed lemur, Lemur catta": 383, + "Maltese dog, Maltese terrier, Maltese": 153, + "Mexican hairless": 268, + "Model T": 661, + "Newfoundland, Newfoundland dog": 256, + "Norfolk terrier": 185, + "Norwegian elkhound, elkhound": 174, + "Norwich terrier": 186, + "Old English sheepdog, bobtail": 229, + "Pekinese, Pekingese, Peke": 154, + "Pembroke, Pembroke Welsh corgi": 263, + "Persian cat": 283, + "Petri dish": 712, + "Polaroid camera, Polaroid Land camera": 732, + "Pomeranian": 259, + "Rhodesian ridgeback": 159, + "Rottweiler": 234, + "Saint Bernard, St Bernard": 247, + "Saluki, gazelle hound": 176, + "Samoyed, Samoyede": 258, + "Scotch terrier, Scottish terrier, Scottie": 199, + "Scottish deerhound, deerhound": 177, + "Sealyham terrier, Sealyham": 190, + "Shetland sheepdog, Shetland sheep dog, Shetland": 230, + "Shih-Tzu": 155, + "Siamese cat, Siamese": 284, + "Siberian husky": 250, + "Staffordshire bullterrier, Staffordshire bull terrier": 179, + "Sussex spaniel": 220, + "Tibetan mastiff": 244, + "Tibetan terrier, chrysanthemum dog": 200, + "Walker hound, Walker foxhound": 166, + "Weimaraner": 178, + "Welsh springer spaniel": 218, + "West Highland white terrier": 203, + "Windsor tie": 906, + "Yorkshire terrier": 187, + "abacus": 398, + "abaya": 399, + "academic gown, academic robe, judge's robe": 400, + "accordion, piano accordion, squeeze box": 401, + "acorn": 988, + "acorn squash": 941, + "acoustic guitar": 402, + "admiral": 321, + "affenpinscher, monkey pinscher, monkey dog": 252, + "agama": 42, + "agaric": 992, + "aircraft carrier, carrier, flattop, attack aircraft carrier": 403, + "airliner": 404, + "airship, dirigible": 405, + "albatross, mollymawk": 146, + "alligator lizard": 44, + "alp": 970, + "altar": 406, + "ambulance": 407, + "amphibian, amphibious vehicle": 408, + "analog clock": 409, + "anemone fish": 393, + "ant, emmet, pismire": 310, + "apiary, bee house": 410, + "apron": 411, + "armadillo": 363, + "artichoke, globe artichoke": 944, + "ashcan, trash can, garbage can, wastebin, ash bin, ash-bin, ashbin, dustbin, trash barrel, trash bin": 412, + "assault rifle, assault gun": 413, + "axolotl, mud puppy, Ambystoma mexicanum": 29, + "baboon": 372, + "backpack, back pack, knapsack, packsack, rucksack, haversack": 414, + "badger": 362, + "bagel, beigel": 931, + "bakery, bakeshop, bakehouse": 415, + "balance beam, beam": 416, + "bald eagle, American eagle, Haliaeetus leucocephalus": 22, + "balloon": 417, + "ballplayer, baseball player": 981, + "ballpoint, ballpoint pen, ballpen, Biro": 418, + "banana": 954, + "banded gecko": 38, + "banjo": 420, + "bannister, banister, balustrade, balusters, handrail": 421, + "barbell": 422, + "barber chair": 423, + "barbershop": 424, + "barn": 425, + "barn spider, Araneus cavaticus": 73, + "barometer": 426, + "barracouta, snoek": 389, + "barrel, cask": 427, + "barrow, garden cart, lawn cart, wheelbarrow": 428, + "baseball": 429, + "basenji": 253, + "basketball": 430, + "basset, basset hound": 161, + "bassinet": 431, + "bassoon": 432, + "bath towel": 434, + "bathing cap, swimming cap": 433, + "bathtub, bathing tub, bath, tub": 435, + "beach wagon, station wagon, wagon, estate car, beach waggon, station waggon, waggon": 436, + "beacon, lighthouse, beacon light, pharos": 437, + "beagle": 162, + "beaker": 438, + "bearskin, busby, shako": 439, + "beaver": 337, + "bee": 309, + "bee eater": 92, + "beer bottle": 440, + "beer glass": 441, + "bell cote, bell cot": 442, + "bell pepper": 945, + "bib": 443, + "bicycle-built-for-two, tandem bicycle, tandem": 444, + "bighorn, bighorn sheep, cimarron, Rocky Mountain bighorn, Rocky Mountain sheep, Ovis canadensis": 349, + "bikini, two-piece": 445, + "binder, ring-binder": 446, + "binoculars, field glasses, opera glasses": 447, + "birdhouse": 448, + "bison": 347, + "bittern": 133, + "black and gold garden spider, Argiope aurantia": 72, + "black grouse": 80, + "black stork, Ciconia nigra": 128, + "black swan, Cygnus atratus": 100, + "black widow, Latrodectus mactans": 75, + "black-and-tan coonhound": 165, + "black-footed ferret, ferret, Mustela nigripes": 359, + "bloodhound, sleuthhound": 163, + "bluetick": 164, + "boa constrictor, Constrictor constrictor": 61, + "boathouse": 449, + "bobsled, bobsleigh, bob": 450, + "bolete": 997, + "bolo tie, bolo, bola tie, bola": 451, + "bonnet, poke bonnet": 452, + "book jacket, dust cover, dust jacket, dust wrapper": 921, + "bookcase": 453, + "bookshop, bookstore, bookstall": 454, + "borzoi, Russian wolfhound": 169, + "bottlecap": 455, + "bow": 456, + "bow tie, bow-tie, bowtie": 457, + "box turtle, box tortoise": 37, + "boxer": 242, + "brain coral": 109, + "brambling, Fringilla montifringilla": 10, + "brass, memorial tablet, plaque": 458, + "brassiere, bra, bandeau": 459, + "breakwater, groin, groyne, mole, bulwark, seawall, jetty": 460, + "breastplate, aegis, egis": 461, + "briard": 226, + "broccoli": 937, + "broom": 462, + "brown bear, bruin, Ursus arctos": 294, + "bubble": 971, + "bucket, pail": 463, + "buckeye, horse chestnut, conker": 990, + "buckle": 464, + "bulbul": 16, + "bull mastiff": 243, + "bullet train, bullet": 466, + "bulletproof vest": 465, + "bullfrog, Rana catesbeiana": 30, + "burrito": 965, + "bustard": 138, + "butcher shop, meat market": 467, + "butternut squash": 942, + "cab, hack, taxi, taxicab": 468, + "cabbage butterfly": 324, + "cairn, cairn terrier": 192, + "caldron, cauldron": 469, + "can opener, tin opener": 473, + "candle, taper, wax light": 470, + "cannon": 471, + "canoe": 472, + "capuchin, ringtail, Cebus capucinus": 378, + "car mirror": 475, + "car wheel": 479, + "carbonara": 959, + "cardigan": 474, + "cardoon": 946, + "carousel, carrousel, merry-go-round, roundabout, whirligig": 476, + "carpenter's kit, tool kit": 477, + "carton": 478, + "cash machine, cash dispenser, automated teller machine, automatic teller machine, automated teller, automatic teller, ATM": 480, + "cassette": 481, + "cassette player": 482, + "castle": 483, + "catamaran": 484, + "cauliflower": 938, + "cello, violoncello": 486, + "cellular telephone, cellular phone, cellphone, cell, mobile phone": 487, + "centipede": 79, + "chain": 488, + "chain mail, ring mail, mail, chain armor, chain armour, ring armor, ring armour": 490, + "chain saw, chainsaw": 491, + "chainlink fence": 489, + "chambered nautilus, pearly nautilus, nautilus": 117, + "cheeseburger": 933, + "cheetah, chetah, Acinonyx jubatus": 293, + "chest": 492, + "chickadee": 19, + "chiffonier, commode": 493, + "chime, bell, gong": 494, + "chimpanzee, chimp, Pan troglodytes": 367, + "china cabinet, china closet": 495, + "chiton, coat-of-mail shell, sea cradle, polyplacophore": 116, + "chocolate sauce, chocolate syrup": 960, + "chow, chow chow": 260, + "church, church building": 497, + "cicada, cicala": 316, + "cinema, movie theater, movie theatre, movie house, picture palace": 498, + "cleaver, meat cleaver, chopper": 499, + "cliff dwelling": 500, + "cliff, drop, drop-off": 972, + "cloak": 501, + "clog, geta, patten, sabot": 502, + "clumber, clumber spaniel": 216, + "cock": 7, + "cocker spaniel, English cocker spaniel, cocker": 219, + "cockroach, roach": 314, + "cocktail shaker": 503, + "coffee mug": 504, + "coffeepot": 505, + "coho, cohoe, coho salmon, blue jack, silver salmon, Oncorhynchus kisutch": 391, + "coil, spiral, volute, whorl, helix": 506, + "collie": 231, + "colobus, colobus monkey": 375, + "combination lock": 507, + "comic book": 917, + "common iguana, iguana, Iguana iguana": 39, + "common newt, Triturus vulgaris": 26, + "computer keyboard, keypad": 508, + "conch": 112, + "confectionery, confectionary, candy store": 509, + "consomme": 925, + "container ship, containership, container vessel": 510, + "convertible": 511, + "coral fungus": 991, + "coral reef": 973, + "corkscrew, bottle screw": 512, + "corn": 987, + "cornet, horn, trumpet, trump": 513, + "coucal": 91, + "cougar, puma, catamount, mountain lion, painter, panther, Felis concolor": 286, + "cowboy boot": 514, + "cowboy hat, ten-gallon hat": 515, + "coyote, prairie wolf, brush wolf, Canis latrans": 272, + "cradle": 516, + "crane": 517, + "crash helmet": 518, + "crate": 519, + "crayfish, crawfish, crawdad, crawdaddy": 124, + "crib, cot": 520, + "cricket": 312, + "croquet ball": 522, + "crossword puzzle, crossword": 918, + "crutch": 523, + "cucumber, cuke": 943, + "cuirass": 524, + "cup": 968, + "curly-coated retriever": 206, + "custard apple": 956, + "daisy": 985, + "dalmatian, coach dog, carriage dog": 251, + "dam, dike, dyke": 525, + "damselfly": 320, + "desk": 526, + "desktop computer": 527, + "dhole, Cuon alpinus": 274, + "dial telephone, dial phone": 528, + "diamondback, diamondback rattlesnake, Crotalus adamanteus": 67, + "diaper, nappy, napkin": 529, + "digital clock": 530, + "digital watch": 531, + "dingo, warrigal, warragal, Canis dingo": 273, + "dining table, board": 532, + "dishrag, dishcloth": 533, + "dishwasher, dish washer, dishwashing machine": 534, + "disk brake, disc brake": 535, + "dock, dockage, docking facility": 536, + "dogsled, dog sled, dog sleigh": 537, + "dome": 538, + "doormat, welcome mat": 539, + "dough": 961, + "dowitcher": 142, + "dragonfly, darning needle, devil's darning needle, sewing needle, snake feeder, snake doctor, mosquito hawk, skeeter hawk": 319, + "drake": 97, + "drilling platform, offshore rig": 540, + "drum, membranophone, tympan": 541, + "drumstick": 542, + "dugong, Dugong dugon": 149, + "dumbbell": 543, + "dung beetle": 305, + "ear, spike, capitulum": 998, + "earthstar": 995, + "echidna, spiny anteater, anteater": 102, + "eel": 390, + "eft": 27, + "eggnog": 969, + "electric fan, blower": 545, + "electric guitar": 546, + "electric locomotive": 547, + "electric ray, crampfish, numbfish, torpedo": 5, + "entertainment center": 548, + "envelope": 549, + "espresso": 967, + "espresso maker": 550, + "face powder": 551, + "feather boa, boa": 552, + "fiddler crab": 120, + "fig": 952, + "file, file cabinet, filing cabinet": 553, + "fire engine, fire truck": 555, + "fire screen, fireguard": 556, + "fireboat": 554, + "flagpole, flagstaff": 557, + "flamingo": 130, + "flat-coated retriever": 205, + "flatworm, platyhelminth": 110, + "flute, transverse flute": 558, + "fly": 308, + "folding chair": 559, + "football helmet": 560, + "forklift": 561, + "fountain": 562, + "fountain pen": 563, + "four-poster": 564, + "fox squirrel, eastern fox squirrel, Sciurus niger": 335, + "freight car": 565, + "frilled lizard, Chlamydosaurus kingi": 43, + "frying pan, frypan, skillet": 567, + "fur coat": 568, + "gar, garfish, garpike, billfish, Lepisosteus osseus": 395, + "garbage truck, dustcart": 569, + "garden spider, Aranea diademata": 74, + "garter snake, grass snake": 57, + "gas pump, gasoline pump, petrol pump, island dispenser": 571, + "gasmask, respirator, gas helmet": 570, + "gazelle": 353, + "geyser": 974, + "giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca": 388, + "giant schnauzer": 197, + "gibbon, Hylobates lar": 368, + "go-kart": 573, + "goblet": 572, + "golden retriever": 207, + "goldfinch, Carduelis carduelis": 11, + "goldfish, Carassius auratus": 1, + "golf ball": 574, + "golfcart, golf cart": 575, + "gondola": 576, + "gong, tam-tam": 577, + "goose": 99, + "gorilla, Gorilla gorilla": 366, + "gown": 578, + "grand piano, grand": 579, + "grasshopper, hopper": 311, + "great grey owl, great gray owl, Strix nebulosa": 24, + "great white shark, white shark, man-eater, man-eating shark, Carcharodon carcharias": 2, + "green lizard, Lacerta viridis": 46, + "green mamba": 64, + "green snake, grass snake": 55, + "greenhouse, nursery, glasshouse": 580, + "grey fox, gray fox, Urocyon cinereoargenteus": 280, + "grey whale, gray whale, devilfish, Eschrichtius gibbosus, Eschrichtius robustus": 147, + "grille, radiator grille": 581, + "grocery store, grocery, food market, market": 582, + "groenendael": 224, + "groom, bridegroom": 982, + "ground beetle, carabid beetle": 302, + "guacamole": 924, + "guenon, guenon monkey": 370, + "guillotine": 583, + "guinea pig, Cavia cobaya": 338, + "gyromitra": 993, + "hair slide": 584, + "hair spray": 585, + "half track": 586, + "hammer": 587, + "hammerhead, hammerhead shark": 4, + "hamper": 588, + "hamster": 333, + "hand blower, blow dryer, blow drier, hair dryer, hair drier": 589, + "hand-held computer, hand-held microcomputer": 590, + "handkerchief, hankie, hanky, hankey": 591, + "hard disc, hard disk, fixed disk": 592, + "hare": 331, + "harmonica, mouth organ, harp, mouth harp": 593, + "harp": 594, + "hartebeest": 351, + "harvester, reaper": 595, + "harvestman, daddy longlegs, Phalangium opilio": 70, + "hatchet": 596, + "hay": 958, + "head cabbage": 936, + "hen": 8, + "hen-of-the-woods, hen of the woods, Polyporus frondosus, Grifola frondosa": 996, + "hermit crab": 125, + "hip, rose hip, rosehip": 989, + "hippopotamus, hippo, river horse, Hippopotamus amphibius": 344, + "hog, pig, grunter, squealer, Sus scrofa": 341, + "hognose snake, puff adder, sand viper": 54, + "holster": 597, + "home theater, home theatre": 598, + "honeycomb": 599, + "hook, claw": 600, + "hoopskirt, crinoline": 601, + "horizontal bar, high bar": 602, + "hornbill": 93, + "horned viper, cerastes, sand viper, horned asp, Cerastes cornutus": 66, + "horse cart, horse-cart": 603, + "hot pot, hotpot": 926, + "hotdog, hot dog, red hot": 934, + "hourglass": 604, + "house finch, linnet, Carpodacus mexicanus": 12, + "howler monkey, howler": 379, + "hummingbird": 94, + "hyena, hyaena": 276, + "iPod": 605, + "ibex, Capra ibex": 350, + "ice bear, polar bear, Ursus Maritimus, Thalarctos maritimus": 296, + "ice cream, icecream": 928, + "ice lolly, lolly, lollipop, popsicle": 929, + "impala, Aepyceros melampus": 352, + "indigo bunting, indigo finch, indigo bird, Passerina cyanea": 14, + "indri, indris, Indri indri, Indri brevicaudatus": 384, + "iron, smoothing iron": 606, + "isopod": 126, + "jacamar": 95, + "jack-o'-lantern": 607, + "jackfruit, jak, jack": 955, + "jaguar, panther, Panthera onca, Felis onca": 290, + "jay": 17, + "jean, blue jean, denim": 608, + "jeep, landrover": 609, + "jellyfish": 107, + "jersey, T-shirt, tee shirt": 610, + "jigsaw puzzle": 611, + "jinrikisha, ricksha, rickshaw": 612, + "joystick": 613, + "junco, snowbird": 13, + "keeshond": 261, + "kelpie": 227, + "killer whale, killer, orca, grampus, sea wolf, Orcinus orca": 148, + "kimono": 614, + "king crab, Alaska crab, Alaskan king crab, Alaska king crab, Paralithodes camtschatica": 121, + "king penguin, Aptenodytes patagonica": 145, + "king snake, kingsnake": 56, + "kit fox, Vulpes macrotis": 278, + "kite": 21, + "knee pad": 615, + "knot": 616, + "koala, koala bear, kangaroo bear, native bear, Phascolarctos cinereus": 105, + "komondor": 228, + "kuvasz": 222, + "lab coat, laboratory coat": 617, + "lacewing, lacewing fly": 318, + "ladle": 618, + "ladybug, ladybeetle, lady beetle, ladybird, ladybird beetle": 301, + "lakeside, lakeshore": 975, + "lampshade, lamp shade": 619, + "langur": 374, + "laptop, laptop computer": 620, + "lawn mower, mower": 621, + "leaf beetle, chrysomelid": 304, + "leafhopper": 317, + "leatherback turtle, leatherback, leathery turtle, Dermochelys coriacea": 34, + "lemon": 951, + "lens cap, lens cover": 622, + "leopard, Panthera pardus": 288, + "lesser panda, red panda, panda, bear cat, cat bear, Ailurus fulgens": 387, + "letter opener, paper knife, paperknife": 623, + "library": 624, + "lifeboat": 625, + "lighter, light, igniter, ignitor": 626, + "limousine, limo": 627, + "limpkin, Aramus pictus": 135, + "liner, ocean liner": 628, + "lion, king of beasts, Panthera leo": 291, + "lionfish": 396, + "lipstick, lip rouge": 629, + "little blue heron, Egretta caerulea": 131, + "llama": 355, + "loggerhead, loggerhead turtle, Caretta caretta": 33, + "long-horned beetle, longicorn, longicorn beetle": 303, + "lorikeet": 90, + "lotion": 631, + "loudspeaker, speaker, speaker unit, loudspeaker system, speaker system": 632, + "loupe, jeweler's loupe": 633, + "lumbermill, sawmill": 634, + "lycaenid, lycaenid butterfly": 326, + "lynx, catamount": 287, + "macaque": 373, + "macaw": 88, + "magnetic compass": 635, + "magpie": 18, + "mailbag, postbag": 636, + "mailbox, letter box": 637, + "maillot": 638, + "maillot, tank suit": 639, + "malamute, malemute, Alaskan malamute": 249, + "malinois": 225, + "manhole cover": 640, + "mantis, mantid": 315, + "maraca": 641, + "marimba, xylophone": 642, + "marmoset": 377, + "marmot": 336, + "mashed potato": 935, + "mask": 643, + "matchstick": 644, + "maypole": 645, + "maze, labyrinth": 646, + "measuring cup": 647, + "meat loaf, meatloaf": 962, + "medicine chest, medicine cabinet": 648, + "meerkat, mierkat": 299, + "megalith, megalithic structure": 649, + "menu": 922, + "microphone, mike": 650, + "microwave, microwave oven": 651, + "military uniform": 652, + "milk can": 653, + "miniature pinscher": 237, + "miniature poodle": 266, + "miniature schnauzer": 196, + "minibus": 654, + "miniskirt, mini": 655, + "minivan": 656, + "mink": 357, + "missile": 657, + "mitten": 658, + "mixing bowl": 659, + "mobile home, manufactured home": 660, + "modem": 662, + "monarch, monarch butterfly, milkweed butterfly, Danaus plexippus": 323, + "monastery": 663, + "mongoose": 298, + "monitor": 664, + "moped": 665, + "mortar": 666, + "mortarboard": 667, + "mosque": 668, + "mosquito net": 669, + "motor scooter, scooter": 670, + "mountain bike, all-terrain bike, off-roader": 671, + "mountain tent": 672, + "mouse, computer mouse": 673, + "mousetrap": 674, + "moving van": 675, + "mud turtle": 35, + "mushroom": 947, + "muzzle": 676, + "nail": 677, + "neck brace": 678, + "necklace": 679, + "nematode, nematode worm, roundworm": 111, + "night snake, Hypsiglena torquata": 60, + "nipple": 680, + "notebook, notebook computer": 681, + "obelisk": 682, + "oboe, hautboy, hautbois": 683, + "ocarina, sweet potato": 684, + "odometer, hodometer, mileometer, milometer": 685, + "oil filter": 686, + "orange": 950, + "orangutan, orang, orangutang, Pongo pygmaeus": 365, + "organ, pipe organ": 687, + "oscilloscope, scope, cathode-ray oscilloscope, CRO": 688, + "ostrich, Struthio camelus": 9, + "otter": 360, + "otterhound, otter hound": 175, + "overskirt": 689, + "ox": 345, + "oxcart": 690, + "oxygen mask": 691, + "oystercatcher, oyster catcher": 143, + "packet": 692, + "paddle, boat paddle": 693, + "paddlewheel, paddle wheel": 694, + "padlock": 695, + "paintbrush": 696, + "pajama, pyjama, pj's, jammies": 697, + "palace": 698, + "panpipe, pandean pipe, syrinx": 699, + "paper towel": 700, + "papillon": 157, + "parachute, chute": 701, + "parallel bars, bars": 702, + "park bench": 703, + "parking meter": 704, + "partridge": 86, + "passenger car, coach, carriage": 705, + "patas, hussar monkey, Erythrocebus patas": 371, + "patio, terrace": 706, + "pay-phone, pay-station": 707, + "peacock": 84, + "pedestal, plinth, footstall": 708, + "pelican": 144, + "pencil box, pencil case": 709, + "pencil sharpener": 710, + "perfume, essence": 711, + "photocopier": 713, + "pick, plectrum, plectron": 714, + "pickelhaube": 715, + "picket fence, paling": 716, + "pickup, pickup truck": 717, + "pier": 718, + "piggy bank, penny bank": 719, + "pill bottle": 720, + "pillow": 721, + "pineapple, ananas": 953, + "ping-pong ball": 722, + "pinwheel": 723, + "pirate, pirate ship": 724, + "pitcher, ewer": 725, + "pizza, pizza pie": 963, + "plane, carpenter's plane, woodworking plane": 726, + "planetarium": 727, + "plastic bag": 728, + "plate": 923, + "plate rack": 729, + "platypus, duckbill, duckbilled platypus, duck-billed platypus, Ornithorhynchus anatinus": 103, + "plow, plough": 730, + "plunger, plumber's helper": 731, + "pole": 733, + "polecat, fitch, foulmart, foumart, Mustela putorius": 358, + "police van, police wagon, paddy wagon, patrol wagon, wagon, black Maria": 734, + "pomegranate": 957, + "poncho": 735, + "pool table, billiard table, snooker table": 736, + "pop bottle, soda bottle": 737, + "porcupine, hedgehog": 334, + "pot, flowerpot": 738, + "potpie": 964, + "potter's wheel": 739, + "power drill": 740, + "prairie chicken, prairie grouse, prairie fowl": 83, + "prayer rug, prayer mat": 741, + "pretzel": 932, + "printer": 742, + "prison, prison house": 743, + "proboscis monkey, Nasalis larvatus": 376, + "projectile, missile": 744, + "projector": 745, + "promontory, headland, head, foreland": 976, + "ptarmigan": 81, + "puck, hockey puck": 746, + "puffer, pufferfish, blowfish, globefish": 397, + "pug, pug-dog": 254, + "punching bag, punch bag, punching ball, punchball": 747, + "purse": 748, + "quail": 85, + "quill, quill pen": 749, + "quilt, comforter, comfort, puff": 750, + "racer, race car, racing car": 751, + "racket, racquet": 752, + "radiator": 753, + "radio telescope, radio reflector": 755, + "radio, wireless": 754, + "rain barrel": 756, + "ram, tup": 348, + "rapeseed": 984, + "recreational vehicle, RV, R.V.": 757, + "red fox, Vulpes vulpes": 277, + "red wine": 966, + "red wolf, maned wolf, Canis rufus, Canis niger": 271, + "red-backed sandpiper, dunlin, Erolia alpina": 140, + "red-breasted merganser, Mergus serrator": 98, + "redbone": 168, + "redshank, Tringa totanus": 141, + "reel": 758, + "reflex camera": 759, + "refrigerator, icebox": 760, + "remote control, remote": 761, + "restaurant, eating house, eating place, eatery": 762, + "revolver, six-gun, six-shooter": 763, + "rhinoceros beetle": 306, + "rifle": 764, + "ringlet, ringlet butterfly": 322, + "ringneck snake, ring-necked snake, ring snake": 53, + "robin, American robin, Turdus migratorius": 15, + "rock beauty, Holocanthus tricolor": 392, + "rock crab, Cancer irroratus": 119, + "rock python, rock snake, Python sebae": 62, + "rocking chair, rocker": 765, + "rotisserie": 766, + "rubber eraser, rubber, pencil eraser": 767, + "ruddy turnstone, Arenaria interpres": 139, + "ruffed grouse, partridge, Bonasa umbellus": 82, + "rugby ball": 768, + "rule, ruler": 769, + "running shoe": 770, + "safe": 771, + "safety pin": 772, + "saltshaker, salt shaker": 773, + "sandal": 774, + "sandbar, sand bar": 977, + "sarong": 775, + "sax, saxophone": 776, + "scabbard": 777, + "scale, weighing machine": 778, + "schipperke": 223, + "school bus": 779, + "schooner": 780, + "scoreboard": 781, + "scorpion": 71, + "screen, CRT screen": 782, + "screw": 783, + "screwdriver": 784, + "scuba diver": 983, + "sea anemone, anemone": 108, + "sea cucumber, holothurian": 329, + "sea lion": 150, + "sea slug, nudibranch": 115, + "sea snake": 65, + "sea urchin": 328, + "seashore, coast, seacoast, sea-coast": 978, + "seat belt, seatbelt": 785, + "sewing machine": 786, + "shield, buckler": 787, + "shoe shop, shoe-shop, shoe store": 788, + "shoji": 789, + "shopping basket": 790, + "shopping cart": 791, + "shovel": 792, + "shower cap": 793, + "shower curtain": 794, + "siamang, Hylobates syndactylus, Symphalangus syndactylus": 369, + "sidewinder, horned rattlesnake, Crotalus cerastes": 68, + "silky terrier, Sydney silky": 201, + "ski": 795, + "ski mask": 796, + "skunk, polecat, wood pussy": 361, + "sleeping bag": 797, + "slide rule, slipstick": 798, + "sliding door": 799, + "slot, one-armed bandit": 800, + "sloth bear, Melursus ursinus, Ursus ursinus": 297, + "slug": 114, + "snail": 113, + "snorkel": 801, + "snow leopard, ounce, Panthera uncia": 289, + "snowmobile": 802, + "snowplow, snowplough": 803, + "soap dispenser": 804, + "soccer ball": 805, + "sock": 806, + "soft-coated wheaten terrier": 202, + "solar dish, solar collector, solar furnace": 807, + "sombrero": 808, + "sorrel": 339, + "soup bowl": 809, + "space bar": 810, + "space heater": 811, + "space shuttle": 812, + "spaghetti squash": 940, + "spatula": 813, + "speedboat": 814, + "spider monkey, Ateles geoffroyi": 381, + "spider web, spider's web": 815, + "spindle": 816, + "spiny lobster, langouste, rock lobster, crawfish, crayfish, sea crawfish": 123, + "spoonbill": 129, + "sports car, sport car": 817, + "spotlight, spot": 818, + "spotted salamander, Ambystoma maculatum": 28, + "squirrel monkey, Saimiri sciureus": 382, + "stage": 819, + "standard poodle": 267, + "standard schnauzer": 198, + "starfish, sea star": 327, + "steam locomotive": 820, + "steel arch bridge": 821, + "steel drum": 822, + "stethoscope": 823, + "stingray": 6, + "stinkhorn, carrion fungus": 994, + "stole": 824, + "stone wall": 825, + "stopwatch, stop watch": 826, + "stove": 827, + "strainer": 828, + "strawberry": 949, + "street sign": 919, + "streetcar, tram, tramcar, trolley, trolley car": 829, + "stretcher": 830, + "studio couch, day bed": 831, + "stupa, tope": 832, + "sturgeon": 394, + "submarine, pigboat, sub, U-boat": 833, + "suit, suit of clothes": 834, + "sulphur butterfly, sulfur butterfly": 325, + "sulphur-crested cockatoo, Kakatoe galerita, Cacatua galerita": 89, + "sundial": 835, + "sunglass": 836, + "sunglasses, dark glasses, shades": 837, + "sunscreen, sunblock, sun blocker": 838, + "suspension bridge": 839, + "swab, swob, mop": 840, + "sweatshirt": 841, + "swimming trunks, bathing trunks": 842, + "swing": 843, + "switch, electric switch, electrical switch": 844, + "syringe": 845, + "tabby, tabby cat": 281, + "table lamp": 846, + "tailed frog, bell toad, ribbed toad, tailed toad, Ascaphus trui": 32, + "tank, army tank, armored combat vehicle, armoured combat vehicle": 847, + "tape player": 848, + "tarantula": 76, + "teapot": 849, + "teddy, teddy bear": 850, + "television, television system": 851, + "tench, Tinca tinca": 0, + "tennis ball": 852, + "terrapin": 36, + "thatch, thatched roof": 853, + "theater curtain, theatre curtain": 854, + "thimble": 855, + "three-toed sloth, ai, Bradypus tridactylus": 364, + "thresher, thrasher, threshing machine": 856, + "throne": 857, + "thunder snake, worm snake, Carphophis amoenus": 52, + "tick": 78, + "tiger beetle": 300, + "tiger cat": 282, + "tiger shark, Galeocerdo cuvieri": 3, + "tiger, Panthera tigris": 292, + "tile roof": 858, + "timber wolf, grey wolf, gray wolf, Canis lupus": 269, + "titi, titi monkey": 380, + "toaster": 859, + "tobacco shop, tobacconist shop, tobacconist": 860, + "toilet seat": 861, + "toilet tissue, toilet paper, bathroom tissue": 999, + "torch": 862, + "totem pole": 863, + "toucan": 96, + "tow truck, tow car, wrecker": 864, + "toy poodle": 265, + "toy terrier": 158, + "toyshop": 865, + "tractor": 866, + "traffic light, traffic signal, stoplight": 920, + "trailer truck, tractor trailer, trucking rig, rig, articulated lorry, semi": 867, + "tray": 868, + "tree frog, tree-frog": 31, + "trench coat": 869, + "triceratops": 51, + "tricycle, trike, velocipede": 870, + "trifle": 927, + "trilobite": 69, + "trimaran": 871, + "tripod": 872, + "triumphal arch": 873, + "trolleybus, trolley coach, trackless trolley": 874, + "trombone": 875, + "tub, vat": 876, + "turnstile": 877, + "tusker": 101, + "typewriter keyboard": 878, + "umbrella": 879, + "unicycle, monocycle": 880, + "upright, upright piano": 881, + "vacuum, vacuum cleaner": 882, + "valley, vale": 979, + "vase": 883, + "vault": 884, + "velvet": 885, + "vending machine": 886, + "vestment": 887, + "viaduct": 888, + "vine snake": 59, + "violin, fiddle": 889, + "vizsla, Hungarian pointer": 211, + "volcano": 980, + "volleyball": 890, + "vulture": 23, + "waffle iron": 891, + "walking stick, walkingstick, stick insect": 313, + "wall clock": 892, + "wallaby, brush kangaroo": 104, + "wallet, billfold, notecase, pocketbook": 893, + "wardrobe, closet, press": 894, + "warplane, military plane": 895, + "warthog": 343, + "washbasin, handbasin, washbowl, lavabo, wash-hand basin": 896, + "washer, automatic washer, washing machine": 897, + "water bottle": 898, + "water buffalo, water ox, Asiatic buffalo, Bubalus bubalis": 346, + "water jug": 899, + "water ouzel, dipper": 20, + "water snake": 58, + "water tower": 900, + "weasel": 356, + "web site, website, internet site, site": 916, + "weevil": 307, + "whippet": 172, + "whiptail, whiptail lizard": 41, + "whiskey jug": 901, + "whistle": 902, + "white stork, Ciconia ciconia": 127, + "white wolf, Arctic wolf, Canis lupus tundrarum": 270, + "wig": 903, + "wild boar, boar, Sus scrofa": 342, + "window screen": 904, + "window shade": 905, + "wine bottle": 907, + "wing": 908, + "wire-haired fox terrier": 188, + "wok": 909, + "wolf spider, hunting spider": 77, + "wombat": 106, + "wood rabbit, cottontail, cottontail rabbit": 330, + "wooden spoon": 910, + "wool, woolen, woollen": 911, + "worm fence, snake fence, snake-rail fence, Virginia fence": 912, + "wreck": 913, + "yawl": 914, + "yellow lady's slipper, yellow lady-slipper, Cypripedium calceolus, Cypripedium parviflorum": 986, + "yurt": 915, + "zebra": 340, + "zucchini, courgette": 939 + }, + "layer_norm_eps": 1e-12, + "mlp_ratio": [ + 4.0, + 4.0, + 4.0 + ], + "model_type": "cvt", + "num_channels": 3, + "num_heads": [ + 1, + 3, + 6 + ], + "num_stages": 3, + "padding_kv": [ + 1, + 1, + 1 + ], + "padding_q": [ + 1, + 1, + 1 + ], + "patch_padding": [ + 2, + 1, + 1 + ], + "patch_sizes": [ + 7, + 3, + 3 + ], + "patch_stride": [ + 4, + 2, + 2 + ], + "pos_embed": [ + false, + false, + false + ], + "qkv_bias": [ + true, + true, + true + ], + "qkv_projection_method": [ + "dw_bn", + "dw_bn", + "dw_bn" + ], + "stride_kv": [ + 2, + 2, + 2 + ], + "stride_q": [ + 1, + 1, + 1 + ], + "torch_dtype": "float32", + "transformers_version": "4.18.0.dev0" +} diff --git a/new_impl/cv/dnns/bert/__init__.py b/new_impl/cv/dnns/bert/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..2becbb7a0ade09e14c8f5adac2cfb06f07416430 --- /dev/null +++ b/new_impl/cv/dnns/bert/__init__.py @@ -0,0 +1,85 @@ +from transformers import BertTokenizer, BertModel, BertConfig +from utils.dl.common.model import set_module +from torch import nn +import torch +from utils.common.log import logger + + +bert_model_tag = 'bert-base-multilingual-cased' + + +class BertForSenCls(nn.Module): + def __init__(self, num_classes): + super(BertForSenCls, self).__init__() + + logger.info(f'init bert for sen cls (using {bert_model_tag})') + self.bert = BertModel.from_pretrained(bert_model_tag) + self.classifier = nn.Linear(768, num_classes) + + def forward(self, **x): + x['return_dict'] = False + + pool_output = self.bert(**x)[-1] + + return self.classifier(pool_output) + + +class BertForTokenCls(nn.Module): + def __init__(self, num_classes): + super(BertForTokenCls, self).__init__() + + logger.info(f'init bert for token cls (using {bert_model_tag})') + self.bert = BertModel.from_pretrained(bert_model_tag) + self.classifier = nn.Linear(768, num_classes) + + def forward(self, **x): + x['return_dict'] = False + + pool_output = self.bert(**x)[0] + + return self.classifier(pool_output) + + +class BertForTranslation(nn.Module): + def __init__(self): + super(BertForTranslation, self).__init__() + + self.bert = BertModel.from_pretrained(bert_model_tag) + + vocab_size = BertConfig.from_pretrained(bert_model_tag).vocab_size + self.decoder = nn.Linear(768, vocab_size) + + logger.info(f'init bert for sen cls (using {bert_model_tag}), vocab size {vocab_size}') + + # https://github.com/huggingface/transformers/blob/66954ea25e342fd451c26ec1c295da0b8692086b/src/transformers/models/bert_generation/modeling_bert_generation.py#L594 + self.decoder.weight.data.normal_(mean=0.0, std=0.02) + + def forward(self, **x): + x['return_dict'] = False + + seq_output = self.bert(**x)[0] + + return self.decoder(seq_output) + + +def bert_base_sen_cls(num_classes): + return BertForSenCls(num_classes) + + +def bert_base_token_cls(num_classes): + return BertForTokenCls(num_classes) + + +def bert_base_translation(no_bert_pooler=False): + # return BertForTranslation() + from transformers import BertTokenizer, BertModel, BertConfig, EncoderDecoderModel, BertGenerationDecoder + encoder = BertModel.from_pretrained(bert_model_tag) + model = BertGenerationDecoder.from_pretrained(bert_model_tag) + model.bert = encoder + + if no_bert_pooler: + logger.info('replace pooler with nn.Identity()') + encoder.pooler = nn.Identity() + + return model + \ No newline at end of file diff --git a/new_impl/cv/dnns/bert/vit_like_bert.py b/new_impl/cv/dnns/bert/vit_like_bert.py new file mode 100644 index 0000000000000000000000000000000000000000..3207ceea1510ecafef8036f5c7f8ae01596c8cde --- /dev/null +++ b/new_impl/cv/dnns/bert/vit_like_bert.py @@ -0,0 +1,277 @@ +from timm.models.vision_transformer import VisionTransformer, Mlp, Block, PatchEmbed, PatchDropout, named_apply, \ + init_weights_vit_timm, get_init_weights_vit, _load_weights, checkpoint_seq + +import torch +from torch import nn + +from functools import partial +from typing import Union, Tuple, Callable, Optional + +import logging +import math +from collections import OrderedDict +from functools import partial +from typing import Callable, List, Optional, Sequence, Tuple, Union + +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.utils.checkpoint +from torch.jit import Final + +from timm.data import IMAGENET_DEFAULT_MEAN, IMAGENET_DEFAULT_STD, IMAGENET_INCEPTION_MEAN, IMAGENET_INCEPTION_STD, \ + OPENAI_CLIP_MEAN, OPENAI_CLIP_STD +from timm.layers import PatchEmbed, Mlp, DropPath, trunc_normal_, lecun_normal_, resample_patch_embed, \ + resample_abs_pos_embed, RmsNorm, PatchDropout, use_fused_attn, SwiGLUPacked + + +class ViTLikeBERT(nn.Module): + + def __init__( + self, + img_size: Union[int, Tuple[int, int]] = 224, + patch_size: Union[int, Tuple[int, int]] = 16, + in_chans: int = 3, + num_classes: int = 1000, + global_pool: str = 'token', + embed_dim: int = 768, + depth: int = 12, + num_heads: int = 12, + mlp_ratio: float = 4., + qkv_bias: bool = True, + qk_norm: bool = False, + init_values: Optional[float] = None, + class_token: bool = True, + no_embed_class: bool = False, + pre_norm: bool = False, + fc_norm: Optional[bool] = None, + drop_rate: float = 0., + pos_drop_rate: float = 0., + patch_drop_rate: float = 0., + proj_drop_rate: float = 0., + attn_drop_rate: float = 0., + drop_path_rate: float = 0., + weight_init: str = '', + embed_layer: Callable = PatchEmbed, + norm_layer: Optional[Callable] = None, + act_layer: Optional[Callable] = None, + block_fn: Callable = Block, + mlp_layer: Callable = Mlp, + ): + """ + Args: + img_size: Input image size. + patch_size: Patch size. + in_chans: Number of image input channels. + num_classes: Mumber of classes for classification head. + global_pool: Type of global pooling for final sequence (default: 'token'). + embed_dim: Transformer embedding dimension. + depth: Depth of transformer. + num_heads: Number of attention heads. + mlp_ratio: Ratio of mlp hidden dim to embedding dim. + qkv_bias: Enable bias for qkv projections if True. + init_values: Layer-scale init values (layer-scale enabled if not None). + class_token: Use class token. + fc_norm: Pre head norm after pool (instead of before), if None, enabled when global_pool == 'avg'. + drop_rate: Head dropout rate. + pos_drop_rate: Position embedding dropout rate. + attn_drop_rate: Attention dropout rate. + drop_path_rate: Stochastic depth rate. + weight_init: Weight initialization scheme. + embed_layer: Patch embedding layer. + norm_layer: Normalization layer. + act_layer: MLP activation layer. + block_fn: Transformer block layer. + """ + super().__init__() + assert global_pool in ('', 'avg', 'token') + assert class_token or global_pool != 'token' + use_fc_norm = global_pool == 'avg' if fc_norm is None else fc_norm + norm_layer = norm_layer or partial(nn.LayerNorm, eps=1e-6) + act_layer = act_layer or nn.GELU + + self.num_classes = num_classes + self.global_pool = global_pool + self.num_features = self.embed_dim = embed_dim # num_features for consistency with other models + self.num_prefix_tokens = 1 if class_token else 0 + self.no_embed_class = no_embed_class + self.grad_checkpointing = False + + self.patch_embed = embed_layer( + img_size=img_size, + patch_size=patch_size, + in_chans=in_chans, + embed_dim=embed_dim, + bias=not pre_norm, # disable bias if pre-norm is used (e.g. CLIP) + ) + num_patches = self.patch_embed.num_patches + + self.cls_token = nn.Parameter(torch.zeros(1, 1, embed_dim)) if class_token else None + embed_len = num_patches if no_embed_class else num_patches + self.num_prefix_tokens + self.pos_embed = nn.Parameter(torch.randn(1, embed_len, embed_dim) * .02) + self.pos_drop = nn.Dropout(p=pos_drop_rate) + if patch_drop_rate > 0: + self.patch_drop = PatchDropout( + patch_drop_rate, + num_prefix_tokens=self.num_prefix_tokens, + ) + else: + self.patch_drop = nn.Identity() + self.norm_pre = norm_layer(embed_dim) if pre_norm else nn.Identity() + + dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth)] # stochastic depth decay rule + self.blocks = nn.Sequential(*[ + block_fn( + dim=embed_dim, + num_heads=num_heads, + mlp_ratio=mlp_ratio, + qkv_bias=qkv_bias, + qk_norm=qk_norm, + init_values=init_values, + proj_drop=proj_drop_rate, + attn_drop=attn_drop_rate, + drop_path=dpr[i], + norm_layer=norm_layer, + act_layer=act_layer, + mlp_layer=mlp_layer, + ) + for i in range(depth)]) + self.norm = norm_layer(embed_dim) if not use_fc_norm else nn.Identity() + + # Classifier Head + self.fc_norm = norm_layer(embed_dim) if use_fc_norm else nn.Identity() + self.head_drop = nn.Dropout(drop_rate) + self.head = nn.Linear(self.embed_dim, num_classes) if num_classes > 0 else nn.Identity() + + if weight_init != 'skip': + self.init_weights(weight_init) + + def init_weights(self, mode=''): + assert mode in ('jax', 'jax_nlhb', 'moco', '') + head_bias = -math.log(self.num_classes) if 'nlhb' in mode else 0. + trunc_normal_(self.pos_embed, std=.02) + if self.cls_token is not None: + nn.init.normal_(self.cls_token, std=1e-6) + named_apply(get_init_weights_vit(mode, head_bias), self) + + def _init_weights(self, m): + # this fn left here for compat with downstream users + init_weights_vit_timm(m) + + @torch.jit.ignore() + def load_pretrained(self, checkpoint_path, prefix=''): + _load_weights(self, checkpoint_path, prefix) + + @torch.jit.ignore + def no_weight_decay(self): + return {'pos_embed', 'cls_token', 'dist_token'} + + @torch.jit.ignore + def group_matcher(self, coarse=False): + return dict( + stem=r'^cls_token|pos_embed|patch_embed', # stem and embed + blocks=[(r'^blocks\.(\d+)', None), (r'^norm', (99999,))] + ) + + @torch.jit.ignore + def set_grad_checkpointing(self, enable=True): + self.grad_checkpointing = enable + + @torch.jit.ignore + def get_classifier(self): + return self.head + + def reset_classifier(self, num_classes: int, global_pool=None): + self.num_classes = num_classes + if global_pool is not None: + assert global_pool in ('', 'avg', 'token') + self.global_pool = global_pool + self.head = nn.Linear(self.embed_dim, num_classes) if num_classes > 0 else nn.Identity() + + def _pos_embed(self, x): + if self.no_embed_class: + # deit-3, updated JAX (big vision) + # position embedding does not overlap with class token, add then concat + x = x + self.pos_embed + if self.cls_token is not None: + x = torch.cat((self.cls_token.expand(x.shape[0], -1, -1), x), dim=1) + else: + # original timm, JAX, and deit vit impl + # pos_embed has entry for class token, concat then add + if self.cls_token is not None: + x = torch.cat((self.cls_token.expand(x.shape[0], -1, -1), x), dim=1) + x = x + self.pos_embed + return self.pos_drop(x) + + def _intermediate_layers( + self, + x: torch.Tensor, + n: Union[int, Sequence] = 1, + ): + outputs, num_blocks = [], len(self.blocks) + take_indices = set(range(num_blocks - n, num_blocks) if isinstance(n, int) else n) + + # forward pass + x = self.patch_embed(x) + x = self._pos_embed(x) + x = self.patch_drop(x) + x = self.norm_pre(x) + for i, blk in enumerate(self.blocks): + x = blk(x) + if i in take_indices: + outputs.append(x) + + return outputs + + def get_intermediate_layers( + self, + x: torch.Tensor, + n: Union[int, Sequence] = 1, + reshape: bool = False, + return_class_token: bool = False, + norm: bool = False, + ) -> Tuple[Union[torch.Tensor, Tuple[torch.Tensor]]]: + """ Intermediate layer accessor (NOTE: This is a WIP experiment). + Inspired by DINO / DINOv2 interface + """ + # take last n blocks if n is an int, if in is a sequence, select by matching indices + outputs = self._intermediate_layers(x, n) + if norm: + outputs = [self.norm(out) for out in outputs] + class_tokens = [out[:, 0:self.num_prefix_tokens] for out in outputs] + outputs = [out[:, self.num_prefix_tokens:] for out in outputs] + + if reshape: + grid_size = self.patch_embed.grid_size + outputs = [ + out.reshape(x.shape[0], grid_size[0], grid_size[1], -1).permute(0, 3, 1, 2).contiguous() + for out in outputs + ] + + if return_class_token: + return tuple(zip(outputs, class_tokens)) + return tuple(outputs) + + def forward_features(self, x): + x = self.patch_embed(x) + x = self._pos_embed(x) + x = self.patch_drop(x) + x = self.norm_pre(x) + if self.grad_checkpointing and not torch.jit.is_scripting(): + x = checkpoint_seq(self.blocks, x) + else: + x = self.blocks(x) + x = self.norm(x) + return x + + def forward_head(self, x, pre_logits: bool = False): + if self.global_pool: + x = x[:, self.num_prefix_tokens:].mean(dim=1) if self.global_pool == 'avg' else x[:, 0] + x = self.fc_norm(x) + x = self.head_drop(x) + return x if pre_logits else self.head(x) + + def forward(self, x): + x = self.forward_features(x) + x = self.forward_head(x) + return x diff --git a/new_impl/cv/dnns/clip/__init__.py b/new_impl/cv/dnns/clip/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..3958ba6967b8cfc4842df7675be6eb97dbd6ddbd --- /dev/null +++ b/new_impl/cv/dnns/clip/__init__.py @@ -0,0 +1,169 @@ +import timm +from timm.models._factory import load_checkpoint +import torch +import os +from typing import List, Union +from torch import nn +from torch.jit import Final +from einops import rearrange, repeat +from einops.layers.torch import Rearrange +from utils.dl.common.model import get_model_device, set_module +import torch.nn.functional as F +from utils.common.log import logger + +from transformers import CLIPProcessor, CLIPModel, CLIPVisionConfig, CLIPConfig +from dnns.clip.custom_clip import CLIPModelCanReceiveTextEmbeds + +import torch.nn.functional as F + + +class Clip_ViTB16(nn.Module): + def __init__(self, img_size): + super(Clip_ViTB16, self).__init__() + + self.processor = CLIPProcessor.from_pretrained("openai/clip-vit-base-patch16") + self.model: CLIPModel = CLIPModelCanReceiveTextEmbeds.from_pretrained("openai/clip-vit-base-patch16") + + self.img_size = img_size + + # reconstruct xx + vm_embed = self.model.vision_model.embeddings + raw_num_patches = vm_embed.num_patches + vm_embed.num_patches = (img_size // self.model.vision_model.embeddings.patch_size) ** 2 + vm_embed.num_positions = vm_embed.num_patches + 1 + vm_embed.register_buffer("position_ids", torch.arange(vm_embed.num_positions).expand((1, -1)), persistent=False) + + logger.info(f'due to changed input image size ({img_size}), num patches are updated from {raw_num_patches} to {vm_embed.num_patches}') + + self.first_inference = True + + def forward(self, images, texts: Union[List[List[str]], torch.Tensor], for_training, disable_return_loss=False, only_return_logits_per_text=False, no_grad_text=False): + + if isinstance(texts[0], str): + inputs = self.processor(text=texts, images=images, return_tensors="pt", padding=True) + else: + # input embeds instead of input ids + # however, original CLIP cannot receive Tensor as input + inputs = self.processor(images=images, return_tensors="pt") + inputs['attention_mask'] = torch.ones((texts.size(0), texts.size(1))) + inputs['input_embeds'] = texts + + if for_training and not disable_return_loss: + inputs['return_loss'] = True + else: + inputs['return_loss'] = False + + inputs['only_return_logits_per_text'] = only_return_logits_per_text + inputs['no_grad_text'] = no_grad_text + + for k, v in inputs.items(): + if isinstance(v, torch.Tensor): + inputs[k] = v.to('cuda') + + if self.first_inference: + logger.info(f'before input size: {inputs["pixel_values"].size()}') + + # print(inputs.keys()) + # print(inputs['pixel_values'].size()) + inputs['pixel_values'] = F.interpolate(inputs['pixel_values'], size=(self.img_size, self.img_size)) + # print(inputs['pixel_values'].size()) + + if self.first_inference: + logger.info(f'after input size: {inputs["pixel_values"].size()}') + self.first_inference = False + + return self.model(**inputs) + +# @torch.no_grad() +# def clip_vit_b_16(): +# # https://huggingface.co/openai/clip-vit-base-patch16 +# model = CLIPModelCanReceiveTextEmbeds.from_pretrained("openai/clip-vit-base-patch16") +# processor = CLIPProcessor.from_pretrained("openai/clip-vit-base-patch16") + +# print(model) + +# from PIL import Image +# import requests +# image = Image.open('/data/zql/datasets/Caltech-256/data/caltech256/256_ObjectCategories/003.backpack/003_0001.jpg') +# inputs = processor(text=["a photo of a dog", "a photo of a backpack", "a photo of a cat"], images=image, return_tensors="pt", padding=True) +# print(inputs) + +# from utils.dl.common.model import LayerActivation2, get_module +# input_embed_hook = LayerActivation2(get_module(model, 'text_model.embeddings')) +# outputs = model(**inputs) +# logits_per_image = outputs.logits_per_image # this is the image-text similarity score +# probs = logits_per_image.softmax(dim=1) +# print(probs) + +# input_embed = input_embed_hook.output +# input_embed_hook.remove() + +# torch.save(input_embed, os.path.join(os.path.dirname(__file__), './test_input_embed.pth')) + +# print('embed', input_embed.size()) + +# del inputs['input_ids'] +# inputs['input_embeds'] = input_embed +# outputs = model(**inputs) +# logits_per_image = outputs.logits_per_image # this is the image-text similarity score +# probs = logits_per_image.softmax(dim=1) +# print(probs) + + +@torch.no_grad() +def clip_vit_b_16(img_size): + # https://huggingface.co/openai/clip-vit-base-patch16 + return Clip_ViTB16(img_size) + + + + +if __name__ == '__main__': + model = clip_vit_b_16().cuda() + # print(model) + # exit() + + + # config = CLIPConfig.from_pretrained('openai/clip-vit-base-patch16') + # print(config) + + # # test 1: single image inference + # from PIL import Image + # import requests + # image = Image.open('/data/zql/datasets/Caltech-256/data/caltech256/256_ObjectCategories/003.backpack/003_0001.jpg') + # text = ["a photo of a dog", "a photo of a backpack", "a photo of a cat"] + + # o = model(image, text, False) + # print(o) + # print(o.logits_per_image.softmax(dim=1)) + + # o = model(image, torch.load('dnns/clip/test_input_embed.pth'), False) + # # print(o) + # print(o.logits_per_image.softmax(dim=1)) + # exit() + + # test 2: normal training using clip loss (batch) + from data import get_dataset, build_dataloader + from torchvision.transforms import Compose, ToTensor, Resize + dataset = get_dataset('Caltech256', '/data/zql/datasets/Caltech-256/data/caltech256/256_ObjectCategories/', 'train', transform=Compose([ + Resize((32, 32)), ToTensor() + ])) + dataloader = build_dataloader(dataset, 8, 0, True, None) + + from PIL import Image + import requests + images, labels = next(iter(dataloader)) + + # torch.save(images, 'dnns/clip/test_image.pth') + classes = dataset.classes + text = [f"a photo of a {classes[i]}" for i in labels] # should be ground truth + print(text) + print(images.size()) + + o = model(images, text, True) + print(o) + print(o.logits_per_image.softmax(dim=1)) + + # o = model(image, torch.load('dnns/clip/test_input_embed.pth'), False) + # # print(o) + # print(o.logits_per_image.softmax(dim=1)) \ No newline at end of file diff --git a/new_impl/cv/dnns/clip/custom_clip.py b/new_impl/cv/dnns/clip/custom_clip.py new file mode 100644 index 0000000000000000000000000000000000000000..a89edbd1bf715b4f06cda60b83c481b69d156e4c --- /dev/null +++ b/new_impl/cv/dnns/clip/custom_clip.py @@ -0,0 +1,177 @@ +from typing import Optional, Tuple, Union +import torch +from transformers.modeling_outputs import BaseModelOutputWithPooling +from transformers.models.clip.configuration_clip import CLIPConfig +from transformers.models.clip.modeling_clip import CLIPModel, CLIPTextTransformer, _make_causal_mask, _expand_mask, clip_loss, CLIPOutput + + +class CLIPTextTransformerCanReceiveEmbed(CLIPTextTransformer): + def forward(self, + input_ids: Optional[torch.Tensor] = None, + input_embeds: Optional[torch.Tensor] = None, # NOTE + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.Tensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None,) -> Union[Tuple, BaseModelOutputWithPooling]: + + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + if input_embeds is None: + if input_ids is None: + raise ValueError("You have to specify input_ids") + + input_shape = input_ids.size() + input_ids = input_ids.view(-1, input_shape[-1]) + + hidden_states = self.embeddings(input_ids=input_ids, position_ids=position_ids) + else: + hidden_states = input_embeds + input_shape = torch.Size([hidden_states.size(0), hidden_states.size(1)]) + + # CLIP's text model uses causal mask, prepare it here. + # https://github.com/openai/CLIP/blob/cfcffb90e69f37bf2ff1e988237a0fbe41f33c04/clip/model.py#L324 + # print(input_shape) + causal_attention_mask = _make_causal_mask(input_shape, hidden_states.dtype, device=hidden_states.device) + # expand attention_mask + if attention_mask is not None: + # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len] + attention_mask = _expand_mask(attention_mask, hidden_states.dtype) + + encoder_outputs = self.encoder( + inputs_embeds=hidden_states, + attention_mask=attention_mask, + causal_attention_mask=causal_attention_mask, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + last_hidden_state = encoder_outputs[0] + last_hidden_state = self.final_layer_norm(last_hidden_state) + + # text_embeds.shape = [batch_size, sequence_length, transformer.width] + # take features from the eot embedding (eot_token is the highest number in each sequence) + # eot embedding pos: input_ids.to(dtype=torch.int, device=last_hidden_state.device).argmax(dim=-1) + # casting to torch.int for onnx compatibility: argmax doesn't support int64 inputs with opset 14 + + if input_ids is not None: + eos_embedding_pos = input_ids.to(dtype=torch.int, device=last_hidden_state.device).argmax(dim=-1) + # print(input_ids, eos_embedding_pos) + else: + # pass + # TODO: is there any exception? + eos_embedding_pos = torch.tensor([input_embeds.size(1) - 1] * input_embeds.size(0), device=last_hidden_state.device) + + pooled_output = last_hidden_state[ + torch.arange(last_hidden_state.shape[0], device=last_hidden_state.device), + eos_embedding_pos + ] + + if not return_dict: + return (last_hidden_state, pooled_output) + encoder_outputs[1:] + + return BaseModelOutputWithPooling( + last_hidden_state=last_hidden_state, + pooler_output=pooled_output, + hidden_states=encoder_outputs.hidden_states, + attentions=encoder_outputs.attentions, + ) + + +class CLIPModelCanReceiveTextEmbeds(CLIPModel): + def __init__(self, config: CLIPConfig): + super().__init__(config) + + self.text_model = CLIPTextTransformerCanReceiveEmbed(config.text_config) + + def forward( + self, + input_ids: Optional[torch.LongTensor] = None, + input_embeds: Optional[torch.LongTensor] = None, + pixel_values: Optional[torch.FloatTensor] = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + return_loss: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + only_return_logits_per_text = False, + no_grad_text = False + ) -> Union[Tuple, CLIPOutput]: + + # Use CLIP model's config for some fields (if specified) instead of those of vision & text components. + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + vision_outputs = self.vision_model( + pixel_values=pixel_values, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + if no_grad_text: + with torch.no_grad(): + text_outputs = self.text_model( + input_ids=input_ids, + input_embeds=input_embeds, + attention_mask=attention_mask, + position_ids=position_ids, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + else: + text_outputs = self.text_model( + input_ids=input_ids, + input_embeds=input_embeds, + attention_mask=attention_mask, + position_ids=position_ids, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + image_embeds = vision_outputs[1] + image_embeds = self.visual_projection(image_embeds) + + text_embeds = text_outputs[1] + text_embeds = self.text_projection(text_embeds) + + # normalized features + image_embeds = image_embeds / image_embeds.norm(p=2, dim=-1, keepdim=True) + text_embeds = text_embeds / text_embeds.norm(p=2, dim=-1, keepdim=True) + + # cosine similarity as logits + logit_scale = self.logit_scale.exp() + logits_per_text = torch.matmul(text_embeds, image_embeds.t()) * logit_scale + logits_per_image = logits_per_text.t() + + if only_return_logits_per_text: + return logits_per_text + + loss = None + if return_loss: + loss = clip_loss(logits_per_text) + + if not return_dict: + output = (logits_per_image, logits_per_text, text_embeds, image_embeds, text_outputs, vision_outputs) + return ((loss,) + output) if loss is not None else output + + return CLIPOutput( + loss=loss, + logits_per_image=logits_per_image, + logits_per_text=logits_per_text, + text_embeds=text_embeds, + image_embeds=image_embeds, + text_model_output=text_outputs, + vision_model_output=vision_outputs, + ) \ No newline at end of file diff --git a/new_impl/cv/dnns/deeplabv3/head.py b/new_impl/cv/dnns/deeplabv3/head.py new file mode 100644 index 0000000000000000000000000000000000000000..34dbd51aa84eb971973e79b11d4dceee8016c12e --- /dev/null +++ b/new_impl/cv/dnns/deeplabv3/head.py @@ -0,0 +1,55 @@ +from torch import nn +from einops import rearrange +import torch.nn.functional as F + +from utils.dl.common.model import get_super_module + + +class DecoderLinear(nn.Module): + def __init__(self, n_cls, patch_size, d_encoder, im_size): + super(DecoderLinear, self).__init__() + + self.d_encoder = d_encoder + self.patch_size = patch_size + self.n_cls = n_cls + self.im_size = im_size + + self.head = nn.Linear(self.d_encoder, n_cls) + + def debug(self): + print(self.head, id(self), 'debug()') + + def forward(self, x): + # print('inside debug') + # self.debug() + # print(x.size()) + #x = x[:, 1:] # remove cls token + # print(x.size()) + + H, W = self.im_size + GS = H // self.patch_size + # print(H, W, GS, self.patch_size) + # print('head', self.head.weight.size(), x.size()) + # print(self.head, 'debug()') + x = self.head(x) + # print(x.size()) + + # (b, HW//ps**2, ps_c) + x = rearrange(x, "b (h w) c -> b c h w", h=GS) + + # print(x.size()) + + masks = x + masks = F.upsample(masks, size=(H, W), mode="bilinear") + + # print(masks.size()) + + return masks + + +def modify_forward_head(): + from types import MethodType + from timm.models.vision_transformer import VisionTransformer + def forward_head(self, x, pre_logits: bool = False): + return self.head(x) + VisionTransformer.forward_head = MethodType(forward_head, VisionTransformer) diff --git a/new_impl/cv/dnns/deeplabv3/init_from_vit.py b/new_impl/cv/dnns/deeplabv3/init_from_vit.py new file mode 100644 index 0000000000000000000000000000000000000000..e06651ddb52407df88a8859c93add2a1d488be70 --- /dev/null +++ b/new_impl/cv/dnns/deeplabv3/init_from_vit.py @@ -0,0 +1,15 @@ +from typing import Callable, Optional, Tuple, Union +from timm.layers import Mlp, PatchEmbed +from timm.models.vision_transformer import Block, VisionTransformer +from .head import DecoderLinear + + +class ViTForSeg(VisionTransformer): + def __init__(self, img_size: int | Tuple[int, int] = 224, patch_size: int | Tuple[int, int] = 16, in_chans: int = 3, num_classes: int = 1000, global_pool: str = 'token', embed_dim: int = 768, depth: int = 12, num_heads: int = 12, mlp_ratio: float = 4, qkv_bias: bool = True, qk_norm: bool = False, init_values: float | None = None, class_token: bool = True, no_embed_class: bool = False, pre_norm: bool = False, fc_norm: bool | None = None, drop_rate: float = 0, pos_drop_rate: float = 0, patch_drop_rate: float = 0, proj_drop_rate: float = 0, attn_drop_rate: float = 0, drop_path_rate: float = 0, weight_init: str = '', embed_layer: Callable[..., Any] = ..., norm_layer: Callable[..., Any] | None = None, act_layer: Callable[..., Any] | None = None, block_fn: Callable[..., Any] = ..., mlp_layer: Callable[..., Any] = ...): + super().__init__(img_size, patch_size, in_chans, num_classes, global_pool, embed_dim, depth, num_heads, mlp_ratio, qkv_bias, qk_norm, init_values, class_token, no_embed_class, pre_norm, fc_norm, drop_rate, pos_drop_rate, patch_drop_rate, proj_drop_rate, attn_drop_rate, drop_path_rate, weight_init, embed_layer, norm_layer, act_layer, block_fn, mlp_layer) + self.head = DecoderLinear(num) + def forward_head(self, x, pre_logits: bool = False): + return self.head(x) + + def init_from_vit(self, vit): + self.load_state_dict(vit.state_dict(), strict=False) diff --git a/new_impl/cv/dnns/vilt/__init__.py b/new_impl/cv/dnns/vilt/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..f7a869c12d554d340666c3dd17a94397d2577333 --- /dev/null +++ b/new_impl/cv/dnns/vilt/__init__.py @@ -0,0 +1,63 @@ +import timm +from timm.models._factory import load_checkpoint +import torch +import os +from typing import List, Union +from torch import nn +from torch.jit import Final +from einops import rearrange, repeat +from einops.layers.torch import Rearrange +from utils.dl.common.model import get_model_device, set_module +import torch.nn.functional as F +from utils.common.log import logger + +from transformers import ViltModel, ViltForQuestionAnswering +import torch.nn.functional as F + + + +def vilt_b_32(num_classes): + """ + Vilt for VQA + + settings based on the dataset VQAv2 (3129 classes): + + 1. use half of classes for LoRA adaptation + 2. use this half of classes for DA evaluation (using corruptions for generating domain shifts), + and use another half of classes for CL evaluation. + """ + + model = ViltForQuestionAnswering.from_pretrained('dandelin/vilt-b32-mlm-itm') + + linear = model.classifier[3] + new_linear = nn.Linear(linear.in_features, num_classes, bias=True) + set_module(model, 'classifier.3', new_linear) + + return model + + +if __name__ == '__main__': + model = vilt_b_32(1565) + + print(model) + + from transformers import ViltProcessor, ViltModel + from PIL import Image + import requests + + # prepare image and text + url = "http://images.cocodataset.org/val2017/000000039769.jpg" + image = Image.open(requests.get(url, stream=True).raw) + text = "hello world" + + processor = ViltProcessor.from_pretrained("dandelin/vilt-b32-mlm") + model = ViltModel.from_pretrained("dandelin/vilt-b32-mlm-itm") + + inputs = processor(image, text, return_tensors="pt") + + print(inputs) + + outputs = model(**inputs) + last_hidden_states = outputs.last_hidden_state + + print(last_hidden_states.shape) \ No newline at end of file diff --git a/new_impl/cv/dnns/vit/__init__.py b/new_impl/cv/dnns/vit/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..86190fb2059d6e65eed064d09c87aed0ddf1e31a --- /dev/null +++ b/new_impl/cv/dnns/vit/__init__.py @@ -0,0 +1,349 @@ +import timm +from timm.models._factory import load_checkpoint +import torch +import os +from torch import nn +from torch.jit import Final +from einops import rearrange, repeat +from einops.layers.torch import Rearrange +from utils.dl.common.model import get_model_device, set_module +import torch.nn.functional as F +from utils.common.log import logger + + +# class SoftmaxIgnoringZero(nn.Module): +# def __init__(self): +# super(SoftmaxIgnoringZero, self).__init__() + +# def forward(self, x: torch.Tensor): +# # non_zero_x_indexes = x.nonzero(as_tuple=True)[0] +# # non_zero_x = x[non_zero_x_indexes] +# # non_zero_x_softmax = F.softmax(non_zero_x, self.dim, _stacklevel=5) +# # res = torch.zeros_like(x) + +# # original: e^i / \sum_i e^i +# # ignoring zero: e^i +# # print(x) + +# non_zero_mask = x != 0 + +# if non_zero_mask.sum() == x.numel(): +# return F.softmax(x, -1) + +# t = non_zero_mask.sum(-1) +# assert t.view(-1).unique().size(0) == 1, f'{t.view(-1).unique()}, {x.size()}' # all vectors in the softmaxed dim has the same number of 0 +# # assert t.view(-1).unique().size(0) <= 2, f'{t.view(-1).unique()}, {x.size()}' # all vectors in the softmaxed dim has the same number of 0 or has no 0 +# non_zero_x = torch.masked_select(x, non_zero_mask) + +# non_zero_x = non_zero_x.view(*(list(x.size())[0: -1] + [t.view(-1)[0].item()])) + +# # print(non_zero_x) + +# non_zero_x_softmax = F.softmax(non_zero_x, -1) + +# a = x.nonzero(as_tuple=True)[-1] +# a = a.view(*non_zero_x_softmax.size()) +# x = x.scatter(x.dim() - 1, a, non_zero_x_softmax) + +# return x + + +class SoftmaxIgnoringZero(nn.Module): + def __init__(self): + super(SoftmaxIgnoringZero, self).__init__() + + def f(self, x): + # return x / (x + 1e-8) + return 1. + + def forward(self, x: torch.Tensor): + res = F.softmax(x, -1) + return res * self.f(x) + + +class PrunableAttention(nn.Module): + """ + https://github.com/lucidrains/vit-pytorch + """ + def __init__(self, dim, heads = 8, dim_head = 64, dropout = 0., qkv_bias = False): + super().__init__() + self.inner_dim = inner_dim = dim_head * heads + project_out = not (heads == 1 and dim_head == dim) + + self.num_heads = heads + self.scale = dim_head ** -0.5 + + self.attend = nn.Softmax(dim = -1) + self.dropout = nn.Dropout(dropout) + + self.qkv = nn.Linear(dim, inner_dim * 3, bias = qkv_bias) + + # self.proj = nn.Sequential( + # nn.Linear(inner_dim, dim), + # nn.Dropout(dropout) + # ) if project_out else nn.Identity() + + self.proj = nn.Linear(inner_dim, dim) if project_out else nn.Identity() + self.proj_dropout = nn.Dropout(dropout) + + def forward(self, x): + # qkv = self.qkv(x).chunk(3, dim = -1) + raw_qkv = self.qkv(x) + + self.inner_dim = (raw_qkv.size(-1) - self.proj.in_features) // 2 + qkv = raw_qkv[:, :, 0: self.inner_dim], raw_qkv[:, :, self.inner_dim: self.inner_dim * 2], raw_qkv[:, :, self.inner_dim * 2:] + + # print('v', qkv[0].size(), qkv[0].sum((0, 1))[0: 10], qkv[0].sum((0, 1)).nonzero(as_tuple=True)[0].size()) + + # raw_v = qkv[2] + # print('after_fbs_q, after_fbs_k', qkv[0].sum((0, 1))[0: 10], qkv[0].sum((0, 1)).nonzero(as_tuple=True)[0].size(), + # qkv[1].sum((0, 1))[0: 10], qkv[1].sum((0, 1)).nonzero(as_tuple=True)[0].size(),) + # print('after_fbs_v', raw_v.size(), raw_v.sum((0, 1))[0: 10], raw_v.sum((0, 1)).nonzero(as_tuple=True)[0].size()) + # print('q, before rearrage', qkv[0].size()) + q, k, v = qkv + # print('raw qkv size', q.size(), k.size(), v.size()) + # exit() + q, k, v = map(lambda t: rearrange(t, 'b n (h d) -> b h n d', h = self.num_heads), qkv) + # print('raw qkv size', q.size(), k.size(), v.size()) + + dots = torch.matmul(q, k.transpose(-1, -2)) * self.scale + + # print('q, k, dots, after rearrage', q.size(), k.transpose(-1, -2).size(), dots.size()) + + attn = self.attend(dots) + # attn = dots + attn = self.dropout(attn) + + # print(attn) + # print('attn', attn.size(), attn.sum((0, 1))[0: 10], attn.sum((0, 1)).nonzero(as_tuple=True)[0].size()) + # print('attn', attn.size(), attn.sum((0, 1))[0: 10], attn.sum((0, 1)).nonzero(as_tuple=True)[0].size()) + # print('v2', v.size()) + out = torch.matmul(attn, v) + # print('out1', out.size()) + # NOTE: just for trial debug + # out = v + + # print('out before rerange', out.size()) + + # print(v.size(), v) + # exit() + + out = rearrange(out, 'b h n d -> b n (h d)') + + # print('out', out.size(), out.sum((0, 1))[0: 10], out.sum((0, 1)).nonzero(as_tuple=True)[0].size()) + # exit() + + res = self.proj_dropout(self.proj(out)) + + # res = self.proj_dropout( + # F.linear(self.proj.weight.T, out.T, self.proj.bias) + # ) + # print(self.proj, self.proj_dropout) + # print('res', res.size(), res.sum((0, 1))[0: 10], res.sum((0, 1)).nonzero(as_tuple=True)[0].size()) + + return res + + +def make_attention_prunable(vit): + for block in vit.blocks: + attn = block.attn + + assert attn.attn_drop.p == attn.proj_drop.p + + prunable_attn = PrunableAttention( + dim=attn.head_dim * attn.num_heads, + heads=attn.num_heads, + dim_head=attn.head_dim, + dropout=attn.attn_drop.p, + qkv_bias=attn.qkv.bias is not None + ) + prunable_attn.qkv.weight.copy_(attn.qkv.weight) + if attn.qkv.bias is not None: + prunable_attn.qkv.bias.copy_(attn.qkv.bias) + prunable_attn.proj.weight.copy_(attn.proj.weight) + prunable_attn.proj.bias.copy_(attn.proj.bias) + + set_module(block, 'attn', prunable_attn) + + +@torch.no_grad() +def vit_l_16(pretrained=True, num_classes=None) -> nn.Module: + # https://huggingface.co/timm/vit_large_patch16_224.augreg_in21k_ft_in1k + res = timm.create_model('vit_large_patch16_224.augreg_in21k_ft_in1k', + num_classes=num_classes) + + if pretrained: + checkpoint_path = os.path.join(os.path.dirname(__file__), + 'weights/vit_large_patch16_224.augreg_in21k_ft_in1k.bin') + def filter_fn(state_dict, _): + if num_classes is None: # use fine-tuned in1k fc head + return state_dict + else: # use a new linear + del state_dict['head.weight'] + del state_dict['head.bias'] + return state_dict + + load_checkpoint(res, checkpoint_path, strict=False, filter_fn=filter_fn) + + res.eval() + input_sample = torch.rand(2, 3, 224, 224) + o1 = res(input_sample) + + make_attention_prunable(res) + res.eval() + o2 = res(input_sample) + + assert ((o1 - o2) ** 2).sum() < 1e-5 + return res + + +from timm.models.vision_transformer import VisionTransformer + +@torch.no_grad() +def vit_b_16(pretrained=True, num_classes=None) -> VisionTransformer: + # https://huggingface.co/timm/vit_base_patch16_224.augreg_in21k_ft_in1k + res = timm.create_model('vit_base_patch16_224.augreg_in21k_ft_in1k', + num_classes=num_classes) + + if pretrained: + checkpoint_path = os.path.join(os.path.dirname(__file__), + 'weights/vit_base_patch16_224.augreg_in21k_ft_in1k.bin') + def filter_fn(state_dict, _): + if num_classes is None: # use fine-tuned in1k fc head + return state_dict + else: # use a new linear + del state_dict['head.weight'] + del state_dict['head.bias'] + return state_dict + + load_checkpoint(res, checkpoint_path, strict=False, filter_fn=filter_fn) + + res.eval() + input_sample = torch.rand(2, 3, 224, 224) + o1 = res(input_sample) + + logger.info(f'make attention prunable') + make_attention_prunable(res) + # logger.info(f'make softmax prunable') + # make_softmax_prunable(res) + + res.eval() + o2 = res(input_sample) + # print(((o1 - o2) ** 2).sum()) + assert ((o1 - o2) ** 2).sum() < 1e-5 + return res + + +def make_softmax_prunable(model): + model.eval() + input_sample = torch.rand(2, 3, 224, 224).to(get_model_device(model)) + o1 = model(input_sample) + + for name, module in model.named_modules(): + if isinstance(module, nn.Softmax): + set_module(model, name, SoftmaxIgnoringZero()) + logger.info(f'make softmax {name} prunable') + + model.eval() + o2 = model(input_sample) + assert ((o1.logits - o2.logits) ** 2).sum() < 1e-5 + return model + + +if __name__ == '__main__': + model = vit_l_16() + model(torch.rand((1, 3, 224, 224))) + + + # from utils.dl.common.data_loader import ImageNetDataLoader + # _, test_loader = ImageNetDataLoader('/data/zql/datasets/imagenet2012/train', '/data/zql/datasets/imagenet2012/val', 512, 8) + + # import torch + # import tqdm + # import torch.nn.functional as F + # def get_accuracy(model, dataloader=test_loader, device='cuda'): + # acc = 0 + # sample_num = 0 + + # model.eval() + # model = model.to(device) + + # with torch.no_grad(): + # pbar = tqdm.tqdm(enumerate(dataloader), total=len(dataloader), dynamic_ncols=True, leave=False) + # for batch_index, (x, y) in pbar: + # x, y = x.to(device), y.to(device) + # output = model(x) + # pred = F.softmax(output, dim=1).argmax(dim=1) + # correct = torch.eq(pred, y).sum().item() + # acc += correct + # sample_num += len(y) + + # pbar.set_description(f'cur_batch_total: {len(y)}, cur_batch_correct: {correct}, ' + # f'cur_batch_acc: {(correct / len(y)):.4f}') + + # acc /= sample_num + # return acc + + # model = model.cuda() + # print(f'vit_l_16 im1k acc: {get_accuracy(model, test_loader, "cuda")}') + + + # softmax = SoftmaxIgnoringZero() + + # x = torch.tensor([[[1, 0, 3], [2, 2, 0]]] * 2).float() + # print(softmax(x)) + + + # model = vit_b_16(True) + # print(get_accuracy(model)) + + # for name, module in model.named_modules(): + # if isinstance(module, nn.Softmax): + # set_module(model, name, SoftmaxIgnoringZero()) + # print(f'{name}') + + # # print(model) + # print(get_accuracy(model)) + + # softmax = SoftmaxIgnoringZero() + # linear = nn.Linear(20, 10) + + # net = nn.Sequential(linear, softmax) + + # optimizer = torch.optim.SGD(net.parameters(), lr=10, momentum=0.9) + + # x = torch.rand((64, 20)) + # y_g = torch.rand((64, 10)) + + # for _ in range(100): + # y = net(x) + # # print(y) + + # loss = F.mse_loss(y, y_g) + + # optimizer.zero_grad() + # loss.backward() + + # # print(linear.weight.grad) + + # optimizer.step() + + # print(loss) + + + softmax = SoftmaxIgnoringZero() + + x = torch.tensor([ + [1, 0, 2], + [4, 0, 9], + [0, 0, 0], + [1, 1, 1] + ]).float() + print(softmax(x)) + + + x = torch.tensor([ + [1, 2], + [4, 9], + ]).float() + print(softmax(x)) \ No newline at end of file diff --git a/new_impl/cv/dnns/vit_old/__init__.py b/new_impl/cv/dnns/vit_old/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..c51e3b6ec689eb4ac489b5772326abbb06e35440 --- /dev/null +++ b/new_impl/cv/dnns/vit_old/__init__.py @@ -0,0 +1,68 @@ +# import os +# import torch +# import pickle + +# from .raw_vit import ViT + + +# def vit_b_16(pretrained_backbone=True): +# vit = ViT( +# image_size = 224, +# patch_size = 16, +# num_classes = 1000, +# dim = 768, # encoder layer/attention input/output size (Hidden Size D in the paper) +# depth = 12, +# heads = 12, # (Heads in the paper) +# dim_head = 64, # attention hidden size (seems be default, never change this) +# mlp_dim = 3072, # mlp layer hidden size (MLP size in the paper) +# dropout = 0., +# emb_dropout = 0. +# ) + +# if pretrained_backbone: +# ckpt = torch.load(os.path.join(os.path.dirname(__file__), 'weights/base_p16_224_backbone.pth')) +# vit.load_state_dict(ckpt) +# return vit + + +# def vit_l_16(pretrained_backbone=True): +# vit = ViT( +# image_size = 224, +# patch_size = 16, +# num_classes = 1000, +# dim = 1024, # encoder layer/attention input/output size (Hidden Size D in the paper) +# depth = 24, +# heads = 16, # (Heads in the paper) +# dim_head = 64, # attention hidden size (seems be default, never change this) +# mlp_dim = 4096, # mlp layer hidden size (MLP size in the paper) +# dropout = 0., +# emb_dropout = 0. +# ) + +# if pretrained_backbone: +# # https://huggingface.co/timm/vit_large_patch16_224.augreg_in21k_ft_in1k +# ckpt = torch.load(os.path.join(os.path.dirname(__file__), 'weights/pytorch_model.bin')) +# # ckpt = pickle.load(f) +# # print(ckpt) +# # exit() +# # ckpt = torch.load(os.path.join(os.path.dirname(__file__), 'weights/large_p16_224_backbone.pth')) +# vit.load_state_dict(ckpt) +# # pass +# return vit + + + +# def vit_h_16(): +# return ViT( +# image_size = 224, +# patch_size = 16, +# num_classes = 1000, +# dim = 1280, # encoder layer/attention input/output size (Hidden Size D in the paper) +# depth = 32, +# heads = 16, # (Heads in the paper) +# dim_head = 64, # attention hidden size (seems be default, never change this) +# mlp_dim = 5120, # mlp layer hidden size (MLP size in the paper) +# dropout = 0., +# emb_dropout = 0. +# ) + diff --git a/new_impl/cv/dnns/vit_old/convert_jax_vit_pth_to_torch_format.py b/new_impl/cv/dnns/vit_old/convert_jax_vit_pth_to_torch_format.py new file mode 100644 index 0000000000000000000000000000000000000000..d6161fffef2db4d4a100a3989293cab42eadccef --- /dev/null +++ b/new_impl/cv/dnns/vit_old/convert_jax_vit_pth_to_torch_format.py @@ -0,0 +1,117 @@ +""" +https://github.com/Sebastian-X/vit-pytorch-with-pretrained-weights +""" + + +import os +import torch + + +def save_weight_dict(root_path, file_name, key, shape): + file_path = os.path.join(root_path, file_name) + fo = open(file_path, "a") + string = key + '\t' + str(shape) + '\n' + fo.write(string) + fo.close() + + +if __name__ == '__main__': + # !!! whether to remain fc layers + backbone = True + + # related input and output files + weight_root_path = os.path.join(os.path.dirname(__file__), './weights') + + # NOTE: modify this + # ---- + # ori_weight_file = 'jx_vit_base_patch16_224_in21k-e5005f0a.pth' + # if backbone: + # trans_weight_file = 'base_p16_224_backbone.pth' + # else: + # trans_weight_file = 'base_p16_224.pth' + + ori_weight_file = 'jx_vit_large_p16_224-4ee7a4dc.pth' + if backbone: + trans_weight_file = 'large_p16_224_backbone.pth' + else: + trans_weight_file = 'large_p16_224.pth' + # ---- + + + + weight_txt_root_path = './models' + if backbone: + weight_txt_file = 'trans_weight_backbone.txt' + else: + weight_txt_file = 'trans_weight.txt' + + + + ori_weight_path = os.path.join(weight_root_path, ori_weight_file) + trans_weight_path = os.path.join(weight_root_path, trans_weight_file) + + ori_weight = torch.load(ori_weight_path) + trans_weight = {} + for key in ori_weight: + div_key = key.split('.') + if div_key[0] == 'norm': + trans_key = 'mlp_head.0.' + div_key[-1] + elif div_key[0] == 'head': + trans_key = 'mlp_head.1.' + div_key[-1] + elif div_key[0] == 'pos_embed': + trans_key = 'pos_embedding' + elif div_key[0] == 'patch_embed': + trans_key = 'to_patch_embedding.2.' + div_key[-1] + elif div_key[0] == 'blocks': + if div_key[2] in ['norm1', 'attn']: + sub_b = '0.' + else: + sub_b = '1.' + + prefix = 'transformer.layers.' + div_key[1] + '.' + sub_b + mod_n = '' + + if div_key[2] in ['norm1', 'norm2']: + mod_n += 'norm.' + else: + mod_n += 'fn.' + if div_key[3] == 'qkv': + mod_n += 'to_qkv.' + elif div_key[3] == 'proj': + mod_n += 'to_out.0.' + elif div_key[3] == 'fc1': + mod_n += 'net.0.' + elif div_key[3] == 'fc2': + mod_n += 'net.3.' + else: + assert ValueError('This should not happen: \'{}\''.format(key)) + + trans_key = prefix + mod_n + div_key[-1] + else: + trans_key = key + + print('{} -> {}'.format(key, trans_key)) + ''' + if trans_key == 'to_patch_embedding.1.weight': + # ipdb.set_trace() + tmp = ori_weight[key].data.contiguous().view(ori_weight[key].shape[0], -1) + trans_weight[trans_key] = tmp + else: + trans_weight[trans_key] = ori_weight[key] + ''' + if backbone: + if 'mlp_head.1' in trans_key: + continue + + + if 'to_patch_embedding' in trans_key: + trans_weight[trans_key] = ori_weight[key].reshape(ori_weight[key].size(0), -1) + else: + trans_weight[trans_key] = ori_weight[key] + + torch.save(trans_weight, trans_weight_path) + print('\nTrans_weight saved to \'{}\''.format(trans_weight_path)) + + # for key in trans_weight: + # save_weight_dict(weight_txt_root_path, weight_txt_file, key, trans_weight[key].shape) + # print('Weight log saved to \'{}\''.format(os.path.join(weight_txt_root_path, weight_txt_file))) \ No newline at end of file diff --git a/new_impl/cv/dnns/vit_old/learn_vit.py b/new_impl/cv/dnns/vit_old/learn_vit.py new file mode 100644 index 0000000000000000000000000000000000000000..d03569a36c8d722da33194d861106622cb164b18 --- /dev/null +++ b/new_impl/cv/dnns/vit_old/learn_vit.py @@ -0,0 +1,188 @@ +import torch +from torch import nn + +from einops import rearrange, repeat +from einops.layers.torch import Rearrange + +# helpers + +def pair(t): + return t if isinstance(t, tuple) else (t, t) + +# classes + +class PreNorm(nn.Module): + def __init__(self, dim, fn): + super().__init__() + self.norm = nn.LayerNorm(dim) + self.fn = fn + def forward(self, x, **kwargs): + return self.fn(self.norm(x), **kwargs) + +class FeedForward(nn.Module): + def __init__(self, dim, hidden_dim, dropout = 0.): + super().__init__() + self.net = nn.Sequential( + nn.Linear(dim, hidden_dim), + nn.GELU(), + nn.Dropout(dropout), + nn.Linear(hidden_dim, dim), + nn.Dropout(dropout) + ) + def forward(self, x): + print(f'ff input size: {x.size()}') + return self.net(x) + +class Attention(nn.Module): + def __init__(self, dim, heads = 8, dim_head = 64, dropout = 0.): + super().__init__() + + # dim_head: qkv output size of each head + self.inner_dim = inner_dim = dim_head * heads + project_out = not (heads == 1 and dim_head == dim) + + self.heads = heads + self.scale = dim_head ** -0.5 + + self.attend = nn.Softmax(dim = -1) + self.dropout = nn.Dropout(dropout) + + self.to_qkv = nn.Linear(dim, inner_dim * 3, bias = False) # to_q: (embed_dim, num_head, dim_head) + + self.to_out = nn.Sequential( + nn.Linear(inner_dim, dim), + nn.Dropout(dropout) + ) if project_out else nn.Identity() + + def forward(self, x): + print(f'attn input size: {x.size()}, to_qkv weight: {self.to_qkv}') + + print(self.inner_dim) + + qkv = self.to_qkv(x).chunk(3, dim = -1) + print([i.size() for i in qkv]) + + q, k, v = map(lambda t: rearrange(t, 'b n (h d) -> b h n d', h = self.heads), qkv) + print(q.size(), k.size(), v.size()) # (batch size 2, num_heads 12, num_patches + 1 65, d: dim_head) + + dots = torch.matmul(q, k.transpose(-1, -2)) * self.scale + + attn = self.attend(dots) + attn = self.dropout(attn) + + out = torch.matmul(attn, v) + + # Attention(Q, K, V) = softmax(QK^T / sqrt(d_k)) * V + + out = rearrange(out, 'b h n d -> b n (h d)') + + print(f'out: {out.size()}') + + res = self.to_out(out) + print(f'linear: {self.to_out}') + + print(f'result (out after linear): {res.size()}') + + return res + +class Transformer(nn.Module): + def __init__(self, dim, depth, heads, dim_head, mlp_dim, dropout = 0.): + super().__init__() + self.layers = nn.ModuleList([]) + for _ in range(depth): + self.layers.append(nn.ModuleList([ + PreNorm(dim, Attention(dim, heads = heads, dim_head = dim_head, dropout = dropout)), + PreNorm(dim, FeedForward(dim, mlp_dim, dropout = dropout)) + ])) + def forward(self, x): + for attn, ff in self.layers: + x = attn(x) + x + x = ff(x) + x + return x + +class ViT(nn.Module): + def __init__(self, *, image_size, patch_size, num_classes, dim, depth, heads, mlp_dim, pool = 'cls', channels = 3, dim_head = 64, dropout = 0., emb_dropout = 0.): + super().__init__() + image_height, image_width = pair(image_size) + patch_height, patch_width = pair(patch_size) + + assert image_height % patch_height == 0 and image_width % patch_width == 0, 'Image dimensions must be divisible by the patch size.' + + num_patches = (image_height // patch_height) * (image_width // patch_width) + self.patch_dim = patch_dim = channels * patch_height * patch_width + assert pool in {'cls', 'mean'}, 'pool type must be either cls (cls token) or mean (mean pooling)' + + self.to_patch_embedding = nn.Sequential( + Rearrange('b c (h p1) (w p2) -> b (h w) (p1 p2 c)', p1 = patch_height, p2 = patch_width), + nn.LayerNorm(patch_dim), + nn.Linear(patch_dim, dim), + nn.LayerNorm(dim), + ) + + self.pos_embedding = nn.Parameter(torch.randn(1, num_patches + 1, dim)) + self.cls_token = nn.Parameter(torch.randn(1, 1, dim)) + self.dropout = nn.Dropout(emb_dropout) + + self.transformer = Transformer(dim, depth, heads, dim_head, mlp_dim, dropout) + + self.pool = pool + self.to_latent = nn.Identity() + + self.mlp_head = nn.Sequential( + nn.LayerNorm(dim), + nn.Linear(dim, num_classes) + ) + + def forward(self, img): + print(f'raw img: {img.size()}') # (B, c, h, w) + + x = self.to_patch_embedding(img) # (B, h*w/p^2, c*p^2) -> (B, h*w/p^2, d) + + print(f'raw patch dim: {self.patch_dim}') + + print(f'patch embeddings: {x.size()}') + + b, n, _ = x.shape # b: batch size, n: # patches + + cls_tokens = repeat(self.cls_token, '1 1 d -> b 1 d', b = b) + print(f'class tokens: {cls_tokens.size()}') + + x = torch.cat((cls_tokens, x), dim=1) + + print(f'class tokens + patch embeddings: {x.size()}') + + # print(self.pos_embedding[:, :(n + 1)].size(), self.pos_embedding.size()) + + x += self.pos_embedding[:, :(n + 1)] + x = self.dropout(x) + + x = self.transformer(x) + + x = x.mean(dim = 1) if self.pool == 'mean' else x[:, 0] + + x = self.to_latent(x) + return self.mlp_head(x) + + +if __name__ == '__main__': + vit_b_32 = ViT( + image_size = 256, + patch_size = 32, + num_classes = 1000, + dim = 1024, # encoder layer/attention input/output size (Hidden Size D in the paper) + depth = 12, + heads = 12, # (Heads in the paper) + dim_head = 64, # attention hidden size (seems be default, never change this) + mlp_dim = 3072, # mlp layer hidden size (MLP size in the paper) + dropout = 0., + emb_dropout = 0. + ) + + with torch.no_grad(): + r = torch.rand((2, 3, 256, 256)) + print(vit_b_32(r).size()) + + import os + torch.save(vit_b_32, './vit_l.pt') + print(os.path.getsize('./vit_l.pt') / 1024**2) + os.remove('./vit_l.pt') \ No newline at end of file diff --git a/new_impl/cv/dnns/vit_old/raw_vit.py b/new_impl/cv/dnns/vit_old/raw_vit.py new file mode 100644 index 0000000000000000000000000000000000000000..0fc5a46b8c4d820305226c12e3827f23c2eb81a7 --- /dev/null +++ b/new_impl/cv/dnns/vit_old/raw_vit.py @@ -0,0 +1,129 @@ +import torch +from torch import nn + +from einops import rearrange, repeat +from einops.layers.torch import Rearrange + +# helpers + +def pair(t): + return t if isinstance(t, tuple) else (t, t) + +# classes + +class PreNorm(nn.Module): + def __init__(self, dim, fn): + super().__init__() + self.norm = nn.LayerNorm(dim) + self.fn = fn + def forward(self, x, **kwargs): + return self.fn(self.norm(x), **kwargs) + +class FeedForward(nn.Module): + def __init__(self, dim, hidden_dim, dropout = 0.): + super().__init__() + self.net = nn.Sequential( + nn.Linear(dim, hidden_dim), + nn.GELU(), + nn.Dropout(dropout), + nn.Linear(hidden_dim, dim), + nn.Dropout(dropout) + ) + def forward(self, x): + return self.net(x) + +class Attention(nn.Module): + def __init__(self, dim, heads = 8, dim_head = 64, dropout = 0.): + super().__init__() + inner_dim = dim_head * heads + project_out = not (heads == 1 and dim_head == dim) + + self.heads = heads + self.scale = dim_head ** -0.5 + + self.attend = nn.Softmax(dim = -1) + self.dropout = nn.Dropout(dropout) + + self.to_qkv = nn.Linear(dim, inner_dim * 3, bias = False) + + self.to_out = nn.Sequential( + nn.Linear(inner_dim, dim), + nn.Dropout(dropout) + ) if project_out else nn.Identity() + + def forward(self, x): + qkv = self.to_qkv(x).chunk(3, dim = -1) + q, k, v = map(lambda t: rearrange(t, 'b n (h d) -> b h n d', h = self.heads), qkv) + + dots = torch.matmul(q, k.transpose(-1, -2)) * self.scale + + attn = self.attend(dots) + attn = self.dropout(attn) + + out = torch.matmul(attn, v) + out = rearrange(out, 'b h n d -> b n (h d)') + return self.to_out(out) + +class Transformer(nn.Module): + def __init__(self, dim, depth, heads, dim_head, mlp_dim, dropout = 0.): + super().__init__() + self.layers = nn.ModuleList([]) + for _ in range(depth): + self.layers.append(nn.ModuleList([ + PreNorm(dim, Attention(dim, heads = heads, dim_head = dim_head, dropout = dropout)), + PreNorm(dim, FeedForward(dim, mlp_dim, dropout = dropout)) + ])) + def forward(self, x): + for attn, ff in self.layers: + x = attn(x) + x + x = ff(x) + x + return x + +class ViT(nn.Module): + def __init__(self, *, image_size, patch_size, num_classes, dim, depth, heads, mlp_dim, pool = 'cls', channels = 3, dim_head = 64, dropout = 0., emb_dropout = 0.): + super().__init__() + image_height, image_width = pair(image_size) + patch_height, patch_width = pair(patch_size) + + assert image_height % patch_height == 0 and image_width % patch_width == 0, 'Image dimensions must be divisible by the patch size.' + + num_patches = (image_height // patch_height) * (image_width // patch_width) + patch_dim = channels * patch_height * patch_width + assert pool in {'cls', 'mean'}, 'pool type must be either cls (cls token) or mean (mean pooling)' + + self.to_patch_embedding = nn.Sequential( + Rearrange('b c (h p1) (w p2) -> b (h w) (p1 p2 c)', p1 = patch_height, p2 = patch_width), + nn.LayerNorm(patch_dim), + nn.Linear(patch_dim, dim), + nn.LayerNorm(dim), + ) + + self.pos_embedding = nn.Parameter(torch.randn(1, num_patches + 1, dim)) + self.cls_token = nn.Parameter(torch.randn(1, 1, dim)) + self.dropout = nn.Dropout(emb_dropout) + + self.transformer = Transformer(dim, depth, heads, dim_head, mlp_dim, dropout) + + self.pool = pool + self.to_latent = nn.Identity() + + self.mlp_head = nn.Sequential( + nn.LayerNorm(dim), + nn.Linear(dim, num_classes) + ) + + def forward(self, img): + x = self.to_patch_embedding(img) + b, n, _ = x.shape + + cls_tokens = repeat(self.cls_token, '1 1 d -> b 1 d', b = b) + x = torch.cat((cls_tokens, x), dim=1) + x += self.pos_embedding[:, :(n + 1)] + x = self.dropout(x) + + x = self.transformer(x) + + x = x.mean(dim = 1) if self.pool == 'mean' else x[:, 0] + + x = self.to_latent(x) + return self.mlp_head(x) \ No newline at end of file diff --git a/new_impl/cv/dnns/vit_old/trans_weight.txt b/new_impl/cv/dnns/vit_old/trans_weight.txt new file mode 100644 index 0000000000000000000000000000000000000000..04e65558712974b87049e55c814df3b45d079b4d --- /dev/null +++ b/new_impl/cv/dnns/vit_old/trans_weight.txt @@ -0,0 +1,154 @@ +cls_token torch.Size([1, 1, 768]) +mlp_head.0.bias torch.Size([768]) +mlp_head.0.weight torch.Size([768]) +transformer.layers.0.0.fn.norm.bias torch.Size([768]) +transformer.layers.0.0.fn.norm.weight torch.Size([768]) +transformer.layers.0.1.fn.norm.bias torch.Size([768]) +transformer.layers.0.1.fn.norm.weight torch.Size([768]) +transformer.layers.0.1.fn.fn.net.0.bias torch.Size([3072]) +transformer.layers.0.1.fn.fn.net.0.weight torch.Size([3072, 768]) +transformer.layers.0.1.fn.fn.net.3.bias torch.Size([768]) +transformer.layers.0.1.fn.fn.net.3.weight torch.Size([768, 3072]) +transformer.layers.0.0.fn.fn.to_out.0.bias torch.Size([768]) +transformer.layers.0.0.fn.fn.to_out.0.weight torch.Size([768, 768]) +transformer.layers.0.0.fn.fn.to_qkv.bias torch.Size([2304]) +transformer.layers.0.0.fn.fn.to_qkv.weight torch.Size([2304, 768]) +transformer.layers.1.0.fn.norm.bias torch.Size([768]) +transformer.layers.1.0.fn.norm.weight torch.Size([768]) +transformer.layers.1.1.fn.norm.bias torch.Size([768]) +transformer.layers.1.1.fn.norm.weight torch.Size([768]) +transformer.layers.1.1.fn.fn.net.0.bias torch.Size([3072]) +transformer.layers.1.1.fn.fn.net.0.weight torch.Size([3072, 768]) +transformer.layers.1.1.fn.fn.net.3.bias torch.Size([768]) +transformer.layers.1.1.fn.fn.net.3.weight torch.Size([768, 3072]) +transformer.layers.1.0.fn.fn.to_out.0.bias torch.Size([768]) +transformer.layers.1.0.fn.fn.to_out.0.weight torch.Size([768, 768]) +transformer.layers.1.0.fn.fn.to_qkv.bias torch.Size([2304]) +transformer.layers.1.0.fn.fn.to_qkv.weight torch.Size([2304, 768]) +transformer.layers.10.0.fn.norm.bias torch.Size([768]) +transformer.layers.10.0.fn.norm.weight torch.Size([768]) +transformer.layers.10.1.fn.norm.bias torch.Size([768]) +transformer.layers.10.1.fn.norm.weight torch.Size([768]) +transformer.layers.10.1.fn.fn.net.0.bias torch.Size([3072]) +transformer.layers.10.1.fn.fn.net.0.weight torch.Size([3072, 768]) +transformer.layers.10.1.fn.fn.net.3.bias torch.Size([768]) +transformer.layers.10.1.fn.fn.net.3.weight torch.Size([768, 3072]) +transformer.layers.10.0.fn.fn.to_out.0.bias torch.Size([768]) +transformer.layers.10.0.fn.fn.to_out.0.weight torch.Size([768, 768]) +transformer.layers.10.0.fn.fn.to_qkv.bias torch.Size([2304]) +transformer.layers.10.0.fn.fn.to_qkv.weight torch.Size([2304, 768]) +transformer.layers.11.0.fn.norm.bias torch.Size([768]) +transformer.layers.11.0.fn.norm.weight torch.Size([768]) +transformer.layers.11.1.fn.norm.bias torch.Size([768]) +transformer.layers.11.1.fn.norm.weight torch.Size([768]) +transformer.layers.11.1.fn.fn.net.0.bias torch.Size([3072]) +transformer.layers.11.1.fn.fn.net.0.weight torch.Size([3072, 768]) +transformer.layers.11.1.fn.fn.net.3.bias torch.Size([768]) +transformer.layers.11.1.fn.fn.net.3.weight torch.Size([768, 3072]) +transformer.layers.11.0.fn.fn.to_out.0.bias torch.Size([768]) +transformer.layers.11.0.fn.fn.to_out.0.weight torch.Size([768, 768]) +transformer.layers.11.0.fn.fn.to_qkv.bias torch.Size([2304]) +transformer.layers.11.0.fn.fn.to_qkv.weight torch.Size([2304, 768]) +transformer.layers.2.0.fn.norm.bias torch.Size([768]) +transformer.layers.2.0.fn.norm.weight torch.Size([768]) +transformer.layers.2.1.fn.norm.bias torch.Size([768]) +transformer.layers.2.1.fn.norm.weight torch.Size([768]) +transformer.layers.2.1.fn.fn.net.0.bias torch.Size([3072]) +transformer.layers.2.1.fn.fn.net.0.weight torch.Size([3072, 768]) +transformer.layers.2.1.fn.fn.net.3.bias torch.Size([768]) +transformer.layers.2.1.fn.fn.net.3.weight torch.Size([768, 3072]) +transformer.layers.2.0.fn.fn.to_out.0.bias torch.Size([768]) +transformer.layers.2.0.fn.fn.to_out.0.weight torch.Size([768, 768]) +transformer.layers.2.0.fn.fn.to_qkv.bias torch.Size([2304]) +transformer.layers.2.0.fn.fn.to_qkv.weight torch.Size([2304, 768]) +transformer.layers.3.0.fn.norm.bias torch.Size([768]) +transformer.layers.3.0.fn.norm.weight torch.Size([768]) +transformer.layers.3.1.fn.norm.bias torch.Size([768]) +transformer.layers.3.1.fn.norm.weight torch.Size([768]) +transformer.layers.3.1.fn.fn.net.0.bias torch.Size([3072]) +transformer.layers.3.1.fn.fn.net.0.weight torch.Size([3072, 768]) +transformer.layers.3.1.fn.fn.net.3.bias torch.Size([768]) +transformer.layers.3.1.fn.fn.net.3.weight torch.Size([768, 3072]) +transformer.layers.3.0.fn.fn.to_out.0.bias torch.Size([768]) +transformer.layers.3.0.fn.fn.to_out.0.weight torch.Size([768, 768]) +transformer.layers.3.0.fn.fn.to_qkv.bias torch.Size([2304]) +transformer.layers.3.0.fn.fn.to_qkv.weight torch.Size([2304, 768]) +transformer.layers.4.0.fn.norm.bias torch.Size([768]) +transformer.layers.4.0.fn.norm.weight torch.Size([768]) +transformer.layers.4.1.fn.norm.bias torch.Size([768]) +transformer.layers.4.1.fn.norm.weight torch.Size([768]) +transformer.layers.4.1.fn.fn.net.0.bias torch.Size([3072]) +transformer.layers.4.1.fn.fn.net.0.weight torch.Size([3072, 768]) +transformer.layers.4.1.fn.fn.net.3.bias torch.Size([768]) +transformer.layers.4.1.fn.fn.net.3.weight torch.Size([768, 3072]) +transformer.layers.4.0.fn.fn.to_out.0.bias torch.Size([768]) +transformer.layers.4.0.fn.fn.to_out.0.weight torch.Size([768, 768]) +transformer.layers.4.0.fn.fn.to_qkv.bias torch.Size([2304]) +transformer.layers.4.0.fn.fn.to_qkv.weight torch.Size([2304, 768]) +transformer.layers.5.0.fn.norm.bias torch.Size([768]) +transformer.layers.5.0.fn.norm.weight torch.Size([768]) +transformer.layers.5.1.fn.norm.bias torch.Size([768]) +transformer.layers.5.1.fn.norm.weight torch.Size([768]) +transformer.layers.5.1.fn.fn.net.0.bias torch.Size([3072]) +transformer.layers.5.1.fn.fn.net.0.weight torch.Size([3072, 768]) +transformer.layers.5.1.fn.fn.net.3.bias torch.Size([768]) +transformer.layers.5.1.fn.fn.net.3.weight torch.Size([768, 3072]) +transformer.layers.5.0.fn.fn.to_out.0.bias torch.Size([768]) +transformer.layers.5.0.fn.fn.to_out.0.weight torch.Size([768, 768]) +transformer.layers.5.0.fn.fn.to_qkv.bias torch.Size([2304]) +transformer.layers.5.0.fn.fn.to_qkv.weight torch.Size([2304, 768]) +transformer.layers.6.0.fn.norm.bias torch.Size([768]) +transformer.layers.6.0.fn.norm.weight torch.Size([768]) +transformer.layers.6.1.fn.norm.bias torch.Size([768]) +transformer.layers.6.1.fn.norm.weight torch.Size([768]) +transformer.layers.6.1.fn.fn.net.0.bias torch.Size([3072]) +transformer.layers.6.1.fn.fn.net.0.weight torch.Size([3072, 768]) +transformer.layers.6.1.fn.fn.net.3.bias torch.Size([768]) +transformer.layers.6.1.fn.fn.net.3.weight torch.Size([768, 3072]) +transformer.layers.6.0.fn.fn.to_out.0.bias torch.Size([768]) +transformer.layers.6.0.fn.fn.to_out.0.weight torch.Size([768, 768]) +transformer.layers.6.0.fn.fn.to_qkv.bias torch.Size([2304]) +transformer.layers.6.0.fn.fn.to_qkv.weight torch.Size([2304, 768]) +transformer.layers.7.0.fn.norm.bias torch.Size([768]) +transformer.layers.7.0.fn.norm.weight torch.Size([768]) +transformer.layers.7.1.fn.norm.bias torch.Size([768]) +transformer.layers.7.1.fn.norm.weight torch.Size([768]) +transformer.layers.7.1.fn.fn.net.0.bias torch.Size([3072]) +transformer.layers.7.1.fn.fn.net.0.weight torch.Size([3072, 768]) +transformer.layers.7.1.fn.fn.net.3.bias torch.Size([768]) +transformer.layers.7.1.fn.fn.net.3.weight torch.Size([768, 3072]) +transformer.layers.7.0.fn.fn.to_out.0.bias torch.Size([768]) +transformer.layers.7.0.fn.fn.to_out.0.weight torch.Size([768, 768]) +transformer.layers.7.0.fn.fn.to_qkv.bias torch.Size([2304]) +transformer.layers.7.0.fn.fn.to_qkv.weight torch.Size([2304, 768]) +transformer.layers.8.0.fn.norm.bias torch.Size([768]) +transformer.layers.8.0.fn.norm.weight torch.Size([768]) +transformer.layers.8.1.fn.norm.bias torch.Size([768]) +transformer.layers.8.1.fn.norm.weight torch.Size([768]) +transformer.layers.8.1.fn.fn.net.0.bias torch.Size([3072]) +transformer.layers.8.1.fn.fn.net.0.weight torch.Size([3072, 768]) +transformer.layers.8.1.fn.fn.net.3.bias torch.Size([768]) +transformer.layers.8.1.fn.fn.net.3.weight torch.Size([768, 3072]) +transformer.layers.8.0.fn.fn.to_out.0.bias torch.Size([768]) +transformer.layers.8.0.fn.fn.to_out.0.weight torch.Size([768, 768]) +transformer.layers.8.0.fn.fn.to_qkv.bias torch.Size([2304]) +transformer.layers.8.0.fn.fn.to_qkv.weight torch.Size([2304, 768]) +transformer.layers.9.0.fn.norm.bias torch.Size([768]) +transformer.layers.9.0.fn.norm.weight torch.Size([768]) +transformer.layers.9.1.fn.norm.bias torch.Size([768]) +transformer.layers.9.1.fn.norm.weight torch.Size([768]) +transformer.layers.9.1.fn.fn.net.0.bias torch.Size([3072]) +transformer.layers.9.1.fn.fn.net.0.weight torch.Size([3072, 768]) +transformer.layers.9.1.fn.fn.net.3.bias torch.Size([768]) +transformer.layers.9.1.fn.fn.net.3.weight torch.Size([768, 3072]) +transformer.layers.9.0.fn.fn.to_out.0.bias torch.Size([768]) +transformer.layers.9.0.fn.fn.to_out.0.weight torch.Size([768, 768]) +transformer.layers.9.0.fn.fn.to_qkv.bias torch.Size([2304]) +transformer.layers.9.0.fn.fn.to_qkv.weight torch.Size([2304, 768]) +pos_embedding torch.Size([1, 197, 768]) +to_patch_embedding.0.bias torch.Size([768]) +to_patch_embedding.0.weight torch.Size([768, 3, 16, 16]) +mlp_head.1.bias torch.Size([21843]) +mlp_head.1.weight torch.Size([21843, 768]) +pre_logits.fc.bias torch.Size([768]) +pre_logits.fc.weight torch.Size([768, 768]) diff --git a/new_impl/cv/dnns/vit_old/vit_w_fbs.py b/new_impl/cv/dnns/vit_old/vit_w_fbs.py new file mode 100644 index 0000000000000000000000000000000000000000..3c6e717778f611fd8ab41be66571df9028de1ada --- /dev/null +++ b/new_impl/cv/dnns/vit_old/vit_w_fbs.py @@ -0,0 +1,336 @@ +from copy import deepcopy +from typing import Optional, Union +import torch +from torch import nn +from einops import rearrange, repeat +from einops.layers.torch import Rearrange + +from raw_vit import ViT, Attention, FeedForward +from utils.dl.common.model import get_model_size, set_module + + +class KTakesAll(nn.Module): + # k means sparsity (the larger k is, the smaller model is) + def __init__(self, k): + super(KTakesAll, self).__init__() + self.k = k + + def forward(self, g: torch.Tensor): + k = int(g.size(1) * self.k) + + i = (-g).topk(k, 1)[1] + t = g.scatter(1, i, 0) + + return t + + +class Abs(nn.Module): + def __init__(self): + super(Abs, self).__init__() + + def forward(self, x): + return x.abs() + + +class SqueezeLast(nn.Module): + def __init__(self): + super(SqueezeLast, self).__init__() + + def forward(self, x): + return x.squeeze(-1) + + +class Linear_WrappedWithFBS(nn.Module): + def __init__(self, linear: nn.Linear, r, k): + super(Linear_WrappedWithFBS, self).__init__() + + self.linear = linear + + # for conv: (B, C_in, H, W) -> (B, C_in) -> (B, C_out) + # for mlp in ViT: (B, #patches, D: dim of patches embedding) -> (B, D) -> (B, C_out) + self.fbs = nn.Sequential( + Rearrange('b n d -> b d n'), + Abs(), + nn.AdaptiveAvgPool1d(1), + SqueezeLast(), + nn.Linear(linear.in_features, linear.out_features // r), + nn.ReLU(), + nn.Linear(linear.out_features // r, linear.out_features), + nn.ReLU(), + KTakesAll(k) + ) + self.k = k + + self.cached_channel_attention = None # (batch_size, dim) + self.use_cached_channel_attention = False + + def forward(self, x): + if self.use_cached_channel_attention and self.cached_channel_attention is not None: + channel_attention = self.cached_channel_attention + else: + channel_attention = self.fbs(x) + self.cached_channel_attention = channel_attention + + raw_res = self.linear(x) + return channel_attention.unsqueeze(1) * raw_res + + +class ToQKV_WrappedWithFBS(nn.Module): + """ + This regards to_q/to_k/to_v as a whole (in fact it consists of multiple heads) and prunes it. + It seems different channels of different heads are pruned according to the input. + This is different from "removing some head" or "removing the same channels in each head". + """ + def __init__(self, to_qkv: nn.Linear, r, k): + super(ToQKV_WrappedWithFBS, self).__init__() + + self.to_qkv = to_qkv + self.fbses = nn.ModuleList([nn.Sequential( + Rearrange('b n d -> b d n'), + Abs(), + nn.AdaptiveAvgPool1d(1), + SqueezeLast(), + nn.Linear(to_qkv.in_features, to_qkv.out_features // 3 // r), + nn.ReLU(), + nn.Linear(to_qkv.out_features // 3 // r, to_qkv.out_features // 3), + nn.ReLU(), + KTakesAll(k) + ) for _ in range(3)]) + self.k = k + + self.cached_channel_attention = None + self.use_cached_channel_attention = False + + def forward(self, x): + if self.use_cached_channel_attention and self.cached_channel_attention is not None: + # print('use cache') + channel_attention = self.cached_channel_attention + else: + # print('dynamic') + channel_attention = torch.cat([fbs(x) for fbs in self.fbses], dim=1) + self.cached_channel_attention = channel_attention + + raw_res = self.to_qkv(x) + return channel_attention.unsqueeze(1) * raw_res + + +def boost_raw_vit_by_fbs(raw_vit: ViT, r, k): + raw_vit = deepcopy(raw_vit) + + raw_vit_model_size = get_model_size(raw_vit, True) + + # set_module(raw_vit.to_patch_embedding, '2', Linear_WrappedWithFBS(raw_vit.to_patch_embedding[2], r, k)) + + for attn, ff in raw_vit.transformer.layers: + attn = attn.fn + ff = ff.fn + + set_module(attn, 'to_qkv', ToQKV_WrappedWithFBS(attn.to_qkv, r, k)) + set_module(ff.net, '0', Linear_WrappedWithFBS(ff.net[0], r, k)) + + boosted_vit_model_size = get_model_size(raw_vit, True) + + print(f'boost_raw_vit_by_fbs() | model size from {raw_vit_model_size:.3f}MB to {boosted_vit_model_size:.3f}MB ' + f'(↑ {((boosted_vit_model_size - raw_vit_model_size) / raw_vit_model_size * 100):.2f}%)') + + return raw_vit + + +def set_boosted_vit_sparsity(boosted_vit: ViT, sparsity: float): + for attn, ff in boosted_vit.transformer.layers: + attn = attn.fn + ff = ff.fn + + q_features = attn.to_qkv.to_qkv.out_features // 3 + + if (q_features - int(q_features * sparsity)) % attn.heads != 0: + # tune sparsity to ensure #unpruned channel % num_heads == 0 + # so that the pruning seems to reduce the dim_head of each head + tuned_sparsity = 1. - int((q_features - int(q_features * sparsity)) / attn.heads) * attn.heads / q_features + print(f'set_boosted_vit_sparsity() | tune sparsity from {sparsity} to {tuned_sparsity}') + sparsity = tuned_sparsity + + attn.to_qkv.k = sparsity + for fbs in attn.to_qkv.fbses: + fbs[-1].k = sparsity + ff.net[0].k = sparsity + ff.net[0].fbs[-1].k = sparsity + + +def set_boosted_vit_inference_via_cached_channel_attentions(boosted_vit: ViT): + for attn, ff in boosted_vit.transformer.layers: + attn = attn.fn + ff = ff.fn + + assert attn.to_qkv.cached_channel_attention is not None + assert ff.net[0].cached_channel_attention is not None + + attn.to_qkv.use_cached_channel_attention = True + ff.net[0].use_cached_channel_attention = True + + +def set_boosted_vit_dynamic_inference(boosted_vit: ViT): + for attn, ff in boosted_vit.transformer.layers: + attn = attn.fn + ff = ff.fn + + attn.to_qkv.use_cached_channel_attention = False + ff.net[0].use_cached_channel_attention = False + + +class StaticFBS(nn.Module): + def __init__(self, static_channel_attention): + super(StaticFBS, self).__init__() + assert static_channel_attention.dim() == 2 and static_channel_attention.size(0) == 1 + self.static_channel_attention = nn.Parameter(static_channel_attention, requires_grad=False) # (1, dim) + + def forward(self, x): + return x * self.static_channel_attention.unsqueeze(1) + + +def extract_surrogate_vit_via_cached_channel_attn(boosted_vit: ViT): + boosted_vit = deepcopy(boosted_vit) + raw_vit_model_size = get_model_size(boosted_vit, True) + + def get_unpruned_indexes_from_channel_attn(channel_attn: torch.Tensor, k): + assert channel_attn.size(0) == 1, 'use A representative sample to generate channel attentions' + + res = channel_attn[0].nonzero(as_tuple=True)[0] # should be one-dim + return res + + for attn, ff in boosted_vit.transformer.layers: + attn = attn.fn + ff_w_norm = ff + ff = ff_w_norm.fn + + # prune to_qkv + to_qkv = attn.to_qkv + to_q_unpruned_indexes = get_unpruned_indexes_from_channel_attn( + to_qkv.cached_channel_attention[:, 0: to_qkv.cached_channel_attention.size(1) // 3], + to_qkv.k + ) + to_q_unpruned_indexes_w_offset = to_q_unpruned_indexes + to_k_unpruned_indexes = get_unpruned_indexes_from_channel_attn( + to_qkv.cached_channel_attention[:, to_qkv.cached_channel_attention.size(1) // 3: to_qkv.cached_channel_attention.size(1) // 3 * 2], + to_qkv.k + ) + to_k_unpruned_indexes_w_offset = to_k_unpruned_indexes + to_qkv.cached_channel_attention.size(1) // 3 + to_v_unpruned_indexes = get_unpruned_indexes_from_channel_attn( + to_qkv.cached_channel_attention[:, to_qkv.cached_channel_attention.size(1) // 3 * 2: ], + to_qkv.k + ) + to_v_unpruned_indexes_w_offset = to_v_unpruned_indexes + to_qkv.cached_channel_attention.size(1) // 3 * 2 + assert to_q_unpruned_indexes.size(0) == to_k_unpruned_indexes.size(0) == to_v_unpruned_indexes.size(0) + to_qkv_unpruned_indexes = torch.cat([to_q_unpruned_indexes_w_offset, to_k_unpruned_indexes_w_offset, to_v_unpruned_indexes_w_offset]) + new_to_qkv = nn.Linear(to_qkv.to_qkv.in_features, to_qkv_unpruned_indexes.size(0), to_qkv.to_qkv.bias is not None) + new_to_qkv.weight.data.copy_(to_qkv.to_qkv.weight.data[to_qkv_unpruned_indexes]) + if to_qkv.to_qkv.bias is not None: + new_to_qkv.bias.data.copy_(to_qkv.to_qkv.bias.data[to_qkv_unpruned_indexes]) + set_module(attn, 'to_qkv', nn.Sequential(new_to_qkv, StaticFBS(to_qkv.cached_channel_attention[:, to_qkv_unpruned_indexes]))) + + # prune to_out + to_out = attn.to_out[0] + new_to_out = nn.Linear(to_v_unpruned_indexes.size(0), to_out.out_features, to_out.bias is not None) + new_to_out.weight.data.copy_(to_out.weight.data[:, to_v_unpruned_indexes]) + if to_out.bias is not None: + new_to_out.bias.data.copy_(to_out.bias.data) + set_module(attn, 'to_out', new_to_out) + + ff_0 = ff.net[0] + ff_0_unpruned_indexes = get_unpruned_indexes_from_channel_attn(ff_0.cached_channel_attention, ff_0.k) + new_ff_0 = nn.Linear(ff_0.linear.in_features, ff_0_unpruned_indexes.size(0), ff_0.linear.bias is not None) + new_ff_0.weight.data.copy_(ff_0.linear.weight.data[ff_0_unpruned_indexes]) + if ff_0.linear.bias is not None: + new_ff_0.bias.data.copy_(ff_0.linear.bias.data[ff_0_unpruned_indexes]) + set_module(ff.net, '0', nn.Sequential(new_ff_0, StaticFBS(ff_0.cached_channel_attention[:, ff_0_unpruned_indexes]))) + + ff_1 = ff.net[3] + new_ff_1 = nn.Linear(ff_0_unpruned_indexes.size(0), ff_1.out_features, ff_1.bias is not None) + new_ff_1.weight.data.copy_(ff_1.weight.data[:, ff_0_unpruned_indexes]) + if ff_1.bias is not None: + new_ff_1.bias.data.copy_(ff_1.bias.data) + set_module(ff.net, '3', new_ff_1) + + pruned_vit_model_size = get_model_size(boosted_vit, True) + + print(f'extract_surrogate_vit_via_cached_channel_attn() | model size from {raw_vit_model_size:.3f}MB to {pruned_vit_model_size:.3f}MB ' + f'({(pruned_vit_model_size / raw_vit_model_size * 100):.2f}%)') + + return boosted_vit + + +if __name__ == '__main__': + from utils.dl.common.env import set_random_seed + set_random_seed(1) + + def verify(vit, sparsity=0.8): + vit.eval() + + with torch.no_grad(): + r = torch.rand((1, 3, 224, 224)) + print(vit(r).size()) + # print(vit) + + boosted_vit = boost_raw_vit_by_fbs(vit, r=32, k=sparsity) + set_boosted_vit_sparsity(boosted_vit, sparsity) + # print(boosted_vit) + with torch.no_grad(): + r = torch.rand((1, 3, 224, 224)) + print(boosted_vit(r).size()) + + # set_boosted_vit_inference_via_cached_channel_attentions(boosted_vit) + r = torch.rand((1, 3, 224, 224)) + boosted_vit.eval() + with torch.no_grad(): + o1 = boosted_vit(r) + + pruned_vit = extract_surrogate_vit_via_cached_channel_attn(boosted_vit) + pruned_vit.eval() + with torch.no_grad(): + o2 = pruned_vit(r) + print('output diff (should be tiny): ', ((o1 - o2) ** 2).sum()) + + # print(pruned_vit) + # print(pruned_vit) + + # vit_b_16 = ViT( + # image_size = 224, + # patch_size = 16, + # num_classes = 1000, + # dim = 768, # encoder layer/attention input/output size (Hidden Size D in the paper) + # depth = 12, + # heads = 12, # (Heads in the paper) + # dim_head = 64, # attention hidden size (seems be default, never change this) + # mlp_dim = 3072, # mlp layer hidden size (MLP size in the paper) + # dropout = 0., + # emb_dropout = 0. + # ) + # verify(vit_b_16) + + vit_l_16 = ViT( + image_size = 224, + patch_size = 16, + num_classes = 1000, + dim = 1024, # encoder layer/attention input/output size (Hidden Size D in the paper) + depth = 24, + heads = 16, # (Heads in the paper) + dim_head = 64, # attention hidden size (seems be default, never change this) + mlp_dim = 4096, # mlp layer hidden size (MLP size in the paper) + dropout = 0., + emb_dropout = 0. + ) + verify(vit_l_16, 0.98) + + # vit_h_16 = ViT( + # image_size = 224, + # patch_size = 16, + # num_classes = 1000, + # dim = 1280, # encoder layer/attention input/output size (Hidden Size D in the paper) + # depth = 32, + # heads = 16, # (Heads in the paper) + # dim_head = 64, # attention hidden size (seems be default, never change this) + # mlp_dim = 5120, # mlp layer hidden size (MLP size in the paper) + # dropout = 0., + # emb_dropout = 0. + # ) + # verify(vit_h_16) \ No newline at end of file diff --git a/new_impl/cv/dnns/yolov3/coco_evaluator.py b/new_impl/cv/dnns/yolov3/coco_evaluator.py new file mode 100644 index 0000000000000000000000000000000000000000..7c960fdebaff5f4ab16b9e1bfe5e4d3c53db2ea9 --- /dev/null +++ b/new_impl/cv/dnns/yolov3/coco_evaluator.py @@ -0,0 +1,372 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +# Copyright (c) Megvii, Inc. and its affiliates. + +import contextlib +import io +import itertools +import json +import tempfile +import time +from loguru import logger +from tabulate import tabulate +from tqdm import tqdm + +import numpy as np + +import torch + +# from core.common.dnn.detection.yolox.yolox.data.datasets import COCO_CLASSES +from .utils import ( + gather, + is_main_process, + postprocess, + synchronize, + time_synchronized, + xyxy2xywh +) +# from core.common.dnn.detection.yolox.yolox.layers import COCOeval_opt as COCOeval + + +def per_class_AR_table(coco_eval, class_names, headers=["class", "AR"], colums=6): + per_class_AR = {} + recalls = coco_eval.eval["recall"] + # dimension of recalls: [TxKxAxM] + # recall has dims (iou, cls, area range, max dets) + assert len(class_names) == recalls.shape[1] + + for idx, name in enumerate(class_names): + recall = recalls[:, idx, 0, -1] + recall = recall[recall > -1] + ar = np.mean(recall) if recall.size else float("nan") + per_class_AR[name] = float(ar * 100) + + num_cols = min(colums, len(per_class_AR) * len(headers)) + result_pair = [x for pair in per_class_AR.items() for x in pair] + row_pair = itertools.zip_longest(*[result_pair[i::num_cols] for i in range(num_cols)]) + table_headers = headers * (num_cols // len(headers)) + table = tabulate( + row_pair, tablefmt="pipe", floatfmt=".3f", headers=table_headers, numalign="left", + ) + return table + + +def per_class_AP_table(coco_eval, class_names, headers=["class", "AP"], colums=6): + per_class_AP = {} + precisions = coco_eval.eval["precision"] + # dimension of precisions: [TxRxKxAxM] + # precision has dims (iou, recall, cls, area range, max dets) + assert len(class_names) == precisions.shape[2] + + for idx, name in enumerate(class_names): + # area range index 0: all area ranges + # max dets index -1: typically 100 per image + precision = precisions[:, :, idx, 0, -1] + precision = precision[precision > -1] + ap = np.mean(precision) if precision.size else float("nan") + per_class_AP[name] = float(ap * 100) + + num_cols = min(colums, len(per_class_AP) * len(headers)) + result_pair = [x for pair in per_class_AP.items() for x in pair] + row_pair = itertools.zip_longest(*[result_pair[i::num_cols] for i in range(num_cols)]) + table_headers = headers * (num_cols // len(headers)) + table = tabulate( + row_pair, tablefmt="pipe", floatfmt=".3f", headers=table_headers, numalign="left", + ) + return table + + +class COCOEvaluator: + """ + COCO AP Evaluation class. All the data in the val2017 dataset are processed + and evaluated by COCO API. + """ + + def __init__( + self, + dataloader, + img_size: int, + confthre: float, + nmsthre: float, + num_classes: int, + testdev: bool = False, + per_class_AP: bool = False, + per_class_AR: bool = False, + ): + """ + Args: + dataloader (Dataloader): evaluate dataloader. + img_size: image size after preprocess. images are resized + to squares whose shape is (img_size, img_size). + confthre: confidence threshold ranging from 0 to 1, which + is defined in the config file. + nmsthre: IoU threshold of non-max supression ranging from 0 to 1. + per_class_AP: Show per class AP during evalution or not. Default to False. + per_class_AR: Show per class AR during evalution or not. Default to False. + """ + self.dataloader = dataloader + self.img_size = img_size + self.confthre = confthre + self.nmsthre = nmsthre + self.num_classes = num_classes + self.testdev = testdev + self.per_class_AP = per_class_AP + self.per_class_AR = per_class_AR + + def evaluate( + self, + model, + distributed=False, + half=False, + trt_file=None, + decoder=None, + test_size=None, + ): + """ + COCO average precision (AP) Evaluation. Iterate inference on the test dataset + and the results are evaluated by COCO API. + + NOTE: This function will change training mode to False, please save states if needed. + + Args: + model : model to evaluate. + + Returns: + ap50_95 (float) : COCO AP of IoU=50:95 + ap50 (float) : COCO AP of IoU=50 + summary (sr): summary info of evaluation. + """ + # TODO half to amp_test + tensor_type = torch.cuda.HalfTensor if half else torch.cuda.FloatTensor + model = model.eval() + if half: + model = model.half() + ids = [] + data_list = [] + progress_bar = iter if is_main_process() else iter + + inference_time = 0 + nms_time = 0 + n_samples = max(len(self.dataloader) - 1, 1) + + if trt_file is not None: + from torch2trt import TRTModule + + model_trt = TRTModule() + model_trt.load_state_dict(torch.load(trt_file)) + + x = torch.ones(1, 3, test_size[0], test_size[1]).cuda() + model(x) + model = model_trt + + import tqdm + for cur_iter, (imgs, _, info_imgs, ids) in tqdm.tqdm(enumerate( + progress_bar(self.dataloader) + ), dynamic_ncols=True, leave=False, total=len(self.dataloader)): + with torch.no_grad(): + imgs = imgs.type(tensor_type) + + # skip the the last iters since batchsize might be not enough for batch inference + is_time_record = cur_iter < len(self.dataloader) - 1 + if is_time_record: + start = time.time() + + outputs = model(imgs) + if decoder is not None: + outputs = decoder(outputs, dtype=outputs.type()) + + if is_time_record: + infer_end = time_synchronized() + inference_time += infer_end - start + + outputs = postprocess( + outputs, self.num_classes, self.confthre, self.nmsthre + ) + if is_time_record: + nms_end = time_synchronized() + nms_time += nms_end - infer_end + + data_list.extend(self.convert_to_coco_format(outputs, info_imgs, ids, imgs)) + + statistics = torch.cuda.FloatTensor([inference_time, nms_time, n_samples]) + if distributed: + data_list = gather(data_list, dst=0) + data_list = list(itertools.chain(*data_list)) + torch.distributed.reduce(statistics, dst=0) + + eval_results = self.evaluate_prediction(data_list, statistics) + synchronize() + return eval_results + + def convert_to_coco_format(self, outputs, info_imgs, ids, input_imgs=None): + data_list = [] + img_i = 0 + for (output, img_h, img_w, img_id, img) in zip( + outputs, info_imgs[0], info_imgs[1], ids, input_imgs + ): + if output is None: + continue + output = output.cpu() + + bboxes = output[:, 0:4] + + # preprocessing: resize + scale = min( + self.img_size[0] / float(img_h), self.img_size[1] / float(img_w) + ) + bboxes /= scale + bboxes = xyxy2xywh(bboxes) + + cls = output[:, 6] + scores = output[:, 4] * output[:, 5] + for ind in range(bboxes.shape[0]): + # print(self.dataloader.dataset.class_ids, cls[ind]) + # implemented by queyu, 2022/08/08 + _d = self.dataloader.dataset + if _d.__class__.__name__ == 'MergedDataset': + # _d = _d.datasets[0] + raise NotImplementedError + from data import ABDataset + if _d.__class__.__name__ == '_AugWrapperForDataset': + _d = _d.raw_dataset + if isinstance(_d, ABDataset): + _d = _d.dataset + if _d.__class__.__name__ == '_SplitDataset': + raise NotImplementedError + _d = _d.underlying_dataset + + class_ids = _d.class_ids + if int(cls[ind]) >= len(class_ids): + raise RuntimeError + label = self.dataloader.dataset.class_ids[-1] + else: + label = class_ids[int(cls[ind])] + pred_data = { + "image_id": int(img_id), + "category_id": label, + "bbox": bboxes[ind].numpy().tolist(), + "score": scores[ind].numpy().item(), + "segmentation": [], + } # COCO json format + data_list.append(pred_data) + + # TODO: debug + # img = input_imgs[ind] + + + # from torchvision.transforms import ToTensor, ToPILImage + # from torchvision.utils import make_grid + # from PIL import Image, ImageDraw + # import matplotlib.pyplot as plt + # import numpy as np + # def draw_bbox(img, bbox, label, f): + # # if f: + # # img = np.uint8(img.permute(1, 2, 0)) + # # img = Image.fromarray(img) + # img = ToPILImage()(img) + # draw = ImageDraw.Draw(img) + # draw.rectangle(bbox, outline=(255, 0, 0), width=6) + # draw.text((bbox[0], bbox[1]), label) + # return ToTensor()(np.array(img)) + + # def xywh2xyxy(bbox): + # x, y, w, h = bbox + # x1, y1 = x, y + # x2, y2 = x + w, y + h + # return x1, y1, x2, y2 + + # img = draw_bbox(img, xywh2xyxy(bboxes[ind].numpy()), str(label), True) + + # img = make_grid([img], 1, normalize=True) + # plt.axis('off') + # img = img.permute(1, 2, 0).numpy() + # plt.imshow(img) + # plt.savefig(f'./tmp-coco-eval-{ind}.png') + # plt.clf() + # img_i += 1 + + # exit(0) + return data_list + + def evaluate_prediction(self, data_dict, statistics): + if not is_main_process(): + return 0, 0, None + + # logger.info("Evaluate in main process...") + + annType = ["segm", "bbox", "keypoints"] + + inference_time = statistics[0].item() + nms_time = statistics[1].item() + n_samples = statistics[2].item() + + a_infer_time = 1000 * inference_time / (n_samples * self.dataloader.batch_size) + a_nms_time = 1000 * nms_time / (n_samples * self.dataloader.batch_size) + + time_info = ", ".join( + [ + "Average {} time: {:.2f} ms".format(k, v) + for k, v in zip( + ["forward", "NMS", "inference"], + [a_infer_time, a_nms_time, (a_infer_time + a_nms_time)], + ) + ] + ) + + info = time_info + "\n" + + # Evaluate the Dt (detection) json comparing with the ground truth + if len(data_dict) > 0: + # cocoGt = self.dataloader.dataset.coco + _d = self.dataloader.dataset + if _d.__class__.__name__ == 'MergedDataset': + # _d = _d.datasets[0] + raise NotImplementedError + from data import ABDataset + if _d.__class__.__name__ == '_AugWrapperForDataset': + _d = _d.raw_dataset + if isinstance(_d, ABDataset): + _d = _d.dataset + if _d.__class__.__name__ == '_SplitDataset': + raise NotImplementedError + _d = _d.underlying_dataset + + cocoGt = _d.coco + + # implemented by queyu, 2022/08/08 + # make cocoGt's label += y_offset + # cocoGt: COCOAPI + + # TODO: since pycocotools can't process dict in py36, write data to json file. + if self.testdev: + json.dump(data_dict, open("./yolox_testdev_2017.json", "w")) + cocoDt = cocoGt.loadRes("./yolox_testdev_2017.json") + else: + _, tmp = tempfile.mkstemp() + json.dump(data_dict, open(tmp, "w")) + cocoDt = cocoGt.loadRes(tmp) + # try: + # from core.common.dnn.detection.yolox.yolox.layers import COCOeval_opt as COCOeval + # except ImportError: + from pycocotools.cocoeval import COCOeval + + logger.warning("Use standard COCOeval.") + + cocoEval = COCOeval(cocoGt, cocoDt, annType[1]) + cocoEval.evaluate() + cocoEval.accumulate() + redirect_string = io.StringIO() + with contextlib.redirect_stdout(redirect_string): + cocoEval.summarize() + info += redirect_string.getvalue() + cat_ids = list(cocoGt.cats.keys()) + cat_names = [cocoGt.cats[catId]['name'] for catId in sorted(cat_ids)] + if self.per_class_AP: + AP_table = per_class_AP_table(cocoEval, class_names=cat_names) + info += "per class AP:\n" + AP_table + "\n" + if self.per_class_AR: + AR_table = per_class_AR_table(cocoEval, class_names=cat_names) + info += "per class AR:\n" + AR_table + "\n" + return cocoEval.stats[0], cocoEval.stats[1], info + else: + return 0, 0, info diff --git a/new_impl/cv/dnns/yolov3/head.py b/new_impl/cv/dnns/yolov3/head.py new file mode 100644 index 0000000000000000000000000000000000000000..1eca44bae9019b1438e01d433afbef0addf0b784 --- /dev/null +++ b/new_impl/cv/dnns/yolov3/head.py @@ -0,0 +1,676 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +# Copyright (c) Megvii Inc. All rights reserved. + +import math +from loguru import logger + +import torch +import torch.nn as nn +import torch.nn.functional as F + +# from core.common.dnn.detection.yolox.yolox.utils import bboxes_iou + +from .losses import IOUloss +from .network_blocks import BaseConv, DWConv + + +def bboxes_iou(bboxes_a, bboxes_b, xyxy=True): + if bboxes_a.shape[1] != 4 or bboxes_b.shape[1] != 4: + raise IndexError + + if xyxy: + tl = torch.max(bboxes_a[:, None, :2], bboxes_b[:, :2]) + br = torch.min(bboxes_a[:, None, 2:], bboxes_b[:, 2:]) + area_a = torch.prod(bboxes_a[:, 2:] - bboxes_a[:, :2], 1) + area_b = torch.prod(bboxes_b[:, 2:] - bboxes_b[:, :2], 1) + else: + tl = torch.max( + (bboxes_a[:, None, :2] - bboxes_a[:, None, 2:] / 2), + (bboxes_b[:, :2] - bboxes_b[:, 2:] / 2), + ) + br = torch.min( + (bboxes_a[:, None, :2] + bboxes_a[:, None, 2:] / 2), + (bboxes_b[:, :2] + bboxes_b[:, 2:] / 2), + ) + + area_a = torch.prod(bboxes_a[:, 2:], 1) + area_b = torch.prod(bboxes_b[:, 2:], 1) + en = (tl < br).type(tl.type()).prod(dim=2) + area_i = torch.prod(br - tl, 2) * en # * ((tl < br).all()) + return area_i / (area_a[:, None] + area_b - area_i) + + +class YOLOXHead(nn.Module): + def __init__( + self, + num_classes, + width=1.0, + strides=[8, 16, 32], + in_channels=[256, 512, 1024], + act="silu", + depthwise=False, + ): + """ + Args: + act (str): activation type of conv. Defalut value: "silu". + depthwise (bool): whether apply depthwise conv in conv branch. Defalut value: False. + """ + super().__init__() + + self.n_anchors = 1 + self.num_classes = num_classes + self.decode_in_inference = True # for deploy, set to False + + self.cls_convs = nn.ModuleList() + self.reg_convs = nn.ModuleList() + self.cls_preds = nn.ModuleList() + self.reg_preds = nn.ModuleList() + self.obj_preds = nn.ModuleList() + self.stems = nn.ModuleList() + Conv = DWConv if depthwise else BaseConv + + for i in range(len(in_channels)): + self.stems.append( + BaseConv( + in_channels=int(in_channels[i] * width), + out_channels=int(256 * width), + ksize=1, + stride=1, + act=act, + ) + ) + self.cls_convs.append( + nn.Sequential( + *[ + Conv( + in_channels=int(256 * width), + out_channels=int(256 * width), + ksize=3, + stride=1, + act=act, + ), + Conv( + in_channels=int(256 * width), + out_channels=int(256 * width), + ksize=3, + stride=1, + act=act, + ), + ] + ) + ) + self.reg_convs.append( + nn.Sequential( + *[ + Conv( + in_channels=int(256 * width), + out_channels=int(256 * width), + ksize=3, + stride=1, + act=act, + ), + Conv( + in_channels=int(256 * width), + out_channels=int(256 * width), + ksize=3, + stride=1, + act=act, + ), + ] + ) + ) + self.cls_preds.append( + nn.Conv2d( + in_channels=int(256 * width), + out_channels=self.n_anchors * self.num_classes, + kernel_size=1, + stride=1, + padding=0, + ) + ) + self.reg_preds.append( + nn.Conv2d( + in_channels=int(256 * width), + out_channels=4, + kernel_size=1, + stride=1, + padding=0, + ) + ) + self.obj_preds.append( + nn.Conv2d( + in_channels=int(256 * width), + out_channels=self.n_anchors * 1, + kernel_size=1, + stride=1, + padding=0, + ) + ) + + self.use_l1 = False + self.l1_loss = nn.L1Loss(reduction="none") + self.bcewithlog_loss = nn.BCEWithLogitsLoss(reduction="none") + self.iou_loss = IOUloss(reduction="none") + self.strides = strides + self.grids = [torch.zeros(1)] * len(in_channels) + + def initialize_biases(self, prior_prob): + for conv in self.cls_preds: + b = conv.bias.view(self.n_anchors, -1) + b.data.fill_(-math.log((1 - prior_prob) / prior_prob)) + conv.bias = torch.nn.Parameter(b.view(-1), requires_grad=True) + + for conv in self.obj_preds: + b = conv.bias.view(self.n_anchors, -1) + b.data.fill_(-math.log((1 - prior_prob) / prior_prob)) + conv.bias = torch.nn.Parameter(b.view(-1), requires_grad=True) + + def forward(self, xin, labels=None, imgs=None): + if isinstance(xin, list): + xin, labels, imgs = xin[0], xin[1], xin[2] + + outputs = [] + origin_preds = [] + x_shifts = [] + y_shifts = [] + expanded_strides = [] + + for k, (cls_conv, reg_conv, stride_this_level, x) in enumerate( + zip(self.cls_convs, self.reg_convs, self.strides, xin) + ): + x = self.stems[k](x) + cls_x = x + reg_x = x + + cls_feat = cls_conv(cls_x) + cls_output = self.cls_preds[k](cls_feat) + # print('cls output size', cls_output.size()) + + reg_feat = reg_conv(reg_x) + reg_output = self.reg_preds[k](reg_feat) + obj_output = self.obj_preds[k](reg_feat) + + # if self.training: + if labels is not None: + output = torch.cat([reg_output, obj_output, cls_output], 1) + output, grid = self.get_output_and_grid( + output, k, stride_this_level, xin[0].type() + ) + x_shifts.append(grid[:, :, 0]) + y_shifts.append(grid[:, :, 1]) + expanded_strides.append( + torch.zeros(1, grid.shape[1]) + .fill_(stride_this_level) + .type_as(xin[0]) + ) + if self.use_l1: + batch_size = reg_output.shape[0] + hsize, wsize = reg_output.shape[-2:] + reg_output = reg_output.view( + batch_size, self.n_anchors, 4, hsize, wsize + ) + reg_output = reg_output.permute(0, 1, 3, 4, 2).reshape( + batch_size, -1, 4 + ) + origin_preds.append(reg_output.clone()) + + else: + output = torch.cat( + [reg_output, obj_output.sigmoid(), cls_output.sigmoid()], 1 + ) + + outputs.append(output) + # print(111) + # if self.training: + if labels is not None: + return self.get_losses( + imgs, + x_shifts, + y_shifts, + expanded_strides, + labels, + torch.cat(outputs, 1), + origin_preds, + dtype=xin[0].dtype, + ) + else: + self.hw = [x.shape[-2:] for x in outputs] + # [batch, n_anchors_all, 85] + outputs = torch.cat( + [x.flatten(start_dim=2) for x in outputs], dim=2 + ).permute(0, 2, 1) + if self.decode_in_inference: + return self.decode_outputs(outputs, dtype=xin[0].type()) + else: + # print(outputs.size()) + return outputs + + def get_output_and_grid(self, output, k, stride, dtype): + grid = self.grids[k] + + batch_size = output.shape[0] + n_ch = 5 + self.num_classes + hsize, wsize = output.shape[-2:] + if grid.shape[2:4] != output.shape[2:4]: + # yv, xv = torch.meshgrid([torch.arange(hsize), torch.arange(wsize)], indexing="ij") + yv, xv = torch.meshgrid([torch.arange(hsize), torch.arange(wsize)]) # fixed bu queyu, 2022/03/14 + grid = torch.stack((xv, yv), 2).view(1, 1, hsize, wsize, 2).type(dtype) + self.grids[k] = grid + + output = output.view(batch_size, self.n_anchors, n_ch, hsize, wsize) + output = output.permute(0, 1, 3, 4, 2).reshape( + batch_size, self.n_anchors * hsize * wsize, -1 + ) + grid = grid.view(1, -1, 2) + output[..., :2] = (output[..., :2] + grid) * stride + output[..., 2:4] = torch.exp(output[..., 2:4]) * stride + return output, grid + + def decode_outputs(self, outputs, dtype): + grids = [] + strides = [] + for (hsize, wsize), stride in zip(self.hw, self.strides): + # yv, xv = torch.meshgrid([torch.arange(hsize), torch.arange(wsize)], indexing="ij") + yv, xv = torch.meshgrid([torch.arange(hsize), torch.arange(wsize)]) # fixed by queyu, 2022/03/14 + grid = torch.stack((xv, yv), 2).view(1, -1, 2) + grids.append(grid) + shape = grid.shape[:2] + strides.append(torch.full((*shape, 1), stride)) + + grids = torch.cat(grids, dim=1).type(dtype) + strides = torch.cat(strides, dim=1).type(dtype) + + outputs[..., :2] = (outputs[..., :2] + grids) * strides + outputs[..., 2:4] = torch.exp(outputs[..., 2:4]) * strides + return outputs + + def get_losses( + self, + imgs, + x_shifts, + y_shifts, + expanded_strides, + labels, + outputs, + origin_preds, + dtype, + ): + bbox_preds = outputs[:, :, :4] # [batch, n_anchors_all, 4] + obj_preds = outputs[:, :, 4].unsqueeze(-1) # [batch, n_anchors_all, 1] + cls_preds = outputs[:, :, 5:] # [batch, n_anchors_all, n_cls] + + # calculate targets + nlabel = (labels.sum(dim=2) > 0).sum(dim=1) # number of objects + + total_num_anchors = outputs.shape[1] + x_shifts = torch.cat(x_shifts, 1) # [1, n_anchors_all] + y_shifts = torch.cat(y_shifts, 1) # [1, n_anchors_all] + expanded_strides = torch.cat(expanded_strides, 1) + if self.use_l1: + origin_preds = torch.cat(origin_preds, 1) + + cls_targets = [] + reg_targets = [] + l1_targets = [] + obj_targets = [] + fg_masks = [] + + num_fg = 0.0 + num_gts = 0.0 + + for batch_idx in range(outputs.shape[0]): + num_gt = int(nlabel[batch_idx]) + num_gts += num_gt + if num_gt == 0: + cls_target = outputs.new_zeros((0, self.num_classes)) + reg_target = outputs.new_zeros((0, 4)) + l1_target = outputs.new_zeros((0, 4)) + obj_target = outputs.new_zeros((total_num_anchors, 1)) + fg_mask = outputs.new_zeros(total_num_anchors).bool() + else: + gt_bboxes_per_image = labels[batch_idx, :num_gt, 1:5] + gt_classes = labels[batch_idx, :num_gt, 0] + bboxes_preds_per_image = bbox_preds[batch_idx] + + try: + ( + gt_matched_classes, + fg_mask, + pred_ious_this_matching, + matched_gt_inds, + num_fg_img, + ) = self.get_assignments( # noqa + batch_idx, + num_gt, + total_num_anchors, + gt_bboxes_per_image, + gt_classes, + bboxes_preds_per_image, + expanded_strides, + x_shifts, + y_shifts, + cls_preds, + bbox_preds, + obj_preds, + labels, + imgs, + ) + except RuntimeError: + logger.error( + "OOM RuntimeError is raised due to the huge memory cost during label assignment. \ + CPU mode is applied in this batch. If you want to avoid this issue, \ + try to reduce the batch size or image size." + ) + torch.cuda.empty_cache() + ( + gt_matched_classes, + fg_mask, + pred_ious_this_matching, + matched_gt_inds, + num_fg_img, + ) = self.get_assignments( # noqa + batch_idx, + num_gt, + total_num_anchors, + gt_bboxes_per_image, + gt_classes, + bboxes_preds_per_image, + expanded_strides, + x_shifts, + y_shifts, + cls_preds, + bbox_preds, + obj_preds, + labels, + imgs, + "cpu", + ) + + torch.cuda.empty_cache() + num_fg += num_fg_img + + cls_target = F.one_hot( + gt_matched_classes.to(torch.int64), self.num_classes + ) * pred_ious_this_matching.unsqueeze(-1) + obj_target = fg_mask.unsqueeze(-1) + reg_target = gt_bboxes_per_image[matched_gt_inds] + if self.use_l1: + l1_target = self.get_l1_target( + outputs.new_zeros((num_fg_img, 4)), + gt_bboxes_per_image[matched_gt_inds], + expanded_strides[0][fg_mask], + x_shifts=x_shifts[0][fg_mask], + y_shifts=y_shifts[0][fg_mask], + ) + + cls_targets.append(cls_target) + reg_targets.append(reg_target) + obj_targets.append(obj_target.to(dtype)) + fg_masks.append(fg_mask) + if self.use_l1: + l1_targets.append(l1_target) + + cls_targets = torch.cat(cls_targets, 0) + reg_targets = torch.cat(reg_targets, 0) + obj_targets = torch.cat(obj_targets, 0) + fg_masks = torch.cat(fg_masks, 0) + if self.use_l1: + l1_targets = torch.cat(l1_targets, 0) + + num_fg = max(num_fg, 1) + loss_iou = ( + self.iou_loss(bbox_preds.view(-1, 4)[fg_masks], reg_targets) + ).sum() / num_fg + loss_obj = ( + self.bcewithlog_loss(obj_preds.view(-1, 1), obj_targets) + ).sum() / num_fg + loss_cls = ( + self.bcewithlog_loss( + cls_preds.view(-1, self.num_classes)[fg_masks], cls_targets + ) + ).sum() / num_fg + if self.use_l1: + loss_l1 = ( + self.l1_loss(origin_preds.view(-1, 4)[fg_masks], l1_targets) + ).sum() / num_fg + else: + loss_l1 = 0.0 + + reg_weight = 5.0 + loss = reg_weight * loss_iou + loss_obj + loss_cls + loss_l1 + + return ( + loss, + reg_weight * loss_iou, + loss_obj, + loss_cls, + loss_l1, + num_fg / max(num_gts, 1), + ) + + def get_l1_target(self, l1_target, gt, stride, x_shifts, y_shifts, eps=1e-8): + l1_target[:, 0] = gt[:, 0] / stride - x_shifts + l1_target[:, 1] = gt[:, 1] / stride - y_shifts + l1_target[:, 2] = torch.log(gt[:, 2] / stride + eps) + l1_target[:, 3] = torch.log(gt[:, 3] / stride + eps) + return l1_target + + @torch.no_grad() + def get_assignments( + self, + batch_idx, + num_gt, + total_num_anchors, + gt_bboxes_per_image, + gt_classes, + bboxes_preds_per_image, + expanded_strides, + x_shifts, + y_shifts, + cls_preds, + bbox_preds, + obj_preds, + labels, + imgs, + mode="gpu", + ): + + if mode == "cpu": + print("------------CPU Mode for This Batch-------------") + gt_bboxes_per_image = gt_bboxes_per_image.cpu().float() + bboxes_preds_per_image = bboxes_preds_per_image.cpu().float() + gt_classes = gt_classes.cpu().float() + expanded_strides = expanded_strides.cpu().float() + x_shifts = x_shifts.cpu() + y_shifts = y_shifts.cpu() + + fg_mask, is_in_boxes_and_center = self.get_in_boxes_info( + gt_bboxes_per_image, + expanded_strides, + x_shifts, + y_shifts, + total_num_anchors, + num_gt, + ) + + bboxes_preds_per_image = bboxes_preds_per_image[fg_mask] + cls_preds_ = cls_preds[batch_idx][fg_mask] + obj_preds_ = obj_preds[batch_idx][fg_mask] + num_in_boxes_anchor = bboxes_preds_per_image.shape[0] + + if mode == "cpu": + gt_bboxes_per_image = gt_bboxes_per_image.cpu() + bboxes_preds_per_image = bboxes_preds_per_image.cpu() + + pair_wise_ious = bboxes_iou(gt_bboxes_per_image, bboxes_preds_per_image, False) + + gt_cls_per_image = ( + F.one_hot(gt_classes.to(torch.int64), self.num_classes) + .float() + .unsqueeze(1) + .repeat(1, num_in_boxes_anchor, 1) + ) + pair_wise_ious_loss = -torch.log(pair_wise_ious + 1e-8) + + if mode == "cpu": + cls_preds_, obj_preds_ = cls_preds_.cpu(), obj_preds_.cpu() + + with torch.cuda.amp.autocast(enabled=False): + cls_preds_ = ( + cls_preds_.float().unsqueeze(0).repeat(num_gt, 1, 1).sigmoid_() + * obj_preds_.float().unsqueeze(0).repeat(num_gt, 1, 1).sigmoid_() + ) + pair_wise_cls_loss = F.binary_cross_entropy( + cls_preds_.sqrt_(), gt_cls_per_image, reduction="none" + ).sum(-1) + del cls_preds_ + + cost = ( + pair_wise_cls_loss + + 3.0 * pair_wise_ious_loss + + 100000.0 * (~is_in_boxes_and_center) + ) + + ( + num_fg, + gt_matched_classes, + pred_ious_this_matching, + matched_gt_inds, + ) = self.dynamic_k_matching(cost, pair_wise_ious, gt_classes, num_gt, fg_mask) + del pair_wise_cls_loss, cost, pair_wise_ious, pair_wise_ious_loss + + if mode == "cpu": + gt_matched_classes = gt_matched_classes.cuda() + fg_mask = fg_mask.cuda() + pred_ious_this_matching = pred_ious_this_matching.cuda() + matched_gt_inds = matched_gt_inds.cuda() + + return ( + gt_matched_classes, + fg_mask, + pred_ious_this_matching, + matched_gt_inds, + num_fg, + ) + + def get_in_boxes_info( + self, + gt_bboxes_per_image, + expanded_strides, + x_shifts, + y_shifts, + total_num_anchors, + num_gt, + ): + expanded_strides_per_image = expanded_strides[0] + x_shifts_per_image = x_shifts[0] * expanded_strides_per_image + y_shifts_per_image = y_shifts[0] * expanded_strides_per_image + x_centers_per_image = ( + (x_shifts_per_image + 0.5 * expanded_strides_per_image) + .unsqueeze(0) + .repeat(num_gt, 1) + ) # [n_anchor] -> [n_gt, n_anchor] + y_centers_per_image = ( + (y_shifts_per_image + 0.5 * expanded_strides_per_image) + .unsqueeze(0) + .repeat(num_gt, 1) + ) + + gt_bboxes_per_image_l = ( + (gt_bboxes_per_image[:, 0] - 0.5 * gt_bboxes_per_image[:, 2]) + .unsqueeze(1) + .repeat(1, total_num_anchors) + ) + gt_bboxes_per_image_r = ( + (gt_bboxes_per_image[:, 0] + 0.5 * gt_bboxes_per_image[:, 2]) + .unsqueeze(1) + .repeat(1, total_num_anchors) + ) + gt_bboxes_per_image_t = ( + (gt_bboxes_per_image[:, 1] - 0.5 * gt_bboxes_per_image[:, 3]) + .unsqueeze(1) + .repeat(1, total_num_anchors) + ) + gt_bboxes_per_image_b = ( + (gt_bboxes_per_image[:, 1] + 0.5 * gt_bboxes_per_image[:, 3]) + .unsqueeze(1) + .repeat(1, total_num_anchors) + ) + + b_l = x_centers_per_image - gt_bboxes_per_image_l + b_r = gt_bboxes_per_image_r - x_centers_per_image + b_t = y_centers_per_image - gt_bboxes_per_image_t + b_b = gt_bboxes_per_image_b - y_centers_per_image + bbox_deltas = torch.stack([b_l, b_t, b_r, b_b], 2) + + is_in_boxes = bbox_deltas.min(dim=-1).values > 0.0 + is_in_boxes_all = is_in_boxes.sum(dim=0) > 0 + # in fixed center + + center_radius = 2.5 + + gt_bboxes_per_image_l = (gt_bboxes_per_image[:, 0]).unsqueeze(1).repeat( + 1, total_num_anchors + ) - center_radius * expanded_strides_per_image.unsqueeze(0) + gt_bboxes_per_image_r = (gt_bboxes_per_image[:, 0]).unsqueeze(1).repeat( + 1, total_num_anchors + ) + center_radius * expanded_strides_per_image.unsqueeze(0) + gt_bboxes_per_image_t = (gt_bboxes_per_image[:, 1]).unsqueeze(1).repeat( + 1, total_num_anchors + ) - center_radius * expanded_strides_per_image.unsqueeze(0) + gt_bboxes_per_image_b = (gt_bboxes_per_image[:, 1]).unsqueeze(1).repeat( + 1, total_num_anchors + ) + center_radius * expanded_strides_per_image.unsqueeze(0) + + c_l = x_centers_per_image - gt_bboxes_per_image_l + c_r = gt_bboxes_per_image_r - x_centers_per_image + c_t = y_centers_per_image - gt_bboxes_per_image_t + c_b = gt_bboxes_per_image_b - y_centers_per_image + center_deltas = torch.stack([c_l, c_t, c_r, c_b], 2) + is_in_centers = center_deltas.min(dim=-1).values > 0.0 + is_in_centers_all = is_in_centers.sum(dim=0) > 0 + + # in boxes and in centers + is_in_boxes_anchor = is_in_boxes_all | is_in_centers_all + + is_in_boxes_and_center = ( + is_in_boxes[:, is_in_boxes_anchor] & is_in_centers[:, is_in_boxes_anchor] + ) + return is_in_boxes_anchor, is_in_boxes_and_center + + def dynamic_k_matching(self, cost, pair_wise_ious, gt_classes, num_gt, fg_mask): + # Dynamic K + # --------------------------------------------------------------- + matching_matrix = torch.zeros_like(cost, dtype=torch.uint8) + + ious_in_boxes_matrix = pair_wise_ious + n_candidate_k = min(10, ious_in_boxes_matrix.size(1)) + topk_ious, _ = torch.topk(ious_in_boxes_matrix, n_candidate_k, dim=1) + dynamic_ks = torch.clamp(topk_ious.sum(1).int(), min=1) + dynamic_ks = dynamic_ks.tolist() + for gt_idx in range(num_gt): + _, pos_idx = torch.topk( + cost[gt_idx], k=dynamic_ks[gt_idx], largest=False + ) + matching_matrix[gt_idx][pos_idx] = 1 + + del topk_ious, dynamic_ks, pos_idx + + anchor_matching_gt = matching_matrix.sum(0) + if (anchor_matching_gt > 1).sum() > 0: + _, cost_argmin = torch.min(cost[:, anchor_matching_gt > 1], dim=0) + matching_matrix[:, anchor_matching_gt > 1] *= 0 + matching_matrix[cost_argmin, anchor_matching_gt > 1] = 1 + fg_mask_inboxes = matching_matrix.sum(0) > 0 + num_fg = fg_mask_inboxes.sum().item() + + fg_mask[fg_mask.clone()] = fg_mask_inboxes + + matched_gt_inds = matching_matrix[:, fg_mask_inboxes].argmax(0) + gt_matched_classes = gt_classes[matched_gt_inds] + + pred_ious_this_matching = (matching_matrix * pair_wise_ious).sum(0)[ + fg_mask_inboxes + ] + return num_fg, gt_matched_classes, pred_ious_this_matching, matched_gt_inds diff --git a/new_impl/cv/dnns/yolov3/losses.py b/new_impl/cv/dnns/yolov3/losses.py new file mode 100644 index 0000000000000000000000000000000000000000..77b4d8ef7660880031f4ef23c82ba3a85b6fd254 --- /dev/null +++ b/new_impl/cv/dnns/yolov3/losses.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +# Copyright (c) Megvii Inc. All rights reserved. + +import torch +import torch.nn as nn + + +class IOUloss(nn.Module): + def __init__(self, reduction="none", loss_type="iou"): + super(IOUloss, self).__init__() + self.reduction = reduction + self.loss_type = loss_type + + def forward(self, pred, target): + assert pred.shape[0] == target.shape[0] + + pred = pred.view(-1, 4) + target = target.view(-1, 4) + tl = torch.max( + (pred[:, :2] - pred[:, 2:] / 2), (target[:, :2] - target[:, 2:] / 2) + ) + br = torch.min( + (pred[:, :2] + pred[:, 2:] / 2), (target[:, :2] + target[:, 2:] / 2) + ) + + area_p = torch.prod(pred[:, 2:], 1) + area_g = torch.prod(target[:, 2:], 1) + + en = (tl < br).type(tl.type()).prod(dim=1) + area_i = torch.prod(br - tl, 1) * en + area_u = area_p + area_g - area_i + iou = (area_i) / (area_u + 1e-16) + + if self.loss_type == "iou": + loss = 1 - iou ** 2 + elif self.loss_type == "giou": + c_tl = torch.min( + (pred[:, :2] - pred[:, 2:] / 2), (target[:, :2] - target[:, 2:] / 2) + ) + c_br = torch.max( + (pred[:, :2] + pred[:, 2:] / 2), (target[:, :2] + target[:, 2:] / 2) + ) + area_c = torch.prod(c_br - c_tl, 1) + giou = iou - (area_c - area_u) / area_c.clamp(1e-16) + loss = 1 - giou.clamp(min=-1.0, max=1.0) + + if self.reduction == "mean": + loss = loss.mean() + elif self.reduction == "sum": + loss = loss.sum() + + return loss diff --git a/new_impl/cv/dnns/yolov3/network_blocks.py b/new_impl/cv/dnns/yolov3/network_blocks.py new file mode 100644 index 0000000000000000000000000000000000000000..68aacfc33208eab072422e0647742006984dfdfd --- /dev/null +++ b/new_impl/cv/dnns/yolov3/network_blocks.py @@ -0,0 +1,210 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +# Copyright (c) Megvii Inc. All rights reserved. + +import torch +import torch.nn as nn + + +class SiLU(nn.Module): + """export-friendly version of nn.SiLU()""" + + @staticmethod + def forward(x): + return x * torch.sigmoid(x) + + +def get_activation(name="silu", inplace=True): + if name == "silu": + module = nn.SiLU(inplace=inplace) + elif name == "relu": + module = nn.ReLU(inplace=inplace) + elif name == "lrelu": + module = nn.LeakyReLU(0.1, inplace=inplace) + else: + raise AttributeError("Unsupported act type: {}".format(name)) + return module + + +class BaseConv(nn.Module): + """A Conv2d -> Batchnorm -> silu/leaky relu block""" + + def __init__( + self, in_channels, out_channels, ksize, stride, groups=1, bias=False, act="silu" + ): + super().__init__() + # same padding + pad = (ksize - 1) // 2 + self.conv = nn.Conv2d( + in_channels, + out_channels, + kernel_size=ksize, + stride=stride, + padding=pad, + groups=groups, + bias=bias, + ) + self.bn = nn.BatchNorm2d(out_channels) + self.act = get_activation(act, inplace=True) + + def forward(self, x): + return self.act(self.bn(self.conv(x))) + + def fuseforward(self, x): + return self.act(self.conv(x)) + + +class DWConv(nn.Module): + """Depthwise Conv + Conv""" + + def __init__(self, in_channels, out_channels, ksize, stride=1, act="silu"): + super().__init__() + self.dconv = BaseConv( + in_channels, + in_channels, + ksize=ksize, + stride=stride, + groups=in_channels, + act=act, + ) + self.pconv = BaseConv( + in_channels, out_channels, ksize=1, stride=1, groups=1, act=act + ) + + def forward(self, x): + x = self.dconv(x) + return self.pconv(x) + + +class Bottleneck(nn.Module): + # Standard bottleneck + def __init__( + self, + in_channels, + out_channels, + shortcut=True, + expansion=0.5, + depthwise=False, + act="silu", + ): + super().__init__() + hidden_channels = int(out_channels * expansion) + Conv = DWConv if depthwise else BaseConv + self.conv1 = BaseConv(in_channels, hidden_channels, 1, stride=1, act=act) + self.conv2 = Conv(hidden_channels, out_channels, 3, stride=1, act=act) + self.use_add = shortcut and in_channels == out_channels + + def forward(self, x): + y = self.conv2(self.conv1(x)) + if self.use_add: + y = y + x + return y + + +class ResLayer(nn.Module): + "Residual layer with `in_channels` inputs." + + def __init__(self, in_channels: int): + super().__init__() + mid_channels = in_channels // 2 + self.layer1 = BaseConv( + in_channels, mid_channels, ksize=1, stride=1, act="lrelu" + ) + self.layer2 = BaseConv( + mid_channels, in_channels, ksize=3, stride=1, act="lrelu" + ) + + def forward(self, x): + out = self.layer2(self.layer1(x)) + return x + out + + +class SPPBottleneck(nn.Module): + """Spatial pyramid pooling layer used in YOLOv3-SPP""" + + def __init__( + self, in_channels, out_channels, kernel_sizes=(5, 9, 13), activation="silu" + ): + super().__init__() + hidden_channels = in_channels // 2 + self.conv1 = BaseConv(in_channels, hidden_channels, 1, stride=1, act=activation) + self.m = nn.ModuleList( + [ + nn.MaxPool2d(kernel_size=ks, stride=1, padding=ks // 2) + for ks in kernel_sizes + ] + ) + conv2_channels = hidden_channels * (len(kernel_sizes) + 1) + self.conv2 = BaseConv(conv2_channels, out_channels, 1, stride=1, act=activation) + + def forward(self, x): + x = self.conv1(x) + x = torch.cat([x] + [m(x) for m in self.m], dim=1) + x = self.conv2(x) + return x + + +class CSPLayer(nn.Module): + """C3 in yolov5, CSP Bottleneck with 3 convolutions""" + + def __init__( + self, + in_channels, + out_channels, + n=1, + shortcut=True, + expansion=0.5, + depthwise=False, + act="silu", + ): + """ + Args: + in_channels (int): input channels. + out_channels (int): output channels. + n (int): number of Bottlenecks. Default value: 1. + """ + # ch_in, ch_out, number, shortcut, groups, expansion + super().__init__() + hidden_channels = int(out_channels * expansion) # hidden channels + self.conv1 = BaseConv(in_channels, hidden_channels, 1, stride=1, act=act) + self.conv2 = BaseConv(in_channels, hidden_channels, 1, stride=1, act=act) + self.conv3 = BaseConv(2 * hidden_channels, out_channels, 1, stride=1, act=act) + module_list = [ + Bottleneck( + hidden_channels, hidden_channels, shortcut, 1.0, depthwise, act=act + ) + for _ in range(n) + ] + self.m = nn.Sequential(*module_list) + + def forward(self, x): + x_1 = self.conv1(x) + x_2 = self.conv2(x) + x_1 = self.m(x_1) + x = torch.cat((x_1, x_2), dim=1) + return self.conv3(x) + + +class Focus(nn.Module): + """Focus width and height information into channel space.""" + + def __init__(self, in_channels, out_channels, ksize=1, stride=1, act="silu"): + super().__init__() + self.conv = BaseConv(in_channels * 4, out_channels, ksize, stride, act=act) + + def forward(self, x): + # shape of x (b,c,w,h) -> y(b,4c,w/2,h/2) + patch_top_left = x[..., ::2, ::2] + patch_top_right = x[..., ::2, 1::2] + patch_bot_left = x[..., 1::2, ::2] + patch_bot_right = x[..., 1::2, 1::2] + x = torch.cat( + ( + patch_top_left, + patch_bot_left, + patch_top_right, + patch_bot_right, + ), + dim=1, + ) + return self.conv(x) diff --git a/new_impl/cv/dnns/yolov3/utils/__init__.py b/new_impl/cv/dnns/yolov3/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..ff8db0dbc2473d2ee7c2b9ad8261183171f08ab1 --- /dev/null +++ b/new_impl/cv/dnns/yolov3/utils/__init__.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +# Copyright (c) Megvii Inc. All rights reserved. + +from .allreduce_norm import * +from .boxes import * +from .checkpoint import load_ckpt, save_checkpoint +from .demo_utils import * +from .dist import * +from .ema import * +from .logger import WandbLogger, setup_logger +from .lr_scheduler import LRScheduler +from .metric import * +from .model_utils import * +from .setup_env import * +from .visualize import * diff --git a/new_impl/cv/dnns/yolov3/utils/allreduce_norm.py b/new_impl/cv/dnns/yolov3/utils/allreduce_norm.py new file mode 100644 index 0000000000000000000000000000000000000000..142c76c78061db6e2c5f4b899bcc5e2f2214f010 --- /dev/null +++ b/new_impl/cv/dnns/yolov3/utils/allreduce_norm.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +# Copyright (c) Megvii Inc. All rights reserved. + +import pickle +from collections import OrderedDict + +import torch +from torch import distributed as dist +from torch import nn + +from .dist import _get_global_gloo_group, get_world_size + +ASYNC_NORM = ( + nn.BatchNorm1d, + nn.BatchNorm2d, + nn.BatchNorm3d, + nn.InstanceNorm1d, + nn.InstanceNorm2d, + nn.InstanceNorm3d, +) + +__all__ = [ + "get_async_norm_states", + "pyobj2tensor", + "tensor2pyobj", + "all_reduce", + "all_reduce_norm", +] + + +def get_async_norm_states(module): + async_norm_states = OrderedDict() + for name, child in module.named_modules(): + if isinstance(child, ASYNC_NORM): + for k, v in child.state_dict().items(): + async_norm_states[".".join([name, k])] = v + return async_norm_states + + +def pyobj2tensor(pyobj, device="cuda"): + """serialize picklable python object to tensor""" + storage = torch.ByteStorage.from_buffer(pickle.dumps(pyobj)) + return torch.ByteTensor(storage).to(device=device) + + +def tensor2pyobj(tensor): + """deserialize tensor to picklable python object""" + return pickle.loads(tensor.cpu().numpy().tobytes()) + + +def _get_reduce_op(op_name): + return { + "sum": dist.ReduceOp.SUM, + "mean": dist.ReduceOp.SUM, + }[op_name.lower()] + + +def all_reduce(py_dict, op="sum", group=None): + """ + Apply all reduce function for python dict object. + NOTE: make sure that every py_dict has the same keys and values are in the same shape. + + Args: + py_dict (dict): dict to apply all reduce op. + op (str): operator, could be "sum" or "mean". + """ + world_size = get_world_size() + if world_size == 1: + return py_dict + if group is None: + group = _get_global_gloo_group() + if dist.get_world_size(group) == 1: + return py_dict + + # all reduce logic across different devices. + py_key = list(py_dict.keys()) + py_key_tensor = pyobj2tensor(py_key) + dist.broadcast(py_key_tensor, src=0) + py_key = tensor2pyobj(py_key_tensor) + + tensor_shapes = [py_dict[k].shape for k in py_key] + tensor_numels = [py_dict[k].numel() for k in py_key] + + flatten_tensor = torch.cat([py_dict[k].flatten() for k in py_key]) + dist.all_reduce(flatten_tensor, op=_get_reduce_op(op)) + if op == "mean": + flatten_tensor /= world_size + + split_tensors = [ + x.reshape(shape) + for x, shape in zip(torch.split(flatten_tensor, tensor_numels), tensor_shapes) + ] + return OrderedDict({k: v for k, v in zip(py_key, split_tensors)}) + + +def all_reduce_norm(module): + """ + All reduce norm statistics in different devices. + """ + states = get_async_norm_states(module) + states = all_reduce(states, op="mean") + module.load_state_dict(states, strict=False) diff --git a/new_impl/cv/dnns/yolov3/utils/boxes.py b/new_impl/cv/dnns/yolov3/utils/boxes.py new file mode 100644 index 0000000000000000000000000000000000000000..a8287a2c243be235cd6504a3b375bde48d3270da --- /dev/null +++ b/new_impl/cv/dnns/yolov3/utils/boxes.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +# Copyright (c) Megvii Inc. All rights reserved. + +import numpy as np + +import torch +import torchvision + +__all__ = [ + "filter_box", + "postprocess", + "bboxes_iou", + "matrix_iou", + "adjust_box_anns", + "xyxy2xywh", + "xyxy2cxcywh", +] + + +def filter_box(output, scale_range): + """ + output: (N, 5+class) shape + """ + min_scale, max_scale = scale_range + w = output[:, 2] - output[:, 0] + h = output[:, 3] - output[:, 1] + keep = (w * h > min_scale * min_scale) & (w * h < max_scale * max_scale) + return output[keep] + + +def postprocess(prediction, num_classes, conf_thre=0.7, nms_thre=0.45, class_agnostic=False): + box_corner = prediction.new(prediction.shape) + box_corner[:, :, 0] = prediction[:, :, 0] - prediction[:, :, 2] / 2 + box_corner[:, :, 1] = prediction[:, :, 1] - prediction[:, :, 3] / 2 + box_corner[:, :, 2] = prediction[:, :, 0] + prediction[:, :, 2] / 2 + box_corner[:, :, 3] = prediction[:, :, 1] + prediction[:, :, 3] / 2 + prediction[:, :, :4] = box_corner[:, :, :4] + + output = [None for _ in range(len(prediction))] + for i, image_pred in enumerate(prediction): + + # If none are remaining => process next image + if not image_pred.size(0): + continue + # Get score and class with highest confidence + class_conf, class_pred = torch.max(image_pred[:, 5: 5 + num_classes], 1, keepdim=True) + + # implemented by queyu, 2022/08/08 + # print(class_pred) + # class_pred -= y_offset + # class_pred[class_pred < 0] = 0 + # print(class_pred) + # exit(0) + + conf_mask = (image_pred[:, 4] * class_conf.squeeze() >= conf_thre).squeeze() + # Detections ordered as (x1, y1, x2, y2, obj_conf, class_conf, class_pred) + detections = torch.cat((image_pred[:, :5], class_conf, class_pred.float()), 1) + detections = detections[conf_mask] + if not detections.size(0): + continue + + if class_agnostic: + nms_out_index = torchvision.ops.nms( + detections[:, :4], + detections[:, 4] * detections[:, 5], + nms_thre, + ) + else: + nms_out_index = torchvision.ops.batched_nms( + detections[:, :4], + detections[:, 4] * detections[:, 5], + detections[:, 6], + nms_thre, + ) + + detections = detections[nms_out_index] + if output[i] is None: + output[i] = detections + else: + output[i] = torch.cat((output[i], detections)) + + return output + + +def bboxes_iou(bboxes_a, bboxes_b, xyxy=True): + if bboxes_a.shape[1] != 4 or bboxes_b.shape[1] != 4: + raise IndexError + + if xyxy: + tl = torch.max(bboxes_a[:, None, :2], bboxes_b[:, :2]) + br = torch.min(bboxes_a[:, None, 2:], bboxes_b[:, 2:]) + area_a = torch.prod(bboxes_a[:, 2:] - bboxes_a[:, :2], 1) + area_b = torch.prod(bboxes_b[:, 2:] - bboxes_b[:, :2], 1) + else: + tl = torch.max( + (bboxes_a[:, None, :2] - bboxes_a[:, None, 2:] / 2), + (bboxes_b[:, :2] - bboxes_b[:, 2:] / 2), + ) + br = torch.min( + (bboxes_a[:, None, :2] + bboxes_a[:, None, 2:] / 2), + (bboxes_b[:, :2] + bboxes_b[:, 2:] / 2), + ) + + area_a = torch.prod(bboxes_a[:, 2:], 1) + area_b = torch.prod(bboxes_b[:, 2:], 1) + en = (tl < br).type(tl.type()).prod(dim=2) + area_i = torch.prod(br - tl, 2) * en # * ((tl < br).all()) + return area_i / (area_a[:, None] + area_b - area_i) + + +def matrix_iou(a, b): + """ + return iou of a and b, numpy version for data augenmentation + """ + lt = np.maximum(a[:, np.newaxis, :2], b[:, :2]) + rb = np.minimum(a[:, np.newaxis, 2:], b[:, 2:]) + + area_i = np.prod(rb - lt, axis=2) * (lt < rb).all(axis=2) + area_a = np.prod(a[:, 2:] - a[:, :2], axis=1) + area_b = np.prod(b[:, 2:] - b[:, :2], axis=1) + return area_i / (area_a[:, np.newaxis] + area_b - area_i + 1e-12) + + +def adjust_box_anns(bbox, scale_ratio, padw, padh, w_max, h_max): + bbox[:, 0::2] = np.clip(bbox[:, 0::2] * scale_ratio + padw, 0, w_max) + bbox[:, 1::2] = np.clip(bbox[:, 1::2] * scale_ratio + padh, 0, h_max) + return bbox + + +def xyxy2xywh(bboxes): + bboxes[:, 2] = bboxes[:, 2] - bboxes[:, 0] + bboxes[:, 3] = bboxes[:, 3] - bboxes[:, 1] + return bboxes + + +def xyxy2cxcywh(bboxes): + bboxes[:, 2] = bboxes[:, 2] - bboxes[:, 0] + bboxes[:, 3] = bboxes[:, 3] - bboxes[:, 1] + bboxes[:, 0] = bboxes[:, 0] + bboxes[:, 2] * 0.5 + bboxes[:, 1] = bboxes[:, 1] + bboxes[:, 3] * 0.5 + return bboxes diff --git a/new_impl/cv/dnns/yolov3/utils/checkpoint.py b/new_impl/cv/dnns/yolov3/utils/checkpoint.py new file mode 100644 index 0000000000000000000000000000000000000000..a0c200e41da9ad8b720369a2181c9642724622ca --- /dev/null +++ b/new_impl/cv/dnns/yolov3/utils/checkpoint.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +# Copyright (c) Megvii Inc. All rights reserved. +import os +import shutil +from loguru import logger + +import torch + + +def load_ckpt(model, ckpt): + model_state_dict = model.state_dict() + load_dict = {} + for key_model, v in model_state_dict.items(): + if key_model not in ckpt: + logger.warning( + "{} is not in the ckpt. Please double check and see if this is desired.".format( + key_model + ) + ) + continue + v_ckpt = ckpt[key_model] + if v.shape != v_ckpt.shape: + logger.warning( + "Shape of {} in checkpoint is {}, while shape of {} in model is {}.".format( + key_model, v_ckpt.shape, key_model, v.shape + ) + ) + continue + load_dict[key_model] = v_ckpt + + model.load_state_dict(load_dict, strict=False) + return model + + +def save_checkpoint(state, is_best, save_dir, model_name=""): + if not os.path.exists(save_dir): + os.makedirs(save_dir) + filename = os.path.join(save_dir, model_name + "_ckpt.pth") + torch.save(state, filename) + if is_best: + best_filename = os.path.join(save_dir, "best_ckpt.pth") + shutil.copyfile(filename, best_filename) diff --git a/new_impl/cv/dnns/yolov3/utils/demo_utils.py b/new_impl/cv/dnns/yolov3/utils/demo_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..71222379497bd4a57d464afb63baebe43e9c447b --- /dev/null +++ b/new_impl/cv/dnns/yolov3/utils/demo_utils.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +# Copyright (c) Megvii Inc. All rights reserved. + +import os + +import numpy as np + +__all__ = ["mkdir", "nms", "multiclass_nms", "demo_postprocess"] + + +def mkdir(path): + if not os.path.exists(path): + os.makedirs(path) + + +def nms(boxes, scores, nms_thr): + """Single class NMS implemented in Numpy.""" + x1 = boxes[:, 0] + y1 = boxes[:, 1] + x2 = boxes[:, 2] + y2 = boxes[:, 3] + + areas = (x2 - x1 + 1) * (y2 - y1 + 1) + order = scores.argsort()[::-1] + + keep = [] + while order.size > 0: + i = order[0] + keep.append(i) + xx1 = np.maximum(x1[i], x1[order[1:]]) + yy1 = np.maximum(y1[i], y1[order[1:]]) + xx2 = np.minimum(x2[i], x2[order[1:]]) + yy2 = np.minimum(y2[i], y2[order[1:]]) + + w = np.maximum(0.0, xx2 - xx1 + 1) + h = np.maximum(0.0, yy2 - yy1 + 1) + inter = w * h + ovr = inter / (areas[i] + areas[order[1:]] - inter) + + inds = np.where(ovr <= nms_thr)[0] + order = order[inds + 1] + + return keep + + +def multiclass_nms(boxes, scores, nms_thr, score_thr, class_agnostic=True): + """Multiclass NMS implemented in Numpy""" + if class_agnostic: + nms_method = multiclass_nms_class_agnostic + else: + nms_method = multiclass_nms_class_aware + return nms_method(boxes, scores, nms_thr, score_thr) + + +def multiclass_nms_class_aware(boxes, scores, nms_thr, score_thr): + """Multiclass NMS implemented in Numpy. Class-aware version.""" + final_dets = [] + num_classes = scores.shape[1] + for cls_ind in range(num_classes): + cls_scores = scores[:, cls_ind] + valid_score_mask = cls_scores > score_thr + if valid_score_mask.sum() == 0: + continue + else: + valid_scores = cls_scores[valid_score_mask] + valid_boxes = boxes[valid_score_mask] + keep = nms(valid_boxes, valid_scores, nms_thr) + if len(keep) > 0: + cls_inds = np.ones((len(keep), 1)) * cls_ind + dets = np.concatenate( + [valid_boxes[keep], valid_scores[keep, None], cls_inds], 1 + ) + final_dets.append(dets) + if len(final_dets) == 0: + return None + return np.concatenate(final_dets, 0) + + +def multiclass_nms_class_agnostic(boxes, scores, nms_thr, score_thr): + """Multiclass NMS implemented in Numpy. Class-agnostic version.""" + cls_inds = scores.argmax(1) + cls_scores = scores[np.arange(len(cls_inds)), cls_inds] + + valid_score_mask = cls_scores > score_thr + if valid_score_mask.sum() == 0: + return None + valid_scores = cls_scores[valid_score_mask] + valid_boxes = boxes[valid_score_mask] + valid_cls_inds = cls_inds[valid_score_mask] + keep = nms(valid_boxes, valid_scores, nms_thr) + if keep: + dets = np.concatenate( + [valid_boxes[keep], valid_scores[keep, None], valid_cls_inds[keep, None]], 1 + ) + return dets + + +def demo_postprocess(outputs, img_size, p6=False): + + grids = [] + expanded_strides = [] + + if not p6: + strides = [8, 16, 32] + else: + strides = [8, 16, 32, 64] + + hsizes = [img_size[0] // stride for stride in strides] + wsizes = [img_size[1] // stride for stride in strides] + + for hsize, wsize, stride in zip(hsizes, wsizes, strides): + xv, yv = np.meshgrid(np.arange(wsize), np.arange(hsize)) + grid = np.stack((xv, yv), 2).reshape(1, -1, 2) + grids.append(grid) + shape = grid.shape[:2] + expanded_strides.append(np.full((*shape, 1), stride)) + + grids = np.concatenate(grids, 1) + expanded_strides = np.concatenate(expanded_strides, 1) + outputs[..., :2] = (outputs[..., :2] + grids) * expanded_strides + outputs[..., 2:4] = np.exp(outputs[..., 2:4]) * expanded_strides + + return outputs diff --git a/new_impl/cv/dnns/yolov3/utils/dist.py b/new_impl/cv/dnns/yolov3/utils/dist.py new file mode 100644 index 0000000000000000000000000000000000000000..e389f42f89d4f3b2d14fbe336e8bf4a0b92add1c --- /dev/null +++ b/new_impl/cv/dnns/yolov3/utils/dist.py @@ -0,0 +1,287 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +# This file mainly comes from +# https://github.com/facebookresearch/detectron2/blob/master/detectron2/utils/comm.py +# Copyright (c) Facebook, Inc. and its affiliates. +# Copyright (c) Megvii Inc. All rights reserved. +""" +This file contains primitives for multi-gpu communication. +This is useful when doing distributed training. +""" + +import functools +import os +import pickle +import time +from contextlib import contextmanager +from loguru import logger + +import numpy as np + +import torch +from torch import distributed as dist + +__all__ = [ + "get_num_devices", + "wait_for_the_master", + "is_main_process", + "synchronize", + "get_world_size", + "get_rank", + "get_local_rank", + "get_local_size", + "time_synchronized", + "gather", + "all_gather", +] + +_LOCAL_PROCESS_GROUP = None + + +def get_num_devices(): + gpu_list = os.getenv('CUDA_VISIBLE_DEVICES', None) + if gpu_list is not None: + return len(gpu_list.split(',')) + else: + devices_list_info = os.popen("nvidia-smi -L") + devices_list_info = devices_list_info.read().strip().split("\n") + return len(devices_list_info) + + +@contextmanager +def wait_for_the_master(local_rank: int): + """ + Make all processes waiting for the master to do some task. + """ + if local_rank > 0: + dist.barrier() + yield + if local_rank == 0: + if not dist.is_available(): + return + if not dist.is_initialized(): + return + else: + dist.barrier() + + +def synchronize(): + """ + Helper function to synchronize (barrier) among all processes when using distributed training + """ + if not dist.is_available(): + return + if not dist.is_initialized(): + return + world_size = dist.get_world_size() + if world_size == 1: + return + dist.barrier() + + +def get_world_size() -> int: + if not dist.is_available(): + return 1 + if not dist.is_initialized(): + return 1 + return dist.get_world_size() + + +def get_rank() -> int: + if not dist.is_available(): + return 0 + if not dist.is_initialized(): + return 0 + return dist.get_rank() + + +def get_local_rank() -> int: + """ + Returns: + The rank of the current process within the local (per-machine) process group. + """ + if _LOCAL_PROCESS_GROUP is None: + return get_rank() + + if not dist.is_available(): + return 0 + if not dist.is_initialized(): + return 0 + return dist.get_rank(group=_LOCAL_PROCESS_GROUP) + + +def get_local_size() -> int: + """ + Returns: + The size of the per-machine process group, i.e. the number of processes per machine. + """ + if not dist.is_available(): + return 1 + if not dist.is_initialized(): + return 1 + return dist.get_world_size(group=_LOCAL_PROCESS_GROUP) + + +def is_main_process() -> bool: + return get_rank() == 0 + + +@functools.lru_cache() +def _get_global_gloo_group(): + """ + Return a process group based on gloo backend, containing all the ranks + The result is cached. + """ + if dist.get_backend() == "nccl": + return dist.new_group(backend="gloo") + else: + return dist.group.WORLD + + +def _serialize_to_tensor(data, group): + backend = dist.get_backend(group) + assert backend in ["gloo", "nccl"] + device = torch.device("cpu" if backend == "gloo" else "cuda") + + buffer = pickle.dumps(data) + if len(buffer) > 1024 ** 3: + logger.warning( + "Rank {} trying to all-gather {:.2f} GB of data on device {}".format( + get_rank(), len(buffer) / (1024 ** 3), device + ) + ) + storage = torch.ByteStorage.from_buffer(buffer) + tensor = torch.ByteTensor(storage).to(device=device) + return tensor + + +def _pad_to_largest_tensor(tensor, group): + """ + Returns: + list[int]: size of the tensor, on each rank + Tensor: padded tensor that has the max size + """ + world_size = dist.get_world_size(group=group) + assert ( + world_size >= 1 + ), "comm.gather/all_gather must be called from ranks within the given group!" + local_size = torch.tensor([tensor.numel()], dtype=torch.int64, device=tensor.device) + size_list = [ + torch.zeros([1], dtype=torch.int64, device=tensor.device) + for _ in range(world_size) + ] + dist.all_gather(size_list, local_size, group=group) + size_list = [int(size.item()) for size in size_list] + + max_size = max(size_list) + + # we pad the tensor because torch all_gather does not support + # gathering tensors of different shapes + if local_size != max_size: + padding = torch.zeros( + (max_size - local_size,), dtype=torch.uint8, device=tensor.device + ) + tensor = torch.cat((tensor, padding), dim=0) + return size_list, tensor + + +def all_gather(data, group=None): + """ + Run all_gather on arbitrary picklable data (not necessarily tensors). + + Args: + data: any picklable object + group: a torch process group. By default, will use a group which + contains all ranks on gloo backend. + Returns: + list[data]: list of data gathered from each rank + """ + if get_world_size() == 1: + return [data] + if group is None: + group = _get_global_gloo_group() + if dist.get_world_size(group) == 1: + return [data] + + tensor = _serialize_to_tensor(data, group) + + size_list, tensor = _pad_to_largest_tensor(tensor, group) + max_size = max(size_list) + + # receiving Tensor from all ranks + tensor_list = [ + torch.empty((max_size,), dtype=torch.uint8, device=tensor.device) + for _ in size_list + ] + dist.all_gather(tensor_list, tensor, group=group) + + data_list = [] + for size, tensor in zip(size_list, tensor_list): + buffer = tensor.cpu().numpy().tobytes()[:size] + data_list.append(pickle.loads(buffer)) + + return data_list + + +def gather(data, dst=0, group=None): + """ + Run gather on arbitrary picklable data (not necessarily tensors). + + Args: + data: any picklable object + dst (int): destination rank + group: a torch process group. By default, will use a group which + contains all ranks on gloo backend. + + Returns: + list[data]: on dst, a list of data gathered from each rank. Otherwise, + an empty list. + """ + if get_world_size() == 1: + return [data] + if group is None: + group = _get_global_gloo_group() + if dist.get_world_size(group=group) == 1: + return [data] + rank = dist.get_rank(group=group) + + tensor = _serialize_to_tensor(data, group) + size_list, tensor = _pad_to_largest_tensor(tensor, group) + + # receiving Tensor from all ranks + if rank == dst: + max_size = max(size_list) + tensor_list = [ + torch.empty((max_size,), dtype=torch.uint8, device=tensor.device) + for _ in size_list + ] + dist.gather(tensor, tensor_list, dst=dst, group=group) + + data_list = [] + for size, tensor in zip(size_list, tensor_list): + buffer = tensor.cpu().numpy().tobytes()[:size] + data_list.append(pickle.loads(buffer)) + return data_list + else: + dist.gather(tensor, [], dst=dst, group=group) + return [] + + +def shared_random_seed(): + """ + Returns: + int: a random number that is the same across all workers. + If workers need a shared RNG, they can use this shared seed to + create one. + All workers must call this function, otherwise it will deadlock. + """ + ints = np.random.randint(2 ** 31) + all_ints = all_gather(ints) + return all_ints[0] + + +def time_synchronized(): + """pytorch-accurate time""" + if torch.cuda.is_available(): + torch.cuda.synchronize() + return time.time() diff --git a/new_impl/cv/dnns/yolov3/utils/ema.py b/new_impl/cv/dnns/yolov3/utils/ema.py new file mode 100644 index 0000000000000000000000000000000000000000..73acbca6796d3cdd07397e657167acdbd5a57647 --- /dev/null +++ b/new_impl/cv/dnns/yolov3/utils/ema.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +# Copyright (c) Megvii Inc. All rights reserved. +import math +from copy import deepcopy + +import torch +import torch.nn as nn + +__all__ = ["ModelEMA", "is_parallel"] + + +def is_parallel(model): + """check if model is in parallel mode.""" + parallel_type = ( + nn.parallel.DataParallel, + nn.parallel.DistributedDataParallel, + ) + return isinstance(model, parallel_type) + + +class ModelEMA: + """ + Model Exponential Moving Average from https://github.com/rwightman/pytorch-image-models + Keep a moving average of everything in the model state_dict (parameters and buffers). + This is intended to allow functionality like + https://www.tensorflow.org/api_docs/python/tf/train/ExponentialMovingAverage + A smoothed version of the weights is necessary for some training schemes to perform well. + This class is sensitive where it is initialized in the sequence of model init, + GPU assignment and distributed training wrappers. + """ + + def __init__(self, model, decay=0.9999, updates=0): + """ + Args: + model (nn.Module): model to apply EMA. + decay (float): ema decay reate. + updates (int): counter of EMA updates. + """ + # Create EMA(FP32) + self.ema = deepcopy(model.module if is_parallel(model) else model).eval() + self.updates = updates + # decay exponential ramp (to help early epochs) + self.decay = lambda x: decay * (1 - math.exp(-x / 2000)) + for p in self.ema.parameters(): + p.requires_grad_(False) + + def update(self, model): + # Update EMA parameters + with torch.no_grad(): + self.updates += 1 + d = self.decay(self.updates) + + msd = ( + model.module.state_dict() if is_parallel(model) else model.state_dict() + ) # model state_dict + for k, v in self.ema.state_dict().items(): + if v.dtype.is_floating_point: + v *= d + v += (1.0 - d) * msd[k].detach() diff --git a/new_impl/cv/dnns/yolov3/utils/logger.py b/new_impl/cv/dnns/yolov3/utils/logger.py new file mode 100644 index 0000000000000000000000000000000000000000..f426e64164915e2708cce2ed6eaee79e9f54109f --- /dev/null +++ b/new_impl/cv/dnns/yolov3/utils/logger.py @@ -0,0 +1,217 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +# Copyright (c) Megvii Inc. All rights reserved. + +import inspect +import os +import sys +from loguru import logger + +import torch + + +def get_caller_name(depth=0): + """ + Args: + depth (int): Depth of caller conext, use 0 for caller depth. + Default value: 0. + + Returns: + str: module name of the caller + """ + # the following logic is a little bit faster than inspect.stack() logic + frame = inspect.currentframe().f_back + for _ in range(depth): + frame = frame.f_back + + return frame.f_globals["__name__"] + + +class StreamToLoguru: + """ + stream object that redirects writes to a logger instance. + """ + + def __init__(self, level="INFO", caller_names=("apex", "pycocotools")): + """ + Args: + level(str): log level string of loguru. Default value: "INFO". + caller_names(tuple): caller names of redirected module. + Default value: (apex, pycocotools). + """ + self.level = level + self.linebuf = "" + self.caller_names = caller_names + + def write(self, buf): + full_name = get_caller_name(depth=1) + module_name = full_name.rsplit(".", maxsplit=-1)[0] + if module_name in self.caller_names: + for line in buf.rstrip().splitlines(): + # use caller level log + logger.opt(depth=2).log(self.level, line.rstrip()) + else: + sys.__stdout__.write(buf) + + def flush(self): + pass + + +def redirect_sys_output(log_level="INFO"): + redirect_logger = StreamToLoguru(log_level) + sys.stderr = redirect_logger + sys.stdout = redirect_logger + + +def setup_logger(save_dir, distributed_rank=0, filename="log.txt", mode="a"): + """setup logger for training and testing. + Args: + save_dir(str): location to save log file + distributed_rank(int): device rank when multi-gpu environment + filename (string): log save name. + mode(str): log file write mode, `append` or `override`. default is `a`. + + Return: + logger instance. + """ + loguru_format = ( + "{time:YYYY-MM-DD HH:mm:ss} | " + "{level: <8} | " + "{name}:{line} - {message}" + ) + + logger.remove() + save_file = os.path.join(save_dir, filename) + if mode == "o" and os.path.exists(save_file): + os.remove(save_file) + # only keep logger in rank0 process + if distributed_rank == 0: + logger.add( + sys.stderr, + format=loguru_format, + level="INFO", + enqueue=True, + ) + logger.add(save_file) + + # redirect stdout/stderr to loguru + redirect_sys_output("INFO") + + +class WandbLogger(object): + """ + Log training runs, datasets, models, and predictions to Weights & Biases. + This logger sends information to W&B at wandb.ai. + By default, this information includes hyperparameters, + system configuration and metrics, model metrics, + and basic data metrics and analyses. + + For more information, please refer to: + https://docs.wandb.ai/guides/track + """ + def __init__(self, + project=None, + name=None, + id=None, + entity=None, + save_dir=None, + config=None, + **kwargs): + """ + Args: + project (str): wandb project name. + name (str): wandb run name. + id (str): wandb run id. + entity (str): wandb entity name. + save_dir (str): save directory. + config (dict): config dict. + **kwargs: other kwargs. + """ + try: + import wandb + self.wandb = wandb + except ModuleNotFoundError: + raise ModuleNotFoundError( + "wandb is not installed." + "Please install wandb using pip install wandb" + ) + + self.project = project + self.name = name + self.id = id + self.save_dir = save_dir + self.config = config + self.kwargs = kwargs + self.entity = entity + self._run = None + self._wandb_init = dict( + project=self.project, + name=self.name, + id=self.id, + entity=self.entity, + dir=self.save_dir, + resume="allow" + ) + self._wandb_init.update(**kwargs) + + _ = self.run + + if self.config: + self.run.config.update(self.config) + self.run.define_metric("epoch") + self.run.define_metric("val/", step_metric="epoch") + + @property + def run(self): + if self._run is None: + if self.wandb.run is not None: + logger.info( + "There is a wandb run already in progress " + "and newly created instances of `WandbLogger` will reuse" + " this run. If this is not desired, call `wandb.finish()`" + "before instantiating `WandbLogger`." + ) + self._run = self.wandb.run + else: + self._run = self.wandb.init(**self._wandb_init) + return self._run + + def log_metrics(self, metrics, step=None): + """ + Args: + metrics (dict): metrics dict. + step (int): step number. + """ + + for k, v in metrics.items(): + if isinstance(v, torch.Tensor): + metrics[k] = v.item() + + if step is not None: + self.run.log(metrics, step=step) + else: + self.run.log(metrics) + + def save_checkpoint(self, save_dir, model_name, is_best): + """ + Args: + save_dir (str): save directory. + model_name (str): model name. + is_best (bool): whether the model is the best model. + """ + filename = os.path.join(save_dir, model_name + "_ckpt.pth") + artifact = self.wandb.Artifact( + name=f"model-{self.run.id}", + type="model" + ) + artifact.add_file(filename, name="model_ckpt.pth") + + aliases = ["latest"] + + if is_best: + aliases.append("best") + + self.run.log_artifact(artifact, aliases=aliases) + + def finish(self): + self.run.finish() diff --git a/new_impl/cv/dnns/yolov3/utils/lr_scheduler.py b/new_impl/cv/dnns/yolov3/utils/lr_scheduler.py new file mode 100644 index 0000000000000000000000000000000000000000..2a1513c22abb5bbcb65447f6ee4bbadebfa9d43f --- /dev/null +++ b/new_impl/cv/dnns/yolov3/utils/lr_scheduler.py @@ -0,0 +1,205 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +# Copyright (c) Megvii Inc. All rights reserved. + +import math +from functools import partial + + +class LRScheduler: + def __init__(self, name, lr, iters_per_epoch, total_epochs, **kwargs): + """ + Supported lr schedulers: [cos, warmcos, multistep] + + Args: + lr (float): learning rate. + iters_per_peoch (int): number of iterations in one epoch. + total_epochs (int): number of epochs in training. + kwargs (dict): + - cos: None + - warmcos: [warmup_epochs, warmup_lr_start (default 1e-6)] + - multistep: [milestones (epochs), gamma (default 0.1)] + """ + + self.lr = lr + self.iters_per_epoch = iters_per_epoch + self.total_epochs = total_epochs + self.total_iters = iters_per_epoch * total_epochs + + self.__dict__.update(kwargs) + + self.lr_func = self._get_lr_func(name) + + def update_lr(self, iters): + return self.lr_func(iters) + + def _get_lr_func(self, name): + if name == "cos": # cosine lr schedule + lr_func = partial(cos_lr, self.lr, self.total_iters) + elif name == "warmcos": + warmup_total_iters = self.iters_per_epoch * self.warmup_epochs + warmup_lr_start = getattr(self, "warmup_lr_start", 1e-6) + lr_func = partial( + warm_cos_lr, + self.lr, + self.total_iters, + warmup_total_iters, + warmup_lr_start, + ) + elif name == "yoloxwarmcos": + warmup_total_iters = self.iters_per_epoch * self.warmup_epochs + no_aug_iters = self.iters_per_epoch * self.no_aug_epochs + warmup_lr_start = getattr(self, "warmup_lr_start", 0) + min_lr_ratio = getattr(self, "min_lr_ratio", 0.2) + lr_func = partial( + yolox_warm_cos_lr, + self.lr, + min_lr_ratio, + self.total_iters, + warmup_total_iters, + warmup_lr_start, + no_aug_iters, + ) + elif name == "yoloxsemiwarmcos": + warmup_lr_start = getattr(self, "warmup_lr_start", 0) + min_lr_ratio = getattr(self, "min_lr_ratio", 0.2) + warmup_total_iters = self.iters_per_epoch * self.warmup_epochs + no_aug_iters = self.iters_per_epoch * self.no_aug_epochs + normal_iters = self.iters_per_epoch * self.semi_epoch + semi_iters = self.iters_per_epoch_semi * ( + self.total_epochs - self.semi_epoch - self.no_aug_epochs + ) + lr_func = partial( + yolox_semi_warm_cos_lr, + self.lr, + min_lr_ratio, + warmup_lr_start, + self.total_iters, + normal_iters, + no_aug_iters, + warmup_total_iters, + semi_iters, + self.iters_per_epoch, + self.iters_per_epoch_semi, + ) + elif name == "multistep": # stepwise lr schedule + milestones = [ + int(self.total_iters * milestone / self.total_epochs) + for milestone in self.milestones + ] + gamma = getattr(self, "gamma", 0.1) + lr_func = partial(multistep_lr, self.lr, milestones, gamma) + else: + raise ValueError("Scheduler version {} not supported.".format(name)) + return lr_func + + +def cos_lr(lr, total_iters, iters): + """Cosine learning rate""" + lr *= 0.5 * (1.0 + math.cos(math.pi * iters / total_iters)) + return lr + + +def warm_cos_lr(lr, total_iters, warmup_total_iters, warmup_lr_start, iters): + """Cosine learning rate with warm up.""" + if iters <= warmup_total_iters: + lr = (lr - warmup_lr_start) * iters / float( + warmup_total_iters + ) + warmup_lr_start + else: + lr *= 0.5 * ( + 1.0 + + math.cos( + math.pi + * (iters - warmup_total_iters) + / (total_iters - warmup_total_iters) + ) + ) + return lr + + +def yolox_warm_cos_lr( + lr, + min_lr_ratio, + total_iters, + warmup_total_iters, + warmup_lr_start, + no_aug_iter, + iters, +): + """Cosine learning rate with warm up.""" + min_lr = lr * min_lr_ratio + if iters <= warmup_total_iters: + # lr = (lr - warmup_lr_start) * iters / float(warmup_total_iters) + warmup_lr_start + lr = (lr - warmup_lr_start) * pow( + iters / float(warmup_total_iters), 2 + ) + warmup_lr_start + elif iters >= total_iters - no_aug_iter: + lr = min_lr + else: + lr = min_lr + 0.5 * (lr - min_lr) * ( + 1.0 + + math.cos( + math.pi + * (iters - warmup_total_iters) + / (total_iters - warmup_total_iters - no_aug_iter) + ) + ) + return lr + + +def yolox_semi_warm_cos_lr( + lr, + min_lr_ratio, + warmup_lr_start, + total_iters, + normal_iters, + no_aug_iters, + warmup_total_iters, + semi_iters, + iters_per_epoch, + iters_per_epoch_semi, + iters, +): + """Cosine learning rate with warm up.""" + min_lr = lr * min_lr_ratio + if iters <= warmup_total_iters: + # lr = (lr - warmup_lr_start) * iters / float(warmup_total_iters) + warmup_lr_start + lr = (lr - warmup_lr_start) * pow( + iters / float(warmup_total_iters), 2 + ) + warmup_lr_start + elif iters >= normal_iters + semi_iters: + lr = min_lr + elif iters <= normal_iters: + lr = min_lr + 0.5 * (lr - min_lr) * ( + 1.0 + + math.cos( + math.pi + * (iters - warmup_total_iters) + / (total_iters - warmup_total_iters - no_aug_iters) + ) + ) + else: + lr = min_lr + 0.5 * (lr - min_lr) * ( + 1.0 + + math.cos( + math.pi + * ( + normal_iters + - warmup_total_iters + + (iters - normal_iters) + * iters_per_epoch + * 1.0 + / iters_per_epoch_semi + ) + / (total_iters - warmup_total_iters - no_aug_iters) + ) + ) + return lr + + +def multistep_lr(lr, milestones, gamma, iters): + """MultiStep learning rate""" + for milestone in milestones: + lr *= gamma if iters >= milestone else 1.0 + return lr diff --git a/new_impl/cv/dnns/yolov3/utils/metric.py b/new_impl/cv/dnns/yolov3/utils/metric.py new file mode 100644 index 0000000000000000000000000000000000000000..10f0e631f9996bb50ac72539b7a6dc91d9560932 --- /dev/null +++ b/new_impl/cv/dnns/yolov3/utils/metric.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# Copyright (c) Megvii Inc. All rights reserved. +import functools +import os +import time +from collections import defaultdict, deque + +import numpy as np + +import torch + +__all__ = [ + "AverageMeter", + "MeterBuffer", + "get_total_and_free_memory_in_Mb", + "occupy_mem", + "gpu_mem_usage", +] + + +def get_total_and_free_memory_in_Mb(cuda_device): + devices_info_str = os.popen( + "nvidia-smi --query-gpu=memory.total,memory.used --format=csv,nounits,noheader" + ) + devices_info = devices_info_str.read().strip().split("\n") + if "CUDA_VISIBLE_DEVICES" in os.environ: + visible_devices = os.environ["CUDA_VISIBLE_DEVICES"].split(',') + cuda_device = int(visible_devices[cuda_device]) + total, used = devices_info[int(cuda_device)].split(",") + return int(total), int(used) + + +def occupy_mem(cuda_device, mem_ratio=0.9): + """ + pre-allocate gpu memory for training to avoid memory Fragmentation. + """ + total, used = get_total_and_free_memory_in_Mb(cuda_device) + max_mem = int(total * mem_ratio) + block_mem = max_mem - used + x = torch.cuda.FloatTensor(256, 1024, block_mem) + del x + time.sleep(5) + + +def gpu_mem_usage(): + """ + Compute the GPU memory usage for the current device (MB). + """ + mem_usage_bytes = torch.cuda.max_memory_allocated() + return mem_usage_bytes / (1024 * 1024) + + +class AverageMeter: + """Track a series of values and provide access to smoothed values over a + window or the global series average. + """ + + def __init__(self, window_size=50): + self._deque = deque(maxlen=window_size) + self._total = 0.0 + self._count = 0 + + def update(self, value): + self._deque.append(value) + self._count += 1 + self._total += value + + @property + def median(self): + d = np.array(list(self._deque)) + return np.median(d) + + @property + def avg(self): + # if deque is empty, nan will be returned. + d = np.array(list(self._deque)) + return d.mean() + + @property + def global_avg(self): + return self._total / max(self._count, 1e-5) + + @property + def latest(self): + return self._deque[-1] if len(self._deque) > 0 else None + + @property + def total(self): + return self._total + + def reset(self): + self._deque.clear() + self._total = 0.0 + self._count = 0 + + def clear(self): + self._deque.clear() + + +class MeterBuffer(defaultdict): + """Computes and stores the average and current value""" + + def __init__(self, window_size=20): + factory = functools.partial(AverageMeter, window_size=window_size) + super().__init__(factory) + + def reset(self): + for v in self.values(): + v.reset() + + def get_filtered_meter(self, filter_key="time"): + return {k: v for k, v in self.items() if filter_key in k} + + def update(self, values=None, **kwargs): + if values is None: + values = {} + values.update(kwargs) + for k, v in values.items(): + if isinstance(v, torch.Tensor): + v = v.detach() + self[k].update(v) + + def clear_meters(self): + for v in self.values(): + v.clear() diff --git a/new_impl/cv/dnns/yolov3/utils/model_utils.py b/new_impl/cv/dnns/yolov3/utils/model_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..4b9b99291bc252d86dd9abb35519534ea9d10faa --- /dev/null +++ b/new_impl/cv/dnns/yolov3/utils/model_utils.py @@ -0,0 +1,185 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +# Copyright (c) Megvii Inc. All rights reserved. + +import contextlib +from copy import deepcopy +from typing import Sequence + +import torch +import torch.nn as nn +from thop import profile + +__all__ = [ + "fuse_conv_and_bn", + "fuse_model", + "get_model_info", + "replace_module", + "freeze_module", + "adjust_status", +] + + +def get_model_info(model: nn.Module, tsize: Sequence[int]) -> str: + stride = 64 + img = torch.zeros((1, 3, stride, stride), device=next(model.parameters()).device) + flops, params = profile(deepcopy(model), inputs=(img,), verbose=False) + params /= 1e6 + flops /= 1e9 + flops *= tsize[0] * tsize[1] / stride / stride * 2 # Gflops + info = "Params: {:.2f}M, Gflops: {:.2f}".format(params, flops) + return info + + +def fuse_conv_and_bn(conv: nn.Conv2d, bn: nn.BatchNorm2d) -> nn.Conv2d: + """ + Fuse convolution and batchnorm layers. + check more info on https://tehnokv.com/posts/fusing-batchnorm-and-conv/ + + Args: + conv (nn.Conv2d): convolution to fuse. + bn (nn.BatchNorm2d): batchnorm to fuse. + + Returns: + nn.Conv2d: fused convolution behaves the same as the input conv and bn. + """ + fusedconv = ( + nn.Conv2d( + conv.in_channels, + conv.out_channels, + kernel_size=conv.kernel_size, + stride=conv.stride, + padding=conv.padding, + groups=conv.groups, + bias=True, + ) + .requires_grad_(False) + .to(conv.weight.device) + ) + + # prepare filters + w_conv = conv.weight.clone().view(conv.out_channels, -1) + w_bn = torch.diag(bn.weight.div(torch.sqrt(bn.eps + bn.running_var))) + fusedconv.weight.copy_(torch.mm(w_bn, w_conv).view(fusedconv.weight.shape)) + + # prepare spatial bias + b_conv = ( + torch.zeros(conv.weight.size(0), device=conv.weight.device) + if conv.bias is None + else conv.bias + ) + b_bn = bn.bias - bn.weight.mul(bn.running_mean).div( + torch.sqrt(bn.running_var + bn.eps) + ) + fusedconv.bias.copy_(torch.mm(w_bn, b_conv.reshape(-1, 1)).reshape(-1) + b_bn) + + return fusedconv + + +def fuse_model(model: nn.Module) -> nn.Module: + """fuse conv and bn in model + + Args: + model (nn.Module): model to fuse + + Returns: + nn.Module: fused model + """ + from yolox.models.network_blocks import BaseConv + + for m in model.modules(): + if type(m) is BaseConv and hasattr(m, "bn"): + m.conv = fuse_conv_and_bn(m.conv, m.bn) # update conv + delattr(m, "bn") # remove batchnorm + m.forward = m.fuseforward # update forward + return model + + +def replace_module(module, replaced_module_type, new_module_type, replace_func=None) -> nn.Module: + """ + Replace given type in module to a new type. mostly used in deploy. + + Args: + module (nn.Module): model to apply replace operation. + replaced_module_type (Type): module type to be replaced. + new_module_type (Type) + replace_func (function): python function to describe replace logic. Defalut value None. + + Returns: + model (nn.Module): module that already been replaced. + """ + + def default_replace_func(replaced_module_type, new_module_type): + return new_module_type() + + if replace_func is None: + replace_func = default_replace_func + + model = module + if isinstance(module, replaced_module_type): + model = replace_func(replaced_module_type, new_module_type) + else: # recurrsively replace + for name, child in module.named_children(): + new_child = replace_module(child, replaced_module_type, new_module_type) + if new_child is not child: # child is already replaced + model.add_module(name, new_child) + + return model + + +def freeze_module(module: nn.Module, name=None) -> nn.Module: + """freeze module inplace + + Args: + module (nn.Module): module to freeze. + name (str, optional): name to freeze. If not given, freeze the whole module. + Note that fuzzy match is not supported. Defaults to None. + + Examples: + freeze the backbone of model + >>> freeze_moudle(model.backbone) + + or freeze the backbone of model by name + >>> freeze_moudle(model, name="backbone") + """ + for param_name, parameter in module.named_parameters(): + if name is None or name in param_name: + parameter.requires_grad = False + + # ensure module like BN and dropout are freezed + for module_name, sub_module in module.named_modules(): + # actually there are no needs to call eval for every single sub_module + if name is None or name in module_name: + sub_module.eval() + + return module + + +@contextlib.contextmanager +def adjust_status(module: nn.Module, training: bool = False) -> nn.Module: + """Adjust module to training/eval mode temporarily. + + Args: + module (nn.Module): module to adjust status. + training (bool): training mode to set. True for train mode, False fro eval mode. + + Examples: + >>> with adjust_status(model, training=False): + ... model(data) + """ + status = {} + + def backup_status(module): + for m in module.modules(): + # save prev status to dict + status[m] = m.training + m.training = training + + def recover_status(module): + for m in module.modules(): + # recover prev status from dict + m.training = status.pop(m) + + backup_status(module) + yield module + recover_status(module) diff --git a/new_impl/cv/dnns/yolov3/utils/setup_env.py b/new_impl/cv/dnns/yolov3/utils/setup_env.py new file mode 100644 index 0000000000000000000000000000000000000000..45289f3245f09e48395ad419d17efffe6846b05c --- /dev/null +++ b/new_impl/cv/dnns/yolov3/utils/setup_env.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +# Copyright (c) Megvii Inc. All rights reserved. + +import os +import subprocess +from loguru import logger + +import cv2 + +from .dist import get_world_size, is_main_process + +__all__ = ["configure_nccl", "configure_module", "configure_omp"] + + +def configure_nccl(): + """Configure multi-machine environment variables of NCCL.""" + os.environ["NCCL_LAUNCH_MODE"] = "PARALLEL" + os.environ["NCCL_IB_HCA"] = subprocess.getoutput( + "pushd /sys/class/infiniband/ > /dev/null; for i in mlx5_*; " + "do cat $i/ports/1/gid_attrs/types/* 2>/dev/null " + "| grep v >/dev/null && echo $i ; done; popd > /dev/null" + ) + os.environ["NCCL_IB_GID_INDEX"] = "3" + os.environ["NCCL_IB_TC"] = "106" + + +def configure_omp(num_threads=1): + """ + If OMP_NUM_THREADS is not configured and world_size is greater than 1, + Configure OMP_NUM_THREADS environment variables of NCCL to `num_thread`. + + Args: + num_threads (int): value of `OMP_NUM_THREADS` to set. + """ + # We set OMP_NUM_THREADS=1 by default, which achieves the best speed on our machines + # feel free to change it for better performance. + if "OMP_NUM_THREADS" not in os.environ and get_world_size() > 1: + os.environ["OMP_NUM_THREADS"] = str(num_threads) + if is_main_process(): + logger.info( + "\n***************************************************************\n" + "We set `OMP_NUM_THREADS` for each process to {} to speed up.\n" + "please further tune the variable for optimal performance.\n" + "***************************************************************".format( + os.environ["OMP_NUM_THREADS"] + ) + ) + + +def configure_module(ulimit_value=8192): + """ + Configure pytorch module environment. setting of ulimit and cv2 will be set. + + Args: + ulimit_value(int): default open file number on linux. Default value: 8192. + """ + # system setting + try: + import resource + + rlimit = resource.getrlimit(resource.RLIMIT_NOFILE) + resource.setrlimit(resource.RLIMIT_NOFILE, (ulimit_value, rlimit[1])) + except Exception: + # Exception might be raised in Windows OS or rlimit reaches max limit number. + # However, set rlimit value might not be necessary. + pass + + # cv2 + # multiprocess might be harmful on performance of torch dataloader + os.environ["OPENCV_OPENCL_RUNTIME"] = "disabled" + try: + cv2.setNumThreads(0) + cv2.ocl.setUseOpenCL(False) + except Exception: + # cv2 version mismatch might rasie exceptions. + pass diff --git a/new_impl/cv/dnns/yolov3/utils/visualize.py b/new_impl/cv/dnns/yolov3/utils/visualize.py new file mode 100644 index 0000000000000000000000000000000000000000..e714a3ee73699141fb4cd8d131d541a6e6625ed6 --- /dev/null +++ b/new_impl/cv/dnns/yolov3/utils/visualize.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +# Copyright (c) Megvii Inc. All rights reserved. + +import cv2 +import numpy as np + +__all__ = ["vis"] + + +def vis(img, boxes, scores, cls_ids, conf=0.5, class_names=None): + + for i in range(len(boxes)): + box = boxes[i] + cls_id = int(cls_ids[i]) + score = scores[i] + if score < conf: + continue + x0 = int(box[0]) + y0 = int(box[1]) + x1 = int(box[2]) + y1 = int(box[3]) + + color = (_COLORS[cls_id] * 255).astype(np.uint8).tolist() + text = '{}:{:.1f}%'.format(class_names[cls_id], score * 100) + txt_color = (0, 0, 0) if np.mean(_COLORS[cls_id]) > 0.5 else (255, 255, 255) + font = cv2.FONT_HERSHEY_SIMPLEX + + txt_size = cv2.getTextSize(text, font, 0.4, 1)[0] + cv2.rectangle(img, (x0, y0), (x1, y1), color, 2) + + txt_bk_color = (_COLORS[cls_id] * 255 * 0.7).astype(np.uint8).tolist() + cv2.rectangle( + img, + (x0, y0 + 1), + (x0 + txt_size[0] + 1, y0 + int(1.5*txt_size[1])), + txt_bk_color, + -1 + ) + cv2.putText(img, text, (x0, y0 + txt_size[1]), font, 0.4, txt_color, thickness=1) + + return img + + +_COLORS = np.array( + [ + 0.000, 0.447, 0.741, + 0.850, 0.325, 0.098, + 0.929, 0.694, 0.125, + 0.494, 0.184, 0.556, + 0.466, 0.674, 0.188, + 0.301, 0.745, 0.933, + 0.635, 0.078, 0.184, + 0.300, 0.300, 0.300, + 0.600, 0.600, 0.600, + 1.000, 0.000, 0.000, + 1.000, 0.500, 0.000, + 0.749, 0.749, 0.000, + 0.000, 1.000, 0.000, + 0.000, 0.000, 1.000, + 0.667, 0.000, 1.000, + 0.333, 0.333, 0.000, + 0.333, 0.667, 0.000, + 0.333, 1.000, 0.000, + 0.667, 0.333, 0.000, + 0.667, 0.667, 0.000, + 0.667, 1.000, 0.000, + 1.000, 0.333, 0.000, + 1.000, 0.667, 0.000, + 1.000, 1.000, 0.000, + 0.000, 0.333, 0.500, + 0.000, 0.667, 0.500, + 0.000, 1.000, 0.500, + 0.333, 0.000, 0.500, + 0.333, 0.333, 0.500, + 0.333, 0.667, 0.500, + 0.333, 1.000, 0.500, + 0.667, 0.000, 0.500, + 0.667, 0.333, 0.500, + 0.667, 0.667, 0.500, + 0.667, 1.000, 0.500, + 1.000, 0.000, 0.500, + 1.000, 0.333, 0.500, + 1.000, 0.667, 0.500, + 1.000, 1.000, 0.500, + 0.000, 0.333, 1.000, + 0.000, 0.667, 1.000, + 0.000, 1.000, 1.000, + 0.333, 0.000, 1.000, + 0.333, 0.333, 1.000, + 0.333, 0.667, 1.000, + 0.333, 1.000, 1.000, + 0.667, 0.000, 1.000, + 0.667, 0.333, 1.000, + 0.667, 0.667, 1.000, + 0.667, 1.000, 1.000, + 1.000, 0.000, 1.000, + 1.000, 0.333, 1.000, + 1.000, 0.667, 1.000, + 0.333, 0.000, 0.000, + 0.500, 0.000, 0.000, + 0.667, 0.000, 0.000, + 0.833, 0.000, 0.000, + 1.000, 0.000, 0.000, + 0.000, 0.167, 0.000, + 0.000, 0.333, 0.000, + 0.000, 0.500, 0.000, + 0.000, 0.667, 0.000, + 0.000, 0.833, 0.000, + 0.000, 1.000, 0.000, + 0.000, 0.000, 0.167, + 0.000, 0.000, 0.333, + 0.000, 0.000, 0.500, + 0.000, 0.000, 0.667, + 0.000, 0.000, 0.833, + 0.000, 0.000, 1.000, + 0.000, 0.000, 0.000, + 0.143, 0.143, 0.143, + 0.286, 0.286, 0.286, + 0.429, 0.429, 0.429, + 0.571, 0.571, 0.571, + 0.714, 0.714, 0.714, + 0.857, 0.857, 0.857, + 0.000, 0.447, 0.741, + 0.314, 0.717, 0.741, + 0.50, 0.5, 0 + ] +).astype(np.float32).reshape(-1, 3) diff --git a/new_impl/cv/dnns/yolov3/vit_yolov3.py b/new_impl/cv/dnns/yolov3/vit_yolov3.py new file mode 100644 index 0000000000000000000000000000000000000000..755d6251c2bbe7520ae895d02aab8ad07209c022 --- /dev/null +++ b/new_impl/cv/dnns/yolov3/vit_yolov3.py @@ -0,0 +1,470 @@ +import torch +from torch import nn +from timm.models.vision_transformer import VisionTransformer +from functools import partial +from einops import rearrange +from dnns.yolov3.yolo_fpn import YOLOFPN +from dnns.yolov3.head import YOLOXHead +from utils.dl.common.model import set_module, get_module +from types import MethodType +import os +from utils.common.log import logger + + +class VisionTransformerYOLOv3(VisionTransformer): + def forward_head(self, x): + # print(222) + return self.head(x) + + def forward_features(self, x): + # print(111) + return self._intermediate_layers(x, n=[len(self.blocks) // 3 - 1, len(self.blocks) // 3 * 2 - 1, len(self.blocks) - 1]) + + def forward(self, x, targets=None): + features = self.forward_features(x) + return self.head(x, features, targets) + + @staticmethod + def init_from_vit(vit: VisionTransformer): + res = VisionTransformerYOLOv3() + + for attr in dir(vit): + # if str(attr) not in ['forward_head', 'forward_features'] and not attr.startswith('__'): + if isinstance(getattr(vit, attr), nn.Module): + # print(attr) + try: + setattr(res, attr, getattr(vit, attr)) + except Exception as e: + print(attr, str(e)) + + return res + + +class Norm2d(nn.Module): + def __init__(self, embed_dim): + super().__init__() + self.ln = nn.LayerNorm(embed_dim, eps=1e-6) + def forward(self, x): + x = x.permute(0, 2, 3, 1) + x = self.ln(x) + x = x.permute(0, 3, 1, 2).contiguous() + return x + + +class ViTYOLOv3Head(nn.Module): + def __init__(self, im_size, patch_size, patch_dim, num_classes, use_bigger_fpns, cls_vit_ckpt_path, init_head): + super(ViTYOLOv3Head, self).__init__() + + self.im_size = im_size + self.patch_size = patch_size + + # target_patch_dim: [256, 512, 512] + # self.change_patchs_dim = nn.ModuleList([nn.Linear(patch_dim, target_patch_dim) for target_patch_dim in [256, 512, 512]]) + + # # input: (1, target_patch_dim, 14, 14) + # # target feature size: {40, 20, 10} + # self.change_features_size = nn.ModuleList([ + # self.get_change_feature_size(cin, cout, t) for t, cin, cout in zip([40, 20, 10], [256, 512, 512], [256, 512, 512]) + # ]) + + embed_dim = 768 + self.before_fpns = nn.ModuleList([ + # nn.Sequential( + # nn.ConvTranspose2d(embed_dim, embed_dim, kernel_size=2, stride=2), + # nn.GroupNorm(embed_dim), + # nn.GELU(), + # nn.ConvTranspose2d(embed_dim, embed_dim, kernel_size=2, stride=2), + # ), + + nn.Sequential( + nn.ConvTranspose2d(embed_dim, embed_dim, kernel_size=2, stride=2), + ), + + nn.Identity(), + + nn.MaxPool2d(kernel_size=2, stride=2) + ]) + + if use_bigger_fpns == 1: + logger.info('use 421x fpns') + self.before_fpns = nn.ModuleList([ + + nn.Sequential( + nn.ConvTranspose2d(embed_dim, embed_dim, kernel_size=2, stride=2), + Norm2d(embed_dim), + nn.GELU(), + nn.ConvTranspose2d(embed_dim, embed_dim, kernel_size=2, stride=2), + ), + + nn.Sequential( + nn.ConvTranspose2d(embed_dim, embed_dim, kernel_size=2, stride=2), + ), + + nn.Identity(), + + # nn.MaxPool2d(kernel_size=2, stride=2) + ]) + if use_bigger_fpns == -1: + logger.info('use 1/0.5/0.25x fpns') + self.before_fpns = nn.ModuleList([ + + # nn.Sequential( + # nn.ConvTranspose2d(embed_dim, embed_dim, kernel_size=2, stride=2), + # ), + + nn.Identity(), + + nn.MaxPool2d(kernel_size=2, stride=2), + + nn.Sequential(nn.MaxPool2d(kernel_size=2, stride=2), nn.MaxPool2d(kernel_size=2, stride=2)) + ]) + + # self.fpn = YOLOFPN() + self.fpn = nn.Identity() + self.head = YOLOXHead(num_classes, in_channels=[768, 768, 768], act='lrelu') + + if init_head: + logger.info('init head') + self.load_pretrained_weight(cls_vit_ckpt_path) + else: + logger.info('do not init head') + + def load_pretrained_weight(self, cls_vit_ckpt_path): + ckpt = torch.load(os.path.join(os.path.dirname(__file__), 'yolox_darknet.pth')) + # for k in [f'head.cls_preds.{i}.{j}' for i in [0, 1, 2] for j in ['weight', 'bias']]: + # del ckpt['model'][k] + removed_k = [f'head.cls_preds.{i}.{j}' for i in [0, 1, 2] for j in ['weight', 'bias']] + for k, v in ckpt['model'].items(): + if 'backbone.backbone' in k: + removed_k += [k] + if 'head.stems' in k and 'conv.weight' in k: + removed_k += [k] + for k in removed_k: + del ckpt['model'][k] + # print(ckpt['model'].keys()) + + new_state_dict = {} + for k, v in ckpt['model'].items(): + new_k = k.replace('backbone', 'fpn') + new_state_dict[new_k] = v + + # cls_vit_ckpt = torch.load(cls_vit_ckpt_path) + # for k, v in cls_vit_ckpt['main'].named_parameters(): + # if not 'qkv.abs' not in k: + # continue + + # new_state_dict[k] = v + # logger.info(f'load {k} from cls vit ckpt') + + self.load_state_dict(new_state_dict, strict=False) + + def get_change_feature_size(self, in_channels, out_channels, target_size): + H, W = self.im_size + GS = H // self.patch_size # 14 + + if target_size == GS: + return nn.Identity() + elif target_size < GS: + return nn.AdaptiveMaxPool2d((target_size, target_size)) + else: + return { + 20: nn.Sequential( + nn.ConvTranspose2d(in_channels, out_channels, kernel_size=(3, 3), stride=2, padding=0), + # nn.BatchNorm2d(out_channels), + # nn.ReLU(), + nn.AdaptiveMaxPool2d((target_size, target_size)) + ), + 40: nn.Sequential( + nn.ConvTranspose2d(in_channels, out_channels, kernel_size=(3, 3), stride=3, padding=1), + # nn.BatchNorm2d(out_channels), + # nn.ReLU(), + ) + }[target_size] + + def forward(self, input_images, x, targets=None): + # print(111) + # NOTE: YOLOX backbone (w/o FPN) output, or FPN input: {'dark3': torch.Size([4, 256, 40, 40]), 'dark4': torch.Size([4, 512, 20, 20]), 'dark5': torch.Size([4, 512, 10, 10])} + # NOTE: YOLOXHead input: [torch.Size([4, 128, 40, 40]), torch.Size([4, 256, 20, 20]), torch.Size([4, 512, 10, 10])] + # print(x) + + # print([i.size() for i in x]) + x = [i[:, 1:] for i in x] + x = [i.permute(0, 2, 1).reshape(input_images.size(0), -1, 14, 14) for i in x] # 14 is hardcode, obtained from timm.layers.patch_embed.py + # print([i.size() for i in x]) + # exit() + + # NOTE: old + # x[0]: torch.Size([1, 196, 768]) + # H, W = self.im_size + # GS = H // self.patch_size # 14 + # xs = [cpd(x) for x, cpd in zip(xs, self.change_patchs_dim)] # (1, 196, target_patch_dim) + # xs = [rearrange(x, "b (h w) c -> b c h w", h=GS) for x in xs] # (1, target_patch_dim, 14, 14) + + # xs = [cfs(x) for x, cfs in zip(xs, self.change_features_size)] + # print([i.size() for i in xs]) + # ---------------- + + xs = [before_fpn(x[-1]) for i, before_fpn in zip(x, self.before_fpns)] + + # print([i.size() for i in xs]) + # exit() + # [torch.Size([1, 768, 28, 28]), torch.Size([1, 768, 14, 14]), torch.Size([1, 768, 7, 7])] + + + xs = self.fpn(xs) + # print('before head', [i.size() for i in xs]) + xs = tuple(xs) + + if targets is not None: + loss, iou_loss, conf_loss, cls_loss, l1_loss, num_fg = self.head(xs, targets, input_images) + return { + "total_loss": loss, + "iou_loss": iou_loss, + "l1_loss": l1_loss, + "conf_loss": conf_loss, + "cls_loss": cls_loss, + "num_fg": num_fg, + } + + return self.head(xs) + + +class ViTYOLOv3Head2(nn.Module): + def __init__(self, im_size, patch_size, patch_dim, num_classes, use_bigger_fpns): + super(ViTYOLOv3Head2, self).__init__() + + self.im_size = im_size + self.patch_size = patch_size + + # target_patch_dim: [256, 512, 512] + # self.change_patchs_dim = nn.ModuleList([nn.Linear(patch_dim, target_patch_dim) for target_patch_dim in [256, 512, 512]]) + + # # input: (1, target_patch_dim, 14, 14) + # # target feature size: {40, 20, 10} + # self.change_features_size = nn.ModuleList([ + # self.get_change_feature_size(cin, cout, t) for t, cin, cout in zip([40, 20, 10], [256, 512, 512], [256, 512, 512]) + # ]) + + embed_dim = 768 + self.before_fpns = nn.ModuleList([ + # nn.Sequential( + # nn.ConvTranspose2d(embed_dim, embed_dim, kernel_size=2, stride=2), + # nn.GroupNorm(embed_dim), + # nn.GELU(), + # nn.ConvTranspose2d(embed_dim, embed_dim, kernel_size=2, stride=2), + # ), + + nn.Sequential( + nn.ConvTranspose2d(embed_dim, embed_dim, kernel_size=2, stride=2), + ), + + nn.Identity(), + + nn.MaxPool2d(kernel_size=2, stride=2) + ]) + + if use_bigger_fpns: + logger.info('use 8/4/2x fpns') + self.before_fpns = nn.ModuleList([ + + nn.Sequential( + nn.ConvTranspose2d(embed_dim, embed_dim, kernel_size=2, stride=2), + Norm2d(embed_dim), + nn.GELU(), + nn.ConvTranspose2d(embed_dim, embed_dim, kernel_size=2, stride=2), + ), + + nn.Sequential( + nn.ConvTranspose2d(embed_dim, embed_dim, kernel_size=2, stride=2), + ), + + nn.Identity(), + + # nn.MaxPool2d(kernel_size=2, stride=2) + ]) + + # self.fpn = YOLOFPN() + self.fpn = nn.Identity() + self.head = YOLOXHead(num_classes, in_channels=[768, 768, 768], act='lrelu') + + self.load_pretrained_weight() + + def load_pretrained_weight(self): + ckpt = torch.load(os.path.join(os.path.dirname(__file__), 'yolox_darknet.pth')) + # for k in [f'head.cls_preds.{i}.{j}' for i in [0, 1, 2] for j in ['weight', 'bias']]: + # del ckpt['model'][k] + removed_k = [f'head.cls_preds.{i}.{j}' for i in [0, 1, 2] for j in ['weight', 'bias']] + for k, v in ckpt['model'].items(): + if 'backbone.backbone' in k: + removed_k += [k] + if 'head.stems' in k and 'conv.weight' in k: + removed_k += [k] + for k in removed_k: + del ckpt['model'][k] + # print(ckpt['model'].keys()) + + new_state_dict = {} + for k, v in ckpt['model'].items(): + new_k = k.replace('backbone', 'fpn') + new_state_dict[new_k] = v + self.load_state_dict(new_state_dict, strict=False) + + def get_change_feature_size(self, in_channels, out_channels, target_size): + H, W = self.im_size + GS = H // self.patch_size # 14 + + if target_size == GS: + return nn.Identity() + elif target_size < GS: + return nn.AdaptiveMaxPool2d((target_size, target_size)) + else: + return { + 20: nn.Sequential( + nn.ConvTranspose2d(in_channels, out_channels, kernel_size=(3, 3), stride=2, padding=0), + # nn.BatchNorm2d(out_channels), + # nn.ReLU(), + nn.AdaptiveMaxPool2d((target_size, target_size)) + ), + 40: nn.Sequential( + nn.ConvTranspose2d(in_channels, out_channels, kernel_size=(3, 3), stride=3, padding=1), + # nn.BatchNorm2d(out_channels), + # nn.ReLU(), + ) + }[target_size] + + def forward(self, input_images, x, targets=None): + # print(111) + # NOTE: YOLOX backbone (w/o FPN) output, or FPN input: {'dark3': torch.Size([4, 256, 40, 40]), 'dark4': torch.Size([4, 512, 20, 20]), 'dark5': torch.Size([4, 512, 10, 10])} + # NOTE: YOLOXHead input: [torch.Size([4, 128, 40, 40]), torch.Size([4, 256, 20, 20]), torch.Size([4, 512, 10, 10])] + # print(x) + + # print([i.size() for i in x]) + x = [i[:, 1:] for i in x] + x = [i.permute(0, 2, 1).reshape(input_images.size(0), -1, 14, 14) for i in x] # 14 is hardcode, obtained from timm.layers.patch_embed.py + # print([i.size() for i in x]) + # exit() + + # NOTE: old + # x[0]: torch.Size([1, 196, 768]) + # H, W = self.im_size + # GS = H // self.patch_size # 14 + # xs = [cpd(x) for x, cpd in zip(xs, self.change_patchs_dim)] # (1, 196, target_patch_dim) + # xs = [rearrange(x, "b (h w) c -> b c h w", h=GS) for x in xs] # (1, target_patch_dim, 14, 14) + + # xs = [cfs(x) for x, cfs in zip(xs, self.change_features_size)] + # print([i.size() for i in xs]) + # ---------------- + + xs = [before_fpn(i) for i, before_fpn in zip(x, self.before_fpns)] + + # print([i.size() for i in xs]) + # exit() + # [torch.Size([1, 768, 28, 28]), torch.Size([1, 768, 14, 14]), torch.Size([1, 768, 7, 7])] + + + xs = self.fpn(xs) + # print('before head', [i.size() for i in xs]) + xs = tuple(xs) + + if targets is not None: + loss, iou_loss, conf_loss, cls_loss, l1_loss, num_fg = self.head(xs, targets, input_images) + return { + "total_loss": loss, + "iou_loss": iou_loss, + "l1_loss": l1_loss, + "conf_loss": conf_loss, + "cls_loss": cls_loss, + "num_fg": num_fg, + } + + return self.head(xs) + + +def _forward_head(self, x): + return self.head(x) + + +# def ensure_forward_head_obj_repoint(self): +# self.forward_head = MethodType(_forward_head, self) + + +@torch.no_grad() +def make_vit_yolov3(vit: VisionTransformer, samples: torch.Tensor, patch_size, patch_dim, num_classes, + use_bigger_fpns=False, use_multi_layer_feature=False, cls_vit_ckpt_path=None, init_head=False): + + assert cls_vit_ckpt_path is None + + # vit -> fpn -> head + + # modify vit.forward() to make it output middle features + # vit.forward_features = partial(vit._intermediate_layers, + # n=[len(vit.blocks) // 3 - 1, len(vit.blocks) // 3 * 2 - 1, len(vit.blocks) - 1]) + # vit.forward_head = _forward_head + # vit.__deepcopy__ = MethodType(ensure_forward_head_obj_repoint, vit) + + vit = VisionTransformerYOLOv3.init_from_vit(vit) + + if not use_multi_layer_feature: + set_module(vit, 'head', ViTYOLOv3Head( + im_size=(samples.size(2), samples.size(3)), + patch_size=patch_size, + patch_dim=patch_dim, + num_classes=num_classes, + use_bigger_fpns=use_bigger_fpns, + cls_vit_ckpt_path=cls_vit_ckpt_path, + init_head=init_head + )) + + + else: + raise NotImplementedError + logger.info('use multi layer feature') + set_module(vit, 'head', ViTYOLOv3Head2( + im_size=(samples.size(2), samples.size(3)), + patch_size=patch_size, + patch_dim=patch_dim, + num_classes=num_classes, + use_bigger_fpns=use_bigger_fpns, + cls_vit_ckpt_path=cls_vit_ckpt_path + )) + + # print(vit) + + vit.eval() + output = vit(samples) + # print([oo.size() for oo in output]) + assert len(output) == samples.size(0) and output[0].size(1) == num_classes + 5, f'{[oo.size() for oo in output]}, {num_classes}' + + return vit + + +if __name__ == '__main__': + from dnns.vit import vit_b_16 + vit_b_16 = vit_b_16() + make_vit_yolov3(vit_b_16, torch.rand((1, 3, 224, 224)), 16, 768, 20) + exit() + + from types import MethodType + + class Student(object): + pass + + def set_name(self, name): + self.name = name + + def get_name(self): + print(self.name) + + s1 = Student() + #将方法绑定到s1和s2实例中 + s1.set_name = MethodType(set_name, s1) + s1.get_name = MethodType(get_name, s1) + s1.set_name('s1') + + from copy import deepcopy + s2 = deepcopy(s1) + s2.get_name() + + s2.set_name('s2') + s1.get_name() + s2.get_name() \ No newline at end of file diff --git a/new_impl/cv/dnns/yolov3/yolo_fpn.py b/new_impl/cv/dnns/yolov3/yolo_fpn.py new file mode 100644 index 0000000000000000000000000000000000000000..1aa786704b0844bcf90380aaebed20689ed0c02f --- /dev/null +++ b/new_impl/cv/dnns/yolov3/yolo_fpn.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +# Copyright (c) Megvii Inc. All rights reserved. + +import torch +import torch.nn as nn + +from .network_blocks import BaseConv + + +class YOLOFPN(nn.Module): + """ + YOLOFPN module. Darknet 53 is the default backbone of this model. + """ + + def __init__( + self + ): + super().__init__() + + # self.backbone = Darknet(depth) + # self.in_features = in_features + + # out 1 + self.out1_cbl = self._make_cbl(512, 256, 1) + self.out1 = self._make_embedding([256, 512], 512 + 256) + + # out 2 + self.out2_cbl = self._make_cbl(256, 128, 1) + self.out2 = self._make_embedding([128, 256], 256 + 128) + + # upsample + self.upsample = nn.Upsample(scale_factor=2, mode="nearest") + + def _make_cbl(self, _in, _out, ks): + return BaseConv(_in, _out, ks, stride=1, act="lrelu") + + def _make_embedding(self, filters_list, in_filters): + m = nn.Sequential( + *[ + self._make_cbl(in_filters, filters_list[0], 1), + self._make_cbl(filters_list[0], filters_list[1], 3), + self._make_cbl(filters_list[1], filters_list[0], 1), + self._make_cbl(filters_list[0], filters_list[1], 3), + self._make_cbl(filters_list[1], filters_list[0], 1), + ] + ) + return m + + def load_pretrained_model(self, filename="./weights/darknet53.mix.pth"): + with open(filename, "rb") as f: + state_dict = torch.load(f, map_location="cpu") + print("loading pretrained weights...") + self.backbone.load_state_dict(state_dict) + + def forward(self, backbone_out_features): + """ + Args: + inputs (Tensor): input image. + + Returns: + Tuple[Tensor]: FPN output features.. + """ + # backbone + # out_features = self.backbone(inputs) + + out_features = backbone_out_features + x2, x1, x0 = out_features + + # yolo branch 1 + x1_in = self.out1_cbl(x0) + x1_in = self.upsample(x1_in) + x1_in = torch.cat([x1_in, x1], 1) + out_dark4 = self.out1(x1_in) + + # yolo branch 2 + x2_in = self.out2_cbl(out_dark4) + x2_in = self.upsample(x2_in) + x2_in = torch.cat([x2_in, x2], 1) + out_dark3 = self.out2(x2_in) + + outputs = (out_dark3, out_dark4, x0) + return outputs \ No newline at end of file diff --git a/new_impl/cv/elasticdnn/api/__pycache__/model.cpython-38.pyc b/new_impl/cv/elasticdnn/api/__pycache__/model.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..43327f0b9640df91906e3bd4d4625e9371a371a5 Binary files /dev/null and b/new_impl/cv/elasticdnn/api/__pycache__/model.cpython-38.pyc differ diff --git a/new_impl/cv/elasticdnn/api/__pycache__/online_model_v2.cpython-38.pyc b/new_impl/cv/elasticdnn/api/__pycache__/online_model_v2.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..471de33a393abde026f5d4f06b1495fbe2ef48fd Binary files /dev/null and b/new_impl/cv/elasticdnn/api/__pycache__/online_model_v2.cpython-38.pyc differ diff --git a/new_impl/cv/elasticdnn/api/algs/__pycache__/md_pretraining_wo_fbs.cpython-38.pyc b/new_impl/cv/elasticdnn/api/algs/__pycache__/md_pretraining_wo_fbs.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ca36018faf97e4bba0a2944a9341226bbbe58081 Binary files /dev/null and b/new_impl/cv/elasticdnn/api/algs/__pycache__/md_pretraining_wo_fbs.cpython-38.pyc differ diff --git a/new_impl/cv/elasticdnn/api/algs/fm_lora.py b/new_impl/cv/elasticdnn/api/algs/fm_lora.py new file mode 100644 index 0000000000000000000000000000000000000000..52444d32329141f831379cbe217e2351ce1b6525 --- /dev/null +++ b/new_impl/cv/elasticdnn/api/algs/fm_lora.py @@ -0,0 +1,153 @@ +from typing import Any, Dict +from schema import Schema +from data import Scenario, MergedDataset +from new_impl.cv.base.alg import BaseAlg +from data import build_dataloader +from ..model import ElasticDNN_OfflineFMModel +from ...model.base import ElasticDNNUtil +import torch.optim +import tqdm +from torch import nn +from utils.dl.common.env import create_tbwriter +import os +import random +import numpy as np +from copy import deepcopy +from utils.dl.common.model import get_module +from utils.common.log import logger + + +class ElasticDNN_FMLoRAAlg(BaseAlg): + def get_required_models_schema(self) -> Schema: + return Schema({ + 'fm': ElasticDNN_OfflineFMModel + }) + + def get_required_hyp_schema(self) -> Schema: + from schema import Optional + return Schema({ + 'launch_tbboard': bool, + + 'samples_size': object, + 'ab_r': int, + + 'train_batch_size': int, + 'val_batch_size': int, + 'num_workers': int, + 'optimizer': str, + 'optimizer_args': dict, + 'scheduler': str, + 'scheduler_args': dict, + 'num_iters': int, + 'val_freq': int, + + Optional('fm_lora_ckpt_path'): str + }) + + def run(self, scenario: Scenario, hyps: Dict) -> Dict[str, Any]: + super().run(scenario, hyps) + assert isinstance(self.models['fm'], ElasticDNN_OfflineFMModel) # for auto completion + + # 1. add LoRA + lora_util = self.models['fm'].get_lora_util() + device = self.models['fm'].device + + sample = hyps['samples_size'] + if isinstance(sample, (tuple, list)) and isinstance(sample[0], int): + sample = torch.rand(hyps['samples_size']).to(device) + lora_util.add_lora_ab_to_fm(self.models['fm'].models_dict['main'], hyps['ab_r'], sample) + + if 'fm_lora_ckpt_path' in hyps.keys() and hyps['fm_lora_ckpt_path'] != '' and hyps['fm_lora_ckpt_path'] is not None: + _ckpt = torch.load(hyps['fm_lora_ckpt_path'])['main'] + + new_state_dict = deepcopy(self.models['fm'].models_dict['main'].state_dict()) + + for n, p in _ckpt.named_parameters(): + if 'qkv.abs' not in n: + continue + + new_state_dict[n] = p + logger.info(f'use {n} from ckpt') + + self.models['fm'].models_dict['main'].load_state_dict(new_state_dict) + + + # 2. train (knowledge distillation, index relationship) + offline_datasets = scenario.get_offline_datasets() + train_dataset = MergedDataset([d['train'] for d in offline_datasets.values()]) + + # debug + # from data.visualize import visualize_classes_in_object_detection + # d = offline_datasets['GTA5Det']['val'] + # class_to_idx_map = {c: d.idx_map[i] for i, c in enumerate(d.classes)} + # print(class_to_idx_map) + # visualize_classes_in_object_detection(d, class_to_idx_map, + # {}, os.path.join(self.res_save_dir, 'debug.png')) + # exit() + val_dataset = MergedDataset([d['val'] for d in offline_datasets.values()]) + train_loader = iter(build_dataloader(train_dataset, hyps['train_batch_size'], hyps['num_workers'], + True, None)) + # train_loader = build_dataloader(train_dataset, hyps['train_batch_size'], hyps['num_workers'], + # True, None) + # if hyps['use_train_loader_for_val']: + # val_loader = build_dataloader(train_dataset, hyps['val_batch_size'], hyps['num_workers'], + # False, False) + # logger.warn('use train loader for val!!!') + # else: + val_loader = build_dataloader(val_dataset, hyps['val_batch_size'], hyps['num_workers'], + False, False) + + lora_params = lora_util.train_only_lora(self.models['fm'].models_dict['main']) + # x = torch.rand(1,3,224,224).to('cuda') + # print(self.models['fm'].models_dict['main'](x).logits) + head_params = self.models['fm'].get_task_head_params() + optimizer = torch.optim.__dict__[hyps['optimizer']](lora_params + head_params, **hyps['optimizer_args']) + scheduler = torch.optim.lr_scheduler.__dict__[hyps['scheduler']](optimizer, **hyps['scheduler_args']) + + fbs_tb_writer = create_tbwriter(os.path.join(self.res_save_dir, 'tb_log'), launch_tbboard=hyps['launch_tbboard']) + pbar = tqdm.tqdm(range(hyps['num_iters']), dynamic_ncols=True) + + best_val_acc = 0 + val_acc = 0 + + for iter_index in pbar: + self.models['fm'].to_train_mode() + #x, y, _ = next(train_loader) + x, y = next(train_loader) + # x, y = train_loader + if isinstance(x, dict) and isinstance(y,torch.Tensor): + for k, v in x.items(): + if isinstance(v, torch.Tensor): + x[k] = v.to(device) + y = y.to(device) + elif isinstance(x, dict) and isinstance(y,dict): + for k, v in x.items(): + if isinstance(v, torch.Tensor): + x[k] = v.to(device) + for k,v in y.items(): + if isinstance(v,torch.Tensor): + y[k] = v.to(device) + else: + x, y = x.to(device), y.to(device) + task_loss = self.models['fm'].forward_to_get_task_loss(x, y) + #task_loss.requires_grad_(True) + optimizer.zero_grad() + task_loss.backward() + optimizer.step() + scheduler.step() + + if (iter_index + 1) % hyps['val_freq'] == 0: + # logger.warn('use train loader for val!!!') + + self.models['fm'].to_eval_mode() + val_acc = self.models['fm'].get_accuracy(val_loader) + + self.models['fm'].save_model(os.path.join(self.res_save_dir, 'models/fm_last.pt')) + if val_acc > best_val_acc: + best_val_acc = val_acc + self.models['fm'].save_model(os.path.join(self.res_save_dir, 'models/fm_best.pt')) + + fbs_tb_writer.add_scalar(f'losses/task_loss', task_loss, iter_index) + fbs_tb_writer.add_scalar(f'accs/val_acc', val_acc, iter_index) + fbs_tb_writer.add_scalar(f'lr', optimizer.param_groups[0]['lr'], iter_index) + pbar.set_description(f'loss: {task_loss:.6f}, val_acc: {val_acc:.4f}') diff --git a/new_impl/cv/elasticdnn/api/algs/fm_lora_cnn.py b/new_impl/cv/elasticdnn/api/algs/fm_lora_cnn.py new file mode 100644 index 0000000000000000000000000000000000000000..8fd69e3c1b36e267042912a5114ce9505a0f8d40 --- /dev/null +++ b/new_impl/cv/elasticdnn/api/algs/fm_lora_cnn.py @@ -0,0 +1,157 @@ +from typing import Any, Dict +from schema import Schema +from data import Scenario, MergedDataset +from new_impl.cv.base.alg import BaseAlg +from data import build_dataloader +from ..model import ElasticDNN_OfflineFMModel +from ...model.base import ElasticDNNUtil +import torch.optim +import tqdm +from torch import nn +from utils.dl.common.env import create_tbwriter +import os +import random +import numpy as np +from copy import deepcopy +from utils.dl.common.model import get_module +from utils.common.log import logger +from torchvision.transforms import Compose + +class ElasticDNN_FMLoRAAlg(BaseAlg): + def get_required_models_schema(self) -> Schema: + return Schema({ + 'fm': ElasticDNN_OfflineFMModel + }) + + def get_required_hyp_schema(self) -> Schema: + from schema import Optional + return Schema({ + 'launch_tbboard': bool, + + 'samples_size': object, + 'ab_r': int, + + 'train_batch_size': int, + 'val_batch_size': int, + 'num_workers': int, + 'optimizer': str, + 'optimizer_args': dict, + 'scheduler': str, + 'scheduler_args': dict, + 'num_iters': int, + 'val_freq': int, + + Optional('fm_lora_ckpt_path'): str, + Optional('transform'): Compose + }) + + def run(self, scenario: Scenario, hyps: Dict, collate_fn=None) -> Dict[str, Any]: + super().run(scenario, hyps) + assert isinstance(self.models['fm'], ElasticDNN_OfflineFMModel) # for auto completion + + # 1. add LoRA + #lora_util = self.models['fm'].get_lora_util() + device = self.models['fm'].device + + sample = hyps['samples_size'] + # if isinstance(sample, (tuple, list)) and isinstance(sample[0], int): + # sample = torch.rand(hyps['samples_size']).to(device) + # lora_util.add_lora_ab_to_fm(self.models['fm'].models_dict['main'], hyps['ab_r'], sample) + + # if 'fm_lora_ckpt_path' in hyps.keys() and hyps['fm_lora_ckpt_path'] != '' and hyps['fm_lora_ckpt_path'] is not None: + # _ckpt = torch.load(hyps['fm_lora_ckpt_path'])['main'] + + # new_state_dict = deepcopy(self.models['fm'].models_dict['main'].state_dict()) + + # for n, p in _ckpt.named_parameters(): + # if 'qkv.abs' not in n: + # continue + + # new_state_dict[n] = p + # logger.info(f'use {n} from ckpt') + + # self.models['fm'].models_dict['main'].load_state_dict(new_state_dict) + + + # 2. train (knowledge distillation, index relationship) + if 'transform' in hyps.keys(): + offline_datasets = scenario.get_offline_datasets(transform=hyps['transform']) + else: + offline_datasets = scenario.get_offline_datasets() + train_dataset = MergedDataset([d['train'] for d in offline_datasets.values()]) + + # debug + # from data.visualize import visualize_classes_in_object_detection + # d = offline_datasets['GTA5Det']['val'] + # class_to_idx_map = {c: d.idx_map[i] for i, c in enumerate(d.classes)} + # print(class_to_idx_map) + # visualize_classes_in_object_detection(d, class_to_idx_map, + # {}, os.path.join(self.res_save_dir, 'debug.png')) + # exit() + val_dataset = MergedDataset([d['val'] for d in offline_datasets.values()]) + train_loader = iter(build_dataloader(train_dataset, hyps['train_batch_size'], hyps['num_workers'], + True, None, collate_fn=collate_fn)) + # train_loader = build_dataloader(train_dataset, hyps['train_batch_size'], hyps['num_workers'], + # True, None) + # if hyps['use_train_loader_for_val']: + # val_loader = build_dataloader(train_dataset, hyps['val_batch_size'], hyps['num_workers'], + # False, False) + # logger.warn('use train loader for val!!!') + # else: + val_loader = build_dataloader(val_dataset, hyps['val_batch_size'], hyps['num_workers'], + False, False, collate_fn=collate_fn) + + #lora_params = lora_util.train_only_lora(self.models['fm'].models_dict['main']) + # x = torch.rand(1,3,224,224).to('cuda') + # print(self.models['fm'].models_dict['main'](x).logits) + head_params = self.models['fm'].get_task_head_params() + optimizer = torch.optim.__dict__[hyps['optimizer']](self.models['fm'].models_dict['main'].parameters(), **hyps['optimizer_args']) + scheduler = torch.optim.lr_scheduler.__dict__[hyps['scheduler']](optimizer, **hyps['scheduler_args']) + + fbs_tb_writer = create_tbwriter(os.path.join(self.res_save_dir, 'tb_log'), launch_tbboard=hyps['launch_tbboard']) + pbar = tqdm.tqdm(range(hyps['num_iters']), dynamic_ncols=True) + + best_val_acc = 0 + val_acc = 0 + + for iter_index in pbar: + self.models['fm'].to_train_mode() + #x, y, _ = next(train_loader) + x, y = next(train_loader) + # x, y = train_loader + if isinstance(x, dict) and isinstance(y,torch.Tensor): + for k, v in x.items(): + if isinstance(v, torch.Tensor): + x[k] = v.to(device) + y = y.to(device) + elif isinstance(x, dict) and isinstance(y,dict): + for k, v in x.items(): + if isinstance(v, torch.Tensor): + x[k] = v.to(device) + for k,v in y.items(): + if isinstance(v,torch.Tensor): + y[k] = v.to(device) + else: + x, y = x.to(device), y.to(device) + task_loss = self.models['fm'].forward_to_get_task_loss(x, y) + #task_loss.requires_grad_(True) + optimizer.zero_grad() + task_loss.backward() + optimizer.step() + scheduler.step() + + if (iter_index + 1) % hyps['val_freq'] == 0: + # logger.warn('use train loader for val!!!') + + self.models['fm'].to_eval_mode() + val_acc = self.models['fm'].get_accuracy(val_loader) + + self.models['fm'].save_model(os.path.join(self.res_save_dir, 'models/fm_last.pt')) + if val_acc > best_val_acc: + best_val_acc = val_acc + self.models['fm'].save_model(os.path.join(self.res_save_dir, 'models/fm_best.pt')) + + fbs_tb_writer.add_scalar(f'losses/task_loss', task_loss, iter_index) + fbs_tb_writer.add_scalar(f'accs/val_acc', val_acc, iter_index) + fbs_tb_writer.add_scalar(f'lr', optimizer.param_groups[0]['lr'], iter_index) + pbar.set_description(f'loss: {task_loss:.6f}, val_acc: {val_acc:.4f}') diff --git a/new_impl/cv/elasticdnn/api/algs/fm_lora_glip.py b/new_impl/cv/elasticdnn/api/algs/fm_lora_glip.py new file mode 100644 index 0000000000000000000000000000000000000000..a9d016039dd1d767be39faade191a656e55b4437 --- /dev/null +++ b/new_impl/cv/elasticdnn/api/algs/fm_lora_glip.py @@ -0,0 +1,154 @@ +from typing import Any, Dict +from schema import Schema +from data import Scenario, MergedDataset +from methods.base.alg import BaseAlg +from data import build_dataloader +from ..model import ElasticDNN_OfflineFMModel +from ...model.base import ElasticDNNUtil +import torch.optim +import tqdm +from torch import nn +from torchvision.transforms import Compose +from utils.dl.common.env import create_tbwriter +import os +import random +import numpy as np +from copy import deepcopy +from utils.dl.common.model import get_module +from utils.common.log import logger + + +class ElasticDNN_FMLoRAAlg(BaseAlg): + def get_required_models_schema(self) -> Schema: + return Schema({ + 'fm': ElasticDNN_OfflineFMModel + }) + + def get_required_hyp_schema(self) -> Schema: + from schema import Optional + return Schema({ + 'launch_tbboard': bool, + + 'samples_size': object, + 'ab_r': int, + + 'train_batch_size': int, + 'val_batch_size': int, + 'num_workers': int, + 'optimizer': str, + 'optimizer_args': dict, + 'scheduler': str, + 'scheduler_args': dict, + 'num_iters': int, + 'val_freq': int, + + Optional('fm_lora_ckpt_path'): str, + Optional('transform'): Compose, + }) + + def run(self, scenario: Scenario, hyps: Dict, collate_fn=None) -> Dict[str, Any]: + super().run(scenario, hyps) + + assert isinstance(self.models['fm'], ElasticDNN_OfflineFMModel) # for auto completion + + # 1. add LoRA + lora_util = self.models['fm'].get_lora_util() + device = self.models['fm'].device + + sample = hyps['samples_size'] + if isinstance(sample, (tuple, list)) and isinstance(sample[0], int): + sample = torch.rand(hyps['samples_size']).to(device) + lora_util.add_lora_ab_to_fm(self.models['fm'].models_dict['main'], hyps['ab_r'], sample) + + if 'fm_lora_ckpt_path' in hyps.keys() and hyps['fm_lora_ckpt_path'] != '' and hyps['fm_lora_ckpt_path'] is not None: + _ckpt = torch.load(hyps['fm_lora_ckpt_path'])['main'] + + new_state_dict = deepcopy(self.models['fm'].models_dict['main'].state_dict()) + + for n, p in _ckpt.named_parameters(): + if 'qkv.abs' not in n: + continue + + new_state_dict[n] = p + logger.info(f'use {n} from ckpt') + + self.models['fm'].models_dict['main'].load_state_dict(new_state_dict) + + + # 2. train (knowledge distillation, index relationship) + if 'transform' in hyps.keys(): + offline_datasets = scenario.get_offline_datasets(transform=hyps['transform']) + else: + offline_datasets = scenario.get_offline_datasets() + train_dataset = MergedDataset([d['train'] for d in offline_datasets.values()]) + + # debug + # from data.visualize import visualize_classes_in_object_detection + # d = offline_datasets['GTA5Det']['val'] + # class_to_idx_map = {c: d.idx_map[i] for i, c in enumerate(d.classes)} + # print(class_to_idx_map) + # visualize_classes_in_object_detection(d, class_to_idx_map, + # {}, os.path.join(self.res_save_dir, 'debug.png')) + # exit() + + val_dataset = MergedDataset([d['val'] for d in offline_datasets.values()]) + train_loader = iter(build_dataloader(train_dataset, hyps['train_batch_size'], hyps['num_workers'], + True, None, collate_fn=collate_fn)) + + # if hyps['use_train_loader_for_val']: + # val_loader = build_dataloader(train_dataset, hyps['val_batch_size'], hyps['num_workers'], + # False, False) + # logger.warn('use train loader for val!!!') + # else: + val_loader = build_dataloader(val_dataset, hyps['val_batch_size'], hyps['num_workers'], + False, False, collate_fn=collate_fn) + + lora_params = lora_util.train_only_lora_and_conv(self.models['fm'].models_dict['main']) + head_params = self.models['fm'].get_task_head_params() + + num_lora_params = sum([np.prod(p.size()) for p in lora_params]) + total_params = sum([np.prod(p.size()) for p in self.models['fm'].models_dict['main'].parameters()]) + logger.info(f'num lora params: {num_lora_params}, total params: {total_params}, ratio: {num_lora_params / total_params}') + + optimizer = torch.optim.__dict__[hyps['optimizer']](lora_params + head_params, **hyps['optimizer_args']) + scheduler = torch.optim.lr_scheduler.__dict__[hyps['scheduler']](optimizer, **hyps['scheduler_args']) + + fbs_tb_writer = create_tbwriter(os.path.join(self.res_save_dir, 'tb_log'), launch_tbboard=hyps['launch_tbboard']) + pbar = tqdm.tqdm(range(hyps['num_iters']), dynamic_ncols=True) + + best_val_acc = 0 + val_acc = 0 + + for iter_index in pbar: + self.models['fm'].to_train_mode() + + x, y = next(train_loader) + + if isinstance(x, dict): + for k, v in x.items(): + if isinstance(v, torch.Tensor): + x[k] = v.to(device) + y = y.to(device) + else: + x, y = x.to(device), y.to(device) + task_loss = self.models['fm'].forward_to_get_task_loss(x, y) + optimizer.zero_grad() + task_loss.backward() + optimizer.step() + scheduler.step() + + if (iter_index + 1) % hyps['val_freq'] == 0: + # logger.warn('use train loader for val!!!') + + self.models['fm'].to_eval_mode() + val_acc = self.models['fm'].get_accuracy(val_loader) + + self.models['fm'].save_model(os.path.join(self.res_save_dir, 'models/fm_last.pt')) + if val_acc > best_val_acc: + best_val_acc = val_acc + self.models['fm'].save_model(os.path.join(self.res_save_dir, 'models/fm_best.pt')) + + fbs_tb_writer.add_scalar(f'losses/task_loss', task_loss, iter_index) + fbs_tb_writer.add_scalar(f'accs/val_acc', val_acc, iter_index) + fbs_tb_writer.add_scalar(f'lr', optimizer.param_groups[0]['lr'], iter_index) + pbar.set_description(f'loss: {task_loss:.6f}, map50: {val_acc:.4f}') diff --git a/new_impl/cv/elasticdnn/api/algs/md_pretraining_bk.py b/new_impl/cv/elasticdnn/api/algs/md_pretraining_bk.py new file mode 100644 index 0000000000000000000000000000000000000000..f8903b7569f54f2dcd364359001de925a3ab9287 --- /dev/null +++ b/new_impl/cv/elasticdnn/api/algs/md_pretraining_bk.py @@ -0,0 +1,292 @@ +from typing import Any, Dict +from schema import Schema, Or +import schema +from data import Scenario, MergedDataset +from methods.base.alg import BaseAlg +from data import build_dataloader +from ..model import ElasticDNN_OfflineFMModel, ElasticDNN_OfflineMDModel +from ...model.base import ElasticDNNUtil +import torch.optim +import tqdm +import torch.nn.functional as F +from torch import nn +from utils.dl.common.env import create_tbwriter +import os +import random +import numpy as np +from copy import deepcopy +from utils.dl.common.model import LayerActivation, get_module +from utils.common.log import logger + + +class ElasticDNN_MDPretrainingAlg(BaseAlg): + """ + TODO: fine-tuned FM -> init MD -> trained MD -> construct indexes (only between similar weights) and fine-tune + """ + def get_required_models_schema(self) -> Schema: + return Schema({ + 'fm': ElasticDNN_OfflineFMModel, + 'md': ElasticDNN_OfflineMDModel + }) + + def get_required_hyp_schema(self) -> Schema: + return Schema({ + 'launch_tbboard': bool, + + 'samples_size': (int, int, int, int), + 'generate_md_width_ratio': int, + + 'FBS_r': int, + 'FBS_ignore_layers': [str], + + 'train_batch_size': int, + 'val_batch_size': int, + 'num_workers': int, + 'optimizer': str, + 'md_optimizer_args': dict, + 'indexes_optimizer_args': dict, + 'scheduler': str, + 'scheduler_args': dict, + 'num_iters': int, + 'val_freq': int, + 'max_sparsity': float, + 'min_sparsity': float, + 'distill_loss_weight': float, + 'index_loss_weight': float, + 'val_num_sparsities': int, + + 'bn_cal_num_iters': int, + + 'index_guided_linear_comb_split_size': Or(int, None) + }) + + def upsample_2d_tensor(self, p: torch.Tensor, target_len: int): + assert p.dim() == 2 # regard 2d weight as (batch_size, 1d_vector_dim) + return F.upsample(p.unsqueeze(1).unsqueeze(3), + size=(target_len, 1), + mode='bilinear').squeeze(3).squeeze(1) + + def two_params_diff_fast(self, trained_p: torch.Tensor, ref_p: torch.Tensor, + index: torch.Tensor, + split_size: int): + + assert trained_p.dim() == ref_p.dim() + assert index.size(0) == trained_p.size(0) and index.size(1) == ref_p.size(0) + + # print(trained_p.size(), ref_p.size(), index.size()) + + ref_p = ref_p.detach() + if trained_p.dim() > 1: + trained_p = trained_p.flatten(1) + ref_p = ref_p.flatten(1) + + # the weight size of master DNN and foundation model may be totally different + + # MD -> FM: upsample first + # FM -> MD: downsample first + if trained_p.size(1) < ref_p.size(1): + trained_p = self.upsample_2d_tensor(trained_p, ref_p.size(1)) + + index = index.unsqueeze(-1) + # linear_combed_ref_p = (ref_p.unsqueeze(0) * index).sum(1) + # else: + + # print(trained_p.size(), ref_p.size(), index.size()) + + if split_size is None: + # old version: huge memory consumption, not recommended (although this is fastest) + # print('old version') + linear_combed_ref_p = (ref_p.unsqueeze(0) * index).sum(1) + + else: + # new version + linear_combed_ref_p = 0 + + cur_split_size = split_size + while index.size(1) % cur_split_size != 0: + cur_split_size -= 1 + # print(cur_split_size) + + for i in range(0, index.size(1), cur_split_size): + # if not isinstance(linear_combed_ref_p, int): + # print(linear_combed_ref_p.size(), ref_p.unsqueeze(0)[:, i: i + cur_split_size].size(), index[:, i: i + cur_split_size].size()) + linear_combed_ref_p += ref_p.unsqueeze(0)[:, i: i + cur_split_size] * index[:, i: i + cur_split_size] + linear_combed_ref_p = linear_combed_ref_p.sum(1) + + diff = (linear_combed_ref_p - trained_p).norm(2) ** 2 + return diff + + def get_index_loss(self, fm, md, indexes, match_fn, split_size): + res = 0. + + for name, p in md.named_parameters(): + if p.dim() == 0: + continue + + raw_p = match_fn(name, fm) + if raw_p is None: + continue + + index = indexes[name] + + # print(name) + res += self.two_params_diff_fast(p, raw_p, index, split_size) + return res + + def bn_cal(self, model: nn.Module, train_loader, num_iters, device): + has_bn = False + for n, m in model.named_modules(): + if isinstance(m, nn.BatchNorm2d): + has_bn = True + break + + if not has_bn: + return {} + + def bn_calibration_init(m): + """ calculating post-statistics of batch normalization """ + if getattr(m, 'track_running_stats', False): + # reset all values for post-statistics + m.reset_running_stats() + # set bn in training mode to update post-statistics + m.training = True + + with torch.no_grad(): + model.eval() + model.apply(bn_calibration_init) + for _ in range(num_iters): + x, _ = next(train_loader) + model(x.to(device)) + model.eval() + + bn_stats = {} + for n, m in model.named_modules(): + if isinstance(m, nn.BatchNorm2d): + bn_stats[n] = m + return bn_stats + + def run(self, scenario: Scenario, hyps: Dict) -> Dict[str, Any]: + super().run(scenario, hyps) + + # sanity check + # a= torch.tensor([[1, 2, 3], [1, 2, 4]]) + # index = torch.tensor([[1, 2, 3], + # [1, 2, 4]]) + # b = torch.tensor([[1, 2, 3], [1, 2, 4], [2, 3, 4]]) + # print(self.two_params_diff_fast(a, b, index, hyps['index_guided_linear_comb_split_size'])) + + assert isinstance(self.models['md'], ElasticDNN_OfflineMDModel) # for auto completion + assert isinstance(self.models['fm'], ElasticDNN_OfflineFMModel) # for auto completion + + # 1. add FBS + device = self.models['md'].device + + if self.models['md'].models_dict['main'] == -1: + logger.info(f'init master DNN by reducing width of an adapted foundation model (already tuned by LoRA)...') + + before_fm_model = deepcopy(self.models['fm'].models_dict['main']) + lora_util = self.models['fm'].get_lora_util() + lora_absorbed_fm_model = lora_util.absorb_lora_and_recover_net_structure(self.models['fm'].models_dict['main'], + torch.rand(hyps['samples_size']).to(device)) + self.models['fm'].models_dict['main'] = lora_absorbed_fm_model + master_dnn = self.models['fm'].generate_md_by_reducing_width(hyps['generate_md_width_ratio'], + torch.rand(hyps['samples_size']).to(device)) + self.models['fm'].models_dict['main'] = before_fm_model + + elastic_dnn_util = self.models['fm'].get_elastic_dnn_util() + master_dnn = elastic_dnn_util.convert_raw_dnn_to_master_dnn_with_perf_test(master_dnn, + hyps['FBS_r'], hyps['FBS_ignore_layers']) + self.models['md'].models_dict['main'] = master_dnn + self.models['md'].to(device) + + # 2. train (knowledge distillation, index relationship) + offline_datasets = scenario.get_offline_datasets() + train_dataset = MergedDataset([d['train'] for d in offline_datasets.values()]) + val_dataset = MergedDataset([d['val'] for d in offline_datasets.values()]) + train_loader = iter(build_dataloader(train_dataset, hyps['train_batch_size'], hyps['num_workers'], + True, None)) + val_loader = build_dataloader(val_dataset, hyps['val_batch_size'], hyps['num_workers'], + False, False) + + # 2.1 train whole master DNN (knowledge distillation) + for p in master_dnn.parameters(): + p.requires_grad = True + self.models['md'].to_train_mode() + + optimizer = torch.optim.__dict__[hyps['optimizer']]([ + {'params': self.models['md'].models_dict['main'].parameters(), **hyps['md_optimizer_args']} + ]) + scheduler = torch.optim.lr_scheduler.__dict__[hyps['scheduler']](optimizer, **hyps['scheduler_args']) + tb_writer = create_tbwriter(os.path.join(self.res_save_dir, 'tb_log'), launch_tbboard=hyps['launch_tbboard']) + pbar = tqdm.tqdm(range(hyps['num_iters']), dynamic_ncols=True) + best_avg_val_acc = 0. + + md_output_hook = None + + for iter_index in pbar: + self.models['md'].to_train_mode() + self.models['fm'].to_eval_mode() + + rand_sparsity = random.random() * (hyps['max_sparsity'] - hyps['min_sparsity']) + hyps['min_sparsity'] + elastic_dnn_util.set_master_dnn_sparsity(self.models['md'].models_dict['main'], rand_sparsity) + + x, y = next(train_loader) + x, y = x.to(device), y.to(device) + + with torch.no_grad(): + fm_output = self.models['fm'].infer(x) + + if md_output_hook is None: + md_output_hook = LayerActivation(self.models['md'].models_dict['main'], False, device) + task_loss = self.models['md'].forward_to_get_task_loss(x, y) + md_output = md_output_hook.output + distill_loss = hyps['distill_loss_weight'] * self.models['md'].get_distill_loss(md_output, fm_output) + total_loss = task_loss + distill_loss + + optimizer.zero_grad() + total_loss.backward() + optimizer.step() + scheduler.step() + + if (iter_index + 1) % hyps['val_freq'] == 0: + + elastic_dnn_util.clear_cached_channel_attention_in_master_dnn(self.models['md'].models_dict['main']) + md_output_hook.remove() + md_output_hook = None + + cur_md = self.models['md'].models_dict['main'] + md_for_test = deepcopy(self.models['md'].models_dict['main']) + val_accs = {} + avg_val_acc = 0. + bn_stats = {} + + for val_sparsity in np.linspace(hyps['min_sparsity'], hyps['max_sparsity'], num=hyps['val_num_sparsities']): + elastic_dnn_util.set_master_dnn_sparsity(md_for_test, val_sparsity) + bn_stats[f'{val_sparsity:.4f}'] = self.bn_cal(md_for_test, train_loader, hyps['bn_cal_num_iters'], device) + self.models['md'].models_dict['main'] = md_for_test + self.models['md'].to_eval_mode() + val_acc = self.models['md'].get_accuracy(val_loader) + + val_accs[f'{val_sparsity:.4f}'] = val_acc + avg_val_acc += val_acc + + avg_val_acc /= hyps['val_num_sparsities'] + + self.models['md'].models_dict['main'] = cur_md + self.models['md'].models_dict['bn_stats'] = bn_stats + + self.models['md'].save_model(os.path.join(self.res_save_dir, 'models/md_last.pt')) + self.models['fm'].save_model(os.path.join(self.res_save_dir, 'models/fm_last.pt')) + + if avg_val_acc > best_avg_val_acc: + best_avg_val_acc = avg_val_acc + self.models['md'].save_model(os.path.join(self.res_save_dir, 'models/md_best.pt')) + self.models['fm'].save_model(os.path.join(self.res_save_dir, 'models/fm_best.pt')) + + tb_writer.add_scalars(f'losses', dict(task=task_loss, distill=distill_loss, total=total_loss), iter_index) + pbar.set_description(f'loss: {total_loss:.6f}') + if (iter_index + 1) >= hyps['val_freq']: + tb_writer.add_scalars(f'accs/val_accs', val_accs, iter_index) + tb_writer.add_scalar(f'accs/avg_val_acc', avg_val_acc, iter_index) + pbar.set_description(f'loss: {total_loss:.6f}, avg_val_acc: {avg_val_acc:.4f}') + \ No newline at end of file diff --git a/new_impl/cv/elasticdnn/api/algs/md_pretraining_index.py b/new_impl/cv/elasticdnn/api/algs/md_pretraining_index.py new file mode 100644 index 0000000000000000000000000000000000000000..a9f71de66b096d8c341a1fd97214803e3f1a137b --- /dev/null +++ b/new_impl/cv/elasticdnn/api/algs/md_pretraining_index.py @@ -0,0 +1,322 @@ +from typing import Any, Dict +from schema import Schema, Or +import schema +from data import Scenario, MergedDataset +from methods.base.alg import BaseAlg +from data import build_dataloader +from ..model import ElasticDNN_OfflineFMModel, ElasticDNN_OfflineMDModel +from ...model.base import ElasticDNNUtil +import torch.optim +import tqdm +import torch.nn.functional as F +from torch import nn +from utils.dl.common.env import create_tbwriter +import os +import random +import numpy as np +from copy import deepcopy +from utils.dl.common.model import LayerActivation, get_module +from utils.common.log import logger +import matplotlib.pyplot as plt + + +class ElasticDNN_MDPretrainingIndexAlg(BaseAlg): + """ + construct indexes between a filter/row of MD and all filters/rows of FM in the same layer + too huge indexes (~1GB), train so slow, hard to optimize + """ + def get_required_models_schema(self) -> Schema: + return Schema({ + 'fm': ElasticDNN_OfflineFMModel, + 'md': ElasticDNN_OfflineMDModel + }) + + def get_required_hyp_schema(self) -> Schema: + return Schema({ + 'launch_tbboard': bool, + + 'samples_size': (int, int, int, int), + + 'train_batch_size': int, + 'val_batch_size': int, + 'num_workers': int, + 'optimizer': str, + 'indexes_optimizer_args': dict, + 'scheduler': str, + 'scheduler_args': dict, + 'num_iters': int, + 'val_freq': int, + 'index_loss_l1_weight': float, + 'val_num_sparsities': int, + + 'bn_cal_num_iters': int, + + 'index_guided_linear_comb_split_size': Or(int, None) + }) + + def upsample_2d_tensor(self, p: torch.Tensor, target_len: int): + assert p.dim() == 2 # regard 2d weight as (batch_size, 1d_vector_dim) + return F.upsample(p.unsqueeze(1).unsqueeze(3), + size=(target_len, 1), + mode='bilinear').squeeze(3).squeeze(1) + + def two_params_diff_fast(self, trained_p: torch.Tensor, ref_p: torch.Tensor, + index: torch.Tensor, + split_size: int): + + assert trained_p.dim() == ref_p.dim() + assert index.size(0) == trained_p.size(0) and index.size(1) == ref_p.size(0) + + # print(trained_p.size(), ref_p.size(), index.size()) + + ref_p = ref_p.detach() + if trained_p.dim() > 1: + trained_p = trained_p.flatten(1) + ref_p = ref_p.flatten(1) + + # the weight size of master DNN and foundation model may be totally different + + # MD -> FM: upsample first + # FM -> MD: downsample first + if trained_p.size(1) < ref_p.size(1): + trained_p = self.upsample_2d_tensor(trained_p, ref_p.size(1)) + + index = index.unsqueeze(-1) + # linear_combed_ref_p = (ref_p.unsqueeze(0) * index).sum(1) + # else: + + # print(trained_p.size(), ref_p.size(), index.size()) + + if split_size is None: + # old version: huge memory consumption, not recommended (although this is fastest) + # print('old version') + linear_combed_ref_p = (ref_p.unsqueeze(0) * index).sum(1) + + else: + # new version + linear_combed_ref_p = 0 + + cur_split_size = split_size + while index.size(1) % cur_split_size != 0: + cur_split_size -= 1 + # print(cur_split_size) + + for i in range(0, index.size(1), cur_split_size): + # if not isinstance(linear_combed_ref_p, int): + # print(linear_combed_ref_p.size(), ref_p.unsqueeze(0)[:, i: i + cur_split_size].size(), index[:, i: i + cur_split_size].size()) + linear_combed_ref_p += ref_p.unsqueeze(0)[:, i: i + cur_split_size] * index[:, i: i + cur_split_size] + linear_combed_ref_p = linear_combed_ref_p.sum(1) + + diff = (linear_combed_ref_p - trained_p).norm(2) ** 2 + return diff + + def get_index_loss(self, fm, md, indexes, match_fn, split_size): + res = 0. + + for name, p in md.named_parameters(): + if name not in indexes.keys(): + continue + # if p.dim() == 0: + # continue + + raw_p = match_fn(name, fm) + # if raw_p is None: + # continue + + index = indexes[name] + + # print(name) + res += self.two_params_diff_fast(p, raw_p, index, split_size) + return res + + def bn_cal(self, model: nn.Module, train_loader, num_iters, device): + has_bn = False + for n, m in model.named_modules(): + if isinstance(m, nn.BatchNorm2d): + has_bn = True + break + + if not has_bn: + return {} + + def bn_calibration_init(m): + """ calculating post-statistics of batch normalization """ + if getattr(m, 'track_running_stats', False): + # reset all values for post-statistics + m.reset_running_stats() + # set bn in training mode to update post-statistics + m.training = True + + with torch.no_grad(): + model.eval() + model.apply(bn_calibration_init) + for _ in range(num_iters): + x, _ = next(train_loader) + model(x.to(device)) + model.eval() + + bn_stats = {} + for n, m in model.named_modules(): + if isinstance(m, nn.BatchNorm2d): + bn_stats[n] = m + return bn_stats + + def run(self, scenario: Scenario, hyps: Dict) -> Dict[str, Any]: + super().run(scenario, hyps) + + # sanity check + # a= torch.tensor([[1, 2, 3], [1, 2, 4]]) + # index = torch.tensor([[1, 2, 3], + # [1, 2, 4]]) + # b = torch.tensor([[1, 2, 3], [1, 2, 4], [2, 3, 4]]) + # print(self.two_params_diff_fast(a, b, index, hyps['index_guided_linear_comb_split_size'])) + + assert isinstance(self.models['md'], ElasticDNN_OfflineMDModel) # for auto completion + assert isinstance(self.models['fm'], ElasticDNN_OfflineFMModel) # for auto completion + + # 1. add FBS + device = self.models['md'].device + + # logger.info(f'init master DNN by reducing width of an adapted foundation model (already tuned by LoRA)...') + + # before_fm_model = deepcopy(self.models['fm'].models_dict['main']) + # lora_util = self.models['fm'].get_lora_util() + # lora_absorbed_fm_model = lora_util.absorb_lora_and_recover_net_structure(self.models['fm'].models_dict['main'], + # torch.rand(hyps['samples_size']).to(device)) + # self.models['fm'].models_dict['main'] = lora_absorbed_fm_model + # master_dnn = self.models['fm'].generate_md_by_reducing_width(hyps['generate_md_width_ratio'], + # torch.rand(hyps['samples_size']).to(device)) + # self.models['fm'].models_dict['main'] = before_fm_model + + elastic_dnn_util = self.models['fm'].get_elastic_dnn_util() + # master_dnn = elastic_dnn_util.convert_raw_dnn_to_master_dnn_with_perf_test(master_dnn, + # hyps['FBS_r'], hyps['FBS_ignore_layers']) + # self.models['md'].models_dict['main'] = master_dnn + # self.models['md'].to(device) + # master_dnn = self.models['md'].models_dict['main'] + + # 2. train (knowledge distillation, index relationship) + offline_datasets = scenario.get_offline_datasets() + train_dataset = MergedDataset([d['train'] for d in offline_datasets.values()]) + val_dataset = MergedDataset([d['val'] for d in offline_datasets.values()]) + train_loader = iter(build_dataloader(train_dataset, hyps['train_batch_size'], hyps['num_workers'], + True, None)) + val_loader = build_dataloader(val_dataset, hyps['val_batch_size'], hyps['num_workers'], + False, False) + + # 2.1 train only FBS (skipped because current md cannot do proper inference) + + # 2.2 train whole master DNN (knowledge distillation, index relationship) + # for p in master_dnn.parameters(): + # p.requires_grad = True + # self.models['md'].to_train_mode() + + indexes = {} + for name, p in self.models['md'].models_dict['main'].named_parameters(): + if p.dim() > 1: + matched_p_in_fm = self.models['md'].get_matched_param_of_fm(name, self.models['fm'].models_dict['main']) + if matched_p_in_fm is None: + continue + indexes[name] = torch.zeros((p.size(0), matched_p_in_fm.size(0))).to(device) + indexes[name].requires_grad = True + + logger.info(f'construct index in layer {name}') + + tmp_indexes_file_path = os.path.join(self.res_save_dir, 'tmp-indexes.pt') + torch.save(indexes, tmp_indexes_file_path) + logger.info(f'generate indexes ({(os.path.getsize(tmp_indexes_file_path) / 1024**2):.3f}MB)') + os.remove(tmp_indexes_file_path) + + optimizer = torch.optim.__dict__[hyps['optimizer']]([ + # {'params': self.models['md'].models_dict['main'].parameters(), **hyps['optimizer_args']}, + {'params': [v for v in indexes.values()], **hyps['indexes_optimizer_args']} + ]) + scheduler = torch.optim.lr_scheduler.__dict__[hyps['scheduler']](optimizer, **hyps['scheduler_args']) + tb_writer = create_tbwriter(os.path.join(self.res_save_dir, 'tb_log'), launch_tbboard=hyps['launch_tbboard']) + pbar = tqdm.tqdm(range(hyps['num_iters']), dynamic_ncols=True) + best_avg_val_acc = 0. + + for p in self.models['md'].models_dict['main'].parameters(): + p.requires_grad = False + for p in self.models['fm'].models_dict['main'].parameters(): + p.requires_grad = False + + for iter_index in pbar: + self.models['md'].to_eval_mode() + self.models['fm'].to_eval_mode() + + # rand_sparsity = random.random() * (hyps['max_sparsity'] - hyps['min_sparsity']) + hyps['min_sparsity'] + # elastic_dnn_util.set_master_dnn_sparsity(self.models['md'].models_dict['main'], rand_sparsity) + + # x, y = next(train_loader) + # x, y = x.to(device), y.to(device) + + # task_loss = self.models['md'].forward_to_get_task_loss(x, y) + # l1_reg_loss = hyps['l1_reg_loss_weight'] * elastic_dnn_util.get_accu_l1_reg_of_raw_channel_attention_in_master_dnn(master_dnn) + index_loss = self.get_index_loss(self.models['fm'].models_dict['main'], + self.models['md'].models_dict['main'], + indexes, + self.models['md'].get_matched_param_of_fm, + hyps['index_guided_linear_comb_split_size']) + index_l1_loss = hyps['index_loss_l1_weight'] * torch.FloatTensor([v.abs().sum() for v in indexes.values()]).sum() + + total_loss = index_loss + index_l1_loss + + optimizer.zero_grad() + total_loss.backward() + optimizer.step() + scheduler.step() + + if (iter_index + 1) % (10) == 0: + # visualize indexes histgoram + # do not use add_histogram because indexes is huge + os.makedirs(os.path.join(self.res_save_dir, 'index_hist'), exist_ok=True) + with torch.no_grad(): + for p_name, index in indexes.items(): + index_hist = index.view(-1).histc(bins=20).detach().cpu().numpy() + plt.bar(list(range(20)), index_hist) + plt.savefig(os.path.join(self.res_save_dir, f'index_hist/{p_name}.png')) + plt.clf() + + if (iter_index + 1) % hyps['val_freq'] == 0: + + elastic_dnn_util.clear_cached_channel_attention_in_master_dnn(self.models['md'].models_dict['main']) + + cur_md = self.models['md'].models_dict['main'] + md_for_test = deepcopy(self.models['md'].models_dict['main']) + val_accs = {} + avg_val_acc = 0. + bn_stats = {} + + for val_sparsity in [0.0, 0.2, 0.4, 0.8]: + elastic_dnn_util.set_master_dnn_sparsity(md_for_test, val_sparsity) + bn_stats[f'{val_sparsity:.4f}'] = self.bn_cal(md_for_test, train_loader, hyps['bn_cal_num_iters'], device) + self.models['md'].models_dict['main'] = md_for_test + self.models['md'].to_eval_mode() + val_acc = self.models['md'].get_accuracy(val_loader) + + val_accs[f'{val_sparsity:.4f}'] = val_acc + avg_val_acc += val_acc + avg_val_acc /= hyps['val_num_sparsities'] + + self.models['md'].models_dict['main'] = cur_md + self.models['md'].models_dict['indexes'] = indexes + self.models['md'].models_dict['bn_stats'] = bn_stats + self.models['fm'].models_dict['indexes'] = indexes + + self.models['md'].save_model(os.path.join(self.res_save_dir, 'models/md_last.pt')) + self.models['fm'].save_model(os.path.join(self.res_save_dir, 'models/fm_last.pt')) + + if avg_val_acc > best_avg_val_acc: + best_avg_val_acc = avg_val_acc + self.models['md'].save_model(os.path.join(self.res_save_dir, 'models/md_best.pt')) + self.models['fm'].save_model(os.path.join(self.res_save_dir, 'models/fm_best.pt')) + + tb_writer.add_scalars(f'losses', dict(index=index_loss, index_l1=index_l1_loss, total=total_loss), iter_index) + pbar.set_description(f'loss: {total_loss:.6f}, index_loss: {index_loss:.6f}, index_l1_loss: {index_l1_loss:.6f}') + if (iter_index + 1) >= hyps['val_freq']: + tb_writer.add_scalars(f'accs/val_accs', val_accs, iter_index) + tb_writer.add_scalar(f'accs/avg_val_acc', avg_val_acc, iter_index) + pbar.set_description(f'loss: {total_loss:.6f}, index_loss: {index_loss:.6f}, index_l1_loss: {index_l1_loss:.6f}, ' + f'avg_val_acc: {avg_val_acc:.4f}') + \ No newline at end of file diff --git a/new_impl/cv/elasticdnn/api/algs/md_pretraining_index_bk_too_slow.py b/new_impl/cv/elasticdnn/api/algs/md_pretraining_index_bk_too_slow.py new file mode 100644 index 0000000000000000000000000000000000000000..8bf1f57d5a0ff71f862a2c7d0aadddfeb41da84d --- /dev/null +++ b/new_impl/cv/elasticdnn/api/algs/md_pretraining_index_bk_too_slow.py @@ -0,0 +1,427 @@ +from typing import Any, Dict +from schema import Schema, Or +import schema +from data import Scenario, MergedDataset +from methods.base.alg import BaseAlg +from data import build_dataloader +from ..model import ElasticDNN_OfflineFMModel, ElasticDNN_OfflineMDModel +from ...model.base import ElasticDNNUtil +import torch.optim +import tqdm +import torch.nn.functional as F +from torch import nn +from utils.dl.common.env import create_tbwriter +import os +import random +import numpy as np +from copy import deepcopy +from utils.dl.common.model import LayerActivation, get_module +from utils.common.log import logger + + +class ElasticDNN_MDPretrainingIndexAlg(BaseAlg): + """ + TODO: fine-tuned FM -> init MD -> trained MD -> construct indexes (only between similar weights) and fine-tune + """ + def get_required_models_schema(self) -> Schema: + return Schema({ + 'fm': ElasticDNN_OfflineFMModel, + 'md': ElasticDNN_OfflineMDModel + }) + + def get_required_hyp_schema(self) -> Schema: + return Schema({ + 'launch_tbboard': bool, + + 'samples_size': (int, int, int, int), + + 'train_batch_size': int, + 'val_batch_size': int, + 'num_workers': int, + 'optimizer': str, + 'optimizer_args': dict, + 'index_optimizer_args': dict, + 'scheduler': str, + 'scheduler_args': dict, + 'num_iters': int, + 'val_freq': int, + 'max_sparsity': float, + 'min_sparsity': float, + 'l1_reg_loss_weight': float, + 'index_loss_weight': float, + 'val_num_sparsities': int, + + 'bn_cal_num_iters': int, + + 'index_1_to_k': int + }) + + def bn_cal(self, model: nn.Module, train_loader, num_iters, device): + has_bn = False + for n, m in model.named_modules(): + if isinstance(m, nn.BatchNorm2d): + has_bn = True + break + + if not has_bn: + return {} + + def bn_calibration_init(m): + """ calculating post-statistics of batch normalization """ + if getattr(m, 'track_running_stats', False): + # reset all values for post-statistics + m.reset_running_stats() + # set bn in training mode to update post-statistics + m.training = True + + with torch.no_grad(): + model.eval() + model.apply(bn_calibration_init) + for _ in range(num_iters): + x, _ = next(train_loader) + model(x.to(device)) + model.eval() + + bn_stats = {} + for n, m in model.named_modules(): + if isinstance(m, nn.BatchNorm2d): + bn_stats[n] = m + return bn_stats + + def upsample_2d_tensor(self, p: torch.Tensor, target_len: int): + assert p.dim() == 2 # regard 2d weight as (batch_size, 1d_vector_dim) + return F.upsample(p.unsqueeze(1).unsqueeze(3), + size=(target_len, 1), + mode='bilinear').squeeze(3).squeeze(1) + + def two_params_diff_fast(self, trained_p: torch.Tensor, ref_p: torch.Tensor, + index: torch.Tensor, + split_size: int): + + assert trained_p.dim() == ref_p.dim() + assert index.size(0) == trained_p.size(0) and index.size(1) == ref_p.size(0) + + # print(trained_p.size(), ref_p.size(), index.size()) + + ref_p = ref_p.detach() + if trained_p.dim() > 1: + trained_p = trained_p.flatten(1) + ref_p = ref_p.flatten(1) + + # the weight size of master DNN and foundation model may be totally different + + # MD -> FM: upsample first + # FM -> MD: downsample first + if trained_p.size(1) < ref_p.size(1): + trained_p = self.upsample_2d_tensor(trained_p, ref_p.size(1)) + + index = index.unsqueeze(-1) + # linear_combed_ref_p = (ref_p.unsqueeze(0) * index).sum(1) + # else: + + # print(trained_p.size(), ref_p.size(), index.size()) + + if split_size is None: + # old version: huge memory consumption, not recommended (although this is fastest) + # print('old version') + linear_combed_ref_p = (ref_p.unsqueeze(0) * index).sum(1) + + else: + # new version + linear_combed_ref_p = 0 + + cur_split_size = split_size + while index.size(1) % cur_split_size != 0: + cur_split_size -= 1 + # print(cur_split_size) + + for i in range(0, index.size(1), cur_split_size): + # if not isinstance(linear_combed_ref_p, int): + # print(linear_combed_ref_p.size(), ref_p.unsqueeze(0)[:, i: i + cur_split_size].size(), index[:, i: i + cur_split_size].size()) + linear_combed_ref_p += ref_p.unsqueeze(0)[:, i: i + cur_split_size] * index[:, i: i + cur_split_size] + linear_combed_ref_p = linear_combed_ref_p.sum(1) + + diff = (linear_combed_ref_p - trained_p).norm(2) ** 2 + return diff + + def two_params_partial_diff(self, trained_p: torch.Tensor, ref_p: torch.Tensor, + index): + + assert trained_p.dim() == ref_p.dim() + # assert index.size(0) == trained_p.size(0) and index.size(1) == ref_p.size(0) + + # print(trained_p.size(), ref_p.size(), index.size()) + + ref_p = ref_p.detach() + if trained_p.dim() > 1: + trained_p = trained_p.flatten(1) + ref_p = ref_p.flatten(1) + + # the weight size of master DNN and foundation model may be totally different + + # MD -> FM: upsample first + # FM -> MD: downsample first + if trained_p.size(1) < ref_p.size(1): + trained_p = self.upsample_2d_tensor(trained_p, ref_p.size(1)) + + elif trained_p.dim() == 1: + trained_p = trained_p.unsqueeze(1) + ref_p = ref_p.unsqueeze(1) + + # index = index.unsqueeze(-1) + # linear_combed_ref_p = (ref_p.unsqueeze(0) * index).sum(1) + # else: + + # res = 0. + # # slow version + # for g1i, (g1, (_index, selected_g2is)) in enumerate(zip(trained_p, index)): + # comb_ref_p = (ref_p[torch.tensor(selected_g2is)] * _index.unsqueeze(1)).sum(0) + # res += ((comb_ref_p - g1) ** 2).sum() + # return res + # ------------- + + # (train_p.size(0), 2, ref_p.size(1)) + # if trained_p.dim() == 2: + + # NOTE: fast version? + selected_ref_p = torch.stack([ref_p[torch.tensor(selected_g2is)] for _, selected_g2is in index]) + # (train_p.size(0), 2) + indexes = torch.stack([_index for _index, _ in index]) + + # print(trained_p.size(), ref_p.size(), selected_ref_p.size(), indexes.size()) + + # (train_p.size(),) + # should be (train_p.size(0), ref_p.size(1)) + linear_combed_ref_p = (selected_ref_p * indexes.unsqueeze(-1)).sum(1) + # ------------- + + # else: + # selected_ref_p = torch.stack([ref_p[torch.tensor(selected_g2is)] for _, selected_g2is in index]) + # # (train_p.size(0), 2) + # indexes = torch.stack([_index for _index, _ in index]) + + # # print(trained_p.size(), ref_p.size(), selected_ref_p.size(), indexes.size()) + + # # (train_p.size(),) + # # should be (train_p.size(0), ref_p.size(1)) + # linear_combed_ref_p = (selected_ref_p * indexes).sum(1) + + diff = (linear_combed_ref_p - trained_p).norm(2) ** 2 + return diff + + # print(trained_p.size(), ref_p.size(), index.size()) + + # if split_size is None: + # # old version: huge memory consumption, not recommended (although this is fastest) + # # print('old version') + # linear_combed_ref_p = (ref_p.unsqueeze(0) * index).sum(1) + + # else: + # # new version + # linear_combed_ref_p = 0 + + # cur_split_size = split_size + # while index.size(1) % cur_split_size != 0: + # cur_split_size -= 1 + # # print(cur_split_size) + + # for i in range(0, index.size(1), cur_split_size): + # # if not isinstance(linear_combed_ref_p, int): + # # print(linear_combed_ref_p.size(), ref_p.unsqueeze(0)[:, i: i + cur_split_size].size(), index[:, i: i + cur_split_size].size()) + # linear_combed_ref_p += ref_p.unsqueeze(0)[:, i: i + cur_split_size] * index[:, i: i + cur_split_size] + # linear_combed_ref_p = linear_combed_ref_p.sum(1) + + # diff = (linear_combed_ref_p - trained_p).norm(2) ** 2 + # return diff + + def get_index_loss(self, fm, md, indexes, match_fn): + res = 0. + + for name, p in md.named_parameters(): + if p.dim() == 0: + continue + + raw_p = match_fn(name, fm) + if raw_p is None: + continue + + index = indexes[name] + + # print(name) + res += self.two_params_partial_diff(p, raw_p, index) + return res + + def run(self, scenario: Scenario, hyps: Dict) -> Dict[str, Any]: + super().run(scenario, hyps) + + assert isinstance(self.models['md'], ElasticDNN_OfflineMDModel) # for auto completion + assert isinstance(self.models['fm'], ElasticDNN_OfflineFMModel) # for auto completion + + # 1. add FBS + device = self.models['md'].device + + # 2. train (knowledge distillation, index relationship) + offline_datasets = scenario.get_offline_datasets() + train_dataset = MergedDataset([d['train'] for d in offline_datasets.values()]) + val_dataset = MergedDataset([d['val'] for d in offline_datasets.values()]) + train_loader = iter(build_dataloader(train_dataset, hyps['train_batch_size'], hyps['num_workers'], + True, None)) + val_loader = build_dataloader(val_dataset, hyps['val_batch_size'], hyps['num_workers'], + False, False) + + # logger.info(f'init master DNN acc w/ FBS (before constructing indexes): {self.models["md"].get_accuracy(val_loader):.4f}') + + master_dnn = self.models['md'].models_dict['main'] + elastic_dnn_util = self.models['fm'].get_elastic_dnn_util() + + with torch.no_grad(): + indexes = {} + trained_indexes = [] + for name, p in self.models['md'].models_dict['main'].named_parameters(): + if p.dim() > 0: + matched_p_in_fm = self.models['md'].get_matched_param_of_fm(name, self.models['fm'].models_dict['main']) + if matched_p_in_fm is None: + continue + + if p.dim() > 1: + p = p.flatten(1) + matched_p_in_fm = matched_p_in_fm.flatten(1) + + if p.size(1) < matched_p_in_fm.size(1): + p = self.upsample_2d_tensor(p, matched_p_in_fm.size(1)) + + indexes[name] = [] + for g1i, g1 in enumerate(p): + selected_g2is = torch.randperm(matched_p_in_fm.size(0))[0: hyps['index_1_to_k']] + index = torch.FloatTensor([1. / hyps['index_1_to_k']] * hyps['index_1_to_k']).to(device) + indexes[name] += [(index, list(selected_g2is))] + trained_indexes += [index] + # for selected_g2i in selected_g2is: + # indexes[name][g1i][selected_g2i] = torch.FloatTensor([1. / hyps['index_1_to_k']]).to(device) + # indexes[name][g1i][selected_g2i].requires_grad = True + # indexes[name][g1i][selected_g2i] = 1. + # trained_indexes += [indexes[name][g1i][selected_g2i]] + + # print(p.size(), selected_g2is) + + # NOTE: only constructing indexes between similar [weight row/filter] pair + # for g1i, g1 in tqdm.tqdm(enumerate(p), dynamic_ncols=True, leave=False, total=len(p)): + # indexes[name][g1i] = {} + + # # similarities = [] + # # for g2i, g2 in enumerate(matched_p_in_fm): + # # similarity = ((g1 - g2) ** 2).sum() + # # similarities += [similarity] + # if p.dim() == 1: + # similarities = ((g1 - matched_p_in_fm) ** 2) + # else: + # similarities = ((g1.unsqueeze(0) - matched_p_in_fm) ** 2).sum(1) + # assert similarities.size(0) == matched_p_in_fm.size(0) + + # most_similar_g2is = similarities.argsort(descending=True) + # accu_similarities = similarities.sort(descending=True)[0].cumsum(0) + + # for ai, accu_sim in enumerate(accu_similarities): + # if accu_sim > accu_similarities[-1] * hyps['index_construct_r']: + # break + + # # selected fm weight rows for constructing indexes + # selected_g2is = most_similar_g2is[0: ai] + # for selected_g2i in selected_g2is: + # indexes[name][g1i][selected_g2i] = torch.FloatTensor([1. / ai]).to(device) + # indexes[name][g1i][selected_g2i].requires_grad = True + # trained_indexes += [indexes[name][g1i][selected_g2i]] + + # num_indexes += 1 + + # index_percent = num_indexes / (matched_p_in_fm.size(0) * p.size(0)) * 100. + # logger.info(f'layer {name}: constructing {index_percent:.3f}% of indexes') + + tmp_indexes_file_path = os.path.join(self.res_save_dir, 'tmp-indexes.pt') + torch.save(indexes, tmp_indexes_file_path) + logger.info(f'# indexes: {len(trained_indexes)}; generate indexes ({(os.path.getsize(tmp_indexes_file_path) / 1024**2):.3f}MB)') + os.remove(tmp_indexes_file_path) + + # 2.1 train whole master DNN (knowledge distillation) + for p in master_dnn.parameters(): + p.requires_grad = True + self.models['md'].to_train_mode() + for p in trained_indexes: + p.requires_grad = True + for p in self.models['fm'].models_dict['main'].parameters(): + p.requires_grad = False + + optimizer = torch.optim.__dict__[hyps['optimizer']]([ + {'params': self.models['md'].models_dict['main'].parameters(), **hyps['optimizer_args']}, + # {'params': trained_indexes, **hyps['index_optimizer_args']} + ]) + scheduler = torch.optim.lr_scheduler.__dict__[hyps['scheduler']](optimizer, **hyps['scheduler_args']) + tb_writer = create_tbwriter(os.path.join(self.res_save_dir, 'tb_log'), launch_tbboard=hyps['launch_tbboard']) + pbar = tqdm.tqdm(range(hyps['num_iters']), dynamic_ncols=True) + best_avg_val_acc = 0. + + for iter_index in pbar: + self.models['md'].to_train_mode() + self.models['fm'].to_eval_mode() + + rand_sparsity = random.random() * (hyps['max_sparsity'] - hyps['min_sparsity']) + hyps['min_sparsity'] + elastic_dnn_util.set_master_dnn_sparsity(self.models['md'].models_dict['main'], rand_sparsity) + + x, y = next(train_loader) + x, y = x.to(device), y.to(device) + + task_loss = self.models['md'].forward_to_get_task_loss(x, y) + l1_reg_loss = hyps['l1_reg_loss_weight'] * elastic_dnn_util.get_accu_l1_reg_of_raw_channel_attention_in_master_dnn(master_dnn) + index_loss = hyps['index_loss_weight'] * self.get_index_loss(self.models['fm'].models_dict['main'], + self.models['md'].models_dict['main'], + indexes, + self.models['md'].get_matched_param_of_fm) + total_loss = task_loss + l1_reg_loss + index_loss + + optimizer.zero_grad() + total_loss.backward() + optimizer.step() + scheduler.step() + + if (iter_index + 1) % hyps['val_freq'] == 0: + + elastic_dnn_util.clear_cached_channel_attention_in_master_dnn(self.models['md'].models_dict['main']) + + cur_md = self.models['md'].models_dict['main'] + md_for_test = deepcopy(self.models['md'].models_dict['main']) + val_accs = {} + avg_val_acc = 0. + bn_stats = {} + + for val_sparsity in np.linspace(hyps['min_sparsity'], hyps['max_sparsity'], num=hyps['val_num_sparsities']): + elastic_dnn_util.set_master_dnn_sparsity(md_for_test, val_sparsity) + bn_stats[f'{val_sparsity:.4f}'] = self.bn_cal(md_for_test, train_loader, hyps['bn_cal_num_iters'], device) + self.models['md'].models_dict['main'] = md_for_test + self.models['md'].to_eval_mode() + val_acc = self.models['md'].get_accuracy(val_loader) + + val_accs[f'{val_sparsity:.4f}'] = val_acc + avg_val_acc += val_acc + + avg_val_acc /= hyps['val_num_sparsities'] + + self.models['md'].models_dict['main'] = cur_md + self.models['md'].models_dict['bn_stats'] = bn_stats + + self.models['md'].save_model(os.path.join(self.res_save_dir, 'models/md_last.pt')) + self.models['fm'].save_model(os.path.join(self.res_save_dir, 'models/fm_last.pt')) + + if avg_val_acc > best_avg_val_acc: + best_avg_val_acc = avg_val_acc + self.models['md'].save_model(os.path.join(self.res_save_dir, 'models/md_best.pt')) + self.models['fm'].save_model(os.path.join(self.res_save_dir, 'models/fm_best.pt')) + + tb_writer.add_scalars(f'losses', dict(task=task_loss, l1_reg=l1_reg_loss, index=index_loss, total=total_loss), iter_index) + pbar.set_description(f'loss: {total_loss:.6f}, task_loss: {task_loss:.6f}, index_loss: {index_loss:.6f}, ' + f'l1_loss: {l1_reg_loss:.6f}') + if (iter_index + 1) >= hyps['val_freq']: + tb_writer.add_scalars(f'accs/val_accs', val_accs, iter_index) + tb_writer.add_scalar(f'accs/avg_val_acc', avg_val_acc, iter_index) + pbar.set_description(f'loss: {total_loss:.6f}, task_loss: {task_loss:.6f}, index_loss: {index_loss:.6f}, ' + f'l1_loss: {l1_reg_loss:.6f}, avg_val_acc: {avg_val_acc:.4f}') + \ No newline at end of file diff --git a/new_impl/cv/elasticdnn/api/algs/md_pretraining_index_cnn.py b/new_impl/cv/elasticdnn/api/algs/md_pretraining_index_cnn.py new file mode 100644 index 0000000000000000000000000000000000000000..b32c64305054eb82a09df3ea63662663d5a919a9 --- /dev/null +++ b/new_impl/cv/elasticdnn/api/algs/md_pretraining_index_cnn.py @@ -0,0 +1,647 @@ +from typing import Any, Dict +from schema import Schema, Or +import schema +from data import Scenario, MergedDataset +from new_impl.cv.base.alg import BaseAlg +from data import build_dataloader +from ..model import ElasticDNN_OfflineFMModel, ElasticDNN_OfflineMDModel +from ...model.base import ElasticDNNUtil +import torch.optim +import tqdm +import torch.nn.functional as F +from torch import nn +from utils.dl.common.env import create_tbwriter +import os +import random +import numpy as np +from copy import deepcopy +from utils.dl.common.model import LayerActivation, get_module +from utils.common.log import logger +from new_impl.cv.elasticdnn.model.base import KTakesAll +from new_impl.cv.resnet.model_fbs import boost_raw_model_with_filter_selection,set_pruning_rate + +# class WeightAffine(nn.Module): +# def __init__(self, a, b, r=16): +# super(WeightAffine, self).__init__() + +# self.a = a +# self.b = b + +# self.w1 = nn.Parameter(torch.rand((b // r, a))) +# self.w2 = nn.Parameter(torch.rand((b, b // r))) + +# self.a_to_b = True + +# nn.init.zeros_(self.w1.data) +# nn.init.zeros_(self.w2.data) + +# def forward1(self, x): +# return F.linear(F.linear(x, self.w1), self.w2) + +# def forward2(self, x): +# return F.linear(F.linear(x, self.w2.T), self.w1.T) + +# def forward(self, x): +# return self.forward1(x) if self.a_to_b else self.forward2(x) + + +class ElasticDNN_MDPretrainingIndexAlg(BaseAlg): + """ + construct indexes between a filter/row of MD and all filters/rows of FM in the same layer + too huge indexes (~1GB), train so slow, hard to optimize + """ + def get_required_models_schema(self) -> Schema: + return Schema({ + 'fm': ElasticDNN_OfflineFMModel, + 'md': ElasticDNN_OfflineMDModel + }) + + def get_required_hyp_schema(self) -> Schema: + return Schema({ + 'launch_tbboard': bool, + + 'samples_size': object, + + 'FBS_r': int, + 'FBS_ignore_layers': [str], + + 'train_batch_size': int, + 'val_batch_size': int, + 'num_workers': int, + 'optimizer': str, + 'optimizer_args': dict, + 'indexes_optimizer_args': dict, + 'scheduler': str, + 'scheduler_args': dict, + 'num_iters': int, + 'val_freq': int, + 'max_sparsity': float, + 'min_sparsity': float, + 'l1_reg_loss_weight': float, + 'index_loss_weight': float, + 'val_num_sparsities': int, + + 'bn_cal_num_iters': int, + + 'index_init': str, + 'index_guided_linear_comb_split_size': Or(int, None) + }) + + # def upsample_2d_tensor(self, p: torch.Tensor, target_len: int, weight_affine: WeightAffine): + # assert p.dim() == 2 # regard 2d weight as (batch_size, 1d_vector_dim) + # # return F.upsample(p.unsqueeze(1).unsqueeze(3), + # # size=(target_len, 1), + # # mode='bilinear').squeeze(3).squeeze(1) + # weight_affine.a_to_b = True + # return weight_affine(p) + + # def downsample_2d_tensor(self, p: torch.Tensor, target_len: int, weight_affine: WeightAffine): + # assert p.dim() == 2 # regard 2d weight as (batch_size, 1d_vector_dim) + # # return F.interpolate(p.unsqueeze(1).unsqueeze(3), + # # size=(target_len, 1), + # # mode='bilinear').squeeze(3).squeeze(1) + # weight_affine.a_to_b = False + # return weight_affine(p) + def two_params_diff_fast(self,trained_p: torch.Tensor, ref_p: torch.Tensor, affine_p: torch.Tensor): + # with torch.cuda.stream(cuda_stream): + assert trained_p.size() == ref_p.size() + assert affine_p.size(0) == trained_p.size(0) and affine_p.size(1) == ref_p.size(0) + + ref_p = ref_p.detach() + if trained_p.dim() > 1: + trained_p = trained_p.flatten(1) + ref_p = ref_p.flatten(1) + + # print(trained_p.size(), ref_p.size()) + # diff = 0. + # for trained_p_i, (trained_p_item, affine_p_item) in enumerate(zip(trained_p, affine_p)): + # linear_combed_ref_p_item = affine_p_item[0:-1].unsqueeze(1).unsqueeze(2).unsqueeze(3) * ref_p + affine_p_item[-1] + # diff += (linear_combed_ref_p_item - trained_p_item).norm(2) ** 2 + # return diff + # affine_p = torch.stack([affine_p] * trained_p.size(1), dim=1) + affine_p = affine_p.unsqueeze(-1) + # print(affine_p.size()) + + # linear_combed_ref_p = (ref_p.unsqueeze(-1) * affine_p[:, :, 0:-1]).sum(dim=(-1,)) + linear_combed_ref_p = (ref_p.unsqueeze(0) * affine_p).sum(1) + + else: + # print(ref_p.size(), affine_p.size(), trained_p.size()) + # + linear_combed_ref_p = (ref_p.unsqueeze(0) * affine_p).sum(1) + # print(linear_combed_ref_p.size()) + # return + + diff = (linear_combed_ref_p - trained_p).norm(2) ** 2 + return diff + + def two_params_diff_fast_no_sample(self, trained_p: torch.Tensor, ref_p: torch.Tensor, + index: torch.Tensor, + split_size: int): + + assert trained_p.dim() == ref_p.dim() + + if trained_p.dim() > 1 and index.size(0) == trained_p.size(1) and index.size(1) == ref_p.size(1): + assert trained_p.dim() == 2 + trained_p = trained_p.T + ref_p = ref_p.T + + assert index.size(0) == trained_p.size(0) and index.size(1) == ref_p.size(0) + + # print(trained_p.size(), ref_p.size(), index.size()) + + ref_p = ref_p.detach() + if trained_p.dim() > 1: + trained_p = trained_p.flatten(1) + ref_p = ref_p.flatten(1) + + # the weight size of master DNN and foundation model may be totally different + + # MD -> FM: upsample first + # FM -> MD: downsample first + # if trained_p.size(1) < ref_p.size(1): + # trained_p = self.upsample_2d_tensor(trained_p, ref_p.size(1), weight_affine) + + index = index.unsqueeze(-1) + # linear_combed_ref_p = (ref_p.unsqueeze(0) * index).sum(1) + # else: + + # print(trained_p.size(), ref_p.size(), index.size()) + + if split_size is None: + # old version: huge memory consumption, not recommended (although this is fastest) + # print('old version') + linear_combed_ref_p = (ref_p.unsqueeze(0) * index).sum(1) + + else: + # new version + linear_combed_ref_p = 0 + + cur_split_size = split_size + while index.size(1) % cur_split_size != 0: + cur_split_size -= 1 + # print(cur_split_size) + + for i in range(0, index.size(1), cur_split_size): + # if not isinstance(linear_combed_ref_p, int): + # print(linear_combed_ref_p.size(), ref_p.unsqueeze(0)[:, i: i + cur_split_size].size(), index[:, i: i + cur_split_size].size()) + linear_combed_ref_p += ref_p.unsqueeze(0)[:, i: i + cur_split_size] * index[:, i: i + cur_split_size] + linear_combed_ref_p = linear_combed_ref_p.sum(1) + + diff = (linear_combed_ref_p - trained_p).norm(2) ** 2 + return diff + + # def two_params_diff_fast_including_upsample(self, trained_p: torch.Tensor, ref_p: torch.Tensor, + # index: torch.Tensor, + # split_size: int, weight_affine: WeightAffine): + + # assert trained_p.dim() == ref_p.dim() + # assert index.size(0) == trained_p.size(0) and index.size(1) == ref_p.size(0) + + # # print(trained_p.size(), ref_p.size(), index.size()) + + # ref_p = ref_p.detach() + # if trained_p.dim() > 1: + # trained_p = trained_p.flatten(1) + # ref_p = ref_p.flatten(1) + + # # the weight size of master DNN and foundation model may be totally different + + # # MD -> FM: upsample first + # # FM -> MD: downsample first + # if trained_p.size(1) < ref_p.size(1): + # trained_p = self.upsample_2d_tensor(trained_p, ref_p.size(1), weight_affine) + + # index = index.unsqueeze(-1) + # # linear_combed_ref_p = (ref_p.unsqueeze(0) * index).sum(1) + # # else: + + # # print(trained_p.size(), ref_p.size(), index.size()) + + # if split_size is None: + # # old version: huge memory consumption, not recommended (although this is fastest) + # # print('old version') + # linear_combed_ref_p = (ref_p.unsqueeze(0) * index).sum(1) + + # else: + # # new version + # linear_combed_ref_p = 0 + + # cur_split_size = split_size + # while index.size(1) % cur_split_size != 0: + # cur_split_size -= 1 + # # print(cur_split_size) + + # for i in range(0, index.size(1), cur_split_size): + # # if not isinstance(linear_combed_ref_p, int): + # # print(linear_combed_ref_p.size(), ref_p.unsqueeze(0)[:, i: i + cur_split_size].size(), index[:, i: i + cur_split_size].size()) + # linear_combed_ref_p += ref_p.unsqueeze(0)[:, i: i + cur_split_size] * index[:, i: i + cur_split_size] + # linear_combed_ref_p = linear_combed_ref_p.sum(1) + + # diff = (linear_combed_ref_p - trained_p).norm(2) ** 2 + # return diff + + # def two_params_diff_fast_including_downsample(self, trained_p: torch.Tensor, ref_p: torch.Tensor, + # index: torch.Tensor, + # split_size: int, weight_affine: WeightAffine): + + # assert trained_p.dim() == ref_p.dim() + # assert index.size(0) == trained_p.size(0) and index.size(1) == ref_p.size(0) + + # # print(trained_p.size(), ref_p.size(), index.size()) + + # ref_p = ref_p.detach() + # if trained_p.dim() > 1: + # trained_p = trained_p.flatten(1) + # ref_p = ref_p.flatten(1) + + # # the weight size of master DNN and foundation model may be totally different + + # # MD -> FM: upsample first + # # FM -> MD: downsample first + # if trained_p.size(1) < ref_p.size(1): + # # trained_p = self.upsample_2d_tensor(trained_p, ref_p.size(1)) + # ref_p = self.downsample_2d_tensor(ref_p, trained_p.size(1), weight_affine) + + # index = index.unsqueeze(-1) + # # linear_combed_ref_p = (ref_p.unsqueeze(0) * index).sum(1) + # # else: + + # # print(trained_p.size(), ref_p.size(), index.size()) + + # if split_size is None: + # # old version: huge memory consumption, not recommended (although this is fastest) + # # print('old version') + # linear_combed_ref_p = (ref_p.unsqueeze(0) * index).sum(1) + + # else: + # # new version + # linear_combed_ref_p = 0 + + # cur_split_size = split_size + # while index.size(1) % cur_split_size != 0: + # cur_split_size -= 1 + # # print(cur_split_size) + + # for i in range(0, index.size(1), cur_split_size): + # # if not isinstance(linear_combed_ref_p, int): + # # print(linear_combed_ref_p.size(), ref_p.unsqueeze(0)[:, i: i + cur_split_size].size(), index[:, i: i + cur_split_size].size()) + # linear_combed_ref_p += ref_p.unsqueeze(0)[:, i: i + cur_split_size] * index[:, i: i + cur_split_size] + # linear_combed_ref_p = linear_combed_ref_p.sum(1) + + # diff = (linear_combed_ref_p - trained_p).norm(2) ** 2 + # return diff + + def get_index_loss(self, fm, md, indexes, match_fn, split_size): + res = 0. + + # for name, p in md.named_parameters(): + # if name not in indexes.keys(): + # continue + # # if p.dim() == 0: + # # continue + + # raw_p = match_fn(name, fm) + # # if raw_p is None: + # # continue + + # index = indexes[name] + for name, p in md.named_parameters(): + if p.dim() == 0: + continue + if 'filter_selection_module' in name: + continue + + # print(name) + + raw_name = name if 'raw_conv2d' not in name else name.replace('raw_conv2d.', '') + + if 'bn' in name: + raw_name = match_fn[raw_name[0: raw_name.index('.bn')]] + '.' + raw_name.split('.')[-1] + # print(name, raw_name) + raw_p = getattr( + get_module(fm, '.'.join(raw_name.split('.')[0:-1])), + raw_name.split('.')[-1] + ) + + index = indexes[raw_name] + # print(name) + # res += (self.two_params_diff_fast_including_upsample(p, raw_p, index, split_size, weight_affines[name]) + \ + # self.two_params_diff_fast_including_downsample(p, raw_p, index, split_size, weight_affines[name])) / 2. + + res += self.two_params_diff_fast(p, raw_p, index) + + return res + + def bn_cal(self, model: nn.Module, train_loader, num_iters, device): + has_bn = False + for n, m in model.named_modules(): + if isinstance(m, nn.BatchNorm2d): + has_bn = True + break + + if not has_bn: + return {} + + def bn_calibration_init(m): + """ calculating post-statistics of batch normalization """ + if getattr(m, 'track_running_stats', False): + # reset all values for post-statistics + m.reset_running_stats() + # set bn in training mode to update post-statistics + m.training = True + + with torch.no_grad(): + model.eval() + model.apply(bn_calibration_init) + for _ in range(num_iters): + x, _ = next(train_loader) + model(x.to(device)) + model.eval() + + bn_stats = {} + for n, m in model.named_modules(): + if isinstance(m, nn.BatchNorm2d): + bn_stats[n] = m + return bn_stats + + def run(self, scenario: Scenario, hyps: Dict) -> Dict[str, Any]: + super().run(scenario, hyps) + + # sanity check + # a= torch.tensor([[1, 2, 3], [1, 2, 4]]) + # index = torch.tensor([[1, 2, 3], + # [1, 2, 4]]) + # b = torch.tensor([[1, 2, 3], [1, 2, 4], [2, 3, 4]]) + # print(self.two_params_diff_fast(a, b, index, hyps['index_guided_linear_comb_split_size'])) + + assert isinstance(self.models['md'], ElasticDNN_OfflineMDModel) # for auto completion + assert isinstance(self.models['fm'], ElasticDNN_OfflineFMModel) # for auto completion + + # 1. add FBS + device = self.models['md'].device + + # logger.info(f'init master DNN by reducing width of an adapted foundation model (already tuned by LoRA)...') + + # before_fm_model = deepcopy(self.models['fm'].models_dict['main']) + # lora_util = self.models['fm'].get_lora_util() + # lora_absorbed_fm_model = lora_util.absorb_lora_and_recover_net_structure(self.models['fm'].models_dict['main'], + # torch.rand(hyps['samples_size']).to(device)) + # self.models['fm'].models_dict['main'] = lora_absorbed_fm_model + # master_dnn = self.models['fm'].generate_md_by_reducing_width(hyps['generate_md_width_ratio'], + # torch.rand(hyps['samples_size']).to(device)) + # self.models['fm'].models_dict['main'] = before_fm_model + + # 2. train (knowledge distillation, index relationship) + offline_datasets = scenario.get_offline_datasets() + train_dataset = MergedDataset([d['train'] for d in offline_datasets.values()]) + val_dataset = MergedDataset([d['val'] for d in offline_datasets.values()]) + train_loader = iter(build_dataloader(train_dataset, hyps['train_batch_size'], hyps['num_workers'], + True, None)) + val_loader = build_dataloader(val_dataset, hyps['val_batch_size'], hyps['num_workers'], + False, False) + + #logger.info(f'master DNN acc before inserting FBS: {self.models["md"].get_accuracy(val_loader):.4f}') + + elastic_dnn_util = self.models['fm'].get_elastic_dnn_util() + + + master_dnn = self.models['md'].models_dict['main'] + elastic_dnn_util = self.models['fm'].get_elastic_dnn_util() + #master_dnn = elastic_dnn_util.convert_raw_dnn_to_master_dnn_with_perf_test(master_dnn, hyps['FBS_r'], hyps['FBS_ignore_layers']).to(device) + + pruned_layers = [] + # for i,block_1 in enumerate(master_dnn.resnet.encoder.stages): + # for j,block_2 in enumerate(block_1.layers): + # for k,block_3 in enumerate(block_2.layer): + # pruned_layers += [f'resnet.encoder.stages.{i}.layers.{j}.layer.{k}.convolution'] + + for i in range(0,3): + pruned_layers += [f'resnet.encoder.stages.{i}.layers.0.layer.0.convolution'] + indexes = {} + for name, p in self.models['md'].models_dict['main'].named_parameters(): + if p.dim() > 0: + indexes[name] = torch.zeros((p.size(0), p.size(0))).to(device) # e.g. for each neuron, y = a_1x_1 + a_2x_2 + b + indexes[name].requires_grad = True + #pruned_layers = ['resnet.encoder.stages.0.layers.0.layer.0.convolution'] + ignore_layers = [layer for layer, m in master_dnn.named_modules() if isinstance(m, nn.Conv2d) and layer not in pruned_layers] + + #ignore_layers = [] + master_dnn,conv_bn_map = boost_raw_model_with_filter_selection(master_dnn, 0., False, ignore_layers, True, (1,3,224,224)) + self.models['md'].models_dict['main'] = master_dnn + #logger.info(f'master DNN acc before inserting FBS: {self.models["md"].get_accuracy(val_loader):.4f}') + # master_dnn = elastic_dnn_util.convert_raw_dnn_to_master_dnn_with_perf_test(master_dnn, + # hyps['FBS_r'], hyps['FBS_ignore_layers']) + # self.models['md'].models_dict['main'] = master_dnn + # self.models['md'].to(device) + # master_dnn = self.models['md'].models_dict['main'] + + + + + + # 2.1 train only FBS (skipped because current md cannot do proper inference) + + # 2.2 train whole master DNN (knowledge distillation, index relationship) + for p in master_dnn.parameters(): + p.requires_grad = True + self.models['md'].to_train_mode() + + _norm = lambda a: (a.T / a.T.sum(0)).T + + + # weight_affines = {} + # weight_affines_trained_p = [] + # for name, p in self.models['md'].models_dict['main'].named_parameters(): + # # if p.dim() > 1: + # # print(name) + # #if('attention.attention.projection_query' in name): print(name) + # logger.debug(f'try: layer {name}, {p.size()}') + # matched_p_in_fm = self.models['md'].get_matched_param_of_fm(name, self.models['fm'].models_dict['main']) + # if matched_p_in_fm is None: + # logger.debug(f'layer {name} no matched fm param') + # continue + # logger.debug(f'layer {name} matched fm param: {matched_p_in_fm.size()}') + # if p.dim() == 1: + # # assert p.size(0) == matched_p_in_fm.size(0), f'{p.size()}, {matched_p_in_fm.size()}' + + # if hyps['index_init'] == 'rand_norm': + # indexes[name] = _norm(torch.rand((p.size(0), matched_p_in_fm.size(0))).to(device)) + # elif hyps['index_init'] == 'zero': + # indexes[name] = torch.zeros((p.size(0), matched_p_in_fm.size(0))).to(device) + # elif hyps['index_init'] == 'randn_norm': + # indexes[name] = _norm(torch.randn((p.size(0), matched_p_in_fm.size(0))).to(device)) + # else: + # raise NotImplementedError + # logger.info(f'construct index {indexes[name].size()} in layer {name} | dim 0') + + # elif p.dim() == 2: + # assert p.size(0) == matched_p_in_fm.size(0) or p.size(1) == matched_p_in_fm.size(1), f'{p.size()}, {matched_p_in_fm.size()}' + + # if p.size(0) == matched_p_in_fm.size(0): + # if hyps['index_init'] == 'rand_norm': + # indexes[name] = _norm(torch.rand((p.size(1), matched_p_in_fm.size(1))).to(device)) + # elif hyps['index_init'] == 'zero': + # indexes[name] = torch.zeros((p.size(1), matched_p_in_fm.size(1))).to(device) + # elif hyps['index_init'] == 'randn_norm': + # indexes[name] = _norm(torch.randn((p.size(1), matched_p_in_fm.size(1))).to(device)) + # else: + # raise NotImplementedError + # logger.info(f'construct index {indexes[name].size()} in layer {name} | dim 1') + + # elif p.size(1) == matched_p_in_fm.size(1): + # if hyps['index_init'] == 'rand_norm': + # indexes[name] = _norm(torch.rand((p.size(0), matched_p_in_fm.size(0))).to(device)) + # elif hyps['index_init'] == 'zero': + # indexes[name] = torch.zeros((p.size(0), matched_p_in_fm.size(0))).to(device) + # elif hyps['index_init'] == 'randn_norm': + # indexes[name] = _norm(torch.randn((p.size(0), matched_p_in_fm.size(0))).to(device)) + # else: + # raise NotImplementedError + # logger.info(f'construct index {indexes[name].size()} in layer {name} | dim 0') + # else: + # raise NotImplementedError + + + # indexes[name].requires_grad = True + + #print() + # for k,v in indexes.items(): + # print(k) + tmp_indexes_file_path = os.path.join(self.res_save_dir, 'tmp-indexes.pt') + torch.save([indexes], tmp_indexes_file_path) + logger.info(f'generate indexes ({(os.path.getsize(tmp_indexes_file_path) / 1024**2):.3f}MB)') + os.remove(tmp_indexes_file_path) + res = [] + #print(self.models['md'].models_dict['main']) + # for n ,m in self.models['md'].models_dict['main'].named_modules(): + # if n.endswith('intermediate.dense.linear'): + # for p in m.parameters(): + # p.requires_grad = True + # res += [p] + # elif n.endswith('intermediate.dense.fbs'): + # for p in m: + # if isinstance(p,nn.Linear): + # for p1 in p.parameters(): + # p1.requires_grad = True + # res += [p1] + # else: + # for p in m.parameters(): + # p.requires_grad = False + optimizer = torch.optim.__dict__[hyps['optimizer']]([ + {'params': self.models['md'].models_dict['main'].parameters(), **hyps['optimizer_args']}, + #{'params': res, **hyps['optimizer_args']}, + {'params': [v for v in indexes.values()], **hyps['indexes_optimizer_args']} + ]) + scheduler = torch.optim.lr_scheduler.__dict__[hyps['scheduler']](optimizer, **hyps['scheduler_args']) + tb_writer = create_tbwriter(os.path.join(self.res_save_dir, 'tb_log'), launch_tbboard=hyps['launch_tbboard']) + pbar = tqdm.tqdm(range(hyps['num_iters']), dynamic_ncols=True) + best_avg_val_acc = 0. + #logger.info(f'master DNN acc before inserting FBS: {self.models["md"].get_accuracy(val_loader):.4f}') + for iter_index in pbar: + self.models['md'].to_train_mode() + self.models['fm'].to_eval_mode() + + rand_sparsity = random.random() * (hyps['max_sparsity'] - hyps['min_sparsity']) + hyps['min_sparsity'] + #elastic_dnn_util.set_master_dnn_sparsity(self.models['md'].models_dict['main'], rand_sparsity) + set_pruning_rate(self.models['md'].models_dict['main'], rand_sparsity) + x, y = next(train_loader) + #x,y,_ = next(train_loader) + # if isinstance(x, dict): + # for k, v in x.items(): + # if isinstance(v, torch.Tensor): + # x[k] = v.to(device) + # y = y.to(device) + # else: + # x, y = x.to(device), y.to(device) + if isinstance(x, dict) and isinstance(y,torch.Tensor): + for k, v in x.items(): + if isinstance(v, torch.Tensor): + x[k] = v.to(device) + y = y.to(device) + elif isinstance(x,dict) and isinstance(y,dict): + for k, v in x.items(): + if isinstance(v, torch.Tensor): + x[k] = v.to(device) + for k, v in y.items(): + if isinstance(v, torch.Tensor): + y[k] = v.to(device) + else: + x, y = x.to(device), y.to(device) + + task_loss = self.models['md'].forward_to_get_task_loss(x, y) + l1_reg_loss = hyps['l1_reg_loss_weight'] * elastic_dnn_util.get_accu_l1_reg_of_raw_channel_attention_in_master_dnn(master_dnn) + index_loss = hyps['index_loss_weight'] * self.get_index_loss(self.models['fm'].models_dict['main'], + self.models['md'].models_dict['main'], + indexes, + #self.models['md'].get_matched_param_of_fm, + conv_bn_map, + hyps['index_guided_linear_comb_split_size']) + total_loss = task_loss + index_loss + l1_reg_loss + # for layer in self.models['md'].models_dict['main'].modules(): + # # if isinstance(layer, DomainDynamicConv2d): + # if layer.__class__.__name__ == 'DomainDynamicConv2d': + # layer.static_w = None + # with torch.no_grad(): + # # ResNetCIFARManager.forward(model, x) + # self.models['md'].models_dict['main'](x) + # for layer in self.models['md'].models_dict['main'].modules(): + # # if isinstance(layer, DomainDynamicConv2d): + # if layer.__class__.__name__ == 'DomainDynamicConv2d': + # layer.static_w = layer.cached_w[0].squeeze().detach() + + optimizer.zero_grad() + total_loss.backward() + optimizer.step() + scheduler.step() + + # for layer in self.models['md'].models_dict['main'].modules(): + # # if isinstance(layer, DomainDynamicConv2d): + # if layer.__class__.__name__ == 'DomainDynamicConv2d': + # layer.static_w = None + + if (iter_index + 1) % hyps['val_freq'] == 0: + + elastic_dnn_util.clear_cached_channel_attention_in_master_dnn(self.models['md'].models_dict['main']) + + cur_md = self.models['md'].models_dict['main'] + md_for_test = self.models['md'].models_dict['main'] + val_accs = {} + avg_val_acc = 0. + bn_stats = {} + #logger.info(f'master DNN acc after inserting FBS: {self.models["md"].get_accuracy(val_loader):.4f}') + for val_sparsity in np.linspace(hyps['min_sparsity'], hyps['max_sparsity'], num=hyps['val_num_sparsities']): + #elastic_dnn_util.set_master_dnn_sparsity(md_for_test, val_sparsity) + set_pruning_rate(md_for_test, val_sparsity) + bn_stats[f'{val_sparsity:.4f}'] = self.bn_cal(md_for_test, train_loader, hyps['bn_cal_num_iters'], device) + self.models['md'].models_dict['main'] = md_for_test + self.models['md'].to_eval_mode() + val_acc = self.models['md'].get_accuracy(val_loader) + #print(self.models['md'].models_dict['main']) + val_accs[f'{val_sparsity:.4f}'] = val_acc + avg_val_acc += val_acc + + avg_val_acc /= hyps['val_num_sparsities'] + + #logger.info(f'master DNN acc before inserting FBS: {self.models["md"].get_accuracy(val_loader):.4f}') + self.models['md'].models_dict['main'] = cur_md + self.models['md'].models_dict['indexes'] = indexes + self.models['md'].models_dict['bn_stats'] = bn_stats + self.models['fm'].models_dict['indexes'] = indexes + + self.models['md'].save_model(os.path.join(self.res_save_dir, 'models/md_last.pt')) + self.models['fm'].save_model(os.path.join(self.res_save_dir, 'models/fm_last.pt')) + + if avg_val_acc > best_avg_val_acc: + best_avg_val_acc = avg_val_acc + self.models['md'].save_model(os.path.join(self.res_save_dir, 'models/md_best.pt')) + self.models['fm'].save_model(os.path.join(self.res_save_dir, 'models/fm_best.pt')) + + tb_writer.add_scalars(f'losses', dict(task=task_loss, index=index_loss, l1=l1_reg_loss, total=total_loss), iter_index) + pbar.set_description(f'loss: {total_loss:.6f}, task_loss: {task_loss:.6f}, index_loss: {index_loss:.6f}, l1_loss: {l1_reg_loss:.6f}') + if (iter_index + 1) >= hyps['val_freq']: + tb_writer.add_scalars(f'accs/val_accs', val_accs, iter_index) + tb_writer.add_scalar(f'accs/avg_val_acc', avg_val_acc, iter_index) + pbar.set_description(f'loss: {total_loss:.6f}, task_loss: {task_loss:.6f}, index_loss: {index_loss:.6f}, l1_loss: {l1_reg_loss:.6f}, ' + f'avg_val_acc: {avg_val_acc:.4f}') + \ No newline at end of file diff --git a/new_impl/cv/elasticdnn/api/algs/md_pretraining_index_v1_train_index_and_md.py b/new_impl/cv/elasticdnn/api/algs/md_pretraining_index_v1_train_index_and_md.py new file mode 100644 index 0000000000000000000000000000000000000000..3eddb2d243b14d3898c6212bcba1b4f94cd045a8 --- /dev/null +++ b/new_impl/cv/elasticdnn/api/algs/md_pretraining_index_v1_train_index_and_md.py @@ -0,0 +1,501 @@ +from typing import Any, Dict +from schema import Schema, Or +import schema +from data import Scenario, MergedDataset +from methods.base.alg import BaseAlg +from data import build_dataloader +from ..model import ElasticDNN_OfflineFMModel, ElasticDNN_OfflineMDModel +from ...model.base import ElasticDNNUtil +import torch.optim +import tqdm +import torch.nn.functional as F +from torch import nn +from utils.dl.common.env import create_tbwriter +import os +import random +import numpy as np +from copy import deepcopy +from utils.dl.common.model import LayerActivation, get_module +from utils.common.log import logger + + +class WeightAffine(nn.Module): + def __init__(self, a, b, r=16): + super(WeightAffine, self).__init__() + + self.a = a + self.b = b + + self.w1 = nn.Parameter(torch.rand((b // r, a))) + self.w2 = nn.Parameter(torch.rand((b, b // r))) + + self.a_to_b = True + + nn.init.zeros_(self.w1.data) + nn.init.zeros_(self.w2.data) + + def forward1(self, x): + return F.linear(F.linear(x, self.w1), self.w2) + + def forward2(self, x): + return F.linear(F.linear(x, self.w2.T), self.w1.T) + + def forward(self, x): + return self.forward1(x) if self.a_to_b else self.forward2(x) + + +class ElasticDNN_MDPretrainingIndexAlg(BaseAlg): + """ + construct indexes between a filter/row of MD and all filters/rows of FM in the same layer + too huge indexes (~1GB), train so slow, hard to optimize + """ + def get_required_models_schema(self) -> Schema: + return Schema({ + 'fm': ElasticDNN_OfflineFMModel, + 'md': ElasticDNN_OfflineMDModel + }) + + def get_required_hyp_schema(self) -> Schema: + return Schema({ + 'launch_tbboard': bool, + + 'samples_size': (int, int, int, int), + + 'train_batch_size': int, + 'val_batch_size': int, + 'num_workers': int, + 'optimizer': str, + 'optimizer_args': dict, + 'indexes_optimizer_args': dict, + 'scheduler': str, + 'scheduler_args': dict, + 'num_iters': int, + 'val_freq': int, + 'max_sparsity': float, + 'min_sparsity': float, + # 'l1_reg_loss_weight': float, + 'index_loss_weight': float, + 'val_num_sparsities': int, + + 'bn_cal_num_iters': int, + + 'index_init': str, + 'index_guided_linear_comb_split_size': Or(int, None) + }) + + # def upsample_2d_tensor(self, p: torch.Tensor, target_len: int, weight_affine: WeightAffine): + # assert p.dim() == 2 # regard 2d weight as (batch_size, 1d_vector_dim) + # # return F.upsample(p.unsqueeze(1).unsqueeze(3), + # # size=(target_len, 1), + # # mode='bilinear').squeeze(3).squeeze(1) + # weight_affine.a_to_b = True + # return weight_affine(p) + + # def downsample_2d_tensor(self, p: torch.Tensor, target_len: int, weight_affine: WeightAffine): + # assert p.dim() == 2 # regard 2d weight as (batch_size, 1d_vector_dim) + # # return F.interpolate(p.unsqueeze(1).unsqueeze(3), + # # size=(target_len, 1), + # # mode='bilinear').squeeze(3).squeeze(1) + # weight_affine.a_to_b = False + # return weight_affine(p) + + def two_params_diff_fast_no_sample(self, trained_p: torch.Tensor, ref_p: torch.Tensor, + index: torch.Tensor, + split_size: int): + + assert trained_p.dim() == ref_p.dim() + + if trained_p.dim() > 1 and index.size(0) == trained_p.size(1) and index.size(1) == ref_p.size(1): + assert trained_p.dim() == 2 + trained_p = trained_p.T + ref_p = ref_p.T + + assert index.size(0) == trained_p.size(0) and index.size(1) == ref_p.size(0) + + # print(trained_p.size(), ref_p.size(), index.size()) + + ref_p = ref_p.detach() + if trained_p.dim() > 1: + trained_p = trained_p.flatten(1) + ref_p = ref_p.flatten(1) + + # the weight size of master DNN and foundation model may be totally different + + # MD -> FM: upsample first + # FM -> MD: downsample first + # if trained_p.size(1) < ref_p.size(1): + # trained_p = self.upsample_2d_tensor(trained_p, ref_p.size(1), weight_affine) + + index = index.unsqueeze(-1) + # linear_combed_ref_p = (ref_p.unsqueeze(0) * index).sum(1) + # else: + + # print(trained_p.size(), ref_p.size(), index.size()) + + if split_size is None: + # old version: huge memory consumption, not recommended (although this is fastest) + # print('old version') + linear_combed_ref_p = (ref_p.unsqueeze(0) * index).sum(1) + + else: + # new version + linear_combed_ref_p = 0 + + cur_split_size = split_size + while index.size(1) % cur_split_size != 0: + cur_split_size -= 1 + # print(cur_split_size) + + for i in range(0, index.size(1), cur_split_size): + # if not isinstance(linear_combed_ref_p, int): + # print(linear_combed_ref_p.size(), ref_p.unsqueeze(0)[:, i: i + cur_split_size].size(), index[:, i: i + cur_split_size].size()) + linear_combed_ref_p += ref_p.unsqueeze(0)[:, i: i + cur_split_size] * index[:, i: i + cur_split_size] + linear_combed_ref_p = linear_combed_ref_p.sum(1) + + diff = (linear_combed_ref_p - trained_p).norm(2) ** 2 + return diff + + # def two_params_diff_fast_including_upsample(self, trained_p: torch.Tensor, ref_p: torch.Tensor, + # index: torch.Tensor, + # split_size: int, weight_affine: WeightAffine): + + # assert trained_p.dim() == ref_p.dim() + # assert index.size(0) == trained_p.size(0) and index.size(1) == ref_p.size(0) + + # # print(trained_p.size(), ref_p.size(), index.size()) + + # ref_p = ref_p.detach() + # if trained_p.dim() > 1: + # trained_p = trained_p.flatten(1) + # ref_p = ref_p.flatten(1) + + # # the weight size of master DNN and foundation model may be totally different + + # # MD -> FM: upsample first + # # FM -> MD: downsample first + # if trained_p.size(1) < ref_p.size(1): + # trained_p = self.upsample_2d_tensor(trained_p, ref_p.size(1), weight_affine) + + # index = index.unsqueeze(-1) + # # linear_combed_ref_p = (ref_p.unsqueeze(0) * index).sum(1) + # # else: + + # # print(trained_p.size(), ref_p.size(), index.size()) + + # if split_size is None: + # # old version: huge memory consumption, not recommended (although this is fastest) + # # print('old version') + # linear_combed_ref_p = (ref_p.unsqueeze(0) * index).sum(1) + + # else: + # # new version + # linear_combed_ref_p = 0 + + # cur_split_size = split_size + # while index.size(1) % cur_split_size != 0: + # cur_split_size -= 1 + # # print(cur_split_size) + + # for i in range(0, index.size(1), cur_split_size): + # # if not isinstance(linear_combed_ref_p, int): + # # print(linear_combed_ref_p.size(), ref_p.unsqueeze(0)[:, i: i + cur_split_size].size(), index[:, i: i + cur_split_size].size()) + # linear_combed_ref_p += ref_p.unsqueeze(0)[:, i: i + cur_split_size] * index[:, i: i + cur_split_size] + # linear_combed_ref_p = linear_combed_ref_p.sum(1) + + # diff = (linear_combed_ref_p - trained_p).norm(2) ** 2 + # return diff + + # def two_params_diff_fast_including_downsample(self, trained_p: torch.Tensor, ref_p: torch.Tensor, + # index: torch.Tensor, + # split_size: int, weight_affine: WeightAffine): + + # assert trained_p.dim() == ref_p.dim() + # assert index.size(0) == trained_p.size(0) and index.size(1) == ref_p.size(0) + + # # print(trained_p.size(), ref_p.size(), index.size()) + + # ref_p = ref_p.detach() + # if trained_p.dim() > 1: + # trained_p = trained_p.flatten(1) + # ref_p = ref_p.flatten(1) + + # # the weight size of master DNN and foundation model may be totally different + + # # MD -> FM: upsample first + # # FM -> MD: downsample first + # if trained_p.size(1) < ref_p.size(1): + # # trained_p = self.upsample_2d_tensor(trained_p, ref_p.size(1)) + # ref_p = self.downsample_2d_tensor(ref_p, trained_p.size(1), weight_affine) + + # index = index.unsqueeze(-1) + # # linear_combed_ref_p = (ref_p.unsqueeze(0) * index).sum(1) + # # else: + + # # print(trained_p.size(), ref_p.size(), index.size()) + + # if split_size is None: + # # old version: huge memory consumption, not recommended (although this is fastest) + # # print('old version') + # linear_combed_ref_p = (ref_p.unsqueeze(0) * index).sum(1) + + # else: + # # new version + # linear_combed_ref_p = 0 + + # cur_split_size = split_size + # while index.size(1) % cur_split_size != 0: + # cur_split_size -= 1 + # # print(cur_split_size) + + # for i in range(0, index.size(1), cur_split_size): + # # if not isinstance(linear_combed_ref_p, int): + # # print(linear_combed_ref_p.size(), ref_p.unsqueeze(0)[:, i: i + cur_split_size].size(), index[:, i: i + cur_split_size].size()) + # linear_combed_ref_p += ref_p.unsqueeze(0)[:, i: i + cur_split_size] * index[:, i: i + cur_split_size] + # linear_combed_ref_p = linear_combed_ref_p.sum(1) + + # diff = (linear_combed_ref_p - trained_p).norm(2) ** 2 + # return diff + + def get_index_loss(self, fm, md, indexes, match_fn, split_size): + res = 0. + + for name, p in md.named_parameters(): + if name not in indexes.keys(): + continue + # if p.dim() == 0: + # continue + + raw_p = match_fn(name, fm) + # if raw_p is None: + # continue + + index = indexes[name] + + # print(name) + # res += (self.two_params_diff_fast_including_upsample(p, raw_p, index, split_size, weight_affines[name]) + \ + # self.two_params_diff_fast_including_downsample(p, raw_p, index, split_size, weight_affines[name])) / 2. + + res += self.two_params_diff_fast_no_sample(p, raw_p, index, split_size) + + return res + + def bn_cal(self, model: nn.Module, train_loader, num_iters, device): + has_bn = False + for n, m in model.named_modules(): + if isinstance(m, nn.BatchNorm2d): + has_bn = True + break + + if not has_bn: + return {} + + def bn_calibration_init(m): + """ calculating post-statistics of batch normalization """ + if getattr(m, 'track_running_stats', False): + # reset all values for post-statistics + m.reset_running_stats() + # set bn in training mode to update post-statistics + m.training = True + + with torch.no_grad(): + model.eval() + model.apply(bn_calibration_init) + for _ in range(num_iters): + x, _ = next(train_loader) + model(x.to(device)) + model.eval() + + bn_stats = {} + for n, m in model.named_modules(): + if isinstance(m, nn.BatchNorm2d): + bn_stats[n] = m + return bn_stats + + def run(self, scenario: Scenario, hyps: Dict) -> Dict[str, Any]: + super().run(scenario, hyps) + + # sanity check + # a= torch.tensor([[1, 2, 3], [1, 2, 4]]) + # index = torch.tensor([[1, 2, 3], + # [1, 2, 4]]) + # b = torch.tensor([[1, 2, 3], [1, 2, 4], [2, 3, 4]]) + # print(self.two_params_diff_fast(a, b, index, hyps['index_guided_linear_comb_split_size'])) + + assert isinstance(self.models['md'], ElasticDNN_OfflineMDModel) # for auto completion + assert isinstance(self.models['fm'], ElasticDNN_OfflineFMModel) # for auto completion + + # 1. add FBS + device = self.models['md'].device + + # logger.info(f'init master DNN by reducing width of an adapted foundation model (already tuned by LoRA)...') + + # before_fm_model = deepcopy(self.models['fm'].models_dict['main']) + # lora_util = self.models['fm'].get_lora_util() + # lora_absorbed_fm_model = lora_util.absorb_lora_and_recover_net_structure(self.models['fm'].models_dict['main'], + # torch.rand(hyps['samples_size']).to(device)) + # self.models['fm'].models_dict['main'] = lora_absorbed_fm_model + # master_dnn = self.models['fm'].generate_md_by_reducing_width(hyps['generate_md_width_ratio'], + # torch.rand(hyps['samples_size']).to(device)) + # self.models['fm'].models_dict['main'] = before_fm_model + + elastic_dnn_util = self.models['fm'].get_elastic_dnn_util() + # master_dnn = elastic_dnn_util.convert_raw_dnn_to_master_dnn_with_perf_test(master_dnn, + # hyps['FBS_r'], hyps['FBS_ignore_layers']) + # self.models['md'].models_dict['main'] = master_dnn + # self.models['md'].to(device) + master_dnn = self.models['md'].models_dict['main'] + + # 2. train (knowledge distillation, index relationship) + offline_datasets = scenario.get_offline_datasets() + train_dataset = MergedDataset([d['train'] for d in offline_datasets.values()]) + val_dataset = MergedDataset([d['val'] for d in offline_datasets.values()]) + train_loader = iter(build_dataloader(train_dataset, hyps['train_batch_size'], hyps['num_workers'], + True, None)) + val_loader = build_dataloader(val_dataset, hyps['val_batch_size'], hyps['num_workers'], + False, False) + + # 2.1 train only FBS (skipped because current md cannot do proper inference) + + # 2.2 train whole master DNN (knowledge distillation, index relationship) + for p in master_dnn.parameters(): + p.requires_grad = True + self.models['md'].to_train_mode() + + _norm = lambda a: (a.T / a.T.sum(0)).T + + indexes = {} + # weight_affines = {} + # weight_affines_trained_p = [] + + for name, p in self.models['md'].models_dict['main'].named_parameters(): + # if p.dim() > 1: + # print(name) + + logger.info(f'try: layer {name}, {p.size()}') + + matched_p_in_fm = self.models['md'].get_matched_param_of_fm(name, self.models['fm'].models_dict['main']) + if matched_p_in_fm is None: + continue + + + + if p.dim() == 1: + assert p.size(0) == matched_p_in_fm.size(0), f'{p.size()}, {matched_p_in_fm.size()}' + + if hyps['index_init'] == 'rand_norm': + indexes[name] = _norm(torch.rand((p.size(0), matched_p_in_fm.size(0))).to(device)) + elif hyps['index_init'] == 'zero': + indexes[name] = torch.zeros((p.size(0), matched_p_in_fm.size(0))).to(device) + elif hyps['index_init'] == 'randn_norm': + indexes[name] = _norm(torch.randn((p.size(0), matched_p_in_fm.size(0))).to(device)) + logger.info(f'construct index {indexes[name].size()} in layer {name} | dim 0') + + elif p.dim() == 2: + assert p.size(0) == matched_p_in_fm.size(0) or p.size(1) == matched_p_in_fm.size(1), f'{p.size()}, {matched_p_in_fm.size()}' + + if p.size(0) == matched_p_in_fm.size(0): + if hyps['index_init'] == 'rand_norm': + indexes[name] = _norm(torch.rand((p.size(1), matched_p_in_fm.size(1))).to(device)) + elif hyps['index_init'] == 'zero': + indexes[name] = torch.zeros((p.size(1), matched_p_in_fm.size(1))).to(device) + elif hyps['index_init'] == 'randn_norm': + indexes[name] = _norm(torch.randn((p.size(1), matched_p_in_fm.size(1))).to(device)) + logger.info(f'construct index {indexes[name].size()} in layer {name} | dim 1') + + elif p.size(1) == matched_p_in_fm.size(1): + if hyps['index_init'] == 'rand_norm': + indexes[name] = _norm(torch.rand((p.size(0), matched_p_in_fm.size(0))).to(device)) + elif hyps['index_init'] == 'zero': + indexes[name] = torch.zeros((p.size(0), matched_p_in_fm.size(0))).to(device) + elif hyps['index_init'] == 'randn_norm': + indexes[name] = _norm(torch.randn((p.size(0), matched_p_in_fm.size(0))).to(device)) + logger.info(f'construct index {indexes[name].size()} in layer {name} | dim 0') + else: + raise NotImplementedError + + + indexes[name].requires_grad = True + + + + tmp_indexes_file_path = os.path.join(self.res_save_dir, 'tmp-indexes.pt') + torch.save([indexes], tmp_indexes_file_path) + logger.info(f'generate indexes ({(os.path.getsize(tmp_indexes_file_path) / 1024**2):.3f}MB)') + os.remove(tmp_indexes_file_path) + + optimizer = torch.optim.__dict__[hyps['optimizer']]([ + {'params': self.models['md'].models_dict['main'].parameters(), **hyps['optimizer_args']}, + {'params': [v for v in indexes.values()], **hyps['indexes_optimizer_args']} + ]) + scheduler = torch.optim.lr_scheduler.__dict__[hyps['scheduler']](optimizer, **hyps['scheduler_args']) + tb_writer = create_tbwriter(os.path.join(self.res_save_dir, 'tb_log'), launch_tbboard=hyps['launch_tbboard']) + pbar = tqdm.tqdm(range(hyps['num_iters']), dynamic_ncols=True) + best_avg_val_acc = 0. + + for iter_index in pbar: + self.models['md'].to_train_mode() + self.models['fm'].to_eval_mode() + + rand_sparsity = random.random() * (hyps['max_sparsity'] - hyps['min_sparsity']) + hyps['min_sparsity'] + elastic_dnn_util.set_master_dnn_sparsity(self.models['md'].models_dict['main'], rand_sparsity) + + x, y = next(train_loader) + x, y = x.to(device), y.to(device) + + task_loss = self.models['md'].forward_to_get_task_loss(x, y) + # l1_reg_loss = hyps['l1_reg_loss_weight'] * elastic_dnn_util.get_accu_l1_reg_of_raw_channel_attention_in_master_dnn(master_dnn) + index_loss = hyps['index_loss_weight'] * self.get_index_loss(self.models['fm'].models_dict['main'], + self.models['md'].models_dict['main'], + indexes, + self.models['md'].get_matched_param_of_fm, + hyps['index_guided_linear_comb_split_size']) + total_loss = task_loss + index_loss + + optimizer.zero_grad() + total_loss.backward() + optimizer.step() + scheduler.step() + + if (iter_index + 1) % hyps['val_freq'] == 0: + + elastic_dnn_util.clear_cached_channel_attention_in_master_dnn(self.models['md'].models_dict['main']) + + cur_md = self.models['md'].models_dict['main'] + md_for_test = deepcopy(self.models['md'].models_dict['main']) + val_accs = {} + avg_val_acc = 0. + bn_stats = {} + + for val_sparsity in np.linspace(hyps['min_sparsity'], hyps['max_sparsity'], num=hyps['val_num_sparsities']): + elastic_dnn_util.set_master_dnn_sparsity(md_for_test, val_sparsity) + bn_stats[f'{val_sparsity:.4f}'] = self.bn_cal(md_for_test, train_loader, hyps['bn_cal_num_iters'], device) + self.models['md'].models_dict['main'] = md_for_test + self.models['md'].to_eval_mode() + val_acc = self.models['md'].get_accuracy(val_loader) + + val_accs[f'{val_sparsity:.4f}'] = val_acc + avg_val_acc += val_acc + + avg_val_acc /= hyps['val_num_sparsities'] + + self.models['md'].models_dict['main'] = cur_md + self.models['md'].models_dict['indexes'] = indexes + self.models['md'].models_dict['bn_stats'] = bn_stats + self.models['fm'].models_dict['indexes'] = indexes + + self.models['md'].save_model(os.path.join(self.res_save_dir, 'models/md_last.pt')) + self.models['fm'].save_model(os.path.join(self.res_save_dir, 'models/fm_last.pt')) + + if avg_val_acc > best_avg_val_acc: + best_avg_val_acc = avg_val_acc + self.models['md'].save_model(os.path.join(self.res_save_dir, 'models/md_best.pt')) + self.models['fm'].save_model(os.path.join(self.res_save_dir, 'models/fm_best.pt')) + + tb_writer.add_scalars(f'losses', dict(task=task_loss, index=index_loss, total=total_loss), iter_index) + pbar.set_description(f'loss: {total_loss:.6f}, task_loss: {task_loss:.6f}, index_loss: {index_loss:.6f}') + if (iter_index + 1) >= hyps['val_freq']: + tb_writer.add_scalars(f'accs/val_accs', val_accs, iter_index) + tb_writer.add_scalar(f'accs/avg_val_acc', avg_val_acc, iter_index) + pbar.set_description(f'loss: {total_loss:.6f}, task_loss: {task_loss:.6f}, index_loss: {index_loss:.6f}, ' + f'avg_val_acc: {avg_val_acc:.4f}') + \ No newline at end of file diff --git a/new_impl/cv/elasticdnn/api/algs/md_pretraining_index_v2_train_index_and_md.py b/new_impl/cv/elasticdnn/api/algs/md_pretraining_index_v2_train_index_and_md.py new file mode 100644 index 0000000000000000000000000000000000000000..7c138c5d16f991caad213195d0a666c1117c50ea --- /dev/null +++ b/new_impl/cv/elasticdnn/api/algs/md_pretraining_index_v2_train_index_and_md.py @@ -0,0 +1,559 @@ +from typing import Any, Dict +from schema import Schema, Or +import schema +from data import Scenario, MergedDataset +from new_impl.cv.base.alg import BaseAlg +from data import build_dataloader +from ..model import ElasticDNN_OfflineFMModel, ElasticDNN_OfflineMDModel +from ...model.base import ElasticDNNUtil +import torch.optim +import tqdm +import torch.nn.functional as F +from torch import nn +from utils.dl.common.env import create_tbwriter +import os +import random +import numpy as np +from copy import deepcopy +from utils.dl.common.model import LayerActivation, get_module +from utils.common.log import logger +from new_impl.cv.elasticdnn.model.base import KTakesAll + + +# class WeightAffine(nn.Module): +# def __init__(self, a, b, r=16): +# super(WeightAffine, self).__init__() + +# self.a = a +# self.b = b + +# self.w1 = nn.Parameter(torch.rand((b // r, a))) +# self.w2 = nn.Parameter(torch.rand((b, b // r))) + +# self.a_to_b = True + +# nn.init.zeros_(self.w1.data) +# nn.init.zeros_(self.w2.data) + +# def forward1(self, x): +# return F.linear(F.linear(x, self.w1), self.w2) + +# def forward2(self, x): +# return F.linear(F.linear(x, self.w2.T), self.w1.T) + +# def forward(self, x): +# return self.forward1(x) if self.a_to_b else self.forward2(x) + + +class ElasticDNN_MDPretrainingIndexAlg(BaseAlg): + """ + construct indexes between a filter/row of MD and all filters/rows of FM in the same layer + too huge indexes (~1GB), train so slow, hard to optimize + """ + def get_required_models_schema(self) -> Schema: + return Schema({ + 'fm': ElasticDNN_OfflineFMModel, + 'md': ElasticDNN_OfflineMDModel + }) + + def get_required_hyp_schema(self) -> Schema: + return Schema({ + 'launch_tbboard': bool, + + 'samples_size': object, + + 'FBS_r': int, + 'FBS_ignore_layers': [str], + + 'train_batch_size': int, + 'val_batch_size': int, + 'num_workers': int, + 'optimizer': str, + 'optimizer_args': dict, + 'indexes_optimizer_args': dict, + 'scheduler': str, + 'scheduler_args': dict, + 'num_iters': int, + 'val_freq': int, + 'max_sparsity': float, + 'min_sparsity': float, + 'l1_reg_loss_weight': float, + 'index_loss_weight': float, + 'val_num_sparsities': int, + + 'bn_cal_num_iters': int, + + 'index_init': str, + 'index_guided_linear_comb_split_size': Or(int, None) + }) + + # def upsample_2d_tensor(self, p: torch.Tensor, target_len: int, weight_affine: WeightAffine): + # assert p.dim() == 2 # regard 2d weight as (batch_size, 1d_vector_dim) + # # return F.upsample(p.unsqueeze(1).unsqueeze(3), + # # size=(target_len, 1), + # # mode='bilinear').squeeze(3).squeeze(1) + # weight_affine.a_to_b = True + # return weight_affine(p) + + # def downsample_2d_tensor(self, p: torch.Tensor, target_len: int, weight_affine: WeightAffine): + # assert p.dim() == 2 # regard 2d weight as (batch_size, 1d_vector_dim) + # # return F.interpolate(p.unsqueeze(1).unsqueeze(3), + # # size=(target_len, 1), + # # mode='bilinear').squeeze(3).squeeze(1) + # weight_affine.a_to_b = False + # return weight_affine(p) + + def two_params_diff_fast_no_sample(self, trained_p: torch.Tensor, ref_p: torch.Tensor, + index: torch.Tensor, + split_size: int): + + assert trained_p.dim() == ref_p.dim() + + if trained_p.dim() > 1 and index.size(0) == trained_p.size(1) and index.size(1) == ref_p.size(1): + assert trained_p.dim() == 2 + trained_p = trained_p.T + ref_p = ref_p.T + + assert index.size(0) == trained_p.size(0) and index.size(1) == ref_p.size(0) + + # print(trained_p.size(), ref_p.size(), index.size()) + + ref_p = ref_p.detach() + if trained_p.dim() > 1: + trained_p = trained_p.flatten(1) + ref_p = ref_p.flatten(1) + + # the weight size of master DNN and foundation model may be totally different + + # MD -> FM: upsample first + # FM -> MD: downsample first + # if trained_p.size(1) < ref_p.size(1): + # trained_p = self.upsample_2d_tensor(trained_p, ref_p.size(1), weight_affine) + + index = index.unsqueeze(-1) + # linear_combed_ref_p = (ref_p.unsqueeze(0) * index).sum(1) + # else: + + # print(trained_p.size(), ref_p.size(), index.size()) + + if split_size is None: + # old version: huge memory consumption, not recommended (although this is fastest) + # print('old version') + linear_combed_ref_p = (ref_p.unsqueeze(0) * index).sum(1) + + else: + # new version + linear_combed_ref_p = 0 + + cur_split_size = split_size + while index.size(1) % cur_split_size != 0: + cur_split_size -= 1 + # print(cur_split_size) + + for i in range(0, index.size(1), cur_split_size): + # if not isinstance(linear_combed_ref_p, int): + # print(linear_combed_ref_p.size(), ref_p.unsqueeze(0)[:, i: i + cur_split_size].size(), index[:, i: i + cur_split_size].size()) + linear_combed_ref_p += ref_p.unsqueeze(0)[:, i: i + cur_split_size] * index[:, i: i + cur_split_size] + linear_combed_ref_p = linear_combed_ref_p.sum(1) + + diff = (linear_combed_ref_p - trained_p).norm(2) ** 2 + return diff + + # def two_params_diff_fast_including_upsample(self, trained_p: torch.Tensor, ref_p: torch.Tensor, + # index: torch.Tensor, + # split_size: int, weight_affine: WeightAffine): + + # assert trained_p.dim() == ref_p.dim() + # assert index.size(0) == trained_p.size(0) and index.size(1) == ref_p.size(0) + + # # print(trained_p.size(), ref_p.size(), index.size()) + + # ref_p = ref_p.detach() + # if trained_p.dim() > 1: + # trained_p = trained_p.flatten(1) + # ref_p = ref_p.flatten(1) + + # # the weight size of master DNN and foundation model may be totally different + + # # MD -> FM: upsample first + # # FM -> MD: downsample first + # if trained_p.size(1) < ref_p.size(1): + # trained_p = self.upsample_2d_tensor(trained_p, ref_p.size(1), weight_affine) + + # index = index.unsqueeze(-1) + # # linear_combed_ref_p = (ref_p.unsqueeze(0) * index).sum(1) + # # else: + + # # print(trained_p.size(), ref_p.size(), index.size()) + + # if split_size is None: + # # old version: huge memory consumption, not recommended (although this is fastest) + # # print('old version') + # linear_combed_ref_p = (ref_p.unsqueeze(0) * index).sum(1) + + # else: + # # new version + # linear_combed_ref_p = 0 + + # cur_split_size = split_size + # while index.size(1) % cur_split_size != 0: + # cur_split_size -= 1 + # # print(cur_split_size) + + # for i in range(0, index.size(1), cur_split_size): + # # if not isinstance(linear_combed_ref_p, int): + # # print(linear_combed_ref_p.size(), ref_p.unsqueeze(0)[:, i: i + cur_split_size].size(), index[:, i: i + cur_split_size].size()) + # linear_combed_ref_p += ref_p.unsqueeze(0)[:, i: i + cur_split_size] * index[:, i: i + cur_split_size] + # linear_combed_ref_p = linear_combed_ref_p.sum(1) + + # diff = (linear_combed_ref_p - trained_p).norm(2) ** 2 + # return diff + + # def two_params_diff_fast_including_downsample(self, trained_p: torch.Tensor, ref_p: torch.Tensor, + # index: torch.Tensor, + # split_size: int, weight_affine: WeightAffine): + + # assert trained_p.dim() == ref_p.dim() + # assert index.size(0) == trained_p.size(0) and index.size(1) == ref_p.size(0) + + # # print(trained_p.size(), ref_p.size(), index.size()) + + # ref_p = ref_p.detach() + # if trained_p.dim() > 1: + # trained_p = trained_p.flatten(1) + # ref_p = ref_p.flatten(1) + + # # the weight size of master DNN and foundation model may be totally different + + # # MD -> FM: upsample first + # # FM -> MD: downsample first + # if trained_p.size(1) < ref_p.size(1): + # # trained_p = self.upsample_2d_tensor(trained_p, ref_p.size(1)) + # ref_p = self.downsample_2d_tensor(ref_p, trained_p.size(1), weight_affine) + + # index = index.unsqueeze(-1) + # # linear_combed_ref_p = (ref_p.unsqueeze(0) * index).sum(1) + # # else: + + # # print(trained_p.size(), ref_p.size(), index.size()) + + # if split_size is None: + # # old version: huge memory consumption, not recommended (although this is fastest) + # # print('old version') + # linear_combed_ref_p = (ref_p.unsqueeze(0) * index).sum(1) + + # else: + # # new version + # linear_combed_ref_p = 0 + + # cur_split_size = split_size + # while index.size(1) % cur_split_size != 0: + # cur_split_size -= 1 + # # print(cur_split_size) + + # for i in range(0, index.size(1), cur_split_size): + # # if not isinstance(linear_combed_ref_p, int): + # # print(linear_combed_ref_p.size(), ref_p.unsqueeze(0)[:, i: i + cur_split_size].size(), index[:, i: i + cur_split_size].size()) + # linear_combed_ref_p += ref_p.unsqueeze(0)[:, i: i + cur_split_size] * index[:, i: i + cur_split_size] + # linear_combed_ref_p = linear_combed_ref_p.sum(1) + + # diff = (linear_combed_ref_p - trained_p).norm(2) ** 2 + # return diff + + def get_index_loss(self, fm, md, indexes, match_fn, split_size): + res = 0. + + for name, p in md.named_parameters(): + if name not in indexes.keys(): + continue + # if p.dim() == 0: + # continue + + raw_p = match_fn(name, fm) + # if raw_p is None: + # continue + + index = indexes[name] + + # print(name) + # res += (self.two_params_diff_fast_including_upsample(p, raw_p, index, split_size, weight_affines[name]) + \ + # self.two_params_diff_fast_including_downsample(p, raw_p, index, split_size, weight_affines[name])) / 2. + + res += self.two_params_diff_fast_no_sample(p, raw_p, index, split_size) + + return res + + def bn_cal(self, model: nn.Module, train_loader, num_iters, device): + has_bn = False + for n, m in model.named_modules(): + if isinstance(m, nn.BatchNorm2d): + has_bn = True + break + + if not has_bn: + return {} + + def bn_calibration_init(m): + """ calculating post-statistics of batch normalization """ + if getattr(m, 'track_running_stats', False): + # reset all values for post-statistics + m.reset_running_stats() + # set bn in training mode to update post-statistics + m.training = True + + with torch.no_grad(): + model.eval() + model.apply(bn_calibration_init) + for _ in range(num_iters): + x, _ = next(train_loader) + model(x.to(device)) + model.eval() + + bn_stats = {} + for n, m in model.named_modules(): + if isinstance(m, nn.BatchNorm2d): + bn_stats[n] = m + return bn_stats + + def run(self, scenario: Scenario, hyps: Dict) -> Dict[str, Any]: + super().run(scenario, hyps) + + # sanity check + # a= torch.tensor([[1, 2, 3], [1, 2, 4]]) + # index = torch.tensor([[1, 2, 3], + # [1, 2, 4]]) + # b = torch.tensor([[1, 2, 3], [1, 2, 4], [2, 3, 4]]) + # print(self.two_params_diff_fast(a, b, index, hyps['index_guided_linear_comb_split_size'])) + + assert isinstance(self.models['md'], ElasticDNN_OfflineMDModel) # for auto completion + assert isinstance(self.models['fm'], ElasticDNN_OfflineFMModel) # for auto completion + + # 1. add FBS + device = self.models['md'].device + + # logger.info(f'init master DNN by reducing width of an adapted foundation model (already tuned by LoRA)...') + + # before_fm_model = deepcopy(self.models['fm'].models_dict['main']) + # lora_util = self.models['fm'].get_lora_util() + # lora_absorbed_fm_model = lora_util.absorb_lora_and_recover_net_structure(self.models['fm'].models_dict['main'], + # torch.rand(hyps['samples_size']).to(device)) + # self.models['fm'].models_dict['main'] = lora_absorbed_fm_model + # master_dnn = self.models['fm'].generate_md_by_reducing_width(hyps['generate_md_width_ratio'], + # torch.rand(hyps['samples_size']).to(device)) + # self.models['fm'].models_dict['main'] = before_fm_model + + # 2. train (knowledge distillation, index relationship) + offline_datasets = scenario.get_offline_datasets() + train_dataset = MergedDataset([d['train'] for d in offline_datasets.values()]) + val_dataset = MergedDataset([d['val'] for d in offline_datasets.values()]) + train_loader = iter(build_dataloader(train_dataset, hyps['train_batch_size'], hyps['num_workers'], + True, None)) + val_loader = build_dataloader(val_dataset, hyps['val_batch_size'], hyps['num_workers'], + False, False) + + #logger.info(f'master DNN acc before inserting FBS: {self.models["md"].get_accuracy(val_loader):.4f}') + + elastic_dnn_util = self.models['fm'].get_elastic_dnn_util() + + + master_dnn = self.models['md'].models_dict['main'] + elastic_dnn_util = self.models['fm'].get_elastic_dnn_util() + master_dnn = elastic_dnn_util.convert_raw_dnn_to_master_dnn_with_perf_test(master_dnn, hyps['FBS_r'], hyps['FBS_ignore_layers']).to(device) + self.models['md'].models_dict['main'] = master_dnn + #logger.info(f'master DNN acc before inserting FBS: {self.models["md"].get_accuracy(val_loader):.4f}') + # master_dnn = elastic_dnn_util.convert_raw_dnn_to_master_dnn_with_perf_test(master_dnn, + # hyps['FBS_r'], hyps['FBS_ignore_layers']) + # self.models['md'].models_dict['main'] = master_dnn + # self.models['md'].to(device) + # master_dnn = self.models['md'].models_dict['main'] + + + + + + # 2.1 train only FBS (skipped because current md cannot do proper inference) + + # 2.2 train whole master DNN (knowledge distillation, index relationship) + for p in master_dnn.parameters(): + p.requires_grad = True + self.models['md'].to_train_mode() + + _norm = lambda a: (a.T / a.T.sum(0)).T + + indexes = {} + # weight_affines = {} + # weight_affines_trained_p = [] + for name, p in self.models['md'].models_dict['main'].named_parameters(): + # if p.dim() > 1: + # print(name) + #if('attention.attention.projection_query' in name): print(name) + logger.debug(f'try: layer {name}, {p.size()}') + matched_p_in_fm = self.models['md'].get_matched_param_of_fm(name, self.models['fm'].models_dict['main']) + if matched_p_in_fm is None: + logger.debug(f'layer {name} no matched fm param') + continue + logger.debug(f'layer {name} matched fm param: {matched_p_in_fm.size()}') + if p.dim() == 1: + # assert p.size(0) == matched_p_in_fm.size(0), f'{p.size()}, {matched_p_in_fm.size()}' + + if hyps['index_init'] == 'rand_norm': + indexes[name] = _norm(torch.rand((p.size(0), matched_p_in_fm.size(0))).to(device)) + elif hyps['index_init'] == 'zero': + indexes[name] = torch.zeros((p.size(0), matched_p_in_fm.size(0))).to(device) + elif hyps['index_init'] == 'randn_norm': + indexes[name] = _norm(torch.randn((p.size(0), matched_p_in_fm.size(0))).to(device)) + else: + raise NotImplementedError + logger.info(f'construct index {indexes[name].size()} in layer {name} | dim 0') + + elif p.dim() == 2: + assert p.size(0) == matched_p_in_fm.size(0) or p.size(1) == matched_p_in_fm.size(1), f'{p.size()}, {matched_p_in_fm.size()}' + + if p.size(0) == matched_p_in_fm.size(0): + if hyps['index_init'] == 'rand_norm': + indexes[name] = _norm(torch.rand((p.size(1), matched_p_in_fm.size(1))).to(device)) + elif hyps['index_init'] == 'zero': + indexes[name] = torch.zeros((p.size(1), matched_p_in_fm.size(1))).to(device) + elif hyps['index_init'] == 'randn_norm': + indexes[name] = _norm(torch.randn((p.size(1), matched_p_in_fm.size(1))).to(device)) + else: + raise NotImplementedError + logger.info(f'construct index {indexes[name].size()} in layer {name} | dim 1') + + elif p.size(1) == matched_p_in_fm.size(1): + if hyps['index_init'] == 'rand_norm': + indexes[name] = _norm(torch.rand((p.size(0), matched_p_in_fm.size(0))).to(device)) + elif hyps['index_init'] == 'zero': + indexes[name] = torch.zeros((p.size(0), matched_p_in_fm.size(0))).to(device) + elif hyps['index_init'] == 'randn_norm': + indexes[name] = _norm(torch.randn((p.size(0), matched_p_in_fm.size(0))).to(device)) + else: + raise NotImplementedError + logger.info(f'construct index {indexes[name].size()} in layer {name} | dim 0') + else: + raise NotImplementedError + + + indexes[name].requires_grad = True + + #print() + tmp_indexes_file_path = os.path.join(self.res_save_dir, 'tmp-indexes.pt') + torch.save([indexes], tmp_indexes_file_path) + logger.info(f'generate indexes ({(os.path.getsize(tmp_indexes_file_path) / 1024**2):.3f}MB)') + os.remove(tmp_indexes_file_path) + res = [] + #print(self.models['md'].models_dict['main']) + # for n ,m in self.models['md'].models_dict['main'].named_modules(): + # if n.endswith('intermediate.dense.linear'): + # for p in m.parameters(): + # p.requires_grad = True + # res += [p] + # elif n.endswith('intermediate.dense.fbs'): + # for p in m: + # if isinstance(p,nn.Linear): + # for p1 in p.parameters(): + # p1.requires_grad = True + # res += [p1] + # else: + # for p in m.parameters(): + # p.requires_grad = False + optimizer = torch.optim.__dict__[hyps['optimizer']]([ + {'params': self.models['md'].models_dict['main'].parameters(), **hyps['optimizer_args']}, + #{'params': res, **hyps['optimizer_args']}, + {'params': [v for v in indexes.values()], **hyps['indexes_optimizer_args']} + ]) + scheduler = torch.optim.lr_scheduler.__dict__[hyps['scheduler']](optimizer, **hyps['scheduler_args']) + tb_writer = create_tbwriter(os.path.join(self.res_save_dir, 'tb_log'), launch_tbboard=hyps['launch_tbboard']) + pbar = tqdm.tqdm(range(hyps['num_iters']), dynamic_ncols=True) + best_avg_val_acc = 0. + #logger.info(f'master DNN acc before inserting FBS: {self.models["md"].get_accuracy(val_loader):.4f}') + for iter_index in pbar: + self.models['md'].to_train_mode() + self.models['fm'].to_eval_mode() + + rand_sparsity = random.random() * (hyps['max_sparsity'] - hyps['min_sparsity']) + hyps['min_sparsity'] + elastic_dnn_util.set_master_dnn_sparsity(self.models['md'].models_dict['main'], rand_sparsity) + + x, y = next(train_loader) + #x,y,_ = next(train_loader) + # if isinstance(x, dict): + # for k, v in x.items(): + # if isinstance(v, torch.Tensor): + # x[k] = v.to(device) + # y = y.to(device) + # else: + # x, y = x.to(device), y.to(device) + if isinstance(x, dict) and isinstance(y,torch.Tensor): + for k, v in x.items(): + if isinstance(v, torch.Tensor): + x[k] = v.to(device) + y = y.to(device) + elif isinstance(x,dict) and isinstance(y,dict): + for k, v in x.items(): + if isinstance(v, torch.Tensor): + x[k] = v.to(device) + for k, v in y.items(): + if isinstance(v, torch.Tensor): + y[k] = v.to(device) + else: + x, y = x.to(device), y.to(device) + + task_loss = self.models['md'].forward_to_get_task_loss(x, y) + l1_reg_loss = hyps['l1_reg_loss_weight'] * elastic_dnn_util.get_accu_l1_reg_of_raw_channel_attention_in_master_dnn(master_dnn) + index_loss = hyps['index_loss_weight'] * self.get_index_loss(self.models['fm'].models_dict['main'], + self.models['md'].models_dict['main'], + indexes, + self.models['md'].get_matched_param_of_fm, + hyps['index_guided_linear_comb_split_size']) + total_loss = task_loss + index_loss + l1_reg_loss + + optimizer.zero_grad() + total_loss.backward() + optimizer.step() + scheduler.step() + + + if (iter_index + 1) % hyps['val_freq'] == 0: + + elastic_dnn_util.clear_cached_channel_attention_in_master_dnn(self.models['md'].models_dict['main']) + + cur_md = self.models['md'].models_dict['main'] + md_for_test = deepcopy(self.models['md'].models_dict['main']) + val_accs = {} + avg_val_acc = 0. + bn_stats = {} + #logger.info(f'master DNN acc after inserting FBS: {self.models["md"].get_accuracy(val_loader):.4f}') + for val_sparsity in np.linspace(hyps['min_sparsity'], hyps['max_sparsity'], num=hyps['val_num_sparsities']): + elastic_dnn_util.set_master_dnn_sparsity(md_for_test, val_sparsity) + bn_stats[f'{val_sparsity:.4f}'] = self.bn_cal(md_for_test, train_loader, hyps['bn_cal_num_iters'], device) + self.models['md'].models_dict['main'] = md_for_test + self.models['md'].to_eval_mode() + val_acc = self.models['md'].get_accuracy(val_loader) + #print(self.models['md'].models_dict['main']) + val_accs[f'{val_sparsity:.4f}'] = val_acc + avg_val_acc += val_acc + + avg_val_acc /= hyps['val_num_sparsities'] + + #logger.info(f'master DNN acc before inserting FBS: {self.models["md"].get_accuracy(val_loader):.4f}') + self.models['md'].models_dict['main'] = cur_md + self.models['md'].models_dict['indexes'] = indexes + self.models['md'].models_dict['bn_stats'] = bn_stats + self.models['fm'].models_dict['indexes'] = indexes + + self.models['md'].save_model(os.path.join(self.res_save_dir, 'models/md_last.pt')) + self.models['fm'].save_model(os.path.join(self.res_save_dir, 'models/fm_last.pt')) + + if avg_val_acc > best_avg_val_acc: + best_avg_val_acc = avg_val_acc + self.models['md'].save_model(os.path.join(self.res_save_dir, 'models/md_best.pt')) + self.models['fm'].save_model(os.path.join(self.res_save_dir, 'models/fm_best.pt')) + + tb_writer.add_scalars(f'losses', dict(task=task_loss, index=index_loss, l1=l1_reg_loss, total=total_loss), iter_index) + pbar.set_description(f'loss: {total_loss:.6f}, task_loss: {task_loss:.6f}, index_loss: {index_loss:.6f}, l1_loss: {l1_reg_loss:.6f}') + if (iter_index + 1) >= hyps['val_freq']: + tb_writer.add_scalars(f'accs/val_accs', val_accs, iter_index) + tb_writer.add_scalar(f'accs/avg_val_acc', avg_val_acc, iter_index) + pbar.set_description(f'loss: {total_loss:.6f}, task_loss: {task_loss:.6f}, index_loss: {index_loss:.6f}, l1_loss: {l1_reg_loss:.6f}, ' + f'avg_val_acc: {avg_val_acc:.4f}') + \ No newline at end of file diff --git a/new_impl/cv/elasticdnn/api/algs/md_pretraining_index_v2_train_index_and_md_glip.py b/new_impl/cv/elasticdnn/api/algs/md_pretraining_index_v2_train_index_and_md_glip.py new file mode 100644 index 0000000000000000000000000000000000000000..938bfc6cff728f69ba61f494d65e8f900e3b11fe --- /dev/null +++ b/new_impl/cv/elasticdnn/api/algs/md_pretraining_index_v2_train_index_and_md_glip.py @@ -0,0 +1,538 @@ +from typing import Any, Dict +from schema import Schema, Or +import schema +from data import Scenario, MergedDataset +from new_impl.cv.base.alg import BaseAlg +from data import build_dataloader +from ..model import ElasticDNN_OfflineFMModel, ElasticDNN_OfflineMDModel +from ...model.base import ElasticDNNUtil +import torch.optim +import tqdm +import torch.nn.functional as F +from torch import nn +from utils.dl.common.env import create_tbwriter +import os +import random +import numpy as np +from copy import deepcopy +from utils.dl.common.model import LayerActivation, get_module +from utils.common.log import logger +from new_impl.cv.elasticdnn.model.base import KTakesAll +from torchvision.transforms import Compose + +# class WeightAffine(nn.Module): +# def __init__(self, a, b, r=16): +# super(WeightAffine, self).__init__() + +# self.a = a +# self.b = b + +# self.w1 = nn.Parameter(torch.rand((b // r, a))) +# self.w2 = nn.Parameter(torch.rand((b, b // r))) + +# self.a_to_b = True + +# nn.init.zeros_(self.w1.data) +# nn.init.zeros_(self.w2.data) + +# def forward1(self, x): +# return F.linear(F.linear(x, self.w1), self.w2) + +# def forward2(self, x): +# return F.linear(F.linear(x, self.w2.T), self.w1.T) + +# def forward(self, x): +# return self.forward1(x) if self.a_to_b else self.forward2(x) + + +class ElasticDNN_MDPretrainingIndexAlg(BaseAlg): + """ + construct indexes between a filter/row of MD and all filters/rows of FM in the same layer + too huge indexes (~1GB), train so slow, hard to optimize + """ + def get_required_models_schema(self) -> Schema: + return Schema({ + 'fm': ElasticDNN_OfflineFMModel, + 'md': ElasticDNN_OfflineMDModel + }) + + def get_required_hyp_schema(self) -> Schema: + from schema import Optional + return Schema({ + 'launch_tbboard': bool, + + 'samples_size': object, + + 'FBS_r': int, + 'FBS_ignore_layers': [str], + + 'train_batch_size': int, + 'val_batch_size': int, + 'num_workers': int, + 'optimizer': str, + 'optimizer_args': dict, + 'indexes_optimizer_args': dict, + 'scheduler': str, + 'scheduler_args': dict, + 'num_iters': int, + 'val_freq': int, + 'max_sparsity': float, + 'min_sparsity': float, + 'l1_reg_loss_weight': float, + 'index_loss_weight': float, + 'val_num_sparsities': int, + + 'bn_cal_num_iters': int, + + 'index_init': str, + 'index_guided_linear_comb_split_size': Or(int, None), + Optional('transform'): Compose, + }) + + # def upsample_2d_tensor(self, p: torch.Tensor, target_len: int, weight_affine: WeightAffine): + # assert p.dim() == 2 # regard 2d weight as (batch_size, 1d_vector_dim) + # # return F.upsample(p.unsqueeze(1).unsqueeze(3), + # # size=(target_len, 1), + # # mode='bilinear').squeeze(3).squeeze(1) + # weight_affine.a_to_b = True + # return weight_affine(p) + + # def downsample_2d_tensor(self, p: torch.Tensor, target_len: int, weight_affine: WeightAffine): + # assert p.dim() == 2 # regard 2d weight as (batch_size, 1d_vector_dim) + # # return F.interpolate(p.unsqueeze(1).unsqueeze(3), + # # size=(target_len, 1), + # # mode='bilinear').squeeze(3).squeeze(1) + # weight_affine.a_to_b = False + # return weight_affine(p) + + def two_params_diff_fast_no_sample(self, trained_p: torch.Tensor, ref_p: torch.Tensor, + index: torch.Tensor, + split_size: int): + + assert trained_p.dim() == ref_p.dim() + + if trained_p.dim() > 1 and index.size(0) == trained_p.size(1) and index.size(1) == ref_p.size(1): + assert trained_p.dim() == 2 + trained_p = trained_p.T + ref_p = ref_p.T + + assert index.size(0) == trained_p.size(0) and index.size(1) == ref_p.size(0) + + # print(trained_p.size(), ref_p.size(), index.size()) + + ref_p = ref_p.detach() + if trained_p.dim() > 1: + trained_p = trained_p.flatten(1) + ref_p = ref_p.flatten(1) + + # the weight size of master DNN and foundation model may be totally different + + # MD -> FM: upsample first + # FM -> MD: downsample first + # if trained_p.size(1) < ref_p.size(1): + # trained_p = self.upsample_2d_tensor(trained_p, ref_p.size(1), weight_affine) + + index = index.unsqueeze(-1) + # linear_combed_ref_p = (ref_p.unsqueeze(0) * index).sum(1) + # else: + + # print(trained_p.size(), ref_p.size(), index.size()) + + if split_size is None: + # old version: huge memory consumption, not recommended (although this is fastest) + # print('old version') + linear_combed_ref_p = (ref_p.unsqueeze(0) * index).sum(1) + + else: + # new version + linear_combed_ref_p = 0 + + cur_split_size = split_size + while index.size(1) % cur_split_size != 0: + cur_split_size -= 1 + # print(cur_split_size) + + for i in range(0, index.size(1), cur_split_size): + # if not isinstance(linear_combed_ref_p, int): + # print(linear_combed_ref_p.size(), ref_p.unsqueeze(0)[:, i: i + cur_split_size].size(), index[:, i: i + cur_split_size].size()) + linear_combed_ref_p += ref_p.unsqueeze(0)[:, i: i + cur_split_size] * index[:, i: i + cur_split_size] + linear_combed_ref_p = linear_combed_ref_p.sum(1) + + diff = (linear_combed_ref_p - trained_p).norm(2) ** 2 + return diff + + # def two_params_diff_fast_including_upsample(self, trained_p: torch.Tensor, ref_p: torch.Tensor, + # index: torch.Tensor, + # split_size: int, weight_affine: WeightAffine): + + # assert trained_p.dim() == ref_p.dim() + # assert index.size(0) == trained_p.size(0) and index.size(1) == ref_p.size(0) + + # # print(trained_p.size(), ref_p.size(), index.size()) + + # ref_p = ref_p.detach() + # if trained_p.dim() > 1: + # trained_p = trained_p.flatten(1) + # ref_p = ref_p.flatten(1) + + # # the weight size of master DNN and foundation model may be totally different + + # # MD -> FM: upsample first + # # FM -> MD: downsample first + # if trained_p.size(1) < ref_p.size(1): + # trained_p = self.upsample_2d_tensor(trained_p, ref_p.size(1), weight_affine) + + # index = index.unsqueeze(-1) + # # linear_combed_ref_p = (ref_p.unsqueeze(0) * index).sum(1) + # # else: + + # # print(trained_p.size(), ref_p.size(), index.size()) + + # if split_size is None: + # # old version: huge memory consumption, not recommended (although this is fastest) + # # print('old version') + # linear_combed_ref_p = (ref_p.unsqueeze(0) * index).sum(1) + + # else: + # # new version + # linear_combed_ref_p = 0 + + # cur_split_size = split_size + # while index.size(1) % cur_split_size != 0: + # cur_split_size -= 1 + # # print(cur_split_size) + + # for i in range(0, index.size(1), cur_split_size): + # # if not isinstance(linear_combed_ref_p, int): + # # print(linear_combed_ref_p.size(), ref_p.unsqueeze(0)[:, i: i + cur_split_size].size(), index[:, i: i + cur_split_size].size()) + # linear_combed_ref_p += ref_p.unsqueeze(0)[:, i: i + cur_split_size] * index[:, i: i + cur_split_size] + # linear_combed_ref_p = linear_combed_ref_p.sum(1) + + # diff = (linear_combed_ref_p - trained_p).norm(2) ** 2 + # return diff + + # def two_params_diff_fast_including_downsample(self, trained_p: torch.Tensor, ref_p: torch.Tensor, + # index: torch.Tensor, + # split_size: int, weight_affine: WeightAffine): + + # assert trained_p.dim() == ref_p.dim() + # assert index.size(0) == trained_p.size(0) and index.size(1) == ref_p.size(0) + + # # print(trained_p.size(), ref_p.size(), index.size()) + + # ref_p = ref_p.detach() + # if trained_p.dim() > 1: + # trained_p = trained_p.flatten(1) + # ref_p = ref_p.flatten(1) + + # # the weight size of master DNN and foundation model may be totally different + + # # MD -> FM: upsample first + # # FM -> MD: downsample first + # if trained_p.size(1) < ref_p.size(1): + # # trained_p = self.upsample_2d_tensor(trained_p, ref_p.size(1)) + # ref_p = self.downsample_2d_tensor(ref_p, trained_p.size(1), weight_affine) + + # index = index.unsqueeze(-1) + # # linear_combed_ref_p = (ref_p.unsqueeze(0) * index).sum(1) + # # else: + + # # print(trained_p.size(), ref_p.size(), index.size()) + + # if split_size is None: + # # old version: huge memory consumption, not recommended (although this is fastest) + # # print('old version') + # linear_combed_ref_p = (ref_p.unsqueeze(0) * index).sum(1) + + # else: + # # new version + # linear_combed_ref_p = 0 + + # cur_split_size = split_size + # while index.size(1) % cur_split_size != 0: + # cur_split_size -= 1 + # # print(cur_split_size) + + # for i in range(0, index.size(1), cur_split_size): + # # if not isinstance(linear_combed_ref_p, int): + # # print(linear_combed_ref_p.size(), ref_p.unsqueeze(0)[:, i: i + cur_split_size].size(), index[:, i: i + cur_split_size].size()) + # linear_combed_ref_p += ref_p.unsqueeze(0)[:, i: i + cur_split_size] * index[:, i: i + cur_split_size] + # linear_combed_ref_p = linear_combed_ref_p.sum(1) + + # diff = (linear_combed_ref_p - trained_p).norm(2) ** 2 + # return diff + + def get_index_loss(self, fm, md, indexes, match_fn, split_size): + res = 0. + + for name, p in md.named_parameters(): + if name not in indexes.keys(): + continue + # if p.dim() == 0: + # continue + + raw_p = match_fn(name, fm) + # if raw_p is None: + # continue + + index = indexes[name] + + # print(name) + # res += (self.two_params_diff_fast_including_upsample(p, raw_p, index, split_size, weight_affines[name]) + \ + # self.two_params_diff_fast_including_downsample(p, raw_p, index, split_size, weight_affines[name])) / 2. + + res += self.two_params_diff_fast_no_sample(p, raw_p, index, split_size) + + return res + + def bn_cal(self, model: nn.Module, train_loader, num_iters, device): + has_bn = False + for n, m in model.named_modules(): + if isinstance(m, nn.BatchNorm2d): + has_bn = True + break + + if not has_bn: + return {} + + def bn_calibration_init(m): + """ calculating post-statistics of batch normalization """ + if getattr(m, 'track_running_stats', False): + # reset all values for post-statistics + m.reset_running_stats() + # set bn in training mode to update post-statistics + m.training = True + + with torch.no_grad(): + model.eval() + model.apply(bn_calibration_init) + for _ in range(num_iters): + x, _ = next(train_loader) + model(x.to(device)) + model.eval() + + bn_stats = {} + for n, m in model.named_modules(): + if isinstance(m, nn.BatchNorm2d): + bn_stats[n] = m + return bn_stats + + def run(self, scenario: Scenario, hyps: Dict, collate_fn=None) -> Dict[str, Any]: + super().run(scenario, hyps) + + # sanity check + # a= torch.tensor([[1, 2, 3], [1, 2, 4]]) + # index = torch.tensor([[1, 2, 3], + # [1, 2, 4]]) + # b = torch.tensor([[1, 2, 3], [1, 2, 4], [2, 3, 4]]) + # print(self.two_params_diff_fast(a, b, index, hyps['index_guided_linear_comb_split_size'])) + + assert isinstance(self.models['md'], ElasticDNN_OfflineMDModel) # for auto completion + assert isinstance(self.models['fm'], ElasticDNN_OfflineFMModel) # for auto completion + + # 1. add FBS + device = self.models['md'].device + + # logger.info(f'init master DNN by reducing width of an adapted foundation model (already tuned by LoRA)...') + + # before_fm_model = deepcopy(self.models['fm'].models_dict['main']) + # lora_util = self.models['fm'].get_lora_util() + # lora_absorbed_fm_model = lora_util.absorb_lora_and_recover_net_structure(self.models['fm'].models_dict['main'], + # torch.rand(hyps['samples_size']).to(device)) + # self.models['fm'].models_dict['main'] = lora_absorbed_fm_model + # master_dnn = self.models['fm'].generate_md_by_reducing_width(hyps['generate_md_width_ratio'], + # torch.rand(hyps['samples_size']).to(device)) + # self.models['fm'].models_dict['main'] = before_fm_model + + # 2. train (knowledge distillation, index relationship) + if 'transform' in hyps.keys(): + offline_datasets = scenario.get_offline_datasets(transform=hyps['transform']) + else: + offline_datasets = scenario.get_offline_datasets() + train_dataset = MergedDataset([d['train'] for d in offline_datasets.values()]) + val_dataset = MergedDataset([d['val'] for d in offline_datasets.values()]) + train_loader = iter(build_dataloader(train_dataset, hyps['train_batch_size'], hyps['num_workers'], + True, None, collate_fn=collate_fn)) + val_loader = build_dataloader(val_dataset, hyps['val_batch_size'], hyps['num_workers'], + False, False, collate_fn=collate_fn) + + logger.info(f'master DNN acc before inserting FBS: {self.models["md"].get_accuracy(val_loader):.4f}') + + elastic_dnn_util = self.models['fm'].get_elastic_dnn_util() + + + master_dnn = self.models['md'].models_dict['main'] + elastic_dnn_util = self.models['fm'].get_elastic_dnn_util() + master_dnn = elastic_dnn_util.convert_raw_dnn_to_master_dnn_with_perf_test(master_dnn, hyps['FBS_r'], hyps['FBS_ignore_layers']).to(device) + self.models['md'].models_dict['main'] = master_dnn + # master_dnn = elastic_dnn_util.convert_raw_dnn_to_master_dnn_with_perf_test(master_dnn, + # hyps['FBS_r'], hyps['FBS_ignore_layers']) + # self.models['md'].models_dict['main'] = master_dnn + # self.models['md'].to(device) + # master_dnn = self.models['md'].models_dict['main'] + + + + + + # 2.1 train only FBS (skipped because current md cannot do proper inference) + + # 2.2 train whole master DNN (knowledge distillation, index relationship) + for p in master_dnn.parameters(): + p.requires_grad = True + self.models['md'].to_train_mode() + + _norm = lambda a: (a.T / a.T.sum(0)).T + + indexes = {} + # weight_affines = {} + # weight_affines_trained_p = [] + + for name, p in self.models['md'].models_dict['main'].named_parameters(): + # if p.dim() > 1: + # print(name) + + logger.debug(f'try: layer {name}, {p.size()}') + + matched_p_in_fm = self.models['md'].get_matched_param_of_fm(name, self.models['fm'].models_dict['main']) + if matched_p_in_fm is None: + logger.debug(f'layer {name} no matched fm param') + continue + logger.debug(f'layer {name} matched fm param: {matched_p_in_fm.size()}') + + + if p.dim() == 1: + # assert p.size(0) == matched_p_in_fm.size(0), f'{p.size()}, {matched_p_in_fm.size()}' + + if hyps['index_init'] == 'rand_norm': + indexes[name] = _norm(torch.rand((p.size(0), matched_p_in_fm.size(0))).to(device)) + elif hyps['index_init'] == 'zero': + indexes[name] = torch.zeros((p.size(0), matched_p_in_fm.size(0))).to(device) + elif hyps['index_init'] == 'randn_norm': + indexes[name] = _norm(torch.randn((p.size(0), matched_p_in_fm.size(0))).to(device)) + else: + raise NotImplementedError + logger.info(f'construct index {indexes[name].size()} in layer {name} | dim 0') + + elif p.dim() == 2: + assert p.size(0) == matched_p_in_fm.size(0) or p.size(1) == matched_p_in_fm.size(1), f'{p.size()}, {matched_p_in_fm.size()}' + + if p.size(0) == matched_p_in_fm.size(0): + if hyps['index_init'] == 'rand_norm': + indexes[name] = _norm(torch.rand((p.size(1), matched_p_in_fm.size(1))).to(device)) + elif hyps['index_init'] == 'zero': + indexes[name] = torch.zeros((p.size(1), matched_p_in_fm.size(1))).to(device) + elif hyps['index_init'] == 'randn_norm': + indexes[name] = _norm(torch.randn((p.size(1), matched_p_in_fm.size(1))).to(device)) + else: + raise NotImplementedError + logger.info(f'construct index {indexes[name].size()} in layer {name} | dim 1') + + elif p.size(1) == matched_p_in_fm.size(1): + if hyps['index_init'] == 'rand_norm': + indexes[name] = _norm(torch.rand((p.size(0), matched_p_in_fm.size(0))).to(device)) + elif hyps['index_init'] == 'zero': + indexes[name] = torch.zeros((p.size(0), matched_p_in_fm.size(0))).to(device) + elif hyps['index_init'] == 'randn_norm': + indexes[name] = _norm(torch.randn((p.size(0), matched_p_in_fm.size(0))).to(device)) + else: + raise NotImplementedError + logger.info(f'construct index {indexes[name].size()} in layer {name} | dim 0') + else: + print('a') + continue + # raise NotImplementedError + + + indexes[name].requires_grad = True + + + + tmp_indexes_file_path = os.path.join(self.res_save_dir, 'tmp-indexes.pt') + torch.save([indexes], tmp_indexes_file_path) + logger.info(f'generate indexes ({(os.path.getsize(tmp_indexes_file_path) / 1024**2):.3f}MB)') + os.remove(tmp_indexes_file_path) + + optimizer = torch.optim.__dict__[hyps['optimizer']]([ + {'params': self.models['md'].models_dict['main'].parameters(), **hyps['optimizer_args']}, + {'params': [v for v in indexes.values()], **hyps['indexes_optimizer_args']} + ]) + scheduler = torch.optim.lr_scheduler.__dict__[hyps['scheduler']](optimizer, **hyps['scheduler_args']) + tb_writer = create_tbwriter(os.path.join(self.res_save_dir, 'tb_log'), launch_tbboard=hyps['launch_tbboard']) + pbar = tqdm.tqdm(range(hyps['num_iters']), dynamic_ncols=True) + best_avg_val_acc = 0. + + for iter_index in pbar: + self.models['md'].to_train_mode() + self.models['fm'].to_eval_mode() + + rand_sparsity = random.random() * (hyps['max_sparsity'] - hyps['min_sparsity']) + hyps['min_sparsity'] + elastic_dnn_util.set_master_dnn_sparsity(self.models['md'].models_dict['main'], rand_sparsity) + + x, y = next(train_loader) + if isinstance(x, dict): + for k, v in x.items(): + if isinstance(v, torch.Tensor): + x[k] = v.to(device) + y = y.to(device) + else: + x, y = x.to(device), y.to(device) + + task_loss = self.models['md'].forward_to_get_task_loss(x, y) + l1_reg_loss = hyps['l1_reg_loss_weight'] * elastic_dnn_util.get_accu_l1_reg_of_raw_channel_attention_in_master_dnn(master_dnn) + index_loss = hyps['index_loss_weight'] * self.get_index_loss(self.models['fm'].models_dict['main'], + self.models['md'].models_dict['main'], + indexes, + self.models['md'].get_matched_param_of_fm, + hyps['index_guided_linear_comb_split_size']) + total_loss = task_loss + index_loss + l1_reg_loss + + optimizer.zero_grad() + total_loss.backward() + optimizer.step() + scheduler.step() + + if (iter_index + 1) % hyps['val_freq'] == 0: + + elastic_dnn_util.clear_cached_channel_attention_in_master_dnn(self.models['md'].models_dict['main']) + + cur_md = self.models['md'].models_dict['main'] + md_for_test = deepcopy(self.models['md'].models_dict['main']) + val_accs = {} + avg_val_acc = 0. + bn_stats = {} + + for val_sparsity in np.linspace(hyps['min_sparsity'], hyps['max_sparsity'], num=hyps['val_num_sparsities']): + elastic_dnn_util.set_master_dnn_sparsity(md_for_test, val_sparsity) + bn_stats[f'{val_sparsity:.4f}'] = self.bn_cal(md_for_test, train_loader, hyps['bn_cal_num_iters'], device) + self.models['md'].models_dict['main'] = md_for_test + self.models['md'].to_eval_mode() + val_acc = self.models['md'].get_accuracy(val_loader, collate_fn=collate_fn) + + val_accs[f'{val_sparsity:.4f}'] = val_acc + avg_val_acc += val_acc + + avg_val_acc /= hyps['val_num_sparsities'] + + self.models['md'].models_dict['main'] = cur_md + self.models['md'].models_dict['indexes'] = indexes + self.models['md'].models_dict['bn_stats'] = bn_stats + self.models['fm'].models_dict['indexes'] = indexes + + self.models['md'].save_model(os.path.join(self.res_save_dir, 'models/md_last.pt')) + self.models['fm'].save_model(os.path.join(self.res_save_dir, 'models/fm_last.pt')) + + if avg_val_acc > best_avg_val_acc: + best_avg_val_acc = avg_val_acc + self.models['md'].save_model(os.path.join(self.res_save_dir, 'models/md_best.pt')) + self.models['fm'].save_model(os.path.join(self.res_save_dir, 'models/fm_best.pt')) + + tb_writer.add_scalars(f'losses', dict(task=task_loss, index=index_loss, l1=l1_reg_loss, total=total_loss), iter_index) + # pbar.set_description(f'loss: {total_loss:.6f}, task_loss: {task_loss:.6f}, index_loss: {index_loss:.6f}, l1_loss: {l1_reg_loss:.6f}') + pbar.set_description(f'loss: {total_loss:.6f}') + if (iter_index + 1) >= hyps['val_freq']: + tb_writer.add_scalars(f'accs/val_accs', val_accs, iter_index) + tb_writer.add_scalar(f'accs/avg_val_acc', avg_val_acc, iter_index) + # pbar.set_description(f'loss: {total_loss:.6f}, task_loss: {task_loss:.6f}, index_loss: {index_loss:.6f}, l1_loss: {l1_reg_loss:.6f}, ' + # f'avg_val_acc: {avg_val_acc:.4f}') + pbar.set_description(f'task_loss: {task_loss:.6f}, index_loss: {index_loss:.6f}, avg_map50: {avg_val_acc:.4f}') \ No newline at end of file diff --git a/new_impl/cv/elasticdnn/api/algs/md_pretraining_v1.py b/new_impl/cv/elasticdnn/api/algs/md_pretraining_v1.py new file mode 100644 index 0000000000000000000000000000000000000000..1b62f0bf6798669a7fc874930055aa2342d31307 --- /dev/null +++ b/new_impl/cv/elasticdnn/api/algs/md_pretraining_v1.py @@ -0,0 +1,316 @@ +from typing import Any, Dict +from schema import Schema, Or +import schema +from data import Scenario, MergedDataset +from methods.base.alg import BaseAlg +from data import build_dataloader +from ..model import ElasticDNN_OfflineFMModel, ElasticDNN_OfflineMDModel +from ...model.base import ElasticDNNUtil +import torch.optim +import tqdm +import torch.nn.functional as F +from torch import nn +from utils.dl.common.env import create_tbwriter +import os +import random +import numpy as np +from copy import deepcopy +from utils.dl.common.model import LayerActivation, get_module +from utils.common.log import logger + + +class ElasticDNN_MDPretrainingAlg(BaseAlg): + """ + construct indexes between a filter/row of MD and all filters/rows of FM in the same layer + too huge indexes (~1GB), train so slow, hard to optimize + """ + def get_required_models_schema(self) -> Schema: + return Schema({ + 'fm': ElasticDNN_OfflineFMModel, + 'md': ElasticDNN_OfflineMDModel + }) + + def get_required_hyp_schema(self) -> Schema: + return Schema({ + 'launch_tbboard': bool, + + 'samples_size': (int, int, int, int), + 'generate_md_width_ratio': int, + + 'FBS_r': int, + 'FBS_ignore_layers': [str], + + 'train_batch_size': int, + 'val_batch_size': int, + 'num_workers': int, + 'optimizer': str, + 'md_optimizer_args': dict, + 'indexes_optimizer_args': dict, + 'scheduler': str, + 'scheduler_args': dict, + 'num_iters': int, + 'val_freq': int, + 'max_sparsity': float, + 'min_sparsity': float, + 'distill_loss_weight': float, + 'index_loss_weight': float, + 'val_num_sparsities': int, + + 'bn_cal_num_iters': int, + + 'index_guided_linear_comb_split_size': Or(int, None) + }) + + def upsample_2d_tensor(self, p: torch.Tensor, target_len: int): + assert p.dim() == 2 # regard 2d weight as (batch_size, 1d_vector_dim) + return F.upsample(p.unsqueeze(1).unsqueeze(3), + size=(target_len, 1), + mode='bilinear').squeeze(3).squeeze(1) + + def two_params_diff_fast(self, trained_p: torch.Tensor, ref_p: torch.Tensor, + index: torch.Tensor, + split_size: int): + + assert trained_p.dim() == ref_p.dim() + assert index.size(0) == trained_p.size(0) and index.size(1) == ref_p.size(0) + + # print(trained_p.size(), ref_p.size(), index.size()) + + ref_p = ref_p.detach() + if trained_p.dim() > 1: + trained_p = trained_p.flatten(1) + ref_p = ref_p.flatten(1) + + # the weight size of master DNN and foundation model may be totally different + + # MD -> FM: upsample first + # FM -> MD: downsample first + if trained_p.size(1) < ref_p.size(1): + trained_p = self.upsample_2d_tensor(trained_p, ref_p.size(1)) + + index = index.unsqueeze(-1) + # linear_combed_ref_p = (ref_p.unsqueeze(0) * index).sum(1) + # else: + + # print(trained_p.size(), ref_p.size(), index.size()) + + if split_size is None: + # old version: huge memory consumption, not recommended (although this is fastest) + # print('old version') + linear_combed_ref_p = (ref_p.unsqueeze(0) * index).sum(1) + + else: + # new version + linear_combed_ref_p = 0 + + cur_split_size = split_size + while index.size(1) % cur_split_size != 0: + cur_split_size -= 1 + # print(cur_split_size) + + for i in range(0, index.size(1), cur_split_size): + # if not isinstance(linear_combed_ref_p, int): + # print(linear_combed_ref_p.size(), ref_p.unsqueeze(0)[:, i: i + cur_split_size].size(), index[:, i: i + cur_split_size].size()) + linear_combed_ref_p += ref_p.unsqueeze(0)[:, i: i + cur_split_size] * index[:, i: i + cur_split_size] + linear_combed_ref_p = linear_combed_ref_p.sum(1) + + diff = (linear_combed_ref_p - trained_p).norm(2) ** 2 + return diff + + def get_index_loss(self, fm, md, indexes, match_fn, split_size): + res = 0. + + for name, p in md.named_parameters(): + if p.dim() == 0: + continue + + raw_p = match_fn(name, fm) + if raw_p is None: + continue + + index = indexes[name] + + # print(name) + res += self.two_params_diff_fast(p, raw_p, index, split_size) + return res + + def bn_cal(self, model: nn.Module, train_loader, num_iters, device): + has_bn = False + for n, m in model.named_modules(): + if isinstance(m, nn.BatchNorm2d): + has_bn = True + break + + if not has_bn: + return {} + + def bn_calibration_init(m): + """ calculating post-statistics of batch normalization """ + if getattr(m, 'track_running_stats', False): + # reset all values for post-statistics + m.reset_running_stats() + # set bn in training mode to update post-statistics + m.training = True + + with torch.no_grad(): + model.eval() + model.apply(bn_calibration_init) + for _ in range(num_iters): + x, _ = next(train_loader) + model(x.to(device)) + model.eval() + + bn_stats = {} + for n, m in model.named_modules(): + if isinstance(m, nn.BatchNorm2d): + bn_stats[n] = m + return bn_stats + + def run(self, scenario: Scenario, hyps: Dict) -> Dict[str, Any]: + super().run(scenario, hyps) + + # sanity check + # a= torch.tensor([[1, 2, 3], [1, 2, 4]]) + # index = torch.tensor([[1, 2, 3], + # [1, 2, 4]]) + # b = torch.tensor([[1, 2, 3], [1, 2, 4], [2, 3, 4]]) + # print(self.two_params_diff_fast(a, b, index, hyps['index_guided_linear_comb_split_size'])) + + assert isinstance(self.models['md'], ElasticDNN_OfflineMDModel) # for auto completion + assert isinstance(self.models['fm'], ElasticDNN_OfflineFMModel) # for auto completion + + # 1. add FBS + device = self.models['md'].device + + logger.info(f'init master DNN by reducing width of an adapted foundation model (already tuned by LoRA)...') + + before_fm_model = deepcopy(self.models['fm'].models_dict['main']) + lora_util = self.models['fm'].get_lora_util() + lora_absorbed_fm_model = lora_util.absorb_lora_and_recover_net_structure(self.models['fm'].models_dict['main'], + torch.rand(hyps['samples_size']).to(device)) + self.models['fm'].models_dict['main'] = lora_absorbed_fm_model + master_dnn = self.models['fm'].generate_md_by_reducing_width(hyps['generate_md_width_ratio'], + torch.rand(hyps['samples_size']).to(device)) + self.models['fm'].models_dict['main'] = before_fm_model + + elastic_dnn_util = self.models['fm'].get_elastic_dnn_util() + master_dnn = elastic_dnn_util.convert_raw_dnn_to_master_dnn_with_perf_test(master_dnn, + hyps['FBS_r'], hyps['FBS_ignore_layers']) + self.models['md'].models_dict['main'] = master_dnn + self.models['md'].to(device) + + # 2. train (knowledge distillation, index relationship) + offline_datasets = scenario.get_offline_datasets() + train_dataset = MergedDataset([d['train'] for d in offline_datasets.values()]) + val_dataset = MergedDataset([d['val'] for d in offline_datasets.values()]) + train_loader = iter(build_dataloader(train_dataset, hyps['train_batch_size'], hyps['num_workers'], + True, None)) + val_loader = build_dataloader(val_dataset, hyps['val_batch_size'], hyps['num_workers'], + False, False) + + # 2.1 train only FBS (skipped because current md cannot do proper inference) + + # 2.2 train whole master DNN (knowledge distillation, index relationship) + for p in master_dnn.parameters(): + p.requires_grad = True + self.models['md'].to_train_mode() + + indexes = {} + for name, p in self.models['md'].models_dict['main'].named_parameters(): + if p.dim() > 0: + matched_p_in_fm = self.models['md'].get_matched_param_of_fm(name, self.models['fm'].models_dict['main']) + if matched_p_in_fm is None: + continue + indexes[name] = torch.zeros((p.size(0), matched_p_in_fm.size(0))).to(device) + indexes[name].requires_grad = True + + tmp_indexes_file_path = os.path.join(self.res_save_dir, 'tmp-indexes.pt') + torch.save(indexes, tmp_indexes_file_path) + logger.info(f'generate indexes ({(os.path.getsize(tmp_indexes_file_path) / 1024**2):.3f}MB)') + os.remove(tmp_indexes_file_path) + + optimizer = torch.optim.__dict__[hyps['optimizer']]([ + {'params': self.models['md'].models_dict['main'].parameters(), **hyps['md_optimizer_args']}, + {'params': [v for v in indexes.values()], **hyps['indexes_optimizer_args']} + ]) + scheduler = torch.optim.lr_scheduler.__dict__[hyps['scheduler']](optimizer, **hyps['scheduler_args']) + tb_writer = create_tbwriter(os.path.join(self.res_save_dir, 'tb_log'), launch_tbboard=hyps['launch_tbboard']) + pbar = tqdm.tqdm(range(hyps['num_iters']), dynamic_ncols=True) + best_avg_val_acc = 0. + + md_output_hook = None + + for iter_index in pbar: + self.models['md'].to_train_mode() + self.models['fm'].to_eval_mode() + + rand_sparsity = random.random() * (hyps['max_sparsity'] - hyps['min_sparsity']) + hyps['min_sparsity'] + elastic_dnn_util.set_master_dnn_sparsity(self.models['md'].models_dict['main'], rand_sparsity) + + x, y = next(train_loader) + x, y = x.to(device), y.to(device) + + with torch.no_grad(): + fm_output = self.models['fm'].infer(x) + + if md_output_hook is None: + md_output_hook = LayerActivation(self.models['md'].models_dict['main'], False, device) + task_loss = self.models['md'].forward_to_get_task_loss(x, y) + md_output = md_output_hook.output + distill_loss = hyps['distill_loss_weight'] * self.models['md'].get_distill_loss(md_output, fm_output) + index_loss = hyps['index_loss_weight'] * self.get_index_loss(self.models['fm'].models_dict['main'], + self.models['md'].models_dict['main'], + indexes, + self.models['md'].get_matched_param_of_fm, + hyps['index_guided_linear_comb_split_size']) + total_loss = task_loss + distill_loss + index_loss + + optimizer.zero_grad() + total_loss.backward() + optimizer.step() + scheduler.step() + + if (iter_index + 1) % hyps['val_freq'] == 0: + + elastic_dnn_util.clear_cached_channel_attention_in_master_dnn(self.models['md'].models_dict['main']) + md_output_hook.remove() + md_output_hook = None + + cur_md = self.models['md'].models_dict['main'] + md_for_test = deepcopy(self.models['md'].models_dict['main']) + val_accs = {} + avg_val_acc = 0. + bn_stats = {} + + for val_sparsity in np.linspace(hyps['min_sparsity'], hyps['max_sparsity'], num=hyps['val_num_sparsities']): + elastic_dnn_util.set_master_dnn_sparsity(md_for_test, val_sparsity) + bn_stats[f'{val_sparsity:.4f}'] = self.bn_cal(md_for_test, train_loader, hyps['bn_cal_num_iters'], device) + self.models['md'].models_dict['main'] = md_for_test + self.models['md'].to_eval_mode() + val_acc = self.models['md'].get_accuracy(val_loader) + + val_accs[f'{val_sparsity:.4f}'] = val_acc + avg_val_acc += val_acc + + avg_val_acc /= hyps['val_num_sparsities'] + + self.models['md'].models_dict['main'] = cur_md + self.models['md'].models_dict['indexes'] = indexes + self.models['md'].models_dict['bn_stats'] = bn_stats + self.models['fm'].models_dict['indexes'] = indexes + + self.models['md'].save_model(os.path.join(self.res_save_dir, 'models/md_last.pt')) + self.models['fm'].save_model(os.path.join(self.res_save_dir, 'models/fm_last.pt')) + + if avg_val_acc > best_avg_val_acc: + best_avg_val_acc = avg_val_acc + self.models['md'].save_model(os.path.join(self.res_save_dir, 'models/md_best.pt')) + self.models['fm'].save_model(os.path.join(self.res_save_dir, 'models/fm_best.pt')) + + tb_writer.add_scalars(f'losses', dict(task=task_loss, distill=distill_loss, index=index_loss, total=total_loss), iter_index) + pbar.set_description(f'loss: {total_loss:.6f}') + if (iter_index + 1) >= hyps['val_freq']: + tb_writer.add_scalars(f'accs/val_accs', val_accs, iter_index) + tb_writer.add_scalar(f'accs/avg_val_acc', avg_val_acc, iter_index) + pbar.set_description(f'loss: {total_loss:.6f}, avg_val_acc: {avg_val_acc:.4f}') + \ No newline at end of file diff --git a/new_impl/cv/elasticdnn/api/algs/md_pretraining_w_fbs.py b/new_impl/cv/elasticdnn/api/algs/md_pretraining_w_fbs.py new file mode 100644 index 0000000000000000000000000000000000000000..53242f1638acd9becf4f56107a712ebbe4ae4ccf --- /dev/null +++ b/new_impl/cv/elasticdnn/api/algs/md_pretraining_w_fbs.py @@ -0,0 +1,192 @@ +from typing import Any, Dict +from schema import Schema, Or +import schema +from data import Scenario, MergedDataset +from methods.base.alg import BaseAlg +from data import build_dataloader +from ..model import ElasticDNN_OfflineFMModel, ElasticDNN_OfflineMDModel +from ...model.base import ElasticDNNUtil +import torch.optim +import tqdm +import torch.nn.functional as F +from torch import nn +from utils.dl.common.env import create_tbwriter +import os +import random +import numpy as np +from copy import deepcopy +from utils.dl.common.model import LayerActivation, get_module +from utils.common.log import logger + + +class ElasticDNN_MDPretrainingWFBSAlg(BaseAlg): + """ + TODO: fine-tuned FM -> init MD -> trained MD -> construct indexes (only between similar weights) and fine-tune + """ + def get_required_models_schema(self) -> Schema: + return Schema({ + 'fm': ElasticDNN_OfflineFMModel, + 'md': ElasticDNN_OfflineMDModel + }) + + def get_required_hyp_schema(self) -> Schema: + return Schema({ + 'launch_tbboard': bool, + + 'samples_size': (int, int, int, int), + 'generate_md_width_ratio': int, + + 'FBS_r': int, + 'FBS_ignore_layers': [str], + + 'train_batch_size': int, + 'val_batch_size': int, + 'num_workers': int, + 'optimizer': str, + 'optimizer_args': dict, + 'scheduler': str, + 'scheduler_args': dict, + 'num_iters': int, + 'val_freq': int, + 'max_sparsity': float, + 'min_sparsity': float, + 'l1_reg_loss_weight': float, + 'val_num_sparsities': int, + + 'bn_cal_num_iters': int + }) + + def bn_cal(self, model: nn.Module, train_loader, num_iters, device): + has_bn = False + for n, m in model.named_modules(): + if isinstance(m, nn.BatchNorm2d): + has_bn = True + break + + if not has_bn: + return {} + + def bn_calibration_init(m): + """ calculating post-statistics of batch normalization """ + if getattr(m, 'track_running_stats', False): + # reset all values for post-statistics + m.reset_running_stats() + # set bn in training mode to update post-statistics + m.training = True + + with torch.no_grad(): + model.eval() + model.apply(bn_calibration_init) + for _ in range(num_iters): + x, _ = next(train_loader) + model(x.to(device)) + model.eval() + + bn_stats = {} + for n, m in model.named_modules(): + if isinstance(m, nn.BatchNorm2d): + bn_stats[n] = m + return bn_stats + + def run(self, scenario: Scenario, hyps: Dict) -> Dict[str, Any]: + super().run(scenario, hyps) + + assert isinstance(self.models['md'], ElasticDNN_OfflineMDModel) # for auto completion + assert isinstance(self.models['fm'], ElasticDNN_OfflineFMModel) # for auto completion + + # 1. add FBS + device = self.models['md'].device + + # 2. train (knowledge distillation, index relationship) + offline_datasets = scenario.get_offline_datasets() + train_dataset = MergedDataset([d['train'] for d in offline_datasets.values()]) + val_dataset = MergedDataset([d['val'] for d in offline_datasets.values()]) + train_loader = iter(build_dataloader(train_dataset, hyps['train_batch_size'], hyps['num_workers'], + True, None)) + val_loader = build_dataloader(val_dataset, hyps['val_batch_size'], hyps['num_workers'], + False, False) + + logger.info(f'master DNN acc before inserting FBS: {self.models["md"].get_accuracy(val_loader):.4f}') + + master_dnn = self.models['md'].models_dict['main'] + elastic_dnn_util = self.models['fm'].get_elastic_dnn_util() + master_dnn = elastic_dnn_util.convert_raw_dnn_to_master_dnn_with_perf_test(master_dnn, hyps['FBS_r'], hyps['FBS_ignore_layers']).to(device) + self.models['md'].models_dict['main'] = master_dnn + + # 2.1 train whole master DNN (knowledge distillation) + for p in master_dnn.parameters(): + p.requires_grad = True + self.models['md'].to_train_mode() + + optimizer = torch.optim.__dict__[hyps['optimizer']]([ + {'params': self.models['md'].models_dict['main'].parameters(), **hyps['optimizer_args']} + ]) + scheduler = torch.optim.lr_scheduler.__dict__[hyps['scheduler']](optimizer, **hyps['scheduler_args']) + tb_writer = create_tbwriter(os.path.join(self.res_save_dir, 'tb_log'), launch_tbboard=hyps['launch_tbboard']) + pbar = tqdm.tqdm(range(hyps['num_iters']), dynamic_ncols=True) + best_avg_val_acc = 0. + + for iter_index in pbar: + self.models['md'].to_train_mode() + self.models['fm'].to_eval_mode() + + rand_sparsity = random.random() * (hyps['max_sparsity'] - hyps['min_sparsity']) + hyps['min_sparsity'] + elastic_dnn_util.set_master_dnn_sparsity(self.models['md'].models_dict['main'], rand_sparsity) + + x, y = next(train_loader) + x, y = x.to(device), y.to(device) + + task_loss = self.models['md'].forward_to_get_task_loss(x, y) + l1_reg_loss = hyps['l1_reg_loss_weight'] * elastic_dnn_util.get_accu_l1_reg_of_raw_channel_attention_in_master_dnn(master_dnn) + total_loss = task_loss + l1_reg_loss + + optimizer.zero_grad() + total_loss.backward() + optimizer.step() + scheduler.step() + + if (iter_index + 1) % hyps['val_freq'] == 0: + + elastic_dnn_util.clear_cached_channel_attention_in_master_dnn(self.models['md'].models_dict['main']) + + cur_md = self.models['md'].models_dict['main'] + md_for_test = deepcopy(self.models['md'].models_dict['main']) + val_accs = {} + avg_val_acc = 0. + bn_stats = {} + + for val_sparsity in np.linspace(hyps['min_sparsity'], hyps['max_sparsity'], num=hyps['val_num_sparsities']): + elastic_dnn_util.set_master_dnn_sparsity(md_for_test, val_sparsity) + bn_stats[f'{val_sparsity:.4f}'] = self.bn_cal(md_for_test, train_loader, hyps['bn_cal_num_iters'], device) + + # generate seperate surrogate DNN + test_sd = elastic_dnn_util.extract_surrogate_dnn_via_samples_with_perf_test(md_for_test, x) + + self.models['md'].models_dict['main'] = test_sd + self.models['md'].to_eval_mode() + val_acc = self.models['md'].get_accuracy(val_loader) + + val_accs[f'{val_sparsity:.4f}'] = val_acc + avg_val_acc += val_acc + + avg_val_acc /= hyps['val_num_sparsities'] + + self.models['md'].models_dict['main'] = cur_md + self.models['md'].models_dict['bn_stats'] = bn_stats + + self.models['md'].save_model(os.path.join(self.res_save_dir, 'models/md_last.pt')) + self.models['fm'].save_model(os.path.join(self.res_save_dir, 'models/fm_last.pt')) + + if avg_val_acc > best_avg_val_acc: + best_avg_val_acc = avg_val_acc + self.models['md'].save_model(os.path.join(self.res_save_dir, 'models/md_best.pt')) + self.models['fm'].save_model(os.path.join(self.res_save_dir, 'models/fm_best.pt')) + + tb_writer.add_scalars(f'losses', dict(task=task_loss, l1_reg=l1_reg_loss, total=total_loss), iter_index) + pbar.set_description(f'loss: {total_loss:.6f}') + if (iter_index + 1) >= hyps['val_freq']: + tb_writer.add_scalars(f'accs/val_accs', val_accs, iter_index) + tb_writer.add_scalar(f'accs/avg_val_acc', avg_val_acc, iter_index) + pbar.set_description(f'loss: {total_loss:.6f}, task_loss: {task_loss:.6f}, ' + f'l1_loss: {l1_reg_loss:.6f}, avg_val_acc: {avg_val_acc:.4f}') + \ No newline at end of file diff --git a/new_impl/cv/elasticdnn/api/algs/md_pretraining_wo_fbs.py b/new_impl/cv/elasticdnn/api/algs/md_pretraining_wo_fbs.py new file mode 100644 index 0000000000000000000000000000000000000000..28034221fc981d30b61b721f364759c9e617d8df --- /dev/null +++ b/new_impl/cv/elasticdnn/api/algs/md_pretraining_wo_fbs.py @@ -0,0 +1,387 @@ +# from typing import Any, Dict +# from schema import Schema, Or +# import schema +# from data import Scenario, MergedDataset +# from methods.base.alg import BaseAlg +# from data import build_dataloader +# from ..model import ElasticDNN_OfflineFMModel, ElasticDNN_OfflineMDModel +# from ...model.base import ElasticDNNUtil +# import torch.optim +# import tqdm +# import torch.nn.functional as F +# from torch import nn +# from utils.dl.common.env import create_tbwriter +# import os +# import random +# import numpy as np +# from copy import deepcopy +# from utils.dl.common.model import LayerActivation2, get_module +# from utils.common.log import logger + + +# class ElasticDNN_MDPretrainingWoFBSAlg(BaseAlg): +# """ +# TODO: fine-tuned FM -> init MD -> trained MD -> construct indexes (only between similar weights) and fine-tune +# """ +# def get_required_models_schema(self) -> Schema: +# return Schema({ +# 'fm': ElasticDNN_OfflineFMModel, +# 'md': ElasticDNN_OfflineMDModel +# }) + +# def get_required_hyp_schema(self) -> Schema: +# return Schema({ +# 'launch_tbboard': bool, + +# 'samples_size': object, +# 'generate_md_width_ratio': int, + +# 'train_batch_size': int, +# 'val_batch_size': int, +# 'num_workers': int, +# 'optimizer': str, +# 'optimizer_args': dict, +# 'scheduler': str, +# 'scheduler_args': dict, +# 'num_iters': int, +# 'val_freq': int, +# 'distill_loss_weight': float +# }) + +# def run(self, scenario: Scenario, hyps: Dict) -> Dict[str, Any]: +# super().run(scenario, hyps) + +# assert isinstance(self.models['md'], ElasticDNN_OfflineMDModel) # for auto completion +# assert isinstance(self.models['fm'], ElasticDNN_OfflineFMModel) # for auto completion + +# # 1. add FBS +# device = self.models['md'].device + +# if self.models['md'].models_dict['main'] == -1: +# logger.info(f'init master DNN by reducing width of an adapted foundation model (already tuned by LoRA)...') + +# before_fm_model = deepcopy(self.models['fm'].models_dict['main']) +# lora_util = self.models['fm'].get_lora_util() + +# sample = hyps['samples_size'] +# if isinstance(sample, (tuple, list)) and isinstance(sample[0], int): +# sample = torch.rand(hyps['samples_size']).to(device) + +# lora_absorbed_fm_model = lora_util.absorb_lora_and_recover_net_structure(self.models['fm'].models_dict['main'], +# sample) +# self.models['fm'].models_dict['main'] = lora_absorbed_fm_model +# master_dnn = self.models['fm'].generate_md_by_reducing_width(hyps['generate_md_width_ratio'], +# sample) +# self.models['fm'].models_dict['main'] = before_fm_model + +# self.models['md'].models_dict['main'] = master_dnn +# self.models['md'].to(device) + + + +# # 2. train (knowledge distillation, index relationship) +# offline_datasets = scenario.get_offline_datasets() +# train_dataset = MergedDataset([d['train'] for d in offline_datasets.values()]) +# val_dataset = MergedDataset([d['val'] for d in offline_datasets.values()]) +# train_loader = iter(build_dataloader(train_dataset, hyps['train_batch_size'], hyps['num_workers'], +# True, None)) +# val_loader = build_dataloader(val_dataset, hyps['val_batch_size'], hyps['num_workers'], +# False, False) + +# # val_acc = self.models['md'].get_accuracy(val_loader) +# # print(val_acc) +# # exit() + +# # 2.1 train whole master DNN (knowledge distillation) +# self.models['md'].to_train_mode() +# for p in master_dnn.parameters(): +# p.requires_grad = True + +# if hasattr(self.models['md'], 'get_trained_params'): +# trained_p = self.models['md'].get_trained_params() +# logger.info(f'use custom trained parameters!!') +# else: +# trained_p = self.models['md'].models_dict['main'].parameters() +# for p in trained_p: +# p.requires_grad = True +# optimizer = torch.optim.__dict__[hyps['optimizer']]([ +# {'params': trained_p, **hyps['optimizer_args']} +# ]) +# scheduler = torch.optim.lr_scheduler.__dict__[hyps['scheduler']](optimizer, **hyps['scheduler_args']) +# tb_writer = create_tbwriter(os.path.join(self.res_save_dir, 'tb_log'), launch_tbboard=hyps['launch_tbboard']) +# pbar = tqdm.tqdm(range(hyps['num_iters']), dynamic_ncols=True) +# best_avg_val_acc = 0. + +# md_output_hook = None + +# for iter_index in pbar: +# self.models['md'].to_train_mode() +# self.models['fm'].to_eval_mode() + +# # rand_sparsity = random.random() * (hyps['max_sparsity'] - hyps['min_sparsity']) + hyps['min_sparsity'] +# # elastic_dnn_util.set_master_dnn_sparsity(self.models['md'].models_dict['main'], rand_sparsity) +# if md_output_hook is None: +# md_output_hook = self.models['md'].get_feature_hook() +# fm_output_hook = self.models['fm'].get_feature_hook() + +# x, y = next(train_loader) +# if isinstance(x, dict): +# for k, v in x.items(): +# if isinstance(v, torch.Tensor): +# x[k] = v.to(device) +# y = y.to(device) +# else: +# x, y = x.to(device), y.to(device) + +# with torch.no_grad(): +# fm_output = self.models['fm'].infer(x) +# task_loss = self.models['md'].forward_to_get_task_loss(x, y) + +# if isinstance(md_output_hook, (tuple, list)): +# distill_loss = 0. +# for h1, h2 in zip(md_output_hook, fm_output_hook): +# md_output = h1.output +# fm_output = h2.output +# distill_loss += hyps['distill_loss_weight'] * self.models['md'].get_distill_loss(md_output, fm_output) +# else: +# md_output = md_output_hook.output +# fm_output = fm_output_hook.output +# distill_loss = hyps['distill_loss_weight'] * self.models['md'].get_distill_loss(md_output, fm_output) + +# total_loss = task_loss + distill_loss + +# optimizer.zero_grad() +# total_loss.backward() + +# # for n, p in self.models['md'].models_dict['main'].named_parameters(): +# # if p.grad is not None: +# # print(n) +# # exit() + +# optimizer.step() +# scheduler.step() + +# if (iter_index + 1) % hyps['val_freq'] == 0: + +# # elastic_dnn_util.clear_cached_channel_attention_in_master_dnn(self.models['md'].models_dict['main']) +# if isinstance(md_output_hook, (tuple, list)): +# [h.remove() for h in md_output_hook] +# [h.remove() for h in fm_output_hook] +# else: +# md_output_hook.remove() +# fm_output_hook.remove() + +# md_output_hook = None +# fm_output_hook = None + +# cur_md = self.models['md'].models_dict['main'] +# md_for_test = deepcopy(self.models['md'].models_dict['main']) +# val_acc = 0. + +# self.models['md'].models_dict['main'] = md_for_test +# self.models['md'].to_eval_mode() +# val_acc = self.models['md'].get_accuracy(val_loader) + +# self.models['md'].models_dict['main'] = cur_md + +# self.models['md'].save_model(os.path.join(self.res_save_dir, 'models/md_last.pt')) +# self.models['fm'].save_model(os.path.join(self.res_save_dir, 'models/fm_last.pt')) + +# if val_acc > best_avg_val_acc: +# best_avg_val_acc = val_acc +# self.models['md'].save_model(os.path.join(self.res_save_dir, 'models/md_best.pt')) +# self.models['fm'].save_model(os.path.join(self.res_save_dir, 'models/fm_best.pt')) + +# tb_writer.add_scalars(f'losses', dict(task=task_loss, distill=distill_loss, total=total_loss), iter_index) +# pbar.set_description(f'loss: {total_loss:.6f}') +# if (iter_index + 1) >= hyps['val_freq']: +# tb_writer.add_scalar(f'accs/val_acc', val_acc, iter_index) +# pbar.set_description(f'loss: {total_loss:.6f}, val_acc: {val_acc:.4f}') + +# code below is commented on 0716 17:49, because of a bug that the loss cannot be gradient decented +# (bug confirmed, why? I dont know :) + + + + + + + + +from typing import Any, Dict +from schema import Schema, Or +import schema +from data import Scenario, MergedDataset +from methods.base.alg import BaseAlg +from data import build_dataloader +from ..model import ElasticDNN_OfflineFMModel, ElasticDNN_OfflineMDModel +from ...model.base import ElasticDNNUtil +import torch.optim +import tqdm +import torch.nn.functional as F +from torch import nn +from utils.dl.common.env import create_tbwriter +import os +import random +import numpy as np +from copy import deepcopy +from utils.dl.common.model import LayerActivation2, get_module +from utils.common.log import logger +from torchvision.transforms import Compose + +class ElasticDNN_MDPretrainingWoFBSAlg(BaseAlg): + """ + TODO: fine-tuned FM -> init MD -> trained MD -> construct indexes (only between similar weights) and fine-tune + """ + def get_required_models_schema(self) -> Schema: + return Schema({ + 'fm': ElasticDNN_OfflineFMModel, + 'md': ElasticDNN_OfflineMDModel + }) + + def get_required_hyp_schema(self) -> Schema: + from schema import Optional + return Schema({ + 'launch_tbboard': bool, + + 'samples_size': any, + 'generate_md_width_ratio': int, + + 'train_batch_size': int, + 'val_batch_size': int, + 'num_workers': int, + 'optimizer': str, + 'optimizer_args': dict, + 'scheduler': str, + 'scheduler_args': dict, + 'num_iters': int, + 'val_freq': int, + 'distill_loss_weight': float, + + Optional('transform'): Compose, + }) + + def run(self, scenario: Scenario, hyps: Dict, collate_fn=None) -> Dict[str, Any]: + super().run(scenario, hyps) + + assert isinstance(self.models['md'], ElasticDNN_OfflineMDModel) # for auto completion + assert isinstance(self.models['fm'], ElasticDNN_OfflineFMModel) # for auto completion + + # 1. add FBS + device = self.models['md'].device + + if self.models['md'].models_dict['main'] == -1: + logger.info(f'init master DNN by reducing width of an adapted foundation model (already tuned by LoRA)...') + + before_fm_model = deepcopy(self.models['fm'].models_dict['main']) + lora_util = self.models['fm'].get_lora_util() + + sample = hyps['samples_size'] + if isinstance(sample, (tuple, list)) and isinstance(sample[0], int): + sample = torch.rand(hyps['samples_size']).to(device) + + lora_absorbed_fm_model = lora_util.absorb_lora_and_recover_net_structure(self.models['fm'].models_dict['main'], + sample) + self.models['fm'].models_dict['main'] = lora_absorbed_fm_model + master_dnn = self.models['fm'].generate_md_by_reducing_width(hyps['generate_md_width_ratio'], + sample) + self.models['fm'].models_dict['main'] = before_fm_model + + self.models['md'].models_dict['main'] = master_dnn + self.models['md'].to(device) + + # 2. train (knowledge distillation, index relationship) + if 'transform' in hyps.keys(): + offline_datasets = scenario.get_offline_datasets(transform=hyps['transform']) + else: + offline_datasets = scenario.get_offline_datasets() + train_dataset = MergedDataset([d['train'] for d in offline_datasets.values()]) + val_dataset = MergedDataset([d['val'] for d in offline_datasets.values()]) + train_loader = iter(build_dataloader(train_dataset, hyps['train_batch_size'], hyps['num_workers'], + True, None, collate_fn=collate_fn)) + val_loader = build_dataloader(val_dataset, hyps['val_batch_size'], hyps['num_workers'], + False, False, collate_fn=collate_fn) + + # logger.info(f'FM acc: {self.models["fm"].get_accuracy(val_loader):.4f}') + + # 2.1 train whole master DNN (knowledge distillation) + for p in master_dnn.parameters(): + p.requires_grad = True + self.models['md'].to_train_mode() + + optimizer = torch.optim.__dict__[hyps['optimizer']]([ + {'params': self.models['md'].models_dict['main'].parameters(), **hyps['optimizer_args']} + ]) + scheduler = torch.optim.lr_scheduler.__dict__[hyps['scheduler']](optimizer, **hyps['scheduler_args']) + tb_writer = create_tbwriter(os.path.join(self.res_save_dir, 'tb_log'), launch_tbboard=hyps['launch_tbboard']) + pbar = tqdm.tqdm(range(hyps['num_iters']), dynamic_ncols=True) + best_avg_val_acc = 0. + + md_output_hook = None + + for iter_index in pbar: + self.models['md'].to_train_mode() + self.models['fm'].to_eval_mode() + + # rand_sparsity = random.random() * (hyps['max_sparsity'] - hyps['min_sparsity']) + hyps['min_sparsity'] + # elastic_dnn_util.set_master_dnn_sparsity(self.models['md'].models_dict['main'], rand_sparsity) + if md_output_hook is None: + md_output_hook = self.models['md'].get_feature_hook() + fm_output_hook = self.models['fm'].get_feature_hook() + + x, y = next(train_loader) + if isinstance(x, dict): + for k, v in x.items(): + if isinstance(v, torch.Tensor): + x[k] = v.to(device) + y = y.to(device) + else: + x, y = x.to(device), y.to(device) + + with torch.no_grad(): + fm_output = self.models['fm'].infer(x) + task_loss = self.models['md'].forward_to_get_task_loss(x, y) + + md_output = md_output_hook.output + fm_output = fm_output_hook.output + + distill_loss = hyps['distill_loss_weight'] * self.models['md'].get_distill_loss(md_output, fm_output) + total_loss = task_loss + distill_loss + + optimizer.zero_grad() + total_loss.backward() + optimizer.step() + scheduler.step() + + if (iter_index + 1) % hyps['val_freq'] == 0: + + # elastic_dnn_util.clear_cached_channel_attention_in_master_dnn(self.models['md'].models_dict['main']) + md_output_hook.remove() + md_output_hook = None + fm_output_hook.remove() + fm_output_hook = None + + cur_md = self.models['md'].models_dict['main'] + md_for_test = deepcopy(self.models['md'].models_dict['main']) + val_acc = 0. + + self.models['md'].models_dict['main'] = md_for_test + self.models['md'].to_eval_mode() + val_acc = self.models['md'].get_accuracy(val_loader) + + self.models['md'].models_dict['main'] = cur_md + + self.models['md'].save_model(os.path.join(self.res_save_dir, 'models/md_last.pt')) + self.models['fm'].save_model(os.path.join(self.res_save_dir, 'models/fm_last.pt')) + + if val_acc > best_avg_val_acc: + best_avg_val_acc = val_acc + self.models['md'].save_model(os.path.join(self.res_save_dir, 'models/md_best.pt')) + self.models['fm'].save_model(os.path.join(self.res_save_dir, 'models/fm_best.pt')) + + tb_writer.add_scalars(f'losses', dict(task=task_loss, distill=distill_loss, total=total_loss), iter_index) + pbar.set_description(f'loss: {total_loss:.6f}') + if (iter_index + 1) >= hyps['val_freq']: + tb_writer.add_scalar(f'accs/val_acc', val_acc, iter_index) + pbar.set_description(f'loss: {total_loss:.6f}, val_acc: {val_acc:.4f}') + \ No newline at end of file diff --git a/new_impl/cv/elasticdnn/api/algs/md_pretraining_wo_fbs_clip_debug.py b/new_impl/cv/elasticdnn/api/algs/md_pretraining_wo_fbs_clip_debug.py new file mode 100644 index 0000000000000000000000000000000000000000..a1b0f3b54fa9bc00d99dcb5de64b929c90d904da --- /dev/null +++ b/new_impl/cv/elasticdnn/api/algs/md_pretraining_wo_fbs_clip_debug.py @@ -0,0 +1,412 @@ +# from typing import Any, Dict +# from schema import Schema, Or +# import schema +# from data import Scenario, MergedDataset +# from methods.base.alg import BaseAlg +# from data import build_dataloader +# from ..model import ElasticDNN_OfflineFMModel, ElasticDNN_OfflineMDModel +# from ...model.base import ElasticDNNUtil +# import torch.optim +# import tqdm +# import torch.nn.functional as F +# from torch import nn +# from utils.dl.common.env import create_tbwriter +# import os +# import random +# import numpy as np +# from copy import deepcopy +# from utils.dl.common.model import LayerActivation2, get_module +# from utils.common.log import logger + + +# class ElasticDNN_MDPretrainingWoFBSAlg(BaseAlg): +# """ +# TODO: fine-tuned FM -> init MD -> trained MD -> construct indexes (only between similar weights) and fine-tune +# """ +# def get_required_models_schema(self) -> Schema: +# return Schema({ +# 'fm': ElasticDNN_OfflineFMModel, +# 'md': ElasticDNN_OfflineMDModel +# }) + +# def get_required_hyp_schema(self) -> Schema: +# return Schema({ +# 'launch_tbboard': bool, + +# 'samples_size': object, +# 'generate_md_width_ratio': int, + +# 'train_batch_size': int, +# 'val_batch_size': int, +# 'num_workers': int, +# 'optimizer': str, +# 'optimizer_args': dict, +# 'scheduler': str, +# 'scheduler_args': dict, +# 'num_iters': int, +# 'val_freq': int, +# 'distill_loss_weight': float +# }) + +# def run(self, scenario: Scenario, hyps: Dict) -> Dict[str, Any]: +# super().run(scenario, hyps) + +# assert isinstance(self.models['md'], ElasticDNN_OfflineMDModel) # for auto completion +# assert isinstance(self.models['fm'], ElasticDNN_OfflineFMModel) # for auto completion + +# # 1. add FBS +# device = self.models['md'].device + +# if self.models['md'].models_dict['main'] == -1: +# logger.info(f'init master DNN by reducing width of an adapted foundation model (already tuned by LoRA)...') + +# before_fm_model = deepcopy(self.models['fm'].models_dict['main']) +# lora_util = self.models['fm'].get_lora_util() + +# sample = hyps['samples_size'] +# if isinstance(sample, (tuple, list)) and isinstance(sample[0], int): +# sample = torch.rand(hyps['samples_size']).to(device) + +# lora_absorbed_fm_model = lora_util.absorb_lora_and_recover_net_structure(self.models['fm'].models_dict['main'], +# sample) +# self.models['fm'].models_dict['main'] = lora_absorbed_fm_model +# master_dnn = self.models['fm'].generate_md_by_reducing_width(hyps['generate_md_width_ratio'], +# sample) +# self.models['fm'].models_dict['main'] = before_fm_model + +# self.models['md'].models_dict['main'] = master_dnn +# self.models['md'].to(device) + + + +# # 2. train (knowledge distillation, index relationship) +# offline_datasets = scenario.get_offline_datasets() +# train_dataset = MergedDataset([d['train'] for d in offline_datasets.values()]) +# val_dataset = MergedDataset([d['val'] for d in offline_datasets.values()]) +# train_loader = iter(build_dataloader(train_dataset, hyps['train_batch_size'], hyps['num_workers'], +# True, None)) +# val_loader = build_dataloader(val_dataset, hyps['val_batch_size'], hyps['num_workers'], +# False, False) + +# # val_acc = self.models['md'].get_accuracy(val_loader) +# # print(val_acc) +# # exit() + +# # 2.1 train whole master DNN (knowledge distillation) +# self.models['md'].to_train_mode() +# for p in master_dnn.parameters(): +# p.requires_grad = True + +# if hasattr(self.models['md'], 'get_trained_params'): +# trained_p = self.models['md'].get_trained_params() +# logger.info(f'use custom trained parameters!!') +# else: +# trained_p = self.models['md'].models_dict['main'].parameters() +# for p in trained_p: +# p.requires_grad = True +# optimizer = torch.optim.__dict__[hyps['optimizer']]([ +# {'params': trained_p, **hyps['optimizer_args']} +# ]) +# scheduler = torch.optim.lr_scheduler.__dict__[hyps['scheduler']](optimizer, **hyps['scheduler_args']) +# tb_writer = create_tbwriter(os.path.join(self.res_save_dir, 'tb_log'), launch_tbboard=hyps['launch_tbboard']) +# pbar = tqdm.tqdm(range(hyps['num_iters']), dynamic_ncols=True) +# best_avg_val_acc = 0. + +# md_output_hook = None + +# for iter_index in pbar: +# self.models['md'].to_train_mode() +# self.models['fm'].to_eval_mode() + +# # rand_sparsity = random.random() * (hyps['max_sparsity'] - hyps['min_sparsity']) + hyps['min_sparsity'] +# # elastic_dnn_util.set_master_dnn_sparsity(self.models['md'].models_dict['main'], rand_sparsity) +# if md_output_hook is None: +# md_output_hook = self.models['md'].get_feature_hook() +# fm_output_hook = self.models['fm'].get_feature_hook() + +# x, y = next(train_loader) +# if isinstance(x, dict): +# for k, v in x.items(): +# if isinstance(v, torch.Tensor): +# x[k] = v.to(device) +# y = y.to(device) +# else: +# x, y = x.to(device), y.to(device) + +# with torch.no_grad(): +# fm_output = self.models['fm'].infer(x) +# task_loss = self.models['md'].forward_to_get_task_loss(x, y) + +# if isinstance(md_output_hook, (tuple, list)): +# distill_loss = 0. +# for h1, h2 in zip(md_output_hook, fm_output_hook): +# md_output = h1.output +# fm_output = h2.output +# distill_loss += hyps['distill_loss_weight'] * self.models['md'].get_distill_loss(md_output, fm_output) +# else: +# md_output = md_output_hook.output +# fm_output = fm_output_hook.output +# distill_loss = hyps['distill_loss_weight'] * self.models['md'].get_distill_loss(md_output, fm_output) + +# total_loss = task_loss + distill_loss + +# optimizer.zero_grad() +# total_loss.backward() + +# # for n, p in self.models['md'].models_dict['main'].named_parameters(): +# # if p.grad is not None: +# # print(n) +# # exit() + +# optimizer.step() +# scheduler.step() + +# if (iter_index + 1) % hyps['val_freq'] == 0: + +# # elastic_dnn_util.clear_cached_channel_attention_in_master_dnn(self.models['md'].models_dict['main']) +# if isinstance(md_output_hook, (tuple, list)): +# [h.remove() for h in md_output_hook] +# [h.remove() for h in fm_output_hook] +# else: +# md_output_hook.remove() +# fm_output_hook.remove() + +# md_output_hook = None +# fm_output_hook = None + +# cur_md = self.models['md'].models_dict['main'] +# md_for_test = deepcopy(self.models['md'].models_dict['main']) +# val_acc = 0. + +# self.models['md'].models_dict['main'] = md_for_test +# self.models['md'].to_eval_mode() +# val_acc = self.models['md'].get_accuracy(val_loader) + +# self.models['md'].models_dict['main'] = cur_md + +# self.models['md'].save_model(os.path.join(self.res_save_dir, 'models/md_last.pt')) +# self.models['fm'].save_model(os.path.join(self.res_save_dir, 'models/fm_last.pt')) + +# if val_acc > best_avg_val_acc: +# best_avg_val_acc = val_acc +# self.models['md'].save_model(os.path.join(self.res_save_dir, 'models/md_best.pt')) +# self.models['fm'].save_model(os.path.join(self.res_save_dir, 'models/fm_best.pt')) + +# tb_writer.add_scalars(f'losses', dict(task=task_loss, distill=distill_loss, total=total_loss), iter_index) +# pbar.set_description(f'loss: {total_loss:.6f}') +# if (iter_index + 1) >= hyps['val_freq']: +# tb_writer.add_scalar(f'accs/val_acc', val_acc, iter_index) +# pbar.set_description(f'loss: {total_loss:.6f}, val_acc: {val_acc:.4f}') + +# code below is commented on 0716 17:49, because of a bug that the loss cannot be gradient decented +# (bug confirmed, why? I dont know :) + + + + + + + + +from typing import Any, Dict +from schema import Schema, Or +import schema +from data import Scenario, MergedDataset +from methods.base.alg import BaseAlg +from data import build_dataloader +from ..model import ElasticDNN_OfflineFMModel, ElasticDNN_OfflineMDModel +from ...model.base import ElasticDNNUtil +import torch.optim +import tqdm +import torch.nn.functional as F +from torch import nn +from utils.dl.common.env import create_tbwriter +import os +import random +import numpy as np +from copy import deepcopy +from utils.dl.common.model import LayerActivation2, get_module +from utils.common.log import logger +from data import split_dataset + + +class ElasticDNN_MDPretrainingWoFBSAlg(BaseAlg): + """ + TODO: fine-tuned FM -> init MD -> trained MD -> construct indexes (only between similar weights) and fine-tune + """ + def get_required_models_schema(self) -> Schema: + return Schema({ + 'fm': ElasticDNN_OfflineFMModel, + 'md': ElasticDNN_OfflineMDModel + }) + + def get_required_hyp_schema(self) -> Schema: + return Schema({ + 'launch_tbboard': bool, + + 'samples_size': any, + 'generate_md_width_ratio': int, + + 'train_batch_size': int, + 'val_batch_size': int, + 'num_workers': int, + 'optimizer': str, + 'optimizer_args': dict, + 'scheduler': str, + 'scheduler_args': dict, + 'num_iters': int, + 'val_freq': int, + 'distill_loss_weight': float + }) + + def run(self, scenario: Scenario, hyps: Dict) -> Dict[str, Any]: + super().run(scenario, hyps) + + assert isinstance(self.models['md'], ElasticDNN_OfflineMDModel) # for auto completion + assert isinstance(self.models['fm'], ElasticDNN_OfflineFMModel) # for auto completion + + # 1. add FBS + device = self.models['md'].device + + if self.models['md'].models_dict['main'] == -1: + logger.info(f'init master DNN by reducing width of an adapted foundation model (already tuned by LoRA)...') + + before_fm_model = deepcopy(self.models['fm'].models_dict['main']) + lora_util = self.models['fm'].get_lora_util() + + sample = hyps['samples_size'] + if isinstance(sample, (tuple, list)) and isinstance(sample[0], int): + sample = torch.rand(hyps['samples_size']).to(device) + + lora_absorbed_fm_model = lora_util.absorb_lora_and_recover_net_structure(self.models['fm'].models_dict['main'], + sample) + self.models['fm'].models_dict['main'] = lora_absorbed_fm_model + master_dnn = self.models['fm'].generate_md_by_reducing_width(hyps['generate_md_width_ratio'], + sample) + self.models['fm'].models_dict['main'] = before_fm_model + + self.models['md'].models_dict['main'] = master_dnn + self.models['md'].to(device) + + # 2. train (knowledge distillation, index relationship) + offline_datasets = scenario.get_offline_datasets() + train_dataset = MergedDataset([d['train'] for d in offline_datasets.values()]) + val_dataset = MergedDataset([d['val'] for d in offline_datasets.values()]) + train_loader = iter(build_dataloader(train_dataset, hyps['train_batch_size'], hyps['num_workers'], + True, None)) + val_loader = build_dataloader(val_dataset, hyps['val_batch_size'], hyps['num_workers'], + False, False) + + # logger.info(f'FM acc: {self.models["fm"].get_accuracy(val_loader):.4f}') + + # 2.1 train whole master DNN (knowledge distillation) + + # self.models['md'].models_dict['main'] = master_dnn = nn.DataParallel(master_dnn.to(device), device_ids=['cuda:0'], output_device=['cuda:0']) + + for p in master_dnn.model.parameters(): + p.requires_grad = True + for p in master_dnn.model.text_model.parameters(): + p.requires_grad = False + self.models['md'].to_train_mode() + + optimizer = torch.optim.__dict__[hyps['optimizer']]([ + {'params': [p for n, p in self.models['md'].models_dict['main'].named_parameters() if 'text_model' not in n], **hyps['optimizer_args']} + ]) + scheduler = torch.optim.lr_scheduler.__dict__[hyps['scheduler']](optimizer, **hyps['scheduler_args']) + tb_writer = create_tbwriter(os.path.join(self.res_save_dir, 'tb_log'), launch_tbboard=hyps['launch_tbboard']) + pbar = tqdm.tqdm(range(hyps['num_iters']), dynamic_ncols=True) + best_avg_val_acc = 0. + + md_output_hook = None + + train_ratio = 0.005 + + for iter_index in pbar: + + if iter_index % 500 == 0: + train_ratio += 0.005 + + train_dataset = MergedDataset([d['train'] for d in offline_datasets.values()]) + train_dataset = split_dataset(train_dataset, max(hyps['train_batch_size'], int(train_ratio * len(train_dataset))))[0] + train_loader = iter(build_dataloader(train_dataset, hyps['train_batch_size'], hyps['num_workers'], True, None)) + + self.models['md'].to_train_mode() + # self.models['fm'].to_eval_mode() + + # rand_sparsity = random.random() * (hyps['max_sparsity'] - hyps['min_sparsity']) + hyps['min_sparsity'] + # elastic_dnn_util.set_master_dnn_sparsity(self.models['md'].models_dict['main'], rand_sparsity) + if md_output_hook is None: + md_output_hook = self.models['md'].get_feature_hook() + fm_output_hook = self.models['fm'].get_feature_hook() + + x, y = next(train_loader) + if isinstance(x, dict): + for k, v in x.items(): + if isinstance(v, torch.Tensor): + x[k] = v.to(device) + y = y.to(device) + else: + x, y = x.to(device), y.to(device) + + # with torch.no_grad(): + # fm_output = self.models['fm'].infer(x) + # task_loss = self.models['md'].forward_to_get_task_loss(x, y) + task_loss = 0. + with torch.no_grad(): + fm_output = self.models['fm'].forward_to_get_task_loss(x, y) + md_output = self.models['md'].forward_to_get_task_loss(x, y) + + if iter_index % 10 == 0: + print(fm_output, md_output) + # md_output = md_output_hook.output + # fm_output = fm_output_hook.output + + distill_loss = hyps['distill_loss_weight'] * self.models['md'].get_distill_loss(md_output, fm_output) + # distill_loss = 0. + + # logit_diff_loss = -((md_output - md_output.mean(dim=1).unsqueeze(1)) ** 2).sum() * 0.01 + logit_diff_loss = 0. + + total_loss = task_loss + distill_loss + logit_diff_loss + + optimizer.zero_grad() + total_loss.backward() + optimizer.step() + scheduler.step() + + if (iter_index + 1) % hyps['val_freq'] == 0: + + # elastic_dnn_util.clear_cached_channel_attention_in_master_dnn(self.models['md'].models_dict['main']) + md_output_hook.remove() + md_output_hook = None + fm_output_hook.remove() + fm_output_hook = None + + cur_md = self.models['md'].models_dict['main'] + md_for_test = deepcopy(self.models['md'].models_dict['main']) + val_acc = 0. + + self.models['md'].models_dict['main'] = md_for_test + self.models['md'].to_eval_mode() + val_acc = self.models['md'].get_accuracy(val_loader) + + self.models['md'].models_dict['main'] = cur_md + + self.models['md'].save_model(os.path.join(self.res_save_dir, 'models/md_last.pt')) + self.models['fm'].save_model(os.path.join(self.res_save_dir, 'models/fm_last.pt')) + + if val_acc > best_avg_val_acc: + best_avg_val_acc = val_acc + self.models['md'].save_model(os.path.join(self.res_save_dir, 'models/md_best.pt')) + self.models['fm'].save_model(os.path.join(self.res_save_dir, 'models/fm_best.pt')) + + tb_writer.add_scalars(f'losses', dict(task=task_loss, distill=distill_loss, logit_diff_loss=logit_diff_loss, total=total_loss), iter_index) + # tb_writer.add_scalar(f'lr', optimizer.param_groups[0]['lr'], iter_index) + tb_writer.add_scalar(f'lr', optimizer.param_groups[0]['lr'], iter_index) + tb_writer.add_scalar(f'train_data_size', len(train_dataset), iter_index) + + pbar.set_description(f'loss: {total_loss:.6f}, logit_diff_loss: {logit_diff_loss:.6f}') + if (iter_index + 1) >= hyps['val_freq']: + tb_writer.add_scalar(f'accs/val_acc', val_acc, iter_index) + pbar.set_description(f'loss: {total_loss:.6f}, logit_diff_loss: {logit_diff_loss:.6f}, val_acc: {val_acc:.4f}') + \ No newline at end of file diff --git a/new_impl/cv/elasticdnn/api/algs/md_pretraining_wo_fbs_glip.py b/new_impl/cv/elasticdnn/api/algs/md_pretraining_wo_fbs_glip.py new file mode 100644 index 0000000000000000000000000000000000000000..3c5e707e325fbdaefaa29e854cf70fab0e402330 --- /dev/null +++ b/new_impl/cv/elasticdnn/api/algs/md_pretraining_wo_fbs_glip.py @@ -0,0 +1,403 @@ +# from typing import Any, Dict +# from schema import Schema, Or +# import schema +# from data import Scenario, MergedDataset +# from methods.base.alg import BaseAlg +# from data import build_dataloader +# from ..model import ElasticDNN_OfflineFMModel, ElasticDNN_OfflineMDModel +# from ...model.base import ElasticDNNUtil +# import torch.optim +# import tqdm +# import torch.nn.functional as F +# from torch import nn +# from utils.dl.common.env import create_tbwriter +# import os +# import random +# import numpy as np +# from copy import deepcopy +# from utils.dl.common.model import LayerActivation2, get_module +# from utils.common.log import logger + + +# class ElasticDNN_MDPretrainingWoFBSAlg(BaseAlg): +# """ +# TODO: fine-tuned FM -> init MD -> trained MD -> construct indexes (only between similar weights) and fine-tune +# """ +# def get_required_models_schema(self) -> Schema: +# return Schema({ +# 'fm': ElasticDNN_OfflineFMModel, +# 'md': ElasticDNN_OfflineMDModel +# }) + +# def get_required_hyp_schema(self) -> Schema: +# return Schema({ +# 'launch_tbboard': bool, + +# 'samples_size': object, +# 'generate_md_width_ratio': int, + +# 'train_batch_size': int, +# 'val_batch_size': int, +# 'num_workers': int, +# 'optimizer': str, +# 'optimizer_args': dict, +# 'scheduler': str, +# 'scheduler_args': dict, +# 'num_iters': int, +# 'val_freq': int, +# 'distill_loss_weight': float +# }) + +# def run(self, scenario: Scenario, hyps: Dict) -> Dict[str, Any]: +# super().run(scenario, hyps) + +# assert isinstance(self.models['md'], ElasticDNN_OfflineMDModel) # for auto completion +# assert isinstance(self.models['fm'], ElasticDNN_OfflineFMModel) # for auto completion + +# # 1. add FBS +# device = self.models['md'].device + +# if self.models['md'].models_dict['main'] == -1: +# logger.info(f'init master DNN by reducing width of an adapted foundation model (already tuned by LoRA)...') + +# before_fm_model = deepcopy(self.models['fm'].models_dict['main']) +# lora_util = self.models['fm'].get_lora_util() + +# sample = hyps['samples_size'] +# if isinstance(sample, (tuple, list)) and isinstance(sample[0], int): +# sample = torch.rand(hyps['samples_size']).to(device) + +# lora_absorbed_fm_model = lora_util.absorb_lora_and_recover_net_structure(self.models['fm'].models_dict['main'], +# sample) +# self.models['fm'].models_dict['main'] = lora_absorbed_fm_model +# master_dnn = self.models['fm'].generate_md_by_reducing_width(hyps['generate_md_width_ratio'], +# sample) +# self.models['fm'].models_dict['main'] = before_fm_model + +# self.models['md'].models_dict['main'] = master_dnn +# self.models['md'].to(device) + + + +# # 2. train (knowledge distillation, index relationship) +# offline_datasets = scenario.get_offline_datasets() +# train_dataset = MergedDataset([d['train'] for d in offline_datasets.values()]) +# val_dataset = MergedDataset([d['val'] for d in offline_datasets.values()]) +# train_loader = iter(build_dataloader(train_dataset, hyps['train_batch_size'], hyps['num_workers'], +# True, None)) +# val_loader = build_dataloader(val_dataset, hyps['val_batch_size'], hyps['num_workers'], +# False, False) + +# # val_acc = self.models['md'].get_accuracy(val_loader) +# # print(val_acc) +# # exit() + +# # 2.1 train whole master DNN (knowledge distillation) +# self.models['md'].to_train_mode() +# for p in master_dnn.parameters(): +# p.requires_grad = True + +# if hasattr(self.models['md'], 'get_trained_params'): +# trained_p = self.models['md'].get_trained_params() +# logger.info(f'use custom trained parameters!!') +# else: +# trained_p = self.models['md'].models_dict['main'].parameters() +# for p in trained_p: +# p.requires_grad = True +# optimizer = torch.optim.__dict__[hyps['optimizer']]([ +# {'params': trained_p, **hyps['optimizer_args']} +# ]) +# scheduler = torch.optim.lr_scheduler.__dict__[hyps['scheduler']](optimizer, **hyps['scheduler_args']) +# tb_writer = create_tbwriter(os.path.join(self.res_save_dir, 'tb_log'), launch_tbboard=hyps['launch_tbboard']) +# pbar = tqdm.tqdm(range(hyps['num_iters']), dynamic_ncols=True) +# best_avg_val_acc = 0. + +# md_output_hook = None + +# for iter_index in pbar: +# self.models['md'].to_train_mode() +# self.models['fm'].to_eval_mode() + +# # rand_sparsity = random.random() * (hyps['max_sparsity'] - hyps['min_sparsity']) + hyps['min_sparsity'] +# # elastic_dnn_util.set_master_dnn_sparsity(self.models['md'].models_dict['main'], rand_sparsity) +# if md_output_hook is None: +# md_output_hook = self.models['md'].get_feature_hook() +# fm_output_hook = self.models['fm'].get_feature_hook() + +# x, y = next(train_loader) +# if isinstance(x, dict): +# for k, v in x.items(): +# if isinstance(v, torch.Tensor): +# x[k] = v.to(device) +# y = y.to(device) +# else: +# x, y = x.to(device), y.to(device) + +# with torch.no_grad(): +# fm_output = self.models['fm'].infer(x) +# task_loss = self.models['md'].forward_to_get_task_loss(x, y) + +# if isinstance(md_output_hook, (tuple, list)): +# distill_loss = 0. +# for h1, h2 in zip(md_output_hook, fm_output_hook): +# md_output = h1.output +# fm_output = h2.output +# distill_loss += hyps['distill_loss_weight'] * self.models['md'].get_distill_loss(md_output, fm_output) +# else: +# md_output = md_output_hook.output +# fm_output = fm_output_hook.output +# distill_loss = hyps['distill_loss_weight'] * self.models['md'].get_distill_loss(md_output, fm_output) + +# total_loss = task_loss + distill_loss + +# optimizer.zero_grad() +# total_loss.backward() + +# # for n, p in self.models['md'].models_dict['main'].named_parameters(): +# # if p.grad is not None: +# # print(n) +# # exit() + +# optimizer.step() +# scheduler.step() + +# if (iter_index + 1) % hyps['val_freq'] == 0: + +# # elastic_dnn_util.clear_cached_channel_attention_in_master_dnn(self.models['md'].models_dict['main']) +# if isinstance(md_output_hook, (tuple, list)): +# [h.remove() for h in md_output_hook] +# [h.remove() for h in fm_output_hook] +# else: +# md_output_hook.remove() +# fm_output_hook.remove() + +# md_output_hook = None +# fm_output_hook = None + +# cur_md = self.models['md'].models_dict['main'] +# md_for_test = deepcopy(self.models['md'].models_dict['main']) +# val_acc = 0. + +# self.models['md'].models_dict['main'] = md_for_test +# self.models['md'].to_eval_mode() +# val_acc = self.models['md'].get_accuracy(val_loader) + +# self.models['md'].models_dict['main'] = cur_md + +# self.models['md'].save_model(os.path.join(self.res_save_dir, 'models/md_last.pt')) +# self.models['fm'].save_model(os.path.join(self.res_save_dir, 'models/fm_last.pt')) + +# if val_acc > best_avg_val_acc: +# best_avg_val_acc = val_acc +# self.models['md'].save_model(os.path.join(self.res_save_dir, 'models/md_best.pt')) +# self.models['fm'].save_model(os.path.join(self.res_save_dir, 'models/fm_best.pt')) + +# tb_writer.add_scalars(f'losses', dict(task=task_loss, distill=distill_loss, total=total_loss), iter_index) +# pbar.set_description(f'loss: {total_loss:.6f}') +# if (iter_index + 1) >= hyps['val_freq']: +# tb_writer.add_scalar(f'accs/val_acc', val_acc, iter_index) +# pbar.set_description(f'loss: {total_loss:.6f}, val_acc: {val_acc:.4f}') + +# code below is commented on 0716 17:49, because of a bug that the loss cannot be gradient decented +# (bug confirmed, why? I dont know :) + + + + + + + + +from typing import Any, Dict +from schema import Schema, Or +import schema +from data import Scenario, MergedDataset +from methods.base.alg import BaseAlg +from data import build_dataloader +from ..model import ElasticDNN_OfflineFMModel, ElasticDNN_OfflineMDModel +from ...model.base import ElasticDNNUtil +import torch.optim +import tqdm +import torch.nn.functional as F +from torch import nn +from utils.dl.common.env import create_tbwriter +import os +import random +import numpy as np +from copy import deepcopy +from utils.dl.common.model import LayerActivation2, get_module +from utils.common.log import logger +from torchvision.transforms import Compose + +class ElasticDNN_MDPretrainingWoFBSAlg(BaseAlg): + """ + TODO: fine-tuned FM -> init MD -> trained MD -> construct indexes (only between similar weights) and fine-tune + """ + def get_required_models_schema(self) -> Schema: + return Schema({ + 'fm': ElasticDNN_OfflineFMModel, + 'md': ElasticDNN_OfflineMDModel + }) + + def get_required_hyp_schema(self) -> Schema: + from schema import Optional + return Schema({ + 'launch_tbboard': bool, + + 'samples_size': any, + 'generate_md_width_ratio': int, + + 'train_batch_size': int, + 'val_batch_size': int, + 'num_workers': int, + 'optimizer': str, + 'optimizer_args': dict, + 'scheduler': str, + 'scheduler_args': dict, + 'num_iters': int, + 'val_freq': int, + 'distill_loss_weight': float, + + Optional('transform'): Compose, + }) + + def run(self, scenario: Scenario, hyps: Dict, collate_fn=None) -> Dict[str, Any]: + super().run(scenario, hyps) + + assert isinstance(self.models['md'], ElasticDNN_OfflineMDModel) # for auto completion + assert isinstance(self.models['fm'], ElasticDNN_OfflineFMModel) # for auto completion + + # 1. add FBS + device = self.models['md'].device + + if self.models['md'].models_dict['main'] == -1: + logger.info(f'init master DNN by reducing width of an adapted foundation model (already tuned by LoRA)...') + + before_fm_model = deepcopy(self.models['fm'].models_dict['main']) + lora_util = self.models['fm'].get_lora_util() + + sample = hyps['samples_size'] + if isinstance(sample, (tuple, list)) and isinstance(sample[0], int): + sample = torch.rand(hyps['samples_size']).to(device) + + lora_absorbed_fm_model = lora_util.absorb_lora_and_recover_net_structure(self.models['fm'].models_dict['main'], + sample) + self.models['fm'].models_dict['main'] = lora_absorbed_fm_model + master_dnn = self.models['fm'].generate_md_by_reducing_width(hyps['generate_md_width_ratio'], + sample) + self.models['fm'].models_dict['main'] = before_fm_model + + self.models['md'].models_dict['main'] = master_dnn + self.models['md'].to(device) + + # 2. train (knowledge distillation, index relationship) + if 'transform' in hyps.keys(): + offline_datasets = scenario.get_offline_datasets(transform=hyps['transform']) + else: + offline_datasets = scenario.get_offline_datasets() + train_dataset = MergedDataset([d['train'] for d in offline_datasets.values()]) + val_dataset = MergedDataset([d['val'] for d in offline_datasets.values()]) + train_loader = iter(build_dataloader(train_dataset, hyps['train_batch_size'], hyps['num_workers'], + True, None, collate_fn=collate_fn)) + val_loader = build_dataloader(val_dataset, hyps['val_batch_size'], hyps['num_workers'], + False, False, collate_fn=collate_fn) + + # logger.info(f'FM acc: {self.models["fm"].get_accuracy(val_loader):.4f}') + + # 2.1 train whole master DNN (knowledge distillation) + for p in master_dnn.parameters(): + p.requires_grad = True + self.models['md'].to_train_mode() + + optimizer = torch.optim.__dict__[hyps['optimizer']]([ + {'params': self.models['md'].models_dict['main'].parameters(), **hyps['optimizer_args']} + ]) + scheduler = torch.optim.lr_scheduler.__dict__[hyps['scheduler']](optimizer, **hyps['scheduler_args']) + tb_writer = create_tbwriter(os.path.join(self.res_save_dir, 'tb_log'), launch_tbboard=hyps['launch_tbboard']) + pbar = tqdm.tqdm(range(hyps['num_iters']), dynamic_ncols=True) + best_avg_val_acc = 0. + + md_output_hooks = None + + for iter_index in pbar: + self.models['md'].to_train_mode() + self.models['fm'].to_eval_mode() + + # rand_sparsity = random.random() * (hyps['max_sparsity'] - hyps['min_sparsity']) + hyps['min_sparsity'] + # elastic_dnn_util.set_master_dnn_sparsity(self.models['md'].models_dict['main'], rand_sparsity) + if md_output_hooks is None: + md_output_hooks = self.models['md'].get_feature_hooks() + fm_output_hooks = self.models['fm'].get_feature_hooks() + + x, y = next(train_loader) + if isinstance(x, dict): + for k, v in x.items(): + if isinstance(v, torch.Tensor): + x[k] = v.to(device) + y = y.to(device) + else: + x, y = x.to(device), y.to(device) + + with torch.no_grad(): + fm_output = self.models['fm'].infer(x) + task_loss = self.models['md'].forward_to_get_task_loss(x, y) + + md_output_vis = md_output_hooks['vis'].output + fm_output_vis = fm_output_hooks['vis'].output + + md_output_lang = md_output_hooks['lang'].output + fm_output_lang = fm_output_hooks['lang'].output + + md_output_cls = md_output_hooks['cls_and_reg'].output[6] + fm_output_cls = fm_output_hooks['cls_and_reg'].output[6] + + md_output_reg = md_output_hooks['cls_and_reg'].output[1] + fm_output_reg = fm_output_hooks['cls_and_reg'].output[1] + + md_outputs = {'vis' : md_output_vis, 'lang' : md_output_lang, 'cls' : md_output_cls, 'reg' : md_output_reg} + fm_outputs = {'vis' : fm_output_vis, 'lang' : fm_output_lang, 'cls' : fm_output_cls, 'reg' : fm_output_reg} + + distill_loss = hyps['distill_loss_weight'] * self.models['md'].get_distill_loss(md_outputs, fm_outputs) + total_loss = task_loss + distill_loss + + optimizer.zero_grad() + total_loss.backward() + optimizer.step() + scheduler.step() + + if (iter_index + 1) % hyps['val_freq'] == 0: + + # elastic_dnn_util.clear_cached_channel_attention_in_master_dnn(self.models['md'].models_dict['main']) + md_output_hooks['vis'].remove() + md_output_hooks['lang'].remove() + md_output_hooks['cls_and_reg'].remove() + md_output_hooks = None + fm_output_hooks['vis'].remove() + fm_output_hooks['lang'].remove() + fm_output_hooks['cls_and_reg'].remove() + fm_output_hooks = None + + cur_md = self.models['md'].models_dict['main'] + md_for_test = deepcopy(self.models['md'].models_dict['main']) + val_acc = 0. + + self.models['md'].models_dict['main'] = md_for_test + self.models['md'].to_eval_mode() + val_acc = self.models['md'].get_accuracy(val_loader) + + self.models['md'].models_dict['main'] = cur_md + + self.models['md'].save_model(os.path.join(self.res_save_dir, 'models/md_last.pt')) + self.models['fm'].save_model(os.path.join(self.res_save_dir, 'models/fm_last.pt')) + + if val_acc > best_avg_val_acc: + best_avg_val_acc = val_acc + self.models['md'].save_model(os.path.join(self.res_save_dir, 'models/md_best.pt')) + self.models['fm'].save_model(os.path.join(self.res_save_dir, 'models/fm_best.pt')) + + tb_writer.add_scalars(f'losses', dict(task=task_loss, distill=distill_loss, total=total_loss), iter_index) + pbar.set_description(f'loss: {total_loss:.6f}') + if (iter_index + 1) >= hyps['val_freq']: + tb_writer.add_scalar(f'accs/val_acc', val_acc, iter_index) + pbar.set_description(f'loss: {total_loss:.6f}, map50: {val_acc:.4f}') + \ No newline at end of file diff --git a/new_impl/cv/elasticdnn/api/model.py b/new_impl/cv/elasticdnn/api/model.py new file mode 100644 index 0000000000000000000000000000000000000000..8983eb1aee65a1d46457695c6f5485892eaed0a6 --- /dev/null +++ b/new_impl/cv/elasticdnn/api/model.py @@ -0,0 +1,889 @@ +from typing import List +import torch +from methods.base.model import BaseModel +import tqdm +from torch import nn +import torch.nn.functional as F +from abc import abstractmethod +from methods.elasticdnn.model.base import ElasticDNNUtil +from methods.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from methods.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util + +from utils.dl.common.model import LayerActivation + + +class ElasticDNN_OfflineFMModel(BaseModel): + def get_required_model_components(self) -> List[str]: + return ['main'] + + @abstractmethod + def generate_md_by_reducing_width(self, reducing_width_ratio, samples: torch.Tensor): + pass + + @abstractmethod + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + raise NotImplementedError + + @abstractmethod + def get_feature_hook(self) -> LayerActivation: + pass + + @abstractmethod + def get_elastic_dnn_util(self) -> ElasticDNNUtil: + pass + + @abstractmethod + def get_lora_util(self) -> FMLoRA_Util: + pass + + @abstractmethod + def get_task_head_params(self): + pass + + +class ElasticDNN_OfflineClsFMModel(ElasticDNN_OfflineFMModel): + def get_accuracy(self, test_loader, *args, **kwargs): + acc = 0 + sample_num = 0 + + self.to_eval_mode() + + with torch.no_grad(): + pbar = tqdm.tqdm(enumerate(test_loader), total=len(test_loader), dynamic_ncols=True, leave=False) + for batch_index, (x, y) in pbar: + x, y = x.to(self.device), y.to(self.device) + output = self.infer(x) + #pred = F.softmax(output.logits, dim=1).argmax(dim=1) + pred = F.softmax(output, dim=1).argmax(dim=1) + #correct = torch.eq(torch.argmax(output.logits,dim = 1), y).sum().item() + correct = torch.eq(pred, y).sum().item() + acc += correct + sample_num += len(y) + + pbar.set_description(f'cur_batch_total: {len(y)}, cur_batch_correct: {correct}, ' + f'cur_batch_acc: {(correct / len(y)):.4f}') + + acc /= sample_num + return acc + + def infer(self, x, *args, **kwargs): + #print(self.models_dict['main'](x)) + return self.models_dict['main'](x) + + +import numpy as np +class StreamSegMetrics: + """ + Stream Metrics for Semantic Segmentation Task + """ + def __init__(self, n_classes): + self.n_classes = n_classes + self.confusion_matrix = np.zeros((n_classes, n_classes)) + + def update(self, label_trues, label_preds): + for lt, lp in zip(label_trues, label_preds): + self.confusion_matrix += self._fast_hist( lt.flatten(), lp.flatten() ) + + @staticmethod + def to_str(results): + string = "\n" + for k, v in results.items(): + if k!="Class IoU": + string += "%s: %f\n"%(k, v) + + return string + + def _fast_hist(self, label_true, label_pred): + mask = (label_true >= 0) & (label_true < self.n_classes) + hist = np.bincount( + self.n_classes * label_true[mask].astype(int) + label_pred[mask], + minlength=self.n_classes ** 2, + ).reshape(self.n_classes, self.n_classes) + return hist + + def get_results(self): + """Returns accuracy score evaluation result. + - overall accuracy + - mean accuracy + - mean IU + - fwavacc + """ + hist = self.confusion_matrix + acc = np.diag(hist).sum() / hist.sum() + acc_cls = np.diag(hist) / hist.sum(axis=1) + acc_cls = np.nanmean(acc_cls) + iu = np.diag(hist) / (hist.sum(axis=1) + hist.sum(axis=0) - np.diag(hist)) + mean_iu = np.nanmean(iu) + freq = hist.sum(axis=1) / hist.sum() + fwavacc = (freq[freq > 0] * iu[freq > 0]).sum() + cls_iu = dict(zip(range(self.n_classes), iu)) + + return { + "Overall Acc": acc, + "Mean Acc": acc_cls, + "FreqW Acc": fwavacc, + "Mean IoU": mean_iu, + "Class IoU": cls_iu, + } + + def reset(self): + self.confusion_matrix = np.zeros((self.n_classes, self.n_classes)) + + +class ElasticDNN_OfflineSegFMModel(ElasticDNN_OfflineFMModel): + def __init__(self, name: str, models_dict_path: str, device: str, num_classes): + super().__init__(name, models_dict_path, device) + self.num_classes = num_classes + + def get_accuracy(self, test_loader, *args, **kwargs): + device = self.device + self.to_eval_mode() + metrics = StreamSegMetrics(self.num_classes) + metrics.reset() + import tqdm + pbar = tqdm.tqdm(enumerate(test_loader), total=len(test_loader), leave=False, dynamic_ncols=True) + with torch.no_grad(): + for batch_index, (x, y) in pbar: + x, y = x.to(device, dtype=x.dtype, non_blocking=True, copy=False), \ + y.to(device, dtype=y.dtype, non_blocking=True, copy=False) + #output = self.infer(x) + output = self.infer(x) + pred = output.detach().max(dim=1)[1].cpu().numpy() + metrics.update((y + 0).cpu().numpy(), pred) + + res = metrics.get_results() + pbar.set_description(f'cur batch mIoU: {res["Mean IoU"]:.4f}') + + res = metrics.get_results() + return res['Mean IoU'] + + def infer(self, x, *args, **kwargs): + return self.models_dict['main'](x) + + +class ElasticDNN_OfflineDetFMModel(ElasticDNN_OfflineFMModel): + def __init__(self, name: str, models_dict_path: str, device: str, num_classes): + super().__init__(name, models_dict_path, device) + self.num_classes = num_classes + + def get_accuracy(self, test_loader, *args, **kwargs): + # print('DeeplabV3: start test acc') + _d = test_loader.dataset + from data import build_dataloader + if _d.__class__.__name__ == 'MergedDataset': + # print('\neval on merged datasets') + datasets = _d.datasets + test_loaders = [build_dataloader(d, test_loader.batch_size, test_loader.num_workers, False, None) for d in datasets] + accs = [self.get_accuracy(loader) for loader in test_loaders] + # print(accs) + return sum(accs) / len(accs) + + # print('dataset len', len(test_loader.dataset)) + + model = self.models_dict['main'] + device = self.device + model.eval() + + # print('# classes', model.num_classes) + + model = model.to(device) + from dnns.yolov3.coco_evaluator import COCOEvaluator + from utils.common.others import HiddenPrints + with torch.no_grad(): + with HiddenPrints(): + evaluator = COCOEvaluator( + dataloader=test_loader, + img_size=(224, 224), + confthre=0.01, + nmsthre=0.65, + num_classes=self.num_classes, + testdev=False + ) + res = evaluator.evaluate(model, False, False) + map50 = res[1] + # print('eval info', res[-1]) + return map50 + + def infer(self, x, *args, **kwargs): + if len(args) > 0: + print(args, len(args)) + return self.models_dict['main'](x, *args) # forward(x, label) + return self.models_dict['main'](x) + + +class ElasticDNN_OfflineSenClsFMModel(ElasticDNN_OfflineFMModel): + def get_accuracy(self, test_loader, *args, **kwargs): + acc = 0 + sample_num = 0 + + self.to_eval_mode() + + with torch.no_grad(): + pbar = tqdm.tqdm(enumerate(test_loader), total=len(test_loader), dynamic_ncols=True, leave=False) + for batch_index, (x, y) in pbar: + for k, v in x.items(): + if isinstance(v, torch.Tensor): + x[k] = v.to(self.device) + y = y.to(self.device) + output = self.infer(x) + pred = F.softmax(output, dim=1).argmax(dim=1) + correct = torch.eq(pred, y).sum().item() + acc += correct + sample_num += len(y) + + pbar.set_description(f'cur_batch_total: {len(y)}, cur_batch_correct: {correct}, ' + f'cur_batch_acc: {(correct / len(y)):.4f}') + + acc /= sample_num + return acc + + def infer(self, x, *args, **kwargs): + return self.models_dict['main'](x) + + +from accelerate.utils.operations import pad_across_processes + + +class ElasticDNN_OfflineTrFMModel(ElasticDNN_OfflineFMModel): + + def get_accuracy(self, test_loader, *args, **kwargs): + # TODO: BLEU + from sacrebleu import corpus_bleu + + acc = 0 + num_batches = 0 + + self.to_eval_mode() + + from data.datasets.sentiment_classification.global_bert_tokenizer import get_tokenizer + tokenizer = get_tokenizer() + + def _decode(o): + # https://github.com/huggingface/transformers/blob/main/examples/research_projects/seq2seq-distillation/finetune.py#L133 + o = tokenizer.batch_decode(o, skip_special_tokens=True) + return [oi.strip() for oi in o] + + with torch.no_grad(): + pbar = tqdm.tqdm(enumerate(test_loader), total=len(test_loader), dynamic_ncols=True, leave=False) + for batch_index, (x, y) in pbar: + for k, v in x.items(): + if isinstance(v, torch.Tensor): + x[k] = v.to(self.device) + label = y.to(self.device) + + # generated_tokens = self.infer(x, generate=True) + + generated_tokens = self.infer(x).logits.argmax(-1) + + # pad tokens + generated_tokens = pad_across_processes( + generated_tokens, dim=1, pad_index=tokenizer.pad_token_id + ) + # pad label + label = pad_across_processes( + label, dim=1, pad_index=tokenizer.pad_token_id + ) + label = label.cpu().numpy() + label = np.where(label != -100, label, tokenizer.pad_token_id) + + decoded_output = _decode(generated_tokens) + decoded_y = _decode(y) + + decoded_y = [decoded_y] + + if batch_index == 0: + print(decoded_y, decoded_output) + + bleu = corpus_bleu(decoded_output, decoded_y).score + pbar.set_description(f'cur_batch_bleu: {bleu:.4f}') + + acc += bleu + num_batches += 1 + + acc /= num_batches + return acc + + def infer(self, x, *args, **kwargs): + if 'token_type_ids' in x.keys(): + del x['token_type_ids'] + + if 'generate' in kwargs: + return self.models_dict['main'].generate( + x['input_ids'], + attention_mask=x["attention_mask"], + max_length=512 + ) + + return self.models_dict['main'](**x) + + +from nltk.metrics import accuracy as nltk_acc + +class ElasticDNN_OfflineTokenClsFMModel(ElasticDNN_OfflineFMModel): + def get_accuracy(self, test_loader, *args, **kwargs): + acc = 0 + sample_num = 0 + + self.to_eval_mode() + + with torch.no_grad(): + pbar = tqdm.tqdm(enumerate(test_loader), total=len(test_loader), dynamic_ncols=True, leave=False) + for batch_index, (x, y) in pbar: + for k, v in x.items(): + if isinstance(v, torch.Tensor): + x[k] = v.to(self.device) + # print(x) + y = y.to(self.device) + output = self.infer(x) + + # torch.Size([16, 512, 43]) torch.Size([16, 512]) + + for oi, yi, xi in zip(output, y, x['input_ids']): + # oi: 512, 43; yi: 512 + seq_len = xi.nonzero().size(0) + + # print(output.size(), y.size()) + + pred = F.softmax(oi, dim=-1).argmax(dim=-1) + correct = torch.eq(pred[1: seq_len], yi[1: seq_len]).sum().item() + + # print(output.size(), y.size()) + + acc += correct + sample_num += seq_len + + pbar.set_description(f'seq_len: {seq_len}, cur_seq_acc: {(correct / seq_len):.4f}') + + acc /= sample_num + return acc + + def infer(self, x, *args, **kwargs): + return self.models_dict['main'](**x) + + +class ElasticDNN_OfflineMMClsFMModel(ElasticDNN_OfflineFMModel): + # def __init__(self, name: str, models_dict_path: str, device: str, class_to_label_idx_map): + # super().__init__(name, models_dict_path, device) + # self.class_to_label_idx_map = class_to_label_idx_map + + def get_accuracy(self, test_loader, *args, **kwargs): + acc = 0 + sample_num = 0 + + self.to_eval_mode() + + batch_size = 1 + + with torch.no_grad(): + pbar = tqdm.tqdm(enumerate(test_loader), total=len(test_loader), dynamic_ncols=True, leave=False) + for batch_index, (x, y) in pbar: + if batch_index * batch_size > 2000: + break + + for k, v in x.items(): + if isinstance(v, torch.Tensor): + x[k] = v.to(self.device) + y = y.to(self.device) + + # print(x) + + raw_texts = x['texts'][:] + x['texts'] = list(set(x['texts'])) + + # print(x['texts']) + + batch_size = len(y) + + x['for_training'] = False + + output = self.infer(x) + + output = output.logits_per_image + + # print(output.size()) + # exit() + + # y = torch.arange(len(y), device=self.device) + y = torch.LongTensor([x['texts'].index(rt) for rt in raw_texts]).to(self.device) + # print(y) + + # exit() + + # print(output.size(), y.size()) + + pred = F.softmax(output, dim=1).argmax(dim=1) + correct = torch.eq(pred, y).sum().item() + acc += correct + sample_num += len(y) + pbar.set_description(f'cur_batch_total: {len(y)}, cur_batch_correct: {correct}, ' + f'cur_batch_acc: {(correct / len(y)):.4f}') + + acc /= sample_num + return acc + + def infer(self, x, *args, **kwargs): + x['for_training'] = self.models_dict['main'].training + return self.models_dict['main'](**x) + + + +class VQAScore: + def __init__(self): + # self.add_state("score", default=torch.tensor(0.0), dist_reduce_fx="sum") + # self.add_state("total", default=torch.tensor(0.0), dist_reduce_fx="sum") + self.score = torch.tensor(0.0) + self.total = torch.tensor(0.0) + + def update(self, logits, target): + logits, target = ( + logits.detach().float().to(self.score.device), + target.detach().float().to(self.score.device), + ) + logits = torch.max(logits, 1)[1] + one_hots = torch.zeros(*target.size()).to(target) + one_hots.scatter_(1, logits.view(-1, 1), 1) + scores = one_hots * target + + self.score += scores.sum() + self.total += len(logits) + + def compute(self): + return self.score / self.total + + + +class ElasticDNN_OfflineVQAFMModel(ElasticDNN_OfflineFMModel): + # def __init__(self, name: str, models_dict_path: str, device: str, class_to_label_idx_map): + # super().__init__(name, models_dict_path, device) + # self.class_to_label_idx_map = class_to_label_idx_map + + def get_accuracy(self, test_loader, *args, **kwargs): + acc = 0 + sample_num = 0 + + vqa_score = VQAScore() + + self.to_eval_mode() + from transformers import AutoProcessor + processor = AutoProcessor.from_pretrained("new_impl/mm/Vis_bert/QuestionAnswering/VisBert_pretrained") + with torch.no_grad(): + pbar = tqdm.tqdm(enumerate(test_loader), total=len(test_loader), dynamic_ncols=True, leave=False) + for batch_index, (x, y, t) in pbar: + for k, v in x.items(): + if isinstance(v, torch.Tensor): + x[k] = v.to(self.device) + if isinstance(y,dict): + for k, v in y.items(): + y[k] = v.to(self.device) + else: + y = y.to(self.device) + + output = self.models_dict['main'].generate(**x) + total = 0 + idx = 0 + for i in output: + val = processor.decode(i, skip_special_tokens=True) + text = t[idx] + if val == text: + total += 1 + idx += 1 + + #vqa_score.update(output, y.labels) + acc = total / (idx+1) + #pbar.set_description(f'cur_batch_total: {len(y['label'])}, cur_batch_acc: {vqa_score.compute():.4f}') + pbar.set_description(f'cur_batch_total: {len(y["labels"])}, cur_batch_acc: {acc:.4f}') + #return vqa_score.compute() + return acc + + def infer(self, x, *args, **kwargs): + return self.models_dict['main'](**x) + + +class ElasticDNN_OfflineMDModel(BaseModel): + def get_required_model_components(self) -> List[str]: + return ['main'] + + @abstractmethod + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + raise NotImplementedError + + @abstractmethod + def get_feature_hook(self) -> LayerActivation: + pass + + @abstractmethod + def get_distill_loss(self, student_output, teacher_output): + pass + + @abstractmethod + def get_matched_param_of_fm(self, self_param_name, fm: nn.Module): + pass + + +class ElasticDNN_OfflineClsMDModel(ElasticDNN_OfflineMDModel): + def get_accuracy(self, test_loader, *args, **kwargs): + acc = 0 + sample_num = 0 + + self.to_eval_mode() + + with torch.no_grad(): + pbar = tqdm.tqdm(enumerate(test_loader), total=len(test_loader), dynamic_ncols=True, leave=False) + for batch_index, (x, y) in pbar: + x, y = x.to(self.device), y.to(self.device) + output = self.infer(x) + #pred = F.softmax(output.logits, dim=1).argmax(dim=1) + pred = F.softmax(output, dim=1).argmax(dim=1) + correct = torch.eq(pred, y).sum().item() + acc += correct + sample_num += len(y) + + pbar.set_description(f'cur_batch_total: {len(y)}, cur_batch_correct: {correct}, ' + f'cur_batch_acc: {(correct / len(y)):.4f}') + + acc /= sample_num + return acc + + def infer(self, x, *args, **kwargs): + return self.models_dict['main'](x) + + +class ElasticDNN_OfflineSegMDModel(ElasticDNN_OfflineMDModel): + def __init__(self, name: str, models_dict_path: str, device: str, num_classes): + super().__init__(name, models_dict_path, device) + self.num_classes = num_classes + + def get_accuracy(self, test_loader, *args, **kwargs): + device = self.device + self.to_eval_mode() + metrics = StreamSegMetrics(self.num_classes) + metrics.reset() + import tqdm + pbar = tqdm.tqdm(enumerate(test_loader), total=len(test_loader), leave=False, dynamic_ncols=True) + with torch.no_grad(): + for batch_index, (x, y) in pbar: + x, y = x.to(device, dtype=x.dtype, non_blocking=True, copy=False), \ + y.to(device, dtype=y.dtype, non_blocking=True, copy=False) + output = self.infer(x) + pred = output.detach().max(dim=1)[1].cpu().numpy() + metrics.update((y + 0).cpu().numpy(), pred) + + res = metrics.get_results() + pbar.set_description(f'cur batch mIoU: {res["Mean IoU"]:.4f}') + + res = metrics.get_results() + return res['Mean IoU'] + + def infer(self, x, *args, **kwargs): + return self.models_dict['main'](x) + + +class ElasticDNN_OfflineDetMDModel(ElasticDNN_OfflineMDModel): + def __init__(self, name: str, models_dict_path: str, device: str, num_classes): + super().__init__(name, models_dict_path, device) + self.num_classes = num_classes + + def get_accuracy(self, test_loader, *args, **kwargs): + # print('DeeplabV3: start test acc') + _d = test_loader.dataset + from data import build_dataloader + if _d.__class__.__name__ == 'MergedDataset': + # print('\neval on merged datasets') + datasets = _d.datasets + test_loaders = [build_dataloader(d, test_loader.batch_size, test_loader.num_workers, False, None) for d in datasets] + accs = [self.get_accuracy(loader) for loader in test_loaders] + # print(accs) + return sum(accs) / len(accs) + + # print('dataset len', len(test_loader.dataset)) + + model = self.models_dict['main'] + device = self.device + model.eval() + + # print('# classes', model.num_classes) + + model = model.to(device) + from dnns.yolov3.coco_evaluator import COCOEvaluator + from utils.common.others import HiddenPrints + with torch.no_grad(): + with HiddenPrints(): + evaluator = COCOEvaluator( + dataloader=test_loader, + img_size=(224, 224), + confthre=0.01, + nmsthre=0.65, + num_classes=self.num_classes, + testdev=False + ) + res = evaluator.evaluate(model, False, False) + map50 = res[1] + # print('eval info', res[-1]) + return map50 + + def infer(self, x, *args, **kwargs): + if len(args) > 0: + return self.models_dict['main'](x, *args) # forward(x, label) + return self.models_dict['main'](x) + + +class ElasticDNN_OfflineSenClsMDModel(ElasticDNN_OfflineMDModel): + def get_accuracy(self, test_loader, *args, **kwargs): + acc = 0 + sample_num = 0 + + self.to_eval_mode() + + with torch.no_grad(): + pbar = tqdm.tqdm(enumerate(test_loader), total=len(test_loader), dynamic_ncols=True, leave=False) + for batch_index, (x, y) in pbar: + for k, v in x.items(): + if isinstance(v, torch.Tensor): + x[k] = v.to(self.device) + y = y.to(self.device) + output = self.infer(x) + pred = F.softmax(output, dim=1).argmax(dim=1) + + print(pred, y) + + correct = torch.eq(pred, y).sum().item() + acc += correct + sample_num += len(y) + + pbar.set_description(f'cur_batch_total: {len(y)}, cur_batch_correct: {correct}, ' + f'cur_batch_acc: {(correct / len(y)):.4f}') + + acc /= sample_num + return acc + + def infer(self, x, *args, **kwargs): + return self.models_dict['main'](**x) + + +class ElasticDNN_OfflineTrMDModel(ElasticDNN_OfflineMDModel): + + def get_accuracy(self, test_loader, *args, **kwargs): + # TODO: BLEU + from sacrebleu import corpus_bleu + + acc = 0 + num_batches = 0 + + self.to_eval_mode() + + from data.datasets.sentiment_classification.global_bert_tokenizer import get_tokenizer + tokenizer = get_tokenizer() + + def _decode(o): + # https://github.com/huggingface/transformers/blob/main/examples/research_projects/seq2seq-distillation/finetune.py#L133 + o = tokenizer.batch_decode(o, skip_special_tokens=True, clean_up_tokenization_spaces=True) + return [oi.strip().replace(' ', '') for oi in o] + + with torch.no_grad(): + pbar = tqdm.tqdm(enumerate(test_loader), total=len(test_loader), dynamic_ncols=True, leave=False) + for batch_index, (x, y) in pbar: + for k, v in x.items(): + if isinstance(v, torch.Tensor): + x[k] = v.to(self.device) + y = y.to(self.device) + + output = self.infer(x) + decoded_output = _decode(output.argmax(-1)) + decoded_y = _decode(y) + + decoded_y = [decoded_y] + + print(x, decoded_y, decoded_output, output.argmax(-1)) + + bleu = corpus_bleu(decoded_output, decoded_y).score + pbar.set_description(f'cur_batch_bleu: {bleu:.4f}') + + acc += bleu + num_batches += 1 + + acc /= num_batches + return acc + + def infer(self, x, *args, **kwargs): + return self.models_dict['main'](**x) + + +class ElasticDNN_OfflineTokenClsMDModel(ElasticDNN_OfflineMDModel): + def get_accuracy(self, test_loader, *args, **kwargs): + acc = 0 + sample_num = 0 + + self.to_eval_mode() + + with torch.no_grad(): + pbar = tqdm.tqdm(enumerate(test_loader), total=len(test_loader), dynamic_ncols=True, leave=False) + for batch_index, (x, y) in pbar: + for k, v in x.items(): + if isinstance(v, torch.Tensor): + x[k] = v.to(self.device) + # print(x) + y = y.to(self.device) + output = self.infer(x) + + # torch.Size([16, 512, 43]) torch.Size([16, 512]) + + for oi, yi, xi in zip(output, y, x['input_ids']): + # oi: 512, 43; yi: 512 + seq_len = xi.nonzero().size(0) + + # print(output.size(), y.size()) + + pred = F.softmax(oi, dim=-1).argmax(dim=-1) + correct = torch.eq(pred[1: seq_len], yi[1: seq_len]).sum().item() + + # print(output.size(), y.size()) + + acc += correct + sample_num += seq_len + + pbar.set_description(f'seq_len: {seq_len}, cur_seq_acc: {(correct / seq_len):.4f}') + + acc /= sample_num + return acc + + def infer(self, x, *args, **kwargs): + return self.models_dict['main'](**x) + + +class ElasticDNN_OfflineMMClsMDModel(ElasticDNN_OfflineMDModel): + def get_accuracy(self, test_loader, *args, **kwargs): + acc = 0 + sample_num = 0 + + self.to_eval_mode() + + batch_size = 1 + + with torch.no_grad(): + pbar = tqdm.tqdm(enumerate(test_loader), total=len(test_loader), dynamic_ncols=True, leave=False) + for batch_index, (x, y) in pbar: + if batch_index * batch_size > 2000: + break + + for k, v in x.items(): + if isinstance(v, torch.Tensor): + x[k] = v.to(self.device) + y = y.to(self.device) + + # print(x) + + raw_texts = x['texts'][:] + x['texts'] = list(set(x['texts'])) + + # print(x['texts']) + + batch_size = len(y) + + x['for_training'] = False + + output = self.infer(x) + + output = output.logits_per_image + + # print(output.size()) + # exit() + + # y = torch.arange(len(y), device=self.device) + y = torch.LongTensor([x['texts'].index(rt) for rt in raw_texts]).to(self.device) + # print(y) + + # exit() + + # print(output.size(), y.size()) + + pred = F.softmax(output, dim=1).argmax(dim=1) + correct = torch.eq(pred, y).sum().item() + acc += correct + sample_num += len(y) + pbar.set_description(f'cur_batch_total: {len(y)}, cur_batch_correct: {correct}, ' + f'cur_batch_acc: {(correct / len(y)):.4f}') + + acc /= sample_num + return acc + + def infer(self, x, *args, **kwargs): + return self.models_dict['main'](**x) + + +# class ElasticDNN_OfflineVQAMDModel(ElasticDNN_OfflineMDModel): +# # def __init__(self, name: str, models_dict_path: str, device: str, class_to_label_idx_map): +# # super().__init__(name, models_dict_path, device) +# # self.class_to_label_idx_map = class_to_label_idx_map + +# def get_accuracy(self, test_loader, *args, **kwargs): +# acc = 0 +# sample_num = 0 + +# vqa_score = VQAScore() + +# self.to_eval_mode() + +# with torch.no_grad(): +# pbar = tqdm.tqdm(enumerate(test_loader), total=len(test_loader), dynamic_ncols=True, leave=False) +# for batch_index, (x, y) in pbar: +# for k, v in x.items(): +# if isinstance(v, torch.Tensor): +# x[k] = v.to(self.device) +# y = y.to(self.device) +# output = self.infer(x) + +# vqa_score.update(output, y) + +# pbar.set_description(f'cur_batch_total: {len(y)}, cur_batch_acc: {vqa_score.compute():.4f}') + +# return vqa_score.compute() + +# def infer(self, x, *args, **kwargs): +# return self.models_dict['main'](**x) + +class ElasticDNN_OfflineVQAMDModel(ElasticDNN_OfflineMDModel): + # def __init__(self, name: str, models_dict_path: str, device: str, class_to_label_idx_map): + # super().__init__(name, models_dict_path, device) + # self.class_to_label_idx_map = class_to_label_idx_map + + def get_accuracy(self, test_loader, *args, **kwargs): + acc = 0 + sample_num = 0 + + vqa_score = VQAScore() + + self.to_eval_mode() + from transformers import AutoProcessor + processor = AutoProcessor.from_pretrained("new_impl/mm/Vis_bert/QuestionAnswering/VisBert_pretrained") + with torch.no_grad(): + pbar = tqdm.tqdm(enumerate(test_loader), total=len(test_loader), dynamic_ncols=True, leave=False) + for batch_index, (x, y, t) in pbar: + for k, v in x.items(): + if isinstance(v, torch.Tensor): + x[k] = v.to(self.device) + if isinstance(y,dict): + for k, v in y.items(): + y[k] = v.to(self.device) + else: + y = y.to(self.device) + + output = self.models_dict['main'].generate(**x) + total = 0 + idx = 0 + for i in output: + val = processor.decode(i, skip_special_tokens=True) + text = t[idx] + if val == text: + total += 1 + idx += 1 + + #vqa_score.update(output, y.labels) + acc = total / (idx+1) + #pbar.set_description(f'cur_batch_total: {len(y['label'])}, cur_batch_acc: {vqa_score.compute():.4f}') + pbar.set_description(f'cur_batch_total: {len(y["labels"])}, cur_batch_acc: {acc:.4f}') + #return vqa_score.compute() + return acc + + def infer(self, x, *args, **kwargs): + return self.models_dict['main'].generate(**x) + \ No newline at end of file diff --git a/new_impl/cv/elasticdnn/api/online_model.py b/new_impl/cv/elasticdnn/api/online_model.py new file mode 100644 index 0000000000000000000000000000000000000000..19426bbce451dae33b270a17ef4e6ca706fe6b01 --- /dev/null +++ b/new_impl/cv/elasticdnn/api/online_model.py @@ -0,0 +1,246 @@ +from copy import deepcopy +from typing import List +import torch +from methods.base.model import BaseModel +import tqdm +from torch import nn +import torch.nn.functional as F +from abc import abstractmethod +from methods.elasticdnn.model.base import ElasticDNNUtil +from methods.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from methods.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from utils.common.log import logger +from utils.dl.common.model import LayerActivation, get_parameter + + +class ElasticDNN_OnlineModel(BaseModel): + def __init__(self, name: str, models_dict_path: str, device: str, ab_options: dict): + super().__init__(name, models_dict_path, device) + + assert [k in ab_options.keys() for k in ['md_to_fm_alpha', 'fm_to_md_alpha']] + self.ab_options = ab_options + + def get_required_model_components(self) -> List[str]: + return ['fm', 'md', 'sd', 'indexes', 'bn_stats'] + + @torch.no_grad() + def generate_sd_by_target_samples(self, target_samples: torch.Tensor): + elastic_dnn_util = self.get_elastic_dnn_util() + sd, unpruned_indexes_of_layers = elastic_dnn_util.extract_surrogate_dnn_via_samples_with_perf_test(self.models_dict['md'], target_samples.to(self.device), True) + logger.debug(f'generate sd: \n{sd}') + return sd, unpruned_indexes_of_layers + + @torch.no_grad() + def _compute_diff(self, old, new): + return (new - old).norm(1) / old.norm(1) + + @torch.no_grad() + def sd_feedback_to_md(self, after_da_sd, unpruned_indexes_of_layers): + self.models_dict['sd'] = after_da_sd + self.before_da_md = deepcopy(self.models_dict['md']) + + logger.info('\n\nsurrogate DNN feedback to master DNN...\n\n') + # one-to-one + + cur_unpruned_indexes = None + cur_unpruned_indexes_name = None + + for p_name, p in self.models_dict['sd'].named_parameters(): + matched_md_param = self.get_md_matched_param_of_sd_param(p_name) + logger.debug(f'if feedback: {p_name}') + if matched_md_param is None: + continue + logger.debug(f'start feedback: {p_name}, {p.size()} -> {matched_md_param.size()}') + # average + # setattr(matched_md_module, matched_md_param_name, (matched_md_param + p) / 2.) + + if p_name in unpruned_indexes_of_layers.keys(): + cur_unpruned_indexes = unpruned_indexes_of_layers[p_name] + cur_unpruned_indexes_name = p_name + + if p.size() != matched_md_param.size(): + logger.debug(f'cur unpruned indexes: {cur_unpruned_indexes_name}, {cur_unpruned_indexes.size()}') + + if p.dim() == 1: # norm + new_p = deepcopy(matched_md_param) + new_p[cur_unpruned_indexes] = p + elif p.dim() == 2: # linear + if p.size(0) < matched_md_param.size(0): # output pruned + new_p = deepcopy(matched_md_param) + new_p[cur_unpruned_indexes] = p + else: # input pruned + new_p = deepcopy(matched_md_param) + new_p[:, cur_unpruned_indexes] = p + p = new_p + + assert p.size() == matched_md_param.size(), f'{p.size()}, {matched_md_param.size()}' + + diff = self._compute_diff(matched_md_param, (matched_md_param + p) / 2.) + matched_md_param.copy_((matched_md_param + p) / 2.) + logger.debug(f'end feedback: {p_name}, diff: {diff:.6f}') + + def infer(self, x, *args, **kwargs): + return self.models_dict['sd'](x) + + def set_sd_sparsity(self, sparsity: float): + elastic_dnn_util = self.get_elastic_dnn_util() + elastic_dnn_util.clear_cached_channel_attention_in_master_dnn(self.models_dict['md']) + elastic_dnn_util.set_master_dnn_sparsity(self.models_dict['md'], sparsity) + + @torch.no_grad() + def md_feedback_to_self_fm(self): + logger.info('\n\nmaster DNN feedback to self foundation model...\n\n') + # one-to-many + + def upsample_2d_tensor(p: torch.Tensor, target_len: int): + assert p.dim() == 2 # regard 2d weight as (batch_size, 1d_vector_dim) + return F.upsample(p.unsqueeze(1).unsqueeze(3), + size=(target_len, 1), + mode='bilinear').squeeze(3).squeeze(1) + + for (p_name, p), before_p in zip(self.models_dict['md'].named_parameters(), self.before_da_md.parameters()): + matched_fm_param = self.get_fm_matched_param_of_md_param(p_name) + logger.debug(f'if feedback: {p_name}') + if matched_fm_param is None: + continue + + index = self.models_dict['indexes'][p_name] + logger.debug(f'start feedback: {p_name}, {p.size()} -> {matched_fm_param.size()}, index: {index.size()}') + + p_update = p - before_p + if p.dim() == 2: + p_update = upsample_2d_tensor(p_update, matched_fm_param.size(1)) + + p_update = p_update.unsqueeze(1) + index = index.unsqueeze(-1) + + # fast + # agg_p_update = (p_update * index).sum(0) + + # balanced agg + agg_p_update = 0 + + cur_split_size = 64 + while index.size(0) % cur_split_size != 0: + cur_split_size -= 1 + + for i in range(0, index.size(0), cur_split_size): + agg_p_update += p_update[i: i + cur_split_size] * index[i: i + cur_split_size] + agg_p_update = agg_p_update.sum(0) + + + else: + agg_p_update = (p_update.unsqueeze(1) * index).sum(0) + + new_fm_param = matched_fm_param + agg_p_update * self.ab_options['md_to_fm_alpha'] + + diff = self._compute_diff(matched_fm_param, new_fm_param) + + # NOTE: matched_fm_param may not be reference, may be a deepcopy!! + # and only here matched_fm_param needs to be updated, so another method dedicated for updating is necessary here + # matched_fm_param.copy_(new_fm_param) + self.update_fm_param(p_name, new_fm_param) + + logger.debug(f'end feedback: {p_name}, diff: {diff:.6f} (md_to_fm_alpha={self.ab_options["md_to_fm_alpha"]:.4f})') + + @abstractmethod + @torch.no_grad() + def update_fm_param(self, md_param_name, cal_new_fm_param_by_md_param): + """ + you should get the reference of fm_param and update it + """ + raise NotImplementedError + + @torch.no_grad() + def aggregate_fms_to_self_fm(self, fms: List[nn.Module]): + # average task-agnositc parameters + logger.info('\n\naggregate foundation models to self foundation model...\n\n') + for p_name, self_p in self.models_dict['fm'].named_parameters(): + logger.debug(f'if aggregate {p_name}') + if 'abs' in p_name or p_name.startswith('norm') or p_name.startswith('head'): + logger.debug(f'{p_name} belongs to LoRA parameters/task-specific head, i.e. task-specific parameters, skip') + continue + all_p = [get_parameter(fm, p_name) for fm in fms] + if any([_p is None for _p in all_p]): + continue + + avg_p = sum(all_p) / len(all_p) + # [_p.copy_(avg_p) for _p in all_p] + + diff = self._compute_diff(self_p, avg_p) + logger.debug(f'aggregate {p_name}, diff {diff:.6f}') + + self_p.copy_(avg_p) + + @torch.no_grad() + def fm_feedback_to_md(self): + logger.info('\n\nself foundation model feedback to master DNN...\n\n') + # one-to-many + + def downsample_2d_tensor(p: torch.Tensor, target_len: int): + assert p.dim() == 2 # regard 2d weight as (batch_size, 1d_vector_dim) + # return F.upsample(p.unsqueeze(1).unsqueeze(3), + # size=(target_len, 1), + # mode='bilinear').squeeze(3).squeeze(1) + return F.interpolate(p.unsqueeze(1).unsqueeze(3), size=(target_len, 1), mode='bilinear').squeeze(3).squeeze(1) + + + for p_name, p in self.models_dict['md'].named_parameters(): + matched_fm_param = self.get_fm_matched_param_of_md_param(p_name) + logger.debug(f'if feedback: {p_name}') + if matched_fm_param is None: + continue + + index = self.models_dict['indexes'][p_name] + logger.debug(f'start feedback: {p_name}, {p.size()} -> {matched_fm_param.size()}, index: {index.size()}') + + if p.dim() == 2: + matched_fm_param = downsample_2d_tensor(matched_fm_param, p.size(1)) + + matched_fm_param = matched_fm_param.unsqueeze(0) + index = index.unsqueeze(-1) + + # fast + # agg_p_update = (p_update * index).sum(0) + + # balanced agg + agg_fm_param = 0 + + cur_split_size = 64 + while index.size(1) % cur_split_size != 0: + cur_split_size -= 1 + + for i in range(0, index.size(1), cur_split_size): + agg_fm_param += matched_fm_param[:, i: i + cur_split_size] * index[:, i: i + cur_split_size] + agg_fm_param = agg_fm_param.sum(1) + # agg_fm_param = downsample_2d_tensor(agg_fm_param, p.size(1)) + + else: + agg_fm_param = (matched_fm_param.unsqueeze(0) * index).sum(1) + + + + diff = self._compute_diff(p, agg_fm_param) + p.copy_(agg_fm_param * self.ab_options['fm_to_md_alpha'] + (1. - self.ab_options['fm_to_md_alpha']) * p) + + logger.debug(f'end feedback: {p_name}, diff: {diff:.6f} (fm_to_md_alpha: {self.ab_options["fm_to_md_alpha"]:.4f})') + + @abstractmethod + def get_elastic_dnn_util(self) -> ElasticDNNUtil: + pass + + @abstractmethod + def get_task_head_params(self): + pass + + @abstractmethod + def get_md_matched_param_of_sd_param(self, sd_param_name): + pass + + @abstractmethod + def get_fm_matched_param_of_md_param(self, md_param_name): + pass + + @abstractmethod + def get_md_matched_param_of_fm_param(self, fm_param_name): + pass \ No newline at end of file diff --git a/new_impl/cv/elasticdnn/api/online_model_v2.py b/new_impl/cv/elasticdnn/api/online_model_v2.py new file mode 100644 index 0000000000000000000000000000000000000000..6c6ae64cb9f4d4d43fb3f1e9ef30a60ba8b150ea --- /dev/null +++ b/new_impl/cv/elasticdnn/api/online_model_v2.py @@ -0,0 +1,268 @@ +from copy import deepcopy +from typing import List +import torch +from new_impl.cv.base.model import BaseModel +import tqdm +from torch import nn +import torch.nn.functional as F +from abc import abstractmethod +from new_impl.cv.elasticdnn.model.base import ElasticDNNUtil +from new_impl.cv.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from new_impl.cv.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from utils.common.log import logger +from utils.dl.common.model import LayerActivation, get_parameter + + +class ElasticDNN_OnlineModel(BaseModel): + def __init__(self, name: str, models_dict_path: str, device: str, ab_options: dict): + super().__init__(name, models_dict_path, device) + + assert [k in ab_options.keys() for k in ['md_to_fm_alpha', 'fm_to_md_alpha']] + self.ab_options = ab_options + + def get_required_model_components(self) -> List[str]: + return ['fm', 'md', 'sd', 'indexes', 'bn_stats'] + + @torch.no_grad() + def generate_sd_by_target_samples(self, target_samples: torch.Tensor): + elastic_dnn_util = self.get_elastic_dnn_util() + + if isinstance(target_samples, dict): + for k, v in target_samples.items(): + if isinstance(v, torch.Tensor): + target_samples[k] = v.to(self.device) + else: + target_samples = target_samples.to(self.device) + sd, unpruned_indexes_of_layers = elastic_dnn_util.extract_surrogate_dnn_via_samples_with_perf_test(self.models_dict['md'], target_samples, True) + logger.debug(f'generate sd: \n{sd}') + return sd, unpruned_indexes_of_layers + + @torch.no_grad() + def _compute_diff(self, old, new): + return (new - old).norm(1) / old.norm(1) + + @torch.no_grad() + def sd_feedback_to_md(self, after_da_sd, unpruned_indexes_of_layers): + self.models_dict['sd'] = after_da_sd + self.before_da_md = deepcopy(self.models_dict['md']) + + logger.info('\n\nsurrogate DNN feedback to master DNN...\n\n') + # one-to-one + + cur_unpruned_indexes = None + cur_unpruned_indexes_name = None + + for p_name, p in self.models_dict['sd'].named_parameters(): + matched_md_param = self.get_md_matched_param_of_sd_param(p_name) + logger.debug(f'if feedback: {p_name}') + if matched_md_param is None: + continue + logger.debug(f'start feedback: {p_name}, {p.size()} -> {matched_md_param.size()}') + # average + # setattr(matched_md_module, matched_md_param_name, (matched_md_param + p) / 2.) + + if p_name in unpruned_indexes_of_layers.keys(): + cur_unpruned_indexes = unpruned_indexes_of_layers[p_name] + cur_unpruned_indexes_name = p_name + + if p.size() != matched_md_param.size(): + logger.debug(f'cur unpruned indexes: {cur_unpruned_indexes_name}, {cur_unpruned_indexes.size()}') + + if p.dim() == 1: # norm + new_p = deepcopy(matched_md_param) + new_p[cur_unpruned_indexes] = p + elif p.dim() == 2: # linear + if p.size(0) < matched_md_param.size(0): # output pruned + new_p = deepcopy(matched_md_param) + new_p[cur_unpruned_indexes] = p + else: # input pruned + new_p = deepcopy(matched_md_param) + new_p[:, cur_unpruned_indexes] = p + p = new_p + + assert p.size() == matched_md_param.size(), f'{p.size()}, {matched_md_param.size()}' + + diff = self._compute_diff(matched_md_param, (matched_md_param + p) / 2.) + matched_md_param.copy_((matched_md_param + p) / 2.) + logger.debug(f'end feedback: {p_name}, diff: {diff:.6f}') + + def infer(self, x, *args, **kwargs): + return self.models_dict['sd'](x) + + def set_sd_sparsity(self, sparsity: float): + elastic_dnn_util = self.get_elastic_dnn_util() + elastic_dnn_util.clear_cached_channel_attention_in_master_dnn(self.models_dict['md']) + elastic_dnn_util.set_master_dnn_sparsity(self.models_dict['md'], sparsity) + + @torch.no_grad() + def md_feedback_to_self_fm(self): + logger.info('\n\nmaster DNN feedback to self foundation model...\n\n') + # one-to-many + + # def upsample_2d_tensor(p: torch.Tensor, target_len: int): + # assert p.dim() == 2 # regard 2d weight as (batch_size, 1d_vector_dim) + # return F.upsample(p.unsqueeze(1).unsqueeze(3), + # size=(target_len, 1), + # mode='bilinear').squeeze(3).squeeze(1) + + for (p_name, p), before_p in zip(self.models_dict['md'].named_parameters(), self.before_da_md.parameters()): + matched_fm_param = self.get_fm_matched_param_of_md_param(p_name) + logger.debug(f'if feedback: {p_name}') + if matched_fm_param is None: + continue + # print(self.models_dict['indexes'].keys()) + index = self.models_dict['indexes'][p_name] + logger.debug(f'start feedback: {p_name}, {p.size()} -> {matched_fm_param.size()}, index: {index.size()}') + + p_update = p - before_p + + t = False + if p.dim() > 1 and index.size(0) == p.size(1) and index.size(1) == matched_fm_param.size(1): + assert p.dim() == 2 + p_update = p_update.T + matched_fm_param = matched_fm_param.T + t = True + logger.debug(f'transpose paramters') + + + if p.dim() == 2: + # p_update = upsample_2d_tensor(p_update, matched_fm_param.size(1)) + + p_update = p_update.unsqueeze(1) + index = index.unsqueeze(-1) + + # fast + # agg_p_update = (p_update * index).sum(0) + + # balanced agg + agg_p_update = 0 + + cur_split_size = 64 + while index.size(0) % cur_split_size != 0: + cur_split_size -= 1 + + for i in range(0, index.size(0), cur_split_size): + agg_p_update += p_update[i: i + cur_split_size] * index[i: i + cur_split_size] + agg_p_update = agg_p_update.sum(0) + + + else: + agg_p_update = (p_update.unsqueeze(1) * index).sum(0) + + new_fm_param = matched_fm_param + agg_p_update * self.ab_options['md_to_fm_alpha'] + + diff = self._compute_diff(matched_fm_param, new_fm_param) + + # NOTE: matched_fm_param may not be reference, may be a deepcopy!! + # and only here matched_fm_param needs to be updated, so another method dedicated for updating is necessary here + # matched_fm_param.copy_(new_fm_param) + self.update_fm_param(p_name, new_fm_param.T if t else new_fm_param) + + logger.debug(f'end feedback: {p_name}, diff: {diff:.6f} (md_to_fm_alpha={self.ab_options["md_to_fm_alpha"]:.4f})') + + @abstractmethod + @torch.no_grad() + def update_fm_param(self, md_param_name, cal_new_fm_param_by_md_param): + """ + you should get the reference of fm_param and update it + """ + raise NotImplementedError + + @torch.no_grad() + def aggregate_fms_to_self_fm(self, fms: List[nn.Module]): + # average task-agnositc parameters + logger.info('\n\naggregate foundation models to self foundation model...\n\n') + for p_name, self_p in self.models_dict['fm'].named_parameters(): + logger.debug(f'if aggregate {p_name}') + if 'abs' in p_name or p_name.startswith('norm') or p_name.startswith('head'): + logger.debug(f'{p_name} belongs to LoRA parameters/task-specific head, i.e. task-specific parameters, skip') + continue + all_p = [get_parameter(fm, p_name) for fm in fms] + if any([_p is None for _p in all_p]): + continue + + avg_p = sum(all_p) / len(all_p) + # [_p.copy_(avg_p) for _p in all_p] + + diff = self._compute_diff(self_p, avg_p) + logger.debug(f'aggregate {p_name}, diff {diff:.6f}') + + self_p.copy_(avg_p) + + @torch.no_grad() + def fm_feedback_to_md(self): + logger.info('\n\nself foundation model feedback to master DNN...\n\n') + # one-to-many + + # def downsample_2d_tensor(p: torch.Tensor, target_len: int): + # assert p.dim() == 2 # regard 2d weight as (batch_size, 1d_vector_dim) + # # return F.upsample(p.unsqueeze(1).unsqueeze(3), + # # size=(target_len, 1), + # # mode='bilinear').squeeze(3).squeeze(1) + # return F.interpolate(p.unsqueeze(1).unsqueeze(3), size=(target_len, 1), mode='bilinear').squeeze(3).squeeze(1) + + + for p_name, p in self.models_dict['md'].named_parameters(): + matched_fm_param = self.get_fm_matched_param_of_md_param(p_name) + logger.debug(f'if feedback: {p_name}') + if matched_fm_param is None: + continue + + index = self.models_dict['indexes'][p_name] + logger.debug(f'start feedback: {p_name}, {p.size()} -> {matched_fm_param.size()}, index: {index.size()}') + + if p.dim() > 1 and index.size(0) == p.size(1) and index.size(1) == matched_fm_param.size(1): + assert p.dim() == 2 + p = p.T + matched_fm_param = matched_fm_param.T + + if p.dim() == 2: + # matched_fm_param = downsample_2d_tensor(matched_fm_param, p.size(1)) + + matched_fm_param = matched_fm_param.unsqueeze(0) + index = index.unsqueeze(-1) + + # fast + # agg_p_update = (p_update * index).sum(0) + + # balanced agg + agg_fm_param = 0 + + cur_split_size = 64 + while index.size(1) % cur_split_size != 0: + cur_split_size -= 1 + + for i in range(0, index.size(1), cur_split_size): + agg_fm_param += matched_fm_param[:, i: i + cur_split_size] * index[:, i: i + cur_split_size] + agg_fm_param = agg_fm_param.sum(1) + # agg_fm_param = downsample_2d_tensor(agg_fm_param, p.size(1)) + + else: + agg_fm_param = (matched_fm_param.unsqueeze(0) * index).sum(1) + + + + diff = self._compute_diff(p, agg_fm_param) + p.copy_(agg_fm_param * self.ab_options['fm_to_md_alpha'] + (1. - self.ab_options['fm_to_md_alpha']) * p) + + logger.debug(f'end feedback: {p_name}, diff: {diff:.6f} (fm_to_md_alpha: {self.ab_options["fm_to_md_alpha"]:.4f})') + + @abstractmethod + def get_elastic_dnn_util(self) -> ElasticDNNUtil: + pass + + @abstractmethod + def get_task_head_params(self): + pass + + @abstractmethod + def get_md_matched_param_of_sd_param(self, sd_param_name): + pass + + @abstractmethod + def get_fm_matched_param_of_md_param(self, md_param_name): + pass + + @abstractmethod + def get_md_matched_param_of_fm_param(self, fm_param_name): + pass \ No newline at end of file diff --git a/new_impl/cv/elasticdnn/model/README.md b/new_impl/cv/elasticdnn/model/README.md new file mode 100644 index 0000000000000000000000000000000000000000..1e979bdba16531dc6decf4fbd618c6d98f347010 --- /dev/null +++ b/new_impl/cv/elasticdnn/model/README.md @@ -0,0 +1,8 @@ +## 20230527 + +only prune to_v (so should divide to_qkv to to_qkv and to_v, and only prune to_v) + +attn (1, 12, 197, 197) * pruned_v (1, 12, 197, int(64\*s)) = out1 (1, 12, 197, int(64\*s)) +out1 rearange to out2 (1, 197, int(768\*s)) + +out2 (1, 197, int(768\*s)) * pruned mlp_fc1 (int(768\*s), int(3072\*s)) \ No newline at end of file diff --git a/new_impl/cv/elasticdnn/model/__init__.py b/new_impl/cv/elasticdnn/model/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..90c653cedcc240c909fe3872390bf325d09a4de7 --- /dev/null +++ b/new_impl/cv/elasticdnn/model/__init__.py @@ -0,0 +1,4 @@ +from .base import ElasticDNNUtil + +from .cnn import ElasticCNNUtil +from .vit import ElasticViTUtil diff --git a/new_impl/cv/elasticdnn/model/__pycache__/__init__.cpython-38.pyc b/new_impl/cv/elasticdnn/model/__pycache__/__init__.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..077189d071c828adb71cadea36ab12ae0aa83a00 Binary files /dev/null and b/new_impl/cv/elasticdnn/model/__pycache__/__init__.cpython-38.pyc differ diff --git a/new_impl/cv/elasticdnn/model/__pycache__/base.cpython-38.pyc b/new_impl/cv/elasticdnn/model/__pycache__/base.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7abe5d347740669897a6bb5b355a19bc41e7a925 Binary files /dev/null and b/new_impl/cv/elasticdnn/model/__pycache__/base.cpython-38.pyc differ diff --git a/new_impl/cv/elasticdnn/model/__pycache__/cnn.cpython-38.pyc b/new_impl/cv/elasticdnn/model/__pycache__/cnn.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..59632465b42d51970fa942238501eac3f78dce35 Binary files /dev/null and b/new_impl/cv/elasticdnn/model/__pycache__/cnn.cpython-38.pyc differ diff --git a/new_impl/cv/elasticdnn/model/__pycache__/vit.cpython-38.pyc b/new_impl/cv/elasticdnn/model/__pycache__/vit.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2dedc38bd8ecb26af4a925040bff631cc6bde986 Binary files /dev/null and b/new_impl/cv/elasticdnn/model/__pycache__/vit.cpython-38.pyc differ diff --git a/new_impl/cv/elasticdnn/model/base.py b/new_impl/cv/elasticdnn/model/base.py new file mode 100644 index 0000000000000000000000000000000000000000..86376da11c937273fc3b16b4c01f2d03c11d2033 --- /dev/null +++ b/new_impl/cv/elasticdnn/model/base.py @@ -0,0 +1,139 @@ +import torch +from torch import nn +from abc import ABC, abstractmethod + +from utils.dl.common.model import get_model_device, get_model_latency, get_model_size +from utils.common.log import logger + + +class KTakesAll(nn.Module): + # k means sparsity (the larger k is, the smaller model is) + def __init__(self, k): + super(KTakesAll, self).__init__() + self.k = k + self.cached_i = None + + def forward(self, g: torch.Tensor): + # k = int(g.size(1) * self.k) + # i = (-g).topk(k, 1)[1] + # t = g.scatter(1, i, 0) + + k = int(g.size(-1) * self.k) + i = (-g).topk(k, -1)[1] + self.cached_i = i + t = g.scatter(-1, i, 0) + + return t + + +class Abs(nn.Module): + def __init__(self): + super(Abs, self).__init__() + + def forward(self, x): + return x.abs() + + +class Layer_WrappedWithFBS(nn.Module): + def __init__(self): + super(Layer_WrappedWithFBS, self).__init__() + + init_sparsity = 0.5 + self.k_takes_all = KTakesAll(init_sparsity) + + self.cached_raw_channel_attention = None + self.cached_channel_attention = None + self.use_cached_channel_attention = False + + +class ElasticDNNUtil(ABC): + @abstractmethod + def convert_raw_dnn_to_master_dnn(self, raw_dnn: nn.Module, r: float, ignore_layers=[]): + raise NotImplementedError + + def convert_raw_dnn_to_master_dnn_with_perf_test(self, raw_dnn: nn.Module, r: float, ignore_layers=[]): + raw_dnn_size = get_model_size(raw_dnn, True) + master_dnn = self.convert_raw_dnn_to_master_dnn(raw_dnn, r, ignore_layers) + master_dnn_size = get_model_size(master_dnn, True) + + logger.info(f'master DNN w/o FBS ({raw_dnn_size:.3f}MB) -> master DNN w/ FBS ({master_dnn_size:.3f}MB) ' + f'(↑ {(((master_dnn_size - raw_dnn_size) / raw_dnn_size) * 100.):.2f}%)') + return master_dnn + + def set_master_dnn_inference_via_cached_channel_attention(self, master_dnn: nn.Module): + for name, module in master_dnn.named_modules(): + if isinstance(module, Layer_WrappedWithFBS): + assert module.cached_channel_attention is not None + module.use_cached_channel_attention = True + + def set_master_dnn_dynamic_inference(self, master_dnn: nn.Module): + for name, module in master_dnn.named_modules(): + if isinstance(module, Layer_WrappedWithFBS): + module.cached_channel_attention = None + module.use_cached_channel_attention = False + + def train_only_fbs_of_master_dnn(self, master_dnn: nn.Module): + fbs_params = [] + for n, p in master_dnn.named_parameters(): + if '.fbs' in n: + fbs_params += [p] + p.requires_grad = True + else: + p.requires_grad = False + return fbs_params + + def get_accu_l1_reg_of_raw_channel_attention_in_master_dnn(self, master_dnn: nn.Module): + res = 0. + for name, module in master_dnn.named_modules(): + if isinstance(module, Layer_WrappedWithFBS): + res += module.cached_raw_channel_attention.norm(1) + return res + + def get_raw_channel_attention_in_master_dnn(self, master_dnn: nn.Module): + res = {} + for name, module in master_dnn.named_modules(): + if isinstance(module, Layer_WrappedWithFBS): + res[name] = module.cached_raw_channel_attention + return res + + def set_master_dnn_sparsity(self, master_dnn: nn.Module, sparsity: float): + assert 0 <= sparsity <= 1., sparsity + for name, module in master_dnn.named_modules(): + if isinstance(module, KTakesAll): + module.k = sparsity + logger.debug(f'set master DNN sparsity to {sparsity}') + + def clear_cached_channel_attention_in_master_dnn(self, master_dnn: nn.Module): + for name, module in master_dnn.named_modules(): + if isinstance(module, Layer_WrappedWithFBS): + module.cached_raw_channel_attention = None + module.cached_channel_attention = None + + @abstractmethod + def select_most_rep_sample(self, master_dnn: nn.Module, samples: torch.Tensor): + raise NotImplementedError + + @abstractmethod + def extract_surrogate_dnn_via_samples(self, master_dnn: nn.Module, samples: torch.Tensor, return_detail=False): + raise NotImplementedError + + def extract_surrogate_dnn_via_samples_with_perf_test(self, master_dnn: nn.Module, samples: torch.Tensor, return_detail=False): + master_dnn_size = get_model_size(master_dnn, True) + master_dnn_latency = get_model_latency(master_dnn, (1, *list(samples.size())[1:]), 50, + get_model_device(master_dnn), 50, False) + + res = self.extract_surrogate_dnn_via_samples(master_dnn, samples, return_detail) + if not return_detail: + surrogate_dnn = res + else: + surrogate_dnn, unpruned_indexes_of_layers = res + surrogate_dnn_size = get_model_size(surrogate_dnn, True) + surrogate_dnn_latency = get_model_latency(surrogate_dnn, (1, *list(samples.size())[1:]), 50, + get_model_device(surrogate_dnn), 50, False) + + logger.info(f'master DNN ({master_dnn_size:.3f}MB, {master_dnn_latency:.4f}s/sample) -> ' + f'surrogate DNN ({surrogate_dnn_size:.3f}MB, {surrogate_dnn_latency:.4f}s/sample)\n' + f'(model size: ↓ {(master_dnn_size / surrogate_dnn_size):.2f}x, ' + f'latency: ↓ {(master_dnn_latency / surrogate_dnn_latency):.2f}x)') + + return res diff --git a/new_impl/cv/elasticdnn/model/bert.py b/new_impl/cv/elasticdnn/model/bert.py new file mode 100644 index 0000000000000000000000000000000000000000..e8f87a6f4943b8193b7d2a3be7223e107a8e22e1 --- /dev/null +++ b/new_impl/cv/elasticdnn/model/bert.py @@ -0,0 +1,359 @@ +from copy import deepcopy +from typing import Optional, Union +import torch +from torch import nn +from einops import rearrange, repeat +from einops.layers.torch import Rearrange +import tqdm + +from utils.dl.common.model import get_model_device, get_model_size, set_module, get_module +from .base import Abs, KTakesAll, ElasticDNNUtil, Layer_WrappedWithFBS +from utils.common.log import logger + + +class SqueezeLast(nn.Module): + def __init__(self): + super(SqueezeLast, self).__init__() + + def forward(self, x): + return x.squeeze(-1) + + +class ProjConv_WrappedWithFBS(Layer_WrappedWithFBS): + def __init__(self, proj: nn.Conv2d, r): + super(ProjConv_WrappedWithFBS, self).__init__() + + self.proj = proj + + # for conv: (B, C_in, H, W) -> (B, C_in) -> (B, C_out) + # for mlp in ViT: (B, #patches, D: dim of patches embedding) -> (B, D) -> (B, C_out) + self.fbs = nn.Sequential( + Abs(), + nn.AdaptiveAvgPool1d(1), + SqueezeLast(), + nn.Linear(proj.in_channels, proj.out_channels // r), + nn.ReLU(), + nn.Linear(proj.out_channels // r, proj.out_channels), + nn.ReLU() + ) + + nn.init.constant_(self.fbs[6].bias, 1.) + nn.init.kaiming_normal_(self.fbs[6].weight) + + + def forward(self, x): + if self.use_cached_channel_attention and self.cached_channel_attention is not None: + channel_attention = self.cached_channel_attention + else: + self.cached_raw_channel_attention = self.fbs(x) + self.cached_channel_attention = self.k_takes_all(self.cached_raw_channel_attention) + + channel_attention = self.cached_channel_attention + + raw_res = self.proj(x) + + return channel_attention.unsqueeze(1) * raw_res # TODO: + + +class Linear_WrappedWithFBS(Layer_WrappedWithFBS): + def __init__(self, linear: nn.Linear, r): + super(Linear_WrappedWithFBS, self).__init__() + + self.linear = linear + + # for conv: (B, C_in, H, W) -> (B, C_in) -> (B, C_out) + # for mlp in ViT: (B, #patches, D: dim of patches embedding) -> (B, D) -> (B, C_out) + self.fbs = nn.Sequential( + Rearrange('b n d -> b d n'), + Abs(), + nn.AdaptiveAvgPool1d(1), + SqueezeLast(), + nn.Linear(linear.in_features, linear.out_features // r), + nn.ReLU(), + nn.Linear(linear.out_features // r, linear.out_features), + nn.ReLU() + ) + + nn.init.constant_(self.fbs[6].bias, 1.) + nn.init.kaiming_normal_(self.fbs[6].weight) + + + def forward(self, x): + if self.use_cached_channel_attention and self.cached_channel_attention is not None: + channel_attention = self.cached_channel_attention + else: + self.cached_raw_channel_attention = self.fbs(x) + self.cached_channel_attention = self.k_takes_all(self.cached_raw_channel_attention) + + channel_attention = self.cached_channel_attention + + raw_res = self.linear(x) + + return channel_attention.unsqueeze(1) * raw_res + + +class ToQKV_WrappedWithFBS(Layer_WrappedWithFBS): + """ + This regards to_q/to_k/to_v as a whole (in fact it consists of multiple heads) and prunes it. + It seems different channels of different heads are pruned according to the input. + This is different from "removing some head" or "removing the same channels in each head". + """ + def __init__(self, to_qkv: nn.Linear, r): + super(ToQKV_WrappedWithFBS, self).__init__() + + # self.to_qkv = to_qkv + + self.to_qk = nn.Linear(to_qkv.in_features, to_qkv.out_features // 3 * 2, bias=to_qkv.bias is not None) + self.to_v = nn.Linear(to_qkv.in_features, to_qkv.out_features // 3, bias=to_qkv.bias is not None) + self.to_qk.weight.data.copy_(to_qkv.weight.data[0: to_qkv.out_features // 3 * 2]) + if to_qkv.bias is not None: + self.to_qk.bias.data.copy_(to_qkv.bias.data[0: to_qkv.out_features // 3 * 2]) + self.to_v.weight.data.copy_(to_qkv.weight.data[to_qkv.out_features // 3 * 2: ]) + if to_qkv.bias is not None: + self.to_v.bias.data.copy_(to_qkv.bias.data[to_qkv.out_features // 3 * 2: ]) + + self.fbs = nn.Sequential( + Rearrange('b n d -> b d n'), + Abs(), + nn.AdaptiveAvgPool1d(1), + SqueezeLast(), + nn.Linear(to_qkv.in_features, to_qkv.out_features // 3 // r), + nn.ReLU(), + # nn.Linear(to_qkv.out_features // 3 // r, to_qkv.out_features // 3), + nn.Linear(to_qkv.out_features // 3 // r, self.to_v.out_features), + nn.ReLU() + ) + + nn.init.constant_(self.fbs[6].bias, 1.) + nn.init.kaiming_normal_(self.fbs[6].weight) + + def forward(self, x): + if self.use_cached_channel_attention and self.cached_channel_attention is not None: + channel_attention = self.cached_channel_attention + else: + self.cached_raw_channel_attention = self.fbs(x) + + # print() + # for attn in self.cached_raw_channel_attention.chunk(3, dim=1)[0: 1]: + # print(self.cached_raw_channel_attention.size(), attn.size()) + # print(self.k_takes_all.k) + # print(attn[0].nonzero(as_tuple=True)[0].size(), attn[0]) + + self.cached_channel_attention = self.k_takes_all(self.cached_raw_channel_attention) + + + # for attn in self.cached_channel_attention.chunk(3, dim=1)[0: 1]: + # print(self.cached_channel_attention.size(), attn.size()) + # print(self.k_takes_all.k) + # print(attn[0].nonzero(as_tuple=True)[0].size(), attn[0]) + # print() + + channel_attention = self.cached_channel_attention + + qk = self.to_qk(x) + v = channel_attention.unsqueeze(1) * self.to_v(x) + return torch.cat([qk, v], dim=-1) + + # qkv = raw_res.chunk(3, dim = -1) + + # raw_v = qkv[2] + # print('raw_k, raw_v', qkv[0].sum((0, 1))[0: 10], qkv[0].sum((0, 1)).nonzero(as_tuple=True)[0].size(), + # qkv[1].sum((0, 1))[0: 10], qkv[1].sum((0, 1)).nonzero(as_tuple=True)[0].size(),) + # print('raw_v', raw_v.size(), raw_v.sum((0, 1))[0: 10], raw_v.sum((0, 1)).nonzero(as_tuple=True)[0].size()) + + # qkv_attn = channel_attention.chunk(3, dim=-1) + # print('attn', [attn[0][0: 10] for attn in qkv_attn]) + # print(channel_attention.unsqueeze(1).size(), raw_res.size()) + # print('fbs', channel_attention.size(), raw_res.size()) + # return channel_attention.unsqueeze(1) * raw_res + + +class StaticFBS(nn.Module): + def __init__(self, static_channel_attention): + super(StaticFBS, self).__init__() + assert static_channel_attention.dim() == 2 and static_channel_attention.size(0) == 1 + self.static_channel_attention = nn.Parameter(static_channel_attention, requires_grad=False) # (1, dim) + + def forward(self, x): + # print('staticfbs', x, self.static_channel_attention.unsqueeze(1)) + return x * self.static_channel_attention.unsqueeze(1) + + +class ElasticBertUtil(ElasticDNNUtil): + def convert_raw_dnn_to_master_dnn(self, raw_dnn: nn.Module, r: float, ignore_layers=[]): + assert len(ignore_layers) == 0, 'not supported yet' + + raw_vit = deepcopy(raw_dnn) + + # set_module(module, 'patch_embed.proj', ProjConv_WrappedWithFBS(module.patch_embed.proj, r)) + + for name, module in raw_vit.named_modules(): + # if name.endswith('attn'): + # set_module(module, 'qkv', ToQKV_WrappedWithFBS(module.qkv, r)) + if name.endswith('intermediate'): + set_module(module, 'dense', Linear_WrappedWithFBS(module.dense, r)) + + return raw_vit + + def set_master_dnn_sparsity(self, master_dnn: nn.Module, sparsity: float): + # for name, module in master_dnn.named_modules(): + # if not name.endswith('attn'): + # continue + + # q_features = module.qkv.to_qk.out_features // 2 + + # if (q_features - int(q_features * sparsity)) % module.num_heads != 0: + # # tune sparsity to ensure #unpruned channel % num_heads == 0 + # # so that the pruning seems to reduce the dim_head of each head + # tuned_sparsity = 1. - int((q_features - int(q_features * sparsity)) / module.num_heads) * module.num_heads / q_features + # logger.debug(f'tune sparsity from {sparsity:.2f} to {tuned_sparsity}') + # sparsity = tuned_sparsity + # break + + return super().set_master_dnn_sparsity(master_dnn, sparsity) + + def select_most_rep_sample(self, master_dnn: nn.Module, samples: torch.Tensor): + # print(samples) + # return samples[0].unsqueeze(0) + res = {k: v[0: 1] for k, v in samples.items()} + return res + + def extract_surrogate_dnn_via_samples(self, master_dnn: nn.Module, samples: torch.Tensor, return_detail=False): + sample = self.select_most_rep_sample(master_dnn, samples) + # assert sample.dim() == 4 and sample.size(0) == 1 + + # print('before') + master_dnn.eval() + self.clear_cached_channel_attention_in_master_dnn(master_dnn) + with torch.no_grad(): + master_dnn_output = master_dnn(**sample) + + # print('after') + + boosted_vit = deepcopy(master_dnn) + + def get_unpruned_indexes_from_channel_attn(channel_attn: torch.Tensor, k): + assert channel_attn.size(0) == 1, 'use A representative sample to generate channel attentions' + + # print('attn_in_unpruned', channel_attn[0][0: 10]) + + res = channel_attn[0].nonzero(as_tuple=True)[0] # should be one-dim + + # res = channel_attn[0].argsort(descending=True)[0: -int(channel_attn.size(1) * k)].sort()[0] + + # g = channel_attn + # k = g.size(1) - int(g.size(1) * k) + # res = g.topk(k, 1)[1][0].sort()[0] + + return res + + unpruned_indexes_of_layers = {} + + # for attn, ff in boosted_vit.transformer.layers: + # for block_i, block in enumerate(boosted_vit.blocks): + for block_i, block in enumerate(boosted_vit.bert.encoder.layer): + # attn = block.attn + # ff = block.mlp + + ff_0 = get_module(block, f'intermediate.dense') + # ff_0_unpruned_indexes = get_unpruned_indexes_from_channel_attn(ff_0.cached_channel_attention, k) + ff_0_pruned_indexes = ff_0.k_takes_all.cached_i[0].sort()[0] + ff_0_unpruned_indexes = torch.LongTensor([ii for ii in range(ff_0.cached_channel_attention.size(1)) if ii not in ff_0_pruned_indexes]) + new_ff_0 = nn.Linear(ff_0.linear.in_features, ff_0_unpruned_indexes.size(0), ff_0.linear.bias is not None) + new_ff_0.weight.data.copy_(ff_0.linear.weight.data[ff_0_unpruned_indexes]) + if ff_0.linear.bias is not None: + new_ff_0.bias.data.copy_(ff_0.linear.bias.data[ff_0_unpruned_indexes]) + set_module(block, 'intermediate.dense', nn.Sequential(new_ff_0, StaticFBS(ff_0.cached_channel_attention[:, ff_0_unpruned_indexes]))) + + ff_1 = get_module(block, f'output.dense') + new_ff_1 = nn.Linear(ff_0_unpruned_indexes.size(0), ff_1.out_features, ff_1.bias is not None) + new_ff_1.weight.data.copy_(ff_1.weight.data[:, ff_0_unpruned_indexes]) + if ff_1.bias is not None: + new_ff_1.bias.data.copy_(ff_1.bias.data) + set_module(block, 'output.dense', new_ff_1) + + unpruned_indexes_of_layers[f'bert.encoder.layer.{block_i}.intermediate.dense.0.weight'] = ff_0_unpruned_indexes + + surrogate_dnn = boosted_vit + surrogate_dnn.eval() + surrogate_dnn = surrogate_dnn.to(get_model_device(master_dnn)) + # logger.debug(surrogate_dnn) + with torch.no_grad(): + surrogate_dnn_output = surrogate_dnn(**sample) + + output_diff = ((surrogate_dnn_output - master_dnn_output) ** 2).sum() + # assert output_diff < 1e-4, output_diff + logger.info(f'output diff of master and surrogate DNN: {output_diff}') + logger.debug(f'example output of master/surrogate: {master_dnn_output.sum(0)[0: 10]}, {surrogate_dnn_output.sum(0)[0: 10]}') + # logger.info(f'\nonly prune mlp!!!!\n') + # logger.info(f'\nonly prune mlp!!!!\n') + + if return_detail: + return boosted_vit, unpruned_indexes_of_layers + + return boosted_vit + + def extract_surrogate_dnn_via_samples_with_perf_test(self, master_dnn: nn.Module, samples: torch.Tensor, return_detail=False): + master_dnn_size = get_model_size(master_dnn, True) + master_dnn_latency = self._get_model_latency(master_dnn, samples, 50, + get_model_device(master_dnn), 50, False) + + res = self.extract_surrogate_dnn_via_samples(master_dnn, samples, return_detail) + if not return_detail: + surrogate_dnn = res + else: + surrogate_dnn, unpruned_indexes_of_layers = res + surrogate_dnn_size = get_model_size(surrogate_dnn, True) + surrogate_dnn_latency = self._get_model_latency(master_dnn, samples, 50, + get_model_device(master_dnn), 50, False) + + logger.info(f'master DNN ({master_dnn_size:.3f}MB, {master_dnn_latency:.4f}s/sample) -> ' + f'surrogate DNN ({surrogate_dnn_size:.3f}MB, {surrogate_dnn_latency:.4f}s/sample)\n' + f'(model size: ↓ {(master_dnn_size / surrogate_dnn_size):.2f}x, ' + f'latency: ↓ {(master_dnn_latency / surrogate_dnn_latency):.2f}x)') + + return res + + def _get_model_latency(self, model: torch.nn.Module, model_input_size, sample_num: int, + device: str, warmup_sample_num: int, return_detail=False): + import time + + if isinstance(model_input_size, tuple): + dummy_input = torch.rand(model_input_size).to(device) + else: + dummy_input = model_input_size + + model = model.to(device) + model.eval() + + # warm up + with torch.no_grad(): + for _ in range(warmup_sample_num): + model(**dummy_input) + + infer_time_list = [] + + if device == 'cuda' or 'cuda' in str(device): + with torch.no_grad(): + for _ in range(sample_num): + s, e = torch.cuda.Event(enable_timing=True), torch.cuda.Event(enable_timing=True) + s.record() + model(**dummy_input) + e.record() + torch.cuda.synchronize() + cur_model_infer_time = s.elapsed_time(e) / 1000. + infer_time_list += [cur_model_infer_time] + + else: + with torch.no_grad(): + for _ in range(sample_num): + start = time.time() + model(**dummy_input) + cur_model_infer_time = time.time() - start + infer_time_list += [cur_model_infer_time] + + avg_infer_time = sum(infer_time_list) / sample_num + + if return_detail: + return avg_infer_time, infer_time_list + return avg_infer_time \ No newline at end of file diff --git a/new_impl/cv/elasticdnn/model/blip.py b/new_impl/cv/elasticdnn/model/blip.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/new_impl/cv/elasticdnn/model/clip.py b/new_impl/cv/elasticdnn/model/clip.py new file mode 100644 index 0000000000000000000000000000000000000000..d7ff70af97a345fe2e7228d239c2ce2dda5ab1c0 --- /dev/null +++ b/new_impl/cv/elasticdnn/model/clip.py @@ -0,0 +1,316 @@ +from copy import deepcopy +from typing import Optional, Union +import torch +from torch import nn +from einops import rearrange, repeat +from einops.layers.torch import Rearrange +import tqdm + +from utils.dl.common.model import get_model_device, get_model_size, set_module, get_module +from .base import Abs, KTakesAll, ElasticDNNUtil, Layer_WrappedWithFBS +from utils.common.log import logger + + +class SqueezeLast(nn.Module): + def __init__(self): + super(SqueezeLast, self).__init__() + + def forward(self, x): + return x.squeeze(-1) + + +class ProjConv_WrappedWithFBS(Layer_WrappedWithFBS): + def __init__(self, proj: nn.Conv2d, r): + super(ProjConv_WrappedWithFBS, self).__init__() + + self.proj = proj + + # for conv: (B, C_in, H, W) -> (B, C_in) -> (B, C_out) + # for mlp in ViT: (B, #patches, D: dim of patches embedding) -> (B, D) -> (B, C_out) + self.fbs = nn.Sequential( + Abs(), + nn.AdaptiveAvgPool1d(1), + SqueezeLast(), + nn.Linear(proj.in_channels, proj.out_channels // r), + nn.ReLU(), + nn.Linear(proj.out_channels // r, proj.out_channels), + nn.ReLU() + ) + + nn.init.constant_(self.fbs[6].bias, 1.) + nn.init.kaiming_normal_(self.fbs[6].weight) + + + def forward(self, x): + if self.use_cached_channel_attention and self.cached_channel_attention is not None: + channel_attention = self.cached_channel_attention + else: + self.cached_raw_channel_attention = self.fbs(x) + self.cached_channel_attention = self.k_takes_all(self.cached_raw_channel_attention) + + channel_attention = self.cached_channel_attention + + raw_res = self.proj(x) + + return channel_attention.unsqueeze(1) * raw_res # TODO: + + +class Linear_WrappedWithFBS(Layer_WrappedWithFBS): + def __init__(self, linear: nn.Linear, r): + super(Linear_WrappedWithFBS, self).__init__() + + self.linear = linear + + # for conv: (B, C_in, H, W) -> (B, C_in) -> (B, C_out) + # for mlp in ViT: (B, #patches, D: dim of patches embedding) -> (B, D) -> (B, C_out) + self.fbs = nn.Sequential( + Rearrange('b n d -> b d n'), + Abs(), + nn.AdaptiveAvgPool1d(1), + SqueezeLast(), + nn.Linear(linear.in_features, linear.out_features // r), + nn.ReLU(), + nn.Linear(linear.out_features // r, linear.out_features), + nn.ReLU() + ) + + nn.init.constant_(self.fbs[6].bias, 1.) + nn.init.kaiming_normal_(self.fbs[6].weight) + + + def forward(self, x): + if self.use_cached_channel_attention and self.cached_channel_attention is not None: + channel_attention = self.cached_channel_attention + else: + self.cached_raw_channel_attention = self.fbs(x) + self.cached_channel_attention = self.k_takes_all(self.cached_raw_channel_attention) + + channel_attention = self.cached_channel_attention + + raw_res = self.linear(x) + + return channel_attention.unsqueeze(1) * raw_res + + +class ToQKV_WrappedWithFBS(Layer_WrappedWithFBS): + """ + This regards to_q/to_k/to_v as a whole (in fact it consists of multiple heads) and prunes it. + It seems different channels of different heads are pruned according to the input. + This is different from "removing some head" or "removing the same channels in each head". + """ + def __init__(self, to_qkv: nn.Linear, r): + super(ToQKV_WrappedWithFBS, self).__init__() + + # self.to_qkv = to_qkv + + self.to_qk = nn.Linear(to_qkv.in_features, to_qkv.out_features // 3 * 2, bias=to_qkv.bias is not None) + self.to_v = nn.Linear(to_qkv.in_features, to_qkv.out_features // 3, bias=to_qkv.bias is not None) + self.to_qk.weight.data.copy_(to_qkv.weight.data[0: to_qkv.out_features // 3 * 2]) + if to_qkv.bias is not None: + self.to_qk.bias.data.copy_(to_qkv.bias.data[0: to_qkv.out_features // 3 * 2]) + self.to_v.weight.data.copy_(to_qkv.weight.data[to_qkv.out_features // 3 * 2: ]) + if to_qkv.bias is not None: + self.to_v.bias.data.copy_(to_qkv.bias.data[to_qkv.out_features // 3 * 2: ]) + + self.fbs = nn.Sequential( + Rearrange('b n d -> b d n'), + Abs(), + nn.AdaptiveAvgPool1d(1), + SqueezeLast(), + nn.Linear(to_qkv.in_features, to_qkv.out_features // 3 // r), + nn.ReLU(), + # nn.Linear(to_qkv.out_features // 3 // r, to_qkv.out_features // 3), + nn.Linear(to_qkv.out_features // 3 // r, self.to_v.out_features), + nn.ReLU() + ) + + nn.init.constant_(self.fbs[6].bias, 1.) + nn.init.kaiming_normal_(self.fbs[6].weight) + + def forward(self, x): + if self.use_cached_channel_attention and self.cached_channel_attention is not None: + channel_attention = self.cached_channel_attention + else: + self.cached_raw_channel_attention = self.fbs(x) + + # print() + # for attn in self.cached_raw_channel_attention.chunk(3, dim=1)[0: 1]: + # print(self.cached_raw_channel_attention.size(), attn.size()) + # print(self.k_takes_all.k) + # print(attn[0].nonzero(as_tuple=True)[0].size(), attn[0]) + + self.cached_channel_attention = self.k_takes_all(self.cached_raw_channel_attention) + + + # for attn in self.cached_channel_attention.chunk(3, dim=1)[0: 1]: + # print(self.cached_channel_attention.size(), attn.size()) + # print(self.k_takes_all.k) + # print(attn[0].nonzero(as_tuple=True)[0].size(), attn[0]) + # print() + + channel_attention = self.cached_channel_attention + + qk = self.to_qk(x) + v = channel_attention.unsqueeze(1) * self.to_v(x) + return torch.cat([qk, v], dim=-1) + + # qkv = raw_res.chunk(3, dim = -1) + + # raw_v = qkv[2] + # print('raw_k, raw_v', qkv[0].sum((0, 1))[0: 10], qkv[0].sum((0, 1)).nonzero(as_tuple=True)[0].size(), + # qkv[1].sum((0, 1))[0: 10], qkv[1].sum((0, 1)).nonzero(as_tuple=True)[0].size(),) + # print('raw_v', raw_v.size(), raw_v.sum((0, 1))[0: 10], raw_v.sum((0, 1)).nonzero(as_tuple=True)[0].size()) + + # qkv_attn = channel_attention.chunk(3, dim=-1) + # print('attn', [attn[0][0: 10] for attn in qkv_attn]) + # print(channel_attention.unsqueeze(1).size(), raw_res.size()) + # print('fbs', channel_attention.size(), raw_res.size()) + # return channel_attention.unsqueeze(1) * raw_res + + +class StaticFBS(nn.Module): + def __init__(self, static_channel_attention): + super(StaticFBS, self).__init__() + assert static_channel_attention.dim() == 2 and static_channel_attention.size(0) == 1 + self.static_channel_attention = nn.Parameter(static_channel_attention, requires_grad=False) # (1, dim) + + def forward(self, x): + # print('staticfbs', x, self.static_channel_attention.unsqueeze(1)) + return x * self.static_channel_attention.unsqueeze(1) + + +class ElasticCLIPUtil(ElasticDNNUtil): + def convert_raw_dnn_to_master_dnn(self, raw_dnn: nn.Module, r: float, ignore_layers=[]): + assert len(ignore_layers) == 0, 'not supported yet' + + raw_vit = deepcopy(raw_dnn) + + # set_module(module, 'patch_embed.proj', ProjConv_WrappedWithFBS(module.patch_embed.proj, r)) + + for name, module in raw_vit.named_modules(): + # if name.endswith('attn'): + # set_module(module, 'qkv', ToQKV_WrappedWithFBS(module.qkv, r)) + if 'vision_model' in name and name.endswith('mlp'): + set_module(module, 'fc1', Linear_WrappedWithFBS(module.fc1, r)) + + return raw_vit + + def set_master_dnn_sparsity(self, master_dnn: nn.Module, sparsity: float): + # for name, module in master_dnn.named_modules(): + # if not name.endswith('attn'): + # continue + + # q_features = module.qkv.to_qk.out_features // 2 + + # if (q_features - int(q_features * sparsity)) % module.num_heads != 0: + # # tune sparsity to ensure #unpruned channel % num_heads == 0 + # # so that the pruning seems to reduce the dim_head of each head + # tuned_sparsity = 1. - int((q_features - int(q_features * sparsity)) / module.num_heads) * module.num_heads / q_features + # logger.debug(f'tune sparsity from {sparsity:.2f} to {tuned_sparsity}') + # sparsity = tuned_sparsity + # break + + return super().set_master_dnn_sparsity(master_dnn, sparsity) + + def select_most_rep_sample(self, master_dnn: nn.Module, samples: torch.Tensor): + return samples[0].unsqueeze(0) + + def extract_surrogate_dnn_via_samples(self, master_dnn: nn.Module, samples: torch.Tensor, return_detail=False): + sample = self.select_most_rep_sample(master_dnn, samples) + assert sample.dim() == 4 and sample.size(0) == 1 + + # print('before') + master_dnn.eval() + self.clear_cached_channel_attention_in_master_dnn(master_dnn) + with torch.no_grad(): + master_dnn_output = master_dnn(sample) + + # print('after') + + boosted_vit = deepcopy(master_dnn) + + def get_unpruned_indexes_from_channel_attn(channel_attn: torch.Tensor, k): + assert channel_attn.size(0) == 1, 'use A representative sample to generate channel attentions' + + # print('attn_in_unpruned', channel_attn[0][0: 10]) + + res = channel_attn[0].nonzero(as_tuple=True)[0] # should be one-dim + + # res = channel_attn[0].argsort(descending=True)[0: -int(channel_attn.size(1) * k)].sort()[0] + + # g = channel_attn + # k = g.size(1) - int(g.size(1) * k) + # res = g.topk(k, 1)[1][0].sort()[0] + + return res + + unpruned_indexes_of_layers = {} + + # for attn, ff in boosted_vit.transformer.layers: + # for block_i, block in enumerate(boosted_vit.blocks): + # for block_i, block in enumerate(boosted_vit.model.text_model.encoder.layers): + # # attn = block.attn + # # ff = block.mlp + + # ff_0 = get_module(block, f'mlp.fc1') + # # ff_0_unpruned_indexes = get_unpruned_indexes_from_channel_attn(ff_0.cached_channel_attention, k) + # ff_0_pruned_indexes = ff_0.k_takes_all.cached_i[0].sort()[0] + # ff_0_unpruned_indexes = torch.LongTensor([ii for ii in range(ff_0.cached_channel_attention.size(1)) if ii not in ff_0_pruned_indexes]) + # new_ff_0 = nn.Linear(ff_0.linear.in_features, ff_0_unpruned_indexes.size(0), ff_0.linear.bias is not None) + # new_ff_0.weight.data.copy_(ff_0.linear.weight.data[ff_0_unpruned_indexes]) + # if ff_0.linear.bias is not None: + # new_ff_0.bias.data.copy_(ff_0.linear.bias.data[ff_0_unpruned_indexes]) + # set_module(block, 'mlp.fc1', nn.Sequential(new_ff_0, StaticFBS(ff_0.cached_channel_attention[:, ff_0_unpruned_indexes]))) + + # ff_1 = get_module(block, f'mlp.fc2') + # new_ff_1 = nn.Linear(ff_0_unpruned_indexes.size(0), ff_1.out_features, ff_1.bias is not None) + # new_ff_1.weight.data.copy_(ff_1.weight.data[:, ff_0_unpruned_indexes]) + # if ff_1.bias is not None: + # new_ff_1.bias.data.copy_(ff_1.bias.data) + # set_module(block, 'mlp.fc2', new_ff_1) + + # unpruned_indexes_of_layers[f'boosted_vit.model.text_model.encoder.layers.{block_i}.mlp.fc1.weight'] = ff_0_unpruned_indexes + + + for block_i, block in enumerate(boosted_vit.model.vision_model.encoder.layers): + # attn = block.attn + # ff = block.mlp + + ff_0 = get_module(block, f'mlp.fc1') + # ff_0_unpruned_indexes = get_unpruned_indexes_from_channel_attn(ff_0.cached_channel_attention, k) + ff_0_pruned_indexes = ff_0.k_takes_all.cached_i[0].sort()[0] + ff_0_unpruned_indexes = torch.LongTensor([ii for ii in range(ff_0.cached_channel_attention.size(1)) if ii not in ff_0_pruned_indexes]) + new_ff_0 = nn.Linear(ff_0.linear.in_features, ff_0_unpruned_indexes.size(0), ff_0.linear.bias is not None) + new_ff_0.weight.data.copy_(ff_0.linear.weight.data[ff_0_unpruned_indexes]) + if ff_0.linear.bias is not None: + new_ff_0.bias.data.copy_(ff_0.linear.bias.data[ff_0_unpruned_indexes]) + set_module(block, 'mlp.fc1', nn.Sequential(new_ff_0, StaticFBS(ff_0.cached_channel_attention[:, ff_0_unpruned_indexes]))) + + ff_1 = get_module(block, f'mlp.fc2') + new_ff_1 = nn.Linear(ff_0_unpruned_indexes.size(0), ff_1.out_features, ff_1.bias is not None) + new_ff_1.weight.data.copy_(ff_1.weight.data[:, ff_0_unpruned_indexes]) + if ff_1.bias is not None: + new_ff_1.bias.data.copy_(ff_1.bias.data) + set_module(block, 'mlp.fc2', new_ff_1) + + unpruned_indexes_of_layers[f'boosted_vit.model.vision_model.encoder.layers.{block_i}.mlp.fc1.weight'] = ff_0_unpruned_indexes + + surrogate_dnn = boosted_vit + surrogate_dnn.eval() + surrogate_dnn = surrogate_dnn.to(get_model_device(master_dnn)) + # logger.debug(surrogate_dnn) + with torch.no_grad(): + surrogate_dnn_output = surrogate_dnn(sample) + + output_diff = ((surrogate_dnn_output - master_dnn_output) ** 2).sum() + # assert output_diff < 1e-4, output_diff + logger.info(f'output diff of master and surrogate DNN: {output_diff}') + logger.debug(f'example output of master/surrogate: {master_dnn_output.sum(0)[0: 10]}, {surrogate_dnn_output.sum(0)[0: 10]}') + # logger.info(f'\nonly prune mlp!!!!\n') + # logger.info(f'\nonly prune mlp!!!!\n') + + if return_detail: + return boosted_vit, unpruned_indexes_of_layers + + return boosted_vit + \ No newline at end of file diff --git a/new_impl/cv/elasticdnn/model/cnn.py b/new_impl/cv/elasticdnn/model/cnn.py new file mode 100644 index 0000000000000000000000000000000000000000..c8b7fd4339921e08e715f6993d700bd8265b866f --- /dev/null +++ b/new_impl/cv/elasticdnn/model/cnn.py @@ -0,0 +1,147 @@ +from typing import Optional +import torch +from copy import deepcopy +from torch import nn +from utils.common.others import get_cur_time_str +from utils.dl.common.model import get_model_device, get_model_latency, get_model_size, get_module, get_super_module, set_module +from utils.common.log import logger +from utils.third_party.nni_new.compression.pytorch.speedup import ModelSpeedup +import os + +from .base import Abs, KTakesAll, Layer_WrappedWithFBS, ElasticDNNUtil + + +class Conv2d_WrappedWithFBS(Layer_WrappedWithFBS): + def __init__(self, raw_conv2d: nn.Conv2d, raw_bn: nn.BatchNorm2d, r): + super(Conv2d_WrappedWithFBS, self).__init__() + + self.fbs = nn.Sequential( + Abs(), + nn.AdaptiveAvgPool2d(1), + nn.Flatten(), + nn.Linear(raw_conv2d.in_channels, raw_conv2d.out_channels // r), + nn.ReLU(), + nn.Linear(raw_conv2d.out_channels // r, raw_conv2d.out_channels), + nn.ReLU() + ) + + self.raw_conv2d = raw_conv2d + self.raw_bn = raw_bn # remember clear the original BNs in the network + + nn.init.constant_(self.fbs[5].bias, 1.) + nn.init.kaiming_normal_(self.fbs[5].weight) + + def forward(self, x): + raw_x = self.raw_bn(self.raw_conv2d(x)) + + if self.use_cached_channel_attention and self.cached_channel_attention is not None: + channel_attention = self.cached_channel_attention + else: + self.cached_raw_channel_attention = self.fbs(x) + self.cached_channel_attention = self.k_takes_all(self.cached_raw_channel_attention) + + channel_attention = self.cached_channel_attention + + return raw_x * channel_attention.unsqueeze(2).unsqueeze(3) + + +class StaticFBS(nn.Module): + def __init__(self, channel_attention: torch.Tensor): + super(StaticFBS, self).__init__() + assert channel_attention.dim() == 1 + self.channel_attention = nn.Parameter(channel_attention.unsqueeze(0).unsqueeze(2).unsqueeze(3), requires_grad=False) + + def forward(self, x): + return x * self.channel_attention + + def __str__(self) -> str: + return f'StaticFBS({len(self.channel_attention.size(1))})' + + +class ElasticCNNUtil(ElasticDNNUtil): + def convert_raw_dnn_to_master_dnn(self, raw_dnn: nn.Module, r: float, ignore_layers=[]): + model = deepcopy(raw_dnn) + + # clear original BNs + num_original_bns = 0 + last_conv_name = None + conv_bn_map = {} + for name, module in model.named_modules(): + if isinstance(module, nn.Conv2d): + last_conv_name = name + if isinstance(module, nn.BatchNorm2d) and (ignore_layers is not None and last_conv_name not in ignore_layers): + num_original_bns += 1 + conv_bn_map[last_conv_name] = name + + num_conv = 0 + for name, module in model.named_modules(): + if isinstance(module, nn.Conv2d) and (ignore_layers is not None and name not in ignore_layers): + set_module(model, name, Conv2d_WrappedWithFBS(module, get_module(model, conv_bn_map[name]), r)) + num_conv += 1 + + assert num_conv == num_original_bns + + for bn_layer in conv_bn_map.values(): + set_module(model, bn_layer, nn.Identity()) + + return model + + def select_most_rep_sample(self, master_dnn: nn.Module, samples: torch.Tensor): + return samples[0].unsqueeze(0) + + def extract_surrogate_dnn_via_samples(self, master_dnn: nn.Module, samples: torch.Tensor): + sample = self.select_most_rep_sample(master_dnn, samples) + assert sample.dim() == 4 and sample.size(0) == 1 + + master_dnn.eval() + with torch.no_grad(): + master_dnn_output = master_dnn(sample) + + pruning_info = {} + pruning_masks = {} + + for layer_name, layer in master_dnn.named_modules(): + if not isinstance(layer, Conv2d_WrappedWithFBS): + continue + + cur_pruning_mask = {'weight': torch.zeros_like(layer.raw_conv2d.weight.data)} + if layer.raw_conv2d.bias is not None: + cur_pruning_mask['bias'] = torch.zeros_like(layer.raw_conv2d.bias.data) + + w = get_module(master_dnn, layer_name).cached_channel_attention.squeeze(0) + unpruned_filters_index = w.nonzero(as_tuple=True)[0] + pruning_info[layer_name] = w + + cur_pruning_mask['weight'][unpruned_filters_index, ...] = 1. + if layer.raw_conv2d.bias is not None: + cur_pruning_mask['bias'][unpruned_filters_index, ...] = 1. + pruning_masks[layer_name + '.0'] = cur_pruning_mask + + surrogate_dnn = deepcopy(master_dnn) + for name, layer in surrogate_dnn.named_modules(): + if not isinstance(layer, Conv2d_WrappedWithFBS): + continue + set_module(surrogate_dnn, name, nn.Sequential(layer.raw_conv2d, layer.raw_bn, nn.Identity())) + + # fixed_pruning_masks = fix_mask_conflict(pruning_masks, fbs_model, sample.size(), None, True, True, True) + tmp_mask_path = f'tmp_mask_{get_cur_time_str()}_{os.getpid()}.pth' + torch.save(pruning_masks, tmp_mask_path) + surrogate_dnn.eval() + model_speedup = ModelSpeedup(surrogate_dnn, sample, tmp_mask_path, sample.device) + model_speedup.speedup_model() + os.remove(tmp_mask_path) + + # add feature boosting module + for layer_name, feature_boosting_w in pruning_info.items(): + feature_boosting_w = feature_boosting_w[feature_boosting_w.nonzero(as_tuple=True)[0]] + set_module(surrogate_dnn, layer_name + '.2', StaticFBS(feature_boosting_w)) + + surrogate_dnn.eval() + with torch.no_grad(): + surrogate_dnn_output = surrogate_dnn(sample) + output_diff = ((surrogate_dnn_output - master_dnn_output) ** 2).sum() + assert output_diff < 1e-4, output_diff + logger.info(f'output diff of master and surrogate DNN: {output_diff}') + + return surrogate_dnn + \ No newline at end of file diff --git a/new_impl/cv/elasticdnn/model/test.py b/new_impl/cv/elasticdnn/model/test.py new file mode 100644 index 0000000000000000000000000000000000000000..be961a0200479a9c98a2776dd678d87ebdb47295 --- /dev/null +++ b/new_impl/cv/elasticdnn/model/test.py @@ -0,0 +1,138 @@ +import torch +from torch import nn + +from methods.elasticdnn.model.base import ElasticDNNUtil + + +def test(raw_dnn: nn.Module, ignore_layers, elastic_dnn_util: ElasticDNNUtil, input_sample: torch.Tensor, sparsity): + + # raw_dnn.eval() + # with torch.no_grad(): + # raw_dnn(input_sample) + + master_dnn = elastic_dnn_util.convert_raw_dnn_to_master_dnn_with_perf_test(raw_dnn, 16, ignore_layers) + # print(master_dnn) + # exit() + + elastic_dnn_util.set_master_dnn_sparsity(master_dnn, sparsity) + + # master_dnn.eval() + # with torch.no_grad(): + # master_dnn(input_sample) + + surrogate_dnn = elastic_dnn_util.extract_surrogate_dnn_via_samples_with_perf_test(master_dnn, input_sample) + + +if __name__ == '__main__': + from utils.dl.common.env import set_random_seed + set_random_seed(1) + + # from torchvision.models import resnet50 + # from methods.elasticdnn.model.cnn import ElasticCNNUtil + # raw_cnn = resnet50() + # prunable_layers = [] + # for i in range(1, 5): + # for j in range([3, 4, 6, 3][i - 1]): + # prunable_layers += [f'layer{i}.{j}.conv1', f'layer{i}.{j}.conv2'] + # ignore_layers = [layer for layer, m in raw_cnn.named_modules() if isinstance(m, nn.Conv2d) and layer not in prunable_layers] + # test(raw_cnn, ignore_layers, ElasticCNNUtil(), torch.rand(1, 3, 224, 224)) + ignore_layers = [] + from methods.elasticdnn.model.vit import ElasticViTUtil + # raw_vit = torch.load('tmp-master-dnn.pt') + raw_vit = torch.load('') + test(raw_vit, ignore_layers, ElasticViTUtil(), torch.rand(16, 3, 224, 224).cuda(), 0.9) + exit() + + + from dnns.vit import vit_b_16 + # from methods.elasticdnn.model.vit_new import ElasticViTUtil + from methods.elasticdnn.model.vit import ElasticViTUtil + # raw_vit = vit_b_16() + + for s in [0.8, 0.9, 0.95]: + raw_vit = vit_b_16().cuda() + + ignore_layers = [] + test(raw_vit, ignore_layers, ElasticViTUtil(), torch.rand(16, 3, 224, 224).cuda(), s) + + # for s in [0, 0.2, 0.4, 0.6, 0.8]: + # pretrained_md_models_dict_path = 'experiments/elasticdnn/vit_b_16/offline/fm_to_md/results/20230518/999999-164524-wo_FBS_trial_dsnet_lr/models/md_best.pt' + # raw_vit = torch.load(pretrained_md_models_dict_path)['main'].cuda() + + # ignore_layers = [] + # test(raw_vit, ignore_layers, ElasticViTUtil(), torch.rand(16, 3, 224, 224).cuda(), s) + # exit() + + + # weight = torch.rand((10, 5)) + # bias = torch.rand(10) + # x = torch.rand((1, 3, 5)) + + # t = torch.randperm(5) + # pruned, unpruned = t[0: 3], t[3: ] + + # mask = torch.ones_like(x) + # mask[:, :, pruned] = 0 + + # print(x, x * mask, (x * mask).sum((0, 1))) + + # import torch.nn.functional as F + # o1 = F.linear(x * mask, weight, bias) + # # print(o1) + + + # o2 = F.linear(x[:, :, unpruned], weight[:, unpruned], bias) + # # print(o2) + + # print(o1.size(), o2.size(), ((o1 - o2) ** 2).sum()) + + + + + # weight = torch.rand((130, 5)) + # bias = torch.rand(130) + # x = torch.rand((1, 3, 5)) + + # t = torch.randperm(5) + # pruned, unpruned = t[0: 3], t[3: ] + + # mask = torch.ones_like(x) + # mask[:, :, pruned] = 0 + + # print(x, x * mask, (x * mask).sum((0, 1))) + + # import torch.nn.functional as F + # o1 = F.linear(x * mask, weight, bias) + # # print(o1) + + + # o2 = F.linear(x[:, :, unpruned], weight[:, unpruned], bias) + # # print(o2) + + # print(o1.size(), o2.size(), ((o1 - o2) ** 2).sum()) + + + + + + # weight = torch.rand((1768, 768)) + # bias = torch.rand(1768) + # x = torch.rand([1, 197, 768]) + + # t = torch.randperm(768) + # unpruned, pruned = t[0: 144], t[144: ] + # unpruned = unpruned.sort()[0] + # pruned = pruned.sort()[0] + + # mask = torch.ones_like(x) + # mask[:, :, pruned] = 0 + + # print(x.sum((0, 1)).size(), (x * mask).sum((0, 1))[0: 10], x[:, :, unpruned].sum((0, 1))[0: 10]) + + # import torch.nn.functional as F + # o1 = F.linear(x * mask, weight, bias) + # o2 = F.linear(x[:, :, unpruned], weight[:, unpruned], bias) + # print(o1.sum((0, 1))[0: 10], o2.sum((0, 1))[0: 10], o1.size(), o2.size(), ((o1 - o2).abs()).sum(), ((o1 - o2) ** 2).sum()) + # unpruned_indexes = torch.randperm(5)[0: 2] + # o2 = F.linear(x[:, unpruned_indexes], weight[:, unpruned_indexes]) + # print(o2) \ No newline at end of file diff --git a/new_impl/cv/elasticdnn/model/vilt.py b/new_impl/cv/elasticdnn/model/vilt.py new file mode 100644 index 0000000000000000000000000000000000000000..f1facec6488bd3be8cabdd10c5bac9290d272167 --- /dev/null +++ b/new_impl/cv/elasticdnn/model/vilt.py @@ -0,0 +1,359 @@ +from copy import deepcopy +from typing import Optional, Union +import torch +from torch import nn +from einops import rearrange, repeat +from einops.layers.torch import Rearrange +import tqdm + +from utils.dl.common.model import get_model_device, get_model_size, set_module, get_module +from .base import Abs, KTakesAll, ElasticDNNUtil, Layer_WrappedWithFBS +from utils.common.log import logger + + +class SqueezeLast(nn.Module): + def __init__(self): + super(SqueezeLast, self).__init__() + + def forward(self, x): + return x.squeeze(-1) + + +class ProjConv_WrappedWithFBS(Layer_WrappedWithFBS): + def __init__(self, proj: nn.Conv2d, r): + super(ProjConv_WrappedWithFBS, self).__init__() + + self.proj = proj + + # for conv: (B, C_in, H, W) -> (B, C_in) -> (B, C_out) + # for mlp in ViT: (B, #patches, D: dim of patches embedding) -> (B, D) -> (B, C_out) + self.fbs = nn.Sequential( + Abs(), + nn.AdaptiveAvgPool1d(1), + SqueezeLast(), + nn.Linear(proj.in_channels, proj.out_channels // r), + nn.ReLU(), + nn.Linear(proj.out_channels // r, proj.out_channels), + nn.ReLU() + ) + + nn.init.constant_(self.fbs[6].bias, 1.) + nn.init.kaiming_normal_(self.fbs[6].weight) + + + def forward(self, x): + if self.use_cached_channel_attention and self.cached_channel_attention is not None: + channel_attention = self.cached_channel_attention + else: + self.cached_raw_channel_attention = self.fbs(x) + self.cached_channel_attention = self.k_takes_all(self.cached_raw_channel_attention) + + channel_attention = self.cached_channel_attention + + raw_res = self.proj(x) + + return channel_attention.unsqueeze(1) * raw_res # TODO: + + +class Linear_WrappedWithFBS(Layer_WrappedWithFBS): + def __init__(self, linear: nn.Linear, r): + super(Linear_WrappedWithFBS, self).__init__() + + self.linear = linear + + # for conv: (B, C_in, H, W) -> (B, C_in) -> (B, C_out) + # for mlp in ViT: (B, #patches, D: dim of patches embedding) -> (B, D) -> (B, C_out) + self.fbs = nn.Sequential( + Rearrange('b n d -> b d n'), + Abs(), + nn.AdaptiveAvgPool1d(1), + SqueezeLast(), + nn.Linear(linear.in_features, linear.out_features // r), + nn.ReLU(), + nn.Linear(linear.out_features // r, linear.out_features), + nn.ReLU() + ) + + nn.init.constant_(self.fbs[6].bias, 1.) + nn.init.kaiming_normal_(self.fbs[6].weight) + + + def forward(self, x): + if self.use_cached_channel_attention and self.cached_channel_attention is not None: + channel_attention = self.cached_channel_attention + else: + self.cached_raw_channel_attention = self.fbs(x) + self.cached_channel_attention = self.k_takes_all(self.cached_raw_channel_attention) + + channel_attention = self.cached_channel_attention + + raw_res = self.linear(x) + + return channel_attention.unsqueeze(1) * raw_res + + +class ToQKV_WrappedWithFBS(Layer_WrappedWithFBS): + """ + This regards to_q/to_k/to_v as a whole (in fact it consists of multiple heads) and prunes it. + It seems different channels of different heads are pruned according to the input. + This is different from "removing some head" or "removing the same channels in each head". + """ + def __init__(self, to_qkv: nn.Linear, r): + super(ToQKV_WrappedWithFBS, self).__init__() + + # self.to_qkv = to_qkv + + self.to_qk = nn.Linear(to_qkv.in_features, to_qkv.out_features // 3 * 2, bias=to_qkv.bias is not None) + self.to_v = nn.Linear(to_qkv.in_features, to_qkv.out_features // 3, bias=to_qkv.bias is not None) + self.to_qk.weight.data.copy_(to_qkv.weight.data[0: to_qkv.out_features // 3 * 2]) + if to_qkv.bias is not None: + self.to_qk.bias.data.copy_(to_qkv.bias.data[0: to_qkv.out_features // 3 * 2]) + self.to_v.weight.data.copy_(to_qkv.weight.data[to_qkv.out_features // 3 * 2: ]) + if to_qkv.bias is not None: + self.to_v.bias.data.copy_(to_qkv.bias.data[to_qkv.out_features // 3 * 2: ]) + + self.fbs = nn.Sequential( + Rearrange('b n d -> b d n'), + Abs(), + nn.AdaptiveAvgPool1d(1), + SqueezeLast(), + nn.Linear(to_qkv.in_features, to_qkv.out_features // 3 // r), + nn.ReLU(), + # nn.Linear(to_qkv.out_features // 3 // r, to_qkv.out_features // 3), + nn.Linear(to_qkv.out_features // 3 // r, self.to_v.out_features), + nn.ReLU() + ) + + nn.init.constant_(self.fbs[6].bias, 1.) + nn.init.kaiming_normal_(self.fbs[6].weight) + + def forward(self, x): + if self.use_cached_channel_attention and self.cached_channel_attention is not None: + channel_attention = self.cached_channel_attention + else: + self.cached_raw_channel_attention = self.fbs(x) + + # print() + # for attn in self.cached_raw_channel_attention.chunk(3, dim=1)[0: 1]: + # print(self.cached_raw_channel_attention.size(), attn.size()) + # print(self.k_takes_all.k) + # print(attn[0].nonzero(as_tuple=True)[0].size(), attn[0]) + + self.cached_channel_attention = self.k_takes_all(self.cached_raw_channel_attention) + + + # for attn in self.cached_channel_attention.chunk(3, dim=1)[0: 1]: + # print(self.cached_channel_attention.size(), attn.size()) + # print(self.k_takes_all.k) + # print(attn[0].nonzero(as_tuple=True)[0].size(), attn[0]) + # print() + + channel_attention = self.cached_channel_attention + + qk = self.to_qk(x) + v = channel_attention.unsqueeze(1) * self.to_v(x) + return torch.cat([qk, v], dim=-1) + + # qkv = raw_res.chunk(3, dim = -1) + + # raw_v = qkv[2] + # print('raw_k, raw_v', qkv[0].sum((0, 1))[0: 10], qkv[0].sum((0, 1)).nonzero(as_tuple=True)[0].size(), + # qkv[1].sum((0, 1))[0: 10], qkv[1].sum((0, 1)).nonzero(as_tuple=True)[0].size(),) + # print('raw_v', raw_v.size(), raw_v.sum((0, 1))[0: 10], raw_v.sum((0, 1)).nonzero(as_tuple=True)[0].size()) + + # qkv_attn = channel_attention.chunk(3, dim=-1) + # print('attn', [attn[0][0: 10] for attn in qkv_attn]) + # print(channel_attention.unsqueeze(1).size(), raw_res.size()) + # print('fbs', channel_attention.size(), raw_res.size()) + # return channel_attention.unsqueeze(1) * raw_res + + +class StaticFBS(nn.Module): + def __init__(self, static_channel_attention): + super(StaticFBS, self).__init__() + assert static_channel_attention.dim() == 2 and static_channel_attention.size(0) == 1 + self.static_channel_attention = nn.Parameter(static_channel_attention, requires_grad=False) # (1, dim) + + def forward(self, x): + # print('staticfbs', x, self.static_channel_attention.unsqueeze(1)) + return x * self.static_channel_attention.unsqueeze(1) + + +class ElasticViltUtil(ElasticDNNUtil): + def convert_raw_dnn_to_master_dnn(self, raw_dnn: nn.Module, r: float, ignore_layers=[]): + assert len(ignore_layers) == 0, 'not supported yet' + + raw_vit = deepcopy(raw_dnn) + + # set_module(module, 'patch_embed.proj', ProjConv_WrappedWithFBS(module.patch_embed.proj, r)) + + for name, module in raw_vit.named_modules(): + # if name.endswith('attn'): + # set_module(module, 'qkv', ToQKV_WrappedWithFBS(module.qkv, r)) + if name.endswith('intermediate'): + set_module(module, 'dense', Linear_WrappedWithFBS(module.dense, r)) + + return raw_vit + + def set_master_dnn_sparsity(self, master_dnn: nn.Module, sparsity: float): + # for name, module in master_dnn.named_modules(): + # if not name.endswith('attn'): + # continue + + # q_features = module.qkv.to_qk.out_features // 2 + + # if (q_features - int(q_features * sparsity)) % module.num_heads != 0: + # # tune sparsity to ensure #unpruned channel % num_heads == 0 + # # so that the pruning seems to reduce the dim_head of each head + # tuned_sparsity = 1. - int((q_features - int(q_features * sparsity)) / module.num_heads) * module.num_heads / q_features + # logger.debug(f'tune sparsity from {sparsity:.2f} to {tuned_sparsity}') + # sparsity = tuned_sparsity + # break + + return super().set_master_dnn_sparsity(master_dnn, sparsity) + + def select_most_rep_sample(self, master_dnn: nn.Module, samples: torch.Tensor): + # print(samples) + # return samples[0].unsqueeze(0) + res = {k: v[0: 1] for k, v in samples.items()} + return res + + def extract_surrogate_dnn_via_samples(self, master_dnn: nn.Module, samples: torch.Tensor, return_detail=False): + sample = self.select_most_rep_sample(master_dnn, samples) + # assert sample.dim() == 4 and sample.size(0) == 1 + + # print('before') + master_dnn.eval() + self.clear_cached_channel_attention_in_master_dnn(master_dnn) + with torch.no_grad(): + master_dnn_output = master_dnn(**sample) + + # print('after') + + boosted_vit = deepcopy(master_dnn) + + def get_unpruned_indexes_from_channel_attn(channel_attn: torch.Tensor, k): + assert channel_attn.size(0) == 1, 'use A representative sample to generate channel attentions' + + # print('attn_in_unpruned', channel_attn[0][0: 10]) + + res = channel_attn[0].nonzero(as_tuple=True)[0] # should be one-dim + + # res = channel_attn[0].argsort(descending=True)[0: -int(channel_attn.size(1) * k)].sort()[0] + + # g = channel_attn + # k = g.size(1) - int(g.size(1) * k) + # res = g.topk(k, 1)[1][0].sort()[0] + + return res + + unpruned_indexes_of_layers = {} + + # for attn, ff in boosted_vit.transformer.layers: + # for block_i, block in enumerate(boosted_vit.blocks): + for block_i, block in enumerate(boosted_vit.vilt.encoder.layer): + # attn = block.attn + # ff = block.mlp + + ff_0 = get_module(block, f'intermediate.dense') + # ff_0_unpruned_indexes = get_unpruned_indexes_from_channel_attn(ff_0.cached_channel_attention, k) + ff_0_pruned_indexes = ff_0.k_takes_all.cached_i[0].sort()[0] + ff_0_unpruned_indexes = torch.LongTensor([ii for ii in range(ff_0.cached_channel_attention.size(1)) if ii not in ff_0_pruned_indexes]) + new_ff_0 = nn.Linear(ff_0.linear.in_features, ff_0_unpruned_indexes.size(0), ff_0.linear.bias is not None) + new_ff_0.weight.data.copy_(ff_0.linear.weight.data[ff_0_unpruned_indexes]) + if ff_0.linear.bias is not None: + new_ff_0.bias.data.copy_(ff_0.linear.bias.data[ff_0_unpruned_indexes]) + set_module(block, 'intermediate.dense', nn.Sequential(new_ff_0, StaticFBS(ff_0.cached_channel_attention[:, ff_0_unpruned_indexes]))) + + ff_1 = get_module(block, f'output.dense') + new_ff_1 = nn.Linear(ff_0_unpruned_indexes.size(0), ff_1.out_features, ff_1.bias is not None) + new_ff_1.weight.data.copy_(ff_1.weight.data[:, ff_0_unpruned_indexes]) + if ff_1.bias is not None: + new_ff_1.bias.data.copy_(ff_1.bias.data) + set_module(block, 'output.dense', new_ff_1) + + unpruned_indexes_of_layers[f'vilt.encoder.layer.{block_i}.intermediate.dense.0.weight'] = ff_0_unpruned_indexes + + surrogate_dnn = boosted_vit + surrogate_dnn.eval() + surrogate_dnn = surrogate_dnn.to(get_model_device(master_dnn)) + # logger.debug(surrogate_dnn) + with torch.no_grad(): + surrogate_dnn_output = surrogate_dnn(**sample) + + output_diff = ((surrogate_dnn_output.logits - master_dnn_output.logits) ** 2).sum() + # assert output_diff < 1e-4, output_diff + logger.info(f'output diff of master and surrogate DNN: {output_diff}') + # logger.debug(f'example output of master/surrogate: {master_dnn_output.sum(0)[0: 10]}, {surrogate_dnn_output.sum(0)[0: 10]}') + # logger.info(f'\nonly prune mlp!!!!\n') + # logger.info(f'\nonly prune mlp!!!!\n') + + if return_detail: + return boosted_vit, unpruned_indexes_of_layers + + return boosted_vit + + def extract_surrogate_dnn_via_samples_with_perf_test(self, master_dnn: nn.Module, samples: torch.Tensor, return_detail=False): + master_dnn_size = get_model_size(master_dnn, True) + master_dnn_latency = self._get_model_latency(master_dnn, samples, 50, + get_model_device(master_dnn), 50, False) + + res = self.extract_surrogate_dnn_via_samples(master_dnn, samples, return_detail) + if not return_detail: + surrogate_dnn = res + else: + surrogate_dnn, unpruned_indexes_of_layers = res + surrogate_dnn_size = get_model_size(surrogate_dnn, True) + surrogate_dnn_latency = self._get_model_latency(master_dnn, samples, 50, + get_model_device(master_dnn), 50, False) + + logger.info(f'master DNN ({master_dnn_size:.3f}MB, {master_dnn_latency:.4f}s/sample) -> ' + f'surrogate DNN ({surrogate_dnn_size:.3f}MB, {surrogate_dnn_latency:.4f}s/sample)\n' + f'(model size: ↓ {(master_dnn_size / surrogate_dnn_size):.2f}x, ' + f'latency: ↓ {(master_dnn_latency / surrogate_dnn_latency):.2f}x)') + + return res + + def _get_model_latency(self, model: torch.nn.Module, model_input_size, sample_num: int, + device: str, warmup_sample_num: int, return_detail=False): + import time + + if isinstance(model_input_size, tuple): + dummy_input = torch.rand(model_input_size).to(device) + else: + dummy_input = model_input_size + + model = model.to(device) + model.eval() + + # warm up + with torch.no_grad(): + for _ in range(warmup_sample_num): + model(**dummy_input) + + infer_time_list = [] + + if device == 'cuda' or 'cuda' in str(device): + with torch.no_grad(): + for _ in range(sample_num): + s, e = torch.cuda.Event(enable_timing=True), torch.cuda.Event(enable_timing=True) + s.record() + model(**dummy_input) + e.record() + torch.cuda.synchronize() + cur_model_infer_time = s.elapsed_time(e) / 1000. + infer_time_list += [cur_model_infer_time] + + else: + with torch.no_grad(): + for _ in range(sample_num): + start = time.time() + model(**dummy_input) + cur_model_infer_time = time.time() - start + infer_time_list += [cur_model_infer_time] + + avg_infer_time = sum(infer_time_list) / sample_num + + if return_detail: + return avg_infer_time, infer_time_list + return avg_infer_time \ No newline at end of file diff --git a/new_impl/cv/elasticdnn/model/vit.py b/new_impl/cv/elasticdnn/model/vit.py new file mode 100644 index 0000000000000000000000000000000000000000..270f2ffcc5ca8a5f3087ed0049a9249c44b53322 --- /dev/null +++ b/new_impl/cv/elasticdnn/model/vit.py @@ -0,0 +1,385 @@ +from copy import deepcopy +from typing import Optional, Union +import torch +from torch import nn +from einops import rearrange, repeat +from einops.layers.torch import Rearrange +import tqdm + +from utils.dl.common.model import get_model_device, get_model_size, set_module,get_module +from .base import Abs, KTakesAll, ElasticDNNUtil, Layer_WrappedWithFBS +from utils.common.log import logger + + +class SqueezeLast(nn.Module): + def __init__(self): + super(SqueezeLast, self).__init__() + + def forward(self, x): + return x.squeeze(-1) + + +class ProjConv_WrappedWithFBS(Layer_WrappedWithFBS): + def __init__(self, proj: nn.Conv2d, r): + super(ProjConv_WrappedWithFBS, self).__init__() + + self.proj = proj + + # for conv: (B, C_in, H, W) -> (B, C_in) -> (B, C_out) + # for mlp in ViT: (B, #patches, D: dim of patches embedding) -> (B, D) -> (B, C_out) + self.fbs = nn.Sequential( + Abs(), + nn.AdaptiveAvgPool1d(1), + SqueezeLast(), + nn.Linear(proj.in_channels, proj.out_channels // r), + nn.ReLU(), + nn.Linear(proj.out_channels // r, proj.out_channels), + nn.ReLU() + ) + + nn.init.constant_(self.fbs[6].bias, 1.) + nn.init.kaiming_normal_(self.fbs[6].weight) + + + def forward(self, x): + if self.use_cached_channel_attention and self.cached_channel_attention is not None: + channel_attention = self.cached_channel_attention + else: + self.cached_raw_channel_attention = self.fbs(x) + self.cached_channel_attention = self.k_takes_all(self.cached_raw_channel_attention) + + channel_attention = self.cached_channel_attention + + raw_res = self.proj(x) + + return channel_attention.unsqueeze(1) * raw_res # TODO: + + +class Linear_WrappedWithFBS(Layer_WrappedWithFBS): + def __init__(self, linear: nn.Linear, r): + super(Linear_WrappedWithFBS, self).__init__() + + self.linear = linear + + # for conv: (B, C_in, H, W) -> (B, C_in) -> (B, C_out) + # for mlp in ViT: (B, #patches, D: dim of patches embedding) -> (B, D) -> (B, C_out) + self.fbs = nn.Sequential( + Rearrange('b n d -> b d n'), + Abs(), + nn.AdaptiveAvgPool1d(1), + SqueezeLast(), + nn.Linear(linear.in_features, linear.out_features // r), + nn.ReLU(), + nn.Linear(linear.out_features // r, linear.out_features), + nn.ReLU() + ) + + nn.init.constant_(self.fbs[6].bias, 1.) + nn.init.kaiming_normal_(self.fbs[6].weight) + + + def forward(self, x): + if self.use_cached_channel_attention and self.cached_channel_attention is not None: + channel_attention = self.cached_channel_attention + else: + self.cached_raw_channel_attention = self.fbs(x) + self.cached_channel_attention = self.k_takes_all(self.cached_raw_channel_attention) + + channel_attention = self.cached_channel_attention + + raw_res = self.linear(x) + + return channel_attention.unsqueeze(1) * raw_res + + +class ToQKV_WrappedWithFBS(Layer_WrappedWithFBS): + """ + This regards to_q/to_k/to_v as a whole (in fact it consists of multiple heads) and prunes it. + It seems different channels of different heads are pruned according to the input. + This is different from "removing some head" or "removing the same channels in each head". + """ + def __init__(self, to_qkv: nn.Linear, r): + super(ToQKV_WrappedWithFBS, self).__init__() + + # self.to_qkv = to_qkv + + self.to_qk = nn.Linear(to_qkv.in_features, to_qkv.out_features // 3 * 2, bias=to_qkv.bias is not None) + self.to_v = nn.Linear(to_qkv.in_features, to_qkv.out_features // 3, bias=to_qkv.bias is not None) + self.to_qk.weight.data.copy_(to_qkv.weight.data[0: to_qkv.out_features // 3 * 2]) + if to_qkv.bias is not None: + self.to_qk.bias.data.copy_(to_qkv.bias.data[0: to_qkv.out_features // 3 * 2]) + self.to_v.weight.data.copy_(to_qkv.weight.data[to_qkv.out_features // 3 * 2: ]) + if to_qkv.bias is not None: + self.to_v.bias.data.copy_(to_qkv.bias.data[to_qkv.out_features // 3 * 2: ]) + + self.fbs = nn.Sequential( + Rearrange('b n d -> b d n'), + Abs(), + nn.AdaptiveAvgPool1d(1), + SqueezeLast(), + nn.Linear(to_qkv.in_features, to_qkv.out_features // 3 // r), + nn.ReLU(), + # nn.Linear(to_qkv.out_features // 3 // r, to_qkv.out_features // 3), + nn.Linear(to_qkv.out_features // 3 // r, self.to_v.out_features), + nn.ReLU() + ) + + nn.init.constant_(self.fbs[6].bias, 1.) + nn.init.kaiming_normal_(self.fbs[6].weight) + + def forward(self, x): + if self.use_cached_channel_attention and self.cached_channel_attention is not None: + channel_attention = self.cached_channel_attention + else: + self.cached_raw_channel_attention = self.fbs(x) + + # print() + # for attn in self.cached_raw_channel_attention.chunk(3, dim=1)[0: 1]: + # print(self.cached_raw_channel_attention.size(), attn.size()) + # print(self.k_takes_all.k) + # print(attn[0].nonzero(as_tuple=True)[0].size(), attn[0]) + + self.cached_channel_attention = self.k_takes_all(self.cached_raw_channel_attention) + + + # for attn in self.cached_channel_attention.chunk(3, dim=1)[0: 1]: + # print(self.cached_channel_attention.size(), attn.size()) + # print(self.k_takes_all.k) + # print(attn[0].nonzero(as_tuple=True)[0].size(), attn[0]) + # print() + + channel_attention = self.cached_channel_attention + + qk = self.to_qk(x) + v = channel_attention.unsqueeze(1) * self.to_v(x) + return torch.cat([qk, v], dim=-1) + + # qkv = raw_res.chunk(3, dim = -1) + + # raw_v = qkv[2] + # print('raw_k, raw_v', qkv[0].sum((0, 1))[0: 10], qkv[0].sum((0, 1)).nonzero(as_tuple=True)[0].size(), + # qkv[1].sum((0, 1))[0: 10], qkv[1].sum((0, 1)).nonzero(as_tuple=True)[0].size(),) + # print('raw_v', raw_v.size(), raw_v.sum((0, 1))[0: 10], raw_v.sum((0, 1)).nonzero(as_tuple=True)[0].size()) + + # qkv_attn = channel_attention.chunk(3, dim=-1) + # print('attn', [attn[0][0: 10] for attn in qkv_attn]) + # print(channel_attention.unsqueeze(1).size(), raw_res.size()) + # print('fbs', channel_attention.size(), raw_res.size()) + # return channel_attention.unsqueeze(1) * raw_res + + +class StaticFBS(nn.Module): + def __init__(self, static_channel_attention): + super(StaticFBS, self).__init__() + assert static_channel_attention.dim() == 2 and static_channel_attention.size(0) == 1 + self.static_channel_attention = nn.Parameter(static_channel_attention, requires_grad=False) # (1, dim) + + def forward(self, x): + # print('staticfbs', x, self.static_channel_attention.unsqueeze(1)) + return x * self.static_channel_attention.unsqueeze(1) + + +class ElasticViTUtil(ElasticDNNUtil): + def convert_raw_dnn_to_master_dnn(self, raw_dnn: nn.Module, r: float, ignore_layers=[]): + assert len(ignore_layers) == 0, 'not supported yet' + + raw_vit = deepcopy(raw_dnn) + + # set_module(module, 'patch_embed.proj', ProjConv_WrappedWithFBS(module.patch_embed.proj, r)) + + for name, module in raw_vit.named_modules(): + # if name.endswith('attn'): + # set_module(module, 'qkv', ToQKV_WrappedWithFBS(module.qkv, r)) + if name.endswith('intermediate'): + set_module(module, 'dense', Linear_WrappedWithFBS(module.dense, r)) + + return raw_vit + + def set_master_dnn_sparsity(self, master_dnn: nn.Module, sparsity: float): + # for name, module in master_dnn.named_modules(): + # if not name.endswith('attn'): + # continue + + # q_features = module.qkv.to_qk.out_features // 2 + + # if (q_features - int(q_features * sparsity)) % module.num_heads != 0: + # # tune sparsity to ensure #unpruned channel % num_heads == 0 + # # so that the pruning seems to reduce the dim_head of each head + # tuned_sparsity = 1. - int((q_features - int(q_features * sparsity)) / module.num_heads) * module.num_heads / q_features + # logger.debug(f'tune sparsity from {sparsity:.2f} to {tuned_sparsity}') + # sparsity = tuned_sparsity + # break + + return super().set_master_dnn_sparsity(master_dnn, sparsity) + + def select_most_rep_sample(self, master_dnn: nn.Module, samples: torch.Tensor): + return samples[0].unsqueeze(0) + + def extract_surrogate_dnn_via_samples(self, master_dnn: nn.Module, samples: torch.Tensor, return_detail=False): + sample = self.select_most_rep_sample(master_dnn, samples) + assert sample.dim() == 4 and sample.size(0) == 1 + + # print('before') + master_dnn.eval() + self.clear_cached_channel_attention_in_master_dnn(master_dnn) + with torch.no_grad(): + master_dnn_output = master_dnn(sample) + + # print('after') + + boosted_vit = deepcopy(master_dnn) + + def get_unpruned_indexes_from_channel_attn(channel_attn: torch.Tensor, k): + assert channel_attn.size(0) == 1, 'use A representative sample to generate channel attentions' + + # print('attn_in_unpruned', channel_attn[0][0: 10]) + + res = channel_attn[0].nonzero(as_tuple=True)[0] # should be one-dim + + # res = channel_attn[0].argsort(descending=True)[0: -int(channel_attn.size(1) * k)].sort()[0] + + # g = channel_attn + # k = g.size(1) - int(g.size(1) * k) + # res = g.topk(k, 1)[1][0].sort()[0] + + return res + + unpruned_indexes_of_layers = {} + + # for attn, ff in boosted_vit.transformer.layers: + for block_i_1, block_1 in enumerate(boosted_vit.cvt.encoder.stages): + for block_i, block in enumerate(block_1.layers): + # # attn1 = block.attention.attention.projection_query + # # attn2 = block.attention.attention.projection_key + # # attn3 = block.attention.attention.projection_value + # ff = block.intermediate + # ff1 = block.output + # ff_0 = ff.dense + # # ff_0_unpruned_indexes = get_unpruned_indexes_from_channel_attn(ff_0.cached_channel_attention, k) + # ff_0_pruned_indexes = ff_0.k_takes_all.cached_i[0].sort()[0] + # ff_0_unpruned_indexes = torch.LongTensor([ii for ii in range(ff_0.cached_channel_attention.size(1)) if ii not in ff_0_pruned_indexes]) + # new_ff_0 = nn.Linear(ff_0.linear.in_features, ff_0_unpruned_indexes.size(0), ff_0.linear.bias is not None) + # new_ff_0.weight.data.copy_(ff_0.linear.weight.data[ff_0_unpruned_indexes]) + # if ff_0.linear.bias is not None: + # new_ff_0.bias.data.copy_(ff_0.linear.bias.data[ff_0_unpruned_indexes]) + # set_module(ff, 'dense', nn.Sequential(new_ff_0, StaticFBS(ff_0.cached_channel_attention[:, ff_0_unpruned_indexes]))) + + + # ff_1 = ff1.dense + # new_ff_1 = nn.Linear(ff_0_unpruned_indexes.size(0), ff_1.out_features, ff_1.bias is not None) + # new_ff_1.weight.data.copy_(ff_1.weight.data[:, ff_0_unpruned_indexes]) + # if ff_1.bias is not None: + # new_ff_1.bias.data.copy_(ff_1.bias.data) + # set_module(ff1, 'dense', new_ff_1) + + ff_0 = get_module(block, f'intermediate.dense') + # ff_0_unpruned_indexes = get_unpruned_indexes_from_channel_attn(ff_0.cached_channel_attention, k) + ff_0_pruned_indexes = ff_0.k_takes_all.cached_i[0].sort()[0] + ff_0_unpruned_indexes = torch.LongTensor([ii for ii in range(ff_0.cached_channel_attention.size(1)) if ii not in ff_0_pruned_indexes]) + new_ff_0 = nn.Linear(ff_0.linear.in_features, ff_0_unpruned_indexes.size(0), ff_0.linear.bias is not None) + new_ff_0.weight.data.copy_(ff_0.linear.weight.data[ff_0_unpruned_indexes]) + if ff_0.linear.bias is not None: + new_ff_0.bias.data.copy_(ff_0.linear.bias.data[ff_0_unpruned_indexes]) + set_module(block, 'intermediate.dense', nn.Sequential(new_ff_0, StaticFBS(ff_0.cached_channel_attention[:, ff_0_unpruned_indexes]))) + + ff_1 = get_module(block, f'output.dense') + new_ff_1 = nn.Linear(ff_0_unpruned_indexes.size(0), ff_1.out_features, ff_1.bias is not None) + new_ff_1.weight.data.copy_(ff_1.weight.data[:, ff_0_unpruned_indexes]) + if ff_1.bias is not None: + new_ff_1.bias.data.copy_(ff_1.bias.data) + set_module(block, 'output.dense', new_ff_1) + unpruned_indexes_of_layers[f'cvt.encoder.stages.{block_i_1}.layers.{block_i}.intermediate.dense.0.weight'] = ff_0_unpruned_indexes + # for block_i, block in enumerate(boosted_vit.blocks): + # attn = block.attn + # ff = block.mlp + + # # prune to_qkv + # # to_qkv = attn.qkv + # # # cached_i = to_qkv.k_takes_all.cached_i + # # k = to_qkv.k_takes_all.k + # # # to_q_unpruned_indexes = get_unpruned_indexes_from_channel_attn( + # # # to_qkv.cached_channel_attention[:, 0: to_qkv.cached_channel_attention.size(1) // 3], k + # # # ) + # # # to_q_unpruned_indexes_w_offset = to_q_unpruned_indexes + # # # to_k_unpruned_indexes = get_unpruned_indexes_from_channel_attn( + # # # to_qkv.cached_channel_attention[:, to_qkv.cached_channel_attention.size(1) // 3: to_qkv.cached_channel_attention.size(1) // 3 * 2], k + # # # ) + # # # to_k_unpruned_indexes_w_offset = to_k_unpruned_indexes + to_qkv.cached_channel_attention.size(1) // 3 + # # # to_v_unpruned_indexes = get_unpruned_indexes_from_channel_attn( + # # # to_qkv.cached_channel_attention, k + # # # ) + # # to_v_pruned_indexes = to_qkv.k_takes_all.cached_i[0].sort()[0] + # # # print(to_v_pruned_indexes.size(), to_qkv.cached_channel_attention.size()) + # # to_v_unpruned_indexes = torch.LongTensor([ii for ii in range(to_qkv.cached_channel_attention.size(1)) if ii not in to_v_pruned_indexes]) + # # # print(to_v_unpruned_indexes.size()) + # # # exit() + # # # to_q_unpruned_indexes = to_qkv.k_takes_all.cached_i[0] + # # # to_q_unpruned_indexes_w_offset = to_q_unpruned_indexes + # # # to_k_unpruned_indexes = to_qkv.k_takes_all.cached_i[1] + # # # to_k_unpruned_indexes_w_offset = to_k_unpruned_indexes + to_qkv.cached_channel_attention.size(1) // 3 + # # # to_v_unpruned_indexes = to_qkv.k_takes_all.cached_i[2] + # # # to_v_unpruned_indexes_w_offset = to_v_unpruned_indexes + to_qkv.cached_channel_attention.size(1) // 3 * 2 + # # # assert to_q_unpruned_indexes_w_offset.size(0) == to_k_unpruned_indexes_w_offset.size(0) == to_v_unpruned_indexes_w_offset.size(0), \ + # # # f'{to_q_unpruned_indexes_w_offset.size(0)}, {to_k_unpruned_indexes_w_offset.size(0)}, {to_v_unpruned_indexes_w_offset.size(0)}' + # # # print('unpruned indexes', to_q_unpruned_indexes[0: 10], to_k_unpruned_indexes[0: 10], to_v_unpruned_indexes[0: 10]) + # # # exit() + # # # print(to_q_unpruned_indexes_w_offset, to_k_unpruned_indexes_w_offset, to_v_unpruned_indexes_w_offset, to_v_unpruned_indexes) + # # # to_qkv_unpruned_indexes = torch.cat([to_q_unpruned_indexes_w_offset, to_k_unpruned_indexes_w_offset, to_v_unpruned_indexes_w_offset]) + # # new_to_qkv = nn.Linear(to_qkv.to_v.in_features, to_qkv.to_v.out_features * 2 + to_v_unpruned_indexes.size(0), to_qkv.to_v.bias is not None) + # # new_to_qkv.weight.data.copy_(torch.cat([to_qkv.to_qk.weight.data, to_qkv.to_v.weight.data[to_v_unpruned_indexes]])) + # # if to_qkv.to_qk.bias is not None: + # # new_to_qkv.bias.data.copy_(torch.cat([to_qkv.to_qk.bias.data, to_qkv.to_v.bias.data[to_v_unpruned_indexes]])) + # # set_module(attn, 'qkv', nn.Sequential(new_to_qkv, StaticFBS(torch.cat([ + # # torch.ones_like(to_qkv.cached_channel_attention), + # # torch.ones_like(to_qkv.cached_channel_attention), + # # to_qkv.cached_channel_attention[:, to_v_unpruned_indexes] + # # ], dim=1)))) + + # # # prune to_out + # # # print('to_v_unpruned_indexes', to_v_unpruned_indexes) + # # to_out = attn.proj + # # new_to_out = nn.Linear(to_v_unpruned_indexes.size(0), to_out.out_features, to_out.bias is not None) + # # new_to_out.weight.data.copy_(to_out.weight.data[:, to_v_unpruned_indexes]) + # # if to_out.bias is not None: + # # new_to_out.bias.data.copy_(to_out.bias.data) + # # # print('to_out copy bias') + # # set_module(attn, 'proj', new_to_out) + + # ff_0 = ff.fc1 + # # ff_0_unpruned_indexes = get_unpruned_indexes_from_channel_attn(ff_0.cached_channel_attention, k) + # ff_0_pruned_indexes = ff_0.k_takes_all.cached_i[0].sort()[0] + # ff_0_unpruned_indexes = torch.LongTensor([ii for ii in range(ff_0.cached_channel_attention.size(1)) if ii not in ff_0_pruned_indexes]) + # new_ff_0 = nn.Linear(ff_0.linear.in_features, ff_0_unpruned_indexes.size(0), ff_0.linear.bias is not None) + # new_ff_0.weight.data.copy_(ff_0.linear.weight.data[ff_0_unpruned_indexes]) + # if ff_0.linear.bias is not None: + # new_ff_0.bias.data.copy_(ff_0.linear.bias.data[ff_0_unpruned_indexes]) + # set_module(ff, 'fc1', nn.Sequential(new_ff_0, StaticFBS(ff_0.cached_channel_attention[:, ff_0_unpruned_indexes]))) + + # ff_1 = ff.fc2 + # new_ff_1 = nn.Linear(ff_0_unpruned_indexes.size(0), ff_1.out_features, ff_1.bias is not None) + # new_ff_1.weight.data.copy_(ff_1.weight.data[:, ff_0_unpruned_indexes]) + # if ff_1.bias is not None: + # new_ff_1.bias.data.copy_(ff_1.bias.data) + # set_module(ff, 'fc2', new_ff_1) + + # unpruned_indexes_of_layers[f'blocks.{block_i}.mlp.fc1.0.weight'] = ff_0_unpruned_indexes + + surrogate_dnn = boosted_vit + surrogate_dnn.eval() + surrogate_dnn = surrogate_dnn.to(get_model_device(master_dnn)) + # logger.debug(surrogate_dnn) + with torch.no_grad(): + surrogate_dnn_output = surrogate_dnn(sample) + + output_diff = ((surrogate_dnn_output.logits - master_dnn_output.logits) ** 2).sum() + # assert output_diff < 1e-4, output_diff + logger.info(f'output diff of master and surrogate DNN: {output_diff}') + logger.debug(f'example output of master/surrogate: {master_dnn_output.logits.sum(0)[0: 10]}, {surrogate_dnn_output.logits.sum(0)[0: 10]}') + # logger.info(f'\nonly prune mlp!!!!\n') + # logger.info(f'\nonly prune mlp!!!!\n') + + if return_detail: + return boosted_vit, unpruned_indexes_of_layers + + return boosted_vit + \ No newline at end of file diff --git a/new_impl/cv/elasticdnn/model/vit_new.py b/new_impl/cv/elasticdnn/model/vit_new.py new file mode 100644 index 0000000000000000000000000000000000000000..3ff7bdfc3bcb9bb3e2c7c96b0385d821dfa6380e --- /dev/null +++ b/new_impl/cv/elasticdnn/model/vit_new.py @@ -0,0 +1,341 @@ +from copy import deepcopy +from typing import Optional, Union +import torch +from torch import nn +from einops import rearrange, repeat +from einops.layers.torch import Rearrange +import tqdm + +from utils.dl.common.model import LayerActivation, get_model_device, get_model_size, set_module +from .base import Abs, KTakesAll, ElasticDNNUtil, Layer_WrappedWithFBS +from utils.common.log import logger + + +class SqueezeLast(nn.Module): + def __init__(self): + super(SqueezeLast, self).__init__() + + def forward(self, x): + return x.squeeze(-1) + + +class ProjConv_WrappedWithFBS(Layer_WrappedWithFBS): + def __init__(self, raw_conv2d: nn.Conv2d, r): + super(ProjConv_WrappedWithFBS, self).__init__() + + self.fbs = nn.Sequential( + Abs(), + nn.AdaptiveAvgPool2d(1), + nn.Flatten(), + nn.Linear(raw_conv2d.in_channels, raw_conv2d.out_channels // r), + nn.ReLU(), + nn.Linear(raw_conv2d.out_channels // r, raw_conv2d.out_channels), + nn.ReLU() + ) + + self.raw_conv2d = raw_conv2d + # self.raw_bn = raw_bn # remember clear the original BNs in the network + + nn.init.constant_(self.fbs[5].bias, 1.) + nn.init.kaiming_normal_(self.fbs[5].weight) + + def forward(self, x): + raw_x = self.raw_conv2d(x) + + if self.use_cached_channel_attention and self.cached_channel_attention is not None: + channel_attention = self.cached_channel_attention + else: + self.cached_raw_channel_attention = self.fbs(x) + self.cached_channel_attention = self.k_takes_all(self.cached_raw_channel_attention) + + channel_attention = self.cached_channel_attention + + return raw_x * channel_attention.unsqueeze(2).unsqueeze(3) + + +class Linear_WrappedWithFBS(Layer_WrappedWithFBS): + def __init__(self, linear: nn.Linear, r): + super(Linear_WrappedWithFBS, self).__init__() + + self.linear = linear + + # for conv: (B, C_in, H, W) -> (B, C_in) -> (B, C_out) + # for mlp in ViT: (B, #patches, D: dim of patches embedding) -> (B, D) -> (B, C_out) + self.fbs = nn.Sequential( + Rearrange('b n d -> b d n'), + Abs(), + nn.AdaptiveAvgPool1d(1), + SqueezeLast(), + nn.Linear(linear.in_features, linear.out_features // r), + nn.ReLU(), + nn.Linear(linear.out_features // r, linear.out_features), + nn.ReLU() + ) + + nn.init.constant_(self.fbs[6].bias, 1.) + nn.init.kaiming_normal_(self.fbs[6].weight) + + + def forward(self, x): + if self.use_cached_channel_attention and self.cached_channel_attention is not None: + channel_attention = self.cached_channel_attention + else: + self.cached_raw_channel_attention = self.fbs(x) + self.cached_channel_attention = self.k_takes_all(self.cached_raw_channel_attention) + + channel_attention = self.cached_channel_attention + + raw_res = self.linear(x) + + return channel_attention.unsqueeze(1) * raw_res + + +# class ToQKV_WrappedWithFBS(Layer_WrappedWithFBS): +# """ +# This regards to_q/to_k/to_v as a whole (in fact it consists of multiple heads) and prunes it. +# It seems different channels of different heads are pruned according to the input. +# This is different from "removing some head" or "removing the same channels in each head". +# """ +# def __init__(self, to_qkv: nn.Linear, r): +# super(ToQKV_WrappedWithFBS, self).__init__() + +# # self.to_qkv = to_qkv + +# self.to_qk = nn.Linear(to_qkv.in_features, to_qkv.out_features // 3 * 2, bias=to_qkv.bias is not None) +# self.to_v = nn.Linear(to_qkv.in_features, to_qkv.out_features // 3, bias=to_qkv.bias is not None) +# self.to_qk.weight.data.copy_(to_qkv.weight.data[0: to_qkv.out_features // 3 * 2]) +# if to_qkv.bias is not None: +# self.to_qk.bias.data.copy_(to_qkv.bias.data[0: to_qkv.out_features // 3 * 2]) +# self.to_v.weight.data.copy_(to_qkv.weight.data[to_qkv.out_features // 3 * 2: ]) +# if to_qkv.bias is not None: +# self.to_v.bias.data.copy_(to_qkv.bias.data[to_qkv.out_features // 3 * 2: ]) + +# self.fbs = nn.Sequential( +# Rearrange('b n d -> b d n'), +# Abs(), +# nn.AdaptiveAvgPool1d(1), +# SqueezeLast(), +# nn.Linear(to_qkv.in_features, to_qkv.out_features // 3 // r), +# nn.ReLU(), +# # nn.Linear(to_qkv.out_features // 3 // r, to_qkv.out_features // 3), +# nn.Linear(to_qkv.out_features // 3 // r, self.to_v.out_features), +# nn.ReLU() +# ) + +# nn.init.constant_(self.fbs[6].bias, 1.) +# nn.init.kaiming_normal_(self.fbs[6].weight) + +# def forward(self, x): +# if self.use_cached_channel_attention and self.cached_channel_attention is not None: +# channel_attention = self.cached_channel_attention +# else: +# self.cached_raw_channel_attention = self.fbs(x) + +# # print() +# # for attn in self.cached_raw_channel_attention.chunk(3, dim=1)[0: 1]: +# # print(self.cached_raw_channel_attention.size(), attn.size()) +# # print(self.k_takes_all.k) +# # print(attn[0].nonzero(as_tuple=True)[0].size(), attn[0]) + +# self.cached_channel_attention = self.k_takes_all(self.cached_raw_channel_attention) + + +# # for attn in self.cached_channel_attention.chunk(3, dim=1)[0: 1]: +# # print(self.cached_channel_attention.size(), attn.size()) +# # print(self.k_takes_all.k) +# # print(attn[0].nonzero(as_tuple=True)[0].size(), attn[0]) +# # print() + +# channel_attention = self.cached_channel_attention + +# qk = self.to_qk(x) +# v = channel_attention.unsqueeze(1) * self.to_v(x) +# return torch.cat([qk, v], dim=-1) + + # qkv = raw_res.chunk(3, dim = -1) + + # raw_v = qkv[2] + # print('raw_k, raw_v', qkv[0].sum((0, 1))[0: 10], qkv[0].sum((0, 1)).nonzero(as_tuple=True)[0].size(), + # qkv[1].sum((0, 1))[0: 10], qkv[1].sum((0, 1)).nonzero(as_tuple=True)[0].size(),) + # print('raw_v', raw_v.size(), raw_v.sum((0, 1))[0: 10], raw_v.sum((0, 1)).nonzero(as_tuple=True)[0].size()) + + # qkv_attn = channel_attention.chunk(3, dim=-1) + # print('attn', [attn[0][0: 10] for attn in qkv_attn]) + # print(channel_attention.unsqueeze(1).size(), raw_res.size()) + # print('fbs', channel_attention.size(), raw_res.size()) + # return channel_attention.unsqueeze(1) * raw_res + + +class LinearStaticFBS(nn.Module): + def __init__(self, static_channel_attention): + super(LinearStaticFBS, self).__init__() + assert static_channel_attention.dim() == 2 and static_channel_attention.size(0) == 1 + self.static_channel_attention = nn.Parameter(static_channel_attention, requires_grad=False) # (1, dim) + + def forward(self, x): + # print('staticfbs', x, self.static_channel_attention.unsqueeze(1)) + return x * self.static_channel_attention.unsqueeze(1) + +from .cnn import StaticFBS as ConvStaticFBS + + +class ElasticViTUtil(ElasticDNNUtil): + def convert_raw_dnn_to_master_dnn(self, raw_dnn: nn.Module, r: float, ignore_layers=[]): + assert len(ignore_layers) == 0, 'not supported yet' + + raw_vit = deepcopy(raw_dnn) + + set_module(raw_vit, 'patch_embed.proj', ProjConv_WrappedWithFBS(raw_vit.patch_embed.proj, r)) + + for name, module in raw_vit.named_modules(): + if name.endswith('mlp'): + set_module(module, 'fc1', Linear_WrappedWithFBS(module.fc1, r)) + + return raw_vit + + # def set_master_dnn_sparsity(self, master_dnn: nn.Module, sparsity: float): + # for name, module in master_dnn.named_modules(): + # if not name.endswith('attn'): + # continue + + # q_features = module.qkv.to_qk.out_features // 2 + + # if (q_features - int(q_features * sparsity)) % module.num_heads != 0: + # # tune sparsity to ensure #unpruned channel % num_heads == 0 + # # so that the pruning seems to reduce the dim_head of each head + # tuned_sparsity = 1. - int((q_features - int(q_features * sparsity)) / module.num_heads) * module.num_heads / q_features + # logger.debug(f'tune sparsity from {sparsity:.2f} to {tuned_sparsity}') + # sparsity = tuned_sparsity + # break + + # return super().set_master_dnn_sparsity(master_dnn, sparsity) + + def select_most_rep_sample(self, master_dnn: nn.Module, samples: torch.Tensor): + return samples[0].unsqueeze(0) + + def extract_surrogate_dnn_via_samples(self, master_dnn: nn.Module, samples: torch.Tensor): + sample = self.select_most_rep_sample(master_dnn, samples) + assert sample.dim() == 4 and sample.size(0) == 1 + + + print('WARN: for debug, modify cls_token and pos_embed') + master_dnn.pos_embed.data = torch.zeros_like(master_dnn.pos_embed.data) + + print('before') + master_dnn.eval() + self.clear_cached_channel_attention_in_master_dnn(master_dnn) + + # debug: add hooks + + hooks = { + 'blocks_input': LayerActivation(master_dnn.blocks, True, 'cuda') + } + + with torch.no_grad(): + master_dnn_output = master_dnn(sample) + + for k, v in hooks.items(): + print(f'{k}: {v.input.size()}') + + print('after') + + boosted_vit = master_dnn + + def get_unpruned_indexes_from_channel_attn(channel_attn: torch.Tensor, k): + assert channel_attn.size(0) == 1, 'use A representative sample to generate channel attentions' + res = channel_attn[0].nonzero(as_tuple=True)[0] + return res + + proj = boosted_vit.patch_embed.proj + proj_unpruned_indexes = get_unpruned_indexes_from_channel_attn( + proj.cached_channel_attention, proj.k_takes_all.k) + + # 1.1 prune proj itself + proj_conv = proj.raw_conv2d + new_proj = nn.Conv2d(proj_conv.in_channels, proj_unpruned_indexes.size(0), proj_conv.kernel_size, proj_conv.stride, proj_conv.padding, + proj_conv.dilation, proj_conv.groups, proj_conv.bias is not None, proj_conv.padding_mode, proj_conv.weight.device) + new_proj.weight.data.copy_(proj_conv.weight.data[proj_unpruned_indexes]) + if new_proj.bias is not None: + new_proj.bias.data.copy_(proj_conv.bias.data[proj_unpruned_indexes]) + set_module(boosted_vit.patch_embed, 'proj', nn.Sequential(new_proj, ConvStaticFBS(proj.cached_channel_attention[0][proj_unpruned_indexes]))) + + # print(boosted_vit.pos_embed.size(), boosted_vit.cls_token.size()) + boosted_vit.pos_embed.data = boosted_vit.pos_embed.data[:, :, proj_unpruned_indexes] + boosted_vit.cls_token.data = boosted_vit.cls_token.data[:, :, proj_unpruned_indexes] + + def reduce_linear_output(raw_linear: nn.Linear, layer_name, unpruned_indexes: torch.Tensor): + new_linear = nn.Linear(raw_linear.in_features, unpruned_indexes.size(0), raw_linear.bias is not None) + new_linear.weight.data.copy_(raw_linear.weight.data[unpruned_indexes]) + if raw_linear.bias is not None: + new_linear.bias.data.copy_(raw_linear.bias.data[unpruned_indexes]) + set_module(boosted_vit, layer_name, new_linear) + + def reduce_linear_input(raw_linear: nn.Linear, layer_name, unpruned_indexes: torch.Tensor): + new_linear = nn.Linear(unpruned_indexes.size(0), raw_linear.out_features, raw_linear.bias is not None) + new_linear.weight.data.copy_(raw_linear.weight.data[:, unpruned_indexes]) + if raw_linear.bias is not None: + new_linear.bias.data.copy_(raw_linear.bias.data) + set_module(boosted_vit, layer_name, new_linear) + + def reduce_norm(raw_norm: nn.LayerNorm, layer_name, unpruned_indexes: torch.Tensor): + new_norm = nn.LayerNorm(unpruned_indexes.size(0), raw_norm.eps, raw_norm.elementwise_affine) + new_norm.weight.data.copy_(raw_norm.weight.data[unpruned_indexes]) + new_norm.bias.data.copy_(raw_norm.bias.data[unpruned_indexes]) + set_module(boosted_vit, layer_name, new_norm) + + # 1.2 prune blocks.x.norm1/to_qkv/proj/fc1/fc2 + for block_i, block in enumerate(boosted_vit.blocks): + attn = block.attn + ff = block.mlp + + reduce_norm(block.norm1, f'blocks.{block_i}.norm1', proj_unpruned_indexes) + reduce_linear_input(attn.qkv, f'blocks.{block_i}.attn.qkv', proj_unpruned_indexes) + reduce_linear_output(attn.proj, f'blocks.{block_i}.attn.proj', proj_unpruned_indexes) + reduce_norm(block.norm2, f'blocks.{block_i}.norm2', proj_unpruned_indexes) + reduce_linear_input(ff.fc1.linear, f'blocks.{block_i}.mlp.fc1.linear', proj_unpruned_indexes) + reduce_linear_output(ff.fc2, f'blocks.{block_i}.mlp.fc2', proj_unpruned_indexes) + + # 1.3 prune norm, head + reduce_norm(boosted_vit.norm, f'norm', proj_unpruned_indexes) + reduce_linear_input(boosted_vit.head, f'head', proj_unpruned_indexes) + + # 2. prune blocks.x.fc1 + for block_i, block in enumerate(boosted_vit.blocks): + attn = block.attn + ff = block.mlp + + fc1 = ff.fc1 + fc1_unpruned_indexes = get_unpruned_indexes_from_channel_attn(fc1.cached_channel_attention, fc1.k_takes_all.k) + fc1_linear = fc1.linear + new_linear = nn.Linear(fc1_linear.in_features, fc1_unpruned_indexes.size(0), fc1_linear.bias is not None) + new_linear.weight.data.copy_(fc1_linear.weight.data[fc1_unpruned_indexes]) + if fc1_linear.bias is not None: + new_linear.bias.data.copy_(fc1_linear.bias.data[fc1_unpruned_indexes]) + set_module(boosted_vit, f'blocks.{block_i}.mlp.fc1', nn.Sequential(new_linear, LinearStaticFBS(fc1.cached_channel_attention[:, fc1_unpruned_indexes]))) + + reduce_linear_input(ff.fc2, f'blocks.{block_i}.mlp.fc2', fc1_unpruned_indexes) + + + + surrogate_dnn = boosted_vit + surrogate_dnn.eval() + surrogate_dnn = surrogate_dnn.to(get_model_device(master_dnn)) + print(surrogate_dnn) + + + hooks = { + 'blocks_input': LayerActivation(surrogate_dnn.blocks, True, 'cuda') + } + + with torch.no_grad(): + surrogate_dnn_output = surrogate_dnn(sample) + + for k, v in hooks.items(): + print(f'{k}: {v.input.size()}') + + output_diff = ((surrogate_dnn_output - master_dnn_output) ** 2).sum() + assert output_diff < 1e-4, output_diff + logger.info(f'output diff of master and surrogate DNN: {output_diff}') + + return boosted_vit + \ No newline at end of file diff --git a/new_impl/cv/elasticdnn/pipeline/offline/fm_lora/__pycache__/base.cpython-38.pyc b/new_impl/cv/elasticdnn/pipeline/offline/fm_lora/__pycache__/base.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b6f158839f434f321635a8e383e9658ac41f8671 Binary files /dev/null and b/new_impl/cv/elasticdnn/pipeline/offline/fm_lora/__pycache__/base.cpython-38.pyc differ diff --git a/new_impl/cv/elasticdnn/pipeline/offline/fm_lora/__pycache__/vit.cpython-38.pyc b/new_impl/cv/elasticdnn/pipeline/offline/fm_lora/__pycache__/vit.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..096ad38bad17a2bb4f4ab0f3c61f5feebbe71eb3 Binary files /dev/null and b/new_impl/cv/elasticdnn/pipeline/offline/fm_lora/__pycache__/vit.cpython-38.pyc differ diff --git a/new_impl/cv/elasticdnn/pipeline/offline/fm_lora/base.py b/new_impl/cv/elasticdnn/pipeline/offline/fm_lora/base.py new file mode 100644 index 0000000000000000000000000000000000000000..6dbbd9ebdfc784f8d195b19d982dfee2db5ea571 --- /dev/null +++ b/new_impl/cv/elasticdnn/pipeline/offline/fm_lora/base.py @@ -0,0 +1,36 @@ +import torch +from torch import nn +from abc import ABC, abstractmethod + +from utils.dl.common.model import get_model_device, get_model_latency, get_model_size +from utils.common.log import logger + + +class LoRA(nn.Linear): + pass + + +class FMLoRA_Util(ABC): + @abstractmethod + def add_lora_ab_to_fm(self, fm: nn.Module, ab_r: int, samples: torch.Tensor): + """ + only applying LoRA to attention weights. + """ + pass + + def train_only_lora(self, fm: nn.Module): + res = [] + for n, m in fm.named_modules(): + if isinstance(m, LoRA): + for p in m.parameters(): + p.requires_grad = True + res += [p] + else: + for p in m.parameters(): + p.requires_grad = False + return res + + @abstractmethod + def absorb_lora_and_recover_net_structure(self, fm: nn.Module): + pass + \ No newline at end of file diff --git a/new_impl/cv/elasticdnn/pipeline/offline/fm_lora/bert.py b/new_impl/cv/elasticdnn/pipeline/offline/fm_lora/bert.py new file mode 100644 index 0000000000000000000000000000000000000000..a54d802afce5f2edeaf741d78a9c504232adcd8c --- /dev/null +++ b/new_impl/cv/elasticdnn/pipeline/offline/fm_lora/bert.py @@ -0,0 +1,81 @@ +import torch +from torch import nn +from abc import ABC, abstractmethod + +from utils.dl.common.model import get_model_device, get_model_latency, get_model_size, set_module +from utils.common.log import logger +from .base import FMLoRA_Util, LoRA + + +class ToQKV_WrappedWithLoRA(nn.Module): + def __init__(self, fc: nn.Linear, ab_r: int): + super(ToQKV_WrappedWithLoRA, self).__init__() + + self.fc = fc + self.ab = self.create_ab_as_linear(fc.weight.data, ab_r) + + def create_ab_as_linear(self, fc_weight: torch.Tensor, ab_r: int): + res = nn.Sequential( + LoRA(fc_weight.size(1), fc_weight.size(0) // ab_r, bias=False), + LoRA(fc_weight.size(0) // ab_r, fc_weight.size(0), bias=False) + ).to(fc_weight.device) + nn.init.kaiming_uniform_(res[0].weight, a=5 ** 0.5) + nn.init.zeros_(res[1].weight) + return res + + def forward(self, x): + x1 = self.fc(x) + x2 = self.ab(x) + return x1 + x2 + + +class FMLoRA_Bert_Util(FMLoRA_Util): + + @torch.no_grad() + def add_lora_ab_to_fm(self, fm: nn.Module, ab_r: int, samples: dict): + fm.eval() + + o1 = fm(**samples) + + for name, module in fm.named_modules(): + if name.endswith(('query', 'key', 'value')): + set_module(fm, name, ToQKV_WrappedWithLoRA(module, ab_r)) + + o2 = fm(**samples) + + if isinstance(o1, tuple): + o1 = o1[-1] + o2 = o2[-1] + output_diff = ((o1 - o2) ** 2).sum() + assert output_diff < 1e-5 + + return fm + + @torch.no_grad() + def absorb_lora_and_recover_net_structure(self, fm: nn.Module, samples: dict): + fm.eval() + # print('absorb lora before') + o1 = fm(**samples) + + for name, module in fm.named_modules(): + if not isinstance(module, ToQKV_WrappedWithLoRA): + continue + + fc = module.fc + ab = module.ab + + fc.weight.add_(ab[1].weight @ ab[0].weight) + + set_module(fm, name, fc) + + # print('absorb lora after') + o2 = fm(**samples) + + if isinstance(o1, tuple): + o1 = o1[-1] + o2 = o2[-1] + output_diff = ((o1 - o2) ** 2).sum() + assert output_diff < 1e-6, output_diff + + return fm + \ No newline at end of file diff --git a/new_impl/cv/elasticdnn/pipeline/offline/fm_lora/clip.py b/new_impl/cv/elasticdnn/pipeline/offline/fm_lora/clip.py new file mode 100644 index 0000000000000000000000000000000000000000..35e38a30187a91371ddfe2535ab83b3cc04da0da --- /dev/null +++ b/new_impl/cv/elasticdnn/pipeline/offline/fm_lora/clip.py @@ -0,0 +1,86 @@ +import torch +from torch import nn +from abc import ABC, abstractmethod + +from utils.dl.common.model import get_model_device, get_model_latency, get_model_size, set_module +from utils.common.log import logger +from .base import FMLoRA_Util, LoRA + + +class ToQKV_WrappedWithLoRA(nn.Module): + def __init__(self, fc: nn.Linear, ab_r: int): + super(ToQKV_WrappedWithLoRA, self).__init__() + + self.fc = fc + self.ab = self.create_ab_as_linear(fc.weight.data, ab_r) + + def create_ab_as_linear(self, fc_weight: torch.Tensor, ab_r: int): + res = nn.Sequential( + LoRA(fc_weight.size(1), fc_weight.size(0) // ab_r, bias=False), + LoRA(fc_weight.size(0) // ab_r, fc_weight.size(0), bias=False) + ).to(fc_weight.device) + nn.init.kaiming_uniform_(res[0].weight, a=5 ** 0.5) + nn.init.zeros_(res[1].weight) + return res + + def forward(self, x): + x1 = self.fc(x) + x2 = self.ab(x) + return x1 + x2 + + +class FMLoRA_CLIP_Util(FMLoRA_Util): + + @torch.no_grad() + def add_lora_ab_to_fm(self, fm: nn.Module, ab_r: int, samples: dict): + fm.eval() + + for k, v in samples.items(): + if isinstance(v, torch.Tensor): + samples[k] = v.to(get_model_device(fm)) + print(k) + + o1 = fm(**samples) + + for name, module in fm.named_modules(): + if name.endswith(('k_proj', 'q_proj', 'v_proj')): + set_module(fm, name, ToQKV_WrappedWithLoRA(module, ab_r)) + + o2 = fm(**samples) + + output_diff = ((o1.logits_per_image - o2.logits_per_image) ** 2).sum() + ((o1.logits_per_text - o2.logits_per_text) ** 2).sum() + assert output_diff < 1e-5 + + return fm + + @torch.no_grad() + def absorb_lora_and_recover_net_structure(self, fm: nn.Module, samples: dict): + fm.eval() + # print('absorb lora before') + + for k, v in samples.items(): + if isinstance(v, torch.Tensor): + samples[k] = v.to(get_model_device(fm)) + print(k) + + o1 = fm(**samples) + + for name, module in fm.named_modules(): + if not isinstance(module, ToQKV_WrappedWithLoRA): + continue + + fc = module.fc + ab = module.ab + + fc.weight.add_(ab[1].weight @ ab[0].weight) + + set_module(fm, name, fc) + + # print('absorb lora after') + o2 = fm(**samples) + + output_diff = ((o1.logits_per_image - o2.logits_per_image) ** 2).sum() + ((o1.logits_per_text - o2.logits_per_text) ** 2).sum() + assert output_diff < 1e-6, output_diff + + return fm + \ No newline at end of file diff --git a/new_impl/cv/elasticdnn/pipeline/offline/fm_lora/vilt.py b/new_impl/cv/elasticdnn/pipeline/offline/fm_lora/vilt.py new file mode 100644 index 0000000000000000000000000000000000000000..244e503be595d01b38dcbb12833535dfdcf3a811 --- /dev/null +++ b/new_impl/cv/elasticdnn/pipeline/offline/fm_lora/vilt.py @@ -0,0 +1,91 @@ +import torch +from torch import nn +from abc import ABC, abstractmethod + +from utils.dl.common.model import get_model_device, get_model_latency, get_model_size, set_module +from utils.common.log import logger +from .base import FMLoRA_Util, LoRA + + +class ToQKV_WrappedWithLoRA(nn.Module): + def __init__(self, fc: nn.Linear, ab_r: int): + super(ToQKV_WrappedWithLoRA, self).__init__() + + self.fc = fc + self.ab = self.create_ab_as_linear(fc.weight.data, ab_r) + + def create_ab_as_linear(self, fc_weight: torch.Tensor, ab_r: int): + res = nn.Sequential( + LoRA(fc_weight.size(1), fc_weight.size(0) // ab_r, bias=False), + LoRA(fc_weight.size(0) // ab_r, fc_weight.size(0), bias=False) + ).to(fc_weight.device) + nn.init.kaiming_uniform_(res[0].weight, a=5 ** 0.5) + nn.init.zeros_(res[1].weight) + return res + + def forward(self, x): + x1 = self.fc(x) + x2 = self.ab(x) + return x1 + x2 + + +class FMLoRA_Vilt_Util(FMLoRA_Util): + + @torch.no_grad() + def add_lora_ab_to_fm(self, fm: nn.Module, ab_r: int, samples: dict): + fm.eval() + + # print(samples) + for k, v in samples.items(): + if isinstance(v, torch.Tensor): + samples[k] = v.to(get_model_device(fm)) + + o1 = fm(**samples) + + for name, module in fm.named_modules(): + if name.endswith(('query', 'key', 'value')): + set_module(fm, name, ToQKV_WrappedWithLoRA(module, ab_r)) + + o2 = fm(**samples) + + if isinstance(o1, tuple): + o1 = o1[-1] + o2 = o2[-1] + output_diff = ((o1.logits - o2.logits) ** 2).sum() + assert output_diff < 1e-5 + + return fm + + @torch.no_grad() + def absorb_lora_and_recover_net_structure(self, fm: nn.Module, samples: dict): + fm.eval() + # print('absorb lora before') + + for k, v in samples.items(): + if isinstance(v, torch.Tensor): + samples[k] = v.to(get_model_device(fm)) + + o1 = fm(**samples) + + for name, module in fm.named_modules(): + if not isinstance(module, ToQKV_WrappedWithLoRA): + continue + + fc = module.fc + ab = module.ab + + fc.weight.add_(ab[1].weight @ ab[0].weight) + + set_module(fm, name, fc) + + # print('absorb lora after') + o2 = fm(**samples) + + if isinstance(o1, tuple): + o1 = o1[-1] + o2 = o2[-1] + output_diff = ((o1.logits - o2.logits) ** 2).sum() + assert output_diff < 1e-6, output_diff + + return fm + \ No newline at end of file diff --git a/new_impl/cv/elasticdnn/pipeline/offline/fm_lora/vit.py b/new_impl/cv/elasticdnn/pipeline/offline/fm_lora/vit.py new file mode 100644 index 0000000000000000000000000000000000000000..eef6c41d1421b282f3a3ae4ef096bd427013822d --- /dev/null +++ b/new_impl/cv/elasticdnn/pipeline/offline/fm_lora/vit.py @@ -0,0 +1,82 @@ +import torch +from torch import nn +from abc import ABC, abstractmethod + +from utils.dl.common.model import get_model_device, get_model_latency, get_model_size, set_module +from utils.common.log import logger +from .base import FMLoRA_Util, LoRA + + +class ToQKV_WrappedWithLoRA(nn.Module): + def __init__(self, qkv: nn.Linear, ab_r: int): + super(ToQKV_WrappedWithLoRA, self).__init__() + + self.qkv = qkv + self.abs = nn.ModuleList([self.create_ab_as_linear(w, ab_r) for w in qkv.weight.data.chunk(3, dim=0)]) + + def create_ab_as_linear(self, fc_weight: torch.Tensor, ab_r: int): + res = nn.Sequential( + LoRA(fc_weight.size(1), fc_weight.size(0) // ab_r, bias=False), + LoRA(fc_weight.size(0) // ab_r, fc_weight.size(0), bias=False) + ).to(fc_weight.device) + nn.init.kaiming_uniform_(res[0].weight, a=5 ** 0.5) + nn.init.zeros_(res[1].weight) + return res + + def forward(self, x): + x1 = self.qkv(x) + x2 = torch.cat([ab(x) for ab in self.abs], dim=-1) + return x1 + x2 + + +class FMLoRA_ViT_Util(FMLoRA_Util): + + @torch.no_grad() + def add_lora_ab_to_fm(self, fm: nn.Module, ab_r: int, samples: torch.Tensor): + fm.eval() + o1 = fm(samples) + + for name, module in fm.named_modules(): + #if not name.endswith('.qkv'): + #if not name.endswith('.projection_key') or not name.endswith('.projection_query') or not name.endswith('.projection_value'): + if name.endswith('.projection_query') or name.endswith('.projection_key') or name.endswith('.projection_value'): + set_module(fm, name, ToQKV_WrappedWithLoRA(module, ab_r)) + + + o2 = fm(samples) + + #output_diff = ((o1 - o2) ** 2).sum() + output_diff = ((o1.logits - o2.logits) ** 2).sum() + assert output_diff < 1e-5 + + return fm + + @torch.no_grad() + def absorb_lora_and_recover_net_structure(self, fm: nn.Module, samples: torch.Tensor): + fm.eval() + # print('absorb lora before') + o1 = fm(samples) + + for name, module in fm.named_modules(): + if not isinstance(module, ToQKV_WrappedWithLoRA): + continue + + qkv = module.qkv + fm_abs = module.abs + + fm_abs_weight = torch.cat([_abs[1].weight @ _abs[0].weight for _abs in fm_abs], dim=0) + qkv.weight.add_(fm_abs_weight) + + set_module(fm, name, qkv) + + # print('absorb lora after') + o2 = fm(samples) + + output_diff = ((o1.logits - o2.logits) ** 2).sum() + # print(o1) + # print(o2) + assert output_diff < 1e-5, output_diff + + return fm + + diff --git a/new_impl/cv/elasticdnn/pipeline/offline/fm_to_md/__pycache__/base.cpython-38.pyc b/new_impl/cv/elasticdnn/pipeline/offline/fm_to_md/__pycache__/base.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..24f1c50372602ea31898fa03d5ed63f9ea9e92c5 Binary files /dev/null and b/new_impl/cv/elasticdnn/pipeline/offline/fm_to_md/__pycache__/base.cpython-38.pyc differ diff --git a/new_impl/cv/elasticdnn/pipeline/offline/fm_to_md/__pycache__/vit.cpython-38.pyc b/new_impl/cv/elasticdnn/pipeline/offline/fm_to_md/__pycache__/vit.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5acd68ec964baa12e299fa4d8897249eef2a99d0 Binary files /dev/null and b/new_impl/cv/elasticdnn/pipeline/offline/fm_to_md/__pycache__/vit.cpython-38.pyc differ diff --git a/new_impl/cv/elasticdnn/pipeline/offline/fm_to_md/base.py b/new_impl/cv/elasticdnn/pipeline/offline/fm_to_md/base.py new file mode 100644 index 0000000000000000000000000000000000000000..0b15fbd8675af7e82bcb90dc76bc2706b8cd5621 --- /dev/null +++ b/new_impl/cv/elasticdnn/pipeline/offline/fm_to_md/base.py @@ -0,0 +1,49 @@ +import torch +from torch import nn +from abc import ABC, abstractmethod + +from utils.dl.common.model import get_model_device, get_model_latency, get_model_size +from utils.common.log import logger + + +class FM_to_MD_Util(ABC): + """ + Foundation Model (FM) to Master DNN (MD), where MD is a narrower FM (with smaller width but the same depth). + + MD is pre-trained by knowledge distillation; + Moreover, we construct the index relationship between FM and MD in this process, + enabling the lightweight knowledge feedback from MD to FM. + + NOTE: 索引建立在master DNN权重通道和LoRA的AB之间 + """ + + @abstractmethod + def init_md_from_fm_by_reducing_width(self, fm: nn.Module, reducing_width_ratio: int) -> nn.Module: + raise NotImplementedError + + def init_md_from_fm_by_reducing_width_with_perf_test(self, fm: nn.Module, reducing_width_ratio: int, + samples: torch.Tensor) -> nn.Module: + fm_size = get_model_size(fm, True) + fm_latency = get_model_latency(fm, (1, *list(samples.size())[1:]), 20, + get_model_device(fm), 20, False) + + master_dnn = self.init_md_from_fm_by_reducing_width(fm, reducing_width_ratio) + master_dnn_size = get_model_size(master_dnn, True) + logger.debug(f'inited master DNN: {master_dnn}') + #print(master_dnn) + # from utils.dl.common.model import get_module + # print('after generating') + # get_module(fm, 'head').debug() + # get_module(master_dnn, 'head').debug() + # print('test master latency') + master_dnn_latency = get_model_latency(master_dnn, (1, *list(samples.size())[1:]), 20, + get_model_device(master_dnn), 20, False) + + logger.info(f'init master DNN (w/o FBS yet) by reducing foundation model\'s width (by {reducing_width_ratio:d}x)') + logger.info(f'foundation model ({fm_size:.3f}MB, {fm_latency:.4f}s/sample) -> ' + f'master DNN ({master_dnn_size:.3f}MB, {master_dnn_latency:.4f}s/sample)\n' + f'(model size: ↓ {(fm_size / master_dnn_size):.2f}x, ' + f'latency: ↓ {(fm_latency / master_dnn_latency):.2f}x)') + + return master_dnn + \ No newline at end of file diff --git a/new_impl/cv/elasticdnn/pipeline/offline/fm_to_md/bert.py b/new_impl/cv/elasticdnn/pipeline/offline/fm_to_md/bert.py new file mode 100644 index 0000000000000000000000000000000000000000..b978953bc3092c0aec1d074b26bbbea38377744c --- /dev/null +++ b/new_impl/cv/elasticdnn/pipeline/offline/fm_to_md/bert.py @@ -0,0 +1,279 @@ + +import torch +from torch import nn +from copy import deepcopy + +from .base import FM_to_MD_Util +from utils.common.log import logger +from utils.dl.common.model import set_module, get_module, get_super_module +from utils.dl.common.model import get_model_device, get_model_latency, get_model_size +from utils.common.log import logger + +from transformers.models.bert.modeling_bert import BertSelfAttention +from transformers import BertConfig + +from typing import Optional, Tuple +import math + +class BertSelfAttentionPrunable(BertSelfAttention): + def __init__(self): + config = BertConfig.from_pretrained('bert-base-multilingual-cased') + super(BertSelfAttentionPrunable, self).__init__(config) + + def transpose_for_scores(self, x): + new_x_shape = x.size()[:-1] + (self.num_attention_heads, -1) + x = x.view(new_x_shape) + return x.permute(0, 2, 1, 3) + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.FloatTensor] = None, + head_mask: Optional[torch.FloatTensor] = None, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + encoder_attention_mask: Optional[torch.FloatTensor] = None, + past_key_value: Optional[Tuple[Tuple[torch.FloatTensor]]] = None, + output_attentions: Optional[bool] = False, + ) -> Tuple[torch.Tensor]: + mixed_query_layer = self.query(hidden_states) + + # If this is instantiated as a cross-attention module, the keys + # and values come from an encoder; the attention mask needs to be + # such that the encoder's padding tokens are not attended to. + is_cross_attention = encoder_hidden_states is not None + + if is_cross_attention and past_key_value is not None: + # reuse k,v, cross_attentions + key_layer = past_key_value[0] + value_layer = past_key_value[1] + attention_mask = encoder_attention_mask + elif is_cross_attention: + key_layer = self.transpose_for_scores(self.key(encoder_hidden_states)) + value_layer = self.transpose_for_scores(self.value(encoder_hidden_states)) + attention_mask = encoder_attention_mask + elif past_key_value is not None: + key_layer = self.transpose_for_scores(self.key(hidden_states)) + value_layer = self.transpose_for_scores(self.value(hidden_states)) + key_layer = torch.cat([past_key_value[0], key_layer], dim=2) + value_layer = torch.cat([past_key_value[1], value_layer], dim=2) + else: + key_layer = self.transpose_for_scores(self.key(hidden_states)) + value_layer = self.transpose_for_scores(self.value(hidden_states)) + + query_layer = self.transpose_for_scores(mixed_query_layer) + + use_cache = past_key_value is not None + if self.is_decoder: + # if cross_attention save Tuple(torch.Tensor, torch.Tensor) of all cross attention key/value_states. + # Further calls to cross_attention layer can then reuse all cross-attention + # key/value_states (first "if" case) + # if uni-directional self-attention (decoder) save Tuple(torch.Tensor, torch.Tensor) of + # all previous decoder key/value_states. Further calls to uni-directional self-attention + # can concat previous decoder key/value_states to current projected key/value_states (third "elif" case) + # if encoder bi-directional self-attention `past_key_value` is always `None` + past_key_value = (key_layer, value_layer) + + # Take the dot product between "query" and "key" to get the raw attention scores. + attention_scores = torch.matmul(query_layer, key_layer.transpose(-1, -2)) + + if self.position_embedding_type == "relative_key" or self.position_embedding_type == "relative_key_query": + query_length, key_length = query_layer.shape[2], key_layer.shape[2] + if use_cache: + position_ids_l = torch.tensor(key_length - 1, dtype=torch.long, device=hidden_states.device).view( + -1, 1 + ) + else: + position_ids_l = torch.arange(query_length, dtype=torch.long, device=hidden_states.device).view(-1, 1) + position_ids_r = torch.arange(key_length, dtype=torch.long, device=hidden_states.device).view(1, -1) + distance = position_ids_l - position_ids_r + + positional_embedding = self.distance_embedding(distance + self.max_position_embeddings - 1) + positional_embedding = positional_embedding.to(dtype=query_layer.dtype) # fp16 compatibility + + if self.position_embedding_type == "relative_key": + relative_position_scores = torch.einsum("bhld,lrd->bhlr", query_layer, positional_embedding) + attention_scores = attention_scores + relative_position_scores + elif self.position_embedding_type == "relative_key_query": + relative_position_scores_query = torch.einsum("bhld,lrd->bhlr", query_layer, positional_embedding) + relative_position_scores_key = torch.einsum("bhrd,lrd->bhlr", key_layer, positional_embedding) + attention_scores = attention_scores + relative_position_scores_query + relative_position_scores_key + + attention_scores = attention_scores / math.sqrt(self.attention_head_size) + if attention_mask is not None: + # Apply the attention mask is (precomputed for all layers in BertModel forward() function) + attention_scores = attention_scores + attention_mask + + # Normalize the attention scores to probabilities. + attention_probs = nn.functional.softmax(attention_scores, dim=-1) + + # This is actually dropping out entire tokens to attend to, which might + # seem a bit unusual, but is taken from the original Transformer paper. + attention_probs = self.dropout(attention_probs) + + # Mask heads if we want to + if head_mask is not None: + attention_probs = attention_probs * head_mask + + context_layer = torch.matmul(attention_probs, value_layer) + + context_layer = context_layer.permute(0, 2, 1, 3).contiguous() + new_context_layer_shape = context_layer.size()[:-2] + (self.query.out_features,) # NOTE: modified + context_layer = context_layer.view(new_context_layer_shape) + + outputs = (context_layer, attention_probs) if output_attentions else (context_layer,) + + if self.is_decoder: + outputs = outputs + (past_key_value,) + return outputs + + @staticmethod + def init_from_exist_self_attn(attn: BertSelfAttention): + # print(attn) + + res = BertSelfAttentionPrunable() + + for attr in dir(attn): + # if str(attr) in ['transpose_for_scores'] or str(attr).startswith('_'): + # continue + # if isinstance(getattr(attn, attr), nn.Module): + # print(attr) + + if isinstance(getattr(attn, attr), nn.Module): + try: + # print(attr, 'ok') + setattr(res, attr, getattr(attn, attr)) + + except Exception as e: + print(attr, str(e)) + + + + return res + +class FM_to_MD_Bert_Util(FM_to_MD_Util): + def init_md_from_fm_by_reducing_width(self, fm: nn.Module, reducing_width_ratio: int) -> nn.Module: + fm_vit = deepcopy(fm) + + for block in fm_vit.bert.encoder.layer: + set_module(block, 'attention.self', BertSelfAttentionPrunable.init_from_exist_self_attn(block.attention.self)) + + def _f(n): + return int(n // reducing_width_ratio) + + # def _rand_indexes(n): + # return torch.randperm(n)[0: int(n // reducing_width_ratio)] + + def l1_max_indexes(p: torch.Tensor, dim=0): + assert dim in [0, 1] + assert p.dim() in [1, 2, 4] + + if dim == 1: + p = p.T + + p_norm = p.abs().contiguous().view(p.size(0), -1).sum(dim=1) + n = p.size(0) + return p_norm.argsort(descending=True)[0: int(n // reducing_width_ratio)].sort()[0] + + for block_i, block in enumerate(fm_vit.bert.encoder.layer): + for k in ['query', 'key', 'value']: + qkv = get_module(block, f'attention.self.{k}') + + new_qkv = nn.Linear(qkv.in_features, _f(qkv.out_features), + qkv.bias is not None, qkv.weight.device) + indexes = l1_max_indexes(qkv.weight.data, 0) + + new_qkv.weight.data.copy_(qkv.weight.data[indexes]) + if qkv.bias is not None: + new_qkv.bias.data.copy_(qkv.bias.data[indexes]) + set_module(block, f'attention.self.{k}', new_qkv) + + proj = get_module(block, f'attention.output.dense') + new_proj = nn.Linear(_f(proj.in_features), proj.out_features, + proj.bias is not None, proj.weight.device) + new_proj.weight.data.copy_(proj.weight.data[:, l1_max_indexes(proj.weight.data, 1)]) + if proj.bias is not None: + new_proj.bias.data.copy_(proj.bias.data) + set_module(block, f'attention.output.dense', new_proj) + + fc1 = get_module(block, f'intermediate.dense') + new_fc1 = nn.Linear(fc1.in_features, _f(fc1.out_features), + fc1.bias is not None, fc1.weight.device) + indexes = l1_max_indexes(fc1.weight.data, 0) + new_fc1.weight.data.copy_(fc1.weight.data[indexes]) + if fc1.bias is not None: + new_fc1.bias.data.copy_(fc1.bias.data[indexes]) + set_module(block, f'intermediate.dense', new_fc1) + + fc2 = get_module(block, f'output.dense') + new_fc2 = nn.Linear(_f(fc2.in_features), fc2.out_features, + fc2.bias is not None, fc2.weight.device) + new_fc2.weight.data.copy_(fc2.weight.data[:, l1_max_indexes(fc2.weight.data, 1)]) + if fc2.bias is not None: + new_fc2.bias.data.copy_(fc2.bias.data) + set_module(block, f'output.dense', new_fc2) + + return fm_vit + + def init_md_from_fm_by_reducing_width_with_perf_test(self, fm: nn.Module, reducing_width_ratio: int, + samples: torch.Tensor) -> nn.Module: + fm_size = get_model_size(fm, True) + fm_latency = self._get_model_latency(fm, samples, 20, + get_model_device(fm), 20, False) + + master_dnn = self.init_md_from_fm_by_reducing_width(fm, reducing_width_ratio) + master_dnn_size = get_model_size(master_dnn, True) + logger.debug(f'inited master DNN: {master_dnn}') + master_dnn_latency = self._get_model_latency(master_dnn, samples, 20, + get_model_device(master_dnn), 20, False) + + logger.info(f'init master DNN (w/o FBS yet) by reducing foundation model\'s width (by {reducing_width_ratio:d}x)') + logger.info(f'foundation model ({fm_size:.3f}MB, {fm_latency:.4f}s/sample) -> ' + f'master DNN ({master_dnn_size:.3f}MB, {master_dnn_latency:.4f}s/sample)\n' + f'(model size: ↓ {(fm_size / master_dnn_size):.2f}x, ' + f'latency: ↓ {(fm_latency / master_dnn_latency):.2f}x)') + + return master_dnn + + def _get_model_latency(self, model: torch.nn.Module, model_input_size, sample_num: int, + device: str, warmup_sample_num: int, return_detail=False): + import time + + if isinstance(model_input_size, tuple): + dummy_input = torch.rand(model_input_size).to(device) + else: + dummy_input = model_input_size + + model = model.to(device) + model.eval() + + # warm up + with torch.no_grad(): + for _ in range(warmup_sample_num): + model(**dummy_input) + + infer_time_list = [] + + if device == 'cuda' or 'cuda' in str(device): + with torch.no_grad(): + for _ in range(sample_num): + s, e = torch.cuda.Event(enable_timing=True), torch.cuda.Event(enable_timing=True) + s.record() + model(**dummy_input) + e.record() + torch.cuda.synchronize() + cur_model_infer_time = s.elapsed_time(e) / 1000. + infer_time_list += [cur_model_infer_time] + + else: + with torch.no_grad(): + for _ in range(sample_num): + start = time.time() + model(**dummy_input) + cur_model_infer_time = time.time() - start + infer_time_list += [cur_model_infer_time] + + avg_infer_time = sum(infer_time_list) / sample_num + + if return_detail: + return avg_infer_time, infer_time_list + return avg_infer_time \ No newline at end of file diff --git a/new_impl/cv/elasticdnn/pipeline/offline/fm_to_md/clip.py b/new_impl/cv/elasticdnn/pipeline/offline/fm_to_md/clip.py new file mode 100644 index 0000000000000000000000000000000000000000..9f35d59e15566def102e68b41d6a8172457f241d --- /dev/null +++ b/new_impl/cv/elasticdnn/pipeline/offline/fm_to_md/clip.py @@ -0,0 +1,672 @@ + +import torch +from torch import nn +from copy import deepcopy + +from .base import FM_to_MD_Util +from utils.common.log import logger +from utils.dl.common.model import set_module, get_module, get_super_module +from utils.dl.common.model import get_model_device, get_model_latency, get_model_size +from utils.common.log import logger +from typing import Optional, Tuple + +from transformers.models.clip.modeling_clip import CLIPAttention +from transformers import CLIPVisionConfig + + +class CLIPAttentionPrunable(CLIPAttention): + """Multi-headed attention from 'Attention Is All You Need' paper""" + + + def __init__(self): + config = CLIPVisionConfig.from_pretrained('openai/clip-vit-base-patch16') + super(CLIPAttentionPrunable, self).__init__(config) + + + # def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int): + # # print(tensor.size(), self.num_heads, self.head_dim, bsz) # torch.Size([1, 197, 192]) 8 64 1 + # # head_dim should be modified + + # # 'b n (h d) -> b h n d', h = self.num_heads + + # if seq_len == -1: + # seq_len = tensor.size(1) + + # # print(tensor.size(), bsz, seq_len, self.num_heads, -1) + # return tensor.view(bsz, seq_len, self.num_heads, -1).transpose(1, 2).contiguous() + + # def forward( + # self, + # hidden_states: torch.Tensor, + # attention_mask: Optional[torch.Tensor] = None, + # causal_attention_mask: Optional[torch.Tensor] = None, + # output_attentions: Optional[bool] = False, + # ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: + # """Input shape: Batch x Time x Channel""" + + # bsz, tgt_len, embed_dim = hidden_states.size() + + # # get query proj + # query_states = self.q_proj(hidden_states) * self.scale + # key_states = self._shape(self.k_proj(hidden_states), -1, bsz) + # value_states = self._shape(self.v_proj(hidden_states), -1, bsz) + + # proj_shape = (-1, tgt_len, self.head_dim) + # # print(proj_shape, key_states.size()) + # query_states = self._shape(query_states, tgt_len, bsz).view(*proj_shape) + # key_states = key_states.view(*proj_shape) + # value_states = value_states.view(*proj_shape) + + # src_len = key_states.size(1) + # attn_weights = torch.bmm(query_states, key_states.transpose(1, 2)) + + # # if attn_weights.size() != (bsz * self.num_heads, tgt_len, src_len): + # # raise ValueError( + # # f"Attention weights should be of size {(bsz * self.num_heads, tgt_len, src_len)}, but is" + # # f" {attn_weights.size()}" + # # ) + + # # apply the causal_attention_mask first + # if causal_attention_mask is not None: + # if causal_attention_mask.size() != (bsz, 1, tgt_len, src_len): + # raise ValueError( + # f"Attention mask should be of size {(bsz, 1, tgt_len, src_len)}, but is" + # f" {causal_attention_mask.size()}" + # ) + # attn_weights = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + causal_attention_mask + # attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len) + + # if attention_mask is not None: + # if attention_mask.size() != (bsz, 1, tgt_len, src_len): + # raise ValueError( + # f"Attention mask should be of size {(bsz, 1, tgt_len, src_len)}, but is {attention_mask.size()}" + # ) + # attn_weights = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + attention_mask + # attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len) + + # attn_weights = nn.functional.softmax(attn_weights, dim=-1) + + # if output_attentions: + # # this operation is a bit akward, but it's required to + # # make sure that attn_weights keeps its gradient. + # # In order to do so, attn_weights have to reshaped + # # twice and have to be reused in the following + # attn_weights_reshaped = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + # attn_weights = attn_weights_reshaped.view(bsz * self.num_heads, tgt_len, src_len) + # else: + # attn_weights_reshaped = None + + # attn_probs = nn.functional.dropout(attn_weights, p=self.dropout, training=self.training) + + # attn_output = torch.bmm(attn_probs, value_states) + + # # if attn_output.size() != (bsz * self.num_heads, tgt_len, self.head_dim): + # # raise ValueError( + # # f"`attn_output` should be of size {(bsz, self.num_heads, tgt_len, self.head_dim)}, but is" + # # f" {attn_output.size()}" + # # ) + + # attn_output = attn_output.view(bsz, self.num_heads, tgt_len, -1) + # attn_output = attn_output.transpose(1, 2) + # attn_output = attn_output.reshape(bsz, tgt_len, -1) + + # attn_output = self.out_proj(attn_output) + + # return attn_output, attn_weights_reshaped + + + def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int): + return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous() + + def _shape_dynamic_head_dim(self, tensor: torch.Tensor, seq_len: int, bsz: int): + return tensor.view(bsz, seq_len, self.num_heads, -1).transpose(1, 2).contiguous() + + def _shape_dynamic_num_head(self, tensor: torch.Tensor, seq_len: int, bsz: int): + return tensor.view(bsz, seq_len, -1, self.head_dim).transpose(1, 2).contiguous() + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + causal_attention_mask: Optional[torch.Tensor] = None, + output_attentions: Optional[bool] = False, + ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: + """Input shape: Batch x Time x Channel""" + + bsz, tgt_len, embed_dim = hidden_states.size() + + # logger.info(f'hidden state size: {hidden_states.size()}') # (64, 197, 768) + + # get query proj + query_states = self.q_proj(hidden_states) * self.scale + key_states = self._shape_dynamic_head_dim(self.k_proj(hidden_states), tgt_len, bsz) + value_states = self._shape_dynamic_head_dim(self.v_proj(hidden_states), tgt_len, bsz) + + # (64, 197, 768), numhead: 12, head_dim: 64, seq_len: 197 + # logger.info(f'key states: {self.k_proj(hidden_states).size()}, bsz: {bsz}, num_heads: {self.num_heads}, head_dim: {self.head_dim}, ' + # f'seq_len: {self.k_proj(hidden_states).numel() / bsz / self.num_heads / self.head_dim}') + # (64, 197, 768), numhead: 12, head_dim: 64, seq_len: 197 + # logger.info(f'value states: {self.v_proj(hidden_states).size()}, bsz: {bsz}, num_heads: {self.num_heads}, head_dim: {self.head_dim}, ' + # f'seq_len: {self.v_proj(hidden_states).numel() / bsz / self.num_heads / self.head_dim}') + + proj_shape = (bsz * self.num_heads, tgt_len, -1) + query_states = self._shape_dynamic_head_dim(query_states, tgt_len, bsz).view(*proj_shape) + + # (64, 12, 197, 64), -1 means 197 + # logger.info(f'query states: {self._shape(query_states, tgt_len, bsz).size()}, ' + # f'-1 in proj_shape: {self._shape(query_states, tgt_len, bsz).numel() / bsz / self.num_heads / self.head_dim}') + + key_states = key_states.view(*proj_shape) + value_states = value_states.view(*proj_shape) + + src_len = key_states.size(1) + attn_weights = torch.bmm(query_states, key_states.transpose(1, 2)) + + if attn_weights.size() != (bsz * self.num_heads, tgt_len, src_len): + raise ValueError( + f"Attention weights should be of size {(bsz * self.num_heads, tgt_len, src_len)}, but is" + f" {attn_weights.size()}" + ) + + # apply the causal_attention_mask first + if causal_attention_mask is not None: + if causal_attention_mask.size() != (bsz, 1, tgt_len, src_len): + raise ValueError( + f"Attention mask should be of size {(bsz, 1, tgt_len, src_len)}, but is" + f" {causal_attention_mask.size()}" + ) + attn_weights = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + causal_attention_mask + attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len) + + if attention_mask is not None: + if attention_mask.size() != (bsz, 1, tgt_len, src_len): + raise ValueError( + f"Attention mask should be of size {(bsz, 1, tgt_len, src_len)}, but is {attention_mask.size()}" + ) + attn_weights = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + attention_mask + attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len) + + attn_weights = nn.functional.softmax(attn_weights, dim=-1) + + if output_attentions: + # this operation is a bit akward, but it's required to + # make sure that attn_weights keeps its gradient. + # In order to do so, attn_weights have to reshaped + # twice and have to be reused in the following + attn_weights_reshaped = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + attn_weights = attn_weights_reshaped.view(bsz * self.num_heads, tgt_len, src_len) + else: + attn_weights_reshaped = None + + attn_probs = nn.functional.dropout(attn_weights, p=self.dropout, training=self.training) + + attn_output = torch.bmm(attn_probs, value_states) + + # if attn_output.size() != (bsz * self.num_heads, tgt_len, self.head_dim): + # raise ValueError( + # f"`attn_output` should be of size {(bsz, self.num_heads, tgt_len, self.head_dim)}, but is" + # f" {attn_output.size()}" + # ) + # print(attn_output.size(), bsz, tgt_len, embed_dim) + attn_output = attn_output.view(bsz, self.num_heads, tgt_len, -1) + attn_output = attn_output.transpose(1, 2) + attn_output = attn_output.reshape(bsz, tgt_len, -1) + + attn_output = self.out_proj(attn_output) + + return attn_output, attn_weights_reshaped + + # reduce num_head + # def forward( + # self, + # hidden_states: torch.Tensor, + # attention_mask: Optional[torch.Tensor] = None, + # causal_attention_mask: Optional[torch.Tensor] = None, + # output_attentions: Optional[bool] = False, + # ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: + # """Input shape: Batch x Time x Channel""" + + # bsz, tgt_len, embed_dim = hidden_states.size() + + # # logger.info(f'hidden state size: {hidden_states.size()}') # (64, 197, 768) + + # # get query proj + # query_states = self.q_proj(hidden_states) * self.scale + # key_states = self._shape_dynamic_num_head(self.k_proj(hidden_states), tgt_len, bsz) + # value_states = self._shape_dynamic_num_head(self.v_proj(hidden_states), tgt_len, bsz) + + # # (64, 197, 768), numhead: 12, head_dim: 64, seq_len: 197 + # # logger.info(f'key states: {self.k_proj(hidden_states).size()}, bsz: {bsz}, num_heads: {self.num_heads}, head_dim: {self.head_dim}, ' + # # f'seq_len: {self.k_proj(hidden_states).numel() / bsz / self.num_heads / self.head_dim}') + # # (64, 197, 768), numhead: 12, head_dim: 64, seq_len: 197 + # # logger.info(f'value states: {self.v_proj(hidden_states).size()}, bsz: {bsz}, num_heads: {self.num_heads}, head_dim: {self.head_dim}, ' + # # f'seq_len: {self.v_proj(hidden_states).numel() / bsz / self.num_heads / self.head_dim}') + + # proj_shape = (-1, tgt_len, self.head_dim) + # query_states = self._shape_dynamic_head_dim(query_states, tgt_len, bsz).view(*proj_shape) + + # # (64, 12, 197, 64), -1 means 197 + # # logger.info(f'query states: {self._shape(query_states, tgt_len, bsz).size()}, ' + # # f'-1 in proj_shape: {self._shape(query_states, tgt_len, bsz).numel() / bsz / self.num_heads / self.head_dim}') + + # key_states = key_states.view(*proj_shape) + # value_states = value_states.view(*proj_shape) + + # src_len = key_states.size(1) + # attn_weights = torch.bmm(query_states, key_states.transpose(1, 2)) + + # # if attn_weights.size() != (bsz * self.num_heads, tgt_len, src_len): + # # raise ValueError( + # # f"Attention weights should be of size {(bsz * self.num_heads, tgt_len, src_len)}, but is" + # # f" {attn_weights.size()}" + # # ) + + # # apply the causal_attention_mask first + # if causal_attention_mask is not None: + # if causal_attention_mask.size() != (bsz, 1, tgt_len, src_len): + # raise ValueError( + # f"Attention mask should be of size {(bsz, 1, tgt_len, src_len)}, but is" + # f" {causal_attention_mask.size()}" + # ) + # attn_weights = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + causal_attention_mask + # attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len) + + # if attention_mask is not None: + # if attention_mask.size() != (bsz, 1, tgt_len, src_len): + # raise ValueError( + # f"Attention mask should be of size {(bsz, 1, tgt_len, src_len)}, but is {attention_mask.size()}" + # ) + # attn_weights = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + attention_mask + # attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len) + + # attn_weights = nn.functional.softmax(attn_weights, dim=-1) + + # if output_attentions: + # # this operation is a bit akward, but it's required to + # # make sure that attn_weights keeps its gradient. + # # In order to do so, attn_weights have to reshaped + # # twice and have to be reused in the following + # attn_weights_reshaped = attn_weights.view(bsz, -1, tgt_len, src_len) + # attn_weights = attn_weights_reshaped.view(-1, tgt_len, src_len) + # else: + # attn_weights_reshaped = None + + # attn_probs = nn.functional.dropout(attn_weights, p=self.dropout, training=self.training) + + # attn_output = torch.bmm(attn_probs, value_states) + + # # if attn_output.size() != (bsz * self.num_heads, tgt_len, self.head_dim): + # # raise ValueError( + # # f"`attn_output` should be of size {(bsz, self.num_heads, tgt_len, self.head_dim)}, but is" + # # f" {attn_output.size()}" + # # ) + # # print(attn_output.size(), bsz, tgt_len, embed_dim) + # attn_output = attn_output.view(bsz, -1, tgt_len, self.head_dim) + # attn_output = attn_output.transpose(1, 2) + # attn_output = attn_output.reshape(bsz, tgt_len, -1) + + # attn_output = self.out_proj(attn_output) + + # return attn_output, attn_weights_reshaped + + + @staticmethod + def init_from_exist_self_attn(attn: CLIPAttention): + # print(attn) + + res = CLIPAttentionPrunable() + + for attr in dir(attn): + # if str(attr) in ['transpose_for_scores'] or str(attr).startswith('_'): + # continue + # if isinstance(getattr(attn, attr), nn.Module): + # print(attr) + + if isinstance(getattr(attn, attr), nn.Module): + try: + # print(attr, 'ok') + setattr(res, attr, getattr(attn, attr)) + + except Exception as e: + print(attr, str(e)) + + + + return res + + +from einops import rearrange, repeat +from einops.layers.torch import Rearrange + +class PrunableAttention(nn.Module): + """ + https://github.com/lucidrains/vit-pytorch + """ + def __init__(self, dim, heads = 8, dim_head = 64, dropout = 0., qkv_bias = False): + super().__init__() + self.inner_dim = inner_dim = dim_head * heads + project_out = not (heads == 1 and dim_head == dim) + + self.num_heads = heads + self.scale = dim_head ** -0.5 + + self.attend = nn.Softmax(dim = -1) + self.dropout = nn.Dropout(dropout) + + self.qkv = nn.Linear(dim, inner_dim * 3, bias = qkv_bias) + + # self.proj = nn.Sequential( + # nn.Linear(inner_dim, dim), + # nn.Dropout(dropout) + # ) if project_out else nn.Identity() + + self.proj = nn.Linear(inner_dim, dim) if project_out else nn.Identity() + self.proj_dropout = nn.Dropout(dropout) + + def forward(self, hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + causal_attention_mask: Optional[torch.Tensor] = None, + output_attentions: Optional[bool] = False,): + + x = hidden_states + assert attention_mask is None + assert causal_attention_mask is None + assert not output_attentions + # qkv = self.qkv(x).chunk(3, dim = -1) + raw_qkv = self.qkv(x) + + self.inner_dim = (raw_qkv.size(-1) - self.proj.in_features) // 2 + qkv = raw_qkv[:, :, 0: self.inner_dim], raw_qkv[:, :, self.inner_dim: self.inner_dim * 2], raw_qkv[:, :, self.inner_dim * 2:] + + # print('v', qkv[0].size(), qkv[0].sum((0, 1))[0: 10], qkv[0].sum((0, 1)).nonzero(as_tuple=True)[0].size()) + + # raw_v = qkv[2] + # print('after_fbs_q, after_fbs_k', qkv[0].sum((0, 1))[0: 10], qkv[0].sum((0, 1)).nonzero(as_tuple=True)[0].size(), + # qkv[1].sum((0, 1))[0: 10], qkv[1].sum((0, 1)).nonzero(as_tuple=True)[0].size(),) + # print('after_fbs_v', raw_v.size(), raw_v.sum((0, 1))[0: 10], raw_v.sum((0, 1)).nonzero(as_tuple=True)[0].size()) + # print('q, before rearrage', qkv[0].size()) + q, k, v = qkv + # print('raw qkv size', q.size(), k.size(), v.size()) + # exit() + q, k, v = map(lambda t: rearrange(t, 'b n (h d) -> b h n d', h = self.num_heads), qkv) + # print('raw qkv size', q.size(), k.size(), v.size()) + + dots = torch.matmul(q, k.transpose(-1, -2)) * self.scale + + # print('q, k, dots, after rearrage', q.size(), k.transpose(-1, -2).size(), dots.size()) + + attn = self.attend(dots) + # attn = dots + attn = self.dropout(attn) + + # print(attn) + # print('attn', attn.size(), attn.sum((0, 1))[0: 10], attn.sum((0, 1)).nonzero(as_tuple=True)[0].size()) + # print('attn', attn.size(), attn.sum((0, 1))[0: 10], attn.sum((0, 1)).nonzero(as_tuple=True)[0].size()) + # print('v2', v.size()) + out = torch.matmul(attn, v) + # print('out1', out.size()) + # NOTE: just for trial debug + # out = v + + # print('out before rerange', out.size()) + + # print(v.size(), v) + # exit() + + out = rearrange(out, 'b h n d -> b n (h d)') + + # print('out', out.size(), out.sum((0, 1))[0: 10], out.sum((0, 1)).nonzero(as_tuple=True)[0].size()) + # exit() + + res = self.proj_dropout(self.proj(out)) + + # res = self.proj_dropout( + # F.linear(self.proj.weight.T, out.T, self.proj.bias) + # ) + # print(self.proj, self.proj_dropout) + # print('res', res.size(), res.sum((0, 1))[0: 10], res.sum((0, 1)).nonzero(as_tuple=True)[0].size()) + + return res, None + + +class FM_to_MD_CLIP_Util(FM_to_MD_Util): + def init_md_from_fm_by_reducing_width(self, fm: nn.Module, reducing_width_ratio: int) -> nn.Module: + fm_vit = deepcopy(fm) + + + # for block in fm_vit.model.text_model.encoder.layers: + # set_module(block, 'self_attn', CLIPAttentionPrunable.init_from_exist_self_attn(block.self_attn)) + + debug_input = torch.rand((1, 3, 32, 32)).cuda() + fm.eval() + o1 = fm.model.vision_model(debug_input).pooler_output + for block in fm_vit.model.vision_model.encoder.layers: + # set_module(block, 'self_attn', CLIPAttentionPrunable.init_from_exist_self_attn(block.self_attn)) + + attn: CLIPAttention = block.self_attn + # from dnns.vit import PrunableAttention + new_attn = PrunableAttention( + dim=768, + heads=12, + dim_head=64, + dropout=0, + qkv_bias=True + ) + new_attn.qkv.weight.data.copy_(torch.cat([ + attn.q_proj.weight, + attn.k_proj.weight, + attn.v_proj.weight + ], dim=0)) + new_attn.qkv.bias.data.copy_(torch.cat([ + attn.q_proj.bias, + attn.k_proj.bias, + attn.v_proj.bias + ], dim=0)) + new_attn.proj.weight.data.copy_(attn.out_proj.weight) + new_attn.proj.bias.data.copy_(attn.out_proj.bias) + set_module(block, 'self_attn', new_attn) + o2 = fm.model.vision_model(debug_input).pooler_output + + # NOTE: bug is here!!! + # although the diff is ZERO, but the logic of CLIPAttentionPrunable is incorrect!!!! + diff = ((o1 - o2) ** 2).sum() + print('diff before/after adding CLIPAttentionPrunable', diff) + assert diff < 1e-4 + + # print('\n\nDEBUG: WITHOUT ADDING CLIPAttentionPrunable\n\n') + + # exit() + + # return fm + def _f(n): + return int(n // reducing_width_ratio) + + # def _rand_indexes(n): + # return torch.randperm(n)[0: int(n // reducing_width_ratio)] + + def l1_max_indexes(p: torch.Tensor, dim=0): + assert dim in [0, 1] + assert p.dim() in [1, 2, 4] + + if dim == 1: + p = p.T + + p_norm = p.abs().contiguous().view(p.size(0), -1).sum(dim=1) + n = p.size(0) + res = p_norm.argsort(descending=True)[0: int(n // reducing_width_ratio)].sort()[0] + # print(res) + return res + + # first_attn = True + + # for block_i, block in enumerate(fm_vit.model.text_model.encoder.layers): + # for k in ['k_proj', 'q_proj', 'v_proj']: + # qkv = get_module(block, f'self_attn.{k}') + + # new_qkv = nn.Linear(qkv.in_features, _f(qkv.out_features), + # qkv.bias is not None, qkv.weight.device) + # indexes = l1_max_indexes(qkv.weight.data, 0) + + # new_qkv.weight.data.copy_(qkv.weight.data[indexes]) + # if qkv.bias is not None: + # new_qkv.bias.data.copy_(qkv.bias.data[indexes]) + # set_module(block, f'self_attn.{k}', new_qkv) + + # proj = block.self_attn.out_proj + # new_proj = nn.Linear(_f(proj.in_features), proj.out_features, + # proj.bias is not None, proj.weight.device) + # new_proj.weight.data.copy_(proj.weight.data[:, l1_max_indexes(proj.weight.data, 1)]) + # if proj.bias is not None: + # new_proj.bias.data.copy_(proj.bias.data) + # set_module(block, f'self_attn.out_proj', new_proj) + + # fc1 = block.mlp.fc1 + # new_fc1 = nn.Linear(fc1.in_features, _f(fc1.out_features), + # fc1.bias is not None, fc1.weight.device) + # indexes = l1_max_indexes(fc1.weight.data, 0) + # new_fc1.weight.data.copy_(fc1.weight.data[indexes]) + # if fc1.bias is not None: + # new_fc1.bias.data.copy_(fc1.bias.data[indexes]) + # set_module(block, f'mlp.fc1', new_fc1) + + # fc2 = block.mlp.fc2 + # new_fc2 = nn.Linear(_f(fc2.in_features), fc2.out_features, + # fc2.bias is not None, fc2.weight.device) + # new_fc2.weight.data.copy_(fc2.weight.data[:, l1_max_indexes(fc2.weight.data, 1)]) + # if fc2.bias is not None: + # new_fc2.bias.data.copy_(fc2.bias.data) + # set_module(block, f'mlp.fc2', new_fc2) + + + for block_i, block in enumerate(fm_vit.model.vision_model.encoder.layers): + # for k in ['k_proj', 'q_proj', 'v_proj']: + # qkv = get_module(block, f'self_attn.{k}') + + # new_qkv = nn.Linear(qkv.in_features, _f(qkv.out_features), + # qkv.bias is not None, qkv.weight.device) + # indexes = l1_max_indexes(qkv.weight.data, 0) + + # new_qkv.weight.data.copy_(qkv.weight.data[indexes]) + # if qkv.bias is not None: + # new_qkv.bias.data.copy_(qkv.bias.data[indexes]) + # set_module(block, f'self_attn.{k}', new_qkv) + + # proj = block.self_attn.out_proj + # new_proj = nn.Linear(_f(proj.in_features), proj.out_features, + # proj.bias is not None, proj.weight.device) + # new_proj.weight.data.copy_(proj.weight.data[:, l1_max_indexes(proj.weight.data, 1)]) + # if proj.bias is not None: + # new_proj.bias.data.copy_(proj.bias.data) + # set_module(block, f'self_attn.out_proj', new_proj) + + + # ------------------ + + qkv = block.self_attn.qkv + new_qkv = nn.Linear(qkv.in_features, _f(qkv.out_features), + qkv.bias is not None, qkv.weight.device) + indexes = l1_max_indexes(qkv.weight.data, 0) + + new_qkv.weight.data.copy_(qkv.weight.data[indexes]) + if qkv.bias is not None: + new_qkv.bias.data.copy_(qkv.bias.data[indexes]) + set_module(block, f'self_attn.qkv', new_qkv) + proj = block.self_attn.proj + new_proj = nn.Linear(_f(proj.in_features), proj.out_features, + proj.bias is not None, proj.weight.device) + new_proj.weight.data.copy_(proj.weight.data[:, l1_max_indexes(proj.weight.data, 1)]) + if proj.bias is not None: + new_proj.bias.data.copy_(proj.bias.data) + set_module(block, f'self_attn.proj', new_proj) + + # -------------------- + + fc1 = block.mlp.fc1 + new_fc1 = nn.Linear(fc1.in_features, _f(fc1.out_features), + fc1.bias is not None, fc1.weight.device) + indexes = l1_max_indexes(fc1.weight.data, 0) + new_fc1.weight.data.copy_(fc1.weight.data[indexes]) + if fc1.bias is not None: + new_fc1.bias.data.copy_(fc1.bias.data[indexes]) + set_module(block, f'mlp.fc1', new_fc1) + + fc2 = block.mlp.fc2 + new_fc2 = nn.Linear(_f(fc2.in_features), fc2.out_features, + fc2.bias is not None, fc2.weight.device) + new_fc2.weight.data.copy_(fc2.weight.data[:, l1_max_indexes(fc2.weight.data, 1)]) + if fc2.bias is not None: + new_fc2.bias.data.copy_(fc2.bias.data) + set_module(block, f'mlp.fc2', new_fc2) + + + return fm_vit + + + def init_md_from_fm_by_reducing_width_with_perf_test(self, fm: nn.Module, reducing_width_ratio: int, + samples: torch.Tensor) -> nn.Module: + fm_size = get_model_size(fm, True) + fm_latency = self._get_model_latency(fm, samples, 20, + get_model_device(fm), 20, False) + + master_dnn = self.init_md_from_fm_by_reducing_width(fm, reducing_width_ratio) + master_dnn_size = get_model_size(master_dnn, True) + logger.debug(f'inited master DNN: {master_dnn}') + # from utils.dl.common.model import get_module + # print('after generating') + # get_module(fm, 'head').debug() + # get_module(master_dnn, 'head').debug() + # print('test master latency') + master_dnn_latency = self._get_model_latency(master_dnn, samples, 20, + get_model_device(master_dnn), 20, False) + + logger.info(f'init master DNN (w/o FBS yet) by reducing foundation model\'s width (by {reducing_width_ratio:d}x)') + logger.info(f'foundation model ({fm_size:.3f}MB, {fm_latency:.4f}s/sample) -> ' + f'master DNN ({master_dnn_size:.3f}MB, {master_dnn_latency:.4f}s/sample)\n' + f'(model size: ↓ {(fm_size / master_dnn_size):.2f}x, ' + f'latency: ↓ {(fm_latency / master_dnn_latency):.2f}x)') + + return master_dnn + + def _get_model_latency(self, model: torch.nn.Module, model_input_size, sample_num: int, + device: str, warmup_sample_num: int, return_detail=False): + import time + + if isinstance(model_input_size, tuple): + dummy_input = torch.rand(model_input_size).to(device) + else: + dummy_input = model_input_size + + model = model.to(device) + model.eval() + + # warm up + with torch.no_grad(): + for _ in range(warmup_sample_num): + model(**dummy_input) + + infer_time_list = [] + + if device == 'cuda' or 'cuda' in str(device): + with torch.no_grad(): + for _ in range(sample_num): + s, e = torch.cuda.Event(enable_timing=True), torch.cuda.Event(enable_timing=True) + s.record() + model(**dummy_input) + e.record() + torch.cuda.synchronize() + cur_model_infer_time = s.elapsed_time(e) / 1000. + infer_time_list += [cur_model_infer_time] + + else: + with torch.no_grad(): + for _ in range(sample_num): + start = time.time() + model(**dummy_input) + cur_model_infer_time = time.time() - start + infer_time_list += [cur_model_infer_time] + + avg_infer_time = sum(infer_time_list) / sample_num + + if return_detail: + return avg_infer_time, infer_time_list + return avg_infer_time \ No newline at end of file diff --git a/new_impl/cv/elasticdnn/pipeline/offline/fm_to_md/test.py b/new_impl/cv/elasticdnn/pipeline/offline/fm_to_md/test.py new file mode 100644 index 0000000000000000000000000000000000000000..df3925ab7042fcf2094e2cb415506528a0469e55 --- /dev/null +++ b/new_impl/cv/elasticdnn/pipeline/offline/fm_to_md/test.py @@ -0,0 +1,28 @@ +from methods.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +import torch + + +def test(fm, fm_to_md_util: FM_to_MD_Util, samples: torch.Tensor): + master_dnn = fm_to_md_util.init_md_from_fm_by_reducing_width_with_perf_test(fm, 4, samples) + torch.save(master_dnn, 'tmp-master-dnn.pt') + +if __name__ == '__main__': + from utils.dl.common.env import set_random_seed + set_random_seed(1) + + # from torchvision.models import resnet50 + # from methods.elasticdnn.model.cnn import ElasticCNNUtil + # raw_cnn = resnet50() + # prunable_layers = [] + # for i in range(1, 5): + # for j in range([3, 4, 6, 3][i - 1]): + # prunable_layers += [f'layer{i}.{j}.conv1', f'layer{i}.{j}.conv2'] + # ignore_layers = [layer for layer, m in raw_cnn.named_modules() if isinstance(m, nn.Conv2d) and layer not in prunable_layers] + # test(raw_cnn, ignore_layers, ElasticCNNUtil(), torch.rand(2, 3, 224, 224)) + + + from dnns.vit import vit_b_16 + from methods.elasticdnn.pipeline.offline.fm_to_md.vit import FM_to_MD_ViT_Util + raw_vit = vit_b_16() + test(raw_vit.cuda(), FM_to_MD_ViT_Util(), torch.rand(2, 3, 224, 224).cuda()) + \ No newline at end of file diff --git a/new_impl/cv/elasticdnn/pipeline/offline/fm_to_md/vilt.py b/new_impl/cv/elasticdnn/pipeline/offline/fm_to_md/vilt.py new file mode 100644 index 0000000000000000000000000000000000000000..80829ed7440ee83860e979a94e7d2a9cf8f79b92 --- /dev/null +++ b/new_impl/cv/elasticdnn/pipeline/offline/fm_to_md/vilt.py @@ -0,0 +1,216 @@ + +import torch +from torch import nn +from copy import deepcopy + +from .base import FM_to_MD_Util +from utils.common.log import logger +from utils.dl.common.model import set_module, get_module, get_super_module +from utils.dl.common.model import get_model_device, get_model_latency, get_model_size +from utils.common.log import logger + +from transformers.models.vilt.modeling_vilt import ViltSelfAttention +from transformers import ViltConfig + +from typing import Optional, Tuple +import math + + +class ViltSelfAttentionPrunable(ViltSelfAttention): + def __init__(self): + config = ViltConfig.from_pretrained('dandelin/vilt-b32-mlm-itm') + super(ViltSelfAttentionPrunable, self).__init__(config) + + def transpose_for_scores(self, x): + new_x_shape = x.size()[:-1] + (self.num_attention_heads, -1) + x = x.view(*new_x_shape) + return x.permute(0, 2, 1, 3) + + def forward(self, hidden_states, attention_mask=None, head_mask=None, output_attentions=False): + mixed_query_layer = self.query(hidden_states) + + key_layer = self.transpose_for_scores(self.key(hidden_states)) + value_layer = self.transpose_for_scores(self.value(hidden_states)) + query_layer = self.transpose_for_scores(mixed_query_layer) + + # Take the dot product between "query" and "key" to get the raw attention scores. + attention_scores = torch.matmul(query_layer, key_layer.transpose(-1, -2)) + attention_scores = attention_scores / math.sqrt(self.attention_head_size) + if attention_mask is not None: + # Apply the attention mask is (precomputed for all layers in BertModel forward() function) + attention_scores = attention_scores + attention_mask + + # Normalize the attention scores to probabilities. + attention_probs = nn.Softmax(dim=-1)(attention_scores) + + # This is actually dropping out entire tokens to attend to, which might + # seem a bit unusual, but is taken from the original Transformer paper. + attention_probs = self.dropout(attention_probs) + + # Mask heads if we want to + if head_mask is not None: + attention_probs = attention_probs * head_mask + + context_layer = torch.matmul(attention_probs, value_layer) + + context_layer = context_layer.permute(0, 2, 1, 3).contiguous() + # new_context_layer_shape = context_layer.size()[:-2] + (self.all_head_size,) + new_context_layer_shape = context_layer.size()[:-2] + (-1,) + context_layer = context_layer.view(*new_context_layer_shape) + + outputs = (context_layer, attention_probs) if output_attentions else (context_layer,) + + return outputs + + @staticmethod + def init_from_exist_self_attn(attn: ViltSelfAttention): + # print(attn) + + res = ViltSelfAttentionPrunable() + + for attr in dir(attn): + # if str(attr) in ['transpose_for_scores'] or str(attr).startswith('_'): + # continue + # if isinstance(getattr(attn, attr), nn.Module): + # print(attr) + + if isinstance(getattr(attn, attr), nn.Module): + try: + # print(attr, 'ok') + setattr(res, attr, getattr(attn, attr)) + + except Exception as e: + print(attr, str(e)) + + + + return res + + +class FM_to_MD_Vilt_Util(FM_to_MD_Util): + def init_md_from_fm_by_reducing_width(self, fm: nn.Module, reducing_width_ratio: int) -> nn.Module: + fm_vit = deepcopy(fm) + + for block in fm_vit.vilt.encoder.layer: + set_module(block, 'attention.attention', ViltSelfAttentionPrunable.init_from_exist_self_attn(block.attention.attention)) + + def _f(n): + return int(n // reducing_width_ratio) + + # def _rand_indexes(n): + # return torch.randperm(n)[0: int(n // reducing_width_ratio)] + + def l1_max_indexes(p: torch.Tensor, dim=0): + assert dim in [0, 1] + assert p.dim() in [1, 2, 4] + + if dim == 1: + p = p.T + + p_norm = p.abs().contiguous().view(p.size(0), -1).sum(dim=1) + n = p.size(0) + return p_norm.argsort(descending=True)[0: int(n // reducing_width_ratio)].sort()[0] + + for block_i, block in enumerate(fm_vit.vilt.encoder.layer): + for k in ['query', 'key', 'value']: + qkv = get_module(block, f'attention.attention.{k}') + + new_qkv = nn.Linear(qkv.in_features, _f(qkv.out_features), + qkv.bias is not None, qkv.weight.device) + indexes = l1_max_indexes(qkv.weight.data, 0) + + new_qkv.weight.data.copy_(qkv.weight.data[indexes]) + if qkv.bias is not None: + new_qkv.bias.data.copy_(qkv.bias.data[indexes]) + set_module(block, f'attention.attention.{k}', new_qkv) + + proj = get_module(block, f'attention.output.dense') + new_proj = nn.Linear(_f(proj.in_features), proj.out_features, + proj.bias is not None, proj.weight.device) + new_proj.weight.data.copy_(proj.weight.data[:, l1_max_indexes(proj.weight.data, 1)]) + if proj.bias is not None: + new_proj.bias.data.copy_(proj.bias.data) + set_module(block, f'attention.output.dense', new_proj) + + fc1 = get_module(block, f'intermediate.dense') + new_fc1 = nn.Linear(fc1.in_features, _f(fc1.out_features), + fc1.bias is not None, fc1.weight.device) + indexes = l1_max_indexes(fc1.weight.data, 0) + new_fc1.weight.data.copy_(fc1.weight.data[indexes]) + if fc1.bias is not None: + new_fc1.bias.data.copy_(fc1.bias.data[indexes]) + set_module(block, f'intermediate.dense', new_fc1) + + fc2 = get_module(block, f'output.dense') + new_fc2 = nn.Linear(_f(fc2.in_features), fc2.out_features, + fc2.bias is not None, fc2.weight.device) + new_fc2.weight.data.copy_(fc2.weight.data[:, l1_max_indexes(fc2.weight.data, 1)]) + if fc2.bias is not None: + new_fc2.bias.data.copy_(fc2.bias.data) + set_module(block, f'output.dense', new_fc2) + + return fm_vit + + def init_md_from_fm_by_reducing_width_with_perf_test(self, fm: nn.Module, reducing_width_ratio: int, + samples: torch.Tensor) -> nn.Module: + fm_size = get_model_size(fm, True) + fm_latency = self._get_model_latency(fm, samples, 20, + get_model_device(fm), 20, False) + + master_dnn = self.init_md_from_fm_by_reducing_width(fm, reducing_width_ratio) + master_dnn_size = get_model_size(master_dnn, True) + logger.debug(f'inited master DNN: {master_dnn}') + master_dnn_latency = self._get_model_latency(master_dnn, samples, 20, + get_model_device(master_dnn), 20, False) + + logger.info(f'init master DNN (w/o FBS yet) by reducing foundation model\'s width (by {reducing_width_ratio:d}x)') + logger.info(f'foundation model ({fm_size:.3f}MB, {fm_latency:.4f}s/sample) -> ' + f'master DNN ({master_dnn_size:.3f}MB, {master_dnn_latency:.4f}s/sample)\n' + f'(model size: ↓ {(fm_size / master_dnn_size):.2f}x, ' + f'latency: ↓ {(fm_latency / master_dnn_latency):.2f}x)') + + return master_dnn + + def _get_model_latency(self, model: torch.nn.Module, model_input_size, sample_num: int, + device: str, warmup_sample_num: int, return_detail=False): + import time + + if isinstance(model_input_size, tuple): + dummy_input = torch.rand(model_input_size).to(device) + else: + dummy_input = model_input_size + + model = model.to(device) + model.eval() + + # warm up + with torch.no_grad(): + for _ in range(warmup_sample_num): + model(**dummy_input) + + infer_time_list = [] + + if device == 'cuda' or 'cuda' in str(device): + with torch.no_grad(): + for _ in range(sample_num): + s, e = torch.cuda.Event(enable_timing=True), torch.cuda.Event(enable_timing=True) + s.record() + model(**dummy_input) + e.record() + torch.cuda.synchronize() + cur_model_infer_time = s.elapsed_time(e) / 1000. + infer_time_list += [cur_model_infer_time] + + else: + with torch.no_grad(): + for _ in range(sample_num): + start = time.time() + model(**dummy_input) + cur_model_infer_time = time.time() - start + infer_time_list += [cur_model_infer_time] + + avg_infer_time = sum(infer_time_list) / sample_num + + if return_detail: + return avg_infer_time, infer_time_list + return avg_infer_time \ No newline at end of file diff --git a/new_impl/cv/elasticdnn/pipeline/offline/fm_to_md/vit.py b/new_impl/cv/elasticdnn/pipeline/offline/fm_to_md/vit.py new file mode 100644 index 0000000000000000000000000000000000000000..f2ccd2ecce1b4af5765c5362eafadc9569b51928 --- /dev/null +++ b/new_impl/cv/elasticdnn/pipeline/offline/fm_to_md/vit.py @@ -0,0 +1,352 @@ + +import torch +from torch import nn +from copy import deepcopy +import math +from .base import FM_to_MD_Util +from utils.common.log import logger +from utils.dl.common.model import set_module, get_module, get_super_module + +from transformers.models.cvt.modeling_cvt import CvtSelfAttention,CvtAttention +from transformers import CvtConfig + + +class CvtSelfAttentionPrunable(CvtSelfAttention): + def __init__(self,embed_dim,num_heads,with_cls_token): + config = CvtConfig.from_pretrained('/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/new_impl/cv/cvt_model') + + super(CvtSelfAttentionPrunable, self).__init__(num_heads = num_heads,embed_dim = embed_dim, + kernel_size = 3,padding_q = config.padding_q, + padding_kv = config.padding_kv,stride_q = config.stride_q, + stride_kv = config.stride_kv,qkv_projection_method = config.qkv_projection_method, + qkv_bias = config.qkv_bias,attention_drop_rate = 0.0,with_cls_token=with_cls_token) + + # def transpose_for_scores(self, x): + # new_x_shape = x.size()[:-1] + (self.num_heads, -1) + # x = x.view(*new_x_shape) + # return x.permute(0, 2, 1, 3) + + def rearrange_for_multi_head_attention(self, hidden_state): + batch_size, hidden_size, _ = hidden_state.shape + head_dim = self.embed_dim // self.num_heads + # rearrange 'b t (h d) -> b h t d' + return hidden_state.view(batch_size, hidden_size, self.num_heads, -1).permute(0, 2, 1, 3) + + def forward(self, hidden_state, height, width): + # mixed_query_layer = self.projection_query(hidden_states) + # key_layer = self.transpose_for_scores(self.projection_key(hidden_states)) + # value_layer = self.transpose_for_scores(self.projection_value(hidden_states)) + # query_layer = self.transpose_for_scores(mixed_query_layer) + + # # Take the dot product between "query" and "key" to get the raw attention scores. + # attention_scores = torch.matmul(query_layer, key_layer.transpose(-1, -2)) + # #attention_scores = attention_scores / math.sqrt(self.attention_head_size) + # if attention_mask is not None: + # # Apply the attention mask is (precomputed for all layers in BertModel forward() function) + # attention_scores = attention_scores + attention_mask + + # # Normalize the attention scores to probabilities. + # attention_probs = nn.Softmax(dim=-1)(attention_scores) + + # # This is actually dropping out entire tokens to attend to, which might + # # seem a bit unusual, but is taken from the original Transformer paper. + # attention_probs = self.dropout(attention_probs) + + # # Mask heads if we want to + # if head_mask is not None: + # attention_probs = attention_probs * head_mask + + # context_layer = torch.matmul(attention_probs, value_layer) + + # context_layer = context_layer.permute(0, 2, 1, 3).contiguous() + # # new_context_layer_shape = context_layer.size()[:-2] + (self.all_head_size,) + # new_context_layer_shape = context_layer.size()[:-2] + (-1,) + # context_layer = context_layer.view(*new_context_layer_shape) + + # outputs = (context_layer, attention_probs) if output_attentions else (context_layer,) + + # return outputs + if self.with_cls_token: + cls_token, hidden_state = torch.split(hidden_state, [1, height * width], 1) + batch_size, hidden_size, num_channels = hidden_state.shape + # rearrange "b (h w) c -> b c h w" + + hidden_state = hidden_state.permute(0, 2, 1).view(batch_size, num_channels, height, width) + key = self.convolution_projection_key(hidden_state) + query = self.convolution_projection_query(hidden_state) + value = self.convolution_projection_value(hidden_state) + + if self.with_cls_token: + query = torch.cat((cls_token, query), dim=1) + key = torch.cat((cls_token, key), dim=1) + value = torch.cat((cls_token, value), dim=1) + + head_dim = self.embed_dim // self.num_heads + query = self.rearrange_for_multi_head_attention(self.projection_query(query)) + key = self.rearrange_for_multi_head_attention(self.projection_key(key)) + value = self.rearrange_for_multi_head_attention(self.projection_value(value)) + + attention_score = torch.einsum("bhlk,bhtk->bhlt", [query, key]) * self.scale + attention_probs = torch.nn.functional.softmax(attention_score, dim=-1) + attention_probs = self.dropout(attention_probs) + + context = torch.einsum("bhlt,bhtv->bhlv", [attention_probs, value]) + # rearrange"b h t d -> b t (h d)" + _, _, hidden_size, _ = context.shape + #context = context.permute(0, 2, 1, 3).contiguous().view(batch_size, hidden_size, self.num_heads * head_dim) + context = context.permute(0, 2, 1, 3).contiguous().view(batch_size, hidden_size, -1) + return context + @staticmethod + def init_from_exist_self_attn(attn: CvtSelfAttention,embed_dim: float,num_heads,cls_token): + # print(attn) + + res = CvtSelfAttentionPrunable(embed_dim,num_heads,with_cls_token = cls_token) + for attr in dir(attn): + # if str(attr) in ['transpose_for_scores'] or str(attr).startswith('_'): + # continue + # if isinstance(getattr(attn, attr), nn.Module): + # print(attr) + + if isinstance(getattr(attn, attr), nn.Module): + try: + #print(attr, 'ok') + setattr(res, attr, getattr(attn, attr)) + + except Exception as e: + print(attr, str(e)) + + return res + + +class FM_to_MD_ViT_Util(FM_to_MD_Util): + def init_md_from_fm_by_reducing_width(self, fm: nn.Module, reducing_width_ratio: int) -> nn.Module: + fm_vit = deepcopy(fm) + config = [64,192,384] + num_heads = [1,3,6] + cls_token = [False,False,True] + for block_1_i,block_1 in enumerate(fm_vit.cvt.encoder.stages): + + for block_i, block in enumerate(block_1.layers): + set_module(block, 'attention.attention', CvtSelfAttentionPrunable.init_from_exist_self_attn(block.attention.attention,config[block_1_i],num_heads[block_1_i],cls_token=cls_token[block_1_i])) + + def _f(n): + return int(n // reducing_width_ratio) + + # def _rand_indexes(n): + # return torch.randperm(n)[0: int(n // reducing_width_ratio)] + + def l1_max_indexes(p: torch.Tensor, dim=0): + assert dim in [0, 1] + assert p.dim() in [1, 2, 4] + + if dim == 1: + p = p.T + + p_norm = p.abs().contiguous().view(p.size(0), -1).sum(dim=1) + n = p.size(0) + return p_norm.argsort(descending=True)[0: int(n // reducing_width_ratio)].sort()[0] + + # first_attn = True + # print(fm_vit) + for block_1_i,block_1 in enumerate(fm_vit.cvt.encoder.stages): + + for block_i, block in enumerate(block_1.layers): + + for k in ['projection_query','projection_key','projection_value']: + qkv = get_module(block, f'attention.attention.{k}') + #print(qkv.weight.data.shape) + new_qkv = nn.Linear(qkv.in_features, _f(qkv.out_features), + qkv.bias is not None, qkv.weight.device) + indexes = l1_max_indexes(qkv.weight.data, 0) + + new_qkv.weight.data.copy_(qkv.weight.data[indexes]) + if qkv.bias is not None: + new_qkv.bias.data.copy_(qkv.bias.data[indexes]) + set_module(block, f'attention.attention.{k}', new_qkv) + + # proj_1 = get_module(block, f'attention.output') + + # new_proj_1 = nn.Linear(_f(proj.in_features), _f(proj.out_features), + # proj.bias is not None, proj.weight.device) + # new_proj_1.weight.data.copy_(proj_1.weight.data[:, l1_max_indexes(proj_1.weight.data, 1)]) + # if proj_1.bias is not None: + # new_proj_1.bias.data.copy_(proj_1.bias.data) + # set_module(block, f'attention.output', new_proj_1) + + proj = get_module(block, f'attention.output.dense') + + new_proj = nn.Linear(_f(proj.in_features), proj.out_features, + proj.bias is not None, proj.weight.device) + new_proj.weight.data.copy_(proj.weight.data[:, l1_max_indexes(proj.weight.data, 1)]) + #new_proj.weight.data.copy_(proj.weight.data[l1_max_indexes(proj.weight.data, 1)]) + if proj.bias is not None: + new_proj.bias.data.copy_(proj.bias.data) + set_module(block, f'attention.output.dense', new_proj) + + fc1 = get_module(block, f'intermediate.dense') + + new_fc1 = nn.Linear(fc1.in_features, _f(fc1.out_features), + fc1.bias is not None, fc1.weight.device) + indexes = l1_max_indexes(fc1.weight.data, 0) + new_fc1.weight.data.copy_(fc1.weight.data[indexes]) + if fc1.bias is not None: + new_fc1.bias.data.copy_(fc1.bias.data[indexes]) + set_module(block, f'intermediate.dense', new_fc1) + + fc2 = get_module(block, f'output.dense') + + new_fc2 = nn.Linear(_f(fc2.in_features), fc2.out_features, + fc2.bias is not None, fc2.weight.device) + new_fc2.weight.data.copy_(fc2.weight.data[:, l1_max_indexes(fc2.weight.data, 1)]) + #new_fc2.weight.data.copy_(fc2.weight.data[l1_max_indexes(fc2.weight.data, 0)]) + if fc2.bias is not None: + new_fc2.bias.data.copy_(fc2.bias.data) + set_module(block, f'output.dense', new_fc2) + return fm_vit + # for block_i, block in enumerate(fm_vit.cvt.encoder.stages.layers): + + + # for k in ['projection_query','projection_key','projection_value']: + # qkv = get_module(block, f'attention.attention.{k}') + + # new_qkv = nn.Linear(qkv.in_features, _f(qkv.out_features), + # qkv.bias is not None, qkv.weight.device) + # indexes = l1_max_indexes(qkv.weight.data, 0) + + # new_qkv.weight.data.copy_(qkv.weight.data[indexes]) + # if qkv.bias is not None: + # new_qkv.bias.data.copy_(qkv.bias.data[indexes]) + # set_module(block, f'attention.attention.{k}', new_qkv) + + # # proj_1 = get_module(block, f'attention.output') + + # # new_proj_1 = nn.Linear(_f(proj.in_features), _f(proj.out_features), + # # proj.bias is not None, proj.weight.device) + # # new_proj_1.weight.data.copy_(proj_1.weight.data[:, l1_max_indexes(proj_1.weight.data, 1)]) + # # if proj_1.bias is not None: + # # new_proj_1.bias.data.copy_(proj_1.bias.data) + # # set_module(block, f'attention.output', new_proj_1) + + # proj = get_module(block, f'attention.output.dense') + + # new_proj = nn.Linear(_f(proj.in_features), proj.out_features, + # proj.bias is not None, proj.weight.device) + # new_proj.weight.data.copy_(proj.weight.data[:, l1_max_indexes(proj.weight.data, 1)]) + # if proj.bias is not None: + # new_proj.bias.data.copy_(proj.bias.data) + # set_module(block, f'attention.output.dense', new_proj) + + # fc1 = get_module(block, f'intermediate.dense') + # new_fc1 = nn.Linear(fc1.in_features, _f(fc1.out_features), + # fc1.bias is not None, fc1.weight.device) + # indexes = l1_max_indexes(fc1.weight.data, 0) + # new_fc1.weight.data.copy_(fc1.weight.data[indexes]) + # if fc1.bias is not None: + # new_fc1.bias.data.copy_(fc1.bias.data[indexes]) + # set_module(block, f'intermediate.dense', new_fc1) + + # fc2 = get_module(block, f'output.dense') + # new_fc2 = nn.Linear(_f(fc2.in_features), fc2.out_features, + # fc2.bias is not None, fc2.weight.device) + # new_fc2.weight.data.copy_(fc2.weight.data[:, l1_max_indexes(fc2.weight.data, 1)]) + # if fc2.bias is not None: + # new_fc2.bias.data.copy_(fc2.bias.data) + # set_module(block, f'output.dense', new_fc2) + # return fm_vit + # for block_i, block in enumerate(fm_vit.blocks): + # qkv = block.attn.qkv + + # new_qkv = nn.Linear(qkv.in_features, _f(qkv.out_features), + # qkv.bias is not None, qkv.weight.device) + # indexes = l1_max_indexes(qkv.weight.data, 0) + + # new_qkv.weight.data.copy_(qkv.weight.data[indexes]) + # if qkv.bias is not None: + # new_qkv.bias.data.copy_(qkv.bias.data[indexes]) + # set_module(fm_vit, f'blocks.{block_i}.attn.qkv', new_qkv) + + # proj = block.attn.proj + # new_proj = nn.Linear(_f(proj.in_features), proj.out_features, + # proj.bias is not None, proj.weight.device) + # new_proj.weight.data.copy_(proj.weight.data[:, l1_max_indexes(proj.weight.data, 1)]) + # if proj.bias is not None: + # new_proj.bias.data.copy_(proj.bias.data) + # set_module(fm_vit, f'blocks.{block_i}.attn.proj', new_proj) + + # fc1 = block.mlp.fc1 + # new_fc1 = nn.Linear(fc1.in_features, _f(fc1.out_features), + # fc1.bias is not None, fc1.weight.device) + # indexes = l1_max_indexes(fc1.weight.data, 0) + # new_fc1.weight.data.copy_(fc1.weight.data[indexes]) + # if fc1.bias is not None: + # new_fc1.bias.data.copy_(fc1.bias.data[indexes]) + # set_module(fm_vit, f'blocks.{block_i}.mlp.fc1', new_fc1) + + # fc2 = block.mlp.fc2 + # new_fc2 = nn.Linear(_f(fc2.in_features), fc2.out_features, + # fc2.bias is not None, fc2.weight.device) + # new_fc2.weight.data.copy_(fc2.weight.data[:, l1_max_indexes(fc2.weight.data, 1)]) + # if fc2.bias is not None: + # new_fc2.bias.data.copy_(fc2.bias.data) + # set_module(fm_vit, f'blocks.{block_i}.mlp.fc2', new_fc2) + # # reduce dim_embedding + # # if name.endswith('patch_embed.proj'): + # # continue + + # # new_layer = nn.Conv2d(module.in_channels, _f(module.out_channels), module.kernel_size, module.stride, + # # module.padding, module.dilation, module.groups, module.bias is not None, module.padding_mode, + # # module.weight.device) + + # # rand_indexes = l1_max_indexes(module.weight.data) + # # new_layer.weight.data.copy_(module.weight.data[rand_indexes]) + # # if new_layer.bias is not None: + # # new_layer.bias.data.copy_(module.bias.data[rand_indexes]) + + # # fm_vit.cls_token.data = fm_vit.cls_token.data[:, :, rand_indexes] + # # fm_vit.pos_embed.data = fm_vit.pos_embed.data[:, :, rand_indexes] + + # # elif isinstance(module, nn.Linear): + + # # if 'head' in name: + # # continue + + # # new_layer = nn.Linear(_f(module.in_features), module.out_features, + # # module.bias is not None, module.weight.device) + # # new_layer.weight.data.copy_(module.weight.data[:, l1_max_indexes(module.weight.data, 1)]) + # # if new_layer.bias is not None: + # # new_layer.bias.data.copy_(module.bias.data) + # # else: + # # first_attn = False + # # if first_attn: + # # first_attn = False + # # new_layer = nn.Linear(module.in_features, _f(module.out_features), + # # module.bias is not None, module.weight.device) + + # # rand_indexes = l1_max_indexes(module.weight.data) + # # new_layer.weight.data.copy_(module.weight.data[rand_indexes]) + # # if new_layer.bias is not None: + # # new_layer.bias.data.copy_(module.bias.data[rand_indexes]) + # # else: + # # new_layer = nn.Linear(_f(module.in_features), _f(module.out_features), + # # module.bias is not None, module.weight.device) + + # # rand_indexes = l1_max_indexes(module.weight.data) + # # new_layer.weight.data.copy_(module.weight.data[rand_indexes][:, l1_max_indexes(module.weight.data, 1)]) + # # if new_layer.bias is not None: + # # new_layer.bias.data.copy_(module.bias.data[rand_indexes]) + + # # elif isinstance(module, nn.LayerNorm) and ('block' in name or name == 'norm' or name == 'norm.0'): + # # new_layer = nn.LayerNorm(_f(module.normalized_shape[0]), eps=module.eps, device=module.weight.device) + # # rand_indexes = l1_max_indexes(module.weight.data) + # # new_layer.weight.data.copy_(module.weight.data[rand_indexes]) + # # new_layer.bias.data.copy_(module.bias.data[rand_indexes]) + + # # else: + # # continue + + # # original_layer_str = str(module) + # # set_module(fm_vit, name, new_layer) + # # logger.debug(f'set_module, {name}, {new_layer}') + # # logger.debug(f'slim {name} from {original_layer_str} to {new_layer}') + + # return fm_vit \ No newline at end of file diff --git a/new_impl/cv/elasticdnn/source-code-line-num.csv b/new_impl/cv/elasticdnn/source-code-line-num.csv new file mode 100644 index 0000000000000000000000000000000000000000..13aa6b900cdf5c6db049587f57afce39de4d3d4a --- /dev/null +++ b/new_impl/cv/elasticdnn/source-code-line-num.csv @@ -0,0 +1,34 @@ +file_path,line_num,file_size +/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/methods/elasticdnn/model/test.py,138,4455 +/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/methods/elasticdnn/model/base.py,139,5797 +/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/methods/elasticdnn/model/vit.py,343,16707 +/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/methods/elasticdnn/model/vit_new.py,341,15609 +/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/methods/elasticdnn/model/clip.py,316,14598 +/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/methods/elasticdnn/model/vilt.py,359,15983 +/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/methods/elasticdnn/model/bert.py,359,15967 +/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/methods/elasticdnn/model/__init__.py,4,98 +/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/methods/elasticdnn/model/cnn.py,147,6260 +/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/methods/elasticdnn/api/algs/fm_lora.py,146,6086 +/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/methods/elasticdnn/api/algs/md_pretraining_index_v2_train_index_and_md.py,529,24650 +/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/methods/elasticdnn/api/algs/md_pretraining_wo_fbs.py,381,17388 +/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/methods/elasticdnn/api/algs/md_pretraining_index_bk_too_slow.py,427,19855 +/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/methods/elasticdnn/api/algs/md_pretraining_index_v1_train_index_and_md.py,501,23379 +/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/methods/elasticdnn/api/algs/md_pretraining_wo_fbs_clip_debug.py,412,19146 +/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/methods/elasticdnn/api/algs/md_pretraining_bk.py,292,13039 +/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/methods/elasticdnn/api/algs/md_pretraining_index.py,322,15329 +/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/methods/elasticdnn/api/algs/md_pretraining_v1.py,316,14640 +/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/methods/elasticdnn/api/algs/md_pretraining_w_fbs.py,192,8653 +/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/methods/elasticdnn/api/online_model_v2.py,269,11807 +/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/methods/elasticdnn/api/model.py,821,29849 +/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/methods/elasticdnn/api/online_model.py,246,10834 +/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/methods/elasticdnn/pipeline/offline/fm_lora/base.py,36,932 +/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/methods/elasticdnn/pipeline/offline/fm_lora/vit.py,79,2484 +/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/methods/elasticdnn/pipeline/offline/fm_lora/clip.py,86,2780 +/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/methods/elasticdnn/pipeline/offline/fm_lora/vilt.py,91,2768 +/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/methods/elasticdnn/pipeline/offline/fm_lora/bert.py,81,2422 +/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/methods/elasticdnn/pipeline/offline/fm_to_md/test.py,28,1180 +/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/methods/elasticdnn/pipeline/offline/fm_to_md/base.py,48,2308 +/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/methods/elasticdnn/pipeline/offline/fm_to_md/vit.py,130,6544 +/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/methods/elasticdnn/pipeline/offline/fm_to_md/clip.py,672,30837 +/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/methods/elasticdnn/pipeline/offline/fm_to_md/vilt.py,216,9424 +/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/methods/elasticdnn/pipeline/offline/fm_to_md/bert.py,279,13316 diff --git a/new_impl/cv/feat_align/__pycache__/main.cpython-38.pyc b/new_impl/cv/feat_align/__pycache__/main.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c0ccce7bdf88e55d85a96c45c8399ea054fcef6a Binary files /dev/null and b/new_impl/cv/feat_align/__pycache__/main.cpython-38.pyc differ diff --git a/new_impl/cv/feat_align/__pycache__/main_gpt_neo.cpython-38.pyc b/new_impl/cv/feat_align/__pycache__/main_gpt_neo.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4531ab3ed4e48dc2c5881869304e1956fde79e88 Binary files /dev/null and b/new_impl/cv/feat_align/__pycache__/main_gpt_neo.cpython-38.pyc differ diff --git a/new_impl/cv/feat_align/__pycache__/mmd.cpython-38.pyc b/new_impl/cv/feat_align/__pycache__/mmd.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2dbf7be3090676f5f3a4d7739adf368a106c6d43 Binary files /dev/null and b/new_impl/cv/feat_align/__pycache__/mmd.cpython-38.pyc differ diff --git a/new_impl/cv/feat_align/main.py b/new_impl/cv/feat_align/main.py new file mode 100644 index 0000000000000000000000000000000000000000..94f59bde3dc2c7a0ff4f1b51928aeb266f62eddd --- /dev/null +++ b/new_impl/cv/feat_align/main.py @@ -0,0 +1,208 @@ +from typing import Any, Dict, List +from schema import Schema +from data import Scenario, MergedDataset +from methods.base.alg import BaseAlg +from methods.base.model import BaseModel +from data import build_dataloader +import torch.optim +import tqdm +import os +import time +from abc import abstractmethod +import matplotlib.pyplot as plt +from torchvision.transforms import Compose + +class OnlineFeatAlignModel(BaseModel): + def get_required_model_components(self) -> List[str]: + return ['main'] + + @abstractmethod + def get_feature_hook(self): + pass + + @abstractmethod + def forward_to_get_task_loss(self, x, y): + pass + + @abstractmethod + def get_trained_params(self): + pass + + @abstractmethod + def get_mmd_loss(self, f1, f2): + pass + + +class FeatAlignAlg(BaseAlg): + def get_required_models_schema(self) -> Schema: + return Schema({ + 'main': OnlineFeatAlignModel + }) + + def get_required_hyp_schema(self) -> Schema: + from schema import Optional + return Schema({ + 'train_batch_size': int, + 'val_batch_size': int, + 'num_workers': int, + 'optimizer': str, + 'optimizer_args': dict, + 'scheduler': str, + 'scheduler_args': dict, + 'num_iters': int, + 'val_freq': int, + 'feat_align_loss_weight': float, + Optional('transform'): Compose, + }) + + def run(self, scenario: Scenario, hyps: Dict, collate_fn=None) -> Dict[str, Any]: + super().run(scenario, hyps) + + assert isinstance(self.models['main'], OnlineFeatAlignModel) # for auto completion + + cur_domain_name = scenario.target_domains_order[scenario.cur_domain_index] + if 'transform' in hyps.keys(): + datasets_for_training = scenario.get_online_cur_domain_datasets_for_training(transform=hyps['transform']) + else: + datasets_for_training = scenario.get_online_cur_domain_datasets_for_training() + train_dataset = datasets_for_training[cur_domain_name]['train'] + val_dataset = datasets_for_training[cur_domain_name]['val'] + if 'transform' in hyps.keys(): + datasets_for_inference = scenario.get_online_cur_domain_datasets_for_inference(transform=hyps['transform']) + else: + datasets_for_inference = scenario.get_online_cur_domain_datasets_for_inference() + test_dataset = datasets_for_inference + + train_loader = iter(build_dataloader(train_dataset, hyps['train_batch_size'], hyps['num_workers'], + True, None, collate_fn=collate_fn)) + test_loader = build_dataloader(test_dataset, hyps['val_batch_size'], hyps['num_workers'], + False, False, collate_fn=collate_fn) + + source_datasets = [d['train'] for n, d in datasets_for_training.items() if n != cur_domain_name] + source_dataset = MergedDataset(source_datasets) + source_train_loader = iter(build_dataloader(source_dataset, hyps['train_batch_size'], hyps['num_workers'], + True, None, collate_fn=collate_fn)) + + # 1. generate surrogate DNN + # for n, m in self.models['main'].models_dict['md'].named_modules(): + # if isinstance(m, nn.Linear): + # m.reset_parameters() + # from utils.dl.common.model import set_module + + # for n, m in self.models['main'].models_dict['md'].named_modules(): + # if m.__class__.__name__ == 'KTakesAll': + # set_module(self.models['main'].models_dict['md'], n, KTakesAll(0.5)) + # self.models['main'].set_sd_sparsity(hyps['sd_sparsity']) + device = self.models['main'].device + # surrogate_dnn = self.models['main'].generate_sd_by_target_samples(next(train_loader)[0].to(device)) + # self.models['sd'] = surrogate_dnn + + # 2. train surrogate DNN + # TODO: train only a part of filters + trained_params = self.models['main'].get_trained_params() + optimizer = torch.optim.__dict__[hyps['optimizer']](trained_params, **hyps['optimizer_args']) + if hyps['scheduler'] != '': + scheduler = torch.optim.lr_scheduler.__dict__[hyps['scheduler']](optimizer, **hyps['scheduler_args']) + else: + scheduler = None + + pbar = tqdm.tqdm(range(hyps['num_iters']), dynamic_ncols=True, desc='da...') + task_losses, mmd_losses = [], [] + accs = [] + + total_train_time = 0. + + feature_hook = self.models['main'].get_feature_hook() + + for iter_index in pbar: + + if iter_index % hyps['val_freq'] == 0: + from data import split_dataset + if 'transform' in hyps.keys(): + cur_test_batch_dataset = split_dataset(test_dataset, hyps['val_batch_size'], iter_index, transform=hyps['transform'])[0] + else: + cur_test_batch_dataset = split_dataset(test_dataset, hyps['val_batch_size'], iter_index)[0] + cur_test_batch_dataloader = build_dataloader(cur_test_batch_dataset, hyps['train_batch_size'], hyps['num_workers'], False, False, collate_fn=collate_fn) + cur_acc = self.models['main'].get_accuracy(cur_test_batch_dataloader) + # cur_acc = self.models['main'].get_accuracy(test_loader) + accs += [{ + 'iter': iter_index, + 'acc': cur_acc + }] + + cur_start_time = time.time() + + self.models['main'].to_train_mode() + + x, _ = next(train_loader) + + if isinstance(x, dict): + for k, v in x.items(): + if isinstance(v, torch.Tensor): + x[k] = v.to(device) + else: + x = x.to(device) + + source_x, source_y = next(source_train_loader) + + if isinstance(source_x, dict): + for k, v in source_x.items(): + if isinstance(v, torch.Tensor): + source_x[k] = v.to(device) + source_y = source_y.to(device) + else: + source_x, source_y = source_x.to(device), source_y.to(device) + + task_loss = self.models['main'].forward_to_get_task_loss(source_x, source_y) + source_features = feature_hook.input + + self.models['main'].infer(x) + target_features = feature_hook.input + if(len(source_features.size())>2): + source_features = torch.squeeze(source_features,-1) + source_features = torch.squeeze(source_features,-1) + target_features = torch.squeeze(target_features,-1) + target_features = torch.squeeze(target_features,-1) + + mmd_loss = hyps['feat_align_loss_weight'] * self.models['main'].get_mmd_loss(source_features, target_features) + + loss = task_loss + mmd_loss + + optimizer.zero_grad() + loss.backward() + optimizer.step() + if scheduler is not None: + scheduler.step() + + pbar.set_description(f'da... | cur_acc: {cur_acc:.4f}, task_loss: {task_loss:.6f}, mmd_loss: {mmd_loss:.6f}') + task_losses += [float(task_loss.cpu().item())] + mmd_losses += [float(mmd_loss.cpu().item())] + + total_train_time += time.time() - cur_start_time + + feature_hook.remove() + + time_usage = total_train_time + + plt.plot(task_losses, label='task') + plt.plot(mmd_losses, label='mmd') + plt.xlabel('iteration') + plt.ylabel('loss') + plt.savefig(os.path.join(self.res_save_dir, 'loss.png')) + plt.clf() + + if 'transform' in hyps.keys(): + cur_test_batch_dataset = split_dataset(test_dataset, hyps['train_batch_size'], iter_index + 1, transform=hyps['transform'])[0] + else: + cur_test_batch_dataset = split_dataset(test_dataset, hyps['train_batch_size'], iter_index + 1)[0] + cur_test_batch_dataloader = build_dataloader(cur_test_batch_dataset, len(cur_test_batch_dataset), hyps['num_workers'], False, False, collate_fn=collate_fn) + cur_acc = self.models['main'].get_accuracy(cur_test_batch_dataloader) + accs += [{ + 'iter': iter_index + 1, + 'acc': cur_acc + }] + + return { + 'accs': accs, + 'time': time_usage + }, self.models \ No newline at end of file diff --git a/new_impl/cv/feat_align/main_glip.py b/new_impl/cv/feat_align/main_glip.py new file mode 100644 index 0000000000000000000000000000000000000000..49657e623720e2c7f1a068a952541385163eefaa --- /dev/null +++ b/new_impl/cv/feat_align/main_glip.py @@ -0,0 +1,233 @@ +from typing import Any, Dict, List +from schema import Schema +from data import Scenario, MergedDataset +from methods.base.alg import BaseAlg +from methods.base.model import BaseModel +from data import build_dataloader +import torch.optim +import tqdm +import os +import time +from abc import abstractmethod +import matplotlib.pyplot as plt +from torchvision.transforms import Compose +import torch.nn.functional as F + +class OnlineFeatAlignModel(BaseModel): + def get_required_model_components(self) -> List[str]: + return ['main'] + + @abstractmethod + def get_feature_hook(self): + pass + + @abstractmethod + def forward_to_get_task_loss(self, x, y): + pass + + @abstractmethod + def get_trained_params(self): + pass + + @abstractmethod + def get_mmd_loss(self, f1, f2): + pass + + +class FeatAlignAlg(BaseAlg): + def get_required_models_schema(self) -> Schema: + return Schema({ + 'main': OnlineFeatAlignModel + }) + + def get_required_hyp_schema(self) -> Schema: + from schema import Optional + return Schema({ + 'train_batch_size': int, + 'val_batch_size': int, + 'num_workers': int, + 'optimizer': str, + 'optimizer_args': dict, + 'scheduler': str, + 'scheduler_args': dict, + 'num_iters': int, + 'val_freq': int, + 'feat_align_loss_weight': float, + Optional('transform'): Compose, + }) + + def run(self, scenario: Scenario, hyps: Dict, collate_fn=None) -> Dict[str, Any]: + super().run(scenario, hyps) + + assert isinstance(self.models['main'], OnlineFeatAlignModel) # for auto completion + + cur_domain_name = scenario.target_domains_order[scenario.cur_domain_index] + if 'transform' in hyps.keys(): + datasets_for_training = scenario.get_online_cur_domain_datasets_for_training(transform=hyps['transform']) + else: + datasets_for_training = scenario.get_online_cur_domain_datasets_for_training() + train_dataset = datasets_for_training[cur_domain_name]['train'] + + if 'transform' in hyps.keys(): + datasets_for_inference = scenario.get_online_cur_domain_datasets_for_inference(transform=hyps['transform']) + else: + datasets_for_inference = scenario.get_online_cur_domain_datasets_for_inference() + test_dataset = datasets_for_inference + + train_loader = iter(build_dataloader(train_dataset, hyps['train_batch_size'], hyps['num_workers'], + True, None, collate_fn=collate_fn)) + # test_loader = build_dataloader(test_dataset, hyps['val_batch_size'], hyps['num_workers'], + # False, False, collate_fn=collate_fn) + + source_datasets = [d['train'] for n, d in datasets_for_training.items() if n != cur_domain_name] + source_dataset = MergedDataset(source_datasets) + # source_dataset_val = [d['val'] for n, d in datasets_for_training.items() if n != cur_domain_name][0] + source_train_loader = iter(build_dataloader(source_dataset, hyps['train_batch_size'], hyps['num_workers'], + True, None, collate_fn=collate_fn)) + # source_val_loader = build_dataloader(source_dataset_val, hyps['val_batch_size'], hyps['num_workers'], + # False, False, collate_fn=collate_fn) + # 1. generate surrogate DNN + # for n, m in self.models['main'].models_dict['md'].named_modules(): + # if isinstance(m, nn.Linear): + # m.reset_parameters() + # from utils.dl.common.model import set_module + + # for n, m in self.models['main'].models_dict['md'].named_modules(): + # if m.__class__.__name__ == 'KTakesAll': + # set_module(self.models['main'].models_dict['md'], n, KTakesAll(0.5)) + # self.models['main'].set_sd_sparsity(hyps['sd_sparsity']) + device = self.models['main'].device + # surrogate_dnn = self.models['main'].generate_sd_by_target_samples(next(train_loader)[0].to(device)) + # self.models['sd'] = surrogate_dnn + + # 2. train surrogate DNN + # TODO: train only a part of filters + trained_params = self.models['main'].get_trained_params() + optimizer = torch.optim.__dict__[hyps['optimizer']](trained_params, **hyps['optimizer_args']) + if hyps['scheduler'] != '': + scheduler = torch.optim.lr_scheduler.__dict__[hyps['scheduler']](optimizer, **hyps['scheduler_args']) + else: + scheduler = None + + pbar = tqdm.tqdm(range(hyps['num_iters']), dynamic_ncols=True, desc='da...') + task_losses, mmd_losses = [], [] + accs = [] + + total_train_time = 0. + + feature_hook = self.models['main'].get_feature_hook() + + num_test = 256 + + for iter_index in pbar: + + if iter_index % hyps['val_freq'] == 0: + from data import split_dataset + if 'transform' in hyps.keys(): + cur_test_batch_dataset = split_dataset(test_dataset, num_test, iter_index, transform=hyps['transform'])[0] + else: + cur_test_batch_dataset = split_dataset(test_dataset, num_test, iter_index)[0] + cur_test_batch_dataloader = build_dataloader(cur_test_batch_dataset, hyps['val_batch_size'], hyps['num_workers'], False, False, collate_fn=collate_fn) + cur_acc = self.models['main'].get_accuracy(cur_test_batch_dataloader) + # cur_source_acc = self.models['main'].get_accuracy(source_val_loader) + accs += [{ + 'iter': iter_index, + 'acc': cur_acc + }] + + cur_start_time = time.time() + + self.models['main'].to_train_mode() + + x, _ = next(train_loader) + + if isinstance(x, dict): + for k, v in x.items(): + if isinstance(v, torch.Tensor): + x[k] = v.to(device) + else: + x = x.to(device) + + source_x, source_y = next(source_train_loader) + + if isinstance(source_x, dict): + for k, v in source_x.items(): + if isinstance(v, torch.Tensor): + source_x[k] = v.to(device) + source_y = source_y.to(device) + else: + source_x, source_y = source_x.to(device), source_y.to(device) + + task_loss = self.models['main'].forward_to_get_task_loss(source_x, source_y) + source_features = feature_hook.input + + source_vis_features = source_features[1] + tlist = [] + for vis in source_vis_features: + tmp = F.max_pool2d(input=vis, kernel_size=2, stride=2, return_indices=False) + tmp = tmp.view(vis.shape[0], -1) + tlist.append(tmp) + source_vis_features = torch.cat(tlist, dim=1) + source_lang_features = source_features[3] + source_lang_aggregate_feature = source_lang_features['aggregate'] + source_lang_embedded_feature = source_lang_features['embedded'].flatten(1) + source_lang_hidden_feature = source_lang_features['hidden'].flatten(1) + source_features = torch.cat([source_vis_features, source_lang_aggregate_feature, source_lang_embedded_feature, source_lang_hidden_feature], dim=1) + self.models['main'].infer(x) + target_features = feature_hook.input + + target_vis_features = target_features[1] + tlist = [] + for vis in target_vis_features: + tmp = F.max_pool2d(input=vis, kernel_size=2, stride=2, return_indices=False) + tmp = tmp.view(vis.shape[0], -1) + tlist.append(tmp) + target_vis_features = torch.cat(tlist, dim=1) + target_lang_features = target_features[3] + target_lang_aggregate_feature = target_lang_features['aggregate'] + target_lang_embedded_feature = target_lang_features['embedded'].flatten(1) + target_lang_hidden_feature = target_lang_features['hidden'].flatten(1) + target_features = torch.cat([target_vis_features, target_lang_aggregate_feature, target_lang_hidden_feature, target_lang_hidden_feature], dim=1) + + mmd_loss = self.models['main'].get_mmd_loss(source_features, target_features) + + loss = task_loss + hyps['feat_align_loss_weight'] * mmd_loss + + optimizer.zero_grad() + loss.backward() + optimizer.step() + if scheduler is not None: + scheduler.step() + + pbar.set_description(f'da... | cur_acc: {cur_acc:.4f}, task_loss: {task_loss:.6f}, mmd_loss: {mmd_loss:.6f}') + task_losses += [float(task_loss.cpu().item())] + mmd_losses += [float(mmd_loss.cpu().item())] + + total_train_time += time.time() - cur_start_time + + feature_hook.remove() + + time_usage = total_train_time + + plt.plot(task_losses, label='task') + plt.plot(mmd_losses, label='mmd') + plt.xlabel('iteration') + plt.ylabel('loss') + plt.savefig(os.path.join(self.res_save_dir, 'loss.png')) + plt.clf() + + if 'transform' in hyps.keys(): + cur_test_batch_dataset = split_dataset(test_dataset, num_test, iter_index + 1, transform=hyps['transform'])[0] + else: + cur_test_batch_dataset = split_dataset(test_dataset, num_test, iter_index + 1)[0] + cur_test_batch_dataloader = build_dataloader(cur_test_batch_dataset, hyps['val_batch_size'], hyps['num_workers'], False, False, collate_fn=collate_fn) + cur_acc = self.models['main'].get_accuracy(cur_test_batch_dataloader) + accs += [{ + 'iter': iter_index + 1, + 'acc': cur_acc + }] + + return { + 'accs': accs, + 'time': time_usage + }, self.models \ No newline at end of file diff --git a/new_impl/cv/feat_align/main_gpt_neo.py b/new_impl/cv/feat_align/main_gpt_neo.py new file mode 100644 index 0000000000000000000000000000000000000000..478cb331ccb47a9557b875585fff872b09c859a3 --- /dev/null +++ b/new_impl/cv/feat_align/main_gpt_neo.py @@ -0,0 +1,217 @@ +from typing import Any, Dict, List +from schema import Schema +from data import Scenario, MergedDataset +from methods.base.alg import BaseAlg +from methods.base.model import BaseModel +from data import build_dataloader +import torch.optim +import tqdm +import os +import time +from abc import abstractmethod +import matplotlib.pyplot as plt +from torchvision.transforms import Compose +import torch.nn.functional as F + +class OnlineFeatAlignModel(BaseModel): + def get_required_model_components(self) -> List[str]: + return ['main'] + + @abstractmethod + def get_feature_hook(self): + pass + + @abstractmethod + def forward_to_get_task_loss(self, x, y): + pass + + @abstractmethod + def get_trained_params(self): + pass + + @abstractmethod + def get_mmd_loss(self, f1, f2): + pass + + +class FeatAlignAlg(BaseAlg): + def get_required_models_schema(self) -> Schema: + return Schema({ + 'main': OnlineFeatAlignModel + }) + + def get_required_hyp_schema(self) -> Schema: + from schema import Optional + return Schema({ + 'train_batch_size': int, + 'val_batch_size': int, + 'num_workers': int, + 'optimizer': str, + 'optimizer_args': dict, + 'scheduler': str, + 'scheduler_args': dict, + 'num_iters': int, + 'val_freq': int, + 'feat_align_loss_weight': float, + Optional('transform'): Compose, + }) + + def run(self, scenario: Scenario, hyps: Dict, collate_fn=None) -> Dict[str, Any]: + super().run(scenario, hyps) + + assert isinstance(self.models['main'], OnlineFeatAlignModel) # for auto completion + + cur_domain_name = scenario.target_domains_order[scenario.cur_domain_index] + if 'transform' in hyps.keys(): + datasets_for_training = scenario.get_online_cur_domain_datasets_for_training(transform=hyps['transform']) + else: + datasets_for_training = scenario.get_online_cur_domain_datasets_for_training() + train_dataset = datasets_for_training[cur_domain_name]['train'] + + if 'transform' in hyps.keys(): + datasets_for_inference = scenario.get_online_cur_domain_datasets_for_inference(transform=hyps['transform']) + else: + datasets_for_inference = scenario.get_online_cur_domain_datasets_for_inference() + test_dataset = datasets_for_inference + test_dataset.dataset.split = 'val' + train_loader = iter(build_dataloader(train_dataset, hyps['train_batch_size'], hyps['num_workers'], + True, None, collate_fn=collate_fn)) + # test_loader = build_dataloader(test_dataset, hyps['val_batch_size'], hyps['num_workers'], + # False, False, collate_fn=collate_fn) + + source_datasets = [d['train'] for n, d in datasets_for_training.items() if n != cur_domain_name] + source_dataset = MergedDataset(source_datasets) + # source_dataset_val = [d['val'] for n, d in datasets_for_training.items() if n != cur_domain_name][0] + source_train_loader = iter(build_dataloader(source_dataset, hyps['train_batch_size'], hyps['num_workers'], + True, None, collate_fn=collate_fn)) + # source_val_loader = build_dataloader(source_dataset_val, hyps['val_batch_size'], hyps['num_workers'], + # False, False, collate_fn=collate_fn) + # 1. generate surrogate DNN + # for n, m in self.models['main'].models_dict['md'].named_modules(): + # if isinstance(m, nn.Linear): + # m.reset_parameters() + # from utils.dl.common.model import set_module + + # for n, m in self.models['main'].models_dict['md'].named_modules(): + # if m.__class__.__name__ == 'KTakesAll': + # set_module(self.models['main'].models_dict['md'], n, KTakesAll(0.5)) + # self.models['main'].set_sd_sparsity(hyps['sd_sparsity']) + device = self.models['main'].device + # surrogate_dnn = self.models['main'].generate_sd_by_target_samples(next(train_loader)[0].to(device)) + # self.models['sd'] = surrogate_dnn + + # 2. train surrogate DNN + # TODO: train only a part of filters + trained_params = self.models['main'].get_trained_params() + optimizer = torch.optim.__dict__[hyps['optimizer']](trained_params, **hyps['optimizer_args']) + if hyps['scheduler'] != '': + scheduler = torch.optim.lr_scheduler.__dict__[hyps['scheduler']](optimizer, **hyps['scheduler_args']) + else: + scheduler = None + + pbar = tqdm.tqdm(range(hyps['num_iters']), dynamic_ncols=True, desc='da...') + task_losses, mmd_losses = [], [] + accs = [] + + total_train_time = 0. + + feature_hook = self.models['main'].get_feature_hook() + + num_test = 100 + + for iter_index in pbar: + + if iter_index % hyps['val_freq'] == 0: + from data import split_dataset + if 'transform' in hyps.keys(): + cur_test_batch_dataset = split_dataset(test_dataset, num_test, iter_index, transform=hyps['transform'])[0] + else: + cur_test_batch_dataset = split_dataset(test_dataset, num_test, iter_index)[0] + cur_test_batch_dataloader = build_dataloader(cur_test_batch_dataset, hyps['val_batch_size'], hyps['num_workers'], False, False, collate_fn=collate_fn) + cur_acc = self.models['main'].get_accuracy(cur_test_batch_dataloader) + # cur_source_acc = self.models['main'].get_accuracy(source_val_loader) + accs += [{ + 'iter': iter_index, + 'acc': cur_acc + }] + + + + cur_start_time = time.time() + + self.models['main'].to_train_mode() + + x, _ = next(train_loader) + + if isinstance(x, dict): + for k, v in x.items(): + if isinstance(v, torch.Tensor): + x[k] = v.to(device) + else: + x = x.to(device) + + source_x, source_y = next(source_train_loader) + + if isinstance(source_x, dict): + for k, v in source_x.items(): + if isinstance(v, torch.Tensor): + source_x[k] = v.to(device) + source_y = source_y.to(device) + else: + source_x, source_y = source_x.to(device), source_y.to(device) + + if len(x) == 0 or len(source_x) == 0: + continue + + task_loss = self.models['main'].forward_to_get_task_loss(source_x, source_y) + source_features = feature_hook.input[0] + source_loss_mask = source_x['attention_mask'].unsqueeze(-1) + source_features = source_loss_mask * source_features + source_features = torch.sum(source_features, dim=1) / torch.sum(source_loss_mask.squeeze(-1), dim=1).unsqueeze(-1) + self.models['main'].infer(x) + target_features = feature_hook.input[0] + target_loss_mask = x['attention_mask'].unsqueeze(-1) + target_features = target_loss_mask * target_features + target_features = torch.sum(target_features, dim=1) / torch.sum(target_loss_mask.squeeze(-1), dim=1).unsqueeze(-1) + mmd_loss = self.models['main'].get_mmd_loss(source_features, target_features) + + loss = task_loss + hyps['feat_align_loss_weight'] * mmd_loss + + optimizer.zero_grad() + loss.backward() + optimizer.step() + if scheduler is not None: + scheduler.step() + + pbar.set_description(f'da... | cur_acc: {cur_acc:.4f}, task_loss: {task_loss:.6f}, mmd_loss: {mmd_loss:.6f}') + task_losses += [float(task_loss.cpu().item())] + mmd_losses += [float(mmd_loss.cpu().item())] + + total_train_time += time.time() - cur_start_time + + feature_hook.remove() + + time_usage = total_train_time + + plt.plot(task_losses, label='task') + plt.plot(mmd_losses, label='mmd') + plt.xlabel('iteration') + plt.ylabel('loss') + plt.savefig(os.path.join(self.res_save_dir, 'loss.png')) + plt.clf() + + if 'transform' in hyps.keys(): + cur_test_batch_dataset = split_dataset(test_dataset, num_test, iter_index + 1, transform=hyps['transform'])[0] + else: + cur_test_batch_dataset = split_dataset(test_dataset, num_test, iter_index + 1)[0] + cur_test_batch_dataloader = build_dataloader(cur_test_batch_dataset, hyps['val_batch_size'], hyps['num_workers'], False, False, collate_fn=collate_fn) + cur_acc = self.models['main'].get_accuracy(cur_test_batch_dataloader) + accs += [{ + 'iter': iter_index + 1, + 'acc': cur_acc + }] + + return { + 'accs': accs, + 'time': time_usage + }, self.models \ No newline at end of file diff --git a/new_impl/cv/feat_align/main_partial.py b/new_impl/cv/feat_align/main_partial.py new file mode 100644 index 0000000000000000000000000000000000000000..853ad67ade7eecf12ff89da616e75eeaca14477c --- /dev/null +++ b/new_impl/cv/feat_align/main_partial.py @@ -0,0 +1,325 @@ +from typing import Any, Dict, List +from schema import Schema +from data import Scenario, MergedDataset +from methods.base.alg import BaseAlg +from methods.base.model import BaseModel +from data import build_dataloader +import torch.optim +import tqdm +import os +import time +from abc import abstractmethod +import matplotlib.pyplot as plt +from copy import deepcopy +from torch import nn + +import torch.optim + + +def tent_as_detector(online_model, x, num_iters=1, lr=1e-4, l1_wd=0., strategy='ours'): + model = online_model.models_dict['main'] + before_model = deepcopy(model) + + # from methods.tent import tent + + optimizer = torch.optim.SGD( + model.parameters(), lr=lr, weight_decay=l1_wd) + + # from .tent import configure_model, forward_and_adapt + # configure_model(model) + output = online_model.infer(x) + entropy = online_model.get_output_entropy(output).mean() + + entropy.backward() + # for _ in range(num_iters): + # forward_and_adapt(x, model, optimizer) + + # entropy_loss = model. + + filters_sen_info = {} + + last_conv_name = None + for (name, m1), m2 in zip(model.named_modules(), before_model.modules()): + if isinstance(m1, nn.Linear): + last_conv_name = name + + if not isinstance(m1, nn.LayerNorm): + continue + + with torch.no_grad(): + features_weight_diff = ((m1.weight.data - m2.weight.data).abs()) + features_bias_diff = ((m1.bias.data - m2.bias.data).abs()) + + features_diff = features_weight_diff + features_bias_diff + + features_diff_order = features_diff.argsort(descending=False) + + if strategy == 'ours': + untrained_filters_index = features_diff_order[: int(len(features_diff) * 0.8)] + elif strategy == 'random': + untrained_filters_index = torch.randperm(len(features_diff))[: int(len(features_diff) * 0.8)] + elif strategy == 'inversed_ours': + untrained_filters_index = features_diff_order.flip(0)[: int(len(features_diff) * 0.8)] + elif strategy == 'none': + untrained_filters_index = None + + filters_sen_info[name] = dict(untrained_filters_index=untrained_filters_index, conv_name=last_conv_name) + + return filters_sen_info + + +class SGDF(torch.optim.SGD): + + @torch.no_grad() + def step(self, p_names, conv_filters_sen_info, filters_sen_info, closure=None): + """Performs a single optimization step. + + Arguments: + closure (callable, optional): A closure that reevaluates the model + and returns the loss. + """ + loss = None + if closure is not None: + with torch.enable_grad(): + loss = closure() + + for group in self.param_groups: + weight_decay = group['weight_decay'] + momentum = group['momentum'] + dampening = group['dampening'] + nesterov = group['nesterov'] + + # assert len([i for i in model.named_parameters()]) == len([j for j in group['params']]) + + for name, p in zip(p_names, group['params']): + if p.grad is None: + continue + + layer_name = '.'.join(name.split('.')[0:-1]) + if layer_name in filters_sen_info.keys(): + untrained_filters_index = filters_sen_info[layer_name]['untrained_filters_index'] + elif layer_name in conv_filters_sen_info.keys(): + untrained_filters_index = conv_filters_sen_info[layer_name]['untrained_filters_index'] + else: + untrained_filters_index = [] + + d_p = p.grad + if weight_decay != 0: + d_p = d_p.add(p, alpha=weight_decay) + if momentum != 0: + param_state = self.state[p] + if 'momentum_buffer' not in param_state: + buf = param_state['momentum_buffer'] = torch.clone(d_p).detach() + else: + buf = param_state['momentum_buffer'] + buf.mul_(momentum).add_(d_p, alpha=1 - dampening) + if nesterov: + d_p = d_p.add(buf, alpha=momentum) + else: + d_p = buf + + try: + d_p[untrained_filters_index] = 0. + p.add_(d_p, alpha=-group['lr']) + except Exception as e: + print('SGDF error', name) + + return loss + + +class OnlineFeatAlignModel(BaseModel): + def get_required_model_components(self) -> List[str]: + return ['main'] + + @abstractmethod + def get_feature_hook(self): + pass + + @abstractmethod + def forward_to_get_task_loss(self, x, y): + pass + + @abstractmethod + def get_trained_params(self): + pass + + @abstractmethod + def get_mmd_loss(self, f1, f2): + pass + + @abstractmethod + def get_output_entropy(self, output): + pass + + +class FeatAlignAlg(BaseAlg): + def get_required_models_schema(self) -> Schema: + return Schema({ + 'main': OnlineFeatAlignModel + }) + + def get_required_hyp_schema(self) -> Schema: + return Schema({ + 'train_batch_size': int, + 'val_batch_size': int, + 'num_workers': int, + 'optimizer': str, + 'optimizer_args': dict, + 'scheduler': str, + 'scheduler_args': dict, + 'num_iters': int, + 'val_freq': int, + 'feat_align_loss_weight': float, + 'trained_neuron_selection_strategy': str + }) + + def run(self, scenario: Scenario, hyps: Dict) -> Dict[str, Any]: + super().run(scenario, hyps) + + assert isinstance(self.models['main'], OnlineFeatAlignModel) # for auto completion + + cur_domain_name = scenario.target_domains_order[scenario.cur_domain_index] + datasets_for_training = scenario.get_online_cur_domain_datasets_for_training() + train_dataset = datasets_for_training[cur_domain_name]['train'] + val_dataset = datasets_for_training[cur_domain_name]['val'] + datasets_for_inference = scenario.get_online_cur_domain_datasets_for_inference() + test_dataset = datasets_for_inference + + train_loader = iter(build_dataloader(train_dataset, hyps['train_batch_size'], hyps['num_workers'], + True, None)) + test_loader = build_dataloader(test_dataset, hyps['val_batch_size'], hyps['num_workers'], + False, False) + + source_datasets = [d['train'] for n, d in datasets_for_training.items() if n != cur_domain_name] + source_dataset = MergedDataset(source_datasets) + source_train_loader = iter(build_dataloader(source_dataset, hyps['train_batch_size'], hyps['num_workers'], + True, None)) + + # 1. generate surrogate DNN + # for n, m in self.models['main'].models_dict['md'].named_modules(): + # if isinstance(m, nn.Linear): + # m.reset_parameters() + # from utils.dl.common.model import set_module + + # for n, m in self.models['main'].models_dict['md'].named_modules(): + # if m.__class__.__name__ == 'KTakesAll': + # set_module(self.models['main'].models_dict['md'], n, KTakesAll(0.5)) + # self.models['main'].set_sd_sparsity(hyps['sd_sparsity']) + device = self.models['main'].device + # surrogate_dnn = self.models['main'].generate_sd_by_target_samples(next(train_loader)[0].to(device)) + # self.models['sd'] = surrogate_dnn + + # 2. train surrogate DNN + # TODO: train only a part of filters + trained_params, p_name = self.models['main'].get_trained_params() + + # optimizer = torch.optim.__dict__[hyps['optimizer']](trained_params, **hyps['optimizer_args']) + + optimizer = SGDF(trained_params, **hyps['optimizer_args']) + + if hyps['scheduler'] != '': + scheduler = torch.optim.lr_scheduler.__dict__[hyps['scheduler']](optimizer, **hyps['scheduler_args']) + else: + scheduler = None + + pbar = tqdm.tqdm(range(hyps['num_iters']), dynamic_ncols=True, desc='da...') + task_losses, mmd_losses = [], [] + accs = [] + + + x, _ = next(train_loader) + if isinstance(x, dict): + for k, v in x.items(): + if isinstance(v, torch.Tensor): + x[k] = v.to(device) + else: + x = x.to(device) + filters_sen_info = tent_as_detector(self.models['main'], x, strategy=hyps['trained_neuron_selection_strategy']) + conv_filters_sen_info = {v['conv_name']: v for _, v in filters_sen_info.items()} + + + total_train_time = 0. + + feature_hook = self.models['main'].get_feature_hook() + + for iter_index in pbar: + + if iter_index % hyps['val_freq'] == 0: + from data import split_dataset + cur_test_batch_dataset = split_dataset(test_dataset, hyps['val_batch_size'], iter_index)[0] + cur_test_batch_dataloader = build_dataloader(cur_test_batch_dataset, hyps['train_batch_size'], hyps['num_workers'], False, False) + cur_acc = self.models['main'].get_accuracy(cur_test_batch_dataloader) + accs += [{ + 'iter': iter_index, + 'acc': cur_acc + }] + + cur_start_time = time.time() + + self.models['main'].to_train_mode() + + x, _ = next(train_loader) + + if isinstance(x, dict): + for k, v in x.items(): + if isinstance(v, torch.Tensor): + x[k] = v.to(device) + else: + x = x.to(device) + + source_x, source_y = next(source_train_loader) + + if isinstance(source_x, dict): + for k, v in source_x.items(): + if isinstance(v, torch.Tensor): + source_x[k] = v.to(device) + source_y = source_y.to(device) + else: + source_x, source_y = source_x.to(device), source_y.to(device) + + task_loss = self.models['main'].forward_to_get_task_loss(source_x, source_y) + source_features = feature_hook.input + + self.models['main'].infer(x) + target_features = feature_hook.input + + mmd_loss = hyps['feat_align_loss_weight'] * self.models['main'].get_mmd_loss(source_features, target_features) + + loss = task_loss + mmd_loss + + optimizer.zero_grad() + loss.backward() + # optimizer.step() + optimizer.step(p_name, conv_filters_sen_info, filters_sen_info) + if scheduler is not None: + scheduler.step() + + pbar.set_description(f'da... | cur_acc: {cur_acc:.4f}, task_loss: {task_loss:.6f}, mmd_loss: {mmd_loss:.6f}') + task_losses += [float(task_loss.cpu().item())] + mmd_losses += [float(mmd_loss.cpu().item())] + + total_train_time += time.time() - cur_start_time + + feature_hook.remove() + + time_usage = total_train_time + + plt.plot(task_losses, label='task') + plt.plot(mmd_losses, label='mmd') + plt.xlabel('iteration') + plt.ylabel('loss') + plt.savefig(os.path.join(self.res_save_dir, 'loss.png')) + plt.clf() + + cur_test_batch_dataset = split_dataset(test_dataset, hyps['train_batch_size'], iter_index + 1)[0] + cur_test_batch_dataloader = build_dataloader(cur_test_batch_dataset, len(cur_test_batch_dataset), hyps['num_workers'], False, False) + cur_acc = self.models['main'].get_accuracy(cur_test_batch_dataloader) + accs += [{ + 'iter': iter_index + 1, + 'acc': cur_acc + }] + + return { + 'accs': accs, + 'time': time_usage + }, self.models \ No newline at end of file diff --git a/new_impl/cv/feat_align/mmd.py b/new_impl/cv/feat_align/mmd.py new file mode 100644 index 0000000000000000000000000000000000000000..647e5600e17cc1d62d2376b28346f5c6e0abc098 --- /dev/null +++ b/new_impl/cv/feat_align/mmd.py @@ -0,0 +1,62 @@ +import torch +import random + +# 参考 +# 最大化均值差异MMD与Numpy/Tensorflow/Pytorch各类代码实现 - 小蔡叔叔开方舟的文章 - 知乎 +# https://zhuanlan.zhihu.com/p/461656480 + +def guassian_kernel(source, target, kernel_mul=2.0, kernel_num=5, fix_sigma=None): + ''' + 将源域数据和目标域数据转化为核矩阵，即上文中的K + Params: + source: 源域数据（n * len(x)) + target: 目标域数据（m * len(y)) + kernel_mul: + kernel_num: 取不同高斯核的数量 + fix_sigma: 不同高斯核的sigma值 + Return: + sum(kernel_val): 多个核矩阵之和 + ''' + n_samples = int(source.size()[0])+int(target.size()[0])# 求矩阵的行数，一般source和target的尺度是一样的，这样便于计算 + total = torch.cat([source, target], dim=0) #将source,target按列方向合并 + #将total复制（n+m）份 + total0 = total.unsqueeze(0).expand(int(total.size(0)), int(total.size(0)), int(total.size(1))) + #将total的每一行都复制成（n+m）行，即每个数据都扩展成（n+m）份 + total1 = total.unsqueeze(1).expand(int(total.size(0)), int(total.size(0)), int(total.size(1))) + #求任意两个数据之间的和，得到的矩阵中坐标（i,j）代表total中第i行数据和第j行数据之间的l2 distance(i==j时为0） + L2_distance = ((total0-total1)**2).sum(2) + #调整高斯核函数的sigma值 + if fix_sigma: + bandwidth = fix_sigma + else: + bandwidth = torch.sum(L2_distance.data) / (n_samples**2-n_samples) + #以fix_sigma为中值，以kernel_mul为倍数取kernel_num个bandwidth值（比如fix_sigma为1时，得到[0.25,0.5,1,2,4] + bandwidth /= kernel_mul ** (kernel_num // 2) + bandwidth_list = [bandwidth * (kernel_mul**i) for i in range(kernel_num)] + #高斯核函数的数学表达式 + kernel_val = [torch.exp(-L2_distance / bandwidth_temp) for bandwidth_temp in bandwidth_list] + #得到最终的核矩阵 + return sum(kernel_val)#/len(kernel_val) + +def mmd_rbf(source, target, kernel_mul=2.0, kernel_num=5, fix_sigma=None): + ''' + 计算源域数据和目标域数据的MMD距离 + Params: + source: 源域数据（n * len(x)) + target: 目标域数据（m * len(y)) + kernel_mul: + kernel_num: 取不同高斯核的数量 + fix_sigma: 不同高斯核的sigma值 + Return: + loss: MMD loss + ''' + batch_size = int(source.size()[0])#一般默认为源域和目标域的batchsize相同 + kernels = guassian_kernel(source, target, + kernel_mul=kernel_mul, kernel_num=kernel_num, fix_sigma=fix_sigma) + #根据式（3）将核矩阵分成4部分 + XX = kernels[:batch_size, :batch_size] + YY = kernels[batch_size:, batch_size:] + XY = kernels[:batch_size, batch_size:] + YX = kernels[batch_size:, :batch_size] + loss = torch.mean(XX + YY - XY -YX) + return loss#因为一般都是n==m，所以L矩阵一般不加入计算 \ No newline at end of file diff --git a/new_impl/cv/glip/object_detection/000000103759.jpg b/new_impl/cv/glip/object_detection/000000103759.jpg new file mode 100644 index 0000000000000000000000000000000000000000..59451fc77a0b75f2ec22a6c1deb070bd618fdb70 Binary files /dev/null and b/new_impl/cv/glip/object_detection/000000103759.jpg differ diff --git a/new_impl/cv/glip/object_detection/9472793441_b7822c00de_z.jpg b/new_impl/cv/glip/object_detection/9472793441_b7822c00de_z.jpg new file mode 100644 index 0000000000000000000000000000000000000000..f9198f2733daf7d93488a2ae5574c1011c889c31 Binary files /dev/null and b/new_impl/cv/glip/object_detection/9472793441_b7822c00de_z.jpg differ diff --git a/new_impl/cv/glip/object_detection/bert-base-uncased/config.json b/new_impl/cv/glip/object_detection/bert-base-uncased/config.json new file mode 100644 index 0000000000000000000000000000000000000000..45a2321a7ecfdaaf60a6c1fd7f5463994cc8907d --- /dev/null +++ b/new_impl/cv/glip/object_detection/bert-base-uncased/config.json @@ -0,0 +1,23 @@ +{ + "architectures": [ + "BertForMaskedLM" + ], + "attention_probs_dropout_prob": 0.1, + "gradient_checkpointing": false, + "hidden_act": "gelu", + "hidden_dropout_prob": 0.1, + "hidden_size": 768, + "initializer_range": 0.02, + "intermediate_size": 3072, + "layer_norm_eps": 1e-12, + "max_position_embeddings": 512, + "model_type": "bert", + "num_attention_heads": 12, + "num_hidden_layers": 12, + "pad_token_id": 0, + "position_embedding_type": "absolute", + "transformers_version": "4.6.0.dev0", + "type_vocab_size": 2, + "use_cache": true, + "vocab_size": 30522 +} diff --git a/new_impl/cv/glip/object_detection/bert-base-uncased/tokenizer.json b/new_impl/cv/glip/object_detection/bert-base-uncased/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..949a6f013d67eb8a5b4b5b46026217b888021b88 --- /dev/null +++ b/new_impl/cv/glip/object_detection/bert-base-uncased/tokenizer.json @@ -0,0 +1 @@ +{"version":"1.0","truncation":null,"padding":null,"added_tokens":[{"id":0,"special":true,"content":"[PAD]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":100,"special":true,"content":"[UNK]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":101,"special":true,"content":"[CLS]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":102,"special":true,"content":"[SEP]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":103,"special":true,"content":"[MASK]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false}],"normalizer":{"type":"BertNormalizer","clean_text":true,"handle_chinese_chars":true,"strip_accents":null,"lowercase":true},"pre_tokenizer":{"type":"BertPreTokenizer"},"post_processor":{"type":"TemplateProcessing","single":[{"SpecialToken":{"id":"[CLS]","type_id":0}},{"Sequence":{"id":"A","type_id":0}},{"SpecialToken":{"id":"[SEP]","type_id":0}}],"pair":[{"SpecialToken":{"id":"[CLS]","type_id":0}},{"Sequence":{"id":"A","type_id":0}},{"SpecialToken":{"id":"[SEP]","type_id":0}},{"Sequence":{"id":"B","type_id":1}},{"SpecialToken":{"id":"[SEP]","type_id":1}}],"special_tokens":{"[CLS]":{"id":"[CLS]","ids":[101],"tokens":["[CLS]"]},"[SEP]":{"id":"[SEP]","ids":[102],"tokens":["[SEP]"]}}},"decoder":{"type":"WordPiece","prefix":"##","cleanup":true},"model":{"unk_token":"[UNK]","continuing_subword_prefix":"##","max_input_chars_per_word":100,"vocab":{"[PAD]":0,"[unused0]":1,"[unused1]":2,"[unused2]":3,"[unused3]":4,"[unused4]":5,"[unused5]":6,"[unused6]":7,"[unused7]":8,"[unused8]":9,"[unused9]":10,"[unused10]":11,"[unused11]":12,"[unused12]":13,"[unused13]":14,"[unused14]":15,"[unused15]":16,"[unused16]":17,"[unused17]":18,"[unused18]":19,"[unused19]":20,"[unused20]":21,"[unused21]":22,"[unused22]":23,"[unused23]":24,"[unused24]":25,"[unused25]":26,"[unused26]":27,"[unused27]":28,"[unused28]":29,"[unused29]":30,"[unused30]":31,"[unused31]":32,"[unused32]":33,"[unused33]":34,"[unused34]":35,"[unused35]":36,"[unused36]":37,"[unused37]":38,"[unused38]":39,"[unused39]":40,"[unused40]":41,"[unused41]":42,"[unused42]":43,"[unused43]":44,"[unused44]":45,"[unused45]":46,"[unused46]":47,"[unused47]":48,"[unused48]":49,"[unused49]":50,"[unused50]":51,"[unused51]":52,"[unused52]":53,"[unused53]":54,"[unused54]":55,"[unused55]":56,"[unused56]":57,"[unused57]":58,"[unused58]":59,"[unused59]":60,"[unused60]":61,"[unused61]":62,"[unused62]":63,"[unused63]":64,"[unused64]":65,"[unused65]":66,"[unused66]":67,"[unused67]":68,"[unused68]":69,"[unused69]":70,"[unused70]":71,"[unused71]":72,"[unused72]":73,"[unused73]":74,"[unused74]":75,"[unused75]":76,"[unused76]":77,"[unused77]":78,"[unused78]":79,"[unused79]":80,"[unused80]":81,"[unused81]":82,"[unused82]":83,"[unused83]":84,"[unused84]":85,"[unused85]":86,"[unused86]":87,"[unused87]":88,"[unused88]":89,"[unused89]":90,"[unused90]":91,"[unused91]":92,"[unused92]":93,"[unused93]":94,"[unused94]":95,"[unused95]":96,"[unused96]":97,"[unused97]":98,"[unused98]":99,"[UNK]":100,"[CLS]":101,"[SEP]":102,"[MASK]":103,"[unused99]":104,"[unused100]":105,"[unused101]":106,"[unused102]":107,"[unused103]":108,"[unused104]":109,"[unused105]":110,"[unused106]":111,"[unused107]":112,"[unused108]":113,"[unused109]":114,"[unused110]":115,"[unused111]":116,"[unused112]":117,"[unused113]":118,"[unused114]":119,"[unused115]":120,"[unused116]":121,"[unused117]":122,"[unused118]":123,"[unused119]":124,"[unused120]":125,"[unused121]":126,"[unused122]":127,"[unused123]":128,"[unused124]":129,"[unused125]":130,"[unused126]":131,"[unused127]":132,"[unused128]":133,"[unused129]":134,"[unused130]":135,"[unused131]":136,"[unused132]":137,"[unused133]":138,"[unused134]":139,"[unused135]":140,"[unused136]":141,"[unused137]":142,"[unused138]":143,"[unused139]":144,"[unused140]":145,"[unused141]":146,"[unused142]":147,"[unused143]":148,"[unused144]":149,"[unused145]":150,"[unused146]":151,"[unused147]":152,"[unused148]":153,"[unused149]":154,"[unused150]":155,"[unused151]":156,"[unused152]":157,"[unused153]":158,"[unused154]":159,"[unused155]":160,"[unused156]":161,"[unused157]":162,"[unused158]":163,"[unused159]":164,"[unused160]":165,"[unused161]":166,"[unused162]":167,"[unused163]":168,"[unused164]":169,"[unused165]":170,"[unused166]":171,"[unused167]":172,"[unused168]":173,"[unused169]":174,"[unused170]":175,"[unused171]":176,"[unused172]":177,"[unused173]":178,"[unused174]":179,"[unused175]":180,"[unused176]":181,"[unused177]":182,"[unused178]":183,"[unused179]":184,"[unused180]":185,"[unused181]":186,"[unused182]":187,"[unused183]":188,"[unused184]":189,"[unused185]":190,"[unused186]":191,"[unused187]":192,"[unused188]":193,"[unused189]":194,"[unused190]":195,"[unused191]":196,"[unused192]":197,"[unused193]":198,"[unused194]":199,"[unused195]":200,"[unused196]":201,"[unused197]":202,"[unused198]":203,"[unused199]":204,"[unused200]":205,"[unused201]":206,"[unused202]":207,"[unused203]":208,"[unused204]":209,"[unused205]":210,"[unused206]":211,"[unused207]":212,"[unused208]":213,"[unused209]":214,"[unused210]":215,"[unused211]":216,"[unused212]":217,"[unused213]":218,"[unused214]":219,"[unused215]":220,"[unused216]":221,"[unused217]":222,"[unused218]":223,"[unused219]":224,"[unused220]":225,"[unused221]":226,"[unused222]":227,"[unused223]":228,"[unused224]":229,"[unused225]":230,"[unused226]":231,"[unused227]":232,"[unused228]":233,"[unused229]":234,"[unused230]":235,"[unused231]":236,"[unused232]":237,"[unused233]":238,"[unused234]":239,"[unused235]":240,"[unused236]":241,"[unused237]":242,"[unused238]":243,"[unused239]":244,"[unused240]":245,"[unused241]":246,"[unused242]":247,"[unused243]":248,"[unused244]":249,"[unused245]":250,"[unused246]":251,"[unused247]":252,"[unused248]":253,"[unused249]":254,"[unused250]":255,"[unused251]":256,"[unused252]":257,"[unused253]":258,"[unused254]":259,"[unused255]":260,"[unused256]":261,"[unused257]":262,"[unused258]":263,"[unused259]":264,"[unused260]":265,"[unused261]":266,"[unused262]":267,"[unused263]":268,"[unused264]":269,"[unused265]":270,"[unused266]":271,"[unused267]":272,"[unused268]":273,"[unused269]":274,"[unused270]":275,"[unused271]":276,"[unused272]":277,"[unused273]":278,"[unused274]":279,"[unused275]":280,"[unused276]":281,"[unused277]":282,"[unused278]":283,"[unused279]":284,"[unused280]":285,"[unused281]":286,"[unused282]":287,"[unused283]":288,"[unused284]":289,"[unused285]":290,"[unused286]":291,"[unused287]":292,"[unused288]":293,"[unused289]":294,"[unused290]":295,"[unused291]":296,"[unused292]":297,"[unused293]":298,"[unused294]":299,"[unused295]":300,"[unused296]":301,"[unused297]":302,"[unused298]":303,"[unused299]":304,"[unused300]":305,"[unused301]":306,"[unused302]":307,"[unused303]":308,"[unused304]":309,"[unused305]":310,"[unused306]":311,"[unused307]":312,"[unused308]":313,"[unused309]":314,"[unused310]":315,"[unused311]":316,"[unused312]":317,"[unused313]":318,"[unused314]":319,"[unused315]":320,"[unused316]":321,"[unused317]":322,"[unused318]":323,"[unused319]":324,"[unused320]":325,"[unused321]":326,"[unused322]":327,"[unused323]":328,"[unused324]":329,"[unused325]":330,"[unused326]":331,"[unused327]":332,"[unused328]":333,"[unused329]":334,"[unused330]":335,"[unused331]":336,"[unused332]":337,"[unused333]":338,"[unused334]":339,"[unused335]":340,"[unused336]":341,"[unused337]":342,"[unused338]":343,"[unused339]":344,"[unused340]":345,"[unused341]":346,"[unused342]":347,"[unused343]":348,"[unused344]":349,"[unused345]":350,"[unused346]":351,"[unused347]":352,"[unused348]":353,"[unused349]":354,"[unused350]":355,"[unused351]":356,"[unused352]":357,"[unused353]":358,"[unused354]":359,"[unused355]":360,"[unused356]":361,"[unused357]":362,"[unused358]":363,"[unused359]":364,"[unused360]":365,"[unused361]":366,"[unused362]":367,"[unused363]":368,"[unused364]":369,"[unused365]":370,"[unused366]":371,"[unused367]":372,"[unused368]":373,"[unused369]":374,"[unused370]":375,"[unused371]":376,"[unused372]":377,"[unused373]":378,"[unused374]":379,"[unused375]":380,"[unused376]":381,"[unused377]":382,"[unused378]":383,"[unused379]":384,"[unused380]":385,"[unused381]":386,"[unused382]":387,"[unused383]":388,"[unused384]":389,"[unused385]":390,"[unused386]":391,"[unused387]":392,"[unused388]":393,"[unused389]":394,"[unused390]":395,"[unused391]":396,"[unused392]":397,"[unused393]":398,"[unused394]":399,"[unused395]":400,"[unused396]":401,"[unused397]":402,"[unused398]":403,"[unused399]":404,"[unused400]":405,"[unused401]":406,"[unused402]":407,"[unused403]":408,"[unused404]":409,"[unused405]":410,"[unused406]":411,"[unused407]":412,"[unused408]":413,"[unused409]":414,"[unused410]":415,"[unused411]":416,"[unused412]":417,"[unused413]":418,"[unused414]":419,"[unused415]":420,"[unused416]":421,"[unused417]":422,"[unused418]":423,"[unused419]":424,"[unused420]":425,"[unused421]":426,"[unused422]":427,"[unused423]":428,"[unused424]":429,"[unused425]":430,"[unused426]":431,"[unused427]":432,"[unused428]":433,"[unused429]":434,"[unused430]":435,"[unused431]":436,"[unused432]":437,"[unused433]":438,"[unused434]":439,"[unused435]":440,"[unused436]":441,"[unused437]":442,"[unused438]":443,"[unused439]":444,"[unused440]":445,"[unused441]":446,"[unused442]":447,"[unused443]":448,"[unused444]":449,"[unused445]":450,"[unused446]":451,"[unused447]":452,"[unused448]":453,"[unused449]":454,"[unused450]":455,"[unused451]":456,"[unused452]":457,"[unused453]":458,"[unused454]":459,"[unused455]":460,"[unused456]":461,"[unused457]":462,"[unused458]":463,"[unused459]":464,"[unused460]":465,"[unused461]":466,"[unused462]":467,"[unused463]":468,"[unused464]":469,"[unused465]":470,"[unused466]":471,"[unused467]":472,"[unused468]":473,"[unused469]":474,"[unused470]":475,"[unused471]":476,"[unused472]":477,"[unused473]":478,"[unused474]":479,"[unused475]":480,"[unused476]":481,"[unused477]":482,"[unused478]":483,"[unused479]":484,"[unused480]":485,"[unused481]":486,"[unused482]":487,"[unused483]":488,"[unused484]":489,"[unused485]":490,"[unused486]":491,"[unused487]":492,"[unused488]":493,"[unused489]":494,"[unused490]":495,"[unused491]":496,"[unused492]":497,"[unused493]":498,"[unused494]":499,"[unused495]":500,"[unused496]":501,"[unused497]":502,"[unused498]":503,"[unused499]":504,"[unused500]":505,"[unused501]":506,"[unused502]":507,"[unused503]":508,"[unused504]":509,"[unused505]":510,"[unused506]":511,"[unused507]":512,"[unused508]":513,"[unused509]":514,"[unused510]":515,"[unused511]":516,"[unused512]":517,"[unused513]":518,"[unused514]":519,"[unused515]":520,"[unused516]":521,"[unused517]":522,"[unused518]":523,"[unused519]":524,"[unused520]":525,"[unused521]":526,"[unused522]":527,"[unused523]":528,"[unused524]":529,"[unused525]":530,"[unused526]":531,"[unused527]":532,"[unused528]":533,"[unused529]":534,"[unused530]":535,"[unused531]":536,"[unused532]":537,"[unused533]":538,"[unused534]":539,"[unused535]":540,"[unused536]":541,"[unused537]":542,"[unused538]":543,"[unused539]":544,"[unused540]":545,"[unused541]":546,"[unused542]":547,"[unused543]":548,"[unused544]":549,"[unused545]":550,"[unused546]":551,"[unused547]":552,"[unused548]":553,"[unused549]":554,"[unused550]":555,"[unused551]":556,"[unused552]":557,"[unused553]":558,"[unused554]":559,"[unused555]":560,"[unused556]":561,"[unused557]":562,"[unused558]":563,"[unused559]":564,"[unused560]":565,"[unused561]":566,"[unused562]":567,"[unused563]":568,"[unused564]":569,"[unused565]":570,"[unused566]":571,"[unused567]":572,"[unused568]":573,"[unused569]":574,"[unused570]":575,"[unused571]":576,"[unused572]":577,"[unused573]":578,"[unused574]":579,"[unused575]":580,"[unused576]":581,"[unused577]":582,"[unused578]":583,"[unused579]":584,"[unused580]":585,"[unused581]":586,"[unused582]":587,"[unused583]":588,"[unused584]":589,"[unused585]":590,"[unused586]":591,"[unused587]":592,"[unused588]":593,"[unused589]":594,"[unused590]":595,"[unused591]":596,"[unused592]":597,"[unused593]":598,"[unused594]":599,"[unused595]":600,"[unused596]":601,"[unused597]":602,"[unused598]":603,"[unused599]":604,"[unused600]":605,"[unused601]":606,"[unused602]":607,"[unused603]":608,"[unused604]":609,"[unused605]":610,"[unused606]":611,"[unused607]":612,"[unused608]":613,"[unused609]":614,"[unused610]":615,"[unused611]":616,"[unused612]":617,"[unused613]":618,"[unused614]":619,"[unused615]":620,"[unused616]":621,"[unused617]":622,"[unused618]":623,"[unused619]":624,"[unused620]":625,"[unused621]":626,"[unused622]":627,"[unused623]":628,"[unused624]":629,"[unused625]":630,"[unused626]":631,"[unused627]":632,"[unused628]":633,"[unused629]":634,"[unused630]":635,"[unused631]":636,"[unused632]":637,"[unused633]":638,"[unused634]":639,"[unused635]":640,"[unused636]":641,"[unused637]":642,"[unused638]":643,"[unused639]":644,"[unused640]":645,"[unused641]":646,"[unused642]":647,"[unused643]":648,"[unused644]":649,"[unused645]":650,"[unused646]":651,"[unused647]":652,"[unused648]":653,"[unused649]":654,"[unused650]":655,"[unused651]":656,"[unused652]":657,"[unused653]":658,"[unused654]":659,"[unused655]":660,"[unused656]":661,"[unused657]":662,"[unused658]":663,"[unused659]":664,"[unused660]":665,"[unused661]":666,"[unused662]":667,"[unused663]":668,"[unused664]":669,"[unused665]":670,"[unused666]":671,"[unused667]":672,"[unused668]":673,"[unused669]":674,"[unused670]":675,"[unused671]":676,"[unused672]":677,"[unused673]":678,"[unused674]":679,"[unused675]":680,"[unused676]":681,"[unused677]":682,"[unused678]":683,"[unused679]":684,"[unused680]":685,"[unused681]":686,"[unused682]":687,"[unused683]":688,"[unused684]":689,"[unused685]":690,"[unused686]":691,"[unused687]":692,"[unused688]":693,"[unused689]":694,"[unused690]":695,"[unused691]":696,"[unused692]":697,"[unused693]":698,"[unused694]":699,"[unused695]":700,"[unused696]":701,"[unused697]":702,"[unused698]":703,"[unused699]":704,"[unused700]":705,"[unused701]":706,"[unused702]":707,"[unused703]":708,"[unused704]":709,"[unused705]":710,"[unused706]":711,"[unused707]":712,"[unused708]":713,"[unused709]":714,"[unused710]":715,"[unused711]":716,"[unused712]":717,"[unused713]":718,"[unused714]":719,"[unused715]":720,"[unused716]":721,"[unused717]":722,"[unused718]":723,"[unused719]":724,"[unused720]":725,"[unused721]":726,"[unused722]":727,"[unused723]":728,"[unused724]":729,"[unused725]":730,"[unused726]":731,"[unused727]":732,"[unused728]":733,"[unused729]":734,"[unused730]":735,"[unused731]":736,"[unused732]":737,"[unused733]":738,"[unused734]":739,"[unused735]":740,"[unused736]":741,"[unused737]":742,"[unused738]":743,"[unused739]":744,"[unused740]":745,"[unused741]":746,"[unused742]":747,"[unused743]":748,"[unused744]":749,"[unused745]":750,"[unused746]":751,"[unused747]":752,"[unused748]":753,"[unused749]":754,"[unused750]":755,"[unused751]":756,"[unused752]":757,"[unused753]":758,"[unused754]":759,"[unused755]":760,"[unused756]":761,"[unused757]":762,"[unused758]":763,"[unused759]":764,"[unused760]":765,"[unused761]":766,"[unused762]":767,"[unused763]":768,"[unused764]":769,"[unused765]":770,"[unused766]":771,"[unused767]":772,"[unused768]":773,"[unused769]":774,"[unused770]":775,"[unused771]":776,"[unused772]":777,"[unused773]":778,"[unused774]":779,"[unused775]":780,"[unused776]":781,"[unused777]":782,"[unused778]":783,"[unused779]":784,"[unused780]":785,"[unused781]":786,"[unused782]":787,"[unused783]":788,"[unused784]":789,"[unused785]":790,"[unused786]":791,"[unused787]":792,"[unused788]":793,"[unused789]":794,"[unused790]":795,"[unused791]":796,"[unused792]":797,"[unused793]":798,"[unused794]":799,"[unused795]":800,"[unused796]":801,"[unused797]":802,"[unused798]":803,"[unused799]":804,"[unused800]":805,"[unused801]":806,"[unused802]":807,"[unused803]":808,"[unused804]":809,"[unused805]":810,"[unused806]":811,"[unused807]":812,"[unused808]":813,"[unused809]":814,"[unused810]":815,"[unused811]":816,"[unused812]":817,"[unused813]":818,"[unused814]":819,"[unused815]":820,"[unused816]":821,"[unused817]":822,"[unused818]":823,"[unused819]":824,"[unused820]":825,"[unused821]":826,"[unused822]":827,"[unused823]":828,"[unused824]":829,"[unused825]":830,"[unused826]":831,"[unused827]":832,"[unused828]":833,"[unused829]":834,"[unused830]":835,"[unused831]":836,"[unused832]":837,"[unused833]":838,"[unused834]":839,"[unused835]":840,"[unused836]":841,"[unused837]":842,"[unused838]":843,"[unused839]":844,"[unused840]":845,"[unused841]":846,"[unused842]":847,"[unused843]":848,"[unused844]":849,"[unused845]":850,"[unused846]":851,"[unused847]":852,"[unused848]":853,"[unused849]":854,"[unused850]":855,"[unused851]":856,"[unused852]":857,"[unused853]":858,"[unused854]":859,"[unused855]":860,"[unused856]":861,"[unused857]":862,"[unused858]":863,"[unused859]":864,"[unused860]":865,"[unused861]":866,"[unused862]":867,"[unused863]":868,"[unused864]":869,"[unused865]":870,"[unused866]":871,"[unused867]":872,"[unused868]":873,"[unused869]":874,"[unused870]":875,"[unused871]":876,"[unused872]":877,"[unused873]":878,"[unused874]":879,"[unused875]":880,"[unused876]":881,"[unused877]":882,"[unused878]":883,"[unused879]":884,"[unused880]":885,"[unused881]":886,"[unused882]":887,"[unused883]":888,"[unused884]":889,"[unused885]":890,"[unused886]":891,"[unused887]":892,"[unused888]":893,"[unused889]":894,"[unused890]":895,"[unused891]":896,"[unused892]":897,"[unused893]":898,"[unused894]":899,"[unused895]":900,"[unused896]":901,"[unused897]":902,"[unused898]":903,"[unused899]":904,"[unused900]":905,"[unused901]":906,"[unused902]":907,"[unused903]":908,"[unused904]":909,"[unused905]":910,"[unused906]":911,"[unused907]":912,"[unused908]":913,"[unused909]":914,"[unused910]":915,"[unused911]":916,"[unused912]":917,"[unused913]":918,"[unused914]":919,"[unused915]":920,"[unused916]":921,"[unused917]":922,"[unused918]":923,"[unused919]":924,"[unused920]":925,"[unused921]":926,"[unused922]":927,"[unused923]":928,"[unused924]":929,"[unused925]":930,"[unused926]":931,"[unused927]":932,"[unused928]":933,"[unused929]":934,"[unused930]":935,"[unused931]":936,"[unused932]":937,"[unused933]":938,"[unused934]":939,"[unused935]":940,"[unused936]":941,"[unused937]":942,"[unused938]":943,"[unused939]":944,"[unused940]":945,"[unused941]":946,"[unused942]":947,"[unused943]":948,"[unused944]":949,"[unused945]":950,"[unused946]":951,"[unused947]":952,"[unused948]":953,"[unused949]":954,"[unused950]":955,"[unused951]":956,"[unused952]":957,"[unused953]":958,"[unused954]":959,"[unused955]":960,"[unused956]":961,"[unused957]":962,"[unused958]":963,"[unused959]":964,"[unused960]":965,"[unused961]":966,"[unused962]":967,"[unused963]":968,"[unused964]":969,"[unused965]":970,"[unused966]":971,"[unused967]":972,"[unused968]":973,"[unused969]":974,"[unused970]":975,"[unused971]":976,"[unused972]":977,"[unused973]":978,"[unused974]":979,"[unused975]":980,"[unused976]":981,"[unused977]":982,"[unused978]":983,"[unused979]":984,"[unused980]":985,"[unused981]":986,"[unused982]":987,"[unused983]":988,"[unused984]":989,"[unused985]":990,"[unused986]":991,"[unused987]":992,"[unused988]":993,"[unused989]":994,"[unused990]":995,"[unused991]":996,"[unused992]":997,"[unused993]":998,"!":999,"\"":1000,"#":1001,"$":1002,"%":1003,"&":1004,"'":1005,"(":1006,")":1007,"*":1008,"+":1009,",":1010,"-":1011,".":1012,"/":1013,"0":1014,"1":1015,"2":1016,"3":1017,"4":1018,"5":1019,"6":1020,"7":1021,"8":1022,"9":1023,":":1024,";":1025,"<":1026,"=":1027,">":1028,"?":1029,"@":1030,"[":1031,"\\":1032,"]":1033,"^":1034,"_":1035,"`":1036,"a":1037,"b":1038,"c":1039,"d":1040,"e":1041,"f":1042,"g":1043,"h":1044,"i":1045,"j":1046,"k":1047,"l":1048,"m":1049,"n":1050,"o":1051,"p":1052,"q":1053,"r":1054,"s":1055,"t":1056,"u":1057,"v":1058,"w":1059,"x":1060,"y":1061,"z":1062,"{":1063,"|":1064,"}":1065,"~":1066,"¡":1067,"¢":1068,"£":1069,"¤":1070,"¥":1071,"¦":1072,"§":1073,"¨":1074,"©":1075,"ª":1076,"«":1077,"¬":1078,"®":1079,"°":1080,"±":1081,"²":1082,"³":1083,"´":1084,"µ":1085,"¶":1086,"·":1087,"¹":1088,"º":1089,"»":1090,"¼":1091,"½":1092,"¾":1093,"¿":1094,"×":1095,"ß":1096,"æ":1097,"ð":1098,"÷":1099,"ø":1100,"þ":1101,"đ":1102,"ħ":1103,"ı":1104,"ł":1105,"ŋ":1106,"œ":1107,"ƒ":1108,"ɐ":1109,"ɑ":1110,"ɒ":1111,"ɔ":1112,"ɕ":1113,"ə":1114,"ɛ":1115,"ɡ":1116,"ɣ":1117,"ɨ":1118,"ɪ":1119,"ɫ":1120,"ɬ":1121,"ɯ":1122,"ɲ":1123,"ɴ":1124,"ɹ":1125,"ɾ":1126,"ʀ":1127,"ʁ":1128,"ʂ":1129,"ʃ":1130,"ʉ":1131,"ʊ":1132,"ʋ":1133,"ʌ":1134,"ʎ":1135,"ʐ":1136,"ʑ":1137,"ʒ":1138,"ʔ":1139,"ʰ":1140,"ʲ":1141,"ʳ":1142,"ʷ":1143,"ʸ":1144,"ʻ":1145,"ʼ":1146,"ʾ":1147,"ʿ":1148,"ˈ":1149,"ː":1150,"ˡ":1151,"ˢ":1152,"ˣ":1153,"ˤ":1154,"α":1155,"β":1156,"γ":1157,"δ":1158,"ε":1159,"ζ":1160,"η":1161,"θ":1162,"ι":1163,"κ":1164,"λ":1165,"μ":1166,"ν":1167,"ξ":1168,"ο":1169,"π":1170,"ρ":1171,"ς":1172,"σ":1173,"τ":1174,"υ":1175,"φ":1176,"χ":1177,"ψ":1178,"ω":1179,"а":1180,"б":1181,"в":1182,"г":1183,"д":1184,"е":1185,"ж":1186,"з":1187,"и":1188,"к":1189,"л":1190,"м":1191,"н":1192,"о":1193,"п":1194,"р":1195,"с":1196,"т":1197,"у":1198,"ф":1199,"х":1200,"ц":1201,"ч":1202,"ш":1203,"щ":1204,"ъ":1205,"ы":1206,"ь":1207,"э":1208,"ю":1209,"я":1210,"ђ":1211,"є":1212,"і":1213,"ј":1214,"љ":1215,"њ":1216,"ћ":1217,"ӏ":1218,"ա":1219,"բ":1220,"գ":1221,"դ":1222,"ե":1223,"թ":1224,"ի":1225,"լ":1226,"կ":1227,"հ":1228,"մ":1229,"յ":1230,"ն":1231,"ո":1232,"պ":1233,"ս":1234,"վ":1235,"տ":1236,"ր":1237,"ւ":1238,"ք":1239,"־":1240,"א":1241,"ב":1242,"ג":1243,"ד":1244,"ה":1245,"ו":1246,"ז":1247,"ח":1248,"ט":1249,"י":1250,"ך":1251,"כ":1252,"ל":1253,"ם":1254,"מ":1255,"ן":1256,"נ":1257,"ס":1258,"ע":1259,"ף":1260,"פ":1261,"ץ":1262,"צ":1263,"ק":1264,"ר":1265,"ש":1266,"ת":1267,"،":1268,"ء":1269,"ا":1270,"ب":1271,"ة":1272,"ت":1273,"ث":1274,"ج":1275,"ح":1276,"خ":1277,"د":1278,"ذ":1279,"ر":1280,"ز":1281,"س":1282,"ش":1283,"ص":1284,"ض":1285,"ط":1286,"ظ":1287,"ع":1288,"غ":1289,"ـ":1290,"ف":1291,"ق":1292,"ك":1293,"ل":1294,"م":1295,"ن":1296,"ه":1297,"و":1298,"ى":1299,"ي":1300,"ٹ":1301,"پ":1302,"چ":1303,"ک":1304,"گ":1305,"ں":1306,"ھ":1307,"ہ":1308,"ی":1309,"ے":1310,"अ":1311,"आ":1312,"उ":1313,"ए":1314,"क":1315,"ख":1316,"ग":1317,"च":1318,"ज":1319,"ट":1320,"ड":1321,"ण":1322,"त":1323,"थ":1324,"द":1325,"ध":1326,"न":1327,"प":1328,"ब":1329,"भ":1330,"म":1331,"य":1332,"र":1333,"ल":1334,"व":1335,"श":1336,"ष":1337,"स":1338,"ह":1339,"ा":1340,"ि":1341,"ी":1342,"ो":1343,"।":1344,"॥":1345,"ং":1346,"অ":1347,"আ":1348,"ই":1349,"উ":1350,"এ":1351,"ও":1352,"ক":1353,"খ":1354,"গ":1355,"চ":1356,"ছ":1357,"জ":1358,"ট":1359,"ড":1360,"ণ":1361,"ত":1362,"থ":1363,"দ":1364,"ধ":1365,"ন":1366,"প":1367,"ব":1368,"ভ":1369,"ম":1370,"য":1371,"র":1372,"ল":1373,"শ":1374,"ষ":1375,"স":1376,"হ":1377,"া":1378,"ি":1379,"ী":1380,"ে":1381,"க":1382,"ச":1383,"ட":1384,"த":1385,"ந":1386,"ன":1387,"ப":1388,"ம":1389,"ய":1390,"ர":1391,"ல":1392,"ள":1393,"வ":1394,"ா":1395,"ி":1396,"ு":1397,"ே":1398,"ை":1399,"ನ":1400,"ರ":1401,"ಾ":1402,"ක":1403,"ය":1404,"ර":1405,"ල":1406,"ව":1407,"ා":1408,"ก":1409,"ง":1410,"ต":1411,"ท":1412,"น":1413,"พ":1414,"ม":1415,"ย":1416,"ร":1417,"ล":1418,"ว":1419,"ส":1420,"อ":1421,"า":1422,"เ":1423,"་":1424,"།":1425,"ག":1426,"ང":1427,"ད":1428,"ན":1429,"པ":1430,"བ":1431,"མ":1432,"འ":1433,"ར":1434,"ལ":1435,"ས":1436,"မ":1437,"ა":1438,"ბ":1439,"გ":1440,"დ":1441,"ე":1442,"ვ":1443,"თ":1444,"ი":1445,"კ":1446,"ლ":1447,"მ":1448,"ნ":1449,"ო":1450,"რ":1451,"ს":1452,"ტ":1453,"უ":1454,"ᄀ":1455,"ᄂ":1456,"ᄃ":1457,"ᄅ":1458,"ᄆ":1459,"ᄇ":1460,"ᄉ":1461,"ᄊ":1462,"ᄋ":1463,"ᄌ":1464,"ᄎ":1465,"ᄏ":1466,"ᄐ":1467,"ᄑ":1468,"ᄒ":1469,"ᅡ":1470,"ᅢ":1471,"ᅥ":1472,"ᅦ":1473,"ᅧ":1474,"ᅩ":1475,"ᅪ":1476,"ᅭ":1477,"ᅮ":1478,"ᅯ":1479,"ᅲ":1480,"ᅳ":1481,"ᅴ":1482,"ᅵ":1483,"ᆨ":1484,"ᆫ":1485,"ᆯ":1486,"ᆷ":1487,"ᆸ":1488,"ᆼ":1489,"ᴬ":1490,"ᴮ":1491,"ᴰ":1492,"ᴵ":1493,"ᴺ":1494,"ᵀ":1495,"ᵃ":1496,"ᵇ":1497,"ᵈ":1498,"ᵉ":1499,"ᵍ":1500,"ᵏ":1501,"ᵐ":1502,"ᵒ":1503,"ᵖ":1504,"ᵗ":1505,"ᵘ":1506,"ᵢ":1507,"ᵣ":1508,"ᵤ":1509,"ᵥ":1510,"ᶜ":1511,"ᶠ":1512,"‐":1513,"‑":1514,"‒":1515,"–":1516,"—":1517,"―":1518,"‖":1519,"‘":1520,"’":1521,"‚":1522,"“":1523,"”":1524,"„":1525,"†":1526,"‡":1527,"•":1528,"…":1529,"‰":1530,"′":1531,"″":1532,"›":1533,"‿":1534,"⁄":1535,"⁰":1536,"ⁱ":1537,"⁴":1538,"⁵":1539,"⁶":1540,"⁷":1541,"⁸":1542,"⁹":1543,"⁺":1544,"⁻":1545,"ⁿ":1546,"₀":1547,"₁":1548,"₂":1549,"₃":1550,"₄":1551,"₅":1552,"₆":1553,"₇":1554,"₈":1555,"₉":1556,"₊":1557,"₍":1558,"₎":1559,"ₐ":1560,"ₑ":1561,"ₒ":1562,"ₓ":1563,"ₕ":1564,"ₖ":1565,"ₗ":1566,"ₘ":1567,"ₙ":1568,"ₚ":1569,"ₛ":1570,"ₜ":1571,"₤":1572,"₩":1573,"€":1574,"₱":1575,"₹":1576,"ℓ":1577,"№":1578,"ℝ":1579,"™":1580,"⅓":1581,"⅔":1582,"←":1583,"↑":1584,"→":1585,"↓":1586,"↔":1587,"↦":1588,"⇄":1589,"⇌":1590,"⇒":1591,"∂":1592,"∅":1593,"∆":1594,"∇":1595,"∈":1596,"−":1597,"∗":1598,"∘":1599,"√":1600,"∞":1601,"∧":1602,"∨":1603,"∩":1604,"∪":1605,"≈":1606,"≡":1607,"≤":1608,"≥":1609,"⊂":1610,"⊆":1611,"⊕":1612,"⊗":1613,"⋅":1614,"─":1615,"│":1616,"■":1617,"▪":1618,"●":1619,"★":1620,"☆":1621,"☉":1622,"♠":1623,"♣":1624,"♥":1625,"♦":1626,"♭":1627,"♯":1628,"⟨":1629,"⟩":1630,"ⱼ":1631,"⺩":1632,"⺼":1633,"⽥":1634,"、":1635,"。":1636,"〈":1637,"〉":1638,"《":1639,"》":1640,"「":1641,"」":1642,"『":1643,"』":1644,"〜":1645,"あ":1646,"い":1647,"う":1648,"え":1649,"お":1650,"か":1651,"き":1652,"く":1653,"け":1654,"こ":1655,"さ":1656,"し":1657,"す":1658,"せ":1659,"そ":1660,"た":1661,"ち":1662,"っ":1663,"つ":1664,"て":1665,"と":1666,"な":1667,"に":1668,"ぬ":1669,"ね":1670,"の":1671,"は":1672,"ひ":1673,"ふ":1674,"へ":1675,"ほ":1676,"ま":1677,"み":1678,"む":1679,"め":1680,"も":1681,"や":1682,"ゆ":1683,"よ":1684,"ら":1685,"り":1686,"る":1687,"れ":1688,"ろ":1689,"を":1690,"ん":1691,"ァ":1692,"ア":1693,"ィ":1694,"イ":1695,"ウ":1696,"ェ":1697,"エ":1698,"オ":1699,"カ":1700,"キ":1701,"ク":1702,"ケ":1703,"コ":1704,"サ":1705,"シ":1706,"ス":1707,"セ":1708,"タ":1709,"チ":1710,"ッ":1711,"ツ":1712,"テ":1713,"ト":1714,"ナ":1715,"ニ":1716,"ノ":1717,"ハ":1718,"ヒ":1719,"フ":1720,"ヘ":1721,"ホ":1722,"マ":1723,"ミ":1724,"ム":1725,"メ":1726,"モ":1727,"ャ":1728,"ュ":1729,"ョ":1730,"ラ":1731,"リ":1732,"ル":1733,"レ":1734,"ロ":1735,"ワ":1736,"ン":1737,"・":1738,"ー":1739,"一":1740,"三":1741,"上":1742,"下":1743,"不":1744,"世":1745,"中":1746,"主":1747,"久":1748,"之":1749,"也":1750,"事":1751,"二":1752,"五":1753,"井":1754,"京":1755,"人":1756,"亻":1757,"仁":1758,"介":1759,"代":1760,"仮":1761,"伊":1762,"会":1763,"佐":1764,"侍":1765,"保":1766,"信":1767,"健":1768,"元":1769,"光":1770,"八":1771,"公":1772,"内":1773,"出":1774,"分":1775,"前":1776,"劉":1777,"力":1778,"加":1779,"勝":1780,"北":1781,"区":1782,"十":1783,"千":1784,"南":1785,"博":1786,"原":1787,"口":1788,"古":1789,"史":1790,"司":1791,"合":1792,"吉":1793,"同":1794,"名":1795,"和":1796,"囗":1797,"四":1798,"国":1799,"國":1800,"土":1801,"地":1802,"坂":1803,"城":1804,"堂":1805,"場":1806,"士":1807,"夏":1808,"外":1809,"大":1810,"天":1811,"太":1812,"夫":1813,"奈":1814,"女":1815,"子":1816,"学":1817,"宀":1818,"宇":1819,"安":1820,"宗":1821,"定":1822,"宣":1823,"宮":1824,"家":1825,"宿":1826,"寺":1827,"將":1828,"小":1829,"尚":1830,"山":1831,"岡":1832,"島":1833,"崎":1834,"川":1835,"州":1836,"巿":1837,"帝":1838,"平":1839,"年":1840,"幸":1841,"广":1842,"弘":1843,"張":1844,"彳":1845,"後":1846,"御":1847,"德":1848,"心":1849,"忄":1850,"志":1851,"忠":1852,"愛":1853,"成":1854,"我":1855,"戦":1856,"戸":1857,"手":1858,"扌":1859,"政":1860,"文":1861,"新":1862,"方":1863,"日":1864,"明":1865,"星":1866,"春":1867,"昭":1868,"智":1869,"曲":1870,"書":1871,"月":1872,"有":1873,"朝":1874,"木":1875,"本":1876,"李":1877,"村":1878,"東":1879,"松":1880,"林":1881,"森":1882,"楊":1883,"樹":1884,"橋":1885,"歌":1886,"止":1887,"正":1888,"武":1889,"比":1890,"氏":1891,"民":1892,"水":1893,"氵":1894,"氷":1895,"永":1896,"江":1897,"沢":1898,"河":1899,"治":1900,"法":1901,"海":1902,"清":1903,"漢":1904,"瀬":1905,"火":1906,"版":1907,"犬":1908,"王":1909,"生":1910,"田":1911,"男":1912,"疒":1913,"発":1914,"白":1915,"的":1916,"皇":1917,"目":1918,"相":1919,"省":1920,"真":1921,"石":1922,"示":1923,"社":1924,"神":1925,"福":1926,"禾":1927,"秀":1928,"秋":1929,"空":1930,"立":1931,"章":1932,"竹":1933,"糹":1934,"美":1935,"義":1936,"耳":1937,"良":1938,"艹":1939,"花":1940,"英":1941,"華":1942,"葉":1943,"藤":1944,"行":1945,"街":1946,"西":1947,"見":1948,"訁":1949,"語":1950,"谷":1951,"貝":1952,"貴":1953,"車":1954,"軍":1955,"辶":1956,"道":1957,"郎":1958,"郡":1959,"部":1960,"都":1961,"里":1962,"野":1963,"金":1964,"鈴":1965,"镇":1966,"長":1967,"門":1968,"間":1969,"阝":1970,"阿":1971,"陳":1972,"陽":1973,"雄":1974,"青":1975,"面":1976,"風":1977,"食":1978,"香":1979,"馬":1980,"高":1981,"龍":1982,"龸":1983,"fi":1984,"fl":1985,"！":1986,"（":1987,"）":1988,"，":1989,"－":1990,"．":1991,"／":1992,"：":1993,"？":1994,"～":1995,"the":1996,"of":1997,"and":1998,"in":1999,"to":2000,"was":2001,"he":2002,"is":2003,"as":2004,"for":2005,"on":2006,"with":2007,"that":2008,"it":2009,"his":2010,"by":2011,"at":2012,"from":2013,"her":2014,"##s":2015,"she":2016,"you":2017,"had":2018,"an":2019,"were":2020,"but":2021,"be":2022,"this":2023,"are":2024,"not":2025,"my":2026,"they":2027,"one":2028,"which":2029,"or":2030,"have":2031,"him":2032,"me":2033,"first":2034,"all":2035,"also":2036,"their":2037,"has":2038,"up":2039,"who":2040,"out":2041,"been":2042,"when":2043,"after":2044,"there":2045,"into":2046,"new":2047,"two":2048,"its":2049,"##a":2050,"time":2051,"would":2052,"no":2053,"what":2054,"about":2055,"said":2056,"we":2057,"over":2058,"then":2059,"other":2060,"so":2061,"more":2062,"##e":2063,"can":2064,"if":2065,"like":2066,"back":2067,"them":2068,"only":2069,"some":2070,"could":2071,"##i":2072,"where":2073,"just":2074,"##ing":2075,"during":2076,"before":2077,"##n":2078,"do":2079,"##o":2080,"made":2081,"school":2082,"through":2083,"than":2084,"now":2085,"years":2086,"most":2087,"world":2088,"may":2089,"between":2090,"down":2091,"well":2092,"three":2093,"##d":2094,"year":2095,"while":2096,"will":2097,"##ed":2098,"##r":2099,"##y":2100,"later":2101,"##t":2102,"city":2103,"under":2104,"around":2105,"did":2106,"such":2107,"being":2108,"used":2109,"state":2110,"people":2111,"part":2112,"know":2113,"against":2114,"your":2115,"many":2116,"second":2117,"university":2118,"both":2119,"national":2120,"##er":2121,"these":2122,"don":2123,"known":2124,"off":2125,"way":2126,"until":2127,"re":2128,"how":2129,"even":2130,"get":2131,"head":2132,"...":2133,"didn":2134,"##ly":2135,"team":2136,"american":2137,"because":2138,"de":2139,"##l":2140,"born":2141,"united":2142,"film":2143,"since":2144,"still":2145,"long":2146,"work":2147,"south":2148,"us":2149,"became":2150,"any":2151,"high":2152,"again":2153,"day":2154,"family":2155,"see":2156,"right":2157,"man":2158,"eyes":2159,"house":2160,"season":2161,"war":2162,"states":2163,"including":2164,"took":2165,"life":2166,"north":2167,"same":2168,"each":2169,"called":2170,"name":2171,"much":2172,"place":2173,"however":2174,"go":2175,"four":2176,"group":2177,"another":2178,"found":2179,"won":2180,"area":2181,"here":2182,"going":2183,"10":2184,"away":2185,"series":2186,"left":2187,"home":2188,"music":2189,"best":2190,"make":2191,"hand":2192,"number":2193,"company":2194,"several":2195,"never":2196,"last":2197,"john":2198,"000":2199,"very":2200,"album":2201,"take":2202,"end":2203,"good":2204,"too":2205,"following":2206,"released":2207,"game":2208,"played":2209,"little":2210,"began":2211,"district":2212,"##m":2213,"old":2214,"want":2215,"those":2216,"side":2217,"held":2218,"own":2219,"early":2220,"county":2221,"ll":2222,"league":2223,"use":2224,"west":2225,"##u":2226,"face":2227,"think":2228,"##es":2229,"2010":2230,"government":2231,"##h":2232,"march":2233,"came":2234,"small":2235,"general":2236,"town":2237,"june":2238,"##on":2239,"line":2240,"based":2241,"something":2242,"##k":2243,"september":2244,"thought":2245,"looked":2246,"along":2247,"international":2248,"2011":2249,"air":2250,"july":2251,"club":2252,"went":2253,"january":2254,"october":2255,"our":2256,"august":2257,"april":2258,"york":2259,"12":2260,"few":2261,"2012":2262,"2008":2263,"east":2264,"show":2265,"member":2266,"college":2267,"2009":2268,"father":2269,"public":2270,"##us":2271,"come":2272,"men":2273,"five":2274,"set":2275,"station":2276,"church":2277,"##c":2278,"next":2279,"former":2280,"november":2281,"room":2282,"party":2283,"located":2284,"december":2285,"2013":2286,"age":2287,"got":2288,"2007":2289,"##g":2290,"system":2291,"let":2292,"love":2293,"2006":2294,"though":2295,"every":2296,"2014":2297,"look":2298,"song":2299,"water":2300,"century":2301,"without":2302,"body":2303,"black":2304,"night":2305,"within":2306,"great":2307,"women":2308,"single":2309,"ve":2310,"building":2311,"large":2312,"population":2313,"river":2314,"named":2315,"band":2316,"white":2317,"started":2318,"##an":2319,"once":2320,"15":2321,"20":2322,"should":2323,"18":2324,"2015":2325,"service":2326,"top":2327,"built":2328,"british":2329,"open":2330,"death":2331,"king":2332,"moved":2333,"local":2334,"times":2335,"children":2336,"february":2337,"book":2338,"why":2339,"11":2340,"door":2341,"need":2342,"president":2343,"order":2344,"final":2345,"road":2346,"wasn":2347,"although":2348,"due":2349,"major":2350,"died":2351,"village":2352,"third":2353,"knew":2354,"2016":2355,"asked":2356,"turned":2357,"st":2358,"wanted":2359,"say":2360,"##p":2361,"together":2362,"received":2363,"main":2364,"son":2365,"served":2366,"different":2367,"##en":2368,"behind":2369,"himself":2370,"felt":2371,"members":2372,"power":2373,"football":2374,"law":2375,"voice":2376,"play":2377,"##in":2378,"near":2379,"park":2380,"history":2381,"30":2382,"having":2383,"2005":2384,"16":2385,"##man":2386,"saw":2387,"mother":2388,"##al":2389,"army":2390,"point":2391,"front":2392,"help":2393,"english":2394,"street":2395,"art":2396,"late":2397,"hands":2398,"games":2399,"award":2400,"##ia":2401,"young":2402,"14":2403,"put":2404,"published":2405,"country":2406,"division":2407,"across":2408,"told":2409,"13":2410,"often":2411,"ever":2412,"french":2413,"london":2414,"center":2415,"six":2416,"red":2417,"2017":2418,"led":2419,"days":2420,"include":2421,"light":2422,"25":2423,"find":2424,"tell":2425,"among":2426,"species":2427,"really":2428,"according":2429,"central":2430,"half":2431,"2004":2432,"form":2433,"original":2434,"gave":2435,"office":2436,"making":2437,"enough":2438,"lost":2439,"full":2440,"opened":2441,"must":2442,"included":2443,"live":2444,"given":2445,"german":2446,"player":2447,"run":2448,"business":2449,"woman":2450,"community":2451,"cup":2452,"might":2453,"million":2454,"land":2455,"2000":2456,"court":2457,"development":2458,"17":2459,"short":2460,"round":2461,"ii":2462,"km":2463,"seen":2464,"class":2465,"story":2466,"always":2467,"become":2468,"sure":2469,"research":2470,"almost":2471,"director":2472,"council":2473,"la":2474,"##2":2475,"career":2476,"things":2477,"using":2478,"island":2479,"##z":2480,"couldn":2481,"car":2482,"##is":2483,"24":2484,"close":2485,"force":2486,"##1":2487,"better":2488,"free":2489,"support":2490,"control":2491,"field":2492,"students":2493,"2003":2494,"education":2495,"married":2496,"##b":2497,"nothing":2498,"worked":2499,"others":2500,"record":2501,"big":2502,"inside":2503,"level":2504,"anything":2505,"continued":2506,"give":2507,"james":2508,"##3":2509,"military":2510,"established":2511,"non":2512,"returned":2513,"feel":2514,"does":2515,"title":2516,"written":2517,"thing":2518,"feet":2519,"william":2520,"far":2521,"co":2522,"association":2523,"hard":2524,"already":2525,"2002":2526,"##ra":2527,"championship":2528,"human":2529,"western":2530,"100":2531,"##na":2532,"department":2533,"hall":2534,"role":2535,"various":2536,"production":2537,"21":2538,"19":2539,"heart":2540,"2001":2541,"living":2542,"fire":2543,"version":2544,"##ers":2545,"##f":2546,"television":2547,"royal":2548,"##4":2549,"produced":2550,"working":2551,"act":2552,"case":2553,"society":2554,"region":2555,"present":2556,"radio":2557,"period":2558,"looking":2559,"least":2560,"total":2561,"keep":2562,"england":2563,"wife":2564,"program":2565,"per":2566,"brother":2567,"mind":2568,"special":2569,"22":2570,"##le":2571,"am":2572,"works":2573,"soon":2574,"##6":2575,"political":2576,"george":2577,"services":2578,"taken":2579,"created":2580,"##7":2581,"further":2582,"able":2583,"reached":2584,"david":2585,"union":2586,"joined":2587,"upon":2588,"done":2589,"important":2590,"social":2591,"information":2592,"either":2593,"##ic":2594,"##x":2595,"appeared":2596,"position":2597,"ground":2598,"lead":2599,"rock":2600,"dark":2601,"election":2602,"23":2603,"board":2604,"france":2605,"hair":2606,"course":2607,"arms":2608,"site":2609,"police":2610,"girl":2611,"instead":2612,"real":2613,"sound":2614,"##v":2615,"words":2616,"moment":2617,"##te":2618,"someone":2619,"##8":2620,"summer":2621,"project":2622,"announced":2623,"san":2624,"less":2625,"wrote":2626,"past":2627,"followed":2628,"##5":2629,"blue":2630,"founded":2631,"al":2632,"finally":2633,"india":2634,"taking":2635,"records":2636,"america":2637,"##ne":2638,"1999":2639,"design":2640,"considered":2641,"northern":2642,"god":2643,"stop":2644,"battle":2645,"toward":2646,"european":2647,"outside":2648,"described":2649,"track":2650,"today":2651,"playing":2652,"language":2653,"28":2654,"call":2655,"26":2656,"heard":2657,"professional":2658,"low":2659,"australia":2660,"miles":2661,"california":2662,"win":2663,"yet":2664,"green":2665,"##ie":2666,"trying":2667,"blood":2668,"##ton":2669,"southern":2670,"science":2671,"maybe":2672,"everything":2673,"match":2674,"square":2675,"27":2676,"mouth":2677,"video":2678,"race":2679,"recorded":2680,"leave":2681,"above":2682,"##9":2683,"daughter":2684,"points":2685,"space":2686,"1998":2687,"museum":2688,"change":2689,"middle":2690,"common":2691,"##0":2692,"move":2693,"tv":2694,"post":2695,"##ta":2696,"lake":2697,"seven":2698,"tried":2699,"elected":2700,"closed":2701,"ten":2702,"paul":2703,"minister":2704,"##th":2705,"months":2706,"start":2707,"chief":2708,"return":2709,"canada":2710,"person":2711,"sea":2712,"release":2713,"similar":2714,"modern":2715,"brought":2716,"rest":2717,"hit":2718,"formed":2719,"mr":2720,"##la":2721,"1997":2722,"floor":2723,"event":2724,"doing":2725,"thomas":2726,"1996":2727,"robert":2728,"care":2729,"killed":2730,"training":2731,"star":2732,"week":2733,"needed":2734,"turn":2735,"finished":2736,"railway":2737,"rather":2738,"news":2739,"health":2740,"sent":2741,"example":2742,"ran":2743,"term":2744,"michael":2745,"coming":2746,"currently":2747,"yes":2748,"forces":2749,"despite":2750,"gold":2751,"areas":2752,"50":2753,"stage":2754,"fact":2755,"29":2756,"dead":2757,"says":2758,"popular":2759,"2018":2760,"originally":2761,"germany":2762,"probably":2763,"developed":2764,"result":2765,"pulled":2766,"friend":2767,"stood":2768,"money":2769,"running":2770,"mi":2771,"signed":2772,"word":2773,"songs":2774,"child":2775,"eventually":2776,"met":2777,"tour":2778,"average":2779,"teams":2780,"minutes":2781,"festival":2782,"current":2783,"deep":2784,"kind":2785,"1995":2786,"decided":2787,"usually":2788,"eastern":2789,"seemed":2790,"##ness":2791,"episode":2792,"bed":2793,"added":2794,"table":2795,"indian":2796,"private":2797,"charles":2798,"route":2799,"available":2800,"idea":2801,"throughout":2802,"centre":2803,"addition":2804,"appointed":2805,"style":2806,"1994":2807,"books":2808,"eight":2809,"construction":2810,"press":2811,"mean":2812,"wall":2813,"friends":2814,"remained":2815,"schools":2816,"study":2817,"##ch":2818,"##um":2819,"institute":2820,"oh":2821,"chinese":2822,"sometimes":2823,"events":2824,"possible":2825,"1992":2826,"australian":2827,"type":2828,"brown":2829,"forward":2830,"talk":2831,"process":2832,"food":2833,"debut":2834,"seat":2835,"performance":2836,"committee":2837,"features":2838,"character":2839,"arts":2840,"herself":2841,"else":2842,"lot":2843,"strong":2844,"russian":2845,"range":2846,"hours":2847,"peter":2848,"arm":2849,"##da":2850,"morning":2851,"dr":2852,"sold":2853,"##ry":2854,"quickly":2855,"directed":2856,"1993":2857,"guitar":2858,"china":2859,"##w":2860,"31":2861,"list":2862,"##ma":2863,"performed":2864,"media":2865,"uk":2866,"players":2867,"smile":2868,"##rs":2869,"myself":2870,"40":2871,"placed":2872,"coach":2873,"province":2874,"towards":2875,"wouldn":2876,"leading":2877,"whole":2878,"boy":2879,"official":2880,"designed":2881,"grand":2882,"census":2883,"##el":2884,"europe":2885,"attack":2886,"japanese":2887,"henry":2888,"1991":2889,"##re":2890,"##os":2891,"cross":2892,"getting":2893,"alone":2894,"action":2895,"lower":2896,"network":2897,"wide":2898,"washington":2899,"japan":2900,"1990":2901,"hospital":2902,"believe":2903,"changed":2904,"sister":2905,"##ar":2906,"hold":2907,"gone":2908,"sir":2909,"hadn":2910,"ship":2911,"##ka":2912,"studies":2913,"academy":2914,"shot":2915,"rights":2916,"below":2917,"base":2918,"bad":2919,"involved":2920,"kept":2921,"largest":2922,"##ist":2923,"bank":2924,"future":2925,"especially":2926,"beginning":2927,"mark":2928,"movement":2929,"section":2930,"female":2931,"magazine":2932,"plan":2933,"professor":2934,"lord":2935,"longer":2936,"##ian":2937,"sat":2938,"walked":2939,"hill":2940,"actually":2941,"civil":2942,"energy":2943,"model":2944,"families":2945,"size":2946,"thus":2947,"aircraft":2948,"completed":2949,"includes":2950,"data":2951,"captain":2952,"##or":2953,"fight":2954,"vocals":2955,"featured":2956,"richard":2957,"bridge":2958,"fourth":2959,"1989":2960,"officer":2961,"stone":2962,"hear":2963,"##ism":2964,"means":2965,"medical":2966,"groups":2967,"management":2968,"self":2969,"lips":2970,"competition":2971,"entire":2972,"lived":2973,"technology":2974,"leaving":2975,"federal":2976,"tournament":2977,"bit":2978,"passed":2979,"hot":2980,"independent":2981,"awards":2982,"kingdom":2983,"mary":2984,"spent":2985,"fine":2986,"doesn":2987,"reported":2988,"##ling":2989,"jack":2990,"fall":2991,"raised":2992,"itself":2993,"stay":2994,"true":2995,"studio":2996,"1988":2997,"sports":2998,"replaced":2999,"paris":3000,"systems":3001,"saint":3002,"leader":3003,"theatre":3004,"whose":3005,"market":3006,"capital":3007,"parents":3008,"spanish":3009,"canadian":3010,"earth":3011,"##ity":3012,"cut":3013,"degree":3014,"writing":3015,"bay":3016,"christian":3017,"awarded":3018,"natural":3019,"higher":3020,"bill":3021,"##as":3022,"coast":3023,"provided":3024,"previous":3025,"senior":3026,"ft":3027,"valley":3028,"organization":3029,"stopped":3030,"onto":3031,"countries":3032,"parts":3033,"conference":3034,"queen":3035,"security":3036,"interest":3037,"saying":3038,"allowed":3039,"master":3040,"earlier":3041,"phone":3042,"matter":3043,"smith":3044,"winning":3045,"try":3046,"happened":3047,"moving":3048,"campaign":3049,"los":3050,"##ley":3051,"breath":3052,"nearly":3053,"mid":3054,"1987":3055,"certain":3056,"girls":3057,"date":3058,"italian":3059,"african":3060,"standing":3061,"fell":3062,"artist":3063,"##ted":3064,"shows":3065,"deal":3066,"mine":3067,"industry":3068,"1986":3069,"##ng":3070,"everyone":3071,"republic":3072,"provide":3073,"collection":3074,"library":3075,"student":3076,"##ville":3077,"primary":3078,"owned":3079,"older":3080,"via":3081,"heavy":3082,"1st":3083,"makes":3084,"##able":3085,"attention":3086,"anyone":3087,"africa":3088,"##ri":3089,"stated":3090,"length":3091,"ended":3092,"fingers":3093,"command":3094,"staff":3095,"skin":3096,"foreign":3097,"opening":3098,"governor":3099,"okay":3100,"medal":3101,"kill":3102,"sun":3103,"cover":3104,"job":3105,"1985":3106,"introduced":3107,"chest":3108,"hell":3109,"feeling":3110,"##ies":3111,"success":3112,"meet":3113,"reason":3114,"standard":3115,"meeting":3116,"novel":3117,"1984":3118,"trade":3119,"source":3120,"buildings":3121,"##land":3122,"rose":3123,"guy":3124,"goal":3125,"##ur":3126,"chapter":3127,"native":3128,"husband":3129,"previously":3130,"unit":3131,"limited":3132,"entered":3133,"weeks":3134,"producer":3135,"operations":3136,"mountain":3137,"takes":3138,"covered":3139,"forced":3140,"related":3141,"roman":3142,"complete":3143,"successful":3144,"key":3145,"texas":3146,"cold":3147,"##ya":3148,"channel":3149,"1980":3150,"traditional":3151,"films":3152,"dance":3153,"clear":3154,"approximately":3155,"500":3156,"nine":3157,"van":3158,"prince":3159,"question":3160,"active":3161,"tracks":3162,"ireland":3163,"regional":3164,"silver":3165,"author":3166,"personal":3167,"sense":3168,"operation":3169,"##ine":3170,"economic":3171,"1983":3172,"holding":3173,"twenty":3174,"isbn":3175,"additional":3176,"speed":3177,"hour":3178,"edition":3179,"regular":3180,"historic":3181,"places":3182,"whom":3183,"shook":3184,"movie":3185,"km²":3186,"secretary":3187,"prior":3188,"report":3189,"chicago":3190,"read":3191,"foundation":3192,"view":3193,"engine":3194,"scored":3195,"1982":3196,"units":3197,"ask":3198,"airport":3199,"property":3200,"ready":3201,"immediately":3202,"lady":3203,"month":3204,"listed":3205,"contract":3206,"##de":3207,"manager":3208,"themselves":3209,"lines":3210,"##ki":3211,"navy":3212,"writer":3213,"meant":3214,"##ts":3215,"runs":3216,"##ro":3217,"practice":3218,"championships":3219,"singer":3220,"glass":3221,"commission":3222,"required":3223,"forest":3224,"starting":3225,"culture":3226,"generally":3227,"giving":3228,"access":3229,"attended":3230,"test":3231,"couple":3232,"stand":3233,"catholic":3234,"martin":3235,"caught":3236,"executive":3237,"##less":3238,"eye":3239,"##ey":3240,"thinking":3241,"chair":3242,"quite":3243,"shoulder":3244,"1979":3245,"hope":3246,"decision":3247,"plays":3248,"defeated":3249,"municipality":3250,"whether":3251,"structure":3252,"offered":3253,"slowly":3254,"pain":3255,"ice":3256,"direction":3257,"##ion":3258,"paper":3259,"mission":3260,"1981":3261,"mostly":3262,"200":3263,"noted":3264,"individual":3265,"managed":3266,"nature":3267,"lives":3268,"plant":3269,"##ha":3270,"helped":3271,"except":3272,"studied":3273,"computer":3274,"figure":3275,"relationship":3276,"issue":3277,"significant":3278,"loss":3279,"die":3280,"smiled":3281,"gun":3282,"ago":3283,"highest":3284,"1972":3285,"##am":3286,"male":3287,"bring":3288,"goals":3289,"mexico":3290,"problem":3291,"distance":3292,"commercial":3293,"completely":3294,"location":3295,"annual":3296,"famous":3297,"drive":3298,"1976":3299,"neck":3300,"1978":3301,"surface":3302,"caused":3303,"italy":3304,"understand":3305,"greek":3306,"highway":3307,"wrong":3308,"hotel":3309,"comes":3310,"appearance":3311,"joseph":3312,"double":3313,"issues":3314,"musical":3315,"companies":3316,"castle":3317,"income":3318,"review":3319,"assembly":3320,"bass":3321,"initially":3322,"parliament":3323,"artists":3324,"experience":3325,"1974":3326,"particular":3327,"walk":3328,"foot":3329,"engineering":3330,"talking":3331,"window":3332,"dropped":3333,"##ter":3334,"miss":3335,"baby":3336,"boys":3337,"break":3338,"1975":3339,"stars":3340,"edge":3341,"remember":3342,"policy":3343,"carried":3344,"train":3345,"stadium":3346,"bar":3347,"sex":3348,"angeles":3349,"evidence":3350,"##ge":3351,"becoming":3352,"assistant":3353,"soviet":3354,"1977":3355,"upper":3356,"step":3357,"wing":3358,"1970":3359,"youth":3360,"financial":3361,"reach":3362,"##ll":3363,"actor":3364,"numerous":3365,"##se":3366,"##st":3367,"nodded":3368,"arrived":3369,"##ation":3370,"minute":3371,"##nt":3372,"believed":3373,"sorry":3374,"complex":3375,"beautiful":3376,"victory":3377,"associated":3378,"temple":3379,"1968":3380,"1973":3381,"chance":3382,"perhaps":3383,"metal":3384,"##son":3385,"1945":3386,"bishop":3387,"##et":3388,"lee":3389,"launched":3390,"particularly":3391,"tree":3392,"le":3393,"retired":3394,"subject":3395,"prize":3396,"contains":3397,"yeah":3398,"theory":3399,"empire":3400,"##ce":3401,"suddenly":3402,"waiting":3403,"trust":3404,"recording":3405,"##to":3406,"happy":3407,"terms":3408,"camp":3409,"champion":3410,"1971":3411,"religious":3412,"pass":3413,"zealand":3414,"names":3415,"2nd":3416,"port":3417,"ancient":3418,"tom":3419,"corner":3420,"represented":3421,"watch":3422,"legal":3423,"anti":3424,"justice":3425,"cause":3426,"watched":3427,"brothers":3428,"45":3429,"material":3430,"changes":3431,"simply":3432,"response":3433,"louis":3434,"fast":3435,"##ting":3436,"answer":3437,"60":3438,"historical":3439,"1969":3440,"stories":3441,"straight":3442,"create":3443,"feature":3444,"increased":3445,"rate":3446,"administration":3447,"virginia":3448,"el":3449,"activities":3450,"cultural":3451,"overall":3452,"winner":3453,"programs":3454,"basketball":3455,"legs":3456,"guard":3457,"beyond":3458,"cast":3459,"doctor":3460,"mm":3461,"flight":3462,"results":3463,"remains":3464,"cost":3465,"effect":3466,"winter":3467,"##ble":3468,"larger":3469,"islands":3470,"problems":3471,"chairman":3472,"grew":3473,"commander":3474,"isn":3475,"1967":3476,"pay":3477,"failed":3478,"selected":3479,"hurt":3480,"fort":3481,"box":3482,"regiment":3483,"majority":3484,"journal":3485,"35":3486,"edward":3487,"plans":3488,"##ke":3489,"##ni":3490,"shown":3491,"pretty":3492,"irish":3493,"characters":3494,"directly":3495,"scene":3496,"likely":3497,"operated":3498,"allow":3499,"spring":3500,"##j":3501,"junior":3502,"matches":3503,"looks":3504,"mike":3505,"houses":3506,"fellow":3507,"##tion":3508,"beach":3509,"marriage":3510,"##ham":3511,"##ive":3512,"rules":3513,"oil":3514,"65":3515,"florida":3516,"expected":3517,"nearby":3518,"congress":3519,"sam":3520,"peace":3521,"recent":3522,"iii":3523,"wait":3524,"subsequently":3525,"cell":3526,"##do":3527,"variety":3528,"serving":3529,"agreed":3530,"please":3531,"poor":3532,"joe":3533,"pacific":3534,"attempt":3535,"wood":3536,"democratic":3537,"piece":3538,"prime":3539,"##ca":3540,"rural":3541,"mile":3542,"touch":3543,"appears":3544,"township":3545,"1964":3546,"1966":3547,"soldiers":3548,"##men":3549,"##ized":3550,"1965":3551,"pennsylvania":3552,"closer":3553,"fighting":3554,"claimed":3555,"score":3556,"jones":3557,"physical":3558,"editor":3559,"##ous":3560,"filled":3561,"genus":3562,"specific":3563,"sitting":3564,"super":3565,"mom":3566,"##va":3567,"therefore":3568,"supported":3569,"status":3570,"fear":3571,"cases":3572,"store":3573,"meaning":3574,"wales":3575,"minor":3576,"spain":3577,"tower":3578,"focus":3579,"vice":3580,"frank":3581,"follow":3582,"parish":3583,"separate":3584,"golden":3585,"horse":3586,"fifth":3587,"remaining":3588,"branch":3589,"32":3590,"presented":3591,"stared":3592,"##id":3593,"uses":3594,"secret":3595,"forms":3596,"##co":3597,"baseball":3598,"exactly":3599,"##ck":3600,"choice":3601,"note":3602,"discovered":3603,"travel":3604,"composed":3605,"truth":3606,"russia":3607,"ball":3608,"color":3609,"kiss":3610,"dad":3611,"wind":3612,"continue":3613,"ring":3614,"referred":3615,"numbers":3616,"digital":3617,"greater":3618,"##ns":3619,"metres":3620,"slightly":3621,"direct":3622,"increase":3623,"1960":3624,"responsible":3625,"crew":3626,"rule":3627,"trees":3628,"troops":3629,"##no":3630,"broke":3631,"goes":3632,"individuals":3633,"hundred":3634,"weight":3635,"creek":3636,"sleep":3637,"memory":3638,"defense":3639,"provides":3640,"ordered":3641,"code":3642,"value":3643,"jewish":3644,"windows":3645,"1944":3646,"safe":3647,"judge":3648,"whatever":3649,"corps":3650,"realized":3651,"growing":3652,"pre":3653,"##ga":3654,"cities":3655,"alexander":3656,"gaze":3657,"lies":3658,"spread":3659,"scott":3660,"letter":3661,"showed":3662,"situation":3663,"mayor":3664,"transport":3665,"watching":3666,"workers":3667,"extended":3668,"##li":3669,"expression":3670,"normal":3671,"##ment":3672,"chart":3673,"multiple":3674,"border":3675,"##ba":3676,"host":3677,"##ner":3678,"daily":3679,"mrs":3680,"walls":3681,"piano":3682,"##ko":3683,"heat":3684,"cannot":3685,"##ate":3686,"earned":3687,"products":3688,"drama":3689,"era":3690,"authority":3691,"seasons":3692,"join":3693,"grade":3694,"##io":3695,"sign":3696,"difficult":3697,"machine":3698,"1963":3699,"territory":3700,"mainly":3701,"##wood":3702,"stations":3703,"squadron":3704,"1962":3705,"stepped":3706,"iron":3707,"19th":3708,"##led":3709,"serve":3710,"appear":3711,"sky":3712,"speak":3713,"broken":3714,"charge":3715,"knowledge":3716,"kilometres":3717,"removed":3718,"ships":3719,"article":3720,"campus":3721,"simple":3722,"##ty":3723,"pushed":3724,"britain":3725,"##ve":3726,"leaves":3727,"recently":3728,"cd":3729,"soft":3730,"boston":3731,"latter":3732,"easy":3733,"acquired":3734,"poland":3735,"##sa":3736,"quality":3737,"officers":3738,"presence":3739,"planned":3740,"nations":3741,"mass":3742,"broadcast":3743,"jean":3744,"share":3745,"image":3746,"influence":3747,"wild":3748,"offer":3749,"emperor":3750,"electric":3751,"reading":3752,"headed":3753,"ability":3754,"promoted":3755,"yellow":3756,"ministry":3757,"1942":3758,"throat":3759,"smaller":3760,"politician":3761,"##by":3762,"latin":3763,"spoke":3764,"cars":3765,"williams":3766,"males":3767,"lack":3768,"pop":3769,"80":3770,"##ier":3771,"acting":3772,"seeing":3773,"consists":3774,"##ti":3775,"estate":3776,"1961":3777,"pressure":3778,"johnson":3779,"newspaper":3780,"jr":3781,"chris":3782,"olympics":3783,"online":3784,"conditions":3785,"beat":3786,"elements":3787,"walking":3788,"vote":3789,"##field":3790,"needs":3791,"carolina":3792,"text":3793,"featuring":3794,"global":3795,"block":3796,"shirt":3797,"levels":3798,"francisco":3799,"purpose":3800,"females":3801,"et":3802,"dutch":3803,"duke":3804,"ahead":3805,"gas":3806,"twice":3807,"safety":3808,"serious":3809,"turning":3810,"highly":3811,"lieutenant":3812,"firm":3813,"maria":3814,"amount":3815,"mixed":3816,"daniel":3817,"proposed":3818,"perfect":3819,"agreement":3820,"affairs":3821,"3rd":3822,"seconds":3823,"contemporary":3824,"paid":3825,"1943":3826,"prison":3827,"save":3828,"kitchen":3829,"label":3830,"administrative":3831,"intended":3832,"constructed":3833,"academic":3834,"nice":3835,"teacher":3836,"races":3837,"1956":3838,"formerly":3839,"corporation":3840,"ben":3841,"nation":3842,"issued":3843,"shut":3844,"1958":3845,"drums":3846,"housing":3847,"victoria":3848,"seems":3849,"opera":3850,"1959":3851,"graduated":3852,"function":3853,"von":3854,"mentioned":3855,"picked":3856,"build":3857,"recognized":3858,"shortly":3859,"protection":3860,"picture":3861,"notable":3862,"exchange":3863,"elections":3864,"1980s":3865,"loved":3866,"percent":3867,"racing":3868,"fish":3869,"elizabeth":3870,"garden":3871,"volume":3872,"hockey":3873,"1941":3874,"beside":3875,"settled":3876,"##ford":3877,"1940":3878,"competed":3879,"replied":3880,"drew":3881,"1948":3882,"actress":3883,"marine":3884,"scotland":3885,"steel":3886,"glanced":3887,"farm":3888,"steve":3889,"1957":3890,"risk":3891,"tonight":3892,"positive":3893,"magic":3894,"singles":3895,"effects":3896,"gray":3897,"screen":3898,"dog":3899,"##ja":3900,"residents":3901,"bus":3902,"sides":3903,"none":3904,"secondary":3905,"literature":3906,"polish":3907,"destroyed":3908,"flying":3909,"founder":3910,"households":3911,"1939":3912,"lay":3913,"reserve":3914,"usa":3915,"gallery":3916,"##ler":3917,"1946":3918,"industrial":3919,"younger":3920,"approach":3921,"appearances":3922,"urban":3923,"ones":3924,"1950":3925,"finish":3926,"avenue":3927,"powerful":3928,"fully":3929,"growth":3930,"page":3931,"honor":3932,"jersey":3933,"projects":3934,"advanced":3935,"revealed":3936,"basic":3937,"90":3938,"infantry":3939,"pair":3940,"equipment":3941,"visit":3942,"33":3943,"evening":3944,"search":3945,"grant":3946,"effort":3947,"solo":3948,"treatment":3949,"buried":3950,"republican":3951,"primarily":3952,"bottom":3953,"owner":3954,"1970s":3955,"israel":3956,"gives":3957,"jim":3958,"dream":3959,"bob":3960,"remain":3961,"spot":3962,"70":3963,"notes":3964,"produce":3965,"champions":3966,"contact":3967,"ed":3968,"soul":3969,"accepted":3970,"ways":3971,"del":3972,"##ally":3973,"losing":3974,"split":3975,"price":3976,"capacity":3977,"basis":3978,"trial":3979,"questions":3980,"##ina":3981,"1955":3982,"20th":3983,"guess":3984,"officially":3985,"memorial":3986,"naval":3987,"initial":3988,"##ization":3989,"whispered":3990,"median":3991,"engineer":3992,"##ful":3993,"sydney":3994,"##go":3995,"columbia":3996,"strength":3997,"300":3998,"1952":3999,"tears":4000,"senate":4001,"00":4002,"card":4003,"asian":4004,"agent":4005,"1947":4006,"software":4007,"44":4008,"draw":4009,"warm":4010,"supposed":4011,"com":4012,"pro":4013,"##il":4014,"transferred":4015,"leaned":4016,"##at":4017,"candidate":4018,"escape":4019,"mountains":4020,"asia":4021,"potential":4022,"activity":4023,"entertainment":4024,"seem":4025,"traffic":4026,"jackson":4027,"murder":4028,"36":4029,"slow":4030,"product":4031,"orchestra":4032,"haven":4033,"agency":4034,"bbc":4035,"taught":4036,"website":4037,"comedy":4038,"unable":4039,"storm":4040,"planning":4041,"albums":4042,"rugby":4043,"environment":4044,"scientific":4045,"grabbed":4046,"protect":4047,"##hi":4048,"boat":4049,"typically":4050,"1954":4051,"1953":4052,"damage":4053,"principal":4054,"divided":4055,"dedicated":4056,"mount":4057,"ohio":4058,"##berg":4059,"pick":4060,"fought":4061,"driver":4062,"##der":4063,"empty":4064,"shoulders":4065,"sort":4066,"thank":4067,"berlin":4068,"prominent":4069,"account":4070,"freedom":4071,"necessary":4072,"efforts":4073,"alex":4074,"headquarters":4075,"follows":4076,"alongside":4077,"des":4078,"simon":4079,"andrew":4080,"suggested":4081,"operating":4082,"learning":4083,"steps":4084,"1949":4085,"sweet":4086,"technical":4087,"begin":4088,"easily":4089,"34":4090,"teeth":4091,"speaking":4092,"settlement":4093,"scale":4094,"##sh":4095,"renamed":4096,"ray":4097,"max":4098,"enemy":4099,"semi":4100,"joint":4101,"compared":4102,"##rd":4103,"scottish":4104,"leadership":4105,"analysis":4106,"offers":4107,"georgia":4108,"pieces":4109,"captured":4110,"animal":4111,"deputy":4112,"guest":4113,"organized":4114,"##lin":4115,"tony":4116,"combined":4117,"method":4118,"challenge":4119,"1960s":4120,"huge":4121,"wants":4122,"battalion":4123,"sons":4124,"rise":4125,"crime":4126,"types":4127,"facilities":4128,"telling":4129,"path":4130,"1951":4131,"platform":4132,"sit":4133,"1990s":4134,"##lo":4135,"tells":4136,"assigned":4137,"rich":4138,"pull":4139,"##ot":4140,"commonly":4141,"alive":4142,"##za":4143,"letters":4144,"concept":4145,"conducted":4146,"wearing":4147,"happen":4148,"bought":4149,"becomes":4150,"holy":4151,"gets":4152,"ocean":4153,"defeat":4154,"languages":4155,"purchased":4156,"coffee":4157,"occurred":4158,"titled":4159,"##q":4160,"declared":4161,"applied":4162,"sciences":4163,"concert":4164,"sounds":4165,"jazz":4166,"brain":4167,"##me":4168,"painting":4169,"fleet":4170,"tax":4171,"nick":4172,"##ius":4173,"michigan":4174,"count":4175,"animals":4176,"leaders":4177,"episodes":4178,"##line":4179,"content":4180,"##den":4181,"birth":4182,"##it":4183,"clubs":4184,"64":4185,"palace":4186,"critical":4187,"refused":4188,"fair":4189,"leg":4190,"laughed":4191,"returning":4192,"surrounding":4193,"participated":4194,"formation":4195,"lifted":4196,"pointed":4197,"connected":4198,"rome":4199,"medicine":4200,"laid":4201,"taylor":4202,"santa":4203,"powers":4204,"adam":4205,"tall":4206,"shared":4207,"focused":4208,"knowing":4209,"yards":4210,"entrance":4211,"falls":4212,"##wa":4213,"calling":4214,"##ad":4215,"sources":4216,"chosen":4217,"beneath":4218,"resources":4219,"yard":4220,"##ite":4221,"nominated":4222,"silence":4223,"zone":4224,"defined":4225,"##que":4226,"gained":4227,"thirty":4228,"38":4229,"bodies":4230,"moon":4231,"##ard":4232,"adopted":4233,"christmas":4234,"widely":4235,"register":4236,"apart":4237,"iran":4238,"premier":4239,"serves":4240,"du":4241,"unknown":4242,"parties":4243,"##les":4244,"generation":4245,"##ff":4246,"continues":4247,"quick":4248,"fields":4249,"brigade":4250,"quiet":4251,"teaching":4252,"clothes":4253,"impact":4254,"weapons":4255,"partner":4256,"flat":4257,"theater":4258,"supreme":4259,"1938":4260,"37":4261,"relations":4262,"##tor":4263,"plants":4264,"suffered":4265,"1936":4266,"wilson":4267,"kids":4268,"begins":4269,"##age":4270,"1918":4271,"seats":4272,"armed":4273,"internet":4274,"models":4275,"worth":4276,"laws":4277,"400":4278,"communities":4279,"classes":4280,"background":4281,"knows":4282,"thanks":4283,"quarter":4284,"reaching":4285,"humans":4286,"carry":4287,"killing":4288,"format":4289,"kong":4290,"hong":4291,"setting":4292,"75":4293,"architecture":4294,"disease":4295,"railroad":4296,"inc":4297,"possibly":4298,"wish":4299,"arthur":4300,"thoughts":4301,"harry":4302,"doors":4303,"density":4304,"##di":4305,"crowd":4306,"illinois":4307,"stomach":4308,"tone":4309,"unique":4310,"reports":4311,"anyway":4312,"##ir":4313,"liberal":4314,"der":4315,"vehicle":4316,"thick":4317,"dry":4318,"drug":4319,"faced":4320,"largely":4321,"facility":4322,"theme":4323,"holds":4324,"creation":4325,"strange":4326,"colonel":4327,"##mi":4328,"revolution":4329,"bell":4330,"politics":4331,"turns":4332,"silent":4333,"rail":4334,"relief":4335,"independence":4336,"combat":4337,"shape":4338,"write":4339,"determined":4340,"sales":4341,"learned":4342,"4th":4343,"finger":4344,"oxford":4345,"providing":4346,"1937":4347,"heritage":4348,"fiction":4349,"situated":4350,"designated":4351,"allowing":4352,"distribution":4353,"hosted":4354,"##est":4355,"sight":4356,"interview":4357,"estimated":4358,"reduced":4359,"##ria":4360,"toronto":4361,"footballer":4362,"keeping":4363,"guys":4364,"damn":4365,"claim":4366,"motion":4367,"sport":4368,"sixth":4369,"stayed":4370,"##ze":4371,"en":4372,"rear":4373,"receive":4374,"handed":4375,"twelve":4376,"dress":4377,"audience":4378,"granted":4379,"brazil":4380,"##well":4381,"spirit":4382,"##ated":4383,"noticed":4384,"etc":4385,"olympic":4386,"representative":4387,"eric":4388,"tight":4389,"trouble":4390,"reviews":4391,"drink":4392,"vampire":4393,"missing":4394,"roles":4395,"ranked":4396,"newly":4397,"household":4398,"finals":4399,"wave":4400,"critics":4401,"##ee":4402,"phase":4403,"massachusetts":4404,"pilot":4405,"unlike":4406,"philadelphia":4407,"bright":4408,"guns":4409,"crown":4410,"organizations":4411,"roof":4412,"42":4413,"respectively":4414,"clearly":4415,"tongue":4416,"marked":4417,"circle":4418,"fox":4419,"korea":4420,"bronze":4421,"brian":4422,"expanded":4423,"sexual":4424,"supply":4425,"yourself":4426,"inspired":4427,"labour":4428,"fc":4429,"##ah":4430,"reference":4431,"vision":4432,"draft":4433,"connection":4434,"brand":4435,"reasons":4436,"1935":4437,"classic":4438,"driving":4439,"trip":4440,"jesus":4441,"cells":4442,"entry":4443,"1920":4444,"neither":4445,"trail":4446,"claims":4447,"atlantic":4448,"orders":4449,"labor":4450,"nose":4451,"afraid":4452,"identified":4453,"intelligence":4454,"calls":4455,"cancer":4456,"attacked":4457,"passing":4458,"stephen":4459,"positions":4460,"imperial":4461,"grey":4462,"jason":4463,"39":4464,"sunday":4465,"48":4466,"swedish":4467,"avoid":4468,"extra":4469,"uncle":4470,"message":4471,"covers":4472,"allows":4473,"surprise":4474,"materials":4475,"fame":4476,"hunter":4477,"##ji":4478,"1930":4479,"citizens":4480,"figures":4481,"davis":4482,"environmental":4483,"confirmed":4484,"shit":4485,"titles":4486,"di":4487,"performing":4488,"difference":4489,"acts":4490,"attacks":4491,"##ov":4492,"existing":4493,"votes":4494,"opportunity":4495,"nor":4496,"shop":4497,"entirely":4498,"trains":4499,"opposite":4500,"pakistan":4501,"##pa":4502,"develop":4503,"resulted":4504,"representatives":4505,"actions":4506,"reality":4507,"pressed":4508,"##ish":4509,"barely":4510,"wine":4511,"conversation":4512,"faculty":4513,"northwest":4514,"ends":4515,"documentary":4516,"nuclear":4517,"stock":4518,"grace":4519,"sets":4520,"eat":4521,"alternative":4522,"##ps":4523,"bag":4524,"resulting":4525,"creating":4526,"surprised":4527,"cemetery":4528,"1919":4529,"drop":4530,"finding":4531,"sarah":4532,"cricket":4533,"streets":4534,"tradition":4535,"ride":4536,"1933":4537,"exhibition":4538,"target":4539,"ear":4540,"explained":4541,"rain":4542,"composer":4543,"injury":4544,"apartment":4545,"municipal":4546,"educational":4547,"occupied":4548,"netherlands":4549,"clean":4550,"billion":4551,"constitution":4552,"learn":4553,"1914":4554,"maximum":4555,"classical":4556,"francis":4557,"lose":4558,"opposition":4559,"jose":4560,"ontario":4561,"bear":4562,"core":4563,"hills":4564,"rolled":4565,"ending":4566,"drawn":4567,"permanent":4568,"fun":4569,"##tes":4570,"##lla":4571,"lewis":4572,"sites":4573,"chamber":4574,"ryan":4575,"##way":4576,"scoring":4577,"height":4578,"1934":4579,"##house":4580,"lyrics":4581,"staring":4582,"55":4583,"officials":4584,"1917":4585,"snow":4586,"oldest":4587,"##tic":4588,"orange":4589,"##ger":4590,"qualified":4591,"interior":4592,"apparently":4593,"succeeded":4594,"thousand":4595,"dinner":4596,"lights":4597,"existence":4598,"fans":4599,"heavily":4600,"41":4601,"greatest":4602,"conservative":4603,"send":4604,"bowl":4605,"plus":4606,"enter":4607,"catch":4608,"##un":4609,"economy":4610,"duty":4611,"1929":4612,"speech":4613,"authorities":4614,"princess":4615,"performances":4616,"versions":4617,"shall":4618,"graduate":4619,"pictures":4620,"effective":4621,"remembered":4622,"poetry":4623,"desk":4624,"crossed":4625,"starring":4626,"starts":4627,"passenger":4628,"sharp":4629,"##ant":4630,"acres":4631,"ass":4632,"weather":4633,"falling":4634,"rank":4635,"fund":4636,"supporting":4637,"check":4638,"adult":4639,"publishing":4640,"heads":4641,"cm":4642,"southeast":4643,"lane":4644,"##burg":4645,"application":4646,"bc":4647,"##ura":4648,"les":4649,"condition":4650,"transfer":4651,"prevent":4652,"display":4653,"ex":4654,"regions":4655,"earl":4656,"federation":4657,"cool":4658,"relatively":4659,"answered":4660,"besides":4661,"1928":4662,"obtained":4663,"portion":4664,"##town":4665,"mix":4666,"##ding":4667,"reaction":4668,"liked":4669,"dean":4670,"express":4671,"peak":4672,"1932":4673,"##tte":4674,"counter":4675,"religion":4676,"chain":4677,"rare":4678,"miller":4679,"convention":4680,"aid":4681,"lie":4682,"vehicles":4683,"mobile":4684,"perform":4685,"squad":4686,"wonder":4687,"lying":4688,"crazy":4689,"sword":4690,"##ping":4691,"attempted":4692,"centuries":4693,"weren":4694,"philosophy":4695,"category":4696,"##ize":4697,"anna":4698,"interested":4699,"47":4700,"sweden":4701,"wolf":4702,"frequently":4703,"abandoned":4704,"kg":4705,"literary":4706,"alliance":4707,"task":4708,"entitled":4709,"##ay":4710,"threw":4711,"promotion":4712,"factory":4713,"tiny":4714,"soccer":4715,"visited":4716,"matt":4717,"fm":4718,"achieved":4719,"52":4720,"defence":4721,"internal":4722,"persian":4723,"43":4724,"methods":4725,"##ging":4726,"arrested":4727,"otherwise":4728,"cambridge":4729,"programming":4730,"villages":4731,"elementary":4732,"districts":4733,"rooms":4734,"criminal":4735,"conflict":4736,"worry":4737,"trained":4738,"1931":4739,"attempts":4740,"waited":4741,"signal":4742,"bird":4743,"truck":4744,"subsequent":4745,"programme":4746,"##ol":4747,"ad":4748,"49":4749,"communist":4750,"details":4751,"faith":4752,"sector":4753,"patrick":4754,"carrying":4755,"laugh":4756,"##ss":4757,"controlled":4758,"korean":4759,"showing":4760,"origin":4761,"fuel":4762,"evil":4763,"1927":4764,"##ent":4765,"brief":4766,"identity":4767,"darkness":4768,"address":4769,"pool":4770,"missed":4771,"publication":4772,"web":4773,"planet":4774,"ian":4775,"anne":4776,"wings":4777,"invited":4778,"##tt":4779,"briefly":4780,"standards":4781,"kissed":4782,"##be":4783,"ideas":4784,"climate":4785,"causing":4786,"walter":4787,"worse":4788,"albert":4789,"articles":4790,"winners":4791,"desire":4792,"aged":4793,"northeast":4794,"dangerous":4795,"gate":4796,"doubt":4797,"1922":4798,"wooden":4799,"multi":4800,"##ky":4801,"poet":4802,"rising":4803,"funding":4804,"46":4805,"communications":4806,"communication":4807,"violence":4808,"copies":4809,"prepared":4810,"ford":4811,"investigation":4812,"skills":4813,"1924":4814,"pulling":4815,"electronic":4816,"##ak":4817,"##ial":4818,"##han":4819,"containing":4820,"ultimately":4821,"offices":4822,"singing":4823,"understanding":4824,"restaurant":4825,"tomorrow":4826,"fashion":4827,"christ":4828,"ward":4829,"da":4830,"pope":4831,"stands":4832,"5th":4833,"flow":4834,"studios":4835,"aired":4836,"commissioned":4837,"contained":4838,"exist":4839,"fresh":4840,"americans":4841,"##per":4842,"wrestling":4843,"approved":4844,"kid":4845,"employed":4846,"respect":4847,"suit":4848,"1925":4849,"angel":4850,"asking":4851,"increasing":4852,"frame":4853,"angry":4854,"selling":4855,"1950s":4856,"thin":4857,"finds":4858,"##nd":4859,"temperature":4860,"statement":4861,"ali":4862,"explain":4863,"inhabitants":4864,"towns":4865,"extensive":4866,"narrow":4867,"51":4868,"jane":4869,"flowers":4870,"images":4871,"promise":4872,"somewhere":4873,"object":4874,"fly":4875,"closely":4876,"##ls":4877,"1912":4878,"bureau":4879,"cape":4880,"1926":4881,"weekly":4882,"presidential":4883,"legislative":4884,"1921":4885,"##ai":4886,"##au":4887,"launch":4888,"founding":4889,"##ny":4890,"978":4891,"##ring":4892,"artillery":4893,"strike":4894,"un":4895,"institutions":4896,"roll":4897,"writers":4898,"landing":4899,"chose":4900,"kevin":4901,"anymore":4902,"pp":4903,"##ut":4904,"attorney":4905,"fit":4906,"dan":4907,"billboard":4908,"receiving":4909,"agricultural":4910,"breaking":4911,"sought":4912,"dave":4913,"admitted":4914,"lands":4915,"mexican":4916,"##bury":4917,"charlie":4918,"specifically":4919,"hole":4920,"iv":4921,"howard":4922,"credit":4923,"moscow":4924,"roads":4925,"accident":4926,"1923":4927,"proved":4928,"wear":4929,"struck":4930,"hey":4931,"guards":4932,"stuff":4933,"slid":4934,"expansion":4935,"1915":4936,"cat":4937,"anthony":4938,"##kin":4939,"melbourne":4940,"opposed":4941,"sub":4942,"southwest":4943,"architect":4944,"failure":4945,"plane":4946,"1916":4947,"##ron":4948,"map":4949,"camera":4950,"tank":4951,"listen":4952,"regarding":4953,"wet":4954,"introduction":4955,"metropolitan":4956,"link":4957,"ep":4958,"fighter":4959,"inch":4960,"grown":4961,"gene":4962,"anger":4963,"fixed":4964,"buy":4965,"dvd":4966,"khan":4967,"domestic":4968,"worldwide":4969,"chapel":4970,"mill":4971,"functions":4972,"examples":4973,"##head":4974,"developing":4975,"1910":4976,"turkey":4977,"hits":4978,"pocket":4979,"antonio":4980,"papers":4981,"grow":4982,"unless":4983,"circuit":4984,"18th":4985,"concerned":4986,"attached":4987,"journalist":4988,"selection":4989,"journey":4990,"converted":4991,"provincial":4992,"painted":4993,"hearing":4994,"aren":4995,"bands":4996,"negative":4997,"aside":4998,"wondered":4999,"knight":5000,"lap":5001,"survey":5002,"ma":5003,"##ow":5004,"noise":5005,"billy":5006,"##ium":5007,"shooting":5008,"guide":5009,"bedroom":5010,"priest":5011,"resistance":5012,"motor":5013,"homes":5014,"sounded":5015,"giant":5016,"##mer":5017,"150":5018,"scenes":5019,"equal":5020,"comic":5021,"patients":5022,"hidden":5023,"solid":5024,"actual":5025,"bringing":5026,"afternoon":5027,"touched":5028,"funds":5029,"wedding":5030,"consisted":5031,"marie":5032,"canal":5033,"sr":5034,"kim":5035,"treaty":5036,"turkish":5037,"recognition":5038,"residence":5039,"cathedral":5040,"broad":5041,"knees":5042,"incident":5043,"shaped":5044,"fired":5045,"norwegian":5046,"handle":5047,"cheek":5048,"contest":5049,"represent":5050,"##pe":5051,"representing":5052,"beauty":5053,"##sen":5054,"birds":5055,"advantage":5056,"emergency":5057,"wrapped":5058,"drawing":5059,"notice":5060,"pink":5061,"broadcasting":5062,"##ong":5063,"somehow":5064,"bachelor":5065,"seventh":5066,"collected":5067,"registered":5068,"establishment":5069,"alan":5070,"assumed":5071,"chemical":5072,"personnel":5073,"roger":5074,"retirement":5075,"jeff":5076,"portuguese":5077,"wore":5078,"tied":5079,"device":5080,"threat":5081,"progress":5082,"advance":5083,"##ised":5084,"banks":5085,"hired":5086,"manchester":5087,"nfl":5088,"teachers":5089,"structures":5090,"forever":5091,"##bo":5092,"tennis":5093,"helping":5094,"saturday":5095,"sale":5096,"applications":5097,"junction":5098,"hip":5099,"incorporated":5100,"neighborhood":5101,"dressed":5102,"ceremony":5103,"##ds":5104,"influenced":5105,"hers":5106,"visual":5107,"stairs":5108,"decades":5109,"inner":5110,"kansas":5111,"hung":5112,"hoped":5113,"gain":5114,"scheduled":5115,"downtown":5116,"engaged":5117,"austria":5118,"clock":5119,"norway":5120,"certainly":5121,"pale":5122,"protected":5123,"1913":5124,"victor":5125,"employees":5126,"plate":5127,"putting":5128,"surrounded":5129,"##ists":5130,"finishing":5131,"blues":5132,"tropical":5133,"##ries":5134,"minnesota":5135,"consider":5136,"philippines":5137,"accept":5138,"54":5139,"retrieved":5140,"1900":5141,"concern":5142,"anderson":5143,"properties":5144,"institution":5145,"gordon":5146,"successfully":5147,"vietnam":5148,"##dy":5149,"backing":5150,"outstanding":5151,"muslim":5152,"crossing":5153,"folk":5154,"producing":5155,"usual":5156,"demand":5157,"occurs":5158,"observed":5159,"lawyer":5160,"educated":5161,"##ana":5162,"kelly":5163,"string":5164,"pleasure":5165,"budget":5166,"items":5167,"quietly":5168,"colorado":5169,"philip":5170,"typical":5171,"##worth":5172,"derived":5173,"600":5174,"survived":5175,"asks":5176,"mental":5177,"##ide":5178,"56":5179,"jake":5180,"jews":5181,"distinguished":5182,"ltd":5183,"1911":5184,"sri":5185,"extremely":5186,"53":5187,"athletic":5188,"loud":5189,"thousands":5190,"worried":5191,"shadow":5192,"transportation":5193,"horses":5194,"weapon":5195,"arena":5196,"importance":5197,"users":5198,"tim":5199,"objects":5200,"contributed":5201,"dragon":5202,"douglas":5203,"aware":5204,"senator":5205,"johnny":5206,"jordan":5207,"sisters":5208,"engines":5209,"flag":5210,"investment":5211,"samuel":5212,"shock":5213,"capable":5214,"clark":5215,"row":5216,"wheel":5217,"refers":5218,"session":5219,"familiar":5220,"biggest":5221,"wins":5222,"hate":5223,"maintained":5224,"drove":5225,"hamilton":5226,"request":5227,"expressed":5228,"injured":5229,"underground":5230,"churches":5231,"walker":5232,"wars":5233,"tunnel":5234,"passes":5235,"stupid":5236,"agriculture":5237,"softly":5238,"cabinet":5239,"regarded":5240,"joining":5241,"indiana":5242,"##ea":5243,"##ms":5244,"push":5245,"dates":5246,"spend":5247,"behavior":5248,"woods":5249,"protein":5250,"gently":5251,"chase":5252,"morgan":5253,"mention":5254,"burning":5255,"wake":5256,"combination":5257,"occur":5258,"mirror":5259,"leads":5260,"jimmy":5261,"indeed":5262,"impossible":5263,"singapore":5264,"paintings":5265,"covering":5266,"##nes":5267,"soldier":5268,"locations":5269,"attendance":5270,"sell":5271,"historian":5272,"wisconsin":5273,"invasion":5274,"argued":5275,"painter":5276,"diego":5277,"changing":5278,"egypt":5279,"##don":5280,"experienced":5281,"inches":5282,"##ku":5283,"missouri":5284,"vol":5285,"grounds":5286,"spoken":5287,"switzerland":5288,"##gan":5289,"reform":5290,"rolling":5291,"ha":5292,"forget":5293,"massive":5294,"resigned":5295,"burned":5296,"allen":5297,"tennessee":5298,"locked":5299,"values":5300,"improved":5301,"##mo":5302,"wounded":5303,"universe":5304,"sick":5305,"dating":5306,"facing":5307,"pack":5308,"purchase":5309,"user":5310,"##pur":5311,"moments":5312,"##ul":5313,"merged":5314,"anniversary":5315,"1908":5316,"coal":5317,"brick":5318,"understood":5319,"causes":5320,"dynasty":5321,"queensland":5322,"establish":5323,"stores":5324,"crisis":5325,"promote":5326,"hoping":5327,"views":5328,"cards":5329,"referee":5330,"extension":5331,"##si":5332,"raise":5333,"arizona":5334,"improve":5335,"colonial":5336,"formal":5337,"charged":5338,"##rt":5339,"palm":5340,"lucky":5341,"hide":5342,"rescue":5343,"faces":5344,"95":5345,"feelings":5346,"candidates":5347,"juan":5348,"##ell":5349,"goods":5350,"6th":5351,"courses":5352,"weekend":5353,"59":5354,"luke":5355,"cash":5356,"fallen":5357,"##om":5358,"delivered":5359,"affected":5360,"installed":5361,"carefully":5362,"tries":5363,"swiss":5364,"hollywood":5365,"costs":5366,"lincoln":5367,"responsibility":5368,"##he":5369,"shore":5370,"file":5371,"proper":5372,"normally":5373,"maryland":5374,"assistance":5375,"jump":5376,"constant":5377,"offering":5378,"friendly":5379,"waters":5380,"persons":5381,"realize":5382,"contain":5383,"trophy":5384,"800":5385,"partnership":5386,"factor":5387,"58":5388,"musicians":5389,"cry":5390,"bound":5391,"oregon":5392,"indicated":5393,"hero":5394,"houston":5395,"medium":5396,"##ure":5397,"consisting":5398,"somewhat":5399,"##ara":5400,"57":5401,"cycle":5402,"##che":5403,"beer":5404,"moore":5405,"frederick":5406,"gotten":5407,"eleven":5408,"worst":5409,"weak":5410,"approached":5411,"arranged":5412,"chin":5413,"loan":5414,"universal":5415,"bond":5416,"fifteen":5417,"pattern":5418,"disappeared":5419,"##ney":5420,"translated":5421,"##zed":5422,"lip":5423,"arab":5424,"capture":5425,"interests":5426,"insurance":5427,"##chi":5428,"shifted":5429,"cave":5430,"prix":5431,"warning":5432,"sections":5433,"courts":5434,"coat":5435,"plot":5436,"smell":5437,"feed":5438,"golf":5439,"favorite":5440,"maintain":5441,"knife":5442,"vs":5443,"voted":5444,"degrees":5445,"finance":5446,"quebec":5447,"opinion":5448,"translation":5449,"manner":5450,"ruled":5451,"operate":5452,"productions":5453,"choose":5454,"musician":5455,"discovery":5456,"confused":5457,"tired":5458,"separated":5459,"stream":5460,"techniques":5461,"committed":5462,"attend":5463,"ranking":5464,"kings":5465,"throw":5466,"passengers":5467,"measure":5468,"horror":5469,"fan":5470,"mining":5471,"sand":5472,"danger":5473,"salt":5474,"calm":5475,"decade":5476,"dam":5477,"require":5478,"runner":5479,"##ik":5480,"rush":5481,"associate":5482,"greece":5483,"##ker":5484,"rivers":5485,"consecutive":5486,"matthew":5487,"##ski":5488,"sighed":5489,"sq":5490,"documents":5491,"steam":5492,"edited":5493,"closing":5494,"tie":5495,"accused":5496,"1905":5497,"##ini":5498,"islamic":5499,"distributed":5500,"directors":5501,"organisation":5502,"bruce":5503,"7th":5504,"breathing":5505,"mad":5506,"lit":5507,"arrival":5508,"concrete":5509,"taste":5510,"08":5511,"composition":5512,"shaking":5513,"faster":5514,"amateur":5515,"adjacent":5516,"stating":5517,"1906":5518,"twin":5519,"flew":5520,"##ran":5521,"tokyo":5522,"publications":5523,"##tone":5524,"obviously":5525,"ridge":5526,"storage":5527,"1907":5528,"carl":5529,"pages":5530,"concluded":5531,"desert":5532,"driven":5533,"universities":5534,"ages":5535,"terminal":5536,"sequence":5537,"borough":5538,"250":5539,"constituency":5540,"creative":5541,"cousin":5542,"economics":5543,"dreams":5544,"margaret":5545,"notably":5546,"reduce":5547,"montreal":5548,"mode":5549,"17th":5550,"ears":5551,"saved":5552,"jan":5553,"vocal":5554,"##ica":5555,"1909":5556,"andy":5557,"##jo":5558,"riding":5559,"roughly":5560,"threatened":5561,"##ise":5562,"meters":5563,"meanwhile":5564,"landed":5565,"compete":5566,"repeated":5567,"grass":5568,"czech":5569,"regularly":5570,"charges":5571,"tea":5572,"sudden":5573,"appeal":5574,"##ung":5575,"solution":5576,"describes":5577,"pierre":5578,"classification":5579,"glad":5580,"parking":5581,"##ning":5582,"belt":5583,"physics":5584,"99":5585,"rachel":5586,"add":5587,"hungarian":5588,"participate":5589,"expedition":5590,"damaged":5591,"gift":5592,"childhood":5593,"85":5594,"fifty":5595,"##red":5596,"mathematics":5597,"jumped":5598,"letting":5599,"defensive":5600,"mph":5601,"##ux":5602,"##gh":5603,"testing":5604,"##hip":5605,"hundreds":5606,"shoot":5607,"owners":5608,"matters":5609,"smoke":5610,"israeli":5611,"kentucky":5612,"dancing":5613,"mounted":5614,"grandfather":5615,"emma":5616,"designs":5617,"profit":5618,"argentina":5619,"##gs":5620,"truly":5621,"li":5622,"lawrence":5623,"cole":5624,"begun":5625,"detroit":5626,"willing":5627,"branches":5628,"smiling":5629,"decide":5630,"miami":5631,"enjoyed":5632,"recordings":5633,"##dale":5634,"poverty":5635,"ethnic":5636,"gay":5637,"##bi":5638,"gary":5639,"arabic":5640,"09":5641,"accompanied":5642,"##one":5643,"##ons":5644,"fishing":5645,"determine":5646,"residential":5647,"acid":5648,"##ary":5649,"alice":5650,"returns":5651,"starred":5652,"mail":5653,"##ang":5654,"jonathan":5655,"strategy":5656,"##ue":5657,"net":5658,"forty":5659,"cook":5660,"businesses":5661,"equivalent":5662,"commonwealth":5663,"distinct":5664,"ill":5665,"##cy":5666,"seriously":5667,"##ors":5668,"##ped":5669,"shift":5670,"harris":5671,"replace":5672,"rio":5673,"imagine":5674,"formula":5675,"ensure":5676,"##ber":5677,"additionally":5678,"scheme":5679,"conservation":5680,"occasionally":5681,"purposes":5682,"feels":5683,"favor":5684,"##and":5685,"##ore":5686,"1930s":5687,"contrast":5688,"hanging":5689,"hunt":5690,"movies":5691,"1904":5692,"instruments":5693,"victims":5694,"danish":5695,"christopher":5696,"busy":5697,"demon":5698,"sugar":5699,"earliest":5700,"colony":5701,"studying":5702,"balance":5703,"duties":5704,"##ks":5705,"belgium":5706,"slipped":5707,"carter":5708,"05":5709,"visible":5710,"stages":5711,"iraq":5712,"fifa":5713,"##im":5714,"commune":5715,"forming":5716,"zero":5717,"07":5718,"continuing":5719,"talked":5720,"counties":5721,"legend":5722,"bathroom":5723,"option":5724,"tail":5725,"clay":5726,"daughters":5727,"afterwards":5728,"severe":5729,"jaw":5730,"visitors":5731,"##ded":5732,"devices":5733,"aviation":5734,"russell":5735,"kate":5736,"##vi":5737,"entering":5738,"subjects":5739,"##ino":5740,"temporary":5741,"swimming":5742,"forth":5743,"smooth":5744,"ghost":5745,"audio":5746,"bush":5747,"operates":5748,"rocks":5749,"movements":5750,"signs":5751,"eddie":5752,"##tz":5753,"ann":5754,"voices":5755,"honorary":5756,"06":5757,"memories":5758,"dallas":5759,"pure":5760,"measures":5761,"racial":5762,"promised":5763,"66":5764,"harvard":5765,"ceo":5766,"16th":5767,"parliamentary":5768,"indicate":5769,"benefit":5770,"flesh":5771,"dublin":5772,"louisiana":5773,"1902":5774,"1901":5775,"patient":5776,"sleeping":5777,"1903":5778,"membership":5779,"coastal":5780,"medieval":5781,"wanting":5782,"element":5783,"scholars":5784,"rice":5785,"62":5786,"limit":5787,"survive":5788,"makeup":5789,"rating":5790,"definitely":5791,"collaboration":5792,"obvious":5793,"##tan":5794,"boss":5795,"ms":5796,"baron":5797,"birthday":5798,"linked":5799,"soil":5800,"diocese":5801,"##lan":5802,"ncaa":5803,"##mann":5804,"offensive":5805,"shell":5806,"shouldn":5807,"waist":5808,"##tus":5809,"plain":5810,"ross":5811,"organ":5812,"resolution":5813,"manufacturing":5814,"adding":5815,"relative":5816,"kennedy":5817,"98":5818,"whilst":5819,"moth":5820,"marketing":5821,"gardens":5822,"crash":5823,"72":5824,"heading":5825,"partners":5826,"credited":5827,"carlos":5828,"moves":5829,"cable":5830,"##zi":5831,"marshall":5832,"##out":5833,"depending":5834,"bottle":5835,"represents":5836,"rejected":5837,"responded":5838,"existed":5839,"04":5840,"jobs":5841,"denmark":5842,"lock":5843,"##ating":5844,"treated":5845,"graham":5846,"routes":5847,"talent":5848,"commissioner":5849,"drugs":5850,"secure":5851,"tests":5852,"reign":5853,"restored":5854,"photography":5855,"##gi":5856,"contributions":5857,"oklahoma":5858,"designer":5859,"disc":5860,"grin":5861,"seattle":5862,"robin":5863,"paused":5864,"atlanta":5865,"unusual":5866,"##gate":5867,"praised":5868,"las":5869,"laughing":5870,"satellite":5871,"hungary":5872,"visiting":5873,"##sky":5874,"interesting":5875,"factors":5876,"deck":5877,"poems":5878,"norman":5879,"##water":5880,"stuck":5881,"speaker":5882,"rifle":5883,"domain":5884,"premiered":5885,"##her":5886,"dc":5887,"comics":5888,"actors":5889,"01":5890,"reputation":5891,"eliminated":5892,"8th":5893,"ceiling":5894,"prisoners":5895,"script":5896,"##nce":5897,"leather":5898,"austin":5899,"mississippi":5900,"rapidly":5901,"admiral":5902,"parallel":5903,"charlotte":5904,"guilty":5905,"tools":5906,"gender":5907,"divisions":5908,"fruit":5909,"##bs":5910,"laboratory":5911,"nelson":5912,"fantasy":5913,"marry":5914,"rapid":5915,"aunt":5916,"tribe":5917,"requirements":5918,"aspects":5919,"suicide":5920,"amongst":5921,"adams":5922,"bone":5923,"ukraine":5924,"abc":5925,"kick":5926,"sees":5927,"edinburgh":5928,"clothing":5929,"column":5930,"rough":5931,"gods":5932,"hunting":5933,"broadway":5934,"gathered":5935,"concerns":5936,"##ek":5937,"spending":5938,"ty":5939,"12th":5940,"snapped":5941,"requires":5942,"solar":5943,"bones":5944,"cavalry":5945,"##tta":5946,"iowa":5947,"drinking":5948,"waste":5949,"index":5950,"franklin":5951,"charity":5952,"thompson":5953,"stewart":5954,"tip":5955,"flash":5956,"landscape":5957,"friday":5958,"enjoy":5959,"singh":5960,"poem":5961,"listening":5962,"##back":5963,"eighth":5964,"fred":5965,"differences":5966,"adapted":5967,"bomb":5968,"ukrainian":5969,"surgery":5970,"corporate":5971,"masters":5972,"anywhere":5973,"##more":5974,"waves":5975,"odd":5976,"sean":5977,"portugal":5978,"orleans":5979,"dick":5980,"debate":5981,"kent":5982,"eating":5983,"puerto":5984,"cleared":5985,"96":5986,"expect":5987,"cinema":5988,"97":5989,"guitarist":5990,"blocks":5991,"electrical":5992,"agree":5993,"involving":5994,"depth":5995,"dying":5996,"panel":5997,"struggle":5998,"##ged":5999,"peninsula":6000,"adults":6001,"novels":6002,"emerged":6003,"vienna":6004,"metro":6005,"debuted":6006,"shoes":6007,"tamil":6008,"songwriter":6009,"meets":6010,"prove":6011,"beating":6012,"instance":6013,"heaven":6014,"scared":6015,"sending":6016,"marks":6017,"artistic":6018,"passage":6019,"superior":6020,"03":6021,"significantly":6022,"shopping":6023,"##tive":6024,"retained":6025,"##izing":6026,"malaysia":6027,"technique":6028,"cheeks":6029,"##ola":6030,"warren":6031,"maintenance":6032,"destroy":6033,"extreme":6034,"allied":6035,"120":6036,"appearing":6037,"##yn":6038,"fill":6039,"advice":6040,"alabama":6041,"qualifying":6042,"policies":6043,"cleveland":6044,"hat":6045,"battery":6046,"smart":6047,"authors":6048,"10th":6049,"soundtrack":6050,"acted":6051,"dated":6052,"lb":6053,"glance":6054,"equipped":6055,"coalition":6056,"funny":6057,"outer":6058,"ambassador":6059,"roy":6060,"possibility":6061,"couples":6062,"campbell":6063,"dna":6064,"loose":6065,"ethan":6066,"supplies":6067,"1898":6068,"gonna":6069,"88":6070,"monster":6071,"##res":6072,"shake":6073,"agents":6074,"frequency":6075,"springs":6076,"dogs":6077,"practices":6078,"61":6079,"gang":6080,"plastic":6081,"easier":6082,"suggests":6083,"gulf":6084,"blade":6085,"exposed":6086,"colors":6087,"industries":6088,"markets":6089,"pan":6090,"nervous":6091,"electoral":6092,"charts":6093,"legislation":6094,"ownership":6095,"##idae":6096,"mac":6097,"appointment":6098,"shield":6099,"copy":6100,"assault":6101,"socialist":6102,"abbey":6103,"monument":6104,"license":6105,"throne":6106,"employment":6107,"jay":6108,"93":6109,"replacement":6110,"charter":6111,"cloud":6112,"powered":6113,"suffering":6114,"accounts":6115,"oak":6116,"connecticut":6117,"strongly":6118,"wright":6119,"colour":6120,"crystal":6121,"13th":6122,"context":6123,"welsh":6124,"networks":6125,"voiced":6126,"gabriel":6127,"jerry":6128,"##cing":6129,"forehead":6130,"mp":6131,"##ens":6132,"manage":6133,"schedule":6134,"totally":6135,"remix":6136,"##ii":6137,"forests":6138,"occupation":6139,"print":6140,"nicholas":6141,"brazilian":6142,"strategic":6143,"vampires":6144,"engineers":6145,"76":6146,"roots":6147,"seek":6148,"correct":6149,"instrumental":6150,"und":6151,"alfred":6152,"backed":6153,"hop":6154,"##des":6155,"stanley":6156,"robinson":6157,"traveled":6158,"wayne":6159,"welcome":6160,"austrian":6161,"achieve":6162,"67":6163,"exit":6164,"rates":6165,"1899":6166,"strip":6167,"whereas":6168,"##cs":6169,"sing":6170,"deeply":6171,"adventure":6172,"bobby":6173,"rick":6174,"jamie":6175,"careful":6176,"components":6177,"cap":6178,"useful":6179,"personality":6180,"knee":6181,"##shi":6182,"pushing":6183,"hosts":6184,"02":6185,"protest":6186,"ca":6187,"ottoman":6188,"symphony":6189,"##sis":6190,"63":6191,"boundary":6192,"1890":6193,"processes":6194,"considering":6195,"considerable":6196,"tons":6197,"##work":6198,"##ft":6199,"##nia":6200,"cooper":6201,"trading":6202,"dear":6203,"conduct":6204,"91":6205,"illegal":6206,"apple":6207,"revolutionary":6208,"holiday":6209,"definition":6210,"harder":6211,"##van":6212,"jacob":6213,"circumstances":6214,"destruction":6215,"##lle":6216,"popularity":6217,"grip":6218,"classified":6219,"liverpool":6220,"donald":6221,"baltimore":6222,"flows":6223,"seeking":6224,"honour":6225,"approval":6226,"92":6227,"mechanical":6228,"till":6229,"happening":6230,"statue":6231,"critic":6232,"increasingly":6233,"immediate":6234,"describe":6235,"commerce":6236,"stare":6237,"##ster":6238,"indonesia":6239,"meat":6240,"rounds":6241,"boats":6242,"baker":6243,"orthodox":6244,"depression":6245,"formally":6246,"worn":6247,"naked":6248,"claire":6249,"muttered":6250,"sentence":6251,"11th":6252,"emily":6253,"document":6254,"77":6255,"criticism":6256,"wished":6257,"vessel":6258,"spiritual":6259,"bent":6260,"virgin":6261,"parker":6262,"minimum":6263,"murray":6264,"lunch":6265,"danny":6266,"printed":6267,"compilation":6268,"keyboards":6269,"false":6270,"blow":6271,"belonged":6272,"68":6273,"raising":6274,"78":6275,"cutting":6276,"##board":6277,"pittsburgh":6278,"##up":6279,"9th":6280,"shadows":6281,"81":6282,"hated":6283,"indigenous":6284,"jon":6285,"15th":6286,"barry":6287,"scholar":6288,"ah":6289,"##zer":6290,"oliver":6291,"##gy":6292,"stick":6293,"susan":6294,"meetings":6295,"attracted":6296,"spell":6297,"romantic":6298,"##ver":6299,"ye":6300,"1895":6301,"photo":6302,"demanded":6303,"customers":6304,"##ac":6305,"1896":6306,"logan":6307,"revival":6308,"keys":6309,"modified":6310,"commanded":6311,"jeans":6312,"##ious":6313,"upset":6314,"raw":6315,"phil":6316,"detective":6317,"hiding":6318,"resident":6319,"vincent":6320,"##bly":6321,"experiences":6322,"diamond":6323,"defeating":6324,"coverage":6325,"lucas":6326,"external":6327,"parks":6328,"franchise":6329,"helen":6330,"bible":6331,"successor":6332,"percussion":6333,"celebrated":6334,"il":6335,"lift":6336,"profile":6337,"clan":6338,"romania":6339,"##ied":6340,"mills":6341,"##su":6342,"nobody":6343,"achievement":6344,"shrugged":6345,"fault":6346,"1897":6347,"rhythm":6348,"initiative":6349,"breakfast":6350,"carbon":6351,"700":6352,"69":6353,"lasted":6354,"violent":6355,"74":6356,"wound":6357,"ken":6358,"killer":6359,"gradually":6360,"filmed":6361,"°c":6362,"dollars":6363,"processing":6364,"94":6365,"remove":6366,"criticized":6367,"guests":6368,"sang":6369,"chemistry":6370,"##vin":6371,"legislature":6372,"disney":6373,"##bridge":6374,"uniform":6375,"escaped":6376,"integrated":6377,"proposal":6378,"purple":6379,"denied":6380,"liquid":6381,"karl":6382,"influential":6383,"morris":6384,"nights":6385,"stones":6386,"intense":6387,"experimental":6388,"twisted":6389,"71":6390,"84":6391,"##ld":6392,"pace":6393,"nazi":6394,"mitchell":6395,"ny":6396,"blind":6397,"reporter":6398,"newspapers":6399,"14th":6400,"centers":6401,"burn":6402,"basin":6403,"forgotten":6404,"surviving":6405,"filed":6406,"collections":6407,"monastery":6408,"losses":6409,"manual":6410,"couch":6411,"description":6412,"appropriate":6413,"merely":6414,"tag":6415,"missions":6416,"sebastian":6417,"restoration":6418,"replacing":6419,"triple":6420,"73":6421,"elder":6422,"julia":6423,"warriors":6424,"benjamin":6425,"julian":6426,"convinced":6427,"stronger":6428,"amazing":6429,"declined":6430,"versus":6431,"merchant":6432,"happens":6433,"output":6434,"finland":6435,"bare":6436,"barbara":6437,"absence":6438,"ignored":6439,"dawn":6440,"injuries":6441,"##port":6442,"producers":6443,"##ram":6444,"82":6445,"luis":6446,"##ities":6447,"kw":6448,"admit":6449,"expensive":6450,"electricity":6451,"nba":6452,"exception":6453,"symbol":6454,"##ving":6455,"ladies":6456,"shower":6457,"sheriff":6458,"characteristics":6459,"##je":6460,"aimed":6461,"button":6462,"ratio":6463,"effectively":6464,"summit":6465,"angle":6466,"jury":6467,"bears":6468,"foster":6469,"vessels":6470,"pants":6471,"executed":6472,"evans":6473,"dozen":6474,"advertising":6475,"kicked":6476,"patrol":6477,"1889":6478,"competitions":6479,"lifetime":6480,"principles":6481,"athletics":6482,"##logy":6483,"birmingham":6484,"sponsored":6485,"89":6486,"rob":6487,"nomination":6488,"1893":6489,"acoustic":6490,"##sm":6491,"creature":6492,"longest":6493,"##tra":6494,"credits":6495,"harbor":6496,"dust":6497,"josh":6498,"##so":6499,"territories":6500,"milk":6501,"infrastructure":6502,"completion":6503,"thailand":6504,"indians":6505,"leon":6506,"archbishop":6507,"##sy":6508,"assist":6509,"pitch":6510,"blake":6511,"arrangement":6512,"girlfriend":6513,"serbian":6514,"operational":6515,"hence":6516,"sad":6517,"scent":6518,"fur":6519,"dj":6520,"sessions":6521,"hp":6522,"refer":6523,"rarely":6524,"##ora":6525,"exists":6526,"1892":6527,"##ten":6528,"scientists":6529,"dirty":6530,"penalty":6531,"burst":6532,"portrait":6533,"seed":6534,"79":6535,"pole":6536,"limits":6537,"rival":6538,"1894":6539,"stable":6540,"alpha":6541,"grave":6542,"constitutional":6543,"alcohol":6544,"arrest":6545,"flower":6546,"mystery":6547,"devil":6548,"architectural":6549,"relationships":6550,"greatly":6551,"habitat":6552,"##istic":6553,"larry":6554,"progressive":6555,"remote":6556,"cotton":6557,"##ics":6558,"##ok":6559,"preserved":6560,"reaches":6561,"##ming":6562,"cited":6563,"86":6564,"vast":6565,"scholarship":6566,"decisions":6567,"cbs":6568,"joy":6569,"teach":6570,"1885":6571,"editions":6572,"knocked":6573,"eve":6574,"searching":6575,"partly":6576,"participation":6577,"gap":6578,"animated":6579,"fate":6580,"excellent":6581,"##ett":6582,"na":6583,"87":6584,"alternate":6585,"saints":6586,"youngest":6587,"##ily":6588,"climbed":6589,"##ita":6590,"##tors":6591,"suggest":6592,"##ct":6593,"discussion":6594,"staying":6595,"choir":6596,"lakes":6597,"jacket":6598,"revenue":6599,"nevertheless":6600,"peaked":6601,"instrument":6602,"wondering":6603,"annually":6604,"managing":6605,"neil":6606,"1891":6607,"signing":6608,"terry":6609,"##ice":6610,"apply":6611,"clinical":6612,"brooklyn":6613,"aim":6614,"catherine":6615,"fuck":6616,"farmers":6617,"figured":6618,"ninth":6619,"pride":6620,"hugh":6621,"evolution":6622,"ordinary":6623,"involvement":6624,"comfortable":6625,"shouted":6626,"tech":6627,"encouraged":6628,"taiwan":6629,"representation":6630,"sharing":6631,"##lia":6632,"##em":6633,"panic":6634,"exact":6635,"cargo":6636,"competing":6637,"fat":6638,"cried":6639,"83":6640,"1920s":6641,"occasions":6642,"pa":6643,"cabin":6644,"borders":6645,"utah":6646,"marcus":6647,"##isation":6648,"badly":6649,"muscles":6650,"##ance":6651,"victorian":6652,"transition":6653,"warner":6654,"bet":6655,"permission":6656,"##rin":6657,"slave":6658,"terrible":6659,"similarly":6660,"shares":6661,"seth":6662,"uefa":6663,"possession":6664,"medals":6665,"benefits":6666,"colleges":6667,"lowered":6668,"perfectly":6669,"mall":6670,"transit":6671,"##ye":6672,"##kar":6673,"publisher":6674,"##ened":6675,"harrison":6676,"deaths":6677,"elevation":6678,"##ae":6679,"asleep":6680,"machines":6681,"sigh":6682,"ash":6683,"hardly":6684,"argument":6685,"occasion":6686,"parent":6687,"leo":6688,"decline":6689,"1888":6690,"contribution":6691,"##ua":6692,"concentration":6693,"1000":6694,"opportunities":6695,"hispanic":6696,"guardian":6697,"extent":6698,"emotions":6699,"hips":6700,"mason":6701,"volumes":6702,"bloody":6703,"controversy":6704,"diameter":6705,"steady":6706,"mistake":6707,"phoenix":6708,"identify":6709,"violin":6710,"##sk":6711,"departure":6712,"richmond":6713,"spin":6714,"funeral":6715,"enemies":6716,"1864":6717,"gear":6718,"literally":6719,"connor":6720,"random":6721,"sergeant":6722,"grab":6723,"confusion":6724,"1865":6725,"transmission":6726,"informed":6727,"op":6728,"leaning":6729,"sacred":6730,"suspended":6731,"thinks":6732,"gates":6733,"portland":6734,"luck":6735,"agencies":6736,"yours":6737,"hull":6738,"expert":6739,"muscle":6740,"layer":6741,"practical":6742,"sculpture":6743,"jerusalem":6744,"latest":6745,"lloyd":6746,"statistics":6747,"deeper":6748,"recommended":6749,"warrior":6750,"arkansas":6751,"mess":6752,"supports":6753,"greg":6754,"eagle":6755,"1880":6756,"recovered":6757,"rated":6758,"concerts":6759,"rushed":6760,"##ano":6761,"stops":6762,"eggs":6763,"files":6764,"premiere":6765,"keith":6766,"##vo":6767,"delhi":6768,"turner":6769,"pit":6770,"affair":6771,"belief":6772,"paint":6773,"##zing":6774,"mate":6775,"##ach":6776,"##ev":6777,"victim":6778,"##ology":6779,"withdrew":6780,"bonus":6781,"styles":6782,"fled":6783,"##ud":6784,"glasgow":6785,"technologies":6786,"funded":6787,"nbc":6788,"adaptation":6789,"##ata":6790,"portrayed":6791,"cooperation":6792,"supporters":6793,"judges":6794,"bernard":6795,"justin":6796,"hallway":6797,"ralph":6798,"##ick":6799,"graduating":6800,"controversial":6801,"distant":6802,"continental":6803,"spider":6804,"bite":6805,"##ho":6806,"recognize":6807,"intention":6808,"mixing":6809,"##ese":6810,"egyptian":6811,"bow":6812,"tourism":6813,"suppose":6814,"claiming":6815,"tiger":6816,"dominated":6817,"participants":6818,"vi":6819,"##ru":6820,"nurse":6821,"partially":6822,"tape":6823,"##rum":6824,"psychology":6825,"##rn":6826,"essential":6827,"touring":6828,"duo":6829,"voting":6830,"civilian":6831,"emotional":6832,"channels":6833,"##king":6834,"apparent":6835,"hebrew":6836,"1887":6837,"tommy":6838,"carrier":6839,"intersection":6840,"beast":6841,"hudson":6842,"##gar":6843,"##zo":6844,"lab":6845,"nova":6846,"bench":6847,"discuss":6848,"costa":6849,"##ered":6850,"detailed":6851,"behalf":6852,"drivers":6853,"unfortunately":6854,"obtain":6855,"##lis":6856,"rocky":6857,"##dae":6858,"siege":6859,"friendship":6860,"honey":6861,"##rian":6862,"1861":6863,"amy":6864,"hang":6865,"posted":6866,"governments":6867,"collins":6868,"respond":6869,"wildlife":6870,"preferred":6871,"operator":6872,"##po":6873,"laura":6874,"pregnant":6875,"videos":6876,"dennis":6877,"suspected":6878,"boots":6879,"instantly":6880,"weird":6881,"automatic":6882,"businessman":6883,"alleged":6884,"placing":6885,"throwing":6886,"ph":6887,"mood":6888,"1862":6889,"perry":6890,"venue":6891,"jet":6892,"remainder":6893,"##lli":6894,"##ci":6895,"passion":6896,"biological":6897,"boyfriend":6898,"1863":6899,"dirt":6900,"buffalo":6901,"ron":6902,"segment":6903,"fa":6904,"abuse":6905,"##era":6906,"genre":6907,"thrown":6908,"stroke":6909,"colored":6910,"stress":6911,"exercise":6912,"displayed":6913,"##gen":6914,"struggled":6915,"##tti":6916,"abroad":6917,"dramatic":6918,"wonderful":6919,"thereafter":6920,"madrid":6921,"component":6922,"widespread":6923,"##sed":6924,"tale":6925,"citizen":6926,"todd":6927,"monday":6928,"1886":6929,"vancouver":6930,"overseas":6931,"forcing":6932,"crying":6933,"descent":6934,"##ris":6935,"discussed":6936,"substantial":6937,"ranks":6938,"regime":6939,"1870":6940,"provinces":6941,"switch":6942,"drum":6943,"zane":6944,"ted":6945,"tribes":6946,"proof":6947,"lp":6948,"cream":6949,"researchers":6950,"volunteer":6951,"manor":6952,"silk":6953,"milan":6954,"donated":6955,"allies":6956,"venture":6957,"principle":6958,"delivery":6959,"enterprise":6960,"##ves":6961,"##ans":6962,"bars":6963,"traditionally":6964,"witch":6965,"reminded":6966,"copper":6967,"##uk":6968,"pete":6969,"inter":6970,"links":6971,"colin":6972,"grinned":6973,"elsewhere":6974,"competitive":6975,"frequent":6976,"##oy":6977,"scream":6978,"##hu":6979,"tension":6980,"texts":6981,"submarine":6982,"finnish":6983,"defending":6984,"defend":6985,"pat":6986,"detail":6987,"1884":6988,"affiliated":6989,"stuart":6990,"themes":6991,"villa":6992,"periods":6993,"tool":6994,"belgian":6995,"ruling":6996,"crimes":6997,"answers":6998,"folded":6999,"licensed":7000,"resort":7001,"demolished":7002,"hans":7003,"lucy":7004,"1881":7005,"lion":7006,"traded":7007,"photographs":7008,"writes":7009,"craig":7010,"##fa":7011,"trials":7012,"generated":7013,"beth":7014,"noble":7015,"debt":7016,"percentage":7017,"yorkshire":7018,"erected":7019,"ss":7020,"viewed":7021,"grades":7022,"confidence":7023,"ceased":7024,"islam":7025,"telephone":7026,"retail":7027,"##ible":7028,"chile":7029,"m²":7030,"roberts":7031,"sixteen":7032,"##ich":7033,"commented":7034,"hampshire":7035,"innocent":7036,"dual":7037,"pounds":7038,"checked":7039,"regulations":7040,"afghanistan":7041,"sung":7042,"rico":7043,"liberty":7044,"assets":7045,"bigger":7046,"options":7047,"angels":7048,"relegated":7049,"tribute":7050,"wells":7051,"attending":7052,"leaf":7053,"##yan":7054,"butler":7055,"romanian":7056,"forum":7057,"monthly":7058,"lisa":7059,"patterns":7060,"gmina":7061,"##tory":7062,"madison":7063,"hurricane":7064,"rev":7065,"##ians":7066,"bristol":7067,"##ula":7068,"elite":7069,"valuable":7070,"disaster":7071,"democracy":7072,"awareness":7073,"germans":7074,"freyja":7075,"##ins":7076,"loop":7077,"absolutely":7078,"paying":7079,"populations":7080,"maine":7081,"sole":7082,"prayer":7083,"spencer":7084,"releases":7085,"doorway":7086,"bull":7087,"##ani":7088,"lover":7089,"midnight":7090,"conclusion":7091,"##sson":7092,"thirteen":7093,"lily":7094,"mediterranean":7095,"##lt":7096,"nhl":7097,"proud":7098,"sample":7099,"##hill":7100,"drummer":7101,"guinea":7102,"##ova":7103,"murphy":7104,"climb":7105,"##ston":7106,"instant":7107,"attributed":7108,"horn":7109,"ain":7110,"railways":7111,"steven":7112,"##ao":7113,"autumn":7114,"ferry":7115,"opponent":7116,"root":7117,"traveling":7118,"secured":7119,"corridor":7120,"stretched":7121,"tales":7122,"sheet":7123,"trinity":7124,"cattle":7125,"helps":7126,"indicates":7127,"manhattan":7128,"murdered":7129,"fitted":7130,"1882":7131,"gentle":7132,"grandmother":7133,"mines":7134,"shocked":7135,"vegas":7136,"produces":7137,"##light":7138,"caribbean":7139,"##ou":7140,"belong":7141,"continuous":7142,"desperate":7143,"drunk":7144,"historically":7145,"trio":7146,"waved":7147,"raf":7148,"dealing":7149,"nathan":7150,"bat":7151,"murmured":7152,"interrupted":7153,"residing":7154,"scientist":7155,"pioneer":7156,"harold":7157,"aaron":7158,"##net":7159,"delta":7160,"attempting":7161,"minority":7162,"mini":7163,"believes":7164,"chorus":7165,"tend":7166,"lots":7167,"eyed":7168,"indoor":7169,"load":7170,"shots":7171,"updated":7172,"jail":7173,"##llo":7174,"concerning":7175,"connecting":7176,"wealth":7177,"##ved":7178,"slaves":7179,"arrive":7180,"rangers":7181,"sufficient":7182,"rebuilt":7183,"##wick":7184,"cardinal":7185,"flood":7186,"muhammad":7187,"whenever":7188,"relation":7189,"runners":7190,"moral":7191,"repair":7192,"viewers":7193,"arriving":7194,"revenge":7195,"punk":7196,"assisted":7197,"bath":7198,"fairly":7199,"breathe":7200,"lists":7201,"innings":7202,"illustrated":7203,"whisper":7204,"nearest":7205,"voters":7206,"clinton":7207,"ties":7208,"ultimate":7209,"screamed":7210,"beijing":7211,"lions":7212,"andre":7213,"fictional":7214,"gathering":7215,"comfort":7216,"radar":7217,"suitable":7218,"dismissed":7219,"hms":7220,"ban":7221,"pine":7222,"wrist":7223,"atmosphere":7224,"voivodeship":7225,"bid":7226,"timber":7227,"##ned":7228,"##nan":7229,"giants":7230,"##ane":7231,"cameron":7232,"recovery":7233,"uss":7234,"identical":7235,"categories":7236,"switched":7237,"serbia":7238,"laughter":7239,"noah":7240,"ensemble":7241,"therapy":7242,"peoples":7243,"touching":7244,"##off":7245,"locally":7246,"pearl":7247,"platforms":7248,"everywhere":7249,"ballet":7250,"tables":7251,"lanka":7252,"herbert":7253,"outdoor":7254,"toured":7255,"derek":7256,"1883":7257,"spaces":7258,"contested":7259,"swept":7260,"1878":7261,"exclusive":7262,"slight":7263,"connections":7264,"##dra":7265,"winds":7266,"prisoner":7267,"collective":7268,"bangladesh":7269,"tube":7270,"publicly":7271,"wealthy":7272,"thai":7273,"##ys":7274,"isolated":7275,"select":7276,"##ric":7277,"insisted":7278,"pen":7279,"fortune":7280,"ticket":7281,"spotted":7282,"reportedly":7283,"animation":7284,"enforcement":7285,"tanks":7286,"110":7287,"decides":7288,"wider":7289,"lowest":7290,"owen":7291,"##time":7292,"nod":7293,"hitting":7294,"##hn":7295,"gregory":7296,"furthermore":7297,"magazines":7298,"fighters":7299,"solutions":7300,"##ery":7301,"pointing":7302,"requested":7303,"peru":7304,"reed":7305,"chancellor":7306,"knights":7307,"mask":7308,"worker":7309,"eldest":7310,"flames":7311,"reduction":7312,"1860":7313,"volunteers":7314,"##tis":7315,"reporting":7316,"##hl":7317,"wire":7318,"advisory":7319,"endemic":7320,"origins":7321,"settlers":7322,"pursue":7323,"knock":7324,"consumer":7325,"1876":7326,"eu":7327,"compound":7328,"creatures":7329,"mansion":7330,"sentenced":7331,"ivan":7332,"deployed":7333,"guitars":7334,"frowned":7335,"involves":7336,"mechanism":7337,"kilometers":7338,"perspective":7339,"shops":7340,"maps":7341,"terminus":7342,"duncan":7343,"alien":7344,"fist":7345,"bridges":7346,"##pers":7347,"heroes":7348,"fed":7349,"derby":7350,"swallowed":7351,"##ros":7352,"patent":7353,"sara":7354,"illness":7355,"characterized":7356,"adventures":7357,"slide":7358,"hawaii":7359,"jurisdiction":7360,"##op":7361,"organised":7362,"##side":7363,"adelaide":7364,"walks":7365,"biology":7366,"se":7367,"##ties":7368,"rogers":7369,"swing":7370,"tightly":7371,"boundaries":7372,"##rie":7373,"prepare":7374,"implementation":7375,"stolen":7376,"##sha":7377,"certified":7378,"colombia":7379,"edwards":7380,"garage":7381,"##mm":7382,"recalled":7383,"##ball":7384,"rage":7385,"harm":7386,"nigeria":7387,"breast":7388,"##ren":7389,"furniture":7390,"pupils":7391,"settle":7392,"##lus":7393,"cuba":7394,"balls":7395,"client":7396,"alaska":7397,"21st":7398,"linear":7399,"thrust":7400,"celebration":7401,"latino":7402,"genetic":7403,"terror":7404,"##cia":7405,"##ening":7406,"lightning":7407,"fee":7408,"witness":7409,"lodge":7410,"establishing":7411,"skull":7412,"##ique":7413,"earning":7414,"hood":7415,"##ei":7416,"rebellion":7417,"wang":7418,"sporting":7419,"warned":7420,"missile":7421,"devoted":7422,"activist":7423,"porch":7424,"worship":7425,"fourteen":7426,"package":7427,"1871":7428,"decorated":7429,"##shire":7430,"housed":7431,"##ock":7432,"chess":7433,"sailed":7434,"doctors":7435,"oscar":7436,"joan":7437,"treat":7438,"garcia":7439,"harbour":7440,"jeremy":7441,"##ire":7442,"traditions":7443,"dominant":7444,"jacques":7445,"##gon":7446,"##wan":7447,"relocated":7448,"1879":7449,"amendment":7450,"sized":7451,"companion":7452,"simultaneously":7453,"volleyball":7454,"spun":7455,"acre":7456,"increases":7457,"stopping":7458,"loves":7459,"belongs":7460,"affect":7461,"drafted":7462,"tossed":7463,"scout":7464,"battles":7465,"1875":7466,"filming":7467,"shoved":7468,"munich":7469,"tenure":7470,"vertical":7471,"romance":7472,"pc":7473,"##cher":7474,"argue":7475,"##ical":7476,"craft":7477,"ranging":7478,"www":7479,"opens":7480,"honest":7481,"tyler":7482,"yesterday":7483,"virtual":7484,"##let":7485,"muslims":7486,"reveal":7487,"snake":7488,"immigrants":7489,"radical":7490,"screaming":7491,"speakers":7492,"firing":7493,"saving":7494,"belonging":7495,"ease":7496,"lighting":7497,"prefecture":7498,"blame":7499,"farmer":7500,"hungry":7501,"grows":7502,"rubbed":7503,"beam":7504,"sur":7505,"subsidiary":7506,"##cha":7507,"armenian":7508,"sao":7509,"dropping":7510,"conventional":7511,"##fer":7512,"microsoft":7513,"reply":7514,"qualify":7515,"spots":7516,"1867":7517,"sweat":7518,"festivals":7519,"##ken":7520,"immigration":7521,"physician":7522,"discover":7523,"exposure":7524,"sandy":7525,"explanation":7526,"isaac":7527,"implemented":7528,"##fish":7529,"hart":7530,"initiated":7531,"connect":7532,"stakes":7533,"presents":7534,"heights":7535,"householder":7536,"pleased":7537,"tourist":7538,"regardless":7539,"slip":7540,"closest":7541,"##ction":7542,"surely":7543,"sultan":7544,"brings":7545,"riley":7546,"preparation":7547,"aboard":7548,"slammed":7549,"baptist":7550,"experiment":7551,"ongoing":7552,"interstate":7553,"organic":7554,"playoffs":7555,"##ika":7556,"1877":7557,"130":7558,"##tar":7559,"hindu":7560,"error":7561,"tours":7562,"tier":7563,"plenty":7564,"arrangements":7565,"talks":7566,"trapped":7567,"excited":7568,"sank":7569,"ho":7570,"athens":7571,"1872":7572,"denver":7573,"welfare":7574,"suburb":7575,"athletes":7576,"trick":7577,"diverse":7578,"belly":7579,"exclusively":7580,"yelled":7581,"1868":7582,"##med":7583,"conversion":7584,"##ette":7585,"1874":7586,"internationally":7587,"computers":7588,"conductor":7589,"abilities":7590,"sensitive":7591,"hello":7592,"dispute":7593,"measured":7594,"globe":7595,"rocket":7596,"prices":7597,"amsterdam":7598,"flights":7599,"tigers":7600,"inn":7601,"municipalities":7602,"emotion":7603,"references":7604,"3d":7605,"##mus":7606,"explains":7607,"airlines":7608,"manufactured":7609,"pm":7610,"archaeological":7611,"1873":7612,"interpretation":7613,"devon":7614,"comment":7615,"##ites":7616,"settlements":7617,"kissing":7618,"absolute":7619,"improvement":7620,"suite":7621,"impressed":7622,"barcelona":7623,"sullivan":7624,"jefferson":7625,"towers":7626,"jesse":7627,"julie":7628,"##tin":7629,"##lu":7630,"grandson":7631,"hi":7632,"gauge":7633,"regard":7634,"rings":7635,"interviews":7636,"trace":7637,"raymond":7638,"thumb":7639,"departments":7640,"burns":7641,"serial":7642,"bulgarian":7643,"scores":7644,"demonstrated":7645,"##ix":7646,"1866":7647,"kyle":7648,"alberta":7649,"underneath":7650,"romanized":7651,"##ward":7652,"relieved":7653,"acquisition":7654,"phrase":7655,"cliff":7656,"reveals":7657,"han":7658,"cuts":7659,"merger":7660,"custom":7661,"##dar":7662,"nee":7663,"gilbert":7664,"graduation":7665,"##nts":7666,"assessment":7667,"cafe":7668,"difficulty":7669,"demands":7670,"swung":7671,"democrat":7672,"jennifer":7673,"commons":7674,"1940s":7675,"grove":7676,"##yo":7677,"completing":7678,"focuses":7679,"sum":7680,"substitute":7681,"bearing":7682,"stretch":7683,"reception":7684,"##py":7685,"reflected":7686,"essentially":7687,"destination":7688,"pairs":7689,"##ched":7690,"survival":7691,"resource":7692,"##bach":7693,"promoting":7694,"doubles":7695,"messages":7696,"tear":7697,"##down":7698,"##fully":7699,"parade":7700,"florence":7701,"harvey":7702,"incumbent":7703,"partial":7704,"framework":7705,"900":7706,"pedro":7707,"frozen":7708,"procedure":7709,"olivia":7710,"controls":7711,"##mic":7712,"shelter":7713,"personally":7714,"temperatures":7715,"##od":7716,"brisbane":7717,"tested":7718,"sits":7719,"marble":7720,"comprehensive":7721,"oxygen":7722,"leonard":7723,"##kov":7724,"inaugural":7725,"iranian":7726,"referring":7727,"quarters":7728,"attitude":7729,"##ivity":7730,"mainstream":7731,"lined":7732,"mars":7733,"dakota":7734,"norfolk":7735,"unsuccessful":7736,"##°":7737,"explosion":7738,"helicopter":7739,"congressional":7740,"##sing":7741,"inspector":7742,"bitch":7743,"seal":7744,"departed":7745,"divine":7746,"##ters":7747,"coaching":7748,"examination":7749,"punishment":7750,"manufacturer":7751,"sink":7752,"columns":7753,"unincorporated":7754,"signals":7755,"nevada":7756,"squeezed":7757,"dylan":7758,"dining":7759,"photos":7760,"martial":7761,"manuel":7762,"eighteen":7763,"elevator":7764,"brushed":7765,"plates":7766,"ministers":7767,"ivy":7768,"congregation":7769,"##len":7770,"slept":7771,"specialized":7772,"taxes":7773,"curve":7774,"restricted":7775,"negotiations":7776,"likes":7777,"statistical":7778,"arnold":7779,"inspiration":7780,"execution":7781,"bold":7782,"intermediate":7783,"significance":7784,"margin":7785,"ruler":7786,"wheels":7787,"gothic":7788,"intellectual":7789,"dependent":7790,"listened":7791,"eligible":7792,"buses":7793,"widow":7794,"syria":7795,"earn":7796,"cincinnati":7797,"collapsed":7798,"recipient":7799,"secrets":7800,"accessible":7801,"philippine":7802,"maritime":7803,"goddess":7804,"clerk":7805,"surrender":7806,"breaks":7807,"playoff":7808,"database":7809,"##ified":7810,"##lon":7811,"ideal":7812,"beetle":7813,"aspect":7814,"soap":7815,"regulation":7816,"strings":7817,"expand":7818,"anglo":7819,"shorter":7820,"crosses":7821,"retreat":7822,"tough":7823,"coins":7824,"wallace":7825,"directions":7826,"pressing":7827,"##oon":7828,"shipping":7829,"locomotives":7830,"comparison":7831,"topics":7832,"nephew":7833,"##mes":7834,"distinction":7835,"honors":7836,"travelled":7837,"sierra":7838,"ibn":7839,"##over":7840,"fortress":7841,"sa":7842,"recognised":7843,"carved":7844,"1869":7845,"clients":7846,"##dan":7847,"intent":7848,"##mar":7849,"coaches":7850,"describing":7851,"bread":7852,"##ington":7853,"beaten":7854,"northwestern":7855,"##ona":7856,"merit":7857,"youtube":7858,"collapse":7859,"challenges":7860,"em":7861,"historians":7862,"objective":7863,"submitted":7864,"virus":7865,"attacking":7866,"drake":7867,"assume":7868,"##ere":7869,"diseases":7870,"marc":7871,"stem":7872,"leeds":7873,"##cus":7874,"##ab":7875,"farming":7876,"glasses":7877,"##lock":7878,"visits":7879,"nowhere":7880,"fellowship":7881,"relevant":7882,"carries":7883,"restaurants":7884,"experiments":7885,"101":7886,"constantly":7887,"bases":7888,"targets":7889,"shah":7890,"tenth":7891,"opponents":7892,"verse":7893,"territorial":7894,"##ira":7895,"writings":7896,"corruption":7897,"##hs":7898,"instruction":7899,"inherited":7900,"reverse":7901,"emphasis":7902,"##vic":7903,"employee":7904,"arch":7905,"keeps":7906,"rabbi":7907,"watson":7908,"payment":7909,"uh":7910,"##ala":7911,"nancy":7912,"##tre":7913,"venice":7914,"fastest":7915,"sexy":7916,"banned":7917,"adrian":7918,"properly":7919,"ruth":7920,"touchdown":7921,"dollar":7922,"boards":7923,"metre":7924,"circles":7925,"edges":7926,"favour":7927,"comments":7928,"ok":7929,"travels":7930,"liberation":7931,"scattered":7932,"firmly":7933,"##ular":7934,"holland":7935,"permitted":7936,"diesel":7937,"kenya":7938,"den":7939,"originated":7940,"##ral":7941,"demons":7942,"resumed":7943,"dragged":7944,"rider":7945,"##rus":7946,"servant":7947,"blinked":7948,"extend":7949,"torn":7950,"##ias":7951,"##sey":7952,"input":7953,"meal":7954,"everybody":7955,"cylinder":7956,"kinds":7957,"camps":7958,"##fe":7959,"bullet":7960,"logic":7961,"##wn":7962,"croatian":7963,"evolved":7964,"healthy":7965,"fool":7966,"chocolate":7967,"wise":7968,"preserve":7969,"pradesh":7970,"##ess":7971,"respective":7972,"1850":7973,"##ew":7974,"chicken":7975,"artificial":7976,"gross":7977,"corresponding":7978,"convicted":7979,"cage":7980,"caroline":7981,"dialogue":7982,"##dor":7983,"narrative":7984,"stranger":7985,"mario":7986,"br":7987,"christianity":7988,"failing":7989,"trent":7990,"commanding":7991,"buddhist":7992,"1848":7993,"maurice":7994,"focusing":7995,"yale":7996,"bike":7997,"altitude":7998,"##ering":7999,"mouse":8000,"revised":8001,"##sley":8002,"veteran":8003,"##ig":8004,"pulls":8005,"theology":8006,"crashed":8007,"campaigns":8008,"legion":8009,"##ability":8010,"drag":8011,"excellence":8012,"customer":8013,"cancelled":8014,"intensity":8015,"excuse":8016,"##lar":8017,"liga":8018,"participating":8019,"contributing":8020,"printing":8021,"##burn":8022,"variable":8023,"##rk":8024,"curious":8025,"bin":8026,"legacy":8027,"renaissance":8028,"##my":8029,"symptoms":8030,"binding":8031,"vocalist":8032,"dancer":8033,"##nie":8034,"grammar":8035,"gospel":8036,"democrats":8037,"ya":8038,"enters":8039,"sc":8040,"diplomatic":8041,"hitler":8042,"##ser":8043,"clouds":8044,"mathematical":8045,"quit":8046,"defended":8047,"oriented":8048,"##heim":8049,"fundamental":8050,"hardware":8051,"impressive":8052,"equally":8053,"convince":8054,"confederate":8055,"guilt":8056,"chuck":8057,"sliding":8058,"##ware":8059,"magnetic":8060,"narrowed":8061,"petersburg":8062,"bulgaria":8063,"otto":8064,"phd":8065,"skill":8066,"##ama":8067,"reader":8068,"hopes":8069,"pitcher":8070,"reservoir":8071,"hearts":8072,"automatically":8073,"expecting":8074,"mysterious":8075,"bennett":8076,"extensively":8077,"imagined":8078,"seeds":8079,"monitor":8080,"fix":8081,"##ative":8082,"journalism":8083,"struggling":8084,"signature":8085,"ranch":8086,"encounter":8087,"photographer":8088,"observation":8089,"protests":8090,"##pin":8091,"influences":8092,"##hr":8093,"calendar":8094,"##all":8095,"cruz":8096,"croatia":8097,"locomotive":8098,"hughes":8099,"naturally":8100,"shakespeare":8101,"basement":8102,"hook":8103,"uncredited":8104,"faded":8105,"theories":8106,"approaches":8107,"dare":8108,"phillips":8109,"filling":8110,"fury":8111,"obama":8112,"##ain":8113,"efficient":8114,"arc":8115,"deliver":8116,"min":8117,"raid":8118,"breeding":8119,"inducted":8120,"leagues":8121,"efficiency":8122,"axis":8123,"montana":8124,"eagles":8125,"##ked":8126,"supplied":8127,"instructions":8128,"karen":8129,"picking":8130,"indicating":8131,"trap":8132,"anchor":8133,"practically":8134,"christians":8135,"tomb":8136,"vary":8137,"occasional":8138,"electronics":8139,"lords":8140,"readers":8141,"newcastle":8142,"faint":8143,"innovation":8144,"collect":8145,"situations":8146,"engagement":8147,"160":8148,"claude":8149,"mixture":8150,"##feld":8151,"peer":8152,"tissue":8153,"logo":8154,"lean":8155,"##ration":8156,"°f":8157,"floors":8158,"##ven":8159,"architects":8160,"reducing":8161,"##our":8162,"##ments":8163,"rope":8164,"1859":8165,"ottawa":8166,"##har":8167,"samples":8168,"banking":8169,"declaration":8170,"proteins":8171,"resignation":8172,"francois":8173,"saudi":8174,"advocate":8175,"exhibited":8176,"armor":8177,"twins":8178,"divorce":8179,"##ras":8180,"abraham":8181,"reviewed":8182,"jo":8183,"temporarily":8184,"matrix":8185,"physically":8186,"pulse":8187,"curled":8188,"##ena":8189,"difficulties":8190,"bengal":8191,"usage":8192,"##ban":8193,"annie":8194,"riders":8195,"certificate":8196,"##pi":8197,"holes":8198,"warsaw":8199,"distinctive":8200,"jessica":8201,"##mon":8202,"mutual":8203,"1857":8204,"customs":8205,"circular":8206,"eugene":8207,"removal":8208,"loaded":8209,"mere":8210,"vulnerable":8211,"depicted":8212,"generations":8213,"dame":8214,"heir":8215,"enormous":8216,"lightly":8217,"climbing":8218,"pitched":8219,"lessons":8220,"pilots":8221,"nepal":8222,"ram":8223,"google":8224,"preparing":8225,"brad":8226,"louise":8227,"renowned":8228,"##₂":8229,"liam":8230,"##ably":8231,"plaza":8232,"shaw":8233,"sophie":8234,"brilliant":8235,"bills":8236,"##bar":8237,"##nik":8238,"fucking":8239,"mainland":8240,"server":8241,"pleasant":8242,"seized":8243,"veterans":8244,"jerked":8245,"fail":8246,"beta":8247,"brush":8248,"radiation":8249,"stored":8250,"warmth":8251,"southeastern":8252,"nate":8253,"sin":8254,"raced":8255,"berkeley":8256,"joke":8257,"athlete":8258,"designation":8259,"trunk":8260,"##low":8261,"roland":8262,"qualification":8263,"archives":8264,"heels":8265,"artwork":8266,"receives":8267,"judicial":8268,"reserves":8269,"##bed":8270,"woke":8271,"installation":8272,"abu":8273,"floating":8274,"fake":8275,"lesser":8276,"excitement":8277,"interface":8278,"concentrated":8279,"addressed":8280,"characteristic":8281,"amanda":8282,"saxophone":8283,"monk":8284,"auto":8285,"##bus":8286,"releasing":8287,"egg":8288,"dies":8289,"interaction":8290,"defender":8291,"ce":8292,"outbreak":8293,"glory":8294,"loving":8295,"##bert":8296,"sequel":8297,"consciousness":8298,"http":8299,"awake":8300,"ski":8301,"enrolled":8302,"##ress":8303,"handling":8304,"rookie":8305,"brow":8306,"somebody":8307,"biography":8308,"warfare":8309,"amounts":8310,"contracts":8311,"presentation":8312,"fabric":8313,"dissolved":8314,"challenged":8315,"meter":8316,"psychological":8317,"lt":8318,"elevated":8319,"rally":8320,"accurate":8321,"##tha":8322,"hospitals":8323,"undergraduate":8324,"specialist":8325,"venezuela":8326,"exhibit":8327,"shed":8328,"nursing":8329,"protestant":8330,"fluid":8331,"structural":8332,"footage":8333,"jared":8334,"consistent":8335,"prey":8336,"##ska":8337,"succession":8338,"reflect":8339,"exile":8340,"lebanon":8341,"wiped":8342,"suspect":8343,"shanghai":8344,"resting":8345,"integration":8346,"preservation":8347,"marvel":8348,"variant":8349,"pirates":8350,"sheep":8351,"rounded":8352,"capita":8353,"sailing":8354,"colonies":8355,"manuscript":8356,"deemed":8357,"variations":8358,"clarke":8359,"functional":8360,"emerging":8361,"boxing":8362,"relaxed":8363,"curse":8364,"azerbaijan":8365,"heavyweight":8366,"nickname":8367,"editorial":8368,"rang":8369,"grid":8370,"tightened":8371,"earthquake":8372,"flashed":8373,"miguel":8374,"rushing":8375,"##ches":8376,"improvements":8377,"boxes":8378,"brooks":8379,"180":8380,"consumption":8381,"molecular":8382,"felix":8383,"societies":8384,"repeatedly":8385,"variation":8386,"aids":8387,"civic":8388,"graphics":8389,"professionals":8390,"realm":8391,"autonomous":8392,"receiver":8393,"delayed":8394,"workshop":8395,"militia":8396,"chairs":8397,"trump":8398,"canyon":8399,"##point":8400,"harsh":8401,"extending":8402,"lovely":8403,"happiness":8404,"##jan":8405,"stake":8406,"eyebrows":8407,"embassy":8408,"wellington":8409,"hannah":8410,"##ella":8411,"sony":8412,"corners":8413,"bishops":8414,"swear":8415,"cloth":8416,"contents":8417,"xi":8418,"namely":8419,"commenced":8420,"1854":8421,"stanford":8422,"nashville":8423,"courage":8424,"graphic":8425,"commitment":8426,"garrison":8427,"##bin":8428,"hamlet":8429,"clearing":8430,"rebels":8431,"attraction":8432,"literacy":8433,"cooking":8434,"ruins":8435,"temples":8436,"jenny":8437,"humanity":8438,"celebrate":8439,"hasn":8440,"freight":8441,"sixty":8442,"rebel":8443,"bastard":8444,"##art":8445,"newton":8446,"##ada":8447,"deer":8448,"##ges":8449,"##ching":8450,"smiles":8451,"delaware":8452,"singers":8453,"##ets":8454,"approaching":8455,"assists":8456,"flame":8457,"##ph":8458,"boulevard":8459,"barrel":8460,"planted":8461,"##ome":8462,"pursuit":8463,"##sia":8464,"consequences":8465,"posts":8466,"shallow":8467,"invitation":8468,"rode":8469,"depot":8470,"ernest":8471,"kane":8472,"rod":8473,"concepts":8474,"preston":8475,"topic":8476,"chambers":8477,"striking":8478,"blast":8479,"arrives":8480,"descendants":8481,"montgomery":8482,"ranges":8483,"worlds":8484,"##lay":8485,"##ari":8486,"span":8487,"chaos":8488,"praise":8489,"##ag":8490,"fewer":8491,"1855":8492,"sanctuary":8493,"mud":8494,"fbi":8495,"##ions":8496,"programmes":8497,"maintaining":8498,"unity":8499,"harper":8500,"bore":8501,"handsome":8502,"closure":8503,"tournaments":8504,"thunder":8505,"nebraska":8506,"linda":8507,"facade":8508,"puts":8509,"satisfied":8510,"argentine":8511,"dale":8512,"cork":8513,"dome":8514,"panama":8515,"##yl":8516,"1858":8517,"tasks":8518,"experts":8519,"##ates":8520,"feeding":8521,"equation":8522,"##las":8523,"##ida":8524,"##tu":8525,"engage":8526,"bryan":8527,"##ax":8528,"um":8529,"quartet":8530,"melody":8531,"disbanded":8532,"sheffield":8533,"blocked":8534,"gasped":8535,"delay":8536,"kisses":8537,"maggie":8538,"connects":8539,"##non":8540,"sts":8541,"poured":8542,"creator":8543,"publishers":8544,"##we":8545,"guided":8546,"ellis":8547,"extinct":8548,"hug":8549,"gaining":8550,"##ord":8551,"complicated":8552,"##bility":8553,"poll":8554,"clenched":8555,"investigate":8556,"##use":8557,"thereby":8558,"quantum":8559,"spine":8560,"cdp":8561,"humor":8562,"kills":8563,"administered":8564,"semifinals":8565,"##du":8566,"encountered":8567,"ignore":8568,"##bu":8569,"commentary":8570,"##maker":8571,"bother":8572,"roosevelt":8573,"140":8574,"plains":8575,"halfway":8576,"flowing":8577,"cultures":8578,"crack":8579,"imprisoned":8580,"neighboring":8581,"airline":8582,"##ses":8583,"##view":8584,"##mate":8585,"##ec":8586,"gather":8587,"wolves":8588,"marathon":8589,"transformed":8590,"##ill":8591,"cruise":8592,"organisations":8593,"carol":8594,"punch":8595,"exhibitions":8596,"numbered":8597,"alarm":8598,"ratings":8599,"daddy":8600,"silently":8601,"##stein":8602,"queens":8603,"colours":8604,"impression":8605,"guidance":8606,"liu":8607,"tactical":8608,"##rat":8609,"marshal":8610,"della":8611,"arrow":8612,"##ings":8613,"rested":8614,"feared":8615,"tender":8616,"owns":8617,"bitter":8618,"advisor":8619,"escort":8620,"##ides":8621,"spare":8622,"farms":8623,"grants":8624,"##ene":8625,"dragons":8626,"encourage":8627,"colleagues":8628,"cameras":8629,"##und":8630,"sucked":8631,"pile":8632,"spirits":8633,"prague":8634,"statements":8635,"suspension":8636,"landmark":8637,"fence":8638,"torture":8639,"recreation":8640,"bags":8641,"permanently":8642,"survivors":8643,"pond":8644,"spy":8645,"predecessor":8646,"bombing":8647,"coup":8648,"##og":8649,"protecting":8650,"transformation":8651,"glow":8652,"##lands":8653,"##book":8654,"dug":8655,"priests":8656,"andrea":8657,"feat":8658,"barn":8659,"jumping":8660,"##chen":8661,"##ologist":8662,"##con":8663,"casualties":8664,"stern":8665,"auckland":8666,"pipe":8667,"serie":8668,"revealing":8669,"ba":8670,"##bel":8671,"trevor":8672,"mercy":8673,"spectrum":8674,"yang":8675,"consist":8676,"governing":8677,"collaborated":8678,"possessed":8679,"epic":8680,"comprises":8681,"blew":8682,"shane":8683,"##ack":8684,"lopez":8685,"honored":8686,"magical":8687,"sacrifice":8688,"judgment":8689,"perceived":8690,"hammer":8691,"mtv":8692,"baronet":8693,"tune":8694,"das":8695,"missionary":8696,"sheets":8697,"350":8698,"neutral":8699,"oral":8700,"threatening":8701,"attractive":8702,"shade":8703,"aims":8704,"seminary":8705,"##master":8706,"estates":8707,"1856":8708,"michel":8709,"wounds":8710,"refugees":8711,"manufacturers":8712,"##nic":8713,"mercury":8714,"syndrome":8715,"porter":8716,"##iya":8717,"##din":8718,"hamburg":8719,"identification":8720,"upstairs":8721,"purse":8722,"widened":8723,"pause":8724,"cared":8725,"breathed":8726,"affiliate":8727,"santiago":8728,"prevented":8729,"celtic":8730,"fisher":8731,"125":8732,"recruited":8733,"byzantine":8734,"reconstruction":8735,"farther":8736,"##mp":8737,"diet":8738,"sake":8739,"au":8740,"spite":8741,"sensation":8742,"##ert":8743,"blank":8744,"separation":8745,"105":8746,"##hon":8747,"vladimir":8748,"armies":8749,"anime":8750,"##lie":8751,"accommodate":8752,"orbit":8753,"cult":8754,"sofia":8755,"archive":8756,"##ify":8757,"##box":8758,"founders":8759,"sustained":8760,"disorder":8761,"honours":8762,"northeastern":8763,"mia":8764,"crops":8765,"violet":8766,"threats":8767,"blanket":8768,"fires":8769,"canton":8770,"followers":8771,"southwestern":8772,"prototype":8773,"voyage":8774,"assignment":8775,"altered":8776,"moderate":8777,"protocol":8778,"pistol":8779,"##eo":8780,"questioned":8781,"brass":8782,"lifting":8783,"1852":8784,"math":8785,"authored":8786,"##ual":8787,"doug":8788,"dimensional":8789,"dynamic":8790,"##san":8791,"1851":8792,"pronounced":8793,"grateful":8794,"quest":8795,"uncomfortable":8796,"boom":8797,"presidency":8798,"stevens":8799,"relating":8800,"politicians":8801,"chen":8802,"barrier":8803,"quinn":8804,"diana":8805,"mosque":8806,"tribal":8807,"cheese":8808,"palmer":8809,"portions":8810,"sometime":8811,"chester":8812,"treasure":8813,"wu":8814,"bend":8815,"download":8816,"millions":8817,"reforms":8818,"registration":8819,"##osa":8820,"consequently":8821,"monitoring":8822,"ate":8823,"preliminary":8824,"brandon":8825,"invented":8826,"ps":8827,"eaten":8828,"exterior":8829,"intervention":8830,"ports":8831,"documented":8832,"log":8833,"displays":8834,"lecture":8835,"sally":8836,"favourite":8837,"##itz":8838,"vermont":8839,"lo":8840,"invisible":8841,"isle":8842,"breed":8843,"##ator":8844,"journalists":8845,"relay":8846,"speaks":8847,"backward":8848,"explore":8849,"midfielder":8850,"actively":8851,"stefan":8852,"procedures":8853,"cannon":8854,"blond":8855,"kenneth":8856,"centered":8857,"servants":8858,"chains":8859,"libraries":8860,"malcolm":8861,"essex":8862,"henri":8863,"slavery":8864,"##hal":8865,"facts":8866,"fairy":8867,"coached":8868,"cassie":8869,"cats":8870,"washed":8871,"cop":8872,"##fi":8873,"announcement":8874,"item":8875,"2000s":8876,"vinyl":8877,"activated":8878,"marco":8879,"frontier":8880,"growled":8881,"curriculum":8882,"##das":8883,"loyal":8884,"accomplished":8885,"leslie":8886,"ritual":8887,"kenny":8888,"##00":8889,"vii":8890,"napoleon":8891,"hollow":8892,"hybrid":8893,"jungle":8894,"stationed":8895,"friedrich":8896,"counted":8897,"##ulated":8898,"platinum":8899,"theatrical":8900,"seated":8901,"col":8902,"rubber":8903,"glen":8904,"1840":8905,"diversity":8906,"healing":8907,"extends":8908,"id":8909,"provisions":8910,"administrator":8911,"columbus":8912,"##oe":8913,"tributary":8914,"te":8915,"assured":8916,"org":8917,"##uous":8918,"prestigious":8919,"examined":8920,"lectures":8921,"grammy":8922,"ronald":8923,"associations":8924,"bailey":8925,"allan":8926,"essays":8927,"flute":8928,"believing":8929,"consultant":8930,"proceedings":8931,"travelling":8932,"1853":8933,"kit":8934,"kerala":8935,"yugoslavia":8936,"buddy":8937,"methodist":8938,"##ith":8939,"burial":8940,"centres":8941,"batman":8942,"##nda":8943,"discontinued":8944,"bo":8945,"dock":8946,"stockholm":8947,"lungs":8948,"severely":8949,"##nk":8950,"citing":8951,"manga":8952,"##ugh":8953,"steal":8954,"mumbai":8955,"iraqi":8956,"robot":8957,"celebrity":8958,"bride":8959,"broadcasts":8960,"abolished":8961,"pot":8962,"joel":8963,"overhead":8964,"franz":8965,"packed":8966,"reconnaissance":8967,"johann":8968,"acknowledged":8969,"introduce":8970,"handled":8971,"doctorate":8972,"developments":8973,"drinks":8974,"alley":8975,"palestine":8976,"##nis":8977,"##aki":8978,"proceeded":8979,"recover":8980,"bradley":8981,"grain":8982,"patch":8983,"afford":8984,"infection":8985,"nationalist":8986,"legendary":8987,"##ath":8988,"interchange":8989,"virtually":8990,"gen":8991,"gravity":8992,"exploration":8993,"amber":8994,"vital":8995,"wishes":8996,"powell":8997,"doctrine":8998,"elbow":8999,"screenplay":9000,"##bird":9001,"contribute":9002,"indonesian":9003,"pet":9004,"creates":9005,"##com":9006,"enzyme":9007,"kylie":9008,"discipline":9009,"drops":9010,"manila":9011,"hunger":9012,"##ien":9013,"layers":9014,"suffer":9015,"fever":9016,"bits":9017,"monica":9018,"keyboard":9019,"manages":9020,"##hood":9021,"searched":9022,"appeals":9023,"##bad":9024,"testament":9025,"grande":9026,"reid":9027,"##war":9028,"beliefs":9029,"congo":9030,"##ification":9031,"##dia":9032,"si":9033,"requiring":9034,"##via":9035,"casey":9036,"1849":9037,"regret":9038,"streak":9039,"rape":9040,"depends":9041,"syrian":9042,"sprint":9043,"pound":9044,"tourists":9045,"upcoming":9046,"pub":9047,"##xi":9048,"tense":9049,"##els":9050,"practiced":9051,"echo":9052,"nationwide":9053,"guild":9054,"motorcycle":9055,"liz":9056,"##zar":9057,"chiefs":9058,"desired":9059,"elena":9060,"bye":9061,"precious":9062,"absorbed":9063,"relatives":9064,"booth":9065,"pianist":9066,"##mal":9067,"citizenship":9068,"exhausted":9069,"wilhelm":9070,"##ceae":9071,"##hed":9072,"noting":9073,"quarterback":9074,"urge":9075,"hectares":9076,"##gue":9077,"ace":9078,"holly":9079,"##tal":9080,"blonde":9081,"davies":9082,"parked":9083,"sustainable":9084,"stepping":9085,"twentieth":9086,"airfield":9087,"galaxy":9088,"nest":9089,"chip":9090,"##nell":9091,"tan":9092,"shaft":9093,"paulo":9094,"requirement":9095,"##zy":9096,"paradise":9097,"tobacco":9098,"trans":9099,"renewed":9100,"vietnamese":9101,"##cker":9102,"##ju":9103,"suggesting":9104,"catching":9105,"holmes":9106,"enjoying":9107,"md":9108,"trips":9109,"colt":9110,"holder":9111,"butterfly":9112,"nerve":9113,"reformed":9114,"cherry":9115,"bowling":9116,"trailer":9117,"carriage":9118,"goodbye":9119,"appreciate":9120,"toy":9121,"joshua":9122,"interactive":9123,"enabled":9124,"involve":9125,"##kan":9126,"collar":9127,"determination":9128,"bunch":9129,"facebook":9130,"recall":9131,"shorts":9132,"superintendent":9133,"episcopal":9134,"frustration":9135,"giovanni":9136,"nineteenth":9137,"laser":9138,"privately":9139,"array":9140,"circulation":9141,"##ovic":9142,"armstrong":9143,"deals":9144,"painful":9145,"permit":9146,"discrimination":9147,"##wi":9148,"aires":9149,"retiring":9150,"cottage":9151,"ni":9152,"##sta":9153,"horizon":9154,"ellen":9155,"jamaica":9156,"ripped":9157,"fernando":9158,"chapters":9159,"playstation":9160,"patron":9161,"lecturer":9162,"navigation":9163,"behaviour":9164,"genes":9165,"georgian":9166,"export":9167,"solomon":9168,"rivals":9169,"swift":9170,"seventeen":9171,"rodriguez":9172,"princeton":9173,"independently":9174,"sox":9175,"1847":9176,"arguing":9177,"entity":9178,"casting":9179,"hank":9180,"criteria":9181,"oakland":9182,"geographic":9183,"milwaukee":9184,"reflection":9185,"expanding":9186,"conquest":9187,"dubbed":9188,"##tv":9189,"halt":9190,"brave":9191,"brunswick":9192,"doi":9193,"arched":9194,"curtis":9195,"divorced":9196,"predominantly":9197,"somerset":9198,"streams":9199,"ugly":9200,"zoo":9201,"horrible":9202,"curved":9203,"buenos":9204,"fierce":9205,"dictionary":9206,"vector":9207,"theological":9208,"unions":9209,"handful":9210,"stability":9211,"chan":9212,"punjab":9213,"segments":9214,"##lly":9215,"altar":9216,"ignoring":9217,"gesture":9218,"monsters":9219,"pastor":9220,"##stone":9221,"thighs":9222,"unexpected":9223,"operators":9224,"abruptly":9225,"coin":9226,"compiled":9227,"associates":9228,"improving":9229,"migration":9230,"pin":9231,"##ose":9232,"compact":9233,"collegiate":9234,"reserved":9235,"##urs":9236,"quarterfinals":9237,"roster":9238,"restore":9239,"assembled":9240,"hurry":9241,"oval":9242,"##cies":9243,"1846":9244,"flags":9245,"martha":9246,"##del":9247,"victories":9248,"sharply":9249,"##rated":9250,"argues":9251,"deadly":9252,"neo":9253,"drawings":9254,"symbols":9255,"performer":9256,"##iel":9257,"griffin":9258,"restrictions":9259,"editing":9260,"andrews":9261,"java":9262,"journals":9263,"arabia":9264,"compositions":9265,"dee":9266,"pierce":9267,"removing":9268,"hindi":9269,"casino":9270,"runway":9271,"civilians":9272,"minds":9273,"nasa":9274,"hotels":9275,"##zation":9276,"refuge":9277,"rent":9278,"retain":9279,"potentially":9280,"conferences":9281,"suburban":9282,"conducting":9283,"##tto":9284,"##tions":9285,"##tle":9286,"descended":9287,"massacre":9288,"##cal":9289,"ammunition":9290,"terrain":9291,"fork":9292,"souls":9293,"counts":9294,"chelsea":9295,"durham":9296,"drives":9297,"cab":9298,"##bank":9299,"perth":9300,"realizing":9301,"palestinian":9302,"finn":9303,"simpson":9304,"##dal":9305,"betty":9306,"##ule":9307,"moreover":9308,"particles":9309,"cardinals":9310,"tent":9311,"evaluation":9312,"extraordinary":9313,"##oid":9314,"inscription":9315,"##works":9316,"wednesday":9317,"chloe":9318,"maintains":9319,"panels":9320,"ashley":9321,"trucks":9322,"##nation":9323,"cluster":9324,"sunlight":9325,"strikes":9326,"zhang":9327,"##wing":9328,"dialect":9329,"canon":9330,"##ap":9331,"tucked":9332,"##ws":9333,"collecting":9334,"##mas":9335,"##can":9336,"##sville":9337,"maker":9338,"quoted":9339,"evan":9340,"franco":9341,"aria":9342,"buying":9343,"cleaning":9344,"eva":9345,"closet":9346,"provision":9347,"apollo":9348,"clinic":9349,"rat":9350,"##ez":9351,"necessarily":9352,"ac":9353,"##gle":9354,"##ising":9355,"venues":9356,"flipped":9357,"cent":9358,"spreading":9359,"trustees":9360,"checking":9361,"authorized":9362,"##sco":9363,"disappointed":9364,"##ado":9365,"notion":9366,"duration":9367,"trumpet":9368,"hesitated":9369,"topped":9370,"brussels":9371,"rolls":9372,"theoretical":9373,"hint":9374,"define":9375,"aggressive":9376,"repeat":9377,"wash":9378,"peaceful":9379,"optical":9380,"width":9381,"allegedly":9382,"mcdonald":9383,"strict":9384,"copyright":9385,"##illa":9386,"investors":9387,"mar":9388,"jam":9389,"witnesses":9390,"sounding":9391,"miranda":9392,"michelle":9393,"privacy":9394,"hugo":9395,"harmony":9396,"##pp":9397,"valid":9398,"lynn":9399,"glared":9400,"nina":9401,"102":9402,"headquartered":9403,"diving":9404,"boarding":9405,"gibson":9406,"##ncy":9407,"albanian":9408,"marsh":9409,"routine":9410,"dealt":9411,"enhanced":9412,"er":9413,"intelligent":9414,"substance":9415,"targeted":9416,"enlisted":9417,"discovers":9418,"spinning":9419,"observations":9420,"pissed":9421,"smoking":9422,"rebecca":9423,"capitol":9424,"visa":9425,"varied":9426,"costume":9427,"seemingly":9428,"indies":9429,"compensation":9430,"surgeon":9431,"thursday":9432,"arsenal":9433,"westminster":9434,"suburbs":9435,"rid":9436,"anglican":9437,"##ridge":9438,"knots":9439,"foods":9440,"alumni":9441,"lighter":9442,"fraser":9443,"whoever":9444,"portal":9445,"scandal":9446,"##ray":9447,"gavin":9448,"advised":9449,"instructor":9450,"flooding":9451,"terrorist":9452,"##ale":9453,"teenage":9454,"interim":9455,"senses":9456,"duck":9457,"teen":9458,"thesis":9459,"abby":9460,"eager":9461,"overcome":9462,"##ile":9463,"newport":9464,"glenn":9465,"rises":9466,"shame":9467,"##cc":9468,"prompted":9469,"priority":9470,"forgot":9471,"bomber":9472,"nicolas":9473,"protective":9474,"360":9475,"cartoon":9476,"katherine":9477,"breeze":9478,"lonely":9479,"trusted":9480,"henderson":9481,"richardson":9482,"relax":9483,"banner":9484,"candy":9485,"palms":9486,"remarkable":9487,"##rio":9488,"legends":9489,"cricketer":9490,"essay":9491,"ordained":9492,"edmund":9493,"rifles":9494,"trigger":9495,"##uri":9496,"##away":9497,"sail":9498,"alert":9499,"1830":9500,"audiences":9501,"penn":9502,"sussex":9503,"siblings":9504,"pursued":9505,"indianapolis":9506,"resist":9507,"rosa":9508,"consequence":9509,"succeed":9510,"avoided":9511,"1845":9512,"##ulation":9513,"inland":9514,"##tie":9515,"##nna":9516,"counsel":9517,"profession":9518,"chronicle":9519,"hurried":9520,"##una":9521,"eyebrow":9522,"eventual":9523,"bleeding":9524,"innovative":9525,"cure":9526,"##dom":9527,"committees":9528,"accounting":9529,"con":9530,"scope":9531,"hardy":9532,"heather":9533,"tenor":9534,"gut":9535,"herald":9536,"codes":9537,"tore":9538,"scales":9539,"wagon":9540,"##oo":9541,"luxury":9542,"tin":9543,"prefer":9544,"fountain":9545,"triangle":9546,"bonds":9547,"darling":9548,"convoy":9549,"dried":9550,"traced":9551,"beings":9552,"troy":9553,"accidentally":9554,"slam":9555,"findings":9556,"smelled":9557,"joey":9558,"lawyers":9559,"outcome":9560,"steep":9561,"bosnia":9562,"configuration":9563,"shifting":9564,"toll":9565,"brook":9566,"performers":9567,"lobby":9568,"philosophical":9569,"construct":9570,"shrine":9571,"aggregate":9572,"boot":9573,"cox":9574,"phenomenon":9575,"savage":9576,"insane":9577,"solely":9578,"reynolds":9579,"lifestyle":9580,"##ima":9581,"nationally":9582,"holdings":9583,"consideration":9584,"enable":9585,"edgar":9586,"mo":9587,"mama":9588,"##tein":9589,"fights":9590,"relegation":9591,"chances":9592,"atomic":9593,"hub":9594,"conjunction":9595,"awkward":9596,"reactions":9597,"currency":9598,"finale":9599,"kumar":9600,"underwent":9601,"steering":9602,"elaborate":9603,"gifts":9604,"comprising":9605,"melissa":9606,"veins":9607,"reasonable":9608,"sunshine":9609,"chi":9610,"solve":9611,"trails":9612,"inhabited":9613,"elimination":9614,"ethics":9615,"huh":9616,"ana":9617,"molly":9618,"consent":9619,"apartments":9620,"layout":9621,"marines":9622,"##ces":9623,"hunters":9624,"bulk":9625,"##oma":9626,"hometown":9627,"##wall":9628,"##mont":9629,"cracked":9630,"reads":9631,"neighbouring":9632,"withdrawn":9633,"admission":9634,"wingspan":9635,"damned":9636,"anthology":9637,"lancashire":9638,"brands":9639,"batting":9640,"forgive":9641,"cuban":9642,"awful":9643,"##lyn":9644,"104":9645,"dimensions":9646,"imagination":9647,"##ade":9648,"dante":9649,"##ship":9650,"tracking":9651,"desperately":9652,"goalkeeper":9653,"##yne":9654,"groaned":9655,"workshops":9656,"confident":9657,"burton":9658,"gerald":9659,"milton":9660,"circus":9661,"uncertain":9662,"slope":9663,"copenhagen":9664,"sophia":9665,"fog":9666,"philosopher":9667,"portraits":9668,"accent":9669,"cycling":9670,"varying":9671,"gripped":9672,"larvae":9673,"garrett":9674,"specified":9675,"scotia":9676,"mature":9677,"luther":9678,"kurt":9679,"rap":9680,"##kes":9681,"aerial":9682,"750":9683,"ferdinand":9684,"heated":9685,"es":9686,"transported":9687,"##shan":9688,"safely":9689,"nonetheless":9690,"##orn":9691,"##gal":9692,"motors":9693,"demanding":9694,"##sburg":9695,"startled":9696,"##brook":9697,"ally":9698,"generate":9699,"caps":9700,"ghana":9701,"stained":9702,"demo":9703,"mentions":9704,"beds":9705,"ap":9706,"afterward":9707,"diary":9708,"##bling":9709,"utility":9710,"##iro":9711,"richards":9712,"1837":9713,"conspiracy":9714,"conscious":9715,"shining":9716,"footsteps":9717,"observer":9718,"cyprus":9719,"urged":9720,"loyalty":9721,"developer":9722,"probability":9723,"olive":9724,"upgraded":9725,"gym":9726,"miracle":9727,"insects":9728,"graves":9729,"1844":9730,"ourselves":9731,"hydrogen":9732,"amazon":9733,"katie":9734,"tickets":9735,"poets":9736,"##pm":9737,"planes":9738,"##pan":9739,"prevention":9740,"witnessed":9741,"dense":9742,"jin":9743,"randy":9744,"tang":9745,"warehouse":9746,"monroe":9747,"bang":9748,"archived":9749,"elderly":9750,"investigations":9751,"alec":9752,"granite":9753,"mineral":9754,"conflicts":9755,"controlling":9756,"aboriginal":9757,"carlo":9758,"##zu":9759,"mechanics":9760,"stan":9761,"stark":9762,"rhode":9763,"skirt":9764,"est":9765,"##berry":9766,"bombs":9767,"respected":9768,"##horn":9769,"imposed":9770,"limestone":9771,"deny":9772,"nominee":9773,"memphis":9774,"grabbing":9775,"disabled":9776,"##als":9777,"amusement":9778,"aa":9779,"frankfurt":9780,"corn":9781,"referendum":9782,"varies":9783,"slowed":9784,"disk":9785,"firms":9786,"unconscious":9787,"incredible":9788,"clue":9789,"sue":9790,"##zhou":9791,"twist":9792,"##cio":9793,"joins":9794,"idaho":9795,"chad":9796,"developers":9797,"computing":9798,"destroyer":9799,"103":9800,"mortal":9801,"tucker":9802,"kingston":9803,"choices":9804,"yu":9805,"carson":9806,"1800":9807,"os":9808,"whitney":9809,"geneva":9810,"pretend":9811,"dimension":9812,"staged":9813,"plateau":9814,"maya":9815,"##une":9816,"freestyle":9817,"##bc":9818,"rovers":9819,"hiv":9820,"##ids":9821,"tristan":9822,"classroom":9823,"prospect":9824,"##hus":9825,"honestly":9826,"diploma":9827,"lied":9828,"thermal":9829,"auxiliary":9830,"feast":9831,"unlikely":9832,"iata":9833,"##tel":9834,"morocco":9835,"pounding":9836,"treasury":9837,"lithuania":9838,"considerably":9839,"1841":9840,"dish":9841,"1812":9842,"geological":9843,"matching":9844,"stumbled":9845,"destroying":9846,"marched":9847,"brien":9848,"advances":9849,"cake":9850,"nicole":9851,"belle":9852,"settling":9853,"measuring":9854,"directing":9855,"##mie":9856,"tuesday":9857,"bassist":9858,"capabilities":9859,"stunned":9860,"fraud":9861,"torpedo":9862,"##list":9863,"##phone":9864,"anton":9865,"wisdom":9866,"surveillance":9867,"ruined":9868,"##ulate":9869,"lawsuit":9870,"healthcare":9871,"theorem":9872,"halls":9873,"trend":9874,"aka":9875,"horizontal":9876,"dozens":9877,"acquire":9878,"lasting":9879,"swim":9880,"hawk":9881,"gorgeous":9882,"fees":9883,"vicinity":9884,"decrease":9885,"adoption":9886,"tactics":9887,"##ography":9888,"pakistani":9889,"##ole":9890,"draws":9891,"##hall":9892,"willie":9893,"burke":9894,"heath":9895,"algorithm":9896,"integral":9897,"powder":9898,"elliott":9899,"brigadier":9900,"jackie":9901,"tate":9902,"varieties":9903,"darker":9904,"##cho":9905,"lately":9906,"cigarette":9907,"specimens":9908,"adds":9909,"##ree":9910,"##ensis":9911,"##inger":9912,"exploded":9913,"finalist":9914,"cia":9915,"murders":9916,"wilderness":9917,"arguments":9918,"nicknamed":9919,"acceptance":9920,"onwards":9921,"manufacture":9922,"robertson":9923,"jets":9924,"tampa":9925,"enterprises":9926,"blog":9927,"loudly":9928,"composers":9929,"nominations":9930,"1838":9931,"ai":9932,"malta":9933,"inquiry":9934,"automobile":9935,"hosting":9936,"viii":9937,"rays":9938,"tilted":9939,"grief":9940,"museums":9941,"strategies":9942,"furious":9943,"euro":9944,"equality":9945,"cohen":9946,"poison":9947,"surrey":9948,"wireless":9949,"governed":9950,"ridiculous":9951,"moses":9952,"##esh":9953,"##room":9954,"vanished":9955,"##ito":9956,"barnes":9957,"attract":9958,"morrison":9959,"istanbul":9960,"##iness":9961,"absent":9962,"rotation":9963,"petition":9964,"janet":9965,"##logical":9966,"satisfaction":9967,"custody":9968,"deliberately":9969,"observatory":9970,"comedian":9971,"surfaces":9972,"pinyin":9973,"novelist":9974,"strictly":9975,"canterbury":9976,"oslo":9977,"monks":9978,"embrace":9979,"ibm":9980,"jealous":9981,"photograph":9982,"continent":9983,"dorothy":9984,"marina":9985,"doc":9986,"excess":9987,"holden":9988,"allegations":9989,"explaining":9990,"stack":9991,"avoiding":9992,"lance":9993,"storyline":9994,"majesty":9995,"poorly":9996,"spike":9997,"dos":9998,"bradford":9999,"raven":10000,"travis":10001,"classics":10002,"proven":10003,"voltage":10004,"pillow":10005,"fists":10006,"butt":10007,"1842":10008,"interpreted":10009,"##car":10010,"1839":10011,"gage":10012,"telegraph":10013,"lens":10014,"promising":10015,"expelled":10016,"casual":10017,"collector":10018,"zones":10019,"##min":10020,"silly":10021,"nintendo":10022,"##kh":10023,"##bra":10024,"downstairs":10025,"chef":10026,"suspicious":10027,"afl":10028,"flies":10029,"vacant":10030,"uganda":10031,"pregnancy":10032,"condemned":10033,"lutheran":10034,"estimates":10035,"cheap":10036,"decree":10037,"saxon":10038,"proximity":10039,"stripped":10040,"idiot":10041,"deposits":10042,"contrary":10043,"presenter":10044,"magnus":10045,"glacier":10046,"im":10047,"offense":10048,"edwin":10049,"##ori":10050,"upright":10051,"##long":10052,"bolt":10053,"##ois":10054,"toss":10055,"geographical":10056,"##izes":10057,"environments":10058,"delicate":10059,"marking":10060,"abstract":10061,"xavier":10062,"nails":10063,"windsor":10064,"plantation":10065,"occurring":10066,"equity":10067,"saskatchewan":10068,"fears":10069,"drifted":10070,"sequences":10071,"vegetation":10072,"revolt":10073,"##stic":10074,"1843":10075,"sooner":10076,"fusion":10077,"opposing":10078,"nato":10079,"skating":10080,"1836":10081,"secretly":10082,"ruin":10083,"lease":10084,"##oc":10085,"edit":10086,"##nne":10087,"flora":10088,"anxiety":10089,"ruby":10090,"##ological":10091,"##mia":10092,"tel":10093,"bout":10094,"taxi":10095,"emmy":10096,"frost":10097,"rainbow":10098,"compounds":10099,"foundations":10100,"rainfall":10101,"assassination":10102,"nightmare":10103,"dominican":10104,"##win":10105,"achievements":10106,"deserve":10107,"orlando":10108,"intact":10109,"armenia":10110,"##nte":10111,"calgary":10112,"valentine":10113,"106":10114,"marion":10115,"proclaimed":10116,"theodore":10117,"bells":10118,"courtyard":10119,"thigh":10120,"gonzalez":10121,"console":10122,"troop":10123,"minimal":10124,"monte":10125,"everyday":10126,"##ence":10127,"##if":10128,"supporter":10129,"terrorism":10130,"buck":10131,"openly":10132,"presbyterian":10133,"activists":10134,"carpet":10135,"##iers":10136,"rubbing":10137,"uprising":10138,"##yi":10139,"cute":10140,"conceived":10141,"legally":10142,"##cht":10143,"millennium":10144,"cello":10145,"velocity":10146,"ji":10147,"rescued":10148,"cardiff":10149,"1835":10150,"rex":10151,"concentrate":10152,"senators":10153,"beard":10154,"rendered":10155,"glowing":10156,"battalions":10157,"scouts":10158,"competitors":10159,"sculptor":10160,"catalogue":10161,"arctic":10162,"ion":10163,"raja":10164,"bicycle":10165,"wow":10166,"glancing":10167,"lawn":10168,"##woman":10169,"gentleman":10170,"lighthouse":10171,"publish":10172,"predicted":10173,"calculated":10174,"##val":10175,"variants":10176,"##gne":10177,"strain":10178,"##ui":10179,"winston":10180,"deceased":10181,"##nus":10182,"touchdowns":10183,"brady":10184,"caleb":10185,"sinking":10186,"echoed":10187,"crush":10188,"hon":10189,"blessed":10190,"protagonist":10191,"hayes":10192,"endangered":10193,"magnitude":10194,"editors":10195,"##tine":10196,"estimate":10197,"responsibilities":10198,"##mel":10199,"backup":10200,"laying":10201,"consumed":10202,"sealed":10203,"zurich":10204,"lovers":10205,"frustrated":10206,"##eau":10207,"ahmed":10208,"kicking":10209,"mit":10210,"treasurer":10211,"1832":10212,"biblical":10213,"refuse":10214,"terrified":10215,"pump":10216,"agrees":10217,"genuine":10218,"imprisonment":10219,"refuses":10220,"plymouth":10221,"##hen":10222,"lou":10223,"##nen":10224,"tara":10225,"trembling":10226,"antarctic":10227,"ton":10228,"learns":10229,"##tas":10230,"crap":10231,"crucial":10232,"faction":10233,"atop":10234,"##borough":10235,"wrap":10236,"lancaster":10237,"odds":10238,"hopkins":10239,"erik":10240,"lyon":10241,"##eon":10242,"bros":10243,"##ode":10244,"snap":10245,"locality":10246,"tips":10247,"empress":10248,"crowned":10249,"cal":10250,"acclaimed":10251,"chuckled":10252,"##ory":10253,"clara":10254,"sends":10255,"mild":10256,"towel":10257,"##fl":10258,"##day":10259,"##а":10260,"wishing":10261,"assuming":10262,"interviewed":10263,"##bal":10264,"##die":10265,"interactions":10266,"eden":10267,"cups":10268,"helena":10269,"##lf":10270,"indie":10271,"beck":10272,"##fire":10273,"batteries":10274,"filipino":10275,"wizard":10276,"parted":10277,"##lam":10278,"traces":10279,"##born":10280,"rows":10281,"idol":10282,"albany":10283,"delegates":10284,"##ees":10285,"##sar":10286,"discussions":10287,"##ex":10288,"notre":10289,"instructed":10290,"belgrade":10291,"highways":10292,"suggestion":10293,"lauren":10294,"possess":10295,"orientation":10296,"alexandria":10297,"abdul":10298,"beats":10299,"salary":10300,"reunion":10301,"ludwig":10302,"alright":10303,"wagner":10304,"intimate":10305,"pockets":10306,"slovenia":10307,"hugged":10308,"brighton":10309,"merchants":10310,"cruel":10311,"stole":10312,"trek":10313,"slopes":10314,"repairs":10315,"enrollment":10316,"politically":10317,"underlying":10318,"promotional":10319,"counting":10320,"boeing":10321,"##bb":10322,"isabella":10323,"naming":10324,"##и":10325,"keen":10326,"bacteria":10327,"listing":10328,"separately":10329,"belfast":10330,"ussr":10331,"450":10332,"lithuanian":10333,"anybody":10334,"ribs":10335,"sphere":10336,"martinez":10337,"cock":10338,"embarrassed":10339,"proposals":10340,"fragments":10341,"nationals":10342,"##fs":10343,"##wski":10344,"premises":10345,"fin":10346,"1500":10347,"alpine":10348,"matched":10349,"freely":10350,"bounded":10351,"jace":10352,"sleeve":10353,"##af":10354,"gaming":10355,"pier":10356,"populated":10357,"evident":10358,"##like":10359,"frances":10360,"flooded":10361,"##dle":10362,"frightened":10363,"pour":10364,"trainer":10365,"framed":10366,"visitor":10367,"challenging":10368,"pig":10369,"wickets":10370,"##fold":10371,"infected":10372,"email":10373,"##pes":10374,"arose":10375,"##aw":10376,"reward":10377,"ecuador":10378,"oblast":10379,"vale":10380,"ch":10381,"shuttle":10382,"##usa":10383,"bach":10384,"rankings":10385,"forbidden":10386,"cornwall":10387,"accordance":10388,"salem":10389,"consumers":10390,"bruno":10391,"fantastic":10392,"toes":10393,"machinery":10394,"resolved":10395,"julius":10396,"remembering":10397,"propaganda":10398,"iceland":10399,"bombardment":10400,"tide":10401,"contacts":10402,"wives":10403,"##rah":10404,"concerto":10405,"macdonald":10406,"albania":10407,"implement":10408,"daisy":10409,"tapped":10410,"sudan":10411,"helmet":10412,"angela":10413,"mistress":10414,"##lic":10415,"crop":10416,"sunk":10417,"finest":10418,"##craft":10419,"hostile":10420,"##ute":10421,"##tsu":10422,"boxer":10423,"fr":10424,"paths":10425,"adjusted":10426,"habit":10427,"ballot":10428,"supervision":10429,"soprano":10430,"##zen":10431,"bullets":10432,"wicked":10433,"sunset":10434,"regiments":10435,"disappear":10436,"lamp":10437,"performs":10438,"app":10439,"##gia":10440,"##oa":10441,"rabbit":10442,"digging":10443,"incidents":10444,"entries":10445,"##cion":10446,"dishes":10447,"##oi":10448,"introducing":10449,"##ati":10450,"##fied":10451,"freshman":10452,"slot":10453,"jill":10454,"tackles":10455,"baroque":10456,"backs":10457,"##iest":10458,"lone":10459,"sponsor":10460,"destiny":10461,"altogether":10462,"convert":10463,"##aro":10464,"consensus":10465,"shapes":10466,"demonstration":10467,"basically":10468,"feminist":10469,"auction":10470,"artifacts":10471,"##bing":10472,"strongest":10473,"twitter":10474,"halifax":10475,"2019":10476,"allmusic":10477,"mighty":10478,"smallest":10479,"precise":10480,"alexandra":10481,"viola":10482,"##los":10483,"##ille":10484,"manuscripts":10485,"##illo":10486,"dancers":10487,"ari":10488,"managers":10489,"monuments":10490,"blades":10491,"barracks":10492,"springfield":10493,"maiden":10494,"consolidated":10495,"electron":10496,"##end":10497,"berry":10498,"airing":10499,"wheat":10500,"nobel":10501,"inclusion":10502,"blair":10503,"payments":10504,"geography":10505,"bee":10506,"cc":10507,"eleanor":10508,"react":10509,"##hurst":10510,"afc":10511,"manitoba":10512,"##yu":10513,"su":10514,"lineup":10515,"fitness":10516,"recreational":10517,"investments":10518,"airborne":10519,"disappointment":10520,"##dis":10521,"edmonton":10522,"viewing":10523,"##row":10524,"renovation":10525,"##cast":10526,"infant":10527,"bankruptcy":10528,"roses":10529,"aftermath":10530,"pavilion":10531,"##yer":10532,"carpenter":10533,"withdrawal":10534,"ladder":10535,"##hy":10536,"discussing":10537,"popped":10538,"reliable":10539,"agreements":10540,"rochester":10541,"##abad":10542,"curves":10543,"bombers":10544,"220":10545,"rao":10546,"reverend":10547,"decreased":10548,"choosing":10549,"107":10550,"stiff":10551,"consulting":10552,"naples":10553,"crawford":10554,"tracy":10555,"ka":10556,"ribbon":10557,"cops":10558,"##lee":10559,"crushed":10560,"deciding":10561,"unified":10562,"teenager":10563,"accepting":10564,"flagship":10565,"explorer":10566,"poles":10567,"sanchez":10568,"inspection":10569,"revived":10570,"skilled":10571,"induced":10572,"exchanged":10573,"flee":10574,"locals":10575,"tragedy":10576,"swallow":10577,"loading":10578,"hanna":10579,"demonstrate":10580,"##ela":10581,"salvador":10582,"flown":10583,"contestants":10584,"civilization":10585,"##ines":10586,"wanna":10587,"rhodes":10588,"fletcher":10589,"hector":10590,"knocking":10591,"considers":10592,"##ough":10593,"nash":10594,"mechanisms":10595,"sensed":10596,"mentally":10597,"walt":10598,"unclear":10599,"##eus":10600,"renovated":10601,"madame":10602,"##cks":10603,"crews":10604,"governmental":10605,"##hin":10606,"undertaken":10607,"monkey":10608,"##ben":10609,"##ato":10610,"fatal":10611,"armored":10612,"copa":10613,"caves":10614,"governance":10615,"grasp":10616,"perception":10617,"certification":10618,"froze":10619,"damp":10620,"tugged":10621,"wyoming":10622,"##rg":10623,"##ero":10624,"newman":10625,"##lor":10626,"nerves":10627,"curiosity":10628,"graph":10629,"115":10630,"##ami":10631,"withdraw":10632,"tunnels":10633,"dull":10634,"meredith":10635,"moss":10636,"exhibits":10637,"neighbors":10638,"communicate":10639,"accuracy":10640,"explored":10641,"raiders":10642,"republicans":10643,"secular":10644,"kat":10645,"superman":10646,"penny":10647,"criticised":10648,"##tch":10649,"freed":10650,"update":10651,"conviction":10652,"wade":10653,"ham":10654,"likewise":10655,"delegation":10656,"gotta":10657,"doll":10658,"promises":10659,"technological":10660,"myth":10661,"nationality":10662,"resolve":10663,"convent":10664,"##mark":10665,"sharon":10666,"dig":10667,"sip":10668,"coordinator":10669,"entrepreneur":10670,"fold":10671,"##dine":10672,"capability":10673,"councillor":10674,"synonym":10675,"blown":10676,"swan":10677,"cursed":10678,"1815":10679,"jonas":10680,"haired":10681,"sofa":10682,"canvas":10683,"keeper":10684,"rivalry":10685,"##hart":10686,"rapper":10687,"speedway":10688,"swords":10689,"postal":10690,"maxwell":10691,"estonia":10692,"potter":10693,"recurring":10694,"##nn":10695,"##ave":10696,"errors":10697,"##oni":10698,"cognitive":10699,"1834":10700,"##²":10701,"claws":10702,"nadu":10703,"roberto":10704,"bce":10705,"wrestler":10706,"ellie":10707,"##ations":10708,"infinite":10709,"ink":10710,"##tia":10711,"presumably":10712,"finite":10713,"staircase":10714,"108":10715,"noel":10716,"patricia":10717,"nacional":10718,"##cation":10719,"chill":10720,"eternal":10721,"tu":10722,"preventing":10723,"prussia":10724,"fossil":10725,"limbs":10726,"##logist":10727,"ernst":10728,"frog":10729,"perez":10730,"rene":10731,"##ace":10732,"pizza":10733,"prussian":10734,"##ios":10735,"##vy":10736,"molecules":10737,"regulatory":10738,"answering":10739,"opinions":10740,"sworn":10741,"lengths":10742,"supposedly":10743,"hypothesis":10744,"upward":10745,"habitats":10746,"seating":10747,"ancestors":10748,"drank":10749,"yield":10750,"hd":10751,"synthesis":10752,"researcher":10753,"modest":10754,"##var":10755,"mothers":10756,"peered":10757,"voluntary":10758,"homeland":10759,"##the":10760,"acclaim":10761,"##igan":10762,"static":10763,"valve":10764,"luxembourg":10765,"alto":10766,"carroll":10767,"fe":10768,"receptor":10769,"norton":10770,"ambulance":10771,"##tian":10772,"johnston":10773,"catholics":10774,"depicting":10775,"jointly":10776,"elephant":10777,"gloria":10778,"mentor":10779,"badge":10780,"ahmad":10781,"distinguish":10782,"remarked":10783,"councils":10784,"precisely":10785,"allison":10786,"advancing":10787,"detection":10788,"crowded":10789,"##10":10790,"cooperative":10791,"ankle":10792,"mercedes":10793,"dagger":10794,"surrendered":10795,"pollution":10796,"commit":10797,"subway":10798,"jeffrey":10799,"lesson":10800,"sculptures":10801,"provider":10802,"##fication":10803,"membrane":10804,"timothy":10805,"rectangular":10806,"fiscal":10807,"heating":10808,"teammate":10809,"basket":10810,"particle":10811,"anonymous":10812,"deployment":10813,"##ple":10814,"missiles":10815,"courthouse":10816,"proportion":10817,"shoe":10818,"sec":10819,"##ller":10820,"complaints":10821,"forbes":10822,"blacks":10823,"abandon":10824,"remind":10825,"sizes":10826,"overwhelming":10827,"autobiography":10828,"natalie":10829,"##awa":10830,"risks":10831,"contestant":10832,"countryside":10833,"babies":10834,"scorer":10835,"invaded":10836,"enclosed":10837,"proceed":10838,"hurling":10839,"disorders":10840,"##cu":10841,"reflecting":10842,"continuously":10843,"cruiser":10844,"graduates":10845,"freeway":10846,"investigated":10847,"ore":10848,"deserved":10849,"maid":10850,"blocking":10851,"phillip":10852,"jorge":10853,"shakes":10854,"dove":10855,"mann":10856,"variables":10857,"lacked":10858,"burden":10859,"accompanying":10860,"que":10861,"consistently":10862,"organizing":10863,"provisional":10864,"complained":10865,"endless":10866,"##rm":10867,"tubes":10868,"juice":10869,"georges":10870,"krishna":10871,"mick":10872,"labels":10873,"thriller":10874,"##uch":10875,"laps":10876,"arcade":10877,"sage":10878,"snail":10879,"##table":10880,"shannon":10881,"fi":10882,"laurence":10883,"seoul":10884,"vacation":10885,"presenting":10886,"hire":10887,"churchill":10888,"surprisingly":10889,"prohibited":10890,"savannah":10891,"technically":10892,"##oli":10893,"170":10894,"##lessly":10895,"testimony":10896,"suited":10897,"speeds":10898,"toys":10899,"romans":10900,"mlb":10901,"flowering":10902,"measurement":10903,"talented":10904,"kay":10905,"settings":10906,"charleston":10907,"expectations":10908,"shattered":10909,"achieving":10910,"triumph":10911,"ceremonies":10912,"portsmouth":10913,"lanes":10914,"mandatory":10915,"loser":10916,"stretching":10917,"cologne":10918,"realizes":10919,"seventy":10920,"cornell":10921,"careers":10922,"webb":10923,"##ulating":10924,"americas":10925,"budapest":10926,"ava":10927,"suspicion":10928,"##ison":10929,"yo":10930,"conrad":10931,"##hai":10932,"sterling":10933,"jessie":10934,"rector":10935,"##az":10936,"1831":10937,"transform":10938,"organize":10939,"loans":10940,"christine":10941,"volcanic":10942,"warrant":10943,"slender":10944,"summers":10945,"subfamily":10946,"newer":10947,"danced":10948,"dynamics":10949,"rhine":10950,"proceeds":10951,"heinrich":10952,"gastropod":10953,"commands":10954,"sings":10955,"facilitate":10956,"easter":10957,"ra":10958,"positioned":10959,"responses":10960,"expense":10961,"fruits":10962,"yanked":10963,"imported":10964,"25th":10965,"velvet":10966,"vic":10967,"primitive":10968,"tribune":10969,"baldwin":10970,"neighbourhood":10971,"donna":10972,"rip":10973,"hay":10974,"pr":10975,"##uro":10976,"1814":10977,"espn":10978,"welcomed":10979,"##aria":10980,"qualifier":10981,"glare":10982,"highland":10983,"timing":10984,"##cted":10985,"shells":10986,"eased":10987,"geometry":10988,"louder":10989,"exciting":10990,"slovakia":10991,"##sion":10992,"##iz":10993,"##lot":10994,"savings":10995,"prairie":10996,"##ques":10997,"marching":10998,"rafael":10999,"tonnes":11000,"##lled":11001,"curtain":11002,"preceding":11003,"shy":11004,"heal":11005,"greene":11006,"worthy":11007,"##pot":11008,"detachment":11009,"bury":11010,"sherman":11011,"##eck":11012,"reinforced":11013,"seeks":11014,"bottles":11015,"contracted":11016,"duchess":11017,"outfit":11018,"walsh":11019,"##sc":11020,"mickey":11021,"##ase":11022,"geoffrey":11023,"archer":11024,"squeeze":11025,"dawson":11026,"eliminate":11027,"invention":11028,"##enberg":11029,"neal":11030,"##eth":11031,"stance":11032,"dealer":11033,"coral":11034,"maple":11035,"retire":11036,"polo":11037,"simplified":11038,"##ht":11039,"1833":11040,"hid":11041,"watts":11042,"backwards":11043,"jules":11044,"##oke":11045,"genesis":11046,"mt":11047,"frames":11048,"rebounds":11049,"burma":11050,"woodland":11051,"moist":11052,"santos":11053,"whispers":11054,"drained":11055,"subspecies":11056,"##aa":11057,"streaming":11058,"ulster":11059,"burnt":11060,"correspondence":11061,"maternal":11062,"gerard":11063,"denis":11064,"stealing":11065,"##load":11066,"genius":11067,"duchy":11068,"##oria":11069,"inaugurated":11070,"momentum":11071,"suits":11072,"placement":11073,"sovereign":11074,"clause":11075,"thames":11076,"##hara":11077,"confederation":11078,"reservation":11079,"sketch":11080,"yankees":11081,"lets":11082,"rotten":11083,"charm":11084,"hal":11085,"verses":11086,"ultra":11087,"commercially":11088,"dot":11089,"salon":11090,"citation":11091,"adopt":11092,"winnipeg":11093,"mist":11094,"allocated":11095,"cairo":11096,"##boy":11097,"jenkins":11098,"interference":11099,"objectives":11100,"##wind":11101,"1820":11102,"portfolio":11103,"armoured":11104,"sectors":11105,"##eh":11106,"initiatives":11107,"##world":11108,"integrity":11109,"exercises":11110,"robe":11111,"tap":11112,"ab":11113,"gazed":11114,"##tones":11115,"distracted":11116,"rulers":11117,"111":11118,"favorable":11119,"jerome":11120,"tended":11121,"cart":11122,"factories":11123,"##eri":11124,"diplomat":11125,"valued":11126,"gravel":11127,"charitable":11128,"##try":11129,"calvin":11130,"exploring":11131,"chang":11132,"shepherd":11133,"terrace":11134,"pdf":11135,"pupil":11136,"##ural":11137,"reflects":11138,"ups":11139,"##rch":11140,"governors":11141,"shelf":11142,"depths":11143,"##nberg":11144,"trailed":11145,"crest":11146,"tackle":11147,"##nian":11148,"##ats":11149,"hatred":11150,"##kai":11151,"clare":11152,"makers":11153,"ethiopia":11154,"longtime":11155,"detected":11156,"embedded":11157,"lacking":11158,"slapped":11159,"rely":11160,"thomson":11161,"anticipation":11162,"iso":11163,"morton":11164,"successive":11165,"agnes":11166,"screenwriter":11167,"straightened":11168,"philippe":11169,"playwright":11170,"haunted":11171,"licence":11172,"iris":11173,"intentions":11174,"sutton":11175,"112":11176,"logical":11177,"correctly":11178,"##weight":11179,"branded":11180,"licked":11181,"tipped":11182,"silva":11183,"ricky":11184,"narrator":11185,"requests":11186,"##ents":11187,"greeted":11188,"supernatural":11189,"cow":11190,"##wald":11191,"lung":11192,"refusing":11193,"employer":11194,"strait":11195,"gaelic":11196,"liner":11197,"##piece":11198,"zoe":11199,"sabha":11200,"##mba":11201,"driveway":11202,"harvest":11203,"prints":11204,"bates":11205,"reluctantly":11206,"threshold":11207,"algebra":11208,"ira":11209,"wherever":11210,"coupled":11211,"240":11212,"assumption":11213,"picks":11214,"##air":11215,"designers":11216,"raids":11217,"gentlemen":11218,"##ean":11219,"roller":11220,"blowing":11221,"leipzig":11222,"locks":11223,"screw":11224,"dressing":11225,"strand":11226,"##lings":11227,"scar":11228,"dwarf":11229,"depicts":11230,"##nu":11231,"nods":11232,"##mine":11233,"differ":11234,"boris":11235,"##eur":11236,"yuan":11237,"flip":11238,"##gie":11239,"mob":11240,"invested":11241,"questioning":11242,"applying":11243,"##ture":11244,"shout":11245,"##sel":11246,"gameplay":11247,"blamed":11248,"illustrations":11249,"bothered":11250,"weakness":11251,"rehabilitation":11252,"##of":11253,"##zes":11254,"envelope":11255,"rumors":11256,"miners":11257,"leicester":11258,"subtle":11259,"kerry":11260,"##ico":11261,"ferguson":11262,"##fu":11263,"premiership":11264,"ne":11265,"##cat":11266,"bengali":11267,"prof":11268,"catches":11269,"remnants":11270,"dana":11271,"##rily":11272,"shouting":11273,"presidents":11274,"baltic":11275,"ought":11276,"ghosts":11277,"dances":11278,"sailors":11279,"shirley":11280,"fancy":11281,"dominic":11282,"##bie":11283,"madonna":11284,"##rick":11285,"bark":11286,"buttons":11287,"gymnasium":11288,"ashes":11289,"liver":11290,"toby":11291,"oath":11292,"providence":11293,"doyle":11294,"evangelical":11295,"nixon":11296,"cement":11297,"carnegie":11298,"embarked":11299,"hatch":11300,"surroundings":11301,"guarantee":11302,"needing":11303,"pirate":11304,"essence":11305,"##bee":11306,"filter":11307,"crane":11308,"hammond":11309,"projected":11310,"immune":11311,"percy":11312,"twelfth":11313,"##ult":11314,"regent":11315,"doctoral":11316,"damon":11317,"mikhail":11318,"##ichi":11319,"lu":11320,"critically":11321,"elect":11322,"realised":11323,"abortion":11324,"acute":11325,"screening":11326,"mythology":11327,"steadily":11328,"##fc":11329,"frown":11330,"nottingham":11331,"kirk":11332,"wa":11333,"minneapolis":11334,"##rra":11335,"module":11336,"algeria":11337,"mc":11338,"nautical":11339,"encounters":11340,"surprising":11341,"statues":11342,"availability":11343,"shirts":11344,"pie":11345,"alma":11346,"brows":11347,"munster":11348,"mack":11349,"soup":11350,"crater":11351,"tornado":11352,"sanskrit":11353,"cedar":11354,"explosive":11355,"bordered":11356,"dixon":11357,"planets":11358,"stamp":11359,"exam":11360,"happily":11361,"##bble":11362,"carriers":11363,"kidnapped":11364,"##vis":11365,"accommodation":11366,"emigrated":11367,"##met":11368,"knockout":11369,"correspondent":11370,"violation":11371,"profits":11372,"peaks":11373,"lang":11374,"specimen":11375,"agenda":11376,"ancestry":11377,"pottery":11378,"spelling":11379,"equations":11380,"obtaining":11381,"ki":11382,"linking":11383,"1825":11384,"debris":11385,"asylum":11386,"##20":11387,"buddhism":11388,"teddy":11389,"##ants":11390,"gazette":11391,"##nger":11392,"##sse":11393,"dental":11394,"eligibility":11395,"utc":11396,"fathers":11397,"averaged":11398,"zimbabwe":11399,"francesco":11400,"coloured":11401,"hissed":11402,"translator":11403,"lynch":11404,"mandate":11405,"humanities":11406,"mackenzie":11407,"uniforms":11408,"lin":11409,"##iana":11410,"##gio":11411,"asset":11412,"mhz":11413,"fitting":11414,"samantha":11415,"genera":11416,"wei":11417,"rim":11418,"beloved":11419,"shark":11420,"riot":11421,"entities":11422,"expressions":11423,"indo":11424,"carmen":11425,"slipping":11426,"owing":11427,"abbot":11428,"neighbor":11429,"sidney":11430,"##av":11431,"rats":11432,"recommendations":11433,"encouraging":11434,"squadrons":11435,"anticipated":11436,"commanders":11437,"conquered":11438,"##oto":11439,"donations":11440,"diagnosed":11441,"##mond":11442,"divide":11443,"##iva":11444,"guessed":11445,"decoration":11446,"vernon":11447,"auditorium":11448,"revelation":11449,"conversations":11450,"##kers":11451,"##power":11452,"herzegovina":11453,"dash":11454,"alike":11455,"protested":11456,"lateral":11457,"herman":11458,"accredited":11459,"mg":11460,"##gent":11461,"freeman":11462,"mel":11463,"fiji":11464,"crow":11465,"crimson":11466,"##rine":11467,"livestock":11468,"##pped":11469,"humanitarian":11470,"bored":11471,"oz":11472,"whip":11473,"##lene":11474,"##ali":11475,"legitimate":11476,"alter":11477,"grinning":11478,"spelled":11479,"anxious":11480,"oriental":11481,"wesley":11482,"##nin":11483,"##hole":11484,"carnival":11485,"controller":11486,"detect":11487,"##ssa":11488,"bowed":11489,"educator":11490,"kosovo":11491,"macedonia":11492,"##sin":11493,"occupy":11494,"mastering":11495,"stephanie":11496,"janeiro":11497,"para":11498,"unaware":11499,"nurses":11500,"noon":11501,"135":11502,"cam":11503,"hopefully":11504,"ranger":11505,"combine":11506,"sociology":11507,"polar":11508,"rica":11509,"##eer":11510,"neill":11511,"##sman":11512,"holocaust":11513,"##ip":11514,"doubled":11515,"lust":11516,"1828":11517,"109":11518,"decent":11519,"cooling":11520,"unveiled":11521,"##card":11522,"1829":11523,"nsw":11524,"homer":11525,"chapman":11526,"meyer":11527,"##gin":11528,"dive":11529,"mae":11530,"reagan":11531,"expertise":11532,"##gled":11533,"darwin":11534,"brooke":11535,"sided":11536,"prosecution":11537,"investigating":11538,"comprised":11539,"petroleum":11540,"genres":11541,"reluctant":11542,"differently":11543,"trilogy":11544,"johns":11545,"vegetables":11546,"corpse":11547,"highlighted":11548,"lounge":11549,"pension":11550,"unsuccessfully":11551,"elegant":11552,"aided":11553,"ivory":11554,"beatles":11555,"amelia":11556,"cain":11557,"dubai":11558,"sunny":11559,"immigrant":11560,"babe":11561,"click":11562,"##nder":11563,"underwater":11564,"pepper":11565,"combining":11566,"mumbled":11567,"atlas":11568,"horns":11569,"accessed":11570,"ballad":11571,"physicians":11572,"homeless":11573,"gestured":11574,"rpm":11575,"freak":11576,"louisville":11577,"corporations":11578,"patriots":11579,"prizes":11580,"rational":11581,"warn":11582,"modes":11583,"decorative":11584,"overnight":11585,"din":11586,"troubled":11587,"phantom":11588,"##ort":11589,"monarch":11590,"sheer":11591,"##dorf":11592,"generals":11593,"guidelines":11594,"organs":11595,"addresses":11596,"##zon":11597,"enhance":11598,"curling":11599,"parishes":11600,"cord":11601,"##kie":11602,"linux":11603,"caesar":11604,"deutsche":11605,"bavaria":11606,"##bia":11607,"coleman":11608,"cyclone":11609,"##eria":11610,"bacon":11611,"petty":11612,"##yama":11613,"##old":11614,"hampton":11615,"diagnosis":11616,"1824":11617,"throws":11618,"complexity":11619,"rita":11620,"disputed":11621,"##₃":11622,"pablo":11623,"##sch":11624,"marketed":11625,"trafficking":11626,"##ulus":11627,"examine":11628,"plague":11629,"formats":11630,"##oh":11631,"vault":11632,"faithful":11633,"##bourne":11634,"webster":11635,"##ox":11636,"highlights":11637,"##ient":11638,"##ann":11639,"phones":11640,"vacuum":11641,"sandwich":11642,"modeling":11643,"##gated":11644,"bolivia":11645,"clergy":11646,"qualities":11647,"isabel":11648,"##nas":11649,"##ars":11650,"wears":11651,"screams":11652,"reunited":11653,"annoyed":11654,"bra":11655,"##ancy":11656,"##rate":11657,"differential":11658,"transmitter":11659,"tattoo":11660,"container":11661,"poker":11662,"##och":11663,"excessive":11664,"resides":11665,"cowboys":11666,"##tum":11667,"augustus":11668,"trash":11669,"providers":11670,"statute":11671,"retreated":11672,"balcony":11673,"reversed":11674,"void":11675,"storey":11676,"preceded":11677,"masses":11678,"leap":11679,"laughs":11680,"neighborhoods":11681,"wards":11682,"schemes":11683,"falcon":11684,"santo":11685,"battlefield":11686,"pad":11687,"ronnie":11688,"thread":11689,"lesbian":11690,"venus":11691,"##dian":11692,"beg":11693,"sandstone":11694,"daylight":11695,"punched":11696,"gwen":11697,"analog":11698,"stroked":11699,"wwe":11700,"acceptable":11701,"measurements":11702,"dec":11703,"toxic":11704,"##kel":11705,"adequate":11706,"surgical":11707,"economist":11708,"parameters":11709,"varsity":11710,"##sberg":11711,"quantity":11712,"ella":11713,"##chy":11714,"##rton":11715,"countess":11716,"generating":11717,"precision":11718,"diamonds":11719,"expressway":11720,"ga":11721,"##ı":11722,"1821":11723,"uruguay":11724,"talents":11725,"galleries":11726,"expenses":11727,"scanned":11728,"colleague":11729,"outlets":11730,"ryder":11731,"lucien":11732,"##ila":11733,"paramount":11734,"##bon":11735,"syracuse":11736,"dim":11737,"fangs":11738,"gown":11739,"sweep":11740,"##sie":11741,"toyota":11742,"missionaries":11743,"websites":11744,"##nsis":11745,"sentences":11746,"adviser":11747,"val":11748,"trademark":11749,"spells":11750,"##plane":11751,"patience":11752,"starter":11753,"slim":11754,"##borg":11755,"toe":11756,"incredibly":11757,"shoots":11758,"elliot":11759,"nobility":11760,"##wyn":11761,"cowboy":11762,"endorsed":11763,"gardner":11764,"tendency":11765,"persuaded":11766,"organisms":11767,"emissions":11768,"kazakhstan":11769,"amused":11770,"boring":11771,"chips":11772,"themed":11773,"##hand":11774,"llc":11775,"constantinople":11776,"chasing":11777,"systematic":11778,"guatemala":11779,"borrowed":11780,"erin":11781,"carey":11782,"##hard":11783,"highlands":11784,"struggles":11785,"1810":11786,"##ifying":11787,"##ced":11788,"wong":11789,"exceptions":11790,"develops":11791,"enlarged":11792,"kindergarten":11793,"castro":11794,"##ern":11795,"##rina":11796,"leigh":11797,"zombie":11798,"juvenile":11799,"##most":11800,"consul":11801,"##nar":11802,"sailor":11803,"hyde":11804,"clarence":11805,"intensive":11806,"pinned":11807,"nasty":11808,"useless":11809,"jung":11810,"clayton":11811,"stuffed":11812,"exceptional":11813,"ix":11814,"apostolic":11815,"230":11816,"transactions":11817,"##dge":11818,"exempt":11819,"swinging":11820,"cove":11821,"religions":11822,"##ash":11823,"shields":11824,"dairy":11825,"bypass":11826,"190":11827,"pursuing":11828,"bug":11829,"joyce":11830,"bombay":11831,"chassis":11832,"southampton":11833,"chat":11834,"interact":11835,"redesignated":11836,"##pen":11837,"nascar":11838,"pray":11839,"salmon":11840,"rigid":11841,"regained":11842,"malaysian":11843,"grim":11844,"publicity":11845,"constituted":11846,"capturing":11847,"toilet":11848,"delegate":11849,"purely":11850,"tray":11851,"drift":11852,"loosely":11853,"striker":11854,"weakened":11855,"trinidad":11856,"mitch":11857,"itv":11858,"defines":11859,"transmitted":11860,"ming":11861,"scarlet":11862,"nodding":11863,"fitzgerald":11864,"fu":11865,"narrowly":11866,"sp":11867,"tooth":11868,"standings":11869,"virtue":11870,"##₁":11871,"##wara":11872,"##cting":11873,"chateau":11874,"gloves":11875,"lid":11876,"##nel":11877,"hurting":11878,"conservatory":11879,"##pel":11880,"sinclair":11881,"reopened":11882,"sympathy":11883,"nigerian":11884,"strode":11885,"advocated":11886,"optional":11887,"chronic":11888,"discharge":11889,"##rc":11890,"suck":11891,"compatible":11892,"laurel":11893,"stella":11894,"shi":11895,"fails":11896,"wage":11897,"dodge":11898,"128":11899,"informal":11900,"sorts":11901,"levi":11902,"buddha":11903,"villagers":11904,"##aka":11905,"chronicles":11906,"heavier":11907,"summoned":11908,"gateway":11909,"3000":11910,"eleventh":11911,"jewelry":11912,"translations":11913,"accordingly":11914,"seas":11915,"##ency":11916,"fiber":11917,"pyramid":11918,"cubic":11919,"dragging":11920,"##ista":11921,"caring":11922,"##ops":11923,"android":11924,"contacted":11925,"lunar":11926,"##dt":11927,"kai":11928,"lisbon":11929,"patted":11930,"1826":11931,"sacramento":11932,"theft":11933,"madagascar":11934,"subtropical":11935,"disputes":11936,"ta":11937,"holidays":11938,"piper":11939,"willow":11940,"mare":11941,"cane":11942,"itunes":11943,"newfoundland":11944,"benny":11945,"companions":11946,"dong":11947,"raj":11948,"observe":11949,"roar":11950,"charming":11951,"plaque":11952,"tibetan":11953,"fossils":11954,"enacted":11955,"manning":11956,"bubble":11957,"tina":11958,"tanzania":11959,"##eda":11960,"##hir":11961,"funk":11962,"swamp":11963,"deputies":11964,"cloak":11965,"ufc":11966,"scenario":11967,"par":11968,"scratch":11969,"metals":11970,"anthem":11971,"guru":11972,"engaging":11973,"specially":11974,"##boat":11975,"dialects":11976,"nineteen":11977,"cecil":11978,"duet":11979,"disability":11980,"messenger":11981,"unofficial":11982,"##lies":11983,"defunct":11984,"eds":11985,"moonlight":11986,"drainage":11987,"surname":11988,"puzzle":11989,"honda":11990,"switching":11991,"conservatives":11992,"mammals":11993,"knox":11994,"broadcaster":11995,"sidewalk":11996,"cope":11997,"##ried":11998,"benson":11999,"princes":12000,"peterson":12001,"##sal":12002,"bedford":12003,"sharks":12004,"eli":12005,"wreck":12006,"alberto":12007,"gasp":12008,"archaeology":12009,"lgbt":12010,"teaches":12011,"securities":12012,"madness":12013,"compromise":12014,"waving":12015,"coordination":12016,"davidson":12017,"visions":12018,"leased":12019,"possibilities":12020,"eighty":12021,"jun":12022,"fernandez":12023,"enthusiasm":12024,"assassin":12025,"sponsorship":12026,"reviewer":12027,"kingdoms":12028,"estonian":12029,"laboratories":12030,"##fy":12031,"##nal":12032,"applies":12033,"verb":12034,"celebrations":12035,"##zzo":12036,"rowing":12037,"lightweight":12038,"sadness":12039,"submit":12040,"mvp":12041,"balanced":12042,"dude":12043,"##vas":12044,"explicitly":12045,"metric":12046,"magnificent":12047,"mound":12048,"brett":12049,"mohammad":12050,"mistakes":12051,"irregular":12052,"##hing":12053,"##ass":12054,"sanders":12055,"betrayed":12056,"shipped":12057,"surge":12058,"##enburg":12059,"reporters":12060,"termed":12061,"georg":12062,"pity":12063,"verbal":12064,"bulls":12065,"abbreviated":12066,"enabling":12067,"appealed":12068,"##are":12069,"##atic":12070,"sicily":12071,"sting":12072,"heel":12073,"sweetheart":12074,"bart":12075,"spacecraft":12076,"brutal":12077,"monarchy":12078,"##tter":12079,"aberdeen":12080,"cameo":12081,"diane":12082,"##ub":12083,"survivor":12084,"clyde":12085,"##aries":12086,"complaint":12087,"##makers":12088,"clarinet":12089,"delicious":12090,"chilean":12091,"karnataka":12092,"coordinates":12093,"1818":12094,"panties":12095,"##rst":12096,"pretending":12097,"ar":12098,"dramatically":12099,"kiev":12100,"bella":12101,"tends":12102,"distances":12103,"113":12104,"catalog":12105,"launching":12106,"instances":12107,"telecommunications":12108,"portable":12109,"lindsay":12110,"vatican":12111,"##eim":12112,"angles":12113,"aliens":12114,"marker":12115,"stint":12116,"screens":12117,"bolton":12118,"##rne":12119,"judy":12120,"wool":12121,"benedict":12122,"plasma":12123,"europa":12124,"spark":12125,"imaging":12126,"filmmaker":12127,"swiftly":12128,"##een":12129,"contributor":12130,"##nor":12131,"opted":12132,"stamps":12133,"apologize":12134,"financing":12135,"butter":12136,"gideon":12137,"sophisticated":12138,"alignment":12139,"avery":12140,"chemicals":12141,"yearly":12142,"speculation":12143,"prominence":12144,"professionally":12145,"##ils":12146,"immortal":12147,"institutional":12148,"inception":12149,"wrists":12150,"identifying":12151,"tribunal":12152,"derives":12153,"gains":12154,"##wo":12155,"papal":12156,"preference":12157,"linguistic":12158,"vince":12159,"operative":12160,"brewery":12161,"##ont":12162,"unemployment":12163,"boyd":12164,"##ured":12165,"##outs":12166,"albeit":12167,"prophet":12168,"1813":12169,"bi":12170,"##rr":12171,"##face":12172,"##rad":12173,"quarterly":12174,"asteroid":12175,"cleaned":12176,"radius":12177,"temper":12178,"##llen":12179,"telugu":12180,"jerk":12181,"viscount":12182,"menu":12183,"##ote":12184,"glimpse":12185,"##aya":12186,"yacht":12187,"hawaiian":12188,"baden":12189,"##rl":12190,"laptop":12191,"readily":12192,"##gu":12193,"monetary":12194,"offshore":12195,"scots":12196,"watches":12197,"##yang":12198,"##arian":12199,"upgrade":12200,"needle":12201,"xbox":12202,"lea":12203,"encyclopedia":12204,"flank":12205,"fingertips":12206,"##pus":12207,"delight":12208,"teachings":12209,"confirm":12210,"roth":12211,"beaches":12212,"midway":12213,"winters":12214,"##iah":12215,"teasing":12216,"daytime":12217,"beverly":12218,"gambling":12219,"bonnie":12220,"##backs":12221,"regulated":12222,"clement":12223,"hermann":12224,"tricks":12225,"knot":12226,"##shing":12227,"##uring":12228,"##vre":12229,"detached":12230,"ecological":12231,"owed":12232,"specialty":12233,"byron":12234,"inventor":12235,"bats":12236,"stays":12237,"screened":12238,"unesco":12239,"midland":12240,"trim":12241,"affection":12242,"##ander":12243,"##rry":12244,"jess":12245,"thoroughly":12246,"feedback":12247,"##uma":12248,"chennai":12249,"strained":12250,"heartbeat":12251,"wrapping":12252,"overtime":12253,"pleaded":12254,"##sworth":12255,"mon":12256,"leisure":12257,"oclc":12258,"##tate":12259,"##ele":12260,"feathers":12261,"angelo":12262,"thirds":12263,"nuts":12264,"surveys":12265,"clever":12266,"gill":12267,"commentator":12268,"##dos":12269,"darren":12270,"rides":12271,"gibraltar":12272,"##nc":12273,"##mu":12274,"dissolution":12275,"dedication":12276,"shin":12277,"meals":12278,"saddle":12279,"elvis":12280,"reds":12281,"chaired":12282,"taller":12283,"appreciation":12284,"functioning":12285,"niece":12286,"favored":12287,"advocacy":12288,"robbie":12289,"criminals":12290,"suffolk":12291,"yugoslav":12292,"passport":12293,"constable":12294,"congressman":12295,"hastings":12296,"vera":12297,"##rov":12298,"consecrated":12299,"sparks":12300,"ecclesiastical":12301,"confined":12302,"##ovich":12303,"muller":12304,"floyd":12305,"nora":12306,"1822":12307,"paved":12308,"1827":12309,"cumberland":12310,"ned":12311,"saga":12312,"spiral":12313,"##flow":12314,"appreciated":12315,"yi":12316,"collaborative":12317,"treating":12318,"similarities":12319,"feminine":12320,"finishes":12321,"##ib":12322,"jade":12323,"import":12324,"##nse":12325,"##hot":12326,"champagne":12327,"mice":12328,"securing":12329,"celebrities":12330,"helsinki":12331,"attributes":12332,"##gos":12333,"cousins":12334,"phases":12335,"ache":12336,"lucia":12337,"gandhi":12338,"submission":12339,"vicar":12340,"spear":12341,"shine":12342,"tasmania":12343,"biting":12344,"detention":12345,"constitute":12346,"tighter":12347,"seasonal":12348,"##gus":12349,"terrestrial":12350,"matthews":12351,"##oka":12352,"effectiveness":12353,"parody":12354,"philharmonic":12355,"##onic":12356,"1816":12357,"strangers":12358,"encoded":12359,"consortium":12360,"guaranteed":12361,"regards":12362,"shifts":12363,"tortured":12364,"collision":12365,"supervisor":12366,"inform":12367,"broader":12368,"insight":12369,"theaters":12370,"armour":12371,"emeritus":12372,"blink":12373,"incorporates":12374,"mapping":12375,"##50":12376,"##ein":12377,"handball":12378,"flexible":12379,"##nta":12380,"substantially":12381,"generous":12382,"thief":12383,"##own":12384,"carr":12385,"loses":12386,"1793":12387,"prose":12388,"ucla":12389,"romeo":12390,"generic":12391,"metallic":12392,"realization":12393,"damages":12394,"mk":12395,"commissioners":12396,"zach":12397,"default":12398,"##ther":12399,"helicopters":12400,"lengthy":12401,"stems":12402,"spa":12403,"partnered":12404,"spectators":12405,"rogue":12406,"indication":12407,"penalties":12408,"teresa":12409,"1801":12410,"sen":12411,"##tric":12412,"dalton":12413,"##wich":12414,"irving":12415,"photographic":12416,"##vey":12417,"dell":12418,"deaf":12419,"peters":12420,"excluded":12421,"unsure":12422,"##vable":12423,"patterson":12424,"crawled":12425,"##zio":12426,"resided":12427,"whipped":12428,"latvia":12429,"slower":12430,"ecole":12431,"pipes":12432,"employers":12433,"maharashtra":12434,"comparable":12435,"va":12436,"textile":12437,"pageant":12438,"##gel":12439,"alphabet":12440,"binary":12441,"irrigation":12442,"chartered":12443,"choked":12444,"antoine":12445,"offs":12446,"waking":12447,"supplement":12448,"##wen":12449,"quantities":12450,"demolition":12451,"regain":12452,"locate":12453,"urdu":12454,"folks":12455,"alt":12456,"114":12457,"##mc":12458,"scary":12459,"andreas":12460,"whites":12461,"##ava":12462,"classrooms":12463,"mw":12464,"aesthetic":12465,"publishes":12466,"valleys":12467,"guides":12468,"cubs":12469,"johannes":12470,"bryant":12471,"conventions":12472,"affecting":12473,"##itt":12474,"drain":12475,"awesome":12476,"isolation":12477,"prosecutor":12478,"ambitious":12479,"apology":12480,"captive":12481,"downs":12482,"atmospheric":12483,"lorenzo":12484,"aisle":12485,"beef":12486,"foul":12487,"##onia":12488,"kidding":12489,"composite":12490,"disturbed":12491,"illusion":12492,"natives":12493,"##ffer":12494,"emi":12495,"rockets":12496,"riverside":12497,"wartime":12498,"painters":12499,"adolf":12500,"melted":12501,"##ail":12502,"uncertainty":12503,"simulation":12504,"hawks":12505,"progressed":12506,"meantime":12507,"builder":12508,"spray":12509,"breach":12510,"unhappy":12511,"regina":12512,"russians":12513,"##urg":12514,"determining":12515,"##tation":12516,"tram":12517,"1806":12518,"##quin":12519,"aging":12520,"##12":12521,"1823":12522,"garion":12523,"rented":12524,"mister":12525,"diaz":12526,"terminated":12527,"clip":12528,"1817":12529,"depend":12530,"nervously":12531,"disco":12532,"owe":12533,"defenders":12534,"shiva":12535,"notorious":12536,"disbelief":12537,"shiny":12538,"worcester":12539,"##gation":12540,"##yr":12541,"trailing":12542,"undertook":12543,"islander":12544,"belarus":12545,"limitations":12546,"watershed":12547,"fuller":12548,"overlooking":12549,"utilized":12550,"raphael":12551,"1819":12552,"synthetic":12553,"breakdown":12554,"klein":12555,"##nate":12556,"moaned":12557,"memoir":12558,"lamb":12559,"practicing":12560,"##erly":12561,"cellular":12562,"arrows":12563,"exotic":12564,"##graphy":12565,"witches":12566,"117":12567,"charted":12568,"rey":12569,"hut":12570,"hierarchy":12571,"subdivision":12572,"freshwater":12573,"giuseppe":12574,"aloud":12575,"reyes":12576,"qatar":12577,"marty":12578,"sideways":12579,"utterly":12580,"sexually":12581,"jude":12582,"prayers":12583,"mccarthy":12584,"softball":12585,"blend":12586,"damien":12587,"##gging":12588,"##metric":12589,"wholly":12590,"erupted":12591,"lebanese":12592,"negro":12593,"revenues":12594,"tasted":12595,"comparative":12596,"teamed":12597,"transaction":12598,"labeled":12599,"maori":12600,"sovereignty":12601,"parkway":12602,"trauma":12603,"gran":12604,"malay":12605,"121":12606,"advancement":12607,"descendant":12608,"2020":12609,"buzz":12610,"salvation":12611,"inventory":12612,"symbolic":12613,"##making":12614,"antarctica":12615,"mps":12616,"##gas":12617,"##bro":12618,"mohammed":12619,"myanmar":12620,"holt":12621,"submarines":12622,"tones":12623,"##lman":12624,"locker":12625,"patriarch":12626,"bangkok":12627,"emerson":12628,"remarks":12629,"predators":12630,"kin":12631,"afghan":12632,"confession":12633,"norwich":12634,"rental":12635,"emerge":12636,"advantages":12637,"##zel":12638,"rca":12639,"##hold":12640,"shortened":12641,"storms":12642,"aidan":12643,"##matic":12644,"autonomy":12645,"compliance":12646,"##quet":12647,"dudley":12648,"atp":12649,"##osis":12650,"1803":12651,"motto":12652,"documentation":12653,"summary":12654,"professors":12655,"spectacular":12656,"christina":12657,"archdiocese":12658,"flashing":12659,"innocence":12660,"remake":12661,"##dell":12662,"psychic":12663,"reef":12664,"scare":12665,"employ":12666,"rs":12667,"sticks":12668,"meg":12669,"gus":12670,"leans":12671,"##ude":12672,"accompany":12673,"bergen":12674,"tomas":12675,"##iko":12676,"doom":12677,"wages":12678,"pools":12679,"##nch":12680,"##bes":12681,"breasts":12682,"scholarly":12683,"alison":12684,"outline":12685,"brittany":12686,"breakthrough":12687,"willis":12688,"realistic":12689,"##cut":12690,"##boro":12691,"competitor":12692,"##stan":12693,"pike":12694,"picnic":12695,"icon":12696,"designing":12697,"commercials":12698,"washing":12699,"villain":12700,"skiing":12701,"micro":12702,"costumes":12703,"auburn":12704,"halted":12705,"executives":12706,"##hat":12707,"logistics":12708,"cycles":12709,"vowel":12710,"applicable":12711,"barrett":12712,"exclaimed":12713,"eurovision":12714,"eternity":12715,"ramon":12716,"##umi":12717,"##lls":12718,"modifications":12719,"sweeping":12720,"disgust":12721,"##uck":12722,"torch":12723,"aviv":12724,"ensuring":12725,"rude":12726,"dusty":12727,"sonic":12728,"donovan":12729,"outskirts":12730,"cu":12731,"pathway":12732,"##band":12733,"##gun":12734,"##lines":12735,"disciplines":12736,"acids":12737,"cadet":12738,"paired":12739,"##40":12740,"sketches":12741,"##sive":12742,"marriages":12743,"##⁺":12744,"folding":12745,"peers":12746,"slovak":12747,"implies":12748,"admired":12749,"##beck":12750,"1880s":12751,"leopold":12752,"instinct":12753,"attained":12754,"weston":12755,"megan":12756,"horace":12757,"##ination":12758,"dorsal":12759,"ingredients":12760,"evolutionary":12761,"##its":12762,"complications":12763,"deity":12764,"lethal":12765,"brushing":12766,"levy":12767,"deserted":12768,"institutes":12769,"posthumously":12770,"delivering":12771,"telescope":12772,"coronation":12773,"motivated":12774,"rapids":12775,"luc":12776,"flicked":12777,"pays":12778,"volcano":12779,"tanner":12780,"weighed":12781,"##nica":12782,"crowds":12783,"frankie":12784,"gifted":12785,"addressing":12786,"granddaughter":12787,"winding":12788,"##rna":12789,"constantine":12790,"gomez":12791,"##front":12792,"landscapes":12793,"rudolf":12794,"anthropology":12795,"slate":12796,"werewolf":12797,"##lio":12798,"astronomy":12799,"circa":12800,"rouge":12801,"dreaming":12802,"sack":12803,"knelt":12804,"drowned":12805,"naomi":12806,"prolific":12807,"tracked":12808,"freezing":12809,"herb":12810,"##dium":12811,"agony":12812,"randall":12813,"twisting":12814,"wendy":12815,"deposit":12816,"touches":12817,"vein":12818,"wheeler":12819,"##bbled":12820,"##bor":12821,"batted":12822,"retaining":12823,"tire":12824,"presently":12825,"compare":12826,"specification":12827,"daemon":12828,"nigel":12829,"##grave":12830,"merry":12831,"recommendation":12832,"czechoslovakia":12833,"sandra":12834,"ng":12835,"roma":12836,"##sts":12837,"lambert":12838,"inheritance":12839,"sheikh":12840,"winchester":12841,"cries":12842,"examining":12843,"##yle":12844,"comeback":12845,"cuisine":12846,"nave":12847,"##iv":12848,"ko":12849,"retrieve":12850,"tomatoes":12851,"barker":12852,"polished":12853,"defining":12854,"irene":12855,"lantern":12856,"personalities":12857,"begging":12858,"tract":12859,"swore":12860,"1809":12861,"175":12862,"##gic":12863,"omaha":12864,"brotherhood":12865,"##rley":12866,"haiti":12867,"##ots":12868,"exeter":12869,"##ete":12870,"##zia":12871,"steele":12872,"dumb":12873,"pearson":12874,"210":12875,"surveyed":12876,"elisabeth":12877,"trends":12878,"##ef":12879,"fritz":12880,"##rf":12881,"premium":12882,"bugs":12883,"fraction":12884,"calmly":12885,"viking":12886,"##birds":12887,"tug":12888,"inserted":12889,"unusually":12890,"##ield":12891,"confronted":12892,"distress":12893,"crashing":12894,"brent":12895,"turks":12896,"resign":12897,"##olo":12898,"cambodia":12899,"gabe":12900,"sauce":12901,"##kal":12902,"evelyn":12903,"116":12904,"extant":12905,"clusters":12906,"quarry":12907,"teenagers":12908,"luna":12909,"##lers":12910,"##ister":12911,"affiliation":12912,"drill":12913,"##ashi":12914,"panthers":12915,"scenic":12916,"libya":12917,"anita":12918,"strengthen":12919,"inscriptions":12920,"##cated":12921,"lace":12922,"sued":12923,"judith":12924,"riots":12925,"##uted":12926,"mint":12927,"##eta":12928,"preparations":12929,"midst":12930,"dub":12931,"challenger":12932,"##vich":12933,"mock":12934,"cf":12935,"displaced":12936,"wicket":12937,"breaths":12938,"enables":12939,"schmidt":12940,"analyst":12941,"##lum":12942,"ag":12943,"highlight":12944,"automotive":12945,"axe":12946,"josef":12947,"newark":12948,"sufficiently":12949,"resembles":12950,"50th":12951,"##pal":12952,"flushed":12953,"mum":12954,"traits":12955,"##ante":12956,"commodore":12957,"incomplete":12958,"warming":12959,"titular":12960,"ceremonial":12961,"ethical":12962,"118":12963,"celebrating":12964,"eighteenth":12965,"cao":12966,"lima":12967,"medalist":12968,"mobility":12969,"strips":12970,"snakes":12971,"##city":12972,"miniature":12973,"zagreb":12974,"barton":12975,"escapes":12976,"umbrella":12977,"automated":12978,"doubted":12979,"differs":12980,"cooled":12981,"georgetown":12982,"dresden":12983,"cooked":12984,"fade":12985,"wyatt":12986,"rna":12987,"jacobs":12988,"carlton":12989,"abundant":12990,"stereo":12991,"boost":12992,"madras":12993,"inning":12994,"##hia":12995,"spur":12996,"ip":12997,"malayalam":12998,"begged":12999,"osaka":13000,"groan":13001,"escaping":13002,"charging":13003,"dose":13004,"vista":13005,"##aj":13006,"bud":13007,"papa":13008,"communists":13009,"advocates":13010,"edged":13011,"tri":13012,"##cent":13013,"resemble":13014,"peaking":13015,"necklace":13016,"fried":13017,"montenegro":13018,"saxony":13019,"goose":13020,"glances":13021,"stuttgart":13022,"curator":13023,"recruit":13024,"grocery":13025,"sympathetic":13026,"##tting":13027,"##fort":13028,"127":13029,"lotus":13030,"randolph":13031,"ancestor":13032,"##rand":13033,"succeeding":13034,"jupiter":13035,"1798":13036,"macedonian":13037,"##heads":13038,"hiking":13039,"1808":13040,"handing":13041,"fischer":13042,"##itive":13043,"garbage":13044,"node":13045,"##pies":13046,"prone":13047,"singular":13048,"papua":13049,"inclined":13050,"attractions":13051,"italia":13052,"pouring":13053,"motioned":13054,"grandma":13055,"garnered":13056,"jacksonville":13057,"corp":13058,"ego":13059,"ringing":13060,"aluminum":13061,"##hausen":13062,"ordering":13063,"##foot":13064,"drawer":13065,"traders":13066,"synagogue":13067,"##play":13068,"##kawa":13069,"resistant":13070,"wandering":13071,"fragile":13072,"fiona":13073,"teased":13074,"var":13075,"hardcore":13076,"soaked":13077,"jubilee":13078,"decisive":13079,"exposition":13080,"mercer":13081,"poster":13082,"valencia":13083,"hale":13084,"kuwait":13085,"1811":13086,"##ises":13087,"##wr":13088,"##eed":13089,"tavern":13090,"gamma":13091,"122":13092,"johan":13093,"##uer":13094,"airways":13095,"amino":13096,"gil":13097,"##ury":13098,"vocational":13099,"domains":13100,"torres":13101,"##sp":13102,"generator":13103,"folklore":13104,"outcomes":13105,"##keeper":13106,"canberra":13107,"shooter":13108,"fl":13109,"beams":13110,"confrontation":13111,"##lling":13112,"##gram":13113,"feb":13114,"aligned":13115,"forestry":13116,"pipeline":13117,"jax":13118,"motorway":13119,"conception":13120,"decay":13121,"##tos":13122,"coffin":13123,"##cott":13124,"stalin":13125,"1805":13126,"escorted":13127,"minded":13128,"##nam":13129,"sitcom":13130,"purchasing":13131,"twilight":13132,"veronica":13133,"additions":13134,"passive":13135,"tensions":13136,"straw":13137,"123":13138,"frequencies":13139,"1804":13140,"refugee":13141,"cultivation":13142,"##iate":13143,"christie":13144,"clary":13145,"bulletin":13146,"crept":13147,"disposal":13148,"##rich":13149,"##zong":13150,"processor":13151,"crescent":13152,"##rol":13153,"bmw":13154,"emphasized":13155,"whale":13156,"nazis":13157,"aurora":13158,"##eng":13159,"dwelling":13160,"hauled":13161,"sponsors":13162,"toledo":13163,"mega":13164,"ideology":13165,"theatres":13166,"tessa":13167,"cerambycidae":13168,"saves":13169,"turtle":13170,"cone":13171,"suspects":13172,"kara":13173,"rusty":13174,"yelling":13175,"greeks":13176,"mozart":13177,"shades":13178,"cocked":13179,"participant":13180,"##tro":13181,"shire":13182,"spit":13183,"freeze":13184,"necessity":13185,"##cos":13186,"inmates":13187,"nielsen":13188,"councillors":13189,"loaned":13190,"uncommon":13191,"omar":13192,"peasants":13193,"botanical":13194,"offspring":13195,"daniels":13196,"formations":13197,"jokes":13198,"1794":13199,"pioneers":13200,"sigma":13201,"licensing":13202,"##sus":13203,"wheelchair":13204,"polite":13205,"1807":13206,"liquor":13207,"pratt":13208,"trustee":13209,"##uta":13210,"forewings":13211,"balloon":13212,"##zz":13213,"kilometre":13214,"camping":13215,"explicit":13216,"casually":13217,"shawn":13218,"foolish":13219,"teammates":13220,"nm":13221,"hassan":13222,"carrie":13223,"judged":13224,"satisfy":13225,"vanessa":13226,"knives":13227,"selective":13228,"cnn":13229,"flowed":13230,"##lice":13231,"eclipse":13232,"stressed":13233,"eliza":13234,"mathematician":13235,"cease":13236,"cultivated":13237,"##roy":13238,"commissions":13239,"browns":13240,"##ania":13241,"destroyers":13242,"sheridan":13243,"meadow":13244,"##rius":13245,"minerals":13246,"##cial":13247,"downstream":13248,"clash":13249,"gram":13250,"memoirs":13251,"ventures":13252,"baha":13253,"seymour":13254,"archie":13255,"midlands":13256,"edith":13257,"fare":13258,"flynn":13259,"invite":13260,"canceled":13261,"tiles":13262,"stabbed":13263,"boulder":13264,"incorporate":13265,"amended":13266,"camden":13267,"facial":13268,"mollusk":13269,"unreleased":13270,"descriptions":13271,"yoga":13272,"grabs":13273,"550":13274,"raises":13275,"ramp":13276,"shiver":13277,"##rose":13278,"coined":13279,"pioneering":13280,"tunes":13281,"qing":13282,"warwick":13283,"tops":13284,"119":13285,"melanie":13286,"giles":13287,"##rous":13288,"wandered":13289,"##inal":13290,"annexed":13291,"nov":13292,"30th":13293,"unnamed":13294,"##ished":13295,"organizational":13296,"airplane":13297,"normandy":13298,"stoke":13299,"whistle":13300,"blessing":13301,"violations":13302,"chased":13303,"holders":13304,"shotgun":13305,"##ctic":13306,"outlet":13307,"reactor":13308,"##vik":13309,"tires":13310,"tearing":13311,"shores":13312,"fortified":13313,"mascot":13314,"constituencies":13315,"nc":13316,"columnist":13317,"productive":13318,"tibet":13319,"##rta":13320,"lineage":13321,"hooked":13322,"oct":13323,"tapes":13324,"judging":13325,"cody":13326,"##gger":13327,"hansen":13328,"kashmir":13329,"triggered":13330,"##eva":13331,"solved":13332,"cliffs":13333,"##tree":13334,"resisted":13335,"anatomy":13336,"protesters":13337,"transparent":13338,"implied":13339,"##iga":13340,"injection":13341,"mattress":13342,"excluding":13343,"##mbo":13344,"defenses":13345,"helpless":13346,"devotion":13347,"##elli":13348,"growl":13349,"liberals":13350,"weber":13351,"phenomena":13352,"atoms":13353,"plug":13354,"##iff":13355,"mortality":13356,"apprentice":13357,"howe":13358,"convincing":13359,"aaa":13360,"swimmer":13361,"barber":13362,"leone":13363,"promptly":13364,"sodium":13365,"def":13366,"nowadays":13367,"arise":13368,"##oning":13369,"gloucester":13370,"corrected":13371,"dignity":13372,"norm":13373,"erie":13374,"##ders":13375,"elders":13376,"evacuated":13377,"sylvia":13378,"compression":13379,"##yar":13380,"hartford":13381,"pose":13382,"backpack":13383,"reasoning":13384,"accepts":13385,"24th":13386,"wipe":13387,"millimetres":13388,"marcel":13389,"##oda":13390,"dodgers":13391,"albion":13392,"1790":13393,"overwhelmed":13394,"aerospace":13395,"oaks":13396,"1795":13397,"showcase":13398,"acknowledge":13399,"recovering":13400,"nolan":13401,"ashe":13402,"hurts":13403,"geology":13404,"fashioned":13405,"disappearance":13406,"farewell":13407,"swollen":13408,"shrug":13409,"marquis":13410,"wimbledon":13411,"124":13412,"rue":13413,"1792":13414,"commemorate":13415,"reduces":13416,"experiencing":13417,"inevitable":13418,"calcutta":13419,"intel":13420,"##court":13421,"murderer":13422,"sticking":13423,"fisheries":13424,"imagery":13425,"bloom":13426,"280":13427,"brake":13428,"##inus":13429,"gustav":13430,"hesitation":13431,"memorable":13432,"po":13433,"viral":13434,"beans":13435,"accidents":13436,"tunisia":13437,"antenna":13438,"spilled":13439,"consort":13440,"treatments":13441,"aye":13442,"perimeter":13443,"##gard":13444,"donation":13445,"hostage":13446,"migrated":13447,"banker":13448,"addiction":13449,"apex":13450,"lil":13451,"trout":13452,"##ously":13453,"conscience":13454,"##nova":13455,"rams":13456,"sands":13457,"genome":13458,"passionate":13459,"troubles":13460,"##lets":13461,"##set":13462,"amid":13463,"##ibility":13464,"##ret":13465,"higgins":13466,"exceed":13467,"vikings":13468,"##vie":13469,"payne":13470,"##zan":13471,"muscular":13472,"##ste":13473,"defendant":13474,"sucking":13475,"##wal":13476,"ibrahim":13477,"fuselage":13478,"claudia":13479,"vfl":13480,"europeans":13481,"snails":13482,"interval":13483,"##garh":13484,"preparatory":13485,"statewide":13486,"tasked":13487,"lacrosse":13488,"viktor":13489,"##lation":13490,"angola":13491,"##hra":13492,"flint":13493,"implications":13494,"employs":13495,"teens":13496,"patrons":13497,"stall":13498,"weekends":13499,"barriers":13500,"scrambled":13501,"nucleus":13502,"tehran":13503,"jenna":13504,"parsons":13505,"lifelong":13506,"robots":13507,"displacement":13508,"5000":13509,"##bles":13510,"precipitation":13511,"##gt":13512,"knuckles":13513,"clutched":13514,"1802":13515,"marrying":13516,"ecology":13517,"marx":13518,"accusations":13519,"declare":13520,"scars":13521,"kolkata":13522,"mat":13523,"meadows":13524,"bermuda":13525,"skeleton":13526,"finalists":13527,"vintage":13528,"crawl":13529,"coordinate":13530,"affects":13531,"subjected":13532,"orchestral":13533,"mistaken":13534,"##tc":13535,"mirrors":13536,"dipped":13537,"relied":13538,"260":13539,"arches":13540,"candle":13541,"##nick":13542,"incorporating":13543,"wildly":13544,"fond":13545,"basilica":13546,"owl":13547,"fringe":13548,"rituals":13549,"whispering":13550,"stirred":13551,"feud":13552,"tertiary":13553,"slick":13554,"goat":13555,"honorable":13556,"whereby":13557,"skip":13558,"ricardo":13559,"stripes":13560,"parachute":13561,"adjoining":13562,"submerged":13563,"synthesizer":13564,"##gren":13565,"intend":13566,"positively":13567,"ninety":13568,"phi":13569,"beaver":13570,"partition":13571,"fellows":13572,"alexis":13573,"prohibition":13574,"carlisle":13575,"bizarre":13576,"fraternity":13577,"##bre":13578,"doubts":13579,"icy":13580,"cbc":13581,"aquatic":13582,"sneak":13583,"sonny":13584,"combines":13585,"airports":13586,"crude":13587,"supervised":13588,"spatial":13589,"merge":13590,"alfonso":13591,"##bic":13592,"corrupt":13593,"scan":13594,"undergo":13595,"##ams":13596,"disabilities":13597,"colombian":13598,"comparing":13599,"dolphins":13600,"perkins":13601,"##lish":13602,"reprinted":13603,"unanimous":13604,"bounced":13605,"hairs":13606,"underworld":13607,"midwest":13608,"semester":13609,"bucket":13610,"paperback":13611,"miniseries":13612,"coventry":13613,"demise":13614,"##leigh":13615,"demonstrations":13616,"sensor":13617,"rotating":13618,"yan":13619,"##hler":13620,"arrange":13621,"soils":13622,"##idge":13623,"hyderabad":13624,"labs":13625,"##dr":13626,"brakes":13627,"grandchildren":13628,"##nde":13629,"negotiated":13630,"rover":13631,"ferrari":13632,"continuation":13633,"directorate":13634,"augusta":13635,"stevenson":13636,"counterpart":13637,"gore":13638,"##rda":13639,"nursery":13640,"rican":13641,"ave":13642,"collectively":13643,"broadly":13644,"pastoral":13645,"repertoire":13646,"asserted":13647,"discovering":13648,"nordic":13649,"styled":13650,"fiba":13651,"cunningham":13652,"harley":13653,"middlesex":13654,"survives":13655,"tumor":13656,"tempo":13657,"zack":13658,"aiming":13659,"lok":13660,"urgent":13661,"##rade":13662,"##nto":13663,"devils":13664,"##ement":13665,"contractor":13666,"turin":13667,"##wl":13668,"##ool":13669,"bliss":13670,"repaired":13671,"simmons":13672,"moan":13673,"astronomical":13674,"cr":13675,"negotiate":13676,"lyric":13677,"1890s":13678,"lara":13679,"bred":13680,"clad":13681,"angus":13682,"pbs":13683,"##ience":13684,"engineered":13685,"posed":13686,"##lk":13687,"hernandez":13688,"possessions":13689,"elbows":13690,"psychiatric":13691,"strokes":13692,"confluence":13693,"electorate":13694,"lifts":13695,"campuses":13696,"lava":13697,"alps":13698,"##ep":13699,"##ution":13700,"##date":13701,"physicist":13702,"woody":13703,"##page":13704,"##ographic":13705,"##itis":13706,"juliet":13707,"reformation":13708,"sparhawk":13709,"320":13710,"complement":13711,"suppressed":13712,"jewel":13713,"##½":13714,"floated":13715,"##kas":13716,"continuity":13717,"sadly":13718,"##ische":13719,"inability":13720,"melting":13721,"scanning":13722,"paula":13723,"flour":13724,"judaism":13725,"safer":13726,"vague":13727,"##lm":13728,"solving":13729,"curb":13730,"##stown":13731,"financially":13732,"gable":13733,"bees":13734,"expired":13735,"miserable":13736,"cassidy":13737,"dominion":13738,"1789":13739,"cupped":13740,"145":13741,"robbery":13742,"facto":13743,"amos":13744,"warden":13745,"resume":13746,"tallest":13747,"marvin":13748,"ing":13749,"pounded":13750,"usd":13751,"declaring":13752,"gasoline":13753,"##aux":13754,"darkened":13755,"270":13756,"650":13757,"sophomore":13758,"##mere":13759,"erection":13760,"gossip":13761,"televised":13762,"risen":13763,"dial":13764,"##eu":13765,"pillars":13766,"##link":13767,"passages":13768,"profound":13769,"##tina":13770,"arabian":13771,"ashton":13772,"silicon":13773,"nail":13774,"##ead":13775,"##lated":13776,"##wer":13777,"##hardt":13778,"fleming":13779,"firearms":13780,"ducked":13781,"circuits":13782,"blows":13783,"waterloo":13784,"titans":13785,"##lina":13786,"atom":13787,"fireplace":13788,"cheshire":13789,"financed":13790,"activation":13791,"algorithms":13792,"##zzi":13793,"constituent":13794,"catcher":13795,"cherokee":13796,"partnerships":13797,"sexuality":13798,"platoon":13799,"tragic":13800,"vivian":13801,"guarded":13802,"whiskey":13803,"meditation":13804,"poetic":13805,"##late":13806,"##nga":13807,"##ake":13808,"porto":13809,"listeners":13810,"dominance":13811,"kendra":13812,"mona":13813,"chandler":13814,"factions":13815,"22nd":13816,"salisbury":13817,"attitudes":13818,"derivative":13819,"##ido":13820,"##haus":13821,"intake":13822,"paced":13823,"javier":13824,"illustrator":13825,"barrels":13826,"bias":13827,"cockpit":13828,"burnett":13829,"dreamed":13830,"ensuing":13831,"##anda":13832,"receptors":13833,"someday":13834,"hawkins":13835,"mattered":13836,"##lal":13837,"slavic":13838,"1799":13839,"jesuit":13840,"cameroon":13841,"wasted":13842,"tai":13843,"wax":13844,"lowering":13845,"victorious":13846,"freaking":13847,"outright":13848,"hancock":13849,"librarian":13850,"sensing":13851,"bald":13852,"calcium":13853,"myers":13854,"tablet":13855,"announcing":13856,"barack":13857,"shipyard":13858,"pharmaceutical":13859,"##uan":13860,"greenwich":13861,"flush":13862,"medley":13863,"patches":13864,"wolfgang":13865,"pt":13866,"speeches":13867,"acquiring":13868,"exams":13869,"nikolai":13870,"##gg":13871,"hayden":13872,"kannada":13873,"##type":13874,"reilly":13875,"##pt":13876,"waitress":13877,"abdomen":13878,"devastated":13879,"capped":13880,"pseudonym":13881,"pharmacy":13882,"fulfill":13883,"paraguay":13884,"1796":13885,"clicked":13886,"##trom":13887,"archipelago":13888,"syndicated":13889,"##hman":13890,"lumber":13891,"orgasm":13892,"rejection":13893,"clifford":13894,"lorraine":13895,"advent":13896,"mafia":13897,"rodney":13898,"brock":13899,"##ght":13900,"##used":13901,"##elia":13902,"cassette":13903,"chamberlain":13904,"despair":13905,"mongolia":13906,"sensors":13907,"developmental":13908,"upstream":13909,"##eg":13910,"##alis":13911,"spanning":13912,"165":13913,"trombone":13914,"basque":13915,"seeded":13916,"interred":13917,"renewable":13918,"rhys":13919,"leapt":13920,"revision":13921,"molecule":13922,"##ages":13923,"chord":13924,"vicious":13925,"nord":13926,"shivered":13927,"23rd":13928,"arlington":13929,"debts":13930,"corpus":13931,"sunrise":13932,"bays":13933,"blackburn":13934,"centimetres":13935,"##uded":13936,"shuddered":13937,"gm":13938,"strangely":13939,"gripping":13940,"cartoons":13941,"isabelle":13942,"orbital":13943,"##ppa":13944,"seals":13945,"proving":13946,"##lton":13947,"refusal":13948,"strengthened":13949,"bust":13950,"assisting":13951,"baghdad":13952,"batsman":13953,"portrayal":13954,"mara":13955,"pushes":13956,"spears":13957,"og":13958,"##cock":13959,"reside":13960,"nathaniel":13961,"brennan":13962,"1776":13963,"confirmation":13964,"caucus":13965,"##worthy":13966,"markings":13967,"yemen":13968,"nobles":13969,"ku":13970,"lazy":13971,"viewer":13972,"catalan":13973,"encompasses":13974,"sawyer":13975,"##fall":13976,"sparked":13977,"substances":13978,"patents":13979,"braves":13980,"arranger":13981,"evacuation":13982,"sergio":13983,"persuade":13984,"dover":13985,"tolerance":13986,"penguin":13987,"cum":13988,"jockey":13989,"insufficient":13990,"townships":13991,"occupying":13992,"declining":13993,"plural":13994,"processed":13995,"projection":13996,"puppet":13997,"flanders":13998,"introduces":13999,"liability":14000,"##yon":14001,"gymnastics":14002,"antwerp":14003,"taipei":14004,"hobart":14005,"candles":14006,"jeep":14007,"wes":14008,"observers":14009,"126":14010,"chaplain":14011,"bundle":14012,"glorious":14013,"##hine":14014,"hazel":14015,"flung":14016,"sol":14017,"excavations":14018,"dumped":14019,"stares":14020,"sh":14021,"bangalore":14022,"triangular":14023,"icelandic":14024,"intervals":14025,"expressing":14026,"turbine":14027,"##vers":14028,"songwriting":14029,"crafts":14030,"##igo":14031,"jasmine":14032,"ditch":14033,"rite":14034,"##ways":14035,"entertaining":14036,"comply":14037,"sorrow":14038,"wrestlers":14039,"basel":14040,"emirates":14041,"marian":14042,"rivera":14043,"helpful":14044,"##some":14045,"caution":14046,"downward":14047,"networking":14048,"##atory":14049,"##tered":14050,"darted":14051,"genocide":14052,"emergence":14053,"replies":14054,"specializing":14055,"spokesman":14056,"convenient":14057,"unlocked":14058,"fading":14059,"augustine":14060,"concentrations":14061,"resemblance":14062,"elijah":14063,"investigator":14064,"andhra":14065,"##uda":14066,"promotes":14067,"bean":14068,"##rrell":14069,"fleeing":14070,"wan":14071,"simone":14072,"announcer":14073,"##ame":14074,"##bby":14075,"lydia":14076,"weaver":14077,"132":14078,"residency":14079,"modification":14080,"##fest":14081,"stretches":14082,"##ast":14083,"alternatively":14084,"nat":14085,"lowe":14086,"lacks":14087,"##ented":14088,"pam":14089,"tile":14090,"concealed":14091,"inferior":14092,"abdullah":14093,"residences":14094,"tissues":14095,"vengeance":14096,"##ided":14097,"moisture":14098,"peculiar":14099,"groove":14100,"zip":14101,"bologna":14102,"jennings":14103,"ninja":14104,"oversaw":14105,"zombies":14106,"pumping":14107,"batch":14108,"livingston":14109,"emerald":14110,"installations":14111,"1797":14112,"peel":14113,"nitrogen":14114,"rama":14115,"##fying":14116,"##star":14117,"schooling":14118,"strands":14119,"responding":14120,"werner":14121,"##ost":14122,"lime":14123,"casa":14124,"accurately":14125,"targeting":14126,"##rod":14127,"underway":14128,"##uru":14129,"hemisphere":14130,"lester":14131,"##yard":14132,"occupies":14133,"2d":14134,"griffith":14135,"angrily":14136,"reorganized":14137,"##owing":14138,"courtney":14139,"deposited":14140,"##dd":14141,"##30":14142,"estadio":14143,"##ifies":14144,"dunn":14145,"exiled":14146,"##ying":14147,"checks":14148,"##combe":14149,"##о":14150,"##fly":14151,"successes":14152,"unexpectedly":14153,"blu":14154,"assessed":14155,"##flower":14156,"##ه":14157,"observing":14158,"sacked":14159,"spiders":14160,"kn":14161,"##tail":14162,"mu":14163,"nodes":14164,"prosperity":14165,"audrey":14166,"divisional":14167,"155":14168,"broncos":14169,"tangled":14170,"adjust":14171,"feeds":14172,"erosion":14173,"paolo":14174,"surf":14175,"directory":14176,"snatched":14177,"humid":14178,"admiralty":14179,"screwed":14180,"gt":14181,"reddish":14182,"##nese":14183,"modules":14184,"trench":14185,"lamps":14186,"bind":14187,"leah":14188,"bucks":14189,"competes":14190,"##nz":14191,"##form":14192,"transcription":14193,"##uc":14194,"isles":14195,"violently":14196,"clutching":14197,"pga":14198,"cyclist":14199,"inflation":14200,"flats":14201,"ragged":14202,"unnecessary":14203,"##hian":14204,"stubborn":14205,"coordinated":14206,"harriet":14207,"baba":14208,"disqualified":14209,"330":14210,"insect":14211,"wolfe":14212,"##fies":14213,"reinforcements":14214,"rocked":14215,"duel":14216,"winked":14217,"embraced":14218,"bricks":14219,"##raj":14220,"hiatus":14221,"defeats":14222,"pending":14223,"brightly":14224,"jealousy":14225,"##xton":14226,"##hm":14227,"##uki":14228,"lena":14229,"gdp":14230,"colorful":14231,"##dley":14232,"stein":14233,"kidney":14234,"##shu":14235,"underwear":14236,"wanderers":14237,"##haw":14238,"##icus":14239,"guardians":14240,"m³":14241,"roared":14242,"habits":14243,"##wise":14244,"permits":14245,"gp":14246,"uranium":14247,"punished":14248,"disguise":14249,"bundesliga":14250,"elise":14251,"dundee":14252,"erotic":14253,"partisan":14254,"pi":14255,"collectors":14256,"float":14257,"individually":14258,"rendering":14259,"behavioral":14260,"bucharest":14261,"ser":14262,"hare":14263,"valerie":14264,"corporal":14265,"nutrition":14266,"proportional":14267,"##isa":14268,"immense":14269,"##kis":14270,"pavement":14271,"##zie":14272,"##eld":14273,"sutherland":14274,"crouched":14275,"1775":14276,"##lp":14277,"suzuki":14278,"trades":14279,"endurance":14280,"operas":14281,"crosby":14282,"prayed":14283,"priory":14284,"rory":14285,"socially":14286,"##urn":14287,"gujarat":14288,"##pu":14289,"walton":14290,"cube":14291,"pasha":14292,"privilege":14293,"lennon":14294,"floods":14295,"thorne":14296,"waterfall":14297,"nipple":14298,"scouting":14299,"approve":14300,"##lov":14301,"minorities":14302,"voter":14303,"dwight":14304,"extensions":14305,"assure":14306,"ballroom":14307,"slap":14308,"dripping":14309,"privileges":14310,"rejoined":14311,"confessed":14312,"demonstrating":14313,"patriotic":14314,"yell":14315,"investor":14316,"##uth":14317,"pagan":14318,"slumped":14319,"squares":14320,"##cle":14321,"##kins":14322,"confront":14323,"bert":14324,"embarrassment":14325,"##aid":14326,"aston":14327,"urging":14328,"sweater":14329,"starr":14330,"yuri":14331,"brains":14332,"williamson":14333,"commuter":14334,"mortar":14335,"structured":14336,"selfish":14337,"exports":14338,"##jon":14339,"cds":14340,"##him":14341,"unfinished":14342,"##rre":14343,"mortgage":14344,"destinations":14345,"##nagar":14346,"canoe":14347,"solitary":14348,"buchanan":14349,"delays":14350,"magistrate":14351,"fk":14352,"##pling":14353,"motivation":14354,"##lier":14355,"##vier":14356,"recruiting":14357,"assess":14358,"##mouth":14359,"malik":14360,"antique":14361,"1791":14362,"pius":14363,"rahman":14364,"reich":14365,"tub":14366,"zhou":14367,"smashed":14368,"airs":14369,"galway":14370,"xii":14371,"conditioning":14372,"honduras":14373,"discharged":14374,"dexter":14375,"##pf":14376,"lionel":14377,"129":14378,"debates":14379,"lemon":14380,"tiffany":14381,"volunteered":14382,"dom":14383,"dioxide":14384,"procession":14385,"devi":14386,"sic":14387,"tremendous":14388,"advertisements":14389,"colts":14390,"transferring":14391,"verdict":14392,"hanover":14393,"decommissioned":14394,"utter":14395,"relate":14396,"pac":14397,"racism":14398,"##top":14399,"beacon":14400,"limp":14401,"similarity":14402,"terra":14403,"occurrence":14404,"ant":14405,"##how":14406,"becky":14407,"capt":14408,"updates":14409,"armament":14410,"richie":14411,"pal":14412,"##graph":14413,"halloween":14414,"mayo":14415,"##ssen":14416,"##bone":14417,"cara":14418,"serena":14419,"fcc":14420,"dolls":14421,"obligations":14422,"##dling":14423,"violated":14424,"lafayette":14425,"jakarta":14426,"exploitation":14427,"##ime":14428,"infamous":14429,"iconic":14430,"##lah":14431,"##park":14432,"kitty":14433,"moody":14434,"reginald":14435,"dread":14436,"spill":14437,"crystals":14438,"olivier":14439,"modeled":14440,"bluff":14441,"equilibrium":14442,"separating":14443,"notices":14444,"ordnance":14445,"extinction":14446,"onset":14447,"cosmic":14448,"attachment":14449,"sammy":14450,"expose":14451,"privy":14452,"anchored":14453,"##bil":14454,"abbott":14455,"admits":14456,"bending":14457,"baritone":14458,"emmanuel":14459,"policeman":14460,"vaughan":14461,"winged":14462,"climax":14463,"dresses":14464,"denny":14465,"polytechnic":14466,"mohamed":14467,"burmese":14468,"authentic":14469,"nikki":14470,"genetics":14471,"grandparents":14472,"homestead":14473,"gaza":14474,"postponed":14475,"metacritic":14476,"una":14477,"##sby":14478,"##bat":14479,"unstable":14480,"dissertation":14481,"##rial":14482,"##cian":14483,"curls":14484,"obscure":14485,"uncovered":14486,"bronx":14487,"praying":14488,"disappearing":14489,"##hoe":14490,"prehistoric":14491,"coke":14492,"turret":14493,"mutations":14494,"nonprofit":14495,"pits":14496,"monaco":14497,"##ي":14498,"##usion":14499,"prominently":14500,"dispatched":14501,"podium":14502,"##mir":14503,"uci":14504,"##uation":14505,"133":14506,"fortifications":14507,"birthplace":14508,"kendall":14509,"##lby":14510,"##oll":14511,"preacher":14512,"rack":14513,"goodman":14514,"##rman":14515,"persistent":14516,"##ott":14517,"countless":14518,"jaime":14519,"recorder":14520,"lexington":14521,"persecution":14522,"jumps":14523,"renewal":14524,"wagons":14525,"##11":14526,"crushing":14527,"##holder":14528,"decorations":14529,"##lake":14530,"abundance":14531,"wrath":14532,"laundry":14533,"£1":14534,"garde":14535,"##rp":14536,"jeanne":14537,"beetles":14538,"peasant":14539,"##sl":14540,"splitting":14541,"caste":14542,"sergei":14543,"##rer":14544,"##ema":14545,"scripts":14546,"##ively":14547,"rub":14548,"satellites":14549,"##vor":14550,"inscribed":14551,"verlag":14552,"scrapped":14553,"gale":14554,"packages":14555,"chick":14556,"potato":14557,"slogan":14558,"kathleen":14559,"arabs":14560,"##culture":14561,"counterparts":14562,"reminiscent":14563,"choral":14564,"##tead":14565,"rand":14566,"retains":14567,"bushes":14568,"dane":14569,"accomplish":14570,"courtesy":14571,"closes":14572,"##oth":14573,"slaughter":14574,"hague":14575,"krakow":14576,"lawson":14577,"tailed":14578,"elias":14579,"ginger":14580,"##ttes":14581,"canopy":14582,"betrayal":14583,"rebuilding":14584,"turf":14585,"##hof":14586,"frowning":14587,"allegiance":14588,"brigades":14589,"kicks":14590,"rebuild":14591,"polls":14592,"alias":14593,"nationalism":14594,"td":14595,"rowan":14596,"audition":14597,"bowie":14598,"fortunately":14599,"recognizes":14600,"harp":14601,"dillon":14602,"horrified":14603,"##oro":14604,"renault":14605,"##tics":14606,"ropes":14607,"##α":14608,"presumed":14609,"rewarded":14610,"infrared":14611,"wiping":14612,"accelerated":14613,"illustration":14614,"##rid":14615,"presses":14616,"practitioners":14617,"badminton":14618,"##iard":14619,"detained":14620,"##tera":14621,"recognizing":14622,"relates":14623,"misery":14624,"##sies":14625,"##tly":14626,"reproduction":14627,"piercing":14628,"potatoes":14629,"thornton":14630,"esther":14631,"manners":14632,"hbo":14633,"##aan":14634,"ours":14635,"bullshit":14636,"ernie":14637,"perennial":14638,"sensitivity":14639,"illuminated":14640,"rupert":14641,"##jin":14642,"##iss":14643,"##ear":14644,"rfc":14645,"nassau":14646,"##dock":14647,"staggered":14648,"socialism":14649,"##haven":14650,"appointments":14651,"nonsense":14652,"prestige":14653,"sharma":14654,"haul":14655,"##tical":14656,"solidarity":14657,"gps":14658,"##ook":14659,"##rata":14660,"igor":14661,"pedestrian":14662,"##uit":14663,"baxter":14664,"tenants":14665,"wires":14666,"medication":14667,"unlimited":14668,"guiding":14669,"impacts":14670,"diabetes":14671,"##rama":14672,"sasha":14673,"pas":14674,"clive":14675,"extraction":14676,"131":14677,"continually":14678,"constraints":14679,"##bilities":14680,"sonata":14681,"hunted":14682,"sixteenth":14683,"chu":14684,"planting":14685,"quote":14686,"mayer":14687,"pretended":14688,"abs":14689,"spat":14690,"##hua":14691,"ceramic":14692,"##cci":14693,"curtains":14694,"pigs":14695,"pitching":14696,"##dad":14697,"latvian":14698,"sore":14699,"dayton":14700,"##sted":14701,"##qi":14702,"patrols":14703,"slice":14704,"playground":14705,"##nted":14706,"shone":14707,"stool":14708,"apparatus":14709,"inadequate":14710,"mates":14711,"treason":14712,"##ija":14713,"desires":14714,"##liga":14715,"##croft":14716,"somalia":14717,"laurent":14718,"mir":14719,"leonardo":14720,"oracle":14721,"grape":14722,"obliged":14723,"chevrolet":14724,"thirteenth":14725,"stunning":14726,"enthusiastic":14727,"##ede":14728,"accounted":14729,"concludes":14730,"currents":14731,"basil":14732,"##kovic":14733,"drought":14734,"##rica":14735,"mai":14736,"##aire":14737,"shove":14738,"posting":14739,"##shed":14740,"pilgrimage":14741,"humorous":14742,"packing":14743,"fry":14744,"pencil":14745,"wines":14746,"smells":14747,"144":14748,"marilyn":14749,"aching":14750,"newest":14751,"clung":14752,"bon":14753,"neighbours":14754,"sanctioned":14755,"##pie":14756,"mug":14757,"##stock":14758,"drowning":14759,"##mma":14760,"hydraulic":14761,"##vil":14762,"hiring":14763,"reminder":14764,"lilly":14765,"investigators":14766,"##ncies":14767,"sour":14768,"##eous":14769,"compulsory":14770,"packet":14771,"##rion":14772,"##graphic":14773,"##elle":14774,"cannes":14775,"##inate":14776,"depressed":14777,"##rit":14778,"heroic":14779,"importantly":14780,"theresa":14781,"##tled":14782,"conway":14783,"saturn":14784,"marginal":14785,"rae":14786,"##xia":14787,"corresponds":14788,"royce":14789,"pact":14790,"jasper":14791,"explosives":14792,"packaging":14793,"aluminium":14794,"##ttered":14795,"denotes":14796,"rhythmic":14797,"spans":14798,"assignments":14799,"hereditary":14800,"outlined":14801,"originating":14802,"sundays":14803,"lad":14804,"reissued":14805,"greeting":14806,"beatrice":14807,"##dic":14808,"pillar":14809,"marcos":14810,"plots":14811,"handbook":14812,"alcoholic":14813,"judiciary":14814,"avant":14815,"slides":14816,"extract":14817,"masculine":14818,"blur":14819,"##eum":14820,"##force":14821,"homage":14822,"trembled":14823,"owens":14824,"hymn":14825,"trey":14826,"omega":14827,"signaling":14828,"socks":14829,"accumulated":14830,"reacted":14831,"attic":14832,"theo":14833,"lining":14834,"angie":14835,"distraction":14836,"primera":14837,"talbot":14838,"##key":14839,"1200":14840,"ti":14841,"creativity":14842,"billed":14843,"##hey":14844,"deacon":14845,"eduardo":14846,"identifies":14847,"proposition":14848,"dizzy":14849,"gunner":14850,"hogan":14851,"##yam":14852,"##pping":14853,"##hol":14854,"ja":14855,"##chan":14856,"jensen":14857,"reconstructed":14858,"##berger":14859,"clearance":14860,"darius":14861,"##nier":14862,"abe":14863,"harlem":14864,"plea":14865,"dei":14866,"circled":14867,"emotionally":14868,"notation":14869,"fascist":14870,"neville":14871,"exceeded":14872,"upwards":14873,"viable":14874,"ducks":14875,"##fo":14876,"workforce":14877,"racer":14878,"limiting":14879,"shri":14880,"##lson":14881,"possesses":14882,"1600":14883,"kerr":14884,"moths":14885,"devastating":14886,"laden":14887,"disturbing":14888,"locking":14889,"##cture":14890,"gal":14891,"fearing":14892,"accreditation":14893,"flavor":14894,"aide":14895,"1870s":14896,"mountainous":14897,"##baum":14898,"melt":14899,"##ures":14900,"motel":14901,"texture":14902,"servers":14903,"soda":14904,"##mb":14905,"herd":14906,"##nium":14907,"erect":14908,"puzzled":14909,"hum":14910,"peggy":14911,"examinations":14912,"gould":14913,"testified":14914,"geoff":14915,"ren":14916,"devised":14917,"sacks":14918,"##law":14919,"denial":14920,"posters":14921,"grunted":14922,"cesar":14923,"tutor":14924,"ec":14925,"gerry":14926,"offerings":14927,"byrne":14928,"falcons":14929,"combinations":14930,"ct":14931,"incoming":14932,"pardon":14933,"rocking":14934,"26th":14935,"avengers":14936,"flared":14937,"mankind":14938,"seller":14939,"uttar":14940,"loch":14941,"nadia":14942,"stroking":14943,"exposing":14944,"##hd":14945,"fertile":14946,"ancestral":14947,"instituted":14948,"##has":14949,"noises":14950,"prophecy":14951,"taxation":14952,"eminent":14953,"vivid":14954,"pol":14955,"##bol":14956,"dart":14957,"indirect":14958,"multimedia":14959,"notebook":14960,"upside":14961,"displaying":14962,"adrenaline":14963,"referenced":14964,"geometric":14965,"##iving":14966,"progression":14967,"##ddy":14968,"blunt":14969,"announce":14970,"##far":14971,"implementing":14972,"##lav":14973,"aggression":14974,"liaison":14975,"cooler":14976,"cares":14977,"headache":14978,"plantations":14979,"gorge":14980,"dots":14981,"impulse":14982,"thickness":14983,"ashamed":14984,"averaging":14985,"kathy":14986,"obligation":14987,"precursor":14988,"137":14989,"fowler":14990,"symmetry":14991,"thee":14992,"225":14993,"hears":14994,"##rai":14995,"undergoing":14996,"ads":14997,"butcher":14998,"bowler":14999,"##lip":15000,"cigarettes":15001,"subscription":15002,"goodness":15003,"##ically":15004,"browne":15005,"##hos":15006,"##tech":15007,"kyoto":15008,"donor":15009,"##erty":15010,"damaging":15011,"friction":15012,"drifting":15013,"expeditions":15014,"hardened":15015,"prostitution":15016,"152":15017,"fauna":15018,"blankets":15019,"claw":15020,"tossing":15021,"snarled":15022,"butterflies":15023,"recruits":15024,"investigative":15025,"coated":15026,"healed":15027,"138":15028,"communal":15029,"hai":15030,"xiii":15031,"academics":15032,"boone":15033,"psychologist":15034,"restless":15035,"lahore":15036,"stephens":15037,"mba":15038,"brendan":15039,"foreigners":15040,"printer":15041,"##pc":15042,"ached":15043,"explode":15044,"27th":15045,"deed":15046,"scratched":15047,"dared":15048,"##pole":15049,"cardiac":15050,"1780":15051,"okinawa":15052,"proto":15053,"commando":15054,"compelled":15055,"oddly":15056,"electrons":15057,"##base":15058,"replica":15059,"thanksgiving":15060,"##rist":15061,"sheila":15062,"deliberate":15063,"stafford":15064,"tidal":15065,"representations":15066,"hercules":15067,"ou":15068,"##path":15069,"##iated":15070,"kidnapping":15071,"lenses":15072,"##tling":15073,"deficit":15074,"samoa":15075,"mouths":15076,"consuming":15077,"computational":15078,"maze":15079,"granting":15080,"smirk":15081,"razor":15082,"fixture":15083,"ideals":15084,"inviting":15085,"aiden":15086,"nominal":15087,"##vs":15088,"issuing":15089,"julio":15090,"pitt":15091,"ramsey":15092,"docks":15093,"##oss":15094,"exhaust":15095,"##owed":15096,"bavarian":15097,"draped":15098,"anterior":15099,"mating":15100,"ethiopian":15101,"explores":15102,"noticing":15103,"##nton":15104,"discarded":15105,"convenience":15106,"hoffman":15107,"endowment":15108,"beasts":15109,"cartridge":15110,"mormon":15111,"paternal":15112,"probe":15113,"sleeves":15114,"interfere":15115,"lump":15116,"deadline":15117,"##rail":15118,"jenks":15119,"bulldogs":15120,"scrap":15121,"alternating":15122,"justified":15123,"reproductive":15124,"nam":15125,"seize":15126,"descending":15127,"secretariat":15128,"kirby":15129,"coupe":15130,"grouped":15131,"smash":15132,"panther":15133,"sedan":15134,"tapping":15135,"##18":15136,"lola":15137,"cheer":15138,"germanic":15139,"unfortunate":15140,"##eter":15141,"unrelated":15142,"##fan":15143,"subordinate":15144,"##sdale":15145,"suzanne":15146,"advertisement":15147,"##ility":15148,"horsepower":15149,"##lda":15150,"cautiously":15151,"discourse":15152,"luigi":15153,"##mans":15154,"##fields":15155,"noun":15156,"prevalent":15157,"mao":15158,"schneider":15159,"everett":15160,"surround":15161,"governorate":15162,"kira":15163,"##avia":15164,"westward":15165,"##take":15166,"misty":15167,"rails":15168,"sustainability":15169,"134":15170,"unused":15171,"##rating":15172,"packs":15173,"toast":15174,"unwilling":15175,"regulate":15176,"thy":15177,"suffrage":15178,"nile":15179,"awe":15180,"assam":15181,"definitions":15182,"travelers":15183,"affordable":15184,"##rb":15185,"conferred":15186,"sells":15187,"undefeated":15188,"beneficial":15189,"torso":15190,"basal":15191,"repeating":15192,"remixes":15193,"##pass":15194,"bahrain":15195,"cables":15196,"fang":15197,"##itated":15198,"excavated":15199,"numbering":15200,"statutory":15201,"##rey":15202,"deluxe":15203,"##lian":15204,"forested":15205,"ramirez":15206,"derbyshire":15207,"zeus":15208,"slamming":15209,"transfers":15210,"astronomer":15211,"banana":15212,"lottery":15213,"berg":15214,"histories":15215,"bamboo":15216,"##uchi":15217,"resurrection":15218,"posterior":15219,"bowls":15220,"vaguely":15221,"##thi":15222,"thou":15223,"preserving":15224,"tensed":15225,"offence":15226,"##inas":15227,"meyrick":15228,"callum":15229,"ridden":15230,"watt":15231,"langdon":15232,"tying":15233,"lowland":15234,"snorted":15235,"daring":15236,"truman":15237,"##hale":15238,"##girl":15239,"aura":15240,"overly":15241,"filing":15242,"weighing":15243,"goa":15244,"infections":15245,"philanthropist":15246,"saunders":15247,"eponymous":15248,"##owski":15249,"latitude":15250,"perspectives":15251,"reviewing":15252,"mets":15253,"commandant":15254,"radial":15255,"##kha":15256,"flashlight":15257,"reliability":15258,"koch":15259,"vowels":15260,"amazed":15261,"ada":15262,"elaine":15263,"supper":15264,"##rth":15265,"##encies":15266,"predator":15267,"debated":15268,"soviets":15269,"cola":15270,"##boards":15271,"##nah":15272,"compartment":15273,"crooked":15274,"arbitrary":15275,"fourteenth":15276,"##ctive":15277,"havana":15278,"majors":15279,"steelers":15280,"clips":15281,"profitable":15282,"ambush":15283,"exited":15284,"packers":15285,"##tile":15286,"nude":15287,"cracks":15288,"fungi":15289,"##е":15290,"limb":15291,"trousers":15292,"josie":15293,"shelby":15294,"tens":15295,"frederic":15296,"##ος":15297,"definite":15298,"smoothly":15299,"constellation":15300,"insult":15301,"baton":15302,"discs":15303,"lingering":15304,"##nco":15305,"conclusions":15306,"lent":15307,"staging":15308,"becker":15309,"grandpa":15310,"shaky":15311,"##tron":15312,"einstein":15313,"obstacles":15314,"sk":15315,"adverse":15316,"elle":15317,"economically":15318,"##moto":15319,"mccartney":15320,"thor":15321,"dismissal":15322,"motions":15323,"readings":15324,"nostrils":15325,"treatise":15326,"##pace":15327,"squeezing":15328,"evidently":15329,"prolonged":15330,"1783":15331,"venezuelan":15332,"je":15333,"marguerite":15334,"beirut":15335,"takeover":15336,"shareholders":15337,"##vent":15338,"denise":15339,"digit":15340,"airplay":15341,"norse":15342,"##bbling":15343,"imaginary":15344,"pills":15345,"hubert":15346,"blaze":15347,"vacated":15348,"eliminating":15349,"##ello":15350,"vine":15351,"mansfield":15352,"##tty":15353,"retrospective":15354,"barrow":15355,"borne":15356,"clutch":15357,"bail":15358,"forensic":15359,"weaving":15360,"##nett":15361,"##witz":15362,"desktop":15363,"citadel":15364,"promotions":15365,"worrying":15366,"dorset":15367,"ieee":15368,"subdivided":15369,"##iating":15370,"manned":15371,"expeditionary":15372,"pickup":15373,"synod":15374,"chuckle":15375,"185":15376,"barney":15377,"##rz":15378,"##ffin":15379,"functionality":15380,"karachi":15381,"litigation":15382,"meanings":15383,"uc":15384,"lick":15385,"turbo":15386,"anders":15387,"##ffed":15388,"execute":15389,"curl":15390,"oppose":15391,"ankles":15392,"typhoon":15393,"##د":15394,"##ache":15395,"##asia":15396,"linguistics":15397,"compassion":15398,"pressures":15399,"grazing":15400,"perfection":15401,"##iting":15402,"immunity":15403,"monopoly":15404,"muddy":15405,"backgrounds":15406,"136":15407,"namibia":15408,"francesca":15409,"monitors":15410,"attracting":15411,"stunt":15412,"tuition":15413,"##ии":15414,"vegetable":15415,"##mates":15416,"##quent":15417,"mgm":15418,"jen":15419,"complexes":15420,"forts":15421,"##ond":15422,"cellar":15423,"bites":15424,"seventeenth":15425,"royals":15426,"flemish":15427,"failures":15428,"mast":15429,"charities":15430,"##cular":15431,"peruvian":15432,"capitals":15433,"macmillan":15434,"ipswich":15435,"outward":15436,"frigate":15437,"postgraduate":15438,"folds":15439,"employing":15440,"##ouse":15441,"concurrently":15442,"fiery":15443,"##tai":15444,"contingent":15445,"nightmares":15446,"monumental":15447,"nicaragua":15448,"##kowski":15449,"lizard":15450,"mal":15451,"fielding":15452,"gig":15453,"reject":15454,"##pad":15455,"harding":15456,"##ipe":15457,"coastline":15458,"##cin":15459,"##nos":15460,"beethoven":15461,"humphrey":15462,"innovations":15463,"##tam":15464,"##nge":15465,"norris":15466,"doris":15467,"solicitor":15468,"huang":15469,"obey":15470,"141":15471,"##lc":15472,"niagara":15473,"##tton":15474,"shelves":15475,"aug":15476,"bourbon":15477,"curry":15478,"nightclub":15479,"specifications":15480,"hilton":15481,"##ndo":15482,"centennial":15483,"dispersed":15484,"worm":15485,"neglected":15486,"briggs":15487,"sm":15488,"font":15489,"kuala":15490,"uneasy":15491,"plc":15492,"##nstein":15493,"##bound":15494,"##aking":15495,"##burgh":15496,"awaiting":15497,"pronunciation":15498,"##bbed":15499,"##quest":15500,"eh":15501,"optimal":15502,"zhu":15503,"raped":15504,"greens":15505,"presided":15506,"brenda":15507,"worries":15508,"##life":15509,"venetian":15510,"marxist":15511,"turnout":15512,"##lius":15513,"refined":15514,"braced":15515,"sins":15516,"grasped":15517,"sunderland":15518,"nickel":15519,"speculated":15520,"lowell":15521,"cyrillic":15522,"communism":15523,"fundraising":15524,"resembling":15525,"colonists":15526,"mutant":15527,"freddie":15528,"usc":15529,"##mos":15530,"gratitude":15531,"##run":15532,"mural":15533,"##lous":15534,"chemist":15535,"wi":15536,"reminds":15537,"28th":15538,"steals":15539,"tess":15540,"pietro":15541,"##ingen":15542,"promoter":15543,"ri":15544,"microphone":15545,"honoured":15546,"rai":15547,"sant":15548,"##qui":15549,"feather":15550,"##nson":15551,"burlington":15552,"kurdish":15553,"terrorists":15554,"deborah":15555,"sickness":15556,"##wed":15557,"##eet":15558,"hazard":15559,"irritated":15560,"desperation":15561,"veil":15562,"clarity":15563,"##rik":15564,"jewels":15565,"xv":15566,"##gged":15567,"##ows":15568,"##cup":15569,"berkshire":15570,"unfair":15571,"mysteries":15572,"orchid":15573,"winced":15574,"exhaustion":15575,"renovations":15576,"stranded":15577,"obe":15578,"infinity":15579,"##nies":15580,"adapt":15581,"redevelopment":15582,"thanked":15583,"registry":15584,"olga":15585,"domingo":15586,"noir":15587,"tudor":15588,"ole":15589,"##atus":15590,"commenting":15591,"behaviors":15592,"##ais":15593,"crisp":15594,"pauline":15595,"probable":15596,"stirling":15597,"wigan":15598,"##bian":15599,"paralympics":15600,"panting":15601,"surpassed":15602,"##rew":15603,"luca":15604,"barred":15605,"pony":15606,"famed":15607,"##sters":15608,"cassandra":15609,"waiter":15610,"carolyn":15611,"exported":15612,"##orted":15613,"andres":15614,"destructive":15615,"deeds":15616,"jonah":15617,"castles":15618,"vacancy":15619,"suv":15620,"##glass":15621,"1788":15622,"orchard":15623,"yep":15624,"famine":15625,"belarusian":15626,"sprang":15627,"##forth":15628,"skinny":15629,"##mis":15630,"administrators":15631,"rotterdam":15632,"zambia":15633,"zhao":15634,"boiler":15635,"discoveries":15636,"##ride":15637,"##physics":15638,"lucius":15639,"disappointing":15640,"outreach":15641,"spoon":15642,"##frame":15643,"qualifications":15644,"unanimously":15645,"enjoys":15646,"regency":15647,"##iidae":15648,"stade":15649,"realism":15650,"veterinary":15651,"rodgers":15652,"dump":15653,"alain":15654,"chestnut":15655,"castile":15656,"censorship":15657,"rumble":15658,"gibbs":15659,"##itor":15660,"communion":15661,"reggae":15662,"inactivated":15663,"logs":15664,"loads":15665,"##houses":15666,"homosexual":15667,"##iano":15668,"ale":15669,"informs":15670,"##cas":15671,"phrases":15672,"plaster":15673,"linebacker":15674,"ambrose":15675,"kaiser":15676,"fascinated":15677,"850":15678,"limerick":15679,"recruitment":15680,"forge":15681,"mastered":15682,"##nding":15683,"leinster":15684,"rooted":15685,"threaten":15686,"##strom":15687,"borneo":15688,"##hes":15689,"suggestions":15690,"scholarships":15691,"propeller":15692,"documentaries":15693,"patronage":15694,"coats":15695,"constructing":15696,"invest":15697,"neurons":15698,"comet":15699,"entirety":15700,"shouts":15701,"identities":15702,"annoying":15703,"unchanged":15704,"wary":15705,"##antly":15706,"##ogy":15707,"neat":15708,"oversight":15709,"##kos":15710,"phillies":15711,"replay":15712,"constance":15713,"##kka":15714,"incarnation":15715,"humble":15716,"skies":15717,"minus":15718,"##acy":15719,"smithsonian":15720,"##chel":15721,"guerrilla":15722,"jar":15723,"cadets":15724,"##plate":15725,"surplus":15726,"audit":15727,"##aru":15728,"cracking":15729,"joanna":15730,"louisa":15731,"pacing":15732,"##lights":15733,"intentionally":15734,"##iri":15735,"diner":15736,"nwa":15737,"imprint":15738,"australians":15739,"tong":15740,"unprecedented":15741,"bunker":15742,"naive":15743,"specialists":15744,"ark":15745,"nichols":15746,"railing":15747,"leaked":15748,"pedal":15749,"##uka":15750,"shrub":15751,"longing":15752,"roofs":15753,"v8":15754,"captains":15755,"neural":15756,"tuned":15757,"##ntal":15758,"##jet":15759,"emission":15760,"medina":15761,"frantic":15762,"codex":15763,"definitive":15764,"sid":15765,"abolition":15766,"intensified":15767,"stocks":15768,"enrique":15769,"sustain":15770,"genoa":15771,"oxide":15772,"##written":15773,"clues":15774,"cha":15775,"##gers":15776,"tributaries":15777,"fragment":15778,"venom":15779,"##rity":15780,"##ente":15781,"##sca":15782,"muffled":15783,"vain":15784,"sire":15785,"laos":15786,"##ingly":15787,"##hana":15788,"hastily":15789,"snapping":15790,"surfaced":15791,"sentiment":15792,"motive":15793,"##oft":15794,"contests":15795,"approximate":15796,"mesa":15797,"luckily":15798,"dinosaur":15799,"exchanges":15800,"propelled":15801,"accord":15802,"bourne":15803,"relieve":15804,"tow":15805,"masks":15806,"offended":15807,"##ues":15808,"cynthia":15809,"##mmer":15810,"rains":15811,"bartender":15812,"zinc":15813,"reviewers":15814,"lois":15815,"##sai":15816,"legged":15817,"arrogant":15818,"rafe":15819,"rosie":15820,"comprise":15821,"handicap":15822,"blockade":15823,"inlet":15824,"lagoon":15825,"copied":15826,"drilling":15827,"shelley":15828,"petals":15829,"##inian":15830,"mandarin":15831,"obsolete":15832,"##inated":15833,"onward":15834,"arguably":15835,"productivity":15836,"cindy":15837,"praising":15838,"seldom":15839,"busch":15840,"discusses":15841,"raleigh":15842,"shortage":15843,"ranged":15844,"stanton":15845,"encouragement":15846,"firstly":15847,"conceded":15848,"overs":15849,"temporal":15850,"##uke":15851,"cbe":15852,"##bos":15853,"woo":15854,"certainty":15855,"pumps":15856,"##pton":15857,"stalked":15858,"##uli":15859,"lizzie":15860,"periodic":15861,"thieves":15862,"weaker":15863,"##night":15864,"gases":15865,"shoving":15866,"chooses":15867,"wc":15868,"##chemical":15869,"prompting":15870,"weights":15871,"##kill":15872,"robust":15873,"flanked":15874,"sticky":15875,"hu":15876,"tuberculosis":15877,"##eb":15878,"##eal":15879,"christchurch":15880,"resembled":15881,"wallet":15882,"reese":15883,"inappropriate":15884,"pictured":15885,"distract":15886,"fixing":15887,"fiddle":15888,"giggled":15889,"burger":15890,"heirs":15891,"hairy":15892,"mechanic":15893,"torque":15894,"apache":15895,"obsessed":15896,"chiefly":15897,"cheng":15898,"logging":15899,"##tag":15900,"extracted":15901,"meaningful":15902,"numb":15903,"##vsky":15904,"gloucestershire":15905,"reminding":15906,"##bay":15907,"unite":15908,"##lit":15909,"breeds":15910,"diminished":15911,"clown":15912,"glove":15913,"1860s":15914,"##ن":15915,"##ug":15916,"archibald":15917,"focal":15918,"freelance":15919,"sliced":15920,"depiction":15921,"##yk":15922,"organism":15923,"switches":15924,"sights":15925,"stray":15926,"crawling":15927,"##ril":15928,"lever":15929,"leningrad":15930,"interpretations":15931,"loops":15932,"anytime":15933,"reel":15934,"alicia":15935,"delighted":15936,"##ech":15937,"inhaled":15938,"xiv":15939,"suitcase":15940,"bernie":15941,"vega":15942,"licenses":15943,"northampton":15944,"exclusion":15945,"induction":15946,"monasteries":15947,"racecourse":15948,"homosexuality":15949,"##right":15950,"##sfield":15951,"##rky":15952,"dimitri":15953,"michele":15954,"alternatives":15955,"ions":15956,"commentators":15957,"genuinely":15958,"objected":15959,"pork":15960,"hospitality":15961,"fencing":15962,"stephan":15963,"warships":15964,"peripheral":15965,"wit":15966,"drunken":15967,"wrinkled":15968,"quentin":15969,"spends":15970,"departing":15971,"chung":15972,"numerical":15973,"spokesperson":15974,"##zone":15975,"johannesburg":15976,"caliber":15977,"killers":15978,"##udge":15979,"assumes":15980,"neatly":15981,"demographic":15982,"abigail":15983,"bloc":15984,"##vel":15985,"mounting":15986,"##lain":15987,"bentley":15988,"slightest":15989,"xu":15990,"recipients":15991,"##jk":15992,"merlin":15993,"##writer":15994,"seniors":15995,"prisons":15996,"blinking":15997,"hindwings":15998,"flickered":15999,"kappa":16000,"##hel":16001,"80s":16002,"strengthening":16003,"appealing":16004,"brewing":16005,"gypsy":16006,"mali":16007,"lashes":16008,"hulk":16009,"unpleasant":16010,"harassment":16011,"bio":16012,"treaties":16013,"predict":16014,"instrumentation":16015,"pulp":16016,"troupe":16017,"boiling":16018,"mantle":16019,"##ffe":16020,"ins":16021,"##vn":16022,"dividing":16023,"handles":16024,"verbs":16025,"##onal":16026,"coconut":16027,"senegal":16028,"340":16029,"thorough":16030,"gum":16031,"momentarily":16032,"##sto":16033,"cocaine":16034,"panicked":16035,"destined":16036,"##turing":16037,"teatro":16038,"denying":16039,"weary":16040,"captained":16041,"mans":16042,"##hawks":16043,"##code":16044,"wakefield":16045,"bollywood":16046,"thankfully":16047,"##16":16048,"cyril":16049,"##wu":16050,"amendments":16051,"##bahn":16052,"consultation":16053,"stud":16054,"reflections":16055,"kindness":16056,"1787":16057,"internally":16058,"##ovo":16059,"tex":16060,"mosaic":16061,"distribute":16062,"paddy":16063,"seeming":16064,"143":16065,"##hic":16066,"piers":16067,"##15":16068,"##mura":16069,"##verse":16070,"popularly":16071,"winger":16072,"kang":16073,"sentinel":16074,"mccoy":16075,"##anza":16076,"covenant":16077,"##bag":16078,"verge":16079,"fireworks":16080,"suppress":16081,"thrilled":16082,"dominate":16083,"##jar":16084,"swansea":16085,"##60":16086,"142":16087,"reconciliation":16088,"##ndi":16089,"stiffened":16090,"cue":16091,"dorian":16092,"##uf":16093,"damascus":16094,"amor":16095,"ida":16096,"foremost":16097,"##aga":16098,"porsche":16099,"unseen":16100,"dir":16101,"##had":16102,"##azi":16103,"stony":16104,"lexi":16105,"melodies":16106,"##nko":16107,"angular":16108,"integer":16109,"podcast":16110,"ants":16111,"inherent":16112,"jaws":16113,"justify":16114,"persona":16115,"##olved":16116,"josephine":16117,"##nr":16118,"##ressed":16119,"customary":16120,"flashes":16121,"gala":16122,"cyrus":16123,"glaring":16124,"backyard":16125,"ariel":16126,"physiology":16127,"greenland":16128,"html":16129,"stir":16130,"avon":16131,"atletico":16132,"finch":16133,"methodology":16134,"ked":16135,"##lent":16136,"mas":16137,"catholicism":16138,"townsend":16139,"branding":16140,"quincy":16141,"fits":16142,"containers":16143,"1777":16144,"ashore":16145,"aragon":16146,"##19":16147,"forearm":16148,"poisoning":16149,"##sd":16150,"adopting":16151,"conquer":16152,"grinding":16153,"amnesty":16154,"keller":16155,"finances":16156,"evaluate":16157,"forged":16158,"lankan":16159,"instincts":16160,"##uto":16161,"guam":16162,"bosnian":16163,"photographed":16164,"workplace":16165,"desirable":16166,"protector":16167,"##dog":16168,"allocation":16169,"intently":16170,"encourages":16171,"willy":16172,"##sten":16173,"bodyguard":16174,"electro":16175,"brighter":16176,"##ν":16177,"bihar":16178,"##chev":16179,"lasts":16180,"opener":16181,"amphibious":16182,"sal":16183,"verde":16184,"arte":16185,"##cope":16186,"captivity":16187,"vocabulary":16188,"yields":16189,"##tted":16190,"agreeing":16191,"desmond":16192,"pioneered":16193,"##chus":16194,"strap":16195,"campaigned":16196,"railroads":16197,"##ович":16198,"emblem":16199,"##dre":16200,"stormed":16201,"501":16202,"##ulous":16203,"marijuana":16204,"northumberland":16205,"##gn":16206,"##nath":16207,"bowen":16208,"landmarks":16209,"beaumont":16210,"##qua":16211,"danube":16212,"##bler":16213,"attorneys":16214,"th":16215,"ge":16216,"flyers":16217,"critique":16218,"villains":16219,"cass":16220,"mutation":16221,"acc":16222,"##0s":16223,"colombo":16224,"mckay":16225,"motif":16226,"sampling":16227,"concluding":16228,"syndicate":16229,"##rell":16230,"neon":16231,"stables":16232,"ds":16233,"warnings":16234,"clint":16235,"mourning":16236,"wilkinson":16237,"##tated":16238,"merrill":16239,"leopard":16240,"evenings":16241,"exhaled":16242,"emil":16243,"sonia":16244,"ezra":16245,"discrete":16246,"stove":16247,"farrell":16248,"fifteenth":16249,"prescribed":16250,"superhero":16251,"##rier":16252,"worms":16253,"helm":16254,"wren":16255,"##duction":16256,"##hc":16257,"expo":16258,"##rator":16259,"hq":16260,"unfamiliar":16261,"antony":16262,"prevents":16263,"acceleration":16264,"fiercely":16265,"mari":16266,"painfully":16267,"calculations":16268,"cheaper":16269,"ign":16270,"clifton":16271,"irvine":16272,"davenport":16273,"mozambique":16274,"##np":16275,"pierced":16276,"##evich":16277,"wonders":16278,"##wig":16279,"##cate":16280,"##iling":16281,"crusade":16282,"ware":16283,"##uel":16284,"enzymes":16285,"reasonably":16286,"mls":16287,"##coe":16288,"mater":16289,"ambition":16290,"bunny":16291,"eliot":16292,"kernel":16293,"##fin":16294,"asphalt":16295,"headmaster":16296,"torah":16297,"aden":16298,"lush":16299,"pins":16300,"waived":16301,"##care":16302,"##yas":16303,"joao":16304,"substrate":16305,"enforce":16306,"##grad":16307,"##ules":16308,"alvarez":16309,"selections":16310,"epidemic":16311,"tempted":16312,"##bit":16313,"bremen":16314,"translates":16315,"ensured":16316,"waterfront":16317,"29th":16318,"forrest":16319,"manny":16320,"malone":16321,"kramer":16322,"reigning":16323,"cookies":16324,"simpler":16325,"absorption":16326,"205":16327,"engraved":16328,"##ffy":16329,"evaluated":16330,"1778":16331,"haze":16332,"146":16333,"comforting":16334,"crossover":16335,"##abe":16336,"thorn":16337,"##rift":16338,"##imo":16339,"##pop":16340,"suppression":16341,"fatigue":16342,"cutter":16343,"##tr":16344,"201":16345,"wurttemberg":16346,"##orf":16347,"enforced":16348,"hovering":16349,"proprietary":16350,"gb":16351,"samurai":16352,"syllable":16353,"ascent":16354,"lacey":16355,"tick":16356,"lars":16357,"tractor":16358,"merchandise":16359,"rep":16360,"bouncing":16361,"defendants":16362,"##yre":16363,"huntington":16364,"##ground":16365,"##oko":16366,"standardized":16367,"##hor":16368,"##hima":16369,"assassinated":16370,"nu":16371,"predecessors":16372,"rainy":16373,"liar":16374,"assurance":16375,"lyrical":16376,"##uga":16377,"secondly":16378,"flattened":16379,"ios":16380,"parameter":16381,"undercover":16382,"##mity":16383,"bordeaux":16384,"punish":16385,"ridges":16386,"markers":16387,"exodus":16388,"inactive":16389,"hesitate":16390,"debbie":16391,"nyc":16392,"pledge":16393,"savoy":16394,"nagar":16395,"offset":16396,"organist":16397,"##tium":16398,"hesse":16399,"marin":16400,"converting":16401,"##iver":16402,"diagram":16403,"propulsion":16404,"pu":16405,"validity":16406,"reverted":16407,"supportive":16408,"##dc":16409,"ministries":16410,"clans":16411,"responds":16412,"proclamation":16413,"##inae":16414,"##ø":16415,"##rea":16416,"ein":16417,"pleading":16418,"patriot":16419,"sf":16420,"birch":16421,"islanders":16422,"strauss":16423,"hates":16424,"##dh":16425,"brandenburg":16426,"concession":16427,"rd":16428,"##ob":16429,"1900s":16430,"killings":16431,"textbook":16432,"antiquity":16433,"cinematography":16434,"wharf":16435,"embarrassing":16436,"setup":16437,"creed":16438,"farmland":16439,"inequality":16440,"centred":16441,"signatures":16442,"fallon":16443,"370":16444,"##ingham":16445,"##uts":16446,"ceylon":16447,"gazing":16448,"directive":16449,"laurie":16450,"##tern":16451,"globally":16452,"##uated":16453,"##dent":16454,"allah":16455,"excavation":16456,"threads":16457,"##cross":16458,"148":16459,"frantically":16460,"icc":16461,"utilize":16462,"determines":16463,"respiratory":16464,"thoughtful":16465,"receptions":16466,"##dicate":16467,"merging":16468,"chandra":16469,"seine":16470,"147":16471,"builders":16472,"builds":16473,"diagnostic":16474,"dev":16475,"visibility":16476,"goddamn":16477,"analyses":16478,"dhaka":16479,"cho":16480,"proves":16481,"chancel":16482,"concurrent":16483,"curiously":16484,"canadians":16485,"pumped":16486,"restoring":16487,"1850s":16488,"turtles":16489,"jaguar":16490,"sinister":16491,"spinal":16492,"traction":16493,"declan":16494,"vows":16495,"1784":16496,"glowed":16497,"capitalism":16498,"swirling":16499,"install":16500,"universidad":16501,"##lder":16502,"##oat":16503,"soloist":16504,"##genic":16505,"##oor":16506,"coincidence":16507,"beginnings":16508,"nissan":16509,"dip":16510,"resorts":16511,"caucasus":16512,"combustion":16513,"infectious":16514,"##eno":16515,"pigeon":16516,"serpent":16517,"##itating":16518,"conclude":16519,"masked":16520,"salad":16521,"jew":16522,"##gr":16523,"surreal":16524,"toni":16525,"##wc":16526,"harmonica":16527,"151":16528,"##gins":16529,"##etic":16530,"##coat":16531,"fishermen":16532,"intending":16533,"bravery":16534,"##wave":16535,"klaus":16536,"titan":16537,"wembley":16538,"taiwanese":16539,"ransom":16540,"40th":16541,"incorrect":16542,"hussein":16543,"eyelids":16544,"jp":16545,"cooke":16546,"dramas":16547,"utilities":16548,"##etta":16549,"##print":16550,"eisenhower":16551,"principally":16552,"granada":16553,"lana":16554,"##rak":16555,"openings":16556,"concord":16557,"##bl":16558,"bethany":16559,"connie":16560,"morality":16561,"sega":16562,"##mons":16563,"##nard":16564,"earnings":16565,"##kara":16566,"##cine":16567,"wii":16568,"communes":16569,"##rel":16570,"coma":16571,"composing":16572,"softened":16573,"severed":16574,"grapes":16575,"##17":16576,"nguyen":16577,"analyzed":16578,"warlord":16579,"hubbard":16580,"heavenly":16581,"behave":16582,"slovenian":16583,"##hit":16584,"##ony":16585,"hailed":16586,"filmmakers":16587,"trance":16588,"caldwell":16589,"skye":16590,"unrest":16591,"coward":16592,"likelihood":16593,"##aging":16594,"bern":16595,"sci":16596,"taliban":16597,"honolulu":16598,"propose":16599,"##wang":16600,"1700":16601,"browser":16602,"imagining":16603,"cobra":16604,"contributes":16605,"dukes":16606,"instinctively":16607,"conan":16608,"violinist":16609,"##ores":16610,"accessories":16611,"gradual":16612,"##amp":16613,"quotes":16614,"sioux":16615,"##dating":16616,"undertake":16617,"intercepted":16618,"sparkling":16619,"compressed":16620,"139":16621,"fungus":16622,"tombs":16623,"haley":16624,"imposing":16625,"rests":16626,"degradation":16627,"lincolnshire":16628,"retailers":16629,"wetlands":16630,"tulsa":16631,"distributor":16632,"dungeon":16633,"nun":16634,"greenhouse":16635,"convey":16636,"atlantis":16637,"aft":16638,"exits":16639,"oman":16640,"dresser":16641,"lyons":16642,"##sti":16643,"joking":16644,"eddy":16645,"judgement":16646,"omitted":16647,"digits":16648,"##cts":16649,"##game":16650,"juniors":16651,"##rae":16652,"cents":16653,"stricken":16654,"une":16655,"##ngo":16656,"wizards":16657,"weir":16658,"breton":16659,"nan":16660,"technician":16661,"fibers":16662,"liking":16663,"royalty":16664,"##cca":16665,"154":16666,"persia":16667,"terribly":16668,"magician":16669,"##rable":16670,"##unt":16671,"vance":16672,"cafeteria":16673,"booker":16674,"camille":16675,"warmer":16676,"##static":16677,"consume":16678,"cavern":16679,"gaps":16680,"compass":16681,"contemporaries":16682,"foyer":16683,"soothing":16684,"graveyard":16685,"maj":16686,"plunged":16687,"blush":16688,"##wear":16689,"cascade":16690,"demonstrates":16691,"ordinance":16692,"##nov":16693,"boyle":16694,"##lana":16695,"rockefeller":16696,"shaken":16697,"banjo":16698,"izzy":16699,"##ense":16700,"breathless":16701,"vines":16702,"##32":16703,"##eman":16704,"alterations":16705,"chromosome":16706,"dwellings":16707,"feudal":16708,"mole":16709,"153":16710,"catalonia":16711,"relics":16712,"tenant":16713,"mandated":16714,"##fm":16715,"fridge":16716,"hats":16717,"honesty":16718,"patented":16719,"raul":16720,"heap":16721,"cruisers":16722,"accusing":16723,"enlightenment":16724,"infants":16725,"wherein":16726,"chatham":16727,"contractors":16728,"zen":16729,"affinity":16730,"hc":16731,"osborne":16732,"piston":16733,"156":16734,"traps":16735,"maturity":16736,"##rana":16737,"lagos":16738,"##zal":16739,"peering":16740,"##nay":16741,"attendant":16742,"dealers":16743,"protocols":16744,"subset":16745,"prospects":16746,"biographical":16747,"##cre":16748,"artery":16749,"##zers":16750,"insignia":16751,"nuns":16752,"endured":16753,"##eration":16754,"recommend":16755,"schwartz":16756,"serbs":16757,"berger":16758,"cromwell":16759,"crossroads":16760,"##ctor":16761,"enduring":16762,"clasped":16763,"grounded":16764,"##bine":16765,"marseille":16766,"twitched":16767,"abel":16768,"choke":16769,"https":16770,"catalyst":16771,"moldova":16772,"italians":16773,"##tist":16774,"disastrous":16775,"wee":16776,"##oured":16777,"##nti":16778,"wwf":16779,"nope":16780,"##piration":16781,"##asa":16782,"expresses":16783,"thumbs":16784,"167":16785,"##nza":16786,"coca":16787,"1781":16788,"cheating":16789,"##ption":16790,"skipped":16791,"sensory":16792,"heidelberg":16793,"spies":16794,"satan":16795,"dangers":16796,"semifinal":16797,"202":16798,"bohemia":16799,"whitish":16800,"confusing":16801,"shipbuilding":16802,"relies":16803,"surgeons":16804,"landings":16805,"ravi":16806,"baku":16807,"moor":16808,"suffix":16809,"alejandro":16810,"##yana":16811,"litre":16812,"upheld":16813,"##unk":16814,"rajasthan":16815,"##rek":16816,"coaster":16817,"insists":16818,"posture":16819,"scenarios":16820,"etienne":16821,"favoured":16822,"appoint":16823,"transgender":16824,"elephants":16825,"poked":16826,"greenwood":16827,"defences":16828,"fulfilled":16829,"militant":16830,"somali":16831,"1758":16832,"chalk":16833,"potent":16834,"##ucci":16835,"migrants":16836,"wink":16837,"assistants":16838,"nos":16839,"restriction":16840,"activism":16841,"niger":16842,"##ario":16843,"colon":16844,"shaun":16845,"##sat":16846,"daphne":16847,"##erated":16848,"swam":16849,"congregations":16850,"reprise":16851,"considerations":16852,"magnet":16853,"playable":16854,"xvi":16855,"##р":16856,"overthrow":16857,"tobias":16858,"knob":16859,"chavez":16860,"coding":16861,"##mers":16862,"propped":16863,"katrina":16864,"orient":16865,"newcomer":16866,"##suke":16867,"temperate":16868,"##pool":16869,"farmhouse":16870,"interrogation":16871,"##vd":16872,"committing":16873,"##vert":16874,"forthcoming":16875,"strawberry":16876,"joaquin":16877,"macau":16878,"ponds":16879,"shocking":16880,"siberia":16881,"##cellular":16882,"chant":16883,"contributors":16884,"##nant":16885,"##ologists":16886,"sped":16887,"absorb":16888,"hail":16889,"1782":16890,"spared":16891,"##hore":16892,"barbados":16893,"karate":16894,"opus":16895,"originates":16896,"saul":16897,"##xie":16898,"evergreen":16899,"leaped":16900,"##rock":16901,"correlation":16902,"exaggerated":16903,"weekday":16904,"unification":16905,"bump":16906,"tracing":16907,"brig":16908,"afb":16909,"pathways":16910,"utilizing":16911,"##ners":16912,"mod":16913,"mb":16914,"disturbance":16915,"kneeling":16916,"##stad":16917,"##guchi":16918,"100th":16919,"pune":16920,"##thy":16921,"decreasing":16922,"168":16923,"manipulation":16924,"miriam":16925,"academia":16926,"ecosystem":16927,"occupational":16928,"rbi":16929,"##lem":16930,"rift":16931,"##14":16932,"rotary":16933,"stacked":16934,"incorporation":16935,"awakening":16936,"generators":16937,"guerrero":16938,"racist":16939,"##omy":16940,"cyber":16941,"derivatives":16942,"culminated":16943,"allie":16944,"annals":16945,"panzer":16946,"sainte":16947,"wikipedia":16948,"pops":16949,"zu":16950,"austro":16951,"##vate":16952,"algerian":16953,"politely":16954,"nicholson":16955,"mornings":16956,"educate":16957,"tastes":16958,"thrill":16959,"dartmouth":16960,"##gating":16961,"db":16962,"##jee":16963,"regan":16964,"differing":16965,"concentrating":16966,"choreography":16967,"divinity":16968,"##media":16969,"pledged":16970,"alexandre":16971,"routing":16972,"gregor":16973,"madeline":16974,"##idal":16975,"apocalypse":16976,"##hora":16977,"gunfire":16978,"culminating":16979,"elves":16980,"fined":16981,"liang":16982,"lam":16983,"programmed":16984,"tar":16985,"guessing":16986,"transparency":16987,"gabrielle":16988,"##gna":16989,"cancellation":16990,"flexibility":16991,"##lining":16992,"accession":16993,"shea":16994,"stronghold":16995,"nets":16996,"specializes":16997,"##rgan":16998,"abused":16999,"hasan":17000,"sgt":17001,"ling":17002,"exceeding":17003,"##₄":17004,"admiration":17005,"supermarket":17006,"##ark":17007,"photographers":17008,"specialised":17009,"tilt":17010,"resonance":17011,"hmm":17012,"perfume":17013,"380":17014,"sami":17015,"threatens":17016,"garland":17017,"botany":17018,"guarding":17019,"boiled":17020,"greet":17021,"puppy":17022,"russo":17023,"supplier":17024,"wilmington":17025,"vibrant":17026,"vijay":17027,"##bius":17028,"paralympic":17029,"grumbled":17030,"paige":17031,"faa":17032,"licking":17033,"margins":17034,"hurricanes":17035,"##gong":17036,"fest":17037,"grenade":17038,"ripping":17039,"##uz":17040,"counseling":17041,"weigh":17042,"##sian":17043,"needles":17044,"wiltshire":17045,"edison":17046,"costly":17047,"##not":17048,"fulton":17049,"tramway":17050,"redesigned":17051,"staffordshire":17052,"cache":17053,"gasping":17054,"watkins":17055,"sleepy":17056,"candidacy":17057,"##group":17058,"monkeys":17059,"timeline":17060,"throbbing":17061,"##bid":17062,"##sos":17063,"berth":17064,"uzbekistan":17065,"vanderbilt":17066,"bothering":17067,"overturned":17068,"ballots":17069,"gem":17070,"##iger":17071,"sunglasses":17072,"subscribers":17073,"hooker":17074,"compelling":17075,"ang":17076,"exceptionally":17077,"saloon":17078,"stab":17079,"##rdi":17080,"carla":17081,"terrifying":17082,"rom":17083,"##vision":17084,"coil":17085,"##oids":17086,"satisfying":17087,"vendors":17088,"31st":17089,"mackay":17090,"deities":17091,"overlooked":17092,"ambient":17093,"bahamas":17094,"felipe":17095,"olympia":17096,"whirled":17097,"botanist":17098,"advertised":17099,"tugging":17100,"##dden":17101,"disciples":17102,"morales":17103,"unionist":17104,"rites":17105,"foley":17106,"morse":17107,"motives":17108,"creepy":17109,"##₀":17110,"soo":17111,"##sz":17112,"bargain":17113,"highness":17114,"frightening":17115,"turnpike":17116,"tory":17117,"reorganization":17118,"##cer":17119,"depict":17120,"biographer":17121,"##walk":17122,"unopposed":17123,"manifesto":17124,"##gles":17125,"institut":17126,"emile":17127,"accidental":17128,"kapoor":17129,"##dam":17130,"kilkenny":17131,"cortex":17132,"lively":17133,"##13":17134,"romanesque":17135,"jain":17136,"shan":17137,"cannons":17138,"##ood":17139,"##ske":17140,"petrol":17141,"echoing":17142,"amalgamated":17143,"disappears":17144,"cautious":17145,"proposes":17146,"sanctions":17147,"trenton":17148,"##ر":17149,"flotilla":17150,"aus":17151,"contempt":17152,"tor":17153,"canary":17154,"cote":17155,"theirs":17156,"##hun":17157,"conceptual":17158,"deleted":17159,"fascinating":17160,"paso":17161,"blazing":17162,"elf":17163,"honourable":17164,"hutchinson":17165,"##eiro":17166,"##outh":17167,"##zin":17168,"surveyor":17169,"tee":17170,"amidst":17171,"wooded":17172,"reissue":17173,"intro":17174,"##ono":17175,"cobb":17176,"shelters":17177,"newsletter":17178,"hanson":17179,"brace":17180,"encoding":17181,"confiscated":17182,"dem":17183,"caravan":17184,"marino":17185,"scroll":17186,"melodic":17187,"cows":17188,"imam":17189,"##adi":17190,"##aneous":17191,"northward":17192,"searches":17193,"biodiversity":17194,"cora":17195,"310":17196,"roaring":17197,"##bers":17198,"connell":17199,"theologian":17200,"halo":17201,"compose":17202,"pathetic":17203,"unmarried":17204,"dynamo":17205,"##oot":17206,"az":17207,"calculation":17208,"toulouse":17209,"deserves":17210,"humour":17211,"nr":17212,"forgiveness":17213,"tam":17214,"undergone":17215,"martyr":17216,"pamela":17217,"myths":17218,"whore":17219,"counselor":17220,"hicks":17221,"290":17222,"heavens":17223,"battleship":17224,"electromagnetic":17225,"##bbs":17226,"stellar":17227,"establishments":17228,"presley":17229,"hopped":17230,"##chin":17231,"temptation":17232,"90s":17233,"wills":17234,"nas":17235,"##yuan":17236,"nhs":17237,"##nya":17238,"seminars":17239,"##yev":17240,"adaptations":17241,"gong":17242,"asher":17243,"lex":17244,"indicator":17245,"sikh":17246,"tobago":17247,"cites":17248,"goin":17249,"##yte":17250,"satirical":17251,"##gies":17252,"characterised":17253,"correspond":17254,"bubbles":17255,"lure":17256,"participates":17257,"##vid":17258,"eruption":17259,"skate":17260,"therapeutic":17261,"1785":17262,"canals":17263,"wholesale":17264,"defaulted":17265,"sac":17266,"460":17267,"petit":17268,"##zzled":17269,"virgil":17270,"leak":17271,"ravens":17272,"256":17273,"portraying":17274,"##yx":17275,"ghetto":17276,"creators":17277,"dams":17278,"portray":17279,"vicente":17280,"##rington":17281,"fae":17282,"namesake":17283,"bounty":17284,"##arium":17285,"joachim":17286,"##ota":17287,"##iser":17288,"aforementioned":17289,"axle":17290,"snout":17291,"depended":17292,"dismantled":17293,"reuben":17294,"480":17295,"##ibly":17296,"gallagher":17297,"##lau":17298,"##pd":17299,"earnest":17300,"##ieu":17301,"##iary":17302,"inflicted":17303,"objections":17304,"##llar":17305,"asa":17306,"gritted":17307,"##athy":17308,"jericho":17309,"##sea":17310,"##was":17311,"flick":17312,"underside":17313,"ceramics":17314,"undead":17315,"substituted":17316,"195":17317,"eastward":17318,"undoubtedly":17319,"wheeled":17320,"chimney":17321,"##iche":17322,"guinness":17323,"cb":17324,"##ager":17325,"siding":17326,"##bell":17327,"traitor":17328,"baptiste":17329,"disguised":17330,"inauguration":17331,"149":17332,"tipperary":17333,"choreographer":17334,"perched":17335,"warmed":17336,"stationary":17337,"eco":17338,"##ike":17339,"##ntes":17340,"bacterial":17341,"##aurus":17342,"flores":17343,"phosphate":17344,"##core":17345,"attacker":17346,"invaders":17347,"alvin":17348,"intersects":17349,"a1":17350,"indirectly":17351,"immigrated":17352,"businessmen":17353,"cornelius":17354,"valves":17355,"narrated":17356,"pill":17357,"sober":17358,"ul":17359,"nationale":17360,"monastic":17361,"applicants":17362,"scenery":17363,"##jack":17364,"161":17365,"motifs":17366,"constitutes":17367,"cpu":17368,"##osh":17369,"jurisdictions":17370,"sd":17371,"tuning":17372,"irritation":17373,"woven":17374,"##uddin":17375,"fertility":17376,"gao":17377,"##erie":17378,"antagonist":17379,"impatient":17380,"glacial":17381,"hides":17382,"boarded":17383,"denominations":17384,"interception":17385,"##jas":17386,"cookie":17387,"nicola":17388,"##tee":17389,"algebraic":17390,"marquess":17391,"bahn":17392,"parole":17393,"buyers":17394,"bait":17395,"turbines":17396,"paperwork":17397,"bestowed":17398,"natasha":17399,"renee":17400,"oceans":17401,"purchases":17402,"157":17403,"vaccine":17404,"215":17405,"##tock":17406,"fixtures":17407,"playhouse":17408,"integrate":17409,"jai":17410,"oswald":17411,"intellectuals":17412,"##cky":17413,"booked":17414,"nests":17415,"mortimer":17416,"##isi":17417,"obsession":17418,"sept":17419,"##gler":17420,"##sum":17421,"440":17422,"scrutiny":17423,"simultaneous":17424,"squinted":17425,"##shin":17426,"collects":17427,"oven":17428,"shankar":17429,"penned":17430,"remarkably":17431,"##я":17432,"slips":17433,"luggage":17434,"spectral":17435,"1786":17436,"collaborations":17437,"louie":17438,"consolidation":17439,"##ailed":17440,"##ivating":17441,"420":17442,"hoover":17443,"blackpool":17444,"harness":17445,"ignition":17446,"vest":17447,"tails":17448,"belmont":17449,"mongol":17450,"skinner":17451,"##nae":17452,"visually":17453,"mage":17454,"derry":17455,"##tism":17456,"##unce":17457,"stevie":17458,"transitional":17459,"##rdy":17460,"redskins":17461,"drying":17462,"prep":17463,"prospective":17464,"##21":17465,"annoyance":17466,"oversee":17467,"##loaded":17468,"fills":17469,"##books":17470,"##iki":17471,"announces":17472,"fda":17473,"scowled":17474,"respects":17475,"prasad":17476,"mystic":17477,"tucson":17478,"##vale":17479,"revue":17480,"springer":17481,"bankrupt":17482,"1772":17483,"aristotle":17484,"salvatore":17485,"habsburg":17486,"##geny":17487,"dal":17488,"natal":17489,"nut":17490,"pod":17491,"chewing":17492,"darts":17493,"moroccan":17494,"walkover":17495,"rosario":17496,"lenin":17497,"punjabi":17498,"##ße":17499,"grossed":17500,"scattering":17501,"wired":17502,"invasive":17503,"hui":17504,"polynomial":17505,"corridors":17506,"wakes":17507,"gina":17508,"portrays":17509,"##cratic":17510,"arid":17511,"retreating":17512,"erich":17513,"irwin":17514,"sniper":17515,"##dha":17516,"linen":17517,"lindsey":17518,"maneuver":17519,"butch":17520,"shutting":17521,"socio":17522,"bounce":17523,"commemorative":17524,"postseason":17525,"jeremiah":17526,"pines":17527,"275":17528,"mystical":17529,"beads":17530,"bp":17531,"abbas":17532,"furnace":17533,"bidding":17534,"consulted":17535,"assaulted":17536,"empirical":17537,"rubble":17538,"enclosure":17539,"sob":17540,"weakly":17541,"cancel":17542,"polly":17543,"yielded":17544,"##emann":17545,"curly":17546,"prediction":17547,"battered":17548,"70s":17549,"vhs":17550,"jacqueline":17551,"render":17552,"sails":17553,"barked":17554,"detailing":17555,"grayson":17556,"riga":17557,"sloane":17558,"raging":17559,"##yah":17560,"herbs":17561,"bravo":17562,"##athlon":17563,"alloy":17564,"giggle":17565,"imminent":17566,"suffers":17567,"assumptions":17568,"waltz":17569,"##itate":17570,"accomplishments":17571,"##ited":17572,"bathing":17573,"remixed":17574,"deception":17575,"prefix":17576,"##emia":17577,"deepest":17578,"##tier":17579,"##eis":17580,"balkan":17581,"frogs":17582,"##rong":17583,"slab":17584,"##pate":17585,"philosophers":17586,"peterborough":17587,"grains":17588,"imports":17589,"dickinson":17590,"rwanda":17591,"##atics":17592,"1774":17593,"dirk":17594,"lan":17595,"tablets":17596,"##rove":17597,"clone":17598,"##rice":17599,"caretaker":17600,"hostilities":17601,"mclean":17602,"##gre":17603,"regimental":17604,"treasures":17605,"norms":17606,"impose":17607,"tsar":17608,"tango":17609,"diplomacy":17610,"variously":17611,"complain":17612,"192":17613,"recognise":17614,"arrests":17615,"1779":17616,"celestial":17617,"pulitzer":17618,"##dus":17619,"bing":17620,"libretto":17621,"##moor":17622,"adele":17623,"splash":17624,"##rite":17625,"expectation":17626,"lds":17627,"confronts":17628,"##izer":17629,"spontaneous":17630,"harmful":17631,"wedge":17632,"entrepreneurs":17633,"buyer":17634,"##ope":17635,"bilingual":17636,"translate":17637,"rugged":17638,"conner":17639,"circulated":17640,"uae":17641,"eaton":17642,"##gra":17643,"##zzle":17644,"lingered":17645,"lockheed":17646,"vishnu":17647,"reelection":17648,"alonso":17649,"##oom":17650,"joints":17651,"yankee":17652,"headline":17653,"cooperate":17654,"heinz":17655,"laureate":17656,"invading":17657,"##sford":17658,"echoes":17659,"scandinavian":17660,"##dham":17661,"hugging":17662,"vitamin":17663,"salute":17664,"micah":17665,"hind":17666,"trader":17667,"##sper":17668,"radioactive":17669,"##ndra":17670,"militants":17671,"poisoned":17672,"ratified":17673,"remark":17674,"campeonato":17675,"deprived":17676,"wander":17677,"prop":17678,"##dong":17679,"outlook":17680,"##tani":17681,"##rix":17682,"##eye":17683,"chiang":17684,"darcy":17685,"##oping":17686,"mandolin":17687,"spice":17688,"statesman":17689,"babylon":17690,"182":17691,"walled":17692,"forgetting":17693,"afro":17694,"##cap":17695,"158":17696,"giorgio":17697,"buffer":17698,"##polis":17699,"planetary":17700,"##gis":17701,"overlap":17702,"terminals":17703,"kinda":17704,"centenary":17705,"##bir":17706,"arising":17707,"manipulate":17708,"elm":17709,"ke":17710,"1770":17711,"ak":17712,"##tad":17713,"chrysler":17714,"mapped":17715,"moose":17716,"pomeranian":17717,"quad":17718,"macarthur":17719,"assemblies":17720,"shoreline":17721,"recalls":17722,"stratford":17723,"##rted":17724,"noticeable":17725,"##evic":17726,"imp":17727,"##rita":17728,"##sque":17729,"accustomed":17730,"supplying":17731,"tents":17732,"disgusted":17733,"vogue":17734,"sipped":17735,"filters":17736,"khz":17737,"reno":17738,"selecting":17739,"luftwaffe":17740,"mcmahon":17741,"tyne":17742,"masterpiece":17743,"carriages":17744,"collided":17745,"dunes":17746,"exercised":17747,"flare":17748,"remembers":17749,"muzzle":17750,"##mobile":17751,"heck":17752,"##rson":17753,"burgess":17754,"lunged":17755,"middleton":17756,"boycott":17757,"bilateral":17758,"##sity":17759,"hazardous":17760,"lumpur":17761,"multiplayer":17762,"spotlight":17763,"jackets":17764,"goldman":17765,"liege":17766,"porcelain":17767,"rag":17768,"waterford":17769,"benz":17770,"attracts":17771,"hopeful":17772,"battling":17773,"ottomans":17774,"kensington":17775,"baked":17776,"hymns":17777,"cheyenne":17778,"lattice":17779,"levine":17780,"borrow":17781,"polymer":17782,"clashes":17783,"michaels":17784,"monitored":17785,"commitments":17786,"denounced":17787,"##25":17788,"##von":17789,"cavity":17790,"##oney":17791,"hobby":17792,"akin":17793,"##holders":17794,"futures":17795,"intricate":17796,"cornish":17797,"patty":17798,"##oned":17799,"illegally":17800,"dolphin":17801,"##lag":17802,"barlow":17803,"yellowish":17804,"maddie":17805,"apologized":17806,"luton":17807,"plagued":17808,"##puram":17809,"nana":17810,"##rds":17811,"sway":17812,"fanny":17813,"łodz":17814,"##rino":17815,"psi":17816,"suspicions":17817,"hanged":17818,"##eding":17819,"initiate":17820,"charlton":17821,"##por":17822,"nak":17823,"competent":17824,"235":17825,"analytical":17826,"annex":17827,"wardrobe":17828,"reservations":17829,"##rma":17830,"sect":17831,"162":17832,"fairfax":17833,"hedge":17834,"piled":17835,"buckingham":17836,"uneven":17837,"bauer":17838,"simplicity":17839,"snyder":17840,"interpret":17841,"accountability":17842,"donors":17843,"moderately":17844,"byrd":17845,"continents":17846,"##cite":17847,"##max":17848,"disciple":17849,"hr":17850,"jamaican":17851,"ping":17852,"nominees":17853,"##uss":17854,"mongolian":17855,"diver":17856,"attackers":17857,"eagerly":17858,"ideological":17859,"pillows":17860,"miracles":17861,"apartheid":17862,"revolver":17863,"sulfur":17864,"clinics":17865,"moran":17866,"163":17867,"##enko":17868,"ile":17869,"katy":17870,"rhetoric":17871,"##icated":17872,"chronology":17873,"recycling":17874,"##hrer":17875,"elongated":17876,"mughal":17877,"pascal":17878,"profiles":17879,"vibration":17880,"databases":17881,"domination":17882,"##fare":17883,"##rant":17884,"matthias":17885,"digest":17886,"rehearsal":17887,"polling":17888,"weiss":17889,"initiation":17890,"reeves":17891,"clinging":17892,"flourished":17893,"impress":17894,"ngo":17895,"##hoff":17896,"##ume":17897,"buckley":17898,"symposium":17899,"rhythms":17900,"weed":17901,"emphasize":17902,"transforming":17903,"##taking":17904,"##gence":17905,"##yman":17906,"accountant":17907,"analyze":17908,"flicker":17909,"foil":17910,"priesthood":17911,"voluntarily":17912,"decreases":17913,"##80":17914,"##hya":17915,"slater":17916,"sv":17917,"charting":17918,"mcgill":17919,"##lde":17920,"moreno":17921,"##iu":17922,"besieged":17923,"zur":17924,"robes":17925,"##phic":17926,"admitting":17927,"api":17928,"deported":17929,"turmoil":17930,"peyton":17931,"earthquakes":17932,"##ares":17933,"nationalists":17934,"beau":17935,"clair":17936,"brethren":17937,"interrupt":17938,"welch":17939,"curated":17940,"galerie":17941,"requesting":17942,"164":17943,"##ested":17944,"impending":17945,"steward":17946,"viper":17947,"##vina":17948,"complaining":17949,"beautifully":17950,"brandy":17951,"foam":17952,"nl":17953,"1660":17954,"##cake":17955,"alessandro":17956,"punches":17957,"laced":17958,"explanations":17959,"##lim":17960,"attribute":17961,"clit":17962,"reggie":17963,"discomfort":17964,"##cards":17965,"smoothed":17966,"whales":17967,"##cene":17968,"adler":17969,"countered":17970,"duffy":17971,"disciplinary":17972,"widening":17973,"recipe":17974,"reliance":17975,"conducts":17976,"goats":17977,"gradient":17978,"preaching":17979,"##shaw":17980,"matilda":17981,"quasi":17982,"striped":17983,"meridian":17984,"cannabis":17985,"cordoba":17986,"certificates":17987,"##agh":17988,"##tering":17989,"graffiti":17990,"hangs":17991,"pilgrims":17992,"repeats":17993,"##ych":17994,"revive":17995,"urine":17996,"etat":17997,"##hawk":17998,"fueled":17999,"belts":18000,"fuzzy":18001,"susceptible":18002,"##hang":18003,"mauritius":18004,"salle":18005,"sincere":18006,"beers":18007,"hooks":18008,"##cki":18009,"arbitration":18010,"entrusted":18011,"advise":18012,"sniffed":18013,"seminar":18014,"junk":18015,"donnell":18016,"processors":18017,"principality":18018,"strapped":18019,"celia":18020,"mendoza":18021,"everton":18022,"fortunes":18023,"prejudice":18024,"starving":18025,"reassigned":18026,"steamer":18027,"##lund":18028,"tuck":18029,"evenly":18030,"foreman":18031,"##ffen":18032,"dans":18033,"375":18034,"envisioned":18035,"slit":18036,"##xy":18037,"baseman":18038,"liberia":18039,"rosemary":18040,"##weed":18041,"electrified":18042,"periodically":18043,"potassium":18044,"stride":18045,"contexts":18046,"sperm":18047,"slade":18048,"mariners":18049,"influx":18050,"bianca":18051,"subcommittee":18052,"##rane":18053,"spilling":18054,"icao":18055,"estuary":18056,"##nock":18057,"delivers":18058,"iphone":18059,"##ulata":18060,"isa":18061,"mira":18062,"bohemian":18063,"dessert":18064,"##sbury":18065,"welcoming":18066,"proudly":18067,"slowing":18068,"##chs":18069,"musee":18070,"ascension":18071,"russ":18072,"##vian":18073,"waits":18074,"##psy":18075,"africans":18076,"exploit":18077,"##morphic":18078,"gov":18079,"eccentric":18080,"crab":18081,"peck":18082,"##ull":18083,"entrances":18084,"formidable":18085,"marketplace":18086,"groom":18087,"bolted":18088,"metabolism":18089,"patton":18090,"robbins":18091,"courier":18092,"payload":18093,"endure":18094,"##ifier":18095,"andes":18096,"refrigerator":18097,"##pr":18098,"ornate":18099,"##uca":18100,"ruthless":18101,"illegitimate":18102,"masonry":18103,"strasbourg":18104,"bikes":18105,"adobe":18106,"##³":18107,"apples":18108,"quintet":18109,"willingly":18110,"niche":18111,"bakery":18112,"corpses":18113,"energetic":18114,"##cliffe":18115,"##sser":18116,"##ards":18117,"177":18118,"centimeters":18119,"centro":18120,"fuscous":18121,"cretaceous":18122,"rancho":18123,"##yde":18124,"andrei":18125,"telecom":18126,"tottenham":18127,"oasis":18128,"ordination":18129,"vulnerability":18130,"presiding":18131,"corey":18132,"cp":18133,"penguins":18134,"sims":18135,"##pis":18136,"malawi":18137,"piss":18138,"##48":18139,"correction":18140,"##cked":18141,"##ffle":18142,"##ryn":18143,"countdown":18144,"detectives":18145,"psychiatrist":18146,"psychedelic":18147,"dinosaurs":18148,"blouse":18149,"##get":18150,"choi":18151,"vowed":18152,"##oz":18153,"randomly":18154,"##pol":18155,"49ers":18156,"scrub":18157,"blanche":18158,"bruins":18159,"dusseldorf":18160,"##using":18161,"unwanted":18162,"##ums":18163,"212":18164,"dominique":18165,"elevations":18166,"headlights":18167,"om":18168,"laguna":18169,"##oga":18170,"1750":18171,"famously":18172,"ignorance":18173,"shrewsbury":18174,"##aine":18175,"ajax":18176,"breuning":18177,"che":18178,"confederacy":18179,"greco":18180,"overhaul":18181,"##screen":18182,"paz":18183,"skirts":18184,"disagreement":18185,"cruelty":18186,"jagged":18187,"phoebe":18188,"shifter":18189,"hovered":18190,"viruses":18191,"##wes":18192,"mandy":18193,"##lined":18194,"##gc":18195,"landlord":18196,"squirrel":18197,"dashed":18198,"##ι":18199,"ornamental":18200,"gag":18201,"wally":18202,"grange":18203,"literal":18204,"spurs":18205,"undisclosed":18206,"proceeding":18207,"yin":18208,"##text":18209,"billie":18210,"orphan":18211,"spanned":18212,"humidity":18213,"indy":18214,"weighted":18215,"presentations":18216,"explosions":18217,"lucian":18218,"##tary":18219,"vaughn":18220,"hindus":18221,"##anga":18222,"##hell":18223,"psycho":18224,"171":18225,"daytona":18226,"protects":18227,"efficiently":18228,"rematch":18229,"sly":18230,"tandem":18231,"##oya":18232,"rebranded":18233,"impaired":18234,"hee":18235,"metropolis":18236,"peach":18237,"godfrey":18238,"diaspora":18239,"ethnicity":18240,"prosperous":18241,"gleaming":18242,"dar":18243,"grossing":18244,"playback":18245,"##rden":18246,"stripe":18247,"pistols":18248,"##tain":18249,"births":18250,"labelled":18251,"##cating":18252,"172":18253,"rudy":18254,"alba":18255,"##onne":18256,"aquarium":18257,"hostility":18258,"##gb":18259,"##tase":18260,"shudder":18261,"sumatra":18262,"hardest":18263,"lakers":18264,"consonant":18265,"creeping":18266,"demos":18267,"homicide":18268,"capsule":18269,"zeke":18270,"liberties":18271,"expulsion":18272,"pueblo":18273,"##comb":18274,"trait":18275,"transporting":18276,"##ddin":18277,"##neck":18278,"##yna":18279,"depart":18280,"gregg":18281,"mold":18282,"ledge":18283,"hangar":18284,"oldham":18285,"playboy":18286,"termination":18287,"analysts":18288,"gmbh":18289,"romero":18290,"##itic":18291,"insist":18292,"cradle":18293,"filthy":18294,"brightness":18295,"slash":18296,"shootout":18297,"deposed":18298,"bordering":18299,"##truct":18300,"isis":18301,"microwave":18302,"tumbled":18303,"sheltered":18304,"cathy":18305,"werewolves":18306,"messy":18307,"andersen":18308,"convex":18309,"clapped":18310,"clinched":18311,"satire":18312,"wasting":18313,"edo":18314,"vc":18315,"rufus":18316,"##jak":18317,"mont":18318,"##etti":18319,"poznan":18320,"##keeping":18321,"restructuring":18322,"transverse":18323,"##rland":18324,"azerbaijani":18325,"slovene":18326,"gestures":18327,"roommate":18328,"choking":18329,"shear":18330,"##quist":18331,"vanguard":18332,"oblivious":18333,"##hiro":18334,"disagreed":18335,"baptism":18336,"##lich":18337,"coliseum":18338,"##aceae":18339,"salvage":18340,"societe":18341,"cory":18342,"locke":18343,"relocation":18344,"relying":18345,"versailles":18346,"ahl":18347,"swelling":18348,"##elo":18349,"cheerful":18350,"##word":18351,"##edes":18352,"gin":18353,"sarajevo":18354,"obstacle":18355,"diverted":18356,"##nac":18357,"messed":18358,"thoroughbred":18359,"fluttered":18360,"utrecht":18361,"chewed":18362,"acquaintance":18363,"assassins":18364,"dispatch":18365,"mirza":18366,"##wart":18367,"nike":18368,"salzburg":18369,"swell":18370,"yen":18371,"##gee":18372,"idle":18373,"ligue":18374,"samson":18375,"##nds":18376,"##igh":18377,"playful":18378,"spawned":18379,"##cise":18380,"tease":18381,"##case":18382,"burgundy":18383,"##bot":18384,"stirring":18385,"skeptical":18386,"interceptions":18387,"marathi":18388,"##dies":18389,"bedrooms":18390,"aroused":18391,"pinch":18392,"##lik":18393,"preferences":18394,"tattoos":18395,"buster":18396,"digitally":18397,"projecting":18398,"rust":18399,"##ital":18400,"kitten":18401,"priorities":18402,"addison":18403,"pseudo":18404,"##guard":18405,"dusk":18406,"icons":18407,"sermon":18408,"##psis":18409,"##iba":18410,"bt":18411,"##lift":18412,"##xt":18413,"ju":18414,"truce":18415,"rink":18416,"##dah":18417,"##wy":18418,"defects":18419,"psychiatry":18420,"offences":18421,"calculate":18422,"glucose":18423,"##iful":18424,"##rized":18425,"##unda":18426,"francaise":18427,"##hari":18428,"richest":18429,"warwickshire":18430,"carly":18431,"1763":18432,"purity":18433,"redemption":18434,"lending":18435,"##cious":18436,"muse":18437,"bruises":18438,"cerebral":18439,"aero":18440,"carving":18441,"##name":18442,"preface":18443,"terminology":18444,"invade":18445,"monty":18446,"##int":18447,"anarchist":18448,"blurred":18449,"##iled":18450,"rossi":18451,"treats":18452,"guts":18453,"shu":18454,"foothills":18455,"ballads":18456,"undertaking":18457,"premise":18458,"cecilia":18459,"affiliates":18460,"blasted":18461,"conditional":18462,"wilder":18463,"minors":18464,"drone":18465,"rudolph":18466,"buffy":18467,"swallowing":18468,"horton":18469,"attested":18470,"##hop":18471,"rutherford":18472,"howell":18473,"primetime":18474,"livery":18475,"penal":18476,"##bis":18477,"minimize":18478,"hydro":18479,"wrecked":18480,"wrought":18481,"palazzo":18482,"##gling":18483,"cans":18484,"vernacular":18485,"friedman":18486,"nobleman":18487,"shale":18488,"walnut":18489,"danielle":18490,"##ection":18491,"##tley":18492,"sears":18493,"##kumar":18494,"chords":18495,"lend":18496,"flipping":18497,"streamed":18498,"por":18499,"dracula":18500,"gallons":18501,"sacrifices":18502,"gamble":18503,"orphanage":18504,"##iman":18505,"mckenzie":18506,"##gible":18507,"boxers":18508,"daly":18509,"##balls":18510,"##ان":18511,"208":18512,"##ific":18513,"##rative":18514,"##iq":18515,"exploited":18516,"slated":18517,"##uity":18518,"circling":18519,"hillary":18520,"pinched":18521,"goldberg":18522,"provost":18523,"campaigning":18524,"lim":18525,"piles":18526,"ironically":18527,"jong":18528,"mohan":18529,"successors":18530,"usaf":18531,"##tem":18532,"##ught":18533,"autobiographical":18534,"haute":18535,"preserves":18536,"##ending":18537,"acquitted":18538,"comparisons":18539,"203":18540,"hydroelectric":18541,"gangs":18542,"cypriot":18543,"torpedoes":18544,"rushes":18545,"chrome":18546,"derive":18547,"bumps":18548,"instability":18549,"fiat":18550,"pets":18551,"##mbe":18552,"silas":18553,"dye":18554,"reckless":18555,"settler":18556,"##itation":18557,"info":18558,"heats":18559,"##writing":18560,"176":18561,"canonical":18562,"maltese":18563,"fins":18564,"mushroom":18565,"stacy":18566,"aspen":18567,"avid":18568,"##kur":18569,"##loading":18570,"vickers":18571,"gaston":18572,"hillside":18573,"statutes":18574,"wilde":18575,"gail":18576,"kung":18577,"sabine":18578,"comfortably":18579,"motorcycles":18580,"##rgo":18581,"169":18582,"pneumonia":18583,"fetch":18584,"##sonic":18585,"axel":18586,"faintly":18587,"parallels":18588,"##oop":18589,"mclaren":18590,"spouse":18591,"compton":18592,"interdisciplinary":18593,"miner":18594,"##eni":18595,"181":18596,"clamped":18597,"##chal":18598,"##llah":18599,"separates":18600,"versa":18601,"##mler":18602,"scarborough":18603,"labrador":18604,"##lity":18605,"##osing":18606,"rutgers":18607,"hurdles":18608,"como":18609,"166":18610,"burt":18611,"divers":18612,"##100":18613,"wichita":18614,"cade":18615,"coincided":18616,"##erson":18617,"bruised":18618,"mla":18619,"##pper":18620,"vineyard":18621,"##ili":18622,"##brush":18623,"notch":18624,"mentioning":18625,"jase":18626,"hearted":18627,"kits":18628,"doe":18629,"##acle":18630,"pomerania":18631,"##ady":18632,"ronan":18633,"seizure":18634,"pavel":18635,"problematic":18636,"##zaki":18637,"domenico":18638,"##ulin":18639,"catering":18640,"penelope":18641,"dependence":18642,"parental":18643,"emilio":18644,"ministerial":18645,"atkinson":18646,"##bolic":18647,"clarkson":18648,"chargers":18649,"colby":18650,"grill":18651,"peeked":18652,"arises":18653,"summon":18654,"##aged":18655,"fools":18656,"##grapher":18657,"faculties":18658,"qaeda":18659,"##vial":18660,"garner":18661,"refurbished":18662,"##hwa":18663,"geelong":18664,"disasters":18665,"nudged":18666,"bs":18667,"shareholder":18668,"lori":18669,"algae":18670,"reinstated":18671,"rot":18672,"##ades":18673,"##nous":18674,"invites":18675,"stainless":18676,"183":18677,"inclusive":18678,"##itude":18679,"diocesan":18680,"til":18681,"##icz":18682,"denomination":18683,"##xa":18684,"benton":18685,"floral":18686,"registers":18687,"##ider":18688,"##erman":18689,"##kell":18690,"absurd":18691,"brunei":18692,"guangzhou":18693,"hitter":18694,"retaliation":18695,"##uled":18696,"##eve":18697,"blanc":18698,"nh":18699,"consistency":18700,"contamination":18701,"##eres":18702,"##rner":18703,"dire":18704,"palermo":18705,"broadcasters":18706,"diaries":18707,"inspire":18708,"vols":18709,"brewer":18710,"tightening":18711,"ky":18712,"mixtape":18713,"hormone":18714,"##tok":18715,"stokes":18716,"##color":18717,"##dly":18718,"##ssi":18719,"pg":18720,"##ometer":18721,"##lington":18722,"sanitation":18723,"##tility":18724,"intercontinental":18725,"apps":18726,"##adt":18727,"¹⁄₂":18728,"cylinders":18729,"economies":18730,"favourable":18731,"unison":18732,"croix":18733,"gertrude":18734,"odyssey":18735,"vanity":18736,"dangling":18737,"##logists":18738,"upgrades":18739,"dice":18740,"middleweight":18741,"practitioner":18742,"##ight":18743,"206":18744,"henrik":18745,"parlor":18746,"orion":18747,"angered":18748,"lac":18749,"python":18750,"blurted":18751,"##rri":18752,"sensual":18753,"intends":18754,"swings":18755,"angled":18756,"##phs":18757,"husky":18758,"attain":18759,"peerage":18760,"precinct":18761,"textiles":18762,"cheltenham":18763,"shuffled":18764,"dai":18765,"confess":18766,"tasting":18767,"bhutan":18768,"##riation":18769,"tyrone":18770,"segregation":18771,"abrupt":18772,"ruiz":18773,"##rish":18774,"smirked":18775,"blackwell":18776,"confidential":18777,"browning":18778,"amounted":18779,"##put":18780,"vase":18781,"scarce":18782,"fabulous":18783,"raided":18784,"staple":18785,"guyana":18786,"unemployed":18787,"glider":18788,"shay":18789,"##tow":18790,"carmine":18791,"troll":18792,"intervene":18793,"squash":18794,"superstar":18795,"##uce":18796,"cylindrical":18797,"len":18798,"roadway":18799,"researched":18800,"handy":18801,"##rium":18802,"##jana":18803,"meta":18804,"lao":18805,"declares":18806,"##rring":18807,"##tadt":18808,"##elin":18809,"##kova":18810,"willem":18811,"shrubs":18812,"napoleonic":18813,"realms":18814,"skater":18815,"qi":18816,"volkswagen":18817,"##ł":18818,"tad":18819,"hara":18820,"archaeologist":18821,"awkwardly":18822,"eerie":18823,"##kind":18824,"wiley":18825,"##heimer":18826,"##24":18827,"titus":18828,"organizers":18829,"cfl":18830,"crusaders":18831,"lama":18832,"usb":18833,"vent":18834,"enraged":18835,"thankful":18836,"occupants":18837,"maximilian":18838,"##gaard":18839,"possessing":18840,"textbooks":18841,"##oran":18842,"collaborator":18843,"quaker":18844,"##ulo":18845,"avalanche":18846,"mono":18847,"silky":18848,"straits":18849,"isaiah":18850,"mustang":18851,"surged":18852,"resolutions":18853,"potomac":18854,"descend":18855,"cl":18856,"kilograms":18857,"plato":18858,"strains":18859,"saturdays":18860,"##olin":18861,"bernstein":18862,"##ype":18863,"holstein":18864,"ponytail":18865,"##watch":18866,"belize":18867,"conversely":18868,"heroine":18869,"perpetual":18870,"##ylus":18871,"charcoal":18872,"piedmont":18873,"glee":18874,"negotiating":18875,"backdrop":18876,"prologue":18877,"##jah":18878,"##mmy":18879,"pasadena":18880,"climbs":18881,"ramos":18882,"sunni":18883,"##holm":18884,"##tner":18885,"##tri":18886,"anand":18887,"deficiency":18888,"hertfordshire":18889,"stout":18890,"##avi":18891,"aperture":18892,"orioles":18893,"##irs":18894,"doncaster":18895,"intrigued":18896,"bombed":18897,"coating":18898,"otis":18899,"##mat":18900,"cocktail":18901,"##jit":18902,"##eto":18903,"amir":18904,"arousal":18905,"sar":18906,"##proof":18907,"##act":18908,"##ories":18909,"dixie":18910,"pots":18911,"##bow":18912,"whereabouts":18913,"159":18914,"##fted":18915,"drains":18916,"bullying":18917,"cottages":18918,"scripture":18919,"coherent":18920,"fore":18921,"poe":18922,"appetite":18923,"##uration":18924,"sampled":18925,"##ators":18926,"##dp":18927,"derrick":18928,"rotor":18929,"jays":18930,"peacock":18931,"installment":18932,"##rro":18933,"advisors":18934,"##coming":18935,"rodeo":18936,"scotch":18937,"##mot":18938,"##db":18939,"##fen":18940,"##vant":18941,"ensued":18942,"rodrigo":18943,"dictatorship":18944,"martyrs":18945,"twenties":18946,"##н":18947,"towed":18948,"incidence":18949,"marta":18950,"rainforest":18951,"sai":18952,"scaled":18953,"##cles":18954,"oceanic":18955,"qualifiers":18956,"symphonic":18957,"mcbride":18958,"dislike":18959,"generalized":18960,"aubrey":18961,"colonization":18962,"##iation":18963,"##lion":18964,"##ssing":18965,"disliked":18966,"lublin":18967,"salesman":18968,"##ulates":18969,"spherical":18970,"whatsoever":18971,"sweating":18972,"avalon":18973,"contention":18974,"punt":18975,"severity":18976,"alderman":18977,"atari":18978,"##dina":18979,"##grant":18980,"##rop":18981,"scarf":18982,"seville":18983,"vertices":18984,"annexation":18985,"fairfield":18986,"fascination":18987,"inspiring":18988,"launches":18989,"palatinate":18990,"regretted":18991,"##rca":18992,"feral":18993,"##iom":18994,"elk":18995,"nap":18996,"olsen":18997,"reddy":18998,"yong":18999,"##leader":19000,"##iae":19001,"garment":19002,"transports":19003,"feng":19004,"gracie":19005,"outrage":19006,"viceroy":19007,"insides":19008,"##esis":19009,"breakup":19010,"grady":19011,"organizer":19012,"softer":19013,"grimaced":19014,"222":19015,"murals":19016,"galicia":19017,"arranging":19018,"vectors":19019,"##rsten":19020,"bas":19021,"##sb":19022,"##cens":19023,"sloan":19024,"##eka":19025,"bitten":19026,"ara":19027,"fender":19028,"nausea":19029,"bumped":19030,"kris":19031,"banquet":19032,"comrades":19033,"detector":19034,"persisted":19035,"##llan":19036,"adjustment":19037,"endowed":19038,"cinemas":19039,"##shot":19040,"sellers":19041,"##uman":19042,"peek":19043,"epa":19044,"kindly":19045,"neglect":19046,"simpsons":19047,"talon":19048,"mausoleum":19049,"runaway":19050,"hangul":19051,"lookout":19052,"##cic":19053,"rewards":19054,"coughed":19055,"acquainted":19056,"chloride":19057,"##ald":19058,"quicker":19059,"accordion":19060,"neolithic":19061,"##qa":19062,"artemis":19063,"coefficient":19064,"lenny":19065,"pandora":19066,"tx":19067,"##xed":19068,"ecstasy":19069,"litter":19070,"segunda":19071,"chairperson":19072,"gemma":19073,"hiss":19074,"rumor":19075,"vow":19076,"nasal":19077,"antioch":19078,"compensate":19079,"patiently":19080,"transformers":19081,"##eded":19082,"judo":19083,"morrow":19084,"penis":19085,"posthumous":19086,"philips":19087,"bandits":19088,"husbands":19089,"denote":19090,"flaming":19091,"##any":19092,"##phones":19093,"langley":19094,"yorker":19095,"1760":19096,"walters":19097,"##uo":19098,"##kle":19099,"gubernatorial":19100,"fatty":19101,"samsung":19102,"leroy":19103,"outlaw":19104,"##nine":19105,"unpublished":19106,"poole":19107,"jakob":19108,"##ᵢ":19109,"##ₙ":19110,"crete":19111,"distorted":19112,"superiority":19113,"##dhi":19114,"intercept":19115,"crust":19116,"mig":19117,"claus":19118,"crashes":19119,"positioning":19120,"188":19121,"stallion":19122,"301":19123,"frontal":19124,"armistice":19125,"##estinal":19126,"elton":19127,"aj":19128,"encompassing":19129,"camel":19130,"commemorated":19131,"malaria":19132,"woodward":19133,"calf":19134,"cigar":19135,"penetrate":19136,"##oso":19137,"willard":19138,"##rno":19139,"##uche":19140,"illustrate":19141,"amusing":19142,"convergence":19143,"noteworthy":19144,"##lma":19145,"##rva":19146,"journeys":19147,"realise":19148,"manfred":19149,"##sable":19150,"410":19151,"##vocation":19152,"hearings":19153,"fiance":19154,"##posed":19155,"educators":19156,"provoked":19157,"adjusting":19158,"##cturing":19159,"modular":19160,"stockton":19161,"paterson":19162,"vlad":19163,"rejects":19164,"electors":19165,"selena":19166,"maureen":19167,"##tres":19168,"uber":19169,"##rce":19170,"swirled":19171,"##num":19172,"proportions":19173,"nanny":19174,"pawn":19175,"naturalist":19176,"parma":19177,"apostles":19178,"awoke":19179,"ethel":19180,"wen":19181,"##bey":19182,"monsoon":19183,"overview":19184,"##inating":19185,"mccain":19186,"rendition":19187,"risky":19188,"adorned":19189,"##ih":19190,"equestrian":19191,"germain":19192,"nj":19193,"conspicuous":19194,"confirming":19195,"##yoshi":19196,"shivering":19197,"##imeter":19198,"milestone":19199,"rumours":19200,"flinched":19201,"bounds":19202,"smacked":19203,"token":19204,"##bei":19205,"lectured":19206,"automobiles":19207,"##shore":19208,"impacted":19209,"##iable":19210,"nouns":19211,"nero":19212,"##leaf":19213,"ismail":19214,"prostitute":19215,"trams":19216,"##lace":19217,"bridget":19218,"sud":19219,"stimulus":19220,"impressions":19221,"reins":19222,"revolves":19223,"##oud":19224,"##gned":19225,"giro":19226,"honeymoon":19227,"##swell":19228,"criterion":19229,"##sms":19230,"##uil":19231,"libyan":19232,"prefers":19233,"##osition":19234,"211":19235,"preview":19236,"sucks":19237,"accusation":19238,"bursts":19239,"metaphor":19240,"diffusion":19241,"tolerate":19242,"faye":19243,"betting":19244,"cinematographer":19245,"liturgical":19246,"specials":19247,"bitterly":19248,"humboldt":19249,"##ckle":19250,"flux":19251,"rattled":19252,"##itzer":19253,"archaeologists":19254,"odor":19255,"authorised":19256,"marshes":19257,"discretion":19258,"##ов":19259,"alarmed":19260,"archaic":19261,"inverse":19262,"##leton":19263,"explorers":19264,"##pine":19265,"drummond":19266,"tsunami":19267,"woodlands":19268,"##minate":19269,"##tland":19270,"booklet":19271,"insanity":19272,"owning":19273,"insert":19274,"crafted":19275,"calculus":19276,"##tore":19277,"receivers":19278,"##bt":19279,"stung":19280,"##eca":19281,"##nched":19282,"prevailing":19283,"travellers":19284,"eyeing":19285,"lila":19286,"graphs":19287,"##borne":19288,"178":19289,"julien":19290,"##won":19291,"morale":19292,"adaptive":19293,"therapist":19294,"erica":19295,"cw":19296,"libertarian":19297,"bowman":19298,"pitches":19299,"vita":19300,"##ional":19301,"crook":19302,"##ads":19303,"##entation":19304,"caledonia":19305,"mutiny":19306,"##sible":19307,"1840s":19308,"automation":19309,"##ß":19310,"flock":19311,"##pia":19312,"ironic":19313,"pathology":19314,"##imus":19315,"remarried":19316,"##22":19317,"joker":19318,"withstand":19319,"energies":19320,"##att":19321,"shropshire":19322,"hostages":19323,"madeleine":19324,"tentatively":19325,"conflicting":19326,"mateo":19327,"recipes":19328,"euros":19329,"ol":19330,"mercenaries":19331,"nico":19332,"##ndon":19333,"albuquerque":19334,"augmented":19335,"mythical":19336,"bel":19337,"freud":19338,"##child":19339,"cough":19340,"##lica":19341,"365":19342,"freddy":19343,"lillian":19344,"genetically":19345,"nuremberg":19346,"calder":19347,"209":19348,"bonn":19349,"outdoors":19350,"paste":19351,"suns":19352,"urgency":19353,"vin":19354,"restraint":19355,"tyson":19356,"##cera":19357,"##selle":19358,"barrage":19359,"bethlehem":19360,"kahn":19361,"##par":19362,"mounts":19363,"nippon":19364,"barony":19365,"happier":19366,"ryu":19367,"makeshift":19368,"sheldon":19369,"blushed":19370,"castillo":19371,"barking":19372,"listener":19373,"taped":19374,"bethel":19375,"fluent":19376,"headlines":19377,"pornography":19378,"rum":19379,"disclosure":19380,"sighing":19381,"mace":19382,"doubling":19383,"gunther":19384,"manly":19385,"##plex":19386,"rt":19387,"interventions":19388,"physiological":19389,"forwards":19390,"emerges":19391,"##tooth":19392,"##gny":19393,"compliment":19394,"rib":19395,"recession":19396,"visibly":19397,"barge":19398,"faults":19399,"connector":19400,"exquisite":19401,"prefect":19402,"##rlin":19403,"patio":19404,"##cured":19405,"elevators":19406,"brandt":19407,"italics":19408,"pena":19409,"173":19410,"wasp":19411,"satin":19412,"ea":19413,"botswana":19414,"graceful":19415,"respectable":19416,"##jima":19417,"##rter":19418,"##oic":19419,"franciscan":19420,"generates":19421,"##dl":19422,"alfredo":19423,"disgusting":19424,"##olate":19425,"##iously":19426,"sherwood":19427,"warns":19428,"cod":19429,"promo":19430,"cheryl":19431,"sino":19432,"##ة":19433,"##escu":19434,"twitch":19435,"##zhi":19436,"brownish":19437,"thom":19438,"ortiz":19439,"##dron":19440,"densely":19441,"##beat":19442,"carmel":19443,"reinforce":19444,"##bana":19445,"187":19446,"anastasia":19447,"downhill":19448,"vertex":19449,"contaminated":19450,"remembrance":19451,"harmonic":19452,"homework":19453,"##sol":19454,"fiancee":19455,"gears":19456,"olds":19457,"angelica":19458,"loft":19459,"ramsay":19460,"quiz":19461,"colliery":19462,"sevens":19463,"##cape":19464,"autism":19465,"##hil":19466,"walkway":19467,"##boats":19468,"ruben":19469,"abnormal":19470,"ounce":19471,"khmer":19472,"##bbe":19473,"zachary":19474,"bedside":19475,"morphology":19476,"punching":19477,"##olar":19478,"sparrow":19479,"convinces":19480,"##35":19481,"hewitt":19482,"queer":19483,"remastered":19484,"rods":19485,"mabel":19486,"solemn":19487,"notified":19488,"lyricist":19489,"symmetric":19490,"##xide":19491,"174":19492,"encore":19493,"passports":19494,"wildcats":19495,"##uni":19496,"baja":19497,"##pac":19498,"mildly":19499,"##ease":19500,"bleed":19501,"commodity":19502,"mounds":19503,"glossy":19504,"orchestras":19505,"##omo":19506,"damian":19507,"prelude":19508,"ambitions":19509,"##vet":19510,"awhile":19511,"remotely":19512,"##aud":19513,"asserts":19514,"imply":19515,"##iques":19516,"distinctly":19517,"modelling":19518,"remedy":19519,"##dded":19520,"windshield":19521,"dani":19522,"xiao":19523,"##endra":19524,"audible":19525,"powerplant":19526,"1300":19527,"invalid":19528,"elemental":19529,"acquisitions":19530,"##hala":19531,"immaculate":19532,"libby":19533,"plata":19534,"smuggling":19535,"ventilation":19536,"denoted":19537,"minh":19538,"##morphism":19539,"430":19540,"differed":19541,"dion":19542,"kelley":19543,"lore":19544,"mocking":19545,"sabbath":19546,"spikes":19547,"hygiene":19548,"drown":19549,"runoff":19550,"stylized":19551,"tally":19552,"liberated":19553,"aux":19554,"interpreter":19555,"righteous":19556,"aba":19557,"siren":19558,"reaper":19559,"pearce":19560,"millie":19561,"##cier":19562,"##yra":19563,"gaius":19564,"##iso":19565,"captures":19566,"##ttering":19567,"dorm":19568,"claudio":19569,"##sic":19570,"benches":19571,"knighted":19572,"blackness":19573,"##ored":19574,"discount":19575,"fumble":19576,"oxidation":19577,"routed":19578,"##ς":19579,"novak":19580,"perpendicular":19581,"spoiled":19582,"fracture":19583,"splits":19584,"##urt":19585,"pads":19586,"topology":19587,"##cats":19588,"axes":19589,"fortunate":19590,"offenders":19591,"protestants":19592,"esteem":19593,"221":19594,"broadband":19595,"convened":19596,"frankly":19597,"hound":19598,"prototypes":19599,"isil":19600,"facilitated":19601,"keel":19602,"##sher":19603,"sahara":19604,"awaited":19605,"bubba":19606,"orb":19607,"prosecutors":19608,"186":19609,"hem":19610,"520":19611,"##xing":19612,"relaxing":19613,"remnant":19614,"romney":19615,"sorted":19616,"slalom":19617,"stefano":19618,"ulrich":19619,"##active":19620,"exemption":19621,"folder":19622,"pauses":19623,"foliage":19624,"hitchcock":19625,"epithet":19626,"204":19627,"criticisms":19628,"##aca":19629,"ballistic":19630,"brody":19631,"hinduism":19632,"chaotic":19633,"youths":19634,"equals":19635,"##pala":19636,"pts":19637,"thicker":19638,"analogous":19639,"capitalist":19640,"improvised":19641,"overseeing":19642,"sinatra":19643,"ascended":19644,"beverage":19645,"##tl":19646,"straightforward":19647,"##kon":19648,"curran":19649,"##west":19650,"bois":19651,"325":19652,"induce":19653,"surveying":19654,"emperors":19655,"sax":19656,"unpopular":19657,"##kk":19658,"cartoonist":19659,"fused":19660,"##mble":19661,"unto":19662,"##yuki":19663,"localities":19664,"##cko":19665,"##ln":19666,"darlington":19667,"slain":19668,"academie":19669,"lobbying":19670,"sediment":19671,"puzzles":19672,"##grass":19673,"defiance":19674,"dickens":19675,"manifest":19676,"tongues":19677,"alumnus":19678,"arbor":19679,"coincide":19680,"184":19681,"appalachian":19682,"mustafa":19683,"examiner":19684,"cabaret":19685,"traumatic":19686,"yves":19687,"bracelet":19688,"draining":19689,"heroin":19690,"magnum":19691,"baths":19692,"odessa":19693,"consonants":19694,"mitsubishi":19695,"##gua":19696,"kellan":19697,"vaudeville":19698,"##fr":19699,"joked":19700,"null":19701,"straps":19702,"probation":19703,"##ław":19704,"ceded":19705,"interfaces":19706,"##pas":19707,"##zawa":19708,"blinding":19709,"viet":19710,"224":19711,"rothschild":19712,"museo":19713,"640":19714,"huddersfield":19715,"##vr":19716,"tactic":19717,"##storm":19718,"brackets":19719,"dazed":19720,"incorrectly":19721,"##vu":19722,"reg":19723,"glazed":19724,"fearful":19725,"manifold":19726,"benefited":19727,"irony":19728,"##sun":19729,"stumbling":19730,"##rte":19731,"willingness":19732,"balkans":19733,"mei":19734,"wraps":19735,"##aba":19736,"injected":19737,"##lea":19738,"gu":19739,"syed":19740,"harmless":19741,"##hammer":19742,"bray":19743,"takeoff":19744,"poppy":19745,"timor":19746,"cardboard":19747,"astronaut":19748,"purdue":19749,"weeping":19750,"southbound":19751,"cursing":19752,"stalls":19753,"diagonal":19754,"##neer":19755,"lamar":19756,"bryce":19757,"comte":19758,"weekdays":19759,"harrington":19760,"##uba":19761,"negatively":19762,"##see":19763,"lays":19764,"grouping":19765,"##cken":19766,"##henko":19767,"affirmed":19768,"halle":19769,"modernist":19770,"##lai":19771,"hodges":19772,"smelling":19773,"aristocratic":19774,"baptized":19775,"dismiss":19776,"justification":19777,"oilers":19778,"##now":19779,"coupling":19780,"qin":19781,"snack":19782,"healer":19783,"##qing":19784,"gardener":19785,"layla":19786,"battled":19787,"formulated":19788,"stephenson":19789,"gravitational":19790,"##gill":19791,"##jun":19792,"1768":19793,"granny":19794,"coordinating":19795,"suites":19796,"##cd":19797,"##ioned":19798,"monarchs":19799,"##cote":19800,"##hips":19801,"sep":19802,"blended":19803,"apr":19804,"barrister":19805,"deposition":19806,"fia":19807,"mina":19808,"policemen":19809,"paranoid":19810,"##pressed":19811,"churchyard":19812,"covert":19813,"crumpled":19814,"creep":19815,"abandoning":19816,"tr":19817,"transmit":19818,"conceal":19819,"barr":19820,"understands":19821,"readiness":19822,"spire":19823,"##cology":19824,"##enia":19825,"##erry":19826,"610":19827,"startling":19828,"unlock":19829,"vida":19830,"bowled":19831,"slots":19832,"##nat":19833,"##islav":19834,"spaced":19835,"trusting":19836,"admire":19837,"rig":19838,"##ink":19839,"slack":19840,"##70":19841,"mv":19842,"207":19843,"casualty":19844,"##wei":19845,"classmates":19846,"##odes":19847,"##rar":19848,"##rked":19849,"amherst":19850,"furnished":19851,"evolve":19852,"foundry":19853,"menace":19854,"mead":19855,"##lein":19856,"flu":19857,"wesleyan":19858,"##kled":19859,"monterey":19860,"webber":19861,"##vos":19862,"wil":19863,"##mith":19864,"##на":19865,"bartholomew":19866,"justices":19867,"restrained":19868,"##cke":19869,"amenities":19870,"191":19871,"mediated":19872,"sewage":19873,"trenches":19874,"ml":19875,"mainz":19876,"##thus":19877,"1800s":19878,"##cula":19879,"##inski":19880,"caine":19881,"bonding":19882,"213":19883,"converts":19884,"spheres":19885,"superseded":19886,"marianne":19887,"crypt":19888,"sweaty":19889,"ensign":19890,"historia":19891,"##br":19892,"spruce":19893,"##post":19894,"##ask":19895,"forks":19896,"thoughtfully":19897,"yukon":19898,"pamphlet":19899,"ames":19900,"##uter":19901,"karma":19902,"##yya":19903,"bryn":19904,"negotiation":19905,"sighs":19906,"incapable":19907,"##mbre":19908,"##ntial":19909,"actresses":19910,"taft":19911,"##mill":19912,"luce":19913,"prevailed":19914,"##amine":19915,"1773":19916,"motionless":19917,"envoy":19918,"testify":19919,"investing":19920,"sculpted":19921,"instructors":19922,"provence":19923,"kali":19924,"cullen":19925,"horseback":19926,"##while":19927,"goodwin":19928,"##jos":19929,"gaa":19930,"norte":19931,"##ldon":19932,"modify":19933,"wavelength":19934,"abd":19935,"214":19936,"skinned":19937,"sprinter":19938,"forecast":19939,"scheduling":19940,"marries":19941,"squared":19942,"tentative":19943,"##chman":19944,"boer":19945,"##isch":19946,"bolts":19947,"swap":19948,"fisherman":19949,"assyrian":19950,"impatiently":19951,"guthrie":19952,"martins":19953,"murdoch":19954,"194":19955,"tanya":19956,"nicely":19957,"dolly":19958,"lacy":19959,"med":19960,"##45":19961,"syn":19962,"decks":19963,"fashionable":19964,"millionaire":19965,"##ust":19966,"surfing":19967,"##ml":19968,"##ision":19969,"heaved":19970,"tammy":19971,"consulate":19972,"attendees":19973,"routinely":19974,"197":19975,"fuse":19976,"saxophonist":19977,"backseat":19978,"malaya":19979,"##lord":19980,"scowl":19981,"tau":19982,"##ishly":19983,"193":19984,"sighted":19985,"steaming":19986,"##rks":19987,"303":19988,"911":19989,"##holes":19990,"##hong":19991,"ching":19992,"##wife":19993,"bless":19994,"conserved":19995,"jurassic":19996,"stacey":19997,"unix":19998,"zion":19999,"chunk":20000,"rigorous":20001,"blaine":20002,"198":20003,"peabody":20004,"slayer":20005,"dismay":20006,"brewers":20007,"nz":20008,"##jer":20009,"det":20010,"##glia":20011,"glover":20012,"postwar":20013,"int":20014,"penetration":20015,"sylvester":20016,"imitation":20017,"vertically":20018,"airlift":20019,"heiress":20020,"knoxville":20021,"viva":20022,"##uin":20023,"390":20024,"macon":20025,"##rim":20026,"##fighter":20027,"##gonal":20028,"janice":20029,"##orescence":20030,"##wari":20031,"marius":20032,"belongings":20033,"leicestershire":20034,"196":20035,"blanco":20036,"inverted":20037,"preseason":20038,"sanity":20039,"sobbing":20040,"##due":20041,"##elt":20042,"##dled":20043,"collingwood":20044,"regeneration":20045,"flickering":20046,"shortest":20047,"##mount":20048,"##osi":20049,"feminism":20050,"##lat":20051,"sherlock":20052,"cabinets":20053,"fumbled":20054,"northbound":20055,"precedent":20056,"snaps":20057,"##mme":20058,"researching":20059,"##akes":20060,"guillaume":20061,"insights":20062,"manipulated":20063,"vapor":20064,"neighbour":20065,"sap":20066,"gangster":20067,"frey":20068,"f1":20069,"stalking":20070,"scarcely":20071,"callie":20072,"barnett":20073,"tendencies":20074,"audi":20075,"doomed":20076,"assessing":20077,"slung":20078,"panchayat":20079,"ambiguous":20080,"bartlett":20081,"##etto":20082,"distributing":20083,"violating":20084,"wolverhampton":20085,"##hetic":20086,"swami":20087,"histoire":20088,"##urus":20089,"liable":20090,"pounder":20091,"groin":20092,"hussain":20093,"larsen":20094,"popping":20095,"surprises":20096,"##atter":20097,"vie":20098,"curt":20099,"##station":20100,"mute":20101,"relocate":20102,"musicals":20103,"authorization":20104,"richter":20105,"##sef":20106,"immortality":20107,"tna":20108,"bombings":20109,"##press":20110,"deteriorated":20111,"yiddish":20112,"##acious":20113,"robbed":20114,"colchester":20115,"cs":20116,"pmid":20117,"ao":20118,"verified":20119,"balancing":20120,"apostle":20121,"swayed":20122,"recognizable":20123,"oxfordshire":20124,"retention":20125,"nottinghamshire":20126,"contender":20127,"judd":20128,"invitational":20129,"shrimp":20130,"uhf":20131,"##icient":20132,"cleaner":20133,"longitudinal":20134,"tanker":20135,"##mur":20136,"acronym":20137,"broker":20138,"koppen":20139,"sundance":20140,"suppliers":20141,"##gil":20142,"4000":20143,"clipped":20144,"fuels":20145,"petite":20146,"##anne":20147,"landslide":20148,"helene":20149,"diversion":20150,"populous":20151,"landowners":20152,"auspices":20153,"melville":20154,"quantitative":20155,"##xes":20156,"ferries":20157,"nicky":20158,"##llus":20159,"doo":20160,"haunting":20161,"roche":20162,"carver":20163,"downed":20164,"unavailable":20165,"##pathy":20166,"approximation":20167,"hiroshima":20168,"##hue":20169,"garfield":20170,"valle":20171,"comparatively":20172,"keyboardist":20173,"traveler":20174,"##eit":20175,"congestion":20176,"calculating":20177,"subsidiaries":20178,"##bate":20179,"serb":20180,"modernization":20181,"fairies":20182,"deepened":20183,"ville":20184,"averages":20185,"##lore":20186,"inflammatory":20187,"tonga":20188,"##itch":20189,"co₂":20190,"squads":20191,"##hea":20192,"gigantic":20193,"serum":20194,"enjoyment":20195,"retailer":20196,"verona":20197,"35th":20198,"cis":20199,"##phobic":20200,"magna":20201,"technicians":20202,"##vati":20203,"arithmetic":20204,"##sport":20205,"levin":20206,"##dation":20207,"amtrak":20208,"chow":20209,"sienna":20210,"##eyer":20211,"backstage":20212,"entrepreneurship":20213,"##otic":20214,"learnt":20215,"tao":20216,"##udy":20217,"worcestershire":20218,"formulation":20219,"baggage":20220,"hesitant":20221,"bali":20222,"sabotage":20223,"##kari":20224,"barren":20225,"enhancing":20226,"murmur":20227,"pl":20228,"freshly":20229,"putnam":20230,"syntax":20231,"aces":20232,"medicines":20233,"resentment":20234,"bandwidth":20235,"##sier":20236,"grins":20237,"chili":20238,"guido":20239,"##sei":20240,"framing":20241,"implying":20242,"gareth":20243,"lissa":20244,"genevieve":20245,"pertaining":20246,"admissions":20247,"geo":20248,"thorpe":20249,"proliferation":20250,"sato":20251,"bela":20252,"analyzing":20253,"parting":20254,"##gor":20255,"awakened":20256,"##isman":20257,"huddled":20258,"secrecy":20259,"##kling":20260,"hush":20261,"gentry":20262,"540":20263,"dungeons":20264,"##ego":20265,"coasts":20266,"##utz":20267,"sacrificed":20268,"##chule":20269,"landowner":20270,"mutually":20271,"prevalence":20272,"programmer":20273,"adolescent":20274,"disrupted":20275,"seaside":20276,"gee":20277,"trusts":20278,"vamp":20279,"georgie":20280,"##nesian":20281,"##iol":20282,"schedules":20283,"sindh":20284,"##market":20285,"etched":20286,"hm":20287,"sparse":20288,"bey":20289,"beaux":20290,"scratching":20291,"gliding":20292,"unidentified":20293,"216":20294,"collaborating":20295,"gems":20296,"jesuits":20297,"oro":20298,"accumulation":20299,"shaping":20300,"mbe":20301,"anal":20302,"##xin":20303,"231":20304,"enthusiasts":20305,"newscast":20306,"##egan":20307,"janata":20308,"dewey":20309,"parkinson":20310,"179":20311,"ankara":20312,"biennial":20313,"towering":20314,"dd":20315,"inconsistent":20316,"950":20317,"##chet":20318,"thriving":20319,"terminate":20320,"cabins":20321,"furiously":20322,"eats":20323,"advocating":20324,"donkey":20325,"marley":20326,"muster":20327,"phyllis":20328,"leiden":20329,"##user":20330,"grassland":20331,"glittering":20332,"iucn":20333,"loneliness":20334,"217":20335,"memorandum":20336,"armenians":20337,"##ddle":20338,"popularized":20339,"rhodesia":20340,"60s":20341,"lame":20342,"##illon":20343,"sans":20344,"bikini":20345,"header":20346,"orbits":20347,"##xx":20348,"##finger":20349,"##ulator":20350,"sharif":20351,"spines":20352,"biotechnology":20353,"strolled":20354,"naughty":20355,"yates":20356,"##wire":20357,"fremantle":20358,"milo":20359,"##mour":20360,"abducted":20361,"removes":20362,"##atin":20363,"humming":20364,"wonderland":20365,"##chrome":20366,"##ester":20367,"hume":20368,"pivotal":20369,"##rates":20370,"armand":20371,"grams":20372,"believers":20373,"elector":20374,"rte":20375,"apron":20376,"bis":20377,"scraped":20378,"##yria":20379,"endorsement":20380,"initials":20381,"##llation":20382,"eps":20383,"dotted":20384,"hints":20385,"buzzing":20386,"emigration":20387,"nearer":20388,"##tom":20389,"indicators":20390,"##ulu":20391,"coarse":20392,"neutron":20393,"protectorate":20394,"##uze":20395,"directional":20396,"exploits":20397,"pains":20398,"loire":20399,"1830s":20400,"proponents":20401,"guggenheim":20402,"rabbits":20403,"ritchie":20404,"305":20405,"hectare":20406,"inputs":20407,"hutton":20408,"##raz":20409,"verify":20410,"##ako":20411,"boilers":20412,"longitude":20413,"##lev":20414,"skeletal":20415,"yer":20416,"emilia":20417,"citrus":20418,"compromised":20419,"##gau":20420,"pokemon":20421,"prescription":20422,"paragraph":20423,"eduard":20424,"cadillac":20425,"attire":20426,"categorized":20427,"kenyan":20428,"weddings":20429,"charley":20430,"##bourg":20431,"entertain":20432,"monmouth":20433,"##lles":20434,"nutrients":20435,"davey":20436,"mesh":20437,"incentive":20438,"practised":20439,"ecosystems":20440,"kemp":20441,"subdued":20442,"overheard":20443,"##rya":20444,"bodily":20445,"maxim":20446,"##nius":20447,"apprenticeship":20448,"ursula":20449,"##fight":20450,"lodged":20451,"rug":20452,"silesian":20453,"unconstitutional":20454,"patel":20455,"inspected":20456,"coyote":20457,"unbeaten":20458,"##hak":20459,"34th":20460,"disruption":20461,"convict":20462,"parcel":20463,"##cl":20464,"##nham":20465,"collier":20466,"implicated":20467,"mallory":20468,"##iac":20469,"##lab":20470,"susannah":20471,"winkler":20472,"##rber":20473,"shia":20474,"phelps":20475,"sediments":20476,"graphical":20477,"robotic":20478,"##sner":20479,"adulthood":20480,"mart":20481,"smoked":20482,"##isto":20483,"kathryn":20484,"clarified":20485,"##aran":20486,"divides":20487,"convictions":20488,"oppression":20489,"pausing":20490,"burying":20491,"##mt":20492,"federico":20493,"mathias":20494,"eileen":20495,"##tana":20496,"kite":20497,"hunched":20498,"##acies":20499,"189":20500,"##atz":20501,"disadvantage":20502,"liza":20503,"kinetic":20504,"greedy":20505,"paradox":20506,"yokohama":20507,"dowager":20508,"trunks":20509,"ventured":20510,"##gement":20511,"gupta":20512,"vilnius":20513,"olaf":20514,"##thest":20515,"crimean":20516,"hopper":20517,"##ej":20518,"progressively":20519,"arturo":20520,"mouthed":20521,"arrondissement":20522,"##fusion":20523,"rubin":20524,"simulcast":20525,"oceania":20526,"##orum":20527,"##stra":20528,"##rred":20529,"busiest":20530,"intensely":20531,"navigator":20532,"cary":20533,"##vine":20534,"##hini":20535,"##bies":20536,"fife":20537,"rowe":20538,"rowland":20539,"posing":20540,"insurgents":20541,"shafts":20542,"lawsuits":20543,"activate":20544,"conor":20545,"inward":20546,"culturally":20547,"garlic":20548,"265":20549,"##eering":20550,"eclectic":20551,"##hui":20552,"##kee":20553,"##nl":20554,"furrowed":20555,"vargas":20556,"meteorological":20557,"rendezvous":20558,"##aus":20559,"culinary":20560,"commencement":20561,"##dition":20562,"quota":20563,"##notes":20564,"mommy":20565,"salaries":20566,"overlapping":20567,"mule":20568,"##iology":20569,"##mology":20570,"sums":20571,"wentworth":20572,"##isk":20573,"##zione":20574,"mainline":20575,"subgroup":20576,"##illy":20577,"hack":20578,"plaintiff":20579,"verdi":20580,"bulb":20581,"differentiation":20582,"engagements":20583,"multinational":20584,"supplemented":20585,"bertrand":20586,"caller":20587,"regis":20588,"##naire":20589,"##sler":20590,"##arts":20591,"##imated":20592,"blossom":20593,"propagation":20594,"kilometer":20595,"viaduct":20596,"vineyards":20597,"##uate":20598,"beckett":20599,"optimization":20600,"golfer":20601,"songwriters":20602,"seminal":20603,"semitic":20604,"thud":20605,"volatile":20606,"evolving":20607,"ridley":20608,"##wley":20609,"trivial":20610,"distributions":20611,"scandinavia":20612,"jiang":20613,"##ject":20614,"wrestled":20615,"insistence":20616,"##dio":20617,"emphasizes":20618,"napkin":20619,"##ods":20620,"adjunct":20621,"rhyme":20622,"##ricted":20623,"##eti":20624,"hopeless":20625,"surrounds":20626,"tremble":20627,"32nd":20628,"smoky":20629,"##ntly":20630,"oils":20631,"medicinal":20632,"padded":20633,"steer":20634,"wilkes":20635,"219":20636,"255":20637,"concessions":20638,"hue":20639,"uniquely":20640,"blinded":20641,"landon":20642,"yahoo":20643,"##lane":20644,"hendrix":20645,"commemorating":20646,"dex":20647,"specify":20648,"chicks":20649,"##ggio":20650,"intercity":20651,"1400":20652,"morley":20653,"##torm":20654,"highlighting":20655,"##oting":20656,"pang":20657,"oblique":20658,"stalled":20659,"##liner":20660,"flirting":20661,"newborn":20662,"1769":20663,"bishopric":20664,"shaved":20665,"232":20666,"currie":20667,"##ush":20668,"dharma":20669,"spartan":20670,"##ooped":20671,"favorites":20672,"smug":20673,"novella":20674,"sirens":20675,"abusive":20676,"creations":20677,"espana":20678,"##lage":20679,"paradigm":20680,"semiconductor":20681,"sheen":20682,"##rdo":20683,"##yen":20684,"##zak":20685,"nrl":20686,"renew":20687,"##pose":20688,"##tur":20689,"adjutant":20690,"marches":20691,"norma":20692,"##enity":20693,"ineffective":20694,"weimar":20695,"grunt":20696,"##gat":20697,"lordship":20698,"plotting":20699,"expenditure":20700,"infringement":20701,"lbs":20702,"refrain":20703,"av":20704,"mimi":20705,"mistakenly":20706,"postmaster":20707,"1771":20708,"##bara":20709,"ras":20710,"motorsports":20711,"tito":20712,"199":20713,"subjective":20714,"##zza":20715,"bully":20716,"stew":20717,"##kaya":20718,"prescott":20719,"1a":20720,"##raphic":20721,"##zam":20722,"bids":20723,"styling":20724,"paranormal":20725,"reeve":20726,"sneaking":20727,"exploding":20728,"katz":20729,"akbar":20730,"migrant":20731,"syllables":20732,"indefinitely":20733,"##ogical":20734,"destroys":20735,"replaces":20736,"applause":20737,"##phine":20738,"pest":20739,"##fide":20740,"218":20741,"articulated":20742,"bertie":20743,"##thing":20744,"##cars":20745,"##ptic":20746,"courtroom":20747,"crowley":20748,"aesthetics":20749,"cummings":20750,"tehsil":20751,"hormones":20752,"titanic":20753,"dangerously":20754,"##ibe":20755,"stadion":20756,"jaenelle":20757,"auguste":20758,"ciudad":20759,"##chu":20760,"mysore":20761,"partisans":20762,"##sio":20763,"lucan":20764,"philipp":20765,"##aly":20766,"debating":20767,"henley":20768,"interiors":20769,"##rano":20770,"##tious":20771,"homecoming":20772,"beyonce":20773,"usher":20774,"henrietta":20775,"prepares":20776,"weeds":20777,"##oman":20778,"ely":20779,"plucked":20780,"##pire":20781,"##dable":20782,"luxurious":20783,"##aq":20784,"artifact":20785,"password":20786,"pasture":20787,"juno":20788,"maddy":20789,"minsk":20790,"##dder":20791,"##ologies":20792,"##rone":20793,"assessments":20794,"martian":20795,"royalist":20796,"1765":20797,"examines":20798,"##mani":20799,"##rge":20800,"nino":20801,"223":20802,"parry":20803,"scooped":20804,"relativity":20805,"##eli":20806,"##uting":20807,"##cao":20808,"congregational":20809,"noisy":20810,"traverse":20811,"##agawa":20812,"strikeouts":20813,"nickelodeon":20814,"obituary":20815,"transylvania":20816,"binds":20817,"depictions":20818,"polk":20819,"trolley":20820,"##yed":20821,"##lard":20822,"breeders":20823,"##under":20824,"dryly":20825,"hokkaido":20826,"1762":20827,"strengths":20828,"stacks":20829,"bonaparte":20830,"connectivity":20831,"neared":20832,"prostitutes":20833,"stamped":20834,"anaheim":20835,"gutierrez":20836,"sinai":20837,"##zzling":20838,"bram":20839,"fresno":20840,"madhya":20841,"##86":20842,"proton":20843,"##lena":20844,"##llum":20845,"##phon":20846,"reelected":20847,"wanda":20848,"##anus":20849,"##lb":20850,"ample":20851,"distinguishing":20852,"##yler":20853,"grasping":20854,"sermons":20855,"tomato":20856,"bland":20857,"stimulation":20858,"avenues":20859,"##eux":20860,"spreads":20861,"scarlett":20862,"fern":20863,"pentagon":20864,"assert":20865,"baird":20866,"chesapeake":20867,"ir":20868,"calmed":20869,"distortion":20870,"fatalities":20871,"##olis":20872,"correctional":20873,"pricing":20874,"##astic":20875,"##gina":20876,"prom":20877,"dammit":20878,"ying":20879,"collaborate":20880,"##chia":20881,"welterweight":20882,"33rd":20883,"pointer":20884,"substitution":20885,"bonded":20886,"umpire":20887,"communicating":20888,"multitude":20889,"paddle":20890,"##obe":20891,"federally":20892,"intimacy":20893,"##insky":20894,"betray":20895,"ssr":20896,"##lett":20897,"##lean":20898,"##lves":20899,"##therapy":20900,"airbus":20901,"##tery":20902,"functioned":20903,"ud":20904,"bearer":20905,"biomedical":20906,"netflix":20907,"##hire":20908,"##nca":20909,"condom":20910,"brink":20911,"ik":20912,"##nical":20913,"macy":20914,"##bet":20915,"flap":20916,"gma":20917,"experimented":20918,"jelly":20919,"lavender":20920,"##icles":20921,"##ulia":20922,"munro":20923,"##mian":20924,"##tial":20925,"rye":20926,"##rle":20927,"60th":20928,"gigs":20929,"hottest":20930,"rotated":20931,"predictions":20932,"fuji":20933,"bu":20934,"##erence":20935,"##omi":20936,"barangay":20937,"##fulness":20938,"##sas":20939,"clocks":20940,"##rwood":20941,"##liness":20942,"cereal":20943,"roe":20944,"wight":20945,"decker":20946,"uttered":20947,"babu":20948,"onion":20949,"xml":20950,"forcibly":20951,"##df":20952,"petra":20953,"sarcasm":20954,"hartley":20955,"peeled":20956,"storytelling":20957,"##42":20958,"##xley":20959,"##ysis":20960,"##ffa":20961,"fibre":20962,"kiel":20963,"auditor":20964,"fig":20965,"harald":20966,"greenville":20967,"##berries":20968,"geographically":20969,"nell":20970,"quartz":20971,"##athic":20972,"cemeteries":20973,"##lr":20974,"crossings":20975,"nah":20976,"holloway":20977,"reptiles":20978,"chun":20979,"sichuan":20980,"snowy":20981,"660":20982,"corrections":20983,"##ivo":20984,"zheng":20985,"ambassadors":20986,"blacksmith":20987,"fielded":20988,"fluids":20989,"hardcover":20990,"turnover":20991,"medications":20992,"melvin":20993,"academies":20994,"##erton":20995,"ro":20996,"roach":20997,"absorbing":20998,"spaniards":20999,"colton":21000,"##founded":21001,"outsider":21002,"espionage":21003,"kelsey":21004,"245":21005,"edible":21006,"##ulf":21007,"dora":21008,"establishes":21009,"##sham":21010,"##tries":21011,"contracting":21012,"##tania":21013,"cinematic":21014,"costello":21015,"nesting":21016,"##uron":21017,"connolly":21018,"duff":21019,"##nology":21020,"mma":21021,"##mata":21022,"fergus":21023,"sexes":21024,"gi":21025,"optics":21026,"spectator":21027,"woodstock":21028,"banning":21029,"##hee":21030,"##fle":21031,"differentiate":21032,"outfielder":21033,"refinery":21034,"226":21035,"312":21036,"gerhard":21037,"horde":21038,"lair":21039,"drastically":21040,"##udi":21041,"landfall":21042,"##cheng":21043,"motorsport":21044,"odi":21045,"##achi":21046,"predominant":21047,"quay":21048,"skins":21049,"##ental":21050,"edna":21051,"harshly":21052,"complementary":21053,"murdering":21054,"##aves":21055,"wreckage":21056,"##90":21057,"ono":21058,"outstretched":21059,"lennox":21060,"munitions":21061,"galen":21062,"reconcile":21063,"470":21064,"scalp":21065,"bicycles":21066,"gillespie":21067,"questionable":21068,"rosenberg":21069,"guillermo":21070,"hostel":21071,"jarvis":21072,"kabul":21073,"volvo":21074,"opium":21075,"yd":21076,"##twined":21077,"abuses":21078,"decca":21079,"outpost":21080,"##cino":21081,"sensible":21082,"neutrality":21083,"##64":21084,"ponce":21085,"anchorage":21086,"atkins":21087,"turrets":21088,"inadvertently":21089,"disagree":21090,"libre":21091,"vodka":21092,"reassuring":21093,"weighs":21094,"##yal":21095,"glide":21096,"jumper":21097,"ceilings":21098,"repertory":21099,"outs":21100,"stain":21101,"##bial":21102,"envy":21103,"##ucible":21104,"smashing":21105,"heightened":21106,"policing":21107,"hyun":21108,"mixes":21109,"lai":21110,"prima":21111,"##ples":21112,"celeste":21113,"##bina":21114,"lucrative":21115,"intervened":21116,"kc":21117,"manually":21118,"##rned":21119,"stature":21120,"staffed":21121,"bun":21122,"bastards":21123,"nairobi":21124,"priced":21125,"##auer":21126,"thatcher":21127,"##kia":21128,"tripped":21129,"comune":21130,"##ogan":21131,"##pled":21132,"brasil":21133,"incentives":21134,"emanuel":21135,"hereford":21136,"musica":21137,"##kim":21138,"benedictine":21139,"biennale":21140,"##lani":21141,"eureka":21142,"gardiner":21143,"rb":21144,"knocks":21145,"sha":21146,"##ael":21147,"##elled":21148,"##onate":21149,"efficacy":21150,"ventura":21151,"masonic":21152,"sanford":21153,"maize":21154,"leverage":21155,"##feit":21156,"capacities":21157,"santana":21158,"##aur":21159,"novelty":21160,"vanilla":21161,"##cter":21162,"##tour":21163,"benin":21164,"##oir":21165,"##rain":21166,"neptune":21167,"drafting":21168,"tallinn":21169,"##cable":21170,"humiliation":21171,"##boarding":21172,"schleswig":21173,"fabian":21174,"bernardo":21175,"liturgy":21176,"spectacle":21177,"sweeney":21178,"pont":21179,"routledge":21180,"##tment":21181,"cosmos":21182,"ut":21183,"hilt":21184,"sleek":21185,"universally":21186,"##eville":21187,"##gawa":21188,"typed":21189,"##dry":21190,"favors":21191,"allegheny":21192,"glaciers":21193,"##rly":21194,"recalling":21195,"aziz":21196,"##log":21197,"parasite":21198,"requiem":21199,"auf":21200,"##berto":21201,"##llin":21202,"illumination":21203,"##breaker":21204,"##issa":21205,"festivities":21206,"bows":21207,"govern":21208,"vibe":21209,"vp":21210,"333":21211,"sprawled":21212,"larson":21213,"pilgrim":21214,"bwf":21215,"leaping":21216,"##rts":21217,"##ssel":21218,"alexei":21219,"greyhound":21220,"hoarse":21221,"##dler":21222,"##oration":21223,"seneca":21224,"##cule":21225,"gaping":21226,"##ulously":21227,"##pura":21228,"cinnamon":21229,"##gens":21230,"##rricular":21231,"craven":21232,"fantasies":21233,"houghton":21234,"engined":21235,"reigned":21236,"dictator":21237,"supervising":21238,"##oris":21239,"bogota":21240,"commentaries":21241,"unnatural":21242,"fingernails":21243,"spirituality":21244,"tighten":21245,"##tm":21246,"canadiens":21247,"protesting":21248,"intentional":21249,"cheers":21250,"sparta":21251,"##ytic":21252,"##iere":21253,"##zine":21254,"widen":21255,"belgarath":21256,"controllers":21257,"dodd":21258,"iaaf":21259,"navarre":21260,"##ication":21261,"defect":21262,"squire":21263,"steiner":21264,"whisky":21265,"##mins":21266,"560":21267,"inevitably":21268,"tome":21269,"##gold":21270,"chew":21271,"##uid":21272,"##lid":21273,"elastic":21274,"##aby":21275,"streaked":21276,"alliances":21277,"jailed":21278,"regal":21279,"##ined":21280,"##phy":21281,"czechoslovak":21282,"narration":21283,"absently":21284,"##uld":21285,"bluegrass":21286,"guangdong":21287,"quran":21288,"criticizing":21289,"hose":21290,"hari":21291,"##liest":21292,"##owa":21293,"skier":21294,"streaks":21295,"deploy":21296,"##lom":21297,"raft":21298,"bose":21299,"dialed":21300,"huff":21301,"##eira":21302,"haifa":21303,"simplest":21304,"bursting":21305,"endings":21306,"ib":21307,"sultanate":21308,"##titled":21309,"franks":21310,"whitman":21311,"ensures":21312,"sven":21313,"##ggs":21314,"collaborators":21315,"forster":21316,"organising":21317,"ui":21318,"banished":21319,"napier":21320,"injustice":21321,"teller":21322,"layered":21323,"thump":21324,"##otti":21325,"roc":21326,"battleships":21327,"evidenced":21328,"fugitive":21329,"sadie":21330,"robotics":21331,"##roud":21332,"equatorial":21333,"geologist":21334,"##iza":21335,"yielding":21336,"##bron":21337,"##sr":21338,"internationale":21339,"mecca":21340,"##diment":21341,"sbs":21342,"skyline":21343,"toad":21344,"uploaded":21345,"reflective":21346,"undrafted":21347,"lal":21348,"leafs":21349,"bayern":21350,"##dai":21351,"lakshmi":21352,"shortlisted":21353,"##stick":21354,"##wicz":21355,"camouflage":21356,"donate":21357,"af":21358,"christi":21359,"lau":21360,"##acio":21361,"disclosed":21362,"nemesis":21363,"1761":21364,"assemble":21365,"straining":21366,"northamptonshire":21367,"tal":21368,"##asi":21369,"bernardino":21370,"premature":21371,"heidi":21372,"42nd":21373,"coefficients":21374,"galactic":21375,"reproduce":21376,"buzzed":21377,"sensations":21378,"zionist":21379,"monsieur":21380,"myrtle":21381,"##eme":21382,"archery":21383,"strangled":21384,"musically":21385,"viewpoint":21386,"antiquities":21387,"bei":21388,"trailers":21389,"seahawks":21390,"cured":21391,"pee":21392,"preferring":21393,"tasmanian":21394,"lange":21395,"sul":21396,"##mail":21397,"##working":21398,"colder":21399,"overland":21400,"lucivar":21401,"massey":21402,"gatherings":21403,"haitian":21404,"##smith":21405,"disapproval":21406,"flaws":21407,"##cco":21408,"##enbach":21409,"1766":21410,"npr":21411,"##icular":21412,"boroughs":21413,"creole":21414,"forums":21415,"techno":21416,"1755":21417,"dent":21418,"abdominal":21419,"streetcar":21420,"##eson":21421,"##stream":21422,"procurement":21423,"gemini":21424,"predictable":21425,"##tya":21426,"acheron":21427,"christoph":21428,"feeder":21429,"fronts":21430,"vendor":21431,"bernhard":21432,"jammu":21433,"tumors":21434,"slang":21435,"##uber":21436,"goaltender":21437,"twists":21438,"curving":21439,"manson":21440,"vuelta":21441,"mer":21442,"peanut":21443,"confessions":21444,"pouch":21445,"unpredictable":21446,"allowance":21447,"theodor":21448,"vascular":21449,"##factory":21450,"bala":21451,"authenticity":21452,"metabolic":21453,"coughing":21454,"nanjing":21455,"##cea":21456,"pembroke":21457,"##bard":21458,"splendid":21459,"36th":21460,"ff":21461,"hourly":21462,"##ahu":21463,"elmer":21464,"handel":21465,"##ivate":21466,"awarding":21467,"thrusting":21468,"dl":21469,"experimentation":21470,"##hesion":21471,"##46":21472,"caressed":21473,"entertained":21474,"steak":21475,"##rangle":21476,"biologist":21477,"orphans":21478,"baroness":21479,"oyster":21480,"stepfather":21481,"##dridge":21482,"mirage":21483,"reefs":21484,"speeding":21485,"##31":21486,"barons":21487,"1764":21488,"227":21489,"inhabit":21490,"preached":21491,"repealed":21492,"##tral":21493,"honoring":21494,"boogie":21495,"captives":21496,"administer":21497,"johanna":21498,"##imate":21499,"gel":21500,"suspiciously":21501,"1767":21502,"sobs":21503,"##dington":21504,"backbone":21505,"hayward":21506,"garry":21507,"##folding":21508,"##nesia":21509,"maxi":21510,"##oof":21511,"##ppe":21512,"ellison":21513,"galileo":21514,"##stand":21515,"crimea":21516,"frenzy":21517,"amour":21518,"bumper":21519,"matrices":21520,"natalia":21521,"baking":21522,"garth":21523,"palestinians":21524,"##grove":21525,"smack":21526,"conveyed":21527,"ensembles":21528,"gardening":21529,"##manship":21530,"##rup":21531,"##stituting":21532,"1640":21533,"harvesting":21534,"topography":21535,"jing":21536,"shifters":21537,"dormitory":21538,"##carriage":21539,"##lston":21540,"ist":21541,"skulls":21542,"##stadt":21543,"dolores":21544,"jewellery":21545,"sarawak":21546,"##wai":21547,"##zier":21548,"fences":21549,"christy":21550,"confinement":21551,"tumbling":21552,"credibility":21553,"fir":21554,"stench":21555,"##bria":21556,"##plication":21557,"##nged":21558,"##sam":21559,"virtues":21560,"##belt":21561,"marjorie":21562,"pba":21563,"##eem":21564,"##made":21565,"celebrates":21566,"schooner":21567,"agitated":21568,"barley":21569,"fulfilling":21570,"anthropologist":21571,"##pro":21572,"restrict":21573,"novi":21574,"regulating":21575,"##nent":21576,"padres":21577,"##rani":21578,"##hesive":21579,"loyola":21580,"tabitha":21581,"milky":21582,"olson":21583,"proprietor":21584,"crambidae":21585,"guarantees":21586,"intercollegiate":21587,"ljubljana":21588,"hilda":21589,"##sko":21590,"ignorant":21591,"hooded":21592,"##lts":21593,"sardinia":21594,"##lidae":21595,"##vation":21596,"frontman":21597,"privileged":21598,"witchcraft":21599,"##gp":21600,"jammed":21601,"laude":21602,"poking":21603,"##than":21604,"bracket":21605,"amazement":21606,"yunnan":21607,"##erus":21608,"maharaja":21609,"linnaeus":21610,"264":21611,"commissioning":21612,"milano":21613,"peacefully":21614,"##logies":21615,"akira":21616,"rani":21617,"regulator":21618,"##36":21619,"grasses":21620,"##rance":21621,"luzon":21622,"crows":21623,"compiler":21624,"gretchen":21625,"seaman":21626,"edouard":21627,"tab":21628,"buccaneers":21629,"ellington":21630,"hamlets":21631,"whig":21632,"socialists":21633,"##anto":21634,"directorial":21635,"easton":21636,"mythological":21637,"##kr":21638,"##vary":21639,"rhineland":21640,"semantic":21641,"taut":21642,"dune":21643,"inventions":21644,"succeeds":21645,"##iter":21646,"replication":21647,"branched":21648,"##pired":21649,"jul":21650,"prosecuted":21651,"kangaroo":21652,"penetrated":21653,"##avian":21654,"middlesbrough":21655,"doses":21656,"bleak":21657,"madam":21658,"predatory":21659,"relentless":21660,"##vili":21661,"reluctance":21662,"##vir":21663,"hailey":21664,"crore":21665,"silvery":21666,"1759":21667,"monstrous":21668,"swimmers":21669,"transmissions":21670,"hawthorn":21671,"informing":21672,"##eral":21673,"toilets":21674,"caracas":21675,"crouch":21676,"kb":21677,"##sett":21678,"295":21679,"cartel":21680,"hadley":21681,"##aling":21682,"alexia":21683,"yvonne":21684,"##biology":21685,"cinderella":21686,"eton":21687,"superb":21688,"blizzard":21689,"stabbing":21690,"industrialist":21691,"maximus":21692,"##gm":21693,"##orus":21694,"groves":21695,"maud":21696,"clade":21697,"oversized":21698,"comedic":21699,"##bella":21700,"rosen":21701,"nomadic":21702,"fulham":21703,"montane":21704,"beverages":21705,"galaxies":21706,"redundant":21707,"swarm":21708,"##rot":21709,"##folia":21710,"##llis":21711,"buckinghamshire":21712,"fen":21713,"bearings":21714,"bahadur":21715,"##rom":21716,"gilles":21717,"phased":21718,"dynamite":21719,"faber":21720,"benoit":21721,"vip":21722,"##ount":21723,"##wd":21724,"booking":21725,"fractured":21726,"tailored":21727,"anya":21728,"spices":21729,"westwood":21730,"cairns":21731,"auditions":21732,"inflammation":21733,"steamed":21734,"##rocity":21735,"##acion":21736,"##urne":21737,"skyla":21738,"thereof":21739,"watford":21740,"torment":21741,"archdeacon":21742,"transforms":21743,"lulu":21744,"demeanor":21745,"fucked":21746,"serge":21747,"##sor":21748,"mckenna":21749,"minas":21750,"entertainer":21751,"##icide":21752,"caress":21753,"originate":21754,"residue":21755,"##sty":21756,"1740":21757,"##ilised":21758,"##org":21759,"beech":21760,"##wana":21761,"subsidies":21762,"##ghton":21763,"emptied":21764,"gladstone":21765,"ru":21766,"firefighters":21767,"voodoo":21768,"##rcle":21769,"het":21770,"nightingale":21771,"tamara":21772,"edmond":21773,"ingredient":21774,"weaknesses":21775,"silhouette":21776,"285":21777,"compatibility":21778,"withdrawing":21779,"hampson":21780,"##mona":21781,"anguish":21782,"giggling":21783,"##mber":21784,"bookstore":21785,"##jiang":21786,"southernmost":21787,"tilting":21788,"##vance":21789,"bai":21790,"economical":21791,"rf":21792,"briefcase":21793,"dreadful":21794,"hinted":21795,"projections":21796,"shattering":21797,"totaling":21798,"##rogate":21799,"analogue":21800,"indicted":21801,"periodical":21802,"fullback":21803,"##dman":21804,"haynes":21805,"##tenberg":21806,"##ffs":21807,"##ishment":21808,"1745":21809,"thirst":21810,"stumble":21811,"penang":21812,"vigorous":21813,"##ddling":21814,"##kor":21815,"##lium":21816,"octave":21817,"##ove":21818,"##enstein":21819,"##inen":21820,"##ones":21821,"siberian":21822,"##uti":21823,"cbn":21824,"repeal":21825,"swaying":21826,"##vington":21827,"khalid":21828,"tanaka":21829,"unicorn":21830,"otago":21831,"plastered":21832,"lobe":21833,"riddle":21834,"##rella":21835,"perch":21836,"##ishing":21837,"croydon":21838,"filtered":21839,"graeme":21840,"tripoli":21841,"##ossa":21842,"crocodile":21843,"##chers":21844,"sufi":21845,"mined":21846,"##tung":21847,"inferno":21848,"lsu":21849,"##phi":21850,"swelled":21851,"utilizes":21852,"£2":21853,"cale":21854,"periodicals":21855,"styx":21856,"hike":21857,"informally":21858,"coop":21859,"lund":21860,"##tidae":21861,"ala":21862,"hen":21863,"qui":21864,"transformations":21865,"disposed":21866,"sheath":21867,"chickens":21868,"##cade":21869,"fitzroy":21870,"sas":21871,"silesia":21872,"unacceptable":21873,"odisha":21874,"1650":21875,"sabrina":21876,"pe":21877,"spokane":21878,"ratios":21879,"athena":21880,"massage":21881,"shen":21882,"dilemma":21883,"##drum":21884,"##riz":21885,"##hul":21886,"corona":21887,"doubtful":21888,"niall":21889,"##pha":21890,"##bino":21891,"fines":21892,"cite":21893,"acknowledging":21894,"bangor":21895,"ballard":21896,"bathurst":21897,"##resh":21898,"huron":21899,"mustered":21900,"alzheimer":21901,"garments":21902,"kinase":21903,"tyre":21904,"warship":21905,"##cp":21906,"flashback":21907,"pulmonary":21908,"braun":21909,"cheat":21910,"kamal":21911,"cyclists":21912,"constructions":21913,"grenades":21914,"ndp":21915,"traveller":21916,"excuses":21917,"stomped":21918,"signalling":21919,"trimmed":21920,"futsal":21921,"mosques":21922,"relevance":21923,"##wine":21924,"wta":21925,"##23":21926,"##vah":21927,"##lter":21928,"hoc":21929,"##riding":21930,"optimistic":21931,"##´s":21932,"deco":21933,"sim":21934,"interacting":21935,"rejecting":21936,"moniker":21937,"waterways":21938,"##ieri":21939,"##oku":21940,"mayors":21941,"gdansk":21942,"outnumbered":21943,"pearls":21944,"##ended":21945,"##hampton":21946,"fairs":21947,"totals":21948,"dominating":21949,"262":21950,"notions":21951,"stairway":21952,"compiling":21953,"pursed":21954,"commodities":21955,"grease":21956,"yeast":21957,"##jong":21958,"carthage":21959,"griffiths":21960,"residual":21961,"amc":21962,"contraction":21963,"laird":21964,"sapphire":21965,"##marine":21966,"##ivated":21967,"amalgamation":21968,"dissolve":21969,"inclination":21970,"lyle":21971,"packaged":21972,"altitudes":21973,"suez":21974,"canons":21975,"graded":21976,"lurched":21977,"narrowing":21978,"boasts":21979,"guise":21980,"wed":21981,"enrico":21982,"##ovsky":21983,"rower":21984,"scarred":21985,"bree":21986,"cub":21987,"iberian":21988,"protagonists":21989,"bargaining":21990,"proposing":21991,"trainers":21992,"voyages":21993,"vans":21994,"fishes":21995,"##aea":21996,"##ivist":21997,"##verance":21998,"encryption":21999,"artworks":22000,"kazan":22001,"sabre":22002,"cleopatra":22003,"hepburn":22004,"rotting":22005,"supremacy":22006,"mecklenburg":22007,"##brate":22008,"burrows":22009,"hazards":22010,"outgoing":22011,"flair":22012,"organizes":22013,"##ctions":22014,"scorpion":22015,"##usions":22016,"boo":22017,"234":22018,"chevalier":22019,"dunedin":22020,"slapping":22021,"##34":22022,"ineligible":22023,"pensions":22024,"##38":22025,"##omic":22026,"manufactures":22027,"emails":22028,"bismarck":22029,"238":22030,"weakening":22031,"blackish":22032,"ding":22033,"mcgee":22034,"quo":22035,"##rling":22036,"northernmost":22037,"xx":22038,"manpower":22039,"greed":22040,"sampson":22041,"clicking":22042,"##ange":22043,"##horpe":22044,"##inations":22045,"##roving":22046,"torre":22047,"##eptive":22048,"##moral":22049,"symbolism":22050,"38th":22051,"asshole":22052,"meritorious":22053,"outfits":22054,"splashed":22055,"biographies":22056,"sprung":22057,"astros":22058,"##tale":22059,"302":22060,"737":22061,"filly":22062,"raoul":22063,"nw":22064,"tokugawa":22065,"linden":22066,"clubhouse":22067,"##apa":22068,"tracts":22069,"romano":22070,"##pio":22071,"putin":22072,"tags":22073,"##note":22074,"chained":22075,"dickson":22076,"gunshot":22077,"moe":22078,"gunn":22079,"rashid":22080,"##tails":22081,"zipper":22082,"##bas":22083,"##nea":22084,"contrasted":22085,"##ply":22086,"##udes":22087,"plum":22088,"pharaoh":22089,"##pile":22090,"aw":22091,"comedies":22092,"ingrid":22093,"sandwiches":22094,"subdivisions":22095,"1100":22096,"mariana":22097,"nokia":22098,"kamen":22099,"hz":22100,"delaney":22101,"veto":22102,"herring":22103,"##words":22104,"possessive":22105,"outlines":22106,"##roup":22107,"siemens":22108,"stairwell":22109,"rc":22110,"gallantry":22111,"messiah":22112,"palais":22113,"yells":22114,"233":22115,"zeppelin":22116,"##dm":22117,"bolivar":22118,"##cede":22119,"smackdown":22120,"mckinley":22121,"##mora":22122,"##yt":22123,"muted":22124,"geologic":22125,"finely":22126,"unitary":22127,"avatar":22128,"hamas":22129,"maynard":22130,"rees":22131,"bog":22132,"contrasting":22133,"##rut":22134,"liv":22135,"chico":22136,"disposition":22137,"pixel":22138,"##erate":22139,"becca":22140,"dmitry":22141,"yeshiva":22142,"narratives":22143,"##lva":22144,"##ulton":22145,"mercenary":22146,"sharpe":22147,"tempered":22148,"navigate":22149,"stealth":22150,"amassed":22151,"keynes":22152,"##lini":22153,"untouched":22154,"##rrie":22155,"havoc":22156,"lithium":22157,"##fighting":22158,"abyss":22159,"graf":22160,"southward":22161,"wolverine":22162,"balloons":22163,"implements":22164,"ngos":22165,"transitions":22166,"##icum":22167,"ambushed":22168,"concacaf":22169,"dormant":22170,"economists":22171,"##dim":22172,"costing":22173,"csi":22174,"rana":22175,"universite":22176,"boulders":22177,"verity":22178,"##llon":22179,"collin":22180,"mellon":22181,"misses":22182,"cypress":22183,"fluorescent":22184,"lifeless":22185,"spence":22186,"##ulla":22187,"crewe":22188,"shepard":22189,"pak":22190,"revelations":22191,"##م":22192,"jolly":22193,"gibbons":22194,"paw":22195,"##dro":22196,"##quel":22197,"freeing":22198,"##test":22199,"shack":22200,"fries":22201,"palatine":22202,"##51":22203,"##hiko":22204,"accompaniment":22205,"cruising":22206,"recycled":22207,"##aver":22208,"erwin":22209,"sorting":22210,"synthesizers":22211,"dyke":22212,"realities":22213,"sg":22214,"strides":22215,"enslaved":22216,"wetland":22217,"##ghan":22218,"competence":22219,"gunpowder":22220,"grassy":22221,"maroon":22222,"reactors":22223,"objection":22224,"##oms":22225,"carlson":22226,"gearbox":22227,"macintosh":22228,"radios":22229,"shelton":22230,"##sho":22231,"clergyman":22232,"prakash":22233,"254":22234,"mongols":22235,"trophies":22236,"oricon":22237,"228":22238,"stimuli":22239,"twenty20":22240,"cantonese":22241,"cortes":22242,"mirrored":22243,"##saurus":22244,"bhp":22245,"cristina":22246,"melancholy":22247,"##lating":22248,"enjoyable":22249,"nuevo":22250,"##wny":22251,"downfall":22252,"schumacher":22253,"##ind":22254,"banging":22255,"lausanne":22256,"rumbled":22257,"paramilitary":22258,"reflex":22259,"ax":22260,"amplitude":22261,"migratory":22262,"##gall":22263,"##ups":22264,"midi":22265,"barnard":22266,"lastly":22267,"sherry":22268,"##hp":22269,"##nall":22270,"keystone":22271,"##kra":22272,"carleton":22273,"slippery":22274,"##53":22275,"coloring":22276,"foe":22277,"socket":22278,"otter":22279,"##rgos":22280,"mats":22281,"##tose":22282,"consultants":22283,"bafta":22284,"bison":22285,"topping":22286,"##km":22287,"490":22288,"primal":22289,"abandonment":22290,"transplant":22291,"atoll":22292,"hideous":22293,"mort":22294,"pained":22295,"reproduced":22296,"tae":22297,"howling":22298,"##turn":22299,"unlawful":22300,"billionaire":22301,"hotter":22302,"poised":22303,"lansing":22304,"##chang":22305,"dinamo":22306,"retro":22307,"messing":22308,"nfc":22309,"domesday":22310,"##mina":22311,"blitz":22312,"timed":22313,"##athing":22314,"##kley":22315,"ascending":22316,"gesturing":22317,"##izations":22318,"signaled":22319,"tis":22320,"chinatown":22321,"mermaid":22322,"savanna":22323,"jameson":22324,"##aint":22325,"catalina":22326,"##pet":22327,"##hers":22328,"cochrane":22329,"cy":22330,"chatting":22331,"##kus":22332,"alerted":22333,"computation":22334,"mused":22335,"noelle":22336,"majestic":22337,"mohawk":22338,"campo":22339,"octagonal":22340,"##sant":22341,"##hend":22342,"241":22343,"aspiring":22344,"##mart":22345,"comprehend":22346,"iona":22347,"paralyzed":22348,"shimmering":22349,"swindon":22350,"rhone":22351,"##eley":22352,"reputed":22353,"configurations":22354,"pitchfork":22355,"agitation":22356,"francais":22357,"gillian":22358,"lipstick":22359,"##ilo":22360,"outsiders":22361,"pontifical":22362,"resisting":22363,"bitterness":22364,"sewer":22365,"rockies":22366,"##edd":22367,"##ucher":22368,"misleading":22369,"1756":22370,"exiting":22371,"galloway":22372,"##nging":22373,"risked":22374,"##heart":22375,"246":22376,"commemoration":22377,"schultz":22378,"##rka":22379,"integrating":22380,"##rsa":22381,"poses":22382,"shrieked":22383,"##weiler":22384,"guineas":22385,"gladys":22386,"jerking":22387,"owls":22388,"goldsmith":22389,"nightly":22390,"penetrating":22391,"##unced":22392,"lia":22393,"##33":22394,"ignited":22395,"betsy":22396,"##aring":22397,"##thorpe":22398,"follower":22399,"vigorously":22400,"##rave":22401,"coded":22402,"kiran":22403,"knit":22404,"zoology":22405,"tbilisi":22406,"##28":22407,"##bered":22408,"repository":22409,"govt":22410,"deciduous":22411,"dino":22412,"growling":22413,"##bba":22414,"enhancement":22415,"unleashed":22416,"chanting":22417,"pussy":22418,"biochemistry":22419,"##eric":22420,"kettle":22421,"repression":22422,"toxicity":22423,"nrhp":22424,"##arth":22425,"##kko":22426,"##bush":22427,"ernesto":22428,"commended":22429,"outspoken":22430,"242":22431,"mca":22432,"parchment":22433,"sms":22434,"kristen":22435,"##aton":22436,"bisexual":22437,"raked":22438,"glamour":22439,"navajo":22440,"a2":22441,"conditioned":22442,"showcased":22443,"##hma":22444,"spacious":22445,"youthful":22446,"##esa":22447,"usl":22448,"appliances":22449,"junta":22450,"brest":22451,"layne":22452,"conglomerate":22453,"enchanted":22454,"chao":22455,"loosened":22456,"picasso":22457,"circulating":22458,"inspect":22459,"montevideo":22460,"##centric":22461,"##kti":22462,"piazza":22463,"spurred":22464,"##aith":22465,"bari":22466,"freedoms":22467,"poultry":22468,"stamford":22469,"lieu":22470,"##ect":22471,"indigo":22472,"sarcastic":22473,"bahia":22474,"stump":22475,"attach":22476,"dvds":22477,"frankenstein":22478,"lille":22479,"approx":22480,"scriptures":22481,"pollen":22482,"##script":22483,"nmi":22484,"overseen":22485,"##ivism":22486,"tides":22487,"proponent":22488,"newmarket":22489,"inherit":22490,"milling":22491,"##erland":22492,"centralized":22493,"##rou":22494,"distributors":22495,"credentials":22496,"drawers":22497,"abbreviation":22498,"##lco":22499,"##xon":22500,"downing":22501,"uncomfortably":22502,"ripe":22503,"##oes":22504,"erase":22505,"franchises":22506,"##ever":22507,"populace":22508,"##bery":22509,"##khar":22510,"decomposition":22511,"pleas":22512,"##tet":22513,"daryl":22514,"sabah":22515,"##stle":22516,"##wide":22517,"fearless":22518,"genie":22519,"lesions":22520,"annette":22521,"##ogist":22522,"oboe":22523,"appendix":22524,"nair":22525,"dripped":22526,"petitioned":22527,"maclean":22528,"mosquito":22529,"parrot":22530,"rpg":22531,"hampered":22532,"1648":22533,"operatic":22534,"reservoirs":22535,"##tham":22536,"irrelevant":22537,"jolt":22538,"summarized":22539,"##fp":22540,"medallion":22541,"##taff":22542,"##−":22543,"clawed":22544,"harlow":22545,"narrower":22546,"goddard":22547,"marcia":22548,"bodied":22549,"fremont":22550,"suarez":22551,"altering":22552,"tempest":22553,"mussolini":22554,"porn":22555,"##isms":22556,"sweetly":22557,"oversees":22558,"walkers":22559,"solitude":22560,"grimly":22561,"shrines":22562,"hk":22563,"ich":22564,"supervisors":22565,"hostess":22566,"dietrich":22567,"legitimacy":22568,"brushes":22569,"expressive":22570,"##yp":22571,"dissipated":22572,"##rse":22573,"localized":22574,"systemic":22575,"##nikov":22576,"gettysburg":22577,"##js":22578,"##uaries":22579,"dialogues":22580,"muttering":22581,"251":22582,"housekeeper":22583,"sicilian":22584,"discouraged":22585,"##frey":22586,"beamed":22587,"kaladin":22588,"halftime":22589,"kidnap":22590,"##amo":22591,"##llet":22592,"1754":22593,"synonymous":22594,"depleted":22595,"instituto":22596,"insulin":22597,"reprised":22598,"##opsis":22599,"clashed":22600,"##ctric":22601,"interrupting":22602,"radcliffe":22603,"insisting":22604,"medici":22605,"1715":22606,"ejected":22607,"playfully":22608,"turbulent":22609,"##47":22610,"starvation":22611,"##rini":22612,"shipment":22613,"rebellious":22614,"petersen":22615,"verification":22616,"merits":22617,"##rified":22618,"cakes":22619,"##charged":22620,"1757":22621,"milford":22622,"shortages":22623,"spying":22624,"fidelity":22625,"##aker":22626,"emitted":22627,"storylines":22628,"harvested":22629,"seismic":22630,"##iform":22631,"cheung":22632,"kilda":22633,"theoretically":22634,"barbie":22635,"lynx":22636,"##rgy":22637,"##tius":22638,"goblin":22639,"mata":22640,"poisonous":22641,"##nburg":22642,"reactive":22643,"residues":22644,"obedience":22645,"##евич":22646,"conjecture":22647,"##rac":22648,"401":22649,"hating":22650,"sixties":22651,"kicker":22652,"moaning":22653,"motown":22654,"##bha":22655,"emancipation":22656,"neoclassical":22657,"##hering":22658,"consoles":22659,"ebert":22660,"professorship":22661,"##tures":22662,"sustaining":22663,"assaults":22664,"obeyed":22665,"affluent":22666,"incurred":22667,"tornadoes":22668,"##eber":22669,"##zow":22670,"emphasizing":22671,"highlanders":22672,"cheated":22673,"helmets":22674,"##ctus":22675,"internship":22676,"terence":22677,"bony":22678,"executions":22679,"legislators":22680,"berries":22681,"peninsular":22682,"tinged":22683,"##aco":22684,"1689":22685,"amplifier":22686,"corvette":22687,"ribbons":22688,"lavish":22689,"pennant":22690,"##lander":22691,"worthless":22692,"##chfield":22693,"##forms":22694,"mariano":22695,"pyrenees":22696,"expenditures":22697,"##icides":22698,"chesterfield":22699,"mandir":22700,"tailor":22701,"39th":22702,"sergey":22703,"nestled":22704,"willed":22705,"aristocracy":22706,"devotees":22707,"goodnight":22708,"raaf":22709,"rumored":22710,"weaponry":22711,"remy":22712,"appropriations":22713,"harcourt":22714,"burr":22715,"riaa":22716,"##lence":22717,"limitation":22718,"unnoticed":22719,"guo":22720,"soaking":22721,"swamps":22722,"##tica":22723,"collapsing":22724,"tatiana":22725,"descriptive":22726,"brigham":22727,"psalm":22728,"##chment":22729,"maddox":22730,"##lization":22731,"patti":22732,"caliph":22733,"##aja":22734,"akron":22735,"injuring":22736,"serra":22737,"##ganj":22738,"basins":22739,"##sari":22740,"astonished":22741,"launcher":22742,"##church":22743,"hilary":22744,"wilkins":22745,"sewing":22746,"##sf":22747,"stinging":22748,"##fia":22749,"##ncia":22750,"underwood":22751,"startup":22752,"##ition":22753,"compilations":22754,"vibrations":22755,"embankment":22756,"jurist":22757,"##nity":22758,"bard":22759,"juventus":22760,"groundwater":22761,"kern":22762,"palaces":22763,"helium":22764,"boca":22765,"cramped":22766,"marissa":22767,"soto":22768,"##worm":22769,"jae":22770,"princely":22771,"##ggy":22772,"faso":22773,"bazaar":22774,"warmly":22775,"##voking":22776,"229":22777,"pairing":22778,"##lite":22779,"##grate":22780,"##nets":22781,"wien":22782,"freaked":22783,"ulysses":22784,"rebirth":22785,"##alia":22786,"##rent":22787,"mummy":22788,"guzman":22789,"jimenez":22790,"stilled":22791,"##nitz":22792,"trajectory":22793,"tha":22794,"woken":22795,"archival":22796,"professions":22797,"##pts":22798,"##pta":22799,"hilly":22800,"shadowy":22801,"shrink":22802,"##bolt":22803,"norwood":22804,"glued":22805,"migrate":22806,"stereotypes":22807,"devoid":22808,"##pheus":22809,"625":22810,"evacuate":22811,"horrors":22812,"infancy":22813,"gotham":22814,"knowles":22815,"optic":22816,"downloaded":22817,"sachs":22818,"kingsley":22819,"parramatta":22820,"darryl":22821,"mor":22822,"##onale":22823,"shady":22824,"commence":22825,"confesses":22826,"kan":22827,"##meter":22828,"##placed":22829,"marlborough":22830,"roundabout":22831,"regents":22832,"frigates":22833,"io":22834,"##imating":22835,"gothenburg":22836,"revoked":22837,"carvings":22838,"clockwise":22839,"convertible":22840,"intruder":22841,"##sche":22842,"banged":22843,"##ogo":22844,"vicky":22845,"bourgeois":22846,"##mony":22847,"dupont":22848,"footing":22849,"##gum":22850,"pd":22851,"##real":22852,"buckle":22853,"yun":22854,"penthouse":22855,"sane":22856,"720":22857,"serviced":22858,"stakeholders":22859,"neumann":22860,"bb":22861,"##eers":22862,"comb":22863,"##gam":22864,"catchment":22865,"pinning":22866,"rallies":22867,"typing":22868,"##elles":22869,"forefront":22870,"freiburg":22871,"sweetie":22872,"giacomo":22873,"widowed":22874,"goodwill":22875,"worshipped":22876,"aspirations":22877,"midday":22878,"##vat":22879,"fishery":22880,"##trick":22881,"bournemouth":22882,"turk":22883,"243":22884,"hearth":22885,"ethanol":22886,"guadalajara":22887,"murmurs":22888,"sl":22889,"##uge":22890,"afforded":22891,"scripted":22892,"##hta":22893,"wah":22894,"##jn":22895,"coroner":22896,"translucent":22897,"252":22898,"memorials":22899,"puck":22900,"progresses":22901,"clumsy":22902,"##race":22903,"315":22904,"candace":22905,"recounted":22906,"##27":22907,"##slin":22908,"##uve":22909,"filtering":22910,"##mac":22911,"howl":22912,"strata":22913,"heron":22914,"leveled":22915,"##ays":22916,"dubious":22917,"##oja":22918,"##т":22919,"##wheel":22920,"citations":22921,"exhibiting":22922,"##laya":22923,"##mics":22924,"##pods":22925,"turkic":22926,"##lberg":22927,"injunction":22928,"##ennial":22929,"##mit":22930,"antibodies":22931,"##44":22932,"organise":22933,"##rigues":22934,"cardiovascular":22935,"cushion":22936,"inverness":22937,"##zquez":22938,"dia":22939,"cocoa":22940,"sibling":22941,"##tman":22942,"##roid":22943,"expanse":22944,"feasible":22945,"tunisian":22946,"algiers":22947,"##relli":22948,"rus":22949,"bloomberg":22950,"dso":22951,"westphalia":22952,"bro":22953,"tacoma":22954,"281":22955,"downloads":22956,"##ours":22957,"konrad":22958,"duran":22959,"##hdi":22960,"continuum":22961,"jett":22962,"compares":22963,"legislator":22964,"secession":22965,"##nable":22966,"##gues":22967,"##zuka":22968,"translating":22969,"reacher":22970,"##gley":22971,"##ła":22972,"aleppo":22973,"##agi":22974,"tc":22975,"orchards":22976,"trapping":22977,"linguist":22978,"versatile":22979,"drumming":22980,"postage":22981,"calhoun":22982,"superiors":22983,"##mx":22984,"barefoot":22985,"leary":22986,"##cis":22987,"ignacio":22988,"alfa":22989,"kaplan":22990,"##rogen":22991,"bratislava":22992,"mori":22993,"##vot":22994,"disturb":22995,"haas":22996,"313":22997,"cartridges":22998,"gilmore":22999,"radiated":23000,"salford":23001,"tunic":23002,"hades":23003,"##ulsive":23004,"archeological":23005,"delilah":23006,"magistrates":23007,"auditioned":23008,"brewster":23009,"charters":23010,"empowerment":23011,"blogs":23012,"cappella":23013,"dynasties":23014,"iroquois":23015,"whipping":23016,"##krishna":23017,"raceway":23018,"truths":23019,"myra":23020,"weaken":23021,"judah":23022,"mcgregor":23023,"##horse":23024,"mic":23025,"refueling":23026,"37th":23027,"burnley":23028,"bosses":23029,"markus":23030,"premio":23031,"query":23032,"##gga":23033,"dunbar":23034,"##economic":23035,"darkest":23036,"lyndon":23037,"sealing":23038,"commendation":23039,"reappeared":23040,"##mun":23041,"addicted":23042,"ezio":23043,"slaughtered":23044,"satisfactory":23045,"shuffle":23046,"##eves":23047,"##thic":23048,"##uj":23049,"fortification":23050,"warrington":23051,"##otto":23052,"resurrected":23053,"fargo":23054,"mane":23055,"##utable":23056,"##lei":23057,"##space":23058,"foreword":23059,"ox":23060,"##aris":23061,"##vern":23062,"abrams":23063,"hua":23064,"##mento":23065,"sakura":23066,"##alo":23067,"uv":23068,"sentimental":23069,"##skaya":23070,"midfield":23071,"##eses":23072,"sturdy":23073,"scrolls":23074,"macleod":23075,"##kyu":23076,"entropy":23077,"##lance":23078,"mitochondrial":23079,"cicero":23080,"excelled":23081,"thinner":23082,"convoys":23083,"perceive":23084,"##oslav":23085,"##urable":23086,"systematically":23087,"grind":23088,"burkina":23089,"287":23090,"##tagram":23091,"ops":23092,"##aman":23093,"guantanamo":23094,"##cloth":23095,"##tite":23096,"forcefully":23097,"wavy":23098,"##jou":23099,"pointless":23100,"##linger":23101,"##tze":23102,"layton":23103,"portico":23104,"superficial":23105,"clerical":23106,"outlaws":23107,"##hism":23108,"burials":23109,"muir":23110,"##inn":23111,"creditors":23112,"hauling":23113,"rattle":23114,"##leg":23115,"calais":23116,"monde":23117,"archers":23118,"reclaimed":23119,"dwell":23120,"wexford":23121,"hellenic":23122,"falsely":23123,"remorse":23124,"##tek":23125,"dough":23126,"furnishings":23127,"##uttered":23128,"gabon":23129,"neurological":23130,"novice":23131,"##igraphy":23132,"contemplated":23133,"pulpit":23134,"nightstand":23135,"saratoga":23136,"##istan":23137,"documenting":23138,"pulsing":23139,"taluk":23140,"##firmed":23141,"busted":23142,"marital":23143,"##rien":23144,"disagreements":23145,"wasps":23146,"##yes":23147,"hodge":23148,"mcdonnell":23149,"mimic":23150,"fran":23151,"pendant":23152,"dhabi":23153,"musa":23154,"##nington":23155,"congratulations":23156,"argent":23157,"darrell":23158,"concussion":23159,"losers":23160,"regrets":23161,"thessaloniki":23162,"reversal":23163,"donaldson":23164,"hardwood":23165,"thence":23166,"achilles":23167,"ritter":23168,"##eran":23169,"demonic":23170,"jurgen":23171,"prophets":23172,"goethe":23173,"eki":23174,"classmate":23175,"buff":23176,"##cking":23177,"yank":23178,"irrational":23179,"##inging":23180,"perished":23181,"seductive":23182,"qur":23183,"sourced":23184,"##crat":23185,"##typic":23186,"mustard":23187,"ravine":23188,"barre":23189,"horizontally":23190,"characterization":23191,"phylogenetic":23192,"boise":23193,"##dit":23194,"##runner":23195,"##tower":23196,"brutally":23197,"intercourse":23198,"seduce":23199,"##bbing":23200,"fay":23201,"ferris":23202,"ogden":23203,"amar":23204,"nik":23205,"unarmed":23206,"##inator":23207,"evaluating":23208,"kyrgyzstan":23209,"sweetness":23210,"##lford":23211,"##oki":23212,"mccormick":23213,"meiji":23214,"notoriety":23215,"stimulate":23216,"disrupt":23217,"figuring":23218,"instructional":23219,"mcgrath":23220,"##zoo":23221,"groundbreaking":23222,"##lto":23223,"flinch":23224,"khorasan":23225,"agrarian":23226,"bengals":23227,"mixer":23228,"radiating":23229,"##sov":23230,"ingram":23231,"pitchers":23232,"nad":23233,"tariff":23234,"##cript":23235,"tata":23236,"##codes":23237,"##emi":23238,"##ungen":23239,"appellate":23240,"lehigh":23241,"##bled":23242,"##giri":23243,"brawl":23244,"duct":23245,"texans":23246,"##ciation":23247,"##ropolis":23248,"skipper":23249,"speculative":23250,"vomit":23251,"doctrines":23252,"stresses":23253,"253":23254,"davy":23255,"graders":23256,"whitehead":23257,"jozef":23258,"timely":23259,"cumulative":23260,"haryana":23261,"paints":23262,"appropriately":23263,"boon":23264,"cactus":23265,"##ales":23266,"##pid":23267,"dow":23268,"legions":23269,"##pit":23270,"perceptions":23271,"1730":23272,"picturesque":23273,"##yse":23274,"periphery":23275,"rune":23276,"wr":23277,"##aha":23278,"celtics":23279,"sentencing":23280,"whoa":23281,"##erin":23282,"confirms":23283,"variance":23284,"425":23285,"moines":23286,"mathews":23287,"spade":23288,"rave":23289,"m1":23290,"fronted":23291,"fx":23292,"blending":23293,"alleging":23294,"reared":23295,"##gl":23296,"237":23297,"##paper":23298,"grassroots":23299,"eroded":23300,"##free":23301,"##physical":23302,"directs":23303,"ordeal":23304,"##sław":23305,"accelerate":23306,"hacker":23307,"rooftop":23308,"##inia":23309,"lev":23310,"buys":23311,"cebu":23312,"devote":23313,"##lce":23314,"specialising":23315,"##ulsion":23316,"choreographed":23317,"repetition":23318,"warehouses":23319,"##ryl":23320,"paisley":23321,"tuscany":23322,"analogy":23323,"sorcerer":23324,"hash":23325,"huts":23326,"shards":23327,"descends":23328,"exclude":23329,"nix":23330,"chaplin":23331,"gaga":23332,"ito":23333,"vane":23334,"##drich":23335,"causeway":23336,"misconduct":23337,"limo":23338,"orchestrated":23339,"glands":23340,"jana":23341,"##kot":23342,"u2":23343,"##mple":23344,"##sons":23345,"branching":23346,"contrasts":23347,"scoop":23348,"longed":23349,"##virus":23350,"chattanooga":23351,"##75":23352,"syrup":23353,"cornerstone":23354,"##tized":23355,"##mind":23356,"##iaceae":23357,"careless":23358,"precedence":23359,"frescoes":23360,"##uet":23361,"chilled":23362,"consult":23363,"modelled":23364,"snatch":23365,"peat":23366,"##thermal":23367,"caucasian":23368,"humane":23369,"relaxation":23370,"spins":23371,"temperance":23372,"##lbert":23373,"occupations":23374,"lambda":23375,"hybrids":23376,"moons":23377,"mp3":23378,"##oese":23379,"247":23380,"rolf":23381,"societal":23382,"yerevan":23383,"ness":23384,"##ssler":23385,"befriended":23386,"mechanized":23387,"nominate":23388,"trough":23389,"boasted":23390,"cues":23391,"seater":23392,"##hom":23393,"bends":23394,"##tangle":23395,"conductors":23396,"emptiness":23397,"##lmer":23398,"eurasian":23399,"adriatic":23400,"tian":23401,"##cie":23402,"anxiously":23403,"lark":23404,"propellers":23405,"chichester":23406,"jock":23407,"ev":23408,"2a":23409,"##holding":23410,"credible":23411,"recounts":23412,"tori":23413,"loyalist":23414,"abduction":23415,"##hoot":23416,"##redo":23417,"nepali":23418,"##mite":23419,"ventral":23420,"tempting":23421,"##ango":23422,"##crats":23423,"steered":23424,"##wice":23425,"javelin":23426,"dipping":23427,"laborers":23428,"prentice":23429,"looming":23430,"titanium":23431,"##ː":23432,"badges":23433,"emir":23434,"tensor":23435,"##ntation":23436,"egyptians":23437,"rash":23438,"denies":23439,"hawthorne":23440,"lombard":23441,"showers":23442,"wehrmacht":23443,"dietary":23444,"trojan":23445,"##reus":23446,"welles":23447,"executing":23448,"horseshoe":23449,"lifeboat":23450,"##lak":23451,"elsa":23452,"infirmary":23453,"nearing":23454,"roberta":23455,"boyer":23456,"mutter":23457,"trillion":23458,"joanne":23459,"##fine":23460,"##oked":23461,"sinks":23462,"vortex":23463,"uruguayan":23464,"clasp":23465,"sirius":23466,"##block":23467,"accelerator":23468,"prohibit":23469,"sunken":23470,"byu":23471,"chronological":23472,"diplomats":23473,"ochreous":23474,"510":23475,"symmetrical":23476,"1644":23477,"maia":23478,"##tology":23479,"salts":23480,"reigns":23481,"atrocities":23482,"##ия":23483,"hess":23484,"bared":23485,"issn":23486,"##vyn":23487,"cater":23488,"saturated":23489,"##cycle":23490,"##isse":23491,"sable":23492,"voyager":23493,"dyer":23494,"yusuf":23495,"##inge":23496,"fountains":23497,"wolff":23498,"##39":23499,"##nni":23500,"engraving":23501,"rollins":23502,"atheist":23503,"ominous":23504,"##ault":23505,"herr":23506,"chariot":23507,"martina":23508,"strung":23509,"##fell":23510,"##farlane":23511,"horrific":23512,"sahib":23513,"gazes":23514,"saetan":23515,"erased":23516,"ptolemy":23517,"##olic":23518,"flushing":23519,"lauderdale":23520,"analytic":23521,"##ices":23522,"530":23523,"navarro":23524,"beak":23525,"gorilla":23526,"herrera":23527,"broom":23528,"guadalupe":23529,"raiding":23530,"sykes":23531,"311":23532,"bsc":23533,"deliveries":23534,"1720":23535,"invasions":23536,"carmichael":23537,"tajikistan":23538,"thematic":23539,"ecumenical":23540,"sentiments":23541,"onstage":23542,"##rians":23543,"##brand":23544,"##sume":23545,"catastrophic":23546,"flanks":23547,"molten":23548,"##arns":23549,"waller":23550,"aimee":23551,"terminating":23552,"##icing":23553,"alternately":23554,"##oche":23555,"nehru":23556,"printers":23557,"outraged":23558,"##eving":23559,"empires":23560,"template":23561,"banners":23562,"repetitive":23563,"za":23564,"##oise":23565,"vegetarian":23566,"##tell":23567,"guiana":23568,"opt":23569,"cavendish":23570,"lucknow":23571,"synthesized":23572,"##hani":23573,"##mada":23574,"finalized":23575,"##ctable":23576,"fictitious":23577,"mayoral":23578,"unreliable":23579,"##enham":23580,"embracing":23581,"peppers":23582,"rbis":23583,"##chio":23584,"##neo":23585,"inhibition":23586,"slashed":23587,"togo":23588,"orderly":23589,"embroidered":23590,"safari":23591,"salty":23592,"236":23593,"barron":23594,"benito":23595,"totaled":23596,"##dak":23597,"pubs":23598,"simulated":23599,"caden":23600,"devin":23601,"tolkien":23602,"momma":23603,"welding":23604,"sesame":23605,"##ept":23606,"gottingen":23607,"hardness":23608,"630":23609,"shaman":23610,"temeraire":23611,"620":23612,"adequately":23613,"pediatric":23614,"##kit":23615,"ck":23616,"assertion":23617,"radicals":23618,"composure":23619,"cadence":23620,"seafood":23621,"beaufort":23622,"lazarus":23623,"mani":23624,"warily":23625,"cunning":23626,"kurdistan":23627,"249":23628,"cantata":23629,"##kir":23630,"ares":23631,"##41":23632,"##clusive":23633,"nape":23634,"townland":23635,"geared":23636,"insulted":23637,"flutter":23638,"boating":23639,"violate":23640,"draper":23641,"dumping":23642,"malmo":23643,"##hh":23644,"##romatic":23645,"firearm":23646,"alta":23647,"bono":23648,"obscured":23649,"##clave":23650,"exceeds":23651,"panorama":23652,"unbelievable":23653,"##train":23654,"preschool":23655,"##essed":23656,"disconnected":23657,"installing":23658,"rescuing":23659,"secretaries":23660,"accessibility":23661,"##castle":23662,"##drive":23663,"##ifice":23664,"##film":23665,"bouts":23666,"slug":23667,"waterway":23668,"mindanao":23669,"##buro":23670,"##ratic":23671,"halves":23672,"##ل":23673,"calming":23674,"liter":23675,"maternity":23676,"adorable":23677,"bragg":23678,"electrification":23679,"mcc":23680,"##dote":23681,"roxy":23682,"schizophrenia":23683,"##body":23684,"munoz":23685,"kaye":23686,"whaling":23687,"239":23688,"mil":23689,"tingling":23690,"tolerant":23691,"##ago":23692,"unconventional":23693,"volcanoes":23694,"##finder":23695,"deportivo":23696,"##llie":23697,"robson":23698,"kaufman":23699,"neuroscience":23700,"wai":23701,"deportation":23702,"masovian":23703,"scraping":23704,"converse":23705,"##bh":23706,"hacking":23707,"bulge":23708,"##oun":23709,"administratively":23710,"yao":23711,"580":23712,"amp":23713,"mammoth":23714,"booster":23715,"claremont":23716,"hooper":23717,"nomenclature":23718,"pursuits":23719,"mclaughlin":23720,"melinda":23721,"##sul":23722,"catfish":23723,"barclay":23724,"substrates":23725,"taxa":23726,"zee":23727,"originals":23728,"kimberly":23729,"packets":23730,"padma":23731,"##ality":23732,"borrowing":23733,"ostensibly":23734,"solvent":23735,"##bri":23736,"##genesis":23737,"##mist":23738,"lukas":23739,"shreveport":23740,"veracruz":23741,"##ь":23742,"##lou":23743,"##wives":23744,"cheney":23745,"tt":23746,"anatolia":23747,"hobbs":23748,"##zyn":23749,"cyclic":23750,"radiant":23751,"alistair":23752,"greenish":23753,"siena":23754,"dat":23755,"independents":23756,"##bation":23757,"conform":23758,"pieter":23759,"hyper":23760,"applicant":23761,"bradshaw":23762,"spores":23763,"telangana":23764,"vinci":23765,"inexpensive":23766,"nuclei":23767,"322":23768,"jang":23769,"nme":23770,"soho":23771,"spd":23772,"##ign":23773,"cradled":23774,"receptionist":23775,"pow":23776,"##43":23777,"##rika":23778,"fascism":23779,"##ifer":23780,"experimenting":23781,"##ading":23782,"##iec":23783,"##region":23784,"345":23785,"jocelyn":23786,"maris":23787,"stair":23788,"nocturnal":23789,"toro":23790,"constabulary":23791,"elgin":23792,"##kker":23793,"msc":23794,"##giving":23795,"##schen":23796,"##rase":23797,"doherty":23798,"doping":23799,"sarcastically":23800,"batter":23801,"maneuvers":23802,"##cano":23803,"##apple":23804,"##gai":23805,"##git":23806,"intrinsic":23807,"##nst":23808,"##stor":23809,"1753":23810,"showtime":23811,"cafes":23812,"gasps":23813,"lviv":23814,"ushered":23815,"##thed":23816,"fours":23817,"restart":23818,"astonishment":23819,"transmitting":23820,"flyer":23821,"shrugs":23822,"##sau":23823,"intriguing":23824,"cones":23825,"dictated":23826,"mushrooms":23827,"medial":23828,"##kovsky":23829,"##elman":23830,"escorting":23831,"gaped":23832,"##26":23833,"godfather":23834,"##door":23835,"##sell":23836,"djs":23837,"recaptured":23838,"timetable":23839,"vila":23840,"1710":23841,"3a":23842,"aerodrome":23843,"mortals":23844,"scientology":23845,"##orne":23846,"angelina":23847,"mag":23848,"convection":23849,"unpaid":23850,"insertion":23851,"intermittent":23852,"lego":23853,"##nated":23854,"endeavor":23855,"kota":23856,"pereira":23857,"##lz":23858,"304":23859,"bwv":23860,"glamorgan":23861,"insults":23862,"agatha":23863,"fey":23864,"##cend":23865,"fleetwood":23866,"mahogany":23867,"protruding":23868,"steamship":23869,"zeta":23870,"##arty":23871,"mcguire":23872,"suspense":23873,"##sphere":23874,"advising":23875,"urges":23876,"##wala":23877,"hurriedly":23878,"meteor":23879,"gilded":23880,"inline":23881,"arroyo":23882,"stalker":23883,"##oge":23884,"excitedly":23885,"revered":23886,"##cure":23887,"earle":23888,"introductory":23889,"##break":23890,"##ilde":23891,"mutants":23892,"puff":23893,"pulses":23894,"reinforcement":23895,"##haling":23896,"curses":23897,"lizards":23898,"stalk":23899,"correlated":23900,"##fixed":23901,"fallout":23902,"macquarie":23903,"##unas":23904,"bearded":23905,"denton":23906,"heaving":23907,"802":23908,"##ocation":23909,"winery":23910,"assign":23911,"dortmund":23912,"##lkirk":23913,"everest":23914,"invariant":23915,"charismatic":23916,"susie":23917,"##elling":23918,"bled":23919,"lesley":23920,"telegram":23921,"sumner":23922,"bk":23923,"##ogen":23924,"##к":23925,"wilcox":23926,"needy":23927,"colbert":23928,"duval":23929,"##iferous":23930,"##mbled":23931,"allotted":23932,"attends":23933,"imperative":23934,"##hita":23935,"replacements":23936,"hawker":23937,"##inda":23938,"insurgency":23939,"##zee":23940,"##eke":23941,"casts":23942,"##yla":23943,"680":23944,"ives":23945,"transitioned":23946,"##pack":23947,"##powering":23948,"authoritative":23949,"baylor":23950,"flex":23951,"cringed":23952,"plaintiffs":23953,"woodrow":23954,"##skie":23955,"drastic":23956,"ape":23957,"aroma":23958,"unfolded":23959,"commotion":23960,"nt":23961,"preoccupied":23962,"theta":23963,"routines":23964,"lasers":23965,"privatization":23966,"wand":23967,"domino":23968,"ek":23969,"clenching":23970,"nsa":23971,"strategically":23972,"showered":23973,"bile":23974,"handkerchief":23975,"pere":23976,"storing":23977,"christophe":23978,"insulting":23979,"316":23980,"nakamura":23981,"romani":23982,"asiatic":23983,"magdalena":23984,"palma":23985,"cruises":23986,"stripping":23987,"405":23988,"konstantin":23989,"soaring":23990,"##berman":23991,"colloquially":23992,"forerunner":23993,"havilland":23994,"incarcerated":23995,"parasites":23996,"sincerity":23997,"##utus":23998,"disks":23999,"plank":24000,"saigon":24001,"##ining":24002,"corbin":24003,"homo":24004,"ornaments":24005,"powerhouse":24006,"##tlement":24007,"chong":24008,"fastened":24009,"feasibility":24010,"idf":24011,"morphological":24012,"usable":24013,"##nish":24014,"##zuki":24015,"aqueduct":24016,"jaguars":24017,"keepers":24018,"##flies":24019,"aleksandr":24020,"faust":24021,"assigns":24022,"ewing":24023,"bacterium":24024,"hurled":24025,"tricky":24026,"hungarians":24027,"integers":24028,"wallis":24029,"321":24030,"yamaha":24031,"##isha":24032,"hushed":24033,"oblivion":24034,"aviator":24035,"evangelist":24036,"friars":24037,"##eller":24038,"monograph":24039,"ode":24040,"##nary":24041,"airplanes":24042,"labourers":24043,"charms":24044,"##nee":24045,"1661":24046,"hagen":24047,"tnt":24048,"rudder":24049,"fiesta":24050,"transcript":24051,"dorothea":24052,"ska":24053,"inhibitor":24054,"maccabi":24055,"retorted":24056,"raining":24057,"encompassed":24058,"clauses":24059,"menacing":24060,"1642":24061,"lineman":24062,"##gist":24063,"vamps":24064,"##ape":24065,"##dick":24066,"gloom":24067,"##rera":24068,"dealings":24069,"easing":24070,"seekers":24071,"##nut":24072,"##pment":24073,"helens":24074,"unmanned":24075,"##anu":24076,"##isson":24077,"basics":24078,"##amy":24079,"##ckman":24080,"adjustments":24081,"1688":24082,"brutality":24083,"horne":24084,"##zell":24085,"sui":24086,"##55":24087,"##mable":24088,"aggregator":24089,"##thal":24090,"rhino":24091,"##drick":24092,"##vira":24093,"counters":24094,"zoom":24095,"##01":24096,"##rting":24097,"mn":24098,"montenegrin":24099,"packard":24100,"##unciation":24101,"##♭":24102,"##kki":24103,"reclaim":24104,"scholastic":24105,"thugs":24106,"pulsed":24107,"##icia":24108,"syriac":24109,"quan":24110,"saddam":24111,"banda":24112,"kobe":24113,"blaming":24114,"buddies":24115,"dissent":24116,"##lusion":24117,"##usia":24118,"corbett":24119,"jaya":24120,"delle":24121,"erratic":24122,"lexie":24123,"##hesis":24124,"435":24125,"amiga":24126,"hermes":24127,"##pressing":24128,"##leen":24129,"chapels":24130,"gospels":24131,"jamal":24132,"##uating":24133,"compute":24134,"revolving":24135,"warp":24136,"##sso":24137,"##thes":24138,"armory":24139,"##eras":24140,"##gol":24141,"antrim":24142,"loki":24143,"##kow":24144,"##asian":24145,"##good":24146,"##zano":24147,"braid":24148,"handwriting":24149,"subdistrict":24150,"funky":24151,"pantheon":24152,"##iculate":24153,"concurrency":24154,"estimation":24155,"improper":24156,"juliana":24157,"##his":24158,"newcomers":24159,"johnstone":24160,"staten":24161,"communicated":24162,"##oco":24163,"##alle":24164,"sausage":24165,"stormy":24166,"##stered":24167,"##tters":24168,"superfamily":24169,"##grade":24170,"acidic":24171,"collateral":24172,"tabloid":24173,"##oped":24174,"##rza":24175,"bladder":24176,"austen":24177,"##ellant":24178,"mcgraw":24179,"##hay":24180,"hannibal":24181,"mein":24182,"aquino":24183,"lucifer":24184,"wo":24185,"badger":24186,"boar":24187,"cher":24188,"christensen":24189,"greenberg":24190,"interruption":24191,"##kken":24192,"jem":24193,"244":24194,"mocked":24195,"bottoms":24196,"cambridgeshire":24197,"##lide":24198,"sprawling":24199,"##bbly":24200,"eastwood":24201,"ghent":24202,"synth":24203,"##buck":24204,"advisers":24205,"##bah":24206,"nominally":24207,"hapoel":24208,"qu":24209,"daggers":24210,"estranged":24211,"fabricated":24212,"towels":24213,"vinnie":24214,"wcw":24215,"misunderstanding":24216,"anglia":24217,"nothin":24218,"unmistakable":24219,"##dust":24220,"##lova":24221,"chilly":24222,"marquette":24223,"truss":24224,"##edge":24225,"##erine":24226,"reece":24227,"##lty":24228,"##chemist":24229,"##connected":24230,"272":24231,"308":24232,"41st":24233,"bash":24234,"raion":24235,"waterfalls":24236,"##ump":24237,"##main":24238,"labyrinth":24239,"queue":24240,"theorist":24241,"##istle":24242,"bharatiya":24243,"flexed":24244,"soundtracks":24245,"rooney":24246,"leftist":24247,"patrolling":24248,"wharton":24249,"plainly":24250,"alleviate":24251,"eastman":24252,"schuster":24253,"topographic":24254,"engages":24255,"immensely":24256,"unbearable":24257,"fairchild":24258,"1620":24259,"dona":24260,"lurking":24261,"parisian":24262,"oliveira":24263,"ia":24264,"indictment":24265,"hahn":24266,"bangladeshi":24267,"##aster":24268,"vivo":24269,"##uming":24270,"##ential":24271,"antonia":24272,"expects":24273,"indoors":24274,"kildare":24275,"harlan":24276,"##logue":24277,"##ogenic":24278,"##sities":24279,"forgiven":24280,"##wat":24281,"childish":24282,"tavi":24283,"##mide":24284,"##orra":24285,"plausible":24286,"grimm":24287,"successively":24288,"scooted":24289,"##bola":24290,"##dget":24291,"##rith":24292,"spartans":24293,"emery":24294,"flatly":24295,"azure":24296,"epilogue":24297,"##wark":24298,"flourish":24299,"##iny":24300,"##tracted":24301,"##overs":24302,"##oshi":24303,"bestseller":24304,"distressed":24305,"receipt":24306,"spitting":24307,"hermit":24308,"topological":24309,"##cot":24310,"drilled":24311,"subunit":24312,"francs":24313,"##layer":24314,"eel":24315,"##fk":24316,"##itas":24317,"octopus":24318,"footprint":24319,"petitions":24320,"ufo":24321,"##say":24322,"##foil":24323,"interfering":24324,"leaking":24325,"palo":24326,"##metry":24327,"thistle":24328,"valiant":24329,"##pic":24330,"narayan":24331,"mcpherson":24332,"##fast":24333,"gonzales":24334,"##ym":24335,"##enne":24336,"dustin":24337,"novgorod":24338,"solos":24339,"##zman":24340,"doin":24341,"##raph":24342,"##patient":24343,"##meyer":24344,"soluble":24345,"ashland":24346,"cuffs":24347,"carole":24348,"pendleton":24349,"whistling":24350,"vassal":24351,"##river":24352,"deviation":24353,"revisited":24354,"constituents":24355,"rallied":24356,"rotate":24357,"loomed":24358,"##eil":24359,"##nting":24360,"amateurs":24361,"augsburg":24362,"auschwitz":24363,"crowns":24364,"skeletons":24365,"##cona":24366,"bonnet":24367,"257":24368,"dummy":24369,"globalization":24370,"simeon":24371,"sleeper":24372,"mandal":24373,"differentiated":24374,"##crow":24375,"##mare":24376,"milne":24377,"bundled":24378,"exasperated":24379,"talmud":24380,"owes":24381,"segregated":24382,"##feng":24383,"##uary":24384,"dentist":24385,"piracy":24386,"props":24387,"##rang":24388,"devlin":24389,"##torium":24390,"malicious":24391,"paws":24392,"##laid":24393,"dependency":24394,"##ergy":24395,"##fers":24396,"##enna":24397,"258":24398,"pistons":24399,"rourke":24400,"jed":24401,"grammatical":24402,"tres":24403,"maha":24404,"wig":24405,"512":24406,"ghostly":24407,"jayne":24408,"##achal":24409,"##creen":24410,"##ilis":24411,"##lins":24412,"##rence":24413,"designate":24414,"##with":24415,"arrogance":24416,"cambodian":24417,"clones":24418,"showdown":24419,"throttle":24420,"twain":24421,"##ception":24422,"lobes":24423,"metz":24424,"nagoya":24425,"335":24426,"braking":24427,"##furt":24428,"385":24429,"roaming":24430,"##minster":24431,"amin":24432,"crippled":24433,"##37":24434,"##llary":24435,"indifferent":24436,"hoffmann":24437,"idols":24438,"intimidating":24439,"1751":24440,"261":24441,"influenza":24442,"memo":24443,"onions":24444,"1748":24445,"bandage":24446,"consciously":24447,"##landa":24448,"##rage":24449,"clandestine":24450,"observes":24451,"swiped":24452,"tangle":24453,"##ener":24454,"##jected":24455,"##trum":24456,"##bill":24457,"##lta":24458,"hugs":24459,"congresses":24460,"josiah":24461,"spirited":24462,"##dek":24463,"humanist":24464,"managerial":24465,"filmmaking":24466,"inmate":24467,"rhymes":24468,"debuting":24469,"grimsby":24470,"ur":24471,"##laze":24472,"duplicate":24473,"vigor":24474,"##tf":24475,"republished":24476,"bolshevik":24477,"refurbishment":24478,"antibiotics":24479,"martini":24480,"methane":24481,"newscasts":24482,"royale":24483,"horizons":24484,"levant":24485,"iain":24486,"visas":24487,"##ischen":24488,"paler":24489,"##around":24490,"manifestation":24491,"snuck":24492,"alf":24493,"chop":24494,"futile":24495,"pedestal":24496,"rehab":24497,"##kat":24498,"bmg":24499,"kerman":24500,"res":24501,"fairbanks":24502,"jarrett":24503,"abstraction":24504,"saharan":24505,"##zek":24506,"1746":24507,"procedural":24508,"clearer":24509,"kincaid":24510,"sash":24511,"luciano":24512,"##ffey":24513,"crunch":24514,"helmut":24515,"##vara":24516,"revolutionaries":24517,"##tute":24518,"creamy":24519,"leach":24520,"##mmon":24521,"1747":24522,"permitting":24523,"nes":24524,"plight":24525,"wendell":24526,"##lese":24527,"contra":24528,"ts":24529,"clancy":24530,"ipa":24531,"mach":24532,"staples":24533,"autopsy":24534,"disturbances":24535,"nueva":24536,"karin":24537,"pontiac":24538,"##uding":24539,"proxy":24540,"venerable":24541,"haunt":24542,"leto":24543,"bergman":24544,"expands":24545,"##helm":24546,"wal":24547,"##pipe":24548,"canning":24549,"celine":24550,"cords":24551,"obesity":24552,"##enary":24553,"intrusion":24554,"planner":24555,"##phate":24556,"reasoned":24557,"sequencing":24558,"307":24559,"harrow":24560,"##chon":24561,"##dora":24562,"marred":24563,"mcintyre":24564,"repay":24565,"tarzan":24566,"darting":24567,"248":24568,"harrisburg":24569,"margarita":24570,"repulsed":24571,"##hur":24572,"##lding":24573,"belinda":24574,"hamburger":24575,"novo":24576,"compliant":24577,"runways":24578,"bingham":24579,"registrar":24580,"skyscraper":24581,"ic":24582,"cuthbert":24583,"improvisation":24584,"livelihood":24585,"##corp":24586,"##elial":24587,"admiring":24588,"##dened":24589,"sporadic":24590,"believer":24591,"casablanca":24592,"popcorn":24593,"##29":24594,"asha":24595,"shovel":24596,"##bek":24597,"##dice":24598,"coiled":24599,"tangible":24600,"##dez":24601,"casper":24602,"elsie":24603,"resin":24604,"tenderness":24605,"rectory":24606,"##ivision":24607,"avail":24608,"sonar":24609,"##mori":24610,"boutique":24611,"##dier":24612,"guerre":24613,"bathed":24614,"upbringing":24615,"vaulted":24616,"sandals":24617,"blessings":24618,"##naut":24619,"##utnant":24620,"1680":24621,"306":24622,"foxes":24623,"pia":24624,"corrosion":24625,"hesitantly":24626,"confederates":24627,"crystalline":24628,"footprints":24629,"shapiro":24630,"tirana":24631,"valentin":24632,"drones":24633,"45th":24634,"microscope":24635,"shipments":24636,"texted":24637,"inquisition":24638,"wry":24639,"guernsey":24640,"unauthorized":24641,"resigning":24642,"760":24643,"ripple":24644,"schubert":24645,"stu":24646,"reassure":24647,"felony":24648,"##ardo":24649,"brittle":24650,"koreans":24651,"##havan":24652,"##ives":24653,"dun":24654,"implicit":24655,"tyres":24656,"##aldi":24657,"##lth":24658,"magnolia":24659,"##ehan":24660,"##puri":24661,"##poulos":24662,"aggressively":24663,"fei":24664,"gr":24665,"familiarity":24666,"##poo":24667,"indicative":24668,"##trust":24669,"fundamentally":24670,"jimmie":24671,"overrun":24672,"395":24673,"anchors":24674,"moans":24675,"##opus":24676,"britannia":24677,"armagh":24678,"##ggle":24679,"purposely":24680,"seizing":24681,"##vao":24682,"bewildered":24683,"mundane":24684,"avoidance":24685,"cosmopolitan":24686,"geometridae":24687,"quartermaster":24688,"caf":24689,"415":24690,"chatter":24691,"engulfed":24692,"gleam":24693,"purge":24694,"##icate":24695,"juliette":24696,"jurisprudence":24697,"guerra":24698,"revisions":24699,"##bn":24700,"casimir":24701,"brew":24702,"##jm":24703,"1749":24704,"clapton":24705,"cloudy":24706,"conde":24707,"hermitage":24708,"278":24709,"simulations":24710,"torches":24711,"vincenzo":24712,"matteo":24713,"##rill":24714,"hidalgo":24715,"booming":24716,"westbound":24717,"accomplishment":24718,"tentacles":24719,"unaffected":24720,"##sius":24721,"annabelle":24722,"flopped":24723,"sloping":24724,"##litz":24725,"dreamer":24726,"interceptor":24727,"vu":24728,"##loh":24729,"consecration":24730,"copying":24731,"messaging":24732,"breaker":24733,"climates":24734,"hospitalized":24735,"1752":24736,"torino":24737,"afternoons":24738,"winfield":24739,"witnessing":24740,"##teacher":24741,"breakers":24742,"choirs":24743,"sawmill":24744,"coldly":24745,"##ege":24746,"sipping":24747,"haste":24748,"uninhabited":24749,"conical":24750,"bibliography":24751,"pamphlets":24752,"severn":24753,"edict":24754,"##oca":24755,"deux":24756,"illnesses":24757,"grips":24758,"##pl":24759,"rehearsals":24760,"sis":24761,"thinkers":24762,"tame":24763,"##keepers":24764,"1690":24765,"acacia":24766,"reformer":24767,"##osed":24768,"##rys":24769,"shuffling":24770,"##iring":24771,"##shima":24772,"eastbound":24773,"ionic":24774,"rhea":24775,"flees":24776,"littered":24777,"##oum":24778,"rocker":24779,"vomiting":24780,"groaning":24781,"champ":24782,"overwhelmingly":24783,"civilizations":24784,"paces":24785,"sloop":24786,"adoptive":24787,"##tish":24788,"skaters":24789,"##vres":24790,"aiding":24791,"mango":24792,"##joy":24793,"nikola":24794,"shriek":24795,"##ignon":24796,"pharmaceuticals":24797,"##mg":24798,"tuna":24799,"calvert":24800,"gustavo":24801,"stocked":24802,"yearbook":24803,"##urai":24804,"##mana":24805,"computed":24806,"subsp":24807,"riff":24808,"hanoi":24809,"kelvin":24810,"hamid":24811,"moors":24812,"pastures":24813,"summons":24814,"jihad":24815,"nectar":24816,"##ctors":24817,"bayou":24818,"untitled":24819,"pleasing":24820,"vastly":24821,"republics":24822,"intellect":24823,"##η":24824,"##ulio":24825,"##tou":24826,"crumbling":24827,"stylistic":24828,"sb":24829,"##ی":24830,"consolation":24831,"frequented":24832,"h₂o":24833,"walden":24834,"widows":24835,"##iens":24836,"404":24837,"##ignment":24838,"chunks":24839,"improves":24840,"288":24841,"grit":24842,"recited":24843,"##dev":24844,"snarl":24845,"sociological":24846,"##arte":24847,"##gul":24848,"inquired":24849,"##held":24850,"bruise":24851,"clube":24852,"consultancy":24853,"homogeneous":24854,"hornets":24855,"multiplication":24856,"pasta":24857,"prick":24858,"savior":24859,"##grin":24860,"##kou":24861,"##phile":24862,"yoon":24863,"##gara":24864,"grimes":24865,"vanishing":24866,"cheering":24867,"reacting":24868,"bn":24869,"distillery":24870,"##quisite":24871,"##vity":24872,"coe":24873,"dockyard":24874,"massif":24875,"##jord":24876,"escorts":24877,"voss":24878,"##valent":24879,"byte":24880,"chopped":24881,"hawke":24882,"illusions":24883,"workings":24884,"floats":24885,"##koto":24886,"##vac":24887,"kv":24888,"annapolis":24889,"madden":24890,"##onus":24891,"alvaro":24892,"noctuidae":24893,"##cum":24894,"##scopic":24895,"avenge":24896,"steamboat":24897,"forte":24898,"illustrates":24899,"erika":24900,"##trip":24901,"570":24902,"dew":24903,"nationalities":24904,"bran":24905,"manifested":24906,"thirsty":24907,"diversified":24908,"muscled":24909,"reborn":24910,"##standing":24911,"arson":24912,"##lessness":24913,"##dran":24914,"##logram":24915,"##boys":24916,"##kushima":24917,"##vious":24918,"willoughby":24919,"##phobia":24920,"286":24921,"alsace":24922,"dashboard":24923,"yuki":24924,"##chai":24925,"granville":24926,"myspace":24927,"publicized":24928,"tricked":24929,"##gang":24930,"adjective":24931,"##ater":24932,"relic":24933,"reorganisation":24934,"enthusiastically":24935,"indications":24936,"saxe":24937,"##lassified":24938,"consolidate":24939,"iec":24940,"padua":24941,"helplessly":24942,"ramps":24943,"renaming":24944,"regulars":24945,"pedestrians":24946,"accents":24947,"convicts":24948,"inaccurate":24949,"lowers":24950,"mana":24951,"##pati":24952,"barrie":24953,"bjp":24954,"outta":24955,"someplace":24956,"berwick":24957,"flanking":24958,"invoked":24959,"marrow":24960,"sparsely":24961,"excerpts":24962,"clothed":24963,"rei":24964,"##ginal":24965,"wept":24966,"##straße":24967,"##vish":24968,"alexa":24969,"excel":24970,"##ptive":24971,"membranes":24972,"aquitaine":24973,"creeks":24974,"cutler":24975,"sheppard":24976,"implementations":24977,"ns":24978,"##dur":24979,"fragrance":24980,"budge":24981,"concordia":24982,"magnesium":24983,"marcelo":24984,"##antes":24985,"gladly":24986,"vibrating":24987,"##rral":24988,"##ggles":24989,"montrose":24990,"##omba":24991,"lew":24992,"seamus":24993,"1630":24994,"cocky":24995,"##ament":24996,"##uen":24997,"bjorn":24998,"##rrick":24999,"fielder":25000,"fluttering":25001,"##lase":25002,"methyl":25003,"kimberley":25004,"mcdowell":25005,"reductions":25006,"barbed":25007,"##jic":25008,"##tonic":25009,"aeronautical":25010,"condensed":25011,"distracting":25012,"##promising":25013,"huffed":25014,"##cala":25015,"##sle":25016,"claudius":25017,"invincible":25018,"missy":25019,"pious":25020,"balthazar":25021,"ci":25022,"##lang":25023,"butte":25024,"combo":25025,"orson":25026,"##dication":25027,"myriad":25028,"1707":25029,"silenced":25030,"##fed":25031,"##rh":25032,"coco":25033,"netball":25034,"yourselves":25035,"##oza":25036,"clarify":25037,"heller":25038,"peg":25039,"durban":25040,"etudes":25041,"offender":25042,"roast":25043,"blackmail":25044,"curvature":25045,"##woods":25046,"vile":25047,"309":25048,"illicit":25049,"suriname":25050,"##linson":25051,"overture":25052,"1685":25053,"bubbling":25054,"gymnast":25055,"tucking":25056,"##mming":25057,"##ouin":25058,"maldives":25059,"##bala":25060,"gurney":25061,"##dda":25062,"##eased":25063,"##oides":25064,"backside":25065,"pinto":25066,"jars":25067,"racehorse":25068,"tending":25069,"##rdial":25070,"baronetcy":25071,"wiener":25072,"duly":25073,"##rke":25074,"barbarian":25075,"cupping":25076,"flawed":25077,"##thesis":25078,"bertha":25079,"pleistocene":25080,"puddle":25081,"swearing":25082,"##nob":25083,"##tically":25084,"fleeting":25085,"prostate":25086,"amulet":25087,"educating":25088,"##mined":25089,"##iti":25090,"##tler":25091,"75th":25092,"jens":25093,"respondents":25094,"analytics":25095,"cavaliers":25096,"papacy":25097,"raju":25098,"##iente":25099,"##ulum":25100,"##tip":25101,"funnel":25102,"271":25103,"disneyland":25104,"##lley":25105,"sociologist":25106,"##iam":25107,"2500":25108,"faulkner":25109,"louvre":25110,"menon":25111,"##dson":25112,"276":25113,"##ower":25114,"afterlife":25115,"mannheim":25116,"peptide":25117,"referees":25118,"comedians":25119,"meaningless":25120,"##anger":25121,"##laise":25122,"fabrics":25123,"hurley":25124,"renal":25125,"sleeps":25126,"##bour":25127,"##icle":25128,"breakout":25129,"kristin":25130,"roadside":25131,"animator":25132,"clover":25133,"disdain":25134,"unsafe":25135,"redesign":25136,"##urity":25137,"firth":25138,"barnsley":25139,"portage":25140,"reset":25141,"narrows":25142,"268":25143,"commandos":25144,"expansive":25145,"speechless":25146,"tubular":25147,"##lux":25148,"essendon":25149,"eyelashes":25150,"smashwords":25151,"##yad":25152,"##bang":25153,"##claim":25154,"craved":25155,"sprinted":25156,"chet":25157,"somme":25158,"astor":25159,"wrocław":25160,"orton":25161,"266":25162,"bane":25163,"##erving":25164,"##uing":25165,"mischief":25166,"##amps":25167,"##sund":25168,"scaling":25169,"terre":25170,"##xious":25171,"impairment":25172,"offenses":25173,"undermine":25174,"moi":25175,"soy":25176,"contiguous":25177,"arcadia":25178,"inuit":25179,"seam":25180,"##tops":25181,"macbeth":25182,"rebelled":25183,"##icative":25184,"##iot":25185,"590":25186,"elaborated":25187,"frs":25188,"uniformed":25189,"##dberg":25190,"259":25191,"powerless":25192,"priscilla":25193,"stimulated":25194,"980":25195,"qc":25196,"arboretum":25197,"frustrating":25198,"trieste":25199,"bullock":25200,"##nified":25201,"enriched":25202,"glistening":25203,"intern":25204,"##adia":25205,"locus":25206,"nouvelle":25207,"ollie":25208,"ike":25209,"lash":25210,"starboard":25211,"ee":25212,"tapestry":25213,"headlined":25214,"hove":25215,"rigged":25216,"##vite":25217,"pollock":25218,"##yme":25219,"thrive":25220,"clustered":25221,"cas":25222,"roi":25223,"gleamed":25224,"olympiad":25225,"##lino":25226,"pressured":25227,"regimes":25228,"##hosis":25229,"##lick":25230,"ripley":25231,"##ophone":25232,"kickoff":25233,"gallon":25234,"rockwell":25235,"##arable":25236,"crusader":25237,"glue":25238,"revolutions":25239,"scrambling":25240,"1714":25241,"grover":25242,"##jure":25243,"englishman":25244,"aztec":25245,"263":25246,"contemplating":25247,"coven":25248,"ipad":25249,"preach":25250,"triumphant":25251,"tufts":25252,"##esian":25253,"rotational":25254,"##phus":25255,"328":25256,"falkland":25257,"##brates":25258,"strewn":25259,"clarissa":25260,"rejoin":25261,"environmentally":25262,"glint":25263,"banded":25264,"drenched":25265,"moat":25266,"albanians":25267,"johor":25268,"rr":25269,"maestro":25270,"malley":25271,"nouveau":25272,"shaded":25273,"taxonomy":25274,"v6":25275,"adhere":25276,"bunk":25277,"airfields":25278,"##ritan":25279,"1741":25280,"encompass":25281,"remington":25282,"tran":25283,"##erative":25284,"amelie":25285,"mazda":25286,"friar":25287,"morals":25288,"passions":25289,"##zai":25290,"breadth":25291,"vis":25292,"##hae":25293,"argus":25294,"burnham":25295,"caressing":25296,"insider":25297,"rudd":25298,"##imov":25299,"##mini":25300,"##rso":25301,"italianate":25302,"murderous":25303,"textual":25304,"wainwright":25305,"armada":25306,"bam":25307,"weave":25308,"timer":25309,"##taken":25310,"##nh":25311,"fra":25312,"##crest":25313,"ardent":25314,"salazar":25315,"taps":25316,"tunis":25317,"##ntino":25318,"allegro":25319,"gland":25320,"philanthropic":25321,"##chester":25322,"implication":25323,"##optera":25324,"esq":25325,"judas":25326,"noticeably":25327,"wynn":25328,"##dara":25329,"inched":25330,"indexed":25331,"crises":25332,"villiers":25333,"bandit":25334,"royalties":25335,"patterned":25336,"cupboard":25337,"interspersed":25338,"accessory":25339,"isla":25340,"kendrick":25341,"entourage":25342,"stitches":25343,"##esthesia":25344,"headwaters":25345,"##ior":25346,"interlude":25347,"distraught":25348,"draught":25349,"1727":25350,"##basket":25351,"biased":25352,"sy":25353,"transient":25354,"triad":25355,"subgenus":25356,"adapting":25357,"kidd":25358,"shortstop":25359,"##umatic":25360,"dimly":25361,"spiked":25362,"mcleod":25363,"reprint":25364,"nellie":25365,"pretoria":25366,"windmill":25367,"##cek":25368,"singled":25369,"##mps":25370,"273":25371,"reunite":25372,"##orous":25373,"747":25374,"bankers":25375,"outlying":25376,"##omp":25377,"##ports":25378,"##tream":25379,"apologies":25380,"cosmetics":25381,"patsy":25382,"##deh":25383,"##ocks":25384,"##yson":25385,"bender":25386,"nantes":25387,"serene":25388,"##nad":25389,"lucha":25390,"mmm":25391,"323":25392,"##cius":25393,"##gli":25394,"cmll":25395,"coinage":25396,"nestor":25397,"juarez":25398,"##rook":25399,"smeared":25400,"sprayed":25401,"twitching":25402,"sterile":25403,"irina":25404,"embodied":25405,"juveniles":25406,"enveloped":25407,"miscellaneous":25408,"cancers":25409,"dq":25410,"gulped":25411,"luisa":25412,"crested":25413,"swat":25414,"donegal":25415,"ref":25416,"##anov":25417,"##acker":25418,"hearst":25419,"mercantile":25420,"##lika":25421,"doorbell":25422,"ua":25423,"vicki":25424,"##alla":25425,"##som":25426,"bilbao":25427,"psychologists":25428,"stryker":25429,"sw":25430,"horsemen":25431,"turkmenistan":25432,"wits":25433,"##national":25434,"anson":25435,"mathew":25436,"screenings":25437,"##umb":25438,"rihanna":25439,"##agne":25440,"##nessy":25441,"aisles":25442,"##iani":25443,"##osphere":25444,"hines":25445,"kenton":25446,"saskatoon":25447,"tasha":25448,"truncated":25449,"##champ":25450,"##itan":25451,"mildred":25452,"advises":25453,"fredrik":25454,"interpreting":25455,"inhibitors":25456,"##athi":25457,"spectroscopy":25458,"##hab":25459,"##kong":25460,"karim":25461,"panda":25462,"##oia":25463,"##nail":25464,"##vc":25465,"conqueror":25466,"kgb":25467,"leukemia":25468,"##dity":25469,"arrivals":25470,"cheered":25471,"pisa":25472,"phosphorus":25473,"shielded":25474,"##riated":25475,"mammal":25476,"unitarian":25477,"urgently":25478,"chopin":25479,"sanitary":25480,"##mission":25481,"spicy":25482,"drugged":25483,"hinges":25484,"##tort":25485,"tipping":25486,"trier":25487,"impoverished":25488,"westchester":25489,"##caster":25490,"267":25491,"epoch":25492,"nonstop":25493,"##gman":25494,"##khov":25495,"aromatic":25496,"centrally":25497,"cerro":25498,"##tively":25499,"##vio":25500,"billions":25501,"modulation":25502,"sedimentary":25503,"283":25504,"facilitating":25505,"outrageous":25506,"goldstein":25507,"##eak":25508,"##kt":25509,"ld":25510,"maitland":25511,"penultimate":25512,"pollard":25513,"##dance":25514,"fleets":25515,"spaceship":25516,"vertebrae":25517,"##nig":25518,"alcoholism":25519,"als":25520,"recital":25521,"##bham":25522,"##ference":25523,"##omics":25524,"m2":25525,"##bm":25526,"trois":25527,"##tropical":25528,"##в":25529,"commemorates":25530,"##meric":25531,"marge":25532,"##raction":25533,"1643":25534,"670":25535,"cosmetic":25536,"ravaged":25537,"##ige":25538,"catastrophe":25539,"eng":25540,"##shida":25541,"albrecht":25542,"arterial":25543,"bellamy":25544,"decor":25545,"harmon":25546,"##rde":25547,"bulbs":25548,"synchronized":25549,"vito":25550,"easiest":25551,"shetland":25552,"shielding":25553,"wnba":25554,"##glers":25555,"##ssar":25556,"##riam":25557,"brianna":25558,"cumbria":25559,"##aceous":25560,"##rard":25561,"cores":25562,"thayer":25563,"##nsk":25564,"brood":25565,"hilltop":25566,"luminous":25567,"carts":25568,"keynote":25569,"larkin":25570,"logos":25571,"##cta":25572,"##ا":25573,"##mund":25574,"##quay":25575,"lilith":25576,"tinted":25577,"277":25578,"wrestle":25579,"mobilization":25580,"##uses":25581,"sequential":25582,"siam":25583,"bloomfield":25584,"takahashi":25585,"274":25586,"##ieving":25587,"presenters":25588,"ringo":25589,"blazed":25590,"witty":25591,"##oven":25592,"##ignant":25593,"devastation":25594,"haydn":25595,"harmed":25596,"newt":25597,"therese":25598,"##peed":25599,"gershwin":25600,"molina":25601,"rabbis":25602,"sudanese":25603,"001":25604,"innate":25605,"restarted":25606,"##sack":25607,"##fus":25608,"slices":25609,"wb":25610,"##shah":25611,"enroll":25612,"hypothetical":25613,"hysterical":25614,"1743":25615,"fabio":25616,"indefinite":25617,"warped":25618,"##hg":25619,"exchanging":25620,"525":25621,"unsuitable":25622,"##sboro":25623,"gallo":25624,"1603":25625,"bret":25626,"cobalt":25627,"homemade":25628,"##hunter":25629,"mx":25630,"operatives":25631,"##dhar":25632,"terraces":25633,"durable":25634,"latch":25635,"pens":25636,"whorls":25637,"##ctuated":25638,"##eaux":25639,"billing":25640,"ligament":25641,"succumbed":25642,"##gly":25643,"regulators":25644,"spawn":25645,"##brick":25646,"##stead":25647,"filmfare":25648,"rochelle":25649,"##nzo":25650,"1725":25651,"circumstance":25652,"saber":25653,"supplements":25654,"##nsky":25655,"##tson":25656,"crowe":25657,"wellesley":25658,"carrot":25659,"##9th":25660,"##movable":25661,"primate":25662,"drury":25663,"sincerely":25664,"topical":25665,"##mad":25666,"##rao":25667,"callahan":25668,"kyiv":25669,"smarter":25670,"tits":25671,"undo":25672,"##yeh":25673,"announcements":25674,"anthologies":25675,"barrio":25676,"nebula":25677,"##islaus":25678,"##shaft":25679,"##tyn":25680,"bodyguards":25681,"2021":25682,"assassinate":25683,"barns":25684,"emmett":25685,"scully":25686,"##mah":25687,"##yd":25688,"##eland":25689,"##tino":25690,"##itarian":25691,"demoted":25692,"gorman":25693,"lashed":25694,"prized":25695,"adventist":25696,"writ":25697,"##gui":25698,"alla":25699,"invertebrates":25700,"##ausen":25701,"1641":25702,"amman":25703,"1742":25704,"align":25705,"healy":25706,"redistribution":25707,"##gf":25708,"##rize":25709,"insulation":25710,"##drop":25711,"adherents":25712,"hezbollah":25713,"vitro":25714,"ferns":25715,"yanking":25716,"269":25717,"php":25718,"registering":25719,"uppsala":25720,"cheerleading":25721,"confines":25722,"mischievous":25723,"tully":25724,"##ross":25725,"49th":25726,"docked":25727,"roam":25728,"stipulated":25729,"pumpkin":25730,"##bry":25731,"prompt":25732,"##ezer":25733,"blindly":25734,"shuddering":25735,"craftsmen":25736,"frail":25737,"scented":25738,"katharine":25739,"scramble":25740,"shaggy":25741,"sponge":25742,"helix":25743,"zaragoza":25744,"279":25745,"##52":25746,"43rd":25747,"backlash":25748,"fontaine":25749,"seizures":25750,"posse":25751,"cowan":25752,"nonfiction":25753,"telenovela":25754,"wwii":25755,"hammered":25756,"undone":25757,"##gpur":25758,"encircled":25759,"irs":25760,"##ivation":25761,"artefacts":25762,"oneself":25763,"searing":25764,"smallpox":25765,"##belle":25766,"##osaurus":25767,"shandong":25768,"breached":25769,"upland":25770,"blushing":25771,"rankin":25772,"infinitely":25773,"psyche":25774,"tolerated":25775,"docking":25776,"evicted":25777,"##col":25778,"unmarked":25779,"##lving":25780,"gnome":25781,"lettering":25782,"litres":25783,"musique":25784,"##oint":25785,"benevolent":25786,"##jal":25787,"blackened":25788,"##anna":25789,"mccall":25790,"racers":25791,"tingle":25792,"##ocene":25793,"##orestation":25794,"introductions":25795,"radically":25796,"292":25797,"##hiff":25798,"##باد":25799,"1610":25800,"1739":25801,"munchen":25802,"plead":25803,"##nka":25804,"condo":25805,"scissors":25806,"##sight":25807,"##tens":25808,"apprehension":25809,"##cey":25810,"##yin":25811,"hallmark":25812,"watering":25813,"formulas":25814,"sequels":25815,"##llas":25816,"aggravated":25817,"bae":25818,"commencing":25819,"##building":25820,"enfield":25821,"prohibits":25822,"marne":25823,"vedic":25824,"civilized":25825,"euclidean":25826,"jagger":25827,"beforehand":25828,"blasts":25829,"dumont":25830,"##arney":25831,"##nem":25832,"740":25833,"conversions":25834,"hierarchical":25835,"rios":25836,"simulator":25837,"##dya":25838,"##lellan":25839,"hedges":25840,"oleg":25841,"thrusts":25842,"shadowed":25843,"darby":25844,"maximize":25845,"1744":25846,"gregorian":25847,"##nded":25848,"##routed":25849,"sham":25850,"unspecified":25851,"##hog":25852,"emory":25853,"factual":25854,"##smo":25855,"##tp":25856,"fooled":25857,"##rger":25858,"ortega":25859,"wellness":25860,"marlon":25861,"##oton":25862,"##urance":25863,"casket":25864,"keating":25865,"ley":25866,"enclave":25867,"##ayan":25868,"char":25869,"influencing":25870,"jia":25871,"##chenko":25872,"412":25873,"ammonia":25874,"erebidae":25875,"incompatible":25876,"violins":25877,"cornered":25878,"##arat":25879,"grooves":25880,"astronauts":25881,"columbian":25882,"rampant":25883,"fabrication":25884,"kyushu":25885,"mahmud":25886,"vanish":25887,"##dern":25888,"mesopotamia":25889,"##lete":25890,"ict":25891,"##rgen":25892,"caspian":25893,"kenji":25894,"pitted":25895,"##vered":25896,"999":25897,"grimace":25898,"roanoke":25899,"tchaikovsky":25900,"twinned":25901,"##analysis":25902,"##awan":25903,"xinjiang":25904,"arias":25905,"clemson":25906,"kazakh":25907,"sizable":25908,"1662":25909,"##khand":25910,"##vard":25911,"plunge":25912,"tatum":25913,"vittorio":25914,"##nden":25915,"cholera":25916,"##dana":25917,"##oper":25918,"bracing":25919,"indifference":25920,"projectile":25921,"superliga":25922,"##chee":25923,"realises":25924,"upgrading":25925,"299":25926,"porte":25927,"retribution":25928,"##vies":25929,"nk":25930,"stil":25931,"##resses":25932,"ama":25933,"bureaucracy":25934,"blackberry":25935,"bosch":25936,"testosterone":25937,"collapses":25938,"greer":25939,"##pathic":25940,"ioc":25941,"fifties":25942,"malls":25943,"##erved":25944,"bao":25945,"baskets":25946,"adolescents":25947,"siegfried":25948,"##osity":25949,"##tosis":25950,"mantra":25951,"detecting":25952,"existent":25953,"fledgling":25954,"##cchi":25955,"dissatisfied":25956,"gan":25957,"telecommunication":25958,"mingled":25959,"sobbed":25960,"6000":25961,"controversies":25962,"outdated":25963,"taxis":25964,"##raus":25965,"fright":25966,"slams":25967,"##lham":25968,"##fect":25969,"##tten":25970,"detectors":25971,"fetal":25972,"tanned":25973,"##uw":25974,"fray":25975,"goth":25976,"olympian":25977,"skipping":25978,"mandates":25979,"scratches":25980,"sheng":25981,"unspoken":25982,"hyundai":25983,"tracey":25984,"hotspur":25985,"restrictive":25986,"##buch":25987,"americana":25988,"mundo":25989,"##bari":25990,"burroughs":25991,"diva":25992,"vulcan":25993,"##6th":25994,"distinctions":25995,"thumping":25996,"##ngen":25997,"mikey":25998,"sheds":25999,"fide":26000,"rescues":26001,"springsteen":26002,"vested":26003,"valuation":26004,"##ece":26005,"##ely":26006,"pinnacle":26007,"rake":26008,"sylvie":26009,"##edo":26010,"almond":26011,"quivering":26012,"##irus":26013,"alteration":26014,"faltered":26015,"##wad":26016,"51st":26017,"hydra":26018,"ticked":26019,"##kato":26020,"recommends":26021,"##dicated":26022,"antigua":26023,"arjun":26024,"stagecoach":26025,"wilfred":26026,"trickle":26027,"pronouns":26028,"##pon":26029,"aryan":26030,"nighttime":26031,"##anian":26032,"gall":26033,"pea":26034,"stitch":26035,"##hei":26036,"leung":26037,"milos":26038,"##dini":26039,"eritrea":26040,"nexus":26041,"starved":26042,"snowfall":26043,"kant":26044,"parasitic":26045,"cot":26046,"discus":26047,"hana":26048,"strikers":26049,"appleton":26050,"kitchens":26051,"##erina":26052,"##partisan":26053,"##itha":26054,"##vius":26055,"disclose":26056,"metis":26057,"##channel":26058,"1701":26059,"tesla":26060,"##vera":26061,"fitch":26062,"1735":26063,"blooded":26064,"##tila":26065,"decimal":26066,"##tang":26067,"##bai":26068,"cyclones":26069,"eun":26070,"bottled":26071,"peas":26072,"pensacola":26073,"basha":26074,"bolivian":26075,"crabs":26076,"boil":26077,"lanterns":26078,"partridge":26079,"roofed":26080,"1645":26081,"necks":26082,"##phila":26083,"opined":26084,"patting":26085,"##kla":26086,"##lland":26087,"chuckles":26088,"volta":26089,"whereupon":26090,"##nche":26091,"devout":26092,"euroleague":26093,"suicidal":26094,"##dee":26095,"inherently":26096,"involuntary":26097,"knitting":26098,"nasser":26099,"##hide":26100,"puppets":26101,"colourful":26102,"courageous":26103,"southend":26104,"stills":26105,"miraculous":26106,"hodgson":26107,"richer":26108,"rochdale":26109,"ethernet":26110,"greta":26111,"uniting":26112,"prism":26113,"umm":26114,"##haya":26115,"##itical":26116,"##utation":26117,"deterioration":26118,"pointe":26119,"prowess":26120,"##ropriation":26121,"lids":26122,"scranton":26123,"billings":26124,"subcontinent":26125,"##koff":26126,"##scope":26127,"brute":26128,"kellogg":26129,"psalms":26130,"degraded":26131,"##vez":26132,"stanisław":26133,"##ructured":26134,"ferreira":26135,"pun":26136,"astonishing":26137,"gunnar":26138,"##yat":26139,"arya":26140,"prc":26141,"gottfried":26142,"##tight":26143,"excursion":26144,"##ographer":26145,"dina":26146,"##quil":26147,"##nare":26148,"huffington":26149,"illustrious":26150,"wilbur":26151,"gundam":26152,"verandah":26153,"##zard":26154,"naacp":26155,"##odle":26156,"constructive":26157,"fjord":26158,"kade":26159,"##naud":26160,"generosity":26161,"thrilling":26162,"baseline":26163,"cayman":26164,"frankish":26165,"plastics":26166,"accommodations":26167,"zoological":26168,"##fting":26169,"cedric":26170,"qb":26171,"motorized":26172,"##dome":26173,"##otted":26174,"squealed":26175,"tackled":26176,"canucks":26177,"budgets":26178,"situ":26179,"asthma":26180,"dail":26181,"gabled":26182,"grasslands":26183,"whimpered":26184,"writhing":26185,"judgments":26186,"##65":26187,"minnie":26188,"pv":26189,"##carbon":26190,"bananas":26191,"grille":26192,"domes":26193,"monique":26194,"odin":26195,"maguire":26196,"markham":26197,"tierney":26198,"##estra":26199,"##chua":26200,"libel":26201,"poke":26202,"speedy":26203,"atrium":26204,"laval":26205,"notwithstanding":26206,"##edly":26207,"fai":26208,"kala":26209,"##sur":26210,"robb":26211,"##sma":26212,"listings":26213,"luz":26214,"supplementary":26215,"tianjin":26216,"##acing":26217,"enzo":26218,"jd":26219,"ric":26220,"scanner":26221,"croats":26222,"transcribed":26223,"##49":26224,"arden":26225,"cv":26226,"##hair":26227,"##raphy":26228,"##lver":26229,"##uy":26230,"357":26231,"seventies":26232,"staggering":26233,"alam":26234,"horticultural":26235,"hs":26236,"regression":26237,"timbers":26238,"blasting":26239,"##ounded":26240,"montagu":26241,"manipulating":26242,"##cit":26243,"catalytic":26244,"1550":26245,"troopers":26246,"##meo":26247,"condemnation":26248,"fitzpatrick":26249,"##oire":26250,"##roved":26251,"inexperienced":26252,"1670":26253,"castes":26254,"##lative":26255,"outing":26256,"314":26257,"dubois":26258,"flicking":26259,"quarrel":26260,"ste":26261,"learners":26262,"1625":26263,"iq":26264,"whistled":26265,"##class":26266,"282":26267,"classify":26268,"tariffs":26269,"temperament":26270,"355":26271,"folly":26272,"liszt":26273,"##yles":26274,"immersed":26275,"jordanian":26276,"ceasefire":26277,"apparel":26278,"extras":26279,"maru":26280,"fished":26281,"##bio":26282,"harta":26283,"stockport":26284,"assortment":26285,"craftsman":26286,"paralysis":26287,"transmitters":26288,"##cola":26289,"blindness":26290,"##wk":26291,"fatally":26292,"proficiency":26293,"solemnly":26294,"##orno":26295,"repairing":26296,"amore":26297,"groceries":26298,"ultraviolet":26299,"##chase":26300,"schoolhouse":26301,"##tua":26302,"resurgence":26303,"nailed":26304,"##otype":26305,"##×":26306,"ruse":26307,"saliva":26308,"diagrams":26309,"##tructing":26310,"albans":26311,"rann":26312,"thirties":26313,"1b":26314,"antennas":26315,"hilarious":26316,"cougars":26317,"paddington":26318,"stats":26319,"##eger":26320,"breakaway":26321,"ipod":26322,"reza":26323,"authorship":26324,"prohibiting":26325,"scoffed":26326,"##etz":26327,"##ttle":26328,"conscription":26329,"defected":26330,"trondheim":26331,"##fires":26332,"ivanov":26333,"keenan":26334,"##adan":26335,"##ciful":26336,"##fb":26337,"##slow":26338,"locating":26339,"##ials":26340,"##tford":26341,"cadiz":26342,"basalt":26343,"blankly":26344,"interned":26345,"rags":26346,"rattling":26347,"##tick":26348,"carpathian":26349,"reassured":26350,"sync":26351,"bum":26352,"guildford":26353,"iss":26354,"staunch":26355,"##onga":26356,"astronomers":26357,"sera":26358,"sofie":26359,"emergencies":26360,"susquehanna":26361,"##heard":26362,"duc":26363,"mastery":26364,"vh1":26365,"williamsburg":26366,"bayer":26367,"buckled":26368,"craving":26369,"##khan":26370,"##rdes":26371,"bloomington":26372,"##write":26373,"alton":26374,"barbecue":26375,"##bians":26376,"justine":26377,"##hri":26378,"##ndt":26379,"delightful":26380,"smartphone":26381,"newtown":26382,"photon":26383,"retrieval":26384,"peugeot":26385,"hissing":26386,"##monium":26387,"##orough":26388,"flavors":26389,"lighted":26390,"relaunched":26391,"tainted":26392,"##games":26393,"##lysis":26394,"anarchy":26395,"microscopic":26396,"hopping":26397,"adept":26398,"evade":26399,"evie":26400,"##beau":26401,"inhibit":26402,"sinn":26403,"adjustable":26404,"hurst":26405,"intuition":26406,"wilton":26407,"cisco":26408,"44th":26409,"lawful":26410,"lowlands":26411,"stockings":26412,"thierry":26413,"##dalen":26414,"##hila":26415,"##nai":26416,"fates":26417,"prank":26418,"tb":26419,"maison":26420,"lobbied":26421,"provocative":26422,"1724":26423,"4a":26424,"utopia":26425,"##qual":26426,"carbonate":26427,"gujarati":26428,"purcell":26429,"##rford":26430,"curtiss":26431,"##mei":26432,"overgrown":26433,"arenas":26434,"mediation":26435,"swallows":26436,"##rnik":26437,"respectful":26438,"turnbull":26439,"##hedron":26440,"##hope":26441,"alyssa":26442,"ozone":26443,"##ʻi":26444,"ami":26445,"gestapo":26446,"johansson":26447,"snooker":26448,"canteen":26449,"cuff":26450,"declines":26451,"empathy":26452,"stigma":26453,"##ags":26454,"##iner":26455,"##raine":26456,"taxpayers":26457,"gui":26458,"volga":26459,"##wright":26460,"##copic":26461,"lifespan":26462,"overcame":26463,"tattooed":26464,"enactment":26465,"giggles":26466,"##ador":26467,"##camp":26468,"barrington":26469,"bribe":26470,"obligatory":26471,"orbiting":26472,"peng":26473,"##enas":26474,"elusive":26475,"sucker":26476,"##vating":26477,"cong":26478,"hardship":26479,"empowered":26480,"anticipating":26481,"estrada":26482,"cryptic":26483,"greasy":26484,"detainees":26485,"planck":26486,"sudbury":26487,"plaid":26488,"dod":26489,"marriott":26490,"kayla":26491,"##ears":26492,"##vb":26493,"##zd":26494,"mortally":26495,"##hein":26496,"cognition":26497,"radha":26498,"319":26499,"liechtenstein":26500,"meade":26501,"richly":26502,"argyle":26503,"harpsichord":26504,"liberalism":26505,"trumpets":26506,"lauded":26507,"tyrant":26508,"salsa":26509,"tiled":26510,"lear":26511,"promoters":26512,"reused":26513,"slicing":26514,"trident":26515,"##chuk":26516,"##gami":26517,"##lka":26518,"cantor":26519,"checkpoint":26520,"##points":26521,"gaul":26522,"leger":26523,"mammalian":26524,"##tov":26525,"##aar":26526,"##schaft":26527,"doha":26528,"frenchman":26529,"nirvana":26530,"##vino":26531,"delgado":26532,"headlining":26533,"##eron":26534,"##iography":26535,"jug":26536,"tko":26537,"1649":26538,"naga":26539,"intersections":26540,"##jia":26541,"benfica":26542,"nawab":26543,"##suka":26544,"ashford":26545,"gulp":26546,"##deck":26547,"##vill":26548,"##rug":26549,"brentford":26550,"frazier":26551,"pleasures":26552,"dunne":26553,"potsdam":26554,"shenzhen":26555,"dentistry":26556,"##tec":26557,"flanagan":26558,"##dorff":26559,"##hear":26560,"chorale":26561,"dinah":26562,"prem":26563,"quezon":26564,"##rogated":26565,"relinquished":26566,"sutra":26567,"terri":26568,"##pani":26569,"flaps":26570,"##rissa":26571,"poly":26572,"##rnet":26573,"homme":26574,"aback":26575,"##eki":26576,"linger":26577,"womb":26578,"##kson":26579,"##lewood":26580,"doorstep":26581,"orthodoxy":26582,"threaded":26583,"westfield":26584,"##rval":26585,"dioceses":26586,"fridays":26587,"subsided":26588,"##gata":26589,"loyalists":26590,"##biotic":26591,"##ettes":26592,"letterman":26593,"lunatic":26594,"prelate":26595,"tenderly":26596,"invariably":26597,"souza":26598,"thug":26599,"winslow":26600,"##otide":26601,"furlongs":26602,"gogh":26603,"jeopardy":26604,"##runa":26605,"pegasus":26606,"##umble":26607,"humiliated":26608,"standalone":26609,"tagged":26610,"##roller":26611,"freshmen":26612,"klan":26613,"##bright":26614,"attaining":26615,"initiating":26616,"transatlantic":26617,"logged":26618,"viz":26619,"##uance":26620,"1723":26621,"combatants":26622,"intervening":26623,"stephane":26624,"chieftain":26625,"despised":26626,"grazed":26627,"317":26628,"cdc":26629,"galveston":26630,"godzilla":26631,"macro":26632,"simulate":26633,"##planes":26634,"parades":26635,"##esses":26636,"960":26637,"##ductive":26638,"##unes":26639,"equator":26640,"overdose":26641,"##cans":26642,"##hosh":26643,"##lifting":26644,"joshi":26645,"epstein":26646,"sonora":26647,"treacherous":26648,"aquatics":26649,"manchu":26650,"responsive":26651,"##sation":26652,"supervisory":26653,"##christ":26654,"##llins":26655,"##ibar":26656,"##balance":26657,"##uso":26658,"kimball":26659,"karlsruhe":26660,"mab":26661,"##emy":26662,"ignores":26663,"phonetic":26664,"reuters":26665,"spaghetti":26666,"820":26667,"almighty":26668,"danzig":26669,"rumbling":26670,"tombstone":26671,"designations":26672,"lured":26673,"outset":26674,"##felt":26675,"supermarkets":26676,"##wt":26677,"grupo":26678,"kei":26679,"kraft":26680,"susanna":26681,"##blood":26682,"comprehension":26683,"genealogy":26684,"##aghan":26685,"##verted":26686,"redding":26687,"##ythe":26688,"1722":26689,"bowing":26690,"##pore":26691,"##roi":26692,"lest":26693,"sharpened":26694,"fulbright":26695,"valkyrie":26696,"sikhs":26697,"##unds":26698,"swans":26699,"bouquet":26700,"merritt":26701,"##tage":26702,"##venting":26703,"commuted":26704,"redhead":26705,"clerks":26706,"leasing":26707,"cesare":26708,"dea":26709,"hazy":26710,"##vances":26711,"fledged":26712,"greenfield":26713,"servicemen":26714,"##gical":26715,"armando":26716,"blackout":26717,"dt":26718,"sagged":26719,"downloadable":26720,"intra":26721,"potion":26722,"pods":26723,"##4th":26724,"##mism":26725,"xp":26726,"attendants":26727,"gambia":26728,"stale":26729,"##ntine":26730,"plump":26731,"asteroids":26732,"rediscovered":26733,"buds":26734,"flea":26735,"hive":26736,"##neas":26737,"1737":26738,"classifications":26739,"debuts":26740,"##eles":26741,"olympus":26742,"scala":26743,"##eurs":26744,"##gno":26745,"##mute":26746,"hummed":26747,"sigismund":26748,"visuals":26749,"wiggled":26750,"await":26751,"pilasters":26752,"clench":26753,"sulfate":26754,"##ances":26755,"bellevue":26756,"enigma":26757,"trainee":26758,"snort":26759,"##sw":26760,"clouded":26761,"denim":26762,"##rank":26763,"##rder":26764,"churning":26765,"hartman":26766,"lodges":26767,"riches":26768,"sima":26769,"##missible":26770,"accountable":26771,"socrates":26772,"regulates":26773,"mueller":26774,"##cr":26775,"1702":26776,"avoids":26777,"solids":26778,"himalayas":26779,"nutrient":26780,"pup":26781,"##jevic":26782,"squat":26783,"fades":26784,"nec":26785,"##lates":26786,"##pina":26787,"##rona":26788,"##ου":26789,"privateer":26790,"tequila":26791,"##gative":26792,"##mpton":26793,"apt":26794,"hornet":26795,"immortals":26796,"##dou":26797,"asturias":26798,"cleansing":26799,"dario":26800,"##rries":26801,"##anta":26802,"etymology":26803,"servicing":26804,"zhejiang":26805,"##venor":26806,"##nx":26807,"horned":26808,"erasmus":26809,"rayon":26810,"relocating":26811,"£10":26812,"##bags":26813,"escalated":26814,"promenade":26815,"stubble":26816,"2010s":26817,"artisans":26818,"axial":26819,"liquids":26820,"mora":26821,"sho":26822,"yoo":26823,"##tsky":26824,"bundles":26825,"oldies":26826,"##nally":26827,"notification":26828,"bastion":26829,"##ths":26830,"sparkle":26831,"##lved":26832,"1728":26833,"leash":26834,"pathogen":26835,"highs":26836,"##hmi":26837,"immature":26838,"880":26839,"gonzaga":26840,"ignatius":26841,"mansions":26842,"monterrey":26843,"sweets":26844,"bryson":26845,"##loe":26846,"polled":26847,"regatta":26848,"brightest":26849,"pei":26850,"rosy":26851,"squid":26852,"hatfield":26853,"payroll":26854,"addict":26855,"meath":26856,"cornerback":26857,"heaviest":26858,"lodging":26859,"##mage":26860,"capcom":26861,"rippled":26862,"##sily":26863,"barnet":26864,"mayhem":26865,"ymca":26866,"snuggled":26867,"rousseau":26868,"##cute":26869,"blanchard":26870,"284":26871,"fragmented":26872,"leighton":26873,"chromosomes":26874,"risking":26875,"##md":26876,"##strel":26877,"##utter":26878,"corinne":26879,"coyotes":26880,"cynical":26881,"hiroshi":26882,"yeomanry":26883,"##ractive":26884,"ebook":26885,"grading":26886,"mandela":26887,"plume":26888,"agustin":26889,"magdalene":26890,"##rkin":26891,"bea":26892,"femme":26893,"trafford":26894,"##coll":26895,"##lun":26896,"##tance":26897,"52nd":26898,"fourier":26899,"upton":26900,"##mental":26901,"camilla":26902,"gust":26903,"iihf":26904,"islamabad":26905,"longevity":26906,"##kala":26907,"feldman":26908,"netting":26909,"##rization":26910,"endeavour":26911,"foraging":26912,"mfa":26913,"orr":26914,"##open":26915,"greyish":26916,"contradiction":26917,"graz":26918,"##ruff":26919,"handicapped":26920,"marlene":26921,"tweed":26922,"oaxaca":26923,"spp":26924,"campos":26925,"miocene":26926,"pri":26927,"configured":26928,"cooks":26929,"pluto":26930,"cozy":26931,"pornographic":26932,"##entes":26933,"70th":26934,"fairness":26935,"glided":26936,"jonny":26937,"lynne":26938,"rounding":26939,"sired":26940,"##emon":26941,"##nist":26942,"remade":26943,"uncover":26944,"##mack":26945,"complied":26946,"lei":26947,"newsweek":26948,"##jured":26949,"##parts":26950,"##enting":26951,"##pg":26952,"293":26953,"finer":26954,"guerrillas":26955,"athenian":26956,"deng":26957,"disused":26958,"stepmother":26959,"accuse":26960,"gingerly":26961,"seduction":26962,"521":26963,"confronting":26964,"##walker":26965,"##going":26966,"gora":26967,"nostalgia":26968,"sabres":26969,"virginity":26970,"wrenched":26971,"##minated":26972,"syndication":26973,"wielding":26974,"eyre":26975,"##56":26976,"##gnon":26977,"##igny":26978,"behaved":26979,"taxpayer":26980,"sweeps":26981,"##growth":26982,"childless":26983,"gallant":26984,"##ywood":26985,"amplified":26986,"geraldine":26987,"scrape":26988,"##ffi":26989,"babylonian":26990,"fresco":26991,"##rdan":26992,"##kney":26993,"##position":26994,"1718":26995,"restricting":26996,"tack":26997,"fukuoka":26998,"osborn":26999,"selector":27000,"partnering":27001,"##dlow":27002,"318":27003,"gnu":27004,"kia":27005,"tak":27006,"whitley":27007,"gables":27008,"##54":27009,"##mania":27010,"mri":27011,"softness":27012,"immersion":27013,"##bots":27014,"##evsky":27015,"1713":27016,"chilling":27017,"insignificant":27018,"pcs":27019,"##uis":27020,"elites":27021,"lina":27022,"purported":27023,"supplemental":27024,"teaming":27025,"##americana":27026,"##dding":27027,"##inton":27028,"proficient":27029,"rouen":27030,"##nage":27031,"##rret":27032,"niccolo":27033,"selects":27034,"##bread":27035,"fluffy":27036,"1621":27037,"gruff":27038,"knotted":27039,"mukherjee":27040,"polgara":27041,"thrash":27042,"nicholls":27043,"secluded":27044,"smoothing":27045,"thru":27046,"corsica":27047,"loaf":27048,"whitaker":27049,"inquiries":27050,"##rrier":27051,"##kam":27052,"indochina":27053,"289":27054,"marlins":27055,"myles":27056,"peking":27057,"##tea":27058,"extracts":27059,"pastry":27060,"superhuman":27061,"connacht":27062,"vogel":27063,"##ditional":27064,"##het":27065,"##udged":27066,"##lash":27067,"gloss":27068,"quarries":27069,"refit":27070,"teaser":27071,"##alic":27072,"##gaon":27073,"20s":27074,"materialized":27075,"sling":27076,"camped":27077,"pickering":27078,"tung":27079,"tracker":27080,"pursuant":27081,"##cide":27082,"cranes":27083,"soc":27084,"##cini":27085,"##typical":27086,"##viere":27087,"anhalt":27088,"overboard":27089,"workout":27090,"chores":27091,"fares":27092,"orphaned":27093,"stains":27094,"##logie":27095,"fenton":27096,"surpassing":27097,"joyah":27098,"triggers":27099,"##itte":27100,"grandmaster":27101,"##lass":27102,"##lists":27103,"clapping":27104,"fraudulent":27105,"ledger":27106,"nagasaki":27107,"##cor":27108,"##nosis":27109,"##tsa":27110,"eucalyptus":27111,"tun":27112,"##icio":27113,"##rney":27114,"##tara":27115,"dax":27116,"heroism":27117,"ina":27118,"wrexham":27119,"onboard":27120,"unsigned":27121,"##dates":27122,"moshe":27123,"galley":27124,"winnie":27125,"droplets":27126,"exiles":27127,"praises":27128,"watered":27129,"noodles":27130,"##aia":27131,"fein":27132,"adi":27133,"leland":27134,"multicultural":27135,"stink":27136,"bingo":27137,"comets":27138,"erskine":27139,"modernized":27140,"canned":27141,"constraint":27142,"domestically":27143,"chemotherapy":27144,"featherweight":27145,"stifled":27146,"##mum":27147,"darkly":27148,"irresistible":27149,"refreshing":27150,"hasty":27151,"isolate":27152,"##oys":27153,"kitchener":27154,"planners":27155,"##wehr":27156,"cages":27157,"yarn":27158,"implant":27159,"toulon":27160,"elects":27161,"childbirth":27162,"yue":27163,"##lind":27164,"##lone":27165,"cn":27166,"rightful":27167,"sportsman":27168,"junctions":27169,"remodeled":27170,"specifies":27171,"##rgh":27172,"291":27173,"##oons":27174,"complimented":27175,"##urgent":27176,"lister":27177,"ot":27178,"##logic":27179,"bequeathed":27180,"cheekbones":27181,"fontana":27182,"gabby":27183,"##dial":27184,"amadeus":27185,"corrugated":27186,"maverick":27187,"resented":27188,"triangles":27189,"##hered":27190,"##usly":27191,"nazareth":27192,"tyrol":27193,"1675":27194,"assent":27195,"poorer":27196,"sectional":27197,"aegean":27198,"##cous":27199,"296":27200,"nylon":27201,"ghanaian":27202,"##egorical":27203,"##weig":27204,"cushions":27205,"forbid":27206,"fusiliers":27207,"obstruction":27208,"somerville":27209,"##scia":27210,"dime":27211,"earrings":27212,"elliptical":27213,"leyte":27214,"oder":27215,"polymers":27216,"timmy":27217,"atm":27218,"midtown":27219,"piloted":27220,"settles":27221,"continual":27222,"externally":27223,"mayfield":27224,"##uh":27225,"enrichment":27226,"henson":27227,"keane":27228,"persians":27229,"1733":27230,"benji":27231,"braden":27232,"pep":27233,"324":27234,"##efe":27235,"contenders":27236,"pepsi":27237,"valet":27238,"##isches":27239,"298":27240,"##asse":27241,"##earing":27242,"goofy":27243,"stroll":27244,"##amen":27245,"authoritarian":27246,"occurrences":27247,"adversary":27248,"ahmedabad":27249,"tangent":27250,"toppled":27251,"dorchester":27252,"1672":27253,"modernism":27254,"marxism":27255,"islamist":27256,"charlemagne":27257,"exponential":27258,"racks":27259,"unicode":27260,"brunette":27261,"mbc":27262,"pic":27263,"skirmish":27264,"##bund":27265,"##lad":27266,"##powered":27267,"##yst":27268,"hoisted":27269,"messina":27270,"shatter":27271,"##ctum":27272,"jedi":27273,"vantage":27274,"##music":27275,"##neil":27276,"clemens":27277,"mahmoud":27278,"corrupted":27279,"authentication":27280,"lowry":27281,"nils":27282,"##washed":27283,"omnibus":27284,"wounding":27285,"jillian":27286,"##itors":27287,"##opped":27288,"serialized":27289,"narcotics":27290,"handheld":27291,"##arm":27292,"##plicity":27293,"intersecting":27294,"stimulating":27295,"##onis":27296,"crate":27297,"fellowships":27298,"hemingway":27299,"casinos":27300,"climatic":27301,"fordham":27302,"copeland":27303,"drip":27304,"beatty":27305,"leaflets":27306,"robber":27307,"brothel":27308,"madeira":27309,"##hedral":27310,"sphinx":27311,"ultrasound":27312,"##vana":27313,"valor":27314,"forbade":27315,"leonid":27316,"villas":27317,"##aldo":27318,"duane":27319,"marquez":27320,"##cytes":27321,"disadvantaged":27322,"forearms":27323,"kawasaki":27324,"reacts":27325,"consular":27326,"lax":27327,"uncles":27328,"uphold":27329,"##hopper":27330,"concepcion":27331,"dorsey":27332,"lass":27333,"##izan":27334,"arching":27335,"passageway":27336,"1708":27337,"researches":27338,"tia":27339,"internationals":27340,"##graphs":27341,"##opers":27342,"distinguishes":27343,"javanese":27344,"divert":27345,"##uven":27346,"plotted":27347,"##listic":27348,"##rwin":27349,"##erik":27350,"##tify":27351,"affirmative":27352,"signifies":27353,"validation":27354,"##bson":27355,"kari":27356,"felicity":27357,"georgina":27358,"zulu":27359,"##eros":27360,"##rained":27361,"##rath":27362,"overcoming":27363,"##dot":27364,"argyll":27365,"##rbin":27366,"1734":27367,"chiba":27368,"ratification":27369,"windy":27370,"earls":27371,"parapet":27372,"##marks":27373,"hunan":27374,"pristine":27375,"astrid":27376,"punta":27377,"##gart":27378,"brodie":27379,"##kota":27380,"##oder":27381,"malaga":27382,"minerva":27383,"rouse":27384,"##phonic":27385,"bellowed":27386,"pagoda":27387,"portals":27388,"reclamation":27389,"##gur":27390,"##odies":27391,"##⁄₄":27392,"parentheses":27393,"quoting":27394,"allergic":27395,"palette":27396,"showcases":27397,"benefactor":27398,"heartland":27399,"nonlinear":27400,"##tness":27401,"bladed":27402,"cheerfully":27403,"scans":27404,"##ety":27405,"##hone":27406,"1666":27407,"girlfriends":27408,"pedersen":27409,"hiram":27410,"sous":27411,"##liche":27412,"##nator":27413,"1683":27414,"##nery":27415,"##orio":27416,"##umen":27417,"bobo":27418,"primaries":27419,"smiley":27420,"##cb":27421,"unearthed":27422,"uniformly":27423,"fis":27424,"metadata":27425,"1635":27426,"ind":27427,"##oted":27428,"recoil":27429,"##titles":27430,"##tura":27431,"##ια":27432,"406":27433,"hilbert":27434,"jamestown":27435,"mcmillan":27436,"tulane":27437,"seychelles":27438,"##frid":27439,"antics":27440,"coli":27441,"fated":27442,"stucco":27443,"##grants":27444,"1654":27445,"bulky":27446,"accolades":27447,"arrays":27448,"caledonian":27449,"carnage":27450,"optimism":27451,"puebla":27452,"##tative":27453,"##cave":27454,"enforcing":27455,"rotherham":27456,"seo":27457,"dunlop":27458,"aeronautics":27459,"chimed":27460,"incline":27461,"zoning":27462,"archduke":27463,"hellenistic":27464,"##oses":27465,"##sions":27466,"candi":27467,"thong":27468,"##ople":27469,"magnate":27470,"rustic":27471,"##rsk":27472,"projective":27473,"slant":27474,"##offs":27475,"danes":27476,"hollis":27477,"vocalists":27478,"##ammed":27479,"congenital":27480,"contend":27481,"gesellschaft":27482,"##ocating":27483,"##pressive":27484,"douglass":27485,"quieter":27486,"##cm":27487,"##kshi":27488,"howled":27489,"salim":27490,"spontaneously":27491,"townsville":27492,"buena":27493,"southport":27494,"##bold":27495,"kato":27496,"1638":27497,"faerie":27498,"stiffly":27499,"##vus":27500,"##rled":27501,"297":27502,"flawless":27503,"realising":27504,"taboo":27505,"##7th":27506,"bytes":27507,"straightening":27508,"356":27509,"jena":27510,"##hid":27511,"##rmin":27512,"cartwright":27513,"berber":27514,"bertram":27515,"soloists":27516,"411":27517,"noses":27518,"417":27519,"coping":27520,"fission":27521,"hardin":27522,"inca":27523,"##cen":27524,"1717":27525,"mobilized":27526,"vhf":27527,"##raf":27528,"biscuits":27529,"curate":27530,"##85":27531,"##anial":27532,"331":27533,"gaunt":27534,"neighbourhoods":27535,"1540":27536,"##abas":27537,"blanca":27538,"bypassed":27539,"sockets":27540,"behold":27541,"coincidentally":27542,"##bane":27543,"nara":27544,"shave":27545,"splinter":27546,"terrific":27547,"##arion":27548,"##erian":27549,"commonplace":27550,"juris":27551,"redwood":27552,"waistband":27553,"boxed":27554,"caitlin":27555,"fingerprints":27556,"jennie":27557,"naturalized":27558,"##ired":27559,"balfour":27560,"craters":27561,"jody":27562,"bungalow":27563,"hugely":27564,"quilt":27565,"glitter":27566,"pigeons":27567,"undertaker":27568,"bulging":27569,"constrained":27570,"goo":27571,"##sil":27572,"##akh":27573,"assimilation":27574,"reworked":27575,"##person":27576,"persuasion":27577,"##pants":27578,"felicia":27579,"##cliff":27580,"##ulent":27581,"1732":27582,"explodes":27583,"##dun":27584,"##inium":27585,"##zic":27586,"lyman":27587,"vulture":27588,"hog":27589,"overlook":27590,"begs":27591,"northwards":27592,"ow":27593,"spoil":27594,"##urer":27595,"fatima":27596,"favorably":27597,"accumulate":27598,"sargent":27599,"sorority":27600,"corresponded":27601,"dispersal":27602,"kochi":27603,"toned":27604,"##imi":27605,"##lita":27606,"internacional":27607,"newfound":27608,"##agger":27609,"##lynn":27610,"##rigue":27611,"booths":27612,"peanuts":27613,"##eborg":27614,"medicare":27615,"muriel":27616,"nur":27617,"##uram":27618,"crates":27619,"millennia":27620,"pajamas":27621,"worsened":27622,"##breakers":27623,"jimi":27624,"vanuatu":27625,"yawned":27626,"##udeau":27627,"carousel":27628,"##hony":27629,"hurdle":27630,"##ccus":27631,"##mounted":27632,"##pod":27633,"rv":27634,"##eche":27635,"airship":27636,"ambiguity":27637,"compulsion":27638,"recapture":27639,"##claiming":27640,"arthritis":27641,"##osomal":27642,"1667":27643,"asserting":27644,"ngc":27645,"sniffing":27646,"dade":27647,"discontent":27648,"glendale":27649,"ported":27650,"##amina":27651,"defamation":27652,"rammed":27653,"##scent":27654,"fling":27655,"livingstone":27656,"##fleet":27657,"875":27658,"##ppy":27659,"apocalyptic":27660,"comrade":27661,"lcd":27662,"##lowe":27663,"cessna":27664,"eine":27665,"persecuted":27666,"subsistence":27667,"demi":27668,"hoop":27669,"reliefs":27670,"710":27671,"coptic":27672,"progressing":27673,"stemmed":27674,"perpetrators":27675,"1665":27676,"priestess":27677,"##nio":27678,"dobson":27679,"ebony":27680,"rooster":27681,"itf":27682,"tortricidae":27683,"##bbon":27684,"##jian":27685,"cleanup":27686,"##jean":27687,"##øy":27688,"1721":27689,"eighties":27690,"taxonomic":27691,"holiness":27692,"##hearted":27693,"##spar":27694,"antilles":27695,"showcasing":27696,"stabilized":27697,"##nb":27698,"gia":27699,"mascara":27700,"michelangelo":27701,"dawned":27702,"##uria":27703,"##vinsky":27704,"extinguished":27705,"fitz":27706,"grotesque":27707,"£100":27708,"##fera":27709,"##loid":27710,"##mous":27711,"barges":27712,"neue":27713,"throbbed":27714,"cipher":27715,"johnnie":27716,"##a1":27717,"##mpt":27718,"outburst":27719,"##swick":27720,"spearheaded":27721,"administrations":27722,"c1":27723,"heartbreak":27724,"pixels":27725,"pleasantly":27726,"##enay":27727,"lombardy":27728,"plush":27729,"##nsed":27730,"bobbie":27731,"##hly":27732,"reapers":27733,"tremor":27734,"xiang":27735,"minogue":27736,"substantive":27737,"hitch":27738,"barak":27739,"##wyl":27740,"kwan":27741,"##encia":27742,"910":27743,"obscene":27744,"elegance":27745,"indus":27746,"surfer":27747,"bribery":27748,"conserve":27749,"##hyllum":27750,"##masters":27751,"horatio":27752,"##fat":27753,"apes":27754,"rebound":27755,"psychotic":27756,"##pour":27757,"iteration":27758,"##mium":27759,"##vani":27760,"botanic":27761,"horribly":27762,"antiques":27763,"dispose":27764,"paxton":27765,"##hli":27766,"##wg":27767,"timeless":27768,"1704":27769,"disregard":27770,"engraver":27771,"hounds":27772,"##bau":27773,"##version":27774,"looted":27775,"uno":27776,"facilitates":27777,"groans":27778,"masjid":27779,"rutland":27780,"antibody":27781,"disqualification":27782,"decatur":27783,"footballers":27784,"quake":27785,"slacks":27786,"48th":27787,"rein":27788,"scribe":27789,"stabilize":27790,"commits":27791,"exemplary":27792,"tho":27793,"##hort":27794,"##chison":27795,"pantry":27796,"traversed":27797,"##hiti":27798,"disrepair":27799,"identifiable":27800,"vibrated":27801,"baccalaureate":27802,"##nnis":27803,"csa":27804,"interviewing":27805,"##iensis":27806,"##raße":27807,"greaves":27808,"wealthiest":27809,"343":27810,"classed":27811,"jogged":27812,"£5":27813,"##58":27814,"##atal":27815,"illuminating":27816,"knicks":27817,"respecting":27818,"##uno":27819,"scrubbed":27820,"##iji":27821,"##dles":27822,"kruger":27823,"moods":27824,"growls":27825,"raider":27826,"silvia":27827,"chefs":27828,"kam":27829,"vr":27830,"cree":27831,"percival":27832,"##terol":27833,"gunter":27834,"counterattack":27835,"defiant":27836,"henan":27837,"ze":27838,"##rasia":27839,"##riety":27840,"equivalence":27841,"submissions":27842,"##fra":27843,"##thor":27844,"bautista":27845,"mechanically":27846,"##heater":27847,"cornice":27848,"herbal":27849,"templar":27850,"##mering":27851,"outputs":27852,"ruining":27853,"ligand":27854,"renumbered":27855,"extravagant":27856,"mika":27857,"blockbuster":27858,"eta":27859,"insurrection":27860,"##ilia":27861,"darkening":27862,"ferocious":27863,"pianos":27864,"strife":27865,"kinship":27866,"##aer":27867,"melee":27868,"##anor":27869,"##iste":27870,"##may":27871,"##oue":27872,"decidedly":27873,"weep":27874,"##jad":27875,"##missive":27876,"##ppel":27877,"354":27878,"puget":27879,"unease":27880,"##gnant":27881,"1629":27882,"hammering":27883,"kassel":27884,"ob":27885,"wessex":27886,"##lga":27887,"bromwich":27888,"egan":27889,"paranoia":27890,"utilization":27891,"##atable":27892,"##idad":27893,"contradictory":27894,"provoke":27895,"##ols":27896,"##ouring":27897,"##tangled":27898,"knesset":27899,"##very":27900,"##lette":27901,"plumbing":27902,"##sden":27903,"##¹":27904,"greensboro":27905,"occult":27906,"sniff":27907,"338":27908,"zev":27909,"beaming":27910,"gamer":27911,"haggard":27912,"mahal":27913,"##olt":27914,"##pins":27915,"mendes":27916,"utmost":27917,"briefing":27918,"gunnery":27919,"##gut":27920,"##pher":27921,"##zh":27922,"##rok":27923,"1679":27924,"khalifa":27925,"sonya":27926,"##boot":27927,"principals":27928,"urbana":27929,"wiring":27930,"##liffe":27931,"##minating":27932,"##rrado":27933,"dahl":27934,"nyu":27935,"skepticism":27936,"np":27937,"townspeople":27938,"ithaca":27939,"lobster":27940,"somethin":27941,"##fur":27942,"##arina":27943,"##−1":27944,"freighter":27945,"zimmerman":27946,"biceps":27947,"contractual":27948,"##herton":27949,"amend":27950,"hurrying":27951,"subconscious":27952,"##anal":27953,"336":27954,"meng":27955,"clermont":27956,"spawning":27957,"##eia":27958,"##lub":27959,"dignitaries":27960,"impetus":27961,"snacks":27962,"spotting":27963,"twigs":27964,"##bilis":27965,"##cz":27966,"##ouk":27967,"libertadores":27968,"nic":27969,"skylar":27970,"##aina":27971,"##firm":27972,"gustave":27973,"asean":27974,"##anum":27975,"dieter":27976,"legislatures":27977,"flirt":27978,"bromley":27979,"trolls":27980,"umar":27981,"##bbies":27982,"##tyle":27983,"blah":27984,"parc":27985,"bridgeport":27986,"crank":27987,"negligence":27988,"##nction":27989,"46th":27990,"constantin":27991,"molded":27992,"bandages":27993,"seriousness":27994,"00pm":27995,"siegel":27996,"carpets":27997,"compartments":27998,"upbeat":27999,"statehood":28000,"##dner":28001,"##edging":28002,"marko":28003,"730":28004,"platt":28005,"##hane":28006,"paving":28007,"##iy":28008,"1738":28009,"abbess":28010,"impatience":28011,"limousine":28012,"nbl":28013,"##talk":28014,"441":28015,"lucille":28016,"mojo":28017,"nightfall":28018,"robbers":28019,"##nais":28020,"karel":28021,"brisk":28022,"calves":28023,"replicate":28024,"ascribed":28025,"telescopes":28026,"##olf":28027,"intimidated":28028,"##reen":28029,"ballast":28030,"specialization":28031,"##sit":28032,"aerodynamic":28033,"caliphate":28034,"rainer":28035,"visionary":28036,"##arded":28037,"epsilon":28038,"##aday":28039,"##onte":28040,"aggregation":28041,"auditory":28042,"boosted":28043,"reunification":28044,"kathmandu":28045,"loco":28046,"robyn":28047,"402":28048,"acknowledges":28049,"appointing":28050,"humanoid":28051,"newell":28052,"redeveloped":28053,"restraints":28054,"##tained":28055,"barbarians":28056,"chopper":28057,"1609":28058,"italiana":28059,"##lez":28060,"##lho":28061,"investigates":28062,"wrestlemania":28063,"##anies":28064,"##bib":28065,"690":28066,"##falls":28067,"creaked":28068,"dragoons":28069,"gravely":28070,"minions":28071,"stupidity":28072,"volley":28073,"##harat":28074,"##week":28075,"musik":28076,"##eries":28077,"##uously":28078,"fungal":28079,"massimo":28080,"semantics":28081,"malvern":28082,"##ahl":28083,"##pee":28084,"discourage":28085,"embryo":28086,"imperialism":28087,"1910s":28088,"profoundly":28089,"##ddled":28090,"jiangsu":28091,"sparkled":28092,"stat":28093,"##holz":28094,"sweatshirt":28095,"tobin":28096,"##iction":28097,"sneered":28098,"##cheon":28099,"##oit":28100,"brit":28101,"causal":28102,"smyth":28103,"##neuve":28104,"diffuse":28105,"perrin":28106,"silvio":28107,"##ipes":28108,"##recht":28109,"detonated":28110,"iqbal":28111,"selma":28112,"##nism":28113,"##zumi":28114,"roasted":28115,"##riders":28116,"tay":28117,"##ados":28118,"##mament":28119,"##mut":28120,"##rud":28121,"840":28122,"completes":28123,"nipples":28124,"cfa":28125,"flavour":28126,"hirsch":28127,"##laus":28128,"calderon":28129,"sneakers":28130,"moravian":28131,"##ksha":28132,"1622":28133,"rq":28134,"294":28135,"##imeters":28136,"bodo":28137,"##isance":28138,"##pre":28139,"##ronia":28140,"anatomical":28141,"excerpt":28142,"##lke":28143,"dh":28144,"kunst":28145,"##tablished":28146,"##scoe":28147,"biomass":28148,"panted":28149,"unharmed":28150,"gael":28151,"housemates":28152,"montpellier":28153,"##59":28154,"coa":28155,"rodents":28156,"tonic":28157,"hickory":28158,"singleton":28159,"##taro":28160,"451":28161,"1719":28162,"aldo":28163,"breaststroke":28164,"dempsey":28165,"och":28166,"rocco":28167,"##cuit":28168,"merton":28169,"dissemination":28170,"midsummer":28171,"serials":28172,"##idi":28173,"haji":28174,"polynomials":28175,"##rdon":28176,"gs":28177,"enoch":28178,"prematurely":28179,"shutter":28180,"taunton":28181,"£3":28182,"##grating":28183,"##inates":28184,"archangel":28185,"harassed":28186,"##asco":28187,"326":28188,"archway":28189,"dazzling":28190,"##ecin":28191,"1736":28192,"sumo":28193,"wat":28194,"##kovich":28195,"1086":28196,"honneur":28197,"##ently":28198,"##nostic":28199,"##ttal":28200,"##idon":28201,"1605":28202,"403":28203,"1716":28204,"blogger":28205,"rents":28206,"##gnan":28207,"hires":28208,"##ikh":28209,"##dant":28210,"howie":28211,"##rons":28212,"handler":28213,"retracted":28214,"shocks":28215,"1632":28216,"arun":28217,"duluth":28218,"kepler":28219,"trumpeter":28220,"##lary":28221,"peeking":28222,"seasoned":28223,"trooper":28224,"##mara":28225,"laszlo":28226,"##iciencies":28227,"##rti":28228,"heterosexual":28229,"##inatory":28230,"##ssion":28231,"indira":28232,"jogging":28233,"##inga":28234,"##lism":28235,"beit":28236,"dissatisfaction":28237,"malice":28238,"##ately":28239,"nedra":28240,"peeling":28241,"##rgeon":28242,"47th":28243,"stadiums":28244,"475":28245,"vertigo":28246,"##ains":28247,"iced":28248,"restroom":28249,"##plify":28250,"##tub":28251,"illustrating":28252,"pear":28253,"##chner":28254,"##sibility":28255,"inorganic":28256,"rappers":28257,"receipts":28258,"watery":28259,"##kura":28260,"lucinda":28261,"##oulos":28262,"reintroduced":28263,"##8th":28264,"##tched":28265,"gracefully":28266,"saxons":28267,"nutritional":28268,"wastewater":28269,"rained":28270,"favourites":28271,"bedrock":28272,"fisted":28273,"hallways":28274,"likeness":28275,"upscale":28276,"##lateral":28277,"1580":28278,"blinds":28279,"prequel":28280,"##pps":28281,"##tama":28282,"deter":28283,"humiliating":28284,"restraining":28285,"tn":28286,"vents":28287,"1659":28288,"laundering":28289,"recess":28290,"rosary":28291,"tractors":28292,"coulter":28293,"federer":28294,"##ifiers":28295,"##plin":28296,"persistence":28297,"##quitable":28298,"geschichte":28299,"pendulum":28300,"quakers":28301,"##beam":28302,"bassett":28303,"pictorial":28304,"buffet":28305,"koln":28306,"##sitor":28307,"drills":28308,"reciprocal":28309,"shooters":28310,"##57":28311,"##cton":28312,"##tees":28313,"converge":28314,"pip":28315,"dmitri":28316,"donnelly":28317,"yamamoto":28318,"aqua":28319,"azores":28320,"demographics":28321,"hypnotic":28322,"spitfire":28323,"suspend":28324,"wryly":28325,"roderick":28326,"##rran":28327,"sebastien":28328,"##asurable":28329,"mavericks":28330,"##fles":28331,"##200":28332,"himalayan":28333,"prodigy":28334,"##iance":28335,"transvaal":28336,"demonstrators":28337,"handcuffs":28338,"dodged":28339,"mcnamara":28340,"sublime":28341,"1726":28342,"crazed":28343,"##efined":28344,"##till":28345,"ivo":28346,"pondered":28347,"reconciled":28348,"shrill":28349,"sava":28350,"##duk":28351,"bal":28352,"cad":28353,"heresy":28354,"jaipur":28355,"goran":28356,"##nished":28357,"341":28358,"lux":28359,"shelly":28360,"whitehall":28361,"##hre":28362,"israelis":28363,"peacekeeping":28364,"##wled":28365,"1703":28366,"demetrius":28367,"ousted":28368,"##arians":28369,"##zos":28370,"beale":28371,"anwar":28372,"backstroke":28373,"raged":28374,"shrinking":28375,"cremated":28376,"##yck":28377,"benign":28378,"towing":28379,"wadi":28380,"darmstadt":28381,"landfill":28382,"parana":28383,"soothe":28384,"colleen":28385,"sidewalks":28386,"mayfair":28387,"tumble":28388,"hepatitis":28389,"ferrer":28390,"superstructure":28391,"##gingly":28392,"##urse":28393,"##wee":28394,"anthropological":28395,"translators":28396,"##mies":28397,"closeness":28398,"hooves":28399,"##pw":28400,"mondays":28401,"##roll":28402,"##vita":28403,"landscaping":28404,"##urized":28405,"purification":28406,"sock":28407,"thorns":28408,"thwarted":28409,"jalan":28410,"tiberius":28411,"##taka":28412,"saline":28413,"##rito":28414,"confidently":28415,"khyber":28416,"sculptors":28417,"##ij":28418,"brahms":28419,"hammersmith":28420,"inspectors":28421,"battista":28422,"fivb":28423,"fragmentation":28424,"hackney":28425,"##uls":28426,"arresting":28427,"exercising":28428,"antoinette":28429,"bedfordshire":28430,"##zily":28431,"dyed":28432,"##hema":28433,"1656":28434,"racetrack":28435,"variability":28436,"##tique":28437,"1655":28438,"austrians":28439,"deteriorating":28440,"madman":28441,"theorists":28442,"aix":28443,"lehman":28444,"weathered":28445,"1731":28446,"decreed":28447,"eruptions":28448,"1729":28449,"flaw":28450,"quinlan":28451,"sorbonne":28452,"flutes":28453,"nunez":28454,"1711":28455,"adored":28456,"downwards":28457,"fable":28458,"rasped":28459,"1712":28460,"moritz":28461,"mouthful":28462,"renegade":28463,"shivers":28464,"stunts":28465,"dysfunction":28466,"restrain":28467,"translit":28468,"327":28469,"pancakes":28470,"##avio":28471,"##cision":28472,"##tray":28473,"351":28474,"vial":28475,"##lden":28476,"bain":28477,"##maid":28478,"##oxide":28479,"chihuahua":28480,"malacca":28481,"vimes":28482,"##rba":28483,"##rnier":28484,"1664":28485,"donnie":28486,"plaques":28487,"##ually":28488,"337":28489,"bangs":28490,"floppy":28491,"huntsville":28492,"loretta":28493,"nikolay":28494,"##otte":28495,"eater":28496,"handgun":28497,"ubiquitous":28498,"##hett":28499,"eras":28500,"zodiac":28501,"1634":28502,"##omorphic":28503,"1820s":28504,"##zog":28505,"cochran":28506,"##bula":28507,"##lithic":28508,"warring":28509,"##rada":28510,"dalai":28511,"excused":28512,"blazers":28513,"mcconnell":28514,"reeling":28515,"bot":28516,"este":28517,"##abi":28518,"geese":28519,"hoax":28520,"taxon":28521,"##bla":28522,"guitarists":28523,"##icon":28524,"condemning":28525,"hunts":28526,"inversion":28527,"moffat":28528,"taekwondo":28529,"##lvis":28530,"1624":28531,"stammered":28532,"##rest":28533,"##rzy":28534,"sousa":28535,"fundraiser":28536,"marylebone":28537,"navigable":28538,"uptown":28539,"cabbage":28540,"daniela":28541,"salman":28542,"shitty":28543,"whimper":28544,"##kian":28545,"##utive":28546,"programmers":28547,"protections":28548,"rm":28549,"##rmi":28550,"##rued":28551,"forceful":28552,"##enes":28553,"fuss":28554,"##tao":28555,"##wash":28556,"brat":28557,"oppressive":28558,"reykjavik":28559,"spartak":28560,"ticking":28561,"##inkles":28562,"##kiewicz":28563,"adolph":28564,"horst":28565,"maui":28566,"protege":28567,"straighten":28568,"cpc":28569,"landau":28570,"concourse":28571,"clements":28572,"resultant":28573,"##ando":28574,"imaginative":28575,"joo":28576,"reactivated":28577,"##rem":28578,"##ffled":28579,"##uising":28580,"consultative":28581,"##guide":28582,"flop":28583,"kaitlyn":28584,"mergers":28585,"parenting":28586,"somber":28587,"##vron":28588,"supervise":28589,"vidhan":28590,"##imum":28591,"courtship":28592,"exemplified":28593,"harmonies":28594,"medallist":28595,"refining":28596,"##rrow":28597,"##ка":28598,"amara":28599,"##hum":28600,"780":28601,"goalscorer":28602,"sited":28603,"overshadowed":28604,"rohan":28605,"displeasure":28606,"secretive":28607,"multiplied":28608,"osman":28609,"##orth":28610,"engravings":28611,"padre":28612,"##kali":28613,"##veda":28614,"miniatures":28615,"mis":28616,"##yala":28617,"clap":28618,"pali":28619,"rook":28620,"##cana":28621,"1692":28622,"57th":28623,"antennae":28624,"astro":28625,"oskar":28626,"1628":28627,"bulldog":28628,"crotch":28629,"hackett":28630,"yucatan":28631,"##sure":28632,"amplifiers":28633,"brno":28634,"ferrara":28635,"migrating":28636,"##gree":28637,"thanking":28638,"turing":28639,"##eza":28640,"mccann":28641,"ting":28642,"andersson":28643,"onslaught":28644,"gaines":28645,"ganga":28646,"incense":28647,"standardization":28648,"##mation":28649,"sentai":28650,"scuba":28651,"stuffing":28652,"turquoise":28653,"waivers":28654,"alloys":28655,"##vitt":28656,"regaining":28657,"vaults":28658,"##clops":28659,"##gizing":28660,"digger":28661,"furry":28662,"memorabilia":28663,"probing":28664,"##iad":28665,"payton":28666,"rec":28667,"deutschland":28668,"filippo":28669,"opaque":28670,"seamen":28671,"zenith":28672,"afrikaans":28673,"##filtration":28674,"disciplined":28675,"inspirational":28676,"##merie":28677,"banco":28678,"confuse":28679,"grafton":28680,"tod":28681,"##dgets":28682,"championed":28683,"simi":28684,"anomaly":28685,"biplane":28686,"##ceptive":28687,"electrode":28688,"##para":28689,"1697":28690,"cleavage":28691,"crossbow":28692,"swirl":28693,"informant":28694,"##lars":28695,"##osta":28696,"afi":28697,"bonfire":28698,"spec":28699,"##oux":28700,"lakeside":28701,"slump":28702,"##culus":28703,"##lais":28704,"##qvist":28705,"##rrigan":28706,"1016":28707,"facades":28708,"borg":28709,"inwardly":28710,"cervical":28711,"xl":28712,"pointedly":28713,"050":28714,"stabilization":28715,"##odon":28716,"chests":28717,"1699":28718,"hacked":28719,"ctv":28720,"orthogonal":28721,"suzy":28722,"##lastic":28723,"gaulle":28724,"jacobite":28725,"rearview":28726,"##cam":28727,"##erted":28728,"ashby":28729,"##drik":28730,"##igate":28731,"##mise":28732,"##zbek":28733,"affectionately":28734,"canine":28735,"disperse":28736,"latham":28737,"##istles":28738,"##ivar":28739,"spielberg":28740,"##orin":28741,"##idium":28742,"ezekiel":28743,"cid":28744,"##sg":28745,"durga":28746,"middletown":28747,"##cina":28748,"customized":28749,"frontiers":28750,"harden":28751,"##etano":28752,"##zzy":28753,"1604":28754,"bolsheviks":28755,"##66":28756,"coloration":28757,"yoko":28758,"##bedo":28759,"briefs":28760,"slabs":28761,"debra":28762,"liquidation":28763,"plumage":28764,"##oin":28765,"blossoms":28766,"dementia":28767,"subsidy":28768,"1611":28769,"proctor":28770,"relational":28771,"jerseys":28772,"parochial":28773,"ter":28774,"##ici":28775,"esa":28776,"peshawar":28777,"cavalier":28778,"loren":28779,"cpi":28780,"idiots":28781,"shamrock":28782,"1646":28783,"dutton":28784,"malabar":28785,"mustache":28786,"##endez":28787,"##ocytes":28788,"referencing":28789,"terminates":28790,"marche":28791,"yarmouth":28792,"##sop":28793,"acton":28794,"mated":28795,"seton":28796,"subtly":28797,"baptised":28798,"beige":28799,"extremes":28800,"jolted":28801,"kristina":28802,"telecast":28803,"##actic":28804,"safeguard":28805,"waldo":28806,"##baldi":28807,"##bular":28808,"endeavors":28809,"sloppy":28810,"subterranean":28811,"##ensburg":28812,"##itung":28813,"delicately":28814,"pigment":28815,"tq":28816,"##scu":28817,"1626":28818,"##ound":28819,"collisions":28820,"coveted":28821,"herds":28822,"##personal":28823,"##meister":28824,"##nberger":28825,"chopra":28826,"##ricting":28827,"abnormalities":28828,"defective":28829,"galician":28830,"lucie":28831,"##dilly":28832,"alligator":28833,"likened":28834,"##genase":28835,"burundi":28836,"clears":28837,"complexion":28838,"derelict":28839,"deafening":28840,"diablo":28841,"fingered":28842,"champaign":28843,"dogg":28844,"enlist":28845,"isotope":28846,"labeling":28847,"mrna":28848,"##erre":28849,"brilliance":28850,"marvelous":28851,"##ayo":28852,"1652":28853,"crawley":28854,"ether":28855,"footed":28856,"dwellers":28857,"deserts":28858,"hamish":28859,"rubs":28860,"warlock":28861,"skimmed":28862,"##lizer":28863,"870":28864,"buick":28865,"embark":28866,"heraldic":28867,"irregularities":28868,"##ajan":28869,"kiara":28870,"##kulam":28871,"##ieg":28872,"antigen":28873,"kowalski":28874,"##lge":28875,"oakley":28876,"visitation":28877,"##mbit":28878,"vt":28879,"##suit":28880,"1570":28881,"murderers":28882,"##miento":28883,"##rites":28884,"chimneys":28885,"##sling":28886,"condemn":28887,"custer":28888,"exchequer":28889,"havre":28890,"##ghi":28891,"fluctuations":28892,"##rations":28893,"dfb":28894,"hendricks":28895,"vaccines":28896,"##tarian":28897,"nietzsche":28898,"biking":28899,"juicy":28900,"##duced":28901,"brooding":28902,"scrolling":28903,"selangor":28904,"##ragan":28905,"352":28906,"annum":28907,"boomed":28908,"seminole":28909,"sugarcane":28910,"##dna":28911,"departmental":28912,"dismissing":28913,"innsbruck":28914,"arteries":28915,"ashok":28916,"batavia":28917,"daze":28918,"kun":28919,"overtook":28920,"##rga":28921,"##tlan":28922,"beheaded":28923,"gaddafi":28924,"holm":28925,"electronically":28926,"faulty":28927,"galilee":28928,"fractures":28929,"kobayashi":28930,"##lized":28931,"gunmen":28932,"magma":28933,"aramaic":28934,"mala":28935,"eastenders":28936,"inference":28937,"messengers":28938,"bf":28939,"##qu":28940,"407":28941,"bathrooms":28942,"##vere":28943,"1658":28944,"flashbacks":28945,"ideally":28946,"misunderstood":28947,"##jali":28948,"##weather":28949,"mendez":28950,"##grounds":28951,"505":28952,"uncanny":28953,"##iii":28954,"1709":28955,"friendships":28956,"##nbc":28957,"sacrament":28958,"accommodated":28959,"reiterated":28960,"logistical":28961,"pebbles":28962,"thumped":28963,"##escence":28964,"administering":28965,"decrees":28966,"drafts":28967,"##flight":28968,"##cased":28969,"##tula":28970,"futuristic":28971,"picket":28972,"intimidation":28973,"winthrop":28974,"##fahan":28975,"interfered":28976,"339":28977,"afar":28978,"francoise":28979,"morally":28980,"uta":28981,"cochin":28982,"croft":28983,"dwarfs":28984,"##bruck":28985,"##dents":28986,"##nami":28987,"biker":28988,"##hner":28989,"##meral":28990,"nano":28991,"##isen":28992,"##ometric":28993,"##pres":28994,"##ан":28995,"brightened":28996,"meek":28997,"parcels":28998,"securely":28999,"gunners":29000,"##jhl":29001,"##zko":29002,"agile":29003,"hysteria":29004,"##lten":29005,"##rcus":29006,"bukit":29007,"champs":29008,"chevy":29009,"cuckoo":29010,"leith":29011,"sadler":29012,"theologians":29013,"welded":29014,"##section":29015,"1663":29016,"jj":29017,"plurality":29018,"xander":29019,"##rooms":29020,"##formed":29021,"shredded":29022,"temps":29023,"intimately":29024,"pau":29025,"tormented":29026,"##lok":29027,"##stellar":29028,"1618":29029,"charred":29030,"ems":29031,"essen":29032,"##mmel":29033,"alarms":29034,"spraying":29035,"ascot":29036,"blooms":29037,"twinkle":29038,"##abia":29039,"##apes":29040,"internment":29041,"obsidian":29042,"##chaft":29043,"snoop":29044,"##dav":29045,"##ooping":29046,"malibu":29047,"##tension":29048,"quiver":29049,"##itia":29050,"hays":29051,"mcintosh":29052,"travers":29053,"walsall":29054,"##ffie":29055,"1623":29056,"beverley":29057,"schwarz":29058,"plunging":29059,"structurally":29060,"m3":29061,"rosenthal":29062,"vikram":29063,"##tsk":29064,"770":29065,"ghz":29066,"##onda":29067,"##tiv":29068,"chalmers":29069,"groningen":29070,"pew":29071,"reckon":29072,"unicef":29073,"##rvis":29074,"55th":29075,"##gni":29076,"1651":29077,"sulawesi":29078,"avila":29079,"cai":29080,"metaphysical":29081,"screwing":29082,"turbulence":29083,"##mberg":29084,"augusto":29085,"samba":29086,"56th":29087,"baffled":29088,"momentary":29089,"toxin":29090,"##urian":29091,"##wani":29092,"aachen":29093,"condoms":29094,"dali":29095,"steppe":29096,"##3d":29097,"##app":29098,"##oed":29099,"##year":29100,"adolescence":29101,"dauphin":29102,"electrically":29103,"inaccessible":29104,"microscopy":29105,"nikita":29106,"##ega":29107,"atv":29108,"##cel":29109,"##enter":29110,"##oles":29111,"##oteric":29112,"##ы":29113,"accountants":29114,"punishments":29115,"wrongly":29116,"bribes":29117,"adventurous":29118,"clinch":29119,"flinders":29120,"southland":29121,"##hem":29122,"##kata":29123,"gough":29124,"##ciency":29125,"lads":29126,"soared":29127,"##ה":29128,"undergoes":29129,"deformation":29130,"outlawed":29131,"rubbish":29132,"##arus":29133,"##mussen":29134,"##nidae":29135,"##rzburg":29136,"arcs":29137,"##ingdon":29138,"##tituted":29139,"1695":29140,"wheelbase":29141,"wheeling":29142,"bombardier":29143,"campground":29144,"zebra":29145,"##lices":29146,"##oj":29147,"##bain":29148,"lullaby":29149,"##ecure":29150,"donetsk":29151,"wylie":29152,"grenada":29153,"##arding":29154,"##ης":29155,"squinting":29156,"eireann":29157,"opposes":29158,"##andra":29159,"maximal":29160,"runes":29161,"##broken":29162,"##cuting":29163,"##iface":29164,"##ror":29165,"##rosis":29166,"additive":29167,"britney":29168,"adultery":29169,"triggering":29170,"##drome":29171,"detrimental":29172,"aarhus":29173,"containment":29174,"jc":29175,"swapped":29176,"vichy":29177,"##ioms":29178,"madly":29179,"##oric":29180,"##rag":29181,"brant":29182,"##ckey":29183,"##trix":29184,"1560":29185,"1612":29186,"broughton":29187,"rustling":29188,"##stems":29189,"##uder":29190,"asbestos":29191,"mentoring":29192,"##nivorous":29193,"finley":29194,"leaps":29195,"##isan":29196,"apical":29197,"pry":29198,"slits":29199,"substitutes":29200,"##dict":29201,"intuitive":29202,"fantasia":29203,"insistent":29204,"unreasonable":29205,"##igen":29206,"##vna":29207,"domed":29208,"hannover":29209,"margot":29210,"ponder":29211,"##zziness":29212,"impromptu":29213,"jian":29214,"lc":29215,"rampage":29216,"stemming":29217,"##eft":29218,"andrey":29219,"gerais":29220,"whichever":29221,"amnesia":29222,"appropriated":29223,"anzac":29224,"clicks":29225,"modifying":29226,"ultimatum":29227,"cambrian":29228,"maids":29229,"verve":29230,"yellowstone":29231,"##mbs":29232,"conservatoire":29233,"##scribe":29234,"adherence":29235,"dinners":29236,"spectra":29237,"imperfect":29238,"mysteriously":29239,"sidekick":29240,"tatar":29241,"tuba":29242,"##aks":29243,"##ifolia":29244,"distrust":29245,"##athan":29246,"##zle":29247,"c2":29248,"ronin":29249,"zac":29250,"##pse":29251,"celaena":29252,"instrumentalist":29253,"scents":29254,"skopje":29255,"##mbling":29256,"comical":29257,"compensated":29258,"vidal":29259,"condor":29260,"intersect":29261,"jingle":29262,"wavelengths":29263,"##urrent":29264,"mcqueen":29265,"##izzly":29266,"carp":29267,"weasel":29268,"422":29269,"kanye":29270,"militias":29271,"postdoctoral":29272,"eugen":29273,"gunslinger":29274,"##ɛ":29275,"faux":29276,"hospice":29277,"##for":29278,"appalled":29279,"derivation":29280,"dwarves":29281,"##elis":29282,"dilapidated":29283,"##folk":29284,"astoria":29285,"philology":29286,"##lwyn":29287,"##otho":29288,"##saka":29289,"inducing":29290,"philanthropy":29291,"##bf":29292,"##itative":29293,"geek":29294,"markedly":29295,"sql":29296,"##yce":29297,"bessie":29298,"indices":29299,"rn":29300,"##flict":29301,"495":29302,"frowns":29303,"resolving":29304,"weightlifting":29305,"tugs":29306,"cleric":29307,"contentious":29308,"1653":29309,"mania":29310,"rms":29311,"##miya":29312,"##reate":29313,"##ruck":29314,"##tucket":29315,"bien":29316,"eels":29317,"marek":29318,"##ayton":29319,"##cence":29320,"discreet":29321,"unofficially":29322,"##ife":29323,"leaks":29324,"##bber":29325,"1705":29326,"332":29327,"dung":29328,"compressor":29329,"hillsborough":29330,"pandit":29331,"shillings":29332,"distal":29333,"##skin":29334,"381":29335,"##tat":29336,"##you":29337,"nosed":29338,"##nir":29339,"mangrove":29340,"undeveloped":29341,"##idia":29342,"textures":29343,"##inho":29344,"##500":29345,"##rise":29346,"ae":29347,"irritating":29348,"nay":29349,"amazingly":29350,"bancroft":29351,"apologetic":29352,"compassionate":29353,"kata":29354,"symphonies":29355,"##lovic":29356,"airspace":29357,"##lch":29358,"930":29359,"gifford":29360,"precautions":29361,"fulfillment":29362,"sevilla":29363,"vulgar":29364,"martinique":29365,"##urities":29366,"looting":29367,"piccolo":29368,"tidy":29369,"##dermott":29370,"quadrant":29371,"armchair":29372,"incomes":29373,"mathematicians":29374,"stampede":29375,"nilsson":29376,"##inking":29377,"##scan":29378,"foo":29379,"quarterfinal":29380,"##ostal":29381,"shang":29382,"shouldered":29383,"squirrels":29384,"##owe":29385,"344":29386,"vinegar":29387,"##bner":29388,"##rchy":29389,"##systems":29390,"delaying":29391,"##trics":29392,"ars":29393,"dwyer":29394,"rhapsody":29395,"sponsoring":29396,"##gration":29397,"bipolar":29398,"cinder":29399,"starters":29400,"##olio":29401,"##urst":29402,"421":29403,"signage":29404,"##nty":29405,"aground":29406,"figurative":29407,"mons":29408,"acquaintances":29409,"duets":29410,"erroneously":29411,"soyuz":29412,"elliptic":29413,"recreated":29414,"##cultural":29415,"##quette":29416,"##ssed":29417,"##tma":29418,"##zcz":29419,"moderator":29420,"scares":29421,"##itaire":29422,"##stones":29423,"##udence":29424,"juniper":29425,"sighting":29426,"##just":29427,"##nsen":29428,"britten":29429,"calabria":29430,"ry":29431,"bop":29432,"cramer":29433,"forsyth":29434,"stillness":29435,"##л":29436,"airmen":29437,"gathers":29438,"unfit":29439,"##umber":29440,"##upt":29441,"taunting":29442,"##rip":29443,"seeker":29444,"streamlined":29445,"##bution":29446,"holster":29447,"schumann":29448,"tread":29449,"vox":29450,"##gano":29451,"##onzo":29452,"strive":29453,"dil":29454,"reforming":29455,"covent":29456,"newbury":29457,"predicting":29458,"##orro":29459,"decorate":29460,"tre":29461,"##puted":29462,"andover":29463,"ie":29464,"asahi":29465,"dept":29466,"dunkirk":29467,"gills":29468,"##tori":29469,"buren":29470,"huskies":29471,"##stis":29472,"##stov":29473,"abstracts":29474,"bets":29475,"loosen":29476,"##opa":29477,"1682":29478,"yearning":29479,"##glio":29480,"##sir":29481,"berman":29482,"effortlessly":29483,"enamel":29484,"napoli":29485,"persist":29486,"##peration":29487,"##uez":29488,"attache":29489,"elisa":29490,"b1":29491,"invitations":29492,"##kic":29493,"accelerating":29494,"reindeer":29495,"boardwalk":29496,"clutches":29497,"nelly":29498,"polka":29499,"starbucks":29500,"##kei":29501,"adamant":29502,"huey":29503,"lough":29504,"unbroken":29505,"adventurer":29506,"embroidery":29507,"inspecting":29508,"stanza":29509,"##ducted":29510,"naia":29511,"taluka":29512,"##pone":29513,"##roids":29514,"chases":29515,"deprivation":29516,"florian":29517,"##jing":29518,"##ppet":29519,"earthly":29520,"##lib":29521,"##ssee":29522,"colossal":29523,"foreigner":29524,"vet":29525,"freaks":29526,"patrice":29527,"rosewood":29528,"triassic":29529,"upstate":29530,"##pkins":29531,"dominates":29532,"ata":29533,"chants":29534,"ks":29535,"vo":29536,"##400":29537,"##bley":29538,"##raya":29539,"##rmed":29540,"555":29541,"agra":29542,"infiltrate":29543,"##ailing":29544,"##ilation":29545,"##tzer":29546,"##uppe":29547,"##werk":29548,"binoculars":29549,"enthusiast":29550,"fujian":29551,"squeak":29552,"##avs":29553,"abolitionist":29554,"almeida":29555,"boredom":29556,"hampstead":29557,"marsden":29558,"rations":29559,"##ands":29560,"inflated":29561,"334":29562,"bonuses":29563,"rosalie":29564,"patna":29565,"##rco":29566,"329":29567,"detachments":29568,"penitentiary":29569,"54th":29570,"flourishing":29571,"woolf":29572,"##dion":29573,"##etched":29574,"papyrus":29575,"##lster":29576,"##nsor":29577,"##toy":29578,"bobbed":29579,"dismounted":29580,"endelle":29581,"inhuman":29582,"motorola":29583,"tbs":29584,"wince":29585,"wreath":29586,"##ticus":29587,"hideout":29588,"inspections":29589,"sanjay":29590,"disgrace":29591,"infused":29592,"pudding":29593,"stalks":29594,"##urbed":29595,"arsenic":29596,"leases":29597,"##hyl":29598,"##rrard":29599,"collarbone":29600,"##waite":29601,"##wil":29602,"dowry":29603,"##bant":29604,"##edance":29605,"genealogical":29606,"nitrate":29607,"salamanca":29608,"scandals":29609,"thyroid":29610,"necessitated":29611,"##!":29612,"##\"":29613,"###":29614,"##$":29615,"##%":29616,"##&":29617,"##'":29618,"##(":29619,"##)":29620,"##*":29621,"##+":29622,"##,":29623,"##-":29624,"##.":29625,"##/":29626,"##:":29627,"##;":29628,"##<":29629,"##=":29630,"##>":29631,"##?":29632,"##@":29633,"##[":29634,"##\\":29635,"##]":29636,"##^":29637,"##_":29638,"##`":29639,"##{":29640,"##|":29641,"##}":29642,"##~":29643,"##¡":29644,"##¢":29645,"##£":29646,"##¤":29647,"##¥":29648,"##¦":29649,"##§":29650,"##¨":29651,"##©":29652,"##ª":29653,"##«":29654,"##¬":29655,"##®":29656,"##±":29657,"##´":29658,"##µ":29659,"##¶":29660,"##·":29661,"##º":29662,"##»":29663,"##¼":29664,"##¾":29665,"##¿":29666,"##æ":29667,"##ð":29668,"##÷":29669,"##þ":29670,"##đ":29671,"##ħ":29672,"##ŋ":29673,"##œ":29674,"##ƒ":29675,"##ɐ":29676,"##ɑ":29677,"##ɒ":29678,"##ɔ":29679,"##ɕ":29680,"##ə":29681,"##ɡ":29682,"##ɣ":29683,"##ɨ":29684,"##ɪ":29685,"##ɫ":29686,"##ɬ":29687,"##ɯ":29688,"##ɲ":29689,"##ɴ":29690,"##ɹ":29691,"##ɾ":29692,"##ʀ":29693,"##ʁ":29694,"##ʂ":29695,"##ʃ":29696,"##ʉ":29697,"##ʊ":29698,"##ʋ":29699,"##ʌ":29700,"##ʎ":29701,"##ʐ":29702,"##ʑ":29703,"##ʒ":29704,"##ʔ":29705,"##ʰ":29706,"##ʲ":29707,"##ʳ":29708,"##ʷ":29709,"##ʸ":29710,"##ʻ":29711,"##ʼ":29712,"##ʾ":29713,"##ʿ":29714,"##ˈ":29715,"##ˡ":29716,"##ˢ":29717,"##ˣ":29718,"##ˤ":29719,"##β":29720,"##γ":29721,"##δ":29722,"##ε":29723,"##ζ":29724,"##θ":29725,"##κ":29726,"##λ":29727,"##μ":29728,"##ξ":29729,"##ο":29730,"##π":29731,"##ρ":29732,"##σ":29733,"##τ":29734,"##υ":29735,"##φ":29736,"##χ":29737,"##ψ":29738,"##ω":29739,"##б":29740,"##г":29741,"##д":29742,"##ж":29743,"##з":29744,"##м":29745,"##п":29746,"##с":29747,"##у":29748,"##ф":29749,"##х":29750,"##ц":29751,"##ч":29752,"##ш":29753,"##щ":29754,"##ъ":29755,"##э":29756,"##ю":29757,"##ђ":29758,"##є":29759,"##і":29760,"##ј":29761,"##љ":29762,"##њ":29763,"##ћ":29764,"##ӏ":29765,"##ա":29766,"##բ":29767,"##գ":29768,"##դ":29769,"##ե":29770,"##թ":29771,"##ի":29772,"##լ":29773,"##կ":29774,"##հ":29775,"##մ":29776,"##յ":29777,"##ն":29778,"##ո":29779,"##պ":29780,"##ս":29781,"##վ":29782,"##տ":29783,"##ր":29784,"##ւ":29785,"##ք":29786,"##־":29787,"##א":29788,"##ב":29789,"##ג":29790,"##ד":29791,"##ו":29792,"##ז":29793,"##ח":29794,"##ט":29795,"##י":29796,"##ך":29797,"##כ":29798,"##ל":29799,"##ם":29800,"##מ":29801,"##ן":29802,"##נ":29803,"##ס":29804,"##ע":29805,"##ף":29806,"##פ":29807,"##ץ":29808,"##צ":29809,"##ק":29810,"##ר":29811,"##ש":29812,"##ת":29813,"##،":29814,"##ء":29815,"##ب":29816,"##ت":29817,"##ث":29818,"##ج":29819,"##ح":29820,"##خ":29821,"##ذ":29822,"##ز":29823,"##س":29824,"##ش":29825,"##ص":29826,"##ض":29827,"##ط":29828,"##ظ":29829,"##ع":29830,"##غ":29831,"##ـ":29832,"##ف":29833,"##ق":29834,"##ك":29835,"##و":29836,"##ى":29837,"##ٹ":29838,"##پ":29839,"##چ":29840,"##ک":29841,"##گ":29842,"##ں":29843,"##ھ":29844,"##ہ":29845,"##ے":29846,"##अ":29847,"##आ":29848,"##उ":29849,"##ए":29850,"##क":29851,"##ख":29852,"##ग":29853,"##च":29854,"##ज":29855,"##ट":29856,"##ड":29857,"##ण":29858,"##त":29859,"##थ":29860,"##द":29861,"##ध":29862,"##न":29863,"##प":29864,"##ब":29865,"##भ":29866,"##म":29867,"##य":29868,"##र":29869,"##ल":29870,"##व":29871,"##श":29872,"##ष":29873,"##स":29874,"##ह":29875,"##ा":29876,"##ि":29877,"##ी":29878,"##ो":29879,"##।":29880,"##॥":29881,"##ং":29882,"##অ":29883,"##আ":29884,"##ই":29885,"##উ":29886,"##এ":29887,"##ও":29888,"##ক":29889,"##খ":29890,"##গ":29891,"##চ":29892,"##ছ":29893,"##জ":29894,"##ট":29895,"##ড":29896,"##ণ":29897,"##ত":29898,"##থ":29899,"##দ":29900,"##ধ":29901,"##ন":29902,"##প":29903,"##ব":29904,"##ভ":29905,"##ম":29906,"##য":29907,"##র":29908,"##ল":29909,"##শ":29910,"##ষ":29911,"##স":29912,"##হ":29913,"##া":29914,"##ি":29915,"##ী":29916,"##ে":29917,"##க":29918,"##ச":29919,"##ட":29920,"##த":29921,"##ந":29922,"##ன":29923,"##ப":29924,"##ம":29925,"##ய":29926,"##ர":29927,"##ல":29928,"##ள":29929,"##வ":29930,"##ா":29931,"##ி":29932,"##ு":29933,"##ே":29934,"##ை":29935,"##ನ":29936,"##ರ":29937,"##ಾ":29938,"##ක":29939,"##ය":29940,"##ර":29941,"##ල":29942,"##ව":29943,"##ා":29944,"##ก":29945,"##ง":29946,"##ต":29947,"##ท":29948,"##น":29949,"##พ":29950,"##ม":29951,"##ย":29952,"##ร":29953,"##ล":29954,"##ว":29955,"##ส":29956,"##อ":29957,"##า":29958,"##เ":29959,"##་":29960,"##།":29961,"##ག":29962,"##ང":29963,"##ད":29964,"##ན":29965,"##པ":29966,"##བ":29967,"##མ":29968,"##འ":29969,"##ར":29970,"##ལ":29971,"##ས":29972,"##မ":29973,"##ა":29974,"##ბ":29975,"##გ":29976,"##დ":29977,"##ე":29978,"##ვ":29979,"##თ":29980,"##ი":29981,"##კ":29982,"##ლ":29983,"##მ":29984,"##ნ":29985,"##ო":29986,"##რ":29987,"##ს":29988,"##ტ":29989,"##უ":29990,"##ᄀ":29991,"##ᄂ":29992,"##ᄃ":29993,"##ᄅ":29994,"##ᄆ":29995,"##ᄇ":29996,"##ᄉ":29997,"##ᄊ":29998,"##ᄋ":29999,"##ᄌ":30000,"##ᄎ":30001,"##ᄏ":30002,"##ᄐ":30003,"##ᄑ":30004,"##ᄒ":30005,"##ᅡ":30006,"##ᅢ":30007,"##ᅥ":30008,"##ᅦ":30009,"##ᅧ":30010,"##ᅩ":30011,"##ᅪ":30012,"##ᅭ":30013,"##ᅮ":30014,"##ᅯ":30015,"##ᅲ":30016,"##ᅳ":30017,"##ᅴ":30018,"##ᅵ":30019,"##ᆨ":30020,"##ᆫ":30021,"##ᆯ":30022,"##ᆷ":30023,"##ᆸ":30024,"##ᆼ":30025,"##ᴬ":30026,"##ᴮ":30027,"##ᴰ":30028,"##ᴵ":30029,"##ᴺ":30030,"##ᵀ":30031,"##ᵃ":30032,"##ᵇ":30033,"##ᵈ":30034,"##ᵉ":30035,"##ᵍ":30036,"##ᵏ":30037,"##ᵐ":30038,"##ᵒ":30039,"##ᵖ":30040,"##ᵗ":30041,"##ᵘ":30042,"##ᵣ":30043,"##ᵤ":30044,"##ᵥ":30045,"##ᶜ":30046,"##ᶠ":30047,"##‐":30048,"##‑":30049,"##‒":30050,"##–":30051,"##—":30052,"##―":30053,"##‖":30054,"##‘":30055,"##’":30056,"##‚":30057,"##“":30058,"##”":30059,"##„":30060,"##†":30061,"##‡":30062,"##•":30063,"##…":30064,"##‰":30065,"##′":30066,"##″":30067,"##›":30068,"##‿":30069,"##⁄":30070,"##⁰":30071,"##ⁱ":30072,"##⁴":30073,"##⁵":30074,"##⁶":30075,"##⁷":30076,"##⁸":30077,"##⁹":30078,"##⁻":30079,"##ⁿ":30080,"##₅":30081,"##₆":30082,"##₇":30083,"##₈":30084,"##₉":30085,"##₊":30086,"##₍":30087,"##₎":30088,"##ₐ":30089,"##ₑ":30090,"##ₒ":30091,"##ₓ":30092,"##ₕ":30093,"##ₖ":30094,"##ₗ":30095,"##ₘ":30096,"##ₚ":30097,"##ₛ":30098,"##ₜ":30099,"##₤":30100,"##₩":30101,"##€":30102,"##₱":30103,"##₹":30104,"##ℓ":30105,"##№":30106,"##ℝ":30107,"##™":30108,"##⅓":30109,"##⅔":30110,"##←":30111,"##↑":30112,"##→":30113,"##↓":30114,"##↔":30115,"##↦":30116,"##⇄":30117,"##⇌":30118,"##⇒":30119,"##∂":30120,"##∅":30121,"##∆":30122,"##∇":30123,"##∈":30124,"##∗":30125,"##∘":30126,"##√":30127,"##∞":30128,"##∧":30129,"##∨":30130,"##∩":30131,"##∪":30132,"##≈":30133,"##≡":30134,"##≤":30135,"##≥":30136,"##⊂":30137,"##⊆":30138,"##⊕":30139,"##⊗":30140,"##⋅":30141,"##─":30142,"##│":30143,"##■":30144,"##▪":30145,"##●":30146,"##★":30147,"##☆":30148,"##☉":30149,"##♠":30150,"##♣":30151,"##♥":30152,"##♦":30153,"##♯":30154,"##⟨":30155,"##⟩":30156,"##ⱼ":30157,"##⺩":30158,"##⺼":30159,"##⽥":30160,"##、":30161,"##。":30162,"##〈":30163,"##〉":30164,"##《":30165,"##》":30166,"##「":30167,"##」":30168,"##『":30169,"##』":30170,"##〜":30171,"##あ":30172,"##い":30173,"##う":30174,"##え":30175,"##お":30176,"##か":30177,"##き":30178,"##く":30179,"##け":30180,"##こ":30181,"##さ":30182,"##し":30183,"##す":30184,"##せ":30185,"##そ":30186,"##た":30187,"##ち":30188,"##っ":30189,"##つ":30190,"##て":30191,"##と":30192,"##な":30193,"##に":30194,"##ぬ":30195,"##ね":30196,"##の":30197,"##は":30198,"##ひ":30199,"##ふ":30200,"##へ":30201,"##ほ":30202,"##ま":30203,"##み":30204,"##む":30205,"##め":30206,"##も":30207,"##や":30208,"##ゆ":30209,"##よ":30210,"##ら":30211,"##り":30212,"##る":30213,"##れ":30214,"##ろ":30215,"##を":30216,"##ん":30217,"##ァ":30218,"##ア":30219,"##ィ":30220,"##イ":30221,"##ウ":30222,"##ェ":30223,"##エ":30224,"##オ":30225,"##カ":30226,"##キ":30227,"##ク":30228,"##ケ":30229,"##コ":30230,"##サ":30231,"##シ":30232,"##ス":30233,"##セ":30234,"##タ":30235,"##チ":30236,"##ッ":30237,"##ツ":30238,"##テ":30239,"##ト":30240,"##ナ":30241,"##ニ":30242,"##ノ":30243,"##ハ":30244,"##ヒ":30245,"##フ":30246,"##ヘ":30247,"##ホ":30248,"##マ":30249,"##ミ":30250,"##ム":30251,"##メ":30252,"##モ":30253,"##ャ":30254,"##ュ":30255,"##ョ":30256,"##ラ":30257,"##リ":30258,"##ル":30259,"##レ":30260,"##ロ":30261,"##ワ":30262,"##ン":30263,"##・":30264,"##ー":30265,"##一":30266,"##三":30267,"##上":30268,"##下":30269,"##不":30270,"##世":30271,"##中":30272,"##主":30273,"##久":30274,"##之":30275,"##也":30276,"##事":30277,"##二":30278,"##五":30279,"##井":30280,"##京":30281,"##人":30282,"##亻":30283,"##仁":30284,"##介":30285,"##代":30286,"##仮":30287,"##伊":30288,"##会":30289,"##佐":30290,"##侍":30291,"##保":30292,"##信":30293,"##健":30294,"##元":30295,"##光":30296,"##八":30297,"##公":30298,"##内":30299,"##出":30300,"##分":30301,"##前":30302,"##劉":30303,"##力":30304,"##加":30305,"##勝":30306,"##北":30307,"##区":30308,"##十":30309,"##千":30310,"##南":30311,"##博":30312,"##原":30313,"##口":30314,"##古":30315,"##史":30316,"##司":30317,"##合":30318,"##吉":30319,"##同":30320,"##名":30321,"##和":30322,"##囗":30323,"##四":30324,"##国":30325,"##國":30326,"##土":30327,"##地":30328,"##坂":30329,"##城":30330,"##堂":30331,"##場":30332,"##士":30333,"##夏":30334,"##外":30335,"##大":30336,"##天":30337,"##太":30338,"##夫":30339,"##奈":30340,"##女":30341,"##子":30342,"##学":30343,"##宀":30344,"##宇":30345,"##安":30346,"##宗":30347,"##定":30348,"##宣":30349,"##宮":30350,"##家":30351,"##宿":30352,"##寺":30353,"##將":30354,"##小":30355,"##尚":30356,"##山":30357,"##岡":30358,"##島":30359,"##崎":30360,"##川":30361,"##州":30362,"##巿":30363,"##帝":30364,"##平":30365,"##年":30366,"##幸":30367,"##广":30368,"##弘":30369,"##張":30370,"##彳":30371,"##後":30372,"##御":30373,"##德":30374,"##心":30375,"##忄":30376,"##志":30377,"##忠":30378,"##愛":30379,"##成":30380,"##我":30381,"##戦":30382,"##戸":30383,"##手":30384,"##扌":30385,"##政":30386,"##文":30387,"##新":30388,"##方":30389,"##日":30390,"##明":30391,"##星":30392,"##春":30393,"##昭":30394,"##智":30395,"##曲":30396,"##書":30397,"##月":30398,"##有":30399,"##朝":30400,"##木":30401,"##本":30402,"##李":30403,"##村":30404,"##東":30405,"##松":30406,"##林":30407,"##森":30408,"##楊":30409,"##樹":30410,"##橋":30411,"##歌":30412,"##止":30413,"##正":30414,"##武":30415,"##比":30416,"##氏":30417,"##民":30418,"##水":30419,"##氵":30420,"##氷":30421,"##永":30422,"##江":30423,"##沢":30424,"##河":30425,"##治":30426,"##法":30427,"##海":30428,"##清":30429,"##漢":30430,"##瀬":30431,"##火":30432,"##版":30433,"##犬":30434,"##王":30435,"##生":30436,"##田":30437,"##男":30438,"##疒":30439,"##発":30440,"##白":30441,"##的":30442,"##皇":30443,"##目":30444,"##相":30445,"##省":30446,"##真":30447,"##石":30448,"##示":30449,"##社":30450,"##神":30451,"##福":30452,"##禾":30453,"##秀":30454,"##秋":30455,"##空":30456,"##立":30457,"##章":30458,"##竹":30459,"##糹":30460,"##美":30461,"##義":30462,"##耳":30463,"##良":30464,"##艹":30465,"##花":30466,"##英":30467,"##華":30468,"##葉":30469,"##藤":30470,"##行":30471,"##街":30472,"##西":30473,"##見":30474,"##訁":30475,"##語":30476,"##谷":30477,"##貝":30478,"##貴":30479,"##車":30480,"##軍":30481,"##辶":30482,"##道":30483,"##郎":30484,"##郡":30485,"##部":30486,"##都":30487,"##里":30488,"##野":30489,"##金":30490,"##鈴":30491,"##镇":30492,"##長":30493,"##門":30494,"##間":30495,"##阝":30496,"##阿":30497,"##陳":30498,"##陽":30499,"##雄":30500,"##青":30501,"##面":30502,"##風":30503,"##食":30504,"##香":30505,"##馬":30506,"##高":30507,"##龍":30508,"##龸":30509,"##fi":30510,"##fl":30511,"##！":30512,"##（":30513,"##）":30514,"##，":30515,"##－":30516,"##．":30517,"##／":30518,"##：":30519,"##？":30520,"##～":30521}}} \ No newline at end of file diff --git a/new_impl/cv/glip/object_detection/bert-base-uncased/tokenizer_config.json b/new_impl/cv/glip/object_detection/bert-base-uncased/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..a661b1a138dac6dc5590367402d100765010ffd6 --- /dev/null +++ b/new_impl/cv/glip/object_detection/bert-base-uncased/tokenizer_config.json @@ -0,0 +1,3 @@ +{ + "do_lower_case": true +} diff --git a/new_impl/cv/glip/object_detection/bert-base-uncased/vocab.txt b/new_impl/cv/glip/object_detection/bert-base-uncased/vocab.txt new file mode 100644 index 0000000000000000000000000000000000000000..fb140275c155a9c7c5a3b3e0e77a9e839594a938 --- /dev/null +++ b/new_impl/cv/glip/object_detection/bert-base-uncased/vocab.txt @@ -0,0 +1,30522 @@ +[PAD] +[unused0] +[unused1] +[unused2] +[unused3] +[unused4] +[unused5] +[unused6] +[unused7] +[unused8] +[unused9] +[unused10] +[unused11] +[unused12] +[unused13] +[unused14] +[unused15] +[unused16] +[unused17] +[unused18] +[unused19] +[unused20] +[unused21] +[unused22] +[unused23] +[unused24] +[unused25] +[unused26] +[unused27] +[unused28] +[unused29] +[unused30] +[unused31] +[unused32] +[unused33] +[unused34] +[unused35] +[unused36] +[unused37] +[unused38] +[unused39] +[unused40] +[unused41] +[unused42] +[unused43] +[unused44] +[unused45] +[unused46] +[unused47] +[unused48] +[unused49] +[unused50] +[unused51] +[unused52] +[unused53] +[unused54] +[unused55] +[unused56] +[unused57] +[unused58] +[unused59] +[unused60] +[unused61] +[unused62] +[unused63] +[unused64] +[unused65] +[unused66] +[unused67] +[unused68] +[unused69] +[unused70] +[unused71] +[unused72] +[unused73] +[unused74] +[unused75] +[unused76] +[unused77] +[unused78] +[unused79] +[unused80] +[unused81] +[unused82] +[unused83] +[unused84] +[unused85] +[unused86] +[unused87] +[unused88] +[unused89] +[unused90] +[unused91] +[unused92] +[unused93] +[unused94] +[unused95] +[unused96] +[unused97] +[unused98] +[UNK] +[CLS] +[SEP] +[MASK] +[unused99] +[unused100] +[unused101] +[unused102] +[unused103] +[unused104] +[unused105] +[unused106] +[unused107] +[unused108] +[unused109] +[unused110] +[unused111] +[unused112] +[unused113] +[unused114] +[unused115] +[unused116] +[unused117] +[unused118] +[unused119] +[unused120] +[unused121] +[unused122] +[unused123] +[unused124] +[unused125] +[unused126] +[unused127] +[unused128] +[unused129] +[unused130] +[unused131] +[unused132] +[unused133] +[unused134] +[unused135] +[unused136] +[unused137] +[unused138] +[unused139] +[unused140] +[unused141] +[unused142] +[unused143] +[unused144] +[unused145] +[unused146] +[unused147] +[unused148] +[unused149] +[unused150] +[unused151] +[unused152] +[unused153] +[unused154] +[unused155] +[unused156] +[unused157] +[unused158] +[unused159] +[unused160] +[unused161] +[unused162] +[unused163] +[unused164] +[unused165] +[unused166] +[unused167] +[unused168] +[unused169] +[unused170] +[unused171] +[unused172] +[unused173] +[unused174] +[unused175] +[unused176] +[unused177] +[unused178] +[unused179] +[unused180] +[unused181] +[unused182] +[unused183] +[unused184] +[unused185] +[unused186] +[unused187] +[unused188] +[unused189] +[unused190] +[unused191] +[unused192] +[unused193] +[unused194] +[unused195] +[unused196] +[unused197] +[unused198] +[unused199] +[unused200] +[unused201] +[unused202] +[unused203] +[unused204] +[unused205] +[unused206] +[unused207] +[unused208] +[unused209] +[unused210] +[unused211] +[unused212] +[unused213] +[unused214] +[unused215] +[unused216] +[unused217] +[unused218] +[unused219] +[unused220] +[unused221] +[unused222] +[unused223] +[unused224] +[unused225] +[unused226] +[unused227] +[unused228] +[unused229] +[unused230] +[unused231] +[unused232] +[unused233] +[unused234] +[unused235] +[unused236] +[unused237] +[unused238] +[unused239] +[unused240] +[unused241] +[unused242] +[unused243] +[unused244] +[unused245] +[unused246] +[unused247] +[unused248] +[unused249] +[unused250] +[unused251] +[unused252] +[unused253] +[unused254] +[unused255] +[unused256] +[unused257] +[unused258] +[unused259] +[unused260] +[unused261] +[unused262] +[unused263] +[unused264] +[unused265] +[unused266] +[unused267] +[unused268] +[unused269] +[unused270] +[unused271] +[unused272] +[unused273] +[unused274] +[unused275] +[unused276] +[unused277] +[unused278] +[unused279] +[unused280] +[unused281] +[unused282] +[unused283] +[unused284] +[unused285] +[unused286] +[unused287] +[unused288] +[unused289] +[unused290] +[unused291] +[unused292] +[unused293] +[unused294] +[unused295] +[unused296] +[unused297] +[unused298] +[unused299] +[unused300] +[unused301] +[unused302] +[unused303] +[unused304] +[unused305] +[unused306] +[unused307] +[unused308] +[unused309] +[unused310] +[unused311] +[unused312] +[unused313] +[unused314] +[unused315] +[unused316] +[unused317] +[unused318] +[unused319] +[unused320] +[unused321] +[unused322] +[unused323] +[unused324] +[unused325] +[unused326] +[unused327] +[unused328] +[unused329] +[unused330] +[unused331] +[unused332] +[unused333] +[unused334] +[unused335] +[unused336] +[unused337] +[unused338] +[unused339] +[unused340] +[unused341] +[unused342] +[unused343] +[unused344] +[unused345] +[unused346] +[unused347] +[unused348] +[unused349] +[unused350] +[unused351] +[unused352] +[unused353] +[unused354] +[unused355] +[unused356] +[unused357] +[unused358] +[unused359] +[unused360] +[unused361] +[unused362] +[unused363] +[unused364] +[unused365] +[unused366] +[unused367] +[unused368] +[unused369] +[unused370] +[unused371] +[unused372] +[unused373] +[unused374] +[unused375] +[unused376] +[unused377] +[unused378] +[unused379] +[unused380] +[unused381] +[unused382] +[unused383] +[unused384] +[unused385] +[unused386] +[unused387] +[unused388] +[unused389] +[unused390] +[unused391] +[unused392] +[unused393] +[unused394] +[unused395] +[unused396] +[unused397] +[unused398] +[unused399] +[unused400] +[unused401] +[unused402] +[unused403] +[unused404] +[unused405] +[unused406] +[unused407] +[unused408] +[unused409] +[unused410] +[unused411] +[unused412] +[unused413] +[unused414] +[unused415] +[unused416] +[unused417] +[unused418] +[unused419] +[unused420] +[unused421] +[unused422] +[unused423] +[unused424] +[unused425] +[unused426] +[unused427] +[unused428] +[unused429] +[unused430] +[unused431] +[unused432] +[unused433] +[unused434] +[unused435] +[unused436] +[unused437] +[unused438] +[unused439] +[unused440] +[unused441] +[unused442] +[unused443] +[unused444] +[unused445] +[unused446] +[unused447] +[unused448] +[unused449] +[unused450] +[unused451] +[unused452] +[unused453] +[unused454] +[unused455] +[unused456] +[unused457] +[unused458] +[unused459] +[unused460] +[unused461] +[unused462] +[unused463] +[unused464] +[unused465] +[unused466] +[unused467] +[unused468] +[unused469] +[unused470] +[unused471] +[unused472] +[unused473] +[unused474] +[unused475] +[unused476] +[unused477] +[unused478] +[unused479] +[unused480] +[unused481] +[unused482] +[unused483] +[unused484] +[unused485] +[unused486] +[unused487] +[unused488] +[unused489] +[unused490] +[unused491] +[unused492] +[unused493] +[unused494] +[unused495] +[unused496] +[unused497] +[unused498] +[unused499] +[unused500] +[unused501] +[unused502] +[unused503] +[unused504] +[unused505] +[unused506] +[unused507] +[unused508] +[unused509] +[unused510] +[unused511] +[unused512] +[unused513] +[unused514] +[unused515] +[unused516] +[unused517] +[unused518] +[unused519] +[unused520] +[unused521] +[unused522] +[unused523] +[unused524] +[unused525] +[unused526] +[unused527] +[unused528] +[unused529] +[unused530] +[unused531] +[unused532] +[unused533] +[unused534] +[unused535] +[unused536] +[unused537] +[unused538] +[unused539] +[unused540] +[unused541] +[unused542] +[unused543] +[unused544] +[unused545] +[unused546] +[unused547] +[unused548] +[unused549] +[unused550] +[unused551] +[unused552] +[unused553] +[unused554] +[unused555] +[unused556] +[unused557] +[unused558] +[unused559] +[unused560] +[unused561] +[unused562] +[unused563] +[unused564] +[unused565] +[unused566] +[unused567] +[unused568] +[unused569] +[unused570] +[unused571] +[unused572] +[unused573] +[unused574] +[unused575] +[unused576] +[unused577] +[unused578] +[unused579] +[unused580] +[unused581] +[unused582] +[unused583] +[unused584] +[unused585] +[unused586] +[unused587] +[unused588] +[unused589] +[unused590] +[unused591] +[unused592] +[unused593] +[unused594] +[unused595] +[unused596] +[unused597] +[unused598] +[unused599] +[unused600] +[unused601] +[unused602] +[unused603] +[unused604] +[unused605] +[unused606] +[unused607] +[unused608] +[unused609] +[unused610] +[unused611] +[unused612] +[unused613] +[unused614] +[unused615] +[unused616] +[unused617] +[unused618] +[unused619] +[unused620] +[unused621] +[unused622] +[unused623] +[unused624] +[unused625] +[unused626] +[unused627] +[unused628] +[unused629] +[unused630] +[unused631] +[unused632] +[unused633] +[unused634] +[unused635] +[unused636] +[unused637] +[unused638] +[unused639] +[unused640] +[unused641] +[unused642] +[unused643] +[unused644] +[unused645] +[unused646] +[unused647] +[unused648] +[unused649] +[unused650] +[unused651] +[unused652] +[unused653] +[unused654] +[unused655] +[unused656] +[unused657] +[unused658] +[unused659] +[unused660] +[unused661] +[unused662] +[unused663] +[unused664] +[unused665] +[unused666] +[unused667] +[unused668] +[unused669] +[unused670] +[unused671] +[unused672] +[unused673] +[unused674] +[unused675] +[unused676] +[unused677] +[unused678] +[unused679] +[unused680] +[unused681] +[unused682] +[unused683] +[unused684] +[unused685] +[unused686] +[unused687] +[unused688] +[unused689] +[unused690] +[unused691] +[unused692] +[unused693] +[unused694] +[unused695] +[unused696] +[unused697] +[unused698] +[unused699] +[unused700] +[unused701] +[unused702] +[unused703] +[unused704] +[unused705] +[unused706] +[unused707] +[unused708] +[unused709] +[unused710] +[unused711] +[unused712] +[unused713] +[unused714] +[unused715] +[unused716] +[unused717] +[unused718] +[unused719] +[unused720] +[unused721] +[unused722] +[unused723] +[unused724] +[unused725] +[unused726] +[unused727] +[unused728] +[unused729] +[unused730] +[unused731] +[unused732] +[unused733] +[unused734] +[unused735] +[unused736] +[unused737] +[unused738] +[unused739] +[unused740] +[unused741] +[unused742] +[unused743] +[unused744] +[unused745] +[unused746] +[unused747] +[unused748] +[unused749] +[unused750] +[unused751] +[unused752] +[unused753] +[unused754] +[unused755] +[unused756] +[unused757] +[unused758] +[unused759] +[unused760] +[unused761] +[unused762] +[unused763] +[unused764] +[unused765] +[unused766] +[unused767] +[unused768] +[unused769] +[unused770] +[unused771] +[unused772] +[unused773] +[unused774] +[unused775] +[unused776] +[unused777] +[unused778] +[unused779] +[unused780] +[unused781] +[unused782] +[unused783] +[unused784] +[unused785] +[unused786] +[unused787] +[unused788] +[unused789] +[unused790] +[unused791] +[unused792] +[unused793] +[unused794] +[unused795] +[unused796] +[unused797] +[unused798] +[unused799] +[unused800] +[unused801] +[unused802] +[unused803] +[unused804] +[unused805] +[unused806] +[unused807] +[unused808] +[unused809] +[unused810] +[unused811] +[unused812] +[unused813] +[unused814] +[unused815] +[unused816] +[unused817] +[unused818] +[unused819] +[unused820] +[unused821] +[unused822] +[unused823] +[unused824] +[unused825] +[unused826] +[unused827] +[unused828] +[unused829] +[unused830] +[unused831] +[unused832] +[unused833] +[unused834] +[unused835] +[unused836] +[unused837] +[unused838] +[unused839] +[unused840] +[unused841] +[unused842] +[unused843] +[unused844] +[unused845] +[unused846] +[unused847] +[unused848] +[unused849] +[unused850] +[unused851] +[unused852] +[unused853] +[unused854] +[unused855] +[unused856] +[unused857] +[unused858] +[unused859] +[unused860] +[unused861] +[unused862] +[unused863] +[unused864] +[unused865] +[unused866] +[unused867] +[unused868] +[unused869] +[unused870] +[unused871] +[unused872] +[unused873] +[unused874] +[unused875] +[unused876] +[unused877] +[unused878] +[unused879] +[unused880] +[unused881] +[unused882] +[unused883] +[unused884] +[unused885] +[unused886] +[unused887] +[unused888] +[unused889] +[unused890] +[unused891] +[unused892] +[unused893] +[unused894] +[unused895] +[unused896] +[unused897] +[unused898] +[unused899] +[unused900] +[unused901] +[unused902] +[unused903] +[unused904] +[unused905] +[unused906] +[unused907] +[unused908] +[unused909] +[unused910] +[unused911] +[unused912] +[unused913] +[unused914] +[unused915] +[unused916] +[unused917] +[unused918] +[unused919] +[unused920] +[unused921] +[unused922] +[unused923] +[unused924] +[unused925] +[unused926] +[unused927] +[unused928] +[unused929] +[unused930] +[unused931] +[unused932] +[unused933] +[unused934] +[unused935] +[unused936] +[unused937] +[unused938] +[unused939] +[unused940] +[unused941] +[unused942] +[unused943] +[unused944] +[unused945] +[unused946] +[unused947] +[unused948] +[unused949] +[unused950] +[unused951] +[unused952] +[unused953] +[unused954] +[unused955] +[unused956] +[unused957] +[unused958] +[unused959] +[unused960] +[unused961] +[unused962] +[unused963] +[unused964] +[unused965] +[unused966] +[unused967] +[unused968] +[unused969] +[unused970] +[unused971] +[unused972] +[unused973] +[unused974] +[unused975] +[unused976] +[unused977] +[unused978] +[unused979] +[unused980] +[unused981] +[unused982] +[unused983] +[unused984] +[unused985] +[unused986] +[unused987] +[unused988] +[unused989] +[unused990] +[unused991] +[unused992] +[unused993] +! +" +# +$ +% +& +' +( +) +* ++ +, +- +. +/ +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +: +; +< += +> +? +@ +[ +\ +] +^ +_ +` +a +b +c +d +e +f +g +h +i +j +k +l +m +n +o +p +q +r +s +t +u +v +w +x +y +z +{ +| +} +~ +¡ +¢ +£ +¤ +¥ +¦ +§ +¨ +© +ª +« +¬ +® +° +± +² +³ +´ +µ +¶ +· +¹ +º +» +¼ +½ +¾ +¿ +× +ß +æ +ð +÷ +ø +þ +đ +ħ +ı +ł +ŋ +œ +ƒ +ɐ +ɑ +ɒ +ɔ +ɕ +ə +ɛ +ɡ +ɣ +ɨ +ɪ +ɫ +ɬ +ɯ +ɲ +ɴ +ɹ +ɾ +ʀ +ʁ +ʂ +ʃ +ʉ +ʊ +ʋ +ʌ +ʎ +ʐ +ʑ +ʒ +ʔ +ʰ +ʲ +ʳ +ʷ +ʸ +ʻ +ʼ +ʾ +ʿ +ˈ +ː +ˡ +ˢ +ˣ +ˤ +α +β +γ +δ +ε +ζ +η +θ +ι +κ +λ +μ +ν +ξ +ο +π +ρ +ς +σ +τ +υ +φ +χ +ψ +ω +а +б +в +г +д +е +ж +з +и +к +л +м +н +о +п +р +с +т +у +ф +х +ц +ч +ш +щ +ъ +ы +ь +э +ю +я +ђ +є +і +ј +љ +њ +ћ +ӏ +ա +բ +գ +դ +ե +թ +ի +լ +կ +հ +մ +յ +ն +ո +պ +ս +վ +տ +ր +ւ +ք +־ +א +ב +ג +ד +ה +ו +ז +ח +ט +י +ך +כ +ל +ם +מ +ן +נ +ס +ע +ף +פ +ץ +צ +ק +ר +ש +ת +، +ء +ا +ب +ة +ت +ث +ج +ح +خ +د +ذ +ر +ز +س +ش +ص +ض +ط +ظ +ع +غ +ـ +ف +ق +ك +ل +م +ن +ه +و +ى +ي +ٹ +پ +چ +ک +گ +ں +ھ +ہ +ی +ے +अ +आ +उ +ए +क +ख +ग +च +ज +ट +ड +ण +त +थ +द +ध +न +प +ब +भ +म +य +र +ल +व +श +ष +स +ह +ा +ि +ी +ो +। +॥ +ং +অ +আ +ই +উ +এ +ও +ক +খ +গ +চ +ছ +জ +ট +ড +ণ +ত +থ +দ +ধ +ন +প +ব +ভ +ম +য +র +ল +শ +ষ +স +হ +া +ি +ী +ে +க +ச +ட +த +ந +ன +ப +ம +ய +ர +ல +ள +வ +ா +ி +ு +ே +ை +ನ +ರ +ಾ +ක +ය +ර +ල +ව +ා +ก +ง +ต +ท +น +พ +ม +ย +ร +ล +ว +ส +อ +า +เ +་ +། +ག +ང +ད +ན +པ +བ +མ +འ +ར +ལ +ས +မ +ა +ბ +გ +დ +ე +ვ +თ +ი +კ +ლ +მ +ნ +ო +რ +ს +ტ +უ +ᄀ +ᄂ +ᄃ +ᄅ +ᄆ +ᄇ +ᄉ +ᄊ +ᄋ +ᄌ +ᄎ +ᄏ +ᄐ +ᄑ +ᄒ +ᅡ +ᅢ +ᅥ +ᅦ +ᅧ +ᅩ +ᅪ +ᅭ +ᅮ +ᅯ +ᅲ +ᅳ +ᅴ +ᅵ +ᆨ +ᆫ +ᆯ +ᆷ +ᆸ +ᆼ +ᴬ +ᴮ +ᴰ +ᴵ +ᴺ +ᵀ +ᵃ +ᵇ +ᵈ +ᵉ +ᵍ +ᵏ +ᵐ +ᵒ +ᵖ +ᵗ +ᵘ +ᵢ +ᵣ +ᵤ +ᵥ +ᶜ +ᶠ +‐ +‑ +‒ +– +— +― +‖ +‘ +’ +‚ +“ +” +„ +† +‡ +• +… +‰ +′ +″ +› +‿ +⁄ +⁰ +ⁱ +⁴ +⁵ +⁶ +⁷ +⁸ +⁹ +⁺ +⁻ +ⁿ +₀ +₁ +₂ +₃ +₄ +₅ +₆ +₇ +₈ +₉ +₊ +₍ +₎ +ₐ +ₑ +ₒ +ₓ +ₕ +ₖ +ₗ +ₘ +ₙ +ₚ +ₛ +ₜ +₤ +₩ +€ +₱ +₹ +ℓ +№ +ℝ +™ +⅓ +⅔ +← +↑ +→ +↓ +↔ +↦ +⇄ +⇌ +⇒ +∂ +∅ +∆ +∇ +∈ +− +∗ +∘ +√ +∞ +∧ +∨ +∩ +∪ +≈ +≡ +≤ +≥ +⊂ +⊆ +⊕ +⊗ +⋅ +─ +│ +■ +▪ +● +★ +☆ +☉ +♠ +♣ +♥ +♦ +♭ +♯ +⟨ +⟩ +ⱼ +⺩ +⺼ +⽥ +、 +。 +〈 +〉 +《 +》 +「 +」 +『 +』 +〜 +あ +い +う +え +お +か +き +く +け +こ +さ +し +す +せ +そ +た +ち +っ +つ +て +と +な +に +ぬ +ね +の +は +ひ +ふ +へ +ほ +ま +み +む +め +も +や +ゆ +よ +ら +り +る +れ +ろ +を +ん +ァ +ア +ィ +イ +ウ +ェ +エ +オ +カ +キ +ク +ケ +コ +サ +シ +ス +セ +タ +チ +ッ +ツ +テ +ト +ナ +ニ +ノ +ハ +ヒ +フ +ヘ +ホ +マ +ミ +ム +メ +モ +ャ +ュ +ョ +ラ +リ +ル +レ +ロ +ワ +ン +・ +ー +一 +三 +上 +下 +不 +世 +中 +主 +久 +之 +也 +事 +二 +五 +井 +京 +人 +亻 +仁 +介 +代 +仮 +伊 +会 +佐 +侍 +保 +信 +健 +元 +光 +八 +公 +内 +出 +分 +前 +劉 +力 +加 +勝 +北 +区 +十 +千 +南 +博 +原 +口 +古 +史 +司 +合 +吉 +同 +名 +和 +囗 +四 +国 +國 +土 +地 +坂 +城 +堂 +場 +士 +夏 +外 +大 +天 +太 +夫 +奈 +女 +子 +学 +宀 +宇 +安 +宗 +定 +宣 +宮 +家 +宿 +寺 +將 +小 +尚 +山 +岡 +島 +崎 +川 +州 +巿 +帝 +平 +年 +幸 +广 +弘 +張 +彳 +後 +御 +德 +心 +忄 +志 +忠 +愛 +成 +我 +戦 +戸 +手 +扌 +政 +文 +新 +方 +日 +明 +星 +春 +昭 +智 +曲 +書 +月 +有 +朝 +木 +本 +李 +村 +東 +松 +林 +森 +楊 +樹 +橋 +歌 +止 +正 +武 +比 +氏 +民 +水 +氵 +氷 +永 +江 +沢 +河 +治 +法 +海 +清 +漢 +瀬 +火 +版 +犬 +王 +生 +田 +男 +疒 +発 +白 +的 +皇 +目 +相 +省 +真 +石 +示 +社 +神 +福 +禾 +秀 +秋 +空 +立 +章 +竹 +糹 +美 +義 +耳 +良 +艹 +花 +英 +華 +葉 +藤 +行 +街 +西 +見 +訁 +語 +谷 +貝 +貴 +車 +軍 +辶 +道 +郎 +郡 +部 +都 +里 +野 +金 +鈴 +镇 +長 +門 +間 +阝 +阿 +陳 +陽 +雄 +青 +面 +風 +食 +香 +馬 +高 +龍 +龸 +fi +fl +！ +（ +） +， +－ +． +／ +： +？ +～ +the +of +and +in +to +was +he +is +as +for +on +with +that +it +his +by +at +from +her +##s +she +you +had +an +were +but +be +this +are +not +my +they +one +which +or +have +him +me +first +all +also +their +has +up +who +out +been +when +after +there +into +new +two +its +##a +time +would +no +what +about +said +we +over +then +other +so +more +##e +can +if +like +back +them +only +some +could +##i +where +just +##ing +during +before +##n +do +##o +made +school +through +than +now +years +most +world +may +between +down +well +three +##d +year +while +will +##ed +##r +##y +later +##t +city +under +around +did +such +being +used +state +people +part +know +against +your +many +second +university +both +national +##er +these +don +known +off +way +until +re +how +even +get +head +... +didn +##ly +team +american +because +de +##l +born +united +film +since +still +long +work +south +us +became +any +high +again +day +family +see +right +man +eyes +house +season +war +states +including +took +life +north +same +each +called +name +much +place +however +go +four +group +another +found +won +area +here +going +10 +away +series +left +home +music +best +make +hand +number +company +several +never +last +john +000 +very +album +take +end +good +too +following +released +game +played +little +began +district +##m +old +want +those +side +held +own +early +county +ll +league +use +west +##u +face +think +##es +2010 +government +##h +march +came +small +general +town +june +##on +line +based +something +##k +september +thought +looked +along +international +2011 +air +july +club +went +january +october +our +august +april +york +12 +few +2012 +2008 +east +show +member +college +2009 +father +public +##us +come +men +five +set +station +church +##c +next +former +november +room +party +located +december +2013 +age +got +2007 +##g +system +let +love +2006 +though +every +2014 +look +song +water +century +without +body +black +night +within +great +women +single +ve +building +large +population +river +named +band +white +started +##an +once +15 +20 +should +18 +2015 +service +top +built +british +open +death +king +moved +local +times +children +february +book +why +11 +door +need +president +order +final +road +wasn +although +due +major +died +village +third +knew +2016 +asked +turned +st +wanted +say +##p +together +received +main +son +served +different +##en +behind +himself +felt +members +power +football +law +voice +play +##in +near +park +history +30 +having +2005 +16 +##man +saw +mother +##al +army +point +front +help +english +street +art +late +hands +games +award +##ia +young +14 +put +published +country +division +across +told +13 +often +ever +french +london +center +six +red +2017 +led +days +include +light +25 +find +tell +among +species +really +according +central +half +2004 +form +original +gave +office +making +enough +lost +full +opened +must +included +live +given +german +player +run +business +woman +community +cup +might +million +land +2000 +court +development +17 +short +round +ii +km +seen +class +story +always +become +sure +research +almost +director +council +la +##2 +career +things +using +island +##z +couldn +car +##is +24 +close +force +##1 +better +free +support +control +field +students +2003 +education +married +##b +nothing +worked +others +record +big +inside +level +anything +continued +give +james +##3 +military +established +non +returned +feel +does +title +written +thing +feet +william +far +co +association +hard +already +2002 +##ra +championship +human +western +100 +##na +department +hall +role +various +production +21 +19 +heart +2001 +living +fire +version +##ers +##f +television +royal +##4 +produced +working +act +case +society +region +present +radio +period +looking +least +total +keep +england +wife +program +per +brother +mind +special +22 +##le +am +works +soon +##6 +political +george +services +taken +created +##7 +further +able +reached +david +union +joined +upon +done +important +social +information +either +##ic +##x +appeared +position +ground +lead +rock +dark +election +23 +board +france +hair +course +arms +site +police +girl +instead +real +sound +##v +words +moment +##te +someone +##8 +summer +project +announced +san +less +wrote +past +followed +##5 +blue +founded +al +finally +india +taking +records +america +##ne +1999 +design +considered +northern +god +stop +battle +toward +european +outside +described +track +today +playing +language +28 +call +26 +heard +professional +low +australia +miles +california +win +yet +green +##ie +trying +blood +##ton +southern +science +maybe +everything +match +square +27 +mouth +video +race +recorded +leave +above +##9 +daughter +points +space +1998 +museum +change +middle +common +##0 +move +tv +post +##ta +lake +seven +tried +elected +closed +ten +paul +minister +##th +months +start +chief +return +canada +person +sea +release +similar +modern +brought +rest +hit +formed +mr +##la +1997 +floor +event +doing +thomas +1996 +robert +care +killed +training +star +week +needed +turn +finished +railway +rather +news +health +sent +example +ran +term +michael +coming +currently +yes +forces +despite +gold +areas +50 +stage +fact +29 +dead +says +popular +2018 +originally +germany +probably +developed +result +pulled +friend +stood +money +running +mi +signed +word +songs +child +eventually +met +tour +average +teams +minutes +festival +current +deep +kind +1995 +decided +usually +eastern +seemed +##ness +episode +bed +added +table +indian +private +charles +route +available +idea +throughout +centre +addition +appointed +style +1994 +books +eight +construction +press +mean +wall +friends +remained +schools +study +##ch +##um +institute +oh +chinese +sometimes +events +possible +1992 +australian +type +brown +forward +talk +process +food +debut +seat +performance +committee +features +character +arts +herself +else +lot +strong +russian +range +hours +peter +arm +##da +morning +dr +sold +##ry +quickly +directed +1993 +guitar +china +##w +31 +list +##ma +performed +media +uk +players +smile +##rs +myself +40 +placed +coach +province +towards +wouldn +leading +whole +boy +official +designed +grand +census +##el +europe +attack +japanese +henry +1991 +##re +##os +cross +getting +alone +action +lower +network +wide +washington +japan +1990 +hospital +believe +changed +sister +##ar +hold +gone +sir +hadn +ship +##ka +studies +academy +shot +rights +below +base +bad +involved +kept +largest +##ist +bank +future +especially +beginning +mark +movement +section +female +magazine +plan +professor +lord +longer +##ian +sat +walked +hill +actually +civil +energy +model +families +size +thus +aircraft +completed +includes +data +captain +##or +fight +vocals +featured +richard +bridge +fourth +1989 +officer +stone +hear +##ism +means +medical +groups +management +self +lips +competition +entire +lived +technology +leaving +federal +tournament +bit +passed +hot +independent +awards +kingdom +mary +spent +fine +doesn +reported +##ling +jack +fall +raised +itself +stay +true +studio +1988 +sports +replaced +paris +systems +saint +leader +theatre +whose +market +capital +parents +spanish +canadian +earth +##ity +cut +degree +writing +bay +christian +awarded +natural +higher +bill +##as +coast +provided +previous +senior +ft +valley +organization +stopped +onto +countries +parts +conference +queen +security +interest +saying +allowed +master +earlier +phone +matter +smith +winning +try +happened +moving +campaign +los +##ley +breath +nearly +mid +1987 +certain +girls +date +italian +african +standing +fell +artist +##ted +shows +deal +mine +industry +1986 +##ng +everyone +republic +provide +collection +library +student +##ville +primary +owned +older +via +heavy +1st +makes +##able +attention +anyone +africa +##ri +stated +length +ended +fingers +command +staff +skin +foreign +opening +governor +okay +medal +kill +sun +cover +job +1985 +introduced +chest +hell +feeling +##ies +success +meet +reason +standard +meeting +novel +1984 +trade +source +buildings +##land +rose +guy +goal +##ur +chapter +native +husband +previously +unit +limited +entered +weeks +producer +operations +mountain +takes +covered +forced +related +roman +complete +successful +key +texas +cold +##ya +channel +1980 +traditional +films +dance +clear +approximately +500 +nine +van +prince +question +active +tracks +ireland +regional +silver +author +personal +sense +operation +##ine +economic +1983 +holding +twenty +isbn +additional +speed +hour +edition +regular +historic +places +whom +shook +movie +km² +secretary +prior +report +chicago +read +foundation +view +engine +scored +1982 +units +ask +airport +property +ready +immediately +lady +month +listed +contract +##de +manager +themselves +lines +##ki +navy +writer +meant +##ts +runs +##ro +practice +championships +singer +glass +commission +required +forest +starting +culture +generally +giving +access +attended +test +couple +stand +catholic +martin +caught +executive +##less +eye +##ey +thinking +chair +quite +shoulder +1979 +hope +decision +plays +defeated +municipality +whether +structure +offered +slowly +pain +ice +direction +##ion +paper +mission +1981 +mostly +200 +noted +individual +managed +nature +lives +plant +##ha +helped +except +studied +computer +figure +relationship +issue +significant +loss +die +smiled +gun +ago +highest +1972 +##am +male +bring +goals +mexico +problem +distance +commercial +completely +location +annual +famous +drive +1976 +neck +1978 +surface +caused +italy +understand +greek +highway +wrong +hotel +comes +appearance +joseph +double +issues +musical +companies +castle +income +review +assembly +bass +initially +parliament +artists +experience +1974 +particular +walk +foot +engineering +talking +window +dropped +##ter +miss +baby +boys +break +1975 +stars +edge +remember +policy +carried +train +stadium +bar +sex +angeles +evidence +##ge +becoming +assistant +soviet +1977 +upper +step +wing +1970 +youth +financial +reach +##ll +actor +numerous +##se +##st +nodded +arrived +##ation +minute +##nt +believed +sorry +complex +beautiful +victory +associated +temple +1968 +1973 +chance +perhaps +metal +##son +1945 +bishop +##et +lee +launched +particularly +tree +le +retired +subject +prize +contains +yeah +theory +empire +##ce +suddenly +waiting +trust +recording +##to +happy +terms +camp +champion +1971 +religious +pass +zealand +names +2nd +port +ancient +tom +corner +represented +watch +legal +anti +justice +cause +watched +brothers +45 +material +changes +simply +response +louis +fast +##ting +answer +60 +historical +1969 +stories +straight +create +feature +increased +rate +administration +virginia +el +activities +cultural +overall +winner +programs +basketball +legs +guard +beyond +cast +doctor +mm +flight +results +remains +cost +effect +winter +##ble +larger +islands +problems +chairman +grew +commander +isn +1967 +pay +failed +selected +hurt +fort +box +regiment +majority +journal +35 +edward +plans +##ke +##ni +shown +pretty +irish +characters +directly +scene +likely +operated +allow +spring +##j +junior +matches +looks +mike +houses +fellow +##tion +beach +marriage +##ham +##ive +rules +oil +65 +florida +expected +nearby +congress +sam +peace +recent +iii +wait +subsequently +cell +##do +variety +serving +agreed +please +poor +joe +pacific +attempt +wood +democratic +piece +prime +##ca +rural +mile +touch +appears +township +1964 +1966 +soldiers +##men +##ized +1965 +pennsylvania +closer +fighting +claimed +score +jones +physical +editor +##ous +filled +genus +specific +sitting +super +mom +##va +therefore +supported +status +fear +cases +store +meaning +wales +minor +spain +tower +focus +vice +frank +follow +parish +separate +golden +horse +fifth +remaining +branch +32 +presented +stared +##id +uses +secret +forms +##co +baseball +exactly +##ck +choice +note +discovered +travel +composed +truth +russia +ball +color +kiss +dad +wind +continue +ring +referred +numbers +digital +greater +##ns +metres +slightly +direct +increase +1960 +responsible +crew +rule +trees +troops +##no +broke +goes +individuals +hundred +weight +creek +sleep +memory +defense +provides +ordered +code +value +jewish +windows +1944 +safe +judge +whatever +corps +realized +growing +pre +##ga +cities +alexander +gaze +lies +spread +scott +letter +showed +situation +mayor +transport +watching +workers +extended +##li +expression +normal +##ment +chart +multiple +border +##ba +host +##ner +daily +mrs +walls +piano +##ko +heat +cannot +##ate +earned +products +drama +era +authority +seasons +join +grade +##io +sign +difficult +machine +1963 +territory +mainly +##wood +stations +squadron +1962 +stepped +iron +19th +##led +serve +appear +sky +speak +broken +charge +knowledge +kilometres +removed +ships +article +campus +simple +##ty +pushed +britain +##ve +leaves +recently +cd +soft +boston +latter +easy +acquired +poland +##sa +quality +officers +presence +planned +nations +mass +broadcast +jean +share +image +influence +wild +offer +emperor +electric +reading +headed +ability +promoted +yellow +ministry +1942 +throat +smaller +politician +##by +latin +spoke +cars +williams +males +lack +pop +80 +##ier +acting +seeing +consists +##ti +estate +1961 +pressure +johnson +newspaper +jr +chris +olympics +online +conditions +beat +elements +walking +vote +##field +needs +carolina +text +featuring +global +block +shirt +levels +francisco +purpose +females +et +dutch +duke +ahead +gas +twice +safety +serious +turning +highly +lieutenant +firm +maria +amount +mixed +daniel +proposed +perfect +agreement +affairs +3rd +seconds +contemporary +paid +1943 +prison +save +kitchen +label +administrative +intended +constructed +academic +nice +teacher +races +1956 +formerly +corporation +ben +nation +issued +shut +1958 +drums +housing +victoria +seems +opera +1959 +graduated +function +von +mentioned +picked +build +recognized +shortly +protection +picture +notable +exchange +elections +1980s +loved +percent +racing +fish +elizabeth +garden +volume +hockey +1941 +beside +settled +##ford +1940 +competed +replied +drew +1948 +actress +marine +scotland +steel +glanced +farm +steve +1957 +risk +tonight +positive +magic +singles +effects +gray +screen +dog +##ja +residents +bus +sides +none +secondary +literature +polish +destroyed +flying +founder +households +1939 +lay +reserve +usa +gallery +##ler +1946 +industrial +younger +approach +appearances +urban +ones +1950 +finish +avenue +powerful +fully +growth +page +honor +jersey +projects +advanced +revealed +basic +90 +infantry +pair +equipment +visit +33 +evening +search +grant +effort +solo +treatment +buried +republican +primarily +bottom +owner +1970s +israel +gives +jim +dream +bob +remain +spot +70 +notes +produce +champions +contact +ed +soul +accepted +ways +del +##ally +losing +split +price +capacity +basis +trial +questions +##ina +1955 +20th +guess +officially +memorial +naval +initial +##ization +whispered +median +engineer +##ful +sydney +##go +columbia +strength +300 +1952 +tears +senate +00 +card +asian +agent +1947 +software +44 +draw +warm +supposed +com +pro +##il +transferred +leaned +##at +candidate +escape +mountains +asia +potential +activity +entertainment +seem +traffic +jackson +murder +36 +slow +product +orchestra +haven +agency +bbc +taught +website +comedy +unable +storm +planning +albums +rugby +environment +scientific +grabbed +protect +##hi +boat +typically +1954 +1953 +damage +principal +divided +dedicated +mount +ohio +##berg +pick +fought +driver +##der +empty +shoulders +sort +thank +berlin +prominent +account +freedom +necessary +efforts +alex +headquarters +follows +alongside +des +simon +andrew +suggested +operating +learning +steps +1949 +sweet +technical +begin +easily +34 +teeth +speaking +settlement +scale +##sh +renamed +ray +max +enemy +semi +joint +compared +##rd +scottish +leadership +analysis +offers +georgia +pieces +captured +animal +deputy +guest +organized +##lin +tony +combined +method +challenge +1960s +huge +wants +battalion +sons +rise +crime +types +facilities +telling +path +1951 +platform +sit +1990s +##lo +tells +assigned +rich +pull +##ot +commonly +alive +##za +letters +concept +conducted +wearing +happen +bought +becomes +holy +gets +ocean +defeat +languages +purchased +coffee +occurred +titled +##q +declared +applied +sciences +concert +sounds +jazz +brain +##me +painting +fleet +tax +nick +##ius +michigan +count +animals +leaders +episodes +##line +content +##den +birth +##it +clubs +64 +palace +critical +refused +fair +leg +laughed +returning +surrounding +participated +formation +lifted +pointed +connected +rome +medicine +laid +taylor +santa +powers +adam +tall +shared +focused +knowing +yards +entrance +falls +##wa +calling +##ad +sources +chosen +beneath +resources +yard +##ite +nominated +silence +zone +defined +##que +gained +thirty +38 +bodies +moon +##ard +adopted +christmas +widely +register +apart +iran +premier +serves +du +unknown +parties +##les +generation +##ff +continues +quick +fields +brigade +quiet +teaching +clothes +impact +weapons +partner +flat +theater +supreme +1938 +37 +relations +##tor +plants +suffered +1936 +wilson +kids +begins +##age +1918 +seats +armed +internet +models +worth +laws +400 +communities +classes +background +knows +thanks +quarter +reaching +humans +carry +killing +format +kong +hong +setting +75 +architecture +disease +railroad +inc +possibly +wish +arthur +thoughts +harry +doors +density +##di +crowd +illinois +stomach +tone +unique +reports +anyway +##ir +liberal +der +vehicle +thick +dry +drug +faced +largely +facility +theme +holds +creation +strange +colonel +##mi +revolution +bell +politics +turns +silent +rail +relief +independence +combat +shape +write +determined +sales +learned +4th +finger +oxford +providing +1937 +heritage +fiction +situated +designated +allowing +distribution +hosted +##est +sight +interview +estimated +reduced +##ria +toronto +footballer +keeping +guys +damn +claim +motion +sport +sixth +stayed +##ze +en +rear +receive +handed +twelve +dress +audience +granted +brazil +##well +spirit +##ated +noticed +etc +olympic +representative +eric +tight +trouble +reviews +drink +vampire +missing +roles +ranked +newly +household +finals +wave +critics +##ee +phase +massachusetts +pilot +unlike +philadelphia +bright +guns +crown +organizations +roof +42 +respectively +clearly +tongue +marked +circle +fox +korea +bronze +brian +expanded +sexual +supply +yourself +inspired +labour +fc +##ah +reference +vision +draft +connection +brand +reasons +1935 +classic +driving +trip +jesus +cells +entry +1920 +neither +trail +claims +atlantic +orders +labor +nose +afraid +identified +intelligence +calls +cancer +attacked +passing +stephen +positions +imperial +grey +jason +39 +sunday +48 +swedish +avoid +extra +uncle +message +covers +allows +surprise +materials +fame +hunter +##ji +1930 +citizens +figures +davis +environmental +confirmed +shit +titles +di +performing +difference +acts +attacks +##ov +existing +votes +opportunity +nor +shop +entirely +trains +opposite +pakistan +##pa +develop +resulted +representatives +actions +reality +pressed +##ish +barely +wine +conversation +faculty +northwest +ends +documentary +nuclear +stock +grace +sets +eat +alternative +##ps +bag +resulting +creating +surprised +cemetery +1919 +drop +finding +sarah +cricket +streets +tradition +ride +1933 +exhibition +target +ear +explained +rain +composer +injury +apartment +municipal +educational +occupied +netherlands +clean +billion +constitution +learn +1914 +maximum +classical +francis +lose +opposition +jose +ontario +bear +core +hills +rolled +ending +drawn +permanent +fun +##tes +##lla +lewis +sites +chamber +ryan +##way +scoring +height +1934 +##house +lyrics +staring +55 +officials +1917 +snow +oldest +##tic +orange +##ger +qualified +interior +apparently +succeeded +thousand +dinner +lights +existence +fans +heavily +41 +greatest +conservative +send +bowl +plus +enter +catch +##un +economy +duty +1929 +speech +authorities +princess +performances +versions +shall +graduate +pictures +effective +remembered +poetry +desk +crossed +starring +starts +passenger +sharp +##ant +acres +ass +weather +falling +rank +fund +supporting +check +adult +publishing +heads +cm +southeast +lane +##burg +application +bc +##ura +les +condition +transfer +prevent +display +ex +regions +earl +federation +cool +relatively +answered +besides +1928 +obtained +portion +##town +mix +##ding +reaction +liked +dean +express +peak +1932 +##tte +counter +religion +chain +rare +miller +convention +aid +lie +vehicles +mobile +perform +squad +wonder +lying +crazy +sword +##ping +attempted +centuries +weren +philosophy +category +##ize +anna +interested +47 +sweden +wolf +frequently +abandoned +kg +literary +alliance +task +entitled +##ay +threw +promotion +factory +tiny +soccer +visited +matt +fm +achieved +52 +defence +internal +persian +43 +methods +##ging +arrested +otherwise +cambridge +programming +villages +elementary +districts +rooms +criminal +conflict +worry +trained +1931 +attempts +waited +signal +bird +truck +subsequent +programme +##ol +ad +49 +communist +details +faith +sector +patrick +carrying +laugh +##ss +controlled +korean +showing +origin +fuel +evil +1927 +##ent +brief +identity +darkness +address +pool +missed +publication +web +planet +ian +anne +wings +invited +##tt +briefly +standards +kissed +##be +ideas +climate +causing +walter +worse +albert +articles +winners +desire +aged +northeast +dangerous +gate +doubt +1922 +wooden +multi +##ky +poet +rising +funding +46 +communications +communication +violence +copies +prepared +ford +investigation +skills +1924 +pulling +electronic +##ak +##ial +##han +containing +ultimately +offices +singing +understanding +restaurant +tomorrow +fashion +christ +ward +da +pope +stands +5th +flow +studios +aired +commissioned +contained +exist +fresh +americans +##per +wrestling +approved +kid +employed +respect +suit +1925 +angel +asking +increasing +frame +angry +selling +1950s +thin +finds +##nd +temperature +statement +ali +explain +inhabitants +towns +extensive +narrow +51 +jane +flowers +images +promise +somewhere +object +fly +closely +##ls +1912 +bureau +cape +1926 +weekly +presidential +legislative +1921 +##ai +##au +launch +founding +##ny +978 +##ring +artillery +strike +un +institutions +roll +writers +landing +chose +kevin +anymore +pp +##ut +attorney +fit +dan +billboard +receiving +agricultural +breaking +sought +dave +admitted +lands +mexican +##bury +charlie +specifically +hole +iv +howard +credit +moscow +roads +accident +1923 +proved +wear +struck +hey +guards +stuff +slid +expansion +1915 +cat +anthony +##kin +melbourne +opposed +sub +southwest +architect +failure +plane +1916 +##ron +map +camera +tank +listen +regarding +wet +introduction +metropolitan +link +ep +fighter +inch +grown +gene +anger +fixed +buy +dvd +khan +domestic +worldwide +chapel +mill +functions +examples +##head +developing +1910 +turkey +hits +pocket +antonio +papers +grow +unless +circuit +18th +concerned +attached +journalist +selection +journey +converted +provincial +painted +hearing +aren +bands +negative +aside +wondered +knight +lap +survey +ma +##ow +noise +billy +##ium +shooting +guide +bedroom +priest +resistance +motor +homes +sounded +giant +##mer +150 +scenes +equal +comic +patients +hidden +solid +actual +bringing +afternoon +touched +funds +wedding +consisted +marie +canal +sr +kim +treaty +turkish +recognition +residence +cathedral +broad +knees +incident +shaped +fired +norwegian +handle +cheek +contest +represent +##pe +representing +beauty +##sen +birds +advantage +emergency +wrapped +drawing +notice +pink +broadcasting +##ong +somehow +bachelor +seventh +collected +registered +establishment +alan +assumed +chemical +personnel +roger +retirement +jeff +portuguese +wore +tied +device +threat +progress +advance +##ised +banks +hired +manchester +nfl +teachers +structures +forever +##bo +tennis +helping +saturday +sale +applications +junction +hip +incorporated +neighborhood +dressed +ceremony +##ds +influenced +hers +visual +stairs +decades +inner +kansas +hung +hoped +gain +scheduled +downtown +engaged +austria +clock +norway +certainly +pale +protected +1913 +victor +employees +plate +putting +surrounded +##ists +finishing +blues +tropical +##ries +minnesota +consider +philippines +accept +54 +retrieved +1900 +concern +anderson +properties +institution +gordon +successfully +vietnam +##dy +backing +outstanding +muslim +crossing +folk +producing +usual +demand +occurs +observed +lawyer +educated +##ana +kelly +string +pleasure +budget +items +quietly +colorado +philip +typical +##worth +derived +600 +survived +asks +mental +##ide +56 +jake +jews +distinguished +ltd +1911 +sri +extremely +53 +athletic +loud +thousands +worried +shadow +transportation +horses +weapon +arena +importance +users +tim +objects +contributed +dragon +douglas +aware +senator +johnny +jordan +sisters +engines +flag +investment +samuel +shock +capable +clark +row +wheel +refers +session +familiar +biggest +wins +hate +maintained +drove +hamilton +request +expressed +injured +underground +churches +walker +wars +tunnel +passes +stupid +agriculture +softly +cabinet +regarded +joining +indiana +##ea +##ms +push +dates +spend +behavior +woods +protein +gently +chase +morgan +mention +burning +wake +combination +occur +mirror +leads +jimmy +indeed +impossible +singapore +paintings +covering +##nes +soldier +locations +attendance +sell +historian +wisconsin +invasion +argued +painter +diego +changing +egypt +##don +experienced +inches +##ku +missouri +vol +grounds +spoken +switzerland +##gan +reform +rolling +ha +forget +massive +resigned +burned +allen +tennessee +locked +values +improved +##mo +wounded +universe +sick +dating +facing +pack +purchase +user +##pur +moments +##ul +merged +anniversary +1908 +coal +brick +understood +causes +dynasty +queensland +establish +stores +crisis +promote +hoping +views +cards +referee +extension +##si +raise +arizona +improve +colonial +formal +charged +##rt +palm +lucky +hide +rescue +faces +95 +feelings +candidates +juan +##ell +goods +6th +courses +weekend +59 +luke +cash +fallen +##om +delivered +affected +installed +carefully +tries +swiss +hollywood +costs +lincoln +responsibility +##he +shore +file +proper +normally +maryland +assistance +jump +constant +offering +friendly +waters +persons +realize +contain +trophy +800 +partnership +factor +58 +musicians +cry +bound +oregon +indicated +hero +houston +medium +##ure +consisting +somewhat +##ara +57 +cycle +##che +beer +moore +frederick +gotten +eleven +worst +weak +approached +arranged +chin +loan +universal +bond +fifteen +pattern +disappeared +##ney +translated +##zed +lip +arab +capture +interests +insurance +##chi +shifted +cave +prix +warning +sections +courts +coat +plot +smell +feed +golf +favorite +maintain +knife +vs +voted +degrees +finance +quebec +opinion +translation +manner +ruled +operate +productions +choose +musician +discovery +confused +tired +separated +stream +techniques +committed +attend +ranking +kings +throw +passengers +measure +horror +fan +mining +sand +danger +salt +calm +decade +dam +require +runner +##ik +rush +associate +greece +##ker +rivers +consecutive +matthew +##ski +sighed +sq +documents +steam +edited +closing +tie +accused +1905 +##ini +islamic +distributed +directors +organisation +bruce +7th +breathing +mad +lit +arrival +concrete +taste +08 +composition +shaking +faster +amateur +adjacent +stating +1906 +twin +flew +##ran +tokyo +publications +##tone +obviously +ridge +storage +1907 +carl +pages +concluded +desert +driven +universities +ages +terminal +sequence +borough +250 +constituency +creative +cousin +economics +dreams +margaret +notably +reduce +montreal +mode +17th +ears +saved +jan +vocal +##ica +1909 +andy +##jo +riding +roughly +threatened +##ise +meters +meanwhile +landed +compete +repeated +grass +czech +regularly +charges +tea +sudden +appeal +##ung +solution +describes +pierre +classification +glad +parking +##ning +belt +physics +99 +rachel +add +hungarian +participate +expedition +damaged +gift +childhood +85 +fifty +##red +mathematics +jumped +letting +defensive +mph +##ux +##gh +testing +##hip +hundreds +shoot +owners +matters +smoke +israeli +kentucky +dancing +mounted +grandfather +emma +designs +profit +argentina +##gs +truly +li +lawrence +cole +begun +detroit +willing +branches +smiling +decide +miami +enjoyed +recordings +##dale +poverty +ethnic +gay +##bi +gary +arabic +09 +accompanied +##one +##ons +fishing +determine +residential +acid +##ary +alice +returns +starred +mail +##ang +jonathan +strategy +##ue +net +forty +cook +businesses +equivalent +commonwealth +distinct +ill +##cy +seriously +##ors +##ped +shift +harris +replace +rio +imagine +formula +ensure +##ber +additionally +scheme +conservation +occasionally +purposes +feels +favor +##and +##ore +1930s +contrast +hanging +hunt +movies +1904 +instruments +victims +danish +christopher +busy +demon +sugar +earliest +colony +studying +balance +duties +##ks +belgium +slipped +carter +05 +visible +stages +iraq +fifa +##im +commune +forming +zero +07 +continuing +talked +counties +legend +bathroom +option +tail +clay +daughters +afterwards +severe +jaw +visitors +##ded +devices +aviation +russell +kate +##vi +entering +subjects +##ino +temporary +swimming +forth +smooth +ghost +audio +bush +operates +rocks +movements +signs +eddie +##tz +ann +voices +honorary +06 +memories +dallas +pure +measures +racial +promised +66 +harvard +ceo +16th +parliamentary +indicate +benefit +flesh +dublin +louisiana +1902 +1901 +patient +sleeping +1903 +membership +coastal +medieval +wanting +element +scholars +rice +62 +limit +survive +makeup +rating +definitely +collaboration +obvious +##tan +boss +ms +baron +birthday +linked +soil +diocese +##lan +ncaa +##mann +offensive +shell +shouldn +waist +##tus +plain +ross +organ +resolution +manufacturing +adding +relative +kennedy +98 +whilst +moth +marketing +gardens +crash +72 +heading +partners +credited +carlos +moves +cable +##zi +marshall +##out +depending +bottle +represents +rejected +responded +existed +04 +jobs +denmark +lock +##ating +treated +graham +routes +talent +commissioner +drugs +secure +tests +reign +restored +photography +##gi +contributions +oklahoma +designer +disc +grin +seattle +robin +paused +atlanta +unusual +##gate +praised +las +laughing +satellite +hungary +visiting +##sky +interesting +factors +deck +poems +norman +##water +stuck +speaker +rifle +domain +premiered +##her +dc +comics +actors +01 +reputation +eliminated +8th +ceiling +prisoners +script +##nce +leather +austin +mississippi +rapidly +admiral +parallel +charlotte +guilty +tools +gender +divisions +fruit +##bs +laboratory +nelson +fantasy +marry +rapid +aunt +tribe +requirements +aspects +suicide +amongst +adams +bone +ukraine +abc +kick +sees +edinburgh +clothing +column +rough +gods +hunting +broadway +gathered +concerns +##ek +spending +ty +12th +snapped +requires +solar +bones +cavalry +##tta +iowa +drinking +waste +index +franklin +charity +thompson +stewart +tip +flash +landscape +friday +enjoy +singh +poem +listening +##back +eighth +fred +differences +adapted +bomb +ukrainian +surgery +corporate +masters +anywhere +##more +waves +odd +sean +portugal +orleans +dick +debate +kent +eating +puerto +cleared +96 +expect +cinema +97 +guitarist +blocks +electrical +agree +involving +depth +dying +panel +struggle +##ged +peninsula +adults +novels +emerged +vienna +metro +debuted +shoes +tamil +songwriter +meets +prove +beating +instance +heaven +scared +sending +marks +artistic +passage +superior +03 +significantly +shopping +##tive +retained +##izing +malaysia +technique +cheeks +##ola +warren +maintenance +destroy +extreme +allied +120 +appearing +##yn +fill +advice +alabama +qualifying +policies +cleveland +hat +battery +smart +authors +10th +soundtrack +acted +dated +lb +glance +equipped +coalition +funny +outer +ambassador +roy +possibility +couples +campbell +dna +loose +ethan +supplies +1898 +gonna +88 +monster +##res +shake +agents +frequency +springs +dogs +practices +61 +gang +plastic +easier +suggests +gulf +blade +exposed +colors +industries +markets +pan +nervous +electoral +charts +legislation +ownership +##idae +mac +appointment +shield +copy +assault +socialist +abbey +monument +license +throne +employment +jay +93 +replacement +charter +cloud +powered +suffering +accounts +oak +connecticut +strongly +wright +colour +crystal +13th +context +welsh +networks +voiced +gabriel +jerry +##cing +forehead +mp +##ens +manage +schedule +totally +remix +##ii +forests +occupation +print +nicholas +brazilian +strategic +vampires +engineers +76 +roots +seek +correct +instrumental +und +alfred +backed +hop +##des +stanley +robinson +traveled +wayne +welcome +austrian +achieve +67 +exit +rates +1899 +strip +whereas +##cs +sing +deeply +adventure +bobby +rick +jamie +careful +components +cap +useful +personality +knee +##shi +pushing +hosts +02 +protest +ca +ottoman +symphony +##sis +63 +boundary +1890 +processes +considering +considerable +tons +##work +##ft +##nia +cooper +trading +dear +conduct +91 +illegal +apple +revolutionary +holiday +definition +harder +##van +jacob +circumstances +destruction +##lle +popularity +grip +classified +liverpool +donald +baltimore +flows +seeking +honour +approval +92 +mechanical +till +happening +statue +critic +increasingly +immediate +describe +commerce +stare +##ster +indonesia +meat +rounds +boats +baker +orthodox +depression +formally +worn +naked +claire +muttered +sentence +11th +emily +document +77 +criticism +wished +vessel +spiritual +bent +virgin +parker +minimum +murray +lunch +danny +printed +compilation +keyboards +false +blow +belonged +68 +raising +78 +cutting +##board +pittsburgh +##up +9th +shadows +81 +hated +indigenous +jon +15th +barry +scholar +ah +##zer +oliver +##gy +stick +susan +meetings +attracted +spell +romantic +##ver +ye +1895 +photo +demanded +customers +##ac +1896 +logan +revival +keys +modified +commanded +jeans +##ious +upset +raw +phil +detective +hiding +resident +vincent +##bly +experiences +diamond +defeating +coverage +lucas +external +parks +franchise +helen +bible +successor +percussion +celebrated +il +lift +profile +clan +romania +##ied +mills +##su +nobody +achievement +shrugged +fault +1897 +rhythm +initiative +breakfast +carbon +700 +69 +lasted +violent +74 +wound +ken +killer +gradually +filmed +°c +dollars +processing +94 +remove +criticized +guests +sang +chemistry +##vin +legislature +disney +##bridge +uniform +escaped +integrated +proposal +purple +denied +liquid +karl +influential +morris +nights +stones +intense +experimental +twisted +71 +84 +##ld +pace +nazi +mitchell +ny +blind +reporter +newspapers +14th +centers +burn +basin +forgotten +surviving +filed +collections +monastery +losses +manual +couch +description +appropriate +merely +tag +missions +sebastian +restoration +replacing +triple +73 +elder +julia +warriors +benjamin +julian +convinced +stronger +amazing +declined +versus +merchant +happens +output +finland +bare +barbara +absence +ignored +dawn +injuries +##port +producers +##ram +82 +luis +##ities +kw +admit +expensive +electricity +nba +exception +symbol +##ving +ladies +shower +sheriff +characteristics +##je +aimed +button +ratio +effectively +summit +angle +jury +bears +foster +vessels +pants +executed +evans +dozen +advertising +kicked +patrol +1889 +competitions +lifetime +principles +athletics +##logy +birmingham +sponsored +89 +rob +nomination +1893 +acoustic +##sm +creature +longest +##tra +credits +harbor +dust +josh +##so +territories +milk +infrastructure +completion +thailand +indians +leon +archbishop +##sy +assist +pitch +blake +arrangement +girlfriend +serbian +operational +hence +sad +scent +fur +dj +sessions +hp +refer +rarely +##ora +exists +1892 +##ten +scientists +dirty +penalty +burst +portrait +seed +79 +pole +limits +rival +1894 +stable +alpha +grave +constitutional +alcohol +arrest +flower +mystery +devil +architectural +relationships +greatly +habitat +##istic +larry +progressive +remote +cotton +##ics +##ok +preserved +reaches +##ming +cited +86 +vast +scholarship +decisions +cbs +joy +teach +1885 +editions +knocked +eve +searching +partly +participation +gap +animated +fate +excellent +##ett +na +87 +alternate +saints +youngest +##ily +climbed +##ita +##tors +suggest +##ct +discussion +staying +choir +lakes +jacket +revenue +nevertheless +peaked +instrument +wondering +annually +managing +neil +1891 +signing +terry +##ice +apply +clinical +brooklyn +aim +catherine +fuck +farmers +figured +ninth +pride +hugh +evolution +ordinary +involvement +comfortable +shouted +tech +encouraged +taiwan +representation +sharing +##lia +##em +panic +exact +cargo +competing +fat +cried +83 +1920s +occasions +pa +cabin +borders +utah +marcus +##isation +badly +muscles +##ance +victorian +transition +warner +bet +permission +##rin +slave +terrible +similarly +shares +seth +uefa +possession +medals +benefits +colleges +lowered +perfectly +mall +transit +##ye +##kar +publisher +##ened +harrison +deaths +elevation +##ae +asleep +machines +sigh +ash +hardly +argument +occasion +parent +leo +decline +1888 +contribution +##ua +concentration +1000 +opportunities +hispanic +guardian +extent +emotions +hips +mason +volumes +bloody +controversy +diameter +steady +mistake +phoenix +identify +violin +##sk +departure +richmond +spin +funeral +enemies +1864 +gear +literally +connor +random +sergeant +grab +confusion +1865 +transmission +informed +op +leaning +sacred +suspended +thinks +gates +portland +luck +agencies +yours +hull +expert +muscle +layer +practical +sculpture +jerusalem +latest +lloyd +statistics +deeper +recommended +warrior +arkansas +mess +supports +greg +eagle +1880 +recovered +rated +concerts +rushed +##ano +stops +eggs +files +premiere +keith +##vo +delhi +turner +pit +affair +belief +paint +##zing +mate +##ach +##ev +victim +##ology +withdrew +bonus +styles +fled +##ud +glasgow +technologies +funded +nbc +adaptation +##ata +portrayed +cooperation +supporters +judges +bernard +justin +hallway +ralph +##ick +graduating +controversial +distant +continental +spider +bite +##ho +recognize +intention +mixing +##ese +egyptian +bow +tourism +suppose +claiming +tiger +dominated +participants +vi +##ru +nurse +partially +tape +##rum +psychology +##rn +essential +touring +duo +voting +civilian +emotional +channels +##king +apparent +hebrew +1887 +tommy +carrier +intersection +beast +hudson +##gar +##zo +lab +nova +bench +discuss +costa +##ered +detailed +behalf +drivers +unfortunately +obtain +##lis +rocky +##dae +siege +friendship +honey +##rian +1861 +amy +hang +posted +governments +collins +respond +wildlife +preferred +operator +##po +laura +pregnant +videos +dennis +suspected +boots +instantly +weird +automatic +businessman +alleged +placing +throwing +ph +mood +1862 +perry +venue +jet +remainder +##lli +##ci +passion +biological +boyfriend +1863 +dirt +buffalo +ron +segment +fa +abuse +##era +genre +thrown +stroke +colored +stress +exercise +displayed +##gen +struggled +##tti +abroad +dramatic +wonderful +thereafter +madrid +component +widespread +##sed +tale +citizen +todd +monday +1886 +vancouver +overseas +forcing +crying +descent +##ris +discussed +substantial +ranks +regime +1870 +provinces +switch +drum +zane +ted +tribes +proof +lp +cream +researchers +volunteer +manor +silk +milan +donated +allies +venture +principle +delivery +enterprise +##ves +##ans +bars +traditionally +witch +reminded +copper +##uk +pete +inter +links +colin +grinned +elsewhere +competitive +frequent +##oy +scream +##hu +tension +texts +submarine +finnish +defending +defend +pat +detail +1884 +affiliated +stuart +themes +villa +periods +tool +belgian +ruling +crimes +answers +folded +licensed +resort +demolished +hans +lucy +1881 +lion +traded +photographs +writes +craig +##fa +trials +generated +beth +noble +debt +percentage +yorkshire +erected +ss +viewed +grades +confidence +ceased +islam +telephone +retail +##ible +chile +m² +roberts +sixteen +##ich +commented +hampshire +innocent +dual +pounds +checked +regulations +afghanistan +sung +rico +liberty +assets +bigger +options +angels +relegated +tribute +wells +attending +leaf +##yan +butler +romanian +forum +monthly +lisa +patterns +gmina +##tory +madison +hurricane +rev +##ians +bristol +##ula +elite +valuable +disaster +democracy +awareness +germans +freyja +##ins +loop +absolutely +paying +populations +maine +sole +prayer +spencer +releases +doorway +bull +##ani +lover +midnight +conclusion +##sson +thirteen +lily +mediterranean +##lt +nhl +proud +sample +##hill +drummer +guinea +##ova +murphy +climb +##ston +instant +attributed +horn +ain +railways +steven +##ao +autumn +ferry +opponent +root +traveling +secured +corridor +stretched +tales +sheet +trinity +cattle +helps +indicates +manhattan +murdered +fitted +1882 +gentle +grandmother +mines +shocked +vegas +produces +##light +caribbean +##ou +belong +continuous +desperate +drunk +historically +trio +waved +raf +dealing +nathan +bat +murmured +interrupted +residing +scientist +pioneer +harold +aaron +##net +delta +attempting +minority +mini +believes +chorus +tend +lots +eyed +indoor +load +shots +updated +jail +##llo +concerning +connecting +wealth +##ved +slaves +arrive +rangers +sufficient +rebuilt +##wick +cardinal +flood +muhammad +whenever +relation +runners +moral +repair +viewers +arriving +revenge +punk +assisted +bath +fairly +breathe +lists +innings +illustrated +whisper +nearest +voters +clinton +ties +ultimate +screamed +beijing +lions +andre +fictional +gathering +comfort +radar +suitable +dismissed +hms +ban +pine +wrist +atmosphere +voivodeship +bid +timber +##ned +##nan +giants +##ane +cameron +recovery +uss +identical +categories +switched +serbia +laughter +noah +ensemble +therapy +peoples +touching +##off +locally +pearl +platforms +everywhere +ballet +tables +lanka +herbert +outdoor +toured +derek +1883 +spaces +contested +swept +1878 +exclusive +slight +connections +##dra +winds +prisoner +collective +bangladesh +tube +publicly +wealthy +thai +##ys +isolated +select +##ric +insisted +pen +fortune +ticket +spotted +reportedly +animation +enforcement +tanks +110 +decides +wider +lowest +owen +##time +nod +hitting +##hn +gregory +furthermore +magazines +fighters +solutions +##ery +pointing +requested +peru +reed +chancellor +knights +mask +worker +eldest +flames +reduction +1860 +volunteers +##tis +reporting +##hl +wire +advisory +endemic +origins +settlers +pursue +knock +consumer +1876 +eu +compound +creatures +mansion +sentenced +ivan +deployed +guitars +frowned +involves +mechanism +kilometers +perspective +shops +maps +terminus +duncan +alien +fist +bridges +##pers +heroes +fed +derby +swallowed +##ros +patent +sara +illness +characterized +adventures +slide +hawaii +jurisdiction +##op +organised +##side +adelaide +walks +biology +se +##ties +rogers +swing +tightly +boundaries +##rie +prepare +implementation +stolen +##sha +certified +colombia +edwards +garage +##mm +recalled +##ball +rage +harm +nigeria +breast +##ren +furniture +pupils +settle +##lus +cuba +balls +client +alaska +21st +linear +thrust +celebration +latino +genetic +terror +##cia +##ening +lightning +fee +witness +lodge +establishing +skull +##ique +earning +hood +##ei +rebellion +wang +sporting +warned +missile +devoted +activist +porch +worship +fourteen +package +1871 +decorated +##shire +housed +##ock +chess +sailed +doctors +oscar +joan +treat +garcia +harbour +jeremy +##ire +traditions +dominant +jacques +##gon +##wan +relocated +1879 +amendment +sized +companion +simultaneously +volleyball +spun +acre +increases +stopping +loves +belongs +affect +drafted +tossed +scout +battles +1875 +filming +shoved +munich +tenure +vertical +romance +pc +##cher +argue +##ical +craft +ranging +www +opens +honest +tyler +yesterday +virtual +##let +muslims +reveal +snake +immigrants +radical +screaming +speakers +firing +saving +belonging +ease +lighting +prefecture +blame +farmer +hungry +grows +rubbed +beam +sur +subsidiary +##cha +armenian +sao +dropping +conventional +##fer +microsoft +reply +qualify +spots +1867 +sweat +festivals +##ken +immigration +physician +discover +exposure +sandy +explanation +isaac +implemented +##fish +hart +initiated +connect +stakes +presents +heights +householder +pleased +tourist +regardless +slip +closest +##ction +surely +sultan +brings +riley +preparation +aboard +slammed +baptist +experiment +ongoing +interstate +organic +playoffs +##ika +1877 +130 +##tar +hindu +error +tours +tier +plenty +arrangements +talks +trapped +excited +sank +ho +athens +1872 +denver +welfare +suburb +athletes +trick +diverse +belly +exclusively +yelled +1868 +##med +conversion +##ette +1874 +internationally +computers +conductor +abilities +sensitive +hello +dispute +measured +globe +rocket +prices +amsterdam +flights +tigers +inn +municipalities +emotion +references +3d +##mus +explains +airlines +manufactured +pm +archaeological +1873 +interpretation +devon +comment +##ites +settlements +kissing +absolute +improvement +suite +impressed +barcelona +sullivan +jefferson +towers +jesse +julie +##tin +##lu +grandson +hi +gauge +regard +rings +interviews +trace +raymond +thumb +departments +burns +serial +bulgarian +scores +demonstrated +##ix +1866 +kyle +alberta +underneath +romanized +##ward +relieved +acquisition +phrase +cliff +reveals +han +cuts +merger +custom +##dar +nee +gilbert +graduation +##nts +assessment +cafe +difficulty +demands +swung +democrat +jennifer +commons +1940s +grove +##yo +completing +focuses +sum +substitute +bearing +stretch +reception +##py +reflected +essentially +destination +pairs +##ched +survival +resource +##bach +promoting +doubles +messages +tear +##down +##fully +parade +florence +harvey +incumbent +partial +framework +900 +pedro +frozen +procedure +olivia +controls +##mic +shelter +personally +temperatures +##od +brisbane +tested +sits +marble +comprehensive +oxygen +leonard +##kov +inaugural +iranian +referring +quarters +attitude +##ivity +mainstream +lined +mars +dakota +norfolk +unsuccessful +##° +explosion +helicopter +congressional +##sing +inspector +bitch +seal +departed +divine +##ters +coaching +examination +punishment +manufacturer +sink +columns +unincorporated +signals +nevada +squeezed +dylan +dining +photos +martial +manuel +eighteen +elevator +brushed +plates +ministers +ivy +congregation +##len +slept +specialized +taxes +curve +restricted +negotiations +likes +statistical +arnold +inspiration +execution +bold +intermediate +significance +margin +ruler +wheels +gothic +intellectual +dependent +listened +eligible +buses +widow +syria +earn +cincinnati +collapsed +recipient +secrets +accessible +philippine +maritime +goddess +clerk +surrender +breaks +playoff +database +##ified +##lon +ideal +beetle +aspect +soap +regulation +strings +expand +anglo +shorter +crosses +retreat +tough +coins +wallace +directions +pressing +##oon +shipping +locomotives +comparison +topics +nephew +##mes +distinction +honors +travelled +sierra +ibn +##over +fortress +sa +recognised +carved +1869 +clients +##dan +intent +##mar +coaches +describing +bread +##ington +beaten +northwestern +##ona +merit +youtube +collapse +challenges +em +historians +objective +submitted +virus +attacking +drake +assume +##ere +diseases +marc +stem +leeds +##cus +##ab +farming +glasses +##lock +visits +nowhere +fellowship +relevant +carries +restaurants +experiments +101 +constantly +bases +targets +shah +tenth +opponents +verse +territorial +##ira +writings +corruption +##hs +instruction +inherited +reverse +emphasis +##vic +employee +arch +keeps +rabbi +watson +payment +uh +##ala +nancy +##tre +venice +fastest +sexy +banned +adrian +properly +ruth +touchdown +dollar +boards +metre +circles +edges +favour +comments +ok +travels +liberation +scattered +firmly +##ular +holland +permitted +diesel +kenya +den +originated +##ral +demons +resumed +dragged +rider +##rus +servant +blinked +extend +torn +##ias +##sey +input +meal +everybody +cylinder +kinds +camps +##fe +bullet +logic +##wn +croatian +evolved +healthy +fool +chocolate +wise +preserve +pradesh +##ess +respective +1850 +##ew +chicken +artificial +gross +corresponding +convicted +cage +caroline +dialogue +##dor +narrative +stranger +mario +br +christianity +failing +trent +commanding +buddhist +1848 +maurice +focusing +yale +bike +altitude +##ering +mouse +revised +##sley +veteran +##ig +pulls +theology +crashed +campaigns +legion +##ability +drag +excellence +customer +cancelled +intensity +excuse +##lar +liga +participating +contributing +printing +##burn +variable +##rk +curious +bin +legacy +renaissance +##my +symptoms +binding +vocalist +dancer +##nie +grammar +gospel +democrats +ya +enters +sc +diplomatic +hitler +##ser +clouds +mathematical +quit +defended +oriented +##heim +fundamental +hardware +impressive +equally +convince +confederate +guilt +chuck +sliding +##ware +magnetic +narrowed +petersburg +bulgaria +otto +phd +skill +##ama +reader +hopes +pitcher +reservoir +hearts +automatically +expecting +mysterious +bennett +extensively +imagined +seeds +monitor +fix +##ative +journalism +struggling +signature +ranch +encounter +photographer +observation +protests +##pin +influences +##hr +calendar +##all +cruz +croatia +locomotive +hughes +naturally +shakespeare +basement +hook +uncredited +faded +theories +approaches +dare +phillips +filling +fury +obama +##ain +efficient +arc +deliver +min +raid +breeding +inducted +leagues +efficiency +axis +montana +eagles +##ked +supplied +instructions +karen +picking +indicating +trap +anchor +practically +christians +tomb +vary +occasional +electronics +lords +readers +newcastle +faint +innovation +collect +situations +engagement +160 +claude +mixture +##feld +peer +tissue +logo +lean +##ration +°f +floors +##ven +architects +reducing +##our +##ments +rope +1859 +ottawa +##har +samples +banking +declaration +proteins +resignation +francois +saudi +advocate +exhibited +armor +twins +divorce +##ras +abraham +reviewed +jo +temporarily +matrix +physically +pulse +curled +##ena +difficulties +bengal +usage +##ban +annie +riders +certificate +##pi +holes +warsaw +distinctive +jessica +##mon +mutual +1857 +customs +circular +eugene +removal +loaded +mere +vulnerable +depicted +generations +dame +heir +enormous +lightly +climbing +pitched +lessons +pilots +nepal +ram +google +preparing +brad +louise +renowned +##₂ +liam +##ably +plaza +shaw +sophie +brilliant +bills +##bar +##nik +fucking +mainland +server +pleasant +seized +veterans +jerked +fail +beta +brush +radiation +stored +warmth +southeastern +nate +sin +raced +berkeley +joke +athlete +designation +trunk +##low +roland +qualification +archives +heels +artwork +receives +judicial +reserves +##bed +woke +installation +abu +floating +fake +lesser +excitement +interface +concentrated +addressed +characteristic +amanda +saxophone +monk +auto +##bus +releasing +egg +dies +interaction +defender +ce +outbreak +glory +loving +##bert +sequel +consciousness +http +awake +ski +enrolled +##ress +handling +rookie +brow +somebody +biography +warfare +amounts +contracts +presentation +fabric +dissolved +challenged +meter +psychological +lt +elevated +rally +accurate +##tha +hospitals +undergraduate +specialist +venezuela +exhibit +shed +nursing +protestant +fluid +structural +footage +jared +consistent +prey +##ska +succession +reflect +exile +lebanon +wiped +suspect +shanghai +resting +integration +preservation +marvel +variant +pirates +sheep +rounded +capita +sailing +colonies +manuscript +deemed +variations +clarke +functional +emerging +boxing +relaxed +curse +azerbaijan +heavyweight +nickname +editorial +rang +grid +tightened +earthquake +flashed +miguel +rushing +##ches +improvements +boxes +brooks +180 +consumption +molecular +felix +societies +repeatedly +variation +aids +civic +graphics +professionals +realm +autonomous +receiver +delayed +workshop +militia +chairs +trump +canyon +##point +harsh +extending +lovely +happiness +##jan +stake +eyebrows +embassy +wellington +hannah +##ella +sony +corners +bishops +swear +cloth +contents +xi +namely +commenced +1854 +stanford +nashville +courage +graphic +commitment +garrison +##bin +hamlet +clearing +rebels +attraction +literacy +cooking +ruins +temples +jenny +humanity +celebrate +hasn +freight +sixty +rebel +bastard +##art +newton +##ada +deer +##ges +##ching +smiles +delaware +singers +##ets +approaching +assists +flame +##ph +boulevard +barrel +planted +##ome +pursuit +##sia +consequences +posts +shallow +invitation +rode +depot +ernest +kane +rod +concepts +preston +topic +chambers +striking +blast +arrives +descendants +montgomery +ranges +worlds +##lay +##ari +span +chaos +praise +##ag +fewer +1855 +sanctuary +mud +fbi +##ions +programmes +maintaining +unity +harper +bore +handsome +closure +tournaments +thunder +nebraska +linda +facade +puts +satisfied +argentine +dale +cork +dome +panama +##yl +1858 +tasks +experts +##ates +feeding +equation +##las +##ida +##tu +engage +bryan +##ax +um +quartet +melody +disbanded +sheffield +blocked +gasped +delay +kisses +maggie +connects +##non +sts +poured +creator +publishers +##we +guided +ellis +extinct +hug +gaining +##ord +complicated +##bility +poll +clenched +investigate +##use +thereby +quantum +spine +cdp +humor +kills +administered +semifinals +##du +encountered +ignore +##bu +commentary +##maker +bother +roosevelt +140 +plains +halfway +flowing +cultures +crack +imprisoned +neighboring +airline +##ses +##view +##mate +##ec +gather +wolves +marathon +transformed +##ill +cruise +organisations +carol +punch +exhibitions +numbered +alarm +ratings +daddy +silently +##stein +queens +colours +impression +guidance +liu +tactical +##rat +marshal +della +arrow +##ings +rested +feared +tender +owns +bitter +advisor +escort +##ides +spare +farms +grants +##ene +dragons +encourage +colleagues +cameras +##und +sucked +pile +spirits +prague +statements +suspension +landmark +fence +torture +recreation +bags +permanently +survivors +pond +spy +predecessor +bombing +coup +##og +protecting +transformation +glow +##lands +##book +dug +priests +andrea +feat +barn +jumping +##chen +##ologist +##con +casualties +stern +auckland +pipe +serie +revealing +ba +##bel +trevor +mercy +spectrum +yang +consist +governing +collaborated +possessed +epic +comprises +blew +shane +##ack +lopez +honored +magical +sacrifice +judgment +perceived +hammer +mtv +baronet +tune +das +missionary +sheets +350 +neutral +oral +threatening +attractive +shade +aims +seminary +##master +estates +1856 +michel +wounds +refugees +manufacturers +##nic +mercury +syndrome +porter +##iya +##din +hamburg +identification +upstairs +purse +widened +pause +cared +breathed +affiliate +santiago +prevented +celtic +fisher +125 +recruited +byzantine +reconstruction +farther +##mp +diet +sake +au +spite +sensation +##ert +blank +separation +105 +##hon +vladimir +armies +anime +##lie +accommodate +orbit +cult +sofia +archive +##ify +##box +founders +sustained +disorder +honours +northeastern +mia +crops +violet +threats +blanket +fires +canton +followers +southwestern +prototype +voyage +assignment +altered +moderate +protocol +pistol +##eo +questioned +brass +lifting +1852 +math +authored +##ual +doug +dimensional +dynamic +##san +1851 +pronounced +grateful +quest +uncomfortable +boom +presidency +stevens +relating +politicians +chen +barrier +quinn +diana +mosque +tribal +cheese +palmer +portions +sometime +chester +treasure +wu +bend +download +millions +reforms +registration +##osa +consequently +monitoring +ate +preliminary +brandon +invented +ps +eaten +exterior +intervention +ports +documented +log +displays +lecture +sally +favourite +##itz +vermont +lo +invisible +isle +breed +##ator +journalists +relay +speaks +backward +explore +midfielder +actively +stefan +procedures +cannon +blond +kenneth +centered +servants +chains +libraries +malcolm +essex +henri +slavery +##hal +facts +fairy +coached +cassie +cats +washed +cop +##fi +announcement +item +2000s +vinyl +activated +marco +frontier +growled +curriculum +##das +loyal +accomplished +leslie +ritual +kenny +##00 +vii +napoleon +hollow +hybrid +jungle +stationed +friedrich +counted +##ulated +platinum +theatrical +seated +col +rubber +glen +1840 +diversity +healing +extends +id +provisions +administrator +columbus +##oe +tributary +te +assured +org +##uous +prestigious +examined +lectures +grammy +ronald +associations +bailey +allan +essays +flute +believing +consultant +proceedings +travelling +1853 +kit +kerala +yugoslavia +buddy +methodist +##ith +burial +centres +batman +##nda +discontinued +bo +dock +stockholm +lungs +severely +##nk +citing +manga +##ugh +steal +mumbai +iraqi +robot +celebrity +bride +broadcasts +abolished +pot +joel +overhead +franz +packed +reconnaissance +johann +acknowledged +introduce +handled +doctorate +developments +drinks +alley +palestine +##nis +##aki +proceeded +recover +bradley +grain +patch +afford +infection +nationalist +legendary +##ath +interchange +virtually +gen +gravity +exploration +amber +vital +wishes +powell +doctrine +elbow +screenplay +##bird +contribute +indonesian +pet +creates +##com +enzyme +kylie +discipline +drops +manila +hunger +##ien +layers +suffer +fever +bits +monica +keyboard +manages +##hood +searched +appeals +##bad +testament +grande +reid +##war +beliefs +congo +##ification +##dia +si +requiring +##via +casey +1849 +regret +streak +rape +depends +syrian +sprint +pound +tourists +upcoming +pub +##xi +tense +##els +practiced +echo +nationwide +guild +motorcycle +liz +##zar +chiefs +desired +elena +bye +precious +absorbed +relatives +booth +pianist +##mal +citizenship +exhausted +wilhelm +##ceae +##hed +noting +quarterback +urge +hectares +##gue +ace +holly +##tal +blonde +davies +parked +sustainable +stepping +twentieth +airfield +galaxy +nest +chip +##nell +tan +shaft +paulo +requirement +##zy +paradise +tobacco +trans +renewed +vietnamese +##cker +##ju +suggesting +catching +holmes +enjoying +md +trips +colt +holder +butterfly +nerve +reformed +cherry +bowling +trailer +carriage +goodbye +appreciate +toy +joshua +interactive +enabled +involve +##kan +collar +determination +bunch +facebook +recall +shorts +superintendent +episcopal +frustration +giovanni +nineteenth +laser +privately +array +circulation +##ovic +armstrong +deals +painful +permit +discrimination +##wi +aires +retiring +cottage +ni +##sta +horizon +ellen +jamaica +ripped +fernando +chapters +playstation +patron +lecturer +navigation +behaviour +genes +georgian +export +solomon +rivals +swift +seventeen +rodriguez +princeton +independently +sox +1847 +arguing +entity +casting +hank +criteria +oakland +geographic +milwaukee +reflection +expanding +conquest +dubbed +##tv +halt +brave +brunswick +doi +arched +curtis +divorced +predominantly +somerset +streams +ugly +zoo +horrible +curved +buenos +fierce +dictionary +vector +theological +unions +handful +stability +chan +punjab +segments +##lly +altar +ignoring +gesture +monsters +pastor +##stone +thighs +unexpected +operators +abruptly +coin +compiled +associates +improving +migration +pin +##ose +compact +collegiate +reserved +##urs +quarterfinals +roster +restore +assembled +hurry +oval +##cies +1846 +flags +martha +##del +victories +sharply +##rated +argues +deadly +neo +drawings +symbols +performer +##iel +griffin +restrictions +editing +andrews +java +journals +arabia +compositions +dee +pierce +removing +hindi +casino +runway +civilians +minds +nasa +hotels +##zation +refuge +rent +retain +potentially +conferences +suburban +conducting +##tto +##tions +##tle +descended +massacre +##cal +ammunition +terrain +fork +souls +counts +chelsea +durham +drives +cab +##bank +perth +realizing +palestinian +finn +simpson +##dal +betty +##ule +moreover +particles +cardinals +tent +evaluation +extraordinary +##oid +inscription +##works +wednesday +chloe +maintains +panels +ashley +trucks +##nation +cluster +sunlight +strikes +zhang +##wing +dialect +canon +##ap +tucked +##ws +collecting +##mas +##can +##sville +maker +quoted +evan +franco +aria +buying +cleaning +eva +closet +provision +apollo +clinic +rat +##ez +necessarily +ac +##gle +##ising +venues +flipped +cent +spreading +trustees +checking +authorized +##sco +disappointed +##ado +notion +duration +trumpet +hesitated +topped +brussels +rolls +theoretical +hint +define +aggressive +repeat +wash +peaceful +optical +width +allegedly +mcdonald +strict +copyright +##illa +investors +mar +jam +witnesses +sounding +miranda +michelle +privacy +hugo +harmony +##pp +valid +lynn +glared +nina +102 +headquartered +diving +boarding +gibson +##ncy +albanian +marsh +routine +dealt +enhanced +er +intelligent +substance +targeted +enlisted +discovers +spinning +observations +pissed +smoking +rebecca +capitol +visa +varied +costume +seemingly +indies +compensation +surgeon +thursday +arsenal +westminster +suburbs +rid +anglican +##ridge +knots +foods +alumni +lighter +fraser +whoever +portal +scandal +##ray +gavin +advised +instructor +flooding +terrorist +##ale +teenage +interim +senses +duck +teen +thesis +abby +eager +overcome +##ile +newport +glenn +rises +shame +##cc +prompted +priority +forgot +bomber +nicolas +protective +360 +cartoon +katherine +breeze +lonely +trusted +henderson +richardson +relax +banner +candy +palms +remarkable +##rio +legends +cricketer +essay +ordained +edmund +rifles +trigger +##uri +##away +sail +alert +1830 +audiences +penn +sussex +siblings +pursued +indianapolis +resist +rosa +consequence +succeed +avoided +1845 +##ulation +inland +##tie +##nna +counsel +profession +chronicle +hurried +##una +eyebrow +eventual +bleeding +innovative +cure +##dom +committees +accounting +con +scope +hardy +heather +tenor +gut +herald +codes +tore +scales +wagon +##oo +luxury +tin +prefer +fountain +triangle +bonds +darling +convoy +dried +traced +beings +troy +accidentally +slam +findings +smelled +joey +lawyers +outcome +steep +bosnia +configuration +shifting +toll +brook +performers +lobby +philosophical +construct +shrine +aggregate +boot +cox +phenomenon +savage +insane +solely +reynolds +lifestyle +##ima +nationally +holdings +consideration +enable +edgar +mo +mama +##tein +fights +relegation +chances +atomic +hub +conjunction +awkward +reactions +currency +finale +kumar +underwent +steering +elaborate +gifts +comprising +melissa +veins +reasonable +sunshine +chi +solve +trails +inhabited +elimination +ethics +huh +ana +molly +consent +apartments +layout +marines +##ces +hunters +bulk +##oma +hometown +##wall +##mont +cracked +reads +neighbouring +withdrawn +admission +wingspan +damned +anthology +lancashire +brands +batting +forgive +cuban +awful +##lyn +104 +dimensions +imagination +##ade +dante +##ship +tracking +desperately +goalkeeper +##yne +groaned +workshops +confident +burton +gerald +milton +circus +uncertain +slope +copenhagen +sophia +fog +philosopher +portraits +accent +cycling +varying +gripped +larvae +garrett +specified +scotia +mature +luther +kurt +rap +##kes +aerial +750 +ferdinand +heated +es +transported +##shan +safely +nonetheless +##orn +##gal +motors +demanding +##sburg +startled +##brook +ally +generate +caps +ghana +stained +demo +mentions +beds +ap +afterward +diary +##bling +utility +##iro +richards +1837 +conspiracy +conscious +shining +footsteps +observer +cyprus +urged +loyalty +developer +probability +olive +upgraded +gym +miracle +insects +graves +1844 +ourselves +hydrogen +amazon +katie +tickets +poets +##pm +planes +##pan +prevention +witnessed +dense +jin +randy +tang +warehouse +monroe +bang +archived +elderly +investigations +alec +granite +mineral +conflicts +controlling +aboriginal +carlo +##zu +mechanics +stan +stark +rhode +skirt +est +##berry +bombs +respected +##horn +imposed +limestone +deny +nominee +memphis +grabbing +disabled +##als +amusement +aa +frankfurt +corn +referendum +varies +slowed +disk +firms +unconscious +incredible +clue +sue +##zhou +twist +##cio +joins +idaho +chad +developers +computing +destroyer +103 +mortal +tucker +kingston +choices +yu +carson +1800 +os +whitney +geneva +pretend +dimension +staged +plateau +maya +##une +freestyle +##bc +rovers +hiv +##ids +tristan +classroom +prospect +##hus +honestly +diploma +lied +thermal +auxiliary +feast +unlikely +iata +##tel +morocco +pounding +treasury +lithuania +considerably +1841 +dish +1812 +geological +matching +stumbled +destroying +marched +brien +advances +cake +nicole +belle +settling +measuring +directing +##mie +tuesday +bassist +capabilities +stunned +fraud +torpedo +##list +##phone +anton +wisdom +surveillance +ruined +##ulate +lawsuit +healthcare +theorem +halls +trend +aka +horizontal +dozens +acquire +lasting +swim +hawk +gorgeous +fees +vicinity +decrease +adoption +tactics +##ography +pakistani +##ole +draws +##hall +willie +burke +heath +algorithm +integral +powder +elliott +brigadier +jackie +tate +varieties +darker +##cho +lately +cigarette +specimens +adds +##ree +##ensis +##inger +exploded +finalist +cia +murders +wilderness +arguments +nicknamed +acceptance +onwards +manufacture +robertson +jets +tampa +enterprises +blog +loudly +composers +nominations +1838 +ai +malta +inquiry +automobile +hosting +viii +rays +tilted +grief +museums +strategies +furious +euro +equality +cohen +poison +surrey +wireless +governed +ridiculous +moses +##esh +##room +vanished +##ito +barnes +attract +morrison +istanbul +##iness +absent +rotation +petition +janet +##logical +satisfaction +custody +deliberately +observatory +comedian +surfaces +pinyin +novelist +strictly +canterbury +oslo +monks +embrace +ibm +jealous +photograph +continent +dorothy +marina +doc +excess +holden +allegations +explaining +stack +avoiding +lance +storyline +majesty +poorly +spike +dos +bradford +raven +travis +classics +proven +voltage +pillow +fists +butt +1842 +interpreted +##car +1839 +gage +telegraph +lens +promising +expelled +casual +collector +zones +##min +silly +nintendo +##kh +##bra +downstairs +chef +suspicious +afl +flies +vacant +uganda +pregnancy +condemned +lutheran +estimates +cheap +decree +saxon +proximity +stripped +idiot +deposits +contrary +presenter +magnus +glacier +im +offense +edwin +##ori +upright +##long +bolt +##ois +toss +geographical +##izes +environments +delicate +marking +abstract +xavier +nails +windsor +plantation +occurring +equity +saskatchewan +fears +drifted +sequences +vegetation +revolt +##stic +1843 +sooner +fusion +opposing +nato +skating +1836 +secretly +ruin +lease +##oc +edit +##nne +flora +anxiety +ruby +##ological +##mia +tel +bout +taxi +emmy +frost +rainbow +compounds +foundations +rainfall +assassination +nightmare +dominican +##win +achievements +deserve +orlando +intact +armenia +##nte +calgary +valentine +106 +marion +proclaimed +theodore +bells +courtyard +thigh +gonzalez +console +troop +minimal +monte +everyday +##ence +##if +supporter +terrorism +buck +openly +presbyterian +activists +carpet +##iers +rubbing +uprising +##yi +cute +conceived +legally +##cht +millennium +cello +velocity +ji +rescued +cardiff +1835 +rex +concentrate +senators +beard +rendered +glowing +battalions +scouts +competitors +sculptor +catalogue +arctic +ion +raja +bicycle +wow +glancing +lawn +##woman +gentleman +lighthouse +publish +predicted +calculated +##val +variants +##gne +strain +##ui +winston +deceased +##nus +touchdowns +brady +caleb +sinking +echoed +crush +hon +blessed +protagonist +hayes +endangered +magnitude +editors +##tine +estimate +responsibilities +##mel +backup +laying +consumed +sealed +zurich +lovers +frustrated +##eau +ahmed +kicking +mit +treasurer +1832 +biblical +refuse +terrified +pump +agrees +genuine +imprisonment +refuses +plymouth +##hen +lou +##nen +tara +trembling +antarctic +ton +learns +##tas +crap +crucial +faction +atop +##borough +wrap +lancaster +odds +hopkins +erik +lyon +##eon +bros +##ode +snap +locality +tips +empress +crowned +cal +acclaimed +chuckled +##ory +clara +sends +mild +towel +##fl +##day +##а +wishing +assuming +interviewed +##bal +##die +interactions +eden +cups +helena +##lf +indie +beck +##fire +batteries +filipino +wizard +parted +##lam +traces +##born +rows +idol +albany +delegates +##ees +##sar +discussions +##ex +notre +instructed +belgrade +highways +suggestion +lauren +possess +orientation +alexandria +abdul +beats +salary +reunion +ludwig +alright +wagner +intimate +pockets +slovenia +hugged +brighton +merchants +cruel +stole +trek +slopes +repairs +enrollment +politically +underlying +promotional +counting +boeing +##bb +isabella +naming +##и +keen +bacteria +listing +separately +belfast +ussr +450 +lithuanian +anybody +ribs +sphere +martinez +cock +embarrassed +proposals +fragments +nationals +##fs +##wski +premises +fin +1500 +alpine +matched +freely +bounded +jace +sleeve +##af +gaming +pier +populated +evident +##like +frances +flooded +##dle +frightened +pour +trainer +framed +visitor +challenging +pig +wickets +##fold +infected +email +##pes +arose +##aw +reward +ecuador +oblast +vale +ch +shuttle +##usa +bach +rankings +forbidden +cornwall +accordance +salem +consumers +bruno +fantastic +toes +machinery +resolved +julius +remembering +propaganda +iceland +bombardment +tide +contacts +wives +##rah +concerto +macdonald +albania +implement +daisy +tapped +sudan +helmet +angela +mistress +##lic +crop +sunk +finest +##craft +hostile +##ute +##tsu +boxer +fr +paths +adjusted +habit +ballot +supervision +soprano +##zen +bullets +wicked +sunset +regiments +disappear +lamp +performs +app +##gia +##oa +rabbit +digging +incidents +entries +##cion +dishes +##oi +introducing +##ati +##fied +freshman +slot +jill +tackles +baroque +backs +##iest +lone +sponsor +destiny +altogether +convert +##aro +consensus +shapes +demonstration +basically +feminist +auction +artifacts +##bing +strongest +twitter +halifax +2019 +allmusic +mighty +smallest +precise +alexandra +viola +##los +##ille +manuscripts +##illo +dancers +ari +managers +monuments +blades +barracks +springfield +maiden +consolidated +electron +##end +berry +airing +wheat +nobel +inclusion +blair +payments +geography +bee +cc +eleanor +react +##hurst +afc +manitoba +##yu +su +lineup +fitness +recreational +investments +airborne +disappointment +##dis +edmonton +viewing +##row +renovation +##cast +infant +bankruptcy +roses +aftermath +pavilion +##yer +carpenter +withdrawal +ladder +##hy +discussing +popped +reliable +agreements +rochester +##abad +curves +bombers +220 +rao +reverend +decreased +choosing +107 +stiff +consulting +naples +crawford +tracy +ka +ribbon +cops +##lee +crushed +deciding +unified +teenager +accepting +flagship +explorer +poles +sanchez +inspection +revived +skilled +induced +exchanged +flee +locals +tragedy +swallow +loading +hanna +demonstrate +##ela +salvador +flown +contestants +civilization +##ines +wanna +rhodes +fletcher +hector +knocking +considers +##ough +nash +mechanisms +sensed +mentally +walt +unclear +##eus +renovated +madame +##cks +crews +governmental +##hin +undertaken +monkey +##ben +##ato +fatal +armored +copa +caves +governance +grasp +perception +certification +froze +damp +tugged +wyoming +##rg +##ero +newman +##lor +nerves +curiosity +graph +115 +##ami +withdraw +tunnels +dull +meredith +moss +exhibits +neighbors +communicate +accuracy +explored +raiders +republicans +secular +kat +superman +penny +criticised +##tch +freed +update +conviction +wade +ham +likewise +delegation +gotta +doll +promises +technological +myth +nationality +resolve +convent +##mark +sharon +dig +sip +coordinator +entrepreneur +fold +##dine +capability +councillor +synonym +blown +swan +cursed +1815 +jonas +haired +sofa +canvas +keeper +rivalry +##hart +rapper +speedway +swords +postal +maxwell +estonia +potter +recurring +##nn +##ave +errors +##oni +cognitive +1834 +##² +claws +nadu +roberto +bce +wrestler +ellie +##ations +infinite +ink +##tia +presumably +finite +staircase +108 +noel +patricia +nacional +##cation +chill +eternal +tu +preventing +prussia +fossil +limbs +##logist +ernst +frog +perez +rene +##ace +pizza +prussian +##ios +##vy +molecules +regulatory +answering +opinions +sworn +lengths +supposedly +hypothesis +upward +habitats +seating +ancestors +drank +yield +hd +synthesis +researcher +modest +##var +mothers +peered +voluntary +homeland +##the +acclaim +##igan +static +valve +luxembourg +alto +carroll +fe +receptor +norton +ambulance +##tian +johnston +catholics +depicting +jointly +elephant +gloria +mentor +badge +ahmad +distinguish +remarked +councils +precisely +allison +advancing +detection +crowded +##10 +cooperative +ankle +mercedes +dagger +surrendered +pollution +commit +subway +jeffrey +lesson +sculptures +provider +##fication +membrane +timothy +rectangular +fiscal +heating +teammate +basket +particle +anonymous +deployment +##ple +missiles +courthouse +proportion +shoe +sec +##ller +complaints +forbes +blacks +abandon +remind +sizes +overwhelming +autobiography +natalie +##awa +risks +contestant +countryside +babies +scorer +invaded +enclosed +proceed +hurling +disorders +##cu +reflecting +continuously +cruiser +graduates +freeway +investigated +ore +deserved +maid +blocking +phillip +jorge +shakes +dove +mann +variables +lacked +burden +accompanying +que +consistently +organizing +provisional +complained +endless +##rm +tubes +juice +georges +krishna +mick +labels +thriller +##uch +laps +arcade +sage +snail +##table +shannon +fi +laurence +seoul +vacation +presenting +hire +churchill +surprisingly +prohibited +savannah +technically +##oli +170 +##lessly +testimony +suited +speeds +toys +romans +mlb +flowering +measurement +talented +kay +settings +charleston +expectations +shattered +achieving +triumph +ceremonies +portsmouth +lanes +mandatory +loser +stretching +cologne +realizes +seventy +cornell +careers +webb +##ulating +americas +budapest +ava +suspicion +##ison +yo +conrad +##hai +sterling +jessie +rector +##az +1831 +transform +organize +loans +christine +volcanic +warrant +slender +summers +subfamily +newer +danced +dynamics +rhine +proceeds +heinrich +gastropod +commands +sings +facilitate +easter +ra +positioned +responses +expense +fruits +yanked +imported +25th +velvet +vic +primitive +tribune +baldwin +neighbourhood +donna +rip +hay +pr +##uro +1814 +espn +welcomed +##aria +qualifier +glare +highland +timing +##cted +shells +eased +geometry +louder +exciting +slovakia +##sion +##iz +##lot +savings +prairie +##ques +marching +rafael +tonnes +##lled +curtain +preceding +shy +heal +greene +worthy +##pot +detachment +bury +sherman +##eck +reinforced +seeks +bottles +contracted +duchess +outfit +walsh +##sc +mickey +##ase +geoffrey +archer +squeeze +dawson +eliminate +invention +##enberg +neal +##eth +stance +dealer +coral +maple +retire +polo +simplified +##ht +1833 +hid +watts +backwards +jules +##oke +genesis +mt +frames +rebounds +burma +woodland +moist +santos +whispers +drained +subspecies +##aa +streaming +ulster +burnt +correspondence +maternal +gerard +denis +stealing +##load +genius +duchy +##oria +inaugurated +momentum +suits +placement +sovereign +clause +thames +##hara +confederation +reservation +sketch +yankees +lets +rotten +charm +hal +verses +ultra +commercially +dot +salon +citation +adopt +winnipeg +mist +allocated +cairo +##boy +jenkins +interference +objectives +##wind +1820 +portfolio +armoured +sectors +##eh +initiatives +##world +integrity +exercises +robe +tap +ab +gazed +##tones +distracted +rulers +111 +favorable +jerome +tended +cart +factories +##eri +diplomat +valued +gravel +charitable +##try +calvin +exploring +chang +shepherd +terrace +pdf +pupil +##ural +reflects +ups +##rch +governors +shelf +depths +##nberg +trailed +crest +tackle +##nian +##ats +hatred +##kai +clare +makers +ethiopia +longtime +detected +embedded +lacking +slapped +rely +thomson +anticipation +iso +morton +successive +agnes +screenwriter +straightened +philippe +playwright +haunted +licence +iris +intentions +sutton +112 +logical +correctly +##weight +branded +licked +tipped +silva +ricky +narrator +requests +##ents +greeted +supernatural +cow +##wald +lung +refusing +employer +strait +gaelic +liner +##piece +zoe +sabha +##mba +driveway +harvest +prints +bates +reluctantly +threshold +algebra +ira +wherever +coupled +240 +assumption +picks +##air +designers +raids +gentlemen +##ean +roller +blowing +leipzig +locks +screw +dressing +strand +##lings +scar +dwarf +depicts +##nu +nods +##mine +differ +boris +##eur +yuan +flip +##gie +mob +invested +questioning +applying +##ture +shout +##sel +gameplay +blamed +illustrations +bothered +weakness +rehabilitation +##of +##zes +envelope +rumors +miners +leicester +subtle +kerry +##ico +ferguson +##fu +premiership +ne +##cat +bengali +prof +catches +remnants +dana +##rily +shouting +presidents +baltic +ought +ghosts +dances +sailors +shirley +fancy +dominic +##bie +madonna +##rick +bark +buttons +gymnasium +ashes +liver +toby +oath +providence +doyle +evangelical +nixon +cement +carnegie +embarked +hatch +surroundings +guarantee +needing +pirate +essence +##bee +filter +crane +hammond +projected +immune +percy +twelfth +##ult +regent +doctoral +damon +mikhail +##ichi +lu +critically +elect +realised +abortion +acute +screening +mythology +steadily +##fc +frown +nottingham +kirk +wa +minneapolis +##rra +module +algeria +mc +nautical +encounters +surprising +statues +availability +shirts +pie +alma +brows +munster +mack +soup +crater +tornado +sanskrit +cedar +explosive +bordered +dixon +planets +stamp +exam +happily +##bble +carriers +kidnapped +##vis +accommodation +emigrated +##met +knockout +correspondent +violation +profits +peaks +lang +specimen +agenda +ancestry +pottery +spelling +equations +obtaining +ki +linking +1825 +debris +asylum +##20 +buddhism +teddy +##ants +gazette +##nger +##sse +dental +eligibility +utc +fathers +averaged +zimbabwe +francesco +coloured +hissed +translator +lynch +mandate +humanities +mackenzie +uniforms +lin +##iana +##gio +asset +mhz +fitting +samantha +genera +wei +rim +beloved +shark +riot +entities +expressions +indo +carmen +slipping +owing +abbot +neighbor +sidney +##av +rats +recommendations +encouraging +squadrons +anticipated +commanders +conquered +##oto +donations +diagnosed +##mond +divide +##iva +guessed +decoration +vernon +auditorium +revelation +conversations +##kers +##power +herzegovina +dash +alike +protested +lateral +herman +accredited +mg +##gent +freeman +mel +fiji +crow +crimson +##rine +livestock +##pped +humanitarian +bored +oz +whip +##lene +##ali +legitimate +alter +grinning +spelled +anxious +oriental +wesley +##nin +##hole +carnival +controller +detect +##ssa +bowed +educator +kosovo +macedonia +##sin +occupy +mastering +stephanie +janeiro +para +unaware +nurses +noon +135 +cam +hopefully +ranger +combine +sociology +polar +rica +##eer +neill +##sman +holocaust +##ip +doubled +lust +1828 +109 +decent +cooling +unveiled +##card +1829 +nsw +homer +chapman +meyer +##gin +dive +mae +reagan +expertise +##gled +darwin +brooke +sided +prosecution +investigating +comprised +petroleum +genres +reluctant +differently +trilogy +johns +vegetables +corpse +highlighted +lounge +pension +unsuccessfully +elegant +aided +ivory +beatles +amelia +cain +dubai +sunny +immigrant +babe +click +##nder +underwater +pepper +combining +mumbled +atlas +horns +accessed +ballad +physicians +homeless +gestured +rpm +freak +louisville +corporations +patriots +prizes +rational +warn +modes +decorative +overnight +din +troubled +phantom +##ort +monarch +sheer +##dorf +generals +guidelines +organs +addresses +##zon +enhance +curling +parishes +cord +##kie +linux +caesar +deutsche +bavaria +##bia +coleman +cyclone +##eria +bacon +petty +##yama +##old +hampton +diagnosis +1824 +throws +complexity +rita +disputed +##₃ +pablo +##sch +marketed +trafficking +##ulus +examine +plague +formats +##oh +vault +faithful +##bourne +webster +##ox +highlights +##ient +##ann +phones +vacuum +sandwich +modeling +##gated +bolivia +clergy +qualities +isabel +##nas +##ars +wears +screams +reunited +annoyed +bra +##ancy +##rate +differential +transmitter +tattoo +container +poker +##och +excessive +resides +cowboys +##tum +augustus +trash +providers +statute +retreated +balcony +reversed +void +storey +preceded +masses +leap +laughs +neighborhoods +wards +schemes +falcon +santo +battlefield +pad +ronnie +thread +lesbian +venus +##dian +beg +sandstone +daylight +punched +gwen +analog +stroked +wwe +acceptable +measurements +dec +toxic +##kel +adequate +surgical +economist +parameters +varsity +##sberg +quantity +ella +##chy +##rton +countess +generating +precision +diamonds +expressway +ga +##ı +1821 +uruguay +talents +galleries +expenses +scanned +colleague +outlets +ryder +lucien +##ila +paramount +##bon +syracuse +dim +fangs +gown +sweep +##sie +toyota +missionaries +websites +##nsis +sentences +adviser +val +trademark +spells +##plane +patience +starter +slim +##borg +toe +incredibly +shoots +elliot +nobility +##wyn +cowboy +endorsed +gardner +tendency +persuaded +organisms +emissions +kazakhstan +amused +boring +chips +themed +##hand +llc +constantinople +chasing +systematic +guatemala +borrowed +erin +carey +##hard +highlands +struggles +1810 +##ifying +##ced +wong +exceptions +develops +enlarged +kindergarten +castro +##ern +##rina +leigh +zombie +juvenile +##most +consul +##nar +sailor +hyde +clarence +intensive +pinned +nasty +useless +jung +clayton +stuffed +exceptional +ix +apostolic +230 +transactions +##dge +exempt +swinging +cove +religions +##ash +shields +dairy +bypass +190 +pursuing +bug +joyce +bombay +chassis +southampton +chat +interact +redesignated +##pen +nascar +pray +salmon +rigid +regained +malaysian +grim +publicity +constituted +capturing +toilet +delegate +purely +tray +drift +loosely +striker +weakened +trinidad +mitch +itv +defines +transmitted +ming +scarlet +nodding +fitzgerald +fu +narrowly +sp +tooth +standings +virtue +##₁ +##wara +##cting +chateau +gloves +lid +##nel +hurting +conservatory +##pel +sinclair +reopened +sympathy +nigerian +strode +advocated +optional +chronic +discharge +##rc +suck +compatible +laurel +stella +shi +fails +wage +dodge +128 +informal +sorts +levi +buddha +villagers +##aka +chronicles +heavier +summoned +gateway +3000 +eleventh +jewelry +translations +accordingly +seas +##ency +fiber +pyramid +cubic +dragging +##ista +caring +##ops +android +contacted +lunar +##dt +kai +lisbon +patted +1826 +sacramento +theft +madagascar +subtropical +disputes +ta +holidays +piper +willow +mare +cane +itunes +newfoundland +benny +companions +dong +raj +observe +roar +charming +plaque +tibetan +fossils +enacted +manning +bubble +tina +tanzania +##eda +##hir +funk +swamp +deputies +cloak +ufc +scenario +par +scratch +metals +anthem +guru +engaging +specially +##boat +dialects +nineteen +cecil +duet +disability +messenger +unofficial +##lies +defunct +eds +moonlight +drainage +surname +puzzle +honda +switching +conservatives +mammals +knox +broadcaster +sidewalk +cope +##ried +benson +princes +peterson +##sal +bedford +sharks +eli +wreck +alberto +gasp +archaeology +lgbt +teaches +securities +madness +compromise +waving +coordination +davidson +visions +leased +possibilities +eighty +jun +fernandez +enthusiasm +assassin +sponsorship +reviewer +kingdoms +estonian +laboratories +##fy +##nal +applies +verb +celebrations +##zzo +rowing +lightweight +sadness +submit +mvp +balanced +dude +##vas +explicitly +metric +magnificent +mound +brett +mohammad +mistakes +irregular +##hing +##ass +sanders +betrayed +shipped +surge +##enburg +reporters +termed +georg +pity +verbal +bulls +abbreviated +enabling +appealed +##are +##atic +sicily +sting +heel +sweetheart +bart +spacecraft +brutal +monarchy +##tter +aberdeen +cameo +diane +##ub +survivor +clyde +##aries +complaint +##makers +clarinet +delicious +chilean +karnataka +coordinates +1818 +panties +##rst +pretending +ar +dramatically +kiev +bella +tends +distances +113 +catalog +launching +instances +telecommunications +portable +lindsay +vatican +##eim +angles +aliens +marker +stint +screens +bolton +##rne +judy +wool +benedict +plasma +europa +spark +imaging +filmmaker +swiftly +##een +contributor +##nor +opted +stamps +apologize +financing +butter +gideon +sophisticated +alignment +avery +chemicals +yearly +speculation +prominence +professionally +##ils +immortal +institutional +inception +wrists +identifying +tribunal +derives +gains +##wo +papal +preference +linguistic +vince +operative +brewery +##ont +unemployment +boyd +##ured +##outs +albeit +prophet +1813 +bi +##rr +##face +##rad +quarterly +asteroid +cleaned +radius +temper +##llen +telugu +jerk +viscount +menu +##ote +glimpse +##aya +yacht +hawaiian +baden +##rl +laptop +readily +##gu +monetary +offshore +scots +watches +##yang +##arian +upgrade +needle +xbox +lea +encyclopedia +flank +fingertips +##pus +delight +teachings +confirm +roth +beaches +midway +winters +##iah +teasing +daytime +beverly +gambling +bonnie +##backs +regulated +clement +hermann +tricks +knot +##shing +##uring +##vre +detached +ecological +owed +specialty +byron +inventor +bats +stays +screened +unesco +midland +trim +affection +##ander +##rry +jess +thoroughly +feedback +##uma +chennai +strained +heartbeat +wrapping +overtime +pleaded +##sworth +mon +leisure +oclc +##tate +##ele +feathers +angelo +thirds +nuts +surveys +clever +gill +commentator +##dos +darren +rides +gibraltar +##nc +##mu +dissolution +dedication +shin +meals +saddle +elvis +reds +chaired +taller +appreciation +functioning +niece +favored +advocacy +robbie +criminals +suffolk +yugoslav +passport +constable +congressman +hastings +vera +##rov +consecrated +sparks +ecclesiastical +confined +##ovich +muller +floyd +nora +1822 +paved +1827 +cumberland +ned +saga +spiral +##flow +appreciated +yi +collaborative +treating +similarities +feminine +finishes +##ib +jade +import +##nse +##hot +champagne +mice +securing +celebrities +helsinki +attributes +##gos +cousins +phases +ache +lucia +gandhi +submission +vicar +spear +shine +tasmania +biting +detention +constitute +tighter +seasonal +##gus +terrestrial +matthews +##oka +effectiveness +parody +philharmonic +##onic +1816 +strangers +encoded +consortium +guaranteed +regards +shifts +tortured +collision +supervisor +inform +broader +insight +theaters +armour +emeritus +blink +incorporates +mapping +##50 +##ein +handball +flexible +##nta +substantially +generous +thief +##own +carr +loses +1793 +prose +ucla +romeo +generic +metallic +realization +damages +mk +commissioners +zach +default +##ther +helicopters +lengthy +stems +spa +partnered +spectators +rogue +indication +penalties +teresa +1801 +sen +##tric +dalton +##wich +irving +photographic +##vey +dell +deaf +peters +excluded +unsure +##vable +patterson +crawled +##zio +resided +whipped +latvia +slower +ecole +pipes +employers +maharashtra +comparable +va +textile +pageant +##gel +alphabet +binary +irrigation +chartered +choked +antoine +offs +waking +supplement +##wen +quantities +demolition +regain +locate +urdu +folks +alt +114 +##mc +scary +andreas +whites +##ava +classrooms +mw +aesthetic +publishes +valleys +guides +cubs +johannes +bryant +conventions +affecting +##itt +drain +awesome +isolation +prosecutor +ambitious +apology +captive +downs +atmospheric +lorenzo +aisle +beef +foul +##onia +kidding +composite +disturbed +illusion +natives +##ffer +emi +rockets +riverside +wartime +painters +adolf +melted +##ail +uncertainty +simulation +hawks +progressed +meantime +builder +spray +breach +unhappy +regina +russians +##urg +determining +##tation +tram +1806 +##quin +aging +##12 +1823 +garion +rented +mister +diaz +terminated +clip +1817 +depend +nervously +disco +owe +defenders +shiva +notorious +disbelief +shiny +worcester +##gation +##yr +trailing +undertook +islander +belarus +limitations +watershed +fuller +overlooking +utilized +raphael +1819 +synthetic +breakdown +klein +##nate +moaned +memoir +lamb +practicing +##erly +cellular +arrows +exotic +##graphy +witches +117 +charted +rey +hut +hierarchy +subdivision +freshwater +giuseppe +aloud +reyes +qatar +marty +sideways +utterly +sexually +jude +prayers +mccarthy +softball +blend +damien +##gging +##metric +wholly +erupted +lebanese +negro +revenues +tasted +comparative +teamed +transaction +labeled +maori +sovereignty +parkway +trauma +gran +malay +121 +advancement +descendant +2020 +buzz +salvation +inventory +symbolic +##making +antarctica +mps +##gas +##bro +mohammed +myanmar +holt +submarines +tones +##lman +locker +patriarch +bangkok +emerson +remarks +predators +kin +afghan +confession +norwich +rental +emerge +advantages +##zel +rca +##hold +shortened +storms +aidan +##matic +autonomy +compliance +##quet +dudley +atp +##osis +1803 +motto +documentation +summary +professors +spectacular +christina +archdiocese +flashing +innocence +remake +##dell +psychic +reef +scare +employ +rs +sticks +meg +gus +leans +##ude +accompany +bergen +tomas +##iko +doom +wages +pools +##nch +##bes +breasts +scholarly +alison +outline +brittany +breakthrough +willis +realistic +##cut +##boro +competitor +##stan +pike +picnic +icon +designing +commercials +washing +villain +skiing +micro +costumes +auburn +halted +executives +##hat +logistics +cycles +vowel +applicable +barrett +exclaimed +eurovision +eternity +ramon +##umi +##lls +modifications +sweeping +disgust +##uck +torch +aviv +ensuring +rude +dusty +sonic +donovan +outskirts +cu +pathway +##band +##gun +##lines +disciplines +acids +cadet +paired +##40 +sketches +##sive +marriages +##⁺ +folding +peers +slovak +implies +admired +##beck +1880s +leopold +instinct +attained +weston +megan +horace +##ination +dorsal +ingredients +evolutionary +##its +complications +deity +lethal +brushing +levy +deserted +institutes +posthumously +delivering +telescope +coronation +motivated +rapids +luc +flicked +pays +volcano +tanner +weighed +##nica +crowds +frankie +gifted +addressing +granddaughter +winding +##rna +constantine +gomez +##front +landscapes +rudolf +anthropology +slate +werewolf +##lio +astronomy +circa +rouge +dreaming +sack +knelt +drowned +naomi +prolific +tracked +freezing +herb +##dium +agony +randall +twisting +wendy +deposit +touches +vein +wheeler +##bbled +##bor +batted +retaining +tire +presently +compare +specification +daemon +nigel +##grave +merry +recommendation +czechoslovakia +sandra +ng +roma +##sts +lambert +inheritance +sheikh +winchester +cries +examining +##yle +comeback +cuisine +nave +##iv +ko +retrieve +tomatoes +barker +polished +defining +irene +lantern +personalities +begging +tract +swore +1809 +175 +##gic +omaha +brotherhood +##rley +haiti +##ots +exeter +##ete +##zia +steele +dumb +pearson +210 +surveyed +elisabeth +trends +##ef +fritz +##rf +premium +bugs +fraction +calmly +viking +##birds +tug +inserted +unusually +##ield +confronted +distress +crashing +brent +turks +resign +##olo +cambodia +gabe +sauce +##kal +evelyn +116 +extant +clusters +quarry +teenagers +luna +##lers +##ister +affiliation +drill +##ashi +panthers +scenic +libya +anita +strengthen +inscriptions +##cated +lace +sued +judith +riots +##uted +mint +##eta +preparations +midst +dub +challenger +##vich +mock +cf +displaced +wicket +breaths +enables +schmidt +analyst +##lum +ag +highlight +automotive +axe +josef +newark +sufficiently +resembles +50th +##pal +flushed +mum +traits +##ante +commodore +incomplete +warming +titular +ceremonial +ethical +118 +celebrating +eighteenth +cao +lima +medalist +mobility +strips +snakes +##city +miniature +zagreb +barton +escapes +umbrella +automated +doubted +differs +cooled +georgetown +dresden +cooked +fade +wyatt +rna +jacobs +carlton +abundant +stereo +boost +madras +inning +##hia +spur +ip +malayalam +begged +osaka +groan +escaping +charging +dose +vista +##aj +bud +papa +communists +advocates +edged +tri +##cent +resemble +peaking +necklace +fried +montenegro +saxony +goose +glances +stuttgart +curator +recruit +grocery +sympathetic +##tting +##fort +127 +lotus +randolph +ancestor +##rand +succeeding +jupiter +1798 +macedonian +##heads +hiking +1808 +handing +fischer +##itive +garbage +node +##pies +prone +singular +papua +inclined +attractions +italia +pouring +motioned +grandma +garnered +jacksonville +corp +ego +ringing +aluminum +##hausen +ordering +##foot +drawer +traders +synagogue +##play +##kawa +resistant +wandering +fragile +fiona +teased +var +hardcore +soaked +jubilee +decisive +exposition +mercer +poster +valencia +hale +kuwait +1811 +##ises +##wr +##eed +tavern +gamma +122 +johan +##uer +airways +amino +gil +##ury +vocational +domains +torres +##sp +generator +folklore +outcomes +##keeper +canberra +shooter +fl +beams +confrontation +##lling +##gram +feb +aligned +forestry +pipeline +jax +motorway +conception +decay +##tos +coffin +##cott +stalin +1805 +escorted +minded +##nam +sitcom +purchasing +twilight +veronica +additions +passive +tensions +straw +123 +frequencies +1804 +refugee +cultivation +##iate +christie +clary +bulletin +crept +disposal +##rich +##zong +processor +crescent +##rol +bmw +emphasized +whale +nazis +aurora +##eng +dwelling +hauled +sponsors +toledo +mega +ideology +theatres +tessa +cerambycidae +saves +turtle +cone +suspects +kara +rusty +yelling +greeks +mozart +shades +cocked +participant +##tro +shire +spit +freeze +necessity +##cos +inmates +nielsen +councillors +loaned +uncommon +omar +peasants +botanical +offspring +daniels +formations +jokes +1794 +pioneers +sigma +licensing +##sus +wheelchair +polite +1807 +liquor +pratt +trustee +##uta +forewings +balloon +##zz +kilometre +camping +explicit +casually +shawn +foolish +teammates +nm +hassan +carrie +judged +satisfy +vanessa +knives +selective +cnn +flowed +##lice +eclipse +stressed +eliza +mathematician +cease +cultivated +##roy +commissions +browns +##ania +destroyers +sheridan +meadow +##rius +minerals +##cial +downstream +clash +gram +memoirs +ventures +baha +seymour +archie +midlands +edith +fare +flynn +invite +canceled +tiles +stabbed +boulder +incorporate +amended +camden +facial +mollusk +unreleased +descriptions +yoga +grabs +550 +raises +ramp +shiver +##rose +coined +pioneering +tunes +qing +warwick +tops +119 +melanie +giles +##rous +wandered +##inal +annexed +nov +30th +unnamed +##ished +organizational +airplane +normandy +stoke +whistle +blessing +violations +chased +holders +shotgun +##ctic +outlet +reactor +##vik +tires +tearing +shores +fortified +mascot +constituencies +nc +columnist +productive +tibet +##rta +lineage +hooked +oct +tapes +judging +cody +##gger +hansen +kashmir +triggered +##eva +solved +cliffs +##tree +resisted +anatomy +protesters +transparent +implied +##iga +injection +mattress +excluding +##mbo +defenses +helpless +devotion +##elli +growl +liberals +weber +phenomena +atoms +plug +##iff +mortality +apprentice +howe +convincing +aaa +swimmer +barber +leone +promptly +sodium +def +nowadays +arise +##oning +gloucester +corrected +dignity +norm +erie +##ders +elders +evacuated +sylvia +compression +##yar +hartford +pose +backpack +reasoning +accepts +24th +wipe +millimetres +marcel +##oda +dodgers +albion +1790 +overwhelmed +aerospace +oaks +1795 +showcase +acknowledge +recovering +nolan +ashe +hurts +geology +fashioned +disappearance +farewell +swollen +shrug +marquis +wimbledon +124 +rue +1792 +commemorate +reduces +experiencing +inevitable +calcutta +intel +##court +murderer +sticking +fisheries +imagery +bloom +280 +brake +##inus +gustav +hesitation +memorable +po +viral +beans +accidents +tunisia +antenna +spilled +consort +treatments +aye +perimeter +##gard +donation +hostage +migrated +banker +addiction +apex +lil +trout +##ously +conscience +##nova +rams +sands +genome +passionate +troubles +##lets +##set +amid +##ibility +##ret +higgins +exceed +vikings +##vie +payne +##zan +muscular +##ste +defendant +sucking +##wal +ibrahim +fuselage +claudia +vfl +europeans +snails +interval +##garh +preparatory +statewide +tasked +lacrosse +viktor +##lation +angola +##hra +flint +implications +employs +teens +patrons +stall +weekends +barriers +scrambled +nucleus +tehran +jenna +parsons +lifelong +robots +displacement +5000 +##bles +precipitation +##gt +knuckles +clutched +1802 +marrying +ecology +marx +accusations +declare +scars +kolkata +mat +meadows +bermuda +skeleton +finalists +vintage +crawl +coordinate +affects +subjected +orchestral +mistaken +##tc +mirrors +dipped +relied +260 +arches +candle +##nick +incorporating +wildly +fond +basilica +owl +fringe +rituals +whispering +stirred +feud +tertiary +slick +goat +honorable +whereby +skip +ricardo +stripes +parachute +adjoining +submerged +synthesizer +##gren +intend +positively +ninety +phi +beaver +partition +fellows +alexis +prohibition +carlisle +bizarre +fraternity +##bre +doubts +icy +cbc +aquatic +sneak +sonny +combines +airports +crude +supervised +spatial +merge +alfonso +##bic +corrupt +scan +undergo +##ams +disabilities +colombian +comparing +dolphins +perkins +##lish +reprinted +unanimous +bounced +hairs +underworld +midwest +semester +bucket +paperback +miniseries +coventry +demise +##leigh +demonstrations +sensor +rotating +yan +##hler +arrange +soils +##idge +hyderabad +labs +##dr +brakes +grandchildren +##nde +negotiated +rover +ferrari +continuation +directorate +augusta +stevenson +counterpart +gore +##rda +nursery +rican +ave +collectively +broadly +pastoral +repertoire +asserted +discovering +nordic +styled +fiba +cunningham +harley +middlesex +survives +tumor +tempo +zack +aiming +lok +urgent +##rade +##nto +devils +##ement +contractor +turin +##wl +##ool +bliss +repaired +simmons +moan +astronomical +cr +negotiate +lyric +1890s +lara +bred +clad +angus +pbs +##ience +engineered +posed +##lk +hernandez +possessions +elbows +psychiatric +strokes +confluence +electorate +lifts +campuses +lava +alps +##ep +##ution +##date +physicist +woody +##page +##ographic +##itis +juliet +reformation +sparhawk +320 +complement +suppressed +jewel +##½ +floated +##kas +continuity +sadly +##ische +inability +melting +scanning +paula +flour +judaism +safer +vague +##lm +solving +curb +##stown +financially +gable +bees +expired +miserable +cassidy +dominion +1789 +cupped +145 +robbery +facto +amos +warden +resume +tallest +marvin +ing +pounded +usd +declaring +gasoline +##aux +darkened +270 +650 +sophomore +##mere +erection +gossip +televised +risen +dial +##eu +pillars +##link +passages +profound +##tina +arabian +ashton +silicon +nail +##ead +##lated +##wer +##hardt +fleming +firearms +ducked +circuits +blows +waterloo +titans +##lina +atom +fireplace +cheshire +financed +activation +algorithms +##zzi +constituent +catcher +cherokee +partnerships +sexuality +platoon +tragic +vivian +guarded +whiskey +meditation +poetic +##late +##nga +##ake +porto +listeners +dominance +kendra +mona +chandler +factions +22nd +salisbury +attitudes +derivative +##ido +##haus +intake +paced +javier +illustrator +barrels +bias +cockpit +burnett +dreamed +ensuing +##anda +receptors +someday +hawkins +mattered +##lal +slavic +1799 +jesuit +cameroon +wasted +tai +wax +lowering +victorious +freaking +outright +hancock +librarian +sensing +bald +calcium +myers +tablet +announcing +barack +shipyard +pharmaceutical +##uan +greenwich +flush +medley +patches +wolfgang +pt +speeches +acquiring +exams +nikolai +##gg +hayden +kannada +##type +reilly +##pt +waitress +abdomen +devastated +capped +pseudonym +pharmacy +fulfill +paraguay +1796 +clicked +##trom +archipelago +syndicated +##hman +lumber +orgasm +rejection +clifford +lorraine +advent +mafia +rodney +brock +##ght +##used +##elia +cassette +chamberlain +despair +mongolia +sensors +developmental +upstream +##eg +##alis +spanning +165 +trombone +basque +seeded +interred +renewable +rhys +leapt +revision +molecule +##ages +chord +vicious +nord +shivered +23rd +arlington +debts +corpus +sunrise +bays +blackburn +centimetres +##uded +shuddered +gm +strangely +gripping +cartoons +isabelle +orbital +##ppa +seals +proving +##lton +refusal +strengthened +bust +assisting +baghdad +batsman +portrayal +mara +pushes +spears +og +##cock +reside +nathaniel +brennan +1776 +confirmation +caucus +##worthy +markings +yemen +nobles +ku +lazy +viewer +catalan +encompasses +sawyer +##fall +sparked +substances +patents +braves +arranger +evacuation +sergio +persuade +dover +tolerance +penguin +cum +jockey +insufficient +townships +occupying +declining +plural +processed +projection +puppet +flanders +introduces +liability +##yon +gymnastics +antwerp +taipei +hobart +candles +jeep +wes +observers +126 +chaplain +bundle +glorious +##hine +hazel +flung +sol +excavations +dumped +stares +sh +bangalore +triangular +icelandic +intervals +expressing +turbine +##vers +songwriting +crafts +##igo +jasmine +ditch +rite +##ways +entertaining +comply +sorrow +wrestlers +basel +emirates +marian +rivera +helpful +##some +caution +downward +networking +##atory +##tered +darted +genocide +emergence +replies +specializing +spokesman +convenient +unlocked +fading +augustine +concentrations +resemblance +elijah +investigator +andhra +##uda +promotes +bean +##rrell +fleeing +wan +simone +announcer +##ame +##bby +lydia +weaver +132 +residency +modification +##fest +stretches +##ast +alternatively +nat +lowe +lacks +##ented +pam +tile +concealed +inferior +abdullah +residences +tissues +vengeance +##ided +moisture +peculiar +groove +zip +bologna +jennings +ninja +oversaw +zombies +pumping +batch +livingston +emerald +installations +1797 +peel +nitrogen +rama +##fying +##star +schooling +strands +responding +werner +##ost +lime +casa +accurately +targeting +##rod +underway +##uru +hemisphere +lester +##yard +occupies +2d +griffith +angrily +reorganized +##owing +courtney +deposited +##dd +##30 +estadio +##ifies +dunn +exiled +##ying +checks +##combe +##о +##fly +successes +unexpectedly +blu +assessed +##flower +##ه +observing +sacked +spiders +kn +##tail +mu +nodes +prosperity +audrey +divisional +155 +broncos +tangled +adjust +feeds +erosion +paolo +surf +directory +snatched +humid +admiralty +screwed +gt +reddish +##nese +modules +trench +lamps +bind +leah +bucks +competes +##nz +##form +transcription +##uc +isles +violently +clutching +pga +cyclist +inflation +flats +ragged +unnecessary +##hian +stubborn +coordinated +harriet +baba +disqualified +330 +insect +wolfe +##fies +reinforcements +rocked +duel +winked +embraced +bricks +##raj +hiatus +defeats +pending +brightly +jealousy +##xton +##hm +##uki +lena +gdp +colorful +##dley +stein +kidney +##shu +underwear +wanderers +##haw +##icus +guardians +m³ +roared +habits +##wise +permits +gp +uranium +punished +disguise +bundesliga +elise +dundee +erotic +partisan +pi +collectors +float +individually +rendering +behavioral +bucharest +ser +hare +valerie +corporal +nutrition +proportional +##isa +immense +##kis +pavement +##zie +##eld +sutherland +crouched +1775 +##lp +suzuki +trades +endurance +operas +crosby +prayed +priory +rory +socially +##urn +gujarat +##pu +walton +cube +pasha +privilege +lennon +floods +thorne +waterfall +nipple +scouting +approve +##lov +minorities +voter +dwight +extensions +assure +ballroom +slap +dripping +privileges +rejoined +confessed +demonstrating +patriotic +yell +investor +##uth +pagan +slumped +squares +##cle +##kins +confront +bert +embarrassment +##aid +aston +urging +sweater +starr +yuri +brains +williamson +commuter +mortar +structured +selfish +exports +##jon +cds +##him +unfinished +##rre +mortgage +destinations +##nagar +canoe +solitary +buchanan +delays +magistrate +fk +##pling +motivation +##lier +##vier +recruiting +assess +##mouth +malik +antique +1791 +pius +rahman +reich +tub +zhou +smashed +airs +galway +xii +conditioning +honduras +discharged +dexter +##pf +lionel +129 +debates +lemon +tiffany +volunteered +dom +dioxide +procession +devi +sic +tremendous +advertisements +colts +transferring +verdict +hanover +decommissioned +utter +relate +pac +racism +##top +beacon +limp +similarity +terra +occurrence +ant +##how +becky +capt +updates +armament +richie +pal +##graph +halloween +mayo +##ssen +##bone +cara +serena +fcc +dolls +obligations +##dling +violated +lafayette +jakarta +exploitation +##ime +infamous +iconic +##lah +##park +kitty +moody +reginald +dread +spill +crystals +olivier +modeled +bluff +equilibrium +separating +notices +ordnance +extinction +onset +cosmic +attachment +sammy +expose +privy +anchored +##bil +abbott +admits +bending +baritone +emmanuel +policeman +vaughan +winged +climax +dresses +denny +polytechnic +mohamed +burmese +authentic +nikki +genetics +grandparents +homestead +gaza +postponed +metacritic +una +##sby +##bat +unstable +dissertation +##rial +##cian +curls +obscure +uncovered +bronx +praying +disappearing +##hoe +prehistoric +coke +turret +mutations +nonprofit +pits +monaco +##ي +##usion +prominently +dispatched +podium +##mir +uci +##uation +133 +fortifications +birthplace +kendall +##lby +##oll +preacher +rack +goodman +##rman +persistent +##ott +countless +jaime +recorder +lexington +persecution +jumps +renewal +wagons +##11 +crushing +##holder +decorations +##lake +abundance +wrath +laundry +£1 +garde +##rp +jeanne +beetles +peasant +##sl +splitting +caste +sergei +##rer +##ema +scripts +##ively +rub +satellites +##vor +inscribed +verlag +scrapped +gale +packages +chick +potato +slogan +kathleen +arabs +##culture +counterparts +reminiscent +choral +##tead +rand +retains +bushes +dane +accomplish +courtesy +closes +##oth +slaughter +hague +krakow +lawson +tailed +elias +ginger +##ttes +canopy +betrayal +rebuilding +turf +##hof +frowning +allegiance +brigades +kicks +rebuild +polls +alias +nationalism +td +rowan +audition +bowie +fortunately +recognizes +harp +dillon +horrified +##oro +renault +##tics +ropes +##α +presumed +rewarded +infrared +wiping +accelerated +illustration +##rid +presses +practitioners +badminton +##iard +detained +##tera +recognizing +relates +misery +##sies +##tly +reproduction +piercing +potatoes +thornton +esther +manners +hbo +##aan +ours +bullshit +ernie +perennial +sensitivity +illuminated +rupert +##jin +##iss +##ear +rfc +nassau +##dock +staggered +socialism +##haven +appointments +nonsense +prestige +sharma +haul +##tical +solidarity +gps +##ook +##rata +igor +pedestrian +##uit +baxter +tenants +wires +medication +unlimited +guiding +impacts +diabetes +##rama +sasha +pas +clive +extraction +131 +continually +constraints +##bilities +sonata +hunted +sixteenth +chu +planting +quote +mayer +pretended +abs +spat +##hua +ceramic +##cci +curtains +pigs +pitching +##dad +latvian +sore +dayton +##sted +##qi +patrols +slice +playground +##nted +shone +stool +apparatus +inadequate +mates +treason +##ija +desires +##liga +##croft +somalia +laurent +mir +leonardo +oracle +grape +obliged +chevrolet +thirteenth +stunning +enthusiastic +##ede +accounted +concludes +currents +basil +##kovic +drought +##rica +mai +##aire +shove +posting +##shed +pilgrimage +humorous +packing +fry +pencil +wines +smells +144 +marilyn +aching +newest +clung +bon +neighbours +sanctioned +##pie +mug +##stock +drowning +##mma +hydraulic +##vil +hiring +reminder +lilly +investigators +##ncies +sour +##eous +compulsory +packet +##rion +##graphic +##elle +cannes +##inate +depressed +##rit +heroic +importantly +theresa +##tled +conway +saturn +marginal +rae +##xia +corresponds +royce +pact +jasper +explosives +packaging +aluminium +##ttered +denotes +rhythmic +spans +assignments +hereditary +outlined +originating +sundays +lad +reissued +greeting +beatrice +##dic +pillar +marcos +plots +handbook +alcoholic +judiciary +avant +slides +extract +masculine +blur +##eum +##force +homage +trembled +owens +hymn +trey +omega +signaling +socks +accumulated +reacted +attic +theo +lining +angie +distraction +primera +talbot +##key +1200 +ti +creativity +billed +##hey +deacon +eduardo +identifies +proposition +dizzy +gunner +hogan +##yam +##pping +##hol +ja +##chan +jensen +reconstructed +##berger +clearance +darius +##nier +abe +harlem +plea +dei +circled +emotionally +notation +fascist +neville +exceeded +upwards +viable +ducks +##fo +workforce +racer +limiting +shri +##lson +possesses +1600 +kerr +moths +devastating +laden +disturbing +locking +##cture +gal +fearing +accreditation +flavor +aide +1870s +mountainous +##baum +melt +##ures +motel +texture +servers +soda +##mb +herd +##nium +erect +puzzled +hum +peggy +examinations +gould +testified +geoff +ren +devised +sacks +##law +denial +posters +grunted +cesar +tutor +ec +gerry +offerings +byrne +falcons +combinations +ct +incoming +pardon +rocking +26th +avengers +flared +mankind +seller +uttar +loch +nadia +stroking +exposing +##hd +fertile +ancestral +instituted +##has +noises +prophecy +taxation +eminent +vivid +pol +##bol +dart +indirect +multimedia +notebook +upside +displaying +adrenaline +referenced +geometric +##iving +progression +##ddy +blunt +announce +##far +implementing +##lav +aggression +liaison +cooler +cares +headache +plantations +gorge +dots +impulse +thickness +ashamed +averaging +kathy +obligation +precursor +137 +fowler +symmetry +thee +225 +hears +##rai +undergoing +ads +butcher +bowler +##lip +cigarettes +subscription +goodness +##ically +browne +##hos +##tech +kyoto +donor +##erty +damaging +friction +drifting +expeditions +hardened +prostitution +152 +fauna +blankets +claw +tossing +snarled +butterflies +recruits +investigative +coated +healed +138 +communal +hai +xiii +academics +boone +psychologist +restless +lahore +stephens +mba +brendan +foreigners +printer +##pc +ached +explode +27th +deed +scratched +dared +##pole +cardiac +1780 +okinawa +proto +commando +compelled +oddly +electrons +##base +replica +thanksgiving +##rist +sheila +deliberate +stafford +tidal +representations +hercules +ou +##path +##iated +kidnapping +lenses +##tling +deficit +samoa +mouths +consuming +computational +maze +granting +smirk +razor +fixture +ideals +inviting +aiden +nominal +##vs +issuing +julio +pitt +ramsey +docks +##oss +exhaust +##owed +bavarian +draped +anterior +mating +ethiopian +explores +noticing +##nton +discarded +convenience +hoffman +endowment +beasts +cartridge +mormon +paternal +probe +sleeves +interfere +lump +deadline +##rail +jenks +bulldogs +scrap +alternating +justified +reproductive +nam +seize +descending +secretariat +kirby +coupe +grouped +smash +panther +sedan +tapping +##18 +lola +cheer +germanic +unfortunate +##eter +unrelated +##fan +subordinate +##sdale +suzanne +advertisement +##ility +horsepower +##lda +cautiously +discourse +luigi +##mans +##fields +noun +prevalent +mao +schneider +everett +surround +governorate +kira +##avia +westward +##take +misty +rails +sustainability +134 +unused +##rating +packs +toast +unwilling +regulate +thy +suffrage +nile +awe +assam +definitions +travelers +affordable +##rb +conferred +sells +undefeated +beneficial +torso +basal +repeating +remixes +##pass +bahrain +cables +fang +##itated +excavated +numbering +statutory +##rey +deluxe +##lian +forested +ramirez +derbyshire +zeus +slamming +transfers +astronomer +banana +lottery +berg +histories +bamboo +##uchi +resurrection +posterior +bowls +vaguely +##thi +thou +preserving +tensed +offence +##inas +meyrick +callum +ridden +watt +langdon +tying +lowland +snorted +daring +truman +##hale +##girl +aura +overly +filing +weighing +goa +infections +philanthropist +saunders +eponymous +##owski +latitude +perspectives +reviewing +mets +commandant +radial +##kha +flashlight +reliability +koch +vowels +amazed +ada +elaine +supper +##rth +##encies +predator +debated +soviets +cola +##boards +##nah +compartment +crooked +arbitrary +fourteenth +##ctive +havana +majors +steelers +clips +profitable +ambush +exited +packers +##tile +nude +cracks +fungi +##е +limb +trousers +josie +shelby +tens +frederic +##ος +definite +smoothly +constellation +insult +baton +discs +lingering +##nco +conclusions +lent +staging +becker +grandpa +shaky +##tron +einstein +obstacles +sk +adverse +elle +economically +##moto +mccartney +thor +dismissal +motions +readings +nostrils +treatise +##pace +squeezing +evidently +prolonged +1783 +venezuelan +je +marguerite +beirut +takeover +shareholders +##vent +denise +digit +airplay +norse +##bbling +imaginary +pills +hubert +blaze +vacated +eliminating +##ello +vine +mansfield +##tty +retrospective +barrow +borne +clutch +bail +forensic +weaving +##nett +##witz +desktop +citadel +promotions +worrying +dorset +ieee +subdivided +##iating +manned +expeditionary +pickup +synod +chuckle +185 +barney +##rz +##ffin +functionality +karachi +litigation +meanings +uc +lick +turbo +anders +##ffed +execute +curl +oppose +ankles +typhoon +##د +##ache +##asia +linguistics +compassion +pressures +grazing +perfection +##iting +immunity +monopoly +muddy +backgrounds +136 +namibia +francesca +monitors +attracting +stunt +tuition +##ии +vegetable +##mates +##quent +mgm +jen +complexes +forts +##ond +cellar +bites +seventeenth +royals +flemish +failures +mast +charities +##cular +peruvian +capitals +macmillan +ipswich +outward +frigate +postgraduate +folds +employing +##ouse +concurrently +fiery +##tai +contingent +nightmares +monumental +nicaragua +##kowski +lizard +mal +fielding +gig +reject +##pad +harding +##ipe +coastline +##cin +##nos +beethoven +humphrey +innovations +##tam +##nge +norris +doris +solicitor +huang +obey +141 +##lc +niagara +##tton +shelves +aug +bourbon +curry +nightclub +specifications +hilton +##ndo +centennial +dispersed +worm +neglected +briggs +sm +font +kuala +uneasy +plc +##nstein +##bound +##aking +##burgh +awaiting +pronunciation +##bbed +##quest +eh +optimal +zhu +raped +greens +presided +brenda +worries +##life +venetian +marxist +turnout +##lius +refined +braced +sins +grasped +sunderland +nickel +speculated +lowell +cyrillic +communism +fundraising +resembling +colonists +mutant +freddie +usc +##mos +gratitude +##run +mural +##lous +chemist +wi +reminds +28th +steals +tess +pietro +##ingen +promoter +ri +microphone +honoured +rai +sant +##qui +feather +##nson +burlington +kurdish +terrorists +deborah +sickness +##wed +##eet +hazard +irritated +desperation +veil +clarity +##rik +jewels +xv +##gged +##ows +##cup +berkshire +unfair +mysteries +orchid +winced +exhaustion +renovations +stranded +obe +infinity +##nies +adapt +redevelopment +thanked +registry +olga +domingo +noir +tudor +ole +##atus +commenting +behaviors +##ais +crisp +pauline +probable +stirling +wigan +##bian +paralympics +panting +surpassed +##rew +luca +barred +pony +famed +##sters +cassandra +waiter +carolyn +exported +##orted +andres +destructive +deeds +jonah +castles +vacancy +suv +##glass +1788 +orchard +yep +famine +belarusian +sprang +##forth +skinny +##mis +administrators +rotterdam +zambia +zhao +boiler +discoveries +##ride +##physics +lucius +disappointing +outreach +spoon +##frame +qualifications +unanimously +enjoys +regency +##iidae +stade +realism +veterinary +rodgers +dump +alain +chestnut +castile +censorship +rumble +gibbs +##itor +communion +reggae +inactivated +logs +loads +##houses +homosexual +##iano +ale +informs +##cas +phrases +plaster +linebacker +ambrose +kaiser +fascinated +850 +limerick +recruitment +forge +mastered +##nding +leinster +rooted +threaten +##strom +borneo +##hes +suggestions +scholarships +propeller +documentaries +patronage +coats +constructing +invest +neurons +comet +entirety +shouts +identities +annoying +unchanged +wary +##antly +##ogy +neat +oversight +##kos +phillies +replay +constance +##kka +incarnation +humble +skies +minus +##acy +smithsonian +##chel +guerrilla +jar +cadets +##plate +surplus +audit +##aru +cracking +joanna +louisa +pacing +##lights +intentionally +##iri +diner +nwa +imprint +australians +tong +unprecedented +bunker +naive +specialists +ark +nichols +railing +leaked +pedal +##uka +shrub +longing +roofs +v8 +captains +neural +tuned +##ntal +##jet +emission +medina +frantic +codex +definitive +sid +abolition +intensified +stocks +enrique +sustain +genoa +oxide +##written +clues +cha +##gers +tributaries +fragment +venom +##rity +##ente +##sca +muffled +vain +sire +laos +##ingly +##hana +hastily +snapping +surfaced +sentiment +motive +##oft +contests +approximate +mesa +luckily +dinosaur +exchanges +propelled +accord +bourne +relieve +tow +masks +offended +##ues +cynthia +##mmer +rains +bartender +zinc +reviewers +lois +##sai +legged +arrogant +rafe +rosie +comprise +handicap +blockade +inlet +lagoon +copied +drilling +shelley +petals +##inian +mandarin +obsolete +##inated +onward +arguably +productivity +cindy +praising +seldom +busch +discusses +raleigh +shortage +ranged +stanton +encouragement +firstly +conceded +overs +temporal +##uke +cbe +##bos +woo +certainty +pumps +##pton +stalked +##uli +lizzie +periodic +thieves +weaker +##night +gases +shoving +chooses +wc +##chemical +prompting +weights +##kill +robust +flanked +sticky +hu +tuberculosis +##eb +##eal +christchurch +resembled +wallet +reese +inappropriate +pictured +distract +fixing +fiddle +giggled +burger +heirs +hairy +mechanic +torque +apache +obsessed +chiefly +cheng +logging +##tag +extracted +meaningful +numb +##vsky +gloucestershire +reminding +##bay +unite +##lit +breeds +diminished +clown +glove +1860s +##ن +##ug +archibald +focal +freelance +sliced +depiction +##yk +organism +switches +sights +stray +crawling +##ril +lever +leningrad +interpretations +loops +anytime +reel +alicia +delighted +##ech +inhaled +xiv +suitcase +bernie +vega +licenses +northampton +exclusion +induction +monasteries +racecourse +homosexuality +##right +##sfield +##rky +dimitri +michele +alternatives +ions +commentators +genuinely +objected +pork +hospitality +fencing +stephan +warships +peripheral +wit +drunken +wrinkled +quentin +spends +departing +chung +numerical +spokesperson +##zone +johannesburg +caliber +killers +##udge +assumes +neatly +demographic +abigail +bloc +##vel +mounting +##lain +bentley +slightest +xu +recipients +##jk +merlin +##writer +seniors +prisons +blinking +hindwings +flickered +kappa +##hel +80s +strengthening +appealing +brewing +gypsy +mali +lashes +hulk +unpleasant +harassment +bio +treaties +predict +instrumentation +pulp +troupe +boiling +mantle +##ffe +ins +##vn +dividing +handles +verbs +##onal +coconut +senegal +340 +thorough +gum +momentarily +##sto +cocaine +panicked +destined +##turing +teatro +denying +weary +captained +mans +##hawks +##code +wakefield +bollywood +thankfully +##16 +cyril +##wu +amendments +##bahn +consultation +stud +reflections +kindness +1787 +internally +##ovo +tex +mosaic +distribute +paddy +seeming +143 +##hic +piers +##15 +##mura +##verse +popularly +winger +kang +sentinel +mccoy +##anza +covenant +##bag +verge +fireworks +suppress +thrilled +dominate +##jar +swansea +##60 +142 +reconciliation +##ndi +stiffened +cue +dorian +##uf +damascus +amor +ida +foremost +##aga +porsche +unseen +dir +##had +##azi +stony +lexi +melodies +##nko +angular +integer +podcast +ants +inherent +jaws +justify +persona +##olved +josephine +##nr +##ressed +customary +flashes +gala +cyrus +glaring +backyard +ariel +physiology +greenland +html +stir +avon +atletico +finch +methodology +ked +##lent +mas +catholicism +townsend +branding +quincy +fits +containers +1777 +ashore +aragon +##19 +forearm +poisoning +##sd +adopting +conquer +grinding +amnesty +keller +finances +evaluate +forged +lankan +instincts +##uto +guam +bosnian +photographed +workplace +desirable +protector +##dog +allocation +intently +encourages +willy +##sten +bodyguard +electro +brighter +##ν +bihar +##chev +lasts +opener +amphibious +sal +verde +arte +##cope +captivity +vocabulary +yields +##tted +agreeing +desmond +pioneered +##chus +strap +campaigned +railroads +##ович +emblem +##dre +stormed +501 +##ulous +marijuana +northumberland +##gn +##nath +bowen +landmarks +beaumont +##qua +danube +##bler +attorneys +th +ge +flyers +critique +villains +cass +mutation +acc +##0s +colombo +mckay +motif +sampling +concluding +syndicate +##rell +neon +stables +ds +warnings +clint +mourning +wilkinson +##tated +merrill +leopard +evenings +exhaled +emil +sonia +ezra +discrete +stove +farrell +fifteenth +prescribed +superhero +##rier +worms +helm +wren +##duction +##hc +expo +##rator +hq +unfamiliar +antony +prevents +acceleration +fiercely +mari +painfully +calculations +cheaper +ign +clifton +irvine +davenport +mozambique +##np +pierced +##evich +wonders +##wig +##cate +##iling +crusade +ware +##uel +enzymes +reasonably +mls +##coe +mater +ambition +bunny +eliot +kernel +##fin +asphalt +headmaster +torah +aden +lush +pins +waived +##care +##yas +joao +substrate +enforce +##grad +##ules +alvarez +selections +epidemic +tempted +##bit +bremen +translates +ensured +waterfront +29th +forrest +manny +malone +kramer +reigning +cookies +simpler +absorption +205 +engraved +##ffy +evaluated +1778 +haze +146 +comforting +crossover +##abe +thorn +##rift +##imo +##pop +suppression +fatigue +cutter +##tr +201 +wurttemberg +##orf +enforced +hovering +proprietary +gb +samurai +syllable +ascent +lacey +tick +lars +tractor +merchandise +rep +bouncing +defendants +##yre +huntington +##ground +##oko +standardized +##hor +##hima +assassinated +nu +predecessors +rainy +liar +assurance +lyrical +##uga +secondly +flattened +ios +parameter +undercover +##mity +bordeaux +punish +ridges +markers +exodus +inactive +hesitate +debbie +nyc +pledge +savoy +nagar +offset +organist +##tium +hesse +marin +converting +##iver +diagram +propulsion +pu +validity +reverted +supportive +##dc +ministries +clans +responds +proclamation +##inae +##ø +##rea +ein +pleading +patriot +sf +birch +islanders +strauss +hates +##dh +brandenburg +concession +rd +##ob +1900s +killings +textbook +antiquity +cinematography +wharf +embarrassing +setup +creed +farmland +inequality +centred +signatures +fallon +370 +##ingham +##uts +ceylon +gazing +directive +laurie +##tern +globally +##uated +##dent +allah +excavation +threads +##cross +148 +frantically +icc +utilize +determines +respiratory +thoughtful +receptions +##dicate +merging +chandra +seine +147 +builders +builds +diagnostic +dev +visibility +goddamn +analyses +dhaka +cho +proves +chancel +concurrent +curiously +canadians +pumped +restoring +1850s +turtles +jaguar +sinister +spinal +traction +declan +vows +1784 +glowed +capitalism +swirling +install +universidad +##lder +##oat +soloist +##genic +##oor +coincidence +beginnings +nissan +dip +resorts +caucasus +combustion +infectious +##eno +pigeon +serpent +##itating +conclude +masked +salad +jew +##gr +surreal +toni +##wc +harmonica +151 +##gins +##etic +##coat +fishermen +intending +bravery +##wave +klaus +titan +wembley +taiwanese +ransom +40th +incorrect +hussein +eyelids +jp +cooke +dramas +utilities +##etta +##print +eisenhower +principally +granada +lana +##rak +openings +concord +##bl +bethany +connie +morality +sega +##mons +##nard +earnings +##kara +##cine +wii +communes +##rel +coma +composing +softened +severed +grapes +##17 +nguyen +analyzed +warlord +hubbard +heavenly +behave +slovenian +##hit +##ony +hailed +filmmakers +trance +caldwell +skye +unrest +coward +likelihood +##aging +bern +sci +taliban +honolulu +propose +##wang +1700 +browser +imagining +cobra +contributes +dukes +instinctively +conan +violinist +##ores +accessories +gradual +##amp +quotes +sioux +##dating +undertake +intercepted +sparkling +compressed +139 +fungus +tombs +haley +imposing +rests +degradation +lincolnshire +retailers +wetlands +tulsa +distributor +dungeon +nun +greenhouse +convey +atlantis +aft +exits +oman +dresser +lyons +##sti +joking +eddy +judgement +omitted +digits +##cts +##game +juniors +##rae +cents +stricken +une +##ngo +wizards +weir +breton +nan +technician +fibers +liking +royalty +##cca +154 +persia +terribly +magician +##rable +##unt +vance +cafeteria +booker +camille +warmer +##static +consume +cavern +gaps +compass +contemporaries +foyer +soothing +graveyard +maj +plunged +blush +##wear +cascade +demonstrates +ordinance +##nov +boyle +##lana +rockefeller +shaken +banjo +izzy +##ense +breathless +vines +##32 +##eman +alterations +chromosome +dwellings +feudal +mole +153 +catalonia +relics +tenant +mandated +##fm +fridge +hats +honesty +patented +raul +heap +cruisers +accusing +enlightenment +infants +wherein +chatham +contractors +zen +affinity +hc +osborne +piston +156 +traps +maturity +##rana +lagos +##zal +peering +##nay +attendant +dealers +protocols +subset +prospects +biographical +##cre +artery +##zers +insignia +nuns +endured +##eration +recommend +schwartz +serbs +berger +cromwell +crossroads +##ctor +enduring +clasped +grounded +##bine +marseille +twitched +abel +choke +https +catalyst +moldova +italians +##tist +disastrous +wee +##oured +##nti +wwf +nope +##piration +##asa +expresses +thumbs +167 +##nza +coca +1781 +cheating +##ption +skipped +sensory +heidelberg +spies +satan +dangers +semifinal +202 +bohemia +whitish +confusing +shipbuilding +relies +surgeons +landings +ravi +baku +moor +suffix +alejandro +##yana +litre +upheld +##unk +rajasthan +##rek +coaster +insists +posture +scenarios +etienne +favoured +appoint +transgender +elephants +poked +greenwood +defences +fulfilled +militant +somali +1758 +chalk +potent +##ucci +migrants +wink +assistants +nos +restriction +activism +niger +##ario +colon +shaun +##sat +daphne +##erated +swam +congregations +reprise +considerations +magnet +playable +xvi +##р +overthrow +tobias +knob +chavez +coding +##mers +propped +katrina +orient +newcomer +##suke +temperate +##pool +farmhouse +interrogation +##vd +committing +##vert +forthcoming +strawberry +joaquin +macau +ponds +shocking +siberia +##cellular +chant +contributors +##nant +##ologists +sped +absorb +hail +1782 +spared +##hore +barbados +karate +opus +originates +saul +##xie +evergreen +leaped +##rock +correlation +exaggerated +weekday +unification +bump +tracing +brig +afb +pathways +utilizing +##ners +mod +mb +disturbance +kneeling +##stad +##guchi +100th +pune +##thy +decreasing +168 +manipulation +miriam +academia +ecosystem +occupational +rbi +##lem +rift +##14 +rotary +stacked +incorporation +awakening +generators +guerrero +racist +##omy +cyber +derivatives +culminated +allie +annals +panzer +sainte +wikipedia +pops +zu +austro +##vate +algerian +politely +nicholson +mornings +educate +tastes +thrill +dartmouth +##gating +db +##jee +regan +differing +concentrating +choreography +divinity +##media +pledged +alexandre +routing +gregor +madeline +##idal +apocalypse +##hora +gunfire +culminating +elves +fined +liang +lam +programmed +tar +guessing +transparency +gabrielle +##gna +cancellation +flexibility +##lining +accession +shea +stronghold +nets +specializes +##rgan +abused +hasan +sgt +ling +exceeding +##₄ +admiration +supermarket +##ark +photographers +specialised +tilt +resonance +hmm +perfume +380 +sami +threatens +garland +botany +guarding +boiled +greet +puppy +russo +supplier +wilmington +vibrant +vijay +##bius +paralympic +grumbled +paige +faa +licking +margins +hurricanes +##gong +fest +grenade +ripping +##uz +counseling +weigh +##sian +needles +wiltshire +edison +costly +##not +fulton +tramway +redesigned +staffordshire +cache +gasping +watkins +sleepy +candidacy +##group +monkeys +timeline +throbbing +##bid +##sos +berth +uzbekistan +vanderbilt +bothering +overturned +ballots +gem +##iger +sunglasses +subscribers +hooker +compelling +ang +exceptionally +saloon +stab +##rdi +carla +terrifying +rom +##vision +coil +##oids +satisfying +vendors +31st +mackay +deities +overlooked +ambient +bahamas +felipe +olympia +whirled +botanist +advertised +tugging +##dden +disciples +morales +unionist +rites +foley +morse +motives +creepy +##₀ +soo +##sz +bargain +highness +frightening +turnpike +tory +reorganization +##cer +depict +biographer +##walk +unopposed +manifesto +##gles +institut +emile +accidental +kapoor +##dam +kilkenny +cortex +lively +##13 +romanesque +jain +shan +cannons +##ood +##ske +petrol +echoing +amalgamated +disappears +cautious +proposes +sanctions +trenton +##ر +flotilla +aus +contempt +tor +canary +cote +theirs +##hun +conceptual +deleted +fascinating +paso +blazing +elf +honourable +hutchinson +##eiro +##outh +##zin +surveyor +tee +amidst +wooded +reissue +intro +##ono +cobb +shelters +newsletter +hanson +brace +encoding +confiscated +dem +caravan +marino +scroll +melodic +cows +imam +##adi +##aneous +northward +searches +biodiversity +cora +310 +roaring +##bers +connell +theologian +halo +compose +pathetic +unmarried +dynamo +##oot +az +calculation +toulouse +deserves +humour +nr +forgiveness +tam +undergone +martyr +pamela +myths +whore +counselor +hicks +290 +heavens +battleship +electromagnetic +##bbs +stellar +establishments +presley +hopped +##chin +temptation +90s +wills +nas +##yuan +nhs +##nya +seminars +##yev +adaptations +gong +asher +lex +indicator +sikh +tobago +cites +goin +##yte +satirical +##gies +characterised +correspond +bubbles +lure +participates +##vid +eruption +skate +therapeutic +1785 +canals +wholesale +defaulted +sac +460 +petit +##zzled +virgil +leak +ravens +256 +portraying +##yx +ghetto +creators +dams +portray +vicente +##rington +fae +namesake +bounty +##arium +joachim +##ota +##iser +aforementioned +axle +snout +depended +dismantled +reuben +480 +##ibly +gallagher +##lau +##pd +earnest +##ieu +##iary +inflicted +objections +##llar +asa +gritted +##athy +jericho +##sea +##was +flick +underside +ceramics +undead +substituted +195 +eastward +undoubtedly +wheeled +chimney +##iche +guinness +cb +##ager +siding +##bell +traitor +baptiste +disguised +inauguration +149 +tipperary +choreographer +perched +warmed +stationary +eco +##ike +##ntes +bacterial +##aurus +flores +phosphate +##core +attacker +invaders +alvin +intersects +a1 +indirectly +immigrated +businessmen +cornelius +valves +narrated +pill +sober +ul +nationale +monastic +applicants +scenery +##jack +161 +motifs +constitutes +cpu +##osh +jurisdictions +sd +tuning +irritation +woven +##uddin +fertility +gao +##erie +antagonist +impatient +glacial +hides +boarded +denominations +interception +##jas +cookie +nicola +##tee +algebraic +marquess +bahn +parole +buyers +bait +turbines +paperwork +bestowed +natasha +renee +oceans +purchases +157 +vaccine +215 +##tock +fixtures +playhouse +integrate +jai +oswald +intellectuals +##cky +booked +nests +mortimer +##isi +obsession +sept +##gler +##sum +440 +scrutiny +simultaneous +squinted +##shin +collects +oven +shankar +penned +remarkably +##я +slips +luggage +spectral +1786 +collaborations +louie +consolidation +##ailed +##ivating +420 +hoover +blackpool +harness +ignition +vest +tails +belmont +mongol +skinner +##nae +visually +mage +derry +##tism +##unce +stevie +transitional +##rdy +redskins +drying +prep +prospective +##21 +annoyance +oversee +##loaded +fills +##books +##iki +announces +fda +scowled +respects +prasad +mystic +tucson +##vale +revue +springer +bankrupt +1772 +aristotle +salvatore +habsburg +##geny +dal +natal +nut +pod +chewing +darts +moroccan +walkover +rosario +lenin +punjabi +##ße +grossed +scattering +wired +invasive +hui +polynomial +corridors +wakes +gina +portrays +##cratic +arid +retreating +erich +irwin +sniper +##dha +linen +lindsey +maneuver +butch +shutting +socio +bounce +commemorative +postseason +jeremiah +pines +275 +mystical +beads +bp +abbas +furnace +bidding +consulted +assaulted +empirical +rubble +enclosure +sob +weakly +cancel +polly +yielded +##emann +curly +prediction +battered +70s +vhs +jacqueline +render +sails +barked +detailing +grayson +riga +sloane +raging +##yah +herbs +bravo +##athlon +alloy +giggle +imminent +suffers +assumptions +waltz +##itate +accomplishments +##ited +bathing +remixed +deception +prefix +##emia +deepest +##tier +##eis +balkan +frogs +##rong +slab +##pate +philosophers +peterborough +grains +imports +dickinson +rwanda +##atics +1774 +dirk +lan +tablets +##rove +clone +##rice +caretaker +hostilities +mclean +##gre +regimental +treasures +norms +impose +tsar +tango +diplomacy +variously +complain +192 +recognise +arrests +1779 +celestial +pulitzer +##dus +bing +libretto +##moor +adele +splash +##rite +expectation +lds +confronts +##izer +spontaneous +harmful +wedge +entrepreneurs +buyer +##ope +bilingual +translate +rugged +conner +circulated +uae +eaton +##gra +##zzle +lingered +lockheed +vishnu +reelection +alonso +##oom +joints +yankee +headline +cooperate +heinz +laureate +invading +##sford +echoes +scandinavian +##dham +hugging +vitamin +salute +micah +hind +trader +##sper +radioactive +##ndra +militants +poisoned +ratified +remark +campeonato +deprived +wander +prop +##dong +outlook +##tani +##rix +##eye +chiang +darcy +##oping +mandolin +spice +statesman +babylon +182 +walled +forgetting +afro +##cap +158 +giorgio +buffer +##polis +planetary +##gis +overlap +terminals +kinda +centenary +##bir +arising +manipulate +elm +ke +1770 +ak +##tad +chrysler +mapped +moose +pomeranian +quad +macarthur +assemblies +shoreline +recalls +stratford +##rted +noticeable +##evic +imp +##rita +##sque +accustomed +supplying +tents +disgusted +vogue +sipped +filters +khz +reno +selecting +luftwaffe +mcmahon +tyne +masterpiece +carriages +collided +dunes +exercised +flare +remembers +muzzle +##mobile +heck +##rson +burgess +lunged +middleton +boycott +bilateral +##sity +hazardous +lumpur +multiplayer +spotlight +jackets +goldman +liege +porcelain +rag +waterford +benz +attracts +hopeful +battling +ottomans +kensington +baked +hymns +cheyenne +lattice +levine +borrow +polymer +clashes +michaels +monitored +commitments +denounced +##25 +##von +cavity +##oney +hobby +akin +##holders +futures +intricate +cornish +patty +##oned +illegally +dolphin +##lag +barlow +yellowish +maddie +apologized +luton +plagued +##puram +nana +##rds +sway +fanny +łodz +##rino +psi +suspicions +hanged +##eding +initiate +charlton +##por +nak +competent +235 +analytical +annex +wardrobe +reservations +##rma +sect +162 +fairfax +hedge +piled +buckingham +uneven +bauer +simplicity +snyder +interpret +accountability +donors +moderately +byrd +continents +##cite +##max +disciple +hr +jamaican +ping +nominees +##uss +mongolian +diver +attackers +eagerly +ideological +pillows +miracles +apartheid +revolver +sulfur +clinics +moran +163 +##enko +ile +katy +rhetoric +##icated +chronology +recycling +##hrer +elongated +mughal +pascal +profiles +vibration +databases +domination +##fare +##rant +matthias +digest +rehearsal +polling +weiss +initiation +reeves +clinging +flourished +impress +ngo +##hoff +##ume +buckley +symposium +rhythms +weed +emphasize +transforming +##taking +##gence +##yman +accountant +analyze +flicker +foil +priesthood +voluntarily +decreases +##80 +##hya +slater +sv +charting +mcgill +##lde +moreno +##iu +besieged +zur +robes +##phic +admitting +api +deported +turmoil +peyton +earthquakes +##ares +nationalists +beau +clair +brethren +interrupt +welch +curated +galerie +requesting +164 +##ested +impending +steward +viper +##vina +complaining +beautifully +brandy +foam +nl +1660 +##cake +alessandro +punches +laced +explanations +##lim +attribute +clit +reggie +discomfort +##cards +smoothed +whales +##cene +adler +countered +duffy +disciplinary +widening +recipe +reliance +conducts +goats +gradient +preaching +##shaw +matilda +quasi +striped +meridian +cannabis +cordoba +certificates +##agh +##tering +graffiti +hangs +pilgrims +repeats +##ych +revive +urine +etat +##hawk +fueled +belts +fuzzy +susceptible +##hang +mauritius +salle +sincere +beers +hooks +##cki +arbitration +entrusted +advise +sniffed +seminar +junk +donnell +processors +principality +strapped +celia +mendoza +everton +fortunes +prejudice +starving +reassigned +steamer +##lund +tuck +evenly +foreman +##ffen +dans +375 +envisioned +slit +##xy +baseman +liberia +rosemary +##weed +electrified +periodically +potassium +stride +contexts +sperm +slade +mariners +influx +bianca +subcommittee +##rane +spilling +icao +estuary +##nock +delivers +iphone +##ulata +isa +mira +bohemian +dessert +##sbury +welcoming +proudly +slowing +##chs +musee +ascension +russ +##vian +waits +##psy +africans +exploit +##morphic +gov +eccentric +crab +peck +##ull +entrances +formidable +marketplace +groom +bolted +metabolism +patton +robbins +courier +payload +endure +##ifier +andes +refrigerator +##pr +ornate +##uca +ruthless +illegitimate +masonry +strasbourg +bikes +adobe +##³ +apples +quintet +willingly +niche +bakery +corpses +energetic +##cliffe +##sser +##ards +177 +centimeters +centro +fuscous +cretaceous +rancho +##yde +andrei +telecom +tottenham +oasis +ordination +vulnerability +presiding +corey +cp +penguins +sims +##pis +malawi +piss +##48 +correction +##cked +##ffle +##ryn +countdown +detectives +psychiatrist +psychedelic +dinosaurs +blouse +##get +choi +vowed +##oz +randomly +##pol +49ers +scrub +blanche +bruins +dusseldorf +##using +unwanted +##ums +212 +dominique +elevations +headlights +om +laguna +##oga +1750 +famously +ignorance +shrewsbury +##aine +ajax +breuning +che +confederacy +greco +overhaul +##screen +paz +skirts +disagreement +cruelty +jagged +phoebe +shifter +hovered +viruses +##wes +mandy +##lined +##gc +landlord +squirrel +dashed +##ι +ornamental +gag +wally +grange +literal +spurs +undisclosed +proceeding +yin +##text +billie +orphan +spanned +humidity +indy +weighted +presentations +explosions +lucian +##tary +vaughn +hindus +##anga +##hell +psycho +171 +daytona +protects +efficiently +rematch +sly +tandem +##oya +rebranded +impaired +hee +metropolis +peach +godfrey +diaspora +ethnicity +prosperous +gleaming +dar +grossing +playback +##rden +stripe +pistols +##tain +births +labelled +##cating +172 +rudy +alba +##onne +aquarium +hostility +##gb +##tase +shudder +sumatra +hardest +lakers +consonant +creeping +demos +homicide +capsule +zeke +liberties +expulsion +pueblo +##comb +trait +transporting +##ddin +##neck +##yna +depart +gregg +mold +ledge +hangar +oldham +playboy +termination +analysts +gmbh +romero +##itic +insist +cradle +filthy +brightness +slash +shootout +deposed +bordering +##truct +isis +microwave +tumbled +sheltered +cathy +werewolves +messy +andersen +convex +clapped +clinched +satire +wasting +edo +vc +rufus +##jak +mont +##etti +poznan +##keeping +restructuring +transverse +##rland +azerbaijani +slovene +gestures +roommate +choking +shear +##quist +vanguard +oblivious +##hiro +disagreed +baptism +##lich +coliseum +##aceae +salvage +societe +cory +locke +relocation +relying +versailles +ahl +swelling +##elo +cheerful +##word +##edes +gin +sarajevo +obstacle +diverted +##nac +messed +thoroughbred +fluttered +utrecht +chewed +acquaintance +assassins +dispatch +mirza +##wart +nike +salzburg +swell +yen +##gee +idle +ligue +samson +##nds +##igh +playful +spawned +##cise +tease +##case +burgundy +##bot +stirring +skeptical +interceptions +marathi +##dies +bedrooms +aroused +pinch +##lik +preferences +tattoos +buster +digitally +projecting +rust +##ital +kitten +priorities +addison +pseudo +##guard +dusk +icons +sermon +##psis +##iba +bt +##lift +##xt +ju +truce +rink +##dah +##wy +defects +psychiatry +offences +calculate +glucose +##iful +##rized +##unda +francaise +##hari +richest +warwickshire +carly +1763 +purity +redemption +lending +##cious +muse +bruises +cerebral +aero +carving +##name +preface +terminology +invade +monty +##int +anarchist +blurred +##iled +rossi +treats +guts +shu +foothills +ballads +undertaking +premise +cecilia +affiliates +blasted +conditional +wilder +minors +drone +rudolph +buffy +swallowing +horton +attested +##hop +rutherford +howell +primetime +livery +penal +##bis +minimize +hydro +wrecked +wrought +palazzo +##gling +cans +vernacular +friedman +nobleman +shale +walnut +danielle +##ection +##tley +sears +##kumar +chords +lend +flipping +streamed +por +dracula +gallons +sacrifices +gamble +orphanage +##iman +mckenzie +##gible +boxers +daly +##balls +##ان +208 +##ific +##rative +##iq +exploited +slated +##uity +circling +hillary +pinched +goldberg +provost +campaigning +lim +piles +ironically +jong +mohan +successors +usaf +##tem +##ught +autobiographical +haute +preserves +##ending +acquitted +comparisons +203 +hydroelectric +gangs +cypriot +torpedoes +rushes +chrome +derive +bumps +instability +fiat +pets +##mbe +silas +dye +reckless +settler +##itation +info +heats +##writing +176 +canonical +maltese +fins +mushroom +stacy +aspen +avid +##kur +##loading +vickers +gaston +hillside +statutes +wilde +gail +kung +sabine +comfortably +motorcycles +##rgo +169 +pneumonia +fetch +##sonic +axel +faintly +parallels +##oop +mclaren +spouse +compton +interdisciplinary +miner +##eni +181 +clamped +##chal +##llah +separates +versa +##mler +scarborough +labrador +##lity +##osing +rutgers +hurdles +como +166 +burt +divers +##100 +wichita +cade +coincided +##erson +bruised +mla +##pper +vineyard +##ili +##brush +notch +mentioning +jase +hearted +kits +doe +##acle +pomerania +##ady +ronan +seizure +pavel +problematic +##zaki +domenico +##ulin +catering +penelope +dependence +parental +emilio +ministerial +atkinson +##bolic +clarkson +chargers +colby +grill +peeked +arises +summon +##aged +fools +##grapher +faculties +qaeda +##vial +garner +refurbished +##hwa +geelong +disasters +nudged +bs +shareholder +lori +algae +reinstated +rot +##ades +##nous +invites +stainless +183 +inclusive +##itude +diocesan +til +##icz +denomination +##xa +benton +floral +registers +##ider +##erman +##kell +absurd +brunei +guangzhou +hitter +retaliation +##uled +##eve +blanc +nh +consistency +contamination +##eres +##rner +dire +palermo +broadcasters +diaries +inspire +vols +brewer +tightening +ky +mixtape +hormone +##tok +stokes +##color +##dly +##ssi +pg +##ometer +##lington +sanitation +##tility +intercontinental +apps +##adt +¹⁄₂ +cylinders +economies +favourable +unison +croix +gertrude +odyssey +vanity +dangling +##logists +upgrades +dice +middleweight +practitioner +##ight +206 +henrik +parlor +orion +angered +lac +python +blurted +##rri +sensual +intends +swings +angled +##phs +husky +attain +peerage +precinct +textiles +cheltenham +shuffled +dai +confess +tasting +bhutan +##riation +tyrone +segregation +abrupt +ruiz +##rish +smirked +blackwell +confidential +browning +amounted +##put +vase +scarce +fabulous +raided +staple +guyana +unemployed +glider +shay +##tow +carmine +troll +intervene +squash +superstar +##uce +cylindrical +len +roadway +researched +handy +##rium +##jana +meta +lao +declares +##rring +##tadt +##elin +##kova +willem +shrubs +napoleonic +realms +skater +qi +volkswagen +##ł +tad +hara +archaeologist +awkwardly +eerie +##kind +wiley +##heimer +##24 +titus +organizers +cfl +crusaders +lama +usb +vent +enraged +thankful +occupants +maximilian +##gaard +possessing +textbooks +##oran +collaborator +quaker +##ulo +avalanche +mono +silky +straits +isaiah +mustang +surged +resolutions +potomac +descend +cl +kilograms +plato +strains +saturdays +##olin +bernstein +##ype +holstein +ponytail +##watch +belize +conversely +heroine +perpetual +##ylus +charcoal +piedmont +glee +negotiating +backdrop +prologue +##jah +##mmy +pasadena +climbs +ramos +sunni +##holm +##tner +##tri +anand +deficiency +hertfordshire +stout +##avi +aperture +orioles +##irs +doncaster +intrigued +bombed +coating +otis +##mat +cocktail +##jit +##eto +amir +arousal +sar +##proof +##act +##ories +dixie +pots +##bow +whereabouts +159 +##fted +drains +bullying +cottages +scripture +coherent +fore +poe +appetite +##uration +sampled +##ators +##dp +derrick +rotor +jays +peacock +installment +##rro +advisors +##coming +rodeo +scotch +##mot +##db +##fen +##vant +ensued +rodrigo +dictatorship +martyrs +twenties +##н +towed +incidence +marta +rainforest +sai +scaled +##cles +oceanic +qualifiers +symphonic +mcbride +dislike +generalized +aubrey +colonization +##iation +##lion +##ssing +disliked +lublin +salesman +##ulates +spherical +whatsoever +sweating +avalon +contention +punt +severity +alderman +atari +##dina +##grant +##rop +scarf +seville +vertices +annexation +fairfield +fascination +inspiring +launches +palatinate +regretted +##rca +feral +##iom +elk +nap +olsen +reddy +yong +##leader +##iae +garment +transports +feng +gracie +outrage +viceroy +insides +##esis +breakup +grady +organizer +softer +grimaced +222 +murals +galicia +arranging +vectors +##rsten +bas +##sb +##cens +sloan +##eka +bitten +ara +fender +nausea +bumped +kris +banquet +comrades +detector +persisted +##llan +adjustment +endowed +cinemas +##shot +sellers +##uman +peek +epa +kindly +neglect +simpsons +talon +mausoleum +runaway +hangul +lookout +##cic +rewards +coughed +acquainted +chloride +##ald +quicker +accordion +neolithic +##qa +artemis +coefficient +lenny +pandora +tx +##xed +ecstasy +litter +segunda +chairperson +gemma +hiss +rumor +vow +nasal +antioch +compensate +patiently +transformers +##eded +judo +morrow +penis +posthumous +philips +bandits +husbands +denote +flaming +##any +##phones +langley +yorker +1760 +walters +##uo +##kle +gubernatorial +fatty +samsung +leroy +outlaw +##nine +unpublished +poole +jakob +##ᵢ +##ₙ +crete +distorted +superiority +##dhi +intercept +crust +mig +claus +crashes +positioning +188 +stallion +301 +frontal +armistice +##estinal +elton +aj +encompassing +camel +commemorated +malaria +woodward +calf +cigar +penetrate +##oso +willard +##rno +##uche +illustrate +amusing +convergence +noteworthy +##lma +##rva +journeys +realise +manfred +##sable +410 +##vocation +hearings +fiance +##posed +educators +provoked +adjusting +##cturing +modular +stockton +paterson +vlad +rejects +electors +selena +maureen +##tres +uber +##rce +swirled +##num +proportions +nanny +pawn +naturalist +parma +apostles +awoke +ethel +wen +##bey +monsoon +overview +##inating +mccain +rendition +risky +adorned +##ih +equestrian +germain +nj +conspicuous +confirming +##yoshi +shivering +##imeter +milestone +rumours +flinched +bounds +smacked +token +##bei +lectured +automobiles +##shore +impacted +##iable +nouns +nero +##leaf +ismail +prostitute +trams +##lace +bridget +sud +stimulus +impressions +reins +revolves +##oud +##gned +giro +honeymoon +##swell +criterion +##sms +##uil +libyan +prefers +##osition +211 +preview +sucks +accusation +bursts +metaphor +diffusion +tolerate +faye +betting +cinematographer +liturgical +specials +bitterly +humboldt +##ckle +flux +rattled +##itzer +archaeologists +odor +authorised +marshes +discretion +##ов +alarmed +archaic +inverse +##leton +explorers +##pine +drummond +tsunami +woodlands +##minate +##tland +booklet +insanity +owning +insert +crafted +calculus +##tore +receivers +##bt +stung +##eca +##nched +prevailing +travellers +eyeing +lila +graphs +##borne +178 +julien +##won +morale +adaptive +therapist +erica +cw +libertarian +bowman +pitches +vita +##ional +crook +##ads +##entation +caledonia +mutiny +##sible +1840s +automation +##ß +flock +##pia +ironic +pathology +##imus +remarried +##22 +joker +withstand +energies +##att +shropshire +hostages +madeleine +tentatively +conflicting +mateo +recipes +euros +ol +mercenaries +nico +##ndon +albuquerque +augmented +mythical +bel +freud +##child +cough +##lica +365 +freddy +lillian +genetically +nuremberg +calder +209 +bonn +outdoors +paste +suns +urgency +vin +restraint +tyson +##cera +##selle +barrage +bethlehem +kahn +##par +mounts +nippon +barony +happier +ryu +makeshift +sheldon +blushed +castillo +barking +listener +taped +bethel +fluent +headlines +pornography +rum +disclosure +sighing +mace +doubling +gunther +manly +##plex +rt +interventions +physiological +forwards +emerges +##tooth +##gny +compliment +rib +recession +visibly +barge +faults +connector +exquisite +prefect +##rlin +patio +##cured +elevators +brandt +italics +pena +173 +wasp +satin +ea +botswana +graceful +respectable +##jima +##rter +##oic +franciscan +generates +##dl +alfredo +disgusting +##olate +##iously +sherwood +warns +cod +promo +cheryl +sino +##ة +##escu +twitch +##zhi +brownish +thom +ortiz +##dron +densely +##beat +carmel +reinforce +##bana +187 +anastasia +downhill +vertex +contaminated +remembrance +harmonic +homework +##sol +fiancee +gears +olds +angelica +loft +ramsay +quiz +colliery +sevens +##cape +autism +##hil +walkway +##boats +ruben +abnormal +ounce +khmer +##bbe +zachary +bedside +morphology +punching +##olar +sparrow +convinces +##35 +hewitt +queer +remastered +rods +mabel +solemn +notified +lyricist +symmetric +##xide +174 +encore +passports +wildcats +##uni +baja +##pac +mildly +##ease +bleed +commodity +mounds +glossy +orchestras +##omo +damian +prelude +ambitions +##vet +awhile +remotely +##aud +asserts +imply +##iques +distinctly +modelling +remedy +##dded +windshield +dani +xiao +##endra +audible +powerplant +1300 +invalid +elemental +acquisitions +##hala +immaculate +libby +plata +smuggling +ventilation +denoted +minh +##morphism +430 +differed +dion +kelley +lore +mocking +sabbath +spikes +hygiene +drown +runoff +stylized +tally +liberated +aux +interpreter +righteous +aba +siren +reaper +pearce +millie +##cier +##yra +gaius +##iso +captures +##ttering +dorm +claudio +##sic +benches +knighted +blackness +##ored +discount +fumble +oxidation +routed +##ς +novak +perpendicular +spoiled +fracture +splits +##urt +pads +topology +##cats +axes +fortunate +offenders +protestants +esteem +221 +broadband +convened +frankly +hound +prototypes +isil +facilitated +keel +##sher +sahara +awaited +bubba +orb +prosecutors +186 +hem +520 +##xing +relaxing +remnant +romney +sorted +slalom +stefano +ulrich +##active +exemption +folder +pauses +foliage +hitchcock +epithet +204 +criticisms +##aca +ballistic +brody +hinduism +chaotic +youths +equals +##pala +pts +thicker +analogous +capitalist +improvised +overseeing +sinatra +ascended +beverage +##tl +straightforward +##kon +curran +##west +bois +325 +induce +surveying +emperors +sax +unpopular +##kk +cartoonist +fused +##mble +unto +##yuki +localities +##cko +##ln +darlington +slain +academie +lobbying +sediment +puzzles +##grass +defiance +dickens +manifest +tongues +alumnus +arbor +coincide +184 +appalachian +mustafa +examiner +cabaret +traumatic +yves +bracelet +draining +heroin +magnum +baths +odessa +consonants +mitsubishi +##gua +kellan +vaudeville +##fr +joked +null +straps +probation +##ław +ceded +interfaces +##pas +##zawa +blinding +viet +224 +rothschild +museo +640 +huddersfield +##vr +tactic +##storm +brackets +dazed +incorrectly +##vu +reg +glazed +fearful +manifold +benefited +irony +##sun +stumbling +##rte +willingness +balkans +mei +wraps +##aba +injected +##lea +gu +syed +harmless +##hammer +bray +takeoff +poppy +timor +cardboard +astronaut +purdue +weeping +southbound +cursing +stalls +diagonal +##neer +lamar +bryce +comte +weekdays +harrington +##uba +negatively +##see +lays +grouping +##cken +##henko +affirmed +halle +modernist +##lai +hodges +smelling +aristocratic +baptized +dismiss +justification +oilers +##now +coupling +qin +snack +healer +##qing +gardener +layla +battled +formulated +stephenson +gravitational +##gill +##jun +1768 +granny +coordinating +suites +##cd +##ioned +monarchs +##cote +##hips +sep +blended +apr +barrister +deposition +fia +mina +policemen +paranoid +##pressed +churchyard +covert +crumpled +creep +abandoning +tr +transmit +conceal +barr +understands +readiness +spire +##cology +##enia +##erry +610 +startling +unlock +vida +bowled +slots +##nat +##islav +spaced +trusting +admire +rig +##ink +slack +##70 +mv +207 +casualty +##wei +classmates +##odes +##rar +##rked +amherst +furnished +evolve +foundry +menace +mead +##lein +flu +wesleyan +##kled +monterey +webber +##vos +wil +##mith +##на +bartholomew +justices +restrained +##cke +amenities +191 +mediated +sewage +trenches +ml +mainz +##thus +1800s +##cula +##inski +caine +bonding +213 +converts +spheres +superseded +marianne +crypt +sweaty +ensign +historia +##br +spruce +##post +##ask +forks +thoughtfully +yukon +pamphlet +ames +##uter +karma +##yya +bryn +negotiation +sighs +incapable +##mbre +##ntial +actresses +taft +##mill +luce +prevailed +##amine +1773 +motionless +envoy +testify +investing +sculpted +instructors +provence +kali +cullen +horseback +##while +goodwin +##jos +gaa +norte +##ldon +modify +wavelength +abd +214 +skinned +sprinter +forecast +scheduling +marries +squared +tentative +##chman +boer +##isch +bolts +swap +fisherman +assyrian +impatiently +guthrie +martins +murdoch +194 +tanya +nicely +dolly +lacy +med +##45 +syn +decks +fashionable +millionaire +##ust +surfing +##ml +##ision +heaved +tammy +consulate +attendees +routinely +197 +fuse +saxophonist +backseat +malaya +##lord +scowl +tau +##ishly +193 +sighted +steaming +##rks +303 +911 +##holes +##hong +ching +##wife +bless +conserved +jurassic +stacey +unix +zion +chunk +rigorous +blaine +198 +peabody +slayer +dismay +brewers +nz +##jer +det +##glia +glover +postwar +int +penetration +sylvester +imitation +vertically +airlift +heiress +knoxville +viva +##uin +390 +macon +##rim +##fighter +##gonal +janice +##orescence +##wari +marius +belongings +leicestershire +196 +blanco +inverted +preseason +sanity +sobbing +##due +##elt +##dled +collingwood +regeneration +flickering +shortest +##mount +##osi +feminism +##lat +sherlock +cabinets +fumbled +northbound +precedent +snaps +##mme +researching +##akes +guillaume +insights +manipulated +vapor +neighbour +sap +gangster +frey +f1 +stalking +scarcely +callie +barnett +tendencies +audi +doomed +assessing +slung +panchayat +ambiguous +bartlett +##etto +distributing +violating +wolverhampton +##hetic +swami +histoire +##urus +liable +pounder +groin +hussain +larsen +popping +surprises +##atter +vie +curt +##station +mute +relocate +musicals +authorization +richter +##sef +immortality +tna +bombings +##press +deteriorated +yiddish +##acious +robbed +colchester +cs +pmid +ao +verified +balancing +apostle +swayed +recognizable +oxfordshire +retention +nottinghamshire +contender +judd +invitational +shrimp +uhf +##icient +cleaner +longitudinal +tanker +##mur +acronym +broker +koppen +sundance +suppliers +##gil +4000 +clipped +fuels +petite +##anne +landslide +helene +diversion +populous +landowners +auspices +melville +quantitative +##xes +ferries +nicky +##llus +doo +haunting +roche +carver +downed +unavailable +##pathy +approximation +hiroshima +##hue +garfield +valle +comparatively +keyboardist +traveler +##eit +congestion +calculating +subsidiaries +##bate +serb +modernization +fairies +deepened +ville +averages +##lore +inflammatory +tonga +##itch +co₂ +squads +##hea +gigantic +serum +enjoyment +retailer +verona +35th +cis +##phobic +magna +technicians +##vati +arithmetic +##sport +levin +##dation +amtrak +chow +sienna +##eyer +backstage +entrepreneurship +##otic +learnt +tao +##udy +worcestershire +formulation +baggage +hesitant +bali +sabotage +##kari +barren +enhancing +murmur +pl +freshly +putnam +syntax +aces +medicines +resentment +bandwidth +##sier +grins +chili +guido +##sei +framing +implying +gareth +lissa +genevieve +pertaining +admissions +geo +thorpe +proliferation +sato +bela +analyzing +parting +##gor +awakened +##isman +huddled +secrecy +##kling +hush +gentry +540 +dungeons +##ego +coasts +##utz +sacrificed +##chule +landowner +mutually +prevalence +programmer +adolescent +disrupted +seaside +gee +trusts +vamp +georgie +##nesian +##iol +schedules +sindh +##market +etched +hm +sparse +bey +beaux +scratching +gliding +unidentified +216 +collaborating +gems +jesuits +oro +accumulation +shaping +mbe +anal +##xin +231 +enthusiasts +newscast +##egan +janata +dewey +parkinson +179 +ankara +biennial +towering +dd +inconsistent +950 +##chet +thriving +terminate +cabins +furiously +eats +advocating +donkey +marley +muster +phyllis +leiden +##user +grassland +glittering +iucn +loneliness +217 +memorandum +armenians +##ddle +popularized +rhodesia +60s +lame +##illon +sans +bikini +header +orbits +##xx +##finger +##ulator +sharif +spines +biotechnology +strolled +naughty +yates +##wire +fremantle +milo +##mour +abducted +removes +##atin +humming +wonderland +##chrome +##ester +hume +pivotal +##rates +armand +grams +believers +elector +rte +apron +bis +scraped +##yria +endorsement +initials +##llation +eps +dotted +hints +buzzing +emigration +nearer +##tom +indicators +##ulu +coarse +neutron +protectorate +##uze +directional +exploits +pains +loire +1830s +proponents +guggenheim +rabbits +ritchie +305 +hectare +inputs +hutton +##raz +verify +##ako +boilers +longitude +##lev +skeletal +yer +emilia +citrus +compromised +##gau +pokemon +prescription +paragraph +eduard +cadillac +attire +categorized +kenyan +weddings +charley +##bourg +entertain +monmouth +##lles +nutrients +davey +mesh +incentive +practised +ecosystems +kemp +subdued +overheard +##rya +bodily +maxim +##nius +apprenticeship +ursula +##fight +lodged +rug +silesian +unconstitutional +patel +inspected +coyote +unbeaten +##hak +34th +disruption +convict +parcel +##cl +##nham +collier +implicated +mallory +##iac +##lab +susannah +winkler +##rber +shia +phelps +sediments +graphical +robotic +##sner +adulthood +mart +smoked +##isto +kathryn +clarified +##aran +divides +convictions +oppression +pausing +burying +##mt +federico +mathias +eileen +##tana +kite +hunched +##acies +189 +##atz +disadvantage +liza +kinetic +greedy +paradox +yokohama +dowager +trunks +ventured +##gement +gupta +vilnius +olaf +##thest +crimean +hopper +##ej +progressively +arturo +mouthed +arrondissement +##fusion +rubin +simulcast +oceania +##orum +##stra +##rred +busiest +intensely +navigator +cary +##vine +##hini +##bies +fife +rowe +rowland +posing +insurgents +shafts +lawsuits +activate +conor +inward +culturally +garlic +265 +##eering +eclectic +##hui +##kee +##nl +furrowed +vargas +meteorological +rendezvous +##aus +culinary +commencement +##dition +quota +##notes +mommy +salaries +overlapping +mule +##iology +##mology +sums +wentworth +##isk +##zione +mainline +subgroup +##illy +hack +plaintiff +verdi +bulb +differentiation +engagements +multinational +supplemented +bertrand +caller +regis +##naire +##sler +##arts +##imated +blossom +propagation +kilometer +viaduct +vineyards +##uate +beckett +optimization +golfer +songwriters +seminal +semitic +thud +volatile +evolving +ridley +##wley +trivial +distributions +scandinavia +jiang +##ject +wrestled +insistence +##dio +emphasizes +napkin +##ods +adjunct +rhyme +##ricted +##eti +hopeless +surrounds +tremble +32nd +smoky +##ntly +oils +medicinal +padded +steer +wilkes +219 +255 +concessions +hue +uniquely +blinded +landon +yahoo +##lane +hendrix +commemorating +dex +specify +chicks +##ggio +intercity +1400 +morley +##torm +highlighting +##oting +pang +oblique +stalled +##liner +flirting +newborn +1769 +bishopric +shaved +232 +currie +##ush +dharma +spartan +##ooped +favorites +smug +novella +sirens +abusive +creations +espana +##lage +paradigm +semiconductor +sheen +##rdo +##yen +##zak +nrl +renew +##pose +##tur +adjutant +marches +norma +##enity +ineffective +weimar +grunt +##gat +lordship +plotting +expenditure +infringement +lbs +refrain +av +mimi +mistakenly +postmaster +1771 +##bara +ras +motorsports +tito +199 +subjective +##zza +bully +stew +##kaya +prescott +1a +##raphic +##zam +bids +styling +paranormal +reeve +sneaking +exploding +katz +akbar +migrant +syllables +indefinitely +##ogical +destroys +replaces +applause +##phine +pest +##fide +218 +articulated +bertie +##thing +##cars +##ptic +courtroom +crowley +aesthetics +cummings +tehsil +hormones +titanic +dangerously +##ibe +stadion +jaenelle +auguste +ciudad +##chu +mysore +partisans +##sio +lucan +philipp +##aly +debating +henley +interiors +##rano +##tious +homecoming +beyonce +usher +henrietta +prepares +weeds +##oman +ely +plucked +##pire +##dable +luxurious +##aq +artifact +password +pasture +juno +maddy +minsk +##dder +##ologies +##rone +assessments +martian +royalist +1765 +examines +##mani +##rge +nino +223 +parry +scooped +relativity +##eli +##uting +##cao +congregational +noisy +traverse +##agawa +strikeouts +nickelodeon +obituary +transylvania +binds +depictions +polk +trolley +##yed +##lard +breeders +##under +dryly +hokkaido +1762 +strengths +stacks +bonaparte +connectivity +neared +prostitutes +stamped +anaheim +gutierrez +sinai +##zzling +bram +fresno +madhya +##86 +proton +##lena +##llum +##phon +reelected +wanda +##anus +##lb +ample +distinguishing +##yler +grasping +sermons +tomato +bland +stimulation +avenues +##eux +spreads +scarlett +fern +pentagon +assert +baird +chesapeake +ir +calmed +distortion +fatalities +##olis +correctional +pricing +##astic +##gina +prom +dammit +ying +collaborate +##chia +welterweight +33rd +pointer +substitution +bonded +umpire +communicating +multitude +paddle +##obe +federally +intimacy +##insky +betray +ssr +##lett +##lean +##lves +##therapy +airbus +##tery +functioned +ud +bearer +biomedical +netflix +##hire +##nca +condom +brink +ik +##nical +macy +##bet +flap +gma +experimented +jelly +lavender +##icles +##ulia +munro +##mian +##tial +rye +##rle +60th +gigs +hottest +rotated +predictions +fuji +bu +##erence +##omi +barangay +##fulness +##sas +clocks +##rwood +##liness +cereal +roe +wight +decker +uttered +babu +onion +xml +forcibly +##df +petra +sarcasm +hartley +peeled +storytelling +##42 +##xley +##ysis +##ffa +fibre +kiel +auditor +fig +harald +greenville +##berries +geographically +nell +quartz +##athic +cemeteries +##lr +crossings +nah +holloway +reptiles +chun +sichuan +snowy +660 +corrections +##ivo +zheng +ambassadors +blacksmith +fielded +fluids +hardcover +turnover +medications +melvin +academies +##erton +ro +roach +absorbing +spaniards +colton +##founded +outsider +espionage +kelsey +245 +edible +##ulf +dora +establishes +##sham +##tries +contracting +##tania +cinematic +costello +nesting +##uron +connolly +duff +##nology +mma +##mata +fergus +sexes +gi +optics +spectator +woodstock +banning +##hee +##fle +differentiate +outfielder +refinery +226 +312 +gerhard +horde +lair +drastically +##udi +landfall +##cheng +motorsport +odi +##achi +predominant +quay +skins +##ental +edna +harshly +complementary +murdering +##aves +wreckage +##90 +ono +outstretched +lennox +munitions +galen +reconcile +470 +scalp +bicycles +gillespie +questionable +rosenberg +guillermo +hostel +jarvis +kabul +volvo +opium +yd +##twined +abuses +decca +outpost +##cino +sensible +neutrality +##64 +ponce +anchorage +atkins +turrets +inadvertently +disagree +libre +vodka +reassuring +weighs +##yal +glide +jumper +ceilings +repertory +outs +stain +##bial +envy +##ucible +smashing +heightened +policing +hyun +mixes +lai +prima +##ples +celeste +##bina +lucrative +intervened +kc +manually +##rned +stature +staffed +bun +bastards +nairobi +priced +##auer +thatcher +##kia +tripped +comune +##ogan +##pled +brasil +incentives +emanuel +hereford +musica +##kim +benedictine +biennale +##lani +eureka +gardiner +rb +knocks +sha +##ael +##elled +##onate +efficacy +ventura +masonic +sanford +maize +leverage +##feit +capacities +santana +##aur +novelty +vanilla +##cter +##tour +benin +##oir +##rain +neptune +drafting +tallinn +##cable +humiliation +##boarding +schleswig +fabian +bernardo +liturgy +spectacle +sweeney +pont +routledge +##tment +cosmos +ut +hilt +sleek +universally +##eville +##gawa +typed +##dry +favors +allegheny +glaciers +##rly +recalling +aziz +##log +parasite +requiem +auf +##berto +##llin +illumination +##breaker +##issa +festivities +bows +govern +vibe +vp +333 +sprawled +larson +pilgrim +bwf +leaping +##rts +##ssel +alexei +greyhound +hoarse +##dler +##oration +seneca +##cule +gaping +##ulously +##pura +cinnamon +##gens +##rricular +craven +fantasies +houghton +engined +reigned +dictator +supervising +##oris +bogota +commentaries +unnatural +fingernails +spirituality +tighten +##tm +canadiens +protesting +intentional +cheers +sparta +##ytic +##iere +##zine +widen +belgarath +controllers +dodd +iaaf +navarre +##ication +defect +squire +steiner +whisky +##mins +560 +inevitably +tome +##gold +chew +##uid +##lid +elastic +##aby +streaked +alliances +jailed +regal +##ined +##phy +czechoslovak +narration +absently +##uld +bluegrass +guangdong +quran +criticizing +hose +hari +##liest +##owa +skier +streaks +deploy +##lom +raft +bose +dialed +huff +##eira +haifa +simplest +bursting +endings +ib +sultanate +##titled +franks +whitman +ensures +sven +##ggs +collaborators +forster +organising +ui +banished +napier +injustice +teller +layered +thump +##otti +roc +battleships +evidenced +fugitive +sadie +robotics +##roud +equatorial +geologist +##iza +yielding +##bron +##sr +internationale +mecca +##diment +sbs +skyline +toad +uploaded +reflective +undrafted +lal +leafs +bayern +##dai +lakshmi +shortlisted +##stick +##wicz +camouflage +donate +af +christi +lau +##acio +disclosed +nemesis +1761 +assemble +straining +northamptonshire +tal +##asi +bernardino +premature +heidi +42nd +coefficients +galactic +reproduce +buzzed +sensations +zionist +monsieur +myrtle +##eme +archery +strangled +musically +viewpoint +antiquities +bei +trailers +seahawks +cured +pee +preferring +tasmanian +lange +sul +##mail +##working +colder +overland +lucivar +massey +gatherings +haitian +##smith +disapproval +flaws +##cco +##enbach +1766 +npr +##icular +boroughs +creole +forums +techno +1755 +dent +abdominal +streetcar +##eson +##stream +procurement +gemini +predictable +##tya +acheron +christoph +feeder +fronts +vendor +bernhard +jammu +tumors +slang +##uber +goaltender +twists +curving +manson +vuelta +mer +peanut +confessions +pouch +unpredictable +allowance +theodor +vascular +##factory +bala +authenticity +metabolic +coughing +nanjing +##cea +pembroke +##bard +splendid +36th +ff +hourly +##ahu +elmer +handel +##ivate +awarding +thrusting +dl +experimentation +##hesion +##46 +caressed +entertained +steak +##rangle +biologist +orphans +baroness +oyster +stepfather +##dridge +mirage +reefs +speeding +##31 +barons +1764 +227 +inhabit +preached +repealed +##tral +honoring +boogie +captives +administer +johanna +##imate +gel +suspiciously +1767 +sobs +##dington +backbone +hayward +garry +##folding +##nesia +maxi +##oof +##ppe +ellison +galileo +##stand +crimea +frenzy +amour +bumper +matrices +natalia +baking +garth +palestinians +##grove +smack +conveyed +ensembles +gardening +##manship +##rup +##stituting +1640 +harvesting +topography +jing +shifters +dormitory +##carriage +##lston +ist +skulls +##stadt +dolores +jewellery +sarawak +##wai +##zier +fences +christy +confinement +tumbling +credibility +fir +stench +##bria +##plication +##nged +##sam +virtues +##belt +marjorie +pba +##eem +##made +celebrates +schooner +agitated +barley +fulfilling +anthropologist +##pro +restrict +novi +regulating +##nent +padres +##rani +##hesive +loyola +tabitha +milky +olson +proprietor +crambidae +guarantees +intercollegiate +ljubljana +hilda +##sko +ignorant +hooded +##lts +sardinia +##lidae +##vation +frontman +privileged +witchcraft +##gp +jammed +laude +poking +##than +bracket +amazement +yunnan +##erus +maharaja +linnaeus +264 +commissioning +milano +peacefully +##logies +akira +rani +regulator +##36 +grasses +##rance +luzon +crows +compiler +gretchen +seaman +edouard +tab +buccaneers +ellington +hamlets +whig +socialists +##anto +directorial +easton +mythological +##kr +##vary +rhineland +semantic +taut +dune +inventions +succeeds +##iter +replication +branched +##pired +jul +prosecuted +kangaroo +penetrated +##avian +middlesbrough +doses +bleak +madam +predatory +relentless +##vili +reluctance +##vir +hailey +crore +silvery +1759 +monstrous +swimmers +transmissions +hawthorn +informing +##eral +toilets +caracas +crouch +kb +##sett +295 +cartel +hadley +##aling +alexia +yvonne +##biology +cinderella +eton +superb +blizzard +stabbing +industrialist +maximus +##gm +##orus +groves +maud +clade +oversized +comedic +##bella +rosen +nomadic +fulham +montane +beverages +galaxies +redundant +swarm +##rot +##folia +##llis +buckinghamshire +fen +bearings +bahadur +##rom +gilles +phased +dynamite +faber +benoit +vip +##ount +##wd +booking +fractured +tailored +anya +spices +westwood +cairns +auditions +inflammation +steamed +##rocity +##acion +##urne +skyla +thereof +watford +torment +archdeacon +transforms +lulu +demeanor +fucked +serge +##sor +mckenna +minas +entertainer +##icide +caress +originate +residue +##sty +1740 +##ilised +##org +beech +##wana +subsidies +##ghton +emptied +gladstone +ru +firefighters +voodoo +##rcle +het +nightingale +tamara +edmond +ingredient +weaknesses +silhouette +285 +compatibility +withdrawing +hampson +##mona +anguish +giggling +##mber +bookstore +##jiang +southernmost +tilting +##vance +bai +economical +rf +briefcase +dreadful +hinted +projections +shattering +totaling +##rogate +analogue +indicted +periodical +fullback +##dman +haynes +##tenberg +##ffs +##ishment +1745 +thirst +stumble +penang +vigorous +##ddling +##kor +##lium +octave +##ove +##enstein +##inen +##ones +siberian +##uti +cbn +repeal +swaying +##vington +khalid +tanaka +unicorn +otago +plastered +lobe +riddle +##rella +perch +##ishing +croydon +filtered +graeme +tripoli +##ossa +crocodile +##chers +sufi +mined +##tung +inferno +lsu +##phi +swelled +utilizes +£2 +cale +periodicals +styx +hike +informally +coop +lund +##tidae +ala +hen +qui +transformations +disposed +sheath +chickens +##cade +fitzroy +sas +silesia +unacceptable +odisha +1650 +sabrina +pe +spokane +ratios +athena +massage +shen +dilemma +##drum +##riz +##hul +corona +doubtful +niall +##pha +##bino +fines +cite +acknowledging +bangor +ballard +bathurst +##resh +huron +mustered +alzheimer +garments +kinase +tyre +warship +##cp +flashback +pulmonary +braun +cheat +kamal +cyclists +constructions +grenades +ndp +traveller +excuses +stomped +signalling +trimmed +futsal +mosques +relevance +##wine +wta +##23 +##vah +##lter +hoc +##riding +optimistic +##´s +deco +sim +interacting +rejecting +moniker +waterways +##ieri +##oku +mayors +gdansk +outnumbered +pearls +##ended +##hampton +fairs +totals +dominating +262 +notions +stairway +compiling +pursed +commodities +grease +yeast +##jong +carthage +griffiths +residual +amc +contraction +laird +sapphire +##marine +##ivated +amalgamation +dissolve +inclination +lyle +packaged +altitudes +suez +canons +graded +lurched +narrowing +boasts +guise +wed +enrico +##ovsky +rower +scarred +bree +cub +iberian +protagonists +bargaining +proposing +trainers +voyages +vans +fishes +##aea +##ivist +##verance +encryption +artworks +kazan +sabre +cleopatra +hepburn +rotting +supremacy +mecklenburg +##brate +burrows +hazards +outgoing +flair +organizes +##ctions +scorpion +##usions +boo +234 +chevalier +dunedin +slapping +##34 +ineligible +pensions +##38 +##omic +manufactures +emails +bismarck +238 +weakening +blackish +ding +mcgee +quo +##rling +northernmost +xx +manpower +greed +sampson +clicking +##ange +##horpe +##inations +##roving +torre +##eptive +##moral +symbolism +38th +asshole +meritorious +outfits +splashed +biographies +sprung +astros +##tale +302 +737 +filly +raoul +nw +tokugawa +linden +clubhouse +##apa +tracts +romano +##pio +putin +tags +##note +chained +dickson +gunshot +moe +gunn +rashid +##tails +zipper +##bas +##nea +contrasted +##ply +##udes +plum +pharaoh +##pile +aw +comedies +ingrid +sandwiches +subdivisions +1100 +mariana +nokia +kamen +hz +delaney +veto +herring +##words +possessive +outlines +##roup +siemens +stairwell +rc +gallantry +messiah +palais +yells +233 +zeppelin +##dm +bolivar +##cede +smackdown +mckinley +##mora +##yt +muted +geologic +finely +unitary +avatar +hamas +maynard +rees +bog +contrasting +##rut +liv +chico +disposition +pixel +##erate +becca +dmitry +yeshiva +narratives +##lva +##ulton +mercenary +sharpe +tempered +navigate +stealth +amassed +keynes +##lini +untouched +##rrie +havoc +lithium +##fighting +abyss +graf +southward +wolverine +balloons +implements +ngos +transitions +##icum +ambushed +concacaf +dormant +economists +##dim +costing +csi +rana +universite +boulders +verity +##llon +collin +mellon +misses +cypress +fluorescent +lifeless +spence +##ulla +crewe +shepard +pak +revelations +##م +jolly +gibbons +paw +##dro +##quel +freeing +##test +shack +fries +palatine +##51 +##hiko +accompaniment +cruising +recycled +##aver +erwin +sorting +synthesizers +dyke +realities +sg +strides +enslaved +wetland +##ghan +competence +gunpowder +grassy +maroon +reactors +objection +##oms +carlson +gearbox +macintosh +radios +shelton +##sho +clergyman +prakash +254 +mongols +trophies +oricon +228 +stimuli +twenty20 +cantonese +cortes +mirrored +##saurus +bhp +cristina +melancholy +##lating +enjoyable +nuevo +##wny +downfall +schumacher +##ind +banging +lausanne +rumbled +paramilitary +reflex +ax +amplitude +migratory +##gall +##ups +midi +barnard +lastly +sherry +##hp +##nall +keystone +##kra +carleton +slippery +##53 +coloring +foe +socket +otter +##rgos +mats +##tose +consultants +bafta +bison +topping +##km +490 +primal +abandonment +transplant +atoll +hideous +mort +pained +reproduced +tae +howling +##turn +unlawful +billionaire +hotter +poised +lansing +##chang +dinamo +retro +messing +nfc +domesday +##mina +blitz +timed +##athing +##kley +ascending +gesturing +##izations +signaled +tis +chinatown +mermaid +savanna +jameson +##aint +catalina +##pet +##hers +cochrane +cy +chatting +##kus +alerted +computation +mused +noelle +majestic +mohawk +campo +octagonal +##sant +##hend +241 +aspiring +##mart +comprehend +iona +paralyzed +shimmering +swindon +rhone +##eley +reputed +configurations +pitchfork +agitation +francais +gillian +lipstick +##ilo +outsiders +pontifical +resisting +bitterness +sewer +rockies +##edd +##ucher +misleading +1756 +exiting +galloway +##nging +risked +##heart +246 +commemoration +schultz +##rka +integrating +##rsa +poses +shrieked +##weiler +guineas +gladys +jerking +owls +goldsmith +nightly +penetrating +##unced +lia +##33 +ignited +betsy +##aring +##thorpe +follower +vigorously +##rave +coded +kiran +knit +zoology +tbilisi +##28 +##bered +repository +govt +deciduous +dino +growling +##bba +enhancement +unleashed +chanting +pussy +biochemistry +##eric +kettle +repression +toxicity +nrhp +##arth +##kko +##bush +ernesto +commended +outspoken +242 +mca +parchment +sms +kristen +##aton +bisexual +raked +glamour +navajo +a2 +conditioned +showcased +##hma +spacious +youthful +##esa +usl +appliances +junta +brest +layne +conglomerate +enchanted +chao +loosened +picasso +circulating +inspect +montevideo +##centric +##kti +piazza +spurred +##aith +bari +freedoms +poultry +stamford +lieu +##ect +indigo +sarcastic +bahia +stump +attach +dvds +frankenstein +lille +approx +scriptures +pollen +##script +nmi +overseen +##ivism +tides +proponent +newmarket +inherit +milling +##erland +centralized +##rou +distributors +credentials +drawers +abbreviation +##lco +##xon +downing +uncomfortably +ripe +##oes +erase +franchises +##ever +populace +##bery +##khar +decomposition +pleas +##tet +daryl +sabah +##stle +##wide +fearless +genie +lesions +annette +##ogist +oboe +appendix +nair +dripped +petitioned +maclean +mosquito +parrot +rpg +hampered +1648 +operatic +reservoirs +##tham +irrelevant +jolt +summarized +##fp +medallion +##taff +##− +clawed +harlow +narrower +goddard +marcia +bodied +fremont +suarez +altering +tempest +mussolini +porn +##isms +sweetly +oversees +walkers +solitude +grimly +shrines +hk +ich +supervisors +hostess +dietrich +legitimacy +brushes +expressive +##yp +dissipated +##rse +localized +systemic +##nikov +gettysburg +##js +##uaries +dialogues +muttering +251 +housekeeper +sicilian +discouraged +##frey +beamed +kaladin +halftime +kidnap +##amo +##llet +1754 +synonymous +depleted +instituto +insulin +reprised +##opsis +clashed +##ctric +interrupting +radcliffe +insisting +medici +1715 +ejected +playfully +turbulent +##47 +starvation +##rini +shipment +rebellious +petersen +verification +merits +##rified +cakes +##charged +1757 +milford +shortages +spying +fidelity +##aker +emitted +storylines +harvested +seismic +##iform +cheung +kilda +theoretically +barbie +lynx +##rgy +##tius +goblin +mata +poisonous +##nburg +reactive +residues +obedience +##евич +conjecture +##rac +401 +hating +sixties +kicker +moaning +motown +##bha +emancipation +neoclassical +##hering +consoles +ebert +professorship +##tures +sustaining +assaults +obeyed +affluent +incurred +tornadoes +##eber +##zow +emphasizing +highlanders +cheated +helmets +##ctus +internship +terence +bony +executions +legislators +berries +peninsular +tinged +##aco +1689 +amplifier +corvette +ribbons +lavish +pennant +##lander +worthless +##chfield +##forms +mariano +pyrenees +expenditures +##icides +chesterfield +mandir +tailor +39th +sergey +nestled +willed +aristocracy +devotees +goodnight +raaf +rumored +weaponry +remy +appropriations +harcourt +burr +riaa +##lence +limitation +unnoticed +guo +soaking +swamps +##tica +collapsing +tatiana +descriptive +brigham +psalm +##chment +maddox +##lization +patti +caliph +##aja +akron +injuring +serra +##ganj +basins +##sari +astonished +launcher +##church +hilary +wilkins +sewing +##sf +stinging +##fia +##ncia +underwood +startup +##ition +compilations +vibrations +embankment +jurist +##nity +bard +juventus +groundwater +kern +palaces +helium +boca +cramped +marissa +soto +##worm +jae +princely +##ggy +faso +bazaar +warmly +##voking +229 +pairing +##lite +##grate +##nets +wien +freaked +ulysses +rebirth +##alia +##rent +mummy +guzman +jimenez +stilled +##nitz +trajectory +tha +woken +archival +professions +##pts +##pta +hilly +shadowy +shrink +##bolt +norwood +glued +migrate +stereotypes +devoid +##pheus +625 +evacuate +horrors +infancy +gotham +knowles +optic +downloaded +sachs +kingsley +parramatta +darryl +mor +##onale +shady +commence +confesses +kan +##meter +##placed +marlborough +roundabout +regents +frigates +io +##imating +gothenburg +revoked +carvings +clockwise +convertible +intruder +##sche +banged +##ogo +vicky +bourgeois +##mony +dupont +footing +##gum +pd +##real +buckle +yun +penthouse +sane +720 +serviced +stakeholders +neumann +bb +##eers +comb +##gam +catchment +pinning +rallies +typing +##elles +forefront +freiburg +sweetie +giacomo +widowed +goodwill +worshipped +aspirations +midday +##vat +fishery +##trick +bournemouth +turk +243 +hearth +ethanol +guadalajara +murmurs +sl +##uge +afforded +scripted +##hta +wah +##jn +coroner +translucent +252 +memorials +puck +progresses +clumsy +##race +315 +candace +recounted +##27 +##slin +##uve +filtering +##mac +howl +strata +heron +leveled +##ays +dubious +##oja +##т +##wheel +citations +exhibiting +##laya +##mics +##pods +turkic +##lberg +injunction +##ennial +##mit +antibodies +##44 +organise +##rigues +cardiovascular +cushion +inverness +##zquez +dia +cocoa +sibling +##tman +##roid +expanse +feasible +tunisian +algiers +##relli +rus +bloomberg +dso +westphalia +bro +tacoma +281 +downloads +##ours +konrad +duran +##hdi +continuum +jett +compares +legislator +secession +##nable +##gues +##zuka +translating +reacher +##gley +##ła +aleppo +##agi +tc +orchards +trapping +linguist +versatile +drumming +postage +calhoun +superiors +##mx +barefoot +leary +##cis +ignacio +alfa +kaplan +##rogen +bratislava +mori +##vot +disturb +haas +313 +cartridges +gilmore +radiated +salford +tunic +hades +##ulsive +archeological +delilah +magistrates +auditioned +brewster +charters +empowerment +blogs +cappella +dynasties +iroquois +whipping +##krishna +raceway +truths +myra +weaken +judah +mcgregor +##horse +mic +refueling +37th +burnley +bosses +markus +premio +query +##gga +dunbar +##economic +darkest +lyndon +sealing +commendation +reappeared +##mun +addicted +ezio +slaughtered +satisfactory +shuffle +##eves +##thic +##uj +fortification +warrington +##otto +resurrected +fargo +mane +##utable +##lei +##space +foreword +ox +##aris +##vern +abrams +hua +##mento +sakura +##alo +uv +sentimental +##skaya +midfield +##eses +sturdy +scrolls +macleod +##kyu +entropy +##lance +mitochondrial +cicero +excelled +thinner +convoys +perceive +##oslav +##urable +systematically +grind +burkina +287 +##tagram +ops +##aman +guantanamo +##cloth +##tite +forcefully +wavy +##jou +pointless +##linger +##tze +layton +portico +superficial +clerical +outlaws +##hism +burials +muir +##inn +creditors +hauling +rattle +##leg +calais +monde +archers +reclaimed +dwell +wexford +hellenic +falsely +remorse +##tek +dough +furnishings +##uttered +gabon +neurological +novice +##igraphy +contemplated +pulpit +nightstand +saratoga +##istan +documenting +pulsing +taluk +##firmed +busted +marital +##rien +disagreements +wasps +##yes +hodge +mcdonnell +mimic +fran +pendant +dhabi +musa +##nington +congratulations +argent +darrell +concussion +losers +regrets +thessaloniki +reversal +donaldson +hardwood +thence +achilles +ritter +##eran +demonic +jurgen +prophets +goethe +eki +classmate +buff +##cking +yank +irrational +##inging +perished +seductive +qur +sourced +##crat +##typic +mustard +ravine +barre +horizontally +characterization +phylogenetic +boise +##dit +##runner +##tower +brutally +intercourse +seduce +##bbing +fay +ferris +ogden +amar +nik +unarmed +##inator +evaluating +kyrgyzstan +sweetness +##lford +##oki +mccormick +meiji +notoriety +stimulate +disrupt +figuring +instructional +mcgrath +##zoo +groundbreaking +##lto +flinch +khorasan +agrarian +bengals +mixer +radiating +##sov +ingram +pitchers +nad +tariff +##cript +tata +##codes +##emi +##ungen +appellate +lehigh +##bled +##giri +brawl +duct +texans +##ciation +##ropolis +skipper +speculative +vomit +doctrines +stresses +253 +davy +graders +whitehead +jozef +timely +cumulative +haryana +paints +appropriately +boon +cactus +##ales +##pid +dow +legions +##pit +perceptions +1730 +picturesque +##yse +periphery +rune +wr +##aha +celtics +sentencing +whoa +##erin +confirms +variance +425 +moines +mathews +spade +rave +m1 +fronted +fx +blending +alleging +reared +##gl +237 +##paper +grassroots +eroded +##free +##physical +directs +ordeal +##sław +accelerate +hacker +rooftop +##inia +lev +buys +cebu +devote +##lce +specialising +##ulsion +choreographed +repetition +warehouses +##ryl +paisley +tuscany +analogy +sorcerer +hash +huts +shards +descends +exclude +nix +chaplin +gaga +ito +vane +##drich +causeway +misconduct +limo +orchestrated +glands +jana +##kot +u2 +##mple +##sons +branching +contrasts +scoop +longed +##virus +chattanooga +##75 +syrup +cornerstone +##tized +##mind +##iaceae +careless +precedence +frescoes +##uet +chilled +consult +modelled +snatch +peat +##thermal +caucasian +humane +relaxation +spins +temperance +##lbert +occupations +lambda +hybrids +moons +mp3 +##oese +247 +rolf +societal +yerevan +ness +##ssler +befriended +mechanized +nominate +trough +boasted +cues +seater +##hom +bends +##tangle +conductors +emptiness +##lmer +eurasian +adriatic +tian +##cie +anxiously +lark +propellers +chichester +jock +ev +2a +##holding +credible +recounts +tori +loyalist +abduction +##hoot +##redo +nepali +##mite +ventral +tempting +##ango +##crats +steered +##wice +javelin +dipping +laborers +prentice +looming +titanium +##ː +badges +emir +tensor +##ntation +egyptians +rash +denies +hawthorne +lombard +showers +wehrmacht +dietary +trojan +##reus +welles +executing +horseshoe +lifeboat +##lak +elsa +infirmary +nearing +roberta +boyer +mutter +trillion +joanne +##fine +##oked +sinks +vortex +uruguayan +clasp +sirius +##block +accelerator +prohibit +sunken +byu +chronological +diplomats +ochreous +510 +symmetrical +1644 +maia +##tology +salts +reigns +atrocities +##ия +hess +bared +issn +##vyn +cater +saturated +##cycle +##isse +sable +voyager +dyer +yusuf +##inge +fountains +wolff +##39 +##nni +engraving +rollins +atheist +ominous +##ault +herr +chariot +martina +strung +##fell +##farlane +horrific +sahib +gazes +saetan +erased +ptolemy +##olic +flushing +lauderdale +analytic +##ices +530 +navarro +beak +gorilla +herrera +broom +guadalupe +raiding +sykes +311 +bsc +deliveries +1720 +invasions +carmichael +tajikistan +thematic +ecumenical +sentiments +onstage +##rians +##brand +##sume +catastrophic +flanks +molten +##arns +waller +aimee +terminating +##icing +alternately +##oche +nehru +printers +outraged +##eving +empires +template +banners +repetitive +za +##oise +vegetarian +##tell +guiana +opt +cavendish +lucknow +synthesized +##hani +##mada +finalized +##ctable +fictitious +mayoral +unreliable +##enham +embracing +peppers +rbis +##chio +##neo +inhibition +slashed +togo +orderly +embroidered +safari +salty +236 +barron +benito +totaled +##dak +pubs +simulated +caden +devin +tolkien +momma +welding +sesame +##ept +gottingen +hardness +630 +shaman +temeraire +620 +adequately +pediatric +##kit +ck +assertion +radicals +composure +cadence +seafood +beaufort +lazarus +mani +warily +cunning +kurdistan +249 +cantata +##kir +ares +##41 +##clusive +nape +townland +geared +insulted +flutter +boating +violate +draper +dumping +malmo +##hh +##romatic +firearm +alta +bono +obscured +##clave +exceeds +panorama +unbelievable +##train +preschool +##essed +disconnected +installing +rescuing +secretaries +accessibility +##castle +##drive +##ifice +##film +bouts +slug +waterway +mindanao +##buro +##ratic +halves +##ل +calming +liter +maternity +adorable +bragg +electrification +mcc +##dote +roxy +schizophrenia +##body +munoz +kaye +whaling +239 +mil +tingling +tolerant +##ago +unconventional +volcanoes +##finder +deportivo +##llie +robson +kaufman +neuroscience +wai +deportation +masovian +scraping +converse +##bh +hacking +bulge +##oun +administratively +yao +580 +amp +mammoth +booster +claremont +hooper +nomenclature +pursuits +mclaughlin +melinda +##sul +catfish +barclay +substrates +taxa +zee +originals +kimberly +packets +padma +##ality +borrowing +ostensibly +solvent +##bri +##genesis +##mist +lukas +shreveport +veracruz +##ь +##lou +##wives +cheney +tt +anatolia +hobbs +##zyn +cyclic +radiant +alistair +greenish +siena +dat +independents +##bation +conform +pieter +hyper +applicant +bradshaw +spores +telangana +vinci +inexpensive +nuclei +322 +jang +nme +soho +spd +##ign +cradled +receptionist +pow +##43 +##rika +fascism +##ifer +experimenting +##ading +##iec +##region +345 +jocelyn +maris +stair +nocturnal +toro +constabulary +elgin +##kker +msc +##giving +##schen +##rase +doherty +doping +sarcastically +batter +maneuvers +##cano +##apple +##gai +##git +intrinsic +##nst +##stor +1753 +showtime +cafes +gasps +lviv +ushered +##thed +fours +restart +astonishment +transmitting +flyer +shrugs +##sau +intriguing +cones +dictated +mushrooms +medial +##kovsky +##elman +escorting +gaped +##26 +godfather +##door +##sell +djs +recaptured +timetable +vila +1710 +3a +aerodrome +mortals +scientology +##orne +angelina +mag +convection +unpaid +insertion +intermittent +lego +##nated +endeavor +kota +pereira +##lz +304 +bwv +glamorgan +insults +agatha +fey +##cend +fleetwood +mahogany +protruding +steamship +zeta +##arty +mcguire +suspense +##sphere +advising +urges +##wala +hurriedly +meteor +gilded +inline +arroyo +stalker +##oge +excitedly +revered +##cure +earle +introductory +##break +##ilde +mutants +puff +pulses +reinforcement +##haling +curses +lizards +stalk +correlated +##fixed +fallout +macquarie +##unas +bearded +denton +heaving +802 +##ocation +winery +assign +dortmund +##lkirk +everest +invariant +charismatic +susie +##elling +bled +lesley +telegram +sumner +bk +##ogen +##к +wilcox +needy +colbert +duval +##iferous +##mbled +allotted +attends +imperative +##hita +replacements +hawker +##inda +insurgency +##zee +##eke +casts +##yla +680 +ives +transitioned +##pack +##powering +authoritative +baylor +flex +cringed +plaintiffs +woodrow +##skie +drastic +ape +aroma +unfolded +commotion +nt +preoccupied +theta +routines +lasers +privatization +wand +domino +ek +clenching +nsa +strategically +showered +bile +handkerchief +pere +storing +christophe +insulting +316 +nakamura +romani +asiatic +magdalena +palma +cruises +stripping +405 +konstantin +soaring +##berman +colloquially +forerunner +havilland +incarcerated +parasites +sincerity +##utus +disks +plank +saigon +##ining +corbin +homo +ornaments +powerhouse +##tlement +chong +fastened +feasibility +idf +morphological +usable +##nish +##zuki +aqueduct +jaguars +keepers +##flies +aleksandr +faust +assigns +ewing +bacterium +hurled +tricky +hungarians +integers +wallis +321 +yamaha +##isha +hushed +oblivion +aviator +evangelist +friars +##eller +monograph +ode +##nary +airplanes +labourers +charms +##nee +1661 +hagen +tnt +rudder +fiesta +transcript +dorothea +ska +inhibitor +maccabi +retorted +raining +encompassed +clauses +menacing +1642 +lineman +##gist +vamps +##ape +##dick +gloom +##rera +dealings +easing +seekers +##nut +##pment +helens +unmanned +##anu +##isson +basics +##amy +##ckman +adjustments +1688 +brutality +horne +##zell +sui +##55 +##mable +aggregator +##thal +rhino +##drick +##vira +counters +zoom +##01 +##rting +mn +montenegrin +packard +##unciation +##♭ +##kki +reclaim +scholastic +thugs +pulsed +##icia +syriac +quan +saddam +banda +kobe +blaming +buddies +dissent +##lusion +##usia +corbett +jaya +delle +erratic +lexie +##hesis +435 +amiga +hermes +##pressing +##leen +chapels +gospels +jamal +##uating +compute +revolving +warp +##sso +##thes +armory +##eras +##gol +antrim +loki +##kow +##asian +##good +##zano +braid +handwriting +subdistrict +funky +pantheon +##iculate +concurrency +estimation +improper +juliana +##his +newcomers +johnstone +staten +communicated +##oco +##alle +sausage +stormy +##stered +##tters +superfamily +##grade +acidic +collateral +tabloid +##oped +##rza +bladder +austen +##ellant +mcgraw +##hay +hannibal +mein +aquino +lucifer +wo +badger +boar +cher +christensen +greenberg +interruption +##kken +jem +244 +mocked +bottoms +cambridgeshire +##lide +sprawling +##bbly +eastwood +ghent +synth +##buck +advisers +##bah +nominally +hapoel +qu +daggers +estranged +fabricated +towels +vinnie +wcw +misunderstanding +anglia +nothin +unmistakable +##dust +##lova +chilly +marquette +truss +##edge +##erine +reece +##lty +##chemist +##connected +272 +308 +41st +bash +raion +waterfalls +##ump +##main +labyrinth +queue +theorist +##istle +bharatiya +flexed +soundtracks +rooney +leftist +patrolling +wharton +plainly +alleviate +eastman +schuster +topographic +engages +immensely +unbearable +fairchild +1620 +dona +lurking +parisian +oliveira +ia +indictment +hahn +bangladeshi +##aster +vivo +##uming +##ential +antonia +expects +indoors +kildare +harlan +##logue +##ogenic +##sities +forgiven +##wat +childish +tavi +##mide +##orra +plausible +grimm +successively +scooted +##bola +##dget +##rith +spartans +emery +flatly +azure +epilogue +##wark +flourish +##iny +##tracted +##overs +##oshi +bestseller +distressed +receipt +spitting +hermit +topological +##cot +drilled +subunit +francs +##layer +eel +##fk +##itas +octopus +footprint +petitions +ufo +##say +##foil +interfering +leaking +palo +##metry +thistle +valiant +##pic +narayan +mcpherson +##fast +gonzales +##ym +##enne +dustin +novgorod +solos +##zman +doin +##raph +##patient +##meyer +soluble +ashland +cuffs +carole +pendleton +whistling +vassal +##river +deviation +revisited +constituents +rallied +rotate +loomed +##eil +##nting +amateurs +augsburg +auschwitz +crowns +skeletons +##cona +bonnet +257 +dummy +globalization +simeon +sleeper +mandal +differentiated +##crow +##mare +milne +bundled +exasperated +talmud +owes +segregated +##feng +##uary +dentist +piracy +props +##rang +devlin +##torium +malicious +paws +##laid +dependency +##ergy +##fers +##enna +258 +pistons +rourke +jed +grammatical +tres +maha +wig +512 +ghostly +jayne +##achal +##creen +##ilis +##lins +##rence +designate +##with +arrogance +cambodian +clones +showdown +throttle +twain +##ception +lobes +metz +nagoya +335 +braking +##furt +385 +roaming +##minster +amin +crippled +##37 +##llary +indifferent +hoffmann +idols +intimidating +1751 +261 +influenza +memo +onions +1748 +bandage +consciously +##landa +##rage +clandestine +observes +swiped +tangle +##ener +##jected +##trum +##bill +##lta +hugs +congresses +josiah +spirited +##dek +humanist +managerial +filmmaking +inmate +rhymes +debuting +grimsby +ur +##laze +duplicate +vigor +##tf +republished +bolshevik +refurbishment +antibiotics +martini +methane +newscasts +royale +horizons +levant +iain +visas +##ischen +paler +##around +manifestation +snuck +alf +chop +futile +pedestal +rehab +##kat +bmg +kerman +res +fairbanks +jarrett +abstraction +saharan +##zek +1746 +procedural +clearer +kincaid +sash +luciano +##ffey +crunch +helmut +##vara +revolutionaries +##tute +creamy +leach +##mmon +1747 +permitting +nes +plight +wendell +##lese +contra +ts +clancy +ipa +mach +staples +autopsy +disturbances +nueva +karin +pontiac +##uding +proxy +venerable +haunt +leto +bergman +expands +##helm +wal +##pipe +canning +celine +cords +obesity +##enary +intrusion +planner +##phate +reasoned +sequencing +307 +harrow +##chon +##dora +marred +mcintyre +repay +tarzan +darting +248 +harrisburg +margarita +repulsed +##hur +##lding +belinda +hamburger +novo +compliant +runways +bingham +registrar +skyscraper +ic +cuthbert +improvisation +livelihood +##corp +##elial +admiring +##dened +sporadic +believer +casablanca +popcorn +##29 +asha +shovel +##bek +##dice +coiled +tangible +##dez +casper +elsie +resin +tenderness +rectory +##ivision +avail +sonar +##mori +boutique +##dier +guerre +bathed +upbringing +vaulted +sandals +blessings +##naut +##utnant +1680 +306 +foxes +pia +corrosion +hesitantly +confederates +crystalline +footprints +shapiro +tirana +valentin +drones +45th +microscope +shipments +texted +inquisition +wry +guernsey +unauthorized +resigning +760 +ripple +schubert +stu +reassure +felony +##ardo +brittle +koreans +##havan +##ives +dun +implicit +tyres +##aldi +##lth +magnolia +##ehan +##puri +##poulos +aggressively +fei +gr +familiarity +##poo +indicative +##trust +fundamentally +jimmie +overrun +395 +anchors +moans +##opus +britannia +armagh +##ggle +purposely +seizing +##vao +bewildered +mundane +avoidance +cosmopolitan +geometridae +quartermaster +caf +415 +chatter +engulfed +gleam +purge +##icate +juliette +jurisprudence +guerra +revisions +##bn +casimir +brew +##jm +1749 +clapton +cloudy +conde +hermitage +278 +simulations +torches +vincenzo +matteo +##rill +hidalgo +booming +westbound +accomplishment +tentacles +unaffected +##sius +annabelle +flopped +sloping +##litz +dreamer +interceptor +vu +##loh +consecration +copying +messaging +breaker +climates +hospitalized +1752 +torino +afternoons +winfield +witnessing +##teacher +breakers +choirs +sawmill +coldly +##ege +sipping +haste +uninhabited +conical +bibliography +pamphlets +severn +edict +##oca +deux +illnesses +grips +##pl +rehearsals +sis +thinkers +tame +##keepers +1690 +acacia +reformer +##osed +##rys +shuffling +##iring +##shima +eastbound +ionic +rhea +flees +littered +##oum +rocker +vomiting +groaning +champ +overwhelmingly +civilizations +paces +sloop +adoptive +##tish +skaters +##vres +aiding +mango +##joy +nikola +shriek +##ignon +pharmaceuticals +##mg +tuna +calvert +gustavo +stocked +yearbook +##urai +##mana +computed +subsp +riff +hanoi +kelvin +hamid +moors +pastures +summons +jihad +nectar +##ctors +bayou +untitled +pleasing +vastly +republics +intellect +##η +##ulio +##tou +crumbling +stylistic +sb +##ی +consolation +frequented +h₂o +walden +widows +##iens +404 +##ignment +chunks +improves +288 +grit +recited +##dev +snarl +sociological +##arte +##gul +inquired +##held +bruise +clube +consultancy +homogeneous +hornets +multiplication +pasta +prick +savior +##grin +##kou +##phile +yoon +##gara +grimes +vanishing +cheering +reacting +bn +distillery +##quisite +##vity +coe +dockyard +massif +##jord +escorts +voss +##valent +byte +chopped +hawke +illusions +workings +floats +##koto +##vac +kv +annapolis +madden +##onus +alvaro +noctuidae +##cum +##scopic +avenge +steamboat +forte +illustrates +erika +##trip +570 +dew +nationalities +bran +manifested +thirsty +diversified +muscled +reborn +##standing +arson +##lessness +##dran +##logram +##boys +##kushima +##vious +willoughby +##phobia +286 +alsace +dashboard +yuki +##chai +granville +myspace +publicized +tricked +##gang +adjective +##ater +relic +reorganisation +enthusiastically +indications +saxe +##lassified +consolidate +iec +padua +helplessly +ramps +renaming +regulars +pedestrians +accents +convicts +inaccurate +lowers +mana +##pati +barrie +bjp +outta +someplace +berwick +flanking +invoked +marrow +sparsely +excerpts +clothed +rei +##ginal +wept +##straße +##vish +alexa +excel +##ptive +membranes +aquitaine +creeks +cutler +sheppard +implementations +ns +##dur +fragrance +budge +concordia +magnesium +marcelo +##antes +gladly +vibrating +##rral +##ggles +montrose +##omba +lew +seamus +1630 +cocky +##ament +##uen +bjorn +##rrick +fielder +fluttering +##lase +methyl +kimberley +mcdowell +reductions +barbed +##jic +##tonic +aeronautical +condensed +distracting +##promising +huffed +##cala +##sle +claudius +invincible +missy +pious +balthazar +ci +##lang +butte +combo +orson +##dication +myriad +1707 +silenced +##fed +##rh +coco +netball +yourselves +##oza +clarify +heller +peg +durban +etudes +offender +roast +blackmail +curvature +##woods +vile +309 +illicit +suriname +##linson +overture +1685 +bubbling +gymnast +tucking +##mming +##ouin +maldives +##bala +gurney +##dda +##eased +##oides +backside +pinto +jars +racehorse +tending +##rdial +baronetcy +wiener +duly +##rke +barbarian +cupping +flawed +##thesis +bertha +pleistocene +puddle +swearing +##nob +##tically +fleeting +prostate +amulet +educating +##mined +##iti +##tler +75th +jens +respondents +analytics +cavaliers +papacy +raju +##iente +##ulum +##tip +funnel +271 +disneyland +##lley +sociologist +##iam +2500 +faulkner +louvre +menon +##dson +276 +##ower +afterlife +mannheim +peptide +referees +comedians +meaningless +##anger +##laise +fabrics +hurley +renal +sleeps +##bour +##icle +breakout +kristin +roadside +animator +clover +disdain +unsafe +redesign +##urity +firth +barnsley +portage +reset +narrows +268 +commandos +expansive +speechless +tubular +##lux +essendon +eyelashes +smashwords +##yad +##bang +##claim +craved +sprinted +chet +somme +astor +wrocław +orton +266 +bane +##erving +##uing +mischief +##amps +##sund +scaling +terre +##xious +impairment +offenses +undermine +moi +soy +contiguous +arcadia +inuit +seam +##tops +macbeth +rebelled +##icative +##iot +590 +elaborated +frs +uniformed +##dberg +259 +powerless +priscilla +stimulated +980 +qc +arboretum +frustrating +trieste +bullock +##nified +enriched +glistening +intern +##adia +locus +nouvelle +ollie +ike +lash +starboard +ee +tapestry +headlined +hove +rigged +##vite +pollock +##yme +thrive +clustered +cas +roi +gleamed +olympiad +##lino +pressured +regimes +##hosis +##lick +ripley +##ophone +kickoff +gallon +rockwell +##arable +crusader +glue +revolutions +scrambling +1714 +grover +##jure +englishman +aztec +263 +contemplating +coven +ipad +preach +triumphant +tufts +##esian +rotational +##phus +328 +falkland +##brates +strewn +clarissa +rejoin +environmentally +glint +banded +drenched +moat +albanians +johor +rr +maestro +malley +nouveau +shaded +taxonomy +v6 +adhere +bunk +airfields +##ritan +1741 +encompass +remington +tran +##erative +amelie +mazda +friar +morals +passions +##zai +breadth +vis +##hae +argus +burnham +caressing +insider +rudd +##imov +##mini +##rso +italianate +murderous +textual +wainwright +armada +bam +weave +timer +##taken +##nh +fra +##crest +ardent +salazar +taps +tunis +##ntino +allegro +gland +philanthropic +##chester +implication +##optera +esq +judas +noticeably +wynn +##dara +inched +indexed +crises +villiers +bandit +royalties +patterned +cupboard +interspersed +accessory +isla +kendrick +entourage +stitches +##esthesia +headwaters +##ior +interlude +distraught +draught +1727 +##basket +biased +sy +transient +triad +subgenus +adapting +kidd +shortstop +##umatic +dimly +spiked +mcleod +reprint +nellie +pretoria +windmill +##cek +singled +##mps +273 +reunite +##orous +747 +bankers +outlying +##omp +##ports +##tream +apologies +cosmetics +patsy +##deh +##ocks +##yson +bender +nantes +serene +##nad +lucha +mmm +323 +##cius +##gli +cmll +coinage +nestor +juarez +##rook +smeared +sprayed +twitching +sterile +irina +embodied +juveniles +enveloped +miscellaneous +cancers +dq +gulped +luisa +crested +swat +donegal +ref +##anov +##acker +hearst +mercantile +##lika +doorbell +ua +vicki +##alla +##som +bilbao +psychologists +stryker +sw +horsemen +turkmenistan +wits +##national +anson +mathew +screenings +##umb +rihanna +##agne +##nessy +aisles +##iani +##osphere +hines +kenton +saskatoon +tasha +truncated +##champ +##itan +mildred +advises +fredrik +interpreting +inhibitors +##athi +spectroscopy +##hab +##kong +karim +panda +##oia +##nail +##vc +conqueror +kgb +leukemia +##dity +arrivals +cheered +pisa +phosphorus +shielded +##riated +mammal +unitarian +urgently +chopin +sanitary +##mission +spicy +drugged +hinges +##tort +tipping +trier +impoverished +westchester +##caster +267 +epoch +nonstop +##gman +##khov +aromatic +centrally +cerro +##tively +##vio +billions +modulation +sedimentary +283 +facilitating +outrageous +goldstein +##eak +##kt +ld +maitland +penultimate +pollard +##dance +fleets +spaceship +vertebrae +##nig +alcoholism +als +recital +##bham +##ference +##omics +m2 +##bm +trois +##tropical +##в +commemorates +##meric +marge +##raction +1643 +670 +cosmetic +ravaged +##ige +catastrophe +eng +##shida +albrecht +arterial +bellamy +decor +harmon +##rde +bulbs +synchronized +vito +easiest +shetland +shielding +wnba +##glers +##ssar +##riam +brianna +cumbria +##aceous +##rard +cores +thayer +##nsk +brood +hilltop +luminous +carts +keynote +larkin +logos +##cta +##ا +##mund +##quay +lilith +tinted +277 +wrestle +mobilization +##uses +sequential +siam +bloomfield +takahashi +274 +##ieving +presenters +ringo +blazed +witty +##oven +##ignant +devastation +haydn +harmed +newt +therese +##peed +gershwin +molina +rabbis +sudanese +001 +innate +restarted +##sack +##fus +slices +wb +##shah +enroll +hypothetical +hysterical +1743 +fabio +indefinite +warped +##hg +exchanging +525 +unsuitable +##sboro +gallo +1603 +bret +cobalt +homemade +##hunter +mx +operatives +##dhar +terraces +durable +latch +pens +whorls +##ctuated +##eaux +billing +ligament +succumbed +##gly +regulators +spawn +##brick +##stead +filmfare +rochelle +##nzo +1725 +circumstance +saber +supplements +##nsky +##tson +crowe +wellesley +carrot +##9th +##movable +primate +drury +sincerely +topical +##mad +##rao +callahan +kyiv +smarter +tits +undo +##yeh +announcements +anthologies +barrio +nebula +##islaus +##shaft +##tyn +bodyguards +2021 +assassinate +barns +emmett +scully +##mah +##yd +##eland +##tino +##itarian +demoted +gorman +lashed +prized +adventist +writ +##gui +alla +invertebrates +##ausen +1641 +amman +1742 +align +healy +redistribution +##gf +##rize +insulation +##drop +adherents +hezbollah +vitro +ferns +yanking +269 +php +registering +uppsala +cheerleading +confines +mischievous +tully +##ross +49th +docked +roam +stipulated +pumpkin +##bry +prompt +##ezer +blindly +shuddering +craftsmen +frail +scented +katharine +scramble +shaggy +sponge +helix +zaragoza +279 +##52 +43rd +backlash +fontaine +seizures +posse +cowan +nonfiction +telenovela +wwii +hammered +undone +##gpur +encircled +irs +##ivation +artefacts +oneself +searing +smallpox +##belle +##osaurus +shandong +breached +upland +blushing +rankin +infinitely +psyche +tolerated +docking +evicted +##col +unmarked +##lving +gnome +lettering +litres +musique +##oint +benevolent +##jal +blackened +##anna +mccall +racers +tingle +##ocene +##orestation +introductions +radically +292 +##hiff +##باد +1610 +1739 +munchen +plead +##nka +condo +scissors +##sight +##tens +apprehension +##cey +##yin +hallmark +watering +formulas +sequels +##llas +aggravated +bae +commencing +##building +enfield +prohibits +marne +vedic +civilized +euclidean +jagger +beforehand +blasts +dumont +##arney +##nem +740 +conversions +hierarchical +rios +simulator +##dya +##lellan +hedges +oleg +thrusts +shadowed +darby +maximize +1744 +gregorian +##nded +##routed +sham +unspecified +##hog +emory +factual +##smo +##tp +fooled +##rger +ortega +wellness +marlon +##oton +##urance +casket +keating +ley +enclave +##ayan +char +influencing +jia +##chenko +412 +ammonia +erebidae +incompatible +violins +cornered +##arat +grooves +astronauts +columbian +rampant +fabrication +kyushu +mahmud +vanish +##dern +mesopotamia +##lete +ict +##rgen +caspian +kenji +pitted +##vered +999 +grimace +roanoke +tchaikovsky +twinned +##analysis +##awan +xinjiang +arias +clemson +kazakh +sizable +1662 +##khand +##vard +plunge +tatum +vittorio +##nden +cholera +##dana +##oper +bracing +indifference +projectile +superliga +##chee +realises +upgrading +299 +porte +retribution +##vies +nk +stil +##resses +ama +bureaucracy +blackberry +bosch +testosterone +collapses +greer +##pathic +ioc +fifties +malls +##erved +bao +baskets +adolescents +siegfried +##osity +##tosis +mantra +detecting +existent +fledgling +##cchi +dissatisfied +gan +telecommunication +mingled +sobbed +6000 +controversies +outdated +taxis +##raus +fright +slams +##lham +##fect +##tten +detectors +fetal +tanned +##uw +fray +goth +olympian +skipping +mandates +scratches +sheng +unspoken +hyundai +tracey +hotspur +restrictive +##buch +americana +mundo +##bari +burroughs +diva +vulcan +##6th +distinctions +thumping +##ngen +mikey +sheds +fide +rescues +springsteen +vested +valuation +##ece +##ely +pinnacle +rake +sylvie +##edo +almond +quivering +##irus +alteration +faltered +##wad +51st +hydra +ticked +##kato +recommends +##dicated +antigua +arjun +stagecoach +wilfred +trickle +pronouns +##pon +aryan +nighttime +##anian +gall +pea +stitch +##hei +leung +milos +##dini +eritrea +nexus +starved +snowfall +kant +parasitic +cot +discus +hana +strikers +appleton +kitchens +##erina +##partisan +##itha +##vius +disclose +metis +##channel +1701 +tesla +##vera +fitch +1735 +blooded +##tila +decimal +##tang +##bai +cyclones +eun +bottled +peas +pensacola +basha +bolivian +crabs +boil +lanterns +partridge +roofed +1645 +necks +##phila +opined +patting +##kla +##lland +chuckles +volta +whereupon +##nche +devout +euroleague +suicidal +##dee +inherently +involuntary +knitting +nasser +##hide +puppets +colourful +courageous +southend +stills +miraculous +hodgson +richer +rochdale +ethernet +greta +uniting +prism +umm +##haya +##itical +##utation +deterioration +pointe +prowess +##ropriation +lids +scranton +billings +subcontinent +##koff +##scope +brute +kellogg +psalms +degraded +##vez +stanisław +##ructured +ferreira +pun +astonishing +gunnar +##yat +arya +prc +gottfried +##tight +excursion +##ographer +dina +##quil +##nare +huffington +illustrious +wilbur +gundam +verandah +##zard +naacp +##odle +constructive +fjord +kade +##naud +generosity +thrilling +baseline +cayman +frankish +plastics +accommodations +zoological +##fting +cedric +qb +motorized +##dome +##otted +squealed +tackled +canucks +budgets +situ +asthma +dail +gabled +grasslands +whimpered +writhing +judgments +##65 +minnie +pv +##carbon +bananas +grille +domes +monique +odin +maguire +markham +tierney +##estra +##chua +libel +poke +speedy +atrium +laval +notwithstanding +##edly +fai +kala +##sur +robb +##sma +listings +luz +supplementary +tianjin +##acing +enzo +jd +ric +scanner +croats +transcribed +##49 +arden +cv +##hair +##raphy +##lver +##uy +357 +seventies +staggering +alam +horticultural +hs +regression +timbers +blasting +##ounded +montagu +manipulating +##cit +catalytic +1550 +troopers +##meo +condemnation +fitzpatrick +##oire +##roved +inexperienced +1670 +castes +##lative +outing +314 +dubois +flicking +quarrel +ste +learners +1625 +iq +whistled +##class +282 +classify +tariffs +temperament +355 +folly +liszt +##yles +immersed +jordanian +ceasefire +apparel +extras +maru +fished +##bio +harta +stockport +assortment +craftsman +paralysis +transmitters +##cola +blindness +##wk +fatally +proficiency +solemnly +##orno +repairing +amore +groceries +ultraviolet +##chase +schoolhouse +##tua +resurgence +nailed +##otype +##× +ruse +saliva +diagrams +##tructing +albans +rann +thirties +1b +antennas +hilarious +cougars +paddington +stats +##eger +breakaway +ipod +reza +authorship +prohibiting +scoffed +##etz +##ttle +conscription +defected +trondheim +##fires +ivanov +keenan +##adan +##ciful +##fb +##slow +locating +##ials +##tford +cadiz +basalt +blankly +interned +rags +rattling +##tick +carpathian +reassured +sync +bum +guildford +iss +staunch +##onga +astronomers +sera +sofie +emergencies +susquehanna +##heard +duc +mastery +vh1 +williamsburg +bayer +buckled +craving +##khan +##rdes +bloomington +##write +alton +barbecue +##bians +justine +##hri +##ndt +delightful +smartphone +newtown +photon +retrieval +peugeot +hissing +##monium +##orough +flavors +lighted +relaunched +tainted +##games +##lysis +anarchy +microscopic +hopping +adept +evade +evie +##beau +inhibit +sinn +adjustable +hurst +intuition +wilton +cisco +44th +lawful +lowlands +stockings +thierry +##dalen +##hila +##nai +fates +prank +tb +maison +lobbied +provocative +1724 +4a +utopia +##qual +carbonate +gujarati +purcell +##rford +curtiss +##mei +overgrown +arenas +mediation +swallows +##rnik +respectful +turnbull +##hedron +##hope +alyssa +ozone +##ʻi +ami +gestapo +johansson +snooker +canteen +cuff +declines +empathy +stigma +##ags +##iner +##raine +taxpayers +gui +volga +##wright +##copic +lifespan +overcame +tattooed +enactment +giggles +##ador +##camp +barrington +bribe +obligatory +orbiting +peng +##enas +elusive +sucker +##vating +cong +hardship +empowered +anticipating +estrada +cryptic +greasy +detainees +planck +sudbury +plaid +dod +marriott +kayla +##ears +##vb +##zd +mortally +##hein +cognition +radha +319 +liechtenstein +meade +richly +argyle +harpsichord +liberalism +trumpets +lauded +tyrant +salsa +tiled +lear +promoters +reused +slicing +trident +##chuk +##gami +##lka +cantor +checkpoint +##points +gaul +leger +mammalian +##tov +##aar +##schaft +doha +frenchman +nirvana +##vino +delgado +headlining +##eron +##iography +jug +tko +1649 +naga +intersections +##jia +benfica +nawab +##suka +ashford +gulp +##deck +##vill +##rug +brentford +frazier +pleasures +dunne +potsdam +shenzhen +dentistry +##tec +flanagan +##dorff +##hear +chorale +dinah +prem +quezon +##rogated +relinquished +sutra +terri +##pani +flaps +##rissa +poly +##rnet +homme +aback +##eki +linger +womb +##kson +##lewood +doorstep +orthodoxy +threaded +westfield +##rval +dioceses +fridays +subsided +##gata +loyalists +##biotic +##ettes +letterman +lunatic +prelate +tenderly +invariably +souza +thug +winslow +##otide +furlongs +gogh +jeopardy +##runa +pegasus +##umble +humiliated +standalone +tagged +##roller +freshmen +klan +##bright +attaining +initiating +transatlantic +logged +viz +##uance +1723 +combatants +intervening +stephane +chieftain +despised +grazed +317 +cdc +galveston +godzilla +macro +simulate +##planes +parades +##esses +960 +##ductive +##unes +equator +overdose +##cans +##hosh +##lifting +joshi +epstein +sonora +treacherous +aquatics +manchu +responsive +##sation +supervisory +##christ +##llins +##ibar +##balance +##uso +kimball +karlsruhe +mab +##emy +ignores +phonetic +reuters +spaghetti +820 +almighty +danzig +rumbling +tombstone +designations +lured +outset +##felt +supermarkets +##wt +grupo +kei +kraft +susanna +##blood +comprehension +genealogy +##aghan +##verted +redding +##ythe +1722 +bowing +##pore +##roi +lest +sharpened +fulbright +valkyrie +sikhs +##unds +swans +bouquet +merritt +##tage +##venting +commuted +redhead +clerks +leasing +cesare +dea +hazy +##vances +fledged +greenfield +servicemen +##gical +armando +blackout +dt +sagged +downloadable +intra +potion +pods +##4th +##mism +xp +attendants +gambia +stale +##ntine +plump +asteroids +rediscovered +buds +flea +hive +##neas +1737 +classifications +debuts +##eles +olympus +scala +##eurs +##gno +##mute +hummed +sigismund +visuals +wiggled +await +pilasters +clench +sulfate +##ances +bellevue +enigma +trainee +snort +##sw +clouded +denim +##rank +##rder +churning +hartman +lodges +riches +sima +##missible +accountable +socrates +regulates +mueller +##cr +1702 +avoids +solids +himalayas +nutrient +pup +##jevic +squat +fades +nec +##lates +##pina +##rona +##ου +privateer +tequila +##gative +##mpton +apt +hornet +immortals +##dou +asturias +cleansing +dario +##rries +##anta +etymology +servicing +zhejiang +##venor +##nx +horned +erasmus +rayon +relocating +£10 +##bags +escalated +promenade +stubble +2010s +artisans +axial +liquids +mora +sho +yoo +##tsky +bundles +oldies +##nally +notification +bastion +##ths +sparkle +##lved +1728 +leash +pathogen +highs +##hmi +immature +880 +gonzaga +ignatius +mansions +monterrey +sweets +bryson +##loe +polled +regatta +brightest +pei +rosy +squid +hatfield +payroll +addict +meath +cornerback +heaviest +lodging +##mage +capcom +rippled +##sily +barnet +mayhem +ymca +snuggled +rousseau +##cute +blanchard +284 +fragmented +leighton +chromosomes +risking +##md +##strel +##utter +corinne +coyotes +cynical +hiroshi +yeomanry +##ractive +ebook +grading +mandela +plume +agustin +magdalene +##rkin +bea +femme +trafford +##coll +##lun +##tance +52nd +fourier +upton +##mental +camilla +gust +iihf +islamabad +longevity +##kala +feldman +netting +##rization +endeavour +foraging +mfa +orr +##open +greyish +contradiction +graz +##ruff +handicapped +marlene +tweed +oaxaca +spp +campos +miocene +pri +configured +cooks +pluto +cozy +pornographic +##entes +70th +fairness +glided +jonny +lynne +rounding +sired +##emon +##nist +remade +uncover +##mack +complied +lei +newsweek +##jured +##parts +##enting +##pg +293 +finer +guerrillas +athenian +deng +disused +stepmother +accuse +gingerly +seduction +521 +confronting +##walker +##going +gora +nostalgia +sabres +virginity +wrenched +##minated +syndication +wielding +eyre +##56 +##gnon +##igny +behaved +taxpayer +sweeps +##growth +childless +gallant +##ywood +amplified +geraldine +scrape +##ffi +babylonian +fresco +##rdan +##kney +##position +1718 +restricting +tack +fukuoka +osborn +selector +partnering +##dlow +318 +gnu +kia +tak +whitley +gables +##54 +##mania +mri +softness +immersion +##bots +##evsky +1713 +chilling +insignificant +pcs +##uis +elites +lina +purported +supplemental +teaming +##americana +##dding +##inton +proficient +rouen +##nage +##rret +niccolo +selects +##bread +fluffy +1621 +gruff +knotted +mukherjee +polgara +thrash +nicholls +secluded +smoothing +thru +corsica +loaf +whitaker +inquiries +##rrier +##kam +indochina +289 +marlins +myles +peking +##tea +extracts +pastry +superhuman +connacht +vogel +##ditional +##het +##udged +##lash +gloss +quarries +refit +teaser +##alic +##gaon +20s +materialized +sling +camped +pickering +tung +tracker +pursuant +##cide +cranes +soc +##cini +##typical +##viere +anhalt +overboard +workout +chores +fares +orphaned +stains +##logie +fenton +surpassing +joyah +triggers +##itte +grandmaster +##lass +##lists +clapping +fraudulent +ledger +nagasaki +##cor +##nosis +##tsa +eucalyptus +tun +##icio +##rney +##tara +dax +heroism +ina +wrexham +onboard +unsigned +##dates +moshe +galley +winnie +droplets +exiles +praises +watered +noodles +##aia +fein +adi +leland +multicultural +stink +bingo +comets +erskine +modernized +canned +constraint +domestically +chemotherapy +featherweight +stifled +##mum +darkly +irresistible +refreshing +hasty +isolate +##oys +kitchener +planners +##wehr +cages +yarn +implant +toulon +elects +childbirth +yue +##lind +##lone +cn +rightful +sportsman +junctions +remodeled +specifies +##rgh +291 +##oons +complimented +##urgent +lister +ot +##logic +bequeathed +cheekbones +fontana +gabby +##dial +amadeus +corrugated +maverick +resented +triangles +##hered +##usly +nazareth +tyrol +1675 +assent +poorer +sectional +aegean +##cous +296 +nylon +ghanaian +##egorical +##weig +cushions +forbid +fusiliers +obstruction +somerville +##scia +dime +earrings +elliptical +leyte +oder +polymers +timmy +atm +midtown +piloted +settles +continual +externally +mayfield +##uh +enrichment +henson +keane +persians +1733 +benji +braden +pep +324 +##efe +contenders +pepsi +valet +##isches +298 +##asse +##earing +goofy +stroll +##amen +authoritarian +occurrences +adversary +ahmedabad +tangent +toppled +dorchester +1672 +modernism +marxism +islamist +charlemagne +exponential +racks +unicode +brunette +mbc +pic +skirmish +##bund +##lad +##powered +##yst +hoisted +messina +shatter +##ctum +jedi +vantage +##music +##neil +clemens +mahmoud +corrupted +authentication +lowry +nils +##washed +omnibus +wounding +jillian +##itors +##opped +serialized +narcotics +handheld +##arm +##plicity +intersecting +stimulating +##onis +crate +fellowships +hemingway +casinos +climatic +fordham +copeland +drip +beatty +leaflets +robber +brothel +madeira +##hedral +sphinx +ultrasound +##vana +valor +forbade +leonid +villas +##aldo +duane +marquez +##cytes +disadvantaged +forearms +kawasaki +reacts +consular +lax +uncles +uphold +##hopper +concepcion +dorsey +lass +##izan +arching +passageway +1708 +researches +tia +internationals +##graphs +##opers +distinguishes +javanese +divert +##uven +plotted +##listic +##rwin +##erik +##tify +affirmative +signifies +validation +##bson +kari +felicity +georgina +zulu +##eros +##rained +##rath +overcoming +##dot +argyll +##rbin +1734 +chiba +ratification +windy +earls +parapet +##marks +hunan +pristine +astrid +punta +##gart +brodie +##kota +##oder +malaga +minerva +rouse +##phonic +bellowed +pagoda +portals +reclamation +##gur +##odies +##⁄₄ +parentheses +quoting +allergic +palette +showcases +benefactor +heartland +nonlinear +##tness +bladed +cheerfully +scans +##ety +##hone +1666 +girlfriends +pedersen +hiram +sous +##liche +##nator +1683 +##nery +##orio +##umen +bobo +primaries +smiley +##cb +unearthed +uniformly +fis +metadata +1635 +ind +##oted +recoil +##titles +##tura +##ια +406 +hilbert +jamestown +mcmillan +tulane +seychelles +##frid +antics +coli +fated +stucco +##grants +1654 +bulky +accolades +arrays +caledonian +carnage +optimism +puebla +##tative +##cave +enforcing +rotherham +seo +dunlop +aeronautics +chimed +incline +zoning +archduke +hellenistic +##oses +##sions +candi +thong +##ople +magnate +rustic +##rsk +projective +slant +##offs +danes +hollis +vocalists +##ammed +congenital +contend +gesellschaft +##ocating +##pressive +douglass +quieter +##cm +##kshi +howled +salim +spontaneously +townsville +buena +southport +##bold +kato +1638 +faerie +stiffly +##vus +##rled +297 +flawless +realising +taboo +##7th +bytes +straightening +356 +jena +##hid +##rmin +cartwright +berber +bertram +soloists +411 +noses +417 +coping +fission +hardin +inca +##cen +1717 +mobilized +vhf +##raf +biscuits +curate +##85 +##anial +331 +gaunt +neighbourhoods +1540 +##abas +blanca +bypassed +sockets +behold +coincidentally +##bane +nara +shave +splinter +terrific +##arion +##erian +commonplace +juris +redwood +waistband +boxed +caitlin +fingerprints +jennie +naturalized +##ired +balfour +craters +jody +bungalow +hugely +quilt +glitter +pigeons +undertaker +bulging +constrained +goo +##sil +##akh +assimilation +reworked +##person +persuasion +##pants +felicia +##cliff +##ulent +1732 +explodes +##dun +##inium +##zic +lyman +vulture +hog +overlook +begs +northwards +ow +spoil +##urer +fatima +favorably +accumulate +sargent +sorority +corresponded +dispersal +kochi +toned +##imi +##lita +internacional +newfound +##agger +##lynn +##rigue +booths +peanuts +##eborg +medicare +muriel +nur +##uram +crates +millennia +pajamas +worsened +##breakers +jimi +vanuatu +yawned +##udeau +carousel +##hony +hurdle +##ccus +##mounted +##pod +rv +##eche +airship +ambiguity +compulsion +recapture +##claiming +arthritis +##osomal +1667 +asserting +ngc +sniffing +dade +discontent +glendale +ported +##amina +defamation +rammed +##scent +fling +livingstone +##fleet +875 +##ppy +apocalyptic +comrade +lcd +##lowe +cessna +eine +persecuted +subsistence +demi +hoop +reliefs +710 +coptic +progressing +stemmed +perpetrators +1665 +priestess +##nio +dobson +ebony +rooster +itf +tortricidae +##bbon +##jian +cleanup +##jean +##øy +1721 +eighties +taxonomic +holiness +##hearted +##spar +antilles +showcasing +stabilized +##nb +gia +mascara +michelangelo +dawned +##uria +##vinsky +extinguished +fitz +grotesque +£100 +##fera +##loid +##mous +barges +neue +throbbed +cipher +johnnie +##a1 +##mpt +outburst +##swick +spearheaded +administrations +c1 +heartbreak +pixels +pleasantly +##enay +lombardy +plush +##nsed +bobbie +##hly +reapers +tremor +xiang +minogue +substantive +hitch +barak +##wyl +kwan +##encia +910 +obscene +elegance +indus +surfer +bribery +conserve +##hyllum +##masters +horatio +##fat +apes +rebound +psychotic +##pour +iteration +##mium +##vani +botanic +horribly +antiques +dispose +paxton +##hli +##wg +timeless +1704 +disregard +engraver +hounds +##bau +##version +looted +uno +facilitates +groans +masjid +rutland +antibody +disqualification +decatur +footballers +quake +slacks +48th +rein +scribe +stabilize +commits +exemplary +tho +##hort +##chison +pantry +traversed +##hiti +disrepair +identifiable +vibrated +baccalaureate +##nnis +csa +interviewing +##iensis +##raße +greaves +wealthiest +343 +classed +jogged +£5 +##58 +##atal +illuminating +knicks +respecting +##uno +scrubbed +##iji +##dles +kruger +moods +growls +raider +silvia +chefs +kam +vr +cree +percival +##terol +gunter +counterattack +defiant +henan +ze +##rasia +##riety +equivalence +submissions +##fra +##thor +bautista +mechanically +##heater +cornice +herbal +templar +##mering +outputs +ruining +ligand +renumbered +extravagant +mika +blockbuster +eta +insurrection +##ilia +darkening +ferocious +pianos +strife +kinship +##aer +melee +##anor +##iste +##may +##oue +decidedly +weep +##jad +##missive +##ppel +354 +puget +unease +##gnant +1629 +hammering +kassel +ob +wessex +##lga +bromwich +egan +paranoia +utilization +##atable +##idad +contradictory +provoke +##ols +##ouring +##tangled +knesset +##very +##lette +plumbing +##sden +##¹ +greensboro +occult +sniff +338 +zev +beaming +gamer +haggard +mahal +##olt +##pins +mendes +utmost +briefing +gunnery +##gut +##pher +##zh +##rok +1679 +khalifa +sonya +##boot +principals +urbana +wiring +##liffe +##minating +##rrado +dahl +nyu +skepticism +np +townspeople +ithaca +lobster +somethin +##fur +##arina +##−1 +freighter +zimmerman +biceps +contractual +##herton +amend +hurrying +subconscious +##anal +336 +meng +clermont +spawning +##eia +##lub +dignitaries +impetus +snacks +spotting +twigs +##bilis +##cz +##ouk +libertadores +nic +skylar +##aina +##firm +gustave +asean +##anum +dieter +legislatures +flirt +bromley +trolls +umar +##bbies +##tyle +blah +parc +bridgeport +crank +negligence +##nction +46th +constantin +molded +bandages +seriousness +00pm +siegel +carpets +compartments +upbeat +statehood +##dner +##edging +marko +730 +platt +##hane +paving +##iy +1738 +abbess +impatience +limousine +nbl +##talk +441 +lucille +mojo +nightfall +robbers +##nais +karel +brisk +calves +replicate +ascribed +telescopes +##olf +intimidated +##reen +ballast +specialization +##sit +aerodynamic +caliphate +rainer +visionary +##arded +epsilon +##aday +##onte +aggregation +auditory +boosted +reunification +kathmandu +loco +robyn +402 +acknowledges +appointing +humanoid +newell +redeveloped +restraints +##tained +barbarians +chopper +1609 +italiana +##lez +##lho +investigates +wrestlemania +##anies +##bib +690 +##falls +creaked +dragoons +gravely +minions +stupidity +volley +##harat +##week +musik +##eries +##uously +fungal +massimo +semantics +malvern +##ahl +##pee +discourage +embryo +imperialism +1910s +profoundly +##ddled +jiangsu +sparkled +stat +##holz +sweatshirt +tobin +##iction +sneered +##cheon +##oit +brit +causal +smyth +##neuve +diffuse +perrin +silvio +##ipes +##recht +detonated +iqbal +selma +##nism +##zumi +roasted +##riders +tay +##ados +##mament +##mut +##rud +840 +completes +nipples +cfa +flavour +hirsch +##laus +calderon +sneakers +moravian +##ksha +1622 +rq +294 +##imeters +bodo +##isance +##pre +##ronia +anatomical +excerpt +##lke +dh +kunst +##tablished +##scoe +biomass +panted +unharmed +gael +housemates +montpellier +##59 +coa +rodents +tonic +hickory +singleton +##taro +451 +1719 +aldo +breaststroke +dempsey +och +rocco +##cuit +merton +dissemination +midsummer +serials +##idi +haji +polynomials +##rdon +gs +enoch +prematurely +shutter +taunton +£3 +##grating +##inates +archangel +harassed +##asco +326 +archway +dazzling +##ecin +1736 +sumo +wat +##kovich +1086 +honneur +##ently +##nostic +##ttal +##idon +1605 +403 +1716 +blogger +rents +##gnan +hires +##ikh +##dant +howie +##rons +handler +retracted +shocks +1632 +arun +duluth +kepler +trumpeter +##lary +peeking +seasoned +trooper +##mara +laszlo +##iciencies +##rti +heterosexual +##inatory +##ssion +indira +jogging +##inga +##lism +beit +dissatisfaction +malice +##ately +nedra +peeling +##rgeon +47th +stadiums +475 +vertigo +##ains +iced +restroom +##plify +##tub +illustrating +pear +##chner +##sibility +inorganic +rappers +receipts +watery +##kura +lucinda +##oulos +reintroduced +##8th +##tched +gracefully +saxons +nutritional +wastewater +rained +favourites +bedrock +fisted +hallways +likeness +upscale +##lateral +1580 +blinds +prequel +##pps +##tama +deter +humiliating +restraining +tn +vents +1659 +laundering +recess +rosary +tractors +coulter +federer +##ifiers +##plin +persistence +##quitable +geschichte +pendulum +quakers +##beam +bassett +pictorial +buffet +koln +##sitor +drills +reciprocal +shooters +##57 +##cton +##tees +converge +pip +dmitri +donnelly +yamamoto +aqua +azores +demographics +hypnotic +spitfire +suspend +wryly +roderick +##rran +sebastien +##asurable +mavericks +##fles +##200 +himalayan +prodigy +##iance +transvaal +demonstrators +handcuffs +dodged +mcnamara +sublime +1726 +crazed +##efined +##till +ivo +pondered +reconciled +shrill +sava +##duk +bal +cad +heresy +jaipur +goran +##nished +341 +lux +shelly +whitehall +##hre +israelis +peacekeeping +##wled +1703 +demetrius +ousted +##arians +##zos +beale +anwar +backstroke +raged +shrinking +cremated +##yck +benign +towing +wadi +darmstadt +landfill +parana +soothe +colleen +sidewalks +mayfair +tumble +hepatitis +ferrer +superstructure +##gingly +##urse +##wee +anthropological +translators +##mies +closeness +hooves +##pw +mondays +##roll +##vita +landscaping +##urized +purification +sock +thorns +thwarted +jalan +tiberius +##taka +saline +##rito +confidently +khyber +sculptors +##ij +brahms +hammersmith +inspectors +battista +fivb +fragmentation +hackney +##uls +arresting +exercising +antoinette +bedfordshire +##zily +dyed +##hema +1656 +racetrack +variability +##tique +1655 +austrians +deteriorating +madman +theorists +aix +lehman +weathered +1731 +decreed +eruptions +1729 +flaw +quinlan +sorbonne +flutes +nunez +1711 +adored +downwards +fable +rasped +1712 +moritz +mouthful +renegade +shivers +stunts +dysfunction +restrain +translit +327 +pancakes +##avio +##cision +##tray +351 +vial +##lden +bain +##maid +##oxide +chihuahua +malacca +vimes +##rba +##rnier +1664 +donnie +plaques +##ually +337 +bangs +floppy +huntsville +loretta +nikolay +##otte +eater +handgun +ubiquitous +##hett +eras +zodiac +1634 +##omorphic +1820s +##zog +cochran +##bula +##lithic +warring +##rada +dalai +excused +blazers +mcconnell +reeling +bot +este +##abi +geese +hoax +taxon +##bla +guitarists +##icon +condemning +hunts +inversion +moffat +taekwondo +##lvis +1624 +stammered +##rest +##rzy +sousa +fundraiser +marylebone +navigable +uptown +cabbage +daniela +salman +shitty +whimper +##kian +##utive +programmers +protections +rm +##rmi +##rued +forceful +##enes +fuss +##tao +##wash +brat +oppressive +reykjavik +spartak +ticking +##inkles +##kiewicz +adolph +horst +maui +protege +straighten +cpc +landau +concourse +clements +resultant +##ando +imaginative +joo +reactivated +##rem +##ffled +##uising +consultative +##guide +flop +kaitlyn +mergers +parenting +somber +##vron +supervise +vidhan +##imum +courtship +exemplified +harmonies +medallist +refining +##rrow +##ка +amara +##hum +780 +goalscorer +sited +overshadowed +rohan +displeasure +secretive +multiplied +osman +##orth +engravings +padre +##kali +##veda +miniatures +mis +##yala +clap +pali +rook +##cana +1692 +57th +antennae +astro +oskar +1628 +bulldog +crotch +hackett +yucatan +##sure +amplifiers +brno +ferrara +migrating +##gree +thanking +turing +##eza +mccann +ting +andersson +onslaught +gaines +ganga +incense +standardization +##mation +sentai +scuba +stuffing +turquoise +waivers +alloys +##vitt +regaining +vaults +##clops +##gizing +digger +furry +memorabilia +probing +##iad +payton +rec +deutschland +filippo +opaque +seamen +zenith +afrikaans +##filtration +disciplined +inspirational +##merie +banco +confuse +grafton +tod +##dgets +championed +simi +anomaly +biplane +##ceptive +electrode +##para +1697 +cleavage +crossbow +swirl +informant +##lars +##osta +afi +bonfire +spec +##oux +lakeside +slump +##culus +##lais +##qvist +##rrigan +1016 +facades +borg +inwardly +cervical +xl +pointedly +050 +stabilization +##odon +chests +1699 +hacked +ctv +orthogonal +suzy +##lastic +gaulle +jacobite +rearview +##cam +##erted +ashby +##drik +##igate +##mise +##zbek +affectionately +canine +disperse +latham +##istles +##ivar +spielberg +##orin +##idium +ezekiel +cid +##sg +durga +middletown +##cina +customized +frontiers +harden +##etano +##zzy +1604 +bolsheviks +##66 +coloration +yoko +##bedo +briefs +slabs +debra +liquidation +plumage +##oin +blossoms +dementia +subsidy +1611 +proctor +relational +jerseys +parochial +ter +##ici +esa +peshawar +cavalier +loren +cpi +idiots +shamrock +1646 +dutton +malabar +mustache +##endez +##ocytes +referencing +terminates +marche +yarmouth +##sop +acton +mated +seton +subtly +baptised +beige +extremes +jolted +kristina +telecast +##actic +safeguard +waldo +##baldi +##bular +endeavors +sloppy +subterranean +##ensburg +##itung +delicately +pigment +tq +##scu +1626 +##ound +collisions +coveted +herds +##personal +##meister +##nberger +chopra +##ricting +abnormalities +defective +galician +lucie +##dilly +alligator +likened +##genase +burundi +clears +complexion +derelict +deafening +diablo +fingered +champaign +dogg +enlist +isotope +labeling +mrna +##erre +brilliance +marvelous +##ayo +1652 +crawley +ether +footed +dwellers +deserts +hamish +rubs +warlock +skimmed +##lizer +870 +buick +embark +heraldic +irregularities +##ajan +kiara +##kulam +##ieg +antigen +kowalski +##lge +oakley +visitation +##mbit +vt +##suit +1570 +murderers +##miento +##rites +chimneys +##sling +condemn +custer +exchequer +havre +##ghi +fluctuations +##rations +dfb +hendricks +vaccines +##tarian +nietzsche +biking +juicy +##duced +brooding +scrolling +selangor +##ragan +352 +annum +boomed +seminole +sugarcane +##dna +departmental +dismissing +innsbruck +arteries +ashok +batavia +daze +kun +overtook +##rga +##tlan +beheaded +gaddafi +holm +electronically +faulty +galilee +fractures +kobayashi +##lized +gunmen +magma +aramaic +mala +eastenders +inference +messengers +bf +##qu +407 +bathrooms +##vere +1658 +flashbacks +ideally +misunderstood +##jali +##weather +mendez +##grounds +505 +uncanny +##iii +1709 +friendships +##nbc +sacrament +accommodated +reiterated +logistical +pebbles +thumped +##escence +administering +decrees +drafts +##flight +##cased +##tula +futuristic +picket +intimidation +winthrop +##fahan +interfered +339 +afar +francoise +morally +uta +cochin +croft +dwarfs +##bruck +##dents +##nami +biker +##hner +##meral +nano +##isen +##ometric +##pres +##ан +brightened +meek +parcels +securely +gunners +##jhl +##zko +agile +hysteria +##lten +##rcus +bukit +champs +chevy +cuckoo +leith +sadler +theologians +welded +##section +1663 +jj +plurality +xander +##rooms +##formed +shredded +temps +intimately +pau +tormented +##lok +##stellar +1618 +charred +ems +essen +##mmel +alarms +spraying +ascot +blooms +twinkle +##abia +##apes +internment +obsidian +##chaft +snoop +##dav +##ooping +malibu +##tension +quiver +##itia +hays +mcintosh +travers +walsall +##ffie +1623 +beverley +schwarz +plunging +structurally +m3 +rosenthal +vikram +##tsk +770 +ghz +##onda +##tiv +chalmers +groningen +pew +reckon +unicef +##rvis +55th +##gni +1651 +sulawesi +avila +cai +metaphysical +screwing +turbulence +##mberg +augusto +samba +56th +baffled +momentary +toxin +##urian +##wani +aachen +condoms +dali +steppe +##3d +##app +##oed +##year +adolescence +dauphin +electrically +inaccessible +microscopy +nikita +##ega +atv +##cel +##enter +##oles +##oteric +##ы +accountants +punishments +wrongly +bribes +adventurous +clinch +flinders +southland +##hem +##kata +gough +##ciency +lads +soared +##ה +undergoes +deformation +outlawed +rubbish +##arus +##mussen +##nidae +##rzburg +arcs +##ingdon +##tituted +1695 +wheelbase +wheeling +bombardier +campground +zebra +##lices +##oj +##bain +lullaby +##ecure +donetsk +wylie +grenada +##arding +##ης +squinting +eireann +opposes +##andra +maximal +runes +##broken +##cuting +##iface +##ror +##rosis +additive +britney +adultery +triggering +##drome +detrimental +aarhus +containment +jc +swapped +vichy +##ioms +madly +##oric +##rag +brant +##ckey +##trix +1560 +1612 +broughton +rustling +##stems +##uder +asbestos +mentoring +##nivorous +finley +leaps +##isan +apical +pry +slits +substitutes +##dict +intuitive +fantasia +insistent +unreasonable +##igen +##vna +domed +hannover +margot +ponder +##zziness +impromptu +jian +lc +rampage +stemming +##eft +andrey +gerais +whichever +amnesia +appropriated +anzac +clicks +modifying +ultimatum +cambrian +maids +verve +yellowstone +##mbs +conservatoire +##scribe +adherence +dinners +spectra +imperfect +mysteriously +sidekick +tatar +tuba +##aks +##ifolia +distrust +##athan +##zle +c2 +ronin +zac +##pse +celaena +instrumentalist +scents +skopje +##mbling +comical +compensated +vidal +condor +intersect +jingle +wavelengths +##urrent +mcqueen +##izzly +carp +weasel +422 +kanye +militias +postdoctoral +eugen +gunslinger +##ɛ +faux +hospice +##for +appalled +derivation +dwarves +##elis +dilapidated +##folk +astoria +philology +##lwyn +##otho +##saka +inducing +philanthropy +##bf +##itative +geek +markedly +sql +##yce +bessie +indices +rn +##flict +495 +frowns +resolving +weightlifting +tugs +cleric +contentious +1653 +mania +rms +##miya +##reate +##ruck +##tucket +bien +eels +marek +##ayton +##cence +discreet +unofficially +##ife +leaks +##bber +1705 +332 +dung +compressor +hillsborough +pandit +shillings +distal +##skin +381 +##tat +##you +nosed +##nir +mangrove +undeveloped +##idia +textures +##inho +##500 +##rise +ae +irritating +nay +amazingly +bancroft +apologetic +compassionate +kata +symphonies +##lovic +airspace +##lch +930 +gifford +precautions +fulfillment +sevilla +vulgar +martinique +##urities +looting +piccolo +tidy +##dermott +quadrant +armchair +incomes +mathematicians +stampede +nilsson +##inking +##scan +foo +quarterfinal +##ostal +shang +shouldered +squirrels +##owe +344 +vinegar +##bner +##rchy +##systems +delaying +##trics +ars +dwyer +rhapsody +sponsoring +##gration +bipolar +cinder +starters +##olio +##urst +421 +signage +##nty +aground +figurative +mons +acquaintances +duets +erroneously +soyuz +elliptic +recreated +##cultural +##quette +##ssed +##tma +##zcz +moderator +scares +##itaire +##stones +##udence +juniper +sighting +##just +##nsen +britten +calabria +ry +bop +cramer +forsyth +stillness +##л +airmen +gathers +unfit +##umber +##upt +taunting +##rip +seeker +streamlined +##bution +holster +schumann +tread +vox +##gano +##onzo +strive +dil +reforming +covent +newbury +predicting +##orro +decorate +tre +##puted +andover +ie +asahi +dept +dunkirk +gills +##tori +buren +huskies +##stis +##stov +abstracts +bets +loosen +##opa +1682 +yearning +##glio +##sir +berman +effortlessly +enamel +napoli +persist +##peration +##uez +attache +elisa +b1 +invitations +##kic +accelerating +reindeer +boardwalk +clutches +nelly +polka +starbucks +##kei +adamant +huey +lough +unbroken +adventurer +embroidery +inspecting +stanza +##ducted +naia +taluka +##pone +##roids +chases +deprivation +florian +##jing +##ppet +earthly +##lib +##ssee +colossal +foreigner +vet +freaks +patrice +rosewood +triassic +upstate +##pkins +dominates +ata +chants +ks +vo +##400 +##bley +##raya +##rmed +555 +agra +infiltrate +##ailing +##ilation +##tzer +##uppe +##werk +binoculars +enthusiast +fujian +squeak +##avs +abolitionist +almeida +boredom +hampstead +marsden +rations +##ands +inflated +334 +bonuses +rosalie +patna +##rco +329 +detachments +penitentiary +54th +flourishing +woolf +##dion +##etched +papyrus +##lster +##nsor +##toy +bobbed +dismounted +endelle +inhuman +motorola +tbs +wince +wreath +##ticus +hideout +inspections +sanjay +disgrace +infused +pudding +stalks +##urbed +arsenic +leases +##hyl +##rrard +collarbone +##waite +##wil +dowry +##bant +##edance +genealogical +nitrate +salamanca +scandals +thyroid +necessitated +##! +##" +### +##$ +##% +##& +##' +##( +##) +##* +##+ +##, +##- +##. +##/ +##: +##; +##< +##= +##> +##? +##@ +##[ +##\ +##] +##^ +##_ +##` +##{ +##| +##} +##~ +##¡ +##¢ +##£ +##¤ +##¥ +##¦ +##§ +##¨ +##© +##ª +##« +##¬ +##® +##± +##´ +##µ +##¶ +##· +##º +##» +##¼ +##¾ +##¿ +##æ +##ð +##÷ +##þ +##đ +##ħ +##ŋ +##œ +##ƒ +##ɐ +##ɑ +##ɒ +##ɔ +##ɕ +##ə +##ɡ +##ɣ +##ɨ +##ɪ +##ɫ +##ɬ +##ɯ +##ɲ +##ɴ +##ɹ +##ɾ +##ʀ +##ʁ +##ʂ +##ʃ +##ʉ +##ʊ +##ʋ +##ʌ +##ʎ +##ʐ +##ʑ +##ʒ +##ʔ +##ʰ +##ʲ +##ʳ +##ʷ +##ʸ +##ʻ +##ʼ +##ʾ +##ʿ +##ˈ +##ˡ +##ˢ +##ˣ +##ˤ +##β +##γ +##δ +##ε +##ζ +##θ +##κ +##λ +##μ +##ξ +##ο +##π +##ρ +##σ +##τ +##υ +##φ +##χ +##ψ +##ω +##б +##г +##д +##ж +##з +##м +##п +##с +##у +##ф +##х +##ц +##ч +##ш +##щ +##ъ +##э +##ю +##ђ +##є +##і +##ј +##љ +##њ +##ћ +##ӏ +##ա +##բ +##գ +##դ +##ե +##թ +##ի +##լ +##կ +##հ +##մ +##յ +##ն +##ո +##պ +##ս +##վ +##տ +##ր +##ւ +##ք +##־ +##א +##ב +##ג +##ד +##ו +##ז +##ח +##ט +##י +##ך +##כ +##ל +##ם +##מ +##ן +##נ +##ס +##ע +##ף +##פ +##ץ +##צ +##ק +##ר +##ש +##ת +##، +##ء +##ب +##ت +##ث +##ج +##ح +##خ +##ذ +##ز +##س +##ش +##ص +##ض +##ط +##ظ +##ع +##غ +##ـ +##ف +##ق +##ك +##و +##ى +##ٹ +##پ +##چ +##ک +##گ +##ں +##ھ +##ہ +##ے +##अ +##आ +##उ +##ए +##क +##ख +##ग +##च +##ज +##ट +##ड +##ण +##त +##थ +##द +##ध +##न +##प +##ब +##भ +##म +##य +##र +##ल +##व +##श +##ष +##स +##ह +##ा +##ि +##ी +##ो +##। +##॥ +##ং +##অ +##আ +##ই +##উ +##এ +##ও +##ক +##খ +##গ +##চ +##ছ +##জ +##ট +##ড +##ণ +##ত +##থ +##দ +##ধ +##ন +##প +##ব +##ভ +##ম +##য +##র +##ল +##শ +##ষ +##স +##হ +##া +##ি +##ী +##ে +##க +##ச +##ட +##த +##ந +##ன +##ப +##ம +##ய +##ர +##ல +##ள +##வ +##ா +##ி +##ு +##ே +##ை +##ನ +##ರ +##ಾ +##ක +##ය +##ර +##ල +##ව +##ා +##ก +##ง +##ต +##ท +##น +##พ +##ม +##ย +##ร +##ล +##ว +##ส +##อ +##า +##เ +##་ +##། +##ག +##ང +##ད +##ན +##པ +##བ +##མ +##འ +##ར +##ལ +##ས +##မ +##ა +##ბ +##გ +##დ +##ე +##ვ +##თ +##ი +##კ +##ლ +##მ +##ნ +##ო +##რ +##ს +##ტ +##უ +##ᄀ +##ᄂ +##ᄃ +##ᄅ +##ᄆ +##ᄇ +##ᄉ +##ᄊ +##ᄋ +##ᄌ +##ᄎ +##ᄏ +##ᄐ +##ᄑ +##ᄒ +##ᅡ +##ᅢ +##ᅥ +##ᅦ +##ᅧ +##ᅩ +##ᅪ +##ᅭ +##ᅮ +##ᅯ +##ᅲ +##ᅳ +##ᅴ +##ᅵ +##ᆨ +##ᆫ +##ᆯ +##ᆷ +##ᆸ +##ᆼ +##ᴬ +##ᴮ +##ᴰ +##ᴵ +##ᴺ +##ᵀ +##ᵃ +##ᵇ +##ᵈ +##ᵉ +##ᵍ +##ᵏ +##ᵐ +##ᵒ +##ᵖ +##ᵗ +##ᵘ +##ᵣ +##ᵤ +##ᵥ +##ᶜ +##ᶠ +##‐ +##‑ +##‒ +##– +##— +##― +##‖ +##‘ +##’ +##‚ +##“ +##” +##„ +##† +##‡ +##• +##… +##‰ +##′ +##″ +##› +##‿ +##⁄ +##⁰ +##ⁱ +##⁴ +##⁵ +##⁶ +##⁷ +##⁸ +##⁹ +##⁻ +##ⁿ +##₅ +##₆ +##₇ +##₈ +##₉ +##₊ +##₍ +##₎ +##ₐ +##ₑ +##ₒ +##ₓ +##ₕ +##ₖ +##ₗ +##ₘ +##ₚ +##ₛ +##ₜ +##₤ +##₩ +##€ +##₱ +##₹ +##ℓ +##№ +##ℝ +##™ +##⅓ +##⅔ +##← +##↑ +##→ +##↓ +##↔ +##↦ +##⇄ +##⇌ +##⇒ +##∂ +##∅ +##∆ +##∇ +##∈ +##∗ +##∘ +##√ +##∞ +##∧ +##∨ +##∩ +##∪ +##≈ +##≡ +##≤ +##≥ +##⊂ +##⊆ +##⊕ +##⊗ +##⋅ +##─ +##│ +##■ +##▪ +##● +##★ +##☆ +##☉ +##♠ +##♣ +##♥ +##♦ +##♯ +##⟨ +##⟩ +##ⱼ +##⺩ +##⺼ +##⽥ +##、 +##。 +##〈 +##〉 +##《 +##》 +##「 +##」 +##『 +##』 +##〜 +##あ +##い +##う +##え +##お +##か +##き +##く +##け +##こ +##さ +##し +##す +##せ +##そ +##た +##ち +##っ +##つ +##て +##と +##な +##に +##ぬ +##ね +##の +##は +##ひ +##ふ +##へ +##ほ +##ま +##み +##む +##め +##も +##や +##ゆ +##よ +##ら +##り +##る +##れ +##ろ +##を +##ん +##ァ +##ア +##ィ +##イ +##ウ +##ェ +##エ +##オ +##カ +##キ +##ク +##ケ +##コ +##サ +##シ +##ス +##セ +##タ +##チ +##ッ +##ツ +##テ +##ト +##ナ +##ニ +##ノ +##ハ +##ヒ +##フ +##ヘ +##ホ +##マ +##ミ +##ム +##メ +##モ +##ャ +##ュ +##ョ +##ラ +##リ +##ル +##レ +##ロ +##ワ +##ン +##・ +##ー +##一 +##三 +##上 +##下 +##不 +##世 +##中 +##主 +##久 +##之 +##也 +##事 +##二 +##五 +##井 +##京 +##人 +##亻 +##仁 +##介 +##代 +##仮 +##伊 +##会 +##佐 +##侍 +##保 +##信 +##健 +##元 +##光 +##八 +##公 +##内 +##出 +##分 +##前 +##劉 +##力 +##加 +##勝 +##北 +##区 +##十 +##千 +##南 +##博 +##原 +##口 +##古 +##史 +##司 +##合 +##吉 +##同 +##名 +##和 +##囗 +##四 +##国 +##國 +##土 +##地 +##坂 +##城 +##堂 +##場 +##士 +##夏 +##外 +##大 +##天 +##太 +##夫 +##奈 +##女 +##子 +##学 +##宀 +##宇 +##安 +##宗 +##定 +##宣 +##宮 +##家 +##宿 +##寺 +##將 +##小 +##尚 +##山 +##岡 +##島 +##崎 +##川 +##州 +##巿 +##帝 +##平 +##年 +##幸 +##广 +##弘 +##張 +##彳 +##後 +##御 +##德 +##心 +##忄 +##志 +##忠 +##愛 +##成 +##我 +##戦 +##戸 +##手 +##扌 +##政 +##文 +##新 +##方 +##日 +##明 +##星 +##春 +##昭 +##智 +##曲 +##書 +##月 +##有 +##朝 +##木 +##本 +##李 +##村 +##東 +##松 +##林 +##森 +##楊 +##樹 +##橋 +##歌 +##止 +##正 +##武 +##比 +##氏 +##民 +##水 +##氵 +##氷 +##永 +##江 +##沢 +##河 +##治 +##法 +##海 +##清 +##漢 +##瀬 +##火 +##版 +##犬 +##王 +##生 +##田 +##男 +##疒 +##発 +##白 +##的 +##皇 +##目 +##相 +##省 +##真 +##石 +##示 +##社 +##神 +##福 +##禾 +##秀 +##秋 +##空 +##立 +##章 +##竹 +##糹 +##美 +##義 +##耳 +##良 +##艹 +##花 +##英 +##華 +##葉 +##藤 +##行 +##街 +##西 +##見 +##訁 +##語 +##谷 +##貝 +##貴 +##車 +##軍 +##辶 +##道 +##郎 +##郡 +##部 +##都 +##里 +##野 +##金 +##鈴 +##镇 +##長 +##門 +##間 +##阝 +##阿 +##陳 +##陽 +##雄 +##青 +##面 +##風 +##食 +##香 +##馬 +##高 +##龍 +##龸 +##fi +##fl +##！ +##（ +##） +##， +##－ +##． +##／ +##： +##？ +##～ diff --git a/new_impl/cv/glip/object_detection/det_baseline.py b/new_impl/cv/glip/object_detection/det_baseline.py new file mode 100644 index 0000000000000000000000000000000000000000..970004849502898d58b236846a963d59e1a15565 --- /dev/null +++ b/new_impl/cv/glip/object_detection/det_baseline.py @@ -0,0 +1,164 @@ +import torch +import torch.nn as nn +from methods.elasticdnn.api.algs.fm_lora import ElasticDNN_FMLoRAAlg +from methods.elasticdnn.api.algs.md_pretraining_wo_fbs import ElasticDNN_MDPretrainingWoFBSAlg +from methods.elasticdnn.model.base import ElasticDNNUtil +from methods.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from glip import ElasticGLIPUtil, FMLoRA_GLIP_Util, FM_to_MD_GLIP_Util, ElasticDNN_OfflineMMDetFMModel, ElasticDNN_OfflineMMDetMDModel +from methods.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from methods.elasticdnn.pipeline.offline.fm_to_md.vit import FM_to_MD_ViT_Util +from methods.elasticdnn.model.vit import ElasticViTUtil +from methods.elasticdnn.api.algs.md_pretraining_index_v2_train_index_and_md import ElasticDNN_MDPretrainingIndexAlg +from utils.dl.common.model import LayerActivation3, get_module, get_parameter +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +import torch.nn.functional as F +from maskrcnn_benchmark.structures.bounding_box import BoxList +import os +from utils.dl.common.loss import CrossEntropyLossSoft +from new_impl.cv.feat_align.main_glip import OnlineFeatAlignModel, FeatAlignAlg +import tqdm +from new_impl.cv.feat_align.mmd import mmd_rbf +from new_impl.cv.utils.baseline_da import baseline_da +from new_impl.cv.elasticdnn.api.online_model_v2 import ElasticDNN_OnlineModel + +os.environ['TOKENIZERS_PARALLELISM'] = 'true' + +torch.cuda.set_device(0) +device = 'cuda' +app_name = 'cls' + +scenario = build_scenario( + source_datasets_name=['MM-COCO2017'], + target_datasets_order=['MM-CityscapesDet', 'MM-GTA5Det'] * 10, + da_mode='close_set', + data_dirs={ + 'MM-COCO2017': '/data/zql/datasets/coco2017', + 'MM-CityscapesDet': '/data/zql/datasets/cityscape', + 'MM-GTA5Det': '/data/zql/datasets/GTA-ls-copy/GTA5', + }, + ) + +class DetOnlineFeatAlignModel(OnlineFeatAlignModel): + def get_trained_params(self): # TODO: elastic fm only train a part of params + #qkv_and_norm_params = [p for n, p in self.models_dict['main'].named_parameters() if 'attention.attention.projection_query' in n or 'attention.attention.projection_key' in n or 'attention.attention.projection_value' in n or 'intermediate.dense' in n or 'output.dense' in n] + qkv_and_norm_params = [p for n, p in self.models_dict['main'].named_parameters()] + return qkv_and_norm_params + + def get_feature_hook(self): + return LayerActivation3(get_module(self.models_dict['main'], 'model.rpn'), False, self.device) + + def forward_to_get_task_loss(self, x, y): + loss_dict = self.infer(x) + losses = sum(loss for loss in loss_dict.values()) + # print(losses) + + return losses + + def get_mmd_loss(self, f1, f2): + # f1 = f1.view(f1.shape[0], -1) + # f2 = f2.view(f2.shape[0], -1) + return mmd_rbf(f1, f2) + + def infer(self, x, *args, **kwargs): + return self.models_dict['main'](**x) + + def get_accuracy(self, test_loader, *args, **kwargs): + # print('DeeplabV3: start test acc') + _d = test_loader.dataset + imgsz = _d.cocods.img_size + cls_num = len(_d.cocods.class_ids) + # num_classes = len(_d.cls_names) + from data import build_dataloader + if _d.__class__.__name__ == 'MergedDataset': + # print('\neval on merged datasets') + datasets = _d.datasets + if self.collate_fn is None: + test_loaders = [build_dataloader(d, test_loader.batch_size, test_loader.num_workers, False, None, collate_fn=None) for d in datasets] + else: + test_loaders = [build_dataloader(d, test_loader.batch_size, test_loader.num_workers, False, None, collate_fn=self.collate_fn) for d in datasets] + accs = [self.get_accuracy(loader) for loader in test_loaders] + # print(accs) + return sum(accs) / len(accs) + + # print('dataset len', len(test_loader.dataset)) + + model = self.models_dict['main'] + device = self.device + model.eval() + + # print('# classes', model.num_classes) + + model = model.to(device) + from evaluator import COCOEvaluator, MMCOCODecoder + from utils.common.others import HiddenPrints + with torch.no_grad(): + with HiddenPrints(): + evaluator = COCOEvaluator( + dataloader=test_loader, + img_size=imgsz, + confthre=0.01, + nmsthre=0.65, + num_classes=cls_num, + testdev=False + ) + res = evaluator.evaluate(model, False, False, decoder=MMCOCODecoder) + map50 = res[1] + # print('eval info', res[-1]) + return map50 + +from glip import glip_model, build_transform, run_ner, collect_mm_fn +cfg_path = 'new_impl/cv/glip/object_detection/pretrained_model/glip_Swin_T_O365_GoldG.yaml' +model_path = 'new_impl/cv/glip/object_detection/pretrained_model/glip_tiny_model_o365_goldg_cc_sbu.pth' +config, _ = glip_model(cfg_path, model_path) +transform = build_transform(config, None) + +da_alg = FeatAlignAlg +from utils.dl.common.lr_scheduler import get_linear_schedule_with_warmup +#from new_impl.cv.model import ClsOnlineFeatAlignModel +da_model = DetOnlineFeatAlignModel( + app_name, + 'new_impl/cv/glip/object_detection/results/det_md_wo_fbs.py/20231129/999999-153230-results/models/md_best.pt', + device +) +da_alg_hyp = { + 'MM-GTA5Det': { + 'train_batch_size': 8, + 'val_batch_size': 1, + 'num_workers': 8, + 'optimizer': 'AdamW', + 'optimizer_args': {'lr': 2e-6, 'betas': [0.9, 0.999], 'weight_decay': 0.01}, + 'scheduler': '', + 'scheduler_args': {}, + 'num_iters': 100, + 'val_freq': 20, + 'feat_align_loss_weight': 0.3, + 'transform':transform + }, + 'MM-CityscapesDet': { + 'train_batch_size': 8, + 'val_batch_size': 1, + 'num_workers': 8, + 'optimizer': 'AdamW', + 'optimizer_args': {'lr': 2e-6, 'betas': [0.9, 0.999], 'weight_decay': 0.01}, + 'scheduler': '', + 'scheduler_args': {}, + 'num_iters': 100, + 'val_freq': 20, + 'feat_align_loss_weight': 0.3, + 'transform':transform + }, +} + + +baseline_da( + app_name, + scenario, + da_alg, + da_alg_hyp, + da_model, + device, + __file__, + "results", + collate_fn=collect_mm_fn +) \ No newline at end of file diff --git a/new_impl/cv/glip/object_detection/det_lora.py b/new_impl/cv/glip/object_detection/det_lora.py new file mode 100644 index 0000000000000000000000000000000000000000..7c8afd4c52399095028d372b36b215d4c93954d5 --- /dev/null +++ b/new_impl/cv/glip/object_detection/det_lora.py @@ -0,0 +1,129 @@ +import torch +from new_impl.cv.elasticdnn.api.algs.fm_lora_glip import ElasticDNN_FMLoRAAlg +from methods.elasticdnn.model.base import ElasticDNNUtil +from methods.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from glip import FMLoRA_GLIP_Util, ElasticDNN_OfflineMMDetFMModel +from methods.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from methods.elasticdnn.pipeline.offline.fm_to_md.vit import FM_to_MD_ViT_Util +from methods.elasticdnn.model.vit import ElasticViTUtil +from utils.dl.common.model import LayerActivation, get_module +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +import torch.nn.functional as F +from maskrcnn_benchmark.structures.bounding_box import BoxList +import os +os.environ['TOKENIZERS_PARALLELISM'] = 'true' + +class ElasticDNN_GLIP_OfflineMMDetFMModel(ElasticDNN_OfflineMMDetFMModel): + def generate_md_by_reducing_width(self, reducing_width_ratio, samples: torch.Tensor): + raise NotImplementedError + + def get_feature_hook(self) -> LayerActivation: + return LayerActivation(get_module(self.models_dict['main'], 'visual_projection'), True, self.device), 'output' + + def get_elastic_dnn_util(self) -> ElasticDNNUtil: + raise NotImplementedError + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + # x: clip-preprocessed images and texts, y: label indexes + x['for_training'] = True + + # for k, v in x.items(): + # if isinstance(v, torch.Tensor): + # print(k, v.size()) + # elif isinstance(v, (list, tuple)): + # print(k, len(v)) + # else: + # print(k, v) + + loss_dict = self.infer(x) + losses = sum(loss for loss in loss_dict.values()) + # print(losses) + + return losses + + def get_lora_util(self) -> FMLoRA_Util: + return FMLoRA_GLIP_Util() + + def get_task_head_params(self): + return [] + + +# class ElasticDNN_CLIP_OfflineMMClsMDModel(ElasticDNN_OfflineMMClsMDModel): +# def get_feature_hook(self) -> LayerActivation: +# return LayerActivation(get_module(self.models_dict['main'], 'visual_projection'), True, self.device), 'output' + +# def forward_to_get_task_loss(self, x, y, *args, **kwargs): +# x['for_training'] = True +# output = self.infer(x) +# return output.loss + + +if __name__ == '__main__': + # 1. init scenario + scenario = build_scenario( + source_datasets_name=['MM-COCO2017'], + target_datasets_order=['MM-CityscapesDet'], + da_mode='close_set', + data_dirs={ + 'MM-COCO2017': '/data/zql/datasets/coco2017', + 'MM-CityscapesDet': '/data/zql/datasets/cityscape' + }, + ) + cfg_path = 'new_impl/cv/glip/object_detection/pretrained_model/glip_Swin_T_O365_GoldG.yaml' + model_path = 'new_impl/cv/glip/object_detection/pretrained_model/glip_tiny_model_o365_goldg_cc_sbu.pth' + # 2. init model + from glip import glip_model, build_transform, run_ner, collect_mm_fn + config, gmodel = glip_model(cfg_path, model_path) + transform = build_transform(config, None) + fm_models_dict_path = save_models_dict_for_init({ + 'main': gmodel + }, __file__, 'fm_glip_pretrained') + device = 'cuda' + + # total_class_to_idx_map = {} + # for v in scenario.all_datasets_e2e_class_to_idx_map.values(): + # for k, v2 in v.items(): + # if k in total_class_to_idx_map.keys(): + # assert total_class_to_idx_map[k] == v2 + # total_class_to_idx_map[k] = v2 + + fm_model = ElasticDNN_GLIP_OfflineMMDetFMModel('fm', fm_models_dict_path, device, collate_fn=collect_mm_fn) + + # 3. init alg + models = { + 'fm': fm_model + } + import sys + fm_lora_alg = ElasticDNN_FMLoRAAlg(models, get_res_save_dir(__file__, tag="results")) + + from PIL import Image, ImageDraw + import requests + import numpy as np + from evaluator import MMCOCODecoder + ori_image = Image.open('new_impl/cv/glip/object_detection/000000103759.jpg').convert("RGB") + image = transform(np.asarray(ori_image)[:, :, [2, 1, 0]]) + text = 'orange. umbrella. ' + targets = BoxList(torch.FloatTensor([[0., 0., 0., 0.]]), image_size=image.size()[1:], mode='xyxy') + targets.add_field('caption', text) + targets.add_field('tokens_positive', run_ner(text)) + targets.add_field('labels', torch.LongTensor([0])) + samples = {'images' : [image], 'targets' : [targets]} + + # 4. run alg + from utils.dl.common.lr_scheduler import get_linear_schedule_with_warmup + fm_lora_alg.run(scenario, hyps={ + 'launch_tbboard': False, + 'transform' : transform, + 'samples_size': samples, + 'ab_r': 8, + 'train_batch_size': 16, + 'val_batch_size': 1, + 'num_workers': 16, + 'optimizer': 'Adam', + 'optimizer_args': {'lr': 1e-4, 'betas': [0.9, 0.999]}, + 'scheduler': 'LambdaLR', + 'scheduler_args': {'lr_lambda': get_linear_schedule_with_warmup(10000, 70000)}, + 'num_iters': 6500, + 'val_freq': 100 + }, collate_fn=collect_mm_fn) \ No newline at end of file diff --git a/new_impl/cv/glip/object_detection/det_md_w_fbs_index.py b/new_impl/cv/glip/object_detection/det_md_w_fbs_index.py new file mode 100644 index 0000000000000000000000000000000000000000..c68f800c0605e872f4a1b4848162ace187e19e06 --- /dev/null +++ b/new_impl/cv/glip/object_detection/det_md_w_fbs_index.py @@ -0,0 +1,231 @@ +import torch +import torch.nn as nn +from methods.elasticdnn.api.algs.fm_lora import ElasticDNN_FMLoRAAlg +from methods.elasticdnn.api.algs.md_pretraining_wo_fbs import ElasticDNN_MDPretrainingWoFBSAlg +from methods.elasticdnn.model.base import ElasticDNNUtil +from methods.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from glip import ElasticGLIPUtil, FMLoRA_GLIP_Util, FM_to_MD_GLIP_Util, ElasticDNN_OfflineMMDetFMModel, ElasticDNN_OfflineMMDetMDModel +from methods.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from methods.elasticdnn.pipeline.offline.fm_to_md.vit import FM_to_MD_ViT_Util +from methods.elasticdnn.model.vit import ElasticViTUtil +from new_impl.cv.elasticdnn.api.algs.md_pretraining_index_v2_train_index_and_md_glip import ElasticDNN_MDPretrainingIndexAlg +from utils.dl.common.model import LayerActivation3, get_module, get_parameter +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +import torch.nn.functional as F +from maskrcnn_benchmark.structures.bounding_box import BoxList +import os +from utils.dl.common.loss import CrossEntropyLossSoft + +os.environ['TOKENIZERS_PARALLELISM'] = 'true' + + +class ElasticDNN_GLIP_OfflineMMDetFMModel(ElasticDNN_OfflineMMDetFMModel): + def generate_md_by_reducing_width(self, reducing_width_ratio, samples: torch.Tensor): + return FM_to_MD_GLIP_Util().init_md_from_fm_by_reducing_width_with_perf_test_2(self.models_dict['main'], + reducing_width_ratio, samples).to(self.device) + + def get_feature_hook(self) -> LayerActivation3: + return LayerActivation3(get_module(self.models_dict['main'], 'model.rpn'), True, self.device) + + def get_elastic_dnn_util(self) -> ElasticDNNUtil: + return ElasticGLIPUtil() + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + loss_dict = self.infer(x) + losses = sum(loss for loss in loss_dict.values()) + # print(losses) + + return losses + + def get_lora_util(self) -> FMLoRA_Util: + return FMLoRA_GLIP_Util() + + def get_task_head_params(self): + head = get_module(self.models_dict['main'], 'classifier') + return list(head.parameters()) + + +class ElasticDNN_GLIP_OfflineMMDetMDModel(ElasticDNN_OfflineMMDetMDModel): + def __init__(self, name: str, models_dict_path: str, device: str, collate_fn=None): + super().__init__(name, models_dict_path, device, collate_fn=collate_fn) + + self.distill_criterion = CrossEntropyLossSoft() + + def get_feature_hook(self) -> LayerActivation3: + return LayerActivation3(get_module(self.models_dict['main'], 'model.rpn'), True, self.device) + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + loss_dict = self.infer(x) + losses = sum(loss for loss in loss_dict.values()) + # print(losses) + + return losses + + def get_distill_loss(self, student_output, teacher_output): + + return self.distill_criterion(student_output, teacher_output) + def get_matched_param_of_fm(self, self_param_name, fm: nn.Module): + # only between qkv.weight, norm.weight/bias + if any([k in self_param_name for k in ['fbs', 'cls_token', 'pos_embed', 'conv']]): + return None + + p = get_parameter(self.models_dict['main'], self_param_name) + + if p.dim() == 0: + return None + # elif p.dim() == 1 and 'norm' in self_param_name and 'weight' in self_param_name: + # return get_parameter(fm, self_param_name) + + # 1. xx.qkv.to_qkv.yy to xx.qkv.qkv.aa and xx.qkv.abs.zz + if ('query' in self_param_name or 'key' in self_param_name or \ + 'value' in self_param_name) and ('weight' in self_param_name): + ss = self_param_name.split('.') + # raise NotImplementedError() # TODO: + fm_qkv_name = '.'.join(ss[0: -1]) + '.fc' + fm_qkv = get_module(fm, fm_qkv_name) + + fm_abs_name = '.'.join(ss[0: -1]) + '.ab' + fm_abs = get_module(fm, fm_abs_name) + + return torch.cat([ + fm_qkv.weight.data, # task-agnositc params + fm_abs[1].weight @ fm_abs[0].weight + ], dim=0) + + elif ('query' in self_param_name or 'key' in self_param_name or \ + 'value' in self_param_name) and ('bias' in self_param_name): + ss = self_param_name.split('.') + + fm_qkv_name = '.'.join(ss[0: -1]) + '.fc.bias' + return get_parameter(fm, fm_qkv_name) + + elif ('attn.qkv' in self_param_name or \ + 'attn.v_proj' in self_param_name or 'attn.l_proj' in self_param_name or 'attn.values_v_proj' in self_param_name or 'attn.values_l_proj' in self_param_name) and ('weight' in self_param_name): + ss = self_param_name.split('.') + + fm_qkv_name = '.'.join(ss[0: -1]) + '.fc' + fm_qkv = get_module(fm, fm_qkv_name) + + fm_abs_name = '.'.join(ss[0: -1]) + '.ab' + fm_abs = [get_module(fm, fm_abs_name)] + + return torch.cat([ + fm_qkv.weight.data, # task-agnositc params + torch.cat([(_abs[0].weight.T @ _abs[1].weight.T).T for _abs in fm_abs], dim=0) # task-specific params (LoRA) + ], dim=0) + + # elif 'to_qkv.bias' in self_param_name: + # ss = self_param_name.split('.') + + # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv.bias' + # return get_parameter(fm, fm_qkv_name) + elif ('attn.qkv' in self_param_name or \ + 'attn.v_proj' in self_param_name or 'attn.l_proj' in self_param_name or 'attn.values_v_proj' in self_param_name or 'attn.values_l_proj' in self_param_name) and ('bias' in self_param_name): + ss = self_param_name.split('.') + + fm_qkv_name = '.'.join(ss[0: -1]) + '.fc.bias' + return get_parameter(fm, fm_qkv_name) + + elif ('intermediate.dense' in self_param_name or 'mlp.fc1' in self_param_name) and ('weight' in self_param_name or 'bias' in self_param_name): + fm_param_name = self_param_name.replace('.linear', '') + return get_parameter(fm, fm_param_name) + + else: + # return get_parameter(fm, self_param_name) + return None + + +if __name__ == '__main__': + from utils.dl.common.env import set_random_seed + set_random_seed(1) + + # 1. init model + from dnns.vit import vit_b_16 + fm_models_dict_path = 'new_impl/cv/glip/object_detection/results/det_md_wo_fbs.py/20231129/999999-153230-results/models/fm_best.pt' + fm_models_dict_path = save_models_dict_for_init(torch.load(fm_models_dict_path), __file__, 'fm_glip_cls_lora') + pretrained_md_models_dict_path = 'new_impl/cv/glip/object_detection/results/det_md_wo_fbs.py/20231129/999999-153230-results/models/md_best.pt' + md_models_dict = torch.load(pretrained_md_models_dict_path) + md_models_dict_path = save_models_dict_for_init(md_models_dict, __file__, 'md_glip_cls_pretrained_wo_fbs') + torch.cuda.set_device(1) + device = 'cuda' + + from glip import glip_model, build_transform, run_ner, collect_mm_fn + + fm_model = ElasticDNN_GLIP_OfflineMMDetFMModel('fm', fm_models_dict_path, device, collate_fn=collect_mm_fn) + md_model = ElasticDNN_GLIP_OfflineMMDetMDModel('md', md_models_dict_path, device, collate_fn=collect_mm_fn) + + # 2. init alg + models = { + 'fm': fm_model, + 'md': md_model + } + + fm_to_md_alg = ElasticDNN_MDPretrainingIndexAlg(models, get_res_save_dir(__file__, "results")) + + # 3. init scenario + scenario = build_scenario( + source_datasets_name=['MM-COCO2017'], + target_datasets_order=['MM-CityscapesDet'], + da_mode='close_set', + data_dirs={ + 'MM-COCO2017': '/data/zql/datasets/coco2017', + 'MM-CityscapesDet': '/data/zql/datasets/cityscape' + }, + ) + + from utils.dl.common.lr_scheduler import get_linear_schedule_with_warmup + from PIL import Image, ImageDraw + import requests + import numpy as np + from evaluator import MMCOCODecoder + cfg_path = 'new_impl/cv/glip/object_detection/pretrained_model/glip_Swin_T_O365_GoldG.yaml' + model_path = 'new_impl/cv/glip/object_detection/pretrained_model/glip_tiny_model_o365_goldg_cc_sbu.pth' + config, _ = glip_model(cfg_path, model_path) + transform = build_transform(config, None) + ori_image = Image.open('new_impl/cv/glip/object_detection/000000103759.jpg').convert("RGB") + image = transform(np.asarray(ori_image)[:, :, [2, 1, 0]]) + text = 'orange. umbrella. ' + targets = BoxList(torch.FloatTensor([[0., 0., 0., 0.]]), image_size=image.size()[1:], mode='xyxy') + targets.add_field('caption', text) + targets.add_field('tokens_positive', run_ner(text)) + targets.add_field('labels', torch.LongTensor([0])) + samples = {'images' : [image], 'targets' : [targets]} + + fm_to_md_alg.run(scenario, hyps={ + 'launch_tbboard': False, + + 'samples_size': samples, + + 'FBS_r': 16, + 'FBS_ignore_layers': [], + + 'train_batch_size': 16, + 'val_batch_size': 1, + 'num_workers': 16, + 'optimizer': 'AdamW', + # 'optimizer_args': {'lr': 1e-4, 'betas': [0.9, 0.999], 'weight_decay': 0.01}, + 'indexes_optimizer_args': {'lr': 3e-4, 'betas': [0.9, 0.999], 'weight_decay': 0.1}, + # 'scheduler': 'StepLR', + # 'scheduler_args': {'step_size': 20000, 'gamma': 0.1}, + # 'optimizer': 'AdamW', + # 'optimizer_args': {'lr': 1e-4, 'betas': [0.9, 0.999], 'weight_decay': 0.01}, + 'optimizer_args': {'lr': 2e-4, 'betas': [0.9, 0.999], 'weight_decay': 0.01},#注意学习率的调整，不同的模型不一样。 + 'transform' : transform, + 'scheduler': 'LambdaLR', + 'scheduler_args': {'lr_lambda': get_linear_schedule_with_warmup(10000, 70000)}, + + 'max_sparsity': 0.9, + 'min_sparsity': 0.3, + 'num_iters': 9000, + 'val_freq': 100, + 'index_loss_weight': 1e-4, + 'l1_reg_loss_weight': 1e-9, + 'val_num_sparsities': 4, + + 'bn_cal_num_iters': 0,#有bn层注意需要加上这个 + + 'index_init': 'zero', + 'index_guided_linear_comb_split_size': 16 + }, collate_fn=collect_mm_fn) + \ No newline at end of file diff --git a/new_impl/cv/glip/object_detection/det_md_wo_fbs.py b/new_impl/cv/glip/object_detection/det_md_wo_fbs.py new file mode 100644 index 0000000000000000000000000000000000000000..097d3765247ab1800a295ee41f9ff0d6c4b7d4ef --- /dev/null +++ b/new_impl/cv/glip/object_detection/det_md_wo_fbs.py @@ -0,0 +1,240 @@ +import torch +import torch.nn as nn +from methods.elasticdnn.api.algs.fm_lora import ElasticDNN_FMLoRAAlg +from new_impl.cv.elasticdnn.api.algs.md_pretraining_wo_fbs_glip import ElasticDNN_MDPretrainingWoFBSAlg +from methods.elasticdnn.model.base import ElasticDNNUtil +from methods.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from glip import FMLoRA_GLIP_Util, FM_to_MD_GLIP_Util, ElasticDNN_OfflineMMDetFMModel, ElasticDNN_OfflineMMDetMDModel +from methods.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from methods.elasticdnn.pipeline.offline.fm_to_md.vit import FM_to_MD_ViT_Util +from methods.elasticdnn.model.vit import ElasticViTUtil +from utils.dl.common.model import LayerActivation3, get_module +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +import torch.nn.functional as F +from maskrcnn_benchmark.structures.bounding_box import BoxList +import os +from utils.dl.common.loss import CrossEntropyLossSoft +os.environ['TOKENIZERS_PARALLELISM'] = 'true' + +class ElasticDNN_GLIP_OfflineMMDetFMModel(ElasticDNN_OfflineMMDetFMModel): + def generate_md_by_reducing_width(self, reducing_width_ratio, samples: torch.Tensor): + return FM_to_MD_GLIP_Util().init_md_from_fm_by_reducing_width_with_perf_test_2(self.models_dict['main'], + reducing_width_ratio, samples).to(self.device) + def get_feature_hook(self): + return None + + def get_feature_hooks(self): + res = {} + res['vis'] = LayerActivation3(get_module(self.models_dict['main'], 'model.backbone'), True, self.device) + res['lang'] = LayerActivation3(get_module(self.models_dict['main'], 'model.language_backbone'), True, self.device) + res['cls_and_reg'] = LayerActivation3(get_module(self.models_dict['main'], 'model.rpn.head'), True, self.device) + return res + + def get_elastic_dnn_util(self) -> ElasticDNNUtil: + return ElasticclipUtil() + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + loss_dict = self.infer(x) + losses = sum(loss for loss in loss_dict.values()) + # print(losses) + + return losses + + def get_lora_util(self) -> FMLoRA_Util: + return FMLoRA_GLIP_Util() + + def get_task_head_params(self): + head = get_module(self.models_dict['main'], 'classifier') + return list(head.parameters()) + + +class ElasticDNN_GLIP_OfflineMMDetMDModel(ElasticDNN_OfflineMMDetMDModel): + def __init__(self, name: str, models_dict_path: str, device: str, collate_fn=None): + super().__init__(name, models_dict_path, device, collate_fn=collate_fn) + self.l2loss = nn.MSELoss() + self.distill_criterion = CrossEntropyLossSoft() + + def get_feature_hook(self): + return None + + def get_feature_hooks(self): + res = {} + res['vis'] = LayerActivation3(get_module(self.models_dict['main'], 'model.backbone'), True, self.device) + res['lang'] = LayerActivation3(get_module(self.models_dict['main'], 'model.language_backbone'), True, self.device) + res['cls_and_reg'] = LayerActivation3(get_module(self.models_dict['main'], 'model.rpn.head'), True, self.device) + return res + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + loss_dict = self.infer(x) + losses = sum(loss for loss in loss_dict.values()) + # print(losses) + + return losses + + def get_distill_loss(self, student_output, teacher_output): + md_output_vis = student_output['vis'] + fm_output_vis = teacher_output['vis'] + + md_output_vis = torch.cat([vis.view(vis.shape[0], 256, -1) for vis in md_output_vis], dim=2) + fm_output_vis = torch.cat([vis.view(vis.shape[0], 256, -1) for vis in fm_output_vis], dim=2) + + vis_loss = self.l2loss(md_output_vis, fm_output_vis) / 10000.0 + + md_output_lang = student_output['lang'] + fm_output_lang = teacher_output['lang'] + + md_output_lang_agg = md_output_lang['aggregate'] + md_output_lang_emb = md_output_lang['embedded'] + md_output_lang_hid = md_output_lang['hidden'] + + fm_output_lang_agg = fm_output_lang['aggregate'] + fm_output_lang_emb = fm_output_lang['embedded'] + fm_output_lang_hid = fm_output_lang['hidden'] + + lang_loss = self.l2loss(md_output_lang_agg, fm_output_lang_agg) + self.l2loss(md_output_lang_emb, fm_output_lang_emb) + self.l2loss(md_output_lang_hid, fm_output_lang_hid) + + md_output_cls = student_output['cls'] + fm_output_cls = teacher_output['cls'] + + md_output_cls = torch.cat(md_output_cls, dim=1) + fm_output_cls = torch.cat(fm_output_cls, dim=1) + + md_output_cls = md_output_cls.view(-1, 256) + fm_output_cls = fm_output_cls.view(-1, 256) + + cls_loss = self.distill_criterion(md_output_cls, fm_output_cls) + + md_output_reg = student_output['reg'] + fm_output_reg = teacher_output['reg'] + + md_output_reg = torch.cat([reg.view(reg.shape[0], 4, -1) for reg in md_output_reg], dim=2) + fm_output_reg = torch.cat([reg.view(reg.shape[0], 4, -1) for reg in fm_output_reg], dim=2) + + reg_loss = self.l2loss(md_output_reg, fm_output_reg) + + loss = vis_loss + lang_loss + cls_loss + reg_loss + + return loss + + + def get_matched_param_of_fm(self, self_param_name, fm: nn.Module): + # only between qkv.weight, norm.weight/bias + if any([k in self_param_name for k in ['fbs', 'cls_token', 'pos_embed']]): + return None + + p = get_parameter(self.models_dict['main'], self_param_name) + if p.dim() == 0: + return None + elif p.dim() == 1 and 'norm' in self_param_name and 'weight' in self_param_name: + return get_parameter(fm, self_param_name) + + # 1. xx.qkv.to_qkv.yy to xx.qkv.qkv.aa and xx.qkv.abs.zz + if ('attention.attention.query' in self_param_name or 'attention.attention.key' in self_param_name or \ + 'attention.attention.value' in self_param_name) and ('weight' in self_param_name): + ss = self_param_name.split('.') + + fm_qkv_name = '.'.join(ss[0: -1]) + '.qkv' + fm_qkv = get_module(fm, fm_qkv_name) + + fm_abs_name = '.'.join(ss[0: -1]) + '.abs' + fm_abs = get_module(fm, fm_abs_name) + + return torch.cat([ + fm_qkv.weight.data, # task-agnositc params + torch.cat([(_abs[0].weight.T @ _abs[1].weight.T).T for _abs in fm_abs], dim=0) # task-specific params (LoRA) + ], dim=0) + + # elif 'to_qkv.bias' in self_param_name: + # ss = self_param_name.split('.') + + # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv.bias' + # return get_parameter(fm, fm_qkv_name) + elif ('attention.attention.query' in self_param_name or 'attention.attention.key' in self_param_name or \ + 'attention.attention.value' in self_param_name) and ('bias' in self_param_name): + ss = self_param_name.split('.') + + fm_qkv_name = '.'.join(ss[0: -1]) + '.qkv.bias' + return get_parameter(fm, fm_qkv_name) + + elif 'intermediate.dense' in self_param_name and 'weight' in self_param_name: + fm_param_name = self_param_name.replace('.linear', '') + return get_parameter(fm, fm_param_name) + + else: + return get_parameter(fm, self_param_name) + #return None + + +if __name__ == '__main__': + from utils.dl.common.env import set_random_seed + set_random_seed(1) + torch.cuda.set_device(0) + # 1. init model + #from dnns.vit import vit_b_16 + fm_models_dict_path = 'new_impl/cv/glip/object_detection/results/det_lora.py/20231128/999999-033639-results/models/fm_best.pt' + fm_models_dict = torch.load(fm_models_dict_path) + #fm_models_dict['main'] = make_softmax_prunable(fm_models_dict['main']) + fm_models_dict_path = save_models_dict_for_init(fm_models_dict, __file__, 'fm_glip_cls_lora') + md_models_dict_path = save_models_dict_for_init({ + 'main': -1 + }, __file__, 'md_glip_cls_lora') + + device = 'cuda' + from glip import glip_model, build_transform, run_ner, collect_mm_fn + fm_model = ElasticDNN_GLIP_OfflineMMDetFMModel('fm', fm_models_dict_path, device, collate_fn=collect_mm_fn) + md_model = ElasticDNN_GLIP_OfflineMMDetMDModel('md', md_models_dict_path, device, collate_fn=collect_mm_fn) + + # 2. init alg + models = { + 'fm': fm_model, + 'md': md_model + } + fm_to_md_alg = ElasticDNN_MDPretrainingWoFBSAlg(models, get_res_save_dir(__file__, "results")) + + # 3. init scenario + scenario = build_scenario( + source_datasets_name=['MM-COCO2017'], + target_datasets_order=['MM-CityscapesDet'], + da_mode='close_set', + data_dirs={ + 'MM-COCO2017': '/data/zql/datasets/coco2017', + 'MM-CityscapesDet': '/data/zql/datasets/cityscape' + }, + ) + + from PIL import Image, ImageDraw + import requests + import numpy as np + from evaluator import MMCOCODecoder + cfg_path = 'new_impl/cv/glip/object_detection/pretrained_model/glip_Swin_T_O365_GoldG.yaml' + model_path = 'new_impl/cv/glip/object_detection/pretrained_model/glip_tiny_model_o365_goldg_cc_sbu.pth' + config, _ = glip_model(cfg_path, model_path) + transform = build_transform(config, None) + ori_image = Image.open('new_impl/cv/glip/object_detection/000000103759.jpg').convert("RGB") + image = transform(np.asarray(ori_image)[:, :, [2, 1, 0]]) + text = 'orange. umbrella. ' + targets = BoxList(torch.FloatTensor([[0., 0., 0., 0.]]), image_size=image.size()[1:], mode='xyxy') + targets.add_field('caption', text) + targets.add_field('tokens_positive', run_ner(text)) + targets.add_field('labels', torch.LongTensor([0])) + samples = {'images' : [image], 'targets' : [targets]} + + from utils.dl.common.lr_scheduler import get_linear_schedule_with_warmup + fm_to_md_alg.run(scenario, hyps={ + 'launch_tbboard': False, + + 'samples_size': samples, + 'generate_md_width_ratio': 4, + 'transform' : transform, + 'train_batch_size': 1, + 'val_batch_size': 1, + 'num_workers': 16, + 'optimizer': 'AdamW', + 'optimizer_args': {'lr': 1e-4, 'betas': [0.9, 0.999], 'weight_decay': 0.01}, + 'scheduler': 'LambdaLR', + 'scheduler_args': {'lr_lambda': get_linear_schedule_with_warmup(10000, 70000)}, + 'num_iters': 50000, + 'val_freq': 3000, + 'distill_loss_weight': 1.0 + }, collate_fn=collect_mm_fn) \ No newline at end of file diff --git a/new_impl/cv/glip/object_detection/det_online.py b/new_impl/cv/glip/object_detection/det_online.py new file mode 100644 index 0000000000000000000000000000000000000000..db7a2405986caa7cf037da09ee31bd00b0ea6137 --- /dev/null +++ b/new_impl/cv/glip/object_detection/det_online.py @@ -0,0 +1,406 @@ +import torch +import torch.nn as nn +from methods.elasticdnn.api.algs.fm_lora import ElasticDNN_FMLoRAAlg +from methods.elasticdnn.api.algs.md_pretraining_wo_fbs import ElasticDNN_MDPretrainingWoFBSAlg +from methods.elasticdnn.model.base import ElasticDNNUtil +from methods.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from glip import ElasticGLIPUtil, FMLoRA_GLIP_Util, FM_to_MD_GLIP_Util, ElasticDNN_OfflineMMDetFMModel, ElasticDNN_OfflineMMDetMDModel +from methods.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from methods.elasticdnn.pipeline.offline.fm_to_md.vit import FM_to_MD_ViT_Util +from methods.elasticdnn.model.vit import ElasticViTUtil +from methods.elasticdnn.api.algs.md_pretraining_index_v2_train_index_and_md import ElasticDNN_MDPretrainingIndexAlg +from utils.dl.common.model import LayerActivation3, get_module, get_parameter +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +import torch.nn.functional as F +from maskrcnn_benchmark.structures.bounding_box import BoxList +import os +from utils.dl.common.loss import CrossEntropyLossSoft +from new_impl.cv.feat_align.main_glip import OnlineFeatAlignModel, FeatAlignAlg +import tqdm +from new_impl.cv.feat_align.mmd import mmd_rbf +from new_impl.cv.utils.elasticfm_da import init_online_model, elasticfm_da +from new_impl.cv.elasticdnn.api.online_model_v2 import ElasticDNN_OnlineModel +from utils.common.log import logger + +os.environ['TOKENIZERS_PARALLELISM'] = 'true' + +torch.cuda.set_device(0) +device = 'cuda' +app_name = 'cls' +sd_sparsity = 0.8 + +settings = { + 'involve_fm': True +} + +scenario = build_scenario( + source_datasets_name=['MM-COCO2017'], + target_datasets_order=['MM-CityscapesDet', 'MM-GTA5Det'] * 10, + da_mode='close_set', + data_dirs={ + 'MM-COCO2017': '/data/zql/datasets/coco2017', + 'MM-CityscapesDet': '/data/zql/datasets/cityscape', + 'MM-GTA5Det': '/data/zql/datasets/GTA-ls-copy/GTA5', + }, + ) + +class ElasticDNN_DetOnlineModel(ElasticDNN_OnlineModel): + def get_accuracy(self, test_loader, *args, **kwargs): + acc = 0 + sample_num = 0 + + self.to_eval_mode() + + with torch.no_grad(): + pbar = tqdm.tqdm(enumerate(test_loader), total=len(test_loader), dynamic_ncols=True, leave=False) + for batch_index, (x, y) in pbar: + x, y = x.to(self.device), y.to(self.device) + output = self.infer(x) + pred = F.softmax(output, dim=1).argmax(dim=1) + correct = torch.eq(pred, y).sum().item() + acc += correct + sample_num += len(y) + + pbar.set_description(f'cur_batch_total: {len(y)}, cur_batch_correct: {correct}, ' + f'cur_batch_acc: {(correct / len(y)):.4f}') + + acc /= sample_num + return acc + + def get_elastic_dnn_util(self) -> ElasticDNNUtil: + return ElasticGLIPUtil() + + def get_fm_matched_param_of_md_param(self, md_param_name): + # only between qkv.weight, norm.weight/bias + self_param_name = md_param_name + fm = self.models_dict['fm'] + # if any([k in self_param_name for k in ['fbs', 'cls_token', 'pos_embed']]): + # return None + + # p = get_parameter(self.models_dict['md'], self_param_name) + # if p.dim() == 0: + # return None + # elif p.dim() == 1 and 'norm' in self_param_name and 'weight' in self_param_name: + # return get_parameter(fm, self_param_name) + + if any([k in self_param_name for k in ['fbs', 'cls_token', 'pos_embed']]): + return None + + p = get_parameter(self.models_dict['md'], self_param_name) + if p.dim() == 0: + return None + # elif p.dim() == 1 and 'layernorm' in self_param_name and 'weight' in self_param_name: + # return get_parameter(fm, self_param_name) + # 1. xx.qkv.to_qkv.yy to xx.qkv.qkv.aa and xx.qkv.abs.zz + # if 'qkv.weight' in self_param_name: + # ss = self_param_name.split('.') + + # fm_qkv_name = '.'.join(ss[0: -1]) + '.qkv' + # fm_qkv = get_module(fm, fm_qkv_name) + + # fm_abs_name = '.'.join(ss[0: -1]) + '.abs' + # fm_abs = get_module(fm, fm_abs_name) + + # # NOTE: unrecoverable operation! multiply LoRA parameters to allow it being updated in update_fm_param() + # # TODO: if fm will be used for inference, _mul_lora_weight will not be applied! + # if not hasattr(fm_abs, '_mul_lora_weight'): + # logger.debug(f'set _mul_lora_weight in {fm_abs_name}') + # setattr(fm_abs, '_mul_lora_weight', + # nn.Parameter(torch.cat([(_abs[0].weight.T @ _abs[1].weight.T).T for _abs in fm_abs], dim=0))) + + # return torch.cat([ + # fm_qkv.weight.data, # task-agnositc params + # fm_abs._mul_lora_weight.data # task-specific params (LoRA) + # ], dim=0) + + # # elif 'to_qkv.bias' in self_param_name: + # # ss = self_param_name.split('.') + + # # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv.bias' + # # return get_parameter(fm, fm_qkv_name) + + # elif 'mlp.fc1' in self_param_name and 'weight' in self_param_name: + # fm_param_name = self_param_name.replace('.linear', '') + # return get_parameter(fm, fm_param_name) + + # elif 'mlp.fc2' in self_param_name and 'weight' in self_param_name: + # fm_param_name = self_param_name + # return get_parameter(fm, fm_param_name) + + # else: + # # return get_parameter(fm, self_param_name) + # return None + if ('attn.qkv' in self_param_name or \ + 'attn.v_proj' in self_param_name or 'attn.l_proj' in self_param_name or 'attn.values_v_proj' in self_param_name or 'attn.values_l_proj' in self_param_name) and ('weight' in self_param_name): + ss = self_param_name.split('.') + + fm_qkv_name = '.'.join(ss[0: -1]) + '.fc' + fm_qkv = get_module(fm, fm_qkv_name) + + fm_abs_name = '.'.join(ss[0: -1]) + '.ab' + fm_abs = get_module(fm, fm_abs_name) + + if not hasattr(fm_abs, '_mul_lora_weight'): + logger.debug(f'set _mul_lora_weight in {fm_abs_name}') + setattr(fm_abs, '_mul_lora_weight', + nn.Parameter(fm_abs[1].weight @ fm_abs[0].weight)) + + return torch.cat([ + fm_qkv.weight.data, # task-agnositc params + fm_abs._mul_lora_weight.data # task-specific params (LoRA) + ], dim=0) + + elif ('attn.qkv' in self_param_name or \ + 'attn.v_proj' in self_param_name or 'attn.l_proj' in self_param_name or 'attn.values_v_proj' in self_param_name or 'attn.values_l_proj' in self_param_name) and ('bias' in self_param_name): + ss = self_param_name.split('.') + + fm_qkv_name = '.'.join(ss[0: -1]) + '.fc.bias' + return get_parameter(fm, fm_qkv_name) + + elif ('query' in self_param_name or 'key' in self_param_name or \ + 'value' in self_param_name) and ('weight' in self_param_name): + ss = self_param_name.split('.') + + fm_qkv_name = '.'.join(ss[0: -1]) + '.fc' + fm_qkv = get_module(fm, fm_qkv_name) + + fm_abs_name = '.'.join(ss[0: -1]) + '.ab' + fm_abs = get_module(fm, fm_abs_name) + + if not hasattr(fm_abs, '_mul_lora_weight'): + logger.debug(f'set _mul_lora_weight in {fm_abs_name}') + setattr(fm_abs, '_mul_lora_weight', + nn.Parameter(fm_abs[1].weight @ fm_abs[0].weight)) + + return torch.cat([ + fm_qkv.weight.data, # task-agnositc params + fm_abs._mul_lora_weight.data # task-specific params (LoRA) + ], dim=0) + + elif ('query' in self_param_name or 'key' in self_param_name or \ + 'value' in self_param_name) and ('bias' in self_param_name): + ss = self_param_name.split('.') + + fm_qkv_name = '.'.join(ss[0: -1]) + '.fc.bias' + return get_parameter(fm, fm_qkv_name) + + elif 'mlp.fc1' in self_param_name and 'weight' in self_param_name: + fm_param_name = self_param_name.replace('.linear', '') + return get_parameter(fm, fm_param_name) + # elif 'mlp.fc2' in self_param_name and 'weight' in self_param_name: + # fm_param_name = self_param_name.replace('.linear', '') + # return get_parameter(fm, fm_param_name) + else: + #return get_parameter(fm, self_param_name) + return None + + + def update_fm_param(self, md_param_name, cal_new_fm_param_by_md_param): + if not ('attn.qkv.weight' in md_param_name or 'attn.v_proj.weight' in md_param_name or \ + 'attn.l_proj.weight' in md_param_name or 'attn.values_v_proj.weight' in md_param_name or \ + 'attn.values_l_proj.weight' in md_param_name or 'query.weight' in md_param_name or 'key.weight' in md_param_name or \ + 'value.weight' in md_param_name): + matched_fm_param_ref = self.get_fm_matched_param_of_md_param(md_param_name) + matched_fm_param_ref.copy_(cal_new_fm_param_by_md_param) + else: + new_fm_attn_weight, new_fm_lora_weight = torch.chunk(cal_new_fm_param_by_md_param, 2, 0) + ss = md_param_name.split('.') + fm = self.models_dict['fm'] + # update task-agnostic parameters + fm_qkv_name = '.'.join(ss[0: -1]) + '.fc' + fm_qkv = get_module(fm, fm_qkv_name) + fm_qkv.weight.data.copy_(new_fm_attn_weight) + + # update task-specific parameters + fm_abs_name = '.'.join(ss[0: -1]) + '.ab' + fm_abs = get_module(fm, fm_abs_name) + fm_abs._mul_lora_weight.data.copy_(new_fm_lora_weight) # TODO: this will not be applied in inference! + + def get_md_matched_param_of_fm_param(self, fm_param_name): + return super().get_md_matched_param_of_fm_param(fm_param_name) + + def get_md_matched_param_of_sd_param(self, sd_param_name): + # raise NotImplementedError + + # only between qkv.weight, norm.weight/bias + self_param_name = sd_param_name + md = self.models_dict['md'] + if any([k in self_param_name for k in ['fbs', 'ab', 'embeddings']]): + return None + + p = get_parameter(self.models_dict['sd'], self_param_name) + if p.dim() == 0: + return None + elif p.dim() == 1 and ('LayerNorm' in self_param_name or 'layernorm' in self_param_name) and 'weight' in self_param_name: + return get_parameter(md, self_param_name) + + # 1. xx.qkv.to_qkv.yy to xx.qkv.qkv.aa and xx.qkv.abs.zz + if ('attn.qkv' in sd_param_name or 'attn.v_proj' in sd_param_name or \ + 'attn.l_proj' in sd_param_name or 'attn.values_v_proj' in sd_param_name or \ + 'attn.values_l_proj' in sd_param_name or 'query' in sd_param_name or 'key' in sd_param_name or \ + 'value' in sd_param_name) and ('weight' in self_param_name): + + + return get_parameter(md, self_param_name) # NOTE: no fbs in qkv! + + # elif 'to_qkv.bias' in self_param_name: + # ss = self_param_name.split('.') + + # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv.bias' + # return get_parameter(fm, fm_qkv_name) + + elif 'mlp.fc1.0.weight' in self_param_name: + fm_param_name = '.'.join(self_param_name.split('.')[0: -2]) + '.linear.weight' + return get_parameter(md, fm_param_name) + + elif 'mlp.fc2' in self_param_name and 'weight' in self_param_name: + fm_param_name = self_param_name + return get_parameter(md, fm_param_name) + + else: + # return get_parameter(fm, self_param_name) + return None + + def get_task_head_params(self): + head = get_module(self.models_dict['sd'], 'classifier') + return list(head.parameters()) + + + +class DetOnlineFeatAlignModel(OnlineFeatAlignModel): + def get_trained_params(self): # TODO: elastic fm only train a part of params + #qkv_and_norm_params = [p for n, p in self.models_dict['main'].named_parameters() if 'attention.attention.projection_query' in n or 'attention.attention.projection_key' in n or 'attention.attention.projection_value' in n or 'intermediate.dense' in n or 'output.dense' in n] + qkv_and_norm_params = [p for n, p in self.models_dict['main'].named_parameters()] + return qkv_and_norm_params + + def get_feature_hook(self): + return LayerActivation3(get_module(self.models_dict['main'], 'model.rpn'), False, self.device) + + def forward_to_get_task_loss(self, x, y): + loss_dict = self.infer(x) + losses = sum(loss for loss in loss_dict.values()) + # print(losses) + + return losses + + def get_mmd_loss(self, f1, f2): + # f1 = f1.view(f1.shape[0], -1) + # f2 = f2.view(f2.shape[0], -1) + return mmd_rbf(f1, f2) + + def infer(self, x, *args, **kwargs): + return self.models_dict['main'](**x) + + def get_accuracy(self, test_loader, *args, **kwargs): + # print('DeeplabV3: start test acc') + _d = test_loader.dataset + imgsz = _d.cocods.img_size + cls_num = len(_d.cocods.class_ids) + # num_classes = len(_d.cls_names) + from data import build_dataloader + if _d.__class__.__name__ == 'MergedDataset': + # print('\neval on merged datasets') + datasets = _d.datasets + if self.collate_fn is None: + test_loaders = [build_dataloader(d, test_loader.batch_size, test_loader.num_workers, False, None, collate_fn=None) for d in datasets] + else: + test_loaders = [build_dataloader(d, test_loader.batch_size, test_loader.num_workers, False, None, collate_fn=self.collate_fn) for d in datasets] + accs = [self.get_accuracy(loader) for loader in test_loaders] + # print(accs) + return sum(accs) / len(accs) + + # print('dataset len', len(test_loader.dataset)) + + model = self.models_dict['main'] + device = self.device + model.eval() + + # print('# classes', model.num_classes) + + model = model.to(device) + from evaluator import COCOEvaluator, MMCOCODecoder + from utils.common.others import HiddenPrints + with torch.no_grad(): + with HiddenPrints(): + evaluator = COCOEvaluator( + dataloader=test_loader, + img_size=imgsz, + confthre=0.01, + nmsthre=0.65, + num_classes=cls_num, + testdev=False + ) + res = evaluator.evaluate(model, False, False, decoder=MMCOCODecoder) + map50 = res[1] + # print('eval info', res[-1]) + return map50 + + + + +#from new_impl.cv.model import ElasticDNN_ClsOnlineModel +elasticfm_model = ElasticDNN_DetOnlineModel('det', init_online_model( + 'new_impl/cv/glip/object_detection/results/det_md_w_fbs_index.py/20231201/999996-195158-results/models/fm_best.pt', + 'new_impl/cv/glip/object_detection/results/det_md_w_fbs_index.py/20231201/999996-195158-results/models/md_best.pt', + 'det', __file__ +), device, { + 'md_to_fm_alpha': 0.1, + 'fm_to_md_alpha': 0.1 +}) + +from glip import glip_model, build_transform, run_ner, collect_mm_fn +cfg_path = 'new_impl/cv/glip/object_detection/pretrained_model/glip_Swin_T_O365_GoldG.yaml' +model_path = 'new_impl/cv/glip/object_detection/pretrained_model/glip_tiny_model_o365_goldg_cc_sbu.pth' +config, _ = glip_model(cfg_path, model_path) +transform = build_transform(config, None) + +da_alg = FeatAlignAlg +from utils.dl.common.lr_scheduler import get_linear_schedule_with_warmup +#from new_impl.cv.model import ClsOnlineFeatAlignModel +da_model = DetOnlineFeatAlignModel +da_alg_hyp = { + 'MM-GTA5Det': { + 'train_batch_size': 8, + 'val_batch_size': 1, + 'num_workers': 8, + 'optimizer': 'AdamW', + 'optimizer_args': {'lr': 5e-7, 'betas': [0.9, 0.999], 'weight_decay': 0.01}, + 'scheduler': '', + 'scheduler_args': {}, + 'num_iters': 100, + 'val_freq': 20, + 'sd_sparsity':0.3, + 'feat_align_loss_weight': 0.0, + 'transform':transform + }, + 'MM-CityscapesDet': { + 'train_batch_size': 8, + 'val_batch_size': 1, + 'num_workers': 8, + 'optimizer': 'AdamW', + 'optimizer_args': {'lr': 5e-7, 'betas': [0.9, 0.999], 'weight_decay': 0.01}, + 'scheduler': '', + 'scheduler_args': {}, + 'num_iters': 100, + 'val_freq': 20, + 'sd_sparsity':0.3, + 'feat_align_loss_weight': 0.0, + 'transform':transform + }, +} + + +elasticfm_da( + [app_name], + [scenario], + [elasticfm_model], + [da_alg], + [da_alg_hyp], + [da_model], + device, + settings, + __file__, + "results", + collate_fn=collect_mm_fn +) \ No newline at end of file diff --git a/new_impl/cv/glip/object_detection/evaluator.py b/new_impl/cv/glip/object_detection/evaluator.py new file mode 100644 index 0000000000000000000000000000000000000000..5f175bb40626035ad6ff5cbfd8fe51aa056e812b --- /dev/null +++ b/new_impl/cv/glip/object_detection/evaluator.py @@ -0,0 +1,410 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +# Copyright (c) Megvii, Inc. and its affiliates. + +import contextlib +import io +import itertools +import json +import tempfile +import time +from loguru import logger +from tabulate import tabulate +from tqdm import tqdm +from data.datasets.object_detection.yolox_data_util.datasets.coco_classes import COCO_CLASSES +import numpy as np +from glip import run_ner +import torch + +# from core.common.dnn.detection.yolox.yolox.data.datasets import COCO_CLASSES +from dnns.yolov3.utils import ( + gather, + is_main_process, + postprocess, + synchronize, + time_synchronized, + xyxy2xywh +) +# from core.common.dnn.detection.yolox.yolox.layers import COCOeval_opt as COCOeval + + +def per_class_AR_table(coco_eval, class_names, headers=["class", "AR"], colums=6): + per_class_AR = {} + recalls = coco_eval.eval["recall"] + # dimension of recalls: [TxKxAxM] + # recall has dims (iou, cls, area range, max dets) + assert len(class_names) == recalls.shape[1] + + for idx, name in enumerate(class_names): + recall = recalls[:, idx, 0, -1] + recall = recall[recall > -1] + ar = np.mean(recall) if recall.size else float("nan") + per_class_AR[name] = float(ar * 100) + + num_cols = min(colums, len(per_class_AR) * len(headers)) + result_pair = [x for pair in per_class_AR.items() for x in pair] + row_pair = itertools.zip_longest(*[result_pair[i::num_cols] for i in range(num_cols)]) + table_headers = headers * (num_cols // len(headers)) + table = tabulate( + row_pair, tablefmt="pipe", floatfmt=".3f", headers=table_headers, numalign="left", + ) + return table + + +def per_class_AP_table(coco_eval, class_names, headers=["class", "AP"], colums=6): + per_class_AP = {} + precisions = coco_eval.eval["precision"] + # dimension of precisions: [TxRxKxAxM] + # precision has dims (iou, recall, cls, area range, max dets) + assert len(class_names) == precisions.shape[2] + + for idx, name in enumerate(class_names): + # area range index 0: all area ranges + # max dets index -1: typically 100 per image + precision = precisions[:, :, idx, 0, -1] + precision = precision[precision > -1] + ap = np.mean(precision) if precision.size else float("nan") + per_class_AP[name] = float(ap * 100) + + num_cols = min(colums, len(per_class_AP) * len(headers)) + result_pair = [x for pair in per_class_AP.items() for x in pair] + row_pair = itertools.zip_longest(*[result_pair[i::num_cols] for i in range(num_cols)]) + table_headers = headers * (num_cols // len(headers)) + table = tabulate( + row_pair, tablefmt="pipe", floatfmt=".3f", headers=table_headers, numalign="left", + ) + return table + + +class COCOEvaluator: + """ + COCO AP Evaluation class. All the data in the val2017 dataset are processed + and evaluated by COCO API. + """ + + def __init__( + self, + dataloader, + img_size: int, + confthre: float, + nmsthre: float, + num_classes: int, + testdev: bool = False, + per_class_AP: bool = False, + per_class_AR: bool = False, + ): + """ + Args: + dataloader (Dataloader): evaluate dataloader. + img_size: image size after preprocess. images are resized + to squares whose shape is (img_size, img_size). + confthre: confidence threshold ranging from 0 to 1, which + is defined in the config file. + nmsthre: IoU threshold of non-max supression ranging from 0 to 1. + per_class_AP: Show per class AP during evalution or not. Default to False. + per_class_AR: Show per class AR during evalution or not. Default to False. + """ + self.dataloader = dataloader + self.img_size = img_size + self.confthre = confthre + self.nmsthre = nmsthre + self.num_classes = num_classes + self.testdev = testdev + self.per_class_AP = per_class_AP + self.per_class_AR = per_class_AR + + def evaluate( + self, + model, + distributed=False, + half=False, + trt_file=None, + decoder=None, + test_size=None, + ): + """ + COCO average precision (AP) Evaluation. Iterate inference on the test dataset + and the results are evaluated by COCO API. + + NOTE: This function will change training mode to False, please save states if needed. + + Args: + model : model to evaluate. + + Returns: + ap50_95 (float) : COCO AP of IoU=50:95 + ap50 (float) : COCO AP of IoU=50 + summary (sr): summary info of evaluation. + """ + # TODO half to amp_test + tensor_type = torch.cuda.HalfTensor if half else torch.cuda.FloatTensor + model = model.eval() + if half: + model = model.half() + ids = [] + data_list = [] + progress_bar = iter if is_main_process() else iter + + inference_time = 0 + nms_time = 0 + n_samples = max(len(self.dataloader) - 1, 1) + + if trt_file is not None: + from torch2trt import TRTModule + + model_trt = TRTModule() + model_trt.load_state_dict(torch.load(trt_file)) + + x = torch.ones(1, 3, test_size[0], test_size[1]).cuda() + model(x) + model = model_trt + + import tqdm + for cur_iter, (dict, _) in tqdm.tqdm(enumerate( + progress_bar(self.dataloader) + ), dynamic_ncols=True, leave=False, total=len(self.dataloader)): + with torch.no_grad(): + imgs = [] + for img in dict['images']: + imgs.append(img.type(tensor_type)) + if len(imgs) == 0: + continue + dict['images'] = imgs + captions = [t.get_field('caption') for t in dict['targets']] + tokens_positives = [run_ner(caption) for caption in captions] + info_imgs = dict.pop('info_imgs') + ids = dict.pop('ids') + # skip the the last iters since batchsize might be not enough for batch inference + is_time_record = cur_iter < len(self.dataloader) - 1 + if is_time_record: + start = time.time() + + outputs, _, _ = model(**dict) + if len(outputs) == 0: + continue + if decoder is not None: + outputs = decoder(outputs) + + if is_time_record: + infer_end = time_synchronized() + inference_time += infer_end - start + + # outputs = postprocess( + # outputs, self.num_classes, self.confthre, self.nmsthre + # ) + if is_time_record: + nms_end = time_synchronized() + nms_time += nms_end - infer_end + + data_list.extend(self.convert_to_coco_format(outputs, info_imgs, ids, imgs, captions, tokens_positives)) + + statistics = torch.cuda.FloatTensor([inference_time, nms_time, n_samples]) + if distributed: + data_list = gather(data_list, dst=0) + data_list = list(itertools.chain(*data_list)) + torch.distributed.reduce(statistics, dst=0) + + eval_results = self.evaluate_prediction(data_list, statistics) + synchronize() + return eval_results + + def convert_to_coco_format(self, outputs, info_imgs, ids, input_imgs=None, captions=None, tokens_positives=None): + ha = 1; + if ha == 1: + print("ERROR: Running Wrong!") + print("Please Run Again After a Long Time...") + data_list = [] + img_i = 0 + for (output, img_info, img_id, img, caption, tokens_positive) in zip( + outputs, info_imgs, ids, input_imgs, captions, tokens_positives + ): + img_h, img_w = img_info + if output is None: + continue + output = output.cpu() + + bboxes = output[:, 0:4] + + # preprocessing: resize + scale = min( + self.img_size[0] / float(img_h), self.img_size[1] / float(img_w) + ) + bboxes /= scale + bboxes = xyxy2xywh(bboxes) + + cls = output[:, 5] - 1 + scores = output[:, 4] + for ind in range(bboxes.shape[0]): + # if bboxes.shape[0] > 1: + # print('a') + # print(self.dataloader.dataset.class_ids, cls[ind]) + # implemented by queyu, 2022/08/08 + _d = self.dataloader.dataset + if _d.__class__.__name__ == 'MergedDataset': + # _d = _d.datasets[0] + raise NotImplementedError + from data import ABDataset + if _d.__class__.__name__ == '_AugWrapperForDataset': + _d = _d.raw_dataset + if isinstance(_d, ABDataset): + _d = _d.dataset + if _d.__class__.__name__ == '_SplitDataset': + raise NotImplementedError + _d = _d.underlying_dataset + + # cls_names = _d.cls_names + # st, end = tokens_positive[int(cls[ind] - 1)][0] + # words = caption.split(". ") + # pos = 0 + # for word in words: + # if pos <= st and end <= pos + len(word): + # cls_name = word + # break + # pos += len(word) + len(". ") + + + # class_ids = _d.class_ids + # if int(cls[ind]) >= len(class_ids): + # raise RuntimeError + # label = self.dataloader.dataset.class_ids[-1] + # else: + # label = class_ids[int(cls[ind])] + pred_data = { + "image_id": int(img_id), + "category_id": int(cls[ind]), + "bbox": bboxes[ind].numpy().tolist(), + "score": scores[ind].numpy().item(), + "segmentation": [], + } # COCO json format + data_list.append(pred_data) + + # TODO: debug + # img = input_imgs[ind] + + + # from torchvision.transforms import ToTensor, ToPILImage + # from torchvision.utils import make_grid + # from PIL import Image, ImageDraw + # import matplotlib.pyplot as plt + # import numpy as np + # def draw_bbox(img, bbox, label, f): + # # if f: + # # img = np.uint8(img.permute(1, 2, 0)) + # # img = Image.fromarray(img) + # img = ToPILImage()(img) + # draw = ImageDraw.Draw(img) + # draw.rectangle(bbox, outline=(255, 0, 0), width=6) + # draw.text((bbox[0], bbox[1]), label) + # return ToTensor()(np.array(img)) + + # def xywh2xyxy(bbox): + # x, y, w, h = bbox + # x1, y1 = x, y + # x2, y2 = x + w, y + h + # return x1, y1, x2, y2 + + # img = draw_bbox(img, xywh2xyxy(bboxes[ind].numpy()), str(label), True) + + # img = make_grid([img], 1, normalize=True) + # plt.axis('off') + # img = img.permute(1, 2, 0).numpy() + # plt.imshow(img) + # plt.savefig(f'./tmp-coco-eval-{ind}.png') + # plt.clf() + # img_i += 1 + + # exit(0) + return data_list + + def evaluate_prediction(self, data_dict, statistics): + if not is_main_process(): + return 0, 0, None + + # logger.info("Evaluate in main process...") + + annType = ["segm", "bbox", "keypoints"] + + inference_time = statistics[0].item() + nms_time = statistics[1].item() + n_samples = statistics[2].item() + + a_infer_time = 1000 * inference_time / (n_samples * self.dataloader.batch_size) + a_nms_time = 1000 * nms_time / (n_samples * self.dataloader.batch_size) + + time_info = ", ".join( + [ + "Average {} time: {:.2f} ms".format(k, v) + for k, v in zip( + ["forward", "NMS", "inference"], + [a_infer_time, a_nms_time, (a_infer_time + a_nms_time)], + ) + ] + ) + + info = time_info + "\n" + + # Evaluate the Dt (detection) json comparing with the ground truth + if len(data_dict) > 0: + # cocoGt = self.dataloader.dataset.coco + _d = self.dataloader.dataset + if _d.__class__.__name__ == 'MergedDataset': + # _d = _d.datasets[0] + raise NotImplementedError + from data import ABDataset + if _d.__class__.__name__ == '_AugWrapperForDataset': + _d = _d.raw_dataset + if isinstance(_d, ABDataset): + _d = _d.dataset + if _d.__class__.__name__ == '_SplitDataset': + raise NotImplementedError + _d = _d.underlying_dataset + + cocoGt = _d.coco + + # implemented by queyu, 2022/08/08 + # make cocoGt's label += y_offset + # cocoGt: COCOAPI + + # TODO: since pycocotools can't process dict in py36, write data to json file. + if self.testdev: + json.dump(data_dict, open("./yolox_testdev_2017.json", "w")) + cocoDt = cocoGt.loadRes(r"./yolox_testdev_2017.json") + else: + _, tmp = tempfile.mkstemp() + json.dump(data_dict, open(tmp, "w")) + cocoDt = cocoGt.loadRes(tmp) + # try: + # from core.common.dnn.detection.yolox.yolox.layers import COCOeval_opt as COCOeval + # except ImportError: + from pycocotools.cocoeval import COCOeval + + # logger.warning("Use standard COCOeval.") + + cocoEval = COCOeval(cocoGt, cocoDt, annType[1]) + cocoEval.evaluate() + cocoEval.accumulate() + redirect_string = io.StringIO() + with contextlib.redirect_stdout(redirect_string): + cocoEval.summarize() + info += redirect_string.getvalue() + cat_ids = list(cocoGt.cats.keys()) + cat_names = [cocoGt.cats[catId]['name'] for catId in sorted(cat_ids)] + if self.per_class_AP: + AP_table = per_class_AP_table(cocoEval, class_names=cat_names) + info += "per class AP:\n" + AP_table + "\n" + if self.per_class_AR: + AR_table = per_class_AR_table(cocoEval, class_names=cat_names) + info += "per class AR:\n" + AR_table + "\n" + return cocoEval.stats[0], cocoEval.stats[1], info + else: + return 0, 0, info + +def MMCOCODecoder(boxes): + labels = boxes.get_field('labels').float().unsqueeze(1) + scores = boxes.get_field('scores').float().unsqueeze(1) + bbox = boxes.bbox + try: + bbox = torch.cat([bbox, scores, labels], dim=1).unsqueeze(0) + except: + print('a') + return bbox \ No newline at end of file diff --git a/new_impl/cv/glip/object_detection/glip.py b/new_impl/cv/glip/object_detection/glip.py new file mode 100644 index 0000000000000000000000000000000000000000..cea538f7933f33712f1e879fe13e3b5a03dbdd55 --- /dev/null +++ b/new_impl/cv/glip/object_detection/glip.py @@ -0,0 +1,1850 @@ +import timm +from timm.models._factory import load_checkpoint +import torch +import os +from typing import List, Union, Optional, Tuple +from torch import nn +from torch.jit import Final +from einops import rearrange, repeat +from einops.layers.torch import Rearrange +from utils.dl.common.model import get_model_device, set_module, get_module, get_model_latency, get_model_size, LayerActivation3 +import torch.nn.functional as F +from utils.common.log import logger +from transformers import AutoTokenizer +import torch.nn.functional as F +from maskrcnn_benchmark.modeling.detector.generalized_vl_rcnn import GeneralizedVLRCNN +from maskrcnn_benchmark.config import cfg +from maskrcnn_benchmark.structures.bounding_box import BoxList +from torchvision import transforms as T +import matplotlib.pyplot as plt +import nltk +import re +from copy import deepcopy +from abc import ABC, abstractmethod +from methods.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from methods.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util, LoRA +from new_impl.cv.elasticdnn.api.model import ElasticDNN_OfflineFMModel, ElasticDNN_OfflineMDModel +from methods.elasticdnn.model.base import Abs, KTakesAll, ElasticDNNUtil, Layer_WrappedWithFBS +from transformers.models.bert.modeling_bert import BertSelfAttention +from transformers import BertConfig +import math +from timm.models.layers import DropPath, to_2tuple, trunc_normal_ + +def collect_mm_fn(batch): + if len(batch[0]) == 2: + dict = {'images' : [], 'targets' : []} + else: + dict = {'images' : [], 'targets' : [], "info_imgs" : [], "ids" : []} + + for item in batch: + if len(item) == 2: + img, new_target = item + if len(new_target) == 0: + continue + dict['images'].append(img) + dict['targets'].append(new_target) + else: + img, new_target, info_imgs, ids = item + if len(new_target) == 0: + continue + dict['images'].append(img) + dict['targets'].append(new_target) + dict['info_imgs'].append(info_imgs) + dict['ids'].append(ids) + + return dict, torch.Tensor([0]) + +def run_ner(caption): + noun_phrases = find_noun_phrases(caption) + noun_phrases = [remove_punctuation(phrase) for phrase in noun_phrases] + noun_phrases = [phrase for phrase in noun_phrases if phrase != ''] + relevant_phrases = noun_phrases + labels = noun_phrases + + tokens_positive = [] + + for entity, label in zip(relevant_phrases, labels): + try: + # search all occurrences and mark them as different entities + for m in re.finditer(entity, caption.lower()): + tokens_positive.append([[m.start(), m.end()]]) + except: + print("noun entities:", noun_phrases) + print("entity:", entity) + print("caption:", caption.lower()) + + return tokens_positive + +def build_transform(cfg, min_image_size): + """ + Creates a basic transformation that was used to train the models + """ + + # we are loading images with OpenCV, so we don't need to convert them + # to BGR, they are already! So all we need to do is to normalize + # by 255 if we want to convert to BGR255 format, or flip the channels + # if we want it to be in RGB in [0-1] range. + if cfg.INPUT.TO_BGR255: + to_bgr_transform = T.Lambda(lambda x: x * 255) + else: + to_bgr_transform = T.Lambda(lambda x: x[[2, 1, 0]]) + + normalize_transform = T.Normalize( + mean=cfg.INPUT.PIXEL_MEAN, std=cfg.INPUT.PIXEL_STD + ) + + transform = T.Compose( + [ + T.ToPILImage(), + T.Resize(min_image_size) if min_image_size is not None else lambda x: x, + T.ToTensor(), + to_bgr_transform, + normalize_transform, + ] + ) + return transform + +def remove_punctuation(text: str) -> str: + punct = ['|', ':', ';', '@', '(', ')', '[', ']', '{', '}', '^', + '\'', '\"', '’', '`', '?', '$', '%', '#', '!', '&', '*', '+', ',', '.' + ] + for p in punct: + text = text.replace(p, '') + return text.strip() + +def create_positive_map_label_to_token_from_positive_map(positive_map, plus=0): + positive_map_label_to_token = {} + for i in range(len(positive_map)): + positive_map_label_to_token[i + plus] = torch.nonzero(positive_map[i], as_tuple=True)[0].tolist() + return positive_map_label_to_token + +def create_positive_map(tokenized, tokens_positive): + """construct a map such that positive_map[i,j] = True iff box i is associated to token j""" + positive_map = torch.zeros((len(tokens_positive), 256), dtype=torch.float) + + for j, tok_list in enumerate(tokens_positive): + for (beg, end) in tok_list: + try: + beg_pos = tokenized.char_to_token(beg) + end_pos = tokenized.char_to_token(end - 1) + except Exception as e: + print("beg:", beg, "end:", end) + print("token_positive:", tokens_positive) + # print("beg_pos:", beg_pos, "end_pos:", end_pos) + raise e + if beg_pos is None: + try: + beg_pos = tokenized.char_to_token(beg + 1) + if beg_pos is None: + beg_pos = tokenized.char_to_token(beg + 2) + except: + beg_pos = None + if end_pos is None: + try: + end_pos = tokenized.char_to_token(end - 2) + if end_pos is None: + end_pos = tokenized.char_to_token(end - 3) + except: + end_pos = None + if beg_pos is None or end_pos is None: + continue + + assert beg_pos is not None and end_pos is not None + positive_map[j, beg_pos: end_pos + 1].fill_(1) + return positive_map / (positive_map.sum(-1)[:, None] + 1e-6) + +def find_noun_phrases(caption: str) -> List[str]: + caption = caption.lower() + tokens = nltk.word_tokenize(caption) + pos_tags = nltk.pos_tag(tokens) + + grammar = "NP: {
?*+}" + cp = nltk.RegexpParser(grammar) + result = cp.parse(pos_tags) + + noun_phrases = list() + for subtree in result.subtrees(): + if subtree.label() == 'NP': + noun_phrases.append(' '.join(t[0] for t in subtree.leaves())) + + return noun_phrases + +class Glip(nn.Module): + def __init__(self, config, pretrain_path, min_image_size=None,confidence_threshold=0.7): + super(Glip, self).__init__() + state_dict = torch.load(pretrain_path)['model'] + self.min_image_size = min_image_size + self.cfg = config + self.confidence_threshold = confidence_threshold + self.tokenizer = AutoTokenizer.from_pretrained(cfg.MODEL.LANGUAGE_BACKBONE.MODEL_PATH) + self.device = torch.device(cfg.MODEL.DEVICE) + for k in list(state_dict.keys()): + if k.startswith('module'): + new_k = k.replace('module.', '') + state_dict[new_k] = state_dict.pop(k) + self.model = GeneralizedVLRCNN(config) + self.model.load_state_dict(state_dict, strict=False) + # self.transform = build_transform(config, min_image_size) + + def forward(self, images, targets, for_training=None): + # img_list = [] + # for image in images: + # img_list.append(self.transform(image).to(self.device)) + + # if isinstance(texts, list): + # # we directly provided a list of category names + # caption_string = "" + # tokens_positive = [] + # seperation_tokens = " . " + # for word in texts: + + # tokens_positive.append([len(caption_string), len(caption_string) + len(word)]) + # caption_string += word + # caption_string += seperation_tokens + + # tokenized = self.tokenizer([caption_string], return_tensors="pt") + # tokens_positive = [tokens_positive] + + # texts = [caption_string] + # print(tokens_positive) + # else: + device = torch.device(cfg.MODEL.DEVICE) + images = [image.to(device) for image in images] + targets = [target.to(device) for target in targets] + texts = [t.get_field("caption") for t in targets if "caption" in t.fields()] + positive_map = [] + # if custom_entity is None: + # tokens_positive = self.run_ner(texts) + # print(tokens_positive) + # process positive map + + if self.training == False: + try: + tokens_positive = run_ner(texts[0]) + except: + print('a') + tokenized = self.tokenizer(texts, return_tensors="pt") + positive_map = create_positive_map(tokenized, tokens_positive) + if self.cfg.MODEL.RPN_ARCHITECTURE == "VLDYHEAD": + plus = 1 + else: + plus = 0 + positive_map = create_positive_map_label_to_token_from_positive_map(positive_map, plus=plus) + else: + for i, text in enumerate(texts): + tokenized = self.tokenizer(text, return_tensors="pt") + tokens_positive = targets[i].get_field('tokens_positive') + positive_map.append(create_positive_map(tokenized, tokens_positive)) + + positive_map = torch.cat(positive_map, dim=0).to(device) + + + if self.training: + proposal_losses = self.model(images, targets, texts, positive_map=positive_map) + return proposal_losses + else: + proposals, token_logits, dot_product_logits = self.model(images, targets, texts, positive_map=positive_map) + proposal = self._post_process(proposals[0]) + return proposal, token_logits, dot_product_logits + + def _post_process_fixed_thresh(self, predictions): + scores = predictions.get_field("scores") + labels = predictions.get_field("labels").tolist() + thresh = scores.clone() + for i, lb in enumerate(labels): + if isinstance(self.confidence_threshold, float): + thresh[i] = self.confidence_threshold + elif len(self.confidence_threshold) == 1: + thresh[i] = self.confidence_threshold[0] + else: + thresh[i] = self.confidence_threshold[lb - 1] + keep = torch.nonzero(scores > thresh).squeeze(1) + predictions = predictions[keep] + + scores = predictions.get_field("scores") + _, idx = scores.sort(0, descending=True) + return predictions[idx] + + def _post_process(self, predictions, threshold=0.5): + scores = predictions.get_field("scores") + labels = predictions.get_field("labels").tolist() + thresh = scores.clone() + for i, lb in enumerate(labels): + if isinstance(self.confidence_threshold, float): + thresh[i] = threshold + elif len(self.confidence_threshold) == 1: + thresh[i] = threshold + else: + thresh[i] = self.confidence_threshold[lb - 1] + keep = torch.nonzero(scores > thresh).squeeze(1) + predictions = predictions[keep] + + scores = predictions.get_field("scores") + _, idx = scores.sort(0, descending=True) + return predictions[idx] + +# @torch.no_grad() +# def clip_vit_b_16(): +# # https://huggingface.co/openai/clip-vit-base-patch16 +# model = CLIPModelCanReceiveTextEmbeds.from_pretrained("openai/clip-vit-base-patch16") +# processor = CLIPProcessor.from_pretrained("openai/clip-vit-base-patch16") + +# print(model) + +# from PIL import Image +# import requests +# image = Image.open('/data/zql/datasets/Caltech-256/data/caltech256/256_ObjectCategories/003.backpack/003_0001.jpg') +# inputs = processor(text=["a photo of a dog", "a photo of a backpack", "a photo of a cat"], images=image, return_tensors="pt", padding=True) +# print(inputs) + +# from utils.dl.common.model import LayerActivation2, get_module +# input_embed_hook = LayerActivation2(get_module(model, 'text_model.embeddings')) +# outputs = model(**inputs) +# logits_per_image = outputs.logits_per_image # this is the image-text similarity score +# probs = logits_per_image.softmax(dim=1) +# print(probs) + +# input_embed = input_embed_hook.output +# input_embed_hook.remove() + +# torch.save(input_embed, os.path.join(os.path.dirname(__file__), './test_input_embed.pth')) + +# print('embed', input_embed.size()) + +# del inputs['input_ids'] +# inputs['input_embeds'] = input_embed +# outputs = model(**inputs) +# logits_per_image = outputs.logits_per_image # this is the image-text similarity score +# probs = logits_per_image.softmax(dim=1) +# print(probs) + + +@torch.no_grad() +def glip_model(config_path, pretrain_path): + # https://huggingface.co/openai/clip-vit-base-patch16 + cfg.merge_from_file(config_path) + return cfg, Glip(cfg, pretrain_path) + +class ToQKV_WrappedWithLoRA(nn.Module): + def __init__(self, fc: nn.Linear, ab_r: int): + super(ToQKV_WrappedWithLoRA, self).__init__() + + self.fc = fc + self.ab = self.create_ab_as_linear(fc.weight.data, ab_r) + + def create_ab_as_linear(self, fc_weight: torch.Tensor, ab_r: int): + res = nn.Sequential( + LoRA(fc_weight.size(1), fc_weight.size(0) // ab_r, bias=False), + LoRA(fc_weight.size(0) // ab_r, fc_weight.size(0), bias=False) + ).to(fc_weight.device) + nn.init.kaiming_uniform_(res[0].weight, a=5 ** 0.5) + nn.init.zeros_(res[1].weight) + return res + + def forward(self, x): + x1 = self.fc(x) + x2 = self.ab(x) + return x1 + x2 + +def get_model_latency_2(model: torch.nn.Module, sample: dict, sample_num: int, + device: str, warmup_sample_num: int, return_detail=False): + """Get the latency (inference time) of a PyTorch model. + + Reference: https://deci.ai/resources/blog/measure-inference-time-deep-neural-networks/ + + Args: + model (torch.nn.Module): A PyTorch model. + model_input_size (Tuple[int]): Typically be `(1, 3, 32, 32)` or `(1, 3, 224, 224)`. + sample_num (int): How many inputs which size is :attr:`model_input_size` will be tested and compute the average latency as result. + device (str): Typically be 'cpu' or 'cuda'. + warmup_sample_num (int): Let model perform some dummy inference to warm up the test environment to avoid measurement loss. + return_detail (bool, optional): Beside the average latency, return all result measured. Defaults to False. + + Returns: + Union[float, Tuple[float, List[float]]]: The average latency (and all lantecy data) of :attr:`model`. + """ + # if isinstance(model_input_size, tuple): + # dummy_input = torch.rand(model_input_size).to(device) + # else: + # dummy_input = model_input_size + + model = model.to(device) + model.eval() + + # warm up + with torch.no_grad(): + for _ in range(warmup_sample_num): + model(**sample) + + infer_time_list = [] + + if device == 'cuda' or 'cuda' in str(device): + with torch.no_grad(): + for _ in range(sample_num): + s, e = torch.cuda.Event(enable_timing=True), torch.cuda.Event(enable_timing=True) + s.record() + model(**sample) + e.record() + torch.cuda.synchronize() + cur_model_infer_time = s.elapsed_time(e) / 1000. + infer_time_list += [cur_model_infer_time] + + else: + with torch.no_grad(): + for _ in range(sample_num): + start = time.time() + model(**sample) + cur_model_infer_time = time.time() - start + infer_time_list += [cur_model_infer_time] + + avg_infer_time = sum(infer_time_list) / sample_num + + if return_detail: + return avg_infer_time, infer_time_list + return avg_infer_time + +class WindowAttention(nn.Module): + """ Window based multi-head self attention (W-MSA) module with relative position bias. + It supports both of shifted and non-shifted window. + Args: + dim (int): Number of input channels. + window_size (tuple[int]): The height and width of the window. + num_heads (int): Number of attention heads. + qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True + qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set + attn_drop (float, optional): Dropout ratio of attention weight. Default: 0.0 + proj_drop (float, optional): Dropout ratio of output. Default: 0.0 + """ + + def __init__(self, dim, window_size, num_heads, qkv_bias=True, qk_scale=None, attn_drop=0., proj_drop=0.): + + super().__init__() + self.dim = dim + self.window_size = window_size # Wh, Ww + self.num_heads = num_heads + head_dim = dim // num_heads + self.scale = qk_scale or head_dim ** -0.5 + + # define a parameter table of relative position bias + self.relative_position_bias_table = nn.Parameter( + torch.zeros((2 * window_size[0] - 1) * (2 * window_size[1] - 1), num_heads)) # 2*Wh-1 * 2*Ww-1, nH + + # get pair-wise relative position index for each token inside the window + coords_h = torch.arange(self.window_size[0]) + coords_w = torch.arange(self.window_size[1]) + coords = torch.stack(torch.meshgrid([coords_h, coords_w])) # 2, Wh, Ww + coords_flatten = torch.flatten(coords, 1) # 2, Wh*Ww + relative_coords = coords_flatten[:, :, None] - coords_flatten[:, None, :] # 2, Wh*Ww, Wh*Ww + relative_coords = relative_coords.permute(1, 2, 0).contiguous() # Wh*Ww, Wh*Ww, 2 + relative_coords[:, :, 0] += self.window_size[0] - 1 # shift to start from 0 + relative_coords[:, :, 1] += self.window_size[1] - 1 + relative_coords[:, :, 0] *= 2 * self.window_size[1] - 1 + relative_position_index = relative_coords.sum(-1) # Wh*Ww, Wh*Ww + self.register_buffer("relative_position_index", relative_position_index) + + self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias) + self.attn_drop = nn.Dropout(attn_drop) + self.proj = nn.Linear(dim, dim) + self.proj_drop = nn.Dropout(proj_drop) + + trunc_normal_(self.relative_position_bias_table, std=.02) + self.softmax = nn.Softmax(dim=-1) + + def forward(self, x, mask=None): + """ Forward function. + Args: + x: input features with shape of (num_windows*B, N, C) + mask: (0/-inf) mask with shape of (num_windows, Wh*Ww, Wh*Ww) or None + """ + B_, N, C = x.shape + qkv = self.qkv(x).reshape(B_, N, 3, self.num_heads, -1).permute(2, 0, 3, 1, 4) + q, k, v = qkv[0], qkv[1], qkv[2] # make torchscript happy (cannot use tensor as tuple) + + q = q * self.scale + attn = (q @ k.transpose(-2, -1)) + + relative_position_bias = self.relative_position_bias_table[self.relative_position_index.view(-1)].view( + self.window_size[0] * self.window_size[1], self.window_size[0] * self.window_size[1], -1) # Wh*Ww,Wh*Ww,nH + relative_position_bias = relative_position_bias.permute(2, 0, 1).contiguous() # nH, Wh*Ww, Wh*Ww + attn = attn + relative_position_bias.unsqueeze(0) + + if mask is not None: + nW = mask.shape[0] + attn = attn.view(B_ // nW, nW, self.num_heads, N, N) + mask.unsqueeze(1).unsqueeze(0) + attn = attn.view(-1, self.num_heads, N, N) + attn = self.softmax(attn) + else: + attn = self.softmax(attn) + + attn = self.attn_drop(attn) + + x = (attn @ v).transpose(1, 2).reshape(B_, N, -1) + x = self.proj(x) + x = self.proj_drop(x) + return x + +class BiMultiHeadAttention(nn.Module): + def __init__(self, v_dim, l_dim, embed_dim, num_heads, dropout=0.1, cfg=None): + super(BiMultiHeadAttention, self).__init__() + + self.embed_dim = embed_dim + self.num_heads = num_heads + self.head_dim = embed_dim // num_heads + self.v_dim = v_dim + self.l_dim = l_dim + + assert ( + self.head_dim * self.num_heads == self.embed_dim + ), f"embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim} and `num_heads`: {self.num_heads})." + self.scale = self.head_dim ** (-0.5) + self.dropout = dropout + + self.v_proj = nn.Linear(self.v_dim, self.embed_dim) + self.l_proj = nn.Linear(self.l_dim, self.embed_dim) + self.values_v_proj = nn.Linear(self.v_dim, self.embed_dim) + self.values_l_proj = nn.Linear(self.l_dim, self.embed_dim) + + self.out_v_proj = nn.Linear(self.embed_dim, self.v_dim) + self.out_l_proj = nn.Linear(self.embed_dim, self.l_dim) + + self.stable_softmax_2d = cfg.MODEL.DYHEAD.FUSE_CONFIG.STABLE_SOFTMAX_2D + self.clamp_min_for_underflow = cfg.MODEL.DYHEAD.FUSE_CONFIG.CLAMP_MIN_FOR_UNDERFLOW + self.clamp_max_for_overflow = cfg.MODEL.DYHEAD.FUSE_CONFIG.CLAMP_MAX_FOR_OVERFLOW + + self._reset_parameters() + + def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int): + return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous() + + def _reset_parameters(self): + nn.init.xavier_uniform_(self.v_proj.weight) + self.v_proj.bias.data.fill_(0) + nn.init.xavier_uniform_(self.l_proj.weight) + self.l_proj.bias.data.fill_(0) + nn.init.xavier_uniform_(self.values_v_proj.weight) + self.values_v_proj.bias.data.fill_(0) + nn.init.xavier_uniform_(self.values_l_proj.weight) + self.values_l_proj.bias.data.fill_(0) + nn.init.xavier_uniform_(self.out_v_proj.weight) + self.out_v_proj.bias.data.fill_(0) + nn.init.xavier_uniform_(self.out_l_proj.weight) + self.out_l_proj.bias.data.fill_(0) + + def forward(self, v, l, attention_mask_l=None): + bsz, tgt_len, embed_dim = v.size() + + query_states = self.v_proj(v) * self.scale + key_states = self._shape(self.l_proj(l), -1, bsz) + value_v_states = self._shape(self.values_v_proj(v), -1, bsz) + value_l_states = self._shape(self.values_l_proj(l), -1, bsz) + + proj_shape = (bsz * self.num_heads, -1, self.head_dim) + query_states = self._shape(query_states, tgt_len, bsz).view(*proj_shape) + key_states = key_states.view(*proj_shape) + value_v_states = value_v_states.view(*proj_shape) + value_l_states = value_l_states.view(*proj_shape) + + src_len = key_states.size(1) + attn_weights = torch.bmm(query_states, key_states.transpose(1, 2)) + + if attn_weights.size() != (bsz * self.num_heads, tgt_len, src_len): + raise ValueError( + f"Attention weights should be of size {(bsz * self.num_heads, tgt_len, src_len)}, but is {attn_weights.size()}" + ) + + # attn_weights_l = nn.functional.softmax(attn_weights.transpose(1, 2), dim=-1) + + if self.stable_softmax_2d: + attn_weights = attn_weights - attn_weights.max() + + if self.clamp_min_for_underflow: + attn_weights = torch.clamp(attn_weights, min=-50000) # Do not increase -50000, data type half has quite limited range + if self.clamp_max_for_overflow: + attn_weights = torch.clamp(attn_weights, max=50000) # Do not increase 50000, data type half has quite limited range + + attn_weights_T = attn_weights.transpose(1, 2) + attn_weights_l = (attn_weights_T - torch.max(attn_weights_T, dim=-1, keepdim=True)[ + 0]) + if self.clamp_min_for_underflow: + attn_weights_l = torch.clamp(attn_weights_l, min=-50000) # Do not increase -50000, data type half has quite limited range + if self.clamp_max_for_overflow: + attn_weights_l = torch.clamp(attn_weights_l, max=50000) # Do not increase 50000, data type half has quite limited range + + attn_weights_l = attn_weights_l.softmax(dim=-1) + + if attention_mask_l is not None: + assert (attention_mask_l.dim() == 2) + attention_mask = attention_mask_l.unsqueeze(1).unsqueeze(1) + attention_mask = attention_mask.expand(bsz, 1, tgt_len, src_len) + attention_mask = attention_mask.masked_fill(attention_mask == 0, -9e15) + + if attention_mask.size() != (bsz, 1, tgt_len, src_len): + raise ValueError( + f"Attention mask should be of size {(bsz, 1, tgt_len, src_len)}" + ) + attn_weights = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + attention_mask + attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len) + + attn_weights_v = nn.functional.softmax(attn_weights, dim=-1) + + attn_probs_v = F.dropout(attn_weights_v, p=self.dropout, training=self.training) + attn_probs_l = F.dropout(attn_weights_l, p=self.dropout, training=self.training) + + attn_output_v = torch.bmm(attn_probs_v, value_l_states) + attn_output_l = torch.bmm(attn_probs_l, value_v_states) + + + if attn_output_v.size() != (bsz * self.num_heads, tgt_len, self.head_dim): + raise ValueError( + f"`attn_output_v` should be of size {(bsz, self.num_heads, tgt_len, self.head_dim)}, but is {attn_output_v.size()}" + ) + + if attn_output_l.size() != (bsz * self.num_heads, src_len, self.head_dim): + raise ValueError( + f"`attn_output_l` should be of size {(bsz, self.num_heads, src_len, self.head_dim)}, but is {attn_output_l.size()}" + ) + + attn_output_v = attn_output_v.view(bsz, self.num_heads, tgt_len, self.head_dim) + attn_output_v = attn_output_v.transpose(1, 2) + attn_output_v = attn_output_v.reshape(bsz, tgt_len, self.embed_dim) + + attn_output_l = attn_output_l.view(bsz, self.num_heads, src_len, self.head_dim) + attn_output_l = attn_output_l.transpose(1, 2) + attn_output_l = attn_output_l.reshape(bsz, src_len, self.embed_dim) + + attn_output_v = self.out_v_proj(attn_output_v) + attn_output_l = self.out_l_proj(attn_output_l) + + return attn_output_v, attn_output_l + +class BertSelfAttentionPrunable(BertSelfAttention): + def __init__(self): + config = BertConfig.from_pretrained('new_impl/cv/glip/object_detection/bert-base-uncased') + super(BertSelfAttentionPrunable, self).__init__(config) + + def transpose_for_scores(self, x): + new_x_shape = x.size()[:-1] + (self.num_attention_heads, -1) + x = x.view(new_x_shape) + return x.permute(0, 2, 1, 3) + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.FloatTensor] = None, + head_mask: Optional[torch.FloatTensor] = None, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + encoder_attention_mask: Optional[torch.FloatTensor] = None, + past_key_value: Optional[Tuple[Tuple[torch.FloatTensor]]] = None, + output_attentions: Optional[bool] = False, + ) -> Tuple[torch.Tensor]: + mixed_query_layer = self.query(hidden_states) + + # If this is instantiated as a cross-attention module, the keys + # and values come from an encoder; the attention mask needs to be + # such that the encoder's padding tokens are not attended to. + is_cross_attention = encoder_hidden_states is not None + + if is_cross_attention and past_key_value is not None: + # reuse k,v, cross_attentions + key_layer = past_key_value[0] + value_layer = past_key_value[1] + attention_mask = encoder_attention_mask + elif is_cross_attention: + key_layer = self.transpose_for_scores(self.key(encoder_hidden_states)) + value_layer = self.transpose_for_scores(self.value(encoder_hidden_states)) + attention_mask = encoder_attention_mask + elif past_key_value is not None: + key_layer = self.transpose_for_scores(self.key(hidden_states)) + value_layer = self.transpose_for_scores(self.value(hidden_states)) + key_layer = torch.cat([past_key_value[0], key_layer], dim=2) + value_layer = torch.cat([past_key_value[1], value_layer], dim=2) + else: + key_layer = self.transpose_for_scores(self.key(hidden_states)) + value_layer = self.transpose_for_scores(self.value(hidden_states)) + + query_layer = self.transpose_for_scores(mixed_query_layer) + + use_cache = past_key_value is not None + if self.is_decoder: + # if cross_attention save Tuple(torch.Tensor, torch.Tensor) of all cross attention key/value_states. + # Further calls to cross_attention layer can then reuse all cross-attention + # key/value_states (first "if" case) + # if uni-directional self-attention (decoder) save Tuple(torch.Tensor, torch.Tensor) of + # all previous decoder key/value_states. Further calls to uni-directional self-attention + # can concat previous decoder key/value_states to current projected key/value_states (third "elif" case) + # if encoder bi-directional self-attention `past_key_value` is always `None` + past_key_value = (key_layer, value_layer) + + # Take the dot product between "query" and "key" to get the raw attention scores. + attention_scores = torch.matmul(query_layer, key_layer.transpose(-1, -2)) + + if self.position_embedding_type == "relative_key" or self.position_embedding_type == "relative_key_query": + query_length, key_length = query_layer.shape[2], key_layer.shape[2] + if use_cache: + position_ids_l = torch.tensor(key_length - 1, dtype=torch.long, device=hidden_states.device).view( + -1, 1 + ) + else: + position_ids_l = torch.arange(query_length, dtype=torch.long, device=hidden_states.device).view(-1, 1) + position_ids_r = torch.arange(key_length, dtype=torch.long, device=hidden_states.device).view(1, -1) + distance = position_ids_l - position_ids_r + + positional_embedding = self.distance_embedding(distance + self.max_position_embeddings - 1) + positional_embedding = positional_embedding.to(dtype=query_layer.dtype) # fp16 compatibility + + if self.position_embedding_type == "relative_key": + relative_position_scores = torch.einsum("bhld,lrd->bhlr", query_layer, positional_embedding) + attention_scores = attention_scores + relative_position_scores + elif self.position_embedding_type == "relative_key_query": + relative_position_scores_query = torch.einsum("bhld,lrd->bhlr", query_layer, positional_embedding) + relative_position_scores_key = torch.einsum("bhrd,lrd->bhlr", key_layer, positional_embedding) + attention_scores = attention_scores + relative_position_scores_query + relative_position_scores_key + + attention_scores = attention_scores / math.sqrt(self.attention_head_size) + if attention_mask is not None: + # Apply the attention mask is (precomputed for all layers in BertModel forward() function) + attention_scores = attention_scores + attention_mask + + # Normalize the attention scores to probabilities. + attention_probs = nn.functional.softmax(attention_scores, dim=-1) + + # This is actually dropping out entire tokens to attend to, which might + # seem a bit unusual, but is taken from the original Transformer paper. + attention_probs = self.dropout(attention_probs) + + # Mask heads if we want to + if head_mask is not None: + attention_probs = attention_probs * head_mask + + context_layer = torch.matmul(attention_probs, value_layer) + + context_layer = context_layer.permute(0, 2, 1, 3).contiguous() + new_context_layer_shape = context_layer.size()[:-2] + (self.query.out_features,) # NOTE: modified + context_layer = context_layer.view(new_context_layer_shape) + + outputs = (context_layer, attention_probs) if output_attentions else (context_layer,) + + if self.is_decoder: + outputs = outputs + (past_key_value,) + return outputs + + @staticmethod + def init_from_exist_self_attn(attn: BertSelfAttention): + # print(attn) + + res = BertSelfAttentionPrunable() + + for attr in dir(attn): + # if str(attr) in ['transpose_for_scores'] or str(attr).startswith('_'): + # continue + # if isinstance(getattr(attn, attr), nn.Module): + # print(attr) + + if isinstance(getattr(attn, attr), nn.Module): + try: + # print(attr, 'ok') + setattr(res, attr, getattr(attn, attr)) + + except Exception as e: + print(attr, str(e)) + + + + return res + +class FM_to_MD_GLIP_Util(FM_to_MD_Util): + def init_md_from_fm_by_reducing_width_with_perf_test_2(self, fm: nn.Module, reducing_width_ratio: int, + samples: torch.Tensor) -> nn.Module: + fm_size = get_model_size(fm, True) + fm_latency = get_model_latency_2(fm, samples, 20, + get_model_device(fm), 20, False) + + master_dnn = self.init_md_from_fm_by_reducing_width(fm, reducing_width_ratio) + master_dnn_size = get_model_size(master_dnn, True) + logger.debug(f'inited master DNN: {master_dnn}') + # from utils.dl.common.model import get_module + # print('after generating') + # get_module(fm, 'head').debug() + # get_module(master_dnn, 'head').debug() + # print('test master latency') + master_dnn_latency = get_model_latency_2(fm, samples, 20, + get_model_device(fm), 20, False) + + logger.info(f'init master DNN (w/o FBS yet) by reducing foundation model\'s width (by {reducing_width_ratio:d}x)') + logger.info(f'foundation model ({fm_size:.3f}MB, {fm_latency:.4f}s/sample) -> ' + f'master DNN ({master_dnn_size:.3f}MB, {master_dnn_latency:.4f}s/sample)\n' + f'(model size: ↓ {(fm_size / master_dnn_size):.2f}x, ' + f'latency: ↓ {(fm_latency / master_dnn_latency):.2f}x)') + + return master_dnn + + def init_md_from_fm_by_reducing_width(self, fm: nn.Module, reducing_width_ratio: int, sparsity=0.0) -> nn.Module: + #sparsity: it is mainly used to make a distilled model used in the baseline algorithm, and the parameter can ensure that the model has the same size as the model used in the online algorithm. + fm_vit = deepcopy(fm) + + def _f(n): + return int(n // reducing_width_ratio) + + # def _rand_indexes(n): + # return torch.randperm(n)[0: int(n // reducing_width_ratio)] + + def l1_max_indexes(p: torch.Tensor, dim=0): + assert dim in [0, 1] + assert p.dim() in [1, 2, 4] + + if dim == 1: + p = p.T + + p_norm = p.abs().contiguous().view(p.size(0), -1).sum(dim=1) + n = p.size(0) + + t1 = p_norm.argsort(descending=True)[0: int(n // reducing_width_ratio)] + t2 = t1.sort()[0] + return p_norm.argsort(descending=True)[0: int(n // reducing_width_ratio)].sort()[0] + + def l1_max_indexes_with_sparsity(p: torch.Tensor, dim=0): + assert dim in [0, 1] + assert p.dim() in [1, 2, 4] + + if dim == 1: + p = p.T + + p_norm = p.abs().contiguous().view(p.size(0), -1).sum(dim=1) + n = p.size(0) + return p_norm.argsort(descending=True)[0: int(n // reducing_width_ratio * (1 - sparsity))].sort()[0] + + for layer_i, layer in enumerate(fm_vit.model.backbone.body.layers): + for block in layer.blocks: + ori_attn = block.attn + new_attn = WindowAttention(ori_attn.dim, ori_attn.window_size, ori_attn.num_heads, True, ori_attn.scale, 0., 0.) + new_attn.relative_position_index = ori_attn.relative_position_index + new_attn.relative_position_bias_table = ori_attn.relative_position_bias_table + new_attn.qkv = ori_attn.qkv + new_attn.attn_drop = ori_attn.attn_drop + new_attn.proj = ori_attn.proj + new_attn.proj_drop = ori_attn.proj_drop + set_module(block, 'attn', new_attn) + + # first_attn = True + for layer_i, layer in enumerate(fm_vit.model.backbone.body.layers): + for block_i, block in enumerate(layer.blocks): + qkv = block.attn.qkv + new_qkv = nn.Linear(qkv.in_features, _f(qkv.out_features), + qkv.bias is not None, qkv.weight.device) + indexes = l1_max_indexes(qkv.weight.data, 0) + new_qkv.weight.data.copy_(qkv.weight.data[indexes]) + if qkv.bias is not None: + new_qkv.bias.data.copy_(qkv.bias.data[indexes]) + # fm_vit.model.backbone.body.layers[0].blocks.0.attn.qkv + set_module(fm_vit, f'model.backbone.body.layers.{layer_i}.blocks.{block_i}.attn.qkv', new_qkv) + + proj = block.attn.proj + new_proj = nn.Linear(_f(proj.in_features), proj.out_features, + proj.bias is not None, proj.weight.device) + new_proj.weight.data.copy_(proj.weight.data[:, l1_max_indexes(proj.weight.data, 1)]) + if proj.bias is not None: + new_proj.bias.data.copy_(proj.bias.data) + set_module(fm_vit, f'model.backbone.body.layers.{layer_i}.blocks.{block_i}.attn.proj', new_proj) + + fc1 = block.mlp.fc1 + new_fc1 = nn.Linear(fc1.in_features, int(_f(fc1.out_features) * (1 - sparsity)), + fc1.bias is not None, fc1.weight.device) + indexes = l1_max_indexes_with_sparsity(fc1.weight.data, 0) + new_fc1.weight.data.copy_(fc1.weight.data[indexes]) + if fc1.bias is not None: + new_fc1.bias.data.copy_(fc1.bias.data[indexes]) + set_module(fm_vit, f'model.backbone.body.layers.{layer_i}.blocks.{block_i}.mlp.fc1', new_fc1) + + fc2 = block.mlp.fc2 + new_fc2 = nn.Linear(int(_f(fc2.in_features) * (1 - sparsity)), fc2.out_features, + fc2.bias is not None, fc2.weight.device) + new_fc2.weight.data.copy_(fc2.weight.data[:, l1_max_indexes_with_sparsity(fc2.weight.data, 1)]) + if fc2.bias is not None: + new_fc2.bias.data.copy_(fc2.bias.data) + set_module(fm_vit, f'model.backbone.body.layers.{layer_i}.blocks.{block_i}.mlp.fc2', new_fc2) + + for block in fm_vit.model.language_backbone.body.model.encoder.layer: + set_module(block, 'attention.self', BertSelfAttentionPrunable.init_from_exist_self_attn(block.attention.self)) + + for block_i, block in enumerate(fm_vit.model.language_backbone.body.model.encoder.layer): + for k in ['query', 'key', 'value']: + qkv = get_module(block, f'attention.self.{k}') + new_qkv = nn.Linear(qkv.in_features, _f(qkv.out_features), + qkv.bias is not None, qkv.weight.device) + indexes = l1_max_indexes(qkv.weight.data, 0) + new_qkv.weight.data.copy_(qkv.weight.data[indexes]) + if qkv.bias is not None: + new_qkv.bias.data.copy_(qkv.bias.data[indexes]) + set_module(block, f'attention.self.{k}', new_qkv) + + proj = get_module(block, f'attention.output.dense') + new_proj = nn.Linear(_f(proj.in_features), proj.out_features, + proj.bias is not None, proj.weight.device) + new_proj.weight.data.copy_(proj.weight.data[:, l1_max_indexes(proj.weight.data, 1)]) + if proj.bias is not None: + new_proj.bias.data.copy_(proj.bias.data) + set_module(block, f'attention.output.dense', new_proj) + + fc1 = get_module(block, f'intermediate.dense') + new_fc1 = nn.Linear(fc1.in_features, int(_f(fc1.out_features) * (1 - sparsity)), + fc1.bias is not None, fc1.weight.device) + indexes = l1_max_indexes_with_sparsity(fc1.weight.data, 0) + new_fc1.weight.data.copy_(fc1.weight.data[indexes]) + if fc1.bias is not None: + new_fc1.bias.data.copy_(fc1.bias.data[indexes]) + set_module(block, f'intermediate.dense', new_fc1) + + fc2 = get_module(block, f'output.dense') + new_fc2 = nn.Linear(int(_f(fc2.in_features) * (1 - sparsity)), fc2.out_features, + fc2.bias is not None, fc2.weight.device) + new_fc2.weight.data.copy_(fc2.weight.data[:, l1_max_indexes_with_sparsity(fc2.weight.data, 1)]) + if fc2.bias is not None: + new_fc2.bias.data.copy_(fc2.bias.data) + set_module(block, f'output.dense', new_fc2) + + for block_i, block in enumerate(fm_vit.model.rpn.head.dyhead_tower): + if block_i % 3 == 0: + tmp = block.b_attn.attn + tmp.head_dim = int(tmp.head_dim // reducing_width_ratio) + tmp.embed_dim = int(tmp.embed_dim // reducing_width_ratio) + set_module(block, 'b_attn.attn', tmp) + for k in ['v_proj', 'l_proj', 'values_v_proj', 'values_l_proj']: + qkv = get_module(block, f'b_attn.attn.{k}') + new_qkv = nn.Linear(qkv.in_features, _f(qkv.out_features), + qkv.bias is not None, qkv.weight.device) + indexes = l1_max_indexes(qkv.weight.data, 0) + new_qkv.weight.data.copy_(qkv.weight.data[indexes]) + if qkv.bias is not None: + new_qkv.bias.data.copy_(qkv.bias.data[indexes]) + set_module(block, f'b_attn.attn.{k}', new_qkv) + + for k in ['out_v_proj', 'out_l_proj']: + qkv = get_module(block, f'b_attn.attn.{k}') + + new_qkv = nn.Linear(_f(qkv.in_features), qkv.out_features, + qkv.bias is not None, qkv.weight.device) + new_qkv.weight.data.copy_(qkv.weight.data[:, l1_max_indexes(qkv.weight.data, 1)]) + if qkv.bias is not None: + new_qkv.bias.data.copy_(qkv.bias.data) + set_module(block, f'b_attn.attn.{k}', new_qkv) + + elif block_i % 3 == 1: + set_module(block, 'attention.self', BertSelfAttentionPrunable.init_from_exist_self_attn(block.attention.self)) + for k in ['query', 'key', 'value']: + qkv = get_module(block, f'attention.self.{k}') + + new_qkv = nn.Linear(qkv.in_features, _f(qkv.out_features), + qkv.bias is not None, qkv.weight.device) + indexes = l1_max_indexes(qkv.weight.data, 0) + + new_qkv.weight.data.copy_(qkv.weight.data[indexes]) + if qkv.bias is not None: + new_qkv.bias.data.copy_(qkv.bias.data[indexes]) + set_module(block, f'attention.self.{k}', new_qkv) + + proj = get_module(block, f'attention.output.dense') + new_proj = nn.Linear(_f(proj.in_features), proj.out_features, + proj.bias is not None, proj.weight.device) + new_proj.weight.data.copy_(proj.weight.data[:, l1_max_indexes(proj.weight.data, 1)]) + if proj.bias is not None: + new_proj.bias.data.copy_(proj.bias.data) + set_module(block, f'attention.output.dense', new_proj) + + fc1 = get_module(block, f'intermediate.dense') + new_fc1 = nn.Linear(fc1.in_features, int(_f(fc1.out_features) * (1 - sparsity)), + fc1.bias is not None, fc1.weight.device) + indexes = l1_max_indexes_with_sparsity(fc1.weight.data, 0) + new_fc1.weight.data.copy_(fc1.weight.data[indexes]) + if fc1.bias is not None: + new_fc1.bias.data.copy_(fc1.bias.data[indexes]) + set_module(block, f'intermediate.dense', new_fc1) + + fc2 = get_module(block, f'output.dense') + new_fc2 = nn.Linear(int(_f(fc2.in_features) * (1 - sparsity)), fc2.out_features, + fc2.bias is not None, fc2.weight.device) + new_fc2.weight.data.copy_(fc2.weight.data[:, l1_max_indexes_with_sparsity(fc2.weight.data, 1)]) + if fc2.bias is not None: + new_fc2.bias.data.copy_(fc2.bias.data) + set_module(block, f'output.dense', new_fc2) + + # reduce dim_embedding + # if name.endswith('patch_embed.proj'): + # continue + + # new_layer = nn.Conv2d(module.in_channels, _f(module.out_channels), module.kernel_size, module.stride, + # module.padding, module.dilation, module.groups, module.bias is not None, module.padding_mode, + # module.weight.device) + + # rand_indexes = l1_max_indexes(module.weight.data) + # new_layer.weight.data.copy_(module.weight.data[rand_indexes]) + # if new_layer.bias is not None: + # new_layer.bias.data.copy_(module.bias.data[rand_indexes]) + + # fm_vit.cls_token.data = fm_vit.cls_token.data[:, :, rand_indexes] + # fm_vit.pos_embed.data = fm_vit.pos_embed.data[:, :, rand_indexes] + + # elif isinstance(module, nn.Linear): + + # if 'head' in name: + # continue + + # new_layer = nn.Linear(_f(module.in_features), module.out_features, + # module.bias is not None, module.weight.device) + # new_layer.weight.data.copy_(module.weight.data[:, l1_max_indexes(module.weight.data, 1)]) + # if new_layer.bias is not None: + # new_layer.bias.data.copy_(module.bias.data) + # else: + # first_attn = False + # if first_attn: + # first_attn = False + # new_layer = nn.Linear(module.in_features, _f(module.out_features), + # module.bias is not None, module.weight.device) + + # rand_indexes = l1_max_indexes(module.weight.data) + # new_layer.weight.data.copy_(module.weight.data[rand_indexes]) + # if new_layer.bias is not None: + # new_layer.bias.data.copy_(module.bias.data[rand_indexes]) + # else: + # new_layer = nn.Linear(_f(module.in_features), _f(module.out_features), + # module.bias is not None, module.weight.device) + + # rand_indexes = l1_max_indexes(module.weight.data) + # new_layer.weight.data.copy_(module.weight.data[rand_indexes][:, l1_max_indexes(module.weight.data, 1)]) + # if new_layer.bias is not None: + # new_layer.bias.data.copy_(module.bias.data[rand_indexes]) + + # elif isinstance(module, nn.LayerNorm) and ('block' in name or name == 'norm' or name == 'norm.0'): + # new_layer = nn.LayerNorm(_f(module.normalized_shape[0]), eps=module.eps, device=module.weight.device) + # rand_indexes = l1_max_indexes(module.weight.data) + # new_layer.weight.data.copy_(module.weight.data[rand_indexes]) + # new_layer.bias.data.copy_(module.bias.data[rand_indexes]) + + # else: + # continue + + # original_layer_str = str(module) + # set_module(fm_vit, name, new_layer) + # logger.debug(f'set_module, {name}, {new_layer}') + # logger.debug(f'slim {name} from {original_layer_str} to {new_layer}') + + return fm_vit + +class FMLoRA_GLIP_Util(FMLoRA_Util): + def train_only_lora_and_conv(self, fm: nn.Module): + res = [] + for n, m in fm.named_modules(): + if isinstance(m, LoRA) or isinstance(m, nn.Conv2d): + for p in m.parameters(): + p.requires_grad = True + res += [p] + else: + for p in m.parameters(): + p.requires_grad = False + return res + + + @torch.no_grad() + def add_lora_ab_to_fm(self, fm: nn.Module, ab_r: int, samples): + fm.eval() + + # samples = {'images' : samples[0], 'targets' : samples[1]} + + for k, v in samples.items(): + if isinstance(v, torch.Tensor) or isinstance(v, BoxList): + samples[k] = v.to(get_model_device(fm)) + print(k) + + _, o1_token_logits, o1_dot_product_logits = fm(**samples) + + mo_list = {k:v for k, v in fm.named_modules()} + + for name, module in fm.named_modules(): + if '.proj' in name or 'out' in name: + continue + if name.endswith(('k_proj', 'q_proj', 'v_proj', 'qkv', 'attn.proj', 'l_proj', 'query', 'key', 'value')): + set_module(fm, name, ToQKV_WrappedWithLoRA(module, ab_r)) + + _, o2_token_logits, o2_dot_product_logits = fm(**samples) + + output_diff = 0. + for o1, o2 in list(zip(o1_dot_product_logits, o2_dot_product_logits)): + output_diff += ((o1 - o2) ** 2).sum() + + if o1_token_logits is not None: + output_diff += ((o1_token_logits - o2_token_logits) ** 2).sum() + assert output_diff < 1e-5 + + return fm + + @torch.no_grad() + def absorb_lora_and_recover_net_structure(self, fm: nn.Module, samples: dict): + fm.eval() + # print('absorb lora before') + + for k, v in samples.items(): + if isinstance(v, torch.Tensor): + samples[k] = v.to(get_model_device(fm)) + print(k) + + _, o1_token_logits, o1_dot_product_logits = fm(**samples) + + for name, module in fm.named_modules(): + if not isinstance(module, ToQKV_WrappedWithLoRA): + continue + + fc = module.fc + ab = module.ab + + fc.weight.add_(ab[1].weight @ ab[0].weight) + + set_module(fm, name, fc) + + # print('absorb lora after') + _, o2_token_logits, o2_dot_product_logits = fm(**samples) + + output_diff = 0. + for o1, o2 in list(zip(o1_dot_product_logits, o2_dot_product_logits)): + output_diff += ((o1 - o2) ** 2).sum() + + if o1_token_logits is not None: + output_diff += ((o1_token_logits - o2_token_logits) ** 2).sum() + assert output_diff < 1e-3, output_diff + + return fm + +class ElasticDNN_OfflineMMDetFMModel(ElasticDNN_OfflineFMModel): + def __init__(self, name: str, models_dict_path: str, device: str, num_classes=10, collate_fn=None): + super().__init__(name, models_dict_path, device) + self.num_classes = num_classes + self.collate_fn = collate_fn + + def get_accuracy(self, test_loader, *args, **kwargs): + # print('DeeplabV3: start test acc') + _d = test_loader.dataset + from data import build_dataloader + if _d.__class__.__name__ == 'MergedDataset': + # print('\neval on merged datasets') + datasets = _d.datasets + if self.collate_fn is None: + test_loaders = [build_dataloader(d, test_loader.batch_size, test_loader.num_workers, False, None, collate_fn=None) for d in datasets] + else: + test_loaders = [build_dataloader(d, test_loader.batch_size, test_loader.num_workers, False, None, collate_fn=self.collate_fn) for d in datasets] + accs = [self.get_accuracy(loader) for loader in test_loaders] + # print(accs) + return sum(accs) / len(accs) + + # print('dataset len', len(test_loader.dataset)) + + model = self.models_dict['main'] + device = self.device + model.eval() + + # print('# classes', model.num_classes) + + model = model.to(device) + from evaluator import COCOEvaluator, MMCOCODecoder + from utils.common.others import HiddenPrints + with torch.no_grad(): + with HiddenPrints(): + evaluator = COCOEvaluator( + dataloader=test_loader, + img_size=(416, 416), + confthre=0.01, + nmsthre=0.65, + num_classes=len(test_loader.dataset.classes), + testdev=True + ) + res = evaluator.evaluate(model, False, False, decoder=MMCOCODecoder) + map50 = res[1] + # print('eval info', res[-1]) + return map50 + + def infer(self, x, *args, **kwargs): + if len(args) > 0: + print(args, len(args)) + return self.models_dict['main'](x, *args) # forward(x, label) + return self.models_dict['main'](**x) + +class ElasticDNN_OfflineMMDetMDModel(ElasticDNN_OfflineMDModel): + def __init__(self, name: str, models_dict_path: str, device: str, num_classes=10, collate_fn=None): + super().__init__(name, models_dict_path, device) + self.num_classes = num_classes + self.collate_fn = collate_fn + + def get_accuracy(self, test_loader, *args, **kwargs): + # print('DeeplabV3: start test acc') + _d = test_loader.dataset + from data import build_dataloader + if _d.__class__.__name__ == 'MergedDataset': + # print('\neval on merged datasets') + datasets = _d.datasets + if self.collate_fn is None: + test_loaders = [build_dataloader(d, test_loader.batch_size, test_loader.num_workers, False, None, collate_fn=None) for d in datasets] + else: + test_loaders = [build_dataloader(d, test_loader.batch_size, test_loader.num_workers, False, None, collate_fn=self.collate_fn) for d in datasets] + accs = [self.get_accuracy(loader) for loader in test_loaders] + # print(accs) + return sum(accs) / len(accs) + + # print('dataset len', len(test_loader.dataset)) + + model = self.models_dict['main'] + device = self.device + model.eval() + + # print('# classes', model.num_classes) + + model = model.to(device) + from evaluator import COCOEvaluator, MMCOCODecoder + from utils.common.others import HiddenPrints + with torch.no_grad(): + with HiddenPrints(): + evaluator = COCOEvaluator( + dataloader=test_loader, + img_size=(416, 416), + confthre=0.01, + nmsthre=0.65, + num_classes=len(test_loader.dataset.classes), + testdev=True + ) + res = evaluator.evaluate(model, False, False, decoder=MMCOCODecoder) + map50 = res[1] + # print('eval info', res[-1]) + return map50 + + def infer(self, x, *args, **kwargs): + if len(args) > 0: + return self.models_dict['main'](x, *args) # forward(x, label) + return self.models_dict['main'](**x) + +class SqueezeLast(nn.Module): + def __init__(self): + super(SqueezeLast, self).__init__() + + def forward(self, x): + return x.squeeze(-1) + + +class ProjConv_WrappedWithFBS(Layer_WrappedWithFBS): + def __init__(self, raw_conv2d: nn.Conv2d, r): + super(ProjConv_WrappedWithFBS, self).__init__() + + self.fbs = nn.Sequential( + Abs(), + nn.AdaptiveAvgPool2d(1), + nn.Flatten(), + nn.Linear(raw_conv2d.in_channels, raw_conv2d.out_channels // r), + nn.ReLU(), + nn.Linear(raw_conv2d.out_channels // r, raw_conv2d.out_channels), + nn.ReLU() + ) + + self.raw_conv2d = raw_conv2d + # self.raw_bn = raw_bn # remember clear the original BNs in the network + + nn.init.constant_(self.fbs[5].bias, 1.) + nn.init.kaiming_normal_(self.fbs[5].weight) + + def forward(self, x): + raw_x = self.raw_conv2d(x) + + if self.use_cached_channel_attention and self.cached_channel_attention is not None: + channel_attention = self.cached_channel_attention + else: + self.cached_raw_channel_attention = self.fbs(x) + self.cached_channel_attention = self.k_takes_all(self.cached_raw_channel_attention) + + channel_attention = self.cached_channel_attention + + return raw_x * channel_attention.unsqueeze(2).unsqueeze(3) + + +class Linear_WrappedWithFBS(Layer_WrappedWithFBS): + def __init__(self, linear: nn.Linear, r): + super(Linear_WrappedWithFBS, self).__init__() + + self.linear = linear + + # for conv: (B, C_in, H, W) -> (B, C_in) -> (B, C_out) + # for mlp in ViT: (B, #patches, D: dim of patches embedding) -> (B, D) -> (B, C_out) + self.fbs = nn.Sequential( + Rearrange('b n d -> b d n'), + Abs(), + nn.AdaptiveAvgPool1d(1), + SqueezeLast(), + nn.Linear(linear.in_features, max(linear.out_features // r, 36)), + nn.ReLU(), + nn.Linear(max(linear.out_features // r, 36), linear.out_features), + nn.ReLU() + ) + + nn.init.constant_(self.fbs[6].bias, 1.) + nn.init.kaiming_normal_(self.fbs[6].weight) + + + def forward(self, x): + if self.use_cached_channel_attention and self.cached_channel_attention is not None: + channel_attention = self.cached_channel_attention + else: + self.cached_raw_channel_attention = self.fbs(x) + self.cached_channel_attention = self.k_takes_all(self.cached_raw_channel_attention) + + channel_attention = self.cached_channel_attention + + raw_res = self.linear(x) + + return channel_attention.unsqueeze(1) * raw_res + + +class ToQKV_WrappedWithFBS(Layer_WrappedWithFBS): + """ + This regards to_q/to_k/to_v as a whole (in fact it consists of multiple heads) and prunes it. + It seems different channels of different heads are pruned according to the input. + This is different from "removing some head" or "removing the same channels in each head". + """ + def __init__(self, to_qkv: nn.Linear, r): + super(ToQKV_WrappedWithFBS, self).__init__() + + # self.to_qkv = to_qkv + + self.to_qk = nn.Linear(to_qkv.in_features, to_qkv.out_features // 3 * 2, bias=to_qkv.bias is not None) + self.to_v = nn.Linear(to_qkv.in_features, to_qkv.out_features // 3, bias=to_qkv.bias is not None) + self.to_qk.weight.data.copy_(to_qkv.weight.data[0: to_qkv.out_features // 3 * 2]) + if to_qkv.bias is not None: + self.to_qk.bias.data.copy_(to_qkv.bias.data[0: to_qkv.out_features // 3 * 2]) + self.to_v.weight.data.copy_(to_qkv.weight.data[to_qkv.out_features // 3 * 2: ]) + if to_qkv.bias is not None: + self.to_v.bias.data.copy_(to_qkv.bias.data[to_qkv.out_features // 3 * 2: ]) + + self.fbs = nn.Sequential( + Rearrange('b n d -> b d n'), + Abs(), + nn.AdaptiveAvgPool1d(1), + SqueezeLast(), + nn.Linear(to_qkv.in_features, to_qkv.out_features // 3 // r), + nn.ReLU(), + # nn.Linear(to_qkv.out_features // 3 // r, to_qkv.out_features // 3), + nn.Linear(to_qkv.out_features // 3 // r, self.to_v.out_features), + nn.ReLU() + ) + + nn.init.constant_(self.fbs[6].bias, 1.) + nn.init.kaiming_normal_(self.fbs[6].weight) + + def forward(self, x): + if self.use_cached_channel_attention and self.cached_channel_attention is not None: + channel_attention = self.cached_channel_attention + else: + self.cached_raw_channel_attention = self.fbs(x) + + # print() + # for attn in self.cached_raw_channel_attention.chunk(3, dim=1)[0: 1]: + # print(self.cached_raw_channel_attention.size(), attn.size()) + # print(self.k_takes_all.k) + # print(attn[0].nonzero(as_tuple=True)[0].size(), attn[0]) + + self.cached_channel_attention = self.k_takes_all(self.cached_raw_channel_attention) + + + # for attn in self.cached_channel_attention.chunk(3, dim=1)[0: 1]: + # print(self.cached_channel_attention.size(), attn.size()) + # print(self.k_takes_all.k) + # print(attn[0].nonzero(as_tuple=True)[0].size(), attn[0]) + # print() + + channel_attention = self.cached_channel_attention + + qk = self.to_qk(x) + v = channel_attention.unsqueeze(1) * self.to_v(x) + return torch.cat([qk, v], dim=-1) + + # qkv = raw_res.chunk(3, dim = -1) + + # raw_v = qkv[2] + # print('raw_k, raw_v', qkv[0].sum((0, 1))[0: 10], qkv[0].sum((0, 1)).nonzero(as_tuple=True)[0].size(), + # qkv[1].sum((0, 1))[0: 10], qkv[1].sum((0, 1)).nonzero(as_tuple=True)[0].size(),) + # print('raw_v', raw_v.size(), raw_v.sum((0, 1))[0: 10], raw_v.sum((0, 1)).nonzero(as_tuple=True)[0].size()) + + # qkv_attn = channel_attention.chunk(3, dim=-1) + # print('attn', [attn[0][0: 10] for attn in qkv_attn]) + # print(channel_attention.unsqueeze(1).size(), raw_res.size()) + # print('fbs', channel_attention.size(), raw_res.size()) + # return channel_attention.unsqueeze(1) * raw_res + + +class StaticFBS(nn.Module): + def __init__(self, static_channel_attention): + super(StaticFBS, self).__init__() + assert static_channel_attention.dim() == 2 and static_channel_attention.size(0) == 1 + self.static_channel_attention = nn.Parameter(static_channel_attention, requires_grad=False) # (1, dim) + + def forward(self, x): + # print('staticfbs', x, self.static_channel_attention.unsqueeze(1)) + return x * self.static_channel_attention.unsqueeze(1) + +class ElasticGLIPUtil(ElasticDNNUtil): + def convert_raw_dnn_to_master_dnn(self, raw_dnn: nn.Module, r: float, ignore_layers=[]): + assert len(ignore_layers) == 0, 'not supported yet' + + raw_vit = deepcopy(raw_dnn) + + + + for name, module in raw_vit.named_modules(): + # if name.endswith('patch_embed'): + # set_module(module, 'proj', ProjConv_WrappedWithFBS(module.proj, r)) + # if name.endswith('attn') and not name.endswith('b_attn.attn') and not name.endswith('b_attn'): + # set_module(module, 'qkv', ToQKV_WrappedWithFBS(module.qkv, r)) + if name.endswith('intermediate'): + set_module(module, 'dense', Linear_WrappedWithFBS(module.dense, r)) + elif name.endswith('mlp'): + set_module(module, 'fc1', Linear_WrappedWithFBS(module.fc1, r)) + + return raw_vit + + def set_master_dnn_sparsity(self, master_dnn: nn.Module, sparsity: float): + # for name, module in master_dnn.named_modules(): + # if not name.endswith('attn'): + # continue + + # q_features = module.qkv.to_qk.out_features // 2 + + # if (q_features - int(q_features * sparsity)) % module.num_heads != 0: + # # tune sparsity to ensure #unpruned channel % num_heads == 0 + # # so that the pruning seems to reduce the dim_head of each head + # tuned_sparsity = 1. - int((q_features - int(q_features * sparsity)) / module.num_heads) * module.num_heads / q_features + # logger.debug(f'tune sparsity from {sparsity:.2f} to {tuned_sparsity}') + # sparsity = tuned_sparsity + # break + + return super().set_master_dnn_sparsity(master_dnn, sparsity) + + def select_most_rep_sample(self, master_dnn: nn.Module, samples: torch.Tensor): + # print(samples) + sample={} + sample['images'] = [samples['images'][0]] + sample['targets'] = [samples['targets'][0]] + # return samples[0].unsqueeze(0) + # res = {k: v[0: 1] for k, v in samples.items()} + return sample + + def extract_surrogate_dnn_via_samples(self, master_dnn: nn.Module, samples: torch.Tensor, return_detail=False):#产生小模型的步骤 + sample = self.select_most_rep_sample(master_dnn, samples) + # assert sample.dim() == 4 and sample.size(0) == 1 + + # print('before') + master_dnn.eval() + self.clear_cached_channel_attention_in_master_dnn(master_dnn) + with torch.no_grad(): + _, o1_token_logits, o1_dot_product_logits = master_dnn(**sample) + # print('after') + + boosted_vit = deepcopy(master_dnn) + + def get_unpruned_indexes_from_channel_attn(channel_attn: torch.Tensor, k): + assert channel_attn.size(0) == 1, 'use A representative sample to generate channel attentions' + + # print('attn_in_unpruned', channel_attn[0][0: 10]) + + res = channel_attn[0].nonzero(as_tuple=True)[0] # should be one-dim + + # res = channel_attn[0].argsort(descending=True)[0: -int(channel_attn.size(1) * k)].sort()[0] + + # g = channel_attn + # k = g.size(1) - int(g.size(1) * k) + # res = g.topk(k, 1)[1][0].sort()[0] + + return res + + unpruned_indexes_of_layers = {} + + # for attn, ff in boosted_vit.transformer.layers: + # for block_i, block in enumerate(boosted_vit.blocks): + for layer_i, layer in enumerate(boosted_vit.model.backbone.body.layers): + for block_i, block in enumerate(layer.blocks): + # attn = block.attn + # ff = block.mlp + + ff_0 = get_module(block, f'mlp.fc1') + # ff_0_unpruned_indexes = get_unpruned_indexes_from_channel_attn(ff_0.cached_channel_attention, k) + ff_0_pruned_indexes = ff_0.k_takes_all.cached_i[0].sort()[0] + ff_0_unpruned_indexes = torch.LongTensor([ii for ii in range(ff_0.cached_channel_attention.size(1)) if ii not in ff_0_pruned_indexes]) + new_ff_0 = nn.Linear(ff_0.linear.in_features, ff_0_unpruned_indexes.size(0), ff_0.linear.bias is not None) + new_ff_0.weight.data.copy_(ff_0.linear.weight.data[ff_0_unpruned_indexes]) + if ff_0.linear.bias is not None: + new_ff_0.bias.data.copy_(ff_0.linear.bias.data[ff_0_unpruned_indexes]) + set_module(block, 'mlp.fc1', nn.Sequential(new_ff_0, StaticFBS(ff_0.cached_channel_attention[:, ff_0_unpruned_indexes]))) + + ff_1 = get_module(block, f'mlp.fc2') + new_ff_1 = nn.Linear(ff_0_unpruned_indexes.size(0), ff_1.out_features, ff_1.bias is not None) + new_ff_1.weight.data.copy_(ff_1.weight.data[:, ff_0_unpruned_indexes]) + if ff_1.bias is not None: + new_ff_1.bias.data.copy_(ff_1.bias.data) + set_module(block, 'mlp.fc2', new_ff_1) + + unpruned_indexes_of_layers[f'model.backbone.body.layers.{layer_i}.blocks.{block_i}.mlp.fc1.0.weight'] = ff_0_unpruned_indexes + # for block_i,block in enumerate(boosted_vit.vision_model.encoder.layers): + + # attn = block.self_attn + # ff = block.mlp + # ff_0 = ff.fc1 + # # ff_0_unpruned_indexes = get_unpruned_indexes_from_channel_attn(ff_0.cached_channel_attention, k) + # ff_0_pruned_indexes = ff_0.k_takes_all.cached_i[0].sort()[0] + # ff_0_unpruned_indexes = torch.LongTensor([ii for ii in range(ff_0.cached_channel_attention.size(1)) if ii not in ff_0_pruned_indexes]) + # new_ff_0 = nn.Linear(ff_0.linear.in_features, ff_0_unpruned_indexes.size(0), ff_0.linear.bias is not None) + # new_ff_0.weight.data.copy_(ff_0.linear.weight.data[ff_0_unpruned_indexes]) + # if ff_0.linear.bias is not None: + # new_ff_0.bias.data.copy_(ff_0.linear.bias.data[ff_0_unpruned_indexes]) + # set_module(ff, 'fc1', nn.Sequential(new_ff_0, StaticFBS(ff_0.cached_channel_attention[:, ff_0_unpruned_indexes]))) + + # ff_1 = ff.fc2 + # new_ff_1 = nn.Linear(ff_0_unpruned_indexes.size(0), ff_1.out_features, ff_1.bias is not None) + # new_ff_1.weight.data.copy_(ff_1.weight.data[:, ff_0_unpruned_indexes]) + # if ff_1.bias is not None: + # new_ff_1.bias.data.copy_(ff_1.bias.data) + # set_module(ff, 'fc2', new_ff_1) + + # unpruned_indexes_of_layers[f'vision_model.encoder.layers.{block_i}.mlp.fc1.0.weight'] = ff_0_unpruned_indexes + + + # for block_i, block in enumerate(boosted_vit.text_decoder.bert.encoder.layer): + # # attn = block.attn + # # ff = block.mlp + + # ff_0 = get_module(block, f'intermediate.dense') + # # ff_0_unpruned_indexes = get_unpruned_indexes_from_channel_attn(ff_0.cached_channel_attention, k) + # ff_0_pruned_indexes = ff_0.k_takes_all.cached_i[0].sort()[0] + # ff_0_unpruned_indexes = torch.LongTensor([ii for ii in range(ff_0.cached_channel_attention.size(1)) if ii not in ff_0_pruned_indexes]) + # new_ff_0 = nn.Linear(ff_0.linear.in_features, ff_0_unpruned_indexes.size(0), ff_0.linear.bias is not None) + # new_ff_0.weight.data.copy_(ff_0.linear.weight.data[ff_0_unpruned_indexes]) + # if ff_0.linear.bias is not None: + # new_ff_0.bias.data.copy_(ff_0.linear.bias.data[ff_0_unpruned_indexes]) + # set_module(block, 'intermediate.dense', nn.Sequential(new_ff_0, StaticFBS(ff_0.cached_channel_attention[:, ff_0_unpruned_indexes]))) + + # ff_1 = get_module(block, f'output.dense') + # new_ff_1 = nn.Linear(ff_0_unpruned_indexes.size(0), ff_1.out_features, ff_1.bias is not None) + # new_ff_1.weight.data.copy_(ff_1.weight.data[:, ff_0_unpruned_indexes]) + # if ff_1.bias is not None: + # new_ff_1.bias.data.copy_(ff_1.bias.data) + # set_module(block, 'output.dense', new_ff_1) + + # unpruned_indexes_of_layers[f'text_decoder.bert.encoder.layer.{block_i}.intermediate.dense.0.weight'] = ff_0_unpruned_indexes + surrogate_dnn = boosted_vit + surrogate_dnn.eval() + surrogate_dnn = surrogate_dnn.to(get_model_device(master_dnn)) + # logger.debug(surrogate_dnn) + with torch.no_grad(): + _, o2_token_logits, o2_dot_product_logits = surrogate_dnn(**sample) + + output_diff = 0. + for o1, o2 in list(zip(o1_dot_product_logits, o2_dot_product_logits)): + output_diff += ((o1 - o2) ** 2).sum() + + if o1_token_logits is not None: + output_diff += ((o1_token_logits - o2_token_logits) ** 2).sum() + # assert output_diff < 1e-4, output_diff + logger.info(f'output diff of master and surrogate DNN: {output_diff}') + # logger.debug(f'example output of master/surrogate: {master_dnn_output.sum(0)[0: 10]}, {surrogate_dnn_output.sum(0)[0: 10]}') + # logger.info(f'\nonly prune mlp!!!!\n') + # logger.info(f'\nonly prune mlp!!!!\n') + + if return_detail: + return boosted_vit, unpruned_indexes_of_layers + + return boosted_vit + + def extract_surrogate_dnn_via_samples_with_perf_test(self, master_dnn: nn.Module, samples, return_detail=False): + master_dnn_size = get_model_size(master_dnn, True) + sample = {} + sample['images'] = [samples['images'][0]] + sample['targets'] = [samples['targets'][0]] + master_dnn_latency = self._get_model_latency(master_dnn, sample, 50, + get_model_device(master_dnn), 50, False) + + res = self.extract_surrogate_dnn_via_samples(master_dnn, samples, return_detail) + if not return_detail: + surrogate_dnn = res + else: + surrogate_dnn, unpruned_indexes_of_layers = res + surrogate_dnn_size = get_model_size(surrogate_dnn, True) + surrogate_dnn_latency = self._get_model_latency(master_dnn, samples, 50, + get_model_device(master_dnn), 50, False) + + logger.info(f'master DNN ({master_dnn_size:.3f}MB, {master_dnn_latency:.4f}s/sample) -> ' + f'surrogate DNN ({surrogate_dnn_size:.3f}MB, {surrogate_dnn_latency:.4f}s/sample)\n' + f'(model size: ↓ {(master_dnn_size / surrogate_dnn_size):.2f}x, ' + f'latency: ↓ {(master_dnn_latency / surrogate_dnn_latency):.2f}x)') + + return res + + def _get_model_latency(self, model: torch.nn.Module, sample, sample_num: int, + device: str, warmup_sample_num: int, return_detail=False): + import time + + model = model.to(device) + model.eval() + sample['images'] = [sample['images'][0]] + sample['targets'] = [sample['targets'][0]] + # warm up + with torch.no_grad(): + for _ in range(warmup_sample_num): + model(**sample) + + infer_time_list = [] + + if device == 'cuda' or 'cuda' in str(device): + with torch.no_grad(): + for _ in range(sample_num): + s, e = torch.cuda.Event(enable_timing=True), torch.cuda.Event(enable_timing=True) + s.record() + model(**sample) + e.record() + torch.cuda.synchronize() + cur_model_infer_time = s.elapsed_time(e) / 1000. + infer_time_list += [cur_model_infer_time] + + else: + with torch.no_grad(): + for _ in range(sample_num): + start = time.time() + model(**sample) + cur_model_infer_time = time.time() - start + infer_time_list += [cur_model_infer_time] + + avg_infer_time = sum(infer_time_list) / sample_num + + if return_detail: + return avg_infer_time, infer_time_list + return avg_infer_time + + +# from typing import Any, Dict +# from schema import Schema, Or +# import schema +# from data import Scenario, MergedDataset +# from methods.base.alg import BaseAlg +# from data import build_dataloader +# from ..model import ElasticDNN_OfflineFMModel, ElasticDNN_OfflineMDModel +# from ...model.base import ElasticDNNUtil +# import torch.optim +# import tqdm +# import torch.nn.functional as F +# from torch import nn +# from utils.dl.common.env import create_tbwriter +# import os +# import random +# import numpy as np +# from copy import deepcopy +# from utils.dl.common.model import LayerActivation2, get_module +# from utils.common.log import logger + + +# class ElasticDNN_Det_MDPretrainingWoFBSAlg(BaseAlg): +# """ +# TODO: fine-tuned FM -> init MD -> trained MD -> construct indexes (only between similar weights) and fine-tune +# """ +# def get_required_models_schema(self) -> Schema: +# return Schema({ +# 'fm': ElasticDNN_OfflineFMModel, +# 'md': ElasticDNN_OfflineMDModel +# }) + +# def get_required_hyp_schema(self) -> Schema: +# return Schema({ +# 'launch_tbboard': bool, + +# 'samples_size': any, +# 'generate_md_width_ratio': int, + +# 'train_batch_size': int, +# 'val_batch_size': int, +# 'num_workers': int, +# 'optimizer': str, +# 'optimizer_args': dict, +# 'scheduler': str, +# 'scheduler_args': dict, +# 'num_iters': int, +# 'val_freq': int, +# 'distill_loss_weight': float +# }) + +# def run(self, scenario: Scenario, hyps: Dict) -> Dict[str, Any]: +# super().run(scenario, hyps) + +# assert isinstance(self.models['md'], ElasticDNN_OfflineMDModel) # for auto completion +# assert isinstance(self.models['fm'], ElasticDNN_OfflineFMModel) # for auto completion + +# # 1. add FBS +# device = self.models['md'].device + +# if self.models['md'].models_dict['main'] == -1: +# logger.info(f'init master DNN by reducing width of an adapted foundation model (already tuned by LoRA)...') + +# before_fm_model = deepcopy(self.models['fm'].models_dict['main']) +# lora_util = self.models['fm'].get_lora_util() + +# sample = hyps['samples_size'] +# if isinstance(sample, (tuple, list)) and isinstance(sample[0], int): +# sample = torch.rand(hyps['samples_size']).to(device) + +# lora_absorbed_fm_model = lora_util.absorb_lora_and_recover_net_structure(self.models['fm'].models_dict['main'], +# sample) +# self.models['fm'].models_dict['main'] = lora_absorbed_fm_model +# master_dnn = self.models['fm'].generate_md_by_reducing_width(hyps['generate_md_width_ratio'], +# sample) +# self.models['fm'].models_dict['main'] = before_fm_model + +# self.models['md'].models_dict['main'] = master_dnn +# self.models['md'].to(device) + +# # 2. train (knowledge distillation, index relationship) +# offline_datasets = scenario.get_offline_datasets() +# train_dataset = MergedDataset([d['train'] for d in offline_datasets.values()]) +# val_dataset = MergedDataset([d['val'] for d in offline_datasets.values()]) +# train_loader = iter(build_dataloader(train_dataset, hyps['train_batch_size'], hyps['num_workers'], +# True, None)) +# val_loader = build_dataloader(val_dataset, hyps['val_batch_size'], hyps['num_workers'], +# False, False) + +# # logger.info(f'FM acc: {self.models["fm"].get_accuracy(val_loader):.4f}') + +# # 2.1 train whole master DNN (knowledge distillation) +# for p in master_dnn.parameters(): +# p.requires_grad = True +# self.models['md'].to_train_mode() + +# optimizer = torch.optim.__dict__[hyps['optimizer']]([ +# {'params': self.models['md'].models_dict['main'].parameters(), **hyps['optimizer_args']} +# ]) +# scheduler = torch.optim.lr_scheduler.__dict__[hyps['scheduler']](optimizer, **hyps['scheduler_args']) +# tb_writer = create_tbwriter(os.path.join(self.res_save_dir, 'tb_log'), launch_tbboard=hyps['launch_tbboard']) +# pbar = tqdm.tqdm(range(hyps['num_iters']), dynamic_ncols=True) +# best_avg_val_acc = 0. + +# md_output_hook = None + +# for iter_index in pbar: +# self.models['md'].to_train_mode() +# self.models['fm'].to_eval_mode() + +# # rand_sparsity = random.random() * (hyps['max_sparsity'] - hyps['min_sparsity']) + hyps['min_sparsity'] +# # elastic_dnn_util.set_master_dnn_sparsity(self.models['md'].models_dict['main'], rand_sparsity) +# if md_output_hook is None: +# md_output_hook = self.models['md'].get_feature_hook() +# fm_output_hook = self.models['fm'].get_feature_hook() + +# x, y = next(train_loader) +# if isinstance(x, dict): +# for k, v in x.items(): +# if isinstance(v, torch.Tensor): +# x[k] = v.to(device) +# y = y.to(device) +# else: +# x, y = x.to(device), y.to(device) + +# with torch.no_grad(): +# fm_output = self.models['fm'].infer(x) +# task_loss = self.models['md'].forward_to_get_task_loss(x, y) + +# md_output = md_output_hook.output +# fm_output = fm_output_hook.output + +# distill_loss = hyps['distill_loss_weight'] * self.models['md'].get_distill_loss(md_output, fm_output) +# total_loss = task_loss + distill_loss + +# optimizer.zero_grad() +# total_loss.backward() +# optimizer.step() +# scheduler.step() + +# if (iter_index + 1) % hyps['val_freq'] == 0: + +# # elastic_dnn_util.clear_cached_channel_attention_in_master_dnn(self.models['md'].models_dict['main']) +# md_output_hook.remove() +# md_output_hook = None +# fm_output_hook.remove() +# fm_output_hook = None + +# cur_md = self.models['md'].models_dict['main'] +# md_for_test = deepcopy(self.models['md'].models_dict['main']) +# val_acc = 0. + +# self.models['md'].models_dict['main'] = md_for_test +# self.models['md'].to_eval_mode() +# val_acc = self.models['md'].get_accuracy(val_loader) + +# self.models['md'].models_dict['main'] = cur_md + +# self.models['md'].save_model(os.path.join(self.res_save_dir, 'models/md_last.pt')) +# self.models['fm'].save_model(os.path.join(self.res_save_dir, 'models/fm_last.pt')) + +# if val_acc > best_avg_val_acc: +# best_avg_val_acc = val_acc +# self.models['md'].save_model(os.path.join(self.res_save_dir, 'models/md_best.pt')) +# self.models['fm'].save_model(os.path.join(self.res_save_dir, 'models/fm_best.pt')) + +# tb_writer.add_scalars(f'losses', dict(task=task_loss, distill=distill_loss, total=total_loss), iter_index) +# pbar.set_description(f'loss: {total_loss:.6f}') +# if (iter_index + 1) >= hyps['val_freq']: +# tb_writer.add_scalar(f'accs/val_acc', val_acc, iter_index) +# pbar.set_description(f'loss: {total_loss:.6f}, val_acc: {val_acc:.4f}') + +# if __name__ == '__main__': +# model = glip_model('new_impl/cv/glip/object_detection/pretrained_model/glip_Swin_T_O365_GoldG.yaml','new_impl/cv/glip/object_detection/pretrained_model/glip_tiny_model_o365_goldg_cc_sbu.pth').cuda() +# model.eval() +# # print(model) +# # exit() + + +# # config = CLIPConfig.from_pretrained('openai/clip-vit-base-patch16') +# # print(config) + +# # # test 1: single image inference + # from PIL import Image, ImageDraw + # import requests + # import numpy as np + # ori_image = Image.open('new_impl/cv/glip/object_detection/9472793441_b7822c00de_z.jpg').convert("RGB") + # image = [np.asarray(ori_image)[:, :, [2, 1, 0]]] + # text = 'sofa . remote . dog . person . car . sky . plane .' + # target = torch.Tensor() + # o = model(image, text) + # o = model._post_process(o[0]) + # print(o) + # bboxes = o.bbox.cpu() + # a = ImageDraw.ImageDraw(ori_image) + # for box in bboxes: + # box = box.int() + # a.rectangle(((box[0], box[1]), (box[2], box[3])), fill=None, outline='red', width=2) + # ori_image.save('test.jpg') +# # print(o.logits_per_image.softmax(dim=1)) + +# # o = model(image, torch.load('dnns/clip/test_input_embed.pth'), False) +# # # print(o) +# # print(o.logits_per_image.softmax(dim=1)) +# # exit() + +# # test 2: normal training using clip loss (batch) +# from data import get_dataset, build_dataloader +# from torchvision.transforms import Compose, ToTensor, Resize +# dataset = get_dataset('Caltech256', '/data/zql/datasets/Caltech-256/data/caltech256/256_ObjectCategories/', 'train', transform=Compose([ +# Resize((32, 32)), ToTensor() +# ])) +# dataloader = build_dataloader(dataset, 8, 0, True, None) + +# from PIL import Image +# import requests +# images, labels = next(iter(dataloader)) + +# # torch.save(images, 'dnns/clip/test_image.pth') +# classes = dataset.classes +# text = [f"a photo of a {classes[i]}" for i in labels] # should be ground truth +# print(text) +# print(images.size()) + +# o = model(images, text, True) +# print(o) +# print(o.logits_per_image.softmax(dim=1)) + +# # o = model(image, torch.load('dnns/clip/test_input_embed.pth'), False) +# # # print(o) +# # print(o.logits_per_image.softmax(dim=1)) \ No newline at end of file diff --git a/new_impl/cv/glip/object_detection/pretrained_model/glip_A_Swin_T_O365.yaml b/new_impl/cv/glip/object_detection/pretrained_model/glip_A_Swin_T_O365.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fac505fe53136bdb7ed75d83beb2662eed1ac2ec --- /dev/null +++ b/new_impl/cv/glip/object_detection/pretrained_model/glip_A_Swin_T_O365.yaml @@ -0,0 +1,101 @@ +MODEL: + META_ARCHITECTURE: "GeneralizedVLRCNN" + WEIGHT: "swin_tiny_patch4_window7_224.pth" + RPN_ONLY: True + RPN_ARCHITECTURE: "VLDYHEAD" + + BACKBONE: + CONV_BODY: "SWINT-FPN-RETINANET" + OUT_CHANNELS: 256 + FREEZE_CONV_BODY_AT: -1 + + LANGUAGE_BACKBONE: + FREEZE: False + MODEL_TYPE: "bert-base-uncased" # "roberta-base", "clip" + MODEL_PATH: "new_impl/cv/glip/object_detection/bert-base-uncased" + MASK_SPECIAL: False + + RPN: + USE_FPN: True + ANCHOR_SIZES: (64, 128, 256, 512, 1024) + ANCHOR_STRIDE: (8, 16, 32, 64, 128) + ASPECT_RATIOS: (1.0,) + SCALES_PER_OCTAVE: 1 + + DYHEAD: + CHANNELS: 256 + NUM_CONVS: 6 + USE_GN: True + USE_DYRELU: True + USE_DFCONV: True + USE_DYFUSE: True + TOPK: 9 # topk for selecting candidate positive samples from each level + SCORE_AGG: "MEAN" + LOG_SCALE: 0.0 + + FUSE_CONFIG: + EARLY_FUSE_ON: False + TYPE: "MHA-B" + USE_CLASSIFICATION_LOSS: False + USE_TOKEN_LOSS: False + USE_CONTRASTIVE_ALIGN_LOSS: False + CONTRASTIVE_HIDDEN_DIM: 64 + USE_DOT_PRODUCT_TOKEN_LOSS: True + USE_FUSED_FEATURES_DOT_PRODUCT: False + USE_LAYER_SCALE: True + CLAMP_MIN_FOR_UNDERFLOW: True + CLAMP_MAX_FOR_OVERFLOW: True + CLAMP_BERTATTN_MIN_FOR_UNDERFLOW: True + CLAMP_BERTATTN_MAX_FOR_OVERFLOW: True + CLAMP_DOT_PRODUCT: True + + USE_CHECKPOINT: True + +TEST: + DURING_TRAINING: False + IMS_PER_BATCH: 64 + +# use for grounding model +DATASETS: + TRAIN: ("object365_dt_train", ) + TEST: ("coco_2017_val", ) + DISABLE_SHUFFLE: False + ADD_DET_PROMPT: False + RANDOM_SAMPLE_NEG: 85 + CONTROL_PROB: (0.0, 0.0, 0.5, 0.0) + + SEPARATION_TOKENS: ". " + +INPUT: + PIXEL_MEAN: [ 103.530, 116.280, 123.675 ] + PIXEL_STD: [ 57.375, 57.120, 58.395 ] + MIN_SIZE_TRAIN: 800 + MAX_SIZE_TRAIN: 1333 + MIN_SIZE_TEST: 800 + MAX_SIZE_TEST: 1333 + +AUGMENT: + MULT_MIN_SIZE_TRAIN: (480,560,640,720,800) + +DATALOADER: + SIZE_DIVISIBILITY: 32 + +SOLVER: + OPTIMIZER: ADAMW + BASE_LR: 0.0001 + LANG_LR: 0.00001 + WEIGHT_DECAY: 0.0001 + STEPS: (0.67, 0.89) + MAX_EPOCH: 30 + IMS_PER_BATCH: 64 + WARMUP_ITERS: 2000 + WARMUP_FACTOR: 0.001 + USE_AMP: True + MODEL_EMA: 0.999 + FIND_UNUSED_PARAMETERS: False + + CLIP_GRADIENTS: + ENABLED: True + CLIP_TYPE: "full_model" + CLIP_VALUE: 1.0 + NORM_TYPE: 2.0 \ No newline at end of file diff --git a/new_impl/cv/glip/object_detection/pretrained_model/glip_Swin_T_O365_GoldG.yaml b/new_impl/cv/glip/object_detection/pretrained_model/glip_Swin_T_O365_GoldG.yaml new file mode 100644 index 0000000000000000000000000000000000000000..edaf7a224677f45f11c8f2eea12efd9680d9117d --- /dev/null +++ b/new_impl/cv/glip/object_detection/pretrained_model/glip_Swin_T_O365_GoldG.yaml @@ -0,0 +1,101 @@ +MODEL: + META_ARCHITECTURE: "GeneralizedVLRCNN" + WEIGHT: "new_impl/cv/glip/object_detection/pretrained_model/glip_tiny_model_o365_goldg_cc_sbu.pth" + RPN_ONLY: True + RPN_ARCHITECTURE: "VLDYHEAD" + + BACKBONE: + CONV_BODY: "SWINT-FPN-RETINANET" + OUT_CHANNELS: 256 + FREEZE_CONV_BODY_AT: -1 + + LANGUAGE_BACKBONE: + FREEZE: False + MODEL_TYPE: "bert-base-uncased" # "roberta-base", "clip" + MODEL_PATH: "new_impl/cv/glip/object_detection/bert-base-uncased" + MASK_SPECIAL: False + + RPN: + USE_FPN: True + ANCHOR_SIZES: (64, 128, 256, 512, 1024) + ANCHOR_STRIDE: (8, 16, 32, 64, 128) + ASPECT_RATIOS: (1.0,) + SCALES_PER_OCTAVE: 1 + + DYHEAD: + CHANNELS: 256 + NUM_CONVS: 6 + USE_GN: True + USE_DYRELU: True + USE_DFCONV: True + USE_DYFUSE: True + TOPK: 9 # topk for selecting candidate positive samples from each level + SCORE_AGG: "MEAN" + LOG_SCALE: 0.0 + + FUSE_CONFIG: + EARLY_FUSE_ON: True + TYPE: "MHA-B" + USE_CLASSIFICATION_LOSS: False + USE_TOKEN_LOSS: False + USE_CONTRASTIVE_ALIGN_LOSS: False + CONTRASTIVE_HIDDEN_DIM: 64 + USE_DOT_PRODUCT_TOKEN_LOSS: True + USE_FUSED_FEATURES_DOT_PRODUCT: True + USE_LAYER_SCALE: True + CLAMP_MIN_FOR_UNDERFLOW: True + CLAMP_MAX_FOR_OVERFLOW: True + CLAMP_BERTATTN_MIN_FOR_UNDERFLOW: True + CLAMP_BERTATTN_MAX_FOR_OVERFLOW: True + CLAMP_DOT_PRODUCT: True + + USE_CHECKPOINT: True + +TEST: + DURING_TRAINING: False + IMS_PER_BATCH: 64 + +# use for grounding model +DATASETS: + TRAIN: ("object365_dt_train", "mixed_train_no_coco", "flickr30k_train", ) + TEST: ("coco_2017_val", ) + DISABLE_SHUFFLE: False + ADD_DET_PROMPT: False + RANDOM_SAMPLE_NEG: 85 + CONTROL_PROB: (0.0, 0.0, 0.5, 0.0) + + SEPARATION_TOKENS: ". " + +INPUT: + PIXEL_MEAN: [ 103.530, 116.280, 123.675 ] + PIXEL_STD: [ 57.375, 57.120, 58.395 ] + MIN_SIZE_TRAIN: 800 + MAX_SIZE_TRAIN: 1333 + MIN_SIZE_TEST: 800 + MAX_SIZE_TEST: 1333 + +AUGMENT: + MULT_MIN_SIZE_TRAIN: (480,560,640,720,800) + +DATALOADER: + SIZE_DIVISIBILITY: 32 + +SOLVER: + OPTIMIZER: ADAMW + BASE_LR: 0.0001 + LANG_LR: 0.00001 + WEIGHT_DECAY: 0.0001 + STEPS: (0.67, 0.89) + MAX_EPOCH: 30 + IMS_PER_BATCH: 64 + WARMUP_ITERS: 2000 + WARMUP_FACTOR: 0.001 + USE_AMP: True + MODEL_EMA: 0.999 + FIND_UNUSED_PARAMETERS: False + + CLIP_GRADIENTS: + ENABLED: True + CLIP_TYPE: "full_model" + CLIP_VALUE: 1.0 + NORM_TYPE: 2.0 \ No newline at end of file diff --git a/new_impl/cv/glip/object_detection/setup.py b/new_impl/cv/glip/object_detection/setup.py new file mode 100644 index 0000000000000000000000000000000000000000..7dd8f13544f56e7153cb38d08242e32e6fcb377d --- /dev/null +++ b/new_impl/cv/glip/object_detection/setup.py @@ -0,0 +1,66 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +#!/usr/bin/env python + +import glob +import os + +import torch +from setuptools import find_packages +from setuptools import setup +from torch.utils.cpp_extension import CUDA_HOME +from torch.utils.cpp_extension import CppExtension +from torch.utils.cpp_extension import CUDAExtension + +requirements = ["torch", "torchvision"] + + +def get_extensions(): + this_dir = os.path.dirname(os.path.abspath(__file__)) + extensions_dir = os.path.join(this_dir, "maskrcnn_benchmark", "csrc") + + main_file = glob.glob(os.path.join(extensions_dir, "*.cpp")) + source_cpu = glob.glob(os.path.join(extensions_dir, "cpu", "*.cpp")) + source_cuda = glob.glob(os.path.join(extensions_dir, "cuda", "*.cu")) + + sources = main_file + source_cpu + extension = CppExtension + + extra_compile_args = {"cxx": []} + define_macros = [] + + if torch.cuda.is_available() and CUDA_HOME is not None: + extension = CUDAExtension + sources += source_cuda + define_macros += [("WITH_CUDA", None)] + extra_compile_args["nvcc"] = [ + "-DCUDA_HAS_FP16=1", + "-D__CUDA_NO_HALF_OPERATORS__", + "-D__CUDA_NO_HALF_CONVERSIONS__", + "-D__CUDA_NO_HALF2_OPERATORS__", + ] + + sources = [os.path.join(extensions_dir, s) for s in sources] + + include_dirs = [extensions_dir] + + ext_modules = [ + extension( + "maskrcnn_benchmark._C", + sources, + include_dirs=include_dirs, + define_macros=define_macros, + extra_compile_args=extra_compile_args, + ) + ] + + return ext_modules + + +setup( + name="maskrcnn_benchmark", + description="object detection in pytorch", + packages=find_packages(exclude=("configs", "tests",)), + # install_requires=requirements, + ext_modules=get_extensions(), + cmdclass={"build_ext": torch.utils.cpp_extension.BuildExtension.with_options(use_ninja=False)}, +) \ No newline at end of file diff --git a/new_impl/cv/model.py b/new_impl/cv/model.py new file mode 100644 index 0000000000000000000000000000000000000000..0438822a002b7666b24b8d36994f1bf501fec359 --- /dev/null +++ b/new_impl/cv/model.py @@ -0,0 +1,256 @@ +from typing import List +from data.dataloader import build_dataloader +# from methods.elasticdnn.api.online_model import ElasticDNN_OnlineModel +from new_impl.cv.elasticdnn.api.online_model_v2 import ElasticDNN_OnlineModel + +import torch +import sys +from torch import nn +from new_impl.cv.elasticdnn.api.model import ElasticDNN_OfflineSegFMModel, ElasticDNN_OfflineSegMDModel +from new_impl.cv.elasticdnn.api.algs.md_pretraining_wo_fbs import ElasticDNN_MDPretrainingWoFBSAlg +from new_impl.cv.elasticdnn.model.base import ElasticDNNUtil +from new_impl.cv.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from new_impl.cv.elasticdnn.pipeline.offline.fm_to_md.vit import FM_to_MD_ViT_Util +from new_impl.cv.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from new_impl.cv.elasticdnn.pipeline.offline.fm_lora.vit import FMLoRA_ViT_Util +from new_impl.cv.elasticdnn.model.vit import ElasticViTUtil +from utils.common.file import ensure_dir +from utils.dl.common.model import LayerActivation, get_module, get_parameter +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +from utils.dl.common.loss import CrossEntropyLossSoft +import torch.nn.functional as F +from utils.dl.common.env import create_tbwriter +import os +from utils.common.log import logger +from utils.common.data_record import write_json +# from methods.shot.shot import OnlineShotModel +from new_impl.cv.feat_align.main import OnlineFeatAlignModel, FeatAlignAlg +import tqdm +from new_impl.cv.feat_align.mmd import mmd_rbf + +class ElasticDNN_ClsOnlineModel(ElasticDNN_OnlineModel): + def get_accuracy(self, test_loader, *args, **kwargs): + acc = 0 + sample_num = 0 + + self.to_eval_mode() + + with torch.no_grad(): + pbar = tqdm.tqdm(enumerate(test_loader), total=len(test_loader), dynamic_ncols=True, leave=False) + for batch_index, (x, y) in pbar: + x, y = x.to(self.device), y.to(self.device) + output = self.infer(x) + pred = F.softmax(output, dim=1).argmax(dim=1) + correct = torch.eq(pred, y).sum().item() + acc += correct + sample_num += len(y) + + pbar.set_description(f'cur_batch_total: {len(y)}, cur_batch_correct: {correct}, ' + f'cur_batch_acc: {(correct / len(y)):.4f}') + + acc /= sample_num + return acc + + def get_elastic_dnn_util(self) -> ElasticDNNUtil: + return ElasticViTUtil() + + def get_fm_matched_param_of_md_param(self, md_param_name): + # only between qkv.weight, norm.weight/bias + self_param_name = md_param_name + fm = self.models_dict['fm'] + # if any([k in self_param_name for k in ['fbs', 'cls_token', 'pos_embed']]): + # return None + + # p = get_parameter(self.models_dict['md'], self_param_name) + # if p.dim() == 0: + # return None + # elif p.dim() == 1 and 'norm' in self_param_name and 'weight' in self_param_name: + # return get_parameter(fm, self_param_name) + + if any([k in self_param_name for k in ['fbs', 'cls_token', 'pos_embed']]): + return None + + p = get_parameter(self.models_dict['md'], self_param_name) + if p.dim() == 0: + return None + elif p.dim() == 1 and 'layernorm' in self_param_name and 'weight' in self_param_name: + return get_parameter(fm, self_param_name) + # 1. xx.qkv.to_qkv.yy to xx.qkv.qkv.aa and xx.qkv.abs.zz + # if 'qkv.weight' in self_param_name: + # ss = self_param_name.split('.') + + # fm_qkv_name = '.'.join(ss[0: -1]) + '.qkv' + # fm_qkv = get_module(fm, fm_qkv_name) + + # fm_abs_name = '.'.join(ss[0: -1]) + '.abs' + # fm_abs = get_module(fm, fm_abs_name) + + # # NOTE: unrecoverable operation! multiply LoRA parameters to allow it being updated in update_fm_param() + # # TODO: if fm will be used for inference, _mul_lora_weight will not be applied! + # if not hasattr(fm_abs, '_mul_lora_weight'): + # logger.debug(f'set _mul_lora_weight in {fm_abs_name}') + # setattr(fm_abs, '_mul_lora_weight', + # nn.Parameter(torch.cat([(_abs[0].weight.T @ _abs[1].weight.T).T for _abs in fm_abs], dim=0))) + + # return torch.cat([ + # fm_qkv.weight.data, # task-agnositc params + # fm_abs._mul_lora_weight.data # task-specific params (LoRA) + # ], dim=0) + + # # elif 'to_qkv.bias' in self_param_name: + # # ss = self_param_name.split('.') + + # # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv.bias' + # # return get_parameter(fm, fm_qkv_name) + + # elif 'mlp.fc1' in self_param_name and 'weight' in self_param_name: + # fm_param_name = self_param_name.replace('.linear', '') + # return get_parameter(fm, fm_param_name) + + # elif 'mlp.fc2' in self_param_name and 'weight' in self_param_name: + # fm_param_name = self_param_name + # return get_parameter(fm, fm_param_name) + + # else: + # # return get_parameter(fm, self_param_name) + # return None + if ('attention.attention.projection_query' in self_param_name or 'attention.attention.projection_key' in self_param_name or \ + 'attention.attention.projection_value' in self_param_name) and ('weight' in self_param_name): + ss = self_param_name.split('.') + + fm_qkv_name = '.'.join(ss[0: -1]) + '.qkv' + fm_qkv = get_module(fm, fm_qkv_name) + + fm_abs_name = '.'.join(ss[0: -1]) + '.abs' + fm_abs = get_module(fm, fm_abs_name) + + if not hasattr(fm_abs, '_mul_lora_weight'): + logger.debug(f'set _mul_lora_weight in {fm_abs_name}') + setattr(fm_abs, '_mul_lora_weight', + nn.Parameter(torch.cat([(_abs[0].weight.T @ _abs[1].weight.T).T for _abs in fm_abs], dim=0))) + + return torch.cat([ + fm_qkv.weight.data, # task-agnositc params + fm_abs._mul_lora_weight.data # task-specific params (LoRA) + ], dim=0) + + elif ('attention.attention.projection_query' in self_param_name or 'attention.attention.projection_key' in self_param_name or \ + 'attention.attention.projection_value' in self_param_name) and ('bias' in self_param_name): + ss = self_param_name.split('.') + + fm_qkv_name = '.'.join(ss[0: -1]) + '.qkv.bias' + return get_parameter(fm, fm_qkv_name) + + elif 'intermediate.dense' in self_param_name and 'weight' in self_param_name: + fm_param_name = self_param_name.replace('.linear', '') + return get_parameter(fm, fm_param_name) + elif 'output.dense' in self_param_name and 'weight' in self_param_name: + fm_param_name = self_param_name.replace('.linear', '') + return get_parameter(fm, fm_param_name) + else: + #return get_parameter(fm, self_param_name) + return None + + + def update_fm_param(self, md_param_name, cal_new_fm_param_by_md_param): + if not ('attention.attention.projection_query.weight' in md_param_name or 'attention.attention.projection_key.weight' in md_param_name or 'attention.attention.projection_value.weight' in md_param_name): + matched_fm_param_ref = self.get_fm_matched_param_of_md_param(md_param_name) + matched_fm_param_ref.copy_(cal_new_fm_param_by_md_param) + else: + new_fm_attn_weight, new_fm_lora_weight = torch.chunk(cal_new_fm_param_by_md_param, 2, 0) + ss = md_param_name.split('.') + fm = self.models_dict['fm'] + # update task-agnostic parameters + fm_qkv_name = '.'.join(ss[0: -1]) + '.qkv' + fm_qkv = get_module(fm, fm_qkv_name) + fm_qkv.weight.data.copy_(new_fm_attn_weight) + + # update task-specific parameters + fm_abs_name = '.'.join(ss[0: -1]) + '.abs' + fm_abs = get_module(fm, fm_abs_name) + fm_abs._mul_lora_weight.data.copy_(new_fm_lora_weight) # TODO: this will not be applied in inference! + + def get_md_matched_param_of_fm_param(self, fm_param_name): + return super().get_md_matched_param_of_fm_param(fm_param_name) + + def get_md_matched_param_of_sd_param(self, sd_param_name): + # only between qkv.weight, norm.weight/bias + self_param_name = sd_param_name + md = self.models_dict['md'] + if any([k in self_param_name for k in ['fbs', 'cls_token', 'pos_embed']]): + return None + + p = get_parameter(self.models_dict['sd'], self_param_name) + if p.dim() == 0: + return None + elif p.dim() == 1 and 'norm' in self_param_name and 'weight' in self_param_name: + return get_parameter(md, self_param_name) + + # 1. xx.qkv.to_qkv.yy to xx.qkv.qkv.aa and xx.qkv.abs.zz + if 'qkv.weight' in self_param_name: + return get_parameter(md, self_param_name) + + # elif 'to_qkv.bias' in self_param_name: + # ss = self_param_name.split('.') + + # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv.bias' + # return get_parameter(fm, fm_qkv_name) + + elif 'mlp.fc1.0.weight' in self_param_name: + fm_param_name = '.'.join(self_param_name.split('.')[0: -2]) + '.linear.weight' + return get_parameter(md, fm_param_name) + + elif 'mlp.fc2' in self_param_name and 'weight' in self_param_name: + fm_param_name = self_param_name + return get_parameter(md, fm_param_name) + + else: + # return get_parameter(fm, self_param_name) + return None + + def get_task_head_params(self): + head = get_module(self.models_dict['sd'], 'head') + return list(head.parameters()) + + + +class ClsOnlineFeatAlignModel(OnlineFeatAlignModel): + def get_trained_params(self): # TODO: elastic fm only train a part of params + #qkv_and_norm_params = [p for n, p in self.models_dict['main'].named_parameters() if 'attention.attention.projection_query' in n or 'attention.attention.projection_key' in n or 'attention.attention.projection_value' in n or 'intermediate.dense' in n or 'output.dense' in n] + qkv_and_norm_params = [p for n, p in self.models_dict['main'].named_parameters()] + return qkv_and_norm_params + + def get_feature_hook(self): + return LayerActivation(get_module(self.models_dict['main'], 'classifier'), False, self.device) + + def forward_to_get_task_loss(self, x, y): + return F.cross_entropy(self.infer(x), y) + + def get_mmd_loss(self, f1, f2): + return mmd_rbf(f1, f2) + + def infer(self, x, *args, **kwargs): + return self.models_dict['main'](x).logits + + def get_accuracy(self, test_loader, *args, **kwargs): + acc = 0 + sample_num = 0 + + self.to_eval_mode() + + with torch.no_grad(): + pbar = tqdm.tqdm(enumerate(test_loader), total=len(test_loader), dynamic_ncols=True, leave=False) + for batch_index, (x, y) in pbar: + x, y = x.to(self.device), y.to(self.device) + output = self.infer(x) + pred = F.softmax(output, dim=1).argmax(dim=1) + correct = torch.eq(pred, y).sum().item() + acc += correct + sample_num += len(y) + + pbar.set_description(f'cur_batch_total: {len(y)}, cur_batch_correct: {correct}, ' + f'cur_batch_acc: {(correct / len(y)):.4f}') + + acc /= sample_num + return acc \ No newline at end of file diff --git a/new_impl/cv/resnet/cls.py b/new_impl/cv/resnet/cls.py new file mode 100644 index 0000000000000000000000000000000000000000..63a061646e1d5260d818d6cbbd850166b7d5946d --- /dev/null +++ b/new_impl/cv/resnet/cls.py @@ -0,0 +1,93 @@ +import torch +from new_impl.cv.elasticdnn.api.model import ElasticDNN_OfflineClsFMModel +#from methods.elasticdnn.api.algs.fm_lora import ElasticDNN_FMLoRAAlg +from new_impl.cv.elasticdnn.api.algs.fm_lora_cnn import ElasticDNN_FMLoRAAlg +from methods.elasticdnn.model.base import ElasticDNNUtil +from new_impl.cv.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +#from beit import FMLoRA_beit_Util +from new_impl.cv.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util + +from new_impl.cv.elasticdnn.model.vit import ElasticViTUtil +from data import build_scenario +import torch.nn.functional as F +from utils.dl.common.model import LayerActivation, get_module +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +# from transformers import CvtForImageClassification +# model = CvtForImageClassification.from_pretrained("/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/new_impl/cv/cvt_model",num_labels=20,ignore_mismatched_sizes=True).to('cuda') + +class ElasticDNN_res_OfflineClsFMModel(ElasticDNN_OfflineClsFMModel): + def generate_md_by_reducing_width(self, reducing_width_ratio, samples: torch.Tensor): + # return FM_to_MD_ViT_Util().init_md_from_fm_by_reducing_width_with_perf_test(self.models_dict['main'], + # reducing_width_ratio, samples) + raise NotImplementedError + def get_feature_hook(self) -> LayerActivation: + return LayerActivation(get_module(self.models_dict['main'], 'classifier'), True, self.device) + + def get_elastic_dnn_util(self) -> ElasticDNNUtil: + raise NotImplementedError + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + #x1 = torch.rand(1,3,224,224).to('cuda:1') + o1 = self.infer(x) + # o2 = self.infer(x1) + # print(o1.logits) + # print(o2.logits) + #print(self.models_dict['main']) + #print(o1.logits.shape) + #print(F.cross_entropy(self.infer(x).logits, y) ) + #formatted_values = [[round(value, 4) for value in row] for row in o1.logits.tolist()] + #return F.cross_entropy(torch.tensor(formatted_values).to('cuda'), y) + return F.cross_entropy(o1.logits, y) #这个是适用于hugging face模型的计算形式，因为它输出的是一个实例化的类，结果封装在类的属性里，你得去给它调出来。 + + def get_lora_util(self) -> FMLoRA_Util: + raise NotImplementedError + #return FMLoRA_beit_Util() + + def get_task_head_params(self): + head = get_module(self.models_dict['main'], 'classifier') + return list(head.parameters()) +if __name__ == '__main__': + scenario = build_scenario( + source_datasets_name=['GTA5Cls', 'SuperviselyPersonCls'], + target_datasets_order=['CityscapesCls', 'BaiduPersonCls'] * 15, + da_mode='close_set', + data_dirs={ + 'GTA5Cls': '/data/zql/datasets/gta5_for_cls_task', + 'SuperviselyPersonCls': '/data/zql/datasets/supervisely_person_for_cls_task', + 'CityscapesCls': '/data/zql/datasets/cityscapes_for_cls_task', + 'BaiduPersonCls': '/data/zql/datasets/baidu_person_for_cls_task' + }, + ) + + from transformers import ResNetForImageClassification + fm_models_dict_path = save_models_dict_for_init({ + 'main':ResNetForImageClassification.from_pretrained('new_impl/cv/resnet/resnet_pretrained',num_labels=scenario.num_classes,ignore_mismatched_sizes=True) + },__file__,'resnet_pretrained') + torch.cuda.set_device(1) + device = 'cuda' + #print(CvtForImageClassification.from_pretrained("/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/new_impl/cv/cvt_model",num_labels=scenario.num_classes,ignore_mismatched_sizes=True)) + fm_model = ElasticDNN_res_OfflineClsFMModel('fm', fm_models_dict_path, device) + #fm_model = CvtForImageClassification.from_pretrained("/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/new_impl/cv/cvt_model",num_labels=scenario.num_classes,ignore_mismatched_sizes=True).to(device) + models = { + 'fm':fm_model + } + import sys + fm_lora_alg = ElasticDNN_FMLoRAAlg(models, get_res_save_dir(__file__, tag=sys.argv[0])) + + from utils.dl.common.lr_scheduler import get_linear_schedule_with_warmup + fm_lora_alg.run(scenario, hyps={ + 'launch_tbboard': False, + + 'samples_size': (1, 3, 224, 224), + 'ab_r': 3,#hugging face中的模型封装得特别严实，自注意力层里面，qkv是分开的，注意这个对应的层数不要设置太高 + 'train_batch_size': 256, + 'val_batch_size': 512, + 'num_workers': 16, + 'optimizer': 'Adam', + 'optimizer_args': {'lr': 1e-3, 'betas': [0.9, 0.999]},#不同的模型，注意调调学习率啊 + 'scheduler': 'LambdaLR', + 'scheduler_args': {'lr_lambda': get_linear_schedule_with_warmup(10000, 70000)}, + 'num_iters': 8000, + 'val_freq': 100 + } +) diff --git a/new_impl/cv/resnet/cls_md_index.py b/new_impl/cv/resnet/cls_md_index.py new file mode 100644 index 0000000000000000000000000000000000000000..0477e8dfbc2ec352c344d7e597c320ed11d7a2ed --- /dev/null +++ b/new_impl/cv/resnet/cls_md_index.py @@ -0,0 +1,277 @@ +import torch +import sys +from torch import nn +from dnns.vit import make_softmax_prunable +from new_impl.cv.elasticdnn.api.model import ElasticDNN_OfflineClsFMModel, ElasticDNN_OfflineClsMDModel +# from methods.elasticdnn.api.algs.md_pretraining_w_fbs import ElasticDNN_MDPretrainingWFBSAlg +from new_impl.cv.elasticdnn.model.base import ElasticDNNUtil +from new_impl.cv.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +# from beit import FM_to_MD_beit_Util +from new_impl.cv.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +#from beit import FMLoRA_beit_Util +from resnet import ElasticresUtil +from utils.dl.common.model import LayerActivation, get_module, get_parameter +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +from utils.dl.common.loss import CrossEntropyLossSoft +import torch.nn.functional as F + + +class ElasticDNN_res_OfflineClsFMModel(ElasticDNN_OfflineClsFMModel): + def generate_md_by_reducing_width(self, reducing_width_ratio, samples: torch.Tensor): + # return FM_to_MD_beit_Util().init_md_from_fm_by_reducing_width_with_perf_test(self.models_dict['main'], + # reducing_width_ratio, samples).to(self.device) + raise NotImplementedError + + def get_feature_hook(self) -> LayerActivation: + return LayerActivation(get_module(self.models_dict['main'], 'classifier'), True, self.device) + + def get_elastic_dnn_util(self) -> ElasticDNNUtil: + return ElasticresUtil() + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + return F.cross_entropy(self.infer(x).logits, y) + + def get_lora_util(self) -> FMLoRA_Util: + #return FMLoRA_beit_Util() + raise NotImplementedError + + def get_task_head_params(self): + head = get_module(self.models_dict['main'], 'classifier') + return list(head.parameters()) + + +class ElasticDNN_res_OfflineClsMDModel(ElasticDNN_OfflineClsMDModel): + def __init__(self, name: str, models_dict_path: str, device: str): + super().__init__(name, models_dict_path, device) + + self.distill_criterion = CrossEntropyLossSoft() + + def get_feature_hook(self) -> LayerActivation: + return LayerActivation(get_module(self.models_dict['main'], 'classifier'), True, self.device) + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + return F.cross_entropy(self.infer(x).logits, y) + + def get_distill_loss(self, student_output, teacher_output): + return self.distill_criterion(student_output, teacher_output) + + def get_matched_param_of_fm(self, self_param_name, fm: nn.Module): + # only between qkv.weight, norm.weight/bias + if any([k in self_param_name for k in ['fbs', 'cls_token', 'pos_embed']]): + return None + + p = get_parameter(self.models_dict['main'], self_param_name) + if p.dim() == 0: + return None + elif p.dim() == 1 and 'layernorm' in self_param_name and 'weight' in self_param_name: + return get_parameter(fm, self_param_name) + + # 1. xx.qkv.to_qkv.yy to xx.qkv.qkv.aa and xx.qkv.abs.zz + # if 'qkv.weight' in self_param_name: + # ss = self_param_name.split('.') + + # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv' + # fm_qkv = get_module(fm, fm_qkv_name) + + # fm_abs_name = '.'.join(ss[0: -2]) + '.abs' + # fm_abs = get_module(fm, fm_abs_name) + + # return torch.cat([ + # fm_qkv.weight.data, # task-agnositc params + # torch.cat([(_abs[0].weight.T @ _abs[1].weight.T).T for _abs in fm_abs], dim=0) # task-specific params (LoRA) + # ], dim=0) + + # # elif 'to_qkv.bias' in self_param_name: + # # ss = self_param_name.split('.') + + # # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv.bias' + # # return get_parameter(fm, fm_qkv_name) + + # elif 'mlp.fc1' in self_param_name and 'weight' in self_param_name: + # fm_param_name = self_param_name.replace('.linear', '') + # return get_parameter(fm, fm_param_name) + + # elif 'mlp.fc2' in self_param_name and 'weight' in self_param_name: + # fm_param_name = self_param_name + # return get_parameter(fm, fm_param_name) + + # else: + # # return get_parameter(fm, self_param_name) + # return None + + # if 'qkv.weight' in self_param_name: + # ss = self_param_name.split('.') + + # fm_qkv_name = '.'.join(ss[0: -1]) + '.qkv' + # fm_qkv = get_module(fm, fm_qkv_name) + + # fm_abs_name = '.'.join(ss[0: -1]) + '.abs' + # fm_abs = get_module(fm, fm_abs_name) + + # return torch.cat([ + # fm_qkv.weight.data, # task-agnositc params + # torch.cat([(_abs[0].weight.T @ _abs[1].weight.T).T for _abs in fm_abs], dim=0) # task-specific params (LoRA) + # ], dim=0) + + # # elif 'to_qkv.bias' in self_param_name: + # # ss = self_param_name.split('.') + + # # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv.bias' + # # return get_parameter(fm, fm_qkv_name) + + # elif 'mlp.fc1' in self_param_name and 'weight' in self_param_name: + # fm_param_name = self_param_name.replace('.linear', '') + # return get_parameter(fm, fm_param_name) + + # elif 'mlp.fc2' in self_param_name and 'weight' in self_param_name: + # fm_param_name = self_param_name + # res = get_parameter(fm, fm_param_name) + # # print('mlp fc2 debug', fm_param_name, res is None) + # return res + + # else: + # # return get_parameter(fm, self_param_name) + # return None + + # def get_matched_param_of_fm(self, self_param_name, fm: nn.Module): + # if any([k in self_param_name for k in ['fbs', 'cls_token', 'pos_embed']]): + # return None + + # # 1. xx.qkv.to_qkv.yy to xx.qkv.qkv.aa and xx.qkv.abs.zz + # if 'to_qkv.weight' in self_param_name: + # ss = self_param_name.split('.') + + # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv' + # fm_qkv = get_module(fm, fm_qkv_name) + + # fm_abs_name = '.'.join(ss[0: -2]) + '.abs' + # fm_abs = get_module(fm, fm_abs_name) + + # return torch.cat([ + # fm_qkv.weight.data, # task-agnositc params + # torch.cat([(_abs[0].weight.T @ _abs[1].weight.T).T for _abs in fm_abs], dim=0) # task-specific params (LoRA) + # ], dim=0) + + # elif 'to_qkv.bias' in self_param_name: + # ss = self_param_name.split('.') + + # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv.bias' + # return get_parameter(fm, fm_qkv_name) + + # elif 'mlp.fc1' in self_param_name: + # fm_param_name = self_param_name.replace('.linear', '') + # return get_parameter(fm, fm_param_name) + + # else: + # return get_parameter(fm, self_param_name) + if ('attention.attention.query' in self_param_name or 'attention.attention.key' in self_param_name or \ + 'attention.attention.value' in self_param_name) and ('weight' in self_param_name): + ss = self_param_name.split('.') + + fm_qkv_name = '.'.join(ss[0: -1]) + '.fc' + fm_qkv = get_module(fm, fm_qkv_name) + + fm_abs_name = '.'.join(ss[0: -1]) + '.ab' + fm_abs = get_module(fm, fm_abs_name) + + return torch.cat([ + fm_qkv.weight.data, # task-agnositc params + fm_abs[1].weight @ fm_abs[0].weight + ], dim=0) + + elif ('attention.attention.query' in self_param_name or 'attention.attention.key' in self_param_name or \ + 'attention.attention.value' in self_param_name) and ('bias' in self_param_name): + ss = self_param_name.split('.') + + fm_qkv_name = '.'.join(ss[0: -1]) + '.fc.bias' + return get_parameter(fm, fm_qkv_name) + + elif 'intermediate.dense' in self_param_name and 'weight' in self_param_name: + fm_param_name = self_param_name.replace('.linear', '') + return get_parameter(fm, fm_param_name) + elif 'output.dense' in self_param_name and 'weight' in self_param_name: + fm_param_name = self_param_name.replace('.linear', '') + return get_parameter(fm, fm_param_name) + else: + #return get_parameter(fm, self_param_name) + return None + + +if __name__ == '__main__': + from utils.dl.common.env import set_random_seed + set_random_seed(1) + + # 1. init model + #from dnns.vit import vit_b_16 + fm_models_dict_path = 'new_impl/cv/resnet/results/cls.py/20231103/999999-100700-/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/new_impl/cv/resnet/cls.py/models/fm_best.pt' + fm_models_dict_path = save_models_dict_for_init(torch.load(fm_models_dict_path), __file__, 'fm_beit_cls_lora') + pretrained_md_models_dict_path = 'new_impl/cv/resnet/results/cls.py/20231103/999999-100700-/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/new_impl/cv/resnet/cls.py/models/fm_best.pt' + md_models_dict = torch.load(pretrained_md_models_dict_path) + md_models_dict_path = save_models_dict_for_init(md_models_dict, __file__, 'md_res_cls_pretrained_wo_fbs') + torch.cuda.set_device(1) + device = 'cuda' + + fm_model = ElasticDNN_res_OfflineClsFMModel('fm', fm_models_dict_path, device) + md_model = ElasticDNN_res_OfflineClsMDModel('md', md_models_dict_path, device) + + # 2. init alg + models = { + 'fm': fm_model, + 'md': md_model + } + from new_impl.cv.elasticdnn.api.algs.md_pretraining_index_cnn import ElasticDNN_MDPretrainingIndexAlg + + fm_to_md_alg = ElasticDNN_MDPretrainingIndexAlg(models, get_res_save_dir(__file__, sys.argv[0])) + + # 3. init scenario + scenario = build_scenario( + source_datasets_name=['GTA5Cls', 'SuperviselyPersonCls'], + target_datasets_order=['CityscapesCls', 'BaiduPersonCls'] * 15, + da_mode='close_set', + data_dirs={ + 'GTA5Cls': '/data/zql/datasets/gta5_for_cls_task', + 'SuperviselyPersonCls': '/data/zql/datasets/supervisely_person_for_cls_task', + 'CityscapesCls': '/data/zql/datasets/cityscapes_for_cls_task', + 'BaiduPersonCls': '/data/zql/datasets/baidu_person_for_cls_task' + }, + ) + + from utils.dl.common.lr_scheduler import get_linear_schedule_with_warmup + fm_to_md_alg.run(scenario, hyps={ + 'launch_tbboard': False, + + 'samples_size': (1, 3, 224, 224), + + 'FBS_r': 16, + 'FBS_ignore_layers': [], + + 'train_batch_size': 128, + 'val_batch_size': 512, + 'num_workers': 16, + 'optimizer': 'AdamW', + # 'optimizer_args': {'lr': 1e-4, 'betas': [0.9, 0.999], 'weight_decay': 0.01}, + 'indexes_optimizer_args': {'lr': 3e-3, 'betas': [0.9, 0.999], 'weight_decay': 0.1}, + # 'scheduler': 'StepLR', + # 'scheduler_args': {'step_size': 20000, 'gamma': 0.1}, + # 'optimizer': 'AdamW', + # 'optimizer_args': {'lr': 1e-4, 'betas': [0.9, 0.999], 'weight_decay': 0.01}, + 'optimizer_args': {'lr': 1e-3, 'betas': [0.9, 0.999], 'weight_decay': 0.01},#注意学习率的调整，不同的模型不一样。 + + 'scheduler': 'LambdaLR', + 'scheduler_args': {'lr_lambda': get_linear_schedule_with_warmup(10000, 70000)}, + + 'max_sparsity': 0.9, + 'min_sparsity': 0.0, + 'num_iters': 60000, + 'val_freq': 900, + 'index_loss_weight': 1e-4, + 'l1_reg_loss_weight': 1e-9, + 'val_num_sparsities': 4, + + 'bn_cal_num_iters': 800,#有bn层注意需要加上这个 + + 'index_init': 'zero', + 'index_guided_linear_comb_split_size': 512 + }) + \ No newline at end of file diff --git a/new_impl/cv/resnet/cls_md_wo_fbs.py b/new_impl/cv/resnet/cls_md_wo_fbs.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/new_impl/cv/resnet/cls_online.py b/new_impl/cv/resnet/cls_online.py new file mode 100644 index 0000000000000000000000000000000000000000..b00d028fb833377025fa521b6f0dcf6d2ebe9ba7 --- /dev/null +++ b/new_impl/cv/resnet/cls_online.py @@ -0,0 +1,347 @@ +from typing import List +from data.dataloader import build_dataloader +# from methods.elasticdnn.api.online_model import ElasticDNN_OnlineModel +from new_impl.cv.elasticdnn.api.online_model_v2 import ElasticDNN_OnlineModel + +import torch +import sys +from torch import nn +from new_impl.cv.elasticdnn.api.model import ElasticDNN_OfflineSegFMModel, ElasticDNN_OfflineSegMDModel +from new_impl.cv.elasticdnn.api.algs.md_pretraining_wo_fbs import ElasticDNN_MDPretrainingWoFBSAlg +from new_impl.cv.elasticdnn.model.base import ElasticDNNUtil +from new_impl.cv.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from new_impl.cv.elasticdnn.pipeline.offline.fm_to_md.vit import FM_to_MD_ViT_Util +from new_impl.cv.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from new_impl.cv.elasticdnn.pipeline.offline.fm_lora.vit import FMLoRA_ViT_Util +from resnet import ElasticresUtil +from utils.common.file import ensure_dir +from utils.dl.common.model import LayerActivation, get_module, get_parameter +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +from utils.dl.common.loss import CrossEntropyLossSoft +import torch.nn.functional as F +from utils.dl.common.env import create_tbwriter +import os +from utils.common.log import logger +from utils.common.data_record import write_json +# from methods.shot.shot import OnlineShotModel +from new_impl.cv.feat_align.main import OnlineFeatAlignModel, FeatAlignAlg +import tqdm +from new_impl.cv.feat_align.mmd import mmd_rbf +from new_impl.cv.utils.elasticfm_da import init_online_model, elasticfm_da +torch.cuda.set_device(1) +device = 'cuda' +app_name = 'cls' +sd_sparsity = 0.8 + +settings = { + 'involve_fm': True +} + +scenario = build_scenario( + source_datasets_name=['GTA5Cls', 'SuperviselyPersonCls'], + target_datasets_order=['CityscapesCls', 'BaiduPersonCls'] * 15, + da_mode='close_set', + data_dirs={ + 'GTA5Cls': '/data/zql/datasets/gta5_for_cls_task', + 'SuperviselyPersonCls': '/data/zql/datasets/supervisely_person_for_cls_task', + 'CityscapesCls': '/data/zql/datasets/cityscapes_for_cls_task', + 'BaiduPersonCls': '/data/zql/datasets/baiduperson_for_cls_task' + }, +) + + +class ElasticDNN_ClsOnlineModel(ElasticDNN_OnlineModel): + def get_accuracy(self, test_loader, *args, **kwargs): + acc = 0 + sample_num = 0 + + self.to_eval_mode() + + with torch.no_grad(): + pbar = tqdm.tqdm(enumerate(test_loader), total=len(test_loader), dynamic_ncols=True, leave=False) + for batch_index, (x, y) in pbar: + x, y = x.to(self.device), y.to(self.device) + output = self.infer(x) + pred = F.softmax(output, dim=1).argmax(dim=1) + correct = torch.eq(pred, y).sum().item() + acc += correct + sample_num += len(y) + + pbar.set_description(f'cur_batch_total: {len(y)}, cur_batch_correct: {correct}, ' + f'cur_batch_acc: {(correct / len(y)):.4f}') + + acc /= sample_num + return acc + + def get_elastic_dnn_util(self) -> ElasticDNNUtil: + return ElasticresUtil() + + def get_fm_matched_param_of_md_param(self, md_param_name): + # only between qkv.weight, norm.weight/bias + self_param_name = md_param_name + fm = self.models_dict['fm'] + # if any([k in self_param_name for k in ['fbs', 'cls_token', 'pos_embed']]): + # return None + + # p = get_parameter(self.models_dict['md'], self_param_name) + # if p.dim() == 0: + # return None + # elif p.dim() == 1 and 'norm' in self_param_name and 'weight' in self_param_name: + # return get_parameter(fm, self_param_name) + + if any([k in self_param_name for k in ['fbs', 'cls_token', 'pos_embed']]): + return None + + p = get_parameter(self.models_dict['md'], self_param_name) + if p.dim() == 0: + return None + elif p.dim() == 1 and 'layernorm' in self_param_name and 'weight' in self_param_name: + return get_parameter(fm, self_param_name) + # 1. xx.qkv.to_qkv.yy to xx.qkv.qkv.aa and xx.qkv.abs.zz + # if 'qkv.weight' in self_param_name: + # ss = self_param_name.split('.') + + # fm_qkv_name = '.'.join(ss[0: -1]) + '.qkv' + # fm_qkv = get_module(fm, fm_qkv_name) + + # fm_abs_name = '.'.join(ss[0: -1]) + '.abs' + # fm_abs = get_module(fm, fm_abs_name) + + # # NOTE: unrecoverable operation! multiply LoRA parameters to allow it being updated in update_fm_param() + # # TODO: if fm will be used for inference, _mul_lora_weight will not be applied! + # if not hasattr(fm_abs, '_mul_lora_weight'): + # logger.debug(f'set _mul_lora_weight in {fm_abs_name}') + # setattr(fm_abs, '_mul_lora_weight', + # nn.Parameter(torch.cat([(_abs[0].weight.T @ _abs[1].weight.T).T for _abs in fm_abs], dim=0))) + + # return torch.cat([ + # fm_qkv.weight.data, # task-agnositc params + # fm_abs._mul_lora_weight.data # task-specific params (LoRA) + # ], dim=0) + + # # elif 'to_qkv.bias' in self_param_name: + # # ss = self_param_name.split('.') + + # # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv.bias' + # # return get_parameter(fm, fm_qkv_name) + + # elif 'mlp.fc1' in self_param_name and 'weight' in self_param_name: + # fm_param_name = self_param_name.replace('.linear', '') + # return get_parameter(fm, fm_param_name) + + # elif 'mlp.fc2' in self_param_name and 'weight' in self_param_name: + # fm_param_name = self_param_name + # return get_parameter(fm, fm_param_name) + + # else: + # # return get_parameter(fm, self_param_name) + # return None + if ('attention.attention.query' in self_param_name or 'attention.attention.key' in self_param_name or \ + 'attention.attention.value' in self_param_name) and ('weight' in self_param_name): + ss = self_param_name.split('.') + + fm_qkv_name = '.'.join(ss[0: -1]) + '.fc' + fm_qkv = get_module(fm, fm_qkv_name) + + fm_abs_name = '.'.join(ss[0: -1]) + '.ab' + fm_abs = get_module(fm, fm_abs_name) + + if not hasattr(fm_abs, '_mul_lora_weight'): + logger.debug(f'set _mul_lora_weight in {fm_abs_name}') + setattr(fm_abs, '_mul_lora_weight', + nn.Parameter(fm_abs[1].weight @ fm_abs[0].weight)) + + return torch.cat([ + fm_qkv.weight.data, # task-agnositc params + fm_abs._mul_lora_weight.data # task-specific params (LoRA) + ], dim=0) + + elif ('attention.attention.query' in self_param_name or 'attention.attention.key' in self_param_name or \ + 'attention.attention.value' in self_param_name) and ('bias' in self_param_name): + ss = self_param_name.split('.') + + fm_qkv_name = '.'.join(ss[0: -1]) + '.fc.bias' + return get_parameter(fm, fm_qkv_name) + + elif 'intermediate.dense' in self_param_name and 'weight' in self_param_name: + fm_param_name = self_param_name.replace('.linear', '') + return get_parameter(fm, fm_param_name) + elif 'output.dense' in self_param_name and 'weight' in self_param_name: + fm_param_name = self_param_name.replace('.linear', '') + return get_parameter(fm, fm_param_name) + else: + #return get_parameter(fm, self_param_name) + return None + + + def update_fm_param(self, md_param_name, cal_new_fm_param_by_md_param): + if not ('attention.attention.query.weight' in md_param_name or 'attention.attention.key.weight' in md_param_name or 'attention.attention.value.weight' in md_param_name): + matched_fm_param_ref = self.get_fm_matched_param_of_md_param(md_param_name) + matched_fm_param_ref.copy_(cal_new_fm_param_by_md_param) + else: + new_fm_attn_weight, new_fm_lora_weight = torch.chunk(cal_new_fm_param_by_md_param, 2, 0) + ss = md_param_name.split('.') + fm = self.models_dict['fm'] + # update task-agnostic parameters + fm_qkv_name = '.'.join(ss[0: -1]) + '.fc' + fm_qkv = get_module(fm, fm_qkv_name) + fm_qkv.weight.data.copy_(new_fm_attn_weight) + + # update task-specific parameters + fm_abs_name = '.'.join(ss[0: -1]) + '.ab' + fm_abs = get_module(fm, fm_abs_name) + fm_abs._mul_lora_weight.data.copy_(new_fm_lora_weight) # TODO: this will not be applied in inference! + + def get_md_matched_param_of_fm_param(self, fm_param_name): + return super().get_md_matched_param_of_fm_param(fm_param_name) + + def get_md_matched_param_of_sd_param(self, sd_param_name): + # raise NotImplementedError + + # only between qkv.weight, norm.weight/bias + self_param_name = sd_param_name + md = self.models_dict['md'] + if any([k in self_param_name for k in ['fbs', 'ab', 'embeddings']]): + return None + + p = get_parameter(self.models_dict['sd'], self_param_name) + if p.dim() == 0: + return None + elif p.dim() == 1 and ('LayerNorm' in self_param_name or 'layernorm' in self_param_name) and 'weight' in self_param_name: + return get_parameter(md, self_param_name) + + # 1. xx.qkv.to_qkv.yy to xx.qkv.qkv.aa and xx.qkv.abs.zz + if ('query' in self_param_name or 'key' in self_param_name or \ + 'value' in self_param_name) and ('weight' in self_param_name): + + + return get_parameter(md, self_param_name) # NOTE: no fbs in qkv! + + # elif 'to_qkv.bias' in self_param_name: + # ss = self_param_name.split('.') + + # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv.bias' + # return get_parameter(fm, fm_qkv_name) + + elif 'intermediate.dense.0.weight' in self_param_name: + fm_param_name = '.'.join(self_param_name.split('.')[0: -2]) + '.linear.weight' + return get_parameter(md, fm_param_name) + + elif 'output.dense' in self_param_name and 'weight' in self_param_name: + fm_param_name = self_param_name + return get_parameter(md, fm_param_name) + + else: + # return get_parameter(fm, self_param_name) + return None + + def get_task_head_params(self): + head = get_module(self.models_dict['sd'], 'classifier') + return list(head.parameters()) + + + +class ClsOnlineFeatAlignModel(OnlineFeatAlignModel): + def get_trained_params(self): # TODO: elastic fm only train a part of params + #qkv_and_norm_params = [p for n, p in self.models_dict['main'].named_parameters() if 'attention.attention.projection_query' in n or 'attention.attention.projection_key' in n or 'attention.attention.projection_value' in n or 'intermediate.dense' in n or 'output.dense' in n] + qkv_and_norm_params = [p for n, p in self.models_dict['main'].named_parameters()] + return qkv_and_norm_params + + def get_feature_hook(self): + return LayerActivation(get_module(self.models_dict['main'], 'classifier'), False, self.device) + + def forward_to_get_task_loss(self, x, y): + return F.cross_entropy(self.infer(x), y) + + def get_mmd_loss(self, f1, f2): + return mmd_rbf(f1, f2) + + def infer(self, x, *args, **kwargs): + return self.models_dict['main'](x).logits + + def get_accuracy(self, test_loader, *args, **kwargs): + acc = 0 + sample_num = 0 + + self.to_eval_mode() + + with torch.no_grad(): + pbar = tqdm.tqdm(enumerate(test_loader), total=len(test_loader), dynamic_ncols=True, leave=False) + for batch_index, (x, y) in pbar: + x, y = x.to(self.device), y.to(self.device) + output = self.infer(x) + pred = F.softmax(output, dim=1).argmax(dim=1) + correct = torch.eq(pred, y).sum().item() + acc += correct + sample_num += len(y) + + pbar.set_description(f'cur_batch_total: {len(y)}, cur_batch_correct: {correct}, ' + f'cur_batch_acc: {(correct / len(y)):.4f}') + + acc /= sample_num + return acc + + + + +#from new_impl.cv.model import ElasticDNN_ClsOnlineModel +elasticfm_model = ElasticDNN_ClsOnlineModel('cls', init_online_model( + # 'experiments/elasticdnn/vit_b_16/offline/fm_to_md/results/cls_md_index.py/20230529/star_999997-154037-only_prune_mlp/models/fm_best.pt', + # 'experiments/elasticdnn/vit_b_16/offline/fm_to_md/results/cls_md_index.py/20230529/star_999997-154037-only_prune_mlp/models/md_best.pt', + #'experiments/elasticdnn/vit_b_16/offline/fm_to_md/cls/results/cls_md_index.py/20230617/999992-101343-lr1e-5_index_bug_fixed/models/fm_best.pt', + #'experiments/elasticdnn/vit_b_16/offline/fm_to_md/cls/results/cls_md_index.py/20230617/999992-101343-lr1e-5_index_bug_fixed/models/md_best.pt', + 'new_impl/cv/resnet/results/cls_md_index.py/20231105/999997-160848-/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/new_impl/cv/resnet/cls_md_index.py/models/fm_best.pt', + 'new_impl/cv/resnet/results/cls_md_index.py/20231105/999997-160848-/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/new_impl/cv/resnet/cls_md_index.py/models/md_best.pt', + 'cls', __file__ +), device, { + 'md_to_fm_alpha': 1, + 'fm_to_md_alpha': 1 +}) + +da_alg = FeatAlignAlg +#from new_impl.cv.model import ClsOnlineFeatAlignModel +da_model = ClsOnlineFeatAlignModel +da_alg_hyp = { + 'CityscapesCls': { + 'train_batch_size': 64, + 'val_batch_size': 512, + 'num_workers': 8, + 'optimizer': 'AdamW', + 'optimizer_args': {'lr': 4e-3/2, 'betas': [0.9, 0.999], 'weight_decay': 0.01},#针对于cvt的online的学习率 + #'optimizer_args': {'lr': 1e-4/2, 'momentum': 0.9}, + 'scheduler': '', + 'scheduler_args': {}, + 'num_iters': 100, + 'val_freq': 20, + 'sd_sparsity':sd_sparsity , + 'feat_align_loss_weight': 3.0 + }, + 'BaiduPersonCls': { + 'train_batch_size': 64, + 'val_batch_size': 512, + 'num_workers': 8, + 'optimizer': 'SGD', + 'optimizer_args': {'lr': 1e-3/2, 'momentum': 0.9}, + 'scheduler': '', + 'scheduler_args': {}, + 'num_iters': 100, + 'val_freq': 20, + 'sd_sparsity': sd_sparsity, + 'feat_align_loss_weight': 0.3 + } +} + + +elasticfm_da( + [app_name], + [scenario], + [elasticfm_model], + [da_alg], + [da_alg_hyp], + [da_model], + device, + settings, + __file__, + sys.argv[0] +) diff --git a/new_impl/cv/resnet/extract_submodel.py b/new_impl/cv/resnet/extract_submodel.py new file mode 100644 index 0000000000000000000000000000000000000000..2184b4b83d437992f200d88df41620689e048d1a --- /dev/null +++ b/new_impl/cv/resnet/extract_submodel.py @@ -0,0 +1,744 @@ +from abc import abstractmethod +from copy import deepcopy +import enum +import torch +from torch import nn +import os + +from .model_fbs import DomainDynamicConv2d +#from methods.utils.data import get_source_dataloader, get_source_normal_aug_dataloader, get_target_dataloaders +#from models.resnet_cifar.model_manager import ResNetCIFARManager +from utils.common.others import get_cur_time_str +from utils.dl.common.env import set_random_seed +from utils.dl.common.model import get_model_latency, get_model_size, get_module, set_module +from utils.common.log import logger +from utils.third_party.nni_new.compression.pytorch.speedup import ModelSpeedup +from utils.third_party.nni_new.compression.pytorch.utils.mask_conflict import GroupMaskConflict, ChannelMaskConflict, CatMaskPadding + + +def fix_mask_conflict(masks, model=None, dummy_input=None, traced=None, fix_group=False, fix_channel=True, fix_padding=False): + if isinstance(masks, str): + # if the input is the path of the mask_file + assert os.path.exists(masks) + masks = torch.load(masks) + assert len(masks) > 0, 'Mask tensor cannot be empty' + # if the user uses the model and dummy_input to trace the model, we + # should get the traced model handly, so that, we only trace the + # model once, GroupMaskConflict and ChannelMaskConflict will reuse + # this traced model. + if traced is None: + assert model is not None and dummy_input is not None + training = model.training + model.eval() + # We need to trace the model in eval mode + traced = torch.jit.trace(model, dummy_input) + model.train(training) + + if fix_group: + fix_group_mask = GroupMaskConflict(masks, model, dummy_input, traced) + masks = fix_group_mask.fix_mask() + if fix_channel: + fix_channel_mask = ChannelMaskConflict(masks, model, dummy_input, traced) + masks = fix_channel_mask.fix_mask() + if fix_padding: + padding_cat_mask = CatMaskPadding(masks, model, dummy_input, traced) + masks = padding_cat_mask.fix_mask() + return masks + + +class FeatureBoosting(nn.Module): + def __init__(self, w: torch.Tensor): + super(FeatureBoosting, self).__init__() + assert w.dim() == 1 + self.w = nn.Parameter(w.unsqueeze(0).unsqueeze(2).unsqueeze(3), requires_grad=False) + + def forward(self, x): + return x * self.w + + +class FBSSubModelExtractor: + def extract_submodel_via_a_sample(self, fbs_model: nn.Module, sample: torch.Tensor): + assert sample.dim() == 4 and sample.size(0) == 1 + + fbs_model.eval() + o1 = fbs_model(sample) + + pruning_info = {} + pruning_masks = {} + + for layer_name, layer in fbs_model.named_modules(): + if not isinstance(layer, DomainDynamicConv2d): + continue + + cur_pruning_mask = {'weight': torch.zeros_like(layer.raw_conv2d.weight.data)} + if layer.raw_conv2d.bias is not None: + cur_pruning_mask['bias'] = torch.zeros_like(layer.raw_conv2d.bias.data) + + w = get_module(fbs_model, layer_name).cached_w.squeeze() + unpruned_filters_index = w.nonzero(as_tuple=True)[0] + pruning_info[layer_name] = w + + cur_pruning_mask['weight'][unpruned_filters_index, ...] = 1. + if layer.raw_conv2d.bias is not None: + cur_pruning_mask['bias'][unpruned_filters_index, ...] = 1. + pruning_masks[layer_name + '.0'] = cur_pruning_mask + + no_gate_model = deepcopy(fbs_model) + for name, layer in no_gate_model.named_modules(): + if not isinstance(layer, DomainDynamicConv2d): + continue + # layer.bn.weight.data.mul_(pruning_info[name]) + set_module(no_gate_model, name, nn.Sequential(layer.raw_conv2d, layer.bn, nn.Identity())) + + # fixed_pruning_masks = fix_mask_conflict(pruning_masks, fbs_model, sample.size(), None, True, True, True) + tmp_mask_path = f'tmp_mask_{get_cur_time_str()}_{os.getpid()}.pth' + torch.save(pruning_masks, tmp_mask_path) + pruned_model = no_gate_model + pruned_model.eval() + model_speedup = ModelSpeedup(pruned_model, sample, tmp_mask_path, sample.device) + model_speedup.speedup_model() + os.remove(tmp_mask_path) + + # add feature boosting module + for layer_name, feature_boosting_w in pruning_info.items(): + feature_boosting_w = feature_boosting_w[feature_boosting_w.nonzero(as_tuple=True)[0]] + set_module(pruned_model, layer_name + '.2', FeatureBoosting(feature_boosting_w)) + + pruned_model_size = get_model_size(pruned_model, True) + pruned_model.eval() + o2 = pruned_model(sample) + diff = ((o1 - o2) ** 2).sum() + logger.info(f'pruned model size: {pruned_model_size:.3f}MB, diff: {diff}') + + return pruned_model + + @abstractmethod + def get_final_w(self, fbs_model: nn.Module, samples: torch.Tensor, layer_name: str, w: torch.Tensor): + pass + + @abstractmethod + def generate_pruning_strategy(self, fbs_model: nn.Module, samples: torch.Tensor): + pass + + def extract_submodel_via_samples(self, fbs_model: nn.Module, samples: torch.Tensor): + assert samples.dim() == 4 + fbs_model = deepcopy(fbs_model) + # fbs_model.eval() + # fbs_model(samples) + self.generate_pruning_strategy(fbs_model, samples) + + pruning_info = {} + pruning_masks = {} + + for layer_name, layer in fbs_model.named_modules(): + + if not isinstance(layer, DomainDynamicConv2d): + continue + + cur_pruning_mask = {'weight': torch.zeros_like(layer.raw_conv2d.weight.data)} + if layer.raw_conv2d.bias is not None: + cur_pruning_mask['bias'] = torch.zeros_like(layer.raw_conv2d.bias.data) + + w = get_module(fbs_model, layer_name).cached_w.squeeze() # 2-dim + w = self.get_final_w(fbs_model, samples, layer_name, w) + + unpruned_filters_index = w.nonzero(as_tuple=True)[0] + pruning_info[layer_name] = w + + cur_pruning_mask['weight'][unpruned_filters_index, ...] = 1. + if layer.raw_conv2d.bias is not None: + cur_pruning_mask['bias'][unpruned_filters_index, ...] = 1. + pruning_masks[layer_name + '.0'] = cur_pruning_mask + + no_gate_model = deepcopy(fbs_model) + for name, layer in no_gate_model.named_modules(): + if not isinstance(layer, DomainDynamicConv2d): + continue + # layer.bn.weight.data.mul_(pruning_info[name]) + set_module(no_gate_model, name, nn.Sequential(layer.raw_conv2d, layer.bn, nn.Identity())) + + # fixed_pruning_masks = fix_mask_conflict(pruning_masks, fbs_model, sample.size(), None, True, True, True) + tmp_mask_path = f'tmp_mask_{get_cur_time_str()}_{os.getpid()}.pth' + torch.save(pruning_masks, tmp_mask_path) + pruned_model = no_gate_model + pruned_model.eval() + model_speedup = ModelSpeedup(pruned_model, samples[0:1], tmp_mask_path, samples.device) + model_speedup.speedup_model() + os.remove(tmp_mask_path) + + # add feature boosting module + for layer_name, feature_boosting_w in pruning_info.items(): + feature_boosting_w = feature_boosting_w[feature_boosting_w.nonzero(as_tuple=True)[0]] + set_module(pruned_model, layer_name + '.2', FeatureBoosting(feature_boosting_w)) + + return pruned_model, pruning_info + + def extract_submodel_via_samples_and_last_submodel(self, fbs_model: nn.Module, samples: torch.Tensor, + last_submodel: nn.Module, last_pruning_info: dict): + assert samples.dim() == 4 + fbs_model = deepcopy(fbs_model) + # fbs_model.eval() + # fbs_model(samples) + self.generate_pruning_strategy(fbs_model, samples) + + pruning_info = {} + pruning_masks = {} + # some tricks + incrementally_updated_layers = [] + + for layer_name, layer in fbs_model.named_modules(): + if not isinstance(layer, DomainDynamicConv2d): + continue + + cur_pruning_mask = {'weight': torch.zeros_like(layer.raw_conv2d.weight.data)} + if layer.raw_conv2d.bias is not None: + cur_pruning_mask['bias'] = torch.zeros_like(layer.raw_conv2d.bias.data) + + w = get_module(fbs_model, layer_name).cached_w.squeeze() # 2-dim + w = self.get_final_w(fbs_model, samples, layer_name, w) + + unpruned_filters_index = w.nonzero(as_tuple=True)[0] + pruning_info[layer_name] = w + + cur_pruning_mask['weight'][unpruned_filters_index, ...] = 1. + if layer.raw_conv2d.bias is not None: + cur_pruning_mask['bias'][unpruned_filters_index, ...] = 1. + pruning_masks[layer_name + '.0'] = cur_pruning_mask + + # some tricks + if last_pruning_info is not None: + last_w = last_pruning_info[layer_name] + intersection_ratio = ((w > 0) * (last_w > 0)).sum() / (last_w > 0).sum() + if intersection_ratio > 0.: + incrementally_updated_layers += [layer_name] # that is, only similar layers are transferable + + no_gate_model = deepcopy(fbs_model) + for name, layer in no_gate_model.named_modules(): + if not isinstance(layer, DomainDynamicConv2d): + continue + # layer.bn.weight.data.mul_(pruning_info[name]) + set_module(no_gate_model, name, nn.Sequential(layer.raw_conv2d, layer.bn, nn.Identity())) + + # fixed_pruning_masks = fix_mask_conflict(pruning_masks, fbs_model, sample.size(), None, True, True, True) + tmp_mask_path = f'tmp_mask_{get_cur_time_str()}_{os.getpid()}.pth' + + torch.save(pruning_masks, tmp_mask_path) + pruned_model = no_gate_model + pruned_model.eval() + model_speedup = ModelSpeedup(pruned_model, samples[0:1], tmp_mask_path, samples.device) + model_speedup.speedup_model() + os.remove(tmp_mask_path) + + # add feature boosting module + for layer_name, feature_boosting_w in pruning_info.items(): + feature_boosting_w = feature_boosting_w[feature_boosting_w.nonzero(as_tuple=True)[0]] + set_module(pruned_model, layer_name + '.2', FeatureBoosting(feature_boosting_w)) + + # some tricks + # incrementally updating (borrow some weights from last_pruned_model) + for layer_name in incrementally_updated_layers: + cur_filter_i, last_filter_i = 0, 0 + for i, (w_factor, last_w_factor) in enumerate(zip(pruning_info[layer_name], last_pruning_info[layer_name])): + if w_factor > 0 and last_w_factor > 0: # the filter is shared + cur_conv2d, last_conv2d = get_module(pruned_model, layer_name + '.0'), get_module(last_submodel, layer_name + '.0') + cur_conv2d.weight.data[cur_filter_i] = last_conv2d.weight.data[last_filter_i] + + cur_bn, last_bn = get_module(pruned_model, layer_name + '.1'), get_module(last_submodel, layer_name + '.1') + cur_bn.weight.data[cur_filter_i] = last_bn.weight.data[last_filter_i] + cur_bn.bias.data[cur_filter_i] = last_bn.bias.data[last_filter_i] + cur_bn.running_mean.data[cur_filter_i] = last_bn.running_mean.data[last_filter_i] + cur_bn.running_var.data[cur_filter_i] = last_bn.running_var.data[last_filter_i] + + cur_fw, last_fw = get_module(pruned_model, layer_name + '.2'), get_module(last_submodel, layer_name + '.2') + cur_fw.w.data[0, cur_filter_i] = last_fw.w.data[0, last_filter_i] + + if w_factor > 0: + cur_filter_i += 1 + if last_w_factor > 0: + last_filter_i += 1 + + return pruned_model, pruning_info + + def absorb_sub_model(self, fbs_model: nn.Module, sub_model: nn.Module, pruning_info: dict, alpha=1.): + if alpha == 0.: + return + for layer_name, feature_boosting_w in pruning_info.items(): + unpruned_filters_index = feature_boosting_w.nonzero(as_tuple=True)[0] + + fbs_layer = get_module(fbs_model, layer_name) + sub_model_layer = get_module(sub_model, layer_name) + + for fi_in_sub_layer, fi_in_fbs_layer in enumerate(unpruned_filters_index): + fbs_layer.raw_conv2d.weight.data[fi_in_fbs_layer] = (1. - alpha) * fbs_layer.raw_conv2d.weight.data[fi_in_fbs_layer] + \ + alpha * sub_model_layer[0].weight.data[fi_in_sub_layer] + for k in ['weight', 'bias', 'running_mean', 'running_var']: + getattr(fbs_layer.bn, k).data[fi_in_fbs_layer] = (1. - alpha) * getattr(fbs_layer.bn, k).data[fi_in_fbs_layer] + \ + alpha * getattr(sub_model_layer[1], k).data[fi_in_sub_layer] + + +class DAFBSSubModelExtractor(FBSSubModelExtractor): + def __init__(self) -> None: + super().__init__() + # self.debug_sample_i = 0 + # self.last_final_ws = None + + @abstractmethod + def generate_pruning_strategy(self, fbs_model: nn.Module, samples: torch.Tensor): + with torch.no_grad(): + fbs_model.eval() + self.cur_output = fbs_model(samples) + + @abstractmethod + def get_final_w(self, fbs_model: nn.Module, samples: torch.Tensor, layer_name: str, w: torch.Tensor): + # import matplotlib.pyplot as plt + # plt.imshow(w.cpu().numpy(), cmap='Greys') + # # plt.colorbar() + # plt.xlabel('Filters') + # plt.ylabel('Samples') + # plt.tight_layout() + # plt.savefig(os.path.join(res_save_dir, f'{layer_name}.png'), dpi=300) + # plt.clf() + # w_sum = w.sum(0) + # w_argsort = w_sum.argsort(descending=True) + # return w[self.debug_sample_i] + # x = self.cur_output + # each_sample_entropy = -(x.softmax(1) * x.log_softmax(1)).sum(1) + + # hardest_sample_index = w.sum(1).argmax() + # return w[hardest_sample_index] + # [0.0828, 0.1017, 0.0575, 0.3081, 0.1511, 0.3634, 0.3388, 0.3942, 0.2475, 0.3371, 0.5837, 0.145, 0.4428, 0.2159, 0.4028] 0.27815999999999996 + + x = self.cur_output + each_sample_entropy = -(x.logits.softmax(1) * x.logits.log_softmax(1)).sum(1) + hardest_sample_index = each_sample_entropy.argmax() + res = w[hardest_sample_index] + return res + + # if self.last_final_ws is not None: + # intersection_ratio = (self.last_final_w == res).sum() / (res > 0).sum() + # print('intersection ratio: ', intersection_ratio) + + # self.last_final_ws[layer_name] = res + + + + # indices = (-w).sum(0).topk((w[0] == 0).sum())[1] + # boosting = w.max(0)[0] + # boosting[indices] = 0. + # return boosting + + # return w[0] + + +def tent_as_detector(model, x, num_iters=1, lr=1e-4, l1_wd=0., strategy='ours'): + model = deepcopy(model) + before_model = deepcopy(model) + + from methods.tent import tent + + optimizer = torch.optim.SGD( + model.parameters(), lr=lr, weight_decay=l1_wd) + from models.resnet_cifar.model_manager import ResNetCIFARManager + tented_model = tent.Tent(model, optimizer, ResNetCIFARManager, steps=num_iters) + + tent.configure_model(model) + tented_model(x) + + filters_sen_info = {} + + last_conv_name = None + for (name, m1), m2 in zip(model.named_modules(), before_model.modules()): + if isinstance(m1, nn.Conv2d): + last_conv_name = name + + if not isinstance(m1, nn.BatchNorm2d): + continue + + with torch.no_grad(): + features_weight_diff = ((m1.weight.data - m2.weight.data).abs()) + features_bias_diff = ((m1.bias.data - m2.bias.data).abs()) + + features_diff = features_weight_diff + features_bias_diff + + features_diff_order = features_diff.argsort(descending=False) + + if strategy == 'ours': + untrained_filters_index = features_diff_order[: int(len(features_diff) * 0.8)] + elif strategy == 'random': + untrained_filters_index = torch.randperm(len(features_diff))[: int(len(features_diff) * 0.8)] + elif strategy == 'inversed_ours': + untrained_filters_index = features_diff_order.flip(0)[: int(len(features_diff) * 0.8)] + elif strategy == 'none': + untrained_filters_index = None + + filters_sen_info[name] = dict(untrained_filters_index=untrained_filters_index, conv_name=last_conv_name) + + return filters_sen_info + + +class SGDF(torch.optim.SGD): + + @torch.no_grad() + def step(self, model, conv_filters_sen_info, filters_sen_info, closure=None): + """Performs a single optimization step. + + Arguments: + closure (callable, optional): A closure that reevaluates the model + and returns the loss. + """ + loss = None + if closure is not None: + with torch.enable_grad(): + loss = closure() + + for group in self.param_groups: + weight_decay = group['weight_decay'] + momentum = group['momentum'] + dampening = group['dampening'] + nesterov = group['nesterov'] + + # assert len([i for i in model.named_parameters()]) == len([j for j in group['params']]) + + for (name, _), p in zip(model.named_parameters(), group['params']): + if p.grad is None: + continue + + layer_name = '.'.join(name.split('.')[0:-1]) + if layer_name in filters_sen_info.keys(): + untrained_filters_index = filters_sen_info[layer_name]['untrained_filters_index'] + elif layer_name in conv_filters_sen_info.keys(): + untrained_filters_index = conv_filters_sen_info[layer_name]['untrained_filters_index'] + else: + untrained_filters_index = [] + + d_p = p.grad + if weight_decay != 0: + d_p = d_p.add(p, alpha=weight_decay) + if momentum != 0: + param_state = self.state[p] + if 'momentum_buffer' not in param_state: + buf = param_state['momentum_buffer'] = torch.clone(d_p).detach() + else: + buf = param_state['momentum_buffer'] + buf.mul_(momentum).add_(d_p, alpha=1 - dampening) + if nesterov: + d_p = d_p.add(buf, alpha=momentum) + else: + d_p = buf + + d_p[untrained_filters_index] = 0. + p.add_(d_p, alpha=-group['lr']) + + return loss + + +if __name__ == '__main__': + set_random_seed(0) + + import sys + tag = sys.argv[1] + # alpha = 0.4 + alpha = 0.2 + # alpha = float(sys.argv[1]) + + fbs_model_path = sys.argv[1] + + cur_time_str = get_cur_time_str() + res_save_dir = f'logs/experiments_trial/CIFAR100C/ours_fbs_more_challenging/{cur_time_str[0:8]}/{cur_time_str[8:]}-{tag}' + os.makedirs(res_save_dir) + + import shutil + shutil.copytree(os.path.dirname(__file__), + os.path.join(res_save_dir, 'method'), ignore=shutil.ignore_patterns('*.pt', '*.pth', 'log', '__pycache__')) + logger.info(f'res save dir: {res_save_dir}') + + # model = torch.load('logs/experiments_trial/CIFAR100C/ours_dynamic_filters/20220801/152138-0.6_l1wd=1e-8/best_model_0.80.pt') + # model = torch.load('logs/experiments_trial/CIFAR100C/ours_dynamic_filters/20220801/232913-sample_subnetwork/best_model_0.80.pt') + model = torch.load(fbs_model_path) + + # model = torch.load('logs/experiments_trial/CIFAR100C/ours_dynamic_filters/20220729/002444-0.4/best_model_0.40.pt') + + # import sys + # sys.path.append('/data/xgf/legodnn_and_domain_adaptation') + xgf_model = torch.load('logs/experiments_trial/CIFAR100C/ours_dynamic_filters/20220731/224212-cifar10_svhn_raw/last_model.pt') + # xgf_model = torch.load('/data/xgf/legodnn_and_domain_adaptation/results_scaling_da/image_classification/CIFAR100C_resnet18/onda/offline_l1/s4/20220607/204211/last_model.pt') + # test_dataloader = get_source_dataloader('CIFAR100', 256, 4, 'test', False, False, False) + # test_dataloader = get_target_dataloaders('CIFAR100C', [7], 128, 4, 'test', False, False, False)[0] # snow, xgf 0.3914 + # test_dataloaders = get_target_dataloaders('CIFAR100C', list(range(15)), 128, 4, 'test', False, False, False) # defocus_blur, xgf 0.2836 + # test_dataloaders = get_target_dataloaders('RotatedCIFAR100', list(range(18)), 128, 4, 'test', False, False, False) + train_dataloaders = [ + get_source_dataloader(dataset_name, 128, 4, 'train', True, None, True) for dataset_name in ['SVHN', 'CIFAR10', 'SVHN'] + ][::-1] * 10 + test_dataloaders = [ + get_source_dataloader('USPS', 128, 4, 'test', False, False, False), + get_source_dataloader('STL10-wo-monkey', 128, 4, 'test', False, False, False), + get_source_dataloader('MNIST', 128, 4, 'test', False, False, False), + ][::-1] * 10 + y_offsets = [10, 0, 10][::-1] * 10 + domain_names = ['USPS', 'STL10', 'MNIST'][::-1] * 10 + # train_dataloader = get_source_dataloader('CIFAR100', 128, 4, 'train', True, None, True) + # acc = ResNetCIFARManager.get_accuracy(model, test_dataloader, 'cuda') + # print(acc) + # baseline_accs = [0.1012, 0.1156, 0.0529, 0.2836, 0.1731, 0.3765, 0.3445, 0.3914, 0.2672, 0.3289, 0.5991, 0.1486, 0.4519, 0.1907, 0.3929] + # accs = [] + + baseline_before, baseline_after, ours_before, ours_after = [], [], [], [] + last_pruned_model, last_pruning_info = None, None + # y_offset = 0 + for ti, (test_dataloader, y_offset) in enumerate(zip(test_dataloaders, y_offsets)): + samples, labels = next(iter(test_dataloader)) + samples, labels = samples.cuda(), labels.cuda() + labels += y_offset + + def bn_cal(_model: nn.Module): + for n, m in _model.named_modules(): + if isinstance(m, nn.BatchNorm2d): + m.reset_running_stats() + m.training = True + m.train() + for _ in range(100): # ~one epoch + x, y = next(train_dataloaders[ti]) + x = x.cuda() + _model(samples) + + def shot(_model: nn.Module, lr=6e-4, num_iters_scale=1, wd=0.): + # print([n for n, p in model.named_parameters()]) + _model.requires_grad_(True) + _model.linear.requires_grad_(False) + import torch.optim + optimizer = torch.optim.SGD([p for p in _model.parameters() if p.requires_grad], lr=lr, momentum=0.9, weight_decay=wd) + device = 'cuda' + + for _ in range(100 * num_iters_scale): + x = samples + _model.train() + output = ResNetCIFARManager.forward(_model, x) + + def Entropy(input_): + entropy = -input_ * torch.log(input_ + 1e-5) + entropy = torch.sum(entropy, dim=1) + return entropy + + softmax_out = nn.Softmax(dim=1)(output) + entropy_loss = torch.mean(Entropy(softmax_out)) + msoftmax = softmax_out.mean(dim=0) + entropy_loss -= torch.sum(-msoftmax * torch.log(msoftmax + 1e-5)) + loss = entropy_loss + + optimizer.zero_grad() + loss.backward() + optimizer.step() + + def shot_w_part_filter(_model: nn.Module, lr=6e-4, num_iters_scale=1, wd=0.): + # print([n for n, p in model.named_parameters()]) + _model.requires_grad_(True) + _model.linear.requires_grad_(False) + import torch.optim + optimizer = SGDF([p for p in _model.parameters() if p.requires_grad], lr=lr, momentum=0.9, weight_decay=wd) + device = 'cuda' + + filters_sen_info = tent_as_detector(_model, samples, strategy='ours') + conv_filters_sen_info = {v['conv_name']: v for _, v in filters_sen_info.items()} + + for _ in range(100 * num_iters_scale): + x = samples + _model.train() + output = ResNetCIFARManager.forward(_model, x) + + def Entropy(input_): + entropy = -input_ * torch.log(input_ + 1e-5) + entropy = torch.sum(entropy, dim=1) + return entropy + + softmax_out = nn.Softmax(dim=1)(output) + entropy_loss = torch.mean(Entropy(softmax_out)) + msoftmax = softmax_out.mean(dim=0) + entropy_loss -= torch.sum(-msoftmax * torch.log(msoftmax + 1e-5)) + loss = entropy_loss + + optimizer.zero_grad() + loss.backward() + optimizer.step(_model, conv_filters_sen_info, filters_sen_info) + + def tent(_model: nn.Module): + from methods.tent import tent + _model = tent.configure_model(_model) + params, param_names = tent.collect_params(_model) + optimizer = torch.optim.Adam(params, lr=1e-4) + tent_model = tent.Tent(_model, optimizer, ResNetCIFARManager, steps=1) + + tent.configure_model(_model) + tent_model(samples) + + def tent_configure_bn(_model): + """Configure model for use with tent.""" + # train mode, because tent optimizes the model to minimize entropy + # _model.train() + # # disable grad, to (re-)enable only what tent updates + # _model.requires_grad_(False) + # configure norm for tent updates: enable grad + force batch statisics + for m in _model.modules(): + if isinstance(m, nn.BatchNorm2d): + m.requires_grad_(True) + # force use of batch stats in train and eval modes + m.track_running_stats = False + m.running_mean = None + m.running_var = None + + # m.track_running_stats = True + # m.momentum = 1.0 + + # # FIXME + # from methods.ours_dynamic_filters.extract_submodel import FeatureBoosting + # # if isinstance(m, FeatureBoosting): + # if m.__class__.__name__ == 'FeatureBoosting': + # m.requires_grad_(True) + + return model + + def sl(_model: nn.Module, lr=6e-4, num_iters_scale=1, wd=0.): + _model.requires_grad_(True) + _model.linear.requires_grad_(False) + import torch.optim + optimizer = torch.optim.SGD([p for p in _model.parameters() if p.requires_grad], lr=lr, momentum=0.9, weight_decay=wd) + device = 'cuda' + + for _ in range(100 * num_iters_scale): + x = samples + _model.train() + loss = ResNetCIFARManager.forward_to_gen_loss(_model, x, labels) + + optimizer.zero_grad() + loss.backward() + optimizer.step() + + model_extractor = DAFBSSubModelExtractor() + model1 = model_extractor.extract_submodel_via_a_sample(model,samples[0]) + pruned_model, pruning_info = model_extractor.extract_submodel_via_samples_and_last_submodel(model, samples, None, None) + # print(pruned_model) + # print(get_model_size(pruned_model, True)) + # bn_cal(pruned_model) + acc = ResNetCIFARManager.get_accuracy(pruned_model, test_dataloader, 'cuda', y_offset) + print(acc) + ours_before += [acc] + # tent(pruned_model) + # bn_cal(pruned_model) + shot_w_part_filter(pruned_model, 6e-4, 1, 1e-3) + # sl(pruned_model) + acc = ResNetCIFARManager.get_accuracy(pruned_model, test_dataloader, 'cuda', y_offset) + print(acc) + ours_after += [acc] + + last_pruned_model, last_pruning_info = deepcopy(pruned_model), deepcopy(pruning_info) + model_extractor.absorb_sub_model(model, pruned_model, pruning_info, alpha) + + # xgf_model = torch.load('/data/xgf/legodnn_and_domain_adaptation/results_scaling_da/image_classification/CIFAR100C_resnet18/onda/offline_l1/s8/20220607/212448/last_model.pt') + # xgf_model = torch.load('/data/xgf/legodnn_and_domain_adaptation/results_scaling_da/image_classification/CIFAR100C_resnet18/onda/offline_l1/s4/20220607/204211/last_model.pt') + + # print(xgf_model) + # acc = ResNetCIFARManager.get_accuracy(xgf_model, test_dataloader, 'cuda', y_offset) + # print(acc) + # baseline_before += [acc] + # # tent(xgf_model) + # shot(xgf_model) + # # sl(xgf_model) + # acc = ResNetCIFARManager.get_accuracy(xgf_model, test_dataloader, 'cuda', y_offset) + # print(acc) + # baseline_after += [acc] + # print() + # diff = acc - baseline_accs[ti] + # print(f'domain {ti}, model size {get_model_size(pruned_model, True):.3f}MB, diff: {diff:.4f}') + # print(accs, sum(accs) / len(accs)) + + import matplotlib.pyplot as plt + from visualize.util import * + set_figure_settings(3) + + def avg(arr): + return sum(arr) / len(arr) + + # plt.plot(list(range(len(test_dataloaders))), baseline_before, lw=2, linestyle='--', color=BLUE, label=f'L1 before DA ({avg(baseline_before):.4f})') + # plt.plot(list(range(len(test_dataloaders))), baseline_after, lw=2, linestyle='-', color=BLUE, label=f'L1 after DA ({avg(baseline_after):.4f})') + plt.plot(list(range(len(test_dataloaders))), ours_before, lw=2, linestyle='--', color=RED, label=f'ours before DA ({avg(ours_before):.4f})') + plt.plot(list(range(len(test_dataloaders))), ours_after, lw=2, linestyle='-', color=RED, label=f'ours after DA ({avg(ours_after):.4f})') + plt.xlabel('domains') + plt.ylabel('accuracy') + plt.xticks(list(range(len(domain_names))), domain_names, rotation=90) + plt.legend(loc=2, bbox_to_anchor=(1.05, 1.0), fontsize=16) + plt.tight_layout() + plt.savefig(os.path.join(res_save_dir, 'main.png'), dpi=300) + plt.clf() + + torch.save((baseline_before, baseline_after, ours_before, ours_after), os.path.join(res_save_dir, 'main.png.data')) + + # with open('./tmp.csv', 'a') as f: + # f.write(f'{alpha:.2f},{avg(baseline_after):.4f},{avg(ours_after):.4f}') + + # std: logs/experiments_trial/CIFAR100C/ours_dynamic_filters/20220730/161404-submodel/main.png + + + # accs = [] + # for i in tqdm.tqdm(range(100)): + # model_extractor.debug_sample_i = i + # pruned_model = model_extractor.extract_submodel_via_samples(model, samples) + # acc = ResNetCIFARManager.get_accuracy(pruned_model, test_dataloader, 'cuda') + # accs += [acc] + + # import matplotlib.pyplot as plt + # plt.plot(list(range(100)), accs) + # plt.savefig('./tmp.png', dpi=300) + # plt.clf() + + + # ------------------------------ + # perf test + + # sample, _ = next(iter(test_dataloader)) + # sample = sample[0: 1].cuda() + + # pruned_model = FBSSubModelExtractor().extract_submodel_via_a_sample(model, sample) + + # bs = 1 + # def perf_test(model, batch_size, device): + # model = model.to(device) + # optimizer = torch.optim.SGD(model.parameters(), lr=1e-3) + + # # warmup + # for _ in range(100): + # rand_input = torch.rand((batch_size, 3, 32, 32)).to(device) + # o = model(rand_input) + + # forward_latency = 0. + # backward_latency = 0. + + # for _ in range(100): + # rand_input = torch.rand((batch_size, 3, 32, 32)).to(device) + # optimizer.zero_grad() + + # s, e = torch.cuda.Event(enable_timing=True), torch.cuda.Event(enable_timing=True) + # s.record() + # o = model(rand_input) + # e.record() + # torch.cuda.synchronize() + # forward_latency += s.elapsed_time(e) / 1000. + + # loss = ((o - 1) ** 2).sum() + + # s, e = torch.cuda.Event(enable_timing=True), torch.cuda.Event(enable_timing=True) + # s.record() + # loss.backward() + # optimizer.step() + # e.record() + # torch.cuda.synchronize() + # backward_latency += s.elapsed_time(e) / 1000. + + # forward_latency /= 100 + # backward_latency /= 100 + + # print(forward_latency, backward_latency) + + # for bs in [1, 128]: + # for device in ['cuda', 'cpu']: + # for m in [model, pruned_model]: + # print(bs, device) + # perf_test(m, bs, device) \ No newline at end of file diff --git a/new_impl/cv/resnet/model_fbs.py b/new_impl/cv/resnet/model_fbs.py new file mode 100644 index 0000000000000000000000000000000000000000..3e6e4fe00e2e97734db9967a4bb82ce608ffacb7 --- /dev/null +++ b/new_impl/cv/resnet/model_fbs.py @@ -0,0 +1,424 @@ +from turtle import forward +from typing import Optional +import torch +import copy +from torch import nn +#from methods.utils.data import get_source_dataloader +from utils.dl.common.model import get_model_device, get_model_latency, get_model_size, get_module, get_super_module, set_module +from utils.common.log import logger + + +""" +No real speedup. +But it's ok because our big model just forward for one time to find the best sub-model. +The sub-model doesn't contain filter selection modules. It's just a normal model. +""" + +class KTakesAll(nn.Module): + def __init__(self, k): + super(KTakesAll, self).__init__() + + self.k = k + + def forward(self, g: torch.Tensor): + # if self.k == 0.: + # t = g + # t = t / torch.sum(t, dim=1).unsqueeze(1) * t.size(1) + # return t.unsqueeze(2).unsqueeze(3) + # t = g + # t = t / torch.sum(t, dim=1).unsqueeze(1) * t.size(1) + # # print('000', t.size()) + # t = t.unsqueeze(2).unsqueeze(3).mean((0, 2, 3)).unsqueeze(0).unsqueeze(2).unsqueeze(3) + # # print('111', t.size()) + # # print(t) + # return t + # # assert x.dim() == 2 + # print(g) + k = int(g.size(1) * self.k) + + i = (-g).topk(k, 1)[1] + t = g.scatter(1, i, 0) + # t = t / torch.sum(t, dim=1).unsqueeze(1) * t.size(1) + # print(t) + + return t.unsqueeze(2).unsqueeze(3) + # g = g.mean(0).unsqueeze(0) + + # k = int(g.size(1) * self.k) + + # i = (-g).topk(k, 1)[1] + # t = g.scatter(1, i, 0) + # t = t / torch.sum(t, dim=1).unsqueeze(1) * t.size(1) + + # return t.unsqueeze(2).unsqueeze(3) + +# class NoiseAdd(nn.Module): +# def __init__(self): +# super(NoiseAdd, self).__init__() + +# self.training = True + +# def forward(self, x): +# if self.training: +# return x + torch.randn_like(x, device=x.device) +# else: +# return x + +class Abs(nn.Module): + def __init__(self): + super(Abs, self).__init__() + + def forward(self, x): + return x.abs() + + +class DomainDynamicConv2d(nn.Module): + def __init__(self, raw_conv2d: nn.Conv2d, raw_bn: nn.BatchNorm2d, k: float, bn_after_fc=False): + super(DomainDynamicConv2d, self).__init__() + + assert not bn_after_fc + + self.filter_selection_module = nn.Sequential( + Abs(), + nn.AdaptiveAvgPool2d(1), + nn.Flatten(), + nn.Linear(raw_conv2d.in_channels, raw_conv2d.out_channels), + # nn.Conv2d(raw_conv2d.in_channels, raw_conv2d.out_channels // 16, kernel_size=1, bias=False), + + # nn.Linear(raw_conv2d.in_channels, raw_conv2d.out_channels // 16), + # nn.BatchNorm1d(raw_conv2d.out_channels // 16) if bn_after_fc else nn.Identity(), + # nn.ReLU(), + # nn.Linear(raw_conv2d.out_channels // 16, raw_conv2d.out_channels), + + # nn.BatchNorm1d(raw_conv2d.out_channels), + nn.ReLU(), + # NoiseAdd(), + # nn.Sigmoid() + # L1RegTrack(), + # KTakesAll(k) + ) + self.k_takes_all = KTakesAll(k) + + self.raw_conv2d = raw_conv2d + self.bn = raw_bn # remember clear the original BNs in the network + + nn.init.constant_(self.filter_selection_module[3].bias, 1.) + nn.init.kaiming_normal_(self.filter_selection_module[3].weight) + + self.cached_raw_w = None + self.l1_reg_of_raw_w = None + self.cached_w = None + self.static_w = None + self.pruning_ratios = None + + + def forward(self, x): + raw_x = self.bn(self.raw_conv2d(x)) + + # if self.k_takes_all.k < 1e-7: + # return raw_x + + if self.static_w is None: + raw_w = self.filter_selection_module(x) + + self.cached_raw_w = raw_w + # self.l1_reg_of_raw_w = raw_w.norm(1, dim=1).mean() + self.l1_reg_of_raw_w = raw_w.norm(1) + + w = self.k_takes_all(raw_w) + + # w = w.unsqueeze(2).unsqueeze(3) + + # if self.training: + # soft_w = torch.max(torch.zeros_like(raw_w), torch.min(torch.ones_like(raw_w), + # 1.2 * (torch.sigmoid(raw_w + torch.randn_like(raw_w))) - 0.1)) + # else: + # soft_w = torch.max(torch.zeros_like(raw_w), torch.min(torch.ones_like(raw_w), + # 1.2 * (torch.sigmoid(raw_w)) - 0.1)) + + # w = soft_w.detach().clone() + # w[w < 0.5] = 0. + # w[w >= 0.5] = 1. + # w = w + soft_w - soft_w.detach() + + # w = w.unsqueeze(2).unsqueeze(3) + # soft_w = soft_w.unsqueeze(2).unsqueeze(3) + # self.l1_reg_of_raw_w = soft_w.norm(1) + + self.cached_w = w + + # print(w.size(), x.size(), raw_x.size()) + else: + w = self.static_w.unsqueeze(0).unsqueeze(2).unsqueeze(3) + + if self.pruning_ratios is not None: + # self.pruning_ratios += [1. - float((w_of_a_asample > 0.).sum() / w_of_a_asample.numel()) for w_of_a_asample in w] + self.pruning_ratios += [torch.sum(w > 0.) / w.numel()] + + return raw_x * w + + # def to_static(self): + # global_w = self.cached_raw_w.detach().topk(0.25, 1)[0].mean(0).unsqueeze(0) + # global_w = self.k_takes_all(global_w).squeeze(0) + # self.static_w = global_w + + # def to_dynamic(self): + # self.static_w = None + + +def boost_raw_model_with_filter_selection(model: nn.Module, init_k: float, bn_after_fc=False, ignore_layers=None, perf_test=True, model_input_size: Optional[tuple]=None): + model = copy.deepcopy(model) + + device = get_model_device(model) + if perf_test: + before_model_size = get_model_size(model, True) + before_model_latency = get_model_latency( + model, model_input_size, 50, device, 50) + + # clear original BNs + num_original_bns = 0 + last_conv_name = None + conv_bn_map = {} + for name, module in model.named_modules(): + if isinstance(module, nn.Conv2d): + last_conv_name = name + if isinstance(module, nn.BatchNorm2d) and (ignore_layers is not None and last_conv_name not in ignore_layers): + # set_module(model, name, nn.Identity()) + num_original_bns += 1 + conv_bn_map[last_conv_name] = name + + num_conv = 0 + for name, module in model.named_modules(): + if isinstance(module, nn.Conv2d) and (ignore_layers is not None and name not in ignore_layers): + set_module(model, name, DomainDynamicConv2d(module, get_module(model, conv_bn_map[name]), init_k, bn_after_fc)) + num_conv += 1 + + assert num_conv == num_original_bns + + for bn_layer in conv_bn_map.values(): + set_module(model, bn_layer, nn.Identity()) + + if perf_test: + after_model_size = get_model_size(model, True) + after_model_latency = get_model_latency( + model, model_input_size, 50, device, 50) + + logger.info(f'raw model -> raw model w/ filter selection:\n' + f'model size: {before_model_size:.3f}MB -> {after_model_size:.3f}MB ' + f'latency: {before_model_latency:.6f}s -> {after_model_latency:.6f}s') + + return model, conv_bn_map + + +def get_l1_reg_in_model(boosted_model): + res = 0. + for name, module in boosted_model.named_modules(): + if isinstance(module, DomainDynamicConv2d): + res += module.l1_reg_of_raw_w + return res + + +def get_cached_w(model): + res = [] + for name, module in model.named_modules(): + if isinstance(module, DomainDynamicConv2d): + res += [module.cached_w] + return torch.cat(res, dim=1) + + +def set_pruning_rate(model, k): + for name, module in model.named_modules(): + if isinstance(module, KTakesAll): + module.k = k + + +def get_cached_raw_w(model): + res = [] + for name, module in model.named_modules(): + if isinstance(module, DomainDynamicConv2d): + res += [module.cached_raw_w] + return torch.cat(res, dim=1) + + +def start_accmu_flops(model): + for name, module in model.named_modules(): + if isinstance(module, DomainDynamicConv2d): + module.pruning_ratios = [] + + +def get_accmu_flops(model): + layer_res = {} + total_res = [] + + for name, module in model.named_modules(): + if isinstance(module, DomainDynamicConv2d): + layer_res[name] = module.pruning_ratios + total_res += module.pruning_ratios + module.pruning_ratios = None + + avg_pruning_ratio = sum(total_res) / len(total_res) + return layer_res, total_res, avg_pruning_ratio + + +def convert_boosted_model_to_static(boosted_model, a_few_data): + boosted_model(a_few_data) + + for name, module in boosted_model.named_modules(): + if isinstance(module, DomainDynamicConv2d): + module.to_static() + # TODO: use fn3 techniques + + +def ensure_boosted_model_to_dynamic(boosted_model): + for name, module in boosted_model.named_modules(): + if isinstance(module, DomainDynamicConv2d): + module.to_dynamic() + + +def train_only_gate(model): + gate_params = [] + for n, p in model.named_parameters(): + if 'filter_selection_module' in n: + gate_params += [p] + else: + p.requires_grad = False + return gate_params + +if __name__ == '__main__': + # rand_input = torch.rand((256, 3, 32, 32)) + # conv = nn.Conv2d(3, 64, 3, 1, 1, bias=False) + # new_conv = DomainDynamicConv2d(conv, 0.1) + + # train_dataloader = get_source_dataloader('CIFAR100', 256, 4, 'train', True, None, True) + # rand_input, _ = next(train_dataloader) + + # start_accmu_flops(new_conv) + + # new_conv(rand_input) + + # _, total_pruning_ratio, avg_pruning_ratio = get_accmu_flops(new_conv) + + # import matplotlib.pyplot as plt + # plt.hist(total_pruning_ratio) + # plt.savefig('./tmp.png') + # plt.clf() + + # print(avg_pruning_ratio) + + + + # with torch.no_grad(): + # conv(rand_input) + # new_conv(rand_input) + + # from torchvision.models import resnet18 + + # model = resnet18() + # boost_raw_model_with_filter_selection(model, 0.5, True, (1, 3, 224, 224)) + + # rand_input = torch.rand((2, 3, 32, 32)) + # conv = nn.Conv2d(3, 4, 3, 1, 1, bias=False) + # w = torch.rand((1, 4)).repeat(2, 1) + + # with torch.no_grad(): + # o1 = conv(rand_input) * w.unsqueeze(2).unsqueeze(3) + # print(w) + + # w = w.mean(0).unsqueeze(1).unsqueeze(2).unsqueeze(3) + # print(w) + # conv.weight.data.mul_(w) + + # o2 = conv(rand_input) + + # diff = ((o1 - o2) ** 2).sum() + # print(diff) + + + # rand_input = torch.rand((2, 3, 32, 32)) + # conv1 = nn.Conv2d(3, 6, 3, 1, 1, bias=False) + # conv2 = nn.Conv2d(3, 3, 3, 1, 1, bias=False, groups=3) + + # print(conv1.weight.data.size(), conv2.weight.data.size()) + + # import time + # import torch + # from utils.dl.common.model import get_model_latency + + # # s, e = torch.cuda.Event(enable_timing=True), torch.cuda.Event(enable_timing=True) + # # s.record() + # # # TODO + # # e.record() + # # torch.cuda.synchronize() + # # time_usage = s.elapsed_time(e) / 1000. + # # print(time_usage) + + # data = [torch.rand((512, 3, 3)).cuda() for _ in range(512)] + # # t1 = time.time() + # # for i in range(300): d = torch.stack(data) + # # t2 = time.time() + # # for i in range(300): d = torch.cat(data).view(512, 512, 3, 3) + # # t3 = time.time() + # # print("torch.stack time: {}, torch.cat time: {}".format(t2 - t1, t3 - t2)) + + # s, e = torch.cuda.Event(enable_timing=True), torch.cuda.Event(enable_timing=True) + # s.record() + # for i in range(300): d = torch.stack(data) + # e.record() + # torch.cuda.synchronize() + # time_usage = s.elapsed_time(e) / 1000. + # print(time_usage) + + # s, e = torch.cuda.Event(enable_timing=True), torch.cuda.Event(enable_timing=True) + # s.record() + # for i in range(300): d = torch.cat(data).view(512, 512, 3, 3) + # e.record() + # torch.cuda.synchronize() + # time_usage = s.elapsed_time(e) / 1000. + # print(time_usage) + + + # from models.resnet_cifar.resnet_cifar_3 import resnet18 + # model = resnet18() + + # full_l1_reg = 0. + # for name, module in model.named_modules(): + # if isinstance(module, nn.Conv2d): + # w = torch.ones((256, module.out_channels)) + # w[:, (module.out_channels // 2):] = 0. + # full_l1_reg += w.norm(1) + + # full_l1_reg /= 2 + + # print(f'{full_l1_reg:.3e}') + + # def f(x): + # # x = x - 0.5 + # return torch.max(torch.zeros_like(x), torch.min(torch.ones_like(x), 1.2 * torch.sigmoid(x) - 0.1)) + + # x = torch.arange(-2, 2, 0.01).float() + # y = f(x) + + # print(f(torch.FloatTensor([0.]))) + # print(f(torch.FloatTensor([0.5]))) + + # import matplotlib.pyplot as plt + + # plt.plot(x, y) + # plt.savefig('./tmp.png') + + # rand_input = torch.rand((256, 3, 32, 32)) + # conv = nn.Conv2d(3, 64, 3, 1, 1, bias=False) + # new_conv = DomainDynamicConv2d(conv, 0.1) + + # new_conv(rand_input) + + # conv = nn.Conv2d(3, 64, 3, 1, 1, bias=False) + # new_conv = DomainDynamicConv2d(conv, nn.BatchNorm2d(64), 0.1) + # print(new_conv.filter_selection_module[5].training) + # new_conv.eval() + # print(new_conv.filter_selection_module[5].training) + + n = KTakesAll(0.6) + + rand_input = torch.rand((1, 5)) + print(n(rand_input)) \ No newline at end of file diff --git a/new_impl/cv/resnet/resnet.py b/new_impl/cv/resnet/resnet.py new file mode 100644 index 0000000000000000000000000000000000000000..89438f117012fb966cbd4ec906619a66f8f5ea8a --- /dev/null +++ b/new_impl/cv/resnet/resnet.py @@ -0,0 +1,1152 @@ +from transformers import BlipForQuestionAnswering, BlipConfig,BlipModel +import torch +from torch import nn +from abc import ABC, abstractmethod +from copy import deepcopy +from typing import Optional, Union +from einops import rearrange, repeat +from einops.layers.torch import Rearrange +import tqdm + +from utils.dl.common.model import get_model_device, get_model_latency, get_model_size, set_module +from utils.dl.common.model import set_module, get_module, get_super_module +from utils.common.log import logger +from new_impl.cv.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util, LoRA +from transformers.models.blip.modeling_blip import BlipAttention +#from transformers.models.blip.modeling_blip_text import BlipTextSelfAttention,BlipTextAttention,BlipTextSelfOutput +from transformers.models.beit.modeling_beit import BeitSelfAttention,BeitConfig +from new_impl.cv.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from new_impl.cv.elasticdnn.model.base import Abs, KTakesAll, ElasticDNNUtil, Layer_WrappedWithFBS + +from typing import Optional, Tuple +import math + +# def blip(num_classes): +# model = BlipForQuestionAnswering.from_pretrained('new_impl/mm/Vis_bert/QuestionAnswering/VisBert_pretrained') +# # linear = model.text_decoder.cls.predictions.decoder +# # new_linear = nn.Linear(linear.in_features,30524,bias = True) +# # set_module(model,'text_decoder.cls.predictions.decoder',new_linear) +# return model +# def blip(num_classes): +# model = BlipForQuestionAnswering.from_pretrained('new_impl/mm/Vis_bert/QuestionAnswering/VisBert_pretrained') +# linear = model.text_decoder.cls.predictions.decoder +# new_linear = nn.Linear(linear.in_features,num_classes,bias = True) +# set_module(model,'text_decoder.cls.predictions.decoder',new_linear) +# return model +# class dinat(nn.Module): +# def __init__(self,num_classes): +# super(dinat,self).__init__() +# self.dinat = DinatModel.from_pretrained('shi-labs/dinat-mini-in1k-224') +# self.classifier = nn.Linear(768,num_classes) + +# def forward(self,**sample): +# output = self.dinat(**sample)[-1]#output the last hidden +# output = self.classifier(output[1]) +# return output + + +class ToQKV_WrappedWithLoRA(nn.Module): + def __init__(self, fc: nn.Linear, ab_r: int): + super(ToQKV_WrappedWithLoRA, self).__init__() + + self.fc = fc + self.ab = self.create_ab_as_linear(fc.weight.data, ab_r) + + def create_ab_as_linear(self, fc_weight: torch.Tensor, ab_r: int): + res = nn.Sequential( + LoRA(fc_weight.size(1), fc_weight.size(0) // ab_r, bias=False), + LoRA(fc_weight.size(0) // ab_r, fc_weight.size(0), bias=False) + ).to(fc_weight.device) + nn.init.kaiming_uniform_(res[0].weight, a=5 ** 0.5) + nn.init.zeros_(res[1].weight) + return res + + def forward(self, x): + x1 = self.fc(x) + x2 = self.ab(x) + return x1 + x2 + + +class FMLoRA_beit_Util(FMLoRA_Util): + + @torch.no_grad() + def add_lora_ab_to_fm(self, fm: nn.Module, ab_r: int, samples: torch.Tensor): + fm.eval() + + # print(samples) + # for k, v in samples.items(): + # if isinstance(v, torch.Tensor): + # samples[k] = v.to(get_model_device(fm)) + + #o1 = fm.generate(**samples) + o1 = fm(samples) + for name, module in fm.named_modules(): + if name.endswith(('query', 'key', 'value')): + set_module(fm, name, ToQKV_WrappedWithLoRA(module, ab_r)) + elif name.endswith('.qkv'): + set_module(fm, name, ToQKV_WrappedWithLoRA(module, ab_r)) + + + #o2 = fm.generate(**samples) + o2 = fm(samples) + if isinstance(o1, tuple): + o1 = o1[-1] + o2 = o2[-1] + output_diff = ((o1.logits - o2.logits) ** 2).sum() + assert output_diff < 1e-5 + return fm + + @torch.no_grad() + def absorb_lora_and_recover_net_structure(self, fm: nn.Module, samples: torch.Tensor): + fm.eval() + # print('absorb lora before') + + # for k, v in samples.items(): + # if isinstance(v, torch.Tensor): + # samples[k] = v.to(get_model_device(fm)) + + o1 = fm(samples) + + for name, module in fm.named_modules(): + if not isinstance(module, ToQKV_WrappedWithLoRA): + continue + + fc = module.fc + ab = module.ab + + fc.weight.add_(ab[1].weight @ ab[0].weight) + + set_module(fm, name, fc) + + # print('absorb lora after') + o2 = fm(samples) + + if isinstance(o1, tuple): + o1 = o1[-1] + o2 = o2[-1] + output_diff = ((o1.logits - o2.logits) ** 2).sum() + assert output_diff < 1e-6, output_diff + + return fm + + +####Here start with Fbs + +# class blipTextAttentionPrunable(BlipTextSelfAttention): +# def __init__(self,is_cross_attention): +# config = BlipConfig.from_pretrained('new_impl/mm/Vis_bert/QuestionAnswering/VisBert_pretrained') +# super(blipTextAttentionPrunable,self).__init__(config.text_config,is_cross_attention) + +# def save_attn_gradients(self, attn_gradients): +# self.attn_gradients = attn_gradients + +# def get_attn_gradients(self): +# return self.attn_gradients + +# def save_attention_map(self, attention_map): +# self.attention_map = attention_map + +# def get_attention_map(self): +# return self.attention_map + +# def transpose_for_scores(self, x): +# new_x_shape = x.size()[:-1] + (self.num_attention_heads, -1) +# x = x.view(*new_x_shape) +# return x.permute(0, 2, 1, 3) + +# def forward( +# self, +# hidden_states: torch.Tensor, +# attention_mask: Optional[torch.FloatTensor] = None, +# head_mask: Optional[torch.FloatTensor] = None, +# encoder_hidden_states: Optional[torch.FloatTensor] = None, +# encoder_attention_mask: Optional[torch.FloatTensor] = None, +# past_key_value: Optional[Tuple[Tuple[torch.FloatTensor]]] = None, +# output_attentions: Optional[bool] = False, +# ) -> Tuple[torch.Tensor]: +# mixed_query_layer = self.query(hidden_states) + +# # If this is instantiated as a cross-attention module, the keys +# # and values come from an encoder; the attention mask needs to be +# # such that the encoder's padding tokens are not attended to. +# is_cross_attention = encoder_hidden_states is not None + +# if is_cross_attention: +# key_layer = self.transpose_for_scores(self.key(encoder_hidden_states)) +# value_layer = self.transpose_for_scores(self.value(encoder_hidden_states)) +# attention_mask = encoder_attention_mask +# elif past_key_value is not None: +# key_layer = self.transpose_for_scores(self.key(hidden_states)) +# value_layer = self.transpose_for_scores(self.value(hidden_states)) +# key_layer = torch.cat([past_key_value[0], key_layer], dim=2) +# value_layer = torch.cat([past_key_value[1], value_layer], dim=2) +# else: +# key_layer = self.transpose_for_scores(self.key(hidden_states)) +# value_layer = self.transpose_for_scores(self.value(hidden_states)) + +# query_layer = self.transpose_for_scores(mixed_query_layer) + +# past_key_value = (key_layer, value_layer) + +# # Take the dot product between "query" and "key" to get the raw attention scores. +# attention_scores = torch.matmul(query_layer, key_layer.transpose(-1, -2)) + +# if self.position_embedding_type == "relative_key" or self.position_embedding_type == "relative_key_query": +# seq_length = hidden_states.size()[1] +# position_ids_l = torch.arange(seq_length, dtype=torch.long, device=hidden_states.device).view(-1, 1) +# position_ids_r = torch.arange(seq_length, dtype=torch.long, device=hidden_states.device).view(1, -1) +# distance = position_ids_l - position_ids_r +# positional_embedding = self.distance_embedding(distance + self.max_position_embeddings - 1) +# positional_embedding = positional_embedding.to(dtype=query_layer.dtype) # fp16 compatibility + +# if self.position_embedding_type == "relative_key": +# relative_position_scores = torch.einsum("bhld,lrd->bhlr", query_layer, positional_embedding) +# attention_scores = attention_scores + relative_position_scores +# elif self.position_embedding_type == "relative_key_query": +# relative_position_scores_query = torch.einsum("bhld,lrd->bhlr", query_layer, positional_embedding) +# relative_position_scores_key = torch.einsum("bhrd,lrd->bhlr", key_layer, positional_embedding) +# attention_scores = attention_scores + relative_position_scores_query + relative_position_scores_key + +# attention_scores = attention_scores / math.sqrt(self.attention_head_size) +# if attention_mask is not None: +# # Apply the attention mask is (precomputed for all layers in BlipTextModel forward() function) +# attention_scores = attention_scores + attention_mask.to(attention_scores.device) + +# # Normalize the attention scores to probabilities. +# attention_probs = nn.Softmax(dim=-1)(attention_scores) + +# # This is actually dropping out entire tokens to attend to, which might +# # seem a bit unusual, but is taken from the original Transformer paper. +# attention_probs_dropped = self.dropout(attention_probs) + +# # Mask heads if we want to +# if head_mask is not None: +# attention_probs_dropped = attention_probs_dropped * head_mask + +# context_layer = torch.matmul(attention_probs_dropped, value_layer) + +# context_layer = context_layer.permute(0, 2, 1, 3).contiguous() +# new_context_layer_shape = context_layer.size()[:-2] + (-1,) +# context_layer = context_layer.view(*new_context_layer_shape) + +# outputs = (context_layer, attention_probs) if output_attentions else (context_layer,) + +# outputs = outputs + (past_key_value,) +# return outputs +# @staticmethod +# def init_from_exist_self_attn(attn: BlipTextSelfAttention,is_cross_attention): +# # print(attn) + +# res = blipTextAttentionPrunable(is_cross_attention) + +# for attr in dir(attn): +# # if str(attr) in ['transpose_for_scores'] or str(attr).startswith('_'): +# # continue +# # if isinstance(getattr(attn, attr), nn.Module): +# # print(attr) + +# if isinstance(getattr(attn, attr), nn.Module): +# try: +# # print(attr, 'ok') +# setattr(res, attr, getattr(attn, attr)) + +# except Exception as e: +# print(attr, str(e)) + + + +# return res + + + +# class blipSelfTextAttentionPrunable(BlipTextAttention): +# def __init__(self, config, is_cross_attention=False): +# self.self = blipTextAttentionPrunable(config, is_cross_attention) +# self.output = BlipTextSelfOutput(config) +# self.pruned_heads = set() +# super(blipSelfTextAttentionPrunable,self).__init__(config) + +# def prune_heads(self, heads): +# if len(heads) == 0: +# return +# heads, index = find_pruneable_heads_and_indices( +# heads, self.self.num_attention_heads, self.self.attention_head_size, self.pruned_heads +# ) + +# # Prune linear layers +# self.self.query = prune_linear_layer(self.self.query, index) +# self.self.key = prune_linear_layer(self.self.key, index) +# self.self.value = prune_linear_layer(self.self.value, index) +# self.output.dense = prune_linear_layer(self.output.dense, index, dim=1) + +# # Update hyper params and store pruned heads +# self.self.num_attention_heads = self.self.num_attention_heads - len(heads) +# self.self.all_head_size = self.self.attention_head_size * self.self.num_attention_heads +# self.pruned_heads = self.pruned_heads.union(heads) + +# def forward( +# self, +# hidden_states: torch.Tensor, +# attention_mask: Optional[torch.FloatTensor] = None, +# head_mask: Optional[torch.FloatTensor] = None, +# encoder_hidden_states: Optional[torch.FloatTensor] = None, +# encoder_attention_mask: Optional[torch.FloatTensor] = None, +# past_key_value: Optional[Tuple[Tuple[torch.FloatTensor]]] = None, +# output_attentions: Optional[bool] = False, +# ) -> Tuple[torch.Tensor]: +# self_outputs = self.self( +# hidden_states, +# attention_mask, +# head_mask, +# encoder_hidden_states, +# encoder_attention_mask, +# past_key_value, +# output_attentions, +# ) +# attention_output = self.output(self_outputs[0], hidden_states) +# outputs = (attention_output,) + self_outputs[1:] # add attentions if we output them +# return outputs +# @staticmethod +# def init_from_exist_self_attn(attn: BlipTextAttention,config,is_cross_attention): +# # print(attn) + +# res = blipTextAttentionPrunable(config,is_cross_attention) + +# for attr in dir(attn): +# # if str(attr) in ['transpose_for_scores'] or str(attr).startswith('_'): +# # continue +# # if isinstance(getattr(attn, attr), nn.Module): +# # print(attr) + +# if isinstance(getattr(attn, attr), nn.Module): +# try: +# # print(attr, 'ok') +# setattr(res, attr, getattr(attn, attr)) + +# except Exception as e: +# print(attr, str(e)) + + + +# return res + + + + + + +# class blipSelfAttentionPrunable(BlipAttention): +# def __init__(self): +# config = BlipConfig.from_pretrained('new_impl/mm/Vis_bert/QuestionAnswering/VisBert_pretrained') +# super(blipSelfAttentionPrunable, self).__init__(config.vision_config) + +# def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int): +# return tensor.view(bsz, seq_len, self.num_heads, -1).transpose(1, 2).contiguous() + +# def forward( +# self, +# hidden_states: torch.Tensor, +# head_mask: Optional[torch.Tensor] = None, +# output_attentions: Optional[bool] = False, +# ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: +# """Input shape: Batch x Time x Channel""" + +# bsz, tgt_len, embed_dim = hidden_states.size() + +# mixed_qkv = ( +# self.qkv(hidden_states) +# .reshape(bsz, tgt_len, 3, self.num_heads, -1) +# .permute(2, 0, 3, 1, 4) +# ) +# query_states, key_states, value_states = mixed_qkv[0], mixed_qkv[1], mixed_qkv[2] + +# # Take the dot product between "query" and "key" to get the raw attention scores. +# attention_scores = torch.matmul(query_states, key_states.transpose(-1, -2)) + +# attention_scores = attention_scores * self.scale + +# # Normalize the attention scores to probabilities. +# attention_probs = nn.functional.softmax(attention_scores, dim=-1) + +# # This is actually dropping out entire tokens to attend to, which might +# # seem a bit unusual, but is taken from the original Transformer paper. +# attention_probs = self.dropout(attention_probs) + +# # Mask heads if we want to +# if head_mask is not None: +# attention_probs = attention_probs * head_mask + +# context_layer = torch.matmul(attention_probs, value_states).permute(0, 2, 1, 3) + +# new_context_layer_shape = context_layer.size()[:-2] + (-1,) +# context_layer = context_layer.reshape(new_context_layer_shape) + +# output = self.projection(context_layer) + +# outputs = (output, attention_probs) if output_attentions else (output, None) + +# return outputs + +# @staticmethod +# def init_from_exist_self_attn(attn: BlipAttention): +# # print(attn) + +# res = blipSelfAttentionPrunable() + +# for attr in dir(attn): +# # if str(attr) in ['transpose_for_scores'] or str(attr).startswith('_'): +# # continue +# # if isinstance(getattr(attn, attr), nn.Module): +# # print(attr) + +# if isinstance(getattr(attn, attr), nn.Module): +# try: +# # print(attr, 'ok') +# setattr(res, attr, getattr(attn, attr)) + +# except Exception as e: +# print(attr, str(e)) + + + +# return res +class BeitSelfAttentionPrunable(BeitSelfAttention): + def __init__(self, config: BeitConfig, window_size: Optional[tuple] = None) -> None: + config = BeitConfig.from_pretrained('new_impl/cv/beit/beit_model') + super(BeitSelfAttentionPrunable, self).__init__(config) + + def transpose_for_scores(self, x): + new_x_shape = x.size()[:-1] + (self.num_attention_heads, -1) + x = x.view(*new_x_shape) + return x.permute(0, 2, 1, 3) + + def forward( + self, + hidden_states: torch.Tensor, + head_mask: Optional[torch.Tensor] = None, + output_attentions: bool = False, + relative_position_bias: Optional["BeitRelativePositionBias"] = None, + ) -> Union[Tuple[torch.Tensor], Tuple[torch.Tensor, torch.Tensor]]: + mixed_query_layer = self.query(hidden_states) + + key_layer = self.transpose_for_scores(self.key(hidden_states)) + value_layer = self.transpose_for_scores(self.value(hidden_states)) + query_layer = self.transpose_for_scores(mixed_query_layer) + + # Take the dot product between "query" and "key" to get the raw attention scores. + attention_scores = torch.matmul(query_layer, key_layer.transpose(-1, -2)) + + attention_scores = attention_scores / math.sqrt(self.attention_head_size) + + # Add relative position bias if present. + if self.relative_position_bias is not None: + attention_scores = attention_scores + self.relative_position_bias().unsqueeze(0) + + # Add shared relative position bias if provided. + if relative_position_bias is not None: + attention_scores = attention_scores + relative_position_bias + + # Normalize the attention scores to probabilities. + attention_probs = nn.functional.softmax(attention_scores, dim=-1) + + # This is actually dropping out entire tokens to attend to, which might + # seem a bit unusual, but is taken from the original Transformer paper. + attention_probs = self.dropout(attention_probs) + + # Mask heads if we want to + if head_mask is not None: + attention_probs = attention_probs * head_mask + + context_layer = torch.matmul(attention_probs, value_layer) + + context_layer = context_layer.permute(0, 2, 1, 3).contiguous() + new_context_layer_shape = context_layer.size()[:-2] + (-1,) + context_layer = context_layer.view(*new_context_layer_shape) + + outputs = (context_layer, attention_probs) if output_attentions else (context_layer,) + + return outputs + + @staticmethod + def init_from_exist_self_attn(attn: BeitSelfAttention,config): + # print(attn) + + res = BeitSelfAttentionPrunable(config) + + for attr in dir(attn): + # if str(attr) in ['transpose_for_scores'] or str(attr).startswith('_'): + # continue + # if isinstance(getattr(attn, attr), nn.Module): + # print(attr) + + if isinstance(getattr(attn, attr), nn.Module): + try: + # print(attr, 'ok') + setattr(res, attr, getattr(attn, attr)) + + except Exception as e: + print(attr, str(e)) + return res + +class FM_to_MD_beit_Util(FM_to_MD_Util): + def init_md_from_fm_by_reducing_width(self, fm: nn.Module, reducing_width_ratio: int) -> nn.Module: + fm_vis = deepcopy(fm) + config = BeitConfig.from_pretrained('new_impl/cv/beit/beit_model') + + for block_i,block in enumerate(fm_vis.beit.encoder.layer): + set_module(block, 'attention.attention', BeitSelfAttentionPrunable.init_from_exist_self_attn(block.attention.attention,config)) + + def _f(n): + return int(n // reducing_width_ratio) + + # def _rand_indexes(n): + # return torch.randperm(n)[0: int(n // reducing_width_ratio)] + + def l1_max_indexes(p: torch.Tensor, dim=0): + assert dim in [0, 1] + assert p.dim() in [1, 2, 4] + + if dim == 1: + p = p.T + + p_norm = p.abs().contiguous().view(p.size(0), -1).sum(dim=1) + n = p.size(0) + return p_norm.argsort(descending=True)[0: int(n // reducing_width_ratio)].sort()[0] + + for block_i, block in enumerate(fm_vis.beit.encoder.layer): + for k in ['query', 'key', 'value']: + qkv = get_module(block, f'attention.attention.{k}') + + new_qkv = nn.Linear(qkv.in_features, _f(qkv.out_features), + qkv.bias is not None, qkv.weight.device) + indexes = l1_max_indexes(qkv.weight.data, 0) + + new_qkv.weight.data.copy_(qkv.weight.data[indexes]) + if qkv.bias is not None: + new_qkv.bias.data.copy_(qkv.bias.data[indexes]) + set_module(block, f'attention.attention.{k}', new_qkv) + + proj = get_module(block, f'attention.output.dense') + new_proj = nn.Linear(_f(proj.in_features), proj.out_features, + proj.bias is not None, proj.weight.device) + new_proj.weight.data.copy_(proj.weight.data[:, l1_max_indexes(proj.weight.data, 1)]) + if proj.bias is not None: + new_proj.bias.data.copy_(proj.bias.data) + set_module(block, f'attention.output.dense', new_proj) + + fc1 = get_module(block, f'intermediate.dense') + new_fc1 = nn.Linear(fc1.in_features, _f(fc1.out_features), + fc1.bias is not None, fc1.weight.device) + indexes = l1_max_indexes(fc1.weight.data, 0) + new_fc1.weight.data.copy_(fc1.weight.data[indexes]) + if fc1.bias is not None: + new_fc1.bias.data.copy_(fc1.bias.data[indexes]) + set_module(block, f'intermediate.dense', new_fc1) + + fc2 = get_module(block, f'output.dense') + new_fc2 = nn.Linear(_f(fc2.in_features), fc2.out_features, + fc2.bias is not None, fc2.weight.device) + new_fc2.weight.data.copy_(fc2.weight.data[:, l1_max_indexes(fc2.weight.data, 1)]) + if fc2.bias is not None: + new_fc2.bias.data.copy_(fc2.bias.data) + set_module(block, f'output.dense', new_fc2) + + + # for block_i, block in enumerate(fm_vis.text_decoder.bert.encoder.layer): + # for k in ['query', 'key', 'value']: + # qkv = get_module(block, f'crossattention.self.{k}') + + # new_qkv = nn.Linear(qkv.in_features, _f(qkv.out_features), + # qkv.bias is not None, qkv.weight.device) + # indexes = l1_max_indexes(qkv.weight.data, 0) + + # new_qkv.weight.data.copy_(qkv.weight.data[indexes]) + # if qkv.bias is not None: + # new_qkv.bias.data.copy_(qkv.bias.data[indexes]) + # set_module(block, f'crossattention.self.{k}', new_qkv) + + # proj = get_module(block, f'crossattention.output.dense') + # new_proj = nn.Linear(_f(proj.in_features), proj.out_features, + # proj.bias is not None, proj.weight.device) + # new_proj.weight.data.copy_(proj.weight.data[:, l1_max_indexes(proj.weight.data, 1)]) + # if proj.bias is not None: + # new_proj.bias.data.copy_(proj.bias.data) + # set_module(block, f'crossattention.output.dense', new_proj) + + + # for block_i, block in enumerate(fm_vis.text_encoder.encoder.layer): + # for k in ['query', 'key', 'value']: + # qkv = get_module(block, f'attention.self.{k}') + + # new_qkv = nn.Linear(qkv.in_features, _f(qkv.out_features), + # qkv.bias is not None, qkv.weight.device) + # indexes = l1_max_indexes(qkv.weight.data, 0) + + # new_qkv.weight.data.copy_(qkv.weight.data[indexes]) + # if qkv.bias is not None: + # new_qkv.bias.data.copy_(qkv.bias.data[indexes]) + # set_module(block, f'attention.self.{k}', new_qkv) + + # proj = get_module(block, f'attention.output.dense') + # new_proj = nn.Linear(_f(proj.in_features), proj.out_features, + # proj.bias is not None, proj.weight.device) + # new_proj.weight.data.copy_(proj.weight.data[:, l1_max_indexes(proj.weight.data, 1)]) + # if proj.bias is not None: + # new_proj.bias.data.copy_(proj.bias.data) + # set_module(block, f'attention.output.dense', new_proj) + + # fc1 = get_module(block, f'intermediate.dense') + # new_fc1 = nn.Linear(fc1.in_features, _f(fc1.out_features), + # fc1.bias is not None, fc1.weight.device) + # indexes = l1_max_indexes(fc1.weight.data, 0) + # new_fc1.weight.data.copy_(fc1.weight.data[indexes]) + # if fc1.bias is not None: + # new_fc1.bias.data.copy_(fc1.bias.data[indexes]) + # set_module(block, f'intermediate.dense', new_fc1) + + # fc2 = get_module(block, f'output.dense') + # new_fc2 = nn.Linear(_f(fc2.in_features), fc2.out_features, + # fc2.bias is not None, fc2.weight.device) + # new_fc2.weight.data.copy_(fc2.weight.data[:, l1_max_indexes(fc2.weight.data, 1)]) + # if fc2.bias is not None: + # new_fc2.bias.data.copy_(fc2.bias.data) + # set_module(block, f'output.dense', new_fc2) + + + # for block_i, block in enumerate(fm_vis.text_encoder.encoder.layer): + # for k in ['query', 'key', 'value']: + # qkv = get_module(block, f'crossattention.self.{k}') + + # new_qkv = nn.Linear(qkv.in_features, _f(qkv.out_features), + # qkv.bias is not None, qkv.weight.device) + # indexes = l1_max_indexes(qkv.weight.data, 0) + + # new_qkv.weight.data.copy_(qkv.weight.data[indexes]) + # if qkv.bias is not None: + # new_qkv.bias.data.copy_(qkv.bias.data[indexes]) + # set_module(block, f'crossattention.self.{k}', new_qkv) + + # proj = get_module(block, f'crossattention.output.dense') + # new_proj = nn.Linear(_f(proj.in_features), proj.out_features, + # proj.bias is not None, proj.weight.device) + # new_proj.weight.data.copy_(proj.weight.data[:, l1_max_indexes(proj.weight.data, 1)]) + # if proj.bias is not None: + # new_proj.bias.data.copy_(proj.bias.data) + # set_module(block, f'crossattention.output.dense', new_proj) + + + + # for block_i, block in enumerate(fm_vis.vision_model.encoder.layers): + # qkv = block.self_attn.qkv + + # new_qkv = nn.Linear(qkv.in_features, _f(qkv.out_features), + # qkv.bias is not None, qkv.weight.device) + # indexes = l1_max_indexes(qkv.weight.data, 0) + + # new_qkv.weight.data.copy_(qkv.weight.data[indexes]) + # if qkv.bias is not None: + # new_qkv.bias.data.copy_(qkv.bias.data[indexes]) + # set_module(fm_vis, f'vision_model.encoder.layers.{block_i}.self_attn.qkv', new_qkv) + + # proj = block.self_attn.projection + # new_proj = nn.Linear(_f(proj.in_features), proj.out_features, + # proj.bias is not None, proj.weight.device) + # new_proj.weight.data.copy_(proj.weight.data[:, l1_max_indexes(proj.weight.data, 1)]) + # if proj.bias is not None: + # new_proj.bias.data.copy_(proj.bias.data) + # set_module(fm_vis, f'vision_model.encoder.layers.{block_i}.self_attn.projection', new_proj) + + # fc1 = block.mlp.fc1 + # new_fc1 = nn.Linear(fc1.in_features, _f(fc1.out_features), + # fc1.bias is not None, fc1.weight.device) + # indexes = l1_max_indexes(fc1.weight.data, 0) + # new_fc1.weight.data.copy_(fc1.weight.data[indexes]) + # if fc1.bias is not None: + # new_fc1.bias.data.copy_(fc1.bias.data[indexes]) + # set_module(fm_vis, f'vision_model.encoder.layers.{block_i}.mlp.fc1', new_fc1) + + # fc2 = block.mlp.fc2 + # new_fc2 = nn.Linear(_f(fc2.in_features), fc2.out_features, + # fc2.bias is not None, fc2.weight.device) + # new_fc2.weight.data.copy_(fc2.weight.data[:, l1_max_indexes(fc2.weight.data, 1)]) + # if fc2.bias is not None: + # new_fc2.bias.data.copy_(fc2.bias.data) + # set_module(fm_vis, f'vision_model.encoder.layers.{block_i}.mlp.fc2', new_fc2) + + return fm_vis + + def init_md_from_fm_by_reducing_width_with_perf_test(self, fm: nn.Module, reducing_width_ratio: int, + samples: torch.Tensor) -> nn.Module: + fm_size = get_model_size(fm, True) + fm_latency = self._get_model_latency(fm, samples, 20, + get_model_device(fm), 20, False) + + master_dnn = self.init_md_from_fm_by_reducing_width(fm, reducing_width_ratio) + master_dnn_size = get_model_size(master_dnn, True) + logger.debug(f'inited master DNN: {master_dnn}') + master_dnn_latency = self._get_model_latency(master_dnn, samples, 20, + get_model_device(master_dnn), 20, False) + + logger.info(f'init master DNN (w/o FBS yet) by reducing foundation model\'s width (by {reducing_width_ratio:d}x)') + logger.info(f'foundation model ({fm_size:.3f}MB, {fm_latency:.4f}s/sample) -> ' + f'master DNN ({master_dnn_size:.3f}MB, {master_dnn_latency:.4f}s/sample)\n' + f'(model size: ↓ {(fm_size / master_dnn_size):.2f}x, ' + f'latency: ↓ {(fm_latency / master_dnn_latency):.2f}x)') + + return master_dnn + + def _get_model_latency(self, model: torch.nn.Module, model_input_size, sample_num: int, + device: str, warmup_sample_num: int, return_detail=False): + import time + + if isinstance(model_input_size, tuple): + dummy_input = torch.rand(model_input_size).to(device) + else: + dummy_input = model_input_size + + model = model.to(device) + model.eval() + + # warm up + with torch.no_grad(): + for _ in range(warmup_sample_num): + model(dummy_input) + + infer_time_list = [] + + if device == 'cuda' or 'cuda' in str(device): + with torch.no_grad(): + for _ in range(sample_num): + s, e = torch.cuda.Event(enable_timing=True), torch.cuda.Event(enable_timing=True) + s.record() + model(dummy_input) + e.record() + torch.cuda.synchronize() + cur_model_infer_time = s.elapsed_time(e) / 1000. + infer_time_list += [cur_model_infer_time] + + else: + with torch.no_grad(): + for _ in range(sample_num): + start = time.time() + model(**dummy_input) + cur_model_infer_time = time.time() - start + infer_time_list += [cur_model_infer_time] + + avg_infer_time = sum(infer_time_list) / sample_num + + if return_detail: + return avg_infer_time, infer_time_list + return avg_infer_time + + +####Here starts with index + +class SqueezeLast(nn.Module): + def __init__(self): + super(SqueezeLast, self).__init__() + + def forward(self, x): + return x.squeeze(-1) + + +class ProjConv_WrappedWithFBS(Layer_WrappedWithFBS): + def __init__(self, proj: nn.Conv2d, r): + super(ProjConv_WrappedWithFBS, self).__init__() + + self.proj = proj + + # for conv: (B, C_in, H, W) -> (B, C_in) -> (B, C_out) + # for mlp in ViT: (B, #patches, D: dim of patches embedding) -> (B, D) -> (B, C_out) + self.fbs = nn.Sequential( + Abs(), + nn.AdaptiveAvgPool1d(1), + SqueezeLast(), + nn.Linear(proj.in_channels, proj.out_channels // r), + nn.ReLU(), + nn.Linear(proj.out_channels // r, proj.out_channels), + nn.ReLU() + ) + + nn.init.constant_(self.fbs[6].bias, 1.) + nn.init.kaiming_normal_(self.fbs[6].weight) + + + def forward(self, x): + if self.use_cached_channel_attention and self.cached_channel_attention is not None: + channel_attention = self.cached_channel_attention + else: + self.cached_raw_channel_attention = self.fbs(x) + self.cached_channel_attention = self.k_takes_all(self.cached_raw_channel_attention) + + channel_attention = self.cached_channel_attention + + raw_res = self.proj(x) + + return channel_attention.unsqueeze(1) * raw_res # TODO: + + +class Linear_WrappedWithFBS(Layer_WrappedWithFBS): + def __init__(self, linear: nn.Linear, r): + super(Linear_WrappedWithFBS, self).__init__() + + self.linear = linear + + # for conv: (B, C_in, H, W) -> (B, C_in) -> (B, C_out) + # for mlp in ViT: (B, #patches, D: dim of patches embedding) -> (B, D) -> (B, C_out) + self.fbs = nn.Sequential( + Rearrange('b n d -> b d n'), + Abs(), + nn.AdaptiveAvgPool1d(1), + SqueezeLast(), + nn.Linear(linear.in_features, linear.out_features // r), + nn.ReLU(), + nn.Linear(linear.out_features // r, linear.out_features), + nn.ReLU() + ) + + nn.init.constant_(self.fbs[6].bias, 1.) + nn.init.kaiming_normal_(self.fbs[6].weight) + + + def forward(self, x): + if self.use_cached_channel_attention and self.cached_channel_attention is not None: + channel_attention = self.cached_channel_attention + else: + self.cached_raw_channel_attention = self.fbs(x) + self.cached_channel_attention = self.k_takes_all(self.cached_raw_channel_attention) + + channel_attention = self.cached_channel_attention + + raw_res = self.linear(x) + + return channel_attention.unsqueeze(1) * raw_res + + +class ToQKV_WrappedWithFBS(Layer_WrappedWithFBS): + """ + This regards to_q/to_k/to_v as a whole (in fact it consists of multiple heads) and prunes it. + It seems different channels of different heads are pruned according to the input. + This is different from "removing some head" or "removing the same channels in each head". + """ + def __init__(self, to_qkv: nn.Linear, r): + super(ToQKV_WrappedWithFBS, self).__init__() + + # self.to_qkv = to_qkv + + self.to_qk = nn.Linear(to_qkv.in_features, to_qkv.out_features // 3 * 2, bias=to_qkv.bias is not None) + self.to_v = nn.Linear(to_qkv.in_features, to_qkv.out_features // 3, bias=to_qkv.bias is not None) + self.to_qk.weight.data.copy_(to_qkv.weight.data[0: to_qkv.out_features // 3 * 2]) + if to_qkv.bias is not None: + self.to_qk.bias.data.copy_(to_qkv.bias.data[0: to_qkv.out_features // 3 * 2]) + self.to_v.weight.data.copy_(to_qkv.weight.data[to_qkv.out_features // 3 * 2: ]) + if to_qkv.bias is not None: + self.to_v.bias.data.copy_(to_qkv.bias.data[to_qkv.out_features // 3 * 2: ]) + + self.fbs = nn.Sequential( + Rearrange('b n d -> b d n'), + Abs(), + nn.AdaptiveAvgPool1d(1), + SqueezeLast(), + nn.Linear(to_qkv.in_features, to_qkv.out_features // 3 // r), + nn.ReLU(), + # nn.Linear(to_qkv.out_features // 3 // r, to_qkv.out_features // 3), + nn.Linear(to_qkv.out_features // 3 // r, self.to_v.out_features), + nn.ReLU() + ) + + nn.init.constant_(self.fbs[6].bias, 1.) + nn.init.kaiming_normal_(self.fbs[6].weight) + + def forward(self, x): + if self.use_cached_channel_attention and self.cached_channel_attention is not None: + channel_attention = self.cached_channel_attention + else: + self.cached_raw_channel_attention = self.fbs(x) + + # print() + # for attn in self.cached_raw_channel_attention.chunk(3, dim=1)[0: 1]: + # print(self.cached_raw_channel_attention.size(), attn.size()) + # print(self.k_takes_all.k) + # print(attn[0].nonzero(as_tuple=True)[0].size(), attn[0]) + + self.cached_channel_attention = self.k_takes_all(self.cached_raw_channel_attention) + + + # for attn in self.cached_channel_attention.chunk(3, dim=1)[0: 1]: + # print(self.cached_channel_attention.size(), attn.size()) + # print(self.k_takes_all.k) + # print(attn[0].nonzero(as_tuple=True)[0].size(), attn[0]) + # print() + + channel_attention = self.cached_channel_attention + + qk = self.to_qk(x) + v = channel_attention.unsqueeze(1) * self.to_v(x) + return torch.cat([qk, v], dim=-1) + + # qkv = raw_res.chunk(3, dim = -1) + + # raw_v = qkv[2] + # print('raw_k, raw_v', qkv[0].sum((0, 1))[0: 10], qkv[0].sum((0, 1)).nonzero(as_tuple=True)[0].size(), + # qkv[1].sum((0, 1))[0: 10], qkv[1].sum((0, 1)).nonzero(as_tuple=True)[0].size(),) + # print('raw_v', raw_v.size(), raw_v.sum((0, 1))[0: 10], raw_v.sum((0, 1)).nonzero(as_tuple=True)[0].size()) + + # qkv_attn = channel_attention.chunk(3, dim=-1) + # print('attn', [attn[0][0: 10] for attn in qkv_attn]) + # print(channel_attention.unsqueeze(1).size(), raw_res.size()) + # print('fbs', channel_attention.size(), raw_res.size()) + # return channel_attention.unsqueeze(1) * raw_res + + +class StaticFBS(nn.Module): + def __init__(self, static_channel_attention): + super(StaticFBS, self).__init__() + assert static_channel_attention.dim() == 2 and static_channel_attention.size(0) == 1 + self.static_channel_attention = nn.Parameter(static_channel_attention, requires_grad=False) # (1, dim) + + def forward(self, x): + # print('staticfbs', x, self.static_channel_attention.unsqueeze(1)) + return x * self.static_channel_attention.unsqueeze(1) + + +class ElasticresUtil(ElasticDNNUtil): + def convert_raw_dnn_to_master_dnn(self, raw_dnn: nn.Module, r: float, ignore_layers=[]): + assert len(ignore_layers) == 0, 'not supported yet' + + raw_vit = deepcopy(raw_dnn) + + # set_module(module, 'patch_embed.proj', ProjConv_WrappedWithFBS(module.patch_embed.proj, r)) + + for name, module in raw_vit.named_modules(): + # if name.endswith('attn'): + # set_module(module, 'qkv', ToQKV_WrappedWithFBS(module.qkv, r)) + # if name.endswith('intermediate'): + # set_module(module, 'dense', Linear_WrappedWithFBS(module.dense, r)) + # elif name.endswith('mlp'): + # set_module(module, 'fc1', Linear_WrappedWithFBS(module.fc1, r)) + if name.endswith('convolution'): + set_module(module,'',ProjConv_WrappedWithFBS(module,r)) + break + + return raw_vit + + def set_master_dnn_sparsity(self, master_dnn: nn.Module, sparsity: float): + # for name, module in master_dnn.named_modules(): + # if not name.endswith('attn'): + # continue + + # q_features = module.qkv.to_qk.out_features // 2 + + # if (q_features - int(q_features * sparsity)) % module.num_heads != 0: + # # tune sparsity to ensure #unpruned channel % num_heads == 0 + # # so that the pruning seems to reduce the dim_head of each head + # tuned_sparsity = 1. - int((q_features - int(q_features * sparsity)) / module.num_heads) * module.num_heads / q_features + # logger.debug(f'tune sparsity from {sparsity:.2f} to {tuned_sparsity}') + # sparsity = tuned_sparsity + # break + + return super().set_master_dnn_sparsity(master_dnn, sparsity) + + def select_most_rep_sample(self, master_dnn: nn.Module, samples: torch.Tensor): + # print(samples) + return samples[0].unsqueeze(0) + # res = {k: v[0: 1] for k, v in samples.items()} + # return res + + def extract_surrogate_dnn_via_samples(self, master_dnn: nn.Module, samples: torch.Tensor, return_detail=False):#产生小模型的步骤 + # sample = self.select_most_rep_sample(master_dnn, samples) + # # assert sample.dim() == 4 and sample.size(0) == 1 + + # # print('before') + # master_dnn.eval() + # self.clear_cached_channel_attention_in_master_dnn(master_dnn) + # with torch.no_grad(): + # master_dnn_output = master_dnn(sample) + + # # print('after') + + # boosted_vit = deepcopy(master_dnn) + + # def get_unpruned_indexes_from_channel_attn(channel_attn: torch.Tensor, k): + # assert channel_attn.size(0) == 1, 'use A representative sample to generate channel attentions' + + # # print('attn_in_unpruned', channel_attn[0][0: 10]) + + # res = channel_attn[0].nonzero(as_tuple=True)[0] # should be one-dim + + # # res = channel_attn[0].argsort(descending=True)[0: -int(channel_attn.size(1) * k)].sort()[0] + + # # g = channel_attn + # # k = g.size(1) - int(g.size(1) * k) + # # res = g.topk(k, 1)[1][0].sort()[0] + + # return res + + # unpruned_indexes_of_layers = {} + + # # for attn, ff in boosted_vit.transformer.layers: + # # for block_i, block in enumerate(boosted_vit.blocks): + # for block_i, block in enumerate(boosted_vit.beit.encoder.layer): + # # attn = block.attn + # # ff = block.mlp + + # ff_0 = get_module(block, f'intermediate.dense') + # # ff_0_unpruned_indexes = get_unpruned_indexes_from_channel_attn(ff_0.cached_channel_attention, k) + # ff_0_pruned_indexes = ff_0.k_takes_all.cached_i[0].sort()[0] + # ff_0_unpruned_indexes = torch.LongTensor([ii for ii in range(ff_0.cached_channel_attention.size(1)) if ii not in ff_0_pruned_indexes]) + # new_ff_0 = nn.Linear(ff_0.linear.in_features, ff_0_unpruned_indexes.size(0), ff_0.linear.bias is not None) + # new_ff_0.weight.data.copy_(ff_0.linear.weight.data[ff_0_unpruned_indexes]) + # if ff_0.linear.bias is not None: + # new_ff_0.bias.data.copy_(ff_0.linear.bias.data[ff_0_unpruned_indexes]) + # set_module(block, 'intermediate.dense', nn.Sequential(new_ff_0, StaticFBS(ff_0.cached_channel_attention[:, ff_0_unpruned_indexes]))) + + # ff_1 = get_module(block, f'output.dense') + # new_ff_1 = nn.Linear(ff_0_unpruned_indexes.size(0), ff_1.out_features, ff_1.bias is not None) + # new_ff_1.weight.data.copy_(ff_1.weight.data[:, ff_0_unpruned_indexes]) + # if ff_1.bias is not None: + # new_ff_1.bias.data.copy_(ff_1.bias.data) + # set_module(block, 'output.dense', new_ff_1) + + # unpruned_indexes_of_layers[f'beit.encoder.layer.{block_i}.intermediate.dense.0.weight'] = ff_0_unpruned_indexes + # # for block_i,block in enumerate(boosted_vit.vision_model.encoder.layers): + + # # attn = block.self_attn + # # ff = block.mlp + # # ff_0 = ff.fc1 + # # # ff_0_unpruned_indexes = get_unpruned_indexes_from_channel_attn(ff_0.cached_channel_attention, k) + # # ff_0_pruned_indexes = ff_0.k_takes_all.cached_i[0].sort()[0] + # # ff_0_unpruned_indexes = torch.LongTensor([ii for ii in range(ff_0.cached_channel_attention.size(1)) if ii not in ff_0_pruned_indexes]) + # # new_ff_0 = nn.Linear(ff_0.linear.in_features, ff_0_unpruned_indexes.size(0), ff_0.linear.bias is not None) + # # new_ff_0.weight.data.copy_(ff_0.linear.weight.data[ff_0_unpruned_indexes]) + # # if ff_0.linear.bias is not None: + # # new_ff_0.bias.data.copy_(ff_0.linear.bias.data[ff_0_unpruned_indexes]) + # # set_module(ff, 'fc1', nn.Sequential(new_ff_0, StaticFBS(ff_0.cached_channel_attention[:, ff_0_unpruned_indexes]))) + + # # ff_1 = ff.fc2 + # # new_ff_1 = nn.Linear(ff_0_unpruned_indexes.size(0), ff_1.out_features, ff_1.bias is not None) + # # new_ff_1.weight.data.copy_(ff_1.weight.data[:, ff_0_unpruned_indexes]) + # # if ff_1.bias is not None: + # # new_ff_1.bias.data.copy_(ff_1.bias.data) + # # set_module(ff, 'fc2', new_ff_1) + + # # unpruned_indexes_of_layers[f'vision_model.encoder.layers.{block_i}.mlp.fc1.0.weight'] = ff_0_unpruned_indexes + + + # # for block_i, block in enumerate(boosted_vit.text_decoder.bert.encoder.layer): + # # # attn = block.attn + # # # ff = block.mlp + + # # ff_0 = get_module(block, f'intermediate.dense') + # # # ff_0_unpruned_indexes = get_unpruned_indexes_from_channel_attn(ff_0.cached_channel_attention, k) + # # ff_0_pruned_indexes = ff_0.k_takes_all.cached_i[0].sort()[0] + # # ff_0_unpruned_indexes = torch.LongTensor([ii for ii in range(ff_0.cached_channel_attention.size(1)) if ii not in ff_0_pruned_indexes]) + # # new_ff_0 = nn.Linear(ff_0.linear.in_features, ff_0_unpruned_indexes.size(0), ff_0.linear.bias is not None) + # # new_ff_0.weight.data.copy_(ff_0.linear.weight.data[ff_0_unpruned_indexes]) + # # if ff_0.linear.bias is not None: + # # new_ff_0.bias.data.copy_(ff_0.linear.bias.data[ff_0_unpruned_indexes]) + # # set_module(block, 'intermediate.dense', nn.Sequential(new_ff_0, StaticFBS(ff_0.cached_channel_attention[:, ff_0_unpruned_indexes]))) + + # # ff_1 = get_module(block, f'output.dense') + # # new_ff_1 = nn.Linear(ff_0_unpruned_indexes.size(0), ff_1.out_features, ff_1.bias is not None) + # # new_ff_1.weight.data.copy_(ff_1.weight.data[:, ff_0_unpruned_indexes]) + # # if ff_1.bias is not None: + # # new_ff_1.bias.data.copy_(ff_1.bias.data) + # # set_module(block, 'output.dense', new_ff_1) + + # # unpruned_indexes_of_layers[f'text_decoder.bert.encoder.layer.{block_i}.intermediate.dense.0.weight'] = ff_0_unpruned_indexes + # surrogate_dnn = boosted_vit + # surrogate_dnn.eval() + # surrogate_dnn = surrogate_dnn.to(get_model_device(master_dnn)) + # # logger.debug(surrogate_dnn) + # with torch.no_grad(): + # surrogate_dnn_output = surrogate_dnn(sample) + + # output_diff = ((surrogate_dnn_output.logits - master_dnn_output.logits) ** 2).sum() + # # assert output_diff < 1e-4, output_diff + # logger.info(f'output diff of master and surrogate DNN: {output_diff}') + # # logger.debug(f'example output of master/surrogate: {master_dnn_output.sum(0)[0: 10]}, {surrogate_dnn_output.sum(0)[0: 10]}') + # # logger.info(f'\nonly prune mlp!!!!\n') + # # logger.info(f'\nonly prune mlp!!!!\n') + + # if return_detail: + # return boosted_vit, unpruned_indexes_of_layers + + # return boosted_vit + from new_impl.cv.resnet.extract_submodel import FBSSubModelExtractor,DAFBSSubModelExtractor + x = DAFBSSubModelExtractor() + boost, indexes = x.extract_submodel_via_samples(master_dnn,samples) + return boost , indexes + + def extract_surrogate_dnn_via_samples_with_perf_test(self, master_dnn: nn.Module, samples: torch.Tensor, return_detail=False): + master_dnn_size = get_model_size(master_dnn, True) + master_dnn_latency = self._get_model_latency(master_dnn, samples, 50, + get_model_device(master_dnn), 50, False) + + res = self.extract_surrogate_dnn_via_samples(master_dnn, samples, return_detail) + if not return_detail: + surrogate_dnn = res + else: + surrogate_dnn, unpruned_indexes_of_layers = res + surrogate_dnn_size = get_model_size(surrogate_dnn, True) + surrogate_dnn_latency = self._get_model_latency(master_dnn, samples, 50, + get_model_device(master_dnn), 50, False) + + logger.info(f'master DNN ({master_dnn_size:.3f}MB, {master_dnn_latency:.4f}s/sample) -> ' + f'surrogate DNN ({surrogate_dnn_size:.3f}MB, {surrogate_dnn_latency:.4f}s/sample)\n' + f'(model size: ↓ {(master_dnn_size / surrogate_dnn_size):.2f}x, ' + f'latency: ↓ {(master_dnn_latency / surrogate_dnn_latency):.2f}x)') + + return res + + def _get_model_latency(self, model: torch.nn.Module, model_input_size, sample_num: int, + device: str, warmup_sample_num: int, return_detail=False): + import time + + if isinstance(model_input_size, tuple): + dummy_input = torch.rand(model_input_size).to(device) + else: + dummy_input = model_input_size + + model = model.to(device) + model.eval() + + # warm up + with torch.no_grad(): + for _ in range(warmup_sample_num): + model(dummy_input) + + infer_time_list = [] + + if device == 'cuda' or 'cuda' in str(device): + with torch.no_grad(): + for _ in range(sample_num): + s, e = torch.cuda.Event(enable_timing=True), torch.cuda.Event(enable_timing=True) + s.record() + model(dummy_input) + e.record() + torch.cuda.synchronize() + cur_model_infer_time = s.elapsed_time(e) / 1000. + infer_time_list += [cur_model_infer_time] + + else: + with torch.no_grad(): + for _ in range(sample_num): + start = time.time() + model(**dummy_input) + cur_model_infer_time = time.time() - start + infer_time_list += [cur_model_infer_time] + + avg_infer_time = sum(infer_time_list) / sample_num + + if return_detail: + return avg_infer_time, infer_time_list + return avg_infer_time + + + +#####Here starts with online + + + + diff --git a/new_impl/cv/resnet/resnet_pretrained/config.json b/new_impl/cv/resnet/resnet_pretrained/config.json new file mode 100644 index 0000000000000000000000000000000000000000..30289c9792e668d73c991829c8842977e2b90539 --- /dev/null +++ b/new_impl/cv/resnet/resnet_pretrained/config.json @@ -0,0 +1,2028 @@ +{ + "architectures": [ + "ResNetForImageClassification" + ], + "depths": [ + 3, + 4, + 6, + 3 + ], + "downsample_in_first_stage": false, + "embedding_size": 64, + "hidden_act": "relu", + "hidden_sizes": [ + 256, + 512, + 1024, + 2048 + ], + "id2label": { + "0": "tench, Tinca tinca", + "1": "goldfish, Carassius auratus", + "2": "great white shark, white shark, man-eater, man-eating shark, Carcharodon carcharias", + "3": "tiger shark, Galeocerdo cuvieri", + "4": "hammerhead, hammerhead shark", + "5": "electric ray, crampfish, numbfish, torpedo", + "6": "stingray", + "7": "cock", + "8": "hen", + "9": "ostrich, Struthio camelus", + "10": "brambling, Fringilla montifringilla", + "11": "goldfinch, Carduelis carduelis", + "12": "house finch, linnet, Carpodacus mexicanus", + "13": "junco, snowbird", + "14": "indigo bunting, indigo finch, indigo bird, Passerina cyanea", + "15": "robin, American robin, Turdus migratorius", + "16": "bulbul", + "17": "jay", + "18": "magpie", + "19": "chickadee", + "20": "water ouzel, dipper", + "21": "kite", + "22": "bald eagle, American eagle, Haliaeetus leucocephalus", + "23": "vulture", + "24": "great grey owl, great gray owl, Strix nebulosa", + "25": "European fire salamander, Salamandra salamandra", + "26": "common newt, Triturus vulgaris", + "27": "eft", + "28": "spotted salamander, Ambystoma maculatum", + "29": "axolotl, mud puppy, Ambystoma mexicanum", + "30": "bullfrog, Rana catesbeiana", + "31": "tree frog, tree-frog", + "32": "tailed frog, bell toad, ribbed toad, tailed toad, Ascaphus trui", + "33": "loggerhead, loggerhead turtle, Caretta caretta", + "34": "leatherback turtle, leatherback, leathery turtle, Dermochelys coriacea", + "35": "mud turtle", + "36": "terrapin", + "37": "box turtle, box tortoise", + "38": "banded gecko", + "39": "common iguana, iguana, Iguana iguana", + "40": "American chameleon, anole, Anolis carolinensis", + "41": "whiptail, whiptail lizard", + "42": "agama", + "43": "frilled lizard, Chlamydosaurus kingi", + "44": "alligator lizard", + "45": "Gila monster, Heloderma suspectum", + "46": "green lizard, Lacerta viridis", + "47": "African chameleon, Chamaeleo chamaeleon", + "48": "Komodo dragon, Komodo lizard, dragon lizard, giant lizard, Varanus komodoensis", + "49": "African crocodile, Nile crocodile, Crocodylus niloticus", + "50": "American alligator, Alligator mississipiensis", + "51": "triceratops", + "52": "thunder snake, worm snake, Carphophis amoenus", + "53": "ringneck snake, ring-necked snake, ring snake", + "54": "hognose snake, puff adder, sand viper", + "55": "green snake, grass snake", + "56": "king snake, kingsnake", + "57": "garter snake, grass snake", + "58": "water snake", + "59": "vine snake", + "60": "night snake, Hypsiglena torquata", + "61": "boa constrictor, Constrictor constrictor", + "62": "rock python, rock snake, Python sebae", + "63": "Indian cobra, Naja naja", + "64": "green mamba", + "65": "sea snake", + "66": "horned viper, cerastes, sand viper, horned asp, Cerastes cornutus", + "67": "diamondback, diamondback rattlesnake, Crotalus adamanteus", + "68": "sidewinder, horned rattlesnake, Crotalus cerastes", + "69": "trilobite", + "70": "harvestman, daddy longlegs, Phalangium opilio", + "71": "scorpion", + "72": "black and gold garden spider, Argiope aurantia", + "73": "barn spider, Araneus cavaticus", + "74": "garden spider, Aranea diademata", + "75": "black widow, Latrodectus mactans", + "76": "tarantula", + "77": "wolf spider, hunting spider", + "78": "tick", + "79": "centipede", + "80": "black grouse", + "81": "ptarmigan", + "82": "ruffed grouse, partridge, Bonasa umbellus", + "83": "prairie chicken, prairie grouse, prairie fowl", + "84": "peacock", + "85": "quail", + "86": "partridge", + "87": "African grey, African gray, Psittacus erithacus", + "88": "macaw", + "89": "sulphur-crested cockatoo, Kakatoe galerita, Cacatua galerita", + "90": "lorikeet", + "91": "coucal", + "92": "bee eater", + "93": "hornbill", + "94": "hummingbird", + "95": "jacamar", + "96": "toucan", + "97": "drake", + "98": "red-breasted merganser, Mergus serrator", + "99": "goose", + "100": "black swan, Cygnus atratus", + "101": "tusker", + "102": "echidna, spiny anteater, anteater", + "103": "platypus, duckbill, duckbilled platypus, duck-billed platypus, Ornithorhynchus anatinus", + "104": "wallaby, brush kangaroo", + "105": "koala, koala bear, kangaroo bear, native bear, Phascolarctos cinereus", + "106": "wombat", + "107": "jellyfish", + "108": "sea anemone, anemone", + "109": "brain coral", + "110": "flatworm, platyhelminth", + "111": "nematode, nematode worm, roundworm", + "112": "conch", + "113": "snail", + "114": "slug", + "115": "sea slug, nudibranch", + "116": "chiton, coat-of-mail shell, sea cradle, polyplacophore", + "117": "chambered nautilus, pearly nautilus, nautilus", + "118": "Dungeness crab, Cancer magister", + "119": "rock crab, Cancer irroratus", + "120": "fiddler crab", + "121": "king crab, Alaska crab, Alaskan king crab, Alaska king crab, Paralithodes camtschatica", + "122": "American lobster, Northern lobster, Maine lobster, Homarus americanus", + "123": "spiny lobster, langouste, rock lobster, crawfish, crayfish, sea crawfish", + "124": "crayfish, crawfish, crawdad, crawdaddy", + "125": "hermit crab", + "126": "isopod", + "127": "white stork, Ciconia ciconia", + "128": "black stork, Ciconia nigra", + "129": "spoonbill", + "130": "flamingo", + "131": "little blue heron, Egretta caerulea", + "132": "American egret, great white heron, Egretta albus", + "133": "bittern", + "134": "crane", + "135": "limpkin, Aramus pictus", + "136": "European gallinule, Porphyrio porphyrio", + "137": "American coot, marsh hen, mud hen, water hen, Fulica americana", + "138": "bustard", + "139": "ruddy turnstone, Arenaria interpres", + "140": "red-backed sandpiper, dunlin, Erolia alpina", + "141": "redshank, Tringa totanus", + "142": "dowitcher", + "143": "oystercatcher, oyster catcher", + "144": "pelican", + "145": "king penguin, Aptenodytes patagonica", + "146": "albatross, mollymawk", + "147": "grey whale, gray whale, devilfish, Eschrichtius gibbosus, Eschrichtius robustus", + "148": "killer whale, killer, orca, grampus, sea wolf, Orcinus orca", + "149": "dugong, Dugong dugon", + "150": "sea lion", + "151": "Chihuahua", + "152": "Japanese spaniel", + "153": "Maltese dog, Maltese terrier, Maltese", + "154": "Pekinese, Pekingese, Peke", + "155": "Shih-Tzu", + "156": "Blenheim spaniel", + "157": "papillon", + "158": "toy terrier", + "159": "Rhodesian ridgeback", + "160": "Afghan hound, Afghan", + "161": "basset, basset hound", + "162": "beagle", + "163": "bloodhound, sleuthhound", + "164": "bluetick", + "165": "black-and-tan coonhound", + "166": "Walker hound, Walker foxhound", + "167": "English foxhound", + "168": "redbone", + "169": "borzoi, Russian wolfhound", + "170": "Irish wolfhound", + "171": "Italian greyhound", + "172": "whippet", + "173": "Ibizan hound, Ibizan Podenco", + "174": "Norwegian elkhound, elkhound", + "175": "otterhound, otter hound", + "176": "Saluki, gazelle hound", + "177": "Scottish deerhound, deerhound", + "178": "Weimaraner", + "179": "Staffordshire bullterrier, Staffordshire bull terrier", + "180": "American Staffordshire terrier, Staffordshire terrier, American pit bull terrier, pit bull terrier", + "181": "Bedlington terrier", + "182": "Border terrier", + "183": "Kerry blue terrier", + "184": "Irish terrier", + "185": "Norfolk terrier", + "186": "Norwich terrier", + "187": "Yorkshire terrier", + "188": "wire-haired fox terrier", + "189": "Lakeland terrier", + "190": "Sealyham terrier, Sealyham", + "191": "Airedale, Airedale terrier", + "192": "cairn, cairn terrier", + "193": "Australian terrier", + "194": "Dandie Dinmont, Dandie Dinmont terrier", + "195": "Boston bull, Boston terrier", + "196": "miniature schnauzer", + "197": "giant schnauzer", + "198": "standard schnauzer", + "199": "Scotch terrier, Scottish terrier, Scottie", + "200": "Tibetan terrier, chrysanthemum dog", + "201": "silky terrier, Sydney silky", + "202": "soft-coated wheaten terrier", + "203": "West Highland white terrier", + "204": "Lhasa, Lhasa apso", + "205": "flat-coated retriever", + "206": "curly-coated retriever", + "207": "golden retriever", + "208": "Labrador retriever", + "209": "Chesapeake Bay retriever", + "210": "German short-haired pointer", + "211": "vizsla, Hungarian pointer", + "212": "English setter", + "213": "Irish setter, red setter", + "214": "Gordon setter", + "215": "Brittany spaniel", + "216": "clumber, clumber spaniel", + "217": "English springer, English springer spaniel", + "218": "Welsh springer spaniel", + "219": "cocker spaniel, English cocker spaniel, cocker", + "220": "Sussex spaniel", + "221": "Irish water spaniel", + "222": "kuvasz", + "223": "schipperke", + "224": "groenendael", + "225": "malinois", + "226": "briard", + "227": "kelpie", + "228": "komondor", + "229": "Old English sheepdog, bobtail", + "230": "Shetland sheepdog, Shetland sheep dog, Shetland", + "231": "collie", + "232": "Border collie", + "233": "Bouvier des Flandres, Bouviers des Flandres", + "234": "Rottweiler", + "235": "German shepherd, German shepherd dog, German police dog, alsatian", + "236": "Doberman, Doberman pinscher", + "237": "miniature pinscher", + "238": "Greater Swiss Mountain dog", + "239": "Bernese mountain dog", + "240": "Appenzeller", + "241": "EntleBucher", + "242": "boxer", + "243": "bull mastiff", + "244": "Tibetan mastiff", + "245": "French bulldog", + "246": "Great Dane", + "247": "Saint Bernard, St Bernard", + "248": "Eskimo dog, husky", + "249": "malamute, malemute, Alaskan malamute", + "250": "Siberian husky", + "251": "dalmatian, coach dog, carriage dog", + "252": "affenpinscher, monkey pinscher, monkey dog", + "253": "basenji", + "254": "pug, pug-dog", + "255": "Leonberg", + "256": "Newfoundland, Newfoundland dog", + "257": "Great Pyrenees", + "258": "Samoyed, Samoyede", + "259": "Pomeranian", + "260": "chow, chow chow", + "261": "keeshond", + "262": "Brabancon griffon", + "263": "Pembroke, Pembroke Welsh corgi", + "264": "Cardigan, Cardigan Welsh corgi", + "265": "toy poodle", + "266": "miniature poodle", + "267": "standard poodle", + "268": "Mexican hairless", + "269": "timber wolf, grey wolf, gray wolf, Canis lupus", + "270": "white wolf, Arctic wolf, Canis lupus tundrarum", + "271": "red wolf, maned wolf, Canis rufus, Canis niger", + "272": "coyote, prairie wolf, brush wolf, Canis latrans", + "273": "dingo, warrigal, warragal, Canis dingo", + "274": "dhole, Cuon alpinus", + "275": "African hunting dog, hyena dog, Cape hunting dog, Lycaon pictus", + "276": "hyena, hyaena", + "277": "red fox, Vulpes vulpes", + "278": "kit fox, Vulpes macrotis", + "279": "Arctic fox, white fox, Alopex lagopus", + "280": "grey fox, gray fox, Urocyon cinereoargenteus", + "281": "tabby, tabby cat", + "282": "tiger cat", + "283": "Persian cat", + "284": "Siamese cat, Siamese", + "285": "Egyptian cat", + "286": "cougar, puma, catamount, mountain lion, painter, panther, Felis concolor", + "287": "lynx, catamount", + "288": "leopard, Panthera pardus", + "289": "snow leopard, ounce, Panthera uncia", + "290": "jaguar, panther, Panthera onca, Felis onca", + "291": "lion, king of beasts, Panthera leo", + "292": "tiger, Panthera tigris", + "293": "cheetah, chetah, Acinonyx jubatus", + "294": "brown bear, bruin, Ursus arctos", + "295": "American black bear, black bear, Ursus americanus, Euarctos americanus", + "296": "ice bear, polar bear, Ursus Maritimus, Thalarctos maritimus", + "297": "sloth bear, Melursus ursinus, Ursus ursinus", + "298": "mongoose", + "299": "meerkat, mierkat", + "300": "tiger beetle", + "301": "ladybug, ladybeetle, lady beetle, ladybird, ladybird beetle", + "302": "ground beetle, carabid beetle", + "303": "long-horned beetle, longicorn, longicorn beetle", + "304": "leaf beetle, chrysomelid", + "305": "dung beetle", + "306": "rhinoceros beetle", + "307": "weevil", + "308": "fly", + "309": "bee", + "310": "ant, emmet, pismire", + "311": "grasshopper, hopper", + "312": "cricket", + "313": "walking stick, walkingstick, stick insect", + "314": "cockroach, roach", + "315": "mantis, mantid", + "316": "cicada, cicala", + "317": "leafhopper", + "318": "lacewing, lacewing fly", + "319": "dragonfly, darning needle, devil's darning needle, sewing needle, snake feeder, snake doctor, mosquito hawk, skeeter hawk", + "320": "damselfly", + "321": "admiral", + "322": "ringlet, ringlet butterfly", + "323": "monarch, monarch butterfly, milkweed butterfly, Danaus plexippus", + "324": "cabbage butterfly", + "325": "sulphur butterfly, sulfur butterfly", + "326": "lycaenid, lycaenid butterfly", + "327": "starfish, sea star", + "328": "sea urchin", + "329": "sea cucumber, holothurian", + "330": "wood rabbit, cottontail, cottontail rabbit", + "331": "hare", + "332": "Angora, Angora rabbit", + "333": "hamster", + "334": "porcupine, hedgehog", + "335": "fox squirrel, eastern fox squirrel, Sciurus niger", + "336": "marmot", + "337": "beaver", + "338": "guinea pig, Cavia cobaya", + "339": "sorrel", + "340": "zebra", + "341": "hog, pig, grunter, squealer, Sus scrofa", + "342": "wild boar, boar, Sus scrofa", + "343": "warthog", + "344": "hippopotamus, hippo, river horse, Hippopotamus amphibius", + "345": "ox", + "346": "water buffalo, water ox, Asiatic buffalo, Bubalus bubalis", + "347": "bison", + "348": "ram, tup", + "349": "bighorn, bighorn sheep, cimarron, Rocky Mountain bighorn, Rocky Mountain sheep, Ovis canadensis", + "350": "ibex, Capra ibex", + "351": "hartebeest", + "352": "impala, Aepyceros melampus", + "353": "gazelle", + "354": "Arabian camel, dromedary, Camelus dromedarius", + "355": "llama", + "356": "weasel", + "357": "mink", + "358": "polecat, fitch, foulmart, foumart, Mustela putorius", + "359": "black-footed ferret, ferret, Mustela nigripes", + "360": "otter", + "361": "skunk, polecat, wood pussy", + "362": "badger", + "363": "armadillo", + "364": "three-toed sloth, ai, Bradypus tridactylus", + "365": "orangutan, orang, orangutang, Pongo pygmaeus", + "366": "gorilla, Gorilla gorilla", + "367": "chimpanzee, chimp, Pan troglodytes", + "368": "gibbon, Hylobates lar", + "369": "siamang, Hylobates syndactylus, Symphalangus syndactylus", + "370": "guenon, guenon monkey", + "371": "patas, hussar monkey, Erythrocebus patas", + "372": "baboon", + "373": "macaque", + "374": "langur", + "375": "colobus, colobus monkey", + "376": "proboscis monkey, Nasalis larvatus", + "377": "marmoset", + "378": "capuchin, ringtail, Cebus capucinus", + "379": "howler monkey, howler", + "380": "titi, titi monkey", + "381": "spider monkey, Ateles geoffroyi", + "382": "squirrel monkey, Saimiri sciureus", + "383": "Madagascar cat, ring-tailed lemur, Lemur catta", + "384": "indri, indris, Indri indri, Indri brevicaudatus", + "385": "Indian elephant, Elephas maximus", + "386": "African elephant, Loxodonta africana", + "387": "lesser panda, red panda, panda, bear cat, cat bear, Ailurus fulgens", + "388": "giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca", + "389": "barracouta, snoek", + "390": "eel", + "391": "coho, cohoe, coho salmon, blue jack, silver salmon, Oncorhynchus kisutch", + "392": "rock beauty, Holocanthus tricolor", + "393": "anemone fish", + "394": "sturgeon", + "395": "gar, garfish, garpike, billfish, Lepisosteus osseus", + "396": "lionfish", + "397": "puffer, pufferfish, blowfish, globefish", + "398": "abacus", + "399": "abaya", + "400": "academic gown, academic robe, judge's robe", + "401": "accordion, piano accordion, squeeze box", + "402": "acoustic guitar", + "403": "aircraft carrier, carrier, flattop, attack aircraft carrier", + "404": "airliner", + "405": "airship, dirigible", + "406": "altar", + "407": "ambulance", + "408": "amphibian, amphibious vehicle", + "409": "analog clock", + "410": "apiary, bee house", + "411": "apron", + "412": "ashcan, trash can, garbage can, wastebin, ash bin, ash-bin, ashbin, dustbin, trash barrel, trash bin", + "413": "assault rifle, assault gun", + "414": "backpack, back pack, knapsack, packsack, rucksack, haversack", + "415": "bakery, bakeshop, bakehouse", + "416": "balance beam, beam", + "417": "balloon", + "418": "ballpoint, ballpoint pen, ballpen, Biro", + "419": "Band Aid", + "420": "banjo", + "421": "bannister, banister, balustrade, balusters, handrail", + "422": "barbell", + "423": "barber chair", + "424": "barbershop", + "425": "barn", + "426": "barometer", + "427": "barrel, cask", + "428": "barrow, garden cart, lawn cart, wheelbarrow", + "429": "baseball", + "430": "basketball", + "431": "bassinet", + "432": "bassoon", + "433": "bathing cap, swimming cap", + "434": "bath towel", + "435": "bathtub, bathing tub, bath, tub", + "436": "beach wagon, station wagon, wagon, estate car, beach waggon, station waggon, waggon", + "437": "beacon, lighthouse, beacon light, pharos", + "438": "beaker", + "439": "bearskin, busby, shako", + "440": "beer bottle", + "441": "beer glass", + "442": "bell cote, bell cot", + "443": "bib", + "444": "bicycle-built-for-two, tandem bicycle, tandem", + "445": "bikini, two-piece", + "446": "binder, ring-binder", + "447": "binoculars, field glasses, opera glasses", + "448": "birdhouse", + "449": "boathouse", + "450": "bobsled, bobsleigh, bob", + "451": "bolo tie, bolo, bola tie, bola", + "452": "bonnet, poke bonnet", + "453": "bookcase", + "454": "bookshop, bookstore, bookstall", + "455": "bottlecap", + "456": "bow", + "457": "bow tie, bow-tie, bowtie", + "458": "brass, memorial tablet, plaque", + "459": "brassiere, bra, bandeau", + "460": "breakwater, groin, groyne, mole, bulwark, seawall, jetty", + "461": "breastplate, aegis, egis", + "462": "broom", + "463": "bucket, pail", + "464": "buckle", + "465": "bulletproof vest", + "466": "bullet train, bullet", + "467": "butcher shop, meat market", + "468": "cab, hack, taxi, taxicab", + "469": "caldron, cauldron", + "470": "candle, taper, wax light", + "471": "cannon", + "472": "canoe", + "473": "can opener, tin opener", + "474": "cardigan", + "475": "car mirror", + "476": "carousel, carrousel, merry-go-round, roundabout, whirligig", + "477": "carpenter's kit, tool kit", + "478": "carton", + "479": "car wheel", + "480": "cash machine, cash dispenser, automated teller machine, automatic teller machine, automated teller, automatic teller, ATM", + "481": "cassette", + "482": "cassette player", + "483": "castle", + "484": "catamaran", + "485": "CD player", + "486": "cello, violoncello", + "487": "cellular telephone, cellular phone, cellphone, cell, mobile phone", + "488": "chain", + "489": "chainlink fence", + "490": "chain mail, ring mail, mail, chain armor, chain armour, ring armor, ring armour", + "491": "chain saw, chainsaw", + "492": "chest", + "493": "chiffonier, commode", + "494": "chime, bell, gong", + "495": "china cabinet, china closet", + "496": "Christmas stocking", + "497": "church, church building", + "498": "cinema, movie theater, movie theatre, movie house, picture palace", + "499": "cleaver, meat cleaver, chopper", + "500": "cliff dwelling", + "501": "cloak", + "502": "clog, geta, patten, sabot", + "503": "cocktail shaker", + "504": "coffee mug", + "505": "coffeepot", + "506": "coil, spiral, volute, whorl, helix", + "507": "combination lock", + "508": "computer keyboard, keypad", + "509": "confectionery, confectionary, candy store", + "510": "container ship, containership, container vessel", + "511": "convertible", + "512": "corkscrew, bottle screw", + "513": "cornet, horn, trumpet, trump", + "514": "cowboy boot", + "515": "cowboy hat, ten-gallon hat", + "516": "cradle", + "517": "crane", + "518": "crash helmet", + "519": "crate", + "520": "crib, cot", + "521": "Crock Pot", + "522": "croquet ball", + "523": "crutch", + "524": "cuirass", + "525": "dam, dike, dyke", + "526": "desk", + "527": "desktop computer", + "528": "dial telephone, dial phone", + "529": "diaper, nappy, napkin", + "530": "digital clock", + "531": "digital watch", + "532": "dining table, board", + "533": "dishrag, dishcloth", + "534": "dishwasher, dish washer, dishwashing machine", + "535": "disk brake, disc brake", + "536": "dock, dockage, docking facility", + "537": "dogsled, dog sled, dog sleigh", + "538": "dome", + "539": "doormat, welcome mat", + "540": "drilling platform, offshore rig", + "541": "drum, membranophone, tympan", + "542": "drumstick", + "543": "dumbbell", + "544": "Dutch oven", + "545": "electric fan, blower", + "546": "electric guitar", + "547": "electric locomotive", + "548": "entertainment center", + "549": "envelope", + "550": "espresso maker", + "551": "face powder", + "552": "feather boa, boa", + "553": "file, file cabinet, filing cabinet", + "554": "fireboat", + "555": "fire engine, fire truck", + "556": "fire screen, fireguard", + "557": "flagpole, flagstaff", + "558": "flute, transverse flute", + "559": "folding chair", + "560": "football helmet", + "561": "forklift", + "562": "fountain", + "563": "fountain pen", + "564": "four-poster", + "565": "freight car", + "566": "French horn, horn", + "567": "frying pan, frypan, skillet", + "568": "fur coat", + "569": "garbage truck, dustcart", + "570": "gasmask, respirator, gas helmet", + "571": "gas pump, gasoline pump, petrol pump, island dispenser", + "572": "goblet", + "573": "go-kart", + "574": "golf ball", + "575": "golfcart, golf cart", + "576": "gondola", + "577": "gong, tam-tam", + "578": "gown", + "579": "grand piano, grand", + "580": "greenhouse, nursery, glasshouse", + "581": "grille, radiator grille", + "582": "grocery store, grocery, food market, market", + "583": "guillotine", + "584": "hair slide", + "585": "hair spray", + "586": "half track", + "587": "hammer", + "588": "hamper", + "589": "hand blower, blow dryer, blow drier, hair dryer, hair drier", + "590": "hand-held computer, hand-held microcomputer", + "591": "handkerchief, hankie, hanky, hankey", + "592": "hard disc, hard disk, fixed disk", + "593": "harmonica, mouth organ, harp, mouth harp", + "594": "harp", + "595": "harvester, reaper", + "596": "hatchet", + "597": "holster", + "598": "home theater, home theatre", + "599": "honeycomb", + "600": "hook, claw", + "601": "hoopskirt, crinoline", + "602": "horizontal bar, high bar", + "603": "horse cart, horse-cart", + "604": "hourglass", + "605": "iPod", + "606": "iron, smoothing iron", + "607": "jack-o'-lantern", + "608": "jean, blue jean, denim", + "609": "jeep, landrover", + "610": "jersey, T-shirt, tee shirt", + "611": "jigsaw puzzle", + "612": "jinrikisha, ricksha, rickshaw", + "613": "joystick", + "614": "kimono", + "615": "knee pad", + "616": "knot", + "617": "lab coat, laboratory coat", + "618": "ladle", + "619": "lampshade, lamp shade", + "620": "laptop, laptop computer", + "621": "lawn mower, mower", + "622": "lens cap, lens cover", + "623": "letter opener, paper knife, paperknife", + "624": "library", + "625": "lifeboat", + "626": "lighter, light, igniter, ignitor", + "627": "limousine, limo", + "628": "liner, ocean liner", + "629": "lipstick, lip rouge", + "630": "Loafer", + "631": "lotion", + "632": "loudspeaker, speaker, speaker unit, loudspeaker system, speaker system", + "633": "loupe, jeweler's loupe", + "634": "lumbermill, sawmill", + "635": "magnetic compass", + "636": "mailbag, postbag", + "637": "mailbox, letter box", + "638": "maillot", + "639": "maillot, tank suit", + "640": "manhole cover", + "641": "maraca", + "642": "marimba, xylophone", + "643": "mask", + "644": "matchstick", + "645": "maypole", + "646": "maze, labyrinth", + "647": "measuring cup", + "648": "medicine chest, medicine cabinet", + "649": "megalith, megalithic structure", + "650": "microphone, mike", + "651": "microwave, microwave oven", + "652": "military uniform", + "653": "milk can", + "654": "minibus", + "655": "miniskirt, mini", + "656": "minivan", + "657": "missile", + "658": "mitten", + "659": "mixing bowl", + "660": "mobile home, manufactured home", + "661": "Model T", + "662": "modem", + "663": "monastery", + "664": "monitor", + "665": "moped", + "666": "mortar", + "667": "mortarboard", + "668": "mosque", + "669": "mosquito net", + "670": "motor scooter, scooter", + "671": "mountain bike, all-terrain bike, off-roader", + "672": "mountain tent", + "673": "mouse, computer mouse", + "674": "mousetrap", + "675": "moving van", + "676": "muzzle", + "677": "nail", + "678": "neck brace", + "679": "necklace", + "680": "nipple", + "681": "notebook, notebook computer", + "682": "obelisk", + "683": "oboe, hautboy, hautbois", + "684": "ocarina, sweet potato", + "685": "odometer, hodometer, mileometer, milometer", + "686": "oil filter", + "687": "organ, pipe organ", + "688": "oscilloscope, scope, cathode-ray oscilloscope, CRO", + "689": "overskirt", + "690": "oxcart", + "691": "oxygen mask", + "692": "packet", + "693": "paddle, boat paddle", + "694": "paddlewheel, paddle wheel", + "695": "padlock", + "696": "paintbrush", + "697": "pajama, pyjama, pj's, jammies", + "698": "palace", + "699": "panpipe, pandean pipe, syrinx", + "700": "paper towel", + "701": "parachute, chute", + "702": "parallel bars, bars", + "703": "park bench", + "704": "parking meter", + "705": "passenger car, coach, carriage", + "706": "patio, terrace", + "707": "pay-phone, pay-station", + "708": "pedestal, plinth, footstall", + "709": "pencil box, pencil case", + "710": "pencil sharpener", + "711": "perfume, essence", + "712": "Petri dish", + "713": "photocopier", + "714": "pick, plectrum, plectron", + "715": "pickelhaube", + "716": "picket fence, paling", + "717": "pickup, pickup truck", + "718": "pier", + "719": "piggy bank, penny bank", + "720": "pill bottle", + "721": "pillow", + "722": "ping-pong ball", + "723": "pinwheel", + "724": "pirate, pirate ship", + "725": "pitcher, ewer", + "726": "plane, carpenter's plane, woodworking plane", + "727": "planetarium", + "728": "plastic bag", + "729": "plate rack", + "730": "plow, plough", + "731": "plunger, plumber's helper", + "732": "Polaroid camera, Polaroid Land camera", + "733": "pole", + "734": "police van, police wagon, paddy wagon, patrol wagon, wagon, black Maria", + "735": "poncho", + "736": "pool table, billiard table, snooker table", + "737": "pop bottle, soda bottle", + "738": "pot, flowerpot", + "739": "potter's wheel", + "740": "power drill", + "741": "prayer rug, prayer mat", + "742": "printer", + "743": "prison, prison house", + "744": "projectile, missile", + "745": "projector", + "746": "puck, hockey puck", + "747": "punching bag, punch bag, punching ball, punchball", + "748": "purse", + "749": "quill, quill pen", + "750": "quilt, comforter, comfort, puff", + "751": "racer, race car, racing car", + "752": "racket, racquet", + "753": "radiator", + "754": "radio, wireless", + "755": "radio telescope, radio reflector", + "756": "rain barrel", + "757": "recreational vehicle, RV, R.V.", + "758": "reel", + "759": "reflex camera", + "760": "refrigerator, icebox", + "761": "remote control, remote", + "762": "restaurant, eating house, eating place, eatery", + "763": "revolver, six-gun, six-shooter", + "764": "rifle", + "765": "rocking chair, rocker", + "766": "rotisserie", + "767": "rubber eraser, rubber, pencil eraser", + "768": "rugby ball", + "769": "rule, ruler", + "770": "running shoe", + "771": "safe", + "772": "safety pin", + "773": "saltshaker, salt shaker", + "774": "sandal", + "775": "sarong", + "776": "sax, saxophone", + "777": "scabbard", + "778": "scale, weighing machine", + "779": "school bus", + "780": "schooner", + "781": "scoreboard", + "782": "screen, CRT screen", + "783": "screw", + "784": "screwdriver", + "785": "seat belt, seatbelt", + "786": "sewing machine", + "787": "shield, buckler", + "788": "shoe shop, shoe-shop, shoe store", + "789": "shoji", + "790": "shopping basket", + "791": "shopping cart", + "792": "shovel", + "793": "shower cap", + "794": "shower curtain", + "795": "ski", + "796": "ski mask", + "797": "sleeping bag", + "798": "slide rule, slipstick", + "799": "sliding door", + "800": "slot, one-armed bandit", + "801": "snorkel", + "802": "snowmobile", + "803": "snowplow, snowplough", + "804": "soap dispenser", + "805": "soccer ball", + "806": "sock", + "807": "solar dish, solar collector, solar furnace", + "808": "sombrero", + "809": "soup bowl", + "810": "space bar", + "811": "space heater", + "812": "space shuttle", + "813": "spatula", + "814": "speedboat", + "815": "spider web, spider's web", + "816": "spindle", + "817": "sports car, sport car", + "818": "spotlight, spot", + "819": "stage", + "820": "steam locomotive", + "821": "steel arch bridge", + "822": "steel drum", + "823": "stethoscope", + "824": "stole", + "825": "stone wall", + "826": "stopwatch, stop watch", + "827": "stove", + "828": "strainer", + "829": "streetcar, tram, tramcar, trolley, trolley car", + "830": "stretcher", + "831": "studio couch, day bed", + "832": "stupa, tope", + "833": "submarine, pigboat, sub, U-boat", + "834": "suit, suit of clothes", + "835": "sundial", + "836": "sunglass", + "837": "sunglasses, dark glasses, shades", + "838": "sunscreen, sunblock, sun blocker", + "839": "suspension bridge", + "840": "swab, swob, mop", + "841": "sweatshirt", + "842": "swimming trunks, bathing trunks", + "843": "swing", + "844": "switch, electric switch, electrical switch", + "845": "syringe", + "846": "table lamp", + "847": "tank, army tank, armored combat vehicle, armoured combat vehicle", + "848": "tape player", + "849": "teapot", + "850": "teddy, teddy bear", + "851": "television, television system", + "852": "tennis ball", + "853": "thatch, thatched roof", + "854": "theater curtain, theatre curtain", + "855": "thimble", + "856": "thresher, thrasher, threshing machine", + "857": "throne", + "858": "tile roof", + "859": "toaster", + "860": "tobacco shop, tobacconist shop, tobacconist", + "861": "toilet seat", + "862": "torch", + "863": "totem pole", + "864": "tow truck, tow car, wrecker", + "865": "toyshop", + "866": "tractor", + "867": "trailer truck, tractor trailer, trucking rig, rig, articulated lorry, semi", + "868": "tray", + "869": "trench coat", + "870": "tricycle, trike, velocipede", + "871": "trimaran", + "872": "tripod", + "873": "triumphal arch", + "874": "trolleybus, trolley coach, trackless trolley", + "875": "trombone", + "876": "tub, vat", + "877": "turnstile", + "878": "typewriter keyboard", + "879": "umbrella", + "880": "unicycle, monocycle", + "881": "upright, upright piano", + "882": "vacuum, vacuum cleaner", + "883": "vase", + "884": "vault", + "885": "velvet", + "886": "vending machine", + "887": "vestment", + "888": "viaduct", + "889": "violin, fiddle", + "890": "volleyball", + "891": "waffle iron", + "892": "wall clock", + "893": "wallet, billfold, notecase, pocketbook", + "894": "wardrobe, closet, press", + "895": "warplane, military plane", + "896": "washbasin, handbasin, washbowl, lavabo, wash-hand basin", + "897": "washer, automatic washer, washing machine", + "898": "water bottle", + "899": "water jug", + "900": "water tower", + "901": "whiskey jug", + "902": "whistle", + "903": "wig", + "904": "window screen", + "905": "window shade", + "906": "Windsor tie", + "907": "wine bottle", + "908": "wing", + "909": "wok", + "910": "wooden spoon", + "911": "wool, woolen, woollen", + "912": "worm fence, snake fence, snake-rail fence, Virginia fence", + "913": "wreck", + "914": "yawl", + "915": "yurt", + "916": "web site, website, internet site, site", + "917": "comic book", + "918": "crossword puzzle, crossword", + "919": "street sign", + "920": "traffic light, traffic signal, stoplight", + "921": "book jacket, dust cover, dust jacket, dust wrapper", + "922": "menu", + "923": "plate", + "924": "guacamole", + "925": "consomme", + "926": "hot pot, hotpot", + "927": "trifle", + "928": "ice cream, icecream", + "929": "ice lolly, lolly, lollipop, popsicle", + "930": "French loaf", + "931": "bagel, beigel", + "932": "pretzel", + "933": "cheeseburger", + "934": "hotdog, hot dog, red hot", + "935": "mashed potato", + "936": "head cabbage", + "937": "broccoli", + "938": "cauliflower", + "939": "zucchini, courgette", + "940": "spaghetti squash", + "941": "acorn squash", + "942": "butternut squash", + "943": "cucumber, cuke", + "944": "artichoke, globe artichoke", + "945": "bell pepper", + "946": "cardoon", + "947": "mushroom", + "948": "Granny Smith", + "949": "strawberry", + "950": "orange", + "951": "lemon", + "952": "fig", + "953": "pineapple, ananas", + "954": "banana", + "955": "jackfruit, jak, jack", + "956": "custard apple", + "957": "pomegranate", + "958": "hay", + "959": "carbonara", + "960": "chocolate sauce, chocolate syrup", + "961": "dough", + "962": "meat loaf, meatloaf", + "963": "pizza, pizza pie", + "964": "potpie", + "965": "burrito", + "966": "red wine", + "967": "espresso", + "968": "cup", + "969": "eggnog", + "970": "alp", + "971": "bubble", + "972": "cliff, drop, drop-off", + "973": "coral reef", + "974": "geyser", + "975": "lakeside, lakeshore", + "976": "promontory, headland, head, foreland", + "977": "sandbar, sand bar", + "978": "seashore, coast, seacoast, sea-coast", + "979": "valley, vale", + "980": "volcano", + "981": "ballplayer, baseball player", + "982": "groom, bridegroom", + "983": "scuba diver", + "984": "rapeseed", + "985": "daisy", + "986": "yellow lady's slipper, yellow lady-slipper, Cypripedium calceolus, Cypripedium parviflorum", + "987": "corn", + "988": "acorn", + "989": "hip, rose hip, rosehip", + "990": "buckeye, horse chestnut, conker", + "991": "coral fungus", + "992": "agaric", + "993": "gyromitra", + "994": "stinkhorn, carrion fungus", + "995": "earthstar", + "996": "hen-of-the-woods, hen of the woods, Polyporus frondosus, Grifola frondosa", + "997": "bolete", + "998": "ear, spike, capitulum", + "999": "toilet tissue, toilet paper, bathroom tissue" + }, + "label2id": { + "Afghan hound, Afghan": 160, + "African chameleon, Chamaeleo chamaeleon": 47, + "African crocodile, Nile crocodile, Crocodylus niloticus": 49, + "African elephant, Loxodonta africana": 386, + "African grey, African gray, Psittacus erithacus": 87, + "African hunting dog, hyena dog, Cape hunting dog, Lycaon pictus": 275, + "Airedale, Airedale terrier": 191, + "American Staffordshire terrier, Staffordshire terrier, American pit bull terrier, pit bull terrier": 180, + "American alligator, Alligator mississipiensis": 50, + "American black bear, black bear, Ursus americanus, Euarctos americanus": 295, + "American chameleon, anole, Anolis carolinensis": 40, + "American coot, marsh hen, mud hen, water hen, Fulica americana": 137, + "American egret, great white heron, Egretta albus": 132, + "American lobster, Northern lobster, Maine lobster, Homarus americanus": 122, + "Angora, Angora rabbit": 332, + "Appenzeller": 240, + "Arabian camel, dromedary, Camelus dromedarius": 354, + "Arctic fox, white fox, Alopex lagopus": 279, + "Australian terrier": 193, + "Band Aid": 419, + "Bedlington terrier": 181, + "Bernese mountain dog": 239, + "Blenheim spaniel": 156, + "Border collie": 232, + "Border terrier": 182, + "Boston bull, Boston terrier": 195, + "Bouvier des Flandres, Bouviers des Flandres": 233, + "Brabancon griffon": 262, + "Brittany spaniel": 215, + "CD player": 485, + "Cardigan, Cardigan Welsh corgi": 264, + "Chesapeake Bay retriever": 209, + "Chihuahua": 151, + "Christmas stocking": 496, + "Crock Pot": 521, + "Dandie Dinmont, Dandie Dinmont terrier": 194, + "Doberman, Doberman pinscher": 236, + "Dungeness crab, Cancer magister": 118, + "Dutch oven": 544, + "Egyptian cat": 285, + "English foxhound": 167, + "English setter": 212, + "English springer, English springer spaniel": 217, + "EntleBucher": 241, + "Eskimo dog, husky": 248, + "European fire salamander, Salamandra salamandra": 25, + "European gallinule, Porphyrio porphyrio": 136, + "French bulldog": 245, + "French horn, horn": 566, + "French loaf": 930, + "German shepherd, German shepherd dog, German police dog, alsatian": 235, + "German short-haired pointer": 210, + "Gila monster, Heloderma suspectum": 45, + "Gordon setter": 214, + "Granny Smith": 948, + "Great Dane": 246, + "Great Pyrenees": 257, + "Greater Swiss Mountain dog": 238, + "Ibizan hound, Ibizan Podenco": 173, + "Indian cobra, Naja naja": 63, + "Indian elephant, Elephas maximus": 385, + "Irish setter, red setter": 213, + "Irish terrier": 184, + "Irish water spaniel": 221, + "Irish wolfhound": 170, + "Italian greyhound": 171, + "Japanese spaniel": 152, + "Kerry blue terrier": 183, + "Komodo dragon, Komodo lizard, dragon lizard, giant lizard, Varanus komodoensis": 48, + "Labrador retriever": 208, + "Lakeland terrier": 189, + "Leonberg": 255, + "Lhasa, Lhasa apso": 204, + "Loafer": 630, + "Madagascar cat, ring-tailed lemur, Lemur catta": 383, + "Maltese dog, Maltese terrier, Maltese": 153, + "Mexican hairless": 268, + "Model T": 661, + "Newfoundland, Newfoundland dog": 256, + "Norfolk terrier": 185, + "Norwegian elkhound, elkhound": 174, + "Norwich terrier": 186, + "Old English sheepdog, bobtail": 229, + "Pekinese, Pekingese, Peke": 154, + "Pembroke, Pembroke Welsh corgi": 263, + "Persian cat": 283, + "Petri dish": 712, + "Polaroid camera, Polaroid Land camera": 732, + "Pomeranian": 259, + "Rhodesian ridgeback": 159, + "Rottweiler": 234, + "Saint Bernard, St Bernard": 247, + "Saluki, gazelle hound": 176, + "Samoyed, Samoyede": 258, + "Scotch terrier, Scottish terrier, Scottie": 199, + "Scottish deerhound, deerhound": 177, + "Sealyham terrier, Sealyham": 190, + "Shetland sheepdog, Shetland sheep dog, Shetland": 230, + "Shih-Tzu": 155, + "Siamese cat, Siamese": 284, + "Siberian husky": 250, + "Staffordshire bullterrier, Staffordshire bull terrier": 179, + "Sussex spaniel": 220, + "Tibetan mastiff": 244, + "Tibetan terrier, chrysanthemum dog": 200, + "Walker hound, Walker foxhound": 166, + "Weimaraner": 178, + "Welsh springer spaniel": 218, + "West Highland white terrier": 203, + "Windsor tie": 906, + "Yorkshire terrier": 187, + "abacus": 398, + "abaya": 399, + "academic gown, academic robe, judge's robe": 400, + "accordion, piano accordion, squeeze box": 401, + "acorn": 988, + "acorn squash": 941, + "acoustic guitar": 402, + "admiral": 321, + "affenpinscher, monkey pinscher, monkey dog": 252, + "agama": 42, + "agaric": 992, + "aircraft carrier, carrier, flattop, attack aircraft carrier": 403, + "airliner": 404, + "airship, dirigible": 405, + "albatross, mollymawk": 146, + "alligator lizard": 44, + "alp": 970, + "altar": 406, + "ambulance": 407, + "amphibian, amphibious vehicle": 408, + "analog clock": 409, + "anemone fish": 393, + "ant, emmet, pismire": 310, + "apiary, bee house": 410, + "apron": 411, + "armadillo": 363, + "artichoke, globe artichoke": 944, + "ashcan, trash can, garbage can, wastebin, ash bin, ash-bin, ashbin, dustbin, trash barrel, trash bin": 412, + "assault rifle, assault gun": 413, + "axolotl, mud puppy, Ambystoma mexicanum": 29, + "baboon": 372, + "backpack, back pack, knapsack, packsack, rucksack, haversack": 414, + "badger": 362, + "bagel, beigel": 931, + "bakery, bakeshop, bakehouse": 415, + "balance beam, beam": 416, + "bald eagle, American eagle, Haliaeetus leucocephalus": 22, + "balloon": 417, + "ballplayer, baseball player": 981, + "ballpoint, ballpoint pen, ballpen, Biro": 418, + "banana": 954, + "banded gecko": 38, + "banjo": 420, + "bannister, banister, balustrade, balusters, handrail": 421, + "barbell": 422, + "barber chair": 423, + "barbershop": 424, + "barn": 425, + "barn spider, Araneus cavaticus": 73, + "barometer": 426, + "barracouta, snoek": 389, + "barrel, cask": 427, + "barrow, garden cart, lawn cart, wheelbarrow": 428, + "baseball": 429, + "basenji": 253, + "basketball": 430, + "basset, basset hound": 161, + "bassinet": 431, + "bassoon": 432, + "bath towel": 434, + "bathing cap, swimming cap": 433, + "bathtub, bathing tub, bath, tub": 435, + "beach wagon, station wagon, wagon, estate car, beach waggon, station waggon, waggon": 436, + "beacon, lighthouse, beacon light, pharos": 437, + "beagle": 162, + "beaker": 438, + "bearskin, busby, shako": 439, + "beaver": 337, + "bee": 309, + "bee eater": 92, + "beer bottle": 440, + "beer glass": 441, + "bell cote, bell cot": 442, + "bell pepper": 945, + "bib": 443, + "bicycle-built-for-two, tandem bicycle, tandem": 444, + "bighorn, bighorn sheep, cimarron, Rocky Mountain bighorn, Rocky Mountain sheep, Ovis canadensis": 349, + "bikini, two-piece": 445, + "binder, ring-binder": 446, + "binoculars, field glasses, opera glasses": 447, + "birdhouse": 448, + "bison": 347, + "bittern": 133, + "black and gold garden spider, Argiope aurantia": 72, + "black grouse": 80, + "black stork, Ciconia nigra": 128, + "black swan, Cygnus atratus": 100, + "black widow, Latrodectus mactans": 75, + "black-and-tan coonhound": 165, + "black-footed ferret, ferret, Mustela nigripes": 359, + "bloodhound, sleuthhound": 163, + "bluetick": 164, + "boa constrictor, Constrictor constrictor": 61, + "boathouse": 449, + "bobsled, bobsleigh, bob": 450, + "bolete": 997, + "bolo tie, bolo, bola tie, bola": 451, + "bonnet, poke bonnet": 452, + "book jacket, dust cover, dust jacket, dust wrapper": 921, + "bookcase": 453, + "bookshop, bookstore, bookstall": 454, + "borzoi, Russian wolfhound": 169, + "bottlecap": 455, + "bow": 456, + "bow tie, bow-tie, bowtie": 457, + "box turtle, box tortoise": 37, + "boxer": 242, + "brain coral": 109, + "brambling, Fringilla montifringilla": 10, + "brass, memorial tablet, plaque": 458, + "brassiere, bra, bandeau": 459, + "breakwater, groin, groyne, mole, bulwark, seawall, jetty": 460, + "breastplate, aegis, egis": 461, + "briard": 226, + "broccoli": 937, + "broom": 462, + "brown bear, bruin, Ursus arctos": 294, + "bubble": 971, + "bucket, pail": 463, + "buckeye, horse chestnut, conker": 990, + "buckle": 464, + "bulbul": 16, + "bull mastiff": 243, + "bullet train, bullet": 466, + "bulletproof vest": 465, + "bullfrog, Rana catesbeiana": 30, + "burrito": 965, + "bustard": 138, + "butcher shop, meat market": 467, + "butternut squash": 942, + "cab, hack, taxi, taxicab": 468, + "cabbage butterfly": 324, + "cairn, cairn terrier": 192, + "caldron, cauldron": 469, + "can opener, tin opener": 473, + "candle, taper, wax light": 470, + "cannon": 471, + "canoe": 472, + "capuchin, ringtail, Cebus capucinus": 378, + "car mirror": 475, + "car wheel": 479, + "carbonara": 959, + "cardigan": 474, + "cardoon": 946, + "carousel, carrousel, merry-go-round, roundabout, whirligig": 476, + "carpenter's kit, tool kit": 477, + "carton": 478, + "cash machine, cash dispenser, automated teller machine, automatic teller machine, automated teller, automatic teller, ATM": 480, + "cassette": 481, + "cassette player": 482, + "castle": 483, + "catamaran": 484, + "cauliflower": 938, + "cello, violoncello": 486, + "cellular telephone, cellular phone, cellphone, cell, mobile phone": 487, + "centipede": 79, + "chain": 488, + "chain mail, ring mail, mail, chain armor, chain armour, ring armor, ring armour": 490, + "chain saw, chainsaw": 491, + "chainlink fence": 489, + "chambered nautilus, pearly nautilus, nautilus": 117, + "cheeseburger": 933, + "cheetah, chetah, Acinonyx jubatus": 293, + "chest": 492, + "chickadee": 19, + "chiffonier, commode": 493, + "chime, bell, gong": 494, + "chimpanzee, chimp, Pan troglodytes": 367, + "china cabinet, china closet": 495, + "chiton, coat-of-mail shell, sea cradle, polyplacophore": 116, + "chocolate sauce, chocolate syrup": 960, + "chow, chow chow": 260, + "church, church building": 497, + "cicada, cicala": 316, + "cinema, movie theater, movie theatre, movie house, picture palace": 498, + "cleaver, meat cleaver, chopper": 499, + "cliff dwelling": 500, + "cliff, drop, drop-off": 972, + "cloak": 501, + "clog, geta, patten, sabot": 502, + "clumber, clumber spaniel": 216, + "cock": 7, + "cocker spaniel, English cocker spaniel, cocker": 219, + "cockroach, roach": 314, + "cocktail shaker": 503, + "coffee mug": 504, + "coffeepot": 505, + "coho, cohoe, coho salmon, blue jack, silver salmon, Oncorhynchus kisutch": 391, + "coil, spiral, volute, whorl, helix": 506, + "collie": 231, + "colobus, colobus monkey": 375, + "combination lock": 507, + "comic book": 917, + "common iguana, iguana, Iguana iguana": 39, + "common newt, Triturus vulgaris": 26, + "computer keyboard, keypad": 508, + "conch": 112, + "confectionery, confectionary, candy store": 509, + "consomme": 925, + "container ship, containership, container vessel": 510, + "convertible": 511, + "coral fungus": 991, + "coral reef": 973, + "corkscrew, bottle screw": 512, + "corn": 987, + "cornet, horn, trumpet, trump": 513, + "coucal": 91, + "cougar, puma, catamount, mountain lion, painter, panther, Felis concolor": 286, + "cowboy boot": 514, + "cowboy hat, ten-gallon hat": 515, + "coyote, prairie wolf, brush wolf, Canis latrans": 272, + "cradle": 516, + "crane": 517, + "crash helmet": 518, + "crate": 519, + "crayfish, crawfish, crawdad, crawdaddy": 124, + "crib, cot": 520, + "cricket": 312, + "croquet ball": 522, + "crossword puzzle, crossword": 918, + "crutch": 523, + "cucumber, cuke": 943, + "cuirass": 524, + "cup": 968, + "curly-coated retriever": 206, + "custard apple": 956, + "daisy": 985, + "dalmatian, coach dog, carriage dog": 251, + "dam, dike, dyke": 525, + "damselfly": 320, + "desk": 526, + "desktop computer": 527, + "dhole, Cuon alpinus": 274, + "dial telephone, dial phone": 528, + "diamondback, diamondback rattlesnake, Crotalus adamanteus": 67, + "diaper, nappy, napkin": 529, + "digital clock": 530, + "digital watch": 531, + "dingo, warrigal, warragal, Canis dingo": 273, + "dining table, board": 532, + "dishrag, dishcloth": 533, + "dishwasher, dish washer, dishwashing machine": 534, + "disk brake, disc brake": 535, + "dock, dockage, docking facility": 536, + "dogsled, dog sled, dog sleigh": 537, + "dome": 538, + "doormat, welcome mat": 539, + "dough": 961, + "dowitcher": 142, + "dragonfly, darning needle, devil's darning needle, sewing needle, snake feeder, snake doctor, mosquito hawk, skeeter hawk": 319, + "drake": 97, + "drilling platform, offshore rig": 540, + "drum, membranophone, tympan": 541, + "drumstick": 542, + "dugong, Dugong dugon": 149, + "dumbbell": 543, + "dung beetle": 305, + "ear, spike, capitulum": 998, + "earthstar": 995, + "echidna, spiny anteater, anteater": 102, + "eel": 390, + "eft": 27, + "eggnog": 969, + "electric fan, blower": 545, + "electric guitar": 546, + "electric locomotive": 547, + "electric ray, crampfish, numbfish, torpedo": 5, + "entertainment center": 548, + "envelope": 549, + "espresso": 967, + "espresso maker": 550, + "face powder": 551, + "feather boa, boa": 552, + "fiddler crab": 120, + "fig": 952, + "file, file cabinet, filing cabinet": 553, + "fire engine, fire truck": 555, + "fire screen, fireguard": 556, + "fireboat": 554, + "flagpole, flagstaff": 557, + "flamingo": 130, + "flat-coated retriever": 205, + "flatworm, platyhelminth": 110, + "flute, transverse flute": 558, + "fly": 308, + "folding chair": 559, + "football helmet": 560, + "forklift": 561, + "fountain": 562, + "fountain pen": 563, + "four-poster": 564, + "fox squirrel, eastern fox squirrel, Sciurus niger": 335, + "freight car": 565, + "frilled lizard, Chlamydosaurus kingi": 43, + "frying pan, frypan, skillet": 567, + "fur coat": 568, + "gar, garfish, garpike, billfish, Lepisosteus osseus": 395, + "garbage truck, dustcart": 569, + "garden spider, Aranea diademata": 74, + "garter snake, grass snake": 57, + "gas pump, gasoline pump, petrol pump, island dispenser": 571, + "gasmask, respirator, gas helmet": 570, + "gazelle": 353, + "geyser": 974, + "giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca": 388, + "giant schnauzer": 197, + "gibbon, Hylobates lar": 368, + "go-kart": 573, + "goblet": 572, + "golden retriever": 207, + "goldfinch, Carduelis carduelis": 11, + "goldfish, Carassius auratus": 1, + "golf ball": 574, + "golfcart, golf cart": 575, + "gondola": 576, + "gong, tam-tam": 577, + "goose": 99, + "gorilla, Gorilla gorilla": 366, + "gown": 578, + "grand piano, grand": 579, + "grasshopper, hopper": 311, + "great grey owl, great gray owl, Strix nebulosa": 24, + "great white shark, white shark, man-eater, man-eating shark, Carcharodon carcharias": 2, + "green lizard, Lacerta viridis": 46, + "green mamba": 64, + "green snake, grass snake": 55, + "greenhouse, nursery, glasshouse": 580, + "grey fox, gray fox, Urocyon cinereoargenteus": 280, + "grey whale, gray whale, devilfish, Eschrichtius gibbosus, Eschrichtius robustus": 147, + "grille, radiator grille": 581, + "grocery store, grocery, food market, market": 582, + "groenendael": 224, + "groom, bridegroom": 982, + "ground beetle, carabid beetle": 302, + "guacamole": 924, + "guenon, guenon monkey": 370, + "guillotine": 583, + "guinea pig, Cavia cobaya": 338, + "gyromitra": 993, + "hair slide": 584, + "hair spray": 585, + "half track": 586, + "hammer": 587, + "hammerhead, hammerhead shark": 4, + "hamper": 588, + "hamster": 333, + "hand blower, blow dryer, blow drier, hair dryer, hair drier": 589, + "hand-held computer, hand-held microcomputer": 590, + "handkerchief, hankie, hanky, hankey": 591, + "hard disc, hard disk, fixed disk": 592, + "hare": 331, + "harmonica, mouth organ, harp, mouth harp": 593, + "harp": 594, + "hartebeest": 351, + "harvester, reaper": 595, + "harvestman, daddy longlegs, Phalangium opilio": 70, + "hatchet": 596, + "hay": 958, + "head cabbage": 936, + "hen": 8, + "hen-of-the-woods, hen of the woods, Polyporus frondosus, Grifola frondosa": 996, + "hermit crab": 125, + "hip, rose hip, rosehip": 989, + "hippopotamus, hippo, river horse, Hippopotamus amphibius": 344, + "hog, pig, grunter, squealer, Sus scrofa": 341, + "hognose snake, puff adder, sand viper": 54, + "holster": 597, + "home theater, home theatre": 598, + "honeycomb": 599, + "hook, claw": 600, + "hoopskirt, crinoline": 601, + "horizontal bar, high bar": 602, + "hornbill": 93, + "horned viper, cerastes, sand viper, horned asp, Cerastes cornutus": 66, + "horse cart, horse-cart": 603, + "hot pot, hotpot": 926, + "hotdog, hot dog, red hot": 934, + "hourglass": 604, + "house finch, linnet, Carpodacus mexicanus": 12, + "howler monkey, howler": 379, + "hummingbird": 94, + "hyena, hyaena": 276, + "iPod": 605, + "ibex, Capra ibex": 350, + "ice bear, polar bear, Ursus Maritimus, Thalarctos maritimus": 296, + "ice cream, icecream": 928, + "ice lolly, lolly, lollipop, popsicle": 929, + "impala, Aepyceros melampus": 352, + "indigo bunting, indigo finch, indigo bird, Passerina cyanea": 14, + "indri, indris, Indri indri, Indri brevicaudatus": 384, + "iron, smoothing iron": 606, + "isopod": 126, + "jacamar": 95, + "jack-o'-lantern": 607, + "jackfruit, jak, jack": 955, + "jaguar, panther, Panthera onca, Felis onca": 290, + "jay": 17, + "jean, blue jean, denim": 608, + "jeep, landrover": 609, + "jellyfish": 107, + "jersey, T-shirt, tee shirt": 610, + "jigsaw puzzle": 611, + "jinrikisha, ricksha, rickshaw": 612, + "joystick": 613, + "junco, snowbird": 13, + "keeshond": 261, + "kelpie": 227, + "killer whale, killer, orca, grampus, sea wolf, Orcinus orca": 148, + "kimono": 614, + "king crab, Alaska crab, Alaskan king crab, Alaska king crab, Paralithodes camtschatica": 121, + "king penguin, Aptenodytes patagonica": 145, + "king snake, kingsnake": 56, + "kit fox, Vulpes macrotis": 278, + "kite": 21, + "knee pad": 615, + "knot": 616, + "koala, koala bear, kangaroo bear, native bear, Phascolarctos cinereus": 105, + "komondor": 228, + "kuvasz": 222, + "lab coat, laboratory coat": 617, + "lacewing, lacewing fly": 318, + "ladle": 618, + "ladybug, ladybeetle, lady beetle, ladybird, ladybird beetle": 301, + "lakeside, lakeshore": 975, + "lampshade, lamp shade": 619, + "langur": 374, + "laptop, laptop computer": 620, + "lawn mower, mower": 621, + "leaf beetle, chrysomelid": 304, + "leafhopper": 317, + "leatherback turtle, leatherback, leathery turtle, Dermochelys coriacea": 34, + "lemon": 951, + "lens cap, lens cover": 622, + "leopard, Panthera pardus": 288, + "lesser panda, red panda, panda, bear cat, cat bear, Ailurus fulgens": 387, + "letter opener, paper knife, paperknife": 623, + "library": 624, + "lifeboat": 625, + "lighter, light, igniter, ignitor": 626, + "limousine, limo": 627, + "limpkin, Aramus pictus": 135, + "liner, ocean liner": 628, + "lion, king of beasts, Panthera leo": 291, + "lionfish": 396, + "lipstick, lip rouge": 629, + "little blue heron, Egretta caerulea": 131, + "llama": 355, + "loggerhead, loggerhead turtle, Caretta caretta": 33, + "long-horned beetle, longicorn, longicorn beetle": 303, + "lorikeet": 90, + "lotion": 631, + "loudspeaker, speaker, speaker unit, loudspeaker system, speaker system": 632, + "loupe, jeweler's loupe": 633, + "lumbermill, sawmill": 634, + "lycaenid, lycaenid butterfly": 326, + "lynx, catamount": 287, + "macaque": 373, + "macaw": 88, + "magnetic compass": 635, + "magpie": 18, + "mailbag, postbag": 636, + "mailbox, letter box": 637, + "maillot": 638, + "maillot, tank suit": 639, + "malamute, malemute, Alaskan malamute": 249, + "malinois": 225, + "manhole cover": 640, + "mantis, mantid": 315, + "maraca": 641, + "marimba, xylophone": 642, + "marmoset": 377, + "marmot": 336, + "mashed potato": 935, + "mask": 643, + "matchstick": 644, + "maypole": 645, + "maze, labyrinth": 646, + "measuring cup": 647, + "meat loaf, meatloaf": 962, + "medicine chest, medicine cabinet": 648, + "meerkat, mierkat": 299, + "megalith, megalithic structure": 649, + "menu": 922, + "microphone, mike": 650, + "microwave, microwave oven": 651, + "military uniform": 652, + "milk can": 653, + "miniature pinscher": 237, + "miniature poodle": 266, + "miniature schnauzer": 196, + "minibus": 654, + "miniskirt, mini": 655, + "minivan": 656, + "mink": 357, + "missile": 657, + "mitten": 658, + "mixing bowl": 659, + "mobile home, manufactured home": 660, + "modem": 662, + "monarch, monarch butterfly, milkweed butterfly, Danaus plexippus": 323, + "monastery": 663, + "mongoose": 298, + "monitor": 664, + "moped": 665, + "mortar": 666, + "mortarboard": 667, + "mosque": 668, + "mosquito net": 669, + "motor scooter, scooter": 670, + "mountain bike, all-terrain bike, off-roader": 671, + "mountain tent": 672, + "mouse, computer mouse": 673, + "mousetrap": 674, + "moving van": 675, + "mud turtle": 35, + "mushroom": 947, + "muzzle": 676, + "nail": 677, + "neck brace": 678, + "necklace": 679, + "nematode, nematode worm, roundworm": 111, + "night snake, Hypsiglena torquata": 60, + "nipple": 680, + "notebook, notebook computer": 681, + "obelisk": 682, + "oboe, hautboy, hautbois": 683, + "ocarina, sweet potato": 684, + "odometer, hodometer, mileometer, milometer": 685, + "oil filter": 686, + "orange": 950, + "orangutan, orang, orangutang, Pongo pygmaeus": 365, + "organ, pipe organ": 687, + "oscilloscope, scope, cathode-ray oscilloscope, CRO": 688, + "ostrich, Struthio camelus": 9, + "otter": 360, + "otterhound, otter hound": 175, + "overskirt": 689, + "ox": 345, + "oxcart": 690, + "oxygen mask": 691, + "oystercatcher, oyster catcher": 143, + "packet": 692, + "paddle, boat paddle": 693, + "paddlewheel, paddle wheel": 694, + "padlock": 695, + "paintbrush": 696, + "pajama, pyjama, pj's, jammies": 697, + "palace": 698, + "panpipe, pandean pipe, syrinx": 699, + "paper towel": 700, + "papillon": 157, + "parachute, chute": 701, + "parallel bars, bars": 702, + "park bench": 703, + "parking meter": 704, + "partridge": 86, + "passenger car, coach, carriage": 705, + "patas, hussar monkey, Erythrocebus patas": 371, + "patio, terrace": 706, + "pay-phone, pay-station": 707, + "peacock": 84, + "pedestal, plinth, footstall": 708, + "pelican": 144, + "pencil box, pencil case": 709, + "pencil sharpener": 710, + "perfume, essence": 711, + "photocopier": 713, + "pick, plectrum, plectron": 714, + "pickelhaube": 715, + "picket fence, paling": 716, + "pickup, pickup truck": 717, + "pier": 718, + "piggy bank, penny bank": 719, + "pill bottle": 720, + "pillow": 721, + "pineapple, ananas": 953, + "ping-pong ball": 722, + "pinwheel": 723, + "pirate, pirate ship": 724, + "pitcher, ewer": 725, + "pizza, pizza pie": 963, + "plane, carpenter's plane, woodworking plane": 726, + "planetarium": 727, + "plastic bag": 728, + "plate": 923, + "plate rack": 729, + "platypus, duckbill, duckbilled platypus, duck-billed platypus, Ornithorhynchus anatinus": 103, + "plow, plough": 730, + "plunger, plumber's helper": 731, + "pole": 733, + "polecat, fitch, foulmart, foumart, Mustela putorius": 358, + "police van, police wagon, paddy wagon, patrol wagon, wagon, black Maria": 734, + "pomegranate": 957, + "poncho": 735, + "pool table, billiard table, snooker table": 736, + "pop bottle, soda bottle": 737, + "porcupine, hedgehog": 334, + "pot, flowerpot": 738, + "potpie": 964, + "potter's wheel": 739, + "power drill": 740, + "prairie chicken, prairie grouse, prairie fowl": 83, + "prayer rug, prayer mat": 741, + "pretzel": 932, + "printer": 742, + "prison, prison house": 743, + "proboscis monkey, Nasalis larvatus": 376, + "projectile, missile": 744, + "projector": 745, + "promontory, headland, head, foreland": 976, + "ptarmigan": 81, + "puck, hockey puck": 746, + "puffer, pufferfish, blowfish, globefish": 397, + "pug, pug-dog": 254, + "punching bag, punch bag, punching ball, punchball": 747, + "purse": 748, + "quail": 85, + "quill, quill pen": 749, + "quilt, comforter, comfort, puff": 750, + "racer, race car, racing car": 751, + "racket, racquet": 752, + "radiator": 753, + "radio telescope, radio reflector": 755, + "radio, wireless": 754, + "rain barrel": 756, + "ram, tup": 348, + "rapeseed": 984, + "recreational vehicle, RV, R.V.": 757, + "red fox, Vulpes vulpes": 277, + "red wine": 966, + "red wolf, maned wolf, Canis rufus, Canis niger": 271, + "red-backed sandpiper, dunlin, Erolia alpina": 140, + "red-breasted merganser, Mergus serrator": 98, + "redbone": 168, + "redshank, Tringa totanus": 141, + "reel": 758, + "reflex camera": 759, + "refrigerator, icebox": 760, + "remote control, remote": 761, + "restaurant, eating house, eating place, eatery": 762, + "revolver, six-gun, six-shooter": 763, + "rhinoceros beetle": 306, + "rifle": 764, + "ringlet, ringlet butterfly": 322, + "ringneck snake, ring-necked snake, ring snake": 53, + "robin, American robin, Turdus migratorius": 15, + "rock beauty, Holocanthus tricolor": 392, + "rock crab, Cancer irroratus": 119, + "rock python, rock snake, Python sebae": 62, + "rocking chair, rocker": 765, + "rotisserie": 766, + "rubber eraser, rubber, pencil eraser": 767, + "ruddy turnstone, Arenaria interpres": 139, + "ruffed grouse, partridge, Bonasa umbellus": 82, + "rugby ball": 768, + "rule, ruler": 769, + "running shoe": 770, + "safe": 771, + "safety pin": 772, + "saltshaker, salt shaker": 773, + "sandal": 774, + "sandbar, sand bar": 977, + "sarong": 775, + "sax, saxophone": 776, + "scabbard": 777, + "scale, weighing machine": 778, + "schipperke": 223, + "school bus": 779, + "schooner": 780, + "scoreboard": 781, + "scorpion": 71, + "screen, CRT screen": 782, + "screw": 783, + "screwdriver": 784, + "scuba diver": 983, + "sea anemone, anemone": 108, + "sea cucumber, holothurian": 329, + "sea lion": 150, + "sea slug, nudibranch": 115, + "sea snake": 65, + "sea urchin": 328, + "seashore, coast, seacoast, sea-coast": 978, + "seat belt, seatbelt": 785, + "sewing machine": 786, + "shield, buckler": 787, + "shoe shop, shoe-shop, shoe store": 788, + "shoji": 789, + "shopping basket": 790, + "shopping cart": 791, + "shovel": 792, + "shower cap": 793, + "shower curtain": 794, + "siamang, Hylobates syndactylus, Symphalangus syndactylus": 369, + "sidewinder, horned rattlesnake, Crotalus cerastes": 68, + "silky terrier, Sydney silky": 201, + "ski": 795, + "ski mask": 796, + "skunk, polecat, wood pussy": 361, + "sleeping bag": 797, + "slide rule, slipstick": 798, + "sliding door": 799, + "slot, one-armed bandit": 800, + "sloth bear, Melursus ursinus, Ursus ursinus": 297, + "slug": 114, + "snail": 113, + "snorkel": 801, + "snow leopard, ounce, Panthera uncia": 289, + "snowmobile": 802, + "snowplow, snowplough": 803, + "soap dispenser": 804, + "soccer ball": 805, + "sock": 806, + "soft-coated wheaten terrier": 202, + "solar dish, solar collector, solar furnace": 807, + "sombrero": 808, + "sorrel": 339, + "soup bowl": 809, + "space bar": 810, + "space heater": 811, + "space shuttle": 812, + "spaghetti squash": 940, + "spatula": 813, + "speedboat": 814, + "spider monkey, Ateles geoffroyi": 381, + "spider web, spider's web": 815, + "spindle": 816, + "spiny lobster, langouste, rock lobster, crawfish, crayfish, sea crawfish": 123, + "spoonbill": 129, + "sports car, sport car": 817, + "spotlight, spot": 818, + "spotted salamander, Ambystoma maculatum": 28, + "squirrel monkey, Saimiri sciureus": 382, + "stage": 819, + "standard poodle": 267, + "standard schnauzer": 198, + "starfish, sea star": 327, + "steam locomotive": 820, + "steel arch bridge": 821, + "steel drum": 822, + "stethoscope": 823, + "stingray": 6, + "stinkhorn, carrion fungus": 994, + "stole": 824, + "stone wall": 825, + "stopwatch, stop watch": 826, + "stove": 827, + "strainer": 828, + "strawberry": 949, + "street sign": 919, + "streetcar, tram, tramcar, trolley, trolley car": 829, + "stretcher": 830, + "studio couch, day bed": 831, + "stupa, tope": 832, + "sturgeon": 394, + "submarine, pigboat, sub, U-boat": 833, + "suit, suit of clothes": 834, + "sulphur butterfly, sulfur butterfly": 325, + "sulphur-crested cockatoo, Kakatoe galerita, Cacatua galerita": 89, + "sundial": 835, + "sunglass": 836, + "sunglasses, dark glasses, shades": 837, + "sunscreen, sunblock, sun blocker": 838, + "suspension bridge": 839, + "swab, swob, mop": 840, + "sweatshirt": 841, + "swimming trunks, bathing trunks": 842, + "swing": 843, + "switch, electric switch, electrical switch": 844, + "syringe": 845, + "tabby, tabby cat": 281, + "table lamp": 846, + "tailed frog, bell toad, ribbed toad, tailed toad, Ascaphus trui": 32, + "tank, army tank, armored combat vehicle, armoured combat vehicle": 847, + "tape player": 848, + "tarantula": 76, + "teapot": 849, + "teddy, teddy bear": 850, + "television, television system": 851, + "tench, Tinca tinca": 0, + "tennis ball": 852, + "terrapin": 36, + "thatch, thatched roof": 853, + "theater curtain, theatre curtain": 854, + "thimble": 855, + "three-toed sloth, ai, Bradypus tridactylus": 364, + "thresher, thrasher, threshing machine": 856, + "throne": 857, + "thunder snake, worm snake, Carphophis amoenus": 52, + "tick": 78, + "tiger beetle": 300, + "tiger cat": 282, + "tiger shark, Galeocerdo cuvieri": 3, + "tiger, Panthera tigris": 292, + "tile roof": 858, + "timber wolf, grey wolf, gray wolf, Canis lupus": 269, + "titi, titi monkey": 380, + "toaster": 859, + "tobacco shop, tobacconist shop, tobacconist": 860, + "toilet seat": 861, + "toilet tissue, toilet paper, bathroom tissue": 999, + "torch": 862, + "totem pole": 863, + "toucan": 96, + "tow truck, tow car, wrecker": 864, + "toy poodle": 265, + "toy terrier": 158, + "toyshop": 865, + "tractor": 866, + "traffic light, traffic signal, stoplight": 920, + "trailer truck, tractor trailer, trucking rig, rig, articulated lorry, semi": 867, + "tray": 868, + "tree frog, tree-frog": 31, + "trench coat": 869, + "triceratops": 51, + "tricycle, trike, velocipede": 870, + "trifle": 927, + "trilobite": 69, + "trimaran": 871, + "tripod": 872, + "triumphal arch": 873, + "trolleybus, trolley coach, trackless trolley": 874, + "trombone": 875, + "tub, vat": 876, + "turnstile": 877, + "tusker": 101, + "typewriter keyboard": 878, + "umbrella": 879, + "unicycle, monocycle": 880, + "upright, upright piano": 881, + "vacuum, vacuum cleaner": 882, + "valley, vale": 979, + "vase": 883, + "vault": 884, + "velvet": 885, + "vending machine": 886, + "vestment": 887, + "viaduct": 888, + "vine snake": 59, + "violin, fiddle": 889, + "vizsla, Hungarian pointer": 211, + "volcano": 980, + "volleyball": 890, + "vulture": 23, + "waffle iron": 891, + "walking stick, walkingstick, stick insect": 313, + "wall clock": 892, + "wallaby, brush kangaroo": 104, + "wallet, billfold, notecase, pocketbook": 893, + "wardrobe, closet, press": 894, + "warplane, military plane": 895, + "warthog": 343, + "washbasin, handbasin, washbowl, lavabo, wash-hand basin": 896, + "washer, automatic washer, washing machine": 897, + "water bottle": 898, + "water buffalo, water ox, Asiatic buffalo, Bubalus bubalis": 346, + "water jug": 899, + "water ouzel, dipper": 20, + "water snake": 58, + "water tower": 900, + "weasel": 356, + "web site, website, internet site, site": 916, + "weevil": 307, + "whippet": 172, + "whiptail, whiptail lizard": 41, + "whiskey jug": 901, + "whistle": 902, + "white stork, Ciconia ciconia": 127, + "white wolf, Arctic wolf, Canis lupus tundrarum": 270, + "wig": 903, + "wild boar, boar, Sus scrofa": 342, + "window screen": 904, + "window shade": 905, + "wine bottle": 907, + "wing": 908, + "wire-haired fox terrier": 188, + "wok": 909, + "wolf spider, hunting spider": 77, + "wombat": 106, + "wood rabbit, cottontail, cottontail rabbit": 330, + "wooden spoon": 910, + "wool, woolen, woollen": 911, + "worm fence, snake fence, snake-rail fence, Virginia fence": 912, + "wreck": 913, + "yawl": 914, + "yellow lady's slipper, yellow lady-slipper, Cypripedium calceolus, Cypripedium parviflorum": 986, + "yurt": 915, + "zebra": 340, + "zucchini, courgette": 939 + }, + "layer_type": "bottleneck", + "model_type": "resnet", + "num_channels": 3, + "torch_dtype": "float32", + "transformers_version": "4.18.0.dev0" +} diff --git a/new_impl/cv/sam/sam.py b/new_impl/cv/sam/sam.py new file mode 100644 index 0000000000000000000000000000000000000000..26cc9176eb819d36b3678629d8b7a30fa4dc3925 --- /dev/null +++ b/new_impl/cv/sam/sam.py @@ -0,0 +1,1342 @@ +from transformers import BlipForQuestionAnswering, BlipConfig,BlipModel,CLIPVisionModel,CLIPConfig +import torch +from torch import nn +from abc import ABC, abstractmethod +from copy import deepcopy +from typing import Optional, Union +from einops import rearrange, repeat +from einops.layers.torch import Rearrange +import tqdm +import torch.nn.functional as F +from utils.dl.common.model import get_model_device, get_model_latency, get_model_size, set_module +from utils.dl.common.model import set_module, get_module, get_super_module +from utils.common.log import logger +from new_impl.cv.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util, LoRA +from transformers.models.blip.modeling_blip import BlipAttention +#from transformers.models.blip.modeling_blip_text import BlipTextSelfAttention,BlipTextAttention,BlipTextSelfOutput +from transformers.models.beit.modeling_beit import BeitSelfAttention,BeitConfig +from transformers.models.clip.modeling_clip import CLIPAttention +from new_impl.cv.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from new_impl.cv.elasticdnn.model.base import Abs, KTakesAll, ElasticDNNUtil, Layer_WrappedWithFBS + +from typing import Optional, Tuple +import math + +from transformers.models.sam.modeling_sam import SamVisionEncoder,SamPreTrainedModel,SamAttention,SamConfig,SamVisionAttention +from new_impl.cv.dnns.deeplabv3.head import DecoderLinear +config = SamConfig.from_pretrained('new_impl/cv/sam/sam_pretrained') + +class Sammodel(SamPreTrainedModel): + def __init__(self, config,num_classes): + config = SamConfig.from_pretrained('new_impl/cv/sam/sam_pretrained') + super(Sammodel,self).__init__(config) + self.vision_encoder = SamVisionEncoder(config.vision_config) + self.head = DecoderLinear(num_classes, 16, 256, (224, 224)).to('cuda') + def forward(self,x): + x = self.vision_encoder(x) + x = rearrange(x[0],'b c h w -> b (h w) c') + output = self.head(x) + return output +# def blip(num_classes): +# model = BlipForQuestionAnswering.from_pretrained('new_impl/mm/Vis_bert/QuestionAnswering/VisBert_pretrained') +# # linear = model.text_decoder.cls.predictions.decoder +# # new_linear = nn.Linear(linear.in_features,30524,bias = True) +# # set_module(model,'text_decoder.cls.predictions.decoder',new_linear) +# return model +# def blip(num_classes): +# model = BlipForQuestionAnswering.from_pretrained('new_impl/mm/Vis_bert/QuestionAnswering/VisBert_pretrained') +# linear = model.text_decoder.cls.predictions.decoder +# new_linear = nn.Linear(linear.in_features,num_classes,bias = True) +# set_module(model,'text_decoder.cls.predictions.decoder',new_linear) +# return model +# class clip(nn.Module): +# def __init__(self,num_classes): +# super(clip,self).__init__() +# self.clip = CLIPVisionModel.from_pretrained('new_impl/cv/clip/pretrained_model') +# self.classifier = nn.Linear(768,num_classes) + +# def forward(self,sample): +# output = self.clip(sample)[-1]#output the last hidden +# output = self.classifier(output) +# return output + + +class ToQKV_WrappedWithLoRA(nn.Module): + def __init__(self, fc: nn.Linear, ab_r: int): + super(ToQKV_WrappedWithLoRA, self).__init__() + + self.fc = fc + self.ab = self.create_ab_as_linear(fc.weight.data, ab_r) + + def create_ab_as_linear(self, fc_weight: torch.Tensor, ab_r: int): + res = nn.Sequential( + LoRA(fc_weight.size(1), fc_weight.size(0) // ab_r, bias=False), + LoRA(fc_weight.size(0) // ab_r, fc_weight.size(0), bias=False) + ).to(fc_weight.device) + nn.init.kaiming_uniform_(res[0].weight, a=5 ** 0.5) + nn.init.zeros_(res[1].weight) + return res + + def forward(self, x): + x1 = self.fc(x) + x2 = self.ab(x) + return x1 + x2 + + +class FMLoRA_sam_Util(FMLoRA_Util): + + @torch.no_grad() + def add_lora_ab_to_fm(self, fm: nn.Module, ab_r: int, samples: torch.Tensor): + fm.eval() + + # print(samples) + # for k, v in samples.items(): + # if isinstance(v, torch.Tensor): + # samples[k] = v.to(get_model_device(fm)) + + #o1 = fm.generate(**samples) + o1 = fm(samples) + for name, module in fm.named_modules(): + if name.endswith(('query', 'key', 'value')): + set_module(fm, name, ToQKV_WrappedWithLoRA(module, ab_r)) + elif name.endswith('.qkv'): + set_module(fm, name, ToQKV_WrappedWithLoRA(module, ab_r)) + elif name.endswith(('k_proj','q_proj','v_proj')): + set_module(fm, name, ToQKV_WrappedWithLoRA(module, ab_r)) + + + #o2 = fm.generate(**samples) + o2 = fm(samples) + if isinstance(o1, tuple): + o1 = o1[-1] + o2 = o2[-1] + output_diff = ((o1 - o2) ** 2).sum() + assert output_diff < 1e-5 + return fm + + @torch.no_grad() + def absorb_lora_and_recover_net_structure(self, fm: nn.Module, samples: torch.Tensor): + fm.eval() + # print('absorb lora before') + + # for k, v in samples.items(): + # if isinstance(v, torch.Tensor): + # samples[k] = v.to(get_model_device(fm)) + + o1 = fm(samples) + + for name, module in fm.named_modules(): + if not isinstance(module, ToQKV_WrappedWithLoRA): + continue + + fc = module.fc + ab = module.ab + + fc.weight.add_(ab[1].weight @ ab[0].weight) + + set_module(fm, name, fc) + + # print('absorb lora after') + o2 = fm(samples) + + if isinstance(o1, tuple): + o1 = o1[-1] + o2 = o2[-1] + output_diff = ((o1 - o2) ** 2).sum() + assert output_diff < 1e-6, output_diff + + return fm + + +####Here start with Fbs + +# class blipTextAttentionPrunable(BlipTextSelfAttention): +# def __init__(self,is_cross_attention): +# config = BlipConfig.from_pretrained('new_impl/mm/Vis_bert/QuestionAnswering/VisBert_pretrained') +# super(blipTextAttentionPrunable,self).__init__(config.text_config,is_cross_attention) + +# def save_attn_gradients(self, attn_gradients): +# self.attn_gradients = attn_gradients + +# def get_attn_gradients(self): +# return self.attn_gradients + +# def save_attention_map(self, attention_map): +# self.attention_map = attention_map + +# def get_attention_map(self): +# return self.attention_map + +# def transpose_for_scores(self, x): +# new_x_shape = x.size()[:-1] + (self.num_attention_heads, -1) +# x = x.view(*new_x_shape) +# return x.permute(0, 2, 1, 3) + +# def forward( +# self, +# hidden_states: torch.Tensor, +# attention_mask: Optional[torch.FloatTensor] = None, +# head_mask: Optional[torch.FloatTensor] = None, +# encoder_hidden_states: Optional[torch.FloatTensor] = None, +# encoder_attention_mask: Optional[torch.FloatTensor] = None, +# past_key_value: Optional[Tuple[Tuple[torch.FloatTensor]]] = None, +# output_attentions: Optional[bool] = False, +# ) -> Tuple[torch.Tensor]: +# mixed_query_layer = self.query(hidden_states) + +# # If this is instantiated as a cross-attention module, the keys +# # and values come from an encoder; the attention mask needs to be +# # such that the encoder's padding tokens are not attended to. +# is_cross_attention = encoder_hidden_states is not None + +# if is_cross_attention: +# key_layer = self.transpose_for_scores(self.key(encoder_hidden_states)) +# value_layer = self.transpose_for_scores(self.value(encoder_hidden_states)) +# attention_mask = encoder_attention_mask +# elif past_key_value is not None: +# key_layer = self.transpose_for_scores(self.key(hidden_states)) +# value_layer = self.transpose_for_scores(self.value(hidden_states)) +# key_layer = torch.cat([past_key_value[0], key_layer], dim=2) +# value_layer = torch.cat([past_key_value[1], value_layer], dim=2) +# else: +# key_layer = self.transpose_for_scores(self.key(hidden_states)) +# value_layer = self.transpose_for_scores(self.value(hidden_states)) + +# query_layer = self.transpose_for_scores(mixed_query_layer) + +# past_key_value = (key_layer, value_layer) + +# # Take the dot product between "query" and "key" to get the raw attention scores. +# attention_scores = torch.matmul(query_layer, key_layer.transpose(-1, -2)) + +# if self.position_embedding_type == "relative_key" or self.position_embedding_type == "relative_key_query": +# seq_length = hidden_states.size()[1] +# position_ids_l = torch.arange(seq_length, dtype=torch.long, device=hidden_states.device).view(-1, 1) +# position_ids_r = torch.arange(seq_length, dtype=torch.long, device=hidden_states.device).view(1, -1) +# distance = position_ids_l - position_ids_r +# positional_embedding = self.distance_embedding(distance + self.max_position_embeddings - 1) +# positional_embedding = positional_embedding.to(dtype=query_layer.dtype) # fp16 compatibility + +# if self.position_embedding_type == "relative_key": +# relative_position_scores = torch.einsum("bhld,lrd->bhlr", query_layer, positional_embedding) +# attention_scores = attention_scores + relative_position_scores +# elif self.position_embedding_type == "relative_key_query": +# relative_position_scores_query = torch.einsum("bhld,lrd->bhlr", query_layer, positional_embedding) +# relative_position_scores_key = torch.einsum("bhrd,lrd->bhlr", key_layer, positional_embedding) +# attention_scores = attention_scores + relative_position_scores_query + relative_position_scores_key + +# attention_scores = attention_scores / math.sqrt(self.attention_head_size) +# if attention_mask is not None: +# # Apply the attention mask is (precomputed for all layers in BlipTextModel forward() function) +# attention_scores = attention_scores + attention_mask.to(attention_scores.device) + +# # Normalize the attention scores to probabilities. +# attention_probs = nn.Softmax(dim=-1)(attention_scores) + +# # This is actually dropping out entire tokens to attend to, which might +# # seem a bit unusual, but is taken from the original Transformer paper. +# attention_probs_dropped = self.dropout(attention_probs) + +# # Mask heads if we want to +# if head_mask is not None: +# attention_probs_dropped = attention_probs_dropped * head_mask + +# context_layer = torch.matmul(attention_probs_dropped, value_layer) + +# context_layer = context_layer.permute(0, 2, 1, 3).contiguous() +# new_context_layer_shape = context_layer.size()[:-2] + (-1,) +# context_layer = context_layer.view(*new_context_layer_shape) + +# outputs = (context_layer, attention_probs) if output_attentions else (context_layer,) + +# outputs = outputs + (past_key_value,) +# return outputs +# @staticmethod +# def init_from_exist_self_attn(attn: BlipTextSelfAttention,is_cross_attention): +# # print(attn) + +# res = blipTextAttentionPrunable(is_cross_attention) + +# for attr in dir(attn): +# # if str(attr) in ['transpose_for_scores'] or str(attr).startswith('_'): +# # continue +# # if isinstance(getattr(attn, attr), nn.Module): +# # print(attr) + +# if isinstance(getattr(attn, attr), nn.Module): +# try: +# # print(attr, 'ok') +# setattr(res, attr, getattr(attn, attr)) + +# except Exception as e: +# print(attr, str(e)) + + + +# return res + + + +# class blipSelfTextAttentionPrunable(BlipTextAttention): +# def __init__(self, config, is_cross_attention=False): +# self.self = blipTextAttentionPrunable(config, is_cross_attention) +# self.output = BlipTextSelfOutput(config) +# self.pruned_heads = set() +# super(blipSelfTextAttentionPrunable,self).__init__(config) + +# def prune_heads(self, heads): +# if len(heads) == 0: +# return +# heads, index = find_pruneable_heads_and_indices( +# heads, self.self.num_attention_heads, self.self.attention_head_size, self.pruned_heads +# ) + +# # Prune linear layers +# self.self.query = prune_linear_layer(self.self.query, index) +# self.self.key = prune_linear_layer(self.self.key, index) +# self.self.value = prune_linear_layer(self.self.value, index) +# self.output.dense = prune_linear_layer(self.output.dense, index, dim=1) + +# # Update hyper params and store pruned heads +# self.self.num_attention_heads = self.self.num_attention_heads - len(heads) +# self.self.all_head_size = self.self.attention_head_size * self.self.num_attention_heads +# self.pruned_heads = self.pruned_heads.union(heads) + +# def forward( +# self, +# hidden_states: torch.Tensor, +# attention_mask: Optional[torch.FloatTensor] = None, +# head_mask: Optional[torch.FloatTensor] = None, +# encoder_hidden_states: Optional[torch.FloatTensor] = None, +# encoder_attention_mask: Optional[torch.FloatTensor] = None, +# past_key_value: Optional[Tuple[Tuple[torch.FloatTensor]]] = None, +# output_attentions: Optional[bool] = False, +# ) -> Tuple[torch.Tensor]: +# self_outputs = self.self( +# hidden_states, +# attention_mask, +# head_mask, +# encoder_hidden_states, +# encoder_attention_mask, +# past_key_value, +# output_attentions, +# ) +# attention_output = self.output(self_outputs[0], hidden_states) +# outputs = (attention_output,) + self_outputs[1:] # add attentions if we output them +# return outputs +# @staticmethod +# def init_from_exist_self_attn(attn: BlipTextAttention,config,is_cross_attention): +# # print(attn) + +# res = blipTextAttentionPrunable(config,is_cross_attention) + +# for attr in dir(attn): +# # if str(attr) in ['transpose_for_scores'] or str(attr).startswith('_'): +# # continue +# # if isinstance(getattr(attn, attr), nn.Module): +# # print(attr) + +# if isinstance(getattr(attn, attr), nn.Module): +# try: +# # print(attr, 'ok') +# setattr(res, attr, getattr(attn, attr)) + +# except Exception as e: +# print(attr, str(e)) + + + +# return res + + + + + + +# class blipSelfAttentionPrunable(BlipAttention): +# def __init__(self): +# config = BlipConfig.from_pretrained('new_impl/mm/Vis_bert/QuestionAnswering/VisBert_pretrained') +# super(blipSelfAttentionPrunable, self).__init__(config.vision_config) + +# def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int): +# return tensor.view(bsz, seq_len, self.num_heads, -1).transpose(1, 2).contiguous() + +# def forward( +# self, +# hidden_states: torch.Tensor, +# head_mask: Optional[torch.Tensor] = None, +# output_attentions: Optional[bool] = False, +# ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: +# """Input shape: Batch x Time x Channel""" + +# bsz, tgt_len, embed_dim = hidden_states.size() + +# mixed_qkv = ( +# self.qkv(hidden_states) +# .reshape(bsz, tgt_len, 3, self.num_heads, -1) +# .permute(2, 0, 3, 1, 4) +# ) +# query_states, key_states, value_states = mixed_qkv[0], mixed_qkv[1], mixed_qkv[2] + +# # Take the dot product between "query" and "key" to get the raw attention scores. +# attention_scores = torch.matmul(query_states, key_states.transpose(-1, -2)) + +# attention_scores = attention_scores * self.scale + +# # Normalize the attention scores to probabilities. +# attention_probs = nn.functional.softmax(attention_scores, dim=-1) + +# # This is actually dropping out entire tokens to attend to, which might +# # seem a bit unusual, but is taken from the original Transformer paper. +# attention_probs = self.dropout(attention_probs) + +# # Mask heads if we want to +# if head_mask is not None: +# attention_probs = attention_probs * head_mask + +# context_layer = torch.matmul(attention_probs, value_states).permute(0, 2, 1, 3) + +# new_context_layer_shape = context_layer.size()[:-2] + (-1,) +# context_layer = context_layer.reshape(new_context_layer_shape) + +# output = self.projection(context_layer) + +# outputs = (output, attention_probs) if output_attentions else (output, None) + +# return outputs + +# @staticmethod +# def init_from_exist_self_attn(attn: BlipAttention): +# # print(attn) + +# res = blipSelfAttentionPrunable() + +# for attr in dir(attn): +# # if str(attr) in ['transpose_for_scores'] or str(attr).startswith('_'): +# # continue +# # if isinstance(getattr(attn, attr), nn.Module): +# # print(attr) + +# if isinstance(getattr(attn, attr), nn.Module): +# try: +# # print(attr, 'ok') +# setattr(res, attr, getattr(attn, attr)) + +# except Exception as e: +# print(attr, str(e)) + + + +# return res +# class CLIPAttentionPrunable(CLIPAttention): +# def __init__(self, config: CLIPConfig, ratio:int) -> None: +# config = CLIPConfig.from_pretrained('new_impl/cv/clip/pretrained_model') +# super(CLIPAttentionPrunable, self).__init__(config.vision_config) +# self.head_dim = self.head_dim // ratio + +# def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int): +# return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous() + +# def forward( +# self, +# hidden_states: torch.Tensor, +# attention_mask: Optional[torch.Tensor] = None, +# causal_attention_mask: Optional[torch.Tensor] = None, +# output_attentions: Optional[bool] = False, +# ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: +# """Input shape: Batch x Time x Channel""" + +# bsz, tgt_len, embed_dim = hidden_states.size() + +# # get query proj +# query_states = self.q_proj(hidden_states) * self.scale +# key_states = self._shape(self.k_proj(hidden_states), -1, bsz) +# value_states = self._shape(self.v_proj(hidden_states), -1, bsz) + +# proj_shape = (bsz * self.num_heads, -1, self.head_dim) +# query_states = self._shape(query_states, tgt_len, bsz).view(*proj_shape) +# key_states = key_states.view(*proj_shape) +# value_states = value_states.view(*proj_shape) + +# src_len = key_states.size(1) +# attn_weights = torch.bmm(query_states, key_states.transpose(1, 2)) + +# if attn_weights.size() != (bsz * self.num_heads, tgt_len, src_len): +# raise ValueError( +# f"Attention weights should be of size {(bsz * self.num_heads, tgt_len, src_len)}, but is" +# f" {attn_weights.size()}" +# ) + +# # apply the causal_attention_mask first +# if causal_attention_mask is not None: +# if causal_attention_mask.size() != (bsz, 1, tgt_len, src_len): +# raise ValueError( +# f"Attention mask should be of size {(bsz, 1, tgt_len, src_len)}, but is" +# f" {causal_attention_mask.size()}" +# ) +# attn_weights = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + causal_attention_mask +# attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len) + +# if attention_mask is not None: +# if attention_mask.size() != (bsz, 1, tgt_len, src_len): +# raise ValueError( +# f"Attention mask should be of size {(bsz, 1, tgt_len, src_len)}, but is {attention_mask.size()}" +# ) +# attn_weights = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + attention_mask +# attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len) + +# attn_weights = nn.functional.softmax(attn_weights, dim=-1) + +# if output_attentions: +# # this operation is a bit akward, but it's required to +# # make sure that attn_weights keeps its gradient. +# # In order to do so, attn_weights have to reshaped +# # twice and have to be reused in the following +# attn_weights_reshaped = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) +# attn_weights = attn_weights_reshaped.view(bsz * self.num_heads, tgt_len, src_len) +# else: +# attn_weights_reshaped = None + +# attn_probs = nn.functional.dropout(attn_weights, p=self.dropout, training=self.training) + +# attn_output = torch.bmm(attn_probs, value_states) + +# if attn_output.size() != (bsz * self.num_heads, tgt_len, self.head_dim): +# raise ValueError( +# f"`attn_output` should be of size {(bsz, self.num_heads, tgt_len, self.head_dim)}, but is" +# f" {attn_output.size()}" +# ) + +# attn_output = attn_output.view(bsz, self.num_heads, tgt_len, self.head_dim) +# attn_output = attn_output.transpose(1, 2) +# attn_output = attn_output.reshape(bsz, tgt_len, -1) + +# attn_output = self.out_proj(attn_output) + +# return attn_output, attn_weights_reshaped + +# @staticmethod +# def init_from_exist_self_attn(attn: CLIPAttention,config,ratio:int): +# # print(attn) + +# res = CLIPAttentionPrunable(config,ratio) + +# for attr in dir(attn): +# # if str(attr) in ['transpose_for_scores'] or str(attr).startswith('_'): +# # continue +# # if isinstance(getattr(attn, attr), nn.Module): +# # print(attr) + +# if isinstance(getattr(attn, attr), nn.Module): +# try: +# # print(attr, 'ok') +# setattr(res, attr, getattr(attn, attr)) + +# except Exception as e: +# print(attr, str(e)) +# return res + +class SamVisionAttentionPrunable(SamVisionAttention): + """Multi-head Attention block with relative position embeddings.""" + + def __init__(self, config,ratio): + super(SamVisionAttentionPrunable,self).__init__(config,window_size=14) + window_size = 14 + head_dim = config.hidden_size // config.num_attention_heads + input_size = ( + (config.image_size // config.patch_size, config.image_size // config.patch_size) + if window_size == 0 + else (window_size, window_size) + ) + self.rel_pos_h = nn.Parameter(torch.zeros(2 * input_size[0] - 1, head_dim//ratio)) + self.rel_pos_w = nn.Parameter(torch.zeros(2 * input_size[1] - 1, head_dim//ratio)) + + def get_rel_pos(self, q_size: int, k_size: int, rel_pos: torch.Tensor) -> torch.Tensor: + """ + Get relative positional embeddings according to the relative positions of + query and key sizes. + + Args: + q_size (int): + size of the query. + k_size (int): + size of key k. + rel_pos (`torch.Tensor`): + relative position embeddings (L, channel). + + Returns: + Extracted positional embeddings according to relative positions. + """ + + max_rel_dist = int(2 * max(q_size, k_size) - 1) + # Interpolate rel pos. + rel_pos_resized = F.interpolate( + rel_pos.reshape(1, rel_pos.shape[0], -1).permute(0, 2, 1), + size=max_rel_dist, + mode="linear", + ) + + rel_pos_resized = rel_pos_resized.reshape(-1, max_rel_dist).permute(1, 0) + + # Scale the coords with short length if shapes for q and k are different. + q_coords = torch.arange(q_size)[:, None] * max(k_size / q_size, 1.0) + k_coords = torch.arange(k_size)[None, :] * max(q_size / k_size, 1.0) + relative_coords = (q_coords - k_coords) + (k_size - 1) * max(q_size / k_size, 1.0) + + return rel_pos_resized[relative_coords.long()] + + def add_decomposed_rel_pos( + self, + attn: torch.Tensor, + query: torch.Tensor, + rel_pos_h: torch.Tensor, + rel_pos_w: torch.Tensor, + q_size: Tuple[int, int], + k_size: Tuple[int, int], + ) -> torch.Tensor: + """ + Calculate decomposed Relative Positional Embeddings from :paper:`mvitv2`. + https://github.com/facebookresearch/mvit/blob/19786631e330df9f3622e5402b4a419a263a2c80/mvit/models/attention.py + + Args: + attn (`torch.Tensor`): + attention map. + query (`torch.Tensor`): + query q in the attention layer with shape (batch_size, query_height * query_width, channel). + rel_pos_h (`torch.Tensor`): + relative position embeddings (Lh, channel) for height axis. + rel_pos_w (`torch.Tensor`): + relative position embeddings (Lw, channel) for width axis. + q_size (tuple): + spatial sequence size of query q with (query_height, query_width). + k_size (tuple): + spatial sequence size of key k with (key_height, key_width). + + Returns: + attn (`torch.Tensor`): + attention map with added relative positional embeddings. + """ + query_height, query_width = q_size + key_height, key_width = k_size + relative_position_height = self.get_rel_pos(query_height, key_height, rel_pos_h) + relative_position_width = self.get_rel_pos(query_width, key_width, rel_pos_w) + + batch_size, _, dim = query.shape + reshaped_query = query.reshape(batch_size, query_height, query_width, dim) + # print(reshaped_query.shape) + # print(relative_position_height.shape) + rel_h = torch.einsum("bhwc,hkc->bhwk", reshaped_query, relative_position_height) + rel_w = torch.einsum("bhwc,wkc->bhwk", reshaped_query, relative_position_width) + attn = attn.reshape(batch_size, query_height, query_width, key_height, key_width) + attn = attn + rel_h[:, :, :, :, None] + rel_w[:, :, :, None, :] + attn = attn.reshape(batch_size, query_height * query_width, key_height * key_width) + return attn + + def forward(self, hidden_states: torch.Tensor, output_attentions=False) -> torch.Tensor: + batch_size, height, width, _ = hidden_states.shape + # qkv with shape (3, batch_size, nHead, height * width, channel) + qkv = ( + self.qkv(hidden_states) + .reshape(batch_size, height * width, 3, self.num_attention_heads, -1) + .permute(2, 0, 3, 1, 4) + ) + # q, k, v with shape (batch_size * nHead, height * width, channel) + query, key, value = qkv.reshape(3, batch_size * self.num_attention_heads, height * width, -1).unbind(0) + + attn_weights = (query * self.scale) @ key.transpose(-2, -1) + + if self.use_rel_pos: + attn_weights = self.add_decomposed_rel_pos( + attn_weights, query, self.rel_pos_h, self.rel_pos_w, (height, width), (height, width) + ) + + attn_weights = torch.nn.functional.softmax(attn_weights, dtype=torch.float32, dim=-1).to(query.dtype) + + attn_probs = nn.functional.dropout(attn_weights, p=self.dropout, training=self.training) + + attn_output = (attn_probs @ value).reshape(batch_size, self.num_attention_heads, height, width, -1) + attn_output = attn_output.permute(0, 2, 3, 1, 4).reshape(batch_size, height, width, -1) + + attn_output = self.proj(attn_output) + + if output_attentions: + outputs = (attn_output, attn_weights) + else: + outputs = (attn_output, None) + + return outputs + @staticmethod + def init_from_exist_self_attn(attn: SamVisionAttention,config,ratio): + # print(attn) + + res = SamVisionAttentionPrunable(config,ratio) + + for attr in dir(attn): + # if str(attr) in ['transpose_for_scores'] or str(attr).startswith('_'): + # continue + # if isinstance(getattr(attn, attr), nn.Module): + # print(attr) + + if isinstance(getattr(attn, attr), nn.Module): + try: + # print(attr, 'ok') + setattr(res, attr, getattr(attn, attr)) + + except Exception as e: + print(attr, str(e)) + return res + +class FM_to_MD_sam_Util(FM_to_MD_Util): + def init_md_from_fm_by_reducing_width(self, fm: nn.Module, reducing_width_ratio: int) -> nn.Module: + fm_vis = deepcopy(fm) + config = SamConfig.from_pretrained('new_impl/cv/sam/sam_pretrained') + + for block_i,block in enumerate(fm_vis.vision_encoder.layers): + set_module(block, 'attn', SamVisionAttentionPrunable.init_from_exist_self_attn(block.attn,config.vision_config,reducing_width_ratio)) + + def _f(n): + return int(n // reducing_width_ratio) + + # def _rand_indexes(n): + # return torch.randperm(n)[0: int(n // reducing_width_ratio)] + + def l1_max_indexes(p: torch.Tensor, dim=0): + assert dim in [0, 1] + assert p.dim() in [1, 2, 4] + + if dim == 1: + p = p.T + + p_norm = p.abs().contiguous().view(p.size(0), -1).sum(dim=1) + n = p.size(0) + return p_norm.argsort(descending=True)[0: int(n // reducing_width_ratio)].sort()[0] + + for block_i, block in enumerate(fm_vis.vision_encoder.layers): + + qkv = get_module(block, f'attn.qkv') + + new_qkv = nn.Linear(qkv.in_features, _f(qkv.out_features), + qkv.bias is not None, qkv.weight.device) + indexes = l1_max_indexes(qkv.weight.data, 0) + + new_qkv.weight.data.copy_(qkv.weight.data[indexes]) + if qkv.bias is not None: + new_qkv.bias.data.copy_(qkv.bias.data[indexes]) + set_module(block, f'attn.qkv', new_qkv) + + proj = get_module(block, f'attn.proj') + new_proj = nn.Linear(_f(proj.in_features), proj.out_features, + proj.bias is not None, proj.weight.device) + new_proj.weight.data.copy_(proj.weight.data[:, l1_max_indexes(proj.weight.data, 1)]) + if proj.bias is not None: + new_proj.bias.data.copy_(proj.bias.data) + set_module(block, f'attn.proj', new_proj) + + fc1 = get_module(block, f'mlp.lin1') + new_fc1 = nn.Linear(fc1.in_features, _f(fc1.out_features), + fc1.bias is not None, fc1.weight.device) + indexes = l1_max_indexes(fc1.weight.data, 0) + new_fc1.weight.data.copy_(fc1.weight.data[indexes]) + if fc1.bias is not None: + new_fc1.bias.data.copy_(fc1.bias.data[indexes]) + set_module(block, f'mlp.lin1', new_fc1) + + fc2 = get_module(block, f'mlp.lin2') + new_fc2 = nn.Linear(_f(fc2.in_features), fc2.out_features, + fc2.bias is not None, fc2.weight.device) + new_fc2.weight.data.copy_(fc2.weight.data[:, l1_max_indexes(fc2.weight.data, 1)]) + if fc2.bias is not None: + new_fc2.bias.data.copy_(fc2.bias.data) + set_module(block, f'mlp.lin2', new_fc2) + + + # for block_i, block in enumerate(fm_vis.text_decoder.bert.encoder.layer): + # for k in ['query', 'key', 'value']: + # qkv = get_module(block, f'crossattention.self.{k}') + + # new_qkv = nn.Linear(qkv.in_features, _f(qkv.out_features), + # qkv.bias is not None, qkv.weight.device) + # indexes = l1_max_indexes(qkv.weight.data, 0) + + # new_qkv.weight.data.copy_(qkv.weight.data[indexes]) + # if qkv.bias is not None: + # new_qkv.bias.data.copy_(qkv.bias.data[indexes]) + # set_module(block, f'crossattention.self.{k}', new_qkv) + + # proj = get_module(block, f'crossattention.output.dense') + # new_proj = nn.Linear(_f(proj.in_features), proj.out_features, + # proj.bias is not None, proj.weight.device) + # new_proj.weight.data.copy_(proj.weight.data[:, l1_max_indexes(proj.weight.data, 1)]) + # if proj.bias is not None: + # new_proj.bias.data.copy_(proj.bias.data) + # set_module(block, f'crossattention.output.dense', new_proj) + + + # for block_i, block in enumerate(fm_vis.text_encoder.encoder.layer): + # for k in ['query', 'key', 'value']: + # qkv = get_module(block, f'attention.self.{k}') + + # new_qkv = nn.Linear(qkv.in_features, _f(qkv.out_features), + # qkv.bias is not None, qkv.weight.device) + # indexes = l1_max_indexes(qkv.weight.data, 0) + + # new_qkv.weight.data.copy_(qkv.weight.data[indexes]) + # if qkv.bias is not None: + # new_qkv.bias.data.copy_(qkv.bias.data[indexes]) + # set_module(block, f'attention.self.{k}', new_qkv) + + # proj = get_module(block, f'attention.output.dense') + # new_proj = nn.Linear(_f(proj.in_features), proj.out_features, + # proj.bias is not None, proj.weight.device) + # new_proj.weight.data.copy_(proj.weight.data[:, l1_max_indexes(proj.weight.data, 1)]) + # if proj.bias is not None: + # new_proj.bias.data.copy_(proj.bias.data) + # set_module(block, f'attention.output.dense', new_proj) + + # fc1 = get_module(block, f'intermediate.dense') + # new_fc1 = nn.Linear(fc1.in_features, _f(fc1.out_features), + # fc1.bias is not None, fc1.weight.device) + # indexes = l1_max_indexes(fc1.weight.data, 0) + # new_fc1.weight.data.copy_(fc1.weight.data[indexes]) + # if fc1.bias is not None: + # new_fc1.bias.data.copy_(fc1.bias.data[indexes]) + # set_module(block, f'intermediate.dense', new_fc1) + + # fc2 = get_module(block, f'output.dense') + # new_fc2 = nn.Linear(_f(fc2.in_features), fc2.out_features, + # fc2.bias is not None, fc2.weight.device) + # new_fc2.weight.data.copy_(fc2.weight.data[:, l1_max_indexes(fc2.weight.data, 1)]) + # if fc2.bias is not None: + # new_fc2.bias.data.copy_(fc2.bias.data) + # set_module(block, f'output.dense', new_fc2) + + + # for block_i, block in enumerate(fm_vis.text_encoder.encoder.layer): + # for k in ['query', 'key', 'value']: + # qkv = get_module(block, f'crossattention.self.{k}') + + # new_qkv = nn.Linear(qkv.in_features, _f(qkv.out_features), + # qkv.bias is not None, qkv.weight.device) + # indexes = l1_max_indexes(qkv.weight.data, 0) + + # new_qkv.weight.data.copy_(qkv.weight.data[indexes]) + # if qkv.bias is not None: + # new_qkv.bias.data.copy_(qkv.bias.data[indexes]) + # set_module(block, f'crossattention.self.{k}', new_qkv) + + # proj = get_module(block, f'crossattention.output.dense') + # new_proj = nn.Linear(_f(proj.in_features), proj.out_features, + # proj.bias is not None, proj.weight.device) + # new_proj.weight.data.copy_(proj.weight.data[:, l1_max_indexes(proj.weight.data, 1)]) + # if proj.bias is not None: + # new_proj.bias.data.copy_(proj.bias.data) + # set_module(block, f'crossattention.output.dense', new_proj) + + + + # for block_i, block in enumerate(fm_vis.vision_model.encoder.layers): + # qkv = block.self_attn.qkv + + # new_qkv = nn.Linear(qkv.in_features, _f(qkv.out_features), + # qkv.bias is not None, qkv.weight.device) + # indexes = l1_max_indexes(qkv.weight.data, 0) + + # new_qkv.weight.data.copy_(qkv.weight.data[indexes]) + # if qkv.bias is not None: + # new_qkv.bias.data.copy_(qkv.bias.data[indexes]) + # set_module(fm_vis, f'vision_model.encoder.layers.{block_i}.self_attn.qkv', new_qkv) + + # proj = block.self_attn.projection + # new_proj = nn.Linear(_f(proj.in_features), proj.out_features, + # proj.bias is not None, proj.weight.device) + # new_proj.weight.data.copy_(proj.weight.data[:, l1_max_indexes(proj.weight.data, 1)]) + # if proj.bias is not None: + # new_proj.bias.data.copy_(proj.bias.data) + # set_module(fm_vis, f'vision_model.encoder.layers.{block_i}.self_attn.projection', new_proj) + + # fc1 = block.mlp.fc1 + # new_fc1 = nn.Linear(fc1.in_features, _f(fc1.out_features), + # fc1.bias is not None, fc1.weight.device) + # indexes = l1_max_indexes(fc1.weight.data, 0) + # new_fc1.weight.data.copy_(fc1.weight.data[indexes]) + # if fc1.bias is not None: + # new_fc1.bias.data.copy_(fc1.bias.data[indexes]) + # set_module(fm_vis, f'vision_model.encoder.layers.{block_i}.mlp.fc1', new_fc1) + + # fc2 = block.mlp.fc2 + # new_fc2 = nn.Linear(_f(fc2.in_features), fc2.out_features, + # fc2.bias is not None, fc2.weight.device) + # new_fc2.weight.data.copy_(fc2.weight.data[:, l1_max_indexes(fc2.weight.data, 1)]) + # if fc2.bias is not None: + # new_fc2.bias.data.copy_(fc2.bias.data) + # set_module(fm_vis, f'vision_model.encoder.layers.{block_i}.mlp.fc2', new_fc2) + + return fm_vis + + def init_md_from_fm_by_reducing_width_with_perf_test(self, fm: nn.Module, reducing_width_ratio: int, + samples: torch.Tensor) -> nn.Module: + fm_size = get_model_size(fm, True) + fm_latency = self._get_model_latency(fm, samples, 20, + get_model_device(fm), 20, False) + + master_dnn = self.init_md_from_fm_by_reducing_width(fm, reducing_width_ratio) + master_dnn_size = get_model_size(master_dnn, True) + logger.debug(f'inited master DNN: {master_dnn}') + master_dnn_latency = self._get_model_latency(master_dnn, samples, 20, + get_model_device(master_dnn), 20, False) + + logger.info(f'init master DNN (w/o FBS yet) by reducing foundation model\'s width (by {reducing_width_ratio:d}x)') + logger.info(f'foundation model ({fm_size:.3f}MB, {fm_latency:.4f}s/sample) -> ' + f'master DNN ({master_dnn_size:.3f}MB, {master_dnn_latency:.4f}s/sample)\n' + f'(model size: ↓ {(fm_size / master_dnn_size):.2f}x, ' + f'latency: ↓ {(fm_latency / master_dnn_latency):.2f}x)') + + return master_dnn + + def _get_model_latency(self, model: torch.nn.Module, model_input_size, sample_num: int, + device: str, warmup_sample_num: int, return_detail=False): + import time + + if isinstance(model_input_size, tuple): + dummy_input = torch.rand(model_input_size).to(device) + else: + dummy_input = model_input_size + + model = model.to(device) + model.eval() + + # warm up + with torch.no_grad(): + for _ in range(warmup_sample_num): + model(dummy_input) + + infer_time_list = [] + + if device == 'cuda' or 'cuda' in str(device): + with torch.no_grad(): + for _ in range(sample_num): + s, e = torch.cuda.Event(enable_timing=True), torch.cuda.Event(enable_timing=True) + s.record() + model(dummy_input) + e.record() + torch.cuda.synchronize() + cur_model_infer_time = s.elapsed_time(e) / 1000. + infer_time_list += [cur_model_infer_time] + + else: + with torch.no_grad(): + for _ in range(sample_num): + start = time.time() + model(**dummy_input) + cur_model_infer_time = time.time() - start + infer_time_list += [cur_model_infer_time] + + avg_infer_time = sum(infer_time_list) / sample_num + + if return_detail: + return avg_infer_time, infer_time_list + return avg_infer_time + + +####Here starts with index + +class SqueezeLast(nn.Module): + def __init__(self): + super(SqueezeLast, self).__init__() + + def forward(self, x): + return x.squeeze(-1) + + +class ProjConv_WrappedWithFBS(Layer_WrappedWithFBS): + def __init__(self, proj: nn.Conv2d, r): + super(ProjConv_WrappedWithFBS, self).__init__() + + self.proj = proj + + # for conv: (B, C_in, H, W) -> (B, C_in) -> (B, C_out) + # for mlp in ViT: (B, #patches, D: dim of patches embedding) -> (B, D) -> (B, C_out) + self.fbs = nn.Sequential( + Abs(), + nn.AdaptiveAvgPool1d(1), + SqueezeLast(), + nn.Linear(proj.in_channels, proj.out_channels // r), + nn.ReLU(), + nn.Linear(proj.out_channels // r, proj.out_channels), + nn.ReLU() + ) + + nn.init.constant_(self.fbs[6].bias, 1.) + nn.init.kaiming_normal_(self.fbs[6].weight) + + + def forward(self, x): + if self.use_cached_channel_attention and self.cached_channel_attention is not None: + channel_attention = self.cached_channel_attention + else: + self.cached_raw_channel_attention = self.fbs(x) + self.cached_channel_attention = self.k_takes_all(self.cached_raw_channel_attention) + + channel_attention = self.cached_channel_attention + + raw_res = self.proj(x) + + return channel_attention.unsqueeze(1) * raw_res # TODO: + + +class Linear_WrappedWithFBS(Layer_WrappedWithFBS): + def __init__(self, linear: nn.Linear, r): + super(Linear_WrappedWithFBS, self).__init__() + + self.linear = linear + + # for conv: (B, C_in, H, W) -> (B, C_in) -> (B, C_out) + # for mlp in ViT: (B, #patches, D: dim of patches embedding) -> (B, D) -> (B, C_out) + self.fbs = nn.Sequential( + #Rearrange('b n d -> b d n'), + Rearrange('b h w d-> b d (h w)'), + Abs(), + nn.AdaptiveAvgPool1d(1), + SqueezeLast(), + nn.Linear(linear.in_features, linear.out_features // r), + nn.ReLU(), + nn.Linear(linear.out_features // r, linear.out_features), + nn.ReLU() + ) + + nn.init.constant_(self.fbs[6].bias, 1.) + nn.init.kaiming_normal_(self.fbs[6].weight) + + + def forward(self, x): + if self.use_cached_channel_attention and self.cached_channel_attention is not None: + channel_attention = self.cached_channel_attention + else: + self.cached_raw_channel_attention = self.fbs(x) + self.cached_channel_attention = self.k_takes_all(self.cached_raw_channel_attention) + + channel_attention = self.cached_channel_attention + + raw_res = self.linear(x) + raw_res = rearrange(raw_res,'b h w d-> b (h w) d') + res = channel_attention.unsqueeze(1) * raw_res + res = rearrange(res,'b (h w) d->b h w d',h=14,w=14) + return res + + +class ToQKV_WrappedWithFBS(Layer_WrappedWithFBS): + """ + This regards to_q/to_k/to_v as a whole (in fact it consists of multiple heads) and prunes it. + It seems different channels of different heads are pruned according to the input. + This is different from "removing some head" or "removing the same channels in each head". + """ + def __init__(self, to_qkv: nn.Linear, r): + super(ToQKV_WrappedWithFBS, self).__init__() + + # self.to_qkv = to_qkv + + self.to_qk = nn.Linear(to_qkv.in_features, to_qkv.out_features // 3 * 2, bias=to_qkv.bias is not None) + self.to_v = nn.Linear(to_qkv.in_features, to_qkv.out_features // 3, bias=to_qkv.bias is not None) + self.to_qk.weight.data.copy_(to_qkv.weight.data[0: to_qkv.out_features // 3 * 2]) + if to_qkv.bias is not None: + self.to_qk.bias.data.copy_(to_qkv.bias.data[0: to_qkv.out_features // 3 * 2]) + self.to_v.weight.data.copy_(to_qkv.weight.data[to_qkv.out_features // 3 * 2: ]) + if to_qkv.bias is not None: + self.to_v.bias.data.copy_(to_qkv.bias.data[to_qkv.out_features // 3 * 2: ]) + + self.fbs = nn.Sequential( + Rearrange('b n d -> b d n'), + Abs(), + nn.AdaptiveAvgPool1d(1), + SqueezeLast(), + nn.Linear(to_qkv.in_features, to_qkv.out_features // 3 // r), + nn.ReLU(), + # nn.Linear(to_qkv.out_features // 3 // r, to_qkv.out_features // 3), + nn.Linear(to_qkv.out_features // 3 // r, self.to_v.out_features), + nn.ReLU() + ) + + nn.init.constant_(self.fbs[6].bias, 1.) + nn.init.kaiming_normal_(self.fbs[6].weight) + + def forward(self, x): + if self.use_cached_channel_attention and self.cached_channel_attention is not None: + channel_attention = self.cached_channel_attention + else: + self.cached_raw_channel_attention = self.fbs(x) + + # print() + # for attn in self.cached_raw_channel_attention.chunk(3, dim=1)[0: 1]: + # print(self.cached_raw_channel_attention.size(), attn.size()) + # print(self.k_takes_all.k) + # print(attn[0].nonzero(as_tuple=True)[0].size(), attn[0]) + + self.cached_channel_attention = self.k_takes_all(self.cached_raw_channel_attention) + + + # for attn in self.cached_channel_attention.chunk(3, dim=1)[0: 1]: + # print(self.cached_channel_attention.size(), attn.size()) + # print(self.k_takes_all.k) + # print(attn[0].nonzero(as_tuple=True)[0].size(), attn[0]) + # print() + + channel_attention = self.cached_channel_attention + + qk = self.to_qk(x) + v = channel_attention.unsqueeze(1) * self.to_v(x) + return torch.cat([qk, v], dim=-1) + + # qkv = raw_res.chunk(3, dim = -1) + + # raw_v = qkv[2] + # print('raw_k, raw_v', qkv[0].sum((0, 1))[0: 10], qkv[0].sum((0, 1)).nonzero(as_tuple=True)[0].size(), + # qkv[1].sum((0, 1))[0: 10], qkv[1].sum((0, 1)).nonzero(as_tuple=True)[0].size(),) + # print('raw_v', raw_v.size(), raw_v.sum((0, 1))[0: 10], raw_v.sum((0, 1)).nonzero(as_tuple=True)[0].size()) + + # qkv_attn = channel_attention.chunk(3, dim=-1) + # print('attn', [attn[0][0: 10] for attn in qkv_attn]) + # print(channel_attention.unsqueeze(1).size(), raw_res.size()) + # print('fbs', channel_attention.size(), raw_res.size()) + # return channel_attention.unsqueeze(1) * raw_res + + +class StaticFBS(nn.Module): + def __init__(self, static_channel_attention): + super(StaticFBS, self).__init__() + assert static_channel_attention.dim() == 2 and static_channel_attention.size(0) == 1 + self.static_channel_attention = nn.Parameter(static_channel_attention, requires_grad=False) # (1, dim) + + def forward(self, x): + # print('staticfbs', x, self.static_channel_attention.unsqueeze(1)) + return x * self.static_channel_attention.unsqueeze(1) + + +class ElasticsamUtil(ElasticDNNUtil): + def convert_raw_dnn_to_master_dnn(self, raw_dnn: nn.Module, r: float, ignore_layers=[]): + assert len(ignore_layers) == 0, 'not supported yet' + + raw_vit = deepcopy(raw_dnn) + + # set_module(module, 'patch_embed.proj', ProjConv_WrappedWithFBS(module.patch_embed.proj, r)) + + for name, module in raw_vit.named_modules(): + # if name.endswith('attn'): + # set_module(module, 'qkv', ToQKV_WrappedWithFBS(module.qkv, r)) + if name.endswith('intermediate'): + set_module(module, 'dense', Linear_WrappedWithFBS(module.dense, r)) + elif name.endswith('mlp'): + set_module(module, 'lin1', Linear_WrappedWithFBS(module.lin1, r)) + + return raw_vit + + def set_master_dnn_sparsity(self, master_dnn: nn.Module, sparsity: float): + # for name, module in master_dnn.named_modules(): + # if not name.endswith('attn'): + # continue + + # q_features = module.qkv.to_qk.out_features // 2 + + # if (q_features - int(q_features * sparsity)) % module.num_heads != 0: + # # tune sparsity to ensure #unpruned channel % num_heads == 0 + # # so that the pruning seems to reduce the dim_head of each head + # tuned_sparsity = 1. - int((q_features - int(q_features * sparsity)) / module.num_heads) * module.num_heads / q_features + # logger.debug(f'tune sparsity from {sparsity:.2f} to {tuned_sparsity}') + # sparsity = tuned_sparsity + # break + + return super().set_master_dnn_sparsity(master_dnn, sparsity) + + def select_most_rep_sample(self, master_dnn: nn.Module, samples: torch.Tensor): + # print(samples) + return samples[0].unsqueeze(0) + # res = {k: v[0: 1] for k, v in samples.items()} + # return res + + def extract_surrogate_dnn_via_samples(self, master_dnn: nn.Module, samples: torch.Tensor, return_detail=False):#产生小模型的步骤 + sample = self.select_most_rep_sample(master_dnn, samples) + # assert sample.dim() == 4 and sample.size(0) == 1 + + # print('before') + master_dnn.eval() + self.clear_cached_channel_attention_in_master_dnn(master_dnn) + with torch.no_grad(): + master_dnn_output = master_dnn(sample) + + # print('after') + + boosted_vit = deepcopy(master_dnn) + + def get_unpruned_indexes_from_channel_attn(channel_attn: torch.Tensor, k): + assert channel_attn.size(0) == 1, 'use A representative sample to generate channel attentions' + + # print('attn_in_unpruned', channel_attn[0][0: 10]) + + res = channel_attn[0].nonzero(as_tuple=True)[0] # should be one-dim + + # res = channel_attn[0].argsort(descending=True)[0: -int(channel_attn.size(1) * k)].sort()[0] + + # g = channel_attn + # k = g.size(1) - int(g.size(1) * k) + # res = g.topk(k, 1)[1][0].sort()[0] + + return res + + unpruned_indexes_of_layers = {} + + # for attn, ff in boosted_vit.transformer.layers: + # for block_i, block in enumerate(boosted_vit.blocks): + for block_i, block in enumerate(boosted_vit.vision_encoder.layers): + # attn = block.attn + # ff = block.mlp + + ff_0 = get_module(block, f'mlp.lin1') + # ff_0_unpruned_indexes = get_unpruned_indexes_from_channel_attn(ff_0.cached_channel_attention, k) + ff_0_pruned_indexes = ff_0.k_takes_all.cached_i[0].sort()[0] + ff_0_unpruned_indexes = torch.LongTensor([ii for ii in range(ff_0.cached_channel_attention.size(1)) if ii not in ff_0_pruned_indexes]) + new_ff_0 = nn.Linear(ff_0.linear.in_features, ff_0_unpruned_indexes.size(0), ff_0.linear.bias is not None) + new_ff_0.weight.data.copy_(ff_0.linear.weight.data[ff_0_unpruned_indexes]) + if ff_0.linear.bias is not None: + new_ff_0.bias.data.copy_(ff_0.linear.bias.data[ff_0_unpruned_indexes]) + set_module(block, 'mlp.lin1', nn.Sequential(new_ff_0, StaticFBS(ff_0.cached_channel_attention[:, ff_0_unpruned_indexes]))) + + ff_1 = get_module(block, f'mlp.lin2') + new_ff_1 = nn.Linear(ff_0_unpruned_indexes.size(0), ff_1.out_features, ff_1.bias is not None) + new_ff_1.weight.data.copy_(ff_1.weight.data[:, ff_0_unpruned_indexes]) + if ff_1.bias is not None: + new_ff_1.bias.data.copy_(ff_1.bias.data) + set_module(block, 'mlp.lin2', new_ff_1) + + unpruned_indexes_of_layers[f'vision_encoder.layers.{block_i}.mlp.lin1.0.weight'] = ff_0_unpruned_indexes + # for block_i,block in enumerate(boosted_vit.vision_model.encoder.layers): + + # attn = block.self_attn + # ff = block.mlp + # ff_0 = ff.fc1 + # # ff_0_unpruned_indexes = get_unpruned_indexes_from_channel_attn(ff_0.cached_channel_attention, k) + # ff_0_pruned_indexes = ff_0.k_takes_all.cached_i[0].sort()[0] + # ff_0_unpruned_indexes = torch.LongTensor([ii for ii in range(ff_0.cached_channel_attention.size(1)) if ii not in ff_0_pruned_indexes]) + # new_ff_0 = nn.Linear(ff_0.linear.in_features, ff_0_unpruned_indexes.size(0), ff_0.linear.bias is not None) + # new_ff_0.weight.data.copy_(ff_0.linear.weight.data[ff_0_unpruned_indexes]) + # if ff_0.linear.bias is not None: + # new_ff_0.bias.data.copy_(ff_0.linear.bias.data[ff_0_unpruned_indexes]) + # set_module(ff, 'fc1', nn.Sequential(new_ff_0, StaticFBS(ff_0.cached_channel_attention[:, ff_0_unpruned_indexes]))) + + # ff_1 = ff.fc2 + # new_ff_1 = nn.Linear(ff_0_unpruned_indexes.size(0), ff_1.out_features, ff_1.bias is not None) + # new_ff_1.weight.data.copy_(ff_1.weight.data[:, ff_0_unpruned_indexes]) + # if ff_1.bias is not None: + # new_ff_1.bias.data.copy_(ff_1.bias.data) + # set_module(ff, 'fc2', new_ff_1) + + # unpruned_indexes_of_layers[f'vision_model.encoder.layers.{block_i}.mlp.fc1.0.weight'] = ff_0_unpruned_indexes + + + # for block_i, block in enumerate(boosted_vit.text_decoder.bert.encoder.layer): + # # attn = block.attn + # # ff = block.mlp + + # ff_0 = get_module(block, f'intermediate.dense') + # # ff_0_unpruned_indexes = get_unpruned_indexes_from_channel_attn(ff_0.cached_channel_attention, k) + # ff_0_pruned_indexes = ff_0.k_takes_all.cached_i[0].sort()[0] + # ff_0_unpruned_indexes = torch.LongTensor([ii for ii in range(ff_0.cached_channel_attention.size(1)) if ii not in ff_0_pruned_indexes]) + # new_ff_0 = nn.Linear(ff_0.linear.in_features, ff_0_unpruned_indexes.size(0), ff_0.linear.bias is not None) + # new_ff_0.weight.data.copy_(ff_0.linear.weight.data[ff_0_unpruned_indexes]) + # if ff_0.linear.bias is not None: + # new_ff_0.bias.data.copy_(ff_0.linear.bias.data[ff_0_unpruned_indexes]) + # set_module(block, 'intermediate.dense', nn.Sequential(new_ff_0, StaticFBS(ff_0.cached_channel_attention[:, ff_0_unpruned_indexes]))) + + # ff_1 = get_module(block, f'output.dense') + # new_ff_1 = nn.Linear(ff_0_unpruned_indexes.size(0), ff_1.out_features, ff_1.bias is not None) + # new_ff_1.weight.data.copy_(ff_1.weight.data[:, ff_0_unpruned_indexes]) + # if ff_1.bias is not None: + # new_ff_1.bias.data.copy_(ff_1.bias.data) + # set_module(block, 'output.dense', new_ff_1) + + # unpruned_indexes_of_layers[f'text_decoder.bert.encoder.layer.{block_i}.intermediate.dense.0.weight'] = ff_0_unpruned_indexes + surrogate_dnn = boosted_vit + surrogate_dnn.eval() + surrogate_dnn = surrogate_dnn.to(get_model_device(master_dnn)) + # logger.debug(surrogate_dnn) + with torch.no_grad(): + surrogate_dnn_output = surrogate_dnn(sample) + + output_diff = ((surrogate_dnn_output - master_dnn_output) ** 2).sum() + # assert output_diff < 1e-4, output_diff + logger.info(f'output diff of master and surrogate DNN: {output_diff}') + # logger.debug(f'example output of master/surrogate: {master_dnn_output.sum(0)[0: 10]}, {surrogate_dnn_output.sum(0)[0: 10]}') + # logger.info(f'\nonly prune mlp!!!!\n') + # logger.info(f'\nonly prune mlp!!!!\n') + + if return_detail: + return boosted_vit, unpruned_indexes_of_layers + + return boosted_vit + + def extract_surrogate_dnn_via_samples_with_perf_test(self, master_dnn: nn.Module, samples: torch.Tensor, return_detail=False): + master_dnn_size = get_model_size(master_dnn, True) + master_dnn_latency = self._get_model_latency(master_dnn, samples, 50, + get_model_device(master_dnn), 50, False) + + res = self.extract_surrogate_dnn_via_samples(master_dnn, samples, return_detail) + if not return_detail: + surrogate_dnn = res + else: + surrogate_dnn, unpruned_indexes_of_layers = res + surrogate_dnn_size = get_model_size(surrogate_dnn, True) + surrogate_dnn_latency = self._get_model_latency(master_dnn, samples, 50, + get_model_device(master_dnn), 50, False) + + logger.info(f'master DNN ({master_dnn_size:.3f}MB, {master_dnn_latency:.4f}s/sample) -> ' + f'surrogate DNN ({surrogate_dnn_size:.3f}MB, {surrogate_dnn_latency:.4f}s/sample)\n' + f'(model size: ↓ {(master_dnn_size / surrogate_dnn_size):.2f}x, ' + f'latency: ↓ {(master_dnn_latency / surrogate_dnn_latency):.2f}x)') + + return res + + def _get_model_latency(self, model: torch.nn.Module, model_input_size, sample_num: int, + device: str, warmup_sample_num: int, return_detail=False): + import time + + if isinstance(model_input_size, tuple): + dummy_input = torch.rand(model_input_size).to(device) + else: + dummy_input = model_input_size + + model = model.to(device) + model.eval() + + # warm up + with torch.no_grad(): + for _ in range(warmup_sample_num): + model(dummy_input) + + infer_time_list = [] + + if device == 'cuda' or 'cuda' in str(device): + with torch.no_grad(): + for _ in range(sample_num): + s, e = torch.cuda.Event(enable_timing=True), torch.cuda.Event(enable_timing=True) + s.record() + model(dummy_input) + e.record() + torch.cuda.synchronize() + cur_model_infer_time = s.elapsed_time(e) / 1000. + infer_time_list += [cur_model_infer_time] + + else: + with torch.no_grad(): + for _ in range(sample_num): + start = time.time() + model(**dummy_input) + cur_model_infer_time = time.time() - start + infer_time_list += [cur_model_infer_time] + + avg_infer_time = sum(infer_time_list) / sample_num + + if return_detail: + return avg_infer_time, infer_time_list + return avg_infer_time + + + +#####Here starts with online diff --git a/new_impl/cv/sam/sam_pretrained/config.json b/new_impl/cv/sam/sam_pretrained/config.json new file mode 100644 index 0000000000000000000000000000000000000000..56bee7113ac71e5fa960709353bc8aee87d9623b --- /dev/null +++ b/new_impl/cv/sam/sam_pretrained/config.json @@ -0,0 +1,249 @@ +{ + "_commit_hash": null, + "_name_or_path": "/tmp/facebook/sam-vit-base", + "architectures": [ + "SamModel" + ], + "initializer_range": 0.02, + "mask_decoder_config": { + "_name_or_path": "", + "add_cross_attention": false, + "architectures": null, + "attention_downsample_rate": 2, + "bad_words_ids": null, + "begin_suppress_tokens": null, + "bos_token_id": null, + "chunk_size_feed_forward": 0, + "cross_attention_hidden_size": null, + "decoder_start_token_id": null, + "diversity_penalty": 0.0, + "do_sample": false, + "early_stopping": false, + "encoder_no_repeat_ngram_size": 0, + "eos_token_id": null, + "exponential_decay_length_penalty": null, + "finetuning_task": null, + "forced_bos_token_id": null, + "forced_eos_token_id": null, + "hidden_act": "relu", + "hidden_size": 256, + "id2label": { + "0": "LABEL_0", + "1": "LABEL_1" + }, + "iou_head_depth": 3, + "iou_head_hidden_dim": 256, + "is_decoder": false, + "is_encoder_decoder": false, + "label2id": { + "LABEL_0": 0, + "LABEL_1": 1 + }, + "layer_norm_eps": 1e-06, + "length_penalty": 1.0, + "max_length": 20, + "min_length": 0, + "mlp_dim": 2048, + "model_type": "", + "no_repeat_ngram_size": 0, + "num_attention_heads": 8, + "num_beam_groups": 1, + "num_beams": 1, + "num_hidden_layers": 2, + "num_multimask_outputs": 3, + "num_return_sequences": 1, + "output_attentions": false, + "output_hidden_states": false, + "output_scores": false, + "pad_token_id": null, + "prefix": null, + "problem_type": null, + "pruned_heads": {}, + "remove_invalid_values": false, + "repetition_penalty": 1.0, + "return_dict": true, + "return_dict_in_generate": false, + "sep_token_id": null, + "suppress_tokens": null, + "task_specific_params": null, + "temperature": 1.0, + "tf_legacy_loss": false, + "tie_encoder_decoder": false, + "tie_word_embeddings": true, + "tokenizer_class": null, + "top_k": 50, + "top_p": 1.0, + "torch_dtype": null, + "torchscript": false, + "transformers_version": "4.29.0.dev0", + "typical_p": 1.0, + "use_bfloat16": false + }, + "model_type": "sam", + "prompt_encoder_config": { + "_name_or_path": "", + "add_cross_attention": false, + "architectures": null, + "bad_words_ids": null, + "begin_suppress_tokens": null, + "bos_token_id": null, + "chunk_size_feed_forward": 0, + "cross_attention_hidden_size": null, + "decoder_start_token_id": null, + "diversity_penalty": 0.0, + "do_sample": false, + "early_stopping": false, + "encoder_no_repeat_ngram_size": 0, + "eos_token_id": null, + "exponential_decay_length_penalty": null, + "finetuning_task": null, + "forced_bos_token_id": null, + "forced_eos_token_id": null, + "hidden_act": "gelu", + "hidden_size": 256, + "id2label": { + "0": "LABEL_0", + "1": "LABEL_1" + }, + "image_embedding_size": 64, + "image_size": 1024, + "is_decoder": false, + "is_encoder_decoder": false, + "label2id": { + "LABEL_0": 0, + "LABEL_1": 1 + }, + "layer_norm_eps": 1e-06, + "length_penalty": 1.0, + "mask_input_channels": 16, + "max_length": 20, + "min_length": 0, + "model_type": "", + "no_repeat_ngram_size": 0, + "num_beam_groups": 1, + "num_beams": 1, + "num_point_embeddings": 4, + "num_return_sequences": 1, + "output_attentions": false, + "output_hidden_states": false, + "output_scores": false, + "pad_token_id": null, + "patch_size": 4, + "prefix": null, + "problem_type": null, + "pruned_heads": {}, + "remove_invalid_values": false, + "repetition_penalty": 1.0, + "return_dict": true, + "return_dict_in_generate": false, + "sep_token_id": null, + "suppress_tokens": null, + "task_specific_params": null, + "temperature": 1.0, + "tf_legacy_loss": false, + "tie_encoder_decoder": false, + "tie_word_embeddings": true, + "tokenizer_class": null, + "top_k": 50, + "top_p": 1.0, + "torch_dtype": null, + "torchscript": false, + "transformers_version": "4.29.0.dev0", + "typical_p": 1.0, + "use_bfloat16": false + }, + "torch_dtype": "float32", + "transformers_version": null, + "vision_config": { + "_name_or_path": "", + "add_cross_attention": false, + "architectures": null, + "attention_dropout": 0.0, + "bad_words_ids": null, + "begin_suppress_tokens": null, + "bos_token_id": null, + "chunk_size_feed_forward": 0, + "cross_attention_hidden_size": null, + "decoder_start_token_id": null, + "diversity_penalty": 0.0, + "do_sample": false, + "dropout": 0.0, + "early_stopping": false, + "encoder_no_repeat_ngram_size": 0, + "eos_token_id": null, + "exponential_decay_length_penalty": null, + "finetuning_task": null, + "forced_bos_token_id": null, + "forced_eos_token_id": null, + "global_attn_indexes": [ + 2, + 5, + 8, + 11 + ], + "hidden_act": "gelu", + "hidden_size": 768, + "id2label": { + "0": "LABEL_0", + "1": "LABEL_1" + }, + "image_size": 224, + "initializer_factor": 1.0, + "initializer_range": 1e-10, + "intermediate_size": 6144, + "is_decoder": false, + "is_encoder_decoder": false, + "label2id": { + "LABEL_0": 0, + "LABEL_1": 1 + }, + "layer_norm_eps": 1e-06, + "length_penalty": 1.0, + "max_length": 20, + "min_length": 0, + "mlp_dim": 3072, + "mlp_ratio": 4.0, + "model_type": "", + "no_repeat_ngram_size": 0, + "num_attention_heads": 12, + "num_beam_groups": 1, + "num_beams": 1, + "num_channels": 3, + "num_hidden_layers": 12, + "num_pos_feats": 128, + "num_return_sequences": 1, + "output_attentions": false, + "output_channels": 256, + "output_hidden_states": false, + "output_scores": false, + "pad_token_id": null, + "patch_size": 16, + "prefix": null, + "problem_type": null, + "projection_dim": 512, + "pruned_heads": {}, + "qkv_bias": true, + "remove_invalid_values": false, + "repetition_penalty": 1.0, + "return_dict": true, + "return_dict_in_generate": false, + "sep_token_id": null, + "suppress_tokens": null, + "task_specific_params": null, + "temperature": 1.0, + "tf_legacy_loss": false, + "tie_encoder_decoder": false, + "tie_word_embeddings": true, + "tokenizer_class": null, + "top_k": 50, + "top_p": 1.0, + "torch_dtype": null, + "torchscript": false, + "transformers_version": "4.29.0.dev0", + "typical_p": 1.0, + "use_abs_pos": true, + "use_bfloat16": false, + "use_rel_pos": true, + "window_size": 14 + } +} diff --git a/new_impl/cv/sam/seg.py b/new_impl/cv/sam/seg.py new file mode 100644 index 0000000000000000000000000000000000000000..9545c957121a41e544a9544abe698f97df4e6363 --- /dev/null +++ b/new_impl/cv/sam/seg.py @@ -0,0 +1,111 @@ +import torch +from new_impl.cv.elasticdnn.api.model import ElasticDNN_OfflineSegFMModel, ElasticDNN_OfflineSegMDModel +from new_impl.cv.elasticdnn.api.algs.fm_lora import ElasticDNN_FMLoRAAlg +from new_impl.cv.elasticdnn.model.base import ElasticDNNUtil +from new_impl.cv.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from sam import FMLoRA_sam_Util +from new_impl.cv.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from sam import FM_to_MD_sam_Util +from sam import ElasticsamUtil +from utils.dl.common.model import LayerActivation, get_module, get_parameter, set_module +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +import torch.nn.functional as F + + +class ElasticDNN_sam_OfflineSegFMModel(ElasticDNN_OfflineSegFMModel): + def generate_md_by_reducing_width(self, reducing_width_ratio, samples: torch.Tensor): + return FM_to_MD_sam_Util().init_md_from_fm_by_reducing_width_with_perf_test(self.models_dict['main'], + reducing_width_ratio, samples) + + def get_feature_hook(self) -> LayerActivation: + return LayerActivation(get_module(self.models_dict['main'], 'head'), True, self.device) + + def get_elastic_dnn_util(self) -> ElasticDNNUtil: + return ElasticsamUtil() + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + return F.cross_entropy(self.infer(x), y) + + def get_lora_util(self) -> FMLoRA_Util: + return FMLoRA_sam_Util() + + def get_task_head_params(self): + head = get_module(self.models_dict['main'], 'head') + return list(head.parameters()) + + +class ElasticDNN_sam_OfflineSegMDModel(ElasticDNN_OfflineSegMDModel): + def get_feature_hook(self) -> LayerActivation: + return LayerActivation(get_module(self.models_dict['main'], 'head'), True, self.device) + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + return F.cross_entropy(self.infer(x).pred_masks, y) + + +if __name__ == '__main__': + # 1. init scenario + scenario = build_scenario( + source_datasets_name=['GTA5', 'SuperviselyPerson'], + target_datasets_order=['Cityscapes', 'BaiduPerson'] * 10, + da_mode='close_set', + data_dirs={ + 'GTA5': '/data/zql/datasets/GTA-ls-copy/GTA5', + 'SuperviselyPerson': '/data/zql/datasets/supervisely_person/Supervisely Person Dataset', + 'Cityscapes': '/data/zql/datasets/cityscape/', + 'BaiduPerson': '/data/zql/datasets/baidu_person/clean_images/' + }, + ) + + # 2. init model\ + torch.cuda.set_device(1) + device = 'cuda' + # from dnns.vit import vit_b_16 + # seg_model = vit_b_16(pretrained=True, num_classes=scenario.num_classes) + # from dnns.deeplabv3.head import DecoderLinear + # head = DecoderLinear(scenario.num_classes, 16, 768, (224, 224)).to(device) + # set_module(seg_model, 'head', head) + + # from types import MethodType + # from timm.models.vision_transformer import VisionTransformer + # def forward_head(self, x, pre_logits: bool = False): + # return self.head(x) + # VisionTransformer.forward_head = MethodType(forward_head, seg_model) + from sam import Sammodel + seg_model = Sammodel.from_pretrained('new_impl/cv/sam/sam_pretrained',ignore_mismatched_sizes=True,num_classes=scenario.num_classes) + # from dnns.deeplabv3.head import DecoderLinear + # head = DecoderLinear(scenario.num_classes, 16, 256, (224, 224)).to(device) + # set_module(seg_model, 'mask_decoder.iou_prediction_head', head) + + fm_models_dict_path = save_models_dict_for_init({ + 'main': seg_model + }, __file__, 'fm_sam_pretrained_with_seg_head') + + fm_model = ElasticDNN_sam_OfflineSegFMModel('fm', fm_models_dict_path, device, scenario.num_classes) + + # 3. init alg + models = { + 'fm': fm_model + } + fm_lora_alg = ElasticDNN_FMLoRAAlg(models, get_res_save_dir(__file__)) + + from PIL import ImageFile + ImageFile.LOAD_TRUNCATED_IMAGES = True + + # 4. run alg + from utils.dl.common.lr_scheduler import get_linear_schedule_with_warmup + fm_lora_alg.run(scenario, hyps={ + 'launch_tbboard': False, + + 'samples_size': (1, 3, 224, 224), + 'ab_r': 8, + 'train_batch_size': 16, + 'val_batch_size': 256, + 'num_workers': 16, + 'optimizer': 'Adam', + 'optimizer_args': {'lr': 5e-3, 'betas': [0.9, 0.999]}, + 'scheduler': 'LambdaLR', + 'scheduler_args': {'lr_lambda': get_linear_schedule_with_warmup(10000, 70000)}, + 'num_iters': 80000, + 'val_freq': 400 + }) \ No newline at end of file diff --git a/new_impl/cv/sam/seg_baseline.py b/new_impl/cv/sam/seg_baseline.py new file mode 100644 index 0000000000000000000000000000000000000000..7097e225cc908dcda9ea4aec51aa69cc230a2f37 --- /dev/null +++ b/new_impl/cv/sam/seg_baseline.py @@ -0,0 +1,138 @@ +from typing import List +from data.dataloader import build_dataloader +# from methods.elasticdnn.api.online_model import ElasticDNN_OnlineModel +from new_impl.cv.elasticdnn.api.online_model_v2 import ElasticDNN_OnlineModel + +import torch +import sys +from torch import nn +from new_impl.cv.elasticdnn.api.model import ElasticDNN_OfflineSegFMModel, ElasticDNN_OfflineSegMDModel +from new_impl.cv.elasticdnn.api.algs.md_pretraining_wo_fbs import ElasticDNN_MDPretrainingWoFBSAlg +from new_impl.cv.elasticdnn.model.base import ElasticDNNUtil +from new_impl.cv.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from sam import FM_to_MD_sam_Util +from new_impl.cv.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from sam import FMLoRA_sam_Util +from sam import ElasticsamUtil +from utils.common.file import ensure_dir +from utils.dl.common.model import LayerActivation, get_module, get_parameter +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +from utils.dl.common.loss import CrossEntropyLossSoft +import torch.nn.functional as F +from utils.dl.common.env import create_tbwriter +import os +from utils.common.log import logger +from utils.common.data_record import write_json +# from methods.shot.shot import OnlineShotModel +from new_impl.cv.feat_align.main import OnlineFeatAlignModel, FeatAlignAlg +import tqdm +from new_impl.cv.feat_align.mmd import mmd_rbf +from new_impl.cv.utils.baseline_da import baseline_da + +device = 'cuda' +app_name = 'cls' + +scenario = build_scenario( + source_datasets_name=['GTA5', 'SuperviselyPerson'], + target_datasets_order=['Cityscapes', 'BaiduPerson'] * 10, + da_mode='close_set', + data_dirs={ + 'GTA5': '/data/zql/datasets/GTA-ls-copy/GTA5', + 'SuperviselyPerson': '/data/zql/datasets/supervisely_person/Supervisely Person Dataset', + 'Cityscapes': '/data/zql/datasets/cityscape/', + 'BaiduPerson': '/data/zql/datasets/baidu_person/clean_images/' + }, + ) +class SegOnlineFeatAlignModel(OnlineFeatAlignModel): + def __init__(self, name: str, models_dict_path: str, device: str, num_classes): + super().__init__(name, models_dict_path, device) + self.num_classes = num_classes + + def get_feature_hook(self): + return LayerActivation(get_module(self.models_dict['main'], 'head'), False, self.device) + + def forward_to_get_task_loss(self, x, y): + return F.cross_entropy(self.infer(x), y) + + def get_mmd_loss(self, f1, f2): + return mmd_rbf(f1.flatten(1), f2.flatten(1)) + + def infer(self, x, *args, **kwargs): + return self.models_dict['main'](x) + + def get_trained_params(self): + qkv_and_norm_params = [p for n, p in self.models_dict['main'].named_parameters() if 'qkv.weight' in n or 'norm' in n or 'mlp' in n] + return qkv_and_norm_params + + def infer(self, x, *args, **kwargs): + return self.models_dict['main'](x) + + def get_accuracy(self, test_loader, *args, **kwargs): + device = self.device + self.to_eval_mode() + from methods.elasticdnn.api.model import StreamSegMetrics + metrics = StreamSegMetrics(self.num_classes) + metrics.reset() + import tqdm + pbar = tqdm.tqdm(enumerate(test_loader), total=len(test_loader), leave=False, dynamic_ncols=True) + with torch.no_grad(): + for batch_index, (x, y) in pbar: + x, y = x.to(device, dtype=x.dtype, non_blocking=True, copy=False), \ + y.to(device, dtype=y.dtype, non_blocking=True, copy=False) + output = self.infer(x) + pred = output.detach().max(dim=1)[1].cpu().numpy() + metrics.update((y + 0).cpu().numpy(), pred) + + res = metrics.get_results() + pbar.set_description(f'cur batch mIoU: {res["Mean Acc"]:.4f}') + + res = metrics.get_results() + return res['Mean Acc'] + + +da_alg = FeatAlignAlg +#from experiments.cua.vit_b_16.online.cls.model import ClsOnlineFeatAlignModel +da_model = SegOnlineFeatAlignModel( + app_name, + 'new_impl/cv/sam/results/seg_wo_fbs.py/20231130/999999-144157/models/md_best.pt', + device, + scenario.num_classes +) +da_alg_hyp = {'Cityscapes': { + 'train_batch_size': 16, + 'val_batch_size': 128, + 'num_workers': 16, + 'optimizer': 'AdamW', + 'optimizer_args': {'lr': 1e-9, 'betas': [0.9, 0.999], 'weight_decay': 0.01}, + 'scheduler': '', + 'scheduler_args': {}, + 'num_iters': 10, + 'val_freq': 20, + # 'sd_sparsity': 0.8, + 'feat_align_loss_weight': 3.0 +}, 'BaiduPerson': { + 'train_batch_size': 16, + 'val_batch_size': 128, + 'num_workers': 16, + 'optimizer': 'AdamW', + 'optimizer_args': {'lr': 1e-2, 'betas': [0.9, 0.999], 'weight_decay': 0.01}, + 'scheduler': '', + 'scheduler_args': {}, + 'num_iters': 10, + 'val_freq': 20, + # 'sd_sparsity': 0.8, + 'feat_align_loss_weight': 0.3 +}} + + +baseline_da( + app_name, + scenario, + da_alg, + da_alg_hyp, + da_model, + device, + __file__, + sys.argv[0] +) diff --git a/new_impl/cv/sam/seg_online.py b/new_impl/cv/sam/seg_online.py new file mode 100644 index 0000000000000000000000000000000000000000..69fad6c179efdb15e70e17ab0324c9376885a8db --- /dev/null +++ b/new_impl/cv/sam/seg_online.py @@ -0,0 +1,309 @@ +from typing import List +from data.dataloader import build_dataloader +# from methods.elasticdnn.api.online_model import ElasticDNN_OnlineModel +from new_impl.cv.elasticdnn.api.online_model_v2 import ElasticDNN_OnlineModel + +import torch +import sys +from torch import nn +from new_impl.cv.elasticdnn.api.model import ElasticDNN_OfflineSegFMModel, ElasticDNN_OfflineSegMDModel +from new_impl.cv.elasticdnn.api.algs.md_pretraining_wo_fbs import ElasticDNN_MDPretrainingWoFBSAlg +from new_impl.cv.elasticdnn.model.base import ElasticDNNUtil +from new_impl.cv.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from new_impl.cv.elasticdnn.pipeline.offline.fm_to_md.vit import FM_to_MD_ViT_Util +from new_impl.cv.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from new_impl.cv.elasticdnn.pipeline.offline.fm_lora.vit import FMLoRA_ViT_Util +from sam import ElasticsamUtil +from utils.common.file import ensure_dir +from utils.dl.common.model import LayerActivation, get_module, get_parameter +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +from utils.dl.common.loss import CrossEntropyLossSoft +import torch.nn.functional as F +from utils.dl.common.env import create_tbwriter +import os +from utils.common.log import logger +from utils.common.data_record import write_json +# from methods.shot.shot import OnlineShotModel +from new_impl.cv.feat_align.main import OnlineFeatAlignModel, FeatAlignAlg +import tqdm +from new_impl.cv.feat_align.mmd import mmd_rbf +from new_impl.cv.utils.elasticfm_da import init_online_model, elasticfm_da +torch.cuda.set_device(1) +device = 'cuda' +app_name = 'seg' +sd_sparsity = 0. + +settings = { + 'involve_fm': True +} + +scenario = build_scenario( + source_datasets_name=['GTA5', 'SuperviselyPerson'], + target_datasets_order=['Cityscapes', 'BaiduPerson'] * 10, + da_mode='close_set', + data_dirs={ + 'GTA5': '/data/zql/datasets/GTA-ls-copy/GTA5', + 'SuperviselyPerson': '/data/zql/datasets/supervisely_person/Supervisely Person Dataset', + 'Cityscapes': '/data/zql/datasets/cityscape/', + 'BaiduPerson': '/data/zql/datasets/baidu_person/clean_images/' + }, +) + + +class ElasticDNN_SegOnlineModel(ElasticDNN_OnlineModel): + def __init__(self, name: str, models_dict_path: str, device: str, ab_options: dict, num_classes: int): + super().__init__(name, models_dict_path, device, ab_options) + self.num_classes = num_classes + + def get_accuracy(self, test_loader, *args, **kwargs): + device = self.device + self.to_eval_mode() + from methods.elasticdnn.api.model import StreamSegMetrics + metrics = StreamSegMetrics(self.num_classes) + metrics.reset() + import tqdm + pbar = tqdm.tqdm(enumerate(test_loader), total=len(test_loader), leave=False, dynamic_ncols=True) + with torch.no_grad(): + for batch_index, (x, y) in pbar: + x, y = x.to(device, dtype=x.dtype, non_blocking=True, copy=False), \ + y.to(device, dtype=y.dtype, non_blocking=True, copy=False) + output = self.infer(x) + pred = output.detach().max(dim=1)[1].cpu().numpy() + metrics.update((y + 0).cpu().numpy(), pred) + + res = metrics.get_results() + pbar.set_description(f'cur batch mIoU: {res["Mean Acc"]:.4f}') + + res = metrics.get_results() + return res['Mean Acc'] + + def get_elastic_dnn_util(self) -> ElasticDNNUtil: + return ElasticsamUtil() + + def get_fm_matched_param_of_md_param(self, md_param_name): + # only between qkv.weight, norm.weight/bias + self_param_name = md_param_name + fm = self.models_dict['fm'] + if any([k in self_param_name for k in ['fbs', 'cls_token', 'pos_embed']]): + return None + + p = get_parameter(self.models_dict['md'], self_param_name) + if p.dim() == 0: + return None + elif p.dim() == 1 and 'layernorm' in self_param_name and 'weight' in self_param_name: + if self_param_name.startswith('norm'): + return None + return get_parameter(fm, self_param_name) + + # 1. xx.qkv.to_qkv.yy to xx.qkv.qkv.aa and xx.qkv.abs.zz + if 'qkv.weight' in self_param_name: + ss = self_param_name.split('.') + + fm_qkv_name = '.'.join(ss[0: -1]) + '.fc' + fm_qkv = get_module(fm, fm_qkv_name) + + fm_abs_name = '.'.join(ss[0: -1]) + '.ab' + fm_abs = get_module(fm, fm_abs_name) + + # NOTE: unrecoverable operation! multiply LoRA parameters to allow it being updated in update_fm_param() + # TODO: if fm will be used for inference, _mul_lora_weight will not be applied! + if not hasattr(fm_abs, '_mul_lora_weight'): + logger.debug(f'set _mul_lora_weight in {fm_abs_name}') + setattr(fm_abs, '_mul_lora_weight', + nn.Parameter(fm_abs[1].weight @ fm_abs[0].weight)) + + return torch.cat([ + fm_qkv.weight.data, # task-agnositc params + fm_abs._mul_lora_weight.data # task-specific params (LoRA) + ], dim=0) + + # elif 'to_qkv.bias' in self_param_name: + # ss = self_param_name.split('.') + + # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv.bias' + # return get_parameter(fm, fm_qkv_name) + + elif 'mlp.lin1' in self_param_name and 'weight' in self_param_name: + fm_param_name = self_param_name.replace('.linear', '') + return get_parameter(fm, fm_param_name) + + elif 'mlp.lin2' in self_param_name and 'weight' in self_param_name: + fm_param_name = self_param_name + return get_parameter(fm, fm_param_name) + + else: + # return get_parameter(fm, self_param_name) + return None + + def update_fm_param(self, md_param_name, cal_new_fm_param_by_md_param): + if not 'qkv.weight' in md_param_name: + matched_fm_param_ref = self.get_fm_matched_param_of_md_param(md_param_name) + matched_fm_param_ref.copy_(cal_new_fm_param_by_md_param) + else: + new_fm_attn_weight, new_fm_lora_weight = torch.chunk(cal_new_fm_param_by_md_param, 2, 0) + + ss = md_param_name.split('.') + fm = self.models_dict['fm'] + + # update task-agnostic parameters + fm_qkv_name = '.'.join(ss[0: -1]) + '.fc' + fm_qkv = get_module(fm, fm_qkv_name) + fm_qkv.weight.data.copy_(new_fm_attn_weight) + + # update task-specific parameters + fm_abs_name = '.'.join(ss[0: -1]) + '.ab' + fm_abs = get_module(fm, fm_abs_name) + fm_abs._mul_lora_weight.data.copy_(new_fm_lora_weight) # TODO: this will not be applied in inference! + + def get_md_matched_param_of_fm_param(self, fm_param_name): + return super().get_md_matched_param_of_fm_param(fm_param_name) + + def get_md_matched_param_of_sd_param(self, sd_param_name): + # only between qkv.weight, norm.weight/bias + self_param_name = sd_param_name + md = self.models_dict['md'] + if any([k in self_param_name for k in ['fbs', 'cls_token', 'pos_embed']]): + return None + + p = get_parameter(self.models_dict['sd'], self_param_name) + if p.dim() == 0: + return None + elif p.dim() == 1 and 'layernorm' in self_param_name and 'weight' in self_param_name: + return get_parameter(md, self_param_name) + + # 1. xx.qkv.to_qkv.yy to xx.qkv.qkv.aa and xx.qkv.abs.zz + if 'qkv.weight' in self_param_name: + return get_parameter(md, self_param_name) + + # elif 'to_qkv.bias' in self_param_name: + # ss = self_param_name.split('.') + + # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv.bias' + # return get_parameter(fm, fm_qkv_name) + + elif 'mlp.lin1.0.weight' in self_param_name: + fm_param_name = '.'.join(self_param_name.split('.')[0: -2]) + '.linear.weight' + return get_parameter(md, fm_param_name) + + elif 'mlp.lin2' in self_param_name and 'weight' in self_param_name: + fm_param_name = self_param_name + return get_parameter(md, fm_param_name) + + else: + # return get_parameter(fm, self_param_name) + return None + + def get_task_head_params(self): + head = get_module(self.models_dict['sd'], 'head') + return list(head.parameters()) + + +class SegOnlineFeatAlignModel(OnlineFeatAlignModel): + def __init__(self, name: str, models_dict_path: str, device: str, num_classes): + super().__init__(name, models_dict_path, device) + self.num_classes = num_classes + + def get_feature_hook(self): + return LayerActivation(get_module(self.models_dict['main'], 'head'), False, self.device) + + def forward_to_get_task_loss(self, x, y): + return F.cross_entropy(self.infer(x), y) + + def get_mmd_loss(self, f1, f2): + return mmd_rbf(f1.flatten(1), f2.flatten(1)) + + def infer(self, x, *args, **kwargs): + return self.models_dict['main'](x) + + def get_trained_params(self): + qkv_and_norm_params = [p for n, p in self.models_dict['main'].named_parameters() if 'qkv.weight' in n or 'norm' in n or 'mlp' in n] + return qkv_and_norm_params + + def infer(self, x, *args, **kwargs): + return self.models_dict['main'](x) + + def get_accuracy(self, test_loader, *args, **kwargs): + device = self.device + self.to_eval_mode() + from methods.elasticdnn.api.model import StreamSegMetrics + metrics = StreamSegMetrics(self.num_classes) + metrics.reset() + import tqdm + pbar = tqdm.tqdm(enumerate(test_loader), total=len(test_loader), leave=False, dynamic_ncols=True) + with torch.no_grad(): + for batch_index, (x, y) in pbar: + x, y = x.to(device, dtype=x.dtype, non_blocking=True, copy=False), \ + y.to(device, dtype=y.dtype, non_blocking=True, copy=False) + output = self.infer(x) + pred = output.detach().max(dim=1)[1].cpu().numpy() + metrics.update((y + 0).cpu().numpy(), pred) + + res = metrics.get_results() + pbar.set_description(f'cur batch mIoU: {res["Mean Acc"]:.4f}') + + res = metrics.get_results() + return res['Mean Acc'] + + + + + + +#from new_impl.cv.model import ElasticDNN_ClsOnlineModel +elasticfm_model = ElasticDNN_SegOnlineModel('cls', init_online_model( + # 'experiments/elasticdnn/vit_b_16/offline/fm_to_md/results/cls_md_index.py/20230529/star_999997-154037-only_prune_mlp/models/fm_best.pt', + # 'experiments/elasticdnn/vit_b_16/offline/fm_to_md/results/cls_md_index.py/20230529/star_999997-154037-only_prune_mlp/models/md_best.pt', + #'experiments/elasticdnn/vit_b_16/offline/fm_to_md/cls/results/cls_md_index.py/20230617/999992-101343-lr1e-5_index_bug_fixed/models/fm_best.pt', + #'experiments/elasticdnn/vit_b_16/offline/fm_to_md/cls/results/cls_md_index.py/20230617/999992-101343-lr1e-5_index_bug_fixed/models/md_best.pt', + 'new_impl/cv/sam/results/seg_wo_index.py/20231125/999999-175801-/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/new_impl/cv/sam/seg_wo_index.py/models/fm_best.pt', + 'new_impl/cv/sam/results/seg_wo_index.py/20231125/999999-175801-/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/new_impl/cv/sam/seg_wo_index.py/models/md_best.pt', + 'seg', __file__ +), device, { + 'md_to_fm_alpha': 0.1, + 'fm_to_md_alpha': 0.1 +},scenario.num_classes) + +da_alg = FeatAlignAlg +from utils.dl.common.lr_scheduler import get_linear_schedule_with_warmup +#from new_impl.cv.model import ClsOnlineFeatAlignModel +da_model = SegOnlineFeatAlignModel +da_alg_hyp = {'Cityscapes': { + 'train_batch_size': 16, + 'val_batch_size': 128, + 'num_workers': 16, + 'optimizer': 'AdamW', + 'optimizer_args': {'lr': 3e-5, 'betas': [0.9, 0.999], 'weight_decay': 0.01}, + 'scheduler': '', + 'scheduler_args': {}, + 'num_iters': 100, + 'val_freq': 20, + 'sd_sparsity': 0.5, + 'feat_align_loss_weight': 0.3 +}, 'BaiduPerson': { + 'train_batch_size': 16, + 'val_batch_size': 128, + 'num_workers': 16, + 'optimizer': 'AdamW', + 'optimizer_args': {'lr': 1e-7,'betas': [0.9, 0.999], 'weight_decay': 0.01}, + 'scheduler': '', + 'scheduler_args': {}, + 'num_iters': 100, + 'val_freq': 20, + 'sd_sparsity': 0.5, + 'feat_align_loss_weight': 0.3 +}} + + +elasticfm_da( + [app_name], + [scenario], + [elasticfm_model], + [da_alg], + [da_alg_hyp], + [da_model], + device, + settings, + __file__, + sys.argv[0] +) diff --git a/new_impl/cv/sam/seg_wo_fbs.py b/new_impl/cv/sam/seg_wo_fbs.py new file mode 100644 index 0000000000000000000000000000000000000000..b6655e7377bc2a9c786e0960c89ea07468f6366d --- /dev/null +++ b/new_impl/cv/sam/seg_wo_fbs.py @@ -0,0 +1,156 @@ +import torch +import sys +from torch import nn +from new_impl.cv.elasticdnn.api.model import ElasticDNN_OfflineSegFMModel, ElasticDNN_OfflineSegMDModel +from new_impl.cv.elasticdnn.api.algs.md_pretraining_wo_fbs import ElasticDNN_MDPretrainingWoFBSAlg +from new_impl.cv.elasticdnn.model.base import ElasticDNNUtil +from new_impl.cv.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from sam import FM_to_MD_sam_Util +from new_impl.cv.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from sam import FMLoRA_sam_Util +from sam import ElasticsamUtil +from utils.dl.common.model import LayerActivation, get_module, get_parameter +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +from utils.dl.common.loss import CrossEntropyLossSoft +import torch.nn.functional as F + + +class ElasticDNN_ViT_OfflineSegFMModel(ElasticDNN_OfflineSegFMModel): + def generate_md_by_reducing_width(self, reducing_width_ratio, samples: torch.Tensor): + return FM_to_MD_sam_Util().init_md_from_fm_by_reducing_width_with_perf_test(self.models_dict['main'], + reducing_width_ratio, samples).to(self.device) + + def get_feature_hook(self) -> LayerActivation: + return LayerActivation(get_module(self.models_dict['main'], 'head'), True, self.device) + + def get_elastic_dnn_util(self) -> ElasticDNNUtil: + return ElasticsamUtil() + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + return F.cross_entropy(self.infer(x), y) + + def get_lora_util(self) -> FMLoRA_Util: + return FMLoRA_sam_Util() + + def get_task_head_params(self): + head = get_module(self.models_dict['main'], 'head') + return list(head.parameters()) + + +class ElasticDNN_ViT_OfflineSegMDModel(ElasticDNN_OfflineSegMDModel): + + def get_feature_hook(self) -> LayerActivation: + return LayerActivation(get_module(self.models_dict['main'], 'head'), True, self.device) + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + return F.cross_entropy(self.infer(x), y) + + def get_distill_loss(self, student_output, teacher_output): + return F.mse_loss(student_output, teacher_output) + + def get_matched_param_of_fm(self, self_param_name, fm: nn.Module): + if any([k in self_param_name for k in ['fbs', 'cls_token', 'pos_embed']]): + return None + + # 1. xx.qkv.to_qkv.yy to xx.qkv.qkv.aa and xx.qkv.abs.zz + if 'to_qkv.weight' in self_param_name: + ss = self_param_name.split('.') + + fm_qkv_name = '.'.join(ss[0: -2]) + '.fc' + fm_qkv = get_module(fm, fm_qkv_name) + + fm_abs_name = '.'.join(ss[0: -2]) + '.ab' + fm_abs = get_module(fm, fm_abs_name) + + return torch.cat([ + fm_qkv.weight.data, # task-agnositc params + torch.cat([(_abs[0].weight.T @ _abs[1].weight.T).T for _abs in fm_abs], dim=0) # task-specific params (LoRA) + ], dim=0) + + elif 'to_qkv.bias' in self_param_name: + ss = self_param_name.split('.') + + fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv.bias' + return get_parameter(fm, fm_qkv_name) + + elif 'mlp.fc1' in self_param_name: + fm_param_name = self_param_name.replace('.linear', '') + return get_parameter(fm, fm_param_name) + + else: + return get_parameter(fm, self_param_name) + + +if __name__ == '__main__': + from utils.dl.common.env import set_random_seed + set_random_seed(1) + + # 3. init scenario + scenario = build_scenario( + source_datasets_name=['GTA5', 'SuperviselyPerson'], + target_datasets_order=['Cityscapes', 'BaiduPerson'] * 10, + da_mode='close_set', + data_dirs={ + 'GTA5': '/data/zql/datasets/GTA-ls-copy/GTA5', + 'SuperviselyPerson': '/data/zql/datasets/supervisely_person/Supervisely Person Dataset', + 'Cityscapes': '/data/zql/datasets/cityscape/', + 'BaiduPerson': '/data/zql/datasets/baidu_person/clean_images/' + }, + ) + + # 1. init model + # from dnns.deeplabv3.head import modify_forward_head + # modify_forward_head() # TODO: bring a bug + from dnns.vit import vit_b_16 + fm_models_dict_path = 'new_impl/cv/sam/results/seg.py/20231123/999983-212616/models/fm_best.pt' + fm_models = torch.load(fm_models_dict_path) + # for n,m in fm_models['main'].named_modules(): + # print(n) + # from utils.dl.common.model import set_module + # set_module( + # fm_models['main'], + # 'norm', + # nn.Sequential( + # get_module(fm_models['main'], 'norm'), + # get_module(fm_models['main'], 'head') + # ) + # ) + # set_module(fm_models['main'], 'head', nn.Identity()) + # fm_models['main'].forward = fm_models['main'].forward_features + + fm_models_dict_path = save_models_dict_for_init(fm_models, __file__, 'fm_sam_seg_lora') + md_models_dict_path = save_models_dict_for_init({ + 'main': -1 + }, __file__, 'md_sam_none') + device = 'cuda' + + fm_model = ElasticDNN_ViT_OfflineSegFMModel('fm', fm_models_dict_path, device, scenario.num_classes) + md_model = ElasticDNN_ViT_OfflineSegMDModel('md', md_models_dict_path, device, scenario.num_classes) + + # 2. init alg + models = { + 'fm': fm_model, + 'md': md_model + } + fm_to_md_alg = ElasticDNN_MDPretrainingWoFBSAlg(models, get_res_save_dir(__file__, None)) + + from utils.dl.common.lr_scheduler import get_linear_schedule_with_warmup + fm_to_md_alg.run(scenario, hyps={ + 'launch_tbboard': False, + + 'samples_size': (1, 3, 224, 224), + 'generate_md_width_ratio': 8, + + 'train_batch_size': 16, + 'val_batch_size': 128, + 'num_workers': 16, + 'optimizer': 'AdamW', + 'optimizer_args': {'lr': 5e-4, 'betas': [0.9, 0.999], 'weight_decay': 0.01}, + 'scheduler': 'LambdaLR', + 'scheduler_args': {'lr_lambda': get_linear_schedule_with_warmup(10000, 70000)}, + 'num_iters': 80000, + 'val_freq': 1000, + 'distill_loss_weight': 1.0 + }) + \ No newline at end of file diff --git a/new_impl/cv/sam/seg_wo_index.py b/new_impl/cv/sam/seg_wo_index.py new file mode 100644 index 0000000000000000000000000000000000000000..538f8e894077357b640fbde58706efa661cef69a --- /dev/null +++ b/new_impl/cv/sam/seg_wo_index.py @@ -0,0 +1,271 @@ +import torch +import sys +from torch import nn +from dnns.vit import make_softmax_prunable +from new_impl.cv.elasticdnn.api.model import ElasticDNN_OfflineClsFMModel, ElasticDNN_OfflineClsMDModel +# from methods.elasticdnn.api.algs.md_pretraining_w_fbs import ElasticDNN_MDPretrainingWFBSAlg +from new_impl.cv.elasticdnn.model.base import ElasticDNNUtil +from new_impl.cv.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from sam import FM_to_MD_sam_Util +from new_impl.cv.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from sam import FMLoRA_sam_Util +from sam import ElasticsamUtil +from utils.dl.common.model import LayerActivation, get_module, get_parameter +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +from utils.dl.common.loss import CrossEntropyLossSoft +import torch.nn.functional as F +from new_impl.cv.elasticdnn.api.model import ElasticDNN_OfflineSegFMModel, ElasticDNN_OfflineSegMDModel + + +class ElasticDNN_sam_OfflineSegFMModel(ElasticDNN_OfflineSegFMModel): + def generate_md_by_reducing_width(self, reducing_width_ratio, samples: torch.Tensor): + return FM_to_MD_sam_Util().init_md_from_fm_by_reducing_width_with_perf_test(self.models_dict['main'], + reducing_width_ratio, samples).to(self.device) + + def get_feature_hook(self) -> LayerActivation: + return LayerActivation(get_module(self.models_dict['main'], 'head'), True, self.device) + + def get_elastic_dnn_util(self) -> ElasticDNNUtil: + return ElasticsamUtil() + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + return F.cross_entropy(self.infer(x), y) + + def get_lora_util(self) -> FMLoRA_Util: + return FMLoRA_sam_Util() + + def get_task_head_params(self): + head = get_module(self.models_dict['main'], 'head') + return list(head.parameters()) + + +class ElasticDNN_sam_OfflineSegMDModel(ElasticDNN_OfflineSegMDModel): + + def get_feature_hook(self) -> LayerActivation: + return LayerActivation(get_module(self.models_dict['main'], 'head'), True, self.device) + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + return F.cross_entropy(self.infer(x), y) + + def get_distill_loss(self, student_output, teacher_output): + return self.distill_criterion(student_output, teacher_output) + + def get_matched_param_of_fm(self, self_param_name, fm: nn.Module): + # only between qkv.weight, norm.weight/bias + if any([k in self_param_name for k in ['fbs', 'cls_token', 'pos_embed']]): + return None + + p = get_parameter(self.models_dict['main'], self_param_name) + if p.dim() == 0: + return None + elif p.dim() == 1 and 'layernorm' in self_param_name and 'weight' in self_param_name: + return get_parameter(fm, self_param_name) + + # 1. xx.qkv.to_qkv.yy to xx.qkv.qkv.aa and xx.qkv.abs.zz + # if 'qkv.weight' in self_param_name: + # ss = self_param_name.split('.') + + # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv' + # fm_qkv = get_module(fm, fm_qkv_name) + + # fm_abs_name = '.'.join(ss[0: -2]) + '.abs' + # fm_abs = get_module(fm, fm_abs_name) + + # return torch.cat([ + # fm_qkv.weight.data, # task-agnositc params + # torch.cat([(_abs[0].weight.T @ _abs[1].weight.T).T for _abs in fm_abs], dim=0) # task-specific params (LoRA) + # ], dim=0) + + # # elif 'to_qkv.bias' in self_param_name: + # # ss = self_param_name.split('.') + + # # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv.bias' + # # return get_parameter(fm, fm_qkv_name) + + # elif 'mlp.fc1' in self_param_name and 'weight' in self_param_name: + # fm_param_name = self_param_name.replace('.linear', '') + # return get_parameter(fm, fm_param_name) + + # elif 'mlp.fc2' in self_param_name and 'weight' in self_param_name: + # fm_param_name = self_param_name + # return get_parameter(fm, fm_param_name) + + # else: + # # return get_parameter(fm, self_param_name) + # return None + + # if 'qkv.weight' in self_param_name: + # ss = self_param_name.split('.') + + # fm_qkv_name = '.'.join(ss[0: -1]) + '.qkv' + # fm_qkv = get_module(fm, fm_qkv_name) + + # fm_abs_name = '.'.join(ss[0: -1]) + '.abs' + # fm_abs = get_module(fm, fm_abs_name) + + # return torch.cat([ + # fm_qkv.weight.data, # task-agnositc params + # torch.cat([(_abs[0].weight.T @ _abs[1].weight.T).T for _abs in fm_abs], dim=0) # task-specific params (LoRA) + # ], dim=0) + + # # elif 'to_qkv.bias' in self_param_name: + # # ss = self_param_name.split('.') + + # # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv.bias' + # # return get_parameter(fm, fm_qkv_name) + + # elif 'mlp.fc1' in self_param_name and 'weight' in self_param_name: + # fm_param_name = self_param_name.replace('.linear', '') + # return get_parameter(fm, fm_param_name) + + # elif 'mlp.fc2' in self_param_name and 'weight' in self_param_name: + # fm_param_name = self_param_name + # res = get_parameter(fm, fm_param_name) + # # print('mlp fc2 debug', fm_param_name, res is None) + # return res + + # else: + # # return get_parameter(fm, self_param_name) + # return None + + # def get_matched_param_of_fm(self, self_param_name, fm: nn.Module): + # if any([k in self_param_name for k in ['fbs', 'cls_token', 'pos_embed']]): + # return None + + # # 1. xx.qkv.to_qkv.yy to xx.qkv.qkv.aa and xx.qkv.abs.zz + # if 'to_qkv.weight' in self_param_name: + # ss = self_param_name.split('.') + + # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv' + # fm_qkv = get_module(fm, fm_qkv_name) + + # fm_abs_name = '.'.join(ss[0: -2]) + '.abs' + # fm_abs = get_module(fm, fm_abs_name) + + # return torch.cat([ + # fm_qkv.weight.data, # task-agnositc params + # torch.cat([(_abs[0].weight.T @ _abs[1].weight.T).T for _abs in fm_abs], dim=0) # task-specific params (LoRA) + # ], dim=0) + + # elif 'to_qkv.bias' in self_param_name: + # ss = self_param_name.split('.') + + # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv.bias' + # return get_parameter(fm, fm_qkv_name) + + # elif 'mlp.fc1' in self_param_name: + # fm_param_name = self_param_name.replace('.linear', '') + # return get_parameter(fm, fm_param_name) + + # else: + # return get_parameter(fm, self_param_name) + if 'attn.qkv' in self_param_name and ('weight' in self_param_name): + ss = self_param_name.split('.') + + fm_qkv_name = '.'.join(ss[0: -1]) + '.fc' + fm_qkv = get_module(fm, fm_qkv_name) + + fm_abs_name = '.'.join(ss[0: -1]) + '.ab' + fm_abs = get_module(fm, fm_abs_name) + + return torch.cat([ + fm_qkv.weight.data, # task-agnositc params + fm_abs[1].weight @ fm_abs[0].weight + ], dim=0) + + elif 'attn.qkv' in self_param_name and ('bias' in self_param_name): + ss = self_param_name.split('.') + + fm_qkv_name = '.'.join(ss[0: -1]) + '.fc.bias' + return get_parameter(fm, fm_qkv_name) + + elif 'mlp.lin1' in self_param_name and 'weight' in self_param_name: + fm_param_name = self_param_name.replace('.linear', '') + return get_parameter(fm, fm_param_name) + elif 'mlp.lin2' in self_param_name and 'weight' in self_param_name: + fm_param_name = self_param_name.replace('.linear', '') + return get_parameter(fm, fm_param_name) + else: + #return get_parameter(fm, self_param_name) + return None + + +if __name__ == '__main__': + from utils.dl.common.env import set_random_seed + set_random_seed(1) + scenario = build_scenario( + source_datasets_name=['GTA5', 'SuperviselyPerson'], + target_datasets_order=['Cityscapes', 'BaiduPerson'] * 10, + da_mode='close_set', + data_dirs={ + 'GTA5': '/data/zql/datasets/GTA-ls-copy/GTA5', + 'SuperviselyPerson': '/data/zql/datasets/supervisely_person/Supervisely Person Dataset', + 'Cityscapes': '/data/zql/datasets/cityscape/', + 'BaiduPerson': '/data/zql/datasets/baidu_person/clean_images/' + }, + ) + + # 1. init model + from dnns.vit import vit_b_16 + fm_models_dict_path = 'new_impl/cv/sam/results/seg_wo_fbs.py/20231125/999997-111727/models/fm_best.pt' + fm_models_dict_path = save_models_dict_for_init(torch.load(fm_models_dict_path), __file__, 'fm_sam_cls_lora') + pretrained_md_models_dict_path = 'new_impl/cv/sam/results/seg_wo_fbs.py/20231125/999997-111727/models/md_best.pt' + md_models_dict = torch.load(pretrained_md_models_dict_path) + md_models_dict_path = save_models_dict_for_init(md_models_dict, __file__, 'md_sam_cls_pretrained_wo_fbs') + torch.cuda.set_device(1) + device = 'cuda' + + fm_model = ElasticDNN_sam_OfflineSegFMModel('fm', fm_models_dict_path, device, scenario.num_classes) + md_model = ElasticDNN_sam_OfflineSegMDModel('md', md_models_dict_path, device,scenario.num_classes) + + + # 2. init alg + models = { + 'fm': fm_model, + 'md': md_model + } + from new_impl.cv.elasticdnn.api.algs.md_pretraining_index_v2_train_index_and_md import ElasticDNN_MDPretrainingIndexAlg + + fm_to_md_alg = ElasticDNN_MDPretrainingIndexAlg(models, get_res_save_dir(__file__, sys.argv[0])) + + # 3. init scenario + + from utils.dl.common.lr_scheduler import get_linear_schedule_with_warmup + fm_to_md_alg.run(scenario, hyps={ + 'launch_tbboard': False, + + 'samples_size': (1, 3, 224, 224), + + 'FBS_r': 16, + 'FBS_ignore_layers': [], + + 'train_batch_size': 128, + 'val_batch_size': 512, + 'num_workers': 16, + 'optimizer': 'AdamW', + # 'optimizer_args': {'lr': 1e-4, 'betas': [0.9, 0.999], 'weight_decay': 0.01}, + 'indexes_optimizer_args': {'lr': 3e-4, 'betas': [0.9, 0.999], 'weight_decay': 0.1}, + # 'scheduler': 'StepLR', + # 'scheduler_args': {'step_size': 20000, 'gamma': 0.1}, + # 'optimizer': 'AdamW', + # 'optimizer_args': {'lr': 1e-4, 'betas': [0.9, 0.999], 'weight_decay': 0.01}, + 'optimizer_args': {'lr': 1e-3, 'betas': [0.9, 0.999], 'weight_decay': 0.01},#注意学习率的调整，不同的模型不一样。 + + 'scheduler': 'LambdaLR', + 'scheduler_args': {'lr_lambda': get_linear_schedule_with_warmup(10000, 70000)}, + + 'max_sparsity': 0.9, + 'min_sparsity': 0.3, + 'num_iters': 60000, + 'val_freq': 400, + 'index_loss_weight': 1e-4, + 'l1_reg_loss_weight': 1e-9, + 'val_num_sparsities': 4, + + 'bn_cal_num_iters': 0,#有bn层注意需要加上这个 + + 'index_init': 'zero', + 'index_guided_linear_comb_split_size': 512 + }) + \ No newline at end of file diff --git a/new_impl/cv/test1.py b/new_impl/cv/test1.py new file mode 100644 index 0000000000000000000000000000000000000000..1c5d709c8f1b6f272ecb303d46d8bc69fa8c0e94 --- /dev/null +++ b/new_impl/cv/test1.py @@ -0,0 +1,28 @@ +# from transformers import CvtModel,CvtConfig,CvtForImageClassification,AutoFeatureExtractor +# import torch +# from PIL import Image +# import requests +# from dnns.vit import vit_b_16 +# torch.cuda.set_device(1) +# device = 'cuda' +# #configuration = CvtConfig(num_labels=5) +# # url = 'http://images.cocodataset.org/val2017/000000039769.jpg' +# # image = Image.open(requests.get(url, stream=True).raw) +# # feature_extractor = AutoFeatureExtractor.from_pretrained('microsoft/cvt-13') +# model = CvtForImageClassification.from_pretrained("/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/new_impl/cv/cvt_model")#这里是规定最终我输出的分类个数，需要注意的是如果linear最终的输出不匹配的话，需要把第三个参数设置为True +# sample = torch.rand((4, 3, 224, 224)).to(device) +# model3 = vit_b_16(pretrained = True,num_classes=20) +# model2 = torch.load("/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/new_impl/cv/entry_model/cvt_pretrained.pt",map_location=device) +# model2['main'].train() +# for n, m in model2['main'].named_modules(): +# print(n) +# if n=='cvt.encoder.stages.2.layers.2.attention.attention.convolution_projection_value.linear_projection': +# print(m) +# elif n== 'cvt.encoder.stages.2.layers.0.intermediate.dense': +# print(m) +# outputs = model2['main'](sample) +# # print(**inputs) + +import numpy +a = [1,2,3,4,5] +print(a[1:]) diff --git a/new_impl/cv/third_party/nni/__init__.py b/new_impl/cv/third_party/nni/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..0630571ae698850bb0a6948f49797b4bdc9f879c --- /dev/null +++ b/new_impl/cv/third_party/nni/__init__.py @@ -0,0 +1,20 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +__version__ = '999.0.0-developing' + +from .runtime.env_vars import dispatcher_env_vars +from .utils import ClassArgsValidator + +if dispatcher_env_vars.SDK_PROCESS != 'dispatcher': + from .trial import * + from .smartparam import * + from .common.nas_utils import training_update + +class NoMoreTrialError(Exception): + def __init__(self, ErrorInfo): + super().__init__(self) + self.errorinfo = ErrorInfo + + def __str__(self): + return self.errorinfo diff --git a/new_impl/cv/third_party/nni/__main__.py b/new_impl/cv/third_party/nni/__main__.py new file mode 100644 index 0000000000000000000000000000000000000000..e3f982f42fe51450c934f4a03bb15c12f7979318 --- /dev/null +++ b/new_impl/cv/third_party/nni/__main__.py @@ -0,0 +1,111 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +import argparse +import logging +import json +import base64 + +from .runtime.common import enable_multi_thread, enable_multi_phase +from .runtime.msg_dispatcher import MsgDispatcher +from .tools.package_utils import create_builtin_class_instance, create_customized_class_instance + +logger = logging.getLogger('nni.main') +logger.debug('START') + +if os.environ.get('COVERAGE_PROCESS_START'): + import coverage + coverage.process_startup() + + +def main(): + parser = argparse.ArgumentParser(description='Dispatcher command line parser') + parser.add_argument('--exp_params', type=str, required=True) + args, _ = parser.parse_known_args() + + exp_params_decode = base64.b64decode(args.exp_params).decode('utf-8') + logger.debug('decoded exp_params: [%s]', exp_params_decode) + exp_params = json.loads(exp_params_decode) + logger.debug('exp_params json obj: [%s]', json.dumps(exp_params, indent=4)) + + if exp_params.get('multiThread'): + enable_multi_thread() + if exp_params.get('multiPhase'): + enable_multi_phase() + + if exp_params.get('advisor') is not None: + # advisor is enabled and starts to run + _run_advisor(exp_params) + else: + # tuner (and assessor) is enabled and starts to run + assert exp_params.get('tuner') is not None + tuner = _create_tuner(exp_params) + if exp_params.get('assessor') is not None: + assessor = _create_assessor(exp_params) + else: + assessor = None + dispatcher = MsgDispatcher(tuner, assessor) + + try: + dispatcher.run() + tuner._on_exit() + if assessor is not None: + assessor._on_exit() + except Exception as exception: + logger.exception(exception) + tuner._on_error() + if assessor is not None: + assessor._on_error() + raise + + +def _run_advisor(exp_params): + if exp_params.get('advisor').get('builtinAdvisorName'): + dispatcher = create_builtin_class_instance( + exp_params.get('advisor').get('builtinAdvisorName'), + exp_params.get('advisor').get('classArgs'), + 'advisors') + else: + dispatcher = create_customized_class_instance(exp_params.get('advisor')) + if dispatcher is None: + raise AssertionError('Failed to create Advisor instance') + try: + dispatcher.run() + except Exception as exception: + logger.exception(exception) + raise + + +def _create_tuner(exp_params): + if exp_params.get('tuner').get('builtinTunerName'): + tuner = create_builtin_class_instance( + exp_params.get('tuner').get('builtinTunerName'), + exp_params.get('tuner').get('classArgs'), + 'tuners') + else: + tuner = create_customized_class_instance(exp_params.get('tuner')) + if tuner is None: + raise AssertionError('Failed to create Tuner instance') + return tuner + + +def _create_assessor(exp_params): + if exp_params.get('assessor').get('builtinAssessorName'): + assessor = create_builtin_class_instance( + exp_params.get('assessor').get('builtinAssessorName'), + exp_params.get('assessor').get('classArgs'), + 'assessors') + else: + assessor = create_customized_class_instance(exp_params.get('assessor')) + if assessor is None: + raise AssertionError('Failed to create Assessor instance') + return assessor + + +if __name__ == '__main__': + try: + main() + except Exception as exception: + logger.exception(exception) + raise diff --git a/new_impl/cv/third_party/nni/algorithms/__init__.py b/new_impl/cv/third_party/nni/algorithms/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/new_impl/cv/third_party/nni/algorithms/compression/__init__.py b/new_impl/cv/third_party/nni/algorithms/compression/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/new_impl/cv/third_party/nni/algorithms/compression/pytorch/__init__.py b/new_impl/cv/third_party/nni/algorithms/compression/pytorch/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/new_impl/cv/third_party/nni/algorithms/compression/pytorch/pruning/__init__.py b/new_impl/cv/third_party/nni/algorithms/compression/pytorch/pruning/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..dfed2bf142cd0cc66e825ce13549a5f8cd68f499 --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/compression/pytorch/pruning/__init__.py @@ -0,0 +1,15 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .finegrained_pruning import * +from .structured_pruning import * +from .apply_compression import apply_compression_results +from .one_shot import * +from .agp import * +# from .lottery_ticket import LotteryTicketPruner +# from .simulated_annealing_pruner import SimulatedAnnealingPruner +# from .net_adapt_pruner import NetAdaptPruner +# from .admm_pruner import ADMMPruner +# from .auto_compress_pruner import AutoCompressPruner +# from .sensitivity_pruner import SensitivityPruner +# from .amc import AMCPruner diff --git a/new_impl/cv/third_party/nni/algorithms/compression/pytorch/pruning/admm_pruner.py b/new_impl/cv/third_party/nni/algorithms/compression/pytorch/pruning/admm_pruner.py new file mode 100644 index 0000000000000000000000000000000000000000..f65f1405e16fd6f4ec942e4ea2781ec9fc82529e --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/compression/pytorch/pruning/admm_pruner.py @@ -0,0 +1,190 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +import torch +from schema import And, Optional + +from nni.compression.pytorch.utils.config_validation import CompressorSchema +from .constants import MASKER_DICT +from .one_shot import OneshotPruner + + +_logger = logging.getLogger(__name__) + + +class ADMMPruner(OneshotPruner): + """ + A Pytorch implementation of ADMM Pruner algorithm. + + Parameters + ---------- + model : torch.nn.Module + Model to be pruned. + config_list : list + List on pruning configs. + trainer : function + Function used for the first subproblem. + Users should write this function as a normal function to train the Pytorch model + and include `model, optimizer, criterion, epoch, callback` as function arguments. + Here `callback` acts as an L2 regulizer as presented in the formula (7) of the original paper. + The logic of `callback` is implemented inside the Pruner, + users are just required to insert `callback()` between `loss.backward()` and `optimizer.step()`. + Example:: + + def trainer(model, criterion, optimizer, epoch, callback): + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + train_loader = ... + model.train() + for batch_idx, (data, target) in enumerate(train_loader): + data, target = data.to(device), target.to(device) + optimizer.zero_grad() + output = model(data) + loss = criterion(output, target) + loss.backward() + # callback should be inserted between loss.backward() and optimizer.step() + if callback: + callback() + optimizer.step() + num_iterations : int + Total number of iterations. + training_epochs : int + Training epochs of the first subproblem. + row : float + Penalty parameters for ADMM training. + base_algo : str + Base pruning algorithm. `level`, `l1` or `l2`, by default `l1`. Given the sparsity distribution among the ops, + the assigned `base_algo` is used to decide which filters/channels/weights to prune. + + """ + + def __init__(self, model, config_list, trainer, num_iterations=30, training_epochs=5, row=1e-4, base_algo='l1'): + self._base_algo = base_algo + + super().__init__(model, config_list) + + self._trainer = trainer + self._num_iterations = num_iterations + self._training_epochs = training_epochs + self._row = row + + self.set_wrappers_attribute("if_calculated", False) + self.masker = MASKER_DICT[self._base_algo](self.bound_model, self) + + def validate_config(self, model, config_list): + """ + Parameters + ---------- + model : torch.nn.Module + Model to be pruned + config_list : list + List on pruning configs + """ + + if self._base_algo == 'level': + schema = CompressorSchema([{ + 'sparsity': And(float, lambda n: 0 < n < 1), + Optional('op_types'): [str], + Optional('op_names'): [str], + }], model, _logger) + elif self._base_algo in ['l1', 'l2']: + schema = CompressorSchema([{ + 'sparsity': And(float, lambda n: 0 < n < 1), + 'op_types': ['Conv2d'], + Optional('op_names'): [str] + }], model, _logger) + + schema.validate(config_list) + + def _projection(self, weight, sparsity): + ''' + Return the Euclidean projection of the weight matrix according to the pruning mode. + + Parameters + ---------- + weight : tensor + original matrix + sparsity : float + the ratio of parameters which need to be set to zero + + Returns + ------- + tensor + the projected matrix + ''' + w_abs = weight.abs() + if self._base_algo == 'level': + k = int(weight.numel() * sparsity) + if k == 0: + mask_weight = torch.ones(weight.shape).type_as(weight) + else: + threshold = torch.topk(w_abs.view(-1), k, largest=False)[0].max() + mask_weight = torch.gt(w_abs, threshold).type_as(weight) + elif self._base_algo in ['l1', 'l2']: + filters = weight.size(0) + num_prune = int(filters * sparsity) + if filters < 2 or num_prune < 1: + mask_weight = torch.ones(weight.size()).type_as(weight).detach() + else: + w_abs_structured = w_abs.view(filters, -1).sum(dim=1) + threshold = torch.topk(w_abs_structured.view(-1), num_prune, largest=False)[0].max() + mask_weight = torch.gt(w_abs_structured, threshold)[:, None, None, None].expand_as(weight).type_as(weight) + + return weight.data.mul(mask_weight) + + def compress(self): + """ + Compress the model with ADMM. + + Returns + ------- + torch.nn.Module + model with specified modules compressed. + """ + _logger.info('Starting ADMM Compression...') + + # initiaze Z, U + # Z_i^0 = W_i^0 + # U_i^0 = 0 + Z = [] + U = [] + for wrapper in self.get_modules_wrapper(): + z = wrapper.module.weight.data + Z.append(z) + U.append(torch.zeros_like(z)) + + optimizer = torch.optim.Adam( + self.bound_model.parameters(), lr=1e-3, weight_decay=5e-5) + + # Loss = cross_entropy + l2 regulization + \Sum_{i=1}^N \row_i ||W_i - Z_i^k + U_i^k||^2 + criterion = torch.nn.CrossEntropyLoss() + + # callback function to do additonal optimization, refer to the deriatives of Formula (7) + def callback(): + for i, wrapper in enumerate(self.get_modules_wrapper()): + wrapper.module.weight.data -= self._row * \ + (wrapper.module.weight.data - Z[i] + U[i]) + + # optimization iteration + for k in range(self._num_iterations): + _logger.info('ADMM iteration : %d', k) + + # step 1: optimize W with AdamOptimizer + for epoch in range(self._training_epochs): + self._trainer(self.bound_model, optimizer=optimizer, + criterion=criterion, epoch=epoch, callback=callback) + + # step 2: update Z, U + # Z_i^{k+1} = projection(W_i^{k+1} + U_i^k) + # U_i^{k+1} = U^k + W_i^{k+1} - Z_i^{k+1} + for i, wrapper in enumerate(self.get_modules_wrapper()): + z = wrapper.module.weight.data + U[i] + Z[i] = self._projection(z, wrapper.config['sparsity']) + U[i] = U[i] + wrapper.module.weight.data - Z[i] + + # apply prune + self.update_mask() + + _logger.info('Compression finished.') + + return self.bound_model diff --git a/new_impl/cv/third_party/nni/algorithms/compression/pytorch/pruning/agp.py b/new_impl/cv/third_party/nni/algorithms/compression/pytorch/pruning/agp.py new file mode 100644 index 0000000000000000000000000000000000000000..a1ee5df069d146a6389d3f7f52c61a192d5fd865 --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/compression/pytorch/pruning/agp.py @@ -0,0 +1,151 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +An automated gradual pruning algorithm that prunes the smallest magnitude +weights to achieve a preset level of network sparsity. +Michael Zhu and Suyog Gupta, "To prune, or not to prune: exploring the +efficacy of pruning for model compression", 2017 NIPS Workshop on Machine +Learning of Phones and other Consumer Devices. +""" + +import logging +import torch +from schema import And, Optional +from .constants import MASKER_DICT +from .....compression.pytorch.utils.config_validation import CompressorSchema +from .....compression.pytorch.compressor import Pruner + +__all__ = ['AGPPruner'] + +logger = logging.getLogger('torch pruner') + +class AGPPruner(Pruner): + """ + Parameters + ---------- + model : torch.nn.Module + Model to be pruned. + config_list : listlist + Supported keys: + - initial_sparsity: This is to specify the sparsity when compressor starts to compress. + - final_sparsity: This is to specify the sparsity when compressor finishes to compress. + - start_epoch: This is to specify the epoch number when compressor starts to compress, default start from epoch 0. + - end_epoch: This is to specify the epoch number when compressor finishes to compress. + - frequency: This is to specify every *frequency* number epochs compressor compress once, default frequency=1. + optimizer: torch.optim.Optimizer + Optimizer used to train model. + pruning_algorithm: str + Algorithms being used to prune model, + choose from `['level', 'slim', 'l1', 'l2', 'fpgm', 'taylorfo', 'apoz', 'mean_activation']`, by default `level` + """ + + def __init__(self, model, config_list, optimizer, pruning_algorithm='level'): + super().__init__(model, config_list, optimizer) + assert isinstance(optimizer, torch.optim.Optimizer), "AGP pruner is an iterative pruner, please pass optimizer of the model to it" + self.masker = MASKER_DICT[pruning_algorithm](model, self) + + self.now_epoch = 0 + self.set_wrappers_attribute("if_calculated", False) + + def validate_config(self, model, config_list): + """ + Parameters + ---------- + model : torch.nn.Module + Model to be pruned + config_list : list + List on pruning configs + """ + schema = CompressorSchema([{ + 'initial_sparsity': And(float, lambda n: 0 <= n <= 1), + 'final_sparsity': And(float, lambda n: 0 <= n <= 1), + 'start_epoch': And(int, lambda n: n >= 0), + 'end_epoch': And(int, lambda n: n >= 0), + 'frequency': And(int, lambda n: n > 0), + Optional('op_types'): [str], + Optional('op_names'): [str] + }], model, logger) + + schema.validate(config_list) + + def calc_mask(self, wrapper, wrapper_idx=None): + """ + Calculate the mask of given layer. + Scale factors with the smallest absolute value in the BN layer are masked. + Parameters + ---------- + wrapper : Module + the layer to instrument the compression operation + wrapper_idx: int + index of this wrapper in pruner's all wrappers + Returns + ------- + dict | None + Dictionary for storing masks, keys of the dict: + 'weight_mask': weight mask tensor + 'bias_mask': bias mask tensor (optional) + """ + + config = wrapper.config + + start_epoch = config.get('start_epoch', 0) + freq = config.get('frequency', 1) + + if wrapper.if_calculated: + return None + if not (self.now_epoch >= start_epoch and (self.now_epoch - start_epoch) % freq == 0): + return None + + target_sparsity = self.compute_target_sparsity(config) + new_mask = self.masker.calc_mask(sparsity=target_sparsity, wrapper=wrapper, wrapper_idx=wrapper_idx) + if new_mask is not None: + wrapper.if_calculated = True + + return new_mask + + def compute_target_sparsity(self, config): + """ + Calculate the sparsity for pruning + Parameters + ---------- + config : dict + Layer's pruning config + Returns + ------- + float + Target sparsity to be pruned + """ + + end_epoch = config.get('end_epoch', 1) + start_epoch = config.get('start_epoch', 0) + freq = config.get('frequency', 1) + final_sparsity = config.get('final_sparsity', 0) + initial_sparsity = config.get('initial_sparsity', 0) + if end_epoch <= start_epoch or initial_sparsity >= final_sparsity: + logger.warning('your end epoch <= start epoch or initial_sparsity >= final_sparsity') + return final_sparsity + + if end_epoch <= self.now_epoch: + return final_sparsity + + span = ((end_epoch - start_epoch - 1) // freq) * freq + assert span > 0 + target_sparsity = (final_sparsity + + (initial_sparsity - final_sparsity) * + (1.0 - ((self.now_epoch - start_epoch) / span)) ** 3) + return target_sparsity + + def update_epoch(self, epoch): + """ + Update epoch + Parameters + ---------- + epoch : int + current training epoch + """ + + if epoch > 0: + self.now_epoch = epoch + for wrapper in self.get_modules_wrapper(): + wrapper.if_calculated = False diff --git a/new_impl/cv/third_party/nni/algorithms/compression/pytorch/pruning/amc/__init__.py b/new_impl/cv/third_party/nni/algorithms/compression/pytorch/pruning/amc/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..3c89a879c6af42ed372f38f3cc11ea1eb9eb6870 --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/compression/pytorch/pruning/amc/__init__.py @@ -0,0 +1,4 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .amc_pruner import AMCPruner diff --git a/new_impl/cv/third_party/nni/algorithms/compression/pytorch/pruning/amc/amc_pruner.py b/new_impl/cv/third_party/nni/algorithms/compression/pytorch/pruning/amc/amc_pruner.py new file mode 100644 index 0000000000000000000000000000000000000000..c2d12429d5fb5d82131b5239ef1992ef100feef6 --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/compression/pytorch/pruning/amc/amc_pruner.py @@ -0,0 +1,296 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +import logging +from copy import deepcopy +from argparse import Namespace +import numpy as np +import torch +from torch.utils.tensorboard import SummaryWriter + +from nni.compression.pytorch.compressor import Pruner +from .channel_pruning_env import ChannelPruningEnv +from .lib.agent import DDPG +from .lib.utils import get_output_folder + +torch.backends.cudnn.deterministic = True + +_logger = logging.getLogger(__name__) + +class AMCPruner(Pruner): + """ + A pytorch implementation of AMC: AutoML for Model Compression and Acceleration on Mobile Devices. + (https://arxiv.org/pdf/1802.03494.pdf) + + Parameters: + model: nn.Module + The model to be pruned. + config_list: list + Configuration list to configure layer pruning. + Supported keys: + - op_types: operation type to be pruned + - op_names: operation name to be pruned + evaluator: function + function to evaluate the pruned model. + The prototype of the function: + >>> def evaluator(val_loader, model): + >>> ... + >>> return acc + val_loader: torch.utils.data.DataLoader + Data loader of validation dataset. + suffix: str + suffix to help you remember what experiment you ran. Default: None. + + # parameters for pruning environment + model_type: str + model type to prune, currently 'mobilenet' and 'mobilenetv2' are supported. Default: mobilenet + flops_ratio: float + preserve flops ratio. Default: 0.5 + lbound: float + minimum weight preserve ratio for each layer. Default: 0.2 + rbound: float + maximum weight preserve ratio for each layer. Default: 1.0 + reward: function + reward function type: + - acc_reward: accuracy * 0.01 + - acc_flops_reward: - (100 - accuracy) * 0.01 * np.log(flops) + Default: acc_reward + # parameters for channel pruning + n_calibration_batches: int + number of batches to extract layer information. Default: 60 + n_points_per_layer: int + number of feature points per layer. Default: 10 + channel_round: int + round channel to multiple of channel_round. Default: 8 + + # parameters for ddpg agent + hidden1: int + hidden num of first fully connect layer. Default: 300 + hidden2: int + hidden num of second fully connect layer. Default: 300 + lr_c: float + learning rate for critic. Default: 1e-3 + lr_a: float + learning rate for actor. Default: 1e-4 + warmup: int + number of episodes without training but only filling the replay memory. During warmup episodes, + random actions ares used for pruning. Default: 100 + discount: float + next Q value discount for deep Q value target. Default: 0.99 + bsize: int + minibatch size for training DDPG agent. Default: 64 + rmsize: int + memory size for each layer. Default: 100 + window_length: int + replay buffer window length. Default: 1 + tau: float + moving average for target network being used by soft_update. Default: 0.99 + # noise + init_delta: float + initial variance of truncated normal distribution + delta_decay: float + delta decay during exploration + + # parameters for training ddpg agent + max_episode_length: int + maximum episode length + output_dir: str + output directory to save log files and model files. Default: ./logs + debug: boolean + debug mode + train_episode: int + train iters each timestep. Default: 800 + epsilon: int + linear decay of exploration policy. Default: 50000 + seed: int + random seed to set for reproduce experiment. Default: None + """ + + def __init__( + self, + model, + config_list, + evaluator, + val_loader, + suffix=None, + model_type='mobilenet', + dataset='cifar10', + flops_ratio=0.5, + lbound=0.2, + rbound=1., + reward='acc_reward', + n_calibration_batches=60, + n_points_per_layer=10, + channel_round=8, + hidden1=300, + hidden2=300, + lr_c=1e-3, + lr_a=1e-4, + warmup=100, + discount=1., + bsize=64, + rmsize=100, + window_length=1, + tau=0.01, + init_delta=0.5, + delta_decay=0.99, + max_episode_length=1e9, + output_dir='./logs', + debug=False, + train_episode=800, + epsilon=50000, + seed=None): + + self.val_loader = val_loader + self.evaluator = evaluator + + if seed is not None: + np.random.seed(seed) + torch.manual_seed(seed) + torch.cuda.manual_seed(seed) + + checkpoint = deepcopy(model.state_dict()) + + super().__init__(model, config_list, optimizer=None) + + # build folder and logs + base_folder_name = '{}_{}_r{}_search'.format(model_type, dataset, flops_ratio) + if suffix is not None: + self.output_dir = os.path.join(output_dir, base_folder_name + '-' + suffix) + else: + self.output_dir = get_output_folder(output_dir, base_folder_name) + + self.env_args = Namespace( + model_type=model_type, + preserve_ratio=flops_ratio, + lbound=lbound, + rbound=rbound, + reward=reward, + n_calibration_batches=n_calibration_batches, + n_points_per_layer=n_points_per_layer, + channel_round=channel_round, + output=self.output_dir + ) + self.env = ChannelPruningEnv( + self, evaluator, val_loader, checkpoint, args=self.env_args) + _logger.info('=> Saving logs to %s', self.output_dir) + self.tfwriter = SummaryWriter(log_dir=self.output_dir) + self.text_writer = open(os.path.join(self.output_dir, 'log.txt'), 'w') + _logger.info('=> Output path: %s...', self.output_dir) + + nb_states = self.env.layer_embedding.shape[1] + nb_actions = 1 # just 1 action here + + rmsize = rmsize * len(self.env.prunable_idx) # for each layer + _logger.info('** Actual replay buffer size: %d', rmsize) + + self.ddpg_args = Namespace( + hidden1=hidden1, + hidden2=hidden2, + lr_c=lr_c, + lr_a=lr_a, + warmup=warmup, + discount=discount, + bsize=bsize, + rmsize=rmsize, + window_length=window_length, + tau=tau, + init_delta=init_delta, + delta_decay=delta_decay, + max_episode_length=max_episode_length, + debug=debug, + train_episode=train_episode, + epsilon=epsilon + ) + self.agent = DDPG(nb_states, nb_actions, self.ddpg_args) + + + def compress(self): + self.train(self.ddpg_args.train_episode, self.agent, self.env, self.output_dir) + + def train(self, num_episode, agent, env, output_dir): + agent.is_training = True + step = episode = episode_steps = 0 + episode_reward = 0. + observation = None + T = [] # trajectory + while episode < num_episode: # counting based on episode + # reset if it is the start of episode + if observation is None: + observation = deepcopy(env.reset()) + agent.reset(observation) + + # agent pick action ... + if episode <= self.ddpg_args.warmup: + action = agent.random_action() + # action = sample_from_truncated_normal_distribution(lower=0., upper=1., mu=env.preserve_ratio, sigma=0.5) + else: + action = agent.select_action(observation, episode=episode) + + # env response with next_observation, reward, terminate_info + observation2, reward, done, info = env.step(action) + + T.append([reward, deepcopy(observation), deepcopy(observation2), action, done]) + + # fix-length, never reach here + # if max_episode_length and episode_steps >= max_episode_length - 1: + # done = True + + # [optional] save intermideate model + if num_episode / 3 <= 1 or episode % int(num_episode / 3) == 0: + agent.save_model(output_dir) + + # update + step += 1 + episode_steps += 1 + episode_reward += reward + observation = deepcopy(observation2) + + if done: # end of episode + _logger.info( + '#%d: episode_reward: %.4f acc: %.4f, ratio: %.4f', + episode, episode_reward, + info['accuracy'], + info['compress_ratio'] + ) + self.text_writer.write( + '#{}: episode_reward:{:.4f} acc: {:.4f}, ratio: {:.4f}\n'.format( + episode, episode_reward, + info['accuracy'], + info['compress_ratio'] + ) + ) + final_reward = T[-1][0] + # print('final_reward: {}'.format(final_reward)) + # agent observe and update policy + for _, s_t, s_t1, a_t, done in T: + agent.observe(final_reward, s_t, s_t1, a_t, done) + if episode > self.ddpg_args.warmup: + agent.update_policy() + + #agent.memory.append( + # observation, + # agent.select_action(observation, episode=episode), + # 0., False + #) + + # reset + observation = None + episode_steps = 0 + episode_reward = 0. + episode += 1 + T = [] + + self.tfwriter.add_scalar('reward/last', final_reward, episode) + self.tfwriter.add_scalar('reward/best', env.best_reward, episode) + self.tfwriter.add_scalar('info/accuracy', info['accuracy'], episode) + self.tfwriter.add_scalar('info/compress_ratio', info['compress_ratio'], episode) + self.tfwriter.add_text('info/best_policy', str(env.best_strategy), episode) + # record the preserve rate for each layer + for i, preserve_rate in enumerate(env.strategy): + self.tfwriter.add_scalar('preserve_rate/{}'.format(i), preserve_rate, episode) + + self.text_writer.write('best reward: {}\n'.format(env.best_reward)) + self.text_writer.write('best policy: {}\n'.format(env.best_strategy)) + self.text_writer.close() diff --git a/new_impl/cv/third_party/nni/algorithms/compression/pytorch/pruning/amc/channel_pruning_env.py b/new_impl/cv/third_party/nni/algorithms/compression/pytorch/pruning/amc/channel_pruning_env.py new file mode 100644 index 0000000000000000000000000000000000000000..443daf7efb2f0fe7b909ab0330bb7a3264d59225 --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/compression/pytorch/pruning/amc/channel_pruning_env.py @@ -0,0 +1,543 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +import logging +import time +import math +import copy +import numpy as np +import torch +import torch.nn as nn + +from nni.compression.pytorch.compressor import PrunerModuleWrapper +from .. import AMCWeightMasker + +_logger = logging.getLogger(__name__) + +# for pruning +def acc_reward(net, acc, flops): + return acc * 0.01 + + +def acc_flops_reward(net, acc, flops): + error = (100 - acc) * 0.01 + return -error * np.log(flops) + + +class ChannelPruningEnv: + """ + Env for channel pruning search. + This class is used to prune model using specified pruner. It prunes one layer when + step() is called. When the last layer is pruned, it evaluate the pruned model using + evaluator, and use the returned value of evaluator as reward of the episode. + + Usage: + env = ChannelPruningEnv(pruner, evaluator, val_loader, checkpoint, env_args) + episode = 0 + T = [] + while episode < num_episode: + action = agent.select_action(observation) + observation2, reward, done, info = env.step(action) + T.append([reward, deepcopy(observation), deepcopy(observation2), action, done]) + + if done: # end of episode, last layer pruned + episode += 1 + # train agent with episode data + for _, s_t, s_t1, a_t, done in T: + agent.observe(final_reward, s_t, s_t1, a_t, done) + agent.update_policy() + T = [] + + Attributes: + prunable_idx: layer indices for pruable layers, the index values are the index + of list(self.model.modules()). Pruable layers are pointwise Conv2d layers and Linear + layers. + buffer_idx: layer indices for buffer layers which refers the depthwise layers. + Each depthwise layer is always followd by a pointwise layer for both mobilenet and + mobilenetv2. The depthwise layer's filters are pruned when its next pointwise layer's + corresponding input channels are pruned. + shared_idx: layer indices for layers which share input. + For example: [[1,4], [8, 10, 15]] means layer 1 and 4 share same input, and layer + 8, 10 and 15 share another input. + layer_embedding: embeddings for each prunable layers, the embedding is used as + observation for DDPG agent. + layer_info_dict: flops and number of parameters of each layer. + min_strategy_dict: key is layer index, value is a tuple, the first value is the minimum + action of input channel, the second value is the minimum action value of output channel. + strategy_dict: key is layer index, value is a tuple, the first value is the action of input + channel, the second value is the action of output channel. + + Parameters: + pruner: Pruner + NNI Pruner instance used to prune model. + evaluator: function + function to evaluate the pruned model. + The prototype of the function: + >>> def evaluator(val_loader, model): + >>> ... + >>> return acc + val_loader: torch.utils.data.DataLoader + Data loader of validation dataset. + checkpoint: dict + checkpoint of the model to be pruned. It is used to reset model at beginning of each + episode. + args: + A Namespace object containing following arguments: + model_type: str + model type to prune, currently 'mobilenet' and 'mobilenetv2' are supported. + flops_ratio: float + preserve flops ratio. + lbound: float + minimum weight preserve ratio for each layer. + rbound: float + maximum weight preserve ratio for each layer. + reward: function + reward function type + + # parameters for channel pruning + n_calibration_batches: int + number of batches to extract layer information. + n_points_per_layer: int + number of feature points per layer. + channel_round: int + round channel to multiple of channel_round. + + """ + def __init__(self, pruner, evaluator, val_loader, checkpoint, args): + self.pruner = pruner + self.model = pruner.bound_model + self.checkpoint = checkpoint + self.batch_size = val_loader.batch_size + self.preserve_ratio = args.preserve_ratio + self.channel_prune_masker = AMCWeightMasker(self.model, self.pruner, args.channel_round) + + # options from args + self.args = args + self.lbound = args.lbound + self.rbound = args.rbound + + self.n_calibration_batches = args.n_calibration_batches + self.n_points_per_layer = args.n_points_per_layer + self.channel_round = args.channel_round + + # sanity check + assert self.preserve_ratio > self.lbound, 'Error! You can not achieve preserve_ratio smaller than lbound!' + + # prepare data + self._val_loader = val_loader + self._validate = evaluator + + # build indexs + self._build_index() + self.n_prunable_layer = len(self.prunable_idx) + + # extract information for preparing + self._extract_layer_information() + + # build embedding (static part) + self._build_state_embedding() + + # build reward + self.reset() # restore weight + self.org_acc = self._validate(self._val_loader, self.model) + _logger.info('=> original acc: %.3f', self.org_acc) + self.org_model_size = sum(self.wsize_list) + _logger.info('=> original weight size: %.4f M param', self.org_model_size * 1. / 1e6) + self.org_flops = sum(self.flops_list) + _logger.info('=> FLOPs:') + _logger.info([self.layer_info_dict[idx]['flops']/1e6 for idx in sorted(self.layer_info_dict.keys())]) + _logger.info('=> original FLOPs: %.4f M', self.org_flops * 1. / 1e6) + + self.expected_preserve_computation = self.preserve_ratio * self.org_flops + + self.reward = eval(args.reward) + + self.best_reward = -math.inf + self.best_strategy = None + self.best_d_prime_list = None + self.best_masks = None + + self.org_w_size = sum(self.wsize_list) + + def step(self, action): + # Pseudo prune and get the corresponding statistics. The real pruning happens till the end of all pseudo pruning + if self.visited[self.cur_ind]: + action = self.strategy_dict[self.prunable_idx[self.cur_ind]][0] + preserve_idx = self.index_buffer[self.cur_ind] + else: + action = self._action_wall(action) # percentage to preserve + preserve_idx = None + # prune and update action + action, d_prime, preserve_idx = self.prune_kernel(self.prunable_idx[self.cur_ind], action, preserve_idx) + if not self.visited[self.cur_ind]: + for group in self.shared_idx: + if self.cur_ind in group: # set the shared ones + for g_idx in group: + self.strategy_dict[self.prunable_idx[g_idx]][0] = action + self.strategy_dict[self.prunable_idx[g_idx - 1]][1] = action + self.visited[g_idx] = True + self.index_buffer[g_idx] = preserve_idx.copy() + + self.strategy.append(action) # save action to strategy + self.d_prime_list.append(d_prime) + + self.strategy_dict[self.prunable_idx[self.cur_ind]][0] = action + if self.cur_ind > 0: + self.strategy_dict[self.prunable_idx[self.cur_ind - 1]][1] = action + + # all the actions are made + if self._is_final_layer(): + assert len(self.strategy) == len(self.prunable_idx) + current_flops = self._cur_flops() + acc_t1 = time.time() + acc = self._validate(self._val_loader, self.model) + acc_t2 = time.time() + self.val_time = acc_t2 - acc_t1 + compress_ratio = current_flops * 1. / self.org_flops + info_set = {'compress_ratio': compress_ratio, 'accuracy': acc, 'strategy': self.strategy.copy()} + reward = self.reward(self, acc, current_flops) + + if reward > self.best_reward: + self.best_reward = reward + self.best_strategy = self.strategy.copy() + self.best_d_prime_list = self.d_prime_list.copy() + best_model = os.path.join(self.args.output, 'best_model.pth') + best_mask = os.path.join(self.args.output, 'best_mask.pth') + self.pruner.export_model(model_path=best_model, mask_path=best_mask) + _logger.info('New best reward: %.4f, acc: %.4f, compress: %.4f', self.best_reward, acc, compress_ratio) + _logger.info('New best policy: %s', self.best_strategy) + _logger.info('New best d primes: %s', self.best_d_prime_list) + obs = self.layer_embedding[self.cur_ind, :].copy() # actually the same as the last state + done = True + return obs, reward, done, info_set + + info_set = None + reward = 0 + done = False + self.visited[self.cur_ind] = True # set to visited + self.cur_ind += 1 # the index of next layer + # build next state (in-place modify) + self.layer_embedding[self.cur_ind][-3] = self._cur_reduced() * 1. / self.org_flops # reduced + self.layer_embedding[self.cur_ind][-2] = sum(self.flops_list[self.cur_ind + 1:]) * 1. / self.org_flops # rest + self.layer_embedding[self.cur_ind][-1] = self.strategy[-1] # last action + obs = self.layer_embedding[self.cur_ind, :].copy() + + return obs, reward, done, info_set + + def reset(self): + # restore env by loading the checkpoint + self.pruner.reset(self.checkpoint) + self.cur_ind = 0 + self.strategy = [] # pruning strategy + self.d_prime_list = [] + self.strategy_dict = copy.deepcopy(self.min_strategy_dict) + # reset layer embeddings + self.layer_embedding[:, -1] = 1. + self.layer_embedding[:, -2] = 0. + self.layer_embedding[:, -3] = 0. + obs = self.layer_embedding[0].copy() + obs[-2] = sum(self.wsize_list[1:]) * 1. / sum(self.wsize_list) + self.extract_time = 0 + self.fit_time = 0 + self.val_time = 0 + # for share index + self.visited = [False] * len(self.prunable_idx) + self.index_buffer = {} + return obs + + def prune_kernel(self, op_idx, preserve_ratio, preserve_idx=None): + m_list = list(self.model.modules()) + op = m_list[op_idx] + assert (0. < preserve_ratio <= 1.) + assert type(op) == PrunerModuleWrapper + if preserve_ratio == 1: # do not prune + if (preserve_idx is None) or (len(preserve_idx) == op.module.weight.size(1)): + return 1., op.module.weight.size(1), None # should be a full index + op.input_feat = self.layer_info_dict[op_idx]['input_feat'] + op.output_feat = self.layer_info_dict[op_idx]['output_feat'] + + masks = self.channel_prune_masker.calc_mask(sparsity=1-preserve_ratio, wrapper=op, preserve_idx=preserve_idx) + m = masks['weight_mask'].cpu().data + if type(op.module) == nn.Conv2d: + d_prime = (m.sum((0, 2, 3)) > 0).sum().item() + preserve_idx = np.nonzero((m.sum((0, 2, 3)) > 0).numpy())[0] + else: + assert type(op.module) == nn.Linear + d_prime = (m.sum(1) > 0).sum().item() + preserve_idx = np.nonzero((m.sum(1) > 0).numpy())[0] + + op.weight_mask = masks['weight_mask'] + if hasattr(op.module, 'bias') and op.module.bias is not None and 'bias_mask' in masks: + op.bias_mask = masks['bias_mask'] + + action = (m == 1).sum().item() / m.numel() + return action, d_prime, preserve_idx + + def _is_final_layer(self): + return self.cur_ind == len(self.prunable_idx) - 1 + + def _action_wall(self, action): + """ + Limit the action generated by DDPG for this layer by two constraints: + 1. The total flops must meet the flops reduce target. + For example: the original flops of entire model is 1000, target flops ratio is 0.5, target flops + is 1000*0.5 = 500. The reduced flops of other layers is 400, so the remaining flops quota is 500-400=100, + if the total original flops of this layer is 250, then the maximum ratio is 100/250 = 0.4. So the + action of this layer can not be greater than 0.4. + 2. The action must be greater than lbound which is stored in self.strategy_dict. + """ + assert len(self.strategy) == self.cur_ind + + action = float(action) + action = np.clip(action, 0, 1) + + other_comp = 0 + this_comp = 0 + for i, idx in enumerate(self.prunable_idx): + flop = self.layer_info_dict[idx]['flops'] + buffer_flop = self._get_buffer_flops(idx) + + if i == self.cur_ind - 1: # TODO: add other member in the set + this_comp += flop * self.strategy_dict[idx][0] + # add buffer (but not influenced by ratio) + other_comp += buffer_flop * self.strategy_dict[idx][0] + elif i == self.cur_ind: + this_comp += flop * self.strategy_dict[idx][1] + # also add buffer here (influenced by ratio) + this_comp += buffer_flop + else: + other_comp += flop * self.strategy_dict[idx][0] * self.strategy_dict[idx][1] + # add buffer + other_comp += buffer_flop * self.strategy_dict[idx][0] # only consider input reduction + + self.expected_min_preserve = other_comp + this_comp * action + max_preserve_ratio = (self.expected_preserve_computation - other_comp) * 1. / this_comp + + action = np.minimum(action, max_preserve_ratio) + action = np.maximum(action, self.strategy_dict[self.prunable_idx[self.cur_ind]][0]) # impossible (should be) + + return action + + def _get_buffer_flops(self, idx): + buffer_idx = self.buffer_dict[idx] + buffer_flop = sum([self.layer_info_dict[_]['flops'] for _ in buffer_idx]) + return buffer_flop + + def _cur_flops(self): + flops = 0 + for idx in self.prunable_idx: + c, n = self.strategy_dict[idx] # input, output pruning ratio + flops += self.layer_info_dict[idx]['flops'] * c * n + # add buffer computation + flops += self._get_buffer_flops(idx) * c # only related to input channel reduction + return flops + + def _cur_reduced(self): + # return the reduced weight + reduced = self.org_flops - self._cur_flops() + return reduced + + def _build_index(self): + """ + Build following information/data for later pruning: + self.prunable_idx: layer indices for pruable layers, the index values are the index + of list(self.model.modules()). Pruable layers are pointwise Conv2d layers and Linear + layers. + self.prunable_ops: prunable modules + self.buffer_idx: layer indices for buffer layers which refers the depthwise layers. + Each depthwise layer is always followd by a pointwise layer for both mobilenet and + mobilenetv2. The depthwise layer's filters are pruned when its next pointwise layer's + corresponding input channels are pruned. + self.shared_idx: layer indices for layers which share input. + For example: [[1,4], [8, 10, 15]] means layer 1 and 4 share same input, and layer + 8, 10 and 15 share another input. + self.org_channels: number of input channels for each layer + self.min_strategy_dict: key is layer index, value is a tuple, the first value is the minimum + action of input channel, the second value is the minimum action value of output channel. + self.strategy_dict: same as self.min_strategy_dict, but it will be updated later. + """ + self.prunable_idx = [] + self.prunable_ops = [] + self.layer_type_dict = {} + self.strategy_dict = {} + self.buffer_dict = {} + this_buffer_list = [] + self.org_channels = [] + # build index and the min strategy dict + for i, m in enumerate(self.model.modules()): + if isinstance(m, PrunerModuleWrapper): + m = m.module + if type(m) == nn.Conv2d and m.groups == m.in_channels: # depth-wise conv, buffer + this_buffer_list.append(i) + else: # really prunable + self.prunable_idx.append(i) + self.prunable_ops.append(m) + self.layer_type_dict[i] = type(m) + self.buffer_dict[i] = this_buffer_list + this_buffer_list = [] # empty + self.org_channels.append(m.in_channels if type(m) == nn.Conv2d else m.in_features) + + self.strategy_dict[i] = [self.lbound, self.lbound] + + self.strategy_dict[self.prunable_idx[0]][0] = 1 # modify the input + self.strategy_dict[self.prunable_idx[-1]][1] = 1 # modify the output + + self.shared_idx = [] + if self.args.model_type == 'mobilenetv2': # TODO: to be tested! Share index for residual connection + connected_idx = [4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32] # to be partitioned + last_ch = -1 + share_group = None + for c_idx in connected_idx: + if self.prunable_ops[c_idx].in_channels != last_ch: # new group + last_ch = self.prunable_ops[c_idx].in_channels + if share_group is not None: + self.shared_idx.append(share_group) + share_group = [c_idx] + else: # same group + share_group.append(c_idx) + self.shared_idx.append(share_group) + _logger.info('=> Conv layers to share channels: %s', self.shared_idx) + + self.min_strategy_dict = copy.deepcopy(self.strategy_dict) + + self.buffer_idx = [] + for _, v in self.buffer_dict.items(): + self.buffer_idx += v + + _logger.info('=> Prunable layer idx: %s', self.prunable_idx) + _logger.info('=> Buffer layer idx: %s', self.buffer_idx) + _logger.info('=> Shared idx: %s', self.shared_idx) + _logger.info('=> Initial min strategy dict: %s', self.min_strategy_dict) + + # added for supporting residual connections during pruning + self.visited = [False] * len(self.prunable_idx) + self.index_buffer = {} + + def _extract_layer_information(self): + m_list = list(self.model.modules()) + + self.data_saver = [] + self.layer_info_dict = dict() + self.wsize_list = [] + self.flops_list = [] + + from .lib.utils import measure_layer_for_pruning + + # extend the forward fn to record layer info + def new_forward(m): + def lambda_forward(x): + m.input_feat = x.clone() + #TODO replace this flops counter with nni.compression.torch.utils.counter.count_flops_params + measure_layer_for_pruning(m, x) + y = m.old_forward(x) + m.output_feat = y.clone() + return y + + return lambda_forward + + device = None + for idx in self.prunable_idx + self.buffer_idx: # get all + m = m_list[idx] + m.old_forward = m.forward + m.forward = new_forward(m) + if device is None and type(m) == PrunerModuleWrapper: + device = m.module.weight.device + + # now let the image flow + _logger.info('=> Extracting information...') + with torch.no_grad(): + for i_b, (inputs, target) in enumerate(self._val_loader): # use image from train set + if i_b == self.n_calibration_batches: + break + self.data_saver.append((inputs.clone(), target.clone())) + input_var = torch.autograd.Variable(inputs).to(device) + + # inference and collect stats + _ = self.model(input_var) + + if i_b == 0: # first batch + for idx in self.prunable_idx + self.buffer_idx: + self.layer_info_dict[idx] = dict() + self.layer_info_dict[idx]['params'] = m_list[idx].params + self.layer_info_dict[idx]['flops'] = m_list[idx].flops + self.wsize_list.append(m_list[idx].params) + self.flops_list.append(m_list[idx].flops) + _logger.info('flops: %s', self.flops_list) + for idx in self.prunable_idx: + f_in_np = m_list[idx].input_feat.data.cpu().numpy() + f_out_np = m_list[idx].output_feat.data.cpu().numpy() + if len(f_in_np.shape) == 4: # conv + if self.prunable_idx.index(idx) == 0: # first conv + f_in2save, f_out2save = None, None + elif m_list[idx].module.weight.size(3) > 1: # normal conv + f_in2save, f_out2save = f_in_np, f_out_np + else: # 1x1 conv + # assert f_out_np.shape[2] == f_in_np.shape[2] # now support k=3 + randx = np.random.randint(0, f_out_np.shape[2] - 0, self.n_points_per_layer) + randy = np.random.randint(0, f_out_np.shape[3] - 0, self.n_points_per_layer) + # input: [N, C, H, W] + self.layer_info_dict[idx][(i_b, 'randx')] = randx.copy() + self.layer_info_dict[idx][(i_b, 'randy')] = randy.copy() + + f_in2save = f_in_np[:, :, randx, randy].copy().transpose(0, 2, 1)\ + .reshape(self.batch_size * self.n_points_per_layer, -1) + + f_out2save = f_out_np[:, :, randx, randy].copy().transpose(0, 2, 1) \ + .reshape(self.batch_size * self.n_points_per_layer, -1) + else: + assert len(f_in_np.shape) == 2 + f_in2save = f_in_np.copy() + f_out2save = f_out_np.copy() + if 'input_feat' not in self.layer_info_dict[idx]: + self.layer_info_dict[idx]['input_feat'] = f_in2save + self.layer_info_dict[idx]['output_feat'] = f_out2save + else: + self.layer_info_dict[idx]['input_feat'] = np.vstack( + (self.layer_info_dict[idx]['input_feat'], f_in2save)) + self.layer_info_dict[idx]['output_feat'] = np.vstack( + (self.layer_info_dict[idx]['output_feat'], f_out2save)) + + def _build_state_embedding(self): + # build the static part of the state embedding + _logger.info('Building state embedding...') + layer_embedding = [] + module_list = list(self.model.modules()) + for i, ind in enumerate(self.prunable_idx): + m = module_list[ind].module + this_state = [] + if type(m) == nn.Conv2d: + this_state.append(i) # index + this_state.append(0) # layer type, 0 for conv + this_state.append(m.in_channels) # in channels + this_state.append(m.out_channels) # out channels + this_state.append(m.stride[0]) # stride + this_state.append(m.kernel_size[0]) # kernel size + this_state.append(np.prod(m.weight.size())) # weight size + elif type(m) == nn.Linear: + this_state.append(i) # index + this_state.append(1) # layer type, 1 for fc + this_state.append(m.in_features) # in channels + this_state.append(m.out_features) # out channels + this_state.append(0) # stride + this_state.append(1) # kernel size + this_state.append(np.prod(m.weight.size())) # weight size + + # this 3 features need to be changed later + this_state.append(0.) # reduced + this_state.append(0.) # rest + this_state.append(1.) # a_{t-1} + layer_embedding.append(np.array(this_state)) + + # normalize the state + layer_embedding = np.array(layer_embedding, 'float') + _logger.info('=> shape of embedding (n_layer * n_dim): %s', layer_embedding.shape) + assert len(layer_embedding.shape) == 2, layer_embedding.shape + for i in range(layer_embedding.shape[1]): + fmin = min(layer_embedding[:, i]) + fmax = max(layer_embedding[:, i]) + if fmax - fmin > 0: + layer_embedding[:, i] = (layer_embedding[:, i] - fmin) / (fmax - fmin) + + self.layer_embedding = layer_embedding + diff --git a/new_impl/cv/third_party/nni/algorithms/compression/pytorch/pruning/amc/lib/__init__.py b/new_impl/cv/third_party/nni/algorithms/compression/pytorch/pruning/amc/lib/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/new_impl/cv/third_party/nni/algorithms/compression/pytorch/pruning/amc/lib/agent.py b/new_impl/cv/third_party/nni/algorithms/compression/pytorch/pruning/amc/lib/agent.py new file mode 100644 index 0000000000000000000000000000000000000000..fe066301b8b5e4325f92a1b98885ae547a7ecee3 --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/compression/pytorch/pruning/amc/lib/agent.py @@ -0,0 +1,232 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import numpy as np + +import torch +import torch.nn as nn +from torch.optim import Adam + +from .memory import SequentialMemory +from .utils import to_numpy, to_tensor + +criterion = nn.MSELoss() +USE_CUDA = torch.cuda.is_available() + + +class Actor(nn.Module): + def __init__(self, nb_states, nb_actions, hidden1=400, hidden2=300): + super(Actor, self).__init__() + self.fc1 = nn.Linear(nb_states, hidden1) + self.fc2 = nn.Linear(hidden1, hidden2) + self.fc3 = nn.Linear(hidden2, nb_actions) + self.relu = nn.ReLU() + self.sigmoid = nn.Sigmoid() + + def forward(self, x): + out = self.fc1(x) + out = self.relu(out) + out = self.fc2(out) + out = self.relu(out) + out = self.fc3(out) + out = self.sigmoid(out) + return out + + +class Critic(nn.Module): + def __init__(self, nb_states, nb_actions, hidden1=400, hidden2=300): + super(Critic, self).__init__() + self.fc11 = nn.Linear(nb_states, hidden1) + self.fc12 = nn.Linear(nb_actions, hidden1) + self.fc2 = nn.Linear(hidden1, hidden2) + self.fc3 = nn.Linear(hidden2, 1) + self.relu = nn.ReLU() + + def forward(self, xs): + x, a = xs + out = self.fc11(x) + self.fc12(a) + out = self.relu(out) + out = self.fc2(out) + out = self.relu(out) + out = self.fc3(out) + return out + + +class DDPG(object): + def __init__(self, nb_states, nb_actions, args): + + self.nb_states = nb_states + self.nb_actions = nb_actions + + # Create Actor and Critic Network + net_cfg = { + 'hidden1': args.hidden1, + 'hidden2': args.hidden2, + # 'init_w': args.init_w + } + self.actor = Actor(self.nb_states, self.nb_actions, **net_cfg) + self.actor_target = Actor(self.nb_states, self.nb_actions, **net_cfg) + self.actor_optim = Adam(self.actor.parameters(), lr=args.lr_a) + + self.critic = Critic(self.nb_states, self.nb_actions, **net_cfg) + self.critic_target = Critic(self.nb_states, self.nb_actions, **net_cfg) + self.critic_optim = Adam(self.critic.parameters(), lr=args.lr_c) + + self.hard_update(self.actor_target, self.actor) # Make sure target is with the same weight + self.hard_update(self.critic_target, self.critic) + + # Create replay buffer + self.memory = SequentialMemory(limit=args.rmsize, window_length=args.window_length) + # self.random_process = OrnsteinUhlenbeckProcess(size=nb_actions, theta=args.ou_theta, mu=args.ou_mu, + # sigma=args.ou_sigma) + + # Hyper-parameters + self.batch_size = args.bsize + self.tau = args.tau + self.discount = args.discount + self.depsilon = 1.0 / args.epsilon + self.lbound = 0. # args.lbound + self.rbound = 1. # args.rbound + + # noise + self.init_delta = args.init_delta + self.delta_decay = args.delta_decay + self.warmup = args.warmup + + # + self.epsilon = 1.0 + # self.s_t = None # Most recent state + # self.a_t = None # Most recent action + self.is_training = True + + # + if USE_CUDA: self.cuda() + + # moving average baseline + self.moving_average = None + self.moving_alpha = 0.5 # based on batch, so small + + def update_policy(self): + # Sample batch + state_batch, action_batch, reward_batch, \ + next_state_batch, terminal_batch = self.memory.sample_and_split(self.batch_size) + + # normalize the reward + batch_mean_reward = np.mean(reward_batch) + if self.moving_average is None: + self.moving_average = batch_mean_reward + else: + self.moving_average += self.moving_alpha * (batch_mean_reward - self.moving_average) + reward_batch -= self.moving_average + # if reward_batch.std() > 0: + # reward_batch /= reward_batch.std() + + # Prepare for the target q batch + with torch.no_grad(): + next_q_values = self.critic_target([ + to_tensor(next_state_batch), + self.actor_target(to_tensor(next_state_batch)), + ]) + + target_q_batch = to_tensor(reward_batch) + \ + self.discount * to_tensor(terminal_batch.astype(np.float)) * next_q_values + + # Critic update + self.critic.zero_grad() + + q_batch = self.critic([to_tensor(state_batch), to_tensor(action_batch)]) + + value_loss = criterion(q_batch, target_q_batch) + value_loss.backward() + self.critic_optim.step() + + # Actor update + self.actor.zero_grad() + + policy_loss = -self.critic([ # pylint: disable=all + to_tensor(state_batch), + self.actor(to_tensor(state_batch)) + ]) + + policy_loss = policy_loss.mean() + policy_loss.backward() + self.actor_optim.step() + + # Target update + self.soft_update(self.actor_target, self.actor) + self.soft_update(self.critic_target, self.critic) + + def eval(self): + self.actor.eval() + self.actor_target.eval() + self.critic.eval() + self.critic_target.eval() + + def cuda(self): + self.actor.cuda() + self.actor_target.cuda() + self.critic.cuda() + self.critic_target.cuda() + + def observe(self, r_t, s_t, s_t1, a_t, done): + if self.is_training: + self.memory.append(s_t, a_t, r_t, done) # save to memory + # self.s_t = s_t1 + + def random_action(self): + action = np.random.uniform(self.lbound, self.rbound, self.nb_actions) + # self.a_t = action + return action + + def select_action(self, s_t, episode): + # assert episode >= self.warmup, 'Episode: {} warmup: {}'.format(episode, self.warmup) + action = to_numpy(self.actor(to_tensor(np.array(s_t).reshape(1, -1)))).squeeze(0) + delta = self.init_delta * (self.delta_decay ** (episode - self.warmup)) + # action += self.is_training * max(self.epsilon, 0) * self.random_process.sample() + action = self.sample_from_truncated_normal_distribution(lower=self.lbound, upper=self.rbound, mu=action, sigma=delta) + action = np.clip(action, self.lbound, self.rbound) + + # self.a_t = action + return action + + def reset(self, obs): + pass + # self.s_t = obs + # self.random_process.reset_states() + + def load_weights(self, output): + if output is None: return + + self.actor.load_state_dict( + torch.load('{}/actor.pkl'.format(output)) + ) + + self.critic.load_state_dict( + torch.load('{}/critic.pkl'.format(output)) + ) + + def save_model(self, output): + torch.save( + self.actor.state_dict(), + '{}/actor.pkl'.format(output) + ) + torch.save( + self.critic.state_dict(), + '{}/critic.pkl'.format(output) + ) + + def soft_update(self, target, source): + for target_param, param in zip(target.parameters(), source.parameters()): + target_param.data.copy_( + target_param.data * (1.0 - self.tau) + param.data * self.tau + ) + + def hard_update(self, target, source): + for target_param, param in zip(target.parameters(), source.parameters()): + target_param.data.copy_(param.data) + + def sample_from_truncated_normal_distribution(self, lower, upper, mu, sigma, size=1): + from scipy import stats + return stats.truncnorm.rvs((lower-mu)/sigma, (upper-mu)/sigma, loc=mu, scale=sigma, size=size) + + diff --git a/new_impl/cv/third_party/nni/algorithms/compression/pytorch/pruning/amc/lib/memory.py b/new_impl/cv/third_party/nni/algorithms/compression/pytorch/pruning/amc/lib/memory.py new file mode 100644 index 0000000000000000000000000000000000000000..57bbcfceb86a20092968c9dc75a618221e119174 --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/compression/pytorch/pruning/amc/lib/memory.py @@ -0,0 +1,227 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from __future__ import absolute_import +from collections import deque, namedtuple +import warnings +import random + +import numpy as np + +# [reference] https://github.com/matthiasplappert/keras-rl/blob/master/rl/memory.py + +# This is to be understood as a transition: Given `state0`, performing `action` +# yields `reward` and results in `state1`, which might be `terminal`. +Experience = namedtuple('Experience', 'state0, action, reward, state1, terminal1') + + +def sample_batch_indexes(low, high, size): + if high - low >= size: + # We have enough data. Draw without replacement, that is each index is unique in the + # batch. We cannot use `np.random.choice` here because it is horribly inefficient as + # the memory grows. See https://github.com/numpy/numpy/issues/2764 for a discussion. + # `random.sample` does the same thing (drawing without replacement) and is way faster. + r = range(low, high) + batch_idxs = random.sample(r, size) + else: + # Not enough data. Help ourselves with sampling from the range, but the same index + # can occur multiple times. This is not good and should be avoided by picking a + # large enough warm-up phase. + warnings.warn( + 'Not enough entries to sample without replacement. ' + 'Consider increasing your warm-up phase to avoid oversampling!') + batch_idxs = np.random.random_integers(low, high - 1, size=size) + assert len(batch_idxs) == size + return batch_idxs + + +class RingBuffer(object): + def __init__(self, maxlen): + self.maxlen = maxlen + self.start = 0 + self.length = 0 + self.data = [None for _ in range(maxlen)] + + def __len__(self): + return self.length + + def __getitem__(self, idx): + if idx < 0 or idx >= self.length: + raise KeyError() + return self.data[(self.start + idx) % self.maxlen] + + def append(self, v): + if self.length < self.maxlen: + # We have space, simply increase the length. + self.length += 1 + elif self.length == self.maxlen: + # No space, "remove" the first item. + self.start = (self.start + 1) % self.maxlen + else: + # This should never happen. + raise RuntimeError() + self.data[(self.start + self.length - 1) % self.maxlen] = v + + +def zeroed_observation(observation): + if hasattr(observation, 'shape'): + return np.zeros(observation.shape) + elif hasattr(observation, '__iter__'): + out = [] + for x in observation: + out.append(zeroed_observation(x)) + return out + else: + return 0. + + +class Memory(object): + def __init__(self, window_length, ignore_episode_boundaries=False): + self.window_length = window_length + self.ignore_episode_boundaries = ignore_episode_boundaries + + self.recent_observations = deque(maxlen=window_length) + self.recent_terminals = deque(maxlen=window_length) + + def sample(self, batch_size, batch_idxs=None): + raise NotImplementedError() + + def append(self, observation, action, reward, terminal, training=True): + self.recent_observations.append(observation) + self.recent_terminals.append(terminal) + + def get_recent_state(self, current_observation): + # This code is slightly complicated by the fact that subsequent observations might be + # from different episodes. We ensure that an experience never spans multiple episodes. + # This is probably not that important in practice but it seems cleaner. + state = [current_observation] + idx = len(self.recent_observations) - 1 + for offset in range(0, self.window_length - 1): + current_idx = idx - offset + current_terminal = self.recent_terminals[current_idx - 1] if current_idx - 1 >= 0 else False + if current_idx < 0 or (not self.ignore_episode_boundaries and current_terminal): + # The previously handled observation was terminal, don't add the current one. + # Otherwise we would leak into a different episode. + break + state.insert(0, self.recent_observations[current_idx]) + while len(state) < self.window_length: + state.insert(0, zeroed_observation(state[0])) + return state + + def get_config(self): + config = { + 'window_length': self.window_length, + 'ignore_episode_boundaries': self.ignore_episode_boundaries, + } + return config + + +class SequentialMemory(Memory): + def __init__(self, limit, **kwargs): + super(SequentialMemory, self).__init__(**kwargs) + + self.limit = limit + + # Do not use deque to implement the memory. This data structure may seem convenient but + # it is way too slow on random access. Instead, we use our own ring buffer implementation. + self.actions = RingBuffer(limit) + self.rewards = RingBuffer(limit) + self.terminals = RingBuffer(limit) + self.observations = RingBuffer(limit) + + def sample(self, batch_size, batch_idxs=None): + if batch_idxs is None: + # Draw random indexes such that we have at least a single entry before each + # index. + batch_idxs = sample_batch_indexes(0, self.nb_entries - 1, size=batch_size) + batch_idxs = np.array(batch_idxs) + 1 + assert np.min(batch_idxs) >= 1 + assert np.max(batch_idxs) < self.nb_entries + assert len(batch_idxs) == batch_size + + # Create experiences + experiences = [] + for idx in batch_idxs: + terminal0 = self.terminals[idx - 2] if idx >= 2 else False + while terminal0: + # Skip this transition because the environment was reset here. Select a new, random + # transition and use this instead. This may cause the batch to contain the same + # transition twice. + idx = sample_batch_indexes(1, self.nb_entries, size=1)[0] + terminal0 = self.terminals[idx - 2] if idx >= 2 else False + assert 1 <= idx < self.nb_entries + + # This code is slightly complicated by the fact that subsequent observations might be + # from different episodes. We ensure that an experience never spans multiple episodes. + # This is probably not that important in practice but it seems cleaner. + state0 = [self.observations[idx - 1]] + for offset in range(0, self.window_length - 1): + current_idx = idx - 2 - offset + current_terminal = self.terminals[current_idx - 1] if current_idx - 1 > 0 else False + if current_idx < 0 or (not self.ignore_episode_boundaries and current_terminal): + # The previously handled observation was terminal, don't add the current one. + # Otherwise we would leak into a different episode. + break + state0.insert(0, self.observations[current_idx]) + while len(state0) < self.window_length: + state0.insert(0, zeroed_observation(state0[0])) + action = self.actions[idx - 1] + reward = self.rewards[idx - 1] + terminal1 = self.terminals[idx - 1] + + # Okay, now we need to create the follow-up state. This is state0 shifted on timestep + # to the right. Again, we need to be careful to not include an observation from the next + # episode if the last state is terminal. + state1 = [np.copy(x) for x in state0[1:]] + state1.append(self.observations[idx]) + + assert len(state0) == self.window_length + assert len(state1) == len(state0) + experiences.append(Experience(state0=state0, action=action, reward=reward, + state1=state1, terminal1=terminal1)) + assert len(experiences) == batch_size + return experiences + + def sample_and_split(self, batch_size, batch_idxs=None): + experiences = self.sample(batch_size, batch_idxs) + + state0_batch = [] + reward_batch = [] + action_batch = [] + terminal1_batch = [] + state1_batch = [] + for e in experiences: + state0_batch.append(e.state0) + state1_batch.append(e.state1) + reward_batch.append(e.reward) + action_batch.append(e.action) + terminal1_batch.append(0. if e.terminal1 else 1.) + + # Prepare and validate parameters. + state0_batch = np.array(state0_batch, 'double').reshape(batch_size, -1) + state1_batch = np.array(state1_batch, 'double').reshape(batch_size, -1) + terminal1_batch = np.array(terminal1_batch, 'double').reshape(batch_size, -1) + reward_batch = np.array(reward_batch, 'double').reshape(batch_size, -1) + action_batch = np.array(action_batch, 'double').reshape(batch_size, -1) + + return state0_batch, action_batch, reward_batch, state1_batch, terminal1_batch + + def append(self, observation, action, reward, terminal, training=True): + super(SequentialMemory, self).append(observation, action, reward, terminal, training=training) + + # This needs to be understood as follows: in `observation`, take `action`, obtain `reward` + # and weather the next state is `terminal` or not. + if training: + self.observations.append(observation) + self.actions.append(action) + self.rewards.append(reward) + self.terminals.append(terminal) + + @property + def nb_entries(self): + return len(self.observations) + + def get_config(self): + config = super(SequentialMemory, self).get_config() + config['limit'] = self.limit + return config diff --git a/new_impl/cv/third_party/nni/algorithms/compression/pytorch/pruning/amc/lib/net_measure.py b/new_impl/cv/third_party/nni/algorithms/compression/pytorch/pruning/amc/lib/net_measure.py new file mode 100644 index 0000000000000000000000000000000000000000..2c9e815116288c11de98b4061b4dbfc3df029a7e --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/compression/pytorch/pruning/amc/lib/net_measure.py @@ -0,0 +1,123 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import torch + +# [reference] https://github.com/ShichenLiu/CondenseNet/blob/master/utils.py + + +def get_num_gen(gen): + return sum(1 for _ in gen) + + +def is_leaf(model): + return get_num_gen(model.children()) == 0 + + +def get_layer_info(layer): + layer_str = str(layer) + type_name = layer_str[:layer_str.find('(')].strip() + return type_name + + +def get_layer_param(model): + import operator + import functools + + return sum([functools.reduce(operator.mul, i.size(), 1) for i in model.parameters()]) + +count_ops = 0 +count_params = 0 + +def measure_layer(layer, x): + global count_ops, count_params + delta_ops = 0 + delta_params = 0 + multi_add = 1 + type_name = get_layer_info(layer) + + # ops_conv + if type_name in ['Conv2d']: + out_h = int((x.size()[2] + 2 * layer.padding[0] - layer.kernel_size[0]) / + layer.stride[0] + 1) + out_w = int((x.size()[3] + 2 * layer.padding[1] - layer.kernel_size[1]) / + layer.stride[1] + 1) + delta_ops = layer.in_channels * layer.out_channels * layer.kernel_size[0] * \ + layer.kernel_size[1] * out_h * out_w / layer.groups * multi_add + delta_params = get_layer_param(layer) + + # ops_nonlinearity + elif type_name in ['ReLU']: + delta_ops = x.numel() / x.size(0) + delta_params = get_layer_param(layer) + + # ops_pooling + elif type_name in ['AvgPool2d']: + in_w = x.size()[2] + kernel_ops = layer.kernel_size * layer.kernel_size + out_w = int((in_w + 2 * layer.padding - layer.kernel_size) / layer.stride + 1) + out_h = int((in_w + 2 * layer.padding - layer.kernel_size) / layer.stride + 1) + delta_ops = x.size()[1] * out_w * out_h * kernel_ops + delta_params = get_layer_param(layer) + + elif type_name in ['AdaptiveAvgPool2d']: + delta_ops = x.size()[1] * x.size()[2] * x.size()[3] + delta_params = get_layer_param(layer) + + # ops_linear + elif type_name in ['Linear']: + weight_ops = layer.weight.numel() * multi_add + bias_ops = layer.bias.numel() + delta_ops = weight_ops + bias_ops + delta_params = get_layer_param(layer) + + # ops_nothing + elif type_name in ['BatchNorm2d', 'Dropout2d', 'DropChannel', 'Dropout']: + delta_params = get_layer_param(layer) + + # unknown layer type + else: + delta_params = get_layer_param(layer) + + count_ops += delta_ops + count_params += delta_params + + return + + +def measure_model(model, H, W, device): + global count_ops, count_params + count_ops = 0 + count_params = 0 + data = torch.zeros(2, 3, H, W).to(device) + + def should_measure(x): + return is_leaf(x) + + def modify_forward(model): + for child in model.children(): + if should_measure(child): + def new_forward(m): + def lambda_forward(x): + measure_layer(m, x) + return m.old_forward(x) + return lambda_forward + child.old_forward = child.forward + child.forward = new_forward(child) + else: + modify_forward(child) + + def restore_forward(model): + for child in model.children(): + # leaf node + if is_leaf(child) and hasattr(child, 'old_forward'): + child.forward = child.old_forward + child.old_forward = None + else: + restore_forward(child) + + modify_forward(model) + model.forward(data) + restore_forward(model) + + return count_ops, count_params diff --git a/new_impl/cv/third_party/nni/algorithms/compression/pytorch/pruning/amc/lib/utils.py b/new_impl/cv/third_party/nni/algorithms/compression/pytorch/pruning/amc/lib/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..f875e8d7d9b363d848aa2424b4f06ddaeb1dc1a6 --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/compression/pytorch/pruning/amc/lib/utils.py @@ -0,0 +1,113 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +import torch + +class TextLogger(object): + """Write log immediately to the disk""" + def __init__(self, filepath): + self.f = open(filepath, 'w') + self.fid = self.f.fileno() + self.filepath = filepath + + def close(self): + self.f.close() + + def write(self, content): + self.f.write(content) + self.f.flush() + os.fsync(self.fid) + + def write_buf(self, content): + self.f.write(content) + + def print_and_write(self, content): + print(content) + self.write(content+'\n') + +def to_numpy(var): + use_cuda = torch.cuda.is_available() + return var.cpu().data.numpy() if use_cuda else var.data.numpy() + + +def to_tensor(ndarray, requires_grad=False): # return a float tensor by default + tensor = torch.from_numpy(ndarray).float() # by default does not require grad + if requires_grad: + tensor.requires_grad_() + return tensor.cuda() if torch.cuda.is_available() else tensor + + +def measure_layer_for_pruning(wrapper, x): + def get_layer_type(layer): + layer_str = str(layer) + return layer_str[:layer_str.find('(')].strip() + + def get_layer_param(model): + import operator + import functools + + return sum([functools.reduce(operator.mul, i.size(), 1) for i in model.parameters()]) + + multi_add = 1 + layer = wrapper.module + type_name = get_layer_type(layer) + + # ops_conv + if type_name in ['Conv2d']: + out_h = int((x.size()[2] + 2 * layer.padding[0] - layer.kernel_size[0]) / + layer.stride[0] + 1) + out_w = int((x.size()[3] + 2 * layer.padding[1] - layer.kernel_size[1]) / + layer.stride[1] + 1) + wrapper.flops = layer.in_channels * layer.out_channels * layer.kernel_size[0] * \ + layer.kernel_size[1] * out_h * out_w / layer.groups * multi_add + wrapper.params = get_layer_param(layer) + # ops_linear + elif type_name in ['Linear']: + weight_ops = layer.weight.numel() * multi_add + bias_ops = layer.bias.numel() + wrapper.flops = weight_ops + bias_ops + wrapper.params = get_layer_param(layer) + return + + +def least_square_sklearn(X, Y): + from sklearn.linear_model import LinearRegression + reg = LinearRegression(fit_intercept=False) + reg.fit(X, Y) + return reg.coef_ + + +def get_output_folder(parent_dir, env_name): + """Return save folder. + Assumes folders in the parent_dir have suffix -run{run + number}. Finds the highest run number and sets the output folder + to that number + 1. This is just convenient so that if you run the + same script multiple times tensorboard can plot all of the results + on the same plots with different names. + Parameters + ---------- + parent_dir: str + Path of the directory containing all experiment runs. + Returns + ------- + parent_dir/run_dir + Path to this run's save directory. + """ + os.makedirs(parent_dir, exist_ok=True) + experiment_id = 0 + for folder_name in os.listdir(parent_dir): + if not os.path.isdir(os.path.join(parent_dir, folder_name)): + continue + try: + folder_name = int(folder_name.split('-run')[-1]) + if folder_name > experiment_id: + experiment_id = folder_name + except: + pass + experiment_id += 1 + + parent_dir = os.path.join(parent_dir, env_name) + parent_dir = parent_dir + '-run{}'.format(experiment_id) + os.makedirs(parent_dir, exist_ok=True) + return parent_dir diff --git a/new_impl/cv/third_party/nni/algorithms/compression/pytorch/pruning/apply_compression.py b/new_impl/cv/third_party/nni/algorithms/compression/pytorch/pruning/apply_compression.py new file mode 100644 index 0000000000000000000000000000000000000000..8e6b023f5b90b8483e21bd0bc575b19b4a4df023 --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/compression/pytorch/pruning/apply_compression.py @@ -0,0 +1,29 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +import torch + +logger = logging.getLogger('torch apply compression') + +def apply_compression_results(model, masks_file, map_location=None): + """ + Apply the masks from ```masks_file``` to the model + Note: this API is for inference, because it simply multiplies weights with + corresponding masks when this API is called. + + Parameters + ---------- + model : torch.nn.Module + The model to be compressed + masks_file : str + The path of the mask file + map_location : str + the device on which masks are placed, same to map_location in ```torch.load``` + """ + masks = torch.load(masks_file, map_location) + for name, module in model.named_modules(): + if name in masks: + module.weight.data = module.weight.data.mul_(masks[name]['weight']) + if hasattr(module, 'bias') and module.bias is not None and 'bias' in masks[name]: + module.bias.data = module.bias.data.mul_(masks[name]['bias']) \ No newline at end of file diff --git a/new_impl/cv/third_party/nni/algorithms/compression/pytorch/pruning/auto_compress_pruner.py b/new_impl/cv/third_party/nni/algorithms/compression/pytorch/pruning/auto_compress_pruner.py new file mode 100644 index 0000000000000000000000000000000000000000..d52c6ec42da02cf84e0dd87c138f30376404682d --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/compression/pytorch/pruning/auto_compress_pruner.py @@ -0,0 +1,251 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +import os +import copy +import torch +from schema import And, Optional + +from nni.utils import OptimizeMode +from nni.compression.pytorch import ModelSpeedup + +from nni.compression.pytorch.compressor import Pruner +from nni.compression.pytorch.utils.config_validation import CompressorSchema +from .simulated_annealing_pruner import SimulatedAnnealingPruner +from .admm_pruner import ADMMPruner + + +_logger = logging.getLogger(__name__) + + +class AutoCompressPruner(Pruner): + """ + A Pytorch implementation of AutoCompress pruning algorithm. + + Parameters + ---------- + model : pytorch model + The model to be pruned. + config_list : list + Supported keys: + - sparsity : The target overall sparsity. + - op_types : The operation type to prune. + trainer : function + Function used for the first subproblem of ADMM Pruner. + Users should write this function as a normal function to train the Pytorch model + and include `model, optimizer, criterion, epoch, callback` as function arguments. + Here `callback` acts as an L2 regulizer as presented in the formula (7) of the original paper. + The logic of `callback` is implemented inside the Pruner, + users are just required to insert `callback()` between `loss.backward()` and `optimizer.step()`. + Example:: + + def trainer(model, criterion, optimizer, epoch, callback): + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + train_loader = ... + model.train() + for batch_idx, (data, target) in enumerate(train_loader): + data, target = data.to(device), target.to(device) + optimizer.zero_grad() + output = model(data) + loss = criterion(output, target) + loss.backward() + # callback should be inserted between loss.backward() and optimizer.step() + if callback: + callback() + optimizer.step() + evaluator : function + function to evaluate the pruned model. + This function should include `model` as the only parameter, and returns a scalar value. + Example:: + + def evaluator(model): + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + val_loader = ... + model.eval() + correct = 0 + with torch.no_grad(): + for data, target in val_loader: + data, target = data.to(device), target.to(device) + output = model(data) + # get the index of the max log-probability + pred = output.argmax(dim=1, keepdim=True) + correct += pred.eq(target.view_as(pred)).sum().item() + accuracy = correct / len(val_loader.dataset) + return accuracy + dummy_input : pytorch tensor + The dummy input for ```jit.trace```, users should put it on right device before pass in. + num_iterations : int + Number of overall iterations. + optimize_mode : str + optimize mode, `maximize` or `minimize`, by default `maximize`. + base_algo : str + Base pruning algorithm. `level`, `l1` or `l2`, by default `l1`. Given the sparsity distribution among the ops, + the assigned `base_algo` is used to decide which filters/channels/weights to prune. + start_temperature : float + Start temperature of the simulated annealing process. + stop_temperature : float + Stop temperature of the simulated annealing process. + cool_down_rate : float + Cool down rate of the temperature. + perturbation_magnitude : float + Initial perturbation magnitude to the sparsities. The magnitude decreases with current temperature. + admm_num_iterations : int + Number of iterations of ADMM Pruner. + admm_training_epochs : int + Training epochs of the first optimization subproblem of ADMMPruner. + row : float + Penalty parameters for ADMM training. + experiment_data_dir : string + PATH to store temporary experiment data. + """ + + def __init__(self, model, config_list, trainer, evaluator, dummy_input, + num_iterations=3, optimize_mode='maximize', base_algo='l1', + # SimulatedAnnealing related + start_temperature=100, stop_temperature=20, cool_down_rate=0.9, perturbation_magnitude=0.35, + # ADMM related + admm_num_iterations=30, admm_training_epochs=5, row=1e-4, + experiment_data_dir='./'): + # original model + self._model_to_prune = model + self._base_algo = base_algo + + self._trainer = trainer + self._evaluator = evaluator + self._dummy_input = dummy_input + self._num_iterations = num_iterations + self._optimize_mode = OptimizeMode(optimize_mode) + + # hyper parameters for SA algorithm + self._start_temperature = start_temperature + self._stop_temperature = stop_temperature + self._cool_down_rate = cool_down_rate + self._perturbation_magnitude = perturbation_magnitude + + # hyper parameters for ADMM algorithm + self._admm_num_iterations = admm_num_iterations + self._admm_training_epochs = admm_training_epochs + self._row = row + + # overall pruning rate + self._sparsity = config_list[0]['sparsity'] + + self._experiment_data_dir = experiment_data_dir + if not os.path.exists(self._experiment_data_dir): + os.makedirs(self._experiment_data_dir) + + def validate_config(self, model, config_list): + """ + Parameters + ---------- + model : torch.nn.Module + Model to be pruned + config_list : list + List on pruning configs + """ + + if self._base_algo == 'level': + schema = CompressorSchema([{ + 'sparsity': And(float, lambda n: 0 < n < 1), + Optional('op_types'): [str], + Optional('op_names'): [str], + }], model, _logger) + elif self._base_algo in ['l1', 'l2']: + schema = CompressorSchema([{ + 'sparsity': And(float, lambda n: 0 < n < 1), + 'op_types': ['Conv2d'], + Optional('op_names'): [str] + }], model, _logger) + + schema.validate(config_list) + + def calc_mask(self, wrapper, **kwargs): + return None + + def compress(self): + """ + Compress the model with AutoCompress. + + Returns + ------- + torch.nn.Module + model with specified modules compressed. + """ + _logger.info('Starting AutoCompress pruning...') + + sparsity_each_round = 1 - pow(1-self._sparsity, 1/self._num_iterations) + + for i in range(self._num_iterations): + _logger.info('Pruning iteration: %d', i) + _logger.info('Target sparsity this round: %s', + 1-pow(1-sparsity_each_round, i+1)) + + # SimulatedAnnealingPruner + _logger.info( + 'Generating sparsities with SimulatedAnnealingPruner...') + SApruner = SimulatedAnnealingPruner( + model=copy.deepcopy(self._model_to_prune), + config_list=[ + {"sparsity": sparsity_each_round, "op_types": ['Conv2d']}], + evaluator=self._evaluator, + optimize_mode=self._optimize_mode, + base_algo=self._base_algo, + start_temperature=self._start_temperature, + stop_temperature=self._stop_temperature, + cool_down_rate=self._cool_down_rate, + perturbation_magnitude=self._perturbation_magnitude, + experiment_data_dir=self._experiment_data_dir) + config_list = SApruner.compress(return_config_list=True) + _logger.info("Generated config_list : %s", config_list) + + # ADMMPruner + _logger.info('Performing structured pruning with ADMMPruner...') + ADMMpruner = ADMMPruner( + model=copy.deepcopy(self._model_to_prune), + config_list=config_list, + trainer=self._trainer, + num_iterations=self._admm_num_iterations, + training_epochs=self._admm_training_epochs, + row=self._row, + base_algo=self._base_algo) + ADMMpruner.compress() + + ADMMpruner.export_model(os.path.join(self._experiment_data_dir, 'model_admm_masked.pth'), os.path.join( + self._experiment_data_dir, 'mask.pth')) + + # use speed up to prune the model before next iteration, because SimulatedAnnealingPruner & ADMMPruner don't take masked models + self._model_to_prune.load_state_dict(torch.load(os.path.join( + self._experiment_data_dir, 'model_admm_masked.pth'))) + + masks_file = os.path.join(self._experiment_data_dir, 'mask.pth') + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + + _logger.info('Speeding up models...') + m_speedup = ModelSpeedup(self._model_to_prune, self._dummy_input, masks_file, device) + m_speedup.speedup_model() + + evaluation_result = self._evaluator(self._model_to_prune) + _logger.info('Evaluation result of the pruned model in iteration %d: %s', i, evaluation_result) + + _logger.info('----------Compression finished--------------') + + os.remove(os.path.join(self._experiment_data_dir, 'model_admm_masked.pth')) + os.remove(os.path.join(self._experiment_data_dir, 'mask.pth')) + + return self._model_to_prune + + def export_model(self, model_path, mask_path=None, onnx_path=None, input_shape=None, device=None): + _logger.info("AutoCompressPruner export directly the pruned model without mask") + + torch.save(self._model_to_prune.state_dict(), model_path) + _logger.info('Model state_dict saved to %s', model_path) + + if onnx_path is not None: + assert input_shape is not None, 'input_shape must be specified to export onnx model' + # input info needed + if device is None: + device = torch.device('cpu') + input_data = torch.Tensor(*input_shape) + torch.onnx.export(self._model_to_prune, input_data.to(device), onnx_path) + _logger.info('Model in onnx with input shape %s saved to %s', input_data.shape, onnx_path) diff --git a/new_impl/cv/third_party/nni/algorithms/compression/pytorch/pruning/constants.py b/new_impl/cv/third_party/nni/algorithms/compression/pytorch/pruning/constants.py new file mode 100644 index 0000000000000000000000000000000000000000..1670384c470ca30fcb2fc5f88bd3658d14010e93 --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/compression/pytorch/pruning/constants.py @@ -0,0 +1,25 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + + +from . import LevelPrunerMasker, SlimPrunerMasker, L1FilterPrunerMasker, \ + L2FilterPrunerMasker, FPGMPrunerMasker, TaylorFOWeightFilterPrunerMasker, \ + ActivationAPoZRankFilterPrunerMasker, ActivationMeanRankFilterPrunerMasker, \ + TRRPrunerMasker, HRankPrunerMasker, PFPMasker + +MASKER_DICT = { + 'level': LevelPrunerMasker, + 'slim': SlimPrunerMasker, + 'l1': L1FilterPrunerMasker, + 'l2': L2FilterPrunerMasker, + 'fpgm': FPGMPrunerMasker, + 'taylorfo': TaylorFOWeightFilterPrunerMasker, + 'apoz': ActivationAPoZRankFilterPrunerMasker, + 'mean_activation': ActivationMeanRankFilterPrunerMasker, + + # implemented by queyu, 2020/11/23 + 'trr': TRRPrunerMasker, + # implemented by queyu, 2021/6/10 + 'hrank': HRankPrunerMasker, + 'pfp': PFPMasker +} diff --git a/new_impl/cv/third_party/nni/algorithms/compression/pytorch/pruning/constants_pruner.py b/new_impl/cv/third_party/nni/algorithms/compression/pytorch/pruning/constants_pruner.py new file mode 100644 index 0000000000000000000000000000000000000000..9448d32d8c1080336339eccc6abe207f105dc8d2 --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/compression/pytorch/pruning/constants_pruner.py @@ -0,0 +1,12 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + + +from .one_shot import LevelPruner, L1FilterPruner, L2FilterPruner, TRRFilterPruner + +PRUNER_DICT = { + 'level': LevelPruner, + 'l1': L1FilterPruner, + 'l2': L2FilterPruner, + 'trr': TRRFilterPruner +} diff --git a/new_impl/cv/third_party/nni/algorithms/compression/pytorch/pruning/finegrained_pruning.py b/new_impl/cv/third_party/nni/algorithms/compression/pytorch/pruning/finegrained_pruning.py new file mode 100644 index 0000000000000000000000000000000000000000..f4aa174233e977eed7b59c6ba65136526a738603 --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/compression/pytorch/pruning/finegrained_pruning.py @@ -0,0 +1,31 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +import torch +from .weight_masker import WeightMasker + +__all__ = ['LevelPrunerMasker'] + +logger = logging.getLogger('torch pruner') + + +class LevelPrunerMasker(WeightMasker): + """ + Prune to an exact pruning level specification + """ + + def calc_mask(self, sparsity, wrapper, wrapper_idx=None): + weight = wrapper.module.weight.data.clone() + if wrapper.weight_mask is not None: + # apply base mask for iterative pruning + weight = weight * wrapper.weight_mask + + w_abs = weight.abs() + k = int(weight.numel() * sparsity) + if k == 0: + return {'weight_mask': torch.ones(weight.shape).type_as(weight)} + threshold = torch.topk(w_abs.view(-1), k, largest=False)[0].max() + mask_weight = torch.gt(w_abs, threshold).type_as(weight) + mask = {'weight_mask': mask_weight} + return mask diff --git a/new_impl/cv/third_party/nni/algorithms/compression/pytorch/pruning/lottery_ticket.py b/new_impl/cv/third_party/nni/algorithms/compression/pytorch/pruning/lottery_ticket.py new file mode 100644 index 0000000000000000000000000000000000000000..b45343457a63e132ea56b82eedb21865da7b5e23 --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/compression/pytorch/pruning/lottery_ticket.py @@ -0,0 +1,146 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import copy +import logging +import torch +from schema import And, Optional +from .....compression.pytorch.utils.config_validation import CompressorSchema +from .....compression.pytorch.compressor import Pruner +from .finegrained_pruning import LevelPrunerMasker + +logger = logging.getLogger('torch pruner') + +class LotteryTicketPruner(Pruner): + """ + Parameters + ---------- + model : pytorch model + The model to be pruned + config_list : list + Supported keys: + - prune_iterations : The number of rounds for the iterative pruning. + - sparsity : The final sparsity when the compression is done. + optimizer : pytorch optimizer + The optimizer for the model + lr_scheduler : pytorch lr scheduler + The lr scheduler for the model if used + reset_weights : bool + Whether reset weights and optimizer at the beginning of each round. + """ + def __init__(self, model, config_list, optimizer=None, lr_scheduler=None, reset_weights=True): + # save init weights and optimizer + self.reset_weights = reset_weights + if self.reset_weights: + self._model = model + self._optimizer = optimizer + self._model_state = copy.deepcopy(model.state_dict()) + self._optimizer_state = copy.deepcopy(optimizer.state_dict()) + self._lr_scheduler = lr_scheduler + if lr_scheduler is not None: + self._scheduler_state = copy.deepcopy(lr_scheduler.state_dict()) + + super().__init__(model, config_list, optimizer) + self.curr_prune_iteration = None + self.prune_iterations = config_list[0]['prune_iterations'] + self.masker = LevelPrunerMasker(model, self) + + def validate_config(self, model, config_list): + """ + Parameters + ---------- + model : torch.nn.Module + Model to be pruned + config_list : list + Supported keys: + - prune_iterations : The number of rounds for the iterative pruning. + - sparsity : The final sparsity when the compression is done. + """ + schema = CompressorSchema([{ + 'sparsity': And(float, lambda n: 0 < n < 1), + 'prune_iterations': And(int, lambda n: n > 0), + Optional('op_types'): [str], + Optional('op_names'): [str] + }], model, logger) + + schema.validate(config_list) + assert len(set([x['prune_iterations'] for x in config_list])) == 1, 'The values of prune_iterations must be equal in your config' + + def _calc_sparsity(self, sparsity): + keep_ratio_once = (1 - sparsity) ** (1 / self.prune_iterations) + curr_keep_ratio = keep_ratio_once ** self.curr_prune_iteration + return max(1 - curr_keep_ratio, 0) + + def _calc_mask(self, wrapper, sparsity): + weight = wrapper.module.weight.data + if self.curr_prune_iteration == 0: + mask = {'weight_mask': torch.ones(weight.shape).type_as(weight)} + else: + curr_sparsity = self._calc_sparsity(sparsity) + mask = self.masker.calc_mask(sparsity=curr_sparsity, wrapper=wrapper) + return mask + + def calc_mask(self, wrapper, **kwargs): + """ + Generate mask for the given ``weight``. + + Parameters + ---------- + wrapper : Module + The layer to be pruned + + Returns + ------- + tensor + The mask for this weight, it is ```None``` because this pruner + calculates and assigns masks in ```prune_iteration_start```, + no need to do anything in this function. + """ + return None + + def get_prune_iterations(self): + """ + Return the range for iterations. + In the first prune iteration, masks are all one, thus, add one more iteration + + Returns + ------- + list + A list for pruning iterations + """ + return range(self.prune_iterations + 1) + + def prune_iteration_start(self): + """ + Control the pruning procedure on updated epoch number. + Should be called at the beginning of the epoch. + """ + if self.curr_prune_iteration is None: + self.curr_prune_iteration = 0 + else: + self.curr_prune_iteration += 1 + assert self.curr_prune_iteration < self.prune_iterations + 1, 'Exceed the configured prune_iterations' + + modules_wrapper = self.get_modules_wrapper() + modules_to_compress = self.get_modules_to_compress() + for layer, config in modules_to_compress: + module_wrapper = None + for wrapper in modules_wrapper: + if wrapper.name == layer.name: + module_wrapper = wrapper + break + assert module_wrapper is not None + + sparsity = config.get('sparsity') + mask = self._calc_mask(module_wrapper, sparsity) + # TODO: directly use weight_mask is not good + module_wrapper.weight_mask = mask['weight_mask'] + # there is no mask for bias + + # reinit weights back to original after new masks are generated + if self.reset_weights: + # should use this member function to reset model weights + self.load_model_state_dict(self._model_state) + self._optimizer.load_state_dict(self._optimizer_state) + if self._lr_scheduler is not None: + self._lr_scheduler.load_state_dict(self._scheduler_state) diff --git a/new_impl/cv/third_party/nni/algorithms/compression/pytorch/pruning/net_adapt_pruner.py b/new_impl/cv/third_party/nni/algorithms/compression/pytorch/pruning/net_adapt_pruner.py new file mode 100644 index 0000000000000000000000000000000000000000..6f55234d5b91059c670e40ecf3cd991f2627c12d --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/compression/pytorch/pruning/net_adapt_pruner.py @@ -0,0 +1,351 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +import os +import copy +import json +import torch +from schema import And, Optional + +from nni.utils import OptimizeMode + +from nni.compression.pytorch.compressor import Pruner +from nni.compression.pytorch.utils.config_validation import CompressorSchema +from nni.compression.pytorch.utils.num_param_counter import get_total_num_weights +from .constants_pruner import PRUNER_DICT + + +_logger = logging.getLogger(__name__) + + +class NetAdaptPruner(Pruner): + """ + A Pytorch implementation of NetAdapt compression algorithm. + + Parameters + ---------- + model : pytorch model + The model to be pruned. + config_list : list + Supported keys: + - sparsity : The target overall sparsity. + - op_types : The operation type to prune. + short_term_fine_tuner : function + function to short-term fine tune the masked model. + This function should include `model` as the only parameter, + and fine tune the model for a short term after each pruning iteration. + Example:: + + def short_term_fine_tuner(model, epoch=3): + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + train_loader = ... + criterion = torch.nn.CrossEntropyLoss() + optimizer = torch.optim.SGD(model.parameters(), lr=0.01) + model.train() + for _ in range(epoch): + for batch_idx, (data, target) in enumerate(train_loader): + data, target = data.to(device), target.to(device) + optimizer.zero_grad() + output = model(data) + loss = criterion(output, target) + loss.backward() + optimizer.step() + evaluator : function + function to evaluate the masked model. + This function should include `model` as the only parameter, and returns a scalar value. + Example:: + + def evaluator(model): + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + val_loader = ... + model.eval() + correct = 0 + with torch.no_grad(): + for data, target in val_loader: + data, target = data.to(device), target.to(device) + output = model(data) + # get the index of the max log-probability + pred = output.argmax(dim=1, keepdim=True) + correct += pred.eq(target.view_as(pred)).sum().item() + accuracy = correct / len(val_loader.dataset) + return accuracy + optimize_mode : str + optimize mode, `maximize` or `minimize`, by default `maximize`. + base_algo : str + Base pruning algorithm. `level`, `l1` or `l2`, by default `l1`. Given the sparsity distribution among the ops, + the assigned `base_algo` is used to decide which filters/channels/weights to prune. + sparsity_per_iteration : float + sparsity to prune in each iteration. + experiment_data_dir : str + PATH to save experiment data, + including the config_list generated for the base pruning algorithm and the performance of the pruned model. + """ + + def __init__(self, model, config_list, short_term_fine_tuner, evaluator, + optimize_mode='maximize', base_algo='l1', sparsity_per_iteration=0.05, experiment_data_dir='./'): + # models used for iterative pruning and evaluation + self._model_to_prune = copy.deepcopy(model) + self._base_algo = base_algo + + super().__init__(model, config_list) + + self._short_term_fine_tuner = short_term_fine_tuner + self._evaluator = evaluator + self._optimize_mode = OptimizeMode(optimize_mode) + + # hyper parameters for NetAdapt algorithm + self._sparsity_per_iteration = sparsity_per_iteration + + # overall pruning rate + self._sparsity = config_list[0]['sparsity'] + + # config_list + self._config_list_generated = [] + + self._experiment_data_dir = experiment_data_dir + if not os.path.exists(self._experiment_data_dir): + os.makedirs(self._experiment_data_dir) + + self._tmp_model_path = os.path.join(self._experiment_data_dir, 'tmp_model.pth') + + def validate_config(self, model, config_list): + """ + Parameters + ---------- + model : torch.nn.Module + Model to be pruned + config_list : list + List on pruning configs + """ + + if self._base_algo == 'level': + schema = CompressorSchema([{ + 'sparsity': And(float, lambda n: 0 < n < 1), + Optional('op_types'): [str], + Optional('op_names'): [str], + }], model, _logger) + elif self._base_algo in ['l1', 'l2']: + schema = CompressorSchema([{ + 'sparsity': And(float, lambda n: 0 < n < 1), + 'op_types': ['Conv2d'], + Optional('op_names'): [str] + }], model, _logger) + + schema.validate(config_list) + + def calc_mask(self, wrapper, **kwargs): + return None + + def _update_config_list(self, config_list, op_name, sparsity): + ''' + update sparsity of op_name in config_list + ''' + config_list_updated = copy.deepcopy(config_list) + + for idx, item in enumerate(config_list): + if op_name in item['op_names']: + config_list_updated[idx]['sparsity'] = sparsity + return config_list_updated + + # if op_name is not in self._config_list_generated, create a new json item + if self._base_algo in ['l1', 'l2']: + config_list_updated.append( + {'sparsity': sparsity, 'op_types': ['Conv2d'], 'op_names': [op_name]}) + elif self._base_algo == 'level': + config_list_updated.append( + {'sparsity': sparsity, 'op_names': [op_name]}) + + return config_list_updated + + def _get_op_num_weights_remained(self, op_name, module): + ''' + Get the number of weights remained after channel pruning with current sparsity + + Returns + ------- + int + remained number of weights of the op + ''' + + # if op is wrapped by the pruner + for wrapper in self.get_modules_wrapper(): + if wrapper.name == op_name: + return wrapper.weight_mask.sum().item() + + # if op is not wrapped by the pruner + return module.weight.data.numel() + + def _get_op_sparsity(self, op_name): + for config in self._config_list_generated: + if 'op_names' in config and op_name in config['op_names']: + return config['sparsity'] + return 0 + + def _calc_num_related_weights(self, op_name): + ''' + Calculate total number weights of the op and the next op, applicable only for models without dependencies among ops + + Parameters + ---------- + op_name : str + + Returns + ------- + int + total number of all the realted (current and the next) op weights + ''' + num_weights = 0 + flag_found = False + previous_name = None + previous_module = None + + for name, module in self._model_to_prune.named_modules(): + if not flag_found and name != op_name and type(module).__name__ in ['Conv2d', 'Linear']: + previous_name = name + previous_module = module + if not flag_found and name == op_name: + _logger.debug("original module found: %s", name) + num_weights = module.weight.data.numel() + + # consider related pruning in this op caused by previous op's pruning + if previous_module: + sparsity_previous_op = self._get_op_sparsity(previous_name) + if sparsity_previous_op: + _logger.debug( + "decrease op's weights by %s due to previous op %s's pruning...", sparsity_previous_op, previous_name) + num_weights *= (1-sparsity_previous_op) + + flag_found = True + continue + if flag_found and type(module).__name__ in ['Conv2d', 'Linear']: + _logger.debug("related module found: %s", name) + # channel/filter pruning crossing is considered here, so only the num_weights after channel pruning is valuable + num_weights += self._get_op_num_weights_remained(name, module) + break + + _logger.debug("num related weights of op %s : %d", op_name, num_weights) + + return num_weights + + def compress(self): + """ + Compress the model. + + Returns + ------- + torch.nn.Module + model with specified modules compressed. + """ + _logger.info('Starting NetAdapt Compression...') + + pruning_iteration = 0 + current_sparsity = 0 + delta_num_weights_per_iteration = \ + int(get_total_num_weights(self._model_to_prune, ['Conv2d', 'Linear']) * self._sparsity_per_iteration) + + # stop condition + while current_sparsity < self._sparsity: + _logger.info('Pruning iteration: %d', pruning_iteration) + + # calculate target sparsity of this iteration + target_sparsity = current_sparsity + self._sparsity_per_iteration + + # variable to store the info of the best layer found in this iteration + best_op = {} + + for wrapper in self.get_modules_wrapper(): + _logger.debug("op name : %s", wrapper.name) + _logger.debug("op weights : %d", wrapper.weight_mask.numel()) + _logger.debug("op left weights : %d", wrapper.weight_mask.sum().item()) + + current_op_sparsity = 1 - wrapper.weight_mask.sum().item() / wrapper.weight_mask.numel() + _logger.debug("current op sparsity : %s", current_op_sparsity) + + # sparsity that this layer needs to prune to satisfy the requirement + target_op_sparsity = current_op_sparsity + delta_num_weights_per_iteration / self._calc_num_related_weights(wrapper.name) + + if target_op_sparsity >= 1: + _logger.info('Layer %s has no enough weights (remained) to prune', wrapper.name) + continue + + config_list = self._update_config_list(self._config_list_generated, wrapper.name, target_op_sparsity) + _logger.debug("config_list used : %s", config_list) + + pruner = PRUNER_DICT[self._base_algo](copy.deepcopy(self._model_to_prune), config_list) + model_masked = pruner.compress() + + # Short-term fine tune the pruned model + self._short_term_fine_tuner(model_masked) + + performance = self._evaluator(model_masked) + _logger.info("Layer : %s, evaluation result after short-term fine tuning : %s", wrapper.name, performance) + + if not best_op \ + or (self._optimize_mode is OptimizeMode.Maximize and performance > best_op['performance']) \ + or (self._optimize_mode is OptimizeMode.Minimize and performance < best_op['performance']): + _logger.debug("updating best layer to %s...", wrapper.name) + # find weight mask of this layer + for w in pruner.get_modules_wrapper(): + if w.name == wrapper.name: + masks = {'weight_mask': w.weight_mask, + 'bias_mask': w.bias_mask} + break + best_op = { + 'op_name': wrapper.name, + 'sparsity': target_op_sparsity, + 'performance': performance, + 'masks': masks + } + + # save model weights + pruner.export_model(self._tmp_model_path) + + if not best_op: + # decrease pruning step + self._sparsity_per_iteration *= 0.5 + _logger.info("No more layers to prune, decrease pruning step to %s", self._sparsity_per_iteration) + continue + + # Pick the best layer to prune, update iterative information + # update config_list + self._config_list_generated = self._update_config_list( + self._config_list_generated, best_op['op_name'], best_op['sparsity']) + + # update weights parameters + self._model_to_prune.load_state_dict(torch.load(self._tmp_model_path)) + + # update mask of the chosen op + for wrapper in self.get_modules_wrapper(): + if wrapper.name == best_op['op_name']: + for k in best_op['masks']: + setattr(wrapper, k, best_op['masks'][k]) + break + + current_sparsity = target_sparsity + _logger.info('Pruning iteration %d finished, current sparsity: %s', pruning_iteration, current_sparsity) + _logger.info('Layer %s seleted with sparsity %s, performance after pruning & short term fine-tuning : %s', + best_op['op_name'], best_op['sparsity'], best_op['performance']) + pruning_iteration += 1 + + self._final_performance = best_op['performance'] + + # load weights parameters + self.load_model_state_dict(torch.load(self._tmp_model_path)) + os.remove(self._tmp_model_path) + + _logger.info('----------Compression finished--------------') + _logger.info('config_list generated: %s', self._config_list_generated) + _logger.info("Performance after pruning: %s", self._final_performance) + _logger.info("Masked sparsity: %.6f", current_sparsity) + + # save best config found and best performance + with open(os.path.join(self._experiment_data_dir, 'search_result.json'), 'w') as jsonfile: + json.dump({ + 'performance': self._final_performance, + 'config_list': json.dumps(self._config_list_generated) + }, jsonfile) + + _logger.info('search history and result saved to foler : %s', self._experiment_data_dir) + + return self.bound_model diff --git a/new_impl/cv/third_party/nni/algorithms/compression/pytorch/pruning/one_shot.py b/new_impl/cv/third_party/nni/algorithms/compression/pytorch/pruning/one_shot.py new file mode 100644 index 0000000000000000000000000000000000000000..f2470d81ba2ca2245bc9ab11aa996d3030af130e --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/compression/pytorch/pruning/one_shot.py @@ -0,0 +1,489 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +from schema import And, Or, Optional, SchemaError +from .....common.graph_utils import TorchModuleGraph +from .....compression.pytorch.utils.shape_dependency import ChannelDependency, GroupDependency +from .constants import MASKER_DICT +from .....compression.pytorch.utils.config_validation import CompressorSchema +from .....compression.pytorch.compressor import Pruner + + +__all__ = ['LevelPruner', 'SlimPruner', 'L1FilterPruner', 'L2FilterPruner', 'FPGMPruner', + 'TaylorFOWeightFilterPruner', 'ActivationAPoZRankFilterPruner', 'ActivationMeanRankFilterPruner', + 'TRRFilterPruner', 'HRankFilterPruner', 'PFPPruner'] + +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) + + +class OneshotPruner(Pruner): + """ + Prune model to an exact pruning level for one time. + """ + + def __init__(self, model, config_list, pruning_algorithm='level', optimizer=None, **algo_kwargs): + """ + Parameters + ---------- + model : torch.nn.Module + Model to be pruned + config_list : list + List on pruning configs + pruning_algorithm: str + algorithms being used to prune model + optimizer: torch.optim.Optimizer + Optimizer used to train model + algo_kwargs: dict + Additional parameters passed to pruning algorithm masker class + """ + + super().__init__(model, config_list, optimizer) + self.set_wrappers_attribute("if_calculated", False) + self.masker = MASKER_DICT[pruning_algorithm]( + model, self, **algo_kwargs) + + def validate_config(self, model, config_list): + """ + Parameters + ---------- + model : torch.nn.Module + Model to be pruned + config_list : list + List on pruning configs + """ + schema = CompressorSchema([{ + 'sparsity': And(float, lambda n: 0 < n < 1), + Optional('op_types'): [str], + Optional('op_names'): [str] + }], model, logger) + + schema.validate(config_list) + + def calc_mask(self, wrapper, wrapper_idx=None): + """ + Calculate the mask of given layer + Parameters + ---------- + wrapper : Module + the module to instrument the compression operation + wrapper_idx: int + index of this wrapper in pruner's all wrappers + Returns + ------- + dict + dictionary for storing masks, keys of the dict: + 'weight_mask': weight mask tensor + 'bias_mask': bias mask tensor (optional) + """ + if wrapper.if_calculated: + return None + + sparsity = wrapper.config['sparsity'] + if not wrapper.if_calculated: + masks = self.masker.calc_mask( + sparsity=sparsity, wrapper=wrapper, wrapper_idx=wrapper_idx) + + # masker.calc_mask returns None means calc_mask is not calculated sucessfully, can try later + if masks is not None: + wrapper.if_calculated = True + return masks + else: + return None + + +class LevelPruner(OneshotPruner): + """ + Parameters + ---------- + model : torch.nn.Module + Model to be pruned + config_list : list + Supported keys: + - sparsity : This is to specify the sparsity operations to be compressed to. + - op_types : Operation types to prune. + optimizer: torch.optim.Optimizer + Optimizer used to train model + """ + + def __init__(self, model, config_list, optimizer=None): + super().__init__(model, config_list, pruning_algorithm='level', optimizer=optimizer) + + +class SlimPruner(OneshotPruner): + """ + Parameters + ---------- + model : torch.nn.Module + Model to be pruned + config_list : list + Supported keys: + - sparsity : This is to specify the sparsity operations to be compressed to. + - op_types : Only BatchNorm2d is supported in Slim Pruner. + optimizer: torch.optim.Optimizer + Optimizer used to train model + """ + + def __init__(self, model, config_list, optimizer=None): + super().__init__(model, config_list, pruning_algorithm='slim', optimizer=optimizer) + + def validate_config(self, model, config_list): + schema = CompressorSchema([{ + 'sparsity': And(float, lambda n: 0 < n < 1), + 'op_types': ['BatchNorm2d'], + Optional('op_names'): [str] + }], model, logger) + + schema.validate(config_list) + + if len(config_list) > 1: + logger.warning('Slim pruner only supports 1 configuration') + + +class _StructuredFilterPruner(OneshotPruner): + """ + _StructuredFilterPruner has two ways to calculate the masks + for conv layers. In the normal way, the _StructuredFilterPruner + will calculate the mask of each layer separately. For example, each + conv layer determine which filters should be pruned according to its L1 + norm. In constrast, in the dependency-aware way, the layers that in a + dependency group will be pruned jointly and these layers will be forced + to prune the same channels. + """ + + def __init__(self, model, config_list, pruning_algorithm, optimizer=None, dependency_aware=False, dummy_input=None, **algo_kwargs): + super().__init__(model, config_list, pruning_algorithm=pruning_algorithm, + optimizer=optimizer, **algo_kwargs) + self.dependency_aware = dependency_aware + # set the dependency-aware switch for the masker + self.masker.dependency_aware = dependency_aware + self.dummy_input = dummy_input + if self.dependency_aware: + errmsg = "When dependency_aware is set, the dummy_input should not be None" + assert self.dummy_input is not None, errmsg + # Get the TorchModuleGraph of the target model + # to trace the model, we need to unwrap the wrappers + self._unwrap_model() + self.graph = TorchModuleGraph(model, dummy_input) + self._wrap_model() + self.channel_depen = ChannelDependency( + traced_model=self.graph.trace) + self.group_depen = GroupDependency(traced_model=self.graph.trace) + self.channel_depen = self.channel_depen.dependency_sets + self.channel_depen = { + name: sets for sets in self.channel_depen for name in sets} + self.group_depen = self.group_depen.dependency_sets + + def update_mask(self): + if not self.dependency_aware: + # if we use the normal way to update the mask, + # then call the update_mask of the father class + super(_StructuredFilterPruner, self).update_mask() + else: + # if we update the mask in a dependency-aware way + # then we call _dependency_update_mask + self._dependency_update_mask() + + def validate_config(self, model, config_list): + schema = CompressorSchema([{ + Optional('sparsity'): And(float, lambda n: 0 < n < 1), + Optional('op_types'): Or(['Conv2d'], ['Conv2d', 'ConvTranspose2d']), + Optional('op_names'): [str], + Optional('exclude'): bool + }], model, logger) + + schema.validate(config_list) + for config in config_list: + if 'exclude' not in config and 'sparsity' not in config: + raise SchemaError('Either sparisty or exclude must be specified!') + + def _dependency_calc_mask(self, wrappers, channel_dsets, wrappers_idx=None): + """ + calculate the masks for the conv layers in the same + channel dependecy set. All the layers passed in have + the same number of channels. + + Parameters + ---------- + wrappers: list + The list of the wrappers that in the same channel dependency + set. + wrappers_idx: list + The list of the indexes of wrapppers. + Returns + ------- + masks: dict + A dict object that contains the masks of the layers in this + dependency group, the key is the name of the convolutional layers. + """ + # The number of the groups for each conv layers + # Note that, this number may be different from its + # original number of groups of filters. + groups = [self.group_depen[_w.name] for _w in wrappers] + sparsities = [_w.config['sparsity'] for _w in wrappers] + masks = self.masker.calc_mask( + sparsities, wrappers, wrappers_idx, channel_dsets=channel_dsets, groups=groups) + if masks is not None: + # if masks is None, then the mask calculation fails. + # for example, in activation related maskers, we should + # pass enough batches of data to the model, so that the + # masks can be calculated successfully. + for _w in wrappers: + _w.if_calculated = True + return masks + + def _dependency_update_mask(self): + """ + In the original update_mask, the wraper of each layer will update its + own mask according to the sparsity specified in the config_list. However, in + the _dependency_update_mask, we may prune several layers at the same + time according the sparsities and the channel/group dependencies. + """ + name2wrapper = {x.name: x for x in self.get_modules_wrapper()} + wrapper2index = {x: i for i, x in enumerate(self.get_modules_wrapper())} + for wrapper in self.get_modules_wrapper(): + if wrapper.if_calculated: + continue + # find all the conv layers that have channel dependecy with this layer + # and prune all these layers at the same time. + _names = [x for x in self.channel_depen[wrapper.name]] + logger.info('Pruning the dependent layers: %s', ','.join(_names)) + _wrappers = [name2wrapper[name] + for name in _names if name in name2wrapper] + _wrapper_idxes = [wrapper2index[_w] for _w in _wrappers] + + masks = self._dependency_calc_mask( + _wrappers, _names, wrappers_idx=_wrapper_idxes) + if masks is not None: + for layer in masks: + for mask_type in masks[layer]: + assert hasattr( + name2wrapper[layer], mask_type), "there is no attribute '%s' in wrapper on %s" % (mask_type, layer) + setattr(name2wrapper[layer], mask_type, masks[layer][mask_type]) + + +class L1FilterPruner(_StructuredFilterPruner): + """ + Parameters + ---------- + model : torch.nn.Module + Model to be pruned + config_list : list + Supported keys: + - sparsity : This is to specify the sparsity operations to be compressed to. + - op_types : Only Conv2d is supported in L1FilterPruner. + optimizer: torch.optim.Optimizer + Optimizer used to train model + dependency_aware: bool + If prune the model in a dependency-aware way. If it is `True`, this pruner will + prune the model according to the l2-norm of weights and the channel-dependency or + group-dependency of the model. In this way, the pruner will force the conv layers + that have dependencies to prune the same channels, so the speedup module can better + harvest the speed benefit from the pruned model. Note that, if this flag is set True + , the dummy_input cannot be None, because the pruner needs a dummy input to trace the + dependency between the conv layers. + dummy_input : torch.Tensor + The dummy input to analyze the topology constraints. Note that, the dummy_input + should on the same device with the model. + """ + + def __init__(self, model, config_list, optimizer=None, dependency_aware=False, dummy_input=None): + super().__init__(model, config_list, pruning_algorithm='l1', optimizer=optimizer, + dependency_aware=dependency_aware, dummy_input=dummy_input) + + +class TRRFilterPruner(_StructuredFilterPruner): + def __init__(self, model, config_list, optimizer=None, dependency_aware=False, train_loader=None, device='cuda'): + super().__init__(model, config_list, pruning_algorithm='trr', optimizer=optimizer, + dependency_aware=dependency_aware, train_loader=train_loader, device=device) + + +class L2FilterPruner(_StructuredFilterPruner): + """ + Parameters + ---------- + model : torch.nn.Module + Model to be pruned + config_list : list + Supported keys: + - sparsity : This is to specify the sparsity operations to be compressed to. + - op_types : Only Conv2d is supported in L2FilterPruner. + optimizer: torch.optim.Optimizer + Optimizer used to train model + dependency_aware: bool + If prune the model in a dependency-aware way. If it is `True`, this pruner will + prune the model according to the l2-norm of weights and the channel-dependency or + group-dependency of the model. In this way, the pruner will force the conv layers + that have dependencies to prune the same channels, so the speedup module can better + harvest the speed benefit from the pruned model. Note that, if this flag is set True + , the dummy_input cannot be None, because the pruner needs a dummy input to trace the + dependency between the conv layers. + dummy_input : torch.Tensor + The dummy input to analyze the topology constraints. Note that, the dummy_input + should on the same device with the model. + """ + + def __init__(self, model, config_list, optimizer=None, dependency_aware=False, dummy_input=None): + super().__init__(model, config_list, pruning_algorithm='l2', optimizer=optimizer, + dependency_aware=dependency_aware, dummy_input=dummy_input) + + +class FPGMPruner(_StructuredFilterPruner): + """ + Parameters + ---------- + model : torch.nn.Module + Model to be pruned + config_list : list + Supported keys: + - sparsity : This is to specify the sparsity operations to be compressed to. + - op_types : Only Conv2d is supported in FPGM Pruner. + optimizer: torch.optim.Optimizer + Optimizer used to train model + dependency_aware: bool + If prune the model in a dependency-aware way. If it is `True`, this pruner will + prune the model according to the l2-norm of weights and the channel-dependency or + group-dependency of the model. In this way, the pruner will force the conv layers + that have dependencies to prune the same channels, so the speedup module can better + harvest the speed benefit from the pruned model. Note that, if this flag is set True + , the dummy_input cannot be None, because the pruner needs a dummy input to trace the + dependency between the conv layers. + dummy_input : torch.Tensor + The dummy input to analyze the topology constraints. Note that, the dummy_input + should on the same device with the model. + """ + + def __init__(self, model, config_list, optimizer=None, dependency_aware=False, dummy_input=None): + super().__init__(model, config_list, pruning_algorithm='fpgm', + dependency_aware=dependency_aware, dummy_input=dummy_input, optimizer=optimizer) + + +class TaylorFOWeightFilterPruner(_StructuredFilterPruner): + """ + Parameters + ---------- + model : torch.nn.Module + Model to be pruned + config_list : list + Supported keys: + - sparsity : How much percentage of convolutional filters are to be pruned. + - op_types : Currently only Conv2d is supported in TaylorFOWeightFilterPruner. + optimizer: torch.optim.Optimizer + Optimizer used to train model + statistics_batch_num: int + The number of batches to statistic the activation. + dependency_aware: bool + If prune the model in a dependency-aware way. If it is `True`, this pruner will + prune the model according to the l2-norm of weights and the channel-dependency or + group-dependency of the model. In this way, the pruner will force the conv layers + that have dependencies to prune the same channels, so the speedup module can better + harvest the speed benefit from the pruned model. Note that, if this flag is set True + , the dummy_input cannot be None, because the pruner needs a dummy input to trace the + dependency between the conv layers. + dummy_input : torch.Tensor + The dummy input to analyze the topology constraints. Note that, the dummy_input + should on the same device with the model. + + """ + + def __init__(self, model, config_list, optimizer=None, statistics_batch_num=1, + dependency_aware=False, dummy_input=None): + super().__init__(model, config_list, pruning_algorithm='taylorfo', + dependency_aware=dependency_aware, dummy_input=dummy_input, + optimizer=optimizer, statistics_batch_num=statistics_batch_num) + + +class ActivationAPoZRankFilterPruner(_StructuredFilterPruner): + """ + Parameters + ---------- + model : torch.nn.Module + Model to be pruned + config_list : list + Supported keys: + - sparsity : How much percentage of convolutional filters are to be pruned. + - op_types : Only Conv2d is supported in ActivationAPoZRankFilterPruner. + optimizer: torch.optim.Optimizer + Optimizer used to train model + activation: str + The activation type. + statistics_batch_num: int + The number of batches to statistic the activation. + dependency_aware: bool + If prune the model in a dependency-aware way. If it is `True`, this pruner will + prune the model according to the l2-norm of weights and the channel-dependency or + group-dependency of the model. In this way, the pruner will force the conv layers + that have dependencies to prune the same channels, so the speedup module can better + harvest the speed benefit from the pruned model. Note that, if this flag is set True + , the dummy_input cannot be None, because the pruner needs a dummy input to trace the + dependency between the conv layers. + dummy_input : torch.Tensor + The dummy input to analyze the topology constraints. Note that, the dummy_input + should on the same device with the model. + + """ + + def __init__(self, model, config_list, optimizer=None, activation='relu', + statistics_batch_num=1, dependency_aware=False, dummy_input=None): + super().__init__(model, config_list, pruning_algorithm='apoz', optimizer=optimizer, + dependency_aware=dependency_aware, dummy_input=dummy_input, + activation=activation, statistics_batch_num=statistics_batch_num) + + +class ActivationMeanRankFilterPruner(_StructuredFilterPruner): + """ + Parameters + ---------- + model : torch.nn.Module + Model to be pruned + config_list : list + Supported keys: + - sparsity : How much percentage of convolutional filters are to be pruned. + - op_types : Only Conv2d is supported in ActivationMeanRankFilterPruner. + optimizer: torch.optim.Optimizer + Optimizer used to train model. + activation: str + The activation type. + statistics_batch_num: int + The number of batches to statistic the activation. + dependency_aware: bool + If prune the model in a dependency-aware way. If it is `True`, this pruner will + prune the model according to the l2-norm of weights and the channel-dependency or + group-dependency of the model. In this way, the pruner will force the conv layers + that have dependencies to prune the same channels, so the speedup module can better + harvest the speed benefit from the pruned model. Note that, if this flag is set True + , the dummy_input cannot be None, because the pruner needs a dummy input to trace the + dependency between the conv layers. + dummy_input : torch.Tensor + The dummy input to analyze the topology constraints. Note that, the dummy_input + should on the same device with the model. + """ + + def __init__(self, model, config_list, optimizer=None, activation='relu', + statistics_batch_num=1, dependency_aware=False, dummy_input=None): + super().__init__(model, config_list, pruning_algorithm='mean_activation', optimizer=optimizer, + dependency_aware=dependency_aware, dummy_input=dummy_input, + activation=activation, statistics_batch_num=statistics_batch_num) + + +# implemented by queyu, 2021/6/10 +class HRankFilterPruner(_StructuredFilterPruner): + def __init__(self, model, config_list, optimizer=None, activation='relu', + statistics_batch_num=1, dependency_aware=False, dummy_input=None): + super().__init__(model, config_list, pruning_algorithm='hrank', optimizer=optimizer, + dependency_aware=dependency_aware, dummy_input=dummy_input, + activation=activation, statistics_batch_num=statistics_batch_num) + + +# implemented by queyu, 2021/6/10 +class PFPPruner(_StructuredFilterPruner): + def __init__(self, model, config_list, optimizer=None, activation='relu', + statistics_batch_num=1, dependency_aware=False, dummy_input=None): + super().__init__(model, config_list, pruning_algorithm='pfp', optimizer=optimizer, + dependency_aware=dependency_aware, dummy_input=dummy_input, + activation=activation, statistics_batch_num=statistics_batch_num) + + self._unwrap_model() + self.graph = TorchModuleGraph(model, dummy_input) + self._wrap_model() diff --git a/new_impl/cv/third_party/nni/algorithms/compression/pytorch/pruning/sensitivity_pruner.py b/new_impl/cv/third_party/nni/algorithms/compression/pytorch/pruning/sensitivity_pruner.py new file mode 100644 index 0000000000000000000000000000000000000000..037e7efc5e9edcd49757ce053bd3b0c1333ae19f --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/compression/pytorch/pruning/sensitivity_pruner.py @@ -0,0 +1,413 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. +import os +import csv +import copy +import json +import logging +import torch + +from schema import And, Optional +from nni.compression.pytorch.compressor import Pruner +from nni.compression.pytorch.utils.config_validation import CompressorSchema +from .constants_pruner import PRUNER_DICT +from nni.compression.pytorch.utils.sensitivity_analysis import SensitivityAnalysis + + +MAX_PRUNE_RATIO_PER_ITER = 0.95 + +_logger = logging.getLogger('Sensitivity_Pruner') +_logger.setLevel(logging.INFO) + +class SensitivityPruner(Pruner): + """ + This function prune the model based on the sensitivity + for each layer. + + Parameters + ---------- + model: torch.nn.Module + model to be compressed + evaluator: function + validation function for the model. This function should return the accuracy + of the validation dataset. The input parameters of evaluator can be specified + in the parameter `eval_args` and 'eval_kwargs' of the compress function if needed. + Example: + >>> def evaluator(model): + >>> device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + >>> val_loader = ... + >>> model.eval() + >>> correct = 0 + >>> with torch.no_grad(): + >>> for data, target in val_loader: + >>> data, target = data.to(device), target.to(device) + >>> output = model(data) + >>> # get the index of the max log-probability + >>> pred = output.argmax(dim=1, keepdim=True) + >>> correct += pred.eq(target.view_as(pred)).sum().item() + >>> accuracy = correct / len(val_loader.dataset) + >>> return accuracy + finetuner: function + finetune function for the model. This parameter is not essential, if is not None, + the sensitivity pruner will finetune the model after pruning in each iteration. + The input parameters of finetuner can be specified in the parameter of compress + called `finetune_args` and `finetune_kwargs` if needed. + Example: + >>> def finetuner(model, epoch=3): + >>> device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + >>> train_loader = ... + >>> criterion = torch.nn.CrossEntropyLoss() + >>> optimizer = torch.optim.SGD(model.parameters(), lr=0.01) + >>> model.train() + >>> for _ in range(epoch): + >>> for _, (data, target) in enumerate(train_loader): + >>> data, target = data.to(device), target.to(device) + >>> optimizer.zero_grad() + >>> output = model(data) + >>> loss = criterion(output, target) + >>> loss.backward() + >>> optimizer.step() + base_algo: str + base pruning algorithm. `level`, `l1` or `l2`, by default `l1`. + sparsity_proportion_calc: function + This function generate the sparsity proportion between the conv layers according to the + sensitivity analysis results. We provide a default function to quantify the sparsity + proportion according to the sensitivity analysis results. Users can also customize + this function according to their needs. The input of this function is a dict, + for example : {'conv1' : {0.1: 0.9, 0.2 : 0.8}, 'conv2' : {0.1: 0.9, 0.2 : 0.8}}, + in which, 'conv1' and is the name of the conv layer, and 0.1:0.9 means when the + sparsity of conv1 is 0.1 (10%), the model's val accuracy equals to 0.9. + sparsity_per_iter: float + The sparsity of the model that the pruner try to prune in each iteration. + acc_drop_threshold : float + The hyperparameter used to quantifiy the sensitivity for each layer. + checkpoint_dir: str + The dir path to save the checkpoints during the pruning. + """ + + def __init__(self, model, config_list, evaluator, + finetuner=None, base_algo='l1', sparsity_proportion_calc=None, + sparsity_per_iter=0.1, acc_drop_threshold=0.05, checkpoint_dir=None): + + self.base_algo = base_algo + self.model = model + super(SensitivityPruner, self).__init__(model, config_list) + # unwrap the model + self._unwrap_model() + _logger.debug(str(self.model)) + self.evaluator = evaluator + self.finetuner = finetuner + self.analyzer = SensitivityAnalysis( + self.model, self.evaluator, prune_type=base_algo, \ + early_stop_mode='dropped', early_stop_value=acc_drop_threshold) + # Get the original accuracy of the pretrained model + self.ori_acc = None + # Copy the original weights before pruning + self.ori_state_dict = copy.deepcopy(self.model.state_dict()) + self.sensitivities = {} + # Save the weight count for each layer + self.weight_count = {} + self.weight_sum = 0 + # Map the layer name to the layer module + self.named_module = {} + + self.Pruner = PRUNER_DICT[self.base_algo] + # Count the total weight count of the model + for name, submodule in self.model.named_modules(): + self.named_module[name] = submodule + if name in self.analyzer.target_layer: + # Currently, only count the weights in the conv layers + # else the fully connected layer (which contains + # the most weights) may make the pruner prune the + # model too hard + # if hasattr(submodule, 'weight'): # Count all the weights of the model + self.weight_count[name] = submodule.weight.data.numel() + self.weight_sum += self.weight_count[name] + # function to generate the sparsity proportion between the conv layers + if sparsity_proportion_calc is None: + self.sparsity_proportion_calc = self._max_prune_ratio + else: + self.sparsity_proportion_calc = sparsity_proportion_calc + # The ratio of remained weights is 1.0 at the begining + self.remained_ratio = 1.0 + self.sparsity_per_iter = sparsity_per_iter + self.acc_drop_threshold = acc_drop_threshold + self.checkpoint_dir = checkpoint_dir + + def validate_config(self, model, config_list): + """ + Parameters + ---------- + model : torch.nn.module + Model to be pruned + config_list : list + List on pruning configs + """ + + if self.base_algo == 'level': + schema = CompressorSchema([{ + 'sparsity': And(float, lambda n: 0 < n < 1), + Optional('op_types'): [str], + Optional('op_names'): [str], + }], model, _logger) + elif self.base_algo in ['l1', 'l2']: + schema = CompressorSchema([{ + 'sparsity': And(float, lambda n: 0 < n < 1), + 'op_types': ['Conv2d'], + Optional('op_names'): [str] + }], model, _logger) + + schema.validate(config_list) + + def load_sensitivity(self, filepath): + """ + load the sensitivity results exported by the sensitivity analyzer + """ + assert os.path.exists(filepath) + with open(filepath, 'r') as csvf: + csv_r = csv.reader(csvf) + header = next(csv_r) + sparsities = [float(x) for x in header[1:]] + sensitivities = {} + for row in csv_r: + layername = row[0] + accuracies = [float(x) for x in row[1:]] + sensitivities[layername] = {} + for i, accuracy in enumerate(accuracies): + sensitivities[layername][sparsities[i]] = accuracy + return sensitivities + + def _max_prune_ratio(self, ori_acc, threshold, sensitivities): + """ + Find the maximum prune ratio for a single layer whose accuracy + drop is lower than the threshold. + + Parameters + ---------- + ori_acc: float + Original accuracy + threshold: float + Accuracy drop threshold + sensitivities: dict + The dict object that stores the sensitivity results for each layer. + For example: {'conv1' : {0.1: 0.9, 0.2 : 0.8}} + Returns + ------- + max_ratios: dict + return the maximum prune ratio for each layer. For example: + {'conv1':0.1, 'conv2':0.2} + """ + max_ratio = {} + for layer in sensitivities: + prune_ratios = sorted(sensitivities[layer].keys()) + last_ratio = 0 + for ratio in prune_ratios: + last_ratio = ratio + cur_acc = sensitivities[layer][ratio] + if cur_acc + threshold < ori_acc: + break + max_ratio[layer] = last_ratio + return max_ratio + + def normalize(self, ratios, target_pruned): + """ + Normalize the prune ratio of each layer according to the + total already pruned ratio and the final target total pruning + ratio + + Parameters + ---------- + ratios: + Dict object that save the prune ratio for each layer + target_pruned: + The amount of the weights expected to be pruned in this + iteration + + Returns + ------- + new_ratios: + return the normalized prune ratios for each layer. + + """ + w_sum = 0 + _Max = 0 + for layername, ratio in ratios.items(): + wcount = self.weight_count[layername] + w_sum += ratio * wcount * \ + (1-self.analyzer.already_pruned[layername]) + target_count = self.weight_sum * target_pruned + for layername in ratios: + ratios[layername] = ratios[layername] * target_count / w_sum + _Max = max(_Max, ratios[layername]) + # Cannot Prune too much in a single iteration + # If a layer's prune ratio is larger than the + # MAX_PRUNE_RATIO_PER_ITER we rescal all prune + # ratios under this threshold + if _Max > MAX_PRUNE_RATIO_PER_ITER: + + for layername in ratios: + ratios[layername] = ratios[layername] * \ + MAX_PRUNE_RATIO_PER_ITER / _Max + return ratios + + def create_cfg(self, ratios): + """ + Generate the cfg_list for the pruner according to the prune ratios. + + Parameters + --------- + ratios: + For example: {'conv1' : 0.2} + + Returns + ------- + cfg_list: + For example: [{'sparsity':0.2, 'op_names':['conv1'], 'op_types':['Conv2d']}] + """ + cfg_list = [] + for layername in ratios: + prune_ratio = ratios[layername] + remain = 1 - self.analyzer.already_pruned[layername] + sparsity = remain * prune_ratio + \ + self.analyzer.already_pruned[layername] + if sparsity > 0: + # Pruner does not allow the prune ratio to be zero + cfg = {'sparsity': sparsity, 'op_names': [ + layername], 'op_types': ['Conv2d']} + cfg_list.append(cfg) + return cfg_list + + def current_sparsity(self): + """ + The sparsity of the weight. + """ + pruned_weight = 0 + for layer_name in self.analyzer.already_pruned: + w_count = self.weight_count[layer_name] + prune_ratio = self.analyzer.already_pruned[layer_name] + pruned_weight += w_count * prune_ratio + return pruned_weight / self.weight_sum + + def compress(self, eval_args=None, eval_kwargs=None, + finetune_args=None, finetune_kwargs=None, resume_sensitivity=None): + """ + This function iteratively prune the model according to the results of + the sensitivity analysis. + + Parameters + ---------- + eval_args: list + eval_kwargs: list& dict + Parameters for the val_funtion, the val_function will be called like + evaluator(*eval_args, **eval_kwargs) + finetune_args: list + finetune_kwargs: dict + Parameters for the finetuner function if needed. + resume_sensitivity: + resume the sensitivity results from this file. + """ + # pylint suggest not use the empty list and dict + # as the default input parameter + if not eval_args: + eval_args = [] + if not eval_kwargs: + eval_kwargs = {} + if not finetune_args: + finetune_args = [] + if not finetune_kwargs: + finetune_kwargs = {} + if self.ori_acc is None: + self.ori_acc = self.evaluator(*eval_args, **eval_kwargs) + assert isinstance(self.ori_acc, float) or isinstance(self.ori_acc, int) + if not resume_sensitivity: + self.sensitivities = self.analyzer.analysis( + val_args=eval_args, val_kwargs=eval_kwargs) + else: + self.sensitivities = self.load_sensitivity(resume_sensitivity) + self.analyzer.sensitivities = self.sensitivities + # the final target sparsity of the model + target_ratio = 1 - self.config_list[0]['sparsity'] + cur_ratio = self.remained_ratio + ori_acc = self.ori_acc + iteration_count = 0 + if self.checkpoint_dir is not None: + os.makedirs(self.checkpoint_dir, exist_ok=True) + modules_wrapper_final = None + while cur_ratio > target_ratio: + iteration_count += 1 + # Each round have three steps: + # 1) Get the current sensitivity for each layer(the sensitivity + # of each layer may change during the pruning) + # 2) Prune each layer according the sensitivies + # 3) finetune the model + _logger.info('Current base accuracy %f', ori_acc) + _logger.info('Remained %f weights', cur_ratio) + # determine the sparsity proportion between different + # layers according to the sensitivity result + proportion = self.sparsity_proportion_calc( + ori_acc, self.acc_drop_threshold, self.sensitivities) + + new_pruneratio = self.normalize(proportion, self.sparsity_per_iter) + cfg_list = self.create_cfg(new_pruneratio) + if not cfg_list: + _logger.error('The threshold is too small, please set a larger threshold') + return self.model + _logger.debug('Pruner Config: %s', str(cfg_list)) + cfg_str = ['%s:%.3f'%(cfg['op_names'][0], cfg['sparsity']) for cfg in cfg_list] + _logger.info('Current Sparsities: %s', ','.join(cfg_str)) + + pruner = self.Pruner(self.model, cfg_list) + pruner.compress() + pruned_acc = self.evaluator(*eval_args, **eval_kwargs) + _logger.info('Accuracy after pruning: %f', pruned_acc) + finetune_acc = pruned_acc + if self.finetuner is not None: + # if the finetune function is None, then skip the finetune + self.finetuner(*finetune_args, **finetune_kwargs) + finetune_acc = self.evaluator(*eval_args, **eval_kwargs) + _logger.info('Accuracy after finetune: %f', finetune_acc) + ori_acc = finetune_acc + # unwrap the pruner + pruner._unwrap_model() + # update the already prune ratio of each layer befor the new + # sensitivity analysis + for layer_cfg in cfg_list: + name = layer_cfg['op_names'][0] + sparsity = layer_cfg['sparsity'] + self.analyzer.already_pruned[name] = sparsity + # update the cur_ratio + cur_ratio = 1 - self.current_sparsity() + modules_wrapper_final = pruner.get_modules_wrapper() + del pruner + _logger.info('Currently remained weights: %f', cur_ratio) + + if self.checkpoint_dir is not None: + checkpoint_name = 'Iter_%d_finetune_acc_%.5f_sparsity_%.4f' % ( + iteration_count, finetune_acc, cur_ratio) + checkpoint_path = os.path.join( + self.checkpoint_dir, '%s.pth' % checkpoint_name) + cfg_path = os.path.join( + self.checkpoint_dir, '%s_pruner.json' % checkpoint_name) + sensitivity_path = os.path.join( + self.checkpoint_dir, '%s_sensitivity.csv' % checkpoint_name) + torch.save(self.model.state_dict(), checkpoint_path) + with open(cfg_path, 'w') as jf: + json.dump(cfg_list, jf) + self.analyzer.export(sensitivity_path) + + if cur_ratio > target_ratio: + # If this is the last prune iteration, skip the time-consuming + # sensitivity analysis + + self.analyzer.load_state_dict(self.model.state_dict()) + self.sensitivities = self.analyzer.analysis( + val_args=eval_args, val_kwargs=eval_kwargs) + + _logger.info('After Pruning: %.2f weights remains', cur_ratio) + self.modules_wrapper = modules_wrapper_final + + self._wrap_model() + return self.model + + def calc_mask(self, wrapper, **kwargs): + return None diff --git a/new_impl/cv/third_party/nni/algorithms/compression/pytorch/pruning/simulated_annealing_pruner.py b/new_impl/cv/third_party/nni/algorithms/compression/pytorch/pruning/simulated_annealing_pruner.py new file mode 100644 index 0000000000000000000000000000000000000000..50f193ba9e3951758c59ee7ef90af1633e4674c4 --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/compression/pytorch/pruning/simulated_annealing_pruner.py @@ -0,0 +1,357 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +import os +import math +import copy +import csv +import json +import numpy as np +from schema import And, Optional + +from nni.utils import OptimizeMode + +from .....compression.pytorch.compressor import Pruner +from .....compression.pytorch.utils.config_validation import CompressorSchema +from .constants_pruner import PRUNER_DICT + + +_logger = logging.getLogger(__name__) + + +class SimulatedAnnealingPruner(Pruner): + """ + A Pytorch implementation of Simulated Annealing compression algorithm. + + Parameters + ---------- + model : pytorch model + The model to be pruned. + config_list : list + Supported keys: + - sparsity : The target overall sparsity. + - op_types : The operation type to prune. + evaluator : function + Function to evaluate the pruned model. + This function should include `model` as the only parameter, and returns a scalar value. + Example:: + + def evaluator(model): + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + val_loader = ... + model.eval() + correct = 0 + with torch.no_grad(): + for data, target in val_loader: + data, target = data.to(device), target.to(device) + output = model(data) + # get the index of the max log-probability + pred = output.argmax(dim=1, keepdim=True) + correct += pred.eq(target.view_as(pred)).sum().item() + accuracy = correct / len(val_loader.dataset) + return accuracy + optimize_mode : str + Optimize mode, `maximize` or `minimize`, by default `maximize`. + base_algo : str + Base pruning algorithm. `level`, `l1` or `l2`, by default `l1`. Given the sparsity distribution among the ops, + the assigned `base_algo` is used to decide which filters/channels/weights to prune. + start_temperature : float + Start temperature of the simulated annealing process. + stop_temperature : float + Stop temperature of the simulated annealing process. + cool_down_rate : float + Cool down rate of the temperature. + perturbation_magnitude : float + Initial perturbation magnitude to the sparsities. The magnitude decreases with current temperature. + experiment_data_dir : string + PATH to save experiment data, + including the config_list generated for the base pruning algorithm, the performance of the pruned model and the pruning history. + + """ + + def __init__(self, model, config_list, evaluator, optimize_mode='maximize', base_algo='l1', + start_temperature=100, stop_temperature=20, cool_down_rate=0.9, perturbation_magnitude=0.35, experiment_data_dir='./'): + # original model + self._model_to_prune = copy.deepcopy(model) + self._base_algo = base_algo + + super().__init__(model, config_list) + + self._evaluator = evaluator + self._optimize_mode = OptimizeMode(optimize_mode) + + # hyper parameters for SA algorithm + self._start_temperature = start_temperature + self._current_temperature = start_temperature + self._stop_temperature = stop_temperature + self._cool_down_rate = cool_down_rate + self._perturbation_magnitude = perturbation_magnitude + + # overall pruning rate + self._sparsity = config_list[0]['sparsity'] + # pruning rates of the layers + self._sparsities = None + + # init current performance & best performance + self._current_performance = -np.inf + self._best_performance = -np.inf + self._best_config_list = [] + + self._search_history = [] + + self._experiment_data_dir = experiment_data_dir + if not os.path.exists(self._experiment_data_dir): + os.makedirs(self._experiment_data_dir) + + def validate_config(self, model, config_list): + """ + Parameters + ---------- + model : torch.nn.Module + Model to be pruned + config_list : list + List on pruning configs + """ + + if self._base_algo == 'level': + schema = CompressorSchema([{ + 'sparsity': And(float, lambda n: 0 < n < 1), + Optional('op_types'): [str], + Optional('op_names'): [str], + }], model, _logger) + elif self._base_algo in ['l1', 'l2']: + schema = CompressorSchema([{ + 'sparsity': And(float, lambda n: 0 < n < 1), + 'op_types': ['Conv2d'], + Optional('op_names'): [str] + }], model, _logger) + + schema.validate(config_list) + + def _sparsities_2_config_list(self, sparsities): + ''' + convert sparsities vector into config_list for LevelPruner or L1FilterPruner + + Parameters + ---------- + sparsities : list + list of sparsities + + Returns + ------- + list of dict + config_list for LevelPruner or L1FilterPruner + ''' + config_list = [] + + sparsities = sorted(sparsities) + self.modules_wrapper = sorted( + self.modules_wrapper, key=lambda wrapper: wrapper.module.weight.data.numel()) + + # a layer with more weights will have no less pruning rate + for idx, wrapper in enumerate(self.get_modules_wrapper()): + # L1Filter Pruner requires to specify op_types + if self._base_algo in ['l1', 'l2']: + config_list.append( + {'sparsity': sparsities[idx], 'op_types': ['Conv2d'], 'op_names': [wrapper.name]}) + elif self._base_algo == 'level': + config_list.append( + {'sparsity': sparsities[idx], 'op_names': [wrapper.name]}) + + config_list = [val for val in config_list if not math.isclose(val['sparsity'], 0, abs_tol=1e-6)] + + return config_list + + def _rescale_sparsities(self, sparsities, target_sparsity): + ''' + Rescale the sparsities list to satisfy the target overall sparsity + + Parameters + ---------- + sparsities : list + + target_sparsity : float + the target overall sparsity + + Returns + ------- + list + the rescaled sparsities + ''' + num_weights = [] + for wrapper in self.get_modules_wrapper(): + num_weights.append(wrapper.module.weight.data.numel()) + + num_weights = sorted(num_weights) + sparsities = sorted(sparsities) + + total_weights = 0 + total_weights_pruned = 0 + + # calculate the scale + for idx, num_weight in enumerate(num_weights): + total_weights += num_weight + total_weights_pruned += int(num_weight*sparsities[idx]) + if total_weights_pruned == 0: + return None + scale = target_sparsity / (total_weights_pruned/total_weights) + + # rescale the sparsities + sparsities = np.asarray(sparsities)*scale + + return sparsities + + def _init_sparsities(self): + ''' + Generate a sorted sparsities vector + ''' + # repeatedly generate a distribution until satisfies the overall sparsity requirement + _logger.info('Gererating sparsities...') + while True: + sparsities = sorted(np.random.uniform( + 0, 1, len(self.get_modules_wrapper()))) + + sparsities = self._rescale_sparsities( + sparsities, target_sparsity=self._sparsity) + + if sparsities is not None and sparsities[0] >= 0 and sparsities[-1] < 1: + _logger.info('Initial sparsities generated : %s', sparsities) + self._sparsities = sparsities + break + + def _generate_perturbations(self): + ''' + Generate perturbation to the current sparsities distribution. + + Returns: + -------- + list + perturbated sparsities + ''' + _logger.info("Gererating perturbations to the current sparsities...") + + # decrease magnitude with current temperature + magnitude = self._current_temperature / \ + self._start_temperature * self._perturbation_magnitude + _logger.info('current perturation magnitude:%s', magnitude) + + while True: + perturbation = np.random.uniform(-magnitude, magnitude, len(self.get_modules_wrapper())) + sparsities = np.clip(0, self._sparsities + perturbation, None) + _logger.debug("sparsities before rescalling:%s", sparsities) + + sparsities = self._rescale_sparsities(sparsities, target_sparsity=self._sparsity) + _logger.debug("sparsities after rescalling:%s", sparsities) + + if sparsities is not None and sparsities[0] >= 0 and sparsities[-1] < 1: + _logger.info("Sparsities perturbated:%s", sparsities) + return sparsities + + def calc_mask(self, wrapper, **kwargs): + return None + + def compress(self, return_config_list=False): + """ + Compress the model with Simulated Annealing. + + Returns + ------- + torch.nn.Module + model with specified modules compressed. + """ + _logger.info('Starting Simulated Annealing Compression...') + + # initiaze a randomized action + pruning_iteration = 0 + self._init_sparsities() + + # stop condition + self._current_temperature = self._start_temperature + while self._current_temperature > self._stop_temperature: + _logger.info('Pruning iteration: %d', pruning_iteration) + _logger.info('Current temperature: %d, Stop temperature: %d', + self._current_temperature, self._stop_temperature) + while True: + # generate perturbation + sparsities_perturbated = self._generate_perturbations() + config_list = self._sparsities_2_config_list( + sparsities_perturbated) + _logger.info( + "config_list for Pruner generated: %s", config_list) + + # fast evaluation + pruner = PRUNER_DICT[self._base_algo](copy.deepcopy(self._model_to_prune), config_list) + model_masked = pruner.compress() + evaluation_result = self._evaluator(model_masked) + + self._search_history.append( + {'sparsity': self._sparsity, 'performance': evaluation_result, 'config_list': config_list}) + + if self._optimize_mode is OptimizeMode.Minimize: + evaluation_result *= -1 + + # if better evaluation result, then accept the perturbation + if evaluation_result > self._current_performance: + self._current_performance = evaluation_result + self._sparsities = sparsities_perturbated + + # save best performance and best params + if evaluation_result > self._best_performance: + _logger.info('updating best model...') + self._best_performance = evaluation_result + self._best_config_list = config_list + + # save the overall best masked model + self.bound_model = model_masked + # the ops with sparsity 0 are not included in this modules_wrapper + modules_wrapper_final = pruner.get_modules_wrapper() + break + # if not, accept with probability e^(-deltaE/current_temperature) + else: + delta_E = np.abs(evaluation_result - + self._current_performance) + probability = math.exp(-1 * delta_E / + self._current_temperature) + if np.random.uniform(0, 1) < probability: + self._current_performance = evaluation_result + self._sparsities = sparsities_perturbated + break + + # cool down + self._current_temperature *= self._cool_down_rate + pruning_iteration += 1 + + _logger.info('----------Compression finished--------------') + _logger.info('Best performance: %s', self._best_performance) + _logger.info('config_list found : %s', + self._best_config_list) + + # save search history + with open(os.path.join(self._experiment_data_dir, 'search_history.csv'), 'w') as csvfile: + writer = csv.DictWriter(csvfile, fieldnames=['sparsity', 'performance', 'config_list']) + writer.writeheader() + for item in self._search_history: + writer.writerow({'sparsity': item['sparsity'], 'performance': item['performance'], 'config_list': json.dumps( + item['config_list'])}) + + # save best config found and best performance + if self._optimize_mode is OptimizeMode.Minimize: + self._best_performance *= -1 + with open(os.path.join(self._experiment_data_dir, 'search_result.json'), 'w+') as jsonfile: + json.dump({ + 'performance': self._best_performance, + 'config_list': json.dumps(self._best_config_list) + }, jsonfile) + + _logger.info('search history and result saved to foler : %s', + self._experiment_data_dir) + + if return_config_list: + return self._best_config_list + + # This should be done only at the final stage, + # because the modules_wrapper with all the ops are used during the annealing process + self.modules_wrapper = modules_wrapper_final + + return self.bound_model diff --git a/new_impl/cv/third_party/nni/algorithms/compression/pytorch/pruning/structured_pruning.py b/new_impl/cv/third_party/nni/algorithms/compression/pytorch/pruning/structured_pruning.py new file mode 100644 index 0000000000000000000000000000000000000000..d1f65bc00e2c50405a245404e79b0a584734940b --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/compression/pytorch/pruning/structured_pruning.py @@ -0,0 +1,1272 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +import math +from zedl.dl.common.model import get_module +from zedl.third_party.nni.common.graph_utils import TorchModuleGraph +import numpy as np +import torch +import random +from .weight_masker import WeightMasker + +# from util.model import LayerActivation + +__all__ = ['L1FilterPrunerMasker', 'L2FilterPrunerMasker', 'FPGMPrunerMasker', + 'TaylorFOWeightFilterPrunerMasker', 'ActivationAPoZRankFilterPrunerMasker', + 'ActivationMeanRankFilterPrunerMasker', 'SlimPrunerMasker', 'AMCWeightMasker', + 'TRRPrunerMasker', 'HRankPrunerMasker', 'PFPMasker'] + +logger = logging.getLogger('torch filter pruners') + + +class StructuredWeightMasker(WeightMasker): + """ + A structured pruning masker base class that prunes convolutional layer filters. + + Parameters + ---------- + model: nn.Module + model to be pruned + pruner: Pruner + A Pruner instance used to prune the model + preserve_round: int + after pruning, preserve filters/channels round to `preserve_round`, for example: + for a Conv2d layer, output channel is 32, sparsity is 0.2, if preserve_round is + 1 (no preserve round), then there will be int(32 * 0.2) = 6 filters pruned, and + 32 - 6 = 26 filters are preserved. If preserve_round is 4, preserved filters will + be round up to 28 (which can be divided by 4) and only 4 filters are pruned. + + """ + + def __init__(self, model, pruner, preserve_round=1, dependency_aware=False): + self.model = model + self.pruner = pruner + self.preserve_round = preserve_round + self.dependency_aware = dependency_aware + + def calc_mask(self, sparsity, wrapper, wrapper_idx=None, **depen_kwargs): + """ + calculate the mask for `wrapper`. + Parameters + ---------- + sparsity: float/list of float + The target sparsity of the wrapper. If we calculate the mask in + the normal way, then sparsity is a float number. In contrast, if + we calculate the mask in the dependency-aware way, sparsity is a + list of float numbers, each float number corressponds to a sparsity + of a layer. + wrapper: PrunerModuleWrapper/list of PrunerModuleWrappers + The wrapper of the target layer. If we calculate the mask in the normal + way, then `wrapper` is an instance of PrunerModuleWrapper, else `wrapper` + is a list of PrunerModuleWrapper. + wrapper_idx: int/list of int + The index of the wrapper. + depen_kwargs: dict + The kw_args for the dependency-aware mode. + """ + if not self.dependency_aware: + # calculate the mask in the normal way, each layer calculate its + # own mask separately + return self._normal_calc_mask(sparsity, wrapper, wrapper_idx) + else: + # if the dependency_aware switch is on, then calculate the mask + # in the dependency-aware way + return self._dependency_calc_mask(sparsity, wrapper, wrapper_idx, **depen_kwargs) + + def _get_current_state(self, sparsity, wrapper, wrapper_idx=None): + """ + Some pruner may prune the layers in a iterative way. In each pruning iteration, + we may get the current state of this wrapper/layer, and continue to prune this layer + based on the current state. This function is to get the current pruning state of the + target wrapper/layer. + Parameters + ---------- + sparsity: float + pruning ratio, preserved weight ratio is `1 - sparsity` + wrapper: PrunerModuleWrapper + layer wrapper of this layer + wrapper_idx: int + index of this wrapper in pruner's all wrappers + Returns + ------- + base_mask: dict + dict object that stores the mask of this wrapper in this iteration, if it is the + first iteration, then we create a new mask with all ones. If there is already a + mask in this wrapper, then we return the existing mask. + weight: tensor + the current weight of this layer + num_prune: int + how many filters we should prune + """ + msg = 'module type {} is not supported!'.format(wrapper.type) + assert wrapper.type == 'Conv2d' or wrapper.type == 'ConvTranspose2d', msg + weight = wrapper.module.weight.data + bias = None + if hasattr(wrapper.module, 'bias') and wrapper.module.bias is not None: + bias = wrapper.module.bias.data + + if wrapper.weight_mask is None: + mask_weight = torch.ones(weight.size()).type_as(weight).detach() + else: + mask_weight = wrapper.weight_mask.clone() + if bias is not None: + if wrapper.bias_mask is None: + mask_bias = torch.ones(bias.size()).type_as(bias).detach() + else: + mask_bias = wrapper.bias_mask.clone() + else: + mask_bias = None + mask = {'weight_mask': mask_weight, 'bias_mask': mask_bias} + + if isinstance(wrapper.module, torch.nn.Conv2d): + num_total = weight.size(0) + elif isinstance(wrapper.module, torch.nn.ConvTranspose2d): + num_total = weight.size(1) + + num_prune = int(num_total * sparsity) + if self.preserve_round > 1: + num_preserve = num_total - num_prune + num_preserve = int( + math.ceil(num_preserve * 1. / self.preserve_round) * self.preserve_round) + if num_preserve > num_total: + num_preserve = int(math.floor( + num_total * 1. / self.preserve_round) * self.preserve_round) + num_prune = num_total - num_preserve + # weight*mask_weight: apply base mask for iterative pruning + return mask, weight * mask_weight, num_prune + + def _normal_calc_mask(self, sparsity, wrapper, wrapper_idx=None): + """ + Calculate the mask of given layer. + Parameters + ---------- + sparsity: float + pruning ratio, preserved weight ratio is `1 - sparsity` + wrapper: PrunerModuleWrapper + layer wrapper of this layer + wrapper_idx: int + index of this wrapper in pruner's all wrappers + Returns + ------- + dict + dictionary for storing masks, keys of the dict: + 'weight_mask': weight mask tensor + 'bias_mask': bias mask tensor (optional) + """ + mask, weight, num_prune = self._get_current_state( + sparsity, wrapper, wrapper_idx) + num_total = weight.size(0) + if num_total < 2 or num_prune < 1: + return mask + + return self.get_mask(mask, weight, num_prune, wrapper, wrapper_idx) + + def _common_channel_to_prune(self, sparsities, wrappers, wrappers_idx, channel_dsets, groups): + """ + Calculate the common channels should be pruned by all the layers in this group. + This function is for filter pruning of Conv layers. if want to support the dependency-aware + mode for others ops, you need to inherit this class and overwrite `_common_channel_to_prune`. + + Parameters + ---------- + sparsities : list + List of float that specify the sparsity for each conv layer. + wrappers : list + List of wrappers + groups : list + The number of the filter groups of each layer. + wrappers_idx : list + The indexes of the wrappers + """ + # sparsity configs for each wrapper + # sparsities = [_w.config['sparsity'] for _w in wrappers] + # check the type of the input wrappers + for _w in wrappers: + msg = 'module type {} is not supported!'.format(_w.type) + assert _w.type == 'Conv2d', msg + # Among the dependent layers, the layer with smallest + # sparsity determines the final benefit of the speedup + # module. To better harvest the speed benefit, we need + # to ensure that these dependent layers have at least + # `min_sparsity` pruned channel are the same. + if len(channel_dsets) == len(wrappers): + # all the layers in the dependency sets are pruned + min_sparsity = min(sparsities) + else: + # not all the layers in the dependency set + # are pruned + min_sparsity = 0 + # donnot prune the channels that we cannot harvest the speed from + sparsities = [min_sparsity] * len(sparsities) + # find the max number of the filter groups of the dependent + # layers. The group constraint of this dependency set is decided + # by the layer with the max groups. + + # should use the least common multiple for all the groups + # the max_group is lower than the channel_count, because + # the number of the filter is always divisible by the number of the group + max_group = np.lcm.reduce(groups) + channel_count = wrappers[0].module.weight.data.size(0) + device = wrappers[0].module.weight.device + channel_sum = torch.zeros(channel_count).to(device) + for _w, _w_idx in zip(wrappers, wrappers_idx): + # calculate the L1/L2 sum for all channels + c_sum = self.get_channel_sum(_w, _w_idx) + + if c_sum is None: + # if the channel sum cannot be calculated + # now, return None + return None + channel_sum += c_sum + + # prune the same `min_sparsity` channels based on channel_sum + # for all the layers in the channel sparsity + target_pruned = int(channel_count * min_sparsity) + # pruned_per_group may be zero, for example dw conv + pruned_per_group = int(target_pruned / max_group) + group_step = int(channel_count / max_group) + + channel_masks = [] + for gid in range(max_group): + _start = gid * group_step + _end = (gid + 1) * group_step + if pruned_per_group > 0: + threshold = torch.topk( + channel_sum[_start: _end], pruned_per_group, largest=False)[0].max() + group_mask = torch.gt(channel_sum[_start:_end], threshold) + else: + group_mask = torch.ones(group_step).to(device) + channel_masks.append(group_mask) + channel_masks = torch.cat(channel_masks, dim=0) + pruned_channel_index = ( + channel_masks == False).nonzero().squeeze(1).tolist() + logger.info('Prune the %s channels for all dependent', + ','.join([str(x) for x in pruned_channel_index])) + return channel_masks + + def _dependency_calc_mask(self, sparsities, wrappers, wrappers_idx, channel_dsets, groups): + """ + Calculate the masks for the layers in the same dependency sets. + Similar to the traditional original calc_mask, _dependency_calc_mask + will prune the target layers based on the L1/L2 norm of the weights. + However, StructuredWeightMasker prunes the filter completely based on the + L1/L2 norm of each filter. In contrast, _dependency_calc_mask + will try to satisfy the channel/group dependency(see nni.compression.torch. + utils.shape_dependency for details). Specifically, _dependency_calc_mask + will try to prune the same channels for the layers that have channel dependency. + In addition, this mask calculator will also ensure that the number of filters + pruned in each group is the same(meet the group dependency). + + Parameters + ---------- + sparsities : list + List of float that specify the sparsity for each conv layer. + wrappers : list + List of wrappers + groups : list + The number of the filter groups of each layer. + wrappers_idx : list + The indexes of the wrappers + """ + channel_masks = self._common_channel_to_prune( + sparsities, wrappers, wrappers_idx, channel_dsets, groups) + # calculate the mask for each layer based on channel_masks, first + # every layer will prune the same channels masked in channel_masks. + # If the sparsity of a layers is larger than min_sparsity, then it + # will continue prune sparsity - min_sparsity channels to meet the sparsity + # config. + masks = {} + for _pos, _w in enumerate(wrappers): + _w_idx = wrappers_idx[_pos] + sparsity = sparsities[_pos] + name = _w.name + + # _tmp_mask = self._normal_calc_mask( + # sparsity, _w, _w_idx, channel_masks) + base_mask, current_weight, num_prune = self._get_current_state( + sparsity, _w, _w_idx) + num_total = current_weight.size(0) + if num_total < 2 or num_prune < 1: + return base_mask + _tmp_mask = self.get_mask( + base_mask, current_weight, num_prune, _w, _w_idx, channel_masks) + + if _tmp_mask is None: + # if the mask calculation fails + return None + masks[name] = _tmp_mask + return masks + + def get_mask(self, base_mask, weight, num_prune, wrapper, wrapper_idx, channel_masks=None): + """ + Calculate the mask of given layer. + Parameters + ---------- + base_mask: dict + The basic mask with the same shape of weight, all item in the basic mask is 1. + weight: tensor + the module weight to be pruned + num_prune: int + Num of filters to prune + wrapper: PrunerModuleWrapper + layer wrapper of this layer + wrapper_idx: int + index of this wrapper in pruner's all wrappers + channel_masks: Tensor + If mask some channels for this layer in advance. In the dependency-aware + mode, before calculating the masks for each layer, we will calculate a common + mask for all the layers in the dependency set. For the pruners that doesnot + support dependency-aware mode, they can just ignore this parameter. + Returns + ------- + dict + dictionary for storing masks + """ + raise NotImplementedError( + '{} get_mask is not implemented'.format(self.__class__.__name__)) + + def get_channel_sum(self, wrapper, wrapper_idx): + """ + Calculate the importance weight for each channel. If want to support the + dependency-aware mode for this one-shot pruner, this function must be + implemented. + Parameters + ---------- + wrapper: PrunerModuleWrapper + layer wrapper of this layer + wrapper_idx: int + index of this wrapper in pruner's all wrappers + Returns + ------- + tensor + Tensor that indicates the importance of each channel + """ + raise NotImplementedError( + '{} get_channel_sum is not implemented'.format(self.__class__.__name__)) + + +class L1FilterPrunerMasker(StructuredWeightMasker): + """ + A structured pruning algorithm that prunes the filters of smallest magnitude + weights sum in the convolution layers to achieve a preset level of network sparsity. + Hao Li, Asim Kadav, Igor Durdanovic, Hanan Samet and Hans Peter Graf, + "PRUNING FILTERS FOR EFFICIENT CONVNETS", 2017 ICLR + https://arxiv.org/abs/1608.08710 + """ + + def get_mask(self, base_mask, weight, num_prune, wrapper, wrapper_idx, channel_masks=None): + if isinstance(wrapper.module, torch.nn.Conv2d): + # get the l1-norm sum for each filter + w_abs_structured = self.get_channel_sum(wrapper, wrapper_idx) + if channel_masks is not None: + # if we need to mask some channels in advance + w_abs_structured = w_abs_structured * channel_masks + threshold = torch.topk(w_abs_structured.view(-1), + num_prune, largest=False)[0].max() + mask_weight = torch.gt(w_abs_structured, threshold)[ + :, None, None, None].expand_as(weight).type_as(weight) + mask_bias = torch.gt(w_abs_structured, threshold).type_as( + weight).detach() if base_mask['bias_mask'] is not None else None + + return {'weight_mask': mask_weight.detach(), 'bias_mask': mask_bias} + elif isinstance(wrapper.module, torch.nn.ConvTranspose2d): + # get the l1-norm sum for each filter + w_abs_structured = self.get_channel_sum(wrapper, wrapper_idx) + if channel_masks is not None: + # if we need to mask some channels in advance + w_abs_structured = w_abs_structured * channel_masks + threshold = torch.topk(w_abs_structured.view(-1), + num_prune, largest=False)[0].max() + mask_weight = torch.gt(w_abs_structured, threshold)[ + None, :, None, None].expand_as(weight).type_as(weight) + mask_bias = torch.gt(w_abs_structured, threshold).type_as( + weight).detach() if base_mask['bias_mask'] is not None else None + + return {'weight_mask': mask_weight.detach(), 'bias_mask': mask_bias} + + def get_channel_sum(self, wrapper, wrapper_idx): + if isinstance(wrapper.module, torch.nn.Conv2d): + weight = wrapper.module.weight.data + filters = weight.shape[0] + w_abs = weight.abs() + w_abs_structured = w_abs.view(filters, -1).sum(dim=1) + return w_abs_structured + elif isinstance(wrapper.module, torch.nn.ConvTranspose2d): + weight = wrapper.module.weight.data + filters = weight.shape[1] + w_abs = weight.abs() + w_abs_structured = w_abs.view(filters, -1).sum(dim=1) + return w_abs_structured + + +class L2FilterPrunerMasker(StructuredWeightMasker): + """ + A structured pruning algorithm that prunes the filters with the + smallest L2 norm of the weights. + """ + + def get_mask(self, base_mask, weight, num_prune, wrapper, wrapper_idx, channel_masks=None): + # get the l2-norm sum for each filter + w_l2_norm = self.get_channel_sum(wrapper, wrapper_idx) + if channel_masks is not None: + # if we need to mask some channels in advance + w_l2_norm = w_l2_norm * channel_masks + threshold = torch.topk( + w_l2_norm.view(-1), num_prune, largest=False)[0].max() + mask_weight = torch.gt(w_l2_norm, threshold)[ + :, None, None, None].expand_as(weight).type_as(weight) + mask_bias = torch.gt(w_l2_norm, threshold).type_as( + weight).detach() if base_mask['bias_mask'] is not None else None + + return {'weight_mask': mask_weight.detach(), 'bias_mask': mask_bias} + + def get_channel_sum(self, wrapper, wrapper_idx): + weight = wrapper.module.weight.data + filters = weight.shape[0] + w = weight.view(filters, -1) + w_l2_norm = torch.sqrt((w ** 2).sum(dim=1)) + return w_l2_norm + + +class TRRPrunerMasker(StructuredWeightMasker): + """ + A filter pruner via Triplet Response Residual (TRR). + B. Fang, X. Zeng, and M. Zhang, + “NestDNN: Resource-aware multi-tenant on-device deep learning for continuous mobile vision,” + in Proc. ACM Mobicom, 2018, pp. 115–127. + + Implemented by queyu, 2020/11/13 + """ + + def __init__(self, model, pruner, preserve_round=1, dependency_aware=False, train_loader=None, device='cuda'): + super().__init__(model, pruner, preserve_round, dependency_aware) + + assert train_loader is not None, 'TRR needs the training dataset' + self.train_loader = train_loader + self.device = device + + self.model.to(device) + + def get_mask(self, base_mask, weight, num_prune, wrapper, wrapper_idx, channel_masks=None): + trrs = self.get_channel_sum(wrapper, wrapper_idx) + if channel_masks is not None: + # if we need to mask some channels in advance + trrs = trrs * channel_masks + + threshold = torch.topk( + trrs, num_prune, largest=False)[0].max() + mask_weight = torch.gt(trrs, threshold)[ + :, None, None, None].expand_as(weight).type_as(weight) + mask_bias = torch.gt(trrs, threshold).type_as( + weight).detach() if base_mask['bias_mask'] is not None else None + + return {'weight_mask': mask_weight.detach(), 'bias_mask': mask_bias} + + def get_channel_sum(self, wrapper, wrapper_idx): + # add hook in the target module + module_activation = LayerActivation(wrapper.module, '', 'cpu') + self.model.eval() + + filter_num = wrapper.module.weight.data.size()[0] + trrs = torch.zeros(filter_num) + data_num = len(self.train_loader.dataset) + cur_data_num = 0 + triplets_num = 0 + + with torch.no_grad(): + for data, target in self.train_loader: + cur_data_num += data.size()[0] + # if cur_data_num > data_num * 0.2: + # continue + + data, target = data.to(self.device), target.to(self.device) + self.model(data) + + # construct {anc, pos, neg} from data and target + # compute TRR from data in module_activation.output + triplets = self._constrcut_trr_triplets(data, target) + triplets_num += len(triplets) + trrs += self._compute_trr(triplets, module_activation.output) + + module_activation.remove() + return trrs + + def _constrcut_trr_triplets(self, data, target): + # return the indexes, not the data + + # randomly select the anc in data + anc = random.randint(0, data.size()[0] - 1) + anc_label = target[anc] + + triplets = [] + poses = [] + negs = [] + + # find the pos, which has the same class with anc + for i, (item, label) in enumerate(zip(data, target)): + if i == anc: + continue + # pos + if label == anc_label: + poses += [i] + # neg + else: + negs += [i] + + # balance the pos and neg + # min_len = min(len(poses), len(negs)) + # poses = poses[0: min_len + 1] + # negs = negs[0: min_len + 1] + + for pos in poses: + for neg in negs: + triplets += [(anc, pos, neg)] + + return triplets + + def _compute_trr(self, triplets, feature_map): + def l2_norm(data): + return (data ** 2).sum(dim=(1, 2)) + + trr = torch.zeros(feature_map.size()[1]) + for triplet in triplets: + anc, pos, neg = triplet + a = l2_norm(feature_map[anc] - feature_map[neg]) + b = l2_norm(feature_map[anc] - feature_map[pos]) + trr_item = a - b + trr += trr_item + + return trr + + +class FPGMPrunerMasker(StructuredWeightMasker): + """ + A filter pruner via geometric median. + "Filter Pruning via Geometric Median for Deep Convolutional Neural Networks Acceleration", + https://arxiv.org/pdf/1811.00250.pdf + """ + + def get_mask(self, base_mask, weight, num_prune, wrapper, wrapper_idx, channel_masks=None): + min_gm_idx = self._get_min_gm_kernel_idx( + num_prune, wrapper, wrapper_idx, channel_masks) + for idx in min_gm_idx: + base_mask['weight_mask'][idx] = 0. + if base_mask['bias_mask'] is not None: + base_mask['bias_mask'][idx] = 0. + return base_mask + + def _get_min_gm_kernel_idx(self, num_prune, wrapper, wrapper_idx, channel_masks): + channel_dist = self.get_channel_sum(wrapper, wrapper_idx) + if channel_masks is not None: + channel_dist = channel_dist * channel_masks + dist_list = [(channel_dist[i], i) + for i in range(channel_dist.size(0))] + min_gm_kernels = sorted(dist_list, key=lambda x: x[0])[:num_prune] + return [x[1] for x in min_gm_kernels] + + def _get_distance_sum(self, weight, out_idx): + """ + Calculate the total distance between a specified filter (by out_idex and in_idx) and + all other filters. + Parameters + ---------- + weight: Tensor + convolutional filter weight + out_idx: int + output channel index of specified filter, this method calculates the total distance + between this specified filter and all other filters. + Returns + ------- + float32 + The total distance + """ + logger.debug('weight size: %s', weight.size()) + assert len(weight.size()) in [3, 4], 'unsupported weight shape' + + w = weight.view(weight.size(0), -1) + anchor_w = w[out_idx].unsqueeze(0).expand(w.size(0), w.size(1)) + x = w - anchor_w + x = (x * x).sum(-1) + x = torch.sqrt(x) + return x.sum() + + def get_channel_sum(self, wrapper, wrapper_idx): + weight = wrapper.module.weight.data + assert len(weight.size()) in [3, 4] + dist_list = [] + for out_i in range(weight.size(0)): + dist_sum = self._get_distance_sum(weight, out_i) + dist_list.append(dist_sum) + return torch.Tensor(dist_list).to(weight.device) + + +class TaylorFOWeightFilterPrunerMasker(StructuredWeightMasker): + """ + A structured pruning algorithm that prunes the filters with the smallest + importance approximations based on the first order taylor expansion on the weight. + Molchanov, Pavlo and Mallya, Arun and Tyree, Stephen and Frosio, Iuri and Kautz, Jan, + "Importance Estimation for Neural Network Pruning", CVPR 2019. + http://jankautz.com/publications/Importance4NNPruning_CVPR19.pdf + """ + + def __init__(self, model, pruner, statistics_batch_num=1): + super().__init__(model, pruner) + self.pruner.statistics_batch_num = statistics_batch_num + self.pruner.set_wrappers_attribute("contribution", None) + self.pruner.iterations = 0 + self.pruner.patch_optimizer(self.calc_contributions) + + def get_mask(self, base_mask, weight, num_prune, wrapper, wrapper_idx, channel_masks=None): + if isinstance(wrapper.module, torch.nn.Conv2d): + channel_contribution = self.get_channel_sum(wrapper, wrapper_idx) + if channel_contribution is None: + # iteration is not enough + return None + if channel_masks is not None: + channel_contribution = channel_contribution * channel_masks + prune_indices = torch.argsort(channel_contribution)[:num_prune] + for idx in prune_indices: + base_mask['weight_mask'][idx] = 0. + if base_mask['bias_mask'] is not None: + base_mask['bias_mask'][idx] = 0. + return base_mask + elif isinstance(wrapper.module, torch.nn.ConvTranspose2d): + channel_contribution = self.get_channel_sum(wrapper, wrapper_idx) + if channel_contribution is None: + # iteration is not enough + return None + if channel_masks is not None: + channel_contribution = channel_contribution * channel_masks + prune_indices = torch.argsort(channel_contribution)[:num_prune] + for idx in prune_indices: + base_mask['weight_mask'][:, idx] = 0. + if base_mask['bias_mask'] is not None: + base_mask['bias_mask'][idx] = 0. + return base_mask + + def calc_contributions(self): + """ + Calculate the estimated importance of filters as a sum of individual contribution + based on the first order taylor expansion. + """ + if self.pruner.iterations >= self.pruner.statistics_batch_num: + return + for wrapper in self.pruner.get_modules_wrapper(): + if isinstance(wrapper.module, torch.nn.Conv2d): + filters = wrapper.module.weight.size(0) + contribution = ( + wrapper.module.weight*wrapper.module.weight.grad).data.pow(2).view(filters, -1).sum(dim=1) + if wrapper.contribution is None: + wrapper.contribution = contribution + else: + wrapper.contribution += contribution + elif isinstance(wrapper.module, torch.nn.ConvTranspose2d): + filters = wrapper.module.weight.size(1) + contribution = ( + wrapper.module.weight*wrapper.module.weight.grad).data.pow(2).view(filters, -1).sum(dim=1) + if wrapper.contribution is None: + wrapper.contribution = contribution + else: + wrapper.contribution += contribution + + self.pruner.iterations += 1 + + def get_channel_sum(self, wrapper, wrapper_idx): + if self.pruner.iterations < self.pruner.statistics_batch_num: + return None + if wrapper.contribution is None: + return None + return wrapper.contribution + + +class ActivationFilterPrunerMasker(StructuredWeightMasker): + def __init__(self, model, pruner, statistics_batch_num=1, activation='relu'): + super().__init__(model, pruner) + self.statistics_batch_num = statistics_batch_num + self.pruner.hook_id = self._add_activation_collector(self.pruner) + + assert activation in ['relu', 'relu6', None] + if activation == 'relu': + self.pruner.activation = torch.nn.functional.relu + elif activation == 'relu6': + self.pruner.activation = torch.nn.functional.relu6 + else: + self.pruner.activation = None + + def _add_activation_collector(self, pruner): + def collector(collected_activation): + def hook(module_, input_, output): + if pruner.activation is not None: + collected_activation.append( + pruner.activation(output.detach().cpu())) + else: + collected_activation.append( + output.detach().cpu()) + return hook + pruner.collected_activation = {} + pruner._fwd_hook_id += 1 + pruner._fwd_hook_handles[pruner._fwd_hook_id] = [] + + for wrapper_idx, wrapper in enumerate(pruner.get_modules_wrapper()): + pruner.collected_activation[wrapper_idx] = [] + handle = wrapper.register_forward_hook( + collector(pruner.collected_activation[wrapper_idx])) + + pruner._fwd_hook_handles[pruner._fwd_hook_id].append(handle) + return pruner._fwd_hook_id + + +class ActivationAPoZRankFilterPrunerMasker(ActivationFilterPrunerMasker): + """ + A structured pruning algorithm that prunes the filters with the + smallest APoZ(average percentage of zeros) of output activations. + Hengyuan Hu, Rui Peng, Yu-Wing Tai and Chi-Keung Tang, + "Network Trimming: A Data-Driven Neuron Pruning Approach towards Efficient Deep Architectures", ICLR 2016. + https://arxiv.org/abs/1607.03250 + """ + + def get_mask(self, base_mask, weight, num_prune, wrapper, wrapper_idx, channel_masks=None): + apoz = self.get_channel_sum(wrapper, wrapper_idx) + if apoz is None: + # the collected activations are not enough + return None + if channel_masks is not None: + apoz = apoz * channel_masks + + prune_indices = torch.argsort(apoz)[:num_prune] + for idx in prune_indices: + base_mask['weight_mask'][idx] = 0. + if base_mask['bias_mask'] is not None: + base_mask['bias_mask'][idx] = 0. + + if self.pruner.hook_id in self.pruner._fwd_hook_handles: + self.pruner.remove_activation_collector(self.pruner.hook_id) + + return base_mask + + def _calc_apoz(self, activations): + """ + Calculate APoZ(average percentage of zeros) of activations. + + Parameters + ---------- + activations : list + Layer's output activations + + Returns + ------- + torch.Tensor + Filter's APoZ(average percentage of zeros) of the activations + """ + activations = torch.cat(activations, 0) + _eq_zero = torch.eq(activations, torch.zeros_like(activations)) + _apoz = torch.sum(_eq_zero, dim=(0, 2, 3), dtype=torch.float64) / \ + torch.numel(_eq_zero[:, 0, :, :]) + return torch.ones_like(_apoz) - _apoz + + def get_channel_sum(self, wrapper, wrapper_idx): + assert wrapper_idx is not None + activations = self.pruner.collected_activation[wrapper_idx] + if len(activations) < self.statistics_batch_num: + # collected activations is not enough + return None + return self._calc_apoz(activations).to(wrapper.module.weight.device) + + +class ActivationMeanRankFilterPrunerMasker(ActivationFilterPrunerMasker): + """ + A structured pruning algorithm that prunes the filters with the + smallest mean value of output activations. + Pavlo Molchanov, Stephen Tyree, Tero Karras, Timo Aila and Jan Kautz, + "Pruning Convolutional Neural Networks for Resource Efficient Inference", ICLR 2017. + https://arxiv.org/abs/1611.06440 + """ + + def get_mask(self, base_mask, weight, num_prune, wrapper, wrapper_idx, channel_masks=None): + + mean_activation = self.get_channel_sum(wrapper, wrapper_idx) + if mean_activation is None: + # the collected activation is not enough + return None + if channel_masks is not None: + mean_activation = mean_activation * channel_masks + + prune_indices = torch.argsort(mean_activation)[:num_prune] + for idx in prune_indices: + base_mask['weight_mask'][idx] = 0. + if base_mask['bias_mask'] is not None: + base_mask['bias_mask'][idx] = 0. + # if len(activations) < self.statistics_batch_num, the code + # cannot reach here + if self.pruner.hook_id in self.pruner._fwd_hook_handles: + self.pruner.remove_activation_collector(self.pruner.hook_id) + + return base_mask + + def _cal_mean_activation(self, activations): + """ + Calculate mean value of activations. + + Parameters + ---------- + activations : list + Layer's output activations + + Returns + ------- + torch.Tensor + Filter's mean value of the output activations + """ + activations = torch.cat(activations, 0) + mean_activation = torch.mean(activations, dim=(0, 2, 3)) + return mean_activation + + def get_channel_sum(self, wrapper, wrapper_idx): + assert wrapper_idx is not None + activations = self.pruner.collected_activation[wrapper_idx] + if len(activations) < self.statistics_batch_num: + return None + # the memory overhead here is acceptable, because only + # the mean_activation tensor returned by _cal_mean_activation + # is transfer to gpu. + return self._cal_mean_activation(activations).to(wrapper.module.weight.device) + + +class SlimPrunerMasker(WeightMasker): + """ + A structured pruning algorithm that prunes channels by pruning the weights of BN layers. + Zhuang Liu, Jianguo Li, Zhiqiang Shen, Gao Huang, Shoumeng Yan and Changshui Zhang + "Learning Efficient Convolutional Networks through Network Slimming", 2017 ICCV + https://arxiv.org/pdf/1708.06519.pdf + """ + + def __init__(self, model, pruner, **kwargs): + super().__init__(model, pruner) + weight_list = [] + for (layer, _) in pruner.get_modules_to_compress(): + weight_list.append(layer.module.weight.data.abs().clone()) + all_bn_weights = torch.cat(weight_list) + k = int(all_bn_weights.shape[0] * pruner.config_list[0]['sparsity']) + self.global_threshold = torch.topk( + all_bn_weights.view(-1), k, largest=False)[0].max() + + def calc_mask(self, sparsity, wrapper, wrapper_idx=None): + assert wrapper.type == 'BatchNorm2d', 'SlimPruner only supports 2d batch normalization layer pruning' + weight = wrapper.module.weight.data.clone() + if wrapper.weight_mask is not None: + # apply base mask for iterative pruning + weight = weight * wrapper.weight_mask + + base_mask = torch.ones(weight.size()).type_as(weight).detach() + mask = {'weight_mask': base_mask.detach( + ), 'bias_mask': base_mask.clone().detach()} + filters = weight.size(0) + num_prune = int(filters * sparsity) + if filters >= 2 and num_prune >= 1: + w_abs = weight.abs() + mask_weight = torch.gt( + w_abs, self.global_threshold).type_as(weight) + mask_bias = mask_weight.clone() + mask = {'weight_mask': mask_weight.detach( + ), 'bias_mask': mask_bias.detach()} + return mask + + +def least_square_sklearn(X, Y): + from sklearn.linear_model import LinearRegression + reg = LinearRegression(fit_intercept=False) + reg.fit(X, Y) + return reg.coef_ + + +class AMCWeightMasker(WeightMasker): + """ + Weight maskser class for AMC pruner. Currently, AMCPruner only supports pruning kernel + size 1x1 pointwise Conv2d layer. Before using this class to prune kernels, AMCPruner + collected input and output feature maps for each layer, the features maps are flattened + and save into wrapper.input_feat and wrapper.output_feat. + + Parameters + ---------- + model: nn.Module + model to be pruned + pruner: Pruner + A Pruner instance used to prune the model + preserve_round: int + after pruning, preserve filters/channels round to `preserve_round`, for example: + for a Conv2d layer, output channel is 32, sparsity is 0.2, if preserve_round is + 1 (no preserve round), then there will be int(32 * 0.2) = 6 filters pruned, and + 32 - 6 = 26 filters are preserved. If preserve_round is 4, preserved filters will + be round up to 28 (which can be divided by 4) and only 4 filters are pruned. + """ + + def __init__(self, model, pruner, preserve_round=1): + self.model = model + self.pruner = pruner + self.preserve_round = preserve_round + + def calc_mask(self, sparsity, wrapper, wrapper_idx=None, preserve_idx=None): + """ + Calculate the mask of given layer. + Parameters + ---------- + sparsity: float + pruning ratio, preserved weight ratio is `1 - sparsity` + wrapper: PrunerModuleWrapper + layer wrapper of this layer + wrapper_idx: int + index of this wrapper in pruner's all wrappers + Returns + ------- + dict + dictionary for storing masks, keys of the dict: + 'weight_mask': weight mask tensor + 'bias_mask': bias mask tensor (optional) + """ + msg = 'module type {} is not supported!'.format(wrapper.type) + assert wrapper.type in ['Conv2d', 'Linear'], msg + weight = wrapper.module.weight.data + bias = None + if hasattr(wrapper.module, 'bias') and wrapper.module.bias is not None: + bias = wrapper.module.bias.data + + if wrapper.weight_mask is None: + mask_weight = torch.ones(weight.size()).type_as(weight).detach() + else: + mask_weight = wrapper.weight_mask.clone() + if bias is not None: + if wrapper.bias_mask is None: + mask_bias = torch.ones(bias.size()).type_as(bias).detach() + else: + mask_bias = wrapper.bias_mask.clone() + else: + mask_bias = None + mask = {'weight_mask': mask_weight, 'bias_mask': mask_bias} + + num_total = weight.size(1) + num_prune = int(num_total * sparsity) + if self.preserve_round > 1: + num_preserve = num_total - num_prune + num_preserve = int( + math.ceil(num_preserve * 1. / self.preserve_round) * self.preserve_round) + if num_preserve > num_total: + num_preserve = num_total + num_prune = num_total - num_preserve + + if (num_total < 2 or num_prune < 1) and preserve_idx is None: + return mask + + return self.get_mask(mask, weight, num_preserve, wrapper, wrapper_idx, preserve_idx) + + def get_mask(self, base_mask, weight, num_preserve, wrapper, wrapper_idx, preserve_idx): + w = weight.data.cpu().numpy() + if wrapper.type == 'Linear': + w = w[:, :, None, None] + + if preserve_idx is None: + importance = np.abs(w).sum((0, 2, 3)) + # sum magnitude along C_in, sort descend + sorted_idx = np.argsort(-importance) + d_prime = num_preserve + preserve_idx = sorted_idx[:d_prime] # to preserve index + else: + d_prime = len(preserve_idx) + + assert len(preserve_idx) == d_prime + mask = np.zeros(w.shape[1], bool) + mask[preserve_idx] = True + + # reconstruct, X, Y <= [N, C] + X, Y = wrapper.input_feat, wrapper.output_feat + masked_X = X[:, mask] + if w.shape[2] == 1: # 1x1 conv or fc + rec_weight = least_square_sklearn(X=masked_X, Y=Y) + rec_weight = rec_weight.reshape(-1, 1, 1, d_prime) # (C_out, K_h, K_w, C_in') + rec_weight = np.transpose(rec_weight, (0, 3, 1, 2)) # (C_out, C_in', K_h, K_w) + + rec_weight_pad = np.zeros_like(w) + # pylint: disable=all + rec_weight_pad[:, mask, :, :] = rec_weight + rec_weight = rec_weight_pad + + if wrapper.type == 'Linear': + rec_weight = rec_weight.squeeze() + assert len(rec_weight.shape) == 2 + + # now assign + wrapper.module.weight.data = torch.from_numpy(rec_weight).to(weight.device) + + mask_weight = torch.zeros_like(weight) + if wrapper.type == 'Linear': + mask_weight[:, preserve_idx] = 1. + if base_mask['bias_mask'] is not None and wrapper.module.bias is not None: + mask_bias = torch.ones_like(wrapper.module.bias) + else: + mask_weight[:, preserve_idx, :, :] = 1. + mask_bias = None + + return {'weight_mask': mask_weight.detach(), 'bias_mask': mask_bias} + + +# implemented by queyu, 2021/6/10 +class HRankPrunerMasker(ActivationFilterPrunerMasker): + + def get_mask(self, base_mask, weight, num_prune, wrapper, wrapper_idx, channel_masks=None): + if isinstance(wrapper.module, torch.nn.Conv2d): + apoz = self.get_channel_sum(wrapper, wrapper_idx) + if apoz is None: + # the collected activations are not enough + return None + if channel_masks is not None: + apoz = apoz * channel_masks + + prune_indices = torch.argsort(apoz)[:num_prune] + for idx in prune_indices: + base_mask['weight_mask'][idx] = 0. + if base_mask['bias_mask'] is not None: + base_mask['bias_mask'][idx] = 0. + + if self.pruner.hook_id in self.pruner._fwd_hook_handles: + self.pruner.remove_activation_collector(self.pruner.hook_id) + + return base_mask + + elif isinstance(wrapper.module, torch.nn.ConvTranspose2d): + apoz = self.get_channel_sum(wrapper, wrapper_idx) + if apoz is None: + # the collected activations are not enough + return None + if channel_masks is not None: + apoz = apoz * channel_masks + + prune_indices = torch.argsort(apoz)[:num_prune] + for idx in prune_indices: + base_mask['weight_mask'][:, idx] = 0. + if base_mask['bias_mask'] is not None: + base_mask['bias_mask'][idx] = 0. + + if self.pruner.hook_id in self.pruner._fwd_hook_handles: + self.pruner.remove_activation_collector(self.pruner.hook_id) + + return base_mask + + def _calc_hrank(self, activations, wrapper): + if isinstance(wrapper.module, torch.nn.Conv2d): + activations = torch.cat(activations, 0) + hrank_stats = torch.zeros((activations.size()[0], activations.size()[1])) + + for activation_index in range(activations.size()[0]): + activation = activations[activation_index] + + for feature_map_index in range(activation.size()[0]): + feature_map = activation[feature_map_index] + + rank = torch.matrix_rank(feature_map) + hrank_stats[activation_index][feature_map_index] = rank + + # print(hrank_stats[0], hrank_stats[1], hrank_stats[2]) + hrank_stats = torch.mean(hrank_stats, dim=0) + # print(hrank_stats) + # raise NotImplementedError() + return hrank_stats + + elif isinstance(wrapper.module, torch.nn.ConvTranspose2d): + activations = torch.cat(activations, 0) + hrank_stats = torch.zeros((activations.size()[0], activations.size()[1])) + + for activation_index in range(activations.size()[0]): + activation = activations[activation_index] + + for feature_map_index in range(activation.size()[0]): + feature_map = activation[feature_map_index] + + rank = torch.matrix_rank(feature_map) + hrank_stats[activation_index][feature_map_index] = rank + + # print(hrank_stats[0], hrank_stats[1], hrank_stats[2]) + hrank_stats = torch.mean(hrank_stats, dim=0) + # print(hrank_stats) + # raise NotImplementedError() + return hrank_stats + + def get_channel_sum(self, wrapper, wrapper_idx): + assert wrapper_idx is not None + activations = self.pruner.collected_activation[wrapper_idx] + if len(activations) < self.statistics_batch_num: + # collected activations is not enough + return None + return self._calc_hrank(activations, wrapper).to(wrapper.module.weight.device) + + +# implemented by queyu, 2021/6/10 +class PFPMasker(ActivationFilterPrunerMasker): + + def get_mask(self, base_mask, weight, num_prune, wrapper, wrapper_idx, channel_masks=None): + if isinstance(wrapper.module, torch.nn.Conv2d): + apoz = self.get_channel_sum(wrapper, wrapper_idx) + if apoz is None: + # the collected activations are not enough + return None + if channel_masks is not None: + apoz = apoz * channel_masks + + prune_indices = torch.argsort(apoz)[:num_prune] + for idx in prune_indices: + base_mask['weight_mask'][idx] = 0. + if base_mask['bias_mask'] is not None: + base_mask['bias_mask'][idx] = 0. + + if self.pruner.hook_id in self.pruner._fwd_hook_handles: + self.pruner.remove_activation_collector(self.pruner.hook_id) + + return base_mask + elif isinstance(wrapper.module, torch.nn.ConvTranspose2d): + apoz = self.get_channel_sum(wrapper, wrapper_idx) + if apoz is None: + # the collected activations are not enough + return None + if channel_masks is not None: + apoz = apoz * channel_masks + + prune_indices = torch.argsort(apoz)[:num_prune] + for idx in prune_indices: + base_mask['weight_mask'][:, idx] = 0. + if base_mask['bias_mask'] is not None: + base_mask['bias_mask'][idx] = 0. + + if self.pruner.hook_id in self.pruner._fwd_hook_handles: + self.pruner.remove_activation_collector(self.pruner.hook_id) + + return base_mask + + def _calc_filter_sen(self, activations, wrapper): + if isinstance(wrapper.module, torch.nn.Conv2d): + activations = torch.cat(activations, 0) + + # find next conv layer + cur_layer_name = '' + self.pruner._unwrap_model() + for name, module in self.model.named_modules(): + if module == wrapper.module: + cur_layer_name = name + break + if cur_layer_name == '': + raise NotImplementedError() + + print(cur_layer_name) + next_conv_layer = None + while True: + successors = self.pruner.graph.find_successors(cur_layer_name) + if len(successors) == 0: + break + + next_layer_name = successors[0] + next_layer = get_module(self.model, next_layer_name) + if isinstance(next_layer, torch.nn.Conv2d): + next_conv_layer = next_layer + break + + cur_layer_name = next_layer_name + + self.pruner._wrap_model() + + next_conv_layer_weight = next_conv_layer.weight.data + unfold_next_conv_layer_weight = next_conv_layer_weight.view((next_conv_layer_weight.size()[0], -1)) + unfold_activations = torch.nn.functional.unfold( + activations, + kernel_size=next_conv_layer.kernel_size, + stride=next_conv_layer.stride, + padding=next_conv_layer.padding, + dilation=next_conv_layer.dilation + ).to(next_conv_layer_weight.device) + + res = torch.zeros((activations.size()[0], next_conv_layer_weight.size()[1])) + + + print(unfold_next_conv_layer_weight.size(), unfold_activations.size()) + + for i in range(unfold_next_conv_layer_weight.size()[0]): + t = unfold_next_conv_layer_weight[i].unsqueeze(0).unsqueeze(-1) * unfold_activations + t = torch.max(t, dim=-1)[0] + t = t.view((activations.size()[0], next_conv_layer_weight.size()[1], -1)) + t = torch.max(t, dim=-1)[0] + t = t.cpu() + res = torch.max(res, t) + + res = torch.max(res, dim=0)[0] + return res + + elif isinstance(wrapper.module, torch.nn.ConvTranspose2d): + activations = torch.cat(activations, 0) + + # find next conv layer + cur_layer_name = '' + self.pruner._unwrap_model() + for name, module in self.model.named_modules(): + if module == wrapper.module: + cur_layer_name = name + break + if cur_layer_name == '': + raise NotImplementedError() + + print(cur_layer_name) + next_conv_layer = None + while True: + successors = self.pruner.graph.find_successors(cur_layer_name) + if len(successors) == 0: + break + + next_layer_name = successors[0] + next_layer = get_module(self.model, next_layer_name) + if isinstance(next_layer, torch.nn.Conv2d): + next_conv_layer = next_layer + break + + cur_layer_name = next_layer_name + + self.pruner._wrap_model() + + next_conv_layer_weight = next_conv_layer.weight.data + unfold_next_conv_layer_weight = next_conv_layer_weight.view((next_conv_layer_weight.size()[0], -1)) + unfold_activations = torch.nn.functional.unfold( + activations, + kernel_size=next_conv_layer.kernel_size, + stride=next_conv_layer.stride, + padding=next_conv_layer.padding, + dilation=next_conv_layer.dilation + ).to(next_conv_layer_weight.device) + + # res = torch.zeros((activations.size()[0], next_conv_layer_weight.size()[1])) + + # for i in range(unfold_next_conv_layer_weight.size()[0]): + # t = unfold_next_conv_layer_weight[i].unsqueeze(0).unsqueeze(-1) * unfold_activations + # t = torch.max(t, dim=-1)[0] + # t = t.view((activations.size()[0], next_conv_layer_weight.size()[1], -1)) + # t = torch.max(t, dim=-1)[0] + # t = t.cpu() + # res = torch.max(res, t) + + # res = torch.max(res, dim=0)[0] + + res = torch.zeros((activations.size()[0], next_conv_layer_weight.size()[0])) + + for i in range(unfold_next_conv_layer_weight.size()[1]): + print(unfold_next_conv_layer_weight.size(), unfold_activations.size()) + t = unfold_next_conv_layer_weight[:, i].unsqueeze(1).unsqueeze(-1) * unfold_activations + t = torch.max(t, dim=-1)[0] + t = t.view((activations.size()[0], next_conv_layer_weight.size()[0], -1)) + t = torch.max(t, dim=-1)[0] + t = t.cpu() + res = torch.max(res, t) + + res = torch.max(res, dim=0)[0] + return res + + def get_channel_sum(self, wrapper, wrapper_idx): + assert wrapper_idx is not None + activations = self.pruner.collected_activation[wrapper_idx] + if len(activations) < self.statistics_batch_num: + # collected activations is not enough + return None + return self._calc_filter_sen(activations, wrapper).to(wrapper.module.weight.device) \ No newline at end of file diff --git a/new_impl/cv/third_party/nni/algorithms/compression/pytorch/pruning/weight_masker.py b/new_impl/cv/third_party/nni/algorithms/compression/pytorch/pruning/weight_masker.py new file mode 100644 index 0000000000000000000000000000000000000000..aec8444ced37bccfa5f05e0a9f81b2843c2a35e4 --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/compression/pytorch/pruning/weight_masker.py @@ -0,0 +1,28 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +class WeightMasker(object): + def __init__(self, model, pruner, **kwargs): + self.model = model + self.pruner = pruner + + def calc_mask(self, sparsity, wrapper, wrapper_idx=None): + """ + Calculate the mask of given layer. + Parameters + ---------- + sparsity: float + pruning ratio, preserved weight ratio is `1 - sparsity` + wrapper: PrunerModuleWrapper + layer wrapper of this layer + wrapper_idx: int + index of this wrapper in pruner's all wrappers + Returns + ------- + dict + dictionary for storing masks, keys of the dict: + 'weight_mask': weight mask tensor + 'bias_mask': bias mask tensor (optional) + """ + + raise NotImplementedError('{} calc_mask is not implemented'.format(self.__class__.__name__)) diff --git a/new_impl/cv/third_party/nni/algorithms/compression/pytorch/quantization/__init__.py b/new_impl/cv/third_party/nni/algorithms/compression/pytorch/quantization/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..0e728dcd3f0817a529eb6801f3ab935de6751348 --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/compression/pytorch/quantization/__init__.py @@ -0,0 +1,4 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .quantizers import * diff --git a/new_impl/cv/third_party/nni/algorithms/compression/pytorch/quantization/quantizers.py b/new_impl/cv/third_party/nni/algorithms/compression/pytorch/quantization/quantizers.py new file mode 100644 index 0000000000000000000000000000000000000000..689b3c56b2262512f32b4018d40fe6b9fb0f0ed6 --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/compression/pytorch/quantization/quantizers.py @@ -0,0 +1,382 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +import copy +import torch +from schema import Schema, And, Or, Optional +from nni.compression.pytorch.utils.config_validation import CompressorSchema +from nni.compression.pytorch.compressor import Quantizer, QuantGrad, QuantType + +__all__ = ['NaiveQuantizer', 'QAT_Quantizer', 'DoReFaQuantizer', 'BNNQuantizer'] + +logger = logging.getLogger(__name__) + + +class NaiveQuantizer(Quantizer): + """quantize weight to 8 bits + """ + + def __init__(self, model, config_list, optimizer=None): + super().__init__(model, config_list, optimizer) + self.layer_scale = {} + + def validate_config(self, model, config_list): + schema = CompressorSchema([{ + Optional('quant_types'): ['weight'], + Optional('quant_bits'): Or(8, {'weight': 8}), + Optional('op_types'): [str], + Optional('op_names'): [str] + }], model, logger) + + schema.validate(config_list) + + def quantize_weight(self, wrapper, **kwargs): + weight = copy.deepcopy(wrapper.module.old_weight.data) + new_scale = weight.abs().max() / 127 + scale = max(self.layer_scale.get(wrapper.name, 0), new_scale) + self.layer_scale[wrapper.name] = scale + orig_type = weight.type() # TODO: user layer + weight = weight.div(scale).type(torch.int8).type(orig_type).mul(scale) + wrapper.module.weight = weight + return weight + +def update_ema(biased_ema, value, decay, step): + """ + calculate biased stat and unbiased stat in each step using exponential moving average method + + Parameters + ---------- + biased_ema : float + previous stat value + value : float + current stat value + decay : float + the weight of previous stat value, larger means smoother curve + step : int + current step + + Returns + ------- + float, float + """ + biased_ema = biased_ema * decay + (1 - decay) * value + unbiased_ema = biased_ema / (1 - decay ** step) # Bias correction + return biased_ema, unbiased_ema + + +def update_quantization_param(bits, rmin, rmax): + """ + calculate the `zero_point` and `scale`. + + Parameters + ---------- + bits : int + quantization bits length + rmin : float + min value of real value + rmax : float + max value of real value + + Returns + ------- + float, float + """ + # extend the [min, max] interval to ensure that it contains 0. + # Otherwise, we would not meet the requirement that 0 be an exactly + # representable value. + rmin = min(rmin, 0) + rmax = max(rmax, 0) + + # the min and max quantized values, as floating-point values + qmin = 0 + qmax = (1 << bits) - 1 + # First determine the scale. + scale = (rmax - rmin) / (qmax - qmin) + + # Zero-point computation. + initial_zero_point = qmin - rmin / scale + + # Now we need to nudge the zero point to be an integer + nudged_zero_point = 0 + if initial_zero_point < qmin: + nudged_zero_point = qmin + elif initial_zero_point > qmax: + nudged_zero_point = qmax + else: + nudged_zero_point = torch.round(initial_zero_point) + + return scale, nudged_zero_point + + +def get_bits_length(config, quant_type): + if isinstance(config["quant_bits"], int): + return config["quant_bits"] + else: + return config["quant_bits"].get(quant_type) + + +class QAT_Quantizer(Quantizer): + """Quantizer defined in: + Quantization and Training of Neural Networks for Efficient Integer-Arithmetic-Only Inference + http://openaccess.thecvf.com/content_cvpr_2018/papers/Jacob_Quantization_and_Training_CVPR_2018_paper.pdf + """ + + def __init__(self, model, config_list, optimizer=None): + """ + Parameters + ---------- + layer : LayerInfo + the layer to quantize + config_list : list of dict + list of configurations for quantization + supported keys for dict: + - quant_types : list of string + type of quantization you want to apply, currently support 'weight', 'input', 'output' + - quant_bits : int or dict of {str : int} + bits length of quantization, key is the quantization type, value is the length, eg. {'weight', 8}, + when the type is int, all quantization types share same bits length + - quant_start_step : int + disable quantization until model are run by certain number of steps, this allows the network to enter a more stable + state where activation quantization ranges do not exclude a significant fraction of values, default value is 0 + - op_types : list of string + types of nn.module you want to apply quantization, eg. 'Conv2d' + """ + super().__init__(model, config_list, optimizer) + self.steps = 1 + modules_to_compress = self.get_modules_to_compress() + for layer, config in modules_to_compress: + layer.module.register_buffer("zero_point", None) + layer.module.register_buffer("scale", None) + if "output" in config.get("quant_types", []): + layer.module.register_buffer('ema_decay', torch.Tensor([0.99])) + layer.module.register_buffer('tracked_min_biased', torch.zeros(1)) + layer.module.register_buffer('tracked_min', torch.zeros(1)) + layer.module.register_buffer('tracked_max_biased', torch.zeros(1)) + layer.module.register_buffer('tracked_max', torch.zeros(1)) + + def validate_config(self, model, config_list): + """ + Parameters + ---------- + model : torch.nn.Module + Model to be pruned + config_list : list of dict + List of configurations + """ + schema = CompressorSchema([{ + Optional('quant_types'): Schema([lambda x: x in ['weight', 'output']]), + Optional('quant_bits'): Or(And(int, lambda n: 0 < n < 32), Schema({ + Optional('weight'): And(int, lambda n: 0 < n < 32), + Optional('output'): And(int, lambda n: 0 < n < 32), + })), + Optional('quant_start_step'): And(int, lambda n: n >= 0), + Optional('op_types'): [str], + Optional('op_names'): [str] + }], model, logger) + + schema.validate(config_list) + + def _quantize(self, bits, op, real_val): + """ + quantize real value. + + Parameters + ---------- + bits : int + quantization bits length + op : torch.nn.Module + target module + real_val : float + real value to be quantized + + Returns + ------- + float + """ + transformed_val = op.zero_point + real_val / op.scale + qmin = 0 + qmax = (1 << bits) - 1 + clamped_val = torch.clamp(transformed_val, qmin, qmax) + quantized_val = torch.round(clamped_val) + return quantized_val + + def _dequantize(self, op, quantized_val): + """ + dequantize quantized value. + Because we simulate quantization in training process, all the computations still happen as float point computations, which means we + first quantize tensors then dequantize them. For more details, please refer to the paper. + + Parameters + ---------- + op : torch.nn.Module + target module + quantized_val : float + quantized_val value to be dequantized + + Returns + ------- + float + """ + real_val = op.scale * (quantized_val - op.zero_point) + return real_val + + def quantize_weight(self, wrapper, **kwargs): + config = wrapper.config + module = wrapper.module + weight = copy.deepcopy(wrapper.module.old_weight.data) + weight_bits = get_bits_length(config, 'weight') + quant_start_step = config.get('quant_start_step', 0) + assert weight_bits >= 1, "quant bits length should be at least 1" + + if quant_start_step > self.steps: + return weight + + # if bias exists, quantize bias to uint32 + if hasattr(wrapper.module, 'bias') and wrapper.module.bias is not None: + bias = wrapper.module.bias.data + bias_bits = 32 + rmin, rmax = torch.min(bias), torch.max(bias) + module.scale, module.zero_point = update_quantization_param(bias_bits, rmin, rmax) + bias = self._quantize(bias_bits, module, bias) + bias = self._dequantize(module, bias) + wrapper.module.bias.data = bias + + + # quantize weight + rmin, rmax = torch.min(weight), torch.max(weight) + module.scale, module.zero_point = update_quantization_param(weight_bits, rmin, rmax) + weight = self._quantize(weight_bits, module, weight) + weight = self._dequantize(module, weight) + wrapper.module.weight = weight + return weight + + def quantize_output(self, output, wrapper, **kwargs): + config = wrapper.config + module = wrapper.module + output_bits = get_bits_length(config, 'output') + quant_start_step = config.get('quant_start_step', 0) + assert output_bits >= 1, "quant bits length should be at least 1" + + if quant_start_step > self.steps: + return output + + current_min, current_max = torch.min(output), torch.max(output) + module.tracked_min_biased, module.tracked_min = update_ema(module.tracked_min_biased, current_min, + module.ema_decay, self.steps) + module.tracked_max_biased, module.tracked_max = update_ema(module.tracked_max_biased, current_max, + module.ema_decay, self.steps) + module.scale, module.zero_point = update_quantization_param(output_bits, module.tracked_min, module.tracked_max) + out = self._quantize(output_bits, module, output) + out = self._dequantize(module, out) + return out + + def fold_bn(self, config, **kwargs): + # TODO simulate folded weight + pass + + def step_with_optimizer(self): + """ + override `compressor` `step` method, quantization only happens after certain number of steps + """ + self.steps += 1 + + +class DoReFaQuantizer(Quantizer): + """Quantizer using the DoReFa scheme, as defined in: + Zhou et al., DoReFa-Net: Training Low Bitwidth Convolutional Neural Networks with Low Bitwidth Gradients + (https://arxiv.org/abs/1606.06160) + """ + + def __init__(self, model, config_list, optimizer=None): + super().__init__(model, config_list, optimizer) + + def validate_config(self, model, config_list): + """ + Parameters + ---------- + model : torch.nn.Module + Model to be pruned + config_list : list of dict + List of configurations + """ + schema = CompressorSchema([{ + Optional('quant_types'): Schema([lambda x: x in ['weight']]), + Optional('quant_bits'): Or(And(int, lambda n: 0 < n < 32), Schema({ + Optional('weight'): And(int, lambda n: 0 < n < 32) + })), + Optional('op_types'): [str], + Optional('op_names'): [str] + }], model, logger) + + schema.validate(config_list) + + def quantize_weight(self, wrapper, **kwargs): + weight = copy.deepcopy(wrapper.module.old_weight.data) + weight_bits = get_bits_length(wrapper.config, 'weight') + weight = weight.tanh() + weight = weight / (2 * weight.abs().max()) + 0.5 + weight = self.quantize(weight, weight_bits) + weight = 2 * weight - 1 + wrapper.module.weight = weight + # wrapper.module.weight.data = weight + return weight + + def quantize(self, input_ri, q_bits): + scale = pow(2, q_bits) - 1 + output = torch.round(input_ri * scale) / scale + return output + + +class ClipGrad(QuantGrad): + @staticmethod + def quant_backward(tensor, grad_output, quant_type): + if quant_type == QuantType.QUANT_OUTPUT: + grad_output[torch.abs(tensor) > 1] = 0 + return grad_output + + +class BNNQuantizer(Quantizer): + """Binarized Neural Networks, as defined in: + Binarized Neural Networks: Training Deep Neural Networks with Weights and Activations Constrained to +1 or -1 + (https://arxiv.org/abs/1602.02830) + """ + + def __init__(self, model, config_list, optimizer=None): + super().__init__(model, config_list, optimizer) + self.quant_grad = ClipGrad + + def validate_config(self, model, config_list): + """ + Parameters + ---------- + model : torch.nn.Module + Model to be pruned + config_list : list of dict + List of configurations + """ + schema = CompressorSchema([{ + Optional('quant_types'): Schema([lambda x: x in ['weight', 'output']]), + Optional('quant_bits'): Or(And(int, lambda n: 0 < n < 32), Schema({ + Optional('weight'): And(int, lambda n: 0 < n < 32), + Optional('output'): And(int, lambda n: 0 < n < 32), + })), + Optional('op_types'): [str], + Optional('op_names'): [str] + }], model, logger) + + schema.validate(config_list) + + def quantize_weight(self, wrapper, **kwargs): + weight = copy.deepcopy(wrapper.module.old_weight.data) + weight = torch.sign(weight) + # remove zeros + weight[weight == 0] = 1 + wrapper.module.weight = weight + return weight + + def quantize_output(self, output, wrapper, **kwargs): + out = torch.sign(output) + # remove zeros + out[out == 0] = 1 + return out diff --git a/new_impl/cv/third_party/nni/algorithms/compression/tensorflow/__init__.py b/new_impl/cv/third_party/nni/algorithms/compression/tensorflow/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/new_impl/cv/third_party/nni/algorithms/compression/tensorflow/pruning/__init__.py b/new_impl/cv/third_party/nni/algorithms/compression/tensorflow/pruning/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..f8ac8ea9b95a732cb581f24c7eebd8ba5739a3c6 --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/compression/tensorflow/pruning/__init__.py @@ -0,0 +1 @@ +from .one_shot import * diff --git a/new_impl/cv/third_party/nni/algorithms/compression/tensorflow/pruning/one_shot.py b/new_impl/cv/third_party/nni/algorithms/compression/tensorflow/pruning/one_shot.py new file mode 100644 index 0000000000000000000000000000000000000000..76dc1915ce1609dd083738f09a4973b66a63c47f --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/compression/tensorflow/pruning/one_shot.py @@ -0,0 +1,71 @@ +import tensorflow as tf + +from nni.compression.tensorflow import Pruner + +__all__ = [ + 'OneshotPruner', + 'LevelPruner', +] + +class OneshotPruner(Pruner): + def __init__(self, model, config_list, pruning_algorithm='level', **algo_kwargs): + super().__init__(model, config_list) + self.set_wrappers_attribute('calculated', False) + self.masker = MASKER_DICT[pruning_algorithm](model, self, **algo_kwargs) + + def validate_config(self, model, config_list): + pass # TODO + + def calc_masks(self, wrapper, wrapper_idx=None): + if wrapper.calculated: + return None + sparsity = wrapper.config['sparsity'] + masks = self.masker.calc_masks(sparsity, wrapper, wrapper_idx) + if masks is not None: + wrapper.calculated = True + return masks + + +class LevelPruner(OneshotPruner): + def __init__(self, model, config_list): + super().__init__(model, config_list, pruning_algorithm='level') + + +class WeightMasker: + def __init__(self, model, pruner, **kwargs): + self.model = model + self.pruner = pruner + + def calc_masks(self, sparsity, wrapper, wrapper_idx=None): + raise NotImplementedError() + + +class LevelPrunerMasker(WeightMasker): + def calc_masks(self, sparsity, wrapper, wrapper_idx=None): + masks = {} + for weight_variable in wrapper.layer.weights: + if 'bias' in weight_variable.name: + continue + + num_prune = int(tf.size(weight_variable).numpy() * sparsity) + if num_prune == 0: + continue + + weight = weight_variable.read_value() + if wrapper.masks.get(weight_variable.name) is not None: + weight = tf.math.multiply(weight, wrapper.masks[weight_variable.name]) + + w_abs = tf.math.abs(weight) + k = tf.size(weight) - num_prune + topk = tf.math.top_k(tf.reshape(w_abs, [-1]), k)[0] + if tf.size(topk) == 0: + mask = tf.zeros_like(weight) + else: + mask = tf.math.greater_equal(w_abs, topk[-1]) + masks[weight_variable.name] = tf.cast(mask, weight.dtype) + return masks + + +MASKER_DICT = { + 'level': LevelPrunerMasker, +} diff --git a/new_impl/cv/third_party/nni/algorithms/feature_engineering/__init__.py b/new_impl/cv/third_party/nni/algorithms/feature_engineering/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/new_impl/cv/third_party/nni/algorithms/feature_engineering/gbdt_selector/__init__.py b/new_impl/cv/third_party/nni/algorithms/feature_engineering/gbdt_selector/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..72970ab8565e11e62bae1242a4a8a00c06993cf7 --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/feature_engineering/gbdt_selector/__init__.py @@ -0,0 +1 @@ +from .gbdt_selector import GBDTSelector \ No newline at end of file diff --git a/new_impl/cv/third_party/nni/algorithms/feature_engineering/gbdt_selector/gbdt_selector.py b/new_impl/cv/third_party/nni/algorithms/feature_engineering/gbdt_selector/gbdt_selector.py new file mode 100644 index 0000000000000000000000000000000000000000..9ee09c25a38f1dca27fb4ab068bbceeaec69a3bd --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/feature_engineering/gbdt_selector/gbdt_selector.py @@ -0,0 +1,115 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# MIT License +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and +# associated documentation files (the "Software"), to deal in the Software without restriction, +# including without limitation the rights to use, copy, modify, merge, publish, distribute, +# sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or +# substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT +# NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT +# OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# ================================================================================================== + +""" +gbdt_selector.py including: + class GBDTSelector +""" + +import random +from sklearn.model_selection import train_test_split + +# pylint: disable=E0401 +import lightgbm as lgb + +from nni.feature_engineering.feature_selector import FeatureSelector + + +class GBDTSelector(FeatureSelector): + + def __init__(self, **kwargs): + self.selected_features_ = None + self.X = None + self.y = None + self.feature_importance = None + self.lgb_params = None + self.eval_ratio = None + self.early_stopping_rounds = None + self.importance_type = None + self.num_boost_round = None + self.model = None + + + def fit(self, X, y, **kwargs): + """ + Fit the training data to FeatureSelector + + Paramters + --------- + X : array-like numpy matrix + The training input samples, which shape is [n_samples, n_features]. + y : array-like numpy matrix + The target values (class labels in classification, real numbers in + regression). Which shape is [n_samples]. + lgb_params : dict + Parameters of lightgbm + eval_ratio : float + The ratio of data size. It's used for split the eval data and train data from self.X. + early_stopping_rounds : int + The early stopping setting in lightgbm. + importance_type : str + Supporting type is 'gain' or 'split'. + num_boost_round : int + num_boost_round in lightgbm. + """ + assert kwargs['lgb_params'] + assert kwargs['eval_ratio'] + assert kwargs['early_stopping_rounds'] + assert kwargs['importance_type'] + assert kwargs['num_boost_round'] + + self.X = X + self.y = y + self.lgb_params = kwargs['lgb_params'] + self.eval_ratio = kwargs['eval_ratio'] + self.early_stopping_rounds = kwargs['early_stopping_rounds'] + self.importance_type = kwargs['importance_type'] + self.num_boost_round = kwargs['num_boost_round'] + + X_train, X_test, y_train, y_test = train_test_split(self.X, + self.y, + test_size=self.eval_ratio, + random_state=random.seed(41)) + lgb_train = lgb.Dataset(X_train, y_train) + lgb_eval = lgb.Dataset(X_test, y_test, reference=lgb_train) + + self.model = lgb.train(self.lgb_params, + lgb_train, + num_boost_round=self.num_boost_round, + valid_sets=lgb_eval, + early_stopping_rounds=self.early_stopping_rounds) + + self.feature_importance = self.model.feature_importance(self.importance_type) + + + def get_selected_features(self, topk): + """ + Fit the training data to FeatureSelector + + Returns + ------- + list : + Return the index of imprtant feature. + """ + assert topk > 0 + + self.selected_features_ = self.feature_importance.argsort()[-topk:][::-1] + + return self.selected_features_ diff --git a/new_impl/cv/third_party/nni/algorithms/feature_engineering/gbdt_selector/requirements.txt b/new_impl/cv/third_party/nni/algorithms/feature_engineering/gbdt_selector/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..b73c05e89d156249906203b67c2e9f786d613738 --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/feature_engineering/gbdt_selector/requirements.txt @@ -0,0 +1 @@ +lightgbm \ No newline at end of file diff --git a/new_impl/cv/third_party/nni/algorithms/feature_engineering/gradient_selector/__init__.py b/new_impl/cv/third_party/nni/algorithms/feature_engineering/gradient_selector/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a43cb7578dcb38cbccc33fa11ab6bafc0a71b1fd --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/feature_engineering/gradient_selector/__init__.py @@ -0,0 +1 @@ +from .gradient_selector import FeatureGradientSelector \ No newline at end of file diff --git a/new_impl/cv/third_party/nni/algorithms/feature_engineering/gradient_selector/constants.py b/new_impl/cv/third_party/nni/algorithms/feature_engineering/gradient_selector/constants.py new file mode 100644 index 0000000000000000000000000000000000000000..0f70e2043af51bcf35c7514a81889203a9017ccb --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/feature_engineering/gradient_selector/constants.py @@ -0,0 +1,100 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# MIT License +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and +# associated documentation files (the "Software"), to deal in the Software without restriction, +# including without limitation the rights to use, copy, modify, merge, publish, distribute, +# sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or +# substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT +# NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT +# OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# ================================================================================================== + + +import numpy as np + + +class StorageLevel: + DISK = 'disk' + SPARSE = 'sparse' + DENSE = 'dense' + + +class DataFormat: + SVM = 'svm' + NUMPY = 'numpy' + ALL_FORMATS = [SVM, NUMPY] + + +class Preprocess: + """ + center the data to mean 0 and create unit variance + center the data to mean 0 + """ + ZSCORE = 'zscore' + CENTER = 'center' + + +class Device: + CUDA = 'cuda' + CPU = 'cpu' + + +class Checkpoint: + MODEL = 'model_state_dict' + OPT = 'optimizer_state_dict' + RNG = 'torch_rng_state' + + +class NanError(ValueError): + pass + + +class Initialization: + ZERO = 'zero' + ON = 'on' + OFF = 'off' + ON_HIGH = 'onhigh' + OFF_HIGH = 'offhigh' + SKLEARN = 'sklearn' + RANDOM = 'random' + VALUE_DICT = {ZERO: 0, + ON: 1, + OFF: -1, + ON_HIGH: 5, + OFF_HIGH: -1, + SKLEARN: None, + RANDOM: None} + + +class Coefficients: + """" + coefficients for sublinear estimator were computed running the sublinear + paper's authors' code + """ + SLE = {1: np.array([0.60355337]), + 2: np.array([1.52705001, -0.34841729]), + 3: np.array([2.90254224, -1.87216745, 0.]), + 4: np.array([4.63445685, -5.19936195, 0., 1.50391676]), + 5: np.array([6.92948049, -14.12216211, 9.4475009, 0., -1.21093546]), + 6: np.array([9.54431082, -28.09414643, 31.84703652, -11.18763791, -1.14175281, 0.]), + 7: np.array([12.54505041, -49.64891525, 79.78828031, -46.72250909, 0., 0., 5.02973646]), + 8: np.array([16.03550163, -84.286182, 196.86078756, -215.36747071, 92.63961263, 0., 0., -4.86280869]), + 9: np.array([19.86409184, -130.76801006, 390.95349861, -570.09210416, 354.77764899, 0., -73.84234865, 0., 10.09148767]), + 10: np.array([2.41117752e+01, -1.94946061e+02, 7.34214614e+02, -1.42851995e+03, 1.41567410e+03, \ + -5.81738134e+02, 0., 0., 3.11664751e+01, 1.05018365e+00]), + 11: np.array([28.75280839, -279.22576729, 1280.46325445, -3104.47148101, 3990.6092248, -2300.29413333, \ + 0., 427.35289033, 0., 0., -42.17587475]), + 12: np.array([33.85141912, -391.4229382, 2184.97827882, -6716.28280208, 11879.75233977, -11739.97267239, \ + 5384.94542245, 0., -674.23291712, 0., 0., 39.37456439])} + + +EPSILON = 1e-8 diff --git a/new_impl/cv/third_party/nni/algorithms/feature_engineering/gradient_selector/fginitialize.py b/new_impl/cv/third_party/nni/algorithms/feature_engineering/gradient_selector/fginitialize.py new file mode 100644 index 0000000000000000000000000000000000000000..6fe28ea5ee5494bc63a1dede2867ed3d2ec9f9a6 --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/feature_engineering/gradient_selector/fginitialize.py @@ -0,0 +1,611 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# MIT License +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and +# associated documentation files (the "Software"), to deal in the Software without restriction, +# including without limitation the rights to use, copy, modify, merge, publish, distribute, +# sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or +# substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT +# NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT +# OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# ================================================================================================== + + +import os +import pickle +import sys +import time + +import numpy as np +import scipy.sparse +from sklearn.datasets import load_svmlight_file + +import torch +from torch.utils.data import DataLoader, Dataset +# pylint: disable=E0611 +from torch.utils.data.dataloader import _SingleProcessDataLoaderIter, _MultiProcessingDataLoaderIter, _utils + +from . import constants +from . import syssettings + +torch.set_default_tensor_type(syssettings.torch.tensortype) +sparsetensor = syssettings.torch.sparse.tensortype + +BYTESPERREAL = 8. +BYTESPERGB = 1024. ** 3 + + +class PrepareData(Dataset): + + def __init__(self, + path_data=None, + data_format=constants.DataFormat.NUMPY, + D=None, N=None, + classification=True, + ordinal=False, + balanced=True, + preprocess=None, + n_to_estimate=None, + MAXMEMGB=syssettings.MAXMEMGB, + set_params=True, + path_mappings=None, + X=None, + y=None, + verbose=0, + n_classes=None, + device=constants.Device.CPU): + """ + Dataset class with helpful features and functions for being included in a dataloader + and managing memory usage. + can read following formats: + svm: svm light format (sklearn.datasets.load_svmlight_file) + numpy: Pass X and y as numpy or sparse arrays + + assumes + 1. if classification, y is in {-1, 1} or continuous and 0 indexed + 2. y can fit into memory + 3. consecutive calls to __getitem__() have consecutive idx values + + notes: + 1. this implementation is not careful wrt/ precise memory reqts. for + example, being able to store one dense row in memory is necessary, + but not sufficient. + 2. for y with 4.2 billion elements, 31.3 GB of memory is necessary + @ 8 bytes/scalar. Use partial fit to avoid loading the entire dataset + at once + 3. disk_size always refer to size of complete data file, even after + a split(). + + + Parameters + ---------- + path_data : str + Path to load data from + data_format : str + File ending for path data. + "numpy" is the default when passing in X and y + D : int + Number of features. + N : int + Number of rows. + classification : bool + If True, problem is classification, else regression. + ordinal: bool + If True, problem is ordinal classification. Requires classification to be True. + balanced : bool + If true, each class is weighted equally in optimization, otherwise + weighted is done via support of each class. Requires classification to be True. + prerocess : str + 'zscore' which refers to centering and normalizing data to unit variance or + 'center' which only centers the data to 0 mean + n_to_estimate : int + Number of rows of data to estimate + MAXMEMGB : float + Maximum allowable size for a minibatch + set_params : bool + Whether or not to determine the statistics of the dataset + path_mappings : str + Used when streaming from disk + X : array-like + Shape = [n_samples, n_features] + The training input samples. + y : array-like + Shape = [n_samples] + The target values (class labels in classification, real numbers in + regression). + verbose : int + Controls the verbosity when fitting. Set to 0 for no printing + 1 or higher for printing every verbose number of gradient steps. + device : str + 'cpu' to run on CPU and 'cuda' to run on GPU. Runs much faster on GPU + n_classes : int + number of classes + """ + + self.path_data = path_data + if self.path_data: + self.disk_size = os.path.getsize(path_data) + else: + assert X is not None, 'X must be specified if no path data' + self.disk_size = X.nbytes if not scipy.sparse.issparse( + X) else X.data.nbytes + assert data_format in constants.DataFormat.ALL_FORMATS, 'Format must in {0}.'.format( + ", ".join(constants.DataFormat.ALL_FORMATS)) + self.format = data_format + self.classification = classification + self.ordinal = ordinal + self.balanced = balanced + self.MAXMEMGB = MAXMEMGB + self.preprocess = preprocess + self.set_params = set_params + self.verbose = verbose + self.n_classes = n_classes + self.device = device + + self.path_data_stats = None + + if D is None: + assert self.disk_size / BYTESPERGB <= self.MAXMEMGB, \ + 'Cannot load data into memory. Supply D.' + + if self.format == constants.DataFormat.SVM: + self.X, self.y = load_svmlight_file(path_data) + elif self.format == constants.DataFormat.NUMPY: + assert X is not None, 'X must be specified in numpy mode' + assert y is not None, 'y must be specified in numpy mode' + self.X = X + self.y = y + if self.n_classes is None: + self.n_classes = np.unique(y).shape[0] + elif self.classification: + assert self.n_classes >= np.unique(y).shape[0], \ + 'n_classes given must be greater than or equal to the number of classes in y' + else: + raise NotImplementedError + self.y = torch.as_tensor(self.y, dtype=torch.get_default_dtype()) + + self.N, self.D = self.X.shape + + # assumes X was returned as a sparse array + self.storage_level = (constants.StorageLevel.SPARSE + if scipy.sparse.issparse(self.X) + else constants.StorageLevel.DENSE) + + else: + assert N is not None, 'Supply N.' + self.N, self.D = N, D + + # assume sparse matrix cannot fit into memory + self.storage_level = constants.StorageLevel.DISK + + self.dense_size_gb = self.get_dense_size() + + # check dense size + self.set_dense_X() + + self.max_rows = int(self.MAXMEMGB * BYTESPERGB / BYTESPERREAL / self.D) + assert self.max_rows, \ + 'Cannot fit one dense row into %d GB memory.' % self.MAXMEMGB + self.max_rows = self.max_batch_size() + sys.stdout.flush() + + if n_to_estimate is None: + self.n_to_estimate = self.max_batch_size() + else: + assert n_to_estimate <= self.N, 'n_to_estimate must be <= N.' + self.n_to_estimate = n_to_estimate + + # initialize disk loader + if self.storage_level == constants.StorageLevel.DISK and self.set_params: + if self.format == constants.DataFormat.SVM: + raise NotImplementedError( + 'Please use partial fit to train on datasets that do not fit in memory') + else: + raise NotImplementedError + + # TODO: use a passed-in RNG here + self.ix_statistics = np.random.permutation(self.N)[:self.n_to_estimate] + self.n_features = self.D + if self.set_params: + if self.verbose: + print('Finding data statistics...', end='') + sys.stdout.flush() + Xmn, sv1, Xsd, ymn, ysd = self.compute_data_stats() + self.set_data_stats(Xmn, sv1, Xsd, ymn, ysd) + if self.verbose: + print() + self.set_return_raw(False) + else: + self.set_return_raw(True) + + self.set_return_np(False) + + # this needs to occur after setting preprocessing params + if (self.storage_level == constants.StorageLevel.DISK and + self.format == constants.DataFormat.SVM and self.set_params): + self.loader.batchsize = 1 + + def get_dense_size(self): + return self.N * self.D * BYTESPERREAL / BYTESPERGB + + def set_dense_X(self): + if self.storage_level != constants.StorageLevel.DISK: + if self.dense_size_gb <= self.MAXMEMGB: + if self.storage_level == constants.StorageLevel.SPARSE: + self.X = self.X.toarray() + self.X = torch.as_tensor( + self.X, dtype=torch.get_default_dtype()) + self.storage_level = constants.StorageLevel.DENSE + + def set_return_np(self, boolean): + + self.return_np = boolean + + def set_return_raw(self, boolean): + + self.return_raw = boolean + + def save_data_stats(self, path_data_stats): + """ + Dumps dataset statistics to pickle file. + """ + + data_stats = { + 'Xmn': self.Xmn, + 'sv1': self.sv1, + 'Xsd': self.Xsd, + 'ymn': self.ymn, + 'ysd': self.ysd, + 'ix_statistics': self.ix_statistics, + } + pickle.dump(data_stats, open(path_data_stats, 'wb')) + + def load_data_stats(self, path_data_stats): + + stats = pickle.load(open(path_data_stats, 'rb')) + self.path_data_stats = path_data_stats + + self.set_data_stats(np.asarray(stats['Xmn']), stats['sv1'], + stats['Xsd'], stats['ymn'], stats['ysd']) + + if self.storage_level == constants.StorageLevel.DISK and hasattr( + self, 'path_mappings'): + if 'ix_statistics' in stats: + self.ix_statistics = stats['ix_statistics'] + else: + self.ix_statistics = range(self.N) + + self.set_return_raw(False) + + def reset(self): + """ + Resets the dataloader. Only implemented for disk StorageLevel. + """ + + if self.storage_level == constants.StorageLevel.DENSE: + pass + elif self.storage_level == constants.StorageLevel.SPARSE: + pass + elif self.storage_level == constants.StorageLevel.DISK: + if self.format == constants.DataFormat.SVM: + self.loader.reset() + else: + raise NotImplementedError + + def todense(self): + + assert hasattr(self, 'Xmn'), 'Set preprocess params first.' + assert len(self) <= self.max_batch_size( + ), 'N must be <= max_batch_size().' + + with torch.no_grad(): + dense, _ = self.split(range(len(self))) + Braw = self.return_raw + Bnp = self.return_np + self.set_return_raw(True) + self.set_return_np(True) + dense.X, dense.y = [], [] + + def f_Xy(X, y): + dense.X.append(X) + dense.y.append(y) + self.apply(f_Xy=f_Xy) + dense.X = dense.X[-1] + dense.y = dense.y[-1] + self.set_return_raw(Braw) + self.set_return_np(Bnp) + dense.storage_level = constants.StorageLevel.DENSE + + return dense + + def split(self, ix): + + assert hasattr(self, 'Xmn'), 'Run set_preprocess_params() first.' + + first = type(self)( + self.path_data, + self.format, + self.D, + N=len(ix), + classification=self.classification, + preprocess=self.preprocess, + n_to_estimate=None, + MAXMEMGB=self.MAXMEMGB, + set_params=False) + second = type(self)( + self.path_data, + self.format, + self.D, + N=self.N - len(ix), + classification=self.classification, + preprocess=self.preprocess, + n_to_estimate=None, + MAXMEMGB=self.MAXMEMGB, + set_params=False) + + first.storage_level = self.storage_level + second.storage_level = self.storage_level + + # copy preprocess params + if not self.classification: + first.ymn = self.ymn + second.ymn = self.ymn + first.ysd = self.ysd + second.ysd = self.ysd + + first.Xmn = self.Xmn + second.Xmn = self.Xmn + first.sv1 = self.sv1 + second.sv1 = self.sv1 + + if self.storage_level == constants.StorageLevel.DISK: + if self.format == constants.DataFormat.SVM: + first.Xsd = self.Xsd + second.Xsd = self.Xsd + else: + raise NotImplementedError + + # initialize data structures + if self.storage_level == constants.StorageLevel.DISK: + if self.format == constants.DataFormat.SVM: + raise NotImplementedError + raise NotImplementedError + elif self.storage_level in [constants.StorageLevel.SPARSE, + constants.StorageLevel.DENSE]: + first.X, first.y = self.X[ix], self.y[ix] + ixsec = list(set(range(self.N)).difference(set(ix))) + second.X, second.y = self.X[ixsec], self.y[ixsec] + + return first, second + + @staticmethod + def sparse_std(X, X_mean): + """ + Calculate the column wise standard deviations of a sparse matrix. + """ + X_copy = X.copy() + X_copy.data **= 2 # square non zero elements + E_x_squared = np.array(X_copy.mean(axis=0)).ravel() + Xsd = np.sqrt(E_x_squared - X_mean**2) + return Xsd + + def compute_data_stats(self): + """ + 1. computes/estimates feature means + 2. if preprocess == 'zscore', computes/estimates feature standard devs + 3. if not classification, computes/estimates target mean/standard dev + 4. estimates largest singular value of data matrix + """ + t = time.time() + X, y = self.X[self.ix_statistics], self.y[self.ix_statistics] + preprocess = self.preprocess + classification = self.classification + + Xmn = (X.mean(dim=0) + if not scipy.sparse.issparse(X) + else np.array(X.mean(axis=0)).ravel()) + + if preprocess == constants.Preprocess.ZSCORE: + Xsd = (X.std(dim=0) + if not scipy.sparse.issparse(X) + else PrepareData.sparse_std(X, Xmn)) + Xsd[Xsd == 0] = 1. + else: + Xsd = 1. + + if preprocess is not None and preprocess: + if preprocess == constants.Preprocess.ZSCORE: + Xc = (X - Xmn) / Xsd + else: + Xc = X - Xmn + else: + Xc = X - Xmn + + sv1 = scipy.sparse.linalg.svds(Xc / ( + torch.sqrt(torch.prod(torch.as_tensor(y.size(), dtype=torch.get_default_dtype()))) + if not scipy.sparse.issparse(X) else y.numpy().size), + k=1, + which='LM', + return_singular_vectors=False) + # avoid runaway sv1 + sv1 = np.array([min(np.finfo(np.float32).max, + sv1[0])]) + + if not classification: + ymn = y.mean() + ysd = y.std() + else: + # TODO: set these, for each class? + ymn = 0. + ysd = 1. + if self.verbose: + print(" computing data statistics took: ", time.time() - t) + + return Xmn, sv1, Xsd, ymn, ysd + + + def set_data_stats(self, Xmn, sv1, Xsd=1., ymn=0., ysd=1.): + """ + Saves dataset stats to self to be used for preprocessing. + """ + + self.Xmn = torch.as_tensor( + Xmn, dtype=torch.get_default_dtype()).to(self.device) + self.sv1 = torch.as_tensor( + sv1, dtype=torch.get_default_dtype()).to(self.device) + self.Xsd = torch.as_tensor( + Xsd, dtype=torch.get_default_dtype()).to(self.device) + self.ymn = torch.as_tensor( + ymn, dtype=torch.get_default_dtype()).to(self.device) + self.ysd = torch.as_tensor( + ysd, dtype=torch.get_default_dtype()).to(self.device) + + + def apply_preprocess(self, X, y): + """ + Faster on gpu device, while dataloading takes up a large portion of the time. + """ + + with torch.no_grad(): + if not self.classification: + y = (y.reshape((-1, 1)) - self.ymn) / self.ysd + else: + y = y.reshape((-1, 1)) + X = (X - self.Xmn) / self.sv1 + + if self.preprocess == constants.Preprocess.ZSCORE: + X /= self.Xsd + + return X, y + + + def max_batch_size(self): + """ + Return the maximum batchsize for the dataset. + """ + + return int(np.min([self.max_rows, self.N])) + + + def apply(self, ix_rows=None, ix_cols=None, f_Xy=None): + + if f_Xy is None: + return + + if ix_rows is None: + ix_rows = range(self.N) + + if ix_cols is None: + ix_cols = range(self.n_features) + + f_Xy((self.X[ix_rows, ix_cols] + if not self.storage_level == constants.StorageLevel.SPARSE + else self.X[ix_rows, ix_cols].toarray()), self.y[ix_rows]) + + + def get_dense_data(self, ix_cols=None, ix_rows=None): + + if ix_cols is None: + ix_cols = range(self.n_features) + + X = [np.zeros((0, len(ix_cols)))] + y = [np.zeros((0, 1))] + Bnp = self.return_np + + def f_Xy(Xb, yb, n): + X[-1] = np.concatenate((X[-1], Xb), axis=0) + y[-1] = np.concatenate((y[-1], yb), axis=0) + self.apply(f_Xy=f_Xy, ix_rows=ix_rows, ix_cols=ix_cols) + self.set_return_np(Bnp) + + return X[-1], y[-1] + + + def __len__(self): + + return self.N + + + def getXy(self, idx): + + if self.storage_level == constants.StorageLevel.DENSE: + X, y = self.X[idx], self.y[idx] + elif self.storage_level == constants.StorageLevel.SPARSE: + # assume subset can fit into memory even if whole matrix cant + X, y = self.X[idx].toarray(), self.y[idx] + else: + raise NotImplementedError + + return X, y + + + def __getitem__(self, idx): + + with torch.no_grad(): + X, y = self.getXy(idx) + X = X.toarray() if scipy.sparse.issparse(X) else X + + X = torch.as_tensor( + X, dtype=torch.get_default_dtype()).to(self.device) + y = torch.as_tensor( + y, dtype=torch.get_default_dtype()).to(self.device) + + if not self.return_raw: + X, y = self.apply_preprocess(X, y) + + if self.classification and ( + self.n_classes is None or self.n_classes == 2): + y[y == 0] = -1 + + if self.return_np: + if constants.Device.CPU not in self.device: + X = X.cpu() + y = y.cpu() + X = X.numpy() + y = y.numpy() + return X, y + + return X, y + + +class ChunkDataLoader(DataLoader): + """ + DataLoader class used to more quickly load a batch of indices at once. + """ + + def __iter__(self): + return _ChunkDataLoaderIter(self) + + +class _ChunkDataLoaderIter: + """ + DataLoaderIter class used to more quickly load a batch of indices at once. + """ + def __init__(self, dataloader): + if dataloader.num_workers == 0: + self.iter = _SingleProcessDataLoaderIter(dataloader) + else: + self.iter = _MultiProcessingDataLoaderIter(dataloader) + + def __next__(self): + # only chunk that is edited from base + if self.iter._num_workers == 0: # same-process loading + indices = next(self.iter._sampler_iter) # may raise StopIteration + if len(indices) > 1: + batch = self.iter._dataset[np.array(indices)] + else: + batch = self.iter._collate_fn([self.iter._dataset[i] for i in indices]) + + if self.iter._pin_memory: + batch = _utils.pin_memory.pin_memory_batch(batch) + return batch + else: + return next(self.iter) diff --git a/new_impl/cv/third_party/nni/algorithms/feature_engineering/gradient_selector/fgtrain.py b/new_impl/cv/third_party/nni/algorithms/feature_engineering/gradient_selector/fgtrain.py new file mode 100644 index 0000000000000000000000000000000000000000..377d72691613b93911e2bbc164e3c677ac31f574 --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/feature_engineering/gradient_selector/fgtrain.py @@ -0,0 +1,228 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# MIT License +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and +# associated documentation files (the "Software"), to deal in the Software without restriction, +# including without limitation the rights to use, copy, modify, merge, publish, distribute, +# sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or +# substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT +# NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT +# OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# ================================================================================================== + + +import time + +import numpy as np +import torch +from sklearn.feature_selection import SelectKBest, \ + f_classif, mutual_info_classif, f_regression, mutual_info_regression + +from . import constants +from . import syssettings +from .learnability import Solver +from .utils import EMA + +torch.set_default_tensor_type(syssettings.torch.tensortype) + + +def get_optim_f_stop(maxiter, maxtime, dftol_stop, freltol_stop, + minibatch=True): + """ + Check stopping conditions. + """ + + discount_factor = 1. / 3 + + total_t = [0.] + df_store = [np.nan] + it_store = [0] + relchange_store = [np.nan] + f_ma = EMA(discount_factor=discount_factor) + df_ma = EMA(discount_factor=discount_factor) + + def f_stop(f0, v0, it, t): + + flag_stop = False + + total_t[-1] += t + g = f0.x.grad.clone().cpu().detach() + df = g.abs().max().numpy().squeeze() + v = v0.clone().cpu().detach() + f = v.numpy().squeeze() + + if it >= maxiter: + flag_stop = True + + elif total_t[-1] >= maxtime: + flag_stop = True + + f_ma.update(f) + df_ma.update(df) + rel_change = f_ma.relchange() + + if ((not minibatch) and (df < dftol_stop)) \ + or (minibatch and (df_ma() < dftol_stop)): + flag_stop = True + + if rel_change < freltol_stop: + flag_stop = True + + if not minibatch: + df_store[-1] = df + else: + df_store[-1] = df_ma() + relchange_store[-1] = rel_change + it_store[-1] = it + + return flag_stop + + return f_stop, {'t': total_t, 'it': it_store, 'df': df_store, + 'relchange': relchange_store} + + +def get_init(data_train, init_type='on', rng=np.random.RandomState(0), prev_score=None): + """ + Initialize the 'x' variable with different settings + """ + + D = data_train.n_features + value_off = constants.Initialization.VALUE_DICT[ + constants.Initialization.OFF] + value_on = constants.Initialization.VALUE_DICT[ + constants.Initialization.ON] + + if prev_score is not None: + x0 = prev_score + elif not isinstance(init_type, str): + x0 = value_off * np.ones(D) + x0[init_type] = value_on + elif init_type.startswith(constants.Initialization.RANDOM): + d = int(init_type.replace(constants.Initialization.RANDOM, '')) + x0 = value_off * np.ones(D) + x0[rng.permutation(D)[:d]] = value_on + elif init_type == constants.Initialization.SKLEARN: + B = data_train.return_raw + X, y = data_train.get_dense_data() + data_train.set_return_raw(B) + ix = train_sk_dense(init_type, X, y, data_train.classification) + x0 = value_off * np.ones(D) + x0[ix] = value_on + elif init_type in constants.Initialization.VALUE_DICT: + x0 = constants.Initialization.VALUE_DICT[init_type] * np.ones(D) + else: + raise NotImplementedError( + 'init_type {0} not supported yet'.format(init_type)) + # pylint: disable=E1102 + return torch.tensor(x0.reshape((-1, 1)), + dtype=torch.get_default_dtype()) + + +def get_checkpoint(S, stop_conds, rng=None, get_state=True): + """ + Save the necessary information into a dictionary + """ + + m = {} + m['ninitfeats'] = S.ninitfeats + m['x0'] = S.x0 + x = S.x.clone().cpu().detach() + m['feats'] = np.where(x.numpy() >= 0)[0] + m.update({k: v[0] for k, v in stop_conds.items()}) + if get_state: + m.update({constants.Checkpoint.MODEL: S.state_dict(), + constants.Checkpoint.OPT: S.opt_train.state_dict(), + constants.Checkpoint.RNG: torch.get_rng_state(), + }) + if rng: + m.update({'rng_state': rng.get_state()}) + + return m + + +def _train(data_train, Nminibatch, order, C, rng, lr_train, debug, maxiter, + maxtime, init, dftol_stop, freltol_stop, dn_log, accum_steps, + path_save, shuffle, device=constants.Device.CPU, + verbose=1, + prev_checkpoint=None, + groups=None, + soft_groups=None): + """ + Main training loop. + """ + + t_init = time.time() + + x0 = get_init(data_train, init, rng) + if isinstance(init, str) and init == constants.Initialization.ZERO: + ninitfeats = -1 + else: + ninitfeats = np.where(x0.detach().numpy() > 0)[0].size + + S = Solver(data_train, order, + Nminibatch=Nminibatch, x0=x0, C=C, + ftransform=lambda x: torch.sigmoid(2 * x), + get_train_opt=lambda p: torch.optim.Adam(p, lr_train), + rng=rng, + accum_steps=accum_steps, + shuffle=shuffle, + groups=groups, + soft_groups=soft_groups, + device=device, + verbose=verbose) + S = S.to(device) + + S.ninitfeats = ninitfeats + S.x0 = x0 + + if prev_checkpoint: + S.load_state_dict(prev_checkpoint[constants.Checkpoint.MODEL]) + S.opt_train.load_state_dict(prev_checkpoint[constants.Checkpoint.OPT]) + torch.set_rng_state(prev_checkpoint[constants.Checkpoint.RNG]) + + minibatch = S.Ntrain != S.Nminibatch + + f_stop, stop_conds = get_optim_f_stop(maxiter, maxtime, dftol_stop, + freltol_stop, minibatch=minibatch) + + if debug: + pass + else: + f_callback = None + stop_conds['t'][-1] = time.time() - t_init + + S.train(f_stop=f_stop, f_callback=f_callback) + + return get_checkpoint(S, stop_conds, rng), S + + +def train_sk_dense(ty, X, y, classification): + if classification: + if ty.startswith('skf'): + d = int(ty.replace('skf', '')) + f_sk = f_classif + elif ty.startswith('skmi'): + d = int(ty.replace('skmi', '')) + f_sk = mutual_info_classif + else: + if ty.startswith('skf'): + d = int(ty.replace('skf', '')) + f_sk = f_regression + elif ty.startswith('skmi'): + d = int(ty.replace('skmi', '')) + f_sk = mutual_info_regression + t = time.time() + clf = SelectKBest(f_sk, k=d) + clf.fit_transform(X, y.squeeze()) + ix = np.argsort(-clf.scores_) + ix = ix[np.where(np.invert(np.isnan(clf.scores_[ix])))[0]][:d] + t = time.time() - t + return {'feats': ix, 't': t} diff --git a/new_impl/cv/third_party/nni/algorithms/feature_engineering/gradient_selector/gradient_selector.py b/new_impl/cv/third_party/nni/algorithms/feature_engineering/gradient_selector/gradient_selector.py new file mode 100644 index 0000000000000000000000000000000000000000..f7cb69f627442cb8b749f792cecfa0fea7526e1b --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/feature_engineering/gradient_selector/gradient_selector.py @@ -0,0 +1,631 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# MIT License +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and +# associated documentation files (the "Software"), to deal in the Software without restriction, +# including without limitation the rights to use, copy, modify, merge, publish, distribute, +# sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or +# substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT +# NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT +# OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# ================================================================================================== + +import time + +import numpy as np +import pandas as pd + +from sklearn.base import BaseEstimator +from sklearn.feature_selection import SelectorMixin +from sklearn.utils.validation import check_is_fitted + +import torch + +from nni.feature_engineering.feature_selector import FeatureSelector +from . import constants +from .fginitialize import PrepareData +from .fgtrain import _train + + +class FeatureGradientSelector(FeatureSelector, BaseEstimator, SelectorMixin): + def __init__(self, + order=4, + penalty=1, + n_features=None, + max_features=None, + learning_rate=1e-1, + init='zero', + n_epochs=1, + shuffle=True, + batch_size=1000, + target_batch_size=1000, + max_time=np.inf, + classification=True, + ordinal=False, + balanced=True, + preprocess='zscore', + soft_grouping=False, + verbose=0, + device='cpu'): + """ + FeatureGradientSelector is a class that selects features for a machine + learning model using a gradient based search. + + Parameters + ---------- + order : int + What order of interactions to include. Higher orders + may be more accurate but increase the run time. 12 is the maximum allowed order. + penatly : int + Constant that multiplies the regularization term. + n_features: int + If None, will automatically choose number of features based on search. + Otherwise, number of top features to select. + max_features : int + If not None, will use the 'elbow method' to determine the number of features + with max_features as the upper limit. + learning_rate : float + init : str + How to initialize the vector of scores. 'zero' is the default. + Options: {'zero', 'on', 'off', 'onhigh', 'offhigh', 'sklearn'} + n_epochs : int + number of epochs to run + shuffle : bool + Shuffle "rows" prior to an epoch. + batch_size : int + Nnumber of "rows" to process at a time + target_batch_size : int + Number of "rows" to accumulate gradients over. + Useful when many rows will not fit into memory but are needed for accurate estimation. + classification : bool + If True, problem is classification, else regression. + ordinal : bool + If True, problem is ordinal classification. Requires classification to be True. + balanced : bool + If true, each class is weighted equally in optimization, otherwise + weighted is done via support of each class. Requires classification to be True. + prerocess : str + 'zscore' which refers to centering and normalizing data to unit variance or + 'center' which only centers the data to 0 mean + soft_grouping : bool + if True, groups represent features that come from the same source. + Used to encourage sparsity of groups and features within groups. + verbose : int + Controls the verbosity when fitting. Set to 0 for no printing + 1 or higher for printing every verbose number of gradient steps. + device : str + 'cpu' to run on CPU and 'cuda' to run on GPU. Runs much faster on GPU + """ + assert order <= 12 and order >= 1, 'order must be an integer between 1 and 12, inclusive' + assert n_features is None or max_features is None, \ + 'only specify one of n_features and max_features at a time' + + self.order = order + self.penalty = penalty + self.n_features = n_features + self.max_features = max_features + self.learning_rate = learning_rate + self.init = init + self.n_epochs = n_epochs + self.shuffle = shuffle + self.batch_size = batch_size + self.target_batch_size = target_batch_size + self.max_time = max_time + self.dftol_stop = -1 + self.freltol_stop = -1 + self.classification = classification + self.ordinal = ordinal + self.balanced = balanced + self.preprocess = preprocess + self.soft_grouping = soft_grouping + self.verbose = verbose + self.device = device + + self.model_ = None + self.scores_ = None + self._prev_checkpoint = None + self._data_train = None + + def partial_fit(self, X, y, + n_classes=None, + groups=None): + """ + Select Features via a gradient based search on (X, y) on the given samples. + Can be called repeatedly with different X and y to handle streaming datasets. + + Parameters + ---------- + X : array-like + Shape = [n_samples, n_features] + The training input samples. + y : array-like + Shape = [n_samples] + The target values (class labels in classification, real numbers in + regression). + n_classes : int + Number of classes + Classes across all calls to partial_fit. + Can be obtained by via `np.unique(y_all).shape[0]`, where y_all is the + target vector of the entire dataset. + This argument is expected for the first call to partial_fit, + otherwise will assume all classes are present in the batch of y given. + It will be ignored in the subsequent calls. + Note that y doesn't need to contain all labels in `classes`. + groups : array-like + Optional, shape = [n_features] + Groups of columns that must be selected as a unit + e.g. [0, 0, 1, 2] specifies the first two columns are part of a group. + This argument is expected for the first call to partial_fit, + otherwise will assume all classes are present in the batch of y given. + It will be ignored in the subsequent calls. + """ + try: + self._partial_fit(X, y, n_classes=n_classes, groups=groups) + except constants.NanError: + if hasattr(self, '_prev_checkpoint'): + # if it's already done some batches successfully just ignore it + print('failed fitting this batch, loss was nan') + else: + # if this is the first batch, reset and try with doubles + if self.verbose: + print('Loss was nan, trying with Doubles') + self._reset() + torch.set_default_tensor_type(torch.DoubleTensor) + self._partial_fit(X, y, n_classes=n_classes, groups=groups) + + return self + + def _partial_fit(self, X, y, n_classes=None, groups=None): + """ + Private function for partial_fit to enable trying floats before doubles. + """ + # pass in X and y in chunks + if hasattr(self, '_data_train'): + # just overwrite the X and y from the new chunk but make them tensors + # keep dataset stats from previous + self._data_train.X = X.values if isinstance(X, pd.DataFrame) else X + self._data_train.N, self._data_train.D = self._data_train.X.shape + self._data_train.dense_size_gb = self._data_train.get_dense_size() + self._data_train.set_dense_X() + + self._data_train.y = y.values if isinstance(y, pd.Series) else y + self._data_train.y = torch.as_tensor( + y, dtype=torch.get_default_dtype()) + else: + data_train = self._prepare_data(X, y, n_classes=n_classes) + self._data_train = data_train + + batch_size, _, accum_steps, max_iter = self._set_batch_size( + self._data_train) + + rng = None # not used + debug = 0 # {0,1} print messages and do other stuff? + dn_logs = None # tensorboard logs; only specify if debug=1 + path_save = None # intermediate models saves; only specify if debug=1 + m, solver = _train(self._data_train, + batch_size, + self.order, + self.penalty, + rng, + self.learning_rate, + debug, + max_iter, + self.max_time, + self.init, + self.dftol_stop, + self.freltol_stop, + dn_logs, + accum_steps, + path_save, + self.shuffle, + device=self.device, + verbose=self.verbose, + prev_checkpoint=self._prev_checkpoint if hasattr( + self, '_prev_checkpoint') else None, + groups=groups if not self.soft_grouping else None, + soft_groups=groups if self.soft_grouping else None) + + self._prev_checkpoint = m + self._process_results(m, solver, X, groups=groups) + return self + + def fit(self, X, y, + groups=None): + """ + Select Features via a gradient based search on (X, y). + + Parameters + ---------- + X : array-like + Shape = [n_samples, n_features] + The training input samples. + y : array-like + Shape = [n_samples] + The target values (class labels in classification, real numbers in + regression). + groups : array-like + Optional, shape = [n_features] + Groups of columns that must be selected as a unit + e.g. [0, 0, 1, 2] specifies the first two columns are part of a group. + """ + try: + self._fit(X, y, groups=groups) + except constants.NanError: + if self.verbose: + print('Loss was nan, trying with Doubles') + torch.set_default_tensor_type(torch.DoubleTensor) + self._fit(X, y, groups=groups) + return self + + def get_selected_features(self): + return self.selected_features_ + + def _prepare_data(self, X, y, n_classes=None): + """ + Returns a PrepareData object. + """ + return PrepareData(X=X.values if isinstance(X, pd.DataFrame) else X, + y=y.values if isinstance(y, pd.Series) else y, + data_format=constants.DataFormat.NUMPY, + classification=int(self.classification), + ordinal=self.ordinal, + balanced=self.balanced, + preprocess=self.preprocess, + verbose=self.verbose, + device=self.device, + n_classes=n_classes) + + def _fit(self, X, y, groups=None): + """ + Private function for fit to enable trying floats before doubles. + """ + data_train = self._prepare_data(X, y) + + batch_size, _, accum_steps, max_iter = self._set_batch_size( + data_train) + + rng = None # not used + debug = 0 # {0,1} print messages and log to tensorboard + dn_logs = None # tensorboard logs; only specify if debug=1 + path_save = None # intermediate models saves; only specify if debug=1 + m, solver = _train(data_train, + batch_size, + self.order, + self.penalty, + rng, + self.learning_rate, + debug, + max_iter, + self.max_time, + self.init, + self.dftol_stop, + self.freltol_stop, + dn_logs, + accum_steps, + path_save, + self.shuffle, + device=self.device, + verbose=self.verbose, + groups=groups if not self.soft_grouping else None, + soft_groups=groups if self.soft_grouping else None) + + self._process_results(m, solver, X, groups=groups) + return self + + def _process_torch_scores(self, scores): + """ + Convert scores into flat numpy arrays. + """ + if constants.Device.CUDA in scores.device.type: + scores = scores.cpu() + return scores.numpy().ravel() + + def _set_batch_size(self, data_train): + """ + Ensures that batch_size is less than the number of rows. + """ + batch_size = min(self.batch_size, data_train.N) + target_batch_size = min(max( + self.batch_size, self.target_batch_size), data_train.N) + accum_steps = max(int(np.ceil(target_batch_size / self.batch_size)), 1) + max_iter = self.n_epochs * (data_train.N // batch_size) + return batch_size, target_batch_size, accum_steps, max_iter + + def _process_results(self, m, solver, X, groups=None): + """ + Process the results of a run into something suitable for transform(). + """ + self.scores_ = self._process_torch_scores( + torch.sigmoid(m[constants.Checkpoint.MODEL]['x'] * 2)) + if self.max_features: + self.max_features = min([self.max_features, self.scores_.shape[0]]) + n_features = self._recommend_number_features(solver) + self.set_n_features(n_features, groups=groups) + elif self.n_features: + self.set_n_features(self.n_features, groups=groups) + else: + self.selected_features_ = m['feats'] + + # subtract elapsed time from max_time + self.max_time -= m['t'] + + self.model_ = m + + return self + + def transform(self, X): + """ + Returns selected features from X. + + Paramters + --------- + X: array-like + Shape = [n_samples, n_features] + The training input samples. + """ + + self._get_support_mask() + if self.selected_features_.shape[0] == 0: + raise ValueError( + 'No Features selected, consider lowering the penalty or specifying n_features') + return (X.iloc[:, self.selected_features_] + if isinstance(X, pd.DataFrame) + else X[:, self.selected_features_]) + + def get_support(self, indices=False): + """ + Get a mask, or integer index, of the features selected. + + Parameters + ---------- + indices : bool + Default False + If True, the return value will be an array of integers, rather than a boolean mask. + + Returns + ------- + list : + returns support: An index that selects the retained features from a feature vector. + If indices is False, this is a boolean array of shape [# input features], + in which an element is True iff its corresponding feature is selected for retention. + If indices is True, this is an integer array of shape [# output features] whose values + are indices into the input feature vector. + """ + self._get_support_mask() + if indices: + return self.selected_features_ + + mask = np.zeros_like(self.scores_, dtype=bool) + # pylint: disable=E1137 + mask[self.selected_features_] = True + return mask + + def inverse_transform(self, X): + """ + Returns transformed X to the original number of column. + This operation is lossy and all columns not in the transformed data + will be returned as columns of 0s. + """ + self._get_support_mask() + X_new = np.zeros((X.shape[0], self.scores_.shape[0])) + X_new[self.selected_features_] = X + return X_new + + def get_params(self, deep=True): + """ + Get parameters for this estimator. + """ + params = self.__dict__ + params = {key: val for (key, val) in params.items() + if not key.endswith('_')} + return params + + def set_params(self, **params): + """ + Set the parameters of this estimator. + """ + for param in params: + if hasattr(self, param): + setattr(self, param, params[param]) + return self + + def fit_transform(self, X, y): + """ + Select features and then return X with the selected features. + + Parameters + ---------- + X : array-like + Shape = [n_samples, n_features] + The training input samples. + y : array-like + Shape = [n_samples] + The target values (class labels in classification, real numbers in + regression). + """ + self.fit(X, y) + return self.transform(X) + + def _get_support_mask(self): + """ + Check if it is fitted. + """ + check_is_fitted(self, 'scores_') + + def _generate_scores(self, solver, xsub, ysub, step_size, feature_order): + """ + Generate forward passes to determine the number of features when max_features is set. + """ + scores = [] + for i in np.arange(1, self.max_features + 1, step_size): + # optimization possible since xsub is growing? + i = int(np.ceil(i)) + # pylint: disable=E1102 + score = solver.f_train(torch.tensor(np.ones(i), + dtype=torch.get_default_dtype() + ).unsqueeze(1).to(self.device), + xsub[:, feature_order[:i]], + ysub) + if constants.Device.CUDA in score.device.type: + score = score.cpu() + # score.numpy()[0][0] + scores.append(score) + return scores + + def set_n_features(self, n, groups=None): + """ + Set the number of features to return after fitting. + """ + self._get_support_mask() + self.n_features = n + return self._set_top_features(groups=groups) + + def _set_top_features(self, groups=None): + """ + Set the selected features after a run. + + With groups, ensures that if any member of a group is selected, all members are selected + """ + self._get_support_mask() + assert self.n_features <= self.scores_.shape[0], \ + 'n_features must be less than or equal to the number of columns in X' + # pylint: disable=E1130 + self.selected_features_ = np.argpartition( + self.scores_, -self.n_features)[-self.n_features:] + if groups is not None and not self.soft_grouping: + selected_feature_set = set(self.selected_features_.tolist()) + for _ in np.unique(groups): + group_members = np.where(groups == groups)[0].tolist() + if selected_feature_set.intersection(group_members): + selected_feature_set.update(group_members) + self.selected_features_ = np.array(list(selected_feature_set)) + self.selected_features_ = np.sort(self.selected_features_) + return self + + def set_top_percentile(self, percentile, groups=None): + """ + Set the percentile of features to return after fitting. + """ + self._get_support_mask() + assert percentile <= 1 and percentile >= 0, \ + 'percentile must between 0 and 1 inclusive' + self.n_features = int(self.scores_.shape[0] * percentile) + return self._set_top_features(groups=groups) + + def _recommend_number_features(self, solver, max_time=None): + """ + Get the recommended number of features by doing forward passes when max_features is set. + """ + max_time = max_time if max_time else self.max_time + if max_time < 0: + max_time = 60 # allow 1 minute extra if we already spent max_time + MAX_FORWARD_PASS = 200 + MAX_FULL_BATCHES = 3 # the forward passes can take longer than the fitting + # if we allow a full epoch of data to be included. By only doing 3 full batches at most + # we get enough accuracy without increasing the time too much. This + # constant may not be optimal + accum_steps = solver.accum_steps + step_size = max(self.max_features / MAX_FORWARD_PASS, 1) + # pylint: disable=E1130 + feature_order = np.argsort(-self.scores_) # note the negative + t = time.time() + + dataloader_iterator = iter(solver.ds_train) + full_scores = [] + # keep_going = True + with torch.no_grad(): + # might want to only consider a batch valid if there are at least + # two classes + for _ in range(accum_steps * MAX_FULL_BATCHES): + scores = [] + try: + xsub, ysub = next(dataloader_iterator) + except StopIteration: + # done with epoch, don't do more than one epoch + break + except Exception as e: + print(e) + break + if max_time and time.time() - t > max_time: + if self.verbose: + print( + "Stoppinn forward passes because they reached max_time: ", + max_time) + if not full_scores: + # no forward passes worked, return half of max_features + return self.max_features // 2 + break + if solver.multiclass: + for target_class in range(solver.n_classes): + ysub_binary = solver.transform_y_into_binary( + ysub, target_class) + scaling_value = solver._get_scaling_value( + ysub, target_class) + if not solver._skip_y_forward(ysub_binary): + scores = self._generate_scores( + solver, xsub, ysub_binary, step_size, feature_order) + # one row will represent one class that is present in the data + # all classes are weighted equally + full_scores.append( + [score * scaling_value for score in scores]) + else: + if not solver._skip_y_forward(ysub): + scores = self._generate_scores( + solver, xsub, ysub, step_size, feature_order) + full_scores.append(scores) + best_index = FeatureGradientSelector._find_best_index_elbow( + full_scores) + if self.verbose: + print("Forward passes took: ", time.time() - t) + # account for step size and off by one (n_features is 1 indexed, not 0 + # ) + return int( + np.ceil( + np.arange( + 1, + self.max_features + + 1, + step_size))[best_index]) + + @staticmethod + def _find_best_index_elbow(full_scores): + """ + Finds the point on the curve that maximizes distance from the line determined by the endpoints. + """ + scores = pd.DataFrame(full_scores).mean(0).values.tolist() + first_point = np.array([0, scores[0]]) + last_point = np.array([len(scores) - 1, scores[-1]]) + elbow_metric = [] + for i in range(len(scores)): + elbow_metric.append( + FeatureGradientSelector._distance_to_line( + first_point, last_point, np.array([i, scores[i]]))) + return np.argmax(elbow_metric) + + @staticmethod + def _distance_to_line(start_point, end_point, new_point): + """ + Calculates the shortest distance from new_point to the line determined by start_point and end_point. + """ + # for calculating elbow method + return np.cross(new_point - start_point, + end_point - start_point) / np.linalg.norm( + end_point - start_point) + + def _reset(self): + """ + Reset the estimator by deleting all private and fit parameters. + """ + params = self.__dict__ + for key, _ in params.items(): + if key.endswith('_') or key.startswith('_'): + delattr(self, key) + return self diff --git a/new_impl/cv/third_party/nni/algorithms/feature_engineering/gradient_selector/learnability.py b/new_impl/cv/third_party/nni/algorithms/feature_engineering/gradient_selector/learnability.py new file mode 100644 index 0000000000000000000000000000000000000000..3a0ab4b39e7bef15101c793f3ec522395c05cd94 --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/feature_engineering/gradient_selector/learnability.py @@ -0,0 +1,534 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# MIT License +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and +# associated documentation files (the "Software"), to deal in the Software without restriction, +# including without limitation the rights to use, copy, modify, merge, publish, distribute, +# sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or +# substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT +# NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT +# OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# ================================================================================================== + +import time + +import numpy as np +import scipy.special +import torch +import torch.nn as nn + +from . import constants +from . import syssettings +from .fginitialize import ChunkDataLoader + +torch.set_default_tensor_type(syssettings.torch.tensortype) +sparsetensor = syssettings.torch.sparse.tensortype + + +def def_train_opt(p): + """ + Return the default optimizer. + """ + return torch.optim.Adam(p, 1e-1, amsgrad=False) + + +def revcumsum(U): + """ + Reverse cumulative sum for faster performance. + """ + return U.flip(dims=[0]).cumsum(dim=0).flip(dims=[0]) + + +def triudr(X, r): + + Zr = torch.zeros_like(X, requires_grad=False) + U = X * r + Zr[:-1] = X[:-1] * revcumsum(U)[1:] + + return Zr + + +def triudl(X, l): + + Zl = torch.zeros_like(X, requires_grad=False) + U = X * l + Zl[1:] = X[1:] * (U.cumsum(dim=0)[:-1]) + + return Zl + + +class ramp(torch.autograd.Function): + """ + Ensures input is between 0 and 1 + """ + + @staticmethod + def forward(ctx, input_data): + ctx.save_for_backward(input_data) + return input_data.clamp(min=0, max=1) + + + @staticmethod + def backward(ctx, grad_output): + input_data, = ctx.saved_tensors + grad_input = grad_output.clone() + grad_input[input_data < 0] = 1e-2 + grad_input[input_data > 1] = -1e-2 + return grad_input + + +class safesqrt(torch.autograd.Function): + """ + Square root without dividing by 0. + """ + @staticmethod + def forward(ctx, input_data): + o = input_data.sqrt() + ctx.save_for_backward(input_data, o) + return o + + + @staticmethod + def backward(ctx, grad_output): + _, o = ctx.saved_tensors + grad_input = grad_output.clone() + grad_input *= 0.5 / (o + constants.EPSILON) + return grad_input + + +class LearnabilityMB(nn.Module): + """ + Calculates the learnability of a set of features. + mini-batch version w/ "left" and "right" multiplies + """ + + + def __init__(self, Nminibatch, D, coeff, groups=None, binary=False, + device=constants.Device.CPU): + super(LearnabilityMB, self).__init__() + + a = coeff / scipy.special.binom(Nminibatch, np.arange(coeff.size) + 2) + self.order = a.size + # pylint: disable=E1102 + self.a = torch.tensor(a, dtype=torch.get_default_dtype(), requires_grad=False) + self.binary = binary + + self.a = self.a.to(device) + + + def ret_val(self, z): + """ + Get the return value based on z. + """ + + if not self.binary: + return 1 - z + + else: + return 0.5 * (1 - safesqrt.apply(ramp.apply(z))) + + + def forward(self, s, X, y): + + l = y.clone() + r = y.clone() + z = 0 + + for i in range(self.order): + if i % 2 == 0: + Z = triudr(X, r) + r = torch.mm(Z, s) + else: + Z = triudl(X, l) + l = torch.mm(Z, s) + if self.a[i] != 0: + # same the computation if a[i] is 0 + p = torch.mm(l.t(), r) + z += self.a[i] * p + return self.ret_val(z) + + +class Solver(nn.Module): + """ + Class that performs the main optimization. + Keeps track of the current x and iterates through data to learn x given the penalty and order. + """ + + def __init__(self, + PreparedData, + order, + Nminibatch=None, + groups=None, + soft_groups=None, + x0=None, + C=1, + ftransform=torch.sigmoid, + get_train_opt=def_train_opt, + accum_steps=1, + rng=np.random.RandomState(0), + max_norm_clip=1., + shuffle=True, + device=constants.Device.CPU, + verbose=1): + """ + + Parameters + ---------- + PreparedData : Dataset of PrepareData class + order : int + What order of interactions to include. Higher orders + may be more accurate but increase the run time. 12 is the maximum allowed order. + Nminibatch : int + Number of rows in a mini batch + groups : array-like + Optional, shape = [n_features] + Groups of columns that must be selected as a unit + e.g. [0, 0, 1, 2] specifies the first two columns are part of a group. + soft_groups : array-like + optional, shape = [n_features] + Groups of columns come from the same source + Used to encourage sparsity of number of sources selected + e.g. [0, 0, 1, 2] specifies the first two columns are part of a group. + x0 : torch.tensor + Optional, initialization of x. + C : float + Penalty parameter. + get_train_opt : function + Function that returns a pytorch optimizer, Adam is the default + accum_steps : int + Number of steps + rng : random state + max_norm_clip : float + Maximum allowable size of the gradient + shuffle : bool + Whether or not to shuffle data within the dataloader + order : int + What order of interactions to include. Higher orders + may be more accurate but increase the run time. 12 is the maximum allowed order. + penalty : int + Constant that multiplies the regularization term. + ftransform : function + Function to transform the x. sigmoid is the default. + device : str + 'cpu' to run on CPU and 'cuda' to run on GPU. Runs much faster on GPU + verbose : int + Controls the verbosity when fitting. Set to 0 for no printing + 1 or higher for printing every verbose number of gradient steps. + """ + super(Solver, self).__init__() + + self.Ntrain, self.D = PreparedData.N, PreparedData.n_features + if groups is not None: + # pylint: disable=E1102 + groups = torch.tensor(groups, dtype=torch.long) + self.groups = groups + else: + self.groups = None + if soft_groups is not None: + # pylint: disable=E1102 + soft_groups = torch.tensor(soft_groups, dtype=torch.long) + self.soft_D = torch.unique(soft_groups).size()[0] + else: + self.soft_D = None + self.soft_groups = soft_groups + + if Nminibatch is None: + Nminibatch = self.Ntrain + else: + if Nminibatch > self.Ntrain: + print('Minibatch larger than sample size.' + + (' Reducing from %d to %d.' + % (Nminibatch, self.Ntrain))) + Nminibatch = self.Ntrain + if Nminibatch > PreparedData.max_rows: + print('Minibatch larger than mem-allowed.' + + (' Reducing from %d to %d.' % (Nminibatch, + PreparedData.max_rows))) + Nminibatch = int(np.min([Nminibatch, PreparedData.max_rows])) + self.Nminibatch = Nminibatch + self.accum_steps = accum_steps + + if x0 is None: + x0 = torch.zeros(self.D, 1, dtype=torch.get_default_dtype()) + self.ftransform = ftransform + self.x = nn.Parameter(x0) + self.max_norm = max_norm_clip + + self.device = device + self.verbose = verbose + + self.multiclass = PreparedData.classification and PreparedData.n_classes and PreparedData.n_classes > 2 + if self.multiclass: + self.n_classes = PreparedData.n_classes + else: + self.n_classes = None + # whether to treat all classes equally + self.balanced = PreparedData.balanced + self.ordinal = PreparedData.ordinal + + if (hasattr(PreparedData, 'mappings') + or PreparedData.storage_level == 'disk'): + num_workers = PreparedData.num_workers + elif PreparedData.storage_level == constants.StorageLevel.DENSE: + num_workers = 0 + else: + num_workers = 0 + + if constants.Device.CUDA in device: + pin_memory = False + else: + pin_memory = False + + if num_workers == 0: + timeout = 0 + else: + timeout = 60 + + self.ds_train = ChunkDataLoader( + PreparedData, + batch_size=self.Nminibatch, + shuffle=shuffle, + drop_last=True, + num_workers=num_workers, + pin_memory=pin_memory, + timeout=timeout) + self.f_train = LearnabilityMB(self.Nminibatch, self.D, + constants.Coefficients.SLE[order], + self.groups, + binary=PreparedData.classification, + device=self.device) + self.opt_train = get_train_opt(torch.nn.ParameterList([self.x])) + self.it = 0 + self.iters_per_epoch = int(np.ceil(len(self.ds_train.dataset) + / self.ds_train.batch_size)) + self.f_train = self.f_train.to(device) + # pylint: disable=E1102 + self.w = torch.tensor( + C / (C + 1), + dtype=torch.get_default_dtype(), requires_grad=False) + self.w = self.w.to(device) + + + def penalty(self, s): + """ + Calculate L1 Penalty. + """ + to_return = torch.sum(s) / self.D + if self.soft_groups is not None: + # if soft_groups, there is an additional penalty for using more + # groups + s_grouped = torch.zeros(self.soft_D, 1, + dtype=torch.get_default_dtype(), + device=self.device) + for group in torch.unique(self.soft_groups): + # groups should be indexed 0 to n_group - 1 + # TODO: consider other functions here + s_grouped[group] = s[self.soft_groups == group].max() + # each component of the penalty contributes .5 + # TODO: could make this a user given parameter + to_return = (to_return + torch.sum(s_grouped) / self.soft_D) * .5 + return to_return + + + def forward_and_backward(self, s, xsub, ysub, retain_graph=False): + """ + Completes the forward operation and computes gradients for learnability and penalty. + """ + f_train = self.f_train(s, xsub, ysub) + pen = self.penalty(s).unsqueeze(0).unsqueeze(0) + # pylint: disable=E1102 + grad_outputs = torch.tensor([[1]], dtype=torch.get_default_dtype(), + device=self.device) + g1, = torch.autograd.grad([f_train], [self.x], grad_outputs, + retain_graph=True) + # pylint: disable=E1102 + grad_outputs = torch.tensor([[1]], dtype=torch.get_default_dtype(), + device=self.device) + g2, = torch.autograd.grad([pen], [self.x], grad_outputs, + retain_graph=retain_graph) + return f_train, pen, g1, g2 + + + def combine_gradient(self, g1, g2): + """ + Combine gradients from learnability and penalty + + Parameters + ---------- + g1 : array-like + gradient from learnability + g2 : array-like + gradient from penalty + """ + to_return = ((1 - self.w) * g1 + self.w * g2) / self.accum_steps + if self.groups is not None: + # each column will get a gradient + # but we can only up or down groups, so the gradient for the group + # should be the average of the gradients of the columns + to_return_grouped = torch.zeros_like(self.x) + for group in torch.unique(self.groups): + to_return_grouped[self.groups == + group] = to_return[self.groups == group].mean() + to_return = to_return_grouped + return to_return + + + def combine_loss(self, f_train, pen): + """ + Combine the learnability and L1 penalty. + """ + return ((1 - self.w) * f_train.detach() + self.w * pen.detach()) \ + / self.accum_steps + + + def transform_y_into_binary(self, ysub, target_class): + """ + Transforms multiclass classification problems into a binary classification problem. + """ + with torch.no_grad(): + ysub_binary = torch.zeros_like(ysub) + if self.ordinal: + # turn ordinal problems into n-1 classifications of is this + # example less than rank k + if target_class == 0: + return None + + ysub_binary[ysub >= target_class] = 1 + ysub_binary[ysub < target_class] = -1 + else: + # turn multiclass problems into n binary classifications + ysub_binary[ysub == target_class] = 1 + ysub_binary[ysub != target_class] = -1 + return ysub_binary + + + def _get_scaling_value(self, ysub, target_class): + """ + Returns the weight given to a class for multiclass classification. + """ + if self.balanced: + if self.ordinal: + return 1 / (torch.unique(ysub).size()[0] - 1) + + return 1 / torch.unique(ysub).size()[0] + else: + if self.ordinal: + this_class_proportion = torch.mean(ysub >= target_class) + normalizing_constant = 0 + for i in range(1, self.n_classes): + normalizing_constant += torch.mean(ysub >= i) + return this_class_proportion / normalizing_constant + else: + return torch.mean(ysub == target_class) + + + def _skip_y_forward(self, y): + """ + Returns boolean of whether to skip the currrent y if there is nothing to be learned from it. + """ + if y is None: + return True + elif torch.unique(y).size()[0] < 2: + return True + else: + return False + + + def train(self, f_callback=None, f_stop=None): + """ + Trains the estimator to determine which features to include. + + Parameters + ---------- + f_callback : function + Function that performs a callback + f_stop: function + Function that tells you when to stop + """ + + t = time.time() + h = torch.zeros([1, 1], dtype=torch.get_default_dtype()) + h = h.to(self.device) + # h_complete is so when we divide by the number of classes + # we only do that for that minibatch if accumulating + h_complete = h.clone() + flag_stop = False + dataloader_iterator = iter(self.ds_train) + self.x.grad = torch.zeros_like(self.x) + while not flag_stop: + try: + xsub, ysub = next(dataloader_iterator) + except StopIteration: + dataloader_iterator = iter(self.ds_train) + xsub, ysub = next(dataloader_iterator) + try: + s = self.ftransform(self.x) + s = s.to(self.device) + if self.multiclass: + # accumulate gradients over each class, classes range from + # 0 to n_classes - 1 + #num_classes_batch = torch.unique(ysub).size()[0] + for target_class in range(self.n_classes): + ysub_binary = self.transform_y_into_binary( + ysub, target_class) + if self._skip_y_forward(ysub_binary): + continue + # should should skip if target class is not included + # but that changes what we divide by + scaling_value = self._get_scaling_value( + ysub, target_class) + f_train, pen, g1, g2 = self.forward_and_backward( + s, xsub, ysub_binary, retain_graph=True) + self.x.grad += self.combine_gradient( + g1, g2) * scaling_value + h += self.combine_loss(f_train, + pen) * scaling_value + else: + if not self._skip_y_forward(ysub): + f_train, pen, g1, g2 = self.forward_and_backward( + s, xsub, ysub) + self.x.grad += self.combine_gradient(g1, g2) + h += self.combine_loss(f_train, pen) + else: + continue + h_complete += h + self.it += 1 + if torch.isnan(h): + raise constants.NanError( + 'Loss is nan, something may be misconfigured') + if self.it % self.accum_steps == 0: + torch.nn.utils.clip_grad_norm_( + torch.nn.ParameterList([self.x]), + max_norm=self.max_norm) + self.opt_train.step() + + t = time.time() - t + if f_stop is not None: + flag_stop = f_stop(self, h, self.it, t) + + if f_callback is not None: + f_callback(self, h, self.it, t) + elif self.verbose and (self.it // self.accum_steps) % self.verbose == 0: + epoch = int(self.it / self.iters_per_epoch) + print( + '[Minibatch: %6d/ Epoch: %3d/ t: %3.3f s] Loss: %0.3f' % + (self.it, epoch, t, h_complete / self.accum_steps)) + + if flag_stop: + break + + self.opt_train.zero_grad() + h = 0 + h_complete = 0 + t = time.time() + except KeyboardInterrupt: + flag_stop = True + break diff --git a/new_impl/cv/third_party/nni/algorithms/feature_engineering/gradient_selector/requirements.txt b/new_impl/cv/third_party/nni/algorithms/feature_engineering/gradient_selector/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..7e2873b558c30216e17b584570b7383623b2931c --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/feature_engineering/gradient_selector/requirements.txt @@ -0,0 +1,4 @@ +numpy==1.14.3 +scikit-learn>=0.23.2 +scipy==1.1.0 +torch==1.1.0 diff --git a/new_impl/cv/third_party/nni/algorithms/feature_engineering/gradient_selector/syssettings.py b/new_impl/cv/third_party/nni/algorithms/feature_engineering/gradient_selector/syssettings.py new file mode 100644 index 0000000000000000000000000000000000000000..df864b316601464a9f35a7b463933b1f05a9fe3f --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/feature_engineering/gradient_selector/syssettings.py @@ -0,0 +1,29 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# MIT License +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and +# associated documentation files (the "Software"), to deal in the Software without restriction, +# including without limitation the rights to use, copy, modify, merge, publish, distribute, +# sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or +# substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT +# NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT +# OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# ================================================================================================== + + +import torch + +# pytorch +torch.tensortype = torch.FloatTensor +torch.sparse.tensortype = torch.sparse.FloatTensor + +# mem +MAXMEMGB = 10 diff --git a/new_impl/cv/third_party/nni/algorithms/feature_engineering/gradient_selector/utils.py b/new_impl/cv/third_party/nni/algorithms/feature_engineering/gradient_selector/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..0ab9b09a25cfb328586c39f00ab9546388b4f8d4 --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/feature_engineering/gradient_selector/utils.py @@ -0,0 +1,78 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# MIT License +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and +# associated documentation files (the "Software"), to deal in the Software without restriction, +# including without limitation the rights to use, copy, modify, merge, publish, distribute, +# sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or +# substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT +# NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT +# OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# ================================================================================================== + + +import numpy as np + +class EMA(): + """ + maintains an exponential moving average + """ + + def __init__(self, f=np.nan, discount_factor=0.1, valid_after=None, + n_iters_relchange=3): + + self.f_ma = [f] + self.fs = [f] + self.gamma = discount_factor + self.rel_change = [np.nan] + if valid_after is None: + self.valid_after = int(1/discount_factor) + else: + self.valid_after = valid_after + self.n_iters_relchange = n_iters_relchange + self.initialized = False + + def reset(self, f): + + self.f_ma = [f] + self.fs = [f] + self.rel_change = [np.nan] + self.initialized = True + + def relchange(self): + + if self.num_updates() > np.max([self.valid_after, + self.n_iters_relchange]): + return np.max(self.rel_change[-self.n_iters_relchange:]) + else: + return np.nan + + def update(self, f_new): + + if not self.initialized: + self.reset(f_new) + else: + self.fs.append(f_new) + self.f_ma.append(self.f_ma[-1]*(1-self.gamma) + self.gamma*f_new) + if self.num_updates() > self.valid_after: + self.rel_change.append(np.abs((self.f_ma[-1]-self.f_ma[-2]) + / self.f_ma[-2])) + + def num_updates(self): + + return len(self.f_ma) + + def __call__(self): + + if self.num_updates() > self.valid_after: + return self.f_ma[-1] + else: + return np.nan diff --git a/new_impl/cv/third_party/nni/algorithms/hpo/__init__.py b/new_impl/cv/third_party/nni/algorithms/hpo/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/new_impl/cv/third_party/nni/algorithms/hpo/batch_tuner/__init__.py b/new_impl/cv/third_party/nni/algorithms/hpo/batch_tuner/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/new_impl/cv/third_party/nni/algorithms/hpo/batch_tuner/batch_tuner.py b/new_impl/cv/third_party/nni/algorithms/hpo/batch_tuner/batch_tuner.py new file mode 100644 index 0000000000000000000000000000000000000000..4f73fce9453161a8f30783f915873ba8e5bb8f31 --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/hpo/batch_tuner/batch_tuner.py @@ -0,0 +1,131 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +batch_tuner.py including: + class BatchTuner +""" + +import logging + +import nni +from nni.tuner import Tuner + +TYPE = '_type' +CHOICE = 'choice' +VALUE = '_value' + +LOGGER = logging.getLogger('batch_tuner_AutoML') + +class BatchTuner(Tuner): + """ + BatchTuner is tuner will running all the configure that user want to run batchly. + + Examples + -------- + The search space only be accepted like: + + :: + + {'combine_params': + { '_type': 'choice', + '_value': '[{...}, {...}, {...}]', + } + } + + """ + + def __init__(self): + self._count = -1 + self._values = [] + + def is_valid(self, search_space): + """ + Check the search space is valid: only contains 'choice' type + + Parameters + ---------- + search_space : dict + + Returns + ------- + None or list + If valid, return candidate values; else return None. + """ + if not len(search_space) == 1: + raise RuntimeError('BatchTuner only supprt one combined-paramreters key.') + + for param in search_space: + param_type = search_space[param][TYPE] + if not param_type == CHOICE: + raise RuntimeError('BatchTuner only supprt \ + one combined-paramreters type is choice.') + + if isinstance(search_space[param][VALUE], list): + return search_space[param][VALUE] + + raise RuntimeError('The combined-paramreters \ + value in BatchTuner is not a list.') + return None + + def update_search_space(self, search_space): + """Update the search space + + Parameters + ---------- + search_space : dict + """ + self._values = self.is_valid(search_space) + + def generate_parameters(self, parameter_id, **kwargs): + """Returns a dict of trial (hyper-)parameters, as a serializable object. + + Parameters + ---------- + parameter_id : int + + Returns + ------- + dict + A candidate parameter group. + """ + self._count += 1 + if self._count > len(self._values) - 1: + raise nni.NoMoreTrialError('no more parameters now.') + return self._values[self._count] + + def receive_trial_result(self, parameter_id, parameters, value, **kwargs): + pass + + def import_data(self, data): + """Import additional data for tuning + + Parameters + ---------- + data: + a list of dictionarys, each of which has at least two keys, 'parameter' and 'value' + """ + if not self._values: + LOGGER.info("Search space has not been initialized, skip this data import") + return + + self._values = self._values[(self._count+1):] + self._count = -1 + + _completed_num = 0 + for trial_info in data: + LOGGER .info("Importing data, current processing \ + progress %s / %s", _completed_num, len(data)) + # simply validate data format + assert "parameter" in trial_info + _params = trial_info["parameter"] + assert "value" in trial_info + _value = trial_info['value'] + if not _value: + LOGGER.info("Useless trial data, value is %s, skip this trial data.", _value) + continue + _completed_num += 1 + if _params in self._values: + self._values.remove(_params) + LOGGER .info("Successfully import data to batch tuner, \ + total data: %d, imported data: %d.", len(data), _completed_num) diff --git a/new_impl/cv/third_party/nni/algorithms/hpo/bohb_advisor/__init__.py b/new_impl/cv/third_party/nni/algorithms/hpo/bohb_advisor/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/new_impl/cv/third_party/nni/algorithms/hpo/bohb_advisor/bohb_advisor.py b/new_impl/cv/third_party/nni/algorithms/hpo/bohb_advisor/bohb_advisor.py new file mode 100644 index 0000000000000000000000000000000000000000..73687abc5c71145e95d8623b64ed82284c9063e3 --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/hpo/bohb_advisor/bohb_advisor.py @@ -0,0 +1,676 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +''' +bohb_advisor.py +''' + +import sys +import math +import logging +import json_tricks +from schema import Schema, Optional +import ConfigSpace as CS +import ConfigSpace.hyperparameters as CSH + +from nni import ClassArgsValidator +from nni.runtime.protocol import CommandType, send +from nni.runtime.msg_dispatcher_base import MsgDispatcherBase +from nni.utils import OptimizeMode, MetricType, extract_scalar_reward +from nni.runtime.common import multi_phase_enabled + +from .config_generator import CG_BOHB + +logger = logging.getLogger('BOHB_Advisor') + +_next_parameter_id = 0 +_KEY = 'TRIAL_BUDGET' +_epsilon = 1e-6 + + +def create_parameter_id(): + """Create an id + + Returns + ------- + int + parameter id + """ + global _next_parameter_id + _next_parameter_id += 1 + return _next_parameter_id - 1 + + +def create_bracket_parameter_id(brackets_id, brackets_curr_decay, increased_id=-1): + """Create a full id for a specific bracket's hyperparameter configuration + + Parameters + ---------- + brackets_id: int + brackets id + brackets_curr_decay: int + brackets curr decay + increased_id: int + increased id + Returns + ------- + int + params id + """ + if increased_id == -1: + increased_id = str(create_parameter_id()) + params_id = '_'.join([str(brackets_id), + str(brackets_curr_decay), + increased_id]) + return params_id + + +class Bracket: + """ + A bracket in BOHB, all the information of a bracket is managed by + an instance of this class. + + Parameters + ---------- + s: int + The current Successive Halving iteration index. + s_max: int + total number of Successive Halving iterations + eta: float + In each iteration, a complete run of sequential halving is executed. In it, + after evaluating each configuration on the same subset size, only a fraction of + 1/eta of them 'advances' to the next round. + max_budget : float + The largest budget to consider. Needs to be larger than min_budget! + The budgets will be geometrically distributed + :math:`a^2 + b^2 = c^2 \\sim \\eta^k` for :math:`k\\in [0, 1, ... , num\\_subsets - 1]`. + optimize_mode: str + optimize mode, 'maximize' or 'minimize' + """ + def __init__(self, s, s_max, eta, max_budget, optimize_mode): + self.s = s + self.s_max = s_max + self.eta = eta + self.max_budget = max_budget + self.optimize_mode = OptimizeMode(optimize_mode) + + self.n = math.ceil((s_max + 1) * eta**s / (s + 1) - _epsilon) + self.r = max_budget / eta**s + self.i = 0 + self.hyper_configs = [] # [ {id: params}, {}, ... ] + self.configs_perf = [] # [ {id: [seq, acc]}, {}, ... ] + self.num_configs_to_run = [] # [ n, n, n, ... ] + self.num_finished_configs = [] # [ n, n, n, ... ] + self.no_more_trial = False + + def is_completed(self): + """check whether this bracket has sent out all the hyperparameter configurations""" + return self.no_more_trial + + def get_n_r(self): + """return the values of n and r for the next round""" + return math.floor(self.n / self.eta**self.i + _epsilon), math.floor(self.r * self.eta**self.i +_epsilon) + + def increase_i(self): + """i means the ith round. Increase i by 1""" + self.i += 1 + + def set_config_perf(self, i, parameter_id, seq, value): + """update trial's latest result with its sequence number, e.g., epoch number or batch number + + Parameters + ---------- + i: int + the ith round + parameter_id: int + the id of the trial/parameter + seq: int + sequence number, e.g., epoch number or batch number + value: int + latest result with sequence number seq + + Returns + ------- + None + """ + if parameter_id in self.configs_perf[i]: + if self.configs_perf[i][parameter_id][0] < seq: + self.configs_perf[i][parameter_id] = [seq, value] + else: + self.configs_perf[i][parameter_id] = [seq, value] + + def inform_trial_end(self, i): + """If the trial is finished and the corresponding round (i.e., i) has all its trials finished, + it will choose the top k trials for the next round (i.e., i+1) + + Parameters + ---------- + i: int + the ith round + + Returns + ------- + new trial or None: + If we have generated new trials after this trial end, we will return a new trial parameters. + Otherwise, we will return None. + """ + global _KEY + self.num_finished_configs[i] += 1 + logger.debug('bracket id: %d, round: %d %d, finished: %d, all: %d', + self.s, self.i, i, self.num_finished_configs[i], self.num_configs_to_run[i]) + if self.num_finished_configs[i] >= self.num_configs_to_run[i] and self.no_more_trial is False: + # choose candidate configs from finished configs to run in the next round + assert self.i == i + 1 + # finish this bracket + if self.i > self.s: + self.no_more_trial = True + return None + this_round_perf = self.configs_perf[i] + if self.optimize_mode is OptimizeMode.Maximize: + sorted_perf = sorted(this_round_perf.items( + ), key=lambda kv: kv[1][1], reverse=True) # reverse + else: + sorted_perf = sorted( + this_round_perf.items(), key=lambda kv: kv[1][1]) + logger.debug( + 'bracket %s next round %s, sorted hyper configs: %s', self.s, self.i, sorted_perf) + next_n, next_r = self.get_n_r() + logger.debug('bracket %s next round %s, next_n=%d, next_r=%d', + self.s, self.i, next_n, next_r) + hyper_configs = dict() + for k in range(next_n): + params_id = sorted_perf[k][0] + params = self.hyper_configs[i][params_id] + params[_KEY] = next_r # modify r + # generate new id + increased_id = params_id.split('_')[-1] + new_id = create_bracket_parameter_id( + self.s, self.i, increased_id) + hyper_configs[new_id] = params + self._record_hyper_configs(hyper_configs) + return [[key, value] for key, value in hyper_configs.items()] + return None + + def get_hyperparameter_configurations(self, num, r, config_generator): + """generate num hyperparameter configurations from search space using Bayesian optimization + + Parameters + ---------- + num: int + the number of hyperparameter configurations + + Returns + ------- + list + a list of hyperparameter configurations. Format: [[key1, value1], [key2, value2], ...] + """ + global _KEY + assert self.i == 0 + hyperparameter_configs = dict() + for _ in range(num): + params_id = create_bracket_parameter_id(self.s, self.i) + params = config_generator.get_config(r) + params[_KEY] = r + hyperparameter_configs[params_id] = params + self._record_hyper_configs(hyperparameter_configs) + return [[key, value] for key, value in hyperparameter_configs.items()] + + def _record_hyper_configs(self, hyper_configs): + """after generating one round of hyperconfigs, this function records the generated hyperconfigs, + creates a dict to record the performance when those hyperconifgs are running, set the number of finished configs + in this round to be 0, and increase the round number. + + Parameters + ---------- + hyper_configs: list + the generated hyperconfigs + """ + self.hyper_configs.append(hyper_configs) + self.configs_perf.append(dict()) + self.num_finished_configs.append(0) + self.num_configs_to_run.append(len(hyper_configs)) + self.increase_i() + +class BOHBClassArgsValidator(ClassArgsValidator): + def validate_class_args(self, **kwargs): + Schema({ + 'optimize_mode': self.choices('optimize_mode', 'maximize', 'minimize'), + Optional('min_budget'): self.range('min_budget', int, 0, 9999), + Optional('max_budget'): self.range('max_budget', int, 0, 9999), + Optional('eta'): self.range('eta', int, 0, 9999), + Optional('min_points_in_model'): self.range('min_points_in_model', int, 0, 9999), + Optional('top_n_percent'): self.range('top_n_percent', int, 1, 99), + Optional('num_samples'): self.range('num_samples', int, 1, 9999), + Optional('random_fraction'): self.range('random_fraction', float, 0, 9999), + Optional('bandwidth_factor'): self.range('bandwidth_factor', float, 0, 9999), + Optional('min_bandwidth'): self.range('min_bandwidth', float, 0, 9999), + }).validate(kwargs) + +class BOHB(MsgDispatcherBase): + """ + BOHB performs robust and efficient hyperparameter optimization + at scale by combining the speed of Hyperband searches with the + guidance and guarantees of convergence of Bayesian Optimization. + Instead of sampling new configurations at random, BOHB uses + kernel density estimators to select promising candidates. + + Parameters + ---------- + optimize_mode: str + optimize mode, 'maximize' or 'minimize' + min_budget: float + The smallest budget to consider. Needs to be positive! + max_budget: float + The largest budget to consider. Needs to be larger than min_budget! + The budgets will be geometrically distributed + :math:`a^2 + b^2 = c^2 \\sim \\eta^k` for :math:`k\\in [0, 1, ... , num\\_subsets - 1]`. + eta: int + In each iteration, a complete run of sequential halving is executed. In it, + after evaluating each configuration on the same subset size, only a fraction of + 1/eta of them 'advances' to the next round. + Must be greater or equal to 2. + min_points_in_model: int + number of observations to start building a KDE. Default 'None' means + dim+1, the bare minimum. + top_n_percent: int + percentage ( between 1 and 99, default 15) of the observations that are considered good. + num_samples: int + number of samples to optimize EI (default 64) + random_fraction: float + fraction of purely random configurations that are sampled from the + prior without the model. + bandwidth_factor: float + to encourage diversity, the points proposed to optimize EI, are sampled + from a 'widened' KDE where the bandwidth is multiplied by this factor (default: 3) + min_bandwidth: float + to keep diversity, even when all (good) samples have the same value for one of the parameters, + a minimum bandwidth (Default: 1e-3) is used instead of zero. + """ + + def __init__(self, + optimize_mode='maximize', + min_budget=1, + max_budget=3, + eta=3, + min_points_in_model=None, + top_n_percent=15, + num_samples=64, + random_fraction=1/3, + bandwidth_factor=3, + min_bandwidth=1e-3): + super(BOHB, self).__init__() + self.optimize_mode = OptimizeMode(optimize_mode) + self.min_budget = min_budget + self.max_budget = max_budget + self.eta = eta + self.min_points_in_model = min_points_in_model + self.top_n_percent = top_n_percent + self.num_samples = num_samples + self.random_fraction = random_fraction + self.bandwidth_factor = bandwidth_factor + self.min_bandwidth = min_bandwidth + + # all the configs waiting for run + self.generated_hyper_configs = [] + # all the completed configs + self.completed_hyper_configs = [] + + self.s_max = math.floor( + math.log(self.max_budget / self.min_budget, self.eta) + _epsilon) + # current bracket(s) number + self.curr_s = self.s_max + # In this case, tuner increases self.credit to issue a trial config sometime later. + self.credit = 0 + self.brackets = dict() + self.search_space = None + # [key, value] = [parameter_id, parameter] + self.parameters = dict() + + # config generator + self.cg = None + + # record the latest parameter_id of the trial job trial_job_id. + # if there is no running parameter_id, self.job_id_para_id_map[trial_job_id] == None + # new trial job is added to this dict and finished trial job is removed from it. + self.job_id_para_id_map = dict() + # record the unsatisfied parameter request from trial jobs + self.unsatisfied_jobs = [] + + def handle_initialize(self, data): + """Initialize Tuner, including creating Bayesian optimization-based parametric models + and search space formations + + Parameters + ---------- + data: search space + search space of this experiment + + Raises + ------ + ValueError + Error: Search space is None + """ + logger.info('start to handle_initialize') + # convert search space jason to ConfigSpace + self.handle_update_search_space(data) + + # generate BOHB config_generator using Bayesian optimization + if self.search_space: + self.cg = CG_BOHB(configspace=self.search_space, + min_points_in_model=self.min_points_in_model, + top_n_percent=self.top_n_percent, + num_samples=self.num_samples, + random_fraction=self.random_fraction, + bandwidth_factor=self.bandwidth_factor, + min_bandwidth=self.min_bandwidth) + else: + raise ValueError('Error: Search space is None') + # generate first brackets + self.generate_new_bracket() + send(CommandType.Initialized, '') + + def generate_new_bracket(self): + """generate a new bracket""" + logger.debug( + 'start to create a new SuccessiveHalving iteration, self.curr_s=%d', self.curr_s) + if self.curr_s < 0: + logger.info("s < 0, Finish this round of Hyperband in BOHB. Generate new round") + self.curr_s = self.s_max + self.brackets[self.curr_s] = Bracket( + s=self.curr_s, s_max=self.s_max, eta=self.eta, + max_budget=self.max_budget, optimize_mode=self.optimize_mode + ) + next_n, next_r = self.brackets[self.curr_s].get_n_r() + logger.debug( + 'new SuccessiveHalving iteration, next_n=%d, next_r=%d', next_n, next_r) + # rewrite with TPE + generated_hyper_configs = self.brackets[self.curr_s].get_hyperparameter_configurations( + next_n, next_r, self.cg) + self.generated_hyper_configs = generated_hyper_configs.copy() + + def handle_request_trial_jobs(self, data): + """recerive the number of request and generate trials + + Parameters + ---------- + data: int + number of trial jobs that nni manager ask to generate + """ + # Receive new request + self.credit += data + + for _ in range(self.credit): + self._request_one_trial_job() + + def _get_one_trial_job(self): + """get one trial job, i.e., one hyperparameter configuration. + + If this function is called, Command will be sent by BOHB: + a. If there is a parameter need to run, will return "NewTrialJob" with a dict: + { + 'parameter_id': id of new hyperparameter + 'parameter_source': 'algorithm' + 'parameters': value of new hyperparameter + } + b. If BOHB don't have parameter waiting, will return "NoMoreTrialJobs" with + { + 'parameter_id': '-1_0_0', + 'parameter_source': 'algorithm', + 'parameters': '' + } + """ + if not self.generated_hyper_configs: + ret = { + 'parameter_id': '-1_0_0', + 'parameter_source': 'algorithm', + 'parameters': '' + } + send(CommandType.NoMoreTrialJobs, json_tricks.dumps(ret)) + return None + assert self.generated_hyper_configs + params = self.generated_hyper_configs.pop(0) + ret = { + 'parameter_id': params[0], + 'parameter_source': 'algorithm', + 'parameters': params[1] + } + self.parameters[params[0]] = params[1] + return ret + + def _request_one_trial_job(self): + """get one trial job, i.e., one hyperparameter configuration. + + If this function is called, Command will be sent by BOHB: + a. If there is a parameter need to run, will return "NewTrialJob" with a dict: + { + 'parameter_id': id of new hyperparameter + 'parameter_source': 'algorithm' + 'parameters': value of new hyperparameter + } + b. If BOHB don't have parameter waiting, will return "NoMoreTrialJobs" with + { + 'parameter_id': '-1_0_0', + 'parameter_source': 'algorithm', + 'parameters': '' + } + """ + ret = self._get_one_trial_job() + if ret is not None: + send(CommandType.NewTrialJob, json_tricks.dumps(ret)) + self.credit -= 1 + + def handle_update_search_space(self, data): + """change json format to ConfigSpace format dict -> configspace + + Parameters + ---------- + data: JSON object + search space of this experiment + """ + search_space = data + cs = CS.ConfigurationSpace() + for var in search_space: + _type = str(search_space[var]["_type"]) + if _type == 'choice': + cs.add_hyperparameter(CSH.CategoricalHyperparameter( + var, choices=search_space[var]["_value"])) + elif _type == 'randint': + cs.add_hyperparameter(CSH.UniformIntegerHyperparameter( + var, lower=search_space[var]["_value"][0], upper=search_space[var]["_value"][1] - 1)) + elif _type == 'uniform': + cs.add_hyperparameter(CSH.UniformFloatHyperparameter( + var, lower=search_space[var]["_value"][0], upper=search_space[var]["_value"][1])) + elif _type == 'quniform': + cs.add_hyperparameter(CSH.UniformFloatHyperparameter( + var, lower=search_space[var]["_value"][0], upper=search_space[var]["_value"][1], + q=search_space[var]["_value"][2])) + elif _type == 'loguniform': + cs.add_hyperparameter(CSH.UniformFloatHyperparameter( + var, lower=search_space[var]["_value"][0], upper=search_space[var]["_value"][1], + log=True)) + elif _type == 'qloguniform': + cs.add_hyperparameter(CSH.UniformFloatHyperparameter( + var, lower=search_space[var]["_value"][0], upper=search_space[var]["_value"][1], + q=search_space[var]["_value"][2], log=True)) + elif _type == 'normal': + cs.add_hyperparameter(CSH.NormalFloatHyperparameter( + var, mu=search_space[var]["_value"][1], sigma=search_space[var]["_value"][2])) + elif _type == 'qnormal': + cs.add_hyperparameter(CSH.NormalFloatHyperparameter( + var, mu=search_space[var]["_value"][1], sigma=search_space[var]["_value"][2], + q=search_space[var]["_value"][3])) + elif _type == 'lognormal': + cs.add_hyperparameter(CSH.NormalFloatHyperparameter( + var, mu=search_space[var]["_value"][1], sigma=search_space[var]["_value"][2], + log=True)) + elif _type == 'qlognormal': + cs.add_hyperparameter(CSH.NormalFloatHyperparameter( + var, mu=search_space[var]["_value"][1], sigma=search_space[var]["_value"][2], + q=search_space[var]["_value"][3], log=True)) + else: + raise ValueError( + 'unrecognized type in search_space, type is {}'.format(_type)) + + self.search_space = cs + + def handle_trial_end(self, data): + """receive the information of trial end and generate next configuaration. + + Parameters + ---------- + data: dict() + it has three keys: trial_job_id, event, hyper_params + trial_job_id: the id generated by training service + event: the job's state + hyper_params: the hyperparameters (a string) generated and returned by tuner + """ + logger.debug('Tuner handle trial end, result is %s', data) + hyper_params = json_tricks.loads(data['hyper_params']) + self._handle_trial_end(hyper_params['parameter_id']) + if data['trial_job_id'] in self.job_id_para_id_map: + del self.job_id_para_id_map[data['trial_job_id']] + + def _send_new_trial(self): + while self.unsatisfied_jobs: + ret = self._get_one_trial_job() + if ret is None: + break + one_unsatisfied = self.unsatisfied_jobs.pop(0) + ret['trial_job_id'] = one_unsatisfied['trial_job_id'] + ret['parameter_index'] = one_unsatisfied['parameter_index'] + # update parameter_id in self.job_id_para_id_map + self.job_id_para_id_map[ret['trial_job_id']] = ret['parameter_id'] + send(CommandType.SendTrialJobParameter, json_tricks.dumps(ret)) + for _ in range(self.credit): + self._request_one_trial_job() + + def _handle_trial_end(self, parameter_id): + s, i, _ = parameter_id.split('_') + hyper_configs = self.brackets[int(s)].inform_trial_end(int(i)) + + if hyper_configs is not None: + logger.debug( + 'bracket %s next round %s, hyper_configs: %s', s, i, hyper_configs) + self.generated_hyper_configs = self.generated_hyper_configs + hyper_configs + # Finish this bracket and generate a new bracket + elif self.brackets[int(s)].no_more_trial: + self.curr_s -= 1 + self.generate_new_bracket() + self._send_new_trial() + + def handle_report_metric_data(self, data): + """reveice the metric data and update Bayesian optimization with final result + + Parameters + ---------- + data: + it is an object which has keys 'parameter_id', 'value', 'trial_job_id', 'type', 'sequence'. + + Raises + ------ + ValueError + Data type not supported + """ + logger.debug('handle report metric data = %s', data) + if 'value' in data: + data['value'] = json_tricks.loads(data['value']) + if data['type'] == MetricType.REQUEST_PARAMETER: + assert multi_phase_enabled() + assert data['trial_job_id'] is not None + assert data['parameter_index'] is not None + assert data['trial_job_id'] in self.job_id_para_id_map + self._handle_trial_end(self.job_id_para_id_map[data['trial_job_id']]) + ret = self._get_one_trial_job() + if ret is None: + self.unsatisfied_jobs.append({'trial_job_id': data['trial_job_id'], 'parameter_index': data['parameter_index']}) + else: + ret['trial_job_id'] = data['trial_job_id'] + ret['parameter_index'] = data['parameter_index'] + # update parameter_id in self.job_id_para_id_map + self.job_id_para_id_map[data['trial_job_id']] = ret['parameter_id'] + send(CommandType.SendTrialJobParameter, json_tricks.dumps(ret)) + else: + assert 'value' in data + value = extract_scalar_reward(data['value']) + if self.optimize_mode is OptimizeMode.Maximize: + reward = -value + else: + reward = value + assert 'parameter_id' in data + s, i, _ = data['parameter_id'].split('_') + logger.debug('bracket id = %s, metrics value = %s, type = %s', s, value, data['type']) + s = int(s) + + # add to self.job_id_para_id_map here, + # because when the first parameter_id is created, trial_job_id is not known yet. + if data['trial_job_id'] in self.job_id_para_id_map: + assert self.job_id_para_id_map[data['trial_job_id']] == data['parameter_id'] + else: + self.job_id_para_id_map[data['trial_job_id']] = data['parameter_id'] + + assert 'type' in data + if data['type'] == MetricType.FINAL: + # and PERIODICAL metric are independent, thus, not comparable. + assert 'sequence' in data + self.brackets[s].set_config_perf( + int(i), data['parameter_id'], sys.maxsize, value) + self.completed_hyper_configs.append(data) + + _parameters = self.parameters[data['parameter_id']] + _parameters.pop(_KEY) + # update BO with loss, max_s budget, hyperparameters + self.cg.new_result(loss=reward, budget=data['sequence'], parameters=_parameters, update_model=True) + elif data['type'] == MetricType.PERIODICAL: + self.brackets[s].set_config_perf( + int(i), data['parameter_id'], data['sequence'], value) + else: + raise ValueError( + 'Data type not supported: {}'.format(data['type'])) + + def handle_add_customized_trial(self, data): + pass + + def handle_import_data(self, data): + """Import additional data for tuning + + Parameters + ---------- + data: + a list of dictionarys, each of which has at least two keys, 'parameter' and 'value' + + Raises + ------ + AssertionError + data doesn't have required key 'parameter' and 'value' + """ + for entry in data: + entry['value'] = json_tricks.loads(entry['value']) + _completed_num = 0 + for trial_info in data: + logger.info("Importing data, current processing progress %s / %s", _completed_num, len(data)) + _completed_num += 1 + assert "parameter" in trial_info + _params = trial_info["parameter"] + assert "value" in trial_info + _value = trial_info['value'] + if not _value: + logger.info("Useless trial data, value is %s, skip this trial data.", _value) + continue + _value = extract_scalar_reward(_value) + budget_exist_flag = False + barely_params = dict() + for keys in _params: + if keys == _KEY: + _budget = _params[keys] + budget_exist_flag = True + else: + barely_params[keys] = _params[keys] + if not budget_exist_flag: + _budget = self.max_budget + logger.info("Set \"TRIAL_BUDGET\" value to %s (max budget)", self.max_budget) + if self.optimize_mode is OptimizeMode.Maximize: + reward = -_value + else: + reward = _value + self.cg.new_result(loss=reward, budget=_budget, parameters=barely_params, update_model=True) + logger.info("Successfully import tuning data to BOHB advisor.") diff --git a/new_impl/cv/third_party/nni/algorithms/hpo/bohb_advisor/config_generator.py b/new_impl/cv/third_party/nni/algorithms/hpo/bohb_advisor/config_generator.py new file mode 100644 index 0000000000000000000000000000000000000000..c6a13c6b35ba3b2db9bbbf8618d3b9a3d9240909 --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/hpo/bohb_advisor/config_generator.py @@ -0,0 +1,344 @@ +# BSD 3-Clause License +# Copyright (c) 2017-2018, ML4AAD +# All rights reserved. + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. + +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. + +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import logging + +import ConfigSpace +import ConfigSpace.hyperparameters +import ConfigSpace.util +import numpy as np +import scipy.stats as sps +import statsmodels.api as sm + +logger = logging.getLogger('BOHB_Advisor') + +class CG_BOHB: + def __init__(self, configspace, min_points_in_model=None, + top_n_percent=15, num_samples=64, random_fraction=1/3, + bandwidth_factor=3, min_bandwidth=1e-3): + """Fits for each given budget a kernel density estimator on the best N percent of the + evaluated configurations on this budget. + + + Parameters: + ----------- + configspace: ConfigSpace + Configuration space object + top_n_percent: int + Determines the percentile of configurations that will be used as training data + for the kernel density estimator, e.g if set to 10 the 10% best configurations will be considered + for training. + min_points_in_model: int + minimum number of datapoints needed to fit a model + num_samples: int + number of samples drawn to optimize EI via sampling + random_fraction: float + fraction of random configurations returned + bandwidth_factor: float + widens the bandwidth for contiuous parameters for proposed points to optimize EI + min_bandwidth: float + to keep diversity, even when all (good) samples have the same value for one of the parameters, + a minimum bandwidth (Default: 1e-3) is used instead of zero. + """ + self.top_n_percent = top_n_percent + self.configspace = configspace + self.bw_factor = bandwidth_factor + self.min_bandwidth = min_bandwidth + + self.min_points_in_model = min_points_in_model + if min_points_in_model is None: + self.min_points_in_model = len(self.configspace.get_hyperparameters())+1 + + if self.min_points_in_model < len(self.configspace.get_hyperparameters())+1: + logger.warning('Invalid min_points_in_model value. Setting it to %i', len(self.configspace.get_hyperparameters()) + 1) + self.min_points_in_model = len(self.configspace.get_hyperparameters()) + 1 + + self.num_samples = num_samples + self.random_fraction = random_fraction + + hps = self.configspace.get_hyperparameters() + + self.kde_vartypes = "" + self.vartypes = [] + + for h in hps: + if hasattr(h, 'choices'): + self.kde_vartypes += 'u' + self.vartypes += [len(h.choices)] + else: + self.kde_vartypes += 'c' + self.vartypes += [0] + + self.vartypes = np.array(self.vartypes, dtype=int) + + # store precomputed probs for the categorical parameters + self.cat_probs = [] + + self.configs = dict() + self.losses = dict() + self.good_config_rankings = dict() + self.kde_models = dict() + + def largest_budget_with_model(self): + if not self.kde_models: + return -float('inf') + return max(self.kde_models.keys()) + + def sample_from_largest_budget(self, info_dict): + """We opted for a single multidimensional KDE compared to the + hierarchy of one-dimensional KDEs used in TPE. The dimensional is + seperated by budget. This function sample a configuration from + largest budget. Firstly we sample "num_samples" configurations, + then prefer one with the largest l(x)/g(x). + + Parameters: + ----------- + info_dict: dict + record the information of this configuration + + Returns + ------- + dict: + new configuration named sample + dict: + info_dict, record the information of this configuration + """ + best = np.inf + best_vector = None + + budget = max(self.kde_models.keys()) + + l = self.kde_models[budget]['good'].pdf + g = self.kde_models[budget]['bad'].pdf + + minimize_me = lambda x: max(1e-32, g(x))/max(l(x), 1e-32) + + kde_good = self.kde_models[budget]['good'] + kde_bad = self.kde_models[budget]['bad'] + + for i in range(self.num_samples): + idx = np.random.randint(0, len(kde_good.data)) + datum = kde_good.data[idx] + vector = [] + + for m, bw, t in zip(datum, kde_good.bw, self.vartypes): + + bw = max(bw, self.min_bandwidth) + if t == 0: + bw = self.bw_factor*bw + vector.append(sps.truncnorm.rvs(-m/bw, (1-m)/bw, loc=m, scale=bw)) + else: + if np.random.rand() < (1-bw): + vector.append(int(m)) + else: + vector.append(np.random.randint(t)) + val = minimize_me(vector) + + if not np.isfinite(val): + logger.warning('sampled vector: %s has EI value %s', vector, val) + logger.warning("data in the KDEs:\n%s\n%s", kde_good.data, kde_bad.data) + logger.warning("bandwidth of the KDEs:\n%s\n%s", kde_good.bw, kde_bad.bw) + logger.warning("l(x) = %s", l(vector)) + logger.warning("g(x) = %s", g(vector)) + + # right now, this happens because a KDE does not contain all values for a categorical parameter + # this cannot be fixed with the statsmodels KDE, so for now, we are just going to evaluate this one + # if the good_kde has a finite value, i.e. there is no config with that value in the bad kde, + # so it shouldn't be terrible. + if np.isfinite(l(vector)): + best_vector = vector + break + + if val < best: + best = val + best_vector = vector + + if best_vector is None: + logger.debug("Sampling based optimization with %i samples failed -> using random configuration", self.num_samples) + sample = self.configspace.sample_configuration().get_dictionary() + info_dict['model_based_pick'] = False + + else: + logger.debug('best_vector: %s, %s, %s, %s', best_vector, best, l(best_vector), g(best_vector)) + for i, _ in enumerate(best_vector): + hp = self.configspace.get_hyperparameter(self.configspace.get_hyperparameter_by_idx(i)) + if isinstance(hp, ConfigSpace.hyperparameters.CategoricalHyperparameter): + best_vector[i] = int(np.rint(best_vector[i])) + sample = ConfigSpace.Configuration(self.configspace, vector=best_vector).get_dictionary() + + sample = ConfigSpace.util.deactivate_inactive_hyperparameters( + configuration_space=self.configspace, + configuration=sample) + info_dict['model_based_pick'] = True + + return sample, info_dict + + def get_config(self, budget): + """Function to sample a new configuration + This function is called inside BOHB to query a new configuration + + Parameters: + ----------- + budget: float + the budget for which this configuration is scheduled + + Returns + ------- + config + return a valid configuration with parameters and budget + """ + logger.debug('start sampling a new configuration.') + sample = None + info_dict = {} + + # If no model is available, sample from prior + # also mix in a fraction of random configs + if not self.kde_models.keys() or np.random.rand() < self.random_fraction: + sample = self.configspace.sample_configuration() + info_dict['model_based_pick'] = False + + if sample is None: + sample, info_dict = self.sample_from_largest_budget(info_dict) + + sample = ConfigSpace.util.deactivate_inactive_hyperparameters( + configuration_space=self.configspace, + configuration=sample.get_dictionary() + ).get_dictionary() + + logger.debug('done sampling a new configuration.') + sample['TRIAL_BUDGET'] = budget + return sample + + def impute_conditional_data(self, array): + return_array = np.zeros(array.shape) + for i in range(array.shape[0]): + datum = np.copy(array[i]) + nan_indices = np.argwhere(np.isnan(datum)).flatten() + while np.any(nan_indices): + nan_idx = nan_indices[0] + valid_indices = np.argwhere(np.isfinite(array[:, nan_idx])).flatten() + if valid_indices: + # pick one of them at random and overwrite all NaN values + row_idx = np.random.choice(valid_indices) + datum[nan_indices] = array[row_idx, nan_indices] + else: + # no good point in the data has this value activated, so fill it with a valid but random value + t = self.vartypes[nan_idx] + if t == 0: + datum[nan_idx] = np.random.rand() + else: + datum[nan_idx] = np.random.randint(t) + nan_indices = np.argwhere(np.isnan(datum)).flatten() + return_array[i, :] = datum + return return_array + + def new_result(self, loss, budget, parameters, update_model=True): + """ + Function to register finished runs. Every time a run has finished, this function should be called + to register it with the loss. + + Parameters: + ----------- + loss: float + the loss of the parameters + budget: float + the budget of the parameters + parameters: dict + the parameters of this trial + update_model: bool + whether use this parameter to update BP model + + Returns + ------- + None + """ + if loss is None: + # One could skip crashed results, but we decided + # assign a +inf loss and count them as bad configurations + loss = np.inf + + if budget not in self.configs.keys(): + self.configs[budget] = [] + self.losses[budget] = [] + + # skip model building if we already have a bigger model + if max(list(self.kde_models.keys()) + [-np.inf]) > budget: + return + + # We want to get a numerical representation of the configuration in the original space + conf = ConfigSpace.Configuration(self.configspace, parameters) + self.configs[budget].append(conf.get_array()) + self.losses[budget].append(loss) + + # skip model building: + # a) if not enough points are available + if len(self.configs[budget]) <= self.min_points_in_model - 1: + logger.debug("Only %i run(s) for budget %f available, need more than %s \ + -> can't build model!", len(self.configs[budget]), budget, self.min_points_in_model+1) + return + # b) during warnm starting when we feed previous results in and only update once + if not update_model: + return + + train_configs = np.array(self.configs[budget]) + train_losses = np.array(self.losses[budget]) + + n_good = max(self.min_points_in_model, (self.top_n_percent * train_configs.shape[0])//100) + n_bad = max(self.min_points_in_model, ((100-self.top_n_percent)*train_configs.shape[0])//100) + + # Refit KDE for the current budget + idx = np.argsort(train_losses) + + train_data_good = self.impute_conditional_data(train_configs[idx[:n_good]]) + train_data_bad = self.impute_conditional_data(train_configs[idx[n_good:n_good+n_bad]]) + + if train_data_good.shape[0] <= train_data_good.shape[1]: + return + if train_data_bad.shape[0] <= train_data_bad.shape[1]: + return + + #more expensive crossvalidation method + #bw_estimation = 'cv_ls' + # quick rule of thumb + bw_estimation = 'normal_reference' + + bad_kde = sm.nonparametric.KDEMultivariate(data=train_data_bad, var_type=self.kde_vartypes, bw=bw_estimation) + good_kde = sm.nonparametric.KDEMultivariate(data=train_data_good, var_type=self.kde_vartypes, bw=bw_estimation) + + bad_kde.bw = np.clip(bad_kde.bw, self.min_bandwidth, None) + good_kde.bw = np.clip(good_kde.bw, self.min_bandwidth, None) + + self.kde_models[budget] = { + 'good': good_kde, + 'bad' : bad_kde + } + + # update probs for the categorical parameters for later sampling + logger.debug('done building a new model for budget %f based on %i/%i split\nBest loss for this budget:%f\n', + budget, n_good, n_bad, np.min(train_losses)) diff --git a/new_impl/cv/third_party/nni/algorithms/hpo/bohb_advisor/requirements.txt b/new_impl/cv/third_party/nni/algorithms/hpo/bohb_advisor/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..4c11edf7f230848d2def263005f08fcc6733596a --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/hpo/bohb_advisor/requirements.txt @@ -0,0 +1,2 @@ +ConfigSpace==0.4.7 +statsmodels==0.10.0 \ No newline at end of file diff --git a/new_impl/cv/third_party/nni/algorithms/hpo/curvefitting_assessor/__init__.py b/new_impl/cv/third_party/nni/algorithms/hpo/curvefitting_assessor/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..664d87fa8870c4535cbe364f49a4e25082d68589 --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/hpo/curvefitting_assessor/__init__.py @@ -0,0 +1,4 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .curvefitting_assessor import CurvefittingAssessor diff --git a/new_impl/cv/third_party/nni/algorithms/hpo/curvefitting_assessor/curvefitting_assessor.py b/new_impl/cv/third_party/nni/algorithms/hpo/curvefitting_assessor/curvefitting_assessor.py new file mode 100644 index 0000000000000000000000000000000000000000..885886e89b83712e7eded05fd89598a9e2232b5a --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/hpo/curvefitting_assessor/curvefitting_assessor.py @@ -0,0 +1,135 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +import datetime +from schema import Schema, Optional + +from nni import ClassArgsValidator +from nni.assessor import Assessor, AssessResult +from nni.utils import extract_scalar_history +from .model_factory import CurveModel + +logger = logging.getLogger('curvefitting_Assessor') + +class CurvefittingClassArgsValidator(ClassArgsValidator): + def validate_class_args(self, **kwargs): + Schema({ + 'epoch_num': self.range('epoch_num', int, 0, 9999), + Optional('start_step'): self.range('start_step', int, 0, 9999), + Optional('threshold'): self.range('threshold', float, 0, 9999), + Optional('gap'): self.range('gap', int, 1, 9999), + }).validate(kwargs) + +class CurvefittingAssessor(Assessor): + """CurvefittingAssessor uses learning curve fitting algorithm to predict the learning curve performance in the future. + It stops a pending trial X at step S if the trial's forecast result at target step is convergence and lower than the + best performance in the history. + + Parameters + ---------- + epoch_num : int + The total number of epoch + start_step : int + only after receiving start_step number of reported intermediate results + threshold : float + The threshold that we decide to early stop the worse performance curve. + """ + + def __init__(self, epoch_num=20, start_step=6, threshold=0.95, gap=1): + if start_step <= 0: + logger.warning('It\'s recommended to set start_step to a positive number') + # Record the target position we predict + self.target_pos = epoch_num + # Start forecasting when historical data reaches start step + self.start_step = start_step + # Record the compared threshold + self.threshold = threshold + # Record the number of gap + self.gap = gap + # Record the number of intermediate result in the lastest judgment + self.last_judgment_num = dict() + # Record the best performance + self.set_best_performance = False + self.completed_best_performance = None + self.trial_history = [] + logger.info('Successfully initials the curvefitting assessor') + + def trial_end(self, trial_job_id, success): + """update the best performance of completed trial job + + Parameters + ---------- + trial_job_id : int + trial job id + success : bool + True if succssfully finish the experiment, False otherwise + """ + if success: + if self.set_best_performance: + self.completed_best_performance = max(self.completed_best_performance, self.trial_history[-1]) + else: + self.set_best_performance = True + self.completed_best_performance = self.trial_history[-1] + logger.info('Updated complted best performance, trial job id: %s', trial_job_id) + else: + logger.info('No need to update, trial job id: %s', trial_job_id) + + def assess_trial(self, trial_job_id, trial_history): + """assess whether a trial should be early stop by curve fitting algorithm + + Parameters + ---------- + trial_job_id : int + trial job id + trial_history : list + The history performance matrix of each trial + + Returns + ------- + bool + AssessResult.Good or AssessResult.Bad + + Raises + ------ + Exception + unrecognize exception in curvefitting_assessor + """ + scalar_trial_history = extract_scalar_history(trial_history) + self.trial_history = scalar_trial_history + if not self.set_best_performance: + return AssessResult.Good + curr_step = len(scalar_trial_history) + if curr_step < self.start_step: + return AssessResult.Good + + if trial_job_id in self.last_judgment_num.keys() and curr_step - self.last_judgment_num[trial_job_id] < self.gap: + return AssessResult.Good + self.last_judgment_num[trial_job_id] = curr_step + + try: + start_time = datetime.datetime.now() + # Predict the final result + curvemodel = CurveModel(self.target_pos) + predict_y = curvemodel.predict(scalar_trial_history) + log_message = "Prediction done. Trial job id = {}, Predict value = {}".format(trial_job_id, predict_y) + if predict_y is None: + logger.info('%s, wait for more information to predict precisely', log_message) + return AssessResult.Good + else: + logger.info(log_message) + standard_performance = self.completed_best_performance * self.threshold + + end_time = datetime.datetime.now() + if (end_time - start_time).seconds > 60: + logger.warning( + 'Curve Fitting Assessor Runtime Exceeds 60s, Trial Id = %s Trial History = %s', + trial_job_id, self.trial_history + ) + + if predict_y > standard_performance: + return AssessResult.Good + return AssessResult.Bad + + except Exception as exception: + logger.exception('unrecognize exception in curvefitting_assessor %s', exception) diff --git a/new_impl/cv/third_party/nni/algorithms/hpo/curvefitting_assessor/curvefunctions.py b/new_impl/cv/third_party/nni/algorithms/hpo/curvefitting_assessor/curvefunctions.py new file mode 100644 index 0000000000000000000000000000000000000000..c7bfc3b9d817df38689d24ff317e1543ac0f88e9 --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/hpo/curvefitting_assessor/curvefunctions.py @@ -0,0 +1,296 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +A family of functions used by CurvefittingAssessor +""" + +import numpy as np + +all_models = {} +model_para = {} +model_para_num = {} + +curve_combination_models = ['vap', 'pow3', 'linear', 'logx_linear', 'dr_hill_zero_background', 'log_power', 'pow4', 'mmf', + 'exp4', 'ilog2', 'weibull', 'janoschek'] + + +def vap(x, a, b, c): + """Vapor pressure model + + Parameters + ---------- + x : int + a : float + b : float + c : float + + Returns + ------- + float + np.exp(a+b/x+c*np.log(x)) + """ + return np.exp(a+b/x+c*np.log(x)) + + +all_models['vap'] = vap +model_para['vap'] = [-0.622028, -0.470050, 0.042322] +model_para_num['vap'] = 3 + + +def pow3(x, c, a, alpha): + """pow3 + + Parameters + ---------- + x : int + c : float + a : float + alpha : float + + Returns + ------- + float + c - a * x**(-alpha) + """ + return c - a * x**(-alpha) + + +all_models['pow3'] = pow3 +model_para['pow3'] = [0.84, 0.52, 0.01] +model_para_num['pow3'] = 3 + + +def linear(x, a, b): + """linear + + Parameters + ---------- + x : int + a : float + b : float + + Returns + ------- + float + a*x + b + """ + return a*x + b + + +all_models['linear'] = linear +model_para['linear'] = [1., 0] +model_para_num['linear'] = 2 + + +def logx_linear(x, a, b): + """logx linear + + Parameters + ---------- + x : int + a : float + b : float + + Returns + ------- + float + a * np.log(x) + b + """ + x = np.log(x) + return a*x + b + + +all_models['logx_linear'] = logx_linear +model_para['logx_linear'] = [0.378106, 0.046506] +model_para_num['logx_linear'] = 2 + + +def dr_hill_zero_background(x, theta, eta, kappa): + """dr hill zero background + + Parameters + ---------- + x : int + theta : float + eta : float + kappa : float + + Returns + ------- + float + (theta* x**eta) / (kappa**eta + x**eta) + """ + return (theta * x**eta) / (kappa**eta + x**eta) + + +all_models['dr_hill_zero_background'] = dr_hill_zero_background +model_para['dr_hill_zero_background'] = [0.772320, 0.586449, 2.460843] +model_para_num['dr_hill_zero_background'] = 3 + + +def log_power(x, a, b, c): + """"logistic power + + Parameters + ---------- + x : int + a : float + b : float + c : float + + Returns + ------- + float + a/(1.+(x/np.exp(b))**c) + """ + return a/(1.+(x/np.exp(b))**c) + + +all_models['log_power'] = log_power +model_para['log_power'] = [0.77, 2.98, -0.51] +model_para_num['log_power'] = 3 + + +def pow4(x, alpha, a, b, c): + """pow4 + + Parameters + ---------- + x : int + alpha : float + a : float + b : float + c : float + + Returns + ------- + float + c - (a*x+b)**-alpha + """ + return c - (a*x+b)**-alpha + + +all_models['pow4'] = pow4 +model_para['pow4'] = [0.1, 200, 0., 0.8] +model_para_num['pow4'] = 4 + + +def mmf(x, alpha, beta, kappa, delta): + """Morgan-Mercer-Flodin + http://www.pisces-conservation.com/growthhelp/index.html?morgan_mercer_floden.htm + + Parameters + ---------- + x : int + alpha : float + beta : float + kappa : float + delta : float + + Returns + ------- + float + alpha - (alpha - beta) / (1. + (kappa * x)**delta) + """ + return alpha - (alpha - beta) / (1. + (kappa * x)**delta) + + +all_models['mmf'] = mmf +model_para['mmf'] = [0.7, 0.1, 0.01, 5] +model_para_num['mmf'] = 4 + + +def exp4(x, c, a, b, alpha): + """exp4 + + Parameters + ---------- + x : int + c : float + a : float + b : float + alpha : float + + Returns + ------- + float + c - np.exp(-a*(x**alpha)+b) + """ + return c - np.exp(-a*(x**alpha)+b) + + +all_models['exp4'] = exp4 +model_para['exp4'] = [0.7, 0.8, -0.8, 0.3] +model_para_num['exp4'] = 4 + + +def ilog2(x, c, a): + """ilog2 + + Parameters + ---------- + x : int + c : float + a : float + + Returns + ------- + float + c - a / np.log(x) + """ + return c - a / np.log(x) + + +all_models['ilog2'] = ilog2 +model_para['ilog2'] = [0.78, 0.43] +model_para_num['ilog2'] = 2 + + +def weibull(x, alpha, beta, kappa, delta): + """Weibull model + http://www.pisces-conservation.com/growthhelp/index.html?morgan_mercer_floden.htm + + Parameters + ---------- + x : int + alpha : float + beta : float + kappa : float + delta : float + + Returns + ------- + float + alpha - (alpha - beta) * np.exp(-(kappa * x)**delta) + """ + return alpha - (alpha - beta) * np.exp(-(kappa * x)**delta) + + +all_models['weibull'] = weibull +model_para['weibull'] = [0.7, 0.1, 0.01, 1] +model_para_num['weibull'] = 4 + + +def janoschek(x, a, beta, k, delta): + """http://www.pisces-conservation.com/growthhelp/janoschek.htm + + Parameters + ---------- + x : int + a : float + beta : float + k : float + delta : float + + Returns + ------- + float + a - (a - beta) * np.exp(-k*x**delta) + """ + return a - (a - beta) * np.exp(-k*x**delta) + + +all_models['janoschek'] = janoschek +model_para['janoschek'] = [0.73, 0.07, 0.355, 0.46] +model_para_num['janoschek'] = 4 diff --git a/new_impl/cv/third_party/nni/algorithms/hpo/curvefitting_assessor/model_factory.py b/new_impl/cv/third_party/nni/algorithms/hpo/curvefitting_assessor/model_factory.py new file mode 100644 index 0000000000000000000000000000000000000000..e6a6ada9976293d98a44d1d763f787a7c7c9bea1 --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/hpo/curvefitting_assessor/model_factory.py @@ -0,0 +1,330 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +import numpy as np +from scipy import optimize +from .curvefunctions import * # pylint: disable=wildcard-import,unused-wildcard-import + +# Number of curve functions we prepared, more details can be found in "curvefunctions.py" +NUM_OF_FUNCTIONS = 12 +# Number of simulation time when we do MCMC sampling +NUM_OF_SIMULATION_TIME = 20 +# Number of samples we select when we do MCMC sampling +NUM_OF_INSTANCE = 10 +# The step size of each noise when we do MCMC sampling +STEP_SIZE = 0.0005 +# Number of least fitting function, if effective function is lower than this number, we will ask for more information +LEAST_FITTED_FUNCTION = 4 + +logger = logging.getLogger('curvefitting_Assessor') + +class CurveModel: + """Build a Curve Model to predict the performance + + Algorithm: https://github.com/Microsoft/nni/blob/master/src/sdk/pynni/nni/curvefitting_assessor/README.md + + Parameters + ---------- + target_pos : int + The point we need to predict + """ + def __init__(self, target_pos): + self.target_pos = target_pos + self.trial_history = [] + self.point_num = 0 + self.effective_model = [] + self.effective_model_num = 0 + self.weight_samples = [] + + def fit_theta(self): + """use least squares to fit all default curves parameter seperately + + Returns + ------- + None + """ + x = range(1, self.point_num + 1) + y = self.trial_history + for i in range(NUM_OF_FUNCTIONS): + model = curve_combination_models[i] + try: + # The maximum number of iterations to fit is 100*(N+1), where N is the number of elements in `x0`. + if model_para_num[model] == 2: + a, b = optimize.curve_fit(all_models[model], x, y)[0] + model_para[model][0] = a + model_para[model][1] = b + elif model_para_num[model] == 3: + a, b, c = optimize.curve_fit(all_models[model], x, y)[0] + model_para[model][0] = a + model_para[model][1] = b + model_para[model][2] = c + elif model_para_num[model] == 4: + a, b, c, d = optimize.curve_fit(all_models[model], x, y)[0] + model_para[model][0] = a + model_para[model][1] = b + model_para[model][2] = c + model_para[model][3] = d + except (RuntimeError, FloatingPointError, OverflowError, ZeroDivisionError): + # Ignore exceptions caused by numerical calculations + pass + except Exception as exception: + logger.critical("Exceptions in fit_theta: %s", exception) + + def filter_curve(self): + """filter the poor performing curve + + Returns + ------- + None + """ + avg = np.sum(self.trial_history) / self.point_num + standard = avg * avg * self.point_num + predict_data = [] + tmp_model = [] + for i in range(NUM_OF_FUNCTIONS): + var = 0 + model = curve_combination_models[i] + for j in range(1, self.point_num + 1): + y = self.predict_y(model, j) + var += (y - self.trial_history[j - 1]) * (y - self.trial_history[j - 1]) + if var < standard: + predict_data.append(y) + tmp_model.append(curve_combination_models[i]) + median = np.median(predict_data) + std = np.std(predict_data) + for model in tmp_model: + y = self.predict_y(model, self.target_pos) + epsilon = self.point_num / 10 * std + if y < median + epsilon and y > median - epsilon: + self.effective_model.append(model) + self.effective_model_num = len(self.effective_model) + logger.info('List of effective model: %s', self.effective_model) + + def predict_y(self, model, pos): + """return the predict y of 'model' when epoch = pos + + Parameters + ---------- + model : string + name of the curve function model + pos : int + the epoch number of the position you want to predict + + Returns + ------- + int + The expected matrix at pos + """ + if model_para_num[model] == 2: + y = all_models[model](pos, model_para[model][0], model_para[model][1]) + elif model_para_num[model] == 3: + y = all_models[model](pos, model_para[model][0], model_para[model][1], model_para[model][2]) + elif model_para_num[model] == 4: + y = all_models[model](pos, model_para[model][0], model_para[model][1], model_para[model][2], model_para[model][3]) + return y + + def f_comb(self, pos, sample): + """return the value of the f_comb when epoch = pos + + Parameters + ---------- + pos : int + the epoch number of the position you want to predict + sample : list + sample is a (1 * NUM_OF_FUNCTIONS) matrix, representing{w1, w2, ... wk} + + Returns + ------- + int + The expected matrix at pos with all the active function's prediction + """ + ret = 0 + for i in range(self.effective_model_num): + model = self.effective_model[i] + y = self.predict_y(model, pos) + ret += sample[i] * y + return ret + + def normalize_weight(self, samples): + """normalize weight + + Parameters + ---------- + samples : list + a collection of sample, it's a (NUM_OF_INSTANCE * NUM_OF_FUNCTIONS) matrix, + representing{{w11, w12, ..., w1k}, {w21, w22, ... w2k}, ...{wk1, wk2,..., wkk}} + + Returns + ------- + list + samples after normalize weight + """ + for i in range(NUM_OF_INSTANCE): + total = 0 + for j in range(self.effective_model_num): + total += samples[i][j] + for j in range(self.effective_model_num): + samples[i][j] /= total + return samples + + def sigma_sq(self, sample): + """returns the value of sigma square, given the weight's sample + + Parameters + ---------- + sample : list + sample is a (1 * NUM_OF_FUNCTIONS) matrix, representing{w1, w2, ... wk} + + Returns + ------- + float + the value of sigma square, given the weight's sample + """ + ret = 0 + for i in range(1, self.point_num + 1): + temp = self.trial_history[i - 1] - self.f_comb(i, sample) + ret += temp * temp + return 1.0 * ret / self.point_num + + def normal_distribution(self, pos, sample): + """returns the value of normal distribution, given the weight's sample and target position + + Parameters + ---------- + pos : int + the epoch number of the position you want to predict + sample : list + sample is a (1 * NUM_OF_FUNCTIONS) matrix, representing{w1, w2, ... wk} + + Returns + ------- + float + the value of normal distribution + """ + curr_sigma_sq = self.sigma_sq(sample) + delta = self.trial_history[pos - 1] - self.f_comb(pos, sample) + return np.exp(np.square(delta) / (-2.0 * curr_sigma_sq)) / np.sqrt(2 * np.pi * np.sqrt(curr_sigma_sq)) + + def likelihood(self, samples): + """likelihood + + Parameters + ---------- + sample : list + sample is a (1 * NUM_OF_FUNCTIONS) matrix, representing{w1, w2, ... wk} + + Returns + ------- + float + likelihood + """ + ret = np.ones(NUM_OF_INSTANCE) + for i in range(NUM_OF_INSTANCE): + for j in range(1, self.point_num + 1): + ret[i] *= self.normal_distribution(j, samples[i]) + return ret + + def prior(self, samples): + """priori distribution + + Parameters + ---------- + samples : list + a collection of sample, it's a (NUM_OF_INSTANCE * NUM_OF_FUNCTIONS) matrix, + representing{{w11, w12, ..., w1k}, {w21, w22, ... w2k}, ...{wk1, wk2,..., wkk}} + + Returns + ------- + float + priori distribution + """ + ret = np.ones(NUM_OF_INSTANCE) + for i in range(NUM_OF_INSTANCE): + for j in range(self.effective_model_num): + if not samples[i][j] > 0: + ret[i] = 0 + if self.f_comb(1, samples[i]) >= self.f_comb(self.target_pos, samples[i]): + ret[i] = 0 + return ret + + def target_distribution(self, samples): + """posterior probability + + Parameters + ---------- + samples : list + a collection of sample, it's a (NUM_OF_INSTANCE * NUM_OF_FUNCTIONS) matrix, + representing{{w11, w12, ..., w1k}, {w21, w22, ... w2k}, ...{wk1, wk2,..., wkk}} + + Returns + ------- + float + posterior probability + """ + curr_likelihood = self.likelihood(samples) + curr_prior = self.prior(samples) + ret = np.ones(NUM_OF_INSTANCE) + for i in range(NUM_OF_INSTANCE): + ret[i] = curr_likelihood[i] * curr_prior[i] + return ret + + def mcmc_sampling(self): + """Adjust the weight of each function using mcmc sampling. + The initial value of each weight is evenly distribute. + Brief introduction: + (1)Definition of sample: + Sample is a (1 * NUM_OF_FUNCTIONS) matrix, representing{w1, w2, ... wk} + (2)Definition of samples: + Samples is a collection of sample, it's a (NUM_OF_INSTANCE * NUM_OF_FUNCTIONS) matrix, + representing{{w11, w12, ..., w1k}, {w21, w22, ... w2k}, ...{wk1, wk2,..., wkk}} + (3)Definition of model: + Model is the function we chose right now. Such as: 'wap', 'weibull'. + (4)Definition of pos: + Pos is the position we want to predict, corresponds to the value of epoch. + + Returns + ------- + None + """ + init_weight = np.ones((self.effective_model_num), dtype=np.float) / self.effective_model_num + self.weight_samples = np.broadcast_to(init_weight, (NUM_OF_INSTANCE, self.effective_model_num)) + for _ in range(NUM_OF_SIMULATION_TIME): + # sample new value from Q(i, j) + new_values = np.random.randn(NUM_OF_INSTANCE, self.effective_model_num) * STEP_SIZE + self.weight_samples + new_values = self.normalize_weight(new_values) + # compute alpha(i, j) = min{1, P(j)Q(j, i)/P(i)Q(i, j)} + alpha = np.minimum(1, self.target_distribution(new_values) / self.target_distribution(self.weight_samples)) + # sample u + u = np.random.rand(NUM_OF_INSTANCE) + # new value + change_value_flag = (u < alpha).astype(np.int) + for j in range(NUM_OF_INSTANCE): + new_values[j] = self.weight_samples[j] * (1 - change_value_flag[j]) + new_values[j] * change_value_flag[j] + self.weight_samples = new_values + + def predict(self, trial_history): + """predict the value of target position + + Parameters + ---------- + trial_history : list + The history performance matrix of each trial. + + Returns + ------- + float + expected final result performance of this hyperparameter config + """ + self.trial_history = trial_history + self.point_num = len(trial_history) + self.fit_theta() + self.filter_curve() + if self.effective_model_num < LEAST_FITTED_FUNCTION: + # different curve's predictions are too scattered, requires more information + return None + self.mcmc_sampling() + ret = 0 + for i in range(NUM_OF_INSTANCE): + ret += self.f_comb(self.target_pos, self.weight_samples[i]) + return ret / NUM_OF_INSTANCE diff --git a/new_impl/cv/third_party/nni/algorithms/hpo/evolution_tuner/__init__.py b/new_impl/cv/third_party/nni/algorithms/hpo/evolution_tuner/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/new_impl/cv/third_party/nni/algorithms/hpo/evolution_tuner/evolution_tuner.py b/new_impl/cv/third_party/nni/algorithms/hpo/evolution_tuner/evolution_tuner.py new file mode 100644 index 0000000000000000000000000000000000000000..9277dcca3caac30134a0b787e4ed23e48295c855 --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/hpo/evolution_tuner/evolution_tuner.py @@ -0,0 +1,283 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +evolution_tuner.py +""" + +import copy +import random +import logging + +from collections import deque +import numpy as np +from schema import Schema, Optional + +import nni +from nni import ClassArgsValidator +from nni.tuner import Tuner +from nni.utils import OptimizeMode, extract_scalar_reward, split_index, json2parameter, json2space + +logger = logging.getLogger(__name__) + +class Individual: + """ + Indicidual class to store the indv info. + + Attributes + ---------- + config : str + Search space. + info : str + The str to save information of individual. + result : float + The final metric of a individual. + """ + + def __init__(self, config=None, info=None, result=None): + """ + Parameters + ---------- + config : str + A config to represent a group of parameters. + info : str + result : float + save_dir : str + """ + self.config = config + self.result = result + self.info = info + + def __str__(self): + return "info: " + str(self.info) + \ + ", config :" + str(self.config) + ", result: " + str(self.result) + +class EvolutionClassArgsValidator(ClassArgsValidator): + def validate_class_args(self, **kwargs): + Schema({ + 'optimize_mode': self.choices('optimize_mode', 'maximize', 'minimize'), + Optional('population_size'): self.range('population_size', int, 0, 99999), + }).validate(kwargs) + +class EvolutionTuner(Tuner): + """ + EvolutionTuner is tuner using navie evolution algorithm. + """ + + def __init__(self, optimize_mode="maximize", population_size=32): + """ + Parameters + ---------- + optimize_mode : str, default 'maximize' + population_size : int + initial population size. The larger population size, + the better evolution performance. + """ + self.optimize_mode = OptimizeMode(optimize_mode) + self.population_size = population_size + + self.searchspace_json = None + self.running_trials = {} + self.num_running_trials = 0 + self.random_state = None + self.population = None + self.space = None + self.credit = 0 # record the unsatisfied trial requests + self.send_trial_callback = None + self.param_ids = deque() + + def update_search_space(self, search_space): + """ + Update search space. + + Search_space contains the information that user pre-defined. + + Parameters + ---------- + search_space : dict + """ + self.searchspace_json = search_space + self.space = json2space(self.searchspace_json) + + self.random_state = np.random.RandomState() + self.population = [] + + for _ in range(self.population_size): + self._random_generate_individual() + + def trial_end(self, parameter_id, success, **kwargs): + """ + To deal with trial failure. If a trial fails, + random generate the parameters and add into the population. + Parameters + ---------- + parameter_id : int + Unique identifier for hyper-parameters used by this trial. + success : bool + True if the trial successfully completed; False if failed or terminated. + **kwargs + Not used + """ + self.num_running_trials -= 1 + logger.info('trial (%d) end', parameter_id) + + if not success: + self.running_trials.pop(parameter_id) + self._random_generate_individual() + + if self.credit > 1: + param_id = self.param_ids.popleft() + config = self._generate_individual(param_id) + logger.debug('Send new trial (%d, %s) for reducing credit', param_id, config) + self.send_trial_callback(param_id, config) + self.credit -= 1 + self.num_running_trials += 1 + + def generate_multiple_parameters(self, parameter_id_list, **kwargs): + """ + Returns multiple sets of trial (hyper-)parameters, as iterable of serializable objects. + Parameters + ---------- + parameter_id_list : list of int + Unique identifiers for each set of requested hyper-parameters. + **kwargs + Not used + Returns + ------- + list + A list of newly generated configurations + """ + + result = [] + if 'st_callback' in kwargs: + self.send_trial_callback = kwargs['st_callback'] + else: + logger.warning('Send trial callback is not found in kwargs. Evolution tuner might not work properly.') + for parameter_id in parameter_id_list: + had_exception = False + try: + logger.debug("generating param for %s", parameter_id) + res = self.generate_parameters(parameter_id, **kwargs) + self.num_running_trials += 1 + except nni.NoMoreTrialError: + had_exception = True + if not had_exception: + result.append(res) + return result + + def _random_generate_individual(self): + is_rand = dict() + for item in self.space: + is_rand[item] = True + + config = json2parameter(self.searchspace_json, is_rand, self.random_state) + self.population.append(Individual(config=config)) + + def _generate_individual(self, parameter_id): + """ + This function will generate the config for a trial. + If at the first generation, randomly generates individuals to satisfy self.population_size. + Otherwise, random choose a pair of individuals and compare their fitnesses. + The worst of the pair will be removed. Copy the best of the pair and mutate it to generate a new individual. + + Parameters + ---------- + parameter_id : int + + Returns + ------- + dict + A group of candaidte parameters that evolution tuner generated. + """ + pos = -1 + + for i in range(len(self.population)): + if self.population[i].result is None: + pos = i + break + + if pos != -1: + indiv = copy.deepcopy(self.population[pos]) + self.population.pop(pos) + else: + random.shuffle(self.population) + # avoid only 1 individual has result + if len(self.population) > 1 and self.population[0].result < self.population[1].result: + self.population[0] = self.population[1] + + # mutation on the worse individual + space = json2space(self.searchspace_json, + self.population[0].config) + is_rand = dict() + mutation_pos = space[random.randint(0, len(space)-1)] + + for i in range(len(self.space)): + is_rand[self.space[i]] = (self.space[i] == mutation_pos) + config = json2parameter( + self.searchspace_json, is_rand, self.random_state, self.population[0].config) + + if len(self.population) > 1: + self.population.pop(1) + + indiv = Individual(config=config) + + # remove "_index" from config and save params-id + self.running_trials[parameter_id] = indiv + config = split_index(indiv.config) + return config + + + def generate_parameters(self, parameter_id, **kwargs): + """ + This function will returns a dict of trial (hyper-)parameters. + If no trial configration for now, self.credit plus 1 to send the config later + + Parameters + ---------- + parameter_id : int + + Returns + ------- + dict + One newly generated configuration. + """ + if not self.population: + raise RuntimeError('The population is empty') + + if self.num_running_trials >= self.population_size: + logger.warning("No enough trial config, population_size is suggested to be larger than trialConcurrency") + self.credit += 1 + self.param_ids.append(parameter_id) + raise nni.NoMoreTrialError('no more parameters now.') + + return self._generate_individual(parameter_id) + + def receive_trial_result(self, parameter_id, parameters, value, **kwargs): + """ + Record the result from a trial + + Parameters + ---------- + parameter_id : int + parameters : dict + value : dict/float + if value is dict, it should have "default" key. + value is final metrics of the trial. + """ + reward = extract_scalar_reward(value) + + if parameter_id not in self.running_trials: + raise RuntimeError('Received parameter_id %s not in running_trials.', parameter_id) + + # restore the paramsters contains "_index" + config = self.running_trials[parameter_id].config + self.running_trials.pop(parameter_id) + + if self.optimize_mode == OptimizeMode.Minimize: + reward = -reward + + indiv = Individual(config=config, result=reward) + self.population.append(indiv) + + def import_data(self, data): + pass diff --git a/new_impl/cv/third_party/nni/algorithms/hpo/gp_tuner/__init__.py b/new_impl/cv/third_party/nni/algorithms/hpo/gp_tuner/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/new_impl/cv/third_party/nni/algorithms/hpo/gp_tuner/gp_tuner.py b/new_impl/cv/third_party/nni/algorithms/hpo/gp_tuner/gp_tuner.py new file mode 100644 index 0000000000000000000000000000000000000000..c4e6e9a89cc6258457f647f80d24464969282357 --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/hpo/gp_tuner/gp_tuner.py @@ -0,0 +1,181 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +GPTuner is a Bayesian Optimization method where Gaussian Process is used for modeling loss functions. + +See :class:`GPTuner` for details. +""" + +import warnings +import logging +import numpy as np +from schema import Schema, Optional + +from sklearn.gaussian_process.kernels import Matern +from sklearn.gaussian_process import GaussianProcessRegressor + +from nni import ClassArgsValidator +from nni.tuner import Tuner +from nni.utils import OptimizeMode, extract_scalar_reward + +from .target_space import TargetSpace +from .util import UtilityFunction, acq_max + +logger = logging.getLogger("GP_Tuner_AutoML") + +class GPClassArgsValidator(ClassArgsValidator): + def validate_class_args(self, **kwargs): + Schema({ + Optional('optimize_mode'): self.choices('optimize_mode', 'maximize', 'minimize'), + Optional('utility'): self.choices('utility', 'ei', 'ucb', 'poi'), + Optional('kappa'): float, + Optional('xi'): float, + Optional('nu'): float, + Optional('alpha'): float, + Optional('cold_start_num'): int, + Optional('selection_num_warm_up'): int, + Optional('selection_num_starting_points'): int, + }).validate(kwargs) + +class GPTuner(Tuner): + """ + GPTuner is a Bayesian Optimization method where Gaussian Process is used for modeling loss functions. + + Parameters + ---------- + optimize_mode : str + optimize mode, 'maximize' or 'minimize', by default 'maximize' + utility : str + utility function (also called 'acquisition funcition') to use, which can be 'ei', 'ucb' or 'poi'. By default 'ei'. + kappa : float + value used by utility function 'ucb'. The bigger kappa is, the more the tuner will be exploratory. By default 5. + xi : float + used by utility function 'ei' and 'poi'. The bigger xi is, the more the tuner will be exploratory. By default 0. + nu : float + used to specify Matern kernel. The smaller nu, the less smooth the approximated function is. By default 2.5. + alpha : float + Used to specify Gaussian Process Regressor. Larger values correspond to increased noise level in the observations. + By default 1e-6. + cold_start_num : int + Number of random exploration to perform before Gaussian Process. By default 10. + selection_num_warm_up : int + Number of random points to evaluate for getting the point which maximizes the acquisition function. By default 100000 + selection_num_starting_points : int + Number of times to run L-BFGS-B from a random starting point after the warmup. By default 250. + """ + + def __init__(self, optimize_mode="maximize", utility='ei', kappa=5, xi=0, nu=2.5, alpha=1e-6, cold_start_num=10, + selection_num_warm_up=100000, selection_num_starting_points=250): + self._optimize_mode = OptimizeMode(optimize_mode) + + # utility function related + self._utility = utility + self._kappa = kappa + self._xi = xi + + # target space + self._space = None + + self._random_state = np.random.RandomState() + + # nu, alpha are GPR related params + self._gp = GaussianProcessRegressor( + kernel=Matern(nu=nu), + alpha=alpha, + normalize_y=True, + n_restarts_optimizer=25, + random_state=self._random_state + ) + # num of random evaluations before GPR + self._cold_start_num = cold_start_num + + # params for acq_max + self._selection_num_warm_up = selection_num_warm_up + self._selection_num_starting_points = selection_num_starting_points + + # num of imported data + self._supplement_data_num = 0 + + def update_search_space(self, search_space): + """ + Update the self.bounds and self.types by the search_space.json file. + + Override of the abstract method in :class:`~nni.tuner.Tuner`. + """ + self._space = TargetSpace(search_space, self._random_state) + + def generate_parameters(self, parameter_id, **kwargs): + """ + Method which provides one set of hyper-parameters. + If the number of trial result is lower than cold_start_number, GPTuner will first randomly generate some parameters. + Otherwise, choose the parameters by the Gussian Process Model. + + Override of the abstract method in :class:`~nni.tuner.Tuner`. + """ + if self._space.len() < self._cold_start_num: + results = self._space.random_sample() + else: + # Sklearn's GP throws a large number of warnings at times, but + # we don't really need to see them here. + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + self._gp.fit(self._space.params, self._space.target) + + util = UtilityFunction( + kind=self._utility, kappa=self._kappa, xi=self._xi) + + results = acq_max( + f_acq=util.utility, + gp=self._gp, + y_max=self._space.target.max(), + bounds=self._space.bounds, + space=self._space, + num_warmup=self._selection_num_warm_up, + num_starting_points=self._selection_num_starting_points + ) + + results = self._space.array_to_params(results) + logger.info("Generate paramageters:\n %s", results) + return results + + def receive_trial_result(self, parameter_id, parameters, value, **kwargs): + """ + Method invoked when a trial reports its final result. + + Override of the abstract method in :class:`~nni.tuner.Tuner`. + """ + value = extract_scalar_reward(value) + if self._optimize_mode == OptimizeMode.Minimize: + value = -value + + logger.info("Received trial result.") + logger.info("value :%s", value) + logger.info("parameter : %s", parameters) + self._space.register(parameters, value) + + def import_data(self, data): + """ + Import additional data for tuning. + + Override of the abstract method in :class:`~nni.tuner.Tuner`. + """ + _completed_num = 0 + for trial_info in data: + logger.info( + "Importing data, current processing progress %s / %s", _completed_num, len(data)) + _completed_num += 1 + assert "parameter" in trial_info + _params = trial_info["parameter"] + assert "value" in trial_info + _value = trial_info['value'] + if not _value: + logger.info( + "Useless trial data, value is %s, skip this trial data.", _value) + continue + self._supplement_data_num += 1 + _parameter_id = '_'.join( + ["ImportData", str(self._supplement_data_num)]) + self.receive_trial_result( + parameter_id=_parameter_id, parameters=_params, value=_value) + logger.info("Successfully import data to GP tuner.") diff --git a/new_impl/cv/third_party/nni/algorithms/hpo/gp_tuner/target_space.py b/new_impl/cv/third_party/nni/algorithms/hpo/gp_tuner/target_space.py new file mode 100644 index 0000000000000000000000000000000000000000..7ee52c0e9969a90f931bd9a9e43b194f354d81c4 --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/hpo/gp_tuner/target_space.py @@ -0,0 +1,295 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +Tool class to hold the param-space coordinates (X) and target values (Y). +""" + +import numpy as np +import nni.parameter_expressions as parameter_expressions + + +def _hashable(params): + """ + Transform list params to tuple format. Ensure that an point is hashable by a python dict. + + Parameters + ---------- + params : numpy array + array format of parameters + + Returns + ------- + tuple + tuple format of parameters + """ + return tuple(map(float, params)) + + +class TargetSpace(): + """ + Holds the param-space coordinates (X) and target values (Y) + + Parameters + ---------- + pbounds : dict + Dictionary with parameters names and legal values. + + random_state : int, RandomState, or None + optionally specify a seed for a random number generator, by default None. + """ + + def __init__(self, pbounds, random_state=None): + self._random_state = random_state + + # Get the name of the parameters + self._keys = sorted(pbounds) + + # Create an array with parameters bounds + self._bounds = np.array( + [item[1] for item in sorted(pbounds.items(), key=lambda x: x[0])] + ) + + # check values type + for _bound in self._bounds: + if _bound['_type'] == 'choice': + try: + [float(val) for val in _bound['_value']] + except ValueError: + raise ValueError("GP Tuner supports only numerical values") + + # preallocated memory for X and Y points + self._params = np.empty(shape=(0, self.dim)) + self._target = np.empty(shape=(0)) + + # keep track of unique points we have seen so far + self._cache = {} + + def __contains__(self, params): + """ + check if a parameter is already registered + + Parameters + ---------- + params : numpy array + + Returns + ------- + bool + True if the parameter is already registered, else false + """ + return _hashable(params) in self._cache + + def len(self): + """ + length of registered params and targets + + Returns + ------- + int + """ + assert len(self._params) == len(self._target) + return len(self._target) + + @property + def params(self): + """ + registered parameters + + Returns + ------- + numpy array + """ + return self._params + + @property + def target(self): + """ + registered target values + + Returns + ------- + numpy array + """ + return self._target + + @property + def dim(self): + """ + dimension of parameters + + Returns + ------- + int + """ + return len(self._keys) + + @property + def keys(self): + """ + keys of parameters + + Returns + ------- + numpy array + """ + return self._keys + + @property + def bounds(self): + """ + bounds of parameters + + Returns + ------- + numpy array + """ + return self._bounds + + def params_to_array(self, params): + """ + dict to array + + Parameters + ---------- + params : dict + dict format of parameters + + Returns + ------- + numpy array + array format of parameters + """ + try: + assert set(params) == set(self.keys) + except AssertionError: + raise ValueError( + "Parameters' keys ({}) do ".format(sorted(params)) + + "not match the expected set of keys ({}).".format(self.keys) + ) + return np.asarray([params[key] for key in self.keys]) + + def array_to_params(self, x): + """ + array to dict + + maintain int type if the paramters is defined as int in search_space.json + Parameters + ---------- + x : numpy array + array format of parameters + + Returns + ------- + dict + dict format of parameters + """ + try: + assert len(x) == len(self.keys) + except AssertionError: + raise ValueError( + "Size of array ({}) is different than the ".format(len(x)) + + "expected number of parameters ({}).".format(self.dim) + ) + + params = {} + for i, _bound in enumerate(self._bounds): + if _bound['_type'] == 'choice' and all(isinstance(val, int) for val in _bound['_value']): + params.update({self.keys[i]: int(x[i])}) + elif _bound['_type'] in ['randint']: + params.update({self.keys[i]: int(x[i])}) + else: + params.update({self.keys[i]: x[i]}) + + return params + + def register(self, params, target): + """ + Append a point and its target value to the known data. + + Parameters + ---------- + params : dict + parameters + + target : float + target function value + """ + + x = self.params_to_array(params) + if x in self: + print('Data point {} is not unique'.format(x)) + + # Insert data into unique dictionary + self._cache[_hashable(x.ravel())] = target + + self._params = np.concatenate([self._params, x.reshape(1, -1)]) + self._target = np.concatenate([self._target, [target]]) + + def random_sample(self): + """ + Creates a random point within the bounds of the space. + + Returns + ------- + numpy array + one groupe of parameter + """ + params = np.empty(self.dim) + for col, _bound in enumerate(self._bounds): + if _bound['_type'] == 'choice': + params[col] = parameter_expressions.choice( + _bound['_value'], self._random_state) + elif _bound['_type'] == 'randint': + params[col] = self._random_state.randint( + _bound['_value'][0], _bound['_value'][1], size=1) + elif _bound['_type'] == 'uniform': + params[col] = parameter_expressions.uniform( + _bound['_value'][0], _bound['_value'][1], self._random_state) + elif _bound['_type'] == 'quniform': + params[col] = parameter_expressions.quniform( + _bound['_value'][0], _bound['_value'][1], _bound['_value'][2], self._random_state) + elif _bound['_type'] == 'loguniform': + params[col] = parameter_expressions.loguniform( + _bound['_value'][0], _bound['_value'][1], self._random_state) + elif _bound['_type'] == 'qloguniform': + params[col] = parameter_expressions.qloguniform( + _bound['_value'][0], _bound['_value'][1], _bound['_value'][2], self._random_state) + + return params + + def max(self): + """ + Get maximum target value found and its corresponding parameters. + + Returns + ------- + dict + target value and parameters, empty dict if nothing registered + """ + try: + res = { + 'target': self.target.max(), + 'params': dict( + zip(self.keys, self.params[self.target.argmax()]) + ) + } + except ValueError: + res = {} + return res + + def res(self): + """ + Get all target values found and corresponding parameters. + + Returns + ------- + list + a list of target values and their corresponding parameters + """ + params = [dict(zip(self.keys, p)) for p in self.params] + + return [ + {"target": target, "params": param} + for target, param in zip(self.target, params) + ] diff --git a/new_impl/cv/third_party/nni/algorithms/hpo/gp_tuner/util.py b/new_impl/cv/third_party/nni/algorithms/hpo/gp_tuner/util.py new file mode 100644 index 0000000000000000000000000000000000000000..6926f988997a1e0b1d0bd6286dd7f091c38fc8da --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/hpo/gp_tuner/util.py @@ -0,0 +1,235 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +utility functions and classes for GPTuner +""" + +import warnings +import numpy as np +from scipy.stats import norm +from scipy.optimize import minimize + + +def _match_val_type(vals, bounds): + """ + Update values in the array, to match their corresponding type, make sure the value is legal. + + Parameters + ---------- + vals : numpy array + values of parameters + bounds : numpy array + list of dictionary which stores parameters names and legal values. + + Returns + ------- + vals_new : list + The closest legal value to the original value + """ + vals_new = [] + + for i, bound in enumerate(bounds): + _type = bound['_type'] + if _type == "choice": + # Find the closest integer in the array, vals_bounds + # pylint: disable=cell-var-from-loop + vals_new.append(min(bound['_value'], key=lambda x: abs(x - vals[i]))) + elif _type in ['quniform', 'randint']: + vals_new.append(np.around(vals[i])) + else: + vals_new.append(vals[i]) + + return vals_new + + +def acq_max(f_acq, gp, y_max, bounds, space, num_warmup, num_starting_points): + """ + A function to find the maximum of the acquisition function + + It uses a combination of random sampling (cheap) and the 'L-BFGS-B' + optimization method. First by sampling ``num_warmup`` points at random, + and then running L-BFGS-B from ``num_starting_points`` random starting points. + + Parameters + ---------- + f_acq : UtilityFunction.utility + The acquisition function object that return its point-wise value. + + gp : GaussianProcessRegressor + A gaussian process fitted to the relevant data. + + y_max : float + The current maximum known value of the target function. + + bounds : numpy array + The variables bounds to limit the search of the acq max. + + num_warmup : int + number of times to randomly sample the aquisition function + + num_starting_points : int + number of times to run scipy.minimize + + Returns + ------- + numpy array + The parameter which achieves max of the acquisition function. + """ + + # Warm up with random points + x_tries = [space.random_sample() + for _ in range(int(num_warmup))] + ys = f_acq(x_tries, gp=gp, y_max=y_max) + x_max = x_tries[ys.argmax()] + max_acq = ys.max() + + + # Explore the parameter space more throughly + x_seeds = [space.random_sample() for _ in range(int(num_starting_points))] + + bounds_minmax = np.array( + [[bound['_value'][0], bound['_value'][-1]] for bound in bounds]) + + for x_try in x_seeds: + # Find the minimum of minus the acquisition function + res = minimize(lambda x: -f_acq(x.reshape(1, -1), gp=gp, y_max=y_max), + x_try.reshape(1, -1), + bounds=bounds_minmax, + method="L-BFGS-B") + + # See if success + if not res.success: + continue + + # Store it if better than previous minimum(maximum). + if max_acq is None or -res.fun[0] >= max_acq: + x_max = _match_val_type(res.x, bounds) + max_acq = -res.fun[0] + + # Clip output to make sure it lies within the bounds. Due to floating + # point technicalities this is not always the case. + return np.clip(x_max, bounds_minmax[:, 0], bounds_minmax[:, 1]) + + +class UtilityFunction(): + """ + A class to compute different acquisition function values. + + Parameters + ---------- + kind : string + specification of utility function to use + kappa : float + parameter usedd for 'ucb' acquisition function + xi : float + parameter usedd for 'ei' and 'poi' acquisition function + """ + + def __init__(self, kind, kappa, xi): + self._kappa = kappa + self._xi = xi + + if kind not in ['ucb', 'ei', 'poi']: + err = "The utility function " \ + "{} has not been implemented, " \ + "please choose one of ucb, ei, or poi.".format(kind) + raise NotImplementedError(err) + self._kind = kind + + def utility(self, x, gp, y_max): + """ + return utility function + + Parameters + ---------- + x : numpy array + parameters + gp : GaussianProcessRegressor + y_max : float + maximum target value observed so far + + Returns + ------- + function + return corresponding function, return None if parameter is illegal + """ + if self._kind == 'ucb': + return self._ucb(x, gp, self._kappa) + if self._kind == 'ei': + return self._ei(x, gp, y_max, self._xi) + if self._kind == 'poi': + return self._poi(x, gp, y_max, self._xi) + return None + + @staticmethod + def _ucb(x, gp, kappa): + """ + Upper Confidence Bound (UCB) utility function + + Parameters + ---------- + x : numpy array + parameters + gp : GaussianProcessRegressor + kappa : float + + Returns + ------- + float + """ + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + mean, std = gp.predict(x, return_std=True) + + return mean + kappa * std + + @staticmethod + def _ei(x, gp, y_max, xi): + """ + Expected Improvement (EI) utility function + + Parameters + ---------- + x : numpy array + parameters + gp : GaussianProcessRegressor + y_max : float + maximum target value observed so far + xi : float + + Returns + ------- + float + """ + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + mean, std = gp.predict(x, return_std=True) + + z = (mean - y_max - xi)/std + return (mean - y_max - xi) * norm.cdf(z) + std * norm.pdf(z) + + @staticmethod + def _poi(x, gp, y_max, xi): + """ + Possibility Of Improvement (POI) utility function + + Parameters + ---------- + x : numpy array + parameters + gp : GaussianProcessRegressor + y_max : float + maximum target value observed so far + xi : float + + Returns + ------- + float + """ + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + mean, std = gp.predict(x, return_std=True) + + z = (mean - y_max - xi)/std + return norm.cdf(z) diff --git a/new_impl/cv/third_party/nni/algorithms/hpo/gridsearch_tuner/__init__.py b/new_impl/cv/third_party/nni/algorithms/hpo/gridsearch_tuner/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..2d97629a4c1a62ea79b489a7ce6bba927d6d4f83 --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/hpo/gridsearch_tuner/__init__.py @@ -0,0 +1,4 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .gridsearch_tuner import GridSearchTuner diff --git a/new_impl/cv/third_party/nni/algorithms/hpo/gridsearch_tuner/gridsearch_tuner.py b/new_impl/cv/third_party/nni/algorithms/hpo/gridsearch_tuner/gridsearch_tuner.py new file mode 100644 index 0000000000000000000000000000000000000000..8a9ab0a4eda1c390d0266f8a6b4d4e9007e70a4b --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/hpo/gridsearch_tuner/gridsearch_tuner.py @@ -0,0 +1,208 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +gridsearch_tuner.py including: + class GridSearchTuner +""" + +import copy +import logging +import numpy as np + +import nni +from nni.tuner import Tuner +from nni.utils import convert_dict2tuple + +TYPE = '_type' +CHOICE = 'choice' +VALUE = '_value' + +logger = logging.getLogger('grid_search_AutoML') + +class GridSearchTuner(Tuner): + """ + GridSearchTuner will search all the possible configures that the user define in the searchSpace. + The only acceptable types of search space are ``choice``, ``quniform``, ``randint`` + + Type ``choice`` will select one of the options. Note that it can also be nested. + + Type ``quniform`` will receive three values [``low``, ``high``, ``q``], + where [``low``, ``high``] specifies a range and ``q`` specifies the interval. + It will be sampled in a way that the first sampled value is ``low``, + and each of the following values is 'interval' larger than the value in front of it. + + Type ``randint`` gives all possible intergers in range[``low``, ``high``). Note that ``high`` is not included. + """ + + def __init__(self): + self.count = -1 + self.expanded_search_space = [] + self.supplement_data = dict() + + def _json2parameter(self, ss_spec): + """ + Generate all possible configs for hyperparameters from hyperparameter space. + + Parameters + ---------- + ss_spec : dict or list + Hyperparameter space or the ``_value`` of a hyperparameter + + Returns + ------- + list or dict + All the candidate choices of hyperparameters. for a hyperparameter, chosen_params + is a list. for multiple hyperparameters (e.g., search space), chosen_params is a dict. + """ + if isinstance(ss_spec, dict): + if '_type' in ss_spec.keys(): + _type = ss_spec['_type'] + _value = ss_spec['_value'] + chosen_params = list() + if _type == 'choice': + for value in _value: + choice = self._json2parameter(value) + if isinstance(choice, list): + chosen_params.extend(choice) + else: + chosen_params.append(choice) + elif _type == 'quniform': + chosen_params = self._parse_quniform(_value) + elif _type == 'randint': + chosen_params = self._parse_randint(_value) + else: + raise RuntimeError("Not supported type: %s" % _type) + else: + chosen_params = dict() + for key in ss_spec.keys(): + chosen_params[key] = self._json2parameter(ss_spec[key]) + return self._expand_parameters(chosen_params) + elif isinstance(ss_spec, list): + chosen_params = list() + for subspec in ss_spec[1:]: + choice = self._json2parameter(subspec) + if isinstance(choice, list): + chosen_params.extend(choice) + else: + chosen_params.append(choice) + chosen_params = list(map(lambda v: {ss_spec[0]: v}, chosen_params)) + else: + chosen_params = copy.deepcopy(ss_spec) + return chosen_params + + def _parse_quniform(self, param_value): + """ + Parse type of quniform parameter and return a list + """ + low, high, q = param_value[0], param_value[1], param_value[2] + return np.clip(np.arange(np.round(low/q), np.round(high/q)+1) * q, low, high) + + def _parse_randint(self, param_value): + """ + Parse type of randint parameter and return a list + """ + if param_value[0] >= param_value[1]: + raise ValueError("Randint should contain at least 1 candidate, but [%s, %s) contains none.", + param_value[0], param_value[1]) + return np.arange(param_value[0], param_value[1]).tolist() + + def _expand_parameters(self, para): + """ + Enumerate all possible combinations of all parameters + + Parameters + ---------- + para : dict + {key1: [v11, v12, ...], key2: [v21, v22, ...], ...} + + Returns + ------- + dict + {{key1: v11, key2: v21, ...}, {key1: v11, key2: v22, ...}, ...} + """ + if len(para) == 1: + for key, values in para.items(): + return list(map(lambda v: {key: v}, values)) + + key = list(para)[0] + values = para.pop(key) + rest_para = self._expand_parameters(para) + ret_para = list() + for val in values: + for config in rest_para: + config[key] = val + ret_para.append(copy.deepcopy(config)) + return ret_para + + def update_search_space(self, search_space): + """ + Check if the search space is valid and expand it: support only ``choice``, ``quniform``, ``randint``. + + Parameters + ---------- + search_space : dict + The format could be referred to search space spec (https://nni.readthedocs.io/en/latest/Tutorial/SearchSpaceSpec.html). + """ + self.expanded_search_space = self._json2parameter(search_space) + + def generate_parameters(self, parameter_id, **kwargs): + """ + Generate parameters for one trial. + + Parameters + ---------- + parameter_id : int + The id for the generated hyperparameter + **kwargs + Not used + + Returns + ------- + dict + One configuration from the expanded search space. + + Raises + ------ + NoMoreTrialError + If all the configurations has been sent, raise :class:`~nni.NoMoreTrialError`. + """ + self.count += 1 + while self.count <= len(self.expanded_search_space) - 1: + _params_tuple = convert_dict2tuple(self.expanded_search_space[self.count]) + if _params_tuple in self.supplement_data: + self.count += 1 + else: + return self.expanded_search_space[self.count] + raise nni.NoMoreTrialError('no more parameters now.') + + def receive_trial_result(self, parameter_id, parameters, value, **kwargs): + """ + Receive a trial's final performance result reported through :func:`~nni.report_final_result` by the trial. + GridSearchTuner does not need trial's results. + """ + pass + + def import_data(self, data): + """ + Import additional data for tuning + + Parameters + ---------- + list + A list of dictionarys, each of which has at least two keys, ``parameter`` and ``value`` + """ + _completed_num = 0 + for trial_info in data: + logger.info("Importing data, current processing progress %s / %s", _completed_num, len(data)) + _completed_num += 1 + assert "parameter" in trial_info + _params = trial_info["parameter"] + assert "value" in trial_info + _value = trial_info['value'] + if not _value: + logger.info("Useless trial data, value is %s, skip this trial data.", _value) + continue + _params_tuple = convert_dict2tuple(_params) + self.supplement_data[_params_tuple] = True + logger.info("Successfully import data to grid search tuner.") diff --git a/new_impl/cv/third_party/nni/algorithms/hpo/hyperband_advisor/__init__.py b/new_impl/cv/third_party/nni/algorithms/hpo/hyperband_advisor/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/new_impl/cv/third_party/nni/algorithms/hpo/hyperband_advisor/hyperband_advisor.py b/new_impl/cv/third_party/nni/algorithms/hpo/hyperband_advisor/hyperband_advisor.py new file mode 100644 index 0000000000000000000000000000000000000000..72e2c28e1a283cb50742c04e4a2e0c1ca41ee8d8 --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/hpo/hyperband_advisor/hyperband_advisor.py @@ -0,0 +1,434 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +hyperband_advisor.py +""" + +import copy +import logging +import math +import sys + +import json_tricks +import numpy as np +from schema import Schema, Optional + +from nni import ClassArgsValidator +from nni.runtime.common import multi_phase_enabled +from nni.runtime.msg_dispatcher_base import MsgDispatcherBase +from nni.runtime.protocol import CommandType, send +from nni.utils import NodeType, OptimizeMode, MetricType, extract_scalar_reward +from nni import parameter_expressions + +_logger = logging.getLogger(__name__) + +_next_parameter_id = 0 +_KEY = 'TRIAL_BUDGET' +_epsilon = 1e-6 + + +def create_parameter_id(): + """Create an id + + Returns + ------- + int + parameter id + """ + global _next_parameter_id + _next_parameter_id += 1 + return _next_parameter_id - 1 + + +def create_bracket_parameter_id(brackets_id, brackets_curr_decay, increased_id=-1): + """Create a full id for a specific bracket's hyperparameter configuration + + Parameters + ---------- + brackets_id: int + brackets id + brackets_curr_decay: + brackets curr decay + increased_id: int + increased id + + Returns + ------- + int + params id + """ + if increased_id == -1: + increased_id = str(create_parameter_id()) + params_id = '_'.join([str(brackets_id), + str(brackets_curr_decay), + increased_id]) + return params_id + + +def json2parameter(ss_spec, random_state): + """Randomly generate values for hyperparameters from hyperparameter space i.e., x. + + Parameters + ---------- + ss_spec: + hyperparameter space + random_state: + random operator to generate random values + + Returns + ------- + Parameter: + Parameters in this experiment + """ + if isinstance(ss_spec, dict): + if NodeType.TYPE in ss_spec.keys(): + _type = ss_spec[NodeType.TYPE] + _value = ss_spec[NodeType.VALUE] + if _type == 'choice': + _index = random_state.randint(len(_value)) + chosen_params = json2parameter(ss_spec[NodeType.VALUE][_index], random_state) + else: + chosen_params = getattr(parameter_expressions, _type)(*(_value + [random_state])) + else: + chosen_params = dict() + for key in ss_spec.keys(): + chosen_params[key] = json2parameter(ss_spec[key], random_state) + elif isinstance(ss_spec, list): + chosen_params = list() + for _, subspec in enumerate(ss_spec): + chosen_params.append(json2parameter(subspec, random_state)) + else: + chosen_params = copy.deepcopy(ss_spec) + return chosen_params + + +class Bracket(): + """A bracket in Hyperband, all the information of a bracket is managed by an instance of this class + + Parameters + ---------- + s: int + The current SH iteration index. + s_max: int + total number of SH iterations + eta: float + In each iteration, a complete run of sequential halving is executed. In it, + after evaluating each configuration on the same subset size, only a fraction of + 1/eta of them 'advances' to the next round. + R: + the budget associated with each stage + optimize_mode: str + optimize mode, 'maximize' or 'minimize' + """ + + def __init__(self, s, s_max, eta, R, optimize_mode): + self.bracket_id = s + self.s_max = s_max + self.eta = eta + self.n = math.ceil((s_max + 1) * (eta ** s) / (s + 1) - _epsilon) + self.r = R / eta ** s + self.i = 0 + self.hyper_configs = [] # [ {id: params}, {}, ... ] + self.configs_perf = [] # [ {id: [seq, acc]}, {}, ... ] + self.num_configs_to_run = [] # [ n, n, n, ... ] + self.num_finished_configs = [] # [ n, n, n, ... ] + self.optimize_mode = OptimizeMode(optimize_mode) + self.no_more_trial = False + + def is_completed(self): + """check whether this bracket has sent out all the hyperparameter configurations""" + return self.no_more_trial + + def get_n_r(self): + """return the values of n and r for the next round""" + return math.floor(self.n / self.eta ** self.i + _epsilon), math.floor(self.r * self.eta ** self.i + _epsilon) + + def increase_i(self): + """i means the ith round. Increase i by 1""" + self.i += 1 + if self.i > self.bracket_id: + self.no_more_trial = True + + def set_config_perf(self, i, parameter_id, seq, value): + """update trial's latest result with its sequence number, e.g., epoch number or batch number + + Parameters + ---------- + i: int + the ith round + parameter_id: int + the id of the trial/parameter + seq: int + sequence number, e.g., epoch number or batch number + value: int + latest result with sequence number seq + + Returns + ------- + None + """ + if parameter_id in self.configs_perf[i]: + if self.configs_perf[i][parameter_id][0] < seq: + self.configs_perf[i][parameter_id] = [seq, value] + else: + self.configs_perf[i][parameter_id] = [seq, value] + + def inform_trial_end(self, i): + """If the trial is finished and the corresponding round (i.e., i) has all its trials finished, + it will choose the top k trials for the next round (i.e., i+1) + + Parameters + ---------- + i: int + the ith round + """ + global _KEY + self.num_finished_configs[i] += 1 + _logger.debug('bracket id: %d, round: %d %d, finished: %d, all: %d', self.bracket_id, self.i, i, + self.num_finished_configs[i], self.num_configs_to_run[i]) + if self.num_finished_configs[i] >= self.num_configs_to_run[i] \ + and self.no_more_trial is False: + # choose candidate configs from finished configs to run in the next round + assert self.i == i + 1 + this_round_perf = self.configs_perf[i] + if self.optimize_mode is OptimizeMode.Maximize: + sorted_perf = sorted(this_round_perf.items(), key=lambda kv: kv[1][1], reverse=True) # reverse + else: + sorted_perf = sorted(this_round_perf.items(), key=lambda kv: kv[1][1]) + _logger.debug('bracket %s next round %s, sorted hyper configs: %s', self.bracket_id, self.i, sorted_perf) + next_n, next_r = self.get_n_r() + _logger.debug('bracket %s next round %s, next_n=%d, next_r=%d', self.bracket_id, self.i, next_n, next_r) + hyper_configs = dict() + for k in range(next_n): + params_id = sorted_perf[k][0] + params = self.hyper_configs[i][params_id] + params[_KEY] = next_r # modify r + # generate new id + increased_id = params_id.split('_')[-1] + new_id = create_bracket_parameter_id(self.bracket_id, self.i, increased_id) + hyper_configs[new_id] = params + self._record_hyper_configs(hyper_configs) + return [[key, value] for key, value in hyper_configs.items()] + return None + + def get_hyperparameter_configurations(self, num, r, searchspace_json, random_state): + """Randomly generate num hyperparameter configurations from search space + + Parameters + ---------- + num: int + the number of hyperparameter configurations + + Returns + ------- + list + a list of hyperparameter configurations. Format: [[key1, value1], [key2, value2], ...] + """ + global _KEY + assert self.i == 0 + hyperparameter_configs = dict() + for _ in range(num): + params_id = create_bracket_parameter_id(self.bracket_id, self.i) + params = json2parameter(searchspace_json, random_state) + params[_KEY] = r + hyperparameter_configs[params_id] = params + self._record_hyper_configs(hyperparameter_configs) + return [[key, value] for key, value in hyperparameter_configs.items()] + + def _record_hyper_configs(self, hyper_configs): + """after generating one round of hyperconfigs, this function records the generated hyperconfigs, + creates a dict to record the performance when those hyperconifgs are running, set the number of finished configs + in this round to be 0, and increase the round number. + + Parameters + ---------- + hyper_configs: list + the generated hyperconfigs + """ + self.hyper_configs.append(hyper_configs) + self.configs_perf.append(dict()) + self.num_finished_configs.append(0) + self.num_configs_to_run.append(len(hyper_configs)) + self.increase_i() + +class HyperbandClassArgsValidator(ClassArgsValidator): + def validate_class_args(self, **kwargs): + Schema({ + 'optimize_mode': self.choices('optimize_mode', 'maximize', 'minimize'), + Optional('R'): int, + Optional('eta'): int + }).validate(kwargs) + +class Hyperband(MsgDispatcherBase): + """Hyperband inherit from MsgDispatcherBase rather than Tuner, because it integrates both tuner's functions and assessor's functions. + This is an implementation that could fully leverage available resources, i.e., high parallelism. + A single execution of Hyperband takes a finite budget of (s_max + 1)B. + + Parameters + ---------- + R: int + the maximum amount of resource that can be allocated to a single configuration + eta: int + the variable that controls the proportion of configurations discarded in each round of SuccessiveHalving + optimize_mode: str + optimize mode, 'maximize' or 'minimize' + """ + + def __init__(self, R=60, eta=3, optimize_mode='maximize'): + """B = (s_max + 1)R""" + super(Hyperband, self).__init__() + self.R = R + self.eta = eta + self.brackets = dict() # dict of Bracket + self.generated_hyper_configs = [] # all the configs waiting for run + self.completed_hyper_configs = [] # all the completed configs + self.s_max = math.floor(math.log(self.R, self.eta) + _epsilon) + self.curr_s = self.s_max + + self.searchspace_json = None + self.random_state = None + self.optimize_mode = OptimizeMode(optimize_mode) + + # This is for the case that nnimanager requests trial config, but tuner cannot provide immediately. + # In this case, tuner increases self.credit to issue a trial config sometime later. + self.credit = 0 + + # record the latest parameter_id of the trial job trial_job_id. + # if there is no running parameter_id, self.job_id_para_id_map[trial_job_id] == None + # new trial job is added to this dict and finished trial job is removed from it. + self.job_id_para_id_map = dict() + + def handle_initialize(self, data): + """callback for initializing the advisor + Parameters + ---------- + data: dict + search space + """ + self.handle_update_search_space(data) + send(CommandType.Initialized, '') + + def handle_request_trial_jobs(self, data): + """ + Parameters + ---------- + data: int + number of trial jobs + """ + for _ in range(data): + ret = self._get_one_trial_job() + send(CommandType.NewTrialJob, json_tricks.dumps(ret)) + + def _get_one_trial_job(self): + """get one trial job, i.e., one hyperparameter configuration.""" + if not self.generated_hyper_configs: + if self.curr_s < 0: + self.curr_s = self.s_max + _logger.debug('create a new bracket, self.curr_s=%d', self.curr_s) + self.brackets[self.curr_s] = Bracket(self.curr_s, self.s_max, self.eta, self.R, self.optimize_mode) + next_n, next_r = self.brackets[self.curr_s].get_n_r() + _logger.debug('new bracket, next_n=%d, next_r=%d', next_n, next_r) + assert self.searchspace_json is not None and self.random_state is not None + generated_hyper_configs = self.brackets[self.curr_s].get_hyperparameter_configurations(next_n, next_r, + self.searchspace_json, + self.random_state) + self.generated_hyper_configs = generated_hyper_configs.copy() + self.curr_s -= 1 + + assert self.generated_hyper_configs + params = self.generated_hyper_configs.pop(0) + ret = { + 'parameter_id': params[0], + 'parameter_source': 'algorithm', + 'parameters': params[1] + } + return ret + + def handle_update_search_space(self, data): + """data: JSON object, which is search space + """ + self.searchspace_json = data + self.random_state = np.random.RandomState() + + def _handle_trial_end(self, parameter_id): + """ + Parameters + ---------- + parameter_id: parameter id of the finished config + """ + bracket_id, i, _ = parameter_id.split('_') + hyper_configs = self.brackets[int(bracket_id)].inform_trial_end(int(i)) + if hyper_configs is not None: + _logger.debug('bracket %s next round %s, hyper_configs: %s', bracket_id, i, hyper_configs) + self.generated_hyper_configs = self.generated_hyper_configs + hyper_configs + + def handle_trial_end(self, data): + """ + Parameters + ---------- + data: dict() + it has three keys: trial_job_id, event, hyper_params + trial_job_id: the id generated by training service + event: the job's state + hyper_params: the hyperparameters (a string) generated and returned by tuner + """ + hyper_params = json_tricks.loads(data['hyper_params']) + self._handle_trial_end(hyper_params['parameter_id']) + if data['trial_job_id'] in self.job_id_para_id_map: + del self.job_id_para_id_map[data['trial_job_id']] + + def handle_report_metric_data(self, data): + """ + Parameters + ---------- + data: + it is an object which has keys 'parameter_id', 'value', 'trial_job_id', 'type', 'sequence'. + + Raises + ------ + ValueError + Data type not supported + """ + if 'value' in data: + data['value'] = json_tricks.loads(data['value']) + if data['type'] == MetricType.REQUEST_PARAMETER: + assert multi_phase_enabled() + assert data['trial_job_id'] is not None + assert data['parameter_index'] is not None + assert data['trial_job_id'] in self.job_id_para_id_map + self._handle_trial_end(self.job_id_para_id_map[data['trial_job_id']]) + ret = self._get_one_trial_job() + if data['trial_job_id'] is not None: + ret['trial_job_id'] = data['trial_job_id'] + if data['parameter_index'] is not None: + ret['parameter_index'] = data['parameter_index'] + self.job_id_para_id_map[data['trial_job_id']] = ret['parameter_id'] + send(CommandType.SendTrialJobParameter, json_tricks.dumps(ret)) + else: + value = extract_scalar_reward(data['value']) + bracket_id, i, _ = data['parameter_id'].split('_') + bracket_id = int(bracket_id) + + # add to self.job_id_para_id_map here, + # because when the first parameter_id is created, trial_job_id is not known yet. + if data['trial_job_id'] in self.job_id_para_id_map: + assert self.job_id_para_id_map[data['trial_job_id']] == data['parameter_id'] + else: + self.job_id_para_id_map[data['trial_job_id']] = data['parameter_id'] + + if data['type'] == MetricType.FINAL: + # sys.maxsize indicates this value is from FINAL metric data, because data['sequence'] from FINAL metric + # and PERIODICAL metric are independent, thus, not comparable. + self.brackets[bracket_id].set_config_perf(int(i), data['parameter_id'], sys.maxsize, value) + self.completed_hyper_configs.append(data) + elif data['type'] == MetricType.PERIODICAL: + self.brackets[bracket_id].set_config_perf(int(i), data['parameter_id'], data['sequence'], value) + else: + raise ValueError('Data type not supported: {}'.format(data['type'])) + + def handle_add_customized_trial(self, data): + pass + + def handle_import_data(self, data): + pass diff --git a/new_impl/cv/third_party/nni/algorithms/hpo/hyperband_advisor/requirements.txt b/new_impl/cv/third_party/nni/algorithms/hpo/hyperband_advisor/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/new_impl/cv/third_party/nni/algorithms/hpo/hyperopt_tuner/__init__.py b/new_impl/cv/third_party/nni/algorithms/hpo/hyperopt_tuner/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/new_impl/cv/third_party/nni/algorithms/hpo/hyperopt_tuner/hyperopt_tuner.py b/new_impl/cv/third_party/nni/algorithms/hpo/hyperopt_tuner/hyperopt_tuner.py new file mode 100644 index 0000000000000000000000000000000000000000..fbe9cda13faac67c060089f68d35048a4109a789 --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/hpo/hyperopt_tuner/hyperopt_tuner.py @@ -0,0 +1,499 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +hyperopt_tuner.py +""" + +import copy +import logging + +import hyperopt as hp +import numpy as np +from schema import Optional, Schema +from nni import ClassArgsValidator +from nni.tuner import Tuner +from nni.utils import NodeType, OptimizeMode, extract_scalar_reward, split_index + +logger = logging.getLogger('hyperopt_AutoML') + + +def json2space(in_x, name=NodeType.ROOT): + """ + Change json to search space in hyperopt. + + Parameters + ---------- + in_x : dict/list/str/int/float + The part of json. + name : str + name could be NodeType.ROOT, NodeType.TYPE, NodeType.VALUE or NodeType.INDEX, NodeType.NAME. + """ + out_y = copy.deepcopy(in_x) + if isinstance(in_x, dict): + if NodeType.TYPE in in_x.keys(): + _type = in_x[NodeType.TYPE] + name = name + '-' + _type + _value = json2space(in_x[NodeType.VALUE], name=name) + if _type == 'choice': + out_y = hp.hp.choice(name, _value) + elif _type == 'randint': + out_y = hp.hp.randint(name, _value[1] - _value[0]) + else: + if _type in ['loguniform', 'qloguniform']: + _value[:2] = np.log(_value[:2]) + out_y = getattr(hp.hp, _type)(name, *_value) + else: + out_y = dict() + for key in in_x.keys(): + out_y[key] = json2space(in_x[key], name + '[%s]' % str(key)) + elif isinstance(in_x, list): + out_y = list() + for i, x_i in enumerate(in_x): + if isinstance(x_i, dict): + if NodeType.NAME not in x_i.keys(): + raise RuntimeError( + '\'_name\' key is not found in this nested search space.' + ) + out_y.append(json2space(x_i, name + '[%d]' % i)) + return out_y + + +def json2parameter(in_x, parameter, name=NodeType.ROOT): + """ + Change json to parameters. + """ + out_y = copy.deepcopy(in_x) + if isinstance(in_x, dict): + if NodeType.TYPE in in_x.keys(): + _type = in_x[NodeType.TYPE] + name = name + '-' + _type + if _type == 'choice': + _index = parameter[name] + out_y = { + NodeType.INDEX: + _index, + NodeType.VALUE: + json2parameter(in_x[NodeType.VALUE][_index], + parameter, + name=name + '[%d]' % _index) + } + else: + if _type in ['quniform', 'qloguniform']: + out_y = np.clip(parameter[name], in_x[NodeType.VALUE][0], in_x[NodeType.VALUE][1]) + elif _type == 'randint': + out_y = parameter[name] + in_x[NodeType.VALUE][0] + else: + out_y = parameter[name] + else: + out_y = dict() + for key in in_x.keys(): + out_y[key] = json2parameter(in_x[key], parameter, + name + '[%s]' % str(key)) + elif isinstance(in_x, list): + out_y = list() + for i, x_i in enumerate(in_x): + if isinstance(x_i, dict): + if NodeType.NAME not in x_i.keys(): + raise RuntimeError( + '\'_name\' key is not found in this nested search space.' + ) + out_y.append(json2parameter(x_i, parameter, name + '[%d]' % i)) + return out_y + + +def json2vals(in_x, vals, out_y, name=NodeType.ROOT): + if isinstance(in_x, dict): + if NodeType.TYPE in in_x.keys(): + _type = in_x[NodeType.TYPE] + name = name + '-' + _type + + try: + out_y[name] = vals[NodeType.INDEX] + # TODO - catch exact Exception + except Exception: + out_y[name] = vals + + if _type == 'choice': + _index = vals[NodeType.INDEX] + json2vals(in_x[NodeType.VALUE][_index], + vals[NodeType.VALUE], + out_y, + name=name + '[%d]' % _index) + if _type == 'randint': + out_y[name] -= in_x[NodeType.VALUE][0] + else: + for key in in_x.keys(): + json2vals(in_x[key], vals[key], out_y, + name + '[%s]' % str(key)) + elif isinstance(in_x, list): + for i, temp in enumerate(in_x): + # nested json + if isinstance(temp, dict): + if NodeType.NAME not in temp.keys(): + raise RuntimeError( + '\'_name\' key is not found in this nested search space.' + ) + else: + json2vals(temp, vals[i], out_y, name + '[%d]' % i) + else: + json2vals(temp, vals[i], out_y, name + '[%d]' % i) + + +def _add_index(in_x, parameter): + """ + change parameters in NNI format to parameters in hyperopt format(This function also support nested dict.). + For example, receive parameters like: + {'dropout_rate': 0.8, 'conv_size': 3, 'hidden_size': 512} + Will change to format in hyperopt, like: + {'dropout_rate': 0.8, 'conv_size': {'_index': 1, '_value': 3}, 'hidden_size': {'_index': 1, '_value': 512}} + """ + if NodeType.TYPE not in in_x: # if at the top level + out_y = dict() + for key, value in parameter.items(): + out_y[key] = _add_index(in_x[key], value) + return out_y + elif isinstance(in_x, dict): + value_type = in_x[NodeType.TYPE] + value_format = in_x[NodeType.VALUE] + if value_type == "choice": + choice_name = parameter[0] if isinstance(parameter, + list) else parameter + for pos, item in enumerate( + value_format): # here value_format is a list + if isinstance( + item, + list): # this format is ["choice_key", format_dict] + choice_key = item[0] + choice_value_format = item[1] + if choice_key == choice_name: + return { + NodeType.INDEX: pos, + NodeType.VALUE: [ + choice_name, + _add_index(choice_value_format, parameter[1]) + ] + } + elif choice_name == item: + return {NodeType.INDEX: pos, NodeType.VALUE: item} + else: + return parameter + return None # note: this is not written by original author, feel free to modify if you think it's incorrect + +class HyperoptClassArgsValidator(ClassArgsValidator): + def validate_class_args(self, **kwargs): + Schema({ + Optional('optimize_mode'): self.choices('optimize_mode', 'maximize', 'minimize'), + Optional('parallel_optimize'): bool, + Optional('constant_liar_type'): self.choices('constant_liar_type', 'min', 'max', 'mean') + }).validate(kwargs) + +class HyperoptTuner(Tuner): + """ + HyperoptTuner is a tuner which using hyperopt algorithm. + """ + + def __init__(self, algorithm_name, optimize_mode='minimize', + parallel_optimize=False, constant_liar_type='min'): + """ + Parameters + ---------- + algorithm_name : str + algorithm_name includes "tpe", "random_search" and anneal". + optimize_mode : str + parallel_optimize : bool + More detail could reference: docs/en_US/Tuner/HyperoptTuner.md + constant_liar_type : str + constant_liar_type including "min", "max" and "mean" + More detail could reference: docs/en_US/Tuner/HyperoptTuner.md + """ + self.algorithm_name = algorithm_name + self.optimize_mode = OptimizeMode(optimize_mode) + self.json = None + self.total_data = {} + self.rval = None + self.supplement_data_num = 0 + + self.parallel = parallel_optimize + if self.parallel: + self.CL_rval = None + self.constant_liar_type = constant_liar_type + self.running_data = [] + self.optimal_y = None + + def _choose_tuner(self, algorithm_name): + """ + Parameters + ---------- + algorithm_name : str + algorithm_name includes "tpe", "random_search" and anneal" + """ + if algorithm_name == 'tpe': + return hp.tpe.suggest + if algorithm_name == 'random_search': + return hp.rand.suggest + if algorithm_name == 'anneal': + return hp.anneal.suggest + raise RuntimeError('Not support tuner algorithm in hyperopt.') + + def update_search_space(self, search_space): + """ + Update search space definition in tuner by search_space in parameters. + + Will called when first setup experiemnt or update search space in WebUI. + + Parameters + ---------- + search_space : dict + """ + self.json = search_space + + search_space_instance = json2space(self.json) + rstate = np.random.RandomState() + trials = hp.Trials() + domain = hp.Domain(None, + search_space_instance, + pass_expr_memo_ctrl=None) + algorithm = self._choose_tuner(self.algorithm_name) + self.rval = hp.FMinIter(algorithm, + domain, + trials, + max_evals=-1, + rstate=rstate, + verbose=0) + self.rval.catch_eval_exceptions = False + + def generate_parameters(self, parameter_id, **kwargs): + """ + Returns a set of trial (hyper-)parameters, as a serializable object. + + Parameters + ---------- + parameter_id : int + + Returns + ------- + params : dict + """ + total_params = self.get_suggestion(random_search=False) + # avoid generating same parameter with concurrent trials because hyperopt doesn't support parallel mode + if total_params in self.total_data.values(): + # but it can cause duplicate parameter rarely + total_params = self.get_suggestion(random_search=True) + self.total_data[parameter_id] = total_params + + if self.parallel: + self.running_data.append(parameter_id) + + params = split_index(total_params) + return params + + def receive_trial_result(self, parameter_id, parameters, value, **kwargs): + """ + Record an observation of the objective function + + Parameters + ---------- + parameter_id : int + parameters : dict + value : dict/float + if value is dict, it should have "default" key. + value is final metrics of the trial. + """ + reward = extract_scalar_reward(value) + # restore the paramsters contains '_index' + if parameter_id not in self.total_data: + raise RuntimeError('Received parameter_id not in total_data.') + params = self.total_data[parameter_id] + + # code for parallel + if self.parallel: + constant_liar = kwargs.get('constant_liar', False) + + if constant_liar: + rval = self.CL_rval + else: + rval = self.rval + # ignore duplicated reported final result (due to aware of intermedate result) + if parameter_id not in self.running_data: + logger.info("Received duplicated final result with parameter id: %s", parameter_id) + return + self.running_data.remove(parameter_id) + + # update the reward of optimal_y + if self.optimal_y is None: + if self.constant_liar_type == 'mean': + self.optimal_y = [reward, 1] + else: + self.optimal_y = reward + else: + if self.constant_liar_type == 'mean': + _sum = self.optimal_y[0] + reward + _number = self.optimal_y[1] + 1 + self.optimal_y = [_sum, _number] + elif self.constant_liar_type == 'min': + self.optimal_y = min(self.optimal_y, reward) + elif self.constant_liar_type == 'max': + self.optimal_y = max(self.optimal_y, reward) + logger.debug("Update optimal_y with reward, optimal_y = %s", self.optimal_y) + else: + rval = self.rval + + + if self.optimize_mode is OptimizeMode.Maximize: + reward = -reward + + domain = rval.domain + trials = rval.trials + + new_id = len(trials) + + rval_specs = [None] + rval_results = [domain.new_result()] + rval_miscs = [dict(tid=new_id, cmd=domain.cmd, workdir=domain.workdir)] + + vals = params + idxs = dict() + + out_y = dict() + json2vals(self.json, vals, out_y) + vals = out_y + for key in domain.params: + if key in [NodeType.VALUE, NodeType.INDEX]: + continue + if key not in vals or vals[key] is None or vals[key] == []: + idxs[key] = vals[key] = [] + else: + idxs[key] = [new_id] + vals[key] = [vals[key]] + + self.miscs_update_idxs_vals(rval_miscs, + idxs, + vals, + idxs_map={new_id: new_id}, + assert_all_vals_used=False) + + trial = trials.new_trial_docs([new_id], rval_specs, rval_results, + rval_miscs)[0] + trial['result'] = {'loss': reward, 'status': 'ok'} + trial['state'] = hp.JOB_STATE_DONE + trials.insert_trial_docs([trial]) + trials.refresh() + + def miscs_update_idxs_vals(self, + miscs, + idxs, + vals, + assert_all_vals_used=True, + idxs_map=None): + """ + Unpack the idxs-vals format into the list of dictionaries that is + `misc`. + + Parameters + ---------- + idxs_map : dict + idxs_map is a dictionary of id->id mappings so that the misc['idxs'] can + contain different numbers than the idxs argument. + """ + if idxs_map is None: + idxs_map = {} + + assert set(idxs.keys()) == set(vals.keys()) + + misc_by_id = {m['tid']: m for m in miscs} + for m in miscs: + m['idxs'] = {key: [] for key in idxs} + m['vals'] = {key: [] for key in idxs} + + for key in idxs: + assert len(idxs[key]) == len(vals[key]) + for tid, val in zip(idxs[key], vals[key]): + tid = idxs_map.get(tid, tid) + if assert_all_vals_used or tid in misc_by_id: + misc_by_id[tid]['idxs'][key] = [tid] + misc_by_id[tid]['vals'][key] = [val] + + def get_suggestion(self, random_search=False): + """ + get suggestion from hyperopt + + Parameters + ---------- + random_search : bool + flag to indicate random search or not (default: {False}) + + Returns + ---------- + total_params : dict + parameter suggestion + """ + if self.parallel and len(self.total_data) > 20 and self.running_data and self.optimal_y is not None: + self.CL_rval = copy.deepcopy(self.rval) + if self.constant_liar_type == 'mean': + _constant_liar_y = self.optimal_y[0] / self.optimal_y[1] + else: + _constant_liar_y = self.optimal_y + for _parameter_id in self.running_data: + self.receive_trial_result(parameter_id=_parameter_id, parameters=None, value=_constant_liar_y, constant_liar=True) + rval = self.CL_rval + + random_state = np.random.randint(2**31 - 1) + else: + rval = self.rval + random_state = rval.rstate.randint(2**31 - 1) + + trials = rval.trials + algorithm = rval.algo + new_ids = rval.trials.new_trial_ids(1) + rval.trials.refresh() + + if random_search: + new_trials = hp.rand.suggest(new_ids, rval.domain, trials, + random_state) + else: + new_trials = algorithm(new_ids, rval.domain, trials, random_state) + rval.trials.refresh() + vals = new_trials[0]['misc']['vals'] + parameter = dict() + for key in vals: + try: + parameter[key] = vals[key][0].item() + except (KeyError, IndexError): + parameter[key] = None + + # remove '_index' from json2parameter and save params-id + total_params = json2parameter(self.json, parameter) + return total_params + + def import_data(self, data): + """ + Import additional data for tuning + + Parameters + ---------- + data: + a list of dictionarys, each of which has at least two keys, 'parameter' and 'value' + """ + _completed_num = 0 + for trial_info in data: + logger.info("Importing data, current processing progress %s / %s", _completed_num, len(data)) + _completed_num += 1 + if self.algorithm_name == 'random_search': + return + assert "parameter" in trial_info + _params = trial_info["parameter"] + assert "value" in trial_info + _value = trial_info['value'] + if not _value: + logger.info("Useless trial data, value is %s, skip this trial data.", _value) + continue + self.supplement_data_num += 1 + _parameter_id = '_'.join( + ["ImportData", str(self.supplement_data_num)]) + self.total_data[_parameter_id] = _add_index(in_x=self.json, + parameter=_params) + self.receive_trial_result(parameter_id=_parameter_id, + parameters=_params, + value=_value) + logger.info("Successfully import data to TPE/Anneal tuner.") diff --git a/new_impl/cv/third_party/nni/algorithms/hpo/medianstop_assessor/__init__.py b/new_impl/cv/third_party/nni/algorithms/hpo/medianstop_assessor/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..47668dc26d43b777e001271defcca3c02a195071 --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/hpo/medianstop_assessor/__init__.py @@ -0,0 +1,4 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .medianstop_assessor import MedianstopAssessor diff --git a/new_impl/cv/third_party/nni/algorithms/hpo/medianstop_assessor/medianstop_assessor.py b/new_impl/cv/third_party/nni/algorithms/hpo/medianstop_assessor/medianstop_assessor.py new file mode 100644 index 0000000000000000000000000000000000000000..56eb82a3c98e0ce65efaa794fa6ebea8ee989566 --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/hpo/medianstop_assessor/medianstop_assessor.py @@ -0,0 +1,125 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +from schema import Schema, Optional + +from nni import ClassArgsValidator +from nni.assessor import Assessor, AssessResult +from nni.utils import extract_scalar_history + +logger = logging.getLogger('medianstop_Assessor') + +class MedianstopClassArgsValidator(ClassArgsValidator): + def validate_class_args(self, **kwargs): + Schema({ + Optional('optimize_mode'): self.choices('optimize_mode', 'maximize', 'minimize'), + Optional('start_step'): self.range('start_step', int, 0, 9999), + }).validate(kwargs) + +class MedianstopAssessor(Assessor): + """MedianstopAssessor is The median stopping rule stops a pending trial X at step S + if the trial’s best objective value by step S is strictly worse than the median value + of the running averages of all completed trials’ objectives reported up to step S + + Parameters + ---------- + optimize_mode : str + optimize mode, 'maximize' or 'minimize' + start_step : int + only after receiving start_step number of reported intermediate results + """ + def __init__(self, optimize_mode='maximize', start_step=0): + self._start_step = start_step + self._running_history = dict() + self._completed_avg_history = dict() + if optimize_mode == 'maximize': + self._high_better = True + elif optimize_mode == 'minimize': + self._high_better = False + else: + self._high_better = True + logger.warning('unrecognized optimize_mode %s', optimize_mode) + + def _update_data(self, trial_job_id, trial_history): + """update data + + Parameters + ---------- + trial_job_id : int + trial job id + trial_history : list + The history performance matrix of each trial + """ + if trial_job_id not in self._running_history: + self._running_history[trial_job_id] = [] + self._running_history[trial_job_id].extend(trial_history[len(self._running_history[trial_job_id]):]) + + def trial_end(self, trial_job_id, success): + """trial_end + + Parameters + ---------- + trial_job_id : int + trial job id + success : bool + True if succssfully finish the experiment, False otherwise + """ + if trial_job_id in self._running_history: + if success: + cnt = 0 + history_sum = 0 + self._completed_avg_history[trial_job_id] = [] + for each in self._running_history[trial_job_id]: + cnt += 1 + history_sum += each + self._completed_avg_history[trial_job_id].append(history_sum / cnt) + self._running_history.pop(trial_job_id) + else: + logger.warning('trial_end: trial_job_id does not exist in running_history') + + def assess_trial(self, trial_job_id, trial_history): + """assess_trial + + Parameters + ---------- + trial_job_id : int + trial job id + trial_history : list + The history performance matrix of each trial + + Returns + ------- + bool + AssessResult.Good or AssessResult.Bad + + Raises + ------ + Exception + unrecognize exception in medianstop_assessor + """ + curr_step = len(trial_history) + if curr_step < self._start_step: + return AssessResult.Good + + scalar_trial_history = extract_scalar_history(trial_history) + self._update_data(trial_job_id, scalar_trial_history) + if self._high_better: + best_history = max(scalar_trial_history) + else: + best_history = min(scalar_trial_history) + + avg_array = [] + for id_ in self._completed_avg_history: + if len(self._completed_avg_history[id_]) >= curr_step: + avg_array.append(self._completed_avg_history[id_][curr_step - 1]) + if avg_array: + avg_array.sort() + if self._high_better: + median = avg_array[(len(avg_array)-1) // 2] + return AssessResult.Bad if best_history < median else AssessResult.Good + else: + median = avg_array[len(avg_array) // 2] + return AssessResult.Bad if best_history > median else AssessResult.Good + else: + return AssessResult.Good diff --git a/new_impl/cv/third_party/nni/algorithms/hpo/medianstop_assessor/test.py b/new_impl/cv/third_party/nni/algorithms/hpo/medianstop_assessor/test.py new file mode 100644 index 0000000000000000000000000000000000000000..8c7d6d927b6b514f7f0117d4424ccceaca92ae6c --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/hpo/medianstop_assessor/test.py @@ -0,0 +1,54 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import argparse +import logging +import random + +from .medianstop_assessor import MedianstopAssessor +from nni.assessor import AssessResult + +logger = logging.getLogger('nni.contrib.medianstop_assessor') +logger.debug('START') + + +def test(): + ''' + tests. + ''' + parser = argparse.ArgumentParser(description='parse command line parameters.') + parser.add_argument('--start_from', type=int, default=10, dest='start_step', + help='Assessing each trial from the step start_step.') + parser.add_argument('--optimize_mode', type=str, default='maximize', + help='Select optimize mode for Tuner: minimize or maximize.') + FLAGS, _ = parser.parse_known_args() + + lcs = [[1,1,1,1,1,1,1,1,1,1], + [2,2,2,2,2,2,2,2,2,2], + [3,3,3,3,3,3,3,3,3,3], + [4,4,4,4,4,4,4,4,4,4]] + #lcs = [[1,1,1,1,1,1,1,1,1,1], + # [1,1,1,1,1,1,1,1,1,1], + # [1,1,1,1,1,1,1,1,1,1]] + + assessor = MedianstopAssessor(FLAGS.optimize_mode, FLAGS.start_step) + for i in range(len(lcs)): + #lc = [] + to_complete = True + for k in range(len(lcs[0])): + #d = random.randint(i*100+0, i*100+100) + #lc.append(d) + ret = assessor.assess_trial(i, lcs[i][:k+1]) + print('result: %d', ret) + if ret == AssessResult.Bad: + assessor.trial_end(i, False) + to_complete = False + break + if to_complete: + assessor.trial_end(i, True) + +try: + test() +except Exception as exception: + logger.exception(exception) + raise diff --git a/new_impl/cv/third_party/nni/algorithms/hpo/metis_tuner/Regression_GMM/CreateModel.py b/new_impl/cv/third_party/nni/algorithms/hpo/metis_tuner/Regression_GMM/CreateModel.py new file mode 100644 index 0000000000000000000000000000000000000000..4846c9a67d1d4579456e5c28ce416b9e274566b4 --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/hpo/metis_tuner/Regression_GMM/CreateModel.py @@ -0,0 +1,44 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +import sys +from operator import itemgetter + +import sklearn.mixture as mm + +sys.path.insert(1, os.path.join(sys.path[0], '..')) + + +def create_model(samples_x, samples_y_aggregation, percentage_goodbatch=0.34): + ''' + Create the Gaussian Mixture Model + ''' + samples = [samples_x[i] + [samples_y_aggregation[i]] + for i in range(0, len(samples_x))] + + # Sorts so that we can get the top samples + samples = sorted(samples, key=itemgetter(-1)) + samples_goodbatch_size = int(len(samples) * percentage_goodbatch) + samples_goodbatch = samples[0:samples_goodbatch_size] + samples_badbatch = samples[samples_goodbatch_size:] + + samples_x_goodbatch = [sample_goodbatch[0:-1] + for sample_goodbatch in samples_goodbatch] + #samples_y_goodbatch = [sample_goodbatch[-1] for sample_goodbatch in samples_goodbatch] + samples_x_badbatch = [sample_badbatch[0:-1] + for sample_badbatch in samples_badbatch] + + # === Trains GMM clustering models === # + #sys.stderr.write("[%s] Train GMM's GMM model\n" % (os.path.basename(__file__))) + bgmm_goodbatch = mm.BayesianGaussianMixture( + n_components=max(1, samples_goodbatch_size - 1)) + bad_n_components = max(1, len(samples_x) - samples_goodbatch_size - 1) + bgmm_badbatch = mm.BayesianGaussianMixture(n_components=bad_n_components) + bgmm_goodbatch.fit(samples_x_goodbatch) + bgmm_badbatch.fit(samples_x_badbatch) + + model = {} + model['clusteringmodel_good'] = bgmm_goodbatch + model['clusteringmodel_bad'] = bgmm_badbatch + return model diff --git a/new_impl/cv/third_party/nni/algorithms/hpo/metis_tuner/Regression_GMM/Selection.py b/new_impl/cv/third_party/nni/algorithms/hpo/metis_tuner/Regression_GMM/Selection.py new file mode 100644 index 0000000000000000000000000000000000000000..0bca96647d4642afc7d46c8a259f1634b4c41d81 --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/hpo/metis_tuner/Regression_GMM/Selection.py @@ -0,0 +1,93 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +import random +import sys + +from .. import lib_acquisition_function +from .. import lib_constraint_summation + +sys.path.insert(1, os.path.join(sys.path[0], '..')) + + +CONSTRAINT_LOWERBOUND = None +CONSTRAINT_UPPERBOUND = None +CONSTRAINT_PARAMS_IDX = [] + + +def _ratio_scores(parameters_value, clusteringmodel_gmm_good, + clusteringmodel_gmm_bad): + ''' + The ratio is smaller the better + ''' + ratio = clusteringmodel_gmm_good.score( + [parameters_value]) / clusteringmodel_gmm_bad.score([parameters_value]) + sigma = 0 + return ratio, sigma + + +def selection_r(x_bounds, + x_types, + clusteringmodel_gmm_good, + clusteringmodel_gmm_bad, + num_starting_points=100, + minimize_constraints_fun=None): + ''' + Select using different types. + ''' + minimize_starting_points = clusteringmodel_gmm_good.sample(n_samples=num_starting_points) + + outputs = selection(x_bounds, x_types, + clusteringmodel_gmm_good, + clusteringmodel_gmm_bad, + minimize_starting_points[0], + minimize_constraints_fun) + + return outputs + + +def selection(x_bounds, + x_types, + clusteringmodel_gmm_good, + clusteringmodel_gmm_bad, + minimize_starting_points, + minimize_constraints_fun=None): + ''' + Select the lowest mu value + ''' + results = lib_acquisition_function.next_hyperparameter_lowest_mu( + _ratio_scores, [clusteringmodel_gmm_good, clusteringmodel_gmm_bad], + x_bounds, x_types, minimize_starting_points, + minimize_constraints_fun=minimize_constraints_fun) + + return results + + +def _rand_with_constraints(x_bounds, x_types): + ''' + Random generate the variable with constraints + ''' + outputs = None + x_bounds_withconstraints = [x_bounds[i] for i in CONSTRAINT_PARAMS_IDX] + x_types_withconstraints = [x_types[i] for i in CONSTRAINT_PARAMS_IDX] + x_val_withconstraints = lib_constraint_summation.rand(x_bounds_withconstraints, + x_types_withconstraints, + CONSTRAINT_LOWERBOUND, + CONSTRAINT_UPPERBOUND) + if x_val_withconstraints is not None: + outputs = [None] * len(x_bounds) + for i, _ in enumerate(CONSTRAINT_PARAMS_IDX): + outputs[CONSTRAINT_PARAMS_IDX[i]] = x_val_withconstraints[i] + for i, _ in enumerate(outputs): + if outputs[i] is None: + outputs[i] = random.randint(x_bounds[i][0], x_bounds[i][1]) + return outputs + + +def _minimize_constraints_fun_summation(x): + ''' + Minimize constraints fun summation + ''' + summation = sum([x[i] for i in CONSTRAINT_PARAMS_IDX]) + return CONSTRAINT_UPPERBOUND >= summation >= CONSTRAINT_LOWERBOUND diff --git a/new_impl/cv/third_party/nni/algorithms/hpo/metis_tuner/Regression_GMM/__init__.py b/new_impl/cv/third_party/nni/algorithms/hpo/metis_tuner/Regression_GMM/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/new_impl/cv/third_party/nni/algorithms/hpo/metis_tuner/Regression_GP/CreateModel.py b/new_impl/cv/third_party/nni/algorithms/hpo/metis_tuner/Regression_GP/CreateModel.py new file mode 100644 index 0000000000000000000000000000000000000000..98d1f22ce3c62c0491f91138e8c1df7ebd5e65ed --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/hpo/metis_tuner/Regression_GP/CreateModel.py @@ -0,0 +1,35 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +import sys +import numpy + +import sklearn.gaussian_process as gp + +sys.path.insert(1, os.path.join(sys.path[0], '..')) + + +def create_model(samples_x, samples_y_aggregation, + n_restarts_optimizer=250, is_white_kernel=False): + ''' + Trains GP regression model + ''' + kernel = gp.kernels.ConstantKernel(constant_value=1, + constant_value_bounds=(1e-12, 1e12)) * \ + gp.kernels.Matern(nu=1.5) + if is_white_kernel is True: + kernel += gp.kernels.WhiteKernel(noise_level=1, noise_level_bounds=(1e-12, 1e12)) + regressor = gp.GaussianProcessRegressor(kernel=kernel, + n_restarts_optimizer=n_restarts_optimizer, + normalize_y=True, + alpha=1e-10) + regressor.fit(numpy.array(samples_x), numpy.array(samples_y_aggregation)) + + model = {} + model['model'] = regressor + model['kernel_prior'] = str(kernel) + model['kernel_posterior'] = str(regressor.kernel_) + model['model_loglikelihood'] = regressor.log_marginal_likelihood(regressor.kernel_.theta) + + return model diff --git a/new_impl/cv/third_party/nni/algorithms/hpo/metis_tuner/Regression_GP/OutlierDetection.py b/new_impl/cv/third_party/nni/algorithms/hpo/metis_tuner/Regression_GP/OutlierDetection.py new file mode 100644 index 0000000000000000000000000000000000000000..f07a93dd3e80a8fd6f9b0e8ceb3e87a5338e915d --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/hpo/metis_tuner/Regression_GP/OutlierDetection.py @@ -0,0 +1,87 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +OutlierDectection.py +""" + +import os +import sys +from multiprocessing.dummy import Pool as ThreadPool + +from . import CreateModel as gp_create_model +from . import Prediction as gp_prediction + +sys.path.insert(1, os.path.join(sys.path[0], '..')) + + +def _outlierDetection_threaded(inputs): + """ + Detect the outlier + """ + [samples_idx, samples_x, samples_y_aggregation] = inputs + sys.stderr.write("[%s] DEBUG: Evaluating %dth of %d samples\n" + % (os.path.basename(__file__), samples_idx + 1, len(samples_x))) + outlier = None + + # Create a diagnostic regression model which removes the sample that we + # want to evaluate + diagnostic_regressor_gp = gp_create_model.create_model( + samples_x[0:samples_idx] + samples_x[samples_idx + 1:], + samples_y_aggregation[0:samples_idx] + samples_y_aggregation[samples_idx + 1:]) + mu, sigma = gp_prediction.predict( + samples_x[samples_idx], diagnostic_regressor_gp['model']) + + # 2.33 is the z-score for 98% confidence level + if abs(samples_y_aggregation[samples_idx] - mu) > (2.33 * sigma): + outlier = {"samples_idx": samples_idx, + "expected_mu": mu, + "expected_sigma": sigma, + "difference": abs(samples_y_aggregation[samples_idx] - mu) - (2.33 * sigma)} + return outlier + + +def outlierDetection_threaded(samples_x, samples_y_aggregation): + """ + Use Multi-thread to detect the outlier + """ + outliers = [] + + threads_inputs = [[samples_idx, samples_x, samples_y_aggregation] + for samples_idx in range(0, len(samples_x))] + threads_pool = ThreadPool(min(4, len(threads_inputs))) + threads_results = threads_pool.map( + _outlierDetection_threaded, threads_inputs) + threads_pool.close() + threads_pool.join() + + for threads_result in threads_results: + if threads_result is not None: + outliers.append(threads_result) + else: + print("Error: threads_result is None.") + + outliers = outliers if outliers else None + return outliers + + +def outlierDetection(samples_x, samples_y_aggregation): + outliers = [] + for samples_idx, _ in enumerate(samples_x): + #sys.stderr.write("[%s] DEBUG: Evaluating %d of %d samples\n" + # \ % (os.path.basename(__file__), samples_idx + 1, len(samples_x))) + diagnostic_regressor_gp = gp_create_model.create_model(\ + samples_x[0:samples_idx] + samples_x[samples_idx + 1:],\ + samples_y_aggregation[0:samples_idx] + samples_y_aggregation[samples_idx + 1:]) + mu, sigma = gp_prediction.predict(samples_x[samples_idx], + diagnostic_regressor_gp['model']) + # 2.33 is the z-score for 98% confidence level + if abs(samples_y_aggregation[samples_idx] - mu) > (2.33 * sigma): + outliers.append({"samples_idx": samples_idx, + "expected_mu": mu, + "expected_sigma": sigma, + "difference": \ + abs(samples_y_aggregation[samples_idx] - mu) - (2.33 * sigma)}) + + outliers = outliers if outliers else None + return outliers diff --git a/new_impl/cv/third_party/nni/algorithms/hpo/metis_tuner/Regression_GP/Prediction.py b/new_impl/cv/third_party/nni/algorithms/hpo/metis_tuner/Regression_GP/Prediction.py new file mode 100644 index 0000000000000000000000000000000000000000..08a32f0e63c86a6fd7dddad558f96c708916e41e --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/hpo/metis_tuner/Regression_GP/Prediction.py @@ -0,0 +1,19 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +import sys + +import numpy + +sys.path.insert(1, os.path.join(sys.path[0], '..')) + + +def predict(parameters_value, regressor_gp): + ''' + Predict by Gaussian Process Model + ''' + parameters_value = numpy.array(parameters_value).reshape(-1, len(parameters_value)) + mu, sigma = regressor_gp.predict(parameters_value, return_std=True) + + return mu[0], sigma[0] diff --git a/new_impl/cv/third_party/nni/algorithms/hpo/metis_tuner/Regression_GP/Selection.py b/new_impl/cv/third_party/nni/algorithms/hpo/metis_tuner/Regression_GP/Selection.py new file mode 100644 index 0000000000000000000000000000000000000000..30a41deec17eec6cd6ddf187f1a23e84a38f57e4 --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/hpo/metis_tuner/Regression_GP/Selection.py @@ -0,0 +1,97 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +import random +import sys + +from .. import lib_acquisition_function +from .. import lib_constraint_summation +from .. import lib_data +from . import Prediction as gp_prediction + +sys.path.insert(1, os.path.join(sys.path[0], '..')) + +CONSTRAINT_LOWERBOUND = None +CONSTRAINT_UPPERBOUND = None +CONSTRAINT_PARAMS_IDX = [] + + +def selection_r(acquisition_function, + samples_y_aggregation, + x_bounds, + x_types, + regressor_gp, + num_starting_points=100, + minimize_constraints_fun=None): + ''' + Selecte R value + ''' + minimize_starting_points = [lib_data.rand(x_bounds, x_types) \ + for i in range(0, num_starting_points)] + outputs = selection(acquisition_function, samples_y_aggregation, + x_bounds, x_types, regressor_gp, + minimize_starting_points, + minimize_constraints_fun=minimize_constraints_fun) + + return outputs + +def selection(acquisition_function, + samples_y_aggregation, + x_bounds, x_types, + regressor_gp, + minimize_starting_points, + minimize_constraints_fun=None): + ''' + selection + ''' + outputs = None + + sys.stderr.write("[%s] Exercise \"%s\" acquisition function\n" \ + % (os.path.basename(__file__), acquisition_function)) + + if acquisition_function == "ei": + outputs = lib_acquisition_function.next_hyperparameter_expected_improvement(\ + gp_prediction.predict, [regressor_gp], x_bounds, x_types, \ + samples_y_aggregation, minimize_starting_points, \ + minimize_constraints_fun=minimize_constraints_fun) + elif acquisition_function == "lc": + outputs = lib_acquisition_function.next_hyperparameter_lowest_confidence(\ + gp_prediction.predict, [regressor_gp], x_bounds, x_types,\ + minimize_starting_points, minimize_constraints_fun=minimize_constraints_fun) + elif acquisition_function == "lm": + outputs = lib_acquisition_function.next_hyperparameter_lowest_mu(\ + gp_prediction.predict, [regressor_gp], x_bounds, x_types,\ + minimize_starting_points, minimize_constraints_fun=minimize_constraints_fun) + return outputs + +def _rand_with_constraints(x_bounds, x_types): + ''' + Random generate with constraints + ''' + outputs = None + + x_bounds_withconstraints = [x_bounds[i] for i in CONSTRAINT_PARAMS_IDX] + x_types_withconstraints = [x_types[i] for i in CONSTRAINT_PARAMS_IDX] + x_val_withconstraints = lib_constraint_summation.rand(x_bounds_withconstraints, + x_types_withconstraints, + CONSTRAINT_LOWERBOUND, + CONSTRAINT_UPPERBOUND) + if x_val_withconstraints is not None: + outputs = [None] * len(x_bounds) + + for i, _ in enumerate(CONSTRAINT_PARAMS_IDX): + outputs[CONSTRAINT_PARAMS_IDX[i]] = x_val_withconstraints[i] + + for i, _ in enumerate(outputs): + if outputs[i] is None: + outputs[i] = random.randint(x_bounds[i][0], x_bounds[i][1]) + return outputs + + +def _minimize_constraints_fun_summation(x): + ''' + Minimize the constraints fun summation + ''' + summation = sum([x[i] for i in CONSTRAINT_PARAMS_IDX]) + return CONSTRAINT_UPPERBOUND >= summation >= CONSTRAINT_LOWERBOUND diff --git a/new_impl/cv/third_party/nni/algorithms/hpo/metis_tuner/Regression_GP/__init__.py b/new_impl/cv/third_party/nni/algorithms/hpo/metis_tuner/Regression_GP/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/new_impl/cv/third_party/nni/algorithms/hpo/metis_tuner/__init__.py b/new_impl/cv/third_party/nni/algorithms/hpo/metis_tuner/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/new_impl/cv/third_party/nni/algorithms/hpo/metis_tuner/lib_acquisition_function.py b/new_impl/cv/third_party/nni/algorithms/hpo/metis_tuner/lib_acquisition_function.py new file mode 100644 index 0000000000000000000000000000000000000000..f1b1edfe0165b61c18753dfa823bc2bc10aa8d05 --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/hpo/metis_tuner/lib_acquisition_function.py @@ -0,0 +1,197 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +lib_acquisition_function.py +""" + +import sys +import numpy + +from scipy.stats import norm +from scipy.optimize import minimize + +from . import lib_data + + +def next_hyperparameter_expected_improvement(fun_prediction, + fun_prediction_args, + x_bounds, x_types, + samples_y_aggregation, + minimize_starting_points, + minimize_constraints_fun=None): + """ + "Expected Improvement" acquisition function + """ + best_x = None + best_acquisition_value = None + x_bounds_minmax = [[i[0], i[-1]] for i in x_bounds] + x_bounds_minmax = numpy.array(x_bounds_minmax) + + for starting_point in numpy.array(minimize_starting_points): + res = minimize(fun=_expected_improvement, + x0=starting_point.reshape(1, -1), + bounds=x_bounds_minmax, + method="L-BFGS-B", + args=(fun_prediction, + fun_prediction_args, + x_bounds, + x_types, + samples_y_aggregation, + minimize_constraints_fun)) + + if (best_acquisition_value is None) or \ + (res.fun < best_acquisition_value): + res.x = numpy.ndarray.tolist(res.x) + res.x = lib_data.match_val_type(res.x, x_bounds, x_types) + if (minimize_constraints_fun is None) or \ + (minimize_constraints_fun(res.x) is True): + best_acquisition_value = res.fun + best_x = res.x + + outputs = None + if best_x is not None: + mu, sigma = fun_prediction(best_x, *fun_prediction_args) + outputs = {'hyperparameter': best_x, 'expected_mu': mu, + 'expected_sigma': sigma, 'acquisition_func': "ei"} + + return outputs + + +def _expected_improvement(x, fun_prediction, fun_prediction_args, + x_bounds, x_types, samples_y_aggregation, + minimize_constraints_fun): + # This is only for step-wise optimization + x = lib_data.match_val_type(x, x_bounds, x_types) + + expected_improvement = sys.maxsize + if (minimize_constraints_fun is None) or ( + minimize_constraints_fun(x) is True): + mu, sigma = fun_prediction(x, *fun_prediction_args) + + loss_optimum = min(samples_y_aggregation) + scaling_factor = -1 + + # In case sigma equals zero + with numpy.errstate(divide="ignore"): + Z = scaling_factor * (mu - loss_optimum) / sigma + expected_improvement = scaling_factor * (mu - loss_optimum) * \ + norm.cdf(Z) + sigma * norm.pdf(Z) + expected_improvement = 0.0 if sigma == 0.0 else expected_improvement + + # We want expected_improvement to be as large as possible + # (i.e., as small as possible for minimize(...)) + expected_improvement = -1 * expected_improvement + return expected_improvement + + +def next_hyperparameter_lowest_confidence(fun_prediction, + fun_prediction_args, + x_bounds, x_types, + minimize_starting_points, + minimize_constraints_fun=None): + """ + "Lowest Confidence" acquisition function + """ + best_x = None + best_acquisition_value = None + x_bounds_minmax = [[i[0], i[-1]] for i in x_bounds] + x_bounds_minmax = numpy.array(x_bounds_minmax) + + for starting_point in numpy.array(minimize_starting_points): + res = minimize(fun=_lowest_confidence, + x0=starting_point.reshape(1, -1), + bounds=x_bounds_minmax, + method="L-BFGS-B", + args=(fun_prediction, + fun_prediction_args, + x_bounds, + x_types, + minimize_constraints_fun)) + + if (best_acquisition_value) is None or ( + res.fun < best_acquisition_value): + res.x = numpy.ndarray.tolist(res.x) + res.x = lib_data.match_val_type(res.x, x_bounds, x_types) + if (minimize_constraints_fun is None) or ( + minimize_constraints_fun(res.x) is True): + best_acquisition_value = res.fun + best_x = res.x + + outputs = None + if best_x is not None: + mu, sigma = fun_prediction(best_x, *fun_prediction_args) + outputs = {'hyperparameter': best_x, 'expected_mu': mu, + 'expected_sigma': sigma, 'acquisition_func': "lc"} + return outputs + + +def _lowest_confidence(x, fun_prediction, fun_prediction_args, + x_bounds, x_types, minimize_constraints_fun): + # This is only for step-wise optimization + x = lib_data.match_val_type(x, x_bounds, x_types) + + ci = sys.maxsize + if (minimize_constraints_fun is None) or ( + minimize_constraints_fun(x) is True): + mu, sigma = fun_prediction(x, *fun_prediction_args) + ci = (sigma * 1.96 * 2) / mu + # We want ci to be as large as possible + # (i.e., as small as possible for minimize(...), + # because this would mean lowest confidence + ci = -1 * ci + + return ci + + +def next_hyperparameter_lowest_mu(fun_prediction, + fun_prediction_args, + x_bounds, x_types, + minimize_starting_points, + minimize_constraints_fun=None): + """ + "Lowest Mu" acquisition function + """ + best_x = None + best_acquisition_value = None + x_bounds_minmax = [[i[0], i[-1]] for i in x_bounds] + x_bounds_minmax = numpy.array(x_bounds_minmax) + + for starting_point in numpy.array(minimize_starting_points): + res = minimize(fun=_lowest_mu, + x0=starting_point.reshape(1, -1), + bounds=x_bounds_minmax, + method="L-BFGS-B", + args=(fun_prediction, fun_prediction_args, + x_bounds, x_types, minimize_constraints_fun)) + + if (best_acquisition_value is None) or ( + res.fun < best_acquisition_value): + res.x = numpy.ndarray.tolist(res.x) + res.x = lib_data.match_val_type(res.x, x_bounds, x_types) + if (minimize_constraints_fun is None) or ( + minimize_constraints_fun(res.x) is True): + best_acquisition_value = res.fun + best_x = res.x + + outputs = None + if best_x is not None: + mu, sigma = fun_prediction(best_x, *fun_prediction_args) + outputs = {'hyperparameter': best_x, 'expected_mu': mu, + 'expected_sigma': sigma, 'acquisition_func': "lm"} + return outputs + + +def _lowest_mu(x, fun_prediction, fun_prediction_args, + x_bounds, x_types, minimize_constraints_fun): + """ + Calculate the lowest mu + """ + # This is only for step-wise optimization + x = lib_data.match_val_type(x, x_bounds, x_types) + + mu = sys.maxsize + if (minimize_constraints_fun is None) or ( + minimize_constraints_fun(x) is True): + mu, _ = fun_prediction(x, *fun_prediction_args) + return mu diff --git a/new_impl/cv/third_party/nni/algorithms/hpo/metis_tuner/lib_constraint_summation.py b/new_impl/cv/third_party/nni/algorithms/hpo/metis_tuner/lib_constraint_summation.py new file mode 100644 index 0000000000000000000000000000000000000000..948be5b3ca3dcace3ae673ce9e12667c093ca7a4 --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/hpo/metis_tuner/lib_constraint_summation.py @@ -0,0 +1,108 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +lib_constraint_summation.py +""" + +import math +import random + +from operator import itemgetter + + +def check_feasibility(x_bounds, lowerbound, upperbound): + ''' + This can have false positives. + For examples, parameters can only be 0 or 5, and the summation constraint is between 6 and 7. + ''' + # x_bounds should be sorted, so even for "discrete_int" type, + # the smallest and the largest number should the first and the last element + x_bounds_lowerbound = sum([x_bound[0] for x_bound in x_bounds]) + x_bounds_upperbound = sum([x_bound[-1] for x_bound in x_bounds]) + + # return ((x_bounds_lowerbound <= lowerbound) and (x_bounds_upperbound >= lowerbound)) or \ + # ((x_bounds_lowerbound <= upperbound) and (x_bounds_upperbound >= upperbound)) + return (x_bounds_lowerbound <= lowerbound <= x_bounds_upperbound) or \ + (x_bounds_lowerbound <= upperbound <= x_bounds_upperbound) + + +def rand(x_bounds, x_types, lowerbound, upperbound, max_retries=100): + ''' + Key idea is that we try to move towards upperbound, by randomly choose one + value for each parameter. However, for the last parameter, + we need to make sure that its value can help us get above lowerbound + ''' + outputs = None + + if check_feasibility(x_bounds, lowerbound, upperbound) is True: + # Order parameters by their range size. We want the smallest range first, + # because the corresponding parameter has less numbers to choose from + x_idx_sorted = [] + for i, _ in enumerate(x_bounds): + if x_types[i] == "discrete_int": + x_idx_sorted.append([i, len(x_bounds[i])]) + elif (x_types[i] == "range_int") or (x_types[i] == "range_continuous"): + x_idx_sorted.append( + [i, math.floor(x_bounds[i][1] - x_bounds[i][0])]) + x_idx_sorted = sorted(x_idx_sorted, key=itemgetter(1)) + + for _ in range(max_retries): + budget_allocated = 0 + outputs = [None] * len(x_bounds) + + for i, _ in enumerate(x_idx_sorted): + x_idx = x_idx_sorted[i][0] + # The amount of unallocated space that we have + budget_max = upperbound - budget_allocated + # NOT the Last x that we need to assign a random number + if i < (len(x_idx_sorted) - 1): + if x_bounds[x_idx][0] <= budget_max: + if x_types[x_idx] == "discrete_int": + # Note the valid integer + temp = [] + for j in x_bounds[x_idx]: + if j <= budget_max: + temp.append(j) + # Randomly pick a number from the integer array + if temp: + outputs[x_idx] = temp[random.randint( + 0, len(temp) - 1)] + + elif (x_types[x_idx] == "range_int") or \ + (x_types[x_idx] == "range_continuous"): + outputs[x_idx] = random.randint( + x_bounds[x_idx][0], min(x_bounds[x_idx][-1], budget_max)) + + else: + # The last x that we need to assign a random number + randint_lowerbound = lowerbound - budget_allocated + randint_lowerbound = 0 if randint_lowerbound < 0 else randint_lowerbound + + # This check: + # is our smallest possible value going to overflow the available budget space, + # and is our largest possible value going to underflow the + # lower bound + if (x_bounds[x_idx][0] <= budget_max) and \ + (x_bounds[x_idx][-1] >= randint_lowerbound): + if x_types[x_idx] == "discrete_int": + temp = [] + for j in x_bounds[x_idx]: + # if (j <= budget_max) and (j >= + # randint_lowerbound): + if randint_lowerbound <= j <= budget_max: + temp.append(j) + if temp: + outputs[x_idx] = temp[random.randint( + 0, len(temp) - 1)] + elif (x_types[x_idx] == "range_int") or \ + (x_types[x_idx] == "range_continuous"): + outputs[x_idx] = random.randint( + randint_lowerbound, min( + x_bounds[x_idx][1], budget_max)) + if outputs[x_idx] is None: + break + budget_allocated += outputs[x_idx] + if None not in outputs: + break + return outputs diff --git a/new_impl/cv/third_party/nni/algorithms/hpo/metis_tuner/lib_data.py b/new_impl/cv/third_party/nni/algorithms/hpo/metis_tuner/lib_data.py new file mode 100644 index 0000000000000000000000000000000000000000..cc3f059514c8c30312acbd0c18ec000803cbe0ad --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/hpo/metis_tuner/lib_data.py @@ -0,0 +1,50 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import math +import random + + +def match_val_type(vals, vals_bounds, vals_types): + ''' + Update values in the array, to match their corresponding type + ''' + vals_new = [] + + for i, _ in enumerate(vals_types): + if vals_types[i] == "discrete_int": + # Find the closest integer in the array, vals_bounds + # pylint: disable=cell-var-from-loop + vals_new.append(min(vals_bounds[i], key=lambda x: abs(x - vals[i]))) + elif vals_types[i] == "range_int": + # Round down to the nearest integer + vals_new.append(math.floor(vals[i])) + elif vals_types[i] == "range_continuous": + # Don't do any processing for continous numbers + vals_new.append(vals[i]) + else: + return None + + return vals_new + + +def rand(x_bounds, x_types): + ''' + Random generate variable value within their bounds + ''' + outputs = [] + + for i, _ in enumerate(x_bounds): + if x_types[i] == "discrete_int": + temp = x_bounds[i][random.randint(0, len(x_bounds[i]) - 1)] + outputs.append(temp) + elif x_types[i] == "range_int": + temp = random.randint(x_bounds[i][0], x_bounds[i][1] - 1) + outputs.append(temp) + elif x_types[i] == "range_continuous": + temp = random.uniform(x_bounds[i][0], x_bounds[i][1]) + outputs.append(temp) + else: + return None + + return outputs diff --git a/new_impl/cv/third_party/nni/algorithms/hpo/metis_tuner/metis_tuner.py b/new_impl/cv/third_party/nni/algorithms/hpo/metis_tuner/metis_tuner.py new file mode 100644 index 0000000000000000000000000000000000000000..1a0670f5e517d3ce1cdf775a2590daea7366f4d9 --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/hpo/metis_tuner/metis_tuner.py @@ -0,0 +1,650 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +metis_tuner.py +""" + +import copy +import logging +import random +import statistics +import warnings +from multiprocessing.dummy import Pool as ThreadPool +import numpy as np +from schema import Schema, Optional + +from nni import ClassArgsValidator +from . import lib_constraint_summation +from . import lib_data +from .Regression_GMM import CreateModel as gmm_create_model +from .Regression_GMM import Selection as gmm_selection +from .Regression_GP import CreateModel as gp_create_model +from .Regression_GP import OutlierDetection as gp_outlier_detection +from .Regression_GP import Prediction as gp_prediction +from .Regression_GP import Selection as gp_selection +from nni.tuner import Tuner +from nni.utils import OptimizeMode, extract_scalar_reward + +logger = logging.getLogger("Metis_Tuner_AutoML") + +NONE_TYPE = '' +CONSTRAINT_LOWERBOUND = None +CONSTRAINT_UPPERBOUND = None +CONSTRAINT_PARAMS_IDX = [] + +class MetisClassArgsValidator(ClassArgsValidator): + def validate_class_args(self, **kwargs): + Schema({ + Optional('optimize_mode'): self.choices('optimize_mode', 'maximize', 'minimize'), + Optional('no_resampling'): bool, + Optional('no_candidates'): bool, + Optional('selection_num_starting_points'): int, + Optional('cold_start_num'): int, + }).validate(kwargs) + +class MetisTuner(Tuner): + """ + Metis Tuner + + More algorithm information you could reference here: + https://www.microsoft.com/en-us/research/publication/metis-robustly-tuning-tail-latencies-cloud-systems/ + + Attributes + ---------- + optimize_mode : str + optimize_mode is a string that including two mode "maximize" and "minimize" + + no_resampling : bool + True or False. + Should Metis consider re-sampling as part of the search strategy? + If you are confident that the training dataset is noise-free, + then you do not need re-sampling. + + no_candidates : bool + True or False. + Should Metis suggest parameters for the next benchmark? + If you do not plan to do more benchmarks, + Metis can skip this step. + + selection_num_starting_points : int + How many times Metis should try to find the global optimal in the search space? + The higher the number, the longer it takes to output the solution. + + cold_start_num : int + Metis need some trial result to get cold start. + when the number of trial result is less than + cold_start_num, Metis will randomly sample hyper-parameter for trial. + + exploration_probability: float + The probability of Metis to select parameter from exploration instead of exploitation. + """ + + def __init__( + self, + optimize_mode="maximize", + no_resampling=True, + no_candidates=False, + selection_num_starting_points=600, + cold_start_num=10, + exploration_probability=0.9): + """ + Parameters + ---------- + optimize_mode : str + optimize_mode is a string that including two mode "maximize" and "minimize" + + no_resampling : bool + True or False. + Should Metis consider re-sampling as part of the search strategy? + If you are confident that the training dataset is noise-free, + then you do not need re-sampling. + + no_candidates : bool + True or False. + Should Metis suggest parameters for the next benchmark? + If you do not plan to do more benchmarks, + Metis can skip this step. + + selection_num_starting_points : int + How many times Metis should try to find the global optimal in the search space? + The higher the number, the longer it takes to output the solution. + + cold_start_num : int + Metis need some trial result to get cold start. + when the number of trial result is less than + cold_start_num, Metis will randomly sample hyper-parameter for trial. + + exploration_probability : float + The probability of Metis to select parameter from exploration instead of exploitation. + + x_bounds : list + The constration of parameters. + + x_types : list + The type of parameters. + """ + + self.samples_x = [] + self.samples_y = [] + self.samples_y_aggregation = [] + self.total_data = [] + self.space = None + self.no_resampling = no_resampling + self.no_candidates = no_candidates + self.optimize_mode = OptimizeMode(optimize_mode) + self.key_order = [] + self.cold_start_num = cold_start_num + self.selection_num_starting_points = selection_num_starting_points + self.exploration_probability = exploration_probability + self.minimize_constraints_fun = None + self.minimize_starting_points = None + self.supplement_data_num = 0 + self.x_bounds = [] + self.x_types = [] + + + def update_search_space(self, search_space): + """ + Update the self.x_bounds and self.x_types by the search_space.json + + Parameters + ---------- + search_space : dict + """ + self.x_bounds = [[] for i in range(len(search_space))] + self.x_types = [NONE_TYPE for i in range(len(search_space))] + + for key in search_space: + self.key_order.append(key) + + key_type = {} + if isinstance(search_space, dict): + for key in search_space: + key_type = search_space[key]['_type'] + key_range = search_space[key]['_value'] + idx = self.key_order.index(key) + if key_type == 'quniform': + if key_range[2] == 1 and key_range[0].is_integer( + ) and key_range[1].is_integer(): + self.x_bounds[idx] = [key_range[0], key_range[1] + 1] + self.x_types[idx] = 'range_int' + else: + low, high, q = key_range + bounds = np.clip( + np.arange( + np.round( + low / q), + np.round( + high / q) + 1) * q, + low, + high) + self.x_bounds[idx] = bounds + self.x_types[idx] = 'discrete_int' + elif key_type == 'randint': + self.x_bounds[idx] = [key_range[0], key_range[1]] + self.x_types[idx] = 'range_int' + elif key_type == 'uniform': + self.x_bounds[idx] = [key_range[0], key_range[1]] + self.x_types[idx] = 'range_continuous' + elif key_type == 'choice': + self.x_bounds[idx] = key_range + + for key_value in key_range: + if not isinstance(key_value, (int, float)): + raise RuntimeError( + "Metis Tuner only support numerical choice.") + + self.x_types[idx] = 'discrete_int' + else: + logger.info( + "Metis Tuner doesn't support this kind of variable: %s", + str(key_type)) + raise RuntimeError( + "Metis Tuner doesn't support this kind of variable: %s" % + str(key_type)) + else: + logger.info("The format of search space is not a dict.") + raise RuntimeError("The format of search space is not a dict.") + + self.minimize_starting_points = _rand_init( + self.x_bounds, self.x_types, self.selection_num_starting_points) + + + def _pack_output(self, init_parameter): + """ + Pack the output + + Parameters + ---------- + init_parameter : dict + + Returns + ------- + output : dict + """ + output = {} + for i, param in enumerate(init_parameter): + output[self.key_order[i]] = param + + return output + + + def generate_parameters(self, parameter_id, **kwargs): + """ + Generate next parameter for trial + + If the number of trial result is lower than cold start number, + metis will first random generate some parameters. + Otherwise, metis will choose the parameters by + the Gussian Process Model and the Gussian Mixture Model. + + Parameters + ---------- + parameter_id : int + + Returns + ------- + result : dict + """ + if len(self.samples_x) < self.cold_start_num: + init_parameter = _rand_init(self.x_bounds, self.x_types, 1)[0] + results = self._pack_output(init_parameter) + else: + self.minimize_starting_points = _rand_init( + self.x_bounds, self.x_types, self.selection_num_starting_points) + results = self._selection( + self.samples_x, + self.samples_y_aggregation, + self.samples_y, + self.x_bounds, + self.x_types, + threshold_samplessize_resampling=( + None if self.no_resampling is True else 50), + no_candidates=self.no_candidates, + minimize_starting_points=self.minimize_starting_points, + minimize_constraints_fun=self.minimize_constraints_fun) + + logger.info("Generate paramageters: \n%s", str(results)) + return results + + + def receive_trial_result(self, parameter_id, parameters, value, **kwargs): + """ + Tuner receive result from trial. + + Parameters + ---------- + parameter_id : int + The id of parameters, generated by nni manager. + parameters : dict + A group of parameters that trial has tried. + value : dict/float + if value is dict, it should have "default" key. + """ + value = extract_scalar_reward(value) + if self.optimize_mode == OptimizeMode.Maximize: + value = -value + + logger.info("Received trial result.") + logger.info("value is : %s", str(value)) + logger.info("parameter is : %s", str(parameters)) + + # parse parameter to sample_x + sample_x = [0 for i in range(len(self.key_order))] + for key in parameters: + idx = self.key_order.index(key) + sample_x[idx] = parameters[key] + + # parse value to sample_y + temp_y = [] + if sample_x in self.samples_x: + idx = self.samples_x.index(sample_x) + temp_y = self.samples_y[idx] + temp_y.append(value) + self.samples_y[idx] = temp_y + + # calculate y aggregation + median = get_median(temp_y) + self.samples_y_aggregation[idx] = [median] + else: + self.samples_x.append(sample_x) + self.samples_y.append([value]) + + # calculate y aggregation + self.samples_y_aggregation.append([value]) + + + def _selection( + self, + samples_x, + samples_y_aggregation, + samples_y, + x_bounds, + x_types, + max_resampling_per_x=3, + threshold_samplessize_exploitation=12, + threshold_samplessize_resampling=50, + no_candidates=False, + minimize_starting_points=None, + minimize_constraints_fun=None): + + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + + next_candidate = None + candidates = [] + samples_size_all = sum([len(i) for i in samples_y]) + samples_size_unique = len(samples_y) + + # ===== STEP 1: Compute the current optimum ===== + gp_model = gp_create_model.create_model( + samples_x, samples_y_aggregation) + lm_current = gp_selection.selection( + "lm", + samples_y_aggregation, + x_bounds, + x_types, + gp_model['model'], + minimize_starting_points, + minimize_constraints_fun=minimize_constraints_fun) + if not lm_current: + return None + logger.info({ + 'hyperparameter': lm_current['hyperparameter'], + 'expected_mu': lm_current['expected_mu'], + 'expected_sigma': lm_current['expected_sigma'], + 'reason': "exploitation_gp" + }) + + if no_candidates is False: + # ===== STEP 2: Get recommended configurations for exploration ==== + results_exploration = gp_selection.selection( + "lc", + samples_y_aggregation, + x_bounds, + x_types, + gp_model['model'], + minimize_starting_points, + minimize_constraints_fun=minimize_constraints_fun) + + if results_exploration is not None: + if _num_past_samples(results_exploration['hyperparameter'], samples_x, samples_y) == 0: + temp_candidate = { + 'hyperparameter': results_exploration['hyperparameter'], + 'expected_mu': results_exploration['expected_mu'], + 'expected_sigma': results_exploration['expected_sigma'], + 'reason': "exploration" + } + candidates.append(temp_candidate) + + logger.info("DEBUG: 1 exploration candidate selected\n") + logger.info(temp_candidate) + else: + logger.info("DEBUG: No suitable exploration candidates were") + + # ===== STEP 3: Get recommended configurations for exploitation === + if samples_size_all >= threshold_samplessize_exploitation: + logger.info("Getting candidates for exploitation...\n") + try: + gmm = gmm_create_model.create_model( + samples_x, samples_y_aggregation) + + if ("discrete_int" in x_types) or ("range_int" in x_types): + results_exploitation = gmm_selection.selection( + x_bounds, + x_types, + gmm['clusteringmodel_good'], + gmm['clusteringmodel_bad'], + minimize_starting_points, + minimize_constraints_fun=minimize_constraints_fun) + else: + # If all parameters are of "range_continuous", + # let's use GMM to generate random starting points + results_exploitation = gmm_selection.selection_r( + x_bounds, + x_types, + gmm['clusteringmodel_good'], + gmm['clusteringmodel_bad'], + num_starting_points=self.selection_num_starting_points, + minimize_constraints_fun=minimize_constraints_fun) + + if results_exploitation is not None: + if _num_past_samples(results_exploitation['hyperparameter'], samples_x, samples_y) == 0: + temp_expected_mu, temp_expected_sigma = \ + gp_prediction.predict(results_exploitation['hyperparameter'], gp_model['model']) + temp_candidate = { + 'hyperparameter': results_exploitation['hyperparameter'], + 'expected_mu': temp_expected_mu, + 'expected_sigma': temp_expected_sigma, + 'reason': "exploitation_gmm" + } + candidates.append(temp_candidate) + + logger.info( + "DEBUG: 1 exploitation_gmm candidate selected\n") + logger.info(temp_candidate) + else: + logger.info( + "DEBUG: No suitable exploitation_gmm candidates were found\n") + + except ValueError as exception: + # The exception: ValueError: Fitting the mixture model failed + # because some components have ill-defined empirical covariance + # (for instance caused by singleton or collapsed samples). + # Try to decrease the number of components, or increase + # reg_covar. + logger.info( + "DEBUG: No suitable exploitation_gmm \ + candidates were found due to exception.") + logger.info(exception) + + # ===== STEP 4: Get a list of outliers ===== + if (threshold_samplessize_resampling is not None) and \ + (samples_size_unique >= threshold_samplessize_resampling): + logger.info("Getting candidates for re-sampling...\n") + results_outliers = gp_outlier_detection.outlierDetection_threaded( + samples_x, samples_y_aggregation) + + if results_outliers is not None: + for results_outlier in results_outliers: # pylint: disable=not-an-iterable + if _num_past_samples(samples_x[results_outlier['samples_idx']], samples_x, samples_y) < max_resampling_per_x: + temp_candidate = {'hyperparameter': samples_x[results_outlier['samples_idx']],\ + 'expected_mu': results_outlier['expected_mu'],\ + 'expected_sigma': results_outlier['expected_sigma'],\ + 'reason': "resampling"} + candidates.append(temp_candidate) + logger.info("DEBUG: %d re-sampling candidates selected\n") + logger.info(temp_candidate) + else: + logger.info( + "DEBUG: No suitable resampling candidates were found\n") + + if candidates: + # ===== STEP 5: Compute the information gain of each candidate + logger.info( + "Evaluating information gain of %d candidates...\n") + next_improvement = 0 + + threads_inputs = [[ + candidate, samples_x, samples_y, x_bounds, x_types, + minimize_constraints_fun, minimize_starting_points + ] for candidate in candidates] + threads_pool = ThreadPool(4) + # Evaluate what would happen if we actually sample each + # candidate + threads_results = threads_pool.map( + _calculate_lowest_mu_threaded, threads_inputs) + threads_pool.close() + threads_pool.join() + + for threads_result in threads_results: + if threads_result['expected_lowest_mu'] < lm_current['expected_mu']: + # Information gain + temp_improvement = threads_result['expected_lowest_mu'] - \ + lm_current['expected_mu'] + + if next_improvement > temp_improvement: + next_improvement = temp_improvement + next_candidate = threads_result['candidate'] + else: + # ===== STEP 6: If we have no candidates, randomly pick one === + logger.info( + "DEBUG: No candidates from exploration, exploitation,\ + and resampling. We will random a candidate for next_candidate\n" + ) + + next_candidate = _rand_with_constraints( + x_bounds, + x_types) if minimize_starting_points is None else minimize_starting_points[0] + next_candidate = lib_data.match_val_type( + next_candidate, x_bounds, x_types) + expected_mu, expected_sigma = gp_prediction.predict( + next_candidate, gp_model['model']) + next_candidate = { + 'hyperparameter': next_candidate, + 'reason': "random", + 'expected_mu': expected_mu, + 'expected_sigma': expected_sigma} + + # STEP 7: If current optimal hyperparameter occurs in the history + # or exploration probability is less than the threshold, take next + # config as exploration step + outputs = self._pack_output(lm_current['hyperparameter']) + ap = random.uniform(0, 1) + if outputs in self.total_data or ap <= self.exploration_probability: + if next_candidate is not None: + outputs = self._pack_output(next_candidate['hyperparameter']) + else: + random_parameter = _rand_init(x_bounds, x_types, 1)[0] + outputs = self._pack_output(random_parameter) + self.total_data.append(outputs) + return outputs + + def import_data(self, data): + """ + Import additional data for tuning + + Parameters + ---------- + data : a list of dict + each of which has at least two keys: 'parameter' and 'value'. + """ + _completed_num = 0 + for trial_info in data: + logger.info("Importing data, current processing progress %s / %s", _completed_num, len(data)) + _completed_num += 1 + assert "parameter" in trial_info + _params = trial_info["parameter"] + assert "value" in trial_info + _value = trial_info['value'] + if not _value: + logger.info("Useless trial data, value is %s, skip this trial data.", _value) + continue + self.supplement_data_num += 1 + _parameter_id = '_'.join( + ["ImportData", str(self.supplement_data_num)]) + self.total_data.append(_params) + self.receive_trial_result( + parameter_id=_parameter_id, + parameters=_params, + value=_value) + logger.info("Successfully import data to metis tuner.") + + +def _rand_with_constraints(x_bounds, x_types): + outputs = None + x_bounds_withconstraints = [x_bounds[i] for i in CONSTRAINT_PARAMS_IDX] + x_types_withconstraints = [x_types[i] for i in CONSTRAINT_PARAMS_IDX] + + x_val_withconstraints = lib_constraint_summation.rand( + x_bounds_withconstraints, + x_types_withconstraints, + CONSTRAINT_LOWERBOUND, + CONSTRAINT_UPPERBOUND) + if not x_val_withconstraints: + outputs = [None] * len(x_bounds) + + for i, _ in enumerate(CONSTRAINT_PARAMS_IDX): + outputs[CONSTRAINT_PARAMS_IDX[i]] = x_val_withconstraints[i] + + for i, output in enumerate(outputs): + if not output: + outputs[i] = random.randint(x_bounds[i][0], x_bounds[i][1]) + return outputs + + +def _calculate_lowest_mu_threaded(inputs): + [candidate, samples_x, samples_y, x_bounds, x_types, + minimize_constraints_fun, minimize_starting_points] = inputs + + outputs = {"candidate": candidate, "expected_lowest_mu": None} + + for expected_mu in [ + candidate['expected_mu'] + + 1.96 * + candidate['expected_sigma'], + candidate['expected_mu'] - + 1.96 * + candidate['expected_sigma']]: + temp_samples_x = copy.deepcopy(samples_x) + temp_samples_y = copy.deepcopy(samples_y) + + try: + idx = temp_samples_x.index(candidate['hyperparameter']) + # This handles the case of re-sampling a potential outlier + temp_samples_y[idx].append(expected_mu) + except ValueError: + temp_samples_x.append(candidate['hyperparameter']) + temp_samples_y.append([expected_mu]) + + # Aggregates multiple observation of the sample sampling points + temp_y_aggregation = [statistics.median( + temp_sample_y) for temp_sample_y in temp_samples_y] + temp_gp = gp_create_model.create_model( + temp_samples_x, temp_y_aggregation) + temp_results = gp_selection.selection( + "lm", + temp_y_aggregation, + x_bounds, + x_types, + temp_gp['model'], + minimize_starting_points, + minimize_constraints_fun=minimize_constraints_fun) + + if outputs["expected_lowest_mu"] is None \ + or outputs["expected_lowest_mu"] > temp_results['expected_mu']: + outputs["expected_lowest_mu"] = temp_results['expected_mu'] + + return outputs + + +def _num_past_samples(x, samples_x, samples_y): + try: + idx = samples_x.index(x) + return len(samples_y[idx]) + except ValueError: + logger.info("x not in sample_x") + return 0 + + +def _rand_init(x_bounds, x_types, selection_num_starting_points): + ''' + Random sample some init seed within bounds. + ''' + return [lib_data.rand(x_bounds, x_types) for i + in range(0, selection_num_starting_points)] + + +def get_median(temp_list): + """ + Return median + """ + num = len(temp_list) + temp_list.sort() + print(temp_list) + if num % 2 == 0: + median = (temp_list[int(num / 2)] + temp_list[int(num / 2) - 1]) / 2 + else: + median = temp_list[int(num / 2)] + return median diff --git a/new_impl/cv/third_party/nni/algorithms/hpo/metis_tuner/requirments.txt b/new_impl/cv/third_party/nni/algorithms/hpo/metis_tuner/requirments.txt new file mode 100644 index 0000000000000000000000000000000000000000..3dfc2232a18c19a544eb3fb5b2f132e185958f98 --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/hpo/metis_tuner/requirments.txt @@ -0,0 +1 @@ +scikit-learn>=0.23.2 diff --git a/new_impl/cv/third_party/nni/algorithms/hpo/networkmorphism_tuner/__init__.py b/new_impl/cv/third_party/nni/algorithms/hpo/networkmorphism_tuner/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/new_impl/cv/third_party/nni/algorithms/hpo/networkmorphism_tuner/bayesian.py b/new_impl/cv/third_party/nni/algorithms/hpo/networkmorphism_tuner/bayesian.py new file mode 100644 index 0000000000000000000000000000000000000000..54c1996dc7708568a7065325d24a8b83fdb5a40f --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/hpo/networkmorphism_tuner/bayesian.py @@ -0,0 +1,482 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import math +import random +from copy import deepcopy +from functools import total_ordering +from queue import PriorityQueue + +import numpy as np +from scipy.linalg import LinAlgError, cho_solve, cholesky, solve_triangular +from scipy.optimize import linear_sum_assignment +from sklearn.metrics.pairwise import rbf_kernel + +from nni.utils import OptimizeMode +from .graph_transformer import transform +from .utils import Constant +from .layers import is_layer + + +def layer_distance(a, b): + """The distance between two layers.""" + # pylint: disable=unidiomatic-typecheck + if not isinstance(a, type(b)): + return 1.0 + if is_layer(a, "Conv"): + att_diff = [ + (a.filters, b.filters), + (a.kernel_size, b.kernel_size), + (a.stride, b.stride), + ] + return attribute_difference(att_diff) + if is_layer(a, "Pooling"): + att_diff = [ + (a.padding, b.padding), + (a.kernel_size, b.kernel_size), + (a.stride, b.stride), + ] + return attribute_difference(att_diff) + return 0.0 + + +def attribute_difference(att_diff): + ''' The attribute distance. + ''' + + ret = 0 + for a_value, b_value in att_diff: + if max(a_value, b_value) == 0: + ret += 0 + else: + ret += abs(a_value - b_value) * 1.0 / max(a_value, b_value) + return ret * 1.0 / len(att_diff) + + +def layers_distance(list_a, list_b): + """The distance between the layers of two neural networks.""" + len_a = len(list_a) + len_b = len(list_b) + f = np.zeros((len_a + 1, len_b + 1)) + f[-1][-1] = 0 + for i in range(-1, len_a): + f[i][-1] = i + 1 + for j in range(-1, len_b): + f[-1][j] = j + 1 + for i in range(len_a): + for j in range(len_b): + f[i][j] = min( + f[i][j - 1] + 1, + f[i - 1][j] + 1, + f[i - 1][j - 1] + layer_distance(list_a[i], list_b[j]), + ) + return f[len_a - 1][len_b - 1] + + +def skip_connection_distance(a, b): + """The distance between two skip-connections.""" + if a[2] != b[2]: + return 1.0 + len_a = abs(a[1] - a[0]) + len_b = abs(b[1] - b[0]) + return (abs(a[0] - b[0]) + abs(len_a - len_b)) / \ + (max(a[0], b[0]) + max(len_a, len_b)) + + +def skip_connections_distance(list_a, list_b): + """The distance between the skip-connections of two neural networks.""" + distance_matrix = np.zeros((len(list_a), len(list_b))) + for i, a in enumerate(list_a): + for j, b in enumerate(list_b): + distance_matrix[i][j] = skip_connection_distance(a, b) + return distance_matrix[linear_sum_assignment(distance_matrix)].sum() + abs( + len(list_a) - len(list_b) + ) + + +def edit_distance(x, y): + """The distance between two neural networks. + Args: + x: An instance of NetworkDescriptor. + y: An instance of NetworkDescriptor + Returns: + The edit-distance between x and y. + """ + + ret = layers_distance(x.layers, y.layers) + ret += Constant.KERNEL_LAMBDA * skip_connections_distance( + x.skip_connections, y.skip_connections + ) + return ret + + +class IncrementalGaussianProcess: + """Gaussian process regressor. + Attributes: + alpha: A hyperparameter. + """ + + def __init__(self): + self.alpha = 1e-10 + self._distance_matrix = None + self._x = None + self._y = None + self._first_fitted = False + self._l_matrix = None + self._alpha_vector = None + + @property + def kernel_matrix(self): + ''' Kernel matric. + ''' + return self._distance_matrix + + def fit(self, train_x, train_y): + """ Fit the regressor with more data. + Args: + train_x: A list of NetworkDescriptor. + train_y: A list of metric values. + """ + if self.first_fitted: + self.incremental_fit(train_x, train_y) + else: + self.first_fit(train_x, train_y) + + def incremental_fit(self, train_x, train_y): + """ Incrementally fit the regressor. """ + if not self._first_fitted: + raise ValueError( + "The first_fit function needs to be called first.") + + train_x, train_y = np.array(train_x), np.array(train_y) + + # Incrementally compute K + up_right_k = edit_distance_matrix(self._x, train_x) + down_left_k = np.transpose(up_right_k) + down_right_k = edit_distance_matrix(train_x) + up_k = np.concatenate((self._distance_matrix, up_right_k), axis=1) + down_k = np.concatenate((down_left_k, down_right_k), axis=1) + temp_distance_matrix = np.concatenate((up_k, down_k), axis=0) + k_matrix = bourgain_embedding_matrix(temp_distance_matrix) + diagonal = np.diag_indices_from(k_matrix) + diagonal = (diagonal[0][-len(train_x):], diagonal[1][-len(train_x):]) + k_matrix[diagonal] += self.alpha + + try: + self._l_matrix = cholesky(k_matrix, lower=True) # Line 2 + except LinAlgError: + return self + + self._x = np.concatenate((self._x, train_x), axis=0) + self._y = np.concatenate((self._y, train_y), axis=0) + self._distance_matrix = temp_distance_matrix + + self._alpha_vector = cho_solve( + (self._l_matrix, True), self._y) # Line 3 + + return self + + @property + def first_fitted(self): + ''' if it is firsr fitted + ''' + return self._first_fitted + + def first_fit(self, train_x, train_y): + """ Fit the regressor for the first time. """ + train_x, train_y = np.array(train_x), np.array(train_y) + + self._x = np.copy(train_x) + self._y = np.copy(train_y) + + self._distance_matrix = edit_distance_matrix(self._x) + k_matrix = bourgain_embedding_matrix(self._distance_matrix) + k_matrix[np.diag_indices_from(k_matrix)] += self.alpha + + self._l_matrix = cholesky(k_matrix, lower=True) # Line 2 + + self._alpha_vector = cho_solve( + (self._l_matrix, True), self._y) # Line 3 + + self._first_fitted = True + return self + + def predict(self, train_x): + """Predict the result. + Args: + train_x: A list of NetworkDescriptor. + Returns: + y_mean: The predicted mean. + y_std: The predicted standard deviation. + """ + k_trans = np.exp(-np.power(edit_distance_matrix(train_x, self._x), 2)) + y_mean = k_trans.dot(self._alpha_vector) # Line 4 (y_mean = f_star) + + # compute inverse K_inv of K based on its Cholesky + # decomposition L and its inverse L_inv + l_inv = solve_triangular( + self._l_matrix.T, np.eye( + self._l_matrix.shape[0])) + k_inv = l_inv.dot(l_inv.T) + # Compute variance of predictive distribution + y_var = np.ones(len(train_x), dtype=np.float) + y_var -= np.einsum("ij,ij->i", np.dot(k_trans, k_inv), k_trans) + + # Check if any of the variances is negative because of + # numerical issues. If yes: set the variance to 0. + y_var_negative = y_var < 0 + if np.any(y_var_negative): + y_var[y_var_negative] = 0.0 + return y_mean, np.sqrt(y_var) + + +def edit_distance_matrix(train_x, train_y=None): + """Calculate the edit distance. + Args: + train_x: A list of neural architectures. + train_y: A list of neural architectures. + Returns: + An edit-distance matrix. + """ + if train_y is None: + ret = np.zeros((train_x.shape[0], train_x.shape[0])) + for x_index, x in enumerate(train_x): + for y_index, y in enumerate(train_x): + if x_index == y_index: + ret[x_index][y_index] = 0 + elif x_index < y_index: + ret[x_index][y_index] = edit_distance(x, y) + else: + ret[x_index][y_index] = ret[y_index][x_index] + return ret + ret = np.zeros((train_x.shape[0], train_y.shape[0])) + for x_index, x in enumerate(train_x): + for y_index, y in enumerate(train_y): + ret[x_index][y_index] = edit_distance(x, y) + return ret + + +def vector_distance(a, b): + """The Euclidean distance between two vectors.""" + a = np.array(a) + b = np.array(b) + return np.linalg.norm(a - b) + + +def bourgain_embedding_matrix(distance_matrix): + """Use Bourgain algorithm to embed the neural architectures based on their edit-distance. + Args: + distance_matrix: A matrix of edit-distances. + Returns: + A matrix of distances after embedding. + """ + distance_matrix = np.array(distance_matrix) + n = len(distance_matrix) + if n == 1: + return distance_matrix + np.random.seed(123) + distort_elements = [] + r = range(n) + k = int(math.ceil(math.log(n) / math.log(2) - 1)) + t = int(math.ceil(math.log(n))) + counter = 0 + for i in range(0, k + 1): + for t in range(t): + s = np.random.choice(r, 2 ** i) + for j in r: + d = min([distance_matrix[j][s] for s in s]) + counter += len(s) + if i == 0 and t == 0: + distort_elements.append([d]) + else: + distort_elements[j].append(d) + return rbf_kernel(distort_elements, distort_elements) + + +class BayesianOptimizer: + """ A Bayesian optimizer for neural architectures. + Attributes: + searcher: The Searcher which is calling the Bayesian optimizer. + t_min: The minimum temperature for simulated annealing. + metric: An instance of the Metric subclasses. + gpr: A GaussianProcessRegressor for bayesian optimization. + beta: The beta in acquisition function. (refer to our paper) + search_tree: The network morphism search tree. + """ + + def __init__(self, searcher, t_min, optimizemode, beta=None): + self.searcher = searcher + self.t_min = t_min + self.optimizemode = optimizemode + self.gpr = IncrementalGaussianProcess() + self.beta = beta if beta is not None else Constant.BETA + self.search_tree = SearchTree() + + def fit(self, x_queue, y_queue): + """ Fit the optimizer with new architectures and performances. + Args: + x_queue: A list of NetworkDescriptor. + y_queue: A list of metric values. + """ + self.gpr.fit(x_queue, y_queue) + + def generate(self, descriptors): + """Generate new architecture. + Args: + descriptors: All the searched neural architectures. + Returns: + graph: An instance of Graph. A morphed neural network with weights. + father_id: The father node ID in the search tree. + """ + model_ids = self.search_tree.adj_list.keys() + + target_graph = None + father_id = None + descriptors = deepcopy(descriptors) + elem_class = Elem + if self.optimizemode is OptimizeMode.Maximize: + elem_class = ReverseElem + + # Initialize the priority queue. + pq = PriorityQueue() + temp_list = [] + for model_id in model_ids: + metric_value = self.searcher.get_metric_value_by_id(model_id) + temp_list.append((metric_value, model_id)) + temp_list = sorted(temp_list) + for metric_value, model_id in temp_list: + graph = self.searcher.load_model_by_id(model_id) + graph.clear_operation_history() + graph.clear_weights() + pq.put(elem_class(metric_value, model_id, graph)) + + t = 1.0 + t_min = self.t_min + alpha = 0.9 + opt_acq = self._get_init_opt_acq_value() + while not pq.empty() and t > t_min: + elem = pq.get() + if self.optimizemode is OptimizeMode.Maximize: + temp_exp = min((elem.metric_value - opt_acq) / t, 1.0) + else: + temp_exp = min((opt_acq - elem.metric_value) / t, 1.0) + ap = math.exp(temp_exp) + if ap >= random.uniform(0, 1): + for temp_graph in transform(elem.graph): + if contain(descriptors, temp_graph.extract_descriptor()): + continue + + temp_acq_value = self.acq(temp_graph) + pq.put( + elem_class( + temp_acq_value, + elem.father_id, + temp_graph)) + descriptors.append(temp_graph.extract_descriptor()) + if self._accept_new_acq_value(opt_acq, temp_acq_value): + opt_acq = temp_acq_value + father_id = elem.father_id + target_graph = deepcopy(temp_graph) + t *= alpha + + # Did not found a not duplicated architecture + if father_id is None: + return None, None + nm_graph = self.searcher.load_model_by_id(father_id) + for args in target_graph.operation_history: + getattr(nm_graph, args[0])(*list(args[1:])) + return nm_graph, father_id + + def acq(self, graph): + ''' estimate the value of generated graph + ''' + mean, std = self.gpr.predict(np.array([graph.extract_descriptor()])) + if self.optimizemode is OptimizeMode.Maximize: + return mean + self.beta * std + return mean - self.beta * std + + def _get_init_opt_acq_value(self): + if self.optimizemode is OptimizeMode.Maximize: + return -np.inf + return np.inf + + def _accept_new_acq_value(self, opt_acq, temp_acq_value): + if temp_acq_value > opt_acq and self.optimizemode is OptimizeMode.Maximize: + return True + if temp_acq_value < opt_acq and not self.optimizemode is OptimizeMode.Maximize: + return True + return False + + def add_child(self, father_id, model_id): + ''' add child to the search tree + Arguments: + father_id {int} -- father id + model_id {int} -- model id + ''' + + self.search_tree.add_child(father_id, model_id) + + +@total_ordering +class Elem: + """Elements to be sorted according to metric value.""" + + def __init__(self, metric_value, father_id, graph): + self.father_id = father_id + self.graph = graph + self.metric_value = metric_value + + def __eq__(self, other): + return self.metric_value == other.metric_value + + def __lt__(self, other): + return self.metric_value < other.metric_value + + +class ReverseElem(Elem): + """Elements to be reversely sorted according to metric value.""" + + def __lt__(self, other): + return self.metric_value > other.metric_value + + +def contain(descriptors, target_descriptor): + """Check if the target descriptor is in the descriptors.""" + for descriptor in descriptors: + if edit_distance(descriptor, target_descriptor) < 1e-5: + return True + return False + + +class SearchTree: + """The network morphism search tree.""" + + def __init__(self): + self.root = None + self.adj_list = {} + + def add_child(self, u, v): + ''' add child to search tree itself. + Arguments: + u {int} -- father id + v {int} -- child id + ''' + + if u == -1: + self.root = v + self.adj_list[v] = [] + return + if v not in self.adj_list[u]: + self.adj_list[u].append(v) + if v not in self.adj_list: + self.adj_list[v] = [] + + def get_dict(self, u=None): + """ A recursive function to return the content of the tree in a dict.""" + if u is None: + return self.get_dict(self.root) + children = [] + for v in self.adj_list[u]: + children.append(self.get_dict(v)) + ret = {"name": u, "children": children} + return ret diff --git a/new_impl/cv/third_party/nni/algorithms/hpo/networkmorphism_tuner/graph.py b/new_impl/cv/third_party/nni/algorithms/hpo/networkmorphism_tuner/graph.py new file mode 100644 index 0000000000000000000000000000000000000000..9c96b6c2f07088aceb0cb029de02cb4b2f16ae32 --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/hpo/networkmorphism_tuner/graph.py @@ -0,0 +1,995 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import json +from collections.abc import Iterable +from copy import deepcopy, copy +from queue import Queue + +import numpy as np +import torch + +from .layer_transformer import ( + add_noise, + wider_bn, + wider_next_conv, + wider_next_dense, + wider_pre_conv, + wider_pre_dense, + init_dense_weight, + init_conv_weight, + init_bn_weight, +) +from .layers import ( + StubAdd, + StubConcatenate, + StubReLU, + get_batch_norm_class, + get_conv_class, + is_layer, + layer_width, + set_keras_weight_to_stub, + set_stub_weight_to_keras, + set_stub_weight_to_torch, + set_torch_weight_to_stub, + to_real_keras_layer, + layer_description_extractor, + layer_description_builder, +) +from .utils import Constant + + +class NetworkDescriptor: + """A class describing the neural architecture for neural network kernel. + It only record the width of convolutional and dense layers, and the skip-connection types and positions. + """ + + CONCAT_CONNECT = "concat" + ADD_CONNECT = "add" + + def __init__(self): + self.skip_connections = [] + self.layers = [] + + @property + def n_layers(self): + return len(self.layers) + + def add_skip_connection(self, u, v, connection_type): + """ Add a skip-connection to the descriptor. + Args: + u: Number of convolutional layers before the starting point. + v: Number of convolutional layers before the ending point. + connection_type: Must be either CONCAT_CONNECT or ADD_CONNECT. + """ + if connection_type not in [self.CONCAT_CONNECT, self.ADD_CONNECT]: + raise ValueError( + "connection_type should be NetworkDescriptor.CONCAT_CONNECT " + "or NetworkDescriptor.ADD_CONNECT." + ) + self.skip_connections.append((u, v, connection_type)) + + def to_json(self): + ''' NetworkDescriptor to json representation + ''' + + skip_list = [] + for u, v, connection_type in self.skip_connections: + skip_list.append({"from": u, "to": v, "type": connection_type}) + return {"node_list": self.layers, "skip_list": skip_list} + + def add_layer(self, layer): + ''' add one layer + ''' + + self.layers.append(layer) + + +class Node: + """A class for intermediate output tensor (node) in the Graph. + Attributes: + shape: A tuple describing the shape of the tensor. + """ + + def __init__(self, shape): + self.shape = shape + + +class Graph: + """A class representing the neural architecture graph of a model. + Graph extracts the neural architecture graph from a model. + Each node in the graph is a intermediate tensor between layers. + Each layer is an edge in the graph. + Notably, multiple edges may refer to the same layer. + (e.g. Add layer is adding two tensor into one tensor. So it is related to two edges.) + Attributes: + weighted: A boolean of whether the weights and biases in the neural network + should be included in the graph. + input_shape: A tuple of integers, which does not include the batch axis. + node_list: A list of integers. The indices of the list are the identifiers. + layer_list: A list of stub layers. The indices of the list are the identifiers. + node_to_id: A dict instance mapping from node integers to their identifiers. + layer_to_id: A dict instance mapping from stub layers to their identifiers. + layer_id_to_input_node_ids: A dict instance mapping from layer identifiers + to their input nodes identifiers. + layer_id_to_output_node_ids: A dict instance mapping from layer identifiers + to their output nodes identifiers. + adj_list: A two dimensional list. The adjacency list of the graph. The first dimension is + identified by tensor identifiers. In each edge list, the elements are two-element tuples + of (tensor identifier, layer identifier). + reverse_adj_list: A reverse adjacent list in the same format as adj_list. + operation_history: A list saving all the network morphism operations. + vis: A dictionary of temporary storage for whether an local operation has been done + during the network morphism. + """ + + def __init__(self, input_shape, weighted=True): + """Initializer for Graph. + """ + self.input_shape = input_shape + self.weighted = weighted + self.node_list = [] + self.layer_list = [] + # node id start with 0 + self.node_to_id = {} + self.layer_to_id = {} + self.layer_id_to_input_node_ids = {} + self.layer_id_to_output_node_ids = {} + self.adj_list = {} + self.reverse_adj_list = {} + self.operation_history = [] + self.n_dim = len(input_shape) - 1 + self.conv = get_conv_class(self.n_dim) + self.batch_norm = get_batch_norm_class(self.n_dim) + + self.vis = None + self._add_node(Node(input_shape)) + + def add_layer(self, layer, input_node_id): + """Add a layer to the Graph. + Args: + layer: An instance of the subclasses of StubLayer in layers.py. + input_node_id: An integer. The ID of the input node of the layer. + Returns: + output_node_id: An integer. The ID of the output node of the layer. + """ + if isinstance(input_node_id, Iterable): + layer.input = list(map(lambda x: self.node_list[x], input_node_id)) + output_node_id = self._add_node(Node(layer.output_shape)) + for node_id in input_node_id: + self._add_edge(layer, node_id, output_node_id) + + else: + layer.input = self.node_list[input_node_id] + output_node_id = self._add_node(Node(layer.output_shape)) + self._add_edge(layer, input_node_id, output_node_id) + + layer.output = self.node_list[output_node_id] + return output_node_id + + def clear_operation_history(self): + self.operation_history = [] + + @property + def n_nodes(self): + """Return the number of nodes in the model.""" + return len(self.node_list) + + @property + def n_layers(self): + """Return the number of layers in the model.""" + return len(self.layer_list) + + def _add_node(self, node): + """Add a new node to node_list and give the node an ID. + Args: + node: An instance of Node. + Returns: + node_id: An integer. + """ + node_id = len(self.node_list) + self.node_to_id[node] = node_id + self.node_list.append(node) + self.adj_list[node_id] = [] + self.reverse_adj_list[node_id] = [] + return node_id + + def _add_edge(self, layer, input_id, output_id): + """Add a new layer to the graph. The nodes should be created in advance.""" + + if layer in self.layer_to_id: + layer_id = self.layer_to_id[layer] + if input_id not in self.layer_id_to_input_node_ids[layer_id]: + self.layer_id_to_input_node_ids[layer_id].append(input_id) + if output_id not in self.layer_id_to_output_node_ids[layer_id]: + self.layer_id_to_output_node_ids[layer_id].append(output_id) + else: + layer_id = len(self.layer_list) + self.layer_list.append(layer) + self.layer_to_id[layer] = layer_id + self.layer_id_to_input_node_ids[layer_id] = [input_id] + self.layer_id_to_output_node_ids[layer_id] = [output_id] + + self.adj_list[input_id].append((output_id, layer_id)) + self.reverse_adj_list[output_id].append((input_id, layer_id)) + + def _redirect_edge(self, u_id, v_id, new_v_id): + """Redirect the layer to a new node. + Change the edge originally from `u_id` to `v_id` into an edge from `u_id` to `new_v_id` + while keeping all other property of the edge the same. + """ + layer_id = None + for index, edge_tuple in enumerate(self.adj_list[u_id]): + if edge_tuple[0] == v_id: + layer_id = edge_tuple[1] + self.adj_list[u_id][index] = (new_v_id, layer_id) + self.layer_list[layer_id].output = self.node_list[new_v_id] + break + + for index, edge_tuple in enumerate(self.reverse_adj_list[v_id]): + if edge_tuple[0] == u_id: + layer_id = edge_tuple[1] + self.reverse_adj_list[v_id].remove(edge_tuple) + break + self.reverse_adj_list[new_v_id].append((u_id, layer_id)) + for index, value in enumerate( + self.layer_id_to_output_node_ids[layer_id]): + if value == v_id: + self.layer_id_to_output_node_ids[layer_id][index] = new_v_id + break + + def _replace_layer(self, layer_id, new_layer): + """Replace the layer with a new layer.""" + old_layer = self.layer_list[layer_id] + new_layer.input = old_layer.input + new_layer.output = old_layer.output + new_layer.output.shape = new_layer.output_shape + self.layer_list[layer_id] = new_layer + self.layer_to_id[new_layer] = layer_id + self.layer_to_id.pop(old_layer) + + @property + def topological_order(self): + """Return the topological order of the node IDs from the input node to the output node.""" + q = Queue() + in_degree = {} + for i in range(self.n_nodes): + in_degree[i] = 0 + for u in range(self.n_nodes): + for v, _ in self.adj_list[u]: + in_degree[v] += 1 + for i in range(self.n_nodes): + if in_degree[i] == 0: + q.put(i) + + order_list = [] + while not q.empty(): + u = q.get() + order_list.append(u) + for v, _ in self.adj_list[u]: + in_degree[v] -= 1 + if in_degree[v] == 0: + q.put(v) + return order_list + + def _get_pooling_layers(self, start_node_id, end_node_id): + """Given two node IDs, return all the pooling layers between them.""" + layer_list = [] + node_list = [start_node_id] + assert self._depth_first_search(end_node_id, layer_list, node_list) + ret = [] + for layer_id in layer_list: + layer = self.layer_list[layer_id] + if is_layer(layer, "Pooling"): + ret.append(layer) + elif is_layer(layer, "Conv") and layer.stride != 1: + ret.append(layer) + return ret + + def _depth_first_search(self, target_id, layer_id_list, node_list): + """Search for all the layers and nodes down the path. + A recursive function to search all the layers and nodes between the node in the node_list + and the node with target_id.""" + assert len(node_list) <= self.n_nodes + u = node_list[-1] + if u == target_id: + return True + + for v, layer_id in self.adj_list[u]: + layer_id_list.append(layer_id) + node_list.append(v) + if self._depth_first_search(target_id, layer_id_list, node_list): + return True + layer_id_list.pop() + node_list.pop() + + return False + + def _search(self, u, start_dim, total_dim, n_add): + """Search the graph for all the layers to be widened caused by an operation. + It is an recursive function with duplication check to avoid deadlock. + It searches from a starting node u until the corresponding layers has been widened. + Args: + u: The starting node ID. + start_dim: The position to insert the additional dimensions. + total_dim: The total number of dimensions the layer has before widening. + n_add: The number of dimensions to add. + """ + if (u, start_dim, total_dim, n_add) in self.vis: + return + self.vis[(u, start_dim, total_dim, n_add)] = True + for v, layer_id in self.adj_list[u]: + layer = self.layer_list[layer_id] + + if is_layer(layer, "Conv"): + new_layer = wider_next_conv( + layer, start_dim, total_dim, n_add, self.weighted + ) + self._replace_layer(layer_id, new_layer) + + elif is_layer(layer, "Dense"): + new_layer = wider_next_dense( + layer, start_dim, total_dim, n_add, self.weighted + ) + self._replace_layer(layer_id, new_layer) + + elif is_layer(layer, "BatchNormalization"): + new_layer = wider_bn( + layer, start_dim, total_dim, n_add, self.weighted) + self._replace_layer(layer_id, new_layer) + self._search(v, start_dim, total_dim, n_add) + + elif is_layer(layer, "Concatenate"): + if self.layer_id_to_input_node_ids[layer_id][1] == u: + # u is on the right of the concat + # next_start_dim += next_total_dim - total_dim + left_dim = self._upper_layer_width( + self.layer_id_to_input_node_ids[layer_id][0] + ) + next_start_dim = start_dim + left_dim + next_total_dim = total_dim + left_dim + else: + next_start_dim = start_dim + next_total_dim = total_dim + self._upper_layer_width( + self.layer_id_to_input_node_ids[layer_id][1] + ) + self._search(v, next_start_dim, next_total_dim, n_add) + + else: + self._search(v, start_dim, total_dim, n_add) + + for v, layer_id in self.reverse_adj_list[u]: + layer = self.layer_list[layer_id] + if is_layer(layer, "Conv"): + new_layer = wider_pre_conv(layer, n_add, self.weighted) + self._replace_layer(layer_id, new_layer) + elif is_layer(layer, "Dense"): + new_layer = wider_pre_dense(layer, n_add, self.weighted) + self._replace_layer(layer_id, new_layer) + elif is_layer(layer, "Concatenate"): + continue + else: + self._search(v, start_dim, total_dim, n_add) + + def _upper_layer_width(self, u): + for v, layer_id in self.reverse_adj_list[u]: + layer = self.layer_list[layer_id] + if is_layer(layer, "Conv") or is_layer(layer, "Dense"): + return layer_width(layer) + elif is_layer(layer, "Concatenate"): + a = self.layer_id_to_input_node_ids[layer_id][0] + b = self.layer_id_to_input_node_ids[layer_id][1] + return self._upper_layer_width(a) + self._upper_layer_width(b) + else: + return self._upper_layer_width(v) + return self.node_list[0].shape[-1] + + def to_deeper_model(self, target_id, new_layer): + """Insert a relu-conv-bn block after the target block. + Args: + target_id: A convolutional layer ID. The new block should be inserted after the block. + new_layer: An instance of StubLayer subclasses. + """ + self.operation_history.append( + ("to_deeper_model", target_id, new_layer)) + input_id = self.layer_id_to_input_node_ids[target_id][0] + output_id = self.layer_id_to_output_node_ids[target_id][0] + if self.weighted: + if is_layer(new_layer, "Dense"): + init_dense_weight(new_layer) + elif is_layer(new_layer, "Conv"): + init_conv_weight(new_layer) + elif is_layer(new_layer, "BatchNormalization"): + init_bn_weight(new_layer) + + self._insert_new_layers([new_layer], input_id, output_id) + + def to_wider_model(self, pre_layer_id, n_add): + """Widen the last dimension of the output of the pre_layer. + Args: + pre_layer_id: The ID of a convolutional layer or dense layer. + n_add: The number of dimensions to add. + """ + self.operation_history.append(("to_wider_model", pre_layer_id, n_add)) + pre_layer = self.layer_list[pre_layer_id] + output_id = self.layer_id_to_output_node_ids[pre_layer_id][0] + dim = layer_width(pre_layer) + self.vis = {} + self._search(output_id, dim, dim, n_add) + # Update the tensor shapes. + for u in self.topological_order: + for v, layer_id in self.adj_list[u]: + self.node_list[v].shape = self.layer_list[layer_id].output_shape + + def _insert_new_layers(self, new_layers, start_node_id, end_node_id): + """Insert the new_layers after the node with start_node_id.""" + new_node_id = self._add_node(deepcopy(self.node_list[end_node_id])) + temp_output_id = new_node_id + for layer in new_layers[:-1]: + temp_output_id = self.add_layer(layer, temp_output_id) + + self._add_edge(new_layers[-1], temp_output_id, end_node_id) + new_layers[-1].input = self.node_list[temp_output_id] + new_layers[-1].output = self.node_list[end_node_id] + self._redirect_edge(start_node_id, end_node_id, new_node_id) + + def _block_end_node(self, layer_id, block_size): + ret = self.layer_id_to_output_node_ids[layer_id][0] + for _ in range(block_size - 2): + ret = self.adj_list[ret][0][0] + return ret + + def _dense_block_end_node(self, layer_id): + return self.layer_id_to_input_node_ids[layer_id][0] + + def _conv_block_end_node(self, layer_id): + """Get the input node ID of the last layer in the block by layer ID. + Return the input node ID of the last layer in the convolutional block. + Args: + layer_id: the convolutional layer ID. + """ + return self._block_end_node(layer_id, Constant.CONV_BLOCK_DISTANCE) + + def to_add_skip_model(self, start_id, end_id): + """Add a weighted add skip-connection from after start node to end node. + Args: + start_id: The convolutional layer ID, after which to start the skip-connection. + end_id: The convolutional layer ID, after which to end the skip-connection. + """ + self.operation_history.append(("to_add_skip_model", start_id, end_id)) + filters_end = self.layer_list[end_id].output.shape[-1] + filters_start = self.layer_list[start_id].output.shape[-1] + start_node_id = self.layer_id_to_output_node_ids[start_id][0] + + pre_end_node_id = self.layer_id_to_input_node_ids[end_id][0] + end_node_id = self.layer_id_to_output_node_ids[end_id][0] + + skip_output_id = self._insert_pooling_layer_chain( + start_node_id, end_node_id) + + # Add the conv layer + new_conv_layer = get_conv_class( + self.n_dim)( + filters_start, + filters_end, + 1) + skip_output_id = self.add_layer(new_conv_layer, skip_output_id) + + # Add the add layer. + add_input_node_id = self._add_node( + deepcopy(self.node_list[end_node_id])) + add_layer = StubAdd() + + self._redirect_edge(pre_end_node_id, end_node_id, add_input_node_id) + self._add_edge(add_layer, add_input_node_id, end_node_id) + self._add_edge(add_layer, skip_output_id, end_node_id) + add_layer.input = [ + self.node_list[add_input_node_id], + self.node_list[skip_output_id], + ] + add_layer.output = self.node_list[end_node_id] + self.node_list[end_node_id].shape = add_layer.output_shape + + # Set weights to the additional conv layer. + if self.weighted: + filter_shape = (1,) * self.n_dim + weights = np.zeros((filters_end, filters_start) + filter_shape) + bias = np.zeros(filters_end) + new_conv_layer.set_weights( + (add_noise(weights, np.array([0, 1])), add_noise( + bias, np.array([0, 1]))) + ) + + def to_concat_skip_model(self, start_id, end_id): + """Add a weighted add concatenate connection from after start node to end node. + Args: + start_id: The convolutional layer ID, after which to start the skip-connection. + end_id: The convolutional layer ID, after which to end the skip-connection. + """ + self.operation_history.append( + ("to_concat_skip_model", start_id, end_id)) + filters_end = self.layer_list[end_id].output.shape[-1] + filters_start = self.layer_list[start_id].output.shape[-1] + start_node_id = self.layer_id_to_output_node_ids[start_id][0] + + pre_end_node_id = self.layer_id_to_input_node_ids[end_id][0] + end_node_id = self.layer_id_to_output_node_ids[end_id][0] + + skip_output_id = self._insert_pooling_layer_chain( + start_node_id, end_node_id) + + concat_input_node_id = self._add_node( + deepcopy(self.node_list[end_node_id])) + self._redirect_edge(pre_end_node_id, end_node_id, concat_input_node_id) + + concat_layer = StubConcatenate() + concat_layer.input = [ + self.node_list[concat_input_node_id], + self.node_list[skip_output_id], + ] + concat_output_node_id = self._add_node(Node(concat_layer.output_shape)) + self._add_edge( + concat_layer, + concat_input_node_id, + concat_output_node_id) + self._add_edge(concat_layer, skip_output_id, concat_output_node_id) + concat_layer.output = self.node_list[concat_output_node_id] + self.node_list[concat_output_node_id].shape = concat_layer.output_shape + + # Add the concatenate layer. + new_conv_layer = get_conv_class(self.n_dim)( + filters_start + filters_end, filters_end, 1 + ) + self._add_edge(new_conv_layer, concat_output_node_id, end_node_id) + new_conv_layer.input = self.node_list[concat_output_node_id] + new_conv_layer.output = self.node_list[end_node_id] + self.node_list[end_node_id].shape = new_conv_layer.output_shape + + if self.weighted: + filter_shape = (1,) * self.n_dim + weights = np.zeros((filters_end, filters_end) + filter_shape) + for i in range(filters_end): + filter_weight = np.zeros((filters_end,) + filter_shape) + center_index = (i,) + (0,) * self.n_dim + filter_weight[center_index] = 1 + weights[i, ...] = filter_weight + weights = np.concatenate( + (weights, np.zeros((filters_end, filters_start) + filter_shape)), axis=1 + ) + bias = np.zeros(filters_end) + new_conv_layer.set_weights( + (add_noise(weights, np.array([0, 1])), add_noise( + bias, np.array([0, 1]))) + ) + + def _insert_pooling_layer_chain(self, start_node_id, end_node_id): + skip_output_id = start_node_id + for layer in self._get_pooling_layers(start_node_id, end_node_id): + new_layer = deepcopy(layer) + if is_layer(new_layer, "Conv"): + filters = self.node_list[start_node_id].shape[-1] + new_layer = get_conv_class(self.n_dim)( + filters, filters, 1, layer.stride) + if self.weighted: + init_conv_weight(new_layer) + else: + new_layer = deepcopy(layer) + skip_output_id = self.add_layer(new_layer, skip_output_id) + skip_output_id = self.add_layer(StubReLU(), skip_output_id) + return skip_output_id + + def extract_descriptor(self): + """Extract the the description of the Graph as an instance of NetworkDescriptor.""" + main_chain = self.get_main_chain() + index_in_main_chain = {} + for index, u in enumerate(main_chain): + index_in_main_chain[u] = index + + ret = NetworkDescriptor() + for u in main_chain: + for v, layer_id in self.adj_list[u]: + if v not in index_in_main_chain: + continue + layer = self.layer_list[layer_id] + copied_layer = copy(layer) + copied_layer.weights = None + ret.add_layer(deepcopy(copied_layer)) + + for u in index_in_main_chain: + for v, layer_id in self.adj_list[u]: + if v not in index_in_main_chain: + temp_u = u + temp_v = v + temp_layer_id = layer_id + skip_type = None + while not ( + temp_v in index_in_main_chain and temp_u in index_in_main_chain): + if is_layer( + self.layer_list[temp_layer_id], "Concatenate"): + skip_type = NetworkDescriptor.CONCAT_CONNECT + if is_layer(self.layer_list[temp_layer_id], "Add"): + skip_type = NetworkDescriptor.ADD_CONNECT + temp_u = temp_v + temp_v, temp_layer_id = self.adj_list[temp_v][0] + ret.add_skip_connection( + index_in_main_chain[u], index_in_main_chain[temp_u], skip_type + ) + + elif index_in_main_chain[v] - index_in_main_chain[u] != 1: + skip_type = None + if is_layer(self.layer_list[layer_id], "Concatenate"): + skip_type = NetworkDescriptor.CONCAT_CONNECT + if is_layer(self.layer_list[layer_id], "Add"): + skip_type = NetworkDescriptor.ADD_CONNECT + ret.add_skip_connection( + index_in_main_chain[u], index_in_main_chain[v], skip_type + ) + + return ret + + def clear_weights(self): + ''' clear weights of the graph + ''' + self.weighted = False + for layer in self.layer_list: + layer.weights = None + + def produce_torch_model(self): + """Build a new Torch model based on the current graph.""" + return TorchModel(self) + + def produce_keras_model(self): + """Build a new keras model based on the current graph.""" + return KerasModel(self).model + + def produce_onnx_model(self): + """Build a new ONNX model based on the current graph.""" + return ONNXModel(self) + + def parsing_onnx_model(self, onnx_model): + '''to do in the future to use the onnx model + ''' + return ONNXModel(onnx_model) + + def produce_json_model(self): + """Build a new Json model based on the current graph.""" + return JSONModel(self).data + + @classmethod + def parsing_json_model(cls, json_model): + '''build a graph from json + ''' + return json_to_graph(json_model) + + def _layer_ids_in_order(self, layer_ids): + node_id_to_order_index = {} + for index, node_id in enumerate(self.topological_order): + node_id_to_order_index[node_id] = index + return sorted( + layer_ids, + key=lambda layer_id: node_id_to_order_index[ + self.layer_id_to_output_node_ids[layer_id][0] + ], + ) + + def _layer_ids_by_type(self, type_str): + return list( + filter( + lambda layer_id: is_layer(self.layer_list[layer_id], type_str), + range(self.n_layers), + ) + ) + + def get_main_chain_layers(self): + """Return a list of layer IDs in the main chain.""" + main_chain = self.get_main_chain() + ret = [] + for u in main_chain: + for v, layer_id in self.adj_list[u]: + if v in main_chain and u in main_chain: + ret.append(layer_id) + return ret + + def _conv_layer_ids_in_order(self): + return list( + filter( + lambda layer_id: is_layer(self.layer_list[layer_id], "Conv"), + self.get_main_chain_layers(), + ) + ) + + def _dense_layer_ids_in_order(self): + return self._layer_ids_in_order(self._layer_ids_by_type("Dense")) + + def deep_layer_ids(self): + ret = [] + for layer_id in self.get_main_chain_layers(): + layer = self.layer_list[layer_id] + if is_layer(layer, "GlobalAveragePooling"): + break + if is_layer(layer, "Add") or is_layer(layer, "Concatenate"): + continue + ret.append(layer_id) + return ret + + def wide_layer_ids(self): + return ( + self._conv_layer_ids_in_order( + )[:-1] + self._dense_layer_ids_in_order()[:-1] + ) + + def skip_connection_layer_ids(self): + return self.deep_layer_ids()[:-1] + + def size(self): + return sum(list(map(lambda x: x.size(), self.layer_list))) + + def get_main_chain(self): + """Returns the main chain node ID list.""" + pre_node = {} + distance = {} + for i in range(self.n_nodes): + distance[i] = 0 + pre_node[i] = i + for i in range(self.n_nodes - 1): + for u in range(self.n_nodes): + for v, _ in self.adj_list[u]: + if distance[u] + 1 > distance[v]: + distance[v] = distance[u] + 1 + pre_node[v] = u + temp_id = 0 + for i in range(self.n_nodes): + if distance[i] > distance[temp_id]: + temp_id = i + ret = [] + for i in range(self.n_nodes + 5): + ret.append(temp_id) + if pre_node[temp_id] == temp_id: + break + temp_id = pre_node[temp_id] + assert temp_id == pre_node[temp_id] + ret.reverse() + return ret + + +class TorchModel(torch.nn.Module): + """A neural network class using pytorch constructed from an instance of Graph.""" + + def __init__(self, graph): + super(TorchModel, self).__init__() + self.graph = graph + self.layers = [] + for layer in graph.layer_list: + self.layers.append(layer.to_real_layer()) + if graph.weighted: + for index, layer in enumerate(self.layers): + set_stub_weight_to_torch(self.graph.layer_list[index], layer) + for index, layer in enumerate(self.layers): + self.add_module(str(index), layer) + + def forward(self, input_tensor): + topo_node_list = self.graph.topological_order + output_id = topo_node_list[-1] + input_id = topo_node_list[0] + + node_list = deepcopy(self.graph.node_list) + node_list[input_id] = input_tensor + + for v in topo_node_list: + for u, layer_id in self.graph.reverse_adj_list[v]: + layer = self.graph.layer_list[layer_id] + torch_layer = self.layers[layer_id] + + if isinstance(layer, (StubAdd, StubConcatenate)): + edge_input_tensor = list( + map( + lambda x: node_list[x], + self.graph.layer_id_to_input_node_ids[layer_id], + ) + ) + else: + edge_input_tensor = node_list[u] + + temp_tensor = torch_layer(edge_input_tensor) + node_list[v] = temp_tensor + return node_list[output_id] + + def set_weight_to_graph(self): + self.graph.weighted = True + for index, layer in enumerate(self.layers): + set_torch_weight_to_stub(layer, self.graph.layer_list[index]) + + +class KerasModel: + def __init__(self, graph): + import keras + + self.graph = graph + self.layers = [] + for layer in graph.layer_list: + self.layers.append(to_real_keras_layer(layer)) + + # Construct the keras graph. + # Input + topo_node_list = self.graph.topological_order + output_id = topo_node_list[-1] + input_id = topo_node_list[0] + input_tensor = keras.layers.Input( + shape=graph.node_list[input_id].shape) + + node_list = deepcopy(self.graph.node_list) + node_list[input_id] = input_tensor + + # Output + for v in topo_node_list: + for u, layer_id in self.graph.reverse_adj_list[v]: + layer = self.graph.layer_list[layer_id] + keras_layer = self.layers[layer_id] + + if isinstance(layer, (StubAdd, StubConcatenate)): + edge_input_tensor = list( + map( + lambda x: node_list[x], + self.graph.layer_id_to_input_node_ids[layer_id], + ) + ) + else: + edge_input_tensor = node_list[u] + + temp_tensor = keras_layer(edge_input_tensor) + node_list[v] = temp_tensor + + output_tensor = node_list[output_id] + output_tensor = keras.layers.Activation("softmax", name="activation_add")( + output_tensor + ) + self.model = keras.models.Model( + inputs=input_tensor, outputs=output_tensor) + + if graph.weighted: + for index, layer in enumerate(self.layers): + set_stub_weight_to_keras(self.graph.layer_list[index], layer) + + def set_weight_to_graph(self): + self.graph.weighted = True + for index, layer in enumerate(self.layers): + set_keras_weight_to_stub(layer, self.graph.layer_list[index]) + + +class ONNXModel: + # to do in the future using onnx ir + def __init__(self, graph): + pass + + +class JSONModel: + def __init__(self, graph): + data = dict() + node_list = list() + layer_list = list() + operation_history = list() + + data["input_shape"] = graph.input_shape + vis = graph.vis + data["vis"] = list(vis.keys()) if vis is not None else None + data["weighted"] = graph.weighted + + for item in graph.operation_history: + if item[0] == "to_deeper_model": + operation_history.append( + [ + item[0], + item[1], + layer_description_extractor(item[2], graph.node_to_id), + ] + ) + else: + operation_history.append(item) + data["operation_history"] = operation_history + data["layer_id_to_input_node_ids"] = graph.layer_id_to_input_node_ids + data["layer_id_to_output_node_ids"] = graph.layer_id_to_output_node_ids + data["adj_list"] = graph.adj_list + data["reverse_adj_list"] = graph.reverse_adj_list + + for node in graph.node_list: + node_id = graph.node_to_id[node] + node_information = node.shape + node_list.append((node_id, node_information)) + + for layer_id, item in enumerate(graph.layer_list): + layer = graph.layer_list[layer_id] + layer_information = layer_description_extractor( + layer, graph.node_to_id) + layer_list.append((layer_id, layer_information)) + + data["node_list"] = node_list + data["layer_list"] = layer_list + + self.data = data + + +def graph_to_onnx(graph, onnx_model_path): + import onnx + # to do in the future using onnx ir + onnx_out = graph.produce_onnx_model() + onnx.save(onnx_out, onnx_model_path) + return onnx_out + + +def onnx_to_graph(onnx_model, input_shape): + # to do in the future using onnx ir + graph = Graph(input_shape, False) + graph.parsing_onnx_model(onnx_model) + return graph + + +def graph_to_json(graph, json_model_path): + json_out = graph.produce_json_model() + with open(json_model_path, "w") as fout: + json.dump(json_out, fout) + json_out = json.dumps(json_out) + return json_out + + +def json_to_graph(json_model: str): + json_model = json.loads(json_model) + # restore graph data from json data + input_shape = tuple(json_model["input_shape"]) + node_list = list() + node_to_id = dict() + id_to_node = dict() + layer_list = list() + layer_to_id = dict() + operation_history = list() + graph = Graph(input_shape, False) + + graph.input_shape = input_shape + vis = json_model["vis"] + graph.vis = { + tuple(item): True for item in vis} if vis is not None else None + graph.weighted = json_model["weighted"] + layer_id_to_input_node_ids = json_model["layer_id_to_input_node_ids"] + graph.layer_id_to_input_node_ids = { + int(k): v for k, v in layer_id_to_input_node_ids.items() + } + layer_id_to_output_node_ids = json_model["layer_id_to_output_node_ids"] + graph.layer_id_to_output_node_ids = { + int(k): v for k, v in layer_id_to_output_node_ids.items() + } + adj_list = {} + for k, v in json_model["adj_list"].items(): + adj_list[int(k)] = [tuple(i) for i in v] + graph.adj_list = adj_list + reverse_adj_list = {} + for k, v in json_model["reverse_adj_list"].items(): + reverse_adj_list[int(k)] = [tuple(i) for i in v] + graph.reverse_adj_list = reverse_adj_list + + for item in json_model["node_list"]: + new_node = Node(tuple(item[1])) + node_id = item[0] + node_list.append(new_node) + node_to_id[new_node] = node_id + id_to_node[node_id] = new_node + + for item in json_model["operation_history"]: + if item[0] == "to_deeper_model": + operation_history.append( + (item[0], item[1], layer_description_builder(item[2], id_to_node)) + ) + else: + operation_history.append(item) + graph.operation_history = operation_history + + for item in json_model["layer_list"]: + new_layer = layer_description_builder(item[1], id_to_node) + layer_id = int(item[0]) + layer_list.append(new_layer) + layer_to_id[new_layer] = layer_id + + graph.node_list = node_list + graph.node_to_id = node_to_id + graph.layer_list = layer_list + graph.layer_to_id = layer_to_id + + return graph diff --git a/new_impl/cv/third_party/nni/algorithms/hpo/networkmorphism_tuner/graph_transformer.py b/new_impl/cv/third_party/nni/algorithms/hpo/networkmorphism_tuner/graph_transformer.py new file mode 100644 index 0000000000000000000000000000000000000000..b03112d63bcc50f5c42e4203dfd4adb70a061119 --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/hpo/networkmorphism_tuner/graph_transformer.py @@ -0,0 +1,167 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from copy import deepcopy + +from random import randrange, sample + +from .graph import NetworkDescriptor +from .layers import ( + StubDense, + StubReLU, + get_batch_norm_class, + get_conv_class, + get_dropout_class, + get_pooling_class, + is_layer, +) +from .utils import Constant + + +def to_wider_graph(graph): + ''' wider graph + ''' + weighted_layer_ids = graph.wide_layer_ids() + weighted_layer_ids = list( + filter( + lambda x: graph.layer_list[x].output.shape[-1], weighted_layer_ids) + ) + wider_layers = sample(weighted_layer_ids, 1) + + for layer_id in wider_layers: + layer = graph.layer_list[layer_id] + if is_layer(layer, "Conv"): + n_add = layer.filters + else: + n_add = layer.units + + graph.to_wider_model(layer_id, n_add) + return graph + + +def to_skip_connection_graph(graph): + ''' skip connection graph + ''' + # The last conv layer cannot be widen since wider operator cannot be done + # over the two sides of flatten. + weighted_layer_ids = graph.skip_connection_layer_ids() + valid_connection = [] + for skip_type in sorted( + [NetworkDescriptor.ADD_CONNECT, NetworkDescriptor.CONCAT_CONNECT]): + for index_a in range(len(weighted_layer_ids)): + for index_b in range(len(weighted_layer_ids))[index_a + 1:]: + valid_connection.append((index_a, index_b, skip_type)) + + if not valid_connection: + return graph + for index_a, index_b, skip_type in sample(valid_connection, 1): + a_id = weighted_layer_ids[index_a] + b_id = weighted_layer_ids[index_b] + if skip_type == NetworkDescriptor.ADD_CONNECT: + graph.to_add_skip_model(a_id, b_id) + else: + graph.to_concat_skip_model(a_id, b_id) + return graph + + +def create_new_layer(layer, n_dim): + ''' create new layer for the graph + ''' + + input_shape = layer.output.shape + dense_deeper_classes = [StubDense, get_dropout_class(n_dim), StubReLU] + conv_deeper_classes = [ + get_conv_class(n_dim), + get_batch_norm_class(n_dim), + StubReLU] + if is_layer(layer, "ReLU"): + conv_deeper_classes = [ + get_conv_class(n_dim), + get_batch_norm_class(n_dim)] + dense_deeper_classes = [StubDense, get_dropout_class(n_dim)] + elif is_layer(layer, "Dropout"): + dense_deeper_classes = [StubDense, StubReLU] + elif is_layer(layer, "BatchNormalization"): + conv_deeper_classes = [get_conv_class(n_dim), StubReLU] + + layer_class = None + if len(input_shape) == 1: + # It is in the dense layer part. + layer_class = sample(dense_deeper_classes, 1)[0] + else: + # It is in the conv layer part. + layer_class = sample(conv_deeper_classes, 1)[0] + + if layer_class == StubDense: + new_layer = StubDense(input_shape[0], input_shape[0]) + + elif layer_class == get_dropout_class(n_dim): + new_layer = layer_class(Constant.DENSE_DROPOUT_RATE) + + elif layer_class == get_conv_class(n_dim): + new_layer = layer_class( + input_shape[-1], input_shape[-1], sample((1, 3, 5), 1)[0], stride=1 + ) + + elif layer_class == get_batch_norm_class(n_dim): + new_layer = layer_class(input_shape[-1]) + + elif layer_class == get_pooling_class(n_dim): + new_layer = layer_class(sample((1, 3, 5), 1)[0]) + + else: + new_layer = layer_class() + + return new_layer + + +def to_deeper_graph(graph): + ''' deeper graph + ''' + + weighted_layer_ids = graph.deep_layer_ids() + if len(weighted_layer_ids) >= Constant.MAX_LAYERS: + return None + + deeper_layer_ids = sample(weighted_layer_ids, 1) + + for layer_id in deeper_layer_ids: + layer = graph.layer_list[layer_id] + new_layer = create_new_layer(layer, graph.n_dim) + graph.to_deeper_model(layer_id, new_layer) + return graph + + +def legal_graph(graph): + '''judge if a graph is legal or not. + ''' + + descriptor = graph.extract_descriptor() + skips = descriptor.skip_connections + if len(skips) != len(set(skips)): + return False + return True + + +def transform(graph): + '''core transform function for graph. + ''' + + graphs = [] + for _ in range(Constant.N_NEIGHBOURS * 2): + random_num = randrange(3) + temp_graph = None + if random_num == 0: + temp_graph = to_deeper_graph(deepcopy(graph)) + elif random_num == 1: + temp_graph = to_wider_graph(deepcopy(graph)) + elif random_num == 2: + temp_graph = to_skip_connection_graph(deepcopy(graph)) + + if temp_graph is not None and temp_graph.size() <= Constant.MAX_MODEL_SIZE: + graphs.append(temp_graph) + + if len(graphs) >= Constant.N_NEIGHBOURS: + break + + return graphs diff --git a/new_impl/cv/third_party/nni/algorithms/hpo/networkmorphism_tuner/layer_transformer.py b/new_impl/cv/third_party/nni/algorithms/hpo/networkmorphism_tuner/layer_transformer.py new file mode 100644 index 0000000000000000000000000000000000000000..6ffd1b20fb0e3b4ef0a6d2e54a02fb7cfa7cfc42 --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/hpo/networkmorphism_tuner/layer_transformer.py @@ -0,0 +1,264 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import numpy as np + +from .layers import ( + StubDense, + StubReLU, + get_batch_norm_class, + get_conv_class, + get_n_dim, +) + +NOISE_RATIO = 1e-4 + + +def deeper_conv_block(conv_layer, kernel_size, weighted=True): + '''deeper conv layer. + ''' + n_dim = get_n_dim(conv_layer) + filter_shape = (kernel_size,) * 2 + n_filters = conv_layer.filters + weight = np.zeros((n_filters, n_filters) + filter_shape) + center = tuple(map(lambda x: int((x - 1) / 2), filter_shape)) + for i in range(n_filters): + filter_weight = np.zeros((n_filters,) + filter_shape) + index = (i,) + center + filter_weight[index] = 1 + weight[i, ...] = filter_weight + bias = np.zeros(n_filters) + new_conv_layer = get_conv_class(n_dim)( + conv_layer.filters, n_filters, kernel_size=kernel_size + ) + bn = get_batch_norm_class(n_dim)(n_filters) + + if weighted: + new_conv_layer.set_weights( + (add_noise(weight, np.array([0, 1])), + add_noise(bias, np.array([0, 1]))) + ) + new_weights = [ + add_noise(np.ones(n_filters, dtype=np.float32), np.array([0, 1])), + add_noise(np.zeros(n_filters, dtype=np.float32), np.array([0, 1])), + add_noise(np.zeros(n_filters, dtype=np.float32), np.array([0, 1])), + add_noise(np.ones(n_filters, dtype=np.float32), np.array([0, 1])), + ] + bn.set_weights(new_weights) + + return [StubReLU(), new_conv_layer, bn] + + +def dense_to_deeper_block(dense_layer, weighted=True): + '''deeper dense layer. + ''' + units = dense_layer.units + weight = np.eye(units) + bias = np.zeros(units) + new_dense_layer = StubDense(units, units) + if weighted: + new_dense_layer.set_weights( + (add_noise(weight, np.array([0, 1])), + add_noise(bias, np.array([0, 1]))) + ) + return [StubReLU(), new_dense_layer] + + +def wider_pre_dense(layer, n_add, weighted=True): + '''wider previous dense layer. + ''' + if not weighted: + return StubDense(layer.input_units, layer.units + n_add) + + n_units2 = layer.units + + teacher_w, teacher_b = layer.get_weights() + rand = np.random.randint(n_units2, size=n_add) + student_w = teacher_w.copy() + student_b = teacher_b.copy() + + # target layer update (i) + for i in range(n_add): + teacher_index = rand[i] + new_weight = teacher_w[teacher_index, :] + new_weight = new_weight[np.newaxis, :] + student_w = np.concatenate( + (student_w, add_noise(new_weight, student_w)), axis=0) + student_b = np.append( + student_b, add_noise( + teacher_b[teacher_index], student_b)) + + new_pre_layer = StubDense(layer.input_units, n_units2 + n_add) + new_pre_layer.set_weights((student_w, student_b)) + + return new_pre_layer + + +def wider_pre_conv(layer, n_add_filters, weighted=True): + '''wider previous conv layer. + ''' + n_dim = get_n_dim(layer) + if not weighted: + return get_conv_class(n_dim)( + layer.input_channel, + layer.filters + n_add_filters, + kernel_size=layer.kernel_size, + ) + + n_pre_filters = layer.filters + rand = np.random.randint(n_pre_filters, size=n_add_filters) + teacher_w, teacher_b = layer.get_weights() + + student_w = teacher_w.copy() + student_b = teacher_b.copy() + # target layer update (i) + for i, _ in enumerate(rand): + teacher_index = rand[i] + new_weight = teacher_w[teacher_index, ...] + new_weight = new_weight[np.newaxis, ...] + student_w = np.concatenate((student_w, new_weight), axis=0) + student_b = np.append(student_b, teacher_b[teacher_index]) + new_pre_layer = get_conv_class(n_dim)( + layer.input_channel, n_pre_filters + n_add_filters, layer.kernel_size + ) + new_pre_layer.set_weights( + (add_noise(student_w, teacher_w), add_noise(student_b, teacher_b)) + ) + return new_pre_layer + + +def wider_next_conv(layer, start_dim, total_dim, n_add, weighted=True): + '''wider next conv layer. + ''' + n_dim = get_n_dim(layer) + if not weighted: + return get_conv_class(n_dim)(layer.input_channel + n_add, + layer.filters, + kernel_size=layer.kernel_size, + stride=layer.stride) + n_filters = layer.filters + teacher_w, teacher_b = layer.get_weights() + + new_weight_shape = list(teacher_w.shape) + new_weight_shape[1] = n_add + new_weight = np.zeros(tuple(new_weight_shape)) + + student_w = np.concatenate((teacher_w[:, :start_dim, ...].copy(), + add_noise(new_weight, teacher_w), + teacher_w[:, start_dim:total_dim, ...].copy()), axis=1) + new_layer = get_conv_class(n_dim)(layer.input_channel + n_add, + n_filters, + kernel_size=layer.kernel_size, + stride=layer.stride) + new_layer.set_weights((student_w, teacher_b)) + return new_layer + + +def wider_bn(layer, start_dim, total_dim, n_add, weighted=True): + '''wider batch norm layer. + ''' + n_dim = get_n_dim(layer) + if not weighted: + return get_batch_norm_class(n_dim)(layer.num_features + n_add) + + weights = layer.get_weights() + + new_weights = [ + add_noise(np.ones(n_add, dtype=np.float32), np.array([0, 1])), + add_noise(np.zeros(n_add, dtype=np.float32), np.array([0, 1])), + add_noise(np.zeros(n_add, dtype=np.float32), np.array([0, 1])), + add_noise(np.ones(n_add, dtype=np.float32), np.array([0, 1])), + ] + + student_w = tuple() + for weight, new_weight in zip(weights, new_weights): + temp_w = weight.copy() + temp_w = np.concatenate( + (temp_w[:start_dim], new_weight, temp_w[start_dim:total_dim]) + ) + student_w += (temp_w,) + new_layer = get_batch_norm_class(n_dim)(layer.num_features + n_add) + new_layer.set_weights(student_w) + return new_layer + + +def wider_next_dense(layer, start_dim, total_dim, n_add, weighted=True): + '''wider next dense layer. + ''' + if not weighted: + return StubDense(layer.input_units + n_add, layer.units) + teacher_w, teacher_b = layer.get_weights() + student_w = teacher_w.copy() + n_units_each_channel = int(teacher_w.shape[1] / total_dim) + + new_weight = np.zeros((teacher_w.shape[0], n_add * n_units_each_channel)) + student_w = np.concatenate( + ( + student_w[:, : start_dim * n_units_each_channel], + add_noise(new_weight, student_w), + student_w[ + :, start_dim * n_units_each_channel: total_dim * n_units_each_channel + ], + ), + axis=1, + ) + + new_layer = StubDense(layer.input_units + n_add, layer.units) + new_layer.set_weights((student_w, teacher_b)) + return new_layer + + +def add_noise(weights, other_weights): + '''add noise to the layer. + ''' + w_range = np.ptp(other_weights.flatten()) + noise_range = NOISE_RATIO * w_range + noise = np.random.uniform(-noise_range / 2.0, + noise_range / 2.0, weights.shape) + return np.add(noise, weights) + + +def init_dense_weight(layer): + '''initilize dense layer weight. + ''' + units = layer.units + weight = np.eye(units) + bias = np.zeros(units) + layer.set_weights( + (add_noise(weight, np.array([0, 1])), + add_noise(bias, np.array([0, 1]))) + ) + + +def init_conv_weight(layer): + '''initilize conv layer weight. + ''' + n_filters = layer.filters + filter_shape = (layer.kernel_size,) * get_n_dim(layer) + weight = np.zeros((n_filters, n_filters) + filter_shape) + + center = tuple(map(lambda x: int((x - 1) / 2), filter_shape)) + for i in range(n_filters): + filter_weight = np.zeros((n_filters,) + filter_shape) + index = (i,) + center + filter_weight[index] = 1 + weight[i, ...] = filter_weight + bias = np.zeros(n_filters) + + layer.set_weights( + (add_noise(weight, np.array([0, 1])), + add_noise(bias, np.array([0, 1]))) + ) + + +def init_bn_weight(layer): + '''initilize batch norm layer weight. + ''' + n_filters = layer.num_features + new_weights = [ + add_noise(np.ones(n_filters, dtype=np.float32), np.array([0, 1])), + add_noise(np.zeros(n_filters, dtype=np.float32), np.array([0, 1])), + add_noise(np.zeros(n_filters, dtype=np.float32), np.array([0, 1])), + add_noise(np.ones(n_filters, dtype=np.float32), np.array([0, 1])), + ] + layer.set_weights(new_weights) diff --git a/new_impl/cv/third_party/nni/algorithms/hpo/networkmorphism_tuner/layers.py b/new_impl/cv/third_party/nni/algorithms/hpo/networkmorphism_tuner/layers.py new file mode 100644 index 0000000000000000000000000000000000000000..a96c87b7801fe2687f61dfd62fdd13019ecd2ee0 --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/hpo/networkmorphism_tuner/layers.py @@ -0,0 +1,862 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from abc import abstractmethod +from collections.abc import Iterable + +import torch +from torch import nn +from torch.nn import functional +from .utils import Constant + + +class AvgPool(nn.Module): + """ + AvgPool Module. + """ + + def __init__(self): + super().__init__() + + @abstractmethod + def forward(self, input_tensor): + pass + + +class GlobalAvgPool1d(AvgPool): + """ + GlobalAvgPool1d Module. + """ + + def forward(self, input_tensor): + return functional.avg_pool1d(input_tensor, input_tensor.size()[2:]).view( + input_tensor.size()[:2] + ) + + +class GlobalAvgPool2d(AvgPool): + """ + GlobalAvgPool2d Module. + """ + + def forward(self, input_tensor): + return functional.avg_pool2d(input_tensor, input_tensor.size()[2:]).view( + input_tensor.size()[:2] + ) + + +class GlobalAvgPool3d(AvgPool): + """ + GlobalAvgPool3d Module. + """ + + def forward(self, input_tensor): + return functional.avg_pool3d(input_tensor, input_tensor.size()[2:]).view( + input_tensor.size()[:2] + ) + + +class StubLayer: + """ + StubLayer Module. Base Module. + """ + + def __init__(self, input_node=None, output_node=None): + self.input = input_node + self.output = output_node + self.weights = None + + def build(self, shape): + """ + build shape. + """ + + def set_weights(self, weights): + """ + set weights. + """ + self.weights = weights + + def import_weights(self, torch_layer): + """ + import weights. + """ + + def import_weights_keras(self, keras_layer): + """ + import weights from keras layer. + """ + + def export_weights(self, torch_layer): + """ + export weights. + """ + + def export_weights_keras(self, keras_layer): + """ + export weights to keras layer. + """ + + def get_weights(self): + """ + get weights. + """ + return self.weights + + def size(self): + """ + size(). + """ + return 0 + + @property + def output_shape(self): + """ + output shape. + """ + return self.input.shape + + def to_real_layer(self): + """ + to real layer. + """ + + def __str__(self): + """ + str() function to print. + """ + return type(self).__name__[4:] + + +class StubWeightBiasLayer(StubLayer): + """ + StubWeightBiasLayer Module to set the bias. + """ + + def import_weights(self, torch_layer): + self.set_weights( + (torch_layer.weight.data.cpu().numpy(), + torch_layer.bias.data.cpu().numpy()) + ) + + def import_weights_keras(self, keras_layer): + self.set_weights(keras_layer.get_weights()) + + def export_weights(self, torch_layer): + torch_layer.weight.data = torch.Tensor(self.weights[0]) + torch_layer.bias.data = torch.Tensor(self.weights[1]) + + def export_weights_keras(self, keras_layer): + keras_layer.set_weights(self.weights) + + +class StubBatchNormalization(StubWeightBiasLayer): + """ + StubBatchNormalization Module. Batch Norm. + """ + + def __init__(self, num_features, input_node=None, output_node=None): + super().__init__(input_node, output_node) + self.num_features = num_features + + def import_weights(self, torch_layer): + self.set_weights( + ( + torch_layer.weight.data.cpu().numpy(), + torch_layer.bias.data.cpu().numpy(), + torch_layer.running_mean.cpu().numpy(), + torch_layer.running_var.cpu().numpy(), + ) + ) + + def export_weights(self, torch_layer): + torch_layer.weight.data = torch.Tensor(self.weights[0]) + torch_layer.bias.data = torch.Tensor(self.weights[1]) + torch_layer.running_mean = torch.Tensor(self.weights[2]) + torch_layer.running_var = torch.Tensor(self.weights[3]) + + def size(self): + return self.num_features * 4 + + @abstractmethod + def to_real_layer(self): + pass + + +class StubBatchNormalization1d(StubBatchNormalization): + """ + StubBatchNormalization1d Module. + """ + + def to_real_layer(self): + return torch.nn.BatchNorm1d(self.num_features) + + +class StubBatchNormalization2d(StubBatchNormalization): + """ + StubBatchNormalization2d Module. + """ + + def to_real_layer(self): + return torch.nn.BatchNorm2d(self.num_features) + + +class StubBatchNormalization3d(StubBatchNormalization): + """ + StubBatchNormalization3d Module. + """ + + def to_real_layer(self): + return torch.nn.BatchNorm3d(self.num_features) + + +class StubDense(StubWeightBiasLayer): + """ + StubDense Module. Linear. + """ + + def __init__(self, input_units, units, input_node=None, output_node=None): + super().__init__(input_node, output_node) + self.input_units = input_units + self.units = units + + @property + def output_shape(self): + return (self.units,) + + def import_weights_keras(self, keras_layer): + self.set_weights( + (keras_layer.get_weights()[0].T, + keras_layer.get_weights()[1])) + + def export_weights_keras(self, keras_layer): + keras_layer.set_weights((self.weights[0].T, self.weights[1])) + + def size(self): + return self.input_units * self.units + self.units + + def to_real_layer(self): + return torch.nn.Linear(self.input_units, self.units) + + +class StubConv(StubWeightBiasLayer): + """ + StubConv Module. Conv. + """ + + def __init__(self, input_channel, filters, kernel_size, + stride=1, input_node=None, output_node=None): + super().__init__(input_node, output_node) + self.input_channel = input_channel + self.filters = filters + self.kernel_size = kernel_size + self.stride = stride + self.padding = int(self.kernel_size / 2) + + @property + def output_shape(self): + ret = list(self.input.shape[:-1]) + for index, dim in enumerate(ret): + ret[index] = ( + int((dim + 2 * self.padding - self.kernel_size) / self.stride) + 1 + ) + ret = ret + [self.filters] + return tuple(ret) + + def import_weights_keras(self, keras_layer): + self.set_weights( + (keras_layer.get_weights()[0].T, + keras_layer.get_weights()[1])) + + def export_weights_keras(self, keras_layer): + keras_layer.set_weights((self.weights[0].T, self.weights[1])) + + def size(self): + return (self.input_channel * self.kernel_size * + self.kernel_size + 1) * self.filters + + @abstractmethod + def to_real_layer(self): + pass + + def __str__(self): + return ( + super().__str__() + + "(" + + ", ".join( + str(item) + for item in [ + self.input_channel, + self.filters, + self.kernel_size, + self.stride, + ] + ) + + ")" + ) + + +class StubConv1d(StubConv): + """ + StubConv1d Module. + """ + + def to_real_layer(self): + return torch.nn.Conv1d( + self.input_channel, + self.filters, + self.kernel_size, + stride=self.stride, + padding=self.padding, + ) + + +class StubConv2d(StubConv): + """ + StubConv2d Module. + """ + + def to_real_layer(self): + return torch.nn.Conv2d( + self.input_channel, + self.filters, + self.kernel_size, + stride=self.stride, + padding=self.padding, + ) + + +class StubConv3d(StubConv): + """ + StubConv3d Module. + """ + + def to_real_layer(self): + return torch.nn.Conv3d( + self.input_channel, + self.filters, + self.kernel_size, + stride=self.stride, + padding=self.padding, + ) + + +class StubAggregateLayer(StubLayer): + """ + StubAggregateLayer Module. + """ + + def __init__(self, input_nodes=None, output_node=None): + if input_nodes is None: + input_nodes = [] + super().__init__(input_nodes, output_node) + + +class StubConcatenate(StubAggregateLayer): + """StubConcatenate Module. + """ + @property + def output_shape(self): + ret = 0 + for current_input in self.input: + ret += current_input.shape[-1] + ret = self.input[0].shape[:-1] + (ret,) + return ret + + def to_real_layer(self): + return TorchConcatenate() + + +class StubAdd(StubAggregateLayer): + """ + StubAdd Module. + """ + @property + def output_shape(self): + return self.input[0].shape + + def to_real_layer(self): + return TorchAdd() + + +class StubFlatten(StubLayer): + """ + StubFlatten Module. + """ + @property + def output_shape(self): + ret = 1 + for dim in self.input.shape: + ret *= dim + return (ret,) + + def to_real_layer(self): + return TorchFlatten() + + +class StubReLU(StubLayer): + """ + StubReLU Module. + """ + + def to_real_layer(self): + return torch.nn.ReLU() + + +class StubSoftmax(StubLayer): + """ + StubSoftmax Module. + """ + + def to_real_layer(self): + return torch.nn.LogSoftmax(dim=1) + + +class StubDropout(StubLayer): + """ + StubDropout Module. + """ + + def __init__(self, rate, input_node=None, output_node=None): + super().__init__(input_node, output_node) + self.rate = rate + + @abstractmethod + def to_real_layer(self): + pass + + +class StubDropout1d(StubDropout): + """ + StubDropout1d Module. + """ + + def to_real_layer(self): + return torch.nn.Dropout(self.rate) + + +class StubDropout2d(StubDropout): + """ + StubDropout2d Module. + """ + + def to_real_layer(self): + return torch.nn.Dropout2d(self.rate) + + +class StubDropout3d(StubDropout): + """ + StubDropout3d Module. + """ + + def to_real_layer(self): + return torch.nn.Dropout3d(self.rate) + + +class StubInput(StubLayer): + """ + StubInput Module. + """ + + def __init__(self, input_node=None, output_node=None): + super().__init__(input_node, output_node) + + +class StubPooling(StubLayer): + """ + StubPooling Module. + """ + + def __init__(self, + kernel_size=None, + stride=None, + padding=0, + input_node=None, + output_node=None): + super().__init__(input_node, output_node) + self.kernel_size = ( + kernel_size if kernel_size is not None else Constant.POOLING_KERNEL_SIZE + ) + self.stride = stride if stride is not None else self.kernel_size + self.padding = padding + + @property + def output_shape(self): + ret = tuple() + for dim in self.input.shape[:-1]: + ret = ret + (max(int(dim / self.kernel_size), 1),) + ret = ret + (self.input.shape[-1],) + return ret + + @abstractmethod + def to_real_layer(self): + pass + + +class StubPooling1d(StubPooling): + """ + StubPooling1d Module. + """ + + def to_real_layer(self): + return torch.nn.MaxPool1d(self.kernel_size, stride=self.stride) + + +class StubPooling2d(StubPooling): + """ + StubPooling2d Module. + """ + + def to_real_layer(self): + return torch.nn.MaxPool2d(self.kernel_size, stride=self.stride) + + +class StubPooling3d(StubPooling): + """ + StubPooling3d Module. + """ + + def to_real_layer(self): + return torch.nn.MaxPool3d(self.kernel_size, stride=self.stride) + + +class StubGlobalPooling(StubLayer): + """ + StubGlobalPooling Module. + """ + + def __init__(self, input_node=None, output_node=None): + super().__init__(input_node, output_node) + + @property + def output_shape(self): + return (self.input.shape[-1],) + + @abstractmethod + def to_real_layer(self): + pass + + +class StubGlobalPooling1d(StubGlobalPooling): + """ + StubGlobalPooling1d Module. + """ + + def to_real_layer(self): + return GlobalAvgPool1d() + + +class StubGlobalPooling2d(StubGlobalPooling): + """ + StubGlobalPooling2d Module. + """ + + def to_real_layer(self): + return GlobalAvgPool2d() + + +class StubGlobalPooling3d(StubGlobalPooling): + """ + StubGlobalPooling3d Module. + """ + + def to_real_layer(self): + return GlobalAvgPool3d() + + +class TorchConcatenate(nn.Module): + """ + TorchConcatenate Module. + """ + + def forward(self, input_list): + return torch.cat(input_list, dim=1) + + +class TorchAdd(nn.Module): + """ + TorchAdd Module. + """ + + def forward(self, input_list): + return input_list[0] + input_list[1] + + +class TorchFlatten(nn.Module): + """ + TorchFlatten Module. + """ + + def forward(self, input_tensor): + return input_tensor.view(input_tensor.size(0), -1) + + +def keras_dropout(layer, rate): + """ + Keras dropout layer. + """ + + from keras import layers + + input_dim = len(layer.input.shape) + if input_dim == 2: + return layers.SpatialDropout1D(rate) + elif input_dim == 3: + return layers.SpatialDropout2D(rate) + elif input_dim == 4: + return layers.SpatialDropout3D(rate) + else: + return layers.Dropout(rate) + + +def to_real_keras_layer(layer): + """ + Real keras layer. + """ + from keras import layers + + if is_layer(layer, "Dense"): + return layers.Dense(layer.units, input_shape=(layer.input_units,)) + if is_layer(layer, "Conv"): + return layers.Conv2D( + layer.filters, + layer.kernel_size, + input_shape=layer.input.shape, + padding="same", + ) # padding + if is_layer(layer, "Pooling"): + return layers.MaxPool2D(2) + if is_layer(layer, "BatchNormalization"): + return layers.BatchNormalization(input_shape=layer.input.shape) + if is_layer(layer, "Concatenate"): + return layers.Concatenate() + if is_layer(layer, "Add"): + return layers.Add() + if is_layer(layer, "Dropout"): + return keras_dropout(layer, layer.rate) + if is_layer(layer, "ReLU"): + return layers.Activation("relu") + if is_layer(layer, "Softmax"): + return layers.Activation("softmax") + if is_layer(layer, "Flatten"): + return layers.Flatten() + if is_layer(layer, "GlobalAveragePooling"): + return layers.GlobalAveragePooling2D() + return None # note: this is not written by original author, feel free to modify if you think it's incorrect + + +def is_layer(layer, layer_type): + """ + Judge the layer type. + + Returns + ------- + bool + boolean -- True or False + """ + + if layer_type == "Input": + return isinstance(layer, StubInput) + elif layer_type == "Conv": + return isinstance(layer, StubConv) + elif layer_type == "Dense": + return isinstance(layer, (StubDense,)) + elif layer_type == "BatchNormalization": + return isinstance(layer, (StubBatchNormalization,)) + elif layer_type == "Concatenate": + return isinstance(layer, (StubConcatenate,)) + elif layer_type == "Add": + return isinstance(layer, (StubAdd,)) + elif layer_type == "Pooling": + return isinstance(layer, StubPooling) + elif layer_type == "Dropout": + return isinstance(layer, (StubDropout,)) + elif layer_type == "Softmax": + return isinstance(layer, (StubSoftmax,)) + elif layer_type == "ReLU": + return isinstance(layer, (StubReLU,)) + elif layer_type == "Flatten": + return isinstance(layer, (StubFlatten,)) + elif layer_type == "GlobalAveragePooling": + return isinstance(layer, StubGlobalPooling) + return None # note: this is not written by original author, feel free to modify if you think it's incorrect + + +def layer_description_extractor(layer, node_to_id): + """ + Get layer description. + """ + + layer_input = layer.input + layer_output = layer.output + if layer_input is not None: + if isinstance(layer_input, Iterable): + layer_input = list(map(lambda x: node_to_id[x], layer_input)) + else: + layer_input = node_to_id[layer_input] + + if layer_output is not None: + layer_output = node_to_id[layer_output] + + if isinstance(layer, StubConv): + return ( + type(layer).__name__, + layer_input, + layer_output, + layer.input_channel, + layer.filters, + layer.kernel_size, + layer.stride, + layer.padding, + ) + elif isinstance(layer, (StubDense,)): + return [ + type(layer).__name__, + layer_input, + layer_output, + layer.input_units, + layer.units, + ] + elif isinstance(layer, (StubBatchNormalization,)): + return (type(layer).__name__, layer_input, + layer_output, layer.num_features) + elif isinstance(layer, (StubDropout,)): + return (type(layer).__name__, layer_input, layer_output, layer.rate) + elif isinstance(layer, StubPooling): + return ( + type(layer).__name__, + layer_input, + layer_output, + layer.kernel_size, + layer.stride, + layer.padding, + ) + else: + return (type(layer).__name__, layer_input, layer_output) + + +def layer_description_builder(layer_information, id_to_node): + """build layer from description. + """ + layer_type = layer_information[0] + + layer_input_ids = layer_information[1] + if isinstance(layer_input_ids, Iterable): + layer_input = list(map(lambda x: id_to_node[x], layer_input_ids)) + else: + layer_input = id_to_node[layer_input_ids] + layer_output = id_to_node[layer_information[2]] + if layer_type.startswith("StubConv"): + input_channel = layer_information[3] + filters = layer_information[4] + kernel_size = layer_information[5] + stride = layer_information[6] + return globals()[layer_type]( + input_channel, filters, kernel_size, stride, layer_input, layer_output + ) + elif layer_type.startswith("StubDense"): + input_units = layer_information[3] + units = layer_information[4] + return globals()[layer_type](input_units, units, layer_input, layer_output) + elif layer_type.startswith("StubBatchNormalization"): + num_features = layer_information[3] + return globals()[layer_type](num_features, layer_input, layer_output) + elif layer_type.startswith("StubDropout"): + rate = layer_information[3] + return globals()[layer_type](rate, layer_input, layer_output) + elif layer_type.startswith("StubPooling"): + kernel_size = layer_information[3] + stride = layer_information[4] + padding = layer_information[5] + return globals()[layer_type](kernel_size, stride, padding, layer_input, layer_output) + else: + return globals()[layer_type](layer_input, layer_output) + + +def layer_width(layer): + """ + Get layer width. + """ + + if is_layer(layer, "Dense"): + return layer.units + if is_layer(layer, "Conv"): + return layer.filters + raise TypeError("The layer should be either Dense or Conv layer.") + + +def set_torch_weight_to_stub(torch_layer, stub_layer): + stub_layer.import_weights(torch_layer) + + +def set_keras_weight_to_stub(keras_layer, stub_layer): + stub_layer.import_weights_keras(keras_layer) + + +def set_stub_weight_to_torch(stub_layer, torch_layer): + stub_layer.export_weights(torch_layer) + + +def set_stub_weight_to_keras(stub_layer, keras_layer): + stub_layer.export_weights_keras(keras_layer) + + +def get_conv_class(n_dim): + conv_class_list = [StubConv1d, StubConv2d, StubConv3d] + return conv_class_list[n_dim - 1] + + +def get_dropout_class(n_dim): + dropout_class_list = [StubDropout1d, StubDropout2d, StubDropout3d] + return dropout_class_list[n_dim - 1] + + +def get_global_avg_pooling_class(n_dim): + global_avg_pooling_class_list = [ + StubGlobalPooling1d, + StubGlobalPooling2d, + StubGlobalPooling3d, + ] + return global_avg_pooling_class_list[n_dim - 1] + + +def get_pooling_class(n_dim): + pooling_class_list = [StubPooling1d, StubPooling2d, StubPooling3d] + return pooling_class_list[n_dim - 1] + + +def get_batch_norm_class(n_dim): + batch_norm_class_list = [ + StubBatchNormalization1d, + StubBatchNormalization2d, + StubBatchNormalization3d, + ] + return batch_norm_class_list[n_dim - 1] + + +def get_n_dim(layer): + if isinstance(layer, ( + StubConv1d, + StubDropout1d, + StubGlobalPooling1d, + StubPooling1d, + StubBatchNormalization1d, + )): + return 1 + if isinstance(layer, ( + StubConv2d, + StubDropout2d, + StubGlobalPooling2d, + StubPooling2d, + StubBatchNormalization2d, + )): + return 2 + if isinstance(layer, ( + StubConv3d, + StubDropout3d, + StubGlobalPooling3d, + StubPooling3d, + StubBatchNormalization3d, + )): + return 3 + return -1 diff --git a/new_impl/cv/third_party/nni/algorithms/hpo/networkmorphism_tuner/networkmorphism_tuner.py b/new_impl/cv/third_party/nni/algorithms/hpo/networkmorphism_tuner/networkmorphism_tuner.py new file mode 100644 index 0000000000000000000000000000000000000000..6a73cad3c6aa345a4e075b571b731417b1bc1615 --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/hpo/networkmorphism_tuner/networkmorphism_tuner.py @@ -0,0 +1,328 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +networkmorphsim_tuner.py +""" + +import logging +import os +from schema import Optional, Schema +from nni.tuner import Tuner +from nni.utils import OptimizeMode, extract_scalar_reward +from .bayesian import BayesianOptimizer +from .nn import CnnGenerator, MlpGenerator +from .utils import Constant +from .graph import graph_to_json, json_to_graph +from nni import ClassArgsValidator + +logger = logging.getLogger("NetworkMorphism_AutoML") + +class NetworkMorphismClassArgsValidator(ClassArgsValidator): + def validate_class_args(self, **kwargs): + Schema({ + Optional('optimize_mode'): self.choices('optimize_mode', 'maximize', 'minimize'), + Optional('task'): self.choices('task', 'cv', 'nlp', 'common'), + Optional('input_width'): int, + Optional('input_channel'): int, + Optional('n_output_node'): int + }).validate(kwargs) + +class NetworkMorphismTuner(Tuner): + """ + NetworkMorphismTuner is a tuner which using network morphism techniques. + + Attributes + ---------- + n_classes : int + The class number or output node number (default: ``10``) + input_shape : tuple + A tuple including: (input_width, input_width, input_channel) + t_min : float + The minimum temperature for simulated annealing. (default: ``Constant.T_MIN``) + beta : float + The beta in acquisition function. (default: ``Constant.BETA``) + algorithm_name : str + algorithm name used in the network morphism (default: ``"Bayesian"``) + optimize_mode : str + optimize mode "minimize" or "maximize" (default: ``"minimize"``) + verbose : bool + verbose to print the log (default: ``True``) + bo : BayesianOptimizer + The optimizer used in networkmorphsim tuner. + max_model_size : int + max model size to the graph (default: ``Constant.MAX_MODEL_SIZE``) + default_model_len : int + default model length (default: ``Constant.MODEL_LEN``) + default_model_width : int + default model width (default: ``Constant.MODEL_WIDTH``) + search_space : dict + """ + + def __init__( + self, + task="cv", + input_width=32, + input_channel=3, + n_output_node=10, + algorithm_name="Bayesian", + optimize_mode="maximize", + path="model_path", + verbose=True, + beta=Constant.BETA, + t_min=Constant.T_MIN, + max_model_size=Constant.MAX_MODEL_SIZE, + default_model_len=Constant.MODEL_LEN, + default_model_width=Constant.MODEL_WIDTH, + ): + """ + initilizer of the NetworkMorphismTuner. + """ + + if not os.path.exists(path): + os.makedirs(path) + self.path = os.path.join(os.getcwd(), path) + if task == "cv": + self.generators = [CnnGenerator] + elif task == "common": + self.generators = [MlpGenerator] + else: + raise NotImplementedError( + '{} task not supported in List ["cv","common"]') + + self.n_classes = n_output_node + self.input_shape = (input_width, input_width, input_channel) + + self.t_min = t_min + self.beta = beta + self.algorithm_name = algorithm_name + self.optimize_mode = OptimizeMode(optimize_mode) + self.json = None + self.total_data = {} + self.verbose = verbose + self.model_count = 0 + + self.bo = BayesianOptimizer( + self, self.t_min, self.optimize_mode, self.beta) + self.training_queue = [] + self.descriptors = [] + self.history = [] + + self.max_model_size = max_model_size + self.default_model_len = default_model_len + self.default_model_width = default_model_width + + self.search_space = dict() + + + def update_search_space(self, search_space): + """ + Update search space definition in tuner by search_space in neural architecture. + """ + self.search_space = search_space + + def generate_parameters(self, parameter_id, **kwargs): + """ + Returns a set of trial neural architecture, as a serializable object. + + Parameters + ---------- + parameter_id : int + """ + if not self.history: + self.init_search() + + new_father_id = None + generated_graph = None + if not self.training_queue: + new_father_id, generated_graph = self.generate() + new_model_id = self.model_count + self.model_count += 1 + self.training_queue.append( + (generated_graph, new_father_id, new_model_id)) + self.descriptors.append(generated_graph.extract_descriptor()) + + graph, father_id, model_id = self.training_queue.pop(0) + + # from graph to json + json_model_path = os.path.join(self.path, str(model_id) + ".json") + json_out = graph_to_json(graph, json_model_path) + self.total_data[parameter_id] = (json_out, father_id, model_id) + + return json_out + + def receive_trial_result(self, parameter_id, parameters, value, **kwargs): + """ + Record an observation of the objective function. + + Parameters + ---------- + parameter_id : int + the id of a group of paramters that generated by nni manager. + parameters : dict + A group of parameters. + value : dict/float + if value is dict, it should have "default" key. + """ + reward = extract_scalar_reward(value) + + if parameter_id not in self.total_data: + raise RuntimeError("Received parameter_id not in total_data.") + + (_, father_id, model_id) = self.total_data[parameter_id] + + graph = self.bo.searcher.load_model_by_id(model_id) + + # to use the value and graph + self.add_model(reward, model_id) + self.update(father_id, graph, reward, model_id) + + + def init_search(self): + """ + Call the generators to generate the initial architectures for the search. + """ + if self.verbose: + logger.info("Initializing search.") + for generator in self.generators: + graph = generator(self.n_classes, self.input_shape).generate( + self.default_model_len, self.default_model_width + ) + model_id = self.model_count + self.model_count += 1 + self.training_queue.append((graph, -1, model_id)) + self.descriptors.append(graph.extract_descriptor()) + + if self.verbose: + logger.info("Initialization finished.") + + + def generate(self): + """ + Generate the next neural architecture. + + Returns + ------- + other_info : any object + Anything to be saved in the training queue together with the architecture. + generated_graph : Graph + An instance of Graph. + """ + generated_graph, new_father_id = self.bo.generate(self.descriptors) + if new_father_id is None: + new_father_id = 0 + generated_graph = self.generators[0]( + self.n_classes, self.input_shape + ).generate(self.default_model_len, self.default_model_width) + + return new_father_id, generated_graph + + def update(self, other_info, graph, metric_value, model_id): + """ + Update the controller with evaluation result of a neural architecture. + + Parameters + ---------- + other_info: any object + In our case it is the father ID in the search tree. + graph: Graph + An instance of Graph. The trained neural architecture. + metric_value: float + The final evaluated metric value. + model_id: int + """ + father_id = other_info + self.bo.fit([graph.extract_descriptor()], [metric_value]) + self.bo.add_child(father_id, model_id) + + def add_model(self, metric_value, model_id): + """ + Add model to the history, x_queue and y_queue + + Parameters + ---------- + metric_value : float + graph : dict + model_id : int + + Returns + ------- + model : dict + """ + if self.verbose: + logger.info("Saving model.") + + # Update best_model text file + ret = {"model_id": model_id, "metric_value": metric_value} + self.history.append(ret) + if model_id == self.get_best_model_id(): + file = open(os.path.join(self.path, "best_model.txt"), "w") + file.write("best model: " + str(model_id)) + file.close() + return ret + + + def get_best_model_id(self): + """ + Get the best model_id from history using the metric value + """ + + if self.optimize_mode is OptimizeMode.Maximize: + return max(self.history, key=lambda x: x["metric_value"])[ + "model_id"] + return min(self.history, key=lambda x: x["metric_value"])["model_id"] + + + def load_model_by_id(self, model_id): + """ + Get the model by model_id + + Parameters + ---------- + model_id : int + model index + + Returns + ------- + load_model : Graph + the model graph representation + """ + + with open(os.path.join(self.path, str(model_id) + ".json")) as fin: + json_str = fin.read().replace("\n", "") + + load_model = json_to_graph(json_str) + return load_model + + def load_best_model(self): + """ + Get the best model by model id + + Returns + ------- + load_model : Graph + the model graph representation + """ + return self.load_model_by_id(self.get_best_model_id()) + + def get_metric_value_by_id(self, model_id): + """ + Get the model metric valud by its model_id + + Parameters + ---------- + model_id : int + model index + + Returns + ------- + float + the model metric + """ + for item in self.history: + if item["model_id"] == model_id: + return item["metric_value"] + return None + + def import_data(self, data): + pass diff --git a/new_impl/cv/third_party/nni/algorithms/hpo/networkmorphism_tuner/nn.py b/new_impl/cv/third_party/nni/algorithms/hpo/networkmorphism_tuner/nn.py new file mode 100644 index 0000000000000000000000000000000000000000..9e0072f9b39e3a68b865ba850157fe1080791983 --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/hpo/networkmorphism_tuner/nn.py @@ -0,0 +1,166 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from abc import abstractmethod + +from .graph import Graph +from .layers import (StubDense, StubDropout1d, + StubReLU, get_batch_norm_class, + get_conv_class, + get_dropout_class, + get_global_avg_pooling_class, + get_pooling_class) +from .utils import Constant + + +class NetworkGenerator: + """The base class for generating a network. + It can be used to generate a CNN or Multi-Layer Perceptron. + Attributes: + n_output_node: Number of output nodes in the network. + input_shape: A tuple to represent the input shape. + """ + + def __init__(self, n_output_node, input_shape): + self.n_output_node = n_output_node + self.input_shape = input_shape + + @abstractmethod + def generate(self, model_len, model_width): + pass + + +class CnnGenerator(NetworkGenerator): + """A class to generate CNN. + Attributes: + n_dim: `len(self.input_shape) - 1` + conv: A class that represents `(n_dim-1)` dimensional convolution. + dropout: A class that represents `(n_dim-1)` dimensional dropout. + global_avg_pooling: A class that represents `(n_dim-1)` dimensional Global Average Pooling. + pooling: A class that represents `(n_dim-1)` dimensional pooling. + batch_norm: A class that represents `(n_dim-1)` dimensional batch normalization. + """ + + def __init__(self, n_output_node, input_shape): + super(CnnGenerator, self).__init__(n_output_node, input_shape) + self.n_dim = len(self.input_shape) - 1 + if len(self.input_shape) > 4: + raise ValueError("The input dimension is too high.") + if len(self.input_shape) < 2: + raise ValueError("The input dimension is too low.") + self.conv = get_conv_class(self.n_dim) + self.dropout = get_dropout_class(self.n_dim) + self.global_avg_pooling = get_global_avg_pooling_class(self.n_dim) + self.pooling = get_pooling_class(self.n_dim) + self.batch_norm = get_batch_norm_class(self.n_dim) + + def generate(self, model_len=None, model_width=None): + """Generates a CNN. + Args: + model_len: An integer. Number of convolutional layers. + model_width: An integer. Number of filters for the convolutional layers. + Returns: + An instance of the class Graph. Represents the neural architecture graph of the generated model. + """ + + if model_len is None: + model_len = Constant.MODEL_LEN + if model_width is None: + model_width = Constant.MODEL_WIDTH + pooling_len = int(model_len / 4) + graph = Graph(self.input_shape, False) + temp_input_channel = self.input_shape[-1] + output_node_id = 0 + stride = 1 + for i in range(model_len): + output_node_id = graph.add_layer(StubReLU(), output_node_id) + output_node_id = graph.add_layer( + self.batch_norm( + graph.node_list[output_node_id].shape[-1]), output_node_id + ) + output_node_id = graph.add_layer( + self.conv( + temp_input_channel, + model_width, + kernel_size=3, + stride=stride), + output_node_id, + ) + temp_input_channel = model_width + if pooling_len == 0 or ( + (i + 1) % pooling_len == 0 and i != model_len - 1): + output_node_id = graph.add_layer( + self.pooling(), output_node_id) + + output_node_id = graph.add_layer( + self.global_avg_pooling(), output_node_id) + output_node_id = graph.add_layer( + self.dropout(Constant.CONV_DROPOUT_RATE), output_node_id + ) + output_node_id = graph.add_layer( + StubDense(graph.node_list[output_node_id].shape[0], model_width), + output_node_id, + ) + output_node_id = graph.add_layer(StubReLU(), output_node_id) + graph.add_layer( + StubDense( + model_width, + self.n_output_node), + output_node_id) + return graph + + +class MlpGenerator(NetworkGenerator): + """A class to generate Multi-Layer Perceptron. + """ + + def __init__(self, n_output_node, input_shape): + """Initialize the instance. + Args: + n_output_node: An integer. Number of output nodes in the network. + input_shape: A tuple. Input shape of the network. If it is 1D, ensure the value is appended by a comma + in the tuple. + """ + super(MlpGenerator, self).__init__(n_output_node, input_shape) + if len(self.input_shape) > 1: + raise ValueError("The input dimension is too high.") + + def generate(self, model_len=None, model_width=None): + """Generates a Multi-Layer Perceptron. + Args: + model_len: An integer. Number of hidden layers. + model_width: An integer or a list of integers of length `model_len`. If it is a list, it represents the + number of nodes in each hidden layer. If it is an integer, all hidden layers have nodes equal to this + value. + Returns: + An instance of the class Graph. Represents the neural architecture graph of the generated model. + """ + if model_len is None: + model_len = Constant.MODEL_LEN + if model_width is None: + model_width = Constant.MODEL_WIDTH + if isinstance(model_width, list) and not len(model_width) == model_len: + raise ValueError( + "The length of 'model_width' does not match 'model_len'") + elif isinstance(model_width, int): + model_width = [model_width] * model_len + + graph = Graph(self.input_shape, False) + output_node_id = 0 + n_nodes_prev_layer = self.input_shape[0] + for width in model_width: + output_node_id = graph.add_layer( + StubDense(n_nodes_prev_layer, width), output_node_id + ) + output_node_id = graph.add_layer( + StubDropout1d(Constant.MLP_DROPOUT_RATE), output_node_id + ) + output_node_id = graph.add_layer(StubReLU(), output_node_id) + n_nodes_prev_layer = width + + graph.add_layer( + StubDense( + n_nodes_prev_layer, + self.n_output_node), + output_node_id) + return graph diff --git a/new_impl/cv/third_party/nni/algorithms/hpo/networkmorphism_tuner/utils.py b/new_impl/cv/third_party/nni/algorithms/hpo/networkmorphism_tuner/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..0634e7f578bdeb505ce984be971beb3978c78a44 --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/hpo/networkmorphism_tuner/utils.py @@ -0,0 +1,23 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + + +class Constant: + '''Constant for the Tuner. + ''' + MAX_LAYERS = 100 + N_NEIGHBOURS = 8 + MAX_MODEL_SIZE = 1 << 24 + KERNEL_LAMBDA = 1.0 + BETA = 2.576 + MLP_MODEL_LEN = 3 + MLP_MODEL_WIDTH = 5 + MODEL_LEN = 3 + MODEL_WIDTH = 64 + POOLING_KERNEL_SIZE = 2 + DENSE_DROPOUT_RATE = 0.5 + CONV_DROPOUT_RATE = 0.25 + MLP_DROPOUT_RATE = 0.25 + CONV_BLOCK_DISTANCE = 2 + BATCH_SIZE = 128 + T_MIN = 0.0001 diff --git a/new_impl/cv/third_party/nni/algorithms/hpo/pbt_tuner/__init__.py b/new_impl/cv/third_party/nni/algorithms/hpo/pbt_tuner/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/new_impl/cv/third_party/nni/algorithms/hpo/pbt_tuner/pbt_tuner.py b/new_impl/cv/third_party/nni/algorithms/hpo/pbt_tuner/pbt_tuner.py new file mode 100644 index 0000000000000000000000000000000000000000..507c519a2a880ca99833433a6f61278e308a469f --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/hpo/pbt_tuner/pbt_tuner.py @@ -0,0 +1,456 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import copy +import logging +import os +import random +import numpy as np +from schema import Schema, Optional + +import nni +from nni import ClassArgsValidator +import nni.parameter_expressions +from nni.tuner import Tuner +from nni.utils import OptimizeMode, extract_scalar_reward, split_index, json2parameter, json2space + + +logger = logging.getLogger('pbt_tuner_AutoML') + + +def perturbation(hyperparameter_type, value, resample_probablity, uv, ub, lv, lb, random_state): + """ + Perturbation for hyperparameters + + Parameters + ---------- + hyperparameter_type : str + type of hyperparameter + value : list + parameters for sampling hyperparameter + resample_probability : float + probability for resampling + uv : float/int + upper value after perturbation + ub : float/int + upper bound + lv : float/int + lower value after perturbation + lb : float/int + lower bound + random_state : RandomState + random state + """ + if random.random() < resample_probablity: + if hyperparameter_type == "choice": + return value.index(nni.parameter_expressions.choice(value, random_state)) + else: + return getattr(nni.parameter_expressions, hyperparameter_type)(*(value + [random_state])) + else: + if random.random() > 0.5: + return min(uv, ub) + else: + return max(lv, lb) + + +def exploit_and_explore(bot_trial_info, top_trial_info, factor, resample_probability, epoch, search_space): + """ + Replace checkpoint of bot_trial with top, and perturb hyperparameters + + Parameters + ---------- + bot_trial_info : TrialInfo + bottom model whose parameters should be replaced + top_trial_info : TrialInfo + better model + factor : float + factor for perturbation + resample_probability : float + probability for resampling + epoch : int + step of PBTTuner + search_space : dict + search_space to keep perturbed hyperparameters in range + """ + bot_checkpoint_dir = bot_trial_info.checkpoint_dir + top_hyper_parameters = top_trial_info.hyper_parameters + hyper_parameters = copy.deepcopy(top_hyper_parameters) + random_state = np.random.RandomState() + hyper_parameters['load_checkpoint_dir'] = hyper_parameters['save_checkpoint_dir'] + hyper_parameters['save_checkpoint_dir'] = os.path.join(bot_checkpoint_dir, str(epoch)) + for key in hyper_parameters.keys(): + hyper_parameter = hyper_parameters[key] + if key == 'load_checkpoint_dir' or key == 'save_checkpoint_dir': + continue + elif search_space[key]["_type"] == "choice": + choices = search_space[key]["_value"] + ub, uv = len(choices) - 1, choices.index(hyper_parameter) + 1 + lb, lv = 0, choices.index(hyper_parameter) - 1 + elif search_space[key]["_type"] == "randint": + lb, ub = search_space[key]["_value"][:2] + ub -= 1 + uv = hyper_parameter + 1 + lv = hyper_parameter - 1 + elif search_space[key]["_type"] == "uniform": + lb, ub = search_space[key]["_value"][:2] + perturb = (ub - lb) * factor + uv = hyper_parameter + perturb + lv = hyper_parameter - perturb + elif search_space[key]["_type"] == "quniform": + lb, ub, q = search_space[key]["_value"][:3] + multi = round(hyper_parameter / q) + uv = (multi + 1) * q + lv = (multi - 1) * q + elif search_space[key]["_type"] == "loguniform": + lb, ub = search_space[key]["_value"][:2] + perturb = (np.log(ub) - np.log(lb)) * factor + uv = np.exp(min(np.log(hyper_parameter) + perturb, np.log(ub))) + lv = np.exp(max(np.log(hyper_parameter) - perturb, np.log(lb))) + elif search_space[key]["_type"] == "qloguniform": + lb, ub, q = search_space[key]["_value"][:3] + multi = round(hyper_parameter / q) + uv = (multi + 1) * q + lv = (multi - 1) * q + elif search_space[key]["_type"] == "normal": + sigma = search_space[key]["_value"][1] + perturb = sigma * factor + uv = ub = hyper_parameter + perturb + lv = lb = hyper_parameter - perturb + elif search_space[key]["_type"] == "qnormal": + q = search_space[key]["_value"][2] + uv = ub = hyper_parameter + q + lv = lb = hyper_parameter - q + elif search_space[key]["_type"] == "lognormal": + sigma = search_space[key]["_value"][1] + perturb = sigma * factor + uv = ub = np.exp(np.log(hyper_parameter) + perturb) + lv = lb = np.exp(np.log(hyper_parameter) - perturb) + elif search_space[key]["_type"] == "qlognormal": + q = search_space[key]["_value"][2] + uv = ub = hyper_parameter + q + lv, lb = hyper_parameter - q, 1E-10 + else: + logger.warning("Illegal type to perturb: %s", search_space[key]["_type"]) + continue + + if search_space[key]["_type"] == "choice": + idx = perturbation(search_space[key]["_type"], search_space[key]["_value"], + resample_probability, uv, ub, lv, lb, random_state) + hyper_parameters[key] = choices[idx] + else: + hyper_parameters[key] = perturbation(search_space[key]["_type"], search_space[key]["_value"], + resample_probability, uv, ub, lv, lb, random_state) + bot_trial_info.hyper_parameters = hyper_parameters + bot_trial_info.clean_id() + + +class TrialInfo: + """ + Information of each trial, refresh for each epoch + + """ + + def __init__(self, checkpoint_dir=None, hyper_parameters=None, parameter_id=None, score=None): + self.checkpoint_dir = checkpoint_dir + self.hyper_parameters = hyper_parameters + self.parameter_id = parameter_id + self.score = score + + def clean_id(self): + self.parameter_id = None + +class PBTClassArgsValidator(ClassArgsValidator): + def validate_class_args(self, **kwargs): + Schema({ + 'optimize_mode': self.choices('optimize_mode', 'maximize', 'minimize'), + Optional('all_checkpoint_dir'): str, + Optional('population_size'): self.range('population_size', int, 0, 99999), + Optional('factors'): float, + Optional('fraction'): float, + }).validate(kwargs) + +class PBTTuner(Tuner): + def __init__(self, optimize_mode="maximize", all_checkpoint_dir=None, population_size=10, factor=0.2, + resample_probability=0.25, fraction=0.2): + """ + Initialization + + Parameters + ---------- + optimize_mode : str + maximize or minimize + all_checkpoint_dir : str + directory to store training model checkpoint + population_size : int + number of trials for each epoch + factor : float + factor for perturbation + resample_probability : float + probability for resampling + fraction : float + fraction for selecting bottom and top trials + """ + self.optimize_mode = OptimizeMode(optimize_mode) + if all_checkpoint_dir is None: + all_checkpoint_dir = os.getenv('NNI_CHECKPOINT_DIRECTORY') + logger.info("Checkpoint dir is set to %s by default.", all_checkpoint_dir) + self.all_checkpoint_dir = all_checkpoint_dir + self.population_size = population_size + self.factor = factor + self.resample_probability = resample_probability + self.fraction = fraction + # defined in trial code + #self.perturbation_interval = perturbation_interval + + self.population = None + self.pos = -1 + self.param_ids = [] + self.running = {} + self.finished = [] + self.credit = 0 + self.finished_trials = 0 + self.epoch = 0 + + self.searchspace_json = None + self.space = None + + self.send_trial_callback = None + + logger.info('PBT tuner initialization') + + def update_search_space(self, search_space): + """ + Get search space + + Parameters + ---------- + search_space : dict + Search space + """ + logger.info('Update search space %s', search_space) + self.searchspace_json = search_space + self.space = json2space(self.searchspace_json) + + self.random_state = np.random.RandomState() + self.population = [] + is_rand = dict() + + for item in self.space: + is_rand[item] = True + + for i in range(self.population_size): + hyper_parameters = json2parameter( + self.searchspace_json, is_rand, self.random_state) + hyper_parameters = split_index(hyper_parameters) + checkpoint_dir = os.path.join(self.all_checkpoint_dir, str(i)) + hyper_parameters['load_checkpoint_dir'] = os.path.join(checkpoint_dir, str(self.epoch)) + hyper_parameters['save_checkpoint_dir'] = os.path.join(checkpoint_dir, str(self.epoch)) + self.population.append(TrialInfo(checkpoint_dir=checkpoint_dir, hyper_parameters=hyper_parameters)) + + def generate_multiple_parameters(self, parameter_id_list, **kwargs): + """ + Returns multiple sets of trial (hyper-)parameters, as iterable of serializable objects. + + Parameters + ---------- + parameter_id_list : list of int + Unique identifiers for each set of requested hyper-parameters. + These will later be used in :meth:`receive_trial_result`. + **kwargs + Used for send_trial_callback. + + Returns + ------- + list + A list of newly generated configurations + """ + result = [] + self.send_trial_callback = kwargs['st_callback'] + for parameter_id in parameter_id_list: + had_exception = False + try: + logger.debug("generating param for %s", parameter_id) + res = self.generate_parameters(parameter_id, **kwargs) + except nni.NoMoreTrialError: + had_exception = True + if not had_exception: + result.append(res) + return result + + def generate_parameters(self, parameter_id, **kwargs): + """ + Generate parameters, if no trial configration for now, self.credit plus 1 to send the config later + + Parameters + ---------- + parameter_id : int + Unique identifier for requested hyper-parameters. + This will later be used in :meth:`receive_trial_result`. + **kwargs + Not used + + Returns + ------- + dict + One newly generated configuration + + """ + if self.pos == self.population_size - 1: + logger.debug('Credit added by one in parameters request') + self.credit += 1 + self.param_ids.append(parameter_id) + raise nni.NoMoreTrialError('No more parameters now.') + self.pos += 1 + trial_info = self.population[self.pos] + trial_info.parameter_id = parameter_id + self.running[parameter_id] = trial_info + logger.info('Generate parameter : %s', trial_info.hyper_parameters) + return trial_info.hyper_parameters + + def _proceed_next_epoch(self): + """ + """ + logger.info('Proceeding to next epoch') + self.epoch += 1 + self.population = [] + self.pos = -1 + self.running = {} + #exploit and explore + reverse = True if self.optimize_mode == OptimizeMode.Maximize else False + self.finished = sorted(self.finished, key=lambda x: x.score, reverse=reverse) + cutoff = int(np.ceil(self.fraction * len(self.finished))) + tops = self.finished[:cutoff] + bottoms = self.finished[self.finished_trials - cutoff:] + for bottom in bottoms: + top = np.random.choice(tops) + exploit_and_explore(bottom, top, self.factor, self.resample_probability, self.epoch, self.searchspace_json) + for trial in self.finished: + if trial not in bottoms: + trial.clean_id() + trial.hyper_parameters['load_checkpoint_dir'] = trial.hyper_parameters['save_checkpoint_dir'] + trial.hyper_parameters['save_checkpoint_dir'] = os.path.join(trial.checkpoint_dir, str(self.epoch)) + self.finished_trials = 0 + for _ in range(self.population_size): + trial_info = self.finished.pop() + self.population.append(trial_info) + while self.credit > 0 and self.pos + 1 < len(self.population): + self.credit -= 1 + self.pos += 1 + parameter_id = self.param_ids.pop() + trial_info = self.population[self.pos] + trial_info.parameter_id = parameter_id + self.running[parameter_id] = trial_info + self.send_trial_callback(parameter_id, trial_info.hyper_parameters) + + def receive_trial_result(self, parameter_id, parameters, value, **kwargs): + """ + Receive trial's result. if the number of finished trials equals ``self.population_size``, start the next epoch to + train the model. + + Parameters + ---------- + parameter_id : int + Unique identifier of used hyper-parameters, same with :meth:`generate_parameters`. + parameters : dict + Hyper-parameters generated by :meth:`generate_parameters`. + value : dict + Result from trial (the return value of :func:`nni.report_final_result`). + """ + logger.info('Get one trial result, id = %d, value = %s', parameter_id, value) + value = extract_scalar_reward(value) + trial_info = self.running.pop(parameter_id, None) + trial_info.score = value + self.finished.append(trial_info) + self.finished_trials += 1 + if self.finished_trials == self.population_size: + self._proceed_next_epoch() + + def trial_end(self, parameter_id, success, **kwargs): + """ + Deal with trial failure + + Parameters + ---------- + parameter_id : int + Unique identifier for hyper-parameters used by this trial. + success : bool + True if the trial successfully completed; False if failed or terminated. + **kwargs + Unstable parameters which should be ignored by normal users. + """ + if success: + return + if self.optimize_mode == OptimizeMode.Minimize: + value = float('inf') + else: + value = float('-inf') + trial_info = self.running.pop(parameter_id, None) + trial_info.score = value + self.finished.append(trial_info) + self.finished_trials += 1 + if self.finished_trials == self.population_size: + self._proceed_next_epoch() + + def import_data(self, data): + """ + Parameters + ---------- + data : json obj + imported data records + + Returns + ------- + int + the start epoch number after data imported, only used for unittest + """ + if self.running: + logger.warning("Do not support importing data in the middle of experiment") + return + # the following is for experiment resume + _completed_num = 0 + epoch_data_dict = {} + for trial_info in data: + logger.info("Process data record %s / %s", _completed_num, len(data)) + _completed_num += 1 + # simply validate data format + _params = trial_info["parameter"] + _value = trial_info['value'] + # assign fake value for failed trials + if not _value: + logger.info("Useless trial data, value is %s, skip this trial data.", _value) + _value = float('inf') if self.optimize_mode == OptimizeMode.Minimize else float('-inf') + _value = extract_scalar_reward(_value) + if 'save_checkpoint_dir' not in _params: + logger.warning("Invalid data record: save_checkpoint_dir is missing, abandon data import.") + return + epoch_num = int(os.path.basename(_params['save_checkpoint_dir'])) + if epoch_num not in epoch_data_dict: + epoch_data_dict[epoch_num] = [] + epoch_data_dict[epoch_num].append((_params, _value)) + if not epoch_data_dict: + logger.warning("No valid epochs, abandon data import.") + return + # figure out start epoch for resume + max_epoch_num = max(epoch_data_dict, key=int) + if len(epoch_data_dict[max_epoch_num]) < self.population_size: + max_epoch_num -= 1 + # If there is no a single complete round, no data to import, start from scratch + if max_epoch_num < 0: + logger.warning("No completed epoch, abandon data import.") + return + assert len(epoch_data_dict[max_epoch_num]) == self.population_size + # check existence of trial save checkpoint dir + for params, _ in epoch_data_dict[max_epoch_num]: + if not os.path.isdir(params['save_checkpoint_dir']): + logger.warning("save_checkpoint_dir %s does not exist, data will not be resumed", params['save_checkpoint_dir']) + return + # resume data + self.epoch = max_epoch_num + self.finished_trials = self.population_size + for params, value in epoch_data_dict[max_epoch_num]: + checkpoint_dir = os.path.dirname(params['save_checkpoint_dir']) + self.finished.append(TrialInfo(checkpoint_dir=checkpoint_dir, hyper_parameters=params, score=value)) + self._proceed_next_epoch() + logger.info("Successfully import data to PBT tuner, total data: %d, imported data: %d.", len(data), self.population_size) + logger.info("Start from epoch %d ...", self.epoch) + return self.epoch # return for test diff --git a/new_impl/cv/third_party/nni/algorithms/hpo/ppo_tuner/__init__.py b/new_impl/cv/third_party/nni/algorithms/hpo/ppo_tuner/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..ada7e57c23d21f92063544430c6b0d41b05a6680 --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/hpo/ppo_tuner/__init__.py @@ -0,0 +1 @@ +from .ppo_tuner import PPOTuner diff --git a/new_impl/cv/third_party/nni/algorithms/hpo/ppo_tuner/distri.py b/new_impl/cv/third_party/nni/algorithms/hpo/ppo_tuner/distri.py new file mode 100644 index 0000000000000000000000000000000000000000..8a2a5ed20c3db6086c74339428fd25e0cf806f57 --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/hpo/ppo_tuner/distri.py @@ -0,0 +1,183 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +functions for sampling from hidden state +""" + +import tensorflow.compat.v1 as tf +tf.disable_v2_behavior() + +from .util import fc + + +class Pd: + """ + A particular probability distribution + """ + def flatparam(self): + raise NotImplementedError + def mode(self): + raise NotImplementedError + def neglogp(self, x): + # Usually it's easier to define the negative logprob + raise NotImplementedError + def kl(self, other): + raise NotImplementedError + def entropy(self): + raise NotImplementedError + def sample(self): + raise NotImplementedError + def logp(self, x): + return - self.neglogp(x) + def get_shape(self): + return self.flatparam().shape + @property + def shape(self): + return self.get_shape() + def __getitem__(self, idx): + return self.__class__(self.flatparam()[idx]) + +class PdType: + """ + Parametrized family of probability distributions + """ + def pdclass(self): + raise NotImplementedError + def pdfromflat(self, flat, mask, nsteps, size, is_act_model): + return self.pdclass()(flat, mask, nsteps, size, is_act_model) + def pdfromlatent(self, latent_vector, init_scale, init_bias): + raise NotImplementedError + def param_shape(self): + raise NotImplementedError + def sample_shape(self): + raise NotImplementedError + def sample_dtype(self): + raise NotImplementedError + + def param_placeholder(self, prepend_shape, name=None): + return tf.placeholder(dtype=tf.float32, shape=prepend_shape+self.param_shape(), name=name) + def sample_placeholder(self, prepend_shape, name=None): + return tf.placeholder(dtype=self.sample_dtype(), shape=prepend_shape+self.sample_shape(), name=name) + +class CategoricalPd(Pd): + """ + Categorical probability distribution + """ + def __init__(self, logits, mask_npinf, nsteps, size, is_act_model): + self.logits = logits + self.mask_npinf = mask_npinf + self.nsteps = nsteps + self.size = size + self.is_act_model = is_act_model + def flatparam(self): + return self.logits + def mode(self): + return tf.argmax(self.logits, axis=-1) + + @property + def mean(self): + return tf.nn.softmax(self.logits) + def neglogp(self, x): + """ + return tf.nn.sparse_softmax_cross_entropy_with_logits(logits=self.logits, labels=x) + Note: we can't use sparse_softmax_cross_entropy_with_logits because + the implementation does not allow second-order derivatives... + """ + if x.dtype in {tf.uint8, tf.int32, tf.int64}: + # one-hot encoding + x_shape_list = x.shape.as_list() + logits_shape_list = self.logits.get_shape().as_list()[:-1] + for xs, ls in zip(x_shape_list, logits_shape_list): + if xs is not None and ls is not None: + assert xs == ls, 'shape mismatch: {} in x vs {} in logits'.format(xs, ls) + + x = tf.one_hot(x, self.logits.get_shape().as_list()[-1]) + else: + # already encoded + assert x.shape.as_list() == self.logits.shape.as_list() + + return tf.nn.softmax_cross_entropy_with_logits_v2( + logits=self.logits, + labels=x) + + def kl(self, other): + """kl""" + a0 = self.logits - tf.reduce_max(self.logits, axis=-1, keepdims=True) + a1 = other.logits - tf.reduce_max(other.logits, axis=-1, keepdims=True) + ea0 = tf.exp(a0) + ea1 = tf.exp(a1) + z0 = tf.reduce_sum(ea0, axis=-1, keepdims=True) + z1 = tf.reduce_sum(ea1, axis=-1, keepdims=True) + p0 = ea0 / z0 + return tf.reduce_sum(p0 * (a0 - tf.log(z0) - a1 + tf.log(z1)), axis=-1) + + def entropy(self): + """compute entropy""" + a0 = self.logits - tf.reduce_max(self.logits, axis=-1, keepdims=True) + ea0 = tf.exp(a0) + z0 = tf.reduce_sum(ea0, axis=-1, keepdims=True) + p0 = ea0 / z0 + return tf.reduce_sum(p0 * (tf.log(z0) - a0), axis=-1) + + def sample(self): + """sample from logits""" + if not self.is_act_model: + re_res = tf.reshape(self.logits, [-1, self.nsteps, self.size]) + masked_res = tf.math.add(re_res, self.mask_npinf) + re_masked_res = tf.reshape(masked_res, [-1, self.size]) + + u = tf.random_uniform(tf.shape(re_masked_res), dtype=self.logits.dtype) + return tf.argmax(re_masked_res - tf.log(-1*tf.log(u)), axis=-1) + else: + u = tf.random_uniform(tf.shape(self.logits), dtype=self.logits.dtype) + return tf.argmax(self.logits - tf.log(-1*tf.log(u)), axis=-1) + + @classmethod + def fromflat(cls, flat): + return cls(flat) # pylint: disable=no-value-for-parameter + +class CategoricalPdType(PdType): + """ + To create CategoricalPd + """ + def __init__(self, ncat, nsteps, np_mask, is_act_model): + self.ncat = ncat + self.nsteps = nsteps + self.np_mask = np_mask + self.is_act_model = is_act_model + def pdclass(self): + return CategoricalPd + + def pdfromlatent(self, latent_vector, init_scale=1.0, init_bias=0.0): + """add fc and create CategoricalPd""" + pdparam, mask, mask_npinf = _matching_fc(latent_vector, 'pi', self.ncat, self.nsteps, + init_scale=init_scale, init_bias=init_bias, + np_mask=self.np_mask, is_act_model=self.is_act_model) + return self.pdfromflat(pdparam, mask_npinf, self.nsteps, self.ncat, self.is_act_model), pdparam, mask, mask_npinf + + def param_shape(self): + return [self.ncat] + def sample_shape(self): + return [] + def sample_dtype(self): + return tf.int32 + +def _matching_fc(tensor, name, size, nsteps, init_scale, init_bias, np_mask, is_act_model): + """ + Add fc op, and add mask op when not in action mode + """ + if tensor.shape[-1] == size: + assert False + return tensor + else: + mask = tf.get_variable("act_mask", dtype=tf.float32, initializer=np_mask[0], trainable=False) + mask_npinf = tf.get_variable("act_mask_npinf", dtype=tf.float32, initializer=np_mask[1], trainable=False) + res = fc(tensor, name, size, init_scale=init_scale, init_bias=init_bias) + if not is_act_model: + re_res = tf.reshape(res, [-1, nsteps, size]) + masked_res = tf.math.multiply(re_res, mask) + re_masked_res = tf.reshape(masked_res, [-1, size]) + return re_masked_res, mask, mask_npinf + else: + return res, mask, mask_npinf diff --git a/new_impl/cv/third_party/nni/algorithms/hpo/ppo_tuner/model.py b/new_impl/cv/third_party/nni/algorithms/hpo/ppo_tuner/model.py new file mode 100644 index 0000000000000000000000000000000000000000..c6a8479c6d0d06fcf0f7c4b8a64a62ab968988aa --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/hpo/ppo_tuner/model.py @@ -0,0 +1,152 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +the main model of policy/value network +""" + +import tensorflow.compat.v1 as tf +tf.disable_v2_behavior() + +from .util import initialize, get_session + +class Model: + """ + We use this object to : + __init__: + - Creates the step_model + - Creates the train_model + + train(): + - Make the training part (feedforward and retropropagation of gradients) + + save/load(): + - Save load the model + """ + def __init__(self, *, policy, nbatch_act, nbatch_train, + nsteps, ent_coef, vf_coef, max_grad_norm, microbatch_size=None, np_mask=None): + self.sess = sess = get_session() + + with tf.variable_scope('ppo2_model', reuse=tf.AUTO_REUSE): + # CREATE OUR TWO MODELS + # act_model that is used for sampling + act_model = policy(nbatch_act, 1, sess, np_mask=np_mask, is_act_model=True) + + # Train model for training + if microbatch_size is None: + train_model = policy(nbatch_train, nsteps, sess, np_mask=np_mask, is_act_model=False) + else: + train_model = policy(microbatch_size, nsteps, sess, np_mask=np_mask, is_act_model=False) + + # CREATE THE PLACEHOLDERS + self.A = A = train_model.pdtype.sample_placeholder([None]) + self.ADV = ADV = tf.placeholder(tf.float32, [None]) + self.R = R = tf.placeholder(tf.float32, [None]) + # Keep track of old actor + self.OLDNEGLOGPAC = OLDNEGLOGPAC = tf.placeholder(tf.float32, [None]) + # Keep track of old critic + self.OLDVPRED = OLDVPRED = tf.placeholder(tf.float32, [None]) + self.LR = LR = tf.placeholder(tf.float32, []) + # Cliprange + self.CLIPRANGE = CLIPRANGE = tf.placeholder(tf.float32, []) + + neglogpac = train_model.pd.neglogp(A) + + # Calculate the entropy + # Entropy is used to improve exploration by limiting the premature convergence to suboptimal policy. + entropy = tf.reduce_mean(train_model.pd.entropy()) + + # CALCULATE THE LOSS + # Total loss = Policy gradient loss - entropy * entropy coefficient + Value coefficient * value loss + + # Clip the value to reduce variability during Critic training + # Get the predicted value + vpred = train_model.vf + vpredclipped = OLDVPRED + tf.clip_by_value(train_model.vf - OLDVPRED, - CLIPRANGE, CLIPRANGE) + # Unclipped value + vf_losses1 = tf.square(vpred - R) + # Clipped value + vf_losses2 = tf.square(vpredclipped - R) + + vf_loss = .5 * tf.reduce_mean(tf.maximum(vf_losses1, vf_losses2)) + + # Calculate ratio (pi current policy / pi old policy) + ratio = tf.exp(OLDNEGLOGPAC - neglogpac) + + # Defining Loss = - J is equivalent to max J + pg_losses = -ADV * ratio + + pg_losses2 = -ADV * tf.clip_by_value(ratio, 1.0 - CLIPRANGE, 1.0 + CLIPRANGE) + + # Final PG loss + pg_loss = tf.reduce_mean(tf.maximum(pg_losses, pg_losses2)) + approxkl = .5 * tf.reduce_mean(tf.square(neglogpac - OLDNEGLOGPAC)) + clipfrac = tf.reduce_mean(tf.to_float(tf.greater(tf.abs(ratio - 1.0), CLIPRANGE))) + + # Total loss + loss = pg_loss - entropy * ent_coef + vf_loss * vf_coef + + # UPDATE THE PARAMETERS USING LOSS + # 1. Get the model parameters + params = tf.trainable_variables('ppo2_model') + # 2. Build our trainer + self.trainer = tf.train.AdamOptimizer(learning_rate=LR, epsilon=1e-5) + # 3. Calculate the gradients + grads_and_var = self.trainer.compute_gradients(loss, params) + grads, var = zip(*grads_and_var) + + if max_grad_norm is not None: + # Clip the gradients (normalize) + grads, _grad_norm = tf.clip_by_global_norm(grads, max_grad_norm) + grads_and_var = list(zip(grads, var)) + # zip aggregate each gradient with parameters associated + # For instance zip(ABCD, xyza) => Ax, By, Cz, Da + + self.grads = grads + self.var = var + self._train_op = self.trainer.apply_gradients(grads_and_var) + self.loss_names = ['policy_loss', 'value_loss', 'policy_entropy', 'approxkl', 'clipfrac'] + self.stats_list = [pg_loss, vf_loss, entropy, approxkl, clipfrac] + + + self.train_model = train_model + self.act_model = act_model + self.step = act_model.step + self.value = act_model.value + self.initial_state = act_model.initial_state + + initialize() + + def train(self, lr, cliprange, obs, returns, masks, actions, values, neglogpacs, states=None): + """ + Train the model. + Here we calculate advantage A(s,a) = R + yV(s') - V(s) + + Returns + ------- + obj + = R + yV(s') + """ + advs = returns - values + + # Normalize the advantages + advs = (advs - advs.mean()) / (advs.std() + 1e-8) + + td_map = { + self.train_model.X : obs, + self.A : actions, + self.ADV : advs, + self.R : returns, + self.LR : lr, + self.CLIPRANGE : cliprange, + self.OLDNEGLOGPAC : neglogpacs, + self.OLDVPRED : values + } + if states is not None: + td_map[self.train_model.S] = states + td_map[self.train_model.M] = masks + + return self.sess.run( + self.stats_list + [self._train_op], + td_map + )[:-1] diff --git a/new_impl/cv/third_party/nni/algorithms/hpo/ppo_tuner/policy.py b/new_impl/cv/third_party/nni/algorithms/hpo/ppo_tuner/policy.py new file mode 100644 index 0000000000000000000000000000000000000000..a35e514eaef36562c26c6a99e8224b4339ae768c --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/hpo/ppo_tuner/policy.py @@ -0,0 +1,230 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +build policy/value network from model +""" + +import tensorflow.compat.v1 as tf +tf.disable_v2_behavior() + +from .distri import CategoricalPdType +from .util import lstm_model, fc, observation_placeholder, adjust_shape + + +class PolicyWithValue: + """ + Encapsulates fields and methods for RL policy and value function estimation with shared parameters + """ + + def __init__(self, env, observations, latent, estimate_q=False, vf_latent=None, sess=None, np_mask=None, is_act_model=False, **tensors): + """ + Parameters + ---------- + env : obj + RL environment + observations : tensorflow placeholder + Tensorflow placeholder in which the observations will be fed + latent : tensor + Latent state from which policy distribution parameters should be inferred + vf_latent : tensor + Latent state from which value function should be inferred (if None, then latent is used) + sess : tensorflow session + Tensorflow session to run calculations in (if None, default session is used) + **tensors + Tensorflow tensors for additional attributes such as state or mask + """ + + self.X = observations + self.state = tf.constant([]) + self.initial_state = None + self.__dict__.update(tensors) + + vf_latent = vf_latent if vf_latent is not None else latent + + vf_latent = tf.layers.flatten(vf_latent) + latent = tf.layers.flatten(latent) + + # Based on the action space, will select what probability distribution type + self.np_mask = np_mask + self.pdtype = CategoricalPdType(env.action_space.n, env.nsteps, np_mask, is_act_model) + + self.act_latent = latent + self.nh = env.action_space.n + + self.pd, self.pi, self.mask, self.mask_npinf = self.pdtype.pdfromlatent(latent, init_scale=0.01) + + # Take an action + self.action = self.pd.sample() + + # Calculate the neg log of our probability + self.neglogp = self.pd.neglogp(self.action) + self.sess = sess or tf.get_default_session() + + assert estimate_q is False + self.vf = fc(vf_latent, 'vf', 1) + self.vf = self.vf[:, 0] + + if is_act_model: + self._build_model_for_step() + + def _evaluate(self, variables, observation, **extra_feed): + sess = self.sess + feed_dict = {self.X: adjust_shape(self.X, observation)} + for inpt_name, data in extra_feed.items(): + if inpt_name in self.__dict__.keys(): + inpt = self.__dict__[inpt_name] + if isinstance(inpt, tf.Tensor) and inpt._op.type == 'Placeholder': + feed_dict[inpt] = adjust_shape(inpt, data) + + return sess.run(variables, feed_dict) + + def _build_model_for_step(self): + # multiply with weight and apply mask on self.act_latent to generate + self.act_step = step = tf.placeholder(shape=(), dtype=tf.int64, name='act_step') + with tf.variable_scope('pi', reuse=tf.AUTO_REUSE): + from .util import ortho_init + nin = self.act_latent.get_shape()[1].value + w = tf.get_variable("w", [nin, self.nh], initializer=ortho_init(0.01)) + b = tf.get_variable("b", [self.nh], initializer=tf.constant_initializer(0.0)) + logits = tf.matmul(self.act_latent, w)+b + piece = tf.slice(self.mask, [step, 0], [1, self.nh]) + re_piece = tf.reshape(piece, [-1]) + masked_logits = tf.math.multiply(logits, re_piece) + + npinf_piece = tf.slice(self.mask_npinf, [step, 0], [1, self.nh]) + re_npinf_piece = tf.reshape(npinf_piece, [-1]) + + def sample(logits, mask_npinf): + new_logits = tf.math.add(logits, mask_npinf) + u = tf.random_uniform(tf.shape(new_logits), dtype=logits.dtype) + return tf.argmax(new_logits - tf.log(-1*tf.log(u)), axis=-1) + + def neglogp(logits, x): + # return tf.nn.sparse_softmax_cross_entropy_with_logits(logits=self.logits, labels=x) + # Note: we can't use sparse_softmax_cross_entropy_with_logits because + # the implementation does not allow second-order derivatives... + if x.dtype in {tf.uint8, tf.int32, tf.int64}: + # one-hot encoding + x_shape_list = x.shape.as_list() + logits_shape_list = logits.get_shape().as_list()[:-1] + for xs, ls in zip(x_shape_list, logits_shape_list): + if xs is not None and ls is not None: + assert xs == ls, 'shape mismatch: {} in x vs {} in logits'.format(xs, ls) + + x = tf.one_hot(x, logits.get_shape().as_list()[-1]) + else: + # already encoded + assert x.shape.as_list() == logits.shape.as_list() + + return tf.nn.softmax_cross_entropy_with_logits_v2( + logits=logits, + labels=x) + + self.act_action = sample(masked_logits, re_npinf_piece) + self.act_neglogp = neglogp(masked_logits, self.act_action) + + + def step(self, step, observation, **extra_feed): + """ + Compute next action(s) given the observation(s) + + Parameters + ---------- + observation : np array + Observation data (either single or a batch) + **extra_feed + Additional data such as state or mask (names of the arguments should match the ones in constructor, see __init__) + + Returns + ------- + (action, value estimate, next state, negative log likelihood of the action under current policy parameters) tuple + """ + extra_feed['act_step'] = step + a, v, state, neglogp = self._evaluate([self.act_action, self.vf, self.state, self.act_neglogp], observation, **extra_feed) + if state.size == 0: + state = None + return a, v, state, neglogp + + def value(self, ob, *args, **kwargs): + """ + Compute value estimate(s) given the observation(s) + + Parameters + ---------- + observation : np array + Observation data (either single or a batch) + **extra_feed + Additional data such as state or mask (names of the arguments should match the ones in constructor, see __init__) + + Returns + ------- + Value estimate + """ + return self._evaluate(self.vf, ob, *args, **kwargs) + + +def build_lstm_policy(model_config, value_network=None, estimate_q=False, **policy_kwargs): + """ + Build lstm policy and value network, they share the same lstm network. + the parameters all use their default values. + + Parameter + --------- + model_config : obj + Configurations of the model + value_network : obj + The network for value function + estimate_q : bool + Whether to estimate ``q`` + **policy_kwargs + The kwargs for policy network, i.e., lstm model + + Returns + ------- + func + The policy network + """ + policy_network = lstm_model(**policy_kwargs) + + def policy_fn(nbatch=None, nsteps=None, sess=None, observ_placeholder=None, np_mask=None, is_act_model=False): + ob_space = model_config.observation_space + + X = observ_placeholder if observ_placeholder is not None else observation_placeholder(ob_space, batch_size=nbatch) + + extra_tensors = {} + + # encode_observation is not necessary anymore as we use embedding_lookup + encoded_x = X + + with tf.variable_scope('pi', reuse=tf.AUTO_REUSE): + policy_latent = policy_network(encoded_x, 1, model_config.observation_space.n) + if isinstance(policy_latent, tuple): + policy_latent, recurrent_tensors = policy_latent + + if recurrent_tensors is not None: + # recurrent architecture, need a few more steps + nenv = nbatch // nsteps + assert nenv > 0, 'Bad input for recurrent policy: batch size {} smaller than nsteps {}'.format(nbatch, nsteps) + policy_latent, recurrent_tensors = policy_network(encoded_x, nenv, model_config.observation_space.n) + extra_tensors.update(recurrent_tensors) + + _v_net = value_network + + assert _v_net is None or _v_net == 'shared' + vf_latent = policy_latent + + policy = PolicyWithValue( + env=model_config, + observations=X, + latent=policy_latent, + vf_latent=vf_latent, + sess=sess, + estimate_q=estimate_q, + np_mask=np_mask, + is_act_model=is_act_model, + **extra_tensors + ) + return policy + + return policy_fn diff --git a/new_impl/cv/third_party/nni/algorithms/hpo/ppo_tuner/ppo_tuner.py b/new_impl/cv/third_party/nni/algorithms/hpo/ppo_tuner/ppo_tuner.py new file mode 100644 index 0000000000000000000000000000000000000000..33b62d600e3b6bb788edab7db53a1223a846b2eb --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/hpo/ppo_tuner/ppo_tuner.py @@ -0,0 +1,654 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +ppo_tuner.py including: + class PPOTuner +""" + +import copy +import logging +import numpy as np +from gym import spaces +from schema import Schema, Optional + +import nni +from nni import ClassArgsValidator +from nni.tuner import Tuner +from nni.utils import OptimizeMode, extract_scalar_reward + +from .model import Model +from .util import set_global_seeds +from .policy import build_lstm_policy + + +logger = logging.getLogger('ppo_tuner_AutoML') + +def _constfn(val): + """ + Wrap as function + """ + def f(_): + return val + return f + + +class ModelConfig: + """ + Configurations of the PPO model + """ + def __init__(self): + self.observation_space = None + self.action_space = None + self.num_envs = 0 + self.nsteps = 0 + + self.ent_coef = 0.0 + self.lr = 3e-4 + self.vf_coef = 0.5 + self.max_grad_norm = 0.5 + self.gamma = 0.99 + self.lam = 0.95 + self.cliprange = 0.2 + self.embedding_size = None # the embedding is for each action + + self.noptepochs = 4 # number of training epochs per update + self.total_timesteps = 5000 # number of timesteps (i.e. number of actions taken in the environment) + self.nminibatches = 4 # number of training minibatches per update. For recurrent policies, + # should be smaller or equal than number of environments run in parallel. + +class TrialsInfo: + """ + Informations of each trial from one model inference + """ + def __init__(self, obs, actions, values, neglogpacs, dones, last_value, inf_batch_size): + self.iter = 0 + self.obs = obs + self.actions = actions + self.values = values + self.neglogpacs = neglogpacs + self.dones = dones + self.last_value = last_value + + self.rewards = None + self.returns = None + + self.inf_batch_size = inf_batch_size + #self.states = None + + def get_next(self): + """ + Get actions of the next trial + """ + if self.iter >= self.inf_batch_size: + return None, None + actions = [] + for step in self.actions: + actions.append(step[self.iter]) + self.iter += 1 + return self.iter - 1, actions + + def update_rewards(self, rewards, returns): + """ + After the trial is finished, reward and return of this trial is updated + """ + self.rewards = rewards + self.returns = returns + + def convert_shape(self): + """ + Convert shape + """ + def sf01(arr): + """ + swap and then flatten axes 0 and 1 + """ + s = arr.shape + return arr.swapaxes(0, 1).reshape(s[0] * s[1], *s[2:]) + self.obs = sf01(self.obs) + self.returns = sf01(self.returns) + self.dones = sf01(self.dones) + self.actions = sf01(self.actions) + self.values = sf01(self.values) + self.neglogpacs = sf01(self.neglogpacs) + + +class PPOModel: + """ + PPO Model + """ + def __init__(self, model_config, mask): + self.model_config = model_config + self.states = None # initial state of lstm in policy/value network + self.nupdates = None # the number of func train is invoked, used to tune lr and cliprange + self.cur_update = 1 # record the current update + self.np_mask = mask # record the mask of each action within one trial + + set_global_seeds(None) + assert isinstance(self.model_config.lr, float) + self.lr = _constfn(self.model_config.lr) + assert isinstance(self.model_config.cliprange, float) + self.cliprange = _constfn(self.model_config.cliprange) + + # build lstm policy network, value share the same network + policy = build_lstm_policy(model_config) + + # Get the nb of env + nenvs = model_config.num_envs + + # Calculate the batch_size + self.nbatch = nbatch = nenvs * model_config.nsteps # num of record per update + nbatch_train = nbatch // model_config.nminibatches # get batch size + # self.nupdates is used to tune lr and cliprange + self.nupdates = self.model_config.total_timesteps // self.nbatch + + # Instantiate the model object (that creates act_model and train_model) + self.model = Model(policy=policy, nbatch_act=nenvs, nbatch_train=nbatch_train, + nsteps=model_config.nsteps, ent_coef=model_config.ent_coef, vf_coef=model_config.vf_coef, + max_grad_norm=model_config.max_grad_norm, np_mask=self.np_mask) + + self.states = self.model.initial_state + + logger.info('=== finished PPOModel initialization') + + def inference(self, num): + """ + Generate actions along with related info from policy network. + observation is the action of the last step. + + Parameters + ---------- + num: int + The number of trials to generate + + Returns + ------- + mb_obs : list + Observation of the ``num`` configurations + mb_actions : list + Actions of the ``num`` configurations + mb_values : list + Values from the value function of the ``num`` configurations + mb_neglogpacs : list + ``neglogp`` of the ``num`` configurations + mb_dones : list + To show whether the play is done, always ``True`` + last_values : tensorflow tensor + The last values of the ``num`` configurations, got with session run + """ + # Here, we init the lists that will contain the mb of experiences + mb_obs, mb_actions, mb_values, mb_dones, mb_neglogpacs = [], [], [], [], [] + # initial observation + # use the (n+1)th embedding to represent the first step action + first_step_ob = self.model_config.action_space.n + obs = [first_step_ob for _ in range(num)] + dones = [True for _ in range(num)] + states = self.states + # For n in range number of steps + for cur_step in range(self.model_config.nsteps): + # Given observations, get action value and neglopacs + # We already have self.obs because Runner superclass run self.obs[:] = env.reset() on init + actions, values, states, neglogpacs = self.model.step(cur_step, obs, S=states, M=dones) + mb_obs.append(obs.copy()) + mb_actions.append(actions) + mb_values.append(values) + mb_neglogpacs.append(neglogpacs) + mb_dones.append(dones) + + # Take actions in env and look the results + # Infos contains a ton of useful informations + obs[:] = actions + if cur_step == self.model_config.nsteps - 1: + dones = [True for _ in range(num)] + else: + dones = [False for _ in range(num)] + + #batch of steps to batch of rollouts + np_obs = np.asarray(obs) + mb_obs = np.asarray(mb_obs, dtype=np_obs.dtype) + mb_actions = np.asarray(mb_actions) + mb_values = np.asarray(mb_values, dtype=np.float32) + mb_neglogpacs = np.asarray(mb_neglogpacs, dtype=np.float32) + mb_dones = np.asarray(mb_dones, dtype=np.bool) + last_values = self.model.value(np_obs, S=states, M=dones) + + return mb_obs, mb_actions, mb_values, mb_neglogpacs, mb_dones, last_values + + def compute_rewards(self, trials_info, trials_result): + """ + Compute the rewards of the trials in trials_info based on trials_result, + and update the rewards in trials_info + + Parameters + ---------- + trials_info : TrialsInfo + Info of the generated trials + trials_result : list + Final results (e.g., acc) of the generated trials + """ + mb_rewards = np.asarray([trials_result for _ in trials_info.actions], dtype=np.float32) + # discount/bootstrap off value fn + mb_returns = np.zeros_like(mb_rewards) + mb_advs = np.zeros_like(mb_rewards) + lastgaelam = 0 + last_dones = np.asarray([True for _ in trials_result], dtype=np.bool) # ugly + for t in reversed(range(self.model_config.nsteps)): + if t == self.model_config.nsteps - 1: + nextnonterminal = 1.0 - last_dones + nextvalues = trials_info.last_value + else: + nextnonterminal = 1.0 - trials_info.dones[t+1] + nextvalues = trials_info.values[t+1] + delta = mb_rewards[t] + self.model_config.gamma * nextvalues * nextnonterminal - trials_info.values[t] + lastgaelam = delta + self.model_config.gamma * self.model_config.lam * nextnonterminal * lastgaelam + mb_advs[t] = lastgaelam # pylint: disable=unsupported-assignment-operation + mb_returns = mb_advs + trials_info.values + + trials_info.update_rewards(mb_rewards, mb_returns) + trials_info.convert_shape() + + def train(self, trials_info, nenvs): + """ + Train the policy/value network using trials_info + + Parameters + ---------- + trials_info : TrialsInfo + Complete info of the generated trials from the previous inference + nenvs : int + The batch size of the (previous) inference + """ + # keep frac decay for future optimization + if self.cur_update <= self.nupdates: + frac = 1.0 - (self.cur_update - 1.0) / self.nupdates + else: + logger.warning('current update (self.cur_update) %d has exceeded total updates (self.nupdates) %d', + self.cur_update, self.nupdates) + frac = 1.0 - (self.nupdates - 1.0) / self.nupdates + lrnow = self.lr(frac) + cliprangenow = self.cliprange(frac) + self.cur_update += 1 + + states = self.states + + assert states is not None # recurrent version + assert nenvs % self.model_config.nminibatches == 0 + envsperbatch = nenvs // self.model_config.nminibatches + envinds = np.arange(nenvs) + flatinds = np.arange(nenvs * self.model_config.nsteps).reshape(nenvs, self.model_config.nsteps) + for _ in range(self.model_config.noptepochs): + np.random.shuffle(envinds) + for start in range(0, nenvs, envsperbatch): + end = start + envsperbatch + mbenvinds = envinds[start:end] + mbflatinds = flatinds[mbenvinds].ravel() + slices = (arr[mbflatinds] for arr in (trials_info.obs, trials_info.returns, trials_info.dones, + trials_info.actions, trials_info.values, trials_info.neglogpacs)) + mbstates = states[mbenvinds] + self.model.train(lrnow, cliprangenow, *slices, mbstates) + +class PPOClassArgsValidator(ClassArgsValidator): + def validate_class_args(self, **kwargs): + Schema({ + 'optimize_mode': self.choices('optimize_mode', 'maximize', 'minimize'), + Optional('trials_per_update'): self.range('trials_per_update', int, 0, 99999), + Optional('epochs_per_update'): self.range('epochs_per_update', int, 0, 99999), + Optional('minibatch_size'): self.range('minibatch_size', int, 0, 99999), + Optional('ent_coef'): float, + Optional('lr'): float, + Optional('vf_coef'): float, + Optional('max_grad_norm'): float, + Optional('gamma'): float, + Optional('lam'): float, + Optional('cliprange'): float, + }).validate(kwargs) + +class PPOTuner(Tuner): + """ + PPOTuner, the implementation inherits the main logic of the implementation + [ppo2 from openai](https://github.com/openai/baselines/tree/master/baselines/ppo2), and is adapted for NAS scenario. + It uses ``lstm`` for its policy network and value network, policy and value share the same network. + """ + + def __init__(self, optimize_mode, trials_per_update=20, epochs_per_update=4, minibatch_size=4, + ent_coef=0.0, lr=3e-4, vf_coef=0.5, max_grad_norm=0.5, gamma=0.99, lam=0.95, cliprange=0.2): + """ + Initialization, PPO model is not initialized here as search space is not received yet. + + Parameters + ---------- + optimize_mode : str + maximize or minimize + trials_per_update : int + Number of trials to have for each model update + epochs_per_update : int + Number of epochs to run for each model update + minibatch_size : int + Minibatch size (number of trials) for the update + ent_coef : float + Policy entropy coefficient in the optimization objective + lr : float + Learning rate of the model (lstm network), constant + vf_coef : float + Value function loss coefficient in the optimization objective + max_grad_norm : float + Gradient norm clipping coefficient + gamma : float + Discounting factor + lam : float + Advantage estimation discounting factor (lambda in the paper) + cliprange : float + Cliprange in the PPO algorithm, constant + """ + self.optimize_mode = OptimizeMode(optimize_mode) + self.model_config = ModelConfig() + self.model = None + self.search_space = None + self.running_trials = {} # key: parameter_id, value: actions/states/etc. + self.inf_batch_size = trials_per_update # number of trials to generate in one inference + self.first_inf = True # indicate whether it is the first time to inference new trials + self.trials_result = [None for _ in range(self.inf_batch_size)] # results of finished trials + + self.credit = 0 # record the unsatisfied trial requests + self.param_ids = [] + self.finished_trials = 0 + self.chosen_arch_template = {} + + self.actions_spaces = None + self.actions_to_config = None + self.full_act_space = None + self.trials_info = None + + self.all_trials = {} # used to dedup the same trial, key: config, value: final result + + self.model_config.num_envs = self.inf_batch_size + self.model_config.noptepochs = epochs_per_update + self.model_config.nminibatches = minibatch_size + + self.send_trial_callback = None + logger.info('Finished PPOTuner initialization') + + def _process_nas_space(self, search_space): + actions_spaces = [] + actions_to_config = [] + for key, val in search_space.items(): + if val['_type'] == 'layer_choice': + actions_to_config.append((key, 'layer_choice')) + actions_spaces.append(val['_value']) + self.chosen_arch_template[key] = None + elif val['_type'] == 'input_choice': + candidates = val['_value']['candidates'] + n_chosen = val['_value']['n_chosen'] + if n_chosen not in [0, 1, [0, 1]]: + raise ValueError('Optional_input_size can only be 0, 1, or [0, 1], but the pecified one is %s' + % (n_chosen)) + if isinstance(n_chosen, list): + actions_to_config.append((key, 'input_choice')) + # FIXME: risk, candidates might also have None + actions_spaces.append(['None', *candidates]) + self.chosen_arch_template[key] = None + elif n_chosen == 1: + actions_to_config.append((key, 'input_choice')) + actions_spaces.append(candidates) + self.chosen_arch_template[key] = None + elif n_chosen == 0: + self.chosen_arch_template[key] = [] + else: + raise ValueError('Unsupported search space type: %s' % (val['_type'])) + + # calculate observation space + dedup = {} + for step in actions_spaces: + for action in step: + dedup[action] = 1 + full_act_space = [act for act, _ in dedup.items()] + assert len(full_act_space) == len(dedup) + observation_space = len(full_act_space) + nsteps = len(actions_spaces) + + return actions_spaces, actions_to_config, full_act_space, observation_space, nsteps + + def _generate_action_mask(self): + """ + Different step could have different action space. to deal with this case, we merge all the + possible actions into one action space, and use mask to indicate available actions for each step + """ + two_masks = [] + + mask = [] + for acts in self.actions_spaces: + one_mask = [0 for _ in range(len(self.full_act_space))] + for act in acts: + idx = self.full_act_space.index(act) + one_mask[idx] = 1 + mask.append(one_mask) + two_masks.append(mask) + + mask = [] + for acts in self.actions_spaces: + one_mask = [-np.inf for _ in range(len(self.full_act_space))] + for act in acts: + idx = self.full_act_space.index(act) + one_mask[idx] = 0 + mask.append(one_mask) + two_masks.append(mask) + + return np.asarray(two_masks, dtype=np.float32) + + def update_search_space(self, search_space): + """ + Get search space, currently the space only includes that for NAS + + Parameters + ---------- + search_space : dict + Search space for NAS + the format could be referred to search space spec (https://nni.readthedocs.io/en/latest/Tutorial/SearchSpaceSpec.html). + """ + logger.info('update search space %s', search_space) + assert self.search_space is None + self.search_space = search_space + + assert self.model_config.observation_space is None + assert self.model_config.action_space is None + + self.actions_spaces, self.actions_to_config, self.full_act_space, obs_space, nsteps = self._process_nas_space(search_space) + + self.model_config.observation_space = spaces.Discrete(obs_space) + self.model_config.action_space = spaces.Discrete(obs_space) + self.model_config.nsteps = nsteps + + # generate mask in numpy + mask = self._generate_action_mask() + + assert self.model is None + self.model = PPOModel(self.model_config, mask) + + def _actions_to_config(self, actions): + """ + Given actions, to generate the corresponding trial configuration + """ + chosen_arch = copy.deepcopy(self.chosen_arch_template) + for cnt, act in enumerate(actions): + act_name = self.full_act_space[act] + (_key, _type) = self.actions_to_config[cnt] + if _type == 'input_choice': + if act_name == 'None': + chosen_arch[_key] = {'_value': [], '_idx': []} + else: + candidates = self.search_space[_key]['_value']['candidates'] + idx = candidates.index(act_name) + chosen_arch[_key] = {'_value': [act_name], '_idx': [idx]} + elif _type == 'layer_choice': + idx = self.search_space[_key]['_value'].index(act_name) + chosen_arch[_key] = {'_value': act_name, '_idx': idx} + else: + raise ValueError('unrecognized key: {0}'.format(_type)) + return chosen_arch + + def generate_multiple_parameters(self, parameter_id_list, **kwargs): + """ + Returns multiple sets of trial (hyper-)parameters, as iterable of serializable objects. + + Parameters + ---------- + parameter_id_list : list of int + Unique identifiers for each set of requested hyper-parameters. + These will later be used in :meth:`receive_trial_result`. + **kwargs + Not used + + Returns + ------- + list + A list of newly generated configurations + """ + result = [] + self.send_trial_callback = kwargs['st_callback'] + for parameter_id in parameter_id_list: + had_exception = False + try: + logger.debug("generating param for %s", parameter_id) + res = self.generate_parameters(parameter_id, **kwargs) + except nni.NoMoreTrialError: + had_exception = True + if not had_exception: + result.append(res) + return result + + def generate_parameters(self, parameter_id, **kwargs): + """ + Generate parameters, if no trial configration for now, self.credit plus 1 to send the config later + + Parameters + ---------- + parameter_id : int + Unique identifier for requested hyper-parameters. + This will later be used in :meth:`receive_trial_result`. + **kwargs + Not used + + Returns + ------- + dict + One newly generated configuration + + """ + if self.first_inf: + self.trials_result = [None for _ in range(self.inf_batch_size)] + mb_obs, mb_actions, mb_values, mb_neglogpacs, mb_dones, last_values = self.model.inference(self.inf_batch_size) + self.trials_info = TrialsInfo(mb_obs, mb_actions, mb_values, mb_neglogpacs, + mb_dones, last_values, self.inf_batch_size) + self.first_inf = False + + trial_info_idx, actions = self.trials_info.get_next() + if trial_info_idx is None: + logger.debug('Credit added by one in parameters request') + self.credit += 1 + self.param_ids.append(parameter_id) + raise nni.NoMoreTrialError('no more parameters now.') + + self.running_trials[parameter_id] = trial_info_idx + new_config = self._actions_to_config(actions) + return new_config + + def _next_round_inference(self): + """ + Run a inference to generate next batch of configurations + """ + logger.debug('Start next round inference...') + self.finished_trials = 0 + self.model.compute_rewards(self.trials_info, self.trials_result) + self.model.train(self.trials_info, self.inf_batch_size) + self.running_trials = {} + # generate new trials + self.trials_result = [None for _ in range(self.inf_batch_size)] + mb_obs, mb_actions, mb_values, mb_neglogpacs, mb_dones, last_values = self.model.inference(self.inf_batch_size) + self.trials_info = TrialsInfo(mb_obs, mb_actions, + mb_values, mb_neglogpacs, + mb_dones, last_values, + self.inf_batch_size) + logger.debug('Next round inference complete.') + # check credit and submit new trials + for _ in range(self.credit): + trial_info_idx, actions = self.trials_info.get_next() + if trial_info_idx is None: + logger.warning('No enough trial config, trials_per_update is suggested to be larger than trialConcurrency') + break + assert self.param_ids + param_id = self.param_ids.pop() + self.running_trials[param_id] = trial_info_idx + new_config = self._actions_to_config(actions) + self.send_trial_callback(param_id, new_config) + self.credit -= 1 + logger.debug('Send new trial (%d, %s) for reducing credit', param_id, new_config) + + def receive_trial_result(self, parameter_id, parameters, value, **kwargs): + """ + Receive trial's result. if the number of finished trials equals self.inf_batch_size, start the next update to + train the model. + + Parameters + ---------- + parameter_id : int + Unique identifier of used hyper-parameters, same with :meth:`generate_parameters`. + parameters : dict + Hyper-parameters generated by :meth:`generate_parameters`. + value : dict + Result from trial (the return value of :func:`nni.report_final_result`). + """ + trial_info_idx = self.running_trials.pop(parameter_id, None) + assert trial_info_idx is not None + + value = extract_scalar_reward(value) + if self.optimize_mode == OptimizeMode.Minimize: + value = -value + + self.trials_result[trial_info_idx] = value + self.finished_trials += 1 + + logger.debug('receive_trial_result, parameter_id %d, trial_info_idx %d, finished_trials %d, inf_batch_size %d', + parameter_id, trial_info_idx, self.finished_trials, self.inf_batch_size) + if self.finished_trials == self.inf_batch_size: + logger.debug('Start next round inference in receive_trial_result') + self._next_round_inference() + + def trial_end(self, parameter_id, success, **kwargs): + """ + To deal with trial failure. If a trial fails, it is popped out from ``self.running_trials``, + and the final result of this trial is assigned with the average of the finished trials. + + Parameters + ---------- + parameter_id : int + Unique identifier for hyper-parameters used by this trial. + success : bool + True if the trial successfully completed; False if failed or terminated. + **kwargs + Not used + """ + if not success: + if parameter_id not in self.running_trials: + logger.warning('The trial is failed, but self.running_trial does not have this trial') + return + trial_info_idx = self.running_trials.pop(parameter_id, None) + assert trial_info_idx is not None + # use mean of finished trials as the result of this failed trial + values = [val for val in self.trials_result if val is not None] + logger.warning('In trial_end, values: %s', values) + self.trials_result[trial_info_idx] = (sum(values) / len(values)) if values else 0 + self.finished_trials += 1 + if self.finished_trials == self.inf_batch_size: + logger.debug('Start next round inference in trial_end') + self._next_round_inference() + + def import_data(self, data): + """ + Import additional data for tuning, not supported yet. + + Parameters + ---------- + data : list + A list of dictionarys, each of which has at least two keys, ``parameter`` and ``value`` + """ + logger.warning('PPOTuner cannot leverage imported data.') diff --git a/new_impl/cv/third_party/nni/algorithms/hpo/ppo_tuner/requirements.txt b/new_impl/cv/third_party/nni/algorithms/hpo/ppo_tuner/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..a58dfb12f2fb8505cb47c2c88434bdded4553d47 --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/hpo/ppo_tuner/requirements.txt @@ -0,0 +1,2 @@ +enum34 +gym diff --git a/new_impl/cv/third_party/nni/algorithms/hpo/ppo_tuner/util.py b/new_impl/cv/third_party/nni/algorithms/hpo/ppo_tuner/util.py new file mode 100644 index 0000000000000000000000000000000000000000..605292de4002f114e935cb24a87c96921940e959 --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/hpo/ppo_tuner/util.py @@ -0,0 +1,260 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +util functions +""" + +import os +import random +import multiprocessing +import numpy as np +import tensorflow.compat.v1 as tf +tf.disable_v2_behavior() +from gym.spaces import Discrete, Box, MultiDiscrete + +def set_global_seeds(i): + """set global seeds""" + rank = 0 + myseed = i + 1000 * rank if i is not None else None + tf.set_random_seed(myseed) + np.random.seed(myseed) + random.seed(myseed) + +def batch_to_seq(h, nbatch, nsteps, flat=False): + """convert from batch to sequence""" + if flat: + h = tf.reshape(h, [nbatch, nsteps]) + else: + h = tf.reshape(h, [nbatch, nsteps, -1]) + return [tf.squeeze(v, [1]) for v in tf.split(axis=1, num_or_size_splits=nsteps, value=h)] + +def seq_to_batch(h, flat=False): + """convert from sequence to batch""" + shape = h[0].get_shape().as_list() + if not flat: + assert len(shape) > 1 + nh = h[0].get_shape()[-1].value + return tf.reshape(tf.concat(axis=1, values=h), [-1, nh]) + else: + return tf.reshape(tf.stack(values=h, axis=1), [-1]) + +def lstm(xs, ms, s, scope, nh, init_scale=1.0): + """lstm cell""" + _, nin = [v.value for v in xs[0].get_shape()] # the first is nbatch + with tf.variable_scope(scope): + wx = tf.get_variable("wx", [nin, nh*4], initializer=ortho_init(init_scale)) + wh = tf.get_variable("wh", [nh, nh*4], initializer=ortho_init(init_scale)) + b = tf.get_variable("b", [nh*4], initializer=tf.constant_initializer(0.0)) + + c, h = tf.split(axis=1, num_or_size_splits=2, value=s) + for idx, (x, m) in enumerate(zip(xs, ms)): + c = c*(1-m) + h = h*(1-m) + z = tf.matmul(x, wx) + tf.matmul(h, wh) + b + i, f, o, u = tf.split(axis=1, num_or_size_splits=4, value=z) + i = tf.nn.sigmoid(i) + f = tf.nn.sigmoid(f) + o = tf.nn.sigmoid(o) + u = tf.tanh(u) + c = f*c + i*u + h = o*tf.tanh(c) + xs[idx] = h + s = tf.concat(axis=1, values=[c, h]) + return xs, s + +def lstm_model(nlstm=128, layer_norm=False): + """ + Builds LSTM (Long-Short Term Memory) network to be used in a policy. + Note that the resulting function returns not only the output of the LSTM + (i.e. hidden state of lstm for each step in the sequence), but also a dictionary + with auxiliary tensors to be set as policy attributes. + + Specifically, + S is a placeholder to feed current state (LSTM state has to be managed outside policy) + M is a placeholder for the mask (used to mask out observations after the end of the episode, but can be used for other purposes too) + initial_state is a numpy array containing initial lstm state (usually zeros) + state is the output LSTM state (to be fed into S at the next call) + + + An example of usage of lstm-based policy can be found here: common/tests/test_doc_examples.py/test_lstm_example + + Parameters + ---------- + nlstm : int + LSTM hidden state size + layer_norm : bool + if True, layer-normalized version of LSTM is used + + Returns + ------- + function that builds LSTM with a given input tensor / placeholder + """ + + def network_fn(X, nenv=1, obs_size=-1): + with tf.variable_scope("emb", reuse=tf.AUTO_REUSE): + w_emb = tf.get_variable("w_emb", [obs_size+1, 32]) + X = tf.nn.embedding_lookup(w_emb, X) + + nbatch = X.shape[0] + nsteps = nbatch // nenv + + h = tf.layers.flatten(X) + + M = tf.placeholder(tf.float32, [nbatch]) #mask (done t-1) + S = tf.placeholder(tf.float32, [nenv, 2*nlstm]) #states + + xs = batch_to_seq(h, nenv, nsteps) + ms = batch_to_seq(M, nenv, nsteps) + + assert not layer_norm + h5, snew = lstm(xs, ms, S, scope='lstm', nh=nlstm) + + h = seq_to_batch(h5) + initial_state = np.zeros(S.shape.as_list(), dtype=float) + + return h, {'S':S, 'M':M, 'state':snew, 'initial_state':initial_state} + + return network_fn + +def ortho_init(scale=1.0): + """init approach""" + def _ortho_init(shape, dtype, partition_info=None): + #lasagne ortho init for tf + shape = tuple(shape) + if len(shape) == 2: + flat_shape = shape + elif len(shape) == 4: # assumes NHWC + flat_shape = (np.prod(shape[:-1]), shape[-1]) + else: + raise NotImplementedError + a = np.random.normal(0.0, 1.0, flat_shape) + u, _, v = np.linalg.svd(a, full_matrices=False) + q = u if u.shape == flat_shape else v # pick the one with the correct shape + q = q.reshape(shape) + return (scale * q[:shape[0], :shape[1]]).astype(np.float32) + return _ortho_init + +def fc(x, scope, nh, *, init_scale=1.0, init_bias=0.0): + """fully connected op""" + with tf.variable_scope(scope): + nin = x.get_shape()[1].value + w = tf.get_variable("w", [nin, nh], initializer=ortho_init(init_scale)) + b = tf.get_variable("b", [nh], initializer=tf.constant_initializer(init_bias)) + return tf.matmul(x, w)+b + +def _check_shape(placeholder_shape, data_shape): + """ + check if two shapes are compatible (i.e. differ only by dimensions of size 1, or by the batch dimension) + """ + + return True + +# ================================================================ +# Shape adjustment for feeding into tf placeholders +# ================================================================ +def adjust_shape(placeholder, data): + """ + adjust shape of the data to the shape of the placeholder if possible. + If shape is incompatible, AssertionError is thrown + + Parameters + ---------- + placeholder + tensorflow input placeholder + data + input data to be (potentially) reshaped to be fed into placeholder + + Returns + ------- + reshaped data + """ + if not isinstance(data, np.ndarray) and not isinstance(data, list): + return data + if isinstance(data, list): + data = np.array(data) + + placeholder_shape = [x or -1 for x in placeholder.shape.as_list()] + + assert _check_shape(placeholder_shape, data.shape), \ + 'Shape of data {} is not compatible with shape of the placeholder {}'.format(data.shape, placeholder_shape) + + return np.reshape(data, placeholder_shape) + +# ================================================================ +# Global session +# ================================================================ + +def get_session(config=None): + """Get default session or create one with a given config""" + sess = tf.get_default_session() + if sess is None: + sess = make_session(config=config, make_default=True) + return sess + +def make_session(config=None, num_cpu=None, make_default=False, graph=None): + """Returns a session that will use CPU's only""" + if num_cpu is None: + num_cpu = int(os.getenv('RCALL_NUM_CPU', multiprocessing.cpu_count())) + if config is None: + config = tf.ConfigProto( + allow_soft_placement=True, + inter_op_parallelism_threads=num_cpu, + intra_op_parallelism_threads=num_cpu) + config.gpu_options.allow_growth = True + + if make_default: + return tf.InteractiveSession(config=config, graph=graph) + else: + return tf.Session(config=config, graph=graph) + +ALREADY_INITIALIZED = set() + +def initialize(): + """Initialize all the uninitialized variables in the global scope.""" + new_variables = set(tf.global_variables()) - ALREADY_INITIALIZED + get_session().run(tf.variables_initializer(new_variables)) + + ALREADY_INITIALIZED.update(new_variables) + +def observation_placeholder(ob_space, batch_size=None, name='Ob'): + """ + Create placeholder to feed observations into of the size appropriate to the observation space + + Parameters + ---------- + ob_space : gym.Space + observation space + batch_size : int + size of the batch to be fed into input. Can be left None in most cases. + name : str + name of the placeholder + + Returns + ------- + tensorflow placeholder tensor + """ + + assert isinstance(ob_space, (Discrete, Box, MultiDiscrete)), \ + 'Can only deal with Discrete and Box observation spaces for now' + + dtype = ob_space.dtype + if dtype == np.int8: + dtype = np.uint8 + + return tf.placeholder(shape=(batch_size,) + ob_space.shape, dtype=dtype, name=name) + +def explained_variance(ypred, y): + """ + Computes fraction of variance that ypred explains about y. + Returns 1 - Var[y-ypred] / Var[y] + + interpretation: + ev=0 => might as well have predicted zero + ev=1 => perfect prediction + ev<0 => worse than just predicting zero + + """ + assert y.ndim == 1 and ypred.ndim == 1 + vary = np.var(y) + return np.nan if vary == 0 else 1 - np.var(y-ypred)/vary diff --git a/new_impl/cv/third_party/nni/algorithms/hpo/regularized_evolution_tuner/__init__.py b/new_impl/cv/third_party/nni/algorithms/hpo/regularized_evolution_tuner/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..befa4e446b9af62ff87abae8bf35134d5eecb68a --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/hpo/regularized_evolution_tuner/__init__.py @@ -0,0 +1 @@ +from .regularized_evolution_tuner import RegularizedEvolutionTuner diff --git a/new_impl/cv/third_party/nni/algorithms/hpo/regularized_evolution_tuner/regularized_evolution_tuner.py b/new_impl/cv/third_party/nni/algorithms/hpo/regularized_evolution_tuner/regularized_evolution_tuner.py new file mode 100644 index 0000000000000000000000000000000000000000..ac756c33885b8ad150e862c6eb315136fb0d4b71 --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/hpo/regularized_evolution_tuner/regularized_evolution_tuner.py @@ -0,0 +1,172 @@ +import copy +import logging +import random +from collections import deque + +from schema import Schema, Optional +import nni +from nni.tuner import Tuner +from nni import ClassArgsValidator +from nni.utils import OptimizeMode, extract_scalar_reward + +logger = logging.getLogger(__name__) + + +class FinishedIndividual: + def __init__(self, parameter_id, parameters, result): + """ + Parameters + ---------- + parameter_id: int + the index of the parameter + parameters : dict + chosen architecture and parameters + result : float + final metric of the chosen one + """ + self.parameter_id = parameter_id + self.parameters = parameters + self.result = result + + +class EvolutionClassArgsValidator(ClassArgsValidator): + def validate_class_args(self, **kwargs): + Schema({ + 'optimize_mode': self.choices('optimize_mode', 'maximize', 'minimize'), + Optional('population_size'): self.range('population_size', int, 0, 99999), + Optional('sample_size'): self.range('sample_size', int, 0, 9999), + }).validate(kwargs) + + +class RegularizedEvolutionTuner(Tuner): + """ + RegularizedEvolutionTuner is tuner using Evolution NAS Tuner. + See ``Regularized Evolution for Image Classifier Architecture Search`` for details. + + Parameters + --- + optimize_mode: str + whether to maximize metric or not. default: 'maximize' + population_size: int + the maximum number of kept models + sample_size: int + the number of models chosen from population each time when evolution + """ + def __init__(self, optimize_mode="maximize", population_size=100, sample_size=25): + super(RegularizedEvolutionTuner, self).__init__() + self.optimize_mode = OptimizeMode(optimize_mode) + self.population_size = population_size + self.sample_size = sample_size + self.initial_population = deque() + self.population = deque() + self.history = {} + self.search_space = None + self._from_initial = {} # whether the parameter is from initial population + + def generate_parameters(self, parameter_id, **kwargs): + """ + This function will returns a dict of trial (hyper-)parameters, as a serializable object. + + Parameters + --- + parameter_id: int + the index of current set of parameters + """ + if self.initial_population: + arch = self.initial_population.popleft() + self.history[parameter_id] = arch + self._from_initial[parameter_id] = True + return arch + elif self.population: + sample = [] + while len(sample) < self.sample_size: + sample.append(random.choice(list(self.population))) + + candidate = max(sample, key=lambda x: x.result) + arch = self._mutate_model(candidate) + self.history[parameter_id] = arch + self._from_initial[parameter_id] = False + return arch + else: + raise nni.NoMoreTrialError + + def receive_trial_result(self, parameter_id, parameters, value, **kwargs): + """ + Record the result from a trial + + Parameters + ---------- + parameter_id : int + parameters : dict + value : dict/float + if value is dict, it should have "default" key. + value is final metrics of the trial. + """ + reward = extract_scalar_reward(value) + if parameter_id not in self.history: + raise RuntimeError('Received parameter_id not in total_data.') + params = self.history[parameter_id] + + if self.optimize_mode == OptimizeMode.Minimize: + reward = -reward + + self.population.append(FinishedIndividual(parameter_id, params, reward)) + if len(self.population) > self.population_size: + self.population.popleft() + + def update_search_space(self, search_space): + """ + Update search space. + Search_space contains the information that user pre-defined. + + Parameters + ---------- + search_space : dict + """ + logger.info('update search space %s', search_space) + assert self.search_space is None + self.search_space = search_space + + for _, val in search_space.items(): + if val['_type'] != 'layer_choice' and val['_type'] != 'input_choice': + raise ValueError('Unsupported search space type: %s' % (val['_type'])) + + self._generate_initial_population() + + def trial_end(self, parameter_id, success, **kwargs): + if not success: + del self.history[parameter_id] + if self._from_initial[parameter_id]: + self.initial_population.append(self._random_model()) + del self._from_initial[parameter_id] + + def _mutate(self, key, individual): + mutate_val = self.search_space[key] + if mutate_val['_type'] == 'layer_choice': + idx = random.randint(0, len(mutate_val['_value']) - 1) + individual[key] = {'_value': mutate_val['_value'][idx], '_idx': idx} + elif mutate_val['_type'] == 'input_choice': + candidates = mutate_val['_value']['candidates'] + n_chosen = mutate_val['_value']['n_chosen'] + idxs = [random.randint(0, len(candidates) - 1) for _ in range(n_chosen)] + vals = [candidates[k] for k in idxs] + individual[key] = {'_value': vals, '_idx': idxs} + else: + raise KeyError + + def _random_model(self): + individual = {} + for key in self.search_space.keys(): + self._mutate(key, individual) + return individual + + def _mutate_model(self, model): + new_individual = copy.deepcopy(model.parameters) + mutate_key = random.choice(list(new_individual.keys())) + self._mutate(mutate_key, new_individual) + return new_individual + + def _generate_initial_population(self): + while len(self.initial_population) < self.population_size: + self.initial_population.append(self._random_model()) + logger.info('init population done.') diff --git a/new_impl/cv/third_party/nni/algorithms/hpo/smac_tuner/__init__.py b/new_impl/cv/third_party/nni/algorithms/hpo/smac_tuner/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..0b59d5ed1627bc2f27daedb880de62c3952af60c --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/hpo/smac_tuner/__init__.py @@ -0,0 +1,4 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .smac_tuner import SMACTuner diff --git a/new_impl/cv/third_party/nni/algorithms/hpo/smac_tuner/convert_ss_to_scenario.py b/new_impl/cv/third_party/nni/algorithms/hpo/smac_tuner/convert_ss_to_scenario.py new file mode 100644 index 0000000000000000000000000000000000000000..ef817c8cf4e10ad08185910b95bd4adc909b2dd3 --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/hpo/smac_tuner/convert_ss_to_scenario.py @@ -0,0 +1,202 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import json + +import numpy as np + + +def get_json_content(file_path): + """ + Load json file content + + Parameters + ---------- + file_path: + path to the file + + Raises + ------ + TypeError + Error with the file path + """ + try: + with open(file_path, 'r') as file: + return json.load(file) + except TypeError as err: + print('Error: ', err) + return None + + +def generate_pcs(nni_search_space_content): + """ + Generate the Parameter Configuration Space (PCS) which defines the + legal ranges of the parameters to be optimized and their default values. + Generally, the format is: + # parameter_name categorical {value_1, ..., value_N} [default value] + # parameter_name ordinal {value_1, ..., value_N} [default value] + # parameter_name integer [min_value, max_value] [default value] + # parameter_name integer [min_value, max_value] [default value] log + # parameter_name real [min_value, max_value] [default value] + # parameter_name real [min_value, max_value] [default value] log + Reference: https://automl.github.io/SMAC3/stable/options.html + + Parameters + ---------- + nni_search_space_content: search_space + The search space in this experiment in nni + + Returns + ------- + Parameter Configuration Space (PCS) + the legal ranges of the parameters to be optimized and their default values + + Raises + ------ + RuntimeError + unsupported type or value error or incorrect search space + """ + categorical_dict = {} + search_space = nni_search_space_content + + def dump_categorical(fd, key, categories): + choice_len = len(categories) + if key in categorical_dict: + raise RuntimeError( + '%s has already existed, please make sure search space has no duplicate key.' % key) + categorical_dict[key] = search_space[key]['_value'] + fd.write('%s categorical {%s} [0]\n' % (key, ','.join(map(str, range(choice_len))))) + + with open('param_config_space.pcs', 'w') as pcs_fd: + if isinstance(search_space, dict): + for key in search_space.keys(): + if isinstance(search_space[key], dict): + try: + if search_space[key]['_type'] == 'choice': + dump_categorical(pcs_fd, key, search_space[key]['_value']) + elif search_space[key]['_type'] == 'randint': + lower, upper = search_space[key]['_value'] + if lower + 1 == upper: + dump_categorical(pcs_fd, key, [lower]) + else: + pcs_fd.write('%s integer [%d, %d] [%d]\n' % (key, lower, upper - 1, lower)) + elif search_space[key]['_type'] == 'uniform': + low, high = search_space[key]['_value'] + if low == high: + dump_categorical(pcs_fd, key, [low]) + else: + pcs_fd.write('%s real [%s, %s] [%s]\n' % (key, low, high, low)) + elif search_space[key]['_type'] == 'loguniform': + # use np.round here to ensure that the rounded default value is in the range, + # which will be rounded in configure_space package + low, high = list(np.round(np.log(search_space[key]['_value']), 10)) + if low == high: + dump_categorical(pcs_fd, key, [search_space[key]['_value'][0]]) + else: + pcs_fd.write('%s real [%s, %s] [%s]\n' % (key, low, high, low)) + elif search_space[key]['_type'] == 'quniform': + low, high, q = search_space[key]['_value'][0:3] + vals = np.clip(np.arange(np.round(low / q), np.round(high / q) + 1) * q, low, high).tolist() + pcs_fd.write('%s ordinal {%s} [%s]\n' % ( + key, + json.dumps(vals)[1:-1], + json.dumps(vals[0]))) + else: + raise RuntimeError('unsupported _type %s' % search_space[key]['_type']) + except: + raise RuntimeError('_type or _value error.') + else: + raise RuntimeError('incorrect search space.') + return categorical_dict + return None + + +def generate_scenario(ss_content): + """ + Generate the scenario. The scenario-object (smac.scenario.scenario.Scenario) is used to configure SMAC and + can be constructed either by providing an actual scenario-object, or by specifing the options in a scenario file. + Reference: https://automl.github.io/SMAC3/stable/options.html + The format of the scenario file is one option per line: + OPTION1 = VALUE1 + OPTION2 = VALUE2 + ... + Parameters + ---------- + abort_on_first_run_crash: bool + If true, SMAC will abort if the first run of the target algorithm crashes. Default: True, + because trials reported to nni tuner would always in success state + algo: function + Specifies the target algorithm call that SMAC will optimize. Interpreted as a bash-command. + Not required by tuner, but required by nni's training service for running trials + always_race_default: + Race new incumbents always against default configuration + cost_for_crash: + Defines the cost-value for crashed runs on scenarios with quality as run-obj. Default: 2147483647.0. + Trials reported to nni tuner would always in success state + cutoff_time: + Maximum runtime, after which the target algorithm is cancelled. `Required if *run_obj* is runtime` + deterministic: bool + If true, the optimization process will be repeatable. + execdir: + Specifies the path to the execution-directory. Default: . + Trials are executed by nni's training service + feature_file: + Specifies the file with the instance-features. + No features specified or feature file is not supported + initial_incumbent: + DEFAULT is the default from the PCS. Default: DEFAULT. Must be from: [‘DEFAULT’, ‘RANDOM’]. + input_psmac_dirs: + For parallel SMAC, multiple output-directories are used. + Parallelism is supported by nni + instance_file: + Specifies the file with the training-instances. Not supported + intensification_percentage: + The fraction of time to be used on intensification (versus choice of next Configurations). Default: 0.5. + Not supported, trials are controlled by nni's training service and kill be assessor + maxR: int + Maximum number of calls per configuration. Default: 2000. + memory_limit: + Maximum available memory the target algorithm can occupy before being cancelled. + minR: int + Minimum number of calls per configuration. Default: 1. + output_dir: + Specifies the output-directory for all emerging files, such as logging and results. + Default: smac3-output_2018-01-22_15:05:56_807070. + overall_obj: + PARX, where X is an integer defining the penalty imposed on timeouts (i.e. runtimes that exceed the cutoff-time). + Timeout is not supported + paramfile: + Specifies the path to the PCS-file. + run_obj: + Defines what metric to optimize. When optimizing runtime, cutoff_time is required as well. + Must be from: [‘runtime’, ‘quality’]. + runcount_limit: int + Maximum number of algorithm-calls during optimization. Default: inf. + Use default because this is controlled by nni + shared_model: + Whether to run SMAC in parallel mode. Parallelism is supported by nni + test_instance_file: + Specifies the file with the test-instances. Instance is not supported + tuner-timeout: + Maximum amount of CPU-time used for optimization. Not supported + wallclock_limit: int + Maximum amount of wallclock-time used for optimization. Default: inf. + Use default because this is controlled by nni + + Returns + ------- + Scenario: + The scenario-object (smac.scenario.scenario.Scenario) is used to configure SMAC and can be constructed + either by providing an actual scenario-object, or by specifing the options in a scenario file + """ + with open('scenario.txt', 'w') as sce_fd: + sce_fd.write('deterministic = 0\n') + # sce_fd.write('output_dir = \n') + sce_fd.write('paramfile = param_config_space.pcs\n') + sce_fd.write('run_obj = quality\n') + + return generate_pcs(ss_content) + + +if __name__ == '__main__': + generate_scenario('search_space.json') diff --git a/new_impl/cv/third_party/nni/algorithms/hpo/smac_tuner/requirements.txt b/new_impl/cv/third_party/nni/algorithms/hpo/smac_tuner/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..a3027fb6fe5ca4b4725ee5a3db8ec4e53a996166 --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/hpo/smac_tuner/requirements.txt @@ -0,0 +1,2 @@ +git+https://github.com/QuanluZhang/ConfigSpace.git +git+https://github.com/QuanluZhang/SMAC3.git diff --git a/new_impl/cv/third_party/nni/algorithms/hpo/smac_tuner/smac_tuner.py b/new_impl/cv/third_party/nni/algorithms/hpo/smac_tuner/smac_tuner.py new file mode 100644 index 0000000000000000000000000000000000000000..fdaed49d8f2480516ec1e1e62fc43eabdd574049 --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/hpo/smac_tuner/smac_tuner.py @@ -0,0 +1,346 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +smac_tuner.py +""" + +import logging +import sys + +import numpy as np +from schema import Schema, Optional + +from smac.facade.epils_facade import EPILS +from smac.facade.roar_facade import ROAR +from smac.facade.smac_facade import SMAC +from smac.scenario.scenario import Scenario +from smac.utils.io.cmd_reader import CMDReader + +from ConfigSpaceNNI import Configuration + +import nni +from nni import ClassArgsValidator +from nni.tuner import Tuner +from nni.utils import OptimizeMode, extract_scalar_reward + +from .convert_ss_to_scenario import generate_scenario + +logger = logging.getLogger('smac_AutoML') + +class SMACClassArgsValidator(ClassArgsValidator): + def validate_class_args(self, **kwargs): + Schema({ + 'optimize_mode': self.choices('optimize_mode', 'maximize', 'minimize'), + Optional('config_dedup'): bool + }).validate(kwargs) + +class SMACTuner(Tuner): + """ + This is a wrapper of [SMAC](https://github.com/automl/SMAC3) following NNI tuner interface. + It only supports ``SMAC`` mode, and does not support the multiple instances of SMAC3 (i.e., + the same configuration is run multiple times). + """ + def __init__(self, optimize_mode="maximize", config_dedup=False): + """ + Parameters + ---------- + optimize_mode : str + Optimize mode, 'maximize' or 'minimize', by default 'maximize' + config_dedup : bool + If True, the tuner will not generate a configuration that has been already generated. + If False, a configuration may be generated twice, but it is rare for relatively large search space. + """ + self.logger = logger + self.optimize_mode = OptimizeMode(optimize_mode) + self.total_data = {} + self.optimizer = None + self.smbo_solver = None + self.first_one = True + self.update_ss_done = False + self.loguniform_key = set() + self.categorical_dict = {} + self.cs = None + self.dedup = config_dedup + + def _main_cli(self): + """ + Main function of SMAC for CLI interface. Some initializations of the wrapped SMAC are done + in this function. + + Returns + ------- + obj + The object of the SMAC optimizer + """ + self.logger.info("SMAC call: %s", " ".join(sys.argv)) + + cmd_reader = CMDReader() + args, _ = cmd_reader.read_cmd() + + root_logger = logging.getLogger() + root_logger.setLevel(args.verbose_level) + logger_handler = logging.StreamHandler(stream=sys.stdout) + if root_logger.level >= logging.INFO: + formatter = logging.Formatter("%(levelname)s:\t%(message)s") + else: + formatter = logging.Formatter( + "%(asctime)s:%(levelname)s:%(name)s:%(message)s", + "%Y-%m-%d %H:%M:%S") + logger_handler.setFormatter(formatter) + root_logger.addHandler(logger_handler) + # remove default handler + root_logger.removeHandler(root_logger.handlers[0]) + + # Create defaults + rh = None + initial_configs = None + stats = None + incumbent = None + + # Create scenario-object + scen = Scenario(args.scenario_file, []) + self.cs = scen.cs + + if args.mode == "SMAC": + optimizer = SMAC( + scenario=scen, + rng=np.random.RandomState(args.seed), + runhistory=rh, + initial_configurations=initial_configs, + stats=stats, + restore_incumbent=incumbent, + run_id=args.seed) + elif args.mode == "ROAR": + optimizer = ROAR( + scenario=scen, + rng=np.random.RandomState(args.seed), + runhistory=rh, + initial_configurations=initial_configs, + run_id=args.seed) + elif args.mode == "EPILS": + optimizer = EPILS( + scenario=scen, + rng=np.random.RandomState(args.seed), + runhistory=rh, + initial_configurations=initial_configs, + run_id=args.seed) + else: + optimizer = None + + return optimizer + + def update_search_space(self, search_space): + """ + Convert search_space to the format that ``SMAC3`` could recognize, thus, not all the search space types + are supported. In this function, we also do the initialization of `SMAC3`, i.e., calling ``self._main_cli``. + + NOTE: updating search space during experiment running is not supported. + + Parameters + ---------- + search_space : dict + The format could be referred to search space spec (https://nni.readthedocs.io/en/latest/Tutorial/SearchSpaceSpec.html). + """ + self.logger.info('update search space in SMAC.') + if not self.update_ss_done: + self.categorical_dict = generate_scenario(search_space) + if self.categorical_dict is None: + raise RuntimeError('categorical dict is not correctly returned after parsing search space.') + # TODO: this is ugly, we put all the initialization work in this method, because initialization relies + # on search space, also because update_search_space is called at the beginning. + self.optimizer = self._main_cli() + self.smbo_solver = self.optimizer.solver + self.loguniform_key = {key for key in search_space.keys() if search_space[key]['_type'] == 'loguniform'} + self.update_ss_done = True + else: + self.logger.warning('update search space is not supported.') + + def receive_trial_result(self, parameter_id, parameters, value, **kwargs): + """ + Receive a trial's final performance result reported through :func:``nni.report_final_result`` by the trial. + GridSearchTuner does not need trial's results. + + Parameters + ---------- + parameter_id : int + Unique identifier of used hyper-parameters, same with :meth:`generate_parameters`. + parameters : dict + Hyper-parameters generated by :meth:`generate_parameters`. + value : dict + Result from trial (the return value of :func:`nni.report_final_result`). + + Raises + ------ + RuntimeError + Received parameter id not in ``self.total_data`` + """ + reward = extract_scalar_reward(value) + if self.optimize_mode is OptimizeMode.Maximize: + reward = -reward + + if parameter_id not in self.total_data: + raise RuntimeError('Received parameter_id not in total_data.') + if self.first_one: + self.smbo_solver.nni_smac_receive_first_run(self.total_data[parameter_id], reward) + self.first_one = False + else: + self.smbo_solver.nni_smac_receive_runs(self.total_data[parameter_id], reward) + + def param_postprocess(self, challenger_dict): + """ + Postprocessing for a set of hyperparameters includes: + 1. Convert the values of type ``loguniform`` back to their initial range. + 2. Convert ``categorical``: categorical values in search space are changed to list of numbers before, + those original values will be changed back in this function. + + Parameters + ---------- + challenger_dict : dict + challenger dict + + Returns + ------- + dict + dict which stores copy of challengers + """ + converted_dict = {} + for key, value in challenger_dict.items(): + # convert to loguniform + if key in self.loguniform_key: + converted_dict[key] = np.exp(challenger_dict[key]) + # convert categorical back to original value + elif key in self.categorical_dict: + idx = challenger_dict[key] + converted_dict[key] = self.categorical_dict[key][idx] + else: + converted_dict[key] = value + return converted_dict + + def generate_parameters(self, parameter_id, **kwargs): + """ + Generate one instance of hyperparameters (i.e., one configuration). + Get one from SMAC3's ``challengers``. + + Parameters + ---------- + parameter_id : int + Unique identifier for requested hyper-parameters. This will later be used in :meth:`receive_trial_result`. + **kwargs + Not used + + Returns + ------- + dict + One newly generated configuration + """ + if self.first_one: + init_challenger = self.smbo_solver.nni_smac_start() + self.total_data[parameter_id] = init_challenger + return self.param_postprocess(init_challenger.get_dictionary()) + else: + challengers = self.smbo_solver.nni_smac_request_challengers() + challengers_empty = True + for challenger in challengers: + challengers_empty = False + if self.dedup: + match = [v for k, v in self.total_data.items() \ + if v.get_dictionary() == challenger.get_dictionary()] + if match: + continue + self.total_data[parameter_id] = challenger + return self.param_postprocess(challenger.get_dictionary()) + assert challengers_empty is False, 'The case that challengers is empty is not handled.' + self.logger.info('In generate_parameters: No more new parameters.') + raise nni.NoMoreTrialError('No more new parameters.') + + def generate_multiple_parameters(self, parameter_id_list, **kwargs): + """ + Generate mutiple instances of hyperparameters. If it is a first request, + retrieve the instances from initial challengers. While if it is not, request + new challengers and retrieve instances from the requested challengers. + + Parameters + ---------- + parameter_id_list: list of int + Unique identifiers for each set of requested hyper-parameters. + These will later be used in :meth:`receive_trial_result`. + **kwargs + Not used + + Returns + ------- + list + a list of newly generated configurations + """ + if self.first_one: + params = [] + for one_id in parameter_id_list: + init_challenger = self.smbo_solver.nni_smac_start() + self.total_data[one_id] = init_challenger + params.append(self.param_postprocess(init_challenger.get_dictionary())) + else: + challengers = self.smbo_solver.nni_smac_request_challengers() + cnt = 0 + params = [] + for challenger in challengers: + if cnt >= len(parameter_id_list): + break + if self.dedup: + match = [v for k, v in self.total_data.items() \ + if v.get_dictionary() == challenger.get_dictionary()] + if match: + continue + self.total_data[parameter_id_list[cnt]] = challenger + params.append(self.param_postprocess(challenger.get_dictionary())) + cnt += 1 + if self.dedup and not params: + self.logger.info('In generate_multiple_parameters: No more new parameters.') + return params + + def import_data(self, data): + """ + Import additional data for tuning. + + Parameters + ---------- + data : list of dict + Each of which has at least two keys, ``parameter`` and ``value``. + """ + _completed_num = 0 + for trial_info in data: + self.logger.info("Importing data, current processing progress %s / %s", _completed_num, len(data)) + # simply validate data format + assert "parameter" in trial_info + _params = trial_info["parameter"] + assert "value" in trial_info + _value = trial_info['value'] + if not _value: + self.logger.info("Useless trial data, value is %s, skip this trial data.", _value) + continue + _value = extract_scalar_reward(_value) + # convert the keys in loguniform and categorical types + valid_entry = True + for key, value in _params.items(): + if key in self.loguniform_key: + _params[key] = np.log(value) + elif key in self.categorical_dict: + if value in self.categorical_dict[key]: + _params[key] = self.categorical_dict[key].index(value) + else: + self.logger.info("The value %s of key %s is not in search space.", str(value), key) + valid_entry = False + break + if not valid_entry: + continue + # start import this data entry + _completed_num += 1 + config = Configuration(self.cs, values=_params) + if self.optimize_mode is OptimizeMode.Maximize: + _value = -_value + if self.first_one: + self.smbo_solver.nni_smac_receive_first_run(config, _value) + self.first_one = False + else: + self.smbo_solver.nni_smac_receive_runs(config, _value) + self.logger.info("Successfully import data to smac tuner, total data: %d, imported data: %d.", len(data), _completed_num) diff --git a/new_impl/cv/third_party/nni/algorithms/nas/__init__.py b/new_impl/cv/third_party/nni/algorithms/nas/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/new_impl/cv/third_party/nni/algorithms/nas/pytorch/__init__.py b/new_impl/cv/third_party/nni/algorithms/nas/pytorch/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/new_impl/cv/third_party/nni/algorithms/nas/pytorch/cdarts/__init__.py b/new_impl/cv/third_party/nni/algorithms/nas/pytorch/cdarts/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..2d0092784646cef640f40eaeca2df1fe26bd3d4c --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/nas/pytorch/cdarts/__init__.py @@ -0,0 +1,5 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .mutator import RegularizedDartsMutator, RegularizedMutatorParallel, DartsDiscreteMutator +from .trainer import CdartsTrainer \ No newline at end of file diff --git a/new_impl/cv/third_party/nni/algorithms/nas/pytorch/cdarts/mutator.py b/new_impl/cv/third_party/nni/algorithms/nas/pytorch/cdarts/mutator.py new file mode 100644 index 0000000000000000000000000000000000000000..85fe9e1a1c1133d32383c8174d0fa5bdfb92f39a --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/nas/pytorch/cdarts/mutator.py @@ -0,0 +1,143 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import torch + +from apex.parallel import DistributedDataParallel # pylint: disable=import-error +from nni.algorithms.nas.pytorch.darts import DartsMutator # pylint: disable=wrong-import-order +from nni.nas.pytorch.mutables import LayerChoice # pylint: disable=wrong-import-order +from nni.nas.pytorch.mutator import Mutator # pylint: disable=wrong-import-order + + +class RegularizedDartsMutator(DartsMutator): + """ + This is :class:`~nni.algorithms.nas.pytorch.darts.DartsMutator` basically, with two differences. + + 1. Choices can be cut (bypassed). This is done by ``cut_choices``. Cutted choices will not be used in + forward pass and thus consumes no memory. + + 2. Regularization on choices, to prevent the mutator from overfitting on some choices. + """ + + def reset(self): + """ + Warnings + -------- + Renamed :func:`~reset_with_loss` to return regularization loss on reset. + """ + raise ValueError("You should probably call `reset_with_loss`.") + + def cut_choices(self, cut_num=2): + """ + Cut the choices with the smallest weights. + ``cut_num`` should be the accumulative number of cutting, e.g., if first time cutting + is 2, the second time should be 4 to cut another two. + + Parameters + ---------- + cut_num : int + Number of choices to cut, so far. + + Warnings + -------- + Though the parameters are set to :math:`-\infty` to be bypassed, they will still receive gradient of 0, + which introduced ``nan`` problem when calling ``optimizer.step()``. To solve this issue, a simple way is to + reset nan to :math:`-\infty` each time after the parameters are updated. + """ + # `cut_choices` is implemented but not used in current implementation of CdartsTrainer + for mutable in self.mutables: + if isinstance(mutable, LayerChoice): + _, idx = torch.topk(-self.choices[mutable.key], cut_num) + with torch.no_grad(): + for i in idx: + self.choices[mutable.key][i] = -float("inf") + + def reset_with_loss(self): + """ + Resample and return loss. If loss is 0, to avoid device issue, it will return ``None``. + + Currently loss penalty are proportional to the L1-norm of parameters corresponding + to modules if their type name contains certain substrings. These substrings include: ``poolwithoutbn``, + ``identity``, ``dilconv``. + """ + self._cache, reg_loss = self.sample_search() + return reg_loss + + def sample_search(self): + result = super().sample_search() + loss = [] + for mutable in self.mutables: + if isinstance(mutable, LayerChoice): + def need_reg(choice): + return any(t in str(type(choice)).lower() for t in ["poolwithoutbn", "identity", "dilconv"]) + + for i, choice in enumerate(mutable.choices): + if need_reg(choice): + norm = torch.abs(self.choices[mutable.key][i]) + if norm < 1E10: + loss.append(norm) + if not loss: + return result, None + return result, sum(loss) + + def export(self, logger=None): + """ + Export an architecture with logger. Genotype will be printed with logger. + + Returns + ------- + dict + A mapping from mutable keys to decisions. + """ + result = self.sample_final() + if hasattr(self.model, "plot_genotype") and logger is not None: + genotypes = self.model.plot_genotype(result, logger) + return result, genotypes + + +class RegularizedMutatorParallel(DistributedDataParallel): + """ + Parallelize :class:`~RegularizedDartsMutator`. + + This makes :func:`~RegularizedDartsMutator.reset_with_loss` method parallelized, + also allowing :func:`~RegularizedDartsMutator.cut_choices` and :func:`~RegularizedDartsMutator.export` + to be easily accessible. + """ + def reset_with_loss(self): + """ + Parallelized :func:`~RegularizedDartsMutator.reset_with_loss`. + """ + result = self.module.reset_with_loss() + self.callback_queued = False + return result + + def cut_choices(self, *args, **kwargs): + """ + Parallelized :func:`~RegularizedDartsMutator.cut_choices`. + """ + self.module.cut_choices(*args, **kwargs) + + def export(self, logger): + """ + Parallelized :func:`~RegularizedDartsMutator.export`. + """ + return self.module.export(logger) + + +class DartsDiscreteMutator(Mutator): + """ + A mutator that applies the final sampling result of a parent mutator on another model to train. + + Parameters + ---------- + model : nn.Module + The model to apply the mutator. + parent_mutator : Mutator + The mutator that provides ``sample_final`` method, that will be called to get the architecture. + """ + def __init__(self, model, parent_mutator): + super().__init__(model) + self.__dict__["parent_mutator"] = parent_mutator # avoid parameters to be included + + def sample_search(self): + return self.parent_mutator.sample_final() diff --git a/new_impl/cv/third_party/nni/algorithms/nas/pytorch/cdarts/trainer.py b/new_impl/cv/third_party/nni/algorithms/nas/pytorch/cdarts/trainer.py new file mode 100644 index 0000000000000000000000000000000000000000..1a5174216fc4ffdc224e9cd6bd1483322005fe01 --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/nas/pytorch/cdarts/trainer.py @@ -0,0 +1,275 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import json +import logging +import os + +import torch +import torch.nn as nn +import torch.nn.functional as F +import apex # pylint: disable=import-error +from apex.parallel import DistributedDataParallel # pylint: disable=import-error +from .mutator import RegularizedDartsMutator, RegularizedMutatorParallel, DartsDiscreteMutator # pylint: disable=wrong-import-order +from nni.nas.pytorch.utils import AverageMeterGroup # pylint: disable=wrong-import-order + +from .utils import CyclicIterator, TorchTensorEncoder, accuracy, reduce_metrics + +PHASE_SMALL = "small" +PHASE_LARGE = "large" + + +class InteractiveKLLoss(nn.Module): + def __init__(self, temperature): + super().__init__() + self.temperature = temperature + # self.kl_loss = nn.KLDivLoss(reduction = 'batchmean') + self.kl_loss = nn.KLDivLoss() + + def forward(self, student, teacher): + return self.kl_loss(F.log_softmax(student / self.temperature, dim=1), + F.softmax(teacher / self.temperature, dim=1)) + + +class CdartsTrainer(object): + """ + CDARTS trainer. + + Parameters + ---------- + model_small : nn.Module + PyTorch model to be trained. This is the search network of CDARTS. + model_large : nn.Module + PyTorch model to be trained. This is the evaluation network of CDARTS. + criterion : callable + Receives logits and ground truth label, return a loss tensor, e.g., ``nn.CrossEntropyLoss()``. + loaders : list of torch.utils.data.DataLoader + List of train data and valid data loaders, for training weights and architecture weights respectively. + samplers : list of torch.utils.data.Sampler + List of train data and valid data samplers. This can be PyTorch standard samplers if not distributed. + In distributed mode, sampler needs to have ``set_epoch`` method. Refer to data utils in CDARTS example for details. + logger : logging.Logger + The logger for logging. Will use nni logger by default (if logger is ``None``). + regular_coeff : float + The coefficient of regular loss. + regular_ratio : float + The ratio of regular loss. + warmup_epochs : int + The epochs to warmup the search network + fix_head : bool + ``True`` if fixing the paramters of auxiliary heads, else unfix the paramters of auxiliary heads. + epochs : int + Number of epochs planned for training. + steps_per_epoch : int + Steps of one epoch. + loss_alpha : float + The loss coefficient. + loss_T : float + The loss coefficient. + distributed : bool + ``True`` if using distributed training, else non-distributed training. + log_frequency : int + Step count per logging. + grad_clip : float + Gradient clipping for weights. + interactive_type : string + ``kl`` or ``smoothl1``. + output_path : string + Log storage path. + w_lr : float + Learning rate of the search network parameters. + w_momentum : float + Momentum of the search and the evaluation network. + w_weight_decay : float + The weight decay the search and the evaluation network parameters. + alpha_lr : float + Learning rate of the architecture parameters. + alpha_weight_decay : float + The weight decay the architecture parameters. + nasnet_lr : float + Learning rate of the evaluation network parameters. + local_rank : int + The number of thread. + share_module : bool + ``True`` if sharing the stem and auxiliary heads, else not sharing these modules. + """ + def __init__(self, model_small, model_large, criterion, loaders, samplers, logger=None, + regular_coeff=5, regular_ratio=0.2, warmup_epochs=2, fix_head=True, + epochs=32, steps_per_epoch=None, loss_alpha=2, loss_T=2, distributed=True, + log_frequency=10, grad_clip=5.0, interactive_type='kl', output_path='./outputs', + w_lr=0.2, w_momentum=0.9, w_weight_decay=3e-4, alpha_lr=0.2, alpha_weight_decay=1e-4, + nasnet_lr=0.2, local_rank=0, share_module=True): + if logger is None: + logger = logging.getLogger(__name__) + train_loader, valid_loader = loaders + train_sampler, valid_sampler = samplers + self.train_loader = CyclicIterator(train_loader, train_sampler, distributed) + self.valid_loader = CyclicIterator(valid_loader, valid_sampler, distributed) + + self.regular_coeff = regular_coeff + self.regular_ratio = regular_ratio + self.warmup_epochs = warmup_epochs + self.fix_head = fix_head + self.epochs = epochs + self.steps_per_epoch = steps_per_epoch + if self.steps_per_epoch is None: + self.steps_per_epoch = min(len(self.train_loader), len(self.valid_loader)) + self.loss_alpha = loss_alpha + self.grad_clip = grad_clip + if interactive_type == "kl": + self.interactive_loss = InteractiveKLLoss(loss_T) + elif interactive_type == "smoothl1": + self.interactive_loss = nn.SmoothL1Loss() + self.loss_T = loss_T + self.distributed = distributed + self.log_frequency = log_frequency + self.main_proc = not distributed or local_rank == 0 + + self.logger = logger + self.checkpoint_dir = output_path + if self.main_proc: + os.makedirs(self.checkpoint_dir, exist_ok=True) + if distributed: + torch.distributed.barrier() + + self.model_small = model_small + self.model_large = model_large + if self.fix_head: + for param in self.model_small.aux_head.parameters(): + param.requires_grad = False + for param in self.model_large.aux_head.parameters(): + param.requires_grad = False + + self.mutator_small = RegularizedDartsMutator(self.model_small).cuda() + self.mutator_large = DartsDiscreteMutator(self.model_large, self.mutator_small).cuda() + self.criterion = criterion + + self.optimizer_small = torch.optim.SGD(self.model_small.parameters(), w_lr, + momentum=w_momentum, weight_decay=w_weight_decay) + self.optimizer_large = torch.optim.SGD(self.model_large.parameters(), nasnet_lr, + momentum=w_momentum, weight_decay=w_weight_decay) + self.optimizer_alpha = torch.optim.Adam(self.mutator_small.parameters(), alpha_lr, + betas=(0.5, 0.999), weight_decay=alpha_weight_decay) + + if distributed: + apex.parallel.convert_syncbn_model(self.model_small) + apex.parallel.convert_syncbn_model(self.model_large) + self.model_small = DistributedDataParallel(self.model_small, delay_allreduce=True) + self.model_large = DistributedDataParallel(self.model_large, delay_allreduce=True) + self.mutator_small = RegularizedMutatorParallel(self.mutator_small, delay_allreduce=True) + if share_module: + self.model_small.callback_queued = True + self.model_large.callback_queued = True + # mutator large never gets optimized, so do not need parallelized + + def _warmup(self, phase, epoch): + assert phase in [PHASE_SMALL, PHASE_LARGE] + if phase == PHASE_SMALL: + model, optimizer = self.model_small, self.optimizer_small + elif phase == PHASE_LARGE: + model, optimizer = self.model_large, self.optimizer_large + model.train() + meters = AverageMeterGroup() + for step in range(self.steps_per_epoch): + x, y = next(self.train_loader) + x, y = x.cuda(), y.cuda() + + optimizer.zero_grad() + logits_main, _ = model(x) + loss = self.criterion(logits_main, y) + loss.backward() + + self._clip_grad_norm(model) + optimizer.step() + prec1, prec5 = accuracy(logits_main, y, topk=(1, 5)) + metrics = {"prec1": prec1, "prec5": prec5, "loss": loss} + metrics = reduce_metrics(metrics, self.distributed) + meters.update(metrics) + if self.main_proc and (step % self.log_frequency == 0 or step + 1 == self.steps_per_epoch): + self.logger.info("Epoch [%d/%d] Step [%d/%d] (%s) %s", epoch + 1, self.epochs, + step + 1, self.steps_per_epoch, phase, meters) + + def _clip_grad_norm(self, model): + if isinstance(model, DistributedDataParallel): + nn.utils.clip_grad_norm_(model.module.parameters(), self.grad_clip) + else: + nn.utils.clip_grad_norm_(model.parameters(), self.grad_clip) + + def _reset_nan(self, parameters): + with torch.no_grad(): + for param in parameters: + for i, p in enumerate(param): + if p != p: # equivalent to `isnan(p)` + param[i] = float("-inf") + + def _joint_train(self, epoch): + self.model_large.train() + self.model_small.train() + meters = AverageMeterGroup() + for step in range(self.steps_per_epoch): + trn_x, trn_y = next(self.train_loader) + val_x, val_y = next(self.valid_loader) + trn_x, trn_y = trn_x.cuda(), trn_y.cuda() + val_x, val_y = val_x.cuda(), val_y.cuda() + + # step 1. optimize architecture + self.optimizer_alpha.zero_grad() + self.optimizer_large.zero_grad() + reg_decay = max(self.regular_coeff * (1 - float(epoch - self.warmup_epochs) / ( + (self.epochs - self.warmup_epochs) * self.regular_ratio)), 0) + loss_regular = self.mutator_small.reset_with_loss() + if loss_regular: + loss_regular *= reg_decay + logits_search, emsemble_logits_search = self.model_small(val_x) + logits_main, emsemble_logits_main = self.model_large(val_x) + loss_cls = (self.criterion(logits_search, val_y) + self.criterion(logits_main, val_y)) / self.loss_alpha + loss_interactive = self.interactive_loss(emsemble_logits_search, emsemble_logits_main) * (self.loss_T ** 2) * self.loss_alpha + loss = loss_cls + loss_interactive + loss_regular + loss.backward() + self._clip_grad_norm(self.model_large) + self.optimizer_large.step() + self.optimizer_alpha.step() + # NOTE: need to call here `self._reset_nan(self.mutator_small.parameters())` if `cut_choices` + + # step 2. optimize op weights + self.optimizer_small.zero_grad() + with torch.no_grad(): + # resample architecture since parameters have been changed + self.mutator_small.reset_with_loss() + logits_search_train, _ = self.model_small(trn_x) + loss_weight = self.criterion(logits_search_train, trn_y) + loss_weight.backward() + self._clip_grad_norm(self.model_small) + self.optimizer_small.step() + + metrics = {"loss_cls": loss_cls, "loss_interactive": loss_interactive, + "loss_regular": loss_regular, "loss_weight": loss_weight} + metrics = reduce_metrics(metrics, self.distributed) + meters.update(metrics) + + if self.main_proc and (step % self.log_frequency == 0 or step + 1 == self.steps_per_epoch): + self.logger.info("Epoch [%d/%d] Step [%d/%d] (joint) %s", epoch + 1, self.epochs, + step + 1, self.steps_per_epoch, meters) + + def train(self): + for epoch in range(self.epochs): + if epoch < self.warmup_epochs: + with torch.no_grad(): # otherwise grads will be retained on the architecture params + self.mutator_small.reset_with_loss() + self._warmup(PHASE_SMALL, epoch) + else: + with torch.no_grad(): + self.mutator_large.reset() + self._warmup(PHASE_LARGE, epoch) + self._joint_train(epoch) + + self.export(os.path.join(self.checkpoint_dir, "epoch_{:02d}.json".format(epoch)), + os.path.join(self.checkpoint_dir, "epoch_{:02d}.genotypes".format(epoch))) + + def export(self, file, genotype_file): + if self.main_proc: + mutator_export, genotypes = self.mutator_small.export(self.logger) + with open(file, "w") as f: + json.dump(mutator_export, f, indent=2, sort_keys=True, cls=TorchTensorEncoder) + with open(genotype_file, "w") as f: + f.write(str(genotypes)) diff --git a/new_impl/cv/third_party/nni/algorithms/nas/pytorch/cdarts/utils.py b/new_impl/cv/third_party/nni/algorithms/nas/pytorch/cdarts/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..780f6fdc0eecdd2fb00593bad1ef17ba2d8f7d39 --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/nas/pytorch/cdarts/utils.py @@ -0,0 +1,76 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import json +import os + +import torch +import torch.distributed as dist + + +class CyclicIterator: + def __init__(self, loader, sampler, distributed): + self.loader = loader + self.sampler = sampler + self.epoch = 0 + self.distributed = distributed + self._next_epoch() + + def _next_epoch(self): + if self.distributed: + self.sampler.set_epoch(self.epoch) + self.iterator = iter(self.loader) + self.epoch += 1 + + def __len__(self): + return len(self.loader) + + def __iter__(self): + return self + + def __next__(self): + try: + return next(self.iterator) + except StopIteration: + self._next_epoch() + return next(self.iterator) + + +class TorchTensorEncoder(json.JSONEncoder): + def default(self, o): # pylint: disable=method-hidden + if isinstance(o, torch.Tensor): + return o.tolist() + return super().default(o) + + +def accuracy(output, target, topk=(1,)): + """ Computes the precision@k for the specified values of k """ + maxk = max(topk) + batch_size = target.size(0) + + _, pred = output.topk(maxk, 1, True, True) + pred = pred.t() + # one-hot case + if target.ndimension() > 1: + target = target.max(1)[1] + + correct = pred.eq(target.view(1, -1).expand_as(pred)) + + res = [] + for k in topk: + correct_k = correct[:k].view(-1).float().sum(0) + res.append(correct_k.mul_(1.0 / batch_size)) + return res + + +def reduce_tensor(tensor): + rt = tensor.clone() + dist.all_reduce(rt, op=dist.ReduceOp.SUM) + rt /= float(os.environ["WORLD_SIZE"]) + return rt + + +def reduce_metrics(metrics, distributed=False): + if distributed: + return {k: reduce_tensor(v).item() for k, v in metrics.items()} + return {k: v.item() for k, v in metrics.items()} diff --git a/new_impl/cv/third_party/nni/algorithms/nas/pytorch/classic_nas/__init__.py b/new_impl/cv/third_party/nni/algorithms/nas/pytorch/classic_nas/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..ec3f5a4894a0460d4a0582a0d6cd43af9bed77e2 --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/nas/pytorch/classic_nas/__init__.py @@ -0,0 +1,4 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .mutator import get_and_apply_next_architecture diff --git a/new_impl/cv/third_party/nni/algorithms/nas/pytorch/classic_nas/mutator.py b/new_impl/cv/third_party/nni/algorithms/nas/pytorch/classic_nas/mutator.py new file mode 100644 index 0000000000000000000000000000000000000000..7254a8b0b4a34b5913dfc71d11f7364b86fc04b8 --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/nas/pytorch/classic_nas/mutator.py @@ -0,0 +1,221 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import json +import logging +import os +import sys + +import torch + +import nni +from nni.runtime.env_vars import trial_env_vars +from nni.nas.pytorch.mutables import LayerChoice, InputChoice, MutableScope +from nni.nas.pytorch.mutator import Mutator + +logger = logging.getLogger(__name__) + +NNI_GEN_SEARCH_SPACE = "NNI_GEN_SEARCH_SPACE" +LAYER_CHOICE = "layer_choice" +INPUT_CHOICE = "input_choice" + + +def get_and_apply_next_architecture(model): + """ + Wrapper of :class:`~nni.nas.pytorch.classic_nas.mutator.ClassicMutator` to make it more meaningful, + similar to ``get_next_parameter`` for HPO. + + It will generate search space based on ``model``. + If env ``NNI_GEN_SEARCH_SPACE`` exists, this is in dry run mode for + generating search space for the experiment. + If not, there are still two mode, one is nni experiment mode where users + use ``nnictl`` to start an experiment. The other is standalone mode + where users directly run the trial command, this mode chooses the first + one(s) for each LayerChoice and InputChoice. + + Parameters + ---------- + model : nn.Module + User's model with search space (e.g., LayerChoice, InputChoice) embedded in it. + """ + ClassicMutator(model) + + +class ClassicMutator(Mutator): + """ + This mutator is to apply the architecture chosen from tuner. + It implements the forward function of LayerChoice and InputChoice, + to only activate the chosen ones. + + Parameters + ---------- + model : nn.Module + User's model with search space (e.g., LayerChoice, InputChoice) embedded in it. + """ + + def __init__(self, model): + super(ClassicMutator, self).__init__(model) + self._chosen_arch = {} + self._search_space = self._generate_search_space() + if NNI_GEN_SEARCH_SPACE in os.environ: + # dry run for only generating search space + self._dump_search_space(os.environ[NNI_GEN_SEARCH_SPACE]) + sys.exit(0) + + if trial_env_vars.NNI_PLATFORM is None: + logger.warning("This is in standalone mode, the chosen are the first one(s).") + self._chosen_arch = self._standalone_generate_chosen() + else: + # get chosen arch from tuner + self._chosen_arch = nni.get_next_parameter() + if self._chosen_arch is None: + if trial_env_vars.NNI_PLATFORM == "unittest": + # happens if NNI_PLATFORM is intentionally set, e.g., in UT + logger.warning("`NNI_PLATFORM` is set but `param` is None. Falling back to standalone mode.") + self._chosen_arch = self._standalone_generate_chosen() + else: + raise RuntimeError("Chosen architecture is None. This may be a platform error.") + self.reset() + + def _sample_layer_choice(self, mutable, idx, value, search_space_item): + """ + Convert layer choice to tensor representation. + + Parameters + ---------- + mutable : Mutable + idx : int + Number `idx` of list will be selected. + value : str + The verbose representation of the selected value. + search_space_item : list + The list for corresponding search space. + """ + # doesn't support multihot for layer choice yet + onehot_list = [False] * len(mutable) + assert 0 <= idx < len(mutable) and search_space_item[idx] == value, \ + "Index '{}' in search space '{}' is not '{}'".format(idx, search_space_item, value) + onehot_list[idx] = True + return torch.tensor(onehot_list, dtype=torch.bool) # pylint: disable=not-callable + + def _sample_input_choice(self, mutable, idx, value, search_space_item): + """ + Convert input choice to tensor representation. + + Parameters + ---------- + mutable : Mutable + idx : int + Number `idx` of list will be selected. + value : str + The verbose representation of the selected value. + search_space_item : list + The list for corresponding search space. + """ + candidate_repr = search_space_item["candidates"] + multihot_list = [False] * mutable.n_candidates + for i, v in zip(idx, value): + assert 0 <= i < mutable.n_candidates and candidate_repr[i] == v, \ + "Index '{}' in search space '{}' is not '{}'".format(i, candidate_repr, v) + assert not multihot_list[i], "'{}' is selected twice in '{}', which is not allowed.".format(i, idx) + multihot_list[i] = True + return torch.tensor(multihot_list, dtype=torch.bool) # pylint: disable=not-callable + + def sample_search(self): + """ + See :meth:`sample_final`. + """ + return self.sample_final() + + def sample_final(self): + """ + Convert the chosen arch and apply it on model. + """ + assert set(self._chosen_arch.keys()) == set(self._search_space.keys()), \ + "Unmatched keys, expected keys '{}' from search space, found '{}'.".format(self._search_space.keys(), + self._chosen_arch.keys()) + result = dict() + for mutable in self.mutables: + if isinstance(mutable, (LayerChoice, InputChoice)): + assert mutable.key in self._chosen_arch, \ + "Expected '{}' in chosen arch, but not found.".format(mutable.key) + data = self._chosen_arch[mutable.key] + assert isinstance(data, dict) and "_value" in data and "_idx" in data, \ + "'{}' is not a valid choice.".format(data) + if isinstance(mutable, LayerChoice): + result[mutable.key] = self._sample_layer_choice(mutable, data["_idx"], data["_value"], + self._search_space[mutable.key]["_value"]) + elif isinstance(mutable, InputChoice): + result[mutable.key] = self._sample_input_choice(mutable, data["_idx"], data["_value"], + self._search_space[mutable.key]["_value"]) + elif isinstance(mutable, MutableScope): + logger.info("Mutable scope '%s' is skipped during parsing choices.", mutable.key) + else: + raise TypeError("Unsupported mutable type: '%s'." % type(mutable)) + return result + + def _standalone_generate_chosen(self): + """ + Generate the chosen architecture for standalone mode, + i.e., choose the first one(s) for LayerChoice and InputChoice. + :: + { key_name: {"_value": "conv1", + "_idx": 0} } + { key_name: {"_value": ["in1"], + "_idx": [0]} } + Returns + ------- + dict + the chosen architecture + """ + chosen_arch = {} + for key, val in self._search_space.items(): + if val["_type"] == LAYER_CHOICE: + choices = val["_value"] + chosen_arch[key] = {"_value": choices[0], "_idx": 0} + elif val["_type"] == INPUT_CHOICE: + choices = val["_value"]["candidates"] + n_chosen = val["_value"]["n_chosen"] + if n_chosen is None: + n_chosen = len(choices) + chosen_arch[key] = {"_value": choices[:n_chosen], "_idx": list(range(n_chosen))} + else: + raise ValueError("Unknown key '%s' and value '%s'." % (key, val)) + return chosen_arch + + def _generate_search_space(self): + """ + Generate search space from mutables. + Here is the search space format: + :: + { key_name: {"_type": "layer_choice", + "_value": ["conv1", "conv2"]} } + { key_name: {"_type": "input_choice", + "_value": {"candidates": ["in1", "in2"], + "n_chosen": 1}} } + Returns + ------- + dict + the generated search space + """ + search_space = {} + for mutable in self.mutables: + # for now we only generate flattened search space + if isinstance(mutable, LayerChoice): + key = mutable.key + val = mutable.names + search_space[key] = {"_type": LAYER_CHOICE, "_value": val} + elif isinstance(mutable, InputChoice): + key = mutable.key + search_space[key] = {"_type": INPUT_CHOICE, + "_value": {"candidates": mutable.choose_from, + "n_chosen": mutable.n_chosen}} + elif isinstance(mutable, MutableScope): + logger.info("Mutable scope '%s' is skipped during generating search space.", mutable.key) + else: + raise TypeError("Unsupported mutable type: '%s'." % type(mutable)) + return search_space + + def _dump_search_space(self, file_path): + with open(file_path, "w") as ss_file: + json.dump(self._search_space, ss_file, sort_keys=True, indent=2) diff --git a/new_impl/cv/third_party/nni/algorithms/nas/pytorch/darts/__init__.py b/new_impl/cv/third_party/nni/algorithms/nas/pytorch/darts/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..1a22790fb90b37591dd58f590a3b528b78b8257e --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/nas/pytorch/darts/__init__.py @@ -0,0 +1,5 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .mutator import DartsMutator +from .trainer import DartsTrainer diff --git a/new_impl/cv/third_party/nni/algorithms/nas/pytorch/darts/mutator.py b/new_impl/cv/third_party/nni/algorithms/nas/pytorch/darts/mutator.py new file mode 100644 index 0000000000000000000000000000000000000000..a4c3898a9b531ea160ea64a17d2e345f6b67d040 --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/nas/pytorch/darts/mutator.py @@ -0,0 +1,85 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from nni.nas.pytorch.mutator import Mutator +from nni.nas.pytorch.mutables import LayerChoice, InputChoice + +_logger = logging.getLogger(__name__) + + +class DartsMutator(Mutator): + """ + Connects the model in a DARTS (differentiable) way. + + An extra connection is automatically inserted for each LayerChoice, when this connection is selected, there is no + op on this LayerChoice (namely a ``ZeroOp``), in which case, every element in the exported choice list is ``false`` + (not chosen). + + All input choice will be fully connected in the search phase. On exporting, the input choice will choose inputs based + on keys in ``choose_from``. If the keys were to be keys of LayerChoices, the top logit of the corresponding LayerChoice + will join the competition of input choice to compete against other logits. Otherwise, the logit will be assumed 0. + + It's possible to cut branches by setting parameter ``choices`` in a particular position to ``-inf``. After softmax, the + value would be 0. Framework will ignore 0 values and not connect. Note that the gradient on the ``-inf`` location will + be 0. Since manipulations with ``-inf`` will be ``nan``, you need to handle the gradient update phase carefully. + + Attributes + ---------- + choices: ParameterDict + dict that maps keys of LayerChoices to weighted-connection float tensors. + """ + def __init__(self, model): + super().__init__(model) + self.choices = nn.ParameterDict() + for mutable in self.mutables: + if isinstance(mutable, LayerChoice): + self.choices[mutable.key] = nn.Parameter(1.0E-3 * torch.randn(mutable.length + 1)) + + def device(self): + for v in self.choices.values(): + return v.device + + def sample_search(self): + result = dict() + for mutable in self.mutables: + if isinstance(mutable, LayerChoice): + result[mutable.key] = F.softmax(self.choices[mutable.key], dim=-1)[:-1] + elif isinstance(mutable, InputChoice): + result[mutable.key] = torch.ones(mutable.n_candidates, dtype=torch.bool, device=self.device()) + return result + + def sample_final(self): + result = dict() + edges_max = dict() + for mutable in self.mutables: + if isinstance(mutable, LayerChoice): + max_val, index = torch.max(F.softmax(self.choices[mutable.key], dim=-1)[:-1], 0) + edges_max[mutable.key] = max_val + result[mutable.key] = F.one_hot(index, num_classes=len(mutable)).view(-1).bool() + for mutable in self.mutables: + if isinstance(mutable, InputChoice): + if mutable.n_chosen is not None: + weights = [] + for src_key in mutable.choose_from: + if src_key not in edges_max: + _logger.warning("InputChoice.NO_KEY in '%s' is weighted 0 when selecting inputs.", mutable.key) + weights.append(edges_max.get(src_key, 0.)) + weights = torch.tensor(weights) # pylint: disable=not-callable + _, topk_edge_indices = torch.topk(weights, mutable.n_chosen) + selected_multihot = [] + for i, src_key in enumerate(mutable.choose_from): + if i not in topk_edge_indices and src_key in result: + # If an edge is never selected, there is no need to calculate any op on this edge. + # This is to eliminate redundant calculation. + result[src_key] = torch.zeros_like(result[src_key]) + selected_multihot.append(i in topk_edge_indices) + result[mutable.key] = torch.tensor(selected_multihot, dtype=torch.bool, device=self.device()) # pylint: disable=not-callable + else: + result[mutable.key] = torch.ones(mutable.n_candidates, dtype=torch.bool, device=self.device()) # pylint: disable=not-callable + return result diff --git a/new_impl/cv/third_party/nni/algorithms/nas/pytorch/darts/trainer.py b/new_impl/cv/third_party/nni/algorithms/nas/pytorch/darts/trainer.py new file mode 100644 index 0000000000000000000000000000000000000000..3873035415d8bdbe5f0f73b2f2fa958d36b470a5 --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/nas/pytorch/darts/trainer.py @@ -0,0 +1,214 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import copy +import logging + +import torch +import torch.nn as nn +from nni.nas.pytorch.trainer import Trainer +from nni.nas.pytorch.utils import AverageMeterGroup + +from .mutator import DartsMutator + +logger = logging.getLogger(__name__) + + +class DartsTrainer(Trainer): + """ + DARTS trainer. + + Parameters + ---------- + model : nn.Module + PyTorch model to be trained. + loss : callable + Receives logits and ground truth label, return a loss tensor. + metrics : callable + Receives logits and ground truth label, return a dict of metrics. + optimizer : Optimizer + The optimizer used for optimizing the model. + num_epochs : int + Number of epochs planned for training. + dataset_train : Dataset + Dataset for training. Will be split for training weights and architecture weights. + dataset_valid : Dataset + Dataset for testing. + mutator : DartsMutator + Use in case of customizing your own DartsMutator. By default will instantiate a DartsMutator. + batch_size : int + Batch size. + workers : int + Workers for data loading. + device : torch.device + ``torch.device("cpu")`` or ``torch.device("cuda")``. + log_frequency : int + Step count per logging. + callbacks : list of Callback + list of callbacks to trigger at events. + arc_learning_rate : float + Learning rate of architecture parameters. + unrolled : float + ``True`` if using second order optimization, else first order optimization. + """ + def __init__(self, model, loss, metrics, + optimizer, num_epochs, dataset_train, dataset_valid, + mutator=None, batch_size=64, workers=4, device=None, log_frequency=None, + callbacks=None, arc_learning_rate=3.0E-4, unrolled=False): + super().__init__(model, mutator if mutator is not None else DartsMutator(model), + loss, metrics, optimizer, num_epochs, dataset_train, dataset_valid, + batch_size, workers, device, log_frequency, callbacks) + + self.ctrl_optim = torch.optim.Adam(self.mutator.parameters(), arc_learning_rate, betas=(0.5, 0.999), + weight_decay=1.0E-3) + self.unrolled = unrolled + + n_train = len(self.dataset_train) + split = n_train // 2 + indices = list(range(n_train)) + train_sampler = torch.utils.data.sampler.SubsetRandomSampler(indices[:split]) + valid_sampler = torch.utils.data.sampler.SubsetRandomSampler(indices[split:]) + self.train_loader = torch.utils.data.DataLoader(self.dataset_train, + batch_size=batch_size, + sampler=train_sampler, + num_workers=workers) + self.valid_loader = torch.utils.data.DataLoader(self.dataset_train, + batch_size=batch_size, + sampler=valid_sampler, + num_workers=workers) + self.test_loader = torch.utils.data.DataLoader(self.dataset_valid, + batch_size=batch_size, + num_workers=workers) + + def train_one_epoch(self, epoch): + self.model.train() + self.mutator.train() + meters = AverageMeterGroup() + for step, ((trn_X, trn_y), (val_X, val_y)) in enumerate(zip(self.train_loader, self.valid_loader)): + trn_X, trn_y = trn_X.to(self.device), trn_y.to(self.device) + val_X, val_y = val_X.to(self.device), val_y.to(self.device) + + # phase 1. architecture step + self.ctrl_optim.zero_grad() + if self.unrolled: + self._unrolled_backward(trn_X, trn_y, val_X, val_y) + else: + self._backward(val_X, val_y) + self.ctrl_optim.step() + + # phase 2: child network step + self.optimizer.zero_grad() + logits, loss = self._logits_and_loss(trn_X, trn_y) + loss.backward() + nn.utils.clip_grad_norm_(self.model.parameters(), 5.) # gradient clipping + self.optimizer.step() + + metrics = self.metrics(logits, trn_y) + metrics["loss"] = loss.item() + meters.update(metrics) + if self.log_frequency is not None and step % self.log_frequency == 0: + logger.info("Epoch [%s/%s] Step [%s/%s] %s", epoch + 1, + self.num_epochs, step + 1, len(self.train_loader), meters) + + def validate_one_epoch(self, epoch): + self.model.eval() + self.mutator.eval() + meters = AverageMeterGroup() + with torch.no_grad(): + self.mutator.reset() + for step, (X, y) in enumerate(self.test_loader): + X, y = X.to(self.device), y.to(self.device) + logits = self.model(X) + metrics = self.metrics(logits, y) + meters.update(metrics) + if self.log_frequency is not None and step % self.log_frequency == 0: + logger.info("Epoch [%s/%s] Step [%s/%s] %s", epoch + 1, + self.num_epochs, step + 1, len(self.test_loader), meters) + + def _logits_and_loss(self, X, y): + self.mutator.reset() + logits = self.model(X) + loss = self.loss(logits, y) + self._write_graph_status() + return logits, loss + + def _backward(self, val_X, val_y): + """ + Simple backward with gradient descent + """ + _, loss = self._logits_and_loss(val_X, val_y) + loss.backward() + + def _unrolled_backward(self, trn_X, trn_y, val_X, val_y): + """ + Compute unrolled loss and backward its gradients + """ + backup_params = copy.deepcopy(tuple(self.model.parameters())) + + # do virtual step on training data + lr = self.optimizer.param_groups[0]["lr"] + momentum = self.optimizer.param_groups[0]["momentum"] + weight_decay = self.optimizer.param_groups[0]["weight_decay"] + self._compute_virtual_model(trn_X, trn_y, lr, momentum, weight_decay) + + # calculate unrolled loss on validation data + # keep gradients for model here for compute hessian + _, loss = self._logits_and_loss(val_X, val_y) + w_model, w_ctrl = tuple(self.model.parameters()), tuple(self.mutator.parameters()) + w_grads = torch.autograd.grad(loss, w_model + w_ctrl) + d_model, d_ctrl = w_grads[:len(w_model)], w_grads[len(w_model):] + + # compute hessian and final gradients + hessian = self._compute_hessian(backup_params, d_model, trn_X, trn_y) + with torch.no_grad(): + for param, d, h in zip(w_ctrl, d_ctrl, hessian): + # gradient = dalpha - lr * hessian + param.grad = d - lr * h + + # restore weights + self._restore_weights(backup_params) + + def _compute_virtual_model(self, X, y, lr, momentum, weight_decay): + """ + Compute unrolled weights w` + """ + # don't need zero_grad, using autograd to calculate gradients + _, loss = self._logits_and_loss(X, y) + gradients = torch.autograd.grad(loss, self.model.parameters()) + with torch.no_grad(): + for w, g in zip(self.model.parameters(), gradients): + m = self.optimizer.state[w].get("momentum_buffer", 0.) + w = w - lr * (momentum * m + g + weight_decay * w) + + def _restore_weights(self, backup_params): + with torch.no_grad(): + for param, backup in zip(self.model.parameters(), backup_params): + param.copy_(backup) + + def _compute_hessian(self, backup_params, dw, trn_X, trn_y): + """ + dw = dw` { L_val(w`, alpha) } + w+ = w + eps * dw + w- = w - eps * dw + hessian = (dalpha { L_trn(w+, alpha) } - dalpha { L_trn(w-, alpha) }) / (2*eps) + eps = 0.01 / ||dw|| + """ + self._restore_weights(backup_params) + norm = torch.cat([w.view(-1) for w in dw]).norm() + eps = 0.01 / norm + if norm < 1E-8: + logger.warning("In computing hessian, norm is smaller than 1E-8, cause eps to be %.6f.", norm.item()) + + dalphas = [] + for e in [eps, -2. * eps]: + # w+ = w + eps*dw`, w- = w - eps*dw` + with torch.no_grad(): + for p, d in zip(self.model.parameters(), dw): + p += e * d + + _, loss = self._logits_and_loss(trn_X, trn_y) + dalphas.append(torch.autograd.grad(loss, self.mutator.parameters())) + + dalpha_pos, dalpha_neg = dalphas # dalpha { L_trn(w+) }, # dalpha { L_trn(w-) } + hessian = [(p - n) / 2. * eps for p, n in zip(dalpha_pos, dalpha_neg)] + return hessian diff --git a/new_impl/cv/third_party/nni/algorithms/nas/pytorch/enas/__init__.py b/new_impl/cv/third_party/nni/algorithms/nas/pytorch/enas/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d3372836ebba2387ade58161218aad731433e46b --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/nas/pytorch/enas/__init__.py @@ -0,0 +1,5 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .mutator import EnasMutator +from .trainer import EnasTrainer diff --git a/new_impl/cv/third_party/nni/algorithms/nas/pytorch/enas/mutator.py b/new_impl/cv/third_party/nni/algorithms/nas/pytorch/enas/mutator.py new file mode 100644 index 0000000000000000000000000000000000000000..7fdba26b99bf72551293fdb77ff61beb974bba8f --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/nas/pytorch/enas/mutator.py @@ -0,0 +1,197 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from nni.nas.pytorch.mutator import Mutator +from nni.nas.pytorch.mutables import LayerChoice, InputChoice, MutableScope + + +class StackedLSTMCell(nn.Module): + def __init__(self, layers, size, bias): + super().__init__() + self.lstm_num_layers = layers + self.lstm_modules = nn.ModuleList([nn.LSTMCell(size, size, bias=bias) + for _ in range(self.lstm_num_layers)]) + + def forward(self, inputs, hidden): + prev_h, prev_c = hidden + next_h, next_c = [], [] + for i, m in enumerate(self.lstm_modules): + curr_h, curr_c = m(inputs, (prev_h[i], prev_c[i])) + next_c.append(curr_c) + next_h.append(curr_h) + # current implementation only supports batch size equals 1, + # but the algorithm does not necessarily have this limitation + inputs = curr_h[-1].view(1, -1) + return next_h, next_c + + +class EnasMutator(Mutator): + """ + A mutator that mutates the graph with RL. + + Parameters + ---------- + model : nn.Module + PyTorch model. + lstm_size : int + Controller LSTM hidden units. + lstm_num_layers : int + Number of layers for stacked LSTM. + tanh_constant : float + Logits will be equal to ``tanh_constant * tanh(logits)``. Don't use ``tanh`` if this value is ``None``. + cell_exit_extra_step : bool + If true, RL controller will perform an extra step at the exit of each MutableScope, dump the hidden state + and mark it as the hidden state of this MutableScope. This is to align with the original implementation of paper. + skip_target : float + Target probability that skipconnect will appear. + temperature : float + Temperature constant that divides the logits. + branch_bias : float + Manual bias applied to make some operations more likely to be chosen. + Currently this is implemented with a hardcoded match rule that aligns with original repo. + If a mutable has a ``reduce`` in its key, all its op choices + that contains `conv` in their typename will receive a bias of ``+self.branch_bias`` initially; while others + receive a bias of ``-self.branch_bias``. + entropy_reduction : str + Can be one of ``sum`` and ``mean``. How the entropy of multi-input-choice is reduced. + """ + + def __init__(self, model, lstm_size=64, lstm_num_layers=1, tanh_constant=1.5, cell_exit_extra_step=False, + skip_target=0.4, temperature=None, branch_bias=0.25, entropy_reduction="sum"): + super().__init__(model) + self.lstm_size = lstm_size + self.lstm_num_layers = lstm_num_layers + self.tanh_constant = tanh_constant + self.temperature = temperature + self.cell_exit_extra_step = cell_exit_extra_step + self.skip_target = skip_target + self.branch_bias = branch_bias + + self.lstm = StackedLSTMCell(self.lstm_num_layers, self.lstm_size, False) + self.attn_anchor = nn.Linear(self.lstm_size, self.lstm_size, bias=False) + self.attn_query = nn.Linear(self.lstm_size, self.lstm_size, bias=False) + self.v_attn = nn.Linear(self.lstm_size, 1, bias=False) + self.g_emb = nn.Parameter(torch.randn(1, self.lstm_size) * 0.1) + self.skip_targets = nn.Parameter(torch.tensor([1.0 - self.skip_target, self.skip_target]), requires_grad=False) # pylint: disable=not-callable + assert entropy_reduction in ["sum", "mean"], "Entropy reduction must be one of sum and mean." + self.entropy_reduction = torch.sum if entropy_reduction == "sum" else torch.mean + self.cross_entropy_loss = nn.CrossEntropyLoss(reduction="none") + self.bias_dict = nn.ParameterDict() + + self.max_layer_choice = 0 + for mutable in self.mutables: + if isinstance(mutable, LayerChoice): + if self.max_layer_choice == 0: + self.max_layer_choice = len(mutable) + assert self.max_layer_choice == len(mutable), \ + "ENAS mutator requires all layer choice have the same number of candidates." + # We are judging by keys and module types to add biases to layer choices. Needs refactor. + if "reduce" in mutable.key: + def is_conv(choice): + return "conv" in str(type(choice)).lower() + bias = torch.tensor([self.branch_bias if is_conv(choice) else -self.branch_bias # pylint: disable=not-callable + for choice in mutable]) + self.bias_dict[mutable.key] = nn.Parameter(bias, requires_grad=False) + + self.embedding = nn.Embedding(self.max_layer_choice + 1, self.lstm_size) + self.soft = nn.Linear(self.lstm_size, self.max_layer_choice, bias=False) + + def sample_search(self): + self._initialize() + self._sample(self.mutables) + return self._choices + + def sample_final(self): + return self.sample_search() + + def _sample(self, tree): + mutable = tree.mutable + if isinstance(mutable, LayerChoice) and mutable.key not in self._choices: + self._choices[mutable.key] = self._sample_layer_choice(mutable) + elif isinstance(mutable, InputChoice) and mutable.key not in self._choices: + self._choices[mutable.key] = self._sample_input_choice(mutable) + for child in tree.children: + self._sample(child) + if isinstance(mutable, MutableScope) and mutable.key not in self._anchors_hid: + if self.cell_exit_extra_step: + self._lstm_next_step() + self._mark_anchor(mutable.key) + + def _initialize(self): + self._choices = dict() + self._anchors_hid = dict() + self._inputs = self.g_emb.data + self._c = [torch.zeros((1, self.lstm_size), + dtype=self._inputs.dtype, + device=self._inputs.device) for _ in range(self.lstm_num_layers)] + self._h = [torch.zeros((1, self.lstm_size), + dtype=self._inputs.dtype, + device=self._inputs.device) for _ in range(self.lstm_num_layers)] + self.sample_log_prob = 0 + self.sample_entropy = 0 + self.sample_skip_penalty = 0 + + def _lstm_next_step(self): + self._h, self._c = self.lstm(self._inputs, (self._h, self._c)) + + def _mark_anchor(self, key): + self._anchors_hid[key] = self._h[-1] + + def _sample_layer_choice(self, mutable): + self._lstm_next_step() + logit = self.soft(self._h[-1]) + if self.temperature is not None: + logit /= self.temperature + if self.tanh_constant is not None: + logit = self.tanh_constant * torch.tanh(logit) + if mutable.key in self.bias_dict: + logit += self.bias_dict[mutable.key] + branch_id = torch.multinomial(F.softmax(logit, dim=-1), 1).view(-1) + log_prob = self.cross_entropy_loss(logit, branch_id) + self.sample_log_prob += self.entropy_reduction(log_prob) + entropy = (log_prob * torch.exp(-log_prob)).detach() # pylint: disable=invalid-unary-operand-type + self.sample_entropy += self.entropy_reduction(entropy) + self._inputs = self.embedding(branch_id) + return F.one_hot(branch_id, num_classes=self.max_layer_choice).bool().view(-1) + + def _sample_input_choice(self, mutable): + query, anchors = [], [] + for label in mutable.choose_from: + if label not in self._anchors_hid: + self._lstm_next_step() + self._mark_anchor(label) # empty loop, fill not found + query.append(self.attn_anchor(self._anchors_hid[label])) + anchors.append(self._anchors_hid[label]) + query = torch.cat(query, 0) + query = torch.tanh(query + self.attn_query(self._h[-1])) + query = self.v_attn(query) + if self.temperature is not None: + query /= self.temperature + if self.tanh_constant is not None: + query = self.tanh_constant * torch.tanh(query) + + if mutable.n_chosen is None: + logit = torch.cat([-query, query], 1) # pylint: disable=invalid-unary-operand-type + + skip = torch.multinomial(F.softmax(logit, dim=-1), 1).view(-1) + skip_prob = torch.sigmoid(logit) + kl = torch.sum(skip_prob * torch.log(skip_prob / self.skip_targets)) + self.sample_skip_penalty += kl + log_prob = self.cross_entropy_loss(logit, skip) + self._inputs = (torch.matmul(skip.float(), torch.cat(anchors, 0)) / (1. + torch.sum(skip))).unsqueeze(0) + else: + assert mutable.n_chosen == 1, "Input choice must select exactly one or any in ENAS." + logit = query.view(1, -1) + index = torch.multinomial(F.softmax(logit, dim=-1), 1).view(-1) + skip = F.one_hot(index, num_classes=mutable.n_candidates).view(-1) + log_prob = self.cross_entropy_loss(logit, index) + self._inputs = anchors[index.item()] + + self.sample_log_prob += self.entropy_reduction(log_prob) + entropy = (log_prob * torch.exp(-log_prob)).detach() # pylint: disable=invalid-unary-operand-type + self.sample_entropy += self.entropy_reduction(entropy) + return skip.bool() diff --git a/new_impl/cv/third_party/nni/algorithms/nas/pytorch/enas/trainer.py b/new_impl/cv/third_party/nni/algorithms/nas/pytorch/enas/trainer.py new file mode 100644 index 0000000000000000000000000000000000000000..5e7a966580a107ceb6b9fe88f888dafefb8b69e7 --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/nas/pytorch/enas/trainer.py @@ -0,0 +1,209 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +from itertools import cycle + +import torch +import torch.nn as nn +import torch.optim as optim + +from nni.nas.pytorch.trainer import Trainer +from nni.nas.pytorch.utils import AverageMeterGroup, to_device +from .mutator import EnasMutator + +logger = logging.getLogger(__name__) + + +class EnasTrainer(Trainer): + """ + ENAS trainer. + + Parameters + ---------- + model : nn.Module + PyTorch model to be trained. + loss : callable + Receives logits and ground truth label, return a loss tensor. + metrics : callable + Receives logits and ground truth label, return a dict of metrics. + reward_function : callable + Receives logits and ground truth label, return a tensor, which will be feeded to RL controller as reward. + optimizer : Optimizer + The optimizer used for optimizing the model. + num_epochs : int + Number of epochs planned for training. + dataset_train : Dataset + Dataset for training. Will be split for training weights and architecture weights. + dataset_valid : Dataset + Dataset for testing. + mutator : EnasMutator + Use when customizing your own mutator or a mutator with customized parameters. + batch_size : int + Batch size. + workers : int + Workers for data loading. + device : torch.device + ``torch.device("cpu")`` or ``torch.device("cuda")``. + log_frequency : int + Step count per logging. + callbacks : list of Callback + list of callbacks to trigger at events. + entropy_weight : float + Weight of sample entropy loss. + skip_weight : float + Weight of skip penalty loss. + baseline_decay : float + Decay factor of baseline. New baseline will be equal to ``baseline_decay * baseline_old + reward * (1 - baseline_decay)``. + child_steps : int + How many mini-batches for model training per epoch. + mutator_lr : float + Learning rate for RL controller. + mutator_steps_aggregate : int + Number of steps that will be aggregated into one mini-batch for RL controller. + mutator_steps : int + Number of mini-batches for each epoch of RL controller learning. + aux_weight : float + Weight of auxiliary head loss. ``aux_weight * aux_loss`` will be added to total loss. + test_arc_per_epoch : int + How many architectures are chosen for direct test after each epoch. + """ + def __init__(self, model, loss, metrics, reward_function, + optimizer, num_epochs, dataset_train, dataset_valid, + mutator=None, batch_size=64, workers=4, device=None, log_frequency=None, callbacks=None, + entropy_weight=0.0001, skip_weight=0.8, baseline_decay=0.999, child_steps=500, + mutator_lr=0.00035, mutator_steps_aggregate=20, mutator_steps=50, aux_weight=0.4, + test_arc_per_epoch=1): + super().__init__(model, mutator if mutator is not None else EnasMutator(model), + loss, metrics, optimizer, num_epochs, dataset_train, dataset_valid, + batch_size, workers, device, log_frequency, callbacks) + self.reward_function = reward_function + self.mutator_optim = optim.Adam(self.mutator.parameters(), lr=mutator_lr) + self.batch_size = batch_size + self.workers = workers + + self.entropy_weight = entropy_weight + self.skip_weight = skip_weight + self.baseline_decay = baseline_decay + self.baseline = 0. + self.mutator_steps_aggregate = mutator_steps_aggregate + self.mutator_steps = mutator_steps + self.child_steps = child_steps + self.aux_weight = aux_weight + self.test_arc_per_epoch = test_arc_per_epoch + + self.init_dataloader() + + def init_dataloader(self): + n_train = len(self.dataset_train) + split = n_train // 10 + indices = list(range(n_train)) + train_sampler = torch.utils.data.sampler.SubsetRandomSampler(indices[:-split]) + valid_sampler = torch.utils.data.sampler.SubsetRandomSampler(indices[-split:]) + self.train_loader = torch.utils.data.DataLoader(self.dataset_train, + batch_size=self.batch_size, + sampler=train_sampler, + num_workers=self.workers) + self.valid_loader = torch.utils.data.DataLoader(self.dataset_train, + batch_size=self.batch_size, + sampler=valid_sampler, + num_workers=self.workers) + self.test_loader = torch.utils.data.DataLoader(self.dataset_valid, + batch_size=self.batch_size, + num_workers=self.workers) + self.train_loader = cycle(self.train_loader) + self.valid_loader = cycle(self.valid_loader) + + def train_one_epoch(self, epoch): + # Sample model and train + self.model.train() + self.mutator.eval() + meters = AverageMeterGroup() + for step in range(1, self.child_steps + 1): + x, y = next(self.train_loader) + x, y = to_device(x, self.device), to_device(y, self.device) + self.optimizer.zero_grad() + + with torch.no_grad(): + self.mutator.reset() + self._write_graph_status() + logits = self.model(x) + + if isinstance(logits, tuple): + logits, aux_logits = logits + aux_loss = self.loss(aux_logits, y) + else: + aux_loss = 0. + metrics = self.metrics(logits, y) + loss = self.loss(logits, y) + loss = loss + self.aux_weight * aux_loss + loss.backward() + nn.utils.clip_grad_norm_(self.model.parameters(), 5.) + self.optimizer.step() + metrics["loss"] = loss.item() + meters.update(metrics) + + if self.log_frequency is not None and step % self.log_frequency == 0: + logger.info("Model Epoch [%d/%d] Step [%d/%d] %s", epoch + 1, + self.num_epochs, step, self.child_steps, meters) + + # Train sampler (mutator) + self.model.eval() + self.mutator.train() + meters = AverageMeterGroup() + for mutator_step in range(1, self.mutator_steps + 1): + self.mutator_optim.zero_grad() + for step in range(1, self.mutator_steps_aggregate + 1): + x, y = next(self.valid_loader) + x, y = to_device(x, self.device), to_device(y, self.device) + + self.mutator.reset() + with torch.no_grad(): + logits = self.model(x) + self._write_graph_status() + metrics = self.metrics(logits, y) + reward = self.reward_function(logits, y) + if self.entropy_weight: + reward += self.entropy_weight * self.mutator.sample_entropy.item() + self.baseline = self.baseline * self.baseline_decay + reward * (1 - self.baseline_decay) + loss = self.mutator.sample_log_prob * (reward - self.baseline) + if self.skip_weight: + loss += self.skip_weight * self.mutator.sample_skip_penalty + metrics["reward"] = reward + metrics["loss"] = loss.item() + metrics["ent"] = self.mutator.sample_entropy.item() + metrics["log_prob"] = self.mutator.sample_log_prob.item() + metrics["baseline"] = self.baseline + metrics["skip"] = self.mutator.sample_skip_penalty + + loss /= self.mutator_steps_aggregate + loss.backward() + meters.update(metrics) + + cur_step = step + (mutator_step - 1) * self.mutator_steps_aggregate + if self.log_frequency is not None and cur_step % self.log_frequency == 0: + logger.info("RL Epoch [%d/%d] Step [%d/%d] [%d/%d] %s", epoch + 1, self.num_epochs, + mutator_step, self.mutator_steps, step, self.mutator_steps_aggregate, + meters) + + nn.utils.clip_grad_norm_(self.mutator.parameters(), 5.) + self.mutator_optim.step() + + def validate_one_epoch(self, epoch): + with torch.no_grad(): + for arc_id in range(self.test_arc_per_epoch): + meters = AverageMeterGroup() + for x, y in self.test_loader: + x, y = to_device(x, self.device), to_device(y, self.device) + self.mutator.reset() + logits = self.model(x) + if isinstance(logits, tuple): + logits, _ = logits + metrics = self.metrics(logits, y) + loss = self.loss(logits, y) + metrics["loss"] = loss.item() + meters.update(metrics) + + logger.info("Test Epoch [%d/%d] Arc [%d/%d] Summary %s", + epoch + 1, self.num_epochs, arc_id + 1, self.test_arc_per_epoch, + meters.summary()) diff --git a/new_impl/cv/third_party/nni/algorithms/nas/pytorch/pdarts/__init__.py b/new_impl/cv/third_party/nni/algorithms/nas/pytorch/pdarts/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d1d17764ba159b35bcc38efa82a2a30dc2366b76 --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/nas/pytorch/pdarts/__init__.py @@ -0,0 +1,4 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .trainer import PdartsTrainer diff --git a/new_impl/cv/third_party/nni/algorithms/nas/pytorch/pdarts/mutator.py b/new_impl/cv/third_party/nni/algorithms/nas/pytorch/pdarts/mutator.py new file mode 100644 index 0000000000000000000000000000000000000000..09ad51c5e471b4acae51d513335682328924b90a --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/nas/pytorch/pdarts/mutator.py @@ -0,0 +1,93 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import copy + +import numpy as np +import torch +from torch import nn + +from nni.algorithms.nas.pytorch.darts import DartsMutator +from nni.nas.pytorch.mutables import LayerChoice + + +class PdartsMutator(DartsMutator): + """ + It works with PdartsTrainer to calculate ops weights, + and drop weights in different PDARTS epochs. + """ + + def __init__(self, model, pdarts_epoch_index, pdarts_num_to_drop, switches={}): + self.pdarts_epoch_index = pdarts_epoch_index + self.pdarts_num_to_drop = pdarts_num_to_drop + if switches is None: + self.switches = {} + else: + self.switches = switches + + super(PdartsMutator, self).__init__(model) + + # this loop go through mutables with different keys, + # it's mainly to update length of choices. + for mutable in self.mutables: + if isinstance(mutable, LayerChoice): + + switches = self.switches.get(mutable.key, [True for j in range(len(mutable))]) + choices = self.choices[mutable.key] + + operations_count = np.sum(switches) + # +1 and -1 are caused by zero operation in darts network + # the zero operation is not in choices list in network, but its weight are in, + # so it needs one more weights and switch for zero. + self.choices[mutable.key] = nn.Parameter(1.0E-3 * torch.randn(operations_count + 1)) + self.switches[mutable.key] = switches + + # update LayerChoice instances in model, + # it's physically remove dropped choices operations. + for module in self.model.modules(): + if isinstance(module, LayerChoice): + switches = self.switches.get(module.key) + choices = self.choices[module.key] + if len(module) > len(choices): + # from last to first, so that it won't effect previous indexes after removed one. + for index in range(len(switches)-1, -1, -1): + if switches[index] == False: + del module[index] + assert len(module) <= len(choices), "Failed to remove dropped choices." + + def export(self): + # Cannot rely on super().export() because P-DARTS has deleted some of the choices and has misaligned length. + results = super().sample_final() + for mutable in self.mutables: + if isinstance(mutable, LayerChoice): + # As some operations are dropped physically, + # so it needs to fill back false to track dropped operations. + trained_result = results[mutable.key] + trained_index = 0 + switches = self.switches[mutable.key] + result = torch.Tensor(switches).bool() + for index in range(len(result)): + if result[index]: + result[index] = trained_result[trained_index] + trained_index += 1 + results[mutable.key] = result + return results + + def drop_paths(self): + """ + This method is called when a PDARTS epoch is finished. + It prepares switches for next epoch. + candidate operations with False switch will be doppped in next epoch. + """ + all_switches = copy.deepcopy(self.switches) + for key in all_switches: + switches = all_switches[key] + idxs = [] + for j in range(len(switches)): + if switches[j]: + idxs.append(j) + sorted_weights = self.choices[key].data.cpu().numpy()[:-1] + drop = np.argsort(sorted_weights)[:self.pdarts_num_to_drop[self.pdarts_epoch_index]] + for idx in drop: + switches[idxs[idx]] = False + return all_switches diff --git a/new_impl/cv/third_party/nni/algorithms/nas/pytorch/pdarts/trainer.py b/new_impl/cv/third_party/nni/algorithms/nas/pytorch/pdarts/trainer.py new file mode 100644 index 0000000000000000000000000000000000000000..7f23a6e222731ae6edc5c034f77eafaaf71b4c5e --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/nas/pytorch/pdarts/trainer.py @@ -0,0 +1,86 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import json +import logging + +from nni.nas.pytorch.callbacks import LRSchedulerCallback +from nni.algorithms.nas.pytorch.darts import DartsTrainer +from nni.nas.pytorch.trainer import BaseTrainer, TorchTensorEncoder + +from .mutator import PdartsMutator + +logger = logging.getLogger(__name__) + + +class PdartsTrainer(BaseTrainer): + """ + This trainer implements the PDARTS algorithm. + PDARTS bases on DARTS algorithm, and provides a network growth approach to find deeper and better network. + This class relies on pdarts_num_layers and pdarts_num_to_drop parameters to control how network grows. + pdarts_num_layers means how many layers more than first epoch. + pdarts_num_to_drop means how many candidate operations should be dropped in each epoch. + So that the grew network can in similar size. + """ + + def __init__(self, model_creator, init_layers, metrics, + num_epochs, dataset_train, dataset_valid, + pdarts_num_layers=[0, 6, 12], pdarts_num_to_drop=[3, 2, 1], + mutator=None, batch_size=64, workers=4, device=None, log_frequency=None, callbacks=None, unrolled=False): + super(PdartsTrainer, self).__init__() + self.model_creator = model_creator + self.init_layers = init_layers + self.pdarts_num_layers = pdarts_num_layers + self.pdarts_num_to_drop = pdarts_num_to_drop + self.pdarts_epoch = len(pdarts_num_to_drop) + self.darts_parameters = { + "metrics": metrics, + "num_epochs": num_epochs, + "dataset_train": dataset_train, + "dataset_valid": dataset_valid, + "batch_size": batch_size, + "workers": workers, + "device": device, + "log_frequency": log_frequency, + "unrolled": unrolled + } + self.callbacks = callbacks if callbacks is not None else [] + + def train(self): + + switches = None + for epoch in range(self.pdarts_epoch): + + layers = self.init_layers+self.pdarts_num_layers[epoch] + model, criterion, optim, lr_scheduler = self.model_creator(layers) + self.mutator = PdartsMutator(model, epoch, self.pdarts_num_to_drop, switches) + + for callback in self.callbacks: + callback.build(model, self.mutator, self) + callback.on_epoch_begin(epoch) + + darts_callbacks = [] + if lr_scheduler is not None: + darts_callbacks.append(LRSchedulerCallback(lr_scheduler)) + + self.trainer = DartsTrainer(model, mutator=self.mutator, loss=criterion, optimizer=optim, + callbacks=darts_callbacks, **self.darts_parameters) + logger.info("start pdarts training epoch %s...", epoch) + + self.trainer.train() + + switches = self.mutator.drop_paths() + + for callback in self.callbacks: + callback.on_epoch_end(epoch) + + def validate(self): + self.trainer.validate() + + def export(self, file): + mutator_export = self.mutator.export() + with open(file, "w") as f: + json.dump(mutator_export, f, indent=2, sort_keys=True, cls=TorchTensorEncoder) + + def checkpoint(self): + raise NotImplementedError("Not implemented yet") diff --git a/new_impl/cv/third_party/nni/algorithms/nas/pytorch/proxylessnas/__init__.py b/new_impl/cv/third_party/nni/algorithms/nas/pytorch/proxylessnas/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..26feedba7d553c32d61ea9139620a68fca7c12d0 --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/nas/pytorch/proxylessnas/__init__.py @@ -0,0 +1,2 @@ +from .mutator import ProxylessNasMutator +from .trainer import ProxylessNasTrainer diff --git a/new_impl/cv/third_party/nni/algorithms/nas/pytorch/proxylessnas/mutator.py b/new_impl/cv/third_party/nni/algorithms/nas/pytorch/proxylessnas/mutator.py new file mode 100644 index 0000000000000000000000000000000000000000..881a6b44038d5d6f2c97b2b106036954401a9cd3 --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/nas/pytorch/proxylessnas/mutator.py @@ -0,0 +1,478 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import math +import torch +from torch import nn as nn +from torch.nn import functional as F +import numpy as np + +from nni.nas.pytorch.base_mutator import BaseMutator +from nni.nas.pytorch.mutables import LayerChoice +from .utils import detach_variable + +class ArchGradientFunction(torch.autograd.Function): + @staticmethod + def forward(ctx, x, binary_gates, run_func, backward_func): + ctx.run_func = run_func + ctx.backward_func = backward_func + + detached_x = detach_variable(x) + with torch.enable_grad(): + output = run_func(detached_x) + ctx.save_for_backward(detached_x, output) + return output.data + + @staticmethod + def backward(ctx, grad_output): + detached_x, output = ctx.saved_tensors + + grad_x = torch.autograd.grad(output, detached_x, grad_output, only_inputs=True) + # compute gradients w.r.t. binary_gates + binary_grads = ctx.backward_func(detached_x.data, output.data, grad_output.data) + + return grad_x[0], binary_grads, None, None + +class MixedOp(nn.Module): + """ + This class is to instantiate and manage info of one LayerChoice. + It includes architecture weights, binary weights, and member functions + operating the weights. + + forward_mode: + forward/backward mode for LayerChoice: None, two, full, and full_v2. + For training architecture weights, we use full_v2 by default, and for training + model weights, we use None. + """ + forward_mode = None + def __init__(self, mutable): + """ + Parameters + ---------- + mutable : LayerChoice + A LayerChoice in user model + """ + super(MixedOp, self).__init__() + self.ap_path_alpha = nn.Parameter(torch.Tensor(len(mutable))) + self.ap_path_wb = nn.Parameter(torch.Tensor(len(mutable))) + self.ap_path_alpha.requires_grad = False + self.ap_path_wb.requires_grad = False + self.active_index = [0] + self.inactive_index = None + self.log_prob = None + self.current_prob_over_ops = None + self.n_choices = len(mutable) + + def get_ap_path_alpha(self): + return self.ap_path_alpha + + def to_requires_grad(self): + self.ap_path_alpha.requires_grad = True + self.ap_path_wb.requires_grad = True + + def to_disable_grad(self): + self.ap_path_alpha.requires_grad = False + self.ap_path_wb.requires_grad = False + + def forward(self, mutable, x): + """ + Define forward of LayerChoice. For 'full_v2', backward is also defined. + The 'two' mode is explained in section 3.2.1 in the paper. + The 'full_v2' mode is explained in Appendix D in the paper. + + Parameters + ---------- + mutable : LayerChoice + this layer's mutable + x : tensor + inputs of this layer, only support one input + + Returns + ------- + output: tensor + output of this layer + """ + if MixedOp.forward_mode == 'full' or MixedOp.forward_mode == 'two': + output = 0 + for _i in self.active_index: + oi = self.candidate_ops[_i](x) + output = output + self.ap_path_wb[_i] * oi + for _i in self.inactive_index: + oi = self.candidate_ops[_i](x) + output = output + self.ap_path_wb[_i] * oi.detach() + elif MixedOp.forward_mode == 'full_v2': + def run_function(key, candidate_ops, active_id): + def forward(_x): + return candidate_ops[active_id](_x) + return forward + + def backward_function(key, candidate_ops, active_id, binary_gates): + def backward(_x, _output, grad_output): + binary_grads = torch.zeros_like(binary_gates.data) + with torch.no_grad(): + for k in range(len(candidate_ops)): + if k != active_id: + out_k = candidate_ops[k](_x.data) + else: + out_k = _output.data + grad_k = torch.sum(out_k * grad_output) + binary_grads[k] = grad_k + return binary_grads + return backward + output = ArchGradientFunction.apply( + x, self.ap_path_wb, run_function(mutable.key, list(mutable), self.active_index[0]), + backward_function(mutable.key, list(mutable), self.active_index[0], self.ap_path_wb)) + else: + output = self.active_op(mutable)(x) + return output + + @property + def probs_over_ops(self): + """ + Apply softmax on alpha to generate probability distribution + + Returns + ------- + pytorch tensor + probability distribution + """ + probs = F.softmax(self.ap_path_alpha, dim=0) # softmax to probability + return probs + + @property + def chosen_index(self): + """ + choose the op with max prob + + Returns + ------- + int + index of the chosen one + numpy.float32 + prob of the chosen one + """ + probs = self.probs_over_ops.data.cpu().numpy() + index = int(np.argmax(probs)) + return index, probs[index] + + def active_op(self, mutable): + """ + assume only one path is active + + Returns + ------- + PyTorch module + the chosen operation + """ + return mutable[self.active_index[0]] + + @property + def active_op_index(self): + """ + return active op's index, the active op is sampled + + Returns + ------- + int + index of the active op + """ + return self.active_index[0] + + def set_chosen_op_active(self): + """ + set chosen index, active and inactive indexes + """ + chosen_idx, _ = self.chosen_index + self.active_index = [chosen_idx] + self.inactive_index = [_i for _i in range(0, chosen_idx)] + \ + [_i for _i in range(chosen_idx + 1, self.n_choices)] + + def binarize(self, mutable): + """ + Sample based on alpha, and set binary weights accordingly. + ap_path_wb is set in this function, which is called binarize. + + Parameters + ---------- + mutable : LayerChoice + this layer's mutable + """ + self.log_prob = None + # reset binary gates + self.ap_path_wb.data.zero_() + probs = self.probs_over_ops + if MixedOp.forward_mode == 'two': + # sample two ops according to probs + sample_op = torch.multinomial(probs.data, 2, replacement=False) + probs_slice = F.softmax(torch.stack([ + self.ap_path_alpha[idx] for idx in sample_op + ]), dim=0) + self.current_prob_over_ops = torch.zeros_like(probs) + for i, idx in enumerate(sample_op): + self.current_prob_over_ops[idx] = probs_slice[i] + # choose one to be active and the other to be inactive according to probs_slice + c = torch.multinomial(probs_slice.data, 1)[0] # 0 or 1 + active_op = sample_op[c].item() + inactive_op = sample_op[1-c].item() + self.active_index = [active_op] + self.inactive_index = [inactive_op] + # set binary gate + self.ap_path_wb.data[active_op] = 1.0 + else: + sample = torch.multinomial(probs, 1)[0].item() + self.active_index = [sample] + self.inactive_index = [_i for _i in range(0, sample)] + \ + [_i for _i in range(sample + 1, len(mutable))] + self.log_prob = torch.log(probs[sample]) + self.current_prob_over_ops = probs + self.ap_path_wb.data[sample] = 1.0 + # avoid over-regularization + for choice in mutable: + for _, param in choice.named_parameters(): + param.grad = None + + @staticmethod + def delta_ij(i, j): + if i == j: + return 1 + else: + return 0 + + def set_arch_param_grad(self, mutable): + """ + Calculate alpha gradient for this LayerChoice. + It is calculated using gradient of binary gate, probs of ops. + """ + binary_grads = self.ap_path_wb.grad.data + if self.active_op(mutable).is_zero_layer(): + self.ap_path_alpha.grad = None + return + if self.ap_path_alpha.grad is None: + self.ap_path_alpha.grad = torch.zeros_like(self.ap_path_alpha.data) + if MixedOp.forward_mode == 'two': + involved_idx = self.active_index + self.inactive_index + probs_slice = F.softmax(torch.stack([ + self.ap_path_alpha[idx] for idx in involved_idx + ]), dim=0).data + for i in range(2): + for j in range(2): + origin_i = involved_idx[i] + origin_j = involved_idx[j] + self.ap_path_alpha.grad.data[origin_i] += \ + binary_grads[origin_j] * probs_slice[j] * (MixedOp.delta_ij(i, j) - probs_slice[i]) + for _i, idx in enumerate(self.active_index): + self.active_index[_i] = (idx, self.ap_path_alpha.data[idx].item()) + for _i, idx in enumerate(self.inactive_index): + self.inactive_index[_i] = (idx, self.ap_path_alpha.data[idx].item()) + else: + probs = self.probs_over_ops.data + for i in range(self.n_choices): + for j in range(self.n_choices): + self.ap_path_alpha.grad.data[i] += binary_grads[j] * probs[j] * (MixedOp.delta_ij(i, j) - probs[i]) + return + + def rescale_updated_arch_param(self): + """ + rescale architecture weights for the 'two' mode. + """ + if not isinstance(self.active_index[0], tuple): + assert self.active_op.is_zero_layer() + return + involved_idx = [idx for idx, _ in (self.active_index + self.inactive_index)] + old_alphas = [alpha for _, alpha in (self.active_index + self.inactive_index)] + new_alphas = [self.ap_path_alpha.data[idx] for idx in involved_idx] + + offset = math.log( + sum([math.exp(alpha) for alpha in new_alphas]) / sum([math.exp(alpha) for alpha in old_alphas]) + ) + + for idx in involved_idx: + self.ap_path_alpha.data[idx] -= offset + + +class ProxylessNasMutator(BaseMutator): + """ + This mutator initializes and operates all the LayerChoices of the input model. + It is for the corresponding trainer to control the training process of LayerChoices, + coordinating with whole training process. + """ + def __init__(self, model): + """ + Init a MixedOp instance for each mutable i.e., LayerChoice. + And register the instantiated MixedOp in corresponding LayerChoice. + If does not register it in LayerChoice, DataParallel does not work then, + because architecture weights are not included in the DataParallel model. + When MixedOPs are registered, we use ```requires_grad``` to control + whether calculate gradients of architecture weights. + + Parameters + ---------- + model : pytorch model + The model that users want to tune, it includes search space defined with nni nas apis + """ + super(ProxylessNasMutator, self).__init__(model) + self._unused_modules = None + self.mutable_list = [] + for mutable in self.undedup_mutables: + self.mutable_list.append(mutable) + mutable.registered_module = MixedOp(mutable) + + def on_forward_layer_choice(self, mutable, *args, **kwargs): + """ + Callback of layer choice forward. This function defines the forward + logic of the input mutable. So mutable is only interface, its real + implementation is defined in mutator. + + Parameters + ---------- + mutable: LayerChoice + forward logic of this input mutable + args: list of torch.Tensor + inputs of this mutable + kwargs: dict + inputs of this mutable + + Returns + ------- + torch.Tensor + output of this mutable, i.e., LayerChoice + int + index of the chosen op + """ + # FIXME: return mask, to be consistent with other algorithms + idx = mutable.registered_module.active_op_index + return mutable.registered_module(mutable, *args, **kwargs), idx + + def reset_binary_gates(self): + """ + For each LayerChoice, binarize binary weights + based on alpha to only activate one op. + It traverses all the mutables in the model to do this. + """ + for mutable in self.undedup_mutables: + mutable.registered_module.binarize(mutable) + + def set_chosen_op_active(self): + """ + For each LayerChoice, set the op with highest alpha as the chosen op. + Usually used for validation. + """ + for mutable in self.undedup_mutables: + mutable.registered_module.set_chosen_op_active() + + def num_arch_params(self): + """ + The number of mutables, i.e., LayerChoice + + Returns + ------- + int + the number of LayerChoice in user model + """ + return len(self.mutable_list) + + def set_arch_param_grad(self): + """ + For each LayerChoice, calculate gradients for architecture weights, i.e., alpha + """ + for mutable in self.undedup_mutables: + mutable.registered_module.set_arch_param_grad(mutable) + + def get_architecture_parameters(self): + """ + Get all the architecture parameters. + + yield + ----- + PyTorch Parameter + Return ap_path_alpha of the traversed mutable + """ + for mutable in self.undedup_mutables: + yield mutable.registered_module.get_ap_path_alpha() + + def change_forward_mode(self, mode): + """ + Update forward mode of MixedOps, as training architecture weights and + model weights use different forward modes. + """ + MixedOp.forward_mode = mode + + def get_forward_mode(self): + """ + Get forward mode of MixedOp + + Returns + ------- + string + the current forward mode of MixedOp + """ + return MixedOp.forward_mode + + def rescale_updated_arch_param(self): + """ + Rescale architecture weights in 'two' mode. + """ + for mutable in self.undedup_mutables: + mutable.registered_module.rescale_updated_arch_param() + + def unused_modules_off(self): + """ + Remove unused modules for each mutables. + The removed modules are kept in ```self._unused_modules``` for resume later. + """ + self._unused_modules = [] + for mutable in self.undedup_mutables: + mixed_op = mutable.registered_module + unused = {} + if self.get_forward_mode() in ['full', 'two', 'full_v2']: + involved_index = mixed_op.active_index + mixed_op.inactive_index + else: + involved_index = mixed_op.active_index + for i in range(mixed_op.n_choices): + if i not in involved_index: + unused[i] = mutable[i] + mutable[i] = None + self._unused_modules.append(unused) + + def unused_modules_back(self): + """ + Resume the removed modules back. + """ + if self._unused_modules is None: + return + for m, unused in zip(self.mutable_list, self._unused_modules): + for i in unused: + m[i] = unused[i] + self._unused_modules = None + + def arch_requires_grad(self): + """ + Make architecture weights require gradient + """ + for mutable in self.undedup_mutables: + mutable.registered_module.to_requires_grad() + + def arch_disable_grad(self): + """ + Disable gradient of architecture weights, i.e., does not + calcuate gradient for them. + """ + for mutable in self.undedup_mutables: + mutable.registered_module.to_disable_grad() + + def sample_final(self): + """ + Generate the final chosen architecture. + + Returns + ------- + dict + the choice of each mutable, i.e., LayerChoice + """ + result = dict() + for mutable in self.undedup_mutables: + assert isinstance(mutable, LayerChoice) + index, _ = mutable.registered_module.chosen_index + # pylint: disable=not-callable + result[mutable.key] = F.one_hot(torch.tensor(index), num_classes=len(mutable)).view(-1).bool() + return result diff --git a/new_impl/cv/third_party/nni/algorithms/nas/pytorch/proxylessnas/trainer.py b/new_impl/cv/third_party/nni/algorithms/nas/pytorch/proxylessnas/trainer.py new file mode 100644 index 0000000000000000000000000000000000000000..d9c86a6a9f098792a4731db32dd140ce3708ea8f --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/nas/pytorch/proxylessnas/trainer.py @@ -0,0 +1,500 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import math +import time +import json +import logging + +import torch +from torch import nn as nn + +from nni.nas.pytorch.base_trainer import BaseTrainer +from nni.nas.pytorch.trainer import TorchTensorEncoder +from nni.nas.pytorch.utils import AverageMeter +from .mutator import ProxylessNasMutator +from .utils import cross_entropy_with_label_smoothing, accuracy + +logger = logging.getLogger(__name__) + +class ProxylessNasTrainer(BaseTrainer): + def __init__(self, model, model_optim, device, + train_loader, valid_loader, label_smoothing=0.1, + n_epochs=120, init_lr=0.025, binary_mode='full_v2', + arch_init_type='normal', arch_init_ratio=1e-3, + arch_optim_lr=1e-3, arch_weight_decay=0, + grad_update_arch_param_every=5, grad_update_steps=1, + warmup=True, warmup_epochs=25, + arch_valid_frequency=1, + load_ckpt=False, ckpt_path=None, arch_path=None): + """ + Parameters + ---------- + model : pytorch model + the user model, which has mutables + model_optim : pytorch optimizer + the user defined optimizer + device : pytorch device + the devices to train/search the model + train_loader : pytorch data loader + data loader for the training set + valid_loader : pytorch data loader + data loader for the validation set + label_smoothing : float + for label smoothing + n_epochs : int + number of epochs to train/search + init_lr : float + init learning rate for training the model + binary_mode : str + the forward/backward mode for the binary weights in mutator + arch_init_type : str + the way to init architecture parameters + arch_init_ratio : float + the ratio to init architecture parameters + arch_optim_lr : float + learning rate of the architecture parameters optimizer + arch_weight_decay : float + weight decay of the architecture parameters optimizer + grad_update_arch_param_every : int + update architecture weights every this number of minibatches + grad_update_steps : int + during each update of architecture weights, the number of steps to train + warmup : bool + whether to do warmup + warmup_epochs : int + the number of epochs to do during warmup + arch_valid_frequency : int + frequency of printing validation result + load_ckpt : bool + whether load checkpoint + ckpt_path : str + checkpoint path, if load_ckpt is True, ckpt_path cannot be None + arch_path : str + the path to store chosen architecture + """ + self.model = model + self.model_optim = model_optim + self.train_loader = train_loader + self.valid_loader = valid_loader + self.device = device + self.n_epochs = n_epochs + self.init_lr = init_lr + self.warmup = warmup + self.warmup_epochs = warmup_epochs + self.arch_valid_frequency = arch_valid_frequency + self.label_smoothing = label_smoothing + + self.train_batch_size = train_loader.batch_sampler.batch_size + self.valid_batch_size = valid_loader.batch_sampler.batch_size + # update architecture parameters every this number of minibatches + self.grad_update_arch_param_every = grad_update_arch_param_every + # the number of steps per architecture parameter update + self.grad_update_steps = grad_update_steps + self.binary_mode = binary_mode + + self.load_ckpt = load_ckpt + self.ckpt_path = ckpt_path + self.arch_path = arch_path + + # init mutator + self.mutator = ProxylessNasMutator(model) + + # DataParallel should be put behind the init of mutator + self.model = torch.nn.DataParallel(self.model) + self.model.to(self.device) + + # iter of valid dataset for training architecture weights + self._valid_iter = None + # init architecture weights + self._init_arch_params(arch_init_type, arch_init_ratio) + # build architecture optimizer + self.arch_optimizer = torch.optim.Adam(self.mutator.get_architecture_parameters(), + arch_optim_lr, + weight_decay=arch_weight_decay, + betas=(0, 0.999), + eps=1e-8) + + self.criterion = nn.CrossEntropyLoss() + self.warmup_curr_epoch = 0 + self.train_curr_epoch = 0 + + def _init_arch_params(self, init_type='normal', init_ratio=1e-3): + """ + Initialize architecture weights + """ + for param in self.mutator.get_architecture_parameters(): + if init_type == 'normal': + param.data.normal_(0, init_ratio) + elif init_type == 'uniform': + param.data.uniform_(-init_ratio, init_ratio) + else: + raise NotImplementedError + + def _validate(self): + """ + Do validation. During validation, LayerChoices use the chosen active op. + + Returns + ------- + float, float, float + average loss, average top1 accuracy, average top5 accuracy + """ + self.valid_loader.batch_sampler.batch_size = self.valid_batch_size + self.valid_loader.batch_sampler.drop_last = False + + self.mutator.set_chosen_op_active() + # remove unused modules to save memory + self.mutator.unused_modules_off() + # test on validation set under train mode + self.model.train() + batch_time = AverageMeter('batch_time') + losses = AverageMeter('losses') + top1 = AverageMeter('top1') + top5 = AverageMeter('top5') + end = time.time() + with torch.no_grad(): + for i, (images, labels) in enumerate(self.valid_loader): + images, labels = images.to(self.device), labels.to(self.device) + output = self.model(images) + loss = self.criterion(output, labels) + acc1, acc5 = accuracy(output, labels, topk=(1, 5)) + losses.update(loss, images.size(0)) + top1.update(acc1[0], images.size(0)) + top5.update(acc5[0], images.size(0)) + # measure elapsed time + batch_time.update(time.time() - end) + end = time.time() + + if i % 10 == 0 or i + 1 == len(self.valid_loader): + test_log = 'Valid' + ': [{0}/{1}]\t'\ + 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'\ + 'Loss {loss.val:.4f} ({loss.avg:.4f})\t'\ + 'Top-1 acc {top1.val:.3f} ({top1.avg:.3f})'.\ + format(i, len(self.valid_loader) - 1, batch_time=batch_time, loss=losses, top1=top1) + # return top5: + test_log += '\tTop-5 acc {top5.val:.3f} ({top5.avg:.3f})'.format(top5=top5) + logger.info(test_log) + self.mutator.unused_modules_back() + return losses.avg, top1.avg, top5.avg + + def _warm_up(self): + """ + Warm up the model, during warm up, architecture weights are not trained. + """ + lr_max = 0.05 + data_loader = self.train_loader + nBatch = len(data_loader) + T_total = self.warmup_epochs * nBatch # total num of batches + + for epoch in range(self.warmup_curr_epoch, self.warmup_epochs): + logger.info('\n--------Warmup epoch: %d--------\n', epoch + 1) + batch_time = AverageMeter('batch_time') + data_time = AverageMeter('data_time') + losses = AverageMeter('losses') + top1 = AverageMeter('top1') + top5 = AverageMeter('top5') + # switch to train mode + self.model.train() + + end = time.time() + logger.info('warm_up epoch: %d', epoch) + for i, (images, labels) in enumerate(data_loader): + data_time.update(time.time() - end) + # lr + T_cur = epoch * nBatch + i + warmup_lr = 0.5 * lr_max * (1 + math.cos(math.pi * T_cur / T_total)) + for param_group in self.model_optim.param_groups: + param_group['lr'] = warmup_lr + images, labels = images.to(self.device), labels.to(self.device) + # compute output + self.mutator.reset_binary_gates() # random sample binary gates + self.mutator.unused_modules_off() # remove unused module for speedup + output = self.model(images) + if self.label_smoothing > 0: + loss = cross_entropy_with_label_smoothing(output, labels, self.label_smoothing) + else: + loss = self.criterion(output, labels) + # measure accuracy and record loss + acc1, acc5 = accuracy(output, labels, topk=(1, 5)) + losses.update(loss, images.size(0)) + top1.update(acc1[0], images.size(0)) + top5.update(acc5[0], images.size(0)) + # compute gradient and do SGD step + self.model.zero_grad() + loss.backward() + self.model_optim.step() + # unused modules back + self.mutator.unused_modules_back() + # measure elapsed time + batch_time.update(time.time() - end) + end = time.time() + + if i % 10 == 0 or i + 1 == nBatch: + batch_log = 'Warmup Train [{0}][{1}/{2}]\t' \ + 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' \ + 'Data {data_time.val:.3f} ({data_time.avg:.3f})\t' \ + 'Loss {losses.val:.4f} ({losses.avg:.4f})\t' \ + 'Top-1 acc {top1.val:.3f} ({top1.avg:.3f})\t' \ + 'Top-5 acc {top5.val:.3f} ({top5.avg:.3f})\tlr {lr:.5f}'. \ + format(epoch + 1, i, nBatch - 1, batch_time=batch_time, data_time=data_time, + losses=losses, top1=top1, top5=top5, lr=warmup_lr) + logger.info(batch_log) + val_loss, val_top1, val_top5 = self._validate() + val_log = 'Warmup Valid [{0}/{1}]\tloss {2:.3f}\ttop-1 acc {3:.3f}\ttop-5 acc {4:.3f}\t' \ + 'Train top-1 {top1.avg:.3f}\ttop-5 {top5.avg:.3f}M'. \ + format(epoch + 1, self.warmup_epochs, val_loss, val_top1, val_top5, top1=top1, top5=top5) + logger.info(val_log) + self.save_checkpoint() + self.warmup_curr_epoch += 1 + + def _get_update_schedule(self, nBatch): + """ + Generate schedule for training architecture weights. Key means after which minibatch + to update architecture weights, value means how many steps for the update. + + Parameters + ---------- + nBatch : int + the total number of minibatches in one epoch + + Returns + ------- + dict + the schedule for updating architecture weights + """ + schedule = {} + for i in range(nBatch): + if (i + 1) % self.grad_update_arch_param_every == 0: + schedule[i] = self.grad_update_steps + return schedule + + def _calc_learning_rate(self, epoch, batch=0, nBatch=None): + """ + Update learning rate. + """ + T_total = self.n_epochs * nBatch + T_cur = epoch * nBatch + batch + lr = 0.5 * self.init_lr * (1 + math.cos(math.pi * T_cur / T_total)) + return lr + + def _adjust_learning_rate(self, optimizer, epoch, batch=0, nBatch=None): + """ + Adjust learning of a given optimizer and return the new learning rate + + Parameters + ---------- + optimizer : pytorch optimizer + the used optimizer + epoch : int + the current epoch number + batch : int + the current minibatch + nBatch : int + the total number of minibatches in one epoch + + Returns + ------- + float + the adjusted learning rate + """ + new_lr = self._calc_learning_rate(epoch, batch, nBatch) + for param_group in optimizer.param_groups: + param_group['lr'] = new_lr + return new_lr + + def _train(self): + """ + Train the model, it trains model weights and architecute weights. + Architecture weights are trained according to the schedule. + Before updating architecture weights, ```requires_grad``` is enabled. + Then, it is disabled after the updating, in order not to update + architecture weights when training model weights. + """ + nBatch = len(self.train_loader) + arch_param_num = self.mutator.num_arch_params() + binary_gates_num = self.mutator.num_arch_params() + logger.info('#arch_params: %d\t#binary_gates: %d', arch_param_num, binary_gates_num) + + update_schedule = self._get_update_schedule(nBatch) + + for epoch in range(self.train_curr_epoch, self.n_epochs): + logger.info('\n--------Train epoch: %d--------\n', epoch + 1) + batch_time = AverageMeter('batch_time') + data_time = AverageMeter('data_time') + losses = AverageMeter('losses') + top1 = AverageMeter('top1') + top5 = AverageMeter('top5') + # switch to train mode + self.model.train() + + end = time.time() + for i, (images, labels) in enumerate(self.train_loader): + data_time.update(time.time() - end) + lr = self._adjust_learning_rate(self.model_optim, epoch, batch=i, nBatch=nBatch) + # train weight parameters + images, labels = images.to(self.device), labels.to(self.device) + self.mutator.reset_binary_gates() + self.mutator.unused_modules_off() + output = self.model(images) + if self.label_smoothing > 0: + loss = cross_entropy_with_label_smoothing(output, labels, self.label_smoothing) + else: + loss = self.criterion(output, labels) + acc1, acc5 = accuracy(output, labels, topk=(1, 5)) + losses.update(loss, images.size(0)) + top1.update(acc1[0], images.size(0)) + top5.update(acc5[0], images.size(0)) + self.model.zero_grad() + loss.backward() + self.model_optim.step() + self.mutator.unused_modules_back() + if epoch > 0: + for _ in range(update_schedule.get(i, 0)): + start_time = time.time() + # GradientArchSearchConfig + self.mutator.arch_requires_grad() + arch_loss, exp_value = self._gradient_step() + self.mutator.arch_disable_grad() + used_time = time.time() - start_time + log_str = 'Architecture [%d-%d]\t Time %.4f\t Loss %.4f\t null %s' % \ + (epoch + 1, i, used_time, arch_loss, exp_value) + logger.info(log_str) + batch_time.update(time.time() - end) + end = time.time() + # training log + if i % 10 == 0 or i + 1 == nBatch: + batch_log = 'Train [{0}][{1}/{2}]\t' \ + 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' \ + 'Data Time {data_time.val:.3f} ({data_time.avg:.3f})\t' \ + 'Loss {losses.val:.4f} ({losses.avg:.4f})\t' \ + 'Top-1 acc {top1.val:.3f} ({top1.avg:.3f})\t' \ + 'Top-5 acc {top5.val:.3f} ({top5.avg:.3f})\tlr {lr:.5f}'. \ + format(epoch + 1, i, nBatch - 1, batch_time=batch_time, data_time=data_time, + losses=losses, top1=top1, top5=top5, lr=lr) + logger.info(batch_log) + # validate + if (epoch + 1) % self.arch_valid_frequency == 0: + val_loss, val_top1, val_top5 = self._validate() + val_log = 'Valid [{0}]\tloss {1:.3f}\ttop-1 acc {2:.3f} \ttop-5 acc {3:.3f}\t' \ + 'Train top-1 {top1.avg:.3f}\ttop-5 {top5.avg:.3f}'. \ + format(epoch + 1, val_loss, val_top1, val_top5, top1=top1, top5=top5) + logger.info(val_log) + self.save_checkpoint() + self.train_curr_epoch += 1 + + def _valid_next_batch(self): + """ + Get next one minibatch from validation set + + Returns + ------- + (tensor, tensor) + the tuple of images and labels + """ + if self._valid_iter is None: + self._valid_iter = iter(self.valid_loader) + try: + data = next(self._valid_iter) + except StopIteration: + self._valid_iter = iter(self.valid_loader) + data = next(self._valid_iter) + return data + + def _gradient_step(self): + """ + This gradient step is for updating architecture weights. + Mutator is intensively used in this function to operate on + architecture weights. + + Returns + ------- + float, None + loss of the model, None + """ + # use the same batch size as train batch size for architecture weights + self.valid_loader.batch_sampler.batch_size = self.train_batch_size + self.valid_loader.batch_sampler.drop_last = True + self.model.train() + self.mutator.change_forward_mode(self.binary_mode) + time1 = time.time() # time + # sample a batch of data from validation set + images, labels = self._valid_next_batch() + images, labels = images.to(self.device), labels.to(self.device) + time2 = time.time() # time + self.mutator.reset_binary_gates() + self.mutator.unused_modules_off() + output = self.model(images) + time3 = time.time() + ce_loss = self.criterion(output, labels) + expected_value = None + loss = ce_loss + self.model.zero_grad() + loss.backward() + self.mutator.set_arch_param_grad() + self.arch_optimizer.step() + if self.mutator.get_forward_mode() == 'two': + self.mutator.rescale_updated_arch_param() + self.mutator.unused_modules_back() + self.mutator.change_forward_mode(None) + time4 = time.time() + logger.info('(%.4f, %.4f, %.4f)', time2 - time1, time3 - time2, time4 - time3) + return loss.data.item(), expected_value.item() if expected_value is not None else None + + def save_checkpoint(self): + """ + Save checkpoint of the whole model. Saving model weights and architecture weights in + ```ckpt_path```, and saving currently chosen architecture in ```arch_path```. + """ + if self.ckpt_path: + state = { + 'warmup_curr_epoch': self.warmup_curr_epoch, + 'train_curr_epoch': self.train_curr_epoch, + 'model': self.model.state_dict(), + 'optim': self.model_optim.state_dict(), + 'arch_optim': self.arch_optimizer.state_dict() + } + torch.save(state, self.ckpt_path) + if self.arch_path: + self.export(self.arch_path) + + def load_checkpoint(self): + """ + Load the checkpoint from ```ckpt_path```. + """ + assert self.ckpt_path is not None, "If load_ckpt is not None, ckpt_path should not be None" + ckpt = torch.load(self.ckpt_path) + self.warmup_curr_epoch = ckpt['warmup_curr_epoch'] + self.train_curr_epoch = ckpt['train_curr_epoch'] + self.model.load_state_dict(ckpt['model']) + self.model_optim.load_state_dict(ckpt['optim']) + self.arch_optimizer.load_state_dict(ckpt['arch_optim']) + + def train(self): + """ + Train the whole model. + """ + if self.load_ckpt: + self.load_checkpoint() + if self.warmup: + self._warm_up() + self._train() + + def export(self, file_name): + """ + Export the chosen architecture into a file + + Parameters + ---------- + file_name : str + the file that stores exported chosen architecture + """ + exported_arch = self.mutator.sample_final() + with open(file_name, 'w') as f: + json.dump(exported_arch, f, indent=2, sort_keys=True, cls=TorchTensorEncoder) + + def validate(self): + raise NotImplementedError + + def checkpoint(self): + raise NotImplementedError diff --git a/new_impl/cv/third_party/nni/algorithms/nas/pytorch/proxylessnas/utils.py b/new_impl/cv/third_party/nni/algorithms/nas/pytorch/proxylessnas/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..b703810d3b703a33bd6bbe8422c557a1f669146b --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/nas/pytorch/proxylessnas/utils.py @@ -0,0 +1,78 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import torch +import torch.nn as nn + +def detach_variable(inputs): + """ + Detach variables + + Parameters + ---------- + inputs : pytorch tensors + pytorch tensors + """ + if isinstance(inputs, tuple): + return tuple([detach_variable(x) for x in inputs]) + else: + x = inputs.detach() + x.requires_grad = inputs.requires_grad + return x + +def cross_entropy_with_label_smoothing(pred, target, label_smoothing=0.1): + """ + Parameters + ---------- + pred : pytorch tensor + predicted value + target : pytorch tensor + label + label_smoothing : float + the degree of label smoothing + + Returns + ------- + pytorch tensor + cross entropy + """ + logsoftmax = nn.LogSoftmax() + n_classes = pred.size(1) + # convert to one-hot + target = torch.unsqueeze(target, 1) + soft_target = torch.zeros_like(pred) + soft_target.scatter_(1, target, 1) + # label smoothing + soft_target = soft_target * (1 - label_smoothing) + label_smoothing / n_classes + return torch.mean(torch.sum(- soft_target * logsoftmax(pred), 1)) + +def accuracy(output, target, topk=(1,)): + """ + Computes the precision@k for the specified values of k + + Parameters + ---------- + output : pytorch tensor + output, e.g., predicted value + target : pytorch tensor + label + topk : tuple + specify top1 and top5 + + Returns + ------- + list + accuracy of top1 and top5 + """ + maxk = max(topk) + batch_size = target.size(0) + + _, pred = output.topk(maxk, 1, True, True) + pred = pred.t() + correct = pred.eq(target.view(1, -1).expand_as(pred)) + + res = [] + for k in topk: + correct_k = correct[:k].view(-1).float().sum(0, keepdim=True) + res.append(correct_k.mul_(100.0 / batch_size)) + return res diff --git a/new_impl/cv/third_party/nni/algorithms/nas/pytorch/random/__init__.py b/new_impl/cv/third_party/nni/algorithms/nas/pytorch/random/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..8a52b8e030549b08ed636c47559c4a144c337446 --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/nas/pytorch/random/__init__.py @@ -0,0 +1 @@ +from .mutator import RandomMutator \ No newline at end of file diff --git a/new_impl/cv/third_party/nni/algorithms/nas/pytorch/random/mutator.py b/new_impl/cv/third_party/nni/algorithms/nas/pytorch/random/mutator.py new file mode 100644 index 0000000000000000000000000000000000000000..f302db56c0cdbd611f648342cc26760d344958c4 --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/nas/pytorch/random/mutator.py @@ -0,0 +1,36 @@ +import torch +import torch.nn.functional as F + +from nni.nas.pytorch.mutator import Mutator +from nni.nas.pytorch.mutables import LayerChoice, InputChoice + + +class RandomMutator(Mutator): + """ + Random mutator that samples a random candidate in the search space each time ``reset()``. + It uses random function in PyTorch, so users can set seed in PyTorch to ensure deterministic behavior. + """ + + def sample_search(self): + """ + Sample a random candidate. + """ + result = dict() + for mutable in self.mutables: + if isinstance(mutable, LayerChoice): + gen_index = torch.randint(high=len(mutable), size=(1, )) + result[mutable.key] = F.one_hot(gen_index, num_classes=len(mutable)).view(-1).bool() + elif isinstance(mutable, InputChoice): + if mutable.n_chosen is None: + result[mutable.key] = torch.randint(high=2, size=(mutable.n_candidates,)).view(-1).bool() + else: + perm = torch.randperm(mutable.n_candidates) + mask = [i in perm[:mutable.n_chosen] for i in range(mutable.n_candidates)] + result[mutable.key] = torch.tensor(mask, dtype=torch.bool) # pylint: disable=not-callable + return result + + def sample_final(self): + """ + Same as :meth:`sample_search`. + """ + return self.sample_search() diff --git a/new_impl/cv/third_party/nni/algorithms/nas/pytorch/spos/__init__.py b/new_impl/cv/third_party/nni/algorithms/nas/pytorch/spos/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..ed432b0845154c4745aa82c8e6a8bad4290237aa --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/nas/pytorch/spos/__init__.py @@ -0,0 +1,6 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .evolution import SPOSEvolution +from .mutator import SPOSSupernetTrainingMutator +from .trainer import SPOSSupernetTrainer diff --git a/new_impl/cv/third_party/nni/algorithms/nas/pytorch/spos/evolution.py b/new_impl/cv/third_party/nni/algorithms/nas/pytorch/spos/evolution.py new file mode 100644 index 0000000000000000000000000000000000000000..bd099e276ebfd4aa68215dbbd25e6443a45b9dd1 --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/nas/pytorch/spos/evolution.py @@ -0,0 +1,223 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import json +import logging +import os +import re +from collections import deque + +import numpy as np +from nni.tuner import Tuner +from nni.algorithms.nas.pytorch.classic_nas.mutator import LAYER_CHOICE, INPUT_CHOICE + + +_logger = logging.getLogger(__name__) + + +class SPOSEvolution(Tuner): + """ + SPOS evolution tuner. + + Parameters + ---------- + max_epochs : int + Maximum number of epochs to run. + num_select : int + Number of survival candidates of each epoch. + num_population : int + Number of candidates at the start of each epoch. If candidates generated by + crossover and mutation are not enough, the rest will be filled with random + candidates. + m_prob : float + The probability of mutation. + num_crossover : int + Number of candidates generated by crossover in each epoch. + num_mutation : int + Number of candidates generated by mutation in each epoch. + """ + + def __init__(self, max_epochs=20, num_select=10, num_population=50, m_prob=0.1, + num_crossover=25, num_mutation=25): + assert num_population >= num_select + self.max_epochs = max_epochs + self.num_select = num_select + self.num_population = num_population + self.m_prob = m_prob + self.num_crossover = num_crossover + self.num_mutation = num_mutation + self.epoch = 0 + self.candidates = [] + self.search_space = None + self.random_state = np.random.RandomState(0) + + # async status + self._to_evaluate_queue = deque() + self._sending_parameter_queue = deque() + self._pending_result_ids = set() + self._reward_dict = dict() + self._id2candidate = dict() + self._st_callback = None + + def update_search_space(self, search_space): + """ + Handle the initialization/update event of search space. + """ + self._search_space = search_space + self._next_round() + + def _next_round(self): + _logger.info("Epoch %d, generating...", self.epoch) + if self.epoch == 0: + self._get_random_population() + self.export_results(self.candidates) + else: + best_candidates = self._select_top_candidates() + self.export_results(best_candidates) + if self.epoch >= self.max_epochs: + return + self.candidates = self._get_mutation(best_candidates) + self._get_crossover(best_candidates) + self._get_random_population() + self.epoch += 1 + + def _random_candidate(self): + chosen_arch = dict() + for key, val in self._search_space.items(): + if val["_type"] == LAYER_CHOICE: + choices = val["_value"] + index = self.random_state.randint(len(choices)) + chosen_arch[key] = {"_value": choices[index], "_idx": index} + elif val["_type"] == INPUT_CHOICE: + raise NotImplementedError("Input choice is not implemented yet.") + return chosen_arch + + def _add_to_evaluate_queue(self, cand): + _logger.info("Generate candidate %s, adding to eval queue.", self._get_architecture_repr(cand)) + self._reward_dict[self._hashcode(cand)] = 0. + self._to_evaluate_queue.append(cand) + + def _get_random_population(self): + while len(self.candidates) < self.num_population: + cand = self._random_candidate() + if self._is_legal(cand): + _logger.info("Random candidate generated.") + self._add_to_evaluate_queue(cand) + self.candidates.append(cand) + + def _get_crossover(self, best): + result = [] + for _ in range(10 * self.num_crossover): + cand_p1 = best[self.random_state.randint(len(best))] + cand_p2 = best[self.random_state.randint(len(best))] + assert cand_p1.keys() == cand_p2.keys() + cand = {k: cand_p1[k] if self.random_state.randint(2) == 0 else cand_p2[k] + for k in cand_p1.keys()} + if self._is_legal(cand): + result.append(cand) + self._add_to_evaluate_queue(cand) + if len(result) >= self.num_crossover: + break + _logger.info("Found %d architectures with crossover.", len(result)) + return result + + def _get_mutation(self, best): + result = [] + for _ in range(10 * self.num_mutation): + cand = best[self.random_state.randint(len(best))].copy() + mutation_sample = np.random.random_sample(len(cand)) + for s, k in zip(mutation_sample, cand): + if s < self.m_prob: + choices = self._search_space[k]["_value"] + index = self.random_state.randint(len(choices)) + cand[k] = {"_value": choices[index], "_idx": index} + if self._is_legal(cand): + result.append(cand) + self._add_to_evaluate_queue(cand) + if len(result) >= self.num_mutation: + break + _logger.info("Found %d architectures with mutation.", len(result)) + return result + + def _get_architecture_repr(self, cand): + return re.sub(r"\".*?\": \{\"_idx\": (\d+), \"_value\": \".*?\"\}", r"\1", + self._hashcode(cand)) + + def _is_legal(self, cand): + if self._hashcode(cand) in self._reward_dict: + return False + return True + + def _select_top_candidates(self): + reward_query = lambda cand: self._reward_dict[self._hashcode(cand)] + _logger.info("All candidate rewards: %s", list(map(reward_query, self.candidates))) + result = sorted(self.candidates, key=reward_query, reverse=True)[:self.num_select] + _logger.info("Best candidate rewards: %s", list(map(reward_query, result))) + return result + + @staticmethod + def _hashcode(d): + return json.dumps(d, sort_keys=True) + + def _bind_and_send_parameters(self): + """ + There are two types of resources: parameter ids and candidates. This function is called at + necessary times to bind these resources to send new trials with st_callback. + """ + result = [] + while self._sending_parameter_queue and self._to_evaluate_queue: + parameter_id = self._sending_parameter_queue.popleft() + parameters = self._to_evaluate_queue.popleft() + self._id2candidate[parameter_id] = parameters + result.append(parameters) + self._pending_result_ids.add(parameter_id) + self._st_callback(parameter_id, parameters) + _logger.info("Send parameter [%d] %s.", parameter_id, self._get_architecture_repr(parameters)) + return result + + def generate_multiple_parameters(self, parameter_id_list, **kwargs): + """ + Callback function necessary to implement a tuner. This will put more parameter ids into the + parameter id queue. + """ + if "st_callback" in kwargs and self._st_callback is None: + self._st_callback = kwargs["st_callback"] + for parameter_id in parameter_id_list: + self._sending_parameter_queue.append(parameter_id) + self._bind_and_send_parameters() + return [] # always not use this. might induce problem of over-sending + + def receive_trial_result(self, parameter_id, parameters, value, **kwargs): + """ + Callback function. Receive a trial result. + """ + _logger.info("Candidate %d, reported reward %f", parameter_id, value) + self._reward_dict[self._hashcode(self._id2candidate[parameter_id])] = value + + def trial_end(self, parameter_id, success, **kwargs): + """ + Callback function when a trial is ended and resource is released. + """ + self._pending_result_ids.remove(parameter_id) + if not self._pending_result_ids and not self._to_evaluate_queue: + # a new epoch now + self._next_round() + assert self._st_callback is not None + self._bind_and_send_parameters() + + def export_results(self, result): + """ + Export a number of candidates to `checkpoints` dir. + + Parameters + ---------- + result : dict + Chosen architectures to be exported. + """ + os.makedirs("checkpoints", exist_ok=True) + for i, cand in enumerate(result): + converted = dict() + for cand_key, cand_val in cand.items(): + onehot = [k == cand_val["_idx"] for k in range(len(self._search_space[cand_key]["_value"]))] + converted[cand_key] = onehot + with open(os.path.join("checkpoints", "%03d_%03d.json" % (self.epoch, i)), "w") as fp: + json.dump(converted, fp) diff --git a/new_impl/cv/third_party/nni/algorithms/nas/pytorch/spos/mutator.py b/new_impl/cv/third_party/nni/algorithms/nas/pytorch/spos/mutator.py new file mode 100644 index 0000000000000000000000000000000000000000..1a803cb2e820b0df2dd8b04b3d387af68d3d2680 --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/nas/pytorch/spos/mutator.py @@ -0,0 +1,66 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging + +import numpy as np +from nni.algorithms.nas.pytorch.random import RandomMutator + +_logger = logging.getLogger(__name__) + + +class SPOSSupernetTrainingMutator(RandomMutator): + """ + A random mutator with flops limit. + + Parameters + ---------- + model : nn.Module + PyTorch model. + flops_func : callable + Callable that takes a candidate from `sample_search` and returns its candidate. When `flops_func` + is None, functions related to flops will be deactivated. + flops_lb : number + Lower bound of flops. + flops_ub : number + Upper bound of flops. + flops_bin_num : number + Number of bins divided for the interval of flops to ensure the uniformity. Bigger number will be more + uniform, but the sampling will be slower. + flops_sample_timeout : int + Maximum number of attempts to sample before giving up and use a random candidate. + """ + def __init__(self, model, flops_func=None, flops_lb=None, flops_ub=None, + flops_bin_num=7, flops_sample_timeout=500): + + super().__init__(model) + self._flops_func = flops_func + if self._flops_func is not None: + self._flops_bin_num = flops_bin_num + self._flops_bins = [flops_lb + (flops_ub - flops_lb) / flops_bin_num * i for i in range(flops_bin_num + 1)] + self._flops_sample_timeout = flops_sample_timeout + + def sample_search(self): + """ + Sample a candidate for training. When `flops_func` is not None, candidates will be sampled uniformly + relative to flops. + + Returns + ------- + dict + """ + if self._flops_func is not None: + for times in range(self._flops_sample_timeout): + idx = np.random.randint(self._flops_bin_num) + cand = super().sample_search() + if self._flops_bins[idx] <= self._flops_func(cand) <= self._flops_bins[idx + 1]: + _logger.debug("Sampled candidate flops %f in %d times.", cand, times) + return cand + _logger.warning("Failed to sample a flops-valid candidate within %d tries.", self._flops_sample_timeout) + return super().sample_search() + + def sample_final(self): + """ + Implement only to suffice the interface of Mutator. + """ + return self.sample_search() diff --git a/new_impl/cv/third_party/nni/algorithms/nas/pytorch/spos/trainer.py b/new_impl/cv/third_party/nni/algorithms/nas/pytorch/spos/trainer.py new file mode 100644 index 0000000000000000000000000000000000000000..c0ca933991688de973b4f739558c4c3ac0410ef4 --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/nas/pytorch/spos/trainer.py @@ -0,0 +1,95 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging + +import torch +from nni.nas.pytorch.trainer import Trainer +from nni.nas.pytorch.utils import AverageMeterGroup + +from .mutator import SPOSSupernetTrainingMutator + +logger = logging.getLogger(__name__) + + +class SPOSSupernetTrainer(Trainer): + """ + This trainer trains a supernet that can be used for evolution search. + + Parameters + ---------- + model : nn.Module + Model with mutables. + mutator : Mutator + A mutator object that has been initialized with the model. + loss : callable + Called with logits and targets. Returns a loss tensor. + metrics : callable + Returns a dict that maps metrics keys to metrics data. + optimizer : Optimizer + Optimizer that optimizes the model. + num_epochs : int + Number of epochs of training. + train_loader : iterable + Data loader of training. Raise ``StopIteration`` when one epoch is exhausted. + dataset_valid : iterable + Data loader of validation. Raise ``StopIteration`` when one epoch is exhausted. + batch_size : int + Batch size. + workers: int + Number of threads for data preprocessing. Not used for this trainer. Maybe removed in future. + device : torch.device + Device object. Either ``torch.device("cuda")`` or ``torch.device("cpu")``. When ``None``, trainer will + automatic detects GPU and selects GPU first. + log_frequency : int + Number of mini-batches to log metrics. + callbacks : list of Callback + Callbacks to plug into the trainer. See Callbacks. + """ + + def __init__(self, model, loss, metrics, + optimizer, num_epochs, train_loader, valid_loader, + mutator=None, batch_size=64, workers=4, device=None, log_frequency=None, + callbacks=None): + assert torch.cuda.is_available() + super().__init__(model, mutator if mutator is not None else SPOSSupernetTrainingMutator(model), + loss, metrics, optimizer, num_epochs, None, None, + batch_size, workers, device, log_frequency, callbacks) + + self.train_loader = train_loader + self.valid_loader = valid_loader + + def train_one_epoch(self, epoch): + self.model.train() + meters = AverageMeterGroup() + for step, (x, y) in enumerate(self.train_loader): + x, y = x.to(self.device), y.to(self.device) + self.optimizer.zero_grad() + self.mutator.reset() + logits = self.model(x) + loss = self.loss(logits, y) + loss.backward() + self.optimizer.step() + + metrics = self.metrics(logits, y) + metrics["loss"] = loss.item() + meters.update(metrics) + if self.log_frequency is not None and step % self.log_frequency == 0: + logger.info("Epoch [%s/%s] Step [%s/%s] %s", epoch + 1, + self.num_epochs, step + 1, len(self.train_loader), meters) + + def validate_one_epoch(self, epoch): + self.model.eval() + meters = AverageMeterGroup() + with torch.no_grad(): + for step, (x, y) in enumerate(self.valid_loader): + x, y = x.to(self.device), y.to(self.device) + self.mutator.reset() + logits = self.model(x) + loss = self.loss(logits, y) + metrics = self.metrics(logits, y) + metrics["loss"] = loss.item() + meters.update(metrics) + if self.log_frequency is not None and step % self.log_frequency == 0: + logger.info("Epoch [%s/%s] Validation Step [%s/%s] %s", epoch + 1, + self.num_epochs, step + 1, len(self.valid_loader), meters) diff --git a/new_impl/cv/third_party/nni/algorithms/nas/tensorflow/__init__.py b/new_impl/cv/third_party/nni/algorithms/nas/tensorflow/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/new_impl/cv/third_party/nni/algorithms/nas/tensorflow/classic_nas/__init__.py b/new_impl/cv/third_party/nni/algorithms/nas/tensorflow/classic_nas/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..ec3f5a4894a0460d4a0582a0d6cd43af9bed77e2 --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/nas/tensorflow/classic_nas/__init__.py @@ -0,0 +1,4 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .mutator import get_and_apply_next_architecture diff --git a/new_impl/cv/third_party/nni/algorithms/nas/tensorflow/classic_nas/mutator.py b/new_impl/cv/third_party/nni/algorithms/nas/tensorflow/classic_nas/mutator.py new file mode 100644 index 0000000000000000000000000000000000000000..fad4987fed55bc9a1460089ee0fd3b92a63ee83f --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/nas/tensorflow/classic_nas/mutator.py @@ -0,0 +1,215 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import json +import logging +import os +import sys + +import tensorflow as tf + +import nni +from nni.runtime.env_vars import trial_env_vars +from nni.nas.tensorflow.mutables import LayerChoice, InputChoice, MutableScope +from nni.nas.tensorflow.mutator import Mutator + +logger = logging.getLogger(__name__) + +NNI_GEN_SEARCH_SPACE = "NNI_GEN_SEARCH_SPACE" +LAYER_CHOICE = "layer_choice" +INPUT_CHOICE = "input_choice" + + +def get_and_apply_next_architecture(model): + """ + Wrapper of :class:`~nni.nas.tensorflow.classic_nas.mutator.ClassicMutator` to make it more meaningful, + similar to ``get_next_parameter`` for HPO. + Tt will generate search space based on ``model``. + If env ``NNI_GEN_SEARCH_SPACE`` exists, this is in dry run mode for + generating search space for the experiment. + If not, there are still two mode, one is nni experiment mode where users + use ``nnictl`` to start an experiment. The other is standalone mode + where users directly run the trial command, this mode chooses the first + one(s) for each LayerChoice and InputChoice. + Parameters + ---------- + model : nn.Module + User's model with search space (e.g., LayerChoice, InputChoice) embedded in it. + """ + ClassicMutator(model) + + +class ClassicMutator(Mutator): + """ + This mutator is to apply the architecture chosen from tuner. + It implements the forward function of LayerChoice and InputChoice, + to only activate the chosen ones. + Parameters + ---------- + model : nn.Module + User's model with search space (e.g., LayerChoice, InputChoice) embedded in it. + """ + + def __init__(self, model): + super(ClassicMutator, self).__init__(model) + self._chosen_arch = {} + self._search_space = self._generate_search_space() + if NNI_GEN_SEARCH_SPACE in os.environ: + # dry run for only generating search space + self._dump_search_space(os.environ[NNI_GEN_SEARCH_SPACE]) + sys.exit(0) + + if trial_env_vars.NNI_PLATFORM is None: + logger.warning("This is in standalone mode, the chosen are the first one(s).") + self._chosen_arch = self._standalone_generate_chosen() + else: + # get chosen arch from tuner + self._chosen_arch = nni.get_next_parameter() + if self._chosen_arch is None: + if trial_env_vars.NNI_PLATFORM == "unittest": + # happens if NNI_PLATFORM is intentionally set, e.g., in UT + logger.warning("`NNI_PLATFORM` is set but `param` is None. Falling back to standalone mode.") + self._chosen_arch = self._standalone_generate_chosen() + else: + raise RuntimeError("Chosen architecture is None. This may be a platform error.") + self.reset() + + def _sample_layer_choice(self, mutable, idx, value, search_space_item): + """ + Convert layer choice to tensor representation. + Parameters + ---------- + mutable : Mutable + idx : int + Number `idx` of list will be selected. + value : str + The verbose representation of the selected value. + search_space_item : list + The list for corresponding search space. + """ + # doesn't support multihot for layer choice yet + assert 0 <= idx < len(mutable) and search_space_item[idx] == value, \ + "Index '{}' in search space '{}' is not '{}'".format(idx, search_space_item, value) + mask = tf.one_hot(idx, len(mutable)) + return tf.cast(tf.reshape(mask, [-1]), tf.bool) + + def _sample_input_choice(self, mutable, idx, value, search_space_item): + """ + Convert input choice to tensor representation. + Parameters + ---------- + mutable : Mutable + idx : int + Number `idx` of list will be selected. + value : str + The verbose representation of the selected value. + search_space_item : list + The list for corresponding search space. + """ + candidate_repr = search_space_item["candidates"] + multihot_list = [False] * mutable.n_candidates + for i, v in zip(idx, value): + assert 0 <= i < mutable.n_candidates and candidate_repr[i] == v, \ + "Index '{}' in search space '{}' is not '{}'".format(i, candidate_repr, v) + assert not multihot_list[i], "'{}' is selected twice in '{}', which is not allowed.".format(i, idx) + multihot_list[i] = True + return tf.cast(multihot_list, tf.bool) # pylint: disable=not-callable + + def sample_search(self): + """ + See :meth:`sample_final`. + """ + return self.sample_final() + + def sample_final(self): + """ + Convert the chosen arch and apply it on model. + """ + assert set(self._chosen_arch.keys()) == set(self._search_space.keys()), \ + "Unmatched keys, expected keys '{}' from search space, found '{}'.".format(self._search_space.keys(), + self._chosen_arch.keys()) + result = dict() + for mutable in self.mutables: + if isinstance(mutable, (LayerChoice, InputChoice)): + assert mutable.key in self._chosen_arch, \ + "Expected '{}' in chosen arch, but not found.".format(mutable.key) + data = self._chosen_arch[mutable.key] + assert isinstance(data, dict) and "_value" in data and "_idx" in data, \ + "'{}' is not a valid choice.".format(data) + if isinstance(mutable, LayerChoice): + result[mutable.key] = self._sample_layer_choice(mutable, data["_idx"], data["_value"], + self._search_space[mutable.key]["_value"]) + elif isinstance(mutable, InputChoice): + result[mutable.key] = self._sample_input_choice(mutable, data["_idx"], data["_value"], + self._search_space[mutable.key]["_value"]) + elif isinstance(mutable, MutableScope): + logger.info("Mutable scope '%s' is skipped during parsing choices.", mutable.key) + else: + raise TypeError("Unsupported mutable type: '%s'." % type(mutable)) + return result + + def _standalone_generate_chosen(self): + """ + Generate the chosen architecture for standalone mode, + i.e., choose the first one(s) for LayerChoice and InputChoice. + :: + { key_name: {"_value": "conv1", + "_idx": 0} } + { key_name: {"_value": ["in1"], + "_idx": [0]} } + Returns + ------- + dict + the chosen architecture + """ + chosen_arch = {} + for key, val in self._search_space.items(): + if val["_type"] == LAYER_CHOICE: + choices = val["_value"] + chosen_arch[key] = {"_value": choices[0], "_idx": 0} + elif val["_type"] == INPUT_CHOICE: + choices = val["_value"]["candidates"] + n_chosen = val["_value"]["n_chosen"] + if n_chosen is None: + n_chosen = len(choices) + chosen_arch[key] = {"_value": choices[:n_chosen], "_idx": list(range(n_chosen))} + else: + raise ValueError("Unknown key '%s' and value '%s'." % (key, val)) + return chosen_arch + + def _generate_search_space(self): + """ + Generate search space from mutables. + Here is the search space format: + :: + { key_name: {"_type": "layer_choice", + "_value": ["conv1", "conv2"]} } + { key_name: {"_type": "input_choice", + "_value": {"candidates": ["in1", "in2"], + "n_chosen": 1}} } + Returns + ------- + dict + the generated search space + """ + search_space = {} + for mutable in self.mutables: + # for now we only generate flattened search space + if isinstance(mutable, LayerChoice): + key = mutable.key + val = mutable.names + search_space[key] = {"_type": LAYER_CHOICE, "_value": val} + elif isinstance(mutable, InputChoice): + key = mutable.key + search_space[key] = {"_type": INPUT_CHOICE, + "_value": {"candidates": mutable.choose_from, + "n_chosen": mutable.n_chosen}} + elif isinstance(mutable, MutableScope): + logger.info("Mutable scope '%s' is skipped during generating search space.", mutable.key) + else: + raise TypeError("Unsupported mutable type: '%s'." % type(mutable)) + return search_space + + def _dump_search_space(self, file_path): + with open(file_path, "w") as ss_file: + json.dump(self._search_space, ss_file, sort_keys=True, indent=2) diff --git a/new_impl/cv/third_party/nni/algorithms/nas/tensorflow/enas/__init__.py b/new_impl/cv/third_party/nni/algorithms/nas/tensorflow/enas/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d3372836ebba2387ade58161218aad731433e46b --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/nas/tensorflow/enas/__init__.py @@ -0,0 +1,5 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .mutator import EnasMutator +from .trainer import EnasTrainer diff --git a/new_impl/cv/third_party/nni/algorithms/nas/tensorflow/enas/mutator.py b/new_impl/cv/third_party/nni/algorithms/nas/tensorflow/enas/mutator.py new file mode 100644 index 0000000000000000000000000000000000000000..de43195fa2e6eef6b8c1991cf8d56998d7abc4fb --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/nas/tensorflow/enas/mutator.py @@ -0,0 +1,160 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import tensorflow as tf +from tensorflow.keras.layers import Dense, Embedding, LSTMCell, RNN +from tensorflow.keras.losses import SparseCategoricalCrossentropy, Reduction + +from nni.nas.tensorflow.mutator import Mutator +from nni.nas.tensorflow.mutables import LayerChoice, InputChoice, MutableScope + + +class EnasMutator(Mutator): + def __init__(self, model, + lstm_size=64, + lstm_num_layers=1, + tanh_constant=1.5, + cell_exit_extra_step=False, + skip_target=0.4, + temperature=None, + branch_bias=0.25, + entropy_reduction='sum'): + super().__init__(model) + self.tanh_constant = tanh_constant + self.temperature = temperature + self.cell_exit_extra_step = cell_exit_extra_step + + cells = [LSTMCell(units=lstm_size, use_bias=False) for _ in range(lstm_num_layers)] + self.lstm = RNN(cells, stateful=True) + self.g_emb = tf.random.normal((1, 1, lstm_size)) * 0.1 + self.skip_targets = tf.constant([1.0 - skip_target, skip_target]) + + self.max_layer_choice = 0 + self.bias_dict = {} + for mutable in self.mutables: + if isinstance(mutable, LayerChoice): + if self.max_layer_choice == 0: + self.max_layer_choice = len(mutable) + assert self.max_layer_choice == len(mutable), \ + "ENAS mutator requires all layer choice have the same number of candidates." + if 'reduce' in mutable.key: + bias = [] + for choice in mutable.choices: + if 'conv' in str(type(choice)).lower(): + bias.append(branch_bias) + else: + bias.append(-branch_bias) + self.bias_dict[mutable.key] = tf.constant(bias) + + # exposed for trainer + self.sample_log_prob = 0 + self.sample_entropy = 0 + self.sample_skip_penalty = 0 + + # internal nn layers + self.embedding = Embedding(self.max_layer_choice + 1, lstm_size) + self.soft = Dense(self.max_layer_choice, use_bias=False) + self.attn_anchor = Dense(lstm_size, use_bias=False) + self.attn_query = Dense(lstm_size, use_bias=False) + self.v_attn = Dense(1, use_bias=False) + assert entropy_reduction in ['sum', 'mean'], 'Entropy reduction must be one of sum and mean.' + self.entropy_reduction = tf.reduce_sum if entropy_reduction == 'sum' else tf.reduce_mean + self.cross_entropy_loss = SparseCategoricalCrossentropy(from_logits=True, reduction=Reduction.NONE) + + self._first_sample = True + + def sample_search(self): + self._initialize() + self._sample(self.mutables) + self._first_sample = False + return self._choices + + def sample_final(self): + return self.sample_search() + + def _sample(self, tree): + mutable = tree.mutable + if isinstance(mutable, LayerChoice) and mutable.key not in self._choices: + self._choices[mutable.key] = self._sample_layer_choice(mutable) + elif isinstance(mutable, InputChoice) and mutable.key not in self._choices: + self._choices[mutable.key] = self._sample_input_choice(mutable) + for child in tree.children: + self._sample(child) + if self.cell_exit_extra_step and isinstance(mutable, MutableScope) and mutable.key not in self._anchors_hid: + self._anchors_hid[mutable.key] = self.lstm(self._inputs, 1) + + def _initialize(self): + self._choices = {} + self._anchors_hid = {} + self._inputs = self.g_emb + # seems the `input_shape` parameter of RNN does not work + # workaround it by omitting `reset_states` for first run + if not self._first_sample: + self.lstm.reset_states() + self.sample_log_prob = 0 + self.sample_entropy = 0 + self.sample_skip_penalty = 0 + + def _sample_layer_choice(self, mutable): + logit = self.soft(self.lstm(self._inputs)) + if self.temperature is not None: + logit /= self.temperature + if self.tanh_constant is not None: + logit = self.tanh_constant * tf.tanh(logit) + if mutable.key in self.bias_dict: + logit += self.bias_dict[mutable.key] + softmax_logit = tf.math.log(tf.nn.softmax(logit, axis=-1)) + branch_id = tf.reshape(tf.random.categorical(softmax_logit, num_samples=1), [1]) + log_prob = self.cross_entropy_loss(branch_id, logit) + self.sample_log_prob += self.entropy_reduction(log_prob) + entropy = log_prob * tf.math.exp(-log_prob) + self.sample_entropy += self.entropy_reduction(entropy) + self._inputs = tf.reshape(self.embedding(branch_id), [1, 1, -1]) + mask = tf.one_hot(branch_id, self.max_layer_choice) + return tf.cast(tf.reshape(mask, [-1]), tf.bool) + + def _sample_input_choice(self, mutable): + query, anchors = [], [] + for label in mutable.choose_from: + if label not in self._anchors_hid: + self._anchors_hid[label] = self.lstm(self._inputs) + query.append(self.attn_anchor(self._anchors_hid[label])) + anchors.append(self._anchors_hid[label]) + query = tf.concat(query, axis=0) + query = tf.tanh(query + self.attn_query(anchors[-1])) + query = self.v_attn(query) + + if self.temperature is not None: + query /= self.temperature + if self.tanh_constant is not None: + query = self.tanh_constant * tf.tanh(query) + + if mutable.n_chosen is None: + logit = tf.concat([-query, query], axis=1) + softmax_logit = tf.math.log(tf.nn.softmax(logit, axis=-1)) + skip = tf.reshape(tf.random.categorical(softmax_logit, num_samples=1), [-1]) + skip_prob = tf.math.sigmoid(logit) + kl = tf.reduce_sum(skip_prob * tf.math.log(skip_prob / self.skip_targets)) + self.sample_skip_penalty += kl + log_prob = self.cross_entropy_loss(skip, logit) + + skip = tf.cast(skip, tf.float32) + inputs = tf.tensordot(skip, tf.concat(anchors, 0), 1) / (1. + tf.reduce_sum(skip)) + self._inputs = tf.reshape(inputs, [1, 1, -1]) + + else: + assert mutable.n_chosen == 1, "Input choice must select exactly one or any in ENAS." + logit = tf.reshape(query, [1, -1]) + softmax_logit = tf.math.log(tf.nn.softmax(logit, axis=-1)) + index = tf.reshape(tf.random.categorical(softmax_logit, num_samples=1), [-1]) + skip = tf.reshape(tf.one_hot(index, mutable.n_candidates), [-1]) + # when the size is 1, tf does not accept tensor here, complaining the shape is wrong + # but using a numpy array seems fine + log_prob = self.cross_entropy_loss(logit, query.numpy()) + self._inputs = tf.reshape(anchors[index.numpy()[0]], [1, 1, -1]) + + self.sample_log_prob += self.entropy_reduction(log_prob) + entropy = log_prob * tf.exp(-log_prob) + self.sample_entropy += self.entropy_reduction(entropy) + assert len(skip) == mutable.n_candidates, (skip, mutable.n_candidates, mutable.n_chosen) + return tf.cast(skip, tf.bool) diff --git a/new_impl/cv/third_party/nni/algorithms/nas/tensorflow/enas/trainer.py b/new_impl/cv/third_party/nni/algorithms/nas/tensorflow/enas/trainer.py new file mode 100644 index 0000000000000000000000000000000000000000..c20f20ab1a68b2c97e6bf27be0f3567016995f1d --- /dev/null +++ b/new_impl/cv/third_party/nni/algorithms/nas/tensorflow/enas/trainer.py @@ -0,0 +1,203 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging + +import tensorflow as tf +from tensorflow.keras.optimizers import Adam + +from nni.nas.tensorflow.utils import AverageMeterGroup, fill_zero_grads + +from .mutator import EnasMutator + +logger = logging.getLogger(__name__) + + +class EnasTrainer: + def __init__( + self, + model, + loss, + metrics, + reward_function, + optimizer, + batch_size, + num_epochs, + dataset_train, + dataset_valid, + log_frequency=100, + entropy_weight=0.0001, + skip_weight=0.8, + baseline_decay=0.999, + child_steps=500, + mutator_lr=0.00035, + mutator_steps=50, + mutator_steps_aggregate=20, + aux_weight=0.4, + test_arc_per_epoch=1, + ): + self.model = model + self.loss = loss + self.metrics = metrics + self.reward_function = reward_function + self.optimizer = optimizer + self.batch_size = batch_size + self.num_epochs = num_epochs + + x, y = dataset_train + split = int(len(x) * 0.9) + self.train_set = tf.data.Dataset.from_tensor_slices((x[:split], y[:split])) + self.valid_set = tf.data.Dataset.from_tensor_slices((x[split:], y[split:])) + self.test_set = tf.data.Dataset.from_tensor_slices(dataset_valid) + + self.log_frequency = log_frequency + self.entropy_weight = entropy_weight + self.skip_weight = skip_weight + self.baseline_decay = baseline_decay + self.child_steps = child_steps + self.mutator_lr = mutator_lr + self.mutator_steps = mutator_steps + self.mutator_steps_aggregate = mutator_steps_aggregate + self.aux_weight = aux_weight + self.test_arc_per_epoch = test_arc_per_epoch + + self.mutator = EnasMutator(model) + self.mutator_optim = Adam(learning_rate=self.mutator_lr) + + self.baseline = 0.0 + + def train(self, validate=True): + for epoch in range(self.num_epochs): + logger.info("Epoch %d Training", epoch + 1) + self.train_one_epoch(epoch) + logger.info("Epoch %d Validating", epoch + 1) + self.validate_one_epoch(epoch) + + def validate(self): + self.validate_one_epoch(-1) + + def train_one_epoch(self, epoch): + train_loader, valid_loader = self._create_train_loader() + + # Sample model and train + meters = AverageMeterGroup() + + for step in range(1, self.child_steps + 1): + x, y = next(train_loader) + self.mutator.reset() + + with tf.GradientTape() as tape: + logits = self.model(x, training=True) + if isinstance(logits, tuple): + logits, aux_logits = logits + aux_loss = self.loss(aux_logits, y) + else: + aux_loss = 0.0 + metrics = self.metrics(y, logits) + loss = self.loss(y, logits) + self.aux_weight * aux_loss + + grads = tape.gradient(loss, self.model.trainable_weights) + grads = fill_zero_grads(grads, self.model.trainable_weights) + grads, _ = tf.clip_by_global_norm(grads, 5.0) + self.optimizer.apply_gradients(zip(grads, self.model.trainable_weights)) + + metrics["loss"] = tf.reduce_mean(loss).numpy() + meters.update(metrics) + + if self.log_frequency and step % self.log_frequency == 0: + logger.info( + "Model Epoch [%d/%d] Step [%d/%d] %s", + epoch + 1, + self.num_epochs, + step, + self.child_steps, + meters, + ) + + # Train sampler (mutator) + meters = AverageMeterGroup() + for mutator_step in range(1, self.mutator_steps + 1): + grads_list = [] + for step in range(1, self.mutator_steps_aggregate + 1): + with tf.GradientTape() as tape: + x, y = next(valid_loader) + self.mutator.reset() + + logits = self.model(x, training=False) + metrics = self.metrics(y, logits) + reward = ( + self.reward_function(y, logits) + + self.entropy_weight * self.mutator.sample_entropy + ) + self.baseline = self.baseline * self.baseline_decay + reward * ( + 1 - self.baseline_decay + ) + loss = self.mutator.sample_log_prob * (reward - self.baseline) + loss += self.skip_weight * self.mutator.sample_skip_penalty + + meters.update( + { + "reward": reward, + "loss": tf.reduce_mean(loss).numpy(), + "ent": self.mutator.sample_entropy.numpy(), + "log_prob": self.mutator.sample_log_prob.numpy(), + "baseline": self.baseline, + "skip": self.mutator.sample_skip_penalty, + } + ) + + cur_step = step + (mutator_step - 1) * self.mutator_steps_aggregate + if self.log_frequency and cur_step % self.log_frequency == 0: + logger.info( + "RL Epoch [%d/%d] Step [%d/%d] [%d/%d] %s", + epoch + 1, + self.num_epochs, + mutator_step, + self.mutator_steps, + step, + self.mutator_steps_aggregate, + meters, + ) + + grads = tape.gradient(loss, self.mutator.trainable_weights) + grads = fill_zero_grads(grads, self.mutator.trainable_weights) + grads_list.append(grads) + total_grads = [ + tf.math.add_n(weight_grads) for weight_grads in zip(*grads_list) + ] + total_grads, _ = tf.clip_by_global_norm(total_grads, 5.0) + self.mutator_optim.apply_gradients( + zip(total_grads, self.mutator.trainable_weights) + ) + + def validate_one_epoch(self, epoch): + test_loader = self._create_validate_loader() + + for arc_id in range(self.test_arc_per_epoch): + meters = AverageMeterGroup() + for x, y in test_loader: + self.mutator.reset() + logits = self.model(x, training=False) + if isinstance(logits, tuple): + logits, _ = logits + metrics = self.metrics(y, logits) + loss = self.loss(y, logits) + metrics["loss"] = tf.reduce_mean(loss).numpy() + meters.update(metrics) + + logger.info( + "Test Epoch [%d/%d] Arc [%d/%d] Summary %s", + epoch + 1, + self.num_epochs, + arc_id + 1, + self.test_arc_per_epoch, + meters.summary(), + ) + + def _create_train_loader(self): + train_set = self.train_set.shuffle(1000000).repeat().batch(self.batch_size) + test_set = self.valid_set.shuffle(1000000).repeat().batch(self.batch_size) + return iter(train_set), iter(test_set) + + def _create_validate_loader(self): + return iter(self.test_set.shuffle(1000000).batch(self.batch_size)) diff --git a/new_impl/cv/third_party/nni/assessor.py b/new_impl/cv/third_party/nni/assessor.py new file mode 100644 index 0000000000000000000000000000000000000000..b70995cbad6b479d8462967fa17c0f4b7fef8ce9 --- /dev/null +++ b/new_impl/cv/third_party/nni/assessor.py @@ -0,0 +1,124 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +Assessor analyzes trial's intermediate results (e.g., periodically evaluated accuracy on test dataset) +to tell whether this trial can be early stopped or not. + +See :class:`Assessor`' specification and ``docs/en_US/assessors.rst`` for details. +""" + +from enum import Enum +import logging + +from .recoverable import Recoverable + +__all__ = ['AssessResult', 'Assessor'] + +_logger = logging.getLogger(__name__) + + +class AssessResult(Enum): + """ + Enum class for :meth:`Assessor.assess_trial` return value. + """ + + Good = True + """The trial works well.""" + + Bad = False + """The trial works poorly and should be early stopped.""" + + +class Assessor(Recoverable): + """ + Assessor analyzes trial's intermediate results (e.g., periodically evaluated accuracy on test dataset) + to tell whether this trial can be early stopped or not. + + This is the abstract base class for all assessors. + Early stopping algorithms should inherit this class and override :meth:`assess_trial` method, + which receives intermediate results from trials and give an assessing result. + + If :meth:`assess_trial` returns :obj:`AssessResult.Bad` for a trial, + it hints NNI framework that the trial is likely to result in a poor final accuracy, + and therefore should be killed to save resource. + + If an accessor want's to be notified when a trial ends, it can also override :meth:`trial_end`. + + To write a new assessor, you can reference :class:`~nni.medianstop_assessor.MedianstopAssessor`'s code as an example. + + See Also + -------- + Builtin assessors: + :class:`~nni.algorithms.hpo.medianstop_assessor.MedianstopAssessor` + :class:`~nni.algorithms.hpo.curvefitting_assessor.CurvefittingAssessor` + """ + + def assess_trial(self, trial_job_id, trial_history): + """ + Abstract method for determining whether a trial should be killed. Must override. + + The NNI framework has little guarantee on ``trial_history``. + This method is not guaranteed to be invoked for each time ``trial_history`` get updated. + It is also possible that a trial's history keeps updating after receiving a bad result. + And if the trial failed and retried, ``trial_history`` may be inconsistent with its previous value. + + The only guarantee is that ``trial_history`` is always growing. + It will not be empty and will always be longer than previous value. + + This is an example of how :meth:`assess_trial` get invoked sequentially: + + :: + + trial_job_id | trial_history | return value + ------------ | --------------- | ------------ + Trial_A | [1.0, 2.0] | Good + Trial_B | [1.5, 1.3] | Bad + Trial_B | [1.5, 1.3, 1.9] | Good + Trial_A | [0.9, 1.8, 2.3] | Good + + Parameters + ---------- + trial_job_id : str + Unique identifier of the trial. + trial_history : list + Intermediate results of this trial. The element type is decided by trial code. + + Returns + ------- + AssessResult + :obj:`AssessResult.Good` or :obj:`AssessResult.Bad`. + """ + raise NotImplementedError('Assessor: assess_trial not implemented') + + def trial_end(self, trial_job_id, success): + """ + Abstract method invoked when a trial is completed or terminated. Do nothing by default. + + Parameters + ---------- + trial_job_id : str + Unique identifier of the trial. + success : bool + True if the trial successfully completed; False if failed or terminated. + """ + + def load_checkpoint(self): + """ + Internal API under revising, not recommended for end users. + """ + checkpoin_path = self.get_checkpoint_path() + _logger.info('Load checkpoint ignored by assessor, checkpoint path: %s', checkpoin_path) + + def save_checkpoint(self): + """ + Internal API under revising, not recommended for end users. + """ + checkpoin_path = self.get_checkpoint_path() + _logger.info('Save checkpoint ignored by assessor, checkpoint path: %s', checkpoin_path) + + def _on_exit(self): + pass + + def _on_error(self): + pass diff --git a/new_impl/cv/third_party/nni/common/__init__.py b/new_impl/cv/third_party/nni/common/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/new_impl/cv/third_party/nni/common/graph_utils.py b/new_impl/cv/third_party/nni/common/graph_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..0037afba9f11864213e10e9ab187e8647b36d425 --- /dev/null +++ b/new_impl/cv/third_party/nni/common/graph_utils.py @@ -0,0 +1,789 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + + +import logging +import queue +import re +from collections import defaultdict +import torch +from torch.utils.tensorboard._pytorch_graph import NodePy, NodePyIO, NodePyOP, GraphPy +CLASSTYPE_KIND = 'ClassType' +GETATTR_KIND = 'prim::GetAttr' +CAT_KIND = 'aten::cat' +LIST_CONSTRUCT_KIND = 'prim::ListConstruct' +LIST_UNPACK_KIND = 'prim::ListUnpack' +TUPLE_CONSTRUCT_KIND = 'prim::TupleConstruct' +TUPLE_UNPACK_KIND = 'prim::TupleUnpack' + +_logger = logging.getLogger(__name__) + + +def build_module_graph(model, dummy_input): + return TorchModuleGraph(model, dummy_input) + + +def build_graph(model, dummy_input, verbose=False): + g = TorchProtoGraph(model, dummy_input, verbose) + return g.graph_def, g.stepstats + + +def parse_traced_name(module_name): + prefix = 'TracedModule[' + suffix = ']' + if module_name.startswith(prefix) and module_name.endswith(suffix): + module_name = module_name[len(prefix):-len(suffix)] + return module_name + + +class TorchGraph: + """ + This class is to extract pytorch model topology graph by tracing + """ + + def __init__(self, model=None, dummy_input=None, traced_model=None): + """ + Parameters + ---------- + model : pytorch model + The model user wants to speed up + dummy_input : pytorch tensor + The dummy input for ```jit.trace```, users should put it on right device before pass in + traced_model : torch._C.torch.jit.TopLevelTracedModule + An alredy traced model, if traced_model is not None, then TorchGraph will build the graph + based on this traced model and won't trace the model again. + """ + assert torch.__version__ >= '1.3.1' + # check if the input is legal + if traced_model is not None: + assert isinstance(traced_model, torch.jit.TopLevelTracedModule) + self.trace = traced_model + # it's ok if the graph is already unpacked + torch._C._jit_pass_inline(self.trace.graph) + elif model is not None and dummy_input is not None: + self.bound_model = model + self._trace(model, dummy_input) + else: + raise Exception( + 'Please provide model & dummy_input or the traced_model as inputs') + + def _trace(self, model, dummy_input): + with torch.onnx.set_training(model, False): + # import torch.jit + self.trace = torch.jit.trace(model, dummy_input, check_trace=False) + torch._C._jit_pass_inline(self.trace.graph) + + +class TorchProtoGraph(TorchGraph): + """ + Generates model graph for pytorch models in protobuf, this implementation + is borrowed from pytorch v1.4.0, and fixed following issues: + https://github.com/pytorch/pytorch/issues/33691 + https://github.com/pytorch/pytorch/issues/33670 + + """ + + def __init__(self, model, dummy_input, verbose=False): + super().__init__(model, dummy_input) + + from tensorboard.compat.proto.config_pb2 import RunMetadata + from tensorboard.compat.proto.graph_pb2 import GraphDef + from tensorboard.compat.proto.step_stats_pb2 import StepStats, DeviceStepStats + from tensorboard.compat.proto.versions_pb2 import VersionDef + + list_of_nodes = self.parse(self.trace.graph, self.trace, dummy_input) + if verbose: + print(self.trace.graph) + self.stepstats = RunMetadata(step_stats=StepStats( + dev_stats=[DeviceStepStats(device="/device:CPU:0")])) + self.graph_def = GraphDef( + node=list_of_nodes, versions=VersionDef(producer=22)) + + def parse(self, graph, trace, args=None, omit_useless_nodes=True): + """This method parses an optimized PyTorch model graph and produces + a list of nodes and node stats for eventual conversion to TensorBoard + protobuf format. + + Args: + graph (PyTorch module): The model graph to be parsed. + trace (PyTorch JIT TracedModule): The model trace to be parsed. + args (tuple): input tensor[s] for the model. + omit_useless_nodes (boolean): Whether to remove nodes from the graph. + """ + nodes_py = GraphPy() + for node in graph.inputs(): + if omit_useless_nodes: + if not node.uses(): # number of user of the node (= number of outputs/ fanout) + continue + + if node.type().kind() != CLASSTYPE_KIND: + nodes_py.append(NodePyIO(node, 'input')) + + attr_to_scope = dict() + + def node_to_name(d): + return str(d).split(":")[0].strip() + for node in graph.nodes(): + if node.kind() == GETATTR_KIND: + attr_name = node.s('name') + node_name = node_to_name(node) + parent = node.input().node() + # If the parent node is not the top-level "self" node + if parent.kind() == GETATTR_KIND: + parent_scope = attr_to_scope[node_to_name(parent)] + attr_scope = parent_scope.split('/')[-1] + attr_to_scope[node_name] = '{}/{}.{}'.format( + parent_scope, attr_scope, attr_name) + else: + attr_to_scope[node_name] = '__module.{}'.format(attr_name) + # We don't need classtype nodes; scope will provide this information + if node.output().type().kind() != CLASSTYPE_KIND: + node_py = NodePyOP(node) + node_py.scopeName = attr_to_scope[node_name] + nodes_py.append(node_py) + else: + nodes_py.append(NodePyOP(node)) + + # Create sink nodes for output ops + for i, node in enumerate(graph.outputs()): + node_py = NodePyIO(node, 'output') + node_py.debugName = "output.{}".format(i + 1) + node_py.inputs = [node.debugName()] + nodes_py.append(node_py) + + alias_to_name = dict() + base_name = parse_traced_name(trace._name) + for name, module in trace.named_modules(prefix='__module'): + mod_name = parse_traced_name(module._name) + attr_name = name.split('.')[-1] + alias_to_name[name] = '{}[{}]'.format(mod_name, attr_name) + + for node in nodes_py.nodes_op: + module_aliases = node.scopeName.split('/')[-1].split('.') + module_name = '' + for i, alias in enumerate(module_aliases): + if i == 0: + module_name = alias + node.scopeName = base_name + else: + module_name += '.' + alias + node.scopeName += '/' + \ + (alias_to_name[module_name] + if module_name in alias_to_name else alias) + + nodes_py.populate_namespace_from_OP_to_IO() + return nodes_py.to_proto() + + +class NodePyGroup(NodePy): + """ + This class is used to represent a graph node which consists of multiple jit traced nodes. In a pytorch trace graph, + there are multiple nodes are traced for one torch.nn.Module object, we group them together to form a single node to + represent the torch.nn.Module object. We also group some functional call trace nodes together to form a new node. + """ + + def __init__(self, name, unique_name, node_type, op_type, node_cpps, inputs=None, outputs=None, key_node=None): + """ + Parameters: + ----------- + name: str + node name, such as `conv1`, `backbone.classifier` + unique_name: str + A global unique name for current node. Due to some modules, + such as relu, may be reused several times, so the scopename + is not suitable as the global unique identifier, so we add a + unique_name for each node as the global unique identifier. + We should use the unique_name to traverset the module graph. + node_type: str + `module` or `func` + op_type: str + operation type, such as `Conv2d`, `aten::view` + node_cpps: list of torch._C.Node + jit trace nodes which are included in this new node + inputs: list of str + All the inputs of this node, each element is debugName of one input + outputs: list of str + All the outputs of this node, each element is debugName of one output + key_node: torch._C.Node + The key node of this NodePyGroup. + """ + super(NodePyGroup, self).__init__(name, []) + self.node_cpps = node_cpps + self.name = name + self.unique_name = unique_name + self.op_type = op_type + self.type = node_type + self.nodes = [] + self.auxiliary = None + self.add_nodes(node_cpps) + self.inputs = inputs + self.outputs = outputs + # The core node in this NodePyGroup + self.key_node = key_node + + def add_nodes(self, node_cpps): + for node_cpp in node_cpps: + nodepy = NodePyOP(node_cpp) + nodepy.name = node_cpp.scopeName() + '_' + node_cpp.kind() + self.nodes.append(nodepy) + + def sub_node_names(self): + return [x.name for x in self.nodes] + + def __repr__(self): + return 'name: {}, type: {}, op_type: {}, sub_nodes: {}, inputs: {}, outputs: {}, aux: {}'.format( + self.name, self.type, self.op_type, self.sub_node_names(), + self.inputs, self.outputs, self.auxiliary + ) + + +class TorchModuleGraph(TorchGraph): + """ + Generates model graph, each node is created from single or multiple jit trace nodes. + """ + + def __init__(self, model=None, dummy_input=None, traced_model=None): + super().__init__(model, dummy_input, traced_model) + self.global_count = 0 + self.name_to_node, self.input_to_node, self.output_to_node = self._build_graph() + self._extract_auxiliary_info() + + def _expand_key_func_node(self, node, nodes, input_to_node, output_to_node, + module_type): + """ + For trace graph nodes, some nodes are not in modules, these nodes are usually generated by + the functions directly called in module ```forward```. For such nodes, some of them are + trivial op which are label by ```prim::```, some of them are not such ops which is call + non-prim ops. This function is to merge neighbor prim ops to a non-prim op, to construct + a node. + + Parameters + ---------- + node : trace graph node + The non-prim node to expand + nodes : list of trace graph node + All the trace graph nodes within the same scope as the non-prim node + input_to_node : dict + key: input name, value: a node that uses this input + output_to_node : dict + key: output name, value: a node that generates this output + module_type : str + can be 'module' or 'func' + + Returns + ------- + node + the expanded non-prim node + """ + # TODO: scope name could be empty + node_name = '.'.join([self._get_module_name( + node.scopeName()), node.kind(), str(self.global_count)]) + unique_name = node_name + _logger.debug("expand non-prim node, node name: %s", node_name) + self.global_count += 1 + op_type = node.kind() + node_group = [node] + inputs = list() + outputs = list() + node_queue = queue.Queue() + node_queue.put(node) + while not node_queue.empty(): + curr_node = node_queue.get() + for _input in curr_node.inputs(): + input_name = _input.debugName() + if input_name in output_to_node and output_to_node[input_name] in nodes: + predecessor_node = output_to_node[input_name] + if not self._is_key_func(predecessor_node): + node_group.append(predecessor_node) + node_queue.put(predecessor_node) + else: + inputs.append(input_name) + else: + inputs.append(input_name) + for output in node.outputs(): + outputs.append(output.debugName()) + nodepy = NodePyGroup(node_name, unique_name, module_type, op_type, + node_group, inputs=inputs, outputs=outputs, key_node=node) + return nodepy + + def _expand_module_node(self, node, node_name, unique_name, op_type, nodes, + input_to_node, output_to_node, module_type): + """ + merge the adjacent nodes of the module. The difference between the + _expand_module_node and _expand_non_prim_node is that, the _expand_non_prim_node + only merge the prim:: nodes into the aten:: node, in contrast,the _expand_module_node + will merge all adjacent nodes into a same nodepy group. + + Parameters + ---------- + node : trace graph node + The non-prim node to expand + node_name : str + specify the node_name for NodePyGroup + unique_name : str + unique_name for the NodePyGroup + op_type : str + specify the op_type for the NodePyGroup + nodes : list of trace graph node + All the trace graph nodes within the same scope as the non-prim node + input_to_node : dict + key: input name, value: a node that uses this input + output_to_node : dict + key: output name, value: a node that generates this output + module_type : str + can be 'module' or 'func' + Returns + ------- + node + the expanded non-prim node + + """ + _logger.debug("expand module node, node name: %s", node_name) + self.global_count += 1 + if not op_type: + op_type = node.kind() + node_group = [node] + inputs = list() + outputs = list() + node_queue = queue.Queue() + node_queue.put(node) + visited = {node} + while not node_queue.empty(): + curr_node = node_queue.get() + for _input in curr_node.inputs(): + input_name = _input.debugName() + if input_name in output_to_node and output_to_node[input_name] in nodes: + predecessor_node = output_to_node[input_name] + if predecessor_node not in visited: + node_group.append(predecessor_node) + node_queue.put(predecessor_node) + visited.add(predecessor_node) + else: + inputs.append(input_name) + for _output in curr_node.outputs(): + output_name = _output.debugName() + if output_name in input_to_node and input_to_node[output_name] in nodes: + successor_node = input_to_node[output_name] + if successor_node not in visited: + node_group.append(successor_node) + node_queue.put(successor_node) + visited.add(successor_node) + else: + outputs.append(output_name) + + nodepy = NodePyGroup(node_name, unique_name, module_type, op_type, + node_group, inputs=inputs, outputs=outputs) + return nodepy + + def _extract_cat_info(self, node_group, cpp_node): + """ + Extract the detail information of the cat operation, + such the order of the input tensor, the shape of each + input tensor, the output shape, and the cat dimension. + + Parameters + ---------- + node_group : NodePyGroup + cpp_node: torch._C.Node + It should be ```aten::cat``` node + + Returns + ------- + dict + Include auxiliary information for the cat operation. + This dict objec has four keys: 'cat_dim', 'out_shape', + 'in_order' and 'in_shape'. cat_dim is the dimension of + the cat operation to concat the input tensors. out_shape + is the shape of the output tensor of the cat operation. + in_order is an ordered list which contains the corresponding + parent operaion nodes of the input tensors. in_shape is also + an ordered list that contains the input shapes of the input + tensor. + """ + # only suport the cat operation + assert cpp_node.kind() == CAT_KIND + cat_info = {} + # get the shape of the output tensor + t_output = cpp_node.output() + out_shape = t_output.type().sizes() + cat_info['out_shape'] = out_shape + # get the cat dimension + inputs = cpp_node.inputs() + cat_dim = list(inputs)[1].toIValue() + cat_info['cat_dim'] = cat_dim + # get the order of the input tensors + # To get the order of the input tensors, we need + # to be aware of the topology of the model, which + # means we should extract the auxiliary information + # after the build_index function. + input_order = [] + list_construct_cpp = list(cpp_node.inputs())[0].node() + input_tensors = list(list_construct_cpp.inputs()) + for _tensor in input_tensors: + debug_name = _tensor.debugName() + input_order.append(self.output_to_node[debug_name].unique_name) + cat_info['in_order'] = input_order + input_shapes = [t.type().sizes() for t in input_tensors] + cat_info['in_shape'] = input_shapes + return cat_info + + def _extract_linear_shape_info(self, node_group): + """ + Extract linear shape input/output tensor shape info from its aten::addmm op. + + Parameters + ---------- + node_group : NodePyGroup + NodePyGroup object associated with the linear module. + + Returns + ------- + dict + Include shape of input tensor and shape of output tensor + """ + for cpp_node in node_group.node_cpps: + if cpp_node.kind() == 'aten::addmm': + # https://github.com/pytorch/pytorch/blob/1.6/torch/nn/functional.py#L1682 + # inputs of aten::addmm: + # inputs[0] is bias + # inputs[1] is input data + # inputs[2] is weight + t_input = list(cpp_node.inputs())[1] + t_output = cpp_node.output() + assert isinstance(t_input.type(), torch._C.TensorType) + assert isinstance(t_output.type(), torch._C.TensorType) + in_shape = t_input.type().sizes() + out_shape = t_output.type().sizes() + return {'in_shape': in_shape, 'out_shape': out_shape} + return None + + def _extract_shape_info(self, node): + """ + Extract the shape information of ```aten::view``` node + + Parameters + ---------- + node : trace graph node + It should be ```aten::view``` node + + Returns + ------- + dict + Include shape of input tensor and shape of output tensor + """ + t_input = None + for _input in node.inputs(): + t_input = _input + break + t_output = node.output() + assert isinstance(t_input.type(), torch._C.TensorType) + assert isinstance(t_output.type(), torch._C.TensorType) + in_shape = t_input.type().sizes() + out_shape = t_output.type().sizes() + return {'in_shape': in_shape, 'out_shape': out_shape} + + def _extract_leaf_modules(self): + """ + Extract leaf modules from the given graph. Leaf module means it does not have submodules. + To extract leaf modules because only leaf module can be replaced. And shape inference can + be done in leaf module level. Other shape inference is done in lower level i.e., + operation level. + + Returns + ------- + list + a list of scope name of all the leaf modules + """ + def is_parent(name1, name2): + """ + check if name1 is parent node of name2, for example: + name1: aa.bb, name2: aa.bb.cc, return True + name1: aa.b, name2: aa.bb, return False + """ + parts1, parts2 = name1.split('.'), name2.split('.') + if len(parts1) >= len(parts2): + return False + for i, _ in enumerate(parts1): + if parts2[i] != parts1[i]: + return False + return True + module_names = sorted([x[0] + for x in self.trace.named_modules() if x[0]]) + leaf_nodes = [] + for i, name in enumerate(module_names): + if i + 1 >= len(module_names) or not is_parent(name, module_names[i + 1]): + leaf_nodes.append(name) + return leaf_nodes + + def _get_module_name(self, scope_name): + """ + Retrieve module name from scope name. + Parameters: + ----------- + scope_name: str + scope_name of a graph node, for example: + for pytorch 1.3.1: MyModel/BackboneModel[backbone]/Conv2d[conv2] + for pytorch 1.4.0: __module.backbone/__module.backbone.conv2 + + Returns: + ------- + str + module name, such as backbone.conv2 + """ + if torch.__version__ >= '1.4.0': + return scope_name.split('/')[-1].replace('__module.', '') + else: + return '.'.join(re.findall(r'\[(.*?)\]', scope_name)) + + def _build_index(self, nodes_op): + name_to_node = dict() + input_to_node = defaultdict(list) + output_to_node = dict() + for node in nodes_op: + name_to_node[node.unique_name] = node + for _input in node.inputs: + input_to_node[_input].append(node) + for output in node.outputs: + assert not output in output_to_node, \ + "One output cannot be generated by multiple nodes" + output_to_node[output] = node + return name_to_node, input_to_node, output_to_node + + def _is_key_func(self, node_cpp): + """ + Judge if a cpp node is a key function node. + If so, we should not merge this node into the + adjacent node. + """ + if node_cpp.kind().startswith('aten::'): + # the nodes that start with 'aten' are key function + # nodes + return True + if node_cpp.kind() in [LIST_UNPACK_KIND, TUPLE_UNPACK_KIND]: + # We cannot merge the List/Tuple + # Unpack func into other nodes, else it + # may lead to a graph construction error. + # The reason why we donnot take the construct node + # also as a key node is that `cat` operation node need + # the last(previous) visited node to infer the mask. If + # we take the Construct node as the important node, the + # predecessor of the `cat` node will always be a construct + # node, which means we cannot infer the mask for the cat + # operation. + return True + return False + + def unpack_manually(self): + """ + Unpack the tensor tuple or tensor list manually, + and remove the ListUnpack/TupleUnpack node from + the graph. Note: this function will change the + graph structure. + """ + if hasattr(self, 'unpacked'): + # if already unpacked the tuple/list manually + return + for node in self.nodes_py.nodes_op: + if node.op_type in [TUPLE_UNPACK_KIND, LIST_UNPACK_KIND]: + unpack_cpp = node.key_node + last_cpp = list(unpack_cpp.inputs())[0].node() + if last_cpp.kind() in [TUPLE_CONSTRUCT_KIND, LIST_CONSTRUCT_KIND]: + # we need check if the tensor tuple or tensor list is produced + # by a list/tuple construct node. If so, we can unpack the tuple + # or list manunally. + _logger.debug('List/Tuple Construct Node(cpp) %s', str(last_cpp)) + _logger.debug('List/Tuple Unpack Node(cpp) %s', str(unpack_cpp)) + assert len(list(unpack_cpp.outputs())) == len(list(last_cpp.inputs())) + errmsg = '%s Input number: %d if inconsistent with the output number %d' % (unpack_cpp, \ + len(node.inputs), len(list(last_cpp.inputs()))) + + assert len(node.inputs) == len(list(last_cpp.inputs())), errmsg + for _debug_input, _debug_output in zip(node.inputs, node.outputs): + # _debug_input = _input.debugName() + # _debug_output = _output.debugName() + if _debug_input in self.input_to_node and _debug_output in self.input_to_node: + # input_to_node[_debug_input] is a list of NodePyGroup, because + # one tensor can be used as input for multiple nodes at the same time. + + # note that, in this case, the construct cpp node and unpack cpp node + # will be merged into the same NodePyGroup, so we remove the `node` from + # input_to_node[_debug_input] and directly connect this tensor to the + # input_to_node[_debug_output] + self.input_to_node[_debug_input].remove(node) + # add the following nodes of _output into the input_to_node[_debug_input] + self.input_to_node[_debug_input].extend(self.input_to_node[_debug_output]) + # just remove the _debug_output from the grapgh index. So that we can also skip + # the construct and tuple + if _debug_output in self.input_to_node: + for following_node in self.input_to_node[_debug_output]: + _tmp_index = following_node.inputs.index(_debug_output) + following_node.inputs[_tmp_index] = _debug_input + + + self.unpacked = True + + def _build_graph(self): + """ + Build graph using our defined format from jit trace. + There are basically three steps: first, construct necessary information (data structures), + second, extract all the modules to convert to node, Third, extract all functions to convert + to node. + + Returns + ------- + dict + use name to index nodes, key: node name, value: node + dict + use input (its name) to index nodes, + key: input, value: list of nodes that take this input + dict + use output (its name) to index nodes, + key: output, value: node that generates this output + """ + omit_useless_nodes = True + graph = self.trace.graph + # _logger.debug(graph) + # build output mapping, from output debugName to its node + output_to_node = {x.debugName(): n for n in graph.nodes() + for x in n.outputs()} + # build input mapping, from input debugName to its node + input_to_node = {x.debugName(): n for n in graph.nodes() + for x in n.inputs()} + # build module mapping, from module name to all nodes (as list) under this module scope + module_to_nodes = defaultdict(list) + # the mapping of function (non-module in forward) to nodes, key is scope name + func_to_nodes = defaultdict(list) + + nodes_py = GraphPy() + for node in graph.inputs(): + if omit_useless_nodes: + if not node.uses(): # number of user of the node (= number of outputs/ fanout) + continue + + if node.type().kind() != 'ClassType': + nodes_py.append(NodePyIO(node, 'input')) + + self.leaf_modules = self._extract_leaf_modules() + module_to_type = {name: parse_traced_name( + module._name) for name, module in self.trace.named_modules()} + + # associate module name with their trace graph nodes + for node in graph.nodes(): + module_name = self._get_module_name(node.scopeName()) + if module_name in self.leaf_modules: + module_to_nodes[module_name].append(node) + else: + func_to_nodes[node.scopeName()].append(node) + # build node group for module + for module_name, node_cpps in module_to_nodes.items(): + use_count = 0 + merged = set() + for node in node_cpps: + if node not in merged: + # modules that have same scope name may have different locations in the + # graph. Futhermore, there are also lots of prim:: nodes that in node_cpps, + # so we also need to call the expand_module_node. + unique_name = module_name + if use_count > 0: + unique_name = module_name + '.%d' % use_count + node_group = self._expand_module_node( + node, module_name, unique_name, module_to_type[module_name], + node_cpps, input_to_node, output_to_node, 'module') + nodes_py.nodes_op.append(node_group) + use_count += 1 + merged.update(node_group.node_cpps) + + # each scope_name may have multiple funcs, we split them and create node for each of them + # build node group for torch.nn.functional + for _, nodes in func_to_nodes.items(): + # extract non prim:: nodes + key_func_nodes = list() + for node in nodes: + if self._is_key_func(node): + # find the key function nodes + key_func_nodes.append(node) + # for each non prim node, expand it + for node in key_func_nodes: + node_group = self._expand_key_func_node( + node, nodes, input_to_node, output_to_node, 'func') + nodes_py.nodes_op.append(node_group) + # get shape infor for view (aten::view) func + # if node_group.op_type in ['aten::view', 'aten::flatten']: + # node_group.auxiliary = self._extract_shape_info(node) + + for node in graph.outputs(): # Create sink nodes for output ops + node_py = NodePyIO(node, 'output') + nodes_py.append(node_py) + + self.nodes_py = nodes_py + # build index + return self._build_index(self.nodes_py.nodes_op) + + def _extract_auxiliary_info(self): + """ + Extract the auxiliary information for the nodegroups + if necessary. For example, view/flatten operations may + need the shape of the input tensor and output tensor. + """ + # extract the input & output shape for the view and flatten + for node_group in self.nodes_py.nodes_op: + if node_group.op_type in ['aten::view', 'aten::flatten', 'aten::mean', 'aten::reshape']: + # get shape infor for view (aten::view) func + cpp_node = list(filter(lambda x: x.kind() == node_group.op_type, + node_group.node_cpps))[0] + node_group.auxiliary = self._extract_shape_info(cpp_node) + elif node_group.op_type == 'Linear': + node_group.auxiliary = self._extract_linear_shape_info(node_group) + elif node_group.op_type == CAT_KIND: + # get the detail information for cat func + cpp_node = list(filter(lambda x: x.kind() == node_group.op_type, + node_group.node_cpps))[0] + node_group.auxiliary = self._extract_cat_info( + node_group, cpp_node) + + def find_predecessors(self, unique_name): + """ + Find predecessor node of the given node + + Parameters + ---------- + unique_name : str + The unique name of the node + + Returns + ------- + list + a list of nodes who are the given node's predecessor + """ + predecessors = [] + for _input in self.name_to_node[unique_name].inputs: + if not _input in self.output_to_node: + _logger.debug("cannot find node with %s as its output", _input) + else: + node_py = self.output_to_node[_input] + predecessors.append(node_py.unique_name) + return predecessors + + def find_successors(self, unique_name): + """ + Find successor nodes of the given node + + Parameters + ---------- + unique_name : str + The unique name of the node + + Returns + ------- + list + a list of nodes who are the given node's successor + """ + successors = [] + for output in self.name_to_node[unique_name].outputs: + if output not in self.input_to_node: + # may reach the output of the whole graph + continue + nodes_py = self.input_to_node[output] + for node_py in nodes_py: + successors.append(node_py.unique_name) + return successors diff --git a/new_impl/cv/third_party/nni/common/nas_utils.py b/new_impl/cv/third_party/nni/common/nas_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..d7f050ec9bb2046b782bf699e1bd3ae7925a31a1 --- /dev/null +++ b/new_impl/cv/third_party/nni/common/nas_utils.py @@ -0,0 +1,317 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import functools +import logging + +from .. import trial + + +_logger = logging.getLogger(__name__) +_MUTABLE_LAYER_SPACE_PREFIX = "_mutable_layer" +_namespace = {} +_tf_variables = {} +_arch_logits_list = [] +_optimizer = None +_train_op = None + + +def classic_mode( + mutable_id, + mutable_layer_id, + funcs, + funcs_args, + fixed_inputs, + optional_inputs, + optional_input_size): + '''Execute the chosen function and inputs directly. + In this mode, the trial code is only running the chosen subgraph (i.e., the chosen ops and inputs), + without touching the full model graph.''' + if trial.get_current_parameter() is None: + trial.get_next_parameter() + + chosen_layer, chosen_inputs = _get_layer_and_inputs_from_tuner(mutable_id, mutable_layer_id, + list(optional_inputs.keys())) + real_chosen_inputs = [optional_inputs[input_name] for input_name in chosen_inputs] + layer_out = funcs[chosen_layer]([fixed_inputs, real_chosen_inputs], **funcs_args[chosen_layer]) + + return layer_out + + +def enas_mode( + mutable_id, + mutable_layer_id, + funcs, + funcs_args, + fixed_inputs, + optional_inputs, + optional_input_size, + tf): + '''For enas mode, we build the full model graph in trial but only run a subgraph。 + This is implemented by masking inputs and branching ops. + Specifically, based on the received subgraph (through nni.get_next_parameter), + it can be known which inputs should be masked and which op should be executed.''' + name_prefix = "{}_{}".format(mutable_id, mutable_layer_id) + # store namespace + _namespace[mutable_id] = True + _namespace[name_prefix] = dict() + _namespace[name_prefix]['funcs'] = list(funcs) + _namespace[name_prefix]['optional_inputs'] = list(optional_inputs) + # create tensorflow variables as 1/0 signals used to form subgraph + name_for_optional_inputs = name_prefix + '_optional_inputs' + name_for_funcs = name_prefix + '_funcs' + _tf_variables[name_prefix] = dict() + _tf_variables[name_prefix]['optional_inputs'] = tf.get_variable( + name_for_optional_inputs, + [len(optional_inputs)], + dtype=tf.bool, + trainable=False + ) + _tf_variables[name_prefix]['funcs'] = tf.get_variable( + name_for_funcs, [], dtype=tf.int64, trainable=False) + + # get real values using their variable names + real_optional_inputs_value = [optional_inputs[name] + for name in _namespace[name_prefix]['optional_inputs']] + real_func_value = [funcs[name] + for name in _namespace[name_prefix]['funcs']] + real_funcs_args = [funcs_args[name] + for name in _namespace[name_prefix]['funcs']] + # build tensorflow graph of geting chosen inputs by masking + real_chosen_inputs = tf.boolean_mask( + real_optional_inputs_value, _tf_variables[name_prefix]['optional_inputs']) + # build tensorflow graph of different branches by using tf.case + branches = dict() + func_output = None + for func_id in range(len(funcs)): + func_output = real_func_value[func_id]([fixed_inputs, real_chosen_inputs], **real_funcs_args[func_id]) + branches[tf.equal(_tf_variables[name_prefix]['funcs'], func_id)] = lambda: func_output + layer_out = tf.case(branches, exclusive=True, default=lambda: func_output) + + return layer_out + + +def oneshot_mode( + mutable_id, + mutable_layer_id, + funcs, + funcs_args, + fixed_inputs, + optional_inputs, + optional_input_size, + tf): + '''Similar to enas mode, oneshot mode also builds the full model graph. + The difference is that oneshot mode does not receive subgraph. + Instead, it uses dropout to randomly dropout inputs and ops.''' + # NNI requires to get_next_parameter before report a result. But the parameter will not be used in this mode + if trial.get_current_parameter() is None: + trial.get_next_parameter() + optional_inputs = list(optional_inputs.values()) + inputs_num = len(optional_inputs) + # Calculate dropout rate according to the formular r^(1/k), where r is a hyper-parameter and k is the number of inputs + if inputs_num > 0: + rate = 0.01 ** (1 / inputs_num) + noise_shape = [inputs_num] + [1] * len(optional_inputs[0].get_shape()) + optional_inputs = tf.nn.dropout( + optional_inputs, rate=rate, noise_shape=noise_shape) + optional_inputs = [optional_inputs[idx] for idx in range(inputs_num)] + layer_outs = [func([fixed_inputs, optional_inputs], **funcs_args[func_name]) + for func_name, func in funcs.items()] + output_num = len(layer_outs) + rate = 0.01 ** (1 / output_num) + noise_shape = [output_num] + [1] * len(layer_outs[0].get_shape()) + layer_outs = tf.nn.dropout(layer_outs, rate=rate, noise_shape=noise_shape) + layer_out = tf.reduce_sum(layer_outs, axis=0) + + return layer_out + + +def darts_mode( + mutable_id, + mutable_layer_id, + funcs, + funcs_args, + fixed_inputs, + optional_inputs, + optional_input_size, + tf): + optional_inputs = list(optional_inputs.values()) + layer_outs = [func([fixed_inputs, optional_inputs], **funcs_args[func_name]) + for func_name, func in funcs.items()] + # Create architecture weights for every func(op) + var_name = "{}_{}_arch_weights".format(mutable_id, mutable_layer_id) + arch_logits = tf.get_variable(var_name, shape=[len(funcs)], trainable=False) + _arch_logits_list.append(arch_logits) + arch_weights = tf.nn.softmax(arch_logits) + layer_out = tf.add_n([arch_weights[idx] * out for idx, out in enumerate(layer_outs)]) + + return layer_out + + +def reload_tensorflow_variables(tf, session): + '''In Enas mode, this function reload every signal varaible created in `enas_mode` function so + the whole tensorflow graph will be changed into certain subgraph recerived from Tuner. + --------------- + session: the tensorflow session created by users + tf: tensorflow module + ''' + subgraph_from_tuner = trial.get_next_parameter() + mutable_layers = set() + for subgraph_key in subgraph_from_tuner: + if "/" in subgraph_key: + # has to remove the last, could be layer_choice or whatever + mutable_id, mutable_layer_id = _decompose_general_key(subgraph_key[:subgraph_key.rfind("/")]) + if mutable_id is not None: + mutable_layers.add((mutable_id, mutable_layer_id)) + mutable_layers = sorted(list(mutable_layers)) + for mutable_id, mutable_layer_id in mutable_layers: + if mutable_id not in _namespace: + _logger.warning("%s not found in name space", mutable_id) + continue + name_prefix = "{}_{}".format(mutable_id, mutable_layer_id) + # get optional inputs names + optional_inputs = _namespace[name_prefix]['optional_inputs'] + # extract layer information from the subgraph sampled by tuner + chosen_layer, chosen_inputs = _get_layer_and_inputs_from_tuner(mutable_id, mutable_layer_id, optional_inputs) + chosen_layer = _namespace[name_prefix]['funcs'].index(chosen_layer) + chosen_inputs = [1 if inp in chosen_inputs else 0 for inp in optional_inputs] + # load these information into pre-defined tensorflow variables + _tf_variables[name_prefix]['funcs'].load(chosen_layer, session) + _tf_variables[name_prefix]['optional_inputs'].load( + chosen_inputs, session) + + +def _construct_general_key(mutable_id, mutable_layer_id): + # Mutable layer key in a general (search space) format + # that is, prefix/mutable_id/mutable_layer_id + return _MUTABLE_LAYER_SPACE_PREFIX + "/" + mutable_id + "/" + mutable_layer_id + + +def _decompose_general_key(key): + # inverse operation of above + if not key.startswith(_MUTABLE_LAYER_SPACE_PREFIX): + return None, None + else: + _, mutable_id, mutable_layer_id = key.split("/", maxsplit=2) + return mutable_id, mutable_layer_id + + +def darts_training(tf, session, loss, feed_dict): + global _optimizer, _train_op + if _optimizer is None: + _optimizer = tf.MomentumOptimizer(learning_rate=0.025) + # TODO: Calculate loss + grads_and_vars = _optimizer.compute_gradients(loss, _arch_logits_list) + _train_op = _optimizer.apply_gradients(grads_and_vars) + session.run(_train_op) + + +def training_update(nas_mode, tf=None, session=None, loss=None, feed_dict=None): + if nas_mode == 'darts_mode': + darts_training(tf, session, loss, feed_dict) + elif nas_mode == 'enas_mode': + reload_tensorflow_variables(tf, session) + + +def _get_layer_and_inputs_from_tuner(mutable_id, mutable_layer_id, optional_inputs): + # optional_inputs should be name(key)s of the optional inputs + try: + mutable_block = trial.get_current_parameter(mutable_id) + + # There is a NAS tuner + chosen_layer = mutable_block[mutable_layer_id]["chosen_layer"] + chosen_inputs = mutable_block[mutable_layer_id]["chosen_inputs"] + except KeyError: + # Try to find converted NAS parameters + params = trial.get_current_parameter() + expected_prefix = _construct_general_key(mutable_id, mutable_layer_id) + chosen_layer = params[expected_prefix + "/layer_choice"] + + # find how many to choose + optional_input_size = int(params[expected_prefix + "/optional_input_size"]) # convert uniform to randint + + # find who to choose, can duplicate + optional_input_state = params[expected_prefix + "/optional_input_chosen_state"] + chosen_inputs = [] + # make sure dict -> list produce stable result by sorting + optional_inputs_keys = sorted(optional_inputs) + for _ in range(optional_input_size): + chosen_inputs.append(optional_inputs_keys[optional_input_state % len(optional_inputs)]) + optional_input_state //= len(optional_inputs) + + _logger.info("%s_%s: layer: %s, optional inputs: %s", mutable_id, mutable_layer_id, chosen_layer, chosen_inputs) + return chosen_layer, chosen_inputs + + +def convert_nas_search_space(search_space): + """ + Args: + param search_space: raw search space + return: the new search space, mutable_layers will be converted into choice + """ + if not isinstance(search_space, dict): + return search_space + ret = dict() + for k, v in search_space.items(): + if "_type" not in v: + # this should not happen + _logger.warning("There is no _type in one of your search space values with key '%s'" + ". Please check your search space", k) + ret[k] = v + elif v["_type"] != "mutable_layer": + ret[k] = v + else: + _logger.info("Converting mutable_layer search space with key '%s'", k) + # v["_value"] looks like {'mutable_layer_1': {'layer_choice': ...} ...} + values = v["_value"] + for layer_name, layer_data in values.items(): + # there should be at most layer_choice, optional_inputs, optional_input_size in layer_data + + # add "_mutable_layer" as prefix so that they can be recovered later + layer_key = _construct_general_key(k, layer_name) + + if layer_data.get("layer_choice"): # filter out empty choice and no choice + layer_choice = layer_data["layer_choice"] + else: + raise ValueError("No layer choice found in %s" % layer_key) + + if layer_data.get("optional_input_size"): + input_size = layer_data["optional_input_size"] + if isinstance(input_size, int): + input_size = [input_size, input_size] + if input_size[0] > input_size[1] or input_size[0] < 0: + _logger.error("Might not be able to handle optional_input_size < 0, please double check") + input_size[1] += 1 + else: + _logger.info("Optional input choices are set to empty by default in %s", layer_key) + input_size = [0, 1] + + if layer_data.get("optional_inputs"): + total_state_size = len(layer_data["optional_inputs"]) ** (input_size[1] - 1) + else: + _logger.info("Optional inputs not found in %s", layer_key) + total_state_size = 1 + + converted = { + layer_key + "/layer_choice": { + "_type": "choice", "_value": layer_choice + }, + layer_key + "/optional_input_size": { + "_type": "randint", "_value": input_size + }, + layer_key + "/optional_input_chosen_state": { + "_type": "randint", "_value": [0, total_state_size] + } + } + _logger.info(converted) + ret.update(converted) + + return ret + + +def rewrite_nas_space(func): + @functools.wraps(func) + def wrap(self, search_space): + search_space = convert_nas_search_space(search_space) + return func(self, search_space) + return wrap diff --git a/new_impl/cv/third_party/nni/compression/__init__.py b/new_impl/cv/third_party/nni/compression/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/new_impl/cv/third_party/nni/compression/pytorch/__init__.py b/new_impl/cv/third_party/nni/compression/pytorch/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e20f284fef2cd06fa7cd4c1cf8ce43914195e5fc --- /dev/null +++ b/new_impl/cv/third_party/nni/compression/pytorch/__init__.py @@ -0,0 +1,5 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .speedup import ModelSpeedup +from .compressor import Compressor, Pruner, Quantizer diff --git a/new_impl/cv/third_party/nni/compression/pytorch/compressor.py b/new_impl/cv/third_party/nni/compression/pytorch/compressor.py new file mode 100644 index 0000000000000000000000000000000000000000..b21589530206794dd506f2137ae8131bae83be80 --- /dev/null +++ b/new_impl/cv/third_party/nni/compression/pytorch/compressor.py @@ -0,0 +1,636 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import types +import logging +import torch +from . import default_layers + +_logger = logging.getLogger(__name__) + + +class LayerInfo: + def __init__(self, name, module): + self.module = module + self.name = name + self.type = type(module).__name__ + +def _setattr(model, name, module): + name_list = name.split(".") + for name in name_list[:-1]: + model = getattr(model, name) + setattr(model, name_list[-1], module) + + +class Compressor: + """ + Abstract base PyTorch compressor + """ + + def __init__(self, model, config_list, optimizer=None): + """ + Record necessary info in class members + + Parameters + ---------- + model : pytorch model + the model user wants to compress + config_list : list + the configurations that users specify for compression + optimizer: pytorch optimizer + optimizer used to train the model + """ + assert isinstance(model, torch.nn.Module) + self.validate_config(model, config_list) + + self.bound_model = model + self.config_list = config_list + self.optimizer = optimizer + + self.modules_to_compress = None + self.modules_wrapper = [] + self.is_wrapped = False + + self._fwd_hook_handles = {} + self._fwd_hook_id = 0 + + self.reset() + + if not self.modules_wrapper: + _logger.warning('Nothing is configured to compress, please check your model and config_list') + + def validate_config(self, model, config_list): + """ + subclass can optionally implement this method to check if config_list if valid + """ + pass + + def reset(self, checkpoint=None): + """ + reset model state dict and model wrapper + """ + self._unwrap_model() + if checkpoint is not None: + self.bound_model.load_state_dict(checkpoint) + + self.modules_to_compress = None + self.modules_wrapper = [] + + for layer, config in self._detect_modules_to_compress(): + wrapper = self._wrap_modules(layer, config) + self.modules_wrapper.append(wrapper) + + self._wrap_model() + + def _detect_modules_to_compress(self): + """ + detect all modules should be compressed, and save the result in `self.modules_to_compress`. + The model will be instrumented and user should never edit it after calling this method. + """ + if self.modules_to_compress is None: + self.modules_to_compress = [] + for name, module in self.bound_model.named_modules(): + if module == self.bound_model: + continue + layer = LayerInfo(name, module) + config = self.select_config(layer) + if config is not None: + self.modules_to_compress.append((layer, config)) + return self.modules_to_compress + + def _wrap_model(self): + """ + wrap all modules that needed to be compressed + + """ + for wrapper in reversed(self.get_modules_wrapper()): + _setattr(self.bound_model, wrapper.name, wrapper) + self.is_wrapped = True + + def _unwrap_model(self): + """ + unwrap all modules that needed to be compressed + + """ + for wrapper in self.get_modules_wrapper(): + _setattr(self.bound_model, wrapper.name, wrapper.module) + self.is_wrapped = False + + def compress(self): + """ + Compress the model with algorithm implemented by subclass. + + The model will be instrumented and user should never edit it after calling this method. + `self.modules_to_compress` records all the to-be-compressed layers + + Returns + ------- + torch.nn.Module + model with specified modules compressed. + """ + return self.bound_model + + def set_wrappers_attribute(self, name, value): + """ + To register attributes used in wrapped module's forward method. + If the type of the value is Torch.tensor, then this value is registered as a buffer in wrapper, + which will be saved by model.state_dict. Otherwise, this value is just a regular variable in wrapper. + + Parameters + ---------- + name : str + name of the variable + value: any + value of the variable + """ + for wrapper in self.get_modules_wrapper(): + if isinstance(value, torch.Tensor): + wrapper.register_buffer(name, value.clone()) + else: + setattr(wrapper, name, value) + + def get_modules_to_compress(self): + """ + To obtain all the to-be-compressed modules. + + Returns + ------- + list + a list of the layers, each of which is a tuple (`layer`, `config`), + `layer` is `LayerInfo`, `config` is a `dict` + """ + return self.modules_to_compress + + def get_modules_wrapper(self): + """ + To obtain all the wrapped modules. + + Returns + ------- + list + a list of the wrapped modules + """ + return self.modules_wrapper + + def select_config(self, layer): + """ + Find the configuration for `layer` by parsing `self.config_list` + + Parameters + ---------- + layer : LayerInfo + one layer + + Returns + ------- + config or None + the retrieved configuration for this layer, if None, this layer should + not be compressed + """ + ret = None + for config in self.config_list: + config = config.copy() + # expand config if key `default` is in config['op_types'] + if 'op_types' in config and 'default' in config['op_types']: + expanded_op_types = [] + for op_type in config['op_types']: + if op_type == 'default': + expanded_op_types.extend(default_layers.weighted_modules) + else: + expanded_op_types.append(op_type) + config['op_types'] = expanded_op_types + + # check if condition is satisified + if 'op_types' in config and layer.type not in config['op_types']: + continue + if 'op_names' in config and layer.name not in config['op_names']: + continue + + ret = config + if ret is None or 'exclude' in ret: + return None + return ret + + def update_epoch(self, epoch): + """ + If user want to update model every epoch, user can override this method. + This method should be called at the beginning of each epoch + + Parameters + ---------- + epoch : num + the current epoch number + """ + pass + + def _wrap_modules(self, layer, config): + """ + This method is implemented in the subclasses, i.e., `Pruner` and `Quantizer` + + Parameters + ---------- + layer : LayerInfo + the layer to instrument the compression operation + config : dict + the configuration for compressing this layer + """ + raise NotImplementedError() + + + def add_activation_collector(self, collector): + self._fwd_hook_id += 1 + self._fwd_hook_handles[self._fwd_hook_id] = [] + for wrapper in self.get_modules_wrapper(): + handle = wrapper.register_forward_hook(collector) + self._fwd_hook_handles[self._fwd_hook_id].append(handle) + return self._fwd_hook_id + + def remove_activation_collector(self, fwd_hook_id): + if fwd_hook_id not in self._fwd_hook_handles: + raise ValueError("%s is not a valid collector id" % str(fwd_hook_id)) + for handle in self._fwd_hook_handles[fwd_hook_id]: + handle.remove() + del self._fwd_hook_handles[fwd_hook_id] + + def patch_optimizer(self, *tasks): + def patch_step(old_step): + def new_step(_, *args, **kwargs): + # call origin optimizer step method + output = old_step(*args, **kwargs) + # calculate mask + for task in tasks: + task() + return output + return new_step + if self.optimizer is not None: + self.optimizer.step = types.MethodType(patch_step(self.optimizer.step), self.optimizer) + +class PrunerModuleWrapper(torch.nn.Module): + def __init__(self, module, module_name, module_type, config, pruner): + """ + Wrap an module to enable data parallel, forward method customization and buffer registeration. + + Parameters + ---------- + module : pytorch module + the module user wants to compress + config : dict + the configurations that users specify for compression + module_name : str + the name of the module to compress, wrapper module shares same name + module_type : str + the type of the module to compress + pruner ： Pruner + the pruner used to calculate mask + """ + super().__init__() + # origin layer information + self.module = module + self.name = module_name + self.type = module_type + # config and pruner + self.config = config + self.pruner = pruner + + # register buffer for mask + self.register_buffer("weight_mask", torch.ones(self.module.weight.shape)) + if hasattr(self.module, 'bias') and self.module.bias is not None: + self.register_buffer("bias_mask", torch.ones(self.module.bias.shape)) + else: + self.register_buffer("bias_mask", None) + + def forward(self, *inputs): + # apply mask to weight, bias + self.module.weight.data = self.module.weight.data.mul_(self.weight_mask) + if hasattr(self.module, 'bias') and self.module.bias is not None: + self.module.bias.data = self.module.bias.data.mul_(self.bias_mask) + return self.module(*inputs) + +class Pruner(Compressor): + """ + Prune to an exact pruning level specification + + Attributes + ---------- + mask_dict : dict + Dictionary for saving masks, `key` should be layer name and + `value` should be a tensor which has the same shape with layer's weight + + """ + + def __init__(self, model, config_list, optimizer=None): + super().__init__(model, config_list, optimizer) + if optimizer is not None: + self.patch_optimizer(self.update_mask) + + def compress(self): + self.update_mask() + return self.bound_model + + def update_mask(self): + for wrapper_idx, wrapper in enumerate(self.get_modules_wrapper()): + masks = self.calc_mask(wrapper, wrapper_idx=wrapper_idx) + if masks is not None: + for k in masks: + assert hasattr(wrapper, k), "there is no attribute '%s' in wrapper" % k + setattr(wrapper, k, masks[k]) + + def calc_mask(self, wrapper, **kwargs): + """ + Pruners should overload this method to provide mask for weight tensors. + The mask must have the same shape and type comparing to the weight. + It will be applied with `mul()` operation on the weight. + This method is effectively hooked to `forward()` method of the model. + + Parameters + ---------- + wrapper : Module + calculate mask for `wrapper.module`'s weight + """ + raise NotImplementedError("Pruners must overload calc_mask()") + + def _wrap_modules(self, layer, config): + """ + Create a wrapper module to replace the original one. + + Parameters + ---------- + layer : LayerInfo + the layer to instrument the mask + config : dict + the configuration for generating the mask + """ + _logger.debug("Module detected to compress : %s.", layer.name) + wrapper = PrunerModuleWrapper(layer.module, layer.name, layer.type, config, self) + assert hasattr(layer.module, 'weight'), "module %s does not have 'weight' attribute" % layer.name + # move newly registered buffers to the same device of weight + wrapper.to(layer.module.weight.device) + return wrapper + + def export_model(self, model_path, mask_path=None, onnx_path=None, input_shape=None, device=None): + """ + Export pruned model weights, masks and onnx model(optional) + + Parameters + ---------- + model_path : str + path to save pruned model state_dict + mask_path : str + (optional) path to save mask dict + onnx_path : str + (optional) path to save onnx model + input_shape : list or tuple + input shape to onnx model + device : torch.device + device of the model, used to place the dummy input tensor for exporting onnx file. + the tensor is placed on cpu if ```device``` is None + """ + assert model_path is not None, 'model_path must be specified' + mask_dict = {} + self._unwrap_model() # used for generating correct state_dict name without wrapper state + + for wrapper in self.get_modules_wrapper(): + weight_mask = wrapper.weight_mask + bias_mask = wrapper.bias_mask + if weight_mask is not None: + mask_sum = weight_mask.sum().item() + mask_num = weight_mask.numel() + _logger.debug('Layer: %s Sparsity: %.4f', wrapper.name, 1 - mask_sum / mask_num) + wrapper.module.weight.data = wrapper.module.weight.data.mul(weight_mask) + if bias_mask is not None: + wrapper.module.bias.data = wrapper.module.bias.data.mul(bias_mask) + # save mask to dict + mask_dict[wrapper.name] = {"weight": weight_mask, "bias": bias_mask} + + torch.save(self.bound_model.state_dict(), model_path) + _logger.info('Model state_dict saved to %s', model_path) + if mask_path is not None: + torch.save(mask_dict, mask_path) + _logger.info('Mask dict saved to %s', mask_path) + if onnx_path is not None: + assert input_shape is not None, 'input_shape must be specified to export onnx model' + # input info needed + if device is None: + device = torch.device('cpu') + input_data = torch.Tensor(*input_shape) + torch.onnx.export(self.bound_model, input_data.to(device), onnx_path) + _logger.info('Model in onnx with input shape %s saved to %s', input_data.shape, onnx_path) + + self._wrap_model() + + def load_model_state_dict(self, model_state): + """ + Load the state dict saved from unwrapped model. + + Parameters: + ----------- + model_state : dict + state dict saved from unwrapped model + """ + if self.is_wrapped: + self._unwrap_model() + self.bound_model.load_state_dict(model_state) + self._wrap_model() + else: + self.bound_model.load_state_dict(model_state) + +class QuantizerModuleWrapper(torch.nn.Module): + def __init__(self, module, module_name, module_type, config, quantizer): + """ + Wrap an module to enable data parallel, forward method customization and buffer registeration. + + Parameters + ---------- + module : pytorch module + the module user wants to compress + config : dict + the configurations that users specify for compression + module_name : str + the name of the module to compress, wrapper module shares same name + module_type : str + the type of the module to compress + quantizer ：quantizer + the quantizer used to calculate mask + """ + super().__init__() + # origin layer information + self.module = module + self.name = module_name + self.type = module_type + # config and pruner + self.config = config + self.quantizer = quantizer + + # register buffer and parameter + # old_weight is used to store origin weight and weight is used to store quantized weight + # the reason why weight is buffer instead of parameter is because in pytorch parameter is used as leaf + # if weight is leaf , then old_weight can not be updated. + if 'weight' in config['quant_types']: + if not _check_weight(self.module): + _logger.warning('Module %s does not have parameter "weight"', self.name) + else: + self.module.register_parameter('old_weight', torch.nn.Parameter(self.module.weight)) + delattr(self.module, 'weight') + self.module.register_buffer('weight', self.module.old_weight) + + def forward(self, *inputs): + if 'input' in self.config['quant_types']: + inputs = self.quantizer.quant_grad.apply( + inputs, + QuantType.QUANT_INPUT, + self) + + if 'weight' in self.config['quant_types'] and _check_weight(self.module): + self.quantizer.quant_grad.apply( + self.module.old_weight, + QuantType.QUANT_WEIGHT, + self) + result = self.module(*inputs) + else: + result = self.module(*inputs) + + if 'output' in self.config['quant_types']: + result = self.quantizer.quant_grad.apply( + result, + QuantType.QUANT_OUTPUT, + self) + return result + return result + +class Quantizer(Compressor): + """ + Base quantizer for pytorch quantizer + """ + + def __init__(self, model, config_list, optimizer=None): + super().__init__(model, config_list, optimizer) + self.quant_grad = QuantGrad + if self.optimizer is not None: + self.patch_optimizer(self.step_with_optimizer) + for wrapper in self.get_modules_wrapper(): + if 'weight' in wrapper.config['quant_types']: + # old_weight is registered to keep track of weight before quantization + # and it is trainable, therefore, it should be added to optimizer. + self.optimizer.add_param_group({"params": wrapper.module.old_weight}) + + def quantize_weight(self, weight, wrapper, **kwargs): + """ + quantize should overload this method to quantize weight. + This method is effectively hooked to :meth:`forward` of the model. + Parameters + ---------- + weight : Tensor + weight that needs to be quantized + wrapper : QuantizerModuleWrapper + the wrapper for origin module + """ + raise NotImplementedError('Quantizer must overload quantize_weight()') + + def quantize_output(self, output, wrapper, **kwargs): + """ + quantize should overload this method to quantize output. + This method is effectively hooked to :meth:`forward` of the model. + Parameters + ---------- + output : Tensor + output that needs to be quantized + wrapper : QuantizerModuleWrapper + the wrapper for origin module + """ + raise NotImplementedError('Quantizer must overload quantize_output()') + + def quantize_input(self, *inputs, wrapper, **kwargs): + """ + quantize should overload this method to quantize input. + This method is effectively hooked to :meth:`forward` of the model. + Parameters + ---------- + inputs : Tensor + inputs that needs to be quantized + wrapper : QuantizerModuleWrapper + the wrapper for origin module + """ + raise NotImplementedError('Quantizer must overload quantize_input()') + + + def _wrap_modules(self, layer, config): + """ + Create a wrapper forward function to replace the original one. + Parameters + ---------- + layer : LayerInfo + the layer to instrument the mask + config : dict + the configuration for quantization + """ + assert 'quant_types' in config, 'must provide quant_types in config' + assert isinstance(config['quant_types'], list), 'quant_types must be list type' + assert 'quant_bits' in config, 'must provide quant_bits in config' + assert isinstance(config['quant_bits'], int) or isinstance(config['quant_bits'], dict), 'quant_bits must be dict type or int type' + + if isinstance(config['quant_bits'], dict): + for quant_type in config['quant_types']: + assert quant_type in config['quant_bits'], 'bits length for %s must be specified in quant_bits dict' % quant_type + + return QuantizerModuleWrapper(layer.module, layer.name, layer.type, config, self) + + def step_with_optimizer(self): + pass + +class QuantType: + """ + Enum class for quantization type. + """ + QUANT_INPUT = 0 + QUANT_WEIGHT = 1 + QUANT_OUTPUT = 2 + + +class QuantGrad(torch.autograd.Function): + """ + Base class for overriding backward function of quantization operation. + """ + @staticmethod + def quant_backward(tensor, grad_output, quant_type): + """ + This method should be overrided by subclass to provide customized backward function, + default implementation is Straight-Through Estimator + Parameters + ---------- + tensor : Tensor + input of quantization operation + grad_output : Tensor + gradient of the output of quantization operation + quant_type : QuantType + the type of quantization, it can be `QuantType.QUANT_INPUT`, `QuantType.QUANT_WEIGHT`, `QuantType.QUANT_OUTPUT`, + you can define different behavior for different types. + Returns + ------- + tensor + gradient of the input of quantization operation + """ + return grad_output + + @staticmethod + def forward(ctx, tensor, quant_type, wrapper, **kwargs): + ctx.save_for_backward(tensor, torch.Tensor([quant_type])) + if quant_type == QuantType.QUANT_INPUT: + return wrapper.quantizer.quantize_input(tensor, wrapper, **kwargs) + elif quant_type == QuantType.QUANT_WEIGHT: + return wrapper.quantizer.quantize_weight(wrapper, **kwargs) + elif quant_type == QuantType.QUANT_OUTPUT: + return wrapper.quantizer.quantize_output(tensor, wrapper, **kwargs) + else: + raise ValueError("unrecognized QuantType.") + + @classmethod + def backward(cls, ctx, grad_output): + tensor, quant_type = ctx.saved_variables + output = cls.quant_backward(tensor, grad_output, quant_type) + return output, None, None, None + +def _check_weight(module): + try: + return isinstance(module.weight.data, torch.Tensor) + except AttributeError: + return False diff --git a/new_impl/cv/third_party/nni/compression/pytorch/default_layers.py b/new_impl/cv/third_party/nni/compression/pytorch/default_layers.py new file mode 100644 index 0000000000000000000000000000000000000000..4d7e6d8aed84ad76c9404f301117fb8a00d9a570 --- /dev/null +++ b/new_impl/cv/third_party/nni/compression/pytorch/default_layers.py @@ -0,0 +1,9 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +weighted_modules = [ + 'Conv1d', 'Conv2d', 'Conv3d', 'ConvTranspose1d', 'ConvTranspose2d', 'ConvTranspose3d', + 'Linear', 'Bilinear', + 'PReLU', + 'Embedding', 'EmbeddingBag', +] diff --git a/new_impl/cv/third_party/nni/compression/pytorch/speedup/__init__.py b/new_impl/cv/third_party/nni/compression/pytorch/speedup/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..cef8ebd76c0d99b1810512afe28723b5eeccb9fc --- /dev/null +++ b/new_impl/cv/third_party/nni/compression/pytorch/speedup/__init__.py @@ -0,0 +1 @@ +from .compressor import ModelSpeedup \ No newline at end of file diff --git a/new_impl/cv/third_party/nni/compression/pytorch/speedup/compress_modules.py b/new_impl/cv/third_party/nni/compression/pytorch/speedup/compress_modules.py new file mode 100644 index 0000000000000000000000000000000000000000..c2e61ba04372eeef1ee44c927756e2a049778dca --- /dev/null +++ b/new_impl/cv/third_party/nni/compression/pytorch/speedup/compress_modules.py @@ -0,0 +1,276 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +import torch +from .infer_shape import ModuleMasks + +_logger = logging.getLogger(__name__) + +replace_module = { + 'BatchNorm2d': lambda module, mask: replace_batchnorm2d(module, mask), + 'Conv2d': lambda module, mask: replace_conv2d(module, mask), + 'ConvTranspose2d': lambda module, mask: replace_conv2dt(module, mask), + 'MaxPool2d': lambda module, mask: no_replace(module, mask), + 'AvgPool2d': lambda module, mask: no_replace(module, mask), + 'AdaptiveAvgPool2d': lambda module, mask: no_replace(module, mask), + 'ReLU': lambda module, mask: no_replace(module, mask), + 'ReLU6': lambda module, mask: no_replace(module, mask), + 'LeakyReLU': lambda module, mask: no_replace(module, mask), + 'Sigmoid': lambda module, mask: no_replace(module, mask), + 'Linear': lambda module, mask: replace_linear(module, mask), + 'Dropout': lambda module, mask: no_replace(module, mask), + 'Dropout2d': lambda module, mask: no_replace(module, mask), + 'Dropout3d': lambda module, mask: no_replace(module, mask), + + 'EltwiseAdd': lambda module, mask: no_replace(module, mask), + 'TwoAdd': lambda module, mask: no_replace(module, mask), + + # yolov3 + 'ZeroPad2d': lambda module, mask: no_replace(module, mask), + + 'Tanh': lambda module, mask: no_replace(module, mask) +} + +def no_replace(module, mask): + """ + No need to replace + """ + _logger.debug("no need to replace") + return module + +def replace_linear(linear, mask): + """ + Parameters + ---------- + linear : torch.nn.Linear + The linear module to be replace + mask : ModuleMasks + The masks of this module + + Returns + ------- + torch.nn.Linear + The new linear module + """ + assert isinstance(mask, ModuleMasks) + assert mask.input_mask is not None + assert mask.output_mask is None + assert not mask.param_masks + index = mask.input_mask.mask_index[-1] + in_features = index.size()[0] + _logger.debug("replace linear with new in_features: %d", in_features) + new_linear = torch.nn.Linear(in_features=in_features, + out_features=linear.out_features, + bias=linear.bias is not None) + new_linear.to(linear.weight.device) + new_linear.weight.data = torch.index_select(linear.weight.data, -1, index.to(linear.weight.device)) + if linear.bias is not None: + new_linear.bias.data.copy_(linear.bias.data) + return new_linear + +def replace_batchnorm2d(norm, mask): + """ + Parameters + ---------- + norm : torch.nn.BatchNorm2d + The batchnorm module to be replace + mask : ModuleMasks + The masks of this module + + Returns + ------- + torch.nn.BatchNorm2d + The new batchnorm module + """ + assert isinstance(mask, ModuleMasks) + assert 'weight' in mask.param_masks and 'bias' in mask.param_masks + index = mask.param_masks['weight'].mask_index[0] + num_features = index.size()[0] + _logger.debug("replace batchnorm2d with num_features: %d", num_features) + new_norm = torch.nn.BatchNorm2d(num_features=num_features, + eps=norm.eps, + momentum=norm.momentum, + affine=norm.affine, + track_running_stats=norm.track_running_stats) + # assign weights + new_norm.weight.data = torch.index_select(norm.weight.data, 0, index) + new_norm.bias.data = torch.index_select(norm.bias.data, 0, index) + if norm.track_running_stats: + new_norm.running_mean.data = torch.index_select(norm.running_mean.data, 0, index) + new_norm.running_var.data = torch.index_select(norm.running_var.data, 0, index) + return new_norm + +def replace_conv2d(conv, mask): + """ + Parameters + ---------- + conv : torch.nn.Conv2d + The conv2d module to be replaced + mask : ModuleMasks + The masks of this module + + Returns + ------- + torch.nn.Conv2d + The new conv2d module + """ + assert isinstance(mask, ModuleMasks) + if mask.input_mask is None: + in_channels = conv.in_channels + else: + in_channels_index = mask.input_mask.mask_index[1] + in_channels = in_channels_index.size()[0] + if mask.output_mask is None: + out_channels = conv.out_channels + else: + out_channels_index = mask.output_mask.mask_index[1] + out_channels = out_channels_index.size()[0] + groups = conv.groups + if conv.in_channels == conv.out_channels == conv.groups: + # remove groups for depthwise layers + assert in_channels == out_channels + groups = in_channels + _logger.debug("replace conv2d %s with in_channels: %d, out_channels: %d", mask.module_name, in_channels, out_channels) + new_conv = torch.nn.Conv2d(in_channels=in_channels, + out_channels=out_channels, + kernel_size=conv.kernel_size, + stride=conv.stride, + padding=conv.padding, + dilation=conv.dilation, + groups=groups, + bias=conv.bias is not None, + padding_mode=conv.padding_mode) + + new_conv.to(conv.weight.device) + tmp_weight_data = tmp_bias_data = None + + if mask.output_mask is not None: + tmp_weight_data = torch.index_select(conv.weight.data, 0, out_channels_index) + if conv.bias is not None: + tmp_bias_data = torch.index_select(conv.bias.data, 0, out_channels_index) + else: + tmp_weight_data = conv.weight.data + # For the convolutional layers that have more than one group + # we need to copy the weight group by group, because the input + # channal is also divided into serveral groups and each group + # filter may have different input channel indexes. + input_step = int(conv.in_channels / conv.groups) + in_channels_group = int(in_channels / groups) + filter_step = int(out_channels / groups) + if mask.input_mask is not None and not (in_channels == out_channels == groups): + for groupid in range(conv.groups): + start = groupid * input_step + end = (groupid + 1) * input_step + current_input_index = list(filter(lambda x: start <= x and x < end, in_channels_index.tolist())) + if not current_input_index: + # there is no kept channel in current group + continue + # shift the global index into the group index + current_input_index = [x-start for x in current_input_index] + # if the groups is larger than 1, the input channels of each + # group should be pruned evenly. + assert len(current_input_index) == in_channels_group, \ + 'Input channels of each group are not pruned evenly' + current_input_index = torch.tensor(current_input_index).to(tmp_weight_data.device) # pylint: disable=not-callable + f_start = groupid * filter_step + f_end = (groupid + 1) * filter_step + new_conv.weight.data[f_start:f_end] = torch.index_select(tmp_weight_data[f_start:f_end], 1, current_input_index) + else: + new_conv.weight.data.copy_(tmp_weight_data) + + if conv.bias is not None: + new_conv.bias.data.copy_(conv.bias.data if tmp_bias_data is None else tmp_bias_data) + + return new_conv + + +def replace_conv2dt(convtrans, mask): + """ + We need anothor replace function for + convtranspose2d, because the layout of + the weight is different from traditional + conv layers. The layout of the weight is [N_in, N_out, ksize_1, ksize_2] + Parameters + ---------- + convtrans : torch.nn.ConvTranspose2d + The conv2d module to be replaced + mask : ModuleMasks + The masks of this module + Returns + ------- + torch.nn.ConvTranspose2d + The new conv2d module + """ + assert isinstance(mask, ModuleMasks) + assert isinstance(convtrans, torch.nn.ConvTranspose2d) + if mask.input_mask is None: + in_channels = convtrans.in_channels + else: + in_channels_index = mask.input_mask.mask_index[1] + in_channels = in_channels_index.size(0) + if mask.output_mask is None: + out_channels = convtrans.out_channels + else: + out_channels_index = mask.output_mask.mask_index[1] + out_channels = out_channels_index.size(0) + groups = convtrans.groups + # print('convtrans {} {} {}'.format(in_channels, out_channels, groups)) + # check if can remove the whole group of filters + if convtrans.in_channels == convtrans.out_channels == convtrans.groups: + # remove groups for depthwise layers + # this needs the group dependency to be fixed before the speedup + assert in_channels == out_channels + # groups = in_channels + _logger.debug('Replace convtranspose2d %s with in_channels:%d out_channels:%d', + mask.module_name, in_channels, out_channels) + new_convtrans = torch.nn.ConvTranspose2d(in_channels=in_channels, + out_channels=out_channels, + kernel_size=convtrans.kernel_size, + stride=convtrans.stride, + padding=convtrans.padding, + dilation=convtrans.dilation, + groups=groups, + bias=convtrans.bias is not None, + padding_mode=convtrans.padding_mode) + new_convtrans.to(convtrans.weight.device) + tmp_weight_data = None + if mask.input_mask is not None: + # in convtranspose2d we need to select the input channel first + tmp_weight_data = torch.index_select( + convtrans.weight.data, 0, in_channels_index) + else: + tmp_weight_data = convtrans.weight.data + # we need to handle the output channel group by group like the conv layer + out_step = int(convtrans.out_channels / convtrans.groups) + out_channel_group = int(out_channels/groups) + new_in_per_group = int(in_channels/groups) + + if mask.output_mask is not None and not(in_channels == out_channels == groups): + for groupid in range(convtrans.groups): + start = groupid * out_step + end = (groupid + 1) * out_step + current_output_index = list( + filter(lambda x: start <= x and x < end, out_channels_index.tolist())) + # we need to shift the index into the group-wise + current_output_index = [x-start for x in current_output_index] + if not current_output_index: + # No kept channel in the current group + raise Exception( + " Donnot support removing the whole group filter except in the depth-wise conv temporarily") + assert len(current_output_index) == out_channel_group, \ + 'Output channel of each group should be the same after pruning' + current_output_index = torch.tensor(current_output_index).to(tmp_weight_data.device) # pylint: disable=not-callable + new_start = groupid * new_in_per_group + new_end = (groupid + 1) * new_in_per_group + new_convtrans.weight.data[new_start:new_end] = torch.index_select( + tmp_weight_data[new_start:new_end], 1, current_output_index) + else: + new_convtrans.weight.data.copy_(tmp_weight_data) + if convtrans.bias is not None: + if mask.output_mask is not None: + new_convtrans.bias.data[:] = torch.index_select( + convtrans.bias.data, 0, out_channels_index) + else: + new_convtrans.bias.data.copy_(convtrans.bias.data) + return new_convtrans \ No newline at end of file diff --git a/new_impl/cv/third_party/nni/compression/pytorch/speedup/compressor.py b/new_impl/cv/third_party/nni/compression/pytorch/speedup/compressor.py new file mode 100644 index 0000000000000000000000000000000000000000..a6d3590b1746f6d577b608fcd7a2ede938890293 --- /dev/null +++ b/new_impl/cv/third_party/nni/compression/pytorch/speedup/compressor.py @@ -0,0 +1,187 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +import torch +from ..utils.mask_conflict import fix_mask_conflict +from ..utils.utils import get_module_by_name +from .compress_modules import replace_module +from .infer_shape import ModuleMasks, infer_from_mask, infer_from_inshape, infer_from_outshape, set_conv_prune_dim + +_logger = logging.getLogger(__name__) + + +class ModelSpeedup: + """ + This class is to speedup the model with provided weight mask + """ + + def __init__(self, model, dummy_input, masks_file, map_location=None): + """ + Parameters + ---------- + model : pytorch model + The model user wants to speed up + dummy_input : pytorch tensor + The dummy input for ```jit.trace```, users should put it on right device before pass in + masks_file : str + The path of user provided mask file + map_location : str + the device on which masks are placed, same to map_location in ```torch.load``` + """ + from ....common.graph_utils import build_module_graph + + self.bound_model = model + self.masks = torch.load(masks_file, map_location) + self.inferred_masks = dict() # key: module_name, value: ModuleMasks + self.dummy_input = dummy_input + self.torch_graph = build_module_graph(model, dummy_input) + + def infer_module_mask(self, module_name, last_module, mask=None, in_shape=None, out_shape=None): + """ + Infer input shape / output shape based on the module's weight mask / input shape / output shape. + + For a module: + Infer its input and output shape from its weight mask + Infer its output shape from its input shape + Infer its input shape from its output shape + + If its input shape is changed, continue infering its predecessors + If its output shape is changed, continue infering its successors + + Parameters + ---------- + module_name : str + The name of the node + last_module : str + The name of last visited node + mask : tensor of mask or ModuleMasks + Mask of the weights in this node (i.e., module) + in_shape : ModuleMasks + Input shape of this node + out_shape : ModuleMasks + Output shape of this node + """ + input_cmask = output_cmask = None + if module_name in self.inferred_masks: + module_masks = self.inferred_masks[module_name] + else: + _, m = get_module_by_name(self.bound_model, module_name) + module_masks = ModuleMasks(module_name, m) + self.inferred_masks[module_name] = module_masks + + m_type = self.torch_graph.name_to_node[module_name].op_type + _logger.debug("infer mask of module %s with op_type %s", module_name, m_type) + if mask is not None: + _logger.debug("mask is not None") + if not m_type in infer_from_mask: + raise RuntimeError( + "Has not supported infering input/output shape from mask for module/function: `{}`, {}" + .format(m_type, module_name)) + if m_type in ['Linear']: + input_cmask, output_cmask = infer_from_mask[m_type]( + module_masks, mask, self.torch_graph.name_to_node[module_name].auxiliary + ) + else: + input_cmask, output_cmask = infer_from_mask[m_type](module_masks, mask) + if in_shape is not None: + _logger.debug("in_shape is not None") + if not m_type in infer_from_inshape: + raise RuntimeError( + "Has not supported infering output shape from input shape for module/function: `{}`, {}" + .format(m_type, module_name)) + if m_type in ['aten::view', 'aten::flatten', 'aten::mean', 'aten::reshape']: + output_cmask = infer_from_inshape[m_type](module_masks, + in_shape, + self.torch_graph.name_to_node[module_name].auxiliary) + elif m_type in ['aten::cat']: + # To calculate the mask for concat operation, the output shape + # , cat dimension, and the order of the input parameters. + output_cmask = infer_from_inshape[m_type](module_masks, + in_shape, + self.torch_graph.name_to_node[module_name].auxiliary, + last_module) + else: + output_cmask = infer_from_inshape[m_type](module_masks, in_shape) + if out_shape is not None: + _logger.debug("out_shape is not None") + if not m_type in infer_from_outshape: + raise RuntimeError( + "Has not supported infering input shape from output shape for module/function: `{}`, {}" + .format(m_type, module_name)) + if m_type in ['aten::view', 'aten::flatten', 'aten::mean', 'aten::reshape']: + input_cmask = infer_from_outshape[m_type](module_masks, out_shape, self.torch_graph.name_to_node[module_name].auxiliary) + else: + input_cmask = infer_from_outshape[m_type](module_masks, out_shape) + + if input_cmask: + predecessors = self.torch_graph.find_predecessors(module_name) + for _module_name in predecessors: + self.infer_module_mask(_module_name, module_name, out_shape=input_cmask) + if output_cmask: + successors = self.torch_graph.find_successors(module_name) + for _module_name in successors: + self.infer_module_mask(_module_name, module_name, in_shape=output_cmask) + + def infer_modules_masks(self): + """ + Do shape inference of involved modules, including the shape of weights, inputs, output + """ + for module_name, mask in self.masks.items(): + _logger.debug('Start mask inference from %s', module_name) + if module_name not in self.torch_graph.name_to_node: + # this module is not traced in the torch_graph, + # jit.trace only correctly records functions and + # modules which are not data dependent (e.g., do + # not have conditionals on data in tensors) + # so, if a node is not traced, we just skip it. + _logger.warning('%s has mask, but not found in the traced graph, just skip it.', module_name) + continue + self.infer_module_mask(module_name, None, mask=mask) + + def replace_compressed_modules(self): + """ + Replace all the modules that have changed (weights/inputs/output) shape. + The new module is created using the same arguments of the to-be-replaced module, + and correctly inherits its weights. + + NOTE: ```func``` type cannot be replaced as it is not a module, thus, one limitation + is that ```func``` should be not required to be replaced. + """ + for module_name in self.inferred_masks: + g_node = self.torch_graph.name_to_node[module_name] + _logger.debug("replace %s, in %s type, with op_type %s", + module_name, g_node.type, g_node.op_type) + if g_node.type == 'module': + super_module, leaf_module = get_module_by_name(self.bound_model, g_node.name) + m_type = g_node.op_type + if not m_type in replace_module: + raise RuntimeError("Has not supported replacing the module: `{}`".format(m_type)) + _logger.info("replace module (name: %s, op_type: %s)", g_node.name, m_type) + compressed_module = replace_module[m_type](leaf_module, self.inferred_masks[module_name]) + setattr(super_module, g_node.name.split('.')[-1], compressed_module) + elif g_node.type == 'func': + _logger.info("Warning: cannot replace (name: %s, op_type: %s) which is func type", + module_name, g_node.op_type) + else: + raise RuntimeError("Unsupported node type: {}".format(g_node.type)) + + def speedup_model(self): + """ + There are basically two steps: + first, do mask/shape inference, + second, replace modules + """ + training = self.bound_model.training + _logger.info("start to speed up the model") + + _logger.info("fix the mask conflict of the interdependent layers") + _, conv_prune_dim = fix_mask_conflict(self.masks, self.bound_model, self.dummy_input) + set_conv_prune_dim(conv_prune_dim) + + _logger.info("infer module masks...") + self.infer_modules_masks() + _logger.info("replace compressed modules...") + self.replace_compressed_modules() + self.bound_model.train(training) + _logger.info("speedup done") diff --git a/new_impl/cv/third_party/nni/compression/pytorch/speedup/infer_shape.py b/new_impl/cv/third_party/nni/compression/pytorch/speedup/infer_shape.py new file mode 100644 index 0000000000000000000000000000000000000000..8a675cf731107f5addf5d693919d5bc06340a853 --- /dev/null +++ b/new_impl/cv/third_party/nni/compression/pytorch/speedup/infer_shape.py @@ -0,0 +1,1150 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. +""" +For each operation or module, there are two functions. +One is given output shape, infer its input shape and initialization parameters (e.g., weight's shape) +The other is given input shape, infer its output shape and initialization parameters (e.g., weight's shape) +""" + +import logging +import torch + +_logger = logging.getLogger(__name__) + +conv_prune_dim = -1 + +def set_conv_prune_dim(dim): + """ + Parameters: + dim: int + 0: filter pruning + 1: channel pruning + """ + global conv_prune_dim + conv_prune_dim = dim + +class CoarseMask: + """ + Coarse grained mask for a given tensor, here tensor could be weights, + input tensor, or output tensor + """ + + def __init__(self, num_dim): + """ + Parameters + ---------- + num_dim : int + The number of dimensions of the tensor that will be masked + """ + self.mask_index = [None for _ in range(num_dim)] + + def add_index_mask(self, dim, index): + """ + Add mask for the specified dimension + + Parameters + ---------- + dim : int + The dimension to add mask + index : tensor + The mask for this dimension, its a 1 dimension tensor which specifies + the index of the elements that are not pruned + """ + self.mask_index[dim] = index + + @staticmethod + def merge_index(index_a, index_b): + """ + Parameters + ---------- + index_a : tensor + One index (1-dimension) tensor + index_b : tensor + The other index (1-dimension) tensor + + Returns + ------- + tensor + The merged index (1-dimension) tensor + Note that: the output tensor will be moved + to the same device as index_a. + """ + device = index_a.device + s = set() + for num in index_a.tolist(): + # we need to transfer the tensor to list here + # first, directly traversing the tensor by for + # loop will return the list of tensor(x) object, + # even the value are the same, but they are different + # tensor objects, so the set will contains multiple + # tensor objects that has the same value. For example + # for num in torch.ones(2): + # s.add(num) + # s will be {tensor(1), tensor(1)} + s.add(num) + for num in index_b.tolist(): + s.add(num) + # move the output tensor to the same device with index_a + return torch.tensor(sorted(s)).to(device) # pylint: disable=not-callable + + def merge(self, cmask): + """ + Merge another CoarseMask + + Parameters + ---------- + cmask : CoarseMask + Another CoarseMask to merge + + Returns + ------- + list + The member variable ```mask_index``` + """ + assert isinstance(cmask, CoarseMask) + assert len(self.mask_index) == len(cmask.mask_index), \ + "Only masks with the same number of dimensions can be merged" + for i, index in enumerate(self.mask_index): + if index is None: + self.mask_index[i] = cmask.mask_index[i] + elif cmask.mask_index[i] is not None: + self.mask_index[i] = CoarseMask.merge_index(self.mask_index[i], + cmask.mask_index[i]) + return self.mask_index + + def __repr__(self): + return 'mask_index: {}'.format(self.mask_index) + + def eq_on_dim(self, other, dim): + assert isinstance(other, CoarseMask) + if self.mask_index[dim] is None and other.mask_index[dim] is None: + return True + elif isinstance(self.mask_index[dim], torch.Tensor) \ + and isinstance(other.mask_index[dim], torch.Tensor): + return torch.equal(self.mask_index[dim], other.mask_index[dim]) + else: + return False + + def __eq__(self, other): + assert isinstance(other, CoarseMask) + if len(self.mask_index) != len(other.mask_index): + return False + for i in range(len(self.mask_index)): + if not self.eq_on_dim(other, i): + return False + return True + + def __lt__(self, other): + """ + Judge if the mask is a subset of another CoarseMask. + """ + assert isinstance(other, CoarseMask) + for dim, _ in enumerate(self.mask_index): + # if self has more dimensions + if dim >= len(other.mask_index): + return False + if self.mask_index[dim] is None: + # if no mask on this dimension, then we have less + # masks then the other CoraseMask. + continue + elif other.mask_index[dim] is None: + return False + else: + s1 = set(self.mask_index[dim].tolist()) + s2 = set(other.mask_index[dim].tolist()) + if not s1 < s2: + return False + return True + + def __le__(self, other): + """ + Return if self's mask is less or equal to other's mask. + """ + assert isinstance(other, CoarseMask) + if self.__lt__(other) or self.__eq__(other): + return True + return False + + def __ne__(self, other): + return not self.__eq__(other) + + +class ModuleMasks: + """ + The masks of a module, including the masks for weights, inputs, output + """ + + def __init__(self, module_name, module=None): + """ + Parameters + ---------- + module_name : str + The name of the module or function + """ + self.module_name = module_name + self.module = module + self.param_masks = dict() + self.input_mask = None + self.output_mask = None + + def set_param_masks(self, name, mask): + """ + Parameters + ---------- + name : str + The name of the weight + mask : CoarseMask + The mask for this weight + """ + self.param_masks[name] = mask + + def set_input_mask(self, mask): + """ + Parameters + ---------- + mask : CoarseMask + The mask for input + """ + self.input_mask = mask + + def set_output_mask(self, mask): + """ + Parameters + ---------- + mask : CoarseMask + The mask for output + """ + self.output_mask = mask + + def __repr__(self): + return 'module_name: {}, input_mask: {}, output_mask: {}, param_masks: {}'.format( + self.module_name, self.input_mask, self.output_mask, self.param_masks + ) + + +""" +Infer input and output shape of a module/function from its weight mask +""" +infer_from_mask = { + 'BatchNorm2d': lambda module_masks, mask: batchnorm2d_mask(module_masks, mask), + 'Conv2d': lambda module_masks, mask: conv2d_mask(module_masks, mask), + 'Linear': lambda module_masks, mask, shape: linear_mask(module_masks, mask, shape), + + 'ConvTranspose2d': lambda module_masks, mask: convtranspose2d_mask(module_masks, mask) +} + +""" +Infer output and weight shape of a module/function from its input shape +""" +infer_from_inshape = { + 'ReLU': lambda module_masks, mask: relu_inshape(module_masks, mask), + 'ReLU6': lambda module_masks, mask: relu_inshape(module_masks, mask), + 'LeakyReLU': lambda module_masks, mask: relu_inshape(module_masks, mask), + 'Sigmoid': lambda module_masks, mask: relu_inshape(module_masks, mask), + 'aten::relu': lambda module_masks, mask: relu_inshape(module_masks, mask), + 'aten::tanh': lambda module_masks, mask: relu_inshape(module_masks, mask), + 'aten::tanh_': lambda module_masks, mask: relu_inshape(module_masks, mask), + 'aten::hardtanh': lambda module_masks, mask: relu_inshape(module_masks, mask), + 'aten::hardtanh_': lambda module_masks, mask: relu_inshape(module_masks, mask), + 'aten::relu_': lambda module_masks, mask: relu_inshape(module_masks, mask), + 'aten::sigmoid': lambda module_masks, mask: relu_inshape(module_masks, mask), + 'Conv2d': lambda module_masks, mask: conv2d_inshape(module_masks, mask), + 'ConvTranspose2d': lambda module_masks, mask: conv2dt_inshape(module_masks, mask), + 'MaxPool2d': lambda module_masks, mask: maxpool2d_inshape(module_masks, mask), + 'aten::max_pool2d': lambda module_masks, mask: maxpool2d_inshape(module_masks, mask), + 'aten::avg_pool2d': lambda module_masks, mask: maxpool2d_inshape(module_masks, mask), + 'aten::adaptive_avg_pool2d': lambda module_masks, mask: maxpool2d_inshape(module_masks, mask), + 'AvgPool2d': lambda module_masks, mask: maxpool2d_inshape(module_masks, mask), + 'AdaptiveAvgPool2d': lambda module_masks, mask: maxpool2d_inshape(module_masks, mask), + 'aten::size': lambda module_masks, mask: size_inshape(module_masks, mask), + 'aten::view': lambda module_masks, mask, shape: view_inshape(module_masks, mask, shape), + 'aten::reshape': lambda module_masks, mask, shape: view_inshape(module_masks, mask, shape), + # support only start_dim=1 + 'aten::flatten': lambda module_masks, mask, shape: view_inshape(module_masks, mask, shape), + 'Linear': lambda module_masks, mask: linear_inshape(module_masks, mask), + 'BatchNorm2d': lambda module_masks, mask: batchnorm2d_inshape(module_masks, mask), + 'aten::add_': lambda module_masks, mask: add_inshape(module_masks, mask), + 'aten::add': lambda module_mask, mask: add_inshape(module_mask, mask), + # mul has the similar behaviour with add, they both request + # the input tesors to have the same shape + 'aten::mul': lambda module_mask, mask: add_inshape(module_mask, mask), + 'aten::mul_': lambda module_mask, mask: add_inshape(module_mask, mask), + 'aten::cat': lambda module_mask, mask, cat_info, last_visited: cat_inshape(module_mask, mask, cat_info, last_visited), + 'aten::mean': lambda module_masks, mask, shape: mean_inshape(module_masks, mask, shape), + 'Dropout': lambda module_masks, mask: dropout_inshape(module_masks, mask), + 'Dropout2d': lambda module_masks, mask: dropout_inshape(module_masks, mask), + 'aten::dropout': lambda module_masks, mask: dropout_inshape(module_masks, mask), + + 'EltwiseAdd': lambda module_mask, mask: eltwise_add_inshape(module_mask, mask), + 'TwoAdd': lambda module_masks, mask: add_inshape(module_masks, mask), + + # yolov3 + 'ZeroPad2d': lambda module_masks, mask: maxpool2d_inshape(module_masks, mask), + + 'Tanh': lambda module_masks, mask: relu_inshape(module_masks, mask) +} + +""" +Infer input and weight shape of a module/function from its output shape +""" +infer_from_outshape = { + 'Conv2d': lambda module_masks, mask: conv2d_outshape(module_masks, mask), + 'BatchNorm2d': lambda module_masks, mask: batchnorm2d_outshape(module_masks, mask), + + 'MaxPool2d': lambda module_masks, mask: maxpool2d_outshape(module_masks, mask), + 'aten::max_pool2d': lambda module_masks, mask: maxpool2d_outshape(module_masks, mask), + 'aten::avg_pool2d': lambda module_masks, mask: maxpool2d_outshape(module_masks, mask), + 'aten::adaptive_avg_pool2d': lambda module_masks, mask: maxpool2d_outshape(module_masks, mask), + 'AvgPool2d': lambda module_masks, mask: maxpool2d_outshape(module_masks, mask), + 'AdaptiveAvgPool2d': lambda module_masks, mask: maxpool2d_outshape(module_masks, mask), + + 'ReLU': lambda module_masks, mask: relu_outshape(module_masks, mask), + 'ReLU6': lambda module_masks, mask: relu_outshape(module_masks, mask), + 'aten::relu': lambda module_masks, mask: relu_outshape(module_masks, mask), + 'aten::tanh': lambda module_masks, mask: relu_outshape(module_masks, mask), + 'aten::tanh_': lambda module_masks, mask: relu_outshape(module_masks, mask), + 'aten::hardtanh': lambda module_masks, mask: relu_outshape(module_masks, mask), + 'aten::hardtanh_': lambda module_masks, mask: relu_outshape(module_masks, mask), + 'aten::relu_': lambda module_masks, mask: relu_outshape(module_masks, mask), + + 'aten::add_': lambda module_masks, mask: add_outshape(module_masks, mask), + 'aten::add': lambda module_mask, mask: add_outshape(module_mask, mask), + 'aten::flatten': lambda module_mask, mask, shape: view_outshape(module_mask, mask, shape), + 'aten::view': lambda module_masks, mask, shape: view_outshape(module_masks, mask, shape), + 'aten::reshape': lambda module_masks, mask, shape: view_outshape(module_masks, mask, shape), + 'aten::mean': lambda module_masks, mask, shape: mean_outshape(module_masks, mask, shape), + 'Dropout': lambda module_masks, mask: dropout_outshape(module_masks, mask), + 'Dropout2d': lambda module_masks, mask: dropout_outshape(module_masks, mask), + 'aten::dropout': lambda module_masks, mask: dropout_outshape(module_masks, mask) +} + +def dropout_inshape(module_masks, mask): + if module_masks.input_mask is None: + module_masks.set_input_mask(mask) + module_masks.set_output_mask(mask) + return module_masks.output_mask + # if alreay visited + assert module_masks.input_mask <= mask + # It should be the same, we pass the masks by the reference(not the value), + # so they acutually are two references of the same object(mask, + # module_masks.input_mask). So we should continue pass the mask + # to the following nodes even module_masks.input_mask == mask. + # if pass the mask by copy.deepcopy(), then we can stop when + # module_masks.input_mask == mask. + # if module_masks.input_mask == mask: + # return None + module_masks.set_input_mask(mask) + module_masks.set_output_mask(mask) + return module_masks.output_mask + +def dropout_outshape(module_masks, mask): + if module_masks.output_mask is None: + module_masks.set_output_mask(mask) + module_masks.set_input_mask(mask) + return module_masks.input_mask + # if alreay visited + assert all(module_masks.output_mask.mask_index[1] == mask.mask_index[1]) + + return module_masks.output_mask + +def cat_inshape(module_masks, mask, cat_info, last_visited): + """ + Inference the output mask of the cat operation from the + input mask. + + Parameters + ---------- + module_masks : ModuleMasks + The ModuleMasks instance of the Conv2d + mask : CoarseMask + The mask of its input tensor + cat_info: dict + Dict object that records the necessary information + of cat operation, such as the order of the input + tensors. + last_visited: str + The unique_name of the last visited node group. + + Returns + ------- + CoarseMask + The mask of its output tensor + + """ + assert isinstance(mask, CoarseMask) + out_shape = cat_info['out_shape'] + cat_dim = cat_info['cat_dim'] + in_order = cat_info['in_order'] + in_shape = cat_info['in_shape'] + if module_masks.output_mask is None: + # First visit to this cat node + # initialize the mask based on + # the number of the output channel. + output_mask = CoarseMask(num_dim=len(out_shape)) + for dim, _ in enumerate(out_shape): + if dim == cat_dim: + if mask.mask_index[dim] is None: + continue + device = mask.mask_index[dim].device + # calculate the offset of the mask + pos = in_order.index(last_visited) + offsets = [in_shape[i][cat_dim] + for i, _ in enumerate(in_shape)] + offset = 0 + for i in range(pos): + offset += offsets[i] + _tmp_mask = (mask.mask_index[dim] + offset).to(device) + output_mask.mask_index[dim] = _tmp_mask + else: + # directly copy the mask + if mask.mask_index[dim] is not None: + output_mask.mask_index[dim] = mask.mask_index[dim].data.clone( + ) + module_masks.set_output_mask(output_mask) + + return module_masks.output_mask + # If this cat node is already visited, we need + # validating if the mask is legel, for cat operation, + # the mask on the 'cat_dim' dimension should be stitched + # together. In the other dimensions, the mask should be + # the same, else the mask is not legal. + for dim, _ in enumerate(out_shape): + if dim == cat_dim: + if mask.mask_index[dim] is None: + continue + pos = in_order.index(last_visited) + offsets = [in_shape[i][cat_dim] for i, _ in enumerate(in_shape)] + offset = 0 + for i in range(pos): + offset += offsets[i] + device = mask.mask_index[dim].device + new_mask = mask.mask_index[dim] + offset + module_masks.output_mask.mask_index[dim] = CoarseMask.merge_index( + module_masks.output_mask.mask_index[dim], new_mask).to(device) + else: + assert module_masks.output_mask.eq_on_dim(mask, dim) + + return module_masks.output_mask + + +def add_inshape(module_masks, mask): + """ + Inference the output mask of the add operation from the + input mask. + """ + assert isinstance(mask, CoarseMask) + if module_masks.input_mask is None: + module_masks.set_input_mask(mask) + module_masks.set_output_mask(mask) + # module_masks.input_mask = mask + return mask + # If alreay visited, validate if have the conflict + # if the mask is different with previous input_mask + # then there is a mask confilct. + if mask != module_masks.input_mask: + raise Exception('Mask conflict happenes!') + return None + + +def eltwise_add_inshape(module_masks, mask): + """ + Inference the output mask of the add operation from the + input mask. + """ + print(mask) + print(module_masks) + assert isinstance(mask, CoarseMask) + if module_masks.input_mask is None: + module_masks.set_input_mask(mask) + module_masks.set_output_mask(mask) + # module_masks.input_mask = mask + return mask + + # # If alreay visited, validate if have the conflict + # # if the mask is different with previous input_mask + # # then there is a mask confilct. + if mask != module_masks.input_mask: + raise Exception('Mask conflict happenes!') + return mask + + +def add_outshape(module_masks, mask): + """ + Inference the input mask of the add operation from the + output mask. + """ + assert isinstance(mask, CoarseMask) + + if module_masks.output_mask is None: + module_masks.set_output_mask(mask) + module_masks.set_input_mask(mask) + return mask + else: + assert all(module_masks.output_mask.mask_index[1] == mask.mask_index[1]) + return mask + +def batchnorm2d_inshape(module_masks, mask): + """ + We assume only the second dimension has coarse grained mask + + Parameters + ---------- + module_masks : ModuleMasks + The ModuleMasks instance of the batchnorm2d + mask : CoarseMask + The mask of its input tensor + + Returns + ------- + CoarseMask + The mask of its output tensor + """ + assert isinstance(mask, CoarseMask) + assert mask.mask_index[1] is not None + assert mask.mask_index[0] is None + assert mask.mask_index[2] is None + assert mask.mask_index[3] is None + module_masks.set_input_mask(mask) + module_masks.set_output_mask(mask) + weight_cmask = CoarseMask(num_dim=1) + weight_cmask.add_index_mask(dim=0, index=mask.mask_index[1]) + module_masks.set_param_masks('weight', weight_cmask) + module_masks.set_param_masks('bias', weight_cmask) + return mask + +def batchnorm2d_outshape(module_masks, mask): + """ + We assume only the second dimension has coarse grained mask + + Parameters + ---------- + module_masks : ModuleMasks + The ModuleMasks instance of the batchnorm2d + mask : CoarseMask + The mask of its input tensor + + Returns + ------- + CoarseMask + The mask of its output tensor + """ + assert isinstance(mask, CoarseMask) + assert len(mask.mask_index) in [2, 4] + assert mask.mask_index[1] is not None + assert mask.mask_index[0] is None + module_masks.set_input_mask(mask) + module_masks.set_output_mask(mask) + weight_cmask = CoarseMask(num_dim=1) + weight_cmask.add_index_mask(dim=0, index=mask.mask_index[1]) + module_masks.set_param_masks('weight', weight_cmask) + module_masks.set_param_masks('bias', weight_cmask) + return mask + + +def linear_inshape(module_masks, mask): + """ + Coarse grained input mask does not change the shape of weights and output tensor + + Parameters + ---------- + module_masks : ModuleMasks + The ModuleMasks instance of the linear + mask : CoarseMask + The mask of its input tensor + + Returns + ------- + CoarseMask + The mask of its output tensor, ```None``` means shape of output tensor is not changed + """ + assert isinstance(mask, CoarseMask) + assert mask.mask_index[0] is None + if module_masks.input_mask is not None: + assert module_masks.input_mask <= mask + module_masks.set_input_mask(mask) + return None + + +def view_inshape(module_masks, mask, shape): + """ + This is a limited support + + TODO: consider replace tensor.view with nn.Flatten, because tensor.view is not + included in module, thus, cannot be replaced by our framework. + + Parameters + ---------- + module_masks : ModuleMasks + The ModuleMasks instance of the ```view``` op + mask : CoarseMask + The mask of its input tensor + shape : dict + Original shape of its input and output tensors + + Returns + ------- + CoarseMask + The mask of its output tensor + """ + # NOTE: the case constrained by the following four asserts + assert shape['in_shape'][0] == shape['out_shape'][0] + assert len(shape['in_shape']) == 4 + assert len(shape['out_shape']) == 2 + assert shape['out_shape'][1] == shape['in_shape'][1] * \ + shape['in_shape'][2]*shape['in_shape'][3] + + assert isinstance(mask, CoarseMask) + assert mask.mask_index[1] is not None + assert mask.mask_index[0] is None + assert mask.mask_index[2] is None + assert mask.mask_index[3] is None + # due to the cat operation, the same node may be + # accessed more than once + if module_masks.input_mask is not None: + assert module_masks.input_mask <= mask + module_masks.set_input_mask(mask) + output_cmask = CoarseMask(num_dim=2) + index = [] + step_size = shape['in_shape'][2] * shape['in_shape'][3] + for loc in mask.mask_index[1]: + index.extend([loc * step_size + i for i in range(step_size)]) + output_cmask.add_index_mask(dim=1, index=torch.tensor(index).to(mask.mask_index[1].device)) # pylint: disable=not-callable + module_masks.set_output_mask(output_cmask) + return output_cmask + +def view_outshape(module_masks, mask, shape): + """ + Parameters + ---------- + module_masks : ModuleMasks + The ModuleMasks instance of the ```flatten``` op + mask : CoarseMask + The mask of its input tensor + shape : dict + Original shape of its input and output tensors + Returns + ------- + CoarseMask + The mask of its output tensor + """ + # NOTE: the case constrained by the following four asserts + assert shape['in_shape'][0] == shape['out_shape'][0] + assert len(shape['in_shape']) == 4 + assert len(shape['out_shape']) == 2 + assert shape['out_shape'][1] == shape['in_shape'][1] * \ + shape['in_shape'][2]*shape['in_shape'][3] + + assert isinstance(mask, CoarseMask) + assert mask.mask_index[1] is not None + assert mask.mask_index[0] is None + + module_masks.set_output_mask(mask) + input_cmask = CoarseMask(num_dim=4) + index = [] + step_size = shape['in_shape'][2] * shape['in_shape'][3] + for loc in mask.mask_index[1]: + index.extend([loc * step_size + i for i in range(step_size)]) + input_cmask.add_index_mask(dim=1, index=torch.tensor(index).to(mask.mask_index[1].device)) # pylint: disable=not-callable + module_masks.set_input_mask(input_cmask) + + return input_cmask + +def size_inshape(module_masks, mask): + """ + No need to do anything for this ```size``` op + """ + return None + +def mean_inshape(module_masks, mask, shape): + """ + Similar to view operation, currently mask inference only supports + the mean operation on the 3rd and 4th dimensions. + """ + assert shape['in_shape'][0] == shape['out_shape'][0] + assert shape['out_shape'][1] == shape['in_shape'][1] + assert len(shape['in_shape']) == 4 + assert len(shape['out_shape']) == 2 + + assert isinstance(mask, CoarseMask) + assert mask.mask_index[1] is not None + assert mask.mask_index[0] is None + assert mask.mask_index[2] is None + assert mask.mask_index[3] is None + module_masks.set_input_mask(mask) + + output_cmask = CoarseMask(num_dim=2) + output_cmask.add_index_mask(dim=1, index=mask.mask_index[1]) + module_masks.set_output_mask(output_cmask) + return output_cmask + +def mean_outshape(module_masks, mask, shape): + """ + Similar to view operation, currently mask inference only supports + the mean operation on the 3rd and 4th dimensions. + """ + assert shape['in_shape'][0] == shape['out_shape'][0] + assert shape['out_shape'][1] == shape['in_shape'][1] + assert len(shape['in_shape']) == 4 + assert len(shape['out_shape']) == 2 + + assert isinstance(mask, CoarseMask) + assert mask.mask_index[1] is not None + assert mask.mask_index[0] is None + module_masks.set_output_mask(mask) + + input_cmask = CoarseMask(num_dim=4) + input_cmask.add_index_mask(dim=1, index=mask.mask_index[1]) + module_masks.set_input_mask(input_cmask) + return input_cmask + +def maxpool2d_inshape(module_masks, mask): + """ + Assume only the second dimension is masked + + Parameters + ---------- + module_masks : ModuleMasks + The ModuleMasks instance of the maxpool2d + mask : CoarseMask + The mask of its input tensor + + Returns + ------- + CoarseMask + The mask of its output tensor + """ + assert isinstance(mask, CoarseMask) + assert mask.mask_index[1] is not None + assert mask.mask_index[0] is None + assert mask.mask_index[2] is None + assert mask.mask_index[3] is None + if module_masks.input_mask is not None: + assert module_masks.input_mask <= mask + # assert module_masks.input_mask is None + module_masks.set_input_mask(mask) + module_masks.set_output_mask(mask) + return mask + +def maxpool2d_outshape(module_masks, mask): + """ + Assume only the second dimension is masked + + Parameters + ---------- + module_masks : ModuleMasks + The ModuleMasks instance of the maxpool2d + mask : CoarseMask + The mask of its input tensor + + Returns + ------- + CoarseMask + The mask of its output tensor + """ + assert isinstance(mask, CoarseMask) + assert mask.mask_index[1] is not None + assert mask.mask_index[0] is None + + module_masks.set_input_mask(mask) + module_masks.set_output_mask(mask) + return mask + +def relu_inshape(module_masks, mask): + """ + Parameters + ---------- + module_masks : ModuleMasks + The ModuleMasks instance of the relu + mask : CoarseMask + The mask of its input tensor + + Returns + ------- + CoarseMask + The mask of its output tensor + """ + assert isinstance(mask, CoarseMask) + if module_masks.input_mask is not None: + # mask conflict should be solved before speedup + assert module_masks.input_mask <= mask + # assert module_masks.input_mask is None, "A relu op can only be processed once" + module_masks.set_input_mask(mask) + module_masks.set_output_mask(mask) + return mask + +def relu_outshape(module_masks, mask): + """ + Parameters + ---------- + module_masks : ModuleMasks + The ModuleMasks instance of the relu + mask : CoarseMask + The mask of its input tensor + + Returns + ------- + CoarseMask + The mask of its output tensor + """ + assert isinstance(mask, CoarseMask) + if module_masks.output_mask is not None: + # mask conflict should be solved before speedup + assert all(module_masks.output_mask.mask_index[1] == mask.mask_index[1]) + module_masks.set_input_mask(mask) + module_masks.set_output_mask(mask) + return mask + +def batchnorm2d_mask(module_masks, mask): + """ + Infer input and output shape from weight mask + Parameters + ---------- + module_masks : ModuleMasks + The ModuleMasks instance of the batchnorm2d + mask : dict + The mask of its weights, from the user provided mask file + Returns + ------- + CoarseMask, CoarseMask + The mask of its input tensor, the mask of its output tensor + """ + assert 'weight' in mask and 'bias' in mask + sum_mask = mask['weight'] + mask['bias'] + nonzero_index = torch.nonzero(sum_mask, as_tuple=True)[0] + # infer shape of parameters + param_cmask = CoarseMask(num_dim=1) + param_cmask.add_index_mask(dim=0, index=nonzero_index) + module_masks.set_param_masks('weight', param_cmask) + module_masks.set_param_masks('bias', param_cmask) + # infer shape of input tensor + input_cmask = CoarseMask(num_dim=4) + input_cmask.add_index_mask(dim=1, + index=torch.nonzero(mask['weight'], as_tuple=True)[0]) + module_masks.set_input_mask(input_cmask) + # infer shape of output tensor + output_cmask = CoarseMask(num_dim=4) + output_cmask.add_index_mask(dim=1, index=nonzero_index) + module_masks.set_output_mask(output_cmask) + return input_cmask, output_cmask + +def linear_mask(module_masks, mask, shape): + """ + Infer input and output shape from weight mask with limitations: + Only support infer input mask + + Parameters + ---------- + module_masks : ModuleMasks + The ModuleMasks instance of the Linear + mask : dict + The mask of its weights, from the user provided mask file + shape: dict + Shape of its input and output tensors + Returns + ------- + CoarseMask, CoarseMask + The mask of its input tensor, the mask of its output tensor + """ + + assert 'weight' in mask + num_input_dim = len(shape['in_shape']) + + # Input data of Linear module can have multiple dimensions. + # here we only support infer coarse mask on the first dimension (dimension 0) + nonzero_index = torch.nonzero(mask['weight'].sum(0), as_tuple=True)[0] + + # infer shape of input tensor + input_cmask = CoarseMask(num_dim=num_input_dim) + input_cmask.add_index_mask(dim=num_input_dim-1, index=nonzero_index) + + module_masks.set_input_mask(input_cmask) + return input_cmask, None + +def conv2d_mask(module_masks, mask): + """ + Infer input and output shape from weight mask + + Parameters + ---------- + module_masks : ModuleMasks + The ModuleMasks instance of the conv2d + mask : dict + The mask of its weights, from the user provided mask file + + Returns + ------- + CoarseMask, CoarseMask + The mask of its input tensor, the mask of its output tensor + """ + def convert_to_coarse_mask(mask, dim=0): + """ + Parameters + ---------- + mask : dict + Weight mask from user provided mask file + dim: int + 0: filter pruning + 1: channel pruning + + Returns + ------- + LongTensor, CoarseMask, CoarseMask + Index of the masked dimension, weight mask, bias mask + """ + assert 'weight' in mask + assert isinstance(mask['weight'], torch.Tensor) + assert dim in [0, 1] + + weight_mask = mask['weight'] + + sum_idx = (1, 2, 3) if dim == 0 else (0, 2, 3) + index = torch.nonzero(weight_mask.abs().sum(sum_idx) != 0, as_tuple=True)[0] + if len(index) == weight_mask.shape[dim]: # full mask + index = None + + if index is None: + return None, None, None + else: + index = index.long().to(weight_mask.device) + weight_cmask = CoarseMask(num_dim=4) + weight_cmask.add_index_mask(dim=dim, index=index) + bias_cmask = None + if dim == 0 and 'bias' in mask and mask['bias'] is not None: + bias_index = torch.nonzero(mask['bias'], as_tuple=True)[0] + assert torch.all(torch.eq(index, bias_index)), \ + "bias mask should be consistent with weight mask" + bias_cmask = CoarseMask(num_dim=1) + bias_cmask.add_index_mask(dim=0, index=bias_index) + return index, weight_cmask, bias_cmask + + index, weight_cmask, bias_cmask = convert_to_coarse_mask(mask, dim=conv_prune_dim) + + if index is None: + # TODO: fine grained mask speedup + return None, None + # deal with coarse grain mask + # mask conflict should be solved by fix_mask_conflict before speedup + if 'weight' in module_masks.param_masks: + assert module_masks.param_masks['weight'] == weight_cmask + else: + module_masks.set_param_masks('weight', weight_cmask) + if conv_prune_dim == 0: + module_masks.set_param_masks('bias', bias_cmask) + + io_cmask = CoarseMask(num_dim=4) + io_cmask.add_index_mask(dim=1, index=index) + + if conv_prune_dim == 0: + if module_masks.output_mask is None: + module_masks.set_output_mask(io_cmask) + else: + assert module_masks.output_mask == io_cmask + return None, module_masks.output_mask + else: + if module_masks.input_mask is None: + module_masks.set_input_mask(io_cmask) + else: + assert module_masks.input_mask == io_cmask + return module_masks.input_mask, None + + +def convtranspose2d_mask(module_masks, mask): + # TODO support the Convtranspose2d Pruning for the L1FilterPruner + # raise Exception( + # "Current Filter pruner cannot prune the ConvTranspose2d, will support pruning ConvTranspose2d later") + """ + Infer input and output shape from weight mask + + Parameters + ---------- + module_masks : ModuleMasks + The ModuleMasks instance of the conv2d + mask : dict + The mask of its weights, from the user provided mask file + + Returns + ------- + CoarseMask, CoarseMask + The mask of its input tensor, the mask of its output tensor + """ + def convert_to_coarse_mask(mask, dim=0): + """ + Parameters + ---------- + mask : dict + Weight mask from user provided mask file + dim: int + 0: filter pruning + 1: channel pruning + + Returns + ------- + LongTensor, CoarseMask, CoarseMask + Index of the masked dimension, weight mask, bias mask + """ + assert 'weight' in mask + assert isinstance(mask['weight'], torch.Tensor) + assert dim in [0, 1] + + weight_mask = mask['weight'] + + sum_idx = (1, 2, 3) if dim == 0 else (0, 2, 3) + index = torch.nonzero(weight_mask.abs().sum(sum_idx) != 0, as_tuple=True)[0] + if len(index) == weight_mask.shape[dim]: # full mask + index = None + + if index is None: + return None, None, None + else: + index = index.long().to(weight_mask.device) + weight_cmask = CoarseMask(num_dim=4) + weight_cmask.add_index_mask(dim=dim, index=index) + bias_cmask = None + if dim == 0 and 'bias' in mask and mask['bias'] is not None: + bias_index = torch.nonzero(mask['bias'], as_tuple=True)[0] + assert torch.all(torch.eq(index, bias_index)), \ + "bias mask should be consistent with weight mask" + bias_cmask = CoarseMask(num_dim=1) + bias_cmask.add_index_mask(dim=0, index=bias_index) + return index, weight_cmask, bias_cmask + + convt_prune_dim = 1 + index, weight_cmask, bias_cmask = convert_to_coarse_mask(mask, dim=convt_prune_dim) + + if index is None: + # TODO: fine grained mask speedup + return None, None + # deal with coarse grain mask + # mask conflict should be solved by fix_mask_conflict before speedup + if 'weight' in module_masks.param_masks: + assert module_masks.param_masks['weight'] == weight_cmask + else: + module_masks.set_param_masks('weight', weight_cmask) + if conv_prune_dim == 0: + module_masks.set_param_masks('bias', bias_cmask) + + io_cmask = CoarseMask(num_dim=4) + io_cmask.add_index_mask(dim=1, index=index) + + if conv_prune_dim == 0: + if module_masks.output_mask is None: + module_masks.set_output_mask(io_cmask) + else: + assert module_masks.output_mask == io_cmask + return None, module_masks.output_mask + else: + if module_masks.input_mask is None: + module_masks.set_input_mask(io_cmask) + else: + assert module_masks.input_mask == io_cmask + return module_masks.input_mask, None + + +def conv2d_inshape(module_masks, mask): + """ + Shape change of input tensor does not affect the shape of its output tensor + Parameters + ---------- + module_masks : ModuleMasks + The ModuleMasks instance of the conv2d + mask : CoarseMask + The mask of its input tensor + Returns + ------- + CoarseMask + The mask of its output tensor + """ + assert isinstance(mask, CoarseMask) + if module_masks.input_mask is None: + module_masks.set_input_mask(mask) + else: + # the same conv layer may be accessed more + # than once, such as a concat operation. + # mask conflict should be solved by fix_mask_conflict before speedup + assert module_masks.input_mask == mask + + # shape changes pass through depths wise conv layers + m = module_masks.module + if m.in_channels == m.out_channels == m.groups: + module_masks.output_mask = mask + module_masks.input_mask = mask + return mask + return None + + +def conv2dt_inshape(module_masks, mask): + """ + Shape change of input tensor does not affect the shape of its output tensor + Parameters + ---------- + module_masks : ModuleMasks + The ModuleMasks instance of the conv2d + mask : CoarseMask + The mask of its input tensor + Returns + ------- + CoarseMask + The mask of its output tensor + """ + assert isinstance(mask, CoarseMask) + if module_masks.input_mask is None: + module_masks.set_input_mask(mask) + else: + # the same conv layer may be accessed more + # than once, such as a concat operation. + # mask conflict should be solved by fix_mask_conflict before speedup + assert module_masks.input_mask == mask + + # shape changes pass through depths wise conv layers + m = module_masks.module + if m.in_channels == m.out_channels == m.groups: + module_masks.output_mask = mask + module_masks.input_mask = mask + return mask + return None + + +def conv2d_outshape(module_masks, mask): + """ + Assume only the second dimension is masked + + Parameters + ---------- + module_masks : ModuleMasks + The ModuleMasks instance of the conv2d + mask : CoarseMask + The mask of its output tensor + + Returns + ------- + CoarseMask + The mask of its input tensor + """ + assert isinstance(mask, CoarseMask) + assert mask.mask_index[1] is not None + assert mask.mask_index[0] is None + assert mask.mask_index[2] is None + assert mask.mask_index[3] is None + + if module_masks.output_mask is None: + module_masks.output_mask = mask + else: + # mask conflict should be solved by fix_mask_conflict before speedup + # mask and module_masks.output_mask may have different number of dimensions + # since they could be passed by linear or conv2d + assert all(module_masks.output_mask.mask_index[1] == mask.mask_index[1]) + + weight_cmask = CoarseMask(num_dim=4) + weight_cmask.add_index_mask(dim=0, index=mask.mask_index[1]) + bias_cmask = CoarseMask(num_dim=1) + bias_cmask.add_index_mask(dim=0, index=mask.mask_index[1]) + module_masks.set_param_masks('weight', weight_cmask) + module_masks.set_param_masks('bias', bias_cmask) + + # shape changes pass through depths wise conv layers + m = module_masks.module + if m.in_channels == m.out_channels == m.groups: + module_masks.output_mask = mask + module_masks.input_mask = mask + return mask + return None diff --git a/new_impl/cv/third_party/nni/compression/pytorch/utils/__init__.py b/new_impl/cv/third_party/nni/compression/pytorch/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/new_impl/cv/third_party/nni/compression/pytorch/utils/config_validation.py b/new_impl/cv/third_party/nni/compression/pytorch/utils/config_validation.py new file mode 100644 index 0000000000000000000000000000000000000000..835cdf17f6f3daa7f5fac82d4a8ca8a2719fcbb0 --- /dev/null +++ b/new_impl/cv/third_party/nni/compression/pytorch/utils/config_validation.py @@ -0,0 +1,54 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from schema import Schema, And, SchemaError + +def validate_op_names(model, op_names, logger): + found_names = set(map(lambda x: x[0], model.named_modules())) + + not_found_op_names = list(set(op_names) - found_names) + if not_found_op_names: + logger.warning('op_names %s not found in model', not_found_op_names) + + return True + +def validate_op_types(model, op_types, logger): + found_types = set(['default']) | set(map(lambda x: type(x[1]).__name__, model.named_modules())) + + not_found_op_types = list(set(op_types) - found_types) + if not_found_op_types: + # logger.warning('op_types %s not found in model', not_found_op_types) + pass + + return True + +def validate_op_types_op_names(data): + if not ('op_types' in data or 'op_names' in data): + raise SchemaError('Either op_types or op_names must be specified.') + return True + +class CompressorSchema: + def __init__(self, data_schema, model, logger): + assert isinstance(data_schema, list) and len(data_schema) <= 1 + self.data_schema = data_schema + self.compressor_schema = Schema(self._modify_schema(data_schema, model, logger)) + + def _modify_schema(self, data_schema, model, logger): + if not data_schema: + return data_schema + + for k in data_schema[0]: + old_schema = data_schema[0][k] + if k == 'op_types' or (isinstance(k, Schema) and k._schema == 'op_types'): + new_schema = And(old_schema, lambda n: validate_op_types(model, n, logger)) + data_schema[0][k] = new_schema + if k == 'op_names' or (isinstance(k, Schema) and k._schema == 'op_names'): + new_schema = And(old_schema, lambda n: validate_op_names(model, n, logger)) + data_schema[0][k] = new_schema + + data_schema[0] = And(data_schema[0], lambda d: validate_op_types_op_names(d)) + + return data_schema + + def validate(self, data): + self.compressor_schema.validate(data) diff --git a/new_impl/cv/third_party/nni/compression/pytorch/utils/counter.py b/new_impl/cv/third_party/nni/compression/pytorch/utils/counter.py new file mode 100644 index 0000000000000000000000000000000000000000..6061e8a8d0a4adea3401fbeab6dd50e8f5f3073a --- /dev/null +++ b/new_impl/cv/third_party/nni/compression/pytorch/utils/counter.py @@ -0,0 +1,135 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import torch +import torch.nn as nn +from nni.compression.pytorch.compressor import PrunerModuleWrapper + +try: + from thop import profile +except Exception as e: + print('thop is not found, please install the python package: thop') + raise + + +def count_flops_params(model: nn.Module, input_size, custom_ops=None, verbose=True): + """ + Count FLOPs and Params of the given model. + This function would identify the mask on the module + and take the pruned shape into consideration. + Note that, for sturctured pruning, we only identify + the remained filters according to its mask, which + not taking the pruned input channels into consideration, + so the calculated FLOPs will be larger than real number. + + Parameters + --------- + model : nn.Module + target model. + input_size: list, tuple + the input shape of data + custom_ops: dict + a mapping of (module: custom operation) + the custom operation will overwrite the default operation. + for reference, please see ``custom_mask_ops``. + + Returns + ------- + flops: float + total flops of the model + params: + total params of the model + """ + + assert input_size is not None + + device = next(model.parameters()).device + inputs = torch.randn(input_size).to(device) + + hook_module_list = [] + if custom_ops is None: + custom_ops = {} + custom_mask_ops.update(custom_ops) + prev_m = None + for m in model.modules(): + weight_mask = None + m_type = type(m) + if m_type in custom_mask_ops: + if isinstance(prev_m, PrunerModuleWrapper): + weight_mask = prev_m.weight_mask + + m.register_buffer('weight_mask', weight_mask) + hook_module_list.append(m) + prev_m = m + + flops, params = profile(model, inputs=(inputs, ), custom_ops=custom_mask_ops, verbose=verbose) + + + for m in hook_module_list: + m._buffers.pop("weight_mask") + # Remove registerd buffer on the model, and fixed following issue: + # https://github.com/Lyken17/pytorch-OpCounter/issues/96 + for m in model.modules(): + if 'total_ops' in m._buffers: + m._buffers.pop("total_ops") + if 'total_params' in m._buffers: + m._buffers.pop("total_params") + + return flops, params + +def count_convNd_mask(m, x, y): + """ + The forward hook to count FLOPs and Parameters of convolution operation. + Parameters + ---------- + m : torch.nn.Module + convolution module to calculate the FLOPs and Parameters + x : torch.Tensor + input data + y : torch.Tensor + output data + """ + output_channel = y.size()[1] + output_size = torch.zeros(y.size()[2:]).numel() + kernel_size = torch.zeros(m.weight.size()[2:]).numel() + + bias_flops = 1 if m.bias is not None else 0 + + if m.weight_mask is not None: + output_channel = m.weight_mask.sum() // (m.in_channels * kernel_size) + + total_ops = output_channel * output_size * (m.in_channels // m.groups * kernel_size + bias_flops) + + m.total_ops += torch.DoubleTensor([int(total_ops)]) + + +def count_linear_mask(m, x, y): + """ + The forward hook to count FLOPs and Parameters of linear transformation. + Parameters + ---------- + m : torch.nn.Module + linear to calculate the FLOPs and Parameters + x : torch.Tensor + input data + y : torch.Tensor + output data + """ + output_channel = y.numel() + + bias_flops = 1 if m.bias is not None else 0 + + if m.weight_mask is not None: + output_channel = m.weight_mask.sum() // m.in_features + + total_ops = output_channel * (m.in_features + bias_flops) + + m.total_ops += torch.DoubleTensor([int(total_ops)]) + + +custom_mask_ops = { + nn.Conv1d: count_convNd_mask, + nn.Conv2d: count_convNd_mask, + nn.Conv3d: count_convNd_mask, + nn.Linear: count_linear_mask, +} diff --git a/new_impl/cv/third_party/nni/compression/pytorch/utils/mask_conflict.py b/new_impl/cv/third_party/nni/compression/pytorch/utils/mask_conflict.py new file mode 100644 index 0000000000000000000000000000000000000000..dff2e97c63c30d268c59c1fe4859c8d5e24918f4 --- /dev/null +++ b/new_impl/cv/third_party/nni/compression/pytorch/utils/mask_conflict.py @@ -0,0 +1,370 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. +import os +import logging +import torch +import numpy as np +from .shape_dependency import ChannelDependency, GroupDependency, CatPaddingDependency, InputChannelDependency +from .utils import get_module_by_name +# logging.basicConfig(level = logging.DEBUG) +_logger = logging.getLogger(__name__) + +def fix_mask_conflict(masks, model=None, dummy_input=None, traced=None): + """ + MaskConflict fix the mask conflict for the channel dependencies + and group dependency. + + Parameters + ---------- + masks : dict/str + A dict object that stores the masks or the path of the mask file + model : torch.nn.Module + model to fix the mask conflict + dummy_input : torch.Tensor + input example to trace the model + traced : torch._C.torch.jit.TopLevelTracedModule + the traced model of the target model, is this parameter is not None, + we donnot use the model and dummpy_input to get the trace graph. + """ + if isinstance(masks, str): + # if the input is the path of the mask_file + assert os.path.exists(masks) + masks = torch.load(masks) + # if the user uses the model and dummy_input to trace the model, we + # should get the traced model handly, so that, we only trace the + # model once, GroupMaskConflict and ChannelMaskConflict will reuse + # this traced model. + if traced is None: + assert model is not None and dummy_input is not None + with torch.onnx.set_training(model, False): + # We need to trace the model in this way, else it will have problems + traced = torch.jit.trace(model, dummy_input, check_trace=False) + + fix_group_mask = GroupMaskConflict(masks, model, dummy_input, traced) + masks = fix_group_mask.fix_mask() + fix_channel_mask = ChannelMaskConflict(masks, model, dummy_input, traced) + masks = fix_channel_mask.fix_mask() + padding_cat_mask = CatMaskPadding(masks, model, dummy_input, traced) + masks = padding_cat_mask.fix_mask() + return masks, fix_channel_mask.conv_prune_dim + +class MaskFix: + def __init__(self, masks, model=None, dummy_input=None, traced=None): + # check if the parameters are valid + parameter_valid = False + if traced is not None: + parameter_valid = True + elif (model is not None) and (dummy_input is not None): + parameter_valid = True + if not parameter_valid: + raise Exception('The input parameters is invalid!') + self.model = model + self.dummy_input = dummy_input + self.traced = traced + self.masks = masks + + def fix_mask(self): + raise NotImplementedError + + def export(self, path): + """ + Export the masks after fixing the conflict to file. + """ + torch.save(self.masks, path) + +class CatMaskPadding(MaskFix): + def __init__(self, masks, model, dummy_input=None, traced=None): + """ + CatMaskPadding find the layers whose output tensor is passed + to the same cat operation. The cat operation concatnates the + masks of the input tensors as the output mask, so when some + of the input layers of the cat operation are not pruned, we still + need to pass the masks of these non-pruned layers(the mask are + all ones) to the cat operation to ensure the shape of the output + mask is right. + + Parameters + ---------- + masks : dict + a dict object that stores the masks + model : torch.nn.Module + model to fix the mask conflict + dummy_input : torch.Tensor + input example to trace the model + traced : torch._C.torch.jit.TopLevelTracedModule + the traced model of the target model, is this parameter is not None, + we donnot use the model and dummpy_input to get the trace graph. + """ + super(CatMaskPadding, self).__init__(masks, model, dummy_input, traced) + + def fix_mask(self): + cat_padding_depen = CatPaddingDependency(self.model, self.dummy_input, self.traced) + name_to_module = {} + for name, module in self.model.named_modules(): + name_to_module[name] = module + depen = cat_padding_depen.dependency_sets + for layers in depen: + device = None + count = 0 + for layer in layers: + if layer in self.masks: + count += 1 + if device is None: + device = self.masks[layer]['weight'].device + if count == 0: + # no layer is pruned + continue + elif count == len(layers): + # all the layers have been pruned + continue + # pad the mask for the non-pruned layers + for layer in layers: + if layer in self.masks: + continue + module = name_to_module[layer] + w_shape = module.weight.data.size() + w_mask = torch.ones(w_shape).to(device) + b_mask = None + if hasattr(module, 'bias') and module.bias is not None: + # module.bias may be None + b_shape = module.bias.data.size() + b_mask = torch.ones(b_shape).to(device) + self.masks[layer] = {'weight':w_mask, 'bias':b_mask} + return self.masks + + + +class GroupMaskConflict(MaskFix): + def __init__(self, masks, model=None, dummy_input=None, traced=None): + """ + GroupMaskConflict fix the mask conflict between the layers that + has group dependecy with each other. + + Parameters + ---------- + masks : dict + a dict object that stores the masks + model : torch.nn.Module + model to fix the mask conflict + dummy_input : torch.Tensor + input example to trace the model + traced : torch._C.torch.jit.TopLevelTracedModule + the traced model of the target model, is this parameter is not None, + we donnot use the model and dummpy_input to get the trace graph. + """ + super(GroupMaskConflict, self).__init__(masks, model, dummy_input, traced) + + + def fix_mask(self): + """ + Fix the mask conflict before the mask inference for the layers that + has group dependencies. This function should be called before the + mask inference of the 'speedup' module. + """ + group_depen = GroupDependency(self.model, self.dummy_input, self.traced) + depens = group_depen.dependency + _logger.info(depens) + for layername in depens: + group = depens[layername] + if layername not in self.masks: + # this layer not pruned + continue + w_mask = self.masks[layername]['weight'] + shape = w_mask.size() + count = np.prod(shape[1:]) + all_ones = (w_mask.flatten(1).sum(-1) == count).nonzero().squeeze(1).tolist() + all_zeros = (w_mask.flatten(1).sum(-1) == 0).nonzero().squeeze(1).tolist() + if len(all_ones) + len(all_zeros) < w_mask.size(0): + # In fine-grained pruning, skip this layer + _logger.info('Layers %s using fine-grained pruning', layername) + continue + assert shape[0] % group == 0 + # Find the number of masked filter for each group (mini_masked). + # Because we have to keep the pruned filter can still + # be divided into the same number of groups, so we only can + # prune mini_masked filters for each group. + step = shape[0] / group + group_masked = [] + for i in range(group): + _start = step * i + _end = step * (i+1) + _tmp_list = list(filter(lambda x: _start <= x and x < _end, all_zeros)) + group_masked.append(_tmp_list) + mini_masked = min([len(x) for x in group_masked]) + for gm in group_masked: + for i in range(mini_masked, len(gm)): + # To keep the output channel number still being divisible to + # groups, we set the masks of following filters to be zero. + pos = gm[i] + self.masks[layername]['weight'][pos] = torch.ones(shape[1:]) + if hasattr(self.masks[layername], 'bias'): + self.masks[layername]['bias'][pos] = 1 + return self.masks + + + +class ChannelMaskConflict(MaskFix): + def __init__(self, masks, model=None, dummy_input=None, traced=None): + """ + ChannelMaskConflict fix the mask conflict between the layers that + has channel dependecy with each other. + + Parameters + ---------- + masks : dict + a dict object that stores the masks + model : torch.nn.Module + model to fix the mask conflict + dummy_input : torch.Tensor + input example to trace the model + graph : torch._C.torch.jit.TopLevelTracedModule + the traced graph of the target model, is this parameter is not None, + we donnot use the model and dummpy_input to get the trace graph. + """ + super(ChannelMaskConflict, self).__init__(masks, model, dummy_input, traced) + self.conv_prune_dim = detect_mask_prune_dim(masks, model) + _logger.info('detected conv prune dim: %s', self.conv_prune_dim) + + def fix_mask(self): + """ + Fix the mask conflict before the mask inference for the layers that + has shape dependencies. This function should be called before the + mask inference of the 'speedup' module. Only structured pruning masks + are supported. + """ + if self.conv_prune_dim == 0: + channel_depen = ChannelDependency(self.model, self.dummy_input, self.traced) + else: + channel_depen = InputChannelDependency(self.model, self.dummy_input, self.traced) + depen_sets = channel_depen.dependency_sets + sum_idx = (1, 2, 3) if self.conv_prune_dim == 0 else (0, 2, 3) + for dset in depen_sets: + if len(dset) <= 1: + continue + # channel_masks is a list, each element is None or a vector, for example: + # [[0, 1, 1, 0, 0], [0, 0, 1, 1, 0], None], None means no channel + # is pruned. + channel_masks = [] + fine_grained = False + for name in dset: + if name in self.masks: + _, m = get_module_by_name(self.model, name) + assert m is not None + mask = self.masks[name]['weight'] + if type(m).__name__ == 'Conv2d': + channel_mask = (mask.abs().sum(sum_idx) != 0).int() + channel_masks.append(channel_mask) + if (channel_mask.sum() * (mask.numel() / mask.shape[self.conv_prune_dim])).item() != (mask > 0).sum().item(): + fine_grained = True + elif type(m).__name__ == 'Linear': + channel_masks.append((mask.abs().sum(0) != 0).int()) + elif type(m).__name__ == 'BatchNorm2d': + channel_masks.append(mask.int()) + else: + raise RuntimeError('unsupported module type: {type(m).__name__}') + else: + # no mask means not pruned, equivlent to full masks + channel_masks.append(None) + if fine_grained: + _logger.info('fine-grained mask detected, skip solving conflict for this set: %s', dset) + continue + if all(x is None for x in channel_masks): + continue + num_channels_list = [len(x) for x in channel_masks if x is not None] + # number of channels in same set should be identical + assert len(set(num_channels_list)) == 1 + num_channels = num_channels_list[0] + + for i, dim_mask in enumerate(channel_masks): + if dim_mask is None: + # temporarily bug fix by queyu, 2020/11/22 + channel_masks[i] = torch.ones(num_channels).to('cuda').int() + + # merge masks with 'or' + merged_channel_mask = channel_masks[0].clone() + for i in range(1, len(channel_masks)): + merged_channel_mask = ((merged_channel_mask + channel_masks[i]) != 0).int() + + merged_index = torch.nonzero(merged_channel_mask, as_tuple=True)[0] + + for name in dset: + if name not in self.masks: + assert all(merged_channel_mask) + continue + orig_mask = self.masks[name]['weight'] + _, m = get_module_by_name(self.model, name) + new_mask = torch.zeros_like(orig_mask) + if type(m).__name__ == 'Conv2d': + if self.conv_prune_dim == 0: + new_mask[merged_index, :, :, :] = 1. + else: + new_mask[:, merged_index, :, :] = 1. + elif type(m).__name__ == 'Linear': + new_mask[:, merged_index] = 1. + elif type(m).__name__ == 'BatchNorm2d': + new_mask = merged_index.type_as(orig_mask) + else: + raise RuntimeError('unsupported module type: {type(m).__name__}') + + self.masks[name]['weight'] = new_mask + if 'bias' in self.masks[name] and self.masks[name]['bias'] is not None: + if type(m).__name__ == 'Conv2d': + assert self.conv_prune_dim == 0 + self.masks[name]['bias'] = merged_channel_mask.type_as(self.masks[name]['bias']) + + return self.masks + +def detect_mask_prune_dim(masks, model): + """ + Detect how the masks of convolutional layers are pruned. + + Parameters + ---------- + masks: dict + A dict object that stores the masks. + model: nn.Module + Model object which the mask can be applied on. + + Returns: + ------- + How the masks of convolutional layers are pruned, this depends on pruning algorithms, it should + return 1 for masks generated by AMCPruner, and returns 0 for masks generated by the rest + NNI builtin pruners. + 0: filter pruning, prune filters of weights which causes channels of output feature maps are pruned. + 1: channel pruning, prune kernels corresponding to each input channels which causes channels of + input feature maps are pruned. + """ + dim0_preserved, dim1_preserved = 0., 0. + dim0_num, dim1_num = 0., 0. + for module_name in masks: + _, m = get_module_by_name(model, module_name) + if m is None or type(m).__name__ != 'Conv2d': + continue + + mask = masks[module_name]['weight'].clone() + assert (mask >= 0).sum() == mask.numel(), \ + "mask values should be greater than or equal to 0." + mask = (mask > 0).int() + mask = mask.view(mask.shape[0], mask.shape[1], -1) + dim0_mask = (mask.sum((1, 2)) > 0).int() + dim1_mask = (mask.sum((0, 2)) > 0).int() + dim0_preserved += dim0_mask.sum().item() + dim1_preserved += dim1_mask.sum().item() + dim0_num += len(dim0_mask) + dim1_num += len(dim1_mask) + + if dim0_num == 0 or dim1_num == 0: + _logger.warning('no multi-dimension masks found.') + return 0 + + dim0_sparsity, dim1_sparsity = 1. - dim0_preserved / dim0_num, 1. - dim1_preserved / dim1_num + _logger.info('dim0 sparsity: %f', dim0_sparsity) + _logger.info('dim1 sparsity: %f', dim1_sparsity) + + if dim0_sparsity == dim1_sparsity == 0.: + _logger.warning('nothing masked.') + + if dim0_sparsity > 0 and dim1_sparsity > 0: + _logger.warning('both dim0 and dim1 masks found.') + + return 0 if dim0_sparsity >= dim1_sparsity else 1 diff --git a/new_impl/cv/third_party/nni/compression/pytorch/utils/num_param_counter.py b/new_impl/cv/third_party/nni/compression/pytorch/utils/num_param_counter.py new file mode 100644 index 0000000000000000000000000000000000000000..89ad0979943126c45112d8b865d87ec12cdb1961 --- /dev/null +++ b/new_impl/cv/third_party/nni/compression/pytorch/utils/num_param_counter.py @@ -0,0 +1,16 @@ +def get_total_num_weights(model, op_types=['default']): + ''' + calculate the total number of weights + + Returns + ------- + int + total weights of all the op considered + ''' + num_weights = 0 + for _, module in model.named_modules(): + if module == model: + continue + if 'default' in op_types or type(module).__name__ in op_types: + num_weights += module.weight.data.numel() + return num_weights \ No newline at end of file diff --git a/new_impl/cv/third_party/nni/compression/pytorch/utils/sensitivity_analysis.py b/new_impl/cv/third_party/nni/compression/pytorch/utils/sensitivity_analysis.py new file mode 100644 index 0000000000000000000000000000000000000000..e6ca7a171c38c1acabceb1137033ca21d10ab5a3 --- /dev/null +++ b/new_impl/cv/third_party/nni/compression/pytorch/utils/sensitivity_analysis.py @@ -0,0 +1,255 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import copy +import csv +import logging +from collections import OrderedDict + +import numpy as np +import torch.nn as nn + +# FIXME: I don't know where "utils" should be +SUPPORTED_OP_NAME = ['Conv2d', 'Conv1d'] +SUPPORTED_OP_TYPE = [getattr(nn, name) for name in SUPPORTED_OP_NAME] + +logger = logging.getLogger('Sensitivity_Analysis') +logger.setLevel(logging.INFO) + + +class SensitivityAnalysis: + def __init__(self, model, val_func, sparsities=None, prune_type='l1', early_stop_mode=None, early_stop_value=None, + train_loader=None): + """ + Perform sensitivity analysis for this model. + Parameters + ---------- + model : torch.nn.Module + the model to perform sensitivity analysis + val_func : function + validation function for the model. Due to + different models may need different dataset/criterion + , therefore the user need to cover this part by themselves. + In the val_func, the model should be tested on the validation dateset, + and the validation accuracy/loss should be returned as the output of val_func. + There are no restrictions on the input parameters of the val_function. + User can use the val_args, val_kwargs parameters in analysis + to pass all the parameters that val_func needed. + sparsities : list + The sparsity list provided by users. This parameter is set when the user + only wants to test some specific sparsities. In the sparsity list, each element + is a sparsity value which means how much weight the pruner should prune. Take + [0.25, 0.5, 0.75] for an example, the SensitivityAnalysis will prune 25% 50% 75% + weights gradually for each layer. + prune_type : str + The pruner type used to prune the conv layers, default is 'l1', + and 'l2', 'fine-grained' is also supported. + early_stop_mode : str + If this flag is set, the sensitivity analysis + for a conv layer will early stop when the validation metric( + for example, accurracy/loss) has alreay meet the threshold. We + support four different early stop modes: minimize, maximize, dropped, + raised. The default value is None, which means the analysis won't stop + until all given sparsities are tested. This option should be used with + early_stop_value together. + + minimize: The analysis stops when the validation metric return by the val_func + lower than early_stop_value. + maximize: The analysis stops when the validation metric return by the val_func + larger than early_stop_value. + dropped: The analysis stops when the validation metric has dropped by early_stop_value. + raised: The analysis stops when the validation metric has raised by early_stop_value. + early_stop_value : float + This value is used as the threshold for different earlystop modes. + This value is effective only when the early_stop_mode is set. + + """ + from nni.algorithms.compression.pytorch.pruning.constants_pruner import PRUNER_DICT + + self.model = model + self.val_func = val_func + self.target_layer = OrderedDict() + self.ori_state_dict = copy.deepcopy(self.model.state_dict()) + self.target_layer = {} + self.sensitivities = {} + if sparsities is not None: + self.sparsities = sorted(sparsities) + else: + self.sparsities = np.arange(0.1, 1.0, 0.1) + self.sparsities = [np.round(x, 2) for x in self.sparsities] + self.Pruner = PRUNER_DICT[prune_type] + self.early_stop_mode = early_stop_mode + self.early_stop_value = early_stop_value + self.ori_metric = None # original validation metric for the model + # already_pruned is for the iterative sensitivity analysis + # For example, sensitivity_pruner iteratively prune the target + # model according to the sensitivity. After each round of + # pruning, the sensitivity_pruner will test the new sensitivity + # for each layer + self.already_pruned = {} + self.model_parse() + + self.train_loader = train_loader + + @property + def layers_count(self): + return len(self.target_layer) + + def model_parse(self): + for name, submodel in self.model.named_modules(): + for op_type in SUPPORTED_OP_TYPE: + if isinstance(submodel, op_type): + self.target_layer[name] = submodel + self.already_pruned[name] = 0 + + def _need_to_stop(self, ori_metric, cur_metric): + """ + Judge if meet the stop conditon(early_stop, min_threshold, + max_threshold). + Parameters + ---------- + ori_metric : float + original validation metric + cur_metric : float + current validation metric + + Returns + ------- + stop : bool + if stop the sensitivity analysis + """ + if self.early_stop_mode is None: + # early stop mode is not enable + return False + assert self.early_stop_value is not None + if self.early_stop_mode == 'minimize': + if cur_metric < self.early_stop_value: + return True + elif self.early_stop_mode == 'maximize': + if cur_metric > self.early_stop_value: + return True + elif self.early_stop_mode == 'dropped': + if cur_metric < ori_metric - self.early_stop_value: + return True + elif self.early_stop_mode == 'raised': + if cur_metric > ori_metric + self.early_stop_value: + return True + return False + + def analysis(self, val_args=None, val_kwargs=None, specified_layers=None): + """ + This function analyze the sensitivity to pruning for + each conv layer in the target model. + If start and end are not set, we analyze all the conv + layers by default. Users can specify several layers to + analyze or parallelize the analysis process easily through + the start and end parameter. + + Parameters + ---------- + val_args : list + args for the val_function + val_kwargs : dict + kwargs for the val_funtion + specified_layers : list + list of layer names to analyze sensitivity. + If this variable is set, then only analyze + the conv layers that specified in the list. + User can also use this option to parallelize + the sensitivity analysis easily. + Returns + ------- + sensitivities : dict + dict object that stores the trajectory of the + accuracy/loss when the prune ratio changes + """ + if val_args is None: + val_args = [] + if val_kwargs is None: + val_kwargs = {} + # Get the original validation metric(accuracy/loss) before pruning + # Get the accuracy baseline before starting the analysis. + self.ori_metric = self.val_func(*val_args, **val_kwargs) + namelist = list(self.target_layer.keys()) + if specified_layers is not None: + # only analyze several specified conv layers + namelist = list(filter(lambda x: x in specified_layers, namelist)) + for name in namelist: + self.sensitivities[name] = {} + for sparsity in self.sparsities: + # here the sparsity is the relative sparsity of the + # the remained weights + # Calculate the actual prune ratio based on the already pruned ratio + real_sparsity = ( + 1.0 - self.already_pruned[name]) * sparsity + self.already_pruned[name] + # TODO In current L1/L2 Filter Pruner, the 'op_types' is still necessary + # I think the L1/L2 Pruner should specify the op_types automaticlly + # according to the op_names + cfg = [{'sparsity': real_sparsity, 'op_names': [ + name], 'op_types': ['Conv2d']}] + if self.train_loader: + pruner = self.Pruner(self.model, cfg, train_loader=self.train_loader) + else: + pruner = self.Pruner(self.model, cfg) + pruner.compress() + val_metric = self.val_func(*val_args, **val_kwargs) + logger.info('Layer: %s Sparsity: %.2f Validation Metric: %.4f', + name, real_sparsity, val_metric) + + self.sensitivities[name][sparsity] = val_metric + pruner._unwrap_model() + del pruner + # check if the current metric meet the stop condition + if self._need_to_stop(self.ori_metric, val_metric): + break + + # reset the weights pruned by the pruner, because the + # input sparsities is sorted, so we donnot need to reset + # weight of the layer when the sparsity changes, instead, + # we only need reset the weight when the pruning layer changes. + self.model.load_state_dict(self.ori_state_dict) + + return self.sensitivities + + def export(self, filepath): + """ + Export the results of the sensitivity analysis + to a csv file. The firstline of the csv file describe the content + structure. The first line is constructed by 'layername' and sparsity + list. Each line below records the validation metric returned by val_func + when this layer is under different sparsities. Note that, due to the early_stop + option, some layers may not have the metrics under all sparsities. + + layername, 0.25, 0.5, 0.75 + conv1, 0.6, 0.55 + conv2, 0.61, 0.57, 0.56 + + Parameters + ---------- + filepath : str + Path of the output file + """ + str_sparsities = [str(x) for x in self.sparsities] + header = ['layername'] + str_sparsities + with open(filepath, 'w') as csvf: + csv_w = csv.writer(csvf) + csv_w.writerow(header) + for layername in self.sensitivities: + row = [] + row.append(layername) + for sparsity in sorted(self.sensitivities[layername].keys()): + row.append(self.sensitivities[layername][sparsity]) + csv_w.writerow(row) + + def update_already_pruned(self, layername, ratio): + """ + Set the already pruned ratio for the target layer. + """ + self.already_pruned[layername] = ratio + + def load_state_dict(self, state_dict): + """ + Update the weight of the model + """ + self.ori_state_dict = copy.deepcopy(state_dict) + self.model.load_state_dict(self.ori_state_dict) diff --git a/new_impl/cv/third_party/nni/compression/pytorch/utils/shape_dependency.py b/new_impl/cv/third_party/nni/compression/pytorch/utils/shape_dependency.py new file mode 100644 index 0000000000000000000000000000000000000000..c68d09e7561777e1a3fdfe6f63926f58c45b0c25 --- /dev/null +++ b/new_impl/cv/third_party/nni/compression/pytorch/utils/shape_dependency.py @@ -0,0 +1,489 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import csv +import logging + +__all__ = ['ChannelDependency', 'GroupDependency', 'CatPaddingDependency', 'InputChannelDependency'] + +CONV_TYPE = 'aten::_convolution' +ADD_TYPES = ['aten::add', 'aten::add_'] +CAT_TYPE = 'aten::cat' +logger = logging.getLogger('Shape_Dependency') +RESHAPE_OPS = [CAT_TYPE, 'aten::view', 'aten::reshape', 'aten::flatten', 'aten::mean'] + +class Dependency: + def __init__(self, model=None, dummy_input=None, traced_model=None): + """ + Build the graph for the model. + """ + from ....common.graph_utils import TorchModuleGraph + + # check if the input is legal + if traced_model is None: + # user should provide model & dummy_input to trace + # the model or a already traced model + assert model is not None and dummy_input is not None + self.graph = TorchModuleGraph(model, dummy_input, traced_model) + self.dependency = dict() + self.build_dependency() + + def build_dependency(self): + raise NotImplementedError + + def export(self, filepath): + raise NotImplementedError + +class ChannelDependency(Dependency): + def __init__(self, model=None, dummy_input=None, traced_model=None): + """ + This model analyze the channel dependencies between the conv + layers in a model. + + Parameters + ---------- + model : torch.nn.Module + The model to be analyzed. + data : torch.Tensor + The example input data to trace the network architecture. + traced_model : torch._C.Graph + if we alreay has the traced graph of the target model, we donnot + need to trace the model again. + """ + super(ChannelDependency, self).__init__(model, dummy_input, traced_model) + + def _get_parent_layers(self, node): + """ + Find the nearest father conv layers for the target node. + + Parameters + --------- + node : torch._C.Node + target node. + + Returns + ------- + parent_layers: list + nearest father conv/linear layers for the target worknode. + """ + parent_layers = [] + queue = [] + queue.append(node) + while queue: + curnode = queue.pop(0) + if curnode.op_type == 'Conv2d' or curnode.op_type == 'Linear': + # find the first met conv + parent_layers.append(curnode.name) + continue + parents = self.graph.find_predecessors(curnode.unique_name) + parents = [self.graph.name_to_node[name] for name in parents] + for parent in parents: + queue.append(parent) + return parent_layers + + def build_dependency(self): + """ + Build the channel dependency for the conv layers + in the model. + """ + # unpack the tuple/list manually before analyze the + # channel dependency + self.graph.unpack_manually() + for node in self.graph.nodes_py.nodes_op: + parent_layers = [] + # find the node that contains aten::add + # or aten::cat operations + if node.op_type in ADD_TYPES: + parent_layers = self._get_parent_layers(node) + elif node.op_type == CAT_TYPE: + # To determine if this cat operation will introduce channel + # dependency, we need the specific input parameters of the cat + # opertion. To get the input parameters of the cat opertion, we + # need to traverse all the cpp_nodes included by this NodePyGroup, + # because, TorchModuleGraph merges the important nodes and the adjacent + # unimportant nodes (nodes started with prim::attr, for example) into a + # NodepyGroup. + cat_dim = None + for cnode in node.node_cpps: + if cnode.kind() == CAT_TYPE: + cat_dim = list(cnode.inputs())[1].toIValue() + break + if cat_dim != 1: + parent_layers = self._get_parent_layers(node) + dependency_set = set(parent_layers) + # merge the dependencies + for parent in parent_layers: + if parent in self.dependency: + dependency_set.update(self.dependency[parent]) + # save the dependencies + for _node in dependency_set: + self.dependency[_node] = dependency_set + + + def export(self, filepath): + """ + export the channel dependencies as a csv file. + The layers at the same line have output channel + dependencies with each other. For example, + layer1.1.conv2, conv1, and layer1.0.conv2 have + output channel dependencies with each other, which + means the output channel(filters) numbers of these + three layers should be same with each other, otherwise + the model may has shape conflict. + + Output example: + Dependency Set,Convolutional Layers + Set 1,layer1.1.conv2,layer1.0.conv2,conv1 + Set 2,layer1.0.conv1 + Set 3,layer1.1.conv1 + """ + header = ['Dependency Set', 'Layers'] + setid = 0 + visited = set() + with open(filepath, 'w') as csvf: + csv_w = csv.writer(csvf, delimiter=',') + csv_w.writerow(header) + for node in self.graph.nodes_py.nodes_op: + if node.op_type != 'Conv2d' or node in visited: + continue + setid += 1 + row = ['Set %d' % setid] + if node.name not in self.dependency: + visited.add(node) + row.append(node.name) + else: + for other in self.dependency[node.name]: + visited.add(self.graph.name_to_node[other]) + row.append(other) + csv_w.writerow(row) + + @property + def dependency_sets(self): + """ + Get the list of the dependency set. + + Returns + ------- + dependency_sets : list + list of the dependency sets. For example, + [set(['conv1', 'conv2']), set(['conv3', 'conv4'])] + + """ + d_sets = [] + visited = set() + for node in self.graph.nodes_py.nodes_op: + if node.op_type != 'Conv2d' or node in visited: + continue + tmp_set = set() + if node.name not in self.dependency: + visited.add(node) + tmp_set.add(node.name) + else: + for other in self.dependency[node.name]: + visited.add(self.graph.name_to_node[other]) + tmp_set.add(other) + d_sets.append(tmp_set) + return d_sets + +def reshape_break_channel_dependency(op_node): + """ + The reshape operations such as (reshape, view, flatten) may break + the channel dependency. We need to check the input parameters of + these reshape operations to check if this reshape node will break + the channel dependency. However, it's complicated to analyze the the input + parameters for each reshape function and infer if it will break the channel + dependency. So currently, we just check if the input channel and the output + channel is the same, if so, then we can say the original reshape function + doesn't want to change the number of the channels, which means the channel + dependency is not broken. In contrast, the original reshap operation wants + to change the number of channels, so it breaks the channel dependency. + + Parameters + ---------- + opnode: NodePyOP + A Op node of the graph. + Returns + ------- + bool + If this operation will break the channel dependency. + """ + in_shape = op_node.auxiliary['in_shape'] + out_shape = op_node.auxiliary['out_shape'] + in_channel = in_shape[1] + out_channel = out_shape[1] + return in_channel != out_channel + +class InputChannelDependency(ChannelDependency): + """ + Some pruners may prune the input channel of the convolutional + layers. While pruning the input channel of the convolutional layers, + the layers that share the same input tensor should prune the same + channels, and we say these layers that share the same input tensor/channel + has the input channel dependency. If we only prune the input channel of one + layer in the dependency set, there will be a shape conflict for the other + layers in the same dependency set, which may trigger a runtime error. + Here we judge whether the application will truncate the dependency by analyzing + whether the number of channels before and after the operation has changed. + If not, the input channel dependency will be passed to the following nodes. + """ + + def __init__(self, model, dummy_input=None, traced_model=None): + """ + This model analyze the input channel dependencies between the conv + layers in a model. + + Parameters + ---------- + model : torch.nn.Module + The model to be analyzed. + data : torch.Tensor + The example input data to trace the network architecture. + traced_model : torch._C.Graph + if we alreay has the traced graph of the target model, we donnot + need to trace the model again. + """ + super(InputChannelDependency, self).__init__(model, dummy_input, traced_model) + + def _get_following_convs(self, tensor): + queue = [] + key_layers = [] + queue.extend(self.graph.input_to_node[tensor]) + while queue: + curnode = queue.pop(0) + if curnode.op_type == 'Conv2d' or curnode.op_type == 'Linear': + # find the first met conv + key_layers.append(curnode.name) + continue + elif curnode.op_type in RESHAPE_OPS: + # check if the reshape operation will break the channel dependency + if reshape_break_channel_dependency(curnode): + # reshape operations also breaks the dependency relationship + continue + successors = self.graph.find_successors(curnode.unique_name) + successors = [self.graph.name_to_node[name] for name in successors] + for layer in successors: + queue.append(layer) + return key_layers + + def build_dependency(self): + """ + Build the input channel dependencies. + The `InputChannelDependency` indicates the layers that have + dependencies when pruning the input channel of the conv layers. + In contrast, `ChannelDependency` indicates the dependent layers + when pruning the output channles of conv layers (for example, L1FilterPruner). + """ + # unpack the tuple or list manually + self.graph.unpack_manually() + for tensor in self.graph.input_to_node: + # start from this tensor, find all the conv layers that + # take this tensor as input. Similar to the `ChannelDependency` + # the conv layer will truncate the dependencies + layers = self._get_following_convs(tensor) + dependency_set = set(layers) + for layer in layers: + if layer in self.dependency: + dependency_set.update(self.dependency[layer]) + for layer in dependency_set: + self.dependency[layer] = dependency_set + + +class CatPaddingDependency(ChannelDependency): + def __init__(self, model=None, dummy_input=None, traced_model=None): + super(CatPaddingDependency, self).__init__(model, dummy_input, traced_model) + + def build_dependency(self): + """ + Build the cat padding dependencies. + If the output features of several layers are stitched together + by cat operation, then these layers have cat padding dependencies. + This is because when inferring the cat mask, we need all the input + masks for the cat operation. At this time we need to know the source + of all input vectors of a cat operation. + """ + for node in self.graph.nodes_py.nodes_op: + parent_layers = [] + if node.op_type == CAT_TYPE: + parent_layers = self._get_parent_layers(node) + dependency_set = set(parent_layers) + # merge the dependencies + for parent in parent_layers: + if parent in self.dependency: + dependency_set.update(self.dependency[parent]) + # save the dependencies + for _node in dependency_set: + self.dependency[_node] = dependency_set + + @property + def dependency_sets(self): + d_sets = [] + visited = set() + for nodename in self.dependency: + if nodename in visited: + continue + d_sets.append(self.dependency[nodename]) + return d_sets + + def export(self, filepath): + """ + Export the dependencies into a file. + In the output file, each line contains a set of layers + whose output features are stitched together by the cat + operation. + + output example: + Dependency Set, Layers + set1, Conv1, Conv2 + set2, Conv3, Conv4 + """ + header = ['Dependency Set', 'Layers'] + setid = 0 + with open(filepath, 'w') as csvf: + csv_w = csv.writer(csvf, delimiter=',') + csv_w.writerow(header) + for layers in self.dependency_sets: + setid += 1 + row = ['Set %d' % setid] + row.extend(list(layers)) + csv_w.writerow(row) + +class GroupDependency(Dependency): + def __init__(self, model=None, dummy_input=None, traced_model=None): + """ + This model analyze the group dependencis between the conv + layers in a model. + + Parameters + ---------- + model : torch.nn.Module + The model to be analyzed. + data : torch.Tensor + The example input data to trace the network architecture. + traced_model : torch._C.Graph + if we alreay has the traced graph of the target model, we donnot + need to trace the model again. + """ + super(GroupDependency, self).__init__(model, dummy_input, traced_model) + + def _get_parent_convs(self, node): + """ + Find the nearest father conv layers for the target node. + + Parameters + --------- + node : torch._C.Node + target node. + + Returns + ------- + parent_layers : list + nearest father conv layers for the target node. Due to the group + dependency only exists between the conv layers, so we only find + the parent conv layers. + """ + parent_layers = [] + # the input node is a Conv node + predeessors = self.graph.find_predecessors(node.unique_name) + predeessors = [self.graph.name_to_node[x] for x in predeessors] + queue = predeessors + while queue: + curnode = queue.pop(0) + if curnode.op_type == 'Conv2d': + # find the first met conv + parent_layers.append(curnode.name) + continue + parents = self.graph.find_predecessors(curnode.unique_name) + parents = [self.graph.name_to_node[name] for name in parents] + for parent in parents: + queue.append(parent) + return parent_layers + + def _get_conv_groups(self, node_group): + """ + Get the number of groups for a convolutional layer. + + Parameters + ---------- + node_group : NodePyGroup + target node. + + Returns + ------- + group : int + the number of the groups of the target conv layer. + """ + cpp_conv = list(filter(lambda x: x.kind() == CONV_TYPE, node_group.node_cpps)) + assert len(cpp_conv) == 1 + cpp_conv = cpp_conv[0] + inputs = list(cpp_conv.inputs()) + # get the number of the group from the input parameters + group = inputs[8].toIValue() + return group + + def build_dependency(self): + """ + Build the channel dependency for the conv layers + in the model. This function return the group number + of each conv layers. Note that, here, the group count + of conv layers may be larger than their originl groups. + This is because that the input channel will also be grouped + for the group conv layers. To make this clear, assume we + have two group conv layers: conv1(group=2), conv2(group=4). + conv2 takes the output features of conv1 as input. + Then we have to the filters of conv1 can still be + divided into 4 groups after filter pruning, because + the input channels of conv2 shoule be divided into + 4 groups. + + Returns + ------- + self.dependency : dict + key: the name of conv layers, value: the minimum value that the number of + filters should be divisible to. + """ + for node in self.graph.nodes_py.nodes_op: + if node.op_type == 'Conv2d': + group = self._get_conv_groups(node) + if node.name in self.dependency: + # the conv layer whose group is larger than 1 will require that + # it's number of output channel to be divisible by the number of group. + self.dependency[node.name] = max(self.dependency[node.name], group) + else: + self.dependency[node.name] = group + if group > 1: + # for the conv layer whose group is larger than 1, it will require the number + # of output channels of their parent conv layer to be divisible by group. + parent_convs = self._get_parent_convs(node) + for parent in parent_convs: + if parent in self.dependency: + self.dependency[parent] = max(self.dependency[parent], group) + else: + self.dependency[parent] = group + return self.dependency + + def export(self, filepath): + """ + export the group dependency to a csv file. + Each line describes a convolution layer, the + first part of each line is the Pytorch module + name of the conv layer. The second part of each + line is the group count of the filters in this layer. + Note that, the group count may be larger than this + layers original group number. + + output example: + Conv layer, Groups + Conv1, 1 + Conv2, 2 + Conv3, 4 + """ + header = ['Conv Layer Name', 'Group'] + with open(filepath, 'w') as csvf: + csv_w = csv.writer(csvf, delimiter=',') + csv_w.writerow(header) + for name in self.dependency: + group = self.dependency[name] + csv_w.writerow([name, group]) + @property + def dependency_sets(self): + return self.dependency diff --git a/new_impl/cv/third_party/nni/compression/pytorch/utils/utils.py b/new_impl/cv/third_party/nni/compression/pytorch/utils/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..c687c5e2a6bf971433e6298ee92a86be5c23b6d4 --- /dev/null +++ b/new_impl/cv/third_party/nni/compression/pytorch/utils/utils.py @@ -0,0 +1,30 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +def get_module_by_name(model, module_name): + """ + Get a module specified by its module name + + Parameters + ---------- + model : pytorch model + the pytorch model from which to get its module + module_name : str + the name of the required module + + Returns + ------- + module, module + the parent module of the required module, the required module + """ + name_list = module_name.split(".") + for name in name_list[:-1]: + if hasattr(model, name): + model = getattr(model, name) + else: + return None, None + if hasattr(model, name_list[-1]): + leaf_module = getattr(model, name_list[-1]) + return model, leaf_module + else: + return None, None diff --git a/new_impl/cv/third_party/nni/compression/tensorflow/__init__.py b/new_impl/cv/third_party/nni/compression/tensorflow/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d05fade2f1140c68bb78969f89aaa8529414f3ca --- /dev/null +++ b/new_impl/cv/third_party/nni/compression/tensorflow/__init__.py @@ -0,0 +1,4 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .compressor import Compressor, Pruner diff --git a/new_impl/cv/third_party/nni/compression/tensorflow/compressor.py b/new_impl/cv/third_party/nni/compression/tensorflow/compressor.py new file mode 100644 index 0000000000000000000000000000000000000000..7f9b1bc6aed33f2922dff409982906799a8e636e --- /dev/null +++ b/new_impl/cv/third_party/nni/compression/tensorflow/compressor.py @@ -0,0 +1,256 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +Abstract base classes for TensorFlow model compression. +""" + +import logging + +import tensorflow as tf +assert tf.__version__.startswith('2'), 'NNI model compression only supports TensorFlow v2.x' + +from . import default_layers + +_logger = logging.getLogger(__name__) + + +class Compressor: + """ + Common base class for all compressors. + + This class is designed for other base classes. + Algorithms should inherit ``Pruner`` or ``Quantizer`` instead. + + Attributes + ---------- + compressed_model : tf.keras.Model + Compressed user model. + wrappers : list of tf.keras.Model + A wrapper is an instrumented TF ``Layer``, in ``Model`` format. + + Parameters + ---------- + model : tf.keras.Model + The user model to be compressed. + config_list : list of JSON object + User configuration. The format is detailed in tutorial. + LayerWrapperClass : a class derive from Model + The class used to instrument layers. + """ + + def __init__(self, model, config_list, LayerWrapperClass): + assert isinstance(model, tf.keras.Model) + self.validate_config(model, config_list) + + self._original_model = model + self._config_list = config_list + self._wrapper_class = LayerWrapperClass + self._wrappers = {} # key: id(layer) , value: Wrapper(layer) + + self.compressed_model = self._instrument(model) + self.wrappers = list(self._wrappers.values()) + + if not self.wrappers: + _logger.warning('Nothing is configured to compress, please check your model and config list') + + def set_wrappers_attribute(self, name, value): + """ + Call ``setattr`` on all wrappers. + """ + for wrapper in self.wrappers: + setattr(wrapper, name, value) + + def validate_config(self, model, config_list): + """ + Compression algorithm should overload this function to validate configuration. + """ + pass + + + def _instrument(self, layer): + if isinstance(layer, tf.keras.Sequential): + return self._instrument_sequential(layer) + if isinstance(layer, tf.keras.Model): + return self._instrument_model(layer) + + # a layer can be referenced in multiple attributes of a model, + # but should only be instrumented once + if id(layer) in self._wrappers: + return self._wrappers[id(layer)] + + config = self._select_config(layer) + if config is not None: + wrapper = self._wrapper_class(layer, config, self) + self._wrappers[id(layer)] = wrapper + return wrapper + + return layer + + def _instrument_sequential(self, seq): + layers = list(seq.layers) # seq.layers is read-only property + need_rebuild = False + for i, layer in enumerate(layers): + new_layer = self._instrument(layer) + if new_layer is not layer: + layers[i] = new_layer + need_rebuild = True + return tf.keras.Sequential(layers) if need_rebuild else seq + + def _instrument_model(self, model): + for key, value in list(model.__dict__.items()): # avoid "dictionary keys changed during iteration" + if isinstance(value, tf.keras.layers.Layer): + new_layer = self._instrument(value) + if new_layer is not value: + setattr(model, key, new_layer) + elif isinstance(value, list): + for i, item in enumerate(value): + if isinstance(item, tf.keras.layers.Layer): + value[i] = self._instrument(item) + return model + + + def _select_config(self, layer): + # Find the last matching config block for given layer. + # Returns None if the layer should not be compressed. + layer_type = type(layer).__name__ + last_match = None + for config in self._config_list: + if 'op_types' in config: + match = layer_type in config['op_types'] + match_default = 'default' in config['op_types'] and layer_type in default_layers.weighted_modules + if not match and not match_default: + continue + if 'op_names' in config and layer.name not in config['op_names']: + continue + last_match = config + if last_match is None or 'exclude' in last_match: + return None + return last_match + + +class Pruner(Compressor): + """ + Base class for pruning algorithms. + + End users should use ``compress`` and callback APIs (WIP) to prune their models. + + The underlying model is instrumented upon initialization of pruner object. + So if you want to pre-train the model, train it before creating pruner object. + + The compressed model can only execute in eager mode. + + Algorithm developers should override ``calc_masks`` method to specify pruning strategy. + + Parameters + ---------- + model : tf.keras.Model + The user model to prune. + config_list : list of JSON object + User configuration. The format is detailed in tutorial. + """ + def __init__(self, model, config_list): + super().__init__(model, config_list, PrunerLayerWrapper) + #self.callback = PrunerCallback(self) + + def compress(self): + """ + Apply compression on a pre-trained model. + + If you want to prune the model during training, use callback API (WIP) instead. + + Returns + ------- + tf.keras.Model + The compressed model. + """ + self._update_mask() + return self.compressed_model + + def calc_masks(self, wrapper, **kwargs): + """ + Abstract method to be overridden by algorithm. End users should ignore it. + + If the callback is set up, this method will be invoked at end of each training minibatch. + If not, it will only be called when end user invokes ``compress``. + + Parameters + ---------- + wrapper : PrunerLayerWrapper + The instrumented layer. + **kwargs + Reserved for forward compatibility. + + Returns + ------- + dict of (str, tf.Tensor), or None + The key is weight ``Variable``'s name. The value is a mask ``Tensor`` of weight's shape and dtype. + If a weight's key does not appear in the return value, that weight will not be pruned. + Returning ``None`` means the mask is not changed since last time. + Weight names are globally unique, e.g. `model/conv_1/kernel:0`. + """ + # TODO: maybe it should be able to calc on weight-granularity, beside from layer-granularity + raise NotImplementedError("Pruners must overload calc_masks()") + + def _update_mask(self): + for wrapper_idx, wrapper in enumerate(self.wrappers): + masks = self.calc_masks(wrapper, wrapper_idx=wrapper_idx) + if masks is not None: + wrapper.masks = masks + + +class PrunerLayerWrapper(tf.keras.Model): + """ + Instrumented TF layer. + + Wrappers will be passed to pruner's ``calc_masks`` API, + and the pruning algorithm should use wrapper's attributes to calculate masks. + + Once instrumented, underlying layer's weights will get **modified** by masks before forward pass. + + Attributes + ---------- + layer_info : LayerInfo + All static information of the original layer. + layer : tf.keras.layers.Layer + The original layer. + config : JSON object + Selected configuration. The format is detailed in tutorial. + pruner : Pruner + Bound pruner object. + masks : dict of (str, tf.Tensor) + Current masks. The key is weight's name and the value is mask tensor. + On initialization, `masks` is an empty dict, which means no weight is pruned. + Afterwards, `masks` is the last return value of ``Pruner.calc_masks``. + See ``Pruner.calc_masks`` for details. + """ + def __init__(self, layer, config, pruner): + super().__init__() + self.layer = layer + self.config = config + self.pruner = pruner + self.masks = {} + _logger.info('Layer detected to compress: %s', self.layer.name) + + def call(self, *inputs): + new_weights = [] + for weight in self.layer.weights: + mask = self.masks.get(weight.name) + if mask is not None: + new_weights.append(tf.math.multiply(weight, mask)) + else: + new_weights.append(weight) + if new_weights and not hasattr(new_weights[0], 'numpy'): + raise RuntimeError('NNI: Compressed model can only run in eager mode') + self.layer.set_weights([weight.numpy() for weight in new_weights]) + return self.layer(*inputs) + + +# TODO: designed to replace `patch_optimizer` +#class PrunerCallback(tf.keras.callbacks.Callback): +# def __init__(self, pruner): +# super().__init__() +# self._pruner = pruner +# +# def on_train_batch_end(self, batch, logs=None): +# self._pruner.update_mask() diff --git a/new_impl/cv/third_party/nni/compression/tensorflow/default_layers.py b/new_impl/cv/third_party/nni/compression/tensorflow/default_layers.py new file mode 100644 index 0000000000000000000000000000000000000000..0c729bd883f1623d28ad279de053a39a5742109c --- /dev/null +++ b/new_impl/cv/third_party/nni/compression/tensorflow/default_layers.py @@ -0,0 +1,9 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +weighted_modules = [ + 'Conv1D', 'Conv2D', 'Conv3D', 'Conv1DTranspose', 'Conv2DTranspose', 'Conv3DTranspose', + 'Dense', + 'PReLU', + 'Embedding', +] diff --git a/new_impl/cv/third_party/nni/experiment/__init__.py b/new_impl/cv/third_party/nni/experiment/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..20b244fe62c321d36dda1f646f8abe94ae8e6b58 --- /dev/null +++ b/new_impl/cv/third_party/nni/experiment/__init__.py @@ -0,0 +1,4 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .nni_client import * diff --git a/new_impl/cv/third_party/nni/experiment/nni_client.py b/new_impl/cv/third_party/nni/experiment/nni_client.py new file mode 100644 index 0000000000000000000000000000000000000000..1dddafd219e546667edec498fc161f9aef7f1845 --- /dev/null +++ b/new_impl/cv/third_party/nni/experiment/nni_client.py @@ -0,0 +1,515 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" A python wrapper for nni rest api + +Example: + +from nni.experiment import Experiment + +exp = Experiment() +exp.start_experiment('../../../../examples/trials/mnist-pytorch/config.yml') + +exp.update_concurrency(3) + +print(exp.get_experiment_status()) +print(exp.get_job_statistics()) +print(exp.list_trial_jobs()) + +exp.stop_experiment() + +""" + +import sys +import os +import subprocess +import re +import json +import requests + +__all__ = [ + 'Experiment', + 'TrialResult', + 'TrialMetricData', + 'TrialHyperParameters', + 'TrialJob' +] + +EXPERIMENT_PATH = 'experiment' +STATUS_PATH = 'check-status' +JOB_STATISTICS_PATH = 'job-statistics' +TRIAL_JOBS_PATH = 'trial-jobs' +METRICS_PATH = 'metric-data' +EXPORT_DATA_PATH = 'export-data' +API_ROOT_PATH = 'api/v1/nni' + +def _nni_rest_get(endpoint, api_path, response_type='json'): + _check_endpoint(endpoint) + uri = '{}/{}/{}'.format(endpoint.strip('/'), API_ROOT_PATH, api_path) + res = requests.get(uri) + if _http_succeed(res.status_code): + if response_type == 'json': + return res.json() + elif response_type == 'text': + return res.text + else: + raise RuntimeError('Incorrect response_type') + else: + return None + +def _http_succeed(status_code): + return status_code // 100 == 2 + +def _create_process(cmd): + if sys.platform == 'win32': + process = subprocess.Popen(cmd, stdout=subprocess.PIPE, creationflags=subprocess.CREATE_NEW_PROCESS_GROUP) + else: + process = subprocess.Popen(cmd, stdout=subprocess.PIPE) + + while process.poll() is None: + output = process.stdout.readline() + if output: + print(output.decode('utf-8').strip()) + return process.returncode + +def _check_endpoint(endpoint): + if endpoint is None: + raise RuntimeError("This instance hasn't been connect to an experiment.") + +class TrialResult: + """ + TrialResult stores the result information of a trial job. + + Parameters + ---------- + json_obj: dict + Json object that stores the result information. + + Attributes + ---------- + parameter: dict + Hyper parameters for this trial. + value: serializable object, usually a number, or a dict with key "default" and other extra keys + Final result. + trialJobId: str + Trial job id. + """ + def __init__(self, json_obj): + self.parameter = None + self.value = None + self.trialJobId = None + for key in json_obj.keys(): + if key == 'id': + setattr(self, 'trialJobId', json_obj[key]) + elif hasattr(self, key): + setattr(self, key, json_obj[key]) + self.value = json.loads(self.value) + + def __repr__(self): + return "TrialResult(parameter: {} value: {} trialJobId: {})".format(self.parameter, self.value, self.trialJobId) + +class TrialMetricData: + """ + TrialMetricData stores the metric data of a trial job. + A trial job may have both intermediate metric and final metric. + + Parameters + ---------- + json_obj: dict + Json object that stores the metric data. + + Attributes + ---------- + timestamp: int + Time stamp. + trialJobId: str + Trial job id. + parameterId: int + Parameter id. + type: str + Metric type, `PERIODICAL` for intermediate result and `FINAL` for final result. + sequence: int + Sequence number in this trial. + data: serializable object, usually a number, or a dict with key "default" and other extra keys + Metric data. + """ + def __init__(self, json_obj): + self.timestamp = None + self.trialJobId = None + self.parameterId = None + self.type = None + self.sequence = None + self.data = None + for key in json_obj.keys(): + setattr(self, key, json_obj[key]) + self.data = json.loads(json.loads(self.data)) + + def __repr__(self): + return "TrialMetricData(timestamp: {} trialJobId: {} parameterId: {} type: {} sequence: {} data: {})" \ + .format(self.timestamp, self.trialJobId, self.parameterId, self.type, self.sequence, self.data) + +class TrialHyperParameters: + """ + TrialHyperParameters stores the hyper parameters of a trial job. + + Parameters + ---------- + json_obj: dict + Json object that stores the hyper parameters. + + Attributes + ---------- + parameter_id: int + Parameter id. + parameter_source: str + Parameter source. + parameters: dict + Hyper parameters. + parameter_index: int + Parameter index. + """ + def __init__(self, json_obj): + self.parameter_id = None + self.parameter_source = None + self.parameters = None + self.parameter_index = None + for key in json_obj.keys(): + if hasattr(self, key): + setattr(self, key, json_obj[key]) + + def __repr__(self): + return "TrialHyperParameters(parameter_id: {} parameter_source: {} parameters: {} parameter_index: {})" \ + .format(self.parameter_id, self.parameter_source, self.parameters, self.parameter_index) + +class TrialJob: + """ + TrialJob stores the information of a trial job. + + Parameters + ---------- + json_obj: dict + json object that stores the hyper parameters + + Attributes + ---------- + trialJobId: str + Trial job id. + status: str + Job status. + hyperParameters: list of `nni.experiment.TrialHyperParameters` + See `nni.experiment.TrialHyperParameters`. + logPath: str + Log path. + startTime: int + Job start time (timestamp). + endTime: int + Job end time (timestamp). + finalMetricData: list of `nni.experiment.TrialMetricData` + See `nni.experiment.TrialMetricData`. + parameter_index: int + Parameter index. + """ + def __init__(self, json_obj): + self.trialJobId = None + self.status = None + self.hyperParameters = None + self.logPath = None + self.startTime = None + self.endTime = None + self.finalMetricData = None + self.stderrPath = None + for key in json_obj.keys(): + if key == 'id': + setattr(self, 'trialJobId', json_obj[key]) + elif hasattr(self, key): + setattr(self, key, json_obj[key]) + if self.hyperParameters: + self.hyperParameters = [TrialHyperParameters(json.loads(e)) for e in self.hyperParameters] + if self.finalMetricData: + self.finalMetricData = [TrialMetricData(e) for e in self.finalMetricData] + + def __repr__(self): + return ("TrialJob(trialJobId: {} status: {} hyperParameters: {} logPath: {} startTime: {} " + "endTime: {} finalMetricData: {} stderrPath: {})") \ + .format(self.trialJobId, self.status, self.hyperParameters, self.logPath, + self.startTime, self.endTime, self.finalMetricData, self.stderrPath) + +class Experiment: + def __init__(self): + self._endpoint = None + self._exp_id = None + self._port = None + + @property + def endpoint(self): + return self._endpoint + + @property + def exp_id(self): + return self._exp_id + + @property + def port(self): + return self._port + + def _exec_command(self, cmd, port=None): + if self._endpoint is not None: + raise RuntimeError('This instance has been connected to an experiment.') + if _create_process(cmd) != 0: + raise RuntimeError('Failed to establish experiment, please check your config.') + else: + if port: + self._port = port + else: + self._port = 8080 + self._endpoint = 'http://localhost:{}'.format(self._port) + self._exp_id = self.get_experiment_profile()['id'] + + def start_experiment(self, config_file, port=None, debug=False): + """ + Start an experiment with specified configuration file and connect to it. + + Parameters + ---------- + config_file: str + Path to the config file. + port: int + The port of restful server, bigger than 1024. + debug: boolean + Set debug mode. + """ + cmd = 'nnictl create --config {}'.format(config_file).split(' ') + if port: + cmd += '--port {}'.format(port).split(' ') + if debug: + cmd += ['--debug'] + self._exec_command(cmd, port) + + def resume_experiment(self, exp_id, port=None, debug=False): + """ + Resume a stopped experiment with specified experiment id + + Parameters + ---------- + exp_id: str + Experiment id. + port: int + The port of restful server, bigger than 1024. + debug: boolean + Set debug mode. + """ + cmd = 'nnictl resume {}'.format(exp_id).split(' ') + if port: + cmd += '--port {}'.format(port).split(' ') + if debug: + cmd += ['--debug'] + self._exec_command(cmd, port) + + def view_experiment(self, exp_id, port=None): + """ + View a stopped experiment with specified experiment id. + + Parameters + ---------- + exp_id: str + Experiment id. + port: int + The port of restful server, bigger than 1024. + """ + cmd = 'nnictl view {}'.format(exp_id).split(' ') + if port: + cmd += '--port {}'.format(port).split(' ') + self._exec_command(cmd, port) + + def connect_experiment(self, endpoint): + """ + Connect to an existing experiment. + + Parameters + ---------- + endpoint: str + The endpoint of nni rest server, i.e, the url of Web UI. Should be a format like `http://ip:port`. + """ + if self._endpoint is not None: + raise RuntimeError('This instance has been connected to an experiment.') + self._endpoint = endpoint + try: + self._exp_id = self.get_experiment_profile()['id'] + except TypeError: + raise RuntimeError('Invalid experiment endpoint.') + self._port = int(re.search(r':[0-9]+', self._endpoint).group().replace(':', '')) + + def stop_experiment(self): + """Stop the experiment. + """ + _check_endpoint(self._endpoint) + cmd = 'nnictl stop {}'.format(self._exp_id).split(' ') + if _create_process(cmd) != 0: + raise RuntimeError('Failed to stop experiment.') + self._endpoint = None + self._exp_id = None + self._port = None + + def update_searchspace(self, filename): + """ + Update the experiment's search space. + + Parameters + ---------- + filename: str + Path to the searchspace file. + """ + _check_endpoint(self._endpoint) + cmd = 'nnictl update searchspace {} --filename {}'.format(self._exp_id, filename).split(' ') + if _create_process(cmd) != 0: + raise RuntimeError('Failed to update searchspace.') + + def update_concurrency(self, value): + """ + Update an experiment's concurrency + + Parameters + ---------- + value: int + New concurrency value. + """ + _check_endpoint(self._endpoint) + cmd = 'nnictl update concurrency {} --value {}'.format(self._exp_id, value).split(' ') + if _create_process(cmd) != 0: + raise RuntimeError('Failed to update concurrency.') + + def update_duration(self, value): + """ + Update an experiment's duration + + Parameters + ---------- + value: str + Strings like '1m' for one minute or '2h' for two hours. + SUFFIX may be 's' for seconds, 'm' for minutes, 'h' for hours or 'd' for days. + """ + _check_endpoint(self._endpoint) + cmd = 'nnictl update duration {} --value {}'.format(self._exp_id, value).split(' ') + if _create_process(cmd) != 0: + raise RuntimeError('Failed to update duration.') + + def update_trailnum(self, value): + """ + Update an experiment's maxtrialnum + + Parameters + ---------- + value: int + New trailnum value. + """ + _check_endpoint(self._endpoint) + cmd = 'nnictl update trialnum {} --value {}'.format(self._exp_id, value).split(' ') + if _create_process(cmd) != 0: + raise RuntimeError('Failed to update trailnum.') + + def get_experiment_status(self): + """ + Return experiment status as a dict. + + Returns + ---------- + dict + Experiment status. + """ + _check_endpoint(self._endpoint) + return _nni_rest_get(self._endpoint, STATUS_PATH) + + def get_trial_job(self, trial_job_id): + """ + Return a trial job. + + Parameters + ---------- + trial_job_id: str + Trial job id. + + Returns + ---------- + nnicli.TrialJob + A `nnicli.TrialJob` instance corresponding to `trial_job_id`. + """ + _check_endpoint(self._endpoint) + assert trial_job_id is not None + trial_job = _nni_rest_get(self._endpoint, os.path.join(TRIAL_JOBS_PATH, trial_job_id)) + return TrialJob(trial_job) + + def list_trial_jobs(self): + """ + Return information for all trial jobs as a list. + + Returns + ---------- + list + List of `nnicli.TrialJob`. + """ + _check_endpoint(self._endpoint) + trial_jobs = _nni_rest_get(self._endpoint, TRIAL_JOBS_PATH) + return [TrialJob(e) for e in trial_jobs] + + def get_job_statistics(self): + """ + Return trial job statistics information as a dict. + + Returns + ---------- + list + Job statistics information. + """ + _check_endpoint(self._endpoint) + return _nni_rest_get(self._endpoint, JOB_STATISTICS_PATH) + + def get_job_metrics(self, trial_job_id=None): + """ + Return trial job metrics. + + Parameters + ---------- + trial_job_id: str + trial job id. if this parameter is None, all trail jobs' metrics will be returned. + + Returns + ---------- + dict + Each key is a trialJobId, the corresponding value is a list of `nnicli.TrialMetricData`. + """ + _check_endpoint(self._endpoint) + api_path = METRICS_PATH if trial_job_id is None else os.path.join(METRICS_PATH, trial_job_id) + output = {} + trail_metrics = _nni_rest_get(self._endpoint, api_path) + for metric in trail_metrics: + trial_id = metric["trialJobId"] + if trial_id not in output: + output[trial_id] = [TrialMetricData(metric)] + else: + output[trial_id].append(TrialMetricData(metric)) + return output + + def export_data(self): + """ + Return exported information for all trial jobs. + + Returns + ---------- + list + List of `nnicli.TrialResult`. + """ + _check_endpoint(self._endpoint) + trial_results = _nni_rest_get(self._endpoint, EXPORT_DATA_PATH) + return [TrialResult(e) for e in trial_results] + + def get_experiment_profile(self): + """ + Return experiment profile as a dict. + + Returns + ---------- + dict + The profile of the experiment. + """ + _check_endpoint(self._endpoint) + return _nni_rest_get(self._endpoint, EXPERIMENT_PATH) diff --git a/new_impl/cv/third_party/nni/parameter_expressions.py b/new_impl/cv/third_party/nni/parameter_expressions.py new file mode 100644 index 0000000000000000000000000000000000000000..adff923f7ea1cb4debe114c22376522adb8d8a6d --- /dev/null +++ b/new_impl/cv/third_party/nni/parameter_expressions.py @@ -0,0 +1,108 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +''' +parameter_expression.py +''' + +import numpy as np + + +def choice(options, random_state): + ''' + options: 1-D array-like or int + random_state: an object of numpy.random.RandomState + ''' + return random_state.choice(options) + + +def randint(lower, upper, random_state): + ''' + Generate a random integer from `lower` (inclusive) to `upper` (exclusive). + lower: an int that represent an lower bound + upper: an int that represent an upper bound + random_state: an object of numpy.random.RandomState + ''' + return random_state.randint(lower, upper) + + +def uniform(low, high, random_state): + ''' + low: an float that represent an lower bound + high: an float that represent an upper bound + random_state: an object of numpy.random.RandomState + ''' + assert high >= low, 'Upper bound must be larger than lower bound' + return random_state.uniform(low, high) + + +def quniform(low, high, q, random_state): + ''' + low: an float that represent an lower bound + high: an float that represent an upper bound + q: sample step + random_state: an object of numpy.random.RandomState + ''' + return np.clip(np.round(uniform(low, high, random_state) / q) * q, low, high) + + +def loguniform(low, high, random_state): + ''' + low: an float that represent an lower bound + high: an float that represent an upper bound + random_state: an object of numpy.random.RandomState + ''' + assert low > 0, 'Lower bound must be positive' + return np.exp(uniform(np.log(low), np.log(high), random_state)) + + +def qloguniform(low, high, q, random_state): + ''' + low: an float that represent an lower bound + high: an float that represent an upper bound + q: sample step + random_state: an object of numpy.random.RandomState + ''' + return np.clip(np.round(loguniform(low, high, random_state) / q) * q, low, high) + + +def normal(mu, sigma, random_state): + ''' + The probability density function of the normal distribution, + first derived by De Moivre and 200 years later by both Gauss and Laplace independently. + mu: float or array_like of floats + Mean (“centre”) of the distribution. + sigma: float or array_like of floats + Standard deviation (spread or “width”) of the distribution. + random_state: an object of numpy.random.RandomState + ''' + return random_state.normal(mu, sigma) + + +def qnormal(mu, sigma, q, random_state): + ''' + mu: float or array_like of floats + sigma: float or array_like of floats + q: sample step + random_state: an object of numpy.random.RandomState + ''' + return np.round(normal(mu, sigma, random_state) / q) * q + + +def lognormal(mu, sigma, random_state): + ''' + mu: float or array_like of floats + sigma: float or array_like of floats + random_state: an object of numpy.random.RandomState + ''' + return np.exp(normal(mu, sigma, random_state)) + + +def qlognormal(mu, sigma, q, random_state): + ''' + mu: float or array_like of floats + sigma: float or array_like of floats + q: sample step + random_state: an object of numpy.random.RandomState + ''' + return np.round(lognormal(mu, sigma, random_state) / q) * q diff --git a/new_impl/cv/third_party/nni/recoverable.py b/new_impl/cv/third_party/nni/recoverable.py new file mode 100644 index 0000000000000000000000000000000000000000..70f11e634dc74886c0d379ca7fc86202654bf6ad --- /dev/null +++ b/new_impl/cv/third_party/nni/recoverable.py @@ -0,0 +1,18 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os + +class Recoverable: + + def load_checkpoint(self): + pass + + def save_checkpoint(self): + pass + + def get_checkpoint_path(self): + ckp_path = os.getenv('NNI_CHECKPOINT_DIRECTORY') + if ckp_path is not None and os.path.isdir(ckp_path): + return ckp_path + return None diff --git a/new_impl/cv/third_party/nni/runtime/__init__.py b/new_impl/cv/third_party/nni/runtime/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/new_impl/cv/third_party/nni/runtime/common.py b/new_impl/cv/third_party/nni/runtime/common.py new file mode 100644 index 0000000000000000000000000000000000000000..ec5ef10162b0936a402c0f1468f314e3eccebfdc --- /dev/null +++ b/new_impl/cv/third_party/nni/runtime/common.py @@ -0,0 +1,108 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from datetime import datetime +from io import TextIOBase +import logging +import os +import sys +import time + +log_level_map = { + 'fatal': logging.FATAL, + 'error': logging.ERROR, + 'warning': logging.WARNING, + 'info': logging.INFO, + 'debug': logging.DEBUG +} + +_time_format = '%m/%d/%Y, %I:%M:%S %p' + +# FIXME +# This hotfix the bug that querying installed tuners with `package_utils` will activate dispatcher logger. +# This behavior depends on underlying implementation of `nnictl` and is likely to break in future. +_logger_initialized = False + +class _LoggerFileWrapper(TextIOBase): + def __init__(self, logger_file): + self.file = logger_file + + def write(self, s): + if s != '\n': + cur_time = datetime.now().strftime(_time_format) + self.file.write('[{}] PRINT '.format(cur_time) + s + '\n') + self.file.flush() + return len(s) + +def init_logger(logger_file_path, log_level_name='info'): + """Initialize root logger. + This will redirect anything from logging.getLogger() as well as stdout to specified file. + logger_file_path: path of logger file (path-like object). + """ + global _logger_initialized + if _logger_initialized: + return + _logger_initialized = True + + if os.environ.get('NNI_PLATFORM') == 'unittest': + return # fixme: launching logic needs refactor + + log_level = log_level_map.get(log_level_name, logging.INFO) + logger_file = open(logger_file_path, 'w') + fmt = '[%(asctime)s] %(levelname)s (%(name)s/%(threadName)s) %(message)s' + logging.Formatter.converter = time.localtime + formatter = logging.Formatter(fmt, _time_format) + handler = logging.StreamHandler(logger_file) + handler.setFormatter(formatter) + + root_logger = logging.getLogger() + root_logger.addHandler(handler) + root_logger.setLevel(log_level) + + # these modules are too verbose + logging.getLogger('matplotlib').setLevel(log_level) + + sys.stdout = _LoggerFileWrapper(logger_file) + +def init_standalone_logger(): + """ + Initialize root logger for standalone mode. + This will set NNI's log level to INFO and print its log to stdout. + """ + global _logger_initialized + if _logger_initialized: + return + _logger_initialized = True + + fmt = '[%(asctime)s] %(levelname)s (%(name)s) %(message)s' + formatter = logging.Formatter(fmt, _time_format) + handler = logging.StreamHandler(sys.stdout) + handler.setFormatter(formatter) + nni_logger = logging.getLogger('nni') + nni_logger.addHandler(handler) + nni_logger.setLevel(logging.INFO) + nni_logger.propagate = False + + # Following line does not affect NNI loggers, but without this user's logger won't be able to + # print log even it's level is set to INFO, so we do it for user's convenience. + # If this causes any issue in future, remove it and use `logging.info` instead of + # `logging.getLogger('xxx')` in all examples. + logging.basicConfig() + + +_multi_thread = False +_multi_phase = False + +def enable_multi_thread(): + global _multi_thread + _multi_thread = True + +def multi_thread_enabled(): + return _multi_thread + +def enable_multi_phase(): + global _multi_phase + _multi_phase = True + +def multi_phase_enabled(): + return _multi_phase diff --git a/new_impl/cv/third_party/nni/runtime/env_vars.py b/new_impl/cv/third_party/nni/runtime/env_vars.py new file mode 100644 index 0000000000000000000000000000000000000000..5227956012f2f8b7b888f859a0faaa324f2800e4 --- /dev/null +++ b/new_impl/cv/third_party/nni/runtime/env_vars.py @@ -0,0 +1,33 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +from collections import namedtuple + + +_trial_env_var_names = [ + 'NNI_PLATFORM', + 'NNI_EXP_ID', + 'NNI_TRIAL_JOB_ID', + 'NNI_SYS_DIR', + 'NNI_OUTPUT_DIR', + 'NNI_TRIAL_SEQ_ID', + 'MULTI_PHASE' +] + +_dispatcher_env_var_names = [ + 'SDK_PROCESS', + 'NNI_MODE', + 'NNI_CHECKPOINT_DIRECTORY', + 'NNI_LOG_DIRECTORY', + 'NNI_LOG_LEVEL', + 'NNI_INCLUDE_INTERMEDIATE_RESULTS' +] + +def _load_env_vars(env_var_names): + env_var_dict = {k: os.environ.get(k) for k in env_var_names} + return namedtuple('EnvVars', env_var_names)(**env_var_dict) + +trial_env_vars = _load_env_vars(_trial_env_var_names) + +dispatcher_env_vars = _load_env_vars(_dispatcher_env_var_names) diff --git a/new_impl/cv/third_party/nni/runtime/msg_dispatcher.py b/new_impl/cv/third_party/nni/runtime/msg_dispatcher.py new file mode 100644 index 0000000000000000000000000000000000000000..4f24294fc750f6f5a341d0a28388c7dcef3c6d1b --- /dev/null +++ b/new_impl/cv/third_party/nni/runtime/msg_dispatcher.py @@ -0,0 +1,239 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +from collections import defaultdict +import json_tricks + +from nni import NoMoreTrialError +from .protocol import CommandType, send +from .msg_dispatcher_base import MsgDispatcherBase +from nni.assessor import AssessResult +from .common import multi_thread_enabled, multi_phase_enabled +from .env_vars import dispatcher_env_vars +from ..utils import MetricType, to_json + +_logger = logging.getLogger(__name__) + +# Assessor global variables +_trial_history = defaultdict(dict) +'''key: trial job ID; value: intermediate results, mapping from sequence number to data''' + +_ended_trials = set() +'''trial_job_id of all ended trials. +We need this because NNI manager may send metrics after reporting a trial ended. +TODO: move this logic to NNI manager +''' + + +def _sort_history(history): + ret = [] + for i, _ in enumerate(history): + if i in history: + ret.append(history[i]) + else: + break + return ret + + +# Tuner global variables +_next_parameter_id = 0 +_trial_params = {} +'''key: trial job ID; value: parameters''' +_customized_parameter_ids = set() + + +def _create_parameter_id(): + global _next_parameter_id + _next_parameter_id += 1 + return _next_parameter_id - 1 + + +def _pack_parameter(parameter_id, params, customized=False, trial_job_id=None, parameter_index=None): + _trial_params[parameter_id] = params + ret = { + 'parameter_id': parameter_id, + 'parameter_source': 'customized' if customized else 'algorithm', + 'parameters': params + } + if trial_job_id is not None: + ret['trial_job_id'] = trial_job_id + if parameter_index is not None: + ret['parameter_index'] = parameter_index + else: + ret['parameter_index'] = 0 + return to_json(ret) + + +class MsgDispatcher(MsgDispatcherBase): + def __init__(self, tuner, assessor=None): + super(MsgDispatcher, self).__init__() + self.tuner = tuner + self.assessor = assessor + if assessor is None: + _logger.debug('Assessor is not configured') + + def load_checkpoint(self): + self.tuner.load_checkpoint() + if self.assessor is not None: + self.assessor.load_checkpoint() + + def save_checkpoint(self): + self.tuner.save_checkpoint() + if self.assessor is not None: + self.assessor.save_checkpoint() + + def handle_initialize(self, data): + """Data is search space + """ + self.tuner.update_search_space(data) + send(CommandType.Initialized, '') + + def send_trial_callback(self, id_, params): + """For tuner to issue trial config when the config is generated + """ + send(CommandType.NewTrialJob, _pack_parameter(id_, params)) + + def handle_request_trial_jobs(self, data): + # data: number or trial jobs + ids = [_create_parameter_id() for _ in range(data)] + _logger.debug("requesting for generating params of %s", ids) + params_list = self.tuner.generate_multiple_parameters(ids, st_callback=self.send_trial_callback) + + for i, _ in enumerate(params_list): + send(CommandType.NewTrialJob, _pack_parameter(ids[i], params_list[i])) + # when parameters is None. + if len(params_list) < len(ids): + send(CommandType.NoMoreTrialJobs, _pack_parameter(ids[0], '')) + + def handle_update_search_space(self, data): + self.tuner.update_search_space(data) + + def handle_import_data(self, data): + """Import additional data for tuning + data: a list of dictionaries, each of which has at least two keys, 'parameter' and 'value' + """ + for entry in data: + entry['value'] = entry['value'] if type(entry['value']) is str else json_tricks.dumps(entry['value']) + entry['value'] = json_tricks.loads(entry['value']) + self.tuner.import_data(data) + + def handle_add_customized_trial(self, data): + # data: parameters + id_ = _create_parameter_id() + _customized_parameter_ids.add(id_) + + def handle_report_metric_data(self, data): + """ + data: a dict received from nni_manager, which contains: + - 'parameter_id': id of the trial + - 'value': metric value reported by nni.report_final_result() + - 'type': report type, support {'FINAL', 'PERIODICAL'} + """ + # metrics value is dumped as json string in trial, so we need to decode it here + if 'value' in data: + data['value'] = json_tricks.loads(data['value']) + if data['type'] == MetricType.FINAL: + self._handle_final_metric_data(data) + elif data['type'] == MetricType.PERIODICAL: + if self.assessor is not None: + self._handle_intermediate_metric_data(data) + elif data['type'] == MetricType.REQUEST_PARAMETER: + assert multi_phase_enabled() + assert data['trial_job_id'] is not None + assert data['parameter_index'] is not None + param_id = _create_parameter_id() + try: + param = self.tuner.generate_parameters(param_id, trial_job_id=data['trial_job_id']) + except NoMoreTrialError: + param = None + send(CommandType.SendTrialJobParameter, _pack_parameter(param_id, param, trial_job_id=data['trial_job_id'], + parameter_index=data['parameter_index'])) + else: + raise ValueError('Data type not supported: {}'.format(data['type'])) + + def handle_trial_end(self, data): + """ + data: it has three keys: trial_job_id, event, hyper_params + - trial_job_id: the id generated by training service + - event: the job's state + - hyper_params: the hyperparameters generated and returned by tuner + """ + trial_job_id = data['trial_job_id'] + _ended_trials.add(trial_job_id) + if trial_job_id in _trial_history: + _trial_history.pop(trial_job_id) + if self.assessor is not None: + self.assessor.trial_end(trial_job_id, data['event'] == 'SUCCEEDED') + if self.tuner is not None: + self.tuner.trial_end(json_tricks.loads(data['hyper_params'])['parameter_id'], data['event'] == 'SUCCEEDED') + + def _handle_final_metric_data(self, data): + """Call tuner to process final results + """ + id_ = data['parameter_id'] + value = data['value'] + if id_ is None or id_ in _customized_parameter_ids: + if not hasattr(self.tuner, '_accept_customized'): + self.tuner._accept_customized = False + if not self.tuner._accept_customized: + _logger.info('Customized trial job %s ignored by tuner', id_) + return + customized = True + else: + customized = False + self.tuner.receive_trial_result(id_, _trial_params[id_], value, customized=customized, + trial_job_id=data.get('trial_job_id')) + + def _handle_intermediate_metric_data(self, data): + """Call assessor to process intermediate results + """ + if data['type'] != MetricType.PERIODICAL: + return + if self.assessor is None: + return + + trial_job_id = data['trial_job_id'] + if trial_job_id in _ended_trials: + return + + history = _trial_history[trial_job_id] + history[data['sequence']] = data['value'] + ordered_history = _sort_history(history) + if len(ordered_history) < data['sequence']: # no user-visible update since last time + return + + try: + result = self.assessor.assess_trial(trial_job_id, ordered_history) + except Exception as e: + _logger.error('Assessor error') + _logger.exception(e) + + if isinstance(result, bool): + result = AssessResult.Good if result else AssessResult.Bad + elif not isinstance(result, AssessResult): + msg = 'Result of Assessor.assess_trial must be an object of AssessResult, not %s' + raise RuntimeError(msg % type(result)) + + if result is AssessResult.Bad: + _logger.debug('BAD, kill %s', trial_job_id) + send(CommandType.KillTrialJob, json_tricks.dumps(trial_job_id)) + # notify tuner + _logger.debug('env var: NNI_INCLUDE_INTERMEDIATE_RESULTS: [%s]', + dispatcher_env_vars.NNI_INCLUDE_INTERMEDIATE_RESULTS) + if dispatcher_env_vars.NNI_INCLUDE_INTERMEDIATE_RESULTS == 'true': + self._earlystop_notify_tuner(data) + else: + _logger.debug('GOOD') + + def _earlystop_notify_tuner(self, data): + """Send last intermediate result as final result to tuner in case the + trial is early stopped. + """ + _logger.debug('Early stop notify tuner data: [%s]', data) + data['type'] = MetricType.FINAL + if multi_thread_enabled(): + self._handle_final_metric_data(data) + else: + data['value'] = to_json(data['value']) + self.enqueue_command(CommandType.ReportMetricData, data) diff --git a/new_impl/cv/third_party/nni/runtime/msg_dispatcher_base.py b/new_impl/cv/third_party/nni/runtime/msg_dispatcher_base.py new file mode 100644 index 0000000000000000000000000000000000000000..66af52df2853267a36fa2e8a52525cac001d9872 --- /dev/null +++ b/new_impl/cv/third_party/nni/runtime/msg_dispatcher_base.py @@ -0,0 +1,247 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import threading +import logging +from multiprocessing.dummy import Pool as ThreadPool +from queue import Queue, Empty +import json_tricks + +from .common import multi_thread_enabled +from .env_vars import dispatcher_env_vars +from ..utils import init_dispatcher_logger +from ..recoverable import Recoverable +from .protocol import CommandType, receive + +init_dispatcher_logger() + +_logger = logging.getLogger(__name__) + +QUEUE_LEN_WARNING_MARK = 20 +_worker_fast_exit_on_terminate = True + + +class MsgDispatcherBase(Recoverable): + """This is where tuners and assessors are not defined yet. + Inherits this class to make your own advisor. + """ + + def __init__(self): + if multi_thread_enabled(): + self.pool = ThreadPool() + self.thread_results = [] + else: + self.stopping = False + self.default_command_queue = Queue() + self.assessor_command_queue = Queue() + self.default_worker = threading.Thread(target=self.command_queue_worker, args=(self.default_command_queue,)) + self.assessor_worker = threading.Thread(target=self.command_queue_worker, + args=(self.assessor_command_queue,)) + self.default_worker.start() + self.assessor_worker.start() + self.worker_exceptions = [] + + def run(self): + """Run the tuner. + This function will never return unless raise. + """ + _logger.info('Start dispatcher') + if dispatcher_env_vars.NNI_MODE == 'resume': + self.load_checkpoint() + + while True: + command, data = receive() + if data: + data = json_tricks.loads(data) + + if command is None or command is CommandType.Terminate: + break + if multi_thread_enabled(): + result = self.pool.map_async(self.process_command_thread, [(command, data)]) + self.thread_results.append(result) + if any([thread_result.ready() and not thread_result.successful() for thread_result in + self.thread_results]): + _logger.debug('Caught thread exception') + break + else: + self.enqueue_command(command, data) + if self.worker_exceptions: + break + + _logger.info('Dispatcher exiting...') + self.stopping = True + if multi_thread_enabled(): + self.pool.close() + self.pool.join() + else: + self.default_worker.join() + self.assessor_worker.join() + + _logger.info('Terminated by NNI manager') + + def command_queue_worker(self, command_queue): + """Process commands in command queues. + """ + while True: + try: + # set timeout to ensure self.stopping is checked periodically + command, data = command_queue.get(timeout=3) + try: + self.process_command(command, data) + except Exception as e: + _logger.exception(e) + self.worker_exceptions.append(e) + break + except Empty: + pass + if self.stopping and (_worker_fast_exit_on_terminate or command_queue.empty()): + break + + def enqueue_command(self, command, data): + """Enqueue command into command queues + """ + if command == CommandType.TrialEnd or ( + command == CommandType.ReportMetricData and data['type'] == 'PERIODICAL'): + self.assessor_command_queue.put((command, data)) + else: + self.default_command_queue.put((command, data)) + + qsize = self.default_command_queue.qsize() + if qsize >= QUEUE_LEN_WARNING_MARK: + _logger.warning('default queue length: %d', qsize) + + qsize = self.assessor_command_queue.qsize() + if qsize >= QUEUE_LEN_WARNING_MARK: + _logger.warning('assessor queue length: %d', qsize) + + def process_command_thread(self, request): + """Worker thread to process a command. + """ + command, data = request + if multi_thread_enabled(): + try: + self.process_command(command, data) + except Exception as e: + _logger.exception(str(e)) + raise + else: + pass + + def process_command(self, command, data): + _logger.debug('process_command: command: [%s], data: [%s]', command, data) + + command_handlers = { + # Tuner commands: + CommandType.Initialize: self.handle_initialize, + CommandType.RequestTrialJobs: self.handle_request_trial_jobs, + CommandType.UpdateSearchSpace: self.handle_update_search_space, + CommandType.ImportData: self.handle_import_data, + CommandType.AddCustomizedTrialJob: self.handle_add_customized_trial, + + # Tuner/Assessor commands: + CommandType.ReportMetricData: self.handle_report_metric_data, + + CommandType.TrialEnd: self.handle_trial_end, + CommandType.Ping: self.handle_ping, + } + if command not in command_handlers: + raise AssertionError('Unsupported command: {}'.format(command)) + command_handlers[command](data) + + def handle_ping(self, data): + pass + + def handle_initialize(self, data): + """Initialize search space and tuner, if any + This method is meant to be called only once for each experiment, after calling this method, + dispatcher should `send(CommandType.Initialized, '')`, to set the status of the experiment to be "INITIALIZED". + Parameters + ---------- + data: dict + search space + """ + raise NotImplementedError('handle_initialize not implemented') + + def handle_request_trial_jobs(self, data): + """The message dispatcher is demanded to generate ``data`` trial jobs. + These trial jobs should be sent via ``send(CommandType.NewTrialJob, json_tricks.dumps(parameter))``, + where ``parameter`` will be received by NNI Manager and eventually accessible to trial jobs as "next parameter". + Semantically, message dispatcher should do this ``send`` exactly ``data`` times. + + The JSON sent by this method should follow the format of + + :: + + { + "parameter_id": 42 + "parameters": { + // this will be received by trial + }, + "parameter_source": "algorithm" // optional + } + + Parameters + ---------- + data: int + number of trial jobs + """ + raise NotImplementedError('handle_request_trial_jobs not implemented') + + def handle_update_search_space(self, data): + """This method will be called when search space is updated. + It's recommended to call this method in `handle_initialize` to initialize search space. + *No need to* notify NNI Manager when this update is done. + Parameters + ---------- + data: dict + search space + """ + raise NotImplementedError('handle_update_search_space not implemented') + + def handle_import_data(self, data): + """Import previous data when experiment is resumed. + Parameters + ---------- + data: list + a list of dictionaries, each of which has at least two keys, 'parameter' and 'value' + """ + raise NotImplementedError('handle_import_data not implemented') + + def handle_add_customized_trial(self, data): + """Experimental API. Not recommended for usage. + """ + raise NotImplementedError('handle_add_customized_trial not implemented') + + def handle_report_metric_data(self, data): + """Called when metric data is reported or new parameters are requested (for multiphase). + When new parameters are requested, this method should send a new parameter. + + Parameters + ---------- + data: dict + a dict which contains 'parameter_id', 'value', 'trial_job_id', 'type', 'sequence'. + type: can be `MetricType.REQUEST_PARAMETER`, `MetricType.FINAL` or `MetricType.PERIODICAL`. + `REQUEST_PARAMETER` is used to request new parameters for multiphase trial job. In this case, + the dict will contain additional keys: `trial_job_id`, `parameter_index`. Refer to `msg_dispatcher.py` + as an example. + + Raises + ------ + ValueError + Data type is not supported + """ + raise NotImplementedError('handle_report_metric_data not implemented') + + def handle_trial_end(self, data): + """Called when the state of one of the trials is changed + + Parameters + ---------- + data: dict + a dict with keys: trial_job_id, event, hyper_params. + trial_job_id: the id generated by training service. + event: the job’s state. + hyper_params: the string that is sent by message dispatcher during the creation of trials. + + """ + raise NotImplementedError('handle_trial_end not implemented') diff --git a/new_impl/cv/third_party/nni/runtime/platform/__init__.py b/new_impl/cv/third_party/nni/runtime/platform/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..84f04a9862a91931ccf7cd18fee93268af86bfbf --- /dev/null +++ b/new_impl/cv/third_party/nni/runtime/platform/__init__.py @@ -0,0 +1,15 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from ..env_vars import trial_env_vars, dispatcher_env_vars + +assert dispatcher_env_vars.SDK_PROCESS != 'dispatcher' + +if trial_env_vars.NNI_PLATFORM is None: + from .standalone import * +elif trial_env_vars.NNI_PLATFORM == 'unittest': + from .test import * +elif trial_env_vars.NNI_PLATFORM in ('local', 'remote', 'pai', 'kubeflow', 'frameworkcontroller', 'paiYarn', 'dlts', 'aml'): + from .local import * +else: + raise RuntimeError('Unknown platform %s' % trial_env_vars.NNI_PLATFORM) diff --git a/new_impl/cv/third_party/nni/runtime/platform/local.py b/new_impl/cv/third_party/nni/runtime/platform/local.py new file mode 100644 index 0000000000000000000000000000000000000000..5d8124d3ff11e8c6b136132ca6d65d8dd2d73e35 --- /dev/null +++ b/new_impl/cv/third_party/nni/runtime/platform/local.py @@ -0,0 +1,86 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +import sys +import json +import time +import subprocess + +from ..common import init_logger +from ..env_vars import trial_env_vars +from nni.utils import to_json + +_sysdir = trial_env_vars.NNI_SYS_DIR +if not os.path.exists(os.path.join(_sysdir, '.nni')): + os.makedirs(os.path.join(_sysdir, '.nni')) +_metric_file = open(os.path.join(_sysdir, '.nni', 'metrics'), 'wb') + +_outputdir = trial_env_vars.NNI_OUTPUT_DIR +if not os.path.exists(_outputdir): + os.makedirs(_outputdir) + +_nni_platform = trial_env_vars.NNI_PLATFORM +if _nni_platform == 'local': + _log_file_path = os.path.join(_outputdir, 'trial.log') + init_logger(_log_file_path) + +_multiphase = trial_env_vars.MULTI_PHASE + +_param_index = 0 + +def request_next_parameter(): + metric = to_json({ + 'trial_job_id': trial_env_vars.NNI_TRIAL_JOB_ID, + 'type': 'REQUEST_PARAMETER', + 'sequence': 0, + 'parameter_index': _param_index + }) + send_metric(metric) + +def get_next_parameter(): + global _param_index + params_file_name = '' + if _multiphase in ('true', 'True'): + params_file_name = ('parameter_{}.cfg'.format(_param_index), 'parameter.cfg')[_param_index == 0] + else: + if _param_index > 0: + return None + elif _param_index == 0: + params_file_name = 'parameter.cfg' + else: + raise AssertionError('_param_index value ({}) should >=0'.format(_param_index)) + + params_filepath = os.path.join(_sysdir, params_file_name) + if not os.path.isfile(params_filepath): + request_next_parameter() + while not (os.path.isfile(params_filepath) and os.path.getsize(params_filepath) > 0): + time.sleep(3) + params_file = open(params_filepath, 'r') + params = json.load(params_file) + _param_index += 1 + return params + +def send_metric(string): + if _nni_platform != 'local': + assert len(string) < 1000000, 'Metric too long' + print("NNISDK_MEb'%s'" % (string), flush=True) + else: + data = (string + '\n').encode('utf8') + assert len(data) < 1000000, 'Metric too long' + _metric_file.write(b'ME%06d%b' % (len(data), data)) + _metric_file.flush() + if sys.platform == "win32": + file = open(_metric_file.name) + file.close() + else: + subprocess.run(['touch', _metric_file.name], check=True) + +def get_experiment_id(): + return trial_env_vars.NNI_EXP_ID + +def get_trial_id(): + return trial_env_vars.NNI_TRIAL_JOB_ID + +def get_sequence_id(): + return int(trial_env_vars.NNI_TRIAL_SEQ_ID) diff --git a/new_impl/cv/third_party/nni/runtime/platform/standalone.py b/new_impl/cv/third_party/nni/runtime/platform/standalone.py new file mode 100644 index 0000000000000000000000000000000000000000..27c2e94db20c309a411619b7484f8c238d9b29a6 --- /dev/null +++ b/new_impl/cv/third_party/nni/runtime/platform/standalone.py @@ -0,0 +1,44 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +import json_tricks + +from ..common import init_standalone_logger + +__all__ = [ + 'get_next_parameter', + 'get_experiment_id', + 'get_trial_id', + 'get_sequence_id', + 'send_metric', +] + +init_standalone_logger() +_logger = logging.getLogger('nni') + + +def get_next_parameter(): + _logger.warning('Requesting parameter without NNI framework, returning empty dict') + return { + 'parameter_id': None, + 'parameters': {} + } + +def get_experiment_id(): + return 'STANDALONE' + +def get_trial_id(): + return 'STANDALONE' + +def get_sequence_id(): + return 0 + +def send_metric(string): + metric = json_tricks.loads(string) + if metric['type'] == 'FINAL': + _logger.info('Final result: %s', metric['value']) + elif metric['type'] == 'PERIODICAL': + _logger.info('Intermediate result: %s (Index %s)', metric['value'], metric['sequence']) + else: + _logger.error('Unexpected metric: %s', string) diff --git a/new_impl/cv/third_party/nni/runtime/platform/test.py b/new_impl/cv/third_party/nni/runtime/platform/test.py new file mode 100644 index 0000000000000000000000000000000000000000..9bc9651eb0a953420b00c50408248939f2e5ecb6 --- /dev/null +++ b/new_impl/cv/third_party/nni/runtime/platform/test.py @@ -0,0 +1,39 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +# pylint: skip-file + +import copy +import json_tricks + + +_params = None +_last_metric = None + + +def get_next_parameter(): + return _params + +def get_experiment_id(): + return 'fakeidex' + +def get_trial_id(): + return 'fakeidtr' + +def get_sequence_id(): + return 0 + +def send_metric(string): + global _last_metric + _last_metric = string + + +def init_params(params): + global _params + _params = copy.deepcopy(params) + +def get_last_metric(): + metrics = json_tricks.loads(_last_metric) + metrics['value'] = json_tricks.loads(metrics['value']) + + return metrics diff --git a/new_impl/cv/third_party/nni/runtime/protocol.py b/new_impl/cv/third_party/nni/runtime/protocol.py new file mode 100644 index 0000000000000000000000000000000000000000..57ea7fbc0b5e99b7534808b4d4541185a23c4d8a --- /dev/null +++ b/new_impl/cv/third_party/nni/runtime/protocol.py @@ -0,0 +1,71 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +import os +import threading +from enum import Enum + + +class CommandType(Enum): + # in + Initialize = b'IN' + RequestTrialJobs = b'GE' + ReportMetricData = b'ME' + UpdateSearchSpace = b'SS' + ImportData = b'FD' + AddCustomizedTrialJob = b'AD' + TrialEnd = b'EN' + Terminate = b'TE' + Ping = b'PI' + + # out + Initialized = b'ID' + NewTrialJob = b'TR' + SendTrialJobParameter = b'SP' + NoMoreTrialJobs = b'NO' + KillTrialJob = b'KI' + +_lock = threading.Lock() +try: + if os.environ.get('NNI_PLATFORM') != 'unittest': + _in_file = open(3, 'rb') + _out_file = open(4, 'wb') +except OSError: + _msg = 'IPC pipeline not exists, maybe you are importing tuner/assessor from trial code?' + logging.getLogger(__name__).warning(_msg) + + +def send(command, data): + """Send command to Training Service. + command: CommandType object. + data: string payload. + """ + global _lock + try: + _lock.acquire() + data = data.encode('utf8') + msg = b'%b%014d%b' % (command.value, len(data), data) + logging.getLogger(__name__).debug('Sending command, data: [%s]', msg) + _out_file.write(msg) + _out_file.flush() + finally: + _lock.release() + + +def receive(): + """Receive a command from Training Service. + Returns a tuple of command (CommandType) and payload (str) + """ + header = _in_file.read(16) + logging.getLogger(__name__).debug('Received command, header: [%s]', header) + if header is None or len(header) < 16: + # Pipe EOF encountered + logging.getLogger(__name__).debug('Pipe EOF encountered') + return None, None + length = int(header[2:]) + data = _in_file.read(length) + command = CommandType(header[:2]) + data = data.decode('utf8') + logging.getLogger(__name__).debug('Received command, data: [%s]', data) + return command, data diff --git a/new_impl/cv/third_party/nni/smartparam.py b/new_impl/cv/third_party/nni/smartparam.py new file mode 100644 index 0000000000000000000000000000000000000000..dde0ac2bd64e8b22e2948909b116c33aefda7561 --- /dev/null +++ b/new_impl/cv/third_party/nni/smartparam.py @@ -0,0 +1,148 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import numpy as np + +from .runtime.env_vars import trial_env_vars +from . import trial +from . import parameter_expressions as param_exp +from .common.nas_utils import classic_mode, enas_mode, oneshot_mode, darts_mode + + +__all__ = [ + 'choice', + 'randint', + 'uniform', + 'quniform', + 'loguniform', + 'qloguniform', + 'normal', + 'qnormal', + 'lognormal', + 'qlognormal', + 'function_choice', + 'mutable_layer' +] + + +if trial_env_vars.NNI_PLATFORM is None: + def choice(*options, name=None): + return param_exp.choice(options, np.random.RandomState()) + + def randint(lower, upper, name=None): + return param_exp.randint(lower, upper, np.random.RandomState()) + + def uniform(low, high, name=None): + return param_exp.uniform(low, high, np.random.RandomState()) + + def quniform(low, high, q, name=None): + assert high > low, 'Upper bound must be larger than lower bound' + return param_exp.quniform(low, high, q, np.random.RandomState()) + + def loguniform(low, high, name=None): + assert low > 0, 'Lower bound must be positive' + return param_exp.loguniform(low, high, np.random.RandomState()) + + def qloguniform(low, high, q, name=None): + return param_exp.qloguniform(low, high, q, np.random.RandomState()) + + def normal(mu, sigma, name=None): + return param_exp.normal(mu, sigma, np.random.RandomState()) + + def qnormal(mu, sigma, q, name=None): + return param_exp.qnormal(mu, sigma, q, np.random.RandomState()) + + def lognormal(mu, sigma, name=None): + return param_exp.lognormal(mu, sigma, np.random.RandomState()) + + def qlognormal(mu, sigma, q, name=None): + return param_exp.qlognormal(mu, sigma, q, np.random.RandomState()) + + def function_choice(*funcs, name=None): + return param_exp.choice(funcs, np.random.RandomState())() + + def mutable_layer(): + raise RuntimeError('Cannot call nni.mutable_layer in this mode') + +else: + + def choice(options, name=None, key=None): + return options[_get_param(key)] + + def randint(lower, upper, name=None, key=None): + return _get_param(key) + + def uniform(low, high, name=None, key=None): + return _get_param(key) + + def quniform(low, high, q, name=None, key=None): + return _get_param(key) + + def loguniform(low, high, name=None, key=None): + return _get_param(key) + + def qloguniform(low, high, q, name=None, key=None): + return _get_param(key) + + def normal(mu, sigma, name=None, key=None): + return _get_param(key) + + def qnormal(mu, sigma, q, name=None, key=None): + return _get_param(key) + + def lognormal(mu, sigma, name=None, key=None): + return _get_param(key) + + def qlognormal(mu, sigma, q, name=None, key=None): + return _get_param(key) + + def function_choice(funcs, name=None, key=None): + return funcs[_get_param(key)]() + + def mutable_layer( + mutable_id, + mutable_layer_id, + funcs, + funcs_args, + fixed_inputs, + optional_inputs, + optional_input_size, + mode='classic_mode', + tf=None): + '''execute the chosen function and inputs. + Below is an example of chosen function and inputs: + { + "mutable_id": { + "mutable_layer_id": { + "chosen_layer": "pool", + "chosen_inputs": ["out1", "out3"] + } + } + } + Parameters: + --------------- + mutable_id: the name of this mutable_layer block (which could have multiple mutable layers) + mutable_layer_id: the name of a mutable layer in this block + funcs: dict of function calls + funcs_args: + fixed_inputs: + optional_inputs: dict of optional inputs + optional_input_size: number of candidate inputs to be chosen + tf: tensorflow module + ''' + args = (mutable_id, mutable_layer_id, funcs, funcs_args, fixed_inputs, optional_inputs, optional_input_size) + if mode == 'classic_mode': + return classic_mode(*args) + assert tf is not None, 'Internal Error: Tensorflow should not be None in modes other than classic_mode' + if mode == 'enas_mode': + return enas_mode(*args, tf) + if mode == 'oneshot_mode': + return oneshot_mode(*args, tf) + if mode == 'darts_mode': + return darts_mode(*args, tf) + raise RuntimeError('Unrecognized mode: %s' % mode) + + def _get_param(key): + if trial.get_current_parameter() is None: + trial.get_next_parameter() + return trial.get_current_parameter(key) diff --git a/new_impl/cv/third_party/nni/tools/__init__.py b/new_impl/cv/third_party/nni/tools/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/new_impl/cv/third_party/nni/tools/annotation/.gitignore b/new_impl/cv/third_party/nni/tools/annotation/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..36e264cf443988bf3101b2467e83b259bcd5a4ad --- /dev/null +++ b/new_impl/cv/third_party/nni/tools/annotation/.gitignore @@ -0,0 +1 @@ +_generated diff --git a/new_impl/cv/third_party/nni/tools/annotation/__init__.py b/new_impl/cv/third_party/nni/tools/annotation/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d9ba405c8844591ec083921c90ba2a1d67d52cf8 --- /dev/null +++ b/new_impl/cv/third_party/nni/tools/annotation/__init__.py @@ -0,0 +1,141 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +import sys +import shutil +import json + +from . import code_generator +from . import search_space_generator +from . import specific_code_generator + + +__all__ = ['generate_search_space', 'expand_annotations'] + +slash = '/' +if sys.platform == "win32": + slash = '\\' + +def generate_search_space(code_dir): + """Generate search space from Python source code. + Return a serializable search space object. + code_dir: directory path of source files (str) + """ + code_dir = str(code_dir) + search_space = {} + + if code_dir.endswith(slash): + code_dir = code_dir[:-1] + + for subdir, _, files in os.walk(code_dir): + # generate module name from path + if subdir == code_dir: + package = '' + else: + assert subdir.startswith(code_dir + slash), subdir + prefix_len = len(code_dir) + 1 + package = subdir[prefix_len:].replace(slash, '.') + '.' + + for file_name in files: + if file_name.endswith('.py'): + path = os.path.join(subdir, file_name) + module = package + file_name[:-3] + search_space.update(_generate_file_search_space(path, module)) + + return search_space + +def _generate_file_search_space(path, module): + with open(path) as src: + try: + search_space, code = search_space_generator.generate(module, src.read()) + except Exception as exc: # pylint: disable=broad-except + if exc.args: + raise RuntimeError(path + ' ' + '\n'.join(exc.args)) + else: + raise RuntimeError('Failed to generate search space for %s: %r' % (path, exc)) + with open(path, 'w') as dst: + dst.write(code) + return search_space + + +def expand_annotations(src_dir, dst_dir, exp_id='', trial_id='', nas_mode=None): + """Expand annotations in user code. + Return dst_dir if annotation detected; return src_dir if not. + src_dir: directory path of user code (str) + dst_dir: directory to place generated files (str) + nas_mode: the mode of NAS given that NAS interface is used + """ + src_dir, dst_dir = str(src_dir), str(dst_dir) + + if src_dir[-1] == slash: + src_dir = src_dir[:-1] + + if dst_dir[-1] == slash: + dst_dir = dst_dir[:-1] + + annotated = False + + for src_subdir, dirs, files in os.walk(src_dir): + assert src_subdir.startswith(src_dir) + dst_subdir = src_subdir.replace(src_dir, dst_dir, 1) + os.makedirs(dst_subdir, exist_ok=True) + + # generate module name from path + if src_subdir == src_dir: + package = '' + else: + assert src_subdir.startswith(src_dir + slash), src_subdir + prefix_len = len(src_dir) + 1 + package = src_subdir[prefix_len:].replace(slash, '.') + '.' + + for file_name in files: + src_path = os.path.join(src_subdir, file_name) + dst_path = os.path.join(dst_subdir, file_name) + if file_name.endswith('.py'): + if trial_id == '': + annotated |= _expand_file_annotations(src_path, dst_path, nas_mode) + else: + module = package + file_name[:-3] + annotated |= _generate_specific_file(src_path, dst_path, exp_id, trial_id, module) + else: + shutil.copyfile(src_path, dst_path) + + for dir_name in dirs: + os.makedirs(os.path.join(dst_subdir, dir_name), exist_ok=True) + + return dst_dir if annotated else src_dir + +def _expand_file_annotations(src_path, dst_path, nas_mode): + with open(src_path) as src, open(dst_path, 'w') as dst: + try: + annotated_code = code_generator.parse(src.read(), nas_mode) + if annotated_code is None: + shutil.copyfile(src_path, dst_path) + return False + dst.write(annotated_code) + return True + + except Exception as exc: # pylint: disable=broad-except + if exc.args: + raise RuntimeError(src_path + ' ' + '\n'.join(str(arg) for arg in exc.args)) + else: + raise RuntimeError('Failed to expand annotations for %s: %r' % (src_path, exc)) + +def _generate_specific_file(src_path, dst_path, exp_id, trial_id, module): + with open(src_path) as src, open(dst_path, 'w') as dst: + try: + with open(os.path.expanduser('~/nni-experiments/%s/trials/%s/parameter.cfg'%(exp_id, trial_id))) as fd: + para_cfg = json.load(fd) + annotated_code = specific_code_generator.parse(src.read(), para_cfg["parameters"], module) + if annotated_code is None: + shutil.copyfile(src_path, dst_path) + return False + dst.write(annotated_code) + return True + + except Exception as exc: # pylint: disable=broad-except + if exc.args: + raise RuntimeError(src_path + ' ' + '\n'.join(str(arg) for arg in exc.args)) + else: + raise RuntimeError('Failed to expand annotations for %s: %r' % (src_path, exc)) diff --git a/new_impl/cv/third_party/nni/tools/annotation/code_generator.py b/new_impl/cv/third_party/nni/tools/annotation/code_generator.py new file mode 100644 index 0000000000000000000000000000000000000000..c924e4195072eeff3e112b9ca5e01cb1df41c178 --- /dev/null +++ b/new_impl/cv/third_party/nni/tools/annotation/code_generator.py @@ -0,0 +1,369 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import ast +import astor + +from .utils import ast_Num, ast_Str, lineno + +# pylint: disable=unidiomatic-typecheck + +def parse_annotation_mutable_layers(code, lineno, nas_mode): + """Parse the string of mutable layers in annotation. + Return a list of AST Expr nodes + code: annotation string (excluding '@') + nas_mode: the mode of NAS + """ + module = ast.parse(code) + assert type(module) is ast.Module, 'internal error #1' + assert len(module.body) == 1, 'Annotation mutable_layers contains more than one expression' + assert type(module.body[0]) is ast.Expr, 'Annotation is not expression' + call = module.body[0].value + nodes = [] + mutable_id = 'mutable_block_' + str(lineno) + mutable_layer_cnt = 0 + for arg in call.args: + fields = {'layer_choice': False, + 'fixed_inputs': False, + 'optional_inputs': False, + 'optional_input_size': False, + 'layer_output': False} + for k, value in zip(arg.keys, arg.values): + if k.id == 'layer_choice': + assert not fields['layer_choice'], 'Duplicated field: layer_choice' + assert type(value) is ast.List, 'Value of layer_choice should be a list' + call_funcs_keys = [] + call_funcs_values = [] + call_kwargs_values = [] + for call in value.elts: + assert type(call) is ast.Call, 'Element in layer_choice should be function call' + call_name = astor.to_source(call).strip() + call_funcs_keys.append(ast_Str(s=call_name)) + call_funcs_values.append(call.func) + assert not call.args, 'Number of args without keyword should be zero' + kw_args = [] + kw_values = [] + for kw in call.keywords: + kw_args.append(ast_Str(s=kw.arg)) + kw_values.append(kw.value) + call_kwargs_values.append(ast.Dict(keys=kw_args, values=kw_values)) + call_funcs = ast.Dict(keys=call_funcs_keys, values=call_funcs_values) + call_kwargs = ast.Dict(keys=call_funcs_keys, values=call_kwargs_values) + fields['layer_choice'] = True + elif k.id == 'fixed_inputs': + assert not fields['fixed_inputs'], 'Duplicated field: fixed_inputs' + assert type(value) is ast.List, 'Value of fixed_inputs should be a list' + fixed_inputs = value + fields['fixed_inputs'] = True + elif k.id == 'optional_inputs': + assert not fields['optional_inputs'], 'Duplicated field: optional_inputs' + assert type(value) is ast.List, 'Value of optional_inputs should be a list' + var_names = [ast_Str(s=astor.to_source(var).strip()) for var in value.elts] + optional_inputs = ast.Dict(keys=var_names, values=value.elts) + fields['optional_inputs'] = True + elif k.id == 'optional_input_size': + assert not fields['optional_input_size'], 'Duplicated field: optional_input_size' + assert type(value) is ast_Num or type(value) is ast.List, \ + 'Value of optional_input_size should be a number or list' + optional_input_size = value + fields['optional_input_size'] = True + elif k.id == 'layer_output': + assert not fields['layer_output'], 'Duplicated field: layer_output' + assert type(value) is ast.Name, 'Value of layer_output should be ast.Name type' + layer_output = value + fields['layer_output'] = True + else: + raise AssertionError('Unexpected field in mutable layer') + # make call for this mutable layer + assert fields['layer_choice'], 'layer_choice must exist' + assert fields['layer_output'], 'layer_output must exist' + mutable_layer_id = 'mutable_layer_' + str(mutable_layer_cnt) + mutable_layer_cnt += 1 + target_call_attr = ast.Attribute(value=ast.Name(id='nni', ctx=ast.Load()), attr='mutable_layer', ctx=ast.Load()) + target_call_args = [ast_Str(s=mutable_id), + ast_Str(s=mutable_layer_id), + call_funcs, + call_kwargs] + if fields['fixed_inputs']: + target_call_args.append(fixed_inputs) + else: + target_call_args.append(ast.List(elts=[])) + if fields['optional_inputs']: + target_call_args.append(optional_inputs) + assert fields['optional_input_size'], 'optional_input_size must exist when optional_inputs exists' + target_call_args.append(optional_input_size) + else: + target_call_args.append(ast.Dict(keys=[], values=[])) + target_call_args.append(ast_Num(n=0)) + target_call_args.append(ast_Str(s=nas_mode)) + if nas_mode in ['enas_mode', 'oneshot_mode', 'darts_mode']: + target_call_args.append(ast.Name(id='tensorflow')) + target_call = ast.Call(func=target_call_attr, args=target_call_args, keywords=[]) + node = ast.Assign(targets=[layer_output], value=target_call) + nodes.append(node) + return nodes + + +def parse_annotation(code): + """Parse an annotation string. + Return an AST Expr node. + code: annotation string (excluding '@') + """ + module = ast.parse(code) + assert type(module) is ast.Module, 'internal error #1' + assert len(module.body) == 1, 'Annotation contains more than one expression' + assert type(module.body[0]) is ast.Expr, 'Annotation is not expression' + return module.body[0] + + +def parse_annotation_function(code, func_name): + """Parse an annotation function. + Return the value of `name` keyword argument and the AST Call node. + func_name: expected function name + """ + expr = parse_annotation(code) + call = expr.value + assert type(call) is ast.Call, 'Annotation is not a function call' + + assert type(call.func) is ast.Attribute, 'Unexpected annotation function' + assert type(call.func.value) is ast.Name, 'Invalid annotation function name' + assert call.func.value.id == 'nni', 'Annotation is not a NNI function' + assert call.func.attr == func_name, 'internal error #2' + + assert len(call.keywords) == 1, 'Annotation function contains more than one keyword argument' + assert call.keywords[0].arg == 'name', 'Annotation keyword argument is not "name"' + name = call.keywords[0].value + + return name, call + + +def parse_nni_variable(code): + """Parse `nni.variable` expression. + Return the name argument and AST node of annotated expression. + code: annotation string + """ + name, call = parse_annotation_function(code, 'variable') + + assert len(call.args) == 1, 'nni.variable contains more than one arguments' + arg = call.args[0] + assert type(arg) is ast.Call, 'Value of nni.variable is not a function call' + assert type(arg.func) is ast.Attribute, 'nni.variable value is not a NNI function' + assert type(arg.func.value) is ast.Name, 'nni.variable value is not a NNI function' + assert arg.func.value.id == 'nni', 'nni.variable value is not a NNI function' + + name_str = astor.to_source(name).strip() + keyword_arg = ast.keyword(arg='name', value=ast_Str(s=name_str)) + arg.keywords.append(keyword_arg) + if arg.func.attr == 'choice': + convert_args_to_dict(arg) + + return name, arg + + +def parse_nni_function(code): + """Parse `nni.function_choice` expression. + Return the AST node of annotated expression and a list of dumped function call expressions. + code: annotation string + """ + name, call = parse_annotation_function(code, 'function_choice') + funcs = [ast.dump(func, False) for func in call.args] + convert_args_to_dict(call, with_lambda=True) + + name_str = astor.to_source(name).strip() + call.keywords[0].value = ast_Str(s=name_str) + + return call, funcs + + +def convert_args_to_dict(call, with_lambda=False): + """Convert all args to a dict such that every key and value in the dict is the same as the value of the arg. + Return the AST Call node with only one arg that is the dictionary + """ + keys, values = list(), list() + for arg in call.args: + if type(arg) in [ast_Str, ast_Num]: + arg_value = arg + else: + # if arg is not a string or a number, we use its source code as the key + arg_value = astor.to_source(arg).strip('\n"') + arg_value = ast_Str(str(arg_value)) + arg = make_lambda(arg) if with_lambda else arg + keys.append(arg_value) + values.append(arg) + del call.args[:] + call.args.append(ast.Dict(keys=keys, values=values)) + + return call + + +def make_lambda(call): + """Wrap an AST Call node to lambda expression node. + call: ast.Call node + """ + empty_args = ast.arguments(args=[], vararg=None, kwarg=None, defaults=[]) + return ast.Lambda(args=empty_args, body=call) + + +def test_variable_equal(node1, node2): + """Test whether two variables are the same.""" + if type(node1) is not type(node2): + return False + if isinstance(node1, ast.AST): + for k, v in vars(node1).items(): + if k in ('lineno', 'col_offset', 'ctx', 'end_lineno', 'end_col_offset'): + continue + if not test_variable_equal(v, getattr(node2, k)): + return False + return True + if isinstance(node1, list): + if len(node1) != len(node2): + return False + return all(test_variable_equal(n1, n2) for n1, n2 in zip(node1, node2)) + + return node1 == node2 + + +def replace_variable_node(node, annotation): + """Replace a node annotated by `nni.variable`. + node: the AST node to replace + annotation: annotation string + """ + assert type(node) is ast.Assign, 'nni.variable is not annotating assignment expression' + assert len(node.targets) == 1, 'Annotated assignment has more than one left-hand value' + name, expr = parse_nni_variable(annotation) + assert test_variable_equal(node.targets[0], name), 'Annotated variable has wrong name' + node.value = expr + return node + + +def replace_function_node(node, annotation): + """Replace a node annotated by `nni.function_choice`. + node: the AST node to replace + annotation: annotation string + """ + target, funcs = parse_nni_function(annotation) + FuncReplacer(funcs, target).visit(node) + return node + + +class FuncReplacer(ast.NodeTransformer): + """To replace target function call expressions in a node annotated by `nni.function_choice`""" + + def __init__(self, funcs, target): + """Constructor. + funcs: list of dumped function call expressions to replace + target: use this AST node to replace matching expressions + """ + self.funcs = set(funcs) + self.target = target + + def visit_Call(self, node): # pylint: disable=invalid-name + if ast.dump(node, False) in self.funcs: + return self.target + return node + + +class Transformer(ast.NodeTransformer): + """Transform original code to annotated code""" + + def __init__(self, nas_mode=None): + self.stack = [] + self.last_line = 0 + self.annotated = False + self.nas_mode = nas_mode + + def visit(self, node): + if isinstance(node, (ast.expr, ast.stmt)): + self.last_line = lineno(node) + + # do nothing for root + if not self.stack: + return self._visit_children(node) + + annotation = self.stack[-1] + + # this is a standalone string, may be an annotation + if type(node) is ast.Expr and type(node.value) is ast_Str: + # must not annotate an annotation string + assert annotation is None, 'Annotating an annotation' + return self._visit_string(node) + + if annotation is not None: # this expression is annotated + self.stack[-1] = None # so next expression is not + if annotation.startswith('nni.variable'): + return replace_variable_node(node, annotation) + if annotation.startswith('nni.function_choice'): + return replace_function_node(node, annotation) + + return self._visit_children(node) + + def _visit_string(self, node): + string = node.value.s + if string.startswith('@nni.'): + self.annotated = True + else: + return node # not an annotation, ignore it + + if string.startswith('@nni.training_update'): + expr = parse_annotation(string[1:]) + call_node = expr.value + call_node.args.insert(0, ast_Str(s=self.nas_mode)) + return expr + + if string.startswith('@nni.report_intermediate_result') \ + or string.startswith('@nni.report_final_result') \ + or string.startswith('@nni.get_next_parameter'): + return parse_annotation(string[1:]) # expand annotation string to code + + if string.startswith('@nni.mutable_layers'): + nodes = parse_annotation_mutable_layers(string[1:], lineno(node), self.nas_mode) + return nodes + + if string.startswith('@nni.variable') \ + or string.startswith('@nni.function_choice'): + self.stack[-1] = string[1:] # mark that the next expression is annotated + return None + + raise AssertionError('Unexpected annotation function') + + def _visit_children(self, node): + self.stack.append(None) + self.generic_visit(node) + annotation = self.stack.pop() + assert annotation is None, 'Annotation has no target' + return node + + +def parse(code, nas_mode=None): + """Annotate user code. + Return annotated code (str) if annotation detected; return None if not. + code: original user code (str), + nas_mode: the mode of NAS given that NAS interface is used + """ + try: + ast_tree = ast.parse(code) + except Exception: + raise RuntimeError('Bad Python code') + + transformer = Transformer(nas_mode) + try: + transformer.visit(ast_tree) + except AssertionError as exc: + raise RuntimeError('%d: %s' % (ast_tree.last_line, exc.args[0])) + + if not transformer.annotated: + return None + + last_future_import = -1 + import_nni = ast.Import(names=[ast.alias(name='nni', asname=None)]) + nodes = ast_tree.body + for i, _ in enumerate(nodes): + if type(nodes[i]) is ast.ImportFrom and nodes[i].module == '__future__': + last_future_import = i + nodes.insert(last_future_import + 1, import_nni) + # enas, oneshot and darts modes for tensorflow need tensorflow module, so we import it here + if nas_mode in ['enas_mode', 'oneshot_mode', 'darts_mode']: + import_tf = ast.Import(names=[ast.alias(name='tensorflow', asname=None)]) + nodes.insert(last_future_import + 1, import_tf) + + return astor.to_source(ast_tree) diff --git a/new_impl/cv/third_party/nni/tools/annotation/search_space_generator.py b/new_impl/cv/third_party/nni/tools/annotation/search_space_generator.py new file mode 100644 index 0000000000000000000000000000000000000000..a0a19f53dff601faa682f48b3f1ee574f21b9acf --- /dev/null +++ b/new_impl/cv/third_party/nni/tools/annotation/search_space_generator.py @@ -0,0 +1,135 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import ast +import numbers + +import astor + +from .utils import ast_Num, ast_Str, lineno + +# pylint: disable=unidiomatic-typecheck + + +# list of functions related to search space generating +_ss_funcs = [ + 'choice', + 'randint', + 'uniform', + 'quniform', + 'loguniform', + 'qloguniform', + 'normal', + 'qnormal', + 'lognormal', + 'qlognormal', + 'function_choice', + 'mutable_layer' +] + + +class SearchSpaceGenerator(ast.NodeTransformer): + """Generate search space from smart parater APIs""" + + def __init__(self, module_name): + self.module_name = module_name + self.search_space = {} + self.last_line = 0 # last parsed line, useful for error reporting + + def generate_mutable_layer_search_space(self, args): + mutable_block = args[0].s + mutable_layer = args[1].s + key = self.module_name + '/' + mutable_block + args[0].s = key + if key not in self.search_space: + self.search_space[key] = {'_type': 'mutable_layer', '_value': {}} + self.search_space[key]['_value'][mutable_layer] = { + 'layer_choice': [k.s for k in args[2].keys], + 'optional_inputs': [k.s for k in args[5].keys], + 'optional_input_size': args[6].n if isinstance(args[6], ast_Num) else [args[6].elts[0].n, args[6].elts[1].n] + } + + def visit_Call(self, node): # pylint: disable=invalid-name + self.generic_visit(node) + + # ignore if the function is not 'nni.*' + if type(node.func) is not ast.Attribute: + return node + if type(node.func.value) is not ast.Name: + return node + if node.func.value.id != 'nni': + return node + + # ignore if its not a search space function (e.g. `report_final_result`) + func = node.func.attr + if func not in _ss_funcs: + return node + + self.last_line = lineno(node) + + if func == 'mutable_layer': + self.generate_mutable_layer_search_space(node.args) + return node + + if node.keywords: + # there is a `name` argument + assert len(node.keywords) == 1, 'Smart parameter has keyword argument other than "name"' + assert node.keywords[0].arg == 'name', 'Smart paramater\'s keyword argument is not "name"' + assert type(node.keywords[0].value) is ast_Str, 'Smart parameter\'s name must be string literal' + name = node.keywords[0].value.s + specified_name = True + else: + # generate the missing name automatically + name = '__line' + str(str(node.args[-1].lineno)) + specified_name = False + node.keywords = list() + + if func in ('choice', 'function_choice'): + # we will use keys in the dict as the choices, which is generated by code_generator according to the args given by user + assert len(node.args) == 1, 'Smart parameter has arguments other than dict' + # check if it is a number or a string and get its value accordingly + args = [key.n if type(key) is ast_Num else key.s for key in node.args[0].keys] + else: + # arguments of other functions must be literal number + assert all(isinstance(ast.literal_eval(astor.to_source(arg)), numbers.Real) for arg in node.args), \ + 'Smart parameter\'s arguments must be number literals' + args = [ast.literal_eval(astor.to_source(arg)) for arg in node.args] + + key = self.module_name + '/' + name + '/' + func + # store key in ast.Call + node.keywords.append(ast.keyword(arg='key', value=ast_Str(s=key))) + + if func == 'function_choice': + func = 'choice' + value = {'_type': func, '_value': args} + + if specified_name: + # multiple functions with same name must have identical arguments + old = self.search_space.get(key) + assert old is None or old == value, 'Different smart parameters have same name' + else: + # generated name must not duplicate + assert key not in self.search_space, 'Only one smart parameter is allowed in a line' + + self.search_space[key] = value + + return node + + +def generate(module_name, code): + """Generate search space. + Return a serializable search space object. + module_name: name of the module (str) + code: user code (str) + """ + try: + ast_tree = ast.parse(code) + except Exception: + raise RuntimeError('Bad Python code') + + visitor = SearchSpaceGenerator(module_name) + try: + visitor.visit(ast_tree) + except AssertionError as exc: + raise RuntimeError('%d: %s' % (visitor.last_line, exc.args[0])) + return visitor.search_space, astor.to_source(ast_tree) diff --git a/new_impl/cv/third_party/nni/tools/annotation/specific_code_generator.py b/new_impl/cv/third_party/nni/tools/annotation/specific_code_generator.py new file mode 100644 index 0000000000000000000000000000000000000000..5ddd362205dfa909d0077315ebe5dd9b8ade239f --- /dev/null +++ b/new_impl/cv/third_party/nni/tools/annotation/specific_code_generator.py @@ -0,0 +1,354 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import ast +import astor +from nni.tools.nnictl.common_utils import print_warning + +from .utils import ast_Num, ast_Str, lineno + +# pylint: disable=unidiomatic-typecheck + +para_cfg = None +prefix_name = None + + +def parse_annotation_mutable_layers(code, lineno): + """Parse the string of mutable layers in annotation. + Return a list of AST Expr nodes + code: annotation string (excluding '@') + """ + module = ast.parse(code) + assert type(module) is ast.Module, 'internal error #1' + assert len(module.body) == 1, 'Annotation mutable_layers contains more than one expression' + assert type(module.body[0]) is ast.Expr, 'Annotation is not expression' + call = module.body[0].value + nodes = [] + mutable_id = prefix_name + '/mutable_block_' + str(lineno) + mutable_layer_cnt = 0 + for arg in call.args: + fields = {'layer_choice': False, + 'fixed_inputs': False, + 'optional_inputs': False, + 'optional_input_size': False, + 'layer_output': False} + mutable_layer_id = 'mutable_layer_' + str(mutable_layer_cnt) + mutable_layer_cnt += 1 + func_call = None + for k, value in zip(arg.keys, arg.values): + if k.id == 'layer_choice': + assert not fields['layer_choice'], 'Duplicated field: layer_choice' + assert type(value) is ast.List, 'Value of layer_choice should be a list' + for call in value.elts: + assert type(call) is ast.Call, 'Element in layer_choice should be function call' + call_name = astor.to_source(call).strip() + if call_name == para_cfg[mutable_id][mutable_layer_id]['chosen_layer']: + func_call = call + assert not call.args, 'Number of args without keyword should be zero' + break + fields['layer_choice'] = True + elif k.id == 'fixed_inputs': + assert not fields['fixed_inputs'], 'Duplicated field: fixed_inputs' + assert type(value) is ast.List, 'Value of fixed_inputs should be a list' + fixed_inputs = value + fields['fixed_inputs'] = True + elif k.id == 'optional_inputs': + assert not fields['optional_inputs'], 'Duplicated field: optional_inputs' + assert type(value) is ast.List, 'Value of optional_inputs should be a list' + var_names = [astor.to_source(var).strip() for var in value.elts] + chosen_inputs = para_cfg[mutable_id][mutable_layer_id]['chosen_inputs'] + elts = [] + for i in chosen_inputs: + index = var_names.index(i) + elts.append(value.elts[index]) + optional_inputs = ast.List(elts=elts) + fields['optional_inputs'] = True + elif k.id == 'optional_input_size': + pass + elif k.id == 'layer_output': + assert not fields['layer_output'], 'Duplicated field: layer_output' + assert type(value) is ast.Name, 'Value of layer_output should be ast.Name type' + layer_output = value + fields['layer_output'] = True + else: + raise AssertionError('Unexpected field in mutable layer') + # make call for this mutable layer + assert fields['layer_choice'], 'layer_choice must exist' + assert fields['layer_output'], 'layer_output must exist' + + if not fields['fixed_inputs']: + fixed_inputs = ast.List(elts=[]) + if not fields['optional_inputs']: + optional_inputs = ast.List(elts=[]) + inputs = ast.List(elts=[fixed_inputs, optional_inputs]) + + func_call.args.append(inputs) + node = ast.Assign(targets=[layer_output], value=func_call) + nodes.append(node) + return nodes + + +def parse_annotation(code): + """Parse an annotation string. + Return an AST Expr node. + code: annotation string (excluding '@') + """ + module = ast.parse(code) + assert type(module) is ast.Module, 'internal error #1' + assert len(module.body) == 1, 'Annotation contains more than one expression' + assert type(module.body[0]) is ast.Expr, 'Annotation is not expression' + return module.body[0] + + +def parse_annotation_function(code, func_name): + """Parse an annotation function. + Return the value of `name` keyword argument and the AST Call node. + func_name: expected function name + """ + expr = parse_annotation(code) + call = expr.value + assert type(call) is ast.Call, 'Annotation is not a function call' + + assert type(call.func) is ast.Attribute, 'Unexpected annotation function' + assert type(call.func.value) is ast.Name, 'Invalid annotation function name' + assert call.func.value.id == 'nni', 'Annotation is not a NNI function' + assert call.func.attr == func_name, 'internal error #2' + + assert len(call.keywords) == 1, 'Annotation function contains more than one keyword argument' + assert call.keywords[0].arg == 'name', 'Annotation keyword argument is not "name"' + name = call.keywords[0].value + + return name, call + + +def parse_nni_variable(code): + """Parse `nni.variable` expression. + Return the name argument and AST node of annotated expression. + code: annotation string + """ + name, call = parse_annotation_function(code, 'variable') + + assert len(call.args) == 1, 'nni.variable contains more than one arguments' + arg = call.args[0] + assert type(arg) is ast.Call, 'Value of nni.variable is not a function call' + assert type(arg.func) is ast.Attribute, 'nni.variable value is not a NNI function' + assert type(arg.func.value) is ast.Name, 'nni.variable value is not a NNI function' + assert arg.func.value.id == 'nni', 'nni.variable value is not a NNI function' + + name_str = astor.to_source(name).strip() + keyword_arg = ast.keyword(arg='name', value=ast_Str(s=name_str)) + arg.keywords.append(keyword_arg) + if arg.func.attr == 'choice': + convert_args_to_dict(arg) + + return name, arg + + +def parse_nni_function(code): + """Parse `nni.function_choice` expression. + Return the AST node of annotated expression and a list of dumped function call expressions. + code: annotation string + """ + name, call = parse_annotation_function(code, 'function_choice') + funcs = [ast.dump(func, False) for func in call.args] + convert_args_to_dict(call, with_lambda=True) + + name_str = astor.to_source(name).strip() + call.keywords[0].value = ast_Str(s=name_str) + + return call, funcs + + +def convert_args_to_dict(call, with_lambda=False): + """Convert all args to a dict such that every key and value in the dict is the same as the value of the arg. + Return the AST Call node with only one arg that is the dictionary + """ + keys, values = list(), list() + for arg in call.args: + if type(arg) in [ast_Str, ast_Num]: + arg_value = arg + else: + # if arg is not a string or a number, we use its source code as the key + arg_value = astor.to_source(arg).strip('\n"') + arg_value = ast_Str(str(arg_value)) + arg = make_lambda(arg) if with_lambda else arg + keys.append(arg_value) + values.append(arg) + del call.args[:] + call.args.append(ast.Dict(keys=keys, values=values)) + + return call + + +def make_lambda(call): + """Wrap an AST Call node to lambda expression node. + call: ast.Call node + """ + empty_args = ast.arguments(args=[], vararg=None, kwarg=None, defaults=[]) + return ast.Lambda(args=empty_args, body=call) + + +def test_variable_equal(node1, node2): + """Test whether two variables are the same.""" + if type(node1) is not type(node2): + return False + if isinstance(node1, ast.AST): + for k, v in vars(node1).items(): + if k in ('lineno', 'col_offset', 'ctx', 'end_lineno', 'end_col_offset'): + continue + if not test_variable_equal(v, getattr(node2, k)): + return False + return True + if isinstance(node1, list): + if len(node1) != len(node2): + return False + return all(test_variable_equal(n1, n2) for n1, n2 in zip(node1, node2)) + + return node1 == node2 + + +def replace_variable_node(node, annotation): + """Replace a node annotated by `nni.variable`. + node: the AST node to replace + annotation: annotation string + """ + assert type(node) is ast.Assign, 'nni.variable is not annotating assignment expression' + assert len(node.targets) == 1, 'Annotated assignment has more than one left-hand value' + name, expr = parse_nni_variable(annotation) + assert test_variable_equal(node.targets[0], name), 'Annotated variable has wrong name' + node.value = expr + return node + + +def replace_function_node(node, annotation): + """Replace a node annotated by `nni.function_choice`. + node: the AST node to replace + annotation: annotation string + """ + target, funcs = parse_nni_function(annotation) + FuncReplacer(funcs, target).visit(node) + return node + + +class FuncReplacer(ast.NodeTransformer): + """To replace target function call expressions in a node annotated by `nni.function_choice`""" + + def __init__(self, funcs, target): + """Constructor. + funcs: list of dumped function call expressions to replace + target: use this AST node to replace matching expressions + """ + self.funcs = set(funcs) + self.target = target + + def visit_Call(self, node): # pylint: disable=invalid-name + if ast.dump(node, False) in self.funcs: + return self.target + return node + + +class Transformer(ast.NodeTransformer): + """Transform original code to annotated code""" + + def __init__(self): + self.stack = [] + self.last_line = 0 + self.annotated = False + + def visit(self, node): + if isinstance(node, (ast.expr, ast.stmt)): + self.last_line = lineno(node) + + # do nothing for root + if not self.stack: + return self._visit_children(node) + + annotation = self.stack[-1] + + # this is a standalone string, may be an annotation + if type(node) is ast.Expr and type(node.value) is ast_Str: + # must not annotate an annotation string + assert annotation is None, 'Annotating an annotation' + return self._visit_string(node) + + if annotation is not None: # this expression is annotated + self.stack[-1] = None # so next expression is not + if annotation.startswith('nni.variable'): + return replace_variable_node(node, annotation) + if annotation.startswith('nni.function_choice'): + return replace_function_node(node, annotation) + + return self._visit_children(node) + + def _visit_string(self, node): + string = node.value.s + if string.startswith('@nni.'): + self.annotated = True + else: + return node # not an annotation, ignore it + + if string.startswith('@nni.get_next_parameter'): + deprecated_message = "'@nni.get_next_parameter' is deprecated in annotation due to inconvenience. " \ + "Please remove this line in the trial code." + print_warning(deprecated_message) + return ast.Expr(value=ast.Call(func=ast.Name(id='print', ctx=ast.Load()), + args=[ast_Str(s='Get next parameter here...')], keywords=[])) + + if string.startswith('@nni.training_update'): + return ast.Expr(value=ast.Call(func=ast.Name(id='print', ctx=ast.Load()), + args=[ast_Str(s='Training update here...')], keywords=[])) + + if string.startswith('@nni.report_intermediate_result'): + module = ast.parse(string[1:]) + arg = module.body[0].value.args[0] + return ast.Expr(value=ast.Call(func=ast.Name(id='print', ctx=ast.Load()), + args=[ast_Str(s='nni.report_intermediate_result: '), arg], keywords=[])) + + if string.startswith('@nni.report_final_result'): + module = ast.parse(string[1:]) + arg = module.body[0].value.args[0] + return ast.Expr(value=ast.Call(func=ast.Name(id='print', ctx=ast.Load()), + args=[ast_Str(s='nni.report_final_result: '), arg], keywords=[])) + + if string.startswith('@nni.mutable_layers'): + return parse_annotation_mutable_layers(string[1:], lineno(node)) + + if string.startswith('@nni.variable') \ + or string.startswith('@nni.function_choice'): + self.stack[-1] = string[1:] # mark that the next expression is annotated + return None + + raise AssertionError('Unexpected annotation function') + + def _visit_children(self, node): + self.stack.append(None) + self.generic_visit(node) + annotation = self.stack.pop() + assert annotation is None, 'Annotation has no target' + return node + + +def parse(code, para, module): + """Annotate user code. + Return annotated code (str) if annotation detected; return None if not. + code: original user code (str) + """ + global para_cfg + global prefix_name + para_cfg = para + prefix_name = module + try: + ast_tree = ast.parse(code) + except Exception: + raise RuntimeError('Bad Python code') + + transformer = Transformer() + try: + transformer.visit(ast_tree) + except AssertionError as exc: + raise RuntimeError('%d: %s' % (ast_tree.last_line, exc.args[0])) + + if not transformer.annotated: + return None + + return astor.to_source(ast_tree) diff --git a/new_impl/cv/third_party/nni/tools/annotation/utils.py b/new_impl/cv/third_party/nni/tools/annotation/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..176621c2cf3d9211d626b182277ef61f170880f9 --- /dev/null +++ b/new_impl/cv/third_party/nni/tools/annotation/utils.py @@ -0,0 +1,22 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import ast +from sys import version_info + + +if version_info >= (3, 8): + ast_Num = ast_Str = ast_Bytes = ast_NameConstant = ast_Ellipsis = ast.Constant + + def lineno(ast_node): + return ast_node.end_lineno + +else: + ast_Num = ast.Num + ast_Str = ast.Str + ast_Bytes = ast.Bytes + ast_NameConstant = ast.NameConstant + ast_Ellipsis = ast.Ellipsis + + def lineno(ast_node): + return ast_node.lineno diff --git a/new_impl/cv/third_party/nni/tools/gpu_tool/__init__.py b/new_impl/cv/third_party/nni/tools/gpu_tool/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/new_impl/cv/third_party/nni/tools/gpu_tool/gpu_metrics_collector.py b/new_impl/cv/third_party/nni/tools/gpu_tool/gpu_metrics_collector.py new file mode 100644 index 0000000000000000000000000000000000000000..8e50cf9ec84533c1b8ded1799918934677eff058 --- /dev/null +++ b/new_impl/cv/third_party/nni/tools/gpu_tool/gpu_metrics_collector.py @@ -0,0 +1,82 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import json +import os +import subprocess +import sys +import time +import traceback + +from xml.dom import minidom + + +def main(argv): + metrics_output_dir = os.environ['METRIC_OUTPUT_DIR'] + + cmd = 'nvidia-smi -q -x'.split() + while(True): + try: + smi_output = subprocess.check_output(cmd) + except Exception: + traceback.print_exc() + gen_empty_gpu_metric(metrics_output_dir) + break + parse_nvidia_smi_result(smi_output, metrics_output_dir) + # TODO: change to sleep time configurable via arguments + time.sleep(5) + + +def parse_nvidia_smi_result(smi, outputDir): + try: + old_umask = os.umask(0) + xmldoc = minidom.parseString(smi) + gpuList = xmldoc.getElementsByTagName('gpu') + with open(os.path.join(outputDir, "gpu_metrics"), 'a') as outputFile: + outPut = {} + outPut["Timestamp"] = time.asctime(time.localtime()) + outPut["gpuCount"] = len(gpuList) + outPut["gpuInfos"] = [] + for gpuIndex, gpu in enumerate(gpuList): + gpuInfo = {} + gpuInfo['index'] = gpuIndex + gpuInfo['gpuUtil'] = gpu.getElementsByTagName('utilization')[0]\ + .getElementsByTagName('gpu_util')[0]\ + .childNodes[0].data.replace("%", "").strip() + gpuInfo['gpuMemUtil'] = gpu.getElementsByTagName('utilization')[0]\ + .getElementsByTagName('memory_util')[0]\ + .childNodes[0].data.replace("%", "").strip() + processes = gpu.getElementsByTagName('processes') + runningProNumber = len(processes[0].getElementsByTagName('process_info')) + gpuInfo['activeProcessNum'] = runningProNumber + + outPut["gpuInfos"].append(gpuInfo) + print(outPut) + outputFile.write("{}\n".format(json.dumps(outPut, sort_keys=True))) + outputFile.flush() + except Exception as error: + # e_info = sys.exc_info() + print('gpu_metrics_collector error: %s' % error) + finally: + os.umask(old_umask) + + +def gen_empty_gpu_metric(outputDir): + try: + old_umask = os.umask(0) + with open(os.path.join(outputDir, "gpu_metrics"), 'a') as outputFile: + outPut = {} + outPut["Timestamp"] = time.asctime(time.localtime()) + outPut["gpuCount"] = 0 + outPut["gpuInfos"] = [] + print(outPut) + outputFile.write("{}\n".format(json.dumps(outPut, sort_keys=True))) + outputFile.flush() + except Exception: + traceback.print_exc() + finally: + os.umask(old_umask) + + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/new_impl/cv/third_party/nni/tools/nnictl/__init__.py b/new_impl/cv/third_party/nni/tools/nnictl/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/new_impl/cv/third_party/nni/tools/nnictl/command_utils.py b/new_impl/cv/third_party/nni/tools/nnictl/command_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..2bbcc883d1cef450fef9b723dc8ca71d34daa095 --- /dev/null +++ b/new_impl/cv/third_party/nni/tools/nnictl/command_utils.py @@ -0,0 +1,82 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from subprocess import call, check_output +import sys +import os +import signal +import psutil +from .common_utils import print_error + + +def check_output_command(file_path, head=None, tail=None): + """call check_output command to read content from a file""" + if os.path.exists(file_path): + if sys.platform == 'win32': + cmds = ['powershell.exe', 'type', file_path] + if head: + cmds += ['|', 'select', '-first', str(head)] + elif tail: + cmds += ['|', 'select', '-last', str(tail)] + return check_output(cmds, shell=True).decode('utf-8') + else: + cmds = ['cat', file_path] + if head: + cmds = ['head', '-' + str(head), file_path] + elif tail: + cmds = ['tail', '-' + str(tail), file_path] + return check_output(cmds, shell=False).decode('utf-8') + else: + print_error('{0} does not exist!'.format(file_path)) + exit(1) + + +def kill_command(pid): + """kill command""" + if sys.platform == 'win32': + process = psutil.Process(pid=pid) + process.send_signal(signal.CTRL_BREAK_EVENT) + else: + cmds = ['kill', str(pid)] + call(cmds) + + +def install_package_command(package_name): + """ + Install python package from pip. + + Parameters + ---------- + package_name: str + The name of package to be installed. + """ + call(_get_pip_install() + [package_name], shell=False) + + +def install_requirements_command(requirements_path): + """ + Install packages from `requirements.txt` in `requirements_path`. + + Parameters + ---------- + requirements_path: str + Path to the directory that contains `requirements.txt`. + """ + return call(_get_pip_install() + ["-r", requirements_path], shell=False) + + +def _get_pip_install(): + python = "python" if sys.platform == "win32" else "python3" + ret = [python, "-m", "pip", "install"] + if "CONDA_DEFAULT_ENV" not in os.environ and "VIRTUAL_ENV" not in os.environ and \ + (sys.platform != "win32" and os.getuid() != 0): # on unix and not running in root + ret.append("--user") # not in virtualenv or conda + return ret + +def call_pip_install(source): + return call(_get_pip_install() + [source]) + +def call_pip_uninstall(module_name): + python = "python" if sys.platform == "win32" else "python3" + cmd = [python, "-m", "pip", "uninstall", module_name] + return call(cmd) diff --git a/new_impl/cv/third_party/nni/tools/nnictl/common_utils.py b/new_impl/cv/third_party/nni/tools/nnictl/common_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..2edbf667dff3234715281bbd1111b14380831282 --- /dev/null +++ b/new_impl/cv/third_party/nni/tools/nnictl/common_utils.py @@ -0,0 +1,97 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +import sys +import json +import tempfile +import socket +import string +import random +import ruamel.yaml as yaml +import psutil +from colorama import Fore + +from .constants import ERROR_INFO, NORMAL_INFO, WARNING_INFO + +def get_yml_content(file_path): + '''Load yaml file content''' + try: + with open(file_path, 'r') as file: + return yaml.load(file, Loader=yaml.Loader) + except yaml.scanner.ScannerError as err: + print_error('yaml file format error!') + print_error(err) + exit(1) + except Exception as exception: + print_error(exception) + exit(1) + +def get_json_content(file_path): + '''Load json file content''' + try: + with open(file_path, 'r') as file: + return json.load(file) + except TypeError as err: + print_error('json file format error!') + print_error(err) + return None + + +def print_error(*content): + '''Print error information to screen''' + print(Fore.RED + ERROR_INFO + ' '.join([str(c) for c in content]) + Fore.RESET) + +def print_green(*content): + '''Print information to screen in green''' + print(Fore.GREEN + ' '.join([str(c) for c in content]) + Fore.RESET) + +def print_normal(*content): + '''Print error information to screen''' + print(NORMAL_INFO, *content) + +def print_warning(*content): + '''Print warning information to screen''' + print(Fore.YELLOW + WARNING_INFO + ' '.join([str(c) for c in content]) + Fore.RESET) + +def detect_process(pid): + '''Detect if a process is alive''' + try: + process = psutil.Process(pid) + return process.is_running() + except: + return False + +def detect_port(port): + '''Detect if the port is used''' + socket_test = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + try: + socket_test.connect(('127.0.0.1', int(port))) + socket_test.close() + return True + except: + return False + +def get_user(): + if sys.platform == 'win32': + return os.environ['USERNAME'] + else: + return os.environ['USER'] + +def check_tensorboard_version(): + try: + import tensorboard + return tensorboard.__version__ + except: + print_error('import tensorboard error!') + exit(1) + +def generate_temp_dir(): + '''generate a temp folder''' + def generate_folder_name(): + return os.path.join(tempfile.gettempdir(), 'nni', ''.join(random.sample(string.ascii_letters + string.digits, 8))) + temp_dir = generate_folder_name() + while os.path.exists(temp_dir): + temp_dir = generate_folder_name() + os.makedirs(temp_dir) + return temp_dir diff --git a/new_impl/cv/third_party/nni/tools/nnictl/config_schema.py b/new_impl/cv/third_party/nni/tools/nnictl/config_schema.py new file mode 100644 index 0000000000000000000000000000000000000000..d3201635955ed656e12339ca54109aab9ab3656a --- /dev/null +++ b/new_impl/cv/third_party/nni/tools/nnictl/config_schema.py @@ -0,0 +1,539 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import json +import logging +import os +import netifaces +from schema import Schema, And, Optional, Regex, Or, SchemaError +from nni.tools.package_utils import create_validator_instance, get_all_builtin_names, get_builtin_algo_meta +from .constants import SCHEMA_TYPE_ERROR, SCHEMA_RANGE_ERROR, SCHEMA_PATH_ERROR +from .common_utils import get_yml_content, print_warning + + +def setType(key, valueType): + '''check key type''' + return And(valueType, error=SCHEMA_TYPE_ERROR % (key, valueType.__name__)) + + +def setChoice(key, *args): + '''check choice''' + return And(lambda n: n in args, error=SCHEMA_RANGE_ERROR % (key, str(args))) + + +def setNumberRange(key, keyType, start, end): + '''check number range''' + return And( + And(keyType, error=SCHEMA_TYPE_ERROR % (key, keyType.__name__)), + And(lambda n: start <= n <= end, error=SCHEMA_RANGE_ERROR % (key, '(%s,%s)' % (start, end))), + ) + + +def setPathCheck(key): + '''check if path exist''' + return And(os.path.exists, error=SCHEMA_PATH_ERROR % key) + + +class AlgoSchema: + """ + This class is the schema of 'tuner', 'assessor' and 'advisor' sections of experiment configuraion file. + For example: + AlgoSchema('tuner') creates the schema of tuner section. + """ + + def __init__(self, algo_type): + """ + Parameters: + ----------- + algo_type: str + One of ['tuner', 'assessor', 'advisor']. + 'tuner': This AlgoSchema class create the schema of tuner section. + 'assessor': This AlgoSchema class create the schema of assessor section. + 'advisor': This AlgoSchema class create the schema of advisor section. + """ + assert algo_type in ['tuner', 'assessor', 'advisor'] + self.algo_type = algo_type + self.algo_schema = { + Optional('codeDir'): setPathCheck('codeDir'), + Optional('classFileName'): setType('classFileName', str), + Optional('className'): setType('className', str), + Optional('classArgs'): dict, + Optional('includeIntermediateResults'): setType('includeIntermediateResults', bool), + Optional('gpuIndices'): Or(int, And(str, lambda x: len([int(i) for i in x.split(',')]) > 0), error='gpuIndex format error!'), + } + self.builtin_keys = { + 'tuner': 'builtinTunerName', + 'assessor': 'builtinAssessorName', + 'advisor': 'builtinAdvisorName' + } + self.builtin_name_schema = {} + for k, n in self.builtin_keys.items(): + self.builtin_name_schema[k] = {Optional(n): setChoice(n, *get_all_builtin_names(k+'s'))} + + self.customized_keys = set(['codeDir', 'classFileName', 'className']) + + def validate_class_args(self, class_args, algo_type, builtin_name): + if not builtin_name or not class_args: + return + meta = get_builtin_algo_meta(algo_type+'s', builtin_name) + if meta and 'accept_class_args' in meta and meta['accept_class_args'] == False: + raise SchemaError('classArgs is not allowed.') + + logging.getLogger('nni.protocol').setLevel(logging.ERROR) # we know IPC is not there, don't complain + validator = create_validator_instance(algo_type+'s', builtin_name) + if validator: + try: + validator.validate_class_args(**class_args) + except Exception as e: + raise SchemaError(str(e)) + + def missing_customized_keys(self, data): + return self.customized_keys - set(data.keys()) + + def validate_extras(self, data, algo_type): + builtin_key = self.builtin_keys[algo_type] + if (builtin_key in data) and (set(data.keys()) & self.customized_keys): + raise SchemaError('{} and {} cannot be specified at the same time.'.format( + builtin_key, set(data.keys()) & self.customized_keys + )) + + if self.missing_customized_keys(data) and builtin_key not in data: + raise SchemaError('Either customized {} ({}) or builtin {} ({}) must be set.'.format( + algo_type, self.customized_keys, algo_type, builtin_key)) + + if not self.missing_customized_keys(data): + class_file_name = os.path.join(data['codeDir'], data['classFileName']) + if not os.path.isfile(class_file_name): + raise SchemaError('classFileName {} not found.'.format(class_file_name)) + + builtin_name = data.get(builtin_key) + class_args = data.get('classArgs') + self.validate_class_args(class_args, algo_type, builtin_name) + + def validate(self, data): + self.algo_schema.update(self.builtin_name_schema[self.algo_type]) + Schema(self.algo_schema).validate(data) + self.validate_extras(data, self.algo_type) + + +common_schema = { + 'authorName': setType('authorName', str), + 'experimentName': setType('experimentName', str), + Optional('description'): setType('description', str), + 'trialConcurrency': setNumberRange('trialConcurrency', int, 1, 99999), + Optional('maxExecDuration'): And(Regex(r'^[1-9][0-9]*[s|m|h|d]$', error='ERROR: maxExecDuration format is [digit]{s,m,h,d}')), + Optional('maxTrialNum'): setNumberRange('maxTrialNum', int, 1, 99999), + 'trainingServicePlatform': setChoice( + 'trainingServicePlatform', 'remote', 'local', 'pai', 'kubeflow', 'frameworkcontroller', 'paiYarn', 'dlts', 'aml'), + Optional('searchSpacePath'): And(os.path.exists, error=SCHEMA_PATH_ERROR % 'searchSpacePath'), + Optional('multiPhase'): setType('multiPhase', bool), + Optional('multiThread'): setType('multiThread', bool), + Optional('nniManagerIp'): setType('nniManagerIp', str), + Optional('logDir'): And(os.path.isdir, error=SCHEMA_PATH_ERROR % 'logDir'), + Optional('debug'): setType('debug', bool), + Optional('versionCheck'): setType('versionCheck', bool), + Optional('logLevel'): setChoice('logLevel', 'trace', 'debug', 'info', 'warning', 'error', 'fatal'), + Optional('logCollection'): setChoice('logCollection', 'http', 'none'), + 'useAnnotation': setType('useAnnotation', bool), + Optional('tuner'): AlgoSchema('tuner'), + Optional('advisor'): AlgoSchema('advisor'), + Optional('assessor'): AlgoSchema('assessor'), + Optional('localConfig'): { + Optional('gpuIndices'): Or(int, And(str, lambda x: len([int(i) for i in x.split(',')]) > 0), error='gpuIndex format error!'), + Optional('maxTrialNumPerGpu'): setType('maxTrialNumPerGpu', int), + Optional('useActiveGpu'): setType('useActiveGpu', bool) + } +} + +common_trial_schema = { + 'trial': { + 'command': setType('command', str), + 'codeDir': setPathCheck('codeDir'), + Optional('gpuNum'): setNumberRange('gpuNum', int, 0, 99999), + Optional('nasMode'): setChoice('nasMode', 'classic_mode', 'enas_mode', 'oneshot_mode', 'darts_mode') + } +} + +pai_yarn_trial_schema = { + 'trial': { + 'command': setType('command', str), + 'codeDir': setPathCheck('codeDir'), + 'gpuNum': setNumberRange('gpuNum', int, 0, 99999), + 'cpuNum': setNumberRange('cpuNum', int, 0, 99999), + 'memoryMB': setType('memoryMB', int), + 'image': setType('image', str), + Optional('authFile'): And(os.path.exists, error=SCHEMA_PATH_ERROR % 'authFile'), + Optional('shmMB'): setType('shmMB', int), + Optional('dataDir'): And(Regex(r'hdfs://(([0-9]{1,3}.){3}[0-9]{1,3})(:[0-9]{2,5})?(/.*)?'), + error='ERROR: dataDir format error, dataDir format is hdfs://xxx.xxx.xxx.xxx:xxx'), + Optional('outputDir'): And(Regex(r'hdfs://(([0-9]{1,3}.){3}[0-9]{1,3})(:[0-9]{2,5})?(/.*)?'), + error='ERROR: outputDir format error, outputDir format is hdfs://xxx.xxx.xxx.xxx:xxx'), + Optional('virtualCluster'): setType('virtualCluster', str), + Optional('nasMode'): setChoice('nasMode', 'classic_mode', 'enas_mode', 'oneshot_mode', 'darts_mode'), + Optional('portList'): [{ + "label": setType('label', str), + "beginAt": setType('beginAt', int), + "portNumber": setType('portNumber', int) + }] + } +} + +pai_yarn_config_schema = { + 'paiYarnConfig': Or({ + 'userName': setType('userName', str), + 'passWord': setType('passWord', str), + 'host': setType('host', str) + }, { + 'userName': setType('userName', str), + 'token': setType('token', str), + 'host': setType('host', str) + }) +} + + +pai_trial_schema = { + 'trial': { + 'codeDir': setPathCheck('codeDir'), + 'nniManagerNFSMountPath': setPathCheck('nniManagerNFSMountPath'), + 'containerNFSMountPath': setType('containerNFSMountPath', str), + Optional('command'): setType('command', str), + Optional('gpuNum'): setNumberRange('gpuNum', int, 0, 99999), + Optional('cpuNum'): setNumberRange('cpuNum', int, 0, 99999), + Optional('memoryMB'): setType('memoryMB', int), + Optional('image'): setType('image', str), + Optional('virtualCluster'): setType('virtualCluster', str), + Optional('paiStorageConfigName'): setType('paiStorageConfigName', str), + Optional('paiConfigPath'): And(os.path.exists, error=SCHEMA_PATH_ERROR % 'paiConfigPath') + } +} + +pai_config_schema = { + 'paiConfig': { + 'userName': setType('userName', str), + Or('passWord', 'token', only_one=True): str, + 'host': setType('host', str), + Optional('reuse'): setType('reuse', bool), + Optional('gpuNum'): setNumberRange('gpuNum', int, 0, 99999), + Optional('cpuNum'): setNumberRange('cpuNum', int, 0, 99999), + Optional('memoryMB'): setType('memoryMB', int), + Optional('maxTrialNumPerGpu'): setType('maxTrialNumPerGpu', int), + Optional('useActiveGpu'): setType('useActiveGpu', bool), + } +} + +dlts_trial_schema = { + 'trial': { + 'command': setType('command', str), + 'codeDir': setPathCheck('codeDir'), + 'gpuNum': setNumberRange('gpuNum', int, 0, 99999), + 'image': setType('image', str), + } +} + +dlts_config_schema = { + 'dltsConfig': { + 'dashboard': setType('dashboard', str), + + Optional('cluster'): setType('cluster', str), + Optional('team'): setType('team', str), + + Optional('email'): setType('email', str), + Optional('password'): setType('password', str), + } +} + +aml_trial_schema = { + 'trial': { + 'codeDir': setPathCheck('codeDir'), + 'command': setType('command', str), + 'image': setType('image', str), + Optional('gpuNum'): setNumberRange('gpuNum', int, 0, 99999), + } +} + +aml_config_schema = { + 'amlConfig': { + 'subscriptionId': setType('subscriptionId', str), + 'resourceGroup': setType('resourceGroup', str), + 'workspaceName': setType('workspaceName', str), + 'computeTarget': setType('computeTarget', str), + Optional('maxTrialNumPerGpu'): setType('maxTrialNumPerGpu', int), + Optional('useActiveGpu'): setType('useActiveGpu', bool), + } +} + +kubeflow_trial_schema = { + 'trial': { + 'codeDir': setPathCheck('codeDir'), + Optional('nasMode'): setChoice('nasMode', 'classic_mode', 'enas_mode', 'oneshot_mode', 'darts_mode'), + Optional('ps'): { + 'replicas': setType('replicas', int), + 'command': setType('command', str), + 'gpuNum': setNumberRange('gpuNum', int, 0, 99999), + 'cpuNum': setNumberRange('cpuNum', int, 0, 99999), + 'memoryMB': setType('memoryMB', int), + 'image': setType('image', str), + Optional('privateRegistryAuthPath'): And(os.path.exists, error=SCHEMA_PATH_ERROR % 'privateRegistryAuthPath') + }, + Optional('master'): { + 'replicas': setType('replicas', int), + 'command': setType('command', str), + 'gpuNum': setNumberRange('gpuNum', int, 0, 99999), + 'cpuNum': setNumberRange('cpuNum', int, 0, 99999), + 'memoryMB': setType('memoryMB', int), + 'image': setType('image', str), + Optional('privateRegistryAuthPath'): And(os.path.exists, error=SCHEMA_PATH_ERROR % 'privateRegistryAuthPath') + }, + Optional('worker'): { + 'replicas': setType('replicas', int), + 'command': setType('command', str), + 'gpuNum': setNumberRange('gpuNum', int, 0, 99999), + 'cpuNum': setNumberRange('cpuNum', int, 0, 99999), + 'memoryMB': setType('memoryMB', int), + 'image': setType('image', str), + Optional('privateRegistryAuthPath'): And(os.path.exists, error=SCHEMA_PATH_ERROR % 'privateRegistryAuthPath') + } + } +} + +kubeflow_config_schema = { + 'kubeflowConfig': Or({ + 'operator': setChoice('operator', 'tf-operator', 'pytorch-operator'), + 'apiVersion': setType('apiVersion', str), + Optional('storage'): setChoice('storage', 'nfs', 'azureStorage'), + 'nfs': { + 'server': setType('server', str), + 'path': setType('path', str) + } + }, { + 'operator': setChoice('operator', 'tf-operator', 'pytorch-operator'), + 'apiVersion': setType('apiVersion', str), + Optional('storage'): setChoice('storage', 'nfs', 'azureStorage'), + 'keyVault': { + 'vaultName': And(Regex('([0-9]|[a-z]|[A-Z]|-){1,127}'), + error='ERROR: vaultName format error, vaultName support using (0-9|a-z|A-Z|-)'), + 'name': And(Regex('([0-9]|[a-z]|[A-Z]|-){1,127}'), + error='ERROR: name format error, name support using (0-9|a-z|A-Z|-)') + }, + 'azureStorage': { + 'accountName': And(Regex('([0-9]|[a-z]|[A-Z]|-){3,31}'), + error='ERROR: accountName format error, accountName support using (0-9|a-z|A-Z|-)'), + 'azureShare': And(Regex('([0-9]|[a-z]|[A-Z]|-){3,63}'), + error='ERROR: azureShare format error, azureShare support using (0-9|a-z|A-Z|-)') + }, + Optional('uploadRetryCount'): setNumberRange('uploadRetryCount', int, 1, 99999) + }) +} + +frameworkcontroller_trial_schema = { + 'trial': { + 'codeDir': setPathCheck('codeDir'), + 'taskRoles': [{ + 'name': setType('name', str), + 'taskNum': setType('taskNum', int), + 'frameworkAttemptCompletionPolicy': { + 'minFailedTaskCount': setType('minFailedTaskCount', int), + 'minSucceededTaskCount': setType('minSucceededTaskCount', int), + }, + 'command': setType('command', str), + 'gpuNum': setNumberRange('gpuNum', int, 0, 99999), + 'cpuNum': setNumberRange('cpuNum', int, 0, 99999), + 'memoryMB': setType('memoryMB', int), + 'image': setType('image', str), + Optional('privateRegistryAuthPath'): And(os.path.exists, error=SCHEMA_PATH_ERROR % 'privateRegistryAuthPath') + }] + } +} + +frameworkcontroller_config_schema = { + 'frameworkcontrollerConfig': Or({ + Optional('storage'): setChoice('storage', 'nfs', 'azureStorage'), + Optional('serviceAccountName'): setType('serviceAccountName', str), + 'nfs': { + 'server': setType('server', str), + 'path': setType('path', str) + } + }, { + Optional('storage'): setChoice('storage', 'nfs', 'azureStorage'), + Optional('serviceAccountName'): setType('serviceAccountName', str), + 'keyVault': { + 'vaultName': And(Regex('([0-9]|[a-z]|[A-Z]|-){1,127}'), + error='ERROR: vaultName format error, vaultName support using (0-9|a-z|A-Z|-)'), + 'name': And(Regex('([0-9]|[a-z]|[A-Z]|-){1,127}'), + error='ERROR: name format error, name support using (0-9|a-z|A-Z|-)') + }, + 'azureStorage': { + 'accountName': And(Regex('([0-9]|[a-z]|[A-Z]|-){3,31}'), + error='ERROR: accountName format error, accountName support using (0-9|a-z|A-Z|-)'), + 'azureShare': And(Regex('([0-9]|[a-z]|[A-Z]|-){3,63}'), + error='ERROR: azureShare format error, azureShare support using (0-9|a-z|A-Z|-)') + }, + Optional('uploadRetryCount'): setNumberRange('uploadRetryCount', int, 1, 99999) + }) +} + +remote_config_schema = { + Optional('remoteConfig'): { + 'reuse': setType('reuse', bool) + } +} + +machine_list_schema = { + 'machineList': [Or( + { + 'ip': setType('ip', str), + Optional('port'): setNumberRange('port', int, 1, 65535), + 'username': setType('username', str), + 'sshKeyPath': setPathCheck('sshKeyPath'), + Optional('passphrase'): setType('passphrase', str), + Optional('gpuIndices'): Or(int, And(str, lambda x: len([int(i) for i in x.split(',')]) > 0), error='gpuIndex format error!'), + Optional('maxTrialNumPerGpu'): setType('maxTrialNumPerGpu', int), + Optional('useActiveGpu'): setType('useActiveGpu', bool), + Optional('preCommand'): setType('preCommand', str) + }, + { + 'ip': setType('ip', str), + Optional('port'): setNumberRange('port', int, 1, 65535), + 'username': setType('username', str), + 'passwd': setType('passwd', str), + Optional('gpuIndices'): Or(int, And(str, lambda x: len([int(i) for i in x.split(',')]) > 0), error='gpuIndex format error!'), + Optional('maxTrialNumPerGpu'): setType('maxTrialNumPerGpu', int), + Optional('useActiveGpu'): setType('useActiveGpu', bool), + Optional('preCommand'): setType('preCommand', str) + })] +} + +training_service_schema_dict = { + 'local': Schema({**common_schema, **common_trial_schema}), + 'remote': Schema({**common_schema, **common_trial_schema, **machine_list_schema, **remote_config_schema}), + 'pai': Schema({**common_schema, **pai_trial_schema, **pai_config_schema}), + 'paiYarn': Schema({**common_schema, **pai_yarn_trial_schema, **pai_yarn_config_schema}), + 'kubeflow': Schema({**common_schema, **kubeflow_trial_schema, **kubeflow_config_schema}), + 'frameworkcontroller': Schema({**common_schema, **frameworkcontroller_trial_schema, **frameworkcontroller_config_schema}), + 'aml': Schema({**common_schema, **aml_trial_schema, **aml_config_schema}), + 'dlts': Schema({**common_schema, **dlts_trial_schema, **dlts_config_schema}), +} + + +class NNIConfigSchema: + def validate(self, data): + train_service = data['trainingServicePlatform'] + Schema(common_schema['trainingServicePlatform']).validate(train_service) + train_service_schema = training_service_schema_dict[train_service] + train_service_schema.validate(data) + self.validate_extras(data) + + def validate_extras(self, experiment_config): + self.validate_tuner_adivosr_assessor(experiment_config) + self.validate_pai_trial_conifg(experiment_config) + self.validate_kubeflow_operators(experiment_config) + self.validate_eth0_device(experiment_config) + + def validate_tuner_adivosr_assessor(self, experiment_config): + if experiment_config.get('advisor'): + if experiment_config.get('assessor') or experiment_config.get('tuner'): + raise SchemaError('advisor could not be set with assessor or tuner simultaneously!') + self.validate_annotation_content(experiment_config, 'advisor', 'builtinAdvisorName') + else: + if not experiment_config.get('tuner'): + raise SchemaError('Please provide tuner spec!') + self.validate_annotation_content(experiment_config, 'tuner', 'builtinTunerName') + + def validate_search_space_content(self, experiment_config): + '''Validate searchspace content, + if the searchspace file is not json format or its values does not contain _type and _value which must be specified, + it will not be a valid searchspace file''' + try: + search_space_content = json.load(open(experiment_config.get('searchSpacePath'), 'r')) + for value in search_space_content.values(): + if not value.get('_type') or not value.get('_value'): + raise SchemaError('please use _type and _value to specify searchspace!') + except Exception as e: + raise SchemaError('searchspace file is not a valid json format! ' + str(e)) + + def validate_kubeflow_operators(self, experiment_config): + '''Validate whether the kubeflow operators are valid''' + if experiment_config.get('kubeflowConfig'): + if experiment_config.get('kubeflowConfig').get('operator') == 'tf-operator': + if experiment_config.get('trial').get('master') is not None: + raise SchemaError('kubeflow with tf-operator can not set master') + if experiment_config.get('trial').get('worker') is None: + raise SchemaError('kubeflow with tf-operator must set worker') + elif experiment_config.get('kubeflowConfig').get('operator') == 'pytorch-operator': + if experiment_config.get('trial').get('ps') is not None: + raise SchemaError('kubeflow with pytorch-operator can not set ps') + if experiment_config.get('trial').get('master') is None: + raise SchemaError('kubeflow with pytorch-operator must set master') + + if experiment_config.get('kubeflowConfig').get('storage') == 'nfs': + if experiment_config.get('kubeflowConfig').get('nfs') is None: + raise SchemaError('please set nfs configuration!') + elif experiment_config.get('kubeflowConfig').get('storage') == 'azureStorage': + if experiment_config.get('kubeflowConfig').get('azureStorage') is None: + raise SchemaError('please set azureStorage configuration!') + elif experiment_config.get('kubeflowConfig').get('storage') is None: + if experiment_config.get('kubeflowConfig').get('azureStorage'): + raise SchemaError('please set storage type!') + + def validate_annotation_content(self, experiment_config, spec_key, builtin_name): + ''' + Valid whether useAnnotation and searchSpacePath is coexist + spec_key: 'advisor' or 'tuner' + builtin_name: 'builtinAdvisorName' or 'builtinTunerName' + ''' + if experiment_config.get('useAnnotation'): + if experiment_config.get('searchSpacePath'): + raise SchemaError('If you set useAnnotation=true, please leave searchSpacePath empty') + else: + # validate searchSpaceFile + if experiment_config[spec_key].get(builtin_name) == 'NetworkMorphism': + return + if experiment_config[spec_key].get(builtin_name): + if experiment_config.get('searchSpacePath') is None: + raise SchemaError('Please set searchSpacePath!') + self.validate_search_space_content(experiment_config) + + def validate_pai_config_path(self, experiment_config): + '''validate paiConfigPath field''' + if experiment_config.get('trainingServicePlatform') == 'pai': + if experiment_config.get('trial', {}).get('paiConfigPath'): + # validate commands + pai_config = get_yml_content(experiment_config['trial']['paiConfigPath']) + taskRoles_dict = pai_config.get('taskRoles') + if not taskRoles_dict: + raise SchemaError('Please set taskRoles in paiConfigPath config file!') + else: + pai_trial_fields_required_list = ['image', 'paiStorageConfigName', 'command'] + for trial_field in pai_trial_fields_required_list: + if experiment_config['trial'].get(trial_field) is None: + raise SchemaError('Please set {0} in trial configuration,\ + or set additional pai configuration file path in paiConfigPath!'.format(trial_field)) + pai_resource_fields_required_list = ['gpuNum', 'cpuNum', 'memoryMB'] + for required_field in pai_resource_fields_required_list: + if experiment_config['trial'].get(required_field) is None and \ + experiment_config['paiConfig'].get(required_field) is None: + raise SchemaError('Please set {0} in trial or paiConfig configuration,\ + or set additional pai configuration file path in paiConfigPath!'.format(required_field)) + + def validate_pai_trial_conifg(self, experiment_config): + '''validate the trial config in pai platform''' + if experiment_config.get('trainingServicePlatform') in ['pai', 'paiYarn']: + if experiment_config.get('trial').get('shmMB') and \ + experiment_config['trial']['shmMB'] > experiment_config['trial']['memoryMB']: + raise SchemaError('shmMB should be no more than memoryMB!') + # backward compatibility + warning_information = '{0} is not supported in NNI anymore, please remove the field in config file!\ + please refer https://github.com/microsoft/nni/blob/master/docs/en_US/TrainingService/PaiMode.md#run-an-experiment\ + for the practices of how to get data and output model in trial code' + if experiment_config.get('trial').get('dataDir'): + print_warning(warning_information.format('dataDir')) + if experiment_config.get('trial').get('outputDir'): + print_warning(warning_information.format('outputDir')) + self.validate_pai_config_path(experiment_config) + + def validate_eth0_device(self, experiment_config): + '''validate whether the machine has eth0 device''' + if experiment_config.get('trainingServicePlatform') not in ['local'] \ + and not experiment_config.get('nniManagerIp') \ + and 'eth0' not in netifaces.interfaces(): + raise SchemaError('This machine does not contain eth0 network device, please set nniManagerIp in config file!') diff --git a/new_impl/cv/third_party/nni/tools/nnictl/config_utils.py b/new_impl/cv/third_party/nni/tools/nnictl/config_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..434bdebeca55fcfc80af6a233ebb57ba00ea9859 --- /dev/null +++ b/new_impl/cv/third_party/nni/tools/nnictl/config_utils.py @@ -0,0 +1,112 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +import json +import shutil +from .constants import NNICTL_HOME_DIR +from .command_utils import print_error + +class Config: + '''a util class to load and save config''' + def __init__(self, file_path, home_dir=NNICTL_HOME_DIR): + config_path = os.path.join(home_dir, str(file_path)) + os.makedirs(config_path, exist_ok=True) + self.config_file = os.path.join(config_path, '.config') + self.config = self.read_file() + + def get_all_config(self): + '''get all of config values''' + return json.dumps(self.config, indent=4, sort_keys=True, separators=(',', ':')) + + def set_config(self, key, value): + '''set {key:value} paris to self.config''' + self.config = self.read_file() + self.config[key] = value + self.write_file() + + def get_config(self, key): + '''get a value according to key''' + return self.config.get(key) + + def write_file(self): + '''save config to local file''' + if self.config: + try: + with open(self.config_file, 'w') as file: + json.dump(self.config, file) + except IOError as error: + print('Error:', error) + return + + def read_file(self): + '''load config from local file''' + if os.path.exists(self.config_file): + try: + with open(self.config_file, 'r') as file: + return json.load(file) + except ValueError: + return {} + return {} + +class Experiments: + '''Maintain experiment list''' + def __init__(self, home_dir=NNICTL_HOME_DIR): + os.makedirs(home_dir, exist_ok=True) + self.experiment_file = os.path.join(home_dir, '.experiment') + self.experiments = self.read_file() + + def add_experiment(self, expId, port, startTime, file_name, platform, experiment_name, endTime='N/A', status='INITIALIZED'): + '''set {key:value} paris to self.experiment''' + self.experiments[expId] = {} + self.experiments[expId]['port'] = port + self.experiments[expId]['startTime'] = startTime + self.experiments[expId]['endTime'] = endTime + self.experiments[expId]['status'] = status + self.experiments[expId]['fileName'] = file_name + self.experiments[expId]['platform'] = platform + self.experiments[expId]['experimentName'] = experiment_name + self.write_file() + + def update_experiment(self, expId, key, value): + '''Update experiment''' + if expId not in self.experiments: + return False + self.experiments[expId][key] = value + self.write_file() + return True + + def remove_experiment(self, expId): + '''remove an experiment by id''' + if expId in self.experiments: + fileName = self.experiments.pop(expId).get('fileName') + if fileName: + logPath = os.path.join(NNICTL_HOME_DIR, fileName) + try: + shutil.rmtree(logPath) + except FileNotFoundError: + print_error('{0} does not exist.'.format(logPath)) + self.write_file() + + def get_all_experiments(self): + '''return all of experiments''' + return self.experiments + + def write_file(self): + '''save config to local file''' + try: + with open(self.experiment_file, 'w') as file: + json.dump(self.experiments, file) + except IOError as error: + print('Error:', error) + return '' + + def read_file(self): + '''load config from local file''' + if os.path.exists(self.experiment_file): + try: + with open(self.experiment_file, 'r') as file: + return json.load(file) + except ValueError: + return {} + return {} diff --git a/new_impl/cv/third_party/nni/tools/nnictl/constants.py b/new_impl/cv/third_party/nni/tools/nnictl/constants.py new file mode 100644 index 0000000000000000000000000000000000000000..0654473ed4548caf3cd9d80d49ce4624c10d6b66 --- /dev/null +++ b/new_impl/cv/third_party/nni/tools/nnictl/constants.py @@ -0,0 +1,102 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +from colorama import Fore + +NNICTL_HOME_DIR = os.path.join(os.path.expanduser('~'), '.local', 'nnictl') + +NNI_HOME_DIR = os.path.join(os.path.expanduser('~'), 'nni-experiments') + +ERROR_INFO = 'ERROR: ' +NORMAL_INFO = 'INFO: ' +WARNING_INFO = 'WARNING: ' + +DEFAULT_REST_PORT = 8080 +REST_TIME_OUT = 20 + +EXPERIMENT_SUCCESS_INFO = Fore.GREEN + 'Successfully started experiment!\n' + Fore.RESET + \ + '------------------------------------------------------------------------------------\n' \ + 'The experiment id is %s\n'\ + 'The Web UI urls are: %s\n' \ + '------------------------------------------------------------------------------------\n\n' \ + 'You can use these commands to get more information about the experiment\n' \ + '------------------------------------------------------------------------------------\n' \ + ' commands description\n' \ + '1. nnictl experiment show show the information of experiments\n' \ + '2. nnictl trial ls list all of trial jobs\n' \ + '3. nnictl top monitor the status of running experiments\n' \ + '4. nnictl log stderr show stderr log content\n' \ + '5. nnictl log stdout show stdout log content\n' \ + '6. nnictl stop stop an experiment\n' \ + '7. nnictl trial kill kill a trial job by id\n' \ + '8. nnictl --help get help information about nnictl\n' \ + '------------------------------------------------------------------------------------\n' \ + 'Command reference document https://nni.readthedocs.io/en/latest/Tutorial/Nnictl.html\n' \ + '------------------------------------------------------------------------------------\n' + +LOG_HEADER = '-----------------------------------------------------------------------\n' \ + ' Experiment start time %s\n' \ + '-----------------------------------------------------------------------\n' + +EXPERIMENT_START_FAILED_INFO = 'There is an experiment running in the port %d, please stop it first or set another port!\n' \ + 'You could use \'nnictl stop --port [PORT]\' command to stop an experiment!\nOr you could ' \ + 'use \'nnictl create --config [CONFIG_PATH] --port [PORT]\' to set port!\n' + +EXPERIMENT_INFORMATION_FORMAT = '----------------------------------------------------------------------------------------\n' \ + ' Experiment information\n' \ + '%s\n' \ + '----------------------------------------------------------------------------------------\n' + +EXPERIMENT_DETAIL_FORMAT = 'Id: %s Name: %s Status: %s Port: %s Platform: %s StartTime: %s EndTime: %s\n' + +EXPERIMENT_MONITOR_INFO = 'Id: %s Status: %s Port: %s Platform: %s \n' \ + 'StartTime: %s Duration: %s' + +TRIAL_MONITOR_HEAD = '-------------------------------------------------------------------------------------\n' + \ + '%-15s %-25s %-25s %-15s \n' % ('trialId', 'startTime', 'endTime', 'status') + \ + '-------------------------------------------------------------------------------------' + +TRIAL_MONITOR_CONTENT = '%-15s %-25s %-25s %-15s' + +TRIAL_MONITOR_TAIL = '-------------------------------------------------------------------------------------\n\n\n' + +INSTALLABLE_PACKAGE_META = { + 'SMAC': { + 'type': 'tuner', + 'class_name': 'nni.smac_tuner.smac_tuner.SMACTuner', + 'code_sub_dir': 'smac_tuner', + 'class_args_validator': 'nni.smac_tuner.smac_tuner.SMACClassArgsValidator' + }, + 'BOHB': { + 'type': 'advisor', + 'class_name': 'nni.bohb_advisor.bohb_advisor.BOHB', + 'code_sub_dir': 'bohb_advisor', + 'class_args_validator': 'nni.bohb_advisor.bohb_advisor.BOHBClassArgsValidator' + }, + 'PPOTuner': { + 'type': 'tuner', + 'class_name': 'nni.ppo_tuner.ppo_tuner.PPOTuner', + 'code_sub_dir': 'ppo_tuner', + 'class_args_validator': 'nni.ppo_tuner.ppo_tuner.PPOClassArgsValidator' + } +} + +TUNERS_SUPPORTING_IMPORT_DATA = { + 'TPE', + 'Anneal', + 'GridSearch', + 'MetisTuner', + 'BOHB', + 'SMAC', + 'BatchTuner' +} + +TUNERS_NO_NEED_TO_IMPORT_DATA = { + 'Random', + 'Hyperband' +} + +SCHEMA_TYPE_ERROR = '%s should be %s type!' +SCHEMA_RANGE_ERROR = '%s should be in range of %s!' +SCHEMA_PATH_ERROR = '%s path not exist!' diff --git a/new_impl/cv/third_party/nni/tools/nnictl/launcher.py b/new_impl/cv/third_party/nni/tools/nnictl/launcher.py new file mode 100644 index 0000000000000000000000000000000000000000..fa0aa3baab43ae1d3cf1f66643c2e28a3475e264 --- /dev/null +++ b/new_impl/cv/third_party/nni/tools/nnictl/launcher.py @@ -0,0 +1,605 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import json +import os +import sys +import string +import random +import time +import tempfile +from subprocess import Popen, check_call, CalledProcessError, PIPE, STDOUT +from nni.tools.annotation import expand_annotations, generate_search_space +from nni.tools.package_utils import get_builtin_module_class_name +import nni_node +from .launcher_utils import validate_all_content +from .rest_utils import rest_put, rest_post, check_rest_server, check_response +from .url_utils import cluster_metadata_url, experiment_url, get_local_urls +from .config_utils import Config, Experiments +from .common_utils import get_yml_content, get_json_content, print_error, print_normal, \ + detect_port, get_user + +from .constants import NNICTL_HOME_DIR, ERROR_INFO, REST_TIME_OUT, EXPERIMENT_SUCCESS_INFO, LOG_HEADER, INSTALLABLE_PACKAGE_META +from .command_utils import check_output_command, kill_command +from .nnictl_utils import update_experiment + +def get_log_path(config_file_name): + '''generate stdout and stderr log path''' + stdout_full_path = os.path.join(NNICTL_HOME_DIR, config_file_name, 'stdout') + stderr_full_path = os.path.join(NNICTL_HOME_DIR, config_file_name, 'stderr') + return stdout_full_path, stderr_full_path + +def print_log_content(config_file_name): + '''print log information''' + stdout_full_path, stderr_full_path = get_log_path(config_file_name) + print_normal(' Stdout:') + print(check_output_command(stdout_full_path)) + print('\n\n') + print_normal(' Stderr:') + print(check_output_command(stderr_full_path)) + +def start_rest_server(port, platform, mode, config_file_name, foreground=False, experiment_id=None, log_dir=None, log_level=None): + '''Run nni manager process''' + if detect_port(port): + print_error('Port %s is used by another process, please reset the port!\n' \ + 'You could use \'nnictl create --help\' to get help information' % port) + exit(1) + + if (platform != 'local') and detect_port(int(port) + 1): + print_error('PAI mode need an additional adjacent port %d, and the port %d is used by another process!\n' \ + 'You could set another port to start experiment!\n' \ + 'You could use \'nnictl create --help\' to get help information' % ((int(port) + 1), (int(port) + 1))) + exit(1) + + print_normal('Starting restful server...') + + entry_dir = nni_node.__path__[0] + if (not entry_dir) or (not os.path.exists(entry_dir)): + print_error('Fail to find nni under python library') + exit(1) + entry_file = os.path.join(entry_dir, 'main.js') + + if sys.platform == 'win32': + node_command = os.path.join(entry_dir, 'node.exe') + else: + node_command = 'node' + cmds = [node_command, '--max-old-space-size=4096', entry_file, '--port', str(port), '--mode', platform] + if mode == 'view': + cmds += ['--start_mode', 'resume'] + cmds += ['--readonly', 'true'] + else: + cmds += ['--start_mode', mode] + if log_dir is not None: + cmds += ['--log_dir', log_dir] + if log_level is not None: + cmds += ['--log_level', log_level] + if mode in ['resume', 'view']: + cmds += ['--experiment_id', experiment_id] + if foreground: + cmds += ['--foreground', 'true'] + stdout_full_path, stderr_full_path = get_log_path(config_file_name) + with open(stdout_full_path, 'a+') as stdout_file, open(stderr_full_path, 'a+') as stderr_file: + time_now = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())) + #add time information in the header of log files + log_header = LOG_HEADER % str(time_now) + stdout_file.write(log_header) + stderr_file.write(log_header) + if sys.platform == 'win32': + from subprocess import CREATE_NEW_PROCESS_GROUP + if foreground: + process = Popen(cmds, cwd=entry_dir, stdout=PIPE, stderr=STDOUT, creationflags=CREATE_NEW_PROCESS_GROUP) + else: + process = Popen(cmds, cwd=entry_dir, stdout=stdout_file, stderr=stderr_file, creationflags=CREATE_NEW_PROCESS_GROUP) + else: + if foreground: + process = Popen(cmds, cwd=entry_dir, stdout=PIPE, stderr=PIPE) + else: + process = Popen(cmds, cwd=entry_dir, stdout=stdout_file, stderr=stderr_file) + return process, str(time_now) + +def set_trial_config(experiment_config, port, config_file_name): + '''set trial configuration''' + request_data = dict() + request_data['trial_config'] = experiment_config['trial'] + response = rest_put(cluster_metadata_url(port), json.dumps(request_data), REST_TIME_OUT) + if check_response(response): + return True + else: + print('Error message is {}'.format(response.text)) + _, stderr_full_path = get_log_path(config_file_name) + if response: + with open(stderr_full_path, 'a+') as fout: + fout.write(json.dumps(json.loads(response.text), indent=4, sort_keys=True, separators=(',', ':'))) + return False + +def set_local_config(experiment_config, port, config_file_name): + '''set local configuration''' + request_data = dict() + if experiment_config.get('localConfig'): + request_data['local_config'] = experiment_config['localConfig'] + if request_data['local_config']: + if request_data['local_config'].get('gpuIndices') and isinstance(request_data['local_config'].get('gpuIndices'), int): + request_data['local_config']['gpuIndices'] = str(request_data['local_config'].get('gpuIndices')) + if request_data['local_config'].get('maxTrialNumOnEachGpu'): + request_data['local_config']['maxTrialNumOnEachGpu'] = request_data['local_config'].get('maxTrialNumOnEachGpu') + if request_data['local_config'].get('useActiveGpu'): + request_data['local_config']['useActiveGpu'] = request_data['local_config'].get('useActiveGpu') + response = rest_put(cluster_metadata_url(port), json.dumps(request_data), REST_TIME_OUT) + err_message = '' + if not response or not check_response(response): + if response is not None: + err_message = response.text + _, stderr_full_path = get_log_path(config_file_name) + with open(stderr_full_path, 'a+') as fout: + fout.write(json.dumps(json.loads(err_message), indent=4, sort_keys=True, separators=(',', ':'))) + return False, err_message + + return set_trial_config(experiment_config, port, config_file_name), None + +def set_remote_config(experiment_config, port, config_file_name): + '''Call setClusterMetadata to pass trial''' + #set machine_list + request_data = dict() + if experiment_config.get('remoteConfig'): + request_data['remote_config'] = experiment_config['remoteConfig'] + else: + request_data['remote_config'] = {'reuse': False} + request_data['machine_list'] = experiment_config['machineList'] + if request_data['machine_list']: + for i in range(len(request_data['machine_list'])): + if isinstance(request_data['machine_list'][i].get('gpuIndices'), int): + request_data['machine_list'][i]['gpuIndices'] = str(request_data['machine_list'][i].get('gpuIndices')) + # It needs to connect all remote machines, the time out of connection is 30 seconds. + # So timeout of this place should be longer. + response = rest_put(cluster_metadata_url(port), json.dumps(request_data), 60, True) + err_message = '' + if not response or not check_response(response): + if response is not None: + err_message = response.text + _, stderr_full_path = get_log_path(config_file_name) + with open(stderr_full_path, 'a+') as fout: + fout.write(json.dumps(json.loads(err_message), indent=4, sort_keys=True, separators=(',', ':'))) + return False, err_message + result, message = setNNIManagerIp(experiment_config, port, config_file_name) + if not result: + return result, message + #set trial_config + return set_trial_config(experiment_config, port, config_file_name), err_message + +def setNNIManagerIp(experiment_config, port, config_file_name): + '''set nniManagerIp''' + if experiment_config.get('nniManagerIp') is None: + return True, None + ip_config_dict = dict() + ip_config_dict['nni_manager_ip'] = {'nniManagerIp': experiment_config['nniManagerIp']} + response = rest_put(cluster_metadata_url(port), json.dumps(ip_config_dict), REST_TIME_OUT) + err_message = None + if not response or not response.status_code == 200: + if response is not None: + err_message = response.text + _, stderr_full_path = get_log_path(config_file_name) + with open(stderr_full_path, 'a+') as fout: + fout.write(json.dumps(json.loads(err_message), indent=4, sort_keys=True, separators=(',', ':'))) + return False, err_message + return True, None + +def set_pai_config(experiment_config, port, config_file_name): + '''set pai configuration''' + pai_config_data = dict() + pai_config_data['pai_config'] = experiment_config['paiConfig'] + response = rest_put(cluster_metadata_url(port), json.dumps(pai_config_data), REST_TIME_OUT) + err_message = None + if not response or not response.status_code == 200: + if response is not None: + err_message = response.text + _, stderr_full_path = get_log_path(config_file_name) + with open(stderr_full_path, 'a+') as fout: + fout.write(json.dumps(json.loads(err_message), indent=4, sort_keys=True, separators=(',', ':'))) + return False, err_message + result, message = setNNIManagerIp(experiment_config, port, config_file_name) + if not result: + return result, message + #set trial_config + return set_trial_config(experiment_config, port, config_file_name), err_message + +def set_pai_yarn_config(experiment_config, port, config_file_name): + '''set paiYarn configuration''' + pai_yarn_config_data = dict() + pai_yarn_config_data['pai_yarn_config'] = experiment_config['paiYarnConfig'] + response = rest_put(cluster_metadata_url(port), json.dumps(pai_yarn_config_data), REST_TIME_OUT) + err_message = None + if not response or not response.status_code == 200: + if response is not None: + err_message = response.text + _, stderr_full_path = get_log_path(config_file_name) + with open(stderr_full_path, 'a+') as fout: + fout.write(json.dumps(json.loads(err_message), indent=4, sort_keys=True, separators=(',', ':'))) + return False, err_message + result, message = setNNIManagerIp(experiment_config, port, config_file_name) + if not result: + return result, message + #set trial_config + return set_trial_config(experiment_config, port, config_file_name), err_message + +def set_kubeflow_config(experiment_config, port, config_file_name): + '''set kubeflow configuration''' + kubeflow_config_data = dict() + kubeflow_config_data['kubeflow_config'] = experiment_config['kubeflowConfig'] + response = rest_put(cluster_metadata_url(port), json.dumps(kubeflow_config_data), REST_TIME_OUT) + err_message = None + if not response or not response.status_code == 200: + if response is not None: + err_message = response.text + _, stderr_full_path = get_log_path(config_file_name) + with open(stderr_full_path, 'a+') as fout: + fout.write(json.dumps(json.loads(err_message), indent=4, sort_keys=True, separators=(',', ':'))) + return False, err_message + result, message = setNNIManagerIp(experiment_config, port, config_file_name) + if not result: + return result, message + #set trial_config + return set_trial_config(experiment_config, port, config_file_name), err_message + +def set_frameworkcontroller_config(experiment_config, port, config_file_name): + '''set kubeflow configuration''' + frameworkcontroller_config_data = dict() + frameworkcontroller_config_data['frameworkcontroller_config'] = experiment_config['frameworkcontrollerConfig'] + response = rest_put(cluster_metadata_url(port), json.dumps(frameworkcontroller_config_data), REST_TIME_OUT) + err_message = None + if not response or not response.status_code == 200: + if response is not None: + err_message = response.text + _, stderr_full_path = get_log_path(config_file_name) + with open(stderr_full_path, 'a+') as fout: + fout.write(json.dumps(json.loads(err_message), indent=4, sort_keys=True, separators=(',', ':'))) + return False, err_message + result, message = setNNIManagerIp(experiment_config, port, config_file_name) + if not result: + return result, message + #set trial_config + return set_trial_config(experiment_config, port, config_file_name), err_message + +def set_dlts_config(experiment_config, port, config_file_name): + '''set dlts configuration''' + dlts_config_data = dict() + dlts_config_data['dlts_config'] = experiment_config['dltsConfig'] + response = rest_put(cluster_metadata_url(port), json.dumps(dlts_config_data), REST_TIME_OUT) + err_message = None + if not response or not response.status_code == 200: + if response is not None: + err_message = response.text + _, stderr_full_path = get_log_path(config_file_name) + with open(stderr_full_path, 'a+') as fout: + fout.write(json.dumps(json.loads(err_message), indent=4, sort_keys=True, separators=(',', ':'))) + return False, err_message + result, message = setNNIManagerIp(experiment_config, port, config_file_name) + if not result: + return result, message + #set trial_config + return set_trial_config(experiment_config, port, config_file_name), err_message + +def set_aml_config(experiment_config, port, config_file_name): + '''set aml configuration''' + aml_config_data = dict() + aml_config_data['aml_config'] = experiment_config['amlConfig'] + response = rest_put(cluster_metadata_url(port), json.dumps(aml_config_data), REST_TIME_OUT) + err_message = None + if not response or not response.status_code == 200: + if response is not None: + err_message = response.text + _, stderr_full_path = get_log_path(config_file_name) + with open(stderr_full_path, 'a+') as fout: + fout.write(json.dumps(json.loads(err_message), indent=4, sort_keys=True, separators=(',', ':'))) + return False, err_message + result, message = setNNIManagerIp(experiment_config, port, config_file_name) + if not result: + return result, message + #set trial_config + return set_trial_config(experiment_config, port, config_file_name), err_message + +def set_experiment(experiment_config, mode, port, config_file_name): + '''Call startExperiment (rest POST /experiment) with yaml file content''' + request_data = dict() + request_data['authorName'] = experiment_config['authorName'] + request_data['experimentName'] = experiment_config['experimentName'] + request_data['trialConcurrency'] = experiment_config['trialConcurrency'] + request_data['maxExecDuration'] = experiment_config['maxExecDuration'] + request_data['maxTrialNum'] = experiment_config['maxTrialNum'] + request_data['searchSpace'] = experiment_config.get('searchSpace') + request_data['trainingServicePlatform'] = experiment_config.get('trainingServicePlatform') + if experiment_config.get('description'): + request_data['description'] = experiment_config['description'] + if experiment_config.get('multiPhase'): + request_data['multiPhase'] = experiment_config.get('multiPhase') + if experiment_config.get('multiThread'): + request_data['multiThread'] = experiment_config.get('multiThread') + if experiment_config.get('advisor'): + request_data['advisor'] = experiment_config['advisor'] + if request_data['advisor'].get('gpuNum'): + print_error('gpuNum is deprecated, please use gpuIndices instead.') + if request_data['advisor'].get('gpuIndices') and isinstance(request_data['advisor'].get('gpuIndices'), int): + request_data['advisor']['gpuIndices'] = str(request_data['advisor'].get('gpuIndices')) + else: + request_data['tuner'] = experiment_config['tuner'] + if request_data['tuner'].get('gpuNum'): + print_error('gpuNum is deprecated, please use gpuIndices instead.') + if request_data['tuner'].get('gpuIndices') and isinstance(request_data['tuner'].get('gpuIndices'), int): + request_data['tuner']['gpuIndices'] = str(request_data['tuner'].get('gpuIndices')) + if 'assessor' in experiment_config: + request_data['assessor'] = experiment_config['assessor'] + if request_data['assessor'].get('gpuNum'): + print_error('gpuNum is deprecated, please remove it from your config file.') + #debug mode should disable version check + if experiment_config.get('debug') is not None: + request_data['versionCheck'] = not experiment_config.get('debug') + #validate version check + if experiment_config.get('versionCheck') is not None: + request_data['versionCheck'] = experiment_config.get('versionCheck') + if experiment_config.get('logCollection'): + request_data['logCollection'] = experiment_config.get('logCollection') + request_data['clusterMetaData'] = [] + if experiment_config['trainingServicePlatform'] == 'local': + request_data['clusterMetaData'].append( + {'key':'codeDir', 'value':experiment_config['trial']['codeDir']}) + request_data['clusterMetaData'].append( + {'key': 'command', 'value': experiment_config['trial']['command']}) + elif experiment_config['trainingServicePlatform'] == 'remote': + request_data['clusterMetaData'].append( + {'key': 'machine_list', 'value': experiment_config['machineList']}) + request_data['clusterMetaData'].append( + {'key': 'trial_config', 'value': experiment_config['trial']}) + if not experiment_config.get('remoteConfig'): + # set default value of reuse in remoteConfig to False + experiment_config['remoteConfig'] = {'reuse': False} + request_data['clusterMetaData'].append( + {'key': 'remote_config', 'value': experiment_config['remoteConfig']}) + elif experiment_config['trainingServicePlatform'] == 'pai': + request_data['clusterMetaData'].append( + {'key': 'pai_config', 'value': experiment_config['paiConfig']}) + request_data['clusterMetaData'].append( + {'key': 'trial_config', 'value': experiment_config['trial']}) + elif experiment_config['trainingServicePlatform'] == 'paiYarn': + request_data['clusterMetaData'].append( + {'key': 'pai_yarn_config', 'value': experiment_config['paiYarnConfig']}) + request_data['clusterMetaData'].append( + {'key': 'trial_config', 'value': experiment_config['trial']}) + elif experiment_config['trainingServicePlatform'] == 'kubeflow': + request_data['clusterMetaData'].append( + {'key': 'kubeflow_config', 'value': experiment_config['kubeflowConfig']}) + request_data['clusterMetaData'].append( + {'key': 'trial_config', 'value': experiment_config['trial']}) + elif experiment_config['trainingServicePlatform'] == 'frameworkcontroller': + request_data['clusterMetaData'].append( + {'key': 'frameworkcontroller_config', 'value': experiment_config['frameworkcontrollerConfig']}) + request_data['clusterMetaData'].append( + {'key': 'trial_config', 'value': experiment_config['trial']}) + elif experiment_config['trainingServicePlatform'] == 'aml': + request_data['clusterMetaData'].append( + {'key': 'aml_config', 'value': experiment_config['amlConfig']}) + request_data['clusterMetaData'].append( + {'key': 'trial_config', 'value': experiment_config['trial']}) + response = rest_post(experiment_url(port), json.dumps(request_data), REST_TIME_OUT, show_error=True) + if check_response(response): + return response + else: + _, stderr_full_path = get_log_path(config_file_name) + if response is not None: + with open(stderr_full_path, 'a+') as fout: + fout.write(json.dumps(json.loads(response.text), indent=4, sort_keys=True, separators=(',', ':'))) + print_error('Setting experiment error, error message is {}'.format(response.text)) + return None + +def set_platform_config(platform, experiment_config, port, config_file_name, rest_process): + '''call set_cluster_metadata for specific platform''' + print_normal('Setting {0} config...'.format(platform)) + config_result, err_msg = None, None + if platform == 'local': + config_result, err_msg = set_local_config(experiment_config, port, config_file_name) + elif platform == 'remote': + config_result, err_msg = set_remote_config(experiment_config, port, config_file_name) + elif platform == 'pai': + config_result, err_msg = set_pai_config(experiment_config, port, config_file_name) + elif platform == 'paiYarn': + config_result, err_msg = set_pai_yarn_config(experiment_config, port, config_file_name) + elif platform == 'kubeflow': + config_result, err_msg = set_kubeflow_config(experiment_config, port, config_file_name) + elif platform == 'frameworkcontroller': + config_result, err_msg = set_frameworkcontroller_config(experiment_config, port, config_file_name) + elif platform == 'dlts': + config_result, err_msg = set_dlts_config(experiment_config, port, config_file_name) + elif platform == 'aml': + config_result, err_msg = set_aml_config(experiment_config, port, config_file_name) + else: + raise Exception(ERROR_INFO % 'Unsupported platform!') + exit(1) + if config_result: + print_normal('Successfully set {0} config!'.format(platform)) + else: + print_error('Failed! Error is: {}'.format(err_msg)) + try: + kill_command(rest_process.pid) + except Exception: + raise Exception(ERROR_INFO % 'Rest server stopped!') + exit(1) + +def launch_experiment(args, experiment_config, mode, config_file_name, experiment_id=None): + '''follow steps to start rest server and start experiment''' + nni_config = Config(config_file_name) + # check packages for tuner + package_name, module_name = None, None + if experiment_config.get('tuner') and experiment_config['tuner'].get('builtinTunerName'): + package_name = experiment_config['tuner']['builtinTunerName'] + module_name, _ = get_builtin_module_class_name('tuners', package_name) + elif experiment_config.get('advisor') and experiment_config['advisor'].get('builtinAdvisorName'): + package_name = experiment_config['advisor']['builtinAdvisorName'] + module_name, _ = get_builtin_module_class_name('advisors', package_name) + if package_name and module_name: + try: + stdout_full_path, stderr_full_path = get_log_path(config_file_name) + with open(stdout_full_path, 'a+') as stdout_file, open(stderr_full_path, 'a+') as stderr_file: + check_call([sys.executable, '-c', 'import %s'%(module_name)], stdout=stdout_file, stderr=stderr_file) + except CalledProcessError: + print_error('some errors happen when import package %s.' %(package_name)) + print_log_content(config_file_name) + if package_name in INSTALLABLE_PACKAGE_META: + print_error('If %s is not installed, it should be installed through '\ + '\'nnictl package install --name %s\''%(package_name, package_name)) + exit(1) + log_dir = experiment_config['logDir'] if experiment_config.get('logDir') else None + log_level = experiment_config['logLevel'] if experiment_config.get('logLevel') else None + #view experiment mode do not need debug function, when view an experiment, there will be no new logs created + foreground = False + if mode != 'view': + foreground = args.foreground + if log_level not in ['trace', 'debug'] and (args.debug or experiment_config.get('debug') is True): + log_level = 'debug' + # start rest server + rest_process, start_time = start_rest_server(args.port, experiment_config['trainingServicePlatform'], \ + mode, config_file_name, foreground, experiment_id, log_dir, log_level) + nni_config.set_config('restServerPid', rest_process.pid) + # Deal with annotation + if experiment_config.get('useAnnotation'): + path = os.path.join(tempfile.gettempdir(), get_user(), 'nni', 'annotation') + if not os.path.isdir(path): + os.makedirs(path) + path = tempfile.mkdtemp(dir=path) + nas_mode = experiment_config['trial'].get('nasMode', 'classic_mode') + code_dir = expand_annotations(experiment_config['trial']['codeDir'], path, nas_mode=nas_mode) + experiment_config['trial']['codeDir'] = code_dir + search_space = generate_search_space(code_dir) + experiment_config['searchSpace'] = json.dumps(search_space) + assert search_space, ERROR_INFO % 'Generated search space is empty' + elif experiment_config.get('searchSpacePath'): + search_space = get_json_content(experiment_config.get('searchSpacePath')) + experiment_config['searchSpace'] = json.dumps(search_space) + else: + experiment_config['searchSpace'] = json.dumps('') + + # check rest server + running, _ = check_rest_server(args.port) + if running: + print_normal('Successfully started Restful server!') + else: + print_error('Restful server start failed!') + print_log_content(config_file_name) + try: + kill_command(rest_process.pid) + except Exception: + raise Exception(ERROR_INFO % 'Rest server stopped!') + exit(1) + if mode != 'view': + # set platform configuration + set_platform_config(experiment_config['trainingServicePlatform'], experiment_config, args.port,\ + config_file_name, rest_process) + + # start a new experiment + print_normal('Starting experiment...') + # set debug configuration + if mode != 'view' and experiment_config.get('debug') is None: + experiment_config['debug'] = args.debug + response = set_experiment(experiment_config, mode, args.port, config_file_name) + if response: + if experiment_id is None: + experiment_id = json.loads(response.text).get('experiment_id') + nni_config.set_config('experimentId', experiment_id) + else: + print_error('Start experiment failed!') + print_log_content(config_file_name) + try: + kill_command(rest_process.pid) + except Exception: + raise Exception(ERROR_INFO % 'Restful server stopped!') + exit(1) + if experiment_config.get('nniManagerIp'): + web_ui_url_list = ['{0}:{1}'.format(experiment_config['nniManagerIp'], str(args.port))] + else: + web_ui_url_list = get_local_urls(args.port) + nni_config.set_config('webuiUrl', web_ui_url_list) + + # save experiment information + nnictl_experiment_config = Experiments() + nnictl_experiment_config.add_experiment(experiment_id, args.port, start_time, config_file_name, + experiment_config['trainingServicePlatform'], + experiment_config['experimentName']) + + print_normal(EXPERIMENT_SUCCESS_INFO % (experiment_id, ' '.join(web_ui_url_list))) + if mode != 'view' and args.foreground: + try: + while True: + log_content = rest_process.stdout.readline().strip().decode('utf-8') + print(log_content) + except KeyboardInterrupt: + kill_command(rest_process.pid) + print_normal('Stopping experiment...') + +def create_experiment(args): + '''start a new experiment''' + config_file_name = ''.join(random.sample(string.ascii_letters + string.digits, 8)) + nni_config = Config(config_file_name) + config_path = os.path.abspath(args.config) + if not os.path.exists(config_path): + print_error('Please set correct config path!') + exit(1) + experiment_config = get_yml_content(config_path) + try: + validate_all_content(experiment_config, config_path) + except Exception as e: + print_error(e) + exit(1) + + nni_config.set_config('experimentConfig', experiment_config) + nni_config.set_config('restServerPort', args.port) + try: + launch_experiment(args, experiment_config, 'new', config_file_name) + except Exception as exception: + nni_config = Config(config_file_name) + restServerPid = nni_config.get_config('restServerPid') + if restServerPid: + kill_command(restServerPid) + print_error(exception) + exit(1) + +def manage_stopped_experiment(args, mode): + '''view a stopped experiment''' + update_experiment() + experiment_config = Experiments() + experiment_dict = experiment_config.get_all_experiments() + experiment_id = None + #find the latest stopped experiment + if not args.id: + print_error('Please set experiment id! \nYou could use \'nnictl {0} id\' to {0} a stopped experiment!\n' \ + 'You could use \'nnictl experiment list --all\' to show all experiments!'.format(mode)) + exit(1) + else: + if experiment_dict.get(args.id) is None: + print_error('Id %s not exist!' % args.id) + exit(1) + if experiment_dict[args.id]['status'] != 'STOPPED': + print_error('Only stopped experiments can be {0}ed!'.format(mode)) + exit(1) + experiment_id = args.id + print_normal('{0} experiment {1}...'.format(mode, experiment_id)) + nni_config = Config(experiment_dict[experiment_id]['fileName']) + experiment_config = nni_config.get_config('experimentConfig') + experiment_id = nni_config.get_config('experimentId') + new_config_file_name = ''.join(random.sample(string.ascii_letters + string.digits, 8)) + new_nni_config = Config(new_config_file_name) + new_nni_config.set_config('experimentConfig', experiment_config) + new_nni_config.set_config('restServerPort', args.port) + try: + launch_experiment(args, experiment_config, mode, new_config_file_name, experiment_id) + except Exception as exception: + nni_config = Config(new_config_file_name) + restServerPid = nni_config.get_config('restServerPid') + if restServerPid: + kill_command(restServerPid) + print_error(exception) + exit(1) + +def view_experiment(args): + '''view a stopped experiment''' + manage_stopped_experiment(args, 'view') + +def resume_experiment(args): + '''resume an experiment''' + manage_stopped_experiment(args, 'resume') diff --git a/new_impl/cv/third_party/nni/tools/nnictl/launcher_utils.py b/new_impl/cv/third_party/nni/tools/nnictl/launcher_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..48dd2779a36ee27e4c1845587b216e32760a33ad --- /dev/null +++ b/new_impl/cv/third_party/nni/tools/nnictl/launcher_utils.py @@ -0,0 +1,114 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +from schema import SchemaError +from .config_schema import NNIConfigSchema +from .common_utils import print_normal + +def expand_path(experiment_config, key): + '''Change '~' to user home directory''' + if experiment_config.get(key): + experiment_config[key] = os.path.expanduser(experiment_config[key]) + +def parse_relative_path(root_path, experiment_config, key): + '''Change relative path to absolute path''' + if experiment_config.get(key) and not os.path.isabs(experiment_config.get(key)): + absolute_path = os.path.join(root_path, experiment_config.get(key)) + print_normal('expand %s: %s to %s ' % (key, experiment_config[key], absolute_path)) + experiment_config[key] = absolute_path + +def parse_time(time): + '''Change the time to seconds''' + unit = time[-1] + if unit not in ['s', 'm', 'h', 'd']: + raise SchemaError('the unit of time could only from {s, m, h, d}') + time = time[:-1] + if not time.isdigit(): + raise SchemaError('time format error!') + parse_dict = {'s':1, 'm':60, 'h':3600, 'd':86400} + return int(time) * parse_dict[unit] + +def parse_path(experiment_config, config_path): + '''Parse path in config file''' + expand_path(experiment_config, 'searchSpacePath') + if experiment_config.get('trial'): + expand_path(experiment_config['trial'], 'codeDir') + if experiment_config['trial'].get('authFile'): + expand_path(experiment_config['trial'], 'authFile') + if experiment_config['trial'].get('ps'): + if experiment_config['trial']['ps'].get('privateRegistryAuthPath'): + expand_path(experiment_config['trial']['ps'], 'privateRegistryAuthPath') + if experiment_config['trial'].get('master'): + if experiment_config['trial']['master'].get('privateRegistryAuthPath'): + expand_path(experiment_config['trial']['master'], 'privateRegistryAuthPath') + if experiment_config['trial'].get('worker'): + if experiment_config['trial']['worker'].get('privateRegistryAuthPath'): + expand_path(experiment_config['trial']['worker'], 'privateRegistryAuthPath') + if experiment_config['trial'].get('taskRoles'): + for index in range(len(experiment_config['trial']['taskRoles'])): + if experiment_config['trial']['taskRoles'][index].get('privateRegistryAuthPath'): + expand_path(experiment_config['trial']['taskRoles'][index], 'privateRegistryAuthPath') + if experiment_config.get('tuner'): + expand_path(experiment_config['tuner'], 'codeDir') + if experiment_config.get('assessor'): + expand_path(experiment_config['assessor'], 'codeDir') + if experiment_config.get('advisor'): + expand_path(experiment_config['advisor'], 'codeDir') + if experiment_config.get('machineList'): + for index in range(len(experiment_config['machineList'])): + expand_path(experiment_config['machineList'][index], 'sshKeyPath') + if experiment_config['trial'].get('paiConfigPath'): + expand_path(experiment_config['trial'], 'paiConfigPath') + + #if users use relative path, convert it to absolute path + root_path = os.path.dirname(config_path) + if experiment_config.get('searchSpacePath'): + parse_relative_path(root_path, experiment_config, 'searchSpacePath') + if experiment_config.get('trial'): + parse_relative_path(root_path, experiment_config['trial'], 'codeDir') + if experiment_config['trial'].get('authFile'): + parse_relative_path(root_path, experiment_config['trial'], 'authFile') + if experiment_config['trial'].get('ps'): + if experiment_config['trial']['ps'].get('privateRegistryAuthPath'): + parse_relative_path(root_path, experiment_config['trial']['ps'], 'privateRegistryAuthPath') + if experiment_config['trial'].get('master'): + if experiment_config['trial']['master'].get('privateRegistryAuthPath'): + parse_relative_path(root_path, experiment_config['trial']['master'], 'privateRegistryAuthPath') + if experiment_config['trial'].get('worker'): + if experiment_config['trial']['worker'].get('privateRegistryAuthPath'): + parse_relative_path(root_path, experiment_config['trial']['worker'], 'privateRegistryAuthPath') + if experiment_config['trial'].get('taskRoles'): + for index in range(len(experiment_config['trial']['taskRoles'])): + if experiment_config['trial']['taskRoles'][index].get('privateRegistryAuthPath'): + parse_relative_path(root_path, experiment_config['trial']['taskRoles'][index], 'privateRegistryAuthPath') + if experiment_config.get('tuner'): + parse_relative_path(root_path, experiment_config['tuner'], 'codeDir') + if experiment_config.get('assessor'): + parse_relative_path(root_path, experiment_config['assessor'], 'codeDir') + if experiment_config.get('advisor'): + parse_relative_path(root_path, experiment_config['advisor'], 'codeDir') + if experiment_config.get('machineList'): + for index in range(len(experiment_config['machineList'])): + parse_relative_path(root_path, experiment_config['machineList'][index], 'sshKeyPath') + if experiment_config['trial'].get('paiConfigPath'): + parse_relative_path(root_path, experiment_config['trial'], 'paiConfigPath') + +def set_default_values(experiment_config): + if experiment_config.get('maxExecDuration') is None: + experiment_config['maxExecDuration'] = '999d' + if experiment_config.get('maxTrialNum') is None: + experiment_config['maxTrialNum'] = 99999 + if experiment_config['trainingServicePlatform'] == 'remote': + for index in range(len(experiment_config['machineList'])): + if experiment_config['machineList'][index].get('port') is None: + experiment_config['machineList'][index]['port'] = 22 + +def validate_all_content(experiment_config, config_path): + '''Validate whether experiment_config is valid''' + parse_path(experiment_config, config_path) + set_default_values(experiment_config) + + NNIConfigSchema().validate(experiment_config) + + experiment_config['maxExecDuration'] = parse_time(experiment_config['maxExecDuration']) diff --git a/new_impl/cv/third_party/nni/tools/nnictl/nnictl.py b/new_impl/cv/third_party/nni/tools/nnictl/nnictl.py new file mode 100644 index 0000000000000000000000000000000000000000..cc5d4cdd11675f9336877d34edf1d71796c3f2e8 --- /dev/null +++ b/new_impl/cv/third_party/nni/tools/nnictl/nnictl.py @@ -0,0 +1,258 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import argparse +import os +import pkg_resources +from colorama import init +from .common_utils import print_error +from .launcher import create_experiment, resume_experiment, view_experiment +from .updater import update_searchspace, update_concurrency, update_duration, update_trialnum, import_data +from .nnictl_utils import stop_experiment, trial_ls, trial_kill, list_experiment, experiment_status,\ + log_trial, experiment_clean, platform_clean, experiment_list, \ + monitor_experiment, export_trials_data, trial_codegen, webui_url, \ + get_config, log_stdout, log_stderr, search_space_auto_gen, webui_nas, \ + save_experiment, load_experiment +from .package_management import package_install, package_uninstall, package_show, package_list +from .constants import DEFAULT_REST_PORT +from .tensorboard_utils import start_tensorboard, stop_tensorboard +init(autoreset=True) + +if os.environ.get('COVERAGE_PROCESS_START'): + import coverage + coverage.process_startup() + +def nni_info(*args): + if args[0].version: + try: + print(pkg_resources.get_distribution('nni').version) + except pkg_resources.ResolutionError: + print_error('Get version failed, please use `pip3 list | grep nni` to check nni version!') + else: + print('please run "nnictl {positional argument} --help" to see nnictl guidance') + +def parse_args(): + '''Definite the arguments users need to follow and input''' + parser = argparse.ArgumentParser(prog='nnictl', description='use nnictl command to control nni experiments') + parser.add_argument('--version', '-v', action='store_true') + parser.set_defaults(func=nni_info) + + # create subparsers for args with sub values + subparsers = parser.add_subparsers() + + # parse the command of auto generating search space + parser_start = subparsers.add_parser('ss_gen', help='automatically generate search space file from trial code') + parser_start.add_argument('--trial_command', '-t', required=True, dest='trial_command', help='the command for running trial code') + parser_start.add_argument('--trial_dir', '-d', default='./', dest='trial_dir', help='the directory for running the command') + parser_start.add_argument('--file', '-f', default='nni_auto_gen_search_space.json', dest='file', help='the path of search space file') + parser_start.set_defaults(func=search_space_auto_gen) + + # parse start command + parser_start = subparsers.add_parser('create', help='create a new experiment') + parser_start.add_argument('--config', '-c', required=True, dest='config', help='the path of yaml config file') + parser_start.add_argument('--port', '-p', default=DEFAULT_REST_PORT, dest='port', help='the port of restful server') + parser_start.add_argument('--debug', '-d', action='store_true', help=' set debug mode') + parser_start.add_argument('--foreground', '-f', action='store_true', help=' set foreground mode, print log content to terminal') + parser_start.set_defaults(func=create_experiment) + + # parse resume command + parser_resume = subparsers.add_parser('resume', help='resume a new experiment') + parser_resume.add_argument('id', nargs='?', help='The id of the experiment you want to resume') + parser_resume.add_argument('--port', '-p', default=DEFAULT_REST_PORT, dest='port', help='the port of restful server') + parser_resume.add_argument('--debug', '-d', action='store_true', help=' set debug mode') + parser_resume.add_argument('--foreground', '-f', action='store_true', help=' set foreground mode, print log content to terminal') + parser_resume.set_defaults(func=resume_experiment) + + # parse view command + parser_view = subparsers.add_parser('view', help='view a stopped experiment') + parser_view.add_argument('id', nargs='?', help='The id of the experiment you want to view') + parser_view.add_argument('--port', '-p', default=DEFAULT_REST_PORT, dest='port', help='the port of restful server') + parser_view.set_defaults(func=view_experiment) + + # parse update command + parser_updater = subparsers.add_parser('update', help='update the experiment') + #add subparsers for parser_updater + parser_updater_subparsers = parser_updater.add_subparsers() + parser_updater_searchspace = parser_updater_subparsers.add_parser('searchspace', help='update searchspace') + parser_updater_searchspace.add_argument('id', nargs='?', help='the id of experiment') + parser_updater_searchspace.add_argument('--filename', '-f', required=True) + parser_updater_searchspace.set_defaults(func=update_searchspace) + parser_updater_concurrency = parser_updater_subparsers.add_parser('concurrency', help='update concurrency') + parser_updater_concurrency.add_argument('id', nargs='?', help='the id of experiment') + parser_updater_concurrency.add_argument('--value', '-v', required=True) + parser_updater_concurrency.set_defaults(func=update_concurrency) + parser_updater_duration = parser_updater_subparsers.add_parser('duration', help='update duration') + parser_updater_duration.add_argument('id', nargs='?', help='the id of experiment') + parser_updater_duration.add_argument('--value', '-v', required=True, help='the unit of time should in {\'s\', \'m\', \'h\', \'d\'}') + parser_updater_duration.set_defaults(func=update_duration) + parser_updater_trialnum = parser_updater_subparsers.add_parser('trialnum', help='update maxtrialnum') + parser_updater_trialnum.add_argument('id', nargs='?', help='the id of experiment') + parser_updater_trialnum.add_argument('--value', '-v', required=True) + parser_updater_trialnum.set_defaults(func=update_trialnum) + + #parse stop command + parser_stop = subparsers.add_parser('stop', help='stop the experiment') + parser_stop.add_argument('id', nargs='?', help='the id of experiment, use \'all\' to stop all running experiments') + parser_stop.add_argument('--port', '-p', dest='port', help='the port of restful server') + parser_stop.add_argument('--all', '-a', action='store_true', help='stop all of experiments') + parser_stop.set_defaults(func=stop_experiment) + + #parse trial command + parser_trial = subparsers.add_parser('trial', help='get trial information') + #add subparsers for parser_trial + parser_trial_subparsers = parser_trial.add_subparsers() + parser_trial_ls = parser_trial_subparsers.add_parser('ls', help='list trial jobs') + parser_trial_ls.add_argument('id', nargs='?', help='the id of experiment') + parser_trial_ls.add_argument('--head', type=int, help='list the highest experiments on the default metric') + parser_trial_ls.add_argument('--tail', type=int, help='list the lowest experiments on the default metric') + parser_trial_ls.set_defaults(func=trial_ls) + parser_trial_kill = parser_trial_subparsers.add_parser('kill', help='kill trial jobs') + parser_trial_kill.add_argument('id', nargs='?', help='the id of experiment') + parser_trial_kill.add_argument('--trial_id', '-T', required=True, dest='trial_id', help='the id of trial to be killed') + parser_trial_kill.set_defaults(func=trial_kill) + parser_trial_codegen = parser_trial_subparsers.add_parser('codegen', help='generate trial code for a specific trial') + parser_trial_codegen.add_argument('id', nargs='?', help='the id of experiment') + parser_trial_codegen.add_argument('--trial_id', '-T', required=True, dest='trial_id', help='the id of trial to do code generation') + parser_trial_codegen.set_defaults(func=trial_codegen) + + #parse experiment command + parser_experiment = subparsers.add_parser('experiment', help='get experiment information') + #add subparsers for parser_experiment + parser_experiment_subparsers = parser_experiment.add_subparsers() + parser_experiment_show = parser_experiment_subparsers.add_parser('show', help='show the information of experiment') + parser_experiment_show.add_argument('id', nargs='?', help='the id of experiment') + parser_experiment_show.set_defaults(func=list_experiment) + parser_experiment_status = parser_experiment_subparsers.add_parser('status', help='show the status of experiment') + parser_experiment_status.add_argument('id', nargs='?', help='the id of experiment') + parser_experiment_status.set_defaults(func=experiment_status) + parser_experiment_list = parser_experiment_subparsers.add_parser('list', help='list all of running experiment ids') + parser_experiment_list.add_argument('--all', action='store_true', default=False, help='list all of experiments') + parser_experiment_list.set_defaults(func=experiment_list) + parser_experiment_clean = parser_experiment_subparsers.add_parser('delete', help='clean up the experiment data') + parser_experiment_clean.add_argument('id', nargs='?', help='the id of experiment') + parser_experiment_clean.add_argument('--all', action='store_true', default=False, help='delete all of experiments') + parser_experiment_clean.set_defaults(func=experiment_clean) + #import tuning data + parser_import_data = parser_experiment_subparsers.add_parser('import', help='import additional data') + parser_import_data.add_argument('id', nargs='?', help='the id of experiment') + parser_import_data.add_argument('--filename', '-f', required=True) + parser_import_data.set_defaults(func=import_data) + #export trial data + parser_trial_export = parser_experiment_subparsers.add_parser('export', help='export trial job results to csv or json') + parser_trial_export.add_argument('id', nargs='?', help='the id of experiment') + parser_trial_export.add_argument('--type', '-t', choices=['json', 'csv'], required=True, dest='type', help='target file type') + parser_trial_export.add_argument('--filename', '-f', required=True, dest='path', help='target file path') + parser_trial_export.add_argument('--intermediate', '-i', action='store_true', + default=False, help='are intermediate results included') + parser_trial_export.set_defaults(func=export_trials_data) + #save an NNI experiment + parser_save_experiment = parser_experiment_subparsers.add_parser('save', help='save an experiment') + parser_save_experiment.add_argument('id', nargs='?', help='the id of experiment') + parser_save_experiment.add_argument('--path', '-p', required=False, help='the folder path to store nni experiment data, \ + default current working directory') + parser_save_experiment.add_argument('--saveCodeDir', '-s', action='store_true', default=False, help='save codeDir data \ + of the experiment') + parser_save_experiment.set_defaults(func=save_experiment) + #load an NNI experiment + parser_load_experiment = parser_experiment_subparsers.add_parser('load', help='load an experiment') + parser_load_experiment.add_argument('--path', '-p', required=True, help='the path of nni package file') + parser_load_experiment.add_argument('--codeDir', '-c', required=True, help='the path of codeDir for loaded experiment, \ + this path will also put the code in the loaded experiment package') + parser_load_experiment.add_argument('--logDir', '-l', required=False, help='the path of logDir for loaded experiment') + parser_load_experiment.add_argument('--searchSpacePath', '-s', required=False, help='the path of search space file for \ + loaded experiment, this path contains file name. Default in $codeDir/search_space.json') + parser_load_experiment.set_defaults(func=load_experiment) + + #parse platform command + parser_platform = subparsers.add_parser('platform', help='get platform information') + #add subparsers for parser_platform + parser_platform_subparsers = parser_platform.add_subparsers() + parser_platform_clean = parser_platform_subparsers.add_parser('clean', help='clean up the platform data') + parser_platform_clean.add_argument('--config', '-c', required=True, dest='config', help='the path of yaml config file') + parser_platform_clean.set_defaults(func=platform_clean) + + #TODO:finish webui function + #parse board command + parser_webui = subparsers.add_parser('webui', help='get web ui information') + #add subparsers for parser_board + parser_webui_subparsers = parser_webui.add_subparsers() + parser_webui_url = parser_webui_subparsers.add_parser('url', help='show the url of web ui') + parser_webui_url.add_argument('id', nargs='?', help='the id of experiment') + parser_webui_url.set_defaults(func=webui_url) + parser_webui_nas = parser_webui_subparsers.add_parser('nas', help='show nas ui') + parser_webui_nas.add_argument('--port', default=6060, type=int, help='port of nas ui') + parser_webui_nas.add_argument('--logdir', default='.', type=str, help='the logdir where nas ui will read data') + parser_webui_nas.set_defaults(func=webui_nas) + + #parse config command + parser_config = subparsers.add_parser('config', help='get config information') + parser_config_subparsers = parser_config.add_subparsers() + parser_config_show = parser_config_subparsers.add_parser('show', help='show the information of config') + parser_config_show.add_argument('id', nargs='?', help='the id of experiment') + parser_config_show.set_defaults(func=get_config) + + #parse log command + parser_log = subparsers.add_parser('log', help='get log information') + # add subparsers for parser_log + parser_log_subparsers = parser_log.add_subparsers() + parser_log_stdout = parser_log_subparsers.add_parser('stdout', help='get stdout information') + parser_log_stdout.add_argument('id', nargs='?', help='the id of experiment') + parser_log_stdout.add_argument('--tail', '-T', dest='tail', type=int, help='get tail -100 content of stdout') + parser_log_stdout.add_argument('--head', '-H', dest='head', type=int, help='get head -100 content of stdout') + parser_log_stdout.add_argument('--path', action='store_true', default=False, help='get the path of stdout file') + parser_log_stdout.set_defaults(func=log_stdout) + parser_log_stderr = parser_log_subparsers.add_parser('stderr', help='get stderr information') + parser_log_stderr.add_argument('id', nargs='?', help='the id of experiment') + parser_log_stderr.add_argument('--tail', '-T', dest='tail', type=int, help='get tail -100 content of stderr') + parser_log_stderr.add_argument('--head', '-H', dest='head', type=int, help='get head -100 content of stderr') + parser_log_stderr.add_argument('--path', action='store_true', default=False, help='get the path of stderr file') + parser_log_stderr.set_defaults(func=log_stderr) + parser_log_trial = parser_log_subparsers.add_parser('trial', help='get trial log path') + parser_log_trial.add_argument('id', nargs='?', help='the id of experiment') + parser_log_trial.add_argument('--trial_id', '-T', dest='trial_id', help='find trial log path by id') + parser_log_trial.set_defaults(func=log_trial) + + #parse package command + parser_package = subparsers.add_parser('package', help='control nni tuner and assessor packages') + # add subparsers for parser_package + parser_package_subparsers = parser_package.add_subparsers() + parser_package_install = parser_package_subparsers.add_parser('install', help='install packages') + parser_package_install.add_argument('source', nargs='?', help='installation source, can be a directory or whl file') + parser_package_install.add_argument('--name', '-n', dest='name', help='package name to be installed', required=False) + parser_package_install.set_defaults(func=package_install) + + parser_package_uninstall = parser_package_subparsers.add_parser('uninstall', help='uninstall packages') + parser_package_uninstall.add_argument('name', nargs=1, help='package name to be uninstalled') + parser_package_uninstall.set_defaults(func=package_uninstall) + + parser_package_show = parser_package_subparsers.add_parser('show', help='show the information of packages') + parser_package_show.add_argument('name', nargs=1, help='builtin name of the package') + parser_package_show.set_defaults(func=package_show) + + parser_package_list = parser_package_subparsers.add_parser('list', help='list installed packages') + parser_package_list.add_argument('--all', action='store_true', help='list all builtin packages') + parser_package_list.set_defaults(func=package_list) + + #parse tensorboard command + parser_tensorboard = subparsers.add_parser('tensorboard', help='manage tensorboard') + parser_tensorboard_subparsers = parser_tensorboard.add_subparsers() + parser_tensorboard_start = parser_tensorboard_subparsers.add_parser('start', help='start tensorboard') + parser_tensorboard_start.add_argument('id', nargs='?', help='the id of experiment') + parser_tensorboard_start.add_argument('--trial_id', '-T', dest='trial_id', help='the id of trial') + parser_tensorboard_start.add_argument('--port', dest='port', default=6006, help='the port to start tensorboard') + parser_tensorboard_start.set_defaults(func=start_tensorboard) + parser_tensorboard_stop = parser_tensorboard_subparsers.add_parser('stop', help='stop tensorboard') + parser_tensorboard_stop.add_argument('id', nargs='?', help='the id of experiment') + parser_tensorboard_stop.set_defaults(func=stop_tensorboard) + + #parse top command + parser_top = subparsers.add_parser('top', help='monitor the experiment') + parser_top.add_argument('--time', '-t', dest='time', type=int, default=3, help='the time interval to update the experiment status, ' \ + 'the unit is second') + parser_top.set_defaults(func=monitor_experiment) + + args = parser.parse_args() + args.func(args) + +if __name__ == '__main__': + parse_args() diff --git a/new_impl/cv/third_party/nni/tools/nnictl/nnictl_utils.py b/new_impl/cv/third_party/nni/tools/nnictl/nnictl_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..3427d698b074ef87837903179d3e8b3b3205962a --- /dev/null +++ b/new_impl/cv/third_party/nni/tools/nnictl/nnictl_utils.py @@ -0,0 +1,995 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import csv +import os +import sys +import json +import time +import re +import shutil +import subprocess +from functools import cmp_to_key +from datetime import datetime, timezone +from subprocess import Popen +from pyhdfs import HdfsClient +from nni.tools.annotation import expand_annotations +import nni_node +from .rest_utils import rest_get, rest_delete, check_rest_server_quick, check_response +from .url_utils import trial_jobs_url, experiment_url, trial_job_id_url, export_data_url, metric_data_url +from .config_utils import Config, Experiments +from .constants import NNICTL_HOME_DIR, NNI_HOME_DIR, EXPERIMENT_INFORMATION_FORMAT, EXPERIMENT_DETAIL_FORMAT, \ + EXPERIMENT_MONITOR_INFO, TRIAL_MONITOR_HEAD, TRIAL_MONITOR_CONTENT, TRIAL_MONITOR_TAIL, REST_TIME_OUT +from .common_utils import print_normal, print_error, print_warning, detect_process, get_yml_content, generate_temp_dir +from .command_utils import check_output_command, kill_command +from .ssh_utils import create_ssh_sftp_client, remove_remote_directory + +def get_experiment_time(port): + '''get the startTime and endTime of an experiment''' + response = rest_get(experiment_url(port), REST_TIME_OUT) + if response and check_response(response): + content = convert_time_stamp_to_date(json.loads(response.text)) + return content.get('startTime'), content.get('endTime') + return None, None + +def get_experiment_status(port): + '''get the status of an experiment''' + result, response = check_rest_server_quick(port) + if result: + return json.loads(response.text).get('status') + return None + +def update_experiment(): + '''Update the experiment status in config file''' + experiment_config = Experiments() + experiment_dict = experiment_config.get_all_experiments() + if not experiment_dict: + return None + for key in experiment_dict.keys(): + if isinstance(experiment_dict[key], dict): + if experiment_dict[key].get('status') != 'STOPPED': + nni_config = Config(experiment_dict[key]['fileName']) + rest_pid = nni_config.get_config('restServerPid') + if not detect_process(rest_pid): + experiment_config.update_experiment(key, 'status', 'STOPPED') + continue + rest_port = nni_config.get_config('restServerPort') + startTime, endTime = get_experiment_time(rest_port) + if startTime: + experiment_config.update_experiment(key, 'startTime', startTime) + if endTime: + experiment_config.update_experiment(key, 'endTime', endTime) + status = get_experiment_status(rest_port) + if status: + experiment_config.update_experiment(key, 'status', status) + +def check_experiment_id(args, update=True): + '''check if the id is valid + ''' + if update: + update_experiment() + experiment_config = Experiments() + experiment_dict = experiment_config.get_all_experiments() + if not experiment_dict: + print_normal('There is no experiment running...') + return None + if not args.id: + running_experiment_list = [] + for key in experiment_dict.keys(): + if isinstance(experiment_dict[key], dict): + if experiment_dict[key].get('status') != 'STOPPED': + running_experiment_list.append(key) + elif isinstance(experiment_dict[key], list): + # if the config file is old version, remove the configuration from file + experiment_config.remove_experiment(key) + if len(running_experiment_list) > 1: + print_error('There are multiple experiments, please set the experiment id...') + experiment_information = "" + for key in running_experiment_list: + experiment_information += EXPERIMENT_DETAIL_FORMAT % (key, + experiment_dict[key].get('experimentName', 'N/A'), + experiment_dict[key]['status'], + experiment_dict[key]['port'], + experiment_dict[key].get('platform'), + experiment_dict[key]['startTime'], + experiment_dict[key]['endTime']) + print(EXPERIMENT_INFORMATION_FORMAT % experiment_information) + exit(1) + elif not running_experiment_list: + print_error('There is no experiment running.') + return None + else: + return running_experiment_list[0] + if experiment_dict.get(args.id): + return args.id + else: + print_error('Id not correct.') + return None + +def parse_ids(args): + '''Parse the arguments for nnictl stop + 1.If port is provided and id is not specified, return the id who owns the port + 2.If both port and id are provided, return the id if it owns the port, otherwise fail + 3.If there is an id specified, return the corresponding id + 4.If there is no id specified, and there is an experiment running, return the id, or return Error + 5.If the id matches an experiment, nnictl will return the id. + 6.If the id ends with *, nnictl will match all ids matchs the regular + 7.If the id does not exist but match the prefix of an experiment id, nnictl will return the matched id + 8.If the id does not exist but match multiple prefix of the experiment ids, nnictl will give id information + ''' + update_experiment() + experiment_config = Experiments() + experiment_dict = experiment_config.get_all_experiments() + if not experiment_dict: + print_normal('Experiment is not running...') + return None + result_list = [] + running_experiment_list = [] + for key in experiment_dict.keys(): + if isinstance(experiment_dict[key], dict): + if experiment_dict[key].get('status') != 'STOPPED': + running_experiment_list.append(key) + elif isinstance(experiment_dict[key], list): + # if the config file is old version, remove the configuration from file + experiment_config.remove_experiment(key) + if args.all: + return running_experiment_list + if args.port is not None: + for key in running_experiment_list: + if str(experiment_dict[key]['port']) == args.port: + result_list.append(key) + if args.id and result_list and args.id != result_list[0]: + print_error('Experiment id and resful server port not match') + exit(1) + elif not args.id: + if len(running_experiment_list) > 1: + print_error('There are multiple experiments, please set the experiment id...') + experiment_information = "" + for key in running_experiment_list: + experiment_information += EXPERIMENT_DETAIL_FORMAT % (key, + experiment_dict[key].get('experimentName', 'N/A'), + experiment_dict[key]['status'], + experiment_dict[key]['port'], + experiment_dict[key].get('platform'), + experiment_dict[key]['startTime'], + experiment_dict[key]['endTime']) + print(EXPERIMENT_INFORMATION_FORMAT % experiment_information) + exit(1) + else: + result_list = running_experiment_list + elif args.id.endswith('*'): + for expId in running_experiment_list: + if expId.startswith(args.id[:-1]): + result_list.append(expId) + elif args.id in running_experiment_list: + result_list.append(args.id) + else: + for expId in running_experiment_list: + if expId.startswith(args.id): + result_list.append(expId) + if len(result_list) > 1: + print_error(args.id + ' is ambiguous, please choose ' + ' '.join(result_list)) + return None + if not result_list and (args.id or args.port): + print_error('There are no experiments matched, please set correct experiment id or restful server port') + elif not result_list: + print_error('There is no experiment running...') + return result_list + +def get_config_filename(args): + '''get the file name of config file''' + experiment_id = check_experiment_id(args) + if experiment_id is None: + print_error('Please set correct experiment id.') + exit(1) + experiment_config = Experiments() + experiment_dict = experiment_config.get_all_experiments() + return experiment_dict[experiment_id]['fileName'] + +def get_experiment_port(args): + '''get the port of experiment''' + experiment_id = check_experiment_id(args) + if experiment_id is None: + print_error('Please set correct experiment id.') + exit(1) + experiment_config = Experiments() + experiment_dict = experiment_config.get_all_experiments() + return experiment_dict[experiment_id]['port'] + +def convert_time_stamp_to_date(content): + '''Convert time stamp to date time format''' + start_time_stamp = content.get('startTime') + end_time_stamp = content.get('endTime') + if start_time_stamp: + start_time = datetime.fromtimestamp(start_time_stamp // 1000, timezone.utc).astimezone().strftime("%Y/%m/%d %H:%M:%S") + content['startTime'] = str(start_time) + if end_time_stamp: + end_time = datetime.fromtimestamp(end_time_stamp // 1000, timezone.utc).astimezone().strftime("%Y/%m/%d %H:%M:%S") + content['endTime'] = str(end_time) + return content + +def check_rest(args): + '''check if restful server is running''' + nni_config = Config(get_config_filename(args)) + rest_port = nni_config.get_config('restServerPort') + running, _ = check_rest_server_quick(rest_port) + if running: + print_normal('Restful server is running...') + else: + print_normal('Restful server is not running...') + return running + +def stop_experiment(args): + '''Stop the experiment which is running''' + if args.id and args.id == 'all': + print_warning('\'nnictl stop all\' is abolished, please use \'nnictl stop --all\' to stop all of experiments!') + exit(1) + experiment_id_list = parse_ids(args) + if experiment_id_list: + experiment_config = Experiments() + experiment_dict = experiment_config.get_all_experiments() + for experiment_id in experiment_id_list: + print_normal('Stopping experiment %s' % experiment_id) + nni_config = Config(experiment_dict[experiment_id]['fileName']) + rest_pid = nni_config.get_config('restServerPid') + if rest_pid: + kill_command(rest_pid) + tensorboard_pid_list = nni_config.get_config('tensorboardPidList') + if tensorboard_pid_list: + for tensorboard_pid in tensorboard_pid_list: + try: + kill_command(tensorboard_pid) + except Exception as exception: + print_error(exception) + nni_config.set_config('tensorboardPidList', []) + print_normal('Stop experiment success.') + experiment_config.update_experiment(experiment_id, 'status', 'STOPPED') + time_now = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())) + experiment_config.update_experiment(experiment_id, 'endTime', str(time_now)) + +def trial_ls(args): + '''List trial''' + def final_metric_data_cmp(lhs, rhs): + metric_l = json.loads(json.loads(lhs['finalMetricData'][0]['data'])) + metric_r = json.loads(json.loads(rhs['finalMetricData'][0]['data'])) + if isinstance(metric_l, float): + return metric_l - metric_r + elif isinstance(metric_l, dict): + return metric_l['default'] - metric_r['default'] + else: + print_error('Unexpected data format. Please check your data.') + raise ValueError + + if args.head and args.tail: + print_error('Head and tail cannot be set at the same time.') + return + nni_config = Config(get_config_filename(args)) + rest_port = nni_config.get_config('restServerPort') + rest_pid = nni_config.get_config('restServerPid') + if not detect_process(rest_pid): + print_error('Experiment is not running...') + return + running, response = check_rest_server_quick(rest_port) + if running: + response = rest_get(trial_jobs_url(rest_port), REST_TIME_OUT) + if response and check_response(response): + content = json.loads(response.text) + if args.head: + assert args.head > 0, 'The number of requested data must be greater than 0.' + content = sorted(filter(lambda x: 'finalMetricData' in x, content), + key=cmp_to_key(final_metric_data_cmp), reverse=True)[:args.head] + elif args.tail: + assert args.tail > 0, 'The number of requested data must be greater than 0.' + content = sorted(filter(lambda x: 'finalMetricData' in x, content), + key=cmp_to_key(final_metric_data_cmp))[:args.tail] + for index, value in enumerate(content): + content[index] = convert_time_stamp_to_date(value) + print(json.dumps(content, indent=4, sort_keys=True, separators=(',', ':'))) + return content + else: + print_error('List trial failed...') + else: + print_error('Restful server is not running...') + return None + +def trial_kill(args): + '''List trial''' + nni_config = Config(get_config_filename(args)) + rest_port = nni_config.get_config('restServerPort') + rest_pid = nni_config.get_config('restServerPid') + if not detect_process(rest_pid): + print_error('Experiment is not running...') + return + running, _ = check_rest_server_quick(rest_port) + if running: + response = rest_delete(trial_job_id_url(rest_port, args.trial_id), REST_TIME_OUT) + if response and check_response(response): + print(response.text) + return True + else: + print_error('Kill trial job failed...') + else: + print_error('Restful server is not running...') + return False + +def trial_codegen(args): + '''Generate code for a specific trial''' + print_warning('Currently, this command is only for nni nas programming interface.') + exp_id = check_experiment_id(args) + nni_config = Config(get_config_filename(args)) + if not nni_config.get_config('experimentConfig')['useAnnotation']: + print_error('The experiment is not using annotation') + exit(1) + code_dir = nni_config.get_config('experimentConfig')['trial']['codeDir'] + expand_annotations(code_dir, './exp_%s_trial_%s_code'%(exp_id, args.trial_id), exp_id, args.trial_id) + +def list_experiment(args): + '''Get experiment information''' + nni_config = Config(get_config_filename(args)) + rest_port = nni_config.get_config('restServerPort') + rest_pid = nni_config.get_config('restServerPid') + if not detect_process(rest_pid): + print_error('Experiment is not running...') + return + running, _ = check_rest_server_quick(rest_port) + if running: + response = rest_get(experiment_url(rest_port), REST_TIME_OUT) + if response and check_response(response): + content = convert_time_stamp_to_date(json.loads(response.text)) + print(json.dumps(content, indent=4, sort_keys=True, separators=(',', ':'))) + return content + else: + print_error('List experiment failed...') + else: + print_error('Restful server is not running...') + return None + +def experiment_status(args): + '''Show the status of experiment''' + nni_config = Config(get_config_filename(args)) + rest_port = nni_config.get_config('restServerPort') + result, response = check_rest_server_quick(rest_port) + if not result: + print_normal('Restful server is not running...') + else: + print(json.dumps(json.loads(response.text), indent=4, sort_keys=True, separators=(',', ':'))) + return result + +def log_internal(args, filetype): + '''internal function to call get_log_content''' + file_name = get_config_filename(args) + if filetype == 'stdout': + file_full_path = os.path.join(NNICTL_HOME_DIR, file_name, 'stdout') + else: + file_full_path = os.path.join(NNICTL_HOME_DIR, file_name, 'stderr') + print(check_output_command(file_full_path, head=args.head, tail=args.tail)) + +def log_stdout(args): + '''get stdout log''' + log_internal(args, 'stdout') + +def log_stderr(args): + '''get stderr log''' + log_internal(args, 'stderr') + +def log_trial(args): + ''''get trial log path''' + trial_id_path_dict = {} + trial_id_list = [] + nni_config = Config(get_config_filename(args)) + rest_port = nni_config.get_config('restServerPort') + rest_pid = nni_config.get_config('restServerPid') + if not detect_process(rest_pid): + print_error('Experiment is not running...') + return + running, response = check_rest_server_quick(rest_port) + if running: + response = rest_get(trial_jobs_url(rest_port), REST_TIME_OUT) + if response and check_response(response): + content = json.loads(response.text) + for trial in content: + trial_id_list.append(trial.get('id')) + if trial.get('logPath'): + trial_id_path_dict[trial.get('id')] = trial['logPath'] + else: + print_error('Restful server is not running...') + exit(1) + if args.trial_id: + if args.trial_id not in trial_id_list: + print_error('Trial id {0} not correct, please check your command!'.format(args.trial_id)) + exit(1) + if trial_id_path_dict.get(args.trial_id): + print_normal('id:' + args.trial_id + ' path:' + trial_id_path_dict[args.trial_id]) + else: + print_error('Log path is not available yet, please wait...') + exit(1) + else: + print_normal('All of trial log info:') + for key in trial_id_path_dict: + print_normal('id:' + key + ' path:' + trial_id_path_dict[key]) + if not trial_id_path_dict: + print_normal('None') + +def get_config(args): + '''get config info''' + nni_config = Config(get_config_filename(args)) + print(nni_config.get_all_config()) + +def webui_url(args): + '''show the url of web ui''' + nni_config = Config(get_config_filename(args)) + print_normal('{0} {1}'.format('Web UI url:', ' '.join(nni_config.get_config('webuiUrl')))) + +def webui_nas(args): + '''launch nas ui''' + print_normal('Starting NAS UI...') + try: + entry_dir = nni_node.__path__[0] + entry_file = os.path.join(entry_dir, 'nasui', 'server.js') + if sys.platform == 'win32': + node_command = os.path.join(entry_dir, 'node.exe') + else: + node_command = 'node' + cmds = [node_command, '--max-old-space-size=4096', entry_file, '--port', str(args.port), '--logdir', args.logdir] + subprocess.run(cmds, cwd=entry_dir) + except KeyboardInterrupt: + pass + +def local_clean(directory): + '''clean up local data''' + print_normal('removing folder {0}'.format(directory)) + try: + shutil.rmtree(directory) + except FileNotFoundError: + print_error('{0} does not exist.'.format(directory)) + +def remote_clean(machine_list, experiment_id=None): + '''clean up remote data''' + for machine in machine_list: + passwd = machine.get('passwd') + userName = machine.get('username') + host = machine.get('ip') + port = machine.get('port') + sshKeyPath = machine.get('sshKeyPath') + passphrase = machine.get('passphrase') + if experiment_id: + remote_dir = '/' + '/'.join(['tmp', 'nni-experiments', experiment_id]) + else: + remote_dir = '/' + '/'.join(['tmp', 'nni-experiments']) + sftp = create_ssh_sftp_client(host, port, userName, passwd, sshKeyPath, passphrase) + print_normal('removing folder {0}'.format(host + ':' + str(port) + remote_dir)) + remove_remote_directory(sftp, remote_dir) + +def hdfs_clean(host, user_name, output_dir, experiment_id=None): + '''clean up hdfs data''' + hdfs_client = HdfsClient(hosts='{0}:80'.format(host), user_name=user_name, webhdfs_path='/webhdfs/api/v1', timeout=5) + if experiment_id: + full_path = '/' + '/'.join([user_name, 'nni', 'experiments', experiment_id]) + else: + full_path = '/' + '/'.join([user_name, 'nni', 'experiments']) + print_normal('removing folder {0} in hdfs'.format(full_path)) + hdfs_client.delete(full_path, recursive=True) + if output_dir: + pattern = re.compile('hdfs://(?P([0-9]{1,3}.){3}[0-9]{1,3})(:[0-9]{2,5})?(?P/.*)?') + match_result = pattern.match(output_dir) + if match_result: + output_host = match_result.group('host') + output_dir = match_result.group('baseDir') + #check if the host is valid + if output_host != host: + print_warning('The host in {0} is not consistent with {1}'.format(output_dir, host)) + else: + if experiment_id: + output_dir = output_dir + '/' + experiment_id + print_normal('removing folder {0} in hdfs'.format(output_dir)) + hdfs_client.delete(output_dir, recursive=True) + +def experiment_clean(args): + '''clean up the experiment data''' + experiment_id_list = [] + experiment_config = Experiments() + experiment_dict = experiment_config.get_all_experiments() + if args.all: + experiment_id_list = list(experiment_dict.keys()) + else: + if args.id is None: + print_error('please set experiment id.') + exit(1) + if args.id not in experiment_dict: + print_error('Cannot find experiment {0}.'.format(args.id)) + exit(1) + experiment_id_list.append(args.id) + while True: + print('INFO: This action will delete experiment {0}, and it\'s not recoverable.'.format(' '.join(experiment_id_list))) + inputs = input('INFO: do you want to continue?[y/N]:') + if not inputs.lower() or inputs.lower() in ['n', 'no']: + exit(0) + elif inputs.lower() not in ['y', 'n', 'yes', 'no']: + print_warning('please input Y or N.') + else: + break + for experiment_id in experiment_id_list: + nni_config = Config(experiment_dict[experiment_id]['fileName']) + platform = nni_config.get_config('experimentConfig').get('trainingServicePlatform') + experiment_id = nni_config.get_config('experimentId') + if platform == 'remote': + machine_list = nni_config.get_config('experimentConfig').get('machineList') + remote_clean(machine_list, experiment_id) + elif platform == 'pai': + host = nni_config.get_config('experimentConfig').get('paiConfig').get('host') + user_name = nni_config.get_config('experimentConfig').get('paiConfig').get('userName') + output_dir = nni_config.get_config('experimentConfig').get('trial').get('outputDir') + hdfs_clean(host, user_name, output_dir, experiment_id) + elif platform != 'local': + #TODO: support all platforms + print_warning('platform {0} clean up not supported yet.'.format(platform)) + exit(0) + #clean local data + local_base_dir = nni_config.get_config('experimentConfig').get('logDir') + if not local_base_dir: + local_base_dir = NNI_HOME_DIR + local_experiment_dir = os.path.join(local_base_dir, experiment_id) + experiment_folder_name_list = ['checkpoint', 'db', 'log', 'trials'] + for folder_name in experiment_folder_name_list: + local_clean(os.path.join(local_experiment_dir, folder_name)) + if not os.listdir(local_experiment_dir): + local_clean(local_experiment_dir) + experiment_config = Experiments() + print_normal('removing metadata of experiment {0}'.format(experiment_id)) + experiment_config.remove_experiment(experiment_id) + print_normal('Done.') + +def get_platform_dir(config_content): + '''get the dir list to be deleted''' + platform = config_content.get('trainingServicePlatform') + dir_list = [] + if platform == 'remote': + machine_list = config_content.get('machineList') + for machine in machine_list: + host = machine.get('ip') + port = machine.get('port') + dir_list.append(host + ':' + str(port) + '/tmp/nni') + elif platform == 'pai': + host = config_content.get('paiConfig').get('host') + user_name = config_content.get('paiConfig').get('userName') + output_dir = config_content.get('trial').get('outputDir') + dir_list.append('server: {0}, path: {1}/nni'.format(host, user_name)) + if output_dir: + dir_list.append(output_dir) + return dir_list + +def platform_clean(args): + '''clean up the experiment data''' + config_path = os.path.abspath(args.config) + if not os.path.exists(config_path): + print_error('Please set correct config path.') + exit(1) + config_content = get_yml_content(config_path) + platform = config_content.get('trainingServicePlatform') + if platform == 'local': + print_normal('it doesn’t need to clean local platform.') + exit(0) + if platform not in ['remote', 'pai']: + print_normal('platform {0} not supported.'.format(platform)) + exit(0) + update_experiment() + dir_list = get_platform_dir(config_content) + if not dir_list: + print_normal('No folder of NNI caches is found.') + exit(1) + while True: + print_normal('This command will remove below folders of NNI caches. If other users are using experiments' \ + ' on below hosts, it will be broken.') + for value in dir_list: + print(' ' + value) + inputs = input('INFO: do you want to continue?[y/N]:') + if not inputs.lower() or inputs.lower() in ['n', 'no']: + exit(0) + elif inputs.lower() not in ['y', 'n', 'yes', 'no']: + print_warning('please input Y or N.') + else: + break + if platform == 'remote': + machine_list = config_content.get('machineList') + remote_clean(machine_list, None) + elif platform == 'pai': + host = config_content.get('paiConfig').get('host') + user_name = config_content.get('paiConfig').get('userName') + output_dir = config_content.get('trial').get('outputDir') + hdfs_clean(host, user_name, output_dir, None) + print_normal('Done.') + +def experiment_list(args): + '''get the information of all experiments''' + update_experiment() + experiment_config = Experiments() + experiment_dict = experiment_config.get_all_experiments() + if not experiment_dict: + print_normal('Cannot find experiments.') + exit(1) + experiment_id_list = [] + if args.all: + for key in experiment_dict.keys(): + experiment_id_list.append(key) + else: + for key in experiment_dict.keys(): + if experiment_dict[key]['status'] != 'STOPPED': + experiment_id_list.append(key) + if not experiment_id_list: + print_warning('There is no experiment running...\nYou can use \'nnictl experiment list --all\' to list all experiments.') + experiment_information = "" + for key in experiment_id_list: + experiment_information += EXPERIMENT_DETAIL_FORMAT % (key, + experiment_dict[key].get('experimentName', 'N/A'), + experiment_dict[key]['status'], + experiment_dict[key]['port'], + experiment_dict[key].get('platform'), + experiment_dict[key]['startTime'], + experiment_dict[key]['endTime']) + print(EXPERIMENT_INFORMATION_FORMAT % experiment_information) + return experiment_id_list + +def get_time_interval(time1, time2): + '''get the interval of two times''' + try: + #convert time to timestamp + time1 = time.mktime(time.strptime(time1, '%Y/%m/%d %H:%M:%S')) + time2 = time.mktime(time.strptime(time2, '%Y/%m/%d %H:%M:%S')) + seconds = (datetime.fromtimestamp(time2) - datetime.fromtimestamp(time1)).seconds + #convert seconds to day:hour:minute:second + days = seconds / 86400 + seconds %= 86400 + hours = seconds / 3600 + seconds %= 3600 + minutes = seconds / 60 + seconds %= 60 + return '%dd %dh %dm %ds' % (days, hours, minutes, seconds) + except: + return 'N/A' + +def show_experiment_info(): + '''show experiment information in monitor''' + update_experiment() + experiment_config = Experiments() + experiment_dict = experiment_config.get_all_experiments() + if not experiment_dict: + print('There is no experiment running...') + exit(1) + experiment_id_list = [] + for key in experiment_dict.keys(): + if experiment_dict[key]['status'] != 'STOPPED': + experiment_id_list.append(key) + if not experiment_id_list: + print_warning('There is no experiment running...') + return + for key in experiment_id_list: + print(EXPERIMENT_MONITOR_INFO % (key, experiment_dict[key]['status'], experiment_dict[key]['port'], \ + experiment_dict[key].get('platform'), experiment_dict[key]['startTime'], \ + get_time_interval(experiment_dict[key]['startTime'], experiment_dict[key]['endTime']))) + print(TRIAL_MONITOR_HEAD) + running, response = check_rest_server_quick(experiment_dict[key]['port']) + if running: + response = rest_get(trial_jobs_url(experiment_dict[key]['port']), REST_TIME_OUT) + if response and check_response(response): + content = json.loads(response.text) + for index, value in enumerate(content): + content[index] = convert_time_stamp_to_date(value) + print(TRIAL_MONITOR_CONTENT % (content[index].get('id'), content[index].get('startTime'), \ + content[index].get('endTime'), content[index].get('status'))) + print(TRIAL_MONITOR_TAIL) + +def set_monitor(auto_exit, time_interval, port=None, pid=None): + '''set the experiment monitor engine''' + while True: + try: + if sys.platform == 'win32': + os.system('cls') + else: + os.system('clear') + update_experiment() + show_experiment_info() + if auto_exit: + status = get_experiment_status(port) + if status in ['DONE', 'ERROR', 'STOPPED']: + print_normal('Experiment status is {0}.'.format(status)) + print_normal('Stopping experiment...') + kill_command(pid) + print_normal('Stop experiment success.') + exit(0) + time.sleep(time_interval) + except KeyboardInterrupt: + if auto_exit: + print_normal('Stopping experiment...') + kill_command(pid) + print_normal('Stop experiment success.') + else: + print_normal('Exiting...') + exit(0) + except Exception as exception: + print_error(exception) + exit(1) + +def monitor_experiment(args): + '''monitor the experiment''' + if args.time <= 0: + print_error('please input a positive integer as time interval, the unit is second.') + exit(1) + set_monitor(False, args.time) + +def export_trials_data(args): + '''export experiment metadata and intermediate results to json or csv + ''' + def groupby_trial_id(intermediate_results): + sorted(intermediate_results, key=lambda x: x['timestamp']) + groupby = dict() + for content in intermediate_results: + groupby.setdefault(content['trialJobId'], []).append(json.loads(content['data'])) + return groupby + + nni_config = Config(get_config_filename(args)) + rest_port = nni_config.get_config('restServerPort') + rest_pid = nni_config.get_config('restServerPid') + + if not detect_process(rest_pid): + print_error('Experiment is not running...') + return + running, response = check_rest_server_quick(rest_port) + if not running: + print_error('Restful server is not running') + return + response = rest_get(export_data_url(rest_port), 20) + if response is not None and check_response(response): + content = json.loads(response.text) + if args.intermediate: + intermediate_results_response = rest_get(metric_data_url(rest_port), REST_TIME_OUT) + if not intermediate_results_response or not check_response(intermediate_results_response): + print_error('Error getting intermediate results.') + return + intermediate_results = groupby_trial_id(json.loads(intermediate_results_response.text)) + for record in content: + record['intermediate'] = intermediate_results[record['id']] + if args.type == 'json': + with open(args.path, 'w') as file: + file.write(json.dumps(content)) + elif args.type == 'csv': + trial_records = [] + for record in content: + formated_record = dict() + if args.intermediate: + formated_record['intermediate'] = '[' + ','.join(record['intermediate']) + ']' + record_value = json.loads(record['value']) + if not isinstance(record_value, (float, int)): + formated_record.update({**record['parameter'], **record_value, **{'id': record['id']}}) + else: + formated_record.update({**record['parameter'], **{'reward': record_value, 'id': record['id']}}) + trial_records.append(formated_record) + if not trial_records: + print_error('No trial results collected! Please check your trial log...') + exit(0) + with open(args.path, 'w', newline='') as file: + writer = csv.DictWriter(file, set.union(*[set(r.keys()) for r in trial_records])) + writer.writeheader() + writer.writerows(trial_records) + else: + print_error('Unknown type: %s' % args.type) + return + else: + print_error('Export failed...') + +def search_space_auto_gen(args): + '''dry run trial code to generate search space file''' + trial_dir = os.path.expanduser(args.trial_dir) + file_path = os.path.expanduser(args.file) + if not os.path.isabs(file_path): + file_path = os.path.join(os.getcwd(), file_path) + assert os.path.exists(trial_dir) + if os.path.exists(file_path): + print_warning('%s already exists, will be overwritten.' % file_path) + print_normal('Dry run to generate search space...') + Popen(args.trial_command, cwd=trial_dir, env=dict(os.environ, NNI_GEN_SEARCH_SPACE=file_path), shell=True).wait() + if not os.path.exists(file_path): + print_warning('Expected search space file \'{}\' generated, but not found.'.format(file_path)) + else: + print_normal('Generate search space done: \'{}\'.'.format(file_path)) + +def save_experiment(args): + '''save experiment data to a zip file''' + experiment_config = Experiments() + experiment_dict = experiment_config.get_all_experiments() + if args.id is None: + print_error('Please set experiment id.') + exit(1) + if args.id not in experiment_dict: + print_error('Cannot find experiment {0}.'.format(args.id)) + exit(1) + if experiment_dict[args.id].get('status') != 'STOPPED': + print_error('Can only save stopped experiment!') + exit(1) + print_normal('Saving...') + nni_config = Config(experiment_dict[args.id]['fileName']) + logDir = os.path.join(NNI_HOME_DIR, args.id) + if nni_config.get_config('logDir'): + logDir = os.path.join(nni_config.get_config('logDir'), args.id) + temp_root_dir = generate_temp_dir() + + # Step1. Copy logDir to temp folder + if not os.path.exists(logDir): + print_error('logDir: %s does not exist!' % logDir) + exit(1) + temp_experiment_dir = os.path.join(temp_root_dir, 'experiment') + shutil.copytree(logDir, temp_experiment_dir) + + # Step2. Copy nnictl metadata to temp folder + temp_nnictl_dir = os.path.join(temp_root_dir, 'nnictl') + os.makedirs(temp_nnictl_dir, exist_ok=True) + try: + with open(os.path.join(temp_nnictl_dir, '.experiment'), 'w') as file: + experiment_dict[args.id]['id'] = args.id + json.dump(experiment_dict[args.id], file) + except IOError: + print_error('Write file to %s failed!' % os.path.join(temp_nnictl_dir, '.experiment')) + exit(1) + nnictl_config_dir = os.path.join(NNICTL_HOME_DIR, experiment_dict[args.id]['fileName']) + shutil.copytree(nnictl_config_dir, os.path.join(temp_nnictl_dir, experiment_dict[args.id]['fileName'])) + + # Step3. Copy code dir + if args.saveCodeDir: + temp_code_dir = os.path.join(temp_root_dir, 'code') + shutil.copytree(nni_config.get_config('experimentConfig')['trial']['codeDir'], temp_code_dir) + + # Step4. Copy searchSpace file + search_space_path = nni_config.get_config('experimentConfig').get('searchSpacePath') + if search_space_path: + if not os.path.exists(search_space_path): + print_warning('search space %s does not exist!' % search_space_path) + else: + temp_search_space_dir = os.path.join(temp_root_dir, 'searchSpace') + os.makedirs(temp_search_space_dir, exist_ok=True) + search_space_name = os.path.basename(search_space_path) + shutil.copyfile(search_space_path, os.path.join(temp_search_space_dir, search_space_name)) + + # Step5. Archive folder + zip_package_name = 'nni_experiment_%s' % args.id + if args.path: + os.makedirs(args.path, exist_ok=True) + zip_package_name = os.path.join(args.path, zip_package_name) + shutil.make_archive(zip_package_name, 'zip', temp_root_dir) + print_normal('Save to %s.zip success!' % zip_package_name) + + # Step5. Cleanup temp data + shutil.rmtree(temp_root_dir) + +def load_experiment(args): + '''load experiment data''' + package_path = os.path.expanduser(args.path) + if not os.path.exists(args.path): + print_error('file path %s does not exist!' % args.path) + exit(1) + if args.searchSpacePath and os.path.isdir(args.searchSpacePath): + print_error('search space path should be a full path with filename, not a directory!') + exit(1) + temp_root_dir = generate_temp_dir() + shutil.unpack_archive(package_path, temp_root_dir) + print_normal('Loading...') + # Step1. Validation + if not os.path.exists(args.codeDir): + print_error('Invalid: codeDir path does not exist!') + exit(1) + if args.logDir: + if not os.path.exists(args.logDir): + print_error('Invalid: logDir path does not exist!') + exit(1) + experiment_temp_dir = os.path.join(temp_root_dir, 'experiment') + if not os.path.exists(os.path.join(experiment_temp_dir, 'db')): + print_error('Invalid archive file: db file does not exist!') + shutil.rmtree(temp_root_dir) + exit(1) + nnictl_temp_dir = os.path.join(temp_root_dir, 'nnictl') + if not os.path.exists(os.path.join(nnictl_temp_dir, '.experiment')): + print_error('Invalid archive file: nnictl metadata file does not exist!') + shutil.rmtree(temp_root_dir) + exit(1) + try: + with open(os.path.join(nnictl_temp_dir, '.experiment'), 'r') as file: + experiment_metadata = json.load(file) + except ValueError as err: + print_error('Invalid nnictl metadata file: %s' % err) + shutil.rmtree(temp_root_dir) + exit(1) + experiment_config = Experiments() + experiment_dict = experiment_config.get_all_experiments() + experiment_id = experiment_metadata.get('id') + if experiment_id in experiment_dict: + print_error('Invalid: experiment id already exist!') + shutil.rmtree(temp_root_dir) + exit(1) + if not os.path.exists(os.path.join(nnictl_temp_dir, experiment_metadata.get('fileName'))): + print_error('Invalid: experiment metadata does not exist!') + shutil.rmtree(temp_root_dir) + exit(1) + + # Step2. Copy nnictl metadata + src_path = os.path.join(nnictl_temp_dir, experiment_metadata.get('fileName')) + dest_path = os.path.join(NNICTL_HOME_DIR, experiment_metadata.get('fileName')) + if os.path.exists(dest_path): + shutil.rmtree(dest_path) + shutil.copytree(src_path, dest_path) + + # Step3. Copy experiment data + nni_config = Config(experiment_metadata.get('fileName')) + nnictl_exp_config = nni_config.get_config('experimentConfig') + if args.logDir: + logDir = args.logDir + nnictl_exp_config['logDir'] = logDir + else: + if nnictl_exp_config.get('logDir'): + logDir = nnictl_exp_config['logDir'] + else: + logDir = NNI_HOME_DIR + os.rename(os.path.join(temp_root_dir, 'experiment'), os.path.join(temp_root_dir, experiment_id)) + src_path = os.path.join(os.path.join(temp_root_dir, experiment_id)) + dest_path = os.path.join(os.path.join(logDir, experiment_id)) + if os.path.exists(dest_path): + shutil.rmtree(dest_path) + shutil.copytree(src_path, dest_path) + + # Step4. Copy code dir + codeDir = os.path.expanduser(args.codeDir) + if not os.path.isabs(codeDir): + codeDir = os.path.join(os.getcwd(), codeDir) + print_normal('Expand codeDir to %s' % codeDir) + nnictl_exp_config['trial']['codeDir'] = codeDir + archive_code_dir = os.path.join(temp_root_dir, 'code') + if os.path.exists(archive_code_dir): + file_list = os.listdir(archive_code_dir) + for file_name in file_list: + src_path = os.path.join(archive_code_dir, file_name) + target_path = os.path.join(codeDir, file_name) + if os.path.exists(target_path): + print_error('Copy %s failed, %s exist!' % (file_name, target_path)) + continue + if os.path.isdir(src_path): + shutil.copytree(src_path, target_path) + else: + shutil.copy(src_path, target_path) + + # Step5. Copy searchSpace file + archive_search_space_dir = os.path.join(temp_root_dir, 'searchSpace') + if args.searchSpacePath: + target_path = os.path.expanduser(args.searchSpacePath) + else: + # set default path to codeDir + target_path = os.path.join(codeDir, 'search_space.json') + if not os.path.isabs(target_path): + target_path = os.path.join(os.getcwd(), target_path) + print_normal('Expand search space path to %s' % target_path) + nnictl_exp_config['searchSpacePath'] = target_path + # if the path already has a search space file, use the original one, otherwise use archived one + if not os.path.isfile(target_path): + if len(os.listdir(archive_search_space_dir)) == 0: + print_error('Archive file does not contain search space file!') + exit(1) + else: + for file in os.listdir(archive_search_space_dir): + source_path = os.path.join(archive_search_space_dir, file) + os.makedirs(os.path.dirname(target_path), exist_ok=True) + shutil.copyfile(source_path, target_path) + break + elif not args.searchSpacePath: + print_warning('%s exist, will not load search_space file' % target_path) + + # Step6. Create experiment metadata + nni_config.set_config('experimentConfig', nnictl_exp_config) + experiment_config.add_experiment(experiment_id, + experiment_metadata.get('port'), + experiment_metadata.get('startTime'), + experiment_metadata.get('fileName'), + experiment_metadata.get('platform'), + experiment_metadata.get('experimentName'), + experiment_metadata.get('endTime'), + experiment_metadata.get('status')) + print_normal('Load experiment %s succsss!' % experiment_id) + + # Step6. Cleanup temp data + shutil.rmtree(temp_root_dir) + diff --git a/new_impl/cv/third_party/nni/tools/nnictl/package_management.py b/new_impl/cv/third_party/nni/tools/nnictl/package_management.py new file mode 100644 index 0000000000000000000000000000000000000000..5eef3407522839b477e19b8fdadadcac19110e1c --- /dev/null +++ b/new_impl/cv/third_party/nni/tools/nnictl/package_management.py @@ -0,0 +1,184 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +from collections import defaultdict +import json +import pkginfo +import nni +from nni.tools.package_utils import read_installed_package_meta, get_installed_package_meta, \ + write_package_meta, get_builtin_algo_meta, get_not_installable_builtin_names, ALGO_TYPES + +from .constants import INSTALLABLE_PACKAGE_META +from .common_utils import print_error, print_green +from .command_utils import install_requirements_command, call_pip_install, call_pip_uninstall + +PACKAGE_TYPES = ['tuner', 'assessor', 'advisor'] + +def install_by_name(package_name): + if package_name not in INSTALLABLE_PACKAGE_META: + raise RuntimeError('{} is not found in installable packages!'.format(package_name)) + + requirements_path = os.path.join(nni.__path__[0], 'algorithms/hpo', INSTALLABLE_PACKAGE_META[package_name]['code_sub_dir'], 'requirements.txt') + assert os.path.exists(requirements_path) + + return install_requirements_command(requirements_path) + +def package_install(args): + '''install packages''' + installed = False + try: + if args.name: + if install_by_name(args.name) == 0: + package_meta = {} + package_meta['type'] = INSTALLABLE_PACKAGE_META[args.name]['type'] + package_meta['name'] = args.name + package_meta['class_name'] = INSTALLABLE_PACKAGE_META[args.name]['class_name'] + package_meta['class_args_validator'] = INSTALLABLE_PACKAGE_META[args.name]['class_args_validator'] + save_package_meta_data(package_meta) + print_green('{} installed!'.format(args.name)) + installed = True + else: + package_meta = get_nni_meta(args.source) + if package_meta: + if call_pip_install(args.source) == 0: + save_package_meta_data(package_meta) + print_green('{} installed!'.format(package_meta['name'])) + installed = True + except Exception as e: + print_error(e) + if not installed: + print_error('installation failed!') + +def package_uninstall(args): + '''uninstall packages''' + name = args.name[0] + if name in get_not_installable_builtin_names(): + print_error('{} can not be uninstalled!'.format(name)) + exit(1) + meta = get_installed_package_meta(None, name) + if meta is None: + print_error('package {} not found!'.format(name)) + return + if 'installed_package' in meta: + call_pip_uninstall(meta['installed_package']) + if remove_package_meta_data(name): + print_green('{} uninstalled sucessfully!'.format(name)) + else: + print_error('Failed to uninstall {}!'.format(name)) + +def package_show(args): + '''show specified packages''' + builtin_name = args.name[0] + meta = get_builtin_algo_meta(builtin_name=builtin_name) + if meta: + print(json.dumps(meta, indent=4)) + else: + print_error('package {} not found'.format(builtin_name)) + +def print_package_list(meta): + print('+-----------------+------------+-----------+--------=-------------+------------------------------------------+') + print('| Name | Type | Installed | Class Name | Module Name |') + print('+-----------------+------------+-----------+----------------------+------------------------------------------+') + MAX_MODULE_NAME = 38 + for t in ['tuners', 'assessors', 'advisors']: + for p in meta[t]: + module_name = '.'.join(p['class_name'].split('.')[:-1]) + if len(module_name) > MAX_MODULE_NAME: + module_name = module_name[:MAX_MODULE_NAME-3] + '...' + class_name = p['class_name'].split('.')[-1] + print('| {:15s} | {:10s} | {:9s} | {:20s} | {:40s} |'.format(p['name'], t, p['installed'], class_name, module_name[:38])) + print('+-----------------+------------+-----------+----------------------+------------------------------------------+') + +def package_list(args): + '''list all packages''' + if args.all: + meta = get_builtin_algo_meta() + else: + meta = read_installed_package_meta() + + installed_names = defaultdict(list) + for t in ['tuners', 'assessors', 'advisors']: + for p in meta[t]: + p['installed'] = 'Yes' + installed_names[t].append(p['name']) + for k, v in INSTALLABLE_PACKAGE_META.items(): + t = v['type']+'s' + if k not in installed_names[t]: + meta[t].append({ + 'name': k, + 'class_name': v['class_name'], + 'class_args_validator': v['class_args_validator'], + 'installed': 'No' + }) + + print_package_list(meta) + +def save_package_meta_data(meta_data): + assert meta_data['type'] in PACKAGE_TYPES + assert 'name' in meta_data + assert 'class_name' in meta_data + + config = read_installed_package_meta() + + if meta_data['name'] in [x['name'] for x in config[meta_data['type']+'s']]: + raise ValueError('name %s already installed' % meta_data['name']) + + package_meta = {k: meta_data[k] for k in ['name', 'class_name', 'class_args_validator'] if k in meta_data} + if 'package_name' in meta_data: + package_meta['installed_package'] = meta_data['package_name'] + config[meta_data['type']+'s'].append(package_meta) + write_package_meta(config) + +def remove_package_meta_data(name): + config = read_installed_package_meta() + + updated = False + for t in ALGO_TYPES: + for meta in config[t]: + if meta['name'] == name: + config[t].remove(meta) + updated = True + if updated: + write_package_meta(config) + return True + return False + +def get_nni_meta(source): + if not os.path.exists(source): + print_error('{} does not exist'.format(source)) + return None + + if os.path.isdir(source): + if not os.path.exists(os.path.join(source, 'setup.py')): + print_error('setup.py not found') + return None + pkg = pkginfo.Develop(source) + else: + if not source.endswith('.whl'): + print_error('File name {} must ends with \'.whl\''.format(source)) + return False + pkg = pkginfo.Wheel(source) + + classifiers = pkg.classifiers + meta = parse_classifiers(classifiers) + meta['package_name'] = pkg.name + return meta + +def parse_classifiers(classifiers): + parts = [] + for c in classifiers: + if c.startswith('NNI Package'): + parts = [x.strip() for x in c.split('::')] + break + if len(parts) < 4 or not all(parts): + raise ValueError('Can not find correct NNI meta data in package classifiers.') + meta = { + 'type': parts[1], + 'name': parts[2], + 'class_name': parts[3] + } + if len(parts) >= 5: + meta['class_args_validator'] = parts[4] + + return meta diff --git a/new_impl/cv/third_party/nni/tools/nnictl/rest_utils.py b/new_impl/cv/third_party/nni/tools/nnictl/rest_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..e98c9a839245ca0775406302dd6bf40bf8a2ae35 --- /dev/null +++ b/new_impl/cv/third_party/nni/tools/nnictl/rest_utils.py @@ -0,0 +1,77 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import time +import requests +from .url_utils import check_status_url +from .constants import REST_TIME_OUT +from .common_utils import print_error + +def rest_put(url, data, timeout, show_error=False): + '''Call rest put method''' + try: + response = requests.put(url, headers={'Accept': 'application/json', 'Content-Type': 'application/json'},\ + data=data, timeout=timeout) + return response + except Exception as exception: + if show_error: + print_error(exception) + return None + +def rest_post(url, data, timeout, show_error=False): + '''Call rest post method''' + try: + response = requests.post(url, headers={'Accept': 'application/json', 'Content-Type': 'application/json'},\ + data=data, timeout=timeout) + return response + except Exception as exception: + if show_error: + print_error(exception) + return None + +def rest_get(url, timeout, show_error=False): + '''Call rest get method''' + try: + response = requests.get(url, timeout=timeout) + return response + except Exception as exception: + if show_error: + print_error(exception) + return None + +def rest_delete(url, timeout, show_error=False): + '''Call rest delete method''' + try: + response = requests.delete(url, timeout=timeout) + return response + except Exception as exception: + if show_error: + print_error(exception) + return None + +def check_rest_server(rest_port): + '''Check if restful server is ready''' + retry_count = 20 + for _ in range(retry_count): + response = rest_get(check_status_url(rest_port), REST_TIME_OUT) + if response: + if response.status_code == 200: + return True, response + else: + return False, response + else: + time.sleep(1) + return False, response + +def check_rest_server_quick(rest_port): + '''Check if restful server is ready, only check once''' + response = rest_get(check_status_url(rest_port), 5) + if response and response.status_code == 200: + return True, response + return False, None + +def check_response(response): + '''Check if a response is success according to status_code''' + if response and response.status_code == 200: + return True + return False diff --git a/new_impl/cv/third_party/nni/tools/nnictl/ssh_utils.py b/new_impl/cv/third_party/nni/tools/nnictl/ssh_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..e3f26a8e24c1be67bf86cfa0192481b245c93f97 --- /dev/null +++ b/new_impl/cv/third_party/nni/tools/nnictl/ssh_utils.py @@ -0,0 +1,60 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +from .common_utils import print_error +from .command_utils import install_package_command + +def check_environment(): + '''check if paramiko is installed''' + try: + import paramiko + except: + install_package_command('paramiko') + import paramiko + return paramiko + +def copy_remote_directory_to_local(sftp, remote_path, local_path): + '''copy remote directory to local machine''' + try: + os.makedirs(local_path, exist_ok=True) + files = sftp.listdir(remote_path) + for file in files: + remote_full_path = os.path.join(remote_path, file) + local_full_path = os.path.join(local_path, file) + try: + if sftp.listdir(remote_full_path): + copy_remote_directory_to_local(sftp, remote_full_path, local_full_path) + except: + sftp.get(remote_full_path, local_full_path) + except Exception: + pass + +def create_ssh_sftp_client(host_ip, port, username, password, ssh_key_path, passphrase): + '''create ssh client''' + try: + paramiko = check_environment() + conn = paramiko.Transport(host_ip, port) + if ssh_key_path is not None: + ssh_key = paramiko.RSAKey.from_private_key_file(ssh_key_path, password=passphrase) + conn.connect(username=username, pkey=ssh_key) + else: + conn.connect(username=username, password=password) + sftp = paramiko.SFTPClient.from_transport(conn) + return sftp + except Exception as exception: + print_error('Create ssh client error %s\n' % exception) + +def remove_remote_directory(sftp, directory): + '''remove a directory in remote machine''' + try: + files = sftp.listdir(directory) + for file in files: + filepath = '/'.join([directory, file]) + try: + sftp.remove(filepath) + except IOError: + remove_remote_directory(sftp, filepath) + sftp.rmdir(directory) + except IOError as err: + print_error(err) diff --git a/new_impl/cv/third_party/nni/tools/nnictl/tensorboard_utils.py b/new_impl/cv/third_party/nni/tools/nnictl/tensorboard_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..adf85f3a741588e8c82bd23bc0b89f2108e59d51 --- /dev/null +++ b/new_impl/cv/third_party/nni/tools/nnictl/tensorboard_utils.py @@ -0,0 +1,147 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +import json +import re +import tempfile +from subprocess import call, Popen +from .rest_utils import rest_get, check_rest_server_quick, check_response +from .config_utils import Config, Experiments +from .url_utils import trial_jobs_url, get_local_urls +from .constants import REST_TIME_OUT +from .common_utils import print_normal, print_error, print_green, detect_process, detect_port, check_tensorboard_version +from .nnictl_utils import check_experiment_id, check_experiment_id +from .ssh_utils import create_ssh_sftp_client, copy_remote_directory_to_local + +def parse_log_path(args, trial_content): + '''parse log path''' + path_list = [] + host_list = [] + for trial in trial_content: + if args.trial_id and args.trial_id != 'all' and trial.get('id') != args.trial_id: + continue + pattern = r'(?P.+)://(?P.+):(?P.*)' + match = re.search(pattern, trial['logPath']) + if match: + path_list.append(match.group('path')) + host_list.append(match.group('host')) + if not path_list: + print_error('Trial id %s error!' % args.trial_id) + exit(1) + return path_list, host_list + +def copy_data_from_remote(args, nni_config, trial_content, path_list, host_list, temp_nni_path): + '''use ssh client to copy data from remote machine to local machien''' + machine_list = nni_config.get_config('experimentConfig').get('machineList') + machine_dict = {} + local_path_list = [] + for machine in machine_list: + machine_dict[machine['ip']] = {'port': machine['port'], 'passwd': machine['passwd'], 'username': machine['username'], + 'sshKeyPath': machine.get('sshKeyPath'), 'passphrase': machine.get('passphrase')} + for index, host in enumerate(host_list): + local_path = os.path.join(temp_nni_path, trial_content[index].get('id')) + local_path_list.append(local_path) + print_normal('Copying log data from %s to %s' % (host + ':' + path_list[index], local_path)) + sftp = create_ssh_sftp_client(host, machine_dict[host]['port'], machine_dict[host]['username'], machine_dict[host]['passwd'], + machine_dict[host]['sshKeyPath'], machine_dict[host]['passphrase']) + copy_remote_directory_to_local(sftp, path_list[index], local_path) + print_normal('Copy done!') + return local_path_list + +def get_path_list(args, nni_config, trial_content, temp_nni_path): + '''get path list according to different platform''' + path_list, host_list = parse_log_path(args, trial_content) + platform = nni_config.get_config('experimentConfig').get('trainingServicePlatform') + if platform == 'local': + print_normal('Log path: %s' % ' '.join(path_list)) + return path_list + elif platform == 'remote': + path_list = copy_data_from_remote(args, nni_config, trial_content, path_list, host_list, temp_nni_path) + print_normal('Log path: %s' % ' '.join(path_list)) + return path_list + else: + print_error('Not supported platform!') + exit(1) + +def format_tensorboard_log_path(path_list): + new_path_list = [] + for index, value in enumerate(path_list): + new_path_list.append('name%d:%s' % (index + 1, value)) + return ','.join(new_path_list) + +def start_tensorboard_process(args, nni_config, path_list, temp_nni_path): + '''call cmds to start tensorboard process in local machine''' + if detect_port(args.port): + print_error('Port %s is used by another process, please reset port!' % str(args.port)) + exit(1) + with open(os.path.join(temp_nni_path, 'tensorboard_stdout'), 'a+') as stdout_file, \ + open(os.path.join(temp_nni_path, 'tensorboard_stderr'), 'a+') as stderr_file: + log_dir_cmd = '--logdir_spec' if check_tensorboard_version() >= '2.0' else '--logdir' + cmds = ['tensorboard', log_dir_cmd, format_tensorboard_log_path(path_list), '--port', str(args.port)] + tensorboard_process = Popen(cmds, stdout=stdout_file, stderr=stderr_file) + url_list = get_local_urls(args.port) + print_green('Start tensorboard success!') + print_normal('Tensorboard urls: ' + ' '.join(url_list)) + tensorboard_process_pid_list = nni_config.get_config('tensorboardPidList') + if tensorboard_process_pid_list is None: + tensorboard_process_pid_list = [tensorboard_process.pid] + else: + tensorboard_process_pid_list.append(tensorboard_process.pid) + nni_config.set_config('tensorboardPidList', tensorboard_process_pid_list) + +def stop_tensorboard(args): + '''stop tensorboard''' + experiment_id = check_experiment_id(args) + experiment_config = Experiments() + experiment_dict = experiment_config.get_all_experiments() + config_file_name = experiment_dict[experiment_id]['fileName'] + nni_config = Config(config_file_name) + tensorboard_pid_list = nni_config.get_config('tensorboardPidList') + if tensorboard_pid_list: + for tensorboard_pid in tensorboard_pid_list: + try: + cmds = ['kill', '-9', str(tensorboard_pid)] + call(cmds) + except Exception as exception: + print_error(exception) + nni_config.set_config('tensorboardPidList', []) + print_normal('Stop tensorboard success!') + else: + print_error('No tensorboard configuration!') + + +def start_tensorboard(args): + '''start tensorboard''' + experiment_id = check_experiment_id(args) + experiment_config = Experiments() + experiment_dict = experiment_config.get_all_experiments() + config_file_name = experiment_dict[experiment_id]['fileName'] + nni_config = Config(config_file_name) + rest_port = nni_config.get_config('restServerPort') + rest_pid = nni_config.get_config('restServerPid') + if not detect_process(rest_pid): + print_error('Experiment is not running...') + return + running, response = check_rest_server_quick(rest_port) + trial_content = None + if running: + response = rest_get(trial_jobs_url(rest_port), REST_TIME_OUT) + if response and check_response(response): + trial_content = json.loads(response.text) + else: + print_error('List trial failed...') + else: + print_error('Restful server is not running...') + if not trial_content: + print_error('No trial information!') + exit(1) + if len(trial_content) > 1 and not args.trial_id: + print_error('There are multiple trials, please set trial id!') + exit(1) + experiment_id = nni_config.get_config('experimentId') + temp_nni_path = os.path.join(tempfile.gettempdir(), 'nni', experiment_id) + os.makedirs(temp_nni_path, exist_ok=True) + + path_list = get_path_list(args, nni_config, trial_content, temp_nni_path) + start_tensorboard_process(args, nni_config, path_list, temp_nni_path) diff --git a/new_impl/cv/third_party/nni/tools/nnictl/updater.py b/new_impl/cv/third_party/nni/tools/nnictl/updater.py new file mode 100644 index 0000000000000000000000000000000000000000..071a167f70084a5007d817427ec139ca2cd54343 --- /dev/null +++ b/new_impl/cv/third_party/nni/tools/nnictl/updater.py @@ -0,0 +1,148 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import json +import os +from .rest_utils import rest_put, rest_post, rest_get, check_rest_server_quick, check_response +from .url_utils import experiment_url, import_data_url +from .config_utils import Config +from .common_utils import get_json_content, print_normal, print_error, print_warning +from .nnictl_utils import get_experiment_port, get_config_filename, detect_process +from .launcher_utils import parse_time +from .constants import REST_TIME_OUT, TUNERS_SUPPORTING_IMPORT_DATA, TUNERS_NO_NEED_TO_IMPORT_DATA + +def validate_digit(value, start, end): + '''validate if a digit is valid''' + if not str(value).isdigit() or int(value) < start or int(value) > end: + raise ValueError('value (%s) must be a digit from %s to %s' % (value, start, end)) + +def validate_file(path): + '''validate if a file exist''' + if not os.path.exists(path): + raise FileNotFoundError('%s is not a valid file path' % path) + +def validate_dispatcher(args): + '''validate if the dispatcher of the experiment supports importing data''' + nni_config = Config(get_config_filename(args)).get_config('experimentConfig') + if nni_config.get('tuner') and nni_config['tuner'].get('builtinTunerName'): + dispatcher_name = nni_config['tuner']['builtinTunerName'] + elif nni_config.get('advisor') and nni_config['advisor'].get('builtinAdvisorName'): + dispatcher_name = nni_config['advisor']['builtinAdvisorName'] + else: # otherwise it should be a customized one + return + if dispatcher_name not in TUNERS_SUPPORTING_IMPORT_DATA: + if dispatcher_name in TUNERS_NO_NEED_TO_IMPORT_DATA: + print_warning("There is no need to import data for %s" % dispatcher_name) + exit(0) + else: + print_error("%s does not support importing addtional data" % dispatcher_name) + exit(1) + +def load_search_space(path): + '''load search space content''' + content = json.dumps(get_json_content(path)) + if not content: + raise ValueError('searchSpace file should not be empty') + return content + +def get_query_type(key): + '''get update query type''' + if key == 'trialConcurrency': + return '?update_type=TRIAL_CONCURRENCY' + if key == 'maxExecDuration': + return '?update_type=MAX_EXEC_DURATION' + if key == 'searchSpace': + return '?update_type=SEARCH_SPACE' + if key == 'maxTrialNum': + return '?update_type=MAX_TRIAL_NUM' + +def update_experiment_profile(args, key, value): + '''call restful server to update experiment profile''' + nni_config = Config(get_config_filename(args)) + rest_port = nni_config.get_config('restServerPort') + running, _ = check_rest_server_quick(rest_port) + if running: + response = rest_get(experiment_url(rest_port), REST_TIME_OUT) + if response and check_response(response): + experiment_profile = json.loads(response.text) + experiment_profile['params'][key] = value + response = rest_put(experiment_url(rest_port)+get_query_type(key), json.dumps(experiment_profile), REST_TIME_OUT) + if response and check_response(response): + return response + else: + print_error('Restful server is not running...') + return None + +def update_searchspace(args): + validate_file(args.filename) + content = load_search_space(args.filename) + args.port = get_experiment_port(args) + if args.port is not None: + if update_experiment_profile(args, 'searchSpace', content): + print_normal('Update %s success!' % 'searchSpace') + else: + print_error('Update %s failed!' % 'searchSpace') + + +def update_concurrency(args): + validate_digit(args.value, 1, 1000) + args.port = get_experiment_port(args) + if args.port is not None: + if update_experiment_profile(args, 'trialConcurrency', int(args.value)): + print_normal('Update %s success!' % 'concurrency') + else: + print_error('Update %s failed!' % 'concurrency') + +def update_duration(args): + #parse time, change time unit to seconds + args.value = parse_time(args.value) + args.port = get_experiment_port(args) + if args.port is not None: + if update_experiment_profile(args, 'maxExecDuration', int(args.value)): + print_normal('Update %s success!' % 'duration') + else: + print_error('Update %s failed!' % 'duration') + +def update_trialnum(args): + validate_digit(args.value, 1, 999999999) + if update_experiment_profile(args, 'maxTrialNum', int(args.value)): + print_normal('Update %s success!' % 'trialnum') + else: + print_error('Update %s failed!' % 'trialnum') + +def import_data(args): + '''import additional data to the experiment''' + validate_file(args.filename) + validate_dispatcher(args) + content = load_search_space(args.filename) + + nni_config = Config(get_config_filename(args)) + rest_port = nni_config.get_config('restServerPort') + rest_pid = nni_config.get_config('restServerPid') + if not detect_process(rest_pid): + print_error('Experiment is not running...') + return + running, _ = check_rest_server_quick(rest_port) + if not running: + print_error('Restful server is not running') + return + + args.port = rest_port + if args.port is not None: + if import_data_to_restful_server(args, content): + pass + else: + print_error('Import data failed!') + +def import_data_to_restful_server(args, content): + '''call restful server to import data to the experiment''' + nni_config = Config(get_config_filename(args)) + rest_port = nni_config.get_config('restServerPort') + running, _ = check_rest_server_quick(rest_port) + if running: + response = rest_post(import_data_url(rest_port), content, REST_TIME_OUT) + if response and check_response(response): + return response + else: + print_error('Restful server is not running...') + return None diff --git a/new_impl/cv/third_party/nni/tools/nnictl/url_utils.py b/new_impl/cv/third_party/nni/tools/nnictl/url_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..59a28837a667fb910a6aa54715bb841b352dbada --- /dev/null +++ b/new_impl/cv/third_party/nni/tools/nnictl/url_utils.py @@ -0,0 +1,78 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import socket +import psutil + +BASE_URL = 'http://localhost' + +API_ROOT_URL = '/api/v1/nni' + +EXPERIMENT_API = '/experiment' + +CLUSTER_METADATA_API = '/experiment/cluster-metadata' + +IMPORT_DATA_API = '/experiment/import-data' + +CHECK_STATUS_API = '/check-status' + +TRIAL_JOBS_API = '/trial-jobs' + +EXPORT_DATA_API = '/export-data' + +TENSORBOARD_API = '/tensorboard' + +METRIC_DATA_API = '/metric-data' + +def metric_data_url(port): + '''get metric_data url''' + return '{0}:{1}{2}{3}'.format(BASE_URL, port, API_ROOT_URL, METRIC_DATA_API) + +def check_status_url(port): + '''get check_status url''' + return '{0}:{1}{2}{3}'.format(BASE_URL, port, API_ROOT_URL, CHECK_STATUS_API) + + +def cluster_metadata_url(port): + '''get cluster_metadata_url''' + return '{0}:{1}{2}{3}'.format(BASE_URL, port, API_ROOT_URL, CLUSTER_METADATA_API) + + +def import_data_url(port): + '''get import_data_url''' + return '{0}:{1}{2}{3}'.format(BASE_URL, port, API_ROOT_URL, IMPORT_DATA_API) + + +def experiment_url(port): + '''get experiment_url''' + return '{0}:{1}{2}{3}'.format(BASE_URL, port, API_ROOT_URL, EXPERIMENT_API) + + +def trial_jobs_url(port): + '''get trial_jobs url''' + return '{0}:{1}{2}{3}'.format(BASE_URL, port, API_ROOT_URL, TRIAL_JOBS_API) + + +def trial_job_id_url(port, job_id): + '''get trial_jobs with id url''' + return '{0}:{1}{2}{3}/{4}'.format(BASE_URL, port, API_ROOT_URL, TRIAL_JOBS_API, job_id) + + +def export_data_url(port): + '''get export_data url''' + return '{0}:{1}{2}{3}'.format(BASE_URL, port, API_ROOT_URL, EXPORT_DATA_API) + + +def tensorboard_url(port): + '''get tensorboard url''' + return '{0}:{1}{2}{3}'.format(BASE_URL, port, API_ROOT_URL, TENSORBOARD_API) + + +def get_local_urls(port): + '''get urls of local machine''' + url_list = [] + for _, info in psutil.net_if_addrs().items(): + for addr in info: + if socket.AddressFamily.AF_INET == addr.family: + url_list.append('http://{}:{}'.format(addr.address, port)) + return url_list diff --git a/new_impl/cv/third_party/nni/tools/package_utils/__init__.py b/new_impl/cv/third_party/nni/tools/package_utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9a1c004053fc3935cffdfa6a634b726b4052c07e --- /dev/null +++ b/new_impl/cv/third_party/nni/tools/package_utils/__init__.py @@ -0,0 +1,321 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from collections import defaultdict +import importlib +import os +from pathlib import Path +import sys + +import ruamel.yaml as yaml + +import nni +from .constants import BuiltinAlgorithms + +ALGO_TYPES = ['tuners', 'assessors', 'advisors'] + +def get_all_builtin_names(algo_type): + """Get all valid builtin names, including: + 1. BuiltinAlgorithms which is pre-installed. + 2. User installed packages in /config/installed_packages.yml + + Parameters + ---------- + algo_type: str + can be one of 'tuners', 'assessors' or 'advisors' + + Returns: list of string + ------- + All builtin names of specified type, for example, if algo_type is 'tuners', returns + all builtin tuner names. + """ + assert algo_type in ALGO_TYPES + merged_dict = _get_merged_builtin_dict() + + builtin_names = [x['name'] for x in merged_dict[algo_type]] + return builtin_names + +def get_not_installable_builtin_names(algo_type=None): + """Get builtin names in BuiltinAlgorithms which do not need to be installed + and can be used once NNI is installed. + + Parameters + ---------- + algo_type: str | None + can be one of 'tuners', 'assessors', 'advisors' or None + + Returns: list of string + ------- + All builtin names of specified type, for example, if algo_type is 'tuners', returns + all builtin tuner names. + If algo_type is None, returns all builtin names of all types. + """ + if algo_type is None: + meta = BuiltinAlgorithms + else: + assert algo_type in ALGO_TYPES + meta = { + algo_type: BuiltinAlgorithms[algo_type] + } + names = [] + for t in ALGO_TYPES: + if t in meta: + names.extend([x['name'] for x in meta[t]]) + return names + +def get_builtin_algo_meta(algo_type=None, builtin_name=None): + """ Get meta information of builtin algorithms from: + 1. Pre-installed BuiltinAlgorithms + 2. User installed packages in /config/installed_packages.yml + + Parameters + ---------- + algo_type: str | None + can be one of 'tuners', 'assessors', 'advisors' or None + builtin_name: str | None + builtin name. + + Returns: dict | list of dict | None + ------- + If builtin_name is specified, returns meta information of speicified builtin + alogorithms, for example: + { + 'name': 'Random', + 'class_name': 'nni.hyperopt_tuner.hyperopt_tuner.HyperoptTuner', + 'class_args': { + 'algorithm_name': 'random_search' + }, + 'accept_class_args': False, + 'class_args_validator': 'nni.hyperopt_tuner.hyperopt_tuner.HyperoptClassArgsValidator' + } + If builtin_name is None, returns multiple meta information in a list. + """ + merged_dict = _get_merged_builtin_dict() + + if algo_type is None and builtin_name is None: + return merged_dict + + if algo_type: + assert algo_type in ALGO_TYPES + metas = merged_dict[algo_type] + else: + metas = merged_dict['tuners'] + merged_dict['assessors'] + merged_dict['advisors'] + if builtin_name: + for m in metas: + if m['name'] == builtin_name: + return m + else: + return metas + + return None + +def get_installed_package_meta(algo_type, builtin_name): + """ Get meta information of user installed algorithms from: + /config/installed_packages.yml + + Parameters + ---------- + algo_type: str | None + can be one of 'tuners', 'assessors', 'advisors' or None + builtin_name: str + builtin name. + + Returns: dict | None + ------- + Returns meta information of speicified builtin alogorithms, for example: + { + 'class_args_validator': 'nni.smac_tuner.smac_tuner.SMACClassArgsValidator', + 'class_name': 'nni.smac_tuner.smac_tuner.SMACTuner', + 'name': 'SMAC' + } + """ + assert builtin_name is not None + if algo_type: + assert algo_type in ALGO_TYPES + config = read_installed_package_meta() + + candidates = [] + if algo_type: + candidates = config[algo_type] + else: + for algo_type in ALGO_TYPES: + candidates.extend(config[algo_type]) + for meta in candidates: + if meta['name'] == builtin_name: + return meta + return None + +def _parse_full_class_name(full_class_name): + if not full_class_name: + return None, None + parts = full_class_name.split('.') + module_name, class_name = '.'.join(parts[:-1]), parts[-1] + return module_name, class_name + +def get_builtin_module_class_name(algo_type, builtin_name): + """Get module name and class name of all builtin algorithms + + Parameters + ---------- + algo_type: str + can be one of 'tuners', 'assessors', 'advisors' + builtin_name: str + builtin name. + + Returns: tuple + ------- + tuple of (module name, class name) + """ + assert algo_type in ALGO_TYPES + assert builtin_name is not None + meta = get_builtin_algo_meta(algo_type, builtin_name) + if not meta: + return None, None + return _parse_full_class_name(meta['class_name']) + +def create_validator_instance(algo_type, builtin_name): + """Create instance of validator class + + Parameters + ---------- + algo_type: str + can be one of 'tuners', 'assessors', 'advisors' + builtin_name: str + builtin name. + + Returns: object | None + ------- + Returns validator class instance. + If specified validator class does not exist, returns None. + """ + assert algo_type in ALGO_TYPES + assert builtin_name is not None + meta = get_builtin_algo_meta(algo_type, builtin_name) + if not meta or 'class_args_validator' not in meta: + return None + module_name, class_name = _parse_full_class_name(meta['class_args_validator']) + class_module = importlib.import_module(module_name) + class_constructor = getattr(class_module, class_name) + + return class_constructor() + +def create_builtin_class_instance(builtin_name, input_class_args, algo_type): + """Create instance of builtin algorithms + + Parameters + ---------- + builtin_name: str + builtin name. + input_class_args: dict + kwargs for builtin class constructor + algo_type: str + can be one of 'tuners', 'assessors', 'advisors' + + Returns: object + ------- + Returns builtin class instance. + """ + assert algo_type in ALGO_TYPES + if builtin_name not in get_all_builtin_names(algo_type): + raise RuntimeError('Builtin name is not found: {}'.format(builtin_name)) + + def parse_algo_meta(algo_meta, input_class_args): + """ + 1. parse class_name field in meta data into module name and class name, + for example: + parse class_name 'nni.hyperopt_tuner.hyperopt_tuner.HyperoptTuner' in meta data into: + module name: nni.hyperopt_tuner.hyperopt_tuner + class name: HyperoptTuner + 2. merge user specified class args together with builtin class args. + """ + assert algo_meta + module_name, class_name = _parse_full_class_name(algo_meta['class_name']) + + class_args = {} + if 'class_args' in algo_meta: + class_args = algo_meta['class_args'] + if input_class_args is not None: + class_args.update(input_class_args) + + return module_name, class_name, class_args + + algo_meta = get_builtin_algo_meta(algo_type, builtin_name) + module_name, class_name, class_args = parse_algo_meta(algo_meta, input_class_args) + + if importlib.util.find_spec(module_name) is None: + raise RuntimeError('Builtin module can not be loaded: {}'.format(module_name)) + + class_module = importlib.import_module(module_name) + class_constructor = getattr(class_module, class_name) + + instance = class_constructor(**class_args) + + return instance + +def create_customized_class_instance(class_params): + """Create instance of customized algorithms + + Parameters + ---------- + class_params: dict + class_params should contains following keys: + codeDir: code directory + classFileName: python file name of the class + className: class name + classArgs (optional): kwargs pass to class constructor + Returns: object + ------- + Returns customized class instance. + """ + + code_dir = class_params.get('codeDir') + class_filename = class_params.get('classFileName') + class_name = class_params.get('className') + class_args = class_params.get('classArgs') + + if not os.path.isfile(os.path.join(code_dir, class_filename)): + raise ValueError('Class file not found: {}'.format( + os.path.join(code_dir, class_filename))) + sys.path.append(code_dir) + module_name = os.path.splitext(class_filename)[0] + class_module = importlib.import_module(module_name) + class_constructor = getattr(class_module, class_name) + + if class_args is None: + class_args = {} + instance = class_constructor(**class_args) + + return instance + +def get_package_config_path(): + # FIXME: this might not be the desired location + config_dir = Path(nni.__path__[0]).parent / 'nni_config' + if not os.path.exists(config_dir): + os.makedirs(config_dir, exist_ok=True) + return os.path.join(config_dir, 'installed_packages.yml') + +def read_installed_package_meta(): + config_file = get_package_config_path() + if os.path.exists(config_file): + with open(config_file, 'r') as f: + config = yaml.load(f, Loader=yaml.Loader) + else: + config = defaultdict(list) + for t in ALGO_TYPES: + if t not in config: + config[t] = [] + return config + +def write_package_meta(config): + config_file = get_package_config_path() + with open(config_file, 'w') as f: + f.write(yaml.dump(dict(config), default_flow_style=False)) + +def _get_merged_builtin_dict(): + def merge_meta_dict(d1, d2): + res = defaultdict(list) + for t in ALGO_TYPES: + res[t] = d1[t] + d2[t] + return res + + return merge_meta_dict(BuiltinAlgorithms, read_installed_package_meta()) diff --git a/new_impl/cv/third_party/nni/tools/package_utils/constants.py b/new_impl/cv/third_party/nni/tools/package_utils/constants.py new file mode 100644 index 0000000000000000000000000000000000000000..952310b5f964e54e2ece8b0ce4e144dce41b4afd --- /dev/null +++ b/new_impl/cv/third_party/nni/tools/package_utils/constants.py @@ -0,0 +1,91 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +BuiltinAlgorithms = { + 'tuners': [ + { + 'name': 'TPE', + 'class_name': 'nni.algorithms.hpo.hyperopt_tuner.hyperopt_tuner.HyperoptTuner', + 'class_args': { + 'algorithm_name': 'tpe' + }, + 'class_args_validator': 'nni.algorithms.hpo.hyperopt_tuner.hyperopt_tuner.HyperoptClassArgsValidator' + }, + { + 'name': 'Random', + 'class_name': 'nni.algorithms.hpo.hyperopt_tuner.hyperopt_tuner.HyperoptTuner', + 'class_args': { + 'algorithm_name': 'random_search' + }, + 'accept_class_args': False, + 'class_args_validator': 'nni.algorithms.hpo.hyperopt_tuner.hyperopt_tuner.HyperoptClassArgsValidator' + }, + { + 'name': 'Anneal', + 'class_name': 'nni.algorithms.hpo.hyperopt_tuner.hyperopt_tuner.HyperoptTuner', + 'class_args': { + 'algorithm_name': 'anneal' + }, + 'class_args_validator': 'nni.algorithms.hpo.hyperopt_tuner.hyperopt_tuner.HyperoptClassArgsValidator' + }, + { + 'name': 'Evolution', + 'class_name': 'nni.algorithms.hpo.evolution_tuner.evolution_tuner.EvolutionTuner', + 'class_args_validator': 'nni.algorithms.hpo.evolution_tuner.evolution_tuner.EvolutionClassArgsValidator' + }, + { + 'name': 'BatchTuner', + 'class_name': 'nni.algorithms.hpo.batch_tuner.batch_tuner.BatchTuner', + 'accept_class_args': False, + }, + { + 'name': 'GridSearch', + 'class_name': 'nni.algorithms.hpo.gridsearch_tuner.gridsearch_tuner.GridSearchTuner', + 'accept_class_args': False, + }, + { + 'name': 'NetworkMorphism', + 'class_name': 'nni.algorithms.hpo.networkmorphism_tuner.networkmorphism_tuner.NetworkMorphismTuner', + 'class_args_validator': 'nni.algorithms.hpo.networkmorphism_tuner.networkmorphism_tuner.NetworkMorphismClassArgsValidator' + }, + { + 'name': 'MetisTuner', + 'class_name': 'nni.algorithms.hpo.metis_tuner.metis_tuner.MetisTuner', + 'class_args_validator': 'nni.algorithms.hpo.metis_tuner.metis_tuner.MetisClassArgsValidator' + }, + { + 'name': 'GPTuner', + 'class_name': 'nni.algorithms.hpo.gp_tuner.gp_tuner.GPTuner', + 'class_args_validator': 'nni.algorithms.hpo.gp_tuner.gp_tuner.GPClassArgsValidator' + }, + { + 'name': 'PBTTuner', + 'class_name': 'nni.algorithms.hpo.pbt_tuner.pbt_tuner.PBTTuner', + 'class_args_validator': 'nni.algorithms.hpo.pbt_tuner.pbt_tuner.PBTClassArgsValidator' + }, + { + 'name': 'RegularizedEvolutionTuner', + 'class_name': 'nni.algorithms.hpo.regularized_evolution_tuner.regularized_evolution_tuner.RegularizedEvolutionTuner', + 'class_args_validator': 'nni.algorithms.hpo.regularized_evolution_tuner.regularized_evolution_tuner.EvolutionClassArgsValidator' + } + ], + 'assessors': [ + { + 'name': 'Medianstop', + 'class_name': 'nni.algorithms.hpo.medianstop_assessor.medianstop_assessor.MedianstopAssessor', + 'class_args_validator': 'nni.algorithms.hpo.medianstop_assessor.medianstop_assessor.MedianstopClassArgsValidator' + }, + { + 'name': 'Curvefitting', + 'class_name': 'nni.algorithms.hpo.curvefitting_assessor.curvefitting_assessor.CurvefittingAssessor', + 'class_args_validator': 'nni.algorithms.hpo.curvefitting_assessor.curvefitting_assessor.CurvefittingClassArgsValidator' + }, + ], + 'advisors': [ + { + 'name': 'Hyperband', + 'class_name': 'nni.algorithms.hpo.hyperband_advisor.hyperband_advisor.Hyperband', + 'class_args_validator': 'nni.algorithms.hpo.hyperband_advisor.hyperband_advisor.HyperbandClassArgsValidator' + } + ] +} diff --git a/new_impl/cv/third_party/nni/tools/trial_tool/__init__.py b/new_impl/cv/third_party/nni/tools/trial_tool/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/new_impl/cv/third_party/nni/tools/trial_tool/aml_channel.py b/new_impl/cv/third_party/nni/tools/trial_tool/aml_channel.py new file mode 100644 index 0000000000000000000000000000000000000000..c8e1d7484a427b80b657d1c889c3a1cbe4aafecc --- /dev/null +++ b/new_impl/cv/third_party/nni/tools/trial_tool/aml_channel.py @@ -0,0 +1,47 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from azureml.core.run import Run # pylint: disable=import-error +from .base_channel import BaseChannel +from .log_utils import LogType, nni_log + + +class AMLChannel(BaseChannel): + def __init__(self, args): + self.args = args + self.run = Run.get_context() + super(AMLChannel, self).__init__(args) + self.current_message_index = -1 + + def _inner_open(self): + pass + + def _inner_close(self): + pass + + def _inner_send(self, message): + try: + self.run.log('trial_runner', message.decode('utf8')) + except Exception as exception: + nni_log(LogType.Error, 'meet unhandled exception when send message: %s' % exception) + + def _inner_receive(self): + messages = [] + message_dict = self.run.get_metrics() + if 'nni_manager' not in message_dict: + return [] + message_list = message_dict['nni_manager'] + if not message_list: + return messages + if type(message_list) is list: + if self.current_message_index < len(message_list) - 1: + messages = message_list[self.current_message_index + 1 : len(message_list)] + self.current_message_index = len(message_list) - 1 + elif self.current_message_index == -1: + messages = [message_list] + self.current_message_index += 1 + newMessage = [] + for message in messages: + # receive message is string, to get consistent result, encode it here. + newMessage.append(message.encode('utf8')) + return newMessage diff --git a/new_impl/cv/third_party/nni/tools/trial_tool/base_channel.py b/new_impl/cv/third_party/nni/tools/trial_tool/base_channel.py new file mode 100644 index 0000000000000000000000000000000000000000..b9d3392abc0b15ce7eb90b07df4107902fb65d9d --- /dev/null +++ b/new_impl/cv/third_party/nni/tools/trial_tool/base_channel.py @@ -0,0 +1,158 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import json +import threading +import time +from abc import ABC, abstractmethod +from queue import Empty, Queue + +from .log_utils import LogType, nni_log +from .commands import CommandType + +INTERVAL_SECONDS = 0.5 + + +class BaseChannel(ABC): + def __init__(self, args): + self.is_keep_parsed = args.node_count > 1 + self.args = args + self.node_id = self.args.node_id + + @abstractmethod + def _inner_send(self, message): + pass + + @abstractmethod + def _inner_receive(self): + return [] + + @abstractmethod + def _inner_open(self): + pass + + @abstractmethod + def _inner_close(self): + pass + + def open(self): + # initialize receive, send threads. + self.is_running = True + self.receive_queue = Queue() + self.receive_thread = threading.Thread(target=self._receive_loop) + self.receive_thread.start() + self.send_queue = Queue() + self.send_thread = threading.Thread(target=self._send_loop) + self.send_thread.start() + + self._inner_open() + + client_info = { + "isReady": True, + "runnerId": self.args.runner_id, + "expId": self.args.exp_id, + } + nni_log(LogType.Info, 'Channel: send ready information %s' % client_info) + self.send(CommandType.Initialized, client_info) + + def close(self): + self.is_running = False + try: + self._inner_close() + except Exception as err: + # ignore any error on closing + print("error on closing channel: %s" % err) + + def send(self, command, data): + """Send command to Training Service. + command: CommandType object. + data: string payload. + the message is sent synchronized. + """ + data["node"] = self.node_id + data = json.dumps(data) + data = data.encode('utf8') + message = b'%b%014d%b' % (command.value, len(data), data) + self.send_queue.put(message) + + def sent(self): + return self.send_queue.qsize() == 0 + + def received(self): + return self.receive_queue.qsize() > 0 + + def receive(self): + """Receive a command from Training Service. + Returns a tuple of command (CommandType) and payload (str) + """ + command = None + data = None + + try: + command_content = self.receive_queue.get(False) + if command_content is not None: + if (len(command_content) < 16): + # invalid header + nni_log(LogType.Error, 'incorrect command is found, command must be greater than 16 bytes!') + return None, None + header = command_content[:16] + command = CommandType(header[:2]) + length = int(header[2:]) + if (len(command_content)-16 != length): + nni_log(LogType.Error, 'incorrect command length, length {}, actual data length is {}, header {}.' + .format(length, len(command_content)-16, header)) + return None, None + data = command_content[16:16+length] + data = json.loads(data.decode('utf8')) + if self.node_id is None: + nni_log(LogType.Info, 'Received command, header: [%s], data: [%s]' % (header, data)) + else: + nni_log(LogType.Info, 'Received command(%s), header: [%s], data: [%s]' % (self.node_id, header, data)) + except Empty: + # do nothing, if no command received. + pass + except Exception as identifier: + nni_log(LogType.Error, 'meet unhandled exception in base_channel: %s' % identifier) + return command, data + + def _fetch_message(self, buffer, has_new_line=False): + messages = [] + while(len(buffer)) >= 16: + header = buffer[:16] + length = int(header[2:]) + + message_length = length+16 + total_length = message_length + if has_new_line: + total_length += 1 + + # break, if buffer is too short. + if len(buffer) < total_length: + break + data = buffer[16:message_length] + if has_new_line and 10 != buffer[total_length-1]: + nni_log(LogType.Error, 'end of message should be \\n, but got {}'.format(self.in_cache[total_length-1])) + buffer = buffer[total_length:] + messages.append(header + data) + + return messages, buffer + + def _receive_loop(self): + while (self.is_running): + messages = self._inner_receive() + if messages is not None: + for message in messages: + self.receive_queue.put(message) + time.sleep(INTERVAL_SECONDS) + + def _send_loop(self): + while (self.is_running): + message = None + try: + # no sleep, since it's a block call with INTERVAL_SECONDS second timeout + message = self.send_queue.get(True, INTERVAL_SECONDS) + except Empty: + # do nothing, if no command received. + pass + if message is not None: + self._inner_send(message) diff --git a/new_impl/cv/third_party/nni/tools/trial_tool/commands.py b/new_impl/cv/third_party/nni/tools/trial_tool/commands.py new file mode 100644 index 0000000000000000000000000000000000000000..86b10a2fe9a85d17bdfb4d9ec57b0b6cceb250da --- /dev/null +++ b/new_impl/cv/third_party/nni/tools/trial_tool/commands.py @@ -0,0 +1,25 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from enum import Enum + + +class CommandType(Enum): + Initialize = b'IN' + RequestTrialJobs = b'GE' + ReportMetricData = b'ME' + ReportGpuInfo = b'GI' + UpdateSearchSpace = b'SS' + ImportData = b'FD' + AddCustomizedTrialJob = b'AD' + TrialEnd = b'EN' + Terminate = b'TE' + Ping = b'PI' + + Initialized = b'ID' + NewTrialJob = b'TR' + SendTrialJobParameter = b'SP' + NoMoreTrialJobs = b'NO' + KillTrialJob = b'KI' + StdOut = b'SO' + VersionCheck = b'VC' diff --git a/new_impl/cv/third_party/nni/tools/trial_tool/constants.py b/new_impl/cv/third_party/nni/tools/trial_tool/constants.py new file mode 100644 index 0000000000000000000000000000000000000000..bef401337039d46aa23fc1e5816b58978d07e104 --- /dev/null +++ b/new_impl/cv/third_party/nni/tools/trial_tool/constants.py @@ -0,0 +1,24 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os + +API_ROOT_URL = '/api/v1/nni-pai' + +BASE_URL = 'http://{}' + +LOG_DIR = os.environ['NNI_OUTPUT_DIR'] + +NNI_PLATFORM = os.environ['NNI_PLATFORM'] + +STDOUT_FULL_PATH = os.path.join(LOG_DIR, 'stdout') + +STDERR_FULL_PATH = os.path.join(LOG_DIR, 'stderr') + +STDOUT_API = '/stdout' +VERSION_API = '/version' +PARAMETER_META_API = '/parameter-file-meta' +NNI_SYS_DIR = os.environ['NNI_SYS_DIR'] +NNI_TRIAL_JOB_ID = os.environ['NNI_TRIAL_JOB_ID'] +NNI_EXP_ID = os.environ['NNI_EXP_ID'] +MULTI_PHASE = os.environ['MULTI_PHASE'] diff --git a/new_impl/cv/third_party/nni/tools/trial_tool/file_channel.py b/new_impl/cv/third_party/nni/tools/trial_tool/file_channel.py new file mode 100644 index 0000000000000000000000000000000000000000..9a431d25f7ee3a4eeff973667d345ce2ddada1cb --- /dev/null +++ b/new_impl/cv/third_party/nni/tools/trial_tool/file_channel.py @@ -0,0 +1,75 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os + +from .base_channel import BaseChannel + +command_path = "./commands" +runner_commands_file_name_prefix = "runner_commands" +manager_commands_file_name = "manager_commands.txt" + + +class FileChannel(BaseChannel): + + def __init__(self, args): + self.node_id = args.node_id + self.out_file = None + self.in_file = None + self.in_offset = 0 + self.in_cache = b"" + + super(FileChannel, self).__init__(args) + + def _inner_open(self): + pass + + def _inner_close(self): + if self.out_file is not None: + self.out_file.close() + self.out_file = None + if self.in_file is not None: + self.in_file.close() + self.in_file = None + + def _inner_send(self, message): + if self.out_file is None: + if not os.path.exists(command_path): + os.makedirs(command_path, exist_ok=True) + + if self.node_id is None: + file_name = os.path.join(command_path, "%s.txt" % runner_commands_file_name_prefix) + else: + file_name = os.path.join(command_path, "%s_%s.txt" % ( + runner_commands_file_name_prefix, self.node_id)) + self.out_file = open(file_name, "ab") + + self.out_file.write(message) + self.out_file.write(b'\n') + self.out_file.flush() + + def _open_manager_command(self): + full_name = os.path.join(command_path, manager_commands_file_name) + + if self.in_file is not None and self.in_file.closed: + self.in_file = None + + if self.in_file is None and os.path.exists(full_name): + self.in_file = open(full_name, "rb") + self.in_file.seek(self.in_offset) + + def _inner_receive(self): + messages = [] + + if self.in_file is None: + self._open_manager_command() + if self.in_file is not None: + self.in_file.seek(0, os.SEEK_END) + new_offset = self.in_file.tell() + self.in_file.seek(self.in_offset, os.SEEK_SET) + count = new_offset - self.in_offset + if count > 0: + self.in_cache += self.in_file.read(count) + self.in_offset = new_offset + messages, self.in_cache = self._fetch_message(self.in_cache, True) + return messages diff --git a/new_impl/cv/third_party/nni/tools/trial_tool/gpu.py b/new_impl/cv/third_party/nni/tools/trial_tool/gpu.py new file mode 100644 index 0000000000000000000000000000000000000000..48dab4b182a6ecd08b1108c297d206663ce9bb40 --- /dev/null +++ b/new_impl/cv/third_party/nni/tools/trial_tool/gpu.py @@ -0,0 +1,69 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import subprocess +import time +import traceback +from xml.dom import minidom + + +def collect_gpu_usage(node_id): + cmd = 'nvidia-smi -q -x'.split() + info = None + try: + smi_output = subprocess.check_output(cmd) + info = parse_nvidia_smi_result(smi_output) + except Exception: + traceback.print_exc() + info = gen_empty_gpu_metric() + return info + + +def parse_nvidia_smi_result(smi): + try: + output = {} + xmldoc = minidom.parseString(smi) + gpuList = xmldoc.getElementsByTagName('gpu') + output["Timestamp"] = time.asctime(time.localtime()) + output["gpuCount"] = len(gpuList) + output["gpuInfos"] = [] + for gpuIndex, gpu in enumerate(gpuList): + gpuInfo = {} + gpuInfo['index'] = gpuIndex + gpuInfo['gpuUtil'] = gpu.getElementsByTagName('utilization')[0]\ + .getElementsByTagName('gpu_util')[0]\ + .childNodes[0].data.replace("%", "").strip() + gpuInfo['gpuMemUtil'] = gpu.getElementsByTagName('utilization')[0]\ + .getElementsByTagName('memory_util')[0]\ + .childNodes[0].data.replace("%", "").strip() + processes = gpu.getElementsByTagName('processes') + runningProNumber = len(processes[0].getElementsByTagName('process_info')) + gpuInfo['activeProcessNum'] = runningProNumber + + gpuInfo['gpuType'] = gpu.getElementsByTagName('product_name')[0]\ + .childNodes[0].data + memUsage = gpu.getElementsByTagName('fb_memory_usage')[0] + gpuInfo['gpuMemTotal'] = memUsage.getElementsByTagName('total')[0]\ + .childNodes[0].data.replace("MiB", "").strip() + gpuInfo['gpuMemUsed'] = memUsage.getElementsByTagName('used')[0]\ + .childNodes[0].data.replace("MiB", "").strip() + gpuInfo['gpuMemFree'] = memUsage.getElementsByTagName('free')[0]\ + .childNodes[0].data.replace("MiB", "").strip() + + output["gpuInfos"].append(gpuInfo) + except Exception: + traceback.print_exc() + output = {} + return output + + +def gen_empty_gpu_metric(): + try: + output = {} + output["Timestamp"] = time.asctime(time.localtime()) + output["gpuCount"] = 0 + output["gpuInfos"] = [] + except Exception: + traceback.print_exc() + output = {} + return output diff --git a/new_impl/cv/third_party/nni/tools/trial_tool/hdfsClientUtility.py b/new_impl/cv/third_party/nni/tools/trial_tool/hdfsClientUtility.py new file mode 100644 index 0000000000000000000000000000000000000000..05d4ea0d85675b5c763bf27f29db50b7900e1d03 --- /dev/null +++ b/new_impl/cv/third_party/nni/tools/trial_tool/hdfsClientUtility.py @@ -0,0 +1,92 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +import posixpath +from .log_utils import LogType, nni_log + +def copyHdfsDirectoryToLocal(hdfsDirectory, localDirectory, hdfsClient): + '''Copy directory from HDFS to local''' + if not os.path.exists(localDirectory): + os.makedirs(localDirectory) + try: + listing = hdfsClient.list_status(hdfsDirectory) + except Exception as exception: + nni_log(LogType.Error, 'List hdfs directory {0} error: {1}'.format(hdfsDirectory, str(exception))) + raise exception + + for f in listing: + if f.type == 'DIRECTORY': + subHdfsDirectory = posixpath.join(hdfsDirectory, f.pathSuffix) + subLocalDirectory = os.path.join(localDirectory, f.pathSuffix) + copyHdfsDirectoryToLocal(subHdfsDirectory, subLocalDirectory, hdfsClient) + elif f.type == 'FILE': + hdfsFilePath = posixpath.join(hdfsDirectory, f.pathSuffix) + localFilePath = os.path.join(localDirectory, f.pathSuffix) + copyHdfsFileToLocal(hdfsFilePath, localFilePath, hdfsClient) + else: + raise AssertionError('unexpected type {}'.format(f.type)) + +def copyHdfsFileToLocal(hdfsFilePath, localFilePath, hdfsClient, override=True): + '''Copy file from HDFS to local''' + if not hdfsClient.exists(hdfsFilePath): + raise Exception('HDFS file {} does not exist!'.format(hdfsFilePath)) + try: + file_status = hdfsClient.get_file_status(hdfsFilePath) + if file_status.type != 'FILE': + raise Exception('HDFS file path {} is not a file'.format(hdfsFilePath)) + except Exception as exception: + nni_log(LogType.Error, 'Get hdfs file {0} status error: {1}'.format(hdfsFilePath, str(exception))) + raise exception + + if os.path.exists(localFilePath) and override: + os.remove(localFilePath) + try: + hdfsClient.copy_to_local(hdfsFilePath, localFilePath) + except Exception as exception: + nni_log(LogType.Error, 'Copy hdfs file {0} to {1} error: {2}'.format(hdfsFilePath, localFilePath, str(exception))) + raise exception + nni_log(LogType.Info, 'Successfully copied hdfs file {0} to {1}, {2} bytes'.format(hdfsFilePath, localFilePath, file_status.length)) + +def copyDirectoryToHdfs(localDirectory, hdfsDirectory, hdfsClient): + '''Copy directory from local to HDFS''' + if not os.path.exists(localDirectory): + raise Exception('Local Directory does not exist!') + hdfsClient.mkdirs(hdfsDirectory) + result = True + for file in os.listdir(localDirectory): + file_path = os.path.join(localDirectory, file) + if os.path.isdir(file_path): + hdfs_directory = os.path.join(hdfsDirectory, file) + try: + result = result and copyDirectoryToHdfs(file_path, hdfs_directory, hdfsClient) + except Exception as exception: + nni_log(LogType.Error, + 'Copy local directory {0} to hdfs directory {1} error: {2}'.format(file_path, hdfs_directory, str(exception))) + result = False + else: + hdfs_file_path = os.path.join(hdfsDirectory, file) + try: + result = result and copyFileToHdfs(file_path, hdfs_file_path, hdfsClient) + except Exception as exception: + nni_log(LogType.Error, 'Copy local file {0} to hdfs {1} error: {2}'.format(file_path, hdfs_file_path, str(exception))) + result = False + return result + +def copyFileToHdfs(localFilePath, hdfsFilePath, hdfsClient, override=True): + '''Copy a local file to HDFS directory''' + if not os.path.exists(localFilePath): + raise Exception('Local file Path does not exist!') + if os.path.isdir(localFilePath): + raise Exception('localFile should not a directory!') + if hdfsClient.exists(hdfsFilePath): + if override: + hdfsClient.delete(hdfsFilePath) + else: + return False + try: + hdfsClient.copy_from_local(localFilePath, hdfsFilePath) + return True + except Exception as exception: + nni_log(LogType.Error, 'Copy local file {0} to hdfs file {1} error: {2}'.format(localFilePath, hdfsFilePath, str(exception))) + return False diff --git a/new_impl/cv/third_party/nni/tools/trial_tool/log_utils.py b/new_impl/cv/third_party/nni/tools/trial_tool/log_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..8d5b3d94c0b60d60b978b94d967f092bfa9a1c56 --- /dev/null +++ b/new_impl/cv/third_party/nni/tools/trial_tool/log_utils.py @@ -0,0 +1,219 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +import sys +import json +import logging +import logging.handlers +import time +import threading +import re + +from datetime import datetime +from enum import Enum, unique +from logging import StreamHandler + +from queue import Queue + +from .rest_utils import rest_post +from .url_utils import gen_send_stdout_url +from .commands import CommandType + + +@unique +class LogType(Enum): + Trace = 'TRACE' + Debug = 'DEBUG' + Info = 'INFO' + Warning = 'WARNING' + Error = 'ERROR' + Fatal = 'FATAL' + + +@unique +class StdOutputType(Enum): + Stdout = 'stdout', + Stderr = 'stderr' + + +def nni_log(log_type, log_message): + '''Log message into stdout''' + dt = datetime.now() + print('[{0}] {1} {2}'.format(dt, log_type.value, log_message), flush=True) + + +class NNIRestLogHanlder(StreamHandler): + def __init__(self, host, port, tag, trial_id, channel, std_output_type=StdOutputType.Stdout): + StreamHandler.__init__(self) + self.host = host + self.port = port + self.tag = tag + self.std_output_type = std_output_type + self.trial_id = trial_id + self.channel = channel + self.orig_stdout = sys.__stdout__ + self.orig_stderr = sys.__stderr__ + + def emit(self, record): + log_entry = {} + log_entry['tag'] = self.tag + log_entry['stdOutputType'] = self.std_output_type.name + log_entry['msg'] = self.format(record) + + try: + if self.channel is None: + rest_post(gen_send_stdout_url(self.host, self.port), json.dumps(log_entry), 10, True) + else: + if self.trial_id is not None: + log_entry["trial"] = self.trial_id + self.channel.send(CommandType.StdOut, log_entry) + except Exception as e: + self.orig_stderr.write(str(e) + '\n') + self.orig_stderr.flush() + + +class RemoteLogger(object): + """ + NNI remote logger + """ + + def __init__(self, syslog_host, syslog_port, tag, std_output_type, log_collection, trial_id=None, channel=None, log_level=logging.INFO): + ''' + constructor + ''' + logger_name = 'nni_syslog_{}'.format(tag) + # to prevent multiple trial logged in same logger + if trial_id is not None: + logger_name = '{}_{}'.format(logger_name, trial_id) + self.logger = logging.getLogger(logger_name) + self.log_level = log_level + self.logger.setLevel(self.log_level) + self.pipeReader = None + self.handler = NNIRestLogHanlder(syslog_host, syslog_port, tag, trial_id, channel) + self.logger.addHandler(self.handler) + if std_output_type == StdOutputType.Stdout: + self.orig_stdout = sys.__stdout__ + else: + self.orig_stdout = sys.__stderr__ + self.log_collection = log_collection + + def get_pipelog_reader(self): + ''' + Get pipe for remote logger + ''' + self.pipeReader = PipeLogReader(self.logger, self.log_collection, logging.INFO) + return self.pipeReader + + def flush(self): + ''' + Add flush in handler + ''' + for handler in self.logger.handlers: + handler.flush() + + def write(self, buf): + ''' + Write buffer data into logger/stdout + ''' + for line in buf.rstrip().splitlines(): + self.orig_stdout.write(line.rstrip() + '\n') + self.orig_stdout.flush() + try: + self.logger.log(self.log_level, line.rstrip()) + except Exception: + pass + + def close(self): + ''' + Close handlers and resources + ''' + if self.pipeReader is not None: + self.pipeReader.set_process_exit() + for handler in self.logger.handlers: + handler.close() + self.logger.removeHandler(handler) + + +class PipeLogReader(threading.Thread): + """ + The reader thread reads log data from pipe + """ + + def __init__(self, logger, log_collection, log_level=logging.INFO): + """Setup the object with a logger and a loglevel + and start the thread + """ + threading.Thread.__init__(self) + self.queue = Queue() + self.logger = logger + self.daemon = False + self.log_level = log_level + self.fdRead, self.fdWrite = os.pipe() + self.pipeReader = os.fdopen(self.fdRead) + self.orig_stdout = sys.__stdout__ + self._is_read_completed = False + self.process_exit = False + self.log_collection = log_collection + self.log_pattern = re.compile(r'NNISDK_MEb\'.*\'$') + + def _populateQueue(stream, queue): + ''' + Collect lines from 'stream' and put them in 'quque'. + ''' + time.sleep(1) + while True: + cur_process_exit = self.process_exit + try: + line = self.queue.get(True, 5) + try: + self.logger.log(self.log_level, line.rstrip()) + except Exception: + pass + except Exception: + if cur_process_exit == True: + self._is_read_completed = True + break + + self.pip_log_reader_thread = threading.Thread(target=_populateQueue, args=(self.pipeReader, self.queue)) + self.pip_log_reader_thread.daemon = True + self.start() + self.pip_log_reader_thread.start() + + def fileno(self): + """Return the write file descriptor of the pipe + """ + return self.fdWrite + + def run(self): + """Run the thread, logging everything. + If the log_collection is 'none', the log content will not be enqueued + """ + for line in iter(self.pipeReader.readline, ''): + self.orig_stdout.write(line.rstrip() + '\n') + self.orig_stdout.flush() + + if self.log_collection == 'none': + search_result = self.log_pattern.search(line) + if search_result: + metrics = search_result.group(0) + self.queue.put(metrics+'\n') + else: + self.queue.put(line) + + self.pipeReader.close() + + def close(self): + """Close the write end of the pipe. + """ + os.close(self.fdWrite) + + @property + def is_read_completed(self): + """Return if read is completed + """ + return self._is_read_completed + + def set_process_exit(self): + self.process_exit = True + return self.process_exit diff --git a/new_impl/cv/third_party/nni/tools/trial_tool/rest_utils.py b/new_impl/cv/third_party/nni/tools/trial_tool/rest_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..959209c7470bdb5dfe1ca32af91ba85757dd51e0 --- /dev/null +++ b/new_impl/cv/third_party/nni/tools/trial_tool/rest_utils.py @@ -0,0 +1,44 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import requests + +def rest_get(url, timeout): + '''Call rest get method''' + try: + response = requests.get(url, timeout=timeout) + return response + except Exception as e: + print('Get exception {0} when sending http get to url {1}'.format(str(e), url)) + return None + +def rest_post(url, data, timeout, rethrow_exception=False): + '''Call rest post method''' + try: + response = requests.post(url, headers={'Accept': 'application/json', 'Content-Type': 'application/json'},\ + data=data, timeout=timeout) + return response + except Exception as e: + if rethrow_exception is True: + raise + print('Get exception {0} when sending http post to url {1}'.format(str(e), url)) + return None + +def rest_put(url, data, timeout): + '''Call rest put method''' + try: + response = requests.put(url, headers={'Accept': 'application/json', 'Content-Type': 'application/json'},\ + data=data, timeout=timeout) + return response + except Exception as e: + print('Get exception {0} when sending http put to url {1}'.format(str(e), url)) + return None + +def rest_delete(url, timeout): + '''Call rest delete method''' + try: + response = requests.delete(url, timeout=timeout) + return response + except Exception as e: + print('Get exception {0} when sending http delete to url {1}'.format(str(e), url)) + return None diff --git a/new_impl/cv/third_party/nni/tools/trial_tool/trial.py b/new_impl/cv/third_party/nni/tools/trial_tool/trial.py new file mode 100644 index 0000000000000000000000000000000000000000..037b210cb0a50d69e89d3c7d62cf0bd0521886f8 --- /dev/null +++ b/new_impl/cv/third_party/nni/tools/trial_tool/trial.py @@ -0,0 +1,159 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import ctypes +import os +import shlex +import tarfile +import time +from datetime import datetime +from subprocess import Popen + +import psutil + +from .log_utils import LogType, RemoteLogger, StdOutputType, nni_log +from .commands import CommandType + +trial_output_path_name = ".nni" + + +class Trial: + def __init__(self, args, data): + self.process = None + self.data = data + self.args = args + self.command_channel = args.command_channel + self.trial_syslogger_stdout = None + + global NNI_TRIAL_JOB_ID + self.id = data["trialId"] + if self.id is None: + raise Exception("trial_id is not found in %s" % data) + os.environ['NNI_TRIAL_JOB_ID'] = self.id + NNI_TRIAL_JOB_ID = self.id + + # for multiple nodes. If it's None, it means single node. + self.node_id = args.node_id + if self.node_id is None: + self.name = self.id + else: + self.name = "%s_%s" % (self.id, self.node_id) + + def run(self): + # redirect trial's stdout and stderr to syslog + self.trial_syslogger_stdout = RemoteLogger(self.args.nnimanager_ip, self.args.nnimanager_port, 'trial', StdOutputType.Stdout, + self.args.log_collection, self.id, self.args.command_channel) + + nni_log(LogType.Info, "%s: start to run trial" % self.name) + + trial_working_dir = os.path.realpath(os.path.join(os.curdir, "..", "..", "trials", self.id)) + self.trial_output_dir = os.path.join(trial_working_dir, trial_output_path_name) + trial_code_dir = os.path.join(trial_working_dir, "code") + trial_nnioutput_dir = os.path.join(trial_working_dir, "nnioutput") + + environ = os.environ.copy() + environ['NNI_TRIAL_SEQ_ID'] = str(self.data["sequenceId"]) + environ['NNI_OUTPUT_DIR'] = os.path.join(trial_working_dir, "nnioutput") + environ['NNI_SYS_DIR'] = trial_working_dir + self.working_dir = trial_working_dir + + # prepare code and parameters + prepared_flag_file_name = os.path.join(trial_working_dir, "trial_prepared") + if not os.path.exists(trial_working_dir): + os.makedirs(trial_working_dir, exist_ok=True) + + os.makedirs(self.trial_output_dir, exist_ok=True) + os.makedirs(trial_nnioutput_dir, exist_ok=True) + # prepare code + os.makedirs(trial_code_dir, exist_ok=True) + with tarfile.open(os.path.join("..", "nni-code.tar.gz"), "r:gz") as tar: + tar.extractall(trial_code_dir) + + # save parameters + nni_log(LogType.Info, '%s: saving parameter %s' % (self.name, self.data["parameter"]["value"])) + parameter_file_name = os.path.join(trial_working_dir, "parameter.cfg") + with open(parameter_file_name, "w") as parameter_file: + parameter_file.write(self.data["parameter"]["value"]) + + # ready flag + with open(prepared_flag_file_name, "w") as prepared_flag_file: + prepared_flag_file.write("%s" % (int(datetime.now().timestamp() * 1000))) + + # make sure code prepared by other node. + if self.node_id is not None: + while True: + if os.path.exists(prepared_flag_file_name): + break + time.sleep(0.1) + + trial_command = self.args.trial_command + + gpuIndices = self.data.get('gpuIndices') + if (gpuIndices is not None): + trial_command = 'CUDA_VISIBLE_DEVICES="%s " %s' % (gpuIndices, trial_command) + + self.log_pipe_stdout = self.trial_syslogger_stdout.get_pipelog_reader() + self.process = Popen(trial_command, shell=True, stdout=self.log_pipe_stdout, + stderr=self.log_pipe_stdout, cwd=trial_code_dir, env=dict(environ)) + nni_log(LogType.Info, '{0}: spawns a subprocess (pid {1}) to run command: {2}'. + format(self.name, self.process.pid, shlex.split(trial_command))) + + def save_parameter_file(self, command_data): + parameters = command_data["parameters"] + file_index = int(parameters["index"]) + if file_index == 0: + parameter_file_name = "parameter.cfg" + else: + parameter_file_name = "parameter_{}.cfg".format(file_index) + parameter_file_name = os.path.join(self.working_dir, parameter_file_name) + with open(parameter_file_name, "w") as parameter_file: + nni_log(LogType.Info, '%s: saving parameter %s' % (self.name, parameters["value"])) + parameter_file.write(parameters["value"]) + + def is_running(self): + if (self.process is None): + return False + + retCode = self.process.poll() + # child worker process exits and all stdout data is read + if retCode is not None and self.log_pipe_stdout.set_process_exit() and self.log_pipe_stdout.is_read_completed == True: + # In Windows, the retCode -1 is 4294967295. It's larger than c_long, and raise OverflowError. + # So covert it to int32. + retCode = ctypes.c_long(retCode).value + nni_log(LogType.Info, '{0}: subprocess terminated. Exit code is {1}.'.format(self.name, retCode)) + + end_time = int(datetime.now().timestamp() * 1000) + end_message = { + "code": retCode, + "time": end_time, + "trial": self.id, + } + self.command_channel.send(CommandType.TrialEnd, end_message) + self.cleanup() + return False + else: + return True + + def kill(self, trial_id=None): + if trial_id == self.id or trial_id is None: + if self.process is not None: + try: + nni_log(LogType.Info, "%s: killing trial" % self.name) + for child in psutil.Process(self.process.pid).children(True): + child.kill() + self.process.kill() + except psutil.NoSuchProcess: + nni_log(LogType.Info, "kill trial %s failed: %s does not exist!" % (trial_id, self.process.pid)) + except Exception as ex: + nni_log(LogType.Error, "kill trial %s failed: %s " % (trial_id, str(ex))) + self.cleanup() + + def cleanup(self): + nni_log(LogType.Info, "%s: clean up trial" % self.name) + self.process = None + if self.log_pipe_stdout is not None: + self.log_pipe_stdout.set_process_exit() + self.log_pipe_stdout = None + if self.trial_syslogger_stdout is not None: + self.trial_syslogger_stdout.close() + self.trial_syslogger_stdout = None diff --git a/new_impl/cv/third_party/nni/tools/trial_tool/trial_keeper.py b/new_impl/cv/third_party/nni/tools/trial_tool/trial_keeper.py new file mode 100644 index 0000000000000000000000000000000000000000..08688973e000f7809421b0010ac9efe2bd0c4777 --- /dev/null +++ b/new_impl/cv/third_party/nni/tools/trial_tool/trial_keeper.py @@ -0,0 +1,246 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import argparse +import ctypes +import json +import logging +import os +import re +import shlex +import sys +import threading +import time +from subprocess import Popen + +import pkg_resources +from pyhdfs import HdfsClient + +from .constants import (LOG_DIR, MULTI_PHASE, NNI_EXP_ID, NNI_PLATFORM, + NNI_SYS_DIR, NNI_TRIAL_JOB_ID) +from .hdfsClientUtility import (copyDirectoryToHdfs, copyHdfsDirectoryToLocal, + copyHdfsFileToLocal) +from .log_utils import LogType, RemoteLogger, StdOutputType, nni_log +from .rest_utils import rest_get, rest_post +from .url_utils import gen_parameter_meta_url, gen_send_version_url + +logger = logging.getLogger('trial_keeper') +regular = re.compile('v?(?P[0-9](\.[0-9]){0,1}).*') + +_hdfs_client = None + + +def get_hdfs_client(args): + global _hdfs_client + + if _hdfs_client is not None: + return _hdfs_client + # backward compatibility + hdfs_host = None + + if args.hdfs_host: + hdfs_host = args.hdfs_host + elif args.pai_hdfs_host: + hdfs_host = args.pai_hdfs_host + else: + return None + + if hdfs_host is not None and args.nni_hdfs_exp_dir is not None: + try: + if args.webhdfs_path: + _hdfs_client = HdfsClient(hosts='{0}:80'.format(hdfs_host), user_name=args.pai_user_name, + webhdfs_path=args.webhdfs_path, timeout=5) + else: + # backward compatibility + _hdfs_client = HdfsClient(hosts='{0}:{1}'.format(hdfs_host, '50070'), user_name=args.pai_user_name, + timeout=5) + except Exception as e: + nni_log(LogType.Error, 'Create HDFS client error: ' + str(e)) + raise e + return _hdfs_client + + +def main_loop(args): + '''main loop logic for trial keeper''' + + if not os.path.exists(LOG_DIR): + os.makedirs(LOG_DIR) + + trial_keeper_syslogger = RemoteLogger(args.nnimanager_ip, args.nnimanager_port, 'trial_keeper', + StdOutputType.Stdout, args.log_collection) + # redirect trial keeper's stdout and stderr to syslog + trial_syslogger_stdout = RemoteLogger(args.nnimanager_ip, args.nnimanager_port, 'trial', StdOutputType.Stdout, + args.log_collection) + sys.stdout = sys.stderr = trial_keeper_syslogger + hdfs_output_dir = None + + if args.hdfs_output_dir: + hdfs_output_dir = args.hdfs_output_dir + elif args.pai_hdfs_output_dir: + hdfs_output_dir = args.pai_hdfs_output_dir + + hdfs_client = get_hdfs_client(args) + + if hdfs_client is not None: + copyHdfsDirectoryToLocal(args.nni_hdfs_exp_dir, os.getcwd(), hdfs_client) + + if args.job_id_file: + with open(args.job_id_file, 'w') as job_file: + job_file.write("%d" % os.getpid()) + + # Notice: We don't appoint env, which means subprocess wil inherit current environment and that is expected behavior + log_pipe_stdout = trial_syslogger_stdout.get_pipelog_reader() + process = Popen(args.trial_command, shell=True, stdout=log_pipe_stdout, stderr=log_pipe_stdout) + nni_log(LogType.Info, 'Trial keeper spawns a subprocess (pid {0}) to run command: {1}'.format(process.pid, + shlex.split( + args.trial_command))) + + while True: + retCode = process.poll() + # child worker process exits and all stdout data is read + if retCode is not None and log_pipe_stdout.set_process_exit() and log_pipe_stdout.is_read_completed == True: + # In Windows, the retCode -1 is 4294967295. It's larger than c_long, and raise OverflowError. + # So covert it to int32. + retCode = ctypes.c_long(retCode).value + nni_log(LogType.Info, 'subprocess terminated. Exit code is {}. Quit'.format(retCode)) + if hdfs_output_dir is not None: + # Copy local directory to hdfs for OpenPAI + nni_local_output_dir = os.environ['NNI_OUTPUT_DIR'] + try: + if copyDirectoryToHdfs(nni_local_output_dir, hdfs_output_dir, hdfs_client): + nni_log(LogType.Info, + 'copy directory from {0} to {1} success!'.format(nni_local_output_dir, hdfs_output_dir)) + else: + nni_log(LogType.Info, + 'copy directory from {0} to {1} failed!'.format(nni_local_output_dir, hdfs_output_dir)) + except Exception as e: + nni_log(LogType.Error, 'HDFS copy directory got exception: ' + str(e)) + raise e + + # Exit as the retCode of subprocess(trial) + exit(retCode) + break + + time.sleep(2) + + +def trial_keeper_help_info(*args): + print('please run --help to see guidance') + + +def check_version(args): + try: + trial_keeper_version = pkg_resources.get_distribution('nni').version + except pkg_resources.ResolutionError as err: + # package nni does not exist, try nni-tool package + nni_log(LogType.Error, 'Package nni does not exist!') + os._exit(1) + if not args.nni_manager_version: + # skip version check + nni_log(LogType.Warning, 'Skipping version check!') + else: + try: + trial_keeper_version = regular.search(trial_keeper_version).group('version') + nni_log(LogType.Info, 'trial_keeper_version is {0}'.format(trial_keeper_version)) + nni_manager_version = regular.search(args.nni_manager_version).group('version') + nni_log(LogType.Info, 'nni_manager_version is {0}'.format(nni_manager_version)) + log_entry = {} + if trial_keeper_version != nni_manager_version: + nni_log(LogType.Error, 'Version does not match!') + error_message = 'NNIManager version is {0}, TrialKeeper version is {1}, NNI version does not match!'.format( + nni_manager_version, trial_keeper_version) + log_entry['tag'] = 'VCFail' + log_entry['msg'] = error_message + rest_post(gen_send_version_url(args.nnimanager_ip, args.nnimanager_port), json.dumps(log_entry), 10, + False) + os._exit(1) + else: + nni_log(LogType.Info, 'Version match!') + log_entry['tag'] = 'VCSuccess' + rest_post(gen_send_version_url(args.nnimanager_ip, args.nnimanager_port), json.dumps(log_entry), 10, + False) + except AttributeError as err: + nni_log(LogType.Error, err) + + +def is_multi_phase(): + return MULTI_PHASE and (MULTI_PHASE in ['True', 'true']) + + +def download_parameter(meta_list, args): + """ + Download parameter file to local working directory. + meta_list format is defined in paiJobRestServer.ts + example meta_list: + [ + {"experimentId":"yWFJarYa","trialId":"UpPkl","filePath":"/chec/nni/experiments/yWFJarYa/trials/UpPkl/parameter_1.cfg"}, + {"experimentId":"yWFJarYa","trialId":"aIUMA","filePath":"/chec/nni/experiments/yWFJarYa/trials/aIUMA/parameter_1.cfg"} + ] + """ + nni_log(LogType.Debug, str(meta_list)) + nni_log(LogType.Debug, + 'NNI_SYS_DIR: {}, trial Id: {}, experiment ID: {}'.format(NNI_SYS_DIR, NNI_TRIAL_JOB_ID, NNI_EXP_ID)) + nni_log(LogType.Debug, 'NNI_SYS_DIR files: {}'.format(os.listdir(NNI_SYS_DIR))) + for meta in meta_list: + if meta['experimentId'] == NNI_EXP_ID and meta['trialId'] == NNI_TRIAL_JOB_ID: + param_fp = os.path.join(NNI_SYS_DIR, os.path.basename(meta['filePath'])) + if not os.path.exists(param_fp): + hdfs_client = get_hdfs_client(args) + copyHdfsFileToLocal(meta['filePath'], param_fp, hdfs_client, override=False) + + +def fetch_parameter_file(args): + class FetchThread(threading.Thread): + def __init__(self, args): + super(FetchThread, self).__init__() + self.args = args + + def run(self): + uri = gen_parameter_meta_url(self.args.nnimanager_ip, self.args.nnimanager_port) + nni_log(LogType.Info, uri) + + while True: + res = rest_get(uri, 10) + nni_log(LogType.Debug, 'status code: {}'.format(res.status_code)) + if res.status_code == 200: + meta_list = res.json() + download_parameter(meta_list, self.args) + else: + nni_log(LogType.Warning, 'rest response: {}'.format(str(res))) + time.sleep(5) + + fetch_file_thread = FetchThread(args) + fetch_file_thread.start() + + +if __name__ == '__main__': + '''NNI Trial Keeper main function''' + PARSER = argparse.ArgumentParser() + PARSER.set_defaults(func=trial_keeper_help_info) + PARSER.add_argument('--trial_command', type=str, help='Command to launch trial process') + PARSER.add_argument('--nnimanager_ip', type=str, default='localhost', help='NNI manager rest server IP') + PARSER.add_argument('--nnimanager_port', type=str, default='8081', help='NNI manager rest server port') + PARSER.add_argument('--pai_hdfs_output_dir', type=str, help='the output dir of pai_hdfs') # backward compatibility + PARSER.add_argument('--hdfs_output_dir', type=str, help='the output dir of hdfs') + PARSER.add_argument('--pai_hdfs_host', type=str, help='the host of pai_hdfs') # backward compatibility + PARSER.add_argument('--hdfs_host', type=str, help='the host of hdfs') + PARSER.add_argument('--pai_user_name', type=str, help='the username of hdfs') + PARSER.add_argument('--nni_hdfs_exp_dir', type=str, help='nni experiment directory in hdfs') + PARSER.add_argument('--webhdfs_path', type=str, help='the webhdfs path used in webhdfs URL') + PARSER.add_argument('--nni_manager_version', type=str, help='the nni version transmitted from nniManager') + PARSER.add_argument('--log_collection', type=str, help='set the way to collect log in trialkeeper') + PARSER.add_argument('--job_id_file', type=str, help='set job id file for operating and monitoring job.') + args, unknown = PARSER.parse_known_args() + if args.trial_command is None: + exit(1) + check_version(args) + try: + if NNI_PLATFORM == 'paiYarn' and is_multi_phase(): + fetch_parameter_file(args) + main_loop(args) + except SystemExit as se: + nni_log(LogType.Info, 'NNI trial keeper exit with code {}'.format(se.code)) + os._exit(se.code) + except Exception as e: + nni_log(LogType.Error, 'Exit trial keeper with code 1 because Exception: {} is catched'.format(str(e))) + os._exit(1) diff --git a/new_impl/cv/third_party/nni/tools/trial_tool/trial_runner.py b/new_impl/cv/third_party/nni/tools/trial_tool/trial_runner.py new file mode 100644 index 0000000000000000000000000000000000000000..8ee5c69bf7c07f04fbad3088456a24894d2032a9 --- /dev/null +++ b/new_impl/cv/third_party/nni/tools/trial_tool/trial_runner.py @@ -0,0 +1,255 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import argparse +import json +import os +import random +import re +import sys +import time +import traceback +from datetime import datetime, timedelta + +import pkg_resources + +from .gpu import collect_gpu_usage + +idle_timeout_seconds = 10 * 60 +gpu_refressh_interval_seconds = 5 +regular = re.compile('v?(?P[0-9](\.[0-9]){0,1}).*') +trial_runner_syslogger = None + + +def main_loop(args): + '''main loop logic for trial runner''' + idle_last_time = datetime.now() + gpu_refresh_last_time = datetime.now() - timedelta(minutes=1) + + try: + if args.job_pid_file: + with open(args.job_pid_file, 'w') as job_file: + job_file.write("%d" % os.getpid()) + + trials = dict() + + command_channel = args.command_channel + # command loop + while True: + command_type, command_data = command_channel.receive() + if command_type == CommandType.NewTrialJob: + trial_id = command_data["trialId"] + if trial_id in trials.keys(): + trial = trials[trial_id] + if trial.is_running(): + raise Exception('trial %s is running already, cannot start a new one' % trial.id) + else: + del trials[trial_id] + trial = Trial(args, command_data) + trial.run() + trials[trial_id] = trial + elif command_type == CommandType.KillTrialJob: + trial_id = command_data + if trial_id in trials.keys(): + trial = trials[trial_id] + trial.kill(command_data) + elif command_type == CommandType.SendTrialJobParameter: + trial_id = command_data["trialId"] + if trial_id in trials.keys(): + trial = trials[trial_id] + trial.save_parameter_file(command_data) + elif command_type is not None: + raise Exception("unknown command %s" % command_type) + + trial_list = list(trials.values()) + for trial in trial_list: + if trial is not None and trial.is_running(): + idle_last_time = datetime.now() + else: + del trials[trial.id] + + if (datetime.now() - idle_last_time).seconds > idle_timeout_seconds: + nni_log(LogType.Info, "trial runner is idle more than {0} seconds, so exit.".format( + idle_timeout_seconds)) + break + + if args.enable_gpu_collect and (datetime.now() - gpu_refresh_last_time).seconds > gpu_refressh_interval_seconds: + # collect gpu information + gpu_info = collect_gpu_usage(args.node_id) + command_channel.send(CommandType.ReportGpuInfo, gpu_info) + gpu_refresh_last_time = datetime.now() + time.sleep(0.5) + except Exception as ex: + traceback.print_exc() + raise ex + finally: + nni_log(LogType.Info, "main_loop exits.") + + trial_list = list(trials.values()) + for trial in trial_list: + trial.kill() + del trials[trial.id] + # wait to send commands + for _ in range(10): + if command_channel.sent(): + break + time.sleep(1) + command_channel.close() + + +def trial_runner_help_info(*args): + print('please run --help to see guidance') + + +def check_version(args): + try: + trial_runner_version = pkg_resources.get_distribution('nni').version + except pkg_resources.ResolutionError as err: + # package nni does not exist, try nni-tool package + nni_log(LogType.Error, 'Package nni does not exist!') + os._exit(1) + if not args.nni_manager_version: + # skip version check + nni_log(LogType.Warning, 'Skipping version check!') + else: + try: + command_channel = args.command_channel + trial_runner_version = regular.search(trial_runner_version).group('version') + nni_log(LogType.Info, '{0}: runner_version is {1}'.format(args.node_id, trial_runner_version)) + nni_manager_version = regular.search(args.nni_manager_version).group('version') + nni_log(LogType.Info, '{0}: nni_manager_version is {1}'.format(args.node_id, nni_manager_version)) + log_entry = {} + if trial_runner_version != nni_manager_version: + nni_log(LogType.Error, '{0}: Version does not match!'.format(args.node_id)) + error_message = '{0}: NNIManager version is {1}, Trial runner version is {2}, NNI version does not match!'.format( + args.node_id, nni_manager_version, trial_runner_version) + log_entry['tag'] = 'VCFail' + log_entry['msg'] = error_message + command_channel.send(CommandType.VersionCheck, log_entry) + while not command_channel.sent(): + time.sleep(1) + os._exit(1) + else: + nni_log(LogType.Info, '{0}: Version match!'.format(args.node_id)) + log_entry['tag'] = 'VCSuccess' + command_channel.send(CommandType.VersionCheck, log_entry) + except AttributeError as err: + nni_log(LogType.Error, '{0}: {1}'.format(args.node_id, err)) + +if __name__ == '__main__': + + '''NNI Trial Runner main function''' + PARSER = argparse.ArgumentParser() + PARSER.set_defaults(func=trial_runner_help_info) + PARSER.add_argument('--trial_command', type=str, help='Command to launch trial process') + PARSER.add_argument('--nnimanager_ip', type=str, help='NNI manager rest server IP') + PARSER.add_argument('--nnimanager_port', type=str, help='NNI manager rest server port') + PARSER.add_argument('--nni_manager_version', type=str, help='the nni version transmitted from nniManager') + PARSER.add_argument('--log_collection', type=str, help='set the way to collect log in trial runner') + PARSER.add_argument('--node_count', type=int, help='number of nodes, it determines how to consume command and save code file') + PARSER.add_argument('--job_pid_file', type=str, help='save trial runner process pid') + args, unknown = PARSER.parse_known_args() + + setting_file = "settings.json" + if not os.path.exists(setting_file): + setting_file = "../{}".format(setting_file) + if os.path.exists(setting_file): + with open(setting_file, 'r') as fp: + settings = json.load(fp) + print("setting is {}".format(settings)) + else: + print("not found setting file") + + args.exp_id = settings["experimentId"] + args.platform = settings["platform"] + # runner_id is unique runner in experiment + args.runner_id = os.path.basename(os.path.realpath(os.path.curdir)) + args.runner_name = "runner_"+args.runner_id + args.enable_gpu_collect = settings["enableGpuCollector"] + args.command_channel = settings["commandChannel"] + + if args.trial_command is None: + args.trial_command = settings["command"] + if args.nnimanager_ip is None: + args.nnimanager_ip = settings["nniManagerIP"] + if args.nnimanager_port is None: + args.nnimanager_port = settings["nniManagerPort"] + if args.nni_manager_version is None: + args.nni_manager_version = settings["nniManagerVersion"] + if args.log_collection is None: + args.log_collection = settings["logCollection"] + if args.node_count is None: + # default has only one node. + args.node_count = 1 + + os.environ['NNI_OUTPUT_DIR'] = os.curdir + "/nnioutput" + os.environ['NNI_PLATFORM'] = args.platform + os.environ['NNI_SYS_DIR'] = os.curdir + os.environ['NNI_EXP_ID'] = args.exp_id + os.environ['MULTI_PHASE'] = "true" + os.environ['NNI_TRIAL_JOB_ID'] = "runner" + + from .log_utils import LogType, RemoteLogger, StdOutputType, nni_log + from .trial import Trial + from .file_channel import FileChannel + from .web_channel import WebChannel + from .commands import CommandType + + is_multi_node = args.node_count > 1 + + if (is_multi_node): + # for multiple nodes, create a file to get a unique id. + while True: + node_id = random.randint(0, 10000) + unique_check_file_name = "node_%s" % (node_id) + if not os.path.exists(unique_check_file_name): + break + with open(unique_check_file_name, "w") as unique_check_file: + unique_check_file.write("%s" % (int(datetime.now().timestamp() * 1000))) + args.node_id = node_id + else: + # node id is unique in the runner + args.node_id = None + + # init command channel + command_channel = None + if args.command_channel == "file": + command_channel = FileChannel(args) + elif args.command_channel == 'aml': + from .aml_channel import AMLChannel + command_channel = AMLChannel(args) + else: + command_channel = WebChannel(args) + command_channel.open() + + nni_log(LogType.Info, "command channel is {}, actual type is {}".format(args.command_channel, type(command_channel))) + args.command_channel = command_channel + + trial_runner_syslogger = RemoteLogger(args.nnimanager_ip, args.nnimanager_port, 'runner', + StdOutputType.Stdout, args.log_collection, args.runner_name, command_channel) + sys.stdout = sys.stderr = trial_runner_syslogger + nni_log(LogType.Info, "{}: merged args is {}".format(args.node_id, args)) + + if args.trial_command is None: + nni_log(LogType.Error, "{}: no command is found.".format(args.node_id)) + os._exit(1) + check_version(args) + try: + main_loop(args) + except SystemExit as se: + nni_log(LogType.Info, '{}: NNI trial runner exit with code {}'.format(args.node_id, se.code)) + + # try best to send latest errors to server + timeout = 10 + while not command_channel.sent() and timeout > 0: + timeout -= 1 + time.sleep(1) + os._exit(se.code) + finally: + if trial_runner_syslogger is not None: + if trial_runner_syslogger.pipeReader is not None: + trial_runner_syslogger.pipeReader.set_process_exit() + trial_runner_syslogger.close() + + # the process doesn't exit even main loop exit. So exit it explictly. + os._exit(0) diff --git a/new_impl/cv/third_party/nni/tools/trial_tool/url_utils.py b/new_impl/cv/third_party/nni/tools/trial_tool/url_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7942c62fb5b4d8d3734ddf8c23a21b0f379d76ec --- /dev/null +++ b/new_impl/cv/third_party/nni/tools/trial_tool/url_utils.py @@ -0,0 +1,19 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .constants import API_ROOT_URL, BASE_URL, STDOUT_API, NNI_TRIAL_JOB_ID, NNI_EXP_ID, VERSION_API, PARAMETER_META_API + + +def gen_send_stdout_url(ip, port): + '''Generate send stdout url''' + return '{0}:{1}{2}{3}/{4}/{5}'.format(BASE_URL.format(ip), port, API_ROOT_URL, STDOUT_API, NNI_EXP_ID, NNI_TRIAL_JOB_ID) + + +def gen_send_version_url(ip, port): + '''Generate send error url''' + return '{0}:{1}{2}{3}/{4}/{5}'.format(BASE_URL.format(ip), port, API_ROOT_URL, VERSION_API, NNI_EXP_ID, NNI_TRIAL_JOB_ID) + + +def gen_parameter_meta_url(ip, port): + '''Generate send error url''' + return '{0}:{1}{2}{3}'.format(BASE_URL.format(ip), port, API_ROOT_URL, PARAMETER_META_API) diff --git a/new_impl/cv/third_party/nni/tools/trial_tool/web_channel.py b/new_impl/cv/third_party/nni/tools/trial_tool/web_channel.py new file mode 100644 index 0000000000000000000000000000000000000000..87901e1163a5fbcf82a2bb05c701735a346333fc --- /dev/null +++ b/new_impl/cv/third_party/nni/tools/trial_tool/web_channel.py @@ -0,0 +1,57 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import asyncio +import os +import websockets + +from .base_channel import BaseChannel +from .log_utils import LogType, nni_log + + +class WebChannel(BaseChannel): + + def __init__(self, args): + self.node_id = args.node_id + self.args = args + self.client = None + self.in_cache = b"" + self.timeout = 10 + + super(WebChannel, self).__init__(args) + + self._event_loop = None + + def _inner_open(self): + url = "ws://{}:{}".format(self.args.nnimanager_ip, self.args.nnimanager_port) + try: + connect = asyncio.wait_for(websockets.connect(url), self.timeout) + self._event_loop = asyncio.get_event_loop() + client = self._event_loop.run_until_complete(connect) + self.client = client + nni_log(LogType.Info, 'WebChannel: connected with info %s' % url) + except asyncio.TimeoutError: + nni_log(LogType.Error, 'connect to %s timeout! Please make sure NNIManagerIP configured correctly, and accessable.' % url) + os._exit(1) + + def _inner_close(self): + if self.client is not None: + self.client.close() + self.client = None + if self._event_loop.is_running(): + self._event_loop.stop() + self._event_loop = None + + def _inner_send(self, message): + loop = asyncio.new_event_loop() + loop.run_until_complete(self.client.send(message)) + + def _inner_receive(self): + messages = [] + if self.client is not None: + received = self._event_loop.run_until_complete(self.client.recv()) + # receive message is string, to get consistent result, encode it here. + self.in_cache += received.encode("utf8") + messages, self.in_cache = self._fetch_message(self.in_cache) + + return messages diff --git a/new_impl/cv/third_party/nni/trial.py b/new_impl/cv/third_party/nni/trial.py new file mode 100644 index 0000000000000000000000000000000000000000..cdb2b1e683ef970455c5b207b7b0f755dfc72d41 --- /dev/null +++ b/new_impl/cv/third_party/nni/trial.py @@ -0,0 +1,141 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .utils import to_json +from .runtime.env_vars import trial_env_vars +from .runtime import platform + + +__all__ = [ + 'get_next_parameter', + 'get_current_parameter', + 'report_intermediate_result', + 'report_final_result', + 'get_experiment_id', + 'get_trial_id', + 'get_sequence_id' +] + + +_params = None +_experiment_id = platform.get_experiment_id() +_trial_id = platform.get_trial_id() +_sequence_id = platform.get_sequence_id() + + +def get_next_parameter(): + """ + Get the hyper paremeters generated by tuner. For a multiphase experiment, it returns a new group of hyper + parameters at each call of get_next_parameter. For a non-multiphase (multiPhase is not configured or set to False) + experiment, it returns hyper parameters only on the first call for each trial job, it returns None since second call. + This API should be called only once in each trial job of an experiment which is not specified as multiphase. + + Returns + ------- + dict + A dict object contains the hyper parameters generated by tuner, the keys of the dict are defined in + search space. Returns None if no more hyper parameters can be generated by tuner. + """ + global _params + _params = platform.get_next_parameter() + if _params is None: + return None + return _params['parameters'] + +def get_current_parameter(tag=None): + """ + Get current hyper parameters generated by tuner. It returns the same group of hyper parameters as the last + call of get_next_parameter returns. + + Parameters + ---------- + tag: str + hyper parameter key + """ + global _params + if _params is None: + return None + if tag is None: + return _params['parameters'] + return _params['parameters'][tag] + +def get_experiment_id(): + """ + Get experiment ID. + + Returns + ------- + str + Identifier of current experiment + """ + return _experiment_id + +def get_trial_id(): + """ + Get trial job ID which is string identifier of a trial job, for example 'MoXrp'. In one experiment, each trial + job has an unique string ID. + + Returns + ------- + str + Identifier of current trial job which is calling this API. + """ + return _trial_id + +def get_sequence_id(): + """ + Get trial job sequence nubmer. A sequence number is an integer value assigned to each trial job base on the + order they are submitted, incremental starting from 0. In one experiment, both trial job ID and sequence number + are unique for each trial job, they are of different data types. + + Returns + ------- + int + Sequence number of current trial job which is calling this API. + """ + return _sequence_id + +_intermediate_seq = 0 + +def report_intermediate_result(metric): + """ + Reports intermediate result to NNI. + + Parameters + ---------- + metric: + serializable object. + """ + global _intermediate_seq + assert _params or trial_env_vars.NNI_PLATFORM is None, \ + 'nni.get_next_parameter() needs to be called before report_intermediate_result' + metric = to_json({ + 'parameter_id': _params['parameter_id'] if _params else None, + 'trial_job_id': trial_env_vars.NNI_TRIAL_JOB_ID, + 'type': 'PERIODICAL', + 'sequence': _intermediate_seq, + 'value': to_json(metric) + }) + _intermediate_seq += 1 + platform.send_metric(metric) + +def report_final_result(metric): + """ + Reports final result to NNI. + + Parameters + ---------- + metric: serializable object + Usually (for built-in tuners to work), it should be a number, or + a dict with key "default" (a number), and any other extra keys. + """ + assert _params or trial_env_vars.NNI_PLATFORM is None, \ + 'nni.get_next_parameter() needs to be called before report_final_result' + metric = to_json({ + 'parameter_id': _params['parameter_id'] if _params else None, + 'trial_job_id': trial_env_vars.NNI_TRIAL_JOB_ID, + 'type': 'FINAL', + 'sequence': 0, + 'value': to_json(metric) + }) + platform.send_metric(metric) diff --git a/new_impl/cv/third_party/nni/tuner.py b/new_impl/cv/third_party/nni/tuner.py new file mode 100644 index 0000000000000000000000000000000000000000..4fbcc011d0c55676e9bc9c7e906d7655bb5badeb --- /dev/null +++ b/new_impl/cv/third_party/nni/tuner.py @@ -0,0 +1,223 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +Tuner is an AutoML algorithm, which generates a new configuration for the next try. +A new trial will run with this configuration. + +See :class:`Tuner`' specification and ``docs/en_US/tuners.rst`` for details. +""" + +import logging + +import nni + +from .recoverable import Recoverable + +__all__ = ['Tuner'] + +_logger = logging.getLogger(__name__) + + +class Tuner(Recoverable): + """ + Tuner is an AutoML algorithm, which generates a new configuration for the next try. + A new trial will run with this configuration. + + This is the abstract base class for all tuners. + Tuning algorithms should inherit this class and override :meth:`update_search_space`, :meth:`receive_trial_result`, + as well as :meth:`generate_parameters` or :meth:`generate_multiple_parameters`. + + After initializing, NNI will first call :meth:`update_search_space` to tell tuner the feasible region, + and then call :meth:`generate_parameters` one or more times to request for hyper-parameter configurations. + + The framework will train several models with given configuration. + When one of them is finished, the final accuracy will be reported to :meth:`receive_trial_result`. + And then another configuration will be reqeusted and trained, util the whole experiment finish. + + If a tuner want's to know when a trial ends, it can also override :meth:`trial_end`. + + Tuners use *parameter ID* to track trials. + In tuner context, there is a one-to-one mapping between parameter ID and trial. + When the framework ask tuner to generate hyper-parameters for a new trial, + an ID has already been assigned and can be recorded in :meth:`generate_parameters`. + Later when the trial ends, the ID will be reported to :meth:`trial_end`, + and :meth:`receive_trial_result` if it has a final result. + Parameter IDs are unique integers. + + The type/format of search space and hyper-parameters are not limited, + as long as they are JSON-serializable and in sync with trial code. + For HPO tuners, however, there is a widely shared common interface, + which supports ``choice``, ``randint``, ``uniform``, and so on. + See ``docs/en_US/Tutorial/SearchSpaceSpec.md`` for details of this interface. + + [WIP] For advanced tuners which take advantage of trials' intermediate results, + an ``Advisor`` interface is under development. + + See Also + -------- + Builtin tuners: + :class:`~nni.algorithms.hpo.hyperopt_tuner.hyperopt_tuner.HyperoptTuner` + :class:`~nni.algorithms.hpo.evolution_tuner.evolution_tuner.EvolutionTuner` + :class:`~nni.algorithms.hpo.smac_tuner.SMACTuner` + :class:`~nni.algorithms.hpo.gridsearch_tuner.GridSearchTuner` + :class:`~nni.algorithms.hpo.networkmorphism_tuner.networkmorphism_tuner.NetworkMorphismTuner` + :class:`~nni.algorithms.hpo.metis_tuner.mets_tuner.MetisTuner` + :class:`~nni.algorithms.hpo.ppo_tuner.PPOTuner` + :class:`~nni.algorithms.hpo.gp_tuner.gp_tuner.GPTuner` + """ + + def generate_parameters(self, parameter_id, **kwargs): + """ + Abstract method which provides a set of hyper-parameters. + + This method will get called when the framework is about to launch a new trial, + if user does not override :meth:`generate_multiple_parameters`. + + The return value of this method will be received by trials via :func:`nni.get_next_parameter`. + It should fit in the search space, though the framework will not verify this. + + User code must override either this method or :meth:`generate_multiple_parameters`. + + Parameters + ---------- + parameter_id : int + Unique identifier for requested hyper-parameters. This will later be used in :meth:`receive_trial_result`. + **kwargs + Unstable parameters which should be ignored by normal users. + + Returns + ------- + any + The hyper-parameters, a dict in most cases, but could be any JSON-serializable type when needed. + + Raises + ------ + nni.NoMoreTrialError + If the search space is fully explored, tuner can raise this exception. + """ + # FIXME: some tuners raise NoMoreTrialError when they are waiting for more trial results + # we need to design a new exception for this purpose + raise NotImplementedError('Tuner: generate_parameters not implemented') + + def generate_multiple_parameters(self, parameter_id_list, **kwargs): + """ + Callback method which provides multiple sets of hyper-parameters. + + This method will get called when the framework is about to launch one or more new trials. + + If user does not override this method, it will invoke :meth:`generate_parameters` on each parameter ID. + + See :meth:`generate_parameters` for details. + + User code must override either this method or :meth:`generate_parameters`. + + Parameters + ---------- + parameter_id_list : list of int + Unique identifiers for each set of requested hyper-parameters. + These will later be used in :meth:`receive_trial_result`. + **kwargs + Unstable parameters which should be ignored by normal users. + + Returns + ------- + list + List of hyper-parameters. An empty list indicates there are no more trials. + """ + result = [] + for parameter_id in parameter_id_list: + try: + _logger.debug("generating param for %s", parameter_id) + res = self.generate_parameters(parameter_id, **kwargs) + except nni.NoMoreTrialError: + return result + result.append(res) + return result + + def receive_trial_result(self, parameter_id, parameters, value, **kwargs): + """ + Abstract method invoked when a trial reports its final result. Must override. + + This method only listens to results of algorithm-generated hyper-parameters. + Currently customized trials added from web UI will not report result to this method. + + Parameters + ---------- + parameter_id : int + Unique identifier of used hyper-parameters, same with :meth:`generate_parameters`. + parameters + Hyper-parameters generated by :meth:`generate_parameters`. + value + Result from trial (the return value of :func:`nni.report_final_result`). + **kwargs + Unstable parameters which should be ignored by normal users. + """ + raise NotImplementedError('Tuner: receive_trial_result not implemented') + + def _accept_customized_trials(self, accept=True): + # FIXME: because Tuner is designed as interface, this API should not be here + + # Enable or disable receiving results of user-added hyper-parameters. + # By default `receive_trial_result()` will only receive results of algorithm-generated hyper-parameters. + # If tuners want to receive those of customized parameters as well, they can call this function in `__init__()`. + + # pylint: disable=attribute-defined-outside-init + self._accept_customized = accept + + def trial_end(self, parameter_id, success, **kwargs): + """ + Abstract method invoked when a trial is completed or terminated. Do nothing by default. + + Parameters + ---------- + parameter_id : int + Unique identifier for hyper-parameters used by this trial. + success : bool + True if the trial successfully completed; False if failed or terminated. + **kwargs + Unstable parameters which should be ignored by normal users. + """ + + def update_search_space(self, search_space): + """ + Abstract method for updating the search space. Must override. + + Tuners are advised to support updating search space at run-time. + If a tuner can only set search space once before generating first hyper-parameters, + it should explicitly document this behaviour. + + Parameters + ---------- + search_space + JSON object defined by experiment owner. + """ + raise NotImplementedError('Tuner: update_search_space not implemented') + + def load_checkpoint(self): + """ + Internal API under revising, not recommended for end users. + """ + checkpoin_path = self.get_checkpoint_path() + _logger.info('Load checkpoint ignored by tuner, checkpoint path: %s', checkpoin_path) + + def save_checkpoint(self): + """ + Internal API under revising, not recommended for end users. + """ + checkpoin_path = self.get_checkpoint_path() + _logger.info('Save checkpoint ignored by tuner, checkpoint path: %s', checkpoin_path) + + def import_data(self, data): + """ + Internal API under revising, not recommended for end users. + """ + # Import additional data for tuning + # data: a list of dictionarys, each of which has at least two keys, 'parameter' and 'value' + pass + + def _on_exit(self): + pass + + def _on_error(self): + pass diff --git a/new_impl/cv/third_party/nni/utils.py b/new_impl/cv/third_party/nni/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..fe782dae2c945d62153511baa4b343c01226d5fa --- /dev/null +++ b/new_impl/cv/third_party/nni/utils.py @@ -0,0 +1,319 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +import copy +import functools +from enum import Enum, unique +import json_tricks +from schema import And + +from . import parameter_expressions +from .runtime.common import init_logger +from .runtime.env_vars import dispatcher_env_vars + + +to_json = functools.partial(json_tricks.dumps, allow_nan=True) + +@unique +class OptimizeMode(Enum): + """Optimize Mode class + + if OptimizeMode is 'minimize', it means the tuner need to minimize the reward + that received from Trial. + + if OptimizeMode is 'maximize', it means the tuner need to maximize the reward + that received from Trial. + """ + Minimize = 'minimize' + Maximize = 'maximize' + + +class NodeType: + """Node Type class + """ + ROOT = 'root' + TYPE = '_type' + VALUE = '_value' + INDEX = '_index' + NAME = '_name' + + +class MetricType: + """The types of metric data + """ + FINAL = 'FINAL' + PERIODICAL = 'PERIODICAL' + REQUEST_PARAMETER = 'REQUEST_PARAMETER' + + +def split_index(params): + """ + Delete index infromation from params + """ + if isinstance(params, dict): + if NodeType.INDEX in params.keys(): + return split_index(params[NodeType.VALUE]) + result = {} + for key in params: + result[key] = split_index(params[key]) + return result + else: + return params + + +def extract_scalar_reward(value, scalar_key='default'): + """ + Extract scalar reward from trial result. + + Parameters + ---------- + value : int, float, dict + the reported final metric data + scalar_key : str + the key name that indicates the numeric number + + Raises + ------ + RuntimeError + Incorrect final result: the final result should be float/int, + or a dict which has a key named "default" whose value is float/int. + """ + if isinstance(value, (float, int)): + reward = value + elif isinstance(value, dict) and scalar_key in value and isinstance(value[scalar_key], (float, int)): + reward = value[scalar_key] + else: + raise RuntimeError('Incorrect final result: the final result should be float/int, ' \ + 'or a dict which has a key named "default" whose value is float/int.') + return reward + + +def extract_scalar_history(trial_history, scalar_key='default'): + """ + Extract scalar value from a list of intermediate results. + + Parameters + ---------- + trial_history : list + accumulated intermediate results of a trial + scalar_key : str + the key name that indicates the numeric number + + Raises + ------ + RuntimeError + Incorrect final result: the final result should be float/int, + or a dict which has a key named "default" whose value is float/int. + """ + return [extract_scalar_reward(ele, scalar_key) for ele in trial_history] + + +def convert_dict2tuple(value): + """ + convert dict type to tuple to solve unhashable problem. + """ + if isinstance(value, dict): + for _keys in value: + value[_keys] = convert_dict2tuple(value[_keys]) + return tuple(sorted(value.items())) + return value + + +def init_dispatcher_logger(): + """ + Initialize dispatcher logging configuration + """ + logger_file_path = 'dispatcher.log' + if dispatcher_env_vars.NNI_LOG_DIRECTORY is not None: + logger_file_path = os.path.join(dispatcher_env_vars.NNI_LOG_DIRECTORY, logger_file_path) + init_logger(logger_file_path, dispatcher_env_vars.NNI_LOG_LEVEL) + + +def json2space(x, oldy=None, name=NodeType.ROOT): + """ + Change search space from json format to hyperopt format + + """ + y = list() + if isinstance(x, dict): + if NodeType.TYPE in x.keys(): + _type = x[NodeType.TYPE] + name = name + '-' + _type + if _type == 'choice': + if oldy is not None: + _index = oldy[NodeType.INDEX] + y += json2space(x[NodeType.VALUE][_index], + oldy[NodeType.VALUE], name=name+'[%d]' % _index) + else: + y += json2space(x[NodeType.VALUE], None, name=name) + y.append(name) + else: + for key in x.keys(): + y += json2space(x[key], oldy[key] if oldy else None, name+"[%s]" % str(key)) + elif isinstance(x, list): + for i, x_i in enumerate(x): + if isinstance(x_i, dict): + if NodeType.NAME not in x_i.keys(): + raise RuntimeError('\'_name\' key is not found in this nested search space.') + y += json2space(x_i, oldy[i] if oldy else None, name + "[%d]" % i) + return y + + +def json2parameter(x, is_rand, random_state, oldy=None, Rand=False, name=NodeType.ROOT): + """ + Json to pramaters. + + """ + if isinstance(x, dict): + if NodeType.TYPE in x.keys(): + _type = x[NodeType.TYPE] + _value = x[NodeType.VALUE] + name = name + '-' + _type + Rand |= is_rand[name] + if Rand is True: + if _type == 'choice': + _index = random_state.randint(len(_value)) + y = { + NodeType.INDEX: _index, + NodeType.VALUE: json2parameter( + x[NodeType.VALUE][_index], + is_rand, + random_state, + None, + Rand, + name=name+"[%d]" % _index + ) + } + else: + y = getattr(parameter_expressions, _type)(*(_value + [random_state])) + else: + y = copy.deepcopy(oldy) + else: + y = dict() + for key in x.keys(): + y[key] = json2parameter( + x[key], + is_rand, + random_state, + oldy[key] if oldy else None, + Rand, + name + "[%s]" % str(key) + ) + elif isinstance(x, list): + y = list() + for i, x_i in enumerate(x): + if isinstance(x_i, dict): + if NodeType.NAME not in x_i.keys(): + raise RuntimeError('\'_name\' key is not found in this nested search space.') + y.append(json2parameter( + x_i, + is_rand, + random_state, + oldy[i] if oldy else None, + Rand, + name + "[%d]" % i + )) + else: + y = copy.deepcopy(x) + return y + +def merge_parameter(base_params, override_params): + """ + Update the parameters in ``base_params`` with ``override_params``. + Can be useful to override parsed command line arguments. + + Parameters + ---------- + base_params : namespace or dict + Base parameters. A key-value mapping. + override_params : dict or None + Parameters to override. Usually the parameters got from ``get_next_parameters()``. + When it is none, nothing will happen. + + Returns + ------- + namespace or dict + The updated ``base_params``. Note that ``base_params`` will be updated inplace. The return value is + only for convenience. + """ + if override_params is None: + return base_params + is_dict = isinstance(base_params, dict) + for k, v in override_params.items(): + if is_dict: + if k not in base_params: + raise ValueError('Key \'%s\' not found in base parameters.' % k) + if type(base_params[k]) != type(v) and base_params[k] is not None: + raise TypeError('Expected \'%s\' in override parameters to have type \'%s\', but found \'%s\'.' % + (k, type(base_params[k]), type(v))) + base_params[k] = v + else: + if not hasattr(base_params, k): + raise ValueError('Key \'%s\' not found in base parameters.' % k) + if type(getattr(base_params, k)) != type(v) and getattr(base_params, k) is not None: + raise TypeError('Expected \'%s\' in override parameters to have type \'%s\', but found \'%s\'.' % + (k, type(getattr(base_params, k)), type(v))) + setattr(base_params, k, v) + return base_params + +class ClassArgsValidator(object): + """ + NNI tuners/assessors/adivisors accept a `classArgs` parameter in experiment configuration file. + This ClassArgsValidator interface is used to validate the classArgs section in exeperiment + configuration file. + """ + def validate_class_args(self, **kwargs): + """ + Validate the classArgs configuration in experiment configuration file. + + Parameters + ---------- + kwargs: dict + kwargs passed to tuner/assessor/advisor constructor + + Raises: + Raise an execption if the kwargs is invalid. + """ + pass + + def choices(self, key, *args): + """ + Utility method to create a scheme to check whether the `key` is one of the `args`. + + Parameters: + ---------- + key: str + key name of the data to be validated + args: list of str + list of the choices + + Returns: Schema + -------- + A scheme to check whether the `key` is one of the `args`. + """ + return And(lambda n: n in args, error='%s should be in [%s]!' % (key, str(args))) + + def range(self, key, keyType, start, end): + """ + Utility method to create a schema to check whether the `key` is in the range of [start, end]. + + Parameters: + ---------- + key: str + key name of the data to be validated + keyType: type + python data type, such as int, float + start: type is specified by keyType + start of the range + end: type is specified by keyType + end of the range + + Returns: Schema + -------- + A scheme to check whether the `key` is in the range of [start, end]. + """ + return And( + And(keyType, error='%s should be %s type!' % (key, keyType.__name__)), + And(lambda n: start <= n <= end, error='%s should be in range of (%s, %s)!' % (key, start, end)) + ) diff --git a/new_impl/cv/third_party/nni_new/__init__.py b/new_impl/cv/third_party/nni_new/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..c15ce55dfa09284f4ebffea6a951633f6ec50618 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/__init__.py @@ -0,0 +1,26 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +try: + from .version import __version__ # pylint: disable=import-error +except ModuleNotFoundError: + __version__ = '999.dev0' + +from .runtime.log import init_logger +init_logger() + +from .runtime.env_vars import dispatcher_env_vars +from .utils import ClassArgsValidator + +if dispatcher_env_vars.SDK_PROCESS != 'dispatcher': + from .trial import * + from .smartparam import * + from .common.nas_utils import training_update + +class NoMoreTrialError(Exception): + def __init__(self, ErrorInfo): + super().__init__(self) + self.errorinfo = ErrorInfo + + def __str__(self): + return self.errorinfo diff --git a/new_impl/cv/third_party/nni_new/__main__.py b/new_impl/cv/third_party/nni_new/__main__.py new file mode 100644 index 0000000000000000000000000000000000000000..80a7125e25877fe856e7fa59faeb6a449cf9d0a2 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/__main__.py @@ -0,0 +1,116 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +import argparse +import logging +import json +import base64 + +from .runtime.common import enable_multi_thread +from .runtime.msg_dispatcher import MsgDispatcher +from .tools.package_utils import create_builtin_class_instance, create_customized_class_instance + +logger = logging.getLogger('nni.main') +logger.debug('START') + +if os.environ.get('COVERAGE_PROCESS_START'): + import coverage + coverage.process_startup() + + +def main(): + parser = argparse.ArgumentParser(description='Dispatcher command line parser') + parser.add_argument('--exp_params', type=str, required=True) + args, _ = parser.parse_known_args() + + exp_params_decode = base64.b64decode(args.exp_params).decode('utf-8') + logger.debug('decoded exp_params: [%s]', exp_params_decode) + exp_params = json.loads(exp_params_decode) + logger.debug('exp_params json obj: [%s]', json.dumps(exp_params, indent=4)) + + if exp_params.get('deprecated', {}).get('multiThread'): + enable_multi_thread() + + if 'trainingServicePlatform' in exp_params: # config schema is v1 + from types import SimpleNamespace + from .experiment.config.convert import convert_algo + for algo_type in ['tuner', 'assessor', 'advisor']: + if algo_type in exp_params: + exp_params[algo_type] = convert_algo(algo_type, exp_params, SimpleNamespace()).json() + + if exp_params.get('advisor') is not None: + # advisor is enabled and starts to run + _run_advisor(exp_params) + else: + # tuner (and assessor) is enabled and starts to run + assert exp_params.get('tuner') is not None + tuner = _create_tuner(exp_params) + if exp_params.get('assessor') is not None: + assessor = _create_assessor(exp_params) + else: + assessor = None + dispatcher = MsgDispatcher(tuner, assessor) + + try: + dispatcher.run() + tuner._on_exit() + if assessor is not None: + assessor._on_exit() + except Exception as exception: + logger.exception(exception) + tuner._on_error() + if assessor is not None: + assessor._on_error() + raise + + +def _run_advisor(exp_params): + if exp_params.get('advisor').get('name'): + dispatcher = create_builtin_class_instance( + exp_params['advisor']['name'], + exp_params['advisor'].get('classArgs'), + 'advisors') + else: + dispatcher = create_customized_class_instance(exp_params.get('advisor')) + if dispatcher is None: + raise AssertionError('Failed to create Advisor instance') + try: + dispatcher.run() + except Exception as exception: + logger.exception(exception) + raise + + +def _create_tuner(exp_params): + if exp_params['tuner'].get('name'): + tuner = create_builtin_class_instance( + exp_params['tuner']['name'], + exp_params['tuner'].get('classArgs'), + 'tuners') + else: + tuner = create_customized_class_instance(exp_params['tuner']) + if tuner is None: + raise AssertionError('Failed to create Tuner instance') + return tuner + + +def _create_assessor(exp_params): + if exp_params['assessor'].get('name'): + assessor = create_builtin_class_instance( + exp_params['assessor']['name'], + exp_params['assessor'].get('classArgs'), + 'assessors') + else: + assessor = create_customized_class_instance(exp_params['assessor']) + if assessor is None: + raise AssertionError('Failed to create Assessor instance') + return assessor + + +if __name__ == '__main__': + try: + main() + except Exception as exception: + logger.exception(exception) + raise diff --git a/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/auto_compress/__init__.py b/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/auto_compress/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..8026bdd94c3b100fb9ee9d568b095ce4a702d4a9 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/auto_compress/__init__.py @@ -0,0 +1,6 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .experiment import AutoCompressionExperiment +from .interface import AbstractAutoCompressionModule +from .utils import AutoCompressionSearchSpaceGenerator diff --git a/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/auto_compress/auto_compress_engine.py b/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/auto_compress/auto_compress_engine.py new file mode 100644 index 0000000000000000000000000000000000000000..b1af71ec7fbf31ee3e3a28976aa6eb0f2fabfbfa --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/auto_compress/auto_compress_engine.py @@ -0,0 +1,164 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +from typing import Optional, Callable + +import json_tricks +from torch.nn import Module +from torch.optim import Optimizer + +import nni +from .constants import PRUNER_DICT, QUANTIZER_DICT +from .interface import BaseAutoCompressionEngine, AbstractAutoCompressionModule +from .utils import import_ + +_logger = logging.getLogger(__name__) +_logger.setLevel(logging.INFO) + +class AutoCompressionEngine(BaseAutoCompressionEngine): + @classmethod + def __convert_compact_pruner_params_to_config_list(cls, compact_config: dict) -> list: + config_dict = {} + for key, value in compact_config.items(): + _, op_types, op_names, var_name = key.split('::') + config_dict.setdefault((op_types, op_names), {}) + config_dict[(op_types, op_names)][var_name] = value + + config_list = [] + for key, config in config_dict.items(): + op_types, op_names = key + op_types = op_types.split(':') if op_types else [] + op_names = op_names.split(':') if op_names else [] + if op_types: + config['op_types'] = op_types + if op_names: + config['op_names'] = op_names + if 'op_types' in config or 'op_names' in config: + config_list.append(config) + + return config_list + + @classmethod + def __convert_compact_quantizer_params_to_config_list(cls, compact_config: dict) -> list: + config_dict = {} + for key, value in compact_config.items(): + _, quant_types, op_types, op_names, var_name = key.split('::') + config_dict.setdefault((quant_types, op_types, op_names), {}) + config_dict[(quant_types, op_types, op_names)][var_name] = value + + config_list = [] + for key, config in config_dict.items(): + quant_types, op_types, op_names = key + quant_types = quant_types.split(':') + op_types = op_types.split(':') + op_names = op_names.split(':') + if quant_types: + config['quant_types'] = quant_types + else: + continue + if op_types: + config['op_types'] = op_types + if op_names: + config['op_names'] = op_names + if 'op_types' in config or 'op_names' in config: + config_list.append(config) + + return config_list + + @classmethod + def _convert_compact_params_to_config_list(cls, compressor_type: str, compact_config: dict) -> list: + func_dict = { + 'pruner': cls.__convert_compact_pruner_params_to_config_list, + 'quantizer': cls.__convert_compact_quantizer_params_to_config_list + } + return func_dict[compressor_type](compact_config) + + @classmethod + def __compress_pruning(cls, algorithm_name: str, + model: Module, + config_list: list, + optimizer_factory: Optional[Callable], + criterion: Optional[Callable], + sparsifying_trainer: Optional[Callable[[Module, Optimizer, Callable, int], None]], + finetuning_trainer: Optional[Callable[[Module, Optimizer, Callable, int], None]], + finetuning_epochs: int, + **compressor_parameter_dict) -> Module: + if algorithm_name in ['level', 'l1', 'l2', 'fpgm']: + pruner = PRUNER_DICT[algorithm_name](model, config_list, **compressor_parameter_dict) + elif algorithm_name in ['slim', 'taylorfo', 'apoz', 'mean_activation']: + optimizer = None if optimizer_factory is None else optimizer_factory(model.parameters()) + pruner = PRUNER_DICT[algorithm_name](model, config_list, optimizer, sparsifying_trainer, criterion, **compressor_parameter_dict) + else: + raise ValueError('Unsupported compression algorithm: {}.'.format(algorithm_name)) + compressed_model = pruner.compress() + if finetuning_trainer is not None: + # note that in pruning process, finetuning will use an un-patched optimizer + optimizer = optimizer_factory(compressed_model.parameters()) + for i in range(finetuning_epochs): + finetuning_trainer(compressed_model, optimizer, criterion, i) + pruner.get_pruned_weights() + return compressed_model + + @classmethod + def __compress_quantization(cls, algorithm_name: str, + model: Module, + config_list: list, + optimizer_factory: Optional[Callable], + criterion: Optional[Callable], + sparsifying_trainer: Optional[Callable[[Module, Optimizer, Callable, int], None]], + finetuning_trainer: Optional[Callable[[Module, Optimizer, Callable, int], None]], + finetuning_epochs: int, + **compressor_parameter_dict) -> Module: + optimizer = None if optimizer_factory is None else optimizer_factory(model.parameters()) + quantizer = QUANTIZER_DICT[algorithm_name](model, config_list, optimizer, **compressor_parameter_dict) + compressed_model = quantizer.compress() + if finetuning_trainer is not None: + # note that in quantization process, finetuning will use a patched optimizer + for i in range(finetuning_epochs): + finetuning_trainer(compressed_model, optimizer, criterion, i) + return compressed_model + + @classmethod + def _compress(cls, compressor_type: str, + algorithm_name: str, + model: Module, + config_list: list, + optimizer_factory: Optional[Callable], + criterion: Optional[Callable], + sparsifying_trainer: Optional[Callable[[Module, Optimizer, Callable, int], None]], + finetuning_trainer: Optional[Callable[[Module, Optimizer, Callable, int], None]], + finetuning_epochs: int, + **compressor_parameter_dict) -> Module: + func_dict = { + 'pruner': cls.__compress_pruning, + 'quantizer': cls.__compress_quantization + } + _logger.info('%s compressor config_list:\n%s', algorithm_name, json_tricks.dumps(config_list, indent=4)) + compressed_model = func_dict[compressor_type](algorithm_name, model, config_list, optimizer_factory, criterion, sparsifying_trainer, + finetuning_trainer, finetuning_epochs, **compressor_parameter_dict) + return compressed_model + + @classmethod + def trial_execute_compress(cls, module_name): + auto_compress_module: AbstractAutoCompressionModule = import_(module_name) + + algorithm_config = nni.get_next_parameter()['algorithm_name'] + algorithm_name = algorithm_config['_name'] + compact_config = {k: v for k, v in algorithm_config.items() if k.startswith('config_list::')} + parameter_dict = {k.split('parameter::')[1]: v for k, v in algorithm_config.items() if k.startswith('parameter::')} + + compressor_type = 'quantizer' if algorithm_name in QUANTIZER_DICT else 'pruner' + + config_list = cls._convert_compact_params_to_config_list(compressor_type, compact_config) + + model, evaluator = auto_compress_module.model(), auto_compress_module.evaluator() + optimizer_factory, criterion = auto_compress_module.optimizer_factory(), auto_compress_module.criterion() + sparsifying_trainer = auto_compress_module.sparsifying_trainer(algorithm_name) + finetuning_trainer = auto_compress_module.post_compress_finetuning_trainer(algorithm_name) + finetuning_epochs = auto_compress_module.post_compress_finetuning_epochs(algorithm_name) + + compressed_model = cls._compress(compressor_type, algorithm_name, model, config_list, optimizer_factory, + criterion, sparsifying_trainer, finetuning_trainer, finetuning_epochs, **parameter_dict) + + nni.report_final_result(evaluator(compressed_model)) diff --git a/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/auto_compress/constants.py b/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/auto_compress/constants.py new file mode 100644 index 0000000000000000000000000000000000000000..bd36871cd8ee0656b9c072d8396fae46a70a4768 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/auto_compress/constants.py @@ -0,0 +1,25 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from ..pruning import LevelPruner, SlimPruner, L1FilterPruner, L2FilterPruner, FPGMPruner, TaylorFOWeightFilterPruner, \ + ActivationAPoZRankFilterPruner, ActivationMeanRankFilterPruner +from ..quantization.quantizers import NaiveQuantizer, QAT_Quantizer, DoReFaQuantizer, BNNQuantizer + + +PRUNER_DICT = { + 'level': LevelPruner, + 'slim': SlimPruner, + 'l1': L1FilterPruner, + 'l2': L2FilterPruner, + 'fpgm': FPGMPruner, + 'taylorfo': TaylorFOWeightFilterPruner, + 'apoz': ActivationAPoZRankFilterPruner, + 'mean_activation': ActivationMeanRankFilterPruner +} + +QUANTIZER_DICT = { + 'naive': NaiveQuantizer, + 'qat': QAT_Quantizer, + 'dorefa': DoReFaQuantizer, + 'bnn': BNNQuantizer +} diff --git a/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/auto_compress/experiment.py b/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/auto_compress/experiment.py new file mode 100644 index 0000000000000000000000000000000000000000..6859e60a128e9e41d29a1c39b557c4f120303326 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/auto_compress/experiment.py @@ -0,0 +1,70 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import inspect +from pathlib import Path, PurePath +from typing import overload, Union, List + +from numpy import tri + +from nni.experiment import Experiment, ExperimentConfig +from nni.algorithms.compression.pytorch.auto_compress.interface import AbstractAutoCompressionModule + + +class AutoCompressionExperiment(Experiment): + + @overload + def __init__(self, auto_compress_module: AbstractAutoCompressionModule, config: ExperimentConfig) -> None: + """ + Prepare an experiment. + + Use `Experiment.run()` to launch it. + + Parameters + ---------- + auto_compress_module + The module provided by the user implements the `AbstractAutoCompressionModule` interfaces. + Remember put the module file under `trial_code_directory`. + config + Experiment configuration. + """ + ... + + @overload + def __init__(self, auto_compress_module: AbstractAutoCompressionModule, training_service: Union[str, List[str]]) -> None: + """ + Prepare an experiment, leaving configuration fields to be set later. + + Example usage:: + + experiment = Experiment(auto_compress_module, 'remote') + experiment.config.trial_command = 'python3 trial.py' + experiment.config.machines.append(RemoteMachineConfig(ip=..., user_name=...)) + ... + experiment.run(8080) + + Parameters + ---------- + auto_compress_module + The module provided by the user implements the `AbstractAutoCompressionModule` interfaces. + Remember put the module file under `trial_code_directory`. + training_service + Name of training service. + Supported value: "local", "remote", "openpai", "aml", "kubeflow", "frameworkcontroller", "adl" and hybrid training service. + """ + ... + + def __init__(self, auto_compress_module: AbstractAutoCompressionModule, config=None, training_service=None): + super().__init__(config, training_service) + + self.module_file_path = str(PurePath(inspect.getfile(auto_compress_module))) + self.module_name = auto_compress_module.__name__ + + def start(self, port: int, debug: bool) -> None: + trial_code_directory = str(PurePath(Path(self.config.trial_code_directory).absolute())) + '/' + assert self.module_file_path.startswith(trial_code_directory), 'The file path of the user-provided module should under trial_code_directory.' + relative_module_path = self.module_file_path.split(trial_code_directory)[1] + # only support linux, need refactor? + command = 'python3 -m nni.algorithms.compression.pytorch.auto_compress.trial_entry --module_file_name {} --module_class_name {}' + self.config.trial_command = command.format(relative_module_path, self.module_name) + super().start(port=port, debug=debug) diff --git a/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/auto_compress/interface.py b/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/auto_compress/interface.py new file mode 100644 index 0000000000000000000000000000000000000000..208d0ef6a15369b32c97c0df609f973c650697e4 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/auto_compress/interface.py @@ -0,0 +1,122 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from abc import ABC, abstractmethod +from typing import Optional, Callable, Iterable + +from torch.nn import Module +from torch.optim import Optimizer + + +class BaseAutoCompressionEngine(ABC): + @classmethod + @abstractmethod + def trial_execute_compress(cls): + """ + Execute the compressing trial. + """ + pass + + +class AbstractAutoCompressionModule(ABC): + """ + The abstract container that user need to implement. + """ + @classmethod + @abstractmethod + def model(cls) -> Module: + """ + Returns + ------- + torch.nn.Module + Model to be compress. + """ + pass + + @classmethod + @abstractmethod + def evaluator(cls) -> Callable[[Module], float]: + """ + Returns + ------- + function + The function used to evaluate the compressed model, return a scalar. + """ + pass + + @classmethod + @abstractmethod + def optimizer_factory(cls) -> Optional[Callable[[Iterable], Optimizer]]: + """ + Returns + ------- + Optional[Callable[[Iterable], Optimizer]] + Optimizer factory function. Input is a iterable value, i.e. `model.parameters()`. + Output is the `torch.optim.Optimizer` instance. + """ + pass + + @classmethod + @abstractmethod + def criterion(cls) -> Optional[Callable]: + """ + Returns + ------- + Optional[Callable] + The criterion function used to train the model. + """ + pass + + @classmethod + @abstractmethod + def sparsifying_trainer(cls, compress_algorithm_name: str) -> Optional[Callable[[Module, Optimizer, Callable, int], None]]: + """ + The trainer is used in sparsifying process. + + Parameters + ---------- + compress_algorithm_name: str + The name of pruner and quantizer, i.e. 'level', 'l1', 'qat'. + + Returns + ------- + Optional[Callable[[Module, Optimizer, Callable, int], None]] + Used to train model in compress stage, include `model, optimizer, criterion, current_epoch` as function arguments. + """ + pass + + @classmethod + @abstractmethod + def post_compress_finetuning_trainer(cls, compress_algorithm_name: str) -> Optional[Callable[[Module, Optimizer, Callable, int], None]]: + """ + The trainer is used in post-compress finetuning process. + + Parameters + ---------- + compress_algorithm_name: str + The name of pruner and quantizer, i.e. 'level', 'l1', 'qat'. + + Returns + ------- + Optional[Callable[[Module, Optimizer, Callable, int], None]] + Used to train model in finetune stage, include `model, optimizer, criterion, current_epoch` as function arguments. + """ + pass + + @classmethod + @abstractmethod + def post_compress_finetuning_epochs(cls, compress_algorithm_name: str) -> int: + """ + The epochs in post-compress finetuning process. + + Parameters + ---------- + compress_algorithm_name: str + The name of pruner and quantizer, i.e. 'level', 'l1', 'qat'. + + Returns + ------- + int + The finetuning epoch number. + """ + pass diff --git a/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/auto_compress/trial_entry.py b/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/auto_compress/trial_entry.py new file mode 100644 index 0000000000000000000000000000000000000000..c502f4a1854df5388c2f451fec42b6a7b05af903 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/auto_compress/trial_entry.py @@ -0,0 +1,21 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +Entrypoint for trials. +""" +import argparse +from pathlib import Path +import re + +from .auto_compress_engine import AutoCompressionEngine + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='trial entry for auto compression.') + parser.add_argument('--module_file_name', required=True, dest='module_file_name', help='the path of auto compression module file') + parser.add_argument('--module_class_name', required=True, dest='module_class_name', help='the name of auto compression module') + args = parser.parse_args() + + module_name = Path(args.module_file_name).as_posix() + module_name = re.sub(re.escape('.py') + '$', '', module_name).replace('/', '.') + '.' + args.module_class_name + AutoCompressionEngine.trial_execute_compress(module_name) diff --git a/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/auto_compress/utils.py b/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/auto_compress/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..fe46db63e9c1b23d7e0c5766e6a89b01e7053fb9 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/auto_compress/utils.py @@ -0,0 +1,116 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from typing import Any + +from .constants import PRUNER_DICT, QUANTIZER_DICT + + +class AutoCompressionSearchSpaceGenerator: + """ + For convenient generation of search space that can be used by tuner. + """ + + def __init__(self): + self.algorithm_choice_list = [] + + def add_config(self, algorithm_name: str, config_list: list, **algo_kwargs): + """ + This function used for distinguish algorithm type is pruning or quantization. + Then call `self._add_pruner_config()` or `self._add_quantizer_config()`. + """ + if algorithm_name in PRUNER_DICT: + self._add_pruner_config(algorithm_name, config_list, **algo_kwargs) + if algorithm_name in QUANTIZER_DICT: + self._add_quantizer_config(algorithm_name, config_list, **algo_kwargs) + + def _add_pruner_config(self, pruner_name: str, config_list: list, **algo_kwargs): + """ + Parameters + ---------- + pruner_name + Supported pruner name: 'level', 'slim', 'l1', 'l2', 'fpgm', 'taylorfo', 'apoz', 'mean_activation'. + config_list + Except 'op_types' and 'op_names', other config value can be written as `{'_type': ..., '_value': ...}`. + **algo_kwargs + The additional pruner parameters except 'model', 'config_list', 'optimizer', 'trainer', 'criterion'. + i.e., you can set `statistics_batch_num={'_type': 'choice', '_value': [1, 2, 3]}` in TaylorFOWeightFilterPruner or just `statistics_batch_num=1`. + """ + sub_search_space = {'_name': pruner_name} + for config in config_list: + op_types = config.pop('op_types', []) + op_names = config.pop('op_names', []) + key_prefix = 'config_list::{}::{}'.format(':'.join(op_types), ':'.join(op_names)) + for var_name, var_search_space in config.items(): + sub_search_space['{}::{}'.format(key_prefix, var_name)] = self._wrap_single_value(var_search_space) + for parameter_name, parameter_search_space in algo_kwargs.items(): + key_prefix = 'parameter' + sub_search_space['{}::{}'.format(key_prefix, parameter_name)] = self._wrap_single_value(parameter_search_space) + self.algorithm_choice_list.append(sub_search_space) + + def _add_quantizer_config(self, quantizer_name: str, config_list: list, **algo_kwargs): + """ + Parameters + ---------- + quantizer_name + Supported pruner name: 'naive', 'qat', 'dorefa', 'bnn'. + config_list + Except 'quant_types', 'op_types' and 'op_names', other config value can be written as `{'_type': ..., '_value': ...}`. + **algo_kwargs + The additional pruner parameters except 'model', 'config_list', 'optimizer'. + """ + sub_search_space = {'_name': quantizer_name} + for config in config_list: + quant_types = config.pop('quant_types', []) + op_types = config.pop('op_types', []) + op_names = config.pop('op_names', []) + key_prefix = 'config_list::{}::{}::{}'.format(':'.join(quant_types), ':'.join(op_types), ':'.join(op_names)) + for var_name, var_search_space in config.items(): + sub_search_space['{}::{}'.format(key_prefix, var_name)] = self._wrap_single_value(var_search_space) + for parameter_name, parameter_search_space in algo_kwargs.items(): + key_prefix = 'parameter' + sub_search_space['{}::{}'.format(key_prefix, parameter_name)] = self._wrap_single_value(parameter_search_space) + self.algorithm_choice_list.append(sub_search_space) + + def dumps(self) -> dict: + """ + Dump the search space as a dict. + """ + search_space = { + 'algorithm_name': { + '_type': 'choice', + '_value': self.algorithm_choice_list + } + } + return search_space + + @classmethod + def loads(cls, search_space: dict): + """ + Return a AutoCompressionSearchSpaceGenerator instance load from a search space dict. + """ + generator = AutoCompressionSearchSpaceGenerator() + generator.algorithm_choice_list = search_space['algorithm_name']['_value'] + return generator + + def _wrap_single_value(self, value) -> dict: + if not isinstance(value, dict): + converted_value = { + '_type': 'choice', + '_value': [value] + } + elif '_type' not in value: + converted_value = {} + for k, v in value.items(): + converted_value[k] = self._wrap_single_value(v) + else: + converted_value = value + return converted_value + + +def import_(target: str, allow_none: bool = False) -> Any: + if target is None: + return None + path, identifier = target.rsplit('.', 1) + module = __import__(path, globals(), locals(), [identifier]) + return getattr(module, identifier) diff --git a/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/pruning/__init__.py b/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/pruning/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..f93cfaeed13b9882ee1c33106ca85af31d148425 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/pruning/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .finegrained_pruning_masker import * +from .structured_pruning_masker import * +from .one_shot_pruner import * +from .iterative_pruner import * +# from .lottery_ticket import LotteryTicketPruner +# from .simulated_annealing_pruner import SimulatedAnnealingPruner +# from .net_adapt_pruner import NetAdaptPruner +# from .auto_compress_pruner import AutoCompressPruner +# from .sensitivity_pruner import SensitivityPruner +# from .amc import AMCPruner diff --git a/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/pruning/amc/__init__.py b/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/pruning/amc/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..3c89a879c6af42ed372f38f3cc11ea1eb9eb6870 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/pruning/amc/__init__.py @@ -0,0 +1,4 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .amc_pruner import AMCPruner diff --git a/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/pruning/amc/amc_pruner.py b/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/pruning/amc/amc_pruner.py new file mode 100644 index 0000000000000000000000000000000000000000..c2d12429d5fb5d82131b5239ef1992ef100feef6 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/pruning/amc/amc_pruner.py @@ -0,0 +1,296 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +import logging +from copy import deepcopy +from argparse import Namespace +import numpy as np +import torch +from torch.utils.tensorboard import SummaryWriter + +from nni.compression.pytorch.compressor import Pruner +from .channel_pruning_env import ChannelPruningEnv +from .lib.agent import DDPG +from .lib.utils import get_output_folder + +torch.backends.cudnn.deterministic = True + +_logger = logging.getLogger(__name__) + +class AMCPruner(Pruner): + """ + A pytorch implementation of AMC: AutoML for Model Compression and Acceleration on Mobile Devices. + (https://arxiv.org/pdf/1802.03494.pdf) + + Parameters: + model: nn.Module + The model to be pruned. + config_list: list + Configuration list to configure layer pruning. + Supported keys: + - op_types: operation type to be pruned + - op_names: operation name to be pruned + evaluator: function + function to evaluate the pruned model. + The prototype of the function: + >>> def evaluator(val_loader, model): + >>> ... + >>> return acc + val_loader: torch.utils.data.DataLoader + Data loader of validation dataset. + suffix: str + suffix to help you remember what experiment you ran. Default: None. + + # parameters for pruning environment + model_type: str + model type to prune, currently 'mobilenet' and 'mobilenetv2' are supported. Default: mobilenet + flops_ratio: float + preserve flops ratio. Default: 0.5 + lbound: float + minimum weight preserve ratio for each layer. Default: 0.2 + rbound: float + maximum weight preserve ratio for each layer. Default: 1.0 + reward: function + reward function type: + - acc_reward: accuracy * 0.01 + - acc_flops_reward: - (100 - accuracy) * 0.01 * np.log(flops) + Default: acc_reward + # parameters for channel pruning + n_calibration_batches: int + number of batches to extract layer information. Default: 60 + n_points_per_layer: int + number of feature points per layer. Default: 10 + channel_round: int + round channel to multiple of channel_round. Default: 8 + + # parameters for ddpg agent + hidden1: int + hidden num of first fully connect layer. Default: 300 + hidden2: int + hidden num of second fully connect layer. Default: 300 + lr_c: float + learning rate for critic. Default: 1e-3 + lr_a: float + learning rate for actor. Default: 1e-4 + warmup: int + number of episodes without training but only filling the replay memory. During warmup episodes, + random actions ares used for pruning. Default: 100 + discount: float + next Q value discount for deep Q value target. Default: 0.99 + bsize: int + minibatch size for training DDPG agent. Default: 64 + rmsize: int + memory size for each layer. Default: 100 + window_length: int + replay buffer window length. Default: 1 + tau: float + moving average for target network being used by soft_update. Default: 0.99 + # noise + init_delta: float + initial variance of truncated normal distribution + delta_decay: float + delta decay during exploration + + # parameters for training ddpg agent + max_episode_length: int + maximum episode length + output_dir: str + output directory to save log files and model files. Default: ./logs + debug: boolean + debug mode + train_episode: int + train iters each timestep. Default: 800 + epsilon: int + linear decay of exploration policy. Default: 50000 + seed: int + random seed to set for reproduce experiment. Default: None + """ + + def __init__( + self, + model, + config_list, + evaluator, + val_loader, + suffix=None, + model_type='mobilenet', + dataset='cifar10', + flops_ratio=0.5, + lbound=0.2, + rbound=1., + reward='acc_reward', + n_calibration_batches=60, + n_points_per_layer=10, + channel_round=8, + hidden1=300, + hidden2=300, + lr_c=1e-3, + lr_a=1e-4, + warmup=100, + discount=1., + bsize=64, + rmsize=100, + window_length=1, + tau=0.01, + init_delta=0.5, + delta_decay=0.99, + max_episode_length=1e9, + output_dir='./logs', + debug=False, + train_episode=800, + epsilon=50000, + seed=None): + + self.val_loader = val_loader + self.evaluator = evaluator + + if seed is not None: + np.random.seed(seed) + torch.manual_seed(seed) + torch.cuda.manual_seed(seed) + + checkpoint = deepcopy(model.state_dict()) + + super().__init__(model, config_list, optimizer=None) + + # build folder and logs + base_folder_name = '{}_{}_r{}_search'.format(model_type, dataset, flops_ratio) + if suffix is not None: + self.output_dir = os.path.join(output_dir, base_folder_name + '-' + suffix) + else: + self.output_dir = get_output_folder(output_dir, base_folder_name) + + self.env_args = Namespace( + model_type=model_type, + preserve_ratio=flops_ratio, + lbound=lbound, + rbound=rbound, + reward=reward, + n_calibration_batches=n_calibration_batches, + n_points_per_layer=n_points_per_layer, + channel_round=channel_round, + output=self.output_dir + ) + self.env = ChannelPruningEnv( + self, evaluator, val_loader, checkpoint, args=self.env_args) + _logger.info('=> Saving logs to %s', self.output_dir) + self.tfwriter = SummaryWriter(log_dir=self.output_dir) + self.text_writer = open(os.path.join(self.output_dir, 'log.txt'), 'w') + _logger.info('=> Output path: %s...', self.output_dir) + + nb_states = self.env.layer_embedding.shape[1] + nb_actions = 1 # just 1 action here + + rmsize = rmsize * len(self.env.prunable_idx) # for each layer + _logger.info('** Actual replay buffer size: %d', rmsize) + + self.ddpg_args = Namespace( + hidden1=hidden1, + hidden2=hidden2, + lr_c=lr_c, + lr_a=lr_a, + warmup=warmup, + discount=discount, + bsize=bsize, + rmsize=rmsize, + window_length=window_length, + tau=tau, + init_delta=init_delta, + delta_decay=delta_decay, + max_episode_length=max_episode_length, + debug=debug, + train_episode=train_episode, + epsilon=epsilon + ) + self.agent = DDPG(nb_states, nb_actions, self.ddpg_args) + + + def compress(self): + self.train(self.ddpg_args.train_episode, self.agent, self.env, self.output_dir) + + def train(self, num_episode, agent, env, output_dir): + agent.is_training = True + step = episode = episode_steps = 0 + episode_reward = 0. + observation = None + T = [] # trajectory + while episode < num_episode: # counting based on episode + # reset if it is the start of episode + if observation is None: + observation = deepcopy(env.reset()) + agent.reset(observation) + + # agent pick action ... + if episode <= self.ddpg_args.warmup: + action = agent.random_action() + # action = sample_from_truncated_normal_distribution(lower=0., upper=1., mu=env.preserve_ratio, sigma=0.5) + else: + action = agent.select_action(observation, episode=episode) + + # env response with next_observation, reward, terminate_info + observation2, reward, done, info = env.step(action) + + T.append([reward, deepcopy(observation), deepcopy(observation2), action, done]) + + # fix-length, never reach here + # if max_episode_length and episode_steps >= max_episode_length - 1: + # done = True + + # [optional] save intermideate model + if num_episode / 3 <= 1 or episode % int(num_episode / 3) == 0: + agent.save_model(output_dir) + + # update + step += 1 + episode_steps += 1 + episode_reward += reward + observation = deepcopy(observation2) + + if done: # end of episode + _logger.info( + '#%d: episode_reward: %.4f acc: %.4f, ratio: %.4f', + episode, episode_reward, + info['accuracy'], + info['compress_ratio'] + ) + self.text_writer.write( + '#{}: episode_reward:{:.4f} acc: {:.4f}, ratio: {:.4f}\n'.format( + episode, episode_reward, + info['accuracy'], + info['compress_ratio'] + ) + ) + final_reward = T[-1][0] + # print('final_reward: {}'.format(final_reward)) + # agent observe and update policy + for _, s_t, s_t1, a_t, done in T: + agent.observe(final_reward, s_t, s_t1, a_t, done) + if episode > self.ddpg_args.warmup: + agent.update_policy() + + #agent.memory.append( + # observation, + # agent.select_action(observation, episode=episode), + # 0., False + #) + + # reset + observation = None + episode_steps = 0 + episode_reward = 0. + episode += 1 + T = [] + + self.tfwriter.add_scalar('reward/last', final_reward, episode) + self.tfwriter.add_scalar('reward/best', env.best_reward, episode) + self.tfwriter.add_scalar('info/accuracy', info['accuracy'], episode) + self.tfwriter.add_scalar('info/compress_ratio', info['compress_ratio'], episode) + self.tfwriter.add_text('info/best_policy', str(env.best_strategy), episode) + # record the preserve rate for each layer + for i, preserve_rate in enumerate(env.strategy): + self.tfwriter.add_scalar('preserve_rate/{}'.format(i), preserve_rate, episode) + + self.text_writer.write('best reward: {}\n'.format(env.best_reward)) + self.text_writer.write('best policy: {}\n'.format(env.best_strategy)) + self.text_writer.close() diff --git a/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/pruning/amc/channel_pruning_env.py b/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/pruning/amc/channel_pruning_env.py new file mode 100644 index 0000000000000000000000000000000000000000..428f7e7532da931abcce5d71ac0847c7234b30aa --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/pruning/amc/channel_pruning_env.py @@ -0,0 +1,543 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +import logging +import time +import math +import copy +import numpy as np +import torch +import torch.nn as nn + +from nni.compression.pytorch.compressor import PrunerModuleWrapper +from .. import AMCWeightMasker + +_logger = logging.getLogger(__name__) + +# for pruning +def acc_reward(net, acc, flops): + return acc * 0.01 + + +def acc_flops_reward(net, acc, flops): + error = (100 - acc) * 0.01 + return -error * np.log(flops) + + +class ChannelPruningEnv: + """ + Env for channel pruning search. + This class is used to prune model using specified pruner. It prunes one layer when + step() is called. When the last layer is pruned, it evaluate the pruned model using + evaluator, and use the returned value of evaluator as reward of the episode. + + Usage: + env = ChannelPruningEnv(pruner, evaluator, val_loader, checkpoint, env_args) + episode = 0 + T = [] + while episode < num_episode: + action = agent.select_action(observation) + observation2, reward, done, info = env.step(action) + T.append([reward, deepcopy(observation), deepcopy(observation2), action, done]) + + if done: # end of episode, last layer pruned + episode += 1 + # train agent with episode data + for _, s_t, s_t1, a_t, done in T: + agent.observe(final_reward, s_t, s_t1, a_t, done) + agent.update_policy() + T = [] + + Attributes: + prunable_idx: layer indices for pruable layers, the index values are the index + of list(self.model.modules()). Pruable layers are pointwise Conv2d layers and Linear + layers. + buffer_idx: layer indices for buffer layers which refers the depthwise layers. + Each depthwise layer is always followd by a pointwise layer for both mobilenet and + mobilenetv2. The depthwise layer's filters are pruned when its next pointwise layer's + corresponding input channels are pruned. + shared_idx: layer indices for layers which share input. + For example: [[1,4], [8, 10, 15]] means layer 1 and 4 share same input, and layer + 8, 10 and 15 share another input. + layer_embedding: embeddings for each prunable layers, the embedding is used as + observation for DDPG agent. + layer_info_dict: flops and number of parameters of each layer. + min_strategy_dict: key is layer index, value is a tuple, the first value is the minimum + action of input channel, the second value is the minimum action value of output channel. + strategy_dict: key is layer index, value is a tuple, the first value is the action of input + channel, the second value is the action of output channel. + + Parameters: + pruner: Pruner + NNI Pruner instance used to prune model. + evaluator: function + function to evaluate the pruned model. + The prototype of the function: + >>> def evaluator(val_loader, model): + >>> ... + >>> return acc + val_loader: torch.utils.data.DataLoader + Data loader of validation dataset. + checkpoint: dict + checkpoint of the model to be pruned. It is used to reset model at beginning of each + episode. + args: + A Namespace object containing following arguments: + model_type: str + model type to prune, currently 'mobilenet', 'mobilenetv2' and 'resnet' are supported. + flops_ratio: float + preserve flops ratio. + lbound: float + minimum weight preserve ratio for each layer. + rbound: float + maximum weight preserve ratio for each layer. + reward: function + reward function type + + # parameters for channel pruning + n_calibration_batches: int + number of batches to extract layer information. + n_points_per_layer: int + number of feature points per layer. + channel_round: int + round channel to multiple of channel_round. + + """ + def __init__(self, pruner, evaluator, val_loader, checkpoint, args): + self.pruner = pruner + self.model = pruner.bound_model + self.checkpoint = checkpoint + self.batch_size = val_loader.batch_size + self.preserve_ratio = args.preserve_ratio + self.channel_prune_masker = AMCWeightMasker(self.model, self.pruner, args.channel_round) + + # options from args + self.args = args + self.lbound = args.lbound + self.rbound = args.rbound + + self.n_calibration_batches = args.n_calibration_batches + self.n_points_per_layer = args.n_points_per_layer + self.channel_round = args.channel_round + + # sanity check + assert self.preserve_ratio > self.lbound, 'Error! You can not achieve preserve_ratio smaller than lbound!' + + # prepare data + self._val_loader = val_loader + self._validate = evaluator + + # build indexs + self._build_index() + self.n_prunable_layer = len(self.prunable_idx) + + # extract information for preparing + self._extract_layer_information() + + # build embedding (static part) + self._build_state_embedding() + + # build reward + self.reset() # restore weight + self.org_acc = self._validate(self._val_loader, self.model) + _logger.info('=> original acc: %.3f', self.org_acc) + self.org_model_size = sum(self.wsize_list) + _logger.info('=> original weight size: %.4f M param', self.org_model_size * 1. / 1e6) + self.org_flops = sum(self.flops_list) + _logger.info('=> FLOPs:') + _logger.info([self.layer_info_dict[idx]['flops']/1e6 for idx in sorted(self.layer_info_dict.keys())]) + _logger.info('=> original FLOPs: %.4f M', self.org_flops * 1. / 1e6) + + self.expected_preserve_computation = self.preserve_ratio * self.org_flops + + self.reward = eval(args.reward) + + self.best_reward = -math.inf + self.best_strategy = None + self.best_d_prime_list = None + self.best_masks = None + + self.org_w_size = sum(self.wsize_list) + + def step(self, action): + # Pseudo prune and get the corresponding statistics. The real pruning happens till the end of all pseudo pruning + if self.visited[self.cur_ind]: + action = self.strategy_dict[self.prunable_idx[self.cur_ind]][0] + preserve_idx = self.index_buffer[self.cur_ind] + else: + action = self._action_wall(action) # percentage to preserve + preserve_idx = None + # prune and update action + action, d_prime, preserve_idx = self.prune_kernel(self.prunable_idx[self.cur_ind], action, preserve_idx) + if not self.visited[self.cur_ind]: + for group in self.shared_idx: + if self.cur_ind in group: # set the shared ones + for g_idx in group: + self.strategy_dict[self.prunable_idx[g_idx]][0] = action + self.strategy_dict[self.prunable_idx[g_idx - 1]][1] = action + self.visited[g_idx] = True + self.index_buffer[g_idx] = preserve_idx.copy() + + self.strategy.append(action) # save action to strategy + self.d_prime_list.append(d_prime) + + self.strategy_dict[self.prunable_idx[self.cur_ind]][0] = action + if self.cur_ind > 0: + self.strategy_dict[self.prunable_idx[self.cur_ind - 1]][1] = action + + # all the actions are made + if self._is_final_layer(): + assert len(self.strategy) == len(self.prunable_idx) + current_flops = self._cur_flops() + acc_t1 = time.time() + acc = self._validate(self._val_loader, self.model) + acc_t2 = time.time() + self.val_time = acc_t2 - acc_t1 + compress_ratio = current_flops * 1. / self.org_flops + info_set = {'compress_ratio': compress_ratio, 'accuracy': acc, 'strategy': self.strategy.copy()} + reward = self.reward(self, acc, current_flops) + + if reward > self.best_reward: + self.best_reward = reward + self.best_strategy = self.strategy.copy() + self.best_d_prime_list = self.d_prime_list.copy() + best_model = os.path.join(self.args.output, 'best_model.pth') + best_mask = os.path.join(self.args.output, 'best_mask.pth') + self.pruner.export_model(model_path=best_model, mask_path=best_mask) + _logger.info('New best reward: %.4f, acc: %.4f, compress: %.4f', self.best_reward, acc, compress_ratio) + _logger.info('New best policy: %s', self.best_strategy) + _logger.info('New best d primes: %s', self.best_d_prime_list) + obs = self.layer_embedding[self.cur_ind, :].copy() # actually the same as the last state + done = True + return obs, reward, done, info_set + + info_set = None + reward = 0 + done = False + self.visited[self.cur_ind] = True # set to visited + self.cur_ind += 1 # the index of next layer + # build next state (in-place modify) + self.layer_embedding[self.cur_ind][-3] = self._cur_reduced() * 1. / self.org_flops # reduced + self.layer_embedding[self.cur_ind][-2] = sum(self.flops_list[self.cur_ind + 1:]) * 1. / self.org_flops # rest + self.layer_embedding[self.cur_ind][-1] = self.strategy[-1] # last action + obs = self.layer_embedding[self.cur_ind, :].copy() + + return obs, reward, done, info_set + + def reset(self): + # restore env by loading the checkpoint + self.pruner.reset(self.checkpoint) + self.cur_ind = 0 + self.strategy = [] # pruning strategy + self.d_prime_list = [] + self.strategy_dict = copy.deepcopy(self.min_strategy_dict) + # reset layer embeddings + self.layer_embedding[:, -1] = 1. + self.layer_embedding[:, -2] = 0. + self.layer_embedding[:, -3] = 0. + obs = self.layer_embedding[0].copy() + obs[-2] = sum(self.wsize_list[1:]) * 1. / sum(self.wsize_list) + self.extract_time = 0 + self.fit_time = 0 + self.val_time = 0 + # for share index + self.visited = [False] * len(self.prunable_idx) + self.index_buffer = {} + return obs + + def prune_kernel(self, op_idx, preserve_ratio, preserve_idx=None): + m_list = list(self.model.modules()) + op = m_list[op_idx] + assert (0. < preserve_ratio <= 1.) + assert type(op) == PrunerModuleWrapper + if preserve_ratio == 1: # do not prune + if (preserve_idx is None) or (len(preserve_idx) == op.module.weight.size(1)): + return 1., op.module.weight.size(1), None # should be a full index + op.input_feat = self.layer_info_dict[op_idx]['input_feat'] + op.output_feat = self.layer_info_dict[op_idx]['output_feat'] + + masks = self.channel_prune_masker.calc_mask(sparsity=1-preserve_ratio, wrapper=op, preserve_idx=preserve_idx) + m = masks['weight_mask'].cpu().data + if type(op.module) == nn.Conv2d: + d_prime = (m.sum((0, 2, 3)) > 0).sum().item() + preserve_idx = np.nonzero((m.sum((0, 2, 3)) > 0).numpy())[0] + else: + assert type(op.module) == nn.Linear + d_prime = (m.sum(1) > 0).sum().item() + preserve_idx = np.nonzero((m.sum(1) > 0).numpy())[0] + + op.weight_mask = masks['weight_mask'] + if hasattr(op.module, 'bias') and op.module.bias is not None and 'bias_mask' in masks: + op.bias_mask = masks['bias_mask'] + + action = (m == 1).sum().item() / m.numel() + return action, d_prime, preserve_idx + + def _is_final_layer(self): + return self.cur_ind == len(self.prunable_idx) - 1 + + def _action_wall(self, action): + """ + Limit the action generated by DDPG for this layer by two constraints: + 1. The total flops must meet the flops reduce target. + For example: the original flops of entire model is 1000, target flops ratio is 0.5, target flops + is 1000*0.5 = 500. The reduced flops of other layers is 400, so the remaining flops quota is 500-400=100, + if the total original flops of this layer is 250, then the maximum ratio is 100/250 = 0.4. So the + action of this layer can not be greater than 0.4. + 2. The action must be greater than lbound which is stored in self.strategy_dict. + """ + assert len(self.strategy) == self.cur_ind + + action = float(action) + action = np.clip(action, 0, 1) + + other_comp = 0 + this_comp = 0 + for i, idx in enumerate(self.prunable_idx): + flop = self.layer_info_dict[idx]['flops'] + buffer_flop = self._get_buffer_flops(idx) + + if i == self.cur_ind - 1: # TODO: add other member in the set + this_comp += flop * self.strategy_dict[idx][0] + # add buffer (but not influenced by ratio) + other_comp += buffer_flop * self.strategy_dict[idx][0] + elif i == self.cur_ind: + this_comp += flop * self.strategy_dict[idx][1] + # also add buffer here (influenced by ratio) + this_comp += buffer_flop + else: + other_comp += flop * self.strategy_dict[idx][0] * self.strategy_dict[idx][1] + # add buffer + other_comp += buffer_flop * self.strategy_dict[idx][0] # only consider input reduction + + self.expected_min_preserve = other_comp + this_comp * action + max_preserve_ratio = (self.expected_preserve_computation - other_comp) * 1. / this_comp + + action = np.minimum(action, max_preserve_ratio) + action = np.maximum(action, self.strategy_dict[self.prunable_idx[self.cur_ind]][0]) # impossible (should be) + + return action + + def _get_buffer_flops(self, idx): + buffer_idx = self.buffer_dict[idx] + buffer_flop = sum([self.layer_info_dict[_]['flops'] for _ in buffer_idx]) + return buffer_flop + + def _cur_flops(self): + flops = 0 + for idx in self.prunable_idx: + c, n = self.strategy_dict[idx] # input, output pruning ratio + flops += self.layer_info_dict[idx]['flops'] * c * n + # add buffer computation + flops += self._get_buffer_flops(idx) * c # only related to input channel reduction + return flops + + def _cur_reduced(self): + # return the reduced weight + reduced = self.org_flops - self._cur_flops() + return reduced + + def _build_index(self): + """ + Build following information/data for later pruning: + self.prunable_idx: layer indices for pruable layers, the index values are the index + of list(self.model.modules()). Pruable layers are pointwise Conv2d layers and Linear + layers. + self.prunable_ops: prunable modules + self.buffer_idx: layer indices for buffer layers which refers the depthwise layers. + Each depthwise layer is always followd by a pointwise layer for both mobilenet and + mobilenetv2. The depthwise layer's filters are pruned when its next pointwise layer's + corresponding input channels are pruned. + self.shared_idx: layer indices for layers which share input. + For example: [[1,4], [8, 10, 15]] means layer 1 and 4 share same input, and layer + 8, 10 and 15 share another input. + self.org_channels: number of input channels for each layer + self.min_strategy_dict: key is layer index, value is a tuple, the first value is the minimum + action of input channel, the second value is the minimum action value of output channel. + self.strategy_dict: same as self.min_strategy_dict, but it will be updated later. + """ + self.prunable_idx = [] + self.prunable_ops = [] + self.layer_type_dict = {} + self.strategy_dict = {} + self.buffer_dict = {} + this_buffer_list = [] + self.org_channels = [] + # build index and the min strategy dict + for i, m in enumerate(self.model.modules()): + if isinstance(m, PrunerModuleWrapper): + m = m.module + if type(m) == nn.Conv2d and m.groups == m.in_channels: # depth-wise conv, buffer + this_buffer_list.append(i) + else: # really prunable + self.prunable_idx.append(i) + self.prunable_ops.append(m) + self.layer_type_dict[i] = type(m) + self.buffer_dict[i] = this_buffer_list + this_buffer_list = [] # empty + self.org_channels.append(m.in_channels if type(m) == nn.Conv2d else m.in_features) + + self.strategy_dict[i] = [self.lbound, self.lbound] + + self.strategy_dict[self.prunable_idx[0]][0] = 1 # modify the input + self.strategy_dict[self.prunable_idx[-1]][1] = 1 # modify the output + + self.shared_idx = [] + if self.args.model_type == 'mobilenetv2': # TODO: to be tested! Share index for residual connection + connected_idx = [4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32] # to be partitioned + last_ch = -1 + share_group = None + for c_idx in connected_idx: + if self.prunable_ops[c_idx].in_channels != last_ch: # new group + last_ch = self.prunable_ops[c_idx].in_channels + if share_group is not None: + self.shared_idx.append(share_group) + share_group = [c_idx] + else: # same group + share_group.append(c_idx) + self.shared_idx.append(share_group) + _logger.info('=> Conv layers to share channels: %s', self.shared_idx) + + self.min_strategy_dict = copy.deepcopy(self.strategy_dict) + + self.buffer_idx = [] + for _, v in self.buffer_dict.items(): + self.buffer_idx += v + + _logger.info('=> Prunable layer idx: %s', self.prunable_idx) + _logger.info('=> Buffer layer idx: %s', self.buffer_idx) + _logger.info('=> Shared idx: %s', self.shared_idx) + _logger.info('=> Initial min strategy dict: %s', self.min_strategy_dict) + + # added for supporting residual connections during pruning + self.visited = [False] * len(self.prunable_idx) + self.index_buffer = {} + + def _extract_layer_information(self): + m_list = list(self.model.modules()) + + self.data_saver = [] + self.layer_info_dict = dict() + self.wsize_list = [] + self.flops_list = [] + + from .lib.utils import measure_layer_for_pruning + + # extend the forward fn to record layer info + def new_forward(m): + def lambda_forward(x): + m.input_feat = x.clone() + #TODO replace this flops counter with nni.compression.torch.utils.counter.count_flops_params + measure_layer_for_pruning(m, x) + y = m.old_forward(x) + m.output_feat = y.clone() + return y + + return lambda_forward + + device = None + for idx in self.prunable_idx + self.buffer_idx: # get all + m = m_list[idx] + m.old_forward = m.forward + m.forward = new_forward(m) + if device is None and type(m) == PrunerModuleWrapper: + device = m.module.weight.device + + # now let the image flow + _logger.info('=> Extracting information...') + with torch.no_grad(): + for i_b, (inputs, target) in enumerate(self._val_loader): # use image from train set + if i_b == self.n_calibration_batches: + break + self.data_saver.append((inputs.clone(), target.clone())) + input_var = torch.autograd.Variable(inputs).to(device) + + # inference and collect stats + _ = self.model(input_var) + + if i_b == 0: # first batch + for idx in self.prunable_idx + self.buffer_idx: + self.layer_info_dict[idx] = dict() + self.layer_info_dict[idx]['params'] = m_list[idx].params + self.layer_info_dict[idx]['flops'] = m_list[idx].flops + self.wsize_list.append(m_list[idx].params) + self.flops_list.append(m_list[idx].flops) + _logger.info('flops: %s', self.flops_list) + for idx in self.prunable_idx: + f_in_np = m_list[idx].input_feat.data.cpu().numpy() + f_out_np = m_list[idx].output_feat.data.cpu().numpy() + if len(f_in_np.shape) == 4: # conv + if self.prunable_idx.index(idx) == 0: # first conv + f_in2save, f_out2save = None, None + elif m_list[idx].module.weight.size(3) > 1: # normal conv + f_in2save, f_out2save = f_in_np, f_out_np + else: # 1x1 conv + # assert f_out_np.shape[2] == f_in_np.shape[2] # now support k=3 + randx = np.random.randint(0, f_out_np.shape[2] - 0, self.n_points_per_layer) + randy = np.random.randint(0, f_out_np.shape[3] - 0, self.n_points_per_layer) + # input: [N, C, H, W] + self.layer_info_dict[idx][(i_b, 'randx')] = randx.copy() + self.layer_info_dict[idx][(i_b, 'randy')] = randy.copy() + + f_in2save = f_in_np[:, :, randx, randy].copy().transpose(0, 2, 1)\ + .reshape(self.batch_size * self.n_points_per_layer, -1) + + f_out2save = f_out_np[:, :, randx, randy].copy().transpose(0, 2, 1) \ + .reshape(self.batch_size * self.n_points_per_layer, -1) + else: + assert len(f_in_np.shape) == 2 + f_in2save = f_in_np.copy() + f_out2save = f_out_np.copy() + if 'input_feat' not in self.layer_info_dict[idx]: + self.layer_info_dict[idx]['input_feat'] = f_in2save + self.layer_info_dict[idx]['output_feat'] = f_out2save + else: + self.layer_info_dict[idx]['input_feat'] = np.vstack( + (self.layer_info_dict[idx]['input_feat'], f_in2save)) + self.layer_info_dict[idx]['output_feat'] = np.vstack( + (self.layer_info_dict[idx]['output_feat'], f_out2save)) + + def _build_state_embedding(self): + # build the static part of the state embedding + _logger.info('Building state embedding...') + layer_embedding = [] + module_list = list(self.model.modules()) + for i, ind in enumerate(self.prunable_idx): + m = module_list[ind].module + this_state = [] + if type(m) == nn.Conv2d: + this_state.append(i) # index + this_state.append(0) # layer type, 0 for conv + this_state.append(m.in_channels) # in channels + this_state.append(m.out_channels) # out channels + this_state.append(m.stride[0]) # stride + this_state.append(m.kernel_size[0]) # kernel size + this_state.append(np.prod(m.weight.size())) # weight size + elif type(m) == nn.Linear: + this_state.append(i) # index + this_state.append(1) # layer type, 1 for fc + this_state.append(m.in_features) # in channels + this_state.append(m.out_features) # out channels + this_state.append(0) # stride + this_state.append(1) # kernel size + this_state.append(np.prod(m.weight.size())) # weight size + + # this 3 features need to be changed later + this_state.append(0.) # reduced + this_state.append(0.) # rest + this_state.append(1.) # a_{t-1} + layer_embedding.append(np.array(this_state)) + + # normalize the state + layer_embedding = np.array(layer_embedding, 'float') + _logger.info('=> shape of embedding (n_layer * n_dim): %s', layer_embedding.shape) + assert len(layer_embedding.shape) == 2, layer_embedding.shape + for i in range(layer_embedding.shape[1]): + fmin = min(layer_embedding[:, i]) + fmax = max(layer_embedding[:, i]) + if fmax - fmin > 0: + layer_embedding[:, i] = (layer_embedding[:, i] - fmin) / (fmax - fmin) + + self.layer_embedding = layer_embedding + diff --git a/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/pruning/amc/lib/__init__.py b/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/pruning/amc/lib/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/pruning/amc/lib/agent.py b/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/pruning/amc/lib/agent.py new file mode 100644 index 0000000000000000000000000000000000000000..fe066301b8b5e4325f92a1b98885ae547a7ecee3 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/pruning/amc/lib/agent.py @@ -0,0 +1,232 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import numpy as np + +import torch +import torch.nn as nn +from torch.optim import Adam + +from .memory import SequentialMemory +from .utils import to_numpy, to_tensor + +criterion = nn.MSELoss() +USE_CUDA = torch.cuda.is_available() + + +class Actor(nn.Module): + def __init__(self, nb_states, nb_actions, hidden1=400, hidden2=300): + super(Actor, self).__init__() + self.fc1 = nn.Linear(nb_states, hidden1) + self.fc2 = nn.Linear(hidden1, hidden2) + self.fc3 = nn.Linear(hidden2, nb_actions) + self.relu = nn.ReLU() + self.sigmoid = nn.Sigmoid() + + def forward(self, x): + out = self.fc1(x) + out = self.relu(out) + out = self.fc2(out) + out = self.relu(out) + out = self.fc3(out) + out = self.sigmoid(out) + return out + + +class Critic(nn.Module): + def __init__(self, nb_states, nb_actions, hidden1=400, hidden2=300): + super(Critic, self).__init__() + self.fc11 = nn.Linear(nb_states, hidden1) + self.fc12 = nn.Linear(nb_actions, hidden1) + self.fc2 = nn.Linear(hidden1, hidden2) + self.fc3 = nn.Linear(hidden2, 1) + self.relu = nn.ReLU() + + def forward(self, xs): + x, a = xs + out = self.fc11(x) + self.fc12(a) + out = self.relu(out) + out = self.fc2(out) + out = self.relu(out) + out = self.fc3(out) + return out + + +class DDPG(object): + def __init__(self, nb_states, nb_actions, args): + + self.nb_states = nb_states + self.nb_actions = nb_actions + + # Create Actor and Critic Network + net_cfg = { + 'hidden1': args.hidden1, + 'hidden2': args.hidden2, + # 'init_w': args.init_w + } + self.actor = Actor(self.nb_states, self.nb_actions, **net_cfg) + self.actor_target = Actor(self.nb_states, self.nb_actions, **net_cfg) + self.actor_optim = Adam(self.actor.parameters(), lr=args.lr_a) + + self.critic = Critic(self.nb_states, self.nb_actions, **net_cfg) + self.critic_target = Critic(self.nb_states, self.nb_actions, **net_cfg) + self.critic_optim = Adam(self.critic.parameters(), lr=args.lr_c) + + self.hard_update(self.actor_target, self.actor) # Make sure target is with the same weight + self.hard_update(self.critic_target, self.critic) + + # Create replay buffer + self.memory = SequentialMemory(limit=args.rmsize, window_length=args.window_length) + # self.random_process = OrnsteinUhlenbeckProcess(size=nb_actions, theta=args.ou_theta, mu=args.ou_mu, + # sigma=args.ou_sigma) + + # Hyper-parameters + self.batch_size = args.bsize + self.tau = args.tau + self.discount = args.discount + self.depsilon = 1.0 / args.epsilon + self.lbound = 0. # args.lbound + self.rbound = 1. # args.rbound + + # noise + self.init_delta = args.init_delta + self.delta_decay = args.delta_decay + self.warmup = args.warmup + + # + self.epsilon = 1.0 + # self.s_t = None # Most recent state + # self.a_t = None # Most recent action + self.is_training = True + + # + if USE_CUDA: self.cuda() + + # moving average baseline + self.moving_average = None + self.moving_alpha = 0.5 # based on batch, so small + + def update_policy(self): + # Sample batch + state_batch, action_batch, reward_batch, \ + next_state_batch, terminal_batch = self.memory.sample_and_split(self.batch_size) + + # normalize the reward + batch_mean_reward = np.mean(reward_batch) + if self.moving_average is None: + self.moving_average = batch_mean_reward + else: + self.moving_average += self.moving_alpha * (batch_mean_reward - self.moving_average) + reward_batch -= self.moving_average + # if reward_batch.std() > 0: + # reward_batch /= reward_batch.std() + + # Prepare for the target q batch + with torch.no_grad(): + next_q_values = self.critic_target([ + to_tensor(next_state_batch), + self.actor_target(to_tensor(next_state_batch)), + ]) + + target_q_batch = to_tensor(reward_batch) + \ + self.discount * to_tensor(terminal_batch.astype(np.float)) * next_q_values + + # Critic update + self.critic.zero_grad() + + q_batch = self.critic([to_tensor(state_batch), to_tensor(action_batch)]) + + value_loss = criterion(q_batch, target_q_batch) + value_loss.backward() + self.critic_optim.step() + + # Actor update + self.actor.zero_grad() + + policy_loss = -self.critic([ # pylint: disable=all + to_tensor(state_batch), + self.actor(to_tensor(state_batch)) + ]) + + policy_loss = policy_loss.mean() + policy_loss.backward() + self.actor_optim.step() + + # Target update + self.soft_update(self.actor_target, self.actor) + self.soft_update(self.critic_target, self.critic) + + def eval(self): + self.actor.eval() + self.actor_target.eval() + self.critic.eval() + self.critic_target.eval() + + def cuda(self): + self.actor.cuda() + self.actor_target.cuda() + self.critic.cuda() + self.critic_target.cuda() + + def observe(self, r_t, s_t, s_t1, a_t, done): + if self.is_training: + self.memory.append(s_t, a_t, r_t, done) # save to memory + # self.s_t = s_t1 + + def random_action(self): + action = np.random.uniform(self.lbound, self.rbound, self.nb_actions) + # self.a_t = action + return action + + def select_action(self, s_t, episode): + # assert episode >= self.warmup, 'Episode: {} warmup: {}'.format(episode, self.warmup) + action = to_numpy(self.actor(to_tensor(np.array(s_t).reshape(1, -1)))).squeeze(0) + delta = self.init_delta * (self.delta_decay ** (episode - self.warmup)) + # action += self.is_training * max(self.epsilon, 0) * self.random_process.sample() + action = self.sample_from_truncated_normal_distribution(lower=self.lbound, upper=self.rbound, mu=action, sigma=delta) + action = np.clip(action, self.lbound, self.rbound) + + # self.a_t = action + return action + + def reset(self, obs): + pass + # self.s_t = obs + # self.random_process.reset_states() + + def load_weights(self, output): + if output is None: return + + self.actor.load_state_dict( + torch.load('{}/actor.pkl'.format(output)) + ) + + self.critic.load_state_dict( + torch.load('{}/critic.pkl'.format(output)) + ) + + def save_model(self, output): + torch.save( + self.actor.state_dict(), + '{}/actor.pkl'.format(output) + ) + torch.save( + self.critic.state_dict(), + '{}/critic.pkl'.format(output) + ) + + def soft_update(self, target, source): + for target_param, param in zip(target.parameters(), source.parameters()): + target_param.data.copy_( + target_param.data * (1.0 - self.tau) + param.data * self.tau + ) + + def hard_update(self, target, source): + for target_param, param in zip(target.parameters(), source.parameters()): + target_param.data.copy_(param.data) + + def sample_from_truncated_normal_distribution(self, lower, upper, mu, sigma, size=1): + from scipy import stats + return stats.truncnorm.rvs((lower-mu)/sigma, (upper-mu)/sigma, loc=mu, scale=sigma, size=size) + + diff --git a/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/pruning/amc/lib/memory.py b/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/pruning/amc/lib/memory.py new file mode 100644 index 0000000000000000000000000000000000000000..57bbcfceb86a20092968c9dc75a618221e119174 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/pruning/amc/lib/memory.py @@ -0,0 +1,227 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from __future__ import absolute_import +from collections import deque, namedtuple +import warnings +import random + +import numpy as np + +# [reference] https://github.com/matthiasplappert/keras-rl/blob/master/rl/memory.py + +# This is to be understood as a transition: Given `state0`, performing `action` +# yields `reward` and results in `state1`, which might be `terminal`. +Experience = namedtuple('Experience', 'state0, action, reward, state1, terminal1') + + +def sample_batch_indexes(low, high, size): + if high - low >= size: + # We have enough data. Draw without replacement, that is each index is unique in the + # batch. We cannot use `np.random.choice` here because it is horribly inefficient as + # the memory grows. See https://github.com/numpy/numpy/issues/2764 for a discussion. + # `random.sample` does the same thing (drawing without replacement) and is way faster. + r = range(low, high) + batch_idxs = random.sample(r, size) + else: + # Not enough data. Help ourselves with sampling from the range, but the same index + # can occur multiple times. This is not good and should be avoided by picking a + # large enough warm-up phase. + warnings.warn( + 'Not enough entries to sample without replacement. ' + 'Consider increasing your warm-up phase to avoid oversampling!') + batch_idxs = np.random.random_integers(low, high - 1, size=size) + assert len(batch_idxs) == size + return batch_idxs + + +class RingBuffer(object): + def __init__(self, maxlen): + self.maxlen = maxlen + self.start = 0 + self.length = 0 + self.data = [None for _ in range(maxlen)] + + def __len__(self): + return self.length + + def __getitem__(self, idx): + if idx < 0 or idx >= self.length: + raise KeyError() + return self.data[(self.start + idx) % self.maxlen] + + def append(self, v): + if self.length < self.maxlen: + # We have space, simply increase the length. + self.length += 1 + elif self.length == self.maxlen: + # No space, "remove" the first item. + self.start = (self.start + 1) % self.maxlen + else: + # This should never happen. + raise RuntimeError() + self.data[(self.start + self.length - 1) % self.maxlen] = v + + +def zeroed_observation(observation): + if hasattr(observation, 'shape'): + return np.zeros(observation.shape) + elif hasattr(observation, '__iter__'): + out = [] + for x in observation: + out.append(zeroed_observation(x)) + return out + else: + return 0. + + +class Memory(object): + def __init__(self, window_length, ignore_episode_boundaries=False): + self.window_length = window_length + self.ignore_episode_boundaries = ignore_episode_boundaries + + self.recent_observations = deque(maxlen=window_length) + self.recent_terminals = deque(maxlen=window_length) + + def sample(self, batch_size, batch_idxs=None): + raise NotImplementedError() + + def append(self, observation, action, reward, terminal, training=True): + self.recent_observations.append(observation) + self.recent_terminals.append(terminal) + + def get_recent_state(self, current_observation): + # This code is slightly complicated by the fact that subsequent observations might be + # from different episodes. We ensure that an experience never spans multiple episodes. + # This is probably not that important in practice but it seems cleaner. + state = [current_observation] + idx = len(self.recent_observations) - 1 + for offset in range(0, self.window_length - 1): + current_idx = idx - offset + current_terminal = self.recent_terminals[current_idx - 1] if current_idx - 1 >= 0 else False + if current_idx < 0 or (not self.ignore_episode_boundaries and current_terminal): + # The previously handled observation was terminal, don't add the current one. + # Otherwise we would leak into a different episode. + break + state.insert(0, self.recent_observations[current_idx]) + while len(state) < self.window_length: + state.insert(0, zeroed_observation(state[0])) + return state + + def get_config(self): + config = { + 'window_length': self.window_length, + 'ignore_episode_boundaries': self.ignore_episode_boundaries, + } + return config + + +class SequentialMemory(Memory): + def __init__(self, limit, **kwargs): + super(SequentialMemory, self).__init__(**kwargs) + + self.limit = limit + + # Do not use deque to implement the memory. This data structure may seem convenient but + # it is way too slow on random access. Instead, we use our own ring buffer implementation. + self.actions = RingBuffer(limit) + self.rewards = RingBuffer(limit) + self.terminals = RingBuffer(limit) + self.observations = RingBuffer(limit) + + def sample(self, batch_size, batch_idxs=None): + if batch_idxs is None: + # Draw random indexes such that we have at least a single entry before each + # index. + batch_idxs = sample_batch_indexes(0, self.nb_entries - 1, size=batch_size) + batch_idxs = np.array(batch_idxs) + 1 + assert np.min(batch_idxs) >= 1 + assert np.max(batch_idxs) < self.nb_entries + assert len(batch_idxs) == batch_size + + # Create experiences + experiences = [] + for idx in batch_idxs: + terminal0 = self.terminals[idx - 2] if idx >= 2 else False + while terminal0: + # Skip this transition because the environment was reset here. Select a new, random + # transition and use this instead. This may cause the batch to contain the same + # transition twice. + idx = sample_batch_indexes(1, self.nb_entries, size=1)[0] + terminal0 = self.terminals[idx - 2] if idx >= 2 else False + assert 1 <= idx < self.nb_entries + + # This code is slightly complicated by the fact that subsequent observations might be + # from different episodes. We ensure that an experience never spans multiple episodes. + # This is probably not that important in practice but it seems cleaner. + state0 = [self.observations[idx - 1]] + for offset in range(0, self.window_length - 1): + current_idx = idx - 2 - offset + current_terminal = self.terminals[current_idx - 1] if current_idx - 1 > 0 else False + if current_idx < 0 or (not self.ignore_episode_boundaries and current_terminal): + # The previously handled observation was terminal, don't add the current one. + # Otherwise we would leak into a different episode. + break + state0.insert(0, self.observations[current_idx]) + while len(state0) < self.window_length: + state0.insert(0, zeroed_observation(state0[0])) + action = self.actions[idx - 1] + reward = self.rewards[idx - 1] + terminal1 = self.terminals[idx - 1] + + # Okay, now we need to create the follow-up state. This is state0 shifted on timestep + # to the right. Again, we need to be careful to not include an observation from the next + # episode if the last state is terminal. + state1 = [np.copy(x) for x in state0[1:]] + state1.append(self.observations[idx]) + + assert len(state0) == self.window_length + assert len(state1) == len(state0) + experiences.append(Experience(state0=state0, action=action, reward=reward, + state1=state1, terminal1=terminal1)) + assert len(experiences) == batch_size + return experiences + + def sample_and_split(self, batch_size, batch_idxs=None): + experiences = self.sample(batch_size, batch_idxs) + + state0_batch = [] + reward_batch = [] + action_batch = [] + terminal1_batch = [] + state1_batch = [] + for e in experiences: + state0_batch.append(e.state0) + state1_batch.append(e.state1) + reward_batch.append(e.reward) + action_batch.append(e.action) + terminal1_batch.append(0. if e.terminal1 else 1.) + + # Prepare and validate parameters. + state0_batch = np.array(state0_batch, 'double').reshape(batch_size, -1) + state1_batch = np.array(state1_batch, 'double').reshape(batch_size, -1) + terminal1_batch = np.array(terminal1_batch, 'double').reshape(batch_size, -1) + reward_batch = np.array(reward_batch, 'double').reshape(batch_size, -1) + action_batch = np.array(action_batch, 'double').reshape(batch_size, -1) + + return state0_batch, action_batch, reward_batch, state1_batch, terminal1_batch + + def append(self, observation, action, reward, terminal, training=True): + super(SequentialMemory, self).append(observation, action, reward, terminal, training=training) + + # This needs to be understood as follows: in `observation`, take `action`, obtain `reward` + # and weather the next state is `terminal` or not. + if training: + self.observations.append(observation) + self.actions.append(action) + self.rewards.append(reward) + self.terminals.append(terminal) + + @property + def nb_entries(self): + return len(self.observations) + + def get_config(self): + config = super(SequentialMemory, self).get_config() + config['limit'] = self.limit + return config diff --git a/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/pruning/amc/lib/net_measure.py b/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/pruning/amc/lib/net_measure.py new file mode 100644 index 0000000000000000000000000000000000000000..2c9e815116288c11de98b4061b4dbfc3df029a7e --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/pruning/amc/lib/net_measure.py @@ -0,0 +1,123 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import torch + +# [reference] https://github.com/ShichenLiu/CondenseNet/blob/master/utils.py + + +def get_num_gen(gen): + return sum(1 for _ in gen) + + +def is_leaf(model): + return get_num_gen(model.children()) == 0 + + +def get_layer_info(layer): + layer_str = str(layer) + type_name = layer_str[:layer_str.find('(')].strip() + return type_name + + +def get_layer_param(model): + import operator + import functools + + return sum([functools.reduce(operator.mul, i.size(), 1) for i in model.parameters()]) + +count_ops = 0 +count_params = 0 + +def measure_layer(layer, x): + global count_ops, count_params + delta_ops = 0 + delta_params = 0 + multi_add = 1 + type_name = get_layer_info(layer) + + # ops_conv + if type_name in ['Conv2d']: + out_h = int((x.size()[2] + 2 * layer.padding[0] - layer.kernel_size[0]) / + layer.stride[0] + 1) + out_w = int((x.size()[3] + 2 * layer.padding[1] - layer.kernel_size[1]) / + layer.stride[1] + 1) + delta_ops = layer.in_channels * layer.out_channels * layer.kernel_size[0] * \ + layer.kernel_size[1] * out_h * out_w / layer.groups * multi_add + delta_params = get_layer_param(layer) + + # ops_nonlinearity + elif type_name in ['ReLU']: + delta_ops = x.numel() / x.size(0) + delta_params = get_layer_param(layer) + + # ops_pooling + elif type_name in ['AvgPool2d']: + in_w = x.size()[2] + kernel_ops = layer.kernel_size * layer.kernel_size + out_w = int((in_w + 2 * layer.padding - layer.kernel_size) / layer.stride + 1) + out_h = int((in_w + 2 * layer.padding - layer.kernel_size) / layer.stride + 1) + delta_ops = x.size()[1] * out_w * out_h * kernel_ops + delta_params = get_layer_param(layer) + + elif type_name in ['AdaptiveAvgPool2d']: + delta_ops = x.size()[1] * x.size()[2] * x.size()[3] + delta_params = get_layer_param(layer) + + # ops_linear + elif type_name in ['Linear']: + weight_ops = layer.weight.numel() * multi_add + bias_ops = layer.bias.numel() + delta_ops = weight_ops + bias_ops + delta_params = get_layer_param(layer) + + # ops_nothing + elif type_name in ['BatchNorm2d', 'Dropout2d', 'DropChannel', 'Dropout']: + delta_params = get_layer_param(layer) + + # unknown layer type + else: + delta_params = get_layer_param(layer) + + count_ops += delta_ops + count_params += delta_params + + return + + +def measure_model(model, H, W, device): + global count_ops, count_params + count_ops = 0 + count_params = 0 + data = torch.zeros(2, 3, H, W).to(device) + + def should_measure(x): + return is_leaf(x) + + def modify_forward(model): + for child in model.children(): + if should_measure(child): + def new_forward(m): + def lambda_forward(x): + measure_layer(m, x) + return m.old_forward(x) + return lambda_forward + child.old_forward = child.forward + child.forward = new_forward(child) + else: + modify_forward(child) + + def restore_forward(model): + for child in model.children(): + # leaf node + if is_leaf(child) and hasattr(child, 'old_forward'): + child.forward = child.old_forward + child.old_forward = None + else: + restore_forward(child) + + modify_forward(model) + model.forward(data) + restore_forward(model) + + return count_ops, count_params diff --git a/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/pruning/amc/lib/utils.py b/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/pruning/amc/lib/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..f875e8d7d9b363d848aa2424b4f06ddaeb1dc1a6 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/pruning/amc/lib/utils.py @@ -0,0 +1,113 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +import torch + +class TextLogger(object): + """Write log immediately to the disk""" + def __init__(self, filepath): + self.f = open(filepath, 'w') + self.fid = self.f.fileno() + self.filepath = filepath + + def close(self): + self.f.close() + + def write(self, content): + self.f.write(content) + self.f.flush() + os.fsync(self.fid) + + def write_buf(self, content): + self.f.write(content) + + def print_and_write(self, content): + print(content) + self.write(content+'\n') + +def to_numpy(var): + use_cuda = torch.cuda.is_available() + return var.cpu().data.numpy() if use_cuda else var.data.numpy() + + +def to_tensor(ndarray, requires_grad=False): # return a float tensor by default + tensor = torch.from_numpy(ndarray).float() # by default does not require grad + if requires_grad: + tensor.requires_grad_() + return tensor.cuda() if torch.cuda.is_available() else tensor + + +def measure_layer_for_pruning(wrapper, x): + def get_layer_type(layer): + layer_str = str(layer) + return layer_str[:layer_str.find('(')].strip() + + def get_layer_param(model): + import operator + import functools + + return sum([functools.reduce(operator.mul, i.size(), 1) for i in model.parameters()]) + + multi_add = 1 + layer = wrapper.module + type_name = get_layer_type(layer) + + # ops_conv + if type_name in ['Conv2d']: + out_h = int((x.size()[2] + 2 * layer.padding[0] - layer.kernel_size[0]) / + layer.stride[0] + 1) + out_w = int((x.size()[3] + 2 * layer.padding[1] - layer.kernel_size[1]) / + layer.stride[1] + 1) + wrapper.flops = layer.in_channels * layer.out_channels * layer.kernel_size[0] * \ + layer.kernel_size[1] * out_h * out_w / layer.groups * multi_add + wrapper.params = get_layer_param(layer) + # ops_linear + elif type_name in ['Linear']: + weight_ops = layer.weight.numel() * multi_add + bias_ops = layer.bias.numel() + wrapper.flops = weight_ops + bias_ops + wrapper.params = get_layer_param(layer) + return + + +def least_square_sklearn(X, Y): + from sklearn.linear_model import LinearRegression + reg = LinearRegression(fit_intercept=False) + reg.fit(X, Y) + return reg.coef_ + + +def get_output_folder(parent_dir, env_name): + """Return save folder. + Assumes folders in the parent_dir have suffix -run{run + number}. Finds the highest run number and sets the output folder + to that number + 1. This is just convenient so that if you run the + same script multiple times tensorboard can plot all of the results + on the same plots with different names. + Parameters + ---------- + parent_dir: str + Path of the directory containing all experiment runs. + Returns + ------- + parent_dir/run_dir + Path to this run's save directory. + """ + os.makedirs(parent_dir, exist_ok=True) + experiment_id = 0 + for folder_name in os.listdir(parent_dir): + if not os.path.isdir(os.path.join(parent_dir, folder_name)): + continue + try: + folder_name = int(folder_name.split('-run')[-1]) + if folder_name > experiment_id: + experiment_id = folder_name + except: + pass + experiment_id += 1 + + parent_dir = os.path.join(parent_dir, env_name) + parent_dir = parent_dir + '-run{}'.format(experiment_id) + os.makedirs(parent_dir, exist_ok=True) + return parent_dir diff --git a/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/pruning/apply_compression.py b/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/pruning/apply_compression.py new file mode 100644 index 0000000000000000000000000000000000000000..8e6b023f5b90b8483e21bd0bc575b19b4a4df023 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/pruning/apply_compression.py @@ -0,0 +1,29 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +import torch + +logger = logging.getLogger('torch apply compression') + +def apply_compression_results(model, masks_file, map_location=None): + """ + Apply the masks from ```masks_file``` to the model + Note: this API is for inference, because it simply multiplies weights with + corresponding masks when this API is called. + + Parameters + ---------- + model : torch.nn.Module + The model to be compressed + masks_file : str + The path of the mask file + map_location : str + the device on which masks are placed, same to map_location in ```torch.load``` + """ + masks = torch.load(masks_file, map_location) + for name, module in model.named_modules(): + if name in masks: + module.weight.data = module.weight.data.mul_(masks[name]['weight']) + if hasattr(module, 'bias') and module.bias is not None and 'bias' in masks[name]: + module.bias.data = module.bias.data.mul_(masks[name]['bias']) \ No newline at end of file diff --git a/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/pruning/auto_compress_pruner.py b/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/pruning/auto_compress_pruner.py new file mode 100644 index 0000000000000000000000000000000000000000..207a8aa2f9b0a963fa3a38e7a11940ba9c1f6cda --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/pruning/auto_compress_pruner.py @@ -0,0 +1,237 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +import os +import copy +import torch +from schema import And, Optional + +from nni.utils import OptimizeMode +from nni.compression.pytorch import ModelSpeedup + +from nni.compression.pytorch.compressor import Pruner +from nni.compression.pytorch.utils.config_validation import CompressorSchema +from .simulated_annealing_pruner import SimulatedAnnealingPruner +from .iterative_pruner import ADMMPruner + +_logger = logging.getLogger(__name__) + + +class AutoCompressPruner(Pruner): + """ + A Pytorch implementation of AutoCompress pruning algorithm. + + Parameters + ---------- + model : pytorch model + The model to be pruned. + config_list : list + Supported keys: + - sparsity : The target overall sparsity. + - op_types : The operation type to prune. + trainer : function + Function used for the first subproblem of ADMM Pruner. + Users should write this function as a normal function to train the Pytorch model + and include `model, optimizer, criterion, epoch` as function arguments. + criterion: function + Function used to calculate the loss between the target and the output. By default, we use CrossEntropyLoss. + For example, you can use ``torch.nn.CrossEntropyLoss()`` as input. + evaluator : function + function to evaluate the pruned model. + This function should include `model` as the only parameter, and returns a scalar value. + Example:: + + def evaluator(model): + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + val_loader = ... + model.eval() + correct = 0 + with torch.no_grad(): + for data, target in val_loader: + data, target = data.to(device), target.to(device) + output = model(data) + # get the index of the max log-probability + pred = output.argmax(dim=1, keepdim=True) + correct += pred.eq(target.view_as(pred)).sum().item() + accuracy = correct / len(val_loader.dataset) + return accuracy + dummy_input : pytorch tensor + The dummy input for ```jit.trace```, users should put it on right device before pass in. + num_iterations : int + Number of overall iterations. + optimize_mode : str + optimize mode, `maximize` or `minimize`, by default `maximize`. + base_algo : str + Base pruning algorithm. `level`, `l1`, `l2` or `fpgm`, by default `l1`. Given the sparsity distribution among + the ops, the assigned `base_algo` is used to decide which filters/channels/weights to prune. + start_temperature : float + Start temperature of the simulated annealing process. + stop_temperature : float + Stop temperature of the simulated annealing process. + cool_down_rate : float + Cool down rate of the temperature. + perturbation_magnitude : float + Initial perturbation magnitude to the sparsities. The magnitude decreases with current temperature. + admm_num_iterations : int + Number of iterations of ADMM Pruner. + admm_epochs_per_iteration : int + Training epochs of the first optimization subproblem of ADMMPruner. + row : float + Penalty parameters for ADMM training. + experiment_data_dir : string + PATH to store temporary experiment data. + """ + + def __init__(self, model, config_list, trainer, evaluator, dummy_input, criterion=torch.nn.CrossEntropyLoss(), + num_iterations=3, optimize_mode='maximize', base_algo='l1', + # SimulatedAnnealing related + start_temperature=100, stop_temperature=20, cool_down_rate=0.9, perturbation_magnitude=0.35, + # ADMM related + admm_num_iterations=30, admm_epochs_per_iteration=5, row=1e-4, + experiment_data_dir='./'): + # original model + self._model_to_prune = model + self._base_algo = base_algo + + self._trainer = trainer + self._criterion = criterion + self._evaluator = evaluator + self._dummy_input = dummy_input + self._num_iterations = num_iterations + self._optimize_mode = OptimizeMode(optimize_mode) + + # hyper parameters for SA algorithm + self._start_temperature = start_temperature + self._stop_temperature = stop_temperature + self._cool_down_rate = cool_down_rate + self._perturbation_magnitude = perturbation_magnitude + + # hyper parameters for ADMM algorithm + self._admm_num_iterations = admm_num_iterations + self._admm_epochs_per_iteration = admm_epochs_per_iteration + self._row = row + + # overall pruning rate + self._sparsity = config_list[0]['sparsity'] + + self._experiment_data_dir = experiment_data_dir + if not os.path.exists(self._experiment_data_dir): + os.makedirs(self._experiment_data_dir) + + def validate_config(self, model, config_list): + """ + Parameters + ---------- + model : torch.nn.Module + Model to be pruned + config_list : list + List on pruning configs + """ + + if self._base_algo == 'level': + schema = CompressorSchema([{ + 'sparsity': And(float, lambda n: 0 < n < 1), + Optional('op_types'): [str], + Optional('op_names'): [str], + }], model, _logger) + elif self._base_algo in ['l1', 'l2', 'fpgm']: + schema = CompressorSchema([{ + 'sparsity': And(float, lambda n: 0 < n < 1), + 'op_types': ['Conv2d'], + Optional('op_names'): [str] + }], model, _logger) + + schema.validate(config_list) + + def calc_mask(self, wrapper, **kwargs): + return None + + def compress(self): + """ + Compress the model with AutoCompress. + + Returns + ------- + torch.nn.Module + model with specified modules compressed. + """ + _logger.info('Starting AutoCompress pruning...') + + sparsity_each_round = 1 - pow(1 - self._sparsity, 1 / self._num_iterations) + + for i in range(self._num_iterations): + _logger.info('Pruning iteration: %d', i) + _logger.info('Target sparsity this round: %s', + 1 - pow(1 - sparsity_each_round, i + 1)) + + # SimulatedAnnealingPruner + _logger.info( + 'Generating sparsities with SimulatedAnnealingPruner...') + SApruner = SimulatedAnnealingPruner( + model=copy.deepcopy(self._model_to_prune), + config_list=[ + {"sparsity": sparsity_each_round, "op_types": ['Conv2d']}], + evaluator=self._evaluator, + optimize_mode=self._optimize_mode, + base_algo=self._base_algo, + start_temperature=self._start_temperature, + stop_temperature=self._stop_temperature, + cool_down_rate=self._cool_down_rate, + perturbation_magnitude=self._perturbation_magnitude, + experiment_data_dir=self._experiment_data_dir) + config_list = SApruner.compress(return_config_list=True) + _logger.info("Generated config_list : %s", config_list) + + # ADMMPruner + _logger.info('Performing structured pruning with ADMMPruner...') + ADMMpruner = ADMMPruner( + model=copy.deepcopy(self._model_to_prune), + config_list=config_list, + criterion=self._criterion, + trainer=self._trainer, + num_iterations=self._admm_num_iterations, + epochs_per_iteration=self._admm_epochs_per_iteration, + row=self._row, + base_algo=self._base_algo) + ADMMpruner.compress() + + ADMMpruner.export_model(os.path.join(self._experiment_data_dir, 'model_admm_masked.pth'), os.path.join( + self._experiment_data_dir, 'mask.pth')) + + # use speed up to prune the model before next iteration, + # because SimulatedAnnealingPruner & ADMMPruner don't take masked models + self._model_to_prune.load_state_dict(torch.load(os.path.join( + self._experiment_data_dir, 'model_admm_masked.pth'))) + + masks_file = os.path.join(self._experiment_data_dir, 'mask.pth') + device = next(self._model_to_prune.parameters()).device + + _logger.info('Speeding up models...') + m_speedup = ModelSpeedup(self._model_to_prune, self._dummy_input, masks_file, device) + m_speedup.speedup_model() + + evaluation_result = self._evaluator(self._model_to_prune) + _logger.info('Evaluation result of the pruned model in iteration %d: %s', i, evaluation_result) + + _logger.info('----------Compression finished--------------') + + os.remove(os.path.join(self._experiment_data_dir, 'model_admm_masked.pth')) + os.remove(os.path.join(self._experiment_data_dir, 'mask.pth')) + + return self._model_to_prune + + def export_model(self, model_path, mask_path=None, onnx_path=None, input_shape=None, device=None): + _logger.info("AutoCompressPruner export directly the pruned model without mask") + + torch.save(self._model_to_prune.state_dict(), model_path) + _logger.info('Model state_dict saved to %s', model_path) + + if onnx_path is not None: + assert input_shape is not None, 'input_shape must be specified to export onnx model' + # input info needed + if device is None: + device = torch.device('cpu') + input_data = torch.Tensor(*input_shape) + torch.onnx.export(self._model_to_prune, input_data.to(device), onnx_path) + _logger.info('Model in onnx with input shape %s saved to %s', input_data.shape, onnx_path) diff --git a/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/pruning/constants.py b/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/pruning/constants.py new file mode 100644 index 0000000000000000000000000000000000000000..24b84340cc6f5b0c362ab45288638a43926809b7 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/pruning/constants.py @@ -0,0 +1,18 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + + +from . import LevelPrunerMasker, SlimPrunerMasker, L1FilterPrunerMasker, \ + L2FilterPrunerMasker, FPGMPrunerMasker, TaylorFOWeightFilterPrunerMasker, \ + ActivationAPoZRankFilterPrunerMasker, ActivationMeanRankFilterPrunerMasker + +MASKER_DICT = { + 'level': LevelPrunerMasker, + 'slim': SlimPrunerMasker, + 'l1': L1FilterPrunerMasker, + 'l2': L2FilterPrunerMasker, + 'fpgm': FPGMPrunerMasker, + 'taylorfo': TaylorFOWeightFilterPrunerMasker, + 'apoz': ActivationAPoZRankFilterPrunerMasker, + 'mean_activation': ActivationMeanRankFilterPrunerMasker +} diff --git a/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/pruning/constants_pruner.py b/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/pruning/constants_pruner.py new file mode 100644 index 0000000000000000000000000000000000000000..55ba9506f3ba6a528a6dfdf87b6c878d61176c9f --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/pruning/constants_pruner.py @@ -0,0 +1,12 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + + +from .one_shot_pruner import LevelPruner, L1FilterPruner, L2FilterPruner, FPGMPruner + +PRUNER_DICT = { + 'level': LevelPruner, + 'l1': L1FilterPruner, + 'l2': L2FilterPruner, + 'fpgm': FPGMPruner +} diff --git a/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/pruning/dependency_aware_pruner.py b/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/pruning/dependency_aware_pruner.py new file mode 100644 index 0000000000000000000000000000000000000000..60f94ee317e8ab70f7079b8a8228da24b968ea10 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/pruning/dependency_aware_pruner.py @@ -0,0 +1,162 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +from schema import And, Optional, SchemaError +from .....common.graph_utils import TorchModuleGraph +from .....compression.pytorch.utils.shape_dependency import ChannelDependency, GroupDependency +from .....compression.pytorch.utils.config_validation import CompressorSchema +from .....compression.pytorch.compressor import Pruner +from .constants import MASKER_DICT + +__all__ = ['DependencyAwarePruner'] + +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) + + +class DependencyAwarePruner(Pruner): + """ + DependencyAwarePruner has two ways to calculate the masks + for conv layers. In the normal way, the DependencyAwarePruner + will calculate the mask of each layer separately. For example, each + conv layer determine which filters should be pruned according to its L1 + norm. In constrast, in the dependency-aware way, the layers that in a + dependency group will be pruned jointly and these layers will be forced + to prune the same channels. + """ + + def __init__(self, model, config_list, optimizer=None, pruning_algorithm='level', dependency_aware=False, + dummy_input=None, **algo_kwargs): + super().__init__(model, config_list=config_list, optimizer=optimizer) + + self.dependency_aware = dependency_aware + self.dummy_input = dummy_input + + if self.dependency_aware: + if not self._supported_dependency_aware(): + raise ValueError('This pruner does not support dependency aware!') + + errmsg = "When dependency_aware is set, the dummy_input should not be None" + assert self.dummy_input is not None, errmsg + # Get the TorchModuleGraph of the target model + # to trace the model, we need to unwrap the wrappers + self._unwrap_model() + self.graph = TorchModuleGraph(model, dummy_input) + self._wrap_model() + self.channel_depen = ChannelDependency( + traced_model=self.graph.trace) + self.group_depen = GroupDependency(traced_model=self.graph.trace) + self.channel_depen = self.channel_depen.dependency_sets + self.channel_depen = { + name: sets for sets in self.channel_depen for name in sets} + self.group_depen = self.group_depen.dependency_sets + + self.masker = MASKER_DICT[pruning_algorithm]( + model, self, **algo_kwargs) + # set the dependency-aware switch for the masker + self.masker.dependency_aware = dependency_aware + self.set_wrappers_attribute("if_calculated", False) + + def calc_mask(self, wrapper, wrapper_idx=None): + if not wrapper.if_calculated: + sparsity = wrapper.config['sparsity'] + masks = self.masker.calc_mask( + sparsity=sparsity, wrapper=wrapper, wrapper_idx=wrapper_idx) + + # masker.calc_mask returns None means calc_mask is not calculated sucessfully, can try later + if masks is not None: + wrapper.if_calculated = True + return masks + else: + return None + + def update_mask(self): + if not self.dependency_aware: + # if we use the normal way to update the mask, + # then call the update_mask of the father class + super(DependencyAwarePruner, self).update_mask() + else: + # if we update the mask in a dependency-aware way + # then we call _dependency_update_mask + self._dependency_update_mask() + + def validate_config(self, model, config_list): + schema = CompressorSchema([{ + Optional('sparsity'): And(float, lambda n: 0 < n < 1), + Optional('op_types'): ['Conv2d'], + Optional('op_names'): [str], + Optional('exclude'): bool + }], model, logger) + + schema.validate(config_list) + for config in config_list: + if 'exclude' not in config and 'sparsity' not in config: + raise SchemaError('Either sparisty or exclude must be specified!') + + def _supported_dependency_aware(self): + raise NotImplementedError + + def _dependency_calc_mask(self, wrappers, channel_dsets, wrappers_idx=None): + """ + calculate the masks for the conv layers in the same + channel dependecy set. All the layers passed in have + the same number of channels. + + Parameters + ---------- + wrappers: list + The list of the wrappers that in the same channel dependency + set. + wrappers_idx: list + The list of the indexes of wrapppers. + Returns + ------- + masks: dict + A dict object that contains the masks of the layers in this + dependency group, the key is the name of the convolutional layers. + """ + # The number of the groups for each conv layers + # Note that, this number may be different from its + # original number of groups of filters. + groups = [self.group_depen[_w.name] for _w in wrappers] + sparsities = [_w.config['sparsity'] for _w in wrappers] + masks = self.masker.calc_mask( + sparsities, wrappers, wrappers_idx, channel_dsets=channel_dsets, groups=groups) + if masks is not None: + # if masks is None, then the mask calculation fails. + # for example, in activation related maskers, we should + # pass enough batches of data to the model, so that the + # masks can be calculated successfully. + for _w in wrappers: + _w.if_calculated = True + return masks + + def _dependency_update_mask(self): + """ + In the original update_mask, the wraper of each layer will update its + own mask according to the sparsity specified in the config_list. However, in + the _dependency_update_mask, we may prune several layers at the same + time according the sparsities and the channel/group dependencies. + """ + name2wrapper = {x.name: x for x in self.get_modules_wrapper()} + wrapper2index = {x: i for i, x in enumerate(self.get_modules_wrapper())} + for wrapper in self.get_modules_wrapper(): + if wrapper.if_calculated: + continue + # find all the conv layers that have channel dependecy with this layer + # and prune all these layers at the same time. + _names = [x for x in self.channel_depen[wrapper.name]] + # logger.info('Pruning the dependent layers: %s', ','.join(_names)) + _wrappers = [name2wrapper[name] + for name in _names if name in name2wrapper] + _wrapper_idxes = [wrapper2index[_w] for _w in _wrappers] + + masks = self._dependency_calc_mask( + _wrappers, _names, wrappers_idx=_wrapper_idxes) + if masks is not None: + for layer in masks: + for mask_type in masks[layer]: + assert hasattr(name2wrapper[layer], mask_type), "there is no attribute '%s' in wrapper on %s" \ + % (mask_type, layer) + setattr(name2wrapper[layer], mask_type, masks[layer][mask_type]) diff --git a/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/pruning/finegrained_pruning_masker.py b/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/pruning/finegrained_pruning_masker.py new file mode 100644 index 0000000000000000000000000000000000000000..f4aa174233e977eed7b59c6ba65136526a738603 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/pruning/finegrained_pruning_masker.py @@ -0,0 +1,31 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +import torch +from .weight_masker import WeightMasker + +__all__ = ['LevelPrunerMasker'] + +logger = logging.getLogger('torch pruner') + + +class LevelPrunerMasker(WeightMasker): + """ + Prune to an exact pruning level specification + """ + + def calc_mask(self, sparsity, wrapper, wrapper_idx=None): + weight = wrapper.module.weight.data.clone() + if wrapper.weight_mask is not None: + # apply base mask for iterative pruning + weight = weight * wrapper.weight_mask + + w_abs = weight.abs() + k = int(weight.numel() * sparsity) + if k == 0: + return {'weight_mask': torch.ones(weight.shape).type_as(weight)} + threshold = torch.topk(w_abs.view(-1), k, largest=False)[0].max() + mask_weight = torch.gt(w_abs, threshold).type_as(weight) + mask = {'weight_mask': mask_weight} + return mask diff --git a/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/pruning/iterative_pruner.py b/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/pruning/iterative_pruner.py new file mode 100644 index 0000000000000000000000000000000000000000..65a98694d2650112524c1c73a393ad7dfa0569d3 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/pruning/iterative_pruner.py @@ -0,0 +1,596 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +import copy +import torch +from schema import And, Optional +from .....compression.pytorch.utils.config_validation import CompressorSchema +from .constants import MASKER_DICT +from .dependency_aware_pruner import DependencyAwarePruner + +__all__ = ['AGPPruner', 'ADMMPruner', 'SlimPruner', 'TaylorFOWeightFilterPruner', 'ActivationAPoZRankFilterPruner', + 'ActivationMeanRankFilterPruner'] + +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) + + +class IterativePruner(DependencyAwarePruner): + """ + Prune model during the training process. + """ + + def __init__(self, model, config_list, optimizer=None, pruning_algorithm='slim', trainer=None, criterion=None, + num_iterations=20, epochs_per_iteration=5, dependency_aware=False, dummy_input=None, **algo_kwargs): + """ + Parameters + ---------- + model: torch.nn.Module + Model to be pruned + config_list: list + List on pruning configs + optimizer: torch.optim.Optimizer + Optimizer used to train model + pruning_algorithm: str + algorithms being used to prune model + trainer: function + Function used to train the model. + Users should write this function as a normal function to train the Pytorch model + and include `model, optimizer, criterion, epoch` as function arguments. + criterion: function + Function used to calculate the loss between the target and the output. + For example, you can use ``torch.nn.CrossEntropyLoss()`` as input. + num_iterations: int + Total number of iterations in pruning process. We will calculate mask at the end of an iteration. + epochs_per_iteration: Union[int, list] + The number of training epochs for each iteration. `int` represents the same value for each iteration. + `list` represents the specific value for each iteration. + dependency_aware: bool + If prune the model in a dependency-aware way. + dummy_input: torch.Tensor + The dummy input to analyze the topology constraints. Note that, + the dummy_input should on the same device with the model. + algo_kwargs: dict + Additional parameters passed to pruning algorithm masker class + """ + super().__init__(model, config_list, optimizer, pruning_algorithm, dependency_aware, dummy_input, **algo_kwargs) + + if isinstance(epochs_per_iteration, list): + assert len(epochs_per_iteration) == num_iterations, 'num_iterations should equal to the length of epochs_per_iteration' + self.epochs_per_iteration = epochs_per_iteration + else: + assert num_iterations > 0, 'num_iterations should >= 1' + self.epochs_per_iteration = [epochs_per_iteration] * num_iterations + + self._validate_iteration_params() + + self._trainer = trainer + self._criterion = criterion + + def _fresh_calculated(self): + for wrapper in self.get_modules_wrapper(): + wrapper.if_calculated = False + + def _validate_iteration_params(self): + assert all(num >= 0 for num in self.epochs_per_iteration), 'all epoch number need >= 0' + + def compress(self): + training = self.bound_model.training + self.bound_model.train() + for _, epochs_num in enumerate(self.epochs_per_iteration): + self._fresh_calculated() + for epoch in range(epochs_num): + self._trainer(self.bound_model, optimizer=self.optimizer, criterion=self._criterion, epoch=epoch) + # NOTE: workaround for statistics_batch_num bigger than max batch number in one epoch, need refactor + if hasattr(self.masker, 'statistics_batch_num') and hasattr(self, 'iterations'): + if self.iterations < self.masker.statistics_batch_num: + self.iterations = self.masker.statistics_batch_num + self.update_mask() + self.bound_model.train(training) + + return self.bound_model + + +class AGPPruner(IterativePruner): + """ + Parameters + ---------- + model : torch.nn.Module + Model to be pruned. + config_list : listlist + Supported keys: + - sparsity : This is to specify the sparsity operations to be compressed to. + - op_types : See supported type in your specific pruning algorithm. + optimizer: torch.optim.Optimizer + Optimizer used to train model. + trainer: function + Function to train the model + criterion: function + Function used to calculate the loss between the target and the output. + For example, you can use ``torch.nn.CrossEntropyLoss()`` as input. + num_iterations: int + Total number of iterations in pruning process. We will calculate mask at the end of an iteration. + epochs_per_iteration: int + The number of training epochs for each iteration. + pruning_algorithm: str + Algorithms being used to prune model, + choose from `['level', 'slim', 'l1', 'l2', 'fpgm', 'taylorfo', 'apoz', 'mean_activation']`, by default `level` + """ + + def __init__(self, model, config_list, optimizer, trainer, criterion, num_iterations=10, epochs_per_iteration=1, pruning_algorithm='level'): + super().__init__(model, config_list, optimizer=optimizer, trainer=trainer, criterion=criterion, + num_iterations=num_iterations, epochs_per_iteration=epochs_per_iteration) + assert isinstance(optimizer, torch.optim.Optimizer), "AGP pruner is an iterative pruner, please pass optimizer of the model to it" + self.masker = MASKER_DICT[pruning_algorithm](model, self) + self.now_epoch = 0 + self.freq = epochs_per_iteration + self.end_epoch = epochs_per_iteration * num_iterations + self.set_wrappers_attribute("if_calculated", False) + + def validate_config(self, model, config_list): + """ + Parameters + ---------- + model : torch.nn.Module + Model to be pruned + config_list : list + List on pruning configs + """ + schema = CompressorSchema([{ + 'sparsity': And(float, lambda n: 0 <= n <= 1), + Optional('op_types'): [str], + Optional('op_names'): [str] + }], model, logger) + + schema.validate(config_list) + + def _supported_dependency_aware(self): + return False + + def calc_mask(self, wrapper, wrapper_idx=None): + """ + Calculate the mask of given layer. + Scale factors with the smallest absolute value in the BN layer are masked. + Parameters + ---------- + wrapper : Module + the layer to instrument the compression operation + wrapper_idx: int + index of this wrapper in pruner's all wrappers + Returns + ------- + dict | None + Dictionary for storing masks, keys of the dict: + 'weight_mask': weight mask tensor + 'bias_mask': bias mask tensor (optional) + """ + + config = wrapper.config + + if wrapper.if_calculated: + return None + + if not self.now_epoch % self.freq == 0: + return None + + target_sparsity = self.compute_target_sparsity(config) + new_mask = self.masker.calc_mask(sparsity=target_sparsity, wrapper=wrapper, wrapper_idx=wrapper_idx) + + if new_mask is not None: + wrapper.if_calculated = True + + return new_mask + + def compute_target_sparsity(self, config): + """ + Calculate the sparsity for pruning + Parameters + ---------- + config : dict + Layer's pruning config + Returns + ------- + float + Target sparsity to be pruned + """ + + initial_sparsity = 0 + self.target_sparsity = final_sparsity = config.get('sparsity', 0) + + if initial_sparsity >= final_sparsity: + logger.warning('your initial_sparsity >= final_sparsity') + return final_sparsity + + if self.end_epoch == 1 or self.end_epoch <= self.now_epoch: + return final_sparsity + + span = ((self.end_epoch - 1) // self.freq) * self.freq + assert span > 0 + self.target_sparsity = (final_sparsity + (initial_sparsity - final_sparsity) * (1.0 - (self.now_epoch / span)) ** 3) + return self.target_sparsity + + def update_epoch(self, epoch): + """ + Update epoch + Parameters + ---------- + epoch : int + current training epoch + """ + + if epoch > 0: + self.now_epoch = epoch + for wrapper in self.get_modules_wrapper(): + wrapper.if_calculated = False + + # TODO: need refactor + def compress(self): + training = self.bound_model.training + self.bound_model.train() + + for epoch in range(self.end_epoch): + self.update_epoch(epoch) + self._trainer(self.bound_model, optimizer=self.optimizer, criterion=self._criterion, epoch=epoch) + self.update_mask() + logger.info(f'sparsity is {self.target_sparsity:.2f} at epoch {epoch}') + self.get_pruned_weights() + + self.bound_model.train(training) + + return self.bound_model + + +class ADMMPruner(IterativePruner): + """ + A Pytorch implementation of ADMM Pruner algorithm. + + Parameters + ---------- + model : torch.nn.Module + Model to be pruned. + config_list : list + List on pruning configs. + trainer : function + Function used for the first subproblem. + Users should write this function as a normal function to train the Pytorch model + and include `model, optimizer, criterion, epoch` as function arguments. + criterion: function + Function used to calculate the loss between the target and the output. By default, we use CrossEntropyLoss in ADMMPruner. + For example, you can use ``torch.nn.CrossEntropyLoss()`` as input. + num_iterations: int + Total number of iterations in pruning process. We will calculate mask after we finish all iterations in ADMMPruner. + epochs_per_iteration: int + Training epochs of the first subproblem. + row : float + Penalty parameters for ADMM training. + base_algo : str + Base pruning algorithm. `level`, `l1`, `l2` or `fpgm`, by default `l1`. Given the sparsity distribution among + the ops, the assigned `base_algo` is used to decide which filters/channels/weights to prune. + """ + + def __init__(self, model, config_list, trainer, criterion=torch.nn.CrossEntropyLoss(), + num_iterations=30, epochs_per_iteration=5, row=1e-4, base_algo='l1'): + self._base_algo = base_algo + + super().__init__(model, config_list) + + self._trainer = trainer + self.optimizer = torch.optim.Adam( + self.bound_model.parameters(), lr=1e-3, weight_decay=5e-5) + self._criterion = criterion + self._num_iterations = num_iterations + self._training_epochs = epochs_per_iteration + self._row = row + + self.set_wrappers_attribute("if_calculated", False) + self.masker = MASKER_DICT[self._base_algo](self.bound_model, self) + + self.patch_optimizer_before(self._callback) + + def validate_config(self, model, config_list): + """ + Parameters + ---------- + model : torch.nn.Module + Model to be pruned + config_list : list + List on pruning configs + """ + + if self._base_algo == 'level': + schema = CompressorSchema([{ + 'sparsity': And(float, lambda n: 0 < n < 1), + Optional('op_types'): [str], + Optional('op_names'): [str], + }], model, logger) + elif self._base_algo in ['l1', 'l2', 'fpgm']: + schema = CompressorSchema([{ + 'sparsity': And(float, lambda n: 0 < n < 1), + 'op_types': ['Conv2d'], + Optional('op_names'): [str] + }], model, logger) + + schema.validate(config_list) + + def _supported_dependency_aware(self): + return False + + def _projection(self, weight, sparsity, wrapper): + ''' + Return the Euclidean projection of the weight matrix according to the pruning mode. + + Parameters + ---------- + weight : tensor + original matrix + sparsity : float + the ratio of parameters which need to be set to zero + wrapper: PrunerModuleWrapper + layer wrapper of this layer + + Returns + ------- + tensor + the projected matrix + ''' + wrapper_copy = copy.deepcopy(wrapper) + wrapper_copy.module.weight.data = weight + return weight.data.mul(self.masker.calc_mask(sparsity, wrapper_copy)['weight_mask']) + + def _callback(self): + # callback function to do additonal optimization, refer to the deriatives of Formula (7) + for i, wrapper in enumerate(self.get_modules_wrapper()): + wrapper.module.weight.data -= self._row * \ + (wrapper.module.weight.data - self.Z[i] + self.U[i]) + + def compress(self): + """ + Compress the model with ADMM. + + Returns + ------- + torch.nn.Module + model with specified modules compressed. + """ + logger.info('Starting ADMM Compression...') + + # initiaze Z, U + # Z_i^0 = W_i^0 + # U_i^0 = 0 + self.Z = [] + self.U = [] + for wrapper in self.get_modules_wrapper(): + z = wrapper.module.weight.data + self.Z.append(z) + self.U.append(torch.zeros_like(z)) + + # Loss = cross_entropy + l2 regulization + \Sum_{i=1}^N \row_i ||W_i - Z_i^k + U_i^k||^2 + # optimization iteration + for k in range(self._num_iterations): + logger.info('ADMM iteration : %d', k) + + # step 1: optimize W with AdamOptimizer + for epoch in range(self._training_epochs): + self._trainer(self.bound_model, optimizer=self.optimizer, criterion=self._criterion, epoch=epoch) + + # step 2: update Z, U + # Z_i^{k+1} = projection(W_i^{k+1} + U_i^k) + # U_i^{k+1} = U^k + W_i^{k+1} - Z_i^{k+1} + for i, wrapper in enumerate(self.get_modules_wrapper()): + z = wrapper.module.weight.data + self.U[i] + self.Z[i] = self._projection(z, wrapper.config['sparsity'], wrapper) + self.U[i] = self.U[i] + wrapper.module.weight.data - self.Z[i] + + # apply prune + self.update_mask() + + logger.info('Compression finished.') + + return self.bound_model + + +class SlimPruner(IterativePruner): + """ + Parameters + ---------- + model : torch.nn.Module + Model to be pruned + config_list : list + Supported keys: + - sparsity : This is to specify the sparsity operations to be compressed to. + - op_types : Only BatchNorm2d is supported in Slim Pruner. + optimizer : torch.optim.Optimizer + Optimizer used to train model + trainer : function + Function used to sparsify BatchNorm2d scaling factors. + Users should write this function as a normal function to train the Pytorch model + and include `model, optimizer, criterion, epoch` as function arguments. + criterion : function + Function used to calculate the loss between the target and the output. + For example, you can use ``torch.nn.CrossEntropyLoss()`` as input. + sparsifying_training_epochs: int + The number of channel sparsity regularization training epochs before pruning. + scale : float + Penalty parameters for sparsification. + dependency_aware: bool + If prune the model in a dependency-aware way. If it is `True`, this pruner will + prune the model according to the l2-norm of weights and the channel-dependency or + group-dependency of the model. In this way, the pruner will force the conv layers + that have dependencies to prune the same channels, so the speedup module can better + harvest the speed benefit from the pruned model. Note that, if this flag is set True + , the dummy_input cannot be None, because the pruner needs a dummy input to trace the + dependency between the conv layers. + dummy_input : torch.Tensor + The dummy input to analyze the topology constraints. Note that, the dummy_input + should on the same device with the model. + """ + + def __init__(self, model, config_list, optimizer, trainer, criterion, sparsifying_training_epochs=10, scale=0.0001, + dependency_aware=False, dummy_input=None): + super().__init__(model, config_list, optimizer=optimizer, pruning_algorithm='slim', trainer=trainer, criterion=criterion, + num_iterations=1, epochs_per_iteration=sparsifying_training_epochs, dependency_aware=dependency_aware, + dummy_input=dummy_input) + self.scale = scale + self.patch_optimizer_before(self._callback) + + def validate_config(self, model, config_list): + schema = CompressorSchema([{ + 'sparsity': And(float, lambda n: 0 < n < 1), + 'op_types': ['BatchNorm2d'], + Optional('op_names'): [str] + }], model, logger) + + schema.validate(config_list) + + if len(config_list) > 1: + logger.warning('Slim pruner only supports 1 configuration') + + def _supported_dependency_aware(self): + return True + + def _callback(self): + for _, wrapper in enumerate(self.get_modules_wrapper()): + wrapper.module.weight.grad.data.add_(self.scale * torch.sign(wrapper.module.weight.data)) + + +class TaylorFOWeightFilterPruner(IterativePruner): + """ + Parameters + ---------- + model : torch.nn.Module + Model to be pruned + config_list : list + Supported keys: + - sparsity : How much percentage of convolutional filters are to be pruned. + - op_types : Currently only Conv2d is supported in TaylorFOWeightFilterPruner. + optimizer: torch.optim.Optimizer + Optimizer used to train model + trainer : function + Function used to sparsify BatchNorm2d scaling factors. + Users should write this function as a normal function to train the Pytorch model + and include `model, optimizer, criterion, epoch` as function arguments. + criterion : function + Function used to calculate the loss between the target and the output. + For example, you can use ``torch.nn.CrossEntropyLoss()`` as input. + sparsifying_training_batches: int + The number of batches to collect the contributions. Note that the number need to be less than the maximum batch number in one epoch. + dependency_aware: bool + If prune the model in a dependency-aware way. If it is `True`, this pruner will + prune the model according to the l2-norm of weights and the channel-dependency or + group-dependency of the model. In this way, the pruner will force the conv layers + that have dependencies to prune the same channels, so the speedup module can better + harvest the speed benefit from the pruned model. Note that, if this flag is set True + , the dummy_input cannot be None, because the pruner needs a dummy input to trace the + dependency between the conv layers. + dummy_input : torch.Tensor + The dummy input to analyze the topology constraints. Note that, the dummy_input + should on the same device with the model. + """ + + def __init__(self, model, config_list, optimizer, trainer, criterion, sparsifying_training_batches=1, + dependency_aware=False, dummy_input=None): + super().__init__(model, config_list, optimizer=optimizer, pruning_algorithm='taylorfo', trainer=trainer, + criterion=criterion, statistics_batch_num=sparsifying_training_batches, num_iterations=1, + epochs_per_iteration=1, dependency_aware=dependency_aware, + dummy_input=dummy_input) + + def _supported_dependency_aware(self): + return True + + +class ActivationAPoZRankFilterPruner(IterativePruner): + """ + Parameters + ---------- + model : torch.nn.Module + Model to be pruned + config_list : list + Supported keys: + - sparsity : How much percentage of convolutional filters are to be pruned. + - op_types : Only Conv2d is supported in ActivationAPoZRankFilterPruner. + optimizer: torch.optim.Optimizer + Optimizer used to train model + trainer: function + Function used to train the model. + Users should write this function as a normal function to train the Pytorch model + and include `model, optimizer, criterion, epoch` as function arguments. + criterion : function + Function used to calculate the loss between the target and the output. + For example, you can use ``torch.nn.CrossEntropyLoss()`` as input. + activation: str + The activation type. + sparsifying_training_batches: int + The number of batches to collect the contributions. Note that the number need to be less than the maximum batch number in one epoch. + dependency_aware: bool + If prune the model in a dependency-aware way. If it is `True`, this pruner will + prune the model according to the l2-norm of weights and the channel-dependency or + group-dependency of the model. In this way, the pruner will force the conv layers + that have dependencies to prune the same channels, so the speedup module can better + harvest the speed benefit from the pruned model. Note that, if this flag is set True + , the dummy_input cannot be None, because the pruner needs a dummy input to trace the + dependency between the conv layers. + dummy_input : torch.Tensor + The dummy input to analyze the topology constraints. Note that, the dummy_input + should on the same device with the model. + + """ + + def __init__(self, model, config_list, optimizer, trainer, criterion, activation='relu', + sparsifying_training_batches=1, dependency_aware=False, dummy_input=None): + super().__init__(model, config_list, pruning_algorithm='apoz', optimizer=optimizer, trainer=trainer, + criterion=criterion, dependency_aware=dependency_aware, dummy_input=dummy_input, + activation=activation, statistics_batch_num=sparsifying_training_batches, num_iterations=1, + epochs_per_iteration=1) + self.patch_optimizer(self.update_mask) + + def _supported_dependency_aware(self): + return True + + +class ActivationMeanRankFilterPruner(IterativePruner): + """ + Parameters + ---------- + model : torch.nn.Module + Model to be pruned + config_list : list + Supported keys: + - sparsity : How much percentage of convolutional filters are to be pruned. + - op_types : Only Conv2d is supported in ActivationMeanRankFilterPruner. + optimizer: torch.optim.Optimizer + Optimizer used to train model. + trainer: function + Function used to train the model. + Users should write this function as a normal function to train the Pytorch model + and include `model, optimizer, criterion, epoch` as function arguments. + criterion : function + Function used to calculate the loss between the target and the output. + For example, you can use ``torch.nn.CrossEntropyLoss()`` as input. + activation: str + The activation type. + sparsifying_training_batches: int + The number of batches to collect the contributions. Note that the number need to be less than the maximum batch number in one epoch. + dependency_aware: bool + If prune the model in a dependency-aware way. If it is `True`, this pruner will + prune the model according to the l2-norm of weights and the channel-dependency or + group-dependency of the model. In this way, the pruner will force the conv layers + that have dependencies to prune the same channels, so the speedup module can better + harvest the speed benefit from the pruned model. Note that, if this flag is set True + , the dummy_input cannot be None, because the pruner needs a dummy input to trace the + dependency between the conv layers. + dummy_input : torch.Tensor + The dummy input to analyze the topology constraints. Note that, the dummy_input + should on the same device with the model. + """ + + def __init__(self, model, config_list, optimizer, trainer, criterion, activation='relu', + sparsifying_training_batches=1, dependency_aware=False, dummy_input=None): + super().__init__(model, config_list, pruning_algorithm='mean_activation', optimizer=optimizer, trainer=trainer, + criterion=criterion, dependency_aware=dependency_aware, dummy_input=dummy_input, + activation=activation, statistics_batch_num=sparsifying_training_batches, num_iterations=1, + epochs_per_iteration=1) + self.patch_optimizer(self.update_mask) + + def _supported_dependency_aware(self): + return True diff --git a/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/pruning/lottery_ticket.py b/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/pruning/lottery_ticket.py new file mode 100644 index 0000000000000000000000000000000000000000..caa1c831e6025ab69d0e606df850584959aaa617 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/pruning/lottery_ticket.py @@ -0,0 +1,146 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import copy +import logging +import torch +from schema import And, Optional +from nni.compression.pytorch.utils.config_validation import CompressorSchema +from nni.compression.pytorch.compressor import Pruner +from .finegrained_pruning_masker import LevelPrunerMasker + +logger = logging.getLogger('torch pruner') + +class LotteryTicketPruner(Pruner): + """ + Parameters + ---------- + model : pytorch model + The model to be pruned + config_list : list + Supported keys: + - prune_iterations : The number of rounds for the iterative pruning. + - sparsity : The final sparsity when the compression is done. + optimizer : pytorch optimizer + The optimizer for the model + lr_scheduler : pytorch lr scheduler + The lr scheduler for the model if used + reset_weights : bool + Whether reset weights and optimizer at the beginning of each round. + """ + def __init__(self, model, config_list, optimizer=None, lr_scheduler=None, reset_weights=True): + # save init weights and optimizer + self.reset_weights = reset_weights + if self.reset_weights: + self._model = model + self._optimizer = optimizer + self._model_state = copy.deepcopy(model.state_dict()) + self._optimizer_state = copy.deepcopy(optimizer.state_dict()) + self._lr_scheduler = lr_scheduler + if lr_scheduler is not None: + self._scheduler_state = copy.deepcopy(lr_scheduler.state_dict()) + + super().__init__(model, config_list, optimizer) + self.curr_prune_iteration = None + self.prune_iterations = config_list[0]['prune_iterations'] + self.masker = LevelPrunerMasker(model, self) + + def validate_config(self, model, config_list): + """ + Parameters + ---------- + model : torch.nn.Module + Model to be pruned + config_list : list + Supported keys: + - prune_iterations : The number of rounds for the iterative pruning. + - sparsity : The final sparsity when the compression is done. + """ + schema = CompressorSchema([{ + 'sparsity': And(float, lambda n: 0 < n < 1), + 'prune_iterations': And(int, lambda n: n > 0), + Optional('op_types'): [str], + Optional('op_names'): [str] + }], model, logger) + + schema.validate(config_list) + assert len(set([x['prune_iterations'] for x in config_list])) == 1, 'The values of prune_iterations must be equal in your config' + + def _calc_sparsity(self, sparsity): + keep_ratio_once = (1 - sparsity) ** (1 / self.prune_iterations) + curr_keep_ratio = keep_ratio_once ** self.curr_prune_iteration + return max(1 - curr_keep_ratio, 0) + + def _calc_mask(self, wrapper, sparsity): + weight = wrapper.module.weight.data + if self.curr_prune_iteration == 0: + mask = {'weight_mask': torch.ones(weight.shape).type_as(weight)} + else: + curr_sparsity = self._calc_sparsity(sparsity) + mask = self.masker.calc_mask(sparsity=curr_sparsity, wrapper=wrapper) + return mask + + def calc_mask(self, wrapper, **kwargs): + """ + Generate mask for the given ``weight``. + + Parameters + ---------- + wrapper : Module + The layer to be pruned + + Returns + ------- + tensor + The mask for this weight, it is ```None``` because this pruner + calculates and assigns masks in ```prune_iteration_start```, + no need to do anything in this function. + """ + return None + + def get_prune_iterations(self): + """ + Return the range for iterations. + In the first prune iteration, masks are all one, thus, add one more iteration + + Returns + ------- + list + A list for pruning iterations + """ + return range(self.prune_iterations + 1) + + def prune_iteration_start(self): + """ + Control the pruning procedure on updated epoch number. + Should be called at the beginning of the epoch. + """ + if self.curr_prune_iteration is None: + self.curr_prune_iteration = 0 + else: + self.curr_prune_iteration += 1 + assert self.curr_prune_iteration < self.prune_iterations + 1, 'Exceed the configured prune_iterations' + + modules_wrapper = self.get_modules_wrapper() + modules_to_compress = self.get_modules_to_compress() + for layer, config in modules_to_compress: + module_wrapper = None + for wrapper in modules_wrapper: + if wrapper.name == layer.name: + module_wrapper = wrapper + break + assert module_wrapper is not None + + sparsity = config.get('sparsity') + mask = self._calc_mask(module_wrapper, sparsity) + # TODO: directly use weight_mask is not good + module_wrapper.weight_mask = mask['weight_mask'] + # there is no mask for bias + + # reinit weights back to original after new masks are generated + if self.reset_weights: + # should use this member function to reset model weights + self.load_model_state_dict(self._model_state) + self._optimizer.load_state_dict(self._optimizer_state) + if self._lr_scheduler is not None: + self._lr_scheduler.load_state_dict(self._scheduler_state) diff --git a/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/pruning/net_adapt_pruner.py b/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/pruning/net_adapt_pruner.py new file mode 100644 index 0000000000000000000000000000000000000000..08416319eaaa6b4fb2ff4e2214f73d80e211020d --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/pruning/net_adapt_pruner.py @@ -0,0 +1,351 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +import os +import copy +import json +import torch +from schema import And, Optional + +from nni.utils import OptimizeMode + +from nni.compression.pytorch.compressor import Pruner +from nni.compression.pytorch.utils.config_validation import CompressorSchema +from nni.compression.pytorch.utils.num_param_counter import get_total_num_weights +from .constants_pruner import PRUNER_DICT + + +_logger = logging.getLogger(__name__) + + +class NetAdaptPruner(Pruner): + """ + A Pytorch implementation of NetAdapt compression algorithm. + + Parameters + ---------- + model : pytorch model + The model to be pruned. + config_list : list + Supported keys: + - sparsity : The target overall sparsity. + - op_types : The operation type to prune. + short_term_fine_tuner : function + function to short-term fine tune the masked model. + This function should include `model` as the only parameter, + and fine tune the model for a short term after each pruning iteration. + Example:: + + def short_term_fine_tuner(model, epoch=3): + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + train_loader = ... + criterion = torch.nn.CrossEntropyLoss() + optimizer = torch.optim.SGD(model.parameters(), lr=0.01) + model.train() + for _ in range(epoch): + for batch_idx, (data, target) in enumerate(train_loader): + data, target = data.to(device), target.to(device) + optimizer.zero_grad() + output = model(data) + loss = criterion(output, target) + loss.backward() + optimizer.step() + evaluator : function + function to evaluate the masked model. + This function should include `model` as the only parameter, and returns a scalar value. + Example:: + + def evaluator(model): + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + val_loader = ... + model.eval() + correct = 0 + with torch.no_grad(): + for data, target in val_loader: + data, target = data.to(device), target.to(device) + output = model(data) + # get the index of the max log-probability + pred = output.argmax(dim=1, keepdim=True) + correct += pred.eq(target.view_as(pred)).sum().item() + accuracy = correct / len(val_loader.dataset) + return accuracy + optimize_mode : str + optimize mode, `maximize` or `minimize`, by default `maximize`. + base_algo : str + Base pruning algorithm. `level`, `l1`, `l2` or `fpgm`, by default `l1`. Given the sparsity distribution among the ops, + the assigned `base_algo` is used to decide which filters/channels/weights to prune. + sparsity_per_iteration : float + sparsity to prune in each iteration. + experiment_data_dir : str + PATH to save experiment data, + including the config_list generated for the base pruning algorithm and the performance of the pruned model. + """ + + def __init__(self, model, config_list, short_term_fine_tuner, evaluator, + optimize_mode='maximize', base_algo='l1', sparsity_per_iteration=0.05, experiment_data_dir='./'): + # models used for iterative pruning and evaluation + self._model_to_prune = copy.deepcopy(model) + self._base_algo = base_algo + + super().__init__(model, config_list) + + self._short_term_fine_tuner = short_term_fine_tuner + self._evaluator = evaluator + self._optimize_mode = OptimizeMode(optimize_mode) + + # hyper parameters for NetAdapt algorithm + self._sparsity_per_iteration = sparsity_per_iteration + + # overall pruning rate + self._sparsity = config_list[0]['sparsity'] + + # config_list + self._config_list_generated = [] + + self._experiment_data_dir = experiment_data_dir + if not os.path.exists(self._experiment_data_dir): + os.makedirs(self._experiment_data_dir) + + self._tmp_model_path = os.path.join(self._experiment_data_dir, 'tmp_model.pth') + + def validate_config(self, model, config_list): + """ + Parameters + ---------- + model : torch.nn.Module + Model to be pruned + config_list : list + List on pruning configs + """ + + if self._base_algo == 'level': + schema = CompressorSchema([{ + 'sparsity': And(float, lambda n: 0 < n < 1), + Optional('op_types'): [str], + Optional('op_names'): [str], + }], model, _logger) + elif self._base_algo in ['l1', 'l2', 'fpgm']: + schema = CompressorSchema([{ + 'sparsity': And(float, lambda n: 0 < n < 1), + 'op_types': ['Conv2d'], + Optional('op_names'): [str] + }], model, _logger) + + schema.validate(config_list) + + def calc_mask(self, wrapper, **kwargs): + return None + + def _update_config_list(self, config_list, op_name, sparsity): + ''' + update sparsity of op_name in config_list + ''' + config_list_updated = copy.deepcopy(config_list) + + for idx, item in enumerate(config_list): + if op_name in item['op_names']: + config_list_updated[idx]['sparsity'] = sparsity + return config_list_updated + + # if op_name is not in self._config_list_generated, create a new json item + if self._base_algo in ['l1', 'l2', 'fpgm']: + config_list_updated.append( + {'sparsity': sparsity, 'op_types': ['Conv2d'], 'op_names': [op_name]}) + elif self._base_algo == 'level': + config_list_updated.append( + {'sparsity': sparsity, 'op_names': [op_name]}) + + return config_list_updated + + def _get_op_num_weights_remained(self, op_name, module): + ''' + Get the number of weights remained after channel pruning with current sparsity + + Returns + ------- + int + remained number of weights of the op + ''' + + # if op is wrapped by the pruner + for wrapper in self.get_modules_wrapper(): + if wrapper.name == op_name: + return wrapper.weight_mask.sum().item() + + # if op is not wrapped by the pruner + return module.weight.data.numel() + + def _get_op_sparsity(self, op_name): + for config in self._config_list_generated: + if 'op_names' in config and op_name in config['op_names']: + return config['sparsity'] + return 0 + + def _calc_num_related_weights(self, op_name): + ''' + Calculate total number weights of the op and the next op, applicable only for models without dependencies among ops + + Parameters + ---------- + op_name : str + + Returns + ------- + int + total number of all the realted (current and the next) op weights + ''' + num_weights = 0 + flag_found = False + previous_name = None + previous_module = None + + for name, module in self._model_to_prune.named_modules(): + if not flag_found and name != op_name and type(module).__name__ in ['Conv2d', 'Linear']: + previous_name = name + previous_module = module + if not flag_found and name == op_name: + _logger.debug("original module found: %s", name) + num_weights = module.weight.data.numel() + + # consider related pruning in this op caused by previous op's pruning + if previous_module: + sparsity_previous_op = self._get_op_sparsity(previous_name) + if sparsity_previous_op: + _logger.debug( + "decrease op's weights by %s due to previous op %s's pruning...", sparsity_previous_op, previous_name) + num_weights *= (1-sparsity_previous_op) + + flag_found = True + continue + if flag_found and type(module).__name__ in ['Conv2d', 'Linear']: + _logger.debug("related module found: %s", name) + # channel/filter pruning crossing is considered here, so only the num_weights after channel pruning is valuable + num_weights += self._get_op_num_weights_remained(name, module) + break + + _logger.debug("num related weights of op %s : %d", op_name, num_weights) + + return num_weights + + def compress(self): + """ + Compress the model. + + Returns + ------- + torch.nn.Module + model with specified modules compressed. + """ + _logger.info('Starting NetAdapt Compression...') + + pruning_iteration = 0 + current_sparsity = 0 + delta_num_weights_per_iteration = \ + int(get_total_num_weights(self._model_to_prune, ['Conv2d', 'Linear']) * self._sparsity_per_iteration) + + # stop condition + while current_sparsity < self._sparsity: + _logger.info('Pruning iteration: %d', pruning_iteration) + + # calculate target sparsity of this iteration + target_sparsity = current_sparsity + self._sparsity_per_iteration + + # variable to store the info of the best layer found in this iteration + best_op = {} + + for wrapper in self.get_modules_wrapper(): + _logger.debug("op name : %s", wrapper.name) + _logger.debug("op weights : %d", wrapper.weight_mask.numel()) + _logger.debug("op left weights : %d", wrapper.weight_mask.sum().item()) + + current_op_sparsity = 1 - wrapper.weight_mask.sum().item() / wrapper.weight_mask.numel() + _logger.debug("current op sparsity : %s", current_op_sparsity) + + # sparsity that this layer needs to prune to satisfy the requirement + target_op_sparsity = current_op_sparsity + delta_num_weights_per_iteration / self._calc_num_related_weights(wrapper.name) + + if target_op_sparsity >= 1: + _logger.info('Layer %s has no enough weights (remained) to prune', wrapper.name) + continue + + config_list = self._update_config_list(self._config_list_generated, wrapper.name, target_op_sparsity) + _logger.debug("config_list used : %s", config_list) + + pruner = PRUNER_DICT[self._base_algo](copy.deepcopy(self._model_to_prune), config_list) + model_masked = pruner.compress() + + # Short-term fine tune the pruned model + self._short_term_fine_tuner(model_masked) + + performance = self._evaluator(model_masked) + _logger.info("Layer : %s, evaluation result after short-term fine tuning : %s", wrapper.name, performance) + + if not best_op \ + or (self._optimize_mode is OptimizeMode.Maximize and performance > best_op['performance']) \ + or (self._optimize_mode is OptimizeMode.Minimize and performance < best_op['performance']): + _logger.debug("updating best layer to %s...", wrapper.name) + # find weight mask of this layer + for w in pruner.get_modules_wrapper(): + if w.name == wrapper.name: + masks = {'weight_mask': w.weight_mask, + 'bias_mask': w.bias_mask} + break + best_op = { + 'op_name': wrapper.name, + 'sparsity': target_op_sparsity, + 'performance': performance, + 'masks': masks + } + + # save model weights + pruner.export_model(self._tmp_model_path) + + if not best_op: + # decrease pruning step + self._sparsity_per_iteration *= 0.5 + _logger.info("No more layers to prune, decrease pruning step to %s", self._sparsity_per_iteration) + continue + + # Pick the best layer to prune, update iterative information + # update config_list + self._config_list_generated = self._update_config_list( + self._config_list_generated, best_op['op_name'], best_op['sparsity']) + + # update weights parameters + self._model_to_prune.load_state_dict(torch.load(self._tmp_model_path)) + + # update mask of the chosen op + for wrapper in self.get_modules_wrapper(): + if wrapper.name == best_op['op_name']: + for k in best_op['masks']: + setattr(wrapper, k, best_op['masks'][k]) + break + + current_sparsity = target_sparsity + _logger.info('Pruning iteration %d finished, current sparsity: %s', pruning_iteration, current_sparsity) + _logger.info('Layer %s seleted with sparsity %s, performance after pruning & short term fine-tuning : %s', + best_op['op_name'], best_op['sparsity'], best_op['performance']) + pruning_iteration += 1 + + self._final_performance = best_op['performance'] + + # load weights parameters + self.load_model_state_dict(torch.load(self._tmp_model_path)) + os.remove(self._tmp_model_path) + + _logger.info('----------Compression finished--------------') + _logger.info('config_list generated: %s', self._config_list_generated) + _logger.info("Performance after pruning: %s", self._final_performance) + _logger.info("Masked sparsity: %.6f", current_sparsity) + + # save best config found and best performance + with open(os.path.join(self._experiment_data_dir, 'search_result.json'), 'w') as jsonfile: + json.dump({ + 'performance': self._final_performance, + 'config_list': json.dumps(self._config_list_generated) + }, jsonfile) + + _logger.info('search history and result saved to foler : %s', self._experiment_data_dir) + + return self.bound_model diff --git a/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/pruning/one_shot_pruner.py b/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/pruning/one_shot_pruner.py new file mode 100644 index 0000000000000000000000000000000000000000..3deec003fe4dcd759d7c8a90e385175b29385a01 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/pruning/one_shot_pruner.py @@ -0,0 +1,169 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +from schema import And, Optional + +from .....compression.pytorch.utils.config_validation import CompressorSchema +from .dependency_aware_pruner import DependencyAwarePruner + +__all__ = ['LevelPruner', 'L1FilterPruner', 'L2FilterPruner', 'FPGMPruner'] + +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) + + +class OneshotPruner(DependencyAwarePruner): + """ + Prune model to an exact pruning level for one time. + """ + + def __init__(self, model, config_list, pruning_algorithm='level', dependency_aware=False, dummy_input=None, + **algo_kwargs): + """ + Parameters + ---------- + model : torch.nn.Module + Model to be pruned + config_list : list + List on pruning configs + pruning_algorithm: str + algorithms being used to prune model + dependency_aware: bool + If prune the model in a dependency-aware way. + dummy_input : torch.Tensor + The dummy input to analyze the topology constraints. Note that, + the dummy_input should on the same device with the model. + algo_kwargs: dict + Additional parameters passed to pruning algorithm masker class + """ + super().__init__(model, config_list, None, pruning_algorithm, dependency_aware, dummy_input, **algo_kwargs) + + def validate_config(self, model, config_list): + """ + Parameters + ---------- + model : torch.nn.Module + Model to be pruned + config_list : list + List on pruning configs + """ + schema = CompressorSchema([{ + 'sparsity': And(float, lambda n: 0 < n < 1), + Optional('op_types'): [str], + Optional('op_names'): [str] + }], model, logger) + + schema.validate(config_list) + + +class LevelPruner(OneshotPruner): + """ + Parameters + ---------- + model : torch.nn.Module + Model to be pruned + config_list : list + Supported keys: + - sparsity : This is to specify the sparsity operations to be compressed to. + - op_types : Operation types to prune. + """ + + def __init__(self, model, config_list): + super().__init__(model, config_list, pruning_algorithm='level') + + def _supported_dependency_aware(self): + return False + + +class L1FilterPruner(OneshotPruner): + """ + Parameters + ---------- + model : torch.nn.Module + Model to be pruned + config_list : list + Supported keys: + - sparsity : This is to specify the sparsity operations to be compressed to. + - op_types : Only Conv2d is supported in L1FilterPruner. + dependency_aware: bool + If prune the model in a dependency-aware way. If it is `True`, this pruner will + prune the model according to the l2-norm of weights and the channel-dependency or + group-dependency of the model. In this way, the pruner will force the conv layers + that have dependencies to prune the same channels, so the speedup module can better + harvest the speed benefit from the pruned model. Note that, if this flag is set True + , the dummy_input cannot be None, because the pruner needs a dummy input to trace the + dependency between the conv layers. + dummy_input : torch.Tensor + The dummy input to analyze the topology constraints. Note that, the dummy_input + should on the same device with the model. + """ + + def __init__(self, model, config_list, dependency_aware=False, dummy_input=None): + super().__init__(model, config_list, pruning_algorithm='l1', dependency_aware=dependency_aware, + dummy_input=dummy_input) + + def _supported_dependency_aware(self): + return True + + +class L2FilterPruner(OneshotPruner): + """ + Parameters + ---------- + model : torch.nn.Module + Model to be pruned + config_list : list + Supported keys: + - sparsity : This is to specify the sparsity operations to be compressed to. + - op_types : Only Conv2d is supported in L2FilterPruner. + dependency_aware: bool + If prune the model in a dependency-aware way. If it is `True`, this pruner will + prune the model according to the l2-norm of weights and the channel-dependency or + group-dependency of the model. In this way, the pruner will force the conv layers + that have dependencies to prune the same channels, so the speedup module can better + harvest the speed benefit from the pruned model. Note that, if this flag is set True + , the dummy_input cannot be None, because the pruner needs a dummy input to trace the + dependency between the conv layers. + dummy_input : torch.Tensor + The dummy input to analyze the topology constraints. Note that, the dummy_input + should on the same device with the model. + """ + + def __init__(self, model, config_list, dependency_aware=False, dummy_input=None): + super().__init__(model, config_list, pruning_algorithm='l2', dependency_aware=dependency_aware, + dummy_input=dummy_input) + + def _supported_dependency_aware(self): + return True + + +class FPGMPruner(OneshotPruner): + """ + Parameters + ---------- + model : torch.nn.Module + Model to be pruned + config_list : list + Supported keys: + - sparsity : This is to specify the sparsity operations to be compressed to. + - op_types : Only Conv2d is supported in FPGM Pruner. + dependency_aware: bool + If prune the model in a dependency-aware way. If it is `True`, this pruner will + prune the model according to the l2-norm of weights and the channel-dependency or + group-dependency of the model. In this way, the pruner will force the conv layers + that have dependencies to prune the same channels, so the speedup module can better + harvest the speed benefit from the pruned model. Note that, if this flag is set True + , the dummy_input cannot be None, because the pruner needs a dummy input to trace the + dependency between the conv layers. + dummy_input : torch.Tensor + The dummy input to analyze the topology constraints. Note that, the dummy_input + should on the same device with the model. + """ + + def __init__(self, model, config_list, dependency_aware=False, dummy_input=None): + super().__init__(model, config_list, pruning_algorithm='fpgm', dependency_aware=dependency_aware, + dummy_input=dummy_input) + + def _supported_dependency_aware(self): + return True diff --git a/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/pruning/sensitivity_pruner.py b/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/pruning/sensitivity_pruner.py new file mode 100644 index 0000000000000000000000000000000000000000..ed4d791abd95e8df45ba92fe579e92c9eb163c60 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/pruning/sensitivity_pruner.py @@ -0,0 +1,413 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. +import os +import csv +import copy +import json +import logging +import torch + +from schema import And, Optional +from nni.compression.pytorch.compressor import Pruner +from nni.compression.pytorch.utils.config_validation import CompressorSchema +from .constants_pruner import PRUNER_DICT +from nni.compression.pytorch.utils.sensitivity_analysis import SensitivityAnalysis + + +MAX_PRUNE_RATIO_PER_ITER = 0.95 + +_logger = logging.getLogger('Sensitivity_Pruner') +_logger.setLevel(logging.INFO) + +class SensitivityPruner(Pruner): + """ + This function prune the model based on the sensitivity + for each layer. + + Parameters + ---------- + model: torch.nn.Module + model to be compressed + evaluator: function + validation function for the model. This function should return the accuracy + of the validation dataset. The input parameters of evaluator can be specified + in the parameter `eval_args` and 'eval_kwargs' of the compress function if needed. + Example: + >>> def evaluator(model): + >>> device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + >>> val_loader = ... + >>> model.eval() + >>> correct = 0 + >>> with torch.no_grad(): + >>> for data, target in val_loader: + >>> data, target = data.to(device), target.to(device) + >>> output = model(data) + >>> # get the index of the max log-probability + >>> pred = output.argmax(dim=1, keepdim=True) + >>> correct += pred.eq(target.view_as(pred)).sum().item() + >>> accuracy = correct / len(val_loader.dataset) + >>> return accuracy + finetuner: function + finetune function for the model. This parameter is not essential, if is not None, + the sensitivity pruner will finetune the model after pruning in each iteration. + The input parameters of finetuner can be specified in the parameter of compress + called `finetune_args` and `finetune_kwargs` if needed. + Example: + >>> def finetuner(model, epoch=3): + >>> device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + >>> train_loader = ... + >>> criterion = torch.nn.CrossEntropyLoss() + >>> optimizer = torch.optim.SGD(model.parameters(), lr=0.01) + >>> model.train() + >>> for _ in range(epoch): + >>> for _, (data, target) in enumerate(train_loader): + >>> data, target = data.to(device), target.to(device) + >>> optimizer.zero_grad() + >>> output = model(data) + >>> loss = criterion(output, target) + >>> loss.backward() + >>> optimizer.step() + base_algo: str + base pruning algorithm. `level`, `l1`, `l2` or `fpgm`, by default `l1`. + sparsity_proportion_calc: function + This function generate the sparsity proportion between the conv layers according to the + sensitivity analysis results. We provide a default function to quantify the sparsity + proportion according to the sensitivity analysis results. Users can also customize + this function according to their needs. The input of this function is a dict, + for example : {'conv1' : {0.1: 0.9, 0.2 : 0.8}, 'conv2' : {0.1: 0.9, 0.2 : 0.8}}, + in which, 'conv1' and is the name of the conv layer, and 0.1:0.9 means when the + sparsity of conv1 is 0.1 (10%), the model's val accuracy equals to 0.9. + sparsity_per_iter: float + The sparsity of the model that the pruner try to prune in each iteration. + acc_drop_threshold : float + The hyperparameter used to quantifiy the sensitivity for each layer. + checkpoint_dir: str + The dir path to save the checkpoints during the pruning. + """ + + def __init__(self, model, config_list, evaluator, + finetuner=None, base_algo='l1', sparsity_proportion_calc=None, + sparsity_per_iter=0.1, acc_drop_threshold=0.05, checkpoint_dir=None): + + self.base_algo = base_algo + self.model = model + super(SensitivityPruner, self).__init__(model, config_list) + # unwrap the model + self._unwrap_model() + _logger.debug(str(self.model)) + self.evaluator = evaluator + self.finetuner = finetuner + self.analyzer = SensitivityAnalysis( + self.model, self.evaluator, prune_type=base_algo, \ + early_stop_mode='dropped', early_stop_value=acc_drop_threshold) + # Get the original accuracy of the pretrained model + self.ori_acc = None + # Copy the original weights before pruning + self.ori_state_dict = copy.deepcopy(self.model.state_dict()) + self.sensitivities = {} + # Save the weight count for each layer + self.weight_count = {} + self.weight_sum = 0 + # Map the layer name to the layer module + self.named_module = {} + + self.Pruner = PRUNER_DICT[self.base_algo] + # Count the total weight count of the model + for name, submodule in self.model.named_modules(): + self.named_module[name] = submodule + if name in self.analyzer.target_layer: + # Currently, only count the weights in the conv layers + # else the fully connected layer (which contains + # the most weights) may make the pruner prune the + # model too hard + # if hasattr(submodule, 'weight'): # Count all the weights of the model + self.weight_count[name] = submodule.weight.data.numel() + self.weight_sum += self.weight_count[name] + # function to generate the sparsity proportion between the conv layers + if sparsity_proportion_calc is None: + self.sparsity_proportion_calc = self._max_prune_ratio + else: + self.sparsity_proportion_calc = sparsity_proportion_calc + # The ratio of remained weights is 1.0 at the begining + self.remained_ratio = 1.0 + self.sparsity_per_iter = sparsity_per_iter + self.acc_drop_threshold = acc_drop_threshold + self.checkpoint_dir = checkpoint_dir + + def validate_config(self, model, config_list): + """ + Parameters + ---------- + model : torch.nn.module + Model to be pruned + config_list : list + List on pruning configs + """ + + if self.base_algo == 'level': + schema = CompressorSchema([{ + 'sparsity': And(float, lambda n: 0 < n < 1), + Optional('op_types'): [str], + Optional('op_names'): [str], + }], model, _logger) + elif self.base_algo in ['l1', 'l2', 'fpgm']: + schema = CompressorSchema([{ + 'sparsity': And(float, lambda n: 0 < n < 1), + 'op_types': ['Conv2d'], + Optional('op_names'): [str] + }], model, _logger) + + schema.validate(config_list) + + def load_sensitivity(self, filepath): + """ + load the sensitivity results exported by the sensitivity analyzer + """ + assert os.path.exists(filepath) + with open(filepath, 'r') as csvf: + csv_r = csv.reader(csvf) + header = next(csv_r) + sparsities = [float(x) for x in header[1:]] + sensitivities = {} + for row in csv_r: + layername = row[0] + accuracies = [float(x) for x in row[1:]] + sensitivities[layername] = {} + for i, accuracy in enumerate(accuracies): + sensitivities[layername][sparsities[i]] = accuracy + return sensitivities + + def _max_prune_ratio(self, ori_acc, threshold, sensitivities): + """ + Find the maximum prune ratio for a single layer whose accuracy + drop is lower than the threshold. + + Parameters + ---------- + ori_acc: float + Original accuracy + threshold: float + Accuracy drop threshold + sensitivities: dict + The dict object that stores the sensitivity results for each layer. + For example: {'conv1' : {0.1: 0.9, 0.2 : 0.8}} + Returns + ------- + max_ratios: dict + return the maximum prune ratio for each layer. For example: + {'conv1':0.1, 'conv2':0.2} + """ + max_ratio = {} + for layer in sensitivities: + prune_ratios = sorted(sensitivities[layer].keys()) + last_ratio = 0 + for ratio in prune_ratios: + last_ratio = ratio + cur_acc = sensitivities[layer][ratio] + if cur_acc + threshold < ori_acc: + break + max_ratio[layer] = last_ratio + return max_ratio + + def normalize(self, ratios, target_pruned): + """ + Normalize the prune ratio of each layer according to the + total already pruned ratio and the final target total pruning + ratio + + Parameters + ---------- + ratios: + Dict object that save the prune ratio for each layer + target_pruned: + The amount of the weights expected to be pruned in this + iteration + + Returns + ------- + new_ratios: + return the normalized prune ratios for each layer. + + """ + w_sum = 0 + _Max = 0 + for layername, ratio in ratios.items(): + wcount = self.weight_count[layername] + w_sum += ratio * wcount * \ + (1-self.analyzer.already_pruned[layername]) + target_count = self.weight_sum * target_pruned + for layername in ratios: + ratios[layername] = ratios[layername] * target_count / w_sum + _Max = max(_Max, ratios[layername]) + # Cannot Prune too much in a single iteration + # If a layer's prune ratio is larger than the + # MAX_PRUNE_RATIO_PER_ITER we rescal all prune + # ratios under this threshold + if _Max > MAX_PRUNE_RATIO_PER_ITER: + + for layername in ratios: + ratios[layername] = ratios[layername] * \ + MAX_PRUNE_RATIO_PER_ITER / _Max + return ratios + + def create_cfg(self, ratios): + """ + Generate the cfg_list for the pruner according to the prune ratios. + + Parameters + --------- + ratios: + For example: {'conv1' : 0.2} + + Returns + ------- + cfg_list: + For example: [{'sparsity':0.2, 'op_names':['conv1'], 'op_types':['Conv2d']}] + """ + cfg_list = [] + for layername in ratios: + prune_ratio = ratios[layername] + remain = 1 - self.analyzer.already_pruned[layername] + sparsity = remain * prune_ratio + \ + self.analyzer.already_pruned[layername] + if sparsity > 0: + # Pruner does not allow the prune ratio to be zero + cfg = {'sparsity': sparsity, 'op_names': [ + layername], 'op_types': ['Conv2d']} + cfg_list.append(cfg) + return cfg_list + + def current_sparsity(self): + """ + The sparsity of the weight. + """ + pruned_weight = 0 + for layer_name in self.analyzer.already_pruned: + w_count = self.weight_count[layer_name] + prune_ratio = self.analyzer.already_pruned[layer_name] + pruned_weight += w_count * prune_ratio + return pruned_weight / self.weight_sum + + def compress(self, eval_args=None, eval_kwargs=None, + finetune_args=None, finetune_kwargs=None, resume_sensitivity=None): + """ + This function iteratively prune the model according to the results of + the sensitivity analysis. + + Parameters + ---------- + eval_args: list + eval_kwargs: list& dict + Parameters for the val_funtion, the val_function will be called like + evaluator(\*eval_args, \*\*eval_kwargs) + finetune_args: list + finetune_kwargs: dict + Parameters for the finetuner function if needed. + resume_sensitivity: + resume the sensitivity results from this file. + """ + # pylint suggest not use the empty list and dict + # as the default input parameter + if not eval_args: + eval_args = [] + if not eval_kwargs: + eval_kwargs = {} + if not finetune_args: + finetune_args = [] + if not finetune_kwargs: + finetune_kwargs = {} + if self.ori_acc is None: + self.ori_acc = self.evaluator(*eval_args, **eval_kwargs) + assert isinstance(self.ori_acc, float) or isinstance(self.ori_acc, int) + if not resume_sensitivity: + self.sensitivities = self.analyzer.analysis( + val_args=eval_args, val_kwargs=eval_kwargs) + else: + self.sensitivities = self.load_sensitivity(resume_sensitivity) + self.analyzer.sensitivities = self.sensitivities + # the final target sparsity of the model + target_ratio = 1 - self.config_list[0]['sparsity'] + cur_ratio = self.remained_ratio + ori_acc = self.ori_acc + iteration_count = 0 + if self.checkpoint_dir is not None: + os.makedirs(self.checkpoint_dir, exist_ok=True) + modules_wrapper_final = None + while cur_ratio > target_ratio: + iteration_count += 1 + # Each round have three steps: + # 1) Get the current sensitivity for each layer(the sensitivity + # of each layer may change during the pruning) + # 2) Prune each layer according the sensitivies + # 3) finetune the model + _logger.info('Current base accuracy %f', ori_acc) + _logger.info('Remained %f weights', cur_ratio) + # determine the sparsity proportion between different + # layers according to the sensitivity result + proportion = self.sparsity_proportion_calc( + ori_acc, self.acc_drop_threshold, self.sensitivities) + + new_pruneratio = self.normalize(proportion, self.sparsity_per_iter) + cfg_list = self.create_cfg(new_pruneratio) + if not cfg_list: + _logger.error('The threshold is too small, please set a larger threshold') + return self.model + _logger.debug('Pruner Config: %s', str(cfg_list)) + cfg_str = ['%s:%.3f'%(cfg['op_names'][0], cfg['sparsity']) for cfg in cfg_list] + _logger.info('Current Sparsities: %s', ','.join(cfg_str)) + + pruner = self.Pruner(self.model, cfg_list) + pruner.compress() + pruned_acc = self.evaluator(*eval_args, **eval_kwargs) + _logger.info('Accuracy after pruning: %f', pruned_acc) + finetune_acc = pruned_acc + if self.finetuner is not None: + # if the finetune function is None, then skip the finetune + self.finetuner(*finetune_args, **finetune_kwargs) + finetune_acc = self.evaluator(*eval_args, **eval_kwargs) + _logger.info('Accuracy after finetune: %f', finetune_acc) + ori_acc = finetune_acc + # unwrap the pruner + pruner._unwrap_model() + # update the already prune ratio of each layer befor the new + # sensitivity analysis + for layer_cfg in cfg_list: + name = layer_cfg['op_names'][0] + sparsity = layer_cfg['sparsity'] + self.analyzer.already_pruned[name] = sparsity + # update the cur_ratio + cur_ratio = 1 - self.current_sparsity() + modules_wrapper_final = pruner.get_modules_wrapper() + del pruner + _logger.info('Currently remained weights: %f', cur_ratio) + + if self.checkpoint_dir is not None: + checkpoint_name = 'Iter_%d_finetune_acc_%.5f_sparsity_%.4f' % ( + iteration_count, finetune_acc, cur_ratio) + checkpoint_path = os.path.join( + self.checkpoint_dir, '%s.pth' % checkpoint_name) + cfg_path = os.path.join( + self.checkpoint_dir, '%s_pruner.json' % checkpoint_name) + sensitivity_path = os.path.join( + self.checkpoint_dir, '%s_sensitivity.csv' % checkpoint_name) + torch.save(self.model.state_dict(), checkpoint_path) + with open(cfg_path, 'w') as jf: + json.dump(cfg_list, jf) + self.analyzer.export(sensitivity_path) + + if cur_ratio > target_ratio: + # If this is the last prune iteration, skip the time-consuming + # sensitivity analysis + + self.analyzer.load_state_dict(self.model.state_dict()) + self.sensitivities = self.analyzer.analysis( + val_args=eval_args, val_kwargs=eval_kwargs) + + _logger.info('After Pruning: %.2f weights remains', cur_ratio) + self.modules_wrapper = modules_wrapper_final + + self._wrap_model() + return self.model + + def calc_mask(self, wrapper, **kwargs): + return None diff --git a/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/pruning/simulated_annealing_pruner.py b/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/pruning/simulated_annealing_pruner.py new file mode 100644 index 0000000000000000000000000000000000000000..b371b2c6fbb29f765fd6c6aa728d3dc98460d0c3 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/pruning/simulated_annealing_pruner.py @@ -0,0 +1,357 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +import os +import math +import copy +import csv +import json +import numpy as np +from schema import And, Optional + +from nni.utils import OptimizeMode + +from nni.compression.pytorch.compressor import Pruner +from nni.compression.pytorch.utils.config_validation import CompressorSchema +from .constants_pruner import PRUNER_DICT + + +_logger = logging.getLogger(__name__) + + +class SimulatedAnnealingPruner(Pruner): + """ + A Pytorch implementation of Simulated Annealing compression algorithm. + + Parameters + ---------- + model : pytorch model + The model to be pruned. + config_list : list + Supported keys: + - sparsity : The target overall sparsity. + - op_types : The operation type to prune. + evaluator : function + Function to evaluate the pruned model. + This function should include `model` as the only parameter, and returns a scalar value. + Example:: + + def evaluator(model): + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + val_loader = ... + model.eval() + correct = 0 + with torch.no_grad(): + for data, target in val_loader: + data, target = data.to(device), target.to(device) + output = model(data) + # get the index of the max log-probability + pred = output.argmax(dim=1, keepdim=True) + correct += pred.eq(target.view_as(pred)).sum().item() + accuracy = correct / len(val_loader.dataset) + return accuracy + optimize_mode : str + Optimize mode, `maximize` or `minimize`, by default `maximize`. + base_algo : str + Base pruning algorithm. `level`, `l1`, `l2` or `fpgm`, by default `l1`. Given the sparsity distribution among the ops, + the assigned `base_algo` is used to decide which filters/channels/weights to prune. + start_temperature : float + Start temperature of the simulated annealing process. + stop_temperature : float + Stop temperature of the simulated annealing process. + cool_down_rate : float + Cool down rate of the temperature. + perturbation_magnitude : float + Initial perturbation magnitude to the sparsities. The magnitude decreases with current temperature. + experiment_data_dir : string + PATH to save experiment data, + including the config_list generated for the base pruning algorithm, the performance of the pruned model and the pruning history. + + """ + + def __init__(self, model, config_list, evaluator, optimize_mode='maximize', base_algo='l1', + start_temperature=100, stop_temperature=20, cool_down_rate=0.9, perturbation_magnitude=0.35, experiment_data_dir='./'): + # original model + self._model_to_prune = copy.deepcopy(model) + self._base_algo = base_algo + + super().__init__(model, config_list) + + self._evaluator = evaluator + self._optimize_mode = OptimizeMode(optimize_mode) + + # hyper parameters for SA algorithm + self._start_temperature = start_temperature + self._current_temperature = start_temperature + self._stop_temperature = stop_temperature + self._cool_down_rate = cool_down_rate + self._perturbation_magnitude = perturbation_magnitude + + # overall pruning rate + self._sparsity = config_list[0]['sparsity'] + # pruning rates of the layers + self._sparsities = None + + # init current performance & best performance + self._current_performance = -np.inf + self._best_performance = -np.inf + self._best_config_list = [] + + self._search_history = [] + + self._experiment_data_dir = experiment_data_dir + if not os.path.exists(self._experiment_data_dir): + os.makedirs(self._experiment_data_dir) + + def validate_config(self, model, config_list): + """ + Parameters + ---------- + model : torch.nn.Module + Model to be pruned + config_list : list + List on pruning configs + """ + + if self._base_algo == 'level': + schema = CompressorSchema([{ + 'sparsity': And(float, lambda n: 0 < n < 1), + Optional('op_types'): [str], + Optional('op_names'): [str], + }], model, _logger) + elif self._base_algo in ['l1', 'l2', 'fpgm']: + schema = CompressorSchema([{ + 'sparsity': And(float, lambda n: 0 < n < 1), + 'op_types': ['Conv2d'], + Optional('op_names'): [str] + }], model, _logger) + + schema.validate(config_list) + + def _sparsities_2_config_list(self, sparsities): + ''' + convert sparsities vector into config_list for LevelPruner or L1FilterPruner + + Parameters + ---------- + sparsities : list + list of sparsities + + Returns + ------- + list of dict + config_list for LevelPruner or L1FilterPruner + ''' + config_list = [] + + sparsities = sorted(sparsities) + self.modules_wrapper = sorted( + self.modules_wrapper, key=lambda wrapper: wrapper.module.weight.data.numel()) + + # a layer with more weights will have no less pruning rate + for idx, wrapper in enumerate(self.get_modules_wrapper()): + # L1Filter Pruner requires to specify op_types + if self._base_algo in ['l1', 'l2', 'fpgm']: + config_list.append( + {'sparsity': sparsities[idx], 'op_types': ['Conv2d'], 'op_names': [wrapper.name]}) + elif self._base_algo == 'level': + config_list.append( + {'sparsity': sparsities[idx], 'op_names': [wrapper.name]}) + + config_list = [val for val in config_list if not math.isclose(val['sparsity'], 0, abs_tol=1e-6)] + + return config_list + + def _rescale_sparsities(self, sparsities, target_sparsity): + ''' + Rescale the sparsities list to satisfy the target overall sparsity + + Parameters + ---------- + sparsities : list + + target_sparsity : float + the target overall sparsity + + Returns + ------- + list + the rescaled sparsities + ''' + num_weights = [] + for wrapper in self.get_modules_wrapper(): + num_weights.append(wrapper.module.weight.data.numel()) + + num_weights = sorted(num_weights) + sparsities = sorted(sparsities) + + total_weights = 0 + total_weights_pruned = 0 + + # calculate the scale + for idx, num_weight in enumerate(num_weights): + total_weights += num_weight + total_weights_pruned += int(num_weight*sparsities[idx]) + if total_weights_pruned == 0: + return None + scale = target_sparsity / (total_weights_pruned/total_weights) + + # rescale the sparsities + sparsities = np.asarray(sparsities)*scale + + return sparsities + + def _init_sparsities(self): + ''' + Generate a sorted sparsities vector + ''' + # repeatedly generate a distribution until satisfies the overall sparsity requirement + _logger.info('Gererating sparsities...') + while True: + sparsities = sorted(np.random.uniform( + 0, 1, len(self.get_modules_wrapper()))) + + sparsities = self._rescale_sparsities( + sparsities, target_sparsity=self._sparsity) + + if sparsities is not None and sparsities[0] >= 0 and sparsities[-1] < 1: + _logger.info('Initial sparsities generated : %s', sparsities) + self._sparsities = sparsities + break + + def _generate_perturbations(self): + ''' + Generate perturbation to the current sparsities distribution. + + Returns: + -------- + list + perturbated sparsities + ''' + _logger.info("Gererating perturbations to the current sparsities...") + + # decrease magnitude with current temperature + magnitude = self._current_temperature / \ + self._start_temperature * self._perturbation_magnitude + _logger.info('current perturation magnitude:%s', magnitude) + + while True: + perturbation = np.random.uniform(-magnitude, magnitude, len(self.get_modules_wrapper())) + sparsities = np.clip(0, self._sparsities + perturbation, None) + _logger.debug("sparsities before rescalling:%s", sparsities) + + sparsities = self._rescale_sparsities(sparsities, target_sparsity=self._sparsity) + _logger.debug("sparsities after rescalling:%s", sparsities) + + if sparsities is not None and sparsities[0] >= 0 and sparsities[-1] < 1: + _logger.info("Sparsities perturbated:%s", sparsities) + return sparsities + + def calc_mask(self, wrapper, **kwargs): + return None + + def compress(self, return_config_list=False): + """ + Compress the model with Simulated Annealing. + + Returns + ------- + torch.nn.Module + model with specified modules compressed. + """ + _logger.info('Starting Simulated Annealing Compression...') + + # initiaze a randomized action + pruning_iteration = 0 + self._init_sparsities() + + # stop condition + self._current_temperature = self._start_temperature + while self._current_temperature > self._stop_temperature: + _logger.info('Pruning iteration: %d', pruning_iteration) + _logger.info('Current temperature: %d, Stop temperature: %d', + self._current_temperature, self._stop_temperature) + while True: + # generate perturbation + sparsities_perturbated = self._generate_perturbations() + config_list = self._sparsities_2_config_list( + sparsities_perturbated) + _logger.info( + "config_list for Pruner generated: %s", config_list) + + # fast evaluation + pruner = PRUNER_DICT[self._base_algo](copy.deepcopy(self._model_to_prune), config_list) + model_masked = pruner.compress() + evaluation_result = self._evaluator(model_masked) + + self._search_history.append( + {'sparsity': self._sparsity, 'performance': evaluation_result, 'config_list': config_list}) + + if self._optimize_mode is OptimizeMode.Minimize: + evaluation_result *= -1 + + # if better evaluation result, then accept the perturbation + if evaluation_result > self._current_performance: + self._current_performance = evaluation_result + self._sparsities = sparsities_perturbated + + # save best performance and best params + if evaluation_result > self._best_performance: + _logger.info('updating best model...') + self._best_performance = evaluation_result + self._best_config_list = config_list + + # save the overall best masked model + self.bound_model = model_masked + # the ops with sparsity 0 are not included in this modules_wrapper + modules_wrapper_final = pruner.get_modules_wrapper() + break + # if not, accept with probability e^(-deltaE/current_temperature) + else: + delta_E = np.abs(evaluation_result - + self._current_performance) + probability = math.exp(-1 * delta_E / + self._current_temperature) + if np.random.uniform(0, 1) < probability: + self._current_performance = evaluation_result + self._sparsities = sparsities_perturbated + break + + # cool down + self._current_temperature *= self._cool_down_rate + pruning_iteration += 1 + + _logger.info('----------Compression finished--------------') + _logger.info('Best performance: %s', self._best_performance) + _logger.info('config_list found : %s', + self._best_config_list) + + # save search history + with open(os.path.join(self._experiment_data_dir, 'search_history.csv'), 'w') as csvfile: + writer = csv.DictWriter(csvfile, fieldnames=['sparsity', 'performance', 'config_list']) + writer.writeheader() + for item in self._search_history: + writer.writerow({'sparsity': item['sparsity'], 'performance': item['performance'], 'config_list': json.dumps( + item['config_list'])}) + + # save best config found and best performance + if self._optimize_mode is OptimizeMode.Minimize: + self._best_performance *= -1 + with open(os.path.join(self._experiment_data_dir, 'search_result.json'), 'w+') as jsonfile: + json.dump({ + 'performance': self._best_performance, + 'config_list': json.dumps(self._best_config_list) + }, jsonfile) + + _logger.info('search history and result saved to foler : %s', + self._experiment_data_dir) + + if return_config_list: + return self._best_config_list + + # This should be done only at the final stage, + # because the modules_wrapper with all the ops are used during the annealing process + self.modules_wrapper = modules_wrapper_final + + return self.bound_model diff --git a/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/pruning/structured_pruning_masker.py b/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/pruning/structured_pruning_masker.py new file mode 100644 index 0000000000000000000000000000000000000000..96e61d355dda01dc5dee8ff505289f5d29c156fc --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/pruning/structured_pruning_masker.py @@ -0,0 +1,965 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +import math +import numpy as np +import torch +from .weight_masker import WeightMasker + +__all__ = ['L1FilterPrunerMasker', 'L2FilterPrunerMasker', 'FPGMPrunerMasker', + 'TaylorFOWeightFilterPrunerMasker', 'ActivationAPoZRankFilterPrunerMasker', + 'ActivationMeanRankFilterPrunerMasker', 'SlimPrunerMasker', 'AMCWeightMasker'] + +logger = logging.getLogger('torch filter pruners') + + +class StructuredWeightMasker(WeightMasker): + """ + A structured pruning masker base class that prunes convolutional layer filters. + + Parameters + ---------- + model: nn.Module + model to be pruned + pruner: Pruner + A Pruner instance used to prune the model + preserve_round: int + after pruning, preserve filters/channels round to `preserve_round`, for example: + for a Conv2d layer, output channel is 32, sparsity is 0.2, if preserve_round is + 1 (no preserve round), then there will be int(32 * 0.2) = 6 filters pruned, and + 32 - 6 = 26 filters are preserved. If preserve_round is 4, preserved filters will + be round up to 28 (which can be divided by 4) and only 4 filters are pruned. + + """ + + def __init__(self, model, pruner, preserve_round=1, dependency_aware=False): + self.model = model + self.pruner = pruner + self.preserve_round = preserve_round + self.dependency_aware = dependency_aware + + def calc_mask(self, sparsity, wrapper, wrapper_idx=None, **depen_kwargs): + """ + calculate the mask for `wrapper`. + + Parameters + ---------- + sparsity: float/list of float + The target sparsity of the wrapper. If we calculate the mask in + the normal way, then sparsity is a float number. In contrast, if + we calculate the mask in the dependency-aware way, sparsity is a + list of float numbers, each float number corressponds to a sparsity + of a layer. + wrapper: PrunerModuleWrapper/list of PrunerModuleWrappers + The wrapper of the target layer. If we calculate the mask in the normal + way, then `wrapper` is an instance of PrunerModuleWrapper, else `wrapper` + is a list of PrunerModuleWrapper. + wrapper_idx: int/list of int + The index of the wrapper. + depen_kwargs: dict + The kw_args for the dependency-aware mode. + """ + if not self.dependency_aware: + # calculate the mask in the normal way, each layer calculate its + # own mask separately + return self._normal_calc_mask(sparsity, wrapper, wrapper_idx) + else: + # if the dependency_aware switch is on, then calculate the mask + # in the dependency-aware way + return self._dependency_calc_mask(sparsity, wrapper, wrapper_idx, **depen_kwargs) + + def _get_current_state(self, sparsity, wrapper, wrapper_idx=None): + """ + Some pruner may prune the layers in a iterative way. In each pruning iteration, + we may get the current state of this wrapper/layer, and continue to prune this layer + based on the current state. This function is to get the current pruning state of the + target wrapper/layer. + Parameters + ---------- + sparsity: float + pruning ratio, preserved weight ratio is `1 - sparsity` + wrapper: PrunerModuleWrapper + layer wrapper of this layer + wrapper_idx: int + index of this wrapper in pruner's all wrappers + Returns + ------- + base_mask: dict + dict object that stores the mask of this wrapper in this iteration, if it is the + first iteration, then we create a new mask with all ones. If there is already a + mask in this wrapper, then we return the existing mask. + weight: tensor + the current weight of this layer + num_prune: int + how many filters we should prune + """ + msg = 'module type {} is not supported!'.format(wrapper.type) + # assert wrapper.type == 'Conv2d', msg + weight = wrapper.module.weight.data + bias = None + if hasattr(wrapper.module, 'bias') and wrapper.module.bias is not None: + bias = wrapper.module.bias.data + + if wrapper.weight_mask is None: + mask_weight = torch.ones(weight.size()).type_as(weight).detach() + else: + mask_weight = wrapper.weight_mask.clone() + if bias is not None: + if wrapper.bias_mask is None: + mask_bias = torch.ones(bias.size()).type_as(bias).detach() + else: + mask_bias = wrapper.bias_mask.clone() + else: + mask_bias = None + mask = {'weight_mask': mask_weight, 'bias_mask': mask_bias} + + # num_total = weight.size(0) + + if isinstance(wrapper.module, torch.nn.Conv2d): + num_total = weight.size(0) + elif isinstance(wrapper.module, torch.nn.ConvTranspose2d): + num_total = weight.size(1) + + num_prune = int(num_total * sparsity) + if self.preserve_round > 1: + num_preserve = num_total - num_prune + num_preserve = int( + math.ceil(num_preserve * 1. / self.preserve_round) * self.preserve_round) + if num_preserve > num_total: + num_preserve = int(math.floor( + num_total * 1. / self.preserve_round) * self.preserve_round) + num_prune = num_total - num_preserve + # weight*mask_weight: apply base mask for iterative pruning + return mask, weight * mask_weight, num_prune + + def _normal_calc_mask(self, sparsity, wrapper, wrapper_idx=None): + """ + Calculate the mask of given layer. + Parameters + ---------- + sparsity: float + pruning ratio, preserved weight ratio is `1 - sparsity` + wrapper: PrunerModuleWrapper + layer wrapper of this layer + wrapper_idx: int + index of this wrapper in pruner's all wrappers + Returns + ------- + dict + dictionary for storing masks, keys of the dict: + 'weight_mask': weight mask tensor + 'bias_mask': bias mask tensor (optional) + """ + mask, weight, num_prune = self._get_current_state( + sparsity, wrapper, wrapper_idx) + num_total = weight.size(0) + if num_total < 2 or num_prune < 1: + return mask + + return self.get_mask(mask, weight, num_prune, wrapper, wrapper_idx) + + def _common_channel_to_prune(self, sparsities, wrappers, wrappers_idx, channel_dsets, groups): + """ + Calculate the common channels should be pruned by all the layers in this group. + This function is for filter pruning of Conv layers. if want to support the dependency-aware + mode for others ops, you need to inherit this class and overwrite `_common_channel_to_prune`. + + Parameters + ---------- + sparsities : list + List of float that specify the sparsity for each conv layer. + wrappers : list + List of wrappers + groups : list + The number of the filter groups of each layer. + wrappers_idx : list + The indexes of the wrappers + """ + # sparsity configs for each wrapper + # sparsities = [_w.config['sparsity'] for _w in wrappers] + # check the type of the input wrappers + for _w in wrappers: + msg = 'module type {} is not supported!'.format(_w.type) + assert _w.type == 'Conv2d', msg + # Among the dependent layers, the layer with smallest + # sparsity determines the final benefit of the speedup + # module. To better harvest the speed benefit, we need + # to ensure that these dependent layers have at least + # `min_sparsity` pruned channel are the same. + if len(channel_dsets) == len(wrappers): + # all the layers in the dependency sets are pruned + min_sparsity = min(sparsities) + else: + # not all the layers in the dependency set + # are pruned + min_sparsity = 0 + # donnot prune the channels that we cannot harvest the speed from + sparsities = [min_sparsity] * len(sparsities) + # find the max number of the filter groups of the dependent + # layers. The group constraint of this dependency set is decided + # by the layer with the max groups. + + # should use the least common multiple for all the groups + # the max_group is lower than the channel_count, because + # the number of the filter is always divisible by the number of the group + max_group = np.lcm.reduce(groups) + channel_count = wrappers[0].module.weight.data.size(0) + device = wrappers[0].module.weight.device + channel_sum = torch.zeros(channel_count).to(device) + for _w, _w_idx in zip(wrappers, wrappers_idx): + # calculate the L1/L2 sum for all channels + c_sum = self.get_channel_sum(_w, _w_idx) + + if c_sum is None: + # if the channel sum cannot be calculated + # now, return None + return None + channel_sum += c_sum + + # prune the same `min_sparsity` channels based on channel_sum + # for all the layers in the channel sparsity + target_pruned = int(channel_count * min_sparsity) + # pruned_per_group may be zero, for example dw conv + pruned_per_group = int(target_pruned / max_group) + group_step = int(channel_count / max_group) + + channel_masks = [] + for gid in range(max_group): + _start = gid * group_step + _end = (gid + 1) * group_step + if pruned_per_group > 0: + threshold = torch.topk( + channel_sum[_start: _end], pruned_per_group, largest=False)[0].max() + group_mask = torch.gt(channel_sum[_start:_end], threshold) + else: + group_mask = torch.ones(group_step).to(device) + channel_masks.append(group_mask) + channel_masks = torch.cat(channel_masks, dim=0) + pruned_channel_index = ( + channel_masks == False).nonzero().squeeze(1).tolist() + logger.info('Prune the %s channels for all dependent', + ','.join([str(x) for x in pruned_channel_index])) + return channel_masks + + def _dependency_calc_mask(self, sparsities, wrappers, wrappers_idx, channel_dsets, groups): + """ + Calculate the masks for the layers in the same dependency sets. + Similar to the traditional original calc_mask, _dependency_calc_mask + will prune the target layers based on the L1/L2 norm of the weights. + However, StructuredWeightMasker prunes the filter completely based on the + L1/L2 norm of each filter. In contrast, _dependency_calc_mask + will try to satisfy the channel/group dependency(see nni.compression.torch. + utils.shape_dependency for details). Specifically, _dependency_calc_mask + will try to prune the same channels for the layers that have channel dependency. + In addition, this mask calculator will also ensure that the number of filters + pruned in each group is the same(meet the group dependency). + + Parameters + ---------- + sparsities : list + List of float that specify the sparsity for each conv layer. + wrappers : list + List of wrappers + groups : list + The number of the filter groups of each layer. + wrappers_idx : list + The indexes of the wrappers + """ + channel_masks = self._common_channel_to_prune( + sparsities, wrappers, wrappers_idx, channel_dsets, groups) + # calculate the mask for each layer based on channel_masks, first + # every layer will prune the same channels masked in channel_masks. + # If the sparsity of a layers is larger than min_sparsity, then it + # will continue prune sparsity - min_sparsity channels to meet the sparsity + # config. + masks = {} + for _pos, _w in enumerate(wrappers): + _w_idx = wrappers_idx[_pos] + sparsity = sparsities[_pos] + name = _w.name + + # _tmp_mask = self._normal_calc_mask( + # sparsity, _w, _w_idx, channel_masks) + base_mask, current_weight, num_prune = self._get_current_state( + sparsity, _w, _w_idx) + num_total = current_weight.size(0) + if num_total < 2 or num_prune < 1: + masks[name] = base_mask + continue + _tmp_mask = self.get_mask( + base_mask, current_weight, num_prune, _w, _w_idx, channel_masks) + + if _tmp_mask is None: + # if the mask calculation fails + return None + masks[name] = _tmp_mask + return masks + + def get_mask(self, base_mask, weight, num_prune, wrapper, wrapper_idx, channel_masks=None): + """ + Calculate the mask of given layer. + + Parameters + ---------- + base_mask: dict + The basic mask with the same shape of weight, all item in the basic mask is 1. + weight: tensor + the module weight to be pruned + num_prune: int + Num of filters to prune + wrapper: PrunerModuleWrapper + layer wrapper of this layer + wrapper_idx: int + index of this wrapper in pruner's all wrappers + channel_masks: Tensor + If mask some channels for this layer in advance. In the dependency-aware + mode, before calculating the masks for each layer, we will calculate a common + mask for all the layers in the dependency set. For the pruners that doesnot + support dependency-aware mode, they can just ignore this parameter. + + Returns + ------- + dict + dictionary for storing masks + """ + raise NotImplementedError( + '{} get_mask is not implemented'.format(self.__class__.__name__)) + + def get_channel_sum(self, wrapper, wrapper_idx): + """ + Calculate the importance weight for each channel. If want to support the + dependency-aware mode for this one-shot pruner, this function must be + implemented. + Parameters + ---------- + wrapper: PrunerModuleWrapper + layer wrapper of this layer + wrapper_idx: int + index of this wrapper in pruner's all wrappers + Returns + ------- + tensor + Tensor that indicates the importance of each channel + """ + raise NotImplementedError( + '{} get_channel_sum is not implemented'.format(self.__class__.__name__)) + + +class L1FilterPrunerMasker(StructuredWeightMasker): + """ + A structured pruning algorithm that prunes the filters of smallest magnitude + weights sum in the convolution layers to achieve a preset level of network sparsity. + Hao Li, Asim Kadav, Igor Durdanovic, Hanan Samet and Hans Peter Graf, + "PRUNING FILTERS FOR EFFICIENT CONVNETS", 2017 ICLR + https://arxiv.org/abs/1608.08710 + """ + + def get_mask(self, base_mask, weight, num_prune, wrapper, wrapper_idx, channel_masks=None): + # get the l1-norm sum for each filter + # w_abs_structured = self.get_channel_sum(wrapper, wrapper_idx) + # if channel_masks is not None: + # # if we need to mask some channels in advance + # w_abs_structured = w_abs_structured * channel_masks + # threshold = torch.topk(w_abs_structured.view(-1), + # num_prune, largest=False)[0].max() + # mask_weight = torch.gt(w_abs_structured, threshold)[ + # :, None, None, None].expand_as(weight).type_as(weight) + # mask_bias = torch.gt(w_abs_structured, threshold).type_as( + # weight).detach() if base_mask['bias_mask'] is not None else None + + # return {'weight_mask': mask_weight.detach(), 'bias_mask': mask_bias} + if isinstance(wrapper.module, torch.nn.Conv2d): + # get the l1-norm sum for each filter + w_abs_structured = self.get_channel_sum(wrapper, wrapper_idx) + if channel_masks is not None: + # if we need to mask some channels in advance + w_abs_structured = w_abs_structured * channel_masks + threshold = torch.topk(w_abs_structured.view(-1), + num_prune, largest=False)[0].max() + mask_weight = torch.gt(w_abs_structured, threshold)[ + :, None, None, None].expand_as(weight).type_as(weight) + mask_bias = torch.gt(w_abs_structured, threshold).type_as( + weight).detach() if base_mask['bias_mask'] is not None else None + + return {'weight_mask': mask_weight.detach(), 'bias_mask': mask_bias} + elif isinstance(wrapper.module, torch.nn.ConvTranspose2d): + # get the l1-norm sum for each filter + w_abs_structured = self.get_channel_sum(wrapper, wrapper_idx) + if channel_masks is not None: + # if we need to mask some channels in advance + w_abs_structured = w_abs_structured * channel_masks + threshold = torch.topk(w_abs_structured.view(-1), + num_prune, largest=False)[0].max() + mask_weight = torch.gt(w_abs_structured, threshold)[ + None, :, None, None].expand_as(weight).type_as(weight) + mask_bias = torch.gt(w_abs_structured, threshold).type_as( + weight).detach() if base_mask['bias_mask'] is not None else None + + return {'weight_mask': mask_weight.detach(), 'bias_mask': mask_bias} + + def get_channel_sum(self, wrapper, wrapper_idx): + # weight = wrapper.module.weight.data + # filters = weight.shape[0] + # w_abs = weight.abs() + # w_abs_structured = w_abs.view(filters, -1).sum(dim=1) + # return w_abs_structured + if isinstance(wrapper.module, torch.nn.Conv2d): + weight = wrapper.module.weight.data + filters = weight.shape[0] + w_abs = weight.abs() + w_abs_structured = w_abs.view(filters, -1).sum(dim=1) + return w_abs_structured + elif isinstance(wrapper.module, torch.nn.ConvTranspose2d): + weight = wrapper.module.weight.data + filters = weight.shape[1] + w_abs = weight.abs() + w_abs_structured = w_abs.view(filters, -1).sum(dim=1) + return w_abs_structured + + +class L2FilterPrunerMasker(StructuredWeightMasker): + """ + A structured pruning algorithm that prunes the filters with the + smallest L2 norm of the weights. + """ + + def get_mask(self, base_mask, weight, num_prune, wrapper, wrapper_idx, channel_masks=None): + # get the l2-norm sum for each filter + w_l2_norm = self.get_channel_sum(wrapper, wrapper_idx) + if channel_masks is not None: + # if we need to mask some channels in advance + w_l2_norm = w_l2_norm * channel_masks + threshold = torch.topk( + w_l2_norm.view(-1), num_prune, largest=False)[0].max() + mask_weight = torch.gt(w_l2_norm, threshold)[ + :, None, None, None].expand_as(weight).type_as(weight) + mask_bias = torch.gt(w_l2_norm, threshold).type_as( + weight).detach() if base_mask['bias_mask'] is not None else None + + return {'weight_mask': mask_weight.detach(), 'bias_mask': mask_bias} + + def get_channel_sum(self, wrapper, wrapper_idx): + weight = wrapper.module.weight.data + filters = weight.shape[0] + w = weight.view(filters, -1) + w_l2_norm = torch.sqrt((w ** 2).sum(dim=1)) + return w_l2_norm + + +class FPGMPrunerMasker(StructuredWeightMasker): + """ + A filter pruner via geometric median. + "Filter Pruning via Geometric Median for Deep Convolutional Neural Networks Acceleration", + https://arxiv.org/pdf/1811.00250.pdf + """ + + def get_mask(self, base_mask, weight, num_prune, wrapper, wrapper_idx, channel_masks=None): + if isinstance(wrapper.module, torch.nn.Conv2d): + min_gm_idx = self._get_min_gm_kernel_idx( + num_prune, wrapper, wrapper_idx, channel_masks) + for idx in min_gm_idx: + base_mask['weight_mask'][idx] = 0. + if base_mask['bias_mask'] is not None: + base_mask['bias_mask'][idx] = 0. + return base_mask + elif isinstance(wrapper.module, torch.nn.ConvTranspose2d): + min_gm_idx = self._get_min_gm_kernel_idx( + num_prune, wrapper, wrapper_idx, channel_masks) + for idx in min_gm_idx: + base_mask['weight_mask'][:, idx] = 0. + if base_mask['bias_mask'] is not None: + base_mask['bias_mask'][idx] = 0. + return base_mask + + def _get_min_gm_kernel_idx(self, num_prune, wrapper, wrapper_idx, channel_masks): + if isinstance(wrapper.module, torch.nn.Conv2d): + channel_dist = self.get_channel_sum(wrapper, wrapper_idx) + if channel_masks is not None: + channel_dist = channel_dist * channel_masks + dist_list = [(channel_dist[i], i) + for i in range(channel_dist.size(0))] + min_gm_kernels = sorted(dist_list, key=lambda x: x[0])[:num_prune] + return [x[1] for x in min_gm_kernels] + elif isinstance(wrapper.module, torch.nn.ConvTranspose2d): + channel_dist = self.get_channel_sum(wrapper, wrapper_idx) + if channel_masks is not None: + channel_dist = channel_dist * channel_masks + dist_list = [(channel_dist[i], i) + for i in range(channel_dist.size(0))] + min_gm_kernels = sorted(dist_list, key=lambda x: x[0])[:num_prune] + return [x[1] for x in min_gm_kernels] + + def _get_distance_sum(self, weight, out_idx, wrapper): + """ + Calculate the total distance between a specified filter (by out_idex and in_idx) and + all other filters. + Parameters + ---------- + weight: Tensor + convolutional filter weight + out_idx: int + output channel index of specified filter, this method calculates the total distance + between this specified filter and all other filters. + Returns + ------- + float32 + The total distance + """ + if isinstance(wrapper.module, torch.nn.Conv2d): + logger.debug('weight size: %s', weight.size()) + assert len(weight.size()) in [3, 4], 'unsupported weight shape' + + w = weight.view(weight.size(0), -1) + anchor_w = w[out_idx].unsqueeze(0).expand(w.size(0), w.size(1)) + x = w - anchor_w + x = (x * x).sum(-1) + x = torch.sqrt(x) + return x.sum() + elif isinstance(wrapper.module, torch.nn.ConvTranspose2d): + logger.debug('weight size: %s', weight.size()) + assert len(weight.size()) in [3, 4], 'unsupported weight shape' + + w = weight.view(weight.size(1), -1) + anchor_w = w[out_idx].unsqueeze(0).expand(w.size(0), w.size(1)) + x = w - anchor_w + x = (x * x).sum(-1) + x = torch.sqrt(x) + return x.sum() + + + def get_channel_sum(self, wrapper, wrapper_idx): + if isinstance(wrapper.module, torch.nn.Conv2d): + weight = wrapper.module.weight.data + assert len(weight.size()) in [3, 4] + dist_list = [] + for out_i in range(weight.size(0)): + dist_sum = self._get_distance_sum(weight, out_i, wrapper) + dist_list.append(dist_sum) + return torch.Tensor(dist_list).to(weight.device) + elif isinstance(wrapper.module, torch.nn.ConvTranspose2d): + weight = wrapper.module.weight.data + assert len(weight.size()) in [3, 4] + dist_list = [] + for out_i in range(weight.size(1)): + dist_sum = self._get_distance_sum(weight, out_i, wrapper) + dist_list.append(dist_sum) + return torch.Tensor(dist_list).to(weight.device) + + +class TaylorFOWeightFilterPrunerMasker(StructuredWeightMasker): + """ + A structured pruning algorithm that prunes the filters with the smallest + importance approximations based on the first order taylor expansion on the weight. + Molchanov, Pavlo and Mallya, Arun and Tyree, Stephen and Frosio, Iuri and Kautz, Jan, + "Importance Estimation for Neural Network Pruning", CVPR 2019. + http://jankautz.com/publications/Importance4NNPruning_CVPR19.pdf + """ + + def __init__(self, model, pruner, statistics_batch_num=1): + super().__init__(model, pruner) + self.statistics_batch_num = statistics_batch_num + self.pruner.iterations = 0 + self.pruner.set_wrappers_attribute("contribution", None) + self.pruner.patch_optimizer(self.calc_contributions) + + def get_mask(self, base_mask, weight, num_prune, wrapper, wrapper_idx, channel_masks=None): + if isinstance(wrapper.module, torch.nn.Conv2d): + channel_contribution = self.get_channel_sum(wrapper, wrapper_idx) + if channel_contribution is None: + # iteration is not enough + return None + if channel_masks is not None: + channel_contribution = channel_contribution * channel_masks + prune_indices = torch.argsort(channel_contribution)[:num_prune] + for idx in prune_indices: + base_mask['weight_mask'][idx] = 0. + if base_mask['bias_mask'] is not None: + base_mask['bias_mask'][idx] = 0. + return base_mask + elif isinstance(wrapper.module, torch.nn.ConvTranspose2d): + channel_contribution = self.get_channel_sum(wrapper, wrapper_idx) + if channel_contribution is None: + # iteration is not enough + return None + if channel_masks is not None: + channel_contribution = channel_contribution * channel_masks + prune_indices = torch.argsort(channel_contribution)[:num_prune] + for idx in prune_indices: + base_mask['weight_mask'][:, idx] = 0. + if base_mask['bias_mask'] is not None: + base_mask['bias_mask'][idx] = 0. + return base_mask + + def calc_contributions(self): + """ + Calculate the estimated importance of filters as a sum of individual contribution + based on the first order taylor expansion. + """ + if self.pruner.iterations >= self.statistics_batch_num: + return + + for wrapper in self.pruner.get_modules_wrapper(): + if isinstance(wrapper.module, torch.nn.Conv2d): + filters = wrapper.module.weight.size(0) + contribution = ( + wrapper.module.weight * wrapper.module.weight.grad).data.pow(2).view(filters, -1).sum(dim=1) + if wrapper.contribution is None: + wrapper.contribution = contribution + else: + wrapper.contribution += contribution + + elif isinstance(wrapper.module, torch.nn.ConvTranspose2d): + filters = wrapper.module.weight.size(1) + contribution = ( + wrapper.module.weight * wrapper.module.weight.grad).data.pow(2).view(filters, -1).sum(dim=0) + if wrapper.contribution is None: + wrapper.contribution = contribution + else: + wrapper.contribution += contribution + + self.pruner.iterations += 1 + + def get_channel_sum(self, wrapper, wrapper_idx): + if self.pruner.iterations < self.statistics_batch_num: + return None + if wrapper.contribution is None: + return None + return wrapper.contribution + + +class ActivationFilterPrunerMasker(StructuredWeightMasker): + def __init__(self, model, pruner, statistics_batch_num=1, activation='relu'): + super().__init__(model, pruner) + self.statistics_batch_num = statistics_batch_num + self.pruner.hook_id = self._add_activation_collector(self.pruner) + self.pruner.iterations = 0 + self.pruner.patch_optimizer(self._iteration_counter) + + assert activation in ['relu', 'relu6'] + if activation == 'relu': + self.pruner.activation = torch.nn.functional.relu + elif activation == 'relu6': + self.pruner.activation = torch.nn.functional.relu6 + else: + self.pruner.activation = None + + def _iteration_counter(self): + self.pruner.iterations += 1 + + def _add_activation_collector(self, pruner): + def collector(collected_activation): + def hook(module_, input_, output): + collected_activation.append( + pruner.activation(output.detach().cpu())) + return hook + pruner.collected_activation = {} + pruner._fwd_hook_id += 1 + pruner._fwd_hook_handles[pruner._fwd_hook_id] = [] + + for wrapper_idx, wrapper in enumerate(pruner.get_modules_wrapper()): + pruner.collected_activation[wrapper_idx] = [] + handle = wrapper.register_forward_hook( + collector(pruner.collected_activation[wrapper_idx])) + + pruner._fwd_hook_handles[pruner._fwd_hook_id].append(handle) + return pruner._fwd_hook_id + + +class ActivationAPoZRankFilterPrunerMasker(ActivationFilterPrunerMasker): + """ + A structured pruning algorithm that prunes the filters with the + smallest APoZ(average percentage of zeros) of output activations. + Hengyuan Hu, Rui Peng, Yu-Wing Tai and Chi-Keung Tang, + "Network Trimming: A Data-Driven Neuron Pruning Approach towards Efficient Deep Architectures", ICLR 2016. + https://arxiv.org/abs/1607.03250 + """ + + def get_mask(self, base_mask, weight, num_prune, wrapper, wrapper_idx, channel_masks=None): + apoz = self.get_channel_sum(wrapper, wrapper_idx) + if apoz is None: + # the collected activations are not enough + return None + if channel_masks is not None: + apoz = apoz * channel_masks + + prune_indices = torch.argsort(apoz)[:num_prune] + for idx in prune_indices: + base_mask['weight_mask'][idx] = 0. + if base_mask['bias_mask'] is not None: + base_mask['bias_mask'][idx] = 0. + + if self.pruner.hook_id in self.pruner._fwd_hook_handles: + self.pruner.remove_activation_collector(self.pruner.hook_id) + + return base_mask + + def _calc_apoz(self, activations): + """ + Calculate APoZ(average percentage of zeros) of activations. + + Parameters + ---------- + activations : list + Layer's output activations + + Returns + ------- + torch.Tensor + Filter's APoZ(average percentage of zeros) of the activations + """ + activations = torch.cat(activations, 0) + _eq_zero = torch.eq(activations, torch.zeros_like(activations)) + _apoz = torch.sum(_eq_zero, dim=(0, 2, 3), dtype=torch.float64) / \ + torch.numel(_eq_zero[:, 0, :, :]) + return torch.ones_like(_apoz) - _apoz + + def get_channel_sum(self, wrapper, wrapper_idx): + assert wrapper_idx is not None + activations = self.pruner.collected_activation[wrapper_idx] + if len(activations) < self.statistics_batch_num: + # collected activations is not enough + return None + return self._calc_apoz(activations).to(wrapper.module.weight.device) + + +class ActivationMeanRankFilterPrunerMasker(ActivationFilterPrunerMasker): + """ + A structured pruning algorithm that prunes the filters with the + smallest mean value of output activations. + Pavlo Molchanov, Stephen Tyree, Tero Karras, Timo Aila and Jan Kautz, + "Pruning Convolutional Neural Networks for Resource Efficient Inference", ICLR 2017. + https://arxiv.org/abs/1611.06440 + """ + + def get_mask(self, base_mask, weight, num_prune, wrapper, wrapper_idx, channel_masks=None): + + mean_activation = self.get_channel_sum(wrapper, wrapper_idx) + if mean_activation is None: + # the collected activation is not enough + return None + if channel_masks is not None: + mean_activation = mean_activation * channel_masks + + prune_indices = torch.argsort(mean_activation)[:num_prune] + for idx in prune_indices: + base_mask['weight_mask'][idx] = 0. + if base_mask['bias_mask'] is not None: + base_mask['bias_mask'][idx] = 0. + # if len(activations) < self.statistics_batch_num, the code + # cannot reach here + if self.pruner.hook_id in self.pruner._fwd_hook_handles: + self.pruner.remove_activation_collector(self.pruner.hook_id) + + return base_mask + + def _cal_mean_activation(self, activations): + """ + Calculate mean value of activations. + + Parameters + ---------- + activations : list + Layer's output activations + + Returns + ------- + torch.Tensor + Filter's mean value of the output activations + """ + activations = torch.cat(activations, 0) + mean_activation = torch.mean(activations, dim=(0, 2, 3)) + return mean_activation + + def get_channel_sum(self, wrapper, wrapper_idx): + assert wrapper_idx is not None + activations = self.pruner.collected_activation[wrapper_idx] + if len(activations) < self.statistics_batch_num: + return None + # the memory overhead here is acceptable, because only + # the mean_activation tensor returned by _cal_mean_activation + # is transfer to gpu. + return self._cal_mean_activation(activations).to(wrapper.module.weight.device) + + +class SlimPrunerMasker(WeightMasker): + """ + A structured pruning algorithm that prunes channels by pruning the weights of BN layers. + Zhuang Liu, Jianguo Li, Zhiqiang Shen, Gao Huang, Shoumeng Yan and Changshui Zhang + "Learning Efficient Convolutional Networks through Network Slimming", 2017 ICCV + https://arxiv.org/pdf/1708.06519.pdf + """ + + def __init__(self, model, pruner, **kwargs): + super().__init__(model, pruner) + self.global_threshold = None + + def _get_global_threshold(self): + weight_list = [] + for (layer, _) in self.pruner.get_modules_to_compress(): + weight_list.append(layer.module.weight.data.abs().clone()) + all_bn_weights = torch.cat(weight_list) + k = int(all_bn_weights.shape[0] * self.pruner.config_list[0]['sparsity']) + self.global_threshold = torch.topk( + all_bn_weights.view(-1), k, largest=False)[0].max() + print(f'set global threshold to {self.global_threshold}') + + def calc_mask(self, sparsity, wrapper, wrapper_idx=None): + assert wrapper.type == 'BatchNorm2d', 'SlimPruner only supports 2d batch normalization layer pruning' + + if self.global_threshold is None: + self._get_global_threshold() + + weight = wrapper.module.weight.data.clone() + if wrapper.weight_mask is not None: + # apply base mask for iterative pruning + weight = weight * wrapper.weight_mask + + base_mask = torch.ones(weight.size()).type_as(weight).detach() + mask = {'weight_mask': base_mask.detach( + ), 'bias_mask': base_mask.clone().detach()} + filters = weight.size(0) + num_prune = int(filters * sparsity) + if filters >= 2 and num_prune >= 1: + w_abs = weight.abs() + mask_weight = torch.gt( + w_abs, self.global_threshold).type_as(weight) + mask_bias = mask_weight.clone() + mask = {'weight_mask': mask_weight.detach( + ), 'bias_mask': mask_bias.detach()} + return mask + +def least_square_sklearn(X, Y): + from sklearn.linear_model import LinearRegression + reg = LinearRegression(fit_intercept=False) + reg.fit(X, Y) + return reg.coef_ + + +class AMCWeightMasker(WeightMasker): + """ + Weight maskser class for AMC pruner. Currently, AMCPruner only supports pruning kernel + size 1x1 pointwise Conv2d layer. Before using this class to prune kernels, AMCPruner + collected input and output feature maps for each layer, the features maps are flattened + and save into wrapper.input_feat and wrapper.output_feat. + + Parameters + ---------- + model: nn.Module + model to be pruned + pruner: Pruner + A Pruner instance used to prune the model + preserve_round: int + after pruning, preserve filters/channels round to `preserve_round`, for example: + for a Conv2d layer, output channel is 32, sparsity is 0.2, if preserve_round is + 1 (no preserve round), then there will be int(32 * 0.2) = 6 filters pruned, and + 32 - 6 = 26 filters are preserved. If preserve_round is 4, preserved filters will + be round up to 28 (which can be divided by 4) and only 4 filters are pruned. + """ + + def __init__(self, model, pruner, preserve_round=1): + self.model = model + self.pruner = pruner + self.preserve_round = preserve_round + + def calc_mask(self, sparsity, wrapper, wrapper_idx=None, preserve_idx=None): + """ + Calculate the mask of given layer. + Parameters + ---------- + sparsity: float + pruning ratio, preserved weight ratio is `1 - sparsity` + wrapper: PrunerModuleWrapper + layer wrapper of this layer + wrapper_idx: int + index of this wrapper in pruner's all wrappers + Returns + ------- + dict + dictionary for storing masks, keys of the dict: + 'weight_mask': weight mask tensor + 'bias_mask': bias mask tensor (optional) + """ + msg = 'module type {} is not supported!'.format(wrapper.type) + assert wrapper.type in ['Conv2d', 'Linear'], msg + weight = wrapper.module.weight.data + bias = None + if hasattr(wrapper.module, 'bias') and wrapper.module.bias is not None: + bias = wrapper.module.bias.data + + if wrapper.weight_mask is None: + mask_weight = torch.ones(weight.size()).type_as(weight).detach() + else: + mask_weight = wrapper.weight_mask.clone() + if bias is not None: + if wrapper.bias_mask is None: + mask_bias = torch.ones(bias.size()).type_as(bias).detach() + else: + mask_bias = wrapper.bias_mask.clone() + else: + mask_bias = None + mask = {'weight_mask': mask_weight, 'bias_mask': mask_bias} + + num_total = weight.size(1) + num_prune = int(num_total * sparsity) + if self.preserve_round > 1: + num_preserve = num_total - num_prune + num_preserve = int( + math.ceil(num_preserve * 1. / self.preserve_round) * self.preserve_round) + if num_preserve > num_total: + num_preserve = num_total + num_prune = num_total - num_preserve + + if (num_total < 2 or num_prune < 1) and preserve_idx is None: + return mask + + return self.get_mask(mask, weight, num_preserve, wrapper, wrapper_idx, preserve_idx) + + def get_mask(self, base_mask, weight, num_preserve, wrapper, wrapper_idx, preserve_idx): + w = weight.data.cpu().numpy() + if wrapper.type == 'Linear': + w = w[:, :, None, None] + + if preserve_idx is None: + importance = np.abs(w).sum((0, 2, 3)) + # sum magnitude along C_in, sort descend + sorted_idx = np.argsort(-importance) + d_prime = num_preserve + preserve_idx = sorted_idx[:d_prime] # to preserve index + else: + d_prime = len(preserve_idx) + + assert len(preserve_idx) == d_prime + mask = np.zeros(w.shape[1], bool) + mask[preserve_idx] = True + + # reconstruct, X, Y <= [N, C] + X, Y = wrapper.input_feat, wrapper.output_feat + masked_X = X[:, mask] + if w.shape[2] == 1: # 1x1 conv or fc + rec_weight = least_square_sklearn(X=masked_X, Y=Y) + rec_weight = rec_weight.reshape(-1, 1, 1, d_prime) # (C_out, K_h, K_w, C_in') + rec_weight = np.transpose(rec_weight, (0, 3, 1, 2)) # (C_out, C_in', K_h, K_w) + + rec_weight_pad = np.zeros_like(w) + # pylint: disable=all + rec_weight_pad[:, mask, :, :] = rec_weight + rec_weight = rec_weight_pad + + if wrapper.type == 'Linear': + rec_weight = rec_weight.squeeze() + assert len(rec_weight.shape) == 2 + + # now assign + wrapper.module.weight.data = torch.from_numpy(rec_weight).to(weight.device) + + mask_weight = torch.zeros_like(weight) + if wrapper.type == 'Linear': + mask_weight[:, preserve_idx] = 1. + if base_mask['bias_mask'] is not None and wrapper.module.bias is not None: + mask_bias = torch.ones_like(wrapper.module.bias) + else: + mask_weight[:, preserve_idx, :, :] = 1. + mask_bias = None + + return {'weight_mask': mask_weight.detach(), 'bias_mask': mask_bias} diff --git a/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/pruning/weight_masker.py b/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/pruning/weight_masker.py new file mode 100644 index 0000000000000000000000000000000000000000..aec8444ced37bccfa5f05e0a9f81b2843c2a35e4 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/pruning/weight_masker.py @@ -0,0 +1,28 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +class WeightMasker(object): + def __init__(self, model, pruner, **kwargs): + self.model = model + self.pruner = pruner + + def calc_mask(self, sparsity, wrapper, wrapper_idx=None): + """ + Calculate the mask of given layer. + Parameters + ---------- + sparsity: float + pruning ratio, preserved weight ratio is `1 - sparsity` + wrapper: PrunerModuleWrapper + layer wrapper of this layer + wrapper_idx: int + index of this wrapper in pruner's all wrappers + Returns + ------- + dict + dictionary for storing masks, keys of the dict: + 'weight_mask': weight mask tensor + 'bias_mask': bias mask tensor (optional) + """ + + raise NotImplementedError('{} calc_mask is not implemented'.format(self.__class__.__name__)) diff --git a/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/quantization/__init__.py b/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/quantization/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..0e728dcd3f0817a529eb6801f3ab935de6751348 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/quantization/__init__.py @@ -0,0 +1,4 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .quantizers import * diff --git a/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/quantization/quantizers.py b/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/quantization/quantizers.py new file mode 100644 index 0000000000000000000000000000000000000000..dbd5e5b3c34048a1a92b3623735ffd48fa5b45d8 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/compression/pytorch/quantization/quantizers.py @@ -0,0 +1,772 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +import copy +import torch +from schema import Schema, And, Or, Optional +from nni.compression.pytorch.utils.config_validation import CompressorSchema +from nni.compression.pytorch.compressor import Quantizer, QuantForward, QuantGrad, QuantType + +__all__ = ['NaiveQuantizer', 'QAT_Quantizer', 'DoReFaQuantizer', 'BNNQuantizer', 'LsqQuantizer'] + +logger = logging.getLogger(__name__) + + +class NaiveQuantizer(Quantizer): + """quantize weight to 8 bits + """ + + def __init__(self, model, config_list, optimizer=None): + super().__init__(model, config_list, optimizer) + self.layer_scale = {} + + def validate_config(self, model, config_list): + schema = CompressorSchema([{ + Optional('quant_types'): ['weight'], + Optional('quant_bits'): Or(8, {'weight': 8}), + Optional('op_types'): [str], + Optional('op_names'): [str] + }], model, logger) + + schema.validate(config_list) + + def quantize_weight(self, wrapper, **kwargs): + weight = copy.deepcopy(wrapper.module.old_weight.data) + new_scale = weight.abs().max() / 127 + scale = max(self.layer_scale.get(wrapper.name, 0), new_scale) + self.layer_scale[wrapper.name] = scale + orig_type = weight.type() # TODO: user layer + weight = weight.div(scale).type(torch.int8).type(orig_type).mul(scale) + wrapper.module.weight = weight + return weight + +def update_ema(biased_ema, value, decay): + """ + calculate biased stat and unbiased stat in each step using exponential moving average method + + Parameters + ---------- + biased_ema : float + previous stat value + value : float + current stat value + decay : float + the weight of previous stat value, larger means smoother curve + + Returns + ------- + float, float + """ + biased_ema = biased_ema * decay + (1 - decay) * value + return biased_ema + + +def update_quantization_param(bits, rmin, rmax): + """ + calculate the `zero_point` and `scale`. + + Parameters + ---------- + bits : int + quantization bits length + rmin : Tensor + min value of real value + rmax : Tensor + max value of real value + + Returns + ------- + float, float + """ + # extend the [min, max] interval to ensure that it contains 0. + # Otherwise, we would not meet the requirement that 0 be an exactly + # representable value. + rmin = torch.min(rmin, torch.Tensor([0]).to(rmin.device)) + rmax = torch.max(rmax, torch.Tensor([0]).to(rmin.device)) + qmin = torch.Tensor([0]).to(rmin.device) + qmax = torch.Tensor([(1 << bits) - 1]).to(rmin.device) + + # First determine the scale. + scale = (rmax - rmin) / (qmax - qmin) + + # Zero-point computation. + initial_zero_point = qmin - rmin / scale + + # Now we need to nudge the zero point to be an integer + if initial_zero_point < qmin: + nudged_zero_point = qmin + elif initial_zero_point > qmax: + nudged_zero_point = qmax + else: + nudged_zero_point = torch.round(initial_zero_point) + + return scale, nudged_zero_point + + +def get_bits_length(config, quant_type): + if isinstance(config["quant_bits"], int): + return config["quant_bits"] + else: + return config["quant_bits"].get(quant_type) + +class QATGrad(QuantGrad): + @staticmethod + def quant_backward(tensor, grad_output, quant_type, scale, zero_point, qmin, qmax): + tensor_q = QuantGrad._quantize(tensor, scale, zero_point) + mask = (tensor_q < qmin) | (tensor_q > qmax) + grad_output[mask] = 0 + return grad_output + + +class QAT_Quantizer(Quantizer): + """Quantizer defined in: + Quantization and Training of Neural Networks for Efficient Integer-Arithmetic-Only Inference + http://openaccess.thecvf.com/content_cvpr_2018/papers/Jacob_Quantization_and_Training_CVPR_2018_paper.pdf + """ + + def __init__(self, model, config_list, optimizer=None): + """ + Parameters + ---------- + layer : LayerInfo + the layer to quantize + config_list : list of dict + list of configurations for quantization + supported keys for dict: + - quant_types : list of string + type of quantization you want to apply, currently support 'weight', 'input', 'output' + - quant_bits : int or dict of {str : int} + bits length of quantization, key is the quantization type, value is the length, eg. {'weight', 8}, + when the type is int, all quantization types share same bits length + - quant_start_step : int + disable quantization until model are run by certain number of steps, this allows the network to enter a more stable + state where activation quantization ranges do not exclude a significant fraction of values, default value is 0 + - op_types : list of string + types of nn.module you want to apply quantization, eg. 'Conv2d' + """ + super().__init__(model, config_list, optimizer) + self.quant_grad = QATGrad.apply + modules_to_compress = self.get_modules_to_compress() + device = next(model.parameters()).device + self.bound_model.register_buffer("steps", torch.Tensor([1])) + for layer, config in modules_to_compress: + layer.module.register_buffer("zero_point", torch.Tensor([0.0])) + layer.module.register_buffer("scale", torch.Tensor([1.0])) + layer.module.register_buffer('ema_decay', torch.Tensor([0.99])) + if "weight" in config.get("quant_types", []): + layer.module.register_buffer('weight_bit', torch.zeros(1)) + layer.module.register_buffer('tracked_min_input', torch.zeros(1)) + layer.module.register_buffer('tracked_max_input', torch.zeros(1)) + if "output" in config.get("quant_types", []): + layer.module.register_buffer('activation_bit', torch.zeros(1)) + layer.module.register_buffer('tracked_min_activation', torch.zeros(1)) + layer.module.register_buffer('tracked_max_activation', torch.zeros(1)) + self.bound_model.to(device) + + def _del_simulated_attr(self, module): + """ + delete redundant parameters in quantize module + """ + del_attr_list = ['old_weight', 'ema_decay', 'tracked_min_activation', 'tracked_max_activation', 'tracked_min_input', \ + 'tracked_max_input', 'scale', 'zero_point', 'weight_bit', 'activation_bit'] + for attr in del_attr_list: + if hasattr(module, attr): + delattr(module, attr) + + def validate_config(self, model, config_list): + """ + Parameters + ---------- + model : torch.nn.Module + Model to be pruned + config_list : list of dict + List of configurations + """ + schema = CompressorSchema([{ + Optional('quant_types'): Schema([lambda x: x in ['weight', 'output']]), + Optional('quant_bits'): Or(And(int, lambda n: 0 < n < 32), Schema({ + Optional('weight'): And(int, lambda n: 0 < n < 32), + Optional('output'): And(int, lambda n: 0 < n < 32), + })), + Optional('quant_start_step'): And(int, lambda n: n >= 0), + Optional('op_types'): [str], + Optional('op_names'): [str] + }], model, logger) + + schema.validate(config_list) + + def _quantize(self, bits, op, real_val): + """ + quantize real value. + + Parameters + ---------- + bits : int + quantization bits length + op : torch.nn.Module + target module + real_val : Tensor + real value to be quantized + + Returns + ------- + Tensor + """ + op.zero_point = op.zero_point.to(real_val.device) + op.scale = op.scale.to(real_val.device) + transformed_val = op.zero_point + real_val / op.scale + qmin = 0 + qmax = (1 << bits) - 1 + clamped_val = torch.clamp(transformed_val, qmin, qmax) + quantized_val = torch.round(clamped_val) + return quantized_val + + def _dequantize(self, op, quantized_val): + """ + dequantize quantized value. + Because we simulate quantization in training process, all the computations still happen as float point computations, which means we + first quantize tensors then dequantize them. For more details, please refer to the paper. + + Parameters + ---------- + op : torch.nn.Module + target module + quantized_val : float + quantized_val value to be dequantized + + Returns + ------- + float + """ + real_val = op.scale * (quantized_val - op.zero_point) + return real_val + + def quantize_weight(self, wrapper, **kwargs): + config = wrapper.config + module = wrapper.module + input = kwargs['input_tensor'] + weight = copy.deepcopy(wrapper.module.old_weight.data) + weight_bits = get_bits_length(config, 'weight') + quant_start_step = config.get('quant_start_step', 0) + assert weight_bits >= 1, "quant bits length should be at least 1" + + # we dont update weight in evaluation stage + if quant_start_step > self.bound_model.steps: + module.tracked_min_input, module.tracked_max_input = torch.min(input), torch.max(input) + return weight + + if not wrapper.training: + return weight + + current_min, current_max = torch.min(input), torch.max(input) + module.tracked_min_input = update_ema(module.tracked_min_input, current_min, + module.ema_decay) + module.tracked_max_input = update_ema(module.tracked_max_input, current_max, + module.ema_decay) + + # if bias exists, quantize bias to uint32 + if hasattr(wrapper.module, 'bias') and wrapper.module.bias is not None: + bias = wrapper.module.bias.data + bias_bits = 32 + rmin, rmax = torch.min(bias), torch.max(bias) + module.scale, module.zero_point = update_quantization_param(bias_bits, rmin, rmax) + bias = self._quantize(bias_bits, module, bias) + bias = self._dequantize(module, bias) + wrapper.module.bias.data = bias + + + # quantize weight + rmin, rmax = torch.min(weight), torch.max(weight) + module.scale, module.zero_point = update_quantization_param(weight_bits, rmin, rmax) + weight = self._quantize(weight_bits, module, weight) + weight = self._dequantize(module, weight) + module.weight_bit = torch.Tensor([weight_bits]) + wrapper.module.weight = weight + return weight + + def quantize_output(self, output, wrapper, **kwargs): + config = wrapper.config + module = wrapper.module + output_bits = get_bits_length(config, 'output') + module.activation_bit = torch.Tensor([output_bits]) + quant_start_step = config.get('quant_start_step', 0) + assert output_bits >= 1, "quant bits length should be at least 1" + + if quant_start_step > self.bound_model.steps: + module.tracked_min_activation, module.tracked_max_activation = torch.min(output), torch.max(output) + return output + + # we dont update output quantization parameters in evaluation stage + if wrapper.training: + current_min, current_max = torch.min(output), torch.max(output) + module.tracked_min_activation = update_ema(module.tracked_min_activation, current_min, + module.ema_decay) + module.tracked_max_activation = update_ema(module.tracked_max_activation, current_max, + module.ema_decay) + module.scale, module.zero_point = update_quantization_param(output_bits, module.tracked_min_activation, module.tracked_max_activation) + out = self._quantize(output_bits, module, output) + out = self._dequantize(module, out) + return out + + def export_model(self, model_path, calibration_path=None, onnx_path=None, input_shape=None, device=None): + """ + Export quantized model weights and calibration parameters(optional) + + Parameters + ---------- + model_path : str + path to save quantized model weight + calibration_path : str + (optional) path to save quantize parameters after calibration + onnx_path : str + (optional) path to save onnx model + input_shape : list or tuple + input shape to onnx model + device : torch.device + device of the model, used to place the dummy input tensor for exporting onnx file. + the tensor is placed on cpu if ```device``` is None + + Returns + ------- + Dict + """ + assert model_path is not None, 'model_path must be specified' + self._unwrap_model() + calibration_config = {} + + for name, module in self.bound_model.named_modules(): + if hasattr(module, 'weight_bit') or hasattr(module, 'activation_bit'): + calibration_config[name] = {} + if hasattr(module, 'weight_bit'): + calibration_config[name]['weight_bit'] = int(module.weight_bit) + calibration_config[name]['tracked_min_input'] = float(module.tracked_min_input) + calibration_config[name]['tracked_max_input'] = float(module.tracked_max_input) + if hasattr(module, 'activation_bit'): + calibration_config[name]['activation_bit'] = int(module.activation_bit) + calibration_config[name]['tracked_min_activation'] = float(module.tracked_min_activation) + calibration_config[name]['tracked_max_activation'] = float(module.tracked_max_activation) + self._del_simulated_attr(module) + + self.export_model_save(self.bound_model, model_path, calibration_config, calibration_path, onnx_path, input_shape, device) + + return calibration_config + + def fold_bn(self, config, **kwargs): + # TODO simulate folded weight + pass + + def step_with_optimizer(self): + """ + override `compressor` `step` method, quantization only happens after certain number of steps + """ + self.bound_model.steps += 1 + + +class DoReFaQuantizer(Quantizer): + """Quantizer using the DoReFa scheme, as defined in: + Zhou et al., DoReFa-Net: Training Low Bitwidth Convolutional Neural Networks with Low Bitwidth Gradients + (https://arxiv.org/abs/1606.06160) + """ + + def __init__(self, model, config_list, optimizer=None): + super().__init__(model, config_list, optimizer) + device = next(model.parameters()).device + modules_to_compress = self.get_modules_to_compress() + for layer, config in modules_to_compress: + if "weight" in config.get("quant_types", []): + layer.module.register_buffer('weight_bit', torch.zeros(1)) + self.bound_model.to(device) + + def _del_simulated_attr(self, module): + """ + delete redundant parameters in quantize module + """ + del_attr_list = ['old_weight', 'weight_bit'] + for attr in del_attr_list: + if hasattr(module, attr): + delattr(module, attr) + + def validate_config(self, model, config_list): + """ + Parameters + ---------- + model : torch.nn.Module + Model to be pruned + config_list : list of dict + List of configurations + """ + schema = CompressorSchema([{ + Optional('quant_types'): Schema([lambda x: x in ['weight']]), + Optional('quant_bits'): Or(And(int, lambda n: 0 < n < 32), Schema({ + Optional('weight'): And(int, lambda n: 0 < n < 32) + })), + Optional('op_types'): [str], + Optional('op_names'): [str] + }], model, logger) + + schema.validate(config_list) + + def quantize_weight(self, wrapper, **kwargs): + weight = copy.deepcopy(wrapper.module.old_weight.data) + weight_bits = get_bits_length(wrapper.config, 'weight') + weight = weight.tanh() + weight = weight / (2 * weight.abs().max()) + 0.5 + weight = self.quantize(weight, weight_bits) + weight = 2 * weight - 1 + wrapper.module.weight = weight + wrapper.module.weight_bit = torch.Tensor([weight_bits]) + # wrapper.module.weight.data = weight + return weight + + def quantize(self, input_ri, q_bits): + scale = pow(2, q_bits) - 1 + output = torch.round(input_ri * scale) / scale + return output + + def export_model(self, model_path, calibration_path=None, onnx_path=None, input_shape=None, device=None): + """ + Export quantized model weights and calibration parameters(optional) + + Parameters + ---------- + model_path : str + path to save quantized model weight + calibration_path : str + (optional) path to save quantize parameters after calibration + onnx_path : str + (optional) path to save onnx model + input_shape : list or tuple + input shape to onnx model + device : torch.device + device of the model, used to place the dummy input tensor for exporting onnx file. + the tensor is placed on cpu if ```device``` is None + + Returns + ------- + Dict + """ + assert model_path is not None, 'model_path must be specified' + self._unwrap_model() + calibration_config = {} + + for name, module in self.bound_model.named_modules(): + if hasattr(module, 'weight_bit'): + calibration_config[name] = {} + calibration_config[name]['weight_bit'] = int(module.weight_bit) + self._del_simulated_attr(module) + + self.export_model_save(self.bound_model, model_path, calibration_config, calibration_path, onnx_path, input_shape, device) + + return calibration_config + + +class ClipGrad(QuantGrad): + @staticmethod + def quant_backward(tensor, grad_output, quant_type, scale, zero_point, qmin, qmax): + if quant_type == QuantType.QUANT_OUTPUT: + grad_output[torch.abs(tensor) > 1] = 0 + return grad_output + + +class BNNQuantizer(Quantizer): + """Binarized Neural Networks, as defined in: + Binarized Neural Networks: Training Deep Neural Networks with Weights and Activations Constrained to +1 or -1 + (https://arxiv.org/abs/1602.02830) + """ + + def __init__(self, model, config_list, optimizer=None): + super().__init__(model, config_list, optimizer) + device = next(model.parameters()).device + self.quant_grad = ClipGrad.apply + modules_to_compress = self.get_modules_to_compress() + for layer, config in modules_to_compress: + if "weight" in config.get("quant_types", []): + layer.module.register_buffer('weight_bit', torch.zeros(1)) + self.bound_model.to(device) + + def _del_simulated_attr(self, module): + """ + delete redundant parameters in quantize module + """ + del_attr_list = ['old_weight', 'weight_bit'] + for attr in del_attr_list: + if hasattr(module, attr): + delattr(module, attr) + + def validate_config(self, model, config_list): + """ + Parameters + ---------- + model : torch.nn.Module + Model to be pruned + config_list : list of dict + List of configurations + """ + schema = CompressorSchema([{ + Optional('quant_types'): Schema([lambda x: x in ['weight', 'output']]), + Optional('quant_bits'): Or(And(int, lambda n: 0 < n < 32), Schema({ + Optional('weight'): And(int, lambda n: 0 < n < 32), + Optional('output'): And(int, lambda n: 0 < n < 32), + })), + Optional('op_types'): [str], + Optional('op_names'): [str] + }], model, logger) + + schema.validate(config_list) + + def quantize_weight(self, wrapper, **kwargs): + weight = copy.deepcopy(wrapper.module.old_weight.data) + weight = torch.sign(weight) + # remove zeros + weight[weight == 0] = 1 + wrapper.module.weight = weight + wrapper.module.weight_bit = torch.Tensor([1.0]) + return weight + + def quantize_output(self, output, wrapper, **kwargs): + out = torch.sign(output) + # remove zeros + out[out == 0] = 1 + return out + + def export_model(self, model_path, calibration_path=None, onnx_path=None, input_shape=None, device=None): + """ + Export quantized model weights and calibration parameters(optional) + + Parameters + ---------- + model_path : str + path to save quantized model weight + calibration_path : str + (optional) path to save quantize parameters after calibration + onnx_path : str + (optional) path to save onnx model + input_shape : list or tuple + input shape to onnx model + device : torch.device + device of the model, used to place the dummy input tensor for exporting onnx file. + the tensor is placed on cpu if ```device``` is None + + Returns + ------- + Dict + """ + assert model_path is not None, 'model_path must be specified' + self._unwrap_model() + calibration_config = {} + + for name, module in self.bound_model.named_modules(): + if hasattr(module, 'weight_bit'): + calibration_config[name] = {} + calibration_config[name]['weight_bit'] = int(module.weight_bit) + self._del_simulated_attr(module) + + self.export_model_save(self.bound_model, model_path, calibration_config, calibration_path, onnx_path, input_shape, device) + + return calibration_config + + +class LsqQuantizer(Quantizer): + """Quantizer defined in: + Learned Step Size Quantization (ICLR 2020) + https://arxiv.org/pdf/1902.08153.pdf + """ + + def __init__(self, model, config_list, optimizer=None): + """ + Parameters + ---------- + model : torch.nn.Module + the model to be quantized + config_list : list of dict + list of configurations for quantization + supported keys for dict: + - quant_types : list of string + type of quantization you want to apply, currently support 'weight', 'input', 'output' + - quant_bits : int or dict of {str : int} + bits length of quantization, key is the quantization type, value is the length, eg. {'weight': 8}, + when the type is int, all quantization types share same bits length + - quant_start_step : int + disable quantization until model are run by certain number of steps, this allows the network to enter a more stable + state where activation quantization ranges do not exclude a significant fraction of values, default value is 0 + - op_types : list of string + types of nn.module you want to apply quantization, eg. 'Conv2d' + """ + super().__init__(model, config_list, optimizer) + device = next(model.parameters()).device + self.quant_grad = QuantForward() + modules_to_compress = self.get_modules_to_compress() + self.bound_model.register_buffer("steps", torch.Tensor([1])) + for layer, config in modules_to_compress: + if "weight" in config.get("quant_types", []): + layer.module.register_parameter("weight_scale", torch.nn.Parameter(torch.Tensor([1.0]))) + # todo: support per-channel quantization for weight since TensorRT use it for conv weight + q_bit = get_bits_length(config, "weight") + layer.module.register_buffer('weight_bit', torch.Tensor([q_bit])) + qmax = 2 ** (q_bit - 1) - 1 + qmin = -2 ** (q_bit - 1) + init_weight_scale = layer.module.weight.data.detach().abs().mean() * 2 / (qmax ** 0.5) + layer.module.weight_scale = torch.nn.Parameter(init_weight_scale) + layer.module.weight_qmax = qmax + layer.module.weight_qmin = qmin + + self.optimizer.add_param_group({"params": layer.module.weight_scale}) + + if "output" in config.get("quant_types", []): + # scale of activation will be initialized using the first batch data + layer.module.register_parameter("output_scale", torch.nn.Parameter(torch.Tensor([1.0]))) + q_bit = get_bits_length(config, "output") + layer.module.register_buffer('output_bit', torch.Tensor([q_bit])) + qmax = 2 ** (q_bit - 1) - 1 + qmin = -2 ** (q_bit - 1) + layer.module.output_qmax = qmax + layer.module.output_qmin = qmin + + self.optimizer.add_param_group({"params": layer.module.output_scale}) + + if "input" in config.get("quant_types", []): + # scale of input will be initialized using the first batch data + layer.module.register_parameter("input_scale", torch.nn.Parameter(torch.Tensor([1.0]))) + q_bit = get_bits_length(config, "input") + layer.module.register_buffer('input_bit', torch.Tensor([q_bit])) + qmax = 2 ** (q_bit - 1) - 1 + qmin = -2 ** (q_bit - 1) + layer.module.input_qmax = qmax + layer.module.input_qmin = qmin + + self.optimizer.add_param_group({"params": layer.module.input_scale}) + + self.bound_model.to(device) + + @staticmethod + def grad_scale(x, scale): + """ + Used to scale the gradient. Give tensor `x`, we have `y=grad_scale(x, scale)=x` in the forward pass, + which means that this function will not change the value of `x`. In the backward pass, we have: + + :math:`\frac{\alpha_L}{\alpha_x}=\frac{\alpha_L}{\alpha_y}*\frac{\alpha_y}{\alpha_x}=sclae*\frac{\alpha_L}{\alpha_x}` + + This means that the origin gradient of x is scaled by a factor of `scale`. Applying this function + to a nn.Parameter will scale the gradient of it without changing its value. + """ + y = x + y_grad = x * scale + return (y - y_grad).detach() + y_grad + + @staticmethod + def round_pass(x): + """ + A simple way to achieve STE operation. + """ + y = x.round() + y_grad = x + return (y - y_grad).detach() + y_grad + + def quantize(self, x, scale, qmin, qmax): + grad_scale_factor = 1.0 / ((qmax * x.numel()) ** 0.5) + scale = self.grad_scale(scale, grad_scale_factor) + x = x / scale + x = torch.clamp(x, qmin, qmax) + x = self.round_pass(x) + x = x * scale + return x + + def quantize_weight(self, wrapper, **kwargs): + module = wrapper.module + + # todo: add support for quantize bias. If we use TensorRT as backend, there is no need to quantize + # bias + old_weight = module.old_weight + weight = self.quantize(old_weight, module.weight_scale, module.weight_qmin, module.weight_qmax) + module.weight = weight + return weight + + def quantize_output(self, output, wrapper, **kwargs): + module = wrapper.module + + # initialize the scale + if self.bound_model.steps == 1: + qmax = module.output_qmax + init_oup_scale = output.data.detach().abs().mean() * 2 / (qmax ** 0.5) + module.output_scale.data = init_oup_scale + + output = self.quantize(output, module.output_scale, module.output_qmin, module.output_qmax) + return output + + def quantize_input(self, *inputs, wrapper, **kwargs): + # This is hacky since it is not recommended to modify a tuple + # NB: support layers with multi inputs + module = wrapper.module + # initialize the scale + if self.bound_model.steps == 1: + qmax = module.input_qmax + init_oup_scale = inputs[0].data.detach().abs().mean() * 2 / (qmax ** 0.5) + module.input_scale.data = init_oup_scale + + new_input = self.quantize(inputs[0], module.input_scale, module.input_qmin, module.input_qmax) + list_inp = list(inputs) + list_inp[0] = new_input + return tuple(list_inp) + + def export_model(self, model_path, calibration_path=None, onnx_path=None, input_shape=None, device=None): + """ + Export quantized model weights and calibration parameters(optional) + + Parameters + ---------- + model_path : str + path to save quantized model weight + calibration_path : str + (optional) path to save quantize parameters after calibration + onnx_path : str + (optional) path to save onnx model + input_shape : list or tuple + input shape to onnx model + device : torch.device + device of the model, used to place the dummy input tensor for exporting onnx file. + the tensor is placed on cpu if ```device``` is None + + Returns + ------- + Dict + """ + assert model_path is not None, 'model_path must be specified' + self._unwrap_model() + calibration_config = {} + + for name, module in self.bound_model.named_modules(): + if hasattr(module, 'input_bit') or hasattr(module, 'output_bit'): + calibration_config[name] = {} + if hasattr(module, 'weight_bit'): + calibration_config[name]['weight_bit'] = int(module.weight_bit) + abs_max_input = float(module.input_scale * module.input_qmax) + calibration_config[name]['tracked_min_input'] = -abs_max_input + calibration_config[name]['tracked_max_input'] = abs_max_input + if hasattr(module, 'output_bit'): + calibration_config[name]['activation_bit'] = int(module.output_bit) + abs_max_output = float(module.output_scale * module.output_qmax) + calibration_config[name]['tracked_min_activation'] = -abs_max_output + calibration_config[name]['tracked_max_activation'] = abs_max_output + self._del_simulated_attr(module) + + self.export_model_save(self.bound_model, model_path, calibration_config, calibration_path, onnx_path, + input_shape, device) + + return calibration_config + + def _del_simulated_attr(self, module): + """ + delete redundant parameters in quantize module + """ + del_attr_list = ['old_weight', 'tracked_min_input', 'tracked_max_input', 'tracked_min_activation', \ + 'tracked_max_activation', 'output_scale', 'input_scale', 'weight_scale','weight_bit', 'output_bit', 'input_bit'] + for attr in del_attr_list: + if hasattr(module, attr): + delattr(module, attr) + + def step_with_optimizer(self): + """ + override `compressor` `step` method, quantization only happens after certain number of steps + """ + self.bound_model.steps += 1 diff --git a/new_impl/cv/third_party/nni_new/algorithms/compression/tensorflow/pruning/__init__.py b/new_impl/cv/third_party/nni_new/algorithms/compression/tensorflow/pruning/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..c535fd75123f8d37fba1084a35cb8615db426175 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/compression/tensorflow/pruning/__init__.py @@ -0,0 +1 @@ +from .one_shot_pruner import * diff --git a/new_impl/cv/third_party/nni_new/algorithms/compression/tensorflow/pruning/one_shot_pruner.py b/new_impl/cv/third_party/nni_new/algorithms/compression/tensorflow/pruning/one_shot_pruner.py new file mode 100644 index 0000000000000000000000000000000000000000..2c1e1e3e0fa15884aa449246bc1a266c977513c6 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/compression/tensorflow/pruning/one_shot_pruner.py @@ -0,0 +1,110 @@ +import tensorflow as tf + +from nni.compression.tensorflow import Pruner + +__all__ = [ + 'LevelPruner', + 'SlimPruner', +] + +class OneshotPruner(Pruner): + def __init__(self, model, config_list, masker_class, **algo_kwargs): + super().__init__(model, config_list) + self.set_wrappers_attribute('calculated', False) + self.masker = masker_class(model, self, **algo_kwargs) + + def validate_config(self, model, config_list): + pass # TODO + + def calc_masks(self, wrapper, wrapper_idx=None): + if wrapper.calculated: + return None + sparsity = wrapper.config['sparsity'] + masks = self.masker.calc_masks(sparsity, wrapper, wrapper_idx) + if masks is not None: + wrapper.calculated = True + return masks + + +class LevelPruner(OneshotPruner): + def __init__(self, model, config_list): + super().__init__(model, config_list, LevelPrunerMasker) + + +class SlimPruner(OneshotPruner): + def __init__(self, model, config_list): + super().__init__(model, config_list, SlimPrunerMasker) + + +class WeightMasker: + def __init__(self, model, pruner, **kwargs): + self.model = model + self.pruner = pruner + + def calc_masks(self, sparsity, wrapper, wrapper_idx=None): + raise NotImplementedError() + + +class LevelPrunerMasker(WeightMasker): + def calc_masks(self, sparsity, wrapper, wrapper_idx=None): + masks = {} + for weight_variable in wrapper.layer.weights: + if 'bias' in weight_variable.name: + continue + + num_prune = int(tf.size(weight_variable).numpy() * sparsity) + if num_prune == 0: + continue + + weight = weight_variable.read_value() + if wrapper.masks.get(weight_variable.name) is not None: + weight = tf.math.multiply(weight, wrapper.masks[weight_variable.name]) + + w_abs = tf.math.abs(weight) + k = tf.size(weight) - num_prune + topk = tf.math.top_k(tf.reshape(w_abs, [-1]), k).values + if tf.size(topk) == 0: + mask = tf.zeros_like(weight) + else: + mask = tf.math.greater_equal(w_abs, topk[-1]) + masks[weight_variable.name] = tf.cast(mask, weight.dtype) + return masks + +class SlimPrunerMasker(WeightMasker): + def __init__(self, model, pruner, **kwargs): + super().__init__(model, pruner) + weight_list = [] + for wrapper in pruner.wrappers: + weights = [w for w in wrapper.layer.weights if '/gamma:' in w.name] + assert len(weights) == 1, f'Bad weights: {[w.name for w in wrapper.layer.weights]}' + weight_list.append(tf.math.abs(weights[0].read_value())) + all_bn_weights = tf.concat(weight_list, 0) + k = int(all_bn_weights.shape[0] * pruner.wrappers[0].config['sparsity']) + top_k = -tf.math.top_k(-tf.reshape(all_bn_weights, [-1]), k).values + self.global_threshold = top_k.numpy()[-1] + + def calc_masks(self, sparsity, wrapper, wrapper_idx=None): + assert isinstance(wrapper.layer, tf.keras.layers.BatchNormalization), \ + 'SlimPruner only supports 2D batch normalization layer pruning' + + weight = None + weight_name = None + bias_name = None + + for variable in wrapper.layer.weights: + if '/gamma:' in variable.name: + weight = variable.read_value() + weight_name = variable.name + elif '/beta:' in variable.name: + bias_name = variable.name + + assert weight is not None + if wrapper.masks.get(weight_name) is not None: + weight *= wrapper.masks[weight_name] + + mask = tf.cast(tf.math.abs(weight) > self.global_threshold, weight.dtype) + + masks = {weight_name: mask} + if bias_name: + masks[bias_name] = mask + return masks diff --git a/new_impl/cv/third_party/nni_new/algorithms/feature_engineering/gbdt_selector/__init__.py b/new_impl/cv/third_party/nni_new/algorithms/feature_engineering/gbdt_selector/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..72970ab8565e11e62bae1242a4a8a00c06993cf7 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/feature_engineering/gbdt_selector/__init__.py @@ -0,0 +1 @@ +from .gbdt_selector import GBDTSelector \ No newline at end of file diff --git a/new_impl/cv/third_party/nni_new/algorithms/feature_engineering/gbdt_selector/gbdt_selector.py b/new_impl/cv/third_party/nni_new/algorithms/feature_engineering/gbdt_selector/gbdt_selector.py new file mode 100644 index 0000000000000000000000000000000000000000..9ee09c25a38f1dca27fb4ab068bbceeaec69a3bd --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/feature_engineering/gbdt_selector/gbdt_selector.py @@ -0,0 +1,115 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# MIT License +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and +# associated documentation files (the "Software"), to deal in the Software without restriction, +# including without limitation the rights to use, copy, modify, merge, publish, distribute, +# sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or +# substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT +# NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT +# OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# ================================================================================================== + +""" +gbdt_selector.py including: + class GBDTSelector +""" + +import random +from sklearn.model_selection import train_test_split + +# pylint: disable=E0401 +import lightgbm as lgb + +from nni.feature_engineering.feature_selector import FeatureSelector + + +class GBDTSelector(FeatureSelector): + + def __init__(self, **kwargs): + self.selected_features_ = None + self.X = None + self.y = None + self.feature_importance = None + self.lgb_params = None + self.eval_ratio = None + self.early_stopping_rounds = None + self.importance_type = None + self.num_boost_round = None + self.model = None + + + def fit(self, X, y, **kwargs): + """ + Fit the training data to FeatureSelector + + Paramters + --------- + X : array-like numpy matrix + The training input samples, which shape is [n_samples, n_features]. + y : array-like numpy matrix + The target values (class labels in classification, real numbers in + regression). Which shape is [n_samples]. + lgb_params : dict + Parameters of lightgbm + eval_ratio : float + The ratio of data size. It's used for split the eval data and train data from self.X. + early_stopping_rounds : int + The early stopping setting in lightgbm. + importance_type : str + Supporting type is 'gain' or 'split'. + num_boost_round : int + num_boost_round in lightgbm. + """ + assert kwargs['lgb_params'] + assert kwargs['eval_ratio'] + assert kwargs['early_stopping_rounds'] + assert kwargs['importance_type'] + assert kwargs['num_boost_round'] + + self.X = X + self.y = y + self.lgb_params = kwargs['lgb_params'] + self.eval_ratio = kwargs['eval_ratio'] + self.early_stopping_rounds = kwargs['early_stopping_rounds'] + self.importance_type = kwargs['importance_type'] + self.num_boost_round = kwargs['num_boost_round'] + + X_train, X_test, y_train, y_test = train_test_split(self.X, + self.y, + test_size=self.eval_ratio, + random_state=random.seed(41)) + lgb_train = lgb.Dataset(X_train, y_train) + lgb_eval = lgb.Dataset(X_test, y_test, reference=lgb_train) + + self.model = lgb.train(self.lgb_params, + lgb_train, + num_boost_round=self.num_boost_round, + valid_sets=lgb_eval, + early_stopping_rounds=self.early_stopping_rounds) + + self.feature_importance = self.model.feature_importance(self.importance_type) + + + def get_selected_features(self, topk): + """ + Fit the training data to FeatureSelector + + Returns + ------- + list : + Return the index of imprtant feature. + """ + assert topk > 0 + + self.selected_features_ = self.feature_importance.argsort()[-topk:][::-1] + + return self.selected_features_ diff --git a/new_impl/cv/third_party/nni_new/algorithms/feature_engineering/gbdt_selector/requirements.txt b/new_impl/cv/third_party/nni_new/algorithms/feature_engineering/gbdt_selector/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..b73c05e89d156249906203b67c2e9f786d613738 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/feature_engineering/gbdt_selector/requirements.txt @@ -0,0 +1 @@ +lightgbm \ No newline at end of file diff --git a/new_impl/cv/third_party/nni_new/algorithms/feature_engineering/gradient_selector/__init__.py b/new_impl/cv/third_party/nni_new/algorithms/feature_engineering/gradient_selector/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a43cb7578dcb38cbccc33fa11ab6bafc0a71b1fd --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/feature_engineering/gradient_selector/__init__.py @@ -0,0 +1 @@ +from .gradient_selector import FeatureGradientSelector \ No newline at end of file diff --git a/new_impl/cv/third_party/nni_new/algorithms/feature_engineering/gradient_selector/constants.py b/new_impl/cv/third_party/nni_new/algorithms/feature_engineering/gradient_selector/constants.py new file mode 100644 index 0000000000000000000000000000000000000000..0f70e2043af51bcf35c7514a81889203a9017ccb --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/feature_engineering/gradient_selector/constants.py @@ -0,0 +1,100 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# MIT License +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and +# associated documentation files (the "Software"), to deal in the Software without restriction, +# including without limitation the rights to use, copy, modify, merge, publish, distribute, +# sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or +# substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT +# NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT +# OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# ================================================================================================== + + +import numpy as np + + +class StorageLevel: + DISK = 'disk' + SPARSE = 'sparse' + DENSE = 'dense' + + +class DataFormat: + SVM = 'svm' + NUMPY = 'numpy' + ALL_FORMATS = [SVM, NUMPY] + + +class Preprocess: + """ + center the data to mean 0 and create unit variance + center the data to mean 0 + """ + ZSCORE = 'zscore' + CENTER = 'center' + + +class Device: + CUDA = 'cuda' + CPU = 'cpu' + + +class Checkpoint: + MODEL = 'model_state_dict' + OPT = 'optimizer_state_dict' + RNG = 'torch_rng_state' + + +class NanError(ValueError): + pass + + +class Initialization: + ZERO = 'zero' + ON = 'on' + OFF = 'off' + ON_HIGH = 'onhigh' + OFF_HIGH = 'offhigh' + SKLEARN = 'sklearn' + RANDOM = 'random' + VALUE_DICT = {ZERO: 0, + ON: 1, + OFF: -1, + ON_HIGH: 5, + OFF_HIGH: -1, + SKLEARN: None, + RANDOM: None} + + +class Coefficients: + """" + coefficients for sublinear estimator were computed running the sublinear + paper's authors' code + """ + SLE = {1: np.array([0.60355337]), + 2: np.array([1.52705001, -0.34841729]), + 3: np.array([2.90254224, -1.87216745, 0.]), + 4: np.array([4.63445685, -5.19936195, 0., 1.50391676]), + 5: np.array([6.92948049, -14.12216211, 9.4475009, 0., -1.21093546]), + 6: np.array([9.54431082, -28.09414643, 31.84703652, -11.18763791, -1.14175281, 0.]), + 7: np.array([12.54505041, -49.64891525, 79.78828031, -46.72250909, 0., 0., 5.02973646]), + 8: np.array([16.03550163, -84.286182, 196.86078756, -215.36747071, 92.63961263, 0., 0., -4.86280869]), + 9: np.array([19.86409184, -130.76801006, 390.95349861, -570.09210416, 354.77764899, 0., -73.84234865, 0., 10.09148767]), + 10: np.array([2.41117752e+01, -1.94946061e+02, 7.34214614e+02, -1.42851995e+03, 1.41567410e+03, \ + -5.81738134e+02, 0., 0., 3.11664751e+01, 1.05018365e+00]), + 11: np.array([28.75280839, -279.22576729, 1280.46325445, -3104.47148101, 3990.6092248, -2300.29413333, \ + 0., 427.35289033, 0., 0., -42.17587475]), + 12: np.array([33.85141912, -391.4229382, 2184.97827882, -6716.28280208, 11879.75233977, -11739.97267239, \ + 5384.94542245, 0., -674.23291712, 0., 0., 39.37456439])} + + +EPSILON = 1e-8 diff --git a/new_impl/cv/third_party/nni_new/algorithms/feature_engineering/gradient_selector/fginitialize.py b/new_impl/cv/third_party/nni_new/algorithms/feature_engineering/gradient_selector/fginitialize.py new file mode 100644 index 0000000000000000000000000000000000000000..6fe28ea5ee5494bc63a1dede2867ed3d2ec9f9a6 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/feature_engineering/gradient_selector/fginitialize.py @@ -0,0 +1,611 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# MIT License +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and +# associated documentation files (the "Software"), to deal in the Software without restriction, +# including without limitation the rights to use, copy, modify, merge, publish, distribute, +# sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or +# substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT +# NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT +# OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# ================================================================================================== + + +import os +import pickle +import sys +import time + +import numpy as np +import scipy.sparse +from sklearn.datasets import load_svmlight_file + +import torch +from torch.utils.data import DataLoader, Dataset +# pylint: disable=E0611 +from torch.utils.data.dataloader import _SingleProcessDataLoaderIter, _MultiProcessingDataLoaderIter, _utils + +from . import constants +from . import syssettings + +torch.set_default_tensor_type(syssettings.torch.tensortype) +sparsetensor = syssettings.torch.sparse.tensortype + +BYTESPERREAL = 8. +BYTESPERGB = 1024. ** 3 + + +class PrepareData(Dataset): + + def __init__(self, + path_data=None, + data_format=constants.DataFormat.NUMPY, + D=None, N=None, + classification=True, + ordinal=False, + balanced=True, + preprocess=None, + n_to_estimate=None, + MAXMEMGB=syssettings.MAXMEMGB, + set_params=True, + path_mappings=None, + X=None, + y=None, + verbose=0, + n_classes=None, + device=constants.Device.CPU): + """ + Dataset class with helpful features and functions for being included in a dataloader + and managing memory usage. + can read following formats: + svm: svm light format (sklearn.datasets.load_svmlight_file) + numpy: Pass X and y as numpy or sparse arrays + + assumes + 1. if classification, y is in {-1, 1} or continuous and 0 indexed + 2. y can fit into memory + 3. consecutive calls to __getitem__() have consecutive idx values + + notes: + 1. this implementation is not careful wrt/ precise memory reqts. for + example, being able to store one dense row in memory is necessary, + but not sufficient. + 2. for y with 4.2 billion elements, 31.3 GB of memory is necessary + @ 8 bytes/scalar. Use partial fit to avoid loading the entire dataset + at once + 3. disk_size always refer to size of complete data file, even after + a split(). + + + Parameters + ---------- + path_data : str + Path to load data from + data_format : str + File ending for path data. + "numpy" is the default when passing in X and y + D : int + Number of features. + N : int + Number of rows. + classification : bool + If True, problem is classification, else regression. + ordinal: bool + If True, problem is ordinal classification. Requires classification to be True. + balanced : bool + If true, each class is weighted equally in optimization, otherwise + weighted is done via support of each class. Requires classification to be True. + prerocess : str + 'zscore' which refers to centering and normalizing data to unit variance or + 'center' which only centers the data to 0 mean + n_to_estimate : int + Number of rows of data to estimate + MAXMEMGB : float + Maximum allowable size for a minibatch + set_params : bool + Whether or not to determine the statistics of the dataset + path_mappings : str + Used when streaming from disk + X : array-like + Shape = [n_samples, n_features] + The training input samples. + y : array-like + Shape = [n_samples] + The target values (class labels in classification, real numbers in + regression). + verbose : int + Controls the verbosity when fitting. Set to 0 for no printing + 1 or higher for printing every verbose number of gradient steps. + device : str + 'cpu' to run on CPU and 'cuda' to run on GPU. Runs much faster on GPU + n_classes : int + number of classes + """ + + self.path_data = path_data + if self.path_data: + self.disk_size = os.path.getsize(path_data) + else: + assert X is not None, 'X must be specified if no path data' + self.disk_size = X.nbytes if not scipy.sparse.issparse( + X) else X.data.nbytes + assert data_format in constants.DataFormat.ALL_FORMATS, 'Format must in {0}.'.format( + ", ".join(constants.DataFormat.ALL_FORMATS)) + self.format = data_format + self.classification = classification + self.ordinal = ordinal + self.balanced = balanced + self.MAXMEMGB = MAXMEMGB + self.preprocess = preprocess + self.set_params = set_params + self.verbose = verbose + self.n_classes = n_classes + self.device = device + + self.path_data_stats = None + + if D is None: + assert self.disk_size / BYTESPERGB <= self.MAXMEMGB, \ + 'Cannot load data into memory. Supply D.' + + if self.format == constants.DataFormat.SVM: + self.X, self.y = load_svmlight_file(path_data) + elif self.format == constants.DataFormat.NUMPY: + assert X is not None, 'X must be specified in numpy mode' + assert y is not None, 'y must be specified in numpy mode' + self.X = X + self.y = y + if self.n_classes is None: + self.n_classes = np.unique(y).shape[0] + elif self.classification: + assert self.n_classes >= np.unique(y).shape[0], \ + 'n_classes given must be greater than or equal to the number of classes in y' + else: + raise NotImplementedError + self.y = torch.as_tensor(self.y, dtype=torch.get_default_dtype()) + + self.N, self.D = self.X.shape + + # assumes X was returned as a sparse array + self.storage_level = (constants.StorageLevel.SPARSE + if scipy.sparse.issparse(self.X) + else constants.StorageLevel.DENSE) + + else: + assert N is not None, 'Supply N.' + self.N, self.D = N, D + + # assume sparse matrix cannot fit into memory + self.storage_level = constants.StorageLevel.DISK + + self.dense_size_gb = self.get_dense_size() + + # check dense size + self.set_dense_X() + + self.max_rows = int(self.MAXMEMGB * BYTESPERGB / BYTESPERREAL / self.D) + assert self.max_rows, \ + 'Cannot fit one dense row into %d GB memory.' % self.MAXMEMGB + self.max_rows = self.max_batch_size() + sys.stdout.flush() + + if n_to_estimate is None: + self.n_to_estimate = self.max_batch_size() + else: + assert n_to_estimate <= self.N, 'n_to_estimate must be <= N.' + self.n_to_estimate = n_to_estimate + + # initialize disk loader + if self.storage_level == constants.StorageLevel.DISK and self.set_params: + if self.format == constants.DataFormat.SVM: + raise NotImplementedError( + 'Please use partial fit to train on datasets that do not fit in memory') + else: + raise NotImplementedError + + # TODO: use a passed-in RNG here + self.ix_statistics = np.random.permutation(self.N)[:self.n_to_estimate] + self.n_features = self.D + if self.set_params: + if self.verbose: + print('Finding data statistics...', end='') + sys.stdout.flush() + Xmn, sv1, Xsd, ymn, ysd = self.compute_data_stats() + self.set_data_stats(Xmn, sv1, Xsd, ymn, ysd) + if self.verbose: + print() + self.set_return_raw(False) + else: + self.set_return_raw(True) + + self.set_return_np(False) + + # this needs to occur after setting preprocessing params + if (self.storage_level == constants.StorageLevel.DISK and + self.format == constants.DataFormat.SVM and self.set_params): + self.loader.batchsize = 1 + + def get_dense_size(self): + return self.N * self.D * BYTESPERREAL / BYTESPERGB + + def set_dense_X(self): + if self.storage_level != constants.StorageLevel.DISK: + if self.dense_size_gb <= self.MAXMEMGB: + if self.storage_level == constants.StorageLevel.SPARSE: + self.X = self.X.toarray() + self.X = torch.as_tensor( + self.X, dtype=torch.get_default_dtype()) + self.storage_level = constants.StorageLevel.DENSE + + def set_return_np(self, boolean): + + self.return_np = boolean + + def set_return_raw(self, boolean): + + self.return_raw = boolean + + def save_data_stats(self, path_data_stats): + """ + Dumps dataset statistics to pickle file. + """ + + data_stats = { + 'Xmn': self.Xmn, + 'sv1': self.sv1, + 'Xsd': self.Xsd, + 'ymn': self.ymn, + 'ysd': self.ysd, + 'ix_statistics': self.ix_statistics, + } + pickle.dump(data_stats, open(path_data_stats, 'wb')) + + def load_data_stats(self, path_data_stats): + + stats = pickle.load(open(path_data_stats, 'rb')) + self.path_data_stats = path_data_stats + + self.set_data_stats(np.asarray(stats['Xmn']), stats['sv1'], + stats['Xsd'], stats['ymn'], stats['ysd']) + + if self.storage_level == constants.StorageLevel.DISK and hasattr( + self, 'path_mappings'): + if 'ix_statistics' in stats: + self.ix_statistics = stats['ix_statistics'] + else: + self.ix_statistics = range(self.N) + + self.set_return_raw(False) + + def reset(self): + """ + Resets the dataloader. Only implemented for disk StorageLevel. + """ + + if self.storage_level == constants.StorageLevel.DENSE: + pass + elif self.storage_level == constants.StorageLevel.SPARSE: + pass + elif self.storage_level == constants.StorageLevel.DISK: + if self.format == constants.DataFormat.SVM: + self.loader.reset() + else: + raise NotImplementedError + + def todense(self): + + assert hasattr(self, 'Xmn'), 'Set preprocess params first.' + assert len(self) <= self.max_batch_size( + ), 'N must be <= max_batch_size().' + + with torch.no_grad(): + dense, _ = self.split(range(len(self))) + Braw = self.return_raw + Bnp = self.return_np + self.set_return_raw(True) + self.set_return_np(True) + dense.X, dense.y = [], [] + + def f_Xy(X, y): + dense.X.append(X) + dense.y.append(y) + self.apply(f_Xy=f_Xy) + dense.X = dense.X[-1] + dense.y = dense.y[-1] + self.set_return_raw(Braw) + self.set_return_np(Bnp) + dense.storage_level = constants.StorageLevel.DENSE + + return dense + + def split(self, ix): + + assert hasattr(self, 'Xmn'), 'Run set_preprocess_params() first.' + + first = type(self)( + self.path_data, + self.format, + self.D, + N=len(ix), + classification=self.classification, + preprocess=self.preprocess, + n_to_estimate=None, + MAXMEMGB=self.MAXMEMGB, + set_params=False) + second = type(self)( + self.path_data, + self.format, + self.D, + N=self.N - len(ix), + classification=self.classification, + preprocess=self.preprocess, + n_to_estimate=None, + MAXMEMGB=self.MAXMEMGB, + set_params=False) + + first.storage_level = self.storage_level + second.storage_level = self.storage_level + + # copy preprocess params + if not self.classification: + first.ymn = self.ymn + second.ymn = self.ymn + first.ysd = self.ysd + second.ysd = self.ysd + + first.Xmn = self.Xmn + second.Xmn = self.Xmn + first.sv1 = self.sv1 + second.sv1 = self.sv1 + + if self.storage_level == constants.StorageLevel.DISK: + if self.format == constants.DataFormat.SVM: + first.Xsd = self.Xsd + second.Xsd = self.Xsd + else: + raise NotImplementedError + + # initialize data structures + if self.storage_level == constants.StorageLevel.DISK: + if self.format == constants.DataFormat.SVM: + raise NotImplementedError + raise NotImplementedError + elif self.storage_level in [constants.StorageLevel.SPARSE, + constants.StorageLevel.DENSE]: + first.X, first.y = self.X[ix], self.y[ix] + ixsec = list(set(range(self.N)).difference(set(ix))) + second.X, second.y = self.X[ixsec], self.y[ixsec] + + return first, second + + @staticmethod + def sparse_std(X, X_mean): + """ + Calculate the column wise standard deviations of a sparse matrix. + """ + X_copy = X.copy() + X_copy.data **= 2 # square non zero elements + E_x_squared = np.array(X_copy.mean(axis=0)).ravel() + Xsd = np.sqrt(E_x_squared - X_mean**2) + return Xsd + + def compute_data_stats(self): + """ + 1. computes/estimates feature means + 2. if preprocess == 'zscore', computes/estimates feature standard devs + 3. if not classification, computes/estimates target mean/standard dev + 4. estimates largest singular value of data matrix + """ + t = time.time() + X, y = self.X[self.ix_statistics], self.y[self.ix_statistics] + preprocess = self.preprocess + classification = self.classification + + Xmn = (X.mean(dim=0) + if not scipy.sparse.issparse(X) + else np.array(X.mean(axis=0)).ravel()) + + if preprocess == constants.Preprocess.ZSCORE: + Xsd = (X.std(dim=0) + if not scipy.sparse.issparse(X) + else PrepareData.sparse_std(X, Xmn)) + Xsd[Xsd == 0] = 1. + else: + Xsd = 1. + + if preprocess is not None and preprocess: + if preprocess == constants.Preprocess.ZSCORE: + Xc = (X - Xmn) / Xsd + else: + Xc = X - Xmn + else: + Xc = X - Xmn + + sv1 = scipy.sparse.linalg.svds(Xc / ( + torch.sqrt(torch.prod(torch.as_tensor(y.size(), dtype=torch.get_default_dtype()))) + if not scipy.sparse.issparse(X) else y.numpy().size), + k=1, + which='LM', + return_singular_vectors=False) + # avoid runaway sv1 + sv1 = np.array([min(np.finfo(np.float32).max, + sv1[0])]) + + if not classification: + ymn = y.mean() + ysd = y.std() + else: + # TODO: set these, for each class? + ymn = 0. + ysd = 1. + if self.verbose: + print(" computing data statistics took: ", time.time() - t) + + return Xmn, sv1, Xsd, ymn, ysd + + + def set_data_stats(self, Xmn, sv1, Xsd=1., ymn=0., ysd=1.): + """ + Saves dataset stats to self to be used for preprocessing. + """ + + self.Xmn = torch.as_tensor( + Xmn, dtype=torch.get_default_dtype()).to(self.device) + self.sv1 = torch.as_tensor( + sv1, dtype=torch.get_default_dtype()).to(self.device) + self.Xsd = torch.as_tensor( + Xsd, dtype=torch.get_default_dtype()).to(self.device) + self.ymn = torch.as_tensor( + ymn, dtype=torch.get_default_dtype()).to(self.device) + self.ysd = torch.as_tensor( + ysd, dtype=torch.get_default_dtype()).to(self.device) + + + def apply_preprocess(self, X, y): + """ + Faster on gpu device, while dataloading takes up a large portion of the time. + """ + + with torch.no_grad(): + if not self.classification: + y = (y.reshape((-1, 1)) - self.ymn) / self.ysd + else: + y = y.reshape((-1, 1)) + X = (X - self.Xmn) / self.sv1 + + if self.preprocess == constants.Preprocess.ZSCORE: + X /= self.Xsd + + return X, y + + + def max_batch_size(self): + """ + Return the maximum batchsize for the dataset. + """ + + return int(np.min([self.max_rows, self.N])) + + + def apply(self, ix_rows=None, ix_cols=None, f_Xy=None): + + if f_Xy is None: + return + + if ix_rows is None: + ix_rows = range(self.N) + + if ix_cols is None: + ix_cols = range(self.n_features) + + f_Xy((self.X[ix_rows, ix_cols] + if not self.storage_level == constants.StorageLevel.SPARSE + else self.X[ix_rows, ix_cols].toarray()), self.y[ix_rows]) + + + def get_dense_data(self, ix_cols=None, ix_rows=None): + + if ix_cols is None: + ix_cols = range(self.n_features) + + X = [np.zeros((0, len(ix_cols)))] + y = [np.zeros((0, 1))] + Bnp = self.return_np + + def f_Xy(Xb, yb, n): + X[-1] = np.concatenate((X[-1], Xb), axis=0) + y[-1] = np.concatenate((y[-1], yb), axis=0) + self.apply(f_Xy=f_Xy, ix_rows=ix_rows, ix_cols=ix_cols) + self.set_return_np(Bnp) + + return X[-1], y[-1] + + + def __len__(self): + + return self.N + + + def getXy(self, idx): + + if self.storage_level == constants.StorageLevel.DENSE: + X, y = self.X[idx], self.y[idx] + elif self.storage_level == constants.StorageLevel.SPARSE: + # assume subset can fit into memory even if whole matrix cant + X, y = self.X[idx].toarray(), self.y[idx] + else: + raise NotImplementedError + + return X, y + + + def __getitem__(self, idx): + + with torch.no_grad(): + X, y = self.getXy(idx) + X = X.toarray() if scipy.sparse.issparse(X) else X + + X = torch.as_tensor( + X, dtype=torch.get_default_dtype()).to(self.device) + y = torch.as_tensor( + y, dtype=torch.get_default_dtype()).to(self.device) + + if not self.return_raw: + X, y = self.apply_preprocess(X, y) + + if self.classification and ( + self.n_classes is None or self.n_classes == 2): + y[y == 0] = -1 + + if self.return_np: + if constants.Device.CPU not in self.device: + X = X.cpu() + y = y.cpu() + X = X.numpy() + y = y.numpy() + return X, y + + return X, y + + +class ChunkDataLoader(DataLoader): + """ + DataLoader class used to more quickly load a batch of indices at once. + """ + + def __iter__(self): + return _ChunkDataLoaderIter(self) + + +class _ChunkDataLoaderIter: + """ + DataLoaderIter class used to more quickly load a batch of indices at once. + """ + def __init__(self, dataloader): + if dataloader.num_workers == 0: + self.iter = _SingleProcessDataLoaderIter(dataloader) + else: + self.iter = _MultiProcessingDataLoaderIter(dataloader) + + def __next__(self): + # only chunk that is edited from base + if self.iter._num_workers == 0: # same-process loading + indices = next(self.iter._sampler_iter) # may raise StopIteration + if len(indices) > 1: + batch = self.iter._dataset[np.array(indices)] + else: + batch = self.iter._collate_fn([self.iter._dataset[i] for i in indices]) + + if self.iter._pin_memory: + batch = _utils.pin_memory.pin_memory_batch(batch) + return batch + else: + return next(self.iter) diff --git a/new_impl/cv/third_party/nni_new/algorithms/feature_engineering/gradient_selector/fgtrain.py b/new_impl/cv/third_party/nni_new/algorithms/feature_engineering/gradient_selector/fgtrain.py new file mode 100644 index 0000000000000000000000000000000000000000..377d72691613b93911e2bbc164e3c677ac31f574 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/feature_engineering/gradient_selector/fgtrain.py @@ -0,0 +1,228 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# MIT License +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and +# associated documentation files (the "Software"), to deal in the Software without restriction, +# including without limitation the rights to use, copy, modify, merge, publish, distribute, +# sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or +# substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT +# NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT +# OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# ================================================================================================== + + +import time + +import numpy as np +import torch +from sklearn.feature_selection import SelectKBest, \ + f_classif, mutual_info_classif, f_regression, mutual_info_regression + +from . import constants +from . import syssettings +from .learnability import Solver +from .utils import EMA + +torch.set_default_tensor_type(syssettings.torch.tensortype) + + +def get_optim_f_stop(maxiter, maxtime, dftol_stop, freltol_stop, + minibatch=True): + """ + Check stopping conditions. + """ + + discount_factor = 1. / 3 + + total_t = [0.] + df_store = [np.nan] + it_store = [0] + relchange_store = [np.nan] + f_ma = EMA(discount_factor=discount_factor) + df_ma = EMA(discount_factor=discount_factor) + + def f_stop(f0, v0, it, t): + + flag_stop = False + + total_t[-1] += t + g = f0.x.grad.clone().cpu().detach() + df = g.abs().max().numpy().squeeze() + v = v0.clone().cpu().detach() + f = v.numpy().squeeze() + + if it >= maxiter: + flag_stop = True + + elif total_t[-1] >= maxtime: + flag_stop = True + + f_ma.update(f) + df_ma.update(df) + rel_change = f_ma.relchange() + + if ((not minibatch) and (df < dftol_stop)) \ + or (minibatch and (df_ma() < dftol_stop)): + flag_stop = True + + if rel_change < freltol_stop: + flag_stop = True + + if not minibatch: + df_store[-1] = df + else: + df_store[-1] = df_ma() + relchange_store[-1] = rel_change + it_store[-1] = it + + return flag_stop + + return f_stop, {'t': total_t, 'it': it_store, 'df': df_store, + 'relchange': relchange_store} + + +def get_init(data_train, init_type='on', rng=np.random.RandomState(0), prev_score=None): + """ + Initialize the 'x' variable with different settings + """ + + D = data_train.n_features + value_off = constants.Initialization.VALUE_DICT[ + constants.Initialization.OFF] + value_on = constants.Initialization.VALUE_DICT[ + constants.Initialization.ON] + + if prev_score is not None: + x0 = prev_score + elif not isinstance(init_type, str): + x0 = value_off * np.ones(D) + x0[init_type] = value_on + elif init_type.startswith(constants.Initialization.RANDOM): + d = int(init_type.replace(constants.Initialization.RANDOM, '')) + x0 = value_off * np.ones(D) + x0[rng.permutation(D)[:d]] = value_on + elif init_type == constants.Initialization.SKLEARN: + B = data_train.return_raw + X, y = data_train.get_dense_data() + data_train.set_return_raw(B) + ix = train_sk_dense(init_type, X, y, data_train.classification) + x0 = value_off * np.ones(D) + x0[ix] = value_on + elif init_type in constants.Initialization.VALUE_DICT: + x0 = constants.Initialization.VALUE_DICT[init_type] * np.ones(D) + else: + raise NotImplementedError( + 'init_type {0} not supported yet'.format(init_type)) + # pylint: disable=E1102 + return torch.tensor(x0.reshape((-1, 1)), + dtype=torch.get_default_dtype()) + + +def get_checkpoint(S, stop_conds, rng=None, get_state=True): + """ + Save the necessary information into a dictionary + """ + + m = {} + m['ninitfeats'] = S.ninitfeats + m['x0'] = S.x0 + x = S.x.clone().cpu().detach() + m['feats'] = np.where(x.numpy() >= 0)[0] + m.update({k: v[0] for k, v in stop_conds.items()}) + if get_state: + m.update({constants.Checkpoint.MODEL: S.state_dict(), + constants.Checkpoint.OPT: S.opt_train.state_dict(), + constants.Checkpoint.RNG: torch.get_rng_state(), + }) + if rng: + m.update({'rng_state': rng.get_state()}) + + return m + + +def _train(data_train, Nminibatch, order, C, rng, lr_train, debug, maxiter, + maxtime, init, dftol_stop, freltol_stop, dn_log, accum_steps, + path_save, shuffle, device=constants.Device.CPU, + verbose=1, + prev_checkpoint=None, + groups=None, + soft_groups=None): + """ + Main training loop. + """ + + t_init = time.time() + + x0 = get_init(data_train, init, rng) + if isinstance(init, str) and init == constants.Initialization.ZERO: + ninitfeats = -1 + else: + ninitfeats = np.where(x0.detach().numpy() > 0)[0].size + + S = Solver(data_train, order, + Nminibatch=Nminibatch, x0=x0, C=C, + ftransform=lambda x: torch.sigmoid(2 * x), + get_train_opt=lambda p: torch.optim.Adam(p, lr_train), + rng=rng, + accum_steps=accum_steps, + shuffle=shuffle, + groups=groups, + soft_groups=soft_groups, + device=device, + verbose=verbose) + S = S.to(device) + + S.ninitfeats = ninitfeats + S.x0 = x0 + + if prev_checkpoint: + S.load_state_dict(prev_checkpoint[constants.Checkpoint.MODEL]) + S.opt_train.load_state_dict(prev_checkpoint[constants.Checkpoint.OPT]) + torch.set_rng_state(prev_checkpoint[constants.Checkpoint.RNG]) + + minibatch = S.Ntrain != S.Nminibatch + + f_stop, stop_conds = get_optim_f_stop(maxiter, maxtime, dftol_stop, + freltol_stop, minibatch=minibatch) + + if debug: + pass + else: + f_callback = None + stop_conds['t'][-1] = time.time() - t_init + + S.train(f_stop=f_stop, f_callback=f_callback) + + return get_checkpoint(S, stop_conds, rng), S + + +def train_sk_dense(ty, X, y, classification): + if classification: + if ty.startswith('skf'): + d = int(ty.replace('skf', '')) + f_sk = f_classif + elif ty.startswith('skmi'): + d = int(ty.replace('skmi', '')) + f_sk = mutual_info_classif + else: + if ty.startswith('skf'): + d = int(ty.replace('skf', '')) + f_sk = f_regression + elif ty.startswith('skmi'): + d = int(ty.replace('skmi', '')) + f_sk = mutual_info_regression + t = time.time() + clf = SelectKBest(f_sk, k=d) + clf.fit_transform(X, y.squeeze()) + ix = np.argsort(-clf.scores_) + ix = ix[np.where(np.invert(np.isnan(clf.scores_[ix])))[0]][:d] + t = time.time() - t + return {'feats': ix, 't': t} diff --git a/new_impl/cv/third_party/nni_new/algorithms/feature_engineering/gradient_selector/gradient_selector.py b/new_impl/cv/third_party/nni_new/algorithms/feature_engineering/gradient_selector/gradient_selector.py new file mode 100644 index 0000000000000000000000000000000000000000..f7cb69f627442cb8b749f792cecfa0fea7526e1b --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/feature_engineering/gradient_selector/gradient_selector.py @@ -0,0 +1,631 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# MIT License +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and +# associated documentation files (the "Software"), to deal in the Software without restriction, +# including without limitation the rights to use, copy, modify, merge, publish, distribute, +# sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or +# substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT +# NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT +# OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# ================================================================================================== + +import time + +import numpy as np +import pandas as pd + +from sklearn.base import BaseEstimator +from sklearn.feature_selection import SelectorMixin +from sklearn.utils.validation import check_is_fitted + +import torch + +from nni.feature_engineering.feature_selector import FeatureSelector +from . import constants +from .fginitialize import PrepareData +from .fgtrain import _train + + +class FeatureGradientSelector(FeatureSelector, BaseEstimator, SelectorMixin): + def __init__(self, + order=4, + penalty=1, + n_features=None, + max_features=None, + learning_rate=1e-1, + init='zero', + n_epochs=1, + shuffle=True, + batch_size=1000, + target_batch_size=1000, + max_time=np.inf, + classification=True, + ordinal=False, + balanced=True, + preprocess='zscore', + soft_grouping=False, + verbose=0, + device='cpu'): + """ + FeatureGradientSelector is a class that selects features for a machine + learning model using a gradient based search. + + Parameters + ---------- + order : int + What order of interactions to include. Higher orders + may be more accurate but increase the run time. 12 is the maximum allowed order. + penatly : int + Constant that multiplies the regularization term. + n_features: int + If None, will automatically choose number of features based on search. + Otherwise, number of top features to select. + max_features : int + If not None, will use the 'elbow method' to determine the number of features + with max_features as the upper limit. + learning_rate : float + init : str + How to initialize the vector of scores. 'zero' is the default. + Options: {'zero', 'on', 'off', 'onhigh', 'offhigh', 'sklearn'} + n_epochs : int + number of epochs to run + shuffle : bool + Shuffle "rows" prior to an epoch. + batch_size : int + Nnumber of "rows" to process at a time + target_batch_size : int + Number of "rows" to accumulate gradients over. + Useful when many rows will not fit into memory but are needed for accurate estimation. + classification : bool + If True, problem is classification, else regression. + ordinal : bool + If True, problem is ordinal classification. Requires classification to be True. + balanced : bool + If true, each class is weighted equally in optimization, otherwise + weighted is done via support of each class. Requires classification to be True. + prerocess : str + 'zscore' which refers to centering and normalizing data to unit variance or + 'center' which only centers the data to 0 mean + soft_grouping : bool + if True, groups represent features that come from the same source. + Used to encourage sparsity of groups and features within groups. + verbose : int + Controls the verbosity when fitting. Set to 0 for no printing + 1 or higher for printing every verbose number of gradient steps. + device : str + 'cpu' to run on CPU and 'cuda' to run on GPU. Runs much faster on GPU + """ + assert order <= 12 and order >= 1, 'order must be an integer between 1 and 12, inclusive' + assert n_features is None or max_features is None, \ + 'only specify one of n_features and max_features at a time' + + self.order = order + self.penalty = penalty + self.n_features = n_features + self.max_features = max_features + self.learning_rate = learning_rate + self.init = init + self.n_epochs = n_epochs + self.shuffle = shuffle + self.batch_size = batch_size + self.target_batch_size = target_batch_size + self.max_time = max_time + self.dftol_stop = -1 + self.freltol_stop = -1 + self.classification = classification + self.ordinal = ordinal + self.balanced = balanced + self.preprocess = preprocess + self.soft_grouping = soft_grouping + self.verbose = verbose + self.device = device + + self.model_ = None + self.scores_ = None + self._prev_checkpoint = None + self._data_train = None + + def partial_fit(self, X, y, + n_classes=None, + groups=None): + """ + Select Features via a gradient based search on (X, y) on the given samples. + Can be called repeatedly with different X and y to handle streaming datasets. + + Parameters + ---------- + X : array-like + Shape = [n_samples, n_features] + The training input samples. + y : array-like + Shape = [n_samples] + The target values (class labels in classification, real numbers in + regression). + n_classes : int + Number of classes + Classes across all calls to partial_fit. + Can be obtained by via `np.unique(y_all).shape[0]`, where y_all is the + target vector of the entire dataset. + This argument is expected for the first call to partial_fit, + otherwise will assume all classes are present in the batch of y given. + It will be ignored in the subsequent calls. + Note that y doesn't need to contain all labels in `classes`. + groups : array-like + Optional, shape = [n_features] + Groups of columns that must be selected as a unit + e.g. [0, 0, 1, 2] specifies the first two columns are part of a group. + This argument is expected for the first call to partial_fit, + otherwise will assume all classes are present in the batch of y given. + It will be ignored in the subsequent calls. + """ + try: + self._partial_fit(X, y, n_classes=n_classes, groups=groups) + except constants.NanError: + if hasattr(self, '_prev_checkpoint'): + # if it's already done some batches successfully just ignore it + print('failed fitting this batch, loss was nan') + else: + # if this is the first batch, reset and try with doubles + if self.verbose: + print('Loss was nan, trying with Doubles') + self._reset() + torch.set_default_tensor_type(torch.DoubleTensor) + self._partial_fit(X, y, n_classes=n_classes, groups=groups) + + return self + + def _partial_fit(self, X, y, n_classes=None, groups=None): + """ + Private function for partial_fit to enable trying floats before doubles. + """ + # pass in X and y in chunks + if hasattr(self, '_data_train'): + # just overwrite the X and y from the new chunk but make them tensors + # keep dataset stats from previous + self._data_train.X = X.values if isinstance(X, pd.DataFrame) else X + self._data_train.N, self._data_train.D = self._data_train.X.shape + self._data_train.dense_size_gb = self._data_train.get_dense_size() + self._data_train.set_dense_X() + + self._data_train.y = y.values if isinstance(y, pd.Series) else y + self._data_train.y = torch.as_tensor( + y, dtype=torch.get_default_dtype()) + else: + data_train = self._prepare_data(X, y, n_classes=n_classes) + self._data_train = data_train + + batch_size, _, accum_steps, max_iter = self._set_batch_size( + self._data_train) + + rng = None # not used + debug = 0 # {0,1} print messages and do other stuff? + dn_logs = None # tensorboard logs; only specify if debug=1 + path_save = None # intermediate models saves; only specify if debug=1 + m, solver = _train(self._data_train, + batch_size, + self.order, + self.penalty, + rng, + self.learning_rate, + debug, + max_iter, + self.max_time, + self.init, + self.dftol_stop, + self.freltol_stop, + dn_logs, + accum_steps, + path_save, + self.shuffle, + device=self.device, + verbose=self.verbose, + prev_checkpoint=self._prev_checkpoint if hasattr( + self, '_prev_checkpoint') else None, + groups=groups if not self.soft_grouping else None, + soft_groups=groups if self.soft_grouping else None) + + self._prev_checkpoint = m + self._process_results(m, solver, X, groups=groups) + return self + + def fit(self, X, y, + groups=None): + """ + Select Features via a gradient based search on (X, y). + + Parameters + ---------- + X : array-like + Shape = [n_samples, n_features] + The training input samples. + y : array-like + Shape = [n_samples] + The target values (class labels in classification, real numbers in + regression). + groups : array-like + Optional, shape = [n_features] + Groups of columns that must be selected as a unit + e.g. [0, 0, 1, 2] specifies the first two columns are part of a group. + """ + try: + self._fit(X, y, groups=groups) + except constants.NanError: + if self.verbose: + print('Loss was nan, trying with Doubles') + torch.set_default_tensor_type(torch.DoubleTensor) + self._fit(X, y, groups=groups) + return self + + def get_selected_features(self): + return self.selected_features_ + + def _prepare_data(self, X, y, n_classes=None): + """ + Returns a PrepareData object. + """ + return PrepareData(X=X.values if isinstance(X, pd.DataFrame) else X, + y=y.values if isinstance(y, pd.Series) else y, + data_format=constants.DataFormat.NUMPY, + classification=int(self.classification), + ordinal=self.ordinal, + balanced=self.balanced, + preprocess=self.preprocess, + verbose=self.verbose, + device=self.device, + n_classes=n_classes) + + def _fit(self, X, y, groups=None): + """ + Private function for fit to enable trying floats before doubles. + """ + data_train = self._prepare_data(X, y) + + batch_size, _, accum_steps, max_iter = self._set_batch_size( + data_train) + + rng = None # not used + debug = 0 # {0,1} print messages and log to tensorboard + dn_logs = None # tensorboard logs; only specify if debug=1 + path_save = None # intermediate models saves; only specify if debug=1 + m, solver = _train(data_train, + batch_size, + self.order, + self.penalty, + rng, + self.learning_rate, + debug, + max_iter, + self.max_time, + self.init, + self.dftol_stop, + self.freltol_stop, + dn_logs, + accum_steps, + path_save, + self.shuffle, + device=self.device, + verbose=self.verbose, + groups=groups if not self.soft_grouping else None, + soft_groups=groups if self.soft_grouping else None) + + self._process_results(m, solver, X, groups=groups) + return self + + def _process_torch_scores(self, scores): + """ + Convert scores into flat numpy arrays. + """ + if constants.Device.CUDA in scores.device.type: + scores = scores.cpu() + return scores.numpy().ravel() + + def _set_batch_size(self, data_train): + """ + Ensures that batch_size is less than the number of rows. + """ + batch_size = min(self.batch_size, data_train.N) + target_batch_size = min(max( + self.batch_size, self.target_batch_size), data_train.N) + accum_steps = max(int(np.ceil(target_batch_size / self.batch_size)), 1) + max_iter = self.n_epochs * (data_train.N // batch_size) + return batch_size, target_batch_size, accum_steps, max_iter + + def _process_results(self, m, solver, X, groups=None): + """ + Process the results of a run into something suitable for transform(). + """ + self.scores_ = self._process_torch_scores( + torch.sigmoid(m[constants.Checkpoint.MODEL]['x'] * 2)) + if self.max_features: + self.max_features = min([self.max_features, self.scores_.shape[0]]) + n_features = self._recommend_number_features(solver) + self.set_n_features(n_features, groups=groups) + elif self.n_features: + self.set_n_features(self.n_features, groups=groups) + else: + self.selected_features_ = m['feats'] + + # subtract elapsed time from max_time + self.max_time -= m['t'] + + self.model_ = m + + return self + + def transform(self, X): + """ + Returns selected features from X. + + Paramters + --------- + X: array-like + Shape = [n_samples, n_features] + The training input samples. + """ + + self._get_support_mask() + if self.selected_features_.shape[0] == 0: + raise ValueError( + 'No Features selected, consider lowering the penalty or specifying n_features') + return (X.iloc[:, self.selected_features_] + if isinstance(X, pd.DataFrame) + else X[:, self.selected_features_]) + + def get_support(self, indices=False): + """ + Get a mask, or integer index, of the features selected. + + Parameters + ---------- + indices : bool + Default False + If True, the return value will be an array of integers, rather than a boolean mask. + + Returns + ------- + list : + returns support: An index that selects the retained features from a feature vector. + If indices is False, this is a boolean array of shape [# input features], + in which an element is True iff its corresponding feature is selected for retention. + If indices is True, this is an integer array of shape [# output features] whose values + are indices into the input feature vector. + """ + self._get_support_mask() + if indices: + return self.selected_features_ + + mask = np.zeros_like(self.scores_, dtype=bool) + # pylint: disable=E1137 + mask[self.selected_features_] = True + return mask + + def inverse_transform(self, X): + """ + Returns transformed X to the original number of column. + This operation is lossy and all columns not in the transformed data + will be returned as columns of 0s. + """ + self._get_support_mask() + X_new = np.zeros((X.shape[0], self.scores_.shape[0])) + X_new[self.selected_features_] = X + return X_new + + def get_params(self, deep=True): + """ + Get parameters for this estimator. + """ + params = self.__dict__ + params = {key: val for (key, val) in params.items() + if not key.endswith('_')} + return params + + def set_params(self, **params): + """ + Set the parameters of this estimator. + """ + for param in params: + if hasattr(self, param): + setattr(self, param, params[param]) + return self + + def fit_transform(self, X, y): + """ + Select features and then return X with the selected features. + + Parameters + ---------- + X : array-like + Shape = [n_samples, n_features] + The training input samples. + y : array-like + Shape = [n_samples] + The target values (class labels in classification, real numbers in + regression). + """ + self.fit(X, y) + return self.transform(X) + + def _get_support_mask(self): + """ + Check if it is fitted. + """ + check_is_fitted(self, 'scores_') + + def _generate_scores(self, solver, xsub, ysub, step_size, feature_order): + """ + Generate forward passes to determine the number of features when max_features is set. + """ + scores = [] + for i in np.arange(1, self.max_features + 1, step_size): + # optimization possible since xsub is growing? + i = int(np.ceil(i)) + # pylint: disable=E1102 + score = solver.f_train(torch.tensor(np.ones(i), + dtype=torch.get_default_dtype() + ).unsqueeze(1).to(self.device), + xsub[:, feature_order[:i]], + ysub) + if constants.Device.CUDA in score.device.type: + score = score.cpu() + # score.numpy()[0][0] + scores.append(score) + return scores + + def set_n_features(self, n, groups=None): + """ + Set the number of features to return after fitting. + """ + self._get_support_mask() + self.n_features = n + return self._set_top_features(groups=groups) + + def _set_top_features(self, groups=None): + """ + Set the selected features after a run. + + With groups, ensures that if any member of a group is selected, all members are selected + """ + self._get_support_mask() + assert self.n_features <= self.scores_.shape[0], \ + 'n_features must be less than or equal to the number of columns in X' + # pylint: disable=E1130 + self.selected_features_ = np.argpartition( + self.scores_, -self.n_features)[-self.n_features:] + if groups is not None and not self.soft_grouping: + selected_feature_set = set(self.selected_features_.tolist()) + for _ in np.unique(groups): + group_members = np.where(groups == groups)[0].tolist() + if selected_feature_set.intersection(group_members): + selected_feature_set.update(group_members) + self.selected_features_ = np.array(list(selected_feature_set)) + self.selected_features_ = np.sort(self.selected_features_) + return self + + def set_top_percentile(self, percentile, groups=None): + """ + Set the percentile of features to return after fitting. + """ + self._get_support_mask() + assert percentile <= 1 and percentile >= 0, \ + 'percentile must between 0 and 1 inclusive' + self.n_features = int(self.scores_.shape[0] * percentile) + return self._set_top_features(groups=groups) + + def _recommend_number_features(self, solver, max_time=None): + """ + Get the recommended number of features by doing forward passes when max_features is set. + """ + max_time = max_time if max_time else self.max_time + if max_time < 0: + max_time = 60 # allow 1 minute extra if we already spent max_time + MAX_FORWARD_PASS = 200 + MAX_FULL_BATCHES = 3 # the forward passes can take longer than the fitting + # if we allow a full epoch of data to be included. By only doing 3 full batches at most + # we get enough accuracy without increasing the time too much. This + # constant may not be optimal + accum_steps = solver.accum_steps + step_size = max(self.max_features / MAX_FORWARD_PASS, 1) + # pylint: disable=E1130 + feature_order = np.argsort(-self.scores_) # note the negative + t = time.time() + + dataloader_iterator = iter(solver.ds_train) + full_scores = [] + # keep_going = True + with torch.no_grad(): + # might want to only consider a batch valid if there are at least + # two classes + for _ in range(accum_steps * MAX_FULL_BATCHES): + scores = [] + try: + xsub, ysub = next(dataloader_iterator) + except StopIteration: + # done with epoch, don't do more than one epoch + break + except Exception as e: + print(e) + break + if max_time and time.time() - t > max_time: + if self.verbose: + print( + "Stoppinn forward passes because they reached max_time: ", + max_time) + if not full_scores: + # no forward passes worked, return half of max_features + return self.max_features // 2 + break + if solver.multiclass: + for target_class in range(solver.n_classes): + ysub_binary = solver.transform_y_into_binary( + ysub, target_class) + scaling_value = solver._get_scaling_value( + ysub, target_class) + if not solver._skip_y_forward(ysub_binary): + scores = self._generate_scores( + solver, xsub, ysub_binary, step_size, feature_order) + # one row will represent one class that is present in the data + # all classes are weighted equally + full_scores.append( + [score * scaling_value for score in scores]) + else: + if not solver._skip_y_forward(ysub): + scores = self._generate_scores( + solver, xsub, ysub, step_size, feature_order) + full_scores.append(scores) + best_index = FeatureGradientSelector._find_best_index_elbow( + full_scores) + if self.verbose: + print("Forward passes took: ", time.time() - t) + # account for step size and off by one (n_features is 1 indexed, not 0 + # ) + return int( + np.ceil( + np.arange( + 1, + self.max_features + + 1, + step_size))[best_index]) + + @staticmethod + def _find_best_index_elbow(full_scores): + """ + Finds the point on the curve that maximizes distance from the line determined by the endpoints. + """ + scores = pd.DataFrame(full_scores).mean(0).values.tolist() + first_point = np.array([0, scores[0]]) + last_point = np.array([len(scores) - 1, scores[-1]]) + elbow_metric = [] + for i in range(len(scores)): + elbow_metric.append( + FeatureGradientSelector._distance_to_line( + first_point, last_point, np.array([i, scores[i]]))) + return np.argmax(elbow_metric) + + @staticmethod + def _distance_to_line(start_point, end_point, new_point): + """ + Calculates the shortest distance from new_point to the line determined by start_point and end_point. + """ + # for calculating elbow method + return np.cross(new_point - start_point, + end_point - start_point) / np.linalg.norm( + end_point - start_point) + + def _reset(self): + """ + Reset the estimator by deleting all private and fit parameters. + """ + params = self.__dict__ + for key, _ in params.items(): + if key.endswith('_') or key.startswith('_'): + delattr(self, key) + return self diff --git a/new_impl/cv/third_party/nni_new/algorithms/feature_engineering/gradient_selector/learnability.py b/new_impl/cv/third_party/nni_new/algorithms/feature_engineering/gradient_selector/learnability.py new file mode 100644 index 0000000000000000000000000000000000000000..3a0ab4b39e7bef15101c793f3ec522395c05cd94 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/feature_engineering/gradient_selector/learnability.py @@ -0,0 +1,534 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# MIT License +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and +# associated documentation files (the "Software"), to deal in the Software without restriction, +# including without limitation the rights to use, copy, modify, merge, publish, distribute, +# sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or +# substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT +# NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT +# OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# ================================================================================================== + +import time + +import numpy as np +import scipy.special +import torch +import torch.nn as nn + +from . import constants +from . import syssettings +from .fginitialize import ChunkDataLoader + +torch.set_default_tensor_type(syssettings.torch.tensortype) +sparsetensor = syssettings.torch.sparse.tensortype + + +def def_train_opt(p): + """ + Return the default optimizer. + """ + return torch.optim.Adam(p, 1e-1, amsgrad=False) + + +def revcumsum(U): + """ + Reverse cumulative sum for faster performance. + """ + return U.flip(dims=[0]).cumsum(dim=0).flip(dims=[0]) + + +def triudr(X, r): + + Zr = torch.zeros_like(X, requires_grad=False) + U = X * r + Zr[:-1] = X[:-1] * revcumsum(U)[1:] + + return Zr + + +def triudl(X, l): + + Zl = torch.zeros_like(X, requires_grad=False) + U = X * l + Zl[1:] = X[1:] * (U.cumsum(dim=0)[:-1]) + + return Zl + + +class ramp(torch.autograd.Function): + """ + Ensures input is between 0 and 1 + """ + + @staticmethod + def forward(ctx, input_data): + ctx.save_for_backward(input_data) + return input_data.clamp(min=0, max=1) + + + @staticmethod + def backward(ctx, grad_output): + input_data, = ctx.saved_tensors + grad_input = grad_output.clone() + grad_input[input_data < 0] = 1e-2 + grad_input[input_data > 1] = -1e-2 + return grad_input + + +class safesqrt(torch.autograd.Function): + """ + Square root without dividing by 0. + """ + @staticmethod + def forward(ctx, input_data): + o = input_data.sqrt() + ctx.save_for_backward(input_data, o) + return o + + + @staticmethod + def backward(ctx, grad_output): + _, o = ctx.saved_tensors + grad_input = grad_output.clone() + grad_input *= 0.5 / (o + constants.EPSILON) + return grad_input + + +class LearnabilityMB(nn.Module): + """ + Calculates the learnability of a set of features. + mini-batch version w/ "left" and "right" multiplies + """ + + + def __init__(self, Nminibatch, D, coeff, groups=None, binary=False, + device=constants.Device.CPU): + super(LearnabilityMB, self).__init__() + + a = coeff / scipy.special.binom(Nminibatch, np.arange(coeff.size) + 2) + self.order = a.size + # pylint: disable=E1102 + self.a = torch.tensor(a, dtype=torch.get_default_dtype(), requires_grad=False) + self.binary = binary + + self.a = self.a.to(device) + + + def ret_val(self, z): + """ + Get the return value based on z. + """ + + if not self.binary: + return 1 - z + + else: + return 0.5 * (1 - safesqrt.apply(ramp.apply(z))) + + + def forward(self, s, X, y): + + l = y.clone() + r = y.clone() + z = 0 + + for i in range(self.order): + if i % 2 == 0: + Z = triudr(X, r) + r = torch.mm(Z, s) + else: + Z = triudl(X, l) + l = torch.mm(Z, s) + if self.a[i] != 0: + # same the computation if a[i] is 0 + p = torch.mm(l.t(), r) + z += self.a[i] * p + return self.ret_val(z) + + +class Solver(nn.Module): + """ + Class that performs the main optimization. + Keeps track of the current x and iterates through data to learn x given the penalty and order. + """ + + def __init__(self, + PreparedData, + order, + Nminibatch=None, + groups=None, + soft_groups=None, + x0=None, + C=1, + ftransform=torch.sigmoid, + get_train_opt=def_train_opt, + accum_steps=1, + rng=np.random.RandomState(0), + max_norm_clip=1., + shuffle=True, + device=constants.Device.CPU, + verbose=1): + """ + + Parameters + ---------- + PreparedData : Dataset of PrepareData class + order : int + What order of interactions to include. Higher orders + may be more accurate but increase the run time. 12 is the maximum allowed order. + Nminibatch : int + Number of rows in a mini batch + groups : array-like + Optional, shape = [n_features] + Groups of columns that must be selected as a unit + e.g. [0, 0, 1, 2] specifies the first two columns are part of a group. + soft_groups : array-like + optional, shape = [n_features] + Groups of columns come from the same source + Used to encourage sparsity of number of sources selected + e.g. [0, 0, 1, 2] specifies the first two columns are part of a group. + x0 : torch.tensor + Optional, initialization of x. + C : float + Penalty parameter. + get_train_opt : function + Function that returns a pytorch optimizer, Adam is the default + accum_steps : int + Number of steps + rng : random state + max_norm_clip : float + Maximum allowable size of the gradient + shuffle : bool + Whether or not to shuffle data within the dataloader + order : int + What order of interactions to include. Higher orders + may be more accurate but increase the run time. 12 is the maximum allowed order. + penalty : int + Constant that multiplies the regularization term. + ftransform : function + Function to transform the x. sigmoid is the default. + device : str + 'cpu' to run on CPU and 'cuda' to run on GPU. Runs much faster on GPU + verbose : int + Controls the verbosity when fitting. Set to 0 for no printing + 1 or higher for printing every verbose number of gradient steps. + """ + super(Solver, self).__init__() + + self.Ntrain, self.D = PreparedData.N, PreparedData.n_features + if groups is not None: + # pylint: disable=E1102 + groups = torch.tensor(groups, dtype=torch.long) + self.groups = groups + else: + self.groups = None + if soft_groups is not None: + # pylint: disable=E1102 + soft_groups = torch.tensor(soft_groups, dtype=torch.long) + self.soft_D = torch.unique(soft_groups).size()[0] + else: + self.soft_D = None + self.soft_groups = soft_groups + + if Nminibatch is None: + Nminibatch = self.Ntrain + else: + if Nminibatch > self.Ntrain: + print('Minibatch larger than sample size.' + + (' Reducing from %d to %d.' + % (Nminibatch, self.Ntrain))) + Nminibatch = self.Ntrain + if Nminibatch > PreparedData.max_rows: + print('Minibatch larger than mem-allowed.' + + (' Reducing from %d to %d.' % (Nminibatch, + PreparedData.max_rows))) + Nminibatch = int(np.min([Nminibatch, PreparedData.max_rows])) + self.Nminibatch = Nminibatch + self.accum_steps = accum_steps + + if x0 is None: + x0 = torch.zeros(self.D, 1, dtype=torch.get_default_dtype()) + self.ftransform = ftransform + self.x = nn.Parameter(x0) + self.max_norm = max_norm_clip + + self.device = device + self.verbose = verbose + + self.multiclass = PreparedData.classification and PreparedData.n_classes and PreparedData.n_classes > 2 + if self.multiclass: + self.n_classes = PreparedData.n_classes + else: + self.n_classes = None + # whether to treat all classes equally + self.balanced = PreparedData.balanced + self.ordinal = PreparedData.ordinal + + if (hasattr(PreparedData, 'mappings') + or PreparedData.storage_level == 'disk'): + num_workers = PreparedData.num_workers + elif PreparedData.storage_level == constants.StorageLevel.DENSE: + num_workers = 0 + else: + num_workers = 0 + + if constants.Device.CUDA in device: + pin_memory = False + else: + pin_memory = False + + if num_workers == 0: + timeout = 0 + else: + timeout = 60 + + self.ds_train = ChunkDataLoader( + PreparedData, + batch_size=self.Nminibatch, + shuffle=shuffle, + drop_last=True, + num_workers=num_workers, + pin_memory=pin_memory, + timeout=timeout) + self.f_train = LearnabilityMB(self.Nminibatch, self.D, + constants.Coefficients.SLE[order], + self.groups, + binary=PreparedData.classification, + device=self.device) + self.opt_train = get_train_opt(torch.nn.ParameterList([self.x])) + self.it = 0 + self.iters_per_epoch = int(np.ceil(len(self.ds_train.dataset) + / self.ds_train.batch_size)) + self.f_train = self.f_train.to(device) + # pylint: disable=E1102 + self.w = torch.tensor( + C / (C + 1), + dtype=torch.get_default_dtype(), requires_grad=False) + self.w = self.w.to(device) + + + def penalty(self, s): + """ + Calculate L1 Penalty. + """ + to_return = torch.sum(s) / self.D + if self.soft_groups is not None: + # if soft_groups, there is an additional penalty for using more + # groups + s_grouped = torch.zeros(self.soft_D, 1, + dtype=torch.get_default_dtype(), + device=self.device) + for group in torch.unique(self.soft_groups): + # groups should be indexed 0 to n_group - 1 + # TODO: consider other functions here + s_grouped[group] = s[self.soft_groups == group].max() + # each component of the penalty contributes .5 + # TODO: could make this a user given parameter + to_return = (to_return + torch.sum(s_grouped) / self.soft_D) * .5 + return to_return + + + def forward_and_backward(self, s, xsub, ysub, retain_graph=False): + """ + Completes the forward operation and computes gradients for learnability and penalty. + """ + f_train = self.f_train(s, xsub, ysub) + pen = self.penalty(s).unsqueeze(0).unsqueeze(0) + # pylint: disable=E1102 + grad_outputs = torch.tensor([[1]], dtype=torch.get_default_dtype(), + device=self.device) + g1, = torch.autograd.grad([f_train], [self.x], grad_outputs, + retain_graph=True) + # pylint: disable=E1102 + grad_outputs = torch.tensor([[1]], dtype=torch.get_default_dtype(), + device=self.device) + g2, = torch.autograd.grad([pen], [self.x], grad_outputs, + retain_graph=retain_graph) + return f_train, pen, g1, g2 + + + def combine_gradient(self, g1, g2): + """ + Combine gradients from learnability and penalty + + Parameters + ---------- + g1 : array-like + gradient from learnability + g2 : array-like + gradient from penalty + """ + to_return = ((1 - self.w) * g1 + self.w * g2) / self.accum_steps + if self.groups is not None: + # each column will get a gradient + # but we can only up or down groups, so the gradient for the group + # should be the average of the gradients of the columns + to_return_grouped = torch.zeros_like(self.x) + for group in torch.unique(self.groups): + to_return_grouped[self.groups == + group] = to_return[self.groups == group].mean() + to_return = to_return_grouped + return to_return + + + def combine_loss(self, f_train, pen): + """ + Combine the learnability and L1 penalty. + """ + return ((1 - self.w) * f_train.detach() + self.w * pen.detach()) \ + / self.accum_steps + + + def transform_y_into_binary(self, ysub, target_class): + """ + Transforms multiclass classification problems into a binary classification problem. + """ + with torch.no_grad(): + ysub_binary = torch.zeros_like(ysub) + if self.ordinal: + # turn ordinal problems into n-1 classifications of is this + # example less than rank k + if target_class == 0: + return None + + ysub_binary[ysub >= target_class] = 1 + ysub_binary[ysub < target_class] = -1 + else: + # turn multiclass problems into n binary classifications + ysub_binary[ysub == target_class] = 1 + ysub_binary[ysub != target_class] = -1 + return ysub_binary + + + def _get_scaling_value(self, ysub, target_class): + """ + Returns the weight given to a class for multiclass classification. + """ + if self.balanced: + if self.ordinal: + return 1 / (torch.unique(ysub).size()[0] - 1) + + return 1 / torch.unique(ysub).size()[0] + else: + if self.ordinal: + this_class_proportion = torch.mean(ysub >= target_class) + normalizing_constant = 0 + for i in range(1, self.n_classes): + normalizing_constant += torch.mean(ysub >= i) + return this_class_proportion / normalizing_constant + else: + return torch.mean(ysub == target_class) + + + def _skip_y_forward(self, y): + """ + Returns boolean of whether to skip the currrent y if there is nothing to be learned from it. + """ + if y is None: + return True + elif torch.unique(y).size()[0] < 2: + return True + else: + return False + + + def train(self, f_callback=None, f_stop=None): + """ + Trains the estimator to determine which features to include. + + Parameters + ---------- + f_callback : function + Function that performs a callback + f_stop: function + Function that tells you when to stop + """ + + t = time.time() + h = torch.zeros([1, 1], dtype=torch.get_default_dtype()) + h = h.to(self.device) + # h_complete is so when we divide by the number of classes + # we only do that for that minibatch if accumulating + h_complete = h.clone() + flag_stop = False + dataloader_iterator = iter(self.ds_train) + self.x.grad = torch.zeros_like(self.x) + while not flag_stop: + try: + xsub, ysub = next(dataloader_iterator) + except StopIteration: + dataloader_iterator = iter(self.ds_train) + xsub, ysub = next(dataloader_iterator) + try: + s = self.ftransform(self.x) + s = s.to(self.device) + if self.multiclass: + # accumulate gradients over each class, classes range from + # 0 to n_classes - 1 + #num_classes_batch = torch.unique(ysub).size()[0] + for target_class in range(self.n_classes): + ysub_binary = self.transform_y_into_binary( + ysub, target_class) + if self._skip_y_forward(ysub_binary): + continue + # should should skip if target class is not included + # but that changes what we divide by + scaling_value = self._get_scaling_value( + ysub, target_class) + f_train, pen, g1, g2 = self.forward_and_backward( + s, xsub, ysub_binary, retain_graph=True) + self.x.grad += self.combine_gradient( + g1, g2) * scaling_value + h += self.combine_loss(f_train, + pen) * scaling_value + else: + if not self._skip_y_forward(ysub): + f_train, pen, g1, g2 = self.forward_and_backward( + s, xsub, ysub) + self.x.grad += self.combine_gradient(g1, g2) + h += self.combine_loss(f_train, pen) + else: + continue + h_complete += h + self.it += 1 + if torch.isnan(h): + raise constants.NanError( + 'Loss is nan, something may be misconfigured') + if self.it % self.accum_steps == 0: + torch.nn.utils.clip_grad_norm_( + torch.nn.ParameterList([self.x]), + max_norm=self.max_norm) + self.opt_train.step() + + t = time.time() - t + if f_stop is not None: + flag_stop = f_stop(self, h, self.it, t) + + if f_callback is not None: + f_callback(self, h, self.it, t) + elif self.verbose and (self.it // self.accum_steps) % self.verbose == 0: + epoch = int(self.it / self.iters_per_epoch) + print( + '[Minibatch: %6d/ Epoch: %3d/ t: %3.3f s] Loss: %0.3f' % + (self.it, epoch, t, h_complete / self.accum_steps)) + + if flag_stop: + break + + self.opt_train.zero_grad() + h = 0 + h_complete = 0 + t = time.time() + except KeyboardInterrupt: + flag_stop = True + break diff --git a/new_impl/cv/third_party/nni_new/algorithms/feature_engineering/gradient_selector/requirements.txt b/new_impl/cv/third_party/nni_new/algorithms/feature_engineering/gradient_selector/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..7e2873b558c30216e17b584570b7383623b2931c --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/feature_engineering/gradient_selector/requirements.txt @@ -0,0 +1,4 @@ +numpy==1.14.3 +scikit-learn>=0.23.2 +scipy==1.1.0 +torch==1.1.0 diff --git a/new_impl/cv/third_party/nni_new/algorithms/feature_engineering/gradient_selector/syssettings.py b/new_impl/cv/third_party/nni_new/algorithms/feature_engineering/gradient_selector/syssettings.py new file mode 100644 index 0000000000000000000000000000000000000000..df864b316601464a9f35a7b463933b1f05a9fe3f --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/feature_engineering/gradient_selector/syssettings.py @@ -0,0 +1,29 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# MIT License +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and +# associated documentation files (the "Software"), to deal in the Software without restriction, +# including without limitation the rights to use, copy, modify, merge, publish, distribute, +# sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or +# substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT +# NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT +# OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# ================================================================================================== + + +import torch + +# pytorch +torch.tensortype = torch.FloatTensor +torch.sparse.tensortype = torch.sparse.FloatTensor + +# mem +MAXMEMGB = 10 diff --git a/new_impl/cv/third_party/nni_new/algorithms/feature_engineering/gradient_selector/utils.py b/new_impl/cv/third_party/nni_new/algorithms/feature_engineering/gradient_selector/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..0ab9b09a25cfb328586c39f00ab9546388b4f8d4 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/feature_engineering/gradient_selector/utils.py @@ -0,0 +1,78 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# MIT License +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and +# associated documentation files (the "Software"), to deal in the Software without restriction, +# including without limitation the rights to use, copy, modify, merge, publish, distribute, +# sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or +# substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT +# NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT +# OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# ================================================================================================== + + +import numpy as np + +class EMA(): + """ + maintains an exponential moving average + """ + + def __init__(self, f=np.nan, discount_factor=0.1, valid_after=None, + n_iters_relchange=3): + + self.f_ma = [f] + self.fs = [f] + self.gamma = discount_factor + self.rel_change = [np.nan] + if valid_after is None: + self.valid_after = int(1/discount_factor) + else: + self.valid_after = valid_after + self.n_iters_relchange = n_iters_relchange + self.initialized = False + + def reset(self, f): + + self.f_ma = [f] + self.fs = [f] + self.rel_change = [np.nan] + self.initialized = True + + def relchange(self): + + if self.num_updates() > np.max([self.valid_after, + self.n_iters_relchange]): + return np.max(self.rel_change[-self.n_iters_relchange:]) + else: + return np.nan + + def update(self, f_new): + + if not self.initialized: + self.reset(f_new) + else: + self.fs.append(f_new) + self.f_ma.append(self.f_ma[-1]*(1-self.gamma) + self.gamma*f_new) + if self.num_updates() > self.valid_after: + self.rel_change.append(np.abs((self.f_ma[-1]-self.f_ma[-2]) + / self.f_ma[-2])) + + def num_updates(self): + + return len(self.f_ma) + + def __call__(self): + + if self.num_updates() > self.valid_after: + return self.f_ma[-1] + else: + return np.nan diff --git a/new_impl/cv/third_party/nni_new/algorithms/hpo/batch_tuner.py b/new_impl/cv/third_party/nni_new/algorithms/hpo/batch_tuner.py new file mode 100644 index 0000000000000000000000000000000000000000..4f73fce9453161a8f30783f915873ba8e5bb8f31 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/hpo/batch_tuner.py @@ -0,0 +1,131 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +batch_tuner.py including: + class BatchTuner +""" + +import logging + +import nni +from nni.tuner import Tuner + +TYPE = '_type' +CHOICE = 'choice' +VALUE = '_value' + +LOGGER = logging.getLogger('batch_tuner_AutoML') + +class BatchTuner(Tuner): + """ + BatchTuner is tuner will running all the configure that user want to run batchly. + + Examples + -------- + The search space only be accepted like: + + :: + + {'combine_params': + { '_type': 'choice', + '_value': '[{...}, {...}, {...}]', + } + } + + """ + + def __init__(self): + self._count = -1 + self._values = [] + + def is_valid(self, search_space): + """ + Check the search space is valid: only contains 'choice' type + + Parameters + ---------- + search_space : dict + + Returns + ------- + None or list + If valid, return candidate values; else return None. + """ + if not len(search_space) == 1: + raise RuntimeError('BatchTuner only supprt one combined-paramreters key.') + + for param in search_space: + param_type = search_space[param][TYPE] + if not param_type == CHOICE: + raise RuntimeError('BatchTuner only supprt \ + one combined-paramreters type is choice.') + + if isinstance(search_space[param][VALUE], list): + return search_space[param][VALUE] + + raise RuntimeError('The combined-paramreters \ + value in BatchTuner is not a list.') + return None + + def update_search_space(self, search_space): + """Update the search space + + Parameters + ---------- + search_space : dict + """ + self._values = self.is_valid(search_space) + + def generate_parameters(self, parameter_id, **kwargs): + """Returns a dict of trial (hyper-)parameters, as a serializable object. + + Parameters + ---------- + parameter_id : int + + Returns + ------- + dict + A candidate parameter group. + """ + self._count += 1 + if self._count > len(self._values) - 1: + raise nni.NoMoreTrialError('no more parameters now.') + return self._values[self._count] + + def receive_trial_result(self, parameter_id, parameters, value, **kwargs): + pass + + def import_data(self, data): + """Import additional data for tuning + + Parameters + ---------- + data: + a list of dictionarys, each of which has at least two keys, 'parameter' and 'value' + """ + if not self._values: + LOGGER.info("Search space has not been initialized, skip this data import") + return + + self._values = self._values[(self._count+1):] + self._count = -1 + + _completed_num = 0 + for trial_info in data: + LOGGER .info("Importing data, current processing \ + progress %s / %s", _completed_num, len(data)) + # simply validate data format + assert "parameter" in trial_info + _params = trial_info["parameter"] + assert "value" in trial_info + _value = trial_info['value'] + if not _value: + LOGGER.info("Useless trial data, value is %s, skip this trial data.", _value) + continue + _completed_num += 1 + if _params in self._values: + self._values.remove(_params) + LOGGER .info("Successfully import data to batch tuner, \ + total data: %d, imported data: %d.", len(data), _completed_num) diff --git a/new_impl/cv/third_party/nni_new/algorithms/hpo/bohb_advisor/__init__.py b/new_impl/cv/third_party/nni_new/algorithms/hpo/bohb_advisor/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..0ebb442e5cbe24103750f94c0ed931dc93b5803a --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/hpo/bohb_advisor/__init__.py @@ -0,0 +1 @@ +from .bohb_advisor import BOHB, BOHBClassArgsValidator diff --git a/new_impl/cv/third_party/nni_new/algorithms/hpo/bohb_advisor/bohb_advisor.py b/new_impl/cv/third_party/nni_new/algorithms/hpo/bohb_advisor/bohb_advisor.py new file mode 100644 index 0000000000000000000000000000000000000000..73687abc5c71145e95d8623b64ed82284c9063e3 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/hpo/bohb_advisor/bohb_advisor.py @@ -0,0 +1,676 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +''' +bohb_advisor.py +''' + +import sys +import math +import logging +import json_tricks +from schema import Schema, Optional +import ConfigSpace as CS +import ConfigSpace.hyperparameters as CSH + +from nni import ClassArgsValidator +from nni.runtime.protocol import CommandType, send +from nni.runtime.msg_dispatcher_base import MsgDispatcherBase +from nni.utils import OptimizeMode, MetricType, extract_scalar_reward +from nni.runtime.common import multi_phase_enabled + +from .config_generator import CG_BOHB + +logger = logging.getLogger('BOHB_Advisor') + +_next_parameter_id = 0 +_KEY = 'TRIAL_BUDGET' +_epsilon = 1e-6 + + +def create_parameter_id(): + """Create an id + + Returns + ------- + int + parameter id + """ + global _next_parameter_id + _next_parameter_id += 1 + return _next_parameter_id - 1 + + +def create_bracket_parameter_id(brackets_id, brackets_curr_decay, increased_id=-1): + """Create a full id for a specific bracket's hyperparameter configuration + + Parameters + ---------- + brackets_id: int + brackets id + brackets_curr_decay: int + brackets curr decay + increased_id: int + increased id + Returns + ------- + int + params id + """ + if increased_id == -1: + increased_id = str(create_parameter_id()) + params_id = '_'.join([str(brackets_id), + str(brackets_curr_decay), + increased_id]) + return params_id + + +class Bracket: + """ + A bracket in BOHB, all the information of a bracket is managed by + an instance of this class. + + Parameters + ---------- + s: int + The current Successive Halving iteration index. + s_max: int + total number of Successive Halving iterations + eta: float + In each iteration, a complete run of sequential halving is executed. In it, + after evaluating each configuration on the same subset size, only a fraction of + 1/eta of them 'advances' to the next round. + max_budget : float + The largest budget to consider. Needs to be larger than min_budget! + The budgets will be geometrically distributed + :math:`a^2 + b^2 = c^2 \\sim \\eta^k` for :math:`k\\in [0, 1, ... , num\\_subsets - 1]`. + optimize_mode: str + optimize mode, 'maximize' or 'minimize' + """ + def __init__(self, s, s_max, eta, max_budget, optimize_mode): + self.s = s + self.s_max = s_max + self.eta = eta + self.max_budget = max_budget + self.optimize_mode = OptimizeMode(optimize_mode) + + self.n = math.ceil((s_max + 1) * eta**s / (s + 1) - _epsilon) + self.r = max_budget / eta**s + self.i = 0 + self.hyper_configs = [] # [ {id: params}, {}, ... ] + self.configs_perf = [] # [ {id: [seq, acc]}, {}, ... ] + self.num_configs_to_run = [] # [ n, n, n, ... ] + self.num_finished_configs = [] # [ n, n, n, ... ] + self.no_more_trial = False + + def is_completed(self): + """check whether this bracket has sent out all the hyperparameter configurations""" + return self.no_more_trial + + def get_n_r(self): + """return the values of n and r for the next round""" + return math.floor(self.n / self.eta**self.i + _epsilon), math.floor(self.r * self.eta**self.i +_epsilon) + + def increase_i(self): + """i means the ith round. Increase i by 1""" + self.i += 1 + + def set_config_perf(self, i, parameter_id, seq, value): + """update trial's latest result with its sequence number, e.g., epoch number or batch number + + Parameters + ---------- + i: int + the ith round + parameter_id: int + the id of the trial/parameter + seq: int + sequence number, e.g., epoch number or batch number + value: int + latest result with sequence number seq + + Returns + ------- + None + """ + if parameter_id in self.configs_perf[i]: + if self.configs_perf[i][parameter_id][0] < seq: + self.configs_perf[i][parameter_id] = [seq, value] + else: + self.configs_perf[i][parameter_id] = [seq, value] + + def inform_trial_end(self, i): + """If the trial is finished and the corresponding round (i.e., i) has all its trials finished, + it will choose the top k trials for the next round (i.e., i+1) + + Parameters + ---------- + i: int + the ith round + + Returns + ------- + new trial or None: + If we have generated new trials after this trial end, we will return a new trial parameters. + Otherwise, we will return None. + """ + global _KEY + self.num_finished_configs[i] += 1 + logger.debug('bracket id: %d, round: %d %d, finished: %d, all: %d', + self.s, self.i, i, self.num_finished_configs[i], self.num_configs_to_run[i]) + if self.num_finished_configs[i] >= self.num_configs_to_run[i] and self.no_more_trial is False: + # choose candidate configs from finished configs to run in the next round + assert self.i == i + 1 + # finish this bracket + if self.i > self.s: + self.no_more_trial = True + return None + this_round_perf = self.configs_perf[i] + if self.optimize_mode is OptimizeMode.Maximize: + sorted_perf = sorted(this_round_perf.items( + ), key=lambda kv: kv[1][1], reverse=True) # reverse + else: + sorted_perf = sorted( + this_round_perf.items(), key=lambda kv: kv[1][1]) + logger.debug( + 'bracket %s next round %s, sorted hyper configs: %s', self.s, self.i, sorted_perf) + next_n, next_r = self.get_n_r() + logger.debug('bracket %s next round %s, next_n=%d, next_r=%d', + self.s, self.i, next_n, next_r) + hyper_configs = dict() + for k in range(next_n): + params_id = sorted_perf[k][0] + params = self.hyper_configs[i][params_id] + params[_KEY] = next_r # modify r + # generate new id + increased_id = params_id.split('_')[-1] + new_id = create_bracket_parameter_id( + self.s, self.i, increased_id) + hyper_configs[new_id] = params + self._record_hyper_configs(hyper_configs) + return [[key, value] for key, value in hyper_configs.items()] + return None + + def get_hyperparameter_configurations(self, num, r, config_generator): + """generate num hyperparameter configurations from search space using Bayesian optimization + + Parameters + ---------- + num: int + the number of hyperparameter configurations + + Returns + ------- + list + a list of hyperparameter configurations. Format: [[key1, value1], [key2, value2], ...] + """ + global _KEY + assert self.i == 0 + hyperparameter_configs = dict() + for _ in range(num): + params_id = create_bracket_parameter_id(self.s, self.i) + params = config_generator.get_config(r) + params[_KEY] = r + hyperparameter_configs[params_id] = params + self._record_hyper_configs(hyperparameter_configs) + return [[key, value] for key, value in hyperparameter_configs.items()] + + def _record_hyper_configs(self, hyper_configs): + """after generating one round of hyperconfigs, this function records the generated hyperconfigs, + creates a dict to record the performance when those hyperconifgs are running, set the number of finished configs + in this round to be 0, and increase the round number. + + Parameters + ---------- + hyper_configs: list + the generated hyperconfigs + """ + self.hyper_configs.append(hyper_configs) + self.configs_perf.append(dict()) + self.num_finished_configs.append(0) + self.num_configs_to_run.append(len(hyper_configs)) + self.increase_i() + +class BOHBClassArgsValidator(ClassArgsValidator): + def validate_class_args(self, **kwargs): + Schema({ + 'optimize_mode': self.choices('optimize_mode', 'maximize', 'minimize'), + Optional('min_budget'): self.range('min_budget', int, 0, 9999), + Optional('max_budget'): self.range('max_budget', int, 0, 9999), + Optional('eta'): self.range('eta', int, 0, 9999), + Optional('min_points_in_model'): self.range('min_points_in_model', int, 0, 9999), + Optional('top_n_percent'): self.range('top_n_percent', int, 1, 99), + Optional('num_samples'): self.range('num_samples', int, 1, 9999), + Optional('random_fraction'): self.range('random_fraction', float, 0, 9999), + Optional('bandwidth_factor'): self.range('bandwidth_factor', float, 0, 9999), + Optional('min_bandwidth'): self.range('min_bandwidth', float, 0, 9999), + }).validate(kwargs) + +class BOHB(MsgDispatcherBase): + """ + BOHB performs robust and efficient hyperparameter optimization + at scale by combining the speed of Hyperband searches with the + guidance and guarantees of convergence of Bayesian Optimization. + Instead of sampling new configurations at random, BOHB uses + kernel density estimators to select promising candidates. + + Parameters + ---------- + optimize_mode: str + optimize mode, 'maximize' or 'minimize' + min_budget: float + The smallest budget to consider. Needs to be positive! + max_budget: float + The largest budget to consider. Needs to be larger than min_budget! + The budgets will be geometrically distributed + :math:`a^2 + b^2 = c^2 \\sim \\eta^k` for :math:`k\\in [0, 1, ... , num\\_subsets - 1]`. + eta: int + In each iteration, a complete run of sequential halving is executed. In it, + after evaluating each configuration on the same subset size, only a fraction of + 1/eta of them 'advances' to the next round. + Must be greater or equal to 2. + min_points_in_model: int + number of observations to start building a KDE. Default 'None' means + dim+1, the bare minimum. + top_n_percent: int + percentage ( between 1 and 99, default 15) of the observations that are considered good. + num_samples: int + number of samples to optimize EI (default 64) + random_fraction: float + fraction of purely random configurations that are sampled from the + prior without the model. + bandwidth_factor: float + to encourage diversity, the points proposed to optimize EI, are sampled + from a 'widened' KDE where the bandwidth is multiplied by this factor (default: 3) + min_bandwidth: float + to keep diversity, even when all (good) samples have the same value for one of the parameters, + a minimum bandwidth (Default: 1e-3) is used instead of zero. + """ + + def __init__(self, + optimize_mode='maximize', + min_budget=1, + max_budget=3, + eta=3, + min_points_in_model=None, + top_n_percent=15, + num_samples=64, + random_fraction=1/3, + bandwidth_factor=3, + min_bandwidth=1e-3): + super(BOHB, self).__init__() + self.optimize_mode = OptimizeMode(optimize_mode) + self.min_budget = min_budget + self.max_budget = max_budget + self.eta = eta + self.min_points_in_model = min_points_in_model + self.top_n_percent = top_n_percent + self.num_samples = num_samples + self.random_fraction = random_fraction + self.bandwidth_factor = bandwidth_factor + self.min_bandwidth = min_bandwidth + + # all the configs waiting for run + self.generated_hyper_configs = [] + # all the completed configs + self.completed_hyper_configs = [] + + self.s_max = math.floor( + math.log(self.max_budget / self.min_budget, self.eta) + _epsilon) + # current bracket(s) number + self.curr_s = self.s_max + # In this case, tuner increases self.credit to issue a trial config sometime later. + self.credit = 0 + self.brackets = dict() + self.search_space = None + # [key, value] = [parameter_id, parameter] + self.parameters = dict() + + # config generator + self.cg = None + + # record the latest parameter_id of the trial job trial_job_id. + # if there is no running parameter_id, self.job_id_para_id_map[trial_job_id] == None + # new trial job is added to this dict and finished trial job is removed from it. + self.job_id_para_id_map = dict() + # record the unsatisfied parameter request from trial jobs + self.unsatisfied_jobs = [] + + def handle_initialize(self, data): + """Initialize Tuner, including creating Bayesian optimization-based parametric models + and search space formations + + Parameters + ---------- + data: search space + search space of this experiment + + Raises + ------ + ValueError + Error: Search space is None + """ + logger.info('start to handle_initialize') + # convert search space jason to ConfigSpace + self.handle_update_search_space(data) + + # generate BOHB config_generator using Bayesian optimization + if self.search_space: + self.cg = CG_BOHB(configspace=self.search_space, + min_points_in_model=self.min_points_in_model, + top_n_percent=self.top_n_percent, + num_samples=self.num_samples, + random_fraction=self.random_fraction, + bandwidth_factor=self.bandwidth_factor, + min_bandwidth=self.min_bandwidth) + else: + raise ValueError('Error: Search space is None') + # generate first brackets + self.generate_new_bracket() + send(CommandType.Initialized, '') + + def generate_new_bracket(self): + """generate a new bracket""" + logger.debug( + 'start to create a new SuccessiveHalving iteration, self.curr_s=%d', self.curr_s) + if self.curr_s < 0: + logger.info("s < 0, Finish this round of Hyperband in BOHB. Generate new round") + self.curr_s = self.s_max + self.brackets[self.curr_s] = Bracket( + s=self.curr_s, s_max=self.s_max, eta=self.eta, + max_budget=self.max_budget, optimize_mode=self.optimize_mode + ) + next_n, next_r = self.brackets[self.curr_s].get_n_r() + logger.debug( + 'new SuccessiveHalving iteration, next_n=%d, next_r=%d', next_n, next_r) + # rewrite with TPE + generated_hyper_configs = self.brackets[self.curr_s].get_hyperparameter_configurations( + next_n, next_r, self.cg) + self.generated_hyper_configs = generated_hyper_configs.copy() + + def handle_request_trial_jobs(self, data): + """recerive the number of request and generate trials + + Parameters + ---------- + data: int + number of trial jobs that nni manager ask to generate + """ + # Receive new request + self.credit += data + + for _ in range(self.credit): + self._request_one_trial_job() + + def _get_one_trial_job(self): + """get one trial job, i.e., one hyperparameter configuration. + + If this function is called, Command will be sent by BOHB: + a. If there is a parameter need to run, will return "NewTrialJob" with a dict: + { + 'parameter_id': id of new hyperparameter + 'parameter_source': 'algorithm' + 'parameters': value of new hyperparameter + } + b. If BOHB don't have parameter waiting, will return "NoMoreTrialJobs" with + { + 'parameter_id': '-1_0_0', + 'parameter_source': 'algorithm', + 'parameters': '' + } + """ + if not self.generated_hyper_configs: + ret = { + 'parameter_id': '-1_0_0', + 'parameter_source': 'algorithm', + 'parameters': '' + } + send(CommandType.NoMoreTrialJobs, json_tricks.dumps(ret)) + return None + assert self.generated_hyper_configs + params = self.generated_hyper_configs.pop(0) + ret = { + 'parameter_id': params[0], + 'parameter_source': 'algorithm', + 'parameters': params[1] + } + self.parameters[params[0]] = params[1] + return ret + + def _request_one_trial_job(self): + """get one trial job, i.e., one hyperparameter configuration. + + If this function is called, Command will be sent by BOHB: + a. If there is a parameter need to run, will return "NewTrialJob" with a dict: + { + 'parameter_id': id of new hyperparameter + 'parameter_source': 'algorithm' + 'parameters': value of new hyperparameter + } + b. If BOHB don't have parameter waiting, will return "NoMoreTrialJobs" with + { + 'parameter_id': '-1_0_0', + 'parameter_source': 'algorithm', + 'parameters': '' + } + """ + ret = self._get_one_trial_job() + if ret is not None: + send(CommandType.NewTrialJob, json_tricks.dumps(ret)) + self.credit -= 1 + + def handle_update_search_space(self, data): + """change json format to ConfigSpace format dict -> configspace + + Parameters + ---------- + data: JSON object + search space of this experiment + """ + search_space = data + cs = CS.ConfigurationSpace() + for var in search_space: + _type = str(search_space[var]["_type"]) + if _type == 'choice': + cs.add_hyperparameter(CSH.CategoricalHyperparameter( + var, choices=search_space[var]["_value"])) + elif _type == 'randint': + cs.add_hyperparameter(CSH.UniformIntegerHyperparameter( + var, lower=search_space[var]["_value"][0], upper=search_space[var]["_value"][1] - 1)) + elif _type == 'uniform': + cs.add_hyperparameter(CSH.UniformFloatHyperparameter( + var, lower=search_space[var]["_value"][0], upper=search_space[var]["_value"][1])) + elif _type == 'quniform': + cs.add_hyperparameter(CSH.UniformFloatHyperparameter( + var, lower=search_space[var]["_value"][0], upper=search_space[var]["_value"][1], + q=search_space[var]["_value"][2])) + elif _type == 'loguniform': + cs.add_hyperparameter(CSH.UniformFloatHyperparameter( + var, lower=search_space[var]["_value"][0], upper=search_space[var]["_value"][1], + log=True)) + elif _type == 'qloguniform': + cs.add_hyperparameter(CSH.UniformFloatHyperparameter( + var, lower=search_space[var]["_value"][0], upper=search_space[var]["_value"][1], + q=search_space[var]["_value"][2], log=True)) + elif _type == 'normal': + cs.add_hyperparameter(CSH.NormalFloatHyperparameter( + var, mu=search_space[var]["_value"][1], sigma=search_space[var]["_value"][2])) + elif _type == 'qnormal': + cs.add_hyperparameter(CSH.NormalFloatHyperparameter( + var, mu=search_space[var]["_value"][1], sigma=search_space[var]["_value"][2], + q=search_space[var]["_value"][3])) + elif _type == 'lognormal': + cs.add_hyperparameter(CSH.NormalFloatHyperparameter( + var, mu=search_space[var]["_value"][1], sigma=search_space[var]["_value"][2], + log=True)) + elif _type == 'qlognormal': + cs.add_hyperparameter(CSH.NormalFloatHyperparameter( + var, mu=search_space[var]["_value"][1], sigma=search_space[var]["_value"][2], + q=search_space[var]["_value"][3], log=True)) + else: + raise ValueError( + 'unrecognized type in search_space, type is {}'.format(_type)) + + self.search_space = cs + + def handle_trial_end(self, data): + """receive the information of trial end and generate next configuaration. + + Parameters + ---------- + data: dict() + it has three keys: trial_job_id, event, hyper_params + trial_job_id: the id generated by training service + event: the job's state + hyper_params: the hyperparameters (a string) generated and returned by tuner + """ + logger.debug('Tuner handle trial end, result is %s', data) + hyper_params = json_tricks.loads(data['hyper_params']) + self._handle_trial_end(hyper_params['parameter_id']) + if data['trial_job_id'] in self.job_id_para_id_map: + del self.job_id_para_id_map[data['trial_job_id']] + + def _send_new_trial(self): + while self.unsatisfied_jobs: + ret = self._get_one_trial_job() + if ret is None: + break + one_unsatisfied = self.unsatisfied_jobs.pop(0) + ret['trial_job_id'] = one_unsatisfied['trial_job_id'] + ret['parameter_index'] = one_unsatisfied['parameter_index'] + # update parameter_id in self.job_id_para_id_map + self.job_id_para_id_map[ret['trial_job_id']] = ret['parameter_id'] + send(CommandType.SendTrialJobParameter, json_tricks.dumps(ret)) + for _ in range(self.credit): + self._request_one_trial_job() + + def _handle_trial_end(self, parameter_id): + s, i, _ = parameter_id.split('_') + hyper_configs = self.brackets[int(s)].inform_trial_end(int(i)) + + if hyper_configs is not None: + logger.debug( + 'bracket %s next round %s, hyper_configs: %s', s, i, hyper_configs) + self.generated_hyper_configs = self.generated_hyper_configs + hyper_configs + # Finish this bracket and generate a new bracket + elif self.brackets[int(s)].no_more_trial: + self.curr_s -= 1 + self.generate_new_bracket() + self._send_new_trial() + + def handle_report_metric_data(self, data): + """reveice the metric data and update Bayesian optimization with final result + + Parameters + ---------- + data: + it is an object which has keys 'parameter_id', 'value', 'trial_job_id', 'type', 'sequence'. + + Raises + ------ + ValueError + Data type not supported + """ + logger.debug('handle report metric data = %s', data) + if 'value' in data: + data['value'] = json_tricks.loads(data['value']) + if data['type'] == MetricType.REQUEST_PARAMETER: + assert multi_phase_enabled() + assert data['trial_job_id'] is not None + assert data['parameter_index'] is not None + assert data['trial_job_id'] in self.job_id_para_id_map + self._handle_trial_end(self.job_id_para_id_map[data['trial_job_id']]) + ret = self._get_one_trial_job() + if ret is None: + self.unsatisfied_jobs.append({'trial_job_id': data['trial_job_id'], 'parameter_index': data['parameter_index']}) + else: + ret['trial_job_id'] = data['trial_job_id'] + ret['parameter_index'] = data['parameter_index'] + # update parameter_id in self.job_id_para_id_map + self.job_id_para_id_map[data['trial_job_id']] = ret['parameter_id'] + send(CommandType.SendTrialJobParameter, json_tricks.dumps(ret)) + else: + assert 'value' in data + value = extract_scalar_reward(data['value']) + if self.optimize_mode is OptimizeMode.Maximize: + reward = -value + else: + reward = value + assert 'parameter_id' in data + s, i, _ = data['parameter_id'].split('_') + logger.debug('bracket id = %s, metrics value = %s, type = %s', s, value, data['type']) + s = int(s) + + # add to self.job_id_para_id_map here, + # because when the first parameter_id is created, trial_job_id is not known yet. + if data['trial_job_id'] in self.job_id_para_id_map: + assert self.job_id_para_id_map[data['trial_job_id']] == data['parameter_id'] + else: + self.job_id_para_id_map[data['trial_job_id']] = data['parameter_id'] + + assert 'type' in data + if data['type'] == MetricType.FINAL: + # and PERIODICAL metric are independent, thus, not comparable. + assert 'sequence' in data + self.brackets[s].set_config_perf( + int(i), data['parameter_id'], sys.maxsize, value) + self.completed_hyper_configs.append(data) + + _parameters = self.parameters[data['parameter_id']] + _parameters.pop(_KEY) + # update BO with loss, max_s budget, hyperparameters + self.cg.new_result(loss=reward, budget=data['sequence'], parameters=_parameters, update_model=True) + elif data['type'] == MetricType.PERIODICAL: + self.brackets[s].set_config_perf( + int(i), data['parameter_id'], data['sequence'], value) + else: + raise ValueError( + 'Data type not supported: {}'.format(data['type'])) + + def handle_add_customized_trial(self, data): + pass + + def handle_import_data(self, data): + """Import additional data for tuning + + Parameters + ---------- + data: + a list of dictionarys, each of which has at least two keys, 'parameter' and 'value' + + Raises + ------ + AssertionError + data doesn't have required key 'parameter' and 'value' + """ + for entry in data: + entry['value'] = json_tricks.loads(entry['value']) + _completed_num = 0 + for trial_info in data: + logger.info("Importing data, current processing progress %s / %s", _completed_num, len(data)) + _completed_num += 1 + assert "parameter" in trial_info + _params = trial_info["parameter"] + assert "value" in trial_info + _value = trial_info['value'] + if not _value: + logger.info("Useless trial data, value is %s, skip this trial data.", _value) + continue + _value = extract_scalar_reward(_value) + budget_exist_flag = False + barely_params = dict() + for keys in _params: + if keys == _KEY: + _budget = _params[keys] + budget_exist_flag = True + else: + barely_params[keys] = _params[keys] + if not budget_exist_flag: + _budget = self.max_budget + logger.info("Set \"TRIAL_BUDGET\" value to %s (max budget)", self.max_budget) + if self.optimize_mode is OptimizeMode.Maximize: + reward = -_value + else: + reward = _value + self.cg.new_result(loss=reward, budget=_budget, parameters=barely_params, update_model=True) + logger.info("Successfully import tuning data to BOHB advisor.") diff --git a/new_impl/cv/third_party/nni_new/algorithms/hpo/bohb_advisor/config_generator.py b/new_impl/cv/third_party/nni_new/algorithms/hpo/bohb_advisor/config_generator.py new file mode 100644 index 0000000000000000000000000000000000000000..c6a13c6b35ba3b2db9bbbf8618d3b9a3d9240909 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/hpo/bohb_advisor/config_generator.py @@ -0,0 +1,344 @@ +# BSD 3-Clause License +# Copyright (c) 2017-2018, ML4AAD +# All rights reserved. + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. + +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. + +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import logging + +import ConfigSpace +import ConfigSpace.hyperparameters +import ConfigSpace.util +import numpy as np +import scipy.stats as sps +import statsmodels.api as sm + +logger = logging.getLogger('BOHB_Advisor') + +class CG_BOHB: + def __init__(self, configspace, min_points_in_model=None, + top_n_percent=15, num_samples=64, random_fraction=1/3, + bandwidth_factor=3, min_bandwidth=1e-3): + """Fits for each given budget a kernel density estimator on the best N percent of the + evaluated configurations on this budget. + + + Parameters: + ----------- + configspace: ConfigSpace + Configuration space object + top_n_percent: int + Determines the percentile of configurations that will be used as training data + for the kernel density estimator, e.g if set to 10 the 10% best configurations will be considered + for training. + min_points_in_model: int + minimum number of datapoints needed to fit a model + num_samples: int + number of samples drawn to optimize EI via sampling + random_fraction: float + fraction of random configurations returned + bandwidth_factor: float + widens the bandwidth for contiuous parameters for proposed points to optimize EI + min_bandwidth: float + to keep diversity, even when all (good) samples have the same value for one of the parameters, + a minimum bandwidth (Default: 1e-3) is used instead of zero. + """ + self.top_n_percent = top_n_percent + self.configspace = configspace + self.bw_factor = bandwidth_factor + self.min_bandwidth = min_bandwidth + + self.min_points_in_model = min_points_in_model + if min_points_in_model is None: + self.min_points_in_model = len(self.configspace.get_hyperparameters())+1 + + if self.min_points_in_model < len(self.configspace.get_hyperparameters())+1: + logger.warning('Invalid min_points_in_model value. Setting it to %i', len(self.configspace.get_hyperparameters()) + 1) + self.min_points_in_model = len(self.configspace.get_hyperparameters()) + 1 + + self.num_samples = num_samples + self.random_fraction = random_fraction + + hps = self.configspace.get_hyperparameters() + + self.kde_vartypes = "" + self.vartypes = [] + + for h in hps: + if hasattr(h, 'choices'): + self.kde_vartypes += 'u' + self.vartypes += [len(h.choices)] + else: + self.kde_vartypes += 'c' + self.vartypes += [0] + + self.vartypes = np.array(self.vartypes, dtype=int) + + # store precomputed probs for the categorical parameters + self.cat_probs = [] + + self.configs = dict() + self.losses = dict() + self.good_config_rankings = dict() + self.kde_models = dict() + + def largest_budget_with_model(self): + if not self.kde_models: + return -float('inf') + return max(self.kde_models.keys()) + + def sample_from_largest_budget(self, info_dict): + """We opted for a single multidimensional KDE compared to the + hierarchy of one-dimensional KDEs used in TPE. The dimensional is + seperated by budget. This function sample a configuration from + largest budget. Firstly we sample "num_samples" configurations, + then prefer one with the largest l(x)/g(x). + + Parameters: + ----------- + info_dict: dict + record the information of this configuration + + Returns + ------- + dict: + new configuration named sample + dict: + info_dict, record the information of this configuration + """ + best = np.inf + best_vector = None + + budget = max(self.kde_models.keys()) + + l = self.kde_models[budget]['good'].pdf + g = self.kde_models[budget]['bad'].pdf + + minimize_me = lambda x: max(1e-32, g(x))/max(l(x), 1e-32) + + kde_good = self.kde_models[budget]['good'] + kde_bad = self.kde_models[budget]['bad'] + + for i in range(self.num_samples): + idx = np.random.randint(0, len(kde_good.data)) + datum = kde_good.data[idx] + vector = [] + + for m, bw, t in zip(datum, kde_good.bw, self.vartypes): + + bw = max(bw, self.min_bandwidth) + if t == 0: + bw = self.bw_factor*bw + vector.append(sps.truncnorm.rvs(-m/bw, (1-m)/bw, loc=m, scale=bw)) + else: + if np.random.rand() < (1-bw): + vector.append(int(m)) + else: + vector.append(np.random.randint(t)) + val = minimize_me(vector) + + if not np.isfinite(val): + logger.warning('sampled vector: %s has EI value %s', vector, val) + logger.warning("data in the KDEs:\n%s\n%s", kde_good.data, kde_bad.data) + logger.warning("bandwidth of the KDEs:\n%s\n%s", kde_good.bw, kde_bad.bw) + logger.warning("l(x) = %s", l(vector)) + logger.warning("g(x) = %s", g(vector)) + + # right now, this happens because a KDE does not contain all values for a categorical parameter + # this cannot be fixed with the statsmodels KDE, so for now, we are just going to evaluate this one + # if the good_kde has a finite value, i.e. there is no config with that value in the bad kde, + # so it shouldn't be terrible. + if np.isfinite(l(vector)): + best_vector = vector + break + + if val < best: + best = val + best_vector = vector + + if best_vector is None: + logger.debug("Sampling based optimization with %i samples failed -> using random configuration", self.num_samples) + sample = self.configspace.sample_configuration().get_dictionary() + info_dict['model_based_pick'] = False + + else: + logger.debug('best_vector: %s, %s, %s, %s', best_vector, best, l(best_vector), g(best_vector)) + for i, _ in enumerate(best_vector): + hp = self.configspace.get_hyperparameter(self.configspace.get_hyperparameter_by_idx(i)) + if isinstance(hp, ConfigSpace.hyperparameters.CategoricalHyperparameter): + best_vector[i] = int(np.rint(best_vector[i])) + sample = ConfigSpace.Configuration(self.configspace, vector=best_vector).get_dictionary() + + sample = ConfigSpace.util.deactivate_inactive_hyperparameters( + configuration_space=self.configspace, + configuration=sample) + info_dict['model_based_pick'] = True + + return sample, info_dict + + def get_config(self, budget): + """Function to sample a new configuration + This function is called inside BOHB to query a new configuration + + Parameters: + ----------- + budget: float + the budget for which this configuration is scheduled + + Returns + ------- + config + return a valid configuration with parameters and budget + """ + logger.debug('start sampling a new configuration.') + sample = None + info_dict = {} + + # If no model is available, sample from prior + # also mix in a fraction of random configs + if not self.kde_models.keys() or np.random.rand() < self.random_fraction: + sample = self.configspace.sample_configuration() + info_dict['model_based_pick'] = False + + if sample is None: + sample, info_dict = self.sample_from_largest_budget(info_dict) + + sample = ConfigSpace.util.deactivate_inactive_hyperparameters( + configuration_space=self.configspace, + configuration=sample.get_dictionary() + ).get_dictionary() + + logger.debug('done sampling a new configuration.') + sample['TRIAL_BUDGET'] = budget + return sample + + def impute_conditional_data(self, array): + return_array = np.zeros(array.shape) + for i in range(array.shape[0]): + datum = np.copy(array[i]) + nan_indices = np.argwhere(np.isnan(datum)).flatten() + while np.any(nan_indices): + nan_idx = nan_indices[0] + valid_indices = np.argwhere(np.isfinite(array[:, nan_idx])).flatten() + if valid_indices: + # pick one of them at random and overwrite all NaN values + row_idx = np.random.choice(valid_indices) + datum[nan_indices] = array[row_idx, nan_indices] + else: + # no good point in the data has this value activated, so fill it with a valid but random value + t = self.vartypes[nan_idx] + if t == 0: + datum[nan_idx] = np.random.rand() + else: + datum[nan_idx] = np.random.randint(t) + nan_indices = np.argwhere(np.isnan(datum)).flatten() + return_array[i, :] = datum + return return_array + + def new_result(self, loss, budget, parameters, update_model=True): + """ + Function to register finished runs. Every time a run has finished, this function should be called + to register it with the loss. + + Parameters: + ----------- + loss: float + the loss of the parameters + budget: float + the budget of the parameters + parameters: dict + the parameters of this trial + update_model: bool + whether use this parameter to update BP model + + Returns + ------- + None + """ + if loss is None: + # One could skip crashed results, but we decided + # assign a +inf loss and count them as bad configurations + loss = np.inf + + if budget not in self.configs.keys(): + self.configs[budget] = [] + self.losses[budget] = [] + + # skip model building if we already have a bigger model + if max(list(self.kde_models.keys()) + [-np.inf]) > budget: + return + + # We want to get a numerical representation of the configuration in the original space + conf = ConfigSpace.Configuration(self.configspace, parameters) + self.configs[budget].append(conf.get_array()) + self.losses[budget].append(loss) + + # skip model building: + # a) if not enough points are available + if len(self.configs[budget]) <= self.min_points_in_model - 1: + logger.debug("Only %i run(s) for budget %f available, need more than %s \ + -> can't build model!", len(self.configs[budget]), budget, self.min_points_in_model+1) + return + # b) during warnm starting when we feed previous results in and only update once + if not update_model: + return + + train_configs = np.array(self.configs[budget]) + train_losses = np.array(self.losses[budget]) + + n_good = max(self.min_points_in_model, (self.top_n_percent * train_configs.shape[0])//100) + n_bad = max(self.min_points_in_model, ((100-self.top_n_percent)*train_configs.shape[0])//100) + + # Refit KDE for the current budget + idx = np.argsort(train_losses) + + train_data_good = self.impute_conditional_data(train_configs[idx[:n_good]]) + train_data_bad = self.impute_conditional_data(train_configs[idx[n_good:n_good+n_bad]]) + + if train_data_good.shape[0] <= train_data_good.shape[1]: + return + if train_data_bad.shape[0] <= train_data_bad.shape[1]: + return + + #more expensive crossvalidation method + #bw_estimation = 'cv_ls' + # quick rule of thumb + bw_estimation = 'normal_reference' + + bad_kde = sm.nonparametric.KDEMultivariate(data=train_data_bad, var_type=self.kde_vartypes, bw=bw_estimation) + good_kde = sm.nonparametric.KDEMultivariate(data=train_data_good, var_type=self.kde_vartypes, bw=bw_estimation) + + bad_kde.bw = np.clip(bad_kde.bw, self.min_bandwidth, None) + good_kde.bw = np.clip(good_kde.bw, self.min_bandwidth, None) + + self.kde_models[budget] = { + 'good': good_kde, + 'bad' : bad_kde + } + + # update probs for the categorical parameters for later sampling + logger.debug('done building a new model for budget %f based on %i/%i split\nBest loss for this budget:%f\n', + budget, n_good, n_bad, np.min(train_losses)) diff --git a/new_impl/cv/third_party/nni_new/algorithms/hpo/curvefitting_assessor/__init__.py b/new_impl/cv/third_party/nni_new/algorithms/hpo/curvefitting_assessor/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..f3ed4d7a513b181e43e8767d28ead3cbc20af139 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/hpo/curvefitting_assessor/__init__.py @@ -0,0 +1,4 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .curvefitting_assessor import CurvefittingAssessor, CurvefittingClassArgsValidator diff --git a/new_impl/cv/third_party/nni_new/algorithms/hpo/curvefitting_assessor/curvefitting_assessor.py b/new_impl/cv/third_party/nni_new/algorithms/hpo/curvefitting_assessor/curvefitting_assessor.py new file mode 100644 index 0000000000000000000000000000000000000000..885886e89b83712e7eded05fd89598a9e2232b5a --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/hpo/curvefitting_assessor/curvefitting_assessor.py @@ -0,0 +1,135 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +import datetime +from schema import Schema, Optional + +from nni import ClassArgsValidator +from nni.assessor import Assessor, AssessResult +from nni.utils import extract_scalar_history +from .model_factory import CurveModel + +logger = logging.getLogger('curvefitting_Assessor') + +class CurvefittingClassArgsValidator(ClassArgsValidator): + def validate_class_args(self, **kwargs): + Schema({ + 'epoch_num': self.range('epoch_num', int, 0, 9999), + Optional('start_step'): self.range('start_step', int, 0, 9999), + Optional('threshold'): self.range('threshold', float, 0, 9999), + Optional('gap'): self.range('gap', int, 1, 9999), + }).validate(kwargs) + +class CurvefittingAssessor(Assessor): + """CurvefittingAssessor uses learning curve fitting algorithm to predict the learning curve performance in the future. + It stops a pending trial X at step S if the trial's forecast result at target step is convergence and lower than the + best performance in the history. + + Parameters + ---------- + epoch_num : int + The total number of epoch + start_step : int + only after receiving start_step number of reported intermediate results + threshold : float + The threshold that we decide to early stop the worse performance curve. + """ + + def __init__(self, epoch_num=20, start_step=6, threshold=0.95, gap=1): + if start_step <= 0: + logger.warning('It\'s recommended to set start_step to a positive number') + # Record the target position we predict + self.target_pos = epoch_num + # Start forecasting when historical data reaches start step + self.start_step = start_step + # Record the compared threshold + self.threshold = threshold + # Record the number of gap + self.gap = gap + # Record the number of intermediate result in the lastest judgment + self.last_judgment_num = dict() + # Record the best performance + self.set_best_performance = False + self.completed_best_performance = None + self.trial_history = [] + logger.info('Successfully initials the curvefitting assessor') + + def trial_end(self, trial_job_id, success): + """update the best performance of completed trial job + + Parameters + ---------- + trial_job_id : int + trial job id + success : bool + True if succssfully finish the experiment, False otherwise + """ + if success: + if self.set_best_performance: + self.completed_best_performance = max(self.completed_best_performance, self.trial_history[-1]) + else: + self.set_best_performance = True + self.completed_best_performance = self.trial_history[-1] + logger.info('Updated complted best performance, trial job id: %s', trial_job_id) + else: + logger.info('No need to update, trial job id: %s', trial_job_id) + + def assess_trial(self, trial_job_id, trial_history): + """assess whether a trial should be early stop by curve fitting algorithm + + Parameters + ---------- + trial_job_id : int + trial job id + trial_history : list + The history performance matrix of each trial + + Returns + ------- + bool + AssessResult.Good or AssessResult.Bad + + Raises + ------ + Exception + unrecognize exception in curvefitting_assessor + """ + scalar_trial_history = extract_scalar_history(trial_history) + self.trial_history = scalar_trial_history + if not self.set_best_performance: + return AssessResult.Good + curr_step = len(scalar_trial_history) + if curr_step < self.start_step: + return AssessResult.Good + + if trial_job_id in self.last_judgment_num.keys() and curr_step - self.last_judgment_num[trial_job_id] < self.gap: + return AssessResult.Good + self.last_judgment_num[trial_job_id] = curr_step + + try: + start_time = datetime.datetime.now() + # Predict the final result + curvemodel = CurveModel(self.target_pos) + predict_y = curvemodel.predict(scalar_trial_history) + log_message = "Prediction done. Trial job id = {}, Predict value = {}".format(trial_job_id, predict_y) + if predict_y is None: + logger.info('%s, wait for more information to predict precisely', log_message) + return AssessResult.Good + else: + logger.info(log_message) + standard_performance = self.completed_best_performance * self.threshold + + end_time = datetime.datetime.now() + if (end_time - start_time).seconds > 60: + logger.warning( + 'Curve Fitting Assessor Runtime Exceeds 60s, Trial Id = %s Trial History = %s', + trial_job_id, self.trial_history + ) + + if predict_y > standard_performance: + return AssessResult.Good + return AssessResult.Bad + + except Exception as exception: + logger.exception('unrecognize exception in curvefitting_assessor %s', exception) diff --git a/new_impl/cv/third_party/nni_new/algorithms/hpo/curvefitting_assessor/curvefunctions.py b/new_impl/cv/third_party/nni_new/algorithms/hpo/curvefitting_assessor/curvefunctions.py new file mode 100644 index 0000000000000000000000000000000000000000..c7bfc3b9d817df38689d24ff317e1543ac0f88e9 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/hpo/curvefitting_assessor/curvefunctions.py @@ -0,0 +1,296 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +A family of functions used by CurvefittingAssessor +""" + +import numpy as np + +all_models = {} +model_para = {} +model_para_num = {} + +curve_combination_models = ['vap', 'pow3', 'linear', 'logx_linear', 'dr_hill_zero_background', 'log_power', 'pow4', 'mmf', + 'exp4', 'ilog2', 'weibull', 'janoschek'] + + +def vap(x, a, b, c): + """Vapor pressure model + + Parameters + ---------- + x : int + a : float + b : float + c : float + + Returns + ------- + float + np.exp(a+b/x+c*np.log(x)) + """ + return np.exp(a+b/x+c*np.log(x)) + + +all_models['vap'] = vap +model_para['vap'] = [-0.622028, -0.470050, 0.042322] +model_para_num['vap'] = 3 + + +def pow3(x, c, a, alpha): + """pow3 + + Parameters + ---------- + x : int + c : float + a : float + alpha : float + + Returns + ------- + float + c - a * x**(-alpha) + """ + return c - a * x**(-alpha) + + +all_models['pow3'] = pow3 +model_para['pow3'] = [0.84, 0.52, 0.01] +model_para_num['pow3'] = 3 + + +def linear(x, a, b): + """linear + + Parameters + ---------- + x : int + a : float + b : float + + Returns + ------- + float + a*x + b + """ + return a*x + b + + +all_models['linear'] = linear +model_para['linear'] = [1., 0] +model_para_num['linear'] = 2 + + +def logx_linear(x, a, b): + """logx linear + + Parameters + ---------- + x : int + a : float + b : float + + Returns + ------- + float + a * np.log(x) + b + """ + x = np.log(x) + return a*x + b + + +all_models['logx_linear'] = logx_linear +model_para['logx_linear'] = [0.378106, 0.046506] +model_para_num['logx_linear'] = 2 + + +def dr_hill_zero_background(x, theta, eta, kappa): + """dr hill zero background + + Parameters + ---------- + x : int + theta : float + eta : float + kappa : float + + Returns + ------- + float + (theta* x**eta) / (kappa**eta + x**eta) + """ + return (theta * x**eta) / (kappa**eta + x**eta) + + +all_models['dr_hill_zero_background'] = dr_hill_zero_background +model_para['dr_hill_zero_background'] = [0.772320, 0.586449, 2.460843] +model_para_num['dr_hill_zero_background'] = 3 + + +def log_power(x, a, b, c): + """"logistic power + + Parameters + ---------- + x : int + a : float + b : float + c : float + + Returns + ------- + float + a/(1.+(x/np.exp(b))**c) + """ + return a/(1.+(x/np.exp(b))**c) + + +all_models['log_power'] = log_power +model_para['log_power'] = [0.77, 2.98, -0.51] +model_para_num['log_power'] = 3 + + +def pow4(x, alpha, a, b, c): + """pow4 + + Parameters + ---------- + x : int + alpha : float + a : float + b : float + c : float + + Returns + ------- + float + c - (a*x+b)**-alpha + """ + return c - (a*x+b)**-alpha + + +all_models['pow4'] = pow4 +model_para['pow4'] = [0.1, 200, 0., 0.8] +model_para_num['pow4'] = 4 + + +def mmf(x, alpha, beta, kappa, delta): + """Morgan-Mercer-Flodin + http://www.pisces-conservation.com/growthhelp/index.html?morgan_mercer_floden.htm + + Parameters + ---------- + x : int + alpha : float + beta : float + kappa : float + delta : float + + Returns + ------- + float + alpha - (alpha - beta) / (1. + (kappa * x)**delta) + """ + return alpha - (alpha - beta) / (1. + (kappa * x)**delta) + + +all_models['mmf'] = mmf +model_para['mmf'] = [0.7, 0.1, 0.01, 5] +model_para_num['mmf'] = 4 + + +def exp4(x, c, a, b, alpha): + """exp4 + + Parameters + ---------- + x : int + c : float + a : float + b : float + alpha : float + + Returns + ------- + float + c - np.exp(-a*(x**alpha)+b) + """ + return c - np.exp(-a*(x**alpha)+b) + + +all_models['exp4'] = exp4 +model_para['exp4'] = [0.7, 0.8, -0.8, 0.3] +model_para_num['exp4'] = 4 + + +def ilog2(x, c, a): + """ilog2 + + Parameters + ---------- + x : int + c : float + a : float + + Returns + ------- + float + c - a / np.log(x) + """ + return c - a / np.log(x) + + +all_models['ilog2'] = ilog2 +model_para['ilog2'] = [0.78, 0.43] +model_para_num['ilog2'] = 2 + + +def weibull(x, alpha, beta, kappa, delta): + """Weibull model + http://www.pisces-conservation.com/growthhelp/index.html?morgan_mercer_floden.htm + + Parameters + ---------- + x : int + alpha : float + beta : float + kappa : float + delta : float + + Returns + ------- + float + alpha - (alpha - beta) * np.exp(-(kappa * x)**delta) + """ + return alpha - (alpha - beta) * np.exp(-(kappa * x)**delta) + + +all_models['weibull'] = weibull +model_para['weibull'] = [0.7, 0.1, 0.01, 1] +model_para_num['weibull'] = 4 + + +def janoschek(x, a, beta, k, delta): + """http://www.pisces-conservation.com/growthhelp/janoschek.htm + + Parameters + ---------- + x : int + a : float + beta : float + k : float + delta : float + + Returns + ------- + float + a - (a - beta) * np.exp(-k*x**delta) + """ + return a - (a - beta) * np.exp(-k*x**delta) + + +all_models['janoschek'] = janoschek +model_para['janoschek'] = [0.73, 0.07, 0.355, 0.46] +model_para_num['janoschek'] = 4 diff --git a/new_impl/cv/third_party/nni_new/algorithms/hpo/curvefitting_assessor/model_factory.py b/new_impl/cv/third_party/nni_new/algorithms/hpo/curvefitting_assessor/model_factory.py new file mode 100644 index 0000000000000000000000000000000000000000..e6a6ada9976293d98a44d1d763f787a7c7c9bea1 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/hpo/curvefitting_assessor/model_factory.py @@ -0,0 +1,330 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +import numpy as np +from scipy import optimize +from .curvefunctions import * # pylint: disable=wildcard-import,unused-wildcard-import + +# Number of curve functions we prepared, more details can be found in "curvefunctions.py" +NUM_OF_FUNCTIONS = 12 +# Number of simulation time when we do MCMC sampling +NUM_OF_SIMULATION_TIME = 20 +# Number of samples we select when we do MCMC sampling +NUM_OF_INSTANCE = 10 +# The step size of each noise when we do MCMC sampling +STEP_SIZE = 0.0005 +# Number of least fitting function, if effective function is lower than this number, we will ask for more information +LEAST_FITTED_FUNCTION = 4 + +logger = logging.getLogger('curvefitting_Assessor') + +class CurveModel: + """Build a Curve Model to predict the performance + + Algorithm: https://github.com/Microsoft/nni/blob/master/src/sdk/pynni/nni/curvefitting_assessor/README.md + + Parameters + ---------- + target_pos : int + The point we need to predict + """ + def __init__(self, target_pos): + self.target_pos = target_pos + self.trial_history = [] + self.point_num = 0 + self.effective_model = [] + self.effective_model_num = 0 + self.weight_samples = [] + + def fit_theta(self): + """use least squares to fit all default curves parameter seperately + + Returns + ------- + None + """ + x = range(1, self.point_num + 1) + y = self.trial_history + for i in range(NUM_OF_FUNCTIONS): + model = curve_combination_models[i] + try: + # The maximum number of iterations to fit is 100*(N+1), where N is the number of elements in `x0`. + if model_para_num[model] == 2: + a, b = optimize.curve_fit(all_models[model], x, y)[0] + model_para[model][0] = a + model_para[model][1] = b + elif model_para_num[model] == 3: + a, b, c = optimize.curve_fit(all_models[model], x, y)[0] + model_para[model][0] = a + model_para[model][1] = b + model_para[model][2] = c + elif model_para_num[model] == 4: + a, b, c, d = optimize.curve_fit(all_models[model], x, y)[0] + model_para[model][0] = a + model_para[model][1] = b + model_para[model][2] = c + model_para[model][3] = d + except (RuntimeError, FloatingPointError, OverflowError, ZeroDivisionError): + # Ignore exceptions caused by numerical calculations + pass + except Exception as exception: + logger.critical("Exceptions in fit_theta: %s", exception) + + def filter_curve(self): + """filter the poor performing curve + + Returns + ------- + None + """ + avg = np.sum(self.trial_history) / self.point_num + standard = avg * avg * self.point_num + predict_data = [] + tmp_model = [] + for i in range(NUM_OF_FUNCTIONS): + var = 0 + model = curve_combination_models[i] + for j in range(1, self.point_num + 1): + y = self.predict_y(model, j) + var += (y - self.trial_history[j - 1]) * (y - self.trial_history[j - 1]) + if var < standard: + predict_data.append(y) + tmp_model.append(curve_combination_models[i]) + median = np.median(predict_data) + std = np.std(predict_data) + for model in tmp_model: + y = self.predict_y(model, self.target_pos) + epsilon = self.point_num / 10 * std + if y < median + epsilon and y > median - epsilon: + self.effective_model.append(model) + self.effective_model_num = len(self.effective_model) + logger.info('List of effective model: %s', self.effective_model) + + def predict_y(self, model, pos): + """return the predict y of 'model' when epoch = pos + + Parameters + ---------- + model : string + name of the curve function model + pos : int + the epoch number of the position you want to predict + + Returns + ------- + int + The expected matrix at pos + """ + if model_para_num[model] == 2: + y = all_models[model](pos, model_para[model][0], model_para[model][1]) + elif model_para_num[model] == 3: + y = all_models[model](pos, model_para[model][0], model_para[model][1], model_para[model][2]) + elif model_para_num[model] == 4: + y = all_models[model](pos, model_para[model][0], model_para[model][1], model_para[model][2], model_para[model][3]) + return y + + def f_comb(self, pos, sample): + """return the value of the f_comb when epoch = pos + + Parameters + ---------- + pos : int + the epoch number of the position you want to predict + sample : list + sample is a (1 * NUM_OF_FUNCTIONS) matrix, representing{w1, w2, ... wk} + + Returns + ------- + int + The expected matrix at pos with all the active function's prediction + """ + ret = 0 + for i in range(self.effective_model_num): + model = self.effective_model[i] + y = self.predict_y(model, pos) + ret += sample[i] * y + return ret + + def normalize_weight(self, samples): + """normalize weight + + Parameters + ---------- + samples : list + a collection of sample, it's a (NUM_OF_INSTANCE * NUM_OF_FUNCTIONS) matrix, + representing{{w11, w12, ..., w1k}, {w21, w22, ... w2k}, ...{wk1, wk2,..., wkk}} + + Returns + ------- + list + samples after normalize weight + """ + for i in range(NUM_OF_INSTANCE): + total = 0 + for j in range(self.effective_model_num): + total += samples[i][j] + for j in range(self.effective_model_num): + samples[i][j] /= total + return samples + + def sigma_sq(self, sample): + """returns the value of sigma square, given the weight's sample + + Parameters + ---------- + sample : list + sample is a (1 * NUM_OF_FUNCTIONS) matrix, representing{w1, w2, ... wk} + + Returns + ------- + float + the value of sigma square, given the weight's sample + """ + ret = 0 + for i in range(1, self.point_num + 1): + temp = self.trial_history[i - 1] - self.f_comb(i, sample) + ret += temp * temp + return 1.0 * ret / self.point_num + + def normal_distribution(self, pos, sample): + """returns the value of normal distribution, given the weight's sample and target position + + Parameters + ---------- + pos : int + the epoch number of the position you want to predict + sample : list + sample is a (1 * NUM_OF_FUNCTIONS) matrix, representing{w1, w2, ... wk} + + Returns + ------- + float + the value of normal distribution + """ + curr_sigma_sq = self.sigma_sq(sample) + delta = self.trial_history[pos - 1] - self.f_comb(pos, sample) + return np.exp(np.square(delta) / (-2.0 * curr_sigma_sq)) / np.sqrt(2 * np.pi * np.sqrt(curr_sigma_sq)) + + def likelihood(self, samples): + """likelihood + + Parameters + ---------- + sample : list + sample is a (1 * NUM_OF_FUNCTIONS) matrix, representing{w1, w2, ... wk} + + Returns + ------- + float + likelihood + """ + ret = np.ones(NUM_OF_INSTANCE) + for i in range(NUM_OF_INSTANCE): + for j in range(1, self.point_num + 1): + ret[i] *= self.normal_distribution(j, samples[i]) + return ret + + def prior(self, samples): + """priori distribution + + Parameters + ---------- + samples : list + a collection of sample, it's a (NUM_OF_INSTANCE * NUM_OF_FUNCTIONS) matrix, + representing{{w11, w12, ..., w1k}, {w21, w22, ... w2k}, ...{wk1, wk2,..., wkk}} + + Returns + ------- + float + priori distribution + """ + ret = np.ones(NUM_OF_INSTANCE) + for i in range(NUM_OF_INSTANCE): + for j in range(self.effective_model_num): + if not samples[i][j] > 0: + ret[i] = 0 + if self.f_comb(1, samples[i]) >= self.f_comb(self.target_pos, samples[i]): + ret[i] = 0 + return ret + + def target_distribution(self, samples): + """posterior probability + + Parameters + ---------- + samples : list + a collection of sample, it's a (NUM_OF_INSTANCE * NUM_OF_FUNCTIONS) matrix, + representing{{w11, w12, ..., w1k}, {w21, w22, ... w2k}, ...{wk1, wk2,..., wkk}} + + Returns + ------- + float + posterior probability + """ + curr_likelihood = self.likelihood(samples) + curr_prior = self.prior(samples) + ret = np.ones(NUM_OF_INSTANCE) + for i in range(NUM_OF_INSTANCE): + ret[i] = curr_likelihood[i] * curr_prior[i] + return ret + + def mcmc_sampling(self): + """Adjust the weight of each function using mcmc sampling. + The initial value of each weight is evenly distribute. + Brief introduction: + (1)Definition of sample: + Sample is a (1 * NUM_OF_FUNCTIONS) matrix, representing{w1, w2, ... wk} + (2)Definition of samples: + Samples is a collection of sample, it's a (NUM_OF_INSTANCE * NUM_OF_FUNCTIONS) matrix, + representing{{w11, w12, ..., w1k}, {w21, w22, ... w2k}, ...{wk1, wk2,..., wkk}} + (3)Definition of model: + Model is the function we chose right now. Such as: 'wap', 'weibull'. + (4)Definition of pos: + Pos is the position we want to predict, corresponds to the value of epoch. + + Returns + ------- + None + """ + init_weight = np.ones((self.effective_model_num), dtype=np.float) / self.effective_model_num + self.weight_samples = np.broadcast_to(init_weight, (NUM_OF_INSTANCE, self.effective_model_num)) + for _ in range(NUM_OF_SIMULATION_TIME): + # sample new value from Q(i, j) + new_values = np.random.randn(NUM_OF_INSTANCE, self.effective_model_num) * STEP_SIZE + self.weight_samples + new_values = self.normalize_weight(new_values) + # compute alpha(i, j) = min{1, P(j)Q(j, i)/P(i)Q(i, j)} + alpha = np.minimum(1, self.target_distribution(new_values) / self.target_distribution(self.weight_samples)) + # sample u + u = np.random.rand(NUM_OF_INSTANCE) + # new value + change_value_flag = (u < alpha).astype(np.int) + for j in range(NUM_OF_INSTANCE): + new_values[j] = self.weight_samples[j] * (1 - change_value_flag[j]) + new_values[j] * change_value_flag[j] + self.weight_samples = new_values + + def predict(self, trial_history): + """predict the value of target position + + Parameters + ---------- + trial_history : list + The history performance matrix of each trial. + + Returns + ------- + float + expected final result performance of this hyperparameter config + """ + self.trial_history = trial_history + self.point_num = len(trial_history) + self.fit_theta() + self.filter_curve() + if self.effective_model_num < LEAST_FITTED_FUNCTION: + # different curve's predictions are too scattered, requires more information + return None + self.mcmc_sampling() + ret = 0 + for i in range(NUM_OF_INSTANCE): + ret += self.f_comb(self.target_pos, self.weight_samples[i]) + return ret / NUM_OF_INSTANCE diff --git a/new_impl/cv/third_party/nni_new/algorithms/hpo/dngo_tuner.py b/new_impl/cv/third_party/nni_new/algorithms/hpo/dngo_tuner.py new file mode 100644 index 0000000000000000000000000000000000000000..276ec75168e41b6714cb51f9bd055ba521fed801 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/hpo/dngo_tuner.py @@ -0,0 +1,117 @@ +import logging + +import numpy as np +import torch +import nni.parameter_expressions as parameter_expressions +from nni import ClassArgsValidator +from nni.tuner import Tuner +from pybnn import DNGO +from torch.distributions import Normal + +_logger = logging.getLogger(__name__) + + +def _random_config(search_space, random_state): + chosen_config = {} + for key, val in search_space.items(): + if val['_type'] == 'choice': + choices = val['_value'] + index = random_state.randint(len(choices)) + if all([isinstance(c, (int, float)) for c in choices]): + chosen_config[key] = choices[index] + else: + raise ValueError('Choices with type other than int and float is not supported.') + elif val['_type'] == 'uniform': + chosen_config[key] = random_state.uniform(val['_value'][0], val['_value'][1]) + elif val['_type'] == 'randint': + chosen_config[key] = random_state.randint( + val['_value'][0], val['_value'][1]) + elif val['_type'] == 'quniform': + chosen_config[key] = parameter_expressions.quniform( + val['_value'][0], val['_value'][1], val['_value'][2], random_state) + elif val['_type'] == 'loguniform': + chosen_config[key] = parameter_expressions.loguniform( + val['_value'][0], val['_value'][1], random_state) + elif val['_type'] == 'qloguniform': + chosen_config[key] = parameter_expressions.qloguniform( + val['_value'][0], val['_value'][1], val['_value'][2], random_state) + else: + raise ValueError('Unknown key %s and value %s' % (key, val)) + return chosen_config + + +class DNGOTuner(Tuner): + + def __init__(self, optimize_mode='maximize', sample_size=1000, trials_per_update=20, num_epochs_per_training=500): + self.searchspace_json = None + self.random_state = None + self.model = DNGO(do_mcmc=False, num_epochs=num_epochs_per_training) + self._model_initialized = False + self.sample_size = sample_size + self.trials_per_update = trials_per_update + self.optimize_mode = optimize_mode + + self.x = [] + self.y = [] + + def receive_trial_result(self, parameter_id, parameters, value, **kwargs): + self.x.append(parameters) + self.y.append(self._get_default_value(value)) + if len(self.y) % self.trials_per_update == 0: + self._update_model() + + def generate_parameters(self, parameter_id, **kwargs): + if not self._model_initialized: + return _random_config(self.searchspace_json, self.random_state) + else: + # random samples and pick best with model + candidate_x = [_random_config(self.searchspace_json, self.random_state) for _ in range(self.sample_size)] + + x_test = np.array([np.array(list(xi.values())) for xi in candidate_x]) + m, v = self.model.predict(x_test) + mean = torch.Tensor(m) + sigma = torch.Tensor(v) + u = (mean - torch.Tensor([0.95]).expand_as(mean)) / sigma + normal = Normal(torch.zeros_like(u), torch.ones_like(u)) + ucdf = normal.cdf(u) + updf = torch.exp(normal.log_prob(u)) + ei = sigma * (updf + u * ucdf) + + if self.optimize_mode == 'maximize': + ind = torch.argmax(ei) + else: + ind = torch.argmin(ei) + new_x = candidate_x[ind] + return new_x + + def update_search_space(self, search_space): + self.searchspace_json = search_space + self.random_state = np.random.RandomState() + + def import_data(self, data): + for d in data: + self.x.append(d['parameter']) + self.y.append(self._get_default_value(d['value'])) + self._update_model() + + def _update_model(self): + _logger.info('Updating model on %d samples', len(self.x)) + x_arr = [] + for x in self.x: + x_arr.append([x[k] for k in sorted(x.keys())]) + self.model.train(np.array(x_arr), np.array(self.y), do_optimize=True) + self._model_initialized = True + + def _get_default_value(self, value): + if isinstance(value, dict) and 'default' in value: + return value['default'] + elif isinstance(value, float): + return value + else: + raise ValueError(f'Unsupported value: {value}') + + +class DNGOClassArgsValidator(ClassArgsValidator): + # DNGO tuner do not have much input arg, so the validation is actually hardly used + def validate_class_args(self, **kwargs): + pass diff --git a/new_impl/cv/third_party/nni_new/algorithms/hpo/evolution_tuner.py b/new_impl/cv/third_party/nni_new/algorithms/hpo/evolution_tuner.py new file mode 100644 index 0000000000000000000000000000000000000000..9277dcca3caac30134a0b787e4ed23e48295c855 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/hpo/evolution_tuner.py @@ -0,0 +1,283 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +evolution_tuner.py +""" + +import copy +import random +import logging + +from collections import deque +import numpy as np +from schema import Schema, Optional + +import nni +from nni import ClassArgsValidator +from nni.tuner import Tuner +from nni.utils import OptimizeMode, extract_scalar_reward, split_index, json2parameter, json2space + +logger = logging.getLogger(__name__) + +class Individual: + """ + Indicidual class to store the indv info. + + Attributes + ---------- + config : str + Search space. + info : str + The str to save information of individual. + result : float + The final metric of a individual. + """ + + def __init__(self, config=None, info=None, result=None): + """ + Parameters + ---------- + config : str + A config to represent a group of parameters. + info : str + result : float + save_dir : str + """ + self.config = config + self.result = result + self.info = info + + def __str__(self): + return "info: " + str(self.info) + \ + ", config :" + str(self.config) + ", result: " + str(self.result) + +class EvolutionClassArgsValidator(ClassArgsValidator): + def validate_class_args(self, **kwargs): + Schema({ + 'optimize_mode': self.choices('optimize_mode', 'maximize', 'minimize'), + Optional('population_size'): self.range('population_size', int, 0, 99999), + }).validate(kwargs) + +class EvolutionTuner(Tuner): + """ + EvolutionTuner is tuner using navie evolution algorithm. + """ + + def __init__(self, optimize_mode="maximize", population_size=32): + """ + Parameters + ---------- + optimize_mode : str, default 'maximize' + population_size : int + initial population size. The larger population size, + the better evolution performance. + """ + self.optimize_mode = OptimizeMode(optimize_mode) + self.population_size = population_size + + self.searchspace_json = None + self.running_trials = {} + self.num_running_trials = 0 + self.random_state = None + self.population = None + self.space = None + self.credit = 0 # record the unsatisfied trial requests + self.send_trial_callback = None + self.param_ids = deque() + + def update_search_space(self, search_space): + """ + Update search space. + + Search_space contains the information that user pre-defined. + + Parameters + ---------- + search_space : dict + """ + self.searchspace_json = search_space + self.space = json2space(self.searchspace_json) + + self.random_state = np.random.RandomState() + self.population = [] + + for _ in range(self.population_size): + self._random_generate_individual() + + def trial_end(self, parameter_id, success, **kwargs): + """ + To deal with trial failure. If a trial fails, + random generate the parameters and add into the population. + Parameters + ---------- + parameter_id : int + Unique identifier for hyper-parameters used by this trial. + success : bool + True if the trial successfully completed; False if failed or terminated. + **kwargs + Not used + """ + self.num_running_trials -= 1 + logger.info('trial (%d) end', parameter_id) + + if not success: + self.running_trials.pop(parameter_id) + self._random_generate_individual() + + if self.credit > 1: + param_id = self.param_ids.popleft() + config = self._generate_individual(param_id) + logger.debug('Send new trial (%d, %s) for reducing credit', param_id, config) + self.send_trial_callback(param_id, config) + self.credit -= 1 + self.num_running_trials += 1 + + def generate_multiple_parameters(self, parameter_id_list, **kwargs): + """ + Returns multiple sets of trial (hyper-)parameters, as iterable of serializable objects. + Parameters + ---------- + parameter_id_list : list of int + Unique identifiers for each set of requested hyper-parameters. + **kwargs + Not used + Returns + ------- + list + A list of newly generated configurations + """ + + result = [] + if 'st_callback' in kwargs: + self.send_trial_callback = kwargs['st_callback'] + else: + logger.warning('Send trial callback is not found in kwargs. Evolution tuner might not work properly.') + for parameter_id in parameter_id_list: + had_exception = False + try: + logger.debug("generating param for %s", parameter_id) + res = self.generate_parameters(parameter_id, **kwargs) + self.num_running_trials += 1 + except nni.NoMoreTrialError: + had_exception = True + if not had_exception: + result.append(res) + return result + + def _random_generate_individual(self): + is_rand = dict() + for item in self.space: + is_rand[item] = True + + config = json2parameter(self.searchspace_json, is_rand, self.random_state) + self.population.append(Individual(config=config)) + + def _generate_individual(self, parameter_id): + """ + This function will generate the config for a trial. + If at the first generation, randomly generates individuals to satisfy self.population_size. + Otherwise, random choose a pair of individuals and compare their fitnesses. + The worst of the pair will be removed. Copy the best of the pair and mutate it to generate a new individual. + + Parameters + ---------- + parameter_id : int + + Returns + ------- + dict + A group of candaidte parameters that evolution tuner generated. + """ + pos = -1 + + for i in range(len(self.population)): + if self.population[i].result is None: + pos = i + break + + if pos != -1: + indiv = copy.deepcopy(self.population[pos]) + self.population.pop(pos) + else: + random.shuffle(self.population) + # avoid only 1 individual has result + if len(self.population) > 1 and self.population[0].result < self.population[1].result: + self.population[0] = self.population[1] + + # mutation on the worse individual + space = json2space(self.searchspace_json, + self.population[0].config) + is_rand = dict() + mutation_pos = space[random.randint(0, len(space)-1)] + + for i in range(len(self.space)): + is_rand[self.space[i]] = (self.space[i] == mutation_pos) + config = json2parameter( + self.searchspace_json, is_rand, self.random_state, self.population[0].config) + + if len(self.population) > 1: + self.population.pop(1) + + indiv = Individual(config=config) + + # remove "_index" from config and save params-id + self.running_trials[parameter_id] = indiv + config = split_index(indiv.config) + return config + + + def generate_parameters(self, parameter_id, **kwargs): + """ + This function will returns a dict of trial (hyper-)parameters. + If no trial configration for now, self.credit plus 1 to send the config later + + Parameters + ---------- + parameter_id : int + + Returns + ------- + dict + One newly generated configuration. + """ + if not self.population: + raise RuntimeError('The population is empty') + + if self.num_running_trials >= self.population_size: + logger.warning("No enough trial config, population_size is suggested to be larger than trialConcurrency") + self.credit += 1 + self.param_ids.append(parameter_id) + raise nni.NoMoreTrialError('no more parameters now.') + + return self._generate_individual(parameter_id) + + def receive_trial_result(self, parameter_id, parameters, value, **kwargs): + """ + Record the result from a trial + + Parameters + ---------- + parameter_id : int + parameters : dict + value : dict/float + if value is dict, it should have "default" key. + value is final metrics of the trial. + """ + reward = extract_scalar_reward(value) + + if parameter_id not in self.running_trials: + raise RuntimeError('Received parameter_id %s not in running_trials.', parameter_id) + + # restore the paramsters contains "_index" + config = self.running_trials[parameter_id].config + self.running_trials.pop(parameter_id) + + if self.optimize_mode == OptimizeMode.Minimize: + reward = -reward + + indiv = Individual(config=config, result=reward) + self.population.append(indiv) + + def import_data(self, data): + pass diff --git a/new_impl/cv/third_party/nni_new/algorithms/hpo/gp_tuner/__init__.py b/new_impl/cv/third_party/nni_new/algorithms/hpo/gp_tuner/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..17bedd38f4ee1fefea0a3d73ab1b5d3fb7ef2aed --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/hpo/gp_tuner/__init__.py @@ -0,0 +1 @@ +from .gp_tuner import GPTuner, GPClassArgsValidator diff --git a/new_impl/cv/third_party/nni_new/algorithms/hpo/gp_tuner/gp_tuner.py b/new_impl/cv/third_party/nni_new/algorithms/hpo/gp_tuner/gp_tuner.py new file mode 100644 index 0000000000000000000000000000000000000000..c4e6e9a89cc6258457f647f80d24464969282357 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/hpo/gp_tuner/gp_tuner.py @@ -0,0 +1,181 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +GPTuner is a Bayesian Optimization method where Gaussian Process is used for modeling loss functions. + +See :class:`GPTuner` for details. +""" + +import warnings +import logging +import numpy as np +from schema import Schema, Optional + +from sklearn.gaussian_process.kernels import Matern +from sklearn.gaussian_process import GaussianProcessRegressor + +from nni import ClassArgsValidator +from nni.tuner import Tuner +from nni.utils import OptimizeMode, extract_scalar_reward + +from .target_space import TargetSpace +from .util import UtilityFunction, acq_max + +logger = logging.getLogger("GP_Tuner_AutoML") + +class GPClassArgsValidator(ClassArgsValidator): + def validate_class_args(self, **kwargs): + Schema({ + Optional('optimize_mode'): self.choices('optimize_mode', 'maximize', 'minimize'), + Optional('utility'): self.choices('utility', 'ei', 'ucb', 'poi'), + Optional('kappa'): float, + Optional('xi'): float, + Optional('nu'): float, + Optional('alpha'): float, + Optional('cold_start_num'): int, + Optional('selection_num_warm_up'): int, + Optional('selection_num_starting_points'): int, + }).validate(kwargs) + +class GPTuner(Tuner): + """ + GPTuner is a Bayesian Optimization method where Gaussian Process is used for modeling loss functions. + + Parameters + ---------- + optimize_mode : str + optimize mode, 'maximize' or 'minimize', by default 'maximize' + utility : str + utility function (also called 'acquisition funcition') to use, which can be 'ei', 'ucb' or 'poi'. By default 'ei'. + kappa : float + value used by utility function 'ucb'. The bigger kappa is, the more the tuner will be exploratory. By default 5. + xi : float + used by utility function 'ei' and 'poi'. The bigger xi is, the more the tuner will be exploratory. By default 0. + nu : float + used to specify Matern kernel. The smaller nu, the less smooth the approximated function is. By default 2.5. + alpha : float + Used to specify Gaussian Process Regressor. Larger values correspond to increased noise level in the observations. + By default 1e-6. + cold_start_num : int + Number of random exploration to perform before Gaussian Process. By default 10. + selection_num_warm_up : int + Number of random points to evaluate for getting the point which maximizes the acquisition function. By default 100000 + selection_num_starting_points : int + Number of times to run L-BFGS-B from a random starting point after the warmup. By default 250. + """ + + def __init__(self, optimize_mode="maximize", utility='ei', kappa=5, xi=0, nu=2.5, alpha=1e-6, cold_start_num=10, + selection_num_warm_up=100000, selection_num_starting_points=250): + self._optimize_mode = OptimizeMode(optimize_mode) + + # utility function related + self._utility = utility + self._kappa = kappa + self._xi = xi + + # target space + self._space = None + + self._random_state = np.random.RandomState() + + # nu, alpha are GPR related params + self._gp = GaussianProcessRegressor( + kernel=Matern(nu=nu), + alpha=alpha, + normalize_y=True, + n_restarts_optimizer=25, + random_state=self._random_state + ) + # num of random evaluations before GPR + self._cold_start_num = cold_start_num + + # params for acq_max + self._selection_num_warm_up = selection_num_warm_up + self._selection_num_starting_points = selection_num_starting_points + + # num of imported data + self._supplement_data_num = 0 + + def update_search_space(self, search_space): + """ + Update the self.bounds and self.types by the search_space.json file. + + Override of the abstract method in :class:`~nni.tuner.Tuner`. + """ + self._space = TargetSpace(search_space, self._random_state) + + def generate_parameters(self, parameter_id, **kwargs): + """ + Method which provides one set of hyper-parameters. + If the number of trial result is lower than cold_start_number, GPTuner will first randomly generate some parameters. + Otherwise, choose the parameters by the Gussian Process Model. + + Override of the abstract method in :class:`~nni.tuner.Tuner`. + """ + if self._space.len() < self._cold_start_num: + results = self._space.random_sample() + else: + # Sklearn's GP throws a large number of warnings at times, but + # we don't really need to see them here. + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + self._gp.fit(self._space.params, self._space.target) + + util = UtilityFunction( + kind=self._utility, kappa=self._kappa, xi=self._xi) + + results = acq_max( + f_acq=util.utility, + gp=self._gp, + y_max=self._space.target.max(), + bounds=self._space.bounds, + space=self._space, + num_warmup=self._selection_num_warm_up, + num_starting_points=self._selection_num_starting_points + ) + + results = self._space.array_to_params(results) + logger.info("Generate paramageters:\n %s", results) + return results + + def receive_trial_result(self, parameter_id, parameters, value, **kwargs): + """ + Method invoked when a trial reports its final result. + + Override of the abstract method in :class:`~nni.tuner.Tuner`. + """ + value = extract_scalar_reward(value) + if self._optimize_mode == OptimizeMode.Minimize: + value = -value + + logger.info("Received trial result.") + logger.info("value :%s", value) + logger.info("parameter : %s", parameters) + self._space.register(parameters, value) + + def import_data(self, data): + """ + Import additional data for tuning. + + Override of the abstract method in :class:`~nni.tuner.Tuner`. + """ + _completed_num = 0 + for trial_info in data: + logger.info( + "Importing data, current processing progress %s / %s", _completed_num, len(data)) + _completed_num += 1 + assert "parameter" in trial_info + _params = trial_info["parameter"] + assert "value" in trial_info + _value = trial_info['value'] + if not _value: + logger.info( + "Useless trial data, value is %s, skip this trial data.", _value) + continue + self._supplement_data_num += 1 + _parameter_id = '_'.join( + ["ImportData", str(self._supplement_data_num)]) + self.receive_trial_result( + parameter_id=_parameter_id, parameters=_params, value=_value) + logger.info("Successfully import data to GP tuner.") diff --git a/new_impl/cv/third_party/nni_new/algorithms/hpo/gp_tuner/target_space.py b/new_impl/cv/third_party/nni_new/algorithms/hpo/gp_tuner/target_space.py new file mode 100644 index 0000000000000000000000000000000000000000..7ee52c0e9969a90f931bd9a9e43b194f354d81c4 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/hpo/gp_tuner/target_space.py @@ -0,0 +1,295 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +Tool class to hold the param-space coordinates (X) and target values (Y). +""" + +import numpy as np +import nni.parameter_expressions as parameter_expressions + + +def _hashable(params): + """ + Transform list params to tuple format. Ensure that an point is hashable by a python dict. + + Parameters + ---------- + params : numpy array + array format of parameters + + Returns + ------- + tuple + tuple format of parameters + """ + return tuple(map(float, params)) + + +class TargetSpace(): + """ + Holds the param-space coordinates (X) and target values (Y) + + Parameters + ---------- + pbounds : dict + Dictionary with parameters names and legal values. + + random_state : int, RandomState, or None + optionally specify a seed for a random number generator, by default None. + """ + + def __init__(self, pbounds, random_state=None): + self._random_state = random_state + + # Get the name of the parameters + self._keys = sorted(pbounds) + + # Create an array with parameters bounds + self._bounds = np.array( + [item[1] for item in sorted(pbounds.items(), key=lambda x: x[0])] + ) + + # check values type + for _bound in self._bounds: + if _bound['_type'] == 'choice': + try: + [float(val) for val in _bound['_value']] + except ValueError: + raise ValueError("GP Tuner supports only numerical values") + + # preallocated memory for X and Y points + self._params = np.empty(shape=(0, self.dim)) + self._target = np.empty(shape=(0)) + + # keep track of unique points we have seen so far + self._cache = {} + + def __contains__(self, params): + """ + check if a parameter is already registered + + Parameters + ---------- + params : numpy array + + Returns + ------- + bool + True if the parameter is already registered, else false + """ + return _hashable(params) in self._cache + + def len(self): + """ + length of registered params and targets + + Returns + ------- + int + """ + assert len(self._params) == len(self._target) + return len(self._target) + + @property + def params(self): + """ + registered parameters + + Returns + ------- + numpy array + """ + return self._params + + @property + def target(self): + """ + registered target values + + Returns + ------- + numpy array + """ + return self._target + + @property + def dim(self): + """ + dimension of parameters + + Returns + ------- + int + """ + return len(self._keys) + + @property + def keys(self): + """ + keys of parameters + + Returns + ------- + numpy array + """ + return self._keys + + @property + def bounds(self): + """ + bounds of parameters + + Returns + ------- + numpy array + """ + return self._bounds + + def params_to_array(self, params): + """ + dict to array + + Parameters + ---------- + params : dict + dict format of parameters + + Returns + ------- + numpy array + array format of parameters + """ + try: + assert set(params) == set(self.keys) + except AssertionError: + raise ValueError( + "Parameters' keys ({}) do ".format(sorted(params)) + + "not match the expected set of keys ({}).".format(self.keys) + ) + return np.asarray([params[key] for key in self.keys]) + + def array_to_params(self, x): + """ + array to dict + + maintain int type if the paramters is defined as int in search_space.json + Parameters + ---------- + x : numpy array + array format of parameters + + Returns + ------- + dict + dict format of parameters + """ + try: + assert len(x) == len(self.keys) + except AssertionError: + raise ValueError( + "Size of array ({}) is different than the ".format(len(x)) + + "expected number of parameters ({}).".format(self.dim) + ) + + params = {} + for i, _bound in enumerate(self._bounds): + if _bound['_type'] == 'choice' and all(isinstance(val, int) for val in _bound['_value']): + params.update({self.keys[i]: int(x[i])}) + elif _bound['_type'] in ['randint']: + params.update({self.keys[i]: int(x[i])}) + else: + params.update({self.keys[i]: x[i]}) + + return params + + def register(self, params, target): + """ + Append a point and its target value to the known data. + + Parameters + ---------- + params : dict + parameters + + target : float + target function value + """ + + x = self.params_to_array(params) + if x in self: + print('Data point {} is not unique'.format(x)) + + # Insert data into unique dictionary + self._cache[_hashable(x.ravel())] = target + + self._params = np.concatenate([self._params, x.reshape(1, -1)]) + self._target = np.concatenate([self._target, [target]]) + + def random_sample(self): + """ + Creates a random point within the bounds of the space. + + Returns + ------- + numpy array + one groupe of parameter + """ + params = np.empty(self.dim) + for col, _bound in enumerate(self._bounds): + if _bound['_type'] == 'choice': + params[col] = parameter_expressions.choice( + _bound['_value'], self._random_state) + elif _bound['_type'] == 'randint': + params[col] = self._random_state.randint( + _bound['_value'][0], _bound['_value'][1], size=1) + elif _bound['_type'] == 'uniform': + params[col] = parameter_expressions.uniform( + _bound['_value'][0], _bound['_value'][1], self._random_state) + elif _bound['_type'] == 'quniform': + params[col] = parameter_expressions.quniform( + _bound['_value'][0], _bound['_value'][1], _bound['_value'][2], self._random_state) + elif _bound['_type'] == 'loguniform': + params[col] = parameter_expressions.loguniform( + _bound['_value'][0], _bound['_value'][1], self._random_state) + elif _bound['_type'] == 'qloguniform': + params[col] = parameter_expressions.qloguniform( + _bound['_value'][0], _bound['_value'][1], _bound['_value'][2], self._random_state) + + return params + + def max(self): + """ + Get maximum target value found and its corresponding parameters. + + Returns + ------- + dict + target value and parameters, empty dict if nothing registered + """ + try: + res = { + 'target': self.target.max(), + 'params': dict( + zip(self.keys, self.params[self.target.argmax()]) + ) + } + except ValueError: + res = {} + return res + + def res(self): + """ + Get all target values found and corresponding parameters. + + Returns + ------- + list + a list of target values and their corresponding parameters + """ + params = [dict(zip(self.keys, p)) for p in self.params] + + return [ + {"target": target, "params": param} + for target, param in zip(self.target, params) + ] diff --git a/new_impl/cv/third_party/nni_new/algorithms/hpo/gp_tuner/util.py b/new_impl/cv/third_party/nni_new/algorithms/hpo/gp_tuner/util.py new file mode 100644 index 0000000000000000000000000000000000000000..6926f988997a1e0b1d0bd6286dd7f091c38fc8da --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/hpo/gp_tuner/util.py @@ -0,0 +1,235 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +utility functions and classes for GPTuner +""" + +import warnings +import numpy as np +from scipy.stats import norm +from scipy.optimize import minimize + + +def _match_val_type(vals, bounds): + """ + Update values in the array, to match their corresponding type, make sure the value is legal. + + Parameters + ---------- + vals : numpy array + values of parameters + bounds : numpy array + list of dictionary which stores parameters names and legal values. + + Returns + ------- + vals_new : list + The closest legal value to the original value + """ + vals_new = [] + + for i, bound in enumerate(bounds): + _type = bound['_type'] + if _type == "choice": + # Find the closest integer in the array, vals_bounds + # pylint: disable=cell-var-from-loop + vals_new.append(min(bound['_value'], key=lambda x: abs(x - vals[i]))) + elif _type in ['quniform', 'randint']: + vals_new.append(np.around(vals[i])) + else: + vals_new.append(vals[i]) + + return vals_new + + +def acq_max(f_acq, gp, y_max, bounds, space, num_warmup, num_starting_points): + """ + A function to find the maximum of the acquisition function + + It uses a combination of random sampling (cheap) and the 'L-BFGS-B' + optimization method. First by sampling ``num_warmup`` points at random, + and then running L-BFGS-B from ``num_starting_points`` random starting points. + + Parameters + ---------- + f_acq : UtilityFunction.utility + The acquisition function object that return its point-wise value. + + gp : GaussianProcessRegressor + A gaussian process fitted to the relevant data. + + y_max : float + The current maximum known value of the target function. + + bounds : numpy array + The variables bounds to limit the search of the acq max. + + num_warmup : int + number of times to randomly sample the aquisition function + + num_starting_points : int + number of times to run scipy.minimize + + Returns + ------- + numpy array + The parameter which achieves max of the acquisition function. + """ + + # Warm up with random points + x_tries = [space.random_sample() + for _ in range(int(num_warmup))] + ys = f_acq(x_tries, gp=gp, y_max=y_max) + x_max = x_tries[ys.argmax()] + max_acq = ys.max() + + + # Explore the parameter space more throughly + x_seeds = [space.random_sample() for _ in range(int(num_starting_points))] + + bounds_minmax = np.array( + [[bound['_value'][0], bound['_value'][-1]] for bound in bounds]) + + for x_try in x_seeds: + # Find the minimum of minus the acquisition function + res = minimize(lambda x: -f_acq(x.reshape(1, -1), gp=gp, y_max=y_max), + x_try.reshape(1, -1), + bounds=bounds_minmax, + method="L-BFGS-B") + + # See if success + if not res.success: + continue + + # Store it if better than previous minimum(maximum). + if max_acq is None or -res.fun[0] >= max_acq: + x_max = _match_val_type(res.x, bounds) + max_acq = -res.fun[0] + + # Clip output to make sure it lies within the bounds. Due to floating + # point technicalities this is not always the case. + return np.clip(x_max, bounds_minmax[:, 0], bounds_minmax[:, 1]) + + +class UtilityFunction(): + """ + A class to compute different acquisition function values. + + Parameters + ---------- + kind : string + specification of utility function to use + kappa : float + parameter usedd for 'ucb' acquisition function + xi : float + parameter usedd for 'ei' and 'poi' acquisition function + """ + + def __init__(self, kind, kappa, xi): + self._kappa = kappa + self._xi = xi + + if kind not in ['ucb', 'ei', 'poi']: + err = "The utility function " \ + "{} has not been implemented, " \ + "please choose one of ucb, ei, or poi.".format(kind) + raise NotImplementedError(err) + self._kind = kind + + def utility(self, x, gp, y_max): + """ + return utility function + + Parameters + ---------- + x : numpy array + parameters + gp : GaussianProcessRegressor + y_max : float + maximum target value observed so far + + Returns + ------- + function + return corresponding function, return None if parameter is illegal + """ + if self._kind == 'ucb': + return self._ucb(x, gp, self._kappa) + if self._kind == 'ei': + return self._ei(x, gp, y_max, self._xi) + if self._kind == 'poi': + return self._poi(x, gp, y_max, self._xi) + return None + + @staticmethod + def _ucb(x, gp, kappa): + """ + Upper Confidence Bound (UCB) utility function + + Parameters + ---------- + x : numpy array + parameters + gp : GaussianProcessRegressor + kappa : float + + Returns + ------- + float + """ + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + mean, std = gp.predict(x, return_std=True) + + return mean + kappa * std + + @staticmethod + def _ei(x, gp, y_max, xi): + """ + Expected Improvement (EI) utility function + + Parameters + ---------- + x : numpy array + parameters + gp : GaussianProcessRegressor + y_max : float + maximum target value observed so far + xi : float + + Returns + ------- + float + """ + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + mean, std = gp.predict(x, return_std=True) + + z = (mean - y_max - xi)/std + return (mean - y_max - xi) * norm.cdf(z) + std * norm.pdf(z) + + @staticmethod + def _poi(x, gp, y_max, xi): + """ + Possibility Of Improvement (POI) utility function + + Parameters + ---------- + x : numpy array + parameters + gp : GaussianProcessRegressor + y_max : float + maximum target value observed so far + xi : float + + Returns + ------- + float + """ + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + mean, std = gp.predict(x, return_std=True) + + z = (mean - y_max - xi)/std + return norm.cdf(z) diff --git a/new_impl/cv/third_party/nni_new/algorithms/hpo/gridsearch_tuner.py b/new_impl/cv/third_party/nni_new/algorithms/hpo/gridsearch_tuner.py new file mode 100644 index 0000000000000000000000000000000000000000..4787b3f0eebfdf44a0fc52fd362c328ec0708f07 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/hpo/gridsearch_tuner.py @@ -0,0 +1,208 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +gridsearch_tuner.py including: + class GridSearchTuner +""" + +import copy +import logging +import numpy as np + +import nni +from nni.tuner import Tuner +from nni.utils import convert_dict2tuple + +TYPE = '_type' +CHOICE = 'choice' +VALUE = '_value' + +logger = logging.getLogger('grid_search_AutoML') + +class GridSearchTuner(Tuner): + """ + GridSearchTuner will search all the possible configures that the user define in the searchSpace. + The only acceptable types of search space are ``choice``, ``quniform``, ``randint`` + + Type ``choice`` will select one of the options. Note that it can also be nested. + + Type ``quniform`` will receive three values [``low``, ``high``, ``q``], + where [``low``, ``high``] specifies a range and ``q`` specifies the interval. + It will be sampled in a way that the first sampled value is ``low``, + and each of the following values is 'interval' larger than the value in front of it. + + Type ``randint`` gives all possible intergers in range[``low``, ``high``). Note that ``high`` is not included. + """ + + def __init__(self): + self.count = -1 + self.expanded_search_space = [] + self.supplement_data = dict() + + def _json2parameter(self, ss_spec): + """ + Generate all possible configs for hyperparameters from hyperparameter space. + + Parameters + ---------- + ss_spec : dict or list + Hyperparameter space or the ``_value`` of a hyperparameter + + Returns + ------- + list or dict + All the candidate choices of hyperparameters. for a hyperparameter, chosen_params + is a list. for multiple hyperparameters (e.g., search space), chosen_params is a dict. + """ + if isinstance(ss_spec, dict): + if '_type' in ss_spec.keys(): + _type = ss_spec['_type'] + _value = ss_spec['_value'] + chosen_params = list() + if _type == 'choice': + for value in _value: + choice = self._json2parameter(value) + if isinstance(choice, list): + chosen_params.extend(choice) + else: + chosen_params.append(choice) + elif _type == 'quniform': + chosen_params = self._parse_quniform(_value) + elif _type == 'randint': + chosen_params = self._parse_randint(_value) + else: + raise RuntimeError("Not supported type: %s" % _type) + else: + chosen_params = dict() + for key in ss_spec.keys(): + chosen_params[key] = self._json2parameter(ss_spec[key]) + return self._expand_parameters(chosen_params) + elif isinstance(ss_spec, list): + chosen_params = list() + for subspec in ss_spec[1:]: + choice = self._json2parameter(subspec) + if isinstance(choice, list): + chosen_params.extend(choice) + else: + chosen_params.append(choice) + chosen_params = list(map(lambda v: {ss_spec[0]: v}, chosen_params)) + else: + chosen_params = copy.deepcopy(ss_spec) + return chosen_params + + def _parse_quniform(self, param_value): + """ + Parse type of quniform parameter and return a list + """ + low, high, q = param_value[0], param_value[1], param_value[2] + return np.clip(np.arange(np.round(low/q), np.round(high/q)+1) * q, low, high) + + def _parse_randint(self, param_value): + """ + Parse type of randint parameter and return a list + """ + if param_value[0] >= param_value[1]: + raise ValueError("Randint should contain at least 1 candidate, but [%s, %s) contains none.", + param_value[0], param_value[1]) + return np.arange(param_value[0], param_value[1]).tolist() + + def _expand_parameters(self, para): + """ + Enumerate all possible combinations of all parameters + + Parameters + ---------- + para : dict + {key1: [v11, v12, ...], key2: [v21, v22, ...], ...} + + Returns + ------- + dict + {{key1: v11, key2: v21, ...}, {key1: v11, key2: v22, ...}, ...} + """ + if len(para) == 1: + for key, values in para.items(): + return list(map(lambda v: {key: v}, values)) + + key = list(para)[0] + values = para.pop(key) + rest_para = self._expand_parameters(para) + ret_para = list() + for val in values: + for config in rest_para: + config[key] = val + ret_para.append(copy.deepcopy(config)) + return ret_para + + def update_search_space(self, search_space): + """ + Check if the search space is valid and expand it: support only ``choice``, ``quniform``, ``randint``. + + Parameters + ---------- + search_space : dict + The format could be referred to search space spec (https://nni.readthedocs.io/en/latest/Tutorial/SearchSpaceSpec.html). + """ + self.expanded_search_space = self._json2parameter(search_space) + + def generate_parameters(self, parameter_id, **kwargs): + """ + Generate parameters for one trial. + + Parameters + ---------- + parameter_id : int + The id for the generated hyperparameter + **kwargs + Not used + + Returns + ------- + dict + One configuration from the expanded search space. + + Raises + ------ + NoMoreTrialError + If all the configurations has been sent, raise :class:`~nni.NoMoreTrialError`. + """ + self.count += 1 + while self.count <= len(self.expanded_search_space) - 1: + _params_tuple = convert_dict2tuple(copy.deepcopy(self.expanded_search_space[self.count])) + if _params_tuple in self.supplement_data: + self.count += 1 + else: + return self.expanded_search_space[self.count] + raise nni.NoMoreTrialError('no more parameters now.') + + def receive_trial_result(self, parameter_id, parameters, value, **kwargs): + """ + Receive a trial's final performance result reported through :func:`~nni.report_final_result` by the trial. + GridSearchTuner does not need trial's results. + """ + pass + + def import_data(self, data): + """ + Import additional data for tuning + + Parameters + ---------- + list + A list of dictionarys, each of which has at least two keys, ``parameter`` and ``value`` + """ + _completed_num = 0 + for trial_info in data: + logger.info("Importing data, current processing progress %s / %s", _completed_num, len(data)) + _completed_num += 1 + assert "parameter" in trial_info + _params = trial_info["parameter"] + assert "value" in trial_info + _value = trial_info['value'] + if not _value: + logger.info("Useless trial data, value is %s, skip this trial data.", _value) + continue + _params_tuple = convert_dict2tuple(copy.deepcopy(_params)) + self.supplement_data[_params_tuple] = True + logger.info("Successfully import data to grid search tuner.") diff --git a/new_impl/cv/third_party/nni_new/algorithms/hpo/hyperband_advisor.py b/new_impl/cv/third_party/nni_new/algorithms/hpo/hyperband_advisor.py new file mode 100644 index 0000000000000000000000000000000000000000..10b9b643bc1afbdf5d3988d6664e5925f3c2c371 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/hpo/hyperband_advisor.py @@ -0,0 +1,464 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +hyperband_advisor.py +""" + +import copy +import logging +import math +import sys + +import json_tricks +import numpy as np +from schema import Schema, Optional + +from nni import ClassArgsValidator +from nni.runtime.common import multi_phase_enabled +from nni.runtime.msg_dispatcher_base import MsgDispatcherBase +from nni.runtime.protocol import CommandType, send +from nni.utils import NodeType, OptimizeMode, MetricType, extract_scalar_reward +from nni import parameter_expressions + +_logger = logging.getLogger(__name__) + +_next_parameter_id = 0 +_KEY = 'TRIAL_BUDGET' +_epsilon = 1e-6 + + +def create_parameter_id(): + """Create an id + + Returns + ------- + int + parameter id + """ + global _next_parameter_id + _next_parameter_id += 1 + return _next_parameter_id - 1 + + +def create_bracket_parameter_id(brackets_id, brackets_curr_decay, increased_id=-1): + """Create a full id for a specific bracket's hyperparameter configuration + + Parameters + ---------- + brackets_id: string + brackets id + brackets_curr_decay: + brackets curr decay + increased_id: int + increased id + + Returns + ------- + int + params id + """ + if increased_id == -1: + increased_id = str(create_parameter_id()) + params_id = '_'.join([brackets_id, + str(brackets_curr_decay), + increased_id]) + return params_id + + +def json2parameter(ss_spec, random_state): + """Randomly generate values for hyperparameters from hyperparameter space i.e., x. + + Parameters + ---------- + ss_spec: + hyperparameter space + random_state: + random operator to generate random values + + Returns + ------- + Parameter: + Parameters in this experiment + """ + if isinstance(ss_spec, dict): + if NodeType.TYPE in ss_spec.keys(): + _type = ss_spec[NodeType.TYPE] + _value = ss_spec[NodeType.VALUE] + if _type == 'choice': + _index = random_state.randint(len(_value)) + chosen_params = json2parameter(ss_spec[NodeType.VALUE][_index], random_state) + else: + chosen_params = getattr(parameter_expressions, _type)(*(_value + [random_state])) + else: + chosen_params = dict() + for key in ss_spec.keys(): + chosen_params[key] = json2parameter(ss_spec[key], random_state) + elif isinstance(ss_spec, list): + chosen_params = list() + for _, subspec in enumerate(ss_spec): + chosen_params.append(json2parameter(subspec, random_state)) + else: + chosen_params = copy.deepcopy(ss_spec) + return chosen_params + + +class Bracket(): + """A bracket in Hyperband, all the information of a bracket is managed by an instance of this class + + Parameters + ---------- + bracket_id: string + The id of this bracket, usually be set as '{Hyperband index}-{SH iteration index}' + s: int + The current SH iteration index. + s_max: int + total number of SH iterations + eta: float + In each iteration, a complete run of sequential halving is executed. In it, + after evaluating each configuration on the same subset size, only a fraction of + 1/eta of them 'advances' to the next round. + R: + the budget associated with each stage + optimize_mode: str + optimize mode, 'maximize' or 'minimize' + """ + + def __init__(self, bracket_id, s, s_max, eta, R, optimize_mode): + self.bracket_id = bracket_id + self.s = s + self.s_max = s_max + self.eta = eta + self.n = math.ceil((s_max + 1) * (eta ** s) / (s + 1) - _epsilon) + self.r = R / eta ** s + self.i = 0 + self.hyper_configs = [] # [ {id: params}, {}, ... ] + self.configs_perf = [] # [ {id: [seq, acc]}, {}, ... ] + self.num_configs_to_run = [] # [ n, n, n, ... ] + self.num_finished_configs = [] # [ n, n, n, ... ] + self.optimize_mode = OptimizeMode(optimize_mode) + self.no_more_trial = False + + def is_completed(self): + """check whether this bracket has sent out all the hyperparameter configurations""" + return self.no_more_trial + + def get_n_r(self): + """return the values of n and r for the next round""" + return math.floor(self.n / self.eta ** self.i + _epsilon), math.floor(self.r * self.eta ** self.i + _epsilon) + + def increase_i(self): + """i means the ith round. Increase i by 1""" + self.i += 1 + if self.i > self.s: + self.no_more_trial = True + + def set_config_perf(self, i, parameter_id, seq, value): + """update trial's latest result with its sequence number, e.g., epoch number or batch number + + Parameters + ---------- + i: int + the ith round + parameter_id: int + the id of the trial/parameter + seq: int + sequence number, e.g., epoch number or batch number + value: int + latest result with sequence number seq + + Returns + ------- + None + """ + if parameter_id in self.configs_perf[i]: + if self.configs_perf[i][parameter_id][0] < seq: + self.configs_perf[i][parameter_id] = [seq, value] + else: + self.configs_perf[i][parameter_id] = [seq, value] + + def inform_trial_end(self, i): + """If the trial is finished and the corresponding round (i.e., i) has all its trials finished, + it will choose the top k trials for the next round (i.e., i+1) + + Parameters + ---------- + i: int + the ith round + """ + global _KEY + self.num_finished_configs[i] += 1 + _logger.debug('bracket id: %d, round: %d %d, finished: %d, all: %d', self.bracket_id, self.i, i, + self.num_finished_configs[i], self.num_configs_to_run[i]) + if self.num_finished_configs[i] >= self.num_configs_to_run[i] \ + and self.no_more_trial is False: + # choose candidate configs from finished configs to run in the next round + assert self.i == i + 1 + this_round_perf = self.configs_perf[i] + if self.optimize_mode is OptimizeMode.Maximize: + sorted_perf = sorted(this_round_perf.items(), key=lambda kv: kv[1][1], reverse=True) # reverse + else: + sorted_perf = sorted(this_round_perf.items(), key=lambda kv: kv[1][1]) + _logger.debug('bracket %s next round %s, sorted hyper configs: %s', self.bracket_id, self.i, sorted_perf) + next_n, next_r = self.get_n_r() + _logger.debug('bracket %s next round %s, next_n=%d, next_r=%d', self.bracket_id, self.i, next_n, next_r) + hyper_configs = dict() + for k in range(next_n): + params_id = sorted_perf[k][0] + params = self.hyper_configs[i][params_id] + params[_KEY] = next_r # modify r + # generate new id + increased_id = params_id.split('_')[-1] + new_id = create_bracket_parameter_id(self.bracket_id, self.i, increased_id) + hyper_configs[new_id] = params + self._record_hyper_configs(hyper_configs) + return [[key, value] for key, value in hyper_configs.items()] + return None + + def get_hyperparameter_configurations(self, num, r, searchspace_json, random_state): + """Randomly generate num hyperparameter configurations from search space + + Parameters + ---------- + num: int + the number of hyperparameter configurations + + Returns + ------- + list + a list of hyperparameter configurations. Format: [[key1, value1], [key2, value2], ...] + """ + global _KEY + assert self.i == 0 + hyperparameter_configs = dict() + for _ in range(num): + params_id = create_bracket_parameter_id(self.bracket_id, self.i) + params = json2parameter(searchspace_json, random_state) + params[_KEY] = r + hyperparameter_configs[params_id] = params + self._record_hyper_configs(hyperparameter_configs) + return [[key, value] for key, value in hyperparameter_configs.items()] + + def _record_hyper_configs(self, hyper_configs): + """after generating one round of hyperconfigs, this function records the generated hyperconfigs, + creates a dict to record the performance when those hyperconifgs are running, set the number of finished configs + in this round to be 0, and increase the round number. + + Parameters + ---------- + hyper_configs: list + the generated hyperconfigs + """ + self.hyper_configs.append(hyper_configs) + self.configs_perf.append(dict()) + self.num_finished_configs.append(0) + self.num_configs_to_run.append(len(hyper_configs)) + self.increase_i() + +class HyperbandClassArgsValidator(ClassArgsValidator): + def validate_class_args(self, **kwargs): + Schema({ + 'optimize_mode': self.choices('optimize_mode', 'maximize', 'minimize'), + Optional('exec_mode'): self.choices('exec_mode', 'serial', 'parallelism'), + Optional('R'): int, + Optional('eta'): int + }).validate(kwargs) + +class Hyperband(MsgDispatcherBase): + """Hyperband inherit from MsgDispatcherBase rather than Tuner, because it integrates both tuner's functions and assessor's functions. + This is an implementation that could fully leverage available resources or follow the algorithm process, i.e., high parallelism or serial. + A single execution of Hyperband takes a finite budget of (s_max + 1)B. + + Parameters + ---------- + R: int + the maximum amount of resource that can be allocated to a single configuration + eta: int + the variable that controls the proportion of configurations discarded in each round of SuccessiveHalving + optimize_mode: str + optimize mode, 'maximize' or 'minimize' + exec_mode: str + execution mode, 'serial' or 'parallelism' + """ + + def __init__(self, R=60, eta=3, optimize_mode='maximize', exec_mode='parallelism'): + """B = (s_max + 1)R""" + super(Hyperband, self).__init__() + self.R = R + self.eta = eta + self.brackets = dict() # dict of Bracket + self.generated_hyper_configs = [] # all the configs waiting for run + self.completed_hyper_configs = [] # all the completed configs + self.s_max = math.floor(math.log(self.R, self.eta) + _epsilon) + self.curr_s = self.s_max + self.curr_hb = 0 + self.exec_mode = exec_mode + self.curr_bracket_id = None + + self.searchspace_json = None + self.random_state = None + self.optimize_mode = OptimizeMode(optimize_mode) + + # This is for the case that nnimanager requests trial config, but tuner cannot provide immediately. + # In this case, tuner increases self.credit to issue a trial config sometime later. + self.credit = 0 + + # record the latest parameter_id of the trial job trial_job_id. + # if there is no running parameter_id, self.job_id_para_id_map[trial_job_id] == None + # new trial job is added to this dict and finished trial job is removed from it. + self.job_id_para_id_map = dict() + + def handle_initialize(self, data): + """callback for initializing the advisor + Parameters + ---------- + data: dict + search space + """ + self.handle_update_search_space(data) + send(CommandType.Initialized, '') + + def handle_request_trial_jobs(self, data): + """ + Parameters + ---------- + data: int + number of trial jobs + """ + self.credit += data + + for _ in range(self.credit): + self._request_one_trial_job() + + def _request_one_trial_job(self): + ret = self._get_one_trial_job() + if ret is not None: + send(CommandType.NewTrialJob, json_tricks.dumps(ret)) + self.credit -= 1 + + def _get_one_trial_job(self): + """get one trial job, i.e., one hyperparameter configuration.""" + if not self.generated_hyper_configs: + if self.exec_mode == 'parallelism' or \ + (self.exec_mode == 'serial' and (self.curr_bracket_id is None or self.brackets[self.curr_bracket_id].is_completed())): + if self.curr_s < 0: + self.curr_s = self.s_max + self.curr_hb += 1 + _logger.debug('create a new bracket, self.curr_hb=%d, self.curr_s=%d', self.curr_hb, self.curr_s) + self.curr_bracket_id = '{}-{}'.format(self.curr_hb, self.curr_s) + self.brackets[self.curr_bracket_id] = Bracket(self.curr_bracket_id, self.curr_s, self.s_max, self.eta, self.R, self.optimize_mode) + next_n, next_r = self.brackets[self.curr_bracket_id].get_n_r() + _logger.debug('new bracket, next_n=%d, next_r=%d', next_n, next_r) + assert self.searchspace_json is not None and self.random_state is not None + generated_hyper_configs = self.brackets[self.curr_bracket_id].get_hyperparameter_configurations(next_n, next_r, + self.searchspace_json, + self.random_state) + self.generated_hyper_configs = generated_hyper_configs.copy() + self.curr_s -= 1 + else: + ret = { + 'parameter_id': '-1_0_0', + 'parameter_source': 'algorithm', + 'parameters': '' + } + send(CommandType.NoMoreTrialJobs, json_tricks.dumps(ret)) + return None + + assert self.generated_hyper_configs + params = self.generated_hyper_configs.pop(0) + ret = { + 'parameter_id': params[0], + 'parameter_source': 'algorithm', + 'parameters': params[1] + } + return ret + + def handle_update_search_space(self, data): + """data: JSON object, which is search space + """ + self.searchspace_json = data + self.random_state = np.random.RandomState() + + def _handle_trial_end(self, parameter_id): + """ + Parameters + ---------- + parameter_id: parameter id of the finished config + """ + bracket_id, i, _ = parameter_id.split('_') + hyper_configs = self.brackets[bracket_id].inform_trial_end(int(i)) + if hyper_configs is not None: + _logger.debug('bracket %s next round %s, hyper_configs: %s', bracket_id, i, hyper_configs) + self.generated_hyper_configs = self.generated_hyper_configs + hyper_configs + for _ in range(self.credit): + self._request_one_trial_job() + + def handle_trial_end(self, data): + """ + Parameters + ---------- + data: dict() + it has three keys: trial_job_id, event, hyper_params + trial_job_id: the id generated by training service + event: the job's state + hyper_params: the hyperparameters (a string) generated and returned by tuner + """ + hyper_params = json_tricks.loads(data['hyper_params']) + self._handle_trial_end(hyper_params['parameter_id']) + if data['trial_job_id'] in self.job_id_para_id_map: + del self.job_id_para_id_map[data['trial_job_id']] + + def handle_report_metric_data(self, data): + """ + Parameters + ---------- + data: + it is an object which has keys 'parameter_id', 'value', 'trial_job_id', 'type', 'sequence'. + + Raises + ------ + ValueError + Data type not supported + """ + if 'value' in data: + data['value'] = json_tricks.loads(data['value']) + # multiphase? need to check + if data['type'] == MetricType.REQUEST_PARAMETER: + assert multi_phase_enabled() + assert data['trial_job_id'] is not None + assert data['parameter_index'] is not None + assert data['trial_job_id'] in self.job_id_para_id_map + self._handle_trial_end(self.job_id_para_id_map[data['trial_job_id']]) + ret = self._get_one_trial_job() + if data['trial_job_id'] is not None: + ret['trial_job_id'] = data['trial_job_id'] + if data['parameter_index'] is not None: + ret['parameter_index'] = data['parameter_index'] + self.job_id_para_id_map[data['trial_job_id']] = ret['parameter_id'] + send(CommandType.SendTrialJobParameter, json_tricks.dumps(ret)) + else: + value = extract_scalar_reward(data['value']) + bracket_id, i, _ = data['parameter_id'].split('_') + + # add to self.job_id_para_id_map here, + # because when the first parameter_id is created, trial_job_id is not known yet. + if data['trial_job_id'] in self.job_id_para_id_map: + assert self.job_id_para_id_map[data['trial_job_id']] == data['parameter_id'] + else: + self.job_id_para_id_map[data['trial_job_id']] = data['parameter_id'] + + if data['type'] == MetricType.FINAL: + # sys.maxsize indicates this value is from FINAL metric data, because data['sequence'] from FINAL metric + # and PERIODICAL metric are independent, thus, not comparable. + self.brackets[bracket_id].set_config_perf(int(i), data['parameter_id'], sys.maxsize, value) + self.completed_hyper_configs.append(data) + elif data['type'] == MetricType.PERIODICAL: + self.brackets[bracket_id].set_config_perf(int(i), data['parameter_id'], data['sequence'], value) + else: + raise ValueError('Data type not supported: {}'.format(data['type'])) + + def handle_add_customized_trial(self, data): + pass + + def handle_import_data(self, data): + pass diff --git a/new_impl/cv/third_party/nni_new/algorithms/hpo/hyperopt_tuner.py b/new_impl/cv/third_party/nni_new/algorithms/hpo/hyperopt_tuner.py new file mode 100644 index 0000000000000000000000000000000000000000..fbe9cda13faac67c060089f68d35048a4109a789 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/hpo/hyperopt_tuner.py @@ -0,0 +1,499 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +hyperopt_tuner.py +""" + +import copy +import logging + +import hyperopt as hp +import numpy as np +from schema import Optional, Schema +from nni import ClassArgsValidator +from nni.tuner import Tuner +from nni.utils import NodeType, OptimizeMode, extract_scalar_reward, split_index + +logger = logging.getLogger('hyperopt_AutoML') + + +def json2space(in_x, name=NodeType.ROOT): + """ + Change json to search space in hyperopt. + + Parameters + ---------- + in_x : dict/list/str/int/float + The part of json. + name : str + name could be NodeType.ROOT, NodeType.TYPE, NodeType.VALUE or NodeType.INDEX, NodeType.NAME. + """ + out_y = copy.deepcopy(in_x) + if isinstance(in_x, dict): + if NodeType.TYPE in in_x.keys(): + _type = in_x[NodeType.TYPE] + name = name + '-' + _type + _value = json2space(in_x[NodeType.VALUE], name=name) + if _type == 'choice': + out_y = hp.hp.choice(name, _value) + elif _type == 'randint': + out_y = hp.hp.randint(name, _value[1] - _value[0]) + else: + if _type in ['loguniform', 'qloguniform']: + _value[:2] = np.log(_value[:2]) + out_y = getattr(hp.hp, _type)(name, *_value) + else: + out_y = dict() + for key in in_x.keys(): + out_y[key] = json2space(in_x[key], name + '[%s]' % str(key)) + elif isinstance(in_x, list): + out_y = list() + for i, x_i in enumerate(in_x): + if isinstance(x_i, dict): + if NodeType.NAME not in x_i.keys(): + raise RuntimeError( + '\'_name\' key is not found in this nested search space.' + ) + out_y.append(json2space(x_i, name + '[%d]' % i)) + return out_y + + +def json2parameter(in_x, parameter, name=NodeType.ROOT): + """ + Change json to parameters. + """ + out_y = copy.deepcopy(in_x) + if isinstance(in_x, dict): + if NodeType.TYPE in in_x.keys(): + _type = in_x[NodeType.TYPE] + name = name + '-' + _type + if _type == 'choice': + _index = parameter[name] + out_y = { + NodeType.INDEX: + _index, + NodeType.VALUE: + json2parameter(in_x[NodeType.VALUE][_index], + parameter, + name=name + '[%d]' % _index) + } + else: + if _type in ['quniform', 'qloguniform']: + out_y = np.clip(parameter[name], in_x[NodeType.VALUE][0], in_x[NodeType.VALUE][1]) + elif _type == 'randint': + out_y = parameter[name] + in_x[NodeType.VALUE][0] + else: + out_y = parameter[name] + else: + out_y = dict() + for key in in_x.keys(): + out_y[key] = json2parameter(in_x[key], parameter, + name + '[%s]' % str(key)) + elif isinstance(in_x, list): + out_y = list() + for i, x_i in enumerate(in_x): + if isinstance(x_i, dict): + if NodeType.NAME not in x_i.keys(): + raise RuntimeError( + '\'_name\' key is not found in this nested search space.' + ) + out_y.append(json2parameter(x_i, parameter, name + '[%d]' % i)) + return out_y + + +def json2vals(in_x, vals, out_y, name=NodeType.ROOT): + if isinstance(in_x, dict): + if NodeType.TYPE in in_x.keys(): + _type = in_x[NodeType.TYPE] + name = name + '-' + _type + + try: + out_y[name] = vals[NodeType.INDEX] + # TODO - catch exact Exception + except Exception: + out_y[name] = vals + + if _type == 'choice': + _index = vals[NodeType.INDEX] + json2vals(in_x[NodeType.VALUE][_index], + vals[NodeType.VALUE], + out_y, + name=name + '[%d]' % _index) + if _type == 'randint': + out_y[name] -= in_x[NodeType.VALUE][0] + else: + for key in in_x.keys(): + json2vals(in_x[key], vals[key], out_y, + name + '[%s]' % str(key)) + elif isinstance(in_x, list): + for i, temp in enumerate(in_x): + # nested json + if isinstance(temp, dict): + if NodeType.NAME not in temp.keys(): + raise RuntimeError( + '\'_name\' key is not found in this nested search space.' + ) + else: + json2vals(temp, vals[i], out_y, name + '[%d]' % i) + else: + json2vals(temp, vals[i], out_y, name + '[%d]' % i) + + +def _add_index(in_x, parameter): + """ + change parameters in NNI format to parameters in hyperopt format(This function also support nested dict.). + For example, receive parameters like: + {'dropout_rate': 0.8, 'conv_size': 3, 'hidden_size': 512} + Will change to format in hyperopt, like: + {'dropout_rate': 0.8, 'conv_size': {'_index': 1, '_value': 3}, 'hidden_size': {'_index': 1, '_value': 512}} + """ + if NodeType.TYPE not in in_x: # if at the top level + out_y = dict() + for key, value in parameter.items(): + out_y[key] = _add_index(in_x[key], value) + return out_y + elif isinstance(in_x, dict): + value_type = in_x[NodeType.TYPE] + value_format = in_x[NodeType.VALUE] + if value_type == "choice": + choice_name = parameter[0] if isinstance(parameter, + list) else parameter + for pos, item in enumerate( + value_format): # here value_format is a list + if isinstance( + item, + list): # this format is ["choice_key", format_dict] + choice_key = item[0] + choice_value_format = item[1] + if choice_key == choice_name: + return { + NodeType.INDEX: pos, + NodeType.VALUE: [ + choice_name, + _add_index(choice_value_format, parameter[1]) + ] + } + elif choice_name == item: + return {NodeType.INDEX: pos, NodeType.VALUE: item} + else: + return parameter + return None # note: this is not written by original author, feel free to modify if you think it's incorrect + +class HyperoptClassArgsValidator(ClassArgsValidator): + def validate_class_args(self, **kwargs): + Schema({ + Optional('optimize_mode'): self.choices('optimize_mode', 'maximize', 'minimize'), + Optional('parallel_optimize'): bool, + Optional('constant_liar_type'): self.choices('constant_liar_type', 'min', 'max', 'mean') + }).validate(kwargs) + +class HyperoptTuner(Tuner): + """ + HyperoptTuner is a tuner which using hyperopt algorithm. + """ + + def __init__(self, algorithm_name, optimize_mode='minimize', + parallel_optimize=False, constant_liar_type='min'): + """ + Parameters + ---------- + algorithm_name : str + algorithm_name includes "tpe", "random_search" and anneal". + optimize_mode : str + parallel_optimize : bool + More detail could reference: docs/en_US/Tuner/HyperoptTuner.md + constant_liar_type : str + constant_liar_type including "min", "max" and "mean" + More detail could reference: docs/en_US/Tuner/HyperoptTuner.md + """ + self.algorithm_name = algorithm_name + self.optimize_mode = OptimizeMode(optimize_mode) + self.json = None + self.total_data = {} + self.rval = None + self.supplement_data_num = 0 + + self.parallel = parallel_optimize + if self.parallel: + self.CL_rval = None + self.constant_liar_type = constant_liar_type + self.running_data = [] + self.optimal_y = None + + def _choose_tuner(self, algorithm_name): + """ + Parameters + ---------- + algorithm_name : str + algorithm_name includes "tpe", "random_search" and anneal" + """ + if algorithm_name == 'tpe': + return hp.tpe.suggest + if algorithm_name == 'random_search': + return hp.rand.suggest + if algorithm_name == 'anneal': + return hp.anneal.suggest + raise RuntimeError('Not support tuner algorithm in hyperopt.') + + def update_search_space(self, search_space): + """ + Update search space definition in tuner by search_space in parameters. + + Will called when first setup experiemnt or update search space in WebUI. + + Parameters + ---------- + search_space : dict + """ + self.json = search_space + + search_space_instance = json2space(self.json) + rstate = np.random.RandomState() + trials = hp.Trials() + domain = hp.Domain(None, + search_space_instance, + pass_expr_memo_ctrl=None) + algorithm = self._choose_tuner(self.algorithm_name) + self.rval = hp.FMinIter(algorithm, + domain, + trials, + max_evals=-1, + rstate=rstate, + verbose=0) + self.rval.catch_eval_exceptions = False + + def generate_parameters(self, parameter_id, **kwargs): + """ + Returns a set of trial (hyper-)parameters, as a serializable object. + + Parameters + ---------- + parameter_id : int + + Returns + ------- + params : dict + """ + total_params = self.get_suggestion(random_search=False) + # avoid generating same parameter with concurrent trials because hyperopt doesn't support parallel mode + if total_params in self.total_data.values(): + # but it can cause duplicate parameter rarely + total_params = self.get_suggestion(random_search=True) + self.total_data[parameter_id] = total_params + + if self.parallel: + self.running_data.append(parameter_id) + + params = split_index(total_params) + return params + + def receive_trial_result(self, parameter_id, parameters, value, **kwargs): + """ + Record an observation of the objective function + + Parameters + ---------- + parameter_id : int + parameters : dict + value : dict/float + if value is dict, it should have "default" key. + value is final metrics of the trial. + """ + reward = extract_scalar_reward(value) + # restore the paramsters contains '_index' + if parameter_id not in self.total_data: + raise RuntimeError('Received parameter_id not in total_data.') + params = self.total_data[parameter_id] + + # code for parallel + if self.parallel: + constant_liar = kwargs.get('constant_liar', False) + + if constant_liar: + rval = self.CL_rval + else: + rval = self.rval + # ignore duplicated reported final result (due to aware of intermedate result) + if parameter_id not in self.running_data: + logger.info("Received duplicated final result with parameter id: %s", parameter_id) + return + self.running_data.remove(parameter_id) + + # update the reward of optimal_y + if self.optimal_y is None: + if self.constant_liar_type == 'mean': + self.optimal_y = [reward, 1] + else: + self.optimal_y = reward + else: + if self.constant_liar_type == 'mean': + _sum = self.optimal_y[0] + reward + _number = self.optimal_y[1] + 1 + self.optimal_y = [_sum, _number] + elif self.constant_liar_type == 'min': + self.optimal_y = min(self.optimal_y, reward) + elif self.constant_liar_type == 'max': + self.optimal_y = max(self.optimal_y, reward) + logger.debug("Update optimal_y with reward, optimal_y = %s", self.optimal_y) + else: + rval = self.rval + + + if self.optimize_mode is OptimizeMode.Maximize: + reward = -reward + + domain = rval.domain + trials = rval.trials + + new_id = len(trials) + + rval_specs = [None] + rval_results = [domain.new_result()] + rval_miscs = [dict(tid=new_id, cmd=domain.cmd, workdir=domain.workdir)] + + vals = params + idxs = dict() + + out_y = dict() + json2vals(self.json, vals, out_y) + vals = out_y + for key in domain.params: + if key in [NodeType.VALUE, NodeType.INDEX]: + continue + if key not in vals or vals[key] is None or vals[key] == []: + idxs[key] = vals[key] = [] + else: + idxs[key] = [new_id] + vals[key] = [vals[key]] + + self.miscs_update_idxs_vals(rval_miscs, + idxs, + vals, + idxs_map={new_id: new_id}, + assert_all_vals_used=False) + + trial = trials.new_trial_docs([new_id], rval_specs, rval_results, + rval_miscs)[0] + trial['result'] = {'loss': reward, 'status': 'ok'} + trial['state'] = hp.JOB_STATE_DONE + trials.insert_trial_docs([trial]) + trials.refresh() + + def miscs_update_idxs_vals(self, + miscs, + idxs, + vals, + assert_all_vals_used=True, + idxs_map=None): + """ + Unpack the idxs-vals format into the list of dictionaries that is + `misc`. + + Parameters + ---------- + idxs_map : dict + idxs_map is a dictionary of id->id mappings so that the misc['idxs'] can + contain different numbers than the idxs argument. + """ + if idxs_map is None: + idxs_map = {} + + assert set(idxs.keys()) == set(vals.keys()) + + misc_by_id = {m['tid']: m for m in miscs} + for m in miscs: + m['idxs'] = {key: [] for key in idxs} + m['vals'] = {key: [] for key in idxs} + + for key in idxs: + assert len(idxs[key]) == len(vals[key]) + for tid, val in zip(idxs[key], vals[key]): + tid = idxs_map.get(tid, tid) + if assert_all_vals_used or tid in misc_by_id: + misc_by_id[tid]['idxs'][key] = [tid] + misc_by_id[tid]['vals'][key] = [val] + + def get_suggestion(self, random_search=False): + """ + get suggestion from hyperopt + + Parameters + ---------- + random_search : bool + flag to indicate random search or not (default: {False}) + + Returns + ---------- + total_params : dict + parameter suggestion + """ + if self.parallel and len(self.total_data) > 20 and self.running_data and self.optimal_y is not None: + self.CL_rval = copy.deepcopy(self.rval) + if self.constant_liar_type == 'mean': + _constant_liar_y = self.optimal_y[0] / self.optimal_y[1] + else: + _constant_liar_y = self.optimal_y + for _parameter_id in self.running_data: + self.receive_trial_result(parameter_id=_parameter_id, parameters=None, value=_constant_liar_y, constant_liar=True) + rval = self.CL_rval + + random_state = np.random.randint(2**31 - 1) + else: + rval = self.rval + random_state = rval.rstate.randint(2**31 - 1) + + trials = rval.trials + algorithm = rval.algo + new_ids = rval.trials.new_trial_ids(1) + rval.trials.refresh() + + if random_search: + new_trials = hp.rand.suggest(new_ids, rval.domain, trials, + random_state) + else: + new_trials = algorithm(new_ids, rval.domain, trials, random_state) + rval.trials.refresh() + vals = new_trials[0]['misc']['vals'] + parameter = dict() + for key in vals: + try: + parameter[key] = vals[key][0].item() + except (KeyError, IndexError): + parameter[key] = None + + # remove '_index' from json2parameter and save params-id + total_params = json2parameter(self.json, parameter) + return total_params + + def import_data(self, data): + """ + Import additional data for tuning + + Parameters + ---------- + data: + a list of dictionarys, each of which has at least two keys, 'parameter' and 'value' + """ + _completed_num = 0 + for trial_info in data: + logger.info("Importing data, current processing progress %s / %s", _completed_num, len(data)) + _completed_num += 1 + if self.algorithm_name == 'random_search': + return + assert "parameter" in trial_info + _params = trial_info["parameter"] + assert "value" in trial_info + _value = trial_info['value'] + if not _value: + logger.info("Useless trial data, value is %s, skip this trial data.", _value) + continue + self.supplement_data_num += 1 + _parameter_id = '_'.join( + ["ImportData", str(self.supplement_data_num)]) + self.total_data[_parameter_id] = _add_index(in_x=self.json, + parameter=_params) + self.receive_trial_result(parameter_id=_parameter_id, + parameters=_params, + value=_value) + logger.info("Successfully import data to TPE/Anneal tuner.") diff --git a/new_impl/cv/third_party/nni_new/algorithms/hpo/medianstop_assessor.py b/new_impl/cv/third_party/nni_new/algorithms/hpo/medianstop_assessor.py new file mode 100644 index 0000000000000000000000000000000000000000..56eb82a3c98e0ce65efaa794fa6ebea8ee989566 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/hpo/medianstop_assessor.py @@ -0,0 +1,125 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +from schema import Schema, Optional + +from nni import ClassArgsValidator +from nni.assessor import Assessor, AssessResult +from nni.utils import extract_scalar_history + +logger = logging.getLogger('medianstop_Assessor') + +class MedianstopClassArgsValidator(ClassArgsValidator): + def validate_class_args(self, **kwargs): + Schema({ + Optional('optimize_mode'): self.choices('optimize_mode', 'maximize', 'minimize'), + Optional('start_step'): self.range('start_step', int, 0, 9999), + }).validate(kwargs) + +class MedianstopAssessor(Assessor): + """MedianstopAssessor is The median stopping rule stops a pending trial X at step S + if the trial’s best objective value by step S is strictly worse than the median value + of the running averages of all completed trials’ objectives reported up to step S + + Parameters + ---------- + optimize_mode : str + optimize mode, 'maximize' or 'minimize' + start_step : int + only after receiving start_step number of reported intermediate results + """ + def __init__(self, optimize_mode='maximize', start_step=0): + self._start_step = start_step + self._running_history = dict() + self._completed_avg_history = dict() + if optimize_mode == 'maximize': + self._high_better = True + elif optimize_mode == 'minimize': + self._high_better = False + else: + self._high_better = True + logger.warning('unrecognized optimize_mode %s', optimize_mode) + + def _update_data(self, trial_job_id, trial_history): + """update data + + Parameters + ---------- + trial_job_id : int + trial job id + trial_history : list + The history performance matrix of each trial + """ + if trial_job_id not in self._running_history: + self._running_history[trial_job_id] = [] + self._running_history[trial_job_id].extend(trial_history[len(self._running_history[trial_job_id]):]) + + def trial_end(self, trial_job_id, success): + """trial_end + + Parameters + ---------- + trial_job_id : int + trial job id + success : bool + True if succssfully finish the experiment, False otherwise + """ + if trial_job_id in self._running_history: + if success: + cnt = 0 + history_sum = 0 + self._completed_avg_history[trial_job_id] = [] + for each in self._running_history[trial_job_id]: + cnt += 1 + history_sum += each + self._completed_avg_history[trial_job_id].append(history_sum / cnt) + self._running_history.pop(trial_job_id) + else: + logger.warning('trial_end: trial_job_id does not exist in running_history') + + def assess_trial(self, trial_job_id, trial_history): + """assess_trial + + Parameters + ---------- + trial_job_id : int + trial job id + trial_history : list + The history performance matrix of each trial + + Returns + ------- + bool + AssessResult.Good or AssessResult.Bad + + Raises + ------ + Exception + unrecognize exception in medianstop_assessor + """ + curr_step = len(trial_history) + if curr_step < self._start_step: + return AssessResult.Good + + scalar_trial_history = extract_scalar_history(trial_history) + self._update_data(trial_job_id, scalar_trial_history) + if self._high_better: + best_history = max(scalar_trial_history) + else: + best_history = min(scalar_trial_history) + + avg_array = [] + for id_ in self._completed_avg_history: + if len(self._completed_avg_history[id_]) >= curr_step: + avg_array.append(self._completed_avg_history[id_][curr_step - 1]) + if avg_array: + avg_array.sort() + if self._high_better: + median = avg_array[(len(avg_array)-1) // 2] + return AssessResult.Bad if best_history < median else AssessResult.Good + else: + median = avg_array[len(avg_array) // 2] + return AssessResult.Bad if best_history > median else AssessResult.Good + else: + return AssessResult.Good diff --git a/new_impl/cv/third_party/nni_new/algorithms/hpo/metis_tuner/Regression_GMM/CreateModel.py b/new_impl/cv/third_party/nni_new/algorithms/hpo/metis_tuner/Regression_GMM/CreateModel.py new file mode 100644 index 0000000000000000000000000000000000000000..4846c9a67d1d4579456e5c28ce416b9e274566b4 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/hpo/metis_tuner/Regression_GMM/CreateModel.py @@ -0,0 +1,44 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +import sys +from operator import itemgetter + +import sklearn.mixture as mm + +sys.path.insert(1, os.path.join(sys.path[0], '..')) + + +def create_model(samples_x, samples_y_aggregation, percentage_goodbatch=0.34): + ''' + Create the Gaussian Mixture Model + ''' + samples = [samples_x[i] + [samples_y_aggregation[i]] + for i in range(0, len(samples_x))] + + # Sorts so that we can get the top samples + samples = sorted(samples, key=itemgetter(-1)) + samples_goodbatch_size = int(len(samples) * percentage_goodbatch) + samples_goodbatch = samples[0:samples_goodbatch_size] + samples_badbatch = samples[samples_goodbatch_size:] + + samples_x_goodbatch = [sample_goodbatch[0:-1] + for sample_goodbatch in samples_goodbatch] + #samples_y_goodbatch = [sample_goodbatch[-1] for sample_goodbatch in samples_goodbatch] + samples_x_badbatch = [sample_badbatch[0:-1] + for sample_badbatch in samples_badbatch] + + # === Trains GMM clustering models === # + #sys.stderr.write("[%s] Train GMM's GMM model\n" % (os.path.basename(__file__))) + bgmm_goodbatch = mm.BayesianGaussianMixture( + n_components=max(1, samples_goodbatch_size - 1)) + bad_n_components = max(1, len(samples_x) - samples_goodbatch_size - 1) + bgmm_badbatch = mm.BayesianGaussianMixture(n_components=bad_n_components) + bgmm_goodbatch.fit(samples_x_goodbatch) + bgmm_badbatch.fit(samples_x_badbatch) + + model = {} + model['clusteringmodel_good'] = bgmm_goodbatch + model['clusteringmodel_bad'] = bgmm_badbatch + return model diff --git a/new_impl/cv/third_party/nni_new/algorithms/hpo/metis_tuner/Regression_GMM/Selection.py b/new_impl/cv/third_party/nni_new/algorithms/hpo/metis_tuner/Regression_GMM/Selection.py new file mode 100644 index 0000000000000000000000000000000000000000..0bca96647d4642afc7d46c8a259f1634b4c41d81 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/hpo/metis_tuner/Regression_GMM/Selection.py @@ -0,0 +1,93 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +import random +import sys + +from .. import lib_acquisition_function +from .. import lib_constraint_summation + +sys.path.insert(1, os.path.join(sys.path[0], '..')) + + +CONSTRAINT_LOWERBOUND = None +CONSTRAINT_UPPERBOUND = None +CONSTRAINT_PARAMS_IDX = [] + + +def _ratio_scores(parameters_value, clusteringmodel_gmm_good, + clusteringmodel_gmm_bad): + ''' + The ratio is smaller the better + ''' + ratio = clusteringmodel_gmm_good.score( + [parameters_value]) / clusteringmodel_gmm_bad.score([parameters_value]) + sigma = 0 + return ratio, sigma + + +def selection_r(x_bounds, + x_types, + clusteringmodel_gmm_good, + clusteringmodel_gmm_bad, + num_starting_points=100, + minimize_constraints_fun=None): + ''' + Select using different types. + ''' + minimize_starting_points = clusteringmodel_gmm_good.sample(n_samples=num_starting_points) + + outputs = selection(x_bounds, x_types, + clusteringmodel_gmm_good, + clusteringmodel_gmm_bad, + minimize_starting_points[0], + minimize_constraints_fun) + + return outputs + + +def selection(x_bounds, + x_types, + clusteringmodel_gmm_good, + clusteringmodel_gmm_bad, + minimize_starting_points, + minimize_constraints_fun=None): + ''' + Select the lowest mu value + ''' + results = lib_acquisition_function.next_hyperparameter_lowest_mu( + _ratio_scores, [clusteringmodel_gmm_good, clusteringmodel_gmm_bad], + x_bounds, x_types, minimize_starting_points, + minimize_constraints_fun=minimize_constraints_fun) + + return results + + +def _rand_with_constraints(x_bounds, x_types): + ''' + Random generate the variable with constraints + ''' + outputs = None + x_bounds_withconstraints = [x_bounds[i] for i in CONSTRAINT_PARAMS_IDX] + x_types_withconstraints = [x_types[i] for i in CONSTRAINT_PARAMS_IDX] + x_val_withconstraints = lib_constraint_summation.rand(x_bounds_withconstraints, + x_types_withconstraints, + CONSTRAINT_LOWERBOUND, + CONSTRAINT_UPPERBOUND) + if x_val_withconstraints is not None: + outputs = [None] * len(x_bounds) + for i, _ in enumerate(CONSTRAINT_PARAMS_IDX): + outputs[CONSTRAINT_PARAMS_IDX[i]] = x_val_withconstraints[i] + for i, _ in enumerate(outputs): + if outputs[i] is None: + outputs[i] = random.randint(x_bounds[i][0], x_bounds[i][1]) + return outputs + + +def _minimize_constraints_fun_summation(x): + ''' + Minimize constraints fun summation + ''' + summation = sum([x[i] for i in CONSTRAINT_PARAMS_IDX]) + return CONSTRAINT_UPPERBOUND >= summation >= CONSTRAINT_LOWERBOUND diff --git a/new_impl/cv/third_party/nni_new/algorithms/hpo/metis_tuner/Regression_GMM/__init__.py b/new_impl/cv/third_party/nni_new/algorithms/hpo/metis_tuner/Regression_GMM/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/new_impl/cv/third_party/nni_new/algorithms/hpo/metis_tuner/Regression_GP/CreateModel.py b/new_impl/cv/third_party/nni_new/algorithms/hpo/metis_tuner/Regression_GP/CreateModel.py new file mode 100644 index 0000000000000000000000000000000000000000..98d1f22ce3c62c0491f91138e8c1df7ebd5e65ed --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/hpo/metis_tuner/Regression_GP/CreateModel.py @@ -0,0 +1,35 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +import sys +import numpy + +import sklearn.gaussian_process as gp + +sys.path.insert(1, os.path.join(sys.path[0], '..')) + + +def create_model(samples_x, samples_y_aggregation, + n_restarts_optimizer=250, is_white_kernel=False): + ''' + Trains GP regression model + ''' + kernel = gp.kernels.ConstantKernel(constant_value=1, + constant_value_bounds=(1e-12, 1e12)) * \ + gp.kernels.Matern(nu=1.5) + if is_white_kernel is True: + kernel += gp.kernels.WhiteKernel(noise_level=1, noise_level_bounds=(1e-12, 1e12)) + regressor = gp.GaussianProcessRegressor(kernel=kernel, + n_restarts_optimizer=n_restarts_optimizer, + normalize_y=True, + alpha=1e-10) + regressor.fit(numpy.array(samples_x), numpy.array(samples_y_aggregation)) + + model = {} + model['model'] = regressor + model['kernel_prior'] = str(kernel) + model['kernel_posterior'] = str(regressor.kernel_) + model['model_loglikelihood'] = regressor.log_marginal_likelihood(regressor.kernel_.theta) + + return model diff --git a/new_impl/cv/third_party/nni_new/algorithms/hpo/metis_tuner/Regression_GP/OutlierDetection.py b/new_impl/cv/third_party/nni_new/algorithms/hpo/metis_tuner/Regression_GP/OutlierDetection.py new file mode 100644 index 0000000000000000000000000000000000000000..f07a93dd3e80a8fd6f9b0e8ceb3e87a5338e915d --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/hpo/metis_tuner/Regression_GP/OutlierDetection.py @@ -0,0 +1,87 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +OutlierDectection.py +""" + +import os +import sys +from multiprocessing.dummy import Pool as ThreadPool + +from . import CreateModel as gp_create_model +from . import Prediction as gp_prediction + +sys.path.insert(1, os.path.join(sys.path[0], '..')) + + +def _outlierDetection_threaded(inputs): + """ + Detect the outlier + """ + [samples_idx, samples_x, samples_y_aggregation] = inputs + sys.stderr.write("[%s] DEBUG: Evaluating %dth of %d samples\n" + % (os.path.basename(__file__), samples_idx + 1, len(samples_x))) + outlier = None + + # Create a diagnostic regression model which removes the sample that we + # want to evaluate + diagnostic_regressor_gp = gp_create_model.create_model( + samples_x[0:samples_idx] + samples_x[samples_idx + 1:], + samples_y_aggregation[0:samples_idx] + samples_y_aggregation[samples_idx + 1:]) + mu, sigma = gp_prediction.predict( + samples_x[samples_idx], diagnostic_regressor_gp['model']) + + # 2.33 is the z-score for 98% confidence level + if abs(samples_y_aggregation[samples_idx] - mu) > (2.33 * sigma): + outlier = {"samples_idx": samples_idx, + "expected_mu": mu, + "expected_sigma": sigma, + "difference": abs(samples_y_aggregation[samples_idx] - mu) - (2.33 * sigma)} + return outlier + + +def outlierDetection_threaded(samples_x, samples_y_aggregation): + """ + Use Multi-thread to detect the outlier + """ + outliers = [] + + threads_inputs = [[samples_idx, samples_x, samples_y_aggregation] + for samples_idx in range(0, len(samples_x))] + threads_pool = ThreadPool(min(4, len(threads_inputs))) + threads_results = threads_pool.map( + _outlierDetection_threaded, threads_inputs) + threads_pool.close() + threads_pool.join() + + for threads_result in threads_results: + if threads_result is not None: + outliers.append(threads_result) + else: + print("Error: threads_result is None.") + + outliers = outliers if outliers else None + return outliers + + +def outlierDetection(samples_x, samples_y_aggregation): + outliers = [] + for samples_idx, _ in enumerate(samples_x): + #sys.stderr.write("[%s] DEBUG: Evaluating %d of %d samples\n" + # \ % (os.path.basename(__file__), samples_idx + 1, len(samples_x))) + diagnostic_regressor_gp = gp_create_model.create_model(\ + samples_x[0:samples_idx] + samples_x[samples_idx + 1:],\ + samples_y_aggregation[0:samples_idx] + samples_y_aggregation[samples_idx + 1:]) + mu, sigma = gp_prediction.predict(samples_x[samples_idx], + diagnostic_regressor_gp['model']) + # 2.33 is the z-score for 98% confidence level + if abs(samples_y_aggregation[samples_idx] - mu) > (2.33 * sigma): + outliers.append({"samples_idx": samples_idx, + "expected_mu": mu, + "expected_sigma": sigma, + "difference": \ + abs(samples_y_aggregation[samples_idx] - mu) - (2.33 * sigma)}) + + outliers = outliers if outliers else None + return outliers diff --git a/new_impl/cv/third_party/nni_new/algorithms/hpo/metis_tuner/Regression_GP/Prediction.py b/new_impl/cv/third_party/nni_new/algorithms/hpo/metis_tuner/Regression_GP/Prediction.py new file mode 100644 index 0000000000000000000000000000000000000000..08a32f0e63c86a6fd7dddad558f96c708916e41e --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/hpo/metis_tuner/Regression_GP/Prediction.py @@ -0,0 +1,19 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +import sys + +import numpy + +sys.path.insert(1, os.path.join(sys.path[0], '..')) + + +def predict(parameters_value, regressor_gp): + ''' + Predict by Gaussian Process Model + ''' + parameters_value = numpy.array(parameters_value).reshape(-1, len(parameters_value)) + mu, sigma = regressor_gp.predict(parameters_value, return_std=True) + + return mu[0], sigma[0] diff --git a/new_impl/cv/third_party/nni_new/algorithms/hpo/metis_tuner/Regression_GP/Selection.py b/new_impl/cv/third_party/nni_new/algorithms/hpo/metis_tuner/Regression_GP/Selection.py new file mode 100644 index 0000000000000000000000000000000000000000..68383ed0f2e039d4fd77c64730d06932f8b85b77 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/hpo/metis_tuner/Regression_GP/Selection.py @@ -0,0 +1,97 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +import random +import sys + +from .. import lib_acquisition_function +from .. import lib_constraint_summation +from .. import lib_data +from . import Prediction as gp_prediction + +sys.path.insert(1, os.path.join(sys.path[0], '..')) + +CONSTRAINT_LOWERBOUND = None +CONSTRAINT_UPPERBOUND = None +CONSTRAINT_PARAMS_IDX = [] + + +def selection_r(acquisition_function, + samples_y_aggregation, + x_bounds, + x_types, + regressor_gp, + num_starting_points=100, + minimize_constraints_fun=None): + ''' + Selecte R value + ''' + minimize_starting_points = [lib_data.rand(x_bounds, x_types) \ + for i in range(0, num_starting_points)] + outputs = selection(acquisition_function, samples_y_aggregation, + x_bounds, x_types, regressor_gp, + minimize_starting_points, + minimize_constraints_fun=minimize_constraints_fun) + + return outputs + +def selection(acquisition_function, + samples_y_aggregation, + x_bounds, x_types, + regressor_gp, + minimize_starting_points, + minimize_constraints_fun=None): + ''' + selection + ''' + outputs = None + + sys.stderr.write("[%s] Exercise \"%s\" acquisition function\n" \ + % (os.path.basename(__file__), acquisition_function)) + + if acquisition_function == "ei": + outputs = lib_acquisition_function.next_hyperparameter_expected_improvement(\ + gp_prediction.predict, [regressor_gp], x_bounds, x_types, \ + samples_y_aggregation, minimize_starting_points, \ + minimize_constraints_fun=minimize_constraints_fun) + elif acquisition_function == "lc": + outputs = lib_acquisition_function.next_hyperparameter_lowest_confidence(\ + gp_prediction.predict, [regressor_gp], x_bounds, x_types,\ + minimize_starting_points, minimize_constraints_fun=minimize_constraints_fun) + elif acquisition_function == "lm": + outputs = lib_acquisition_function.next_hyperparameter_lowest_mu(\ + gp_prediction.predict, [regressor_gp], x_bounds, x_types,\ + minimize_starting_points, minimize_constraints_fun=minimize_constraints_fun) + return outputs + +def _rand_with_constraints(x_bounds, x_types): + ''' + Random generate with constraints + ''' + outputs = None + + x_bounds_withconstraints = [x_bounds[i] for i in CONSTRAINT_PARAMS_IDX] + x_types_withconstraints = [x_types[i] for i in CONSTRAINT_PARAMS_IDX] + x_val_withconstraints = lib_constraint_summation.rand(x_bounds_withconstraints, + x_types_withconstraints, + CONSTRAINT_LOWERBOUND, + CONSTRAINT_UPPERBOUND) + if x_val_withconstraints is not None: + outputs = [None] * len(x_bounds) + + for i, _ in enumerate(CONSTRAINT_PARAMS_IDX): + outputs[CONSTRAINT_PARAMS_IDX[i]] = x_val_withconstraints[i] + + for i, _ in enumerate(outputs): + if outputs[i] is None: + outputs[i] = random.randint(x_bounds[i][0], x_bounds[i][1]) + return outputs + + +def _minimize_constraints_fun_summation(x): + ''' + Minimize the constraints fun summation + ''' + summation = sum([x[i] for i in CONSTRAINT_PARAMS_IDX]) + return CONSTRAINT_UPPERBOUND >= summation >= CONSTRAINT_LOWERBOUND diff --git a/new_impl/cv/third_party/nni_new/algorithms/hpo/metis_tuner/Regression_GP/__init__.py b/new_impl/cv/third_party/nni_new/algorithms/hpo/metis_tuner/Regression_GP/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/new_impl/cv/third_party/nni_new/algorithms/hpo/metis_tuner/__init__.py b/new_impl/cv/third_party/nni_new/algorithms/hpo/metis_tuner/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..f4f9ceba61960bab8b741d760448815a93b3ae0b --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/hpo/metis_tuner/__init__.py @@ -0,0 +1 @@ +from .metis_tuner import MetisTuner, MetisClassArgsValidator diff --git a/new_impl/cv/third_party/nni_new/algorithms/hpo/metis_tuner/lib_acquisition_function.py b/new_impl/cv/third_party/nni_new/algorithms/hpo/metis_tuner/lib_acquisition_function.py new file mode 100644 index 0000000000000000000000000000000000000000..f1b1edfe0165b61c18753dfa823bc2bc10aa8d05 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/hpo/metis_tuner/lib_acquisition_function.py @@ -0,0 +1,197 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +lib_acquisition_function.py +""" + +import sys +import numpy + +from scipy.stats import norm +from scipy.optimize import minimize + +from . import lib_data + + +def next_hyperparameter_expected_improvement(fun_prediction, + fun_prediction_args, + x_bounds, x_types, + samples_y_aggregation, + minimize_starting_points, + minimize_constraints_fun=None): + """ + "Expected Improvement" acquisition function + """ + best_x = None + best_acquisition_value = None + x_bounds_minmax = [[i[0], i[-1]] for i in x_bounds] + x_bounds_minmax = numpy.array(x_bounds_minmax) + + for starting_point in numpy.array(minimize_starting_points): + res = minimize(fun=_expected_improvement, + x0=starting_point.reshape(1, -1), + bounds=x_bounds_minmax, + method="L-BFGS-B", + args=(fun_prediction, + fun_prediction_args, + x_bounds, + x_types, + samples_y_aggregation, + minimize_constraints_fun)) + + if (best_acquisition_value is None) or \ + (res.fun < best_acquisition_value): + res.x = numpy.ndarray.tolist(res.x) + res.x = lib_data.match_val_type(res.x, x_bounds, x_types) + if (minimize_constraints_fun is None) or \ + (minimize_constraints_fun(res.x) is True): + best_acquisition_value = res.fun + best_x = res.x + + outputs = None + if best_x is not None: + mu, sigma = fun_prediction(best_x, *fun_prediction_args) + outputs = {'hyperparameter': best_x, 'expected_mu': mu, + 'expected_sigma': sigma, 'acquisition_func': "ei"} + + return outputs + + +def _expected_improvement(x, fun_prediction, fun_prediction_args, + x_bounds, x_types, samples_y_aggregation, + minimize_constraints_fun): + # This is only for step-wise optimization + x = lib_data.match_val_type(x, x_bounds, x_types) + + expected_improvement = sys.maxsize + if (minimize_constraints_fun is None) or ( + minimize_constraints_fun(x) is True): + mu, sigma = fun_prediction(x, *fun_prediction_args) + + loss_optimum = min(samples_y_aggregation) + scaling_factor = -1 + + # In case sigma equals zero + with numpy.errstate(divide="ignore"): + Z = scaling_factor * (mu - loss_optimum) / sigma + expected_improvement = scaling_factor * (mu - loss_optimum) * \ + norm.cdf(Z) + sigma * norm.pdf(Z) + expected_improvement = 0.0 if sigma == 0.0 else expected_improvement + + # We want expected_improvement to be as large as possible + # (i.e., as small as possible for minimize(...)) + expected_improvement = -1 * expected_improvement + return expected_improvement + + +def next_hyperparameter_lowest_confidence(fun_prediction, + fun_prediction_args, + x_bounds, x_types, + minimize_starting_points, + minimize_constraints_fun=None): + """ + "Lowest Confidence" acquisition function + """ + best_x = None + best_acquisition_value = None + x_bounds_minmax = [[i[0], i[-1]] for i in x_bounds] + x_bounds_minmax = numpy.array(x_bounds_minmax) + + for starting_point in numpy.array(minimize_starting_points): + res = minimize(fun=_lowest_confidence, + x0=starting_point.reshape(1, -1), + bounds=x_bounds_minmax, + method="L-BFGS-B", + args=(fun_prediction, + fun_prediction_args, + x_bounds, + x_types, + minimize_constraints_fun)) + + if (best_acquisition_value) is None or ( + res.fun < best_acquisition_value): + res.x = numpy.ndarray.tolist(res.x) + res.x = lib_data.match_val_type(res.x, x_bounds, x_types) + if (minimize_constraints_fun is None) or ( + minimize_constraints_fun(res.x) is True): + best_acquisition_value = res.fun + best_x = res.x + + outputs = None + if best_x is not None: + mu, sigma = fun_prediction(best_x, *fun_prediction_args) + outputs = {'hyperparameter': best_x, 'expected_mu': mu, + 'expected_sigma': sigma, 'acquisition_func': "lc"} + return outputs + + +def _lowest_confidence(x, fun_prediction, fun_prediction_args, + x_bounds, x_types, minimize_constraints_fun): + # This is only for step-wise optimization + x = lib_data.match_val_type(x, x_bounds, x_types) + + ci = sys.maxsize + if (minimize_constraints_fun is None) or ( + minimize_constraints_fun(x) is True): + mu, sigma = fun_prediction(x, *fun_prediction_args) + ci = (sigma * 1.96 * 2) / mu + # We want ci to be as large as possible + # (i.e., as small as possible for minimize(...), + # because this would mean lowest confidence + ci = -1 * ci + + return ci + + +def next_hyperparameter_lowest_mu(fun_prediction, + fun_prediction_args, + x_bounds, x_types, + minimize_starting_points, + minimize_constraints_fun=None): + """ + "Lowest Mu" acquisition function + """ + best_x = None + best_acquisition_value = None + x_bounds_minmax = [[i[0], i[-1]] for i in x_bounds] + x_bounds_minmax = numpy.array(x_bounds_minmax) + + for starting_point in numpy.array(minimize_starting_points): + res = minimize(fun=_lowest_mu, + x0=starting_point.reshape(1, -1), + bounds=x_bounds_minmax, + method="L-BFGS-B", + args=(fun_prediction, fun_prediction_args, + x_bounds, x_types, minimize_constraints_fun)) + + if (best_acquisition_value is None) or ( + res.fun < best_acquisition_value): + res.x = numpy.ndarray.tolist(res.x) + res.x = lib_data.match_val_type(res.x, x_bounds, x_types) + if (minimize_constraints_fun is None) or ( + minimize_constraints_fun(res.x) is True): + best_acquisition_value = res.fun + best_x = res.x + + outputs = None + if best_x is not None: + mu, sigma = fun_prediction(best_x, *fun_prediction_args) + outputs = {'hyperparameter': best_x, 'expected_mu': mu, + 'expected_sigma': sigma, 'acquisition_func': "lm"} + return outputs + + +def _lowest_mu(x, fun_prediction, fun_prediction_args, + x_bounds, x_types, minimize_constraints_fun): + """ + Calculate the lowest mu + """ + # This is only for step-wise optimization + x = lib_data.match_val_type(x, x_bounds, x_types) + + mu = sys.maxsize + if (minimize_constraints_fun is None) or ( + minimize_constraints_fun(x) is True): + mu, _ = fun_prediction(x, *fun_prediction_args) + return mu diff --git a/new_impl/cv/third_party/nni_new/algorithms/hpo/metis_tuner/lib_constraint_summation.py b/new_impl/cv/third_party/nni_new/algorithms/hpo/metis_tuner/lib_constraint_summation.py new file mode 100644 index 0000000000000000000000000000000000000000..948be5b3ca3dcace3ae673ce9e12667c093ca7a4 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/hpo/metis_tuner/lib_constraint_summation.py @@ -0,0 +1,108 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +lib_constraint_summation.py +""" + +import math +import random + +from operator import itemgetter + + +def check_feasibility(x_bounds, lowerbound, upperbound): + ''' + This can have false positives. + For examples, parameters can only be 0 or 5, and the summation constraint is between 6 and 7. + ''' + # x_bounds should be sorted, so even for "discrete_int" type, + # the smallest and the largest number should the first and the last element + x_bounds_lowerbound = sum([x_bound[0] for x_bound in x_bounds]) + x_bounds_upperbound = sum([x_bound[-1] for x_bound in x_bounds]) + + # return ((x_bounds_lowerbound <= lowerbound) and (x_bounds_upperbound >= lowerbound)) or \ + # ((x_bounds_lowerbound <= upperbound) and (x_bounds_upperbound >= upperbound)) + return (x_bounds_lowerbound <= lowerbound <= x_bounds_upperbound) or \ + (x_bounds_lowerbound <= upperbound <= x_bounds_upperbound) + + +def rand(x_bounds, x_types, lowerbound, upperbound, max_retries=100): + ''' + Key idea is that we try to move towards upperbound, by randomly choose one + value for each parameter. However, for the last parameter, + we need to make sure that its value can help us get above lowerbound + ''' + outputs = None + + if check_feasibility(x_bounds, lowerbound, upperbound) is True: + # Order parameters by their range size. We want the smallest range first, + # because the corresponding parameter has less numbers to choose from + x_idx_sorted = [] + for i, _ in enumerate(x_bounds): + if x_types[i] == "discrete_int": + x_idx_sorted.append([i, len(x_bounds[i])]) + elif (x_types[i] == "range_int") or (x_types[i] == "range_continuous"): + x_idx_sorted.append( + [i, math.floor(x_bounds[i][1] - x_bounds[i][0])]) + x_idx_sorted = sorted(x_idx_sorted, key=itemgetter(1)) + + for _ in range(max_retries): + budget_allocated = 0 + outputs = [None] * len(x_bounds) + + for i, _ in enumerate(x_idx_sorted): + x_idx = x_idx_sorted[i][0] + # The amount of unallocated space that we have + budget_max = upperbound - budget_allocated + # NOT the Last x that we need to assign a random number + if i < (len(x_idx_sorted) - 1): + if x_bounds[x_idx][0] <= budget_max: + if x_types[x_idx] == "discrete_int": + # Note the valid integer + temp = [] + for j in x_bounds[x_idx]: + if j <= budget_max: + temp.append(j) + # Randomly pick a number from the integer array + if temp: + outputs[x_idx] = temp[random.randint( + 0, len(temp) - 1)] + + elif (x_types[x_idx] == "range_int") or \ + (x_types[x_idx] == "range_continuous"): + outputs[x_idx] = random.randint( + x_bounds[x_idx][0], min(x_bounds[x_idx][-1], budget_max)) + + else: + # The last x that we need to assign a random number + randint_lowerbound = lowerbound - budget_allocated + randint_lowerbound = 0 if randint_lowerbound < 0 else randint_lowerbound + + # This check: + # is our smallest possible value going to overflow the available budget space, + # and is our largest possible value going to underflow the + # lower bound + if (x_bounds[x_idx][0] <= budget_max) and \ + (x_bounds[x_idx][-1] >= randint_lowerbound): + if x_types[x_idx] == "discrete_int": + temp = [] + for j in x_bounds[x_idx]: + # if (j <= budget_max) and (j >= + # randint_lowerbound): + if randint_lowerbound <= j <= budget_max: + temp.append(j) + if temp: + outputs[x_idx] = temp[random.randint( + 0, len(temp) - 1)] + elif (x_types[x_idx] == "range_int") or \ + (x_types[x_idx] == "range_continuous"): + outputs[x_idx] = random.randint( + randint_lowerbound, min( + x_bounds[x_idx][1], budget_max)) + if outputs[x_idx] is None: + break + budget_allocated += outputs[x_idx] + if None not in outputs: + break + return outputs diff --git a/new_impl/cv/third_party/nni_new/algorithms/hpo/metis_tuner/lib_data.py b/new_impl/cv/third_party/nni_new/algorithms/hpo/metis_tuner/lib_data.py new file mode 100644 index 0000000000000000000000000000000000000000..cc3f059514c8c30312acbd0c18ec000803cbe0ad --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/hpo/metis_tuner/lib_data.py @@ -0,0 +1,50 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import math +import random + + +def match_val_type(vals, vals_bounds, vals_types): + ''' + Update values in the array, to match their corresponding type + ''' + vals_new = [] + + for i, _ in enumerate(vals_types): + if vals_types[i] == "discrete_int": + # Find the closest integer in the array, vals_bounds + # pylint: disable=cell-var-from-loop + vals_new.append(min(vals_bounds[i], key=lambda x: abs(x - vals[i]))) + elif vals_types[i] == "range_int": + # Round down to the nearest integer + vals_new.append(math.floor(vals[i])) + elif vals_types[i] == "range_continuous": + # Don't do any processing for continous numbers + vals_new.append(vals[i]) + else: + return None + + return vals_new + + +def rand(x_bounds, x_types): + ''' + Random generate variable value within their bounds + ''' + outputs = [] + + for i, _ in enumerate(x_bounds): + if x_types[i] == "discrete_int": + temp = x_bounds[i][random.randint(0, len(x_bounds[i]) - 1)] + outputs.append(temp) + elif x_types[i] == "range_int": + temp = random.randint(x_bounds[i][0], x_bounds[i][1] - 1) + outputs.append(temp) + elif x_types[i] == "range_continuous": + temp = random.uniform(x_bounds[i][0], x_bounds[i][1]) + outputs.append(temp) + else: + return None + + return outputs diff --git a/new_impl/cv/third_party/nni_new/algorithms/hpo/metis_tuner/metis_tuner.py b/new_impl/cv/third_party/nni_new/algorithms/hpo/metis_tuner/metis_tuner.py new file mode 100644 index 0000000000000000000000000000000000000000..1a0670f5e517d3ce1cdf775a2590daea7366f4d9 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/hpo/metis_tuner/metis_tuner.py @@ -0,0 +1,650 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +metis_tuner.py +""" + +import copy +import logging +import random +import statistics +import warnings +from multiprocessing.dummy import Pool as ThreadPool +import numpy as np +from schema import Schema, Optional + +from nni import ClassArgsValidator +from . import lib_constraint_summation +from . import lib_data +from .Regression_GMM import CreateModel as gmm_create_model +from .Regression_GMM import Selection as gmm_selection +from .Regression_GP import CreateModel as gp_create_model +from .Regression_GP import OutlierDetection as gp_outlier_detection +from .Regression_GP import Prediction as gp_prediction +from .Regression_GP import Selection as gp_selection +from nni.tuner import Tuner +from nni.utils import OptimizeMode, extract_scalar_reward + +logger = logging.getLogger("Metis_Tuner_AutoML") + +NONE_TYPE = '' +CONSTRAINT_LOWERBOUND = None +CONSTRAINT_UPPERBOUND = None +CONSTRAINT_PARAMS_IDX = [] + +class MetisClassArgsValidator(ClassArgsValidator): + def validate_class_args(self, **kwargs): + Schema({ + Optional('optimize_mode'): self.choices('optimize_mode', 'maximize', 'minimize'), + Optional('no_resampling'): bool, + Optional('no_candidates'): bool, + Optional('selection_num_starting_points'): int, + Optional('cold_start_num'): int, + }).validate(kwargs) + +class MetisTuner(Tuner): + """ + Metis Tuner + + More algorithm information you could reference here: + https://www.microsoft.com/en-us/research/publication/metis-robustly-tuning-tail-latencies-cloud-systems/ + + Attributes + ---------- + optimize_mode : str + optimize_mode is a string that including two mode "maximize" and "minimize" + + no_resampling : bool + True or False. + Should Metis consider re-sampling as part of the search strategy? + If you are confident that the training dataset is noise-free, + then you do not need re-sampling. + + no_candidates : bool + True or False. + Should Metis suggest parameters for the next benchmark? + If you do not plan to do more benchmarks, + Metis can skip this step. + + selection_num_starting_points : int + How many times Metis should try to find the global optimal in the search space? + The higher the number, the longer it takes to output the solution. + + cold_start_num : int + Metis need some trial result to get cold start. + when the number of trial result is less than + cold_start_num, Metis will randomly sample hyper-parameter for trial. + + exploration_probability: float + The probability of Metis to select parameter from exploration instead of exploitation. + """ + + def __init__( + self, + optimize_mode="maximize", + no_resampling=True, + no_candidates=False, + selection_num_starting_points=600, + cold_start_num=10, + exploration_probability=0.9): + """ + Parameters + ---------- + optimize_mode : str + optimize_mode is a string that including two mode "maximize" and "minimize" + + no_resampling : bool + True or False. + Should Metis consider re-sampling as part of the search strategy? + If you are confident that the training dataset is noise-free, + then you do not need re-sampling. + + no_candidates : bool + True or False. + Should Metis suggest parameters for the next benchmark? + If you do not plan to do more benchmarks, + Metis can skip this step. + + selection_num_starting_points : int + How many times Metis should try to find the global optimal in the search space? + The higher the number, the longer it takes to output the solution. + + cold_start_num : int + Metis need some trial result to get cold start. + when the number of trial result is less than + cold_start_num, Metis will randomly sample hyper-parameter for trial. + + exploration_probability : float + The probability of Metis to select parameter from exploration instead of exploitation. + + x_bounds : list + The constration of parameters. + + x_types : list + The type of parameters. + """ + + self.samples_x = [] + self.samples_y = [] + self.samples_y_aggregation = [] + self.total_data = [] + self.space = None + self.no_resampling = no_resampling + self.no_candidates = no_candidates + self.optimize_mode = OptimizeMode(optimize_mode) + self.key_order = [] + self.cold_start_num = cold_start_num + self.selection_num_starting_points = selection_num_starting_points + self.exploration_probability = exploration_probability + self.minimize_constraints_fun = None + self.minimize_starting_points = None + self.supplement_data_num = 0 + self.x_bounds = [] + self.x_types = [] + + + def update_search_space(self, search_space): + """ + Update the self.x_bounds and self.x_types by the search_space.json + + Parameters + ---------- + search_space : dict + """ + self.x_bounds = [[] for i in range(len(search_space))] + self.x_types = [NONE_TYPE for i in range(len(search_space))] + + for key in search_space: + self.key_order.append(key) + + key_type = {} + if isinstance(search_space, dict): + for key in search_space: + key_type = search_space[key]['_type'] + key_range = search_space[key]['_value'] + idx = self.key_order.index(key) + if key_type == 'quniform': + if key_range[2] == 1 and key_range[0].is_integer( + ) and key_range[1].is_integer(): + self.x_bounds[idx] = [key_range[0], key_range[1] + 1] + self.x_types[idx] = 'range_int' + else: + low, high, q = key_range + bounds = np.clip( + np.arange( + np.round( + low / q), + np.round( + high / q) + 1) * q, + low, + high) + self.x_bounds[idx] = bounds + self.x_types[idx] = 'discrete_int' + elif key_type == 'randint': + self.x_bounds[idx] = [key_range[0], key_range[1]] + self.x_types[idx] = 'range_int' + elif key_type == 'uniform': + self.x_bounds[idx] = [key_range[0], key_range[1]] + self.x_types[idx] = 'range_continuous' + elif key_type == 'choice': + self.x_bounds[idx] = key_range + + for key_value in key_range: + if not isinstance(key_value, (int, float)): + raise RuntimeError( + "Metis Tuner only support numerical choice.") + + self.x_types[idx] = 'discrete_int' + else: + logger.info( + "Metis Tuner doesn't support this kind of variable: %s", + str(key_type)) + raise RuntimeError( + "Metis Tuner doesn't support this kind of variable: %s" % + str(key_type)) + else: + logger.info("The format of search space is not a dict.") + raise RuntimeError("The format of search space is not a dict.") + + self.minimize_starting_points = _rand_init( + self.x_bounds, self.x_types, self.selection_num_starting_points) + + + def _pack_output(self, init_parameter): + """ + Pack the output + + Parameters + ---------- + init_parameter : dict + + Returns + ------- + output : dict + """ + output = {} + for i, param in enumerate(init_parameter): + output[self.key_order[i]] = param + + return output + + + def generate_parameters(self, parameter_id, **kwargs): + """ + Generate next parameter for trial + + If the number of trial result is lower than cold start number, + metis will first random generate some parameters. + Otherwise, metis will choose the parameters by + the Gussian Process Model and the Gussian Mixture Model. + + Parameters + ---------- + parameter_id : int + + Returns + ------- + result : dict + """ + if len(self.samples_x) < self.cold_start_num: + init_parameter = _rand_init(self.x_bounds, self.x_types, 1)[0] + results = self._pack_output(init_parameter) + else: + self.minimize_starting_points = _rand_init( + self.x_bounds, self.x_types, self.selection_num_starting_points) + results = self._selection( + self.samples_x, + self.samples_y_aggregation, + self.samples_y, + self.x_bounds, + self.x_types, + threshold_samplessize_resampling=( + None if self.no_resampling is True else 50), + no_candidates=self.no_candidates, + minimize_starting_points=self.minimize_starting_points, + minimize_constraints_fun=self.minimize_constraints_fun) + + logger.info("Generate paramageters: \n%s", str(results)) + return results + + + def receive_trial_result(self, parameter_id, parameters, value, **kwargs): + """ + Tuner receive result from trial. + + Parameters + ---------- + parameter_id : int + The id of parameters, generated by nni manager. + parameters : dict + A group of parameters that trial has tried. + value : dict/float + if value is dict, it should have "default" key. + """ + value = extract_scalar_reward(value) + if self.optimize_mode == OptimizeMode.Maximize: + value = -value + + logger.info("Received trial result.") + logger.info("value is : %s", str(value)) + logger.info("parameter is : %s", str(parameters)) + + # parse parameter to sample_x + sample_x = [0 for i in range(len(self.key_order))] + for key in parameters: + idx = self.key_order.index(key) + sample_x[idx] = parameters[key] + + # parse value to sample_y + temp_y = [] + if sample_x in self.samples_x: + idx = self.samples_x.index(sample_x) + temp_y = self.samples_y[idx] + temp_y.append(value) + self.samples_y[idx] = temp_y + + # calculate y aggregation + median = get_median(temp_y) + self.samples_y_aggregation[idx] = [median] + else: + self.samples_x.append(sample_x) + self.samples_y.append([value]) + + # calculate y aggregation + self.samples_y_aggregation.append([value]) + + + def _selection( + self, + samples_x, + samples_y_aggregation, + samples_y, + x_bounds, + x_types, + max_resampling_per_x=3, + threshold_samplessize_exploitation=12, + threshold_samplessize_resampling=50, + no_candidates=False, + minimize_starting_points=None, + minimize_constraints_fun=None): + + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + + next_candidate = None + candidates = [] + samples_size_all = sum([len(i) for i in samples_y]) + samples_size_unique = len(samples_y) + + # ===== STEP 1: Compute the current optimum ===== + gp_model = gp_create_model.create_model( + samples_x, samples_y_aggregation) + lm_current = gp_selection.selection( + "lm", + samples_y_aggregation, + x_bounds, + x_types, + gp_model['model'], + minimize_starting_points, + minimize_constraints_fun=minimize_constraints_fun) + if not lm_current: + return None + logger.info({ + 'hyperparameter': lm_current['hyperparameter'], + 'expected_mu': lm_current['expected_mu'], + 'expected_sigma': lm_current['expected_sigma'], + 'reason': "exploitation_gp" + }) + + if no_candidates is False: + # ===== STEP 2: Get recommended configurations for exploration ==== + results_exploration = gp_selection.selection( + "lc", + samples_y_aggregation, + x_bounds, + x_types, + gp_model['model'], + minimize_starting_points, + minimize_constraints_fun=minimize_constraints_fun) + + if results_exploration is not None: + if _num_past_samples(results_exploration['hyperparameter'], samples_x, samples_y) == 0: + temp_candidate = { + 'hyperparameter': results_exploration['hyperparameter'], + 'expected_mu': results_exploration['expected_mu'], + 'expected_sigma': results_exploration['expected_sigma'], + 'reason': "exploration" + } + candidates.append(temp_candidate) + + logger.info("DEBUG: 1 exploration candidate selected\n") + logger.info(temp_candidate) + else: + logger.info("DEBUG: No suitable exploration candidates were") + + # ===== STEP 3: Get recommended configurations for exploitation === + if samples_size_all >= threshold_samplessize_exploitation: + logger.info("Getting candidates for exploitation...\n") + try: + gmm = gmm_create_model.create_model( + samples_x, samples_y_aggregation) + + if ("discrete_int" in x_types) or ("range_int" in x_types): + results_exploitation = gmm_selection.selection( + x_bounds, + x_types, + gmm['clusteringmodel_good'], + gmm['clusteringmodel_bad'], + minimize_starting_points, + minimize_constraints_fun=minimize_constraints_fun) + else: + # If all parameters are of "range_continuous", + # let's use GMM to generate random starting points + results_exploitation = gmm_selection.selection_r( + x_bounds, + x_types, + gmm['clusteringmodel_good'], + gmm['clusteringmodel_bad'], + num_starting_points=self.selection_num_starting_points, + minimize_constraints_fun=minimize_constraints_fun) + + if results_exploitation is not None: + if _num_past_samples(results_exploitation['hyperparameter'], samples_x, samples_y) == 0: + temp_expected_mu, temp_expected_sigma = \ + gp_prediction.predict(results_exploitation['hyperparameter'], gp_model['model']) + temp_candidate = { + 'hyperparameter': results_exploitation['hyperparameter'], + 'expected_mu': temp_expected_mu, + 'expected_sigma': temp_expected_sigma, + 'reason': "exploitation_gmm" + } + candidates.append(temp_candidate) + + logger.info( + "DEBUG: 1 exploitation_gmm candidate selected\n") + logger.info(temp_candidate) + else: + logger.info( + "DEBUG: No suitable exploitation_gmm candidates were found\n") + + except ValueError as exception: + # The exception: ValueError: Fitting the mixture model failed + # because some components have ill-defined empirical covariance + # (for instance caused by singleton or collapsed samples). + # Try to decrease the number of components, or increase + # reg_covar. + logger.info( + "DEBUG: No suitable exploitation_gmm \ + candidates were found due to exception.") + logger.info(exception) + + # ===== STEP 4: Get a list of outliers ===== + if (threshold_samplessize_resampling is not None) and \ + (samples_size_unique >= threshold_samplessize_resampling): + logger.info("Getting candidates for re-sampling...\n") + results_outliers = gp_outlier_detection.outlierDetection_threaded( + samples_x, samples_y_aggregation) + + if results_outliers is not None: + for results_outlier in results_outliers: # pylint: disable=not-an-iterable + if _num_past_samples(samples_x[results_outlier['samples_idx']], samples_x, samples_y) < max_resampling_per_x: + temp_candidate = {'hyperparameter': samples_x[results_outlier['samples_idx']],\ + 'expected_mu': results_outlier['expected_mu'],\ + 'expected_sigma': results_outlier['expected_sigma'],\ + 'reason': "resampling"} + candidates.append(temp_candidate) + logger.info("DEBUG: %d re-sampling candidates selected\n") + logger.info(temp_candidate) + else: + logger.info( + "DEBUG: No suitable resampling candidates were found\n") + + if candidates: + # ===== STEP 5: Compute the information gain of each candidate + logger.info( + "Evaluating information gain of %d candidates...\n") + next_improvement = 0 + + threads_inputs = [[ + candidate, samples_x, samples_y, x_bounds, x_types, + minimize_constraints_fun, minimize_starting_points + ] for candidate in candidates] + threads_pool = ThreadPool(4) + # Evaluate what would happen if we actually sample each + # candidate + threads_results = threads_pool.map( + _calculate_lowest_mu_threaded, threads_inputs) + threads_pool.close() + threads_pool.join() + + for threads_result in threads_results: + if threads_result['expected_lowest_mu'] < lm_current['expected_mu']: + # Information gain + temp_improvement = threads_result['expected_lowest_mu'] - \ + lm_current['expected_mu'] + + if next_improvement > temp_improvement: + next_improvement = temp_improvement + next_candidate = threads_result['candidate'] + else: + # ===== STEP 6: If we have no candidates, randomly pick one === + logger.info( + "DEBUG: No candidates from exploration, exploitation,\ + and resampling. We will random a candidate for next_candidate\n" + ) + + next_candidate = _rand_with_constraints( + x_bounds, + x_types) if minimize_starting_points is None else minimize_starting_points[0] + next_candidate = lib_data.match_val_type( + next_candidate, x_bounds, x_types) + expected_mu, expected_sigma = gp_prediction.predict( + next_candidate, gp_model['model']) + next_candidate = { + 'hyperparameter': next_candidate, + 'reason': "random", + 'expected_mu': expected_mu, + 'expected_sigma': expected_sigma} + + # STEP 7: If current optimal hyperparameter occurs in the history + # or exploration probability is less than the threshold, take next + # config as exploration step + outputs = self._pack_output(lm_current['hyperparameter']) + ap = random.uniform(0, 1) + if outputs in self.total_data or ap <= self.exploration_probability: + if next_candidate is not None: + outputs = self._pack_output(next_candidate['hyperparameter']) + else: + random_parameter = _rand_init(x_bounds, x_types, 1)[0] + outputs = self._pack_output(random_parameter) + self.total_data.append(outputs) + return outputs + + def import_data(self, data): + """ + Import additional data for tuning + + Parameters + ---------- + data : a list of dict + each of which has at least two keys: 'parameter' and 'value'. + """ + _completed_num = 0 + for trial_info in data: + logger.info("Importing data, current processing progress %s / %s", _completed_num, len(data)) + _completed_num += 1 + assert "parameter" in trial_info + _params = trial_info["parameter"] + assert "value" in trial_info + _value = trial_info['value'] + if not _value: + logger.info("Useless trial data, value is %s, skip this trial data.", _value) + continue + self.supplement_data_num += 1 + _parameter_id = '_'.join( + ["ImportData", str(self.supplement_data_num)]) + self.total_data.append(_params) + self.receive_trial_result( + parameter_id=_parameter_id, + parameters=_params, + value=_value) + logger.info("Successfully import data to metis tuner.") + + +def _rand_with_constraints(x_bounds, x_types): + outputs = None + x_bounds_withconstraints = [x_bounds[i] for i in CONSTRAINT_PARAMS_IDX] + x_types_withconstraints = [x_types[i] for i in CONSTRAINT_PARAMS_IDX] + + x_val_withconstraints = lib_constraint_summation.rand( + x_bounds_withconstraints, + x_types_withconstraints, + CONSTRAINT_LOWERBOUND, + CONSTRAINT_UPPERBOUND) + if not x_val_withconstraints: + outputs = [None] * len(x_bounds) + + for i, _ in enumerate(CONSTRAINT_PARAMS_IDX): + outputs[CONSTRAINT_PARAMS_IDX[i]] = x_val_withconstraints[i] + + for i, output in enumerate(outputs): + if not output: + outputs[i] = random.randint(x_bounds[i][0], x_bounds[i][1]) + return outputs + + +def _calculate_lowest_mu_threaded(inputs): + [candidate, samples_x, samples_y, x_bounds, x_types, + minimize_constraints_fun, minimize_starting_points] = inputs + + outputs = {"candidate": candidate, "expected_lowest_mu": None} + + for expected_mu in [ + candidate['expected_mu'] + + 1.96 * + candidate['expected_sigma'], + candidate['expected_mu'] - + 1.96 * + candidate['expected_sigma']]: + temp_samples_x = copy.deepcopy(samples_x) + temp_samples_y = copy.deepcopy(samples_y) + + try: + idx = temp_samples_x.index(candidate['hyperparameter']) + # This handles the case of re-sampling a potential outlier + temp_samples_y[idx].append(expected_mu) + except ValueError: + temp_samples_x.append(candidate['hyperparameter']) + temp_samples_y.append([expected_mu]) + + # Aggregates multiple observation of the sample sampling points + temp_y_aggregation = [statistics.median( + temp_sample_y) for temp_sample_y in temp_samples_y] + temp_gp = gp_create_model.create_model( + temp_samples_x, temp_y_aggregation) + temp_results = gp_selection.selection( + "lm", + temp_y_aggregation, + x_bounds, + x_types, + temp_gp['model'], + minimize_starting_points, + minimize_constraints_fun=minimize_constraints_fun) + + if outputs["expected_lowest_mu"] is None \ + or outputs["expected_lowest_mu"] > temp_results['expected_mu']: + outputs["expected_lowest_mu"] = temp_results['expected_mu'] + + return outputs + + +def _num_past_samples(x, samples_x, samples_y): + try: + idx = samples_x.index(x) + return len(samples_y[idx]) + except ValueError: + logger.info("x not in sample_x") + return 0 + + +def _rand_init(x_bounds, x_types, selection_num_starting_points): + ''' + Random sample some init seed within bounds. + ''' + return [lib_data.rand(x_bounds, x_types) for i + in range(0, selection_num_starting_points)] + + +def get_median(temp_list): + """ + Return median + """ + num = len(temp_list) + temp_list.sort() + print(temp_list) + if num % 2 == 0: + median = (temp_list[int(num / 2)] + temp_list[int(num / 2) - 1]) / 2 + else: + median = temp_list[int(num / 2)] + return median diff --git a/new_impl/cv/third_party/nni_new/algorithms/hpo/metis_tuner/requirments.txt b/new_impl/cv/third_party/nni_new/algorithms/hpo/metis_tuner/requirments.txt new file mode 100644 index 0000000000000000000000000000000000000000..3dfc2232a18c19a544eb3fb5b2f132e185958f98 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/hpo/metis_tuner/requirments.txt @@ -0,0 +1 @@ +scikit-learn>=0.23.2 diff --git a/new_impl/cv/third_party/nni_new/algorithms/hpo/networkmorphism_tuner/__init__.py b/new_impl/cv/third_party/nni_new/algorithms/hpo/networkmorphism_tuner/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..b60da9c3892a06392397b7b8549de6615f4569cf --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/hpo/networkmorphism_tuner/__init__.py @@ -0,0 +1 @@ +from .networkmorphism_tuner import NetworkMorphismTuner, NetworkMorphismClassArgsValidator diff --git a/new_impl/cv/third_party/nni_new/algorithms/hpo/networkmorphism_tuner/bayesian.py b/new_impl/cv/third_party/nni_new/algorithms/hpo/networkmorphism_tuner/bayesian.py new file mode 100644 index 0000000000000000000000000000000000000000..54c1996dc7708568a7065325d24a8b83fdb5a40f --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/hpo/networkmorphism_tuner/bayesian.py @@ -0,0 +1,482 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import math +import random +from copy import deepcopy +from functools import total_ordering +from queue import PriorityQueue + +import numpy as np +from scipy.linalg import LinAlgError, cho_solve, cholesky, solve_triangular +from scipy.optimize import linear_sum_assignment +from sklearn.metrics.pairwise import rbf_kernel + +from nni.utils import OptimizeMode +from .graph_transformer import transform +from .utils import Constant +from .layers import is_layer + + +def layer_distance(a, b): + """The distance between two layers.""" + # pylint: disable=unidiomatic-typecheck + if not isinstance(a, type(b)): + return 1.0 + if is_layer(a, "Conv"): + att_diff = [ + (a.filters, b.filters), + (a.kernel_size, b.kernel_size), + (a.stride, b.stride), + ] + return attribute_difference(att_diff) + if is_layer(a, "Pooling"): + att_diff = [ + (a.padding, b.padding), + (a.kernel_size, b.kernel_size), + (a.stride, b.stride), + ] + return attribute_difference(att_diff) + return 0.0 + + +def attribute_difference(att_diff): + ''' The attribute distance. + ''' + + ret = 0 + for a_value, b_value in att_diff: + if max(a_value, b_value) == 0: + ret += 0 + else: + ret += abs(a_value - b_value) * 1.0 / max(a_value, b_value) + return ret * 1.0 / len(att_diff) + + +def layers_distance(list_a, list_b): + """The distance between the layers of two neural networks.""" + len_a = len(list_a) + len_b = len(list_b) + f = np.zeros((len_a + 1, len_b + 1)) + f[-1][-1] = 0 + for i in range(-1, len_a): + f[i][-1] = i + 1 + for j in range(-1, len_b): + f[-1][j] = j + 1 + for i in range(len_a): + for j in range(len_b): + f[i][j] = min( + f[i][j - 1] + 1, + f[i - 1][j] + 1, + f[i - 1][j - 1] + layer_distance(list_a[i], list_b[j]), + ) + return f[len_a - 1][len_b - 1] + + +def skip_connection_distance(a, b): + """The distance between two skip-connections.""" + if a[2] != b[2]: + return 1.0 + len_a = abs(a[1] - a[0]) + len_b = abs(b[1] - b[0]) + return (abs(a[0] - b[0]) + abs(len_a - len_b)) / \ + (max(a[0], b[0]) + max(len_a, len_b)) + + +def skip_connections_distance(list_a, list_b): + """The distance between the skip-connections of two neural networks.""" + distance_matrix = np.zeros((len(list_a), len(list_b))) + for i, a in enumerate(list_a): + for j, b in enumerate(list_b): + distance_matrix[i][j] = skip_connection_distance(a, b) + return distance_matrix[linear_sum_assignment(distance_matrix)].sum() + abs( + len(list_a) - len(list_b) + ) + + +def edit_distance(x, y): + """The distance between two neural networks. + Args: + x: An instance of NetworkDescriptor. + y: An instance of NetworkDescriptor + Returns: + The edit-distance between x and y. + """ + + ret = layers_distance(x.layers, y.layers) + ret += Constant.KERNEL_LAMBDA * skip_connections_distance( + x.skip_connections, y.skip_connections + ) + return ret + + +class IncrementalGaussianProcess: + """Gaussian process regressor. + Attributes: + alpha: A hyperparameter. + """ + + def __init__(self): + self.alpha = 1e-10 + self._distance_matrix = None + self._x = None + self._y = None + self._first_fitted = False + self._l_matrix = None + self._alpha_vector = None + + @property + def kernel_matrix(self): + ''' Kernel matric. + ''' + return self._distance_matrix + + def fit(self, train_x, train_y): + """ Fit the regressor with more data. + Args: + train_x: A list of NetworkDescriptor. + train_y: A list of metric values. + """ + if self.first_fitted: + self.incremental_fit(train_x, train_y) + else: + self.first_fit(train_x, train_y) + + def incremental_fit(self, train_x, train_y): + """ Incrementally fit the regressor. """ + if not self._first_fitted: + raise ValueError( + "The first_fit function needs to be called first.") + + train_x, train_y = np.array(train_x), np.array(train_y) + + # Incrementally compute K + up_right_k = edit_distance_matrix(self._x, train_x) + down_left_k = np.transpose(up_right_k) + down_right_k = edit_distance_matrix(train_x) + up_k = np.concatenate((self._distance_matrix, up_right_k), axis=1) + down_k = np.concatenate((down_left_k, down_right_k), axis=1) + temp_distance_matrix = np.concatenate((up_k, down_k), axis=0) + k_matrix = bourgain_embedding_matrix(temp_distance_matrix) + diagonal = np.diag_indices_from(k_matrix) + diagonal = (diagonal[0][-len(train_x):], diagonal[1][-len(train_x):]) + k_matrix[diagonal] += self.alpha + + try: + self._l_matrix = cholesky(k_matrix, lower=True) # Line 2 + except LinAlgError: + return self + + self._x = np.concatenate((self._x, train_x), axis=0) + self._y = np.concatenate((self._y, train_y), axis=0) + self._distance_matrix = temp_distance_matrix + + self._alpha_vector = cho_solve( + (self._l_matrix, True), self._y) # Line 3 + + return self + + @property + def first_fitted(self): + ''' if it is firsr fitted + ''' + return self._first_fitted + + def first_fit(self, train_x, train_y): + """ Fit the regressor for the first time. """ + train_x, train_y = np.array(train_x), np.array(train_y) + + self._x = np.copy(train_x) + self._y = np.copy(train_y) + + self._distance_matrix = edit_distance_matrix(self._x) + k_matrix = bourgain_embedding_matrix(self._distance_matrix) + k_matrix[np.diag_indices_from(k_matrix)] += self.alpha + + self._l_matrix = cholesky(k_matrix, lower=True) # Line 2 + + self._alpha_vector = cho_solve( + (self._l_matrix, True), self._y) # Line 3 + + self._first_fitted = True + return self + + def predict(self, train_x): + """Predict the result. + Args: + train_x: A list of NetworkDescriptor. + Returns: + y_mean: The predicted mean. + y_std: The predicted standard deviation. + """ + k_trans = np.exp(-np.power(edit_distance_matrix(train_x, self._x), 2)) + y_mean = k_trans.dot(self._alpha_vector) # Line 4 (y_mean = f_star) + + # compute inverse K_inv of K based on its Cholesky + # decomposition L and its inverse L_inv + l_inv = solve_triangular( + self._l_matrix.T, np.eye( + self._l_matrix.shape[0])) + k_inv = l_inv.dot(l_inv.T) + # Compute variance of predictive distribution + y_var = np.ones(len(train_x), dtype=np.float) + y_var -= np.einsum("ij,ij->i", np.dot(k_trans, k_inv), k_trans) + + # Check if any of the variances is negative because of + # numerical issues. If yes: set the variance to 0. + y_var_negative = y_var < 0 + if np.any(y_var_negative): + y_var[y_var_negative] = 0.0 + return y_mean, np.sqrt(y_var) + + +def edit_distance_matrix(train_x, train_y=None): + """Calculate the edit distance. + Args: + train_x: A list of neural architectures. + train_y: A list of neural architectures. + Returns: + An edit-distance matrix. + """ + if train_y is None: + ret = np.zeros((train_x.shape[0], train_x.shape[0])) + for x_index, x in enumerate(train_x): + for y_index, y in enumerate(train_x): + if x_index == y_index: + ret[x_index][y_index] = 0 + elif x_index < y_index: + ret[x_index][y_index] = edit_distance(x, y) + else: + ret[x_index][y_index] = ret[y_index][x_index] + return ret + ret = np.zeros((train_x.shape[0], train_y.shape[0])) + for x_index, x in enumerate(train_x): + for y_index, y in enumerate(train_y): + ret[x_index][y_index] = edit_distance(x, y) + return ret + + +def vector_distance(a, b): + """The Euclidean distance between two vectors.""" + a = np.array(a) + b = np.array(b) + return np.linalg.norm(a - b) + + +def bourgain_embedding_matrix(distance_matrix): + """Use Bourgain algorithm to embed the neural architectures based on their edit-distance. + Args: + distance_matrix: A matrix of edit-distances. + Returns: + A matrix of distances after embedding. + """ + distance_matrix = np.array(distance_matrix) + n = len(distance_matrix) + if n == 1: + return distance_matrix + np.random.seed(123) + distort_elements = [] + r = range(n) + k = int(math.ceil(math.log(n) / math.log(2) - 1)) + t = int(math.ceil(math.log(n))) + counter = 0 + for i in range(0, k + 1): + for t in range(t): + s = np.random.choice(r, 2 ** i) + for j in r: + d = min([distance_matrix[j][s] for s in s]) + counter += len(s) + if i == 0 and t == 0: + distort_elements.append([d]) + else: + distort_elements[j].append(d) + return rbf_kernel(distort_elements, distort_elements) + + +class BayesianOptimizer: + """ A Bayesian optimizer for neural architectures. + Attributes: + searcher: The Searcher which is calling the Bayesian optimizer. + t_min: The minimum temperature for simulated annealing. + metric: An instance of the Metric subclasses. + gpr: A GaussianProcessRegressor for bayesian optimization. + beta: The beta in acquisition function. (refer to our paper) + search_tree: The network morphism search tree. + """ + + def __init__(self, searcher, t_min, optimizemode, beta=None): + self.searcher = searcher + self.t_min = t_min + self.optimizemode = optimizemode + self.gpr = IncrementalGaussianProcess() + self.beta = beta if beta is not None else Constant.BETA + self.search_tree = SearchTree() + + def fit(self, x_queue, y_queue): + """ Fit the optimizer with new architectures and performances. + Args: + x_queue: A list of NetworkDescriptor. + y_queue: A list of metric values. + """ + self.gpr.fit(x_queue, y_queue) + + def generate(self, descriptors): + """Generate new architecture. + Args: + descriptors: All the searched neural architectures. + Returns: + graph: An instance of Graph. A morphed neural network with weights. + father_id: The father node ID in the search tree. + """ + model_ids = self.search_tree.adj_list.keys() + + target_graph = None + father_id = None + descriptors = deepcopy(descriptors) + elem_class = Elem + if self.optimizemode is OptimizeMode.Maximize: + elem_class = ReverseElem + + # Initialize the priority queue. + pq = PriorityQueue() + temp_list = [] + for model_id in model_ids: + metric_value = self.searcher.get_metric_value_by_id(model_id) + temp_list.append((metric_value, model_id)) + temp_list = sorted(temp_list) + for metric_value, model_id in temp_list: + graph = self.searcher.load_model_by_id(model_id) + graph.clear_operation_history() + graph.clear_weights() + pq.put(elem_class(metric_value, model_id, graph)) + + t = 1.0 + t_min = self.t_min + alpha = 0.9 + opt_acq = self._get_init_opt_acq_value() + while not pq.empty() and t > t_min: + elem = pq.get() + if self.optimizemode is OptimizeMode.Maximize: + temp_exp = min((elem.metric_value - opt_acq) / t, 1.0) + else: + temp_exp = min((opt_acq - elem.metric_value) / t, 1.0) + ap = math.exp(temp_exp) + if ap >= random.uniform(0, 1): + for temp_graph in transform(elem.graph): + if contain(descriptors, temp_graph.extract_descriptor()): + continue + + temp_acq_value = self.acq(temp_graph) + pq.put( + elem_class( + temp_acq_value, + elem.father_id, + temp_graph)) + descriptors.append(temp_graph.extract_descriptor()) + if self._accept_new_acq_value(opt_acq, temp_acq_value): + opt_acq = temp_acq_value + father_id = elem.father_id + target_graph = deepcopy(temp_graph) + t *= alpha + + # Did not found a not duplicated architecture + if father_id is None: + return None, None + nm_graph = self.searcher.load_model_by_id(father_id) + for args in target_graph.operation_history: + getattr(nm_graph, args[0])(*list(args[1:])) + return nm_graph, father_id + + def acq(self, graph): + ''' estimate the value of generated graph + ''' + mean, std = self.gpr.predict(np.array([graph.extract_descriptor()])) + if self.optimizemode is OptimizeMode.Maximize: + return mean + self.beta * std + return mean - self.beta * std + + def _get_init_opt_acq_value(self): + if self.optimizemode is OptimizeMode.Maximize: + return -np.inf + return np.inf + + def _accept_new_acq_value(self, opt_acq, temp_acq_value): + if temp_acq_value > opt_acq and self.optimizemode is OptimizeMode.Maximize: + return True + if temp_acq_value < opt_acq and not self.optimizemode is OptimizeMode.Maximize: + return True + return False + + def add_child(self, father_id, model_id): + ''' add child to the search tree + Arguments: + father_id {int} -- father id + model_id {int} -- model id + ''' + + self.search_tree.add_child(father_id, model_id) + + +@total_ordering +class Elem: + """Elements to be sorted according to metric value.""" + + def __init__(self, metric_value, father_id, graph): + self.father_id = father_id + self.graph = graph + self.metric_value = metric_value + + def __eq__(self, other): + return self.metric_value == other.metric_value + + def __lt__(self, other): + return self.metric_value < other.metric_value + + +class ReverseElem(Elem): + """Elements to be reversely sorted according to metric value.""" + + def __lt__(self, other): + return self.metric_value > other.metric_value + + +def contain(descriptors, target_descriptor): + """Check if the target descriptor is in the descriptors.""" + for descriptor in descriptors: + if edit_distance(descriptor, target_descriptor) < 1e-5: + return True + return False + + +class SearchTree: + """The network morphism search tree.""" + + def __init__(self): + self.root = None + self.adj_list = {} + + def add_child(self, u, v): + ''' add child to search tree itself. + Arguments: + u {int} -- father id + v {int} -- child id + ''' + + if u == -1: + self.root = v + self.adj_list[v] = [] + return + if v not in self.adj_list[u]: + self.adj_list[u].append(v) + if v not in self.adj_list: + self.adj_list[v] = [] + + def get_dict(self, u=None): + """ A recursive function to return the content of the tree in a dict.""" + if u is None: + return self.get_dict(self.root) + children = [] + for v in self.adj_list[u]: + children.append(self.get_dict(v)) + ret = {"name": u, "children": children} + return ret diff --git a/new_impl/cv/third_party/nni_new/algorithms/hpo/networkmorphism_tuner/graph.py b/new_impl/cv/third_party/nni_new/algorithms/hpo/networkmorphism_tuner/graph.py new file mode 100644 index 0000000000000000000000000000000000000000..9c96b6c2f07088aceb0cb029de02cb4b2f16ae32 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/hpo/networkmorphism_tuner/graph.py @@ -0,0 +1,995 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import json +from collections.abc import Iterable +from copy import deepcopy, copy +from queue import Queue + +import numpy as np +import torch + +from .layer_transformer import ( + add_noise, + wider_bn, + wider_next_conv, + wider_next_dense, + wider_pre_conv, + wider_pre_dense, + init_dense_weight, + init_conv_weight, + init_bn_weight, +) +from .layers import ( + StubAdd, + StubConcatenate, + StubReLU, + get_batch_norm_class, + get_conv_class, + is_layer, + layer_width, + set_keras_weight_to_stub, + set_stub_weight_to_keras, + set_stub_weight_to_torch, + set_torch_weight_to_stub, + to_real_keras_layer, + layer_description_extractor, + layer_description_builder, +) +from .utils import Constant + + +class NetworkDescriptor: + """A class describing the neural architecture for neural network kernel. + It only record the width of convolutional and dense layers, and the skip-connection types and positions. + """ + + CONCAT_CONNECT = "concat" + ADD_CONNECT = "add" + + def __init__(self): + self.skip_connections = [] + self.layers = [] + + @property + def n_layers(self): + return len(self.layers) + + def add_skip_connection(self, u, v, connection_type): + """ Add a skip-connection to the descriptor. + Args: + u: Number of convolutional layers before the starting point. + v: Number of convolutional layers before the ending point. + connection_type: Must be either CONCAT_CONNECT or ADD_CONNECT. + """ + if connection_type not in [self.CONCAT_CONNECT, self.ADD_CONNECT]: + raise ValueError( + "connection_type should be NetworkDescriptor.CONCAT_CONNECT " + "or NetworkDescriptor.ADD_CONNECT." + ) + self.skip_connections.append((u, v, connection_type)) + + def to_json(self): + ''' NetworkDescriptor to json representation + ''' + + skip_list = [] + for u, v, connection_type in self.skip_connections: + skip_list.append({"from": u, "to": v, "type": connection_type}) + return {"node_list": self.layers, "skip_list": skip_list} + + def add_layer(self, layer): + ''' add one layer + ''' + + self.layers.append(layer) + + +class Node: + """A class for intermediate output tensor (node) in the Graph. + Attributes: + shape: A tuple describing the shape of the tensor. + """ + + def __init__(self, shape): + self.shape = shape + + +class Graph: + """A class representing the neural architecture graph of a model. + Graph extracts the neural architecture graph from a model. + Each node in the graph is a intermediate tensor between layers. + Each layer is an edge in the graph. + Notably, multiple edges may refer to the same layer. + (e.g. Add layer is adding two tensor into one tensor. So it is related to two edges.) + Attributes: + weighted: A boolean of whether the weights and biases in the neural network + should be included in the graph. + input_shape: A tuple of integers, which does not include the batch axis. + node_list: A list of integers. The indices of the list are the identifiers. + layer_list: A list of stub layers. The indices of the list are the identifiers. + node_to_id: A dict instance mapping from node integers to their identifiers. + layer_to_id: A dict instance mapping from stub layers to their identifiers. + layer_id_to_input_node_ids: A dict instance mapping from layer identifiers + to their input nodes identifiers. + layer_id_to_output_node_ids: A dict instance mapping from layer identifiers + to their output nodes identifiers. + adj_list: A two dimensional list. The adjacency list of the graph. The first dimension is + identified by tensor identifiers. In each edge list, the elements are two-element tuples + of (tensor identifier, layer identifier). + reverse_adj_list: A reverse adjacent list in the same format as adj_list. + operation_history: A list saving all the network morphism operations. + vis: A dictionary of temporary storage for whether an local operation has been done + during the network morphism. + """ + + def __init__(self, input_shape, weighted=True): + """Initializer for Graph. + """ + self.input_shape = input_shape + self.weighted = weighted + self.node_list = [] + self.layer_list = [] + # node id start with 0 + self.node_to_id = {} + self.layer_to_id = {} + self.layer_id_to_input_node_ids = {} + self.layer_id_to_output_node_ids = {} + self.adj_list = {} + self.reverse_adj_list = {} + self.operation_history = [] + self.n_dim = len(input_shape) - 1 + self.conv = get_conv_class(self.n_dim) + self.batch_norm = get_batch_norm_class(self.n_dim) + + self.vis = None + self._add_node(Node(input_shape)) + + def add_layer(self, layer, input_node_id): + """Add a layer to the Graph. + Args: + layer: An instance of the subclasses of StubLayer in layers.py. + input_node_id: An integer. The ID of the input node of the layer. + Returns: + output_node_id: An integer. The ID of the output node of the layer. + """ + if isinstance(input_node_id, Iterable): + layer.input = list(map(lambda x: self.node_list[x], input_node_id)) + output_node_id = self._add_node(Node(layer.output_shape)) + for node_id in input_node_id: + self._add_edge(layer, node_id, output_node_id) + + else: + layer.input = self.node_list[input_node_id] + output_node_id = self._add_node(Node(layer.output_shape)) + self._add_edge(layer, input_node_id, output_node_id) + + layer.output = self.node_list[output_node_id] + return output_node_id + + def clear_operation_history(self): + self.operation_history = [] + + @property + def n_nodes(self): + """Return the number of nodes in the model.""" + return len(self.node_list) + + @property + def n_layers(self): + """Return the number of layers in the model.""" + return len(self.layer_list) + + def _add_node(self, node): + """Add a new node to node_list and give the node an ID. + Args: + node: An instance of Node. + Returns: + node_id: An integer. + """ + node_id = len(self.node_list) + self.node_to_id[node] = node_id + self.node_list.append(node) + self.adj_list[node_id] = [] + self.reverse_adj_list[node_id] = [] + return node_id + + def _add_edge(self, layer, input_id, output_id): + """Add a new layer to the graph. The nodes should be created in advance.""" + + if layer in self.layer_to_id: + layer_id = self.layer_to_id[layer] + if input_id not in self.layer_id_to_input_node_ids[layer_id]: + self.layer_id_to_input_node_ids[layer_id].append(input_id) + if output_id not in self.layer_id_to_output_node_ids[layer_id]: + self.layer_id_to_output_node_ids[layer_id].append(output_id) + else: + layer_id = len(self.layer_list) + self.layer_list.append(layer) + self.layer_to_id[layer] = layer_id + self.layer_id_to_input_node_ids[layer_id] = [input_id] + self.layer_id_to_output_node_ids[layer_id] = [output_id] + + self.adj_list[input_id].append((output_id, layer_id)) + self.reverse_adj_list[output_id].append((input_id, layer_id)) + + def _redirect_edge(self, u_id, v_id, new_v_id): + """Redirect the layer to a new node. + Change the edge originally from `u_id` to `v_id` into an edge from `u_id` to `new_v_id` + while keeping all other property of the edge the same. + """ + layer_id = None + for index, edge_tuple in enumerate(self.adj_list[u_id]): + if edge_tuple[0] == v_id: + layer_id = edge_tuple[1] + self.adj_list[u_id][index] = (new_v_id, layer_id) + self.layer_list[layer_id].output = self.node_list[new_v_id] + break + + for index, edge_tuple in enumerate(self.reverse_adj_list[v_id]): + if edge_tuple[0] == u_id: + layer_id = edge_tuple[1] + self.reverse_adj_list[v_id].remove(edge_tuple) + break + self.reverse_adj_list[new_v_id].append((u_id, layer_id)) + for index, value in enumerate( + self.layer_id_to_output_node_ids[layer_id]): + if value == v_id: + self.layer_id_to_output_node_ids[layer_id][index] = new_v_id + break + + def _replace_layer(self, layer_id, new_layer): + """Replace the layer with a new layer.""" + old_layer = self.layer_list[layer_id] + new_layer.input = old_layer.input + new_layer.output = old_layer.output + new_layer.output.shape = new_layer.output_shape + self.layer_list[layer_id] = new_layer + self.layer_to_id[new_layer] = layer_id + self.layer_to_id.pop(old_layer) + + @property + def topological_order(self): + """Return the topological order of the node IDs from the input node to the output node.""" + q = Queue() + in_degree = {} + for i in range(self.n_nodes): + in_degree[i] = 0 + for u in range(self.n_nodes): + for v, _ in self.adj_list[u]: + in_degree[v] += 1 + for i in range(self.n_nodes): + if in_degree[i] == 0: + q.put(i) + + order_list = [] + while not q.empty(): + u = q.get() + order_list.append(u) + for v, _ in self.adj_list[u]: + in_degree[v] -= 1 + if in_degree[v] == 0: + q.put(v) + return order_list + + def _get_pooling_layers(self, start_node_id, end_node_id): + """Given two node IDs, return all the pooling layers between them.""" + layer_list = [] + node_list = [start_node_id] + assert self._depth_first_search(end_node_id, layer_list, node_list) + ret = [] + for layer_id in layer_list: + layer = self.layer_list[layer_id] + if is_layer(layer, "Pooling"): + ret.append(layer) + elif is_layer(layer, "Conv") and layer.stride != 1: + ret.append(layer) + return ret + + def _depth_first_search(self, target_id, layer_id_list, node_list): + """Search for all the layers and nodes down the path. + A recursive function to search all the layers and nodes between the node in the node_list + and the node with target_id.""" + assert len(node_list) <= self.n_nodes + u = node_list[-1] + if u == target_id: + return True + + for v, layer_id in self.adj_list[u]: + layer_id_list.append(layer_id) + node_list.append(v) + if self._depth_first_search(target_id, layer_id_list, node_list): + return True + layer_id_list.pop() + node_list.pop() + + return False + + def _search(self, u, start_dim, total_dim, n_add): + """Search the graph for all the layers to be widened caused by an operation. + It is an recursive function with duplication check to avoid deadlock. + It searches from a starting node u until the corresponding layers has been widened. + Args: + u: The starting node ID. + start_dim: The position to insert the additional dimensions. + total_dim: The total number of dimensions the layer has before widening. + n_add: The number of dimensions to add. + """ + if (u, start_dim, total_dim, n_add) in self.vis: + return + self.vis[(u, start_dim, total_dim, n_add)] = True + for v, layer_id in self.adj_list[u]: + layer = self.layer_list[layer_id] + + if is_layer(layer, "Conv"): + new_layer = wider_next_conv( + layer, start_dim, total_dim, n_add, self.weighted + ) + self._replace_layer(layer_id, new_layer) + + elif is_layer(layer, "Dense"): + new_layer = wider_next_dense( + layer, start_dim, total_dim, n_add, self.weighted + ) + self._replace_layer(layer_id, new_layer) + + elif is_layer(layer, "BatchNormalization"): + new_layer = wider_bn( + layer, start_dim, total_dim, n_add, self.weighted) + self._replace_layer(layer_id, new_layer) + self._search(v, start_dim, total_dim, n_add) + + elif is_layer(layer, "Concatenate"): + if self.layer_id_to_input_node_ids[layer_id][1] == u: + # u is on the right of the concat + # next_start_dim += next_total_dim - total_dim + left_dim = self._upper_layer_width( + self.layer_id_to_input_node_ids[layer_id][0] + ) + next_start_dim = start_dim + left_dim + next_total_dim = total_dim + left_dim + else: + next_start_dim = start_dim + next_total_dim = total_dim + self._upper_layer_width( + self.layer_id_to_input_node_ids[layer_id][1] + ) + self._search(v, next_start_dim, next_total_dim, n_add) + + else: + self._search(v, start_dim, total_dim, n_add) + + for v, layer_id in self.reverse_adj_list[u]: + layer = self.layer_list[layer_id] + if is_layer(layer, "Conv"): + new_layer = wider_pre_conv(layer, n_add, self.weighted) + self._replace_layer(layer_id, new_layer) + elif is_layer(layer, "Dense"): + new_layer = wider_pre_dense(layer, n_add, self.weighted) + self._replace_layer(layer_id, new_layer) + elif is_layer(layer, "Concatenate"): + continue + else: + self._search(v, start_dim, total_dim, n_add) + + def _upper_layer_width(self, u): + for v, layer_id in self.reverse_adj_list[u]: + layer = self.layer_list[layer_id] + if is_layer(layer, "Conv") or is_layer(layer, "Dense"): + return layer_width(layer) + elif is_layer(layer, "Concatenate"): + a = self.layer_id_to_input_node_ids[layer_id][0] + b = self.layer_id_to_input_node_ids[layer_id][1] + return self._upper_layer_width(a) + self._upper_layer_width(b) + else: + return self._upper_layer_width(v) + return self.node_list[0].shape[-1] + + def to_deeper_model(self, target_id, new_layer): + """Insert a relu-conv-bn block after the target block. + Args: + target_id: A convolutional layer ID. The new block should be inserted after the block. + new_layer: An instance of StubLayer subclasses. + """ + self.operation_history.append( + ("to_deeper_model", target_id, new_layer)) + input_id = self.layer_id_to_input_node_ids[target_id][0] + output_id = self.layer_id_to_output_node_ids[target_id][0] + if self.weighted: + if is_layer(new_layer, "Dense"): + init_dense_weight(new_layer) + elif is_layer(new_layer, "Conv"): + init_conv_weight(new_layer) + elif is_layer(new_layer, "BatchNormalization"): + init_bn_weight(new_layer) + + self._insert_new_layers([new_layer], input_id, output_id) + + def to_wider_model(self, pre_layer_id, n_add): + """Widen the last dimension of the output of the pre_layer. + Args: + pre_layer_id: The ID of a convolutional layer or dense layer. + n_add: The number of dimensions to add. + """ + self.operation_history.append(("to_wider_model", pre_layer_id, n_add)) + pre_layer = self.layer_list[pre_layer_id] + output_id = self.layer_id_to_output_node_ids[pre_layer_id][0] + dim = layer_width(pre_layer) + self.vis = {} + self._search(output_id, dim, dim, n_add) + # Update the tensor shapes. + for u in self.topological_order: + for v, layer_id in self.adj_list[u]: + self.node_list[v].shape = self.layer_list[layer_id].output_shape + + def _insert_new_layers(self, new_layers, start_node_id, end_node_id): + """Insert the new_layers after the node with start_node_id.""" + new_node_id = self._add_node(deepcopy(self.node_list[end_node_id])) + temp_output_id = new_node_id + for layer in new_layers[:-1]: + temp_output_id = self.add_layer(layer, temp_output_id) + + self._add_edge(new_layers[-1], temp_output_id, end_node_id) + new_layers[-1].input = self.node_list[temp_output_id] + new_layers[-1].output = self.node_list[end_node_id] + self._redirect_edge(start_node_id, end_node_id, new_node_id) + + def _block_end_node(self, layer_id, block_size): + ret = self.layer_id_to_output_node_ids[layer_id][0] + for _ in range(block_size - 2): + ret = self.adj_list[ret][0][0] + return ret + + def _dense_block_end_node(self, layer_id): + return self.layer_id_to_input_node_ids[layer_id][0] + + def _conv_block_end_node(self, layer_id): + """Get the input node ID of the last layer in the block by layer ID. + Return the input node ID of the last layer in the convolutional block. + Args: + layer_id: the convolutional layer ID. + """ + return self._block_end_node(layer_id, Constant.CONV_BLOCK_DISTANCE) + + def to_add_skip_model(self, start_id, end_id): + """Add a weighted add skip-connection from after start node to end node. + Args: + start_id: The convolutional layer ID, after which to start the skip-connection. + end_id: The convolutional layer ID, after which to end the skip-connection. + """ + self.operation_history.append(("to_add_skip_model", start_id, end_id)) + filters_end = self.layer_list[end_id].output.shape[-1] + filters_start = self.layer_list[start_id].output.shape[-1] + start_node_id = self.layer_id_to_output_node_ids[start_id][0] + + pre_end_node_id = self.layer_id_to_input_node_ids[end_id][0] + end_node_id = self.layer_id_to_output_node_ids[end_id][0] + + skip_output_id = self._insert_pooling_layer_chain( + start_node_id, end_node_id) + + # Add the conv layer + new_conv_layer = get_conv_class( + self.n_dim)( + filters_start, + filters_end, + 1) + skip_output_id = self.add_layer(new_conv_layer, skip_output_id) + + # Add the add layer. + add_input_node_id = self._add_node( + deepcopy(self.node_list[end_node_id])) + add_layer = StubAdd() + + self._redirect_edge(pre_end_node_id, end_node_id, add_input_node_id) + self._add_edge(add_layer, add_input_node_id, end_node_id) + self._add_edge(add_layer, skip_output_id, end_node_id) + add_layer.input = [ + self.node_list[add_input_node_id], + self.node_list[skip_output_id], + ] + add_layer.output = self.node_list[end_node_id] + self.node_list[end_node_id].shape = add_layer.output_shape + + # Set weights to the additional conv layer. + if self.weighted: + filter_shape = (1,) * self.n_dim + weights = np.zeros((filters_end, filters_start) + filter_shape) + bias = np.zeros(filters_end) + new_conv_layer.set_weights( + (add_noise(weights, np.array([0, 1])), add_noise( + bias, np.array([0, 1]))) + ) + + def to_concat_skip_model(self, start_id, end_id): + """Add a weighted add concatenate connection from after start node to end node. + Args: + start_id: The convolutional layer ID, after which to start the skip-connection. + end_id: The convolutional layer ID, after which to end the skip-connection. + """ + self.operation_history.append( + ("to_concat_skip_model", start_id, end_id)) + filters_end = self.layer_list[end_id].output.shape[-1] + filters_start = self.layer_list[start_id].output.shape[-1] + start_node_id = self.layer_id_to_output_node_ids[start_id][0] + + pre_end_node_id = self.layer_id_to_input_node_ids[end_id][0] + end_node_id = self.layer_id_to_output_node_ids[end_id][0] + + skip_output_id = self._insert_pooling_layer_chain( + start_node_id, end_node_id) + + concat_input_node_id = self._add_node( + deepcopy(self.node_list[end_node_id])) + self._redirect_edge(pre_end_node_id, end_node_id, concat_input_node_id) + + concat_layer = StubConcatenate() + concat_layer.input = [ + self.node_list[concat_input_node_id], + self.node_list[skip_output_id], + ] + concat_output_node_id = self._add_node(Node(concat_layer.output_shape)) + self._add_edge( + concat_layer, + concat_input_node_id, + concat_output_node_id) + self._add_edge(concat_layer, skip_output_id, concat_output_node_id) + concat_layer.output = self.node_list[concat_output_node_id] + self.node_list[concat_output_node_id].shape = concat_layer.output_shape + + # Add the concatenate layer. + new_conv_layer = get_conv_class(self.n_dim)( + filters_start + filters_end, filters_end, 1 + ) + self._add_edge(new_conv_layer, concat_output_node_id, end_node_id) + new_conv_layer.input = self.node_list[concat_output_node_id] + new_conv_layer.output = self.node_list[end_node_id] + self.node_list[end_node_id].shape = new_conv_layer.output_shape + + if self.weighted: + filter_shape = (1,) * self.n_dim + weights = np.zeros((filters_end, filters_end) + filter_shape) + for i in range(filters_end): + filter_weight = np.zeros((filters_end,) + filter_shape) + center_index = (i,) + (0,) * self.n_dim + filter_weight[center_index] = 1 + weights[i, ...] = filter_weight + weights = np.concatenate( + (weights, np.zeros((filters_end, filters_start) + filter_shape)), axis=1 + ) + bias = np.zeros(filters_end) + new_conv_layer.set_weights( + (add_noise(weights, np.array([0, 1])), add_noise( + bias, np.array([0, 1]))) + ) + + def _insert_pooling_layer_chain(self, start_node_id, end_node_id): + skip_output_id = start_node_id + for layer in self._get_pooling_layers(start_node_id, end_node_id): + new_layer = deepcopy(layer) + if is_layer(new_layer, "Conv"): + filters = self.node_list[start_node_id].shape[-1] + new_layer = get_conv_class(self.n_dim)( + filters, filters, 1, layer.stride) + if self.weighted: + init_conv_weight(new_layer) + else: + new_layer = deepcopy(layer) + skip_output_id = self.add_layer(new_layer, skip_output_id) + skip_output_id = self.add_layer(StubReLU(), skip_output_id) + return skip_output_id + + def extract_descriptor(self): + """Extract the the description of the Graph as an instance of NetworkDescriptor.""" + main_chain = self.get_main_chain() + index_in_main_chain = {} + for index, u in enumerate(main_chain): + index_in_main_chain[u] = index + + ret = NetworkDescriptor() + for u in main_chain: + for v, layer_id in self.adj_list[u]: + if v not in index_in_main_chain: + continue + layer = self.layer_list[layer_id] + copied_layer = copy(layer) + copied_layer.weights = None + ret.add_layer(deepcopy(copied_layer)) + + for u in index_in_main_chain: + for v, layer_id in self.adj_list[u]: + if v not in index_in_main_chain: + temp_u = u + temp_v = v + temp_layer_id = layer_id + skip_type = None + while not ( + temp_v in index_in_main_chain and temp_u in index_in_main_chain): + if is_layer( + self.layer_list[temp_layer_id], "Concatenate"): + skip_type = NetworkDescriptor.CONCAT_CONNECT + if is_layer(self.layer_list[temp_layer_id], "Add"): + skip_type = NetworkDescriptor.ADD_CONNECT + temp_u = temp_v + temp_v, temp_layer_id = self.adj_list[temp_v][0] + ret.add_skip_connection( + index_in_main_chain[u], index_in_main_chain[temp_u], skip_type + ) + + elif index_in_main_chain[v] - index_in_main_chain[u] != 1: + skip_type = None + if is_layer(self.layer_list[layer_id], "Concatenate"): + skip_type = NetworkDescriptor.CONCAT_CONNECT + if is_layer(self.layer_list[layer_id], "Add"): + skip_type = NetworkDescriptor.ADD_CONNECT + ret.add_skip_connection( + index_in_main_chain[u], index_in_main_chain[v], skip_type + ) + + return ret + + def clear_weights(self): + ''' clear weights of the graph + ''' + self.weighted = False + for layer in self.layer_list: + layer.weights = None + + def produce_torch_model(self): + """Build a new Torch model based on the current graph.""" + return TorchModel(self) + + def produce_keras_model(self): + """Build a new keras model based on the current graph.""" + return KerasModel(self).model + + def produce_onnx_model(self): + """Build a new ONNX model based on the current graph.""" + return ONNXModel(self) + + def parsing_onnx_model(self, onnx_model): + '''to do in the future to use the onnx model + ''' + return ONNXModel(onnx_model) + + def produce_json_model(self): + """Build a new Json model based on the current graph.""" + return JSONModel(self).data + + @classmethod + def parsing_json_model(cls, json_model): + '''build a graph from json + ''' + return json_to_graph(json_model) + + def _layer_ids_in_order(self, layer_ids): + node_id_to_order_index = {} + for index, node_id in enumerate(self.topological_order): + node_id_to_order_index[node_id] = index + return sorted( + layer_ids, + key=lambda layer_id: node_id_to_order_index[ + self.layer_id_to_output_node_ids[layer_id][0] + ], + ) + + def _layer_ids_by_type(self, type_str): + return list( + filter( + lambda layer_id: is_layer(self.layer_list[layer_id], type_str), + range(self.n_layers), + ) + ) + + def get_main_chain_layers(self): + """Return a list of layer IDs in the main chain.""" + main_chain = self.get_main_chain() + ret = [] + for u in main_chain: + for v, layer_id in self.adj_list[u]: + if v in main_chain and u in main_chain: + ret.append(layer_id) + return ret + + def _conv_layer_ids_in_order(self): + return list( + filter( + lambda layer_id: is_layer(self.layer_list[layer_id], "Conv"), + self.get_main_chain_layers(), + ) + ) + + def _dense_layer_ids_in_order(self): + return self._layer_ids_in_order(self._layer_ids_by_type("Dense")) + + def deep_layer_ids(self): + ret = [] + for layer_id in self.get_main_chain_layers(): + layer = self.layer_list[layer_id] + if is_layer(layer, "GlobalAveragePooling"): + break + if is_layer(layer, "Add") or is_layer(layer, "Concatenate"): + continue + ret.append(layer_id) + return ret + + def wide_layer_ids(self): + return ( + self._conv_layer_ids_in_order( + )[:-1] + self._dense_layer_ids_in_order()[:-1] + ) + + def skip_connection_layer_ids(self): + return self.deep_layer_ids()[:-1] + + def size(self): + return sum(list(map(lambda x: x.size(), self.layer_list))) + + def get_main_chain(self): + """Returns the main chain node ID list.""" + pre_node = {} + distance = {} + for i in range(self.n_nodes): + distance[i] = 0 + pre_node[i] = i + for i in range(self.n_nodes - 1): + for u in range(self.n_nodes): + for v, _ in self.adj_list[u]: + if distance[u] + 1 > distance[v]: + distance[v] = distance[u] + 1 + pre_node[v] = u + temp_id = 0 + for i in range(self.n_nodes): + if distance[i] > distance[temp_id]: + temp_id = i + ret = [] + for i in range(self.n_nodes + 5): + ret.append(temp_id) + if pre_node[temp_id] == temp_id: + break + temp_id = pre_node[temp_id] + assert temp_id == pre_node[temp_id] + ret.reverse() + return ret + + +class TorchModel(torch.nn.Module): + """A neural network class using pytorch constructed from an instance of Graph.""" + + def __init__(self, graph): + super(TorchModel, self).__init__() + self.graph = graph + self.layers = [] + for layer in graph.layer_list: + self.layers.append(layer.to_real_layer()) + if graph.weighted: + for index, layer in enumerate(self.layers): + set_stub_weight_to_torch(self.graph.layer_list[index], layer) + for index, layer in enumerate(self.layers): + self.add_module(str(index), layer) + + def forward(self, input_tensor): + topo_node_list = self.graph.topological_order + output_id = topo_node_list[-1] + input_id = topo_node_list[0] + + node_list = deepcopy(self.graph.node_list) + node_list[input_id] = input_tensor + + for v in topo_node_list: + for u, layer_id in self.graph.reverse_adj_list[v]: + layer = self.graph.layer_list[layer_id] + torch_layer = self.layers[layer_id] + + if isinstance(layer, (StubAdd, StubConcatenate)): + edge_input_tensor = list( + map( + lambda x: node_list[x], + self.graph.layer_id_to_input_node_ids[layer_id], + ) + ) + else: + edge_input_tensor = node_list[u] + + temp_tensor = torch_layer(edge_input_tensor) + node_list[v] = temp_tensor + return node_list[output_id] + + def set_weight_to_graph(self): + self.graph.weighted = True + for index, layer in enumerate(self.layers): + set_torch_weight_to_stub(layer, self.graph.layer_list[index]) + + +class KerasModel: + def __init__(self, graph): + import keras + + self.graph = graph + self.layers = [] + for layer in graph.layer_list: + self.layers.append(to_real_keras_layer(layer)) + + # Construct the keras graph. + # Input + topo_node_list = self.graph.topological_order + output_id = topo_node_list[-1] + input_id = topo_node_list[0] + input_tensor = keras.layers.Input( + shape=graph.node_list[input_id].shape) + + node_list = deepcopy(self.graph.node_list) + node_list[input_id] = input_tensor + + # Output + for v in topo_node_list: + for u, layer_id in self.graph.reverse_adj_list[v]: + layer = self.graph.layer_list[layer_id] + keras_layer = self.layers[layer_id] + + if isinstance(layer, (StubAdd, StubConcatenate)): + edge_input_tensor = list( + map( + lambda x: node_list[x], + self.graph.layer_id_to_input_node_ids[layer_id], + ) + ) + else: + edge_input_tensor = node_list[u] + + temp_tensor = keras_layer(edge_input_tensor) + node_list[v] = temp_tensor + + output_tensor = node_list[output_id] + output_tensor = keras.layers.Activation("softmax", name="activation_add")( + output_tensor + ) + self.model = keras.models.Model( + inputs=input_tensor, outputs=output_tensor) + + if graph.weighted: + for index, layer in enumerate(self.layers): + set_stub_weight_to_keras(self.graph.layer_list[index], layer) + + def set_weight_to_graph(self): + self.graph.weighted = True + for index, layer in enumerate(self.layers): + set_keras_weight_to_stub(layer, self.graph.layer_list[index]) + + +class ONNXModel: + # to do in the future using onnx ir + def __init__(self, graph): + pass + + +class JSONModel: + def __init__(self, graph): + data = dict() + node_list = list() + layer_list = list() + operation_history = list() + + data["input_shape"] = graph.input_shape + vis = graph.vis + data["vis"] = list(vis.keys()) if vis is not None else None + data["weighted"] = graph.weighted + + for item in graph.operation_history: + if item[0] == "to_deeper_model": + operation_history.append( + [ + item[0], + item[1], + layer_description_extractor(item[2], graph.node_to_id), + ] + ) + else: + operation_history.append(item) + data["operation_history"] = operation_history + data["layer_id_to_input_node_ids"] = graph.layer_id_to_input_node_ids + data["layer_id_to_output_node_ids"] = graph.layer_id_to_output_node_ids + data["adj_list"] = graph.adj_list + data["reverse_adj_list"] = graph.reverse_adj_list + + for node in graph.node_list: + node_id = graph.node_to_id[node] + node_information = node.shape + node_list.append((node_id, node_information)) + + for layer_id, item in enumerate(graph.layer_list): + layer = graph.layer_list[layer_id] + layer_information = layer_description_extractor( + layer, graph.node_to_id) + layer_list.append((layer_id, layer_information)) + + data["node_list"] = node_list + data["layer_list"] = layer_list + + self.data = data + + +def graph_to_onnx(graph, onnx_model_path): + import onnx + # to do in the future using onnx ir + onnx_out = graph.produce_onnx_model() + onnx.save(onnx_out, onnx_model_path) + return onnx_out + + +def onnx_to_graph(onnx_model, input_shape): + # to do in the future using onnx ir + graph = Graph(input_shape, False) + graph.parsing_onnx_model(onnx_model) + return graph + + +def graph_to_json(graph, json_model_path): + json_out = graph.produce_json_model() + with open(json_model_path, "w") as fout: + json.dump(json_out, fout) + json_out = json.dumps(json_out) + return json_out + + +def json_to_graph(json_model: str): + json_model = json.loads(json_model) + # restore graph data from json data + input_shape = tuple(json_model["input_shape"]) + node_list = list() + node_to_id = dict() + id_to_node = dict() + layer_list = list() + layer_to_id = dict() + operation_history = list() + graph = Graph(input_shape, False) + + graph.input_shape = input_shape + vis = json_model["vis"] + graph.vis = { + tuple(item): True for item in vis} if vis is not None else None + graph.weighted = json_model["weighted"] + layer_id_to_input_node_ids = json_model["layer_id_to_input_node_ids"] + graph.layer_id_to_input_node_ids = { + int(k): v for k, v in layer_id_to_input_node_ids.items() + } + layer_id_to_output_node_ids = json_model["layer_id_to_output_node_ids"] + graph.layer_id_to_output_node_ids = { + int(k): v for k, v in layer_id_to_output_node_ids.items() + } + adj_list = {} + for k, v in json_model["adj_list"].items(): + adj_list[int(k)] = [tuple(i) for i in v] + graph.adj_list = adj_list + reverse_adj_list = {} + for k, v in json_model["reverse_adj_list"].items(): + reverse_adj_list[int(k)] = [tuple(i) for i in v] + graph.reverse_adj_list = reverse_adj_list + + for item in json_model["node_list"]: + new_node = Node(tuple(item[1])) + node_id = item[0] + node_list.append(new_node) + node_to_id[new_node] = node_id + id_to_node[node_id] = new_node + + for item in json_model["operation_history"]: + if item[0] == "to_deeper_model": + operation_history.append( + (item[0], item[1], layer_description_builder(item[2], id_to_node)) + ) + else: + operation_history.append(item) + graph.operation_history = operation_history + + for item in json_model["layer_list"]: + new_layer = layer_description_builder(item[1], id_to_node) + layer_id = int(item[0]) + layer_list.append(new_layer) + layer_to_id[new_layer] = layer_id + + graph.node_list = node_list + graph.node_to_id = node_to_id + graph.layer_list = layer_list + graph.layer_to_id = layer_to_id + + return graph diff --git a/new_impl/cv/third_party/nni_new/algorithms/hpo/networkmorphism_tuner/graph_transformer.py b/new_impl/cv/third_party/nni_new/algorithms/hpo/networkmorphism_tuner/graph_transformer.py new file mode 100644 index 0000000000000000000000000000000000000000..b03112d63bcc50f5c42e4203dfd4adb70a061119 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/hpo/networkmorphism_tuner/graph_transformer.py @@ -0,0 +1,167 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from copy import deepcopy + +from random import randrange, sample + +from .graph import NetworkDescriptor +from .layers import ( + StubDense, + StubReLU, + get_batch_norm_class, + get_conv_class, + get_dropout_class, + get_pooling_class, + is_layer, +) +from .utils import Constant + + +def to_wider_graph(graph): + ''' wider graph + ''' + weighted_layer_ids = graph.wide_layer_ids() + weighted_layer_ids = list( + filter( + lambda x: graph.layer_list[x].output.shape[-1], weighted_layer_ids) + ) + wider_layers = sample(weighted_layer_ids, 1) + + for layer_id in wider_layers: + layer = graph.layer_list[layer_id] + if is_layer(layer, "Conv"): + n_add = layer.filters + else: + n_add = layer.units + + graph.to_wider_model(layer_id, n_add) + return graph + + +def to_skip_connection_graph(graph): + ''' skip connection graph + ''' + # The last conv layer cannot be widen since wider operator cannot be done + # over the two sides of flatten. + weighted_layer_ids = graph.skip_connection_layer_ids() + valid_connection = [] + for skip_type in sorted( + [NetworkDescriptor.ADD_CONNECT, NetworkDescriptor.CONCAT_CONNECT]): + for index_a in range(len(weighted_layer_ids)): + for index_b in range(len(weighted_layer_ids))[index_a + 1:]: + valid_connection.append((index_a, index_b, skip_type)) + + if not valid_connection: + return graph + for index_a, index_b, skip_type in sample(valid_connection, 1): + a_id = weighted_layer_ids[index_a] + b_id = weighted_layer_ids[index_b] + if skip_type == NetworkDescriptor.ADD_CONNECT: + graph.to_add_skip_model(a_id, b_id) + else: + graph.to_concat_skip_model(a_id, b_id) + return graph + + +def create_new_layer(layer, n_dim): + ''' create new layer for the graph + ''' + + input_shape = layer.output.shape + dense_deeper_classes = [StubDense, get_dropout_class(n_dim), StubReLU] + conv_deeper_classes = [ + get_conv_class(n_dim), + get_batch_norm_class(n_dim), + StubReLU] + if is_layer(layer, "ReLU"): + conv_deeper_classes = [ + get_conv_class(n_dim), + get_batch_norm_class(n_dim)] + dense_deeper_classes = [StubDense, get_dropout_class(n_dim)] + elif is_layer(layer, "Dropout"): + dense_deeper_classes = [StubDense, StubReLU] + elif is_layer(layer, "BatchNormalization"): + conv_deeper_classes = [get_conv_class(n_dim), StubReLU] + + layer_class = None + if len(input_shape) == 1: + # It is in the dense layer part. + layer_class = sample(dense_deeper_classes, 1)[0] + else: + # It is in the conv layer part. + layer_class = sample(conv_deeper_classes, 1)[0] + + if layer_class == StubDense: + new_layer = StubDense(input_shape[0], input_shape[0]) + + elif layer_class == get_dropout_class(n_dim): + new_layer = layer_class(Constant.DENSE_DROPOUT_RATE) + + elif layer_class == get_conv_class(n_dim): + new_layer = layer_class( + input_shape[-1], input_shape[-1], sample((1, 3, 5), 1)[0], stride=1 + ) + + elif layer_class == get_batch_norm_class(n_dim): + new_layer = layer_class(input_shape[-1]) + + elif layer_class == get_pooling_class(n_dim): + new_layer = layer_class(sample((1, 3, 5), 1)[0]) + + else: + new_layer = layer_class() + + return new_layer + + +def to_deeper_graph(graph): + ''' deeper graph + ''' + + weighted_layer_ids = graph.deep_layer_ids() + if len(weighted_layer_ids) >= Constant.MAX_LAYERS: + return None + + deeper_layer_ids = sample(weighted_layer_ids, 1) + + for layer_id in deeper_layer_ids: + layer = graph.layer_list[layer_id] + new_layer = create_new_layer(layer, graph.n_dim) + graph.to_deeper_model(layer_id, new_layer) + return graph + + +def legal_graph(graph): + '''judge if a graph is legal or not. + ''' + + descriptor = graph.extract_descriptor() + skips = descriptor.skip_connections + if len(skips) != len(set(skips)): + return False + return True + + +def transform(graph): + '''core transform function for graph. + ''' + + graphs = [] + for _ in range(Constant.N_NEIGHBOURS * 2): + random_num = randrange(3) + temp_graph = None + if random_num == 0: + temp_graph = to_deeper_graph(deepcopy(graph)) + elif random_num == 1: + temp_graph = to_wider_graph(deepcopy(graph)) + elif random_num == 2: + temp_graph = to_skip_connection_graph(deepcopy(graph)) + + if temp_graph is not None and temp_graph.size() <= Constant.MAX_MODEL_SIZE: + graphs.append(temp_graph) + + if len(graphs) >= Constant.N_NEIGHBOURS: + break + + return graphs diff --git a/new_impl/cv/third_party/nni_new/algorithms/hpo/networkmorphism_tuner/layer_transformer.py b/new_impl/cv/third_party/nni_new/algorithms/hpo/networkmorphism_tuner/layer_transformer.py new file mode 100644 index 0000000000000000000000000000000000000000..6ffd1b20fb0e3b4ef0a6d2e54a02fb7cfa7cfc42 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/hpo/networkmorphism_tuner/layer_transformer.py @@ -0,0 +1,264 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import numpy as np + +from .layers import ( + StubDense, + StubReLU, + get_batch_norm_class, + get_conv_class, + get_n_dim, +) + +NOISE_RATIO = 1e-4 + + +def deeper_conv_block(conv_layer, kernel_size, weighted=True): + '''deeper conv layer. + ''' + n_dim = get_n_dim(conv_layer) + filter_shape = (kernel_size,) * 2 + n_filters = conv_layer.filters + weight = np.zeros((n_filters, n_filters) + filter_shape) + center = tuple(map(lambda x: int((x - 1) / 2), filter_shape)) + for i in range(n_filters): + filter_weight = np.zeros((n_filters,) + filter_shape) + index = (i,) + center + filter_weight[index] = 1 + weight[i, ...] = filter_weight + bias = np.zeros(n_filters) + new_conv_layer = get_conv_class(n_dim)( + conv_layer.filters, n_filters, kernel_size=kernel_size + ) + bn = get_batch_norm_class(n_dim)(n_filters) + + if weighted: + new_conv_layer.set_weights( + (add_noise(weight, np.array([0, 1])), + add_noise(bias, np.array([0, 1]))) + ) + new_weights = [ + add_noise(np.ones(n_filters, dtype=np.float32), np.array([0, 1])), + add_noise(np.zeros(n_filters, dtype=np.float32), np.array([0, 1])), + add_noise(np.zeros(n_filters, dtype=np.float32), np.array([0, 1])), + add_noise(np.ones(n_filters, dtype=np.float32), np.array([0, 1])), + ] + bn.set_weights(new_weights) + + return [StubReLU(), new_conv_layer, bn] + + +def dense_to_deeper_block(dense_layer, weighted=True): + '''deeper dense layer. + ''' + units = dense_layer.units + weight = np.eye(units) + bias = np.zeros(units) + new_dense_layer = StubDense(units, units) + if weighted: + new_dense_layer.set_weights( + (add_noise(weight, np.array([0, 1])), + add_noise(bias, np.array([0, 1]))) + ) + return [StubReLU(), new_dense_layer] + + +def wider_pre_dense(layer, n_add, weighted=True): + '''wider previous dense layer. + ''' + if not weighted: + return StubDense(layer.input_units, layer.units + n_add) + + n_units2 = layer.units + + teacher_w, teacher_b = layer.get_weights() + rand = np.random.randint(n_units2, size=n_add) + student_w = teacher_w.copy() + student_b = teacher_b.copy() + + # target layer update (i) + for i in range(n_add): + teacher_index = rand[i] + new_weight = teacher_w[teacher_index, :] + new_weight = new_weight[np.newaxis, :] + student_w = np.concatenate( + (student_w, add_noise(new_weight, student_w)), axis=0) + student_b = np.append( + student_b, add_noise( + teacher_b[teacher_index], student_b)) + + new_pre_layer = StubDense(layer.input_units, n_units2 + n_add) + new_pre_layer.set_weights((student_w, student_b)) + + return new_pre_layer + + +def wider_pre_conv(layer, n_add_filters, weighted=True): + '''wider previous conv layer. + ''' + n_dim = get_n_dim(layer) + if not weighted: + return get_conv_class(n_dim)( + layer.input_channel, + layer.filters + n_add_filters, + kernel_size=layer.kernel_size, + ) + + n_pre_filters = layer.filters + rand = np.random.randint(n_pre_filters, size=n_add_filters) + teacher_w, teacher_b = layer.get_weights() + + student_w = teacher_w.copy() + student_b = teacher_b.copy() + # target layer update (i) + for i, _ in enumerate(rand): + teacher_index = rand[i] + new_weight = teacher_w[teacher_index, ...] + new_weight = new_weight[np.newaxis, ...] + student_w = np.concatenate((student_w, new_weight), axis=0) + student_b = np.append(student_b, teacher_b[teacher_index]) + new_pre_layer = get_conv_class(n_dim)( + layer.input_channel, n_pre_filters + n_add_filters, layer.kernel_size + ) + new_pre_layer.set_weights( + (add_noise(student_w, teacher_w), add_noise(student_b, teacher_b)) + ) + return new_pre_layer + + +def wider_next_conv(layer, start_dim, total_dim, n_add, weighted=True): + '''wider next conv layer. + ''' + n_dim = get_n_dim(layer) + if not weighted: + return get_conv_class(n_dim)(layer.input_channel + n_add, + layer.filters, + kernel_size=layer.kernel_size, + stride=layer.stride) + n_filters = layer.filters + teacher_w, teacher_b = layer.get_weights() + + new_weight_shape = list(teacher_w.shape) + new_weight_shape[1] = n_add + new_weight = np.zeros(tuple(new_weight_shape)) + + student_w = np.concatenate((teacher_w[:, :start_dim, ...].copy(), + add_noise(new_weight, teacher_w), + teacher_w[:, start_dim:total_dim, ...].copy()), axis=1) + new_layer = get_conv_class(n_dim)(layer.input_channel + n_add, + n_filters, + kernel_size=layer.kernel_size, + stride=layer.stride) + new_layer.set_weights((student_w, teacher_b)) + return new_layer + + +def wider_bn(layer, start_dim, total_dim, n_add, weighted=True): + '''wider batch norm layer. + ''' + n_dim = get_n_dim(layer) + if not weighted: + return get_batch_norm_class(n_dim)(layer.num_features + n_add) + + weights = layer.get_weights() + + new_weights = [ + add_noise(np.ones(n_add, dtype=np.float32), np.array([0, 1])), + add_noise(np.zeros(n_add, dtype=np.float32), np.array([0, 1])), + add_noise(np.zeros(n_add, dtype=np.float32), np.array([0, 1])), + add_noise(np.ones(n_add, dtype=np.float32), np.array([0, 1])), + ] + + student_w = tuple() + for weight, new_weight in zip(weights, new_weights): + temp_w = weight.copy() + temp_w = np.concatenate( + (temp_w[:start_dim], new_weight, temp_w[start_dim:total_dim]) + ) + student_w += (temp_w,) + new_layer = get_batch_norm_class(n_dim)(layer.num_features + n_add) + new_layer.set_weights(student_w) + return new_layer + + +def wider_next_dense(layer, start_dim, total_dim, n_add, weighted=True): + '''wider next dense layer. + ''' + if not weighted: + return StubDense(layer.input_units + n_add, layer.units) + teacher_w, teacher_b = layer.get_weights() + student_w = teacher_w.copy() + n_units_each_channel = int(teacher_w.shape[1] / total_dim) + + new_weight = np.zeros((teacher_w.shape[0], n_add * n_units_each_channel)) + student_w = np.concatenate( + ( + student_w[:, : start_dim * n_units_each_channel], + add_noise(new_weight, student_w), + student_w[ + :, start_dim * n_units_each_channel: total_dim * n_units_each_channel + ], + ), + axis=1, + ) + + new_layer = StubDense(layer.input_units + n_add, layer.units) + new_layer.set_weights((student_w, teacher_b)) + return new_layer + + +def add_noise(weights, other_weights): + '''add noise to the layer. + ''' + w_range = np.ptp(other_weights.flatten()) + noise_range = NOISE_RATIO * w_range + noise = np.random.uniform(-noise_range / 2.0, + noise_range / 2.0, weights.shape) + return np.add(noise, weights) + + +def init_dense_weight(layer): + '''initilize dense layer weight. + ''' + units = layer.units + weight = np.eye(units) + bias = np.zeros(units) + layer.set_weights( + (add_noise(weight, np.array([0, 1])), + add_noise(bias, np.array([0, 1]))) + ) + + +def init_conv_weight(layer): + '''initilize conv layer weight. + ''' + n_filters = layer.filters + filter_shape = (layer.kernel_size,) * get_n_dim(layer) + weight = np.zeros((n_filters, n_filters) + filter_shape) + + center = tuple(map(lambda x: int((x - 1) / 2), filter_shape)) + for i in range(n_filters): + filter_weight = np.zeros((n_filters,) + filter_shape) + index = (i,) + center + filter_weight[index] = 1 + weight[i, ...] = filter_weight + bias = np.zeros(n_filters) + + layer.set_weights( + (add_noise(weight, np.array([0, 1])), + add_noise(bias, np.array([0, 1]))) + ) + + +def init_bn_weight(layer): + '''initilize batch norm layer weight. + ''' + n_filters = layer.num_features + new_weights = [ + add_noise(np.ones(n_filters, dtype=np.float32), np.array([0, 1])), + add_noise(np.zeros(n_filters, dtype=np.float32), np.array([0, 1])), + add_noise(np.zeros(n_filters, dtype=np.float32), np.array([0, 1])), + add_noise(np.ones(n_filters, dtype=np.float32), np.array([0, 1])), + ] + layer.set_weights(new_weights) diff --git a/new_impl/cv/third_party/nni_new/algorithms/hpo/networkmorphism_tuner/layers.py b/new_impl/cv/third_party/nni_new/algorithms/hpo/networkmorphism_tuner/layers.py new file mode 100644 index 0000000000000000000000000000000000000000..a96c87b7801fe2687f61dfd62fdd13019ecd2ee0 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/hpo/networkmorphism_tuner/layers.py @@ -0,0 +1,862 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from abc import abstractmethod +from collections.abc import Iterable + +import torch +from torch import nn +from torch.nn import functional +from .utils import Constant + + +class AvgPool(nn.Module): + """ + AvgPool Module. + """ + + def __init__(self): + super().__init__() + + @abstractmethod + def forward(self, input_tensor): + pass + + +class GlobalAvgPool1d(AvgPool): + """ + GlobalAvgPool1d Module. + """ + + def forward(self, input_tensor): + return functional.avg_pool1d(input_tensor, input_tensor.size()[2:]).view( + input_tensor.size()[:2] + ) + + +class GlobalAvgPool2d(AvgPool): + """ + GlobalAvgPool2d Module. + """ + + def forward(self, input_tensor): + return functional.avg_pool2d(input_tensor, input_tensor.size()[2:]).view( + input_tensor.size()[:2] + ) + + +class GlobalAvgPool3d(AvgPool): + """ + GlobalAvgPool3d Module. + """ + + def forward(self, input_tensor): + return functional.avg_pool3d(input_tensor, input_tensor.size()[2:]).view( + input_tensor.size()[:2] + ) + + +class StubLayer: + """ + StubLayer Module. Base Module. + """ + + def __init__(self, input_node=None, output_node=None): + self.input = input_node + self.output = output_node + self.weights = None + + def build(self, shape): + """ + build shape. + """ + + def set_weights(self, weights): + """ + set weights. + """ + self.weights = weights + + def import_weights(self, torch_layer): + """ + import weights. + """ + + def import_weights_keras(self, keras_layer): + """ + import weights from keras layer. + """ + + def export_weights(self, torch_layer): + """ + export weights. + """ + + def export_weights_keras(self, keras_layer): + """ + export weights to keras layer. + """ + + def get_weights(self): + """ + get weights. + """ + return self.weights + + def size(self): + """ + size(). + """ + return 0 + + @property + def output_shape(self): + """ + output shape. + """ + return self.input.shape + + def to_real_layer(self): + """ + to real layer. + """ + + def __str__(self): + """ + str() function to print. + """ + return type(self).__name__[4:] + + +class StubWeightBiasLayer(StubLayer): + """ + StubWeightBiasLayer Module to set the bias. + """ + + def import_weights(self, torch_layer): + self.set_weights( + (torch_layer.weight.data.cpu().numpy(), + torch_layer.bias.data.cpu().numpy()) + ) + + def import_weights_keras(self, keras_layer): + self.set_weights(keras_layer.get_weights()) + + def export_weights(self, torch_layer): + torch_layer.weight.data = torch.Tensor(self.weights[0]) + torch_layer.bias.data = torch.Tensor(self.weights[1]) + + def export_weights_keras(self, keras_layer): + keras_layer.set_weights(self.weights) + + +class StubBatchNormalization(StubWeightBiasLayer): + """ + StubBatchNormalization Module. Batch Norm. + """ + + def __init__(self, num_features, input_node=None, output_node=None): + super().__init__(input_node, output_node) + self.num_features = num_features + + def import_weights(self, torch_layer): + self.set_weights( + ( + torch_layer.weight.data.cpu().numpy(), + torch_layer.bias.data.cpu().numpy(), + torch_layer.running_mean.cpu().numpy(), + torch_layer.running_var.cpu().numpy(), + ) + ) + + def export_weights(self, torch_layer): + torch_layer.weight.data = torch.Tensor(self.weights[0]) + torch_layer.bias.data = torch.Tensor(self.weights[1]) + torch_layer.running_mean = torch.Tensor(self.weights[2]) + torch_layer.running_var = torch.Tensor(self.weights[3]) + + def size(self): + return self.num_features * 4 + + @abstractmethod + def to_real_layer(self): + pass + + +class StubBatchNormalization1d(StubBatchNormalization): + """ + StubBatchNormalization1d Module. + """ + + def to_real_layer(self): + return torch.nn.BatchNorm1d(self.num_features) + + +class StubBatchNormalization2d(StubBatchNormalization): + """ + StubBatchNormalization2d Module. + """ + + def to_real_layer(self): + return torch.nn.BatchNorm2d(self.num_features) + + +class StubBatchNormalization3d(StubBatchNormalization): + """ + StubBatchNormalization3d Module. + """ + + def to_real_layer(self): + return torch.nn.BatchNorm3d(self.num_features) + + +class StubDense(StubWeightBiasLayer): + """ + StubDense Module. Linear. + """ + + def __init__(self, input_units, units, input_node=None, output_node=None): + super().__init__(input_node, output_node) + self.input_units = input_units + self.units = units + + @property + def output_shape(self): + return (self.units,) + + def import_weights_keras(self, keras_layer): + self.set_weights( + (keras_layer.get_weights()[0].T, + keras_layer.get_weights()[1])) + + def export_weights_keras(self, keras_layer): + keras_layer.set_weights((self.weights[0].T, self.weights[1])) + + def size(self): + return self.input_units * self.units + self.units + + def to_real_layer(self): + return torch.nn.Linear(self.input_units, self.units) + + +class StubConv(StubWeightBiasLayer): + """ + StubConv Module. Conv. + """ + + def __init__(self, input_channel, filters, kernel_size, + stride=1, input_node=None, output_node=None): + super().__init__(input_node, output_node) + self.input_channel = input_channel + self.filters = filters + self.kernel_size = kernel_size + self.stride = stride + self.padding = int(self.kernel_size / 2) + + @property + def output_shape(self): + ret = list(self.input.shape[:-1]) + for index, dim in enumerate(ret): + ret[index] = ( + int((dim + 2 * self.padding - self.kernel_size) / self.stride) + 1 + ) + ret = ret + [self.filters] + return tuple(ret) + + def import_weights_keras(self, keras_layer): + self.set_weights( + (keras_layer.get_weights()[0].T, + keras_layer.get_weights()[1])) + + def export_weights_keras(self, keras_layer): + keras_layer.set_weights((self.weights[0].T, self.weights[1])) + + def size(self): + return (self.input_channel * self.kernel_size * + self.kernel_size + 1) * self.filters + + @abstractmethod + def to_real_layer(self): + pass + + def __str__(self): + return ( + super().__str__() + + "(" + + ", ".join( + str(item) + for item in [ + self.input_channel, + self.filters, + self.kernel_size, + self.stride, + ] + ) + + ")" + ) + + +class StubConv1d(StubConv): + """ + StubConv1d Module. + """ + + def to_real_layer(self): + return torch.nn.Conv1d( + self.input_channel, + self.filters, + self.kernel_size, + stride=self.stride, + padding=self.padding, + ) + + +class StubConv2d(StubConv): + """ + StubConv2d Module. + """ + + def to_real_layer(self): + return torch.nn.Conv2d( + self.input_channel, + self.filters, + self.kernel_size, + stride=self.stride, + padding=self.padding, + ) + + +class StubConv3d(StubConv): + """ + StubConv3d Module. + """ + + def to_real_layer(self): + return torch.nn.Conv3d( + self.input_channel, + self.filters, + self.kernel_size, + stride=self.stride, + padding=self.padding, + ) + + +class StubAggregateLayer(StubLayer): + """ + StubAggregateLayer Module. + """ + + def __init__(self, input_nodes=None, output_node=None): + if input_nodes is None: + input_nodes = [] + super().__init__(input_nodes, output_node) + + +class StubConcatenate(StubAggregateLayer): + """StubConcatenate Module. + """ + @property + def output_shape(self): + ret = 0 + for current_input in self.input: + ret += current_input.shape[-1] + ret = self.input[0].shape[:-1] + (ret,) + return ret + + def to_real_layer(self): + return TorchConcatenate() + + +class StubAdd(StubAggregateLayer): + """ + StubAdd Module. + """ + @property + def output_shape(self): + return self.input[0].shape + + def to_real_layer(self): + return TorchAdd() + + +class StubFlatten(StubLayer): + """ + StubFlatten Module. + """ + @property + def output_shape(self): + ret = 1 + for dim in self.input.shape: + ret *= dim + return (ret,) + + def to_real_layer(self): + return TorchFlatten() + + +class StubReLU(StubLayer): + """ + StubReLU Module. + """ + + def to_real_layer(self): + return torch.nn.ReLU() + + +class StubSoftmax(StubLayer): + """ + StubSoftmax Module. + """ + + def to_real_layer(self): + return torch.nn.LogSoftmax(dim=1) + + +class StubDropout(StubLayer): + """ + StubDropout Module. + """ + + def __init__(self, rate, input_node=None, output_node=None): + super().__init__(input_node, output_node) + self.rate = rate + + @abstractmethod + def to_real_layer(self): + pass + + +class StubDropout1d(StubDropout): + """ + StubDropout1d Module. + """ + + def to_real_layer(self): + return torch.nn.Dropout(self.rate) + + +class StubDropout2d(StubDropout): + """ + StubDropout2d Module. + """ + + def to_real_layer(self): + return torch.nn.Dropout2d(self.rate) + + +class StubDropout3d(StubDropout): + """ + StubDropout3d Module. + """ + + def to_real_layer(self): + return torch.nn.Dropout3d(self.rate) + + +class StubInput(StubLayer): + """ + StubInput Module. + """ + + def __init__(self, input_node=None, output_node=None): + super().__init__(input_node, output_node) + + +class StubPooling(StubLayer): + """ + StubPooling Module. + """ + + def __init__(self, + kernel_size=None, + stride=None, + padding=0, + input_node=None, + output_node=None): + super().__init__(input_node, output_node) + self.kernel_size = ( + kernel_size if kernel_size is not None else Constant.POOLING_KERNEL_SIZE + ) + self.stride = stride if stride is not None else self.kernel_size + self.padding = padding + + @property + def output_shape(self): + ret = tuple() + for dim in self.input.shape[:-1]: + ret = ret + (max(int(dim / self.kernel_size), 1),) + ret = ret + (self.input.shape[-1],) + return ret + + @abstractmethod + def to_real_layer(self): + pass + + +class StubPooling1d(StubPooling): + """ + StubPooling1d Module. + """ + + def to_real_layer(self): + return torch.nn.MaxPool1d(self.kernel_size, stride=self.stride) + + +class StubPooling2d(StubPooling): + """ + StubPooling2d Module. + """ + + def to_real_layer(self): + return torch.nn.MaxPool2d(self.kernel_size, stride=self.stride) + + +class StubPooling3d(StubPooling): + """ + StubPooling3d Module. + """ + + def to_real_layer(self): + return torch.nn.MaxPool3d(self.kernel_size, stride=self.stride) + + +class StubGlobalPooling(StubLayer): + """ + StubGlobalPooling Module. + """ + + def __init__(self, input_node=None, output_node=None): + super().__init__(input_node, output_node) + + @property + def output_shape(self): + return (self.input.shape[-1],) + + @abstractmethod + def to_real_layer(self): + pass + + +class StubGlobalPooling1d(StubGlobalPooling): + """ + StubGlobalPooling1d Module. + """ + + def to_real_layer(self): + return GlobalAvgPool1d() + + +class StubGlobalPooling2d(StubGlobalPooling): + """ + StubGlobalPooling2d Module. + """ + + def to_real_layer(self): + return GlobalAvgPool2d() + + +class StubGlobalPooling3d(StubGlobalPooling): + """ + StubGlobalPooling3d Module. + """ + + def to_real_layer(self): + return GlobalAvgPool3d() + + +class TorchConcatenate(nn.Module): + """ + TorchConcatenate Module. + """ + + def forward(self, input_list): + return torch.cat(input_list, dim=1) + + +class TorchAdd(nn.Module): + """ + TorchAdd Module. + """ + + def forward(self, input_list): + return input_list[0] + input_list[1] + + +class TorchFlatten(nn.Module): + """ + TorchFlatten Module. + """ + + def forward(self, input_tensor): + return input_tensor.view(input_tensor.size(0), -1) + + +def keras_dropout(layer, rate): + """ + Keras dropout layer. + """ + + from keras import layers + + input_dim = len(layer.input.shape) + if input_dim == 2: + return layers.SpatialDropout1D(rate) + elif input_dim == 3: + return layers.SpatialDropout2D(rate) + elif input_dim == 4: + return layers.SpatialDropout3D(rate) + else: + return layers.Dropout(rate) + + +def to_real_keras_layer(layer): + """ + Real keras layer. + """ + from keras import layers + + if is_layer(layer, "Dense"): + return layers.Dense(layer.units, input_shape=(layer.input_units,)) + if is_layer(layer, "Conv"): + return layers.Conv2D( + layer.filters, + layer.kernel_size, + input_shape=layer.input.shape, + padding="same", + ) # padding + if is_layer(layer, "Pooling"): + return layers.MaxPool2D(2) + if is_layer(layer, "BatchNormalization"): + return layers.BatchNormalization(input_shape=layer.input.shape) + if is_layer(layer, "Concatenate"): + return layers.Concatenate() + if is_layer(layer, "Add"): + return layers.Add() + if is_layer(layer, "Dropout"): + return keras_dropout(layer, layer.rate) + if is_layer(layer, "ReLU"): + return layers.Activation("relu") + if is_layer(layer, "Softmax"): + return layers.Activation("softmax") + if is_layer(layer, "Flatten"): + return layers.Flatten() + if is_layer(layer, "GlobalAveragePooling"): + return layers.GlobalAveragePooling2D() + return None # note: this is not written by original author, feel free to modify if you think it's incorrect + + +def is_layer(layer, layer_type): + """ + Judge the layer type. + + Returns + ------- + bool + boolean -- True or False + """ + + if layer_type == "Input": + return isinstance(layer, StubInput) + elif layer_type == "Conv": + return isinstance(layer, StubConv) + elif layer_type == "Dense": + return isinstance(layer, (StubDense,)) + elif layer_type == "BatchNormalization": + return isinstance(layer, (StubBatchNormalization,)) + elif layer_type == "Concatenate": + return isinstance(layer, (StubConcatenate,)) + elif layer_type == "Add": + return isinstance(layer, (StubAdd,)) + elif layer_type == "Pooling": + return isinstance(layer, StubPooling) + elif layer_type == "Dropout": + return isinstance(layer, (StubDropout,)) + elif layer_type == "Softmax": + return isinstance(layer, (StubSoftmax,)) + elif layer_type == "ReLU": + return isinstance(layer, (StubReLU,)) + elif layer_type == "Flatten": + return isinstance(layer, (StubFlatten,)) + elif layer_type == "GlobalAveragePooling": + return isinstance(layer, StubGlobalPooling) + return None # note: this is not written by original author, feel free to modify if you think it's incorrect + + +def layer_description_extractor(layer, node_to_id): + """ + Get layer description. + """ + + layer_input = layer.input + layer_output = layer.output + if layer_input is not None: + if isinstance(layer_input, Iterable): + layer_input = list(map(lambda x: node_to_id[x], layer_input)) + else: + layer_input = node_to_id[layer_input] + + if layer_output is not None: + layer_output = node_to_id[layer_output] + + if isinstance(layer, StubConv): + return ( + type(layer).__name__, + layer_input, + layer_output, + layer.input_channel, + layer.filters, + layer.kernel_size, + layer.stride, + layer.padding, + ) + elif isinstance(layer, (StubDense,)): + return [ + type(layer).__name__, + layer_input, + layer_output, + layer.input_units, + layer.units, + ] + elif isinstance(layer, (StubBatchNormalization,)): + return (type(layer).__name__, layer_input, + layer_output, layer.num_features) + elif isinstance(layer, (StubDropout,)): + return (type(layer).__name__, layer_input, layer_output, layer.rate) + elif isinstance(layer, StubPooling): + return ( + type(layer).__name__, + layer_input, + layer_output, + layer.kernel_size, + layer.stride, + layer.padding, + ) + else: + return (type(layer).__name__, layer_input, layer_output) + + +def layer_description_builder(layer_information, id_to_node): + """build layer from description. + """ + layer_type = layer_information[0] + + layer_input_ids = layer_information[1] + if isinstance(layer_input_ids, Iterable): + layer_input = list(map(lambda x: id_to_node[x], layer_input_ids)) + else: + layer_input = id_to_node[layer_input_ids] + layer_output = id_to_node[layer_information[2]] + if layer_type.startswith("StubConv"): + input_channel = layer_information[3] + filters = layer_information[4] + kernel_size = layer_information[5] + stride = layer_information[6] + return globals()[layer_type]( + input_channel, filters, kernel_size, stride, layer_input, layer_output + ) + elif layer_type.startswith("StubDense"): + input_units = layer_information[3] + units = layer_information[4] + return globals()[layer_type](input_units, units, layer_input, layer_output) + elif layer_type.startswith("StubBatchNormalization"): + num_features = layer_information[3] + return globals()[layer_type](num_features, layer_input, layer_output) + elif layer_type.startswith("StubDropout"): + rate = layer_information[3] + return globals()[layer_type](rate, layer_input, layer_output) + elif layer_type.startswith("StubPooling"): + kernel_size = layer_information[3] + stride = layer_information[4] + padding = layer_information[5] + return globals()[layer_type](kernel_size, stride, padding, layer_input, layer_output) + else: + return globals()[layer_type](layer_input, layer_output) + + +def layer_width(layer): + """ + Get layer width. + """ + + if is_layer(layer, "Dense"): + return layer.units + if is_layer(layer, "Conv"): + return layer.filters + raise TypeError("The layer should be either Dense or Conv layer.") + + +def set_torch_weight_to_stub(torch_layer, stub_layer): + stub_layer.import_weights(torch_layer) + + +def set_keras_weight_to_stub(keras_layer, stub_layer): + stub_layer.import_weights_keras(keras_layer) + + +def set_stub_weight_to_torch(stub_layer, torch_layer): + stub_layer.export_weights(torch_layer) + + +def set_stub_weight_to_keras(stub_layer, keras_layer): + stub_layer.export_weights_keras(keras_layer) + + +def get_conv_class(n_dim): + conv_class_list = [StubConv1d, StubConv2d, StubConv3d] + return conv_class_list[n_dim - 1] + + +def get_dropout_class(n_dim): + dropout_class_list = [StubDropout1d, StubDropout2d, StubDropout3d] + return dropout_class_list[n_dim - 1] + + +def get_global_avg_pooling_class(n_dim): + global_avg_pooling_class_list = [ + StubGlobalPooling1d, + StubGlobalPooling2d, + StubGlobalPooling3d, + ] + return global_avg_pooling_class_list[n_dim - 1] + + +def get_pooling_class(n_dim): + pooling_class_list = [StubPooling1d, StubPooling2d, StubPooling3d] + return pooling_class_list[n_dim - 1] + + +def get_batch_norm_class(n_dim): + batch_norm_class_list = [ + StubBatchNormalization1d, + StubBatchNormalization2d, + StubBatchNormalization3d, + ] + return batch_norm_class_list[n_dim - 1] + + +def get_n_dim(layer): + if isinstance(layer, ( + StubConv1d, + StubDropout1d, + StubGlobalPooling1d, + StubPooling1d, + StubBatchNormalization1d, + )): + return 1 + if isinstance(layer, ( + StubConv2d, + StubDropout2d, + StubGlobalPooling2d, + StubPooling2d, + StubBatchNormalization2d, + )): + return 2 + if isinstance(layer, ( + StubConv3d, + StubDropout3d, + StubGlobalPooling3d, + StubPooling3d, + StubBatchNormalization3d, + )): + return 3 + return -1 diff --git a/new_impl/cv/third_party/nni_new/algorithms/hpo/networkmorphism_tuner/networkmorphism_tuner.py b/new_impl/cv/third_party/nni_new/algorithms/hpo/networkmorphism_tuner/networkmorphism_tuner.py new file mode 100644 index 0000000000000000000000000000000000000000..385028506db5a5f5b3b2ecf951542be6d7cbde26 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/hpo/networkmorphism_tuner/networkmorphism_tuner.py @@ -0,0 +1,328 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +networkmorphsim_tuner.py +""" + +import logging +import os +from schema import Optional, Schema +from nni.tuner import Tuner +from nni.utils import OptimizeMode, extract_scalar_reward +from .bayesian import BayesianOptimizer +from .nn import CnnGenerator, MlpGenerator +from .utils import Constant +from .graph import graph_to_json, json_to_graph +from nni import ClassArgsValidator + +logger = logging.getLogger("NetworkMorphism_AutoML") + +class NetworkMorphismClassArgsValidator(ClassArgsValidator): + def validate_class_args(self, **kwargs): + Schema({ + Optional('optimize_mode'): self.choices('optimize_mode', 'maximize', 'minimize'), + Optional('task'): self.choices('task', 'cv', 'nlp', 'common'), + Optional('input_width'): int, + Optional('input_channel'): int, + Optional('n_output_node'): int + }).validate(kwargs) + +class NetworkMorphismTuner(Tuner): + """ + NetworkMorphismTuner is a tuner which using network morphism techniques. + + Attributes + ---------- + n_classes : int + The class number or output node number (default: ``10``) + input_shape : tuple + A tuple including: (input_width, input_width, input_channel) + t_min : float + The minimum temperature for simulated annealing. (default: ``Constant.T_MIN``) + beta : float + The beta in acquisition function. (default: ``Constant.BETA``) + algorithm_name : str + algorithm name used in the network morphism (default: ``"Bayesian"``) + optimize_mode : str + optimize mode "minimize" or "maximize" (default: ``"minimize"``) + verbose : bool + verbose to print the log (default: ``True``) + bo : BayesianOptimizer + The optimizer used in networkmorphsim tuner. + max_model_size : int + max model size to the graph (default: ``Constant.MAX_MODEL_SIZE``) + default_model_len : int + default model length (default: ``Constant.MODEL_LEN``) + default_model_width : int + default model width (default: ``Constant.MODEL_WIDTH``) + search_space : dict + """ + + def __init__( + self, + task="cv", + input_width=32, + input_channel=3, + n_output_node=10, + algorithm_name="Bayesian", + optimize_mode="maximize", + path="model_path", + verbose=True, + beta=Constant.BETA, + t_min=Constant.T_MIN, + max_model_size=Constant.MAX_MODEL_SIZE, + default_model_len=Constant.MODEL_LEN, + default_model_width=Constant.MODEL_WIDTH, + ): + """ + initilizer of the NetworkMorphismTuner. + """ + + if not os.path.exists(path): + os.makedirs(path) + self.path = os.path.join(os.getcwd(), path) + if task == "cv": + self.generators = [CnnGenerator] + elif task == "common": + self.generators = [MlpGenerator] + else: + raise NotImplementedError( + '{} task not supported in List ["cv","common"]') + + self.n_classes = n_output_node + self.input_shape = (input_width, input_width, input_channel) + + self.t_min = t_min + self.beta = beta + self.algorithm_name = algorithm_name + self.optimize_mode = OptimizeMode(optimize_mode) + self.json = None + self.total_data = {} + self.verbose = verbose + self.model_count = 0 + + self.bo = BayesianOptimizer( + self, self.t_min, self.optimize_mode, self.beta) + self.training_queue = [] + self.descriptors = [] + self.history = [] + + self.max_model_size = max_model_size + self.default_model_len = default_model_len + self.default_model_width = default_model_width + + self.search_space = dict() + + + def update_search_space(self, search_space): + """ + Update search space definition in tuner by search_space in neural architecture. + """ + self.search_space = search_space + + def generate_parameters(self, parameter_id, **kwargs): + """ + Returns a set of trial neural architecture, as a serializable object. + + Parameters + ---------- + parameter_id : int + """ + if not self.history: + self.init_search() + + new_father_id = None + generated_graph = None + if not self.training_queue: + new_father_id, generated_graph = self.generate() + new_model_id = self.model_count + self.model_count += 1 + self.training_queue.append( + (generated_graph, new_father_id, new_model_id)) + self.descriptors.append(generated_graph.extract_descriptor()) + + graph, father_id, model_id = self.training_queue.pop(0) + + # from graph to json + json_model_path = os.path.join(self.path, str(model_id) + ".json") + json_out = graph_to_json(graph, json_model_path) + self.total_data[parameter_id] = (json_out, father_id, model_id) + + return json_out + + def receive_trial_result(self, parameter_id, parameters, value, **kwargs): + """ + Record an observation of the objective function. + + Parameters + ---------- + parameter_id : int + the id of a group of paramters that generated by nni manager. + parameters : dict + A group of parameters. + value : dict/float + if value is dict, it should have "default" key. + """ + reward = extract_scalar_reward(value) + + if parameter_id not in self.total_data: + raise RuntimeError("Received parameter_id not in total_data.") + + (_, father_id, model_id) = self.total_data[parameter_id] + + graph = self.bo.searcher.load_model_by_id(model_id) + + # to use the value and graph + self.add_model(reward, model_id) + self.update(father_id, graph, reward, model_id) + + + def init_search(self): + """ + Call the generators to generate the initial architectures for the search. + """ + if self.verbose: + logger.info("Initializing search.") + for generator in self.generators: + graph = generator(self.n_classes, self.input_shape).generate( + self.default_model_len, self.default_model_width + ) + model_id = self.model_count + self.model_count += 1 + self.training_queue.append((graph, -1, model_id)) + self.descriptors.append(graph.extract_descriptor()) + + if self.verbose: + logger.info("Initialization finished.") + + + def generate(self): + """ + Generate the next neural architecture. + + Returns + ------- + other_info : any object + Anything to be saved in the training queue together with the architecture. + generated_graph : Graph + An instance of Graph. + """ + generated_graph, new_father_id = self.bo.generate(self.descriptors) + if new_father_id is None: + new_father_id = 0 + generated_graph = self.generators[0]( + self.n_classes, self.input_shape + ).generate(self.default_model_len, self.default_model_width) + + return new_father_id, generated_graph + + def update(self, other_info, graph, metric_value, model_id): + """ + Update the controller with evaluation result of a neural architecture. + + Parameters + ---------- + other_info: any object + In our case it is the father ID in the search tree. + graph: graph.Graph + An instance of Graph. The trained neural architecture. + metric_value: float + The final evaluated metric value. + model_id: int + """ + father_id = other_info + self.bo.fit([graph.extract_descriptor()], [metric_value]) + self.bo.add_child(father_id, model_id) + + def add_model(self, metric_value, model_id): + """ + Add model to the history, x_queue and y_queue + + Parameters + ---------- + metric_value : float + graph : dict + model_id : int + + Returns + ------- + model : dict + """ + if self.verbose: + logger.info("Saving model.") + + # Update best_model text file + ret = {"model_id": model_id, "metric_value": metric_value} + self.history.append(ret) + if model_id == self.get_best_model_id(): + file = open(os.path.join(self.path, "best_model.txt"), "w") + file.write("best model: " + str(model_id)) + file.close() + return ret + + + def get_best_model_id(self): + """ + Get the best model_id from history using the metric value + """ + + if self.optimize_mode is OptimizeMode.Maximize: + return max(self.history, key=lambda x: x["metric_value"])[ + "model_id"] + return min(self.history, key=lambda x: x["metric_value"])["model_id"] + + + def load_model_by_id(self, model_id): + """ + Get the model by model_id + + Parameters + ---------- + model_id : int + model index + + Returns + ------- + load_model : graph.Graph + the model graph representation + """ + + with open(os.path.join(self.path, str(model_id) + ".json")) as fin: + json_str = fin.read().replace("\n", "") + + load_model = json_to_graph(json_str) + return load_model + + def load_best_model(self): + """ + Get the best model by model id + + Returns + ------- + load_model : graph.Graph + the model graph representation + """ + return self.load_model_by_id(self.get_best_model_id()) + + def get_metric_value_by_id(self, model_id): + """ + Get the model metric valud by its model_id + + Parameters + ---------- + model_id : int + model index + + Returns + ------- + float + the model metric + """ + for item in self.history: + if item["model_id"] == model_id: + return item["metric_value"] + return None + + def import_data(self, data): + pass diff --git a/new_impl/cv/third_party/nni_new/algorithms/hpo/networkmorphism_tuner/nn.py b/new_impl/cv/third_party/nni_new/algorithms/hpo/networkmorphism_tuner/nn.py new file mode 100644 index 0000000000000000000000000000000000000000..9e0072f9b39e3a68b865ba850157fe1080791983 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/hpo/networkmorphism_tuner/nn.py @@ -0,0 +1,166 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from abc import abstractmethod + +from .graph import Graph +from .layers import (StubDense, StubDropout1d, + StubReLU, get_batch_norm_class, + get_conv_class, + get_dropout_class, + get_global_avg_pooling_class, + get_pooling_class) +from .utils import Constant + + +class NetworkGenerator: + """The base class for generating a network. + It can be used to generate a CNN or Multi-Layer Perceptron. + Attributes: + n_output_node: Number of output nodes in the network. + input_shape: A tuple to represent the input shape. + """ + + def __init__(self, n_output_node, input_shape): + self.n_output_node = n_output_node + self.input_shape = input_shape + + @abstractmethod + def generate(self, model_len, model_width): + pass + + +class CnnGenerator(NetworkGenerator): + """A class to generate CNN. + Attributes: + n_dim: `len(self.input_shape) - 1` + conv: A class that represents `(n_dim-1)` dimensional convolution. + dropout: A class that represents `(n_dim-1)` dimensional dropout. + global_avg_pooling: A class that represents `(n_dim-1)` dimensional Global Average Pooling. + pooling: A class that represents `(n_dim-1)` dimensional pooling. + batch_norm: A class that represents `(n_dim-1)` dimensional batch normalization. + """ + + def __init__(self, n_output_node, input_shape): + super(CnnGenerator, self).__init__(n_output_node, input_shape) + self.n_dim = len(self.input_shape) - 1 + if len(self.input_shape) > 4: + raise ValueError("The input dimension is too high.") + if len(self.input_shape) < 2: + raise ValueError("The input dimension is too low.") + self.conv = get_conv_class(self.n_dim) + self.dropout = get_dropout_class(self.n_dim) + self.global_avg_pooling = get_global_avg_pooling_class(self.n_dim) + self.pooling = get_pooling_class(self.n_dim) + self.batch_norm = get_batch_norm_class(self.n_dim) + + def generate(self, model_len=None, model_width=None): + """Generates a CNN. + Args: + model_len: An integer. Number of convolutional layers. + model_width: An integer. Number of filters for the convolutional layers. + Returns: + An instance of the class Graph. Represents the neural architecture graph of the generated model. + """ + + if model_len is None: + model_len = Constant.MODEL_LEN + if model_width is None: + model_width = Constant.MODEL_WIDTH + pooling_len = int(model_len / 4) + graph = Graph(self.input_shape, False) + temp_input_channel = self.input_shape[-1] + output_node_id = 0 + stride = 1 + for i in range(model_len): + output_node_id = graph.add_layer(StubReLU(), output_node_id) + output_node_id = graph.add_layer( + self.batch_norm( + graph.node_list[output_node_id].shape[-1]), output_node_id + ) + output_node_id = graph.add_layer( + self.conv( + temp_input_channel, + model_width, + kernel_size=3, + stride=stride), + output_node_id, + ) + temp_input_channel = model_width + if pooling_len == 0 or ( + (i + 1) % pooling_len == 0 and i != model_len - 1): + output_node_id = graph.add_layer( + self.pooling(), output_node_id) + + output_node_id = graph.add_layer( + self.global_avg_pooling(), output_node_id) + output_node_id = graph.add_layer( + self.dropout(Constant.CONV_DROPOUT_RATE), output_node_id + ) + output_node_id = graph.add_layer( + StubDense(graph.node_list[output_node_id].shape[0], model_width), + output_node_id, + ) + output_node_id = graph.add_layer(StubReLU(), output_node_id) + graph.add_layer( + StubDense( + model_width, + self.n_output_node), + output_node_id) + return graph + + +class MlpGenerator(NetworkGenerator): + """A class to generate Multi-Layer Perceptron. + """ + + def __init__(self, n_output_node, input_shape): + """Initialize the instance. + Args: + n_output_node: An integer. Number of output nodes in the network. + input_shape: A tuple. Input shape of the network. If it is 1D, ensure the value is appended by a comma + in the tuple. + """ + super(MlpGenerator, self).__init__(n_output_node, input_shape) + if len(self.input_shape) > 1: + raise ValueError("The input dimension is too high.") + + def generate(self, model_len=None, model_width=None): + """Generates a Multi-Layer Perceptron. + Args: + model_len: An integer. Number of hidden layers. + model_width: An integer or a list of integers of length `model_len`. If it is a list, it represents the + number of nodes in each hidden layer. If it is an integer, all hidden layers have nodes equal to this + value. + Returns: + An instance of the class Graph. Represents the neural architecture graph of the generated model. + """ + if model_len is None: + model_len = Constant.MODEL_LEN + if model_width is None: + model_width = Constant.MODEL_WIDTH + if isinstance(model_width, list) and not len(model_width) == model_len: + raise ValueError( + "The length of 'model_width' does not match 'model_len'") + elif isinstance(model_width, int): + model_width = [model_width] * model_len + + graph = Graph(self.input_shape, False) + output_node_id = 0 + n_nodes_prev_layer = self.input_shape[0] + for width in model_width: + output_node_id = graph.add_layer( + StubDense(n_nodes_prev_layer, width), output_node_id + ) + output_node_id = graph.add_layer( + StubDropout1d(Constant.MLP_DROPOUT_RATE), output_node_id + ) + output_node_id = graph.add_layer(StubReLU(), output_node_id) + n_nodes_prev_layer = width + + graph.add_layer( + StubDense( + n_nodes_prev_layer, + self.n_output_node), + output_node_id) + return graph diff --git a/new_impl/cv/third_party/nni_new/algorithms/hpo/networkmorphism_tuner/utils.py b/new_impl/cv/third_party/nni_new/algorithms/hpo/networkmorphism_tuner/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..0634e7f578bdeb505ce984be971beb3978c78a44 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/hpo/networkmorphism_tuner/utils.py @@ -0,0 +1,23 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + + +class Constant: + '''Constant for the Tuner. + ''' + MAX_LAYERS = 100 + N_NEIGHBOURS = 8 + MAX_MODEL_SIZE = 1 << 24 + KERNEL_LAMBDA = 1.0 + BETA = 2.576 + MLP_MODEL_LEN = 3 + MLP_MODEL_WIDTH = 5 + MODEL_LEN = 3 + MODEL_WIDTH = 64 + POOLING_KERNEL_SIZE = 2 + DENSE_DROPOUT_RATE = 0.5 + CONV_DROPOUT_RATE = 0.25 + MLP_DROPOUT_RATE = 0.25 + CONV_BLOCK_DISTANCE = 2 + BATCH_SIZE = 128 + T_MIN = 0.0001 diff --git a/new_impl/cv/third_party/nni_new/algorithms/hpo/pbt_tuner.py b/new_impl/cv/third_party/nni_new/algorithms/hpo/pbt_tuner.py new file mode 100644 index 0000000000000000000000000000000000000000..507c519a2a880ca99833433a6f61278e308a469f --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/hpo/pbt_tuner.py @@ -0,0 +1,456 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import copy +import logging +import os +import random +import numpy as np +from schema import Schema, Optional + +import nni +from nni import ClassArgsValidator +import nni.parameter_expressions +from nni.tuner import Tuner +from nni.utils import OptimizeMode, extract_scalar_reward, split_index, json2parameter, json2space + + +logger = logging.getLogger('pbt_tuner_AutoML') + + +def perturbation(hyperparameter_type, value, resample_probablity, uv, ub, lv, lb, random_state): + """ + Perturbation for hyperparameters + + Parameters + ---------- + hyperparameter_type : str + type of hyperparameter + value : list + parameters for sampling hyperparameter + resample_probability : float + probability for resampling + uv : float/int + upper value after perturbation + ub : float/int + upper bound + lv : float/int + lower value after perturbation + lb : float/int + lower bound + random_state : RandomState + random state + """ + if random.random() < resample_probablity: + if hyperparameter_type == "choice": + return value.index(nni.parameter_expressions.choice(value, random_state)) + else: + return getattr(nni.parameter_expressions, hyperparameter_type)(*(value + [random_state])) + else: + if random.random() > 0.5: + return min(uv, ub) + else: + return max(lv, lb) + + +def exploit_and_explore(bot_trial_info, top_trial_info, factor, resample_probability, epoch, search_space): + """ + Replace checkpoint of bot_trial with top, and perturb hyperparameters + + Parameters + ---------- + bot_trial_info : TrialInfo + bottom model whose parameters should be replaced + top_trial_info : TrialInfo + better model + factor : float + factor for perturbation + resample_probability : float + probability for resampling + epoch : int + step of PBTTuner + search_space : dict + search_space to keep perturbed hyperparameters in range + """ + bot_checkpoint_dir = bot_trial_info.checkpoint_dir + top_hyper_parameters = top_trial_info.hyper_parameters + hyper_parameters = copy.deepcopy(top_hyper_parameters) + random_state = np.random.RandomState() + hyper_parameters['load_checkpoint_dir'] = hyper_parameters['save_checkpoint_dir'] + hyper_parameters['save_checkpoint_dir'] = os.path.join(bot_checkpoint_dir, str(epoch)) + for key in hyper_parameters.keys(): + hyper_parameter = hyper_parameters[key] + if key == 'load_checkpoint_dir' or key == 'save_checkpoint_dir': + continue + elif search_space[key]["_type"] == "choice": + choices = search_space[key]["_value"] + ub, uv = len(choices) - 1, choices.index(hyper_parameter) + 1 + lb, lv = 0, choices.index(hyper_parameter) - 1 + elif search_space[key]["_type"] == "randint": + lb, ub = search_space[key]["_value"][:2] + ub -= 1 + uv = hyper_parameter + 1 + lv = hyper_parameter - 1 + elif search_space[key]["_type"] == "uniform": + lb, ub = search_space[key]["_value"][:2] + perturb = (ub - lb) * factor + uv = hyper_parameter + perturb + lv = hyper_parameter - perturb + elif search_space[key]["_type"] == "quniform": + lb, ub, q = search_space[key]["_value"][:3] + multi = round(hyper_parameter / q) + uv = (multi + 1) * q + lv = (multi - 1) * q + elif search_space[key]["_type"] == "loguniform": + lb, ub = search_space[key]["_value"][:2] + perturb = (np.log(ub) - np.log(lb)) * factor + uv = np.exp(min(np.log(hyper_parameter) + perturb, np.log(ub))) + lv = np.exp(max(np.log(hyper_parameter) - perturb, np.log(lb))) + elif search_space[key]["_type"] == "qloguniform": + lb, ub, q = search_space[key]["_value"][:3] + multi = round(hyper_parameter / q) + uv = (multi + 1) * q + lv = (multi - 1) * q + elif search_space[key]["_type"] == "normal": + sigma = search_space[key]["_value"][1] + perturb = sigma * factor + uv = ub = hyper_parameter + perturb + lv = lb = hyper_parameter - perturb + elif search_space[key]["_type"] == "qnormal": + q = search_space[key]["_value"][2] + uv = ub = hyper_parameter + q + lv = lb = hyper_parameter - q + elif search_space[key]["_type"] == "lognormal": + sigma = search_space[key]["_value"][1] + perturb = sigma * factor + uv = ub = np.exp(np.log(hyper_parameter) + perturb) + lv = lb = np.exp(np.log(hyper_parameter) - perturb) + elif search_space[key]["_type"] == "qlognormal": + q = search_space[key]["_value"][2] + uv = ub = hyper_parameter + q + lv, lb = hyper_parameter - q, 1E-10 + else: + logger.warning("Illegal type to perturb: %s", search_space[key]["_type"]) + continue + + if search_space[key]["_type"] == "choice": + idx = perturbation(search_space[key]["_type"], search_space[key]["_value"], + resample_probability, uv, ub, lv, lb, random_state) + hyper_parameters[key] = choices[idx] + else: + hyper_parameters[key] = perturbation(search_space[key]["_type"], search_space[key]["_value"], + resample_probability, uv, ub, lv, lb, random_state) + bot_trial_info.hyper_parameters = hyper_parameters + bot_trial_info.clean_id() + + +class TrialInfo: + """ + Information of each trial, refresh for each epoch + + """ + + def __init__(self, checkpoint_dir=None, hyper_parameters=None, parameter_id=None, score=None): + self.checkpoint_dir = checkpoint_dir + self.hyper_parameters = hyper_parameters + self.parameter_id = parameter_id + self.score = score + + def clean_id(self): + self.parameter_id = None + +class PBTClassArgsValidator(ClassArgsValidator): + def validate_class_args(self, **kwargs): + Schema({ + 'optimize_mode': self.choices('optimize_mode', 'maximize', 'minimize'), + Optional('all_checkpoint_dir'): str, + Optional('population_size'): self.range('population_size', int, 0, 99999), + Optional('factors'): float, + Optional('fraction'): float, + }).validate(kwargs) + +class PBTTuner(Tuner): + def __init__(self, optimize_mode="maximize", all_checkpoint_dir=None, population_size=10, factor=0.2, + resample_probability=0.25, fraction=0.2): + """ + Initialization + + Parameters + ---------- + optimize_mode : str + maximize or minimize + all_checkpoint_dir : str + directory to store training model checkpoint + population_size : int + number of trials for each epoch + factor : float + factor for perturbation + resample_probability : float + probability for resampling + fraction : float + fraction for selecting bottom and top trials + """ + self.optimize_mode = OptimizeMode(optimize_mode) + if all_checkpoint_dir is None: + all_checkpoint_dir = os.getenv('NNI_CHECKPOINT_DIRECTORY') + logger.info("Checkpoint dir is set to %s by default.", all_checkpoint_dir) + self.all_checkpoint_dir = all_checkpoint_dir + self.population_size = population_size + self.factor = factor + self.resample_probability = resample_probability + self.fraction = fraction + # defined in trial code + #self.perturbation_interval = perturbation_interval + + self.population = None + self.pos = -1 + self.param_ids = [] + self.running = {} + self.finished = [] + self.credit = 0 + self.finished_trials = 0 + self.epoch = 0 + + self.searchspace_json = None + self.space = None + + self.send_trial_callback = None + + logger.info('PBT tuner initialization') + + def update_search_space(self, search_space): + """ + Get search space + + Parameters + ---------- + search_space : dict + Search space + """ + logger.info('Update search space %s', search_space) + self.searchspace_json = search_space + self.space = json2space(self.searchspace_json) + + self.random_state = np.random.RandomState() + self.population = [] + is_rand = dict() + + for item in self.space: + is_rand[item] = True + + for i in range(self.population_size): + hyper_parameters = json2parameter( + self.searchspace_json, is_rand, self.random_state) + hyper_parameters = split_index(hyper_parameters) + checkpoint_dir = os.path.join(self.all_checkpoint_dir, str(i)) + hyper_parameters['load_checkpoint_dir'] = os.path.join(checkpoint_dir, str(self.epoch)) + hyper_parameters['save_checkpoint_dir'] = os.path.join(checkpoint_dir, str(self.epoch)) + self.population.append(TrialInfo(checkpoint_dir=checkpoint_dir, hyper_parameters=hyper_parameters)) + + def generate_multiple_parameters(self, parameter_id_list, **kwargs): + """ + Returns multiple sets of trial (hyper-)parameters, as iterable of serializable objects. + + Parameters + ---------- + parameter_id_list : list of int + Unique identifiers for each set of requested hyper-parameters. + These will later be used in :meth:`receive_trial_result`. + **kwargs + Used for send_trial_callback. + + Returns + ------- + list + A list of newly generated configurations + """ + result = [] + self.send_trial_callback = kwargs['st_callback'] + for parameter_id in parameter_id_list: + had_exception = False + try: + logger.debug("generating param for %s", parameter_id) + res = self.generate_parameters(parameter_id, **kwargs) + except nni.NoMoreTrialError: + had_exception = True + if not had_exception: + result.append(res) + return result + + def generate_parameters(self, parameter_id, **kwargs): + """ + Generate parameters, if no trial configration for now, self.credit plus 1 to send the config later + + Parameters + ---------- + parameter_id : int + Unique identifier for requested hyper-parameters. + This will later be used in :meth:`receive_trial_result`. + **kwargs + Not used + + Returns + ------- + dict + One newly generated configuration + + """ + if self.pos == self.population_size - 1: + logger.debug('Credit added by one in parameters request') + self.credit += 1 + self.param_ids.append(parameter_id) + raise nni.NoMoreTrialError('No more parameters now.') + self.pos += 1 + trial_info = self.population[self.pos] + trial_info.parameter_id = parameter_id + self.running[parameter_id] = trial_info + logger.info('Generate parameter : %s', trial_info.hyper_parameters) + return trial_info.hyper_parameters + + def _proceed_next_epoch(self): + """ + """ + logger.info('Proceeding to next epoch') + self.epoch += 1 + self.population = [] + self.pos = -1 + self.running = {} + #exploit and explore + reverse = True if self.optimize_mode == OptimizeMode.Maximize else False + self.finished = sorted(self.finished, key=lambda x: x.score, reverse=reverse) + cutoff = int(np.ceil(self.fraction * len(self.finished))) + tops = self.finished[:cutoff] + bottoms = self.finished[self.finished_trials - cutoff:] + for bottom in bottoms: + top = np.random.choice(tops) + exploit_and_explore(bottom, top, self.factor, self.resample_probability, self.epoch, self.searchspace_json) + for trial in self.finished: + if trial not in bottoms: + trial.clean_id() + trial.hyper_parameters['load_checkpoint_dir'] = trial.hyper_parameters['save_checkpoint_dir'] + trial.hyper_parameters['save_checkpoint_dir'] = os.path.join(trial.checkpoint_dir, str(self.epoch)) + self.finished_trials = 0 + for _ in range(self.population_size): + trial_info = self.finished.pop() + self.population.append(trial_info) + while self.credit > 0 and self.pos + 1 < len(self.population): + self.credit -= 1 + self.pos += 1 + parameter_id = self.param_ids.pop() + trial_info = self.population[self.pos] + trial_info.parameter_id = parameter_id + self.running[parameter_id] = trial_info + self.send_trial_callback(parameter_id, trial_info.hyper_parameters) + + def receive_trial_result(self, parameter_id, parameters, value, **kwargs): + """ + Receive trial's result. if the number of finished trials equals ``self.population_size``, start the next epoch to + train the model. + + Parameters + ---------- + parameter_id : int + Unique identifier of used hyper-parameters, same with :meth:`generate_parameters`. + parameters : dict + Hyper-parameters generated by :meth:`generate_parameters`. + value : dict + Result from trial (the return value of :func:`nni.report_final_result`). + """ + logger.info('Get one trial result, id = %d, value = %s', parameter_id, value) + value = extract_scalar_reward(value) + trial_info = self.running.pop(parameter_id, None) + trial_info.score = value + self.finished.append(trial_info) + self.finished_trials += 1 + if self.finished_trials == self.population_size: + self._proceed_next_epoch() + + def trial_end(self, parameter_id, success, **kwargs): + """ + Deal with trial failure + + Parameters + ---------- + parameter_id : int + Unique identifier for hyper-parameters used by this trial. + success : bool + True if the trial successfully completed; False if failed or terminated. + **kwargs + Unstable parameters which should be ignored by normal users. + """ + if success: + return + if self.optimize_mode == OptimizeMode.Minimize: + value = float('inf') + else: + value = float('-inf') + trial_info = self.running.pop(parameter_id, None) + trial_info.score = value + self.finished.append(trial_info) + self.finished_trials += 1 + if self.finished_trials == self.population_size: + self._proceed_next_epoch() + + def import_data(self, data): + """ + Parameters + ---------- + data : json obj + imported data records + + Returns + ------- + int + the start epoch number after data imported, only used for unittest + """ + if self.running: + logger.warning("Do not support importing data in the middle of experiment") + return + # the following is for experiment resume + _completed_num = 0 + epoch_data_dict = {} + for trial_info in data: + logger.info("Process data record %s / %s", _completed_num, len(data)) + _completed_num += 1 + # simply validate data format + _params = trial_info["parameter"] + _value = trial_info['value'] + # assign fake value for failed trials + if not _value: + logger.info("Useless trial data, value is %s, skip this trial data.", _value) + _value = float('inf') if self.optimize_mode == OptimizeMode.Minimize else float('-inf') + _value = extract_scalar_reward(_value) + if 'save_checkpoint_dir' not in _params: + logger.warning("Invalid data record: save_checkpoint_dir is missing, abandon data import.") + return + epoch_num = int(os.path.basename(_params['save_checkpoint_dir'])) + if epoch_num not in epoch_data_dict: + epoch_data_dict[epoch_num] = [] + epoch_data_dict[epoch_num].append((_params, _value)) + if not epoch_data_dict: + logger.warning("No valid epochs, abandon data import.") + return + # figure out start epoch for resume + max_epoch_num = max(epoch_data_dict, key=int) + if len(epoch_data_dict[max_epoch_num]) < self.population_size: + max_epoch_num -= 1 + # If there is no a single complete round, no data to import, start from scratch + if max_epoch_num < 0: + logger.warning("No completed epoch, abandon data import.") + return + assert len(epoch_data_dict[max_epoch_num]) == self.population_size + # check existence of trial save checkpoint dir + for params, _ in epoch_data_dict[max_epoch_num]: + if not os.path.isdir(params['save_checkpoint_dir']): + logger.warning("save_checkpoint_dir %s does not exist, data will not be resumed", params['save_checkpoint_dir']) + return + # resume data + self.epoch = max_epoch_num + self.finished_trials = self.population_size + for params, value in epoch_data_dict[max_epoch_num]: + checkpoint_dir = os.path.dirname(params['save_checkpoint_dir']) + self.finished.append(TrialInfo(checkpoint_dir=checkpoint_dir, hyper_parameters=params, score=value)) + self._proceed_next_epoch() + logger.info("Successfully import data to PBT tuner, total data: %d, imported data: %d.", len(data), self.population_size) + logger.info("Start from epoch %d ...", self.epoch) + return self.epoch # return for test diff --git a/new_impl/cv/third_party/nni_new/algorithms/hpo/ppo_tuner/__init__.py b/new_impl/cv/third_party/nni_new/algorithms/hpo/ppo_tuner/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..854090c93e71c710dec79c570953ac1fb920dcdc --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/hpo/ppo_tuner/__init__.py @@ -0,0 +1 @@ +from .ppo_tuner import PPOTuner, PPOClassArgsValidator diff --git a/new_impl/cv/third_party/nni_new/algorithms/hpo/ppo_tuner/distri.py b/new_impl/cv/third_party/nni_new/algorithms/hpo/ppo_tuner/distri.py new file mode 100644 index 0000000000000000000000000000000000000000..8a2a5ed20c3db6086c74339428fd25e0cf806f57 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/hpo/ppo_tuner/distri.py @@ -0,0 +1,183 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +functions for sampling from hidden state +""" + +import tensorflow.compat.v1 as tf +tf.disable_v2_behavior() + +from .util import fc + + +class Pd: + """ + A particular probability distribution + """ + def flatparam(self): + raise NotImplementedError + def mode(self): + raise NotImplementedError + def neglogp(self, x): + # Usually it's easier to define the negative logprob + raise NotImplementedError + def kl(self, other): + raise NotImplementedError + def entropy(self): + raise NotImplementedError + def sample(self): + raise NotImplementedError + def logp(self, x): + return - self.neglogp(x) + def get_shape(self): + return self.flatparam().shape + @property + def shape(self): + return self.get_shape() + def __getitem__(self, idx): + return self.__class__(self.flatparam()[idx]) + +class PdType: + """ + Parametrized family of probability distributions + """ + def pdclass(self): + raise NotImplementedError + def pdfromflat(self, flat, mask, nsteps, size, is_act_model): + return self.pdclass()(flat, mask, nsteps, size, is_act_model) + def pdfromlatent(self, latent_vector, init_scale, init_bias): + raise NotImplementedError + def param_shape(self): + raise NotImplementedError + def sample_shape(self): + raise NotImplementedError + def sample_dtype(self): + raise NotImplementedError + + def param_placeholder(self, prepend_shape, name=None): + return tf.placeholder(dtype=tf.float32, shape=prepend_shape+self.param_shape(), name=name) + def sample_placeholder(self, prepend_shape, name=None): + return tf.placeholder(dtype=self.sample_dtype(), shape=prepend_shape+self.sample_shape(), name=name) + +class CategoricalPd(Pd): + """ + Categorical probability distribution + """ + def __init__(self, logits, mask_npinf, nsteps, size, is_act_model): + self.logits = logits + self.mask_npinf = mask_npinf + self.nsteps = nsteps + self.size = size + self.is_act_model = is_act_model + def flatparam(self): + return self.logits + def mode(self): + return tf.argmax(self.logits, axis=-1) + + @property + def mean(self): + return tf.nn.softmax(self.logits) + def neglogp(self, x): + """ + return tf.nn.sparse_softmax_cross_entropy_with_logits(logits=self.logits, labels=x) + Note: we can't use sparse_softmax_cross_entropy_with_logits because + the implementation does not allow second-order derivatives... + """ + if x.dtype in {tf.uint8, tf.int32, tf.int64}: + # one-hot encoding + x_shape_list = x.shape.as_list() + logits_shape_list = self.logits.get_shape().as_list()[:-1] + for xs, ls in zip(x_shape_list, logits_shape_list): + if xs is not None and ls is not None: + assert xs == ls, 'shape mismatch: {} in x vs {} in logits'.format(xs, ls) + + x = tf.one_hot(x, self.logits.get_shape().as_list()[-1]) + else: + # already encoded + assert x.shape.as_list() == self.logits.shape.as_list() + + return tf.nn.softmax_cross_entropy_with_logits_v2( + logits=self.logits, + labels=x) + + def kl(self, other): + """kl""" + a0 = self.logits - tf.reduce_max(self.logits, axis=-1, keepdims=True) + a1 = other.logits - tf.reduce_max(other.logits, axis=-1, keepdims=True) + ea0 = tf.exp(a0) + ea1 = tf.exp(a1) + z0 = tf.reduce_sum(ea0, axis=-1, keepdims=True) + z1 = tf.reduce_sum(ea1, axis=-1, keepdims=True) + p0 = ea0 / z0 + return tf.reduce_sum(p0 * (a0 - tf.log(z0) - a1 + tf.log(z1)), axis=-1) + + def entropy(self): + """compute entropy""" + a0 = self.logits - tf.reduce_max(self.logits, axis=-1, keepdims=True) + ea0 = tf.exp(a0) + z0 = tf.reduce_sum(ea0, axis=-1, keepdims=True) + p0 = ea0 / z0 + return tf.reduce_sum(p0 * (tf.log(z0) - a0), axis=-1) + + def sample(self): + """sample from logits""" + if not self.is_act_model: + re_res = tf.reshape(self.logits, [-1, self.nsteps, self.size]) + masked_res = tf.math.add(re_res, self.mask_npinf) + re_masked_res = tf.reshape(masked_res, [-1, self.size]) + + u = tf.random_uniform(tf.shape(re_masked_res), dtype=self.logits.dtype) + return tf.argmax(re_masked_res - tf.log(-1*tf.log(u)), axis=-1) + else: + u = tf.random_uniform(tf.shape(self.logits), dtype=self.logits.dtype) + return tf.argmax(self.logits - tf.log(-1*tf.log(u)), axis=-1) + + @classmethod + def fromflat(cls, flat): + return cls(flat) # pylint: disable=no-value-for-parameter + +class CategoricalPdType(PdType): + """ + To create CategoricalPd + """ + def __init__(self, ncat, nsteps, np_mask, is_act_model): + self.ncat = ncat + self.nsteps = nsteps + self.np_mask = np_mask + self.is_act_model = is_act_model + def pdclass(self): + return CategoricalPd + + def pdfromlatent(self, latent_vector, init_scale=1.0, init_bias=0.0): + """add fc and create CategoricalPd""" + pdparam, mask, mask_npinf = _matching_fc(latent_vector, 'pi', self.ncat, self.nsteps, + init_scale=init_scale, init_bias=init_bias, + np_mask=self.np_mask, is_act_model=self.is_act_model) + return self.pdfromflat(pdparam, mask_npinf, self.nsteps, self.ncat, self.is_act_model), pdparam, mask, mask_npinf + + def param_shape(self): + return [self.ncat] + def sample_shape(self): + return [] + def sample_dtype(self): + return tf.int32 + +def _matching_fc(tensor, name, size, nsteps, init_scale, init_bias, np_mask, is_act_model): + """ + Add fc op, and add mask op when not in action mode + """ + if tensor.shape[-1] == size: + assert False + return tensor + else: + mask = tf.get_variable("act_mask", dtype=tf.float32, initializer=np_mask[0], trainable=False) + mask_npinf = tf.get_variable("act_mask_npinf", dtype=tf.float32, initializer=np_mask[1], trainable=False) + res = fc(tensor, name, size, init_scale=init_scale, init_bias=init_bias) + if not is_act_model: + re_res = tf.reshape(res, [-1, nsteps, size]) + masked_res = tf.math.multiply(re_res, mask) + re_masked_res = tf.reshape(masked_res, [-1, size]) + return re_masked_res, mask, mask_npinf + else: + return res, mask, mask_npinf diff --git a/new_impl/cv/third_party/nni_new/algorithms/hpo/ppo_tuner/model.py b/new_impl/cv/third_party/nni_new/algorithms/hpo/ppo_tuner/model.py new file mode 100644 index 0000000000000000000000000000000000000000..c6a8479c6d0d06fcf0f7c4b8a64a62ab968988aa --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/hpo/ppo_tuner/model.py @@ -0,0 +1,152 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +the main model of policy/value network +""" + +import tensorflow.compat.v1 as tf +tf.disable_v2_behavior() + +from .util import initialize, get_session + +class Model: + """ + We use this object to : + __init__: + - Creates the step_model + - Creates the train_model + + train(): + - Make the training part (feedforward and retropropagation of gradients) + + save/load(): + - Save load the model + """ + def __init__(self, *, policy, nbatch_act, nbatch_train, + nsteps, ent_coef, vf_coef, max_grad_norm, microbatch_size=None, np_mask=None): + self.sess = sess = get_session() + + with tf.variable_scope('ppo2_model', reuse=tf.AUTO_REUSE): + # CREATE OUR TWO MODELS + # act_model that is used for sampling + act_model = policy(nbatch_act, 1, sess, np_mask=np_mask, is_act_model=True) + + # Train model for training + if microbatch_size is None: + train_model = policy(nbatch_train, nsteps, sess, np_mask=np_mask, is_act_model=False) + else: + train_model = policy(microbatch_size, nsteps, sess, np_mask=np_mask, is_act_model=False) + + # CREATE THE PLACEHOLDERS + self.A = A = train_model.pdtype.sample_placeholder([None]) + self.ADV = ADV = tf.placeholder(tf.float32, [None]) + self.R = R = tf.placeholder(tf.float32, [None]) + # Keep track of old actor + self.OLDNEGLOGPAC = OLDNEGLOGPAC = tf.placeholder(tf.float32, [None]) + # Keep track of old critic + self.OLDVPRED = OLDVPRED = tf.placeholder(tf.float32, [None]) + self.LR = LR = tf.placeholder(tf.float32, []) + # Cliprange + self.CLIPRANGE = CLIPRANGE = tf.placeholder(tf.float32, []) + + neglogpac = train_model.pd.neglogp(A) + + # Calculate the entropy + # Entropy is used to improve exploration by limiting the premature convergence to suboptimal policy. + entropy = tf.reduce_mean(train_model.pd.entropy()) + + # CALCULATE THE LOSS + # Total loss = Policy gradient loss - entropy * entropy coefficient + Value coefficient * value loss + + # Clip the value to reduce variability during Critic training + # Get the predicted value + vpred = train_model.vf + vpredclipped = OLDVPRED + tf.clip_by_value(train_model.vf - OLDVPRED, - CLIPRANGE, CLIPRANGE) + # Unclipped value + vf_losses1 = tf.square(vpred - R) + # Clipped value + vf_losses2 = tf.square(vpredclipped - R) + + vf_loss = .5 * tf.reduce_mean(tf.maximum(vf_losses1, vf_losses2)) + + # Calculate ratio (pi current policy / pi old policy) + ratio = tf.exp(OLDNEGLOGPAC - neglogpac) + + # Defining Loss = - J is equivalent to max J + pg_losses = -ADV * ratio + + pg_losses2 = -ADV * tf.clip_by_value(ratio, 1.0 - CLIPRANGE, 1.0 + CLIPRANGE) + + # Final PG loss + pg_loss = tf.reduce_mean(tf.maximum(pg_losses, pg_losses2)) + approxkl = .5 * tf.reduce_mean(tf.square(neglogpac - OLDNEGLOGPAC)) + clipfrac = tf.reduce_mean(tf.to_float(tf.greater(tf.abs(ratio - 1.0), CLIPRANGE))) + + # Total loss + loss = pg_loss - entropy * ent_coef + vf_loss * vf_coef + + # UPDATE THE PARAMETERS USING LOSS + # 1. Get the model parameters + params = tf.trainable_variables('ppo2_model') + # 2. Build our trainer + self.trainer = tf.train.AdamOptimizer(learning_rate=LR, epsilon=1e-5) + # 3. Calculate the gradients + grads_and_var = self.trainer.compute_gradients(loss, params) + grads, var = zip(*grads_and_var) + + if max_grad_norm is not None: + # Clip the gradients (normalize) + grads, _grad_norm = tf.clip_by_global_norm(grads, max_grad_norm) + grads_and_var = list(zip(grads, var)) + # zip aggregate each gradient with parameters associated + # For instance zip(ABCD, xyza) => Ax, By, Cz, Da + + self.grads = grads + self.var = var + self._train_op = self.trainer.apply_gradients(grads_and_var) + self.loss_names = ['policy_loss', 'value_loss', 'policy_entropy', 'approxkl', 'clipfrac'] + self.stats_list = [pg_loss, vf_loss, entropy, approxkl, clipfrac] + + + self.train_model = train_model + self.act_model = act_model + self.step = act_model.step + self.value = act_model.value + self.initial_state = act_model.initial_state + + initialize() + + def train(self, lr, cliprange, obs, returns, masks, actions, values, neglogpacs, states=None): + """ + Train the model. + Here we calculate advantage A(s,a) = R + yV(s') - V(s) + + Returns + ------- + obj + = R + yV(s') + """ + advs = returns - values + + # Normalize the advantages + advs = (advs - advs.mean()) / (advs.std() + 1e-8) + + td_map = { + self.train_model.X : obs, + self.A : actions, + self.ADV : advs, + self.R : returns, + self.LR : lr, + self.CLIPRANGE : cliprange, + self.OLDNEGLOGPAC : neglogpacs, + self.OLDVPRED : values + } + if states is not None: + td_map[self.train_model.S] = states + td_map[self.train_model.M] = masks + + return self.sess.run( + self.stats_list + [self._train_op], + td_map + )[:-1] diff --git a/new_impl/cv/third_party/nni_new/algorithms/hpo/ppo_tuner/policy.py b/new_impl/cv/third_party/nni_new/algorithms/hpo/ppo_tuner/policy.py new file mode 100644 index 0000000000000000000000000000000000000000..a35e514eaef36562c26c6a99e8224b4339ae768c --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/hpo/ppo_tuner/policy.py @@ -0,0 +1,230 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +build policy/value network from model +""" + +import tensorflow.compat.v1 as tf +tf.disable_v2_behavior() + +from .distri import CategoricalPdType +from .util import lstm_model, fc, observation_placeholder, adjust_shape + + +class PolicyWithValue: + """ + Encapsulates fields and methods for RL policy and value function estimation with shared parameters + """ + + def __init__(self, env, observations, latent, estimate_q=False, vf_latent=None, sess=None, np_mask=None, is_act_model=False, **tensors): + """ + Parameters + ---------- + env : obj + RL environment + observations : tensorflow placeholder + Tensorflow placeholder in which the observations will be fed + latent : tensor + Latent state from which policy distribution parameters should be inferred + vf_latent : tensor + Latent state from which value function should be inferred (if None, then latent is used) + sess : tensorflow session + Tensorflow session to run calculations in (if None, default session is used) + **tensors + Tensorflow tensors for additional attributes such as state or mask + """ + + self.X = observations + self.state = tf.constant([]) + self.initial_state = None + self.__dict__.update(tensors) + + vf_latent = vf_latent if vf_latent is not None else latent + + vf_latent = tf.layers.flatten(vf_latent) + latent = tf.layers.flatten(latent) + + # Based on the action space, will select what probability distribution type + self.np_mask = np_mask + self.pdtype = CategoricalPdType(env.action_space.n, env.nsteps, np_mask, is_act_model) + + self.act_latent = latent + self.nh = env.action_space.n + + self.pd, self.pi, self.mask, self.mask_npinf = self.pdtype.pdfromlatent(latent, init_scale=0.01) + + # Take an action + self.action = self.pd.sample() + + # Calculate the neg log of our probability + self.neglogp = self.pd.neglogp(self.action) + self.sess = sess or tf.get_default_session() + + assert estimate_q is False + self.vf = fc(vf_latent, 'vf', 1) + self.vf = self.vf[:, 0] + + if is_act_model: + self._build_model_for_step() + + def _evaluate(self, variables, observation, **extra_feed): + sess = self.sess + feed_dict = {self.X: adjust_shape(self.X, observation)} + for inpt_name, data in extra_feed.items(): + if inpt_name in self.__dict__.keys(): + inpt = self.__dict__[inpt_name] + if isinstance(inpt, tf.Tensor) and inpt._op.type == 'Placeholder': + feed_dict[inpt] = adjust_shape(inpt, data) + + return sess.run(variables, feed_dict) + + def _build_model_for_step(self): + # multiply with weight and apply mask on self.act_latent to generate + self.act_step = step = tf.placeholder(shape=(), dtype=tf.int64, name='act_step') + with tf.variable_scope('pi', reuse=tf.AUTO_REUSE): + from .util import ortho_init + nin = self.act_latent.get_shape()[1].value + w = tf.get_variable("w", [nin, self.nh], initializer=ortho_init(0.01)) + b = tf.get_variable("b", [self.nh], initializer=tf.constant_initializer(0.0)) + logits = tf.matmul(self.act_latent, w)+b + piece = tf.slice(self.mask, [step, 0], [1, self.nh]) + re_piece = tf.reshape(piece, [-1]) + masked_logits = tf.math.multiply(logits, re_piece) + + npinf_piece = tf.slice(self.mask_npinf, [step, 0], [1, self.nh]) + re_npinf_piece = tf.reshape(npinf_piece, [-1]) + + def sample(logits, mask_npinf): + new_logits = tf.math.add(logits, mask_npinf) + u = tf.random_uniform(tf.shape(new_logits), dtype=logits.dtype) + return tf.argmax(new_logits - tf.log(-1*tf.log(u)), axis=-1) + + def neglogp(logits, x): + # return tf.nn.sparse_softmax_cross_entropy_with_logits(logits=self.logits, labels=x) + # Note: we can't use sparse_softmax_cross_entropy_with_logits because + # the implementation does not allow second-order derivatives... + if x.dtype in {tf.uint8, tf.int32, tf.int64}: + # one-hot encoding + x_shape_list = x.shape.as_list() + logits_shape_list = logits.get_shape().as_list()[:-1] + for xs, ls in zip(x_shape_list, logits_shape_list): + if xs is not None and ls is not None: + assert xs == ls, 'shape mismatch: {} in x vs {} in logits'.format(xs, ls) + + x = tf.one_hot(x, logits.get_shape().as_list()[-1]) + else: + # already encoded + assert x.shape.as_list() == logits.shape.as_list() + + return tf.nn.softmax_cross_entropy_with_logits_v2( + logits=logits, + labels=x) + + self.act_action = sample(masked_logits, re_npinf_piece) + self.act_neglogp = neglogp(masked_logits, self.act_action) + + + def step(self, step, observation, **extra_feed): + """ + Compute next action(s) given the observation(s) + + Parameters + ---------- + observation : np array + Observation data (either single or a batch) + **extra_feed + Additional data such as state or mask (names of the arguments should match the ones in constructor, see __init__) + + Returns + ------- + (action, value estimate, next state, negative log likelihood of the action under current policy parameters) tuple + """ + extra_feed['act_step'] = step + a, v, state, neglogp = self._evaluate([self.act_action, self.vf, self.state, self.act_neglogp], observation, **extra_feed) + if state.size == 0: + state = None + return a, v, state, neglogp + + def value(self, ob, *args, **kwargs): + """ + Compute value estimate(s) given the observation(s) + + Parameters + ---------- + observation : np array + Observation data (either single or a batch) + **extra_feed + Additional data such as state or mask (names of the arguments should match the ones in constructor, see __init__) + + Returns + ------- + Value estimate + """ + return self._evaluate(self.vf, ob, *args, **kwargs) + + +def build_lstm_policy(model_config, value_network=None, estimate_q=False, **policy_kwargs): + """ + Build lstm policy and value network, they share the same lstm network. + the parameters all use their default values. + + Parameter + --------- + model_config : obj + Configurations of the model + value_network : obj + The network for value function + estimate_q : bool + Whether to estimate ``q`` + **policy_kwargs + The kwargs for policy network, i.e., lstm model + + Returns + ------- + func + The policy network + """ + policy_network = lstm_model(**policy_kwargs) + + def policy_fn(nbatch=None, nsteps=None, sess=None, observ_placeholder=None, np_mask=None, is_act_model=False): + ob_space = model_config.observation_space + + X = observ_placeholder if observ_placeholder is not None else observation_placeholder(ob_space, batch_size=nbatch) + + extra_tensors = {} + + # encode_observation is not necessary anymore as we use embedding_lookup + encoded_x = X + + with tf.variable_scope('pi', reuse=tf.AUTO_REUSE): + policy_latent = policy_network(encoded_x, 1, model_config.observation_space.n) + if isinstance(policy_latent, tuple): + policy_latent, recurrent_tensors = policy_latent + + if recurrent_tensors is not None: + # recurrent architecture, need a few more steps + nenv = nbatch // nsteps + assert nenv > 0, 'Bad input for recurrent policy: batch size {} smaller than nsteps {}'.format(nbatch, nsteps) + policy_latent, recurrent_tensors = policy_network(encoded_x, nenv, model_config.observation_space.n) + extra_tensors.update(recurrent_tensors) + + _v_net = value_network + + assert _v_net is None or _v_net == 'shared' + vf_latent = policy_latent + + policy = PolicyWithValue( + env=model_config, + observations=X, + latent=policy_latent, + vf_latent=vf_latent, + sess=sess, + estimate_q=estimate_q, + np_mask=np_mask, + is_act_model=is_act_model, + **extra_tensors + ) + return policy + + return policy_fn diff --git a/new_impl/cv/third_party/nni_new/algorithms/hpo/ppo_tuner/ppo_tuner.py b/new_impl/cv/third_party/nni_new/algorithms/hpo/ppo_tuner/ppo_tuner.py new file mode 100644 index 0000000000000000000000000000000000000000..33b62d600e3b6bb788edab7db53a1223a846b2eb --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/hpo/ppo_tuner/ppo_tuner.py @@ -0,0 +1,654 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +ppo_tuner.py including: + class PPOTuner +""" + +import copy +import logging +import numpy as np +from gym import spaces +from schema import Schema, Optional + +import nni +from nni import ClassArgsValidator +from nni.tuner import Tuner +from nni.utils import OptimizeMode, extract_scalar_reward + +from .model import Model +from .util import set_global_seeds +from .policy import build_lstm_policy + + +logger = logging.getLogger('ppo_tuner_AutoML') + +def _constfn(val): + """ + Wrap as function + """ + def f(_): + return val + return f + + +class ModelConfig: + """ + Configurations of the PPO model + """ + def __init__(self): + self.observation_space = None + self.action_space = None + self.num_envs = 0 + self.nsteps = 0 + + self.ent_coef = 0.0 + self.lr = 3e-4 + self.vf_coef = 0.5 + self.max_grad_norm = 0.5 + self.gamma = 0.99 + self.lam = 0.95 + self.cliprange = 0.2 + self.embedding_size = None # the embedding is for each action + + self.noptepochs = 4 # number of training epochs per update + self.total_timesteps = 5000 # number of timesteps (i.e. number of actions taken in the environment) + self.nminibatches = 4 # number of training minibatches per update. For recurrent policies, + # should be smaller or equal than number of environments run in parallel. + +class TrialsInfo: + """ + Informations of each trial from one model inference + """ + def __init__(self, obs, actions, values, neglogpacs, dones, last_value, inf_batch_size): + self.iter = 0 + self.obs = obs + self.actions = actions + self.values = values + self.neglogpacs = neglogpacs + self.dones = dones + self.last_value = last_value + + self.rewards = None + self.returns = None + + self.inf_batch_size = inf_batch_size + #self.states = None + + def get_next(self): + """ + Get actions of the next trial + """ + if self.iter >= self.inf_batch_size: + return None, None + actions = [] + for step in self.actions: + actions.append(step[self.iter]) + self.iter += 1 + return self.iter - 1, actions + + def update_rewards(self, rewards, returns): + """ + After the trial is finished, reward and return of this trial is updated + """ + self.rewards = rewards + self.returns = returns + + def convert_shape(self): + """ + Convert shape + """ + def sf01(arr): + """ + swap and then flatten axes 0 and 1 + """ + s = arr.shape + return arr.swapaxes(0, 1).reshape(s[0] * s[1], *s[2:]) + self.obs = sf01(self.obs) + self.returns = sf01(self.returns) + self.dones = sf01(self.dones) + self.actions = sf01(self.actions) + self.values = sf01(self.values) + self.neglogpacs = sf01(self.neglogpacs) + + +class PPOModel: + """ + PPO Model + """ + def __init__(self, model_config, mask): + self.model_config = model_config + self.states = None # initial state of lstm in policy/value network + self.nupdates = None # the number of func train is invoked, used to tune lr and cliprange + self.cur_update = 1 # record the current update + self.np_mask = mask # record the mask of each action within one trial + + set_global_seeds(None) + assert isinstance(self.model_config.lr, float) + self.lr = _constfn(self.model_config.lr) + assert isinstance(self.model_config.cliprange, float) + self.cliprange = _constfn(self.model_config.cliprange) + + # build lstm policy network, value share the same network + policy = build_lstm_policy(model_config) + + # Get the nb of env + nenvs = model_config.num_envs + + # Calculate the batch_size + self.nbatch = nbatch = nenvs * model_config.nsteps # num of record per update + nbatch_train = nbatch // model_config.nminibatches # get batch size + # self.nupdates is used to tune lr and cliprange + self.nupdates = self.model_config.total_timesteps // self.nbatch + + # Instantiate the model object (that creates act_model and train_model) + self.model = Model(policy=policy, nbatch_act=nenvs, nbatch_train=nbatch_train, + nsteps=model_config.nsteps, ent_coef=model_config.ent_coef, vf_coef=model_config.vf_coef, + max_grad_norm=model_config.max_grad_norm, np_mask=self.np_mask) + + self.states = self.model.initial_state + + logger.info('=== finished PPOModel initialization') + + def inference(self, num): + """ + Generate actions along with related info from policy network. + observation is the action of the last step. + + Parameters + ---------- + num: int + The number of trials to generate + + Returns + ------- + mb_obs : list + Observation of the ``num`` configurations + mb_actions : list + Actions of the ``num`` configurations + mb_values : list + Values from the value function of the ``num`` configurations + mb_neglogpacs : list + ``neglogp`` of the ``num`` configurations + mb_dones : list + To show whether the play is done, always ``True`` + last_values : tensorflow tensor + The last values of the ``num`` configurations, got with session run + """ + # Here, we init the lists that will contain the mb of experiences + mb_obs, mb_actions, mb_values, mb_dones, mb_neglogpacs = [], [], [], [], [] + # initial observation + # use the (n+1)th embedding to represent the first step action + first_step_ob = self.model_config.action_space.n + obs = [first_step_ob for _ in range(num)] + dones = [True for _ in range(num)] + states = self.states + # For n in range number of steps + for cur_step in range(self.model_config.nsteps): + # Given observations, get action value and neglopacs + # We already have self.obs because Runner superclass run self.obs[:] = env.reset() on init + actions, values, states, neglogpacs = self.model.step(cur_step, obs, S=states, M=dones) + mb_obs.append(obs.copy()) + mb_actions.append(actions) + mb_values.append(values) + mb_neglogpacs.append(neglogpacs) + mb_dones.append(dones) + + # Take actions in env and look the results + # Infos contains a ton of useful informations + obs[:] = actions + if cur_step == self.model_config.nsteps - 1: + dones = [True for _ in range(num)] + else: + dones = [False for _ in range(num)] + + #batch of steps to batch of rollouts + np_obs = np.asarray(obs) + mb_obs = np.asarray(mb_obs, dtype=np_obs.dtype) + mb_actions = np.asarray(mb_actions) + mb_values = np.asarray(mb_values, dtype=np.float32) + mb_neglogpacs = np.asarray(mb_neglogpacs, dtype=np.float32) + mb_dones = np.asarray(mb_dones, dtype=np.bool) + last_values = self.model.value(np_obs, S=states, M=dones) + + return mb_obs, mb_actions, mb_values, mb_neglogpacs, mb_dones, last_values + + def compute_rewards(self, trials_info, trials_result): + """ + Compute the rewards of the trials in trials_info based on trials_result, + and update the rewards in trials_info + + Parameters + ---------- + trials_info : TrialsInfo + Info of the generated trials + trials_result : list + Final results (e.g., acc) of the generated trials + """ + mb_rewards = np.asarray([trials_result for _ in trials_info.actions], dtype=np.float32) + # discount/bootstrap off value fn + mb_returns = np.zeros_like(mb_rewards) + mb_advs = np.zeros_like(mb_rewards) + lastgaelam = 0 + last_dones = np.asarray([True for _ in trials_result], dtype=np.bool) # ugly + for t in reversed(range(self.model_config.nsteps)): + if t == self.model_config.nsteps - 1: + nextnonterminal = 1.0 - last_dones + nextvalues = trials_info.last_value + else: + nextnonterminal = 1.0 - trials_info.dones[t+1] + nextvalues = trials_info.values[t+1] + delta = mb_rewards[t] + self.model_config.gamma * nextvalues * nextnonterminal - trials_info.values[t] + lastgaelam = delta + self.model_config.gamma * self.model_config.lam * nextnonterminal * lastgaelam + mb_advs[t] = lastgaelam # pylint: disable=unsupported-assignment-operation + mb_returns = mb_advs + trials_info.values + + trials_info.update_rewards(mb_rewards, mb_returns) + trials_info.convert_shape() + + def train(self, trials_info, nenvs): + """ + Train the policy/value network using trials_info + + Parameters + ---------- + trials_info : TrialsInfo + Complete info of the generated trials from the previous inference + nenvs : int + The batch size of the (previous) inference + """ + # keep frac decay for future optimization + if self.cur_update <= self.nupdates: + frac = 1.0 - (self.cur_update - 1.0) / self.nupdates + else: + logger.warning('current update (self.cur_update) %d has exceeded total updates (self.nupdates) %d', + self.cur_update, self.nupdates) + frac = 1.0 - (self.nupdates - 1.0) / self.nupdates + lrnow = self.lr(frac) + cliprangenow = self.cliprange(frac) + self.cur_update += 1 + + states = self.states + + assert states is not None # recurrent version + assert nenvs % self.model_config.nminibatches == 0 + envsperbatch = nenvs // self.model_config.nminibatches + envinds = np.arange(nenvs) + flatinds = np.arange(nenvs * self.model_config.nsteps).reshape(nenvs, self.model_config.nsteps) + for _ in range(self.model_config.noptepochs): + np.random.shuffle(envinds) + for start in range(0, nenvs, envsperbatch): + end = start + envsperbatch + mbenvinds = envinds[start:end] + mbflatinds = flatinds[mbenvinds].ravel() + slices = (arr[mbflatinds] for arr in (trials_info.obs, trials_info.returns, trials_info.dones, + trials_info.actions, trials_info.values, trials_info.neglogpacs)) + mbstates = states[mbenvinds] + self.model.train(lrnow, cliprangenow, *slices, mbstates) + +class PPOClassArgsValidator(ClassArgsValidator): + def validate_class_args(self, **kwargs): + Schema({ + 'optimize_mode': self.choices('optimize_mode', 'maximize', 'minimize'), + Optional('trials_per_update'): self.range('trials_per_update', int, 0, 99999), + Optional('epochs_per_update'): self.range('epochs_per_update', int, 0, 99999), + Optional('minibatch_size'): self.range('minibatch_size', int, 0, 99999), + Optional('ent_coef'): float, + Optional('lr'): float, + Optional('vf_coef'): float, + Optional('max_grad_norm'): float, + Optional('gamma'): float, + Optional('lam'): float, + Optional('cliprange'): float, + }).validate(kwargs) + +class PPOTuner(Tuner): + """ + PPOTuner, the implementation inherits the main logic of the implementation + [ppo2 from openai](https://github.com/openai/baselines/tree/master/baselines/ppo2), and is adapted for NAS scenario. + It uses ``lstm`` for its policy network and value network, policy and value share the same network. + """ + + def __init__(self, optimize_mode, trials_per_update=20, epochs_per_update=4, minibatch_size=4, + ent_coef=0.0, lr=3e-4, vf_coef=0.5, max_grad_norm=0.5, gamma=0.99, lam=0.95, cliprange=0.2): + """ + Initialization, PPO model is not initialized here as search space is not received yet. + + Parameters + ---------- + optimize_mode : str + maximize or minimize + trials_per_update : int + Number of trials to have for each model update + epochs_per_update : int + Number of epochs to run for each model update + minibatch_size : int + Minibatch size (number of trials) for the update + ent_coef : float + Policy entropy coefficient in the optimization objective + lr : float + Learning rate of the model (lstm network), constant + vf_coef : float + Value function loss coefficient in the optimization objective + max_grad_norm : float + Gradient norm clipping coefficient + gamma : float + Discounting factor + lam : float + Advantage estimation discounting factor (lambda in the paper) + cliprange : float + Cliprange in the PPO algorithm, constant + """ + self.optimize_mode = OptimizeMode(optimize_mode) + self.model_config = ModelConfig() + self.model = None + self.search_space = None + self.running_trials = {} # key: parameter_id, value: actions/states/etc. + self.inf_batch_size = trials_per_update # number of trials to generate in one inference + self.first_inf = True # indicate whether it is the first time to inference new trials + self.trials_result = [None for _ in range(self.inf_batch_size)] # results of finished trials + + self.credit = 0 # record the unsatisfied trial requests + self.param_ids = [] + self.finished_trials = 0 + self.chosen_arch_template = {} + + self.actions_spaces = None + self.actions_to_config = None + self.full_act_space = None + self.trials_info = None + + self.all_trials = {} # used to dedup the same trial, key: config, value: final result + + self.model_config.num_envs = self.inf_batch_size + self.model_config.noptepochs = epochs_per_update + self.model_config.nminibatches = minibatch_size + + self.send_trial_callback = None + logger.info('Finished PPOTuner initialization') + + def _process_nas_space(self, search_space): + actions_spaces = [] + actions_to_config = [] + for key, val in search_space.items(): + if val['_type'] == 'layer_choice': + actions_to_config.append((key, 'layer_choice')) + actions_spaces.append(val['_value']) + self.chosen_arch_template[key] = None + elif val['_type'] == 'input_choice': + candidates = val['_value']['candidates'] + n_chosen = val['_value']['n_chosen'] + if n_chosen not in [0, 1, [0, 1]]: + raise ValueError('Optional_input_size can only be 0, 1, or [0, 1], but the pecified one is %s' + % (n_chosen)) + if isinstance(n_chosen, list): + actions_to_config.append((key, 'input_choice')) + # FIXME: risk, candidates might also have None + actions_spaces.append(['None', *candidates]) + self.chosen_arch_template[key] = None + elif n_chosen == 1: + actions_to_config.append((key, 'input_choice')) + actions_spaces.append(candidates) + self.chosen_arch_template[key] = None + elif n_chosen == 0: + self.chosen_arch_template[key] = [] + else: + raise ValueError('Unsupported search space type: %s' % (val['_type'])) + + # calculate observation space + dedup = {} + for step in actions_spaces: + for action in step: + dedup[action] = 1 + full_act_space = [act for act, _ in dedup.items()] + assert len(full_act_space) == len(dedup) + observation_space = len(full_act_space) + nsteps = len(actions_spaces) + + return actions_spaces, actions_to_config, full_act_space, observation_space, nsteps + + def _generate_action_mask(self): + """ + Different step could have different action space. to deal with this case, we merge all the + possible actions into one action space, and use mask to indicate available actions for each step + """ + two_masks = [] + + mask = [] + for acts in self.actions_spaces: + one_mask = [0 for _ in range(len(self.full_act_space))] + for act in acts: + idx = self.full_act_space.index(act) + one_mask[idx] = 1 + mask.append(one_mask) + two_masks.append(mask) + + mask = [] + for acts in self.actions_spaces: + one_mask = [-np.inf for _ in range(len(self.full_act_space))] + for act in acts: + idx = self.full_act_space.index(act) + one_mask[idx] = 0 + mask.append(one_mask) + two_masks.append(mask) + + return np.asarray(two_masks, dtype=np.float32) + + def update_search_space(self, search_space): + """ + Get search space, currently the space only includes that for NAS + + Parameters + ---------- + search_space : dict + Search space for NAS + the format could be referred to search space spec (https://nni.readthedocs.io/en/latest/Tutorial/SearchSpaceSpec.html). + """ + logger.info('update search space %s', search_space) + assert self.search_space is None + self.search_space = search_space + + assert self.model_config.observation_space is None + assert self.model_config.action_space is None + + self.actions_spaces, self.actions_to_config, self.full_act_space, obs_space, nsteps = self._process_nas_space(search_space) + + self.model_config.observation_space = spaces.Discrete(obs_space) + self.model_config.action_space = spaces.Discrete(obs_space) + self.model_config.nsteps = nsteps + + # generate mask in numpy + mask = self._generate_action_mask() + + assert self.model is None + self.model = PPOModel(self.model_config, mask) + + def _actions_to_config(self, actions): + """ + Given actions, to generate the corresponding trial configuration + """ + chosen_arch = copy.deepcopy(self.chosen_arch_template) + for cnt, act in enumerate(actions): + act_name = self.full_act_space[act] + (_key, _type) = self.actions_to_config[cnt] + if _type == 'input_choice': + if act_name == 'None': + chosen_arch[_key] = {'_value': [], '_idx': []} + else: + candidates = self.search_space[_key]['_value']['candidates'] + idx = candidates.index(act_name) + chosen_arch[_key] = {'_value': [act_name], '_idx': [idx]} + elif _type == 'layer_choice': + idx = self.search_space[_key]['_value'].index(act_name) + chosen_arch[_key] = {'_value': act_name, '_idx': idx} + else: + raise ValueError('unrecognized key: {0}'.format(_type)) + return chosen_arch + + def generate_multiple_parameters(self, parameter_id_list, **kwargs): + """ + Returns multiple sets of trial (hyper-)parameters, as iterable of serializable objects. + + Parameters + ---------- + parameter_id_list : list of int + Unique identifiers for each set of requested hyper-parameters. + These will later be used in :meth:`receive_trial_result`. + **kwargs + Not used + + Returns + ------- + list + A list of newly generated configurations + """ + result = [] + self.send_trial_callback = kwargs['st_callback'] + for parameter_id in parameter_id_list: + had_exception = False + try: + logger.debug("generating param for %s", parameter_id) + res = self.generate_parameters(parameter_id, **kwargs) + except nni.NoMoreTrialError: + had_exception = True + if not had_exception: + result.append(res) + return result + + def generate_parameters(self, parameter_id, **kwargs): + """ + Generate parameters, if no trial configration for now, self.credit plus 1 to send the config later + + Parameters + ---------- + parameter_id : int + Unique identifier for requested hyper-parameters. + This will later be used in :meth:`receive_trial_result`. + **kwargs + Not used + + Returns + ------- + dict + One newly generated configuration + + """ + if self.first_inf: + self.trials_result = [None for _ in range(self.inf_batch_size)] + mb_obs, mb_actions, mb_values, mb_neglogpacs, mb_dones, last_values = self.model.inference(self.inf_batch_size) + self.trials_info = TrialsInfo(mb_obs, mb_actions, mb_values, mb_neglogpacs, + mb_dones, last_values, self.inf_batch_size) + self.first_inf = False + + trial_info_idx, actions = self.trials_info.get_next() + if trial_info_idx is None: + logger.debug('Credit added by one in parameters request') + self.credit += 1 + self.param_ids.append(parameter_id) + raise nni.NoMoreTrialError('no more parameters now.') + + self.running_trials[parameter_id] = trial_info_idx + new_config = self._actions_to_config(actions) + return new_config + + def _next_round_inference(self): + """ + Run a inference to generate next batch of configurations + """ + logger.debug('Start next round inference...') + self.finished_trials = 0 + self.model.compute_rewards(self.trials_info, self.trials_result) + self.model.train(self.trials_info, self.inf_batch_size) + self.running_trials = {} + # generate new trials + self.trials_result = [None for _ in range(self.inf_batch_size)] + mb_obs, mb_actions, mb_values, mb_neglogpacs, mb_dones, last_values = self.model.inference(self.inf_batch_size) + self.trials_info = TrialsInfo(mb_obs, mb_actions, + mb_values, mb_neglogpacs, + mb_dones, last_values, + self.inf_batch_size) + logger.debug('Next round inference complete.') + # check credit and submit new trials + for _ in range(self.credit): + trial_info_idx, actions = self.trials_info.get_next() + if trial_info_idx is None: + logger.warning('No enough trial config, trials_per_update is suggested to be larger than trialConcurrency') + break + assert self.param_ids + param_id = self.param_ids.pop() + self.running_trials[param_id] = trial_info_idx + new_config = self._actions_to_config(actions) + self.send_trial_callback(param_id, new_config) + self.credit -= 1 + logger.debug('Send new trial (%d, %s) for reducing credit', param_id, new_config) + + def receive_trial_result(self, parameter_id, parameters, value, **kwargs): + """ + Receive trial's result. if the number of finished trials equals self.inf_batch_size, start the next update to + train the model. + + Parameters + ---------- + parameter_id : int + Unique identifier of used hyper-parameters, same with :meth:`generate_parameters`. + parameters : dict + Hyper-parameters generated by :meth:`generate_parameters`. + value : dict + Result from trial (the return value of :func:`nni.report_final_result`). + """ + trial_info_idx = self.running_trials.pop(parameter_id, None) + assert trial_info_idx is not None + + value = extract_scalar_reward(value) + if self.optimize_mode == OptimizeMode.Minimize: + value = -value + + self.trials_result[trial_info_idx] = value + self.finished_trials += 1 + + logger.debug('receive_trial_result, parameter_id %d, trial_info_idx %d, finished_trials %d, inf_batch_size %d', + parameter_id, trial_info_idx, self.finished_trials, self.inf_batch_size) + if self.finished_trials == self.inf_batch_size: + logger.debug('Start next round inference in receive_trial_result') + self._next_round_inference() + + def trial_end(self, parameter_id, success, **kwargs): + """ + To deal with trial failure. If a trial fails, it is popped out from ``self.running_trials``, + and the final result of this trial is assigned with the average of the finished trials. + + Parameters + ---------- + parameter_id : int + Unique identifier for hyper-parameters used by this trial. + success : bool + True if the trial successfully completed; False if failed or terminated. + **kwargs + Not used + """ + if not success: + if parameter_id not in self.running_trials: + logger.warning('The trial is failed, but self.running_trial does not have this trial') + return + trial_info_idx = self.running_trials.pop(parameter_id, None) + assert trial_info_idx is not None + # use mean of finished trials as the result of this failed trial + values = [val for val in self.trials_result if val is not None] + logger.warning('In trial_end, values: %s', values) + self.trials_result[trial_info_idx] = (sum(values) / len(values)) if values else 0 + self.finished_trials += 1 + if self.finished_trials == self.inf_batch_size: + logger.debug('Start next round inference in trial_end') + self._next_round_inference() + + def import_data(self, data): + """ + Import additional data for tuning, not supported yet. + + Parameters + ---------- + data : list + A list of dictionarys, each of which has at least two keys, ``parameter`` and ``value`` + """ + logger.warning('PPOTuner cannot leverage imported data.') diff --git a/new_impl/cv/third_party/nni_new/algorithms/hpo/ppo_tuner/util.py b/new_impl/cv/third_party/nni_new/algorithms/hpo/ppo_tuner/util.py new file mode 100644 index 0000000000000000000000000000000000000000..605292de4002f114e935cb24a87c96921940e959 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/hpo/ppo_tuner/util.py @@ -0,0 +1,260 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +util functions +""" + +import os +import random +import multiprocessing +import numpy as np +import tensorflow.compat.v1 as tf +tf.disable_v2_behavior() +from gym.spaces import Discrete, Box, MultiDiscrete + +def set_global_seeds(i): + """set global seeds""" + rank = 0 + myseed = i + 1000 * rank if i is not None else None + tf.set_random_seed(myseed) + np.random.seed(myseed) + random.seed(myseed) + +def batch_to_seq(h, nbatch, nsteps, flat=False): + """convert from batch to sequence""" + if flat: + h = tf.reshape(h, [nbatch, nsteps]) + else: + h = tf.reshape(h, [nbatch, nsteps, -1]) + return [tf.squeeze(v, [1]) for v in tf.split(axis=1, num_or_size_splits=nsteps, value=h)] + +def seq_to_batch(h, flat=False): + """convert from sequence to batch""" + shape = h[0].get_shape().as_list() + if not flat: + assert len(shape) > 1 + nh = h[0].get_shape()[-1].value + return tf.reshape(tf.concat(axis=1, values=h), [-1, nh]) + else: + return tf.reshape(tf.stack(values=h, axis=1), [-1]) + +def lstm(xs, ms, s, scope, nh, init_scale=1.0): + """lstm cell""" + _, nin = [v.value for v in xs[0].get_shape()] # the first is nbatch + with tf.variable_scope(scope): + wx = tf.get_variable("wx", [nin, nh*4], initializer=ortho_init(init_scale)) + wh = tf.get_variable("wh", [nh, nh*4], initializer=ortho_init(init_scale)) + b = tf.get_variable("b", [nh*4], initializer=tf.constant_initializer(0.0)) + + c, h = tf.split(axis=1, num_or_size_splits=2, value=s) + for idx, (x, m) in enumerate(zip(xs, ms)): + c = c*(1-m) + h = h*(1-m) + z = tf.matmul(x, wx) + tf.matmul(h, wh) + b + i, f, o, u = tf.split(axis=1, num_or_size_splits=4, value=z) + i = tf.nn.sigmoid(i) + f = tf.nn.sigmoid(f) + o = tf.nn.sigmoid(o) + u = tf.tanh(u) + c = f*c + i*u + h = o*tf.tanh(c) + xs[idx] = h + s = tf.concat(axis=1, values=[c, h]) + return xs, s + +def lstm_model(nlstm=128, layer_norm=False): + """ + Builds LSTM (Long-Short Term Memory) network to be used in a policy. + Note that the resulting function returns not only the output of the LSTM + (i.e. hidden state of lstm for each step in the sequence), but also a dictionary + with auxiliary tensors to be set as policy attributes. + + Specifically, + S is a placeholder to feed current state (LSTM state has to be managed outside policy) + M is a placeholder for the mask (used to mask out observations after the end of the episode, but can be used for other purposes too) + initial_state is a numpy array containing initial lstm state (usually zeros) + state is the output LSTM state (to be fed into S at the next call) + + + An example of usage of lstm-based policy can be found here: common/tests/test_doc_examples.py/test_lstm_example + + Parameters + ---------- + nlstm : int + LSTM hidden state size + layer_norm : bool + if True, layer-normalized version of LSTM is used + + Returns + ------- + function that builds LSTM with a given input tensor / placeholder + """ + + def network_fn(X, nenv=1, obs_size=-1): + with tf.variable_scope("emb", reuse=tf.AUTO_REUSE): + w_emb = tf.get_variable("w_emb", [obs_size+1, 32]) + X = tf.nn.embedding_lookup(w_emb, X) + + nbatch = X.shape[0] + nsteps = nbatch // nenv + + h = tf.layers.flatten(X) + + M = tf.placeholder(tf.float32, [nbatch]) #mask (done t-1) + S = tf.placeholder(tf.float32, [nenv, 2*nlstm]) #states + + xs = batch_to_seq(h, nenv, nsteps) + ms = batch_to_seq(M, nenv, nsteps) + + assert not layer_norm + h5, snew = lstm(xs, ms, S, scope='lstm', nh=nlstm) + + h = seq_to_batch(h5) + initial_state = np.zeros(S.shape.as_list(), dtype=float) + + return h, {'S':S, 'M':M, 'state':snew, 'initial_state':initial_state} + + return network_fn + +def ortho_init(scale=1.0): + """init approach""" + def _ortho_init(shape, dtype, partition_info=None): + #lasagne ortho init for tf + shape = tuple(shape) + if len(shape) == 2: + flat_shape = shape + elif len(shape) == 4: # assumes NHWC + flat_shape = (np.prod(shape[:-1]), shape[-1]) + else: + raise NotImplementedError + a = np.random.normal(0.0, 1.0, flat_shape) + u, _, v = np.linalg.svd(a, full_matrices=False) + q = u if u.shape == flat_shape else v # pick the one with the correct shape + q = q.reshape(shape) + return (scale * q[:shape[0], :shape[1]]).astype(np.float32) + return _ortho_init + +def fc(x, scope, nh, *, init_scale=1.0, init_bias=0.0): + """fully connected op""" + with tf.variable_scope(scope): + nin = x.get_shape()[1].value + w = tf.get_variable("w", [nin, nh], initializer=ortho_init(init_scale)) + b = tf.get_variable("b", [nh], initializer=tf.constant_initializer(init_bias)) + return tf.matmul(x, w)+b + +def _check_shape(placeholder_shape, data_shape): + """ + check if two shapes are compatible (i.e. differ only by dimensions of size 1, or by the batch dimension) + """ + + return True + +# ================================================================ +# Shape adjustment for feeding into tf placeholders +# ================================================================ +def adjust_shape(placeholder, data): + """ + adjust shape of the data to the shape of the placeholder if possible. + If shape is incompatible, AssertionError is thrown + + Parameters + ---------- + placeholder + tensorflow input placeholder + data + input data to be (potentially) reshaped to be fed into placeholder + + Returns + ------- + reshaped data + """ + if not isinstance(data, np.ndarray) and not isinstance(data, list): + return data + if isinstance(data, list): + data = np.array(data) + + placeholder_shape = [x or -1 for x in placeholder.shape.as_list()] + + assert _check_shape(placeholder_shape, data.shape), \ + 'Shape of data {} is not compatible with shape of the placeholder {}'.format(data.shape, placeholder_shape) + + return np.reshape(data, placeholder_shape) + +# ================================================================ +# Global session +# ================================================================ + +def get_session(config=None): + """Get default session or create one with a given config""" + sess = tf.get_default_session() + if sess is None: + sess = make_session(config=config, make_default=True) + return sess + +def make_session(config=None, num_cpu=None, make_default=False, graph=None): + """Returns a session that will use CPU's only""" + if num_cpu is None: + num_cpu = int(os.getenv('RCALL_NUM_CPU', multiprocessing.cpu_count())) + if config is None: + config = tf.ConfigProto( + allow_soft_placement=True, + inter_op_parallelism_threads=num_cpu, + intra_op_parallelism_threads=num_cpu) + config.gpu_options.allow_growth = True + + if make_default: + return tf.InteractiveSession(config=config, graph=graph) + else: + return tf.Session(config=config, graph=graph) + +ALREADY_INITIALIZED = set() + +def initialize(): + """Initialize all the uninitialized variables in the global scope.""" + new_variables = set(tf.global_variables()) - ALREADY_INITIALIZED + get_session().run(tf.variables_initializer(new_variables)) + + ALREADY_INITIALIZED.update(new_variables) + +def observation_placeholder(ob_space, batch_size=None, name='Ob'): + """ + Create placeholder to feed observations into of the size appropriate to the observation space + + Parameters + ---------- + ob_space : gym.Space + observation space + batch_size : int + size of the batch to be fed into input. Can be left None in most cases. + name : str + name of the placeholder + + Returns + ------- + tensorflow placeholder tensor + """ + + assert isinstance(ob_space, (Discrete, Box, MultiDiscrete)), \ + 'Can only deal with Discrete and Box observation spaces for now' + + dtype = ob_space.dtype + if dtype == np.int8: + dtype = np.uint8 + + return tf.placeholder(shape=(batch_size,) + ob_space.shape, dtype=dtype, name=name) + +def explained_variance(ypred, y): + """ + Computes fraction of variance that ypred explains about y. + Returns 1 - Var[y-ypred] / Var[y] + + interpretation: + ev=0 => might as well have predicted zero + ev=1 => perfect prediction + ev<0 => worse than just predicting zero + + """ + assert y.ndim == 1 and ypred.ndim == 1 + vary = np.var(y) + return np.nan if vary == 0 else 1 - np.var(y-ypred)/vary diff --git a/new_impl/cv/third_party/nni_new/algorithms/hpo/regularized_evolution_tuner.py b/new_impl/cv/third_party/nni_new/algorithms/hpo/regularized_evolution_tuner.py new file mode 100644 index 0000000000000000000000000000000000000000..ac756c33885b8ad150e862c6eb315136fb0d4b71 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/hpo/regularized_evolution_tuner.py @@ -0,0 +1,172 @@ +import copy +import logging +import random +from collections import deque + +from schema import Schema, Optional +import nni +from nni.tuner import Tuner +from nni import ClassArgsValidator +from nni.utils import OptimizeMode, extract_scalar_reward + +logger = logging.getLogger(__name__) + + +class FinishedIndividual: + def __init__(self, parameter_id, parameters, result): + """ + Parameters + ---------- + parameter_id: int + the index of the parameter + parameters : dict + chosen architecture and parameters + result : float + final metric of the chosen one + """ + self.parameter_id = parameter_id + self.parameters = parameters + self.result = result + + +class EvolutionClassArgsValidator(ClassArgsValidator): + def validate_class_args(self, **kwargs): + Schema({ + 'optimize_mode': self.choices('optimize_mode', 'maximize', 'minimize'), + Optional('population_size'): self.range('population_size', int, 0, 99999), + Optional('sample_size'): self.range('sample_size', int, 0, 9999), + }).validate(kwargs) + + +class RegularizedEvolutionTuner(Tuner): + """ + RegularizedEvolutionTuner is tuner using Evolution NAS Tuner. + See ``Regularized Evolution for Image Classifier Architecture Search`` for details. + + Parameters + --- + optimize_mode: str + whether to maximize metric or not. default: 'maximize' + population_size: int + the maximum number of kept models + sample_size: int + the number of models chosen from population each time when evolution + """ + def __init__(self, optimize_mode="maximize", population_size=100, sample_size=25): + super(RegularizedEvolutionTuner, self).__init__() + self.optimize_mode = OptimizeMode(optimize_mode) + self.population_size = population_size + self.sample_size = sample_size + self.initial_population = deque() + self.population = deque() + self.history = {} + self.search_space = None + self._from_initial = {} # whether the parameter is from initial population + + def generate_parameters(self, parameter_id, **kwargs): + """ + This function will returns a dict of trial (hyper-)parameters, as a serializable object. + + Parameters + --- + parameter_id: int + the index of current set of parameters + """ + if self.initial_population: + arch = self.initial_population.popleft() + self.history[parameter_id] = arch + self._from_initial[parameter_id] = True + return arch + elif self.population: + sample = [] + while len(sample) < self.sample_size: + sample.append(random.choice(list(self.population))) + + candidate = max(sample, key=lambda x: x.result) + arch = self._mutate_model(candidate) + self.history[parameter_id] = arch + self._from_initial[parameter_id] = False + return arch + else: + raise nni.NoMoreTrialError + + def receive_trial_result(self, parameter_id, parameters, value, **kwargs): + """ + Record the result from a trial + + Parameters + ---------- + parameter_id : int + parameters : dict + value : dict/float + if value is dict, it should have "default" key. + value is final metrics of the trial. + """ + reward = extract_scalar_reward(value) + if parameter_id not in self.history: + raise RuntimeError('Received parameter_id not in total_data.') + params = self.history[parameter_id] + + if self.optimize_mode == OptimizeMode.Minimize: + reward = -reward + + self.population.append(FinishedIndividual(parameter_id, params, reward)) + if len(self.population) > self.population_size: + self.population.popleft() + + def update_search_space(self, search_space): + """ + Update search space. + Search_space contains the information that user pre-defined. + + Parameters + ---------- + search_space : dict + """ + logger.info('update search space %s', search_space) + assert self.search_space is None + self.search_space = search_space + + for _, val in search_space.items(): + if val['_type'] != 'layer_choice' and val['_type'] != 'input_choice': + raise ValueError('Unsupported search space type: %s' % (val['_type'])) + + self._generate_initial_population() + + def trial_end(self, parameter_id, success, **kwargs): + if not success: + del self.history[parameter_id] + if self._from_initial[parameter_id]: + self.initial_population.append(self._random_model()) + del self._from_initial[parameter_id] + + def _mutate(self, key, individual): + mutate_val = self.search_space[key] + if mutate_val['_type'] == 'layer_choice': + idx = random.randint(0, len(mutate_val['_value']) - 1) + individual[key] = {'_value': mutate_val['_value'][idx], '_idx': idx} + elif mutate_val['_type'] == 'input_choice': + candidates = mutate_val['_value']['candidates'] + n_chosen = mutate_val['_value']['n_chosen'] + idxs = [random.randint(0, len(candidates) - 1) for _ in range(n_chosen)] + vals = [candidates[k] for k in idxs] + individual[key] = {'_value': vals, '_idx': idxs} + else: + raise KeyError + + def _random_model(self): + individual = {} + for key in self.search_space.keys(): + self._mutate(key, individual) + return individual + + def _mutate_model(self, model): + new_individual = copy.deepcopy(model.parameters) + mutate_key = random.choice(list(new_individual.keys())) + self._mutate(mutate_key, new_individual) + return new_individual + + def _generate_initial_population(self): + while len(self.initial_population) < self.population_size: + self.initial_population.append(self._random_model()) + logger.info('init population done.') diff --git a/new_impl/cv/third_party/nni_new/algorithms/hpo/smac_tuner/__init__.py b/new_impl/cv/third_party/nni_new/algorithms/hpo/smac_tuner/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9ac4c4bc4809aa0d04d67127a25ebfefa577be3c --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/hpo/smac_tuner/__init__.py @@ -0,0 +1,4 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .smac_tuner import SMACTuner, SMACClassArgsValidator diff --git a/new_impl/cv/third_party/nni_new/algorithms/hpo/smac_tuner/convert_ss_to_scenario.py b/new_impl/cv/third_party/nni_new/algorithms/hpo/smac_tuner/convert_ss_to_scenario.py new file mode 100644 index 0000000000000000000000000000000000000000..ef817c8cf4e10ad08185910b95bd4adc909b2dd3 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/hpo/smac_tuner/convert_ss_to_scenario.py @@ -0,0 +1,202 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import json + +import numpy as np + + +def get_json_content(file_path): + """ + Load json file content + + Parameters + ---------- + file_path: + path to the file + + Raises + ------ + TypeError + Error with the file path + """ + try: + with open(file_path, 'r') as file: + return json.load(file) + except TypeError as err: + print('Error: ', err) + return None + + +def generate_pcs(nni_search_space_content): + """ + Generate the Parameter Configuration Space (PCS) which defines the + legal ranges of the parameters to be optimized and their default values. + Generally, the format is: + # parameter_name categorical {value_1, ..., value_N} [default value] + # parameter_name ordinal {value_1, ..., value_N} [default value] + # parameter_name integer [min_value, max_value] [default value] + # parameter_name integer [min_value, max_value] [default value] log + # parameter_name real [min_value, max_value] [default value] + # parameter_name real [min_value, max_value] [default value] log + Reference: https://automl.github.io/SMAC3/stable/options.html + + Parameters + ---------- + nni_search_space_content: search_space + The search space in this experiment in nni + + Returns + ------- + Parameter Configuration Space (PCS) + the legal ranges of the parameters to be optimized and their default values + + Raises + ------ + RuntimeError + unsupported type or value error or incorrect search space + """ + categorical_dict = {} + search_space = nni_search_space_content + + def dump_categorical(fd, key, categories): + choice_len = len(categories) + if key in categorical_dict: + raise RuntimeError( + '%s has already existed, please make sure search space has no duplicate key.' % key) + categorical_dict[key] = search_space[key]['_value'] + fd.write('%s categorical {%s} [0]\n' % (key, ','.join(map(str, range(choice_len))))) + + with open('param_config_space.pcs', 'w') as pcs_fd: + if isinstance(search_space, dict): + for key in search_space.keys(): + if isinstance(search_space[key], dict): + try: + if search_space[key]['_type'] == 'choice': + dump_categorical(pcs_fd, key, search_space[key]['_value']) + elif search_space[key]['_type'] == 'randint': + lower, upper = search_space[key]['_value'] + if lower + 1 == upper: + dump_categorical(pcs_fd, key, [lower]) + else: + pcs_fd.write('%s integer [%d, %d] [%d]\n' % (key, lower, upper - 1, lower)) + elif search_space[key]['_type'] == 'uniform': + low, high = search_space[key]['_value'] + if low == high: + dump_categorical(pcs_fd, key, [low]) + else: + pcs_fd.write('%s real [%s, %s] [%s]\n' % (key, low, high, low)) + elif search_space[key]['_type'] == 'loguniform': + # use np.round here to ensure that the rounded default value is in the range, + # which will be rounded in configure_space package + low, high = list(np.round(np.log(search_space[key]['_value']), 10)) + if low == high: + dump_categorical(pcs_fd, key, [search_space[key]['_value'][0]]) + else: + pcs_fd.write('%s real [%s, %s] [%s]\n' % (key, low, high, low)) + elif search_space[key]['_type'] == 'quniform': + low, high, q = search_space[key]['_value'][0:3] + vals = np.clip(np.arange(np.round(low / q), np.round(high / q) + 1) * q, low, high).tolist() + pcs_fd.write('%s ordinal {%s} [%s]\n' % ( + key, + json.dumps(vals)[1:-1], + json.dumps(vals[0]))) + else: + raise RuntimeError('unsupported _type %s' % search_space[key]['_type']) + except: + raise RuntimeError('_type or _value error.') + else: + raise RuntimeError('incorrect search space.') + return categorical_dict + return None + + +def generate_scenario(ss_content): + """ + Generate the scenario. The scenario-object (smac.scenario.scenario.Scenario) is used to configure SMAC and + can be constructed either by providing an actual scenario-object, or by specifing the options in a scenario file. + Reference: https://automl.github.io/SMAC3/stable/options.html + The format of the scenario file is one option per line: + OPTION1 = VALUE1 + OPTION2 = VALUE2 + ... + Parameters + ---------- + abort_on_first_run_crash: bool + If true, SMAC will abort if the first run of the target algorithm crashes. Default: True, + because trials reported to nni tuner would always in success state + algo: function + Specifies the target algorithm call that SMAC will optimize. Interpreted as a bash-command. + Not required by tuner, but required by nni's training service for running trials + always_race_default: + Race new incumbents always against default configuration + cost_for_crash: + Defines the cost-value for crashed runs on scenarios with quality as run-obj. Default: 2147483647.0. + Trials reported to nni tuner would always in success state + cutoff_time: + Maximum runtime, after which the target algorithm is cancelled. `Required if *run_obj* is runtime` + deterministic: bool + If true, the optimization process will be repeatable. + execdir: + Specifies the path to the execution-directory. Default: . + Trials are executed by nni's training service + feature_file: + Specifies the file with the instance-features. + No features specified or feature file is not supported + initial_incumbent: + DEFAULT is the default from the PCS. Default: DEFAULT. Must be from: [‘DEFAULT’, ‘RANDOM’]. + input_psmac_dirs: + For parallel SMAC, multiple output-directories are used. + Parallelism is supported by nni + instance_file: + Specifies the file with the training-instances. Not supported + intensification_percentage: + The fraction of time to be used on intensification (versus choice of next Configurations). Default: 0.5. + Not supported, trials are controlled by nni's training service and kill be assessor + maxR: int + Maximum number of calls per configuration. Default: 2000. + memory_limit: + Maximum available memory the target algorithm can occupy before being cancelled. + minR: int + Minimum number of calls per configuration. Default: 1. + output_dir: + Specifies the output-directory for all emerging files, such as logging and results. + Default: smac3-output_2018-01-22_15:05:56_807070. + overall_obj: + PARX, where X is an integer defining the penalty imposed on timeouts (i.e. runtimes that exceed the cutoff-time). + Timeout is not supported + paramfile: + Specifies the path to the PCS-file. + run_obj: + Defines what metric to optimize. When optimizing runtime, cutoff_time is required as well. + Must be from: [‘runtime’, ‘quality’]. + runcount_limit: int + Maximum number of algorithm-calls during optimization. Default: inf. + Use default because this is controlled by nni + shared_model: + Whether to run SMAC in parallel mode. Parallelism is supported by nni + test_instance_file: + Specifies the file with the test-instances. Instance is not supported + tuner-timeout: + Maximum amount of CPU-time used for optimization. Not supported + wallclock_limit: int + Maximum amount of wallclock-time used for optimization. Default: inf. + Use default because this is controlled by nni + + Returns + ------- + Scenario: + The scenario-object (smac.scenario.scenario.Scenario) is used to configure SMAC and can be constructed + either by providing an actual scenario-object, or by specifing the options in a scenario file + """ + with open('scenario.txt', 'w') as sce_fd: + sce_fd.write('deterministic = 0\n') + # sce_fd.write('output_dir = \n') + sce_fd.write('paramfile = param_config_space.pcs\n') + sce_fd.write('run_obj = quality\n') + + return generate_pcs(ss_content) + + +if __name__ == '__main__': + generate_scenario('search_space.json') diff --git a/new_impl/cv/third_party/nni_new/algorithms/hpo/smac_tuner/smac_tuner.py b/new_impl/cv/third_party/nni_new/algorithms/hpo/smac_tuner/smac_tuner.py new file mode 100644 index 0000000000000000000000000000000000000000..fdaed49d8f2480516ec1e1e62fc43eabdd574049 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/hpo/smac_tuner/smac_tuner.py @@ -0,0 +1,346 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +smac_tuner.py +""" + +import logging +import sys + +import numpy as np +from schema import Schema, Optional + +from smac.facade.epils_facade import EPILS +from smac.facade.roar_facade import ROAR +from smac.facade.smac_facade import SMAC +from smac.scenario.scenario import Scenario +from smac.utils.io.cmd_reader import CMDReader + +from ConfigSpaceNNI import Configuration + +import nni +from nni import ClassArgsValidator +from nni.tuner import Tuner +from nni.utils import OptimizeMode, extract_scalar_reward + +from .convert_ss_to_scenario import generate_scenario + +logger = logging.getLogger('smac_AutoML') + +class SMACClassArgsValidator(ClassArgsValidator): + def validate_class_args(self, **kwargs): + Schema({ + 'optimize_mode': self.choices('optimize_mode', 'maximize', 'minimize'), + Optional('config_dedup'): bool + }).validate(kwargs) + +class SMACTuner(Tuner): + """ + This is a wrapper of [SMAC](https://github.com/automl/SMAC3) following NNI tuner interface. + It only supports ``SMAC`` mode, and does not support the multiple instances of SMAC3 (i.e., + the same configuration is run multiple times). + """ + def __init__(self, optimize_mode="maximize", config_dedup=False): + """ + Parameters + ---------- + optimize_mode : str + Optimize mode, 'maximize' or 'minimize', by default 'maximize' + config_dedup : bool + If True, the tuner will not generate a configuration that has been already generated. + If False, a configuration may be generated twice, but it is rare for relatively large search space. + """ + self.logger = logger + self.optimize_mode = OptimizeMode(optimize_mode) + self.total_data = {} + self.optimizer = None + self.smbo_solver = None + self.first_one = True + self.update_ss_done = False + self.loguniform_key = set() + self.categorical_dict = {} + self.cs = None + self.dedup = config_dedup + + def _main_cli(self): + """ + Main function of SMAC for CLI interface. Some initializations of the wrapped SMAC are done + in this function. + + Returns + ------- + obj + The object of the SMAC optimizer + """ + self.logger.info("SMAC call: %s", " ".join(sys.argv)) + + cmd_reader = CMDReader() + args, _ = cmd_reader.read_cmd() + + root_logger = logging.getLogger() + root_logger.setLevel(args.verbose_level) + logger_handler = logging.StreamHandler(stream=sys.stdout) + if root_logger.level >= logging.INFO: + formatter = logging.Formatter("%(levelname)s:\t%(message)s") + else: + formatter = logging.Formatter( + "%(asctime)s:%(levelname)s:%(name)s:%(message)s", + "%Y-%m-%d %H:%M:%S") + logger_handler.setFormatter(formatter) + root_logger.addHandler(logger_handler) + # remove default handler + root_logger.removeHandler(root_logger.handlers[0]) + + # Create defaults + rh = None + initial_configs = None + stats = None + incumbent = None + + # Create scenario-object + scen = Scenario(args.scenario_file, []) + self.cs = scen.cs + + if args.mode == "SMAC": + optimizer = SMAC( + scenario=scen, + rng=np.random.RandomState(args.seed), + runhistory=rh, + initial_configurations=initial_configs, + stats=stats, + restore_incumbent=incumbent, + run_id=args.seed) + elif args.mode == "ROAR": + optimizer = ROAR( + scenario=scen, + rng=np.random.RandomState(args.seed), + runhistory=rh, + initial_configurations=initial_configs, + run_id=args.seed) + elif args.mode == "EPILS": + optimizer = EPILS( + scenario=scen, + rng=np.random.RandomState(args.seed), + runhistory=rh, + initial_configurations=initial_configs, + run_id=args.seed) + else: + optimizer = None + + return optimizer + + def update_search_space(self, search_space): + """ + Convert search_space to the format that ``SMAC3`` could recognize, thus, not all the search space types + are supported. In this function, we also do the initialization of `SMAC3`, i.e., calling ``self._main_cli``. + + NOTE: updating search space during experiment running is not supported. + + Parameters + ---------- + search_space : dict + The format could be referred to search space spec (https://nni.readthedocs.io/en/latest/Tutorial/SearchSpaceSpec.html). + """ + self.logger.info('update search space in SMAC.') + if not self.update_ss_done: + self.categorical_dict = generate_scenario(search_space) + if self.categorical_dict is None: + raise RuntimeError('categorical dict is not correctly returned after parsing search space.') + # TODO: this is ugly, we put all the initialization work in this method, because initialization relies + # on search space, also because update_search_space is called at the beginning. + self.optimizer = self._main_cli() + self.smbo_solver = self.optimizer.solver + self.loguniform_key = {key for key in search_space.keys() if search_space[key]['_type'] == 'loguniform'} + self.update_ss_done = True + else: + self.logger.warning('update search space is not supported.') + + def receive_trial_result(self, parameter_id, parameters, value, **kwargs): + """ + Receive a trial's final performance result reported through :func:``nni.report_final_result`` by the trial. + GridSearchTuner does not need trial's results. + + Parameters + ---------- + parameter_id : int + Unique identifier of used hyper-parameters, same with :meth:`generate_parameters`. + parameters : dict + Hyper-parameters generated by :meth:`generate_parameters`. + value : dict + Result from trial (the return value of :func:`nni.report_final_result`). + + Raises + ------ + RuntimeError + Received parameter id not in ``self.total_data`` + """ + reward = extract_scalar_reward(value) + if self.optimize_mode is OptimizeMode.Maximize: + reward = -reward + + if parameter_id not in self.total_data: + raise RuntimeError('Received parameter_id not in total_data.') + if self.first_one: + self.smbo_solver.nni_smac_receive_first_run(self.total_data[parameter_id], reward) + self.first_one = False + else: + self.smbo_solver.nni_smac_receive_runs(self.total_data[parameter_id], reward) + + def param_postprocess(self, challenger_dict): + """ + Postprocessing for a set of hyperparameters includes: + 1. Convert the values of type ``loguniform`` back to their initial range. + 2. Convert ``categorical``: categorical values in search space are changed to list of numbers before, + those original values will be changed back in this function. + + Parameters + ---------- + challenger_dict : dict + challenger dict + + Returns + ------- + dict + dict which stores copy of challengers + """ + converted_dict = {} + for key, value in challenger_dict.items(): + # convert to loguniform + if key in self.loguniform_key: + converted_dict[key] = np.exp(challenger_dict[key]) + # convert categorical back to original value + elif key in self.categorical_dict: + idx = challenger_dict[key] + converted_dict[key] = self.categorical_dict[key][idx] + else: + converted_dict[key] = value + return converted_dict + + def generate_parameters(self, parameter_id, **kwargs): + """ + Generate one instance of hyperparameters (i.e., one configuration). + Get one from SMAC3's ``challengers``. + + Parameters + ---------- + parameter_id : int + Unique identifier for requested hyper-parameters. This will later be used in :meth:`receive_trial_result`. + **kwargs + Not used + + Returns + ------- + dict + One newly generated configuration + """ + if self.first_one: + init_challenger = self.smbo_solver.nni_smac_start() + self.total_data[parameter_id] = init_challenger + return self.param_postprocess(init_challenger.get_dictionary()) + else: + challengers = self.smbo_solver.nni_smac_request_challengers() + challengers_empty = True + for challenger in challengers: + challengers_empty = False + if self.dedup: + match = [v for k, v in self.total_data.items() \ + if v.get_dictionary() == challenger.get_dictionary()] + if match: + continue + self.total_data[parameter_id] = challenger + return self.param_postprocess(challenger.get_dictionary()) + assert challengers_empty is False, 'The case that challengers is empty is not handled.' + self.logger.info('In generate_parameters: No more new parameters.') + raise nni.NoMoreTrialError('No more new parameters.') + + def generate_multiple_parameters(self, parameter_id_list, **kwargs): + """ + Generate mutiple instances of hyperparameters. If it is a first request, + retrieve the instances from initial challengers. While if it is not, request + new challengers and retrieve instances from the requested challengers. + + Parameters + ---------- + parameter_id_list: list of int + Unique identifiers for each set of requested hyper-parameters. + These will later be used in :meth:`receive_trial_result`. + **kwargs + Not used + + Returns + ------- + list + a list of newly generated configurations + """ + if self.first_one: + params = [] + for one_id in parameter_id_list: + init_challenger = self.smbo_solver.nni_smac_start() + self.total_data[one_id] = init_challenger + params.append(self.param_postprocess(init_challenger.get_dictionary())) + else: + challengers = self.smbo_solver.nni_smac_request_challengers() + cnt = 0 + params = [] + for challenger in challengers: + if cnt >= len(parameter_id_list): + break + if self.dedup: + match = [v for k, v in self.total_data.items() \ + if v.get_dictionary() == challenger.get_dictionary()] + if match: + continue + self.total_data[parameter_id_list[cnt]] = challenger + params.append(self.param_postprocess(challenger.get_dictionary())) + cnt += 1 + if self.dedup and not params: + self.logger.info('In generate_multiple_parameters: No more new parameters.') + return params + + def import_data(self, data): + """ + Import additional data for tuning. + + Parameters + ---------- + data : list of dict + Each of which has at least two keys, ``parameter`` and ``value``. + """ + _completed_num = 0 + for trial_info in data: + self.logger.info("Importing data, current processing progress %s / %s", _completed_num, len(data)) + # simply validate data format + assert "parameter" in trial_info + _params = trial_info["parameter"] + assert "value" in trial_info + _value = trial_info['value'] + if not _value: + self.logger.info("Useless trial data, value is %s, skip this trial data.", _value) + continue + _value = extract_scalar_reward(_value) + # convert the keys in loguniform and categorical types + valid_entry = True + for key, value in _params.items(): + if key in self.loguniform_key: + _params[key] = np.log(value) + elif key in self.categorical_dict: + if value in self.categorical_dict[key]: + _params[key] = self.categorical_dict[key].index(value) + else: + self.logger.info("The value %s of key %s is not in search space.", str(value), key) + valid_entry = False + break + if not valid_entry: + continue + # start import this data entry + _completed_num += 1 + config = Configuration(self.cs, values=_params) + if self.optimize_mode is OptimizeMode.Maximize: + _value = -_value + if self.first_one: + self.smbo_solver.nni_smac_receive_first_run(config, _value) + self.first_one = False + else: + self.smbo_solver.nni_smac_receive_runs(config, _value) + self.logger.info("Successfully import data to smac tuner, total data: %d, imported data: %d.", len(data), _completed_num) diff --git a/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/cdarts/__init__.py b/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/cdarts/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..ab34902e0ec5d4f7cbbf943f2dcaf03d0a7bc97d --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/cdarts/__init__.py @@ -0,0 +1,5 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .mutator import RegularizedDartsMutator, RegularizedMutatorParallel, DartsDiscreteMutator +from .trainer import CdartsTrainer diff --git a/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/cdarts/mutator.py b/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/cdarts/mutator.py new file mode 100644 index 0000000000000000000000000000000000000000..a0bf79040eafb86c336467246521fb9767f60144 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/cdarts/mutator.py @@ -0,0 +1,143 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import torch + +from apex.parallel import DistributedDataParallel # pylint: disable=import-error +from nni.algorithms.nas.pytorch.darts import DartsMutator # pylint: disable=wrong-import-order +from nni.nas.pytorch.mutables import LayerChoice # pylint: disable=wrong-import-order +from nni.nas.pytorch.mutator import Mutator # pylint: disable=wrong-import-order + + +class RegularizedDartsMutator(DartsMutator): + """ + This is :class:`~nni.algorithms.nas.pytorch.darts.DartsMutator` basically, with two differences. + + 1. Choices can be cut (bypassed). This is done by ``cut_choices``. Cutted choices will not be used in + forward pass and thus consumes no memory. + + 2. Regularization on choices, to prevent the mutator from overfitting on some choices. + """ + + def reset(self): + """ + Warnings + -------- + Renamed :func:`~reset_with_loss` to return regularization loss on reset. + """ + raise ValueError("You should probably call `reset_with_loss`.") + + def cut_choices(self, cut_num=2): + """ + Cut the choices with the smallest weights. + ``cut_num`` should be the accumulative number of cutting, e.g., if first time cutting + is 2, the second time should be 4 to cut another two. + + Parameters + ---------- + cut_num : int + Number of choices to cut, so far. + + Warnings + -------- + Though the parameters are set to :math:`-\infty` to be bypassed, they will still receive gradient of 0, + which introduced ``nan`` problem when calling ``optimizer.step()``. To solve this issue, a simple way is to + reset nan to :math:`-\infty` each time after the parameters are updated. + """ + # `cut_choices` is implemented but not used in current implementation of CdartsTrainer + for mutable in self.mutables: + if isinstance(mutable, LayerChoice): + _, idx = torch.topk(-self.choices[mutable.key], cut_num) + with torch.no_grad(): + for i in idx: + self.choices[mutable.key][i] = -float("inf") + + def reset_with_loss(self): + """ + Resample and return loss. If loss is 0, to avoid device issue, it will return ``None``. + + Currently loss penalty are proportional to the L1-norm of parameters corresponding + to modules if their type name contains certain substrings. These substrings include: ``poolwithoutbn``, + ``identity``, ``dilconv``. + """ + self._cache, reg_loss = self.sample_search() + return reg_loss + + def sample_search(self): + result = super().sample_search() + loss = [] + for mutable in self.mutables: + if isinstance(mutable, LayerChoice): + def need_reg(choice): + return any(t in str(type(choice)).lower() for t in ["poolwithoutbn", "identity", "dilconv"]) + + for i, choice in enumerate(mutable.choices): + if need_reg(choice): + norm = torch.abs(self.choices[mutable.key][i]) + if norm < 1E10: + loss.append(norm) + if not loss: + return result, None + return result, sum(loss) + + def export(self, logger=None): + """ + Export an architecture with logger. Genotype will be printed with logger. + + Returns + ------- + dict + A mapping from mutable keys to decisions. + """ + result = self.sample_final() + if hasattr(self.model, "plot_genotype") and logger is not None: + genotypes = self.model.plot_genotype(result, logger) + return result, genotypes + + +class RegularizedMutatorParallel(DistributedDataParallel): + """ + Parallelize :class:`~RegularizedDartsMutator`. + + This makes :func:`~RegularizedDartsMutator.reset_with_loss` method parallelized, + also allowing :func:`~RegularizedDartsMutator.cut_choices` and :func:`~RegularizedDartsMutator.export` + to be easily accessible. + """ + def reset_with_loss(self): + """ + Parallelized :func:`~RegularizedDartsMutator.reset_with_loss`. + """ + result = self.module.reset_with_loss() + self.callback_queued = False + return result + + def cut_choices(self, *args, **kwargs): + """ + Parallelized :func:`~RegularizedDartsMutator.cut_choices`. + """ + self.module.cut_choices(*args, **kwargs) + + def export(self, logger): + """ + Parallelized :func:`~RegularizedDartsMutator.export`. + """ + return self.module.export(logger) + + +class DartsDiscreteMutator(Mutator): + """ + A mutator that applies the final sampling result of a parent mutator on another model to train. + + Parameters + ---------- + model : nn.Module + The model to apply the mutator. + parent_mutator : nni.nas.pytorch.mutator.Mutator + The mutator that provides ``sample_final`` method, that will be called to get the architecture. + """ + def __init__(self, model, parent_mutator): + super().__init__(model) + self.__dict__["parent_mutator"] = parent_mutator # avoid parameters to be included + + def sample_search(self): + return self.parent_mutator.sample_final() diff --git a/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/cdarts/trainer.py b/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/cdarts/trainer.py new file mode 100644 index 0000000000000000000000000000000000000000..1a5174216fc4ffdc224e9cd6bd1483322005fe01 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/cdarts/trainer.py @@ -0,0 +1,275 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import json +import logging +import os + +import torch +import torch.nn as nn +import torch.nn.functional as F +import apex # pylint: disable=import-error +from apex.parallel import DistributedDataParallel # pylint: disable=import-error +from .mutator import RegularizedDartsMutator, RegularizedMutatorParallel, DartsDiscreteMutator # pylint: disable=wrong-import-order +from nni.nas.pytorch.utils import AverageMeterGroup # pylint: disable=wrong-import-order + +from .utils import CyclicIterator, TorchTensorEncoder, accuracy, reduce_metrics + +PHASE_SMALL = "small" +PHASE_LARGE = "large" + + +class InteractiveKLLoss(nn.Module): + def __init__(self, temperature): + super().__init__() + self.temperature = temperature + # self.kl_loss = nn.KLDivLoss(reduction = 'batchmean') + self.kl_loss = nn.KLDivLoss() + + def forward(self, student, teacher): + return self.kl_loss(F.log_softmax(student / self.temperature, dim=1), + F.softmax(teacher / self.temperature, dim=1)) + + +class CdartsTrainer(object): + """ + CDARTS trainer. + + Parameters + ---------- + model_small : nn.Module + PyTorch model to be trained. This is the search network of CDARTS. + model_large : nn.Module + PyTorch model to be trained. This is the evaluation network of CDARTS. + criterion : callable + Receives logits and ground truth label, return a loss tensor, e.g., ``nn.CrossEntropyLoss()``. + loaders : list of torch.utils.data.DataLoader + List of train data and valid data loaders, for training weights and architecture weights respectively. + samplers : list of torch.utils.data.Sampler + List of train data and valid data samplers. This can be PyTorch standard samplers if not distributed. + In distributed mode, sampler needs to have ``set_epoch`` method. Refer to data utils in CDARTS example for details. + logger : logging.Logger + The logger for logging. Will use nni logger by default (if logger is ``None``). + regular_coeff : float + The coefficient of regular loss. + regular_ratio : float + The ratio of regular loss. + warmup_epochs : int + The epochs to warmup the search network + fix_head : bool + ``True`` if fixing the paramters of auxiliary heads, else unfix the paramters of auxiliary heads. + epochs : int + Number of epochs planned for training. + steps_per_epoch : int + Steps of one epoch. + loss_alpha : float + The loss coefficient. + loss_T : float + The loss coefficient. + distributed : bool + ``True`` if using distributed training, else non-distributed training. + log_frequency : int + Step count per logging. + grad_clip : float + Gradient clipping for weights. + interactive_type : string + ``kl`` or ``smoothl1``. + output_path : string + Log storage path. + w_lr : float + Learning rate of the search network parameters. + w_momentum : float + Momentum of the search and the evaluation network. + w_weight_decay : float + The weight decay the search and the evaluation network parameters. + alpha_lr : float + Learning rate of the architecture parameters. + alpha_weight_decay : float + The weight decay the architecture parameters. + nasnet_lr : float + Learning rate of the evaluation network parameters. + local_rank : int + The number of thread. + share_module : bool + ``True`` if sharing the stem and auxiliary heads, else not sharing these modules. + """ + def __init__(self, model_small, model_large, criterion, loaders, samplers, logger=None, + regular_coeff=5, regular_ratio=0.2, warmup_epochs=2, fix_head=True, + epochs=32, steps_per_epoch=None, loss_alpha=2, loss_T=2, distributed=True, + log_frequency=10, grad_clip=5.0, interactive_type='kl', output_path='./outputs', + w_lr=0.2, w_momentum=0.9, w_weight_decay=3e-4, alpha_lr=0.2, alpha_weight_decay=1e-4, + nasnet_lr=0.2, local_rank=0, share_module=True): + if logger is None: + logger = logging.getLogger(__name__) + train_loader, valid_loader = loaders + train_sampler, valid_sampler = samplers + self.train_loader = CyclicIterator(train_loader, train_sampler, distributed) + self.valid_loader = CyclicIterator(valid_loader, valid_sampler, distributed) + + self.regular_coeff = regular_coeff + self.regular_ratio = regular_ratio + self.warmup_epochs = warmup_epochs + self.fix_head = fix_head + self.epochs = epochs + self.steps_per_epoch = steps_per_epoch + if self.steps_per_epoch is None: + self.steps_per_epoch = min(len(self.train_loader), len(self.valid_loader)) + self.loss_alpha = loss_alpha + self.grad_clip = grad_clip + if interactive_type == "kl": + self.interactive_loss = InteractiveKLLoss(loss_T) + elif interactive_type == "smoothl1": + self.interactive_loss = nn.SmoothL1Loss() + self.loss_T = loss_T + self.distributed = distributed + self.log_frequency = log_frequency + self.main_proc = not distributed or local_rank == 0 + + self.logger = logger + self.checkpoint_dir = output_path + if self.main_proc: + os.makedirs(self.checkpoint_dir, exist_ok=True) + if distributed: + torch.distributed.barrier() + + self.model_small = model_small + self.model_large = model_large + if self.fix_head: + for param in self.model_small.aux_head.parameters(): + param.requires_grad = False + for param in self.model_large.aux_head.parameters(): + param.requires_grad = False + + self.mutator_small = RegularizedDartsMutator(self.model_small).cuda() + self.mutator_large = DartsDiscreteMutator(self.model_large, self.mutator_small).cuda() + self.criterion = criterion + + self.optimizer_small = torch.optim.SGD(self.model_small.parameters(), w_lr, + momentum=w_momentum, weight_decay=w_weight_decay) + self.optimizer_large = torch.optim.SGD(self.model_large.parameters(), nasnet_lr, + momentum=w_momentum, weight_decay=w_weight_decay) + self.optimizer_alpha = torch.optim.Adam(self.mutator_small.parameters(), alpha_lr, + betas=(0.5, 0.999), weight_decay=alpha_weight_decay) + + if distributed: + apex.parallel.convert_syncbn_model(self.model_small) + apex.parallel.convert_syncbn_model(self.model_large) + self.model_small = DistributedDataParallel(self.model_small, delay_allreduce=True) + self.model_large = DistributedDataParallel(self.model_large, delay_allreduce=True) + self.mutator_small = RegularizedMutatorParallel(self.mutator_small, delay_allreduce=True) + if share_module: + self.model_small.callback_queued = True + self.model_large.callback_queued = True + # mutator large never gets optimized, so do not need parallelized + + def _warmup(self, phase, epoch): + assert phase in [PHASE_SMALL, PHASE_LARGE] + if phase == PHASE_SMALL: + model, optimizer = self.model_small, self.optimizer_small + elif phase == PHASE_LARGE: + model, optimizer = self.model_large, self.optimizer_large + model.train() + meters = AverageMeterGroup() + for step in range(self.steps_per_epoch): + x, y = next(self.train_loader) + x, y = x.cuda(), y.cuda() + + optimizer.zero_grad() + logits_main, _ = model(x) + loss = self.criterion(logits_main, y) + loss.backward() + + self._clip_grad_norm(model) + optimizer.step() + prec1, prec5 = accuracy(logits_main, y, topk=(1, 5)) + metrics = {"prec1": prec1, "prec5": prec5, "loss": loss} + metrics = reduce_metrics(metrics, self.distributed) + meters.update(metrics) + if self.main_proc and (step % self.log_frequency == 0 or step + 1 == self.steps_per_epoch): + self.logger.info("Epoch [%d/%d] Step [%d/%d] (%s) %s", epoch + 1, self.epochs, + step + 1, self.steps_per_epoch, phase, meters) + + def _clip_grad_norm(self, model): + if isinstance(model, DistributedDataParallel): + nn.utils.clip_grad_norm_(model.module.parameters(), self.grad_clip) + else: + nn.utils.clip_grad_norm_(model.parameters(), self.grad_clip) + + def _reset_nan(self, parameters): + with torch.no_grad(): + for param in parameters: + for i, p in enumerate(param): + if p != p: # equivalent to `isnan(p)` + param[i] = float("-inf") + + def _joint_train(self, epoch): + self.model_large.train() + self.model_small.train() + meters = AverageMeterGroup() + for step in range(self.steps_per_epoch): + trn_x, trn_y = next(self.train_loader) + val_x, val_y = next(self.valid_loader) + trn_x, trn_y = trn_x.cuda(), trn_y.cuda() + val_x, val_y = val_x.cuda(), val_y.cuda() + + # step 1. optimize architecture + self.optimizer_alpha.zero_grad() + self.optimizer_large.zero_grad() + reg_decay = max(self.regular_coeff * (1 - float(epoch - self.warmup_epochs) / ( + (self.epochs - self.warmup_epochs) * self.regular_ratio)), 0) + loss_regular = self.mutator_small.reset_with_loss() + if loss_regular: + loss_regular *= reg_decay + logits_search, emsemble_logits_search = self.model_small(val_x) + logits_main, emsemble_logits_main = self.model_large(val_x) + loss_cls = (self.criterion(logits_search, val_y) + self.criterion(logits_main, val_y)) / self.loss_alpha + loss_interactive = self.interactive_loss(emsemble_logits_search, emsemble_logits_main) * (self.loss_T ** 2) * self.loss_alpha + loss = loss_cls + loss_interactive + loss_regular + loss.backward() + self._clip_grad_norm(self.model_large) + self.optimizer_large.step() + self.optimizer_alpha.step() + # NOTE: need to call here `self._reset_nan(self.mutator_small.parameters())` if `cut_choices` + + # step 2. optimize op weights + self.optimizer_small.zero_grad() + with torch.no_grad(): + # resample architecture since parameters have been changed + self.mutator_small.reset_with_loss() + logits_search_train, _ = self.model_small(trn_x) + loss_weight = self.criterion(logits_search_train, trn_y) + loss_weight.backward() + self._clip_grad_norm(self.model_small) + self.optimizer_small.step() + + metrics = {"loss_cls": loss_cls, "loss_interactive": loss_interactive, + "loss_regular": loss_regular, "loss_weight": loss_weight} + metrics = reduce_metrics(metrics, self.distributed) + meters.update(metrics) + + if self.main_proc and (step % self.log_frequency == 0 or step + 1 == self.steps_per_epoch): + self.logger.info("Epoch [%d/%d] Step [%d/%d] (joint) %s", epoch + 1, self.epochs, + step + 1, self.steps_per_epoch, meters) + + def train(self): + for epoch in range(self.epochs): + if epoch < self.warmup_epochs: + with torch.no_grad(): # otherwise grads will be retained on the architecture params + self.mutator_small.reset_with_loss() + self._warmup(PHASE_SMALL, epoch) + else: + with torch.no_grad(): + self.mutator_large.reset() + self._warmup(PHASE_LARGE, epoch) + self._joint_train(epoch) + + self.export(os.path.join(self.checkpoint_dir, "epoch_{:02d}.json".format(epoch)), + os.path.join(self.checkpoint_dir, "epoch_{:02d}.genotypes".format(epoch))) + + def export(self, file, genotype_file): + if self.main_proc: + mutator_export, genotypes = self.mutator_small.export(self.logger) + with open(file, "w") as f: + json.dump(mutator_export, f, indent=2, sort_keys=True, cls=TorchTensorEncoder) + with open(genotype_file, "w") as f: + f.write(str(genotypes)) diff --git a/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/cdarts/utils.py b/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/cdarts/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..96afa9425633811327c158f54b8c63be95775455 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/cdarts/utils.py @@ -0,0 +1,76 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import json +import os + +import torch +import torch.distributed as dist + + +class CyclicIterator: + def __init__(self, loader, sampler, distributed): + self.loader = loader + self.sampler = sampler + self.epoch = 0 + self.distributed = distributed + self._next_epoch() + + def _next_epoch(self): + if self.distributed: + self.sampler.set_epoch(self.epoch) + self.iterator = iter(self.loader) + self.epoch += 1 + + def __len__(self): + return len(self.loader) + + def __iter__(self): + return self + + def __next__(self): + try: + return next(self.iterator) + except StopIteration: + self._next_epoch() + return next(self.iterator) + + +class TorchTensorEncoder(json.JSONEncoder): + def default(self, o): # pylint: disable=method-hidden + if isinstance(o, torch.Tensor): + return o.tolist() + return super().default(o) + + +def accuracy(output, target, topk=(1,)): + """ Computes the precision@k for the specified values of k """ + maxk = max(topk) + batch_size = target.size(0) + + _, pred = output.topk(maxk, 1, True, True) + pred = pred.t() + # one-hot case + if target.ndimension() > 1: + target = target.max(1)[1] + + correct = pred.eq(target.view(1, -1).expand_as(pred)) + + res = [] + for k in topk: + correct_k = correct[:k].reshape(-1).float().sum(0) + res.append(correct_k.mul_(1.0 / batch_size)) + return res + + +def reduce_tensor(tensor): + rt = tensor.clone() + dist.all_reduce(rt, op=dist.ReduceOp.SUM) + rt /= float(os.environ["WORLD_SIZE"]) + return rt + + +def reduce_metrics(metrics, distributed=False): + if distributed: + return {k: reduce_tensor(v).item() for k, v in metrics.items()} + return {k: v.item() for k, v in metrics.items()} diff --git a/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/classic_nas/__init__.py b/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/classic_nas/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..ec3f5a4894a0460d4a0582a0d6cd43af9bed77e2 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/classic_nas/__init__.py @@ -0,0 +1,4 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .mutator import get_and_apply_next_architecture diff --git a/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/classic_nas/mutator.py b/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/classic_nas/mutator.py new file mode 100644 index 0000000000000000000000000000000000000000..7254a8b0b4a34b5913dfc71d11f7364b86fc04b8 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/classic_nas/mutator.py @@ -0,0 +1,221 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import json +import logging +import os +import sys + +import torch + +import nni +from nni.runtime.env_vars import trial_env_vars +from nni.nas.pytorch.mutables import LayerChoice, InputChoice, MutableScope +from nni.nas.pytorch.mutator import Mutator + +logger = logging.getLogger(__name__) + +NNI_GEN_SEARCH_SPACE = "NNI_GEN_SEARCH_SPACE" +LAYER_CHOICE = "layer_choice" +INPUT_CHOICE = "input_choice" + + +def get_and_apply_next_architecture(model): + """ + Wrapper of :class:`~nni.nas.pytorch.classic_nas.mutator.ClassicMutator` to make it more meaningful, + similar to ``get_next_parameter`` for HPO. + + It will generate search space based on ``model``. + If env ``NNI_GEN_SEARCH_SPACE`` exists, this is in dry run mode for + generating search space for the experiment. + If not, there are still two mode, one is nni experiment mode where users + use ``nnictl`` to start an experiment. The other is standalone mode + where users directly run the trial command, this mode chooses the first + one(s) for each LayerChoice and InputChoice. + + Parameters + ---------- + model : nn.Module + User's model with search space (e.g., LayerChoice, InputChoice) embedded in it. + """ + ClassicMutator(model) + + +class ClassicMutator(Mutator): + """ + This mutator is to apply the architecture chosen from tuner. + It implements the forward function of LayerChoice and InputChoice, + to only activate the chosen ones. + + Parameters + ---------- + model : nn.Module + User's model with search space (e.g., LayerChoice, InputChoice) embedded in it. + """ + + def __init__(self, model): + super(ClassicMutator, self).__init__(model) + self._chosen_arch = {} + self._search_space = self._generate_search_space() + if NNI_GEN_SEARCH_SPACE in os.environ: + # dry run for only generating search space + self._dump_search_space(os.environ[NNI_GEN_SEARCH_SPACE]) + sys.exit(0) + + if trial_env_vars.NNI_PLATFORM is None: + logger.warning("This is in standalone mode, the chosen are the first one(s).") + self._chosen_arch = self._standalone_generate_chosen() + else: + # get chosen arch from tuner + self._chosen_arch = nni.get_next_parameter() + if self._chosen_arch is None: + if trial_env_vars.NNI_PLATFORM == "unittest": + # happens if NNI_PLATFORM is intentionally set, e.g., in UT + logger.warning("`NNI_PLATFORM` is set but `param` is None. Falling back to standalone mode.") + self._chosen_arch = self._standalone_generate_chosen() + else: + raise RuntimeError("Chosen architecture is None. This may be a platform error.") + self.reset() + + def _sample_layer_choice(self, mutable, idx, value, search_space_item): + """ + Convert layer choice to tensor representation. + + Parameters + ---------- + mutable : Mutable + idx : int + Number `idx` of list will be selected. + value : str + The verbose representation of the selected value. + search_space_item : list + The list for corresponding search space. + """ + # doesn't support multihot for layer choice yet + onehot_list = [False] * len(mutable) + assert 0 <= idx < len(mutable) and search_space_item[idx] == value, \ + "Index '{}' in search space '{}' is not '{}'".format(idx, search_space_item, value) + onehot_list[idx] = True + return torch.tensor(onehot_list, dtype=torch.bool) # pylint: disable=not-callable + + def _sample_input_choice(self, mutable, idx, value, search_space_item): + """ + Convert input choice to tensor representation. + + Parameters + ---------- + mutable : Mutable + idx : int + Number `idx` of list will be selected. + value : str + The verbose representation of the selected value. + search_space_item : list + The list for corresponding search space. + """ + candidate_repr = search_space_item["candidates"] + multihot_list = [False] * mutable.n_candidates + for i, v in zip(idx, value): + assert 0 <= i < mutable.n_candidates and candidate_repr[i] == v, \ + "Index '{}' in search space '{}' is not '{}'".format(i, candidate_repr, v) + assert not multihot_list[i], "'{}' is selected twice in '{}', which is not allowed.".format(i, idx) + multihot_list[i] = True + return torch.tensor(multihot_list, dtype=torch.bool) # pylint: disable=not-callable + + def sample_search(self): + """ + See :meth:`sample_final`. + """ + return self.sample_final() + + def sample_final(self): + """ + Convert the chosen arch and apply it on model. + """ + assert set(self._chosen_arch.keys()) == set(self._search_space.keys()), \ + "Unmatched keys, expected keys '{}' from search space, found '{}'.".format(self._search_space.keys(), + self._chosen_arch.keys()) + result = dict() + for mutable in self.mutables: + if isinstance(mutable, (LayerChoice, InputChoice)): + assert mutable.key in self._chosen_arch, \ + "Expected '{}' in chosen arch, but not found.".format(mutable.key) + data = self._chosen_arch[mutable.key] + assert isinstance(data, dict) and "_value" in data and "_idx" in data, \ + "'{}' is not a valid choice.".format(data) + if isinstance(mutable, LayerChoice): + result[mutable.key] = self._sample_layer_choice(mutable, data["_idx"], data["_value"], + self._search_space[mutable.key]["_value"]) + elif isinstance(mutable, InputChoice): + result[mutable.key] = self._sample_input_choice(mutable, data["_idx"], data["_value"], + self._search_space[mutable.key]["_value"]) + elif isinstance(mutable, MutableScope): + logger.info("Mutable scope '%s' is skipped during parsing choices.", mutable.key) + else: + raise TypeError("Unsupported mutable type: '%s'." % type(mutable)) + return result + + def _standalone_generate_chosen(self): + """ + Generate the chosen architecture for standalone mode, + i.e., choose the first one(s) for LayerChoice and InputChoice. + :: + { key_name: {"_value": "conv1", + "_idx": 0} } + { key_name: {"_value": ["in1"], + "_idx": [0]} } + Returns + ------- + dict + the chosen architecture + """ + chosen_arch = {} + for key, val in self._search_space.items(): + if val["_type"] == LAYER_CHOICE: + choices = val["_value"] + chosen_arch[key] = {"_value": choices[0], "_idx": 0} + elif val["_type"] == INPUT_CHOICE: + choices = val["_value"]["candidates"] + n_chosen = val["_value"]["n_chosen"] + if n_chosen is None: + n_chosen = len(choices) + chosen_arch[key] = {"_value": choices[:n_chosen], "_idx": list(range(n_chosen))} + else: + raise ValueError("Unknown key '%s' and value '%s'." % (key, val)) + return chosen_arch + + def _generate_search_space(self): + """ + Generate search space from mutables. + Here is the search space format: + :: + { key_name: {"_type": "layer_choice", + "_value": ["conv1", "conv2"]} } + { key_name: {"_type": "input_choice", + "_value": {"candidates": ["in1", "in2"], + "n_chosen": 1}} } + Returns + ------- + dict + the generated search space + """ + search_space = {} + for mutable in self.mutables: + # for now we only generate flattened search space + if isinstance(mutable, LayerChoice): + key = mutable.key + val = mutable.names + search_space[key] = {"_type": LAYER_CHOICE, "_value": val} + elif isinstance(mutable, InputChoice): + key = mutable.key + search_space[key] = {"_type": INPUT_CHOICE, + "_value": {"candidates": mutable.choose_from, + "n_chosen": mutable.n_chosen}} + elif isinstance(mutable, MutableScope): + logger.info("Mutable scope '%s' is skipped during generating search space.", mutable.key) + else: + raise TypeError("Unsupported mutable type: '%s'." % type(mutable)) + return search_space + + def _dump_search_space(self, file_path): + with open(file_path, "w") as ss_file: + json.dump(self._search_space, ss_file, sort_keys=True, indent=2) diff --git a/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/cream/__init__.py b/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/cream/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..43a038b4670c0e3e5adeb1d34a4ee025970a8b19 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/cream/__init__.py @@ -0,0 +1,4 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .trainer import CreamSupernetTrainer diff --git a/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/cream/trainer.py b/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/cream/trainer.py new file mode 100644 index 0000000000000000000000000000000000000000..50830ce64b1f458c299df755159f91747f8eefe7 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/cream/trainer.py @@ -0,0 +1,404 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +import torch +import logging + +from copy import deepcopy +from nni.nas.pytorch.trainer import Trainer +from nni.nas.pytorch.utils import AverageMeterGroup + +from .utils import accuracy, reduce_metrics + +logger = logging.getLogger(__name__) + + +class CreamSupernetTrainer(Trainer): + """ + This trainer trains a supernet and output prioritized architectures that can be used for other tasks. + + Parameters + ---------- + model : nn.Module + Model with mutables. + loss : callable + Called with logits and targets. Returns a loss tensor. + val_loss : callable + Called with logits and targets for validation only. Returns a loss tensor. + optimizer : Optimizer + Optimizer that optimizes the model. + num_epochs : int + Number of epochs of training. + train_loader : iterablez + Data loader of training. Raise ``StopIteration`` when one epoch is exhausted. + valid_loader : iterablez + Data loader of validation. Raise ``StopIteration`` when one epoch is exhausted. + mutator : Mutator + A mutator object that has been initialized with the model. + batch_size : int + Batch size. + log_frequency : int + Number of mini-batches to log metrics. + meta_sta_epoch : int + start epoch of using meta matching network to pick teacher architecture + update_iter : int + interval of updating meta matching networks + slices : int + batch size of mini training data in the process of training meta matching network + pool_size : int + board size + pick_method : basestring + how to pick teacher network + choice_num : int + number of operations in supernet + sta_num : int + layer number of each stage in supernet (5 stage in supernet) + acc_gap : int + maximum accuracy improvement to omit the limitation of flops + flops_dict : Dict + dictionary of each layer's operations in supernet + flops_fixed : int + flops of fixed part in supernet + local_rank : int + index of current rank + callbacks : list of Callback + Callbacks to plug into the trainer. See Callbacks. + """ + + def __init__(self, model, loss, val_loss, + optimizer, num_epochs, train_loader, valid_loader, + mutator=None, batch_size=64, log_frequency=None, + meta_sta_epoch=20, update_iter=200, slices=2, + pool_size=10, pick_method='meta', choice_num=6, + sta_num=(4, 4, 4, 4, 4), acc_gap=5, + flops_dict=None, flops_fixed=0, local_rank=0, callbacks=None): + assert torch.cuda.is_available() + super(CreamSupernetTrainer, self).__init__(model, mutator, loss, None, + optimizer, num_epochs, None, None, + batch_size, None, None, log_frequency, callbacks) + self.model = model + self.loss = loss + self.val_loss = val_loss + self.train_loader = train_loader + self.valid_loader = valid_loader + self.log_frequency = log_frequency + self.batch_size = batch_size + self.optimizer = optimizer + self.model = model + self.loss = loss + self.num_epochs = num_epochs + self.meta_sta_epoch = meta_sta_epoch + self.update_iter = update_iter + self.slices = slices + self.pick_method = pick_method + self.pool_size = pool_size + self.local_rank = local_rank + self.choice_num = choice_num + self.sta_num = sta_num + self.acc_gap = acc_gap + self.flops_dict = flops_dict + self.flops_fixed = flops_fixed + + self.current_student_arch = None + self.current_teacher_arch = None + self.main_proc = (local_rank == 0) + self.current_epoch = 0 + + self.prioritized_board = [] + + # size of prioritized board + def _board_size(self): + return len(self.prioritized_board) + + # select teacher architecture according to the logit difference + def _select_teacher(self): + self._replace_mutator_cand(self.current_student_arch) + + if self.pick_method == 'top1': + meta_value, teacher_cand = 0.5, sorted( + self.prioritized_board, reverse=True)[0][3] + elif self.pick_method == 'meta': + meta_value, cand_idx, teacher_cand = -1000000000, -1, None + for now_idx, item in enumerate(self.prioritized_board): + inputx = item[4] + output = torch.nn.functional.softmax(self.model(inputx), dim=1) + weight = self.model.module.forward_meta(output - item[5]) + if weight > meta_value: + meta_value = weight + cand_idx = now_idx + teacher_cand = self.prioritized_board[cand_idx][3] + assert teacher_cand is not None + meta_value = torch.nn.functional.sigmoid(-weight) + else: + raise ValueError('Method Not supported') + + return meta_value, teacher_cand + + # check whether to update prioritized board + def _isUpdateBoard(self, prec1, flops): + if self.current_epoch <= self.meta_sta_epoch: + return False + + if len(self.prioritized_board) < self.pool_size: + return True + + if prec1 > self.prioritized_board[-1][1] + self.acc_gap: + return True + + if prec1 > self.prioritized_board[-1][1] and flops < self.prioritized_board[-1][2]: + return True + + return False + + # update prioritized board + def _update_prioritized_board(self, inputs, teacher_output, outputs, prec1, flops): + if self._isUpdateBoard(prec1, flops): + val_prec1 = prec1 + training_data = deepcopy(inputs[:self.slices].detach()) + if len(self.prioritized_board) == 0: + features = deepcopy(outputs[:self.slices].detach()) + else: + features = deepcopy( + teacher_output[:self.slices].detach()) + self.prioritized_board.append( + (val_prec1, + prec1, + flops, + self.current_student_arch, + training_data, + torch.nn.functional.softmax( + features, + dim=1))) + self.prioritized_board = sorted( + self.prioritized_board, reverse=True) + + if len(self.prioritized_board) > self.pool_size: + del self.prioritized_board[-1] + + # only update student network weights + def _update_student_weights_only(self, grad_1): + for weight, grad_item in zip( + self.model.module.rand_parameters(self.current_student_arch), grad_1): + weight.grad = grad_item + torch.nn.utils.clip_grad_norm_( + self.model.module.rand_parameters(self.current_student_arch), 1) + self.optimizer.step() + for weight, grad_item in zip( + self.model.module.rand_parameters(self.current_student_arch), grad_1): + del weight.grad + + # only update meta networks weights + def _update_meta_weights_only(self, teacher_cand, grad_teacher): + for weight, grad_item in zip(self.model.module.rand_parameters( + teacher_cand, self.pick_method == 'meta'), grad_teacher): + weight.grad = grad_item + + # clip gradients + torch.nn.utils.clip_grad_norm_( + self.model.module.rand_parameters( + self.current_student_arch, self.pick_method == 'meta'), 1) + + self.optimizer.step() + for weight, grad_item in zip(self.model.module.rand_parameters( + teacher_cand, self.pick_method == 'meta'), grad_teacher): + del weight.grad + + # simulate sgd updating + def _simulate_sgd_update(self, w, g, optimizer): + return g * optimizer.param_groups[-1]['lr'] + w + + # split training images into several slices + def _get_minibatch_input(self, input): + slice = self.slices + x = deepcopy(input[:slice].clone().detach()) + return x + + # calculate 1st gradient of student architectures + def _calculate_1st_gradient(self, kd_loss): + self.optimizer.zero_grad() + grad = torch.autograd.grad( + kd_loss, + self.model.module.rand_parameters(self.current_student_arch), + create_graph=True) + return grad + + # calculate 2nd gradient of meta networks + def _calculate_2nd_gradient(self, validation_loss, teacher_cand, students_weight): + self.optimizer.zero_grad() + grad_student_val = torch.autograd.grad( + validation_loss, + self.model.module.rand_parameters(self.current_student_arch), + retain_graph=True) + + grad_teacher = torch.autograd.grad( + students_weight[0], + self.model.module.rand_parameters( + teacher_cand, + self.pick_method == 'meta'), + grad_outputs=grad_student_val) + return grad_teacher + + # forward training data + def _forward_training(self, x, meta_value): + self._replace_mutator_cand(self.current_student_arch) + output = self.model(x) + + with torch.no_grad(): + self._replace_mutator_cand(self.current_teacher_arch) + teacher_output = self.model(x) + soft_label = torch.nn.functional.softmax(teacher_output, dim=1) + + kd_loss = meta_value * \ + self._cross_entropy_loss_with_soft_target(output, soft_label) + return kd_loss + + # calculate soft target loss + def _cross_entropy_loss_with_soft_target(self, pred, soft_target): + logsoftmax = torch.nn.LogSoftmax() + return torch.mean(torch.sum(- soft_target * logsoftmax(pred), 1)) + + # forward validation data + def _forward_validation(self, input, target): + slice = self.slices + x = input[slice:slice * 2].clone() + + self._replace_mutator_cand(self.current_student_arch) + output_2 = self.model(x) + + validation_loss = self.loss(output_2, target[slice:slice * 2]) + return validation_loss + + def _isUpdateMeta(self, batch_idx): + isUpdate = True + isUpdate &= (self.current_epoch > self.meta_sta_epoch) + isUpdate &= (batch_idx > 0) + isUpdate &= (batch_idx % self.update_iter == 0) + isUpdate &= (self._board_size() > 0) + return isUpdate + + def _replace_mutator_cand(self, cand): + self.mutator._cache = cand + + # update meta matching networks + def _run_update(self, input, target, batch_idx): + if self._isUpdateMeta(batch_idx): + x = self._get_minibatch_input(input) + + meta_value, teacher_cand = self._select_teacher() + + kd_loss = self._forward_training(x, meta_value) + + # calculate 1st gradient + grad_1st = self._calculate_1st_gradient(kd_loss) + + # simulate updated student weights + students_weight = [ + self._simulate_sgd_update( + p, grad_item, self.optimizer) for p, grad_item in zip( + self.model.module.rand_parameters(self.current_student_arch), grad_1st)] + + # update student weights + self._update_student_weights_only(grad_1st) + + validation_loss = self._forward_validation(input, target) + + # calculate 2nd gradient + grad_teacher = self._calculate_2nd_gradient(validation_loss, teacher_cand, students_weight) + + # update meta matching networks + self._update_meta_weights_only(teacher_cand, grad_teacher) + + # delete internal variants + del grad_teacher, grad_1st, x, validation_loss, kd_loss, students_weight + + def _get_cand_flops(self, cand): + flops = 0 + for block_id, block in enumerate(cand): + if block == 'LayerChoice1' or block_id == 'LayerChoice23': + continue + for idx, choice in enumerate(cand[block]): + flops += self.flops_dict[block_id][idx] * (1 if choice else 0) + return flops + self.flops_fixed + + def train_one_epoch(self, epoch): + self.current_epoch = epoch + meters = AverageMeterGroup() + self.steps_per_epoch = len(self.train_loader) + for step, (input_data, target) in enumerate(self.train_loader): + self.mutator.reset() + self.current_student_arch = self.mutator._cache + + input_data, target = input_data.cuda(), target.cuda() + + # calculate flops of current architecture + cand_flops = self._get_cand_flops(self.mutator._cache) + + # update meta matching network + self._run_update(input_data, target, step) + + if self._board_size() > 0: + # select teacher architecture + meta_value, teacher_cand = self._select_teacher() + self.current_teacher_arch = teacher_cand + + # forward supernet + if self._board_size() == 0 or epoch <= self.meta_sta_epoch: + self._replace_mutator_cand(self.current_student_arch) + output = self.model(input_data) + + loss = self.loss(output, target) + kd_loss, teacher_output, teacher_cand = None, None, None + else: + self._replace_mutator_cand(self.current_student_arch) + output = self.model(input_data) + + gt_loss = self.loss(output, target) + + with torch.no_grad(): + self._replace_mutator_cand(self.current_teacher_arch) + teacher_output = self.model(input_data).detach() + + soft_label = torch.nn.functional.softmax(teacher_output, dim=1) + kd_loss = self._cross_entropy_loss_with_soft_target(output, soft_label) + + loss = (meta_value * kd_loss + (2 - meta_value) * gt_loss) / 2 + + # update network + self.optimizer.zero_grad() + loss.backward() + self.optimizer.step() + + # update metrics + prec1, prec5 = accuracy(output, target, topk=(1, 5)) + metrics = {"prec1": prec1, "prec5": prec5, "loss": loss} + metrics = reduce_metrics(metrics) + meters.update(metrics) + + # update prioritized board + self._update_prioritized_board(input_data, teacher_output, output, metrics['prec1'], cand_flops) + + if self.main_proc and (step % self.log_frequency == 0 or step + 1 == self.steps_per_epoch): + logger.info("Epoch [%d/%d] Step [%d/%d] %s", epoch + 1, self.num_epochs, + step + 1, len(self.train_loader), meters) + + if self.main_proc and self.num_epochs == epoch + 1: + for idx, i in enumerate(self.prioritized_board): + logger.info("No.%s %s", idx, i[:4]) + + def validate_one_epoch(self, epoch): + self.model.eval() + meters = AverageMeterGroup() + with torch.no_grad(): + for step, (x, y) in enumerate(self.valid_loader): + self.mutator.reset() + logits = self.model(x) + loss = self.val_loss(logits, y) + prec1, prec5 = accuracy(logits, y, topk=(1, 5)) + metrics = {"prec1": prec1, "prec5": prec5, "loss": loss} + metrics = reduce_metrics(metrics) + meters.update(metrics) + + if self.log_frequency is not None and step % self.log_frequency == 0: + logger.info("Epoch [%s/%s] Validation Step [%s/%s] %s", epoch + 1, + self.num_epochs, step + 1, len(self.valid_loader), meters) diff --git a/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/cream/utils.py b/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/cream/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7d71faa715b379616c1affa6aaca319977435e65 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/cream/utils.py @@ -0,0 +1,37 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + + +import os +import torch.distributed as dist + + +def accuracy(output, target, topk=(1,)): + """ Computes the precision@k for the specified values of k """ + maxk = max(topk) + batch_size = target.size(0) + + _, pred = output.topk(maxk, 1, True, True) + pred = pred.t() + # one-hot case + if target.ndimension() > 1: + target = target.max(1)[1] + + correct = pred.eq(target.reshape(1, -1).expand_as(pred)) + + res = [] + for k in topk: + correct_k = correct[:k].view(-1).float().sum(0) + res.append(correct_k.mul_(1.0 / batch_size)) + return res + + +def reduce_metrics(metrics): + return {k: reduce_tensor(v).item() for k, v in metrics.items()} + + +def reduce_tensor(tensor): + rt = tensor.clone() + dist.all_reduce(rt, op=dist.ReduceOp.SUM) + rt /= float(os.environ["WORLD_SIZE"]) + return rt diff --git a/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/darts/__init__.py b/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/darts/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..1a22790fb90b37591dd58f590a3b528b78b8257e --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/darts/__init__.py @@ -0,0 +1,5 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .mutator import DartsMutator +from .trainer import DartsTrainer diff --git a/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/darts/mutator.py b/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/darts/mutator.py new file mode 100644 index 0000000000000000000000000000000000000000..a4c3898a9b531ea160ea64a17d2e345f6b67d040 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/darts/mutator.py @@ -0,0 +1,85 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from nni.nas.pytorch.mutator import Mutator +from nni.nas.pytorch.mutables import LayerChoice, InputChoice + +_logger = logging.getLogger(__name__) + + +class DartsMutator(Mutator): + """ + Connects the model in a DARTS (differentiable) way. + + An extra connection is automatically inserted for each LayerChoice, when this connection is selected, there is no + op on this LayerChoice (namely a ``ZeroOp``), in which case, every element in the exported choice list is ``false`` + (not chosen). + + All input choice will be fully connected in the search phase. On exporting, the input choice will choose inputs based + on keys in ``choose_from``. If the keys were to be keys of LayerChoices, the top logit of the corresponding LayerChoice + will join the competition of input choice to compete against other logits. Otherwise, the logit will be assumed 0. + + It's possible to cut branches by setting parameter ``choices`` in a particular position to ``-inf``. After softmax, the + value would be 0. Framework will ignore 0 values and not connect. Note that the gradient on the ``-inf`` location will + be 0. Since manipulations with ``-inf`` will be ``nan``, you need to handle the gradient update phase carefully. + + Attributes + ---------- + choices: ParameterDict + dict that maps keys of LayerChoices to weighted-connection float tensors. + """ + def __init__(self, model): + super().__init__(model) + self.choices = nn.ParameterDict() + for mutable in self.mutables: + if isinstance(mutable, LayerChoice): + self.choices[mutable.key] = nn.Parameter(1.0E-3 * torch.randn(mutable.length + 1)) + + def device(self): + for v in self.choices.values(): + return v.device + + def sample_search(self): + result = dict() + for mutable in self.mutables: + if isinstance(mutable, LayerChoice): + result[mutable.key] = F.softmax(self.choices[mutable.key], dim=-1)[:-1] + elif isinstance(mutable, InputChoice): + result[mutable.key] = torch.ones(mutable.n_candidates, dtype=torch.bool, device=self.device()) + return result + + def sample_final(self): + result = dict() + edges_max = dict() + for mutable in self.mutables: + if isinstance(mutable, LayerChoice): + max_val, index = torch.max(F.softmax(self.choices[mutable.key], dim=-1)[:-1], 0) + edges_max[mutable.key] = max_val + result[mutable.key] = F.one_hot(index, num_classes=len(mutable)).view(-1).bool() + for mutable in self.mutables: + if isinstance(mutable, InputChoice): + if mutable.n_chosen is not None: + weights = [] + for src_key in mutable.choose_from: + if src_key not in edges_max: + _logger.warning("InputChoice.NO_KEY in '%s' is weighted 0 when selecting inputs.", mutable.key) + weights.append(edges_max.get(src_key, 0.)) + weights = torch.tensor(weights) # pylint: disable=not-callable + _, topk_edge_indices = torch.topk(weights, mutable.n_chosen) + selected_multihot = [] + for i, src_key in enumerate(mutable.choose_from): + if i not in topk_edge_indices and src_key in result: + # If an edge is never selected, there is no need to calculate any op on this edge. + # This is to eliminate redundant calculation. + result[src_key] = torch.zeros_like(result[src_key]) + selected_multihot.append(i in topk_edge_indices) + result[mutable.key] = torch.tensor(selected_multihot, dtype=torch.bool, device=self.device()) # pylint: disable=not-callable + else: + result[mutable.key] = torch.ones(mutable.n_candidates, dtype=torch.bool, device=self.device()) # pylint: disable=not-callable + return result diff --git a/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/darts/trainer.py b/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/darts/trainer.py new file mode 100644 index 0000000000000000000000000000000000000000..e2d8e1866b68ad259771140ab0a7b86c43185f67 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/darts/trainer.py @@ -0,0 +1,214 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import copy +import logging + +import torch +import torch.nn as nn +from nni.nas.pytorch.trainer import Trainer +from nni.nas.pytorch.utils import AverageMeterGroup + +from .mutator import DartsMutator + +logger = logging.getLogger(__name__) + + +class DartsTrainer(Trainer): + """ + DARTS trainer. + + Parameters + ---------- + model : nn.Module + PyTorch model to be trained. + loss : callable + Receives logits and ground truth label, return a loss tensor. + metrics : callable + Receives logits and ground truth label, return a dict of metrics. + optimizer : Optimizer + The optimizer used for optimizing the model. + num_epochs : int + Number of epochs planned for training. + dataset_train : Dataset + Dataset for training. Will be split for training weights and architecture weights. + dataset_valid : Dataset + Dataset for testing. + mutator : DartsMutator + Use in case of customizing your own DartsMutator. By default will instantiate a DartsMutator. + batch_size : int + Batch size. + workers : int + Workers for data loading. + device : torch.device + ``torch.device("cpu")`` or ``torch.device("cuda")``. + log_frequency : int + Step count per logging. + callbacks : list of Callback + list of callbacks to trigger at events. + arc_learning_rate : float + Learning rate of architecture parameters. + unrolled : float + ``True`` if using second order optimization, else first order optimization. + """ + def __init__(self, model, loss, metrics, + optimizer, num_epochs, dataset_train, dataset_valid, + mutator=None, batch_size=64, workers=4, device=None, log_frequency=None, + callbacks=None, arc_learning_rate=3.0E-4, unrolled=False): + super().__init__(model, mutator if mutator is not None else DartsMutator(model), + loss, metrics, optimizer, num_epochs, dataset_train, dataset_valid, + batch_size, workers, device, log_frequency, callbacks) + + self.ctrl_optim = torch.optim.Adam(self.mutator.parameters(), arc_learning_rate, betas=(0.5, 0.999), + weight_decay=1.0E-3) + self.unrolled = unrolled + + n_train = len(self.dataset_train) + split = n_train // 2 + indices = list(range(n_train)) + train_sampler = torch.utils.data.sampler.SubsetRandomSampler(indices[:split]) + valid_sampler = torch.utils.data.sampler.SubsetRandomSampler(indices[split:]) + self.train_loader = torch.utils.data.DataLoader(self.dataset_train, + batch_size=batch_size, + sampler=train_sampler, + num_workers=workers) + self.valid_loader = torch.utils.data.DataLoader(self.dataset_train, + batch_size=batch_size, + sampler=valid_sampler, + num_workers=workers) + self.test_loader = torch.utils.data.DataLoader(self.dataset_valid, + batch_size=batch_size, + num_workers=workers) + + def train_one_epoch(self, epoch): + self.model.train() + self.mutator.train() + meters = AverageMeterGroup() + for step, ((trn_X, trn_y), (val_X, val_y)) in enumerate(zip(self.train_loader, self.valid_loader)): + trn_X, trn_y = trn_X.to(self.device), trn_y.to(self.device) + val_X, val_y = val_X.to(self.device), val_y.to(self.device) + + # phase 1. architecture step + self.ctrl_optim.zero_grad() + if self.unrolled: + self._unrolled_backward(trn_X, trn_y, val_X, val_y) + else: + self._backward(val_X, val_y) + self.ctrl_optim.step() + + # phase 2: child network step + self.optimizer.zero_grad() + logits, loss = self._logits_and_loss(trn_X, trn_y) + loss.backward() + nn.utils.clip_grad_norm_(self.model.parameters(), 5.) # gradient clipping + self.optimizer.step() + + metrics = self.metrics(logits, trn_y) + metrics["loss"] = loss.item() + meters.update(metrics) + if self.log_frequency is not None and step % self.log_frequency == 0: + logger.info("Epoch [%s/%s] Step [%s/%s] %s", epoch + 1, + self.num_epochs, step + 1, len(self.train_loader), meters) + + def validate_one_epoch(self, epoch): + self.model.eval() + self.mutator.eval() + meters = AverageMeterGroup() + with torch.no_grad(): + self.mutator.reset() + for step, (X, y) in enumerate(self.test_loader): + X, y = X.to(self.device), y.to(self.device) + logits = self.model(X) + metrics = self.metrics(logits, y) + meters.update(metrics) + if self.log_frequency is not None and step % self.log_frequency == 0: + logger.info("Epoch [%s/%s] Step [%s/%s] %s", epoch + 1, + self.num_epochs, step + 1, len(self.test_loader), meters) + + def _logits_and_loss(self, X, y): + self.mutator.reset() + logits = self.model(X) + loss = self.loss(logits, y) + self._write_graph_status() + return logits, loss + + def _backward(self, val_X, val_y): + """ + Simple backward with gradient descent + """ + _, loss = self._logits_and_loss(val_X, val_y) + loss.backward() + + def _unrolled_backward(self, trn_X, trn_y, val_X, val_y): + """ + Compute unrolled loss and backward its gradients + """ + backup_params = copy.deepcopy(tuple(self.model.parameters())) + + # do virtual step on training data + lr = self.optimizer.param_groups[0]["lr"] + momentum = self.optimizer.param_groups[0]["momentum"] + weight_decay = self.optimizer.param_groups[0]["weight_decay"] + self._compute_virtual_model(trn_X, trn_y, lr, momentum, weight_decay) + + # calculate unrolled loss on validation data + # keep gradients for model here for compute hessian + _, loss = self._logits_and_loss(val_X, val_y) + w_model, w_ctrl = tuple(self.model.parameters()), tuple(self.mutator.parameters()) + w_grads = torch.autograd.grad(loss, w_model + w_ctrl) + d_model, d_ctrl = w_grads[:len(w_model)], w_grads[len(w_model):] + + # compute hessian and final gradients + hessian = self._compute_hessian(backup_params, d_model, trn_X, trn_y) + with torch.no_grad(): + for param, d, h in zip(w_ctrl, d_ctrl, hessian): + # gradient = dalpha - lr * hessian + param.grad = d - lr * h + + # restore weights + self._restore_weights(backup_params) + + def _compute_virtual_model(self, X, y, lr, momentum, weight_decay): + """ + Compute unrolled weights w` + """ + # don't need zero_grad, using autograd to calculate gradients + _, loss = self._logits_and_loss(X, y) + gradients = torch.autograd.grad(loss, self.model.parameters()) + with torch.no_grad(): + for w, g in zip(self.model.parameters(), gradients): + m = self.optimizer.state[w].get("momentum_buffer", 0.) + w = w - lr * (momentum * m + g + weight_decay * w) + + def _restore_weights(self, backup_params): + with torch.no_grad(): + for param, backup in zip(self.model.parameters(), backup_params): + param.copy_(backup) + + def _compute_hessian(self, backup_params, dw, trn_X, trn_y): + """ + dw = dw` { L_val(w`, alpha) } + w+ = w + eps * dw + w- = w - eps * dw + hessian = (dalpha { L_trn(w+, alpha) } - dalpha { L_trn(w-, alpha) }) / (2*eps) + eps = 0.01 / ||dw|| + """ + self._restore_weights(backup_params) + norm = torch.cat([w.view(-1) for w in dw]).norm() + eps = 0.01 / norm + if norm < 1E-8: + logger.warning("In computing hessian, norm is smaller than 1E-8, cause eps to be %.6f.", norm.item()) + + dalphas = [] + for e in [eps, -2. * eps]: + # w+ = w + eps*dw`, w- = w - eps*dw` + with torch.no_grad(): + for p, d in zip(self.model.parameters(), dw): + p += e * d + + _, loss = self._logits_and_loss(trn_X, trn_y) + dalphas.append(torch.autograd.grad(loss, self.mutator.parameters())) + + dalpha_pos, dalpha_neg = dalphas # dalpha { L_trn(w+) }, # dalpha { L_trn(w-) } + hessian = [(p - n) / (2. * eps) for p, n in zip(dalpha_pos, dalpha_neg)] + return hessian diff --git a/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/enas/__init__.py b/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/enas/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d3372836ebba2387ade58161218aad731433e46b --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/enas/__init__.py @@ -0,0 +1,5 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .mutator import EnasMutator +from .trainer import EnasTrainer diff --git a/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/enas/mutator.py b/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/enas/mutator.py new file mode 100644 index 0000000000000000000000000000000000000000..7fdba26b99bf72551293fdb77ff61beb974bba8f --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/enas/mutator.py @@ -0,0 +1,197 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from nni.nas.pytorch.mutator import Mutator +from nni.nas.pytorch.mutables import LayerChoice, InputChoice, MutableScope + + +class StackedLSTMCell(nn.Module): + def __init__(self, layers, size, bias): + super().__init__() + self.lstm_num_layers = layers + self.lstm_modules = nn.ModuleList([nn.LSTMCell(size, size, bias=bias) + for _ in range(self.lstm_num_layers)]) + + def forward(self, inputs, hidden): + prev_h, prev_c = hidden + next_h, next_c = [], [] + for i, m in enumerate(self.lstm_modules): + curr_h, curr_c = m(inputs, (prev_h[i], prev_c[i])) + next_c.append(curr_c) + next_h.append(curr_h) + # current implementation only supports batch size equals 1, + # but the algorithm does not necessarily have this limitation + inputs = curr_h[-1].view(1, -1) + return next_h, next_c + + +class EnasMutator(Mutator): + """ + A mutator that mutates the graph with RL. + + Parameters + ---------- + model : nn.Module + PyTorch model. + lstm_size : int + Controller LSTM hidden units. + lstm_num_layers : int + Number of layers for stacked LSTM. + tanh_constant : float + Logits will be equal to ``tanh_constant * tanh(logits)``. Don't use ``tanh`` if this value is ``None``. + cell_exit_extra_step : bool + If true, RL controller will perform an extra step at the exit of each MutableScope, dump the hidden state + and mark it as the hidden state of this MutableScope. This is to align with the original implementation of paper. + skip_target : float + Target probability that skipconnect will appear. + temperature : float + Temperature constant that divides the logits. + branch_bias : float + Manual bias applied to make some operations more likely to be chosen. + Currently this is implemented with a hardcoded match rule that aligns with original repo. + If a mutable has a ``reduce`` in its key, all its op choices + that contains `conv` in their typename will receive a bias of ``+self.branch_bias`` initially; while others + receive a bias of ``-self.branch_bias``. + entropy_reduction : str + Can be one of ``sum`` and ``mean``. How the entropy of multi-input-choice is reduced. + """ + + def __init__(self, model, lstm_size=64, lstm_num_layers=1, tanh_constant=1.5, cell_exit_extra_step=False, + skip_target=0.4, temperature=None, branch_bias=0.25, entropy_reduction="sum"): + super().__init__(model) + self.lstm_size = lstm_size + self.lstm_num_layers = lstm_num_layers + self.tanh_constant = tanh_constant + self.temperature = temperature + self.cell_exit_extra_step = cell_exit_extra_step + self.skip_target = skip_target + self.branch_bias = branch_bias + + self.lstm = StackedLSTMCell(self.lstm_num_layers, self.lstm_size, False) + self.attn_anchor = nn.Linear(self.lstm_size, self.lstm_size, bias=False) + self.attn_query = nn.Linear(self.lstm_size, self.lstm_size, bias=False) + self.v_attn = nn.Linear(self.lstm_size, 1, bias=False) + self.g_emb = nn.Parameter(torch.randn(1, self.lstm_size) * 0.1) + self.skip_targets = nn.Parameter(torch.tensor([1.0 - self.skip_target, self.skip_target]), requires_grad=False) # pylint: disable=not-callable + assert entropy_reduction in ["sum", "mean"], "Entropy reduction must be one of sum and mean." + self.entropy_reduction = torch.sum if entropy_reduction == "sum" else torch.mean + self.cross_entropy_loss = nn.CrossEntropyLoss(reduction="none") + self.bias_dict = nn.ParameterDict() + + self.max_layer_choice = 0 + for mutable in self.mutables: + if isinstance(mutable, LayerChoice): + if self.max_layer_choice == 0: + self.max_layer_choice = len(mutable) + assert self.max_layer_choice == len(mutable), \ + "ENAS mutator requires all layer choice have the same number of candidates." + # We are judging by keys and module types to add biases to layer choices. Needs refactor. + if "reduce" in mutable.key: + def is_conv(choice): + return "conv" in str(type(choice)).lower() + bias = torch.tensor([self.branch_bias if is_conv(choice) else -self.branch_bias # pylint: disable=not-callable + for choice in mutable]) + self.bias_dict[mutable.key] = nn.Parameter(bias, requires_grad=False) + + self.embedding = nn.Embedding(self.max_layer_choice + 1, self.lstm_size) + self.soft = nn.Linear(self.lstm_size, self.max_layer_choice, bias=False) + + def sample_search(self): + self._initialize() + self._sample(self.mutables) + return self._choices + + def sample_final(self): + return self.sample_search() + + def _sample(self, tree): + mutable = tree.mutable + if isinstance(mutable, LayerChoice) and mutable.key not in self._choices: + self._choices[mutable.key] = self._sample_layer_choice(mutable) + elif isinstance(mutable, InputChoice) and mutable.key not in self._choices: + self._choices[mutable.key] = self._sample_input_choice(mutable) + for child in tree.children: + self._sample(child) + if isinstance(mutable, MutableScope) and mutable.key not in self._anchors_hid: + if self.cell_exit_extra_step: + self._lstm_next_step() + self._mark_anchor(mutable.key) + + def _initialize(self): + self._choices = dict() + self._anchors_hid = dict() + self._inputs = self.g_emb.data + self._c = [torch.zeros((1, self.lstm_size), + dtype=self._inputs.dtype, + device=self._inputs.device) for _ in range(self.lstm_num_layers)] + self._h = [torch.zeros((1, self.lstm_size), + dtype=self._inputs.dtype, + device=self._inputs.device) for _ in range(self.lstm_num_layers)] + self.sample_log_prob = 0 + self.sample_entropy = 0 + self.sample_skip_penalty = 0 + + def _lstm_next_step(self): + self._h, self._c = self.lstm(self._inputs, (self._h, self._c)) + + def _mark_anchor(self, key): + self._anchors_hid[key] = self._h[-1] + + def _sample_layer_choice(self, mutable): + self._lstm_next_step() + logit = self.soft(self._h[-1]) + if self.temperature is not None: + logit /= self.temperature + if self.tanh_constant is not None: + logit = self.tanh_constant * torch.tanh(logit) + if mutable.key in self.bias_dict: + logit += self.bias_dict[mutable.key] + branch_id = torch.multinomial(F.softmax(logit, dim=-1), 1).view(-1) + log_prob = self.cross_entropy_loss(logit, branch_id) + self.sample_log_prob += self.entropy_reduction(log_prob) + entropy = (log_prob * torch.exp(-log_prob)).detach() # pylint: disable=invalid-unary-operand-type + self.sample_entropy += self.entropy_reduction(entropy) + self._inputs = self.embedding(branch_id) + return F.one_hot(branch_id, num_classes=self.max_layer_choice).bool().view(-1) + + def _sample_input_choice(self, mutable): + query, anchors = [], [] + for label in mutable.choose_from: + if label not in self._anchors_hid: + self._lstm_next_step() + self._mark_anchor(label) # empty loop, fill not found + query.append(self.attn_anchor(self._anchors_hid[label])) + anchors.append(self._anchors_hid[label]) + query = torch.cat(query, 0) + query = torch.tanh(query + self.attn_query(self._h[-1])) + query = self.v_attn(query) + if self.temperature is not None: + query /= self.temperature + if self.tanh_constant is not None: + query = self.tanh_constant * torch.tanh(query) + + if mutable.n_chosen is None: + logit = torch.cat([-query, query], 1) # pylint: disable=invalid-unary-operand-type + + skip = torch.multinomial(F.softmax(logit, dim=-1), 1).view(-1) + skip_prob = torch.sigmoid(logit) + kl = torch.sum(skip_prob * torch.log(skip_prob / self.skip_targets)) + self.sample_skip_penalty += kl + log_prob = self.cross_entropy_loss(logit, skip) + self._inputs = (torch.matmul(skip.float(), torch.cat(anchors, 0)) / (1. + torch.sum(skip))).unsqueeze(0) + else: + assert mutable.n_chosen == 1, "Input choice must select exactly one or any in ENAS." + logit = query.view(1, -1) + index = torch.multinomial(F.softmax(logit, dim=-1), 1).view(-1) + skip = F.one_hot(index, num_classes=mutable.n_candidates).view(-1) + log_prob = self.cross_entropy_loss(logit, index) + self._inputs = anchors[index.item()] + + self.sample_log_prob += self.entropy_reduction(log_prob) + entropy = (log_prob * torch.exp(-log_prob)).detach() # pylint: disable=invalid-unary-operand-type + self.sample_entropy += self.entropy_reduction(entropy) + return skip.bool() diff --git a/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/enas/trainer.py b/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/enas/trainer.py new file mode 100644 index 0000000000000000000000000000000000000000..5e7a966580a107ceb6b9fe88f888dafefb8b69e7 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/enas/trainer.py @@ -0,0 +1,209 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +from itertools import cycle + +import torch +import torch.nn as nn +import torch.optim as optim + +from nni.nas.pytorch.trainer import Trainer +from nni.nas.pytorch.utils import AverageMeterGroup, to_device +from .mutator import EnasMutator + +logger = logging.getLogger(__name__) + + +class EnasTrainer(Trainer): + """ + ENAS trainer. + + Parameters + ---------- + model : nn.Module + PyTorch model to be trained. + loss : callable + Receives logits and ground truth label, return a loss tensor. + metrics : callable + Receives logits and ground truth label, return a dict of metrics. + reward_function : callable + Receives logits and ground truth label, return a tensor, which will be feeded to RL controller as reward. + optimizer : Optimizer + The optimizer used for optimizing the model. + num_epochs : int + Number of epochs planned for training. + dataset_train : Dataset + Dataset for training. Will be split for training weights and architecture weights. + dataset_valid : Dataset + Dataset for testing. + mutator : EnasMutator + Use when customizing your own mutator or a mutator with customized parameters. + batch_size : int + Batch size. + workers : int + Workers for data loading. + device : torch.device + ``torch.device("cpu")`` or ``torch.device("cuda")``. + log_frequency : int + Step count per logging. + callbacks : list of Callback + list of callbacks to trigger at events. + entropy_weight : float + Weight of sample entropy loss. + skip_weight : float + Weight of skip penalty loss. + baseline_decay : float + Decay factor of baseline. New baseline will be equal to ``baseline_decay * baseline_old + reward * (1 - baseline_decay)``. + child_steps : int + How many mini-batches for model training per epoch. + mutator_lr : float + Learning rate for RL controller. + mutator_steps_aggregate : int + Number of steps that will be aggregated into one mini-batch for RL controller. + mutator_steps : int + Number of mini-batches for each epoch of RL controller learning. + aux_weight : float + Weight of auxiliary head loss. ``aux_weight * aux_loss`` will be added to total loss. + test_arc_per_epoch : int + How many architectures are chosen for direct test after each epoch. + """ + def __init__(self, model, loss, metrics, reward_function, + optimizer, num_epochs, dataset_train, dataset_valid, + mutator=None, batch_size=64, workers=4, device=None, log_frequency=None, callbacks=None, + entropy_weight=0.0001, skip_weight=0.8, baseline_decay=0.999, child_steps=500, + mutator_lr=0.00035, mutator_steps_aggregate=20, mutator_steps=50, aux_weight=0.4, + test_arc_per_epoch=1): + super().__init__(model, mutator if mutator is not None else EnasMutator(model), + loss, metrics, optimizer, num_epochs, dataset_train, dataset_valid, + batch_size, workers, device, log_frequency, callbacks) + self.reward_function = reward_function + self.mutator_optim = optim.Adam(self.mutator.parameters(), lr=mutator_lr) + self.batch_size = batch_size + self.workers = workers + + self.entropy_weight = entropy_weight + self.skip_weight = skip_weight + self.baseline_decay = baseline_decay + self.baseline = 0. + self.mutator_steps_aggregate = mutator_steps_aggregate + self.mutator_steps = mutator_steps + self.child_steps = child_steps + self.aux_weight = aux_weight + self.test_arc_per_epoch = test_arc_per_epoch + + self.init_dataloader() + + def init_dataloader(self): + n_train = len(self.dataset_train) + split = n_train // 10 + indices = list(range(n_train)) + train_sampler = torch.utils.data.sampler.SubsetRandomSampler(indices[:-split]) + valid_sampler = torch.utils.data.sampler.SubsetRandomSampler(indices[-split:]) + self.train_loader = torch.utils.data.DataLoader(self.dataset_train, + batch_size=self.batch_size, + sampler=train_sampler, + num_workers=self.workers) + self.valid_loader = torch.utils.data.DataLoader(self.dataset_train, + batch_size=self.batch_size, + sampler=valid_sampler, + num_workers=self.workers) + self.test_loader = torch.utils.data.DataLoader(self.dataset_valid, + batch_size=self.batch_size, + num_workers=self.workers) + self.train_loader = cycle(self.train_loader) + self.valid_loader = cycle(self.valid_loader) + + def train_one_epoch(self, epoch): + # Sample model and train + self.model.train() + self.mutator.eval() + meters = AverageMeterGroup() + for step in range(1, self.child_steps + 1): + x, y = next(self.train_loader) + x, y = to_device(x, self.device), to_device(y, self.device) + self.optimizer.zero_grad() + + with torch.no_grad(): + self.mutator.reset() + self._write_graph_status() + logits = self.model(x) + + if isinstance(logits, tuple): + logits, aux_logits = logits + aux_loss = self.loss(aux_logits, y) + else: + aux_loss = 0. + metrics = self.metrics(logits, y) + loss = self.loss(logits, y) + loss = loss + self.aux_weight * aux_loss + loss.backward() + nn.utils.clip_grad_norm_(self.model.parameters(), 5.) + self.optimizer.step() + metrics["loss"] = loss.item() + meters.update(metrics) + + if self.log_frequency is not None and step % self.log_frequency == 0: + logger.info("Model Epoch [%d/%d] Step [%d/%d] %s", epoch + 1, + self.num_epochs, step, self.child_steps, meters) + + # Train sampler (mutator) + self.model.eval() + self.mutator.train() + meters = AverageMeterGroup() + for mutator_step in range(1, self.mutator_steps + 1): + self.mutator_optim.zero_grad() + for step in range(1, self.mutator_steps_aggregate + 1): + x, y = next(self.valid_loader) + x, y = to_device(x, self.device), to_device(y, self.device) + + self.mutator.reset() + with torch.no_grad(): + logits = self.model(x) + self._write_graph_status() + metrics = self.metrics(logits, y) + reward = self.reward_function(logits, y) + if self.entropy_weight: + reward += self.entropy_weight * self.mutator.sample_entropy.item() + self.baseline = self.baseline * self.baseline_decay + reward * (1 - self.baseline_decay) + loss = self.mutator.sample_log_prob * (reward - self.baseline) + if self.skip_weight: + loss += self.skip_weight * self.mutator.sample_skip_penalty + metrics["reward"] = reward + metrics["loss"] = loss.item() + metrics["ent"] = self.mutator.sample_entropy.item() + metrics["log_prob"] = self.mutator.sample_log_prob.item() + metrics["baseline"] = self.baseline + metrics["skip"] = self.mutator.sample_skip_penalty + + loss /= self.mutator_steps_aggregate + loss.backward() + meters.update(metrics) + + cur_step = step + (mutator_step - 1) * self.mutator_steps_aggregate + if self.log_frequency is not None and cur_step % self.log_frequency == 0: + logger.info("RL Epoch [%d/%d] Step [%d/%d] [%d/%d] %s", epoch + 1, self.num_epochs, + mutator_step, self.mutator_steps, step, self.mutator_steps_aggregate, + meters) + + nn.utils.clip_grad_norm_(self.mutator.parameters(), 5.) + self.mutator_optim.step() + + def validate_one_epoch(self, epoch): + with torch.no_grad(): + for arc_id in range(self.test_arc_per_epoch): + meters = AverageMeterGroup() + for x, y in self.test_loader: + x, y = to_device(x, self.device), to_device(y, self.device) + self.mutator.reset() + logits = self.model(x) + if isinstance(logits, tuple): + logits, _ = logits + metrics = self.metrics(logits, y) + loss = self.loss(logits, y) + metrics["loss"] = loss.item() + meters.update(metrics) + + logger.info("Test Epoch [%d/%d] Arc [%d/%d] Summary %s", + epoch + 1, self.num_epochs, arc_id + 1, self.test_arc_per_epoch, + meters.summary()) diff --git a/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/fbnet/__init__.py b/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/fbnet/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..fa15cc64a77a6b7b4573d5a3bb03b9d6895d1f08 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/fbnet/__init__.py @@ -0,0 +1,11 @@ +from __future__ import absolute_import + +from .mutator import FBNetMutator # noqa: F401 +from .trainer import FBNetTrainer # noqa: F401 +from .utils import ( # noqa: F401 + LookUpTable, + NASConfig, + RegularizerLoss, + model_init, + supernet_sample, +) diff --git a/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/fbnet/mutator.py b/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/fbnet/mutator.py new file mode 100644 index 0000000000000000000000000000000000000000..42b46afcabbcefde53af840e8ddbf822d088ccc9 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/fbnet/mutator.py @@ -0,0 +1,268 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from __future__ import absolute_import, division, print_function + +import torch +from torch import nn as nn +from torch.nn import functional as F +import numpy as np + +from nni.nas.pytorch.base_mutator import BaseMutator +from nni.nas.pytorch.mutables import LayerChoice + + +class MixedOp(nn.Module): + """ + This class is to instantiate and manage info of one LayerChoice. + It includes architecture weights and member functions for the weights. + """ + + def __init__(self, mutable, latency): + """ + Parameters + ---------- + mutable : LayerChoice + A LayerChoice in user model + latency : List + performance cost for each op in mutable + """ + super(MixedOp, self).__init__() + self.latency = latency + n_choices = len(mutable) + self.path_alpha = nn.Parameter( + torch.FloatTensor([1.0 / n_choices for i in range(n_choices)]) + ) + self.path_alpha.requires_grad = False + self.temperature = 1.0 + + def get_path_alpha(self): + """Return the architecture parameter.""" + return self.path_alpha + + def get_weighted_latency(self): + """Return the weighted perf_cost of current mutable.""" + soft_masks = self.probs_over_ops() + weighted_latency = sum(m * l for m, l in zip(soft_masks, self.latency)) + return weighted_latency + + def set_temperature(self, temperature): + """ + Set the annealed temperature for gumbel softmax. + + Parameters + ---------- + temperature : float + The annealed temperature for gumbel softmax + """ + self.temperature = temperature + + def to_requires_grad(self): + """Enable gradient calculation.""" + self.path_alpha.requires_grad = True + + def to_disable_grad(self): + """Disable gradient calculation.""" + self.path_alpha.requires_grad = False + + def probs_over_ops(self): + """Apply gumbel softmax to generate probability distribution.""" + return F.gumbel_softmax(self.path_alpha, self.temperature) + + def forward(self, mutable, x): + """ + Define forward of LayerChoice. + + Parameters + ---------- + mutable : LayerChoice + this layer's mutable + x : tensor + inputs of this layer, only support one input + + Returns + ------- + output: tensor + output of this layer + """ + candidate_ops = list(mutable) + soft_masks = self.probs_over_ops() + output = sum(m * op(x) for m, op in zip(soft_masks, candidate_ops)) + + return output + + @property + def chosen_index(self): + """ + choose the op with max prob + + Returns + ------- + int + index of the chosen one + """ + alphas = self.path_alpha.data.detach().cpu().numpy() + index = int(np.argmax(alphas)) + return index + + +class FBNetMutator(BaseMutator): + """ + This mutator initializes and operates all the LayerChoices of the supernet. + It is for the related trainer to control the training flow of LayerChoices, + coordinating with whole training process. + """ + + def __init__(self, model, lookup_table): + """ + Init a MixedOp instance for each mutable i.e., LayerChoice. + And register the instantiated MixedOp in corresponding LayerChoice. + If does not register it in LayerChoice, DataParallel does'nt work then, + for architecture weights are not included in the DataParallel model. + When MixedOPs are registered, we use ```requires_grad``` to control + whether calculate gradients of architecture weights. + + Parameters + ---------- + model : pytorch model + The model that users want to tune, + it includes search space defined with nni nas apis + lookup_table : class + lookup table object to manage model space information, + including candidate ops for each stage as the model space, + input channels/output channels/stride/fm_size as the layer config, + and the performance information for perf_cost accumulation. + + """ + super(FBNetMutator, self).__init__(model) + self.mutable_list = [] + + # Collect the op names of the candidate ops within each mutable + ops_names_mutable = dict() + left = 0 + right = 1 + for stage_name in lookup_table.layer_num: + right = lookup_table.layer_num[stage_name] + stage_ops = lookup_table.lut_ops[stage_name] + ops_names = [op_name for op_name in stage_ops] + + for i in range(left, left + right): + ops_names_mutable[i] = ops_names + left = right + + # Create the mixed op + for i, mutable in enumerate(self.undedup_mutables): + ops_names = ops_names_mutable[i] + latency_mutable = lookup_table.lut_perf[i] + latency = [latency_mutable[op_name] for op_name in ops_names] + self.mutable_list.append(mutable) + mutable.registered_module = MixedOp(mutable, latency) + + def on_forward_layer_choice(self, mutable, *args, **kwargs): + """ + Callback of layer choice forward. This function defines the forward + logic of the input mutable. So mutable is only interface, its real + implementation is defined in mutator. + + Parameters + ---------- + mutable: LayerChoice + forward logic of this input mutable + args: list of torch.Tensor + inputs of this mutable + kwargs: dict + inputs of this mutable + + Returns + ------- + torch.Tensor + output of this mutable, i.e., LayerChoice + int + index of the chosen op + """ + # FIXME: return mask, to be consistent with other algorithms + idx = mutable.registered_module.chosen_index + return mutable.registered_module(mutable, *args, **kwargs), idx + + def num_arch_params(self): + """ + The number of mutables, i.e., LayerChoice + + Returns + ------- + int + the number of LayerChoice in user model + """ + return len(self.mutable_list) + + def get_architecture_parameters(self): + """ + Get all the architecture parameters. + + yield + ----- + PyTorch Parameter + Return path_alpha of the traversed mutable + """ + for mutable in self.undedup_mutables: + yield mutable.registered_module.get_path_alpha() + + def get_weighted_latency(self): + """ + Get the latency weighted by gumbel softmax coefficients. + + yield + ----- + Tuple + Return the weighted_latency of the traversed mutable + """ + for mutable in self.undedup_mutables: + yield mutable.registered_module.get_weighted_latency() + + def set_temperature(self, temperature): + """ + Set the annealed temperature of the op for gumbel softmax. + + Parameters + ---------- + temperature : float + The annealed temperature for gumbel softmax + """ + for mutable in self.undedup_mutables: + mutable.registered_module.set_temperature(temperature) + + def arch_requires_grad(self): + """ + Make architecture weights require gradient + """ + for mutable in self.undedup_mutables: + mutable.registered_module.to_requires_grad() + + def arch_disable_grad(self): + """ + Disable gradient of architecture weights, i.e., does not + calculate gradient for them. + """ + for mutable in self.undedup_mutables: + mutable.registered_module.to_disable_grad() + + def sample_final(self): + """ + Generate the final chosen architecture. + + Returns + ------- + dict + the choice of each mutable, i.e., LayerChoice + """ + result = dict() + for mutable in self.undedup_mutables: + assert isinstance(mutable, LayerChoice) + index = mutable.registered_module.chosen_index + # pylint: disable=not-callable + result[mutable.key] = ( + F.one_hot(torch.tensor(index), num_classes=len(mutable)) + .view(-1) + .bool(), + ) + return result diff --git a/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/fbnet/trainer.py b/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/fbnet/trainer.py new file mode 100644 index 0000000000000000000000000000000000000000..7588ecf5e54877c22ee329c9463da09d52f0e7bf --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/fbnet/trainer.py @@ -0,0 +1,413 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from __future__ import absolute_import, division, print_function + +import json +import os +import time +import torch + +import numpy as np + +from torch.autograd import Variable +from nni.nas.pytorch.base_trainer import BaseTrainer +from nni.nas.pytorch.trainer import TorchTensorEncoder +from nni.nas.pytorch.utils import AverageMeter +from .mutator import FBNetMutator +from .utils import RegularizerLoss, accuracy + + +class FBNetTrainer(BaseTrainer): + def __init__( + self, + model, + model_optim, + criterion, + device, + device_ids, + lookup_table, + train_loader, + valid_loader, + n_epochs=120, + load_ckpt=False, + arch_path=None, + logger=None, + ): + """ + Parameters + ---------- + model : pytorch model + the user model, which has mutables + model_optim : pytorch optimizer + the user defined optimizer + criterion : pytorch loss + the main task loss, nn.CrossEntropyLoss() is for classification + device : pytorch device + the devices to train/search the model + device_ids : list of int + the indexes of devices used for training + lookup_table : class + lookup table object for fbnet training + train_loader : pytorch data loader + data loader for the training set + valid_loader : pytorch data loader + data loader for the validation set + n_epochs : int + number of epochs to train/search + load_ckpt : bool + whether load checkpoint + arch_path : str + the path to store chosen architecture + logger : logger + the logger + """ + self.model = model + self.model_optim = model_optim + self.train_loader = train_loader + self.valid_loader = valid_loader + self.device = device + self.dev_num = len(device_ids) + self.n_epochs = n_epochs + self.lookup_table = lookup_table + self.config = lookup_table.config + self.start_epoch = self.config.start_epoch + self.temp = self.config.init_temperature + self.exp_anneal_rate = self.config.exp_anneal_rate + self.mode = self.config.mode + + self.load_ckpt = load_ckpt + self.arch_path = arch_path + self.logger = logger + + # scheduler of learning rate + self.scheduler = torch.optim.lr_scheduler.CosineAnnealingLR( + model_optim, T_max=n_epochs, last_epoch=-1 + ) + + # init mutator + self.mutator = FBNetMutator(model, lookup_table) + self.mutator.set_temperature(self.temp) + + # DataParallel should be put behind the init of mutator + self.model = torch.nn.DataParallel(self.model, device_ids=device_ids) + self.model.to(device) + + # build architecture optimizer + self.arch_optimizer = torch.optim.AdamW( + self.mutator.get_architecture_parameters(), + self.config.nas_lr, + weight_decay=self.config.nas_weight_decay, + ) + self.reg_loss = RegularizerLoss(config=self.config) + + self.criterion = criterion + self.epoch = 0 + + def _layer_choice_sample(self): + """ + Sample the index of network within layer choice + """ + stages = [stage_name for stage_name in self.lookup_table.layer_num] + stage_lnum = [self.lookup_table.layer_num[stage] for stage in stages] + + # get the choice idx in each layer + choice_ids = list() + layer_id = 0 + for param in self.mutator.get_architecture_parameters(): + param_np = param.cpu().detach().numpy() + op_idx = np.argmax(param_np) + choice_ids.append(op_idx) + self.logger.info( + "layer {}: {}, index: {}".format(layer_id, param_np, op_idx) + ) + layer_id += 1 + + # get the arch_sample + choice_names = list() + layer_id = 0 + for i, stage_name in enumerate(stages): + ops_names = [op for op in self.lookup_table.lut_ops[stage_name]] + for j in range(stage_lnum[i]): + searched_op = ops_names[choice_ids[layer_id]] + choice_names.append(searched_op) + layer_id += 1 + + self.logger.info(choice_names) + return choice_names + + def _get_perf_cost(self, requires_grad=True): + """ + Get the accumulated performance cost. + """ + perf_cost = Variable( + torch.zeros(1), requires_grad=requires_grad + ).to(self.device, non_blocking=True) + + for latency in self.mutator.get_weighted_latency(): + perf_cost = perf_cost + latency + + return perf_cost + + def _validate(self): + """ + Do validation. During validation, LayerChoices use the mixed-op. + + Returns + ------- + float, float, float + average loss, average top1 accuracy, average top5 accuracy + """ + self.valid_loader.batch_sampler.drop_last = False + batch_time = AverageMeter("batch_time") + losses = AverageMeter("losses") + top1 = AverageMeter("top1") + top5 = AverageMeter("top5") + + # test on validation set under eval mode + self.model.eval() + + end = time.time() + with torch.no_grad(): + for i, (images, labels) in enumerate(self.valid_loader): + images = images.to(self.device, non_blocking=True) + labels = labels.to(self.device, non_blocking=True) + + output = self.model(images) + + loss = self.criterion(output, labels) + acc1, acc5 = accuracy(output, labels, topk=(1, 5)) + losses.update(loss, images.size(0)) + top1.update(acc1[0], images.size(0)) + top5.update(acc5[0], images.size(0)) + # measure elapsed time + batch_time.update(time.time() - end) + end = time.time() + + if i % 10 == 0 or i + 1 == len(self.valid_loader): + test_log = ( + "Valid" + ": [{0}/{1}]\t" + "Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t" + "Loss {loss.val:.4f} ({loss.avg:.4f})\t" + "Top-1 acc {top1.val:.3f} ({top1.avg:.3f})\t" + "Top-5 acc {top5.val:.3f} ({top5.avg:.3f})".format( + i, + len(self.valid_loader) - 1, + batch_time=batch_time, + loss=losses, + top1=top1, + top5=top5, + ) + ) + self.logger.info(test_log) + + return losses.avg, top1.avg, top5.avg + + def _train_epoch(self, epoch, optimizer, arch_train=False): + """ + Train one epoch. + """ + batch_time = AverageMeter("batch_time") + data_time = AverageMeter("data_time") + losses = AverageMeter("losses") + top1 = AverageMeter("top1") + top5 = AverageMeter("top5") + + # switch to train mode + self.model.train() + + data_loader = self.valid_loader if arch_train else self.train_loader + end = time.time() + for i, (images, labels) in enumerate(data_loader): + data_time.update(time.time() - end) + images = images.to(self.device, non_blocking=True) + labels = labels.to(self.device, non_blocking=True) + + output = self.model(images) + loss = self.criterion(output, labels) + + # hardware-aware loss + perf_cost = self._get_perf_cost(requires_grad=True) + regu_loss = self.reg_loss(perf_cost) + if self.mode.startswith("mul"): + loss = loss * regu_loss + elif self.mode.startswith("add"): + loss = loss + regu_loss + + # measure accuracy and record loss + acc1, acc5 = accuracy(output, labels, topk=(1, 5)) + losses.update(loss.item(), images.size(0)) + top1.update(acc1[0].item(), images.size(0)) + top5.update(acc5[0].item(), images.size(0)) + # compute gradient and do SGD step + optimizer.zero_grad() + loss.backward() + optimizer.step() + # measure elapsed time + batch_time.update(time.time() - end) + end = time.time() + + if i % 10 == 0: + batch_log = ( + "Warmup Train [{0}][{1}]\t" + "Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t" + "Data {data_time.val:.3f} ({data_time.avg:.3f})\t" + "Loss {losses.val:.4f} ({losses.avg:.4f})\t" + "Top-1 acc {top1.val:.3f} ({top1.avg:.3f})\t" + "Top-5 acc {top5.val:.3f} ({top5.avg:.3f})\t".format( + epoch + 1, + i, + batch_time=batch_time, + data_time=data_time, + losses=losses, + top1=top1, + top5=top5, + ) + ) + self.logger.info(batch_log) + + def _warm_up(self): + """ + Warm up the model, while the architecture weights are not trained. + """ + for epoch in range(self.epoch, self.start_epoch): + self.logger.info("\n--------Warmup epoch: %d--------\n", epoch + 1) + self._train_epoch(epoch, self.model_optim) + # adjust learning rate + self.scheduler.step() + + # validation + val_loss, val_top1, val_top5 = self._validate() + val_log = ( + "Warmup Valid [{0}/{1}]\t" + "loss {2:.3f}\ttop-1 acc {3:.3f}\ttop-5 acc {4:.3f}".format( + epoch + 1, self.warmup_epochs, val_loss, val_top1, val_top5 + ) + ) + self.logger.info(val_log) + + if epoch % 10 == 0: + filename = os.path.join( + self.config.model_dir, "checkpoint_%s.pth" % epoch + ) + self.save_checkpoint(epoch, filename) + + def _train(self): + """ + Train the model, it trains model weights and architecute weights. + Architecture weights are trained according to the schedule. + Before updating architecture weights, ```requires_grad``` is enabled. + Then, it is disabled after the updating, in order not to update + architecture weights when training model weights. + """ + arch_param_num = self.mutator.num_arch_params() + self.logger.info("#arch_params: {}".format(arch_param_num)) + self.epoch = max(self.start_epoch, self.epoch) + + ckpt_path = self.config.model_dir + choice_names = None + top1_best = 0.0 + + for epoch in range(self.epoch, self.n_epochs): + self.logger.info("\n--------Train epoch: %d--------\n", epoch + 1) + # update the weight parameters + self._train_epoch(epoch, self.model_optim) + # adjust learning rate + self.scheduler.step() + + self.logger.info("Update architecture parameters") + # update the architecture parameters + self.mutator.arch_requires_grad() + self._train_epoch(epoch, self.arch_optimizer, True) + self.mutator.arch_disable_grad() + # temperature annealing + self.temp = self.temp * self.exp_anneal_rate + self.mutator.set_temperature(self.temp) + # sample the architecture of sub-network + choice_names = self._layer_choice_sample() + + # validate + val_loss, val_top1, val_top5 = self._validate() + val_log = ( + "Valid [{0}]\t" + "loss {1:.3f}\ttop-1 acc {2:.3f} \ttop-5 acc {3:.3f}".format( + epoch + 1, val_loss, val_top1, val_top5 + ) + ) + self.logger.info(val_log) + + if epoch % 10 == 0: + filename = os.path.join(ckpt_path, "checkpoint_%s.pth" % epoch) + self.save_checkpoint(epoch, filename, choice_names) + + val_top1 = val_top1.cpu().as_numpy() + if val_top1 > top1_best: + filename = os.path.join(ckpt_path, "checkpoint_best.pth") + self.save_checkpoint(epoch, filename, choice_names) + top1_best = val_top1 + + def save_checkpoint(self, epoch, filename, choice_names=None): + """ + Save checkpoint of the whole model. + Saving model weights and architecture weights as ```filename```, + and saving currently chosen architecture in ```arch_path```. + """ + state = { + "model": self.model.state_dict(), + "optim": self.model_optim.state_dict(), + "epoch": epoch, + "arch_sample": choice_names, + } + torch.save(state, filename) + self.logger.info("Save checkpoint to {0:}".format(filename)) + + if self.arch_path: + self.export(self.arch_path) + + def load_checkpoint(self, filename): + """ + Load the checkpoint from ```ckpt_path```. + """ + ckpt = torch.load(filename) + self.epoch = ckpt["epoch"] + self.model.load_state_dict(ckpt["model"]) + self.model_optim.load_state_dict(ckpt["optim"]) + + def train(self): + """ + Train the whole model. + """ + if self.load_ckpt: + ckpt_path = self.config.model_dir + filename = os.path.join(ckpt_path, "checkpoint_best.pth") + if os.path.exists(filename): + self.load_checkpoint(filename) + + if self.epoch < self.start_epoch: + self._warm_up() + self._train() + + def export(self, file_name): + """ + Export the chosen architecture into a file + + Parameters + ---------- + file_name : str + the file that stores exported chosen architecture + """ + exported_arch = self.mutator.sample_final() + with open(file_name, "w") as f: + json.dump( + exported_arch, + f, + indent=2, + sort_keys=True, + cls=TorchTensorEncoder, + ) + + def validate(self): + raise NotImplementedError + + def checkpoint(self): + raise NotImplementedError diff --git a/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/fbnet/utils.py b/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/fbnet/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..332f2e2c620e14f27f5402858d2c8c631816dcd2 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/fbnet/utils.py @@ -0,0 +1,352 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from __future__ import absolute_import, division, print_function + +import gc # noqa: F401 +import os +import timeit +import torch + +import numpy as np +import torch.nn as nn + +from nni.compression.pytorch.utils.counter import count_flops_params + +LUT_FILE = "lut.npy" +LUT_PATH = "lut" + + +class NASConfig: + def __init__( + self, + perf_metric="flops", + lut_load=False, + model_dir=None, + nas_lr=0.01, + nas_weight_decay=5e-4, + mode="mul", + alpha=0.25, + beta=0.6, + start_epoch=50, + init_temperature=5.0, + exp_anneal_rate=np.exp(-0.045), + search_space=None, + ): + # LUT of performance metric + # flops means the multiplies, latency means the time cost on platform + self.perf_metric = perf_metric + assert perf_metric in [ + "flops", + "latency", + ], "perf_metric should be ['flops', 'latency']" + # wether load or create lut file + self.lut_load = lut_load + # necessary dirs + self.lut_en = model_dir is not None + if self.lut_en: + self.model_dir = model_dir + os.makedirs(model_dir, exist_ok=True) + self.lut_path = os.path.join(model_dir, LUT_PATH) + os.makedirs(self.lut_path, exist_ok=True) + # NAS learning setting + self.nas_lr = nas_lr + self.nas_weight_decay = nas_weight_decay + # hardware-aware loss setting + self.mode = mode + assert mode in ["mul", "add"], "mode should be ['mul', 'add']" + self.alpha = alpha + self.beta = beta + # NAS training setting + self.start_epoch = start_epoch + self.init_temperature = init_temperature + self.exp_anneal_rate = exp_anneal_rate + # definition of search blocks and space + self.search_space = search_space + + +class RegularizerLoss(nn.Module): + """Auxilliary loss for hardware-aware NAS.""" + + def __init__(self, config): + """ + Parameters + ---------- + config : class + to manage the configuration for NAS training, and search space etc. + """ + super(RegularizerLoss, self).__init__() + self.mode = config.mode + self.alpha = config.alpha + self.beta = config.beta + + def forward(self, perf_cost, batch_size=1): + """ + Parameters + ---------- + perf_cost : tensor + the accumulated performance cost + batch_size : int + batch size for normalization + + Returns + ------- + output: tensor + the hardware-aware constraint loss + """ + if self.mode == "mul": + log_loss = torch.log(perf_cost / batch_size) ** self.beta + return self.alpha * log_loss + elif self.mode == "add": + linear_loss = (perf_cost / batch_size) ** self.beta + return self.alpha * linear_loss + else: + raise NotImplementedError + + +def accuracy(output, target, topk=(1,)): + """ + Computes the precision@k for the specified values of k + + Parameters + ---------- + output : pytorch tensor + output, e.g., predicted value + target : pytorch tensor + label + topk : tuple + specify top1 and top5 + + Returns + ------- + list + accuracy of top1 and top5 + """ + maxk = max(topk) + batch_size = target.size(0) + + _, pred = output.topk(maxk, 1, True, True) + pred = pred.t() + correct = pred.eq(target.view(1, -1).expand_as(pred)) + + res = [] + for k in topk: + correct_k = correct[:k].view(-1).float().sum(0, keepdim=True) + res.append(correct_k.mul_(100.0 / batch_size)) + return res + + +def supernet_sample(model, state_dict, sampled_arch=[], lookup_table=None): + """ + Initialize the searched sub-model from supernet. + + Parameters + ---------- + model : pytorch model + the created subnet + state_dict : checkpoint + the checkpoint of supernet, including the pre-trained params + sampled_arch : list of str + the searched layer names of the subnet + lookup_table : class + to manage the candidate ops, layer information and layer performance + """ + replace = list() + stages = [stage for stage in lookup_table.layer_num] + stage_lnum = [lookup_table.layer_num[stage] for stage in stages] + + if sampled_arch: + layer_id = 0 + for i, stage in enumerate(stages): + ops_names = [op_name for op_name in lookup_table.lut_ops[stage]] + for j in range(stage_lnum[i]): + searched_op = sampled_arch[layer_id] + op_i = ops_names.index(searched_op) + replace.append( + [ + "blocks.{}.".format(layer_id), + "blocks.{}.op.".format(layer_id), + "blocks.{}.{}.".format(layer_id, op_i), + ] + ) + layer_id += 1 + model_init(model, state_dict, replace=replace) + + +def model_init(model, state_dict, replace=[]): + """Initialize the model from state_dict.""" + prefix = "module." + param_dict = dict() + for k, v in state_dict.items(): + if k.startswith(prefix): + k = k[7:] + param_dict[k] = v + + for k, (name, m) in enumerate(model.named_modules()): + if replace: + for layer_replace in replace: + assert len(layer_replace) == 3, "The elements should be three." + pre_scope, key, replace_key = layer_replace + if pre_scope in name: + name = name.replace(key, replace_key) + + # Copy the state_dict to current model + if (name + ".weight" in param_dict) or ( + name + ".running_mean" in param_dict + ): + if isinstance(m, nn.BatchNorm2d): + shape = m.running_mean.shape + if shape == param_dict[name + ".running_mean"].shape: + if m.weight is not None: + m.weight.data = param_dict[name + ".weight"] + m.bias.data = param_dict[name + ".bias"] + m.running_mean = param_dict[name + ".running_mean"] + m.running_var = param_dict[name + ".running_var"] + + elif isinstance(m, nn.Conv2d) or isinstance(m, nn.Linear): + shape = m.weight.data.shape + if shape == param_dict[name + ".weight"].shape: + m.weight.data = param_dict[name + ".weight"] + if m.bias is not None: + m.bias.data = param_dict[name + ".bias"] + + elif isinstance(m, nn.ConvTranspose2d): + m.weight.data = param_dict[name + ".weight"] + if m.bias is not None: + m.bias.data = param_dict[name + ".bias"] + + +class LookUpTable: + """Build look-up table for NAS.""" + + def __init__(self, config, primitives): + """ + Parameters + ---------- + config : class + to manage the configuration for NAS training, and search space etc. + """ + self.config = config + # definition of search blocks and space + self.search_space = config.search_space + # layers for NAS + self.cnt_layers = len(self.search_space["input_shape"]) + # constructors for each operation + self.lut_ops = { + stage_name: { + op_name: primitives[op_name] + for op_name in self.search_space["stages"][stage_name]["ops"] + } + for stage_name in self.search_space["stages"] + } + self.layer_num = { + stage_name: self.search_space["stages"][stage_name]["layer_num"] + for stage_name in self.search_space["stages"] + } + + # arguments for the ops constructors, input_shapes just for convinience + self.layer_configs, self.layer_in_shapes = self._layer_configs() + + # lookup_table + self.perf_metric = config.perf_metric + + if config.lut_en: + self.lut_perf = None + self.lut_file = os.path.join(config.lut_path, LUT_FILE) + if config.lut_load: + self._load_from_file() + else: + self._create_perfs() + + def _layer_configs(self): + """Generate basic params for different layers.""" + # layer_configs are : c_in, c_out, stride, fm_size + layer_configs = [ + [ + self.search_space["input_shape"][layer_id][0], + self.search_space["channel_size"][layer_id], + self.search_space["strides"][layer_id], + self.search_space["fm_size"][layer_id], + ] + for layer_id in range(self.cnt_layers) + ] + + # layer_in_shapes are (C_in, input_w, input_h) + layer_in_shapes = self.search_space["input_shape"] + + return layer_configs, layer_in_shapes + + def _create_perfs(self, cnt_of_runs=200): + """Create performance cost for each op.""" + if self.perf_metric == "latency": + self.lut_perf = self._calculate_latency(cnt_of_runs) + elif self.perf_metric == "flops": + self.lut_perf = self._calculate_flops() + + self._write_lut_to_file() + + def _calculate_flops(self, eps=0.001): + """FLOPs cost.""" + flops_lut = [{} for i in range(self.cnt_layers)] + layer_id = 0 + + for stage_name in self.lut_ops: + stage_ops = self.lut_ops[stage_name] + ops_num = self.layer_num[stage_name] + + for _ in range(ops_num): + for op_name in stage_ops: + layer_config = self.layer_configs[layer_id] + key_params = {"fm_size": layer_config[3]} + op = stage_ops[op_name](*layer_config[0:3], **key_params) + + # measured in Flops + in_shape = self.layer_in_shapes[layer_id] + x = (1, in_shape[0], in_shape[1], in_shape[2]) + flops, _, _ = count_flops_params(op, x, verbose=False) + flops = eps if flops == 0.0 else flops + flops_lut[layer_id][op_name] = float(flops) + layer_id += 1 + + return flops_lut + + def _calculate_latency(self, cnt_of_runs): + """Latency cost.""" + LATENCY_BATCH_SIZE = 1 + latency_lut = [{} for i in range(self.cnt_layers)] + layer_id = 0 + + for stage_name in self.lut_ops: + stage_ops = self.lut_ops[stage_name] + ops_num = self.layer_num[stage_name] + + for _ in range(ops_num): + for op_name in stage_ops: + layer_config = self.layer_configs[layer_id] + key_params = {"fm_size": layer_config[3]} + op = stage_ops[op_name](*layer_config[0:3], **key_params) + input_data = torch.randn( + (LATENCY_BATCH_SIZE, *self.layer_in_shapes[layer_id]) + ) + globals()["op"], globals()["input_data"] = op, input_data + total_time = timeit.timeit( + "output = op(input_data)", + setup="gc.enable()", + globals=globals(), + number=cnt_of_runs, + ) + # measured in micro-second + latency_lut[layer_id][op_name] = ( + total_time / cnt_of_runs / LATENCY_BATCH_SIZE * 1e6 + ) + layer_id += 1 + + return latency_lut + + def _write_lut_to_file(self): + """Save lut as numpy file.""" + np.save(self.lut_file, self.lut_perf) + + def _load_from_file(self): + """Load numpy file.""" + self.lut_perf = np.load(self.lut_file, allow_pickle=True) diff --git a/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/pdarts/__init__.py b/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/pdarts/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d1d17764ba159b35bcc38efa82a2a30dc2366b76 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/pdarts/__init__.py @@ -0,0 +1,4 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .trainer import PdartsTrainer diff --git a/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/pdarts/mutator.py b/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/pdarts/mutator.py new file mode 100644 index 0000000000000000000000000000000000000000..09ad51c5e471b4acae51d513335682328924b90a --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/pdarts/mutator.py @@ -0,0 +1,93 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import copy + +import numpy as np +import torch +from torch import nn + +from nni.algorithms.nas.pytorch.darts import DartsMutator +from nni.nas.pytorch.mutables import LayerChoice + + +class PdartsMutator(DartsMutator): + """ + It works with PdartsTrainer to calculate ops weights, + and drop weights in different PDARTS epochs. + """ + + def __init__(self, model, pdarts_epoch_index, pdarts_num_to_drop, switches={}): + self.pdarts_epoch_index = pdarts_epoch_index + self.pdarts_num_to_drop = pdarts_num_to_drop + if switches is None: + self.switches = {} + else: + self.switches = switches + + super(PdartsMutator, self).__init__(model) + + # this loop go through mutables with different keys, + # it's mainly to update length of choices. + for mutable in self.mutables: + if isinstance(mutable, LayerChoice): + + switches = self.switches.get(mutable.key, [True for j in range(len(mutable))]) + choices = self.choices[mutable.key] + + operations_count = np.sum(switches) + # +1 and -1 are caused by zero operation in darts network + # the zero operation is not in choices list in network, but its weight are in, + # so it needs one more weights and switch for zero. + self.choices[mutable.key] = nn.Parameter(1.0E-3 * torch.randn(operations_count + 1)) + self.switches[mutable.key] = switches + + # update LayerChoice instances in model, + # it's physically remove dropped choices operations. + for module in self.model.modules(): + if isinstance(module, LayerChoice): + switches = self.switches.get(module.key) + choices = self.choices[module.key] + if len(module) > len(choices): + # from last to first, so that it won't effect previous indexes after removed one. + for index in range(len(switches)-1, -1, -1): + if switches[index] == False: + del module[index] + assert len(module) <= len(choices), "Failed to remove dropped choices." + + def export(self): + # Cannot rely on super().export() because P-DARTS has deleted some of the choices and has misaligned length. + results = super().sample_final() + for mutable in self.mutables: + if isinstance(mutable, LayerChoice): + # As some operations are dropped physically, + # so it needs to fill back false to track dropped operations. + trained_result = results[mutable.key] + trained_index = 0 + switches = self.switches[mutable.key] + result = torch.Tensor(switches).bool() + for index in range(len(result)): + if result[index]: + result[index] = trained_result[trained_index] + trained_index += 1 + results[mutable.key] = result + return results + + def drop_paths(self): + """ + This method is called when a PDARTS epoch is finished. + It prepares switches for next epoch. + candidate operations with False switch will be doppped in next epoch. + """ + all_switches = copy.deepcopy(self.switches) + for key in all_switches: + switches = all_switches[key] + idxs = [] + for j in range(len(switches)): + if switches[j]: + idxs.append(j) + sorted_weights = self.choices[key].data.cpu().numpy()[:-1] + drop = np.argsort(sorted_weights)[:self.pdarts_num_to_drop[self.pdarts_epoch_index]] + for idx in drop: + switches[idxs[idx]] = False + return all_switches diff --git a/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/pdarts/trainer.py b/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/pdarts/trainer.py new file mode 100644 index 0000000000000000000000000000000000000000..7f23a6e222731ae6edc5c034f77eafaaf71b4c5e --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/pdarts/trainer.py @@ -0,0 +1,86 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import json +import logging + +from nni.nas.pytorch.callbacks import LRSchedulerCallback +from nni.algorithms.nas.pytorch.darts import DartsTrainer +from nni.nas.pytorch.trainer import BaseTrainer, TorchTensorEncoder + +from .mutator import PdartsMutator + +logger = logging.getLogger(__name__) + + +class PdartsTrainer(BaseTrainer): + """ + This trainer implements the PDARTS algorithm. + PDARTS bases on DARTS algorithm, and provides a network growth approach to find deeper and better network. + This class relies on pdarts_num_layers and pdarts_num_to_drop parameters to control how network grows. + pdarts_num_layers means how many layers more than first epoch. + pdarts_num_to_drop means how many candidate operations should be dropped in each epoch. + So that the grew network can in similar size. + """ + + def __init__(self, model_creator, init_layers, metrics, + num_epochs, dataset_train, dataset_valid, + pdarts_num_layers=[0, 6, 12], pdarts_num_to_drop=[3, 2, 1], + mutator=None, batch_size=64, workers=4, device=None, log_frequency=None, callbacks=None, unrolled=False): + super(PdartsTrainer, self).__init__() + self.model_creator = model_creator + self.init_layers = init_layers + self.pdarts_num_layers = pdarts_num_layers + self.pdarts_num_to_drop = pdarts_num_to_drop + self.pdarts_epoch = len(pdarts_num_to_drop) + self.darts_parameters = { + "metrics": metrics, + "num_epochs": num_epochs, + "dataset_train": dataset_train, + "dataset_valid": dataset_valid, + "batch_size": batch_size, + "workers": workers, + "device": device, + "log_frequency": log_frequency, + "unrolled": unrolled + } + self.callbacks = callbacks if callbacks is not None else [] + + def train(self): + + switches = None + for epoch in range(self.pdarts_epoch): + + layers = self.init_layers+self.pdarts_num_layers[epoch] + model, criterion, optim, lr_scheduler = self.model_creator(layers) + self.mutator = PdartsMutator(model, epoch, self.pdarts_num_to_drop, switches) + + for callback in self.callbacks: + callback.build(model, self.mutator, self) + callback.on_epoch_begin(epoch) + + darts_callbacks = [] + if lr_scheduler is not None: + darts_callbacks.append(LRSchedulerCallback(lr_scheduler)) + + self.trainer = DartsTrainer(model, mutator=self.mutator, loss=criterion, optimizer=optim, + callbacks=darts_callbacks, **self.darts_parameters) + logger.info("start pdarts training epoch %s...", epoch) + + self.trainer.train() + + switches = self.mutator.drop_paths() + + for callback in self.callbacks: + callback.on_epoch_end(epoch) + + def validate(self): + self.trainer.validate() + + def export(self, file): + mutator_export = self.mutator.export() + with open(file, "w") as f: + json.dump(mutator_export, f, indent=2, sort_keys=True, cls=TorchTensorEncoder) + + def checkpoint(self): + raise NotImplementedError("Not implemented yet") diff --git a/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/proxylessnas/__init__.py b/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/proxylessnas/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..26feedba7d553c32d61ea9139620a68fca7c12d0 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/proxylessnas/__init__.py @@ -0,0 +1,2 @@ +from .mutator import ProxylessNasMutator +from .trainer import ProxylessNasTrainer diff --git a/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/proxylessnas/mutator.py b/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/proxylessnas/mutator.py new file mode 100644 index 0000000000000000000000000000000000000000..881a6b44038d5d6f2c97b2b106036954401a9cd3 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/proxylessnas/mutator.py @@ -0,0 +1,478 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import math +import torch +from torch import nn as nn +from torch.nn import functional as F +import numpy as np + +from nni.nas.pytorch.base_mutator import BaseMutator +from nni.nas.pytorch.mutables import LayerChoice +from .utils import detach_variable + +class ArchGradientFunction(torch.autograd.Function): + @staticmethod + def forward(ctx, x, binary_gates, run_func, backward_func): + ctx.run_func = run_func + ctx.backward_func = backward_func + + detached_x = detach_variable(x) + with torch.enable_grad(): + output = run_func(detached_x) + ctx.save_for_backward(detached_x, output) + return output.data + + @staticmethod + def backward(ctx, grad_output): + detached_x, output = ctx.saved_tensors + + grad_x = torch.autograd.grad(output, detached_x, grad_output, only_inputs=True) + # compute gradients w.r.t. binary_gates + binary_grads = ctx.backward_func(detached_x.data, output.data, grad_output.data) + + return grad_x[0], binary_grads, None, None + +class MixedOp(nn.Module): + """ + This class is to instantiate and manage info of one LayerChoice. + It includes architecture weights, binary weights, and member functions + operating the weights. + + forward_mode: + forward/backward mode for LayerChoice: None, two, full, and full_v2. + For training architecture weights, we use full_v2 by default, and for training + model weights, we use None. + """ + forward_mode = None + def __init__(self, mutable): + """ + Parameters + ---------- + mutable : LayerChoice + A LayerChoice in user model + """ + super(MixedOp, self).__init__() + self.ap_path_alpha = nn.Parameter(torch.Tensor(len(mutable))) + self.ap_path_wb = nn.Parameter(torch.Tensor(len(mutable))) + self.ap_path_alpha.requires_grad = False + self.ap_path_wb.requires_grad = False + self.active_index = [0] + self.inactive_index = None + self.log_prob = None + self.current_prob_over_ops = None + self.n_choices = len(mutable) + + def get_ap_path_alpha(self): + return self.ap_path_alpha + + def to_requires_grad(self): + self.ap_path_alpha.requires_grad = True + self.ap_path_wb.requires_grad = True + + def to_disable_grad(self): + self.ap_path_alpha.requires_grad = False + self.ap_path_wb.requires_grad = False + + def forward(self, mutable, x): + """ + Define forward of LayerChoice. For 'full_v2', backward is also defined. + The 'two' mode is explained in section 3.2.1 in the paper. + The 'full_v2' mode is explained in Appendix D in the paper. + + Parameters + ---------- + mutable : LayerChoice + this layer's mutable + x : tensor + inputs of this layer, only support one input + + Returns + ------- + output: tensor + output of this layer + """ + if MixedOp.forward_mode == 'full' or MixedOp.forward_mode == 'two': + output = 0 + for _i in self.active_index: + oi = self.candidate_ops[_i](x) + output = output + self.ap_path_wb[_i] * oi + for _i in self.inactive_index: + oi = self.candidate_ops[_i](x) + output = output + self.ap_path_wb[_i] * oi.detach() + elif MixedOp.forward_mode == 'full_v2': + def run_function(key, candidate_ops, active_id): + def forward(_x): + return candidate_ops[active_id](_x) + return forward + + def backward_function(key, candidate_ops, active_id, binary_gates): + def backward(_x, _output, grad_output): + binary_grads = torch.zeros_like(binary_gates.data) + with torch.no_grad(): + for k in range(len(candidate_ops)): + if k != active_id: + out_k = candidate_ops[k](_x.data) + else: + out_k = _output.data + grad_k = torch.sum(out_k * grad_output) + binary_grads[k] = grad_k + return binary_grads + return backward + output = ArchGradientFunction.apply( + x, self.ap_path_wb, run_function(mutable.key, list(mutable), self.active_index[0]), + backward_function(mutable.key, list(mutable), self.active_index[0], self.ap_path_wb)) + else: + output = self.active_op(mutable)(x) + return output + + @property + def probs_over_ops(self): + """ + Apply softmax on alpha to generate probability distribution + + Returns + ------- + pytorch tensor + probability distribution + """ + probs = F.softmax(self.ap_path_alpha, dim=0) # softmax to probability + return probs + + @property + def chosen_index(self): + """ + choose the op with max prob + + Returns + ------- + int + index of the chosen one + numpy.float32 + prob of the chosen one + """ + probs = self.probs_over_ops.data.cpu().numpy() + index = int(np.argmax(probs)) + return index, probs[index] + + def active_op(self, mutable): + """ + assume only one path is active + + Returns + ------- + PyTorch module + the chosen operation + """ + return mutable[self.active_index[0]] + + @property + def active_op_index(self): + """ + return active op's index, the active op is sampled + + Returns + ------- + int + index of the active op + """ + return self.active_index[0] + + def set_chosen_op_active(self): + """ + set chosen index, active and inactive indexes + """ + chosen_idx, _ = self.chosen_index + self.active_index = [chosen_idx] + self.inactive_index = [_i for _i in range(0, chosen_idx)] + \ + [_i for _i in range(chosen_idx + 1, self.n_choices)] + + def binarize(self, mutable): + """ + Sample based on alpha, and set binary weights accordingly. + ap_path_wb is set in this function, which is called binarize. + + Parameters + ---------- + mutable : LayerChoice + this layer's mutable + """ + self.log_prob = None + # reset binary gates + self.ap_path_wb.data.zero_() + probs = self.probs_over_ops + if MixedOp.forward_mode == 'two': + # sample two ops according to probs + sample_op = torch.multinomial(probs.data, 2, replacement=False) + probs_slice = F.softmax(torch.stack([ + self.ap_path_alpha[idx] for idx in sample_op + ]), dim=0) + self.current_prob_over_ops = torch.zeros_like(probs) + for i, idx in enumerate(sample_op): + self.current_prob_over_ops[idx] = probs_slice[i] + # choose one to be active and the other to be inactive according to probs_slice + c = torch.multinomial(probs_slice.data, 1)[0] # 0 or 1 + active_op = sample_op[c].item() + inactive_op = sample_op[1-c].item() + self.active_index = [active_op] + self.inactive_index = [inactive_op] + # set binary gate + self.ap_path_wb.data[active_op] = 1.0 + else: + sample = torch.multinomial(probs, 1)[0].item() + self.active_index = [sample] + self.inactive_index = [_i for _i in range(0, sample)] + \ + [_i for _i in range(sample + 1, len(mutable))] + self.log_prob = torch.log(probs[sample]) + self.current_prob_over_ops = probs + self.ap_path_wb.data[sample] = 1.0 + # avoid over-regularization + for choice in mutable: + for _, param in choice.named_parameters(): + param.grad = None + + @staticmethod + def delta_ij(i, j): + if i == j: + return 1 + else: + return 0 + + def set_arch_param_grad(self, mutable): + """ + Calculate alpha gradient for this LayerChoice. + It is calculated using gradient of binary gate, probs of ops. + """ + binary_grads = self.ap_path_wb.grad.data + if self.active_op(mutable).is_zero_layer(): + self.ap_path_alpha.grad = None + return + if self.ap_path_alpha.grad is None: + self.ap_path_alpha.grad = torch.zeros_like(self.ap_path_alpha.data) + if MixedOp.forward_mode == 'two': + involved_idx = self.active_index + self.inactive_index + probs_slice = F.softmax(torch.stack([ + self.ap_path_alpha[idx] for idx in involved_idx + ]), dim=0).data + for i in range(2): + for j in range(2): + origin_i = involved_idx[i] + origin_j = involved_idx[j] + self.ap_path_alpha.grad.data[origin_i] += \ + binary_grads[origin_j] * probs_slice[j] * (MixedOp.delta_ij(i, j) - probs_slice[i]) + for _i, idx in enumerate(self.active_index): + self.active_index[_i] = (idx, self.ap_path_alpha.data[idx].item()) + for _i, idx in enumerate(self.inactive_index): + self.inactive_index[_i] = (idx, self.ap_path_alpha.data[idx].item()) + else: + probs = self.probs_over_ops.data + for i in range(self.n_choices): + for j in range(self.n_choices): + self.ap_path_alpha.grad.data[i] += binary_grads[j] * probs[j] * (MixedOp.delta_ij(i, j) - probs[i]) + return + + def rescale_updated_arch_param(self): + """ + rescale architecture weights for the 'two' mode. + """ + if not isinstance(self.active_index[0], tuple): + assert self.active_op.is_zero_layer() + return + involved_idx = [idx for idx, _ in (self.active_index + self.inactive_index)] + old_alphas = [alpha for _, alpha in (self.active_index + self.inactive_index)] + new_alphas = [self.ap_path_alpha.data[idx] for idx in involved_idx] + + offset = math.log( + sum([math.exp(alpha) for alpha in new_alphas]) / sum([math.exp(alpha) for alpha in old_alphas]) + ) + + for idx in involved_idx: + self.ap_path_alpha.data[idx] -= offset + + +class ProxylessNasMutator(BaseMutator): + """ + This mutator initializes and operates all the LayerChoices of the input model. + It is for the corresponding trainer to control the training process of LayerChoices, + coordinating with whole training process. + """ + def __init__(self, model): + """ + Init a MixedOp instance for each mutable i.e., LayerChoice. + And register the instantiated MixedOp in corresponding LayerChoice. + If does not register it in LayerChoice, DataParallel does not work then, + because architecture weights are not included in the DataParallel model. + When MixedOPs are registered, we use ```requires_grad``` to control + whether calculate gradients of architecture weights. + + Parameters + ---------- + model : pytorch model + The model that users want to tune, it includes search space defined with nni nas apis + """ + super(ProxylessNasMutator, self).__init__(model) + self._unused_modules = None + self.mutable_list = [] + for mutable in self.undedup_mutables: + self.mutable_list.append(mutable) + mutable.registered_module = MixedOp(mutable) + + def on_forward_layer_choice(self, mutable, *args, **kwargs): + """ + Callback of layer choice forward. This function defines the forward + logic of the input mutable. So mutable is only interface, its real + implementation is defined in mutator. + + Parameters + ---------- + mutable: LayerChoice + forward logic of this input mutable + args: list of torch.Tensor + inputs of this mutable + kwargs: dict + inputs of this mutable + + Returns + ------- + torch.Tensor + output of this mutable, i.e., LayerChoice + int + index of the chosen op + """ + # FIXME: return mask, to be consistent with other algorithms + idx = mutable.registered_module.active_op_index + return mutable.registered_module(mutable, *args, **kwargs), idx + + def reset_binary_gates(self): + """ + For each LayerChoice, binarize binary weights + based on alpha to only activate one op. + It traverses all the mutables in the model to do this. + """ + for mutable in self.undedup_mutables: + mutable.registered_module.binarize(mutable) + + def set_chosen_op_active(self): + """ + For each LayerChoice, set the op with highest alpha as the chosen op. + Usually used for validation. + """ + for mutable in self.undedup_mutables: + mutable.registered_module.set_chosen_op_active() + + def num_arch_params(self): + """ + The number of mutables, i.e., LayerChoice + + Returns + ------- + int + the number of LayerChoice in user model + """ + return len(self.mutable_list) + + def set_arch_param_grad(self): + """ + For each LayerChoice, calculate gradients for architecture weights, i.e., alpha + """ + for mutable in self.undedup_mutables: + mutable.registered_module.set_arch_param_grad(mutable) + + def get_architecture_parameters(self): + """ + Get all the architecture parameters. + + yield + ----- + PyTorch Parameter + Return ap_path_alpha of the traversed mutable + """ + for mutable in self.undedup_mutables: + yield mutable.registered_module.get_ap_path_alpha() + + def change_forward_mode(self, mode): + """ + Update forward mode of MixedOps, as training architecture weights and + model weights use different forward modes. + """ + MixedOp.forward_mode = mode + + def get_forward_mode(self): + """ + Get forward mode of MixedOp + + Returns + ------- + string + the current forward mode of MixedOp + """ + return MixedOp.forward_mode + + def rescale_updated_arch_param(self): + """ + Rescale architecture weights in 'two' mode. + """ + for mutable in self.undedup_mutables: + mutable.registered_module.rescale_updated_arch_param() + + def unused_modules_off(self): + """ + Remove unused modules for each mutables. + The removed modules are kept in ```self._unused_modules``` for resume later. + """ + self._unused_modules = [] + for mutable in self.undedup_mutables: + mixed_op = mutable.registered_module + unused = {} + if self.get_forward_mode() in ['full', 'two', 'full_v2']: + involved_index = mixed_op.active_index + mixed_op.inactive_index + else: + involved_index = mixed_op.active_index + for i in range(mixed_op.n_choices): + if i not in involved_index: + unused[i] = mutable[i] + mutable[i] = None + self._unused_modules.append(unused) + + def unused_modules_back(self): + """ + Resume the removed modules back. + """ + if self._unused_modules is None: + return + for m, unused in zip(self.mutable_list, self._unused_modules): + for i in unused: + m[i] = unused[i] + self._unused_modules = None + + def arch_requires_grad(self): + """ + Make architecture weights require gradient + """ + for mutable in self.undedup_mutables: + mutable.registered_module.to_requires_grad() + + def arch_disable_grad(self): + """ + Disable gradient of architecture weights, i.e., does not + calcuate gradient for them. + """ + for mutable in self.undedup_mutables: + mutable.registered_module.to_disable_grad() + + def sample_final(self): + """ + Generate the final chosen architecture. + + Returns + ------- + dict + the choice of each mutable, i.e., LayerChoice + """ + result = dict() + for mutable in self.undedup_mutables: + assert isinstance(mutable, LayerChoice) + index, _ = mutable.registered_module.chosen_index + # pylint: disable=not-callable + result[mutable.key] = F.one_hot(torch.tensor(index), num_classes=len(mutable)).view(-1).bool() + return result diff --git a/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/proxylessnas/trainer.py b/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/proxylessnas/trainer.py new file mode 100644 index 0000000000000000000000000000000000000000..d9c86a6a9f098792a4731db32dd140ce3708ea8f --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/proxylessnas/trainer.py @@ -0,0 +1,500 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import math +import time +import json +import logging + +import torch +from torch import nn as nn + +from nni.nas.pytorch.base_trainer import BaseTrainer +from nni.nas.pytorch.trainer import TorchTensorEncoder +from nni.nas.pytorch.utils import AverageMeter +from .mutator import ProxylessNasMutator +from .utils import cross_entropy_with_label_smoothing, accuracy + +logger = logging.getLogger(__name__) + +class ProxylessNasTrainer(BaseTrainer): + def __init__(self, model, model_optim, device, + train_loader, valid_loader, label_smoothing=0.1, + n_epochs=120, init_lr=0.025, binary_mode='full_v2', + arch_init_type='normal', arch_init_ratio=1e-3, + arch_optim_lr=1e-3, arch_weight_decay=0, + grad_update_arch_param_every=5, grad_update_steps=1, + warmup=True, warmup_epochs=25, + arch_valid_frequency=1, + load_ckpt=False, ckpt_path=None, arch_path=None): + """ + Parameters + ---------- + model : pytorch model + the user model, which has mutables + model_optim : pytorch optimizer + the user defined optimizer + device : pytorch device + the devices to train/search the model + train_loader : pytorch data loader + data loader for the training set + valid_loader : pytorch data loader + data loader for the validation set + label_smoothing : float + for label smoothing + n_epochs : int + number of epochs to train/search + init_lr : float + init learning rate for training the model + binary_mode : str + the forward/backward mode for the binary weights in mutator + arch_init_type : str + the way to init architecture parameters + arch_init_ratio : float + the ratio to init architecture parameters + arch_optim_lr : float + learning rate of the architecture parameters optimizer + arch_weight_decay : float + weight decay of the architecture parameters optimizer + grad_update_arch_param_every : int + update architecture weights every this number of minibatches + grad_update_steps : int + during each update of architecture weights, the number of steps to train + warmup : bool + whether to do warmup + warmup_epochs : int + the number of epochs to do during warmup + arch_valid_frequency : int + frequency of printing validation result + load_ckpt : bool + whether load checkpoint + ckpt_path : str + checkpoint path, if load_ckpt is True, ckpt_path cannot be None + arch_path : str + the path to store chosen architecture + """ + self.model = model + self.model_optim = model_optim + self.train_loader = train_loader + self.valid_loader = valid_loader + self.device = device + self.n_epochs = n_epochs + self.init_lr = init_lr + self.warmup = warmup + self.warmup_epochs = warmup_epochs + self.arch_valid_frequency = arch_valid_frequency + self.label_smoothing = label_smoothing + + self.train_batch_size = train_loader.batch_sampler.batch_size + self.valid_batch_size = valid_loader.batch_sampler.batch_size + # update architecture parameters every this number of minibatches + self.grad_update_arch_param_every = grad_update_arch_param_every + # the number of steps per architecture parameter update + self.grad_update_steps = grad_update_steps + self.binary_mode = binary_mode + + self.load_ckpt = load_ckpt + self.ckpt_path = ckpt_path + self.arch_path = arch_path + + # init mutator + self.mutator = ProxylessNasMutator(model) + + # DataParallel should be put behind the init of mutator + self.model = torch.nn.DataParallel(self.model) + self.model.to(self.device) + + # iter of valid dataset for training architecture weights + self._valid_iter = None + # init architecture weights + self._init_arch_params(arch_init_type, arch_init_ratio) + # build architecture optimizer + self.arch_optimizer = torch.optim.Adam(self.mutator.get_architecture_parameters(), + arch_optim_lr, + weight_decay=arch_weight_decay, + betas=(0, 0.999), + eps=1e-8) + + self.criterion = nn.CrossEntropyLoss() + self.warmup_curr_epoch = 0 + self.train_curr_epoch = 0 + + def _init_arch_params(self, init_type='normal', init_ratio=1e-3): + """ + Initialize architecture weights + """ + for param in self.mutator.get_architecture_parameters(): + if init_type == 'normal': + param.data.normal_(0, init_ratio) + elif init_type == 'uniform': + param.data.uniform_(-init_ratio, init_ratio) + else: + raise NotImplementedError + + def _validate(self): + """ + Do validation. During validation, LayerChoices use the chosen active op. + + Returns + ------- + float, float, float + average loss, average top1 accuracy, average top5 accuracy + """ + self.valid_loader.batch_sampler.batch_size = self.valid_batch_size + self.valid_loader.batch_sampler.drop_last = False + + self.mutator.set_chosen_op_active() + # remove unused modules to save memory + self.mutator.unused_modules_off() + # test on validation set under train mode + self.model.train() + batch_time = AverageMeter('batch_time') + losses = AverageMeter('losses') + top1 = AverageMeter('top1') + top5 = AverageMeter('top5') + end = time.time() + with torch.no_grad(): + for i, (images, labels) in enumerate(self.valid_loader): + images, labels = images.to(self.device), labels.to(self.device) + output = self.model(images) + loss = self.criterion(output, labels) + acc1, acc5 = accuracy(output, labels, topk=(1, 5)) + losses.update(loss, images.size(0)) + top1.update(acc1[0], images.size(0)) + top5.update(acc5[0], images.size(0)) + # measure elapsed time + batch_time.update(time.time() - end) + end = time.time() + + if i % 10 == 0 or i + 1 == len(self.valid_loader): + test_log = 'Valid' + ': [{0}/{1}]\t'\ + 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'\ + 'Loss {loss.val:.4f} ({loss.avg:.4f})\t'\ + 'Top-1 acc {top1.val:.3f} ({top1.avg:.3f})'.\ + format(i, len(self.valid_loader) - 1, batch_time=batch_time, loss=losses, top1=top1) + # return top5: + test_log += '\tTop-5 acc {top5.val:.3f} ({top5.avg:.3f})'.format(top5=top5) + logger.info(test_log) + self.mutator.unused_modules_back() + return losses.avg, top1.avg, top5.avg + + def _warm_up(self): + """ + Warm up the model, during warm up, architecture weights are not trained. + """ + lr_max = 0.05 + data_loader = self.train_loader + nBatch = len(data_loader) + T_total = self.warmup_epochs * nBatch # total num of batches + + for epoch in range(self.warmup_curr_epoch, self.warmup_epochs): + logger.info('\n--------Warmup epoch: %d--------\n', epoch + 1) + batch_time = AverageMeter('batch_time') + data_time = AverageMeter('data_time') + losses = AverageMeter('losses') + top1 = AverageMeter('top1') + top5 = AverageMeter('top5') + # switch to train mode + self.model.train() + + end = time.time() + logger.info('warm_up epoch: %d', epoch) + for i, (images, labels) in enumerate(data_loader): + data_time.update(time.time() - end) + # lr + T_cur = epoch * nBatch + i + warmup_lr = 0.5 * lr_max * (1 + math.cos(math.pi * T_cur / T_total)) + for param_group in self.model_optim.param_groups: + param_group['lr'] = warmup_lr + images, labels = images.to(self.device), labels.to(self.device) + # compute output + self.mutator.reset_binary_gates() # random sample binary gates + self.mutator.unused_modules_off() # remove unused module for speedup + output = self.model(images) + if self.label_smoothing > 0: + loss = cross_entropy_with_label_smoothing(output, labels, self.label_smoothing) + else: + loss = self.criterion(output, labels) + # measure accuracy and record loss + acc1, acc5 = accuracy(output, labels, topk=(1, 5)) + losses.update(loss, images.size(0)) + top1.update(acc1[0], images.size(0)) + top5.update(acc5[0], images.size(0)) + # compute gradient and do SGD step + self.model.zero_grad() + loss.backward() + self.model_optim.step() + # unused modules back + self.mutator.unused_modules_back() + # measure elapsed time + batch_time.update(time.time() - end) + end = time.time() + + if i % 10 == 0 or i + 1 == nBatch: + batch_log = 'Warmup Train [{0}][{1}/{2}]\t' \ + 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' \ + 'Data {data_time.val:.3f} ({data_time.avg:.3f})\t' \ + 'Loss {losses.val:.4f} ({losses.avg:.4f})\t' \ + 'Top-1 acc {top1.val:.3f} ({top1.avg:.3f})\t' \ + 'Top-5 acc {top5.val:.3f} ({top5.avg:.3f})\tlr {lr:.5f}'. \ + format(epoch + 1, i, nBatch - 1, batch_time=batch_time, data_time=data_time, + losses=losses, top1=top1, top5=top5, lr=warmup_lr) + logger.info(batch_log) + val_loss, val_top1, val_top5 = self._validate() + val_log = 'Warmup Valid [{0}/{1}]\tloss {2:.3f}\ttop-1 acc {3:.3f}\ttop-5 acc {4:.3f}\t' \ + 'Train top-1 {top1.avg:.3f}\ttop-5 {top5.avg:.3f}M'. \ + format(epoch + 1, self.warmup_epochs, val_loss, val_top1, val_top5, top1=top1, top5=top5) + logger.info(val_log) + self.save_checkpoint() + self.warmup_curr_epoch += 1 + + def _get_update_schedule(self, nBatch): + """ + Generate schedule for training architecture weights. Key means after which minibatch + to update architecture weights, value means how many steps for the update. + + Parameters + ---------- + nBatch : int + the total number of minibatches in one epoch + + Returns + ------- + dict + the schedule for updating architecture weights + """ + schedule = {} + for i in range(nBatch): + if (i + 1) % self.grad_update_arch_param_every == 0: + schedule[i] = self.grad_update_steps + return schedule + + def _calc_learning_rate(self, epoch, batch=0, nBatch=None): + """ + Update learning rate. + """ + T_total = self.n_epochs * nBatch + T_cur = epoch * nBatch + batch + lr = 0.5 * self.init_lr * (1 + math.cos(math.pi * T_cur / T_total)) + return lr + + def _adjust_learning_rate(self, optimizer, epoch, batch=0, nBatch=None): + """ + Adjust learning of a given optimizer and return the new learning rate + + Parameters + ---------- + optimizer : pytorch optimizer + the used optimizer + epoch : int + the current epoch number + batch : int + the current minibatch + nBatch : int + the total number of minibatches in one epoch + + Returns + ------- + float + the adjusted learning rate + """ + new_lr = self._calc_learning_rate(epoch, batch, nBatch) + for param_group in optimizer.param_groups: + param_group['lr'] = new_lr + return new_lr + + def _train(self): + """ + Train the model, it trains model weights and architecute weights. + Architecture weights are trained according to the schedule. + Before updating architecture weights, ```requires_grad``` is enabled. + Then, it is disabled after the updating, in order not to update + architecture weights when training model weights. + """ + nBatch = len(self.train_loader) + arch_param_num = self.mutator.num_arch_params() + binary_gates_num = self.mutator.num_arch_params() + logger.info('#arch_params: %d\t#binary_gates: %d', arch_param_num, binary_gates_num) + + update_schedule = self._get_update_schedule(nBatch) + + for epoch in range(self.train_curr_epoch, self.n_epochs): + logger.info('\n--------Train epoch: %d--------\n', epoch + 1) + batch_time = AverageMeter('batch_time') + data_time = AverageMeter('data_time') + losses = AverageMeter('losses') + top1 = AverageMeter('top1') + top5 = AverageMeter('top5') + # switch to train mode + self.model.train() + + end = time.time() + for i, (images, labels) in enumerate(self.train_loader): + data_time.update(time.time() - end) + lr = self._adjust_learning_rate(self.model_optim, epoch, batch=i, nBatch=nBatch) + # train weight parameters + images, labels = images.to(self.device), labels.to(self.device) + self.mutator.reset_binary_gates() + self.mutator.unused_modules_off() + output = self.model(images) + if self.label_smoothing > 0: + loss = cross_entropy_with_label_smoothing(output, labels, self.label_smoothing) + else: + loss = self.criterion(output, labels) + acc1, acc5 = accuracy(output, labels, topk=(1, 5)) + losses.update(loss, images.size(0)) + top1.update(acc1[0], images.size(0)) + top5.update(acc5[0], images.size(0)) + self.model.zero_grad() + loss.backward() + self.model_optim.step() + self.mutator.unused_modules_back() + if epoch > 0: + for _ in range(update_schedule.get(i, 0)): + start_time = time.time() + # GradientArchSearchConfig + self.mutator.arch_requires_grad() + arch_loss, exp_value = self._gradient_step() + self.mutator.arch_disable_grad() + used_time = time.time() - start_time + log_str = 'Architecture [%d-%d]\t Time %.4f\t Loss %.4f\t null %s' % \ + (epoch + 1, i, used_time, arch_loss, exp_value) + logger.info(log_str) + batch_time.update(time.time() - end) + end = time.time() + # training log + if i % 10 == 0 or i + 1 == nBatch: + batch_log = 'Train [{0}][{1}/{2}]\t' \ + 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' \ + 'Data Time {data_time.val:.3f} ({data_time.avg:.3f})\t' \ + 'Loss {losses.val:.4f} ({losses.avg:.4f})\t' \ + 'Top-1 acc {top1.val:.3f} ({top1.avg:.3f})\t' \ + 'Top-5 acc {top5.val:.3f} ({top5.avg:.3f})\tlr {lr:.5f}'. \ + format(epoch + 1, i, nBatch - 1, batch_time=batch_time, data_time=data_time, + losses=losses, top1=top1, top5=top5, lr=lr) + logger.info(batch_log) + # validate + if (epoch + 1) % self.arch_valid_frequency == 0: + val_loss, val_top1, val_top5 = self._validate() + val_log = 'Valid [{0}]\tloss {1:.3f}\ttop-1 acc {2:.3f} \ttop-5 acc {3:.3f}\t' \ + 'Train top-1 {top1.avg:.3f}\ttop-5 {top5.avg:.3f}'. \ + format(epoch + 1, val_loss, val_top1, val_top5, top1=top1, top5=top5) + logger.info(val_log) + self.save_checkpoint() + self.train_curr_epoch += 1 + + def _valid_next_batch(self): + """ + Get next one minibatch from validation set + + Returns + ------- + (tensor, tensor) + the tuple of images and labels + """ + if self._valid_iter is None: + self._valid_iter = iter(self.valid_loader) + try: + data = next(self._valid_iter) + except StopIteration: + self._valid_iter = iter(self.valid_loader) + data = next(self._valid_iter) + return data + + def _gradient_step(self): + """ + This gradient step is for updating architecture weights. + Mutator is intensively used in this function to operate on + architecture weights. + + Returns + ------- + float, None + loss of the model, None + """ + # use the same batch size as train batch size for architecture weights + self.valid_loader.batch_sampler.batch_size = self.train_batch_size + self.valid_loader.batch_sampler.drop_last = True + self.model.train() + self.mutator.change_forward_mode(self.binary_mode) + time1 = time.time() # time + # sample a batch of data from validation set + images, labels = self._valid_next_batch() + images, labels = images.to(self.device), labels.to(self.device) + time2 = time.time() # time + self.mutator.reset_binary_gates() + self.mutator.unused_modules_off() + output = self.model(images) + time3 = time.time() + ce_loss = self.criterion(output, labels) + expected_value = None + loss = ce_loss + self.model.zero_grad() + loss.backward() + self.mutator.set_arch_param_grad() + self.arch_optimizer.step() + if self.mutator.get_forward_mode() == 'two': + self.mutator.rescale_updated_arch_param() + self.mutator.unused_modules_back() + self.mutator.change_forward_mode(None) + time4 = time.time() + logger.info('(%.4f, %.4f, %.4f)', time2 - time1, time3 - time2, time4 - time3) + return loss.data.item(), expected_value.item() if expected_value is not None else None + + def save_checkpoint(self): + """ + Save checkpoint of the whole model. Saving model weights and architecture weights in + ```ckpt_path```, and saving currently chosen architecture in ```arch_path```. + """ + if self.ckpt_path: + state = { + 'warmup_curr_epoch': self.warmup_curr_epoch, + 'train_curr_epoch': self.train_curr_epoch, + 'model': self.model.state_dict(), + 'optim': self.model_optim.state_dict(), + 'arch_optim': self.arch_optimizer.state_dict() + } + torch.save(state, self.ckpt_path) + if self.arch_path: + self.export(self.arch_path) + + def load_checkpoint(self): + """ + Load the checkpoint from ```ckpt_path```. + """ + assert self.ckpt_path is not None, "If load_ckpt is not None, ckpt_path should not be None" + ckpt = torch.load(self.ckpt_path) + self.warmup_curr_epoch = ckpt['warmup_curr_epoch'] + self.train_curr_epoch = ckpt['train_curr_epoch'] + self.model.load_state_dict(ckpt['model']) + self.model_optim.load_state_dict(ckpt['optim']) + self.arch_optimizer.load_state_dict(ckpt['arch_optim']) + + def train(self): + """ + Train the whole model. + """ + if self.load_ckpt: + self.load_checkpoint() + if self.warmup: + self._warm_up() + self._train() + + def export(self, file_name): + """ + Export the chosen architecture into a file + + Parameters + ---------- + file_name : str + the file that stores exported chosen architecture + """ + exported_arch = self.mutator.sample_final() + with open(file_name, 'w') as f: + json.dump(exported_arch, f, indent=2, sort_keys=True, cls=TorchTensorEncoder) + + def validate(self): + raise NotImplementedError + + def checkpoint(self): + raise NotImplementedError diff --git a/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/proxylessnas/utils.py b/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/proxylessnas/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..b703810d3b703a33bd6bbe8422c557a1f669146b --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/proxylessnas/utils.py @@ -0,0 +1,78 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import torch +import torch.nn as nn + +def detach_variable(inputs): + """ + Detach variables + + Parameters + ---------- + inputs : pytorch tensors + pytorch tensors + """ + if isinstance(inputs, tuple): + return tuple([detach_variable(x) for x in inputs]) + else: + x = inputs.detach() + x.requires_grad = inputs.requires_grad + return x + +def cross_entropy_with_label_smoothing(pred, target, label_smoothing=0.1): + """ + Parameters + ---------- + pred : pytorch tensor + predicted value + target : pytorch tensor + label + label_smoothing : float + the degree of label smoothing + + Returns + ------- + pytorch tensor + cross entropy + """ + logsoftmax = nn.LogSoftmax() + n_classes = pred.size(1) + # convert to one-hot + target = torch.unsqueeze(target, 1) + soft_target = torch.zeros_like(pred) + soft_target.scatter_(1, target, 1) + # label smoothing + soft_target = soft_target * (1 - label_smoothing) + label_smoothing / n_classes + return torch.mean(torch.sum(- soft_target * logsoftmax(pred), 1)) + +def accuracy(output, target, topk=(1,)): + """ + Computes the precision@k for the specified values of k + + Parameters + ---------- + output : pytorch tensor + output, e.g., predicted value + target : pytorch tensor + label + topk : tuple + specify top1 and top5 + + Returns + ------- + list + accuracy of top1 and top5 + """ + maxk = max(topk) + batch_size = target.size(0) + + _, pred = output.topk(maxk, 1, True, True) + pred = pred.t() + correct = pred.eq(target.view(1, -1).expand_as(pred)) + + res = [] + for k in topk: + correct_k = correct[:k].view(-1).float().sum(0, keepdim=True) + res.append(correct_k.mul_(100.0 / batch_size)) + return res diff --git a/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/random/__init__.py b/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/random/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..b4102266ca2a03d29c1c16c90b688865ce493b7a --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/random/__init__.py @@ -0,0 +1 @@ +from .mutator import RandomMutator diff --git a/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/random/mutator.py b/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/random/mutator.py new file mode 100644 index 0000000000000000000000000000000000000000..f302db56c0cdbd611f648342cc26760d344958c4 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/random/mutator.py @@ -0,0 +1,36 @@ +import torch +import torch.nn.functional as F + +from nni.nas.pytorch.mutator import Mutator +from nni.nas.pytorch.mutables import LayerChoice, InputChoice + + +class RandomMutator(Mutator): + """ + Random mutator that samples a random candidate in the search space each time ``reset()``. + It uses random function in PyTorch, so users can set seed in PyTorch to ensure deterministic behavior. + """ + + def sample_search(self): + """ + Sample a random candidate. + """ + result = dict() + for mutable in self.mutables: + if isinstance(mutable, LayerChoice): + gen_index = torch.randint(high=len(mutable), size=(1, )) + result[mutable.key] = F.one_hot(gen_index, num_classes=len(mutable)).view(-1).bool() + elif isinstance(mutable, InputChoice): + if mutable.n_chosen is None: + result[mutable.key] = torch.randint(high=2, size=(mutable.n_candidates,)).view(-1).bool() + else: + perm = torch.randperm(mutable.n_candidates) + mask = [i in perm[:mutable.n_chosen] for i in range(mutable.n_candidates)] + result[mutable.key] = torch.tensor(mask, dtype=torch.bool) # pylint: disable=not-callable + return result + + def sample_final(self): + """ + Same as :meth:`sample_search`. + """ + return self.sample_search() diff --git a/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/spos/__init__.py b/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/spos/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..ed432b0845154c4745aa82c8e6a8bad4290237aa --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/spos/__init__.py @@ -0,0 +1,6 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .evolution import SPOSEvolution +from .mutator import SPOSSupernetTrainingMutator +from .trainer import SPOSSupernetTrainer diff --git a/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/spos/evolution.py b/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/spos/evolution.py new file mode 100644 index 0000000000000000000000000000000000000000..bd099e276ebfd4aa68215dbbd25e6443a45b9dd1 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/spos/evolution.py @@ -0,0 +1,223 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import json +import logging +import os +import re +from collections import deque + +import numpy as np +from nni.tuner import Tuner +from nni.algorithms.nas.pytorch.classic_nas.mutator import LAYER_CHOICE, INPUT_CHOICE + + +_logger = logging.getLogger(__name__) + + +class SPOSEvolution(Tuner): + """ + SPOS evolution tuner. + + Parameters + ---------- + max_epochs : int + Maximum number of epochs to run. + num_select : int + Number of survival candidates of each epoch. + num_population : int + Number of candidates at the start of each epoch. If candidates generated by + crossover and mutation are not enough, the rest will be filled with random + candidates. + m_prob : float + The probability of mutation. + num_crossover : int + Number of candidates generated by crossover in each epoch. + num_mutation : int + Number of candidates generated by mutation in each epoch. + """ + + def __init__(self, max_epochs=20, num_select=10, num_population=50, m_prob=0.1, + num_crossover=25, num_mutation=25): + assert num_population >= num_select + self.max_epochs = max_epochs + self.num_select = num_select + self.num_population = num_population + self.m_prob = m_prob + self.num_crossover = num_crossover + self.num_mutation = num_mutation + self.epoch = 0 + self.candidates = [] + self.search_space = None + self.random_state = np.random.RandomState(0) + + # async status + self._to_evaluate_queue = deque() + self._sending_parameter_queue = deque() + self._pending_result_ids = set() + self._reward_dict = dict() + self._id2candidate = dict() + self._st_callback = None + + def update_search_space(self, search_space): + """ + Handle the initialization/update event of search space. + """ + self._search_space = search_space + self._next_round() + + def _next_round(self): + _logger.info("Epoch %d, generating...", self.epoch) + if self.epoch == 0: + self._get_random_population() + self.export_results(self.candidates) + else: + best_candidates = self._select_top_candidates() + self.export_results(best_candidates) + if self.epoch >= self.max_epochs: + return + self.candidates = self._get_mutation(best_candidates) + self._get_crossover(best_candidates) + self._get_random_population() + self.epoch += 1 + + def _random_candidate(self): + chosen_arch = dict() + for key, val in self._search_space.items(): + if val["_type"] == LAYER_CHOICE: + choices = val["_value"] + index = self.random_state.randint(len(choices)) + chosen_arch[key] = {"_value": choices[index], "_idx": index} + elif val["_type"] == INPUT_CHOICE: + raise NotImplementedError("Input choice is not implemented yet.") + return chosen_arch + + def _add_to_evaluate_queue(self, cand): + _logger.info("Generate candidate %s, adding to eval queue.", self._get_architecture_repr(cand)) + self._reward_dict[self._hashcode(cand)] = 0. + self._to_evaluate_queue.append(cand) + + def _get_random_population(self): + while len(self.candidates) < self.num_population: + cand = self._random_candidate() + if self._is_legal(cand): + _logger.info("Random candidate generated.") + self._add_to_evaluate_queue(cand) + self.candidates.append(cand) + + def _get_crossover(self, best): + result = [] + for _ in range(10 * self.num_crossover): + cand_p1 = best[self.random_state.randint(len(best))] + cand_p2 = best[self.random_state.randint(len(best))] + assert cand_p1.keys() == cand_p2.keys() + cand = {k: cand_p1[k] if self.random_state.randint(2) == 0 else cand_p2[k] + for k in cand_p1.keys()} + if self._is_legal(cand): + result.append(cand) + self._add_to_evaluate_queue(cand) + if len(result) >= self.num_crossover: + break + _logger.info("Found %d architectures with crossover.", len(result)) + return result + + def _get_mutation(self, best): + result = [] + for _ in range(10 * self.num_mutation): + cand = best[self.random_state.randint(len(best))].copy() + mutation_sample = np.random.random_sample(len(cand)) + for s, k in zip(mutation_sample, cand): + if s < self.m_prob: + choices = self._search_space[k]["_value"] + index = self.random_state.randint(len(choices)) + cand[k] = {"_value": choices[index], "_idx": index} + if self._is_legal(cand): + result.append(cand) + self._add_to_evaluate_queue(cand) + if len(result) >= self.num_mutation: + break + _logger.info("Found %d architectures with mutation.", len(result)) + return result + + def _get_architecture_repr(self, cand): + return re.sub(r"\".*?\": \{\"_idx\": (\d+), \"_value\": \".*?\"\}", r"\1", + self._hashcode(cand)) + + def _is_legal(self, cand): + if self._hashcode(cand) in self._reward_dict: + return False + return True + + def _select_top_candidates(self): + reward_query = lambda cand: self._reward_dict[self._hashcode(cand)] + _logger.info("All candidate rewards: %s", list(map(reward_query, self.candidates))) + result = sorted(self.candidates, key=reward_query, reverse=True)[:self.num_select] + _logger.info("Best candidate rewards: %s", list(map(reward_query, result))) + return result + + @staticmethod + def _hashcode(d): + return json.dumps(d, sort_keys=True) + + def _bind_and_send_parameters(self): + """ + There are two types of resources: parameter ids and candidates. This function is called at + necessary times to bind these resources to send new trials with st_callback. + """ + result = [] + while self._sending_parameter_queue and self._to_evaluate_queue: + parameter_id = self._sending_parameter_queue.popleft() + parameters = self._to_evaluate_queue.popleft() + self._id2candidate[parameter_id] = parameters + result.append(parameters) + self._pending_result_ids.add(parameter_id) + self._st_callback(parameter_id, parameters) + _logger.info("Send parameter [%d] %s.", parameter_id, self._get_architecture_repr(parameters)) + return result + + def generate_multiple_parameters(self, parameter_id_list, **kwargs): + """ + Callback function necessary to implement a tuner. This will put more parameter ids into the + parameter id queue. + """ + if "st_callback" in kwargs and self._st_callback is None: + self._st_callback = kwargs["st_callback"] + for parameter_id in parameter_id_list: + self._sending_parameter_queue.append(parameter_id) + self._bind_and_send_parameters() + return [] # always not use this. might induce problem of over-sending + + def receive_trial_result(self, parameter_id, parameters, value, **kwargs): + """ + Callback function. Receive a trial result. + """ + _logger.info("Candidate %d, reported reward %f", parameter_id, value) + self._reward_dict[self._hashcode(self._id2candidate[parameter_id])] = value + + def trial_end(self, parameter_id, success, **kwargs): + """ + Callback function when a trial is ended and resource is released. + """ + self._pending_result_ids.remove(parameter_id) + if not self._pending_result_ids and not self._to_evaluate_queue: + # a new epoch now + self._next_round() + assert self._st_callback is not None + self._bind_and_send_parameters() + + def export_results(self, result): + """ + Export a number of candidates to `checkpoints` dir. + + Parameters + ---------- + result : dict + Chosen architectures to be exported. + """ + os.makedirs("checkpoints", exist_ok=True) + for i, cand in enumerate(result): + converted = dict() + for cand_key, cand_val in cand.items(): + onehot = [k == cand_val["_idx"] for k in range(len(self._search_space[cand_key]["_value"]))] + converted[cand_key] = onehot + with open(os.path.join("checkpoints", "%03d_%03d.json" % (self.epoch, i)), "w") as fp: + json.dump(converted, fp) diff --git a/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/spos/mutator.py b/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/spos/mutator.py new file mode 100644 index 0000000000000000000000000000000000000000..1a803cb2e820b0df2dd8b04b3d387af68d3d2680 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/spos/mutator.py @@ -0,0 +1,66 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging + +import numpy as np +from nni.algorithms.nas.pytorch.random import RandomMutator + +_logger = logging.getLogger(__name__) + + +class SPOSSupernetTrainingMutator(RandomMutator): + """ + A random mutator with flops limit. + + Parameters + ---------- + model : nn.Module + PyTorch model. + flops_func : callable + Callable that takes a candidate from `sample_search` and returns its candidate. When `flops_func` + is None, functions related to flops will be deactivated. + flops_lb : number + Lower bound of flops. + flops_ub : number + Upper bound of flops. + flops_bin_num : number + Number of bins divided for the interval of flops to ensure the uniformity. Bigger number will be more + uniform, but the sampling will be slower. + flops_sample_timeout : int + Maximum number of attempts to sample before giving up and use a random candidate. + """ + def __init__(self, model, flops_func=None, flops_lb=None, flops_ub=None, + flops_bin_num=7, flops_sample_timeout=500): + + super().__init__(model) + self._flops_func = flops_func + if self._flops_func is not None: + self._flops_bin_num = flops_bin_num + self._flops_bins = [flops_lb + (flops_ub - flops_lb) / flops_bin_num * i for i in range(flops_bin_num + 1)] + self._flops_sample_timeout = flops_sample_timeout + + def sample_search(self): + """ + Sample a candidate for training. When `flops_func` is not None, candidates will be sampled uniformly + relative to flops. + + Returns + ------- + dict + """ + if self._flops_func is not None: + for times in range(self._flops_sample_timeout): + idx = np.random.randint(self._flops_bin_num) + cand = super().sample_search() + if self._flops_bins[idx] <= self._flops_func(cand) <= self._flops_bins[idx + 1]: + _logger.debug("Sampled candidate flops %f in %d times.", cand, times) + return cand + _logger.warning("Failed to sample a flops-valid candidate within %d tries.", self._flops_sample_timeout) + return super().sample_search() + + def sample_final(self): + """ + Implement only to suffice the interface of Mutator. + """ + return self.sample_search() diff --git a/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/spos/trainer.py b/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/spos/trainer.py new file mode 100644 index 0000000000000000000000000000000000000000..7c954e2ad4c913fd16f8e9adf195f2ebb14b1e9b --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/nas/pytorch/spos/trainer.py @@ -0,0 +1,95 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging + +import torch +from nni.nas.pytorch.trainer import Trainer +from nni.nas.pytorch.utils import AverageMeterGroup + +from .mutator import SPOSSupernetTrainingMutator + +logger = logging.getLogger(__name__) + + +class SPOSSupernetTrainer(Trainer): + """ + This trainer trains a supernet that can be used for evolution search. + + Parameters + ---------- + model : nn.Module + Model with mutables. + mutator : nni.nas.pytorch.mutator.Mutator + A mutator object that has been initialized with the model. + loss : callable + Called with logits and targets. Returns a loss tensor. + metrics : callable + Returns a dict that maps metrics keys to metrics data. + optimizer : Optimizer + Optimizer that optimizes the model. + num_epochs : int + Number of epochs of training. + train_loader : iterable + Data loader of training. Raise ``StopIteration`` when one epoch is exhausted. + dataset_valid : iterable + Data loader of validation. Raise ``StopIteration`` when one epoch is exhausted. + batch_size : int + Batch size. + workers: int + Number of threads for data preprocessing. Not used for this trainer. Maybe removed in future. + device : torch.device + Device object. Either ``torch.device("cuda")`` or ``torch.device("cpu")``. When ``None``, trainer will + automatic detects GPU and selects GPU first. + log_frequency : int + Number of mini-batches to log metrics. + callbacks : list of Callback + Callbacks to plug into the trainer. See Callbacks. + """ + + def __init__(self, model, loss, metrics, + optimizer, num_epochs, train_loader, valid_loader, + mutator=None, batch_size=64, workers=4, device=None, log_frequency=None, + callbacks=None): + assert torch.cuda.is_available() + super().__init__(model, mutator if mutator is not None else SPOSSupernetTrainingMutator(model), + loss, metrics, optimizer, num_epochs, None, None, + batch_size, workers, device, log_frequency, callbacks) + + self.train_loader = train_loader + self.valid_loader = valid_loader + + def train_one_epoch(self, epoch): + self.model.train() + meters = AverageMeterGroup() + for step, (x, y) in enumerate(self.train_loader): + x, y = x.to(self.device), y.to(self.device) + self.optimizer.zero_grad() + self.mutator.reset() + logits = self.model(x) + loss = self.loss(logits, y) + loss.backward() + self.optimizer.step() + + metrics = self.metrics(logits, y) + metrics["loss"] = loss.item() + meters.update(metrics) + if self.log_frequency is not None and step % self.log_frequency == 0: + logger.info("Epoch [%s/%s] Step [%s/%s] %s", epoch + 1, + self.num_epochs, step + 1, len(self.train_loader), meters) + + def validate_one_epoch(self, epoch): + self.model.eval() + meters = AverageMeterGroup() + with torch.no_grad(): + for step, (x, y) in enumerate(self.valid_loader): + x, y = x.to(self.device), y.to(self.device) + self.mutator.reset() + logits = self.model(x) + loss = self.loss(logits, y) + metrics = self.metrics(logits, y) + metrics["loss"] = loss.item() + meters.update(metrics) + if self.log_frequency is not None and step % self.log_frequency == 0: + logger.info("Epoch [%s/%s] Validation Step [%s/%s] %s", epoch + 1, + self.num_epochs, step + 1, len(self.valid_loader), meters) diff --git a/new_impl/cv/third_party/nni_new/algorithms/nas/tensorflow/classic_nas/__init__.py b/new_impl/cv/third_party/nni_new/algorithms/nas/tensorflow/classic_nas/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..ec3f5a4894a0460d4a0582a0d6cd43af9bed77e2 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/nas/tensorflow/classic_nas/__init__.py @@ -0,0 +1,4 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .mutator import get_and_apply_next_architecture diff --git a/new_impl/cv/third_party/nni_new/algorithms/nas/tensorflow/classic_nas/mutator.py b/new_impl/cv/third_party/nni_new/algorithms/nas/tensorflow/classic_nas/mutator.py new file mode 100644 index 0000000000000000000000000000000000000000..fad4987fed55bc9a1460089ee0fd3b92a63ee83f --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/nas/tensorflow/classic_nas/mutator.py @@ -0,0 +1,215 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import json +import logging +import os +import sys + +import tensorflow as tf + +import nni +from nni.runtime.env_vars import trial_env_vars +from nni.nas.tensorflow.mutables import LayerChoice, InputChoice, MutableScope +from nni.nas.tensorflow.mutator import Mutator + +logger = logging.getLogger(__name__) + +NNI_GEN_SEARCH_SPACE = "NNI_GEN_SEARCH_SPACE" +LAYER_CHOICE = "layer_choice" +INPUT_CHOICE = "input_choice" + + +def get_and_apply_next_architecture(model): + """ + Wrapper of :class:`~nni.nas.tensorflow.classic_nas.mutator.ClassicMutator` to make it more meaningful, + similar to ``get_next_parameter`` for HPO. + Tt will generate search space based on ``model``. + If env ``NNI_GEN_SEARCH_SPACE`` exists, this is in dry run mode for + generating search space for the experiment. + If not, there are still two mode, one is nni experiment mode where users + use ``nnictl`` to start an experiment. The other is standalone mode + where users directly run the trial command, this mode chooses the first + one(s) for each LayerChoice and InputChoice. + Parameters + ---------- + model : nn.Module + User's model with search space (e.g., LayerChoice, InputChoice) embedded in it. + """ + ClassicMutator(model) + + +class ClassicMutator(Mutator): + """ + This mutator is to apply the architecture chosen from tuner. + It implements the forward function of LayerChoice and InputChoice, + to only activate the chosen ones. + Parameters + ---------- + model : nn.Module + User's model with search space (e.g., LayerChoice, InputChoice) embedded in it. + """ + + def __init__(self, model): + super(ClassicMutator, self).__init__(model) + self._chosen_arch = {} + self._search_space = self._generate_search_space() + if NNI_GEN_SEARCH_SPACE in os.environ: + # dry run for only generating search space + self._dump_search_space(os.environ[NNI_GEN_SEARCH_SPACE]) + sys.exit(0) + + if trial_env_vars.NNI_PLATFORM is None: + logger.warning("This is in standalone mode, the chosen are the first one(s).") + self._chosen_arch = self._standalone_generate_chosen() + else: + # get chosen arch from tuner + self._chosen_arch = nni.get_next_parameter() + if self._chosen_arch is None: + if trial_env_vars.NNI_PLATFORM == "unittest": + # happens if NNI_PLATFORM is intentionally set, e.g., in UT + logger.warning("`NNI_PLATFORM` is set but `param` is None. Falling back to standalone mode.") + self._chosen_arch = self._standalone_generate_chosen() + else: + raise RuntimeError("Chosen architecture is None. This may be a platform error.") + self.reset() + + def _sample_layer_choice(self, mutable, idx, value, search_space_item): + """ + Convert layer choice to tensor representation. + Parameters + ---------- + mutable : Mutable + idx : int + Number `idx` of list will be selected. + value : str + The verbose representation of the selected value. + search_space_item : list + The list for corresponding search space. + """ + # doesn't support multihot for layer choice yet + assert 0 <= idx < len(mutable) and search_space_item[idx] == value, \ + "Index '{}' in search space '{}' is not '{}'".format(idx, search_space_item, value) + mask = tf.one_hot(idx, len(mutable)) + return tf.cast(tf.reshape(mask, [-1]), tf.bool) + + def _sample_input_choice(self, mutable, idx, value, search_space_item): + """ + Convert input choice to tensor representation. + Parameters + ---------- + mutable : Mutable + idx : int + Number `idx` of list will be selected. + value : str + The verbose representation of the selected value. + search_space_item : list + The list for corresponding search space. + """ + candidate_repr = search_space_item["candidates"] + multihot_list = [False] * mutable.n_candidates + for i, v in zip(idx, value): + assert 0 <= i < mutable.n_candidates and candidate_repr[i] == v, \ + "Index '{}' in search space '{}' is not '{}'".format(i, candidate_repr, v) + assert not multihot_list[i], "'{}' is selected twice in '{}', which is not allowed.".format(i, idx) + multihot_list[i] = True + return tf.cast(multihot_list, tf.bool) # pylint: disable=not-callable + + def sample_search(self): + """ + See :meth:`sample_final`. + """ + return self.sample_final() + + def sample_final(self): + """ + Convert the chosen arch and apply it on model. + """ + assert set(self._chosen_arch.keys()) == set(self._search_space.keys()), \ + "Unmatched keys, expected keys '{}' from search space, found '{}'.".format(self._search_space.keys(), + self._chosen_arch.keys()) + result = dict() + for mutable in self.mutables: + if isinstance(mutable, (LayerChoice, InputChoice)): + assert mutable.key in self._chosen_arch, \ + "Expected '{}' in chosen arch, but not found.".format(mutable.key) + data = self._chosen_arch[mutable.key] + assert isinstance(data, dict) and "_value" in data and "_idx" in data, \ + "'{}' is not a valid choice.".format(data) + if isinstance(mutable, LayerChoice): + result[mutable.key] = self._sample_layer_choice(mutable, data["_idx"], data["_value"], + self._search_space[mutable.key]["_value"]) + elif isinstance(mutable, InputChoice): + result[mutable.key] = self._sample_input_choice(mutable, data["_idx"], data["_value"], + self._search_space[mutable.key]["_value"]) + elif isinstance(mutable, MutableScope): + logger.info("Mutable scope '%s' is skipped during parsing choices.", mutable.key) + else: + raise TypeError("Unsupported mutable type: '%s'." % type(mutable)) + return result + + def _standalone_generate_chosen(self): + """ + Generate the chosen architecture for standalone mode, + i.e., choose the first one(s) for LayerChoice and InputChoice. + :: + { key_name: {"_value": "conv1", + "_idx": 0} } + { key_name: {"_value": ["in1"], + "_idx": [0]} } + Returns + ------- + dict + the chosen architecture + """ + chosen_arch = {} + for key, val in self._search_space.items(): + if val["_type"] == LAYER_CHOICE: + choices = val["_value"] + chosen_arch[key] = {"_value": choices[0], "_idx": 0} + elif val["_type"] == INPUT_CHOICE: + choices = val["_value"]["candidates"] + n_chosen = val["_value"]["n_chosen"] + if n_chosen is None: + n_chosen = len(choices) + chosen_arch[key] = {"_value": choices[:n_chosen], "_idx": list(range(n_chosen))} + else: + raise ValueError("Unknown key '%s' and value '%s'." % (key, val)) + return chosen_arch + + def _generate_search_space(self): + """ + Generate search space from mutables. + Here is the search space format: + :: + { key_name: {"_type": "layer_choice", + "_value": ["conv1", "conv2"]} } + { key_name: {"_type": "input_choice", + "_value": {"candidates": ["in1", "in2"], + "n_chosen": 1}} } + Returns + ------- + dict + the generated search space + """ + search_space = {} + for mutable in self.mutables: + # for now we only generate flattened search space + if isinstance(mutable, LayerChoice): + key = mutable.key + val = mutable.names + search_space[key] = {"_type": LAYER_CHOICE, "_value": val} + elif isinstance(mutable, InputChoice): + key = mutable.key + search_space[key] = {"_type": INPUT_CHOICE, + "_value": {"candidates": mutable.choose_from, + "n_chosen": mutable.n_chosen}} + elif isinstance(mutable, MutableScope): + logger.info("Mutable scope '%s' is skipped during generating search space.", mutable.key) + else: + raise TypeError("Unsupported mutable type: '%s'." % type(mutable)) + return search_space + + def _dump_search_space(self, file_path): + with open(file_path, "w") as ss_file: + json.dump(self._search_space, ss_file, sort_keys=True, indent=2) diff --git a/new_impl/cv/third_party/nni_new/algorithms/nas/tensorflow/enas/__init__.py b/new_impl/cv/third_party/nni_new/algorithms/nas/tensorflow/enas/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d3372836ebba2387ade58161218aad731433e46b --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/nas/tensorflow/enas/__init__.py @@ -0,0 +1,5 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .mutator import EnasMutator +from .trainer import EnasTrainer diff --git a/new_impl/cv/third_party/nni_new/algorithms/nas/tensorflow/enas/mutator.py b/new_impl/cv/third_party/nni_new/algorithms/nas/tensorflow/enas/mutator.py new file mode 100644 index 0000000000000000000000000000000000000000..de43195fa2e6eef6b8c1991cf8d56998d7abc4fb --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/nas/tensorflow/enas/mutator.py @@ -0,0 +1,160 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import tensorflow as tf +from tensorflow.keras.layers import Dense, Embedding, LSTMCell, RNN +from tensorflow.keras.losses import SparseCategoricalCrossentropy, Reduction + +from nni.nas.tensorflow.mutator import Mutator +from nni.nas.tensorflow.mutables import LayerChoice, InputChoice, MutableScope + + +class EnasMutator(Mutator): + def __init__(self, model, + lstm_size=64, + lstm_num_layers=1, + tanh_constant=1.5, + cell_exit_extra_step=False, + skip_target=0.4, + temperature=None, + branch_bias=0.25, + entropy_reduction='sum'): + super().__init__(model) + self.tanh_constant = tanh_constant + self.temperature = temperature + self.cell_exit_extra_step = cell_exit_extra_step + + cells = [LSTMCell(units=lstm_size, use_bias=False) for _ in range(lstm_num_layers)] + self.lstm = RNN(cells, stateful=True) + self.g_emb = tf.random.normal((1, 1, lstm_size)) * 0.1 + self.skip_targets = tf.constant([1.0 - skip_target, skip_target]) + + self.max_layer_choice = 0 + self.bias_dict = {} + for mutable in self.mutables: + if isinstance(mutable, LayerChoice): + if self.max_layer_choice == 0: + self.max_layer_choice = len(mutable) + assert self.max_layer_choice == len(mutable), \ + "ENAS mutator requires all layer choice have the same number of candidates." + if 'reduce' in mutable.key: + bias = [] + for choice in mutable.choices: + if 'conv' in str(type(choice)).lower(): + bias.append(branch_bias) + else: + bias.append(-branch_bias) + self.bias_dict[mutable.key] = tf.constant(bias) + + # exposed for trainer + self.sample_log_prob = 0 + self.sample_entropy = 0 + self.sample_skip_penalty = 0 + + # internal nn layers + self.embedding = Embedding(self.max_layer_choice + 1, lstm_size) + self.soft = Dense(self.max_layer_choice, use_bias=False) + self.attn_anchor = Dense(lstm_size, use_bias=False) + self.attn_query = Dense(lstm_size, use_bias=False) + self.v_attn = Dense(1, use_bias=False) + assert entropy_reduction in ['sum', 'mean'], 'Entropy reduction must be one of sum and mean.' + self.entropy_reduction = tf.reduce_sum if entropy_reduction == 'sum' else tf.reduce_mean + self.cross_entropy_loss = SparseCategoricalCrossentropy(from_logits=True, reduction=Reduction.NONE) + + self._first_sample = True + + def sample_search(self): + self._initialize() + self._sample(self.mutables) + self._first_sample = False + return self._choices + + def sample_final(self): + return self.sample_search() + + def _sample(self, tree): + mutable = tree.mutable + if isinstance(mutable, LayerChoice) and mutable.key not in self._choices: + self._choices[mutable.key] = self._sample_layer_choice(mutable) + elif isinstance(mutable, InputChoice) and mutable.key not in self._choices: + self._choices[mutable.key] = self._sample_input_choice(mutable) + for child in tree.children: + self._sample(child) + if self.cell_exit_extra_step and isinstance(mutable, MutableScope) and mutable.key not in self._anchors_hid: + self._anchors_hid[mutable.key] = self.lstm(self._inputs, 1) + + def _initialize(self): + self._choices = {} + self._anchors_hid = {} + self._inputs = self.g_emb + # seems the `input_shape` parameter of RNN does not work + # workaround it by omitting `reset_states` for first run + if not self._first_sample: + self.lstm.reset_states() + self.sample_log_prob = 0 + self.sample_entropy = 0 + self.sample_skip_penalty = 0 + + def _sample_layer_choice(self, mutable): + logit = self.soft(self.lstm(self._inputs)) + if self.temperature is not None: + logit /= self.temperature + if self.tanh_constant is not None: + logit = self.tanh_constant * tf.tanh(logit) + if mutable.key in self.bias_dict: + logit += self.bias_dict[mutable.key] + softmax_logit = tf.math.log(tf.nn.softmax(logit, axis=-1)) + branch_id = tf.reshape(tf.random.categorical(softmax_logit, num_samples=1), [1]) + log_prob = self.cross_entropy_loss(branch_id, logit) + self.sample_log_prob += self.entropy_reduction(log_prob) + entropy = log_prob * tf.math.exp(-log_prob) + self.sample_entropy += self.entropy_reduction(entropy) + self._inputs = tf.reshape(self.embedding(branch_id), [1, 1, -1]) + mask = tf.one_hot(branch_id, self.max_layer_choice) + return tf.cast(tf.reshape(mask, [-1]), tf.bool) + + def _sample_input_choice(self, mutable): + query, anchors = [], [] + for label in mutable.choose_from: + if label not in self._anchors_hid: + self._anchors_hid[label] = self.lstm(self._inputs) + query.append(self.attn_anchor(self._anchors_hid[label])) + anchors.append(self._anchors_hid[label]) + query = tf.concat(query, axis=0) + query = tf.tanh(query + self.attn_query(anchors[-1])) + query = self.v_attn(query) + + if self.temperature is not None: + query /= self.temperature + if self.tanh_constant is not None: + query = self.tanh_constant * tf.tanh(query) + + if mutable.n_chosen is None: + logit = tf.concat([-query, query], axis=1) + softmax_logit = tf.math.log(tf.nn.softmax(logit, axis=-1)) + skip = tf.reshape(tf.random.categorical(softmax_logit, num_samples=1), [-1]) + skip_prob = tf.math.sigmoid(logit) + kl = tf.reduce_sum(skip_prob * tf.math.log(skip_prob / self.skip_targets)) + self.sample_skip_penalty += kl + log_prob = self.cross_entropy_loss(skip, logit) + + skip = tf.cast(skip, tf.float32) + inputs = tf.tensordot(skip, tf.concat(anchors, 0), 1) / (1. + tf.reduce_sum(skip)) + self._inputs = tf.reshape(inputs, [1, 1, -1]) + + else: + assert mutable.n_chosen == 1, "Input choice must select exactly one or any in ENAS." + logit = tf.reshape(query, [1, -1]) + softmax_logit = tf.math.log(tf.nn.softmax(logit, axis=-1)) + index = tf.reshape(tf.random.categorical(softmax_logit, num_samples=1), [-1]) + skip = tf.reshape(tf.one_hot(index, mutable.n_candidates), [-1]) + # when the size is 1, tf does not accept tensor here, complaining the shape is wrong + # but using a numpy array seems fine + log_prob = self.cross_entropy_loss(logit, query.numpy()) + self._inputs = tf.reshape(anchors[index.numpy()[0]], [1, 1, -1]) + + self.sample_log_prob += self.entropy_reduction(log_prob) + entropy = log_prob * tf.exp(-log_prob) + self.sample_entropy += self.entropy_reduction(entropy) + assert len(skip) == mutable.n_candidates, (skip, mutable.n_candidates, mutable.n_chosen) + return tf.cast(skip, tf.bool) diff --git a/new_impl/cv/third_party/nni_new/algorithms/nas/tensorflow/enas/trainer.py b/new_impl/cv/third_party/nni_new/algorithms/nas/tensorflow/enas/trainer.py new file mode 100644 index 0000000000000000000000000000000000000000..c20f20ab1a68b2c97e6bf27be0f3567016995f1d --- /dev/null +++ b/new_impl/cv/third_party/nni_new/algorithms/nas/tensorflow/enas/trainer.py @@ -0,0 +1,203 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging + +import tensorflow as tf +from tensorflow.keras.optimizers import Adam + +from nni.nas.tensorflow.utils import AverageMeterGroup, fill_zero_grads + +from .mutator import EnasMutator + +logger = logging.getLogger(__name__) + + +class EnasTrainer: + def __init__( + self, + model, + loss, + metrics, + reward_function, + optimizer, + batch_size, + num_epochs, + dataset_train, + dataset_valid, + log_frequency=100, + entropy_weight=0.0001, + skip_weight=0.8, + baseline_decay=0.999, + child_steps=500, + mutator_lr=0.00035, + mutator_steps=50, + mutator_steps_aggregate=20, + aux_weight=0.4, + test_arc_per_epoch=1, + ): + self.model = model + self.loss = loss + self.metrics = metrics + self.reward_function = reward_function + self.optimizer = optimizer + self.batch_size = batch_size + self.num_epochs = num_epochs + + x, y = dataset_train + split = int(len(x) * 0.9) + self.train_set = tf.data.Dataset.from_tensor_slices((x[:split], y[:split])) + self.valid_set = tf.data.Dataset.from_tensor_slices((x[split:], y[split:])) + self.test_set = tf.data.Dataset.from_tensor_slices(dataset_valid) + + self.log_frequency = log_frequency + self.entropy_weight = entropy_weight + self.skip_weight = skip_weight + self.baseline_decay = baseline_decay + self.child_steps = child_steps + self.mutator_lr = mutator_lr + self.mutator_steps = mutator_steps + self.mutator_steps_aggregate = mutator_steps_aggregate + self.aux_weight = aux_weight + self.test_arc_per_epoch = test_arc_per_epoch + + self.mutator = EnasMutator(model) + self.mutator_optim = Adam(learning_rate=self.mutator_lr) + + self.baseline = 0.0 + + def train(self, validate=True): + for epoch in range(self.num_epochs): + logger.info("Epoch %d Training", epoch + 1) + self.train_one_epoch(epoch) + logger.info("Epoch %d Validating", epoch + 1) + self.validate_one_epoch(epoch) + + def validate(self): + self.validate_one_epoch(-1) + + def train_one_epoch(self, epoch): + train_loader, valid_loader = self._create_train_loader() + + # Sample model and train + meters = AverageMeterGroup() + + for step in range(1, self.child_steps + 1): + x, y = next(train_loader) + self.mutator.reset() + + with tf.GradientTape() as tape: + logits = self.model(x, training=True) + if isinstance(logits, tuple): + logits, aux_logits = logits + aux_loss = self.loss(aux_logits, y) + else: + aux_loss = 0.0 + metrics = self.metrics(y, logits) + loss = self.loss(y, logits) + self.aux_weight * aux_loss + + grads = tape.gradient(loss, self.model.trainable_weights) + grads = fill_zero_grads(grads, self.model.trainable_weights) + grads, _ = tf.clip_by_global_norm(grads, 5.0) + self.optimizer.apply_gradients(zip(grads, self.model.trainable_weights)) + + metrics["loss"] = tf.reduce_mean(loss).numpy() + meters.update(metrics) + + if self.log_frequency and step % self.log_frequency == 0: + logger.info( + "Model Epoch [%d/%d] Step [%d/%d] %s", + epoch + 1, + self.num_epochs, + step, + self.child_steps, + meters, + ) + + # Train sampler (mutator) + meters = AverageMeterGroup() + for mutator_step in range(1, self.mutator_steps + 1): + grads_list = [] + for step in range(1, self.mutator_steps_aggregate + 1): + with tf.GradientTape() as tape: + x, y = next(valid_loader) + self.mutator.reset() + + logits = self.model(x, training=False) + metrics = self.metrics(y, logits) + reward = ( + self.reward_function(y, logits) + + self.entropy_weight * self.mutator.sample_entropy + ) + self.baseline = self.baseline * self.baseline_decay + reward * ( + 1 - self.baseline_decay + ) + loss = self.mutator.sample_log_prob * (reward - self.baseline) + loss += self.skip_weight * self.mutator.sample_skip_penalty + + meters.update( + { + "reward": reward, + "loss": tf.reduce_mean(loss).numpy(), + "ent": self.mutator.sample_entropy.numpy(), + "log_prob": self.mutator.sample_log_prob.numpy(), + "baseline": self.baseline, + "skip": self.mutator.sample_skip_penalty, + } + ) + + cur_step = step + (mutator_step - 1) * self.mutator_steps_aggregate + if self.log_frequency and cur_step % self.log_frequency == 0: + logger.info( + "RL Epoch [%d/%d] Step [%d/%d] [%d/%d] %s", + epoch + 1, + self.num_epochs, + mutator_step, + self.mutator_steps, + step, + self.mutator_steps_aggregate, + meters, + ) + + grads = tape.gradient(loss, self.mutator.trainable_weights) + grads = fill_zero_grads(grads, self.mutator.trainable_weights) + grads_list.append(grads) + total_grads = [ + tf.math.add_n(weight_grads) for weight_grads in zip(*grads_list) + ] + total_grads, _ = tf.clip_by_global_norm(total_grads, 5.0) + self.mutator_optim.apply_gradients( + zip(total_grads, self.mutator.trainable_weights) + ) + + def validate_one_epoch(self, epoch): + test_loader = self._create_validate_loader() + + for arc_id in range(self.test_arc_per_epoch): + meters = AverageMeterGroup() + for x, y in test_loader: + self.mutator.reset() + logits = self.model(x, training=False) + if isinstance(logits, tuple): + logits, _ = logits + metrics = self.metrics(y, logits) + loss = self.loss(y, logits) + metrics["loss"] = tf.reduce_mean(loss).numpy() + meters.update(metrics) + + logger.info( + "Test Epoch [%d/%d] Arc [%d/%d] Summary %s", + epoch + 1, + self.num_epochs, + arc_id + 1, + self.test_arc_per_epoch, + meters.summary(), + ) + + def _create_train_loader(self): + train_set = self.train_set.shuffle(1000000).repeat().batch(self.batch_size) + test_set = self.valid_set.shuffle(1000000).repeat().batch(self.batch_size) + return iter(train_set), iter(test_set) + + def _create_validate_loader(self): + return iter(self.test_set.shuffle(1000000).batch(self.batch_size)) diff --git a/new_impl/cv/third_party/nni_new/assessor.py b/new_impl/cv/third_party/nni_new/assessor.py new file mode 100644 index 0000000000000000000000000000000000000000..7cd83e9232735f5aee82e1abca702e347505f4cd --- /dev/null +++ b/new_impl/cv/third_party/nni_new/assessor.py @@ -0,0 +1,124 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +Assessor analyzes trial's intermediate results (e.g., periodically evaluated accuracy on test dataset) +to tell whether this trial can be early stopped or not. + +See :class:`Assessor`' specification and ``docs/en_US/assessors.rst`` for details. +""" + +from enum import Enum +import logging + +from .recoverable import Recoverable + +__all__ = ['AssessResult', 'Assessor'] + +_logger = logging.getLogger(__name__) + + +class AssessResult(Enum): + """ + Enum class for :meth:`Assessor.assess_trial` return value. + """ + + Good = True + """The trial works well.""" + + Bad = False + """The trial works poorly and should be early stopped.""" + + +class Assessor(Recoverable): + """ + Assessor analyzes trial's intermediate results (e.g., periodically evaluated accuracy on test dataset) + to tell whether this trial can be early stopped or not. + + This is the abstract base class for all assessors. + Early stopping algorithms should inherit this class and override :meth:`assess_trial` method, + which receives intermediate results from trials and give an assessing result. + + If :meth:`assess_trial` returns :obj:`AssessResult.Bad` for a trial, + it hints NNI framework that the trial is likely to result in a poor final accuracy, + and therefore should be killed to save resource. + + If an assessor want's to be notified when a trial ends, it can also override :meth:`trial_end`. + + To write a new assessor, you can reference :class:`~nni.medianstop_assessor.MedianstopAssessor`'s code as an example. + + See Also + -------- + Builtin assessors: + :class:`~nni.algorithms.hpo.medianstop_assessor.MedianstopAssessor` + :class:`~nni.algorithms.hpo.curvefitting_assessor.CurvefittingAssessor` + """ + + def assess_trial(self, trial_job_id, trial_history): + """ + Abstract method for determining whether a trial should be killed. Must override. + + The NNI framework has little guarantee on ``trial_history``. + This method is not guaranteed to be invoked for each time ``trial_history`` get updated. + It is also possible that a trial's history keeps updating after receiving a bad result. + And if the trial failed and retried, ``trial_history`` may be inconsistent with its previous value. + + The only guarantee is that ``trial_history`` is always growing. + It will not be empty and will always be longer than previous value. + + This is an example of how :meth:`assess_trial` get invoked sequentially: + + :: + + trial_job_id | trial_history | return value + ------------ | --------------- | ------------ + Trial_A | [1.0, 2.0] | Good + Trial_B | [1.5, 1.3] | Bad + Trial_B | [1.5, 1.3, 1.9] | Good + Trial_A | [0.9, 1.8, 2.3] | Good + + Parameters + ---------- + trial_job_id : str + Unique identifier of the trial. + trial_history : list + Intermediate results of this trial. The element type is decided by trial code. + + Returns + ------- + AssessResult + :obj:`AssessResult.Good` or :obj:`AssessResult.Bad`. + """ + raise NotImplementedError('Assessor: assess_trial not implemented') + + def trial_end(self, trial_job_id, success): + """ + Abstract method invoked when a trial is completed or terminated. Do nothing by default. + + Parameters + ---------- + trial_job_id : str + Unique identifier of the trial. + success : bool + True if the trial successfully completed; False if failed or terminated. + """ + + def load_checkpoint(self): + """ + Internal API under revising, not recommended for end users. + """ + checkpoin_path = self.get_checkpoint_path() + _logger.info('Load checkpoint ignored by assessor, checkpoint path: %s', checkpoin_path) + + def save_checkpoint(self): + """ + Internal API under revising, not recommended for end users. + """ + checkpoin_path = self.get_checkpoint_path() + _logger.info('Save checkpoint ignored by assessor, checkpoint path: %s', checkpoin_path) + + def _on_exit(self): + pass + + def _on_error(self): + pass diff --git a/new_impl/cv/third_party/nni_new/common/graph_utils.py b/new_impl/cv/third_party/nni_new/common/graph_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..43fab1da597e9b861dc5bf8dfd3e07661d334883 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/common/graph_utils.py @@ -0,0 +1,831 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + + +import logging +import queue +import re +from collections import defaultdict +import torch +from torch.utils.tensorboard._pytorch_graph import NodePy, NodePyIO, NodePyOP, GraphPy +CLASSTYPE_KIND = 'ClassType' +GETATTR_KIND = 'prim::GetAttr' +CAT_KIND = 'aten::cat' +LIST_CONSTRUCT_KIND = 'prim::ListConstruct' +LIST_UNPACK_KIND = 'prim::ListUnpack' +TUPLE_CONSTRUCT_KIND = 'prim::TupleConstruct' +TUPLE_UNPACK_KIND = 'prim::TupleUnpack' +CONSTANT_KIND = 'prim::Constant' + +_logger = logging.getLogger(__name__) + +def _dummy_debug(*args): + return None +_logger.debug = _dummy_debug + + +def build_module_graph(model, dummy_input): + return TorchModuleGraph(model, dummy_input) + + +def build_graph(model, dummy_input, verbose=False): + g = TorchProtoGraph(model, dummy_input, verbose) + return g.graph_def, g.stepstats + + +def parse_traced_name(module_name): + prefix = 'TracedModule[' + suffix = ']' + if module_name.startswith(prefix) and module_name.endswith(suffix): + module_name = module_name[len(prefix):-len(suffix)] + return module_name + + +class TorchGraph: + """ + This class is to extract pytorch model topology graph by tracing + """ + + def __init__(self, model=None, dummy_input=None, traced_model=None): + """ + Parameters + ---------- + model : pytorch model + The model user wants to speed up + dummy_input : pytorch tensor + The dummy input for ```jit.trace```, users should put it on right device before pass in + traced_model : torch._C.torch.jit.TopLevelTracedModule + An alredy traced model, if traced_model is not None, then TorchGraph will build the graph + based on this traced model and won't trace the model again. + """ + assert torch.__version__ >= '1.3.1' + # check if the input is legal + if traced_model is not None: + assert isinstance(traced_model, torch.jit.TopLevelTracedModule) + self.trace = traced_model + # it's ok if the graph is already unpacked + torch._C._jit_pass_inline(self.trace.graph) + elif model is not None and dummy_input is not None: + self.bound_model = model + self._trace(model, dummy_input) + else: + raise Exception( + 'Please provide model & dummy_input or the traced_model as inputs') + + def _trace(self, model, dummy_input): + training = model.training + model.eval() + self.trace = torch.jit.trace(model, dummy_input) + torch._C._jit_pass_inline(self.trace.graph) + model.train(training) + + +class TorchProtoGraph(TorchGraph): + """ + Generates model graph for pytorch models in protobuf, this implementation + is borrowed from pytorch v1.4.0, and fixed following issues: + https://github.com/pytorch/pytorch/issues/33691 + https://github.com/pytorch/pytorch/issues/33670 + + """ + + def __init__(self, model, dummy_input, verbose=False): + super().__init__(model, dummy_input) + + from tensorboard.compat.proto.config_pb2 import RunMetadata + from tensorboard.compat.proto.graph_pb2 import GraphDef + from tensorboard.compat.proto.step_stats_pb2 import StepStats, DeviceStepStats + from tensorboard.compat.proto.versions_pb2 import VersionDef + + list_of_nodes = self.parse(self.trace.graph, self.trace, dummy_input) + if verbose: + print(self.trace.graph) + self.stepstats = RunMetadata(step_stats=StepStats( + dev_stats=[DeviceStepStats(device="/device:CPU:0")])) + self.graph_def = GraphDef( + node=list_of_nodes, versions=VersionDef(producer=22)) + + def parse(self, graph, trace, args=None, omit_useless_nodes=True): + """This method parses an optimized PyTorch model graph and produces + a list of nodes and node stats for eventual conversion to TensorBoard + protobuf format. + + Args: + graph (PyTorch module): The model graph to be parsed. + trace (PyTorch JIT TracedModule): The model trace to be parsed. + args (tuple): input tensor[s] for the model. + omit_useless_nodes (boolean): Whether to remove nodes from the graph. + """ + nodes_py = GraphPy() + for node in graph.inputs(): + if omit_useless_nodes: + if not node.uses(): # number of user of the node (= number of outputs/ fanout) + continue + + if node.type().kind() != CLASSTYPE_KIND: + nodes_py.append(NodePyIO(node, 'input')) + + attr_to_scope = dict() + + def node_to_name(d): + return str(d).split(":")[0].strip() + for node in graph.nodes(): + if node.kind() == GETATTR_KIND: + attr_name = node.s('name') + node_name = node_to_name(node) + parent = node.input().node() + # If the parent node is not the top-level "self" node + if parent.kind() == GETATTR_KIND: + parent_scope = attr_to_scope[node_to_name(parent)] + attr_scope = parent_scope.split('/')[-1] + attr_to_scope[node_name] = '{}/{}.{}'.format( + parent_scope, attr_scope, attr_name) + else: + attr_to_scope[node_name] = '__module.{}'.format(attr_name) + # We don't need classtype nodes; scope will provide this information + if node.output().type().kind() != CLASSTYPE_KIND: + node_py = NodePyOP(node) + node_py.scopeName = attr_to_scope[node_name] + nodes_py.append(node_py) + else: + nodes_py.append(NodePyOP(node)) + + # Create sink nodes for output ops + for i, node in enumerate(graph.outputs()): + node_py = NodePyIO(node, 'output') + node_py.debugName = "output.{}".format(i + 1) + node_py.inputs = [node.debugName()] + nodes_py.append(node_py) + + alias_to_name = dict() + base_name = parse_traced_name(trace._name) + for name, module in trace.named_modules(prefix='__module'): + mod_name = parse_traced_name(module._name) + attr_name = name.split('.')[-1] + alias_to_name[name] = '{}[{}]'.format(mod_name, attr_name) + + for node in nodes_py.nodes_op: + module_aliases = node.scopeName.split('/')[-1].split('.') + module_name = '' + for i, alias in enumerate(module_aliases): + if i == 0: + module_name = alias + node.scopeName = base_name + else: + module_name += '.' + alias + node.scopeName += '/' + \ + (alias_to_name[module_name] + if module_name in alias_to_name else alias) + + nodes_py.populate_namespace_from_OP_to_IO() + return nodes_py.to_proto() + + +class NodePyGroup(NodePy): + """ + This class is used to represent a graph node which consists of multiple jit traced nodes. In a pytorch trace graph, + there are multiple nodes are traced for one torch.nn.Module object, we group them together to form a single node to + represent the torch.nn.Module object. We also group some functional call trace nodes together to form a new node. + """ + + def __init__(self, name, unique_name, node_type, op_type, node_cpps, inputs=None, outputs=None, key_node=None): + """ + Parameters: + ----------- + name: str + node name, such as `conv1`, `backbone.classifier` + unique_name: str + A global unique name for current node. Due to some modules, + such as relu, may be reused several times, so the scopename + is not suitable as the global unique identifier, so we add a + unique_name for each node as the global unique identifier. + We should use the unique_name to traverset the module graph. + node_type: str + `module` or `func` + op_type: str + operation type, such as `Conv2d`, `aten::view` + node_cpps: list of torch._C.Node + jit trace nodes which are included in this new node + inputs: list of str + All the inputs of this node, each element is debugName of one input + outputs: list of str + All the outputs of this node, each element is debugName of one output + key_node: torch._C.Node + The key node of this NodePyGroup. + """ + super(NodePyGroup, self).__init__(name, []) + self.node_cpps = node_cpps + self.name = name + self.unique_name = unique_name + self.op_type = op_type + self.type = node_type + self.nodes = [] + self.auxiliary = None + self.add_nodes(node_cpps) + self.inputs = inputs + self.outputs = outputs + # The core node in this NodePyGroup + self.key_node = key_node + + def add_nodes(self, node_cpps): + for node_cpp in node_cpps: + nodepy = NodePyOP(node_cpp) + nodepy.name = node_cpp.scopeName() + '_' + node_cpp.kind() + self.nodes.append(nodepy) + + def sub_node_names(self): + return [x.name for x in self.nodes] + + def __repr__(self): + return 'name: {}, type: {}, op_type: {}, sub_nodes: {}, inputs: {}, outputs: {}, aux: {}'.format( + self.name, self.type, self.op_type, self.sub_node_names(), + self.inputs, self.outputs, self.auxiliary + ) + + +class TorchModuleGraph(TorchGraph): + """ + Generates model graph, each node is created from single or multiple jit trace nodes. + """ + + def __init__(self, model=None, dummy_input=None, traced_model=None): + super().__init__(model, dummy_input, traced_model) + self.global_count = 0 + self.name_to_node, self.input_to_node, self.output_to_node = self._build_graph() + self._extract_auxiliary_info() + + def _expand_key_func_node(self, node, nodes, input_to_node, output_to_node, + module_type): + """ + For trace graph nodes, some nodes are not in modules, these nodes are usually generated by + the functions directly called in module ```forward```. For such nodes, some of them are + trivial op which are label by ```prim::```, some of them are not such ops which is call + non-prim ops. This function is to merge neighbor prim ops to a non-prim op, to construct + a node. + + Parameters + ---------- + node : trace graph node + The non-prim node to expand + nodes : list of trace graph node + All the trace graph nodes within the same scope as the non-prim node + input_to_node : dict + key: input name, value: a node that uses this input + output_to_node : dict + key: output name, value: a node that generates this output + module_type : str + can be 'module' or 'func' + + Returns + ------- + node + the expanded non-prim node + """ + # TODO: scope name could be empty + node_name = '.'.join([self._get_module_name( + node.scopeName()), node.kind(), str(self.global_count)]) + unique_name = node_name + _logger.debug("expand non-prim node, node name: %s", node_name) + self.global_count += 1 + op_type = node.kind() + node_group = [node] + inputs = [] + outputs = [] + node_queue = queue.Queue() + node_queue.put(node) + while not node_queue.empty(): + curr_node = node_queue.get() + for _input in curr_node.inputs(): + if _input.node().kind() == CONSTANT_KIND: + continue + input_name = _input.debugName() + if input_name in output_to_node: + for predecessor_node in output_to_node[input_name]: + if predecessor_node in nodes: + if not self._is_key_func(predecessor_node): + if predecessor_node not in node_group: + node_group.append(predecessor_node) + node_queue.put(predecessor_node) + else: + inputs.append(input_name) + else: + inputs.append(input_name) + else: + inputs.append(input_name) + for output in node.outputs(): + if output.node().kind() == CONSTANT_KIND: + continue + outputs.append(output.debugName()) + nodepy = NodePyGroup(node_name, unique_name, module_type, op_type, + node_group, inputs=inputs, outputs=outputs, key_node=node) + return nodepy + + def _expand_module_node(self, node, node_name, unique_name, op_type, nodes, + input_to_node, output_to_node, module_type): + """ + merge the adjacent nodes of the module. The difference between the + _expand_module_node and _expand_non_prim_node is that, the _expand_non_prim_node + only merge the prim:: nodes into the aten:: node, in contrast,the _expand_module_node + will merge all adjacent nodes into a same nodepy group. + + Parameters + ---------- + node : trace graph node + The non-prim node to expand + node_name : str + specify the node_name for NodePyGroup + unique_name : str + unique_name for the NodePyGroup + op_type : str + specify the op_type for the NodePyGroup + nodes : list of trace graph node + All the trace graph nodes within the same scope as the non-prim node + input_to_node : dict + key: input name, value: a node that uses this input + output_to_node : dict + key: output name, value: a node that generates this output + module_type : str + can be 'module' or 'func' + Returns + ------- + node + the expanded non-prim node + + """ + _logger.debug("expand module node, node name: %s", node_name) + self.global_count += 1 + if not op_type: + op_type = node.kind() + node_group = [node] + inputs = [] + outputs = [] + node_queue = queue.Queue() + node_queue.put(node) + visited = {node} + while not node_queue.empty(): + curr_node = node_queue.get() + for _input in curr_node.inputs(): + if _input.node().kind() == CONSTANT_KIND: + continue + input_name = _input.debugName() + if input_name in output_to_node: + for predecessor_node in output_to_node[input_name]: + if predecessor_node in nodes: + if predecessor_node not in visited: + node_group.append(predecessor_node) + node_queue.put(predecessor_node) + visited.add(predecessor_node) + else: + inputs.append(input_name) + else: + inputs.append(input_name) + for _output in curr_node.outputs(): + if _output.node().kind() == CONSTANT_KIND: + continue + output_name = _output.debugName() + if output_name in input_to_node: + for successor_node in input_to_node[output_name]: + if successor_node in nodes: + if successor_node not in visited: + node_group.append(successor_node) + node_queue.put(successor_node) + visited.add(successor_node) + else: + outputs.append(output_name) + else: + outputs.append(output_name) + + nodepy = NodePyGroup(node_name, unique_name, module_type, op_type, + node_group, inputs=list(inputs), outputs=list(outputs)) + return nodepy + + def _extract_cat_info(self, node_group, cpp_node): + """ + Extract the detail information of the cat operation, + such the order of the input tensor, the shape of each + input tensor, the output shape, and the cat dimension. + + Parameters + ---------- + node_group : NodePyGroup + cpp_node: torch._C.Node + It should be ```aten::cat``` node + + Returns + ------- + dict + Include auxiliary information for the cat operation. + This dict objec has four keys: 'cat_dim', 'out_shape', + 'in_order' and 'in_shape'. cat_dim is the dimension of + the cat operation to concat the input tensors. out_shape + is the shape of the output tensor of the cat operation. + in_order is an ordered list which contains the corresponding + parent operaion nodes of the input tensors. in_shape is also + an ordered list that contains the input shapes of the input + tensor. + """ + # only suport the cat operation + assert cpp_node.kind() == CAT_KIND + cat_info = {} + # get the shape of the output tensor + t_output = cpp_node.output() + out_shape = t_output.type().sizes() + cat_info['out_shape'] = out_shape + # get the cat dimension + inputs = cpp_node.inputs() + cat_dim = list(inputs)[1].toIValue() + cat_info['cat_dim'] = cat_dim + # get the order of the input tensors + # To get the order of the input tensors, we need + # to be aware of the topology of the model, which + # means we should extract the auxiliary information + # after the build_index function. + input_order = [] + list_construct_cpp = list(cpp_node.inputs())[0].node() + input_tensors = list(list_construct_cpp.inputs()) + for _tensor in input_tensors: + debug_name = _tensor.debugName() + if debug_name in self.output_to_node: + input_order.append(self.output_to_node[debug_name].unique_name) + else: + # the input tensor may be the input tensor of the whole model + input_order.append(None) + cat_info['in_order'] = input_order + input_shapes = [t.type().sizes() for t in input_tensors] + cat_info['in_shape'] = input_shapes + return cat_info + + def _extract_linear_shape_info(self, node_group): + """ + Extract linear shape input/output tensor shape info from its aten::addmm op. + + Parameters + ---------- + node_group : NodePyGroup + NodePyGroup object associated with the linear module. + + Returns + ------- + dict + Include shape of input tensor and shape of output tensor + """ + for cpp_node in node_group.node_cpps: + if cpp_node.kind() == 'aten::addmm': + # https://github.com/pytorch/pytorch/blob/1.6/torch/nn/functional.py#L1682 + # inputs of aten::addmm: + # inputs[0] is bias + # inputs[1] is input data + # inputs[2] is weight + t_input = list(cpp_node.inputs())[1] + t_output = cpp_node.output() + assert isinstance(t_input.type(), torch._C.TensorType) + assert isinstance(t_output.type(), torch._C.TensorType) + in_shape = t_input.type().sizes() + out_shape = t_output.type().sizes() + return {'in_shape': in_shape, 'out_shape': out_shape} + return None + + def _extract_shape_info(self, node): + """ + Extract the shape information of ```aten::view``` node + + Parameters + ---------- + node : trace graph node + It should be ```aten::view``` node + + Returns + ------- + dict + Include shape of input tensor and shape of output tensor + """ + t_input = None + for _input in node.inputs(): + t_input = _input + break + t_output = node.output() + assert isinstance(t_input.type(), torch._C.TensorType) + assert isinstance(t_output.type(), torch._C.TensorType) + in_shape = t_input.type().sizes() + out_shape = t_output.type().sizes() + return {'in_shape': in_shape, 'out_shape': out_shape} + + def _extract_leaf_modules(self): + """ + Extract leaf modules from the given graph. Leaf module means it does not have submodules. + To extract leaf modules because only leaf module can be replaced. And shape inference can + be done in leaf module level. Other shape inference is done in lower level i.e., + operation level. + + Returns + ------- + list + a list of scope name of all the leaf modules + """ + def is_parent(name1, name2): + """ + check if name1 is parent node of name2, for example: + name1: aa.bb, name2: aa.bb.cc, return True + name1: aa.b, name2: aa.bb, return False + """ + parts1, parts2 = name1.split('.'), name2.split('.') + if len(parts1) >= len(parts2): + return False + for i, _ in enumerate(parts1): + if parts2[i] != parts1[i]: + return False + return True + module_names = sorted([x[0] + for x in self.trace.named_modules() if x[0]]) + leaf_nodes = [] + for i, name in enumerate(module_names): + if i + 1 >= len(module_names) or not is_parent(name, module_names[i + 1]): + leaf_nodes.append(name) + return leaf_nodes + + def _get_module_name(self, scope_name): + """ + Retrieve module name from scope name. + Parameters: + ----------- + scope_name: str + scope_name of a graph node, for example: + for pytorch 1.3.1: MyModel/BackboneModel[backbone]/Conv2d[conv2] + for pytorch 1.4.0: __module.backbone/__module.backbone.conv2 + + Returns: + ------- + str + module name, such as backbone.conv2 + """ + if torch.__version__ >= '1.4.0': + return scope_name.split('/')[-1].replace('__module.', '') + else: + return '.'.join(re.findall(r'\[(.*?)\]', scope_name)) + + def _build_index(self, nodes_op): + name_to_node = dict() + input_to_node = defaultdict(list) + output_to_node = dict() + for node in nodes_op: + name_to_node[node.unique_name] = node + for _input in node.inputs: + # inputs may have duplicate tensors + if node not in input_to_node[_input]: + input_to_node[_input].append(node) + for output in node.outputs: + if output in output_to_node: + assert output_to_node[output] == node, \ + "One output cannot be generated by multiple nodes %s" % output + output_to_node[output] = node + return name_to_node, input_to_node, output_to_node + + def _is_key_func(self, node_cpp): + """ + Judge if a cpp node is a key function node. + If so, we should not merge this node into the + adjacent node. + """ + if node_cpp.kind().startswith('aten::'): + # the nodes that start with 'aten' are key function + # nodes + return True + if node_cpp.kind() in [LIST_UNPACK_KIND, TUPLE_UNPACK_KIND]: + # We cannot merge the List/Tuple + # Unpack func into other nodes, else it + # may lead to a graph construction error. + # The reason why we donnot take the construct node + # also as a key node is that `cat` operation node need + # the last(previous) visited node to infer the mask. If + # we take the Construct node as the important node, the + # predecessor of the `cat` node will always be a construct + # node, which means we cannot infer the mask for the cat + # operation. + return True + return False + + def unpack_manually(self): + """ + Unpack the tensor tuple or tensor list manually, + and remove the ListUnpack/TupleUnpack node from + the graph. Note: this function will change the + graph structure. + """ + if hasattr(self, 'unpacked'): + # if already unpacked the tuple/list manually + return + for node in self.nodes_py.nodes_op: + if node.op_type in [TUPLE_UNPACK_KIND, LIST_UNPACK_KIND]: + unpack_cpp = node.key_node + last_cpp = list(unpack_cpp.inputs())[0].node() + if last_cpp.kind() in [TUPLE_CONSTRUCT_KIND, LIST_CONSTRUCT_KIND]: + # we need check if the tensor tuple or tensor list is produced + # by a list/tuple construct node. If so, we can unpack the tuple + # or list manunally. + _logger.debug('List/Tuple Construct Node(cpp) %s', str(last_cpp)) + _logger.debug('List/Tuple Unpack Node(cpp) %s', str(unpack_cpp)) + assert len(list(unpack_cpp.outputs())) == len(list(last_cpp.inputs())) + errmsg = '%s Input number: %d if inconsistent with the output number %d' % (unpack_cpp, \ + len(node.inputs), len(list(last_cpp.inputs()))) + + assert len(node.inputs) == len(list(last_cpp.inputs())), errmsg + for _debug_input, _debug_output in zip(node.inputs, node.outputs): + if _debug_input in self.input_to_node and _debug_output in self.input_to_node: + # input_to_node[_debug_input] is a list of NodePyGroup, because + # one tensor can be used as input for multiple nodes at the same time. + + # note that, in this case, the construct cpp node and unpack cpp node + # will be merged into the same NodePyGroup, so we remove the `node` from + # input_to_node[_debug_input] and directly connect this tensor to the + # input_to_node[_debug_output] + if node in self.input_to_node[_debug_input]: + self.input_to_node[_debug_input].remove(node) + # add the following nodes of _output into the input_to_node[_debug_input] + self.input_to_node[_debug_input].extend(self.input_to_node[_debug_output]) + # just remove the _debug_output from the grapgh index. So that we can also skip + # the construct and tuple + if _debug_output in self.input_to_node: + for following_node in self.input_to_node[_debug_output]: + _tmp_index = following_node.inputs.index(_debug_output) + following_node.inputs[_tmp_index] = _debug_input + + + self.unpacked = True + + def _build_graph(self): + """ + Build graph using our defined format from jit trace. + There are basically three steps: first, construct necessary information (data structures), + second, extract all the modules to convert to node, Third, extract all functions to convert + to node. + + Returns + ------- + dict + use name to index nodes, key: node name, value: node + dict + use input (its name) to index nodes, + key: input, value: list of nodes that take this input + dict + use output (its name) to index nodes, + key: output, value: node that generates this output + """ + omit_useless_nodes = True + graph = self.trace.graph + _logger.debug(graph) + # build input/output mapping, from input/output debugName to its node + input_to_node = defaultdict(list) + output_to_node = defaultdict(list) + for node in graph.nodes(): + if node.kind() == CONSTANT_KIND: + continue + for x in node.outputs(): + if x.node().kind() == CONSTANT_KIND: + continue + output_to_node[x.debugName()].append(node) + assert len(output_to_node[x.debugName()]) <= 1, "One output cannot be generated by multiple nodes %s" % x.debugName() + for x in node.inputs(): + if x.node().kind() == CONSTANT_KIND: + continue + input_to_node[x.debugName()].append(node) + + # build module mapping, from module name to all nodes (as list) under this module scope + module_to_nodes = defaultdict(list) + # the mapping of function (non-module in forward) to nodes, key is scope name + func_to_nodes = defaultdict(list) + + nodes_py = GraphPy() + for node in graph.inputs(): + if omit_useless_nodes: + if not node.uses(): # number of user of the node (= number of outputs/ fanout) + continue + + if node.type().kind() != 'ClassType': + nodes_py.append(NodePyIO(node, 'input')) + + self.leaf_modules = self._extract_leaf_modules() + module_to_type = {name: parse_traced_name( + module._name) for name, module in self.trace.named_modules()} + + # associate module name with their trace graph nodes + for node in graph.nodes(): + if node.kind() == CONSTANT_KIND: + continue + module_name = self._get_module_name(node.scopeName()) + if module_name in self.leaf_modules: + module_to_nodes[module_name].append(node) + else: + func_to_nodes[node.scopeName()].append(node) + # build node group for module + for module_name, node_cpps in module_to_nodes.items(): + use_count = 0 + merged = set() + for node in node_cpps: + if node not in merged: + # modules that have same scope name may have different locations in the + # graph. Futhermore, there are also lots of prim:: nodes that in node_cpps, + # so we also need to call the expand_module_node. + unique_name = module_name + if use_count > 0: + unique_name = module_name + '.%d' % use_count + node_group = self._expand_module_node( + node, module_name, unique_name, module_to_type[module_name], + node_cpps, input_to_node, output_to_node, 'module') + nodes_py.nodes_op.append(node_group) + use_count += 1 + merged.update(node_group.node_cpps) + + # each scope_name may have multiple funcs, we split them and create node for each of them + # build node group for torch.nn.functional + for _, nodes in func_to_nodes.items(): + # extract non prim:: nodes + key_func_nodes = list() + for node in nodes: + if self._is_key_func(node): + # find the key function nodes + key_func_nodes.append(node) + # for each non prim node, expand it + for node in key_func_nodes: + node_group = self._expand_key_func_node( + node, nodes, input_to_node, output_to_node, 'func') + nodes_py.nodes_op.append(node_group) + # get shape infor for view (aten::view) func + # if node_group.op_type in ['aten::view', 'aten::flatten']: + # node_group.auxiliary = self._extract_shape_info(node) + + for node in graph.outputs(): # Create sink nodes for output ops + node_py = NodePyIO(node, 'output') + nodes_py.append(node_py) + + self.nodes_py = nodes_py + # build index + return self._build_index(self.nodes_py.nodes_op) + + def _extract_auxiliary_info(self): + """ + Extract the auxiliary information for the nodegroups + if necessary. For example, view/flatten operations may + need the shape of the input tensor and output tensor. + """ + # extract the input & output shape for the view and flatten + for node_group in self.nodes_py.nodes_op: + if node_group.op_type in ['aten::view', 'aten::flatten', 'aten::mean', 'aten::reshape']: + # get shape infor for view (aten::view) func + cpp_node = list(filter(lambda x: x.kind() == node_group.op_type, + node_group.node_cpps))[0] + node_group.auxiliary = self._extract_shape_info(cpp_node) + elif node_group.op_type == 'Linear': + node_group.auxiliary = self._extract_linear_shape_info(node_group) + elif node_group.op_type == CAT_KIND: + # get the detail information for cat func + cpp_node = list(filter(lambda x: x.kind() == node_group.op_type, + node_group.node_cpps))[0] + node_group.auxiliary = self._extract_cat_info( + node_group, cpp_node) + + def find_predecessors(self, unique_name): + """ + Find predecessor node of the given node + + Parameters + ---------- + unique_name : str + The unique name of the node + + Returns + ------- + list + a list of nodes who are the given node's predecessor + """ + predecessors = [] + for _input in self.name_to_node[unique_name].inputs: + if not _input in self.output_to_node: + _logger.debug("cannot find node with %s as its output", _input) + else: + node_py = self.output_to_node[_input] + predecessors.append(node_py.unique_name) + return predecessors + + def find_successors(self, unique_name): + """ + Find successor nodes of the given node + + Parameters + ---------- + unique_name : str + The unique name of the node + + Returns + ------- + list + a list of nodes who are the given node's successor + """ + successors = [] + for output in self.name_to_node[unique_name].outputs: + if output not in self.input_to_node: + # may reach the output of the whole graph + continue + nodes_py = self.input_to_node[output] + for node_py in nodes_py: + successors.append(node_py.unique_name) + return successors diff --git a/new_impl/cv/third_party/nni_new/common/nas_utils.py b/new_impl/cv/third_party/nni_new/common/nas_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..d7f050ec9bb2046b782bf699e1bd3ae7925a31a1 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/common/nas_utils.py @@ -0,0 +1,317 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import functools +import logging + +from .. import trial + + +_logger = logging.getLogger(__name__) +_MUTABLE_LAYER_SPACE_PREFIX = "_mutable_layer" +_namespace = {} +_tf_variables = {} +_arch_logits_list = [] +_optimizer = None +_train_op = None + + +def classic_mode( + mutable_id, + mutable_layer_id, + funcs, + funcs_args, + fixed_inputs, + optional_inputs, + optional_input_size): + '''Execute the chosen function and inputs directly. + In this mode, the trial code is only running the chosen subgraph (i.e., the chosen ops and inputs), + without touching the full model graph.''' + if trial.get_current_parameter() is None: + trial.get_next_parameter() + + chosen_layer, chosen_inputs = _get_layer_and_inputs_from_tuner(mutable_id, mutable_layer_id, + list(optional_inputs.keys())) + real_chosen_inputs = [optional_inputs[input_name] for input_name in chosen_inputs] + layer_out = funcs[chosen_layer]([fixed_inputs, real_chosen_inputs], **funcs_args[chosen_layer]) + + return layer_out + + +def enas_mode( + mutable_id, + mutable_layer_id, + funcs, + funcs_args, + fixed_inputs, + optional_inputs, + optional_input_size, + tf): + '''For enas mode, we build the full model graph in trial but only run a subgraph。 + This is implemented by masking inputs and branching ops. + Specifically, based on the received subgraph (through nni.get_next_parameter), + it can be known which inputs should be masked and which op should be executed.''' + name_prefix = "{}_{}".format(mutable_id, mutable_layer_id) + # store namespace + _namespace[mutable_id] = True + _namespace[name_prefix] = dict() + _namespace[name_prefix]['funcs'] = list(funcs) + _namespace[name_prefix]['optional_inputs'] = list(optional_inputs) + # create tensorflow variables as 1/0 signals used to form subgraph + name_for_optional_inputs = name_prefix + '_optional_inputs' + name_for_funcs = name_prefix + '_funcs' + _tf_variables[name_prefix] = dict() + _tf_variables[name_prefix]['optional_inputs'] = tf.get_variable( + name_for_optional_inputs, + [len(optional_inputs)], + dtype=tf.bool, + trainable=False + ) + _tf_variables[name_prefix]['funcs'] = tf.get_variable( + name_for_funcs, [], dtype=tf.int64, trainable=False) + + # get real values using their variable names + real_optional_inputs_value = [optional_inputs[name] + for name in _namespace[name_prefix]['optional_inputs']] + real_func_value = [funcs[name] + for name in _namespace[name_prefix]['funcs']] + real_funcs_args = [funcs_args[name] + for name in _namespace[name_prefix]['funcs']] + # build tensorflow graph of geting chosen inputs by masking + real_chosen_inputs = tf.boolean_mask( + real_optional_inputs_value, _tf_variables[name_prefix]['optional_inputs']) + # build tensorflow graph of different branches by using tf.case + branches = dict() + func_output = None + for func_id in range(len(funcs)): + func_output = real_func_value[func_id]([fixed_inputs, real_chosen_inputs], **real_funcs_args[func_id]) + branches[tf.equal(_tf_variables[name_prefix]['funcs'], func_id)] = lambda: func_output + layer_out = tf.case(branches, exclusive=True, default=lambda: func_output) + + return layer_out + + +def oneshot_mode( + mutable_id, + mutable_layer_id, + funcs, + funcs_args, + fixed_inputs, + optional_inputs, + optional_input_size, + tf): + '''Similar to enas mode, oneshot mode also builds the full model graph. + The difference is that oneshot mode does not receive subgraph. + Instead, it uses dropout to randomly dropout inputs and ops.''' + # NNI requires to get_next_parameter before report a result. But the parameter will not be used in this mode + if trial.get_current_parameter() is None: + trial.get_next_parameter() + optional_inputs = list(optional_inputs.values()) + inputs_num = len(optional_inputs) + # Calculate dropout rate according to the formular r^(1/k), where r is a hyper-parameter and k is the number of inputs + if inputs_num > 0: + rate = 0.01 ** (1 / inputs_num) + noise_shape = [inputs_num] + [1] * len(optional_inputs[0].get_shape()) + optional_inputs = tf.nn.dropout( + optional_inputs, rate=rate, noise_shape=noise_shape) + optional_inputs = [optional_inputs[idx] for idx in range(inputs_num)] + layer_outs = [func([fixed_inputs, optional_inputs], **funcs_args[func_name]) + for func_name, func in funcs.items()] + output_num = len(layer_outs) + rate = 0.01 ** (1 / output_num) + noise_shape = [output_num] + [1] * len(layer_outs[0].get_shape()) + layer_outs = tf.nn.dropout(layer_outs, rate=rate, noise_shape=noise_shape) + layer_out = tf.reduce_sum(layer_outs, axis=0) + + return layer_out + + +def darts_mode( + mutable_id, + mutable_layer_id, + funcs, + funcs_args, + fixed_inputs, + optional_inputs, + optional_input_size, + tf): + optional_inputs = list(optional_inputs.values()) + layer_outs = [func([fixed_inputs, optional_inputs], **funcs_args[func_name]) + for func_name, func in funcs.items()] + # Create architecture weights for every func(op) + var_name = "{}_{}_arch_weights".format(mutable_id, mutable_layer_id) + arch_logits = tf.get_variable(var_name, shape=[len(funcs)], trainable=False) + _arch_logits_list.append(arch_logits) + arch_weights = tf.nn.softmax(arch_logits) + layer_out = tf.add_n([arch_weights[idx] * out for idx, out in enumerate(layer_outs)]) + + return layer_out + + +def reload_tensorflow_variables(tf, session): + '''In Enas mode, this function reload every signal varaible created in `enas_mode` function so + the whole tensorflow graph will be changed into certain subgraph recerived from Tuner. + --------------- + session: the tensorflow session created by users + tf: tensorflow module + ''' + subgraph_from_tuner = trial.get_next_parameter() + mutable_layers = set() + for subgraph_key in subgraph_from_tuner: + if "/" in subgraph_key: + # has to remove the last, could be layer_choice or whatever + mutable_id, mutable_layer_id = _decompose_general_key(subgraph_key[:subgraph_key.rfind("/")]) + if mutable_id is not None: + mutable_layers.add((mutable_id, mutable_layer_id)) + mutable_layers = sorted(list(mutable_layers)) + for mutable_id, mutable_layer_id in mutable_layers: + if mutable_id not in _namespace: + _logger.warning("%s not found in name space", mutable_id) + continue + name_prefix = "{}_{}".format(mutable_id, mutable_layer_id) + # get optional inputs names + optional_inputs = _namespace[name_prefix]['optional_inputs'] + # extract layer information from the subgraph sampled by tuner + chosen_layer, chosen_inputs = _get_layer_and_inputs_from_tuner(mutable_id, mutable_layer_id, optional_inputs) + chosen_layer = _namespace[name_prefix]['funcs'].index(chosen_layer) + chosen_inputs = [1 if inp in chosen_inputs else 0 for inp in optional_inputs] + # load these information into pre-defined tensorflow variables + _tf_variables[name_prefix]['funcs'].load(chosen_layer, session) + _tf_variables[name_prefix]['optional_inputs'].load( + chosen_inputs, session) + + +def _construct_general_key(mutable_id, mutable_layer_id): + # Mutable layer key in a general (search space) format + # that is, prefix/mutable_id/mutable_layer_id + return _MUTABLE_LAYER_SPACE_PREFIX + "/" + mutable_id + "/" + mutable_layer_id + + +def _decompose_general_key(key): + # inverse operation of above + if not key.startswith(_MUTABLE_LAYER_SPACE_PREFIX): + return None, None + else: + _, mutable_id, mutable_layer_id = key.split("/", maxsplit=2) + return mutable_id, mutable_layer_id + + +def darts_training(tf, session, loss, feed_dict): + global _optimizer, _train_op + if _optimizer is None: + _optimizer = tf.MomentumOptimizer(learning_rate=0.025) + # TODO: Calculate loss + grads_and_vars = _optimizer.compute_gradients(loss, _arch_logits_list) + _train_op = _optimizer.apply_gradients(grads_and_vars) + session.run(_train_op) + + +def training_update(nas_mode, tf=None, session=None, loss=None, feed_dict=None): + if nas_mode == 'darts_mode': + darts_training(tf, session, loss, feed_dict) + elif nas_mode == 'enas_mode': + reload_tensorflow_variables(tf, session) + + +def _get_layer_and_inputs_from_tuner(mutable_id, mutable_layer_id, optional_inputs): + # optional_inputs should be name(key)s of the optional inputs + try: + mutable_block = trial.get_current_parameter(mutable_id) + + # There is a NAS tuner + chosen_layer = mutable_block[mutable_layer_id]["chosen_layer"] + chosen_inputs = mutable_block[mutable_layer_id]["chosen_inputs"] + except KeyError: + # Try to find converted NAS parameters + params = trial.get_current_parameter() + expected_prefix = _construct_general_key(mutable_id, mutable_layer_id) + chosen_layer = params[expected_prefix + "/layer_choice"] + + # find how many to choose + optional_input_size = int(params[expected_prefix + "/optional_input_size"]) # convert uniform to randint + + # find who to choose, can duplicate + optional_input_state = params[expected_prefix + "/optional_input_chosen_state"] + chosen_inputs = [] + # make sure dict -> list produce stable result by sorting + optional_inputs_keys = sorted(optional_inputs) + for _ in range(optional_input_size): + chosen_inputs.append(optional_inputs_keys[optional_input_state % len(optional_inputs)]) + optional_input_state //= len(optional_inputs) + + _logger.info("%s_%s: layer: %s, optional inputs: %s", mutable_id, mutable_layer_id, chosen_layer, chosen_inputs) + return chosen_layer, chosen_inputs + + +def convert_nas_search_space(search_space): + """ + Args: + param search_space: raw search space + return: the new search space, mutable_layers will be converted into choice + """ + if not isinstance(search_space, dict): + return search_space + ret = dict() + for k, v in search_space.items(): + if "_type" not in v: + # this should not happen + _logger.warning("There is no _type in one of your search space values with key '%s'" + ". Please check your search space", k) + ret[k] = v + elif v["_type"] != "mutable_layer": + ret[k] = v + else: + _logger.info("Converting mutable_layer search space with key '%s'", k) + # v["_value"] looks like {'mutable_layer_1': {'layer_choice': ...} ...} + values = v["_value"] + for layer_name, layer_data in values.items(): + # there should be at most layer_choice, optional_inputs, optional_input_size in layer_data + + # add "_mutable_layer" as prefix so that they can be recovered later + layer_key = _construct_general_key(k, layer_name) + + if layer_data.get("layer_choice"): # filter out empty choice and no choice + layer_choice = layer_data["layer_choice"] + else: + raise ValueError("No layer choice found in %s" % layer_key) + + if layer_data.get("optional_input_size"): + input_size = layer_data["optional_input_size"] + if isinstance(input_size, int): + input_size = [input_size, input_size] + if input_size[0] > input_size[1] or input_size[0] < 0: + _logger.error("Might not be able to handle optional_input_size < 0, please double check") + input_size[1] += 1 + else: + _logger.info("Optional input choices are set to empty by default in %s", layer_key) + input_size = [0, 1] + + if layer_data.get("optional_inputs"): + total_state_size = len(layer_data["optional_inputs"]) ** (input_size[1] - 1) + else: + _logger.info("Optional inputs not found in %s", layer_key) + total_state_size = 1 + + converted = { + layer_key + "/layer_choice": { + "_type": "choice", "_value": layer_choice + }, + layer_key + "/optional_input_size": { + "_type": "randint", "_value": input_size + }, + layer_key + "/optional_input_chosen_state": { + "_type": "randint", "_value": [0, total_state_size] + } + } + _logger.info(converted) + ret.update(converted) + + return ret + + +def rewrite_nas_space(func): + @functools.wraps(func) + def wrap(self, search_space): + search_space = convert_nas_search_space(search_space) + return func(self, search_space) + return wrap diff --git a/new_impl/cv/third_party/nni_new/compression/__init__.py b/new_impl/cv/third_party/nni_new/compression/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/new_impl/cv/third_party/nni_new/compression/pytorch/__init__.py b/new_impl/cv/third_party/nni_new/compression/pytorch/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..10e2fd050de5630d75be2432fc50aaf95cf86b40 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/compression/pytorch/__init__.py @@ -0,0 +1,6 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .speedup import ModelSpeedup +from .compressor import Compressor, Pruner, Quantizer +from .pruning import apply_compression_results diff --git a/new_impl/cv/third_party/nni_new/compression/pytorch/compressor.py b/new_impl/cv/third_party/nni_new/compression/pytorch/compressor.py new file mode 100644 index 0000000000000000000000000000000000000000..70dbd03f291eaebaf417ce8114545fd80a2ead74 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/compression/pytorch/compressor.py @@ -0,0 +1,799 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import types +import logging +import torch +from . import default_layers + +_logger = logging.getLogger(__name__) + +def _dummy_debug(*args): + return None +_logger.debug = _dummy_debug + +class LayerInfo: + def __init__(self, name, module): + self.module = module + self.name = name + self.type = type(module).__name__ + +def _setattr(model, name, module): + name_list = name.split(".") + for name in name_list[:-1]: + model = getattr(model, name) + setattr(model, name_list[-1], module) + +class Compressor: + """ + Abstract base PyTorch compressor + """ + + def __init__(self, model, config_list, optimizer=None): + """ + Record necessary info in class members + + Parameters + ---------- + model : pytorch model + the model user wants to compress + config_list : list + the configurations that users specify for compression + optimizer: pytorch optimizer + optimizer used to train the model + """ + assert isinstance(model, torch.nn.Module) + self.validate_config(model, config_list) + + self.bound_model = model + self.config_list = config_list + self.optimizer = optimizer + + self.modules_to_compress = None + self.modules_wrapper = [] + self.is_wrapped = False + + self._fwd_hook_handles = {} + self._fwd_hook_id = 0 + + self.reset() + + if not self.modules_wrapper: + _logger.warning('Nothing is configured to compress, please check your model and config_list') + + def validate_config(self, model, config_list): + """ + subclass can optionally implement this method to check if config_list if valid + """ + pass + + def reset(self, checkpoint=None): + """ + reset model state dict and model wrapper + """ + self._unwrap_model() + if checkpoint is not None: + self.bound_model.load_state_dict(checkpoint) + + self.modules_to_compress = None + self.modules_wrapper = [] + + for layer, config in self._detect_modules_to_compress(): + wrapper = self._wrap_modules(layer, config) + self.modules_wrapper.append(wrapper) + + self._wrap_model() + + def _detect_modules_to_compress(self): + """ + detect all modules should be compressed, and save the result in `self.modules_to_compress`. + The model will be instrumented and user should never edit it after calling this method. + """ + if self.modules_to_compress is None: + self.modules_to_compress = [] + for name, module in self.bound_model.named_modules(): + if module == self.bound_model: + continue + layer = LayerInfo(name, module) + config = self.select_config(layer) + if config is not None: + self.modules_to_compress.append((layer, config)) + return self.modules_to_compress + + def _wrap_model(self): + """ + wrap all modules that needed to be compressed + + """ + for wrapper in reversed(self.get_modules_wrapper()): + _setattr(self.bound_model, wrapper.name, wrapper) + self.is_wrapped = True + + def _unwrap_model(self): + """ + unwrap all modules that needed to be compressed + + """ + for wrapper in self.get_modules_wrapper(): + _setattr(self.bound_model, wrapper.name, wrapper.module) + self.is_wrapped = False + + def compress(self): + """ + Compress the model with algorithm implemented by subclass. + + The model will be instrumented and user should never edit it after calling this method. + `self.modules_to_compress` records all the to-be-compressed layers + + Returns + ------- + torch.nn.Module + model with specified modules compressed. + """ + return self.bound_model + + def set_wrappers_attribute(self, name, value): + """ + To register attributes used in wrapped module's forward method. + If the type of the value is Torch.tensor, then this value is registered as a buffer in wrapper, + which will be saved by model.state_dict. Otherwise, this value is just a regular variable in wrapper. + + Parameters + ---------- + name : str + name of the variable + value: any + value of the variable + """ + for wrapper in self.get_modules_wrapper(): + if isinstance(value, torch.Tensor): + wrapper.register_buffer(name, value.clone()) + else: + setattr(wrapper, name, value) + + def get_modules_to_compress(self): + """ + To obtain all the to-be-compressed modules. + + Returns + ------- + list + a list of the layers, each of which is a tuple (`layer`, `config`), + `layer` is `LayerInfo`, `config` is a `dict` + """ + return self.modules_to_compress + + def get_modules_wrapper(self): + """ + To obtain all the wrapped modules. + + Returns + ------- + list + a list of the wrapped modules + """ + return self.modules_wrapper + + def select_config(self, layer): + """ + Find the configuration for `layer` by parsing `self.config_list` + + Parameters + ---------- + layer : LayerInfo + one layer + + Returns + ------- + config or None + the retrieved configuration for this layer, if None, this layer should + not be compressed + """ + ret = None + for config in self.config_list: + config = config.copy() + # expand config if key `default` is in config['op_types'] + if 'op_types' in config and 'default' in config['op_types']: + expanded_op_types = [] + for op_type in config['op_types']: + if op_type == 'default': + expanded_op_types.extend(default_layers.weighted_modules) + else: + expanded_op_types.append(op_type) + config['op_types'] = expanded_op_types + + # check if condition is satisified + if 'op_types' in config and layer.type not in config['op_types']: + continue + if 'op_names' in config and layer.name not in config['op_names']: + continue + + ret = config + if ret is None or 'exclude' in ret: + return None + return ret + + def update_epoch(self, epoch): + """ + If user want to update model every epoch, user can override this method. + This method should be called at the beginning of each epoch + + Parameters + ---------- + epoch : num + the current epoch number + """ + pass + + def _wrap_modules(self, layer, config): + """ + This method is implemented in the subclasses, i.e., `Pruner` and `Quantizer` + + Parameters + ---------- + layer : LayerInfo + the layer to instrument the compression operation + config : dict + the configuration for compressing this layer + """ + raise NotImplementedError() + + def add_activation_collector(self, collector): + self._fwd_hook_id += 1 + self._fwd_hook_handles[self._fwd_hook_id] = [] + for wrapper in self.get_modules_wrapper(): + handle = wrapper.register_forward_hook(collector) + self._fwd_hook_handles[self._fwd_hook_id].append(handle) + return self._fwd_hook_id + + def remove_activation_collector(self, fwd_hook_id): + if fwd_hook_id not in self._fwd_hook_handles: + raise ValueError("%s is not a valid collector id" % str(fwd_hook_id)) + for handle in self._fwd_hook_handles[fwd_hook_id]: + handle.remove() + del self._fwd_hook_handles[fwd_hook_id] + + def patch_optimizer(self, *tasks): + def patch_step(old_step): + def new_step(_, *args, **kwargs): + # call origin optimizer step method + output = old_step(*args, **kwargs) + # calculate mask + for task in tasks: + task() + return output + return new_step + if self.optimizer is not None: + self.optimizer.step = types.MethodType(patch_step(self.optimizer.step), self.optimizer) + + def patch_optimizer_before(self, *tasks): + def patch_step(old_step): + def new_step(_, *args, **kwargs): + for task in tasks: + task() + # call origin optimizer step method + output = old_step(*args, **kwargs) + return output + return new_step + if self.optimizer is not None: + self.optimizer.step = types.MethodType(patch_step(self.optimizer.step), self.optimizer) + +class PrunerModuleWrapper(torch.nn.Module): + def __init__(self, module, module_name, module_type, config, pruner): + """ + Wrap an module to enable data parallel, forward method customization and buffer registeration. + + Parameters + ---------- + module : pytorch module + the module user wants to compress + config : dict + the configurations that users specify for compression + module_name : str + the name of the module to compress, wrapper module shares same name + module_type : str + the type of the module to compress + pruner ： Pruner + the pruner used to calculate mask + """ + super().__init__() + # origin layer information + self.module = module + self.name = module_name + self.type = module_type + # config and pruner + self.config = config + self.pruner = pruner + + # register buffer for mask + self.register_buffer("weight_mask", torch.ones(self.module.weight.shape)) + if hasattr(self.module, 'bias') and self.module.bias is not None: + self.register_buffer("bias_mask", torch.ones(self.module.bias.shape)) + else: + self.register_buffer("bias_mask", None) + + def forward(self, *inputs): + # apply mask to weight, bias + self.module.weight.data = self.module.weight.data.mul_(self.weight_mask) + if hasattr(self.module, 'bias') and self.module.bias is not None: + self.module.bias.data = self.module.bias.data.mul_(self.bias_mask) + return self.module(*inputs) + +class Pruner(Compressor): + """ + Prune to an exact pruning level specification + + Attributes + ---------- + mask_dict : dict + Dictionary for saving masks, `key` should be layer name and + `value` should be a tensor which has the same shape with layer's weight + + """ + + def __init__(self, model, config_list, optimizer=None): + super().__init__(model, config_list, optimizer) + + def compress(self): + self.update_mask() + return self.bound_model + + def update_mask(self): + for wrapper_idx, wrapper in enumerate(self.get_modules_wrapper()): + masks = self.calc_mask(wrapper, wrapper_idx=wrapper_idx) + if masks is not None: + for k in masks: + assert hasattr(wrapper, k), "there is no attribute '%s' in wrapper" % k + setattr(wrapper, k, masks[k]) + + def calc_mask(self, wrapper, **kwargs): + """ + Pruners should overload this method to provide mask for weight tensors. + The mask must have the same shape and type comparing to the weight. + It will be applied with `mul()` operation on the weight. + This method is effectively hooked to `forward()` method of the model. + + Parameters + ---------- + wrapper : Module + calculate mask for `wrapper.module`'s weight + """ + raise NotImplementedError("Pruners must overload calc_mask()") + + def _wrap_modules(self, layer, config): + """ + Create a wrapper module to replace the original one. + + Parameters + ---------- + layer : LayerInfo + the layer to instrument the mask + config : dict + the configuration for generating the mask + """ + _logger.debug("Module detected to compress : %s.", layer.name) + wrapper = PrunerModuleWrapper(layer.module, layer.name, layer.type, config, self) + assert hasattr(layer.module, 'weight'), "module %s does not have 'weight' attribute" % layer.name + # move newly registered buffers to the same device of weight + wrapper.to(layer.module.weight.device) + return wrapper + + def export_model(self, model_path, mask_path=None, onnx_path=None, input_shape=None, device=None): + """ + Export pruned model weights, masks and onnx model(optional) + + Parameters + ---------- + model_path : str + path to save pruned model state_dict + mask_path : str + (optional) path to save mask dict + onnx_path : str + (optional) path to save onnx model + input_shape : list or tuple + input shape to onnx model + device : torch.device + device of the model, used to place the dummy input tensor for exporting onnx file. + the tensor is placed on cpu if ```device``` is None + """ + assert model_path is not None, 'model_path must be specified' + mask_dict = {} + self._unwrap_model() # used for generating correct state_dict name without wrapper state + + for wrapper in self.get_modules_wrapper(): + weight_mask = wrapper.weight_mask + bias_mask = wrapper.bias_mask + if weight_mask is not None: + mask_sum = weight_mask.sum().item() + mask_num = weight_mask.numel() + _logger.debug('Layer: %s Sparsity: %.4f', wrapper.name, 1 - mask_sum / mask_num) + wrapper.module.weight.data = wrapper.module.weight.data.mul(weight_mask) + if bias_mask is not None: + wrapper.module.bias.data = wrapper.module.bias.data.mul(bias_mask) + # save mask to dict + mask_dict[wrapper.name] = {"weight": weight_mask, "bias": bias_mask} + + torch.save(self.bound_model.state_dict(), model_path) + _logger.info('Model state_dict saved to %s', model_path) + if mask_path is not None: + torch.save(mask_dict, mask_path) + _logger.info('Mask dict saved to %s', mask_path) + if onnx_path is not None: + assert input_shape is not None, 'input_shape must be specified to export onnx model' + # input info needed + if device is None: + device = torch.device('cpu') + input_data = torch.Tensor(*input_shape) + torch.onnx.export(self.bound_model, input_data.to(device), onnx_path) + _logger.info('Model in onnx with input shape %s saved to %s', input_data.shape, onnx_path) + + self._wrap_model() + + def load_model_state_dict(self, model_state): + """ + Load the state dict saved from unwrapped model. + + Parameters + ---------- + model_state : dict + state dict saved from unwrapped model + """ + if self.is_wrapped: + self._unwrap_model() + self.bound_model.load_state_dict(model_state) + self._wrap_model() + else: + self.bound_model.load_state_dict(model_state) + + def get_pruned_weights(self, dim=0): + """ + Log the simulated prune sparsity. + + Parameters + ---------- + dim : int + the pruned dim. + """ + for _, wrapper in enumerate(self.get_modules_wrapper()): + weight_mask = wrapper.weight_mask + mask_size = weight_mask.size() + if len(mask_size) == 1: + index = torch.nonzero(weight_mask.abs() != 0).tolist() + else: + sum_idx = list(range(len(mask_size))) + sum_idx.remove(dim) + index = torch.nonzero(weight_mask.abs().sum(sum_idx) != 0).tolist() + _logger.info(f'simulated prune {wrapper.name} remain/total: {len(index)}/{weight_mask.size(dim)}') + + +class QuantizerModuleWrapper(torch.nn.Module): + def __init__(self, module, module_name, module_type, config, quantizer): + """ + Wrap an module to enable data parallel, forward method customization and buffer registeration. + + Parameters + ---------- + module : pytorch module + the module user wants to compress + config : dict + the configurations that users specify for compression + module_name : str + the name of the module to compress, wrapper module shares same name + module_type : str + the type of the module to compress + quantizer ：quantizer + the quantizer used to calculate mask + """ + super().__init__() + # origin layer information + self.module = module + self.name = module_name + self.type = module_type + # config and pruner + self.config = config + self.quantizer = quantizer + + # register buffer and parameter + # old_weight is used to store origin weight and weight is used to store quantized weight + # the reason why weight is buffer instead of parameter is because in pytorch parameter is used as leaf + # if weight is leaf , then old_weight can not be updated. + if 'weight' in config['quant_types']: + if not _check_weight(self.module): + _logger.warning('Module %s does not have parameter "weight"', self.name) + else: + self.module.register_parameter('old_weight', torch.nn.Parameter(self.module.weight)) + delattr(self.module, 'weight') + self.module.register_buffer('weight', self.module.old_weight) + + def forward(self, *inputs): + if 'input' in self.config['quant_types']: + inputs = self.quantizer.quant_grad( + inputs, + QuantType.QUANT_INPUT, + self) + + if 'weight' in self.config['quant_types'] and _check_weight(self.module): + self.quantizer.quant_grad( + self.module.old_weight, + QuantType.QUANT_WEIGHT, + self, inputs[0]) + result = self.module(*inputs) + else: + result = self.module(*inputs) + + if 'output' in self.config['quant_types']: + result = self.quantizer.quant_grad( + result, + QuantType.QUANT_OUTPUT, + self) + return result + + +class Quantizer(Compressor): + """ + Base quantizer for pytorch quantizer + """ + + def __init__(self, model, config_list, optimizer=None): + super().__init__(model, config_list, optimizer) + self.quant_grad = QuantGrad.apply + if self.optimizer is not None: + self.patch_optimizer(self.step_with_optimizer) + for wrapper in self.get_modules_wrapper(): + if 'weight' in wrapper.config['quant_types']: + # old_weight is registered to keep track of weight before quantization + # and it is trainable, therefore, it should be added to optimizer. + self.optimizer.add_param_group({"params": wrapper.module.old_weight}) + + def quantize_weight(self, wrapper, **kwargs): + """ + quantize should overload this method to quantize weight. + This method is effectively hooked to :meth:`forward` of the model. + Parameters + ---------- + wrapper : QuantizerModuleWrapper + the wrapper for origin module + """ + raise NotImplementedError('Quantizer must overload quantize_weight()') + + def quantize_output(self, output, wrapper, **kwargs): + """ + quantize should overload this method to quantize output. + This method is effectively hooked to :meth:`forward` of the model. + Parameters + ---------- + output : Tensor + output that needs to be quantized + wrapper : QuantizerModuleWrapper + the wrapper for origin module + """ + raise NotImplementedError('Quantizer must overload quantize_output()') + + def quantize_input(self, *inputs, wrapper, **kwargs): + """ + quantize should overload this method to quantize input. + This method is effectively hooked to :meth:`forward` of the model. + Parameters + ---------- + inputs : Tensor + inputs that needs to be quantized + wrapper : QuantizerModuleWrapper + the wrapper for origin module + """ + raise NotImplementedError('Quantizer must overload quantize_input()') + + def _wrap_modules(self, layer, config): + """ + Create a wrapper forward function to replace the original one. + Parameters + ---------- + layer : LayerInfo + the layer to instrument the mask + config : dict + the configuration for quantization + """ + assert 'quant_types' in config, 'must provide quant_types in config' + assert isinstance(config['quant_types'], list), 'quant_types must be list type' + assert 'quant_bits' in config, 'must provide quant_bits in config' + assert isinstance(config['quant_bits'], int) or isinstance(config['quant_bits'], dict), 'quant_bits must be dict type or int type' + + if isinstance(config['quant_bits'], dict): + for quant_type in config['quant_types']: + assert quant_type in config['quant_bits'], 'bits length for %s must be specified in quant_bits dict' % quant_type + + return QuantizerModuleWrapper(layer.module, layer.name, layer.type, config, self) + + def export_model_save(self, model, model_path, calibration_config=None, calibration_path=None, onnx_path=None, + input_shape=None, device=None): + """ + This method helps save pytorch model, calibration config, onnx model in quantizer. + + Parameters + ---------- + model : pytorch model + pytorch model to be saved + model_path : str + path to save pytorch + calibration_config: dict + (optional) config of calibration parameters + calibration_path : str + (optional) path to save quantize parameters after calibration + onnx_path : str + (optional) path to save onnx model + input_shape : list or tuple + input shape to onnx model + device : torch.device + device of the model, used to place the dummy input tensor for exporting onnx file. + the tensor is placed on cpu if ```device``` is None + """ + torch.save(model.state_dict(), model_path) + _logger.info('Model state_dict saved to %s', model_path) + if calibration_path is not None: + torch.save(calibration_config, calibration_path) + _logger.info('Mask dict saved to %s', calibration_path) + if onnx_path is not None: + assert input_shape is not None, 'input_shape must be specified to export onnx model' + # input info needed + if device is None: + device = torch.device('cpu') + input_data = torch.Tensor(*input_shape) + torch.onnx.export(self.bound_model, input_data.to(device), onnx_path) + _logger.info('Model in onnx with input shape %s saved to %s', input_data.shape, onnx_path) + + def export_model(self, model_path, calibration_path=None, onnx_path=None, input_shape=None, device=None): + """ + Export quantized model weights and calibration parameters + + Parameters + ---------- + model_path : str + path to save quantized model weight + calibration_path : str + (optional) path to save quantize parameters after calibration + onnx_path : str + (optional) path to save onnx model + input_shape : list or tuple + input shape to onnx model + device : torch.device + device of the model, used to place the dummy input tensor for exporting onnx file. + the tensor is placed on cpu if ```device``` is None + + Returns + ------- + Dict + """ + raise NotImplementedError('Quantizer must overload export_model()') + + def step_with_optimizer(self): + pass + +class QuantType: + """ + Enum class for quantization type. + """ + QUANT_INPUT = 0 + QUANT_WEIGHT = 1 + QUANT_OUTPUT = 2 + +QType_Dict = { + 0: "input", + 1: "weight", + 2: "output" +} + +class QuantGrad(torch.autograd.Function): + """ + Base class for overriding backward function of quantization operation. + """ + @classmethod + def _quantize(cls, x, scale, zero_point): + """ + Reference function for quantizing x -- non-clamped. + Parameters + ---------- + x : Tensor + tensor to be quantized + scale : Tensor + scale for quantizing x + zero_point : Tensor + zero_point for quantizing x + Returns + ------- + tensor + quantized x without clamped + """ + return ((x / scale) + zero_point).round() + + @classmethod + def get_bits_length(cls, config, quant_type): + """ + Get bit for quantize config + Parameters + ---------- + config : Dict + the configuration for quantization + quant_type : str + quant type + Returns + ------- + int + n-bits for quantization configuration + """ + if isinstance(config["quant_bits"], int): + return config["quant_bits"] + else: + return config["quant_bits"].get(quant_type) + + @staticmethod + def quant_backward(tensor, grad_output, quant_type, scale, zero_point, qmin, qmax): + """ + This method should be overrided by subclass to provide customized backward function, + default implementation is Straight-Through Estimator + Parameters + ---------- + tensor : Tensor + input of quantization operation + grad_output : Tensor + gradient of the output of quantization operation + scale : Tensor + the type of quantization, it can be `QuantType.QUANT_INPUT`, `QuantType.QUANT_WEIGHT`, + `QuantType.QUANT_OUTPUT`, you can define different behavior for different types. + zero_point : Tensor + zero_point for quantizing tensor + qmin : Tensor + quant_min for quantizing tensor + qmax : Tensor + quant_max for quantizng tensor + Returns + ------- + tensor + gradient of the input of quantization operation + """ + return grad_output + + @staticmethod + def forward(ctx, tensor, quant_type, wrapper, input_tensor=None, **kwargs): + output = quantize_helper(tensor, quant_type, wrapper, input_tensor, **kwargs) + + bits = QuantGrad.get_bits_length(wrapper.config, QType_Dict[quant_type]) + qmin, qmax = torch.Tensor([0]).to(tensor.device), torch.Tensor([(1 << bits) - 1]).to(tensor.device) + if hasattr(wrapper.module, 'scale') and hasattr(wrapper.module, 'zero_point'): + scale = wrapper.module.scale + zero_point = wrapper.module.zero_point + else: + scale, zero_point = None, None + ctx.save_for_backward(tensor, torch.Tensor([quant_type]), scale, zero_point, qmin, qmax) + return output + + @classmethod + def backward(cls, ctx, grad_output): + tensor, quant_type, scale, zero_point, qmin, qmax = ctx.saved_variables + output = cls.quant_backward(tensor, grad_output, quant_type, scale, zero_point, qmin, qmax) + return output, None, None, None + +def _check_weight(module): + try: + return isinstance(module.weight.data, torch.Tensor) + except AttributeError: + return False + +def quantize_helper(tensor, quant_type, wrapper, input_tensor=None, **kwargs): + if quant_type == QuantType.QUANT_INPUT: + output = wrapper.quantizer.quantize_input(*tensor, wrapper=wrapper, **kwargs) + elif quant_type == QuantType.QUANT_WEIGHT: + output = wrapper.quantizer.quantize_weight(wrapper, input_tensor=input_tensor, **kwargs) + elif quant_type == QuantType.QUANT_OUTPUT: + output = wrapper.quantizer.quantize_output(tensor, wrapper, **kwargs) + else: + raise ValueError("unrecognized QuantType.") + + return output + +class QuantForward(torch.nn.Module): + """ + Base class for executing quantization operations. This is for quantization algorithms + that do not need to customize gradient. + """ + + def forward(self, tensor, quant_type, wrapper, input_tensor=None, **kwargs): + return quantize_helper(tensor, quant_type, wrapper, input_tensor, **kwargs) diff --git a/new_impl/cv/third_party/nni_new/compression/pytorch/default_layers.py b/new_impl/cv/third_party/nni_new/compression/pytorch/default_layers.py new file mode 100644 index 0000000000000000000000000000000000000000..4d7e6d8aed84ad76c9404f301117fb8a00d9a570 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/compression/pytorch/default_layers.py @@ -0,0 +1,9 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +weighted_modules = [ + 'Conv1d', 'Conv2d', 'Conv3d', 'ConvTranspose1d', 'ConvTranspose2d', 'ConvTranspose3d', + 'Linear', 'Bilinear', + 'PReLU', + 'Embedding', 'EmbeddingBag', +] diff --git a/new_impl/cv/third_party/nni_new/compression/pytorch/pruning/__init__.py b/new_impl/cv/third_party/nni_new/compression/pytorch/pruning/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9d3a7d2ca90a76918e1a89508f007903d1bb6485 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/compression/pytorch/pruning/__init__.py @@ -0,0 +1,4 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .apply_compression import apply_compression_results diff --git a/new_impl/cv/third_party/nni_new/compression/pytorch/pruning/apply_compression.py b/new_impl/cv/third_party/nni_new/compression/pytorch/pruning/apply_compression.py new file mode 100644 index 0000000000000000000000000000000000000000..8e6b023f5b90b8483e21bd0bc575b19b4a4df023 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/compression/pytorch/pruning/apply_compression.py @@ -0,0 +1,29 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +import torch + +logger = logging.getLogger('torch apply compression') + +def apply_compression_results(model, masks_file, map_location=None): + """ + Apply the masks from ```masks_file``` to the model + Note: this API is for inference, because it simply multiplies weights with + corresponding masks when this API is called. + + Parameters + ---------- + model : torch.nn.Module + The model to be compressed + masks_file : str + The path of the mask file + map_location : str + the device on which masks are placed, same to map_location in ```torch.load``` + """ + masks = torch.load(masks_file, map_location) + for name, module in model.named_modules(): + if name in masks: + module.weight.data = module.weight.data.mul_(masks[name]['weight']) + if hasattr(module, 'bias') and module.bias is not None and 'bias' in masks[name]: + module.bias.data = module.bias.data.mul_(masks[name]['bias']) \ No newline at end of file diff --git a/new_impl/cv/third_party/nni_new/compression/pytorch/quantization_speedup/__init__.py b/new_impl/cv/third_party/nni_new/compression/pytorch/quantization_speedup/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..636c82a5b045d39281064b80c77cfa60489eb6af --- /dev/null +++ b/new_impl/cv/third_party/nni_new/compression/pytorch/quantization_speedup/__init__.py @@ -0,0 +1 @@ +from .integrated_tensorrt import CalibrateType, ModelSpeedupTensorRT \ No newline at end of file diff --git a/new_impl/cv/third_party/nni_new/compression/pytorch/quantization_speedup/backend.py b/new_impl/cv/third_party/nni_new/compression/pytorch/quantization_speedup/backend.py new file mode 100644 index 0000000000000000000000000000000000000000..7d139d48f84c10557e3c83d2a858da707fe31227 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/compression/pytorch/quantization_speedup/backend.py @@ -0,0 +1,51 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +class BaseModelSpeedup: + """ + Base speedup class for backend engine + """ + def __init__(self, model, config): + """ + Parameters + ---------- + model : pytorch model + The model to speed up by quantization. + config : dict + Config recording bit number and name of layers. + """ + self.model = model + self.config = config + + def inference(self, test_data): + """ + This function should be overrided by subclass to provide inference ability, + which should return output and inference time. + + Parameters + ---------- + test_data : numpy data + test data given to the inference engine + + Returns + ------- + numpy data + output data will be generated after inference + float + latency of such inference process + """ + raise NotImplementedError('Backend engine must overload inference()') + + def compress(self): + """ + This function should be overrided by subclass to build inference + engine which will be used to process input data + """ + raise NotImplementedError('Backend engine must overload compress()') + + def export_quantized_model(self, path): + """ + This function should be overrided by subclass to build inference + engine which will be used to process input data + """ + raise NotImplementedError('Backend engine must overload export_quantized_model()') \ No newline at end of file diff --git a/new_impl/cv/third_party/nni_new/compression/pytorch/quantization_speedup/calibrator.py b/new_impl/cv/third_party/nni_new/compression/pytorch/quantization_speedup/calibrator.py new file mode 100644 index 0000000000000000000000000000000000000000..6bc49622f2e8cb1e818149ec06039bf9ab305c33 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/compression/pytorch/quantization_speedup/calibrator.py @@ -0,0 +1,99 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +import logging +import tensorrt as trt +import pycuda.driver as cuda + +logger = logging.getLogger(__name__) + +class Calibrator(trt.IInt8Calibrator): + def __init__(self, training_data, cache_file, batch_size=64, algorithm=trt.CalibrationAlgoType.ENTROPY_CALIBRATION_2): + """ + Parameters + ---------- + training_data : numpy array + The data using to calibrate quantization model + cache_file : str + The path user want to store calibrate cache file + batch_size : int + The batch_size of calibrating process + algorithm : tensorrt.tensorrt.CalibrationAlgoType + The algorithms of calibrating contains LEGACY_CALIBRATION, + ENTROPY_CALIBRATION, ENTROPY_CALIBRATION_2, MINMAX_CALIBRATION. + Please refer to https://docs.nvidia.com/deeplearning/tensorrt/api/ + python_api/infer/Int8/Calibrator.html for detail + """ + trt.IInt8Calibrator.__init__(self) + + self.algorithm = algorithm + self.cache_file = cache_file + + self.data = training_data + self.batch_size = batch_size + self.current_index = 0 + + # Allocate enough memory for a whole batch. + self.device_input = cuda.mem_alloc(self.data[0].nbytes * self.batch_size) + + def get_algorithm(self): + return self.algorithm + + def get_batch_size(self): + return self.batch_size + + def get_batch(self, names): + """ + This function is used to define the way of feeding calibrating data each batch. + + Parameters + ---------- + names : str + The names of the network inputs for each object in the bindings array + + Returns + ------- + list + A list of device memory pointers set to the memory containing each network + input data, or an empty list if there are no more batches for calibration. + You can allocate these device buffers with pycuda, for example, and then + cast them to int to retrieve the pointer + """ + if self.current_index + self.batch_size > self.data.shape[0]: + return None + + current_batch = int(self.current_index / self.batch_size) + if current_batch % 10 == 0: + logger.info("Calibrating batch %d, containing %d images", current_batch, self.batch_size) + + batch = self.data[self.current_index:self.current_index + self.batch_size].ravel() + cuda.memcpy_htod(self.device_input, batch) + self.current_index += self.batch_size + memory_pointers = [self.device_input] + return memory_pointers + + def read_calibration_cache(self): + """ + If there is a cache, use it instead of calibrating again. Otherwise, implicitly return None. + + Returns + ------- + cache object + A cache object which contains calibration parameters for quantization + """ + if os.path.exists(self.cache_file): + with open(self.cache_file, "rb") as f: + return f.read() + + def write_calibration_cache(self, cache): + """ + Write calibration cache to specific path. + + Parameters + ---------- + cache : str + The calibration cache to write + """ + with open(self.cache_file, "wb") as f: + f.write(cache) \ No newline at end of file diff --git a/new_impl/cv/third_party/nni_new/compression/pytorch/quantization_speedup/frontend_to_onnx.py b/new_impl/cv/third_party/nni_new/compression/pytorch/quantization_speedup/frontend_to_onnx.py new file mode 100644 index 0000000000000000000000000000000000000000..2bbb9f17e112e2d09a3bfcef19822172b0e3efed --- /dev/null +++ b/new_impl/cv/third_party/nni_new/compression/pytorch/quantization_speedup/frontend_to_onnx.py @@ -0,0 +1,148 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import torch +import onnx +import onnx.numpy_helper +""" +The main function of this page is to convert pytorch model to onnx model. +Convertion from pytorch model to onnx model is primary so that a critical +problem is caused that Layer name of pytorch model fail to convert to onnx +layer name directly. To solve it, we wrap pytorch model in new wrapper which +multiply bit number and input before computation of each op. Only in this +way can onnx model get bit number of corresponded layer. +""" + +class LayernameModuleWrapper(torch.nn.Module): + def __init__(self, module, module_bit) -> None: + """ + Parameters + ---------- + module : torch.nn.Module + Layer module of pytorch model + module_bit : int + Bit width setting for module + """ + super().__init__() + self.module = module + self.module_bit = module_bit + + def forward(self, inputs): + inputs = inputs*self.module_bit + inputs = self.module(inputs) + return inputs + +def _setattr(model, name, module): + """ + Parameters + ---------- + model : pytorch model + The model to speed up by quantization + name : str + name of pytorch module + module : torch.nn.Module + Layer module of pytorch model + """ + name_list = name.split(".") + for name in name_list[:-1]: + model = getattr(model, name) + setattr(model, name_list[-1], module) + +def unwrapper(model_onnx, index2name, config): + """ + Fill onnx config and remove wrapper node in onnx + + Parameters + ---------- + model_onnx : onnx model + Onnx model which is converted from pytorch model + index2name : dict + Dictionary of layer index and name + config : dict + Config recording name of layers and calibration parameters + + Returns + ------- + onnx model + Onnx model which is converted from pytorch model + dict + The configuration of onnx model layers and calibration parameters + """ + # Support Gemm, Conv, Relu, Clip(Relu6) and Maxpool + support_op = ['Gemm', 'Conv', 'Relu', 'Clip', 'MaxP'] + idx = 0 + onnx_config = {} + while idx < len(model_onnx.graph.node): + nd = model_onnx.graph.node[idx] + if nd.name[0:4] in support_op and idx > 1: + # Grad constant node and multiply node + const_nd = model_onnx.graph.node[idx-2] + mul_nd = model_onnx.graph.node[idx-1] + # Get index number which is transferred by constant node + index = int(onnx.numpy_helper.to_array(const_nd.attribute[0].t)) + if index != -1: + name = index2name[index] + onnx_config[nd.name] = config[name] + nd.input[0] = mul_nd.input[0] + # Remove constant node and multiply node + model_onnx.graph.node.remove(const_nd) + model_onnx.graph.node.remove(mul_nd) + idx = idx-2 + idx = idx+1 + return model_onnx, onnx_config + +def torch_to_onnx(model, config, input_shape, model_path, input_names, output_names): + """ + Convert torch model to onnx model and get layer bit config of onnx model. + + Parameters + ---------- + model : pytorch model + The model to speed up by quantization + config : dict + Config recording bit number and name of layers + input_shape : tuple + The input shape of model, shall pass it to torch.onnx.export + model_path : str + The path user want to store onnx model which is converted from pytorch model + input_names : list + Input name of onnx model providing for torch.onnx.export to generate onnx model + output_name : list + Output name of onnx model providing for torch.onnx.export to generate onnx model + + Returns + ------- + onnx model + Onnx model which is converted from pytorch model + dict + The configuration of onnx model layers and calibration parameters + """ + # Support Gemm, Conv, Relu, Clip(Relu6) and MaxPool + support_op = [torch.nn.Conv2d, torch.nn.Linear, torch.nn.ReLU, torch.nn.ReLU6, torch.nn.MaxPool2d] + # Transfer bit number to onnx layer by using wrapper + index2name = {} + name2index = {} + if config is not None: + for i, name in enumerate(config.keys()): + index2name[i] = name + name2index[name] = i + for name, module in model.named_modules(): + if config is not None and name in config: + assert type(module) in support_op + wrapper_module = LayernameModuleWrapper(module, name2index[name]) + _setattr(model, name, wrapper_module) + elif type(module) in support_op: + wrapper_module = LayernameModuleWrapper(module, -1) + _setattr(model, name, wrapper_module) + # Convert torch model to onnx model and save it in model_path + dummy_input = torch.randn(input_shape) + model.to('cpu') + torch.onnx.export(model, dummy_input, model_path, verbose=False, input_names=input_names, output_names=output_names, export_params=True) + + # Load onnx model + model_onnx = onnx.load(model_path) + model_onnx, onnx_config = unwrapper(model_onnx, index2name, config) + onnx.save(model_onnx, model_path) + + onnx.checker.check_model(model_onnx) + return model_onnx, onnx_config \ No newline at end of file diff --git a/new_impl/cv/third_party/nni_new/compression/pytorch/quantization_speedup/integrated_tensorrt.py b/new_impl/cv/third_party/nni_new/compression/pytorch/quantization_speedup/integrated_tensorrt.py new file mode 100644 index 0000000000000000000000000000000000000000..c7849774cc36295ae13a2be0442de7cbcf9ff894 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/compression/pytorch/quantization_speedup/integrated_tensorrt.py @@ -0,0 +1,381 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import time +import logging +import tensorrt as trt +import numpy as np +import torch + +from . import frontend_to_onnx as fonnx +from . import calibrator as calibrator +from . import trt_pycuda as common +from .backend import BaseModelSpeedup + +# TRT_LOGGER = trt.Logger(trt.Logger.VERBOSE) +TRT_LOGGER = trt.Logger() +logger = logging.getLogger(__name__) + +class CalibrateType: + LEGACY = trt.CalibrationAlgoType.LEGACY_CALIBRATION + ENTROPY = trt.CalibrationAlgoType.ENTROPY_CALIBRATION + ENTROPY2 = trt.CalibrationAlgoType.ENTROPY_CALIBRATION_2 + MINMAX = trt.CalibrationAlgoType.MINMAX_CALIBRATION + +Precision_Dict = { + 8: trt.float32, + 16: trt.float16, + 32: trt.float32 +} + +def valid_config(config=None): + """ + This function validates the bit setting configuration + """ + if config is None: + return + support_bit = [8, 16, 32] + for name in config.keys(): + if 'weight_bit' in config[name]: + w_bit = config[name]['weight_bit'] + assert w_bit in support_bit, "weight bit should be 8, 16, 32" + if 'activation_bit' in config[name]: + a_bit = config[name]['activation_bit'] + assert a_bit in support_bit, "activation bit should be 8, 16, 32" + +def handle_gemm(network, layer_idx, config): + """ + This function handles special gemm operation due to layer numbers of gemm changed during pytorch->onnx model convertion. + + Parameters + ---------- + network : tensorrt.INetworkDefinition + Represents a TensorRT Network from which the Builder can build an Engine + layer_idx : int + layer index of gemm + config : dict + Config recording bit number and name of layers + """ + layer = network.get_layer(layer_idx) + pre_layer = network.get_layer(layer_idx-1) + next_layer = network.get_layer(layer_idx+1) + # if weight bit exists, set three layers' precision, + # input tensor range and the first two layers' output type + if 'weight_bit' in config[layer.name]: + assert 'tracked_min_input' in config[layer.name] + assert 'tracked_max_input' in config[layer.name] + w_bit = config[layer.name]['weight_bit'] + tracked_min_input = config[layer.name]['tracked_min_input'] + tracked_max_input = config[layer.name]['tracked_max_input'] + # set three layers the same precision + layer.precision = Precision_Dict[w_bit] + pre_layer.precision = Precision_Dict[w_bit] + next_layer.precision = Precision_Dict[w_bit] + # set the first two layers' output type + pre_layer.set_output_type(0, Precision_Dict[w_bit]) + layer.set_output_type(0, Precision_Dict[w_bit]) + pre_in_tensor = pre_layer.get_input(0) + in_tensor = layer.get_input(0) + next_in_tensor = next_layer.get_input(0) + # set three layers' input tensor range + pre_in_tensor.dynamic_range = (tracked_min_input, tracked_max_input) + in_tensor.dynamic_range = (tracked_min_input, tracked_max_input) + next_in_tensor.dynamic_range = (tracked_min_input, tracked_max_input) + + # if activation bit exists, set the last layer's output type output tensor range + if 'activation_bit' in config[layer.name]: + assert 'tracked_min_activation' in config[layer.name] + assert 'tracked_max_activation' in config[layer.name] + a_bit = config[layer.name]['activation_bit'] + tracked_min_activation = config[layer.name]['tracked_min_activation'] + tracked_max_activation = config[layer.name]['tracked_max_activation'] + # set the last layer's output type + next_layer.set_output_type(0, Precision_Dict[a_bit]) + next_out_tensor = next_layer.get_output(0) + # set the last layer's output tensor range + next_out_tensor.dynamic_range = (tracked_min_activation, tracked_max_activation) + +def build_engine(model_file, config=None, extra_layer_bit=32, strict_datatype=False, calib=None): + """ + This function builds an engine from an onnx model with calibration process. + + Parameters + ---------- + model_file : str + The path of onnx model + config : dict + Config recording bit number and name of layers + extra_layer_bit : int + Other layers which are not in config will be quantized to corresponding bit number + strict_datatype : bool + Whether constrain layer bit to the number given in config or not. If true, all the layer + will be set to given bit strictly. Otherwise, these layers will be set automatically by + tensorrt + calib : numpy array + The data using to calibrate quantization model + + Returns + ------- + tensorrt.ICudaEngine + An ICudaEngine for executing inference on a built network + """ + with trt.Builder(TRT_LOGGER) as builder, builder.create_network(common.EXPLICIT_BATCH) as network, \ + trt.OnnxParser(network, TRT_LOGGER) as parser: + # Attention that, builder should be set to 1 because of the implementation of allocate_buffer + builder.max_batch_size = 1 + builder.max_workspace_size = common.GiB(4) + + if extra_layer_bit == 32 and config is None: + pass + elif extra_layer_bit == 16 and config is None: + builder.fp16_mode = True + elif extra_layer_bit == 8 and config is None: + # entire model in 8bit mode + builder.int8_mode = True + else: + builder.int8_mode = True + builder.fp16_mode = True + builder.strict_type_constraints = strict_datatype + + valid_config(config) + + # Parse onnx model + with open(model_file, 'rb') as model: + if not parser.parse(model.read()): + logger.error('ERROR: Fail to parse the ONNX file.') + for error in range(parser.num_errors): + logger.error(parser.get_error(error)) + return None + + if calib is not None: + builder.int8_calibrator = calib + # This design may not be correct if output more than one + for i in range(network.num_layers): + if config is None: + break + layer = network.get_layer(i) + if layer.name in config: + w_bit = config[layer.name]['weight_bit'] + a_bit = config[layer.name]['activation_bit'] + layer.precision = Precision_Dict[w_bit] + layer.set_output_type(0, Precision_Dict[a_bit]) + else: + # This implementation may be incorrect when output number > 1 + for i in range(network.num_layers): + if config is None: + # no low bit layer need to be set, keep original model + break + layer = network.get_layer(i) + if layer.name not in config: + continue + # layer numbers of gemm changed during pytorch->onnx model convertion, need special handle + if layer.name[0:4] == "Gemm": + handle_gemm(network, i, config) + continue + + # If weight_bit exists in config, set layer precision and layer's input tensor dynamic range. + if 'weight_bit' in config[layer.name]: + assert 'tracked_min_input' in config[layer.name] + assert 'tracked_max_input' in config[layer.name] + w_bit = config[layer.name]['weight_bit'] + tracked_min_input = config[layer.name]['tracked_min_input'] + tracked_max_input = config[layer.name]['tracked_max_input'] + layer.precision = Precision_Dict[w_bit] + in_tensor = layer.get_input(0) + in_tensor.dynamic_range = (tracked_min_input, tracked_max_input) + + # If activation exists in config, set layer output type and layer's output tensor dynamic range. + if 'activation_bit' in config[layer.name]: + assert 'tracked_min_activation' in config[layer.name] + assert 'tracked_max_activation' in config[layer.name] + a_bit = config[layer.name]['activation_bit'] + tracked_min_activation = config[layer.name]['tracked_min_activation'] + tracked_max_activation = config[layer.name]['tracked_max_activation'] + layer.set_output_type(0, Precision_Dict[a_bit]) + out_tensor = layer.get_output(0) + out_tensor.dynamic_range = (tracked_min_activation, tracked_max_activation) + + # Build engine and do int8 calibration. + engine = builder.build_cuda_engine(network) + return engine + +class ModelSpeedupTensorRT(BaseModelSpeedup): + def __init__(self, model, input_shape, config=None, onnx_path="default_model.onnx", extra_layer_bit=32, strict_datatype=True, + calibrate_type=CalibrateType.ENTROPY2, calib_data_loader=None, calibration_cache = "calibration.cache", batchsize=1, + input_names=["actual_input_1"], output_names=["output1"]): + """ + Parameters + ---------- + model : pytorch model + The model to speed up by quantization. + input_shape : tuple + The input shape of model, shall pass it to torch.onnx.export. + config : dict + Config recording bit number and name of layers. + onnx_path : str + The path user want to store onnx model which is converted from pytorch model. + extra_layer_bit : int + Other layers which are not in config will be quantized to corresponding bit number. + strict_datatype : bool + Whether constrain layer bit to the number given in config or not. If true, all the layer + will be set to given bit strictly. Otherwise, these layers will be set automatically by + tensorrt. + calibrate_type : tensorrt.tensorrt.CalibrationAlgoType + The algorithm of calibrating. Please refer to https://docs.nvidia.com/deeplearning/ + tensorrt/api/python_api/infer/Int8/Calibrator.html for detail + calibrate_data : numpy array + The data using to calibrate quantization model + calibration_cache : str + The path user want to store calibrate cache file + batchsize : int + The batch size of calibration and inference + input_names : list + Input name of onnx model providing for torch.onnx.export to generate onnx model + output_name : list + Output name of onnx model providing for torch.onnx.export to generate onnx model + """ + super().__init__(model, config) + self.model = model + self.onnx_path = onnx_path + self.input_shape = input_shape + self.config = config + self.extra_layer_bit = extra_layer_bit + self.strict_datatype = strict_datatype + self.calibrate_type = calibrate_type + self.calib_data_loader = calib_data_loader + self.calibration_cache = calibration_cache + self.batchsize = batchsize + self.input_names = input_names + self.output_names = output_names + self.context = None + self.onnx_config = {} + + def compress(self): + """ + Get onnx config and build tensorrt engine. + """ + assert self.model is not None + assert self.onnx_path is not None + assert self.input_shape is not None + + # Convert pytorch model to onnx model and save onnx model in onnx_path + _, self.onnx_config = fonnx.torch_to_onnx(self.model, self.config, input_shape=self.input_shape, + model_path=self.onnx_path, input_names=self.input_names, output_names=self.output_names) + + if self.calib_data_loader is not None: + assert self.calibrate_type is not None + context = self._tensorrt_build_withcalib(self.onnx_path) + else: + context = self._tensorrt_build_withoutcalib(self.onnx_path) + self.context = context + + def _tensorrt_build_withcalib(self, onnx_path): + """ + Convert pytorch tensor to numpy darray + + Parameters + ---------- + onnx_path : str + The path of onnx model + + Returns + ------- + tensorrt.IExecutionContext + Context for executing inference using an ICudaEngine + """ + calib_data = None + if type(self.calib_data_loader) == torch.utils.data.dataloader.DataLoader: + calib_data_set = [] + for data, _ in self.calib_data_loader: + calib_data_set.append(data) + calib_data = np.concatenate(calib_data_set) + elif type(self.calib_data_loader) == torch.Tensor: + calib_data = self.calib_data_loader.numpy() + else: + raise ValueError("Not support calibration datatype") + calib = calibrator.Calibrator(calib_data, self.calibration_cache, self.batchsize, self.calibrate_type) + + # build inference engine with calibration + engine = build_engine(onnx_path, self.onnx_config, self.extra_layer_bit, self.strict_datatype, calib) + return engine.create_execution_context() + + def _tensorrt_build_withoutcalib(self, onnx_path): + """ + Build inference engine without calibration + + Parameters + ---------- + onnx_path : str + The path of onnx model + + Returns + ------- + tensorrt.IExecutionContext + Context for executing inference using an ICudaEngine + """ + engine = build_engine(onnx_path, self.onnx_config, self.extra_layer_bit, self.strict_datatype) + return engine.create_execution_context() + + def inference(self, test_data): + """ + Do inference by tensorrt builded engine. + + Parameters + ---------- + test_data : pytorch tensor + Model input tensor + """ + # convert pytorch tensor to numpy darray + test_data = test_data.numpy() + # Numpy dtype should be float32 + assert test_data.dtype == np.float32 + elapsed_time = 0 + inputs, outputs, bindings, stream = common.allocate_buffers(self.context.engine) + result = [] + for start_idx in range(0, test_data.shape[0], self.batchsize): + # If the number of images in the test set is not divisible by the batch size, the last batch will be smaller. + # This logic is used for handling that case. + end_idx = min(start_idx + self.batchsize, test_data.shape[0]) + effective_batch_size = end_idx - start_idx + + # Do inference for every batch. + inputs[0].host = test_data[start_idx:start_idx + effective_batch_size] + t1 = time.time() + [output] = common.do_inference_v2(self.context, bindings=bindings, inputs=inputs, outputs=outputs, stream=stream) + elapsed_time += time.time() - t1 + shape = output.shape[0] + output = output[0:int(shape * effective_batch_size / self.batchsize)].reshape(effective_batch_size, -1) + result.append(output.copy()) + # Use argmax to get predictions and then check accuracy + # convert numpy darray to pytorch tensor + result = torch.Tensor(np.concatenate(result)) + return result, elapsed_time + + def export_quantized_model(self, path): + """ + Export TensorRT quantized model engine which only can be loaded by TensorRT deserialize API. + + Parameters + ---------- + path : str + The path of export model + """ + assert path is not None + with open(path, "wb") as f: + f.write(self.context.engine.serialize()) + logger.info("TensorRT engine has been saved to %s", path) + + def load_quantized_model(self, path): + """ + Load TensorRT quantized model engine from specific path. + + Parameters + ---------- + path : str + The path of export model + """ + assert path is not None + with open(path, "rb") as f, trt.Runtime(TRT_LOGGER) as runtime: + engine = runtime.deserialize_cuda_engine(f.read()) + self.context = engine.create_execution_context() + logger.info("Load TensorRT engine from %s successfully.", path) \ No newline at end of file diff --git a/new_impl/cv/third_party/nni_new/compression/pytorch/quantization_speedup/trt_pycuda.py b/new_impl/cv/third_party/nni_new/compression/pytorch/quantization_speedup/trt_pycuda.py new file mode 100644 index 0000000000000000000000000000000000000000..d3f8e1f4c6792c4cded99d2aaad3dfeb47a57686 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/compression/pytorch/quantization_speedup/trt_pycuda.py @@ -0,0 +1,86 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import pycuda.driver as cuda +import pycuda.autoinit # pylint: disable=unused-import +import tensorrt as trt + +EXPLICIT_BATCH = 1 + +def GiB(val): + return val * 1 << 30 + +# Simple helper data class that's a little nicer to use than a 2-tuple. +class HostDeviceMem(object): + def __init__(self, host_mem, device_mem): + """ + This function builds an engine from an onnx model with calibration process. + + Parameters + ---------- + host_mem : host memory + Memory buffers of host + device_mem : device memory + Memory buffers of device + """ + self.host = host_mem + self.device = device_mem + + def __str__(self): + return "Host:\n" + str(self.host) + "\nDevice:\n" + str(self.device) + + def __repr__(self): + return self.__str__() + +def allocate_buffers(engine): + """ + Allocates all buffers required for an engine, i.e. host/device inputs/outputs. + + Parameters + ---------- + engine : tensorrt.ICudaEngine + An ICudaEngine for executing inference on a built network + + Returns + ------- + list + All input HostDeviceMem of an engine + list + All output HostDeviceMem of an engine + GPU bindings + Device bindings + GPU stream + A stream is a sequence of commands (possibly issued by different host threads) that execute in order + """ + inputs = [] + outputs = [] + bindings = [] + stream = cuda.Stream() + for binding in engine: + size = trt.volume(engine.get_binding_shape(binding)) * engine.max_batch_size + dtype = trt.nptype(engine.get_binding_dtype(binding)) + # Allocate host and device buffers + host_mem = cuda.pagelocked_empty(size, dtype) + device_mem = cuda.mem_alloc(host_mem.nbytes) + # Append the device buffer to device bindings. + bindings.append(int(device_mem)) + # Append to the appropriate list. + if engine.binding_is_input(binding): + inputs.append(HostDeviceMem(host_mem, device_mem)) + else: + outputs.append(HostDeviceMem(host_mem, device_mem)) + return inputs, outputs, bindings, stream + +# This function is generalized for multiple inputs/outputs for full dimension networks. +# inputs and outputs are expected to be lists of HostDeviceMem objects. +def do_inference_v2(context, bindings, inputs, outputs, stream): + # Transfer input data to the GPU. + [cuda.memcpy_htod_async(inp.device, inp.host, stream) for inp in inputs] + # Run inference. + context.execute_async_v2(bindings=bindings, stream_handle=stream.handle) + # Transfer predictions back from the GPU. + [cuda.memcpy_dtoh_async(out.host, out.device, stream) for out in outputs] + # Synchronize the stream + stream.synchronize() + # Return only the host outputs. + return [out.host for out in outputs] \ No newline at end of file diff --git a/new_impl/cv/third_party/nni_new/compression/pytorch/speedup/__init__.py b/new_impl/cv/third_party/nni_new/compression/pytorch/speedup/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..cef8ebd76c0d99b1810512afe28723b5eeccb9fc --- /dev/null +++ b/new_impl/cv/third_party/nni_new/compression/pytorch/speedup/__init__.py @@ -0,0 +1 @@ +from .compressor import ModelSpeedup \ No newline at end of file diff --git a/new_impl/cv/third_party/nni_new/compression/pytorch/speedup/compress_modules.py b/new_impl/cv/third_party/nni_new/compression/pytorch/speedup/compress_modules.py new file mode 100644 index 0000000000000000000000000000000000000000..05567986922d7c070f6ed6ac676c3c87f2a82c22 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/compression/pytorch/speedup/compress_modules.py @@ -0,0 +1,294 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +import torch +from .infer_shape import ModuleMasks + +_logger = logging.getLogger(__name__) +def _dummy_debug(*args): + return None +_logger.debug = _dummy_debug + +replace_module = { + 'BatchNorm2d': lambda module, mask: replace_batchnorm2d(module, mask), + 'Conv2d': lambda module, mask: replace_conv2d(module, mask), + 'ConvTranspose2d': lambda module, mask: replace_convtranspose2d(module, mask), + 'MaxPool2d': lambda module, mask: no_replace(module, mask), + 'AvgPool2d': lambda module, mask: no_replace(module, mask), + 'AdaptiveAvgPool2d': lambda module, mask: no_replace(module, mask), + 'ReLU': lambda module, mask: no_replace(module, mask), + 'ReLU6': lambda module, mask: no_replace(module, mask), + 'Sigmoid': lambda module, mask: no_replace(module, mask), + 'Linear': lambda module, mask: replace_linear(module, mask), + 'Dropout': lambda module, mask: no_replace(module, mask), + 'Dropout2d': lambda module, mask: no_replace(module, mask), + 'Dropout3d': lambda module, mask: no_replace(module, mask), + + # implemented by queyu, 2021/06/21 + 'ShuffleBlock': lambda module, mask: no_replace(module, mask), + 'hswish': lambda module, mask: no_replace(module, mask), + 'hsigmoid': lambda module, mask: no_replace(module, mask), + 'LeakyReLU': lambda module, mask: no_replace(module, mask), + 'ZeroPad2d': lambda module, mask: no_replace(module, mask), + + 'SiLU': lambda module, mask: no_replace(module, mask), +} + + +def no_replace(module, mask): + """ + No need to replace + """ + _logger.debug("no need to replace") + return module + + +def replace_linear(linear, mask): + """ + Parameters + ---------- + linear : torch.nn.Linear + The linear module to be replace + mask : ModuleMasks + The masks of this module + + Returns + ------- + torch.nn.Linear + The new linear module + """ + assert isinstance(mask, ModuleMasks) + assert mask.input_mask is not None + assert mask.output_mask is None + assert not mask.param_masks + index = mask.input_mask.mask_index[-1] + in_features = index.size()[0] + _logger.debug("replace linear with new in_features: %d", in_features) + new_linear = torch.nn.Linear(in_features=in_features, + out_features=linear.out_features, + bias=linear.bias is not None) + new_linear.to(linear.weight.device) + new_linear.weight.data = torch.index_select( + linear.weight.data, -1, index.to(linear.weight.device)) + if linear.bias is not None: + new_linear.bias.data.copy_(linear.bias.data) + return new_linear + + +def replace_batchnorm2d(norm, mask): + """ + Parameters + ---------- + norm : torch.nn.BatchNorm2d + The batchnorm module to be replace + mask : ModuleMasks + The masks of this module + + Returns + ------- + torch.nn.BatchNorm2d + The new batchnorm module + """ + assert isinstance(mask, ModuleMasks) + assert 'weight' in mask.param_masks and 'bias' in mask.param_masks + index = mask.param_masks['weight'].mask_index[0].cuda() + num_features = index.size()[0] + _logger.debug("replace batchnorm2d with num_features: %d", num_features) + new_norm = torch.nn.BatchNorm2d(num_features=num_features, + eps=norm.eps, + momentum=norm.momentum, + affine=norm.affine, + track_running_stats=norm.track_running_stats) + # assign weights + new_norm.weight.data = torch.index_select(norm.weight.data, 0, index) + new_norm.bias.data = torch.index_select(norm.bias.data, 0, index) + if norm.track_running_stats: + new_norm.running_mean.data = torch.index_select( + norm.running_mean.data, 0, index) + new_norm.running_var.data = torch.index_select( + norm.running_var.data, 0, index) + return new_norm + + +def replace_conv2d(conv, mask): + """ + Parameters + ---------- + conv : torch.nn.Conv2d + The conv2d module to be replaced + mask : ModuleMasks + The masks of this module + + Returns + ------- + torch.nn.Conv2d + The new conv2d module + """ + assert isinstance(mask, ModuleMasks) + if mask.input_mask is None: + in_channels = conv.in_channels + else: + in_channels_index = mask.input_mask.mask_index[1] + in_channels = in_channels_index.size()[0] + if mask.output_mask is None: + out_channels = conv.out_channels + else: + out_channels_index = mask.output_mask.mask_index[1] + out_channels = out_channels_index.size()[0] + groups = conv.groups + if conv.in_channels == conv.out_channels == conv.groups: + # remove groups for depthwise layers + assert in_channels == out_channels + groups = in_channels + _logger.debug("replace conv2d %s with in_channels: %d, out_channels: %d", + mask.module_name, in_channels, out_channels) + new_conv = torch.nn.Conv2d(in_channels=in_channels, + out_channels=out_channels, + kernel_size=conv.kernel_size, + stride=conv.stride, + padding=conv.padding, + dilation=conv.dilation, + groups=groups, + bias=conv.bias is not None, + padding_mode=conv.padding_mode) + + new_conv.to(conv.weight.device) + tmp_weight_data = tmp_bias_data = None + + if mask.output_mask is not None: + tmp_weight_data = torch.index_select( + conv.weight.data, 0, out_channels_index.cuda()) + if conv.bias is not None: + tmp_bias_data = torch.index_select( + conv.bias.data, 0, out_channels_index.cuda()) + else: + tmp_weight_data = conv.weight.data + # For the convolutional layers that have more than one group + # we need to copy the weight group by group, because the input + # channal is also divided into serveral groups and each group + # filter may have different input channel indexes. + input_step = int(conv.in_channels / conv.groups) + in_channels_group = int(in_channels / groups) + filter_step = int(out_channels / groups) + if mask.input_mask is not None and not (in_channels == out_channels == groups): + for groupid in range(conv.groups): + start = groupid * input_step + end = (groupid + 1) * input_step + current_input_index = list( + filter(lambda x: start <= x and x < end, in_channels_index.tolist())) + if not current_input_index: + # there is no kept channel in current group + # TODO bug here, the groups is directly get from conv.groups, if the whole group is removed, + # then the number of groups in the new_conv also need to change + raise Exception( + " Donnot support removing the whole group filter except in the depth-wise conv temporarily") + # shift the global index into the group index + current_input_index = [x-start for x in current_input_index] + # if the groups is larger than 1, the input channels of each + # group should be pruned evenly. + assert len(current_input_index) == in_channels_group, \ + 'Input channels of each group are not pruned evenly' + current_input_index = torch.tensor(current_input_index).to(tmp_weight_data.device) # pylint: disable=not-callable + f_start = groupid * filter_step + f_end = (groupid + 1) * filter_step + new_conv.weight.data[f_start:f_end] = torch.index_select( + tmp_weight_data[f_start:f_end], 1, current_input_index) + else: + new_conv.weight.data.copy_(tmp_weight_data) + + if conv.bias is not None: + new_conv.bias.data.copy_( + conv.bias.data if tmp_bias_data is None else tmp_bias_data) + + return new_conv + + +def replace_convtranspose2d(convtrans, mask): + """ + We need anothor replace function for + convtranspose2d, because the layout of + the weight is different from traditional + conv layers. The layout of the weight is [N_in, N_out, ksize_1, ksize_2] + Parameters + ---------- + convtrans : torch.nn.ConvTranspose2d + The conv2d module to be replaced + mask : ModuleMasks + The masks of this module + Returns + ------- + torch.nn.ConvTranspose2d + The new conv2d module + """ + assert isinstance(mask, ModuleMasks) + assert isinstance(convtrans, torch.nn.ConvTranspose2d) + if mask.input_mask is None: + in_channels = convtrans.in_channels + else: + in_channels_index = mask.input_mask.mask_index[1] + in_channels = in_channels_index.size(0) + if mask.output_mask is None: + out_channels = convtrans.out_channels + else: + out_channels_index = mask.output_mask.mask_index[1] + out_channels = out_channels_index.size(0) + groups = convtrans.groups + # check if can remove the whole group of filters + if convtrans.in_channels == convtrans.out_channels == convtrans.groups: + # remove groups for depthwise layers + # this needs the group dependency to be fixed before the speedup + assert in_channels == out_channels + groups = in_channels + _logger.info('Replace convtranspose2d %s with in_channels:%d out_channels:%d', + mask.module_name, in_channels, out_channels) + new_convtrans = torch.nn.ConvTranspose2d(in_channels=in_channels, + out_channels=out_channels, + kernel_size=convtrans.kernel_size, + stride=convtrans.stride, + padding=convtrans.padding, + dilation=convtrans.dilation, + groups=groups, + bias=convtrans.bias is not None, + padding_mode=convtrans.padding_mode) + new_convtrans.to(convtrans.weight.device) + tmp_weight_data = None + if mask.input_mask is not None: + # in convtranspose2d we need to select the input channel first + tmp_weight_data = torch.index_select( + convtrans.weight.data, 0, in_channels_index) + else: + tmp_weight_data = convtrans.weight.data + # we need to handle the output channel group by group like the conv layer + out_step = int(convtrans.out_channels / convtrans.groups) + out_channel_group = int(out_channels/groups) + new_in_per_group = int(in_channels/groups) + + if mask.output_mask is not None and not(in_channels == out_channels == groups): + for groupid in range(convtrans.groups): + start = groupid * out_step + end = (groupid + 1) * out_step + current_output_index = list( + filter(lambda x: start <= x and x < end, out_channels_index.tolist())) + # we need to shift the index into the group-wise + current_output_index = [x-start for x in current_output_index] + if not current_output_index: + # No kept channel in the current group + raise Exception( + " Donnot support removing the whole group filter except in the depth-wise conv temporarily") + assert len(current_output_index) == out_channel_group, \ + 'Output channel of each group should be the same after pruning' + current_output_index = torch.tensor(current_output_index).to(tmp_weight_data.device) # pylint: disable=not-callable + new_start = groupid * new_in_per_group + new_end = (groupid + 1) * new_in_per_group + new_convtrans.weight.data[new_start:new_end] = torch.index_select( + tmp_weight_data[new_start:new_end], 1, current_output_index) + else: + new_convtrans.weight.data.copy_(tmp_weight_data) + if convtrans.bias is not None: + if mask.output_mask is not None: + new_convtrans.bias.data[:] = torch.index_select( + convtrans.bias.data, 0, out_channels_index) + else: + new_convtrans.bias.data.copy_(convtrans.bias.data) + return new_convtrans diff --git a/new_impl/cv/third_party/nni_new/compression/pytorch/speedup/compressor.py b/new_impl/cv/third_party/nni_new/compression/pytorch/speedup/compressor.py new file mode 100644 index 0000000000000000000000000000000000000000..ad50aa1e0cf245e2122cddeeef5d199f7244ceb4 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/compression/pytorch/speedup/compressor.py @@ -0,0 +1,192 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +import torch +from ....compression.pytorch.utils.mask_conflict import fix_mask_conflict +from ....compression.pytorch.utils.utils import get_module_by_name +from .compress_modules import replace_module +from .infer_shape import ModuleMasks, infer_from_mask, infer_from_inshape, infer_from_outshape, set_conv_prune_dim + +_logger = logging.getLogger(__name__) +def _dummy_debug(*args): + return None +_logger.debug = _dummy_debug + + +class ModelSpeedup: + """ + This class is to speedup the model with provided weight mask + """ + + def __init__(self, model, dummy_input, masks_file, map_location=None): + """ + Parameters + ---------- + model : pytorch model + The model user wants to speed up + dummy_input : pytorch tensor + The dummy input for ```jit.trace```, users should put it on right device before pass in + masks_file : str + The path of user provided mask file + map_location : str + the device on which masks are placed, same to map_location in ```torch.load``` + """ + from ....common.graph_utils import build_module_graph + + self.bound_model = model + self.masks = torch.load(masks_file, map_location) + self.inferred_masks = dict() # key: module_name, value: ModuleMasks + self.dummy_input = dummy_input + self.torch_graph = build_module_graph(model, dummy_input) + + def infer_module_mask(self, module_name, last_module, mask=None, in_shape=None, out_shape=None): + """ + Infer input shape / output shape based on the module's weight mask / input shape / output shape. + + For a module: + Infer its input and output shape from its weight mask + Infer its output shape from its input shape + Infer its input shape from its output shape + + If its input shape is changed, continue infering its predecessors + If its output shape is changed, continue infering its successors + + Parameters + ---------- + module_name : str + The name of the node + last_module : str + The name of last visited node + mask : tensor of mask or ModuleMasks + Mask of the weights in this node (i.e., module) + in_shape : ModuleMasks + Input shape of this node + out_shape : ModuleMasks + Output shape of this node + """ + input_cmask = output_cmask = None + if module_name in self.inferred_masks: + module_masks = self.inferred_masks[module_name] + else: + _, m = get_module_by_name(self.bound_model, module_name) + module_masks = ModuleMasks(module_name, m) + self.inferred_masks[module_name] = module_masks + + m_type = self.torch_graph.name_to_node[module_name].op_type + _logger.debug("infer mask of module %s with op_type %s", module_name, m_type) + if mask is not None: + _logger.debug("mask is not None") + if not m_type in infer_from_mask: + raise RuntimeError( + "Has not supported infering input/output shape from mask for module/function: `{}`, {}" + .format(m_type, module_name)) + if m_type in ['Linear']: + input_cmask, output_cmask = infer_from_mask[m_type]( + module_masks, mask, self.torch_graph.name_to_node[module_name].auxiliary + ) + else: + input_cmask, output_cmask = infer_from_mask[m_type](module_masks, mask) + if in_shape is not None: + _logger.debug("in_shape is not None") + if not m_type in infer_from_inshape: + raise RuntimeError( + "Has not supported infering output shape from input shape for module/function: `{}`, {}" + .format(m_type, module_name)) + if m_type in ['aten::view', 'aten::flatten', 'aten::mean', 'aten::reshape']: + output_cmask = infer_from_inshape[m_type](module_masks, + in_shape, + self.torch_graph.name_to_node[module_name].auxiliary) + elif m_type in ['aten::cat']: + # To calculate the mask for concat operation, the output shape + # , cat dimension, and the order of the input parameters. + output_cmask = infer_from_inshape[m_type](module_masks, + in_shape, + self.torch_graph.name_to_node[module_name].auxiliary, + last_module) + else: + output_cmask = infer_from_inshape[m_type](module_masks, in_shape) + if out_shape is not None: + _logger.debug("out_shape is not None") + if not m_type in infer_from_outshape: + raise RuntimeError( + "Has not supported infering input shape from output shape for module/function: `{}`, {}" + .format(m_type, module_name)) + if m_type in ['aten::view', 'aten::flatten', 'aten::mean', 'aten::reshape']: + input_cmask = infer_from_outshape[m_type](module_masks, out_shape, self.torch_graph.name_to_node[module_name].auxiliary) + else: + input_cmask = infer_from_outshape[m_type](module_masks, out_shape) + + if input_cmask: + predecessors = self.torch_graph.find_predecessors(module_name) + for _module_name in predecessors: + self.infer_module_mask(_module_name, module_name, out_shape=input_cmask) + if output_cmask: + successors = self.torch_graph.find_successors(module_name) + for _module_name in successors: + self.infer_module_mask(_module_name, module_name, in_shape=output_cmask) + + def infer_modules_masks(self): + """ + Do shape inference of involved modules, including the shape of weights, inputs, output + """ + for module_name, mask in self.masks.items(): + _logger.debug('Start mask inference from %s', module_name) + if module_name not in self.torch_graph.name_to_node: + # this module is not traced in the torch_graph, + # jit.trace only correctly records functions and + # modules which are not data dependent (e.g., do + # not have conditionals on data in tensors) + # so, if a node is not traced, we just skip it. + _logger.warning('%s has mask, but not found in the traced graph, just skip it.', module_name) + continue + self.infer_module_mask(module_name, None, mask=mask) + + def replace_compressed_modules(self): + """ + Replace all the modules that have changed (weights/inputs/output) shape. + The new module is created using the same arguments of the to-be-replaced module, + and correctly inherits its weights. + + NOTE: ```func``` type cannot be replaced as it is not a module, thus, one limitation + is that ```func``` should be not required to be replaced. + """ + for module_name in self.inferred_masks: + g_node = self.torch_graph.name_to_node[module_name] + _logger.debug("replace %s, in %s type, with op_type %s", + module_name, g_node.type, g_node.op_type) + if g_node.type == 'module': + super_module, leaf_module = get_module_by_name(self.bound_model, g_node.name) + m_type = g_node.op_type + if not m_type in replace_module: + raise RuntimeError("Has not supported replacing the module: `{}`".format(m_type)) + _logger.info("replace module (name: %s, op_type: %s)", g_node.name, m_type) + compressed_module = replace_module[m_type](leaf_module, self.inferred_masks[module_name]) + setattr(super_module, g_node.name.split('.')[-1], compressed_module) + elif g_node.type == 'func': + _logger.info("Warning: cannot replace (name: %s, op_type: %s) which is func type", + module_name, g_node.op_type) + else: + raise RuntimeError("Unsupported node type: {}".format(g_node.type)) + + def speedup_model(self): + """ + There are basically two steps: + first, do mask/shape inference, + second, replace modules + """ + training = self.bound_model.training + _logger.info("start to speed up the model") + + _logger.info("fix the mask conflict of the interdependent layers") + fixed_masks, conv_prune_dim = fix_mask_conflict(self.masks, self.bound_model, self.dummy_input) + set_conv_prune_dim(conv_prune_dim) + + _logger.info("infer module masks...") + self.infer_modules_masks() + _logger.info("replace compressed modules...") + self.replace_compressed_modules() + self.bound_model.train(training) + _logger.info("speedup done") + + return fixed_masks diff --git a/new_impl/cv/third_party/nni_new/compression/pytorch/speedup/infer_shape.py b/new_impl/cv/third_party/nni_new/compression/pytorch/speedup/infer_shape.py new file mode 100644 index 0000000000000000000000000000000000000000..e04c50c527148c27033313238dcf6a4676f26949 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/compression/pytorch/speedup/infer_shape.py @@ -0,0 +1,1189 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. +""" +For each operation or module, there are two functions. +One is given output shape, infer its input shape and initialization parameters (e.g., weight's shape) +The other is given input shape, infer its output shape and initialization parameters (e.g., weight's shape) +""" + +import logging +import torch + +_logger = logging.getLogger(__name__) + +conv_prune_dim = -1 + + +def set_conv_prune_dim(dim): + """ + Parameters: + dim: int + 0: filter pruning + 1: channel pruning + """ + global conv_prune_dim + conv_prune_dim = dim + + +class CoarseMask: + """ + Coarse grained mask for a given tensor, here tensor could be weights, + input tensor, or output tensor + """ + + def __init__(self, num_dim): + """ + Parameters + ---------- + num_dim : int + The number of dimensions of the tensor that will be masked + """ + self.mask_index = [None for _ in range(num_dim)] + + def add_index_mask(self, dim, index): + """ + Add mask for the specified dimension + + Parameters + ---------- + dim : int + The dimension to add mask + index : tensor + The mask for this dimension, its a 1 dimension tensor which specifies + the index of the elements that are not pruned + """ + self.mask_index[dim] = index + + @staticmethod + def merge_index(index_a, index_b): + """ + Parameters + ---------- + index_a : tensor + One index (1-dimension) tensor + index_b : tensor + The other index (1-dimension) tensor + + Returns + ------- + tensor + The merged index (1-dimension) tensor + Note that: the output tensor will be moved + to the same device as index_a. + """ + device = index_a.device + s = set() + for num in index_a.tolist(): + # we need to transfer the tensor to list here + # first, directly traversing the tensor by for + # loop will return the list of tensor(x) object, + # even the value are the same, but they are different + # tensor objects, so the set will contains multiple + # tensor objects that has the same value. For example + # for num in torch.ones(2): + # s.add(num) + # s will be {tensor(1), tensor(1)} + s.add(num) + for num in index_b.tolist(): + s.add(num) + # move the output tensor to the same device with index_a + return torch.tensor(sorted(s)).to(device) # pylint: disable=not-callable + + def merge(self, cmask): + """ + Merge another CoarseMask + + Parameters + ---------- + cmask : CoarseMask + Another CoarseMask to merge + + Returns + ------- + list + The member variable ```mask_index``` + """ + assert isinstance(cmask, CoarseMask) + assert len(self.mask_index) == len(cmask.mask_index), \ + "Only masks with the same number of dimensions can be merged" + for i, index in enumerate(self.mask_index): + if index is None: + self.mask_index[i] = cmask.mask_index[i] + elif cmask.mask_index[i] is not None: + self.mask_index[i] = CoarseMask.merge_index(self.mask_index[i], + cmask.mask_index[i]) + return self.mask_index + + def __repr__(self): + return 'mask_index: {}'.format(self.mask_index) + + def eq_on_dim(self, other, dim): + assert isinstance(other, CoarseMask) + if self.mask_index[dim] is None and other.mask_index[dim] is None: + return True + elif isinstance(self.mask_index[dim], torch.Tensor) \ + and isinstance(other.mask_index[dim], torch.Tensor): + return torch.equal(self.mask_index[dim], other.mask_index[dim]) + else: + return False + + def __eq__(self, other): + assert isinstance(other, CoarseMask) + if len(self.mask_index) != len(other.mask_index): + return False + for i in range(len(self.mask_index)): + if not self.eq_on_dim(other, i): + return False + return True + + def __lt__(self, other): + """ + Judge if the mask is a subset of another CoarseMask. + """ + assert isinstance(other, CoarseMask) + for dim, _ in enumerate(self.mask_index): + # if self has more dimensions + if dim >= len(other.mask_index): + return False + if self.mask_index[dim] is None: + # if no mask on this dimension, then we have less + # masks then the other CoraseMask. + continue + elif other.mask_index[dim] is None: + return False + else: + s1 = set(self.mask_index[dim].tolist()) + s2 = set(other.mask_index[dim].tolist()) + if not s1 < s2: + return False + return True + + def __le__(self, other): + """ + Return if self's mask is less or equal to other's mask. + """ + assert isinstance(other, CoarseMask) + if self.__lt__(other) or self.__eq__(other): + return True + return False + + def __ne__(self, other): + return not self.__eq__(other) + + +class ModuleMasks: + """ + The masks of a module, including the masks for weights, inputs, output + """ + + def __init__(self, module_name, module=None): + """ + Parameters + ---------- + module_name : str + The name of the module or function + """ + self.module_name = module_name + self.module = module + self.param_masks = dict() + self.input_mask = None + self.output_mask = None + + def set_param_masks(self, name, mask): + """ + Parameters + ---------- + name : str + The name of the weight + mask : CoarseMask + The mask for this weight + """ + self.param_masks[name] = mask + + def set_input_mask(self, mask): + """ + Parameters + ---------- + mask : CoarseMask + The mask for input + """ + self.input_mask = mask + + def set_output_mask(self, mask): + """ + Parameters + ---------- + mask : CoarseMask + The mask for output + """ + self.output_mask = mask + + def __repr__(self): + return 'module_name: {}, input_mask: {}, output_mask: {}, param_masks: {}'.format( + self.module_name, self.input_mask, self.output_mask, self.param_masks + ) + + +""" +Infer input and output shape of a module/function from its weight mask +""" +infer_from_mask = { + 'BatchNorm2d': lambda module_masks, mask: batchnorm2d_mask(module_masks, mask), + 'Conv2d': lambda module_masks, mask: conv2d_mask(module_masks, mask), + 'ConvTranspose2d': lambda module_masks, mask: convtranspose2d_mask(module_masks, mask), + 'Linear': lambda module_masks, mask, shape: linear_mask(module_masks, mask, shape) +} + +""" +Infer output and weight shape of a module/function from its input shape +""" +infer_from_inshape = { + 'ReLU': lambda module_masks, mask: relu_inshape(module_masks, mask), + 'ReLU6': lambda module_masks, mask: relu_inshape(module_masks, mask), + 'Sigmoid': lambda module_masks, mask: relu_inshape(module_masks, mask), + 'aten::relu': lambda module_masks, mask: relu_inshape(module_masks, mask), + 'aten::tanh': lambda module_masks, mask: relu_inshape(module_masks, mask), + 'aten::tanh_': lambda module_masks, mask: relu_inshape(module_masks, mask), + 'aten::hardtanh': lambda module_masks, mask: relu_inshape(module_masks, mask), + 'aten::hardtanh_': lambda module_masks, mask: relu_inshape(module_masks, mask), + 'aten::relu_': lambda module_masks, mask: relu_inshape(module_masks, mask), + 'aten::sigmoid': lambda module_masks, mask: relu_inshape(module_masks, mask), + 'Conv2d': lambda module_masks, mask: conv2d_inshape(module_masks, mask), + 'ConvTranspose2d': lambda module_masks, mask: convtranspose2d_inshape(module_masks, mask), + 'MaxPool2d': lambda module_masks, mask: maxpool2d_inshape(module_masks, mask), + 'aten::max_pool2d': lambda module_masks, mask: maxpool2d_inshape(module_masks, mask), + 'aten::avg_pool2d': lambda module_masks, mask: maxpool2d_inshape(module_masks, mask), + 'aten::adaptive_avg_pool2d': lambda module_masks, mask: maxpool2d_inshape(module_masks, mask), + 'AvgPool2d': lambda module_masks, mask: maxpool2d_inshape(module_masks, mask), + 'AdaptiveAvgPool2d': lambda module_masks, mask: maxpool2d_inshape(module_masks, mask), + 'aten::size': lambda module_masks, mask: size_inshape(module_masks, mask), + 'aten::view': lambda module_masks, mask, shape: view_inshape(module_masks, mask, shape), + 'aten::reshape': lambda module_masks, mask, shape: view_inshape(module_masks, mask, shape), + # support only start_dim=1 + 'aten::flatten': lambda module_masks, mask, shape: view_inshape(module_masks, mask, shape), + 'Linear': lambda module_masks, mask: linear_inshape(module_masks, mask), + 'BatchNorm2d': lambda module_masks, mask: batchnorm2d_inshape(module_masks, mask), + 'aten::add_': lambda module_masks, mask: add_inshape(module_masks, mask), + 'aten::add': lambda module_mask, mask: add_inshape(module_mask, mask), + # mul has the similar behaviour with add, they both request + # the input tesors to have the same shape + 'aten::mul': lambda module_mask, mask: add_inshape(module_mask, mask), + 'aten::mul_': lambda module_mask, mask: add_inshape(module_mask, mask), + 'aten::div': lambda module_mask, mask: add_inshape(module_mask, mask), + 'aten::div_': lambda module_mask, mask: add_inshape(module_mask, mask), + 'aten::cat': lambda module_mask, mask, cat_info, last_visited: cat_inshape(module_mask, mask, cat_info, last_visited), + 'aten::mean': lambda module_masks, mask, shape: mean_inshape(module_masks, mask, shape), + 'Dropout': lambda module_masks, mask: dropout_inshape(module_masks, mask), + 'Dropout2d': lambda module_masks, mask: dropout_inshape(module_masks, mask), + 'aten::dropout': lambda module_masks, mask: dropout_inshape(module_masks, mask), + 'aten::detach': lambda module_masks, mask: dropout_inshape(module_masks, mask), + + # implemented by queyu, 2021/06/21, + 'ShuffleBlock': lambda module_masks, mask: relu_inshape(module_masks, mask), + 'hswish': lambda module_masks, mask: relu_inshape(module_masks, mask), + 'hsigmoid': lambda module_masks, mask: relu_inshape(module_masks, mask), + 'Swish': lambda module_masks, mask: relu_inshape(module_masks, mask), + # implemented by queyu, 2021/07/04 + 'LeakyReLU': lambda module_masks, mask: relu_inshape(module_masks, mask), + 'ZeroPad2d': lambda module_masks, mask: maxpool2d_inshape(module_masks, mask), + # implemented by queyu, 2022/08/05 + 'aten::upsample_bilinear2d': lambda module_masks, mask: relu_inshape(module_masks, mask), + # implemented by queyu, 2022/08/05 + 'SiLU': lambda module_masks, mask: relu_inshape(module_masks, mask), +} + +""" +Infer input and weight shape of a module/function from its output shape +""" +infer_from_outshape = { + 'Conv2d': lambda module_masks, mask: conv2d_outshape(module_masks, mask), + 'ConvTranspose2d': lambda module_masks, mask: convtranspose2d_outshape(module_masks, mask), + 'BatchNorm2d': lambda module_masks, mask: batchnorm2d_outshape(module_masks, mask), + + 'MaxPool2d': lambda module_masks, mask: maxpool2d_outshape(module_masks, mask), + 'aten::max_pool2d': lambda module_masks, mask: maxpool2d_outshape(module_masks, mask), + 'aten::avg_pool2d': lambda module_masks, mask: maxpool2d_outshape(module_masks, mask), + 'aten::adaptive_avg_pool2d': lambda module_masks, mask: maxpool2d_outshape(module_masks, mask), + 'AvgPool2d': lambda module_masks, mask: maxpool2d_outshape(module_masks, mask), + 'AdaptiveAvgPool2d': lambda module_masks, mask: maxpool2d_outshape(module_masks, mask), + + 'ReLU': lambda module_masks, mask: relu_outshape(module_masks, mask), + 'ReLU6': lambda module_masks, mask: relu_outshape(module_masks, mask), + 'aten::relu': lambda module_masks, mask: relu_outshape(module_masks, mask), + 'aten::tanh': lambda module_masks, mask: relu_outshape(module_masks, mask), + 'aten::tanh_': lambda module_masks, mask: relu_outshape(module_masks, mask), + 'aten::hardtanh': lambda module_masks, mask: relu_outshape(module_masks, mask), + 'aten::hardtanh_': lambda module_masks, mask: relu_outshape(module_masks, mask), + 'aten::relu_': lambda module_masks, mask: relu_outshape(module_masks, mask), + + 'aten::add_': lambda module_masks, mask: add_outshape(module_masks, mask), + 'aten::add': lambda module_mask, mask: add_outshape(module_mask, mask), + 'aten::flatten': lambda module_mask, mask, shape: view_outshape(module_mask, mask, shape), + 'aten::view': lambda module_masks, mask, shape: view_outshape(module_masks, mask, shape), + 'aten::reshape': lambda module_masks, mask, shape: view_outshape(module_masks, mask, shape), + 'aten::mean': lambda module_masks, mask, shape: mean_outshape(module_masks, mask, shape), + 'Dropout': lambda module_masks, mask: dropout_outshape(module_masks, mask), + 'Dropout2d': lambda module_masks, mask: dropout_outshape(module_masks, mask), + 'aten::dropout': lambda module_masks, mask: dropout_outshape(module_masks, mask), + 'aten::detach': lambda module_masks, mask: dropout_outshape(module_masks, mask) +} + + +def dropout_inshape(module_masks, mask): + if module_masks.input_mask is None: + module_masks.set_input_mask(mask) + module_masks.set_output_mask(mask) + return module_masks.output_mask + # if alreay visited + assert module_masks.input_mask <= mask + # It should be the same, we pass the masks by the reference(not the value), + # so they acutually are two references of the same object(mask, + # module_masks.input_mask). So we should continue pass the mask + # to the following nodes even module_masks.input_mask == mask. + # if pass the mask by copy.deepcopy(), then we can stop when + # module_masks.input_mask == mask. + # if module_masks.input_mask == mask: + # return None + module_masks.set_input_mask(mask) + module_masks.set_output_mask(mask) + return module_masks.output_mask + + +def dropout_outshape(module_masks, mask): + if module_masks.output_mask is None: + module_masks.set_output_mask(mask) + module_masks.set_input_mask(mask) + return module_masks.input_mask + # if alreay visited + assert all(module_masks.output_mask.mask_index[1] == mask.mask_index[1]) + + return module_masks.output_mask + + +def cat_inshape(module_masks, mask, cat_info, last_visited): + """ + Inference the output mask of the cat operation from the + input mask. + + Parameters + ---------- + module_masks : ModuleMasks + The ModuleMasks instance of the Conv2d + mask : CoarseMask + The mask of its input tensor + cat_info: dict + Dict object that records the necessary information + of cat operation, such as the order of the input + tensors. + last_visited: str + The unique_name of the last visited node group. + + Returns + ------- + CoarseMask + The mask of its output tensor + + """ + assert isinstance(mask, CoarseMask) + out_shape = cat_info['out_shape'] + cat_dim = cat_info['cat_dim'] + in_order = cat_info['in_order'] + in_shape = cat_info['in_shape'] + if module_masks.output_mask is None: + # First visit to this cat node + # initialize the mask based on + # the number of the output channel. + output_mask = CoarseMask(num_dim=len(out_shape)) + for dim, _ in enumerate(out_shape): + if dim == cat_dim: + if mask.mask_index[dim] is None: + continue + device = mask.mask_index[dim].device + # calculate the offset of the mask + pos = in_order.index(last_visited) + offsets = [in_shape[i][cat_dim] + for i, _ in enumerate(in_shape)] + offset = 0 + for i in range(pos): + offset += offsets[i] + _tmp_mask = (mask.mask_index[dim] + offset).to(device) + output_mask.mask_index[dim] = _tmp_mask + else: + # directly copy the mask + if mask.mask_index[dim] is not None: + output_mask.mask_index[dim] = mask.mask_index[dim].data.clone( + ) + module_masks.set_output_mask(output_mask) + + return module_masks.output_mask + # If this cat node is already visited, we need + # validating if the mask is legel, for cat operation, + # the mask on the 'cat_dim' dimension should be stitched + # together. In the other dimensions, the mask should be + # the same, else the mask is not legal. + for dim, _ in enumerate(out_shape): + if dim == cat_dim: + if mask.mask_index[dim] is None: + continue + pos = in_order.index(last_visited) + offsets = [in_shape[i][cat_dim] for i, _ in enumerate(in_shape)] + offset = 0 + for i in range(pos): + offset += offsets[i] + device = mask.mask_index[dim].device + new_mask = mask.mask_index[dim] + offset + module_masks.output_mask.mask_index[dim] = CoarseMask.merge_index( + module_masks.output_mask.mask_index[dim], new_mask).to(device) + else: + assert module_masks.output_mask.eq_on_dim(mask, dim) + + return module_masks.output_mask + + +def add_inshape(module_masks, mask): + """ + Inference the output mask of the add operation from the + input mask. + """ + assert isinstance(mask, CoarseMask) + if module_masks.input_mask is None: + module_masks.set_input_mask(mask) + module_masks.set_output_mask(mask) + # module_masks.input_mask = mask + return mask + # If alreay visited, validate if have the conflict + # if the mask is different with previous input_mask + # then there is a mask confilct. + if mask != module_masks.input_mask: + raise Exception('Mask conflict happenes!') + return None + + +def add_outshape(module_masks, mask): + """ + Inference the input mask of the add operation from the + output mask. + """ + assert isinstance(mask, CoarseMask) + + if module_masks.output_mask is None: + module_masks.set_output_mask(mask) + module_masks.set_input_mask(mask) + return mask + else: + assert all( + module_masks.output_mask.mask_index[1] == mask.mask_index[1]) + return mask + + +def batchnorm2d_inshape(module_masks, mask): + """ + We assume only the second dimension has coarse grained mask + + Parameters + ---------- + module_masks : ModuleMasks + The ModuleMasks instance of the batchnorm2d + mask : CoarseMask + The mask of its input tensor + + Returns + ------- + CoarseMask + The mask of its output tensor + """ + assert isinstance(mask, CoarseMask) + assert mask.mask_index[1] is not None + assert mask.mask_index[0] is None + assert mask.mask_index[2] is None + assert mask.mask_index[3] is None + module_masks.set_input_mask(mask) + module_masks.set_output_mask(mask) + weight_cmask = CoarseMask(num_dim=1) + weight_cmask.add_index_mask(dim=0, index=mask.mask_index[1]) + module_masks.set_param_masks('weight', weight_cmask) + module_masks.set_param_masks('bias', weight_cmask) + return mask + + +def batchnorm2d_outshape(module_masks, mask): + """ + We assume only the second dimension has coarse grained mask + + Parameters + ---------- + module_masks : ModuleMasks + The ModuleMasks instance of the batchnorm2d + mask : CoarseMask + The mask of its input tensor + + Returns + ------- + CoarseMask + The mask of its output tensor + """ + assert isinstance(mask, CoarseMask) + assert len(mask.mask_index) in [2, 4] + assert mask.mask_index[1] is not None + assert mask.mask_index[0] is None + module_masks.set_input_mask(mask) + module_masks.set_output_mask(mask) + weight_cmask = CoarseMask(num_dim=1) + weight_cmask.add_index_mask(dim=0, index=mask.mask_index[1]) + module_masks.set_param_masks('weight', weight_cmask) + module_masks.set_param_masks('bias', weight_cmask) + return mask + + +def linear_inshape(module_masks, mask): + """ + Coarse grained input mask does not change the shape of weights and output tensor + + Parameters + ---------- + module_masks : ModuleMasks + The ModuleMasks instance of the linear + mask : CoarseMask + The mask of its input tensor + + Returns + ------- + CoarseMask + The mask of its output tensor, ```None``` means shape of output tensor is not changed + """ + assert isinstance(mask, CoarseMask) + assert mask.mask_index[0] is None + if module_masks.input_mask is not None: + assert module_masks.input_mask <= mask + module_masks.set_input_mask(mask) + return None + + +def view_inshape(module_masks, mask, shape): + """ + This is a limited support + + TODO: consider replace tensor.view with nn.Flatten, because tensor.view is not + included in module, thus, cannot be replaced by our framework. + + Parameters + ---------- + module_masks : ModuleMasks + The ModuleMasks instance of the ```view``` op + mask : CoarseMask + The mask of its input tensor + shape : dict + Original shape of its input and output tensors + + Returns + ------- + CoarseMask + The mask of its output tensor + """ + # NOTE: the case constrained by the following four asserts + assert shape['in_shape'][0] == shape['out_shape'][0] + assert len(shape['in_shape']) == 4 + assert len(shape['out_shape']) == 2 + assert shape['out_shape'][1] == shape['in_shape'][1] * \ + shape['in_shape'][2]*shape['in_shape'][3] + + assert isinstance(mask, CoarseMask) + assert mask.mask_index[1] is not None + assert mask.mask_index[0] is None + assert mask.mask_index[2] is None + assert mask.mask_index[3] is None + # due to the cat operation, the same node may be + # accessed more than once + if module_masks.input_mask is not None: + assert module_masks.input_mask <= mask + module_masks.set_input_mask(mask) + output_cmask = CoarseMask(num_dim=2) + index = [] + step_size = shape['in_shape'][2] * shape['in_shape'][3] + for loc in mask.mask_index[1]: + index.extend([loc * step_size + i for i in range(step_size)]) + output_cmask.add_index_mask(dim=1, index=torch.tensor(index).to(mask.mask_index[1].device)) # pylint: disable=not-callable + module_masks.set_output_mask(output_cmask) + return output_cmask + + +def view_outshape(module_masks, mask, shape): + """ + Parameters + ---------- + module_masks : ModuleMasks + The ModuleMasks instance of the ```view``` op + mask : CoarseMask + The mask of its output tensor + shape : dict + Original shape of its input and output tensors + Returns + ------- + CoarseMask + The mask of its input tensor + """ + # NOTE: the case constrained by the following four asserts + assert shape['in_shape'][0] == shape['out_shape'][0] + assert len(shape['in_shape']) == 4 + assert len(shape['out_shape']) == 2 + assert shape['out_shape'][1] == shape['in_shape'][1] * \ + shape['in_shape'][2]*shape['in_shape'][3] + + assert isinstance(mask, CoarseMask) + assert mask.mask_index[1] is not None + assert mask.mask_index[0] is None + + module_masks.set_output_mask(mask) + input_cmask = CoarseMask(num_dim=4) + index = set() + step_size = shape['in_shape'][2] * shape['in_shape'][3] + for loc in mask.mask_index[1]: + index.add(loc // step_size) + index = sorted(list(index)) + input_cmask.add_index_mask(dim=1, index=torch.tensor(index).to(mask.mask_index[1].device)) # pylint: disable=not-callable + module_masks.set_input_mask(input_cmask) + + return input_cmask + + +def size_inshape(module_masks, mask): + """ + No need to do anything for this ```size``` op + """ + return None + + +def mean_inshape(module_masks, mask, shape): + """ + Similar to view operation, currently mask inference only supports + the mean operation on the 3rd and 4th dimensions. + """ + assert shape['in_shape'][0] == shape['out_shape'][0] + assert shape['out_shape'][1] == shape['in_shape'][1] + assert len(shape['in_shape']) == 4 + assert len(shape['out_shape']) == 2 + + assert isinstance(mask, CoarseMask) + assert mask.mask_index[1] is not None + assert mask.mask_index[0] is None + assert mask.mask_index[2] is None + assert mask.mask_index[3] is None + module_masks.set_input_mask(mask) + + output_cmask = CoarseMask(num_dim=2) + output_cmask.add_index_mask(dim=1, index=mask.mask_index[1]) + module_masks.set_output_mask(output_cmask) + return output_cmask + + +def mean_outshape(module_masks, mask, shape): + """ + Similar to view operation, currently mask inference only supports + the mean operation on the 3rd and 4th dimensions. + """ + assert shape['in_shape'][0] == shape['out_shape'][0] + assert shape['out_shape'][1] == shape['in_shape'][1] + assert len(shape['in_shape']) == 4 + assert len(shape['out_shape']) == 2 + + assert isinstance(mask, CoarseMask) + assert mask.mask_index[1] is not None + assert mask.mask_index[0] is None + module_masks.set_output_mask(mask) + + input_cmask = CoarseMask(num_dim=4) + input_cmask.add_index_mask(dim=1, index=mask.mask_index[1]) + module_masks.set_input_mask(input_cmask) + return input_cmask + + +def maxpool2d_inshape(module_masks, mask): + """ + Assume only the second dimension is masked + + Parameters + ---------- + module_masks : ModuleMasks + The ModuleMasks instance of the maxpool2d + mask : CoarseMask + The mask of its input tensor + + Returns + ------- + CoarseMask + The mask of its output tensor + """ + assert isinstance(mask, CoarseMask) + assert mask.mask_index[1] is not None + assert mask.mask_index[0] is None + assert mask.mask_index[2] is None + assert mask.mask_index[3] is None + if module_masks.input_mask is not None: + assert module_masks.input_mask <= mask + # assert module_masks.input_mask is None + module_masks.set_input_mask(mask) + module_masks.set_output_mask(mask) + return mask + + +def maxpool2d_outshape(module_masks, mask): + """ + Assume only the second dimension is masked + + Parameters + ---------- + module_masks : ModuleMasks + The ModuleMasks instance of the maxpool2d + mask : CoarseMask + The mask of its input tensor + + Returns + ------- + CoarseMask + The mask of its output tensor + """ + assert isinstance(mask, CoarseMask) + assert mask.mask_index[1] is not None + assert mask.mask_index[0] is None + + module_masks.set_input_mask(mask) + module_masks.set_output_mask(mask) + return mask + + +def relu_inshape(module_masks, mask): + """ + Parameters + ---------- + module_masks : ModuleMasks + The ModuleMasks instance of the relu + mask : CoarseMask + The mask of its input tensor + + Returns + ------- + CoarseMask + The mask of its output tensor + """ + assert isinstance(mask, CoarseMask) + if module_masks.input_mask is not None: + # mask conflict should be solved before speedup + assert module_masks.input_mask <= mask + # assert module_masks.input_mask is None, "A relu op can only be processed once" + module_masks.set_input_mask(mask) + module_masks.set_output_mask(mask) + return mask + + +def relu_outshape(module_masks, mask): + """ + Parameters + ---------- + module_masks : ModuleMasks + The ModuleMasks instance of the relu + mask : CoarseMask + The mask of its input tensor + + Returns + ------- + CoarseMask + The mask of its output tensor + """ + assert isinstance(mask, CoarseMask) + if module_masks.output_mask is not None: + # mask conflict should be solved before speedup + assert all( + module_masks.output_mask.mask_index[1] == mask.mask_index[1]) + module_masks.set_input_mask(mask) + module_masks.set_output_mask(mask) + return mask + + +def batchnorm2d_mask(module_masks, mask): + """ + Infer input and output shape from weight mask + Parameters + ---------- + module_masks : ModuleMasks + The ModuleMasks instance of the batchnorm2d + mask : dict + The mask of its weights, from the user provided mask file + Returns + ------- + CoarseMask, CoarseMask + The mask of its input tensor, the mask of its output tensor + """ + assert 'weight' in mask and 'bias' in mask + sum_mask = mask['weight'] + mask['bias'] + nonzero_index = torch.nonzero(sum_mask, as_tuple=True)[0] + # infer shape of parameters + param_cmask = CoarseMask(num_dim=1) + param_cmask.add_index_mask(dim=0, index=nonzero_index) + module_masks.set_param_masks('weight', param_cmask) + module_masks.set_param_masks('bias', param_cmask) + # infer shape of input tensor + input_cmask = CoarseMask(num_dim=4) + input_cmask.add_index_mask(dim=1, + index=torch.nonzero(mask['weight'], as_tuple=True)[0]) + module_masks.set_input_mask(input_cmask) + # infer shape of output tensor + output_cmask = CoarseMask(num_dim=4) + output_cmask.add_index_mask(dim=1, index=nonzero_index) + module_masks.set_output_mask(output_cmask) + return input_cmask, output_cmask + + +def linear_mask(module_masks, mask, shape): + """ + Infer input and output shape from weight mask with limitations: + Only support infer input mask + + Parameters + ---------- + module_masks : ModuleMasks + The ModuleMasks instance of the Linear + mask : dict + The mask of its weights, from the user provided mask file + shape: dict + Shape of its input and output tensors + Returns + ------- + CoarseMask, CoarseMask + The mask of its input tensor, the mask of its output tensor + """ + + assert 'weight' in mask + num_input_dim = len(shape['in_shape']) + + # Input data of Linear module can have multiple dimensions. + # here we only support infer coarse mask on the first dimension (dimension 0) + nonzero_index = torch.nonzero(mask['weight'].sum(0), as_tuple=True)[0] + + # infer shape of input tensor + input_cmask = CoarseMask(num_dim=num_input_dim) + input_cmask.add_index_mask(dim=num_input_dim-1, index=nonzero_index) + + module_masks.set_input_mask(input_cmask) + return input_cmask, None + + +def conv2d_mask(module_masks, mask): + """ + Infer input and output shape from weight mask + + Parameters + ---------- + module_masks : ModuleMasks + The ModuleMasks instance of the conv2d + mask : dict + The mask of its weights, from the user provided mask file + + Returns + ------- + CoarseMask, CoarseMask + The mask of its input tensor, the mask of its output tensor + """ + def convert_to_coarse_mask(mask, dim=0): + """ + Parameters + ---------- + mask : dict + Weight mask from user provided mask file + dim: int + 0: filter pruning + 1: channel pruning + + Returns + ------- + LongTensor, CoarseMask, CoarseMask + Index of the masked dimension, weight mask, bias mask + """ + assert 'weight' in mask + assert isinstance(mask['weight'], torch.Tensor) + assert dim in [0, 1] + + weight_mask = mask['weight'] + + sum_idx = (1, 2, 3) if dim == 0 else (0, 2, 3) + index = torch.nonzero(weight_mask.abs().sum( + sum_idx) != 0, as_tuple=True)[0] + + index = index.long().to(weight_mask.device) + weight_cmask = CoarseMask(num_dim=4) + weight_cmask.add_index_mask(dim=dim, index=index) + bias_cmask = None + if dim == 0 and 'bias' in mask and mask['bias'] is not None: + bias_index = torch.nonzero(mask['bias'], as_tuple=True)[0] + assert torch.all(torch.eq(index, bias_index)), \ + "bias mask should be consistent with weight mask" + bias_cmask = CoarseMask(num_dim=1) + bias_cmask.add_index_mask(dim=0, index=bias_index) + return index, weight_cmask, bias_cmask + + index, weight_cmask, bias_cmask = convert_to_coarse_mask( + mask, dim=conv_prune_dim) + + if index is None: + # TODO: fine grained mask speedup + return None, None + # deal with coarse grain mask + # mask conflict should be solved by fix_mask_conflict before speedup + if 'weight' in module_masks.param_masks: + assert module_masks.param_masks['weight'] == weight_cmask + else: + module_masks.set_param_masks('weight', weight_cmask) + if conv_prune_dim == 0: + module_masks.set_param_masks('bias', bias_cmask) + + io_cmask = CoarseMask(num_dim=4) + io_cmask.add_index_mask(dim=1, index=index) + + if conv_prune_dim == 0: + if module_masks.output_mask is None: + module_masks.set_output_mask(io_cmask) + else: + assert module_masks.output_mask == io_cmask + return None, module_masks.output_mask + else: + if module_masks.input_mask is None: + module_masks.set_input_mask(io_cmask) + else: + assert module_masks.input_mask == io_cmask + return module_masks.input_mask, None + + +def conv2d_inshape(module_masks, mask): + """ + Shape change of input tensor does not affect the shape of its output tensor + Parameters + ---------- + module_masks : ModuleMasks + The ModuleMasks instance of the conv2d + mask : CoarseMask + The mask of its input tensor + Returns + ------- + CoarseMask + The mask of its output tensor + """ + assert isinstance(mask, CoarseMask) + if module_masks.input_mask is None: + module_masks.set_input_mask(mask) + else: + # the same conv layer may be accessed more + # than once, such as a concat operation. + # mask conflict should be solved by fix_mask_conflict before speedup + + assert module_masks.input_mask == mask + + # shape changes pass through depths wise conv layers + m = module_masks.module + if m.in_channels == m.out_channels == m.groups: + module_masks.output_mask = mask + module_masks.input_mask = mask + return mask + return None + + +def conv2d_outshape(module_masks, mask): + """ + Assume only the second dimension is masked + + Parameters + ---------- + module_masks : ModuleMasks + The ModuleMasks instance of the conv2d + mask : CoarseMask + The mask of its output tensor + + Returns + ------- + CoarseMask + The mask of its input tensor + """ + assert isinstance(mask, CoarseMask) + assert mask.mask_index[1] is not None + assert mask.mask_index[0] is None + assert mask.mask_index[2] is None + assert mask.mask_index[3] is None + + if module_masks.output_mask is None: + module_masks.output_mask = mask + else: + # mask conflict should be solved by fix_mask_conflict before speedup + # mask and module_masks.output_mask may have different number of dimensions + # since they could be passed by linear or conv2d + assert all( + module_masks.output_mask.mask_index[1] == mask.mask_index[1]) + + weight_cmask = CoarseMask(num_dim=4) + weight_cmask.add_index_mask(dim=0, index=mask.mask_index[1]) + bias_cmask = CoarseMask(num_dim=1) + bias_cmask.add_index_mask(dim=0, index=mask.mask_index[1]) + module_masks.set_param_masks('weight', weight_cmask) + module_masks.set_param_masks('bias', bias_cmask) + + # shape changes pass through depths wise conv layers + m = module_masks.module + if m.in_channels == m.out_channels == m.groups: + module_masks.output_mask = mask + module_masks.input_mask = mask + return mask + return None + + +def convtranspose2d_mask(module_masks, mask): + # TODO support the Convtranspose2d Pruning for the L1FilterPruner + # raise Exception( + # "Current Filter pruner cannot prune the ConvTranspose2d, will support pruning ConvTranspose2d later") + """ + Infer input and output shape from weight mask + + Parameters + ---------- + module_masks : ModuleMasks + The ModuleMasks instance of the conv2d + mask : dict + The mask of its weights, from the user provided mask file + + Returns + ------- + CoarseMask, CoarseMask + The mask of its input tensor, the mask of its output tensor + """ + def convert_to_coarse_mask(mask, dim=0): + """ + Parameters + ---------- + mask : dict + Weight mask from user provided mask file + dim: int + 0: filter pruning + 1: channel pruning + + Returns + ------- + LongTensor, CoarseMask, CoarseMask + Index of the masked dimension, weight mask, bias mask + """ + assert 'weight' in mask + assert isinstance(mask['weight'], torch.Tensor) + assert dim in [0, 1] + + weight_mask = mask['weight'] + + sum_idx = (1, 2, 3) if dim == 0 else (0, 2, 3) + index = torch.nonzero(weight_mask.abs().sum(sum_idx) != 0, as_tuple=True)[0] + if len(index) == weight_mask.shape[dim]: # full mask + index = None + + if index is None: + return None, None, None + else: + index = index.long().to(weight_mask.device) + weight_cmask = CoarseMask(num_dim=4) + weight_cmask.add_index_mask(dim=dim, index=index) + bias_cmask = None + if dim == 0 and 'bias' in mask and mask['bias'] is not None: + bias_index = torch.nonzero(mask['bias'], as_tuple=True)[0] + assert torch.all(torch.eq(index, bias_index)), \ + "bias mask should be consistent with weight mask" + bias_cmask = CoarseMask(num_dim=1) + bias_cmask.add_index_mask(dim=0, index=bias_index) + return index, weight_cmask, bias_cmask + + convt_prune_dim = 1 + index, weight_cmask, bias_cmask = convert_to_coarse_mask(mask, dim=convt_prune_dim) + + if index is None: + # TODO: fine grained mask speedup + return None, None + # deal with coarse grain mask + # mask conflict should be solved by fix_mask_conflict before speedup + if 'weight' in module_masks.param_masks: + assert module_masks.param_masks['weight'] == weight_cmask + else: + module_masks.set_param_masks('weight', weight_cmask) + if conv_prune_dim == 0: + module_masks.set_param_masks('bias', bias_cmask) + + io_cmask = CoarseMask(num_dim=4) + io_cmask.add_index_mask(dim=1, index=index) + + if conv_prune_dim == 0: + if module_masks.output_mask is None: + module_masks.set_output_mask(io_cmask) + else: + assert module_masks.output_mask == io_cmask + return None, module_masks.output_mask + else: + if module_masks.input_mask is None: + module_masks.set_input_mask(io_cmask) + else: + assert module_masks.input_mask == io_cmask + return module_masks.input_mask, None + + +def convtranspose2d_inshape(module_masks, mask): + """ + Shape change of input tensor does not affect the shape of its output tensor + Parameters + ---------- + module_masks : ModuleMasks + The ModuleMasks instance of the conv2d + mask : CoarseMask + The mask of its input tensor + Returns + ------- + CoarseMask + The mask of its output tensor + """ + assert isinstance(mask, CoarseMask) + if module_masks.input_mask is None: + module_masks.set_input_mask(mask) + else: + # the same conv layer may be accessed more + # than once, such as a concat operation. + # mask conflict should be solved by fix_mask_conflict before speedup + assert module_masks.input_mask == mask + + # shape changes pass through depths wise conv layers + m = module_masks.module + if m.in_channels == m.out_channels == m.groups: + module_masks.output_mask = mask + module_masks.input_mask = mask + return mask + return None + + +def convtranspose2d_outshape(module_masks, mask): + assert isinstance(mask, CoarseMask) + assert mask.mask_index[1] is not None + assert mask.mask_index[0] is None + assert mask.mask_index[2] is None + assert mask.mask_index[3] is None + + if module_masks.output_mask is None: + module_masks.output_mask = mask + else: + # mask conflict should be solved by fix_mask_conflict before speedup + # mask and module_masks.output_mask may have different number of dimensions + # since they could be passed by linear or conv2d + assert all( + module_masks.output_mask.mask_index[1] == mask.mask_index[1]) + + weight_cmask = CoarseMask(num_dim=4) + # Note the memory layout of Convtranspose2d is C_in, C_out, k1, k2 + weight_cmask.add_index_mask(dim=1, index=mask.mask_index[1]) + bias_cmask = CoarseMask(num_dim=1) + bias_cmask.add_index_mask(dim=0, index=mask.mask_index[1]) + module_masks.set_param_masks('weight', weight_cmask) + module_masks.set_param_masks('bias', bias_cmask) + + # shape changes pass through depths wise conv layers + m = module_masks.module + if m.in_channels == m.out_channels == m.groups: + module_masks.output_mask = mask + module_masks.input_mask = mask + return mask + return None diff --git a/new_impl/cv/third_party/nni_new/compression/pytorch/utils/__init__.py b/new_impl/cv/third_party/nni_new/compression/pytorch/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/new_impl/cv/third_party/nni_new/compression/pytorch/utils/config_validation.py b/new_impl/cv/third_party/nni_new/compression/pytorch/utils/config_validation.py new file mode 100644 index 0000000000000000000000000000000000000000..3b9f93f96247eae71c59828d76f878d90fafc00f --- /dev/null +++ b/new_impl/cv/third_party/nni_new/compression/pytorch/utils/config_validation.py @@ -0,0 +1,53 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from schema import Schema, And, SchemaError + +def validate_op_names(model, op_names, logger): + found_names = set(map(lambda x: x[0], model.named_modules())) + + not_found_op_names = list(set(op_names) - found_names) + if not_found_op_names: + logger.warning('op_names %s not found in model', not_found_op_names) + + return True + +def validate_op_types(model, op_types, logger): + found_types = set(['default']) | set(map(lambda x: type(x[1]).__name__, model.named_modules())) + + not_found_op_types = list(set(op_types) - found_types) + if not_found_op_types: + logger.warning('op_types %s not found in model', not_found_op_types) + + return True + +def validate_op_types_op_names(data): + if not ('op_types' in data or 'op_names' in data): + raise SchemaError('Either op_types or op_names must be specified.') + return True + +class CompressorSchema: + def __init__(self, data_schema, model, logger): + assert isinstance(data_schema, list) and len(data_schema) <= 1 + self.data_schema = data_schema + self.compressor_schema = Schema(self._modify_schema(data_schema, model, logger)) + + def _modify_schema(self, data_schema, model, logger): + if not data_schema: + return data_schema + + for k in data_schema[0]: + old_schema = data_schema[0][k] + if k == 'op_types' or (isinstance(k, Schema) and k._schema == 'op_types'): + new_schema = And(old_schema, lambda n: validate_op_types(model, n, logger)) + data_schema[0][k] = new_schema + if k == 'op_names' or (isinstance(k, Schema) and k._schema == 'op_names'): + new_schema = And(old_schema, lambda n: validate_op_names(model, n, logger)) + data_schema[0][k] = new_schema + + data_schema[0] = And(data_schema[0], lambda d: validate_op_types_op_names(d)) + + return data_schema + + def validate(self, data): + self.compressor_schema.validate(data) diff --git a/new_impl/cv/third_party/nni_new/compression/pytorch/utils/counter.py b/new_impl/cv/third_party/nni_new/compression/pytorch/utils/counter.py new file mode 100644 index 0000000000000000000000000000000000000000..0302c980558474ea417b32f56d006b896f5d4cad --- /dev/null +++ b/new_impl/cv/third_party/nni_new/compression/pytorch/utils/counter.py @@ -0,0 +1,411 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import functools +from collections import Counter +from prettytable import PrettyTable + +import torch +import torch.nn as nn +from torch.nn.utils.rnn import PackedSequence +from nni.compression.pytorch.compressor import PrunerModuleWrapper + + +__all__ = ['count_flops_params'] + + +def _get_params(m): + return sum([p.numel() for p in m.parameters()]) + + +class ModelProfiler: + + def __init__(self, custom_ops=None, mode='default'): + """ + ModelProfiler is used to share state to hooks. + + Parameters + ---------- + custom_ops: dict + a mapping of (module -> torch.nn.Module : custom operation) + the custom operation is a callback funtion to calculate + the module flops, parameters and the weight shape, it will overwrite the default operation. + for reference, please see ``self.ops``. + mode: + the mode of how to collect information. If the mode is set to `default`, + only the information of convolution, linear and rnn modules will be collected. + If the mode is set to `full`, other operations will also be collected. + """ + self.ops = { + nn.Conv1d: self._count_convNd, + nn.Conv2d: self._count_convNd, + nn.Conv3d: self._count_convNd, + nn.ConvTranspose1d: self._count_convNd, + nn.ConvTranspose2d: self._count_convNd, + nn.ConvTranspose3d: self._count_convNd, + nn.Linear: self._count_linear, + nn.RNNCell: self._count_rnn_cell, + nn.GRUCell: self._count_gru_cell, + nn.LSTMCell: self._count_lstm_cell, + nn.RNN: self._count_rnn, + nn.GRU: self._count_gru, + nn.LSTM: self._count_lstm + } + self._count_bias = False + if mode == 'full': + self.ops.update({ + nn.BatchNorm1d: self._count_bn, + nn.BatchNorm2d: self._count_bn, + nn.BatchNorm3d: self._count_bn, + nn.LeakyReLU: self._count_relu, + nn.AvgPool1d: self._count_avgpool, + nn.AvgPool2d: self._count_avgpool, + nn.AvgPool3d: self._count_avgpool, + nn.AdaptiveAvgPool1d: self._count_adap_avgpool, + nn.AdaptiveAvgPool2d: self._count_adap_avgpool, + nn.AdaptiveAvgPool3d: self._count_adap_avgpool, + nn.Upsample: self._count_upsample, + nn.UpsamplingBilinear2d: self._count_upsample, + nn.UpsamplingNearest2d: self._count_upsample + }) + self._count_bias = True + + if custom_ops is not None: + self.ops.update(custom_ops) + + self.mode = mode + self.results = [] + + def _push_result(self, result): + self.results.append(result) + + def _get_result(self, m, flops): + # assume weight is called `weight`, otherwise it's not applicable + # if user customize the operation, the callback function should + # return the dict result, inluding calculated flops, params and weight_shape. + + result = { + 'flops': flops, + 'params': _get_params(m), + 'weight_shape': tuple(m.weight.size()) if hasattr(m, 'weight') else 0, + } + return result + + def _count_convNd(self, m, x, y): + cin = m.in_channels + kernel_ops = torch.zeros(m.weight.size()[2:]).numel() + output_size = torch.zeros(y.size()[2:]).numel() + cout = y.size()[1] + + if hasattr(m, 'weight_mask'): + cout = m.weight_mask.sum() // (cin * kernel_ops) + + total_ops = cout * output_size * kernel_ops * cin // m.groups # cout x oW x oH + + if self._count_bias: + bias_flops = 1 if m.bias is not None else 0 + total_ops += cout * output_size * bias_flops + + return self._get_result(m, total_ops) + + def _count_linear(self, m, x, y): + out_features = m.out_features + if hasattr(m, 'weight_mask'): + out_features = m.weight_mask.sum() // m.in_features + total_ops = out_features * m.in_features + + if self._count_bias: + bias_flops = 1 if m.bias is not None else 0 + total_ops += out_features * bias_flops + + return self._get_result(m, total_ops) + + def _count_bn(self, m, x, y): + total_ops = 2 * x[0].numel() + return self._get_result(m, total_ops) + + def _count_relu(self, m, x, y): + total_ops = x[0].numel() + return self._get_result(m, total_ops) + + def _count_avgpool(self, m, x, y): + total_ops = y.numel() + return self._get_result(m, total_ops) + + def _count_adap_avgpool(self, m, x, y): + kernel = torch.Tensor([*(x[0].shape[2:])]) // torch.Tensor(list((m.output_size,))).squeeze() + total_add = int(torch.prod(kernel)) + total_div = 1 + kernel_ops = total_add + total_div + num_elements = y.numel() + total_ops = kernel_ops * num_elements + + return self._get_result(m, total_ops) + + def _count_upsample(self, m, x, y): + if m.mode == 'linear': + total_ops = y.nelement() * 5 # 2 muls + 3 add + elif m.mode == 'bilinear': + # https://en.wikipedia.org/wiki/Bilinear_interpolation + total_ops = y.nelement() * 11 # 6 muls + 5 adds + elif m.mode == 'bicubic': + # https://en.wikipedia.org/wiki/Bicubic_interpolation + # Product matrix [4x4] x [4x4] x [4x4] + ops_solve_A = 224 # 128 muls + 96 adds + ops_solve_p = 35 # 16 muls + 12 adds + 4 muls + 3 adds + total_ops = y.nelement() * (ops_solve_A + ops_solve_p) + elif m.mode == 'trilinear': + # https://en.wikipedia.org/wiki/Trilinear_interpolation + # can viewed as 2 bilinear + 1 linear + total_ops = y.nelement() * (13 * 2 + 5) + else: + total_ops = 0 + + return self._get_result(m, total_ops) + + def _count_cell_flops(self, input_size, hidden_size, cell_type): + # h' = \tanh(W_{ih} x + b_{ih} + W_{hh} h + b_{hh}) + total_ops = hidden_size * (input_size + hidden_size) + hidden_size + + if self._count_bias: + total_ops += hidden_size * 2 + + if cell_type == 'rnn': + return total_ops + + if cell_type == 'gru': + # r = \sigma(W_{ir} x + b_{ir} + W_{hr} h + b_{hr}) \\ + # z = \sigma(W_{iz} x + b_{iz} + W_{hz} h + b_{hz}) \\ + # n = \tanh(W_{in} x + b_{in} + r * (W_{hn} h + b_{hn})) \\ + total_ops *= 3 + + # r hadamard : r * (~) + total_ops += hidden_size + + # h' = (1 - z) * n + z * h + # hadamard hadamard add + total_ops += hidden_size * 3 + + elif cell_type == 'lstm': + # i = \sigma(W_{ii} x + b_{ii} + W_{hi} h + b_{hi}) \\ + # f = \sigma(W_{if} x + b_{if} + W_{hf} h + b_{hf}) \\ + # o = \sigma(W_{io} x + b_{io} + W_{ho} h + b_{ho}) \\ + # g = \tanh(W_{ig} x + b_{ig} + W_{hg} h + b_{hg}) \\ + total_ops *= 4 + + # c' = f * c + i * g + # hadamard hadamard add + total_ops += hidden_size * 3 + + # h' = o * \tanh(c') + total_ops += hidden_size + + return total_ops + + + def _count_rnn_cell(self, m, x, y): + total_ops = self._count_cell_flops(m.input_size, m.hidden_size, 'rnn') + batch_size = x[0].size(0) + total_ops *= batch_size + + return self._get_result(m, total_ops) + + def _count_gru_cell(self, m, x, y): + total_ops = self._count_cell_flops(m.input_size, m.hidden_size, 'gru') + batch_size = x[0].size(0) + total_ops *= batch_size + + return self._get_result(m, total_ops) + + def _count_lstm_cell(self, m, x, y): + total_ops = self._count_cell_flops(m.input_size, m.hidden_size, 'lstm') + batch_size = x[0].size(0) + total_ops *= batch_size + + return self._get_result(m, total_ops) + + def _get_bsize_nsteps(self, m, x): + if isinstance(x[0], PackedSequence): + batch_size = torch.max(x[0].batch_sizes) + num_steps = x[0].batch_sizes.size(0) + else: + if m.batch_first: + batch_size = x[0].size(0) + num_steps = x[0].size(1) + else: + batch_size = x[0].size(1) + num_steps = x[0].size(0) + + return batch_size, num_steps + + def _count_rnn_module(self, m, x, y, module_name): + input_size = m.input_size + hidden_size = m.hidden_size + num_layers = m.num_layers + + batch_size, num_steps = self._get_bsize_nsteps(m, x) + total_ops = self._count_cell_flops(input_size, hidden_size, module_name) + + for _ in range(num_layers - 1): + if m.bidirectional: + cell_flops = self._count_cell_flops(hidden_size * 2, hidden_size, module_name) * 2 + else: + cell_flops = self._count_cell_flops(hidden_size, hidden_size,module_name) + total_ops += cell_flops + + total_ops *= num_steps + total_ops *= batch_size + return total_ops + + def _count_rnn(self, m, x, y): + total_ops = self._count_rnn_module(m, x, y, 'rnn') + + return self._get_result(m, total_ops) + + def _count_gru(self, m, x, y): + total_ops = self._count_rnn_module(m, x, y, 'gru') + + return self._get_result(m, total_ops) + + def _count_lstm(self, m, x, y): + total_ops = self._count_rnn_module(m, x, y, 'lstm') + + return self._get_result(m, total_ops) + + + def count_module(self, m, x, y, name): + # assume x is tuple of single tensor + result = self.ops[type(m)](m, x, y) + output_size = y[0].size() if isinstance(y, tuple) else y.size() + + total_result = { + 'name': name, + 'input_size': tuple(x[0].size()), + 'output_size': tuple(output_size), + 'module_type': type(m).__name__, + **result + } + + self._push_result(total_result) + + def sum_flops(self): + return sum([s['flops'] for s in self.results]) + + def sum_params(self): + return sum({s['name']: s['params'] for s in self.results}.values()) + + def format_results(self): + table = PrettyTable() + name_counter = Counter([s['name'] for s in self.results]) + has_multi_use = any(map(lambda v: v > 1, name_counter.values())) + name_counter = Counter() # clear the counter to count from 0 + + headers = [ + 'Index', + 'Name', + 'Type', + 'Weight Shape', + 'FLOPs', + '#Params', + ] + if has_multi_use: + headers.append('#Call') + + table.field_names = headers + for i, result in enumerate(self.results): + row_values = [ + i, + result['name'], + result['module_type'], + str(result['weight_shape']), + result['flops'], + result['params'], + ] + name_counter[result['name']] += 1 + if has_multi_use: + row_values.append(name_counter[result['name']]) + table.add_row(row_values) + return table + + +def count_flops_params(model, x, custom_ops=None, verbose=True, mode='default'): + """ + Count FLOPs and Params of the given model. This function would + identify the mask on the module and take the pruned shape into consideration. + Note that, for sturctured pruning, we only identify the remained filters + according to its mask, and do not take the pruned input channels into consideration, + so the calculated FLOPs will be larger than real number. + + Parameters + --------- + model : nn.Module + Target model. + x : tuple or tensor + The input shape of data (a tuple), a tensor or a tuple of tensor as input data. + custom_ops : dict + A mapping of (module -> torch.nn.Module : custom operation) + the custom operation is a callback funtion to calculate + the module flops and parameters, it will overwrite the default operation. + for reference, please see ``ops`` in ``ModelProfiler``. + verbose : bool + If False, mute detail information about modules. Default is True. + mode : str + the mode of how to collect information. If the mode is set to ``default``, + only the information of convolution and linear will be collected. + If the mode is set to ``full``, other operations will also be collected. + + Returns + ------- + tuple of int, int and dict + Representing total FLOPs, total parameters, and a detailed list of results respectively. + The list of results are a list of dict, each of which contains (name, module_type, weight_shape, + flops, params, input_size, output_size) as its keys. + """ + + assert isinstance(x, tuple) or isinstance(x, torch.Tensor) + assert mode in ['default', 'full'] + + original_device = next(model.parameters()).device + training = model.training + + if isinstance(x, tuple) and all(isinstance(t, int) for t in x): + x = (torch.zeros(x).to(original_device), ) + elif torch.is_tensor(x): + x = (x.to(original_device), ) + else: + x = (t.to(original_device) for t in x) + + handler_collection = [] + profiler = ModelProfiler(custom_ops, mode) + + prev_m = None + for name, m in model.named_modules(): + # dealing with weight mask here + if isinstance(prev_m, PrunerModuleWrapper): + # weight mask is set to weight mask of its parent (wrapper) + weight_mask = prev_m.weight_mask + m.weight_mask = weight_mask + prev_m = m + + if type(m) in profiler.ops: + # if a leaf node + _handler = m.register_forward_hook(functools.partial(profiler.count_module, name=name)) + handler_collection.append(_handler) + + model.eval() + + with torch.no_grad(): + model(*x) + + # restore origin status + model.train(training).to(original_device) + for handler in handler_collection: + handler.remove() + + if verbose: + # get detail information + print(profiler.format_results()) + print(f'FLOPs total: {profiler.sum_flops()}') + print(f'#Params total: {profiler.sum_params()}') + + return profiler.sum_flops(), profiler.sum_params(), profiler.results \ No newline at end of file diff --git a/new_impl/cv/third_party/nni_new/compression/pytorch/utils/mask_conflict.py b/new_impl/cv/third_party/nni_new/compression/pytorch/utils/mask_conflict.py new file mode 100644 index 0000000000000000000000000000000000000000..2ee330c65789a5ef13e0baaf34dcd55545ac307c --- /dev/null +++ b/new_impl/cv/third_party/nni_new/compression/pytorch/utils/mask_conflict.py @@ -0,0 +1,403 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. +import os +import logging +import torch +import numpy as np +from .shape_dependency import ChannelDependency, GroupDependency, CatPaddingDependency, InputChannelDependency +from .utils import get_module_by_name +# logging.basicConfig(level = logging.DEBUG) +_logger = logging.getLogger(__name__) + + +def fix_mask_conflict(masks, model=None, dummy_input=None, traced=None): + """ + MaskConflict fix the mask conflict for the channel dependencies + and group dependency. + + Parameters + ---------- + masks : dict/str + A dict object that stores the masks or the path of the mask file + model : torch.nn.Module + model to fix the mask conflict + dummy_input : torch.Tensor + input example to trace the model + traced : torch._C.torch.jit.TopLevelTracedModule + the traced model of the target model, is this parameter is not None, + we donnot use the model and dummpy_input to get the trace graph. + """ + if isinstance(masks, str): + # if the input is the path of the mask_file + assert os.path.exists(masks) + masks = torch.load(masks) + assert len(masks) > 0, 'Mask tensor cannot be empty' + # if the user uses the model and dummy_input to trace the model, we + # should get the traced model handly, so that, we only trace the + # model once, GroupMaskConflict and ChannelMaskConflict will reuse + # this traced model. + if traced is None: + assert model is not None and dummy_input is not None + training = model.training + model.eval() + # We need to trace the model in eval mode + traced = torch.jit.trace(model, dummy_input) + model.train(training) + + fix_group_mask = GroupMaskConflict(masks, model, dummy_input, traced) + masks = fix_group_mask.fix_mask() + fix_channel_mask = ChannelMaskConflict(masks, model, dummy_input, traced) + masks = fix_channel_mask.fix_mask() + padding_cat_mask = CatMaskPadding(masks, model, dummy_input, traced) + masks = padding_cat_mask.fix_mask() + return masks, fix_channel_mask.conv_prune_dim + + +class MaskFix: + def __init__(self, masks, model=None, dummy_input=None, traced=None): + # check if the parameters are valid + parameter_valid = False + if traced is not None: + parameter_valid = True + elif (model is not None) and (dummy_input is not None): + parameter_valid = True + if not parameter_valid: + raise Exception('The input parameters is invalid!') + self.model = model + self.dummy_input = dummy_input + self.traced = traced + self.masks = masks + + def fix_mask(self): + raise NotImplementedError + + def export(self, path): + """ + Export the masks after fixing the conflict to file. + """ + torch.save(self.masks, path) + + +class CatMaskPadding(MaskFix): + def __init__(self, masks, model, dummy_input=None, traced=None): + """ + CatMaskPadding find the layers whose output tensor is passed + to the same cat operation. The cat operation concatnates the + masks of the input tensors as the output mask, so when some + of the input layers of the cat operation are not pruned, we still + need to pass the masks of these non-pruned layers(the mask are + all ones) to the cat operation to ensure the shape of the output + mask is right. + + Parameters + ---------- + masks : dict + a dict object that stores the masks + model : torch.nn.Module + model to fix the mask conflict + dummy_input : torch.Tensor + input example to trace the model + traced : torch._C.torch.jit.TopLevelTracedModule + the traced model of the target model, is this parameter is not None, + we donnot use the model and dummpy_input to get the trace graph. + """ + super(CatMaskPadding, self).__init__(masks, model, dummy_input, traced) + + def fix_mask(self): + cat_padding_depen = CatPaddingDependency( + self.model, self.dummy_input, self.traced) + name_to_module = {} + for name, module in self.model.named_modules(): + name_to_module[name] = module + depen = cat_padding_depen.dependency_sets + for layers in depen: + device = None + count = 0 + for layer in layers: + if layer in self.masks: + count += 1 + if device is None: + device = self.masks[layer]['weight'].device + if count == 0: + # no layer is pruned + continue + elif count == len(layers): + # all the layers have been pruned + continue + # pad the mask for the non-pruned layers + for layer in layers: + if layer in self.masks: + continue + + module = name_to_module[layer] + w_shape = module.weight.data.size() + w_mask = torch.ones(w_shape).to(device) + b_mask = None + if hasattr(module, 'bias') and module.bias is not None: + # module.bias may be None + b_shape = module.bias.data.size() + b_mask = torch.ones(b_shape).to(device) + self.masks[layer] = {'weight': w_mask, 'bias': b_mask} + + return self.masks + + +class GroupMaskConflict(MaskFix): + def __init__(self, masks, model=None, dummy_input=None, traced=None): + """ + GroupMaskConflict fix the mask conflict between the layers that + has group dependecy with each other. + + Parameters + ---------- + masks : dict + a dict object that stores the masks + model : torch.nn.Module + model to fix the mask conflict + dummy_input : torch.Tensor + input example to trace the model + traced : torch._C.torch.jit.TopLevelTracedModule + the traced model of the target model, is this parameter is not None, + we donnot use the model and dummpy_input to get the trace graph. + """ + super(GroupMaskConflict, self).__init__( + masks, model, dummy_input, traced) + + def fix_mask(self): + """ + Fix the mask conflict before the mask inference for the layers that + has group dependencies. This function should be called before the + mask inference of the 'speedup' module. + """ + group_depen = GroupDependency( + self.model, self.dummy_input, self.traced) + depens = group_depen.dependency + _logger.info(depens) + for layername in depens: + group = depens[layername] + if layername not in self.masks: + # this layer not pruned + continue + w_mask = self.masks[layername]['weight'] + shape = w_mask.size() + count = np.prod(shape[1:]) + all_ones = (w_mask.flatten(1).sum(-1) == count).nonzero(as_tuple=False).squeeze(1).tolist() + all_zeros = (w_mask.flatten(1).sum(-1) == 0).nonzero(as_tuple=False).squeeze(1).tolist() + if len(all_ones) + len(all_zeros) < w_mask.size(0): + # In fine-grained pruning, skip this layer + _logger.info('Layers %s using fine-grained pruning', layername) + continue + assert shape[0] % group == 0 + # Find the number of masked filter for each group (mini_masked). + # Because we have to keep the pruned filter can still + # be divided into the same number of groups, so we only can + # prune mini_masked filters for each group. + step = shape[0] / group + group_masked = [] + for i in range(group): + _start = step * i + _end = step * (i + 1) + _tmp_list = list( + filter(lambda x: _start <= x and x < _end, all_zeros)) + group_masked.append(_tmp_list) + mini_masked = min([len(x) for x in group_masked]) + for gm in group_masked: + for i in range(mini_masked, len(gm)): + # To keep the output channel number still being divisible to + # groups, we set the masks of following filters to be zero. + pos = gm[i] + self.masks[layername]['weight'][pos] = torch.ones( + shape[1:]) + if 'bias' in self.masks[layername] and self.masks[layername]['bias'] is not None: + self.masks[layername]['bias'][pos] = 1 + return self.masks + + +class ChannelMaskConflict(MaskFix): + def __init__(self, masks, model=None, dummy_input=None, traced=None): + """ + ChannelMaskConflict fix the mask conflict between the layers that + has channel dependecy with each other. + + Parameters + ---------- + masks : dict + a dict object that stores the masks + model : torch.nn.Module + model to fix the mask conflict + dummy_input : torch.Tensor + input example to trace the model + graph : torch._C.torch.jit.TopLevelTracedModule + the traced graph of the target model, is this parameter is not None, + we donnot use the model and dummpy_input to get the trace graph. + """ + super(ChannelMaskConflict, self).__init__( + masks, model, dummy_input, traced) + self.conv_prune_dim = detect_mask_prune_dim(masks, model) + _logger.info('detected conv prune dim: %s', self.conv_prune_dim) + + def fix_mask(self): + """ + Fix the mask conflict before the mask inference for the layers that + has shape dependencies. This function should be called before the + mask inference of the 'speedup' module. Only structured pruning masks + are supported. + """ + if self.conv_prune_dim == 0: + channel_depen = ChannelDependency( + self.model, self.dummy_input, self.traced) + else: + channel_depen = InputChannelDependency( + self.model, self.dummy_input, self.traced) + depen_sets = channel_depen.dependency_sets + sum_idx = (1, 2, 3) if self.conv_prune_dim == 0 else (0, 2, 3) + + (_tmp_name, _tmp_tensor) = list(self.masks.items())[0] + device = _tmp_tensor['weight'].device + + for dset in depen_sets: + if len(dset) <= 1: + continue + # channel_masks is a list, each element is None or a vector, for example: + # [[0, 1, 1, 0, 0], [0, 0, 1, 1, 0], None], None means no channel + # is pruned. + channel_masks = [] + fine_grained = False + for name in dset: + if name in self.masks: + _, m = get_module_by_name(self.model, name) + assert m is not None + mask = self.masks[name]['weight'] + if type(m).__name__ == 'Conv2d': + channel_mask = (mask.abs().sum(sum_idx) != 0).int() + channel_masks.append(channel_mask) + if (channel_mask.sum() * (mask.numel() / mask.shape[self.conv_prune_dim])).item() != (mask > 0).sum().item(): + fine_grained = True + elif type(m).__name__ == 'Linear': + channel_masks.append((mask.abs().sum(0) != 0).int()) + elif type(m).__name__ == 'BatchNorm2d': + channel_masks.append(mask.int()) + elif type(m).__name__ == 'ConvTranspose2d': + # convtranspose have difference memory layout, so that we need create + # a tmp_sum_idx for conv_transpose + tmp_sum_idx = ( + 0, 2, 3) if self.conv_prune_dim == 0 else (1, 2, 3) + channel_mask = (mask.abs().sum(tmp_sum_idx) != 0).int() + channel_masks.append(channel_mask) + if (channel_mask.sum() * (mask.numel() / mask.shape[1 - self.conv_prune_dim])).item() != (mask > 0).sum().item(): + fine_grained = True + else: + raise RuntimeError( + f'unsupported module type: {type(m).__name__}') + else: + # no mask means not pruned, equivlent to full masks + channel_masks.append(None) + if fine_grained: + _logger.info( + 'fine-grained mask detected, skip solving conflict for this set: %s', dset) + continue + if all(x is None for x in channel_masks): + continue + num_channels_list = [len(x) + for x in channel_masks if x is not None] + # number of channels in same set should be identical + assert len(set(num_channels_list)) == 1 + num_channels = num_channels_list[0] + + for i, dim_mask in enumerate(channel_masks): + if dim_mask is None: + channel_masks[i] = torch.ones(num_channels).int().to(device) + + # merge masks with 'or' + merged_channel_mask = channel_masks[0].clone() + for i in range(1, len(channel_masks)): + merged_channel_mask = ( + (merged_channel_mask + channel_masks[i]) != 0).int() + + merged_index = torch.nonzero(merged_channel_mask, as_tuple=True)[0] + + for name in dset: + if name not in self.masks: + assert all(merged_channel_mask) + continue + orig_mask = self.masks[name]['weight'] + _, m = get_module_by_name(self.model, name) + new_mask = torch.zeros_like(orig_mask) + if type(m).__name__ == 'Conv2d': + if self.conv_prune_dim == 0: + new_mask[merged_index, :, :, :] = 1. + else: + new_mask[:, merged_index, :, :] = 1. + elif type(m).__name__ == 'Linear': + new_mask[:, merged_index] = 1. + elif type(m).__name__ == 'BatchNorm2d': + new_mask = merged_channel_mask.type_as(orig_mask) + else: + raise RuntimeError( + f'unsupported module type: {type(m).__name__}') + + self.masks[name]['weight'] = new_mask + if 'bias' in self.masks[name] and self.masks[name]['bias'] is not None: + if type(m).__name__ == 'Conv2d': + assert self.conv_prune_dim == 0 + self.masks[name]['bias'] = merged_channel_mask.type_as( + self.masks[name]['bias']) + + return self.masks + + +def detect_mask_prune_dim(masks, model): + """ + Detect how the masks of convolutional layers are pruned. + + Parameters + ---------- + masks: dict + A dict object that stores the masks. + model: nn.Module + Model object which the mask can be applied on. + + Returns: + ------- + How the masks of convolutional layers are pruned, this depends on pruning algorithms, it should + return 1 for masks generated by AMCPruner, and returns 0 for masks generated by the rest + NNI builtin pruners. + 0: filter pruning, prune filters of weights which causes channels of output feature maps are pruned. + 1: channel pruning, prune kernels corresponding to each input channels which causes channels of + input feature maps are pruned. + """ + dim0_preserved, dim1_preserved = 0., 0. + dim0_num, dim1_num = 0., 0. + for module_name in masks: + _, m = get_module_by_name(model, module_name) + if m is None or type(m).__name__ != 'Conv2d': + continue + + mask = masks[module_name]['weight'].clone() + assert (mask >= 0).sum() == mask.numel(), \ + "mask values should be greater than or equal to 0." + mask = (mask > 0).int() + mask = mask.view(mask.shape[0], mask.shape[1], -1) + dim0_mask = (mask.sum((1, 2)) > 0).int() + dim1_mask = (mask.sum((0, 2)) > 0).int() + dim0_preserved += dim0_mask.sum().item() + dim1_preserved += dim1_mask.sum().item() + dim0_num += len(dim0_mask) + dim1_num += len(dim1_mask) + + if dim0_num == 0 or dim1_num == 0: + _logger.warning('no multi-dimension masks found.') + return 0 + + dim0_sparsity, dim1_sparsity = 1. - dim0_preserved / \ + dim0_num, 1. - dim1_preserved / dim1_num + _logger.info('dim0 sparsity: %f', dim0_sparsity) + _logger.info('dim1 sparsity: %f', dim1_sparsity) + + if dim0_sparsity == dim1_sparsity == 0.: + _logger.warning('nothing masked.') + + if dim0_sparsity > 0 and dim1_sparsity > 0: + _logger.warning('both dim0 and dim1 masks found.') + + return 0 if dim0_sparsity >= dim1_sparsity else 1 diff --git a/new_impl/cv/third_party/nni_new/compression/pytorch/utils/num_param_counter.py b/new_impl/cv/third_party/nni_new/compression/pytorch/utils/num_param_counter.py new file mode 100644 index 0000000000000000000000000000000000000000..89ad0979943126c45112d8b865d87ec12cdb1961 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/compression/pytorch/utils/num_param_counter.py @@ -0,0 +1,16 @@ +def get_total_num_weights(model, op_types=['default']): + ''' + calculate the total number of weights + + Returns + ------- + int + total weights of all the op considered + ''' + num_weights = 0 + for _, module in model.named_modules(): + if module == model: + continue + if 'default' in op_types or type(module).__name__ in op_types: + num_weights += module.weight.data.numel() + return num_weights \ No newline at end of file diff --git a/new_impl/cv/third_party/nni_new/compression/pytorch/utils/sensitivity_analysis.py b/new_impl/cv/third_party/nni_new/compression/pytorch/utils/sensitivity_analysis.py new file mode 100644 index 0000000000000000000000000000000000000000..a36a523feb1e3d7e7959d00ad01c8705af4f9373 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/compression/pytorch/utils/sensitivity_analysis.py @@ -0,0 +1,249 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import copy +import csv +import logging +from collections import OrderedDict + +import numpy as np +import torch.nn as nn + +# FIXME: I don't know where "utils" should be +SUPPORTED_OP_NAME = ['Conv2d', 'Conv1d'] +SUPPORTED_OP_TYPE = [getattr(nn, name) for name in SUPPORTED_OP_NAME] + +logger = logging.getLogger('Sensitivity_Analysis') +logger.setLevel(logging.INFO) + + +class SensitivityAnalysis: + def __init__(self, model, val_func, sparsities=None, prune_type='l1', early_stop_mode=None, early_stop_value=None): + """ + Perform sensitivity analysis for this model. + Parameters + ---------- + model : torch.nn.Module + the model to perform sensitivity analysis + val_func : function + validation function for the model. Due to + different models may need different dataset/criterion + , therefore the user need to cover this part by themselves. + In the val_func, the model should be tested on the validation dateset, + and the validation accuracy/loss should be returned as the output of val_func. + There are no restrictions on the input parameters of the val_function. + User can use the val_args, val_kwargs parameters in analysis + to pass all the parameters that val_func needed. + sparsities : list + The sparsity list provided by users. This parameter is set when the user + only wants to test some specific sparsities. In the sparsity list, each element + is a sparsity value which means how much weight the pruner should prune. Take + [0.25, 0.5, 0.75] for an example, the SensitivityAnalysis will prune 25% 50% 75% + weights gradually for each layer. + prune_type : str + The pruner type used to prune the conv layers, default is 'l1', + and 'l2', 'fine-grained' is also supported. + early_stop_mode : str + If this flag is set, the sensitivity analysis + for a conv layer will early stop when the validation metric( + for example, accurracy/loss) has alreay meet the threshold. We + support four different early stop modes: minimize, maximize, dropped, + raised. The default value is None, which means the analysis won't stop + until all given sparsities are tested. This option should be used with + early_stop_value together. + + minimize: The analysis stops when the validation metric return by the val_func + lower than early_stop_value. + maximize: The analysis stops when the validation metric return by the val_func + larger than early_stop_value. + dropped: The analysis stops when the validation metric has dropped by early_stop_value. + raised: The analysis stops when the validation metric has raised by early_stop_value. + early_stop_value : float + This value is used as the threshold for different earlystop modes. + This value is effective only when the early_stop_mode is set. + + """ + from nni.algorithms.compression.pytorch.pruning.constants_pruner import PRUNER_DICT + + self.model = model + self.val_func = val_func + self.target_layer = OrderedDict() + self.ori_state_dict = copy.deepcopy(self.model.state_dict()) + self.target_layer = {} + self.sensitivities = {} + if sparsities is not None: + self.sparsities = sorted(sparsities) + else: + self.sparsities = np.arange(0.1, 1.0, 0.1) + self.sparsities = [np.round(x, 2) for x in self.sparsities] + self.Pruner = PRUNER_DICT[prune_type] + self.early_stop_mode = early_stop_mode + self.early_stop_value = early_stop_value + self.ori_metric = None # original validation metric for the model + # already_pruned is for the iterative sensitivity analysis + # For example, sensitivity_pruner iteratively prune the target + # model according to the sensitivity. After each round of + # pruning, the sensitivity_pruner will test the new sensitivity + # for each layer + self.already_pruned = {} + self.model_parse() + + @property + def layers_count(self): + return len(self.target_layer) + + def model_parse(self): + for name, submodel in self.model.named_modules(): + for op_type in SUPPORTED_OP_TYPE: + if isinstance(submodel, op_type): + self.target_layer[name] = submodel + self.already_pruned[name] = 0 + + def _need_to_stop(self, ori_metric, cur_metric): + """ + Judge if meet the stop conditon(early_stop, min_threshold, + max_threshold). + Parameters + ---------- + ori_metric : float + original validation metric + cur_metric : float + current validation metric + + Returns + ------- + stop : bool + if stop the sensitivity analysis + """ + if self.early_stop_mode is None: + # early stop mode is not enable + return False + assert self.early_stop_value is not None + if self.early_stop_mode == 'minimize': + if cur_metric < self.early_stop_value: + return True + elif self.early_stop_mode == 'maximize': + if cur_metric > self.early_stop_value: + return True + elif self.early_stop_mode == 'dropped': + if cur_metric < ori_metric - self.early_stop_value: + return True + elif self.early_stop_mode == 'raised': + if cur_metric > ori_metric + self.early_stop_value: + return True + return False + + def analysis(self, val_args=None, val_kwargs=None, specified_layers=None): + """ + This function analyze the sensitivity to pruning for + each conv layer in the target model. + If start and end are not set, we analyze all the conv + layers by default. Users can specify several layers to + analyze or parallelize the analysis process easily through + the start and end parameter. + + Parameters + ---------- + val_args : list + args for the val_function + val_kwargs : dict + kwargs for the val_funtion + specified_layers : list + list of layer names to analyze sensitivity. + If this variable is set, then only analyze + the conv layers that specified in the list. + User can also use this option to parallelize + the sensitivity analysis easily. + Returns + ------- + sensitivities : dict + dict object that stores the trajectory of the + accuracy/loss when the prune ratio changes + """ + if val_args is None: + val_args = [] + if val_kwargs is None: + val_kwargs = {} + # Get the original validation metric(accuracy/loss) before pruning + # Get the accuracy baseline before starting the analysis. + self.ori_metric = self.val_func(*val_args, **val_kwargs) + namelist = list(self.target_layer.keys()) + if specified_layers is not None: + # only analyze several specified conv layers + namelist = list(filter(lambda x: x in specified_layers, namelist)) + for name in namelist: + self.sensitivities[name] = {} + for sparsity in self.sparsities: + # here the sparsity is the relative sparsity of the + # the remained weights + # Calculate the actual prune ratio based on the already pruned ratio + real_sparsity = ( + 1.0 - self.already_pruned[name]) * sparsity + self.already_pruned[name] + # TODO In current L1/L2 Filter Pruner, the 'op_types' is still necessary + # I think the L1/L2 Pruner should specify the op_types automaticlly + # according to the op_names + cfg = [{'sparsity': real_sparsity, 'op_names': [ + name], 'op_types': ['Conv2d']}] + pruner = self.Pruner(self.model, cfg) + pruner.compress() + val_metric = self.val_func(*val_args, **val_kwargs) + logger.info('Layer: %s Sparsity: %.2f Validation Metric: %.4f', + name, real_sparsity, val_metric) + + self.sensitivities[name][sparsity] = val_metric + pruner._unwrap_model() + del pruner + # check if the current metric meet the stop condition + if self._need_to_stop(self.ori_metric, val_metric): + break + + # reset the weights pruned by the pruner, because the + # input sparsities is sorted, so we donnot need to reset + # weight of the layer when the sparsity changes, instead, + # we only need reset the weight when the pruning layer changes. + self.model.load_state_dict(self.ori_state_dict) + + return self.sensitivities + + def export(self, filepath): + """ + Export the results of the sensitivity analysis + to a csv file. The firstline of the csv file describe the content + structure. The first line is constructed by 'layername' and sparsity + list. Each line below records the validation metric returned by val_func + when this layer is under different sparsities. Note that, due to the early_stop + option, some layers may not have the metrics under all sparsities. + + layername, 0.25, 0.5, 0.75 + conv1, 0.6, 0.55 + conv2, 0.61, 0.57, 0.56 + + Parameters + ---------- + filepath : str + Path of the output file + """ + str_sparsities = [str(x) for x in self.sparsities] + header = ['layername'] + str_sparsities + with open(filepath, 'w') as csvf: + csv_w = csv.writer(csvf) + csv_w.writerow(header) + for layername in self.sensitivities: + row = [] + row.append(layername) + for sparsity in sorted(self.sensitivities[layername].keys()): + row.append(self.sensitivities[layername][sparsity]) + csv_w.writerow(row) + + def update_already_pruned(self, layername, ratio): + """ + Set the already pruned ratio for the target layer. + """ + self.already_pruned[layername] = ratio + + def load_state_dict(self, state_dict): + """ + Update the weight of the model + """ + self.ori_state_dict = copy.deepcopy(state_dict) + self.model.load_state_dict(self.ori_state_dict) diff --git a/new_impl/cv/third_party/nni_new/compression/pytorch/utils/shape_dependency.py b/new_impl/cv/third_party/nni_new/compression/pytorch/utils/shape_dependency.py new file mode 100644 index 0000000000000000000000000000000000000000..1d3d087def1c0c69230ecc3f2e82db53a28bb557 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/compression/pytorch/utils/shape_dependency.py @@ -0,0 +1,503 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import csv +import logging + +__all__ = ['ChannelDependency', 'GroupDependency', + 'CatPaddingDependency', 'InputChannelDependency'] + +CONV_TYPE = 'aten::_convolution' +ADD_TYPES = ['aten::add', 'aten::add_'] +CAT_TYPE = 'aten::cat' +logger = logging.getLogger('Shape_Dependency') +RESHAPE_OPS = [CAT_TYPE, 'aten::view', + 'aten::reshape', 'aten::flatten', 'aten::mean'] + + +class Dependency: + def __init__(self, model=None, dummy_input=None, traced_model=None): + """ + Build the graph for the model. + """ + from ....common.graph_utils import TorchModuleGraph + + # check if the input is legal + if traced_model is None: + # user should provide model & dummy_input to trace + # the model or a already traced model + assert model is not None and dummy_input is not None + self.graph = TorchModuleGraph(model, dummy_input, traced_model) + self.dependency = dict() + self.build_dependency() + + def build_dependency(self): + raise NotImplementedError + + def export(self, filepath): + raise NotImplementedError + + +class ChannelDependency(Dependency): + def __init__(self, model=None, dummy_input=None, traced_model=None): + """ + This model analyze the channel dependencies between the conv + layers in a model. + + Parameters + ---------- + model : torch.nn.Module + The model to be analyzed. + data : torch.Tensor + The example input data to trace the network architecture. + traced_model : torch._C.Graph + if we alreay has the traced graph of the target model, we donnot + need to trace the model again. + """ + super(ChannelDependency, self).__init__( + model, dummy_input, traced_model) + + def _get_parent_layers(self, node): + """ + Find the nearest father conv layers for the target node. + + Parameters + --------- + node : torch._C.Node + target node. + + Returns + ------- + parent_layers: list + nearest father conv/linear layers for the target worknode. + """ + parent_layers = [] + queue = [] + queue.append(node) + while queue: + curnode = queue.pop(0) + if curnode.op_type == 'Conv2d' or curnode.op_type == 'Linear' or curnode.op_type == 'ConvTranspose2d': + # find the first met conv + parent_layers.append(curnode.name) + continue + parents = self.graph.find_predecessors(curnode.unique_name) + parents = [self.graph.name_to_node[name] for name in parents] + for parent in parents: + queue.append(parent) + return parent_layers + + def build_dependency(self): + """ + Build the channel dependency for the conv layers + in the model. + """ + # unpack the tuple/list manually before analyze the + # channel dependency + self.graph.unpack_manually() + for node in self.graph.nodes_py.nodes_op: + parent_layers = [] + # find the node that contains aten::add + # or aten::cat operations + if node.op_type in ADD_TYPES: + parent_layers = self._get_parent_layers(node) + elif node.op_type == CAT_TYPE: + # To determine if this cat operation will introduce channel + # dependency, we need the specific input parameters of the cat + # opertion. To get the input parameters of the cat opertion, we + # need to traverse all the cpp_nodes included by this NodePyGroup, + # because, TorchModuleGraph merges the important nodes and the adjacent + # unimportant nodes (nodes started with prim::attr, for example) into a + # NodepyGroup. + cat_dim = None + for cnode in node.node_cpps: + if cnode.kind() == CAT_TYPE: + cat_dim = list(cnode.inputs())[1].toIValue() + break + if cat_dim != 1: + parent_layers = self._get_parent_layers(node) + dependency_set = set(parent_layers) + # merge the dependencies + for parent in parent_layers: + if parent in self.dependency: + dependency_set.update(self.dependency[parent]) + # save the dependencies + for _node in dependency_set: + self.dependency[_node] = dependency_set + + def export(self, filepath): + """ + export the channel dependencies as a csv file. + The layers at the same line have output channel + dependencies with each other. For example, + layer1.1.conv2, conv1, and layer1.0.conv2 have + output channel dependencies with each other, which + means the output channel(filters) numbers of these + three layers should be same with each other, otherwise + the model may has shape conflict. + + Output example: + Dependency Set,Convolutional Layers + Set 1,layer1.1.conv2,layer1.0.conv2,conv1 + Set 2,layer1.0.conv1 + Set 3,layer1.1.conv1 + """ + header = ['Dependency Set', 'Layers'] + setid = 0 + visited = set() + with open(filepath, 'w') as csvf: + csv_w = csv.writer(csvf, delimiter=',') + csv_w.writerow(header) + for node in self.graph.nodes_py.nodes_op: + if node.op_type != 'Conv2d' or node in visited: + continue + setid += 1 + row = ['Set %d' % setid] + if node.name not in self.dependency: + visited.add(node) + row.append(node.name) + else: + for other in self.dependency[node.name]: + visited.add(self.graph.name_to_node[other]) + row.append(other) + csv_w.writerow(row) + + @property + def dependency_sets(self): + """ + Get the list of the dependency set. + + Returns + ------- + dependency_sets : list + list of the dependency sets. For example, + [set(['conv1', 'conv2']), set(['conv3', 'conv4'])] + + """ + d_sets = [] + visited = set() + for node in self.graph.nodes_py.nodes_op: + if node.op_type != 'Conv2d' or node in visited: + continue + tmp_set = set() + if node.name not in self.dependency: + visited.add(node) + tmp_set.add(node.name) + else: + for other in self.dependency[node.name]: + visited.add(self.graph.name_to_node[other]) + tmp_set.add(other) + d_sets.append(tmp_set) + return d_sets + + +def reshape_break_channel_dependency(op_node): + """ + The reshape operations such as (reshape, view, flatten) may break + the channel dependency. We need to check the input parameters of + these reshape operations to check if this reshape node will break + the channel dependency. However, it's complicated to analyze the the input + parameters for each reshape function and infer if it will break the channel + dependency. So currently, we just check if the input channel and the output + channel is the same, if so, then we can say the original reshape function + doesn't want to change the number of the channels, which means the channel + dependency is not broken. In contrast, the original reshap operation wants + to change the number of channels, so it breaks the channel dependency. + + Parameters + ---------- + opnode: NodePyOP + A Op node of the graph. + Returns + ------- + bool + If this operation will break the channel dependency. + """ + in_shape = op_node.auxiliary['in_shape'] + out_shape = op_node.auxiliary['out_shape'] + in_channel = in_shape[1] + out_channel = out_shape[1] + return in_channel != out_channel + + +class InputChannelDependency(ChannelDependency): + """ + Some pruners may prune the input channel of the convolutional + layers. While pruning the input channel of the convolutional layers, + the layers that share the same input tensor should prune the same + channels, and we say these layers that share the same input tensor/channel + has the input channel dependency. If we only prune the input channel of one + layer in the dependency set, there will be a shape conflict for the other + layers in the same dependency set, which may trigger a runtime error. + Here we judge whether the application will truncate the dependency by analyzing + whether the number of channels before and after the operation has changed. + If not, the input channel dependency will be passed to the following nodes. + """ + + def __init__(self, model, dummy_input=None, traced_model=None): + """ + This model analyze the input channel dependencies between the conv + layers in a model. + + Parameters + ---------- + model : torch.nn.Module + The model to be analyzed. + data : torch.Tensor + The example input data to trace the network architecture. + traced_model : torch._C.Graph + if we alreay has the traced graph of the target model, we donnot + need to trace the model again. + """ + super(InputChannelDependency, self).__init__( + model, dummy_input, traced_model) + + def _get_following_convs(self, tensor): + queue = [] + key_layers = [] + queue.extend(self.graph.input_to_node[tensor]) + while queue: + curnode = queue.pop(0) + if curnode.op_type == 'Conv2d' or curnode.op_type == 'Linear' or curnode.op_type == 'ConvTranspose2d': + # find the first met conv + key_layers.append(curnode.name) + continue + elif curnode.op_type in RESHAPE_OPS: + # check if the reshape operation will break the channel dependency + if reshape_break_channel_dependency(curnode): + # reshape operations also breaks the dependency relationship + continue + successors = self.graph.find_successors(curnode.unique_name) + successors = [self.graph.name_to_node[name] for name in successors] + for layer in successors: + queue.append(layer) + return key_layers + + def build_dependency(self): + """ + Build the input channel dependencies. + The `InputChannelDependency` indicates the layers that have + dependencies when pruning the input channel of the conv layers. + In contrast, `ChannelDependency` indicates the dependent layers + when pruning the output channles of conv layers (for example, L1FilterPruner). + """ + # unpack the tuple or list manually + self.graph.unpack_manually() + for tensor in self.graph.input_to_node: + # start from this tensor, find all the conv layers that + # take this tensor as input. Similar to the `ChannelDependency` + # the conv layer will truncate the dependencies + layers = self._get_following_convs(tensor) + dependency_set = set(layers) + for layer in layers: + if layer in self.dependency: + dependency_set.update(self.dependency[layer]) + for layer in dependency_set: + self.dependency[layer] = dependency_set + + +class CatPaddingDependency(ChannelDependency): + def __init__(self, model=None, dummy_input=None, traced_model=None): + super(CatPaddingDependency, self).__init__( + model, dummy_input, traced_model) + + def build_dependency(self): + """ + Build the cat padding dependencies. + If the output features of several layers are stitched together + by cat operation, then these layers have cat padding dependencies. + This is because when inferring the cat mask, we need all the input + masks for the cat operation. At this time we need to know the source + of all input vectors of a cat operation. + """ + for node in self.graph.nodes_py.nodes_op: + parent_layers = [] + if node.op_type == CAT_TYPE: + parent_layers = self._get_parent_layers(node) + dependency_set = set(parent_layers) + # merge the dependencies + for parent in parent_layers: + if parent in self.dependency: + dependency_set.update(self.dependency[parent]) + # save the dependencies + for _node in dependency_set: + self.dependency[_node] = dependency_set + + @property + def dependency_sets(self): + d_sets = [] + visited = set() + for nodename in self.dependency: + if nodename in visited: + continue + d_sets.append(self.dependency[nodename]) + return d_sets + + def export(self, filepath): + """ + Export the dependencies into a file. + In the output file, each line contains a set of layers + whose output features are stitched together by the cat + operation. + + output example: + Dependency Set, Layers + set1, Conv1, Conv2 + set2, Conv3, Conv4 + """ + header = ['Dependency Set', 'Layers'] + setid = 0 + with open(filepath, 'w') as csvf: + csv_w = csv.writer(csvf, delimiter=',') + csv_w.writerow(header) + for layers in self.dependency_sets: + setid += 1 + row = ['Set %d' % setid] + row.extend(list(layers)) + csv_w.writerow(row) + + +class GroupDependency(Dependency): + def __init__(self, model=None, dummy_input=None, traced_model=None): + """ + This model analyze the group dependencis between the conv + layers in a model. + + Parameters + ---------- + model : torch.nn.Module + The model to be analyzed. + data : torch.Tensor + The example input data to trace the network architecture. + traced_model : torch._C.Graph + if we alreay has the traced graph of the target model, we donnot + need to trace the model again. + """ + super(GroupDependency, self).__init__(model, dummy_input, traced_model) + + def _get_parent_convs(self, node): + """ + Find the nearest father conv layers for the target node. + + Parameters + --------- + node : torch._C.Node + target node. + + Returns + ------- + parent_layers : list + nearest father conv layers for the target node. Due to the group + dependency only exists between the conv layers, so we only find + the parent conv layers. + """ + parent_layers = [] + # the input node is a Conv node + predeessors = self.graph.find_predecessors(node.unique_name) + predeessors = [self.graph.name_to_node[x] for x in predeessors] + queue = predeessors + while queue: + curnode = queue.pop(0) + if curnode.op_type == 'Conv2d' or curnode.op_type == 'ConvTranspose2d': + # find the first met conv + parent_layers.append(curnode.name) + continue + parents = self.graph.find_predecessors(curnode.unique_name) + parents = [self.graph.name_to_node[name] for name in parents] + for parent in parents: + queue.append(parent) + return parent_layers + + def _get_conv_groups(self, node_group): + """ + Get the number of groups for a convolutional layer. + + Parameters + ---------- + node_group : NodePyGroup + target node. + + Returns + ------- + group : int + the number of the groups of the target conv layer. + """ + cpp_conv = list(filter(lambda x: x.kind() == + CONV_TYPE, node_group.node_cpps)) + assert len(cpp_conv) == 1 + cpp_conv = cpp_conv[0] + inputs = list(cpp_conv.inputs()) + # get the number of the group from the input parameters + group = inputs[8].toIValue() + return group + + def build_dependency(self): + """ + Build the channel dependency for the conv layers + in the model. This function return the group number + of each conv layers. Note that, here, the group count + of conv layers may be larger than their originl groups. + This is because that the input channel will also be grouped + for the group conv layers. To make this clear, assume we + have two group conv layers: conv1(group=2), conv2(group=4). + conv2 takes the output features of conv1 as input. + Then we have to the filters of conv1 can still be + divided into 4 groups after filter pruning, because + the input channels of conv2 shoule be divided into + 4 groups. + + Returns + ------- + self.dependency : dict + key: the name of conv layers, value: the minimum value that the number of + filters should be divisible to. + """ + for node in self.graph.nodes_py.nodes_op: + if node.op_type == 'Conv2d' or node.op_type == 'ConvTranspose2d': + group = self._get_conv_groups(node) + + if node.name in self.dependency: + # the conv layer whose group is larger than 1 will require that + # it's number of output channel to be divisible by the number of group. + self.dependency[node.name] = max( + self.dependency[node.name], group) + else: + self.dependency[node.name] = group + if group > 1: + # for the conv layer whose group is larger than 1, it will require the number + # of output channels of their parent conv layer to be divisible by group. + parent_convs = self._get_parent_convs(node) + for parent in parent_convs: + if parent in self.dependency: + self.dependency[parent] = max( + self.dependency[parent], group) + else: + self.dependency[parent] = group + return self.dependency + + def export(self, filepath): + """ + export the group dependency to a csv file. + Each line describes a convolution layer, the + first part of each line is the Pytorch module + name of the conv layer. The second part of each + line is the group count of the filters in this layer. + Note that, the group count may be larger than this + layers original group number. + + output example: + Conv layer, Groups + Conv1, 1 + Conv2, 2 + Conv3, 4 + """ + header = ['Conv Layer Name', 'Group'] + with open(filepath, 'w') as csvf: + csv_w = csv.writer(csvf, delimiter=',') + csv_w.writerow(header) + for name in self.dependency: + group = self.dependency[name] + csv_w.writerow([name, group]) + + @property + def dependency_sets(self): + return self.dependency diff --git a/new_impl/cv/third_party/nni_new/compression/pytorch/utils/utils.py b/new_impl/cv/third_party/nni_new/compression/pytorch/utils/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..c687c5e2a6bf971433e6298ee92a86be5c23b6d4 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/compression/pytorch/utils/utils.py @@ -0,0 +1,30 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +def get_module_by_name(model, module_name): + """ + Get a module specified by its module name + + Parameters + ---------- + model : pytorch model + the pytorch model from which to get its module + module_name : str + the name of the required module + + Returns + ------- + module, module + the parent module of the required module, the required module + """ + name_list = module_name.split(".") + for name in name_list[:-1]: + if hasattr(model, name): + model = getattr(model, name) + else: + return None, None + if hasattr(model, name_list[-1]): + leaf_module = getattr(model, name_list[-1]) + return model, leaf_module + else: + return None, None diff --git a/new_impl/cv/third_party/nni_new/compression/tensorflow/__init__.py b/new_impl/cv/third_party/nni_new/compression/tensorflow/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d05fade2f1140c68bb78969f89aaa8529414f3ca --- /dev/null +++ b/new_impl/cv/third_party/nni_new/compression/tensorflow/__init__.py @@ -0,0 +1,4 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .compressor import Compressor, Pruner diff --git a/new_impl/cv/third_party/nni_new/compression/tensorflow/compressor.py b/new_impl/cv/third_party/nni_new/compression/tensorflow/compressor.py new file mode 100644 index 0000000000000000000000000000000000000000..bb249807e77c8ad14423dad8c6d09321b64f9e21 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/compression/tensorflow/compressor.py @@ -0,0 +1,338 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +Abstract base classes for TensorFlow model compression. +""" + +import logging + +import tensorflow as tf +assert tf.__version__.startswith('2'), 'NNI model compression only supports TensorFlow v2.x' + +from . import default_layers + +_logger = logging.getLogger(__name__) + + +class Compressor: + """ + Common base class for all compressors. + + This class is designed for other base classes. + Algorithms should inherit ``Pruner`` or ``Quantizer`` instead. + + Attributes + ---------- + compressed_model : tf.keras.Model + Compressed user model. + wrappers : list of tf.keras.Model + A wrapper is an instrumented TF ``Layer``, in ``Model`` format. + + Parameters + ---------- + model : tf.keras.Model + The user model to be compressed. + config_list : list of JSON object + User configuration. The format is detailed in tutorial. + LayerWrapperClass : a class derive from Model + The class used to instrument layers. + """ + + def __init__(self, model, config_list, LayerWrapperClass): + assert isinstance(model, tf.keras.Model) + self.validate_config(model, config_list) + + self._original_model = model + self._config_list = config_list + self._wrapper_class = LayerWrapperClass + self._wrappers = {} # key: id(layer) , value: Wrapper(layer) + + self.compressed_model = self._instrument(model) + self.wrappers = list(self._wrappers.values()) + + if not self.wrappers: + _logger.warning('Nothing is configured to compress, please check your model and config list') + + def set_wrappers_attribute(self, name, value): + """ + Call ``setattr`` on all wrappers. + """ + for wrapper in self.wrappers: + setattr(wrapper, name, value) + + def validate_config(self, model, config_list): + """ + Compression algorithm should overload this function to validate configuration. + """ + pass + + + def _instrument(self, layer): + if isinstance(layer, tf.keras.Sequential): + return self._instrument_sequential(layer) + if isinstance(layer, tf.keras.Model): + return self._instrument_model(layer) + + # a layer can be referenced in multiple attributes of a model, + # but should only be instrumented once + if id(layer) in self._wrappers: + return self._wrappers[id(layer)] + + config = self._select_config(layer) + if config is not None: + wrapper = self._wrapper_class(layer, config, self) + self._wrappers[id(layer)] = wrapper + return wrapper + + return layer + + def _uninstrument(self, layer): + # note that ``self._wrappers`` cache is not cleared here, + # so the same wrapper objects will be recovered in next ``self._instrument()`` call + if isinstance(layer, LayerWrapper): + layer._instrumented = False + return self._uninstrument(layer.layer) + if isinstance(layer, tf.keras.Sequential): + return self._uninstrument_sequential(layer) + if isinstance(layer, tf.keras.Model): + return self._uninstrument_model(layer) + return layer + + def _instrument_sequential(self, seq): + layers = list(seq.layers) # seq.layers is read-only property + need_rebuild = False + for i, layer in enumerate(layers): + new_layer = self._instrument(layer) + if new_layer is not layer: + layers[i] = new_layer + need_rebuild = True + return tf.keras.Sequential(layers) if need_rebuild else seq + + def _uninstrument_sequential(self, seq): + layers = list(seq.layers) + rebuilt = False + for i, layer in enumerate(layers): + orig_layer = self._uninstrument(layer) + if orig_layer is not layer: + layers[i] = orig_layer + rebuilt = True + return tf.keras.Sequential(layers) if rebuilt else seq + + def _instrument_model(self, model): + for key, value in list(model.__dict__.items()): # avoid "dictionary keys changed during iteration" + if isinstance(value, tf.keras.layers.Layer): + new_layer = self._instrument(value) + if new_layer is not value: + setattr(model, key, new_layer) + elif isinstance(value, list): + for i, item in enumerate(value): + if isinstance(item, tf.keras.layers.Layer): + value[i] = self._instrument(item) + return model + + def _uninstrument_model(self, model): + for key, value in list(model.__dict__.items()): + if isinstance(value, tf.keras.layers.Layer): + orig_layer = self._uninstrument(value) + if orig_layer is not value: + setattr(model, key, orig_layer) + elif isinstance(value, list): + for i, item in enumerate(value): + if isinstance(item, tf.keras.layers.Layer): + value[i] = self._uninstrument(item) + return model + + def _select_config(self, layer): + # Find the last matching config block for given layer. + # Returns None if the layer should not be compressed. + layer_type = type(layer).__name__ + last_match = None + for config in self._config_list: + if 'op_types' in config: + match = layer_type in config['op_types'] + match_default = 'default' in config['op_types'] and layer_type in default_layers.weighted_modules + if not match and not match_default: + continue + if 'op_names' in config and layer.name not in config['op_names']: + continue + last_match = config + if last_match is None or 'exclude' in last_match: + return None + return last_match + + +class LayerWrapper(tf.keras.Model): + """ + Abstract base class of layer wrappers. + + Concrete layer wrapper classes must inherit this to support ``isinstance`` check. + """ + def __init__(self): + super().__init__() + self._instrumented = True + + +class Pruner(Compressor): + """ + Base class for pruning algorithms. + + End users should use ``compress`` and callback APIs (WIP) to prune their models. + + The underlying model is instrumented upon initialization of pruner object. + So if you want to pre-train the model, train it before creating pruner object. + + The compressed model can only execute in eager mode. + + Algorithm developers should override ``calc_masks`` method to specify pruning strategy. + + Parameters + ---------- + model : tf.keras.Model + The user model to prune. + config_list : list of JSON object + User configuration. The format is detailed in tutorial. + """ + def __init__(self, model, config_list): + super().__init__(model, config_list, PrunerLayerWrapper) + #self.callback = PrunerCallback(self) + + def compress(self): + """ + Apply compression on a pre-trained model. + + If you want to prune the model during training, use callback API (WIP) instead. + + Returns + ------- + tf.keras.Model + The compressed model. + """ + self._update_mask() + return self.compressed_model + + def export_model(self, model_path, mask_path=None): + """ + Export pruned model and optionally mask tensors. + + Parameters + ---------- + model_path : path-like + The path passed to ``Model.save()``. + You can use ".h5" extension name to export HDF5 format. + mask_path : path-like or None + Export masks to the path when set. + Because Keras cannot save tensors without a ``Model``, + this will create a model, set all masks as its weights, and then save that model. + Masks in saved model will be named by corresponding layer name in compressed model. + + Returns + ------- + None + """ + _logger.info('Saving model to %s', model_path) + input_shape = self.compressed_model._build_input_shape # cannot find a public API + model = self._uninstrument(self.compressed_model) + if input_shape: + model.build(input_shape) + model.save(model_path) + self._instrument(model) + + if mask_path is not None: + _logger.info('Saving masks to %s', mask_path) + # can't find "save raw weights" API in tensorflow, so build a simple model + mask_model = tf.keras.Model() + for wrapper in self.wrappers: + setattr(mask_model, wrapper.layer.name, wrapper.masks) + mask_model.save_weights(mask_path) + + _logger.info('Done') + + def calc_masks(self, wrapper, **kwargs): + """ + Abstract method to be overridden by algorithm. End users should ignore it. + + If the callback is set up, this method will be invoked at end of each training minibatch. + If not, it will only be called when end user invokes ``compress``. + + Parameters + ---------- + wrapper : PrunerLayerWrapper + The instrumented layer. + **kwargs + Reserved for forward compatibility. + + Returns + ------- + dict of (str, tf.Tensor), or None + The key is weight ``Variable``'s name. The value is a mask ``Tensor`` of weight's shape and dtype. + If a weight's key does not appear in the return value, that weight will not be pruned. + Returning ``None`` means the mask is not changed since last time. + Weight names are globally unique, e.g. `model/conv_1/kernel:0`. + """ + # TODO: maybe it should be able to calc on weight-granularity, beside from layer-granularity + raise NotImplementedError("Pruners must overload calc_masks()") + + def _update_mask(self): + for wrapper_idx, wrapper in enumerate(self.wrappers): + masks = self.calc_masks(wrapper, wrapper_idx=wrapper_idx) + if masks is not None: + wrapper.masks = masks + + +class PrunerLayerWrapper(LayerWrapper): + """ + Instrumented TF layer. + + Wrappers will be passed to pruner's ``calc_masks`` API, + and the pruning algorithm should use wrapper's attributes to calculate masks. + + Once instrumented, underlying layer's weights will get **modified** by masks before forward pass. + + Attributes + ---------- + layer : tf.keras.layers.Layer + The original layer. + config : JSON object + Selected configuration. The format is detailed in tutorial. + pruner : Pruner + Bound pruner object. + masks : dict of (str, tf.Tensor) + Current masks. The key is weight's name and the value is mask tensor. + On initialization, `masks` is an empty dict, which means no weight is pruned. + Afterwards, `masks` is the last return value of ``Pruner.calc_masks``. + See ``Pruner.calc_masks`` for details. + """ + def __init__(self, layer, config, pruner): + super().__init__() + self.layer = layer + self.config = config + self.pruner = pruner + self.masks = {} + _logger.info('Layer detected to compress: %s', self.layer.name) + + def call(self, *inputs): + self._update_weights() + return self.layer(*inputs) + + def _update_weights(self): + new_weights = [] + for weight in self.layer.weights: + mask = self.masks.get(weight.name) + if mask is not None: + new_weights.append(tf.math.multiply(weight, mask)) + else: + new_weights.append(weight) + if new_weights and not hasattr(new_weights[0], 'numpy'): + raise RuntimeError('NNI: Compressed model can only run in eager mode') + self.layer.set_weights([weight.numpy() for weight in new_weights]) + + +# TODO: designed to replace `patch_optimizer` +#class PrunerCallback(tf.keras.callbacks.Callback): +# def __init__(self, pruner): +# super().__init__() +# self._pruner = pruner +# +# def on_train_batch_end(self, batch, logs=None): +# self._pruner.update_mask() diff --git a/new_impl/cv/third_party/nni_new/compression/tensorflow/default_layers.py b/new_impl/cv/third_party/nni_new/compression/tensorflow/default_layers.py new file mode 100644 index 0000000000000000000000000000000000000000..0c729bd883f1623d28ad279de053a39a5742109c --- /dev/null +++ b/new_impl/cv/third_party/nni_new/compression/tensorflow/default_layers.py @@ -0,0 +1,9 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +weighted_modules = [ + 'Conv1D', 'Conv2D', 'Conv3D', 'Conv1DTranspose', 'Conv2DTranspose', 'Conv3DTranspose', + 'Dense', + 'PReLU', + 'Embedding', +] diff --git a/new_impl/cv/third_party/nni_new/experiment/__init__.py b/new_impl/cv/third_party/nni_new/experiment/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d59aec4e71220918c30017a1184544524b267944 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/experiment/__init__.py @@ -0,0 +1,6 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .config import * +from .experiment import Experiment +from .data import * diff --git a/new_impl/cv/third_party/nni_new/experiment/config/__init__.py b/new_impl/cv/third_party/nni_new/experiment/config/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..cc7feefdbd81cc0c2d5fa30fa1be548eea7266f7 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/experiment/config/__init__.py @@ -0,0 +1,12 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .common import * +from .local import * +from .remote import * +from .openpai import * +from .aml import * +from .kubeflow import * +from .frameworkcontroller import * +from .adl import * +from .shared_storage import * diff --git a/new_impl/cv/third_party/nni_new/experiment/config/adl.py b/new_impl/cv/third_party/nni_new/experiment/config/adl.py new file mode 100644 index 0000000000000000000000000000000000000000..0f9a856ab62f92850abf30bc57c531be8f506215 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/experiment/config/adl.py @@ -0,0 +1,17 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from dataclasses import dataclass + +from .common import TrainingServiceConfig + +__all__ = ['AdlConfig'] + +@dataclass(init=False) +class AdlConfig(TrainingServiceConfig): + platform: str = 'adl' + docker_image: str = 'msranni/nni:latest' + + _validation_rules = { + 'platform': lambda value: (value == 'adl', 'cannot be modified') + } diff --git a/new_impl/cv/third_party/nni_new/experiment/config/aml.py b/new_impl/cv/third_party/nni_new/experiment/config/aml.py new file mode 100644 index 0000000000000000000000000000000000000000..2fd92e7e76e898d13295edc23171c1abc62e1bd5 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/experiment/config/aml.py @@ -0,0 +1,22 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from dataclasses import dataclass + +from .common import TrainingServiceConfig + +__all__ = ['AmlConfig'] + +@dataclass(init=False) +class AmlConfig(TrainingServiceConfig): + platform: str = 'aml' + subscription_id: str + resource_group: str + workspace_name: str + compute_target: str + docker_image: str = 'msranni/nni:latest' + max_trial_number_per_gpu: int = 1 + + _validation_rules = { + 'platform': lambda value: (value == 'aml', 'cannot be modified') + } diff --git a/new_impl/cv/third_party/nni_new/experiment/config/base.py b/new_impl/cv/third_party/nni_new/experiment/config/base.py new file mode 100644 index 0000000000000000000000000000000000000000..d1fc92a26f1ad2d5b25be748bea917bfaf47d434 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/experiment/config/base.py @@ -0,0 +1,153 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import copy +import dataclasses +from pathlib import Path +from typing import Any, Dict, Optional, Type, TypeVar + +import yaml + +from . import util + +__all__ = ['ConfigBase', 'PathLike'] + +T = TypeVar('T', bound='ConfigBase') + +PathLike = util.PathLike + +def _is_missing(obj: Any) -> bool: + return isinstance(obj, type(dataclasses.MISSING)) + +class ConfigBase: + """ + Base class of config classes. + Subclass may override `_canonical_rules` and `_validation_rules`, + and `validate()` if the logic is complex. + """ + + # Rules to convert field value to canonical format. + # The key is field name. + # The value is callable `value -> canonical_value` + # It is not type-hinted so dataclass won't treat it as field + _canonical_rules = {} # type: ignore + + # Rules to validate field value. + # The key is field name. + # The value is callable `value -> valid` or `value -> (valid, error_message)` + # The rule will be called with canonical format and is only called when `value` is not None. + # `error_message` is used when `valid` is False. + # It will be prepended with class name and field name in exception message. + _validation_rules = {} # type: ignore + + def __init__(self, *, _base_path: Optional[Path] = None, **kwargs): + """ + Initialize a config object and set some fields. + Name of keyword arguments can either be snake_case or camelCase. + They will be converted to snake_case automatically. + If a field is missing and don't have default value, it will be set to `dataclasses.MISSING`. + """ + if 'basepath' in kwargs: + _base_path = kwargs.pop('basepath') + kwargs = {util.case_insensitive(key): value for key, value in kwargs.items()} + if _base_path is None: + _base_path = Path() + for field in dataclasses.fields(self): + value = kwargs.pop(util.case_insensitive(field.name), field.default) + if value is not None and not _is_missing(value): + # relative paths loaded from config file are not relative to pwd + if 'Path' in str(field.type): + value = Path(value).expanduser() + if not value.is_absolute(): + value = _base_path / value + setattr(self, field.name, value) + if kwargs: + cls = type(self).__name__ + fields = ', '.join(kwargs.keys()) + raise ValueError(f'{cls}: Unrecognized fields {fields}') + + @classmethod + def load(cls: Type[T], path: PathLike) -> T: + """ + Load config from YAML (or JSON) file. + Keys in YAML file can either be camelCase or snake_case. + """ + data = yaml.safe_load(open(path)) + if not isinstance(data, dict): + raise ValueError(f'Content of config file {path} is not a dict/object') + return cls(**data, _base_path=Path(path).parent) + + def json(self) -> Dict[str, Any]: + """ + Convert config to JSON object. + The keys of returned object will be camelCase. + """ + self.validate() + return dataclasses.asdict( + self.canonical(), + dict_factory=lambda items: dict((util.camel_case(k), v) for k, v in items if v is not None) + ) + + def canonical(self: T) -> T: + """ + Returns a deep copy, where the fields supporting multiple formats are converted to the canonical format. + Noticeably, relative path may be converted to absolute path. + """ + ret = copy.deepcopy(self) + for field in dataclasses.fields(ret): + key, value = field.name, getattr(ret, field.name) + rule = ret._canonical_rules.get(key) + if rule is not None: + setattr(ret, key, rule(value)) + elif isinstance(value, ConfigBase): + setattr(ret, key, value.canonical()) + # value will be copied twice, should not be a performance issue anyway + elif isinstance(value, Path): + setattr(ret, key, str(value)) + return ret + + def validate(self) -> None: + """ + Validate the config object and raise Exception if it's ill-formed. + """ + class_name = type(self).__name__ + config = self.canonical() + + for field in dataclasses.fields(config): + key, value = field.name, getattr(config, field.name) + + # check existence + if _is_missing(value): + raise ValueError(f'{class_name}: {key} is not set') + + # check type (TODO) + type_name = str(field.type).replace('typing.', '') + optional = any([ + type_name.startswith('Optional['), + type_name.startswith('Union[') and 'None' in type_name, + type_name == 'Any' + ]) + if value is None: + if optional: + continue + else: + raise ValueError(f'{class_name}: {key} cannot be None') + + # check value + rule = config._validation_rules.get(key) + if rule is not None: + try: + result = rule(value) + except Exception: + raise ValueError(f'{class_name}: {key} has bad value {repr(value)}') + + if isinstance(result, bool): + if not result: + raise ValueError(f'{class_name}: {key} ({repr(value)}) is out of range') + else: + if not result[0]: + raise ValueError(f'{class_name}: {key} {result[1]}') + + # check nested config + if isinstance(value, ConfigBase): + value.validate() diff --git a/new_impl/cv/third_party/nni_new/experiment/config/common.py b/new_impl/cv/third_party/nni_new/experiment/config/common.py new file mode 100644 index 0000000000000000000000000000000000000000..c7cd64a7da85a52ba3d2da0816c754acde1e6301 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/experiment/config/common.py @@ -0,0 +1,191 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Dict, List, Optional, Union + +import yaml + +from .base import ConfigBase, PathLike +from . import util + +__all__ = [ + 'ExperimentConfig', + 'AlgorithmConfig', + 'CustomAlgorithmConfig', + 'TrainingServiceConfig', +] + + +@dataclass(init=False) +class _AlgorithmConfig(ConfigBase): + name: Optional[str] = None + class_name: Optional[str] = None + code_directory: Optional[PathLike] = None + class_args: Optional[Dict[str, Any]] = None + + def validate(self): + super().validate() + _validate_algo(self) + + _canonical_rules = {'code_directory': util.canonical_path} + +@dataclass(init=False) +class AlgorithmConfig(_AlgorithmConfig): + name: str + class_args: Optional[Dict[str, Any]] = None + +@dataclass(init=False) +class CustomAlgorithmConfig(_AlgorithmConfig): + class_name: str + code_directory: Optional[PathLike] = '.' + class_args: Optional[Dict[str, Any]] = None + + +class TrainingServiceConfig(ConfigBase): + platform: str + +class SharedStorageConfig(ConfigBase): + storage_type: str + local_mount_point: str + remote_mount_point: str + local_mounted: str + + +@dataclass(init=False) +class ExperimentConfig(ConfigBase): + experiment_name: Optional[str] = None + search_space_file: Optional[PathLike] = None + search_space: Any = None + trial_command: str + trial_code_directory: PathLike = '.' + trial_concurrency: int + trial_gpu_number: Optional[int] = None # TODO: in openpai cannot be None + max_experiment_duration: Optional[str] = None + max_trial_number: Optional[int] = None + nni_manager_ip: Optional[str] = None + use_annotation: bool = False + debug: bool = False + log_level: Optional[str] = None + experiment_working_directory: PathLike = '~/nni-experiments' + tuner_gpu_indices: Union[List[int], str, int, None] = None + tuner: Optional[_AlgorithmConfig] = None + assessor: Optional[_AlgorithmConfig] = None + advisor: Optional[_AlgorithmConfig] = None + training_service: Union[TrainingServiceConfig, List[TrainingServiceConfig]] + shared_storage: Optional[SharedStorageConfig] = None + _deprecated: Optional[Dict[str, Any]] = None + + def __init__(self, training_service_platform: Optional[Union[str, List[str]]] = None, **kwargs): + base_path = kwargs.pop('_base_path', None) + kwargs = util.case_insensitive(kwargs) + if training_service_platform is not None: + assert 'trainingservice' not in kwargs + kwargs['trainingservice'] = util.training_service_config_factory( + platform=training_service_platform, + base_path=base_path + ) + elif isinstance(kwargs.get('trainingservice'), (dict, list)): + # dict means a single training service + # list means hybrid training service + kwargs['trainingservice'] = util.training_service_config_factory( + config=kwargs['trainingservice'], + base_path=base_path + ) + else: + raise RuntimeError('Unsupported Training service configuration!') + super().__init__(_base_path=base_path, **kwargs) + for algo_type in ['tuner', 'assessor', 'advisor']: + if isinstance(kwargs.get(algo_type), dict): + setattr(self, algo_type, _AlgorithmConfig(**kwargs.pop(algo_type))) + + def canonical(self): + ret = super().canonical() + if isinstance(ret.training_service, list): + for i, ts in enumerate(ret.training_service): + ret.training_service[i] = ts.canonical() + return ret + + def validate(self, initialized_tuner: bool = False) -> None: + super().validate() + if initialized_tuner: + _validate_for_exp(self.canonical()) + else: + _validate_for_nnictl(self.canonical()) + if self.trial_gpu_number and hasattr(self.training_service, 'use_active_gpu'): + if self.training_service.use_active_gpu is None: + raise ValueError('Please set "use_active_gpu"') + + def json(self) -> Dict[str, Any]: + obj = super().json() + if obj.get('searchSpaceFile'): + obj['searchSpace'] = yaml.safe_load(open(obj.pop('searchSpaceFile'))) + return obj + +## End of public API ## + + @property + def _canonical_rules(self): + return _canonical_rules + + @property + def _validation_rules(self): + return _validation_rules + + +_canonical_rules = { + 'search_space_file': util.canonical_path, + 'trial_code_directory': util.canonical_path, + 'max_experiment_duration': lambda value: f'{util.parse_time(value)}s' if value is not None else None, + 'experiment_working_directory': util.canonical_path, + 'tuner_gpu_indices': util.canonical_gpu_indices, + 'tuner': lambda config: None if config is None or config.name == '_none_' else config.canonical(), + 'assessor': lambda config: None if config is None or config.name == '_none_' else config.canonical(), + 'advisor': lambda config: None if config is None or config.name == '_none_' else config.canonical(), +} + +_validation_rules = { + 'search_space_file': lambda value: (Path(value).is_file(), f'"{value}" does not exist or is not regular file'), + 'trial_code_directory': lambda value: (Path(value).is_dir(), f'"{value}" does not exist or is not directory'), + 'trial_concurrency': lambda value: value > 0, + 'trial_gpu_number': lambda value: value >= 0, + 'max_experiment_duration': lambda value: util.parse_time(value) > 0, + 'max_trial_number': lambda value: value > 0, + 'log_level': lambda value: value in ["trace", "debug", "info", "warning", "error", "fatal"], + 'tuner_gpu_indices': lambda value: all(i >= 0 for i in value) and len(value) == len(set(value)), + 'training_service': lambda value: (type(value) is not TrainingServiceConfig, 'cannot be abstract base class') +} + +def _validate_for_exp(config: ExperimentConfig) -> None: + # validate experiment for nni.Experiment, where tuner is already initialized outside + if config.use_annotation: + raise ValueError('ExperimentConfig: annotation is not supported in this mode') + if util.count(config.search_space, config.search_space_file) != 1: + raise ValueError('ExperimentConfig: search_space and search_space_file must be set one') + if util.count(config.tuner, config.assessor, config.advisor) != 0: + raise ValueError('ExperimentConfig: tuner, assessor, and advisor must not be set in for this mode') + if config.tuner_gpu_indices is not None: + raise ValueError('ExperimentConfig: tuner_gpu_indices is not supported in this mode') + +def _validate_for_nnictl(config: ExperimentConfig) -> None: + # validate experiment for normal launching approach + if config.use_annotation: + if util.count(config.search_space, config.search_space_file) != 0: + raise ValueError('ExperimentConfig: search_space and search_space_file must not be set with annotationn') + else: + if util.count(config.search_space, config.search_space_file) != 1: + raise ValueError('ExperimentConfig: search_space and search_space_file must be set one') + if util.count(config.tuner, config.advisor) != 1: + raise ValueError('ExperimentConfig: tuner and advisor must be set one') + +def _validate_algo(algo: AlgorithmConfig) -> None: + if algo.name is None: + if algo.class_name is None: + raise ValueError('Missing algorithm name') + if algo.code_directory is not None and not Path(algo.code_directory).is_dir(): + raise ValueError(f'code_directory "{algo.code_directory}" does not exist or is not directory') + else: + if algo.class_name is not None or algo.code_directory is not None: + raise ValueError(f'When name is set for registered algorithm, class_name and code_directory cannot be used') + # TODO: verify algorithm installation and class args diff --git a/new_impl/cv/third_party/nni_new/experiment/config/convert.py b/new_impl/cv/third_party/nni_new/experiment/config/convert.py new file mode 100644 index 0000000000000000000000000000000000000000..0b4b332d08f6433201400e8e880691d460ecb71f --- /dev/null +++ b/new_impl/cv/third_party/nni_new/experiment/config/convert.py @@ -0,0 +1,266 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import copy +import logging + +from .common import ExperimentConfig, AlgorithmConfig, CustomAlgorithmConfig +from .remote import RemoteMachineConfig +from .kubeflow import KubeflowRoleConfig, KubeflowNfsConfig, KubeflowAzureStorageConfig +from .frameworkcontroller import FrameworkControllerRoleConfig +from .shared_storage import NfsConfig, AzureBlobConfig +from . import util + +_logger = logging.getLogger(__name__) + +def to_v2(v1) -> ExperimentConfig: + v1 = copy.deepcopy(v1) + platform = v1.pop('trainingServicePlatform') + assert platform in ['local', 'remote', 'openpai', 'aml'] + v2 = ExperimentConfig(platform) + + _drop_field(v1, 'authorName') + _move_field(v1, v2, 'experimentName', 'experiment_name') + _drop_field(v1, 'description') + _move_field(v1, v2, 'trialConcurrency', 'trial_concurrency') + _move_field(v1, v2, 'maxExecDuration', 'max_experiment_duration') + if isinstance(v2.max_experiment_duration, (int, float)): + v2.max_experiment_duration = str(v2.max_experiment_duration) + 's' + _move_field(v1, v2, 'maxTrialNum', 'max_trial_number') + _move_field(v1, v2, 'searchSpacePath', 'search_space_file') + assert not v1.pop('multiPhase', None), 'Multi-phase is no longer supported' + _deprecate(v1, v2, 'multiThread') + _move_field(v1, v2, 'nniManagerIp', 'nni_manager_ip') + _move_field(v1, v2, 'logDir', 'experiment_working_directory') + _move_field(v1, v2, 'debug', 'debug') + _deprecate(v1, v2, 'versionCheck') + _move_field(v1, v2, 'logLevel', 'log_level') + _deprecate(v1, v2, 'logCollection') + v1.pop('useAnnotation', None) # TODO: how to handle annotation in nni.Experiment? + + if 'trial' in v1: + v1_trial = v1.pop('trial') + _move_field(v1_trial, v2, 'command', 'trial_command') + _move_field(v1_trial, v2, 'codeDir', 'trial_code_directory') + _move_field(v1_trial, v2, 'gpuNum', 'trial_gpu_number') + + for algo_type in ['tuner', 'assessor', 'advisor']: + if algo_type in v1: + convert_algo(algo_type, v1, v2) + + ts = v2.training_service + + if platform == 'local': + local_config = v1.pop('localConfig', {}) + _move_field(local_config, ts, 'gpuIndices', 'gpu_indices') + _move_field(local_config, ts, 'maxTrialNumPerGpu', 'max_trial_number_per_gpu') + _move_field(local_config, ts, 'useActiveGpu', 'use_active_gpu') + assert not local_config, local_config + + if platform == 'remote': + remote_config = v1.pop('remoteConfig', {}) + _move_field(remote_config, ts, 'reuse', 'reuse_mode') + assert not remote_config, remote_config + + ts.machine_list = [] + for v1_machine in v1.pop('machineList'): + v2_machine = RemoteMachineConfig() + ts.machine_list.append(v2_machine) + _move_field(v1_machine, v2_machine, 'ip', 'host') + _move_field(v1_machine, v2_machine, 'port', 'port') + _move_field(v1_machine, v2_machine, 'username', 'user') + _move_field(v1_machine, v2_machine, 'sshKeyPath', 'ssh_key_file') + _move_field(v1_machine, v2_machine, 'passphrase', 'ssh_passphrase') + _move_field(v1_machine, v2_machine, 'gpuIndices', 'gpu_indices') + _move_field(v1_machine, v2_machine, 'maxTrialNumPerGpu', 'max_trial_number_per_gpu') + _move_field(v1_machine, v2_machine, 'useActiveGpu', 'use_active_gpu') + _move_field(v1_machine, v2_machine, 'pythonPath', 'python_path') + _move_field(v1_machine, v2_machine, 'passwd', 'password') + assert not v1_machine, v1_machine + + if platform == 'openpai': + _move_field(v1_trial, ts, 'nniManagerNFSMountPath', 'local_storage_mount_point') + _move_field(v1_trial, ts, 'containerNFSMountPath', 'container_storage_mount_point') + _move_field(v1_trial, ts, 'cpuNum', 'trial_cpu_number') + if 'memoryMB' in v1_trial: + ts.trial_memory_size = str(v1_trial.pop('memoryMB')) + 'mb' + _move_field(v1_trial, ts, 'image', 'docker_image') + _deprecate(v1_trial, v2, 'virtualCluster') + _move_field(v1_trial, ts, 'paiStorageConfigName', 'storage_config_name') + _move_field(v1_trial, ts, 'paiConfigPath', 'openpaiConfigFile') + + pai_config = v1.pop('paiConfig') + _move_field(pai_config, ts, 'userName', 'username') + _deprecate(pai_config, v2, 'password') + _move_field(pai_config, ts, 'token', 'token') + _move_field(pai_config, ts, 'host', 'host') + _move_field(pai_config, ts, 'reuse', 'reuse_mode') + _move_field(pai_config, ts, 'gpuNum', 'trial_gpu_number') + _move_field(pai_config, ts, 'cpuNum', 'trial_cpu_number') + if 'memoryMB' in pai_config: + ts.trial_memory_size = str(pai_config.pop('memoryMB')) + 'mb' + _deprecate(pai_config, v2, 'maxTrialNumPerGpu') + _deprecate(pai_config, v2, 'useActiveGpu') + assert not pai_config, pai_config + + if platform == 'aml': + _move_field(v1_trial, ts, 'image', 'docker_image') + + aml_config = v1.pop('amlConfig', {}) + _move_field(aml_config, ts, 'subscriptionId', 'subscription_id') + _move_field(aml_config, ts, 'resourceGroup', 'resource_group') + _move_field(aml_config, ts, 'workspaceName', 'workspace_name') + _move_field(aml_config, ts, 'computeTarget', 'compute_target') + _move_field(aml_config, ts, 'maxTrialNumPerGpu', 'max_trial_number_per_gpu') + _deprecate(aml_config, v2, 'useActiveGpu') + assert not aml_config, aml_config + + if platform == 'kubeflow': + kf_config = v1.pop('kubeflowConfig') + _move_field(kf_config, ts, 'operator', 'operator') + ps_name = 'ps' if ts.operator != 'pytorch-operator' else 'master' + _move_field(kf_config, ts, 'apiVersion', 'api_version') + + # FIXME: use storage service + storage_name = kf_config.pop('storage', None) + if storage_name is None: + storage_name = 'nfs' if 'nfs' in kf_config else 'azureStorage' + if storage_name == 'nfs': + nfs = kf_config.pop('nfs') + ts.storage = KubeflowNfsConfig(server=nfs['server'], path=nfs['path']) + if storage_name == 'azureStorage': + key_vault = kf_config.pop('keyVault') + azure_storage = kf_config.pop('azureStorage') + ts.storage = KubeflowAzureStorageConfig( + azure_account=azure_storage['accountName'], + azure_share=azure_storage['azureShare'], + key_vault=key_vault['vaultName'], + key_vault_secret=key_vault['name'] + ) + _deprecate(kf_config, v2, 'uploadRetryCount') + + assert not kf_config, kf_config + + _drop_field(v1_trial, 'nasMode') + for role_name in [ps_name, 'worker']: + if role_name not in v1_trial: + continue + v1_role = v1_trial.pop(role_name) + v2_role = KubeflowRoleConfig() + if role_name == 'worker': + ts.worker = v2_role + else: + ts.parameter_server = v2_role + + _move_field(v1_role, v2_role, 'replicas', 'replicas') + _move_field(v1_role, v2_role, 'command', 'command') + _move_field(v1_role, v2_role, 'gpu_num', 'gpu_number') + _move_field(v1_role, v2_role, 'cpu_num', 'cpu_number') + v2_role.memory_size = str(v1_role.pop('memoryMB')) + 'mb' + _move_field(v1_role, v2_role, 'image', 'docker_image') + _deprecate(v1_role, v2, 'privateRegistryAuthPath') + assert not v1_role, v1_role + + if platform == 'frameworkcontroller': + fc_config = v1.pop('frameworkcontroller') + _deprecate(fc_config, v2, 'serviceAccountName') + + storage_name = fc_config.pop('storage', None) + if storage_name is None: + storage_name = 'nfs' if 'nfs' in fc_config else 'azureStorage' + if storage_name == 'nfs': + nfs = fc_config.pop('nfs') + ts.storage = KubeflowNfsConfig(server=nfs['server'], path=nfs['path']) + if storage_name == 'azureStorage': + key_vault = fc_config.pop('keyVault') + azure_storage = fc_config.pop('azureStorage') + ts.storage = KubeflowAzureStorageConfig( + azure_account=azure_storage['accountName'], + azure_share=azure_storage['azureShare'], + key_vault=key_vault['vaultName'], + key_vault_secret=key_vault['name'] + ) + _deprecate(fc_config, v2, 'uploadRetryCount') + + assert not fc_config, fc_config + + _drop_field(v1_trial, 'nasMode') + ts.task_roles = [] + for v1_role in v1_trial.pop('taskRoles', []): + v2_role = FrameworkControllerRoleConfig() + ts.task_roles.append(v2_role) + + _move_field(v1_role, v2_role, 'name', 'name') + _move_field(v1_role, v2_role, 'taskNum', 'task_number') + policy = v1_role.pop('frameworkControllerCompletionPolicy', {}) + _move_field(policy, v2_role, 'minFailedTaskCount', 'attempt_completion_min_failed_tasks') + _move_field(policy, v2_role, 'minSucceededTaskCount', 'attempt_completion_min_succeeded_tasks') + _move_field(v1_role, v2_role, 'command', 'command') + _move_field(v1_role, v2_role, 'gpuNum', 'gpu_number') + _move_field(v1_role, v2_role, 'cpuNum', 'cpu_number') + v2_role.memory_size = str(v1_role.pop('memoryMB')) + 'mb' + _move_field(v1_role, v2_role, 'image', 'docker_image') + _deprecate(v1_role, v2, 'privateRegistryAuthPath') + assert not v1_role, v1_role + + # hybrid mode should always use v2 schema, so no need to handle here + + v1_storage = v1.pop('sharedStorage', None) + if v1_storage: + type_ = v1_storage.pop('storageType') + if type_ == 'NFS': + v2.shared_storage = NfsConfig(**v1_storage) + elif type_ == 'AzureBlob': + v2.shared_storage = AzureBlobConfig(**v1_storage) + else: + raise ValueError(f'bad storage type: {type_}') + + assert not v1_trial, v1_trial + assert not v1, v1 + return v2.canonical() + +def _move_field(v1, v2, v1_key, v2_key): + if v1_key in v1: + value = v1.pop(v1_key, None) + if value is not None: + setattr(v2, v2_key, value) + +def _drop_field(v1, key): + if key in v1: + logging.warning(f'Configuration field {key} is no longer supported and has been ignored') + v1.pop(key) + +# NOTE: fields not yet supported by v2 are also (temporarily) placed here +def _deprecate(v1, v2, key): + if key in v1: + if v2._deprecated is None: + v2._deprecated = {} + v2._deprecated[key] = v1.pop(key) + +def convert_algo(algo_type, v1, v2): + if algo_type not in v1: + return None + v1_algo = v1.pop(algo_type) + + builtin_name = v1_algo.pop(f'builtin{algo_type.title()}Name', None) + class_args = v1_algo.pop('classArgs', None) + + if builtin_name is not None: + v2_algo = AlgorithmConfig(name=builtin_name, class_args=class_args) + + else: + code_directory = util.canonical_path(v1_algo.pop('codeDir')) + class_file_name = v1_algo.pop('classFileName') + assert class_file_name.endswith('.py') + class_name = class_file_name[:-3] + '.' + v1_algo.pop('className') + v2_algo = CustomAlgorithmConfig( + class_name=class_name, + code_directory=code_directory, + class_args=class_args + ) + + setattr(v2, algo_type, v2_algo) + _deprecate(v1_algo, v2, 'includeIntermediateResults') + _move_field(v1_algo, v2, 'gpuIndices', 'tuner_gpu_indices') + assert not v1_algo, v1_algo + return v2_algo diff --git a/new_impl/cv/third_party/nni_new/experiment/config/frameworkcontroller.py b/new_impl/cv/third_party/nni_new/experiment/config/frameworkcontroller.py new file mode 100644 index 0000000000000000000000000000000000000000..fbc1397630609171a9d08ed2f4a3ba555543e3b9 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/experiment/config/frameworkcontroller.py @@ -0,0 +1,72 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from dataclasses import dataclass +from typing import List, Optional + +from .base import ConfigBase +from .common import TrainingServiceConfig +from . import util + +__all__ = [ + 'FrameworkControllerConfig', + 'FrameworkControllerRoleConfig', + 'FrameworkControllerNfsConfig', + 'FrameworkControllerAzureStorageConfig' +] + + +@dataclass(init=False) +class _FrameworkControllerStorageConfig(ConfigBase): + storage: str + server: Optional[str] = None + path: Optional[str] = None + azure_account: Optional[str] = None + azure_share: Optional[str] = None + key_vault: Optional[str] = None + key_vault_secret: Optional[str] = None + +@dataclass(init=False) +class FrameworkControllerNfsConfig(ConfigBase): + storage: str = 'nfs' + server: str + path: str + +@dataclass(init=False) +class FrameworkControllerAzureStorageConfig(ConfigBase): + storage: str = 'azureStorage' + azure_account: str + azure_share: str + key_vault: str + key_vault_secret: str + + +@dataclass(init=False) +class FrameworkControllerRoleConfig(ConfigBase): + name: str + docker_image: str = 'msranni/nni:latest' + task_number: int + command: str + gpu_number: int + cpu_number: int + memory_size: str + attempt_completion_min_failed_tasks: int + attempt_completion_min_succeeded_tasks: int + + +@dataclass(init=False) +class FrameworkControllerConfig(TrainingServiceConfig): + platform: str = 'frameworkcontroller' + service_account_name: str + storage: _FrameworkControllerStorageConfig + task_roles: List[FrameworkControllerRoleConfig] + + def __init__(self, **kwargs): + kwargs = util.case_insensitive(kwargs) + kwargs['storage'] = util.load_config(_FrameworkControllerStorageConfig, kwargs.get('storage')) + kwargs['taskroles'] = util.load_config(FrameworkControllerRoleConfig, kwargs.get('taskroles')) + super().__init__(**kwargs) + + _validation_rules = { + 'platform': lambda value: (value == 'frameworkcontroller', 'cannot be modified') + } diff --git a/new_impl/cv/third_party/nni_new/experiment/config/kubeflow.py b/new_impl/cv/third_party/nni_new/experiment/config/kubeflow.py new file mode 100644 index 0000000000000000000000000000000000000000..aaa15085d4bf4d0970f0042eae33487908761812 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/experiment/config/kubeflow.py @@ -0,0 +1,68 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from dataclasses import dataclass +from typing import Optional + +from .base import ConfigBase +from .common import TrainingServiceConfig +from . import util + +__all__ = ['KubeflowConfig', 'KubeflowRoleConfig', 'KubeflowNfsConfig', 'KubeflowAzureStorageConfig'] + + +@dataclass(init=False) +class _KubeflowStorageConfig(ConfigBase): + storage: str + server: Optional[str] = None + path: Optional[str] = None + azure_account: Optional[str] = None + azure_share: Optional[str] = None + key_vault: Optional[str] = None + key_vault_secret: Optional[str] = None + +@dataclass(init=False) +class KubeflowNfsConfig(_KubeflowStorageConfig): + storage: str = 'nfs' + server: str + path: str + +@dataclass(init=False) +class KubeflowAzureStorageConfig(ConfigBase): + storage: str = 'azureStorage' + azure_account: str + azure_share: str + key_vault: str + key_vault_secret: str + + +@dataclass(init=False) +class KubeflowRoleConfig(ConfigBase): + replicas: int + command: str + gpu_number: int + cpu_number: int + memory_size: str + docker_image: str = 'msranni/nni:latest' + + +@dataclass(init=False) +class KubeflowConfig(TrainingServiceConfig): + platform: str = 'kubeflow' + operator: str + api_version: str + storage: _KubeflowStorageConfig + worker: KubeflowRoleConfig + parameter_server: Optional[KubeflowRoleConfig] = None + + def __init__(self, **kwargs): + kwargs = util.case_insensitive(kwargs) + kwargs['storage'] = util.load_config(_KubeflowStorageConfig, kwargs.get('storage')) + kwargs['worker'] = util.load_config(KubeflowRoleConfig, kwargs.get('worker')) + kwargs['parameterserver'] = util.load_config(KubeflowRoleConfig, kwargs.get('parameterserver')) + super().__init__(**kwargs) + + _validation_rules = { + 'platform': lambda value: (value == 'kubeflow', 'cannot be modified'), + 'operator': lambda value: value in ['tf-operator', 'pytorch-operator'] + } diff --git a/new_impl/cv/third_party/nni_new/experiment/config/local.py b/new_impl/cv/third_party/nni_new/experiment/config/local.py new file mode 100644 index 0000000000000000000000000000000000000000..90b92093fd69f50048cab85c20678be67d7b50aa --- /dev/null +++ b/new_impl/cv/third_party/nni_new/experiment/config/local.py @@ -0,0 +1,27 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from dataclasses import dataclass +from typing import List, Optional, Union + +from .common import TrainingServiceConfig +from . import util + +__all__ = ['LocalConfig'] + +@dataclass(init=False) +class LocalConfig(TrainingServiceConfig): + platform: str = 'local' + use_active_gpu: Optional[bool] = None + max_trial_number_per_gpu: int = 1 + gpu_indices: Union[List[int], str, int, None] = None + + _canonical_rules = { + 'gpu_indices': util.canonical_gpu_indices + } + + _validation_rules = { + 'platform': lambda value: (value == 'local', 'cannot be modified'), + 'max_trial_number_per_gpu': lambda value: value > 0, + 'gpu_indices': lambda value: all(idx >= 0 for idx in value) and len(value) == len(set(value)) + } diff --git a/new_impl/cv/third_party/nni_new/experiment/config/openpai.py b/new_impl/cv/third_party/nni_new/experiment/config/openpai.py new file mode 100644 index 0000000000000000000000000000000000000000..66eecadac7cafdcb1540370a21b8b056f936e31f --- /dev/null +++ b/new_impl/cv/third_party/nni_new/experiment/config/openpai.py @@ -0,0 +1,47 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from dataclasses import dataclass +from pathlib import Path, PurePosixPath +from typing import Any, Dict, Optional + +from .base import PathLike +from .common import TrainingServiceConfig +from . import util + +__all__ = ['OpenpaiConfig'] + +@dataclass(init=False) +class OpenpaiConfig(TrainingServiceConfig): + platform: str = 'openpai' + host: str + username: str + token: str + trial_cpu_number: int + trial_memory_size: str + storage_config_name: str + docker_image: str = 'msranni/nni:latest' + local_storage_mount_point: PathLike + container_storage_mount_point: str + reuse_mode: bool = True + + openpai_config: Optional[Dict[str, Any]] = None + openpai_config_file: Optional[PathLike] = None + + _canonical_rules = { + 'host': lambda value: 'https://' + value if '://' not in value else value, # type: ignore + 'local_storage_mount_point': util.canonical_path, + 'openpai_config_file': util.canonical_path + } + + _validation_rules = { + 'platform': lambda value: (value == 'openpai', 'cannot be modified'), + 'local_storage_mount_point': lambda value: Path(value).is_dir(), + 'container_storage_mount_point': lambda value: (PurePosixPath(value).is_absolute(), 'is not absolute'), + 'openpai_config_file': lambda value: Path(value).is_file() + } + + def validate(self) -> None: + super().validate() + if self.openpai_config is not None and self.openpai_config_file is not None: + raise ValueError('openpai_config and openpai_config_file can only be set one') diff --git a/new_impl/cv/third_party/nni_new/experiment/config/remote.py b/new_impl/cv/third_party/nni_new/experiment/config/remote.py new file mode 100644 index 0000000000000000000000000000000000000000..29c47fcaa8f0f48b1c62b6d7da0342b6ccbff7d9 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/experiment/config/remote.py @@ -0,0 +1,63 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from dataclasses import dataclass +from pathlib import Path +from typing import List, Optional, Union +import warnings + +from .base import ConfigBase, PathLike +from .common import TrainingServiceConfig +from . import util + +__all__ = ['RemoteConfig', 'RemoteMachineConfig'] + +@dataclass(init=False) +class RemoteMachineConfig(ConfigBase): + host: str + port: int = 22 + user: str + password: Optional[str] = None + ssh_key_file: PathLike = None #'~/.ssh/id_rsa' + ssh_passphrase: Optional[str] = None + use_active_gpu: bool = False + max_trial_number_per_gpu: int = 1 + gpu_indices: Union[List[int], str, int, None] = None + python_path: Optional[str] = None + + _canonical_rules = { + 'ssh_key_file': util.canonical_path, + 'gpu_indices': util.canonical_gpu_indices + } + + _validation_rules = { + 'port': lambda value: 0 < value < 65536, + 'max_trial_number_per_gpu': lambda value: value > 0, + 'gpu_indices': lambda value: all(idx >= 0 for idx in value) and len(value) == len(set(value)) + } + + def validate(self): + super().validate() + if self.password is None and not Path(self.ssh_key_file).is_file(): + raise ValueError(f'Password is not provided and cannot find SSH key file "{self.ssh_key_file}"') + if self.password: + warnings.warn('Password will be exposed through web UI in plain text. We recommend to use SSH key file.') + +@dataclass(init=False) +class RemoteConfig(TrainingServiceConfig): + platform: str = 'remote' + reuse_mode: bool = True + machine_list: List[RemoteMachineConfig] + + def __init__(self, **kwargs): + kwargs = util.case_insensitive(kwargs) + kwargs['machinelist'] = util.load_config(RemoteMachineConfig, kwargs.get('machinelist')) + super().__init__(**kwargs) + + _canonical_rules = { + 'machine_list': lambda value: [config.canonical() for config in value] + } + + _validation_rules = { + 'platform': lambda value: (value == 'remote', 'cannot be modified') + } diff --git a/new_impl/cv/third_party/nni_new/experiment/config/shared_storage.py b/new_impl/cv/third_party/nni_new/experiment/config/shared_storage.py new file mode 100644 index 0000000000000000000000000000000000000000..3d4d357764ce8741fcecd64e519b17fd15dd0ba1 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/experiment/config/shared_storage.py @@ -0,0 +1,23 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from dataclasses import dataclass +from typing import Optional + +from .common import SharedStorageConfig + +__all__ = ['NfsConfig', 'AzureBlobConfig'] + +@dataclass(init=False) +class NfsConfig(SharedStorageConfig): + storage_type: str = 'NFS' + nfs_server: str + exported_directory: str + +@dataclass(init=False) +class AzureBlobConfig(SharedStorageConfig): + storage_type: str = 'AzureBlob' + storage_account_name: str + storage_account_key: Optional[str] = None + resource_group_name: Optional[str] = None + container_name: str diff --git a/new_impl/cv/third_party/nni_new/experiment/config/util.py b/new_impl/cv/third_party/nni_new/experiment/config/util.py new file mode 100644 index 0000000000000000000000000000000000000000..62a56f6b00dc082a1b9636ad13d045eefd0680a1 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/experiment/config/util.py @@ -0,0 +1,101 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +Miscellaneous utility functions. +""" + +import importlib +import json +import math +import os.path +from pathlib import Path +from typing import Any, Dict, Optional, Union, List + +import nni.runtime.config + +PathLike = Union[Path, str] + +def case_insensitive(key_or_kwargs: Union[str, Dict[str, Any]]) -> Union[str, Dict[str, Any]]: + if isinstance(key_or_kwargs, str): + return key_or_kwargs.lower().replace('_', '') + else: + return {key.lower().replace('_', ''): value for key, value in key_or_kwargs.items()} + +def camel_case(key: str) -> str: + words = key.strip('_').split('_') + return words[0] + ''.join(word.title() for word in words[1:]) + +def canonical_path(path: Optional[PathLike]) -> Optional[str]: + # Path.resolve() does not work on Windows when file not exist, so use os.path instead + return os.path.abspath(os.path.expanduser(path)) if path is not None else None + +def count(*values) -> int: + return sum(value is not None and value is not False for value in values) + +def training_service_config_factory( + platform: Union[str, List[str]] = None, + config: Union[List, Dict] = None, + base_path: Optional[Path] = None): # -> TrainingServiceConfig + from .common import TrainingServiceConfig + + # import all custom config classes so they can be found in TrainingServiceConfig.__subclasses__() + custom_ts_config_path = nni.runtime.config.get_config_file('training_services.json') + custom_ts_config = json.load(custom_ts_config_path.open()) + for custom_ts_pkg in custom_ts_config.keys(): + pkg = importlib.import_module(custom_ts_pkg) + _config_class = pkg.nni_training_service_info.config_class + + ts_configs = [] + if platform is not None: + assert config is None + platforms = platform if isinstance(platform, list) else [platform] + for cls in TrainingServiceConfig.__subclasses__(): + if cls.platform in platforms: + ts_configs.append(cls()) + if len(ts_configs) < len(platforms): + bad = ', '.join(set(platforms) - set(ts_configs)) + raise RuntimeError(f'Bad training service platform: {bad}') + else: + assert config is not None + supported_platforms = {cls.platform: cls for cls in TrainingServiceConfig.__subclasses__()} + configs = config if isinstance(config, list) else [config] + for conf in configs: + if conf['platform'] not in supported_platforms: + raise RuntimeError(f'Unrecognized platform {conf["platform"]}') + ts_configs.append(supported_platforms[conf['platform']](_base_path=base_path, **conf)) + return ts_configs if len(ts_configs) > 1 else ts_configs[0] + +def load_config(Type, value): + if isinstance(value, list): + return [load_config(Type, item) for item in value] + if isinstance(value, dict): + return Type(**value) + return value + +def strip_optional(type_hint): + return type_hint.__args__[0] if str(type_hint).startswith('typing.Optional[') else type_hint + +def parse_time(time: str, target_unit: str = 's') -> int: + return _parse_unit(time.lower(), target_unit, _time_units) + +def parse_size(size: str, target_unit: str = 'mb') -> int: + return _parse_unit(size.lower(), target_unit, _size_units) + +_time_units = {'d': 24 * 3600, 'h': 3600, 'm': 60, 's': 1} +_size_units = {'gb': 1024 * 1024 * 1024, 'mb': 1024 * 1024, 'kb': 1024} + +def _parse_unit(string, target_unit, all_units): + for unit, factor in all_units.items(): + if string.endswith(unit): + number = string[:-len(unit)] + value = float(number) * factor + return math.ceil(value / all_units[target_unit]) + raise ValueError(f'Unsupported unit in "{string}"') + +def canonical_gpu_indices(indices: Union[List[int], str, int, None]) -> Optional[List[int]]: + if isinstance(indices, str): + return [int(idx) for idx in indices.split(',')] + if isinstance(indices, int): + return [indices] + return indices diff --git a/new_impl/cv/third_party/nni_new/experiment/data.py b/new_impl/cv/third_party/nni_new/experiment/data.py new file mode 100644 index 0000000000000000000000000000000000000000..d58f4671f6e7007f9967f1796ec12b212786e501 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/experiment/data.py @@ -0,0 +1,135 @@ +from dataclasses import dataclass +import json +from typing import List + + +@dataclass +class TrialResult: + """ + TrialResult stores the result information of a trial job. + + Attributes + ---------- + parameter: dict + Hyper parameters for this trial. + value: serializable object, usually a number, or a dict with key "default" and other extra keys + Final result. + trialJobId: str + Trial job id. + """ + parameter: dict + value: dict + trialJobId: str + + def __init__(self, parameter: dict, value: str, trialJobId: str): + self.parameter = parameter + self.value = json.loads(value) + self.trialJobId = trialJobId + + +@dataclass +class TrialMetricData: + """ + TrialMetricData stores the metric data of a trial job. + A trial job may have both intermediate metric and final metric. + + Attributes + ---------- + timestamp: int + Time stamp. + trialJobId: str + Trial job id. + parameterId: int + Parameter id. + type: str + Metric type, `PERIODICAL` for intermediate result and `FINAL` for final result. + sequence: int + Sequence number in this trial. + data: serializable object, usually a number, or a dict with key "default" and other extra keys + Metric data. + """ + timestamp: int + trialJobId: str + parameterId: int + type: str + sequence: int + data: dict + + def __init__(self, timestamp: int, trialJobId: str, parameterId: int, type: str, sequence: int, data: str): # pylint: disable=W0622 + self.timestamp = timestamp + self.trialJobId = trialJobId + self.parameterId = parameterId + self.type = type + self.sequence = sequence + self.data = json.loads(json.loads(data)) + + +@dataclass +class TrialHyperParameters: + """ + TrialHyperParameters stores the hyper parameters of a trial job. + + Attributes + ---------- + parameter_id: int + Parameter id. + parameter_source: str + Parameter source. + parameters: dict + Hyper parameters. + parameter_index: int + Parameter index. + """ + parameter_id: int + parameter_source: str + parameters: dict + parameter_index: int + + +@dataclass +class TrialJob: + """ + TrialJob stores the information of a trial job. + + Attributes + ---------- + trialJobId: str + Trial job id. + status: str + Job status. + hyperParameters: list of `nni.experiment.TrialHyperParameters` + See `nni.experiment.TrialHyperParameters`. + logPath: str + Log path. + startTime: int + Job start time (timestamp). + endTime: int + Job end time (timestamp). + finalMetricData: list of `nni.experiment.TrialMetricData` + See `nni.experiment.TrialMetricData`. + stderrPath: str + Stderr log path. + sequenceId: int + Sequence Id. + """ + trialJobId: str + status: str + hyperParameters: List[TrialHyperParameters] + logPath: str + startTime: int + endTime: int + finalMetricData: List[TrialMetricData] + stderrPath: str + sequenceId: int + + def __init__(self, trialJobId: str, status: str, logPath: str, startTime: int, sequenceId: int, + endTime: int = -1, stderrPath: str = '', hyperParameters: List = [], finalMetricData: List = []): + self.trialJobId = trialJobId + self.status = status + self.hyperParameters = [TrialHyperParameters(**json.loads(e)) for e in hyperParameters] + self.logPath = logPath + self.startTime = startTime + self.endTime = endTime + self.finalMetricData = [TrialMetricData(**e) for e in finalMetricData] + self.stderrPath = stderrPath + self.sequenceId = sequenceId diff --git a/new_impl/cv/third_party/nni_new/experiment/experiment.py b/new_impl/cv/third_party/nni_new/experiment/experiment.py new file mode 100644 index 0000000000000000000000000000000000000000..c7708f4f87a55e39fd55786bf5cbd81c36dfec1a --- /dev/null +++ b/new_impl/cv/third_party/nni_new/experiment/experiment.py @@ -0,0 +1,454 @@ +import atexit +import logging +from pathlib import Path +import socket +from subprocess import Popen +import time +from typing import Optional, Union, List, overload, Any + +import json_tricks +import colorama +import psutil + +import nni.runtime.log + +from .config import ExperimentConfig, AlgorithmConfig +from .data import TrialJob, TrialMetricData, TrialResult +from . import launcher +from . import management +from . import rest +from ..tools.nnictl.command_utils import kill_command + +_logger = logging.getLogger('nni.experiment') + + +class Experiment: + """ + Create and stop an NNI experiment. + + Attributes + ---------- + config + Experiment configuration. + port + Web UI port of the experiment, or `None` if it is not running. + """ + + @overload + def __init__(self, config: ExperimentConfig) -> None: + """ + Prepare an experiment. + + Use `Experiment.run()` to launch it. + + Parameters + ---------- + config + Experiment configuration. + """ + ... + + @overload + def __init__(self, training_service: Union[str, List[str]]) -> None: + """ + Prepare an experiment, leaving configuration fields to be set later. + + Example usage:: + + experiment = Experiment('remote') + experiment.config.trial_command = 'python3 trial.py' + experiment.config.machines.append(RemoteMachineConfig(ip=..., user_name=...)) + ... + experiment.run(8080) + + Parameters + ---------- + training_service + Name of training service. + Supported value: "local", "remote", "openpai", "aml", "kubeflow", "frameworkcontroller", "adl" and hybrid training service. + """ + ... + + def __init__(self, config=None, training_service=None): + nni.runtime.log.init_logger_experiment() + + self.config: Optional[ExperimentConfig] = None + self.id: Optional[str] = None + self.port: Optional[int] = None + self._proc: Optional[Popen] = None + self.mode = 'new' + + args = [config, training_service] # deal with overloading + if isinstance(args[0], (str, list)): + self.config = ExperimentConfig(args[0]) + self.config.tuner = AlgorithmConfig(name='_none_', class_args={}) + self.config.assessor = AlgorithmConfig(name='_none_', class_args={}) + self.config.advisor = AlgorithmConfig(name='_none_', class_args={}) + else: + self.config = args[0] + + def start(self, port: int = 8080, debug: bool = False) -> None: + """ + Start the experiment in background. + + This method will raise exception on failure. + If it returns, the experiment should have been successfully started. + + Parameters + ---------- + port + The port of web UI. + debug + Whether to start in debug mode. + """ + atexit.register(self.stop) + + if self.mode == 'new': + self.id = management.generate_experiment_id() + else: + self.config = launcher.get_stopped_experiment_config(self.id, self.mode) + + if self.config.experiment_working_directory is not None: + log_dir = Path(self.config.experiment_working_directory, self.id, 'log') + else: + log_dir = Path.home() / f'nni-experiments/{self.id}/log' + nni.runtime.log.start_experiment_log(self.id, log_dir, debug) + + self._proc = launcher.start_experiment(self.id, self.config, port, debug, mode=self.mode) + assert self._proc is not None + + self.port = port # port will be None if start up failed + + ips = [self.config.nni_manager_ip] + for interfaces in psutil.net_if_addrs().values(): + for interface in interfaces: + if interface.family == socket.AF_INET: + ips.append(interface.address) + ips = [f'http://{ip}:{port}' for ip in ips if ip] + msg = 'Web UI URLs: ' + colorama.Fore.CYAN + ' '.join(ips) + colorama.Style.RESET_ALL + _logger.info(msg) + + def stop(self) -> None: + """ + Stop background experiment. + """ + _logger.info('Stopping experiment, please wait...') + atexit.unregister(self.stop) + + if self.id is not None: + nni.runtime.log.stop_experiment_log(self.id) + if self._proc is not None: + try: + rest.delete(self.port, '/experiment') + except Exception as e: + _logger.exception(e) + _logger.warning('Cannot gracefully stop experiment, killing NNI process...') + kill_command(self._proc.pid) + + self.id = None + self.port = None + self._proc = None + _logger.info('Experiment stopped') + + def run(self, port: int = 8080, wait_completion: bool = True, debug: bool = False) -> bool: + """ + Run the experiment. + + If wait_completion is True, this function will block until experiment finish or error. + + Return `True` when experiment done; or return `False` when experiment failed. + + Else if wait_completion is False, this function will non-block and return None immediately. + """ + self.start(port, debug) + if wait_completion: + try: + while True: + time.sleep(10) + status = self.get_status() + if status == 'DONE' or status == 'STOPPED': + return True + if status == 'ERROR': + return False + except KeyboardInterrupt: + _logger.warning('KeyboardInterrupt detected') + finally: + self.stop() + + @classmethod + def connect(cls, port: int): + """ + Connect to an existing experiment. + + Parameters + ---------- + port + The port of web UI. + """ + experiment = Experiment() + experiment.port = port + experiment.id = experiment.get_experiment_profile().get('id') + status = experiment.get_status() + pid = experiment.get_experiment_metadata(experiment.id).get('pid') + if pid is None: + _logger.warning('Get experiment pid failed, can not stop experiment by stop().') + else: + experiment._proc = psutil.Process(pid) + _logger.info('Connect to port %d success, experiment id is %s, status is %s.', port, experiment.id, status) + return experiment + + @classmethod + def resume(cls, experiment_id: str, port: int = 8080, wait_completion: bool = True, debug: bool = False): + """ + Resume a stopped experiment. + + Parameters + ---------- + experiment_id + The stopped experiment id. + port + The port of web UI. + wait_completion + If true, run in the foreground. If false, run in the background. + debug + Whether to start in debug mode. + """ + experiment = Experiment() + experiment.id = experiment_id + experiment.mode = 'resume' + experiment.run(port=port, wait_completion=wait_completion, debug=debug) + if not wait_completion: + return experiment + + @classmethod + def view(cls, experiment_id: str, port: int = 8080, non_blocking: bool = False): + """ + View a stopped experiment. + + Parameters + ---------- + experiment_id + The stopped experiment id. + port + The port of web UI. + non_blocking + If false, run in the foreground. If true, run in the background. + """ + debug = False + experiment = Experiment() + experiment.id = experiment_id + experiment.mode = 'view' + experiment.start(port=port, debug=debug) + if non_blocking: + return experiment + else: + try: + while True: + time.sleep(10) + except KeyboardInterrupt: + _logger.warning('KeyboardInterrupt detected') + finally: + experiment.stop() + + def get_status(self) -> str: + """ + Return experiment status as a str. + + Returns + ------- + str + Experiment status. + """ + resp = rest.get(self.port, '/check-status') + return resp['status'] + + def get_trial_job(self, trial_job_id: str): + """ + Return a trial job. + + Parameters + ---------- + trial_job_id: str + Trial job id. + + Returns + ------- + TrialJob + A `TrialJob` instance corresponding to `trial_job_id`. + """ + resp = rest.get(self.port, '/trial-jobs/{}'.format(trial_job_id)) + return TrialJob(**resp) + + def list_trial_jobs(self): + """ + Return information for all trial jobs as a list. + + Returns + ------- + list + List of `TrialJob`. + """ + resp = rest.get(self.port, '/trial-jobs') + return [TrialJob(**trial_job) for trial_job in resp] + + def get_job_statistics(self): + """ + Return trial job statistics information as a dict. + + Returns + ------- + dict + Job statistics information. + """ + resp = rest.get(self.port, '/job-statistics') + return resp + + def get_job_metrics(self, trial_job_id=None): + """ + Return trial job metrics. + + Parameters + ---------- + trial_job_id: str + trial job id. if this parameter is None, all trail jobs' metrics will be returned. + + Returns + ------- + dict + Each key is a trialJobId, the corresponding value is a list of `TrialMetricData`. + """ + api = '/metric-data/{}'.format(trial_job_id) if trial_job_id else '/metric-data' + resp = rest.get(self.port, api) + metric_dict = {} + for metric in resp: + trial_id = metric["trialJobId"] + if trial_id not in metric_dict: + metric_dict[trial_id] = [TrialMetricData(**metric)] + else: + metric_dict[trial_id].append(TrialMetricData(**metric)) + return metric_dict + + def get_experiment_profile(self): + """ + Return experiment profile as a dict. + + Returns + ------- + dict + The profile of the experiment. + """ + resp = rest.get(self.port, '/experiment') + return resp + + def get_experiment_metadata(self, exp_id: str): + """ + Return experiment metadata with specified exp_id as a dict. + + Returns + ------- + dict + The specified experiment metadata. + """ + experiments_metadata = self.get_all_experiments_metadata() + for metadata in experiments_metadata: + if metadata['id'] == exp_id: + return metadata + return {} + + def get_all_experiments_metadata(self): + """ + Return all experiments metadata as a list. + + Returns + ------- + list + The experiments metadata. + """ + resp = rest.get(self.port, '/experiments-info') + return resp + + def export_data(self): + """ + Return exported information for all trial jobs. + + Returns + ------- + list + List of `TrialResult`. + """ + resp = rest.get(self.port, '/export-data') + return [TrialResult(**trial_result) for trial_result in resp] + + def _get_query_type(self, key: str): + if key == 'trialConcurrency': + return '?update_type=TRIAL_CONCURRENCY' + if key == 'maxExecDuration': + return '?update_type=MAX_EXEC_DURATION' + if key == 'searchSpace': + return '?update_type=SEARCH_SPACE' + if key == 'maxTrialNum': + return '?update_type=MAX_TRIAL_NUM' + + def _update_experiment_profile(self, key: str, value: Any): + """ + Update an experiment's profile + + Parameters + ---------- + key: str + One of `['trial_concurrency', 'max_experiment_duration', 'search_space', 'max_trial_number']`. + value: Any + New value of the key. + """ + api = '/experiment{}'.format(self._get_query_type(key)) + experiment_profile = self.get_experiment_profile() + experiment_profile['params'][key] = value + rest.put(self.port, api, experiment_profile) + logging.info('Successfully update %s.', key) + + def update_trial_concurrency(self, value: int): + """ + Update an experiment's trial_concurrency + + Parameters + ---------- + value: int + New trial_concurrency value. + """ + self._update_experiment_profile('trialConcurrency', value) + + def update_max_experiment_duration(self, value: str): + """ + Update an experiment's max_experiment_duration + + Parameters + ---------- + value: str + Strings like '1m' for one minute or '2h' for two hours. + SUFFIX may be 's' for seconds, 'm' for minutes, 'h' for hours or 'd' for days. + """ + self._update_experiment_profile('maxExecDuration', value) + + def update_search_space(self, value: dict): + """ + Update the experiment's search_space. + TODO: support searchspace file. + + Parameters + ---------- + value: dict + New search_space. + """ + value = json_tricks.dumps(value) + self._update_experiment_profile('searchSpace', value) + + def update_max_trial_number(self, value: int): + """ + Update an experiment's max_trial_number + + Parameters + ---------- + value: int + New max_trial_number value. + """ + self._update_experiment_profile('maxTrialNum', value) diff --git a/new_impl/cv/third_party/nni_new/experiment/launcher.py b/new_impl/cv/third_party/nni_new/experiment/launcher.py new file mode 100644 index 0000000000000000000000000000000000000000..7ee9e43242b70e8bef6c86ef4dee87bd52b09064 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/experiment/launcher.py @@ -0,0 +1,177 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import contextlib +import logging +from pathlib import Path +import socket +from subprocess import Popen +import sys +import time +from typing import Optional, Tuple + +import colorama + +import nni_node # pylint: disable=import-error +import nni.runtime.protocol + +from .config import ExperimentConfig +from .pipe import Pipe +from . import rest +from ..tools.nnictl.config_utils import Experiments, Config +from ..tools.nnictl.nnictl_utils import update_experiment + +_logger = logging.getLogger('nni.experiment') + + +def start_experiment(exp_id: str, config: ExperimentConfig, port: int, debug: bool, mode: str = 'new') -> Popen: + proc = None + + config.validate(initialized_tuner=False) + _ensure_port_idle(port) + + if mode != 'view': + if isinstance(config.training_service, list): # hybrid training service + _ensure_port_idle(port + 1, 'Hybrid training service requires an additional port') + elif config.training_service.platform in ['remote', 'openpai', 'kubeflow', 'frameworkcontroller', 'adl']: + _ensure_port_idle(port + 1, f'{config.training_service.platform} requires an additional port') + + try: + _logger.info('Creating experiment, Experiment ID: %s', colorama.Fore.CYAN + exp_id + colorama.Style.RESET_ALL) + start_time, proc = _start_rest_server(config, port, debug, exp_id, mode=mode) + _logger.info('Starting web server...') + _check_rest_server(port) + platform = 'hybrid' if isinstance(config.training_service, list) else config.training_service.platform + _save_experiment_information(exp_id, port, start_time, platform, + config.experiment_name, proc.pid, str(config.experiment_working_directory)) + _logger.info('Setting up...') + rest.post(port, '/experiment', config.json()) + return proc + + except Exception as e: + _logger.error('Create experiment failed') + if proc is not None: + with contextlib.suppress(Exception): + proc.kill() + raise e + +def start_experiment_retiarii(exp_id: str, config: ExperimentConfig, port: int, debug: bool) -> Popen: + pipe = None + proc = None + + config.validate(initialized_tuner=True) + _ensure_port_idle(port) + if isinstance(config.training_service, list): # hybrid training service + _ensure_port_idle(port + 1, 'Hybrid training service requires an additional port') + elif config.training_service.platform in ['remote', 'openpai', 'kubeflow', 'frameworkcontroller', 'adl']: + _ensure_port_idle(port + 1, f'{config.training_service.platform} requires an additional port') + + try: + _logger.info('Creating experiment, Experiment ID: %s', colorama.Fore.CYAN + exp_id + colorama.Style.RESET_ALL) + pipe = Pipe(exp_id) + start_time, proc = _start_rest_server(config, port, debug, exp_id, pipe.path) + _logger.info('Connecting IPC pipe...') + pipe_file = pipe.connect() + nni.runtime.protocol._in_file = pipe_file + nni.runtime.protocol._out_file = pipe_file + _logger.info('Starting web server...') + _check_rest_server(port) + platform = 'hybrid' if isinstance(config.training_service, list) else config.training_service.platform + _save_experiment_information(exp_id, port, start_time, platform, + config.experiment_name, proc.pid, config.experiment_working_directory) + _logger.info('Setting up...') + rest.post(port, '/experiment', config.json()) + return proc, pipe + + except Exception as e: + _logger.error('Create experiment failed') + if proc is not None: + with contextlib.suppress(Exception): + proc.kill() + if pipe is not None: + with contextlib.suppress(Exception): + pipe.close() + raise e + +def _ensure_port_idle(port: int, message: Optional[str] = None) -> None: + sock = socket.socket() + if sock.connect_ex(('localhost', port)) == 0: + sock.close() + message = f'(message)' if message else '' + raise RuntimeError(f'Port {port} is not idle {message}') + + +def _start_rest_server(config: ExperimentConfig, port: int, debug: bool, experiment_id: str, pipe_path: str = None, + mode: str = 'new') -> Tuple[int, Popen]: + if isinstance(config.training_service, list): + ts = 'hybrid' + else: + ts = config.training_service.platform + if ts == 'openpai': + ts = 'pai' + + args = { + 'port': port, + 'mode': ts, + 'experiment_id': experiment_id, + 'start_mode': mode, + 'log_dir': config.experiment_working_directory, + 'log_level': 'debug' if debug else 'info' + } + if pipe_path is not None: + args['dispatcher_pipe'] = pipe_path + + if mode == 'view': + args['start_mode'] = 'resume' + args['readonly'] = 'true' + + node_dir = Path(nni_node.__path__[0]) + node = str(node_dir / ('node.exe' if sys.platform == 'win32' else 'node')) + main_js = str(node_dir / 'main.js') + cmd = [node, '--max-old-space-size=4096', main_js] + for arg_key, arg_value in args.items(): + cmd.append('--' + arg_key) + cmd.append(str(arg_value)) + + if sys.platform == 'win32': + from subprocess import CREATE_NEW_PROCESS_GROUP + proc = Popen(cmd, cwd=node_dir, creationflags=CREATE_NEW_PROCESS_GROUP) + else: + if pipe_path is None: + import os + proc = Popen(cmd, cwd=node_dir, preexec_fn=os.setpgrp) + else: + proc = Popen(cmd, cwd=node_dir) + return int(time.time() * 1000), proc + + +def _check_rest_server(port: int, retry: int = 3) -> None: + for i in range(retry): + with contextlib.suppress(Exception): + rest.get(port, '/check-status') + return + if i > 0: + _logger.warning('Timeout, retry...') + time.sleep(1) + rest.get(port, '/check-status') + + +def _save_experiment_information(experiment_id: str, port: int, start_time: int, platform: str, name: str, pid: int, logDir: str) -> None: + experiments_config = Experiments() + experiments_config.add_experiment(experiment_id, port, start_time, platform, name, pid=pid, logDir=logDir) + + +def get_stopped_experiment_config(exp_id: str, mode: str) -> None: + update_experiment() + experiments_config = Experiments() + experiments_dict = experiments_config.get_all_experiments() + experiment_metadata = experiments_dict.get(exp_id) + if experiment_metadata is None: + _logger.error('Id %s not exist!', exp_id) + return + if experiment_metadata['status'] != 'STOPPED': + _logger.error('Only stopped experiments can be %sed!', mode) + return + experiment_config = Config(exp_id, experiment_metadata['logDir']).get_config() + config = ExperimentConfig(**experiment_config) + return config diff --git a/new_impl/cv/third_party/nni_new/experiment/management.py b/new_impl/cv/third_party/nni_new/experiment/management.py new file mode 100644 index 0000000000000000000000000000000000000000..b15c4d6d2561cc678574fc197a95d1d1d1ffafe3 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/experiment/management.py @@ -0,0 +1,16 @@ +from pathlib import Path +import random +import string + + +def generate_experiment_id() -> str: + return ''.join(random.sample(string.ascii_lowercase + string.digits, 8)) + + +def create_experiment_directory(experiment_id: str) -> Path: + path = Path.home() / 'nni-experiments' / experiment_id + path.mkdir(parents=True, exist_ok=True) + return path + + +# TODO: port shangning's work here, and use it in Experiment.start()/.stop() diff --git a/new_impl/cv/third_party/nni_new/experiment/pipe.py b/new_impl/cv/third_party/nni_new/experiment/pipe.py new file mode 100644 index 0000000000000000000000000000000000000000..e59fd8270bd74b1735cde9bae2983875b85f74e0 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/experiment/pipe.py @@ -0,0 +1,72 @@ +from io import BufferedIOBase +import logging +import os +import sys + +_logger = logging.getLogger(__name__) + +if sys.platform == 'win32': + import _winapi + import msvcrt + + class WindowsPipe: + def __init__(self, experiment_id: str): + self.path: str = r'\\.\pipe\nni-' + experiment_id + self.file = None + + self._handle = _winapi.CreateNamedPipe( + self.path, + _winapi.PIPE_ACCESS_DUPLEX, + _winapi.PIPE_TYPE_MESSAGE | _winapi.PIPE_READMODE_MESSAGE | _winapi.PIPE_WAIT, + 1, + 8192, + 8192, + 0, + _winapi.NULL + ) + + def connect(self) -> BufferedIOBase: + _winapi.ConnectNamedPipe(self._handle, _winapi.NULL) + fd = msvcrt.open_osfhandle(self._handle, 0) + self.file = os.fdopen(fd, 'w+b') + return self.file + + def close(self) -> None: + try: + if self.file is not None: + self.file.close() + except Exception as e: + _logger.debug('Error on closing Windows pipe: %s', e) + + Pipe = WindowsPipe + + +else: + import socket + + from . import management + + class UnixPipe: + def __init__(self, experiment_id: str): + self.path: str = str(management.create_experiment_directory(experiment_id) / 'dispatcher-pipe') + self.file = None + + self._socket = socket.socket(socket.AF_UNIX) + self._socket.bind(self.path) + self._socket.listen(1) # only accepts one connection + + def connect(self) -> BufferedIOBase: + conn, _ = self._socket.accept() + self.file = conn.makefile('rwb') + return self.file + + def close(self) -> None: + try: + if self.file is not None: + self.file.close() + self._socket.close() + os.unlink(self.path) + except Exception as e: + _logger.debug('Error on closing POSIX pipe: %s', e) + + Pipe = UnixPipe diff --git a/new_impl/cv/third_party/nni_new/experiment/rest.py b/new_impl/cv/third_party/nni_new/experiment/rest.py new file mode 100644 index 0000000000000000000000000000000000000000..bdacc7c215ac759fdb551e7d4fa3d6e296e45fd1 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/experiment/rest.py @@ -0,0 +1,35 @@ +import logging +from typing import Any, Optional + +import requests + +_logger = logging.getLogger(__name__) + +url_template = 'http://localhost:{}/api/v1/nni{}' +timeout = 20 + +def request(method: str, port: Optional[int], api: str, data: Any = None) -> Any: + if port is None: + raise RuntimeError('Experiment is not running') + url = url_template.format(port, api) + if data is None: + resp = requests.request(method, url, timeout=timeout) + else: + resp = requests.request(method, url, json=data, timeout=timeout) + if not resp.ok: + _logger.error('rest request %s %s failed: %s %s', method.upper(), url, resp.status_code, resp.text) + resp.raise_for_status() + if method.lower() in ['get', 'post'] and len(resp.content) > 0: + return resp.json() + +def get(port: Optional[int], api: str) -> Any: + return request('get', port, api) + +def post(port: Optional[int], api: str, data: Any) -> Any: + return request('post', port, api, data) + +def put(port: Optional[int], api: str, data: Any) -> None: + request('put', port, api, data) + +def delete(port: Optional[int], api: str) -> None: + request('delete', port, api) diff --git a/new_impl/cv/third_party/nni_new/feature_engineering/__init__.py b/new_impl/cv/third_party/nni_new/feature_engineering/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..8b137891791fe96927ad78e64b0aad7bded08bdc --- /dev/null +++ b/new_impl/cv/third_party/nni_new/feature_engineering/__init__.py @@ -0,0 +1 @@ + diff --git a/new_impl/cv/third_party/nni_new/feature_engineering/feature_selector.py b/new_impl/cv/third_party/nni_new/feature_engineering/feature_selector.py new file mode 100644 index 0000000000000000000000000000000000000000..32021bfb29931f459011edd4ceff3b5e2f899c99 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/feature_engineering/feature_selector.py @@ -0,0 +1,59 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# MIT License +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and +# associated documentation files (the "Software"), to deal in the Software without restriction, +# including without limitation the rights to use, copy, modify, merge, publish, distribute, +# sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or +# substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT +# NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT +# OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# ================================================================================================== + +import logging + +_logger = logging.getLogger(__name__) + + +class FeatureSelector(): + + def __init__(self, **kwargs): + self.selected_features_ = None + self.X = None + self.y = None + + + def fit(self, X, y, **kwargs): + """ + Fit the training data to FeatureSelector + + Paramters + --------- + X : array-like numpy matrix + The training input samples, which shape is [n_samples, n_features]. + y: array-like numpy matrix + The target values (class labels in classification, real numbers in + regression). Which shape is [n_samples]. + """ + self.X = X + self.y = y + + + def get_selected_features(self): + """ + Fit the training data to FeatureSelector + + Returns + ------- + list : + Return the index of imprtant feature. + """ + return self.selected_features_ diff --git a/new_impl/cv/third_party/nni_new/nas/__init__.py b/new_impl/cv/third_party/nni_new/nas/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/new_impl/cv/third_party/nni_new/nas/benchmarks/__init__.py b/new_impl/cv/third_party/nni_new/nas/benchmarks/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/new_impl/cv/third_party/nni_new/nas/benchmarks/constants.py b/new_impl/cv/third_party/nni_new/nas/benchmarks/constants.py new file mode 100644 index 0000000000000000000000000000000000000000..4a52fa8d8fce3d9a70f2ca38299a6667c35d9e55 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/nas/benchmarks/constants.py @@ -0,0 +1,4 @@ +import os + + +DATABASE_DIR = os.environ.get("NASBENCHMARK_DIR", os.path.expanduser("~/.nni/nasbenchmark")) diff --git a/new_impl/cv/third_party/nni_new/nas/benchmarks/nasbench101/__init__.py b/new_impl/cv/third_party/nni_new/nas/benchmarks/nasbench101/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e9aad0552cb9520d82685e28b0269ae85c54595e --- /dev/null +++ b/new_impl/cv/third_party/nni_new/nas/benchmarks/nasbench101/__init__.py @@ -0,0 +1,3 @@ +from .constants import INPUT, OUTPUT, CONV3X3_BN_RELU, CONV1X1_BN_RELU, MAXPOOL3X3 +from .model import Nb101TrialStats, Nb101IntermediateStats, Nb101TrialConfig +from .query import query_nb101_trial_stats diff --git a/new_impl/cv/third_party/nni_new/nas/benchmarks/nasbench101/constants.py b/new_impl/cv/third_party/nni_new/nas/benchmarks/nasbench101/constants.py new file mode 100644 index 0000000000000000000000000000000000000000..c2769f497f4dea6f0b5e1541d2413a8475866f15 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/nas/benchmarks/nasbench101/constants.py @@ -0,0 +1,14 @@ +INPUT = 'input' +OUTPUT = 'output' +CONV3X3_BN_RELU = 'conv3x3-bn-relu' +CONV1X1_BN_RELU = 'conv1x1-bn-relu' +MAXPOOL3X3 = 'maxpool3x3' + + +LABEL2ID = { + INPUT: -1, + OUTPUT: -2, + CONV3X3_BN_RELU: 0, + CONV1X1_BN_RELU: 1, + MAXPOOL3X3: 2 +} diff --git a/new_impl/cv/third_party/nni_new/nas/benchmarks/nasbench101/db_gen.py b/new_impl/cv/third_party/nni_new/nas/benchmarks/nasbench101/db_gen.py new file mode 100644 index 0000000000000000000000000000000000000000..04212ca270228fca0e1b07e728f69f229ff0893e --- /dev/null +++ b/new_impl/cv/third_party/nni_new/nas/benchmarks/nasbench101/db_gen.py @@ -0,0 +1,53 @@ +import argparse + +from tqdm import tqdm +from nasbench import api # pylint: disable=import-error + +from .model import db, Nb101TrialConfig, Nb101TrialStats, Nb101IntermediateStats +from .graph_util import nasbench_format_to_architecture_repr, hash_module + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('input_file', + help='Path to the file to be converted, e.g., nasbench_full.tfrecord') + args = parser.parse_args() + nasbench = api.NASBench(args.input_file) + with db: + db.create_tables([Nb101TrialConfig, Nb101TrialStats, Nb101IntermediateStats]) + for hashval in tqdm(nasbench.hash_iterator(), desc='Dumping data into database'): + metadata, metrics = nasbench.get_metrics_from_hash(hashval) + num_vertices, architecture = nasbench_format_to_architecture_repr( + metadata['module_adjacency'], metadata['module_operations']) + assert hashval == hash_module(architecture, num_vertices) + for epochs in [4, 12, 36, 108]: + trial_config = Nb101TrialConfig.create( + arch=architecture, + num_vertices=num_vertices, + hash=hashval, + num_epochs=epochs + ) + + for seed in range(3): + cur = metrics[epochs][seed] + trial = Nb101TrialStats.create( + config=trial_config, + train_acc=cur['final_train_accuracy'] * 100, + valid_acc=cur['final_validation_accuracy'] * 100, + test_acc=cur['final_test_accuracy'] * 100, + parameters=metadata['trainable_parameters'] / 1e6, + training_time=cur['final_training_time'] * 60 + ) + for t in ['halfway', 'final']: + Nb101IntermediateStats.create( + trial=trial, + current_epoch=epochs // 2 if t == 'halfway' else epochs, + training_time=cur[t + '_training_time'], + train_acc=cur[t + '_train_accuracy'] * 100, + valid_acc=cur[t + '_validation_accuracy'] * 100, + test_acc=cur[t + '_test_accuracy'] * 100 + ) + + +if __name__ == '__main__': + main() diff --git a/new_impl/cv/third_party/nni_new/nas/benchmarks/nasbench101/graph_util.py b/new_impl/cv/third_party/nni_new/nas/benchmarks/nasbench101/graph_util.py new file mode 100644 index 0000000000000000000000000000000000000000..10805685fec3ff7359ec39dc0ae1c019e67950ae --- /dev/null +++ b/new_impl/cv/third_party/nni_new/nas/benchmarks/nasbench101/graph_util.py @@ -0,0 +1,111 @@ +import hashlib + +import numpy as np + +from .constants import INPUT, LABEL2ID, OUTPUT + + +def _labeling_from_architecture(architecture, vertices): + return [INPUT] + [architecture['op{}'.format(i)] for i in range(1, vertices - 1)] + [OUTPUT] + + +def _adjancency_matrix_from_architecture(architecture, vertices): + matrix = np.zeros((vertices, vertices), dtype=np.bool) + for i in range(1, vertices): + for k in architecture['input{}'.format(i)]: + matrix[k, i] = 1 + return matrix + + +def nasbench_format_to_architecture_repr(adjacency_matrix, labeling): + """ + Computes a graph-invariance MD5 hash of the matrix and label pair. + Imported from NAS-Bench-101 repo. + + Parameters + ---------- + adjacency_matrix : np.ndarray + A 2D array of shape NxN, where N is the number of vertices. + ``matrix[u][v]`` is 1 if there is a direct edge from `u` to `v`, + otherwise it will be 0. + labeling : list of str + A list of str that starts with input and ends with output. The intermediate + nodes are chosen from candidate operators. + + Returns + ------- + tuple and int and dict + Converted number of vertices and architecture. + """ + num_vertices = adjacency_matrix.shape[0] + assert len(labeling) == num_vertices + architecture = {} + for i in range(1, num_vertices - 1): + architecture['op{}'.format(i)] = labeling[i] + assert labeling[i] not in [INPUT, OUTPUT] + for i in range(1, num_vertices): + architecture['input{}'.format(i)] = [k for k in range(i) if adjacency_matrix[k, i]] + return num_vertices, architecture + + +def infer_num_vertices(architecture): + """ + Infer number of vertices from an architecture dict. + + Parameters + ---------- + architecture : dict + Architecture in NNI format. + + Returns + ------- + int + Number of vertices. + """ + op_keys = set([k for k in architecture.keys() if k.startswith('op')]) + intermediate_vertices = len(op_keys) + assert op_keys == {'op{}'.format(i) for i in range(1, intermediate_vertices + 1)} + return intermediate_vertices + 2 + + +def hash_module(architecture, vertices): + """ + Computes a graph-invariance MD5 hash of the matrix and label pair. + This snippet is modified from code in NAS-Bench-101 repo. + + Parameters + ---------- + matrix : np.ndarray + Square upper-triangular adjacency matrix. + labeling : list of int + Labels of length equal to both dimensions of matrix. + + Returns + ------- + str + MD5 hash of the matrix and labeling. + """ + labeling = _labeling_from_architecture(architecture, vertices) + labeling = [LABEL2ID[t] for t in labeling] + matrix = _adjancency_matrix_from_architecture(architecture, vertices) + in_edges = np.sum(matrix, axis=0).tolist() + out_edges = np.sum(matrix, axis=1).tolist() + + assert len(in_edges) == len(out_edges) == len(labeling) + hashes = list(zip(out_edges, in_edges, labeling)) + hashes = [hashlib.md5(str(h).encode('utf-8')).hexdigest() for h in hashes] + # Computing this up to the diameter is probably sufficient but since the + # operation is fast, it is okay to repeat more times. + for _ in range(vertices): + new_hashes = [] + for v in range(vertices): + in_neighbors = [hashes[w] for w in range(vertices) if matrix[w, v]] + out_neighbors = [hashes[w] for w in range(vertices) if matrix[v, w]] + new_hashes.append(hashlib.md5( + (''.join(sorted(in_neighbors)) + '|' + + ''.join(sorted(out_neighbors)) + '|' + + hashes[v]).encode('utf-8')).hexdigest()) + hashes = new_hashes + fingerprint = hashlib.md5(str(sorted(hashes)).encode('utf-8')).hexdigest() + + return fingerprint diff --git a/new_impl/cv/third_party/nni_new/nas/benchmarks/nasbench101/model.py b/new_impl/cv/third_party/nni_new/nas/benchmarks/nasbench101/model.py new file mode 100644 index 0000000000000000000000000000000000000000..44ec3f874f5ba58df03b7862bce9fc127a8094aa --- /dev/null +++ b/new_impl/cv/third_party/nni_new/nas/benchmarks/nasbench101/model.py @@ -0,0 +1,102 @@ +import os + +from peewee import CharField, FloatField, ForeignKeyField, IntegerField, Model +from playhouse.sqlite_ext import JSONField, SqliteExtDatabase + +from nni.nas.benchmarks.constants import DATABASE_DIR +from nni.nas.benchmarks.utils import json_dumps + +db = SqliteExtDatabase(os.path.join(DATABASE_DIR, 'nasbench101.db'), autoconnect=True) + + +class Nb101TrialConfig(Model): + """ + Trial config for NAS-Bench-101. + + Attributes + ---------- + arch : dict + A dict with keys ``op1``, ``op2``, ... and ``input1``, ``input2``, ... Vertices are + enumerate from 0. Since node 0 is input node, it is skipped in this dict. Each ``op`` + is one of :const:`nni.nas.benchmark.nasbench101.CONV3X3_BN_RELU`, + :const:`nni.nas.benchmark.nasbench101.CONV1X1_BN_RELU`, and :const:`nni.nas.benchmark.nasbench101.MAXPOOL3X3`. + Each ``input`` is a list of previous nodes. For example ``input5`` can be ``[0, 1, 3]``. + num_vertices : int + Number of vertices (nodes) in one cell. Should be less than or equal to 7 in default setup. + hash : str + Graph-invariant MD5 string for this architecture. + num_epochs : int + Number of epochs planned for this trial. Should be one of 4, 12, 36, 108 in default setup. + """ + + arch = JSONField(json_dumps=json_dumps, index=True) + num_vertices = IntegerField(index=True) + hash = CharField(max_length=64, index=True) + num_epochs = IntegerField(index=True) + + class Meta: + database = db + + +class Nb101TrialStats(Model): + """ + Computation statistics for NAS-Bench-101. Each corresponds to one trial. + Each config has multiple trials with different random seeds, but unfortunately seed for each trial is unavailable. + NAS-Bench-101 trains and evaluates on CIFAR-10 by default. The original training set is divided into + 40k training images and 10k validation images, and the original validation set is used for test only. + + Attributes + ---------- + config : Nb101TrialConfig + Setup for this trial data. + train_acc : float + Final accuracy on training data, ranging from 0 to 100. + valid_acc : float + Final accuracy on validation data, ranging from 0 to 100. + test_acc : float + Final accuracy on test data, ranging from 0 to 100. + parameters : float + Number of trainable parameters in million. + training_time : float + Duration of training in seconds. + """ + config = ForeignKeyField(Nb101TrialConfig, backref='trial_stats', index=True) + train_acc = FloatField() + valid_acc = FloatField() + test_acc = FloatField() + parameters = FloatField() + training_time = FloatField() + + class Meta: + database = db + + +class Nb101IntermediateStats(Model): + """ + Intermediate statistics for NAS-Bench-101. + + Attributes + ---------- + trial : Nb101TrialStats + The exact trial where the intermediate result is produced. + current_epoch : int + Elapsed epochs when evaluation is done. + train_acc : float + Intermediate accuracy on training data, ranging from 0 to 100. + valid_acc : float + Intermediate accuracy on validation data, ranging from 0 to 100. + test_acc : float + Intermediate accuracy on test data, ranging from 0 to 100. + training_time : float + Time elapsed in seconds. + """ + + trial = ForeignKeyField(Nb101TrialStats, backref='intermediates', index=True) + current_epoch = IntegerField(index=True) + train_acc = FloatField() + valid_acc = FloatField() + test_acc = FloatField() + training_time = FloatField() + + class Meta: + database = db diff --git a/new_impl/cv/third_party/nni_new/nas/benchmarks/nasbench101/query.py b/new_impl/cv/third_party/nni_new/nas/benchmarks/nasbench101/query.py new file mode 100644 index 0000000000000000000000000000000000000000..bd43d722f89a0e555cfd8b1843c079c9cf1f4873 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/nas/benchmarks/nasbench101/query.py @@ -0,0 +1,70 @@ +import functools + +from peewee import fn +from playhouse.shortcuts import model_to_dict +from .model import Nb101TrialStats, Nb101TrialConfig +from .graph_util import hash_module, infer_num_vertices + + +def query_nb101_trial_stats(arch, num_epochs, isomorphism=True, reduction=None, include_intermediates=False): + """ + Query trial stats of NAS-Bench-101 given conditions. + + Parameters + ---------- + arch : dict or None + If a dict, it is in the format that is described in + :class:`nni.nas.benchmark.nasbench101.Nb101TrialConfig`. Only trial stats + matched will be returned. If none, all architectures in the database will be matched. + num_epochs : int or None + If int, matching results will be returned. Otherwise a wildcard. + isomorphism : boolean + Whether to match essentially-same architecture, i.e., architecture with the + same graph-invariant hash value. + reduction : str or None + If 'none' or None, all trial stats will be returned directly. + If 'mean', fields in trial stats will be averaged given the same trial config. + include_intermediates : boolean + If true, intermediate results will be returned. + + Returns + ------- + generator of dict + A generator of :class:`nni.nas.benchmark.nasbench101.Nb101TrialStats` objects, + where each of them has been converted into a dict. + """ + fields = [] + if reduction == 'none': + reduction = None + if reduction == 'mean': + for field_name in Nb101TrialStats._meta.sorted_field_names: + if field_name not in ['id', 'config']: + fields.append(fn.AVG(getattr(Nb101TrialStats, field_name)).alias(field_name)) + elif reduction is None: + fields.append(Nb101TrialStats) + else: + raise ValueError('Unsupported reduction: \'%s\'' % reduction) + query = Nb101TrialStats.select(*fields, Nb101TrialConfig).join(Nb101TrialConfig) + conditions = [] + if arch is not None: + if isomorphism: + num_vertices = infer_num_vertices(arch) + conditions.append(Nb101TrialConfig.hash == hash_module(arch, num_vertices)) + else: + conditions.append(Nb101TrialConfig.arch == arch) + if num_epochs is not None: + conditions.append(Nb101TrialConfig.num_epochs == num_epochs) + if conditions: + query = query.where(functools.reduce(lambda a, b: a & b, conditions)) + if reduction is not None: + query = query.group_by(Nb101TrialStats.config) + for trial in query: + if include_intermediates: + data = model_to_dict(trial) + # exclude 'trial' from intermediates as it is already available in data + data['intermediates'] = [ + {k: v for k, v in model_to_dict(t).items() if k != 'trial'} for t in trial.intermediates + ] + yield data + else: + yield model_to_dict(trial) diff --git a/new_impl/cv/third_party/nni_new/nas/benchmarks/nasbench201/__init__.py b/new_impl/cv/third_party/nni_new/nas/benchmarks/nasbench201/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..3ce3d0170d29d2e6c0fbdfdcff17077be72dd420 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/nas/benchmarks/nasbench201/__init__.py @@ -0,0 +1,3 @@ +from .constants import NONE, SKIP_CONNECT, CONV_1X1, CONV_3X3, AVG_POOL_3X3 +from .model import Nb201TrialStats, Nb201IntermediateStats, Nb201TrialConfig +from .query import query_nb201_trial_stats diff --git a/new_impl/cv/third_party/nni_new/nas/benchmarks/nasbench201/constants.py b/new_impl/cv/third_party/nni_new/nas/benchmarks/nasbench201/constants.py new file mode 100644 index 0000000000000000000000000000000000000000..c9d6bce5d112f3d84ae0dd17a780e4a3e83758bd --- /dev/null +++ b/new_impl/cv/third_party/nni_new/nas/benchmarks/nasbench201/constants.py @@ -0,0 +1,12 @@ +NONE = 'none' +SKIP_CONNECT = 'skip_connect' +CONV_1X1 = 'conv_1x1' +CONV_3X3 = 'conv_3x3' +AVG_POOL_3X3 = 'avg_pool_3x3' +PRIMITIVES = [ + NONE, + SKIP_CONNECT, + CONV_1X1, + CONV_3X3, + AVG_POOL_3X3, +] diff --git a/new_impl/cv/third_party/nni_new/nas/benchmarks/nasbench201/db_gen.py b/new_impl/cv/third_party/nni_new/nas/benchmarks/nasbench201/db_gen.py new file mode 100644 index 0000000000000000000000000000000000000000..e2202612b4a76ce7bafd063c62faba7859f18120 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/nas/benchmarks/nasbench201/db_gen.py @@ -0,0 +1,106 @@ +import argparse +import re + +import tqdm +import torch + +from .constants import NONE, SKIP_CONNECT, CONV_1X1, CONV_3X3, AVG_POOL_3X3 +from .model import db, Nb201TrialConfig, Nb201TrialStats, Nb201IntermediateStats + + +def parse_arch_str(arch_str): + mp = { + 'none': NONE, + 'skip_connect': SKIP_CONNECT, + 'nor_conv_1x1': CONV_1X1, + 'nor_conv_3x3': CONV_3X3, + 'avg_pool_3x3': AVG_POOL_3X3 + } + m = re.match(r'\|(.*)~0\|\+\|(.*)~0\|(.*)~1\|\+\|(.*)~0\|(.*)~1\|(.*)~2\|', arch_str) + return { + '0_1': mp[m.group(1)], + '0_2': mp[m.group(2)], + '1_2': mp[m.group(3)], + '0_3': mp[m.group(4)], + '1_3': mp[m.group(5)], + '2_3': mp[m.group(6)] + } + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('input_file', + help='Path to the file to be converted, e.g., NAS-Bench-201-v1_1-096897.pth.') + args = parser.parse_args() + dataset_split = { + 'cifar10-valid': ['train', 'x-valid', 'ori-test', 'ori-test'], + 'cifar10': ['train', 'ori-test', 'ori-test', 'ori-test'], + 'cifar100': ['train', 'x-valid', 'x-test', 'ori-test'], + 'imagenet16-120': ['train', 'x-valid', 'x-test', 'ori-test'], + } + + with db: + db.create_tables([Nb201TrialConfig, Nb201TrialStats, Nb201IntermediateStats]) + print('Loading NAS-Bench-201 pickle...') + nb201_data = torch.load(args.input_file) + print('Dumping architectures...') + for arch_str in nb201_data['meta_archs']: + arch_json = parse_arch_str(arch_str) + for epochs in [12, 200]: + for dataset in Nb201TrialConfig.dataset.choices: + Nb201TrialConfig.create(arch=arch_json, num_epochs=epochs, dataset=dataset, + num_channels=16, num_cells=5) + for arch_info in tqdm.tqdm(nb201_data['arch2infos'].values(), + desc='Processing architecture statistics'): + for epochs_verb, d in arch_info.items(): + if epochs_verb == 'less': + epochs = 12 + else: + epochs = 200 + arch_json = parse_arch_str(d['arch_str']) + for (dataset, seed), r in d['all_results'].items(): + sp = dataset_split[dataset.lower()] + data_parsed = { + 'train_acc': r['train_acc1es'][epochs - 1], + 'valid_acc': r['eval_acc1es']['{}@{}'.format(sp[1], epochs - 1)], + 'test_acc': r['eval_acc1es']['{}@{}'.format(sp[2], epochs - 1)], + 'ori_test_acc': r['eval_acc1es']['{}@{}'.format(sp[3], epochs - 1)], + 'train_loss': r['train_losses'][epochs - 1], + 'valid_loss': r['eval_losses']['{}@{}'.format(sp[1], epochs - 1)], + 'test_loss': r['eval_losses']['{}@{}'.format(sp[2], epochs - 1)], + 'ori_test_loss': r['eval_losses']['{}@{}'.format(sp[3], epochs - 1)], + 'parameters': r['params'], + 'flops': r['flop'], + 'latency': r['latency'][0], + 'training_time': r['train_times'][epochs - 1] * epochs, + 'valid_evaluation_time': r['eval_times']['{}@{}'.format(sp[1], epochs - 1)], + 'test_evaluation_time': r['eval_times']['{}@{}'.format(sp[2], epochs - 1)], + 'ori_test_evaluation_time': r['eval_times']['{}@{}'.format(sp[3], epochs - 1)], + } + config = Nb201TrialConfig.get( + (Nb201TrialConfig.num_epochs == epochs) & + (Nb201TrialConfig.arch == arch_json) & + (Nb201TrialConfig.dataset == dataset.lower()) + ) + trial_stats = Nb201TrialStats.create(config=config, seed=seed, **data_parsed) + intermediate_stats = [] + for epoch in range(epochs): + data_parsed = { + 'train_acc': r['train_acc1es'].get(epoch), + 'valid_acc': r['eval_acc1es'].get('{}@{}'.format(sp[1], epoch)), + 'test_acc': r['eval_acc1es'].get('{}@{}'.format(sp[2], epoch)), + 'ori_test_acc': r['eval_acc1es'].get('{}@{}'.format(sp[3], epoch)), + 'train_loss': r['train_losses'].get(epoch), + 'valid_loss': r['eval_losses'].get('{}@{}'.format(sp[1], epoch)), + 'test_loss': r['eval_losses'].get('{}@{}'.format(sp[2], epoch)), + 'ori_test_loss': r['eval_losses'].get('{}@{}'.format(sp[3], epoch)), + } + if all([v is None for v in data_parsed.values()]): + continue + data_parsed.update(current_epoch=epoch + 1, trial=trial_stats) + intermediate_stats.append(data_parsed) + Nb201IntermediateStats.insert_many(intermediate_stats).execute(db) + + +if __name__ == '__main__': + main() diff --git a/new_impl/cv/third_party/nni_new/nas/benchmarks/nasbench201/model.py b/new_impl/cv/third_party/nni_new/nas/benchmarks/nasbench201/model.py new file mode 100644 index 0000000000000000000000000000000000000000..b390a43091bcedcdca7e599eec38b9c9a31f6147 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/nas/benchmarks/nasbench201/model.py @@ -0,0 +1,160 @@ +import os + +from peewee import CharField, FloatField, ForeignKeyField, IntegerField, Model +from playhouse.sqlite_ext import JSONField, SqliteExtDatabase + +from nni.nas.benchmarks.constants import DATABASE_DIR +from nni.nas.benchmarks.utils import json_dumps + +db = SqliteExtDatabase(os.path.join(DATABASE_DIR, 'nasbench201.db'), autoconnect=True) + + +class Nb201TrialConfig(Model): + """ + Trial config for NAS-Bench-201. + + Attributes + ---------- + arch : dict + A dict with keys ``0_1``, ``0_2``, ``0_3``, ``1_2``, ``1_3``, ``2_3``, each of which + is an operator chosen from :const:`nni.nas.benchmark.nasbench201.NONE`, + :const:`nni.nas.benchmark.nasbench201.SKIP_CONNECT`, + :const:`nni.nas.benchmark.nasbench201.CONV_1X1`, + :const:`nni.nas.benchmark.nasbench201.CONV_3X3` and :const:`nni.nas.benchmark.nasbench201.AVG_POOL_3X3`. + num_epochs : int + Number of epochs planned for this trial. Should be one of 12 and 200. + num_channels: int + Number of channels for initial convolution. 16 by default. + num_cells: int + Number of cells per stage. 5 by default. + dataset: str + Dataset used for training and evaluation. NAS-Bench-201 provides the following 4 options: + ``cifar10-valid`` (training data is splited into 25k for training and 25k for validation, + validation data is used for test), ``cifar10`` (training data is used in training, validation + data is splited into 5k for validation and 5k for testing), ``cifar100`` (same protocol as ``cifar10``), + and ``imagenet16-120`` (a subset of 120 classes in ImageNet, downscaled to 16x16, using training data + for training, 6k images from validation set for validation and the other 6k for testing). + """ + + arch = JSONField(json_dumps=json_dumps, index=True) + num_epochs = IntegerField(index=True) + num_channels = IntegerField() + num_cells = IntegerField() + dataset = CharField(max_length=20, index=True, choices=[ + 'cifar10-valid', # 25k+25k+10k + 'cifar10', # 50k+5k+5k + 'cifar100', # 50k+5k+5k + 'imagenet16-120', + ]) + + class Meta: + database = db + + +class Nb201TrialStats(Model): + """ + Computation statistics for NAS-Bench-201. Each corresponds to one trial. + + Attributes + ---------- + config : Nb201TrialConfig + Setup for this trial data. + seed : int + Random seed selected, for reproduction. + train_acc : float + Final accuracy on training data, ranging from 0 to 100. + valid_acc : float + Final accuracy on validation data, ranging from 0 to 100. + test_acc : float + Final accuracy on test data, ranging from 0 to 100. + ori_test_acc : float + Test accuracy on original validation set (10k for CIFAR and 12k for Imagenet16-120), + ranging from 0 to 100. + train_loss : float or None + Final cross entropy loss on training data. Note that loss could be NaN, in which case + this attributed will be None. + valid_loss : float or None + Final cross entropy loss on validation data. + test_loss : float or None + Final cross entropy loss on test data. + ori_test_loss : float or None + Final cross entropy loss on original validation set. + parameters : float + Number of trainable parameters in million. + latency : float + Latency in seconds. + flops : float + FLOPs in million. + training_time : float + Duration of training in seconds. + valid_evaluation_time : float + Time elapsed to evaluate on validation set. + test_evaluation_time : float + Time elapsed to evaluate on test set. + ori_test_evaluation_time : float + Time elapsed to evaluate on original test set. + """ + config = ForeignKeyField(Nb201TrialConfig, backref='trial_stats', index=True) + seed = IntegerField() + train_acc = FloatField() + valid_acc = FloatField() + test_acc = FloatField() + ori_test_acc = FloatField() # test accuracy of the original test set + train_loss = FloatField(null=True) # possibly nan + valid_loss = FloatField(null=True) + test_loss = FloatField(null=True) + ori_test_loss = FloatField(null=True) + parameters = FloatField() # parameters in million + latency = FloatField() # latency in milliseconds + flops = FloatField() # flops in million + training_time = FloatField() + valid_evaluation_time = FloatField() + test_evaluation_time = FloatField() + ori_test_evaluation_time = FloatField() + + class Meta: + database = db + + +class Nb201IntermediateStats(Model): + """ + Intermediate statistics for NAS-Bench-201. + + Attributes + ---------- + trial : Nb201TrialStats + Corresponding trial. + current_epoch : int + Elapsed epochs. + train_acc : float + Current accuracy on training data, ranging from 0 to 100. + valid_acc : float + Current accuracy on validation data, ranging from 0 to 100. + test_acc : float + Current accuracy on test data, ranging from 0 to 100. + ori_test_acc : float + Test accuracy on original validation set (10k for CIFAR and 12k for Imagenet16-120), + ranging from 0 to 100. + train_loss : float or None + Current cross entropy loss on training data. + valid_loss : float or None + Current cross entropy loss on validation data. + test_loss : float or None + Current cross entropy loss on test data. + ori_test_loss : float or None + Current cross entropy loss on original validation set. + """ + + trial = ForeignKeyField(Nb201TrialStats, backref='intermediates', index=True) + current_epoch = IntegerField(index=True) + train_acc = FloatField(null=True) + valid_acc = FloatField(null=True) + test_acc = FloatField(null=True) + ori_test_acc = FloatField(null=True) + train_loss = FloatField(null=True) + valid_loss = FloatField(null=True) + test_loss = FloatField(null=True) + ori_test_loss = FloatField(null=True) + + class Meta: + database = db diff --git a/new_impl/cv/third_party/nni_new/nas/benchmarks/nasbench201/query.py b/new_impl/cv/third_party/nni_new/nas/benchmarks/nasbench201/query.py new file mode 100644 index 0000000000000000000000000000000000000000..b52548ecd35f897bddc74c42cc129268a6e75b80 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/nas/benchmarks/nasbench201/query.py @@ -0,0 +1,67 @@ +import functools + +from peewee import fn +from playhouse.shortcuts import model_to_dict +from .model import Nb201TrialStats, Nb201TrialConfig + + +def query_nb201_trial_stats(arch, num_epochs, dataset, reduction=None, include_intermediates=False): + """ + Query trial stats of NAS-Bench-201 given conditions. + + Parameters + ---------- + arch : dict or None + If a dict, it is in the format that is described in + :class:`nni.nas.benchmark.nasbench201.Nb201TrialConfig`. Only trial stats + matched will be returned. If none, all architectures in the database will be matched. + num_epochs : int or None + If int, matching results will be returned. Otherwise a wildcard. + dataset : str or None + If specified, can be one of the dataset available in :class:`nni.nas.benchmark.nasbench201.Nb201TrialConfig`. + Otherwise a wildcard. + reduction : str or None + If 'none' or None, all trial stats will be returned directly. + If 'mean', fields in trial stats will be averaged given the same trial config. + include_intermediates : boolean + If true, intermediate results will be returned. + + Returns + ------- + generator of dict + A generator of :class:`nni.nas.benchmark.nasbench201.Nb201TrialStats` objects, + where each of them has been converted into a dict. + """ + fields = [] + if reduction == 'none': + reduction = None + if reduction == 'mean': + for field_name in Nb201TrialStats._meta.sorted_field_names: + if field_name not in ['id', 'config', 'seed']: + fields.append(fn.AVG(getattr(Nb201TrialStats, field_name)).alias(field_name)) + elif reduction is None: + fields.append(Nb201TrialStats) + else: + raise ValueError('Unsupported reduction: \'%s\'' % reduction) + query = Nb201TrialStats.select(*fields, Nb201TrialConfig).join(Nb201TrialConfig) + conditions = [] + if arch is not None: + conditions.append(Nb201TrialConfig.arch == arch) + if num_epochs is not None: + conditions.append(Nb201TrialConfig.num_epochs == num_epochs) + if dataset is not None: + conditions.append(Nb201TrialConfig.dataset == dataset) + if conditions: + query = query.where(functools.reduce(lambda a, b: a & b, conditions)) + if reduction is not None: + query = query.group_by(Nb201TrialStats.config) + for trial in query: + if include_intermediates: + data = model_to_dict(trial) + # exclude 'trial' from intermediates as it is already available in data + data['intermediates'] = [ + {k: v for k, v in model_to_dict(t).items() if k != 'trial'} for t in trial.intermediates + ] + yield data + else: + yield model_to_dict(trial) diff --git a/new_impl/cv/third_party/nni_new/nas/benchmarks/nds/__init__.py b/new_impl/cv/third_party/nni_new/nas/benchmarks/nds/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9d393b86e1948d6ea5fdf8cacee6d4fd645c8434 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/nas/benchmarks/nds/__init__.py @@ -0,0 +1,3 @@ +from .constants import * +from .model import NdsTrialConfig, NdsTrialStats, NdsIntermediateStats +from .query import query_nds_trial_stats diff --git a/new_impl/cv/third_party/nni_new/nas/benchmarks/nds/constants.py b/new_impl/cv/third_party/nni_new/nas/benchmarks/nds/constants.py new file mode 100644 index 0000000000000000000000000000000000000000..164f094f3aacb8ac36a8572c28e31e9419baa416 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/nas/benchmarks/nds/constants.py @@ -0,0 +1,16 @@ +NONE = 'none' +SKIP_CONNECT = 'skip_connect' +AVG_POOL_3X3 = 'avg_pool_3x3' +MAX_POOL_3X3 = 'max_pool_3x3' +MAX_POOL_5X5 = 'max_pool_5x5' +MAX_POOL_7X7 = 'max_pool_7x7' +CONV_1X1 = 'conv_1x1' +CONV_3X3 = 'conv_3x3' +CONV_3X1_1X3 = 'conv_3x1_1x3' +CONV_7X1_1X7 = 'conv_7x1_1x7' +DIL_CONV_3X3 = 'dil_conv_3x3' +DIL_CONV_5X5 = 'dil_conv_5x5' +SEP_CONV_3X3 = 'sep_conv_3x3' +SEP_CONV_5X5 = 'sep_conv_5x5' +SEP_CONV_7X7 = 'sep_conv_7x7' +DIL_SEP_CONV_3X3 = 'dil_sep_conv_3x3' diff --git a/new_impl/cv/third_party/nni_new/nas/benchmarks/nds/db_gen.py b/new_impl/cv/third_party/nni_new/nas/benchmarks/nds/db_gen.py new file mode 100644 index 0000000000000000000000000000000000000000..d4df09d4e3befba746806d5029c38420c237b2d3 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/nas/benchmarks/nds/db_gen.py @@ -0,0 +1,153 @@ +import json +import argparse +import os + +import numpy as np +import tqdm + +from .model import db, NdsTrialConfig, NdsTrialStats, NdsIntermediateStats + + +def inject_item(db, item, proposer, dataset, generator): + if 'genotype' in item['net']: + model_family = 'nas_cell' + num_nodes_normal = len(item['net']['genotype']['normal']) // 2 + num_nodes_reduce = len(item['net']['genotype']['reduce']) // 2 + model_spec = { + 'num_nodes_normal': num_nodes_normal, + 'num_nodes_reduce': num_nodes_reduce, + 'depth': item['net']['depth'], + 'width': item['net']['width'], + 'aux': item['net']['aux'], + 'drop_prob': item['net']['drop_prob'], + } + cell_spec = {} + for cell_type in ['normal', 'reduce']: + for i in range(num_nodes_normal): + for j, label in enumerate(['x', 'y']): + cell_spec['{}_{}_op_{}'.format(cell_type, i, label)] = \ + item['net']['genotype'][cell_type][i * 2 + j][0] + cell_spec['{}_{}_input_{}'.format(cell_type, i, label)] = \ + item['net']['genotype'][cell_type][i * 2 + j][1] + cell_spec['{}_concat'.format(cell_type)] = item['net']['genotype']['{}_concat'.format(cell_type)] + else: + if item['net']['block_type'].startswith('res_bottleneck'): + model_family = 'residual_bottleneck' + elif item['net']['block_type'].startswith('res_basic'): + model_family = 'residual_basic' + elif item['net']['block_type'].startswith('double_plain'): + model_family = 'vanilla' + else: + raise ValueError('Unrecognized block type') + model_spec = {k: v for k, v in item['net'].items() if v and k != 'block_type'} + cell_spec = {} + trial_config, _ = NdsTrialConfig.get_or_create( + model_family=model_family, + model_spec=model_spec, + cell_spec=cell_spec, + proposer=proposer, + base_lr=item['optim']['base_lr'], + weight_decay=item['optim']['wd'], + num_epochs=item['optim']['max_ep'], + dataset=dataset, + generator=generator + ) + assert len(item['train_ep_top1']) == len(item['test_ep_top1']) == trial_config.num_epochs + trial = NdsTrialStats.create( + config=trial_config, + seed=item['rng_seed'], + final_train_acc=100 - item['train_ep_top1'][-1], + final_train_loss=item['train_ep_loss'][-1], + final_test_acc=100 - item['test_ep_top1'][-1], + best_train_acc=100 - min(item['train_ep_top1']), + best_train_loss=np.nanmin(item['train_ep_loss']).item(), + best_test_acc=100 - min(item['test_ep_top1']), + parameters=item['params'] / 1e6, + flops=item['flops'] / 1e6, + iter_time=item['iter_time'] + ) + intermediate_stats = [] + for i in range(trial_config.num_epochs): + intermediate_stats.append({ + 'trial': trial, + 'current_epoch': i + 1, + 'train_loss': item['train_ep_loss'][i], + 'train_acc': 100 - item['train_ep_top1'][i], + 'test_acc': 100 - item['test_ep_top1'][i] + }) + NdsIntermediateStats.insert_many(intermediate_stats).execute(db) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('input_dir', help='Path to extracted NDS data dir.') + args = parser.parse_args() + + sweep_list = [ + 'Amoeba.json', + 'Amoeba_in.json', + 'DARTS.json', + 'DARTS_fix-w-d.json', + 'DARTS_in.json', + 'DARTS_lr-wd.json', + 'DARTS_lr-wd_in.json', + 'ENAS.json', + 'ENAS_fix-w-d.json', + 'ENAS_in.json', + 'NASNet.json', + 'NASNet_in.json', + 'PNAS.json', + 'PNAS_fix-w-d.json', + 'PNAS_in.json', + 'ResNeXt-A.json', + 'ResNeXt-A_in.json', + 'ResNeXt-B.json', + 'ResNeXt-B_in.json', + 'ResNet-B.json', + 'ResNet.json', + 'ResNet_lr-wd.json', + 'ResNet_lr-wd_in.json', + 'ResNet_reruns.json', + 'ResNet_rng1.json', + 'ResNet_rng2.json', + 'ResNet_rng3.json', + 'Vanilla.json', + 'Vanilla_lr-wd.json', + 'Vanilla_lr-wd_in.json', + 'Vanilla_reruns.json', + 'Vanilla_rng1.json', + 'Vanilla_rng2.json', + 'Vanilla_rng3.json' + ] + + with db: + db.create_tables([NdsTrialConfig, NdsTrialStats, NdsIntermediateStats]) + for json_idx, json_file in enumerate(sweep_list, start=1): + if 'fix-w-d' in json_file: + generator = 'fix_w_d' + elif 'lr-wd' in json_file: + generator = 'tune_lr_wd' + else: + generator = 'random' + if '_in' in json_file: + dataset = 'imagenet' + else: + dataset = 'cifar10' + proposer = json_file.split(".")[0].split("_")[0].lower() + with open(os.path.join(args.input_dir, json_file), 'r') as f: + data = json.load(f) + if 'top' in data and 'mid' in data: + for t in tqdm.tqdm(data['top'], + desc='[{}/{}] Processing {} (top)'.format(json_idx, len(sweep_list), json_file)): + inject_item(db, t, proposer, dataset, generator) + for t in tqdm.tqdm(data['mid'], + desc='[{}/{}] Processing {} (mid)'.format(json_idx, len(sweep_list), json_file)): + inject_item(db, t, proposer, dataset, generator) + else: + for job in tqdm.tqdm(data, + desc='[{}/{}] Processing {}'.format(json_idx, len(sweep_list), json_file)): + inject_item(db, job, proposer, dataset, generator) + + +if __name__ == '__main__': + main() diff --git a/new_impl/cv/third_party/nni_new/nas/benchmarks/nds/model.py b/new_impl/cv/third_party/nni_new/nas/benchmarks/nds/model.py new file mode 100644 index 0000000000000000000000000000000000000000..a6ace351d7fbe3aa8f043e8d7e56447807a3a184 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/nas/benchmarks/nds/model.py @@ -0,0 +1,143 @@ +import os + +from peewee import CharField, FloatField, ForeignKeyField, IntegerField, Model +from playhouse.sqlite_ext import JSONField, SqliteExtDatabase + +from nni.nas.benchmarks.constants import DATABASE_DIR +from nni.nas.benchmarks.utils import json_dumps + +db = SqliteExtDatabase(os.path.join(DATABASE_DIR, 'nds.db'), autoconnect=True) + + +class NdsTrialConfig(Model): + """ + Trial config for NDS. + + Attributes + ---------- + model_family : str + Could be ``nas_cell``, ``residual_bottleneck``, ``residual_basic`` or ``vanilla``. + model_spec : dict + If ``model_family`` is ``nas_cell``, it contains ``num_nodes_normal``, ``num_nodes_reduce``, ``depth``, + ``width``, ``aux`` and ``drop_prob``. If ``model_family`` is ``residual_bottleneck``, it contains ``bot_muls``, + ``ds`` (depths), ``num_gs`` (number of groups) and ``ss`` (strides). If ``model_family`` is ``residual_basic`` or + ``vanilla``, it contains ``ds``, ``ss`` and ``ws``. + cell_spec : dict + If ``model_family`` is not ``nas_cell`` it will be an empty dict. Otherwise, it specifies + ``___``, where i ranges from 0 to ``num_nodes_ - 1``. + If it is an ``op``, the value is chosen from the constants specified previously like :const:`nni.nas.benchmark.nds.CONV_1X1`. + If it is i's ``input``, the value range from 0 to ``i + 1``, as ``nas_cell`` uses previous two nodes as inputs, and + node 0 is actually the second node. Refer to NASNet paper for details. Finally, another two key-value pairs + ``normal_concat`` and ``reduce_concat`` specify which nodes are eventually concatenated into output. + dataset : str + Dataset used. Could be ``cifar10`` or ``imagenet``. + generator : str + Can be one of ``random`` which generates configurations at random, while keeping learning rate and weight decay fixed, + ``fix_w_d`` which further keeps ``width`` and ``depth`` fixed, only applicable for ``nas_cell``. ``tune_lr_wd`` which + further tunes learning rate and weight decay. + proposer : str + Paper who has proposed the distribution for random sampling. Available proposers include ``nasnet``, ``darts``, ``enas``, + ``pnas``, ``amoeba``, ``vanilla``, ``resnext-a``, ``resnext-b``, ``resnet``, ``resnet-b`` (ResNet with bottleneck). + See NDS paper for details. + base_lr : float + Initial learning rate. + weight_decay : float + L2 weight decay applied on weights. + num_epochs : int + Number of epochs scheduled, during which learning rate will decay to 0 following cosine annealing. + """ + + model_family = CharField(max_length=20, index=True, choices=[ + 'nas_cell', + 'residual_bottleneck', + 'residual_basic', + 'vanilla', + ]) + model_spec = JSONField(json_dumps=json_dumps, index=True) + cell_spec = JSONField(json_dumps=json_dumps, index=True, null=True) + dataset = CharField(max_length=15, index=True, choices=['cifar10', 'imagenet']) + generator = CharField(max_length=15, index=True, choices=[ + 'random', + 'fix_w_d', + 'tune_lr_wd', + ]) + proposer = CharField(max_length=15, index=True) + base_lr = FloatField() + weight_decay = FloatField() + num_epochs = IntegerField() + + class Meta: + database = db + + +class NdsTrialStats(Model): + """ + Computation statistics for NDS. Each corresponds to one trial. + + Attributes + ---------- + config : NdsTrialConfig + Corresponding config for trial. + seed : int + Random seed selected, for reproduction. + final_train_acc : float + Final accuracy on training data, ranging from 0 to 100. + final_train_loss : float or None + Final cross entropy loss on training data. Could be NaN (None). + final_test_acc : float + Final accuracy on test data, ranging from 0 to 100. + best_train_acc : float + Best accuracy on training data, ranging from 0 to 100. + best_train_loss : float or None + Best cross entropy loss on training data. Could be NaN (None). + best_test_acc : float + Best accuracy on test data, ranging from 0 to 100. + parameters : float + Number of trainable parameters in million. + flops : float + FLOPs in million. + iter_time : float + Seconds elapsed for each iteration. + """ + config = ForeignKeyField(NdsTrialConfig, backref='trial_stats', index=True) + seed = IntegerField() + final_train_acc = FloatField() + final_train_loss = FloatField(null=True) + final_test_acc = FloatField() + best_train_acc = FloatField() + best_train_loss = FloatField(null=True) + best_test_acc = FloatField() + parameters = FloatField() + flops = FloatField() + iter_time = FloatField() + + class Meta: + database = db + + +class NdsIntermediateStats(Model): + """ + Intermediate statistics for NDS. + + Attributes + ---------- + trial : NdsTrialStats + Corresponding trial. + current_epoch : int + Elapsed epochs. + train_loss : float or None + Current cross entropy loss on training data. Can be NaN (None). + train_acc : float + Current accuracy on training data, ranging from 0 to 100. + test_acc : float + Current accuracy on test data, ranging from 0 to 100. + """ + + trial = ForeignKeyField(NdsTrialStats, backref='intermediates', index=True) + current_epoch = IntegerField(index=True) + train_loss = FloatField(null=True) + train_acc = FloatField() + test_acc = FloatField() + + class Meta: + database = db diff --git a/new_impl/cv/third_party/nni_new/nas/benchmarks/nds/query.py b/new_impl/cv/third_party/nni_new/nas/benchmarks/nds/query.py new file mode 100644 index 0000000000000000000000000000000000000000..fe192ba5090f789df2febb4d237475a75f74e445 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/nas/benchmarks/nds/query.py @@ -0,0 +1,74 @@ +import functools + +from peewee import fn +from playhouse.shortcuts import model_to_dict +from .model import NdsTrialStats, NdsTrialConfig + + +def query_nds_trial_stats(model_family, proposer, generator, model_spec, cell_spec, dataset, + num_epochs=None, reduction=None, include_intermediates=False): + """ + Query trial stats of NDS given conditions. + + Parameters + ---------- + model_family : str or None + If str, can be one of the model families available in :class:`nni.nas.benchmark.nds.NdsTrialConfig`. + Otherwise a wildcard. + proposer : str or None + If str, can be one of the proposers available in :class:`nni.nas.benchmark.nds.NdsTrialConfig`. Otherwise a wildcard. + generator : str or None + If str, can be one of the generators available in :class:`nni.nas.benchmark.nds.NdsTrialConfig`. Otherwise a wildcard. + model_spec : dict or None + If specified, can be one of the model spec available in :class:`nni.nas.benchmark.nds.NdsTrialConfig`. + Otherwise a wildcard. + cell_spec : dict or None + If specified, can be one of the cell spec available in :class:`nni.nas.benchmark.nds.NdsTrialConfig`. + Otherwise a wildcard. + dataset : str or None + If str, can be one of the datasets available in :class:`nni.nas.benchmark.nds.NdsTrialConfig`. Otherwise a wildcard. + num_epochs : float or None + If int, matching results will be returned. Otherwise a wildcard. + reduction : str or None + If 'none' or None, all trial stats will be returned directly. + If 'mean', fields in trial stats will be averaged given the same trial config. + include_intermediates : boolean + If true, intermediate results will be returned. + + Returns + ------- + generator of dict + A generator of :class:`nni.nas.benchmark.nds.NdsTrialStats` objects, + where each of them has been converted into a dict. + """ + fields = [] + if reduction == 'none': + reduction = None + if reduction == 'mean': + for field_name in NdsTrialStats._meta.sorted_field_names: + if field_name not in ['id', 'config', 'seed']: + fields.append(fn.AVG(getattr(NdsTrialStats, field_name)).alias(field_name)) + elif reduction is None: + fields.append(NdsTrialStats) + else: + raise ValueError('Unsupported reduction: \'%s\'' % reduction) + query = NdsTrialStats.select(*fields, NdsTrialConfig).join(NdsTrialConfig) + conditions = [] + for field_name in ['model_family', 'proposer', 'generator', 'model_spec', 'cell_spec', + 'dataset', 'num_epochs']: + if locals()[field_name] is not None: + conditions.append(getattr(NdsTrialConfig, field_name) == locals()[field_name]) + if conditions: + query = query.where(functools.reduce(lambda a, b: a & b, conditions)) + if reduction is not None: + query = query.group_by(NdsTrialStats.config) + for trial in query: + if include_intermediates: + data = model_to_dict(trial) + # exclude 'trial' from intermediates as it is already available in data + data['intermediates'] = [ + {k: v for k, v in model_to_dict(t).items() if k != 'trial'} for t in trial.intermediates + ] + yield data + else: + yield model_to_dict(trial) diff --git a/new_impl/cv/third_party/nni_new/nas/benchmarks/nlp/__init__.py b/new_impl/cv/third_party/nni_new/nas/benchmarks/nlp/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9dc929727c4293a778dd5d86b7cf294ed5015ffc --- /dev/null +++ b/new_impl/cv/third_party/nni_new/nas/benchmarks/nlp/__init__.py @@ -0,0 +1,4 @@ +from .model import NlpTrialStats, NlpIntermediateStats, NlpTrialConfig +from .query import query_nlp_trial_stats + + diff --git a/new_impl/cv/third_party/nni_new/nas/benchmarks/nlp/db_gen.py b/new_impl/cv/third_party/nni_new/nas/benchmarks/nlp/db_gen.py new file mode 100644 index 0000000000000000000000000000000000000000..4c1198051438601a53732df9922f6d62993acd32 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/nas/benchmarks/nlp/db_gen.py @@ -0,0 +1,46 @@ +import json +import os +import argparse +import tqdm + +from .model import db, NlpTrialConfig, NlpTrialStats, NlpIntermediateStats + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('input_dir', help='Path to extracted NLP data dir.') + args = parser.parse_args() + with db, tqdm.tqdm(total=len(os.listdir(args.input_dir)), desc="creating tables") as pbar: + db.create_tables([NlpTrialConfig, NlpTrialStats, NlpIntermediateStats]) + json_files = os.listdir(args.input_dir) + for json_file in json_files: + pbar.update(1) + if json_file.endswith('.json'): + log_path = os.path.join(args.input_dir, json_file) + cur = json.load(open(log_path, 'r')) + arch = json.loads(cur['recepie']) + unested_arch = {} + for k in arch.keys(): + # print(k) + unested_arch['{}_op'.format(k)] = arch[k]['op'] + for i in range(len(arch[k]['input'])): + unested_arch['{}_input_{}'.format(k, i)] = arch[k]['input'][i] + config = NlpTrialConfig.create(arch=unested_arch, dataset=cur['data'][5:]) + if cur['status'] == 'OK': + trial_stats = NlpTrialStats.create(config=config, train_loss=cur['train_losses'][-1], val_loss=cur['val_losses'][-1], + test_loss=cur['test_losses'][-1], training_time=cur['wall_times'][-1]) + epochs = 50 + intermediate_stats = [] + for epoch in range(epochs): + epoch_res = { + 'train_loss' : cur['train_losses'][epoch], + 'val_loss' : cur['val_losses'][epoch], + 'test_loss' : cur['test_losses'][epoch], + 'training_time' : cur['wall_times'][epoch] + } + epoch_res.update(current_epoch=epoch + 1, trial=trial_stats) + intermediate_stats.append(epoch_res) + NlpIntermediateStats.insert_many(intermediate_stats).execute(db) + + +if __name__ == '__main__': + main() diff --git a/new_impl/cv/third_party/nni_new/nas/benchmarks/nlp/model.py b/new_impl/cv/third_party/nni_new/nas/benchmarks/nlp/model.py new file mode 100644 index 0000000000000000000000000000000000000000..d83ab7ff237af5fce0c9b1de322f77ff4cc021f1 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/nas/benchmarks/nlp/model.py @@ -0,0 +1,92 @@ +import os + +from peewee import CharField, FloatField, ForeignKeyField, IntegerField, Model +from playhouse.sqlite_ext import JSONField, SqliteExtDatabase + +from nni.nas.benchmarks.utils import json_dumps +from nni.nas.benchmarks.constants import DATABASE_DIR + +db = SqliteExtDatabase(os.path.join(DATABASE_DIR, 'nlp.db'), autoconnect=True) + +class NlpTrialConfig(Model): + """ + Trial config for NLP. epoch_num is fixed at 50. + + Attributes + ---------- + arch: dict + aka recepie in NAS-NLP-Benchmark repo (https://github.com/fmsnew/nas-bench-nlp-release). + an arch has multiple Node, Node_input_n and Node_op. + ``Node`` can be ``node_n`` or ``h_new_n`` or ``f/i/o/j(_act)`` etc. (n is an int number and need not to be consecutive) + ``Node_input_n`` can be ``Node`` or ``x`` etc. + ``Node_op`` can be ``linear`` or ``activation_sigm`` or ``activation_tanh`` or ``elementwise_prod`` + or ``elementwise_sum`` or ``activation_leaky_relu`` ... + e.g., {"h_new_0_input_0":"node_3","h_new_0_input_1":"x","h_new_0_op":"linear","node_2_input_0":"x", + "node_2_input_1":"h_prev_0","node_2_op":"linear","node_3_input_0":"node_2","node_3_op":"activation_leaky_relu"} + dataset: str + Dataset used. Could be ``ptb`` or ``wikitext-2``. + """ + arch = JSONField(json_dumps=json_dumps, index=True) + dataset = CharField(max_length=15, index=True, choices=[ + 'ptb', + 'wikitext-2' + ]) + + class Meta: + database = db + +class NlpTrialStats(Model): + """ + Computation statistics for NAS-NLP-Benchmark. + Each corresponds to one trial result after 50 epoch. + + Attributes + ---------- + config : NlpTrialConfig + Corresponding config for trial. + train_loss : float or None + Final loss on training data. Could be NaN (None). + val_loss : float or None + Final loss on validation data. Could be NaN (None). + test_loss : float or None + Final loss on test data. Could be NaN (None). + training_time : float + Time elapsed in seconds. aka wall_time in in NAS-NLP-Benchmark repo. + """ + config = ForeignKeyField(NlpTrialConfig, backref='trial_stats', index=True) + train_loss = FloatField(null=True) + val_loss = FloatField(null=True) + test_loss = FloatField(null=True) + training_time = FloatField(null=True) + + class Meta: + database = db + +class NlpIntermediateStats(Model): + """ + Computation statistics for NAS-NLP-Benchmark. + Each corresponds to one trial result for 1-50 epoch. + + Attributes + ---------- + config : NlpTrialConfig + Corresponding config for trial. + train_loss : float or None + Final loss on training data. Could be NaN (None). + val_loss : float or None + Final loss on validation data. Could be NaN (None). + test_loss : float or None + Final loss on test data. Could be NaN (None). + training_time : float + Time elapsed in seconds. aka wall_time in in NAS-NLP-Benchmark repo. + """ + trial = ForeignKeyField(NlpTrialStats, backref='intermediates', index=True) + current_epoch = IntegerField(index=True) + train_loss = FloatField(null=True) + val_loss = FloatField(null=True) + test_loss = FloatField(null=True) + training_time = FloatField(null=True) + + class Meta: + database = db + \ No newline at end of file diff --git a/new_impl/cv/third_party/nni_new/nas/benchmarks/nlp/query.py b/new_impl/cv/third_party/nni_new/nas/benchmarks/nlp/query.py new file mode 100644 index 0000000000000000000000000000000000000000..98885896b4d719ac07532a90a317f77a96d0ddc6 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/nas/benchmarks/nlp/query.py @@ -0,0 +1,61 @@ +import functools + +from peewee import fn +from playhouse.shortcuts import model_to_dict +from .model import NlpTrialStats, NlpTrialConfig + +def query_nlp_trial_stats(arch, dataset, reduction=None, include_intermediates=False): + """ + Query trial stats of NLP benchmark given conditions, including config(arch + dataset) and training results after 50 epoch. + + Parameters + ---------- + arch : dict or None + If a dict, it is in the format that is described in + :class:`nni.nas.benchmark.nlp.NlpTrialConfig`. Only trial stats matched will be returned. + If none, all architectures in the database will be matched. + dataset : str or None + If specified, can be one of the dataset available in :class:`nni.nas.benchmark.nlp.NlpTrialConfig`. + Otherwise a wildcard. + reduction : str or None + If 'none' or None, all trial stats will be returned directly. + If 'mean', fields in trial stats will be averaged given the same trial config. + Please note that some trial configs have multiple runs which make "reduction" meaningful, while some may not. + include_intermediates : boolean + If true, intermediate results will be returned. + + Returns + ------- + generator of dict + A generator of :class:`nni.nas.benchmark.nlp.NlpTrialStats` objects, + where each of them has been converted into a dict. + """ + fields = [] + if reduction == 'none': + reduction = None + if reduction == 'mean': + for field_name in NlpTrialStats._meta.sorted_field_names: + if field_name not in ['id', 'config']: + fields.append(fn.AVG(getattr(NlpTrialStats, field_name)).alias(field_name)) + elif reduction is None: + fields.append(NlpTrialStats) + else: + raise ValueError('Unsupported reduction: \'%s\'' % reduction) + query = NlpTrialStats.select(*fields, NlpTrialConfig).join(NlpTrialConfig) + + conditions = [] + if arch is not None: + conditions.append(NlpTrialConfig.arch == arch) + if dataset is not None: + conditions.append(NlpTrialConfig.dataset == dataset) + + for trial in query.where(functools.reduce(lambda a, b: a & b, conditions)): + if include_intermediates: + data = model_to_dict(trial) + # exclude 'trial' from intermediates as it is already available in data + data['intermediates'] = [ + {k: v for k, v in model_to_dict(t).items() if k != 'trial'} for t in trial.intermediates + ] + yield data + else: + yield model_to_dict(trial) \ No newline at end of file diff --git a/new_impl/cv/third_party/nni_new/nas/benchmarks/utils.py b/new_impl/cv/third_party/nni_new/nas/benchmarks/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7189d52f5d8619bb4f1d1fa4ba7f3a2ede33a6a0 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/nas/benchmarks/utils.py @@ -0,0 +1,5 @@ +import functools +import json + + +json_dumps = functools.partial(json.dumps, sort_keys=True) diff --git a/new_impl/cv/third_party/nni_new/nas/pytorch/__init__.py b/new_impl/cv/third_party/nni_new/nas/pytorch/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..4a61f1672d859edda94ed7194e1bc313d824a39b --- /dev/null +++ b/new_impl/cv/third_party/nni_new/nas/pytorch/__init__.py @@ -0,0 +1,6 @@ +from .base_mutator import BaseMutator +from .base_trainer import BaseTrainer +from .fixed import apply_fixed_architecture +from .mutables import Mutable, LayerChoice, InputChoice +from .mutator import Mutator +from .trainer import Trainer diff --git a/new_impl/cv/third_party/nni_new/nas/pytorch/base_mutator.py b/new_impl/cv/third_party/nni_new/nas/pytorch/base_mutator.py new file mode 100644 index 0000000000000000000000000000000000000000..df1a5f9ba8719a5c6619cd4ccc2874c0017a44db --- /dev/null +++ b/new_impl/cv/third_party/nni_new/nas/pytorch/base_mutator.py @@ -0,0 +1,155 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging + +import torch.nn as nn +from nni.nas.pytorch.mutables import Mutable, MutableScope, InputChoice +from nni.nas.pytorch.utils import StructuredMutableTreeNode + +logger = logging.getLogger(__name__) + + +class BaseMutator(nn.Module): + """ + A mutator is responsible for mutating a graph by obtaining the search space from the network and implementing + callbacks that are called in ``forward`` in mutables. + + Parameters + ---------- + model : nn.Module + PyTorch model to apply mutator on. + """ + + def __init__(self, model): + super().__init__() + self.__dict__["model"] = model + self._structured_mutables = self._parse_search_space(self.model) + + def _parse_search_space(self, module, root=None, prefix="", memo=None, nested_detection=None): + if memo is None: + memo = set() + if root is None: + root = StructuredMutableTreeNode(None) + if module not in memo: + memo.add(module) + if isinstance(module, Mutable): + if nested_detection is not None: + raise RuntimeError("Cannot have nested search space. Error at {} in {}" + .format(module, nested_detection)) + module.name = prefix + module.set_mutator(self) + root = root.add_child(module) + if not isinstance(module, MutableScope): + nested_detection = module + if isinstance(module, InputChoice): + for k in module.choose_from: + if k != InputChoice.NO_KEY and k not in [m.key for m in memo if isinstance(m, Mutable)]: + raise RuntimeError("'{}' required by '{}' not found in keys that appeared before, and is not NO_KEY." + .format(k, module.key)) + for name, submodule in module._modules.items(): + if submodule is None: + continue + submodule_prefix = prefix + ("." if prefix else "") + name + self._parse_search_space(submodule, root, submodule_prefix, memo=memo, + nested_detection=nested_detection) + return root + + @property + def mutables(self): + """ + A generator of all modules inheriting :class:`~nni.nas.pytorch.mutables.Mutable`. + Modules are yielded in the order that they are defined in ``__init__``. + For mutables with their keys appearing multiple times, only the first one will appear. + """ + return self._structured_mutables + + @property + def undedup_mutables(self): + return self._structured_mutables.traverse(deduplicate=False) + + def forward(self, *inputs): + """ + Warnings + -------- + Don't call forward of a mutator. + """ + raise RuntimeError("Forward is undefined for mutators.") + + def __setattr__(self, name, value): + if name == "model": + raise AttributeError("Attribute `model` can be set at most once, and you shouldn't use `self.model = model` to " + "include you network, as it will include all parameters in model into the mutator.") + return super().__setattr__(name, value) + + def enter_mutable_scope(self, mutable_scope): + """ + Callback when forward of a MutableScope is entered. + + Parameters + ---------- + mutable_scope : MutableScope + The mutable scope that is entered. + """ + pass + + def exit_mutable_scope(self, mutable_scope): + """ + Callback when forward of a MutableScope is exited. + + Parameters + ---------- + mutable_scope : MutableScope + The mutable scope that is exited. + """ + pass + + def on_forward_layer_choice(self, mutable, *args, **kwargs): + """ + Callbacks of forward in LayerChoice. + + Parameters + ---------- + mutable : nni.nas.pytorch.mutables.LayerChoice + Module whose forward is called. + args : list of torch.Tensor + The arguments of its forward function. + kwargs : dict + The keyword arguments of its forward function. + + Returns + ------- + tuple of torch.Tensor and torch.Tensor + Output tensor and mask. + """ + raise NotImplementedError + + def on_forward_input_choice(self, mutable, tensor_list): + """ + Callbacks of forward in InputChoice. + + Parameters + ---------- + mutable : nni.nas.pytorch.mutables.InputChoice + Mutable that is called. + tensor_list : list of torch.Tensor + The arguments mutable is called with. + + Returns + ------- + tuple of torch.Tensor and torch.Tensor + Output tensor and mask. + """ + raise NotImplementedError + + def export(self): + """ + Export the data of all decisions. This should output the decisions of all the mutables, so that the whole + network can be fully determined with these decisions for further training from scratch. + + Returns + ------- + dict + Mappings from mutable keys to decisions. + """ + raise NotImplementedError diff --git a/new_impl/cv/third_party/nni_new/nas/pytorch/base_trainer.py b/new_impl/cv/third_party/nni_new/nas/pytorch/base_trainer.py new file mode 100644 index 0000000000000000000000000000000000000000..2e7a4a2a23a24cd2886eac12dda80eedfa6076d8 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/nas/pytorch/base_trainer.py @@ -0,0 +1,40 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from abc import ABC, abstractmethod + + +class BaseTrainer(ABC): + + @abstractmethod + def train(self): + """ + Override the method to train. + """ + raise NotImplementedError + + @abstractmethod + def validate(self): + """ + Override the method to validate. + """ + raise NotImplementedError + + @abstractmethod + def export(self, file): + """ + Override the method to export to file. + + Parameters + ---------- + file : str + File path to export to. + """ + raise NotImplementedError + + @abstractmethod + def checkpoint(self): + """ + Override to dump a checkpoint. + """ + raise NotImplementedError diff --git a/new_impl/cv/third_party/nni_new/nas/pytorch/callbacks.py b/new_impl/cv/third_party/nni_new/nas/pytorch/callbacks.py new file mode 100644 index 0000000000000000000000000000000000000000..86a0dc3800745d93939b9ce85dd9cdca3abef166 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/nas/pytorch/callbacks.py @@ -0,0 +1,139 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +import os + +import torch +import torch.nn as nn + +_logger = logging.getLogger(__name__) + + +class Callback: + """ + Callback provides an easy way to react to events like begin/end of epochs. + """ + + def __init__(self): + self.model = None + self.mutator = None + self.trainer = None + + def build(self, model, mutator, trainer): + """ + Callback needs to be built with model, mutator, trainer, to get updates from them. + + Parameters + ---------- + model : nn.Module + Model to be trained. + mutator : nn.Module + Mutator that mutates the model. + trainer : BaseTrainer + Trainer that is to call the callback. + """ + self.model = model + self.mutator = mutator + self.trainer = trainer + + def on_epoch_begin(self, epoch): + """ + Implement this to do something at the begin of epoch. + + Parameters + ---------- + epoch : int + Epoch number, starting from 0. + """ + pass + + def on_epoch_end(self, epoch): + """ + Implement this to do something at the end of epoch. + + Parameters + ---------- + epoch : int + Epoch number, starting from 0. + """ + pass + + def on_batch_begin(self, epoch): + pass + + def on_batch_end(self, epoch): + pass + + +class LRSchedulerCallback(Callback): + """ + Calls scheduler on every epoch ends. + + Parameters + ---------- + scheduler : LRScheduler + Scheduler to be called. + """ + def __init__(self, scheduler, mode="epoch"): + super().__init__() + assert mode == "epoch" + self.scheduler = scheduler + self.mode = mode + + def on_epoch_end(self, epoch): + """ + Call ``self.scheduler.step()`` on epoch end. + """ + self.scheduler.step() + + +class ArchitectureCheckpoint(Callback): + """ + Calls ``trainer.export()`` on every epoch ends. + + Parameters + ---------- + checkpoint_dir : str + Location to save checkpoints. + """ + def __init__(self, checkpoint_dir): + super().__init__() + self.checkpoint_dir = checkpoint_dir + os.makedirs(self.checkpoint_dir, exist_ok=True) + + def on_epoch_end(self, epoch): + """ + Dump to ``/checkpoint_dir/epoch_{number}.json`` on epoch end. + """ + dest_path = os.path.join(self.checkpoint_dir, "epoch_{}.json".format(epoch)) + _logger.info("Saving architecture to %s", dest_path) + self.trainer.export(dest_path) + + +class ModelCheckpoint(Callback): + """ + Calls ``trainer.export()`` on every epoch ends. + + Parameters + ---------- + checkpoint_dir : str + Location to save checkpoints. + """ + def __init__(self, checkpoint_dir): + super().__init__() + self.checkpoint_dir = checkpoint_dir + os.makedirs(self.checkpoint_dir, exist_ok=True) + + def on_epoch_end(self, epoch): + """ + Dump to ``/checkpoint_dir/epoch_{number}.pth.tar`` on every epoch end. + ``DataParallel`` object will have their inside modules exported. + """ + if isinstance(self.model, nn.DataParallel): + state_dict = self.model.module.state_dict() + else: + state_dict = self.model.state_dict() + dest_path = os.path.join(self.checkpoint_dir, "epoch_{}.pth.tar".format(epoch)) + _logger.info("Saving model to %s", dest_path) + torch.save(state_dict, dest_path) diff --git a/new_impl/cv/third_party/nni_new/nas/pytorch/fixed.py b/new_impl/cv/third_party/nni_new/nas/pytorch/fixed.py new file mode 100644 index 0000000000000000000000000000000000000000..9bfa933e80e740ef5ff8832b9d45484bbdc07771 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/nas/pytorch/fixed.py @@ -0,0 +1,147 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import json +import logging + +from .mutables import InputChoice, LayerChoice, MutableScope +from .mutator import Mutator +from .utils import to_list + + +_logger = logging.getLogger(__name__) + + +class FixedArchitecture(Mutator): + """ + Fixed architecture mutator that always selects a certain graph. + + Parameters + ---------- + model : nn.Module + A mutable network. + fixed_arc : dict + Preloaded architecture object. + strict : bool + Force everything that appears in ``fixed_arc`` to be used at least once. + verbose : bool + Print log messages if set to True + """ + + def __init__(self, model, fixed_arc, strict=True, verbose=True): + super().__init__(model) + self._fixed_arc = fixed_arc + self.verbose = verbose + + mutable_keys = set([mutable.key for mutable in self.mutables if not isinstance(mutable, MutableScope)]) + fixed_arc_keys = set(self._fixed_arc.keys()) + if fixed_arc_keys - mutable_keys: + raise RuntimeError("Unexpected keys found in fixed architecture: {}.".format(fixed_arc_keys - mutable_keys)) + if mutable_keys - fixed_arc_keys: + raise RuntimeError("Missing keys in fixed architecture: {}.".format(mutable_keys - fixed_arc_keys)) + self._fixed_arc = self._from_human_readable_architecture(self._fixed_arc) + + def _from_human_readable_architecture(self, human_arc): + # convert from an exported architecture + result_arc = {k: to_list(v) for k, v in human_arc.items()} # there could be tensors, numpy arrays, etc. + # First, convert non-list to list, because there could be {"op1": 0} or {"op1": "conv"}, + # which means {"op1": [0, ]} ir {"op1": ["conv", ]} + result_arc = {k: v if isinstance(v, list) else [v] for k, v in result_arc.items()} + # Second, infer which ones are multi-hot arrays and which ones are in human-readable format. + # This is non-trivial, since if an array in [0, 1], we cannot know for sure it means [false, true] or [true, true]. + # Here, we assume an multihot array has to be a boolean array or a float array and matches the length. + for mutable in self.mutables: + if mutable.key not in result_arc: + continue # skip silently + choice_arr = result_arc[mutable.key] + if all(isinstance(v, bool) for v in choice_arr) or all(isinstance(v, float) for v in choice_arr): + if (isinstance(mutable, LayerChoice) and len(mutable) == len(choice_arr)) or \ + (isinstance(mutable, InputChoice) and mutable.n_candidates == len(choice_arr)): + # multihot, do nothing + continue + if isinstance(mutable, LayerChoice): + choice_arr = [mutable.names.index(val) if isinstance(val, str) else val for val in choice_arr] + choice_arr = [i in choice_arr for i in range(len(mutable))] + elif isinstance(mutable, InputChoice): + choice_arr = [mutable.choose_from.index(val) if isinstance(val, str) else val for val in choice_arr] + choice_arr = [i in choice_arr for i in range(mutable.n_candidates)] + result_arc[mutable.key] = choice_arr + return result_arc + + def sample_search(self): + """ + Always returns the fixed architecture. + """ + return self._fixed_arc + + def sample_final(self): + """ + Always returns the fixed architecture. + """ + return self._fixed_arc + + def replace_layer_choice(self, module=None, prefix=""): + """ + Replace layer choices with selected candidates. It's done with best effort. + In case of weighted choices or multiple choices. if some of the choices on weighted with zero, delete them. + If single choice, replace the module with a normal module. + + Parameters + ---------- + module : nn.Module + Module to be processed. + prefix : str + Module name under global namespace. + """ + if module is None: + module = self.model + for name, mutable in module.named_children(): + global_name = (prefix + "." if prefix else "") + name + if isinstance(mutable, LayerChoice): + chosen = self._fixed_arc[mutable.key] + if sum(chosen) == 1 and max(chosen) == 1 and not mutable.return_mask: + # sum is one, max is one, there has to be an only one + # this is compatible with both integer arrays, boolean arrays and float arrays + if self.verbose: + _logger.info("Replacing %s with candidate number %d.", global_name, chosen.index(1)) + setattr(module, name, mutable[chosen.index(1)]) + else: + if mutable.return_mask and self.verbose: + _logger.info("`return_mask` flag of %s is true. As it relies on the behavior of LayerChoice, " \ + "LayerChoice will not be replaced.") + # remove unused parameters + for ch, n in zip(chosen, mutable.names): + if ch == 0 and not isinstance(ch, float): + setattr(mutable, n, None) + else: + self.replace_layer_choice(mutable, global_name) + + +def apply_fixed_architecture(model, fixed_arc, verbose=True): + """ + Load architecture from `fixed_arc` and apply to model. + + Parameters + ---------- + model : torch.nn.Module + Model with mutables. + fixed_arc : str or dict + Path to the JSON that stores the architecture, or dict that stores the exported architecture. + verbose : bool + Print log messages if set to True + + Returns + ------- + FixedArchitecture + Mutator that is responsible for fixes the graph. + """ + + if isinstance(fixed_arc, str): + with open(fixed_arc) as f: + fixed_arc = json.load(f) + architecture = FixedArchitecture(model, fixed_arc, verbose) + architecture.reset() + + # for the convenience of parameters counting + architecture.replace_layer_choice() + return architecture diff --git a/new_impl/cv/third_party/nni_new/nas/pytorch/mutables.py b/new_impl/cv/third_party/nni_new/nas/pytorch/mutables.py new file mode 100644 index 0000000000000000000000000000000000000000..7fbb655e51e25c0d639746657bec5d4709e50c44 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/nas/pytorch/mutables.py @@ -0,0 +1,344 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +import warnings +from collections import OrderedDict + +import torch.nn as nn + +from nni.nas.pytorch.utils import global_mutable_counting + +logger = logging.getLogger(__name__) + + +class Mutable(nn.Module): + """ + Mutable is designed to function as a normal layer, with all necessary operators' weights. + States and weights of architectures should be included in mutator, instead of the layer itself. + + Mutable has a key, which marks the identity of the mutable. This key can be used by users to share + decisions among different mutables. In mutator's implementation, mutators should use the key to + distinguish different mutables. Mutables that share the same key should be "similar" to each other. + + Currently the default scope for keys is global. By default, the keys uses a global counter from 1 to + produce unique ids. + + Parameters + ---------- + key : str + The key of mutable. + + Notes + ----- + The counter is program level, but mutables are model level. In case multiple models are defined, and + you want to have `counter` starting from 1 in the second model, it's recommended to assign keys manually + instead of using automatic keys. + """ + + def __init__(self, key=None): + super().__init__() + if key is not None: + if not isinstance(key, str): + key = str(key) + logger.warning("Warning: key \"%s\" is not string, converted to string.", key) + self._key = key + else: + self._key = self.__class__.__name__ + str(global_mutable_counting()) + self.init_hook = self.forward_hook = None + + def __deepcopy__(self, memodict=None): + raise NotImplementedError("Deep copy doesn't work for mutables.") + + def __call__(self, *args, **kwargs): + self._check_built() + return super().__call__(*args, **kwargs) + + def set_mutator(self, mutator): + if "mutator" in self.__dict__: + raise RuntimeError("`set_mutator` is called more than once. Did you parse the search space multiple times? " + "Or did you apply multiple fixed architectures?") + self.__dict__["mutator"] = mutator + + @property + def key(self): + """ + Read-only property of key. + """ + return self._key + + @property + def name(self): + """ + After the search space is parsed, it will be the module name of the mutable. + """ + return self._name if hasattr(self, "_name") else self._key + + @name.setter + def name(self, name): + self._name = name + + def _check_built(self): + if not hasattr(self, "mutator"): + raise ValueError( + "Mutator not set for {}. You might have forgotten to initialize and apply your mutator. " + "Or did you initialize a mutable on the fly in forward pass? Move to `__init__` " + "so that trainer can locate all your mutables. See NNI docs for more details.".format(self)) + + +class MutableScope(Mutable): + """ + Mutable scope marks a subgraph/submodule to help mutators make better decisions. + + If not annotated with mutable scope, search space will be flattened as a list. However, some mutators might + need to leverage the concept of a "cell". So if a module is defined as a mutable scope, everything in it will + look like "sub-search-space" in the scope. Scopes can be nested. + + There are two ways mutators can use mutable scope. One is to traverse the search space as a tree during initialization + and reset. The other is to implement `enter_mutable_scope` and `exit_mutable_scope`. They are called before and after + the forward method of the class inheriting mutable scope. + + Mutable scopes are also mutables that are listed in the mutator.mutables (search space), but they are not supposed + to appear in the dict of choices. + + Parameters + ---------- + key : str + Key of mutable scope. + """ + def __init__(self, key): + super().__init__(key=key) + + def _check_built(self): + return True # bypass the test because it's deprecated + + def __call__(self, *args, **kwargs): + if not hasattr(self, 'mutator'): + return super().__call__(*args, **kwargs) + warnings.warn("`MutableScope` is deprecated in Retiarii.", DeprecationWarning) + try: + self._check_built() + self.mutator.enter_mutable_scope(self) + return super().__call__(*args, **kwargs) + finally: + self.mutator.exit_mutable_scope(self) + + +class LayerChoice(Mutable): + """ + Layer choice selects one of the ``op_candidates``, then apply it on inputs and return results. + In rare cases, it can also select zero or many. + + Layer choice does not allow itself to be nested. + + Parameters + ---------- + op_candidates : list of nn.Module or OrderedDict + A module list to be selected from. + reduction : str + ``mean``, ``concat``, ``sum`` or ``none``. Policy if multiples are selected. + If ``none``, a list is returned. ``mean`` returns the average. ``sum`` returns the sum. + ``concat`` concatenate the list at dimension 1. + return_mask : bool + If ``return_mask``, return output tensor and a mask. Otherwise return tensor only. + key : str + Key of the input choice. + + Attributes + ---------- + length : int + Deprecated. Number of ops to choose from. ``len(layer_choice)`` is recommended. + names : list of str + Names of candidates. + choices : list of Module + Deprecated. A list of all candidate modules in the layer choice module. + ``list(layer_choice)`` is recommended, which will serve the same purpose. + + Notes + ----- + ``op_candidates`` can be a list of modules or a ordered dict of named modules, for example, + + .. code-block:: python + + self.op_choice = LayerChoice(OrderedDict([ + ("conv3x3", nn.Conv2d(3, 16, 128)), + ("conv5x5", nn.Conv2d(5, 16, 128)), + ("conv7x7", nn.Conv2d(7, 16, 128)) + ])) + + Elements in layer choice can be modified or deleted. Use ``del self.op_choice["conv5x5"]`` or + ``self.op_choice[1] = nn.Conv3d(...)``. Adding more choices is not supported yet. + """ + + def __init__(self, op_candidates, reduction="sum", return_mask=False, key=None): + super().__init__(key=key) + self.names = [] + if isinstance(op_candidates, OrderedDict): + for name, module in op_candidates.items(): + assert name not in ["length", "reduction", "return_mask", "_key", "key", "names"], \ + "Please don't use a reserved name '{}' for your module.".format(name) + self.add_module(name, module) + self.names.append(name) + elif isinstance(op_candidates, list): + for i, module in enumerate(op_candidates): + self.add_module(str(i), module) + self.names.append(str(i)) + else: + raise TypeError("Unsupported op_candidates type: {}".format(type(op_candidates))) + self.reduction = reduction + self.return_mask = return_mask + + def __getitem__(self, idx): + if isinstance(idx, str): + return self._modules[idx] + return list(self)[idx] + + def __setitem__(self, idx, module): + key = idx if isinstance(idx, str) else self.names[idx] + return setattr(self, key, module) + + def __delitem__(self, idx): + if isinstance(idx, slice): + for key in self.names[idx]: + delattr(self, key) + else: + if isinstance(idx, str): + key, idx = idx, self.names.index(idx) + else: + key = self.names[idx] + delattr(self, key) + del self.names[idx] + + @property + def length(self): + warnings.warn("layer_choice.length is deprecated. Use `len(layer_choice)` instead.", DeprecationWarning) + return len(self) + + def __len__(self): + return len(self.names) + + def __iter__(self): + return map(lambda name: self._modules[name], self.names) + + @property + def choices(self): + warnings.warn("layer_choice.choices is deprecated. Use `list(layer_choice)` instead.", DeprecationWarning) + return list(self) + + def forward(self, *args, **kwargs): + """ + Returns + ------- + tuple of tensors + Output and selection mask. If ``return_mask`` is ``False``, only output is returned. + """ + out, mask = self.mutator.on_forward_layer_choice(self, *args, **kwargs) + if self.return_mask: + return out, mask + return out + + +class InputChoice(Mutable): + """ + Input choice selects ``n_chosen`` inputs from ``choose_from`` (contains ``n_candidates`` keys). For beginners, + use ``n_candidates`` instead of ``choose_from`` is a safe option. To get the most power out of it, you might want to + know about ``choose_from``. + + The keys in ``choose_from`` can be keys that appear in past mutables, or ``NO_KEY`` if there are no suitable ones. + The keys are designed to be the keys of the sources. To help mutators make better decisions, + mutators might be interested in how the tensors to choose from come into place. For example, the tensor is the + output of some operator, some node, some cell, or some module. If this operator happens to be a mutable (e.g., + ``LayerChoice`` or ``InputChoice``), it has a key naturally that can be used as a source key. If it's a + module/submodule, it needs to be annotated with a key: that's where a :class:`MutableScope` is needed. + + In the example below, ``input_choice`` is a 4-choose-any. The first 3 is semantically output of cell1, output of cell2, + output of cell3 with respectively. Notice that an extra max pooling is followed by cell1, indicating x1 is not + "actually" the direct output of cell1. + + .. code-block:: python + + class Cell(MutableScope): + pass + + class Net(nn.Module): + def __init__(self): + self.cell1 = Cell("cell1") + self.cell2 = Cell("cell2") + self.op = LayerChoice([conv3x3(), conv5x5()], key="op") + self.input_choice = InputChoice(choose_from=["cell1", "cell2", "op", InputChoice.NO_KEY]) + + def forward(self, x): + x1 = max_pooling(self.cell1(x)) + x2 = self.cell2(x) + x3 = self.op(x) + x4 = torch.zeros_like(x) + return self.input_choice([x1, x2, x3, x4]) + + Parameters + ---------- + n_candidates : int + Number of inputs to choose from. + choose_from : list of str + List of source keys to choose from. At least of one of ``choose_from`` and ``n_candidates`` must be fulfilled. + If ``n_candidates`` has a value but ``choose_from`` is None, it will be automatically treated as ``n_candidates`` + number of empty string. + n_chosen : int + Recommended inputs to choose. If None, mutator is instructed to select any. + reduction : str + ``mean``, ``concat``, ``sum`` or ``none``. See :class:`LayerChoice`. + return_mask : bool + If ``return_mask``, return output tensor and a mask. Otherwise return tensor only. + key : str + Key of the input choice. + """ + + NO_KEY = "" + + def __init__(self, n_candidates=None, choose_from=None, n_chosen=None, + reduction="sum", return_mask=False, key=None): + super().__init__(key=key) + # precondition check + assert n_candidates is not None or choose_from is not None, "At least one of `n_candidates` and `choose_from`" \ + "must be not None." + if choose_from is not None and n_candidates is None: + n_candidates = len(choose_from) + elif choose_from is None and n_candidates is not None: + choose_from = [self.NO_KEY] * n_candidates + assert n_candidates == len(choose_from), "Number of candidates must be equal to the length of `choose_from`." + assert n_candidates > 0, "Number of candidates must be greater than 0." + assert n_chosen is None or 0 <= n_chosen <= n_candidates, "Expected selected number must be None or no more " \ + "than number of candidates." + + self.n_candidates = n_candidates + self.choose_from = choose_from.copy() + self.n_chosen = n_chosen + self.reduction = reduction + self.return_mask = return_mask + + def forward(self, optional_inputs): + """ + Forward method of LayerChoice. + + Parameters + ---------- + optional_inputs : list or dict + Recommended to be a dict. As a dict, inputs will be converted to a list that follows the order of + ``choose_from`` in initialization. As a list, inputs must follow the semantic order that is the same as + ``choose_from``. + + Returns + ------- + tuple of tensors + Output and selection mask. If ``return_mask`` is ``False``, only output is returned. + """ + optional_input_list = optional_inputs + if isinstance(optional_inputs, dict): + optional_input_list = [optional_inputs[tag] for tag in self.choose_from] + assert isinstance(optional_input_list, list), \ + "Optional input list must be a list, not a {}.".format(type(optional_input_list)) + assert len(optional_inputs) == self.n_candidates, \ + "Length of the input list must be equal to number of candidates." + out, mask = self.mutator.on_forward_input_choice(self, optional_input_list) + if self.return_mask: + return out, mask + return out diff --git a/new_impl/cv/third_party/nni_new/nas/pytorch/mutator.py b/new_impl/cv/third_party/nni_new/nas/pytorch/mutator.py new file mode 100644 index 0000000000000000000000000000000000000000..e1894b52496ae63611178ccdf301cd4f612abc9e --- /dev/null +++ b/new_impl/cv/third_party/nni_new/nas/pytorch/mutator.py @@ -0,0 +1,308 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +from collections import defaultdict + +import numpy as np +import torch + +from .base_mutator import BaseMutator +from .mutables import LayerChoice, InputChoice +from .utils import to_list + +logger = logging.getLogger(__name__) + + +class Mutator(BaseMutator): + + def __init__(self, model): + super().__init__(model) + self._cache = dict() + self._connect_all = False + + def sample_search(self): + """ + Override to implement this method to iterate over mutables and make decisions. + + Returns + ------- + dict + A mapping from key of mutables to decisions. + """ + raise NotImplementedError + + def sample_final(self): + """ + Override to implement this method to iterate over mutables and make decisions that is final + for export and retraining. + + Returns + ------- + dict + A mapping from key of mutables to decisions. + """ + raise NotImplementedError + + def reset(self): + """ + Reset the mutator by call the `sample_search` to resample (for search). Stores the result in a local + variable so that `on_forward_layer_choice` and `on_forward_input_choice` can use the decision directly. + """ + self._cache = self.sample_search() + + def export(self): + """ + Resample (for final) and return results. + + Returns + ------- + dict + A mapping from key of mutables to decisions. + """ + sampled = self.sample_final() + result = dict() + for mutable in self.mutables: + if not isinstance(mutable, (LayerChoice, InputChoice)): + # not supported as built-in + continue + result[mutable.key] = self._convert_mutable_decision_to_human_readable(mutable, sampled.pop(mutable.key)) + if sampled: + raise ValueError("Unexpected keys returned from 'sample_final()': %s", list(sampled.keys())) + return result + + def status(self): + """ + Return current selection status of mutator. + + Returns + ------- + dict + A mapping from key of mutables to decisions. All weights (boolean type and float type) + are converted into real number values. Numpy arrays and tensors are converted into list. + """ + data = dict() + for k, v in self._cache.items(): + if torch.is_tensor(v): + v = v.detach().cpu().numpy().tolist() + if isinstance(v, np.ndarray): + v = v.astype(np.float32).tolist() + data[k] = v + return data + + def graph(self, inputs): + """ + Return model supernet graph. + + Parameters + ---------- + inputs: tuple of tensor + Inputs that will be feeded into the network. + + Returns + ------- + dict + Containing ``node``, in Tensorboard GraphDef format. + Additional key ``mutable`` is a map from key to list of modules. + """ + if not torch.__version__.startswith("1.4"): + logger.warning("Graph is only tested with PyTorch 1.4. Other versions might not work.") + from nni.common.graph_utils import build_graph + from google.protobuf import json_format + # protobuf should be installed as long as tensorboard is installed + try: + self._connect_all = True + graph_def, _ = build_graph(self.model, inputs, verbose=False) + result = json_format.MessageToDict(graph_def) + finally: + self._connect_all = False + + # `mutable` is to map the keys to a list of corresponding modules. + # A key can be linked to multiple modules, use `dedup=False` to find them all. + result["mutable"] = defaultdict(list) + for mutable in self.mutables.traverse(deduplicate=False): + # A module will be represent in the format of + # [{"type": "Net", "name": ""}, {"type": "Cell", "name": "cell1"}, {"type": "Conv2d": "name": "conv"}] + # which will be concatenated into Net/Cell[cell1]/Conv2d[conv] in frontend. + # This format is aligned with the scope name jit gives. + modules = mutable.name.split(".") + path = [ + {"type": self.model.__class__.__name__, "name": ""} + ] + m = self.model + for module in modules: + m = getattr(m, module) + path.append({ + "type": m.__class__.__name__, + "name": module + }) + result["mutable"][mutable.key].append(path) + return result + + def on_forward_layer_choice(self, mutable, *args, **kwargs): + """ + On default, this method retrieves the decision obtained previously, and select certain operations. + Only operations with non-zero weight will be executed. The results will be added to a list. + Then it will reduce the list of all tensor outputs with the policy specified in `mutable.reduction`. + + Parameters + ---------- + mutable : nni.nas.pytorch.mutables.LayerChoice + Layer choice module. + args : list of torch.Tensor + Inputs + kwargs : dict + Inputs + + Returns + ------- + tuple of torch.Tensor and torch.Tensor + Output and mask. + """ + if self._connect_all: + return self._all_connect_tensor_reduction(mutable.reduction, + [op(*args, **kwargs) for op in mutable]), \ + torch.ones(len(mutable)).bool() + + def _map_fn(op, args, kwargs): + return op(*args, **kwargs) + + mask = self._get_decision(mutable) + assert len(mask) == len(mutable), \ + "Invalid mask, expected {} to be of length {}.".format(mask, len(mutable)) + out, mask = self._select_with_mask(_map_fn, [(choice, args, kwargs) for choice in mutable], mask) + return self._tensor_reduction(mutable.reduction, out), mask + + def on_forward_input_choice(self, mutable, tensor_list): + """ + On default, this method retrieves the decision obtained previously, and select certain tensors. + Then it will reduce the list of all tensor outputs with the policy specified in `mutable.reduction`. + + Parameters + ---------- + mutable : nni.nas.pytorch.mutables.InputChoice + Input choice module. + tensor_list : list of torch.Tensor + Tensor list to apply the decision on. + + Returns + ------- + tuple of torch.Tensor and torch.Tensor + Output and mask. + """ + if self._connect_all: + return self._all_connect_tensor_reduction(mutable.reduction, tensor_list), \ + torch.ones(mutable.n_candidates).bool() + mask = self._get_decision(mutable) + assert len(mask) == mutable.n_candidates, \ + "Invalid mask, expected {} to be of length {}.".format(mask, mutable.n_candidates) + out, mask = self._select_with_mask(lambda x: x, [(t,) for t in tensor_list], mask) + return self._tensor_reduction(mutable.reduction, out), mask + + def _select_with_mask(self, map_fn, candidates, mask): + """ + Select masked tensors and return a list of tensors. + + Parameters + ---------- + map_fn : function + Convert candidates to target candidates. Can be simply identity. + candidates : list of torch.Tensor + Tensor list to apply the decision on. + mask : list-like object + Can be a list, an numpy array or a tensor (recommended). Needs to + have the same length as ``candidates``. + + Returns + ------- + tuple of list of torch.Tensor and torch.Tensor + Output and mask. + """ + if (isinstance(mask, list) and len(mask) >= 1 and isinstance(mask[0], bool)) or \ + (isinstance(mask, np.ndarray) and mask.dtype == np.bool) or \ + "BoolTensor" in mask.type(): + out = [map_fn(*cand) for cand, m in zip(candidates, mask) if m] + elif (isinstance(mask, list) and len(mask) >= 1 and isinstance(mask[0], (float, int))) or \ + (isinstance(mask, np.ndarray) and mask.dtype in (np.float32, np.float64, np.int32, np.int64)) or \ + "FloatTensor" in mask.type(): + out = [map_fn(*cand) * m for cand, m in zip(candidates, mask) if m] + else: + raise ValueError("Unrecognized mask '%s'" % mask) + if not torch.is_tensor(mask): + mask = torch.tensor(mask) # pylint: disable=not-callable + return out, mask + + def _tensor_reduction(self, reduction_type, tensor_list): + if reduction_type == "none": + return tensor_list + if not tensor_list: + return None # empty. return None for now + if len(tensor_list) == 1: + return tensor_list[0] + if reduction_type == "sum": + return sum(tensor_list) + if reduction_type == "mean": + return sum(tensor_list) / len(tensor_list) + if reduction_type == "concat": + return torch.cat(tensor_list, dim=1) + raise ValueError("Unrecognized reduction policy: \"{}\"".format(reduction_type)) + + def _all_connect_tensor_reduction(self, reduction_type, tensor_list): + if reduction_type == "none": + return tensor_list + if reduction_type == "concat": + return torch.cat(tensor_list, dim=1) + return torch.stack(tensor_list).sum(0) + + def _get_decision(self, mutable): + """ + By default, this method checks whether `mutable.key` is already in the decision cache, + and returns the result without double-check. + + Parameters + ---------- + mutable : Mutable + + Returns + ------- + object + """ + if mutable.key not in self._cache: + raise ValueError("\"{}\" not found in decision cache.".format(mutable.key)) + result = self._cache[mutable.key] + logger.debug("Decision %s: %s", mutable.key, result) + return result + + def _convert_mutable_decision_to_human_readable(self, mutable, sampled): + # Assert the existence of mutable.key in returned architecture. + # Also check if there is anything extra. + multihot_list = to_list(sampled) + converted = None + # If it's a boolean array, we can do optimization. + if all([t == 0 or t == 1 for t in multihot_list]): + if isinstance(mutable, LayerChoice): + assert len(multihot_list) == len(mutable), \ + "Results returned from 'sample_final()' (%s: %s) either too short or too long." \ + % (mutable.key, multihot_list) + # check if all modules have different names and they indeed have names + if len(set(mutable.names)) == len(mutable) and not all(d.isdigit() for d in mutable.names): + converted = [name for i, name in enumerate(mutable.names) if multihot_list[i]] + else: + converted = [i for i in range(len(multihot_list)) if multihot_list[i]] + if isinstance(mutable, InputChoice): + assert len(multihot_list) == mutable.n_candidates, \ + "Results returned from 'sample_final()' (%s: %s) either too short or too long." \ + % (mutable.key, multihot_list) + # check if all input candidates have different names + if len(set(mutable.choose_from)) == mutable.n_candidates: + converted = [name for i, name in enumerate(mutable.choose_from) if multihot_list[i]] + else: + converted = [i for i in range(len(multihot_list)) if multihot_list[i]] + if converted is not None: + # if only one element, then remove the bracket + if len(converted) == 1: + converted = converted[0] + else: + # do nothing + converted = multihot_list + return converted diff --git a/new_impl/cv/third_party/nni_new/nas/pytorch/nasbench201/__init__.py b/new_impl/cv/third_party/nni_new/nas/pytorch/nasbench201/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..1419b3fa92420753a9c0ebea60944023ff0c313e --- /dev/null +++ b/new_impl/cv/third_party/nni_new/nas/pytorch/nasbench201/__init__.py @@ -0,0 +1 @@ +from .nasbench201 import NASBench201Cell diff --git a/new_impl/cv/third_party/nni_new/nas/pytorch/nasbench201/nasbench201.py b/new_impl/cv/third_party/nni_new/nas/pytorch/nasbench201/nasbench201.py new file mode 100644 index 0000000000000000000000000000000000000000..cd42fa1a2f20b86978f1d7fd44ca630c01d54d4d --- /dev/null +++ b/new_impl/cv/third_party/nni_new/nas/pytorch/nasbench201/nasbench201.py @@ -0,0 +1,72 @@ +from collections import OrderedDict +import torch.nn as nn +from nni.nas.pytorch.mutables import LayerChoice + +from .nasbench201_ops import Pooling, ReLUConvBN, Zero, FactorizedReduce + + +class NASBench201Cell(nn.Module): + """ + Builtin cell structure of NAS Bench 201. One cell contains four nodes. The First node serves as an input node + accepting the output of the previous cell. And other nodes connect to all previous nodes with an edge that + represents an operation chosen from a set to transform the tensor from the source node to the target node. + Every node accepts all its inputs and adds them as its output. + + Parameters + --- + cell_id: str + the name of this cell + C_in: int + the number of input channels of the cell + C_out: int + the number of output channels of the cell + stride: int + stride of all convolution operations in the cell + bn_affine: bool + If set to ``True``, all ``torch.nn.BatchNorm2d`` in this cell will have learnable affine parameters. Default: True + bn_momentum: float + the value used for the running_mean and running_var computation. Default: 0.1 + bn_track_running_stats: bool + When set to ``True``, all ``torch.nn.BatchNorm2d`` in this cell tracks the running mean and variance. Default: True + """ + + def __init__(self, cell_id, C_in, C_out, stride, bn_affine=True, bn_momentum=0.1, bn_track_running_stats=True): + super(NASBench201Cell, self).__init__() + + self.NUM_NODES = 4 + self.layers = nn.ModuleList() + + OPS = lambda layer_idx: OrderedDict([ + ("none", Zero(C_in, C_out, stride)), + ("avg_pool_3x3", Pooling(C_in, C_out, stride if layer_idx == 0 else 1, bn_affine, bn_momentum, + bn_track_running_stats)), + ("conv_3x3", ReLUConvBN(C_in, C_out, 3, stride if layer_idx == 0 else 1, 1, 1, bn_affine, bn_momentum, + bn_track_running_stats)), + ("conv_1x1", ReLUConvBN(C_in, C_out, 1, stride if layer_idx == 0 else 1, 0, 1, bn_affine, bn_momentum, + bn_track_running_stats)), + ("skip_connect", nn.Identity() if stride == 1 and C_in == C_out + else FactorizedReduce(C_in, C_out, stride if layer_idx == 0 else 1, bn_affine, bn_momentum, + bn_track_running_stats)) + ]) + + for i in range(self.NUM_NODES): + node_ops = nn.ModuleList() + for j in range(0, i): + node_ops.append(LayerChoice(OPS(j), key="%d_%d" % (j, i), reduction="mean")) + self.layers.append(node_ops) + self.in_dim = C_in + self.out_dim = C_out + self.cell_id = cell_id + + def forward(self, input): # pylint: disable=W0622 + """ + Parameters + --- + input: torch.tensor + the output of the previous layer + """ + nodes = [input] + for i in range(1, self.NUM_NODES): + node_feature = sum(self.layers[i][k](nodes[k]) for k in range(i)) + nodes.append(node_feature) + return nodes[-1] diff --git a/new_impl/cv/third_party/nni_new/nas/pytorch/nasbench201/nasbench201_ops.py b/new_impl/cv/third_party/nni_new/nas/pytorch/nasbench201/nasbench201_ops.py new file mode 100644 index 0000000000000000000000000000000000000000..aa60f8854fc1cf3169884cdde139eaa97f78831e --- /dev/null +++ b/new_impl/cv/third_party/nni_new/nas/pytorch/nasbench201/nasbench201_ops.py @@ -0,0 +1,146 @@ +import torch +import torch.nn as nn + + +class ReLUConvBN(nn.Module): + """ + Parameters + --- + C_in: int + the number of input channels + C_out: int + the number of output channels + stride: int + stride of the convolution + padding: int + zero-padding added to both sides of the input + dilation: int + spacing between kernel elements + bn_affine: bool + If set to ``True``, ``torch.nn.BatchNorm2d`` will have learnable affine parameters. Default: True + bn_momentun: float + the value used for the running_mean and running_var computation. Default: 0.1 + bn_track_running_stats: bool + When set to ``True``, ``torch.nn.BatchNorm2d`` tracks the running mean and variance. Default: True + """ + def __init__(self, C_in, C_out, kernel_size, stride, padding, dilation, + bn_affine=True, bn_momentum=0.1, bn_track_running_stats=True): + super(ReLUConvBN, self).__init__() + self.op = nn.Sequential( + nn.ReLU(inplace=False), + nn.Conv2d(C_in, C_out, kernel_size, stride=stride, + padding=padding, dilation=dilation, bias=False), + nn.BatchNorm2d(C_out, affine=bn_affine, momentum=bn_momentum, + track_running_stats=bn_track_running_stats) + ) + + def forward(self, x): + """ + Parameters + --- + x: torch.Tensor + input tensor + """ + return self.op(x) + + +class Pooling(nn.Module): + """ + Parameters + --- + C_in: int + the number of input channels + C_out: int + the number of output channels + stride: int + stride of the convolution + bn_affine: bool + If set to ``True``, ``torch.nn.BatchNorm2d`` will have learnable affine parameters. Default: True + bn_momentun: float + the value used for the running_mean and running_var computation. Default: 0.1 + bn_track_running_stats: bool + When set to ``True``, ``torch.nn.BatchNorm2d`` tracks the running mean and variance. Default: True + """ + def __init__(self, C_in, C_out, stride, bn_affine=True, bn_momentum=0.1, bn_track_running_stats=True): + super(Pooling, self).__init__() + if C_in == C_out: + self.preprocess = None + else: + self.preprocess = ReLUConvBN(C_in, C_out, 1, 1, 0, 0, + bn_affine, bn_momentum, bn_track_running_stats) + self.op = nn.AvgPool2d(3, stride=stride, padding=1, count_include_pad=False) + + def forward(self, x): + """ + Parameters + --- + x: torch.Tensor + input tensor + """ + if self.preprocess: + x = self.preprocess(x) + return self.op(x) + + +class Zero(nn.Module): + """ + Parameters + --- + C_in: int + the number of input channels + C_out: int + the number of output channels + stride: int + stride of the convolution + """ + def __init__(self, C_in, C_out, stride): + super(Zero, self).__init__() + self.C_in = C_in + self.C_out = C_out + self.stride = stride + self.is_zero = True + + def forward(self, x): + """ + Parameters + --- + x: torch.Tensor + input tensor + """ + if self.C_in == self.C_out: + if self.stride == 1: + return x.mul(0.) + else: + return x[:, :, ::self.stride, ::self.stride].mul(0.) + else: + shape = list(x.shape) + shape[1] = self.C_out + zeros = x.new_zeros(shape, dtype=x.dtype, device=x.device) + return zeros + + +class FactorizedReduce(nn.Module): + def __init__(self, C_in, C_out, stride, bn_affine=True, bn_momentum=0.1, + bn_track_running_stats=True): + super(FactorizedReduce, self).__init__() + self.stride = stride + self.C_in = C_in + self.C_out = C_out + self.relu = nn.ReLU(inplace=False) + if stride == 2: + C_outs = [C_out // 2, C_out - C_out // 2] + self.convs = nn.ModuleList() + for i in range(2): + self.convs.append(nn.Conv2d(C_in, C_outs[i], 1, stride=stride, padding=0, bias=False)) + self.pad = nn.ConstantPad2d((0, 1, 0, 1), 0) + else: + raise ValueError("Invalid stride : {:}".format(stride)) + self.bn = nn.BatchNorm2d(C_out, affine=bn_affine, momentum=bn_momentum, + track_running_stats=bn_track_running_stats) + + def forward(self, x): + x = self.relu(x) + y = self.pad(x) + out = torch.cat([self.convs[0](x), self.convs[1](y[:, :, 1:, 1:])], dim=1) + out = self.bn(out) + return out diff --git a/new_impl/cv/third_party/nni_new/nas/pytorch/search_space_zoo/__init__.py b/new_impl/cv/third_party/nni_new/nas/pytorch/search_space_zoo/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..59bb3b78d1874808c9ac9ed2c2b625786f16250e --- /dev/null +++ b/new_impl/cv/third_party/nni_new/nas/pytorch/search_space_zoo/__init__.py @@ -0,0 +1,4 @@ +from .darts_cell import DartsCell +from .enas_cell import ENASMicroLayer +from .enas_cell import ENASMacroLayer +from .enas_cell import ENASMacroGeneralModel diff --git a/new_impl/cv/third_party/nni_new/nas/pytorch/search_space_zoo/darts_cell.py b/new_impl/cv/third_party/nni_new/nas/pytorch/search_space_zoo/darts_cell.py new file mode 100644 index 0000000000000000000000000000000000000000..53fca5940c1f85b38cef2d08e8e20c8499728609 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/nas/pytorch/search_space_zoo/darts_cell.py @@ -0,0 +1,112 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from collections import OrderedDict + +import torch +import torch.nn as nn +from nni.nas.pytorch import mutables + +from .darts_ops import PoolBN, SepConv, DilConv, FactorizedReduce, DropPath, StdConv + + +class Node(nn.Module): + def __init__(self, node_id, num_prev_nodes, channels, num_downsample_connect): + """ + builtin Darts Node structure + + Parameters + --- + node_id: str + num_prev_nodes: int + the number of previous nodes in this cell + channels: int + output channels + num_downsample_connect: int + downsample the input node if this cell is reduction cell + """ + super().__init__() + self.ops = nn.ModuleList() + choice_keys = [] + for i in range(num_prev_nodes): + stride = 2 if i < num_downsample_connect else 1 + choice_keys.append("{}_p{}".format(node_id, i)) + self.ops.append( + mutables.LayerChoice(OrderedDict([ + ("maxpool", PoolBN('max', channels, 3, stride, 1, affine=False)), + ("avgpool", PoolBN('avg', channels, 3, stride, 1, affine=False)), + ("skipconnect", + nn.Identity() if stride == 1 else FactorizedReduce(channels, channels, affine=False)), + ("sepconv3x3", SepConv(channels, channels, 3, stride, 1, affine=False)), + ("sepconv5x5", SepConv(channels, channels, 5, stride, 2, affine=False)), + ("dilconv3x3", DilConv(channels, channels, 3, stride, 2, 2, affine=False)), + ("dilconv5x5", DilConv(channels, channels, 5, stride, 4, 2, affine=False)) + ]), key=choice_keys[-1])) + self.drop_path = DropPath() + self.input_switch = mutables.InputChoice(choose_from=choice_keys, n_chosen=2, key="{}_switch".format(node_id)) + + def forward(self, prev_nodes): + assert len(self.ops) == len(prev_nodes) + out = [op(node) for op, node in zip(self.ops, prev_nodes)] + out = [self.drop_path(o) if o is not None else None for o in out] + return self.input_switch(out) + + +class DartsCell(nn.Module): + """ + Builtin Darts Cell structure. There are ``n_nodes`` nodes in one cell, in which the first two nodes' values are + fixed to the results of previous previous cell and previous cell respectively. One node will connect all + the nodes after with predefined operations in a mutable way. The last node accepts five inputs from nodes + before and it concats all inputs in channels as the output of the current cell, and the number of output + channels is ``n_nodes`` times ``channels``. + + Parameters + --- + n_nodes: int + the number of nodes contained in this cell + channels_pp: int + the number of previous previous cell's output channels + channels_p: int + the number of previous cell's output channels + channels: int + the number of output channels for each node + reduction_p: bool + Is previous cell a reduction cell + reduction: bool + is current cell a reduction cell + """ + def __init__(self, n_nodes, channels_pp, channels_p, channels, reduction_p, reduction): + super().__init__() + self.reduction = reduction + self.n_nodes = n_nodes + + # If previous cell is reduction cell, current input size does not match with + # output size of cell[k-2]. So the output[k-2] should be reduced by preprocessing. + if reduction_p: + self.preproc0 = FactorizedReduce(channels_pp, channels, affine=False) + else: + self.preproc0 = StdConv(channels_pp, channels, 1, 1, 0, affine=False) + self.preproc1 = StdConv(channels_p, channels, 1, 1, 0, affine=False) + + # generate dag + self.mutable_ops = nn.ModuleList() + for depth in range(2, self.n_nodes + 2): + self.mutable_ops.append(Node("{}_n{}".format("reduce" if reduction else "normal", depth), + depth, channels, 2 if reduction else 0)) + + def forward(self, pprev, prev): + """ + Parameters + --- + pprev: torch.Tensor + the output of the previous previous layer + prev: torch.Tensor + the output of the previous layer + """ + tensors = [self.preproc0(pprev), self.preproc1(prev)] + for node in self.mutable_ops: + cur_tensor = node(tensors) + tensors.append(cur_tensor) + + output = torch.cat(tensors[2:], dim=1) + return output diff --git a/new_impl/cv/third_party/nni_new/nas/pytorch/search_space_zoo/darts_ops.py b/new_impl/cv/third_party/nni_new/nas/pytorch/search_space_zoo/darts_ops.py new file mode 100644 index 0000000000000000000000000000000000000000..ce5410cfb4a5a466f26e67aaf93d21f90672407c --- /dev/null +++ b/new_impl/cv/third_party/nni_new/nas/pytorch/search_space_zoo/darts_ops.py @@ -0,0 +1,196 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import torch +import torch.nn as nn + + +class DropPath(nn.Module): + def __init__(self, p=0.): + """ + Drop path with probability. + + Parameters + ---------- + p : float + Probability of an path to be zeroed. + """ + super().__init__() + self.p = p + + def forward(self, x): + if self.training and self.p > 0.: + keep_prob = 1. - self.p + # per data point mask + mask = torch.zeros((x.size(0), 1, 1, 1), device=x.device).bernoulli_(keep_prob) + return x / keep_prob * mask + + return x + + +class PoolBN(nn.Module): + """ + AvgPool or MaxPool with BN. ``pool_type`` must be ``max`` or ``avg``. + + Parameters + --- + pool_type: str + choose operation + C: int + number of channels + kernal_size: int + size of the convolving kernel + stride: int + stride of the convolution + padding: int + zero-padding added to both sides of the input + affine: bool + is using affine in BatchNorm + """ + + def __init__(self, pool_type, C, kernel_size, stride, padding, affine=True): + super().__init__() + if pool_type.lower() == 'max': + self.pool = nn.MaxPool2d(kernel_size, stride, padding) + elif pool_type.lower() == 'avg': + self.pool = nn.AvgPool2d(kernel_size, stride, padding, count_include_pad=False) + else: + raise ValueError() + + self.bn = nn.BatchNorm2d(C, affine=affine) + + def forward(self, x): + out = self.pool(x) + out = self.bn(out) + return out + + +class StdConv(nn.Sequential): + """ + Standard conv: ReLU - Conv - BN + + Parameters + --- + C_in: int + the number of input channels + C_out: int + the number of output channels + kernel_size: int + size of the convolution kernel + padding: + zero-padding added to both sides of the input + affine: bool + is using affine in BatchNorm + """ + + def __init__(self, C_in, C_out, kernel_size, stride, padding, affine=True): + super().__init__() + self.net = nn.Sequential + for idx, ops in enumerate((nn.ReLU(), nn.Conv2d(C_in, C_out, kernel_size, stride, padding, bias=False), + nn.BatchNorm2d(C_out, affine=affine))): + self.add_module(str(idx), ops) + + +class FacConv(nn.Module): + """ + Factorized conv: ReLU - Conv(Kx1) - Conv(1xK) - BN + """ + + def __init__(self, C_in, C_out, kernel_length, stride, padding, affine=True): + super().__init__() + self.net = nn.Sequential( + nn.ReLU(), + nn.Conv2d(C_in, C_in, (kernel_length, 1), stride, padding, bias=False), + nn.Conv2d(C_in, C_out, (1, kernel_length), stride, padding, bias=False), + nn.BatchNorm2d(C_out, affine=affine) + ) + + def forward(self, x): + return self.net(x) + + +class DilConv(nn.Module): + """ + (Dilated) depthwise separable conv. + ReLU - (Dilated) depthwise separable - Pointwise - BN. + If dilation == 2, 3x3 conv => 5x5 receptive field, 5x5 conv => 9x9 receptive field. + + Parameters + --- + C_in: int + the number of input channels + C_out: int + the number of output channels + kernal_size: + size of the convolving kernel + padding: + zero-padding added to both sides of the input + dilation: int + spacing between kernel elements. + affine: bool + is using affine in BatchNorm + """ + + def __init__(self, C_in, C_out, kernel_size, stride, padding, dilation, affine=True): + super().__init__() + self.net = nn.Sequential( + nn.ReLU(), + nn.Conv2d(C_in, C_in, kernel_size, stride, padding, dilation=dilation, groups=C_in, + bias=False), + nn.Conv2d(C_in, C_out, 1, stride=1, padding=0, bias=False), + nn.BatchNorm2d(C_out, affine=affine) + ) + + def forward(self, x): + return self.net(x) + + +class SepConv(nn.Module): + """ + Depthwise separable conv. + DilConv(dilation=1) * 2. + + Parameters + --- + C_in: int + the number of input channels + C_out: int + the number of output channels + kernal_size: + size of the convolving kernel + padding: + zero-padding added to both sides of the input + dilation: int + spacing between kernel elements. + affine: bool + is using affine in BatchNorm + """ + + def __init__(self, C_in, C_out, kernel_size, stride, padding, affine=True): + super().__init__() + self.net = nn.Sequential( + DilConv(C_in, C_in, kernel_size, stride, padding, dilation=1, affine=affine), + DilConv(C_in, C_out, kernel_size, 1, padding, dilation=1, affine=affine) + ) + + def forward(self, x): + return self.net(x) + + +class FactorizedReduce(nn.Module): + """ + Reduce feature map size by factorized pointwise (stride=2). + """ + + def __init__(self, C_in, C_out, affine=True): + super().__init__() + self.relu = nn.ReLU() + self.conv1 = nn.Conv2d(C_in, C_out // 2, 1, stride=2, padding=0, bias=False) + self.conv2 = nn.Conv2d(C_in, C_out // 2, 1, stride=2, padding=0, bias=False) + self.bn = nn.BatchNorm2d(C_out, affine=affine) + + def forward(self, x): + x = self.relu(x) + out = torch.cat([self.conv1(x), self.conv2(x[:, :, 1:, 1:])], dim=1) + out = self.bn(out) + return out diff --git a/new_impl/cv/third_party/nni_new/nas/pytorch/search_space_zoo/enas_cell.py b/new_impl/cv/third_party/nni_new/nas/pytorch/search_space_zoo/enas_cell.py new file mode 100644 index 0000000000000000000000000000000000000000..de57d55e23f83bb5a35cd4d3a474c71b8fc90fc4 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/nas/pytorch/search_space_zoo/enas_cell.py @@ -0,0 +1,255 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from nni.nas.pytorch import mutables +from .enas_ops import FactorizedReduce, StdConv, SepConvBN, Pool, ConvBranch, PoolBranch + + +class Cell(nn.Module): + def __init__(self, cell_name, prev_labels, channels): + super().__init__() + self.input_choice = mutables.InputChoice(choose_from=prev_labels, n_chosen=1, return_mask=True, + key=cell_name + "_input") + self.op_choice = mutables.LayerChoice([ + SepConvBN(channels, channels, 3, 1), + SepConvBN(channels, channels, 5, 2), + Pool("avg", 3, 1, 1), + Pool("max", 3, 1, 1), + nn.Identity() + ], key=cell_name + "_op") + + def forward(self, prev_layers): + chosen_input, chosen_mask = self.input_choice(prev_layers) + cell_out = self.op_choice(chosen_input) + return cell_out, chosen_mask + + +class Node(mutables.MutableScope): + def __init__(self, node_name, prev_node_names, channels): + super().__init__(node_name) + self.cell_x = Cell(node_name + "_x", prev_node_names, channels) + self.cell_y = Cell(node_name + "_y", prev_node_names, channels) + + def forward(self, prev_layers): + out_x, mask_x = self.cell_x(prev_layers) + out_y, mask_y = self.cell_y(prev_layers) + return out_x + out_y, mask_x | mask_y + + +class Calibration(nn.Module): + def __init__(self, in_channels, out_channels): + super().__init__() + self.process = None + if in_channels != out_channels: + self.process = StdConv(in_channels, out_channels) + + def forward(self, x): + if self.process is None: + return x + return self.process(x) + + +class ENASMicroLayer(nn.Module): + """ + Builtin EnasMicroLayer. Micro search designs only one building block whose architecture is repeated + throughout the final architecture. A cell has ``num_nodes`` nodes and searches the topology and + operations among them in RL way. The first two nodes in a layer stand for the outputs from previous + previous layer and previous layer respectively. For the following nodes, the controller chooses + two previous nodes and applies two operations respectively for each node. Nodes that are not served + as input for any other node are viewed as the output of the layer. If there are multiple output nodes, + the model will calculate the average of these nodes as the layer output. Every node's output has ``out_channels`` + channels so the result of the layer has the same number of channels as each node. + + Parameters + --- + num_nodes: int + the number of nodes contained in this layer + in_channles_pp: int + the number of previous previous layer's output channels + in_channels_p: int + the number of previous layer's output channels + out_channels: int + output channels of this layer + reduction: bool + is reduction operation empolyed before this layer + """ + def __init__(self, num_nodes, in_channels_pp, in_channels_p, out_channels, reduction): + super().__init__() + self.reduction = reduction + if self.reduction: + self.reduce0 = FactorizedReduce(in_channels_pp, out_channels, affine=False) + self.reduce1 = FactorizedReduce(in_channels_p, out_channels, affine=False) + in_channels_pp = in_channels_p = out_channels + self.preproc0 = Calibration(in_channels_pp, out_channels) + self.preproc1 = Calibration(in_channels_p, out_channels) + + self.num_nodes = num_nodes + name_prefix = "reduce" if reduction else "normal" + self.nodes = nn.ModuleList() + node_labels = [mutables.InputChoice.NO_KEY, mutables.InputChoice.NO_KEY] + for i in range(num_nodes): + node_labels.append("{}_node_{}".format(name_prefix, i)) + self.nodes.append(Node(node_labels[-1], node_labels[:-1], out_channels)) + self.final_conv_w = nn.Parameter(torch.zeros(out_channels, self.num_nodes + 2, out_channels, 1, 1), + requires_grad=True) + self.bn = nn.BatchNorm2d(out_channels, affine=False) + self.reset_parameters() + + def reset_parameters(self): + nn.init.kaiming_normal_(self.final_conv_w) + + def forward(self, pprev, prev): + """ + Parameters + --- + pprev: torch.Tensor + the output of the previous previous layer + prev: torch.Tensor + the output of the previous layer + """ + if self.reduction: + pprev, prev = self.reduce0(pprev), self.reduce1(prev) + pprev_, prev_ = self.preproc0(pprev), self.preproc1(prev) + + prev_nodes_out = [pprev_, prev_] + nodes_used_mask = torch.zeros(self.num_nodes + 2, dtype=torch.bool, device=prev.device) + for i in range(self.num_nodes): + node_out, mask = self.nodes[i](prev_nodes_out) + nodes_used_mask[:mask.size(0)] |= mask.to(node_out.device) + prev_nodes_out.append(node_out) + + unused_nodes = torch.cat([out for used, out in zip(nodes_used_mask, prev_nodes_out) if not used], 1) + unused_nodes = F.relu(unused_nodes) + conv_weight = self.final_conv_w[:, ~nodes_used_mask, :, :, :] + conv_weight = conv_weight.view(conv_weight.size(0), -1, 1, 1) + out = F.conv2d(unused_nodes, conv_weight) + return prev, self.bn(out) + + +class ENASMacroLayer(mutables.MutableScope): + """ + Builtin ENAS Marco Layer. With search space changing to layer level, the controller decides + what operation is employed and the previous layer to connect to for skip connections. The model + is made up of the same layers but the choice of each layer may be different. + + Parameters + --- + key: str + the name of this layer + prev_labels: str + names of all previous layers + in_filters: int + the number of input channels + out_filters: + the number of output channels + """ + def __init__(self, key, prev_labels, in_filters, out_filters): + super().__init__(key) + self.in_filters = in_filters + self.out_filters = out_filters + self.mutable = mutables.LayerChoice([ + ConvBranch(in_filters, out_filters, 3, 1, 1, separable=False), + ConvBranch(in_filters, out_filters, 3, 1, 1, separable=True), + ConvBranch(in_filters, out_filters, 5, 1, 2, separable=False), + ConvBranch(in_filters, out_filters, 5, 1, 2, separable=True), + PoolBranch('avg', in_filters, out_filters, 3, 1, 1), + PoolBranch('max', in_filters, out_filters, 3, 1, 1) + ]) + if prev_labels: + self.skipconnect = mutables.InputChoice(choose_from=prev_labels, n_chosen=None) + else: + self.skipconnect = None + self.batch_norm = nn.BatchNorm2d(out_filters, affine=False) + + def forward(self, prev_list): + """ + Parameters + --- + prev_list: list + The cell selects the last element of the list as input and applies an operation on it. + The cell chooses none/one/multiple tensor(s) as SkipConnect(s) from the list excluding + the last element. + """ + out = self.mutable(prev_list[-1]) + if self.skipconnect is not None: + connection = self.skipconnect(prev_list[:-1]) + if connection is not None: + out += connection + return self.batch_norm(out) + + +class ENASMacroGeneralModel(nn.Module): + """ + The network is made up by stacking ENASMacroLayer. The Macro search space contains these layers. + Each layer chooses an operation from predefined ones and SkipConnect then forms a network. + + Parameters + --- + num_layers: int + The number of layers contained in the network. + out_filters: int + The number of each layer's output channels. + in_channel: int + The number of input's channels. + num_classes: int + The number of classes for classification. + dropout_rate: float + Dropout layer's dropout rate before the final dense layer. + """ + def __init__(self, num_layers=12, out_filters=24, in_channels=3, num_classes=10, + dropout_rate=0.0): + super().__init__() + self.num_layers = num_layers + self.num_classes = num_classes + self.out_filters = out_filters + + self.stem = nn.Sequential( + nn.Conv2d(in_channels, out_filters, 3, 1, 1, bias=False), + nn.BatchNorm2d(out_filters) + ) + + pool_distance = self.num_layers // 3 + self.pool_layers_idx = [pool_distance - 1, 2 * pool_distance - 1] + self.dropout_rate = dropout_rate + self.dropout = nn.Dropout(self.dropout_rate) + + self.layers = nn.ModuleList() + self.pool_layers = nn.ModuleList() + labels = [] + for layer_id in range(self.num_layers): + labels.append("layer_{}".format(layer_id)) + if layer_id in self.pool_layers_idx: + self.pool_layers.append(FactorizedReduce(self.out_filters, self.out_filters)) + self.layers.append(ENASMacroLayer(labels[-1], labels[:-1], self.out_filters, self.out_filters)) + + self.gap = nn.AdaptiveAvgPool2d(1) + self.dense = nn.Linear(self.out_filters, self.num_classes) + + def forward(self, x): + """ + Parameters + --- + x: torch.Tensor + the input of the network + """ + bs = x.size(0) + cur = self.stem(x) + + layers = [cur] + + for layer_id in range(self.num_layers): + cur = self.layers[layer_id](layers) + layers.append(cur) + if layer_id in self.pool_layers_idx: + for i, layer in enumerate(layers): + layers[i] = self.pool_layers[self.pool_layers_idx.index(layer_id)](layer) + cur = layers[-1] + + cur = self.gap(cur).view(bs, -1) + cur = self.dropout(cur) + logits = self.dense(cur) + return logits diff --git a/new_impl/cv/third_party/nni_new/nas/pytorch/search_space_zoo/enas_ops.py b/new_impl/cv/third_party/nni_new/nas/pytorch/search_space_zoo/enas_ops.py new file mode 100644 index 0000000000000000000000000000000000000000..21ecc2da798dd025966a4631d59fb6f652e5da98 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/nas/pytorch/search_space_zoo/enas_ops.py @@ -0,0 +1,171 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import torch +import torch.nn as nn + + +class StdConv(nn.Module): + def __init__(self, C_in, C_out): + super(StdConv, self).__init__() + self.conv = nn.Sequential( + nn.Conv2d(C_in, C_out, 1, stride=1, padding=0, bias=False), + nn.BatchNorm2d(C_out, affine=False), + nn.ReLU() + ) + + def forward(self, x): + return self.conv(x) + + +class PoolBranch(nn.Module): + """ + Pooling structure for Macro search. First pass through a 1x1 Conv, then pooling operation followed by BatchNorm2d. + + Parameters + --- + pool_type: str + only accept ``max`` for MaxPool and ``avg`` for AvgPool + C_in: int + the number of input channels + C_out: int + the number of output channels + kernal_size: int + size of the convolving kernel + stride: int + stride of the convolution + padding: int + zero-padding added to both sides of the input + """ + def __init__(self, pool_type, C_in, C_out, kernel_size, stride, padding, affine=False): + super().__init__() + self.preproc = StdConv(C_in, C_out) + self.pool = Pool(pool_type, kernel_size, stride, padding) + self.bn = nn.BatchNorm2d(C_out, affine=affine) + + def forward(self, x): + out = self.preproc(x) + out = self.pool(out) + out = self.bn(out) + return out + + +class SeparableConv(nn.Module): + def __init__(self, C_in, C_out, kernel_size, stride, padding): + super(SeparableConv, self).__init__() + self.depthwise = nn.Conv2d(C_in, C_in, kernel_size=kernel_size, padding=padding, stride=stride, + groups=C_in, bias=False) + self.pointwise = nn.Conv2d(C_in, C_out, kernel_size=1, bias=False) + + def forward(self, x): + out = self.depthwise(x) + out = self.pointwise(out) + return out + + +class ConvBranch(nn.Module): + """ + Conv structure for Macro search. First pass through a 1x1 Conv, + then Conv operation with kernal_size equals 3 or 5 followed by BatchNorm and ReLU. + + Parameters + --- + C_in: int + the number of input channels + C_out: int + the number of output channels + kernal_size: int + size of the convolving kernel + stride: int + stride of the convolution + padding: int + zero-padding added to both sides of the input + separable: True + is separable Conv is used + """ + def __init__(self, C_in, C_out, kernel_size, stride, padding, separable): + super(ConvBranch, self).__init__() + self.preproc = StdConv(C_in, C_out) + if separable: + self.conv = SeparableConv(C_out, C_out, kernel_size, stride, padding) + else: + self.conv = nn.Conv2d(C_out, C_out, kernel_size, stride=stride, padding=padding) + self.postproc = nn.Sequential( + nn.BatchNorm2d(C_out, affine=False), + nn.ReLU() + ) + + def forward(self, x): + out = self.preproc(x) + out = self.conv(out) + out = self.postproc(out) + return out + + +class FactorizedReduce(nn.Module): + def __init__(self, C_in, C_out, affine=False): + super().__init__() + self.conv1 = nn.Conv2d(C_in, C_out // 2, 1, stride=2, padding=0, bias=False) + self.conv2 = nn.Conv2d(C_in, C_out // 2, 1, stride=2, padding=0, bias=False) + self.bn = nn.BatchNorm2d(C_out, affine=affine) + + def forward(self, x): + out = torch.cat([self.conv1(x), self.conv2(x[:, :, 1:, 1:])], dim=1) + out = self.bn(out) + return out + + +class Pool(nn.Module): + """ + Pooling structure + + Parameters + --- + pool_type: str + only accept ``max`` for MaxPool and ``avg`` for AvgPool + kernal_size: int + size of the convolving kernel + stride: int + stride of the convolution + padding: int + zero-padding added to both sides of the input + """ + def __init__(self, pool_type, kernel_size, stride, padding): + super().__init__() + if pool_type.lower() == 'max': + self.pool = nn.MaxPool2d(kernel_size, stride, padding) + elif pool_type.lower() == 'avg': + self.pool = nn.AvgPool2d(kernel_size, stride, padding, count_include_pad=False) + else: + raise ValueError() + + def forward(self, x): + return self.pool(x) + + +class SepConvBN(nn.Module): + """ + Implement SepConv followed by BatchNorm. The structure is ReLU ==> SepConv ==> BN. + + Parameters + --- + C_in: int + the number of imput channels + C_out: int + the number of output channels + kernal_size: int + size of the convolving kernel + padding: int + zero-padding added to both sides of the input + """ + def __init__(self, C_in, C_out, kernel_size, padding): + super().__init__() + self.relu = nn.ReLU() + self.conv = SeparableConv(C_in, C_out, kernel_size, 1, padding) + self.bn = nn.BatchNorm2d(C_out, affine=True) + + def forward(self, x): + x = self.relu(x) + x = self.conv(x) + x = self.bn(x) + return x diff --git a/new_impl/cv/third_party/nni_new/nas/pytorch/trainer.py b/new_impl/cv/third_party/nni_new/nas/pytorch/trainer.py new file mode 100644 index 0000000000000000000000000000000000000000..6a3881177a63befdf1c32f8b53edc88b4f8fd379 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/nas/pytorch/trainer.py @@ -0,0 +1,194 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import json +import logging +import os +import time +from abc import abstractmethod + +import torch + +from .base_trainer import BaseTrainer + +_logger = logging.getLogger(__name__) + + +class TorchTensorEncoder(json.JSONEncoder): + def default(self, o): # pylint: disable=method-hidden + if isinstance(o, torch.Tensor): + olist = o.tolist() + if "bool" not in o.type().lower() and all(map(lambda d: d == 0 or d == 1, olist)): + _logger.warning("Every element in %s is either 0 or 1. " + "You might consider convert it into bool.", olist) + return olist + return super().default(o) + + +class Trainer(BaseTrainer): + """ + A trainer with some helper functions implemented. To implement a new trainer, + users need to implement :meth:`train_one_epoch`, :meth:`validate_one_epoch` and :meth:`checkpoint`. + + Parameters + ---------- + model : nn.Module + Model with mutables. + mutator : BaseMutator + A mutator object that has been initialized with the model. + loss : callable + Called with logits and targets. Returns a loss tensor. + See `PyTorch loss functions`_ for examples. + metrics : callable + Called with logits and targets. Returns a dict that maps metrics keys to metrics data. For example, + + .. code-block:: python + + def metrics_fn(output, target): + return {"acc1": accuracy(output, target, topk=1), "acc5": accuracy(output, target, topk=5)} + + optimizer : Optimizer + Optimizer that optimizes the model. + num_epochs : int + Number of epochs of training. + dataset_train : torch.utils.data.Dataset + Dataset of training. If not otherwise specified, ``dataset_train`` and ``dataset_valid`` should be standard + PyTorch Dataset. See `torch.utils.data`_ for examples. + dataset_valid : torch.utils.data.Dataset + Dataset of validation/testing. + batch_size : int + Batch size. + workers : int + Number of workers used in data preprocessing. + device : torch.device + Device object. Either ``torch.device("cuda")`` or ``torch.device("cpu")``. When ``None``, trainer will + automatic detects GPU and selects GPU first. + log_frequency : int + Number of mini-batches to log metrics. + callbacks : list of Callback + Callbacks to plug into the trainer. See Callbacks. + + + .. _`PyTorch loss functions`: https://pytorch.org/docs/stable/nn.html#loss-functions + .. _`torch.utils.data`: https://pytorch.org/docs/stable/data.html + """ + def __init__(self, model, mutator, loss, metrics, optimizer, num_epochs, + dataset_train, dataset_valid, batch_size, workers, device, log_frequency, callbacks): + self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") if device is None else device + self.model = model + self.mutator = mutator + self.loss = loss + + self.metrics = metrics + self.optimizer = optimizer + + self.model.to(self.device) + self.mutator.to(self.device) + self.loss.to(self.device) + + self.num_epochs = num_epochs + self.dataset_train = dataset_train + self.dataset_valid = dataset_valid + self.batch_size = batch_size + self.workers = workers + self.log_frequency = log_frequency + self.log_dir = os.path.join("logs", str(time.time())) + os.makedirs(self.log_dir, exist_ok=True) + self.status_writer = open(os.path.join(self.log_dir, "log"), "w") + self.callbacks = callbacks if callbacks is not None else [] + for callback in self.callbacks: + callback.build(self.model, self.mutator, self) + + @abstractmethod + def train_one_epoch(self, epoch): + """ + Train one epoch. + + Parameters + ---------- + epoch : int + Epoch number starting from 0. + """ + pass + + @abstractmethod + def validate_one_epoch(self, epoch): + """ + Validate one epoch. + + Parameters + ---------- + epoch : int + Epoch number starting from 0. + """ + pass + + def train(self, validate=True): + """ + Train ``num_epochs``. + Trigger callbacks at the start and the end of each epoch. + + Parameters + ---------- + validate : bool + If ``true``, will do validation every epoch. + """ + for epoch in range(self.num_epochs): + for callback in self.callbacks: + callback.on_epoch_begin(epoch) + + # training + _logger.info("Epoch %d Training", epoch + 1) + self.train_one_epoch(epoch) + + if validate: + # validation + _logger.info("Epoch %d Validating", epoch + 1) + self.validate_one_epoch(epoch) + + for callback in self.callbacks: + callback.on_epoch_end(epoch) + + def validate(self): + """ + Do one validation. + """ + self.validate_one_epoch(-1) + + def export(self, file): + """ + Call ``mutator.export()`` and dump the architecture to ``file``. + + Parameters + ---------- + file : str + A file path. Expected to be a JSON. + """ + mutator_export = self.mutator.export() + with open(file, "w") as f: + json.dump(mutator_export, f, indent=2, sort_keys=True, cls=TorchTensorEncoder) + + def checkpoint(self): + """ + Return trainer checkpoint. + """ + raise NotImplementedError("Not implemented yet") + + def enable_visualization(self): + """ + Enable visualization. Write graph and training log to folder ``logs/``. + """ + sample = None + for x, _ in self.train_loader: + sample = x.to(self.device)[:2] + break + if sample is None: + _logger.warning("Sample is %s.", sample) + _logger.info("Creating graph json, writing to %s. Visualization enabled.", self.log_dir) + with open(os.path.join(self.log_dir, "graph.json"), "w") as f: + json.dump(self.mutator.graph(sample), f) + self.visualization_enabled = True + + def _write_graph_status(self): + if hasattr(self, "visualization_enabled") and self.visualization_enabled: + print(json.dumps(self.mutator.status()), file=self.status_writer, flush=True) diff --git a/new_impl/cv/third_party/nni_new/nas/pytorch/utils.py b/new_impl/cv/third_party/nni_new/nas/pytorch/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..a3f5aabfb74e5c1d2350f579233ae36fdadb2313 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/nas/pytorch/utils.py @@ -0,0 +1,210 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +from collections import OrderedDict + +import numpy as np +import torch + +_counter = 0 + +_logger = logging.getLogger(__name__) + + +def global_mutable_counting(): + """ + A program level counter starting from 1. + """ + global _counter + _counter += 1 + return _counter + + +def _reset_global_mutable_counting(): + """ + Reset the global mutable counting to count from 1. Useful when defining multiple models with default keys. + """ + global _counter + _counter = 0 + + +def to_device(obj, device): + """ + Move a tensor, tuple, list, or dict onto device. + """ + if torch.is_tensor(obj): + return obj.to(device) + if isinstance(obj, tuple): + return tuple(to_device(t, device) for t in obj) + if isinstance(obj, list): + return [to_device(t, device) for t in obj] + if isinstance(obj, dict): + return {k: to_device(v, device) for k, v in obj.items()} + if isinstance(obj, (int, float, str)): + return obj + raise ValueError("'%s' has unsupported type '%s'" % (obj, type(obj))) + + +def to_list(arr): + if torch.is_tensor(arr): + return arr.cpu().numpy().tolist() + if isinstance(arr, np.ndarray): + return arr.tolist() + if isinstance(arr, (list, tuple)): + return list(arr) + return arr + + +class AverageMeterGroup: + """ + Average meter group for multiple average meters. + """ + + def __init__(self): + self.meters = OrderedDict() + + def update(self, data): + """ + Update the meter group with a dict of metrics. + Non-exist average meters will be automatically created. + """ + for k, v in data.items(): + if k not in self.meters: + self.meters[k] = AverageMeter(k, ":4f") + self.meters[k].update(v) + + def __getattr__(self, item): + return self.meters[item] + + def __getitem__(self, item): + return self.meters[item] + + def __str__(self): + return " ".join(str(v) for v in self.meters.values()) + + def summary(self): + """ + Return a summary string of group data. + """ + return " ".join(v.summary() for v in self.meters.values()) + + +class AverageMeter: + """ + Computes and stores the average and current value. + + Parameters + ---------- + name : str + Name to display. + fmt : str + Format string to print the values. + """ + + def __init__(self, name, fmt=':f'): + self.name = name + self.fmt = fmt + self.reset() + + def reset(self): + """ + Reset the meter. + """ + self.val = 0 + self.avg = 0 + self.sum = 0 + self.count = 0 + + def update(self, val, n=1): + """ + Update with value and weight. + + Parameters + ---------- + val : float or int + The new value to be accounted in. + n : int + The weight of the new value. + """ + self.val = val + self.sum += val * n + self.count += n + self.avg = self.sum / self.count + + def __str__(self): + fmtstr = '{name} {val' + self.fmt + '} ({avg' + self.fmt + '})' + return fmtstr.format(**self.__dict__) + + def summary(self): + fmtstr = '{name}: {avg' + self.fmt + '}' + return fmtstr.format(**self.__dict__) + + +class StructuredMutableTreeNode: + """ + A structured representation of a search space. + A search space comes with a root (with `None` stored in its `mutable`), and a bunch of children in its `children`. + This tree can be seen as a "flattened" version of the module tree. Since nested mutable entity is not supported yet, + the following must be true: each subtree corresponds to a ``MutableScope`` and each leaf corresponds to a + ``Mutable`` (other than ``MutableScope``). + + Parameters + ---------- + mutable : nni.nas.pytorch.mutables.Mutable + The mutable that current node is linked with. + """ + + def __init__(self, mutable): + self.mutable = mutable + self.children = [] + + def add_child(self, mutable): + """ + Add a tree node to the children list of current node. + """ + self.children.append(StructuredMutableTreeNode(mutable)) + return self.children[-1] + + def type(self): + """ + Return the ``type`` of mutable content. + """ + return type(self.mutable) + + def __iter__(self): + return self.traverse() + + def traverse(self, order="pre", deduplicate=True, memo=None): + """ + Return a generator that generates a list of mutables in this tree. + + Parameters + ---------- + order : str + pre or post. If pre, current mutable is yield before children. Otherwise after. + deduplicate : bool + If true, mutables with the same key will not appear after the first appearance. + memo : dict + An auxiliary dict that memorize keys seen before, so that deduplication is possible. + + Returns + ------- + generator of Mutable + """ + if memo is None: + memo = set() + assert order in ["pre", "post"] + if order == "pre": + if self.mutable is not None: + if not deduplicate or self.mutable.key not in memo: + memo.add(self.mutable.key) + yield self.mutable + for child in self.children: + for m in child.traverse(order=order, deduplicate=deduplicate, memo=memo): + yield m + if order == "post": + if self.mutable is not None: + if not deduplicate or self.mutable.key not in memo: + memo.add(self.mutable.key) + yield self.mutable diff --git a/new_impl/cv/third_party/nni_new/nas/tensorflow/__init__.py b/new_impl/cv/third_party/nni_new/nas/tensorflow/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/new_impl/cv/third_party/nni_new/nas/tensorflow/base_mutator.py b/new_impl/cv/third_party/nni_new/nas/tensorflow/base_mutator.py new file mode 100644 index 0000000000000000000000000000000000000000..860680f199278d3fd38910b82e7661b17d2f652e --- /dev/null +++ b/new_impl/cv/third_party/nni_new/nas/tensorflow/base_mutator.py @@ -0,0 +1,73 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from tensorflow.keras import Model + +from .mutables import Mutable, MutableScope, InputChoice +from .utils import StructuredMutableTreeNode + + +class BaseMutator(Model): + def __init__(self, model): + super().__init__() + self.__dict__['model'] = model + self._structured_mutables = self._parse_search_space(self.model) + + def _parse_search_space(self, module, root=None, prefix='', memo=None, nested_detection=None): + if memo is None: + memo = set() + if root is None: + root = StructuredMutableTreeNode(None) + if module not in memo: + memo.add(module) + if isinstance(module, Mutable): + if nested_detection is not None: + raise RuntimeError('Cannot have nested search space. Error at {} in {}' + .format(module, nested_detection)) + module.name = prefix + module.set_mutator(self) + root = root.add_child(module) + if not isinstance(module, MutableScope): + nested_detection = module + if isinstance(module, InputChoice): + for k in module.choose_from: + if k != InputChoice.NO_KEY and k not in [m.key for m in memo if isinstance(m, Mutable)]: + raise RuntimeError('"{}" required by "{}" not found in keys that appeared before, and is not NO_KEY.' + .format(k, module.key)) + for submodule in module.layers: + if not isinstance(submodule, Model): + continue + submodule_prefix = prefix + ('.' if prefix else '') + submodule.name + self._parse_search_space(submodule, root, submodule_prefix, memo=memo, nested_detection=nested_detection) + return root + + @property + def mutables(self): + return self._structured_mutables + + def undedup_mutables(self): + return self._structured_mutables.traverse(deduplicate=False) + + def call(self, *inputs): + raise RuntimeError('Call is undefined for mutators.') + + def __setattr__(self, name, value): + if name == 'model': + raise AttributeError("Attribute `model` can be set at most once, and you shouldn't use `self.model = model` to " + "include your network, as it will include all parameters in model into the mutator.") + return super().__setattr__(name, value) + + def enter_mutable_scope(self, mutable_scope): + pass + + def exit_mutable_scope(self, mutable_scope): + pass + + def on_forward_layer_choice(self, mutable, *inputs): + raise NotImplementedError + + def on_forward_input_choice(self, mutable, tensor_list): + raise NotImplementedError + + def export(self): + raise NotImplementedError diff --git a/new_impl/cv/third_party/nni_new/nas/tensorflow/mutables.py b/new_impl/cv/third_party/nni_new/nas/tensorflow/mutables.py new file mode 100644 index 0000000000000000000000000000000000000000..06183a34c1872804f87887567692fcab6a732816 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/nas/tensorflow/mutables.py @@ -0,0 +1,144 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +from collections import OrderedDict + +from tensorflow.keras import Model + +from .utils import global_mutable_counting + + +_logger = logging.getLogger(__name__) + + +class Mutable(Model): + def __init__(self, key=None): + super().__init__() + if key is None: + self._key = '{}_{}'.format(type(self).__name__, global_mutable_counting()) + elif isinstance(key, str): + self._key = key + else: + self._key = str(key) + _logger.warning('Key "%s" is not string, converted to string.', key) + self.init_hook = None + self.forward_hook = None + + def __deepcopy__(self, memodict=None): + raise NotImplementedError("Deep copy doesn't work for mutables.") + + def set_mutator(self, mutator): + if hasattr(self, 'mutator'): + raise RuntimeError('`set_mutator is called more than once. ' + 'Did you parse the search space multiple times? ' + 'Or did you apply multiple fixed architectures?') + self.mutator = mutator + + def call(self, *inputs): + raise NotImplementedError('Method `call` of Mutable must be overridden') + + def build(self, input_shape): + self._check_built() + + @property + def key(self): + return self._key + + @property + def name(self): + return self._name if hasattr(self, '_name') else self._key + + @name.setter + def name(self, name): + self._name = name + + def _check_built(self): + if not hasattr(self, 'mutator'): + raise ValueError( + "Mutator not set for {}. You might have forgotten to initialize and apply your mutator. " + "Or did you initialize a mutable on the fly in forward pass? Move to `__init__` " + "so that trainer can locate all your mutables. See NNI docs for more details.".format(self)) + + def __repr__(self): + return '{} ({})'.format(self.name, self.key) + + +class MutableScope(Mutable): + def __call__(self, *args, **kwargs): + try: + self.mutator.enter_mutable_scope(self) + return super().__call__(*args, **kwargs) + finally: + self.mutator.exit_mutable_scope(self) + + +class LayerChoice(Mutable): + def __init__(self, op_candidates, reduction='sum', return_mask=False, key=None): + super().__init__(key=key) + self.names = [] + if isinstance(op_candidates, OrderedDict): + for name in op_candidates: + assert name not in ["length", "reduction", "return_mask", "_key", "key", "names"], \ + "Please don't use a reserved name '{}' for your module.".format(name) + self.names.append(name) + elif isinstance(op_candidates, list): + for i, _ in enumerate(op_candidates): + self.names.append(str(i)) + else: + raise TypeError("Unsupported op_candidates type: {}".format(type(op_candidates))) + + self.length = len(op_candidates) + self.choices = op_candidates + self.reduction = reduction + self.return_mask = return_mask + + def call(self, *inputs): + out, mask = self.mutator.on_forward_layer_choice(self, *inputs) + if self.return_mask: + return out, mask + return out + + def build(self, input_shape): + self._check_built() + for op in self.choices: + op.build(input_shape) + + def __len__(self): + return len(self.choices) + + +class InputChoice(Mutable): + NO_KEY = '' + + def __init__(self, n_candidates=None, choose_from=None, n_chosen=None, reduction='sum', return_mask=False, key=None): + super().__init__(key=key) + assert n_candidates is not None or choose_from is not None, \ + 'At least one of `n_candidates` and `choose_from` must be not None.' + if choose_from is not None and n_candidates is None: + n_candidates = len(choose_from) + elif choose_from is None and n_candidates is not None: + choose_from = [self.NO_KEY] * n_candidates + assert n_candidates == len(choose_from), 'Number of candidates must be equal to the length of `choose_from`.' + assert n_candidates > 0, 'Number of candidates must be greater than 0.' + assert n_chosen is None or 0 <= n_chosen <= n_candidates, \ + 'Expected selected number must be None or no more than number of candidates.' + + self.n_candidates = n_candidates + self.choose_from = choose_from.copy() + self.n_chosen = n_chosen + self.reduction = reduction + self.return_mask = return_mask + + def call(self, optional_inputs): + optional_input_list = optional_inputs + if isinstance(optional_inputs, dict): + optional_input_list = [optional_inputs[tag] for tag in self.choose_from] + assert isinstance(optional_input_list, list), \ + 'Optional input list must be a list, not a {}.'.format(type(optional_input_list)) + assert len(optional_inputs) == self.n_candidates, \ + 'Length of the input list must be equal to number of candidates.' + out, mask = self.mutator.on_forward_input_choice(self, optional_input_list) + if self.return_mask: + return out, mask + return out diff --git a/new_impl/cv/third_party/nni_new/nas/tensorflow/mutator.py b/new_impl/cv/third_party/nni_new/nas/tensorflow/mutator.py new file mode 100644 index 0000000000000000000000000000000000000000..b0d2aed684e289b556ccc1388b95477d55c5c2da --- /dev/null +++ b/new_impl/cv/third_party/nni_new/nas/tensorflow/mutator.py @@ -0,0 +1,83 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging + +import tensorflow as tf + +from .base_mutator import BaseMutator + + +_logger = logging.getLogger(__name__) + + +class Mutator(BaseMutator): + def __init__(self, model): + super().__init__(model) + self._cache = {} + + def sample_search(self): + raise NotImplementedError('Method `sample_search` must be overridden') + + def sample_final(self): + raise NotImplementedError('Method `sample_final` must be overriden for exporting') + + def reset(self): + self._cache = self.sample_search() + + def export(self): + return self.sample_final() + + # TODO: status + # TODO: graph + + def on_forward_layer_choice(self, mutable, *inputs): + mask = self._get_decision(mutable) + assert len(mask) == len(mutable), \ + 'Invalid mask, expected {} to be of length {}.'.format(mask, len(mutable)) + out = self._select_with_mask(lambda choice: choice(*inputs), mutable.choices, mask) + return self._tensor_reduction(mutable.reduction, out), mask + + def on_forward_input_choice(self, mutable, tensor_list): + mask = self._get_decision(mutable) + assert len(mask) == mutable.n_candidates, \ + 'Invalid mask, expected {} to be of length {}.'.format(mask, mutable.n_candidates) + out = self._select_with_mask(lambda tensor: tensor, tensor_list, mask) + return self._tensor_reduction(mutable.reduction, out), mask + + def _select_with_mask(self, map_fn, candidates, mask): + if mask.dtype.is_bool: + out = [map_fn(cand) for cand, m in zip(candidates, mask) if m] + elif mask.dtype.is_floating: + out = [map_fn(cand) * m for cand, m in zip(candidates, mask) if m] + else: + raise ValueError('Unrecognized mask, dtype is {}'.format(mask.dtype.name)) + return out + + def _tensor_reduction(self, reduction_type, tensor_list): + if reduction_type == 'none': + return tensor_list + if not tensor_list: + return None + if len(tensor_list) == 1: + return tensor_list[0] + if reduction_type == 'sum': + return sum(tensor_list) + if reduction_type == 'mean': + return sum(tensor_list) / len(tensor_list) + if reduction_type == 'concat': + image_data_format = tf.keras.backend.image_data_format() + if image_data_format == "channels_first": + axis = 0 + else: + axis = -1 + return tf.concat(tensor_list, axis=axis) # pylint: disable=E1120,E1123 + # pylint issue #3613 + raise ValueError('Unrecognized reduction policy: "{}'.format(reduction_type)) + + def _get_decision(self, mutable): + if mutable.key not in self._cache: + raise ValueError('"{}" not found in decision cache.'.format(mutable.key)) + result = self._cache[mutable.key] + _logger.debug('Decision %s: %s', mutable.key, result) + return result diff --git a/new_impl/cv/third_party/nni_new/nas/tensorflow/utils.py b/new_impl/cv/third_party/nni_new/nas/tensorflow/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..0cfc6e815d973774a543e23078692189bfbb90d0 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/nas/tensorflow/utils.py @@ -0,0 +1,93 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import tensorflow as tf + +_counter = 0 + +def global_mutable_counting(): + global _counter + _counter += 1 + return _counter + + +class AverageMeter: + def __init__(self, name): + self.name = name + self.val = 0 + self.avg = 0 + self.sum = 0 + self.count = 0 + + def update(self, val): + self.val = val + self.sum += val + self.count += 1 + self.avg = self.sum / self.count + + def __str__(self): + return '{name} {val:4f} ({avg:4f})'.format(**self.__dict__) + + def summary(self): + return '{name}: {avg:4f}'.format(**self.__dict__) + + +class AverageMeterGroup: + def __init__(self): + self.meters = {} + + def update(self, data): + for k, v in data.items(): + if k not in self.meters: + self.meters[k] = AverageMeter(k) + self.meters[k].update(v) + + def __str__(self): + return ' '.join(str(v) for v in self.meters.values()) + + def summary(self): + return ' '.join(v.summary() for v in self.meters.values()) + + +class StructuredMutableTreeNode: + def __init__(self, mutable): + self.mutable = mutable + self.children = [] + + def add_child(self, mutable): + self.children.append(StructuredMutableTreeNode(mutable)) + return self.children[-1] + + def type(self): + return type(self.mutable) + + def __iter__(self): + return self.traverse() + + def traverse(self, order="pre", deduplicate=True, memo=None): + if memo is None: + memo = set() + assert order in ["pre", "post"] + if order == "pre": + if self.mutable is not None: + if not deduplicate or self.mutable.key not in memo: + memo.add(self.mutable.key) + yield self.mutable + for child in self.children: + for m in child.traverse(order=order, deduplicate=deduplicate, memo=memo): + yield m + if order == "post": + if self.mutable is not None: + if not deduplicate or self.mutable.key not in memo: + memo.add(self.mutable.key) + yield self.mutable + + +def fill_zero_grads(grads, weights): + ret = [] + for grad, weight in zip(grads, weights): + if grad is not None: + ret.append(grad) + else: + ret.append(tf.zeros_like(weight)) + return ret diff --git a/new_impl/cv/third_party/nni_new/parameter_expressions.py b/new_impl/cv/third_party/nni_new/parameter_expressions.py new file mode 100644 index 0000000000000000000000000000000000000000..adff923f7ea1cb4debe114c22376522adb8d8a6d --- /dev/null +++ b/new_impl/cv/third_party/nni_new/parameter_expressions.py @@ -0,0 +1,108 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +''' +parameter_expression.py +''' + +import numpy as np + + +def choice(options, random_state): + ''' + options: 1-D array-like or int + random_state: an object of numpy.random.RandomState + ''' + return random_state.choice(options) + + +def randint(lower, upper, random_state): + ''' + Generate a random integer from `lower` (inclusive) to `upper` (exclusive). + lower: an int that represent an lower bound + upper: an int that represent an upper bound + random_state: an object of numpy.random.RandomState + ''' + return random_state.randint(lower, upper) + + +def uniform(low, high, random_state): + ''' + low: an float that represent an lower bound + high: an float that represent an upper bound + random_state: an object of numpy.random.RandomState + ''' + assert high >= low, 'Upper bound must be larger than lower bound' + return random_state.uniform(low, high) + + +def quniform(low, high, q, random_state): + ''' + low: an float that represent an lower bound + high: an float that represent an upper bound + q: sample step + random_state: an object of numpy.random.RandomState + ''' + return np.clip(np.round(uniform(low, high, random_state) / q) * q, low, high) + + +def loguniform(low, high, random_state): + ''' + low: an float that represent an lower bound + high: an float that represent an upper bound + random_state: an object of numpy.random.RandomState + ''' + assert low > 0, 'Lower bound must be positive' + return np.exp(uniform(np.log(low), np.log(high), random_state)) + + +def qloguniform(low, high, q, random_state): + ''' + low: an float that represent an lower bound + high: an float that represent an upper bound + q: sample step + random_state: an object of numpy.random.RandomState + ''' + return np.clip(np.round(loguniform(low, high, random_state) / q) * q, low, high) + + +def normal(mu, sigma, random_state): + ''' + The probability density function of the normal distribution, + first derived by De Moivre and 200 years later by both Gauss and Laplace independently. + mu: float or array_like of floats + Mean (“centre”) of the distribution. + sigma: float or array_like of floats + Standard deviation (spread or “width”) of the distribution. + random_state: an object of numpy.random.RandomState + ''' + return random_state.normal(mu, sigma) + + +def qnormal(mu, sigma, q, random_state): + ''' + mu: float or array_like of floats + sigma: float or array_like of floats + q: sample step + random_state: an object of numpy.random.RandomState + ''' + return np.round(normal(mu, sigma, random_state) / q) * q + + +def lognormal(mu, sigma, random_state): + ''' + mu: float or array_like of floats + sigma: float or array_like of floats + random_state: an object of numpy.random.RandomState + ''' + return np.exp(normal(mu, sigma, random_state)) + + +def qlognormal(mu, sigma, q, random_state): + ''' + mu: float or array_like of floats + sigma: float or array_like of floats + q: sample step + random_state: an object of numpy.random.RandomState + ''' + return np.round(lognormal(mu, sigma, random_state) / q) * q diff --git a/new_impl/cv/third_party/nni_new/recoverable.py b/new_impl/cv/third_party/nni_new/recoverable.py new file mode 100644 index 0000000000000000000000000000000000000000..70f11e634dc74886c0d379ca7fc86202654bf6ad --- /dev/null +++ b/new_impl/cv/third_party/nni_new/recoverable.py @@ -0,0 +1,18 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os + +class Recoverable: + + def load_checkpoint(self): + pass + + def save_checkpoint(self): + pass + + def get_checkpoint_path(self): + ckp_path = os.getenv('NNI_CHECKPOINT_DIRECTORY') + if ckp_path is not None and os.path.isdir(ckp_path): + return ckp_path + return None diff --git a/new_impl/cv/third_party/nni_new/retiarii/__init__.py b/new_impl/cv/third_party/nni_new/retiarii/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..f4413674600d75345d505d3c4bc8da8a1f91f419 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/retiarii/__init__.py @@ -0,0 +1,8 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .operation import Operation +from .graph import * +from .execution import * +from .mutator import * +from .serializer import basic_unit, json_dump, json_dumps, json_load, json_loads, serialize, serialize_cls, model_wrapper diff --git a/new_impl/cv/third_party/nni_new/retiarii/codegen/__init__.py b/new_impl/cv/third_party/nni_new/retiarii/codegen/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..52f3abc6366f6a48339afa654948af0e651e3287 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/retiarii/codegen/__init__.py @@ -0,0 +1 @@ +from .pytorch import model_to_pytorch_script diff --git a/new_impl/cv/third_party/nni_new/retiarii/codegen/pytorch.py b/new_impl/cv/third_party/nni_new/retiarii/codegen/pytorch.py new file mode 100644 index 0000000000000000000000000000000000000000..17dc45e3a5f6e0ac6dec69472a55238b46623895 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/retiarii/codegen/pytorch.py @@ -0,0 +1,181 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +from typing import List, Tuple, Any + +from ..graph import IllegalGraphError, Edge, Graph, Node, Model + +_logger = logging.getLogger(__name__) + + +def model_to_pytorch_script(model: Model, placement=None) -> str: + graphs = [] + total_pkgs = set() + for name, cell in model.graphs.items(): + import_pkgs, graph_code = graph_to_pytorch_model(name, cell, placement=placement) + graphs.append(graph_code) + total_pkgs.update(import_pkgs) + pkgs_code = '\n'.join(['import {}'.format(pkg) for pkg in total_pkgs]) + return _PyTorchScriptTemplate.format(pkgs_code, '\n\n'.join(graphs)).strip() + + +def _sorted_incoming_edges(node: Node) -> List[Edge]: + edges = [edge for edge in node.graph.edges if edge.tail is node] + _logger.debug('sorted_incoming_edges: %s', str(edges)) + if not edges: + return [] + _logger.debug('all tail_slots are None: %s', str([edge.tail_slot for edge in edges])) + if all(edge.tail_slot is None for edge in edges): + return edges + if all(isinstance(edge.tail_slot, int) for edge in edges): + edges = sorted(edges, key=(lambda edge: edge.tail_slot)) + if [edge.tail_slot for edge in edges] == list(range(len(edges))): + return edges + raise IllegalGraphError(node.graph, 'Node {} has bad inputs'.format(node.name)) + + +def _format_inputs(node: Node) -> Tuple[List[str], List[Any]]: + """ + Format the inputs of a given node + + Parameters + ---------- + node : Node + a graph node, get and format its inputs + + Returns + ------- + list + the list of input names + list + the list of input values, if an input is simple type, record its value, + otherwise the value is None + """ + edges = _sorted_incoming_edges(node) + inputs = [] + inputs_value = [] + for edge in edges: + if edge.head.name == '_inputs': + assert isinstance(edge.head_slot, int) + if edge.head.operation.io_names is not None: + # when input has names, e.g., forward(self, tensor1, tensor2, another_one) + inputs.append(edge.head.operation.io_names[edge.head_slot]) + else: + # when input has no name, e.g., forward(*_inputs) + inputs.append('_inputs[{}]'.format(edge.head_slot)) + inputs_value.append(None) + else: + if edge.head_slot is None: + # when the input comes from a single-output operator + inputs.append('{}'.format(edge.head.name)) + if edge.head.operation.type in ('prim::Constant', 'prim::GetAttr') and \ + 'value' in edge.head.operation.parameters: + inputs_value.append(edge.head.operation.parameters['value']) + else: + inputs_value.append(None) + else: + # when the input comes from a multi-output operator: needs to know which one it comes from + inputs.append('{}[{}]'.format(edge.head.name, edge.head_slot)) + inputs_value.append(None) + return inputs, inputs_value + + +def _remove_prefix(names, graph_name): + """ + variables name (full name space) is too long, + shorten the name by removing the prefix ```graph_name``` + """ + if isinstance(names, list): + converted_names = [] + for name in names: + if name.startswith(graph_name): + converted_names.append(name[len(graph_name):]) + else: + converted_names.append(name) + return converted_names + else: + return names[len(graph_name):] if names.startswith(graph_name) else names + + +def graph_to_pytorch_model(graph_name: str, graph: Graph, placement=None) -> str: + nodes = graph.topo_sort() + + # handle module node and function node differently + # only need to generate code for module here + import_pkgs = set() + node_codes = [] + for node in nodes: + if node.operation: + if node.operation.type == 'shared': + continue + pkg_name = node.operation.get_import_pkg() + if pkg_name is not None: + import_pkgs.add(pkg_name) + node_code = node.operation.to_init_code(_remove_prefix(node.name, graph_name)) + if node_code is not None: + if placement and node in placement and len(node_code) > 0: + node_codes.append(f"{node_code}.to('{placement[node].device}')") + else: + node_codes.append(node_code) + + if graph.input_node.operation.io_names is None: + input_code = '*_inputs' + else: + for name in graph.input_node.operation.io_names: + assert not name.startswith(graph_name) + input_code = ', '.join(graph.input_node.operation.io_names) + + edge_codes = [] + sorted_nodes = graph.topo_sort() + for node in sorted_nodes: + if node.operation: + inputs, inputs_value = _format_inputs(node) + inputs = _remove_prefix(inputs, graph_name) + node_name = _remove_prefix(node.name, graph_name) + submodule_name = node_name + if node.operation.type == 'shared': + submodule_name = _remove_prefix(node.operation.parameters['reference'], graph_name) + edge_codes.append(node.operation.to_forward_code(submodule_name, node_name, inputs, inputs_value)) + + output_names, _ = _format_inputs(graph.output_node) + output_names = _remove_prefix(output_names, graph_name) + if not output_names: + raise RuntimeError('"forward" function should have return value(s): {}, {}, {}'.format(output_names, graph_name, graph.output_node)) + output_code = ', '.join(output_names) + + linebreak = '\n ' + return import_pkgs, _PyTorchModelTemplate.format( + graph_name=('Graph' if graph_name == '_graph' else graph_name), + inputs=input_code, + outputs=output_code, + nodes=linebreak.join(node_codes), + edges=linebreak.join(edge_codes) + ) + + +# TODO: handle imports + +_PyTorchScriptTemplate = ''' +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.optim as optim + +import nni.retiarii.nn.pytorch + +{} + +{} +''' + +_PyTorchModelTemplate = ''' +class {graph_name}(nn.Module): + def __init__(self): + super().__init__() + {nodes} + + def forward(self, {inputs}): + {edges} + return {outputs} +''' diff --git a/new_impl/cv/third_party/nni_new/retiarii/codegen/tensorflow.py b/new_impl/cv/third_party/nni_new/retiarii/codegen/tensorflow.py new file mode 100644 index 0000000000000000000000000000000000000000..ac0e0d7003599560a0222b3675cc0bc7374a4694 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/retiarii/codegen/tensorflow.py @@ -0,0 +1,109 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +# pylint: skip-file + +""" +FIXME +This file is inherited from last version. + +I expect it can work with a few modifications to incorporate with the latest API, but it hasn't +been tested and I'm not sure. +""" + +from ..graph_v2 import IllegalGraphError, Cell, Edge, Graph, Node +from ..operations_tf import Operation +from ..type_utils import * + + +def graph_to_tensorflow_script(graph: Graph) -> str: + graphs = [graph_to_tensorflow_model(name, cell) for name, cell in graph.cell_templates.items()] + return _TensorFlowScriptTemplate.format('\n\n'.join(graphs)).strip() + + +def _sort_incoming_edges(node: Node) -> List[Edge]: + edges = [edge for edge in node.graph.edges if edge.tail is node] + if not edges: + return [] + if all(edge.tail_idx is None for edge in edges): + return edges + if all(isinstance(edge.tail_idx, int) for edge in edges): + edges = sorted(edges, key=(lambda edge: edge.tail_idx)) + if [edge.tail_idx for edge in edges] == list(range(len(edges))): + return edges + raise IllegalGraphError(node.graph, 'Node {} has bad inputs'.format(node.name)) + +def _format_inputs(node: Node) -> str: + edges = _sort_incoming_edges(node) + inputs = [] + for edge in edges: + if edge.head.name == '_inputs': + assert isinstance(edge.head_idx, int) + if node.graph.input_names is not None: + inputs.append(node.graph.input_names[edge.head_idx]) + else: + inputs.append('_inputs[{}]'.format(edge.head_idx)) + else: + if edge.head_idx is None: + inputs.append('{}'.format(edge.head.name)) + else: + inputs.append('{}[{}]'.format(edge.head.name, edge.head_idx)) + return ', '.join(inputs) + + +def graph_to_tensorflow_model(graph_name: str, graph: Graph) -> str: + nodes = graph.topo_sort() + + # handle module node and function node differently + # only need to generate code for module here + node_codes = [] + for node in nodes: + if isinstance(node, Cell): + node_codes.append('self.{} = {}()'.format(node.name, node.template_name)) + else: + node_codes.append('self.{} = {}'.format(node.name, cast(Operation, node.operation).to_tensorflow_init())) + + edge_codes = [] + + for node in nodes: + inputs = _format_inputs(node) + edge_codes.append('{} = self.{}({})'.format(node.name, node.name, inputs)) + + output_code = _format_inputs(graph.output_node) + if not output_code: + output_code = 'None' + + if graph.input_names is None: + input_code = '*_inputs' + else: + input_code = ', '.join(graph.input_names) + + linebreak = '\n ' + return _TensorFlowModelTemplate.format( + graph_name=('Graph' if graph_name == '_graph' else graph_name), + inputs=input_code, + outputs=output_code, + nodes=linebreak.join(node_codes), + edges=linebreak.join(edge_codes) + ) + + +_TensorFlowScriptTemplate = ''' +import tensorflow as tf +import tensorflow.keras as K + +import sdk.custom_ops_tf as CUSTOM + +{} +''' + +_TensorFlowModelTemplate = ''' +class {graph_name}(K.Model): + def __init__(self): + super().__init__() + {nodes} + + def call(self, {inputs}): + {edges} + return {outputs} +''' \ No newline at end of file diff --git a/new_impl/cv/third_party/nni_new/retiarii/converter/README.md b/new_impl/cv/third_party/nni_new/retiarii/converter/README.md new file mode 100644 index 0000000000000000000000000000000000000000..d0f19066b1ac0ed367cd5d8bd488984288f08315 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/retiarii/converter/README.md @@ -0,0 +1,37 @@ +# PyTorch Graph Converter + +## Namespace for PyTorch Graph + +We should have a concrete rule for specifying nodes in graph with namespace. + +Each node has a name, either specified or generated. The nodes in the same hierarchy cannot have the same name. + +* The name of module node natively follows this rule, because we use variable name for instantiated modules like what PyTorch graph does. + +* For the nodes created in `forward` function, we use a global sequence number. + +### Namespace for mutated (new) nodes + +TBD + +## Graph Simplification + +TBD + +## Node Types + +We define concrete type string for each node type. + +## Module's Input Arguments + +We use wrapper to obtain the input arguments of modules. Users need to use our wrapped "nn" and wrapped "Module". + +## Control Flow + +### for loop + +Currently, we only support `ModuleList` (`ModuleDict`) based for loop, which is automatically unfolded by TorchScript. That is to say, we do not support loop in TorchScript for now. + +### if/else + +For now, we only deal with the case that the condition is constant or attribute. In this case, only one branch is kept during generating the graph. \ No newline at end of file diff --git a/new_impl/cv/third_party/nni_new/retiarii/converter/__init__.py b/new_impl/cv/third_party/nni_new/retiarii/converter/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e0fff09f2da1af2f3904ae9ab40939498030a295 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/retiarii/converter/__init__.py @@ -0,0 +1 @@ +from .graph_gen import convert_to_graph diff --git a/new_impl/cv/third_party/nni_new/retiarii/converter/graph_gen.py b/new_impl/cv/third_party/nni_new/retiarii/converter/graph_gen.py new file mode 100644 index 0000000000000000000000000000000000000000..f8b06b887a6772a4d0d77e728c85aa120bbec2c1 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/retiarii/converter/graph_gen.py @@ -0,0 +1,679 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import re + +import torch + +from ..graph import Graph, Model, Node +from ..nn.pytorch import InputChoice, Placeholder +from ..operation import Cell, Operation +from ..serializer import get_init_parameters_or_fail +from ..utils import get_importable_name +from .op_types import MODULE_EXCEPT_LIST, OpTypeName +from .utils import _convert_name, build_full_name + + +class GraphConverter: + def __init__(self): + self.global_seq = 0 + self.global_graph_id = 0 + + def _add_edge_handle_source_node(self, _input, graph_inputs, ir_graph, output_remap, node_index): + if _input in output_remap: + assert output_remap[_input].kind() == 'aten::append' + predecessor_node = output_remap[_input] + assert predecessor_node in node_index, 'predecessor node: {}'.format(predecessor_node) + src_node_idx = None + src_node = node_index[predecessor_node] + assert isinstance(src_node, Node) + elif _input in graph_inputs: + idx = graph_inputs.index(_input) + src_node = ir_graph.input_node + src_node_idx = idx + else: + predecessor_node = _input.node() + assert predecessor_node in node_index, 'predecessor node: {}'.format(predecessor_node) + # find out the index of _input in the outputs of predecessor_node + predecessor_outputs = [_output for _output in predecessor_node.outputs()] + if len(predecessor_outputs) == 1: + idx = None + else: + idx = predecessor_outputs.index(_input) + ir_predecessor_node = node_index[predecessor_node] + src_node_idx = idx + assert isinstance(ir_predecessor_node, Node) + src_node = ir_predecessor_node + return src_node, src_node_idx + + def _add_edge(self, ir_graph, node, graph_inputs, node_index, new_node, output_remap, ignore_first=False): + """ + Parameters + ---------- + ir_graph : Graph + node : torch._C.Node + graph_inputs : List[torch._C.Value] + a list of a script graph's inputs + node_index : Dict + new_node : Node + newly created ir node corresponding to `node` + output_remap : Dict + ignore_first : bool + if it is true, skip the first input + """ + is_single_input = (len([_input for _input in node.inputs()]) - (1 if ignore_first else 0)) == 1 + new_node_input_idx = 0 + for _input in node.inputs(): + if ignore_first: + ignore_first = False + continue + # handle source node + src_node, src_node_idx = self._add_edge_handle_source_node(_input, graph_inputs, ir_graph, output_remap, node_index) + # handle destination node + dst_node = new_node + if is_single_input: + dst_node_idx = None + else: + dst_node_idx = new_node_input_idx + # create edge + ir_graph.add_edge(head=(src_node, src_node_idx), tail=(dst_node, dst_node_idx)) + + new_node_input_idx += 1 + + def create_prim_constant_node(self, ir_graph, node, module_name): + # NOTE: compare with string not type, because the type is defined in pytorch C code. + # `.kind()` can also be used here + if node.outputsAt(0).type().str() == 'None': + attrs = {'type': 'None'} + else: + attrs = {'type': node.outputsAt(0).type().str(), 'value': node.outputsAt(0).toIValue()} + self.global_seq += 1 + new_node = ir_graph.add_node(build_full_name(module_name, OpTypeName.Constant, self.global_seq), + node.kind(), attrs) + return new_node + + def handle_prim_attr_node(self, node, module): + assert node.hasAttribute('name') + value = None + if node.inputsAt(0).debugName() == 'self': + _val = getattr(module, node.s('name')) + # TODO: serialize complex data type, and output proper error message + if isinstance(_val, (int, float, str, bool)): + value = _val + attrs = {'name': node.s('name'), 'input': node.inputsAt(0).debugName(), 'value': value} + return node.kind(), attrs + + def _remove_mangle(self, module_type_str): + return re.sub('\\.___torch_mangle_\\d+', '', module_type_str) + + def remove_unconnected_nodes(self, ir_graph, targeted_type=None): + """ + Parameters + ---------- + ir_graph : Graph + our ir graph representation + targeted_type : str + nodes with ```targeted_type``` will be removed from graph if their fanout is 0. + ```None``` means removing all the nodes whose fanout is 0. + """ + # build index of outputs of Node(s) + node_fanout = set() + for edge in ir_graph.edges: + if edge.head.id not in node_fanout: + node_fanout.add(edge.head.id) + + to_removes = [] + for hidden_node in ir_graph.hidden_nodes: + if hidden_node.id not in node_fanout: + assert isinstance(hidden_node, Node) + if targeted_type is None: + to_removes.append(hidden_node) + elif hidden_node.operation.type == targeted_type: + to_removes.append(hidden_node) + + for hidden_node in to_removes: + hidden_node.remove() + + def handle_graph_nodes(self, script_module, sm_graph, + module, module_name, + ir_model, ir_graph, + shared_module_index=None): + """ + Convert torch script node to our node ir, and build our graph ir + + Parameters + ---------- + script_module : torch.jit.RecursiveScriptModule + the torch script of ```module``` + sm_graph : torch._C.Graph + the graph in torch script + module : nn.Module + the targeted pytorch module + module_name : str + ```module```'s name + ir_model : Model + the whole graph ir + ir_graph : Graph + the graph ir of ```module``` + shared_module_index : dict + it is used for knowing which module has been created an ir node, + if created and invoked again, then the new ir node can simply reference that ir node. + this way we can identify shared modules (i.e., one module invoked multiple times in `forward` function) + + Returns + ------- + dict + the mapping from graph node to our graph ir node + """ + # handle inputs + graph_inputs = [] + for _input in sm_graph.inputs(): + if _input.debugName() == 'self': + assert _input.unique() == 0 + continue + graph_inputs.append(_input) + # TODO: add scope name + ir_graph._add_input(_convert_name(_input.debugName())) + + node_index = {} # graph node to graph ir node + if shared_module_index is None: + shared_module_index = {} + + # some node does not have output but it modifies a variable, for example aten::append + # %17 : Tensor[] = aten::append(%out.1, %16) + # %out.1 is updated, and %17 is None + # we add output to this type of node and connect it to the following node which uses %out.1 + # key: tensor (%out.1), value: node (this node) + output_remap = {} + + # ===================handle control flow: if=================== + def handle_if_condition(cond_tensor): + """ + to calculate the condition, we only deal with the following op types by tracing back + `prim::GetAttr`, `aten::__getitem__`, `prim::Constant`, `aten::eq` + + generate the expression using recursive calls + + NOTE: do not support dynamic graph + """ + def _generate_expr(tensor): + if tensor.node().kind() == 'prim::GetAttr': + return f'({getattr(module, tensor.node().s("name"))})' + elif tensor.node().kind() == 'aten::__getitem__': + t = _generate_expr(tensor.node().inputsAt(0)) + idx = _generate_expr(tensor.node().inputsAt(1)) + return f'({t}[{idx}])' + elif tensor.node().kind() == 'prim::Constant': + return f'{tensor.toIValue()}' + elif tensor.node().kind() == 'aten::eq': + left = _generate_expr(tensor.node().inputsAt(0)) + right = _generate_expr(tensor.node().inputsAt(1)) + return f'({left} == {right})' + elif tensor.node().kind() == 'aten::le': + left = _generate_expr(tensor.node().inputsAt(0)) + right = _generate_expr(tensor.node().inputsAt(1)) + return f'({left} <= {right})' + elif tensor.node().kind() == 'aten::ge': + left = _generate_expr(tensor.node().inputsAt(0)) + right = _generate_expr(tensor.node().inputsAt(1)) + return f'({left} >= {right})' + elif tensor.node().kind() == 'aten::__not__': + value = _generate_expr(tensor.node().inputsAt(0)) + return f'(not {value})' + elif tensor.node().kind() == 'aten::Bool': + value = _generate_expr(tensor.node().inputsAt(0)) + return f'bool({value})' + elif tensor.node().kind() == 'aten::__is__': + left = _generate_expr(tensor.node().inputsAt(0)) + right = _generate_expr(tensor.node().inputsAt(1)) + return f'({left} is {right})' + elif tensor.node().kind() == 'aten::__isnot__': + left = _generate_expr(tensor.node().inputsAt(0)) + right = _generate_expr(tensor.node().inputsAt(1)) + return f'({left} is not {right})' + elif tensor.node().kind() == 'aten::ne': + left = _generate_expr(tensor.node().inputsAt(0)) + right = _generate_expr(tensor.node().inputsAt(1)) + return f'({left} != {right})' + elif tensor.node().kind() == 'aten::gt': + left = _generate_expr(tensor.node().inputsAt(0)) + right = _generate_expr(tensor.node().inputsAt(1)) + return f'({left} > {right})' + elif tensor.node().kind() == 'aten::lt': + left = _generate_expr(tensor.node().inputsAt(0)) + right = _generate_expr(tensor.node().inputsAt(1)) + return f'({left} < {right})' + elif tensor.node().kind() == 'prim::If': + raise RuntimeError('Have not supported `if A and/or B`, please use two `if` statements instead.') + else: + raise RuntimeError(f'Unsupported op type {tensor.node().kind()} in if condition, ' + 'you are suggested to decorate the corresponding class with "@basic_unit".') + expr = _generate_expr(cond_tensor) + return eval(expr) + + def handle_if_node(node): + """ + Parameters + ---------- + node : torch._C.Node + the node from TorchScript graph + + Returns + ------- + Node + the created node ir + """ + # only deal with input of prim::If is constant or attribute for now + # will support constant expression in future + inputs = [i for i in node.inputs()] + assert len(inputs) == 1 + cond = handle_if_condition(inputs[0]) + chosen_block = 0 if cond else 1 + blocks = [block for block in node.blocks()] + assert len(blocks) == 2 + last_block_node = None + for node in blocks[chosen_block].nodes(): + last_block_node = handle_single_node(node) + self.global_seq += 1 + new_node = ir_graph.add_node(build_full_name(module_name, 'noop_identity', self.global_seq), 'noop_identity') + self._add_edge(ir_graph, blocks[chosen_block].returnNode(), graph_inputs, node_index, new_node, output_remap) + last_block_node = new_node + return last_block_node + + # ===================handle function call=================== + def handle_function_callmethod(node): + # get and handle the first input, which should be an nn.Module + assert node.hasAttribute('name') + # NOTE: "forward__0" is hacky, LSTM instance is parsed to call forward__0 in torchscript + if node.s('name') in ['forward', 'forward__0']: + # node.inputsAt(0).type() is + submodule_type_str = self._remove_mangle(node.inputsAt(0).type().str()) + submodule = node.inputsAt(0).node() + assert submodule.kind() == 'prim::GetAttr' + assert submodule.hasAttribute('name') + submodule_name = submodule.s('name') + + if submodule.inputsAt(0).debugName() == 'self': + # module is usually instantiated in __init__. + # when calling a module in forward, + # prim::GetAttr is used to obtain the module in torch script. + # therefore, we do this check for a module. example below: + # %25 : __torch__.xxx = prim::GetAttr[name="input_switch"](%self) + # %27 : Tensor = prim::CallMethod[name="forward"](%25, %out.1) + assert submodule_name in script_module._modules, "submodule_name: {} not in script_module {}".format( + submodule_name, script_module._modules.keys()) + + submodule_full_name = build_full_name(module_name, submodule_name) + submodule_obj = getattr(module, submodule_name) + subgraph, sub_m_attrs = self.convert_module(script_module._modules[submodule_name], + submodule_obj, + submodule_full_name, ir_model) + else: + # %8 : __torch__.nni.retiarii.model_apis.nn.___torch_mangle_37.ModuleList = prim::GetAttr[name="cells"](%self) + # %10 : __torch__.darts_model.Cell = prim::GetAttr[name="0"](%8) + # %s1.4 : Tensor = prim::CallMethod[name="forward"](%10, %4, %4) + if submodule.inputsAt(0).type().name() == 'ModuleList': + # handle ModuleList + predecessor = submodule.inputsAt(0).node() + module_name_space = [submodule_name] + while predecessor.inputsAt(0).debugName() != 'self': + # this is for dealing with nested ModuleList. below is an example + # %3 : __torch__.torch.nn.modules.container.___torch_mangle_0.ModuleList = prim::GetAttr[name="ops"](%self) + # %5 : __torch__.torch.nn.modules.container.ModuleList = prim::GetAttr[name="0"](%3) + # %7 : __torch__.torch.nn.modules.container.ModuleList = prim::GetAttr[name="1"](%3) + # %9 : __torch__.torch.nn.modules.container.ModuleList = prim::GetAttr[name="2"](%3) + # %11 : __torch__.torch.nn.modules.container.ModuleList = prim::GetAttr[name="3"](%3) + # %14 : __torch__.torch.nn.modules.linear.Linear = prim::GetAttr[name="0"](%5) + # %16 : __torch__.torch.nn.modules.linear.Linear = prim::GetAttr[name="1"](%5) + # %state.2 : Tensor = prim::CallMethod[name="forward"](%14, %x.1) # modulelist.py:18:24 + # %state.4 : Tensor = prim::CallMethod[name="forward"](%16, %state.2) # modulelist.py:18:24 + assert predecessor.kind() == 'prim::GetAttr' + module_name_space.append(predecessor.s('name')) + predecessor = predecessor.inputsAt(0).node() + assert predecessor.kind() == 'prim::GetAttr' + assert predecessor.hasAttribute('name') + module_name_space.append(predecessor.s('name')) + submodule_full_name = build_full_name(module_name, list(reversed(module_name_space))) + submodule_obj = module + script_submodule = script_module + for each_name in list(reversed(module_name_space)): + submodule_obj = getattr(submodule_obj, each_name) + script_submodule = script_submodule._modules[each_name] + subgraph, sub_m_attrs = self.convert_module(script_submodule, submodule_obj, submodule_full_name, ir_model) + else: + raise RuntimeError('Unsupported module case: {}'.format(submodule.inputsAt(0).type().str())) + + if submodule_full_name in shared_module_index: + # this module is invoked more than once, the ir node has already been created + # create a reference node for it. + # example: {"name": "conv2", "operation": {"type": "shared", "parameters": {"reference": "conv1"}}} + self.global_seq += 1 + shared_node_name = build_full_name(submodule_full_name, '', self.global_seq) + shared_type_operation = Operation.new('shared', {'reference': submodule_full_name}) + subcell = ir_graph.add_node(shared_node_name, shared_type_operation) + else: + # this module is processed for the first time, build cell for it + if subgraph is None: + # if we do not parse this module's graph, we create Node for this module + subcell = ir_graph.add_node(submodule_full_name, submodule_type_str, sub_m_attrs) + if isinstance(submodule_obj, Placeholder): + subcell.update_label(submodule_obj.label) + elif isinstance(submodule_obj, InputChoice): + subcell.update_label(sub_m_attrs['label']) + else: + # Graph already created, create Cell for it + new_cell = Cell(cell_name=submodule_full_name, parameters=sub_m_attrs) + subcell = ir_graph.add_node(submodule_full_name, new_cell) + shared_module_index[submodule_full_name] = subcell + node_index[node] = subcell + # connect the cell into graph + self._add_edge(ir_graph, node, graph_inputs, node_index, subcell, output_remap, ignore_first=True) + else: + # handle normal member function + assert hasattr(script_module, node.s('name')) + # TODO: support non member functions + assert node.inputsAt(0).debugName() == 'self' + script_method = getattr(script_module, node.s('name')) # + + # step #1: generate graph ir for this method + method_ir_graph = Graph(model=ir_model, graph_id=-100, name='temp_graph', _internal=True) + method_node_index = self.handle_graph_nodes(script_module, script_method.graph, module, + module_name, ir_model, method_ir_graph, shared_module_index) + for _output in script_method.graph.outputs(): + method_ir_graph._add_output(_convert_name(_output.debugName())) + predecessor_node_outputs = [o for o in _output.node().outputs()] + if len(predecessor_node_outputs) == 1: + src_node_idx = None + else: + src_node_idx = predecessor_node_outputs.index(_output) + method_ir_graph.add_edge(head=(method_node_index[_output.node()], src_node_idx), + tail=(method_ir_graph.output_node, None)) + self.refine_graph(method_ir_graph) + + # step #2: merge this graph to its module graph + for h_node in method_ir_graph.hidden_nodes: + h_node.graph = ir_graph + ir_graph.hidden_nodes.append(h_node) + for edge in method_ir_graph.edges: + edge.graph = ir_graph + if edge.head == method_ir_graph.input_node: + # this is a member method, 'self' is the first argument, thus +1 + _input = node.inputsAt(edge.head_slot + 1) + src_node, src_node_idx = self._add_edge_handle_source_node(_input, graph_inputs, ir_graph, output_remap, node_index) + edge.head = src_node + edge.head_slot = src_node_idx + if edge.tail == method_ir_graph.output_node: + # since the following nodes have not been created, skip this edge + # edge.head is the output node of this method + # TODO: check whether there could be multiple output nodes??? + node_index[node] = edge.head + continue + ir_graph.edges.append(edge) + + # ===================handle each single node=================== + def handle_single_node(node): + """ + Parameters + ---------- + node : torch._C.Node + the node from TorchScript graph + + Returns + ------- + Node + the created node ir + """ + if node.kind() == 'prim::CallMethod': + handle_function_callmethod(node) + elif node.kind() == 'prim::CallFunction': + func_type_str = self._remove_mangle(node.inputsAt(0).type().str()) + func = node.inputsAt(0).node() + assert func.kind() == 'prim::Constant' + assert func.hasAttribute('name') + func_name = func.s('name') + # create node for func + self.global_seq += 1 + func_node = ir_graph.add_node(build_full_name(module_name, func_name, self.global_seq), + '{}.{}'.format(func_type_str, func_name)) + node_index[node] = func_node + self._add_edge(ir_graph, node, graph_inputs, node_index, func_node, output_remap, ignore_first=True) + elif node.kind() == 'prim::Constant': + new_node = self.create_prim_constant_node(ir_graph, node, module_name) + node_index[node] = new_node + elif node.kind() in ['prim::ListConstruct', 'prim::ListUnpack', 'prim::TupleConstruct', 'prim::TupleUnpack']: + self.global_seq += 1 + prim_op_name = node.kind().split('::')[-1] + new_node = ir_graph.add_node(build_full_name(module_name, prim_op_name, self.global_seq), node.kind()) + node_index[node] = new_node + self._add_edge(ir_graph, node, graph_inputs, node_index, new_node, output_remap) + elif node.kind() == 'prim::GetAttr': + node_type, attrs = self.handle_prim_attr_node(node, module) + self.global_seq += 1 + new_node = ir_graph.add_node(build_full_name(module_name, OpTypeName.Attr, self.global_seq), + node_type, attrs) + node_index[node] = new_node + elif node.kind() == 'prim::If': + last_block_node = handle_if_node(node) + # last_block_node is None means no node in the branch block + node_index[node] = last_block_node + elif node.kind() == 'prim::Loop': + # refer to https://gist.github.com/liuzhe-lz/90c35d9dd6fd7f3f32544940151ab186 + raise RuntimeError('Loop has not been supported yet!') + elif node.kind().startswith('prim::'): + self.global_seq += 1 + prim_op_name = node.kind().replace('::', '__') + prim_node = ir_graph.add_node(build_full_name(module_name, prim_op_name, self.global_seq), node.kind()) + node_index[node] = prim_node + self._add_edge(ir_graph, node, graph_inputs, node_index, prim_node, output_remap) + elif node.kind() == 'aten::append': + self.global_seq += 1 + aten_op_name = node.kind().replace('::', '__') + aten_node = ir_graph.add_node(build_full_name(module_name, aten_op_name, self.global_seq), node.kind()) + node_index[node] = aten_node + self._add_edge(ir_graph, node, graph_inputs, node_index, aten_node, output_remap) + output_remap[node.inputsAt(0)] = node + elif node.kind().startswith('aten::'): + # handle aten::XXX + self.global_seq += 1 + aten_op_name = node.kind().replace('::', '__') + aten_node = ir_graph.add_node(build_full_name(module_name, aten_op_name, self.global_seq), node.kind()) + node_index[node] = aten_node + self._add_edge(ir_graph, node, graph_inputs, node_index, aten_node, output_remap) + else: + raise RuntimeError('Unsupported kind: {}'.format(node.kind())) + + return node_index[node] + + for node in sm_graph.nodes(): + handle_single_node(node) + + return node_index + + def merge_aten_slices(self, ir_graph): + """ + if there is aten::slice node, merge the consecutive ones together. + ```x[:, :, 1:, 1:]``` in python code will be converted into 4 node in torch script, + each node has 5 inputs: tensor, dim, x, y, z (i.e., x:y:z) + """ + head_slice_nodes = [] + has_slice_node = False + for node in ir_graph.hidden_nodes: + if node.operation.type == 'aten::slice': + has_slice_node = True + for pred in node.predecessors: + if pred.operation.type not in ['aten::slice', 'prim::Constant']: + head_slice_nodes.append(node) + break + if has_slice_node: + assert head_slice_nodes + + for head_node in head_slice_nodes: + slot = 0 + new_slice_node = ir_graph.add_node(build_full_name(head_node.name, 'merged'), OpTypeName.MergedSlice) + if len(head_node.incoming_edges) == 4: + # when slice is for one dimension list, there are only 4 inputs, thus merge is not needed + for edge in head_node.incoming_edges: + edge.tail = new_slice_node + for edge in head_node.outgoing_edges: + edge.head = new_slice_node + ir_graph.hidden_nodes.remove(head_node) + break + assert len(head_node.incoming_edges) == 5 + for edge in head_node.incoming_edges: + edge.tail = new_slice_node + slot += 5 + node = head_node + while len(node.successors) == 1 and node.successors[0].operation.type == 'aten::slice': + suc_node = node.successors[0] + assert len(suc_node.incoming_edges) == 5 + for edge in suc_node.incoming_edges: + if edge.tail_slot == 0: + edge.remove() + else: + edge.tail = new_slice_node + edge.tail_slot = slot + edge.tail_slot - 1 + slot += 4 + ir_graph.hidden_nodes.remove(node) + node = suc_node + + for edge in node.outgoing_edges: + edge.head = new_slice_node + ir_graph.hidden_nodes.remove(node) + + def refine_graph(self, ir_graph): + """ + Do the following process to simplify graph: + 1. remove unconnected constant node + 2. remove unconnected getattr node + """ + # some constant is not used, for example, function name as prim::Constant + self.remove_unconnected_nodes(ir_graph, targeted_type='prim::Constant') + self.remove_unconnected_nodes(ir_graph, targeted_type='prim::GetAttr') + self.merge_aten_slices(ir_graph) + + def _handle_inputchoice(self, module): + return { + 'n_candidates': module.n_candidates, + 'n_chosen': module.n_chosen, + 'reduction': module.reduction, + 'label': module.label + } + + def _handle_valuechoice(self, module): + return { + 'candidates': module.candidates, + 'label': module.label, + 'accessor': module._accessor + } + + def convert_module(self, script_module, module, module_name, ir_model): + """ + Convert a module to its graph ir (i.e., Graph) along with its input arguments + + Parameters + ---------- + script_module : torch.jit.RecursiveScriptModule + the script module of ```module``` obtained with torch.jit.script + module : nn.Module + the targeted module instance + module_name : str + the constructed name space of ```module``` + ir_model : Model + the whole graph ir + + Returns + ------- + Graph + the built graph ir from module, ```None``` means do not further parse the module + dict + the input arguments of this module + """ + + # NOTE: have not supported nested LayerChoice, i.e., a candidate module + # also has LayerChoice or InputChoice or ValueChoice + original_type_name = script_module.original_name + m_attrs = None + if original_type_name in MODULE_EXCEPT_LIST: + pass # do nothing + elif original_type_name == OpTypeName.LayerChoice: + graph = Graph(ir_model, -100, module_name, _internal=True) # graph_id is not used now + candidate_name_list = [f'layerchoice_{module.label}_{cand_name}' for cand_name in module.names] + for cand_name, cand in zip(candidate_name_list, module): + cand_type = '__torch__.' + get_importable_name(cand.__class__) + graph.add_node(cand_name, cand_type, get_init_parameters_or_fail(cand)) + graph._register() + return graph, {'mutation': 'layerchoice', 'label': module.label, 'candidates': candidate_name_list} + elif original_type_name == OpTypeName.InputChoice: + m_attrs = self._handle_inputchoice(module) + elif original_type_name == OpTypeName.ValueChoice: + m_attrs = self._handle_valuechoice(module) + elif original_type_name == OpTypeName.Placeholder: + m_attrs = get_init_parameters_or_fail(module) + elif module.__class__.__module__.startswith('torch.nn') and original_type_name in torch.nn.__dict__: + # this is a basic module from pytorch, no need to parse its graph + m_attrs = get_init_parameters_or_fail(module) + else: + # this module is marked as serialize, won't continue to parse + m_attrs = get_init_parameters_or_fail(module, silently=True) + if m_attrs is not None: + return None, m_attrs + + # handle TorchScript graph + sm_graph = script_module.graph + self.global_graph_id += 1 + ir_graph = Graph(model=ir_model, graph_id=self.global_graph_id, name=module_name, _internal=True) + + # handle graph nodes + node_index = self.handle_graph_nodes(script_module, sm_graph, module, + module_name, ir_model, ir_graph) + + # handle graph outputs + for _output in sm_graph.outputs(): + ir_graph._add_output(_convert_name(_output.debugName())) + predecessor_node_outputs = [o for o in _output.node().outputs()] + if len(predecessor_node_outputs) == 1: + src_node_idx = None + else: + src_node_idx = predecessor_node_outputs.index(_output) + ir_graph.add_edge(head=(node_index[_output.node()], src_node_idx), + tail=(ir_graph.output_node, None)) + + self.refine_graph(ir_graph) + + ir_graph._register() + + # add mutation signal for special modules + if original_type_name == OpTypeName.Repeat: + attrs = { + 'mutation': 'repeat', + 'label': module.label, + 'min_depth': module.min_depth, + 'max_depth': module.max_depth + } + return ir_graph, attrs + + return ir_graph, {} + + +def convert_to_graph(script_module, module): + """ + Convert module to our graph ir, i.e., build a ```Model``` type + + Parameters + ---------- + script_module : torch.jit.RecursiveScriptModule + the script module obtained with torch.jit.script + module : nn.Module + the targeted module instance + + Returns + ------- + Model + the constructed IR model + """ + + model = Model(_internal=True) + module_name = '_model' + GraphConverter().convert_module(script_module, module, module_name, model) + + return model diff --git a/new_impl/cv/third_party/nni_new/retiarii/converter/op_types.py b/new_impl/cv/third_party/nni_new/retiarii/converter/op_types.py new file mode 100644 index 0000000000000000000000000000000000000000..1a4ba5a42db8743a154d3cbe490858a480309e93 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/retiarii/converter/op_types.py @@ -0,0 +1,21 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from enum import Enum + +MODULE_EXCEPT_LIST = ['Sequential'] + + +class OpTypeName(str, Enum): + """ + op type to its type name str + """ + Attr = 'Attr' + Constant = 'Constant' + LayerChoice = 'LayerChoice' + InputChoice = 'InputChoice' + ValueChoice = 'ValueChoice' + Placeholder = 'Placeholder' + MergedSlice = 'MergedSlice' + Repeat = 'Repeat' + Cell = 'Cell' diff --git a/new_impl/cv/third_party/nni_new/retiarii/converter/utils.py b/new_impl/cv/third_party/nni_new/retiarii/converter/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..c43f62176ed70024f62d254770cb5c918c18016c --- /dev/null +++ b/new_impl/cv/third_party/nni_new/retiarii/converter/utils.py @@ -0,0 +1,17 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +def build_full_name(prefix, name, seq=None): + if isinstance(name, list): + name = '__'.join(name) + if seq is None: + return '{}__{}'.format(prefix, name) + else: + return '{}__{}{}'.format(prefix, name, str(seq)) + + +def _convert_name(name: str) -> str: + """ + Convert the names using separator '.' to valid variable name in code + """ + return name.replace('.', '__') diff --git a/new_impl/cv/third_party/nni_new/retiarii/converter/visualize.py b/new_impl/cv/third_party/nni_new/retiarii/converter/visualize.py new file mode 100644 index 0000000000000000000000000000000000000000..2bfe299198d65c29d87aadbd1e8a1001c045992c --- /dev/null +++ b/new_impl/cv/third_party/nni_new/retiarii/converter/visualize.py @@ -0,0 +1,44 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import graphviz + + +def convert_to_visualize(graph_ir, vgraph): + for name, graph in graph_ir.items(): + if name == '_evaluator': + continue + with vgraph.subgraph(name='cluster'+name) as subgraph: + subgraph.attr(color='blue') + cell_node = {} + ioput = {'_inputs': '{}-{}'.format(name, '_'.join(graph['inputs'])), + '_outputs': '{}-{}'.format(name, '_'.join(graph['outputs']))} + subgraph.node(ioput['_inputs']) + subgraph.node(ioput['_outputs']) + for node_name, node_value in graph['nodes'].items(): + value = node_value['operation'] + if value['type'] == '_cell': + cell_input_name = '{}-{}'.format(value['cell_name'], '_'.join(graph_ir[value['cell_name']]['inputs'])) + cell_output_name = '{}-{}'.format(value['cell_name'], '_'.join(graph_ir[value['cell_name']]['outputs'])) + cell_node[node_name] = (cell_input_name, cell_output_name) + print('cell: ', node_name, cell_input_name, cell_output_name) + else: + subgraph.node(node_name) + for edge in graph['edges']: + src = edge['head'][0] + if src == '_inputs': + src = ioput['_inputs'] + elif src in cell_node: + src = cell_node[src][1] + dst = edge['tail'][0] + if dst == '_outputs': + dst = ioput['_outputs'] + elif dst in cell_node: + dst = cell_node[dst][0] + subgraph.edge(src, dst) + + +def visualize_model(graph_ir): + vgraph = graphviz.Digraph('G', filename='vgraph', format='jpg') + convert_to_visualize(graph_ir, vgraph) + vgraph.render() diff --git a/new_impl/cv/third_party/nni_new/retiarii/debug_configs.py b/new_impl/cv/third_party/nni_new/retiarii/debug_configs.py new file mode 100644 index 0000000000000000000000000000000000000000..9b4b1c643b33ae5336a4a0cad436311e5d30e331 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/retiarii/debug_configs.py @@ -0,0 +1,3 @@ +# we will support tensorflow in future release + +framework = 'pytorch' diff --git a/new_impl/cv/third_party/nni_new/retiarii/evaluator/__init__.py b/new_impl/cv/third_party/nni_new/retiarii/evaluator/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..c6fd370b906e8ad1545101c9ab0a8e3d605c0b33 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/retiarii/evaluator/__init__.py @@ -0,0 +1 @@ +from .functional import FunctionalEvaluator diff --git a/new_impl/cv/third_party/nni_new/retiarii/evaluator/functional.py b/new_impl/cv/third_party/nni_new/retiarii/evaluator/functional.py new file mode 100644 index 0000000000000000000000000000000000000000..3a7333284e94823ca9c6cee0eb9fd198c1a9a213 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/retiarii/evaluator/functional.py @@ -0,0 +1,37 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from ..graph import Evaluator + + +class FunctionalEvaluator(Evaluator): + """ + Functional evaluator that directly takes a function and thus should be general. + + Attributes + ---------- + function + The full name of the function. + arguments + Keyword arguments for the function other than model. + """ + + def __init__(self, function, **kwargs): + self.function = function + self.arguments = kwargs + + @staticmethod + def _load(ir): + return FunctionalEvaluator(ir['function'], **ir['arguments']) + + def _dump(self): + return { + 'function': self.function, + 'arguments': self.arguments + } + + def _execute(self, model_cls): + return self.function(model_cls, **self.arguments) + + def __eq__(self, other): + return self.function == other.function and self.arguments == other.arguments diff --git a/new_impl/cv/third_party/nni_new/retiarii/evaluator/pytorch/__init__.py b/new_impl/cv/third_party/nni_new/retiarii/evaluator/pytorch/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..c35431e91a9d71606de7d07625535785a0d6a23b --- /dev/null +++ b/new_impl/cv/third_party/nni_new/retiarii/evaluator/pytorch/__init__.py @@ -0,0 +1,2 @@ +from .base import PyTorchImageClassificationTrainer, PyTorchMultiModelTrainer +from .lightning import * diff --git a/new_impl/cv/third_party/nni_new/retiarii/evaluator/pytorch/base.py b/new_impl/cv/third_party/nni_new/retiarii/evaluator/pytorch/base.py new file mode 100644 index 0000000000000000000000000000000000000000..62d25b4f740e6d806747ac8c79c1b3788582064a --- /dev/null +++ b/new_impl/cv/third_party/nni_new/retiarii/evaluator/pytorch/base.py @@ -0,0 +1,305 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +# This file is deprecated. + +import abc +from typing import Any, List, Dict, Tuple + +import numpy as np +import torch +import torch.nn as nn +from torch.utils.data import DataLoader +from torchvision import datasets, transforms + +import nni + +class BaseTrainer(abc.ABC): + @abc.abstractmethod + def fit(self) -> None: + pass + + +def get_default_transform(dataset: str) -> Any: + """ + To get a default transformation of image for a specific dataset. + This is needed because transform objects can not be directly passed as arguments. + + Parameters + ---------- + dataset : str + Dataset class name. + + Returns + ------- + transform object + """ + if dataset == 'MNIST': + return transforms.Compose([ + transforms.ToTensor(), + transforms.Normalize((0.1307,), (0.3081,)) + ]) + if dataset == 'CIFAR10': + return transforms.Compose([ + transforms.RandomCrop(32, padding=4), + transforms.RandomHorizontalFlip(), + transforms.ToTensor(), + transforms.Normalize((0.4914, 0.4822, 0.4465), + (0.2023, 0.1994, 0.2010)), + ]) + # unsupported dataset, return None + return None + + +class PyTorchImageClassificationTrainer(BaseTrainer): + """ + Image classification trainer for PyTorch. + + A model, along with corresponding dataset, optimizer config is used to initialize the trainer. + The trainer will run for a fixed number of epochs (by default 10), and report the final result. + + TODO + Support scheduler, validate every n epochs, train/valid dataset + + Limitation induced by NNI: kwargs must be serializable to put into a JSON packed in parameters. + """ + + def __init__(self, model, + dataset_cls='MNIST', dataset_kwargs=None, dataloader_kwargs=None, + optimizer_cls='SGD', optimizer_kwargs=None, trainer_kwargs=None): + """Initialization of image classification trainer. + + Parameters + ---------- + model : nn.Module + Model to train. + dataset_cls : str, optional + Dataset class name that is available in ``torchvision.datasets``, by default 'MNIST' + dataset_kwargs : dict, optional + Keyword arguments passed to initialization of dataset class, by default None + dataset_kwargs : dict, optional + Keyword arguments passed to ``torch.utils.data.DataLoader``, by default None + optimizer_cls : str, optional + Optimizer class name that is available in ``torch.optim``, by default 'SGD' + optimizer_kwargs : dict, optional + Keyword arguments passed to initialization of optimizer class, by default None + trainer_kwargs: dict, optional + Keyword arguments passed to trainer. Will be passed to Trainer class in future. Currently, + only the key ``max_epochs`` is useful. + """ + super().__init__() + self._use_cuda = torch.cuda.is_available() + self.model = model + if self._use_cuda: + self.model.cuda() + self._loss_fn = nn.CrossEntropyLoss() + self._train_dataset = getattr(datasets, dataset_cls)(train=True, transform=get_default_transform(dataset_cls), + **(dataset_kwargs or {})) + self._val_dataset = getattr(datasets, dataset_cls)(train=False, transform=get_default_transform(dataset_cls), + **(dataset_kwargs or {})) + self._optimizer = getattr(torch.optim, optimizer_cls)(model.parameters(), **(optimizer_kwargs or {})) + self._trainer_kwargs = trainer_kwargs or {'max_epochs': 10} + + self._train_dataloader = DataLoader(self._train_dataset, **(dataloader_kwargs or {})) + self._val_dataloader = DataLoader(self._val_dataset, **(dataloader_kwargs or {})) + + def _accuracy(self, input, target): # pylint: disable=redefined-builtin + _, predict = torch.max(input.data, 1) + correct = predict.eq(target.data).cpu().sum().item() + return correct / input.size(0) + + def training_step(self, batch: Tuple[torch.Tensor, torch.Tensor], batch_idx: int) -> Dict[str, Any]: + x, y = self.training_step_before_model(batch, batch_idx) + y_hat = self.model(x) + return self.training_step_after_model(x, y, y_hat) + + def training_step_before_model(self, batch: Tuple[torch.Tensor, torch.Tensor], batch_idx: int): + x, y = batch + if self._use_cuda: + x, y = x.cuda(torch.device('cuda:0')), y.cuda(torch.device('cuda:0')) + return x, y + + def training_step_after_model(self, x, y, y_hat): + loss = self._loss_fn(y_hat, y) + return loss + + def validation_step(self, batch: Tuple[torch.Tensor, torch.Tensor], batch_idx: int) -> Dict[str, Any]: + x, y = self.validation_step_before_model(batch, batch_idx) + y_hat = self.model(x) + return self.validation_step_after_model(x, y, y_hat) + + def validation_step_before_model(self, batch: Tuple[torch.Tensor, torch.Tensor], batch_idx: int): + x, y = batch + if self._use_cuda: + x, y = x.cuda(), y.cuda() + return x, y + + def validation_step_after_model(self, x, y, y_hat): + acc = self._accuracy(y_hat, y) + return {'val_acc': acc} + + def validation_epoch_end(self, outputs: List[Dict[str, Any]]) -> Dict[str, Any]: + # We might need dict metrics in future? + avg_acc = np.mean([x['val_acc'] for x in outputs]).item() + nni.report_intermediate_result(avg_acc) + return {'val_acc': avg_acc} + + def _validate(self): + validation_outputs = [] + for i, batch in enumerate(self._val_dataloader): + validation_outputs.append(self.validation_step(batch, i)) + return self.validation_epoch_end(validation_outputs) + + def _train(self): + for i, batch in enumerate(self._train_dataloader): + self._optimizer.zero_grad() + loss = self.training_step(batch, i) + loss.backward() + self._optimizer.step() + + def fit(self) -> None: + for _ in range(self._trainer_kwargs['max_epochs']): + self._train() + self._validate() + # assuming val_acc here + nni.report_final_result(self._validate()['val_acc']) + + +class PyTorchMultiModelTrainer(BaseTrainer): + def __init__(self, multi_model, kwargs=[]): + self.multi_model = multi_model + self.kwargs = kwargs + self._train_dataloaders = [] + self._train_datasets = [] + self._val_dataloaders = [] + self._val_datasets = [] + self._optimizers = [] + self._trainers = [] + self._loss_fn = nn.CrossEntropyLoss() + self.max_steps = self.kwargs['max_steps'] if 'makx_steps' in self.kwargs else None + self.n_model = len(self.kwargs['model_kwargs']) + + for m in self.kwargs['model_kwargs']: + if m['use_input']: + dataset_cls = m['dataset_cls'] + dataset_kwargs = m['dataset_kwargs'] + dataloader_kwargs = m['dataloader_kwargs'] + train_dataset = getattr(datasets, dataset_cls)(train=True, transform=get_default_transform(dataset_cls), + **(dataset_kwargs or {})) + val_dataset = getattr(datasets, dataset_cls)(train=False, transform=get_default_transform(dataset_cls), + **(dataset_kwargs or {})) + train_dataloader = DataLoader(train_dataset, **(dataloader_kwargs or {})) + val_dataloader = DataLoader(val_dataset, **(dataloader_kwargs or {})) + self._train_datasets.append(train_dataset) + self._train_dataloaders.append(train_dataloader) + + self._val_datasets.append(val_dataset) + self._val_dataloaders.append(val_dataloader) + + if m['use_output']: + optimizer_cls = m['optimizer_cls'] + optimizer_kwargs = m['optimizer_kwargs'] + m_header = f"M_{m['model_id']}" + one_model_params = [] + for name, param in multi_model.named_parameters(): + name_prefix = '_'.join(name.split('_')[:2]) + if m_header == name_prefix: + one_model_params.append(param) + + optimizer = getattr(torch.optim, optimizer_cls)(one_model_params, **(optimizer_kwargs or {})) + self._optimizers.append(optimizer) + + def fit(self) -> None: + torch.autograd.set_detect_anomaly(True) + max_epochs = max([x['trainer_kwargs']['max_epochs'] for x in self.kwargs['model_kwargs']]) + for _ in range(max_epochs): + self._train() + self._validate() + nni.report_final_result(self._validate()) + + def _train(self): + for batch_idx, multi_model_batch in enumerate(zip(*self._train_dataloaders)): + for opt in self._optimizers: + opt.zero_grad() + xs = [] + ys = [] + for idx, batch in enumerate(multi_model_batch): + x, y = self.training_step_before_model(batch, batch_idx, f'cuda:{idx}') + xs.append(x) + ys.append(y) + + y_hats = self.multi_model(*xs) + if len(ys) != len(xs): + raise ValueError('len(ys) should be equal to len(xs)') + losses = [] + report_loss = {} + for output_idx, yhat in enumerate(y_hats): + if len(ys) == len(y_hats): + loss = self.training_step_after_model(xs[output_idx], ys[output_idx], yhat) + elif len(ys) == 1: + loss = self.training_step_after_model(xs[0], ys[0].to(yhat.get_device()), yhat) + else: + raise ValueError('len(ys) should be either 1 or len(y_hats)') + losses.append(loss.to("cuda:0")) + report_loss[self.kwargs['model_kwargs'][output_idx]['model_id']] = loss.item() + summed_loss = sum(losses) + summed_loss.backward() + for opt in self._optimizers: + opt.step() + if self.max_steps and batch_idx >= self.max_steps: + return + + def training_step_before_model(self, batch: Tuple[torch.Tensor, torch.Tensor], batch_idx: int, device=None): + x, y = batch + if device: + x, y = x.cuda(torch.device(device)), y.cuda(torch.device(device)) + return x, y + + def training_step_after_model(self, x, y, y_hat): + loss = self._loss_fn(y_hat, y) + return loss + + def _validate(self): + all_val_outputs = {idx: [] for idx in range(self.n_model)} + for batch_idx, multi_model_batch in enumerate(zip(*self._val_dataloaders)): + xs = [] + ys = [] + for idx, batch in enumerate(multi_model_batch): + x, y = self.training_step_before_model(batch, batch_idx, f'cuda:{idx}') + xs.append(x) + ys.append(y) + if len(ys) != len(xs): + raise ValueError('len(ys) should be equal to len(xs)') + + y_hats = self.multi_model(*xs) + + for output_idx, yhat in enumerate(y_hats): + if len(ys) == len(y_hats): + acc = self.validation_step_after_model(xs[output_idx], ys[output_idx], yhat) + elif len(ys) == 1: + acc = self.validation_step_after_model(xs[0], ys[0].to(yhat.get_device()), yhat) + else: + raise ValueError('len(ys) should be either 1 or len(y_hats)') + all_val_outputs[output_idx].append(acc) + + report_acc = {} + for idx in all_val_outputs: + avg_acc = np.mean([x['val_acc'] for x in all_val_outputs[idx]]).item() + report_acc[self.kwargs['model_kwargs'][idx]['model_id']] = avg_acc + nni.report_intermediate_result(report_acc) + return report_acc + + def validation_step_before_model(self, batch: Tuple[torch.Tensor, torch.Tensor], batch_idx: int, device=None): + x, y = batch + if device: + x, y = x.cuda(torch.device(device)), y.cuda(torch.device(device)) + return x, y + + def validation_step_after_model(self, x, y, y_hat): + acc = self._accuracy(y_hat, y) + return {'val_acc': acc} + + def _accuracy(self, input, target): # pylint: disable=redefined-builtin + _, predict = torch.max(input.data, 1) + correct = predict.eq(target.data).cpu().sum().item() + return correct / input.size(0) diff --git a/new_impl/cv/third_party/nni_new/retiarii/evaluator/pytorch/lightning.py b/new_impl/cv/third_party/nni_new/retiarii/evaluator/pytorch/lightning.py new file mode 100644 index 0000000000000000000000000000000000000000..4399844ac61944f2174ade19f78c2b9187efcbb3 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/retiarii/evaluator/pytorch/lightning.py @@ -0,0 +1,266 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import warnings +from typing import Dict, Union, Optional, List + +import pytorch_lightning as pl +import torch.nn as nn +import torch.optim as optim +from torch.utils.data import DataLoader + +import nni +from ...graph import Evaluator +from ...serializer import serialize_cls + + +__all__ = ['LightningModule', 'Trainer', 'DataLoader', 'Lightning', 'Classification', 'Regression'] + + +class LightningModule(pl.LightningModule): + def set_model(self, model): + if isinstance(model, type): + self.model = model() + else: + self.model = model + + +Trainer = serialize_cls(pl.Trainer) +DataLoader = serialize_cls(DataLoader) + + +class Lightning(Evaluator): + """ + Delegate the whole training to PyTorch Lightning. + + Since the arguments passed to the initialization needs to be serialized, ``LightningModule``, ``Trainer`` or + ``DataLoader`` in this file should be used. Another option is to hide dataloader in the Lightning module, in + which case, dataloaders are not required for this class to work. + + Following the programming style of Lightning, metrics sent to NNI should be obtained from ``callback_metrics`` + in trainer. Two hooks are added at the end of validation epoch and the end of ``fit``, respectively. The metric name + and type depend on the specific task. + + Parameters + ---------- + lightning_module : LightningModule + Lightning module that defines the training logic. + trainer : Trainer + Lightning trainer that handles the training. + train_dataloders : DataLoader + Used in ``trainer.fit()``. A PyTorch DataLoader with training samples. + If the ``lightning_module`` has a predefined train_dataloader method this will be skipped. + val_dataloaders : DataLoader or List of DataLoader + Used in ``trainer.fit()``. Either a single PyTorch Dataloader or a list of them, specifying validation samples. + If the ``lightning_module`` has a predefined val_dataloaders method this will be skipped. + """ + + def __init__(self, lightning_module: LightningModule, trainer: Trainer, + train_dataloader: Optional[DataLoader] = None, + val_dataloaders: Union[DataLoader, List[DataLoader], None] = None): + assert isinstance(lightning_module, LightningModule), f'Lightning module must be an instance of {__name__}.LightningModule.' + assert isinstance(trainer, Trainer), f'Trainer must be imported from {__name__}.' + assert _check_dataloader(train_dataloader), f'Wrong dataloader type. Try import DataLoader from {__name__}.' + assert _check_dataloader(val_dataloaders), f'Wrong dataloader type. Try import DataLoader from {__name__}.' + self.module = lightning_module + self.trainer = trainer + self.train_dataloader = train_dataloader + self.val_dataloaders = val_dataloaders + + @staticmethod + def _load(ir): + return Lightning(ir['module'], ir['trainer'], ir['train_dataloader'], ir['val_dataloaders']) + + def _dump(self): + return { + 'module': self.module, + 'trainer': self.trainer, + 'train_dataloader': self.train_dataloader, + 'val_dataloaders': self.val_dataloaders + } + + def _execute(self, model_cls): + return self.fit(model_cls) + + def __eq__(self, other): + return self.function == other.function and self.arguments == other.arguments + + def fit(self, model): + """ + Fit the model with provided dataloader, with Lightning trainer. + + Parameters + ---------- + model : nn.Module + The model to fit. + """ + self.module.set_model(model) + return self.trainer.fit(self.module, self.train_dataloader, self.val_dataloaders) + + +def _check_dataloader(dataloader): + if dataloader is None: + return True + if isinstance(dataloader, list): + return all([_check_dataloader(d) for d in dataloader]) + return isinstance(dataloader, DataLoader) + + +### The following are some commonly used Lightning modules ### + +class _SupervisedLearningModule(LightningModule): + def __init__(self, criterion: nn.Module, metrics: Dict[str, pl.metrics.Metric], + learning_rate: float = 0.001, + weight_decay: float = 0., + optimizer: optim.Optimizer = optim.Adam): + super().__init__() + self.save_hyperparameters('criterion', 'optimizer', 'learning_rate', 'weight_decay') + self.criterion = criterion() + self.optimizer = optimizer + self.metrics = nn.ModuleDict({name: cls() for name, cls in metrics.items()}) + + def forward(self, x): + y_hat = self.model(x) + return y_hat + + def training_step(self, batch, batch_idx): + x, y = batch + y_hat = self(x) + loss = self.criterion(y_hat, y) + self.log('train_loss', loss, prog_bar=True) + for name, metric in self.metrics.items(): + self.log('train_' + name, metric(y_hat, y), prog_bar=True) + return loss + + def validation_step(self, batch, batch_idx): + x, y = batch + y_hat = self(x) + self.log('val_loss', self.criterion(y_hat, y), prog_bar=True) + for name, metric in self.metrics.items(): + self.log('val_' + name, metric(y_hat, y), prog_bar=True) + + def test_step(self, batch, batch_idx): + x, y = batch + y_hat = self(x) + self.log('test_loss', self.criterion(y_hat, y), prog_bar=True) + for name, metric in self.metrics.items(): + self.log('test_' + name, metric(y_hat, y), prog_bar=True) + + def configure_optimizers(self): + return self.optimizer(self.parameters(), lr=self.hparams.learning_rate, weight_decay=self.hparams.weight_decay) + + def on_validation_epoch_end(self): + nni.report_intermediate_result(self._get_validation_metrics()) + + def teardown(self, stage): + if stage == 'fit': + nni.report_final_result(self._get_validation_metrics()) + + def _get_validation_metrics(self): + if len(self.metrics) == 1: + metric_name = next(iter(self.metrics)) + return self.trainer.callback_metrics['val_' + metric_name].item() + else: + warnings.warn('Multiple metrics without "default" is not supported by current framework.') + return {name: self.trainer.callback_metrics['val_' + name].item() for name in self.metrics} + + +class _AccuracyWithLogits(pl.metrics.Accuracy): + def update(self, pred, target): + return super().update(nn.functional.softmax(pred), target) + + +@serialize_cls +class _ClassificationModule(_SupervisedLearningModule): + def __init__(self, criterion: nn.Module = nn.CrossEntropyLoss, + learning_rate: float = 0.001, + weight_decay: float = 0., + optimizer: optim.Optimizer = optim.Adam): + super().__init__(criterion, {'acc': _AccuracyWithLogits}, + learning_rate=learning_rate, weight_decay=weight_decay, optimizer=optimizer) + + +class Classification(Lightning): + """ + Trainer that is used for classification. + + Parameters + ---------- + criterion : nn.Module + Class for criterion module (not an instance). default: ``nn.CrossEntropyLoss`` + learning_rate : float + Learning rate. default: 0.001 + weight_decay : float + L2 weight decay. default: 0 + optimizer : Optimizer + Class for optimizer (not an instance). default: ``Adam`` + train_dataloders : DataLoader + Used in ``trainer.fit()``. A PyTorch DataLoader with training samples. + If the ``lightning_module`` has a predefined train_dataloader method this will be skipped. + val_dataloaders : DataLoader or List of DataLoader + Used in ``trainer.fit()``. Either a single PyTorch Dataloader or a list of them, specifying validation samples. + If the ``lightning_module`` has a predefined val_dataloaders method this will be skipped. + trainer_kwargs : dict + Optional keyword arguments passed to trainer. See + `Lightning documentation `__ for details. + """ + + def __init__(self, criterion: nn.Module = nn.CrossEntropyLoss, + learning_rate: float = 0.001, + weight_decay: float = 0., + optimizer: optim.Optimizer = optim.Adam, + train_dataloader: Optional[DataLoader] = None, + val_dataloaders: Union[DataLoader, List[DataLoader], None] = None, + **trainer_kwargs): + module = _ClassificationModule(criterion=criterion, learning_rate=learning_rate, + weight_decay=weight_decay, optimizer=optimizer) + super().__init__(module, Trainer(**trainer_kwargs), + train_dataloader=train_dataloader, val_dataloaders=val_dataloaders) + + +@serialize_cls +class _RegressionModule(_SupervisedLearningModule): + def __init__(self, criterion: nn.Module = nn.MSELoss, + learning_rate: float = 0.001, + weight_decay: float = 0., + optimizer: optim.Optimizer = optim.Adam): + super().__init__(criterion, {'mse': pl.metrics.MeanSquaredError}, + learning_rate=learning_rate, weight_decay=weight_decay, optimizer=optimizer) + + +class Regression(Lightning): + """ + Trainer that is used for regression. + + Parameters + ---------- + criterion : nn.Module + Class for criterion module (not an instance). default: ``nn.MSELoss`` + learning_rate : float + Learning rate. default: 0.001 + weight_decay : float + L2 weight decay. default: 0 + optimizer : Optimizer + Class for optimizer (not an instance). default: ``Adam`` + train_dataloders : DataLoader + Used in ``trainer.fit()``. A PyTorch DataLoader with training samples. + If the ``lightning_module`` has a predefined train_dataloader method this will be skipped. + val_dataloaders : DataLoader or List of DataLoader + Used in ``trainer.fit()``. Either a single PyTorch Dataloader or a list of them, specifying validation samples. + If the ``lightning_module`` has a predefined val_dataloaders method this will be skipped. + trainer_kwargs : dict + Optional keyword arguments passed to trainer. See + `Lightning documentation `__ for details. + """ + + def __init__(self, criterion: nn.Module = nn.MSELoss, + learning_rate: float = 0.001, + weight_decay: float = 0., + optimizer: optim.Optimizer = optim.Adam, + train_dataloader: Optional[DataLoader] = None, + val_dataloaders: Union[DataLoader, List[DataLoader], None] = None, + **trainer_kwargs): + module = _RegressionModule(criterion=criterion, learning_rate=learning_rate, + weight_decay=weight_decay, optimizer=optimizer) + super().__init__(module, Trainer(**trainer_kwargs), + train_dataloader=train_dataloader, val_dataloaders=val_dataloaders) diff --git a/new_impl/cv/third_party/nni_new/retiarii/execution/__init__.py b/new_impl/cv/third_party/nni_new/retiarii/execution/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..0a0e47b0b01b597936c8c2a8a09cafeda899168b --- /dev/null +++ b/new_impl/cv/third_party/nni_new/retiarii/execution/__init__.py @@ -0,0 +1 @@ +from .api import * diff --git a/new_impl/cv/third_party/nni_new/retiarii/execution/api.py b/new_impl/cv/third_party/nni_new/retiarii/execution/api.py new file mode 100644 index 0000000000000000000000000000000000000000..8027e7e36308c4cf511ba1942264be9bdb2dfef1 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/retiarii/execution/api.py @@ -0,0 +1,74 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import time +from typing import Iterable + +from ..graph import Model, ModelStatus +from .interface import AbstractExecutionEngine +from .listener import DefaultListener + +_execution_engine = None +_default_listener = None + +__all__ = ['get_execution_engine', 'get_and_register_default_listener', + 'list_models', 'submit_models', 'wait_models', 'query_available_resources', + 'set_execution_engine', 'is_stopped_exec', 'budget_exhausted'] + + +def set_execution_engine(engine: AbstractExecutionEngine) -> None: + global _execution_engine + if _execution_engine is None: + _execution_engine = engine + else: + raise RuntimeError('Execution engine is already set.') + + +def get_execution_engine() -> AbstractExecutionEngine: + global _execution_engine + assert _execution_engine is not None, 'You need to set execution engine, before using it.' + return _execution_engine + + +def get_and_register_default_listener(engine: AbstractExecutionEngine) -> DefaultListener: + global _default_listener + if _default_listener is None: + _default_listener = DefaultListener() + engine.register_graph_listener(_default_listener) + return _default_listener + + +def submit_models(*models: Model) -> None: + engine = get_execution_engine() + get_and_register_default_listener(engine) + engine.submit_models(*models) + + +def list_models(*models: Model) -> Iterable[Model]: + engine = get_execution_engine() + get_and_register_default_listener(engine) + return engine.list_models() + + +def wait_models(*models: Model) -> None: + get_and_register_default_listener(get_execution_engine()) + while True: + time.sleep(1) + left_models = [g for g in models if not g.status in (ModelStatus.Trained, ModelStatus.Failed)] + if not left_models: + break + + +def query_available_resources() -> int: + engine = get_execution_engine() + resources = engine.query_available_resource() + return resources if isinstance(resources, int) else len(resources) + + +def is_stopped_exec(model: Model) -> bool: + return model.status in (ModelStatus.Trained, ModelStatus.Failed) + + +def budget_exhausted() -> bool: + engine = get_execution_engine() + return engine.budget_exhausted() diff --git a/new_impl/cv/third_party/nni_new/retiarii/execution/base.py b/new_impl/cv/third_party/nni_new/retiarii/execution/base.py new file mode 100644 index 0000000000000000000000000000000000000000..36d09b505fbe0090a88694034ae6c5aed57f099c --- /dev/null +++ b/new_impl/cv/third_party/nni_new/retiarii/execution/base.py @@ -0,0 +1,128 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +import os +import random +import string +from typing import Any, Dict, Iterable, List + +from .interface import AbstractExecutionEngine, AbstractGraphListener +from .. import codegen, utils +from ..graph import Model, ModelStatus, MetricData, Evaluator +from ..integration_api import send_trial, receive_trial_parameters, get_advisor + +_logger = logging.getLogger(__name__) + +class BaseGraphData: + def __init__(self, model_script: str, evaluator: Evaluator) -> None: + self.model_script = model_script + self.evaluator = evaluator + + def dump(self) -> dict: + return { + 'model_script': self.model_script, + 'evaluator': self.evaluator + } + + @staticmethod + def load(data) -> 'BaseGraphData': + return BaseGraphData(data['model_script'], data['evaluator']) + + +class BaseExecutionEngine(AbstractExecutionEngine): + """ + The execution engine with no optimization at all. + Resource management is implemented in this class. + """ + + def __init__(self) -> None: + """ + Upon initialization, advisor callbacks need to be registered. + Advisor will call the callbacks when the corresponding event has been triggered. + Base execution engine will get those callbacks and broadcast them to graph listener. + """ + self._listeners: List[AbstractGraphListener] = [] + + # register advisor callbacks + advisor = get_advisor() + advisor.send_trial_callback = self._send_trial_callback + advisor.request_trial_jobs_callback = self._request_trial_jobs_callback + advisor.trial_end_callback = self._trial_end_callback + advisor.intermediate_metric_callback = self._intermediate_metric_callback + advisor.final_metric_callback = self._final_metric_callback + + self._running_models: Dict[int, Model] = dict() + self._history: List[Model] = [] + + self.resources = 0 + + def submit_models(self, *models: Model) -> None: + for model in models: + data = self.pack_model_data(model) + self._running_models[send_trial(data.dump())] = model + self._history.append(model) + + def list_models(self) -> Iterable[Model]: + return self._history + + def register_graph_listener(self, listener: AbstractGraphListener) -> None: + self._listeners.append(listener) + + def _send_trial_callback(self, paramater: dict) -> None: + if self.resources <= 0: + # FIXME: should be a warning message here + _logger.debug('There is no available resource, but trial is submitted.') + self.resources -= 1 + _logger.debug('Resource used. Remaining: %d', self.resources) + + def _request_trial_jobs_callback(self, num_trials: int) -> None: + self.resources += num_trials + _logger.debug('New resource available. Remaining: %d', self.resources) + + def _trial_end_callback(self, trial_id: int, success: bool) -> None: + model = self._running_models[trial_id] + if success: + model.status = ModelStatus.Trained + else: + model.status = ModelStatus.Failed + for listener in self._listeners: + listener.on_training_end(model, success) + + def _intermediate_metric_callback(self, trial_id: int, metrics: MetricData) -> None: + model = self._running_models[trial_id] + model.intermediate_metrics.append(metrics) + for listener in self._listeners: + listener.on_intermediate_metric(model, metrics) + + def _final_metric_callback(self, trial_id: int, metrics: MetricData) -> None: + model = self._running_models[trial_id] + model.metric = metrics + for listener in self._listeners: + listener.on_metric(model, metrics) + + def query_available_resource(self) -> int: + return self.resources + + def budget_exhausted(self) -> bool: + advisor = get_advisor() + return advisor.stopping + + @classmethod + def pack_model_data(cls, model: Model) -> Any: + return BaseGraphData(codegen.model_to_pytorch_script(model), model.evaluator) + + @classmethod + def trial_execute_graph(cls) -> None: + """ + Initialize the model, hand it over to trainer. + """ + graph_data = BaseGraphData.load(receive_trial_parameters()) + random_str = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(6)) + file_name = f'_generated_model/{random_str}.py' + os.makedirs(os.path.dirname(file_name), exist_ok=True) + with open(file_name, 'w') as f: + f.write(graph_data.model_script) + model_cls = utils.import_(f'_generated_model.{random_str}._model') + graph_data.evaluator._execute(model_cls) + os.remove(file_name) diff --git a/new_impl/cv/third_party/nni_new/retiarii/execution/cgo_engine.py b/new_impl/cv/third_party/nni_new/retiarii/execution/cgo_engine.py new file mode 100644 index 0000000000000000000000000000000000000000..ee8e59498440a2326cbce12cb233bf854c36f196 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/retiarii/execution/cgo_engine.py @@ -0,0 +1,159 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +from typing import Iterable, List, Dict, Tuple + +from .interface import AbstractExecutionEngine, AbstractGraphListener, WorkerInfo +from .. import codegen, utils +from ..graph import Model, ModelStatus, MetricData +from ..integration_api import send_trial, receive_trial_parameters, get_advisor +from .logical_optimizer.logical_plan import LogicalPlan, PhysicalDevice +from .logical_optimizer.opt_dedup_input import DedupInputOptimizer + +from .base import BaseGraphData + +_logger = logging.getLogger(__name__) + + +class CGOExecutionEngine(AbstractExecutionEngine): + def __init__(self, n_model_per_graph=4) -> None: + self._listeners: List[AbstractGraphListener] = [] + self._running_models: Dict[int, Model] = dict() + self.logical_plan_counter = 0 + self.n_model_per_graph = n_model_per_graph + self._optimizers = [DedupInputOptimizer()] + self._original_models = {} + self._original_model_to_multi_model = {} + + # register advisor callbacks + advisor = get_advisor() + advisor.send_trial_callback = self._send_trial_callback + advisor.request_trial_jobs_callback = self._request_trial_jobs_callback + advisor.trial_end_callback = self._trial_end_callback + advisor.intermediate_metric_callback = self._intermediate_metric_callback + advisor.final_metric_callback = self._final_metric_callback + + def add_optimizer(self, opt): + self._optimizers.append(opt) + + def submit_models(self, *models: List[Model]) -> None: + _logger.info('%d models are submitted', len(models)) + logical = self._build_logical(models) + + for opt in self._optimizers: + opt.convert(logical) + + phy_models_and_placements = self._assemble(logical) + for model, placement, grouped_models in phy_models_and_placements: + data = BaseGraphData(codegen.model_to_pytorch_script(model, placement=placement), + model.evaluator) + for m in grouped_models: + self._original_models[m.model_id] = m + self._original_model_to_multi_model[m.model_id] = model + self._running_models[send_trial(data.dump())] = model + + # for model in models: + # data = BaseGraphData(codegen.model_to_pytorch_script(model), + # model.config['trainer_module'], model.config['trainer_kwargs']) + # self._running_models[send_trial(data.dump())] = model + + def list_models(self) -> Iterable[Model]: + raise NotImplementedError + + def _assemble(self, logical_plan: LogicalPlan) -> List[Tuple[Model, PhysicalDevice]]: + # unique_models = set() + # for node in logical_plan.graph.nodes: + # if node.graph.model not in unique_models: + # unique_models.add(node.graph.model) + # return [m for m in unique_models] + grouped_models: List[Dict[Model, PhysicalDevice]] = AssemblePolicy().group(logical_plan) + phy_models_and_placements = [] + for multi_model in grouped_models: + model, model_placement = logical_plan.assemble(multi_model) + phy_models_and_placements.append((model, model_placement, multi_model.keys())) + return phy_models_and_placements + + def _build_logical(self, models: List[Model]) -> LogicalPlan: + logical_plan = LogicalPlan(plan_id=self.logical_plan_counter) + for model in models: + logical_plan.add_model(model) + self.logical_plan_counter += 1 + return logical_plan + + def register_graph_listener(self, listener: AbstractGraphListener) -> None: + self._listeners.append(listener) + + def _send_trial_callback(self, paramater: dict) -> None: + for listener in self._listeners: + listener.on_resource_used(0) # FIXME: find the real resource id + + def _request_trial_jobs_callback(self, num_trials: int) -> None: + for listener in self._listeners: + listener.on_resource_available([0] * num_trials) # FIXME: find the real resource id + + def _trial_end_callback(self, trial_id: int, success: bool) -> None: + model = self._running_models[trial_id] + if success: + model.status = ModelStatus.Trained + else: + model.status = ModelStatus.Failed + for model_id in self._original_model_to_multi_model: + if self._original_model_to_multi_model[model_id] == model: + original_model = self._original_models[model_id] + if success: + original_model.status = ModelStatus.Trained + else: + original_model.status = ModelStatus.Failed + for listener in self._listeners: + listener.on_training_end(original_model, success) + + def _intermediate_metric_callback(self, trial_id: int, metrics: MetricData) -> None: + # model = self._running_models[trial_id] + merged_metrics = dict(metrics) + for model_id in merged_metrics: + int_model_id = int(model_id) + self._original_models[int_model_id].intermediate_metrics.append(merged_metrics[model_id]) + # model.intermediate_metrics.append(metrics) + for listener in self._listeners: + listener.on_intermediate_metric(self._original_models[int_model_id], merged_metrics[model_id]) + + def _final_metric_callback(self, trial_id: int, metrics: MetricData) -> None: + merged_metrics = dict(metrics) + for model_id in merged_metrics: + int_model_id = int(model_id) + self._original_models[int_model_id].intermediate_metrics.append(merged_metrics[model_id]) + # model.intermediate_metrics.append(metrics) + for listener in self._listeners: + listener.on_metric(self._original_models[int_model_id], merged_metrics[model_id]) + + def query_available_resource(self) -> List[WorkerInfo]: + raise NotImplementedError # move the method from listener to here? + + def budget_exhausted(self) -> bool: + raise NotImplementedError + + @classmethod + def trial_execute_graph(cls) -> None: + """ + Initialize the model, hand it over to trainer. + """ + graph_data = BaseGraphData.load(receive_trial_parameters()) + _logger.info('CGO_ENGINE trial parameters received') + with open('_generated_model.py', 'w') as f: + f.write(graph_data.model_script) + # with open('_debug_graph_data.json', 'w') as f: + # json.dump(graph_data.dump(), f) + trainer_cls = utils.import_(graph_data.training_module) + model_cls = utils.import_(f"_generated_model.{graph_data.training_kwargs['model_cls']}") + trainer_instance = trainer_cls(model_cls(), graph_data.training_kwargs) + trainer_instance.fit() + + +class AssemblePolicy: + @staticmethod + def group(logical_plan): + group_model = {} + for idx, m in enumerate(logical_plan.models): + group_model[m] = PhysicalDevice('server', f'cuda:{idx}') + return [group_model] diff --git a/new_impl/cv/third_party/nni_new/retiarii/execution/interface.py b/new_impl/cv/third_party/nni_new/retiarii/execution/interface.py new file mode 100644 index 0000000000000000000000000000000000000000..ae74e241067640196ca87422b21415f17c023106 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/retiarii/execution/interface.py @@ -0,0 +1,153 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from abc import ABC, abstractmethod, abstractclassmethod +from typing import Any, Iterable, NewType, List, Union + +from ..graph import Model, MetricData + +__all__ = [ + 'GraphData', 'WorkerInfo', + 'AbstractGraphListener', 'AbstractExecutionEngine' +] + + +GraphData = NewType('GraphData', Any) +""" +A _serializable_ internal data type defined by execution engine. + +Execution engine will submit this kind of data through NNI to worker machine, and train it there. + +A `GraphData` object describes a (merged) executable graph. + +This is trial's "hyper-parameter" in NNI's term and will be transfered in JSON format. + +See `AbstractExecutionEngine` for details. +""" + + +WorkerInfo = NewType('WorkerInfo', Any) +""" +To be designed. Discussion needed. + +This describes the properties of a worker machine. (e.g. memory size) +""" + + +class AbstractGraphListener(ABC): + """ + Abstract listener interface to receive graph events. + + Use `AbstractExecutionEngine.register_graph_listener()` to activate a listener. + """ + + @abstractmethod + def on_metric(self, model: Model, metric: MetricData) -> None: + """ + Reports the final metric of a graph. + """ + raise NotImplementedError + + @abstractmethod + def on_intermediate_metric(self, model: Model, metric: MetricData) -> None: + """ + Reports the latest intermediate metric of a trainning graph. + """ + pass + + @abstractmethod + def on_training_end(self, model: Model, success: bool) -> None: + """ + Reports either a graph is fully trained or the training process has failed. + """ + pass + + +class AbstractExecutionEngine(ABC): + """ + The abstract interface of execution engine. + + Most of these APIs are used by strategy, except `trial_execute_graph`, which is invoked by framework in trial. + Strategy will get the singleton execution engine object through a global API, + and use it in either sync or async manner. + + Execution engine is responsible for submitting (maybe-optimized) models to NNI, + and assigning their metrics to the `Model` object after training. + Execution engine is also responsible to launch the graph in trial process, + because it's the only one who understands graph data, or "hyper-parameter" in NNI's term. + + Execution engine will leverage NNI Advisor APIs, which are yet open for discussion. + + In synchronized use case, the strategy will have a loop to call `submit_models` and `wait_models` repeatly, + and will receive metrics from `Model` attributes. + Execution engine could assume that strategy will only submit graph when there are availabe resources (for now). + + In asynchronized use case, the strategy will register a listener to receive events, + while still using `submit_models` to train. + + There will be a `BaseExecutionEngine` subclass. + Inner-graph optimizing is supposed to derive `BaseExecutionEngine`, + while overrides `submit_models` and `trial_execute_graph`. + cross-graph optimizing is supposed to derive `AbstractExectutionEngine` directly, + because in this case APIs like `wait_graph` and `listener.on_training_end` will have unique logic. + + There might be some util functions benefit all optimizing methods, + but non-mandatory utils should not be covered in abstract interface. + """ + + @abstractmethod + def submit_models(self, *models: Model) -> None: + """ + Submit models to NNI. + + This method is supposed to call something like `nni.Advisor.create_trial_job(graph_data)`. + """ + raise NotImplementedError + + @abstractmethod + def list_models(self) -> Iterable[Model]: + """ + Get all models in submitted. + + Execution engine should store a copy of models that have been submitted and return a list of copies in this method. + """ + raise NotImplementedError + + @abstractmethod + def query_available_resource(self) -> Union[List[WorkerInfo], int]: + """ + Returns information of all idle workers. + If no details are available, this may returns a list of "empty" objects, reporting the number of idle workers. + + Could be left unimplemented for first iteration. + """ + raise NotImplementedError + + @abstractmethod + def budget_exhausted(self) -> bool: + """ + Check whether user configured max trial number or max execution duration has been reached + """ + raise NotImplementedError + + @abstractmethod + def register_graph_listener(self, listener: AbstractGraphListener) -> None: + """ + Register a listener to receive graph events. + + Could be left unimplemented for first iteration. + """ + raise NotImplementedError + + @abstractclassmethod + def trial_execute_graph(cls) -> MetricData: + """ + Train graph and returns its metrics, in a separate trial process. + + Each call to `nni.Advisor.create_trial_job(graph_data)` will eventually invoke this method. + + Because this method will be invoked in trial process on training platform, + it has different context from other methods and has no access to global variable or `self`. + However util APIs like `.utils.experiment_config()` should still be available. + """ + raise NotImplementedError diff --git a/new_impl/cv/third_party/nni_new/retiarii/execution/listener.py b/new_impl/cv/third_party/nni_new/retiarii/execution/listener.py new file mode 100644 index 0000000000000000000000000000000000000000..cfda111fae7d837a4824a2365d3ab02a70df29f9 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/retiarii/execution/listener.py @@ -0,0 +1,20 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from ..graph import Model, ModelStatus +from .interface import MetricData, AbstractGraphListener + + +class DefaultListener(AbstractGraphListener): + + def on_metric(self, model: Model, metric: MetricData) -> None: + model.metric = metric + + def on_intermediate_metric(self, model: Model, metric: MetricData) -> None: + model.intermediate_metrics.append(metric) + + def on_training_end(self, model: Model, success: bool) -> None: + if success: + model.status = ModelStatus.Trained + else: + model.status = ModelStatus.Failed diff --git a/new_impl/cv/third_party/nni_new/retiarii/execution/logical_optimizer/__init__.py b/new_impl/cv/third_party/nni_new/retiarii/execution/logical_optimizer/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/new_impl/cv/third_party/nni_new/retiarii/execution/logical_optimizer/interface.py b/new_impl/cv/third_party/nni_new/retiarii/execution/logical_optimizer/interface.py new file mode 100644 index 0000000000000000000000000000000000000000..03e1e84772417ccda1dc39a97a54e4123b183570 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/retiarii/execution/logical_optimizer/interface.py @@ -0,0 +1,14 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from abc import ABC + +from .logical_plan import LogicalPlan + + +class AbstractOptimizer(ABC): + def __init__(self) -> None: + pass + + def convert(self, logical_plan: LogicalPlan) -> None: + raise NotImplementedError diff --git a/new_impl/cv/third_party/nni_new/retiarii/execution/logical_optimizer/logical_plan.py b/new_impl/cv/third_party/nni_new/retiarii/execution/logical_optimizer/logical_plan.py new file mode 100644 index 0000000000000000000000000000000000000000..f68ffc86a66312778c276243668b77b046e16fd3 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/retiarii/execution/logical_optimizer/logical_plan.py @@ -0,0 +1,290 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import copy +from typing import Dict, Tuple, List, Any + +from nni.retiarii.utils import uid +from ...graph import Cell, Edge, Graph, Model, Node +from ...operation import Operation, _IOPseudoOperation + + +class PhysicalDevice: + def __init__(self, server: str, device: str): + self.server = server + self.device = device + + def __eq__(self, o) -> bool: + return self.server == o.server and self.device == o.device + + def __hash__(self) -> int: + return hash(self.server + '_' + self.device) + + +class AbstractLogicalNode(Node): + def __init__(self, graph, node_id, name, operation, _internal=False): + super().__init__(graph, node_id, name, operation, _internal=_internal) + + def assemble(self, multi_model_placement: Dict[Model, PhysicalDevice]) -> Tuple[Node, PhysicalDevice]: + raise NotImplementedError + + def _fork_to(self, graph: Graph): + raise NotImplementedError + + +class LogicalGraph(Graph): + def __init__(self, model: Model, graph_id: int, name: str = None, _internal: bool = False): + super().__init__(model, graph_id, name='logical_' + name, _internal=_internal) + + def _dump(self) -> Any: + nodes_dump = {} + for node in self.hidden_nodes: + if isinstance(node, OriginNode): + nodes_dump[f"{node.original_graph.model.model_id}_{node.name}"] = node._dump( + ) + else: + nodes_dump[f"{node.graph.model.model_id}_{node.name}"] = node._dump() + + edges_dump = [] + for edge in self.edges: + if isinstance(edge.head, OriginNode): + head_info = f'{edge.head.original_graph.model.model_id}_{edge.head.name}' + else: + head_info = edge.head.name + if isinstance(edge.tail, OriginNode): + tail_info = f'{edge.tail.original_graph.model.model_id}_{edge.tail.name}' + else: + tail_info = edge.tail.name + edges_dump.append((head_info, tail_info)) + return { + 'inputs': self.input_node.operation.io_names, + 'outputs': self.output_node.operation.io_names, + 'nodes': nodes_dump, + 'edges': edges_dump + } + + def _fork_to(self, model: Model) -> Graph: + new_graph = Graph(model, self.id, self.name, + _internal=True)._register() + + for node in self.hidden_nodes: + if isinstance(node, AbstractLogicalNode): + node._fork_to(new_graph) + else: + Node(new_graph, node.id, node.name, + node.operation, _internal=True)._register() + + id_to_new_node = {node.__repr__(): node for node in new_graph.nodes} + + for edge in self.edges: + new_head = id_to_new_node[edge.head.__repr__()] + new_tail = id_to_new_node[edge.tail.__repr__()] + Edge((new_head, edge.head_slot), + (new_tail, edge.tail_slot), _internal=True)._register() + + return new_graph + + +class OriginNode(AbstractLogicalNode): + def __init__(self, logical_graph: LogicalGraph, + original_graph: Graph, original_node: Node, + name: str, operation, _internal=False): + super().__init__(logical_graph, original_node.id, name, operation) + self.original_graph = original_graph + self.original_node = original_node + + def assemble(self, multi_model_placement: Dict[Model, PhysicalDevice]) -> Tuple[Node, PhysicalDevice]: + model_id = self.original_node.graph.model.model_id + new_node = Node(self.original_node.graph, self.original_node.id, + f"M_{model_id}_" + + self.original_node.name, + self.original_node.operation) + return new_node, multi_model_placement[self.original_node.graph.model] + + def __repr__(self): + return f'OriginNode(id={self.id}, name={self.name}, \ + operation={self.operation}, origin_model_id={self.original_graph.model.model_id})' + + def _fork_to(self, graph: Graph): + OriginNode(graph, self.original_graph, self.original_node, + self.name, self.operation)._register() + + +class LogicalPlan: + def __init__(self, plan_id=0) -> None: + self.lp_model = Model(_internal=True) + self.id = plan_id + self.logical_graph = LogicalGraph( + self.lp_model, self.id, name=f'{self.id}', _internal=True)._register() + self.lp_model._root_graph_name = self.logical_graph.name + self.models = [] + + def add_model(self, model: Model): + self.models.append(model) + # Only optimize the root graph. + self._merge_graph(model.root_graph) + + def _merge_graph(self, from_graph): + to_graph = self.logical_graph + id_to_new_node = {} # old node ID -> new node object + + for old_node in from_graph.nodes: + new_node = OriginNode(to_graph, old_node.graph, + old_node, old_node.name, + old_node.operation, _internal=True)._register() + id_to_new_node[old_node.id] = new_node + + for edge in from_graph.edges: + new_head = id_to_new_node[edge.head.id] + new_tail = id_to_new_node[edge.tail.id] + Edge((new_head, edge.head_slot), (new_tail, + edge.tail_slot), _internal=True)._register() + + def assemble(self, multi_model_placement: Dict[Model, PhysicalDevice]) \ + -> Tuple[Model, Dict[Node, PhysicalDevice], List[Model]]: + phy_model = Model(_internal=True) # self.lp_model.fork() + phy_graph = self.lp_model.root_graph._fork_to(phy_model) + + # Add a flag to mark multi-model in graph json. + # Multi-model has a list of training configs in kwargs['model_kwargs'] + if len(multi_model_placement) > 1: + phy_model.evaluator.kwargs['is_multi_model'] = True + phy_model.evaluator.kwargs['model_cls'] = phy_graph.name + phy_model.evaluator.kwargs['model_kwargs'] = [] + # FIXME: allow user to specify + phy_model.evaluator.module = 'nni.retiarii.trainer.pytorch.PyTorchMultiModelTrainer' + + # merge sub-graphs + for model in multi_model_placement: + for graph_name in model.graphs: + if graph_name != model._root_graph_name: + model.graphs[graph_name]._fork_to( + phy_model, name_prefix=f'M_{model.model_id}_') + + # When replace logical nodes, merge the training configs when + # input/output nodes are replaced. + evaluator_slot = {} # Model ID -> Slot ID + input_slot_mapping = {} + output_slot_mapping = {} + # Replace all logical nodes to executable physical nodes + hidden_nodes = phy_graph.hidden_nodes.copy() + node_placements = {} + for node in hidden_nodes: + if isinstance(node, OriginNode): + model_id = node.original_graph.model.model_id + if node.original_graph.model not in multi_model_placement: + for edge in node.incoming_edges: + edge.remove() + for edge in node.outgoing_edges: + edge.remove() + node.remove() + continue + + if isinstance(node, AbstractLogicalNode): + new_node, placement = node.assemble(multi_model_placement) + if isinstance(new_node.operation, _IOPseudoOperation): + model_id = new_node.graph.model.model_id + if model_id not in evaluator_slot: + phy_model.evaluator.kwargs['model_kwargs'].append(new_node.graph.model.evaluator.kwargs.copy()) + evaluator_slot[model_id] = len(phy_model.evaluator.kwargs['model_kwargs']) - 1 + slot = evaluator_slot[model_id] + phy_model.evaluator.kwargs['model_kwargs'][slot]['model_id'] = model_id + phy_model.evaluator.kwargs['model_kwargs'][slot]['use_input'] = False + phy_model.evaluator.kwargs['model_kwargs'][slot]['use_output'] = False + else: + slot = evaluator_slot[model_id] + # If a model's inputs/outputs are not used in the multi-model + # the codegen and trainer should not generate and use them + # "use_input" and "use_output" are used to mark whether + # an input/output of a model is used in a multi-model + if new_node.operation.type == '_inputs': + input_slot_mapping[new_node] = slot + phy_model.evaluator.kwargs['model_kwargs'][slot]['use_input'] = True + if new_node.operation.type == '_outputs': + output_slot_mapping[new_node] = slot + phy_model.evaluator.kwargs['model_kwargs'][slot]['use_output'] = True + + self.node_replace(node, new_node) + + if isinstance(new_node.operation, Cell): + old_cell_name = new_node.operation.cell_name + new_node.operation = copy.deepcopy(new_node.operation) + new_node.operation.cell_name = f'M_{model_id}_{old_cell_name}' + node_placements[new_node] = placement + + node.remove() + + # If two nodes are placed on different devices, use ToDevice op to copy the node + existing_edges = phy_graph.edges.copy() + # Avoid a node is copied multiple times on the same device + copied_op: Dict[Tuple(Node, PhysicalDevice), Node] = {} + for edge in existing_edges: + head_placement = node_placements[edge.head] + tail_placement = node_placements[edge.tail] + if head_placement != tail_placement: + if head_placement.server != tail_placement.server: + raise ValueError('Cross-server placement is not supported.') + # Same server different devices + if (edge.head, tail_placement) in copied_op: + to_node = copied_op[(edge.head, tail_placement)] + else: + to_operation = Operation.new('ToDevice', {"device": tail_placement.device}) + to_node = Node(phy_graph, uid(), edge.head.name + "_to_" + edge.tail.name, to_operation)._register() + Edge((edge.head, edge.head_slot), (to_node, None), _internal=True)._register() + copied_op[(edge.head, tail_placement)] = to_node + edge.head = to_node + edge.head_slot = None + + # merge all input nodes into one with multiple slots + input_nodes = [] + for node in phy_graph.hidden_nodes: + if isinstance(node.operation, _IOPseudoOperation) and node.operation.type == '_inputs': + input_nodes.append(node) + + for edge in phy_graph.edges: + if edge.head in input_nodes: + edge.head_slot = input_slot_mapping[edge.head] + edge.head = phy_graph.input_node + + # merge all output nodes into one with multiple slots + output_nodes = [] + for node in phy_graph.hidden_nodes: + if isinstance(node.operation, _IOPseudoOperation) and node.operation.type == '_outputs': + output_nodes.append(node) + + for edge in phy_graph.edges: + if edge.tail in output_nodes: + edge.tail_slot = output_slot_mapping[edge.tail] + edge.tail = phy_graph.output_node + + for node in input_nodes: + node.remove() + for node in output_nodes: + node.remove() + + return phy_model, node_placements + + def node_replace(self, old_node: Node, new_node: Node, input_slot_mapping=None, output_slot_mapping=None): + # TODO: currently, only support single input slot and output slot. + if input_slot_mapping is not None or output_slot_mapping is not None: + raise ValueError('Slot mapping is not supported') + + phy_graph = old_node.graph + new_node.graph = phy_graph + + new_node._register() + + for edge in phy_graph.edges: + if edge.head == old_node: + edge.head = new_node + elif edge.tail == old_node: + edge.tail = new_node + + # after the replacement, there might be multiple duplicated edges + # with the same input and output nodes, which should be de-duplicated + self._remove_duplicated_edges() + + def _remove_duplicated_edges(self): + # TODO: it does not have duplicated edges if only supporting dedup input + # Duplicated edges appear when a chain of prefix nodes are deduplicated + pass diff --git a/new_impl/cv/third_party/nni_new/retiarii/execution/logical_optimizer/opt_dedup_input.py b/new_impl/cv/third_party/nni_new/retiarii/execution/logical_optimizer/opt_dedup_input.py new file mode 100644 index 0000000000000000000000000000000000000000..4b210d157b3b5fe63038fbf5e05c4f82d74e29e7 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/retiarii/execution/logical_optimizer/opt_dedup_input.py @@ -0,0 +1,92 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from typing import List, Dict, Tuple + +from nni.retiarii.utils import uid +from ...graph import Graph, Model, Node +from .interface import AbstractOptimizer +from .logical_plan import (AbstractLogicalNode, LogicalGraph, LogicalPlan, + OriginNode, PhysicalDevice) + +_supported_training_modules = ['nni.retiarii.trainer.pytorch.PyTorchImageClassificationTrainer'] + + +class DedupInputNode(AbstractLogicalNode): + def __init__(self, logical_graph: LogicalGraph, node_id: int, + nodes_to_dedup: List[Node], _internal=False): + super().__init__(logical_graph, node_id, + "Dedup_"+nodes_to_dedup[0].name, + nodes_to_dedup[0].operation) + self.origin_nodes: List[OriginNode] = nodes_to_dedup.copy() + + def assemble(self, multi_model_placement: Dict[Model, PhysicalDevice]) -> Tuple[Node, PhysicalDevice]: + for node in self.origin_nodes: + if node.original_graph.model in multi_model_placement: + new_node = Node(node.original_graph, node.id, + f'M_{node.original_graph.model.model_id}_{node.name}', + node.operation) + return new_node, multi_model_placement[node.original_graph.model] + raise ValueError(f'DedupInputNode {self.name} does not contain nodes from multi_model') + + def _fork_to(self, graph: Graph): + DedupInputNode(graph, self.id, self.origin_nodes)._register() + + def __repr__(self) -> str: + return f'DedupNode(id={self.id}, name={self.name}, \ + len(nodes_to_dedup)={len(self.origin_nodes)}' + + +class DedupInputOptimizer(AbstractOptimizer): + def __init__(self) -> None: + pass + + def _check_deduplicate_by_node(self, root_node, node_to_check): + if root_node == node_to_check: + return True + if root_node.operation.type == '_inputs' and \ + node_to_check.operation.type == '_inputs' and \ + isinstance(root_node, OriginNode) and \ + isinstance(node_to_check, OriginNode): + if root_node.original_graph.model.evaluator.module not in _supported_training_modules: + return False + if root_node.original_graph.model.evaluator == node_to_check.original_graph.model.evaluator: + return True + else: + return False + else: + return False + + def convert(self, logical_plan: LogicalPlan) -> None: + nodes_to_skip = set() + while True: # repeat until the logical_graph converges + input_nodes = logical_plan.logical_graph.get_nodes_by_type("_inputs") + # _PseudoOperation(type_name="_inputs")) + root_node = None + for node in input_nodes: + if node in nodes_to_skip: + continue + root_node = node + break + if root_node == None: + break # end of convert + else: + nodes_to_dedup = [] + for node in input_nodes: + if node in nodes_to_skip: + continue + if self._check_deduplicate_by_node(root_node, node): + nodes_to_dedup.append(node) + assert(len(nodes_to_dedup) >= 1) + if len(nodes_to_dedup) == 1: + assert(nodes_to_dedup[0] == root_node) + nodes_to_skip.add(root_node) + else: + dedup_node = DedupInputNode(logical_plan.logical_graph, uid(), nodes_to_dedup)._register() + for edge in logical_plan.logical_graph.edges: + if edge.head in nodes_to_dedup: + edge.head = dedup_node + if edge.tail in nodes_to_dedup: + edge.tail = dedup_node + for node in nodes_to_dedup: + node.remove() diff --git a/new_impl/cv/third_party/nni_new/retiarii/execution/python.py b/new_impl/cv/third_party/nni_new/retiarii/execution/python.py new file mode 100644 index 0000000000000000000000000000000000000000..2a5e7a11a908dff343c12a96c2361c90daf3a4ac --- /dev/null +++ b/new_impl/cv/third_party/nni_new/retiarii/execution/python.py @@ -0,0 +1,57 @@ +from typing import Dict, Any, List + +from ..graph import Evaluator, Model +from ..integration_api import receive_trial_parameters +from ..utils import ContextStack, import_, get_importable_name +from .base import BaseExecutionEngine + + +class PythonGraphData: + def __init__(self, class_name: str, init_parameters: Dict[str, Any], + mutation: Dict[str, Any], evaluator: Evaluator) -> None: + self.class_name = class_name + self.init_parameters = init_parameters + self.mutation = mutation + self.evaluator = evaluator + + def dump(self) -> dict: + return { + 'class_name': self.class_name, + 'init_parameters': self.init_parameters, + 'mutation': self.mutation, + 'evaluator': self.evaluator + } + + @staticmethod + def load(data) -> 'PythonGraphData': + return PythonGraphData(data['class_name'], data['init_parameters'], data['mutation'], data['evaluator']) + + +class PurePythonExecutionEngine(BaseExecutionEngine): + @classmethod + def pack_model_data(cls, model: Model) -> Any: + mutation = get_mutation_dict(model) + graph_data = PythonGraphData(get_importable_name(model.python_class, relocate_module=True), + model.python_init_params, mutation, model.evaluator) + return graph_data + + @classmethod + def trial_execute_graph(cls) -> None: + graph_data = PythonGraphData.load(receive_trial_parameters()) + + class _model(import_(graph_data.class_name)): + def __init__(self): + super().__init__(**graph_data.init_parameters) + + with ContextStack('fixed', graph_data.mutation): + graph_data.evaluator._execute(_model) + + +def _unpack_if_only_one(ele: List[Any]): + if len(ele) == 1: + return ele[0] + return ele + + +def get_mutation_dict(model: Model): + return {mut.mutator.label: _unpack_if_only_one(mut.samples) for mut in model.history} diff --git a/new_impl/cv/third_party/nni_new/retiarii/experiment/pytorch.py b/new_impl/cv/third_party/nni_new/retiarii/experiment/pytorch.py new file mode 100644 index 0000000000000000000000000000000000000000..0780439cffcd79d9d881b69f932d78df3d916ba7 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/retiarii/experiment/pytorch.py @@ -0,0 +1,353 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import atexit +import logging +import time +from dataclasses import dataclass +import os +from pathlib import Path +import socket +from subprocess import Popen +from threading import Thread +import time +from typing import Any, List, Optional, Union + +import colorama +import psutil + +import torch +import torch.nn as nn +import nni.runtime.log +from nni.experiment import Experiment, TrainingServiceConfig +from nni.experiment import management, launcher, rest +from nni.experiment.config import util +from nni.experiment.config.base import ConfigBase, PathLike +from nni.experiment.pipe import Pipe +from nni.tools.nnictl.command_utils import kill_command + +from ..codegen import model_to_pytorch_script +from ..converter import convert_to_graph +from ..execution import list_models, set_execution_engine +from ..execution.python import get_mutation_dict +from ..graph import Model, Evaluator +from ..integration import RetiariiAdvisor +from ..mutator import Mutator +from ..nn.pytorch.mutator import process_inline_mutation, extract_mutation_from_pt_module +from ..strategy import BaseStrategy +from ..oneshot.interface import BaseOneShotTrainer + +_logger = logging.getLogger(__name__) + + +@dataclass(init=False) +class RetiariiExeConfig(ConfigBase): + experiment_name: Optional[str] = None + search_space: Any = '' # TODO: remove + trial_command: str = '_reserved' + trial_code_directory: PathLike = '.' + trial_concurrency: int + trial_gpu_number: int = 0 + max_experiment_duration: Optional[str] = None + max_trial_number: Optional[int] = None + nni_manager_ip: Optional[str] = None + debug: bool = False + log_level: Optional[str] = None + experiment_working_directory: PathLike = '~/nni-experiments' + # remove configuration of tuner/assessor/advisor + training_service: TrainingServiceConfig + execution_engine: str = 'py' + + def __init__(self, training_service_platform: Optional[str] = None, **kwargs): + super().__init__(**kwargs) + if training_service_platform is not None: + assert 'training_service' not in kwargs + self.training_service = util.training_service_config_factory(platform = training_service_platform) + self.__dict__['trial_command'] = 'python3 -m nni.retiarii.trial_entry py' + + def __setattr__(self, key, value): + fixed_attrs = {'search_space': '', + 'trial_command': '_reserved'} + if key in fixed_attrs and fixed_attrs[key] != value: + raise AttributeError(f'{key} is not supposed to be set in Retiarii mode by users!') + # 'trial_code_directory' is handled differently because the path will be converted to absolute path by us + if key == 'trial_code_directory' and not (value == Path('.') or os.path.isabs(value)): + raise AttributeError(f'{key} is not supposed to be set in Retiarii mode by users!') + if key == 'execution_engine': + assert value in ['base', 'py', 'cgo'], f'The specified execution engine "{value}" is not supported.' + self.__dict__['trial_command'] = 'python3 -m nni.retiarii.trial_entry ' + value + self.__dict__[key] = value + + def validate(self, initialized_tuner: bool = False) -> None: + super().validate() + + @property + def _canonical_rules(self): + return _canonical_rules + + @property + def _validation_rules(self): + return _validation_rules + + +_canonical_rules = { + 'trial_code_directory': util.canonical_path, + 'max_experiment_duration': lambda value: f'{util.parse_time(value)}s' if value is not None else None, + 'experiment_working_directory': util.canonical_path +} + +_validation_rules = { + 'trial_code_directory': lambda value: (Path(value).is_dir(), f'"{value}" does not exist or is not directory'), + 'trial_concurrency': lambda value: value > 0, + 'trial_gpu_number': lambda value: value >= 0, + 'max_experiment_duration': lambda value: util.parse_time(value) > 0, + 'max_trial_number': lambda value: value > 0, + 'log_level': lambda value: value in ["trace", "debug", "info", "warning", "error", "fatal"], + 'training_service': lambda value: (type(value) is not TrainingServiceConfig, 'cannot be abstract base class') +} + +def preprocess_model(base_model, trainer, applied_mutators, full_ir=True): + # TODO: this logic might need to be refactored into execution engine + if full_ir: + try: + script_module = torch.jit.script(base_model) + except Exception as e: + _logger.error('Your base model cannot be parsed by torch.jit.script, please fix the following error:') + raise e + base_model_ir = convert_to_graph(script_module, base_model) + # handle inline mutations + mutators = process_inline_mutation(base_model_ir) + else: + base_model_ir, mutators = extract_mutation_from_pt_module(base_model) + base_model_ir.evaluator = trainer + + if mutators is not None and applied_mutators: + raise RuntimeError('Have not supported mixed usage of LayerChoice/InputChoice and mutators, ' + 'do not use mutators when you use LayerChoice/InputChoice') + if mutators is not None: + applied_mutators = mutators + return base_model_ir, applied_mutators + +def debug_mutated_model(base_model, trainer, applied_mutators): + """ + Locally run only one trial without launching an experiment for debug purpose, then exit. + For example, it can be used to quickly check shape mismatch. + + Specifically, it applies mutators (default to choose the first candidate for the choices) + to generate a new model, then run this model locally. + + Parameters + ---------- + base_model : nni.retiarii.nn.pytorch.nn.Module + the base model + trainer : nni.retiarii.evaluator + the training class of the generated models + applied_mutators : list + a list of mutators that will be applied on the base model for generating a new model + """ + base_model_ir, applied_mutators = preprocess_model(base_model, trainer, applied_mutators) + from ..strategy import _LocalDebugStrategy + strategy = _LocalDebugStrategy() + strategy.run(base_model_ir, applied_mutators) + _logger.info('local debug completed!') + + +class RetiariiExperiment(Experiment): + def __init__(self, base_model: nn.Module, trainer: Union[Evaluator, BaseOneShotTrainer], + applied_mutators: List[Mutator] = None, strategy: BaseStrategy = None): + # TODO: The current design of init interface of Retiarii experiment needs to be reviewed. + self.config: RetiariiExeConfig = None + self.port: Optional[int] = None + + self.base_model = base_model + self.trainer = trainer + self.applied_mutators = applied_mutators + self.strategy = strategy + + self._dispatcher = RetiariiAdvisor() + self._dispatcher_thread: Optional[Thread] = None + self._proc: Optional[Popen] = None + self._pipe: Optional[Pipe] = None + + def _start_strategy(self): + base_model_ir, self.applied_mutators = preprocess_model( + self.base_model, self.trainer, self.applied_mutators, full_ir=self.config.execution_engine != 'py') + + _logger.info('Start strategy...') + self.strategy.run(base_model_ir, self.applied_mutators) + _logger.info('Strategy exit') + # TODO: find out a proper way to show no more trial message on WebUI + #self._dispatcher.mark_experiment_as_ending() + + def start(self, port: int = 8080, debug: bool = False) -> None: + """ + Start the experiment in background. + This method will raise exception on failure. + If it returns, the experiment should have been successfully started. + Parameters + ---------- + port + The port of web UI. + debug + Whether to start in debug mode. + """ + atexit.register(self.stop) + + # we will probably need a execution engine factory to make this clean and elegant + if self.config.execution_engine == 'base': + from ..execution.base import BaseExecutionEngine + engine = BaseExecutionEngine() + elif self.config.execution_engine == 'cgo': + from ..execution.cgo_engine import CGOExecutionEngine + engine = CGOExecutionEngine() + elif self.config.execution_engine == 'py': + from ..execution.python import PurePythonExecutionEngine + engine = PurePythonExecutionEngine() + set_execution_engine(engine) + + self.id = management.generate_experiment_id() + + if self.config.experiment_working_directory is not None: + log_dir = Path(self.config.experiment_working_directory, self.id, 'log') + else: + log_dir = Path.home() / f'nni-experiments/{self.id}/log' + nni.runtime.log.start_experiment_log(self.id, log_dir, debug) + + self._proc, self._pipe = launcher.start_experiment_retiarii(self.id, self.config, port, debug) + assert self._proc is not None + assert self._pipe is not None + + self.port = port # port will be None if start up failed + + # dispatcher must be launched after pipe initialized + # the logic to launch dispatcher in background should be refactored into dispatcher api + self._dispatcher = self._create_dispatcher() + self._dispatcher_thread = Thread(target=self._dispatcher.run) + self._dispatcher_thread.start() + + ips = [self.config.nni_manager_ip] + for interfaces in psutil.net_if_addrs().values(): + for interface in interfaces: + if interface.family == socket.AF_INET: + ips.append(interface.address) + ips = [f'http://{ip}:{port}' for ip in ips if ip] + msg = 'Web UI URLs: ' + colorama.Fore.CYAN + ' '.join(ips) + colorama.Style.RESET_ALL + _logger.info(msg) + + exp_status_checker = Thread(target=self._check_exp_status) + exp_status_checker.start() + self._start_strategy() + # TODO: the experiment should be completed, when strategy exits and there is no running job + _logger.info('Waiting for experiment to become DONE (you can ctrl+c if there is no running trial jobs)...') + exp_status_checker.join() + + def _create_dispatcher(self): + return self._dispatcher + + def run(self, config: RetiariiExeConfig = None, port: int = 8080, debug: bool = False) -> str: + """ + Run the experiment. + This function will block until experiment finish or error. + """ + if isinstance(self.trainer, BaseOneShotTrainer): + self.trainer.fit() + else: + assert config is not None, 'You are using classic search mode, config cannot be None!' + self.config = config + self.start(port, debug) + + def _check_exp_status(self) -> bool: + """ + Run the experiment. + This function will block until experiment finish or error. + Return `True` when experiment done; or return `False` when experiment failed. + """ + try: + while True: + time.sleep(10) + # this if is to deal with the situation that + # nnimanager is cleaned up by ctrl+c first + if self._proc.poll() is None: + status = self.get_status() + else: + return False + if status == 'DONE' or status == 'STOPPED': + return True + if status == 'ERROR': + return False + except KeyboardInterrupt: + _logger.warning('KeyboardInterrupt detected') + finally: + self.stop() + + def stop(self) -> None: + """ + Stop background experiment. + """ + _logger.info('Stopping experiment, please wait...') + atexit.unregister(self.stop) + + if self.id is not None: + nni.runtime.log.stop_experiment_log(self.id) + if self._proc is not None: + try: + # this if is to deal with the situation that + # nnimanager is cleaned up by ctrl+c first + if self._proc.poll() is None: + rest.delete(self.port, '/experiment') + except Exception as e: + _logger.exception(e) + _logger.warning('Cannot gracefully stop experiment, killing NNI process...') + kill_command(self._proc.pid) + + if self._pipe is not None: + self._pipe.close() + if self._dispatcher_thread is not None: + self._dispatcher.stopping = True + self._dispatcher_thread.join(timeout=1) + + self.id = None + self.port = None + self._proc = None + self._pipe = None + self._dispatcher = None + self._dispatcher_thread = None + _logger.info('Experiment stopped') + + def export_top_models(self, top_k: int = 1, optimize_mode: str = 'maximize', formatter: str = 'dict') -> Any: + """ + Export several top performing models. + + For one-shot algorithms, only top-1 is supported. For others, ``optimize_mode`` and ``formatter`` are + available for customization. + + top_k : int + How many models are intended to be exported. + optimize_mode : str + ``maximize`` or ``minimize``. Not supported by one-shot algorithms. + ``optimize_mode`` is likely to be removed and defined in strategy in future. + formatter : str + Support ``code`` and ``dict``. Not supported by one-shot algorithms. + If ``code``, the python code of model will be returned. + If ``dict``, the mutation history will be returned. + """ + if formatter == 'code': + assert self.config.execution_engine != 'py', 'You should use `dict` formatter when using Python execution engine.' + if isinstance(self.trainer, BaseOneShotTrainer): + assert top_k == 1, 'Only support top_k is 1 for now.' + return self.trainer.export() + else: + all_models = filter(lambda m: m.metric is not None, list_models()) + assert optimize_mode in ['maximize', 'minimize'] + all_models = sorted(all_models, key=lambda m: m.metric, reverse=optimize_mode == 'maximize') + assert formatter in ['code', 'dict'], 'Export formatter other than "code" and "dict" is not supported yet.' + if formatter == 'code': + return [model_to_pytorch_script(model) for model in all_models[:top_k]] + elif formatter == 'dict': + return [get_mutation_dict(model) for model in all_models[:top_k]] + + def retrain_model(self, model): + """ + this function retrains the exported model, and test it to output test accuracy + """ + raise NotImplementedError diff --git a/new_impl/cv/third_party/nni_new/retiarii/graph.py b/new_impl/cv/third_party/nni_new/retiarii/graph.py new file mode 100644 index 0000000000000000000000000000000000000000..3eba65805dc6532353cf7c5e35aaf375c07a010a --- /dev/null +++ b/new_impl/cv/third_party/nni_new/retiarii/graph.py @@ -0,0 +1,743 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +Model representation. +""" + +import abc +import json +from enum import Enum +from typing import (Any, Dict, Iterable, List, Optional, Tuple, Type, Union, overload) + +from .operation import Cell, Operation, _IOPseudoOperation +from .utils import get_importable_name, import_, uid + +__all__ = ['Model', 'ModelStatus', 'Graph', 'Node', 'Edge', 'Mutation', 'IllegalGraphError', 'MetricData'] + + +MetricData = Any +""" +Type hint for graph metrics (loss, accuracy, etc). +""" + +EdgeEndpoint = Tuple['Node', Optional[int]] +""" +Type hint for edge's endpoint. The int indicates nodes' order. +""" + + +class Evaluator(abc.ABC): + """ + Evaluator of a model. An evaluator should define where the training code is, and the configuration of + training code. The configuration includes basic runtime information trainer needs to know (such as number of GPUs) + or tune-able parameters (such as learning rate), depending on the implementation of training code. + + Each config should define how it is interpreted in ``_execute()``, taking only one argument which is the mutated model class. + For example, functional evaluator might directly import the function and call the function. + """ + + def __repr__(self): + items = ', '.join(['%s=%r' % (k, v) for k, v in self.__dict__.items()]) + return f'{self.__class__.__name__}({items})' + + @abc.abstractstaticmethod + def _load(ir: Any) -> 'Evaluator': + pass + + @staticmethod + def _load_with_type(type_name: str, ir: Any) -> 'Optional[Evaluator]': + if type_name == '_debug_no_trainer': + return DebugEvaluator() + config_cls = import_(type_name) + assert issubclass(config_cls, Evaluator) + return config_cls._load(ir) + + @abc.abstractmethod + def _dump(self) -> Any: + pass + + @abc.abstractmethod + def _execute(self, model_cls: type) -> Any: + pass + + @abc.abstractmethod + def __eq__(self, other) -> bool: + pass + + +class Model: + """ + Represents a neural network model. + + During mutation, one `Model` object is created for each trainable snapshot. + For example, consider a mutator that insert a node at an edge for each iteration. + In one iteration, the mutator invokes 4 primitives: add node, remove edge, add edge to head, add edge to tail. + These 4 primitives operates in one `Model` object. + When they are all done the model will be set to "frozen" (trainable) status and be submitted to execution engine. + And then a new iteration starts, and a new `Model` object is created by forking last model. + + Attributes + ---------- + python_class + Python class that base model is converted from. + python_init_params + Initialization parameters of python class. + status + See `ModelStatus`. + root_graph + The outermost graph which usually takes dataset as input and feeds output to loss function. + graphs + All graphs (subgraphs) in this model. + evaluator + Model evaluator + history + Mutation history. + `self` is directly mutated from `self.history[-1]`; + `self.history[-1] is mutated from `self.history[-2]`, and so on. + `self.history[0]` is the base graph. + metric + Training result of the model, or `None` if it's not yet trained or has failed to train. + intermediate_metrics + Intermediate training metrics. If the model is not trained, it's an empty list. + """ + + def __init__(self, _internal=False): + assert _internal, '`Model()` is private, use `model.fork()` instead' + self.model_id: int = uid('model') + self.python_class: Optional[Type] = None + self.python_init_params: Optional[Dict[str, Any]] = None + + self.status: ModelStatus = ModelStatus.Mutating + + self._root_graph_name: str = '_model' + self.graphs: Dict[str, Graph] = {} + self.evaluator: Optional[Evaluator] = None + + self.history: List[Model] = [] + + self.metric: Optional[MetricData] = None + self.intermediate_metrics: List[MetricData] = [] + + def __repr__(self): + return f'Model(model_id={self.model_id}, status={self.status}, graphs={list(self.graphs.keys())}, ' + \ + f'evaluator={self.evaluator}, metric={self.metric}, intermediate_metrics={self.intermediate_metrics}, ' + \ + f'python_class={self.python_class})' + + @property + def root_graph(self) -> 'Graph': + return self.graphs[self._root_graph_name] + + def fork(self) -> 'Model': + """ + Create a new model which has same topology, names, and IDs to current one. + + Can only be invoked on a frozen model. + The new model will be in `Mutating` state. + + This API is used in mutator base class. + """ + new_model = Model(_internal=True) + new_model._root_graph_name = self._root_graph_name + new_model.python_class = self.python_class + new_model.python_init_params = self.python_init_params + new_model.graphs = {name: graph._fork_to(new_model) for name, graph in self.graphs.items()} + new_model.evaluator = self.evaluator # TODO this needs a clever copy (not deepcopy) if we need mutation + new_model.history = [*self.history] + # Note: the history is not updated. It will be updated when the model is changed, that is in mutator. + return new_model + + @staticmethod + def _load(ir: Any) -> 'Model': + model = Model(_internal=True) + for graph_name, graph_data in ir.items(): + if graph_name != '_evaluator': + Graph._load(model, graph_name, graph_data)._register() + if '_evaluator' in ir: + model.evaluator = Evaluator._load_with_type(ir['_evaluator']['__type__'], ir['_evaluator']) + return model + + def _dump(self) -> Any: + ret = {name: graph._dump() for name, graph in self.graphs.items()} + if self.evaluator is not None: + ret['_evaluator'] = { + '__type__': get_importable_name(self.evaluator.__class__), + **self.evaluator._dump() + } + return ret + + def get_nodes(self) -> Iterable['Node']: + """ + Traverse through all the nodes. + """ + for graph in self.graphs.values(): + for node in graph.nodes: + yield node + + def get_nodes_by_label(self, label: str) -> List['Node']: + """ + Traverse all the nodes to find the matched node(s) with the given label. + There could be multiple nodes with the same label. Name space name can uniquely + identify a graph or node. + + NOTE: the implementation does not support the class abstration + """ + matched_nodes = [] + for graph in self.graphs.values(): + nodes = graph.get_nodes_by_label(label) + matched_nodes.extend(nodes) + return matched_nodes + + def get_nodes_by_type(self, type_name: str) -> List['Node']: + """ + Traverse all the nodes to find the matched node(s) with the given type. + """ + matched_nodes = [] + for graph in self.graphs.values(): + nodes = graph.get_nodes_by_type(type_name) + matched_nodes.extend(nodes) + return matched_nodes + + def get_node_by_name(self, node_name: str) -> 'Node': + """ + Traverse all the nodes to find the matched node with the given name. + """ + matched_nodes = [] + for graph in self.graphs.values(): + nodes = graph.get_nodes_by_name(node_name) + matched_nodes.extend(nodes) + assert len(matched_nodes) <= 1 + if matched_nodes: + return matched_nodes[0] + else: + return None + + +class ModelStatus(Enum): + """ + The status of model. + + A model is created in `Mutating` status. + When the mutation is done and the model get ready to train, its status becomes `Frozen`. + When training started, the model's status becomes `Training`. + If training is successfully ended, model's `metric` attribute get set and its status becomes `Trained`. + If training failed, the status becomes `Failed`. + """ + Mutating = "mutating" + Frozen = "frozen" + Training = "training" + Trained = "trained" + Failed = "failed" + + +_InputPseudoUid = -1 +_OutputPseudoUid = -2 + + +class Graph: + """ + Graph topology. + + This class simply represents the topology, with no semantic meaning. + All other information like metric, non-graph functions, mutation history, etc should go to `Model`. + + Each graph belongs to and only belongs to one `Model`. + + Attributes + ---------- + model + The model containing (and owning) this graph. + id + Unique ID in the model. + If two models have graphs of identical ID, they are semantically the same graph. + Typically this means one graph is mutated from another, or they are both mutated from one ancestor. + name + Mnemonic name of this graph. It should have an one-to-one mapping with ID. + input_names + Optional mnemonic names of input parameters. + output_names + Optional mnemonic names of output values. + input_node + ... + output_node + ... + hidden_nodes + ... + nodes + All input/output/hidden nodes. + edges + ... + """ + + def __init__(self, model: Model, graph_id: int, name: str = None, _internal: bool = False): + assert _internal, '`Graph()` is private' + + self.model: Model = model + self.id: int = graph_id + self.name: str = name or f'_generated_{graph_id}' + + self.input_node: Node = Node(self, _InputPseudoUid, '_inputs', _IOPseudoOperation('_inputs'), _internal=True) + self.output_node: Node = Node(self, _OutputPseudoUid, '_outputs', _IOPseudoOperation('_outputs'), _internal=True) + self.hidden_nodes: List[Node] = [] + + self.edges: List[Edge] = [] + + def __repr__(self): + return f'Graph(id={self.id}, name={self.name}, ' + \ + f'input_names={self.input_node.operation.io_names}, ' + \ + f'output_names={self.output_node.operation.io_names}, ' + \ + f'num_hidden_nodes={len(self.hidden_nodes)}, num_edges={len(self.edges)})' + + @property + def nodes(self) -> List['Node']: + return [self.input_node, self.output_node] + self.hidden_nodes + + def _add_input(self, input_name) -> None: + if self.input_node.operation.io_names is None: + self.input_node.operation.io_names = [input_name] + else: + self.input_node.operation.io_names.append(input_name) + + def _add_output(self, output_name) -> None: + if self.output_node.operation.io_names is None: + self.output_node.operation.io_names = [output_name] + else: + self.output_node.operation.io_names.append(output_name) + + @overload + def add_node(self, name: str, operation: Operation) -> 'Node': ... + @overload + def add_node(self, name: str, type_name: str, parameters: Dict[str, Any] = {}) -> 'Node': ... + + def add_node(self, name, operation_or_type, parameters={}): + if isinstance(operation_or_type, Operation): + op = operation_or_type + else: + op = Operation.new(operation_or_type, parameters, name) + return Node(self, uid(), name, op, _internal=True)._register() + + @overload + def insert_node_on_edge(self, edge: 'Edge', name: str, operation: Operation) -> 'Node': ... + @overload + def insert_node_on_edge(self, edge: 'Edge', name: str, type_name: str, parameters: Dict[str, Any] = {}) -> 'Node': ... + + def insert_node_on_edge(self, edge, name, operation_or_type, parameters={}) -> 'Node': + if isinstance(operation_or_type, Operation): + op = operation_or_type + else: + op = Operation.new(operation_or_type, parameters, name) + new_node = Node(self, uid(), name, op, _internal=True)._register() + # update edges + self.add_edge((edge.head, edge.head_slot), (new_node, None)) + self.add_edge((new_node, None), (edge.tail, edge.tail_slot)) + self.del_edge(edge) + return new_node + + # mutation + def add_edge(self, head: EdgeEndpoint, tail: EdgeEndpoint) -> 'Edge': + assert head[0].graph is self and tail[0].graph is self + return Edge(head, tail, _internal=True)._register() + + def del_edge(self, edge: 'Edge') -> None: + self.edges.remove(edge) + + def get_node_by_name(self, name: str) -> Optional['Node']: + """ + Returns the node which has specified name; or returns `None` if no node has this name. + """ + found = [node for node in self.nodes if node.name == name] + return found[0] if found else None + + def get_nodes_by_type(self, operation_type: str) -> List['Node']: + """ + Returns nodes whose operation is specified typed. + """ + return [node for node in self.hidden_nodes if node.operation.type == operation_type] + + def get_node_by_id(self, node_id: int) -> Optional['Node']: + """ + Returns the node which has specified name; or returns `None` if no node has this name. + """ + found = [node for node in self.nodes if node.id == node_id] + return found[0] if found else None + + def get_nodes_by_label(self, label: str) -> List['Node']: + return [node for node in self.hidden_nodes if node.label == label] + + def get_nodes_by_name(self, name: str) -> List['Node']: + return [node for node in self.hidden_nodes if node.name == name] + + def topo_sort(self) -> List['Node']: + node_to_fanin = {} + curr_nodes = [] + for node in self.nodes: + fanin = len(node.incoming_edges) + node_to_fanin[node] = fanin + if fanin == 0: + curr_nodes.append(node) + + sorted_nodes = [] + while curr_nodes: + curr_node = curr_nodes.pop(0) + sorted_nodes.append(curr_node) + # use successor_slots because a node may connect to another node multiple times + # to different slots + for successor_slot in curr_node.successor_slots: + successor = successor_slot[0] + node_to_fanin[successor] -= 1 + if node_to_fanin[successor] == 0: + curr_nodes.append(successor) + + for key in node_to_fanin: + assert node_to_fanin[key] == 0, '{}, fanin: {}, predecessor: {}, edges: {}, fanin: {}, keys: {}'.format( + key, + node_to_fanin[key], + key.predecessors[0], + self.edges, + node_to_fanin.values(), + node_to_fanin.keys()) + + return sorted_nodes + + def fork(self) -> 'Graph': + """ + Fork the model and returns corresponding graph in new model. + This shortcut might be helpful because many algorithms only cares about "stem" subgraph instead of whole model. + """ + return self.model.fork().graphs[self.name] + + def __eq__(self, other: object) -> bool: + return self is other + + def _fork_to(self, model: Model, name_prefix='') -> 'Graph': + new_graph = Graph(model, self.id, name_prefix+self.name, _internal=True)._register() + # TODO: use node copy instead + new_graph.input_node.operation.io_names = self.input_node.operation.io_names + new_graph.output_node.operation.io_names = self.output_node.operation.io_names + new_graph.input_node.update_label(self.input_node.label) + new_graph.output_node.update_label(self.output_node.label) + + for node in self.hidden_nodes: + new_node = Node(new_graph, node.id, node.name, node.operation, _internal=True) + new_node.update_label(node.label) + new_node._register() + + id_to_new_node = {node.id: node for node in new_graph.nodes} + + for edge in self.edges: + new_head = id_to_new_node[edge.head.id] + new_tail = id_to_new_node[edge.tail.id] + Edge((new_head, edge.head_slot), (new_tail, edge.tail_slot), _internal=True)._register() + + return new_graph + + def _copy(self) -> 'Graph': + # Copy this graph inside the model. + # The new graph will have identical topology, but its nodes' name and ID will be different. + new_graph = Graph(self.model, uid(), _internal=True)._register() + new_graph.input_node.operation.io_names = self.input_node.operation.io_names + new_graph.output_node.operation.io_names = self.output_node.operation.io_names + new_graph.input_node.update_label(self.input_node.label) + new_graph.output_node.update_label(self.output_node.label) + + id_to_new_node = {} # old node ID -> new node object + + for old_node in self.hidden_nodes: + new_node = Node(new_graph, uid(), None, old_node.operation, _internal=True)._register() + new_node.update_label(old_node.label) + id_to_new_node[old_node.id] = new_node + + for edge in self.edges: + new_head = id_to_new_node[edge.head.id] + new_tail = id_to_new_node[edge.tail.id] + Edge((new_head, edge.head_slot), (new_tail, edge.tail_slot), _internal=True)._register() + + return new_graph + + def _register(self) -> 'Graph': + self.model.graphs[self.name] = self + return self + + @staticmethod + def _load(model: Model, name: str, ir: Any) -> 'Graph': + graph = Graph(model, uid(), name, _internal=True) + graph.input_node.operation.io_names = ir.get('inputs') + graph.output_node.operation.io_names = ir.get('outputs') + for node_name, node_data in ir['nodes'].items(): + Node._load(graph, node_name, node_data)._register() + for edge_data in ir['edges']: + Edge._load(graph, edge_data)._register() + return graph + + def _dump(self) -> Any: + return { + 'inputs': self.input_node.operation.io_names, + 'outputs': self.output_node.operation.io_names, + 'nodes': {node.name: node._dump() for node in self.hidden_nodes}, + 'edges': [edge._dump() for edge in self.edges] + } + + +class Node: + """ + An operation or an opaque subgraph inside a graph. + + Each node belongs to and only belongs to one `Graph`. + Nodes should never be created with constructor. Use `Graph.add_node()` instead. + + The node itself is for topology only. + Information of tensor calculation should all go inside `operation` attribute. + + TODO: parameter of subgraph (cell) + It's easy to assign parameters on cell node, but it's hard to "use" them. + We need to design a way to reference stored cell parameters in inner node operations. + e.g. `self.fc = Linear(self.units)` <- how to express `self.units` in IR? + + Attributes + ---------- + graph + The graph containing this node. + id + Unique ID in the model. + If two models have nodes with same ID, they are semantically the same node. + name + Mnemonic name. It should have an one-to-one mapping with ID. + label + Optional. If two nodes have the same label, they are considered same by the mutator. + operation + ... + cell + Read only shortcut to get the referenced subgraph. + If this node is not a subgraph (is a primitive operation), accessing `cell` will raise an error. + predecessors + Predecessor nodes of this node in the graph. This is an optional mutation helper. + successors + Successor nodes of this node in the graph. This is an optional mutation helper. + incoming_edges + Incoming edges of this node in the graph. This is an optional mutation helper. + outgoing_edges + Outgoing edges of this node in the graph. This is an optional mutation helper. + """ + + def __init__(self, graph, node_id, name, operation, _internal=False): + self.graph: Graph = graph + self.id: int = node_id + self.name: str = name or f'_generated_{node_id}' + # TODO: the operation is likely to be considered editable by end-user and it will be hard to debug + # maybe we should copy it here or make Operation class immutable, in next release + self.operation: Operation = operation + self.label: Optional[str] = None + + def __repr__(self): + return f'Node(id={self.id}, name={self.name}, label={self.label}, operation={self.operation})' + + @property + def predecessors(self) -> List['Node']: + return sorted(set(edge.head for edge in self.incoming_edges), key=(lambda node: node.id)) + + @property + def successors(self) -> List['Node']: + return sorted(set(edge.tail for edge in self.outgoing_edges), key=(lambda node: node.id)) + + @property + def successor_slots(self) -> List[Tuple['Node', Union[int, None]]]: + return set((edge.tail, edge.tail_slot) for edge in self.outgoing_edges) + + @property + def incoming_edges(self) -> List['Edge']: + return [edge for edge in self.graph.edges if edge.tail is self] + + @property + def outgoing_edges(self) -> List['Edge']: + return [edge for edge in self.graph.edges if edge.head is self] + + @property + def cell(self) -> Graph: + assert isinstance(self.operation, Cell) + return self.graph.model.graphs[self.operation.parameters['cell']] + + def update_label(self, label: str) -> None: + self.label = label + + @overload + def update_operation(self, operation: Operation) -> None: ... + @overload + def update_operation(self, type_name: str, parameters: Dict[str, Any] = {}) -> None: ... + + def update_operation(self, operation_or_type, parameters={}): + if isinstance(operation_or_type, Operation): + self.operation = operation_or_type + else: + self.operation = Operation.new(operation_or_type, parameters) + + # mutation + def remove(self) -> None: + assert not self.incoming_edges and not self.outgoing_edges + self.graph.hidden_nodes.remove(self) + + # mutation + def specialize_cell(self) -> Graph: + """ + Only available if the operation is a cell. + Duplicate the cell template and let this node reference to newly created copy. + """ + new_cell = self.cell._copy()._register() + self.operation = Cell(new_cell.name) + return new_cell + + def __eq__(self, other: object) -> bool: + return self is other + + def __hash__(self) -> int: + return hash(id(self)) + + def _register(self) -> 'Node': + self.graph.hidden_nodes.append(self) + return self + + @staticmethod + def _load(graph: Graph, name: str, ir: Any) -> 'Node': + if ir['operation']['type'] == '_cell': + op = Cell(ir['operation']['cell_name'], ir['operation'].get('parameters', {})) + else: + op = Operation.new(ir['operation']['type'], ir['operation'].get('parameters', {})) + node = Node(graph, uid(), name, op) + if 'label' in ir: + node.update_label(ir['label']) + return node + + def _dump(self) -> Any: + ret = {'operation': {'type': self.operation.type, 'parameters': self.operation.parameters}} + if isinstance(self.operation, Cell): + ret['operation']['cell_name'] = self.operation.cell_name + if self.label is not None: + ret['label'] = self.label + return ret + + +class Edge: + """ + A tensor, or "data flow", between two nodes. + + Example forward code snippet: + ``` + a, b, c = split(x) + p = concat(a, c) + q = sum(b, p) + z = relu(q) + ``` + + Edges in above snippet: + + head: (split, 0), tail: (concat, 0) # a in concat + + head: (split, 2), tail: (concat, 1) # c in concat + + head: (split, 1), tail: (sum, -1 or 0) # b in sum + + head: (concat, null), tail: (sum, -1 or 1) # p in sum + + head: (sum, null), tail: (relu, null) # q in relu + + Attributes + ---------- + graph + ... + head + Head node. + tail + Tail node. + head_slot + Index of outputs in head node. + If the node has only one output, this should be `null`. + tail_slot + Index of inputs in tail node. + If the node has only one input, this should be `null`. + If the node does not care about order, this can be `-1`. + """ + + def __init__(self, head: EdgeEndpoint, tail: EdgeEndpoint, _internal: bool = False): + assert _internal, '`Edge()` is private' + self.graph: Graph = head[0].graph + self.head: Node = head[0] + self.tail: Node = tail[0] + self.head_slot: Optional[int] = head[1] + self.tail_slot: Optional[int] = tail[1] + + def __repr__(self): + return f'Edge(head=({self.head}, {self.head_slot}), tail=({self.tail}, {self.tail_slot}))' + + # mutation + def remove(self) -> None: + self.graph.edges.remove(self) + + def _register(self) -> 'Edge': + self.graph.edges.append(self) + return self + + @staticmethod + def _load(graph: Graph, ir: Any) -> 'Edge': + head = graph.get_node_by_name(ir['head'][0]) + tail = graph.get_node_by_name(ir['tail'][0]) + assert head is not None and tail is not None + return Edge((head, ir['head'][1]), (tail, ir['tail'][1]), _internal=True) + + def _dump(self) -> Any: + return { + 'head': [self.head.name, self.head_slot], + 'tail': [self.tail.name, self.tail_slot] + } + + +class Mutation: + """ + An execution of mutation, which consists of four parts: a mutator, a list of decisions (choices), + the model that it comes from, and the model that it becomes. + + In general cases, the mutation logs are not reliable and should not be replayed as the mutators can + be arbitrarily complex. However, for inline mutations, the labels correspond to mutator labels here, + this can be useful for metadata visualization and python execution mode. + + Attributes + ---------- + mutator + Mutator. + samples + Decisions/choices. + from_ + Model that is comes from. + to + Model that it becomes. + """ + + def __init__(self, mutator: 'Mutator', samples: List[Any], from_: Model, to: Model): # noqa: F821 + self.mutator: 'Mutator' = mutator # noqa: F821 + self.samples: List[Any] = samples + self.from_: Model = from_ + self.to: Model = to + + def __repr__(self): + return f'Edge(mutator={self.mutator}, samples={self.samples}, from={self.from_}, to={self.to})' + + +class IllegalGraphError(ValueError): + def __init__(self, graph, *args): + self._debug_dump_graph(graph) + super().__init__(*args) + + @staticmethod + def _debug_dump_graph(graph): + if isinstance(graph, Graph): + graph = graph._dump() + with open('generated/debug.json', 'w') as dump_file: + json.dump(graph, dump_file, indent=4) + + +class DebugEvaluator(Evaluator): + @staticmethod + def _load(ir: Any) -> 'DebugEvaluator': + return DebugEvaluator() + + def _dump(self) -> Any: + return {'__type__': '_debug_no_trainer'} + + def _execute(self, model_cls: type) -> Any: + pass + + def __eq__(self, other) -> bool: + return True diff --git a/new_impl/cv/third_party/nni_new/retiarii/integration.py b/new_impl/cv/third_party/nni_new/retiarii/integration.py new file mode 100644 index 0000000000000000000000000000000000000000..189db5ff5c3df8f178842ee7234c1adaaf964750 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/retiarii/integration.py @@ -0,0 +1,134 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +from typing import Any, Callable + +from nni.runtime.msg_dispatcher_base import MsgDispatcherBase +from nni.runtime.protocol import CommandType, send +from nni.utils import MetricType + +from .graph import MetricData +from .integration_api import register_advisor +from .serializer import json_dumps, json_loads + +_logger = logging.getLogger(__name__) + + +class RetiariiAdvisor(MsgDispatcherBase): + """ + The class is to connect Retiarii components to NNI backend. + + It will function as the main thread when running a Retiarii experiment through NNI. + Strategy will be launched as its thread, who will call APIs in execution engine. Execution + engine will then find the advisor singleton and send payloads to advisor. + + When metrics are sent back, advisor will first receive the payloads, who will call the callback + function (that is a member function in graph listener). + + The conversion advisor provides are minimum. It is only a send/receive module, and execution engine + needs to handle all the rest. + + FIXME + How does advisor exit when strategy exists? + + Attributes + ---------- + send_trial_callback + + request_trial_jobs_callback + + trial_end_callback + + intermediate_metric_callback + + final_metric_callback + """ + + def __init__(self): + super(RetiariiAdvisor, self).__init__() + register_advisor(self) # register the current advisor as the "global only" advisor + self.search_space = None + + self.send_trial_callback: Callable[[dict], None] = None + self.request_trial_jobs_callback: Callable[[int], None] = None + self.trial_end_callback: Callable[[int, bool], None] = None + self.intermediate_metric_callback: Callable[[int, MetricData], None] = None + self.final_metric_callback: Callable[[int, MetricData], None] = None + + self.parameters_count = 0 + + def handle_initialize(self, data): + """callback for initializing the advisor + Parameters + ---------- + data: dict + search space + """ + self.handle_update_search_space(data) + send(CommandType.Initialized, '') + + def send_trial(self, parameters): + """ + Send parameters to NNI. + + Parameters + ---------- + parameters : Any + Any payload. + + Returns + ------- + int + Parameter ID that is assigned to this parameter, + which will be used for identification in future. + """ + self.parameters_count += 1 + new_trial = { + 'parameter_id': self.parameters_count, + 'parameters': parameters, + 'parameter_source': 'algorithm' + } + _logger.debug('New trial sent: %s', new_trial) + send(CommandType.NewTrialJob, json_dumps(new_trial)) + if self.send_trial_callback is not None: + self.send_trial_callback(parameters) # pylint: disable=not-callable + return self.parameters_count + + def mark_experiment_as_ending(self): + send(CommandType.NoMoreTrialJobs, '') + + def handle_request_trial_jobs(self, num_trials): + _logger.debug('Request trial jobs: %s', num_trials) + if self.request_trial_jobs_callback is not None: + self.request_trial_jobs_callback(num_trials) # pylint: disable=not-callable + + def handle_update_search_space(self, data): + _logger.debug('Received search space: %s', data) + self.search_space = data + + def handle_trial_end(self, data): + _logger.debug('Trial end: %s', data) + self.trial_end_callback(json_loads(data['hyper_params'])['parameter_id'], # pylint: disable=not-callable + data['event'] == 'SUCCEEDED') + + def handle_report_metric_data(self, data): + _logger.debug('Metric reported: %s', data) + if data['type'] == MetricType.REQUEST_PARAMETER: + raise ValueError('Request parameter not supported') + elif data['type'] == MetricType.PERIODICAL: + self.intermediate_metric_callback(data['parameter_id'], # pylint: disable=not-callable + self._process_value(data['value'])) + elif data['type'] == MetricType.FINAL: + self.final_metric_callback(data['parameter_id'], # pylint: disable=not-callable + self._process_value(data['value'])) + + @staticmethod + def _process_value(value) -> Any: # hopefully a float + value = json_loads(value) + if isinstance(value, dict): + if 'default' in value: + return value['default'] + else: + return value + return value diff --git a/new_impl/cv/third_party/nni_new/retiarii/integration_api.py b/new_impl/cv/third_party/nni_new/retiarii/integration_api.py new file mode 100644 index 0000000000000000000000000000000000000000..9d96da897ecb9025eaae1bada2684ec01b9334cc --- /dev/null +++ b/new_impl/cv/third_party/nni_new/retiarii/integration_api.py @@ -0,0 +1,49 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import json +from typing import NewType, Any + +import nni + +from .serializer import json_loads + +# NOTE: this is only for passing flake8, we cannot import RetiariiAdvisor +# because it would induce cycled import +RetiariiAdvisor = NewType('RetiariiAdvisor', Any) + +_advisor: 'RetiariiAdvisor' = None + + +def get_advisor() -> 'RetiariiAdvisor': + global _advisor + assert _advisor is not None + return _advisor + + +def register_advisor(advisor: 'RetiariiAdvisor'): + global _advisor + assert _advisor is None + _advisor = advisor + + +def send_trial(parameters: dict) -> int: + """ + Send a new trial. Executed on tuner end. + Return a ID that is the unique identifier for this trial. + """ + return get_advisor().send_trial(parameters) + + +def receive_trial_parameters() -> dict: + """ + Received a new trial. Executed on trial end. + Reload with our json loads because NNI didn't use Retiarii serializer to load the data. + """ + params = nni.get_next_parameter() + params = json_loads(json.dumps(params)) + return params + + +def get_experiment_id() -> str: + return nni.get_experiment_id() diff --git a/new_impl/cv/third_party/nni_new/retiarii/mutator.py b/new_impl/cv/third_party/nni_new/retiarii/mutator.py new file mode 100644 index 0000000000000000000000000000000000000000..e7d570816917e1e48de7a9e07e0cc0ae700a6b2b --- /dev/null +++ b/new_impl/cv/third_party/nni_new/retiarii/mutator.py @@ -0,0 +1,117 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from typing import (Any, Iterable, List, Optional) + +from .graph import Model, Mutation, ModelStatus + + +__all__ = ['Sampler', 'Mutator'] + + +Choice = Any + + +class Sampler: + """ + Handles `Mutator.choice()` calls. + """ + + def choice(self, candidates: List[Choice], mutator: 'Mutator', model: Model, index: int) -> Choice: + raise NotImplementedError() + + def mutation_start(self, mutator: 'Mutator', model: Model) -> None: + pass + + def mutation_end(self, mutator: 'Mutator', model: Model) -> None: + pass + + +class Mutator: + """ + Mutates graphs in model to generate new model. + `Mutator` class will be used in two places: + + 1. Inherit `Mutator` to implement graph mutation logic. + 2. Use `Mutator` subclass to implement NAS strategy. + + In scenario 1, the subclass should implement `Mutator.mutate()` interface with `Mutator.choice()`. + In scenario 2, strategy should use constructor or `Mutator.bind_sampler()` to initialize subclass, + and then use `Mutator.apply()` to mutate model. + For certain mutator subclasses, strategy or sampler can use `Mutator.dry_run()` to predict choice candidates. + # Method names are open for discussion. + + If mutator has a label, in most cases, it means that this mutator is applied to nodes with this label. + """ + + def __init__(self, sampler: Optional[Sampler] = None, label: Optional[str] = None): + self.sampler: Optional[Sampler] = sampler + self.label: Optional[str] = label + self._cur_model: Optional[Model] = None + self._cur_choice_idx: Optional[int] = None + + def bind_sampler(self, sampler: Sampler) -> 'Mutator': + """ + Set the sampler which will handle `Mutator.choice` calls. + """ + self.sampler = sampler + return self + + def apply(self, model: Model) -> Model: + """ + Apply this mutator on a model. + Returns mutated model. + The model will be copied before mutation and the original model will not be modified. + """ + assert self.sampler is not None + copy = model.fork() + self._cur_model = copy + self._cur_choice_idx = 0 + self._cur_samples = [] + self.sampler.mutation_start(self, copy) + self.mutate(copy) + self.sampler.mutation_end(self, copy) + copy.history.append(Mutation(self, self._cur_samples, model, copy)) + copy.status = ModelStatus.Frozen + self._cur_model = None + self._cur_choice_idx = None + return copy + + def dry_run(self, model: Model) -> List[List[Choice]]: + """ + Dry run mutator on a model to collect choice candidates. + If you invoke this method multiple times on same or different models, + it may or may not return identical results, depending on how the subclass implements `Mutator.mutate()`. + """ + sampler_backup = self.sampler + recorder = _RecorderSampler() + self.sampler = recorder + new_model = self.apply(model) + self.sampler = sampler_backup + return recorder.recorded_candidates, new_model + + def mutate(self, model: Model) -> None: + """ + Abstract method to be implemented by subclass. + Mutate a model in place. + """ + raise NotImplementedError() + + def choice(self, candidates: Iterable[Choice]) -> Choice: + """ + Ask sampler to make a choice. + """ + assert self.sampler is not None and self._cur_model is not None and self._cur_choice_idx is not None + ret = self.sampler.choice(list(candidates), self, self._cur_model, self._cur_choice_idx) + self._cur_samples.append(ret) + self._cur_choice_idx += 1 + return ret + + +class _RecorderSampler(Sampler): + def __init__(self): + self.recorded_candidates: List[List[Choice]] = [] + + def choice(self, candidates: List[Choice], *args) -> Choice: + self.recorded_candidates.append(candidates) + return candidates[0] diff --git a/new_impl/cv/third_party/nni_new/retiarii/nn/__init__.py b/new_impl/cv/third_party/nni_new/retiarii/nn/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/new_impl/cv/third_party/nni_new/retiarii/nn/pytorch/__init__.py b/new_impl/cv/third_party/nni_new/retiarii/nn/pytorch/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..5c392164b15c72bebafea0e8c59f369a203f99c3 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/retiarii/nn/pytorch/__init__.py @@ -0,0 +1,3 @@ +from .api import * +from .component import * +from .nn import * diff --git a/new_impl/cv/third_party/nni_new/retiarii/nn/pytorch/api.py b/new_impl/cv/third_party/nni_new/retiarii/nn/pytorch/api.py new file mode 100644 index 0000000000000000000000000000000000000000..69d12fb9080981d87c712488723c4378442a547a --- /dev/null +++ b/new_impl/cv/third_party/nni_new/retiarii/nn/pytorch/api.py @@ -0,0 +1,385 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import copy +import warnings +from collections import OrderedDict +from typing import Any, List, Union, Dict, Optional + +import torch +import torch.nn as nn + +from ...serializer import Translatable, basic_unit +from .utils import generate_new_label, get_fixed_value + + +__all__ = ['LayerChoice', 'InputChoice', 'ValueChoice', 'Placeholder', 'ChosenInputs'] + + +class LayerChoice(nn.Module): + """ + Layer choice selects one of the ``candidates``, then apply it on inputs and return results. + + Layer choice does not allow itself to be nested. + + Parameters + ---------- + candidates : list of nn.Module or OrderedDict + A module list to be selected from. + label : str + Identifier of the layer choice. + + Attributes + ---------- + length : int + Deprecated. Number of ops to choose from. ``len(layer_choice)`` is recommended. + names : list of str + Names of candidates. + choices : list of Module + Deprecated. A list of all candidate modules in the layer choice module. + ``list(layer_choice)`` is recommended, which will serve the same purpose. + + Notes + ----- + ``candidates`` can be a list of modules or a ordered dict of named modules, for example, + + .. code-block:: python + + self.op_choice = LayerChoice(OrderedDict([ + ("conv3x3", nn.Conv2d(3, 16, 128)), + ("conv5x5", nn.Conv2d(5, 16, 128)), + ("conv7x7", nn.Conv2d(7, 16, 128)) + ])) + + Elements in layer choice can be modified or deleted. Use ``del self.op_choice["conv5x5"]`` or + ``self.op_choice[1] = nn.Conv3d(...)``. Adding more choices is not supported yet. + """ + + def __new__(cls, candidates: Union[Dict[str, nn.Module], List[nn.Module]], label: Optional[str] = None, **kwargs): + try: + chosen = get_fixed_value(label) + if isinstance(candidates, list): + return candidates[int(chosen)] + else: + return candidates[chosen] + except AssertionError: + return super().__new__(cls) + + def __init__(self, candidates: Union[Dict[str, nn.Module], List[nn.Module]], label: Optional[str] = None, **kwargs): + super(LayerChoice, self).__init__() + if 'key' in kwargs: + warnings.warn(f'"key" is deprecated. Assuming label.') + label = kwargs['key'] + if 'return_mask' in kwargs: + warnings.warn(f'"return_mask" is deprecated. Ignoring...') + if 'reduction' in kwargs: + warnings.warn(f'"reduction" is deprecated. Ignoring...') + self.candidates = candidates + self._label = generate_new_label(label) + + self.names = [] + if isinstance(candidates, OrderedDict): + for name, module in candidates.items(): + assert name not in ["length", "reduction", "return_mask", "_key", "key", "names"], \ + "Please don't use a reserved name '{}' for your module.".format(name) + self.add_module(name, module) + self.names.append(name) + elif isinstance(candidates, list): + for i, module in enumerate(candidates): + self.add_module(str(i), module) + self.names.append(str(i)) + else: + raise TypeError("Unsupported candidates type: {}".format(type(candidates))) + + @property + def key(self): + return self._key() + + @torch.jit.ignore + def _key(self): + warnings.warn('Using key to access the identifier of LayerChoice is deprecated. Please use label instead.', + category=DeprecationWarning) + return self._label + + @property + def label(self): + return self._label + + def __getitem__(self, idx): + if isinstance(idx, str): + return self._modules[idx] + return list(self)[idx] + + def __setitem__(self, idx, module): + key = idx if isinstance(idx, str) else self.names[idx] + return setattr(self, key, module) + + def __delitem__(self, idx): + if isinstance(idx, slice): + for key in self.names[idx]: + delattr(self, key) + else: + if isinstance(idx, str): + key, idx = idx, self.names.index(idx) + else: + key = self.names[idx] + delattr(self, key) + del self.names[idx] + + def __len__(self): + return len(self.names) + + def __iter__(self): + return map(lambda name: self._modules[name], self.names) + + @property + def choices(self): + return self._choices() + + @torch.jit.ignore + def _choices(self): + warnings.warn("layer_choice.choices is deprecated. Use `list(layer_choice)` instead.", category=DeprecationWarning) + return list(self) + + def forward(self, x): + warnings.warn('You should not run forward of this module directly.') + return x + + def __repr__(self): + return f'LayerChoice({self.candidates}, label={repr(self.label)})' + + +class InputChoice(nn.Module): + """ + Input choice selects ``n_chosen`` inputs from ``choose_from`` (contains ``n_candidates`` keys). + Use ``reduction`` to specify how chosen inputs are reduced into one output. A few options are: + + * ``none``: do nothing and return the list directly. + * ``sum``: summing all the chosen inputs. + * ``mean``: taking the average of all chosen inputs. + * ``concat``: concatenate all chosen inputs at dimension 1. + + We don't support customizing reduction yet. + + Parameters + ---------- + n_candidates : int + Number of inputs to choose from. It is required. + n_chosen : int + Recommended inputs to choose. If None, mutator is instructed to select any. + reduction : str + ``mean``, ``concat``, ``sum`` or ``none``. + label : str + Identifier of the input choice. + """ + + def __new__(cls, n_candidates: int, n_chosen: int = 1, reduction: str = 'sum', label: Optional[str] = None, **kwargs): + try: + return ChosenInputs(get_fixed_value(label), reduction=reduction) + except AssertionError: + return super().__new__(cls) + + def __init__(self, n_candidates: int, n_chosen: int = 1, reduction: str = 'sum', label: Optional[str] = None, **kwargs): + super(InputChoice, self).__init__() + if 'key' in kwargs: + warnings.warn(f'"key" is deprecated. Assuming label.') + label = kwargs['key'] + if 'return_mask' in kwargs: + warnings.warn(f'"return_mask" is deprecated. Ignoring...') + if 'choose_from' in kwargs: + warnings.warn(f'"reduction" is deprecated. Ignoring...') + self.n_candidates = n_candidates + self.n_chosen = n_chosen + self.reduction = reduction + assert self.reduction in ['mean', 'concat', 'sum', 'none'] + self._label = generate_new_label(label) + + @property + def key(self): + return self._key() + + @torch.jit.ignore + def _key(self): + warnings.warn('Using key to access the identifier of InputChoice is deprecated. Please use label instead.', + category=DeprecationWarning) + return self._label + + @property + def label(self): + return self._label + + def forward(self, candidate_inputs: List[torch.Tensor]) -> torch.Tensor: + warnings.warn('You should not run forward of this module directly.') + return candidate_inputs[0] + + def __repr__(self): + return f'InputChoice(n_candidates={self.n_candidates}, n_chosen={self.n_chosen}, ' \ + f'reduction={repr(self.reduction)}, label={repr(self.label)})' + + +class ValueChoice(Translatable, nn.Module): + """ + ValueChoice is to choose one from ``candidates``. + + In most use scenarios, ValueChoice should be passed to the init parameters of a serializable module. For example, + + .. code-block:: python + + class Net(nn.Module): + def __init__(self): + super().__init__() + self.conv = nn.Conv2d(3, nn.ValueChoice([32, 64]), kernel_size=nn.ValueChoice([3, 5, 7])) + + def forward(self, x): + return self.conv(x) + + In case, you want to search a parameter that is used repeatedly, this is also possible by sharing the same value choice instance. + (Sharing the label should have the same effect.) For example, + + .. code-block:: python + + class Net(nn.Module): + def __init__(self): + super().__init__() + hidden_dim = nn.ValueChoice([128, 512]) + self.fc = nn.Sequential( + nn.Linear(64, hidden_dim), + nn.Linear(hidden_dim, 10) + ) + + # the following code has the same effect. + # self.fc = nn.Sequential( + # nn.Linear(64, nn.ValueChoice([128, 512], label='dim')), + # nn.Linear(nn.ValueChoice([128, 512], label='dim'), 10) + # ) + + def forward(self, x): + return self.fc(x) + + Note that ValueChoice should be used directly. Transformations like ``nn.Linear(32, nn.ValueChoice([64, 128]) * 2)`` + are not supported. + + Another common use case is to initialize the values to choose from in init and call the module in forward to get the chosen value. + Usually, this is used to pass a mutable value to a functional API like ``torch.xxx`` or ``nn.functional.xxx```. + For example, + + .. code-block:: python + + class Net(nn.Module): + def __init__(self): + super().__init__() + self.dropout_rate = nn.ValueChoice([0., 1.]) + + def forward(self, x): + return F.dropout(x, self.dropout_rate()) + + Parameters + ---------- + candidates : list + List of values to choose from. + label : str + Identifier of the value choice. + """ + + def __new__(cls, candidates: List[Any], label: Optional[str] = None): + try: + return get_fixed_value(label) + except AssertionError: + return super().__new__(cls) + + def __init__(self, candidates: List[Any], label: Optional[str] = None): + super().__init__() + self.candidates = candidates + self._label = generate_new_label(label) + self._accessor = [] + + @property + def label(self): + return self._label + + def forward(self): + warnings.warn('You should not run forward of this module directly.') + return self.candidates[0] + + def _translate(self): + # Will function as a value when used in serializer. + return self.access(self.candidates[0]) + + def __repr__(self): + return f'ValueChoice({self.candidates}, label={repr(self.label)})' + + def access(self, value): + if not self._accessor: + return value + try: + v = value + for a in self._accessor: + v = v[a] + except KeyError: + raise KeyError(''.join([f'[{a}]' for a in self._accessor]) + f' does not work on {value}') + return v + + def __copy__(self): + return self + + def __deepcopy__(self, memo): + new_item = ValueChoice(self.candidates, self.label) + new_item._accessor = [*self._accessor] + return new_item + + def __getitem__(self, item): + """ + Get a sub-element of value choice. + + The underlying implementation is to clone the current instance, and append item to "accessor", which records all + the history getitem calls. For example, when accessor is ``[a, b, c]``, the value choice will return ``vc[a][b][c]`` + where ``vc`` is the original value choice. + """ + access = copy.deepcopy(self) + access._accessor.append(item) + for candidate in self.candidates: + access.access(candidate) + return access + + +@basic_unit +class Placeholder(nn.Module): + # TODO: docstring + + def __init__(self, label, **related_info): + self.label = label + self.related_info = related_info + super().__init__() + + def forward(self, x): + return x + + +class ChosenInputs(nn.Module): + """ + A module that chooses from a tensor list and outputs a reduced tensor. + The already-chosen version of InputChoice. + """ + + def __init__(self, chosen: Union[List[int], int], reduction: str): + super().__init__() + self.chosen = chosen if isinstance(chosen, list) else [chosen] + self.reduction = reduction + + def forward(self, candidate_inputs): + return self._tensor_reduction(self.reduction, [candidate_inputs[i] for i in self.chosen]) + + def _tensor_reduction(self, reduction_type, tensor_list): + if reduction_type == 'none': + return tensor_list + if not tensor_list: + return None # empty. return None for now + if len(tensor_list) == 1: + return tensor_list[0] + if reduction_type == 'sum': + return sum(tensor_list) + if reduction_type == 'mean': + return sum(tensor_list) / len(tensor_list) + if reduction_type == 'concat': + return torch.cat(tensor_list, dim=1) + raise ValueError(f'Unrecognized reduction policy: "{reduction_type}"') diff --git a/new_impl/cv/third_party/nni_new/retiarii/nn/pytorch/component.py b/new_impl/cv/third_party/nni_new/retiarii/nn/pytorch/component.py new file mode 100644 index 0000000000000000000000000000000000000000..20c6e30de531d1431d61b85af804e8fcb8dbbb93 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/retiarii/nn/pytorch/component.py @@ -0,0 +1,147 @@ +import copy +from typing import Callable, List, Union, Tuple, Optional + +import torch +import torch.nn as nn + +from .api import LayerChoice, InputChoice +from .nn import ModuleList + +from .utils import generate_new_label, get_fixed_value + + +__all__ = ['Repeat', 'Cell'] + + +class Repeat(nn.Module): + """ + Repeat a block by a variable number of times. + + Parameters + ---------- + blocks : function, list of function, module or list of module + The block to be repeated. If not a list, it will be replicated into a list. + If a list, it should be of length ``max_depth``, the modules will be instantiated in order and a prefix will be taken. + If a function, it will be called to instantiate a module. Otherwise the module will be deep-copied. + depth : int or tuple of int + If one number, the block will be repeated by a fixed number of times. If a tuple, it should be (min, max), + meaning that the block will be repeated at least `min` times and at most `max` times. + """ + + def __new__(cls, blocks: Union[Callable[[], nn.Module], List[Callable[[], nn.Module]], nn.Module, List[nn.Module]], + depth: Union[int, Tuple[int, int]], label: Optional[str] = None): + try: + repeat = get_fixed_value(label) + return nn.Sequential(*cls._replicate_and_instantiate(blocks, repeat)) + except AssertionError: + return super().__new__(cls) + + def __init__(self, + blocks: Union[Callable[[], nn.Module], List[Callable[[], nn.Module]], nn.Module, List[nn.Module]], + depth: Union[int, Tuple[int, int]], label: Optional[str] = None): + super().__init__() + self._label = generate_new_label(label) + self.min_depth = depth if isinstance(depth, int) else depth[0] + self.max_depth = depth if isinstance(depth, int) else depth[1] + assert self.max_depth >= self.min_depth > 0 + self.blocks = nn.ModuleList(self._replicate_and_instantiate(blocks, self.max_depth)) + + @property + def label(self): + return self._label + + def forward(self, x): + for block in self.blocks: + x = block(x) + return x + + @staticmethod + def _replicate_and_instantiate(blocks, repeat): + if not isinstance(blocks, list): + if isinstance(blocks, nn.Module): + blocks = [blocks] + [copy.deepcopy(blocks) for _ in range(repeat - 1)] + else: + blocks = [blocks for _ in range(repeat)] + assert len(blocks) > 0 + assert repeat <= len(blocks), f'Not enough blocks to be used. {repeat} expected, only found {len(blocks)}.' + blocks = blocks[:repeat] + if not isinstance(blocks[0], nn.Module): + blocks = [b() for b in blocks] + return blocks + + +class Cell(nn.Module): + """ + Cell structure [zophnas]_ [zophnasnet]_ that is popularly used in NAS literature. + + A cell consists of multiple "nodes". Each node is a sum of multiple operators. Each operator is chosen from + ``op_candidates``, and takes one input from previous nodes and predecessors. Predecessor means the input of cell. + The output of cell is the concatenation of some of the nodes in the cell (currently all the nodes). + + Parameters + ---------- + op_candidates : function or list of module + A list of modules to choose from, or a function that returns a list of modules. + num_nodes : int + Number of nodes in the cell. + num_ops_per_node: int + Number of operators in each node. The output of each node is the sum of all operators in the node. Default: 1. + num_predecessors : int + Number of inputs of the cell. The input to forward should be a list of tensors. Default: 1. + merge_op : str + Currently only ``all`` is supported, which has slight difference with that described in reference. Default: all. + label : str + Identifier of the cell. Cell sharing the same label will semantically share the same choice. + + References + ---------- + .. [zophnas] Barret Zoph, Quoc V. Le, "Neural Architecture Search with Reinforcement Learning". https://arxiv.org/abs/1611.01578 + .. [zophnasnet] Barret Zoph, Vijay Vasudevan, Jonathon Shlens, Quoc V. Le, + "Learning Transferable Architectures for Scalable Image Recognition". https://arxiv.org/abs/1707.07012 + """ + + # TODO: + # Support loose end concat (shape inference on the following cells) + # How to dynamically create convolution with stride as the first node + + def __init__(self, + op_candidates: Union[Callable, List[nn.Module]], + num_nodes: int, + num_ops_per_node: int = 1, + num_predecessors: int = 1, + merge_op: str = 'all', + label: str = None): + super().__init__() + self._label = generate_new_label(label) + self.ops = ModuleList() + self.inputs = ModuleList() + self.num_nodes = num_nodes + self.num_ops_per_node = num_ops_per_node + self.num_predecessors = num_predecessors + for i in range(num_nodes): + self.ops.append(ModuleList()) + self.inputs.append(ModuleList()) + for k in range(num_ops_per_node): + if isinstance(op_candidates, list): + assert len(op_candidates) > 0 and isinstance(op_candidates[0], nn.Module) + ops = copy.deepcopy(op_candidates) + else: + ops = op_candidates() + self.ops[-1].append(LayerChoice(ops, label=f'{self.label}__op_{i}_{k}')) + self.inputs[-1].append(InputChoice(i + num_predecessors, 1, label=f'{self.label}/input_{i}_{k}')) + assert merge_op in ['all'] # TODO: loose_end + self.merge_op = merge_op + + @property + def label(self): + return self._label + + def forward(self, x: List[torch.Tensor]): + states = x + for ops, inps in zip(self.ops, self.inputs): + current_state = [] + for op, inp in zip(ops, inps): + current_state.append(op(inp(states))) + current_state = torch.sum(torch.stack(current_state), 0) + states.append(current_state) + return torch.cat(states[self.num_predecessors:], 1) diff --git a/new_impl/cv/third_party/nni_new/retiarii/nn/pytorch/mutator.py b/new_impl/cv/third_party/nni_new/retiarii/nn/pytorch/mutator.py new file mode 100644 index 0000000000000000000000000000000000000000..4307aeac681bbd631103affa2e21c2c5c7d38f10 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/retiarii/nn/pytorch/mutator.py @@ -0,0 +1,301 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import inspect +from typing import Any, List, Optional, Tuple + +import torch.nn as nn + +from ...mutator import Mutator +from ...graph import Cell, Graph, Model, ModelStatus, Node +from .api import LayerChoice, InputChoice, ValueChoice, Placeholder +from .component import Repeat +from ...utils import uid + + +class LayerChoiceMutator(Mutator): + def __init__(self, nodes: List[Node]): + super().__init__(label=nodes[0].operation.parameters['label']) + self.nodes = nodes + + def mutate(self, model): + candidates = self.nodes[0].operation.parameters['candidates'] + chosen = self.choice(candidates) + for node in self.nodes: + # Each layer choice corresponds to a cell, which is unconnected in the base graph. + # We add the connections here in the mutation logic. + # Thus, the mutated model should not be mutated again. Everything should be based on the original base graph. + target = model.graphs[node.operation.cell_name] + chosen_node = target.get_node_by_name(chosen) + assert chosen_node is not None + target.add_edge((target.input_node, 0), (chosen_node, None)) + target.add_edge((chosen_node, None), (target.output_node, None)) + model.get_node_by_name(node.name).update_operation(Cell(node.operation.cell_name)) + + # remove redundant nodes + for rm_node in list(target.hidden_nodes): # remove from a list on the fly will cause issues + if rm_node.name != chosen_node.name: + rm_node.remove() + + +class InputChoiceMutator(Mutator): + def __init__(self, nodes: List[Node]): + super().__init__(label=nodes[0].operation.parameters['label']) + self.nodes = nodes + + def mutate(self, model): + n_candidates = self.nodes[0].operation.parameters['n_candidates'] + n_chosen = self.nodes[0].operation.parameters['n_chosen'] + candidates = list(range(n_candidates)) + chosen = [self.choice(candidates) for _ in range(n_chosen)] + for node in self.nodes: + target = model.get_node_by_name(node.name) + target.update_operation('__torch__.nni.retiarii.nn.pytorch.ChosenInputs', + {'chosen': chosen, 'reduction': node.operation.parameters['reduction']}) + + +class ValueChoiceMutator(Mutator): + def __init__(self, nodes: List[Node], candidates: List[Any]): + super().__init__(label=nodes[0].operation.parameters['label']) + self.nodes = nodes + self.candidates = candidates + + def mutate(self, model): + chosen = self.choice(self.candidates) + for node in self.nodes: + target = model.get_node_by_name(node.name) + target.update_operation('prim::Constant', {'type': type(chosen).__name__, 'value': chosen}) + + +class ParameterChoiceMutator(Mutator): + def __init__(self, nodes: List[Tuple[Node, str]], candidates: List[Any]): + node, argname = nodes[0] + super().__init__(label=node.operation.parameters[argname].label) + self.nodes = nodes + self.candidates = candidates + + def mutate(self, model): + chosen = self.choice(self.candidates) + for node, argname in self.nodes: + chosen_value = node.operation.parameters[argname].access(chosen) + target = model.get_node_by_name(node.name) + target.update_operation(target.operation.type, {**target.operation.parameters, argname: chosen_value}) + + +class RepeatMutator(Mutator): + def __init__(self, nodes: List[Node]): + # nodes is a subgraph consisting of repeated blocks. + super().__init__(label=nodes[0].operation.parameters['label']) + self.nodes = nodes + + def _retrieve_chain_from_graph(self, graph: Graph) -> List[Node]: + u = graph.input_node + chain = [] + while u != graph.output_node: + if u != graph.input_node: + chain.append(u) + assert len(u.successors) == 1, f'This graph is an illegal chain. {u} has output {u.successor}.' + u = u.successors[0] + return chain + + def mutate(self, model): + min_depth = self.nodes[0].operation.parameters['min_depth'] + max_depth = self.nodes[0].operation.parameters['max_depth'] + if min_depth < max_depth: + chosen_depth = self.choice(list(range(min_depth, max_depth + 1))) + for node in self.nodes: + # the logic here is similar to layer choice. We find cell attached to each node. + target: Graph = model.graphs[node.operation.cell_name] + chain = self._retrieve_chain_from_graph(target) + for edge in chain[chosen_depth - 1].outgoing_edges: + edge.remove() + target.add_edge((chain[chosen_depth - 1], None), (target.output_node, None)) + for rm_node in chain[chosen_depth:]: + for edge in rm_node.outgoing_edges: + edge.remove() + rm_node.remove() + # to delete the unused parameters. + model.get_node_by_name(node.name).update_operation(Cell(node.operation.cell_name)) + + +def process_inline_mutation(model: Model) -> Optional[List[Mutator]]: + applied_mutators = [] + + ic_nodes = _group_by_label(model.get_nodes_by_type('__torch__.nni.retiarii.nn.pytorch.api.InputChoice')) + for node_list in ic_nodes: + assert _is_all_equal(map(lambda node: node.operation.parameters['n_candidates'], node_list)) and \ + _is_all_equal(map(lambda node: node.operation.parameters['n_chosen'], node_list)), \ + 'Input choice with the same label must have the same number of candidates.' + mutator = InputChoiceMutator(node_list) + applied_mutators.append(mutator) + + vc_nodes = _group_by_label(model.get_nodes_by_type('__torch__.nni.retiarii.nn.pytorch.api.ValueChoice')) + for node_list in vc_nodes: + assert _is_all_equal(map(lambda node: node.operation.parameters['candidates'], node_list)), \ + 'Value choice with the same label must have the same candidates.' + mutator = ValueChoiceMutator(node_list, node_list[0].operation.parameters['candidates']) + applied_mutators.append(mutator) + + pc_nodes = [] + for node in model.get_nodes(): + for name, choice in node.operation.parameters.items(): + if isinstance(choice, ValueChoice): + pc_nodes.append((node, name)) + pc_nodes = _group_parameters_by_label(pc_nodes) + for node_list in pc_nodes: + assert _is_all_equal([node.operation.parameters[name].candidates for node, name in node_list]), \ + 'Value choice with the same label must have the same candidates.' + first_node, first_argname = node_list[0] + mutator = ParameterChoiceMutator(node_list, first_node.operation.parameters[first_argname].candidates) + applied_mutators.append(mutator) + + # apply layer choice at last as it will delete some nodes + lc_nodes = _group_by_label(filter(lambda d: d.operation.parameters.get('mutation') == 'layerchoice', + model.get_nodes_by_type('_cell'))) + for node_list in lc_nodes: + assert _is_all_equal(map(lambda node: len(node.operation.parameters['candidates']), node_list)), \ + 'Layer choice with the same label must have the same number of candidates.' + mutator = LayerChoiceMutator(node_list) + applied_mutators.append(mutator) + + repeat_nodes = _group_by_label(filter(lambda d: d.operation.parameters.get('mutation') == 'repeat', + model.get_nodes_by_type('_cell'))) + for node_list in repeat_nodes: + assert _is_all_equal(map(lambda node: node.operation.parameters['max_depth'], node_list)) and \ + _is_all_equal(map(lambda node: node.operation.parameters['min_depth'], node_list)), \ + 'Repeat with the same label must have the same number of candidates.' + mutator = RepeatMutator(node_list) + applied_mutators.append(mutator) + + if applied_mutators: + return applied_mutators + return None + + +# The following are written for pure-python mode + + +class ManyChooseManyMutator(Mutator): + """ + Choose based on labels. Will not affect the model itself. + """ + + def __init__(self, label: Optional[str]): + super().__init__(label=label) + + @staticmethod + def candidates(node): + if 'n_candidates' in node.operation.parameters: + return list(range(node.operation.parameters['n_candidates'])) + else: + return node.operation.parameters['candidates'] + + @staticmethod + def number_of_chosen(node): + if 'n_chosen' in node.operation.parameters: + return node.operation.parameters['n_chosen'] + return 1 + + def mutate(self, model: Model): + # this mutate does not have any effect, but it is recorded in the mutation history + for node in model.get_nodes_by_label(self.label): + for _ in range(self.number_of_chosen(node)): + self.choice(self.candidates(node)) + break + + +def extract_mutation_from_pt_module(pytorch_model: nn.Module) -> Tuple[Model, Optional[List[Mutator]]]: + model = Model(_internal=True) + graph = Graph(model, uid(), '_model', _internal=True)._register() + model.python_class = pytorch_model.__class__ + if len(inspect.signature(model.python_class.__init__).parameters) > 1: + if not hasattr(pytorch_model, '_init_parameters'): + raise ValueError('Please annotate the model with @serialize decorator in python execution mode ' + 'if your model has init parameters.') + model.python_init_params = pytorch_model._init_parameters + else: + model.python_init_params = {} + + for name, module in pytorch_model.named_modules(): + # tricky case: value choice that serves as parameters are stored in _init_parameters + if hasattr(module, '_init_parameters'): + for key, value in module._init_parameters.items(): + if isinstance(value, ValueChoice): + node = graph.add_node(name + '.init.' + key, 'ValueChoice', {'candidates': value.candidates}) + node.label = value.label + + if isinstance(module, (LayerChoice, InputChoice, ValueChoice)): + # TODO: check the label of module and warn if it's auto-generated + pass + if isinstance(module, LayerChoice): + node = graph.add_node(name, 'LayerChoice', {'candidates': module.names}) + node.label = module.label + if isinstance(module, InputChoice): + node = graph.add_node(name, 'InputChoice', + {'n_candidates': module.n_candidates, 'n_chosen': module.n_chosen}) + node.label = module.label + if isinstance(module, ValueChoice): + node = graph.add_node(name, 'ValueChoice', {'candidates': module.candidates}) + node.label = module.label + if isinstance(module, Repeat) and module.min_depth <= module.max_depth: + node = graph.add_node(name, 'Repeat', { + 'candidates': list(range(module.min_depth, module.max_depth + 1)) + }) + node.label = module.label + if isinstance(module, Placeholder): + raise NotImplementedError('Placeholder is not supported in python execution mode.') + + model.status = ModelStatus.Frozen + if not graph.hidden_nodes: + return model, None + + mutators = [] + for nodes in _group_by_label_and_type(graph.hidden_nodes): + assert _is_all_equal(map(lambda n: n.operation.type, nodes)), \ + f'Node with label "{nodes[0].label}" does not all have the same type.' + assert _is_all_equal(map(lambda n: n.operation.parameters, nodes)), \ + f'Node with label "{nodes[0].label}" does not agree on parameters.' + mutators.append(ManyChooseManyMutator(nodes[0].label)) + return model, mutators + + +# utility functions + + +def _is_all_equal(lst): + last = None + for x in lst: + if last is not None and last != x: + return False + last = x + return True + + +def _group_by_label_and_type(nodes: List[Node]) -> List[List[Node]]: + result = {} + for node in nodes: + key = (node.label, node.operation.type) + if key not in result: + result[key] = [] + result[key].append(node) + return list(result.values()) + + +def _group_by_label(nodes: List[Node]) -> List[List[Node]]: + result = {} + for node in nodes: + label = node.operation.parameters['label'] + if label not in result: + result[label] = [] + result[label].append(node) + return list(result.values()) + + +def _group_parameters_by_label(nodes: List[Tuple[Node, str]]) -> List[List[Tuple[Node, str]]]: + result = {} + for node, argname in nodes: + label = node.operation.parameters[argname].label + if label not in result: + result[label] = [] + result[label].append((node, argname)) + return list(result.values()) diff --git a/new_impl/cv/third_party/nni_new/retiarii/nn/pytorch/nn.py b/new_impl/cv/third_party/nni_new/retiarii/nn/pytorch/nn.py new file mode 100644 index 0000000000000000000000000000000000000000..1cbf0f9e6ee6a9be501bd8ef30369dfec910af0a --- /dev/null +++ b/new_impl/cv/third_party/nni_new/retiarii/nn/pytorch/nn.py @@ -0,0 +1,159 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import torch +import torch.nn as nn + +from ...serializer import basic_unit +from ...serializer import transparent_serialize +from ...utils import version_larger_equal + +# NOTE: support pytorch version >= 1.5.0 + +__all__ = [ + 'Module', 'Sequential', 'ModuleList', # TODO: 'ModuleDict', 'ParameterList', 'ParameterDict', + 'Identity', 'Linear', 'Conv1d', 'Conv2d', 'Conv3d', 'ConvTranspose1d', + 'ConvTranspose2d', 'ConvTranspose3d', 'Threshold', 'ReLU', 'Hardtanh', 'ReLU6', + 'Sigmoid', 'Tanh', 'Softmax', 'Softmax2d', 'LogSoftmax', 'ELU', 'SELU', 'CELU', 'GLU', 'GELU', 'Hardshrink', + 'LeakyReLU', 'LogSigmoid', 'Softplus', 'Softshrink', 'MultiheadAttention', 'PReLU', 'Softsign', 'Softmin', + 'Tanhshrink', 'RReLU', 'AvgPool1d', 'AvgPool2d', 'AvgPool3d', 'MaxPool1d', 'MaxPool2d', + 'MaxPool3d', 'MaxUnpool1d', 'MaxUnpool2d', 'MaxUnpool3d', 'FractionalMaxPool2d', "FractionalMaxPool3d", + 'LPPool1d', 'LPPool2d', 'LocalResponseNorm', 'BatchNorm1d', 'BatchNorm2d', 'BatchNorm3d', 'InstanceNorm1d', + 'InstanceNorm2d', 'InstanceNorm3d', 'LayerNorm', 'GroupNorm', 'SyncBatchNorm', + 'Dropout', 'Dropout2d', 'Dropout3d', 'AlphaDropout', 'FeatureAlphaDropout', + 'ReflectionPad1d', 'ReflectionPad2d', 'ReplicationPad2d', 'ReplicationPad1d', 'ReplicationPad3d', + 'CrossMapLRN2d', 'Embedding', 'EmbeddingBag', 'RNNBase', 'RNN', 'LSTM', 'GRU', 'RNNCellBase', 'RNNCell', + 'LSTMCell', 'GRUCell', 'PixelShuffle', 'Upsample', 'UpsamplingNearest2d', 'UpsamplingBilinear2d', + 'PairwiseDistance', 'AdaptiveMaxPool1d', 'AdaptiveMaxPool2d', 'AdaptiveMaxPool3d', 'AdaptiveAvgPool1d', + 'AdaptiveAvgPool2d', 'AdaptiveAvgPool3d', 'TripletMarginLoss', 'ZeroPad2d', 'ConstantPad1d', 'ConstantPad2d', + 'ConstantPad3d', 'Bilinear', 'CosineSimilarity', 'Unfold', 'Fold', + 'AdaptiveLogSoftmaxWithLoss', 'TransformerEncoder', 'TransformerDecoder', + 'TransformerEncoderLayer', 'TransformerDecoderLayer', 'Transformer', + 'Flatten', 'Hardsigmoid' +] + +if version_larger_equal(torch.__version__, '1.6.0'): + __all__.append('Hardswish') + +if version_larger_equal(torch.__version__, '1.7.0'): + __all__.extend(['Unflatten', 'SiLU', 'TripletMarginWithDistanceLoss']) + + +Module = nn.Module + +Sequential = transparent_serialize(nn.Sequential) +ModuleList = transparent_serialize(nn.ModuleList) + +Identity = basic_unit(nn.Identity) +Linear = basic_unit(nn.Linear) +Conv1d = basic_unit(nn.Conv1d) +Conv2d = basic_unit(nn.Conv2d) +Conv3d = basic_unit(nn.Conv3d) +ConvTranspose1d = basic_unit(nn.ConvTranspose1d) +ConvTranspose2d = basic_unit(nn.ConvTranspose2d) +ConvTranspose3d = basic_unit(nn.ConvTranspose3d) +Threshold = basic_unit(nn.Threshold) +ReLU = basic_unit(nn.ReLU) +Hardtanh = basic_unit(nn.Hardtanh) +ReLU6 = basic_unit(nn.ReLU6) +Sigmoid = basic_unit(nn.Sigmoid) +Tanh = basic_unit(nn.Tanh) +Softmax = basic_unit(nn.Softmax) +Softmax2d = basic_unit(nn.Softmax2d) +LogSoftmax = basic_unit(nn.LogSoftmax) +ELU = basic_unit(nn.ELU) +SELU = basic_unit(nn.SELU) +CELU = basic_unit(nn.CELU) +GLU = basic_unit(nn.GLU) +GELU = basic_unit(nn.GELU) +Hardshrink = basic_unit(nn.Hardshrink) +LeakyReLU = basic_unit(nn.LeakyReLU) +LogSigmoid = basic_unit(nn.LogSigmoid) +Softplus = basic_unit(nn.Softplus) +Softshrink = basic_unit(nn.Softshrink) +MultiheadAttention = basic_unit(nn.MultiheadAttention) +PReLU = basic_unit(nn.PReLU) +Softsign = basic_unit(nn.Softsign) +Softmin = basic_unit(nn.Softmin) +Tanhshrink = basic_unit(nn.Tanhshrink) +RReLU = basic_unit(nn.RReLU) +AvgPool1d = basic_unit(nn.AvgPool1d) +AvgPool2d = basic_unit(nn.AvgPool2d) +AvgPool3d = basic_unit(nn.AvgPool3d) +MaxPool1d = basic_unit(nn.MaxPool1d) +MaxPool2d = basic_unit(nn.MaxPool2d) +MaxPool3d = basic_unit(nn.MaxPool3d) +MaxUnpool1d = basic_unit(nn.MaxUnpool1d) +MaxUnpool2d = basic_unit(nn.MaxUnpool2d) +MaxUnpool3d = basic_unit(nn.MaxUnpool3d) +FractionalMaxPool2d = basic_unit(nn.FractionalMaxPool2d) +FractionalMaxPool3d = basic_unit(nn.FractionalMaxPool3d) +LPPool1d = basic_unit(nn.LPPool1d) +LPPool2d = basic_unit(nn.LPPool2d) +LocalResponseNorm = basic_unit(nn.LocalResponseNorm) +BatchNorm1d = basic_unit(nn.BatchNorm1d) +BatchNorm2d = basic_unit(nn.BatchNorm2d) +BatchNorm3d = basic_unit(nn.BatchNorm3d) +InstanceNorm1d = basic_unit(nn.InstanceNorm1d) +InstanceNorm2d = basic_unit(nn.InstanceNorm2d) +InstanceNorm3d = basic_unit(nn.InstanceNorm3d) +LayerNorm = basic_unit(nn.LayerNorm) +GroupNorm = basic_unit(nn.GroupNorm) +SyncBatchNorm = basic_unit(nn.SyncBatchNorm) +Dropout = basic_unit(nn.Dropout) +Dropout2d = basic_unit(nn.Dropout2d) +Dropout3d = basic_unit(nn.Dropout3d) +AlphaDropout = basic_unit(nn.AlphaDropout) +FeatureAlphaDropout = basic_unit(nn.FeatureAlphaDropout) +ReflectionPad1d = basic_unit(nn.ReflectionPad1d) +ReflectionPad2d = basic_unit(nn.ReflectionPad2d) +ReplicationPad2d = basic_unit(nn.ReplicationPad2d) +ReplicationPad1d = basic_unit(nn.ReplicationPad1d) +ReplicationPad3d = basic_unit(nn.ReplicationPad3d) +CrossMapLRN2d = basic_unit(nn.CrossMapLRN2d) +Embedding = basic_unit(nn.Embedding) +EmbeddingBag = basic_unit(nn.EmbeddingBag) +RNNBase = basic_unit(nn.RNNBase) +RNN = basic_unit(nn.RNN) +LSTM = basic_unit(nn.LSTM) +GRU = basic_unit(nn.GRU) +RNNCellBase = basic_unit(nn.RNNCellBase) +RNNCell = basic_unit(nn.RNNCell) +LSTMCell = basic_unit(nn.LSTMCell) +GRUCell = basic_unit(nn.GRUCell) +PixelShuffle = basic_unit(nn.PixelShuffle) +Upsample = basic_unit(nn.Upsample) +UpsamplingNearest2d = basic_unit(nn.UpsamplingNearest2d) +UpsamplingBilinear2d = basic_unit(nn.UpsamplingBilinear2d) +PairwiseDistance = basic_unit(nn.PairwiseDistance) +AdaptiveMaxPool1d = basic_unit(nn.AdaptiveMaxPool1d) +AdaptiveMaxPool2d = basic_unit(nn.AdaptiveMaxPool2d) +AdaptiveMaxPool3d = basic_unit(nn.AdaptiveMaxPool3d) +AdaptiveAvgPool1d = basic_unit(nn.AdaptiveAvgPool1d) +AdaptiveAvgPool2d = basic_unit(nn.AdaptiveAvgPool2d) +AdaptiveAvgPool3d = basic_unit(nn.AdaptiveAvgPool3d) +TripletMarginLoss = basic_unit(nn.TripletMarginLoss) +ZeroPad2d = basic_unit(nn.ZeroPad2d) +ConstantPad1d = basic_unit(nn.ConstantPad1d) +ConstantPad2d = basic_unit(nn.ConstantPad2d) +ConstantPad3d = basic_unit(nn.ConstantPad3d) +Bilinear = basic_unit(nn.Bilinear) +CosineSimilarity = basic_unit(nn.CosineSimilarity) +Unfold = basic_unit(nn.Unfold) +Fold = basic_unit(nn.Fold) +AdaptiveLogSoftmaxWithLoss = basic_unit(nn.AdaptiveLogSoftmaxWithLoss) +TransformerEncoder = basic_unit(nn.TransformerEncoder) +TransformerDecoder = basic_unit(nn.TransformerDecoder) +TransformerEncoderLayer = basic_unit(nn.TransformerEncoderLayer) +TransformerDecoderLayer = basic_unit(nn.TransformerDecoderLayer) +Transformer = basic_unit(nn.Transformer) +Flatten = basic_unit(nn.Flatten) +Hardsigmoid = basic_unit(nn.Hardsigmoid) + +if version_larger_equal(torch.__version__, '1.6.0'): + Hardswish = basic_unit(nn.Hardswish) + +if version_larger_equal(torch.__version__, '1.7.0'): + SiLU = basic_unit(nn.SiLU) + Unflatten = basic_unit(nn.Unflatten) + TripletMarginWithDistanceLoss = basic_unit(nn.TripletMarginWithDistanceLoss) diff --git a/new_impl/cv/third_party/nni_new/retiarii/nn/pytorch/utils.py b/new_impl/cv/third_party/nni_new/retiarii/nn/pytorch/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..352348b997389cfe1d7afc9071f056265115ab2a --- /dev/null +++ b/new_impl/cv/third_party/nni_new/retiarii/nn/pytorch/utils.py @@ -0,0 +1,17 @@ +from typing import Optional + +from ...utils import uid, get_current_context + + +def generate_new_label(label: Optional[str]): + if label is None: + return '_mutation_' + str(uid('mutation')) + return label + + +def get_fixed_value(label: str): + ret = get_current_context('fixed') + try: + return ret[generate_new_label(label)] + except KeyError: + raise KeyError(f'Fixed context with {label} not found. Existing values are: {ret}') diff --git a/new_impl/cv/third_party/nni_new/retiarii/oneshot/__init__.py b/new_impl/cv/third_party/nni_new/retiarii/oneshot/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d4e1dad8512bd03c9475d4546d2b86748222aff2 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/retiarii/oneshot/__init__.py @@ -0,0 +1,4 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .interface import BaseOneShotTrainer diff --git a/new_impl/cv/third_party/nni_new/retiarii/oneshot/interface.py b/new_impl/cv/third_party/nni_new/retiarii/oneshot/interface.py new file mode 100644 index 0000000000000000000000000000000000000000..217bae9096efe518bdef9accc6ce55ef61fedc78 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/retiarii/oneshot/interface.py @@ -0,0 +1,25 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import abc +from typing import Any + + +class BaseOneShotTrainer(abc.ABC): + """ + Build many (possibly all) architectures into a full graph, search (with train) and export the best. + + One-shot trainer has a ``fit`` function with no return value. Trainers should fit and search for the best architecture. + Currently, all the inputs of trainer needs to be manually set before fit (including the search space, data loader + to use training epochs, and etc.). + + It has an extra ``export`` function that exports an object representing the final searched architecture. + """ + + @abc.abstractmethod + def fit(self) -> None: + pass + + @abc.abstractmethod + def export(self) -> Any: + pass diff --git a/new_impl/cv/third_party/nni_new/retiarii/oneshot/pytorch/__init__.py b/new_impl/cv/third_party/nni_new/retiarii/oneshot/pytorch/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6e9456958af9e0b6f78450e8707f9d1ef046d009 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/retiarii/oneshot/pytorch/__init__.py @@ -0,0 +1,8 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .darts import DartsTrainer +from .enas import EnasTrainer +from .proxyless import ProxylessTrainer +from .random import SinglePathTrainer, RandomTrainer +from .utils import replace_input_choice, replace_layer_choice diff --git a/new_impl/cv/third_party/nni_new/retiarii/oneshot/pytorch/darts.py b/new_impl/cv/third_party/nni_new/retiarii/oneshot/pytorch/darts.py new file mode 100644 index 0000000000000000000000000000000000000000..edcb6d7b864a91ef42d08722e4c638d23c691891 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/retiarii/oneshot/pytorch/darts.py @@ -0,0 +1,285 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import copy +import logging + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from ..interface import BaseOneShotTrainer +from .utils import AverageMeterGroup, replace_layer_choice, replace_input_choice + + +_logger = logging.getLogger(__name__) + + +class DartsLayerChoice(nn.Module): + def __init__(self, layer_choice): + super(DartsLayerChoice, self).__init__() + self.name = layer_choice.key + self.op_choices = nn.ModuleDict(layer_choice.named_children()) + self.alpha = nn.Parameter(torch.randn(len(self.op_choices)) * 1e-3) + + def forward(self, *args, **kwargs): + op_results = torch.stack([op(*args, **kwargs) for op in self.op_choices.values()]) + alpha_shape = [-1] + [1] * (len(op_results.size()) - 1) + return torch.sum(op_results * F.softmax(self.alpha, -1).view(*alpha_shape), 0) + + def parameters(self): + for _, p in self.named_parameters(): + yield p + + def named_parameters(self): + for name, p in super(DartsLayerChoice, self).named_parameters(): + if name == 'alpha': + continue + yield name, p + + def export(self): + return torch.argmax(self.alpha).item() + + +class DartsInputChoice(nn.Module): + def __init__(self, input_choice): + super(DartsInputChoice, self).__init__() + self.name = input_choice.key + self.alpha = nn.Parameter(torch.randn(input_choice.n_candidates) * 1e-3) + self.n_chosen = input_choice.n_chosen or 1 + + def forward(self, inputs): + inputs = torch.stack(inputs) + alpha_shape = [-1] + [1] * (len(inputs.size()) - 1) + return torch.sum(inputs * F.softmax(self.alpha, -1).view(*alpha_shape), 0) + + def parameters(self): + for _, p in self.named_parameters(): + yield p + + def named_parameters(self): + for name, p in super(DartsInputChoice, self).named_parameters(): + if name == 'alpha': + continue + yield name, p + + def export(self): + return torch.argsort(-self.alpha).cpu().numpy().tolist()[:self.n_chosen] + + +class DartsTrainer(BaseOneShotTrainer): + """ + DARTS trainer. + + Parameters + ---------- + model : nn.Module + PyTorch model to be trained. + loss : callable + Receives logits and ground truth label, return a loss tensor. + metrics : callable + Receives logits and ground truth label, return a dict of metrics. + optimizer : Optimizer + The optimizer used for optimizing the model. + num_epochs : int + Number of epochs planned for training. + dataset : Dataset + Dataset for training. Will be split for training weights and architecture weights. + grad_clip : float + Gradient clipping. Set to 0 to disable. Default: 5. + learning_rate : float + Learning rate to optimize the model. + batch_size : int + Batch size. + workers : int + Workers for data loading. + device : torch.device + ``torch.device("cpu")`` or ``torch.device("cuda")``. + log_frequency : int + Step count per logging. + arc_learning_rate : float + Learning rate of architecture parameters. + unrolled : float + ``True`` if using second order optimization, else first order optimization. + """ + + def __init__(self, model, loss, metrics, optimizer, + num_epochs, dataset, grad_clip=5., + learning_rate=2.5E-3, batch_size=64, workers=4, + device=None, log_frequency=None, + arc_learning_rate=3.0E-4, unrolled=False): + self.model = model + self.loss = loss + self.metrics = metrics + self.num_epochs = num_epochs + self.dataset = dataset + self.batch_size = batch_size + self.workers = workers + self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') if device is None else device + self.log_frequency = log_frequency + self.model.to(self.device) + + self.nas_modules = [] + replace_layer_choice(self.model, DartsLayerChoice, self.nas_modules) + replace_input_choice(self.model, DartsInputChoice, self.nas_modules) + for _, module in self.nas_modules: + module.to(self.device) + + self.model_optim = optimizer + # use the same architecture weight for modules with duplicated names + ctrl_params = {} + for _, m in self.nas_modules: + if m.name in ctrl_params: + assert m.alpha.size() == ctrl_params[m.name].size(), 'Size of parameters with the same label should be same.' + m.alpha = ctrl_params[m.name] + else: + ctrl_params[m.name] = m.alpha + self.ctrl_optim = torch.optim.Adam(list(ctrl_params.values()), arc_learning_rate, betas=(0.5, 0.999), + weight_decay=1.0E-3) + self.unrolled = unrolled + self.grad_clip = 5. + + self._init_dataloader() + + def _init_dataloader(self): + n_train = len(self.dataset) + split = n_train // 2 + indices = list(range(n_train)) + train_sampler = torch.utils.data.sampler.SubsetRandomSampler(indices[:split]) + valid_sampler = torch.utils.data.sampler.SubsetRandomSampler(indices[split:]) + self.train_loader = torch.utils.data.DataLoader(self.dataset, + batch_size=self.batch_size, + sampler=train_sampler, + num_workers=self.workers) + self.valid_loader = torch.utils.data.DataLoader(self.dataset, + batch_size=self.batch_size, + sampler=valid_sampler, + num_workers=self.workers) + + def _train_one_epoch(self, epoch): + self.model.train() + meters = AverageMeterGroup() + for step, ((trn_X, trn_y), (val_X, val_y)) in enumerate(zip(self.train_loader, self.valid_loader)): + trn_X, trn_y = trn_X.to(self.device), trn_y.to(self.device) + val_X, val_y = val_X.to(self.device), val_y.to(self.device) + + # phase 1. architecture step + self.ctrl_optim.zero_grad() + if self.unrolled: + self._unrolled_backward(trn_X, trn_y, val_X, val_y) + else: + self._backward(val_X, val_y) + self.ctrl_optim.step() + + # phase 2: child network step + self.model_optim.zero_grad() + logits, loss = self._logits_and_loss(trn_X, trn_y) + loss.backward() + if self.grad_clip > 0: + nn.utils.clip_grad_norm_(self.model.parameters(), self.grad_clip) # gradient clipping + self.model_optim.step() + + metrics = self.metrics(logits, trn_y) + metrics['loss'] = loss.item() + meters.update(metrics) + if self.log_frequency is not None and step % self.log_frequency == 0: + _logger.info('Epoch [%s/%s] Step [%s/%s] %s', epoch + 1, + self.num_epochs, step + 1, len(self.train_loader), meters) + + def _logits_and_loss(self, X, y): + logits = self.model(X) + loss = self.loss(logits, y) + return logits, loss + + def _backward(self, val_X, val_y): + """ + Simple backward with gradient descent + """ + _, loss = self._logits_and_loss(val_X, val_y) + loss.backward() + + def _unrolled_backward(self, trn_X, trn_y, val_X, val_y): + """ + Compute unrolled loss and backward its gradients + """ + backup_params = copy.deepcopy(tuple(self.model.parameters())) + + # do virtual step on training data + lr = self.model_optim.param_groups[0]["lr"] + momentum = self.model_optim.param_groups[0]["momentum"] + weight_decay = self.model_optim.param_groups[0]["weight_decay"] + self._compute_virtual_model(trn_X, trn_y, lr, momentum, weight_decay) + + # calculate unrolled loss on validation data + # keep gradients for model here for compute hessian + _, loss = self._logits_and_loss(val_X, val_y) + w_model, w_ctrl = tuple(self.model.parameters()), tuple([c.alpha for c in self.nas_modules]) + w_grads = torch.autograd.grad(loss, w_model + w_ctrl) + d_model, d_ctrl = w_grads[:len(w_model)], w_grads[len(w_model):] + + # compute hessian and final gradients + hessian = self._compute_hessian(backup_params, d_model, trn_X, trn_y) + with torch.no_grad(): + for param, d, h in zip(w_ctrl, d_ctrl, hessian): + # gradient = dalpha - lr * hessian + param.grad = d - lr * h + + # restore weights + self._restore_weights(backup_params) + + def _compute_virtual_model(self, X, y, lr, momentum, weight_decay): + """ + Compute unrolled weights w` + """ + # don't need zero_grad, using autograd to calculate gradients + _, loss = self._logits_and_loss(X, y) + gradients = torch.autograd.grad(loss, self.model.parameters()) + with torch.no_grad(): + for w, g in zip(self.model.parameters(), gradients): + m = self.model_optim.state[w].get('momentum_buffer', 0.) + w = w - lr * (momentum * m + g + weight_decay * w) + + def _restore_weights(self, backup_params): + with torch.no_grad(): + for param, backup in zip(self.model.parameters(), backup_params): + param.copy_(backup) + + def _compute_hessian(self, backup_params, dw, trn_X, trn_y): + """ + dw = dw` { L_val(w`, alpha) } + w+ = w + eps * dw + w- = w - eps * dw + hessian = (dalpha { L_trn(w+, alpha) } - dalpha { L_trn(w-, alpha) }) / (2*eps) + eps = 0.01 / ||dw|| + """ + self._restore_weights(backup_params) + norm = torch.cat([w.view(-1) for w in dw]).norm() + eps = 0.01 / norm + if norm < 1E-8: + _logger.warning('In computing hessian, norm is smaller than 1E-8, cause eps to be %.6f.', norm.item()) + + dalphas = [] + for e in [eps, -2. * eps]: + # w+ = w + eps*dw`, w- = w - eps*dw` + with torch.no_grad(): + for p, d in zip(self.model.parameters(), dw): + p += e * d + + _, loss = self._logits_and_loss(trn_X, trn_y) + dalphas.append(torch.autograd.grad(loss, [c.alpha for c in self.nas_modules])) + + dalpha_pos, dalpha_neg = dalphas # dalpha { L_trn(w+) }, # dalpha { L_trn(w-) } + hessian = [(p - n) / (2. * eps) for p, n in zip(dalpha_pos, dalpha_neg)] + return hessian + + def fit(self): + for i in range(self.num_epochs): + self._train_one_epoch(i) + + @torch.no_grad() + def export(self): + result = dict() + for name, module in self.nas_modules: + if name not in result: + result[name] = module.export() + return result diff --git a/new_impl/cv/third_party/nni_new/retiarii/oneshot/pytorch/enas.py b/new_impl/cv/third_party/nni_new/retiarii/oneshot/pytorch/enas.py new file mode 100644 index 0000000000000000000000000000000000000000..7f03c1dd1015a23f6255ba05a221a179e8d37fde --- /dev/null +++ b/new_impl/cv/third_party/nni_new/retiarii/oneshot/pytorch/enas.py @@ -0,0 +1,336 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.optim as optim + +from ..interface import BaseOneShotTrainer +from .random import PathSamplingLayerChoice, PathSamplingInputChoice +from .utils import AverageMeterGroup, replace_layer_choice, replace_input_choice, to_device + +_logger = logging.getLogger(__name__) + + +class StackedLSTMCell(nn.Module): + def __init__(self, layers, size, bias): + super().__init__() + self.lstm_num_layers = layers + self.lstm_modules = nn.ModuleList([nn.LSTMCell(size, size, bias=bias) + for _ in range(self.lstm_num_layers)]) + + def forward(self, inputs, hidden): + prev_h, prev_c = hidden + next_h, next_c = [], [] + for i, m in enumerate(self.lstm_modules): + curr_h, curr_c = m(inputs, (prev_h[i], prev_c[i])) + next_c.append(curr_c) + next_h.append(curr_h) + # current implementation only supports batch size equals 1, + # but the algorithm does not necessarily have this limitation + inputs = curr_h[-1].view(1, -1) + return next_h, next_c + + +class ReinforceField: + """ + A field with ``name``, with ``total`` choices. ``choose_one`` is true if one and only one is meant to be + selected. Otherwise, any number of choices can be chosen. + """ + + def __init__(self, name, total, choose_one): + self.name = name + self.total = total + self.choose_one = choose_one + + def __repr__(self): + return f'ReinforceField(name={self.name}, total={self.total}, choose_one={self.choose_one})' + + +class ReinforceController(nn.Module): + """ + A controller that mutates the graph with RL. + + Parameters + ---------- + fields : list of ReinforceField + List of fields to choose. + lstm_size : int + Controller LSTM hidden units. + lstm_num_layers : int + Number of layers for stacked LSTM. + tanh_constant : float + Logits will be equal to ``tanh_constant * tanh(logits)``. Don't use ``tanh`` if this value is ``None``. + skip_target : float + Target probability that skipconnect will appear. + temperature : float + Temperature constant that divides the logits. + entropy_reduction : str + Can be one of ``sum`` and ``mean``. How the entropy of multi-input-choice is reduced. + """ + + def __init__(self, fields, lstm_size=64, lstm_num_layers=1, tanh_constant=1.5, + skip_target=0.4, temperature=None, entropy_reduction='sum'): + super(ReinforceController, self).__init__() + self.fields = fields + self.lstm_size = lstm_size + self.lstm_num_layers = lstm_num_layers + self.tanh_constant = tanh_constant + self.temperature = temperature + self.skip_target = skip_target + + self.lstm = StackedLSTMCell(self.lstm_num_layers, self.lstm_size, False) + self.attn_anchor = nn.Linear(self.lstm_size, self.lstm_size, bias=False) + self.attn_query = nn.Linear(self.lstm_size, self.lstm_size, bias=False) + self.v_attn = nn.Linear(self.lstm_size, 1, bias=False) + self.g_emb = nn.Parameter(torch.randn(1, self.lstm_size) * 0.1) + self.skip_targets = nn.Parameter(torch.tensor([1.0 - self.skip_target, self.skip_target]), # pylint: disable=not-callable + requires_grad=False) + assert entropy_reduction in ['sum', 'mean'], 'Entropy reduction must be one of sum and mean.' + self.entropy_reduction = torch.sum if entropy_reduction == 'sum' else torch.mean + self.cross_entropy_loss = nn.CrossEntropyLoss(reduction='none') + self.soft = nn.ModuleDict({ + field.name: nn.Linear(self.lstm_size, field.total, bias=False) for field in fields + }) + self.embedding = nn.ModuleDict({ + field.name: nn.Embedding(field.total, self.lstm_size) for field in fields + }) + + def resample(self): + self._initialize() + result = dict() + for field in self.fields: + result[field.name] = self._sample_single(field) + return result + + def _initialize(self): + self._inputs = self.g_emb.data + self._c = [torch.zeros((1, self.lstm_size), + dtype=self._inputs.dtype, + device=self._inputs.device) for _ in range(self.lstm_num_layers)] + self._h = [torch.zeros((1, self.lstm_size), + dtype=self._inputs.dtype, + device=self._inputs.device) for _ in range(self.lstm_num_layers)] + self.sample_log_prob = 0 + self.sample_entropy = 0 + self.sample_skip_penalty = 0 + + def _lstm_next_step(self): + self._h, self._c = self.lstm(self._inputs, (self._h, self._c)) + + def _sample_single(self, field): + self._lstm_next_step() + logit = self.soft[field.name](self._h[-1]) + if self.temperature is not None: + logit /= self.temperature + if self.tanh_constant is not None: + logit = self.tanh_constant * torch.tanh(logit) + if field.choose_one: + sampled = torch.multinomial(F.softmax(logit, dim=-1), 1).view(-1) + log_prob = self.cross_entropy_loss(logit, sampled) + self._inputs = self.embedding[field.name](sampled) + else: + logit = logit.view(-1, 1) + logit = torch.cat([-logit, logit], 1) # pylint: disable=invalid-unary-operand-type + sampled = torch.multinomial(F.softmax(logit, dim=-1), 1).view(-1) + skip_prob = torch.sigmoid(logit) + kl = torch.sum(skip_prob * torch.log(skip_prob / self.skip_targets)) + self.sample_skip_penalty += kl + log_prob = self.cross_entropy_loss(logit, sampled) + sampled = sampled.nonzero().view(-1) + if sampled.sum().item(): + self._inputs = (torch.sum(self.embedding[field.name](sampled.view(-1)), 0) / (1. + torch.sum(sampled))).unsqueeze(0) + else: + self._inputs = torch.zeros(1, self.lstm_size, device=self.embedding[field.name].weight.device) + + sampled = sampled.detach().numpy().tolist() + self.sample_log_prob += self.entropy_reduction(log_prob) + entropy = (log_prob * torch.exp(-log_prob)).detach() # pylint: disable=invalid-unary-operand-type + self.sample_entropy += self.entropy_reduction(entropy) + if len(sampled) == 1: + sampled = sampled[0] + return sampled + + +class EnasTrainer(BaseOneShotTrainer): + """ + ENAS trainer. + + Parameters + ---------- + model : nn.Module + PyTorch model to be trained. + loss : callable + Receives logits and ground truth label, return a loss tensor. + metrics : callable + Receives logits and ground truth label, return a dict of metrics. + reward_function : callable + Receives logits and ground truth label, return a tensor, which will be feeded to RL controller as reward. + optimizer : Optimizer + The optimizer used for optimizing the model. + num_epochs : int + Number of epochs planned for training. + dataset : Dataset + Dataset for training. Will be split for training weights and architecture weights. + batch_size : int + Batch size. + workers : int + Workers for data loading. + device : torch.device + ``torch.device("cpu")`` or ``torch.device("cuda")``. + log_frequency : int + Step count per logging. + grad_clip : float + Gradient clipping. Set to 0 to disable. Default: 5. + entropy_weight : float + Weight of sample entropy loss. + skip_weight : float + Weight of skip penalty loss. + baseline_decay : float + Decay factor of baseline. New baseline will be equal to ``baseline_decay * baseline_old + reward * (1 - baseline_decay)``. + ctrl_lr : float + Learning rate for RL controller. + ctrl_steps_aggregate : int + Number of steps that will be aggregated into one mini-batch for RL controller. + ctrl_steps : int + Number of mini-batches for each epoch of RL controller learning. + ctrl_kwargs : dict + Optional kwargs that will be passed to :class:`ReinforceController`. + """ + + def __init__(self, model, loss, metrics, reward_function, + optimizer, num_epochs, dataset, + batch_size=64, workers=4, device=None, log_frequency=None, + grad_clip=5., entropy_weight=0.0001, skip_weight=0.8, baseline_decay=0.999, + ctrl_lr=0.00035, ctrl_steps_aggregate=20, ctrl_kwargs=None): + self.model = model + self.loss = loss + self.metrics = metrics + self.optimizer = optimizer + self.num_epochs = num_epochs + self.dataset = dataset + self.batch_size = batch_size + self.workers = workers + self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') if device is None else device + self.log_frequency = log_frequency + + self.nas_modules = [] + replace_layer_choice(self.model, PathSamplingLayerChoice, self.nas_modules) + replace_input_choice(self.model, PathSamplingInputChoice, self.nas_modules) + for _, module in self.nas_modules: + module.to(self.device) + self.model.to(self.device) + + self.nas_fields = [ReinforceField(name, len(module), + isinstance(module, PathSamplingLayerChoice) or module.n_chosen == 1) + for name, module in self.nas_modules] + self.controller = ReinforceController(self.nas_fields, **(ctrl_kwargs or {})) + + self.grad_clip = grad_clip + self.reward_function = reward_function + self.ctrl_optim = optim.Adam(self.controller.parameters(), lr=ctrl_lr) + self.batch_size = batch_size + self.workers = workers + + self.entropy_weight = entropy_weight + self.skip_weight = skip_weight + self.baseline_decay = baseline_decay + self.baseline = 0. + self.ctrl_steps_aggregate = ctrl_steps_aggregate + + self.init_dataloader() + + def init_dataloader(self): + n_train = len(self.dataset) + split = n_train // 2 + indices = list(range(n_train)) + train_sampler = torch.utils.data.sampler.SubsetRandomSampler(indices[:-split]) + valid_sampler = torch.utils.data.sampler.SubsetRandomSampler(indices[-split:]) + self.train_loader = torch.utils.data.DataLoader(self.dataset, + batch_size=self.batch_size, + sampler=train_sampler, + num_workers=self.workers) + self.valid_loader = torch.utils.data.DataLoader(self.dataset, + batch_size=self.batch_size, + sampler=valid_sampler, + num_workers=self.workers) + + def _train_model(self, epoch): + self.model.train() + self.controller.eval() + meters = AverageMeterGroup() + for step, (x, y) in enumerate(self.train_loader): + x, y = to_device(x, self.device), to_device(y, self.device) + self.optimizer.zero_grad() + + self._resample() + logits = self.model(x) + metrics = self.metrics(logits, y) + loss = self.loss(logits, y) + loss.backward() + if self.grad_clip > 0: + nn.utils.clip_grad_norm_(self.model.parameters(), self.grad_clip) + self.optimizer.step() + metrics['loss'] = loss.item() + meters.update(metrics) + + if self.log_frequency is not None and step % self.log_frequency == 0: + _logger.info('Model Epoch [%d/%d] Step [%d/%d] %s', epoch + 1, + self.num_epochs, step + 1, len(self.train_loader), meters) + + def _train_controller(self, epoch): + self.model.eval() + self.controller.train() + meters = AverageMeterGroup() + self.ctrl_optim.zero_grad() + for ctrl_step, (x, y) in enumerate(self.valid_loader): + x, y = to_device(x, self.device), to_device(y, self.device) + + self._resample() + with torch.no_grad(): + logits = self.model(x) + metrics = self.metrics(logits, y) + reward = self.reward_function(logits, y) + if self.entropy_weight: + reward += self.entropy_weight * self.controller.sample_entropy.item() + self.baseline = self.baseline * self.baseline_decay + reward * (1 - self.baseline_decay) + loss = self.controller.sample_log_prob * (reward - self.baseline) + if self.skip_weight: + loss += self.skip_weight * self.controller.sample_skip_penalty + metrics['reward'] = reward + metrics['loss'] = loss.item() + metrics['ent'] = self.controller.sample_entropy.item() + metrics['log_prob'] = self.controller.sample_log_prob.item() + metrics['baseline'] = self.baseline + metrics['skip'] = self.controller.sample_skip_penalty + + loss /= self.ctrl_steps_aggregate + loss.backward() + meters.update(metrics) + + if (ctrl_step + 1) % self.ctrl_steps_aggregate == 0: + if self.grad_clip > 0: + nn.utils.clip_grad_norm_(self.controller.parameters(), self.grad_clip) + self.ctrl_optim.step() + self.ctrl_optim.zero_grad() + + if self.log_frequency is not None and ctrl_step % self.log_frequency == 0: + _logger.info('RL Epoch [%d/%d] Step [%d/%d] %s', epoch + 1, self.num_epochs, + ctrl_step + 1, len(self.valid_loader), meters) + + def _resample(self): + result = self.controller.resample() + for name, module in self.nas_modules: + module.sampled = result[name] + + def fit(self): + for i in range(self.num_epochs): + self._train_model(i) + self._train_controller(i) + + def export(self): + self.controller.eval() + with torch.no_grad(): + return self.controller.resample() diff --git a/new_impl/cv/third_party/nni_new/retiarii/oneshot/pytorch/proxyless.py b/new_impl/cv/third_party/nni_new/retiarii/oneshot/pytorch/proxyless.py new file mode 100644 index 0000000000000000000000000000000000000000..b44d883711c3e5af3d0e42fad8b4effc91a6e281 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/retiarii/oneshot/pytorch/proxyless.py @@ -0,0 +1,227 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from ..interface import BaseOneShotTrainer +from .utils import AverageMeterGroup, replace_layer_choice, replace_input_choice + + +_logger = logging.getLogger(__name__) + + +class ArchGradientFunction(torch.autograd.Function): + @staticmethod + def forward(ctx, x, binary_gates, run_func, backward_func): + ctx.run_func = run_func + ctx.backward_func = backward_func + + detached_x = x.detach() + detached_x.requires_grad = x.requires_grad + with torch.enable_grad(): + output = run_func(detached_x) + ctx.save_for_backward(detached_x, output) + return output.data + + @staticmethod + def backward(ctx, grad_output): + detached_x, output = ctx.saved_tensors + + grad_x = torch.autograd.grad(output, detached_x, grad_output, only_inputs=True) + # compute gradients w.r.t. binary_gates + binary_grads = ctx.backward_func(detached_x.data, output.data, grad_output.data) + + return grad_x[0], binary_grads, None, None + + +class ProxylessLayerChoice(nn.Module): + def __init__(self, ops): + super(ProxylessLayerChoice, self).__init__() + self.ops = nn.ModuleList(ops) + self.alpha = nn.Parameter(torch.randn(len(self.ops)) * 1E-3) + self._binary_gates = nn.Parameter(torch.randn(len(self.ops)) * 1E-3) + self.sampled = None + + def forward(self, *args): + def run_function(ops, active_id): + def forward(_x): + return ops[active_id](_x) + return forward + + def backward_function(ops, active_id, binary_gates): + def backward(_x, _output, grad_output): + binary_grads = torch.zeros_like(binary_gates.data) + with torch.no_grad(): + for k in range(len(ops)): + if k != active_id: + out_k = ops[k](_x.data) + else: + out_k = _output.data + grad_k = torch.sum(out_k * grad_output) + binary_grads[k] = grad_k + return binary_grads + return backward + + assert len(args) == 1 + x = args[0] + return ArchGradientFunction.apply( + x, self._binary_gates, run_function(self.ops, self.sampled), + backward_function(self.ops, self.sampled, self._binary_gates) + ) + + def resample(self): + probs = F.softmax(self.alpha, dim=-1) + sample = torch.multinomial(probs, 1)[0].item() + self.sampled = sample + with torch.no_grad(): + self._binary_gates.zero_() + self._binary_gates.grad = torch.zeros_like(self._binary_gates.data) + self._binary_gates.data[sample] = 1.0 + + def finalize_grad(self): + binary_grads = self._binary_gates.grad + with torch.no_grad(): + if self.alpha.grad is None: + self.alpha.grad = torch.zeros_like(self.alpha.data) + probs = F.softmax(self.alpha, dim=-1) + for i in range(len(self.ops)): + for j in range(len(self.ops)): + self.alpha.grad[i] += binary_grads[j] * probs[j] * (int(i == j) - probs[i]) + + def export(self): + return torch.argmax(self.alpha).item() + + +class ProxylessInputChoice(nn.Module): + def __init__(self, *args, **kwargs): + raise NotImplementedError('Input choice is not supported for ProxylessNAS.') + + +class ProxylessTrainer(BaseOneShotTrainer): + """ + Proxyless trainer. + + Parameters + ---------- + model : nn.Module + PyTorch model to be trained. + loss : callable + Receives logits and ground truth label, return a loss tensor. + metrics : callable + Receives logits and ground truth label, return a dict of metrics. + optimizer : Optimizer + The optimizer used for optimizing the model. + num_epochs : int + Number of epochs planned for training. + dataset : Dataset + Dataset for training. Will be split for training weights and architecture weights. + warmup_epochs : int + Number of epochs to warmup model parameters. + batch_size : int + Batch size. + workers : int + Workers for data loading. + device : torch.device + ``torch.device("cpu")`` or ``torch.device("cuda")``. + log_frequency : int + Step count per logging. + arc_learning_rate : float + Learning rate of architecture parameters. + """ + + def __init__(self, model, loss, metrics, optimizer, + num_epochs, dataset, warmup_epochs=0, + batch_size=64, workers=4, device=None, log_frequency=None, + arc_learning_rate=1.0E-3): + self.model = model + self.loss = loss + self.metrics = metrics + self.optimizer = optimizer + self.num_epochs = num_epochs + self.warmup_epochs = warmup_epochs + self.dataset = dataset + self.batch_size = batch_size + self.workers = workers + self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') if device is None else device + self.log_frequency = log_frequency + self.model.to(self.device) + + self.nas_modules = [] + replace_layer_choice(self.model, ProxylessLayerChoice, self.nas_modules) + replace_input_choice(self.model, ProxylessInputChoice, self.nas_modules) + for _, module in self.nas_modules: + module.to(self.device) + + self.optimizer = optimizer + # we do not support deduplicate control parameters with same label (like DARTS) yet. + self.ctrl_optim = torch.optim.Adam([m.alpha for _, m in self.nas_modules], arc_learning_rate, + weight_decay=0, betas=(0, 0.999), eps=1e-8) + self._init_dataloader() + + def _init_dataloader(self): + n_train = len(self.dataset) + split = n_train // 2 + indices = list(range(n_train)) + train_sampler = torch.utils.data.sampler.SubsetRandomSampler(indices[:split]) + valid_sampler = torch.utils.data.sampler.SubsetRandomSampler(indices[split:]) + self.train_loader = torch.utils.data.DataLoader(self.dataset, + batch_size=self.batch_size, + sampler=train_sampler, + num_workers=self.workers) + self.valid_loader = torch.utils.data.DataLoader(self.dataset, + batch_size=self.batch_size, + sampler=valid_sampler, + num_workers=self.workers) + + def _train_one_epoch(self, epoch): + self.model.train() + meters = AverageMeterGroup() + for step, ((trn_X, trn_y), (val_X, val_y)) in enumerate(zip(self.train_loader, self.valid_loader)): + trn_X, trn_y = trn_X.to(self.device), trn_y.to(self.device) + val_X, val_y = val_X.to(self.device), val_y.to(self.device) + + if epoch >= self.warmup_epochs: + # 1) train architecture parameters + for _, module in self.nas_modules: + module.resample() + self.ctrl_optim.zero_grad() + logits, loss = self._logits_and_loss(val_X, val_y) + loss.backward() + for _, module in self.nas_modules: + module.finalize_grad() + self.ctrl_optim.step() + + # 2) train model parameters + for _, module in self.nas_modules: + module.resample() + self.optimizer.zero_grad() + logits, loss = self._logits_and_loss(trn_X, trn_y) + loss.backward() + self.optimizer.step() + metrics = self.metrics(logits, trn_y) + metrics["loss"] = loss.item() + meters.update(metrics) + if self.log_frequency is not None and step % self.log_frequency == 0: + _logger.info("Epoch [%s/%s] Step [%s/%s] %s", epoch + 1, + self.num_epochs, step + 1, len(self.train_loader), meters) + + def _logits_and_loss(self, X, y): + logits = self.model(X) + loss = self.loss(logits, y) + return logits, loss + + def fit(self): + for i in range(self.num_epochs): + self._train_one_epoch(i) + + @torch.no_grad() + def export(self): + result = dict() + for name, module in self.nas_modules: + if name not in result: + result[name] = module.export() + return result diff --git a/new_impl/cv/third_party/nni_new/retiarii/oneshot/pytorch/random.py b/new_impl/cv/third_party/nni_new/retiarii/oneshot/pytorch/random.py new file mode 100644 index 0000000000000000000000000000000000000000..a82ddada10853dfcf4a0ac2bc22301485954a32b --- /dev/null +++ b/new_impl/cv/third_party/nni_new/retiarii/oneshot/pytorch/random.py @@ -0,0 +1,203 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +import random + +import torch +import torch.nn as nn + +from ..interface import BaseOneShotTrainer +from .utils import AverageMeterGroup, replace_layer_choice, replace_input_choice + + +_logger = logging.getLogger(__name__) + + +def _get_mask(sampled, total): + multihot = [i == sampled or (isinstance(sampled, list) and i in sampled) for i in range(total)] + return torch.tensor(multihot, dtype=torch.bool) # pylint: disable=not-callable + + +class PathSamplingLayerChoice(nn.Module): + """ + Mixed module, in which fprop is decided by exactly one or multiple (sampled) module. + If multiple module is selected, the result will be sumed and returned. + + Attributes + ---------- + sampled : int or list of int + Sampled module indices. + mask : tensor + A multi-hot bool 1D-tensor representing the sampled mask. + """ + + def __init__(self, layer_choice): + super(PathSamplingLayerChoice, self).__init__() + self.op_names = [] + for name, module in layer_choice.named_children(): + self.add_module(name, module) + self.op_names.append(name) + assert self.op_names, 'There has to be at least one op to choose from.' + self.sampled = None # sampled can be either a list of indices or an index + + def forward(self, *args, **kwargs): + assert self.sampled is not None, 'At least one path needs to be sampled before fprop.' + if isinstance(self.sampled, list): + return sum([getattr(self, self.op_names[i])(*args, **kwargs) for i in self.sampled]) # pylint: disable=not-an-iterable + else: + return getattr(self, self.op_names[self.sampled])(*args, **kwargs) # pylint: disable=invalid-sequence-index + + def __len__(self): + return len(self.op_names) + + @property + def mask(self): + return _get_mask(self.sampled, len(self)) + + +class PathSamplingInputChoice(nn.Module): + """ + Mixed input. Take a list of tensor as input, select some of them and return the sum. + + Attributes + ---------- + sampled : int or list of int + Sampled module indices. + mask : tensor + A multi-hot bool 1D-tensor representing the sampled mask. + """ + + def __init__(self, input_choice): + super(PathSamplingInputChoice, self).__init__() + self.n_candidates = input_choice.n_candidates + self.n_chosen = input_choice.n_chosen + self.sampled = None + + def forward(self, input_tensors): + if isinstance(self.sampled, list): + return sum([input_tensors[t] for t in self.sampled]) # pylint: disable=not-an-iterable + else: + return input_tensors[self.sampled] + + def __len__(self): + return self.n_candidates + + @property + def mask(self): + return _get_mask(self.sampled, len(self)) + + +class SinglePathTrainer(BaseOneShotTrainer): + """ + Single-path trainer. Samples a path every time and backpropagates on that path. + + Parameters + ---------- + model : nn.Module + Model with mutables. + loss : callable + Called with logits and targets. Returns a loss tensor. + metrics : callable + Returns a dict that maps metrics keys to metrics data. + optimizer : Optimizer + Optimizer that optimizes the model. + num_epochs : int + Number of epochs of training. + dataset_train : Dataset + Dataset of training. + dataset_valid : Dataset + Dataset of validation. + batch_size : int + Batch size. + workers: int + Number of threads for data preprocessing. Not used for this trainer. Maybe removed in future. + device : torch.device + Device object. Either ``torch.device("cuda")`` or ``torch.device("cpu")``. When ``None``, trainer will + automatic detects GPU and selects GPU first. + log_frequency : int + Number of mini-batches to log metrics. + """ + + def __init__(self, model, loss, metrics, + optimizer, num_epochs, dataset_train, dataset_valid, + mutator=None, batch_size=64, workers=4, device=None, log_frequency=None): + self.model = model + self.loss = loss + self.metrics = metrics + self.optimizer = optimizer + self.num_epochs = num_epochs + self.dataset_train = dataset_train + self.dataset_valid = dataset_valid + self.batch_size = batch_size + self.workers = workers + self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') if device is None else device + self.log_frequency = log_frequency + self.model.to(self.device) + + self.nas_modules = [] + replace_layer_choice(self.model, PathSamplingLayerChoice, self.nas_modules) + replace_input_choice(self.model, PathSamplingInputChoice, self.nas_modules) + for _, module in self.nas_modules: + module.to(self.device) + + self.train_loader = torch.utils.data.DataLoader(self.dataset_train, + batch_size=batch_size, + num_workers=workers) + self.valid_loader = torch.utils.data.DataLoader(self.dataset_valid, + batch_size=batch_size, + num_workers=workers) + + def _resample(self): + result = {} + for name, module in self.nas_modules: + if name not in result: + result[name] = random.randint(0, len(module) - 1) + module.sampled = result[name] + return result + + def _train_one_epoch(self, epoch): + self.model.train() + meters = AverageMeterGroup() + for step, (x, y) in enumerate(self.train_loader): + x, y = x.to(self.device), y.to(self.device) + self.optimizer.zero_grad() + self._resample() + logits = self.model(x) + loss = self.loss(logits, y) + loss.backward() + self.optimizer.step() + + metrics = self.metrics(logits, y) + metrics["loss"] = loss.item() + meters.update(metrics) + if self.log_frequency is not None and step % self.log_frequency == 0: + _logger.info("Epoch [%s/%s] Step [%s/%s] %s", epoch + 1, + self.num_epochs, step + 1, len(self.train_loader), meters) + + def _validate_one_epoch(self, epoch): + self.model.eval() + meters = AverageMeterGroup() + with torch.no_grad(): + for step, (x, y) in enumerate(self.valid_loader): + x, y = x.to(self.device), y.to(self.device) + self._resample() + logits = self.model(x) + loss = self.loss(logits, y) + metrics = self.metrics(logits, y) + metrics["loss"] = loss.item() + meters.update(metrics) + if self.log_frequency is not None and step % self.log_frequency == 0: + _logger.info("Epoch [%s/%s] Validation Step [%s/%s] %s", epoch + 1, + self.num_epochs, step + 1, len(self.valid_loader), meters) + + def fit(self): + for i in range(self.num_epochs): + self._train_one_epoch(i) + self._validate_one_epoch(i) + + def export(self): + return self._resample() + + +RandomTrainer = SinglePathTrainer diff --git a/new_impl/cv/third_party/nni_new/retiarii/oneshot/pytorch/utils.py b/new_impl/cv/third_party/nni_new/retiarii/oneshot/pytorch/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..61a4f916ae626cfd5409064321ccd6915e2868cc --- /dev/null +++ b/new_impl/cv/third_party/nni_new/retiarii/oneshot/pytorch/utils.py @@ -0,0 +1,182 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +from collections import OrderedDict + +import numpy as np +import torch +import nni.retiarii.nn.pytorch as nn +from nni.nas.pytorch.mutables import InputChoice, LayerChoice + +_logger = logging.getLogger(__name__) + + +def to_device(obj, device): + """ + Move a tensor, tuple, list, or dict onto device. + """ + if torch.is_tensor(obj): + return obj.to(device) + if isinstance(obj, tuple): + return tuple(to_device(t, device) for t in obj) + if isinstance(obj, list): + return [to_device(t, device) for t in obj] + if isinstance(obj, dict): + return {k: to_device(v, device) for k, v in obj.items()} + if isinstance(obj, (int, float, str)): + return obj + raise ValueError("'%s' has unsupported type '%s'" % (obj, type(obj))) + + +def to_list(arr): + if torch.is_tensor(arr): + return arr.cpu().numpy().tolist() + if isinstance(arr, np.ndarray): + return arr.tolist() + if isinstance(arr, (list, tuple)): + return list(arr) + return arr + + +class AverageMeterGroup: + """ + Average meter group for multiple average meters. + """ + + def __init__(self): + self.meters = OrderedDict() + + def update(self, data): + """ + Update the meter group with a dict of metrics. + Non-exist average meters will be automatically created. + """ + for k, v in data.items(): + if k not in self.meters: + self.meters[k] = AverageMeter(k, ":4f") + self.meters[k].update(v) + + def __getattr__(self, item): + return self.meters[item] + + def __getitem__(self, item): + return self.meters[item] + + def __str__(self): + return " ".join(str(v) for v in self.meters.values()) + + def summary(self): + """ + Return a summary string of group data. + """ + return " ".join(v.summary() for v in self.meters.values()) + + +class AverageMeter: + """ + Computes and stores the average and current value. + + Parameters + ---------- + name : str + Name to display. + fmt : str + Format string to print the values. + """ + + def __init__(self, name, fmt=':f'): + self.name = name + self.fmt = fmt + self.reset() + + def reset(self): + """ + Reset the meter. + """ + self.val = 0 + self.avg = 0 + self.sum = 0 + self.count = 0 + + def update(self, val, n=1): + """ + Update with value and weight. + + Parameters + ---------- + val : float or int + The new value to be accounted in. + n : int + The weight of the new value. + """ + self.val = val + self.sum += val * n + self.count += n + self.avg = self.sum / self.count + + def __str__(self): + fmtstr = '{name} {val' + self.fmt + '} ({avg' + self.fmt + '})' + return fmtstr.format(**self.__dict__) + + def summary(self): + fmtstr = '{name}: {avg' + self.fmt + '}' + return fmtstr.format(**self.__dict__) + + +def _replace_module_with_type(root_module, init_fn, type_name, modules): + if modules is None: + modules = [] + + def apply(m): + for name, child in m.named_children(): + if isinstance(child, type_name): + setattr(m, name, init_fn(child)) + modules.append((child.key, getattr(m, name))) + else: + apply(child) + + apply(root_module) + return modules + + +def replace_layer_choice(root_module, init_fn, modules=None): + """ + Replace layer choice modules with modules that are initiated with init_fn. + + Parameters + ---------- + root_module : nn.Module + Root module to traverse. + init_fn : Callable + Initializing function. + modules : dict, optional + Update the replaced modules into the dict and check duplicate if provided. + + Returns + ------- + List[Tuple[str, nn.Module]] + A list from layer choice keys (names) and replaced modules. + """ + return _replace_module_with_type(root_module, init_fn, (LayerChoice, nn.LayerChoice), modules) + + +def replace_input_choice(root_module, init_fn, modules=None): + """ + Replace input choice modules with modules that are initiated with init_fn. + + Parameters + ---------- + root_module : nn.Module + Root module to traverse. + init_fn : Callable + Initializing function. + modules : dict, optional + Update the replaced modules into the dict and check duplicate if provided. + + Returns + ------- + List[Tuple[str, nn.Module]] + A list from layer choice keys (names) and replaced modules. + """ + return _replace_module_with_type(root_module, init_fn, (InputChoice, nn.InputChoice), modules) diff --git a/new_impl/cv/third_party/nni_new/retiarii/operation.py b/new_impl/cv/third_party/nni_new/retiarii/operation.py new file mode 100644 index 0000000000000000000000000000000000000000..d97f87f46b374a74455ff3916c59b691cffc6b71 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/retiarii/operation.py @@ -0,0 +1,233 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from typing import (Any, Dict, List) + +from . import debug_configs + +__all__ = ['Operation', 'Cell'] + + +def _convert_name(name: str) -> str: + """ + Convert the names using separator '.' to valid variable name in code + """ + return name.replace('.', '__') + + +class Operation: + """ + Calculation logic of a graph node. + + The constructor is private. Use `Operation.new()` to create operation object. + + `Operation` is a naive record. + Do not "mutate" its attributes or store information relate to specific node. + All complex logic should be implemented in `Node` class. + + Attributes + ---------- + type + Operation type name (e.g. Conv2D). + If it starts with underscore, the "operation" is a special one (e.g. subgraph, input/output). + parameters + Arbitrary key-value parameters (e.g. kernel_size). + """ + + def __init__(self, type_name: str, parameters: Dict[str, Any], _internal: bool = False): + assert _internal, '`Operation()` is private, use `Operation.new()` instead' + self.type: str = type_name + self.parameters: Dict[str, Any] = parameters + + def to_init_code(self, field: str) -> str: + raise NotImplementedError() + + def to_forward_code(self, field: str, output: str, inputs: List[str]) -> str: + raise NotImplementedError() + + def _to_class_name(self) -> str: + raise NotImplementedError() + + def __bool__(self) -> bool: + return True + + @staticmethod + def new(type_name: str, parameters: Dict[str, Any] = {}, cell_name: str = None) -> 'Operation': + if type_name == '_cell': + # NOTE: cell_name is the same as its Node's name, when the cell is wrapped within the node + return Cell(cell_name, parameters) + else: + if debug_configs.framework.lower() in ('torch', 'pytorch'): + from .operation_def import torch_op_def # pylint: disable=unused-import + cls = PyTorchOperation._find_subclass(type_name) + elif debug_configs.framework.lower() in ('tf', 'tensorflow'): + from .operation_def import tf_op_def # pylint: disable=unused-import + cls = TensorFlowOperation._find_subclass(type_name) + else: + raise ValueError(f'Unsupported framework: {debug_configs.framework}') + return cls(type_name, parameters, _internal=True) + + @classmethod + def _find_subclass(cls, subclass_name): + for subclass in cls.__subclasses__(): + if subclass.__name__ == subclass_name: + return subclass + return cls + + def __repr__(self): + type_name = type(self).__name__ + args = [f'{key}={repr(value)}' for key, value in self.parameters.items()] + if type_name != self.type: + args = [f'type="{self.type}"'] + args + return f'{type_name}({", ".join(args)})' + + def __eq__(self, other): + return type(other) is type(self) and other.type == self.type and other.parameters == self.parameters + + +class PyTorchOperation(Operation): + @classmethod + def _find_subclass(cls, subclass_name): + if cls.to_class_name(subclass_name) is not None: + subclass_name = 'ModuleOperator' + if cls.is_functional(subclass_name): + subclass_name = 'FunctionalOperator' + for subclass in cls.__subclasses__(): + if hasattr(subclass, '_ori_type_name') and \ + subclass_name in subclass._ori_type_name: + return subclass + return cls + + @classmethod + def to_class_name(cls, type_name) -> str: + if type_name.startswith('__torch__.'): + return type_name[len('__torch__.'):] + elif type_name.startswith('__mutated__.'): + return type_name[len('__mutated__.'):] + else: + return None + + @classmethod + def is_functional(cls, type_name) -> bool: + return type_name.startswith('Function.') + + def _to_class_name(self) -> str: + if self.type.startswith('__torch__.'): + return self.type[len('__torch__.'):] + elif self.type.startswith('__mutated__.'): + return self.type[len('__mutated__.'):] + else: + return None + + def get_import_pkg(self) -> str: + if self.type.startswith('__torch__.'): + return self.type[len('__torch__.'):].split('.')[0] + elif self.type.startswith('__mutated__.'): + return self.type[len('__mutated__.'):].split('.')[0] + else: + return None + + def to_init_code(self, field: str) -> str: + if self._to_class_name() is not None: + assert 'positional_args' not in self.parameters + kw_params = ', '.join(f'{key}={repr(value)}' for key, value in self.parameters.items()) + return f'self.{field} = {self._to_class_name()}({kw_params})' + return None + + def to_forward_code(self, field: str, output: str, inputs: List[str], inputs_value: List[Any] = None) -> str: + """ + Parameters + ---------- + field : str + the name of member submodule + output : str + the output name (lvalue) of this line of code + inputs : List[str] + variables used in this line of code + inputs_value : List[Any] + some variables are actually constant, their real values are recorded in ```inputs_value```. + if not constant, we simply put None at the corresponding index + + Returns + ------- + str + generated code line + """ + if self.type == 'aten::slice': + raise RuntimeError('not supposed to have aten::slice operation') + else: + raise RuntimeError(f'unsupported operation type: {self.type} ? {self._to_class_name()}') + + +class TensorFlowOperation(Operation): + def _to_class_name(self) -> str: + return 'K.layers.' + self.type + + +class Cell(PyTorchOperation): + """ + TODO: this is pytorch cell + + An operation reference to a subgraph. + + Example code: + ``` + def __init__(...): + ... + self.cell = CustomCell(...) + self.relu = K.layers.ReLU() + ... + + def forward(...): + ... + x = self.cell(x) + ... + ``` + + In above example, node `self.cell`'s operation is `Cell(cell_name='CustomCell')`. + For comparison, `self.relu`'s operation is `Operation(type='ReLU')`. + + TODO: parameters of subgraph (see `Node` class) + + Attributes + ---------- + type + Always "_cell". + parameters + A dict with only one item; the key is "cell" and the value is cell's name. + framework + No real usage. Exists for compatibility with base class. + """ + + def __init__(self, cell_name: str, parameters: Dict[str, Any] = {}): + self.type = '_cell' + self.cell_name = cell_name + self.parameters = parameters + + def _to_class_name(self): + # TODO: ugly, think about how to refactor this part + return _convert_name(self.cell_name) + + def to_forward_code(self, field: str, output: str, inputs: List[str], inputs_value: List[Any] = None) -> str: + return f'{output} = self.{field}({", ".join(inputs)})' + +class _IOPseudoOperation(Operation): + """ + This is the pseudo operation used by I/O nodes. + The benefit is that users no longer need to verify `Node.operation is not None`, + especially in static type checking. + """ + + def __init__(self, type_name: str, io_names: List = None): + assert type_name.startswith('_') + super(_IOPseudoOperation, self).__init__(type_name, {}, True) + self.io_names = io_names + + def to_init_code(self, field: str) -> str: + raise ValueError(f'Cannot generate code for pseudo operation "{self.type}"') + + def to_forward_code(self, field: str, output: str, inputs: List[str]) -> str: + raise ValueError(f'Cannot generate code for pseudo operation "{self.type}"') + + def __bool__(self) -> bool: + return False diff --git a/new_impl/cv/third_party/nni_new/retiarii/operation_def/__init__.py b/new_impl/cv/third_party/nni_new/retiarii/operation_def/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..0b9575cc55a0a035af3de2c1608b60bc7631a676 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/retiarii/operation_def/__init__.py @@ -0,0 +1,7 @@ +""" +Definition of operation types. + +These are currently examples for overriding codegen. + +Feel free to propose better package name or hierarchy. +""" diff --git a/new_impl/cv/third_party/nni_new/retiarii/operation_def/tf_op_def.py b/new_impl/cv/third_party/nni_new/retiarii/operation_def/tf_op_def.py new file mode 100644 index 0000000000000000000000000000000000000000..d7bd64726800028be2d5874beb726a46fb7af694 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/retiarii/operation_def/tf_op_def.py @@ -0,0 +1,11 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from ..operation import TensorFlowOperation + + +class Conv2D(TensorFlowOperation): + def __init__(self, type_name, parameters, _internal): + if 'padding' not in parameters: + parameters['padding'] = 'same' + super().__init__(type_name, parameters, _internal) diff --git a/new_impl/cv/third_party/nni_new/retiarii/operation_def/torch_op_def.py b/new_impl/cv/third_party/nni_new/retiarii/operation_def/torch_op_def.py new file mode 100644 index 0000000000000000000000000000000000000000..bb97069e639af23b179f815759c81423d82f6d62 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/retiarii/operation_def/torch_op_def.py @@ -0,0 +1,429 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from typing import (Any, List) + +import torch + +from ..operation import PyTorchOperation + + +mem_format = [ + 'torch.contiguous_format', # 0 + 'torch.preserve_format', # 1 + 'torch.channels_last', # 2 +] + +# this snippet is copied from torch/onnx/symbolic_helper.py, +# the original definition is in c10/core/ScalarType.h +# This indicates each scalar type's corresponding +scalar_type_to_pytorch_type = [ + 'torch.uint8', # 0 + 'torch.int8', # 1 + 'torch.short', # 2 + 'torch.int', # 3 + 'torch.int64', # 4 + 'torch.half', # 5 + 'torch.float', # 6 + 'torch.double', # 7 + 'torch.complex32', # 8 + 'torch.complex64', # 9 + 'torch.complex128', # 10 + 'torch.bool', # 11 +] + +class NoOpIdentity(PyTorchOperation): + """ + this operator type is added by us + """ + _ori_type_name = ['noop_identity'] + def to_forward_code(self, field: str, output: str, inputs: List[str], inputs_value: List[Any] = None) -> str: + return f'{output} = {", ".join(inputs)}' + +class ModuleOperator(PyTorchOperation): + _ori_type_name = ['ModuleOperator', 'shared'] + def to_forward_code(self, field: str, output: str, inputs: List[str], inputs_value: List[Any] = None) -> str: + return f'{output} = self.{field}({", ".join(inputs)})' + +class FunctionalOperator(PyTorchOperation): + _ori_type_name = ['FunctionalOperator'] + def to_forward_code(self, field: str, output: str, inputs: List[str], inputs_value: List[Any] = None) -> str: + func_name = self.type[len('Function.'):] + if not hasattr(torch.nn.functional, func_name): + raise RuntimeError('For now, we only support calling independent functions from `torch.nn.functional`, ' + f'{func_name} is not in it.') + return f'{output} = F.{func_name}({", ".join(inputs)})' + +class PrimConstant(PyTorchOperation): + _ori_type_name = ['prim::Constant'] + def to_forward_code(self, field: str, output: str, inputs: List[str], inputs_value: List[Any] = None) -> str: + # TODO: refactor this part, maybe we can remove the code gen of prim::Constant + # TODO: deal with all the types + if self.parameters['type'] == 'None': + return f'{output} = None' + elif self.parameters['type'] in ('int', 'float', 'bool', 'int[]'): + return f'{output} = {self.parameters["value"]}' + elif self.parameters['type'] == 'str': + str_val = self.parameters["value"] + return f'{output} = "{str_val}"' + elif self.parameters['type'] == 'Device': + value = self.parameters['value'] + return f'{output} = torch.device("{value}")' + elif self.parameters['type'] in ('dict', 'list', 'tuple'): + # TODO: prim::TupleIndex is not supported yet + return f'{output} = {repr(self.parameters["value"])}' + else: + raise RuntimeError(f'unsupported type of prim::Constant: {self.parameters["type"]}') + +class PrimListConstruct(PyTorchOperation): + _ori_type_name = ['prim::ListConstruct'] + def to_forward_code(self, field: str, output: str, inputs: List[str], inputs_value: List[Any] = None) -> str: + return f'{output} = [{", ".join(inputs)}]' + +class PrimListUnpack(PyTorchOperation): + _ori_type_name = ['prim::ListUnpack'] + def to_forward_code(self, field: str, output: str, inputs: List[str], inputs_value: List[Any] = None) -> str: + return f'{output} = {inputs[0]}' + +class PrimTupleConstruct(PyTorchOperation): + _ori_type_name = ['prim::TupleConstruct'] + def to_forward_code(self, field: str, output: str, inputs: List[str], inputs_value: List[Any] = None) -> str: + return f'{output} = ({", ".join(inputs)})' + +class PrimTupleUnpack(PyTorchOperation): + _ori_type_name = ['prim::TupleUnpack'] + def to_forward_code(self, field: str, output: str, inputs: List[str], inputs_value: List[Any] = None) -> str: + # have single output here, because the following code uses index to access the unpacked values + assert len(inputs) == 1 + return f'{output} = {inputs[0]}' + +class PrimGetAttr(PyTorchOperation): + _ori_type_name = ['prim::GetAttr'] + def to_forward_code(self, field: str, output: str, inputs: List[str], inputs_value: List[Any] = None) -> str: + if self.parameters['value'] is not None: + return f"{output} = {self.parameters['value']}" + else: + return f"{output} = {self.parameters['input']}.{self.parameters['name']}" + +class SimpleMember(PyTorchOperation): + _ori_type_name = ['prim::is_cuda', 'prim::data'] + def to_forward_code(self, field: str, output: str, inputs: List[str], inputs_value: List[Any] = None) -> str: + member_name = self.type.split('::')[-1] + return f'{output} = {inputs[0]}.{member_name}' + +class AtenContiguous(PyTorchOperation): + _ori_type_name = ['aten::contiguous'] + def to_forward_code(self, field: str, output: str, inputs: List[str], inputs_value: List[Any] = None) -> str: + # defined in pytorch/c10/core/MemoryFormat.h + assert inputs_value[1] in [0, 1, 2] + return f'{output} = {inputs[0]}.contiguous(memory_format={mem_format[inputs_value[1]]})' + +class AtenGetitem(PyTorchOperation): + _ori_type_name = ['aten::__getitem__'] + def to_forward_code(self, field: str, output: str, inputs: List[str], inputs_value: List[Any] = None) -> str: + assert len(inputs) == 2 + return f'{output} = {inputs[0]}[{inputs[1]}]' + +class AtenAppend(PyTorchOperation): + _ori_type_name = ['aten::append'] + def to_forward_code(self, field: str, output: str, inputs: List[str], inputs_value: List[Any] = None) -> str: + assert len(inputs) == 2 + return f'_, {output} = {inputs[0]}.append({inputs[1]}), {inputs[0]}' + +class MergedSlice(PyTorchOperation): + _ori_type_name = ['MergedSlice'] + def to_forward_code(self, field: str, output: str, inputs: List[str], inputs_value: List[Any] = None) -> str: + if (len(inputs) - 1) % 4 == 0: + slices = [] + dim = int((len(inputs) - 1) / 4) + for i in range(dim): + slices.append(f'{inputs[i*4+2]}:{inputs[i*4+3]}:{inputs[i*4+4]}') + slice_str = ','.join(slices) + return f'{output} = {inputs[0]}[{slice_str}]' + elif len(inputs) == 4: + # this case is for simple list + return f'{output} = {inputs[0]}[{inputs[1]}:{inputs[2]}:{inputs[3]}]' + else: + raise RuntimeError('Unsupported slice pattern') + +# the following Aten classes means these aten ops are not in torch.Tensor + +class AtenBool(PyTorchOperation): + _ori_type_name = ['aten::Bool'] + def to_forward_code(self, field: str, output: str, inputs: List[str], inputs_value: List[Any] = None) -> str: + return f'{output} = bool({inputs[0]})' + +class AtenNot(PyTorchOperation): + _ori_type_name = ['aten::__not__'] + def to_forward_code(self, field: str, output: str, inputs: List[str], inputs_value: List[Any] = None) -> str: + return f'{output} = not {inputs[0]}' + +class AtenCat(PyTorchOperation): + _ori_type_name = ['aten::cat'] + def to_forward_code(self, field: str, output: str, inputs: List[str], inputs_value: List[Any] = None) -> str: + assert len(inputs) == 2 + return f'{output} = torch.cat({inputs[0]}, dim={inputs[1]})' + +#==================================== + +class AtenTensors(PyTorchOperation): + _ori_type_name = ['aten::full', 'aten::full_like', 'aten::empty_like', + 'aten::ones_like', 'aten::zeros_like', 'aten::rand', + 'aten::randn', 'aten::scalar_tensor', 'aten::new_full', + 'aten::new_empty', 'aten::new_zeros', 'aten::arange', + 'aten::tensor', 'aten::ones', 'aten::zeros'] + + def to_forward_code(self, field: str, output: str, inputs: List[str], inputs_value: List[Any] = None) -> str: + schemas = torch._C._jit_get_schemas_for_operator(self.type) + # match number of inputs + overloaded_defs = [len(s.arguments) for s in schemas] + matched = overloaded_defs.index(len(inputs)) + args_list = [] + for idx, arg in enumerate(schemas[matched].arguments): + if arg.name == 'dtype': + arg_str = f'dtype={scalar_type_to_pytorch_type[inputs_value[idx]]}' if inputs_value[idx] is not None else '' + elif arg.name == 'layout': + if inputs_value[idx] is not None: + arg_str = f'layout=torch.strided' + print('Warning: only support `torch.strided` for now!!!') + else: + arg_str = '' + elif arg.name == 'device': + arg_str = f'device=torch.device({inputs[idx]})' if inputs_value[idx] is not None else '' + elif arg.name == 'memory_format': + arg_str = f'memory_format={mem_format[inputs_value[idx]]}' if inputs_value[idx] is not None else '' + elif arg.name == 'pin_memory': + # TODO: deal with this argument + continue + elif arg.name == 'requires_grad': + arg_str = f'requires_grad={inputs[idx]}' if inputs_value[idx] else '' + elif str(arg.type).startswith('Optional['): + arg_str = f'{arg.name}={inputs[idx]}' + else: + arg_str = f'{inputs[idx]}' + if arg_str != '': + args_list.append(arg_str) + op_name = self.type.split('::')[-1] + if hasattr(torch, op_name): + return f'{output} = torch.{op_name}({", ".join(args_list)})' + else: + return f'{output} = {inputs[0]}.{op_name}({", ".join(args_list[1:])})' + +#==================================== + +class AtenFloordiv(PyTorchOperation): + _ori_type_name = ['aten::floordiv'] + def to_forward_code(self, field: str, output: str, inputs: List[str], inputs_value: List[Any] = None) -> str: + return f'{output} = {inputs[0]} // {inputs[1]}' + +class AtenLen(PyTorchOperation): + _ori_type_name = ['aten::len'] + def to_forward_code(self, field: str, output: str, inputs: List[str], inputs_value: List[Any] = None) -> str: + return f'{output} = len({inputs[0]})' + +class AtenIntImplicit(PyTorchOperation): + _ori_type_name = ['aten::IntImplicit', 'aten::Float', 'aten::Int', 'aten::ScalarImplicit'] + def to_forward_code(self, field: str, output: str, inputs: List[str], inputs_value: List[Any] = None) -> str: + if self.type.endswith('Implicit'): + return f'{output} = {inputs[0]}' + elif self.type == 'aten::Int': + return f'{output} = int({inputs[0]})' + elif self.type == 'aten::Float': + return f'{output} = float({inputs[0]})' + +class AtenIndex(PyTorchOperation): + _ori_type_name = ['aten::index'] + def to_forward_code(self, field: str, output: str, inputs: List[str], inputs_value: List[Any] = None) -> str: + return f'{output} = {inputs[0]}[{inputs[1]}]' + +ManuallyChooseDef = { + 'aten::flatten': [('start_dim', 'int', '0'), ('end_dim', 'int', '-1')], + 'aten::split': [('split_size', 'int', 'None'), ('dim', 'int', '0')] +} + +TensorOpExceptions = { + 'aten::sub': lambda output, inputs: f'{output} = {inputs[0]} - {inputs[1]}', # example: x.size(1) - 3 + 'aten::add': lambda output, inputs: f'{output} = {inputs[0]} + {inputs[1]}' # example: input.shape[0] + 5 +} + +TorchOpExclude = ['aten::Size', 'aten::as_tensor', 'aten::device', + 'aten::manual_seed', 'aten::quantized_gru', 'aten::quantized_lstm', + 'aten::save', 'aten::tensor', 'aten::wait' +] + +def _hidden(name): + return name.startswith('_') and not name.startswith('__') + +def _emit_args(args): + # filter out the `out` argument here + return [(arg.name, str(arg.type), str(arg.default_value)) for arg in args] # if arg.name != 'out' + +def _get_tensor_ops(): + def is_tensor_method(schema): + if len(schema.arguments) == 0: + return False + self = schema.arguments[0] + if self.name != 'self': + return False + if not self.type.isSubtypeOf(torch._C.TensorType.get()): + return False + return True + + op_args = {} + # discover methods + for elem in dir(torch.Tensor): + if not _hidden(elem): + schemas = torch._C._jit_get_schemas_for_operator("aten::" + elem) + for schema in schemas: + if is_tensor_method(schema): + op_name = 'aten::' + elem + args = _emit_args(schema.arguments[1:]) + if op_name in op_args: + op_args[op_name].append(args) + else: + op_args[op_name] = [args] + + return op_args.keys(), op_args + +def _get_torch_ops(): + torch_op_args = {} + for mod in torch.jit._builtins._modules_containing_builtins: + name = mod.__name__ + if name == 'torch._C._nn': + continue + # only process 'torch.XXX' + for elem in dir(mod): + builtin = torch.jit._builtins._find_builtin(getattr(mod, elem)) + if builtin is not None: + schemas = torch._C._jit_get_schemas_for_operator(builtin) + for schema in schemas: + # remove _tan but not __and__ + if not _hidden(elem): + op_name = 'aten::' + elem + if len(schema.arguments) > 0 and schema.arguments[0].name == 'self': + continue + args = _emit_args(schema.arguments) + if op_name in torch_op_args: + torch_op_args[op_name].append(args) + else: + torch_op_args[op_name] = [args] + + return torch_op_args.keys(), torch_op_args + +def _get_torch_ops_exclude_tensor_ops(): + tensor_op_names, _ = _get_tensor_ops() + torch_op_names, torch_ops = _get_torch_ops() + + torch_exclude_ops = {} + for name in torch_op_names: + if name not in tensor_op_names: + if name not in TorchOpExclude: + # exclude the ops that are not in + # https://github.com/pytorch/pytorch/blob/master/aten/src/ATen/native/native_functions.yaml + torch_exclude_ops[name] = torch_ops[name] + + return torch_exclude_ops.keys(), torch_exclude_ops + +class TensorOps(PyTorchOperation): + """ + corresponding to _get_tensor_ops in torch.jit.supported_ops + """ + _ori_type_name, _op_args = _get_tensor_ops() + + comparison_ops = {'aten::eq': '==', 'aten::ne': '!=', 'aten::le': '<=', 'aten::ge': '>=', 'aten::lt': '<', 'aten::gt': '>'} + + @staticmethod + def _get_matched_args(_type, inputs): + def has_same_arg_name(matched): + concated_names = [] + for i, each in enumerate(matched): + name = ','.join([arg[0] for arg in each]) + concated_names.append(name) + for i in range(len(concated_names) - 1): + if concated_names[i] != concated_names[i+1]: + return False + return True + + overloaded_defs = TensorOps._op_args[_type] + matched = [] + for each in overloaded_defs: + # plus 1 because we skip the first argument when generating tensor op def + if len(each) + 1 == len(inputs): + matched.append(each) + if len(matched) == 1: + return matched[0] + elif len(matched) > 1: + # TODO: match with arg's type. manually choose for now + if has_same_arg_name(matched): + # return any one is okay + return matched[0] + elif _type in ManuallyChooseDef: + return ManuallyChooseDef[_type] + else: + raise RuntimeError(f'tensor op type {_type} has more than one matched: {matched}') + else: + if _type in TensorOpExceptions: + return None + raise RuntimeError(f'tensor op type {_type} has no matched') + + def to_forward_code(self, field: str, output: str, inputs: List[str], inputs_value: List[Any] = None) -> str: + # TODO: deal with conditional ops + if self.type in TensorOps.comparison_ops: + return f'{output} = ({inputs[0]} {TensorOps.comparison_ops[self.type]} {inputs[1]})' + matched_args = TensorOps._get_matched_args(self.type, inputs) + if matched_args is None: + return TensorOpExceptions[self.type](output, inputs) + op_name = self.type.split('::')[-1] + args_str = ', '.join([f'{name}={inputs[i+1]}' for i, (name, t, default) in enumerate(matched_args)]) + return f'{output} = {inputs[0]}.{op_name}({args_str})' + +class TorchOps(PyTorchOperation): + """ + corresponding to _get_nn_functional_ops in torch.jit.supported_ops + """ + _ori_type_name, _op_args = _get_torch_ops_exclude_tensor_ops() + # add 'aten::pixel_shuffle' + _op_args['aten::pixel_shuffle'] = [[('input', 'Tensor', 'None'), ('upscale_factor', 'Optional[int]', 'None')]] + _ori_type_name = _op_args.keys() + + @staticmethod + def _get_matched_args(_type, inputs): + def has_same_arg_name(matched): + concated_names = [] + for i, each in enumerate(matched): + name = ','.join([arg[0] for arg in each]) + concated_names.append(name) + for i in range(len(concated_names) - 1): + if concated_names[i] != concated_names[i+1]: + return False + return True + + overloaded_defs = TorchOps._op_args[_type] + matched = [] + for each in overloaded_defs: + if len(each) == len(inputs): + matched.append(each) + if len(matched) == 1: + return matched[0] + elif len(matched) > 1: + # TODO: match with arg's type. manually choose for now + if has_same_arg_name(matched): + # return any one is okay + return matched[0] + else: + raise RuntimeError(f'torch op type {_type} has more than one matched: {matched}') + else: + raise RuntimeError(f'torch op type {_type} has no matched') + + def to_forward_code(self, field: str, output: str, inputs: List[str], inputs_value: List[Any] = None) -> str: + matched_args = TorchOps._get_matched_args(self.type, inputs) + op_name = self.type.split('::')[-1] + args_str = ', '.join([f'{name}={inputs[i]}' if t.startswith('Optional[') else f'{inputs[i]}' \ + for i, (name, t, default) in enumerate(matched_args)]) + return f'{output} = torch.{op_name}({args_str})' + +class AtenAvgpool2d(PyTorchOperation): + # NOTE: it is not included in the above aten ops for unkown reason + _ori_type_name = ['aten::avg_pool2d'] + def to_forward_code(self, field: str, output: str, inputs: List[str], inputs_value: List[Any] = None) -> str: + return f'{output} = F.avg_pool2d({", ".join(inputs)})' \ No newline at end of file diff --git a/new_impl/cv/third_party/nni_new/retiarii/serializer.py b/new_impl/cv/third_party/nni_new/retiarii/serializer.py new file mode 100644 index 0000000000000000000000000000000000000000..efa78243bdb3df2a3edecbc4bb8bf0d77833df0d --- /dev/null +++ b/new_impl/cv/third_party/nni_new/retiarii/serializer.py @@ -0,0 +1,166 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import abc +import functools +import inspect +import types +from typing import Any + +import json_tricks + +from .utils import get_importable_name, get_module_name, import_, reset_uid + + +def get_init_parameters_or_fail(obj, silently=False): + if hasattr(obj, '_init_parameters'): + return obj._init_parameters + elif silently: + return None + else: + raise ValueError(f'Object {obj} needs to be serializable but `_init_parameters` is not available. ' + 'If it is a built-in module (like Conv2d), please import it from retiarii.nn. ' + 'If it is a customized module, please to decorate it with @basic_unit. ' + 'For other complex objects (e.g., trainer, optimizer, dataset, dataloader), ' + 'try to use serialize or @serialize_cls.') + + +### This is a patch of json-tricks to make it more useful to us ### + + +def _serialize_class_instance_encode(obj, primitives=False): + assert not primitives, 'Encoding with primitives is not supported.' + try: # FIXME: raise error + if hasattr(obj, '__class__'): + return { + '__type__': get_importable_name(obj.__class__), + 'arguments': get_init_parameters_or_fail(obj) + } + except ValueError: + pass + return obj + + +def _serialize_class_instance_decode(obj): + if isinstance(obj, dict) and '__type__' in obj and 'arguments' in obj: + return import_(obj['__type__'])(**obj['arguments']) + return obj + + +def _type_encode(obj, primitives=False): + assert not primitives, 'Encoding with primitives is not supported.' + if isinstance(obj, type): + return {'__typename__': get_importable_name(obj, relocate_module=True)} + if isinstance(obj, (types.FunctionType, types.BuiltinFunctionType)): + # This is not reliable for cases like closure, `open`, or objects that is callable but not intended to be serialized. + # https://stackoverflow.com/questions/624926/how-do-i-detect-whether-a-python-variable-is-a-function + return {'__typename__': get_importable_name(obj, relocate_module=True)} + return obj + + +def _type_decode(obj): + if isinstance(obj, dict) and '__typename__' in obj: + return import_(obj['__typename__']) + return obj + + +json_loads = functools.partial(json_tricks.loads, extra_obj_pairs_hooks=[_serialize_class_instance_decode, _type_decode]) +json_dumps = functools.partial(json_tricks.dumps, extra_obj_encoders=[_serialize_class_instance_encode, _type_encode]) +json_load = functools.partial(json_tricks.load, extra_obj_pairs_hooks=[_serialize_class_instance_decode, _type_decode]) +json_dump = functools.partial(json_tricks.dump, extra_obj_encoders=[_serialize_class_instance_encode, _type_encode]) + +### End of json-tricks patch ### + + +class Translatable(abc.ABC): + """ + Inherit this class and implement ``translate`` when the inner class needs a different + parameter from the wrapper class in its init function. + """ + + @abc.abstractmethod + def _translate(self) -> Any: + pass + + +def _create_wrapper_cls(cls, store_init_parameters=True, reset_mutation_uid=False): + class wrapper(cls): + def __init__(self, *args, **kwargs): + if reset_mutation_uid: + reset_uid('mutation') + if store_init_parameters: + argname_list = list(inspect.signature(cls.__init__).parameters.keys())[1:] + full_args = {} + full_args.update(kwargs) + + assert len(args) <= len(argname_list), f'Length of {args} is greater than length of {argname_list}.' + for argname, value in zip(argname_list, args): + full_args[argname] = value + + # translate parameters + args = list(args) + for i, value in enumerate(args): + if isinstance(value, Translatable): + args[i] = value._translate() + for i, value in kwargs.items(): + if isinstance(value, Translatable): + kwargs[i] = value._translate() + + self._init_parameters = full_args + else: + self._init_parameters = {} + + super().__init__(*args, **kwargs) + + wrapper.__module__ = get_module_name(cls) + wrapper.__name__ = cls.__name__ + wrapper.__qualname__ = cls.__qualname__ + wrapper.__init__.__doc__ = cls.__init__.__doc__ + + return wrapper + + +def serialize_cls(cls): + """ + To create an serializable class. + """ + return _create_wrapper_cls(cls) + + +def transparent_serialize(cls): + """ + Wrap a module but does not record parameters. For internal use only. + """ + return _create_wrapper_cls(cls, store_init_parameters=False) + + +def serialize(cls, *args, **kwargs): + """ + To create an serializable instance inline without decorator. For example, + + .. code-block:: python + + self.op = serialize(MyCustomOp, hidden_units=128) + """ + return serialize_cls(cls)(*args, **kwargs) + + +def basic_unit(cls): + """ + To wrap a module as a basic unit, to stop it from parsing and make it mutate-able. + """ + import torch.nn as nn + assert issubclass(cls, nn.Module), 'When using @basic_unit, the class must be a subclass of nn.Module.' + return serialize_cls(cls) + + +def model_wrapper(cls): + """ + Wrap the model if you are using pure-python execution engine. + + The wrapper serves two purposes: + + 1. Capture the init parameters of python class so that it can be re-instantiated in another process. + 2. Reset uid in `mutation` namespace so that each model counts from zero. Can be useful in unittest and other multi-model scenarios. + """ + return _create_wrapper_cls(cls, reset_mutation_uid=True) diff --git a/new_impl/cv/third_party/nni_new/retiarii/strategy/__init__.py b/new_impl/cv/third_party/nni_new/retiarii/strategy/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..04511eaa69b85666e9ebc69b643bc0c74fe6467d --- /dev/null +++ b/new_impl/cv/third_party/nni_new/retiarii/strategy/__init__.py @@ -0,0 +1,9 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .base import BaseStrategy +from .bruteforce import Random, GridSearch +from .evolution import RegularizedEvolution +from .tpe_strategy import TPEStrategy +from .local_debug_strategy import _LocalDebugStrategy +from .rl import PolicyBasedRL diff --git a/new_impl/cv/third_party/nni_new/retiarii/strategy/_rl_impl.py b/new_impl/cv/third_party/nni_new/retiarii/strategy/_rl_impl.py new file mode 100644 index 0000000000000000000000000000000000000000..7e038aab6984c5238549d4aefdd778c609a10bff --- /dev/null +++ b/new_impl/cv/third_party/nni_new/retiarii/strategy/_rl_impl.py @@ -0,0 +1,161 @@ +# This file might cause import error for those who didn't install RL-related dependencies + +import logging +import threading +from multiprocessing.pool import ThreadPool + +import gym +import numpy as np +import torch +import torch.nn as nn + +from gym import spaces +from tianshou.data import to_torch +from tianshou.env.worker import EnvWorker + +from .utils import get_targeted_model +from ..graph import ModelStatus +from ..execution import submit_models, wait_models + + +_logger = logging.getLogger(__name__) +_thread_lock = threading.Lock() + + +class MultiThreadEnvWorker(EnvWorker): + def __init__(self, env_fn): + self.env = env_fn() + self.pool = ThreadPool(processes=1) + super().__init__(env_fn) + + def __getattr__(self, key): + return getattr(self.env, key) + + def reset(self): + return self.env.reset() + + @staticmethod + def wait(*args, **kwargs): + raise NotImplementedError('Async collect is not supported yet.') + + def send_action(self, action) -> None: + # self.result is actually a handle + self.result = self.pool.apply_async(self.env.step, (action,)) + + def get_result(self): + return self.result.get() + + def seed(self, seed): + super().seed(seed) + return self.env.seed(seed) + + def render(self, **kwargs): + return self.env.render(**kwargs) + + def close_env(self) -> None: + self.pool.terminate() + return self.env.close() + + +class ModelEvaluationEnv(gym.Env): + def __init__(self, base_model, mutators, search_space): + self.base_model = base_model + self.mutators = mutators + self.search_space = search_space + self.ss_keys = list(self.search_space.keys()) + self.action_dim = max(map(lambda v: len(v), self.search_space.values())) + self.num_steps = len(self.search_space) + + @property + def observation_space(self): + return spaces.Dict({ + 'action_history': spaces.MultiDiscrete([self.action_dim] * self.num_steps), + 'cur_step': spaces.Discrete(self.num_steps + 1), + 'action_dim': spaces.Discrete(self.action_dim + 1) + }) + + @property + def action_space(self): + return spaces.Discrete(self.action_dim) + + def reset(self): + self.action_history = np.zeros(self.num_steps, dtype=np.int32) + self.cur_step = 0 + self.sample = {} + return { + 'action_history': self.action_history, + 'cur_step': self.cur_step, + 'action_dim': len(self.search_space[self.ss_keys[self.cur_step]]) + } + + def step(self, action): + cur_key = self.ss_keys[self.cur_step] + assert action < len(self.search_space[cur_key]), \ + f'Current action {action} out of range {self.search_space[cur_key]}.' + self.action_history[self.cur_step] = action + self.sample[cur_key] = self.search_space[cur_key][action] + self.cur_step += 1 + obs = { + 'action_history': self.action_history, + 'cur_step': self.cur_step, + 'action_dim': len(self.search_space[self.ss_keys[self.cur_step]]) \ + if self.cur_step < self.num_steps else self.action_dim + } + if self.cur_step == self.num_steps: + with _thread_lock: + model = get_targeted_model(self.base_model, self.mutators, self.sample) + _logger.info(f'New model created: {self.sample}') + submit_models(model) + wait_models(model) + if model.status == ModelStatus.Failed: + return self.reset(), 0., False, {} + rew = model.metric + _logger.info(f'Model metric received as reward: {rew}') + return obs, rew, True, {} + else: + + return obs, 0., False, {} + + +class Preprocessor(nn.Module): + def __init__(self, obs_space, hidden_dim=64, num_layers=1): + super().__init__() + self.action_dim = obs_space['action_history'].nvec[0] + self.hidden_dim = hidden_dim + # first token is [SOS] + self.embedding = nn.Embedding(self.action_dim + 1, hidden_dim) + self.rnn = nn.LSTM(hidden_dim, hidden_dim, num_layers, batch_first=True) + + def forward(self, obs): + seq = nn.functional.pad(obs['action_history'] + 1, (1, 1)) # pad the start token and end token + # end token is used to avoid out-of-range of v_s_. Will not actually affect BP. + seq = self.embedding(seq.long()) + feature, _ = self.rnn(seq) + return feature[torch.arange(len(feature), device=feature.device), obs['cur_step'].long() + 1] + + +class Actor(nn.Module): + def __init__(self, action_space, preprocess): + super().__init__() + self.preprocess = preprocess + self.action_dim = action_space.n + self.linear = nn.Linear(self.preprocess.hidden_dim, self.action_dim) + + def forward(self, obs, **kwargs): + obs = to_torch(obs, device=self.linear.weight.device) + out = self.linear(self.preprocess(obs)) + # to take care of choices with different number of options + mask = torch.arange(self.action_dim).expand(len(out), self.action_dim) >= obs['action_dim'].unsqueeze(1) + out[mask.to(out.device)] = float('-inf') + return nn.functional.softmax(out, dim=-1), kwargs.get('state', None) + + +class Critic(nn.Module): + def __init__(self, preprocess): + super().__init__() + self.preprocess = preprocess + self.linear = nn.Linear(self.preprocess.hidden_dim, 1) + + def forward(self, obs, **kwargs): + obs = to_torch(obs, device=self.linear.weight.device) + return self.linear(self.preprocess(obs)).squeeze(-1) diff --git a/new_impl/cv/third_party/nni_new/retiarii/strategy/base.py b/new_impl/cv/third_party/nni_new/retiarii/strategy/base.py new file mode 100644 index 0000000000000000000000000000000000000000..894da87b784b901346424e503067bae8e276de16 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/retiarii/strategy/base.py @@ -0,0 +1,15 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import abc +from typing import List + +from ..graph import Model +from ..mutator import Mutator + + +class BaseStrategy(abc.ABC): + + @abc.abstractmethod + def run(self, base_model: Model, applied_mutators: List[Mutator]) -> None: + pass diff --git a/new_impl/cv/third_party/nni_new/retiarii/strategy/bruteforce.py b/new_impl/cv/third_party/nni_new/retiarii/strategy/bruteforce.py new file mode 100644 index 0000000000000000000000000000000000000000..971711f0d9ab83c6581cb660c7b1d4430c1d4316 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/retiarii/strategy/bruteforce.py @@ -0,0 +1,124 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import copy +import itertools +import logging +import random +import time +from typing import Any, Dict, List + +from .. import Sampler, submit_models, query_available_resources, budget_exhausted +from .base import BaseStrategy +from .utils import dry_run_for_search_space, get_targeted_model + +_logger = logging.getLogger(__name__) + + +def grid_generator(search_space: Dict[Any, List[Any]], shuffle=True): + keys = list(search_space.keys()) + search_space_values = copy.deepcopy(list(search_space.values())) + if shuffle: + for values in search_space_values: + random.shuffle(values) + for values in itertools.product(*search_space_values): + yield {key: value for key, value in zip(keys, values)} + + +def random_generator(search_space: Dict[Any, List[Any]], dedup=True, retries=500): + keys = list(search_space.keys()) + history = set() + search_space_values = copy.deepcopy(list(search_space.values())) + while True: + for retry_count in range(retries): + selected = [random.choice(v) for v in search_space_values] + if not dedup: + break + selected = tuple(selected) + if selected not in history: + history.add(selected) + break + if retry_count + 1 == retries: + _logger.debug('Random generation has run out of patience. There is nothing to search. Exiting.') + return + yield {key: value for key, value in zip(keys, selected)} + + +class GridSearch(BaseStrategy): + """ + Traverse the search space and try all the possible combinations one by one. + + Parameters + ---------- + shuffle : bool + Shuffle the order in a candidate list, so that they are tried in a random order. Default: true. + """ + + def __init__(self, shuffle=True): + self._polling_interval = 2. + self.shuffle = shuffle + + def run(self, base_model, applied_mutators): + search_space = dry_run_for_search_space(base_model, applied_mutators) + for sample in grid_generator(search_space, shuffle=self.shuffle): + _logger.debug('New model created. Waiting for resource. %s', str(sample)) + while query_available_resources() <= 0: + if budget_exhausted(): + return + time.sleep(self._polling_interval) + submit_models(get_targeted_model(base_model, applied_mutators, sample)) + + +class _RandomSampler(Sampler): + def choice(self, candidates, mutator, model, index): + return random.choice(candidates) + + +class Random(BaseStrategy): + """ + Random search on the search space. + + Parameters + ---------- + variational : bool + Do not dry run to get the full search space. Used when the search space has variational size or candidates. Default: false. + dedup : bool + Do not try the same configuration twice. When variational is true, deduplication is not supported. Default: true. + """ + + def __init__(self, variational=False, dedup=True): + self.variational = variational + self.dedup = dedup + if variational and dedup: + raise ValueError('Dedup is not supported in variational mode.') + self.random_sampler = _RandomSampler() + self._polling_interval = 2. + + def run(self, base_model, applied_mutators): + if self.variational: + _logger.info('Random search running in variational mode.') + sampler = _RandomSampler() + for mutator in applied_mutators: + mutator.bind_sampler(sampler) + while True: + avail_resource = query_available_resources() + if avail_resource > 0: + model = base_model + for mutator in applied_mutators: + model = mutator.apply(model) + _logger.debug('New model created. Applied mutators are: %s', str(applied_mutators)) + submit_models(model) + elif budget_exhausted(): + break + else: + time.sleep(self._polling_interval) + else: + _logger.info('Random search running in fixed size mode. Dedup: %s.', 'on' if self.dedup else 'off') + search_space = dry_run_for_search_space(base_model, applied_mutators) + for sample in random_generator(search_space, dedup=self.dedup): + _logger.debug('New model created. Waiting for resource. %s', str(sample)) + while query_available_resources() <= 0: + if budget_exhausted(): + return + time.sleep(self._polling_interval) + submit_models(get_targeted_model(base_model, applied_mutators, sample)) diff --git a/new_impl/cv/third_party/nni_new/retiarii/strategy/evolution.py b/new_impl/cv/third_party/nni_new/retiarii/strategy/evolution.py new file mode 100644 index 0000000000000000000000000000000000000000..ca13d32da1159ece2cc03832932bf29252d689f6 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/retiarii/strategy/evolution.py @@ -0,0 +1,161 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import collections +import dataclasses +import logging +import random +import time + +from ..execution import query_available_resources, submit_models +from ..graph import ModelStatus +from .base import BaseStrategy +from .utils import dry_run_for_search_space, get_targeted_model + + +_logger = logging.getLogger(__name__) + + +@dataclasses.dataclass +class Individual: + """ + A class that represents an individual. + Holds two attributes, where ``x`` is the model and ``y`` is the metric (e.g., accuracy). + """ + x: dict + y: float + + +class RegularizedEvolution(BaseStrategy): + """ + Algorithm for regularized evolution (i.e. aging evolution). + Follows "Algorithm 1" in Real et al. "Regularized Evolution for Image Classifier Architecture Search". + + Parameters + ---------- + optimize_mode : str + Can be one of "maximize" and "minimize". Default: maximize. + population_size : int + The number of individuals to keep in the population. Default: 100. + cycles : int + The number of cycles (trials) the algorithm should run for. Default: 20000. + sample_size : int + The number of individuals that should participate in each tournament. Default: 25. + mutation_prob : float + Probability that mutation happens in each dim. Default: 0.05 + on_failure : str + Can be one of "ignore" and "worst". If "ignore", simply give up the model and find a new one. + If "worst", mark the model as -inf (if maximize, inf if minimize), so that the algorithm "learns" to avoid such model. + Default: ignore. + """ + + def __init__(self, optimize_mode='maximize', population_size=100, sample_size=25, cycles=20000, + mutation_prob=0.05, on_failure='ignore'): + assert optimize_mode in ['maximize', 'minimize'] + assert on_failure in ['ignore', 'worst'] + assert sample_size < population_size + self.optimize_mode = optimize_mode + self.population_size = population_size + self.sample_size = sample_size + self.cycles = cycles + self.mutation_prob = mutation_prob + self.on_failure = on_failure + + self._worst = float('-inf') if self.optimize_mode == 'maximize' else float('inf') + + self._success_count = 0 + self._population = collections.deque() + self._running_models = [] + self._polling_interval = 2. + + def random(self, search_space): + return {k: random.choice(v) for k, v in search_space.items()} + + def mutate(self, parent, search_space): + child = {} + for k, v in parent.items(): + if random.uniform(0, 1) < self.mutation_prob: + # NOTE: we do not exclude the original choice here for simplicity, + # which is slightly different from the original paper. + child[k] = random.choice(search_space[k]) + else: + child[k] = v + return child + + def best_parent(self): + samples = [p for p in self._population] # copy population + random.shuffle(samples) + samples = list(samples)[:self.sample_size] + if self.optimize_mode == 'maximize': + parent = max(samples, key=lambda sample: sample.y) + else: + parent = min(samples, key=lambda sample: sample.y) + return parent.x + + def run(self, base_model, applied_mutators): + search_space = dry_run_for_search_space(base_model, applied_mutators) + # Run the first population regardless concurrency + _logger.info('Initializing the first population.') + while len(self._population) + len(self._running_models) <= self.population_size: + # try to submit new models + while len(self._population) + len(self._running_models) < self.population_size: + config = self.random(search_space) + self._submit_config(config, base_model, applied_mutators) + # collect results + self._move_succeeded_models_to_population() + self._remove_failed_models_from_running_list() + time.sleep(self._polling_interval) + + if len(self._population) >= self.population_size: + break + + # Resource-aware mutation of models + _logger.info('Running mutations.') + while self._success_count + len(self._running_models) <= self.cycles: + # try to submit new models + while query_available_resources() > 0 and self._success_count + len(self._running_models) < self.cycles: + config = self.mutate(self.best_parent(), search_space) + self._submit_config(config, base_model, applied_mutators) + # collect results + self._move_succeeded_models_to_population() + self._remove_failed_models_from_running_list() + time.sleep(self._polling_interval) + + if self._success_count >= self.cycles: + break + + def _submit_config(self, config, base_model, mutators): + _logger.debug('Model submitted to running queue: %s', config) + model = get_targeted_model(base_model, mutators, config) + submit_models(model) + self._running_models.append((config, model)) + return model + + def _move_succeeded_models_to_population(self): + completed_indices = [] + for i, (config, model) in enumerate(self._running_models): + metric = None + if self.on_failure == 'worst' and model.status == ModelStatus.Failed: + metric = self._worst + elif model.status == ModelStatus.Trained: + metric = model.metric + if metric is not None: + individual = Individual(config, metric) + _logger.debug('Individual created: %s', str(individual)) + self._population.append(individual) + if len(self._population) > self.population_size: + self._population.popleft() + completed_indices.append(i) + for i in completed_indices[::-1]: + # delete from end to start so that the index number will not be affected. + self._success_count += 1 + self._running_models.pop(i) + + def _remove_failed_models_from_running_list(self): + # This is only done when on_failure policy is set to "ignore". + # Otherwise, failed models will be treated as inf when processed. + if self.on_failure == 'ignore': + number_of_failed_models = len([g for g in self._running_models if g[1].status == ModelStatus.Failed]) + self._running_models = [g for g in self._running_models if g[1].status != ModelStatus.Failed] + if number_of_failed_models > 0: + _logger.info('%d failed models are ignored. Will retry.', number_of_failed_models) diff --git a/new_impl/cv/third_party/nni_new/retiarii/strategy/local_debug_strategy.py b/new_impl/cv/third_party/nni_new/retiarii/strategy/local_debug_strategy.py new file mode 100644 index 0000000000000000000000000000000000000000..743d6b2fc62b113c25c8cae748a490f219978deb --- /dev/null +++ b/new_impl/cv/third_party/nni_new/retiarii/strategy/local_debug_strategy.py @@ -0,0 +1,44 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +import os +import random +import string + +from .. import Sampler, codegen, utils +from ..execution.base import BaseGraphData +from .base import BaseStrategy + +_logger = logging.getLogger(__name__) + +class ChooseFirstSampler(Sampler): + def choice(self, candidates, mutator, model, index): + return candidates[0] + +class _LocalDebugStrategy(BaseStrategy): + """ + This class is supposed to be used internally, for debugging trial mutation + """ + + def run_one_model(self, model): + graph_data = BaseGraphData(codegen.model_to_pytorch_script(model), model.evaluator) + random_str = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(6)) + file_name = f'_generated_model/{random_str}.py' + os.makedirs(os.path.dirname(file_name), exist_ok=True) + with open(file_name, 'w') as f: + f.write(graph_data.model_script) + model_cls = utils.import_(f'_generated_model.{random_str}._model') + graph_data.evaluator._execute(model_cls) + os.remove(file_name) + + def run(self, base_model, applied_mutators): + _logger.info('local debug strategy has been started.') + model = base_model + _logger.debug('New model created. Applied mutators: %s', str(applied_mutators)) + choose_first_sampler = ChooseFirstSampler() + for mutator in applied_mutators: + mutator.bind_sampler(choose_first_sampler) + model = mutator.apply(model) + # directly run models + self.run_one_model(model) diff --git a/new_impl/cv/third_party/nni_new/retiarii/strategy/rl.py b/new_impl/cv/third_party/nni_new/retiarii/strategy/rl.py new file mode 100644 index 0000000000000000000000000000000000000000..23a05ed1ed9ad20d7421b82c0e820482c71aef34 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/retiarii/strategy/rl.py @@ -0,0 +1,78 @@ +import logging +from typing import Optional, Callable + +from .base import BaseStrategy +from .utils import dry_run_for_search_space +from ..execution import query_available_resources + +try: + has_tianshou = True + import torch + from tianshou.data import Collector, VectorReplayBuffer + from tianshou.env import BaseVectorEnv + from tianshou.policy import BasePolicy, PPOPolicy # pylint: disable=unused-import + from ._rl_impl import ModelEvaluationEnv, MultiThreadEnvWorker, Preprocessor, Actor, Critic +except ImportError: + has_tianshou = False + + +_logger = logging.getLogger(__name__) + + +class PolicyBasedRL(BaseStrategy): + """ + Algorithm for policy-based reinforcement learning. + This is a wrapper of algorithms provided in tianshou (PPO by default), + and can be easily customized with other algorithms that inherit ``BasePolicy`` (e.g., REINFORCE [1]_). + + Parameters + ---------- + max_collect : int + How many times collector runs to collect trials for RL. Default 100. + trial_per_collect : int + How many trials (trajectories) each time collector collects. + After each collect, trainer will sample batch from replay buffer and do the update. Default: 20. + policy_fn : function + Takes ``ModelEvaluationEnv`` as input and return a policy. See ``_default_policy_fn`` for an example. + + References + ---------- + + .. [1] Barret Zoph and Quoc V. Le, "Neural Architecture Search with Reinforcement Learning". + https://arxiv.org/abs/1611.01578 + """ + + def __init__(self, max_collect: int = 100, trial_per_collect = 20, + policy_fn: Optional[Callable[['ModelEvaluationEnv'], 'BasePolicy']] = None): + if not has_tianshou: + raise ImportError('`tianshou` is required to run RL-based strategy. ' + 'Please use "pip install tianshou" to install it beforehand.') + + self.policy_fn = policy_fn or self._default_policy_fn + self.max_collect = max_collect + self.trial_per_collect = trial_per_collect + + @staticmethod + def _default_policy_fn(env): + net = Preprocessor(env.observation_space) + actor = Actor(env.action_space, net) + critic = Critic(net) + optim = torch.optim.Adam(set(actor.parameters()).union(critic.parameters()), lr=1e-4) + return PPOPolicy(actor, critic, optim, torch.distributions.Categorical, + discount_factor=1., action_space=env.action_space) + + def run(self, base_model, applied_mutators): + search_space = dry_run_for_search_space(base_model, applied_mutators) + concurrency = query_available_resources() + + env_fn = lambda: ModelEvaluationEnv(base_model, applied_mutators, search_space) + policy = self.policy_fn(env_fn()) + + env = BaseVectorEnv([env_fn for _ in range(concurrency)], MultiThreadEnvWorker) + collector = Collector(policy, env, VectorReplayBuffer(20000, len(env))) + + for cur_collect in range(1, self.max_collect + 1): + _logger.info('Collect [%d] Running...', cur_collect) + result = collector.collect(n_episode=self.trial_per_collect) + _logger.info('Collect [%d] Result: %s', cur_collect, str(result)) + policy.update(0, collector.buffer, batch_size=64, repeat=5) diff --git a/new_impl/cv/third_party/nni_new/retiarii/strategy/tpe_strategy.py b/new_impl/cv/third_party/nni_new/retiarii/strategy/tpe_strategy.py new file mode 100644 index 0000000000000000000000000000000000000000..7f55ad302e6ecfe356a242ae92db04b74ad30e27 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/retiarii/strategy/tpe_strategy.py @@ -0,0 +1,94 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +import time + +from nni.algorithms.hpo.hyperopt_tuner import HyperoptTuner + +from .. import Sampler, submit_models, query_available_resources, is_stopped_exec, budget_exhausted +from .base import BaseStrategy + +_logger = logging.getLogger(__name__) + + +class TPESampler(Sampler): + def __init__(self, optimize_mode='minimize'): + self.tpe_tuner = HyperoptTuner('tpe', optimize_mode) + self.cur_sample = None + self.index = None + self.total_parameters = {} + + def update_sample_space(self, sample_space): + search_space = {} + for i, each in enumerate(sample_space): + search_space[str(i)] = {'_type': 'choice', '_value': each} + self.tpe_tuner.update_search_space(search_space) + + def generate_samples(self, model_id): + self.cur_sample = self.tpe_tuner.generate_parameters(model_id) + self.total_parameters[model_id] = self.cur_sample + self.index = 0 + + def receive_result(self, model_id, result): + self.tpe_tuner.receive_trial_result(model_id, self.total_parameters[model_id], result) + + def choice(self, candidates, mutator, model, index): + chosen = self.cur_sample[str(self.index)] + self.index += 1 + return chosen + + +class TPEStrategy(BaseStrategy): + """ + The Tree-structured Parzen Estimator (TPE) [bergstrahpo]_ is a sequential model-based optimization (SMBO) approach. + SMBO methods sequentially construct models to approximate the performance of hyperparameters based on historical measurements, + and then subsequently choose new hyperparameters to test based on this model. + + References + ---------- + + .. [bergstrahpo] Bergstra et al., "Algorithms for Hyper-Parameter Optimization". + https://papers.nips.cc/paper/4443-algorithms-for-hyper-parameter-optimization.pdf + """ + + def __init__(self): + self.tpe_sampler = TPESampler() + self.model_id = 0 + self.running_models = {} + + def run(self, base_model, applied_mutators): + sample_space = [] + new_model = base_model + for mutator in applied_mutators: + recorded_candidates, new_model = mutator.dry_run(new_model) + sample_space.extend(recorded_candidates) + self.tpe_sampler.update_sample_space(sample_space) + + _logger.info('TPE strategy has been started.') + while not budget_exhausted(): + avail_resource = query_available_resources() + if avail_resource > 0: + model = base_model + _logger.debug('New model created. Applied mutators: %s', str(applied_mutators)) + self.tpe_sampler.generate_samples(self.model_id) + for mutator in applied_mutators: + mutator.bind_sampler(self.tpe_sampler) + model = mutator.apply(model) + # run models + submit_models(model) + self.running_models[self.model_id] = model + self.model_id += 1 + else: + time.sleep(2) + + _logger.debug('num of running models: %d', len(self.running_models)) + to_be_deleted = [] + for _id, _model in self.running_models.items(): + if is_stopped_exec(_model): + if _model.metric is not None: + self.tpe_sampler.receive_result(_id, _model.metric) + _logger.debug('tpe receive results: %d, %s', _id, _model.metric) + to_be_deleted.append(_id) + for _id in to_be_deleted: + del self.running_models[_id] diff --git a/new_impl/cv/third_party/nni_new/retiarii/strategy/utils.py b/new_impl/cv/third_party/nni_new/retiarii/strategy/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..8b687c4c211715fd38d361ba9af4f9cbbf573c07 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/retiarii/strategy/utils.py @@ -0,0 +1,32 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import collections +from typing import Dict, Any, List +from ..graph import Model +from ..mutator import Mutator, Sampler + + +class _FixedSampler(Sampler): + def __init__(self, sample): + self.sample = sample + + def choice(self, candidates, mutator, model, index): + return self.sample[(mutator, index)] + + +def dry_run_for_search_space(model: Model, mutators: List[Mutator]) -> Dict[Any, List[Any]]: + search_space = collections.OrderedDict() + for mutator in mutators: + recorded_candidates, model = mutator.dry_run(model) + for i, candidates in enumerate(recorded_candidates): + search_space[(mutator, i)] = candidates + return search_space + + +def get_targeted_model(base_model: Model, mutators: List[Mutator], sample: dict) -> Model: + sampler = _FixedSampler(sample) + model = base_model + for mutator in mutators: + model = mutator.bind_sampler(sampler).apply(model) + return model diff --git a/new_impl/cv/third_party/nni_new/retiarii/trial_entry.py b/new_impl/cv/third_party/nni_new/retiarii/trial_entry.py new file mode 100644 index 0000000000000000000000000000000000000000..7d805dd47f3b2294c24df6895974553ccc3397a8 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/retiarii/trial_entry.py @@ -0,0 +1,25 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +Entrypoint for trials. + +Assuming execution engine is BaseExecutionEngine. +""" +import argparse + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('exec', choices=['base', 'py', 'cgo']) + args = parser.parse_args() + if args.exec == 'base': + from .execution.base import BaseExecutionEngine + engine = BaseExecutionEngine + elif args.exec == 'cgo': + from .execution.cgo_engine import CGOExecutionEngine + engine = CGOExecutionEngine + elif args.exec == 'py': + from .execution.python import PurePythonExecutionEngine + engine = PurePythonExecutionEngine + engine.trial_execute_graph() diff --git a/new_impl/cv/third_party/nni_new/retiarii/utils.py b/new_impl/cv/third_party/nni_new/retiarii/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..c8b02dfba46020aaa8da4bf451817894be95278e --- /dev/null +++ b/new_impl/cv/third_party/nni_new/retiarii/utils.py @@ -0,0 +1,106 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import inspect +import warnings +from collections import defaultdict +from typing import Any, List, Dict +from pathlib import Path + + +def import_(target: str, allow_none: bool = False) -> Any: + if target is None: + return None + path, identifier = target.rsplit('.', 1) + module = __import__(path, globals(), locals(), [identifier]) + return getattr(module, identifier) + + +def version_larger_equal(a: str, b: str) -> bool: + # TODO: refactor later + a = a.split('+')[0] + b = b.split('+')[0] + return tuple(map(int, a.split('.'))) >= tuple(map(int, b.split('.'))) + + +_last_uid = defaultdict(int) + + +def uid(namespace: str = 'default') -> int: + _last_uid[namespace] += 1 + return _last_uid[namespace] + + +def reset_uid(namespace: str = 'default') -> None: + _last_uid[namespace] = 0 + + +def get_module_name(cls_or_func): + module_name = cls_or_func.__module__ + if module_name == '__main__': + # infer the module name with inspect + for frm in inspect.stack(): + if inspect.getmodule(frm[0]).__name__ == '__main__': + # main module found + main_file_path = Path(inspect.getsourcefile(frm[0])) + if main_file_path.parents[0] != Path('.'): + raise RuntimeError(f'You are using "{main_file_path}" to launch your experiment, ' + f'please launch the experiment under the directory where "{main_file_path.name}" is located.') + module_name = main_file_path.stem + break + if module_name == '__main__': + warnings.warn('Callstack exhausted but main module still not found. This will probably cause issues that the ' + 'function/class cannot be imported.') + + # NOTE: this is hacky. As torchscript retrieves LSTM's source code to do something. + # to make LSTM's source code can be found, we should assign original LSTM's __module__ to + # the wrapped LSTM's __module__ + # TODO: find out all the modules that have the same requirement as LSTM + if f'{cls_or_func.__module__}.{cls_or_func.__name__}' == 'torch.nn.modules.rnn.LSTM': + module_name = cls_or_func.__module__ + + return module_name + + +def get_importable_name(cls, relocate_module=False): + module_name = get_module_name(cls) if relocate_module else cls.__module__ + return module_name + '.' + cls.__name__ + + +class ContextStack: + """ + This is to maintain a globally-accessible context envinronment that is visible to everywhere. + + Use ``with ContextStack(namespace, value):`` to initiate, and use ``get_current_context(namespace)`` to + get the corresponding value in the namespace. + """ + + _stack: Dict[str, List[Any]] = defaultdict(list) + + def __init__(self, key: str, value: Any): + self.key = key + self.value = value + + def __enter__(self): + self.push(self.key, self.value) + return self + + def __exit__(self, *args, **kwargs): + self.pop(self.key) + + @classmethod + def push(cls, key: str, value: Any): + cls._stack[key].append(value) + + @classmethod + def pop(cls, key: str) -> None: + cls._stack[key].pop() + + @classmethod + def top(cls, key: str) -> Any: + assert cls._stack[key], 'Context is empty.' + return cls._stack[key][-1] + + +def get_current_context(key: str) -> Any: + return ContextStack.top(key) diff --git a/new_impl/cv/third_party/nni_new/runtime/common.py b/new_impl/cv/third_party/nni_new/runtime/common.py new file mode 100644 index 0000000000000000000000000000000000000000..537a35b55c697dd722f20747e6d2b6c19f0fbc54 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/runtime/common.py @@ -0,0 +1,19 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +_multi_thread = False +_multi_phase = False + +def enable_multi_thread(): + global _multi_thread + _multi_thread = True + +def multi_thread_enabled(): + return _multi_thread + +def enable_multi_phase(): + global _multi_phase + _multi_phase = True + +def multi_phase_enabled(): + return _multi_phase diff --git a/new_impl/cv/third_party/nni_new/runtime/config.py b/new_impl/cv/third_party/nni_new/runtime/config.py new file mode 100644 index 0000000000000000000000000000000000000000..8f1c8e2e10fb690de30dc9d8584905e6a33255fb --- /dev/null +++ b/new_impl/cv/third_party/nni_new/runtime/config.py @@ -0,0 +1,34 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +from pathlib import Path +import shutil +import sys + +import nni + +def get_config_directory() -> Path: + """ + Get NNI config directory. + Create it if not exist. + """ + if sys.prefix != sys.base_prefix or Path(sys.prefix, 'conda-meta').is_dir(): + config_dir = Path(sys.prefix, 'nni') + elif sys.platform == 'win32': + config_dir = Path(os.environ['APPDATA'], 'nni') + else: + config_dir = Path.home() / '.config/nni' + config_dir.mkdir(parents=True, exist_ok=True) + return config_dir + +def get_config_file(name: str) -> Path: + """ + Get an NNI config file. + Copy from `nni/runtime/default_config` if not exist. + """ + config_file = get_config_directory() / name + if not config_file.exists(): + default = Path(nni.__path__[0], 'runtime/default_config', name) + shutil.copyfile(default, config_file) + return config_file diff --git a/new_impl/cv/third_party/nni_new/runtime/default_config/registered_algorithms.yml b/new_impl/cv/third_party/nni_new/runtime/default_config/registered_algorithms.yml new file mode 100644 index 0000000000000000000000000000000000000000..04c87e8de8ad99f71ee452aeab52d465d73fbf2f --- /dev/null +++ b/new_impl/cv/third_party/nni_new/runtime/default_config/registered_algorithms.yml @@ -0,0 +1,82 @@ +advisors: +- builtinName: Hyperband + classArgsValidator: nni.algorithms.hpo.hyperband_advisor.HyperbandClassArgsValidator + className: nni.algorithms.hpo.hyperband_advisor.Hyperband + source: nni +- builtinName: BOHB + classArgsValidator: nni.algorithms.hpo.bohb_advisor.BOHBClassArgsValidator + className: nni.algorithms.hpo.bohb_advisor.BOHB + source: nni +assessors: +- builtinName: Medianstop + classArgsValidator: nni.algorithms.hpo.medianstop_assessor.MedianstopClassArgsValidator + className: nni.algorithms.hpo.medianstop_assessor.MedianstopAssessor + source: nni +- builtinName: Curvefitting + classArgsValidator: nni.algorithms.hpo.curvefitting_assessor.CurvefittingClassArgsValidator + className: nni.algorithms.hpo.curvefitting_assessor.CurvefittingAssessor + source: nni +tuners: +- builtinName: PPOTuner + classArgsValidator: nni.algorithms.hpo.ppo_tuner.PPOClassArgsValidator + className: nni.algorithms.hpo.ppo_tuner.PPOTuner + source: nni +- builtinName: SMAC + classArgsValidator: nni.algorithms.hpo.smac_tuner.SMACClassArgsValidator + className: nni.algorithms.hpo.smac_tuner.SMACTuner + source: nni +- builtinName: TPE + classArgs: + algorithm_name: tpe + classArgsValidator: nni.algorithms.hpo.hyperopt_tuner.HyperoptClassArgsValidator + className: nni.algorithms.hpo.hyperopt_tuner.HyperoptTuner + source: nni +- acceptClassArgs: false + builtinName: Random + classArgs: + algorithm_name: random_search + classArgsValidator: nni.algorithms.hpo.hyperopt_tuner.HyperoptClassArgsValidator + className: nni.algorithms.hpo.hyperopt_tuner.HyperoptTuner + source: nni +- builtinName: Anneal + classArgs: + algorithm_name: anneal + classArgsValidator: nni.algorithms.hpo.hyperopt_tuner.HyperoptClassArgsValidator + className: nni.algorithms.hpo.hyperopt_tuner.HyperoptTuner + source: nni +- builtinName: Evolution + classArgsValidator: nni.algorithms.hpo.evolution_tuner.EvolutionClassArgsValidator + className: nni.algorithms.hpo.evolution_tuner.EvolutionTuner + source: nni +- acceptClassArgs: false + builtinName: BatchTuner + className: nni.algorithms.hpo.batch_tuner.BatchTuner + source: nni +- acceptClassArgs: false + builtinName: GridSearch + className: nni.algorithms.hpo.gridsearch_tuner.GridSearchTuner + source: nni +- builtinName: NetworkMorphism + classArgsValidator: nni.algorithms.hpo.networkmorphism_tuner.NetworkMorphismClassArgsValidator + className: nni.algorithms.hpo.networkmorphism_tuner.NetworkMorphismTuner + source: nni +- builtinName: MetisTuner + classArgsValidator: nni.algorithms.hpo.metis_tuner.MetisClassArgsValidator + className: nni.algorithms.hpo.metis_tuner.MetisTuner + source: nni +- builtinName: GPTuner + classArgsValidator: nni.algorithms.hpo.gp_tuner.GPClassArgsValidator + className: nni.algorithms.hpo.gp_tuner.GPTuner + source: nni +- builtinName: PBTTuner + classArgsValidator: nni.algorithms.hpo.pbt_tuner.PBTClassArgsValidator + className: nni.algorithms.hpo.pbt_tuner.PBTTuner + source: nni +- builtinName: RegularizedEvolutionTuner + classArgsValidator: nni.algorithms.hpo.regularized_evolution_tuner.EvolutionClassArgsValidator + className: nni.algorithms.hpo.regularized_evolution_tuner.RegularizedEvolutionTuner + source: nni +- builtinName: DNGOTuner + classArgsValidator: nni.algorithms.hpo.dngo_tuner.DNGOClassArgsValidator + className: nni.algorithms.hpo.dngo_tuner.DNGOTuner + source: nni diff --git a/new_impl/cv/third_party/nni_new/runtime/default_config/training_services.json b/new_impl/cv/third_party/nni_new/runtime/default_config/training_services.json new file mode 100644 index 0000000000000000000000000000000000000000..0967ef424bce6791893e9a57bb952f80fd536e93 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/runtime/default_config/training_services.json @@ -0,0 +1 @@ +{} diff --git a/new_impl/cv/third_party/nni_new/runtime/env_vars.py b/new_impl/cv/third_party/nni_new/runtime/env_vars.py new file mode 100644 index 0000000000000000000000000000000000000000..810ab2f4f6db96d4ab195d44643f1deaab5f6528 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/runtime/env_vars.py @@ -0,0 +1,34 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +from collections import namedtuple + + +_trial_env_var_names = [ + 'NNI_PLATFORM', + 'NNI_EXP_ID', + 'NNI_TRIAL_JOB_ID', + 'NNI_SYS_DIR', + 'NNI_OUTPUT_DIR', + 'NNI_TRIAL_SEQ_ID', + 'MULTI_PHASE', + 'REUSE_MODE' +] + +_dispatcher_env_var_names = [ + 'SDK_PROCESS', + 'NNI_MODE', + 'NNI_CHECKPOINT_DIRECTORY', + 'NNI_LOG_DIRECTORY', + 'NNI_LOG_LEVEL', + 'NNI_INCLUDE_INTERMEDIATE_RESULTS' +] + +def _load_env_vars(env_var_names): + env_var_dict = {k: os.environ.get(k) for k in env_var_names} + return namedtuple('EnvVars', env_var_names)(**env_var_dict) + +trial_env_vars = _load_env_vars(_trial_env_var_names) + +dispatcher_env_vars = _load_env_vars(_dispatcher_env_var_names) diff --git a/new_impl/cv/third_party/nni_new/runtime/log.py b/new_impl/cv/third_party/nni_new/runtime/log.py new file mode 100644 index 0000000000000000000000000000000000000000..19aa4e061b00773ab3d84025d44e755c866e94db --- /dev/null +++ b/new_impl/cv/third_party/nni_new/runtime/log.py @@ -0,0 +1,172 @@ +from datetime import datetime +from io import TextIOBase +import logging +from logging import FileHandler, Formatter, Handler, StreamHandler +from pathlib import Path +import sys +import time +from typing import Optional + +import colorama + +from .env_vars import dispatcher_env_vars, trial_env_vars + + +handlers = {} + +log_format = '[%(asctime)s] %(levelname)s (%(name)s/%(threadName)s) %(message)s' +time_format = '%Y-%m-%d %H:%M:%S' +formatter = Formatter(log_format, time_format) + + +def init_logger() -> None: + """ + This function will (and should only) get invoked on the first time of importing nni (no matter which submodule). + It will try to detect the running environment and setup logger accordingly. + + The detection should work in most cases but for `nnictl` and `nni.experiment`. + They will be identified as "standalone" mode and must configure the logger by themselves. + """ + colorama.init() + + if dispatcher_env_vars.SDK_PROCESS == 'dispatcher': + _init_logger_dispatcher() + return + + trial_platform = trial_env_vars.NNI_PLATFORM + + if trial_platform == 'unittest': + return + + if trial_platform and not trial_env_vars.REUSE_MODE: + _init_logger_trial() + return + + _init_logger_standalone() + + logging.getLogger('filelock').setLevel(logging.WARNING) + +_exp_log_initialized = False + +def init_logger_experiment() -> None: + """ + Initialize logger for `nni.experiment.Experiment`. + + This function will get invoked after `init_logger()`. + """ + global _exp_log_initialized + if not _exp_log_initialized: + _exp_log_initialized = True + colorful_formatter = Formatter(log_format, time_format) + colorful_formatter.format = _colorful_format + handlers['_default_'].setFormatter(colorful_formatter) + +def start_experiment_log(experiment_id: str, log_directory: Path, debug: bool) -> None: + log_path = _prepare_log_dir(log_directory) / 'dispatcher.log' + log_level = logging.DEBUG if debug else logging.WARNING + _register_handler(FileHandler(log_path), log_level, experiment_id) + +def stop_experiment_log(experiment_id: str) -> None: + if experiment_id in handlers: + logging.getLogger().removeHandler(handlers.pop(experiment_id)) + + +def _init_logger_dispatcher() -> None: + log_level_map = { + 'fatal': logging.CRITICAL, + 'error': logging.ERROR, + 'warning': logging.WARNING, + 'info': logging.WARNING, + 'debug': logging.DEBUG, + 'trace': 0 + } + + log_path = _prepare_log_dir(dispatcher_env_vars.NNI_LOG_DIRECTORY) / 'dispatcher.log' + log_level = log_level_map.get(dispatcher_env_vars.NNI_LOG_LEVEL, logging.WARNING) + _register_handler(FileHandler(log_path), log_level) + + +def _init_logger_trial() -> None: + log_path = _prepare_log_dir(trial_env_vars.NNI_OUTPUT_DIR) / 'trial.log' + log_file = open(log_path, 'a') + _register_handler(StreamHandler(log_file), logging.WARNING) + + if trial_env_vars.NNI_PLATFORM == 'local': + sys.stdout = _LogFileWrapper(log_file) + + +def _init_logger_standalone() -> None: + _register_handler(StreamHandler(sys.stdout), logging.WARNING) + + +def _prepare_log_dir(path: Optional[str]) -> Path: + if path is None: + return Path() + ret = Path(path) + ret.mkdir(parents=True, exist_ok=True) + return ret + +def _register_handler(handler: Handler, level: int, tag: str = '_default_') -> None: + assert tag not in handlers + handlers[tag] = handler + handler.setFormatter(formatter) + logger = logging.getLogger() + logger.addHandler(handler) + logger.setLevel(level) + +def _colorful_format(record): + time = formatter.formatTime(record, time_format) + if not record.name.startswith('nni.'): + return '[{}] ({}) {}'.format(time, record.name, record.msg % record.args) + if record.levelno >= logging.ERROR: + color = colorama.Fore.RED + elif record.levelno >= logging.WARNING: + color = colorama.Fore.YELLOW + elif record.levelno >= logging.WARNING: + color = colorama.Fore.GREEN + else: + color = colorama.Fore.BLUE + msg = color + (record.msg % record.args) + colorama.Style.RESET_ALL + if record.levelno < logging.WARNING: + return '[{}] {}:{} {}'.format(time, record.threadName, record.name, msg) + else: + return '[{}] {}'.format(time, msg) + +class _LogFileWrapper(TextIOBase): + # wrap the logger file so that anything written to it will automatically get formatted + + def __init__(self, log_file: TextIOBase): + self.file: TextIOBase = log_file + self.line_buffer: Optional[str] = None + self.line_start_time: Optional[datetime] = None + + def write(self, s: str) -> int: + cur_time = datetime.now() + if self.line_buffer and (cur_time - self.line_start_time).total_seconds() > 0.1: + self.flush() + + if self.line_buffer: + self.line_buffer += s + else: + self.line_buffer = s + self.line_start_time = cur_time + + if '\n' not in s: + return len(s) + + time_str = cur_time.strftime(time_format) + lines = self.line_buffer.split('\n') + for line in lines[:-1]: + self.file.write(f'[{time_str}] PRINT {line}\n') + self.file.flush() + + self.line_buffer = lines[-1] + self.line_start_time = cur_time + return len(s) + + def flush(self) -> None: + if self.line_buffer: + time_str = self.line_start_time.strftime(time_format) + self.file.write(f'[{time_str}] PRINT {self.line_buffer}\n') + self.file.flush() + self.line_buffer = None diff --git a/new_impl/cv/third_party/nni_new/runtime/log_backup.py b/new_impl/cv/third_party/nni_new/runtime/log_backup.py new file mode 100644 index 0000000000000000000000000000000000000000..0fc97c666f5c2c5bec08d89f16c28efefab2d833 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/runtime/log_backup.py @@ -0,0 +1,172 @@ +from datetime import datetime +from io import TextIOBase +import logging +from logging import FileHandler, Formatter, Handler, StreamHandler +from pathlib import Path +import sys +import time +from typing import Optional + +import colorama + +from .env_vars import dispatcher_env_vars, trial_env_vars + + +handlers = {} + +log_format = '[%(asctime)s] %(levelname)s (%(name)s/%(threadName)s) %(message)s' +time_format = '%Y-%m-%d %H:%M:%S' +formatter = Formatter(log_format, time_format) + + +def init_logger() -> None: + """ + This function will (and should only) get invoked on the first time of importing nni (no matter which submodule). + It will try to detect the running environment and setup logger accordingly. + + The detection should work in most cases but for `nnictl` and `nni.experiment`. + They will be identified as "standalone" mode and must configure the logger by themselves. + """ + colorama.init() + + if dispatcher_env_vars.SDK_PROCESS == 'dispatcher': + _init_logger_dispatcher() + return + + trial_platform = trial_env_vars.NNI_PLATFORM + + if trial_platform == 'unittest': + return + + if trial_platform and not trial_env_vars.REUSE_MODE: + _init_logger_trial() + return + + _init_logger_standalone() + + logging.getLogger('filelock').setLevel(logging.WARNING) + +_exp_log_initialized = False + +def init_logger_experiment() -> None: + """ + Initialize logger for `nni.experiment.Experiment`. + + This function will get invoked after `init_logger()`. + """ + global _exp_log_initialized + if not _exp_log_initialized: + _exp_log_initialized = True + colorful_formatter = Formatter(log_format, time_format) + colorful_formatter.format = _colorful_format + handlers['_default_'].setFormatter(colorful_formatter) + +def start_experiment_log(experiment_id: str, log_directory: Path, debug: bool) -> None: + log_path = _prepare_log_dir(log_directory) / 'dispatcher.log' + log_level = logging.DEBUG if debug else logging.INFO + _register_handler(FileHandler(log_path), log_level, experiment_id) + +def stop_experiment_log(experiment_id: str) -> None: + if experiment_id in handlers: + logging.getLogger().removeHandler(handlers.pop(experiment_id)) + + +def _init_logger_dispatcher() -> None: + log_level_map = { + 'fatal': logging.CRITICAL, + 'error': logging.ERROR, + 'warning': logging.WARNING, + 'info': logging.INFO, + 'debug': logging.DEBUG, + 'trace': 0 + } + + log_path = _prepare_log_dir(dispatcher_env_vars.NNI_LOG_DIRECTORY) / 'dispatcher.log' + log_level = log_level_map.get(dispatcher_env_vars.NNI_LOG_LEVEL, logging.INFO) + _register_handler(FileHandler(log_path), log_level) + + +def _init_logger_trial() -> None: + log_path = _prepare_log_dir(trial_env_vars.NNI_OUTPUT_DIR) / 'trial.log' + log_file = open(log_path, 'a') + _register_handler(StreamHandler(log_file), logging.INFO) + + if trial_env_vars.NNI_PLATFORM == 'local': + sys.stdout = _LogFileWrapper(log_file) + + +def _init_logger_standalone() -> None: + _register_handler(StreamHandler(sys.stdout), logging.INFO) + + +def _prepare_log_dir(path: Optional[str]) -> Path: + if path is None: + return Path() + ret = Path(path) + ret.mkdir(parents=True, exist_ok=True) + return ret + +def _register_handler(handler: Handler, level: int, tag: str = '_default_') -> None: + assert tag not in handlers + handlers[tag] = handler + handler.setFormatter(formatter) + logger = logging.getLogger() + logger.addHandler(handler) + logger.setLevel(level) + +def _colorful_format(record): + time = formatter.formatTime(record, time_format) + if not record.name.startswith('nni.'): + return '[{}] ({}) {}'.format(time, record.name, record.msg % record.args) + if record.levelno >= logging.ERROR: + color = colorama.Fore.RED + elif record.levelno >= logging.WARNING: + color = colorama.Fore.YELLOW + elif record.levelno >= logging.INFO: + color = colorama.Fore.GREEN + else: + color = colorama.Fore.BLUE + msg = color + (record.msg % record.args) + colorama.Style.RESET_ALL + if record.levelno < logging.INFO: + return '[{}] {}:{} {}'.format(time, record.threadName, record.name, msg) + else: + return '[{}] {}'.format(time, msg) + +class _LogFileWrapper(TextIOBase): + # wrap the logger file so that anything written to it will automatically get formatted + + def __init__(self, log_file: TextIOBase): + self.file: TextIOBase = log_file + self.line_buffer: Optional[str] = None + self.line_start_time: Optional[datetime] = None + + def write(self, s: str) -> int: + cur_time = datetime.now() + if self.line_buffer and (cur_time - self.line_start_time).total_seconds() > 0.1: + self.flush() + + if self.line_buffer: + self.line_buffer += s + else: + self.line_buffer = s + self.line_start_time = cur_time + + if '\n' not in s: + return len(s) + + time_str = cur_time.strftime(time_format) + lines = self.line_buffer.split('\n') + for line in lines[:-1]: + self.file.write(f'[{time_str}] PRINT {line}\n') + self.file.flush() + + self.line_buffer = lines[-1] + self.line_start_time = cur_time + return len(s) + + def flush(self) -> None: + if self.line_buffer: + time_str = self.line_start_time.strftime(time_format) + self.file.write(f'[{time_str}] PRINT {self.line_buffer}\n') + self.file.flush() + self.line_buffer = None diff --git a/new_impl/cv/third_party/nni_new/runtime/msg_dispatcher.py b/new_impl/cv/third_party/nni_new/runtime/msg_dispatcher.py new file mode 100644 index 0000000000000000000000000000000000000000..20b9597f9e0df13c4b1f1ac4ad344cce77a41fc1 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/runtime/msg_dispatcher.py @@ -0,0 +1,242 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +from collections import defaultdict +import json_tricks + +from nni import NoMoreTrialError +from .protocol import CommandType, send +from .msg_dispatcher_base import MsgDispatcherBase +from nni.assessor import AssessResult +from .common import multi_thread_enabled, multi_phase_enabled +from .env_vars import dispatcher_env_vars +from ..utils import MetricType, to_json + +_logger = logging.getLogger(__name__) + +# Assessor global variables +_trial_history = defaultdict(dict) +'''key: trial job ID; value: intermediate results, mapping from sequence number to data''' + +_ended_trials = set() +'''trial_job_id of all ended trials. +We need this because NNI manager may send metrics after reporting a trial ended. +TODO: move this logic to NNI manager +''' + + +def _sort_history(history): + ret = [] + for i, _ in enumerate(history): + if i in history: + ret.append(history[i]) + else: + break + return ret + + +# Tuner global variables +_next_parameter_id = 0 +_trial_params = {} +'''key: parameter ID; value: parameters''' +_customized_parameter_ids = set() + + +def _create_parameter_id(): + global _next_parameter_id + _next_parameter_id += 1 + return _next_parameter_id - 1 + + +def _pack_parameter(parameter_id, params, customized=False, trial_job_id=None, parameter_index=None): + _trial_params[parameter_id] = params + ret = { + 'parameter_id': parameter_id, + 'parameter_source': 'customized' if customized else 'algorithm', + 'parameters': params + } + if trial_job_id is not None: + ret['trial_job_id'] = trial_job_id + if parameter_index is not None: + ret['parameter_index'] = parameter_index + else: + ret['parameter_index'] = 0 + return to_json(ret) + + +class MsgDispatcher(MsgDispatcherBase): + def __init__(self, tuner, assessor=None): + super(MsgDispatcher, self).__init__() + self.tuner = tuner + self.assessor = assessor + if assessor is None: + _logger.debug('Assessor is not configured') + + def load_checkpoint(self): + self.tuner.load_checkpoint() + if self.assessor is not None: + self.assessor.load_checkpoint() + + def save_checkpoint(self): + self.tuner.save_checkpoint() + if self.assessor is not None: + self.assessor.save_checkpoint() + + def handle_initialize(self, data): + """Data is search space + """ + self.tuner.update_search_space(data) + send(CommandType.Initialized, '') + + def send_trial_callback(self, id_, params): + """For tuner to issue trial config when the config is generated + """ + send(CommandType.NewTrialJob, _pack_parameter(id_, params)) + + def handle_request_trial_jobs(self, data): + # data: number or trial jobs + ids = [_create_parameter_id() for _ in range(data)] + _logger.debug("requesting for generating params of %s", ids) + params_list = self.tuner.generate_multiple_parameters(ids, st_callback=self.send_trial_callback) + + for i, _ in enumerate(params_list): + send(CommandType.NewTrialJob, _pack_parameter(ids[i], params_list[i])) + # when parameters is None. + if len(params_list) < len(ids): + send(CommandType.NoMoreTrialJobs, _pack_parameter(ids[0], '')) + + def handle_update_search_space(self, data): + self.tuner.update_search_space(data) + + def handle_import_data(self, data): + """Import additional data for tuning + data: a list of dictionaries, each of which has at least two keys, 'parameter' and 'value' + """ + for entry in data: + entry['value'] = entry['value'] if type(entry['value']) is str else json_tricks.dumps(entry['value']) + entry['value'] = json_tricks.loads(entry['value']) + self.tuner.import_data(data) + + def handle_add_customized_trial(self, data): + # data: parameters + id_ = _create_parameter_id() + _customized_parameter_ids.add(id_) + + def handle_report_metric_data(self, data): + """ + data: a dict received from nni_manager, which contains: + - 'parameter_id': id of the trial + - 'value': metric value reported by nni.report_final_result() + - 'type': report type, support {'FINAL', 'PERIODICAL'} + """ + # metrics value is dumped as json string in trial, so we need to decode it here + if 'value' in data: + data['value'] = json_tricks.loads(data['value']) + if data['type'] == MetricType.FINAL: + self._handle_final_metric_data(data) + elif data['type'] == MetricType.PERIODICAL: + if self.assessor is not None: + self._handle_intermediate_metric_data(data) + elif data['type'] == MetricType.REQUEST_PARAMETER: + assert multi_phase_enabled() + assert data['trial_job_id'] is not None + assert data['parameter_index'] is not None + param_id = _create_parameter_id() + try: + param = self.tuner.generate_parameters(param_id, trial_job_id=data['trial_job_id']) + except NoMoreTrialError: + param = None + send(CommandType.SendTrialJobParameter, _pack_parameter(param_id, param, trial_job_id=data['trial_job_id'], + parameter_index=data['parameter_index'])) + else: + raise ValueError('Data type not supported: {}'.format(data['type'])) + + def handle_trial_end(self, data): + """ + data: it has three keys: trial_job_id, event, hyper_params + - trial_job_id: the id generated by training service + - event: the job's state + - hyper_params: the hyperparameters generated and returned by tuner + """ + trial_job_id = data['trial_job_id'] + _ended_trials.add(trial_job_id) + if trial_job_id in _trial_history: + _trial_history.pop(trial_job_id) + if self.assessor is not None: + self.assessor.trial_end(trial_job_id, data['event'] == 'SUCCEEDED') + if self.tuner is not None: + self.tuner.trial_end(json_tricks.loads(data['hyper_params'])['parameter_id'], data['event'] == 'SUCCEEDED') + + def _handle_final_metric_data(self, data): + """Call tuner to process final results + """ + id_ = data['parameter_id'] + value = data['value'] + if id_ is None or id_ in _customized_parameter_ids: + if not hasattr(self.tuner, '_accept_customized'): + self.tuner._accept_customized = False + if not self.tuner._accept_customized: + _logger.info('Customized trial job %s ignored by tuner', id_) + return + customized = True + else: + customized = False + if id_ in _trial_params: + self.tuner.receive_trial_result(id_, _trial_params[id_], value, customized=customized, + trial_job_id=data.get('trial_job_id')) + else: + _logger.warning('Find unknown job parameter id %s, maybe something goes wrong.', _trial_params[id_]) + + def _handle_intermediate_metric_data(self, data): + """Call assessor to process intermediate results + """ + if data['type'] != MetricType.PERIODICAL: + return + if self.assessor is None: + return + + trial_job_id = data['trial_job_id'] + if trial_job_id in _ended_trials: + return + + history = _trial_history[trial_job_id] + history[data['sequence']] = data['value'] + ordered_history = _sort_history(history) + if len(ordered_history) < data['sequence']: # no user-visible update since last time + return + + try: + result = self.assessor.assess_trial(trial_job_id, ordered_history) + except Exception as e: + _logger.error('Assessor error') + _logger.exception(e) + + if isinstance(result, bool): + result = AssessResult.Good if result else AssessResult.Bad + elif not isinstance(result, AssessResult): + msg = 'Result of Assessor.assess_trial must be an object of AssessResult, not %s' + raise RuntimeError(msg % type(result)) + + if result is AssessResult.Bad: + _logger.debug('BAD, kill %s', trial_job_id) + send(CommandType.KillTrialJob, json_tricks.dumps(trial_job_id)) + # notify tuner + _logger.debug('env var: NNI_INCLUDE_INTERMEDIATE_RESULTS: [%s]', + dispatcher_env_vars.NNI_INCLUDE_INTERMEDIATE_RESULTS) + if dispatcher_env_vars.NNI_INCLUDE_INTERMEDIATE_RESULTS == 'true': + self._earlystop_notify_tuner(data) + else: + _logger.debug('GOOD') + + def _earlystop_notify_tuner(self, data): + """Send last intermediate result as final result to tuner in case the + trial is early stopped. + """ + _logger.debug('Early stop notify tuner data: [%s]', data) + data['type'] = MetricType.FINAL + if multi_thread_enabled(): + self._handle_final_metric_data(data) + else: + data['value'] = to_json(data['value']) + self.enqueue_command(CommandType.ReportMetricData, data) diff --git a/new_impl/cv/third_party/nni_new/runtime/msg_dispatcher_base.py b/new_impl/cv/third_party/nni_new/runtime/msg_dispatcher_base.py new file mode 100644 index 0000000000000000000000000000000000000000..cda40f5c3f6ed4521b70757fb2ae6a83dcc31427 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/runtime/msg_dispatcher_base.py @@ -0,0 +1,245 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import threading +import logging +from multiprocessing.dummy import Pool as ThreadPool +from queue import Queue, Empty +import json_tricks + +from .common import multi_thread_enabled +from .env_vars import dispatcher_env_vars +from ..recoverable import Recoverable +from .protocol import CommandType, receive + + +_logger = logging.getLogger(__name__) + +QUEUE_LEN_WARNING_MARK = 20 +_worker_fast_exit_on_terminate = True + + +class MsgDispatcherBase(Recoverable): + """This is where tuners and assessors are not defined yet. + Inherits this class to make your own advisor. + """ + + def __init__(self): + self.stopping = False + if multi_thread_enabled(): + self.pool = ThreadPool() + self.thread_results = [] + else: + self.default_command_queue = Queue() + self.assessor_command_queue = Queue() + self.default_worker = threading.Thread(target=self.command_queue_worker, args=(self.default_command_queue,)) + self.assessor_worker = threading.Thread(target=self.command_queue_worker, + args=(self.assessor_command_queue,)) + self.default_worker.start() + self.assessor_worker.start() + self.worker_exceptions = [] + + def run(self): + """Run the tuner. + This function will never return unless raise. + """ + _logger.info('Dispatcher started') + if dispatcher_env_vars.NNI_MODE == 'resume': + self.load_checkpoint() + + while not self.stopping: + command, data = receive() + if data: + data = json_tricks.loads(data) + + if command is None or command is CommandType.Terminate: + break + if multi_thread_enabled(): + result = self.pool.map_async(self.process_command_thread, [(command, data)]) + self.thread_results.append(result) + if any([thread_result.ready() and not thread_result.successful() for thread_result in + self.thread_results]): + _logger.debug('Caught thread exception') + break + else: + self.enqueue_command(command, data) + if self.worker_exceptions: + break + + _logger.info('Dispatcher exiting...') + self.stopping = True + if multi_thread_enabled(): + self.pool.close() + self.pool.join() + else: + self.default_worker.join() + self.assessor_worker.join() + + _logger.info('Dispatcher terminiated') + + def command_queue_worker(self, command_queue): + """Process commands in command queues. + """ + while True: + try: + # set timeout to ensure self.stopping is checked periodically + command, data = command_queue.get(timeout=3) + try: + self.process_command(command, data) + except Exception as e: + _logger.exception(e) + self.worker_exceptions.append(e) + break + except Empty: + pass + if self.stopping and (_worker_fast_exit_on_terminate or command_queue.empty()): + break + + def enqueue_command(self, command, data): + """Enqueue command into command queues + """ + if command == CommandType.TrialEnd or ( + command == CommandType.ReportMetricData and data['type'] == 'PERIODICAL'): + self.assessor_command_queue.put((command, data)) + else: + self.default_command_queue.put((command, data)) + + qsize = self.default_command_queue.qsize() + if qsize >= QUEUE_LEN_WARNING_MARK: + _logger.warning('default queue length: %d', qsize) + + qsize = self.assessor_command_queue.qsize() + if qsize >= QUEUE_LEN_WARNING_MARK: + _logger.warning('assessor queue length: %d', qsize) + + def process_command_thread(self, request): + """Worker thread to process a command. + """ + command, data = request + if multi_thread_enabled(): + try: + self.process_command(command, data) + except Exception as e: + _logger.exception(str(e)) + raise + else: + pass + + def process_command(self, command, data): + _logger.debug('process_command: command: [%s], data: [%s]', command, data) + + command_handlers = { + # Tuner commands: + CommandType.Initialize: self.handle_initialize, + CommandType.RequestTrialJobs: self.handle_request_trial_jobs, + CommandType.UpdateSearchSpace: self.handle_update_search_space, + CommandType.ImportData: self.handle_import_data, + CommandType.AddCustomizedTrialJob: self.handle_add_customized_trial, + + # Tuner/Assessor commands: + CommandType.ReportMetricData: self.handle_report_metric_data, + + CommandType.TrialEnd: self.handle_trial_end, + CommandType.Ping: self.handle_ping, + } + if command not in command_handlers: + raise AssertionError('Unsupported command: {}'.format(command)) + command_handlers[command](data) + + def handle_ping(self, data): + pass + + def handle_initialize(self, data): + """Initialize search space and tuner, if any + This method is meant to be called only once for each experiment, after calling this method, + dispatcher should `send(CommandType.Initialized, '')`, to set the status of the experiment to be "INITIALIZED". + Parameters + ---------- + data: dict + search space + """ + raise NotImplementedError('handle_initialize not implemented') + + def handle_request_trial_jobs(self, data): + """The message dispatcher is demanded to generate ``data`` trial jobs. + These trial jobs should be sent via ``send(CommandType.NewTrialJob, json_tricks.dumps(parameter))``, + where ``parameter`` will be received by NNI Manager and eventually accessible to trial jobs as "next parameter". + Semantically, message dispatcher should do this ``send`` exactly ``data`` times. + + The JSON sent by this method should follow the format of + + :: + + { + "parameter_id": 42 + "parameters": { + // this will be received by trial + }, + "parameter_source": "algorithm" // optional + } + + Parameters + ---------- + data: int + number of trial jobs + """ + raise NotImplementedError('handle_request_trial_jobs not implemented') + + def handle_update_search_space(self, data): + """This method will be called when search space is updated. + It's recommended to call this method in `handle_initialize` to initialize search space. + *No need to* notify NNI Manager when this update is done. + Parameters + ---------- + data: dict + search space + """ + raise NotImplementedError('handle_update_search_space not implemented') + + def handle_import_data(self, data): + """Import previous data when experiment is resumed. + Parameters + ---------- + data: list + a list of dictionaries, each of which has at least two keys, 'parameter' and 'value' + """ + raise NotImplementedError('handle_import_data not implemented') + + def handle_add_customized_trial(self, data): + """Experimental API. Not recommended for usage. + """ + raise NotImplementedError('handle_add_customized_trial not implemented') + + def handle_report_metric_data(self, data): + """Called when metric data is reported or new parameters are requested (for multiphase). + When new parameters are requested, this method should send a new parameter. + + Parameters + ---------- + data: dict + a dict which contains 'parameter_id', 'value', 'trial_job_id', 'type', 'sequence'. + type: can be `MetricType.REQUEST_PARAMETER`, `MetricType.FINAL` or `MetricType.PERIODICAL`. + `REQUEST_PARAMETER` is used to request new parameters for multiphase trial job. In this case, + the dict will contain additional keys: `trial_job_id`, `parameter_index`. Refer to `msg_dispatcher.py` + as an example. + + Raises + ------ + ValueError + Data type is not supported + """ + raise NotImplementedError('handle_report_metric_data not implemented') + + def handle_trial_end(self, data): + """Called when the state of one of the trials is changed + + Parameters + ---------- + data: dict + a dict with keys: trial_job_id, event, hyper_params. + trial_job_id: the id generated by training service. + event: the job’s state. + hyper_params: the string that is sent by message dispatcher during the creation of trials. + + """ + raise NotImplementedError('handle_trial_end not implemented') diff --git a/new_impl/cv/third_party/nni_new/runtime/platform/__init__.py b/new_impl/cv/third_party/nni_new/runtime/platform/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9f42a16f4351ac1105bdcba0d1d60647f608b476 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/runtime/platform/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from ..env_vars import trial_env_vars, dispatcher_env_vars + +assert dispatcher_env_vars.SDK_PROCESS != 'dispatcher' + +if trial_env_vars.NNI_PLATFORM is None: + from .standalone import * +elif trial_env_vars.NNI_PLATFORM == 'unittest': + from .test import * +else: + from .local import * diff --git a/new_impl/cv/third_party/nni_new/runtime/platform/local.py b/new_impl/cv/third_party/nni_new/runtime/platform/local.py new file mode 100644 index 0000000000000000000000000000000000000000..b1f26462e21ac59b08082a97f1a1535b89a421c5 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/runtime/platform/local.py @@ -0,0 +1,83 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +import sys +import json +import time +import subprocess + +from ..env_vars import trial_env_vars +from nni.utils import to_json + +_sysdir = trial_env_vars.NNI_SYS_DIR +if not os.path.exists(os.path.join(_sysdir, '.nni')): + os.makedirs(os.path.join(_sysdir, '.nni')) +_metric_file = open(os.path.join(_sysdir, '.nni', 'metrics'), 'ab') + +_outputdir = trial_env_vars.NNI_OUTPUT_DIR +if not os.path.exists(_outputdir): + os.makedirs(_outputdir) + +_reuse_mode = trial_env_vars.REUSE_MODE +_nni_platform = trial_env_vars.NNI_PLATFORM + +_multiphase = trial_env_vars.MULTI_PHASE + +_param_index = 0 + +def request_next_parameter(): + metric = to_json({ + 'trial_job_id': trial_env_vars.NNI_TRIAL_JOB_ID, + 'type': 'REQUEST_PARAMETER', + 'sequence': 0, + 'parameter_index': _param_index + }) + send_metric(metric) + +def get_next_parameter(): + global _param_index + params_file_name = '' + if _multiphase in ('true', 'True'): + params_file_name = ('parameter_{}.cfg'.format(_param_index), 'parameter.cfg')[_param_index == 0] + else: + if _param_index > 0: + return None + elif _param_index == 0: + params_file_name = 'parameter.cfg' + else: + raise AssertionError('_param_index value ({}) should >=0'.format(_param_index)) + + params_filepath = os.path.join(_sysdir, params_file_name) + if not os.path.isfile(params_filepath): + request_next_parameter() + while not (os.path.isfile(params_filepath) and os.path.getsize(params_filepath) > 0): + time.sleep(3) + params_file = open(params_filepath, 'r') + params = json.load(params_file) + _param_index += 1 + return params + +def send_metric(string): + if _nni_platform != 'local' or _reuse_mode in ('true', 'True'): + assert len(string) < 1000000, 'Metric too long' + print("NNISDK_MEb'%s'" % (string), flush=True) + else: + data = (string + '\n').encode('utf8') + assert len(data) < 1000000, 'Metric too long' + _metric_file.write(b'ME%06d%b' % (len(data), data)) + _metric_file.flush() + if sys.platform == "win32": + file = open(_metric_file.name) + file.close() + else: + subprocess.run(['touch', _metric_file.name], check=True) + +def get_experiment_id(): + return trial_env_vars.NNI_EXP_ID + +def get_trial_id(): + return trial_env_vars.NNI_TRIAL_JOB_ID + +def get_sequence_id(): + return int(trial_env_vars.NNI_TRIAL_SEQ_ID) diff --git a/new_impl/cv/third_party/nni_new/runtime/platform/standalone.py b/new_impl/cv/third_party/nni_new/runtime/platform/standalone.py new file mode 100644 index 0000000000000000000000000000000000000000..70caad4e6616aef63e55d559d3f32726ffa42437 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/runtime/platform/standalone.py @@ -0,0 +1,52 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import colorama +import logging +import warnings +import json_tricks + +__all__ = [ + 'get_next_parameter', + 'get_experiment_id', + 'get_trial_id', + 'get_sequence_id', + 'send_metric', +] + +_logger = logging.getLogger('nni') + + +def get_next_parameter(): + warning_message = ''.join([ + colorama.Style.BRIGHT, + colorama.Fore.RED, + 'Running NNI code without runtime. ', + 'Check the following tutorial if you are new to NNI: ', + colorama.Fore.YELLOW, + 'https://nni.readthedocs.io/en/stable/Tutorial/QuickStart.html#id1', + colorama.Style.RESET_ALL + ]) + warnings.warn(warning_message, RuntimeWarning) + return { + 'parameter_id': None, + 'parameters': {} + } + +def get_experiment_id(): + return 'STANDALONE' + +def get_trial_id(): + return 'STANDALONE' + +def get_sequence_id(): + return 0 + +def send_metric(string): + metric = json_tricks.loads(string) + if metric['type'] == 'FINAL': + _logger.info('Final result: %s', metric['value']) + elif metric['type'] == 'PERIODICAL': + _logger.info('Intermediate result: %s (Index %s)', metric['value'], metric['sequence']) + else: + _logger.error('Unexpected metric: %s', string) diff --git a/new_impl/cv/third_party/nni_new/runtime/platform/test.py b/new_impl/cv/third_party/nni_new/runtime/platform/test.py new file mode 100644 index 0000000000000000000000000000000000000000..9bc9651eb0a953420b00c50408248939f2e5ecb6 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/runtime/platform/test.py @@ -0,0 +1,39 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +# pylint: skip-file + +import copy +import json_tricks + + +_params = None +_last_metric = None + + +def get_next_parameter(): + return _params + +def get_experiment_id(): + return 'fakeidex' + +def get_trial_id(): + return 'fakeidtr' + +def get_sequence_id(): + return 0 + +def send_metric(string): + global _last_metric + _last_metric = string + + +def init_params(params): + global _params + _params = copy.deepcopy(params) + +def get_last_metric(): + metrics = json_tricks.loads(_last_metric) + metrics['value'] = json_tricks.loads(metrics['value']) + + return metrics diff --git a/new_impl/cv/third_party/nni_new/runtime/protocol.py b/new_impl/cv/third_party/nni_new/runtime/protocol.py new file mode 100644 index 0000000000000000000000000000000000000000..9c5222f0970a71ac6bc7f6fd45e397ebfac2680b --- /dev/null +++ b/new_impl/cv/third_party/nni_new/runtime/protocol.py @@ -0,0 +1,72 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +import os +import threading +from enum import Enum + +_logger = logging.getLogger(__name__) + + +class CommandType(Enum): + # in + Initialize = b'IN' + RequestTrialJobs = b'GE' + ReportMetricData = b'ME' + UpdateSearchSpace = b'SS' + ImportData = b'FD' + AddCustomizedTrialJob = b'AD' + TrialEnd = b'EN' + Terminate = b'TE' + Ping = b'PI' + + # out + Initialized = b'ID' + NewTrialJob = b'TR' + SendTrialJobParameter = b'SP' + NoMoreTrialJobs = b'NO' + KillTrialJob = b'KI' + +_lock = threading.Lock() +try: + if os.environ.get('NNI_PLATFORM') != 'unittest': + _in_file = open(3, 'rb') + _out_file = open(4, 'wb') +except OSError: + _logger.debug('IPC pipeline not exists') + + +def send(command, data): + """Send command to Training Service. + command: CommandType object. + data: string payload. + """ + global _lock + try: + _lock.acquire() + data = data.encode('utf8') + msg = b'%b%014d%b' % (command.value, len(data), data) + _logger.debug('Sending command, data: [%s]', msg) + _out_file.write(msg) + _out_file.flush() + finally: + _lock.release() + + +def receive(): + """Receive a command from Training Service. + Returns a tuple of command (CommandType) and payload (str) + """ + header = _in_file.read(16) + _logger.debug('Received command, header: [%s]', header) + if header is None or len(header) < 16: + # Pipe EOF encountered + _logger.debug('Pipe EOF encountered') + return None, None + length = int(header[2:]) + data = _in_file.read(length) + command = CommandType(header[:2]) + data = data.decode('utf8') + _logger.debug('Received command, data: [%s]', data) + return command, data diff --git a/new_impl/cv/third_party/nni_new/smartparam.py b/new_impl/cv/third_party/nni_new/smartparam.py new file mode 100644 index 0000000000000000000000000000000000000000..dde0ac2bd64e8b22e2948909b116c33aefda7561 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/smartparam.py @@ -0,0 +1,148 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import numpy as np + +from .runtime.env_vars import trial_env_vars +from . import trial +from . import parameter_expressions as param_exp +from .common.nas_utils import classic_mode, enas_mode, oneshot_mode, darts_mode + + +__all__ = [ + 'choice', + 'randint', + 'uniform', + 'quniform', + 'loguniform', + 'qloguniform', + 'normal', + 'qnormal', + 'lognormal', + 'qlognormal', + 'function_choice', + 'mutable_layer' +] + + +if trial_env_vars.NNI_PLATFORM is None: + def choice(*options, name=None): + return param_exp.choice(options, np.random.RandomState()) + + def randint(lower, upper, name=None): + return param_exp.randint(lower, upper, np.random.RandomState()) + + def uniform(low, high, name=None): + return param_exp.uniform(low, high, np.random.RandomState()) + + def quniform(low, high, q, name=None): + assert high > low, 'Upper bound must be larger than lower bound' + return param_exp.quniform(low, high, q, np.random.RandomState()) + + def loguniform(low, high, name=None): + assert low > 0, 'Lower bound must be positive' + return param_exp.loguniform(low, high, np.random.RandomState()) + + def qloguniform(low, high, q, name=None): + return param_exp.qloguniform(low, high, q, np.random.RandomState()) + + def normal(mu, sigma, name=None): + return param_exp.normal(mu, sigma, np.random.RandomState()) + + def qnormal(mu, sigma, q, name=None): + return param_exp.qnormal(mu, sigma, q, np.random.RandomState()) + + def lognormal(mu, sigma, name=None): + return param_exp.lognormal(mu, sigma, np.random.RandomState()) + + def qlognormal(mu, sigma, q, name=None): + return param_exp.qlognormal(mu, sigma, q, np.random.RandomState()) + + def function_choice(*funcs, name=None): + return param_exp.choice(funcs, np.random.RandomState())() + + def mutable_layer(): + raise RuntimeError('Cannot call nni.mutable_layer in this mode') + +else: + + def choice(options, name=None, key=None): + return options[_get_param(key)] + + def randint(lower, upper, name=None, key=None): + return _get_param(key) + + def uniform(low, high, name=None, key=None): + return _get_param(key) + + def quniform(low, high, q, name=None, key=None): + return _get_param(key) + + def loguniform(low, high, name=None, key=None): + return _get_param(key) + + def qloguniform(low, high, q, name=None, key=None): + return _get_param(key) + + def normal(mu, sigma, name=None, key=None): + return _get_param(key) + + def qnormal(mu, sigma, q, name=None, key=None): + return _get_param(key) + + def lognormal(mu, sigma, name=None, key=None): + return _get_param(key) + + def qlognormal(mu, sigma, q, name=None, key=None): + return _get_param(key) + + def function_choice(funcs, name=None, key=None): + return funcs[_get_param(key)]() + + def mutable_layer( + mutable_id, + mutable_layer_id, + funcs, + funcs_args, + fixed_inputs, + optional_inputs, + optional_input_size, + mode='classic_mode', + tf=None): + '''execute the chosen function and inputs. + Below is an example of chosen function and inputs: + { + "mutable_id": { + "mutable_layer_id": { + "chosen_layer": "pool", + "chosen_inputs": ["out1", "out3"] + } + } + } + Parameters: + --------------- + mutable_id: the name of this mutable_layer block (which could have multiple mutable layers) + mutable_layer_id: the name of a mutable layer in this block + funcs: dict of function calls + funcs_args: + fixed_inputs: + optional_inputs: dict of optional inputs + optional_input_size: number of candidate inputs to be chosen + tf: tensorflow module + ''' + args = (mutable_id, mutable_layer_id, funcs, funcs_args, fixed_inputs, optional_inputs, optional_input_size) + if mode == 'classic_mode': + return classic_mode(*args) + assert tf is not None, 'Internal Error: Tensorflow should not be None in modes other than classic_mode' + if mode == 'enas_mode': + return enas_mode(*args, tf) + if mode == 'oneshot_mode': + return oneshot_mode(*args, tf) + if mode == 'darts_mode': + return darts_mode(*args, tf) + raise RuntimeError('Unrecognized mode: %s' % mode) + + def _get_param(key): + if trial.get_current_parameter() is None: + trial.get_next_parameter() + return trial.get_current_parameter(key) diff --git a/new_impl/cv/third_party/nni_new/tools/annotation/.gitignore b/new_impl/cv/third_party/nni_new/tools/annotation/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..36e264cf443988bf3101b2467e83b259bcd5a4ad --- /dev/null +++ b/new_impl/cv/third_party/nni_new/tools/annotation/.gitignore @@ -0,0 +1 @@ +_generated diff --git a/new_impl/cv/third_party/nni_new/tools/annotation/__init__.py b/new_impl/cv/third_party/nni_new/tools/annotation/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d9ba405c8844591ec083921c90ba2a1d67d52cf8 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/tools/annotation/__init__.py @@ -0,0 +1,141 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +import sys +import shutil +import json + +from . import code_generator +from . import search_space_generator +from . import specific_code_generator + + +__all__ = ['generate_search_space', 'expand_annotations'] + +slash = '/' +if sys.platform == "win32": + slash = '\\' + +def generate_search_space(code_dir): + """Generate search space from Python source code. + Return a serializable search space object. + code_dir: directory path of source files (str) + """ + code_dir = str(code_dir) + search_space = {} + + if code_dir.endswith(slash): + code_dir = code_dir[:-1] + + for subdir, _, files in os.walk(code_dir): + # generate module name from path + if subdir == code_dir: + package = '' + else: + assert subdir.startswith(code_dir + slash), subdir + prefix_len = len(code_dir) + 1 + package = subdir[prefix_len:].replace(slash, '.') + '.' + + for file_name in files: + if file_name.endswith('.py'): + path = os.path.join(subdir, file_name) + module = package + file_name[:-3] + search_space.update(_generate_file_search_space(path, module)) + + return search_space + +def _generate_file_search_space(path, module): + with open(path) as src: + try: + search_space, code = search_space_generator.generate(module, src.read()) + except Exception as exc: # pylint: disable=broad-except + if exc.args: + raise RuntimeError(path + ' ' + '\n'.join(exc.args)) + else: + raise RuntimeError('Failed to generate search space for %s: %r' % (path, exc)) + with open(path, 'w') as dst: + dst.write(code) + return search_space + + +def expand_annotations(src_dir, dst_dir, exp_id='', trial_id='', nas_mode=None): + """Expand annotations in user code. + Return dst_dir if annotation detected; return src_dir if not. + src_dir: directory path of user code (str) + dst_dir: directory to place generated files (str) + nas_mode: the mode of NAS given that NAS interface is used + """ + src_dir, dst_dir = str(src_dir), str(dst_dir) + + if src_dir[-1] == slash: + src_dir = src_dir[:-1] + + if dst_dir[-1] == slash: + dst_dir = dst_dir[:-1] + + annotated = False + + for src_subdir, dirs, files in os.walk(src_dir): + assert src_subdir.startswith(src_dir) + dst_subdir = src_subdir.replace(src_dir, dst_dir, 1) + os.makedirs(dst_subdir, exist_ok=True) + + # generate module name from path + if src_subdir == src_dir: + package = '' + else: + assert src_subdir.startswith(src_dir + slash), src_subdir + prefix_len = len(src_dir) + 1 + package = src_subdir[prefix_len:].replace(slash, '.') + '.' + + for file_name in files: + src_path = os.path.join(src_subdir, file_name) + dst_path = os.path.join(dst_subdir, file_name) + if file_name.endswith('.py'): + if trial_id == '': + annotated |= _expand_file_annotations(src_path, dst_path, nas_mode) + else: + module = package + file_name[:-3] + annotated |= _generate_specific_file(src_path, dst_path, exp_id, trial_id, module) + else: + shutil.copyfile(src_path, dst_path) + + for dir_name in dirs: + os.makedirs(os.path.join(dst_subdir, dir_name), exist_ok=True) + + return dst_dir if annotated else src_dir + +def _expand_file_annotations(src_path, dst_path, nas_mode): + with open(src_path) as src, open(dst_path, 'w') as dst: + try: + annotated_code = code_generator.parse(src.read(), nas_mode) + if annotated_code is None: + shutil.copyfile(src_path, dst_path) + return False + dst.write(annotated_code) + return True + + except Exception as exc: # pylint: disable=broad-except + if exc.args: + raise RuntimeError(src_path + ' ' + '\n'.join(str(arg) for arg in exc.args)) + else: + raise RuntimeError('Failed to expand annotations for %s: %r' % (src_path, exc)) + +def _generate_specific_file(src_path, dst_path, exp_id, trial_id, module): + with open(src_path) as src, open(dst_path, 'w') as dst: + try: + with open(os.path.expanduser('~/nni-experiments/%s/trials/%s/parameter.cfg'%(exp_id, trial_id))) as fd: + para_cfg = json.load(fd) + annotated_code = specific_code_generator.parse(src.read(), para_cfg["parameters"], module) + if annotated_code is None: + shutil.copyfile(src_path, dst_path) + return False + dst.write(annotated_code) + return True + + except Exception as exc: # pylint: disable=broad-except + if exc.args: + raise RuntimeError(src_path + ' ' + '\n'.join(str(arg) for arg in exc.args)) + else: + raise RuntimeError('Failed to expand annotations for %s: %r' % (src_path, exc)) diff --git a/new_impl/cv/third_party/nni_new/tools/annotation/code_generator.py b/new_impl/cv/third_party/nni_new/tools/annotation/code_generator.py new file mode 100644 index 0000000000000000000000000000000000000000..c924e4195072eeff3e112b9ca5e01cb1df41c178 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/tools/annotation/code_generator.py @@ -0,0 +1,369 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import ast +import astor + +from .utils import ast_Num, ast_Str, lineno + +# pylint: disable=unidiomatic-typecheck + +def parse_annotation_mutable_layers(code, lineno, nas_mode): + """Parse the string of mutable layers in annotation. + Return a list of AST Expr nodes + code: annotation string (excluding '@') + nas_mode: the mode of NAS + """ + module = ast.parse(code) + assert type(module) is ast.Module, 'internal error #1' + assert len(module.body) == 1, 'Annotation mutable_layers contains more than one expression' + assert type(module.body[0]) is ast.Expr, 'Annotation is not expression' + call = module.body[0].value + nodes = [] + mutable_id = 'mutable_block_' + str(lineno) + mutable_layer_cnt = 0 + for arg in call.args: + fields = {'layer_choice': False, + 'fixed_inputs': False, + 'optional_inputs': False, + 'optional_input_size': False, + 'layer_output': False} + for k, value in zip(arg.keys, arg.values): + if k.id == 'layer_choice': + assert not fields['layer_choice'], 'Duplicated field: layer_choice' + assert type(value) is ast.List, 'Value of layer_choice should be a list' + call_funcs_keys = [] + call_funcs_values = [] + call_kwargs_values = [] + for call in value.elts: + assert type(call) is ast.Call, 'Element in layer_choice should be function call' + call_name = astor.to_source(call).strip() + call_funcs_keys.append(ast_Str(s=call_name)) + call_funcs_values.append(call.func) + assert not call.args, 'Number of args without keyword should be zero' + kw_args = [] + kw_values = [] + for kw in call.keywords: + kw_args.append(ast_Str(s=kw.arg)) + kw_values.append(kw.value) + call_kwargs_values.append(ast.Dict(keys=kw_args, values=kw_values)) + call_funcs = ast.Dict(keys=call_funcs_keys, values=call_funcs_values) + call_kwargs = ast.Dict(keys=call_funcs_keys, values=call_kwargs_values) + fields['layer_choice'] = True + elif k.id == 'fixed_inputs': + assert not fields['fixed_inputs'], 'Duplicated field: fixed_inputs' + assert type(value) is ast.List, 'Value of fixed_inputs should be a list' + fixed_inputs = value + fields['fixed_inputs'] = True + elif k.id == 'optional_inputs': + assert not fields['optional_inputs'], 'Duplicated field: optional_inputs' + assert type(value) is ast.List, 'Value of optional_inputs should be a list' + var_names = [ast_Str(s=astor.to_source(var).strip()) for var in value.elts] + optional_inputs = ast.Dict(keys=var_names, values=value.elts) + fields['optional_inputs'] = True + elif k.id == 'optional_input_size': + assert not fields['optional_input_size'], 'Duplicated field: optional_input_size' + assert type(value) is ast_Num or type(value) is ast.List, \ + 'Value of optional_input_size should be a number or list' + optional_input_size = value + fields['optional_input_size'] = True + elif k.id == 'layer_output': + assert not fields['layer_output'], 'Duplicated field: layer_output' + assert type(value) is ast.Name, 'Value of layer_output should be ast.Name type' + layer_output = value + fields['layer_output'] = True + else: + raise AssertionError('Unexpected field in mutable layer') + # make call for this mutable layer + assert fields['layer_choice'], 'layer_choice must exist' + assert fields['layer_output'], 'layer_output must exist' + mutable_layer_id = 'mutable_layer_' + str(mutable_layer_cnt) + mutable_layer_cnt += 1 + target_call_attr = ast.Attribute(value=ast.Name(id='nni', ctx=ast.Load()), attr='mutable_layer', ctx=ast.Load()) + target_call_args = [ast_Str(s=mutable_id), + ast_Str(s=mutable_layer_id), + call_funcs, + call_kwargs] + if fields['fixed_inputs']: + target_call_args.append(fixed_inputs) + else: + target_call_args.append(ast.List(elts=[])) + if fields['optional_inputs']: + target_call_args.append(optional_inputs) + assert fields['optional_input_size'], 'optional_input_size must exist when optional_inputs exists' + target_call_args.append(optional_input_size) + else: + target_call_args.append(ast.Dict(keys=[], values=[])) + target_call_args.append(ast_Num(n=0)) + target_call_args.append(ast_Str(s=nas_mode)) + if nas_mode in ['enas_mode', 'oneshot_mode', 'darts_mode']: + target_call_args.append(ast.Name(id='tensorflow')) + target_call = ast.Call(func=target_call_attr, args=target_call_args, keywords=[]) + node = ast.Assign(targets=[layer_output], value=target_call) + nodes.append(node) + return nodes + + +def parse_annotation(code): + """Parse an annotation string. + Return an AST Expr node. + code: annotation string (excluding '@') + """ + module = ast.parse(code) + assert type(module) is ast.Module, 'internal error #1' + assert len(module.body) == 1, 'Annotation contains more than one expression' + assert type(module.body[0]) is ast.Expr, 'Annotation is not expression' + return module.body[0] + + +def parse_annotation_function(code, func_name): + """Parse an annotation function. + Return the value of `name` keyword argument and the AST Call node. + func_name: expected function name + """ + expr = parse_annotation(code) + call = expr.value + assert type(call) is ast.Call, 'Annotation is not a function call' + + assert type(call.func) is ast.Attribute, 'Unexpected annotation function' + assert type(call.func.value) is ast.Name, 'Invalid annotation function name' + assert call.func.value.id == 'nni', 'Annotation is not a NNI function' + assert call.func.attr == func_name, 'internal error #2' + + assert len(call.keywords) == 1, 'Annotation function contains more than one keyword argument' + assert call.keywords[0].arg == 'name', 'Annotation keyword argument is not "name"' + name = call.keywords[0].value + + return name, call + + +def parse_nni_variable(code): + """Parse `nni.variable` expression. + Return the name argument and AST node of annotated expression. + code: annotation string + """ + name, call = parse_annotation_function(code, 'variable') + + assert len(call.args) == 1, 'nni.variable contains more than one arguments' + arg = call.args[0] + assert type(arg) is ast.Call, 'Value of nni.variable is not a function call' + assert type(arg.func) is ast.Attribute, 'nni.variable value is not a NNI function' + assert type(arg.func.value) is ast.Name, 'nni.variable value is not a NNI function' + assert arg.func.value.id == 'nni', 'nni.variable value is not a NNI function' + + name_str = astor.to_source(name).strip() + keyword_arg = ast.keyword(arg='name', value=ast_Str(s=name_str)) + arg.keywords.append(keyword_arg) + if arg.func.attr == 'choice': + convert_args_to_dict(arg) + + return name, arg + + +def parse_nni_function(code): + """Parse `nni.function_choice` expression. + Return the AST node of annotated expression and a list of dumped function call expressions. + code: annotation string + """ + name, call = parse_annotation_function(code, 'function_choice') + funcs = [ast.dump(func, False) for func in call.args] + convert_args_to_dict(call, with_lambda=True) + + name_str = astor.to_source(name).strip() + call.keywords[0].value = ast_Str(s=name_str) + + return call, funcs + + +def convert_args_to_dict(call, with_lambda=False): + """Convert all args to a dict such that every key and value in the dict is the same as the value of the arg. + Return the AST Call node with only one arg that is the dictionary + """ + keys, values = list(), list() + for arg in call.args: + if type(arg) in [ast_Str, ast_Num]: + arg_value = arg + else: + # if arg is not a string or a number, we use its source code as the key + arg_value = astor.to_source(arg).strip('\n"') + arg_value = ast_Str(str(arg_value)) + arg = make_lambda(arg) if with_lambda else arg + keys.append(arg_value) + values.append(arg) + del call.args[:] + call.args.append(ast.Dict(keys=keys, values=values)) + + return call + + +def make_lambda(call): + """Wrap an AST Call node to lambda expression node. + call: ast.Call node + """ + empty_args = ast.arguments(args=[], vararg=None, kwarg=None, defaults=[]) + return ast.Lambda(args=empty_args, body=call) + + +def test_variable_equal(node1, node2): + """Test whether two variables are the same.""" + if type(node1) is not type(node2): + return False + if isinstance(node1, ast.AST): + for k, v in vars(node1).items(): + if k in ('lineno', 'col_offset', 'ctx', 'end_lineno', 'end_col_offset'): + continue + if not test_variable_equal(v, getattr(node2, k)): + return False + return True + if isinstance(node1, list): + if len(node1) != len(node2): + return False + return all(test_variable_equal(n1, n2) for n1, n2 in zip(node1, node2)) + + return node1 == node2 + + +def replace_variable_node(node, annotation): + """Replace a node annotated by `nni.variable`. + node: the AST node to replace + annotation: annotation string + """ + assert type(node) is ast.Assign, 'nni.variable is not annotating assignment expression' + assert len(node.targets) == 1, 'Annotated assignment has more than one left-hand value' + name, expr = parse_nni_variable(annotation) + assert test_variable_equal(node.targets[0], name), 'Annotated variable has wrong name' + node.value = expr + return node + + +def replace_function_node(node, annotation): + """Replace a node annotated by `nni.function_choice`. + node: the AST node to replace + annotation: annotation string + """ + target, funcs = parse_nni_function(annotation) + FuncReplacer(funcs, target).visit(node) + return node + + +class FuncReplacer(ast.NodeTransformer): + """To replace target function call expressions in a node annotated by `nni.function_choice`""" + + def __init__(self, funcs, target): + """Constructor. + funcs: list of dumped function call expressions to replace + target: use this AST node to replace matching expressions + """ + self.funcs = set(funcs) + self.target = target + + def visit_Call(self, node): # pylint: disable=invalid-name + if ast.dump(node, False) in self.funcs: + return self.target + return node + + +class Transformer(ast.NodeTransformer): + """Transform original code to annotated code""" + + def __init__(self, nas_mode=None): + self.stack = [] + self.last_line = 0 + self.annotated = False + self.nas_mode = nas_mode + + def visit(self, node): + if isinstance(node, (ast.expr, ast.stmt)): + self.last_line = lineno(node) + + # do nothing for root + if not self.stack: + return self._visit_children(node) + + annotation = self.stack[-1] + + # this is a standalone string, may be an annotation + if type(node) is ast.Expr and type(node.value) is ast_Str: + # must not annotate an annotation string + assert annotation is None, 'Annotating an annotation' + return self._visit_string(node) + + if annotation is not None: # this expression is annotated + self.stack[-1] = None # so next expression is not + if annotation.startswith('nni.variable'): + return replace_variable_node(node, annotation) + if annotation.startswith('nni.function_choice'): + return replace_function_node(node, annotation) + + return self._visit_children(node) + + def _visit_string(self, node): + string = node.value.s + if string.startswith('@nni.'): + self.annotated = True + else: + return node # not an annotation, ignore it + + if string.startswith('@nni.training_update'): + expr = parse_annotation(string[1:]) + call_node = expr.value + call_node.args.insert(0, ast_Str(s=self.nas_mode)) + return expr + + if string.startswith('@nni.report_intermediate_result') \ + or string.startswith('@nni.report_final_result') \ + or string.startswith('@nni.get_next_parameter'): + return parse_annotation(string[1:]) # expand annotation string to code + + if string.startswith('@nni.mutable_layers'): + nodes = parse_annotation_mutable_layers(string[1:], lineno(node), self.nas_mode) + return nodes + + if string.startswith('@nni.variable') \ + or string.startswith('@nni.function_choice'): + self.stack[-1] = string[1:] # mark that the next expression is annotated + return None + + raise AssertionError('Unexpected annotation function') + + def _visit_children(self, node): + self.stack.append(None) + self.generic_visit(node) + annotation = self.stack.pop() + assert annotation is None, 'Annotation has no target' + return node + + +def parse(code, nas_mode=None): + """Annotate user code. + Return annotated code (str) if annotation detected; return None if not. + code: original user code (str), + nas_mode: the mode of NAS given that NAS interface is used + """ + try: + ast_tree = ast.parse(code) + except Exception: + raise RuntimeError('Bad Python code') + + transformer = Transformer(nas_mode) + try: + transformer.visit(ast_tree) + except AssertionError as exc: + raise RuntimeError('%d: %s' % (ast_tree.last_line, exc.args[0])) + + if not transformer.annotated: + return None + + last_future_import = -1 + import_nni = ast.Import(names=[ast.alias(name='nni', asname=None)]) + nodes = ast_tree.body + for i, _ in enumerate(nodes): + if type(nodes[i]) is ast.ImportFrom and nodes[i].module == '__future__': + last_future_import = i + nodes.insert(last_future_import + 1, import_nni) + # enas, oneshot and darts modes for tensorflow need tensorflow module, so we import it here + if nas_mode in ['enas_mode', 'oneshot_mode', 'darts_mode']: + import_tf = ast.Import(names=[ast.alias(name='tensorflow', asname=None)]) + nodes.insert(last_future_import + 1, import_tf) + + return astor.to_source(ast_tree) diff --git a/new_impl/cv/third_party/nni_new/tools/annotation/search_space_generator.py b/new_impl/cv/third_party/nni_new/tools/annotation/search_space_generator.py new file mode 100644 index 0000000000000000000000000000000000000000..a0a19f53dff601faa682f48b3f1ee574f21b9acf --- /dev/null +++ b/new_impl/cv/third_party/nni_new/tools/annotation/search_space_generator.py @@ -0,0 +1,135 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import ast +import numbers + +import astor + +from .utils import ast_Num, ast_Str, lineno + +# pylint: disable=unidiomatic-typecheck + + +# list of functions related to search space generating +_ss_funcs = [ + 'choice', + 'randint', + 'uniform', + 'quniform', + 'loguniform', + 'qloguniform', + 'normal', + 'qnormal', + 'lognormal', + 'qlognormal', + 'function_choice', + 'mutable_layer' +] + + +class SearchSpaceGenerator(ast.NodeTransformer): + """Generate search space from smart parater APIs""" + + def __init__(self, module_name): + self.module_name = module_name + self.search_space = {} + self.last_line = 0 # last parsed line, useful for error reporting + + def generate_mutable_layer_search_space(self, args): + mutable_block = args[0].s + mutable_layer = args[1].s + key = self.module_name + '/' + mutable_block + args[0].s = key + if key not in self.search_space: + self.search_space[key] = {'_type': 'mutable_layer', '_value': {}} + self.search_space[key]['_value'][mutable_layer] = { + 'layer_choice': [k.s for k in args[2].keys], + 'optional_inputs': [k.s for k in args[5].keys], + 'optional_input_size': args[6].n if isinstance(args[6], ast_Num) else [args[6].elts[0].n, args[6].elts[1].n] + } + + def visit_Call(self, node): # pylint: disable=invalid-name + self.generic_visit(node) + + # ignore if the function is not 'nni.*' + if type(node.func) is not ast.Attribute: + return node + if type(node.func.value) is not ast.Name: + return node + if node.func.value.id != 'nni': + return node + + # ignore if its not a search space function (e.g. `report_final_result`) + func = node.func.attr + if func not in _ss_funcs: + return node + + self.last_line = lineno(node) + + if func == 'mutable_layer': + self.generate_mutable_layer_search_space(node.args) + return node + + if node.keywords: + # there is a `name` argument + assert len(node.keywords) == 1, 'Smart parameter has keyword argument other than "name"' + assert node.keywords[0].arg == 'name', 'Smart paramater\'s keyword argument is not "name"' + assert type(node.keywords[0].value) is ast_Str, 'Smart parameter\'s name must be string literal' + name = node.keywords[0].value.s + specified_name = True + else: + # generate the missing name automatically + name = '__line' + str(str(node.args[-1].lineno)) + specified_name = False + node.keywords = list() + + if func in ('choice', 'function_choice'): + # we will use keys in the dict as the choices, which is generated by code_generator according to the args given by user + assert len(node.args) == 1, 'Smart parameter has arguments other than dict' + # check if it is a number or a string and get its value accordingly + args = [key.n if type(key) is ast_Num else key.s for key in node.args[0].keys] + else: + # arguments of other functions must be literal number + assert all(isinstance(ast.literal_eval(astor.to_source(arg)), numbers.Real) for arg in node.args), \ + 'Smart parameter\'s arguments must be number literals' + args = [ast.literal_eval(astor.to_source(arg)) for arg in node.args] + + key = self.module_name + '/' + name + '/' + func + # store key in ast.Call + node.keywords.append(ast.keyword(arg='key', value=ast_Str(s=key))) + + if func == 'function_choice': + func = 'choice' + value = {'_type': func, '_value': args} + + if specified_name: + # multiple functions with same name must have identical arguments + old = self.search_space.get(key) + assert old is None or old == value, 'Different smart parameters have same name' + else: + # generated name must not duplicate + assert key not in self.search_space, 'Only one smart parameter is allowed in a line' + + self.search_space[key] = value + + return node + + +def generate(module_name, code): + """Generate search space. + Return a serializable search space object. + module_name: name of the module (str) + code: user code (str) + """ + try: + ast_tree = ast.parse(code) + except Exception: + raise RuntimeError('Bad Python code') + + visitor = SearchSpaceGenerator(module_name) + try: + visitor.visit(ast_tree) + except AssertionError as exc: + raise RuntimeError('%d: %s' % (visitor.last_line, exc.args[0])) + return visitor.search_space, astor.to_source(ast_tree) diff --git a/new_impl/cv/third_party/nni_new/tools/annotation/specific_code_generator.py b/new_impl/cv/third_party/nni_new/tools/annotation/specific_code_generator.py new file mode 100644 index 0000000000000000000000000000000000000000..5ddd362205dfa909d0077315ebe5dd9b8ade239f --- /dev/null +++ b/new_impl/cv/third_party/nni_new/tools/annotation/specific_code_generator.py @@ -0,0 +1,354 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import ast +import astor +from nni.tools.nnictl.common_utils import print_warning + +from .utils import ast_Num, ast_Str, lineno + +# pylint: disable=unidiomatic-typecheck + +para_cfg = None +prefix_name = None + + +def parse_annotation_mutable_layers(code, lineno): + """Parse the string of mutable layers in annotation. + Return a list of AST Expr nodes + code: annotation string (excluding '@') + """ + module = ast.parse(code) + assert type(module) is ast.Module, 'internal error #1' + assert len(module.body) == 1, 'Annotation mutable_layers contains more than one expression' + assert type(module.body[0]) is ast.Expr, 'Annotation is not expression' + call = module.body[0].value + nodes = [] + mutable_id = prefix_name + '/mutable_block_' + str(lineno) + mutable_layer_cnt = 0 + for arg in call.args: + fields = {'layer_choice': False, + 'fixed_inputs': False, + 'optional_inputs': False, + 'optional_input_size': False, + 'layer_output': False} + mutable_layer_id = 'mutable_layer_' + str(mutable_layer_cnt) + mutable_layer_cnt += 1 + func_call = None + for k, value in zip(arg.keys, arg.values): + if k.id == 'layer_choice': + assert not fields['layer_choice'], 'Duplicated field: layer_choice' + assert type(value) is ast.List, 'Value of layer_choice should be a list' + for call in value.elts: + assert type(call) is ast.Call, 'Element in layer_choice should be function call' + call_name = astor.to_source(call).strip() + if call_name == para_cfg[mutable_id][mutable_layer_id]['chosen_layer']: + func_call = call + assert not call.args, 'Number of args without keyword should be zero' + break + fields['layer_choice'] = True + elif k.id == 'fixed_inputs': + assert not fields['fixed_inputs'], 'Duplicated field: fixed_inputs' + assert type(value) is ast.List, 'Value of fixed_inputs should be a list' + fixed_inputs = value + fields['fixed_inputs'] = True + elif k.id == 'optional_inputs': + assert not fields['optional_inputs'], 'Duplicated field: optional_inputs' + assert type(value) is ast.List, 'Value of optional_inputs should be a list' + var_names = [astor.to_source(var).strip() for var in value.elts] + chosen_inputs = para_cfg[mutable_id][mutable_layer_id]['chosen_inputs'] + elts = [] + for i in chosen_inputs: + index = var_names.index(i) + elts.append(value.elts[index]) + optional_inputs = ast.List(elts=elts) + fields['optional_inputs'] = True + elif k.id == 'optional_input_size': + pass + elif k.id == 'layer_output': + assert not fields['layer_output'], 'Duplicated field: layer_output' + assert type(value) is ast.Name, 'Value of layer_output should be ast.Name type' + layer_output = value + fields['layer_output'] = True + else: + raise AssertionError('Unexpected field in mutable layer') + # make call for this mutable layer + assert fields['layer_choice'], 'layer_choice must exist' + assert fields['layer_output'], 'layer_output must exist' + + if not fields['fixed_inputs']: + fixed_inputs = ast.List(elts=[]) + if not fields['optional_inputs']: + optional_inputs = ast.List(elts=[]) + inputs = ast.List(elts=[fixed_inputs, optional_inputs]) + + func_call.args.append(inputs) + node = ast.Assign(targets=[layer_output], value=func_call) + nodes.append(node) + return nodes + + +def parse_annotation(code): + """Parse an annotation string. + Return an AST Expr node. + code: annotation string (excluding '@') + """ + module = ast.parse(code) + assert type(module) is ast.Module, 'internal error #1' + assert len(module.body) == 1, 'Annotation contains more than one expression' + assert type(module.body[0]) is ast.Expr, 'Annotation is not expression' + return module.body[0] + + +def parse_annotation_function(code, func_name): + """Parse an annotation function. + Return the value of `name` keyword argument and the AST Call node. + func_name: expected function name + """ + expr = parse_annotation(code) + call = expr.value + assert type(call) is ast.Call, 'Annotation is not a function call' + + assert type(call.func) is ast.Attribute, 'Unexpected annotation function' + assert type(call.func.value) is ast.Name, 'Invalid annotation function name' + assert call.func.value.id == 'nni', 'Annotation is not a NNI function' + assert call.func.attr == func_name, 'internal error #2' + + assert len(call.keywords) == 1, 'Annotation function contains more than one keyword argument' + assert call.keywords[0].arg == 'name', 'Annotation keyword argument is not "name"' + name = call.keywords[0].value + + return name, call + + +def parse_nni_variable(code): + """Parse `nni.variable` expression. + Return the name argument and AST node of annotated expression. + code: annotation string + """ + name, call = parse_annotation_function(code, 'variable') + + assert len(call.args) == 1, 'nni.variable contains more than one arguments' + arg = call.args[0] + assert type(arg) is ast.Call, 'Value of nni.variable is not a function call' + assert type(arg.func) is ast.Attribute, 'nni.variable value is not a NNI function' + assert type(arg.func.value) is ast.Name, 'nni.variable value is not a NNI function' + assert arg.func.value.id == 'nni', 'nni.variable value is not a NNI function' + + name_str = astor.to_source(name).strip() + keyword_arg = ast.keyword(arg='name', value=ast_Str(s=name_str)) + arg.keywords.append(keyword_arg) + if arg.func.attr == 'choice': + convert_args_to_dict(arg) + + return name, arg + + +def parse_nni_function(code): + """Parse `nni.function_choice` expression. + Return the AST node of annotated expression and a list of dumped function call expressions. + code: annotation string + """ + name, call = parse_annotation_function(code, 'function_choice') + funcs = [ast.dump(func, False) for func in call.args] + convert_args_to_dict(call, with_lambda=True) + + name_str = astor.to_source(name).strip() + call.keywords[0].value = ast_Str(s=name_str) + + return call, funcs + + +def convert_args_to_dict(call, with_lambda=False): + """Convert all args to a dict such that every key and value in the dict is the same as the value of the arg. + Return the AST Call node with only one arg that is the dictionary + """ + keys, values = list(), list() + for arg in call.args: + if type(arg) in [ast_Str, ast_Num]: + arg_value = arg + else: + # if arg is not a string or a number, we use its source code as the key + arg_value = astor.to_source(arg).strip('\n"') + arg_value = ast_Str(str(arg_value)) + arg = make_lambda(arg) if with_lambda else arg + keys.append(arg_value) + values.append(arg) + del call.args[:] + call.args.append(ast.Dict(keys=keys, values=values)) + + return call + + +def make_lambda(call): + """Wrap an AST Call node to lambda expression node. + call: ast.Call node + """ + empty_args = ast.arguments(args=[], vararg=None, kwarg=None, defaults=[]) + return ast.Lambda(args=empty_args, body=call) + + +def test_variable_equal(node1, node2): + """Test whether two variables are the same.""" + if type(node1) is not type(node2): + return False + if isinstance(node1, ast.AST): + for k, v in vars(node1).items(): + if k in ('lineno', 'col_offset', 'ctx', 'end_lineno', 'end_col_offset'): + continue + if not test_variable_equal(v, getattr(node2, k)): + return False + return True + if isinstance(node1, list): + if len(node1) != len(node2): + return False + return all(test_variable_equal(n1, n2) for n1, n2 in zip(node1, node2)) + + return node1 == node2 + + +def replace_variable_node(node, annotation): + """Replace a node annotated by `nni.variable`. + node: the AST node to replace + annotation: annotation string + """ + assert type(node) is ast.Assign, 'nni.variable is not annotating assignment expression' + assert len(node.targets) == 1, 'Annotated assignment has more than one left-hand value' + name, expr = parse_nni_variable(annotation) + assert test_variable_equal(node.targets[0], name), 'Annotated variable has wrong name' + node.value = expr + return node + + +def replace_function_node(node, annotation): + """Replace a node annotated by `nni.function_choice`. + node: the AST node to replace + annotation: annotation string + """ + target, funcs = parse_nni_function(annotation) + FuncReplacer(funcs, target).visit(node) + return node + + +class FuncReplacer(ast.NodeTransformer): + """To replace target function call expressions in a node annotated by `nni.function_choice`""" + + def __init__(self, funcs, target): + """Constructor. + funcs: list of dumped function call expressions to replace + target: use this AST node to replace matching expressions + """ + self.funcs = set(funcs) + self.target = target + + def visit_Call(self, node): # pylint: disable=invalid-name + if ast.dump(node, False) in self.funcs: + return self.target + return node + + +class Transformer(ast.NodeTransformer): + """Transform original code to annotated code""" + + def __init__(self): + self.stack = [] + self.last_line = 0 + self.annotated = False + + def visit(self, node): + if isinstance(node, (ast.expr, ast.stmt)): + self.last_line = lineno(node) + + # do nothing for root + if not self.stack: + return self._visit_children(node) + + annotation = self.stack[-1] + + # this is a standalone string, may be an annotation + if type(node) is ast.Expr and type(node.value) is ast_Str: + # must not annotate an annotation string + assert annotation is None, 'Annotating an annotation' + return self._visit_string(node) + + if annotation is not None: # this expression is annotated + self.stack[-1] = None # so next expression is not + if annotation.startswith('nni.variable'): + return replace_variable_node(node, annotation) + if annotation.startswith('nni.function_choice'): + return replace_function_node(node, annotation) + + return self._visit_children(node) + + def _visit_string(self, node): + string = node.value.s + if string.startswith('@nni.'): + self.annotated = True + else: + return node # not an annotation, ignore it + + if string.startswith('@nni.get_next_parameter'): + deprecated_message = "'@nni.get_next_parameter' is deprecated in annotation due to inconvenience. " \ + "Please remove this line in the trial code." + print_warning(deprecated_message) + return ast.Expr(value=ast.Call(func=ast.Name(id='print', ctx=ast.Load()), + args=[ast_Str(s='Get next parameter here...')], keywords=[])) + + if string.startswith('@nni.training_update'): + return ast.Expr(value=ast.Call(func=ast.Name(id='print', ctx=ast.Load()), + args=[ast_Str(s='Training update here...')], keywords=[])) + + if string.startswith('@nni.report_intermediate_result'): + module = ast.parse(string[1:]) + arg = module.body[0].value.args[0] + return ast.Expr(value=ast.Call(func=ast.Name(id='print', ctx=ast.Load()), + args=[ast_Str(s='nni.report_intermediate_result: '), arg], keywords=[])) + + if string.startswith('@nni.report_final_result'): + module = ast.parse(string[1:]) + arg = module.body[0].value.args[0] + return ast.Expr(value=ast.Call(func=ast.Name(id='print', ctx=ast.Load()), + args=[ast_Str(s='nni.report_final_result: '), arg], keywords=[])) + + if string.startswith('@nni.mutable_layers'): + return parse_annotation_mutable_layers(string[1:], lineno(node)) + + if string.startswith('@nni.variable') \ + or string.startswith('@nni.function_choice'): + self.stack[-1] = string[1:] # mark that the next expression is annotated + return None + + raise AssertionError('Unexpected annotation function') + + def _visit_children(self, node): + self.stack.append(None) + self.generic_visit(node) + annotation = self.stack.pop() + assert annotation is None, 'Annotation has no target' + return node + + +def parse(code, para, module): + """Annotate user code. + Return annotated code (str) if annotation detected; return None if not. + code: original user code (str) + """ + global para_cfg + global prefix_name + para_cfg = para + prefix_name = module + try: + ast_tree = ast.parse(code) + except Exception: + raise RuntimeError('Bad Python code') + + transformer = Transformer() + try: + transformer.visit(ast_tree) + except AssertionError as exc: + raise RuntimeError('%d: %s' % (ast_tree.last_line, exc.args[0])) + + if not transformer.annotated: + return None + + return astor.to_source(ast_tree) diff --git a/new_impl/cv/third_party/nni_new/tools/annotation/utils.py b/new_impl/cv/third_party/nni_new/tools/annotation/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..176621c2cf3d9211d626b182277ef61f170880f9 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/tools/annotation/utils.py @@ -0,0 +1,22 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import ast +from sys import version_info + + +if version_info >= (3, 8): + ast_Num = ast_Str = ast_Bytes = ast_NameConstant = ast_Ellipsis = ast.Constant + + def lineno(ast_node): + return ast_node.end_lineno + +else: + ast_Num = ast.Num + ast_Str = ast.Str + ast_Bytes = ast.Bytes + ast_NameConstant = ast.NameConstant + ast_Ellipsis = ast.Ellipsis + + def lineno(ast_node): + return ast_node.lineno diff --git a/new_impl/cv/third_party/nni_new/tools/gpu_tool/__init__.py b/new_impl/cv/third_party/nni_new/tools/gpu_tool/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/new_impl/cv/third_party/nni_new/tools/gpu_tool/gpu_metrics_collector.py b/new_impl/cv/third_party/nni_new/tools/gpu_tool/gpu_metrics_collector.py new file mode 100644 index 0000000000000000000000000000000000000000..8e50cf9ec84533c1b8ded1799918934677eff058 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/tools/gpu_tool/gpu_metrics_collector.py @@ -0,0 +1,82 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import json +import os +import subprocess +import sys +import time +import traceback + +from xml.dom import minidom + + +def main(argv): + metrics_output_dir = os.environ['METRIC_OUTPUT_DIR'] + + cmd = 'nvidia-smi -q -x'.split() + while(True): + try: + smi_output = subprocess.check_output(cmd) + except Exception: + traceback.print_exc() + gen_empty_gpu_metric(metrics_output_dir) + break + parse_nvidia_smi_result(smi_output, metrics_output_dir) + # TODO: change to sleep time configurable via arguments + time.sleep(5) + + +def parse_nvidia_smi_result(smi, outputDir): + try: + old_umask = os.umask(0) + xmldoc = minidom.parseString(smi) + gpuList = xmldoc.getElementsByTagName('gpu') + with open(os.path.join(outputDir, "gpu_metrics"), 'a') as outputFile: + outPut = {} + outPut["Timestamp"] = time.asctime(time.localtime()) + outPut["gpuCount"] = len(gpuList) + outPut["gpuInfos"] = [] + for gpuIndex, gpu in enumerate(gpuList): + gpuInfo = {} + gpuInfo['index'] = gpuIndex + gpuInfo['gpuUtil'] = gpu.getElementsByTagName('utilization')[0]\ + .getElementsByTagName('gpu_util')[0]\ + .childNodes[0].data.replace("%", "").strip() + gpuInfo['gpuMemUtil'] = gpu.getElementsByTagName('utilization')[0]\ + .getElementsByTagName('memory_util')[0]\ + .childNodes[0].data.replace("%", "").strip() + processes = gpu.getElementsByTagName('processes') + runningProNumber = len(processes[0].getElementsByTagName('process_info')) + gpuInfo['activeProcessNum'] = runningProNumber + + outPut["gpuInfos"].append(gpuInfo) + print(outPut) + outputFile.write("{}\n".format(json.dumps(outPut, sort_keys=True))) + outputFile.flush() + except Exception as error: + # e_info = sys.exc_info() + print('gpu_metrics_collector error: %s' % error) + finally: + os.umask(old_umask) + + +def gen_empty_gpu_metric(outputDir): + try: + old_umask = os.umask(0) + with open(os.path.join(outputDir, "gpu_metrics"), 'a') as outputFile: + outPut = {} + outPut["Timestamp"] = time.asctime(time.localtime()) + outPut["gpuCount"] = 0 + outPut["gpuInfos"] = [] + print(outPut) + outputFile.write("{}\n".format(json.dumps(outPut, sort_keys=True))) + outputFile.flush() + except Exception: + traceback.print_exc() + finally: + os.umask(old_umask) + + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/new_impl/cv/third_party/nni_new/tools/nnictl/__init__.py b/new_impl/cv/third_party/nni_new/tools/nnictl/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/new_impl/cv/third_party/nni_new/tools/nnictl/algo_management.py b/new_impl/cv/third_party/nni_new/tools/nnictl/algo_management.py new file mode 100644 index 0000000000000000000000000000000000000000..1715e0130085a9c5b6395e1e5921a7daaf286bd8 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/tools/nnictl/algo_management.py @@ -0,0 +1,106 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import importlib +import json +from nni.tools.package_utils import read_registerd_algo_meta, get_registered_algo_meta, \ + write_registered_algo_meta, ALGO_TYPES, parse_full_class_name +from .common_utils import print_error, print_green, get_yml_content + +def read_reg_meta_list(meta_path): + content = get_yml_content(meta_path) + if content.get('algorithms'): + meta_list = content.get('algorithms') + else: + meta_list = [content] + for meta in meta_list: + assert 'algoType' in meta + assert meta['algoType'] in ['tuner', 'assessor', 'advisor'] + assert 'builtinName' in meta + assert 'className' in meta + return meta_list + +def verify_algo_import(meta): + def _do_verify_import(fullName): + module_name, class_name = parse_full_class_name(fullName) + class_module = importlib.import_module(module_name) + getattr(class_module, class_name) + + _do_verify_import(meta['className']) + + if meta.get('classArgsValidator'): + _do_verify_import(meta['classArgsValidator']) + +def algo_reg(args): + meta_list = read_reg_meta_list(args.meta_path) + for meta in meta_list: + old = get_registered_algo_meta(meta['builtinName']) + if old is None: + verify_algo_import(meta) + save_algo_meta_data(meta) + elif old['source'] != 'nni': + verify_algo_import(meta) + print_green(f'Updating exist algorithm') + remove_algo_meta_data(meta['builtinName']) + save_algo_meta_data(meta) + else: + print_error(f'Cannot overwrite builtin algorithm') + print_green('{} registered sucessfully!'.format(meta['builtinName'])) + +def algo_unreg(args): + name = args.name[0] + meta = get_registered_algo_meta(name) + if meta is None: + print_error('builtin algorithms {} not found!'.format(name)) + return + if meta['source'] == 'nni': + print_error('{} is provided by nni, can not be unregistered!'.format(name)) + return + if remove_algo_meta_data(name): + print_green('{} unregistered sucessfully!'.format(name)) + else: + print_error('Failed to unregistered {}!'.format(name)) + +def algo_show(args): + builtin_name = args.name[0] + meta = get_registered_algo_meta(builtin_name) + if meta: + print(json.dumps(meta, indent=4)) + else: + print_error('package {} not found'.format(builtin_name)) + +def algo_list(args): + meta = read_registerd_algo_meta() + print('+-----------------+------------+-----------+--------=-------------+------------------------------------------+') + print('| Name | Type | source | Class Name | Module Name |') + print('+-----------------+------------+-----------+----------------------+------------------------------------------+') + MAX_MODULE_NAME = 38 + for t in ['tuners', 'assessors', 'advisors']: + for p in meta[t]: + module_name = '.'.join(p['className'].split('.')[:-1]) + if len(module_name) > MAX_MODULE_NAME: + module_name = module_name[:MAX_MODULE_NAME-3] + '...' + class_name = p['className'].split('.')[-1] + print('| {:15s} | {:10s} | {:9s} | {:20s} | {:40s} |'.format(p['builtinName'], t, p['source'], class_name, module_name[:38])) + print('+-----------------+------------+-----------+----------------------+------------------------------------------+') + + +def save_algo_meta_data(meta_data): + meta_data['source'] = 'user' + config = read_registerd_algo_meta() + config[meta_data['algoType']+'s'].append(meta_data) + write_registered_algo_meta(config) + +def remove_algo_meta_data(name): + config = read_registerd_algo_meta() + + updated = False + for t in ALGO_TYPES: + for meta in config[t]: + if meta['builtinName'] == name: + config[t].remove(meta) + updated = True + if updated: + write_registered_algo_meta(config) + return True + return False diff --git a/new_impl/cv/third_party/nni_new/tools/nnictl/command_utils.py b/new_impl/cv/third_party/nni_new/tools/nnictl/command_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..2bbcc883d1cef450fef9b723dc8ca71d34daa095 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/tools/nnictl/command_utils.py @@ -0,0 +1,82 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from subprocess import call, check_output +import sys +import os +import signal +import psutil +from .common_utils import print_error + + +def check_output_command(file_path, head=None, tail=None): + """call check_output command to read content from a file""" + if os.path.exists(file_path): + if sys.platform == 'win32': + cmds = ['powershell.exe', 'type', file_path] + if head: + cmds += ['|', 'select', '-first', str(head)] + elif tail: + cmds += ['|', 'select', '-last', str(tail)] + return check_output(cmds, shell=True).decode('utf-8') + else: + cmds = ['cat', file_path] + if head: + cmds = ['head', '-' + str(head), file_path] + elif tail: + cmds = ['tail', '-' + str(tail), file_path] + return check_output(cmds, shell=False).decode('utf-8') + else: + print_error('{0} does not exist!'.format(file_path)) + exit(1) + + +def kill_command(pid): + """kill command""" + if sys.platform == 'win32': + process = psutil.Process(pid=pid) + process.send_signal(signal.CTRL_BREAK_EVENT) + else: + cmds = ['kill', str(pid)] + call(cmds) + + +def install_package_command(package_name): + """ + Install python package from pip. + + Parameters + ---------- + package_name: str + The name of package to be installed. + """ + call(_get_pip_install() + [package_name], shell=False) + + +def install_requirements_command(requirements_path): + """ + Install packages from `requirements.txt` in `requirements_path`. + + Parameters + ---------- + requirements_path: str + Path to the directory that contains `requirements.txt`. + """ + return call(_get_pip_install() + ["-r", requirements_path], shell=False) + + +def _get_pip_install(): + python = "python" if sys.platform == "win32" else "python3" + ret = [python, "-m", "pip", "install"] + if "CONDA_DEFAULT_ENV" not in os.environ and "VIRTUAL_ENV" not in os.environ and \ + (sys.platform != "win32" and os.getuid() != 0): # on unix and not running in root + ret.append("--user") # not in virtualenv or conda + return ret + +def call_pip_install(source): + return call(_get_pip_install() + [source]) + +def call_pip_uninstall(module_name): + python = "python" if sys.platform == "win32" else "python3" + cmd = [python, "-m", "pip", "uninstall", module_name] + return call(cmd) diff --git a/new_impl/cv/third_party/nni_new/tools/nnictl/common_utils.py b/new_impl/cv/third_party/nni_new/tools/nnictl/common_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..25df67cf2e11297ccbf02223432a72c01768868c --- /dev/null +++ b/new_impl/cv/third_party/nni_new/tools/nnictl/common_utils.py @@ -0,0 +1,125 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +import sys +import json +import tempfile +import time +import socket +import string +import random +import glob +from colorama import Fore +import filelock +import psutil +import yaml + +from .constants import ERROR_INFO, NORMAL_INFO, WARNING_INFO + +def get_yml_content(file_path): + '''Load yaml file content''' + try: + with open(file_path, 'r') as file: + return yaml.safe_load(file) + except yaml.scanner.ScannerError as err: + print_error('yaml file format error!') + print_error(err) + exit(1) + except Exception as exception: + print_error(exception) + exit(1) + +def get_json_content(file_path): + '''Load json file content''' + try: + with open(file_path, 'r') as file: + return json.load(file) + except TypeError as err: + print_error('json file format error!') + print_error(err) + return None + + +def print_error(*content): + '''Print error information to screen''' + print(Fore.RED + ERROR_INFO + ' '.join([str(c) for c in content]) + Fore.RESET) + +def print_green(*content): + '''Print information to screen in green''' + print(Fore.GREEN + ' '.join([str(c) for c in content]) + Fore.RESET) + +def print_normal(*content): + '''Print error information to screen''' + print(NORMAL_INFO, *content) + +def print_warning(*content): + '''Print warning information to screen''' + print(Fore.YELLOW + WARNING_INFO + ' '.join([str(c) for c in content]) + Fore.RESET) + +def detect_process(pid): + '''Detect if a process is alive''' + try: + process = psutil.Process(pid) + return process.is_running() + except: + return False + +def detect_port(port): + '''Detect if the port is used''' + socket_test = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + try: + socket_test.connect(('127.0.0.1', int(port))) + socket_test.close() + return True + except: + return False + +def get_user(): + if sys.platform == 'win32': + return os.environ['USERNAME'] + else: + return os.environ['USER'] + +def generate_temp_dir(): + '''generate a temp folder''' + def generate_folder_name(): + return os.path.join(tempfile.gettempdir(), 'nni', ''.join(random.sample(string.ascii_letters + string.digits, 8))) + temp_dir = generate_folder_name() + while os.path.exists(temp_dir): + temp_dir = generate_folder_name() + os.makedirs(temp_dir) + return temp_dir + +class SimplePreemptiveLock(filelock.SoftFileLock): + '''this is a lock support check lock expiration, if you do not need check expiration, you can use SoftFileLock''' + def __init__(self, lock_file, stale=-1): + super(__class__, self).__init__(lock_file, timeout=-1) + self._lock_file_name = '{}.{}'.format(self._lock_file, os.getpid()) + self._stale = stale + + def _acquire(self): + open_mode = os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_TRUNC + try: + lock_file_names = glob.glob(self._lock_file + '.*') + for file_name in lock_file_names: + if os.path.exists(file_name) and (self._stale < 0 or time.time() - os.stat(file_name).st_mtime < self._stale): + return None + fd = os.open(self._lock_file_name, open_mode) + except (IOError, OSError): + pass + else: + self._lock_file_fd = fd + return None + + def _release(self): + os.close(self._lock_file_fd) + self._lock_file_fd = None + try: + os.remove(self._lock_file_name) + except OSError: + pass + return None + +def get_file_lock(path: string, stale=-1): + return SimplePreemptiveLock(path + '.lock', stale=stale) diff --git a/new_impl/cv/third_party/nni_new/tools/nnictl/config_schema.py b/new_impl/cv/third_party/nni_new/tools/nnictl/config_schema.py new file mode 100644 index 0000000000000000000000000000000000000000..65e645b560bf862ff37be5a9f28f5a9e4f7f7e2e --- /dev/null +++ b/new_impl/cv/third_party/nni_new/tools/nnictl/config_schema.py @@ -0,0 +1,641 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import json +import logging +import os + +import netifaces +from schema import And, Optional, Or, Regex, Schema, SchemaError +from nni.tools.package_utils import ( + create_validator_instance, + get_all_builtin_names, + get_registered_algo_meta, +) + +from .common_utils import get_yml_content, print_warning +from .constants import SCHEMA_PATH_ERROR, SCHEMA_RANGE_ERROR, SCHEMA_TYPE_ERROR + + +def setType(key, valueType): + '''check key type''' + return And(valueType, error=SCHEMA_TYPE_ERROR % (key, valueType.__name__)) + + +def setChoice(key, *args): + '''check choice''' + return And(lambda n: n in args, error=SCHEMA_RANGE_ERROR % (key, str(args))) + + +def setNumberRange(key, keyType, start, end): + '''check number range''' + return And( + And(keyType, error=SCHEMA_TYPE_ERROR % (key, keyType.__name__)), + And(lambda n: start <= n <= end, error=SCHEMA_RANGE_ERROR % (key, '(%s,%s)' % (start, end))), + ) + + +def setPathCheck(key): + '''check if path exist''' + return And(os.path.exists, error=SCHEMA_PATH_ERROR % key) + + +class AlgoSchema: + """ + This class is the schema of 'tuner', 'assessor' and 'advisor' sections of experiment configuraion file. + For example: + AlgoSchema('tuner') creates the schema of tuner section. + """ + + def __init__(self, algo_type): + """ + Parameters: + ----------- + algo_type: str + One of ['tuner', 'assessor', 'advisor']. + 'tuner': This AlgoSchema class create the schema of tuner section. + 'assessor': This AlgoSchema class create the schema of assessor section. + 'advisor': This AlgoSchema class create the schema of advisor section. + """ + assert algo_type in ['tuner', 'assessor', 'advisor'] + self.algo_type = algo_type + self.algo_schema = { + Optional('codeDir'): setPathCheck('codeDir'), + Optional('classFileName'): setType('classFileName', str), + Optional('className'): setType('className', str), + Optional('classArgs'): dict, + Optional('includeIntermediateResults'): setType('includeIntermediateResults', bool), + Optional('gpuIndices'): Or(int, And(str, lambda x: len([int(i) for i in x.split(',')]) > 0), error='gpuIndex format error!'), + } + self.builtin_keys = { + 'tuner': 'builtinTunerName', + 'assessor': 'builtinAssessorName', + 'advisor': 'builtinAdvisorName' + } + self.builtin_name_schema = {} + for k, n in self.builtin_keys.items(): + self.builtin_name_schema[k] = {Optional(n): setChoice(n, *get_all_builtin_names(k+'s'))} + + self.customized_keys = set(['codeDir', 'classFileName', 'className']) + + def validate_class_args(self, class_args, algo_type, builtin_name): + if not builtin_name or not class_args: + return + meta = get_registered_algo_meta(builtin_name, algo_type+'s') + if meta and 'acceptClassArgs' in meta and meta['acceptClassArgs'] == False: + raise SchemaError('classArgs is not allowed.') + + logging.getLogger('nni.protocol').setLevel(logging.ERROR) # we know IPC is not there, don't complain + validator = create_validator_instance(algo_type+'s', builtin_name) + if validator: + try: + validator.validate_class_args(**class_args) + except Exception as e: + raise SchemaError(str(e)) + + def missing_customized_keys(self, data): + return self.customized_keys - set(data.keys()) + + def validate_extras(self, data, algo_type): + builtin_key = self.builtin_keys[algo_type] + if (builtin_key in data) and (set(data.keys()) & self.customized_keys): + raise SchemaError('{} and {} cannot be specified at the same time.'.format( + builtin_key, set(data.keys()) & self.customized_keys + )) + + if self.missing_customized_keys(data) and builtin_key not in data: + raise SchemaError('Either customized {} ({}) or builtin {} ({}) must be set.'.format( + algo_type, self.customized_keys, algo_type, builtin_key)) + + if not self.missing_customized_keys(data): + class_file_name = os.path.join(data['codeDir'], data['classFileName']) + if not os.path.isfile(class_file_name): + raise SchemaError('classFileName {} not found.'.format(class_file_name)) + + builtin_name = data.get(builtin_key) + class_args = data.get('classArgs') + self.validate_class_args(class_args, algo_type, builtin_name) + + def validate(self, data): + self.algo_schema.update(self.builtin_name_schema[self.algo_type]) + Schema(self.algo_schema).validate(data) + self.validate_extras(data, self.algo_type) + + +common_schema = { + 'authorName': setType('authorName', str), + 'experimentName': setType('experimentName', str), + Optional('description'): setType('description', str), + 'trialConcurrency': setNumberRange('trialConcurrency', int, 1, 99999), + Optional('maxExecDuration'): And(Regex(r'^[1-9][0-9]*[s|m|h|d]$', error='ERROR: maxExecDuration format is [digit]{s,m,h,d}')), + Optional('maxTrialNum'): setNumberRange('maxTrialNum', int, 1, 99999), + 'trainingServicePlatform': setChoice( + 'trainingServicePlatform', 'remote', 'local', 'pai', 'kubeflow', 'frameworkcontroller', 'dlts', 'aml', 'adl', 'hybrid'), + Optional('searchSpacePath'): And(os.path.exists, error=SCHEMA_PATH_ERROR % 'searchSpacePath'), + Optional('multiPhase'): setType('multiPhase', bool), + Optional('multiThread'): setType('multiThread', bool), + Optional('nniManagerIp'): setType('nniManagerIp', str), + Optional('logDir'): And(os.path.isdir, error=SCHEMA_PATH_ERROR % 'logDir'), + Optional('debug'): setType('debug', bool), + Optional('versionCheck'): setType('versionCheck', bool), + Optional('logLevel'): setChoice('logLevel', 'trace', 'debug', 'info', 'warning', 'error', 'fatal'), + Optional('logCollection'): setChoice('logCollection', 'http', 'none'), + 'useAnnotation': setType('useAnnotation', bool), + Optional('tuner'): AlgoSchema('tuner'), + Optional('advisor'): AlgoSchema('advisor'), + Optional('assessor'): AlgoSchema('assessor'), + Optional('localConfig'): { + Optional('gpuIndices'): Or(int, And(str, lambda x: len([int(i) for i in x.split(',')]) > 0), error='gpuIndex format error!'), + Optional('maxTrialNumPerGpu'): setType('maxTrialNumPerGpu', int), + Optional('useActiveGpu'): setType('useActiveGpu', bool) + }, + Optional('sharedStorage'): { + 'storageType': setChoice('storageType', 'NFS', 'AzureBlob'), + Optional('localMountPoint'): setType('localMountPoint', str), + Optional('remoteMountPoint'): setType('remoteMountPoint', str), + Optional('nfsServer'): setType('nfsServer', str), + Optional('exportedDirectory'): setType('exportedDirectory', str), + Optional('storageAccountName'): setType('storageAccountName', str), + Optional('storageAccountKey'): setType('storageAccountKey', str), + Optional('containerName'): setType('containerName', str), + Optional('resourceGroupName'): setType('resourceGroupName', str), + Optional('localMounted'): setChoice('localMounted', 'usermount', 'nnimount', 'nomount') + } +} + +common_trial_schema = { + 'trial': { + 'command': setType('command', str), + 'codeDir': setPathCheck('codeDir'), + Optional('gpuNum'): setNumberRange('gpuNum', int, 0, 99999), + Optional('nasMode'): setChoice('nasMode', 'classic_mode', 'enas_mode', 'oneshot_mode', 'darts_mode') + } +} + +pai_yarn_trial_schema = { + 'trial': { + 'command': setType('command', str), + 'codeDir': setPathCheck('codeDir'), + 'gpuNum': setNumberRange('gpuNum', int, 0, 99999), + 'cpuNum': setNumberRange('cpuNum', int, 0, 99999), + 'memoryMB': setType('memoryMB', int), + 'image': setType('image', str), + Optional('authFile'): And(os.path.exists, error=SCHEMA_PATH_ERROR % 'authFile'), + Optional('shmMB'): setType('shmMB', int), + Optional('dataDir'): And(Regex(r'hdfs://(([0-9]{1,3}.){3}[0-9]{1,3})(:[0-9]{2,5})?(/.*)?'), + error='ERROR: dataDir format error, dataDir format is hdfs://xxx.xxx.xxx.xxx:xxx'), + Optional('outputDir'): And(Regex(r'hdfs://(([0-9]{1,3}.){3}[0-9]{1,3})(:[0-9]{2,5})?(/.*)?'), + error='ERROR: outputDir format error, outputDir format is hdfs://xxx.xxx.xxx.xxx:xxx'), + Optional('virtualCluster'): setType('virtualCluster', str), + Optional('nasMode'): setChoice('nasMode', 'classic_mode', 'enas_mode', 'oneshot_mode', 'darts_mode'), + Optional('portList'): [{ + 'label': setType('label', str), + 'beginAt': setType('beginAt', int), + 'portNumber': setType('portNumber', int) + }] + } +} + + +pai_trial_schema = { + 'trial': { + 'codeDir': setPathCheck('codeDir'), + 'nniManagerNFSMountPath': setPathCheck('nniManagerNFSMountPath'), + 'containerNFSMountPath': setType('containerNFSMountPath', str), + Optional('command'): setType('command', str), + Optional('gpuNum'): setNumberRange('gpuNum', int, 0, 99999), + Optional('cpuNum'): setNumberRange('cpuNum', int, 0, 99999), + Optional('memoryMB'): setType('memoryMB', int), + Optional('image'): setType('image', str), + Optional('virtualCluster'): setType('virtualCluster', str), + Optional('paiStorageConfigName'): setType('paiStorageConfigName', str), + Optional('paiConfigPath'): And(os.path.exists, error=SCHEMA_PATH_ERROR % 'paiConfigPath') + } +} + +pai_config_schema = { + Optional('paiConfig'): { + 'userName': setType('userName', str), + Or('passWord', 'token', only_one=True): str, + 'host': setType('host', str), + Optional('reuse'): setType('reuse', bool), + Optional('gpuNum'): setNumberRange('gpuNum', int, 0, 99999), + Optional('cpuNum'): setNumberRange('cpuNum', int, 0, 99999), + Optional('memoryMB'): setType('memoryMB', int), + Optional('maxTrialNumPerGpu'): setType('maxTrialNumPerGpu', int), + Optional('useActiveGpu'): setType('useActiveGpu', bool), + } +} + +dlts_trial_schema = { + 'trial': { + 'command': setType('command', str), + 'codeDir': setPathCheck('codeDir'), + 'gpuNum': setNumberRange('gpuNum', int, 0, 99999), + 'image': setType('image', str), + } +} + +dlts_config_schema = { + 'dltsConfig': { + 'dashboard': setType('dashboard', str), + + Optional('cluster'): setType('cluster', str), + Optional('team'): setType('team', str), + + Optional('email'): setType('email', str), + Optional('password'): setType('password', str), + } +} + +aml_trial_schema = { + 'trial': { + 'codeDir': setPathCheck('codeDir'), + 'command': setType('command', str), + 'image': setType('image', str), + Optional('gpuNum'): setNumberRange('gpuNum', int, 0, 99999), + } +} + +aml_config_schema = { + Optional('amlConfig'): { + 'subscriptionId': setType('subscriptionId', str), + 'resourceGroup': setType('resourceGroup', str), + 'workspaceName': setType('workspaceName', str), + 'computeTarget': setType('computeTarget', str), + Optional('maxTrialNumPerGpu'): setType('maxTrialNumPerGpu', int), + Optional('useActiveGpu'): setType('useActiveGpu', bool), + } +} + +hybrid_trial_schema = { + 'trial': { + 'codeDir': setPathCheck('codeDir'), + Optional('nniManagerNFSMountPath'): setPathCheck('nniManagerNFSMountPath'), + Optional('containerNFSMountPath'): setType('containerNFSMountPath', str), + Optional('nasMode'): setChoice('nasMode', 'classic_mode', 'enas_mode', 'oneshot_mode', 'darts_mode'), + 'command': setType('command', str), + Optional('gpuNum'): setNumberRange('gpuNum', int, 0, 99999), + Optional('cpuNum'): setNumberRange('cpuNum', int, 0, 99999), + Optional('memoryMB'): setType('memoryMB', int), + Optional('image'): setType('image', str), + Optional('virtualCluster'): setType('virtualCluster', str), + Optional('paiStorageConfigName'): setType('paiStorageConfigName', str), + Optional('paiConfigPath'): And(os.path.exists, error=SCHEMA_PATH_ERROR % 'paiConfigPath') + } +} + +hybrid_config_schema = { + 'hybridConfig': { + 'trainingServicePlatforms': ['local', 'remote', 'pai', 'aml'] + } +} + +adl_trial_schema = { + 'trial':{ + 'codeDir': setType('codeDir', str), + 'command': setType('command', str), + 'gpuNum': setNumberRange('gpuNum', int, 0, 99999), + 'image': setType('image', str), + Optional('namespace'): setType('namespace', str), + Optional('imagePullSecrets'): [{ + 'name': setType('name', str) + }], + Optional('nfs'): { + 'server': setType('server', str), + 'path': setType('path', str), + 'containerMountPath': setType('containerMountPath', str) + }, + Optional('adaptive'): setType('adaptive', bool), + Optional('checkpoint'): { + 'storageClass': setType('storageClass', str), + 'storageSize': setType('storageSize', str) + }, + Optional('cpuNum'): setNumberRange('cpuNum', int, 0, 99999), + Optional('memorySize'): setType('memorySize', str) + } +} + +kubeflow_trial_schema = { + 'trial': { + 'codeDir': setPathCheck('codeDir'), + Optional('nasMode'): setChoice('nasMode', 'classic_mode', 'enas_mode', 'oneshot_mode', 'darts_mode'), + Optional('ps'): { + 'replicas': setType('replicas', int), + 'command': setType('command', str), + 'gpuNum': setNumberRange('gpuNum', int, 0, 99999), + 'cpuNum': setNumberRange('cpuNum', int, 0, 99999), + 'memoryMB': setType('memoryMB', int), + 'image': setType('image', str), + Optional('privateRegistryAuthPath'): And(os.path.exists, error=SCHEMA_PATH_ERROR % 'privateRegistryAuthPath') + }, + Optional('master'): { + 'replicas': setType('replicas', int), + 'command': setType('command', str), + 'gpuNum': setNumberRange('gpuNum', int, 0, 99999), + 'cpuNum': setNumberRange('cpuNum', int, 0, 99999), + 'memoryMB': setType('memoryMB', int), + 'image': setType('image', str), + Optional('privateRegistryAuthPath'): And(os.path.exists, error=SCHEMA_PATH_ERROR % 'privateRegistryAuthPath') + }, + Optional('worker'): { + 'replicas': setType('replicas', int), + 'command': setType('command', str), + 'gpuNum': setNumberRange('gpuNum', int, 0, 99999), + 'cpuNum': setNumberRange('cpuNum', int, 0, 99999), + 'memoryMB': setType('memoryMB', int), + 'image': setType('image', str), + Optional('privateRegistryAuthPath'): And(os.path.exists, error=SCHEMA_PATH_ERROR % 'privateRegistryAuthPath') + } + } +} + +kubeflow_config_schema = { + 'kubeflowConfig': Or({ + 'operator': setChoice('operator', 'tf-operator', 'pytorch-operator'), + 'apiVersion': setType('apiVersion', str), + Optional('storage'): setChoice('storage', 'nfs', 'azureStorage'), + 'nfs': { + 'server': setType('server', str), + 'path': setType('path', str) + } + }, { + 'operator': setChoice('operator', 'tf-operator', 'pytorch-operator'), + 'apiVersion': setType('apiVersion', str), + Optional('storage'): setChoice('storage', 'nfs', 'azureStorage'), + 'keyVault': { + 'vaultName': And(Regex('([0-9]|[a-z]|[A-Z]|-){1,127}'), + error='ERROR: vaultName format error, vaultName support using (0-9|a-z|A-Z|-)'), + 'name': And(Regex('([0-9]|[a-z]|[A-Z]|-){1,127}'), + error='ERROR: name format error, name support using (0-9|a-z|A-Z|-)') + }, + 'azureStorage': { + 'accountName': And(Regex('([0-9]|[a-z]|[A-Z]|-){3,31}'), + error='ERROR: accountName format error, accountName support using (0-9|a-z|A-Z|-)'), + 'azureShare': And(Regex('([0-9]|[a-z]|[A-Z]|-){3,63}'), + error='ERROR: azureShare format error, azureShare support using (0-9|a-z|A-Z|-)') + }, + Optional('uploadRetryCount'): setNumberRange('uploadRetryCount', int, 1, 99999) + }) +} + +frameworkcontroller_trial_schema = { + 'trial': { + 'codeDir': setPathCheck('codeDir'), + Optional('taskRoles'): [{ + 'name': setType('name', str), + 'taskNum': setType('taskNum', int), + 'frameworkAttemptCompletionPolicy': { + 'minFailedTaskCount': setType('minFailedTaskCount', int), + 'minSucceededTaskCount': setType('minSucceededTaskCount', int), + }, + 'command': setType('command', str), + 'gpuNum': setNumberRange('gpuNum', int, 0, 99999), + 'cpuNum': setNumberRange('cpuNum', int, 0, 99999), + 'memoryMB': setType('memoryMB', int), + 'image': setType('image', str), + Optional('privateRegistryAuthPath'): And(os.path.exists, error=SCHEMA_PATH_ERROR % 'privateRegistryAuthPath') + }] + } +} + +frameworkcontroller_config_schema = { + 'frameworkcontrollerConfig': Or({ + Optional('storage'): setChoice('storage', 'nfs', 'azureStorage', 'pvc'), + Optional('serviceAccountName'): setType('serviceAccountName', str), + 'nfs': { + 'server': setType('server', str), + 'path': setType('path', str) + }, + Optional('namespace'): setType('namespace', str), + Optional('configPath'): setType('configPath', str), + }, { + Optional('storage'): setChoice('storage', 'nfs', 'azureStorage', 'pvc'), + Optional('serviceAccountName'): setType('serviceAccountName', str), + 'configPath': setType('configPath', str), + 'pvc': {'path': setType('server', str)}, + Optional('namespace'): setType('namespace', str), + }, { + Optional('storage'): setChoice('storage', 'nfs', 'azureStorage', 'pvc'), + Optional('serviceAccountName'): setType('serviceAccountName', str), + 'keyVault': { + 'vaultName': And(Regex('([0-9]|[a-z]|[A-Z]|-){1,127}'), + error='ERROR: vaultName format error, vaultName support using (0-9|a-z|A-Z|-)'), + 'name': And(Regex('([0-9]|[a-z]|[A-Z]|-){1,127}'), + error='ERROR: name format error, name support using (0-9|a-z|A-Z|-)') + }, + 'azureStorage': { + 'accountName': And(Regex('([0-9]|[a-z]|[A-Z]|-){3,31}'), + error='ERROR: accountName format error, accountName support using (0-9|a-z|A-Z|-)'), + 'azureShare': And(Regex('([0-9]|[a-z]|[A-Z]|-){3,63}'), + error='ERROR: azureShare format error, azureShare support using (0-9|a-z|A-Z|-)') + }, + Optional('uploadRetryCount'): setNumberRange('uploadRetryCount', int, 1, 99999), + Optional('namespace'): setType('namespace', str), + Optional('configPath'): setType('configPath', str), + }) +} + +remote_config_schema = { + Optional('remoteConfig'): { + 'reuse': setType('reuse', bool) + } +} + +machine_list_schema = { + Optional('machineList'): [Or( + { + 'ip': setType('ip', str), + Optional('port'): setNumberRange('port', int, 1, 65535), + 'username': setType('username', str), + 'sshKeyPath': setPathCheck('sshKeyPath'), + Optional('passphrase'): setType('passphrase', str), + Optional('gpuIndices'): Or(int, And(str, lambda x: len([int(i) for i in x.split(',')]) > 0), error='gpuIndex format error!'), + Optional('maxTrialNumPerGpu'): setType('maxTrialNumPerGpu', int), + Optional('useActiveGpu'): setType('useActiveGpu', bool), + Optional('pythonPath'): setType('pythonPath', str) + }, + { + 'ip': setType('ip', str), + Optional('port'): setNumberRange('port', int, 1, 65535), + 'username': setType('username', str), + 'passwd': setType('passwd', str), + Optional('gpuIndices'): Or(int, And(str, lambda x: len([int(i) for i in x.split(',')]) > 0), error='gpuIndex format error!'), + Optional('maxTrialNumPerGpu'): setType('maxTrialNumPerGpu', int), + Optional('useActiveGpu'): setType('useActiveGpu', bool), + Optional('pythonPath'): setType('pythonPath', str) + })] +} + +training_service_schema_dict = { + 'adl': Schema({**common_schema, **adl_trial_schema}), + 'local': Schema({**common_schema, **common_trial_schema}), + 'remote': Schema({**common_schema, **common_trial_schema, **machine_list_schema, **remote_config_schema}), + 'pai': Schema({**common_schema, **pai_trial_schema, **pai_config_schema}), + 'kubeflow': Schema({**common_schema, **kubeflow_trial_schema, **kubeflow_config_schema}), + 'frameworkcontroller': Schema({**common_schema, **frameworkcontroller_trial_schema, **frameworkcontroller_config_schema}), + 'aml': Schema({**common_schema, **aml_trial_schema, **aml_config_schema}), + 'dlts': Schema({**common_schema, **dlts_trial_schema, **dlts_config_schema}), + 'hybrid': Schema({**common_schema, **hybrid_trial_schema, **hybrid_config_schema, **machine_list_schema, + **pai_config_schema, **aml_config_schema, **remote_config_schema}), +} + + +class NNIConfigSchema: + def validate(self, data): + train_service = data['trainingServicePlatform'] + Schema(common_schema['trainingServicePlatform']).validate(train_service) + train_service_schema = training_service_schema_dict[train_service] + train_service_schema.validate(data) + self.validate_extras(data) + + def validate_extras(self, experiment_config): + self.validate_tuner_adivosr_assessor(experiment_config) + self.validate_pai_trial_conifg(experiment_config) + self.validate_kubeflow_operators(experiment_config) + self.validate_eth0_device(experiment_config) + self.validate_hybrid_platforms(experiment_config) + self.validate_frameworkcontroller_trial_config(experiment_config) + + def validate_tuner_adivosr_assessor(self, experiment_config): + if experiment_config.get('advisor'): + if experiment_config.get('assessor') or experiment_config.get('tuner'): + raise SchemaError('advisor could not be set with assessor or tuner simultaneously!') + self.validate_annotation_content(experiment_config, 'advisor', 'builtinAdvisorName') + else: + if not experiment_config.get('tuner'): + raise SchemaError('Please provide tuner spec!') + self.validate_annotation_content(experiment_config, 'tuner', 'builtinTunerName') + + def validate_search_space_content(self, experiment_config): + '''Validate searchspace content, + if the searchspace file is not json format or its values does not contain _type and _value which must be specified, + it will not be a valid searchspace file''' + try: + search_space_content = json.load(open(experiment_config.get('searchSpacePath'), 'r')) + for value in search_space_content.values(): + if not value.get('_type') or not value.get('_value'): + raise SchemaError('please use _type and _value to specify searchspace!') + except Exception as e: + raise SchemaError('searchspace file is not a valid json format! ' + str(e)) + + def validate_kubeflow_operators(self, experiment_config): + '''Validate whether the kubeflow operators are valid''' + if experiment_config.get('kubeflowConfig'): + if experiment_config.get('kubeflowConfig').get('operator') == 'tf-operator': + if experiment_config.get('trial').get('master') is not None: + raise SchemaError('kubeflow with tf-operator can not set master') + if experiment_config.get('trial').get('worker') is None: + raise SchemaError('kubeflow with tf-operator must set worker') + elif experiment_config.get('kubeflowConfig').get('operator') == 'pytorch-operator': + if experiment_config.get('trial').get('ps') is not None: + raise SchemaError('kubeflow with pytorch-operator can not set ps') + if experiment_config.get('trial').get('master') is None: + raise SchemaError('kubeflow with pytorch-operator must set master') + + if experiment_config.get('kubeflowConfig').get('storage') == 'nfs': + if experiment_config.get('kubeflowConfig').get('nfs') is None: + raise SchemaError('please set nfs configuration!') + elif experiment_config.get('kubeflowConfig').get('storage') == 'azureStorage': + if experiment_config.get('kubeflowConfig').get('azureStorage') is None: + raise SchemaError('please set azureStorage configuration!') + elif experiment_config.get('kubeflowConfig').get('storage') is None: + if experiment_config.get('kubeflowConfig').get('azureStorage'): + raise SchemaError('please set storage type!') + + def validate_annotation_content(self, experiment_config, spec_key, builtin_name): + ''' + Valid whether useAnnotation and searchSpacePath is coexist + spec_key: 'advisor' or 'tuner' + builtin_name: 'builtinAdvisorName' or 'builtinTunerName' + ''' + if experiment_config.get('useAnnotation'): + if experiment_config.get('searchSpacePath'): + raise SchemaError('If you set useAnnotation=true, please leave searchSpacePath empty') + else: + # validate searchSpaceFile + if experiment_config[spec_key].get(builtin_name) == 'NetworkMorphism': + return + if experiment_config[spec_key].get(builtin_name): + if experiment_config.get('searchSpacePath') is None: + raise SchemaError('Please set searchSpacePath!') + self.validate_search_space_content(experiment_config) + + def validate_pai_config_path(self, experiment_config): + '''validate paiConfigPath field''' + if experiment_config.get('trainingServicePlatform') == 'pai': + if experiment_config.get('trial', {}).get('paiConfigPath'): + # validate commands + pai_config = get_yml_content(experiment_config['trial']['paiConfigPath']) + taskRoles_dict = pai_config.get('taskRoles') + if not taskRoles_dict: + raise SchemaError('Please set taskRoles in paiConfigPath config file!') + else: + pai_trial_fields_required_list = ['image', 'paiStorageConfigName', 'command'] + for trial_field in pai_trial_fields_required_list: + if experiment_config['trial'].get(trial_field) is None: + raise SchemaError('Please set {0} in trial configuration,\ + or set additional pai configuration file path in paiConfigPath!'.format(trial_field)) + pai_resource_fields_required_list = ['gpuNum', 'cpuNum', 'memoryMB'] + for required_field in pai_resource_fields_required_list: + if experiment_config['trial'].get(required_field) is None and \ + experiment_config['paiConfig'].get(required_field) is None: + raise SchemaError('Please set {0} in trial or paiConfig configuration,\ + or set additional pai configuration file path in paiConfigPath!'.format(required_field)) + + def validate_pai_trial_conifg(self, experiment_config): + '''validate the trial config in pai platform''' + if experiment_config.get('trainingServicePlatform') in ['pai']: + if experiment_config.get('trial').get('shmMB') and \ + experiment_config['trial']['shmMB'] > experiment_config['trial']['memoryMB']: + raise SchemaError('shmMB should be no more than memoryMB!') + # backward compatibility + warning_information = '{0} is not supported in NNI anymore, please remove the field in config file!\ + please refer https://github.com/microsoft/nni/blob/master/docs/en_US/TrainingService/PaiMode.md#run-an-experiment\ + for the practices of how to get data and output model in trial code' + if experiment_config.get('trial').get('dataDir'): + print_warning(warning_information.format('dataDir')) + if experiment_config.get('trial').get('outputDir'): + print_warning(warning_information.format('outputDir')) + self.validate_pai_config_path(experiment_config) + + def validate_eth0_device(self, experiment_config): + '''validate whether the machine has eth0 device''' + if experiment_config.get('trainingServicePlatform') not in ['local'] \ + and not experiment_config.get('nniManagerIp') \ + and 'eth0' not in netifaces.interfaces(): + raise SchemaError('This machine does not contain eth0 network device, please set nniManagerIp in config file!') + + def validate_hybrid_platforms(self, experiment_config): + required_config_name_map = { + 'remote': 'machineList', + 'aml': 'amlConfig', + 'pai': 'paiConfig' + } + if experiment_config.get('trainingServicePlatform') == 'hybrid': + for platform in experiment_config['hybridConfig']['trainingServicePlatforms']: + config_name = required_config_name_map.get(platform) + if config_name and not experiment_config.get(config_name): + raise SchemaError('Need to set {0} for {1} in hybrid mode!'.format(config_name, platform)) + + def validate_frameworkcontroller_trial_config(self, experiment_config): + if experiment_config.get('trainingServicePlatform') == 'frameworkcontroller': + if not experiment_config.get('trial').get('taskRoles'): + if not experiment_config.get('frameworkcontrollerConfig').get('configPath'): + raise SchemaError("""If no taskRoles are specified a valid custom frameworkcontroller config should + be set using the configPath attribute in frameworkcontrollerConfig!""") + config_content = get_yml_content(experiment_config.get('frameworkcontrollerConfig').get('configPath')) + if not config_content.get('spec').get('taskRoles') or not config_content.get('spec').get('taskRoles'): + raise SchemaError('Invalid frameworkcontroller config! No taskRoles were specified!') + if not config_content.get('spec').get('taskRoles')[0].get('task'): + raise SchemaError('Invalid frameworkcontroller config! No task was specified for taskRole!') + names = [] + for taskRole in config_content.get('spec').get('taskRoles'): + if not "name" in taskRole: + raise SchemaError('Invalid frameworkcontroller config! Name is missing for taskRole!') + names.append(taskRole.get("name")) + if len(names) > len(set(names)): + raise SchemaError('Invalid frameworkcontroller config! Duplicate taskrole names!') + if not config_content.get('metadata').get('name'): + raise SchemaError('Invalid frameworkcontroller config! No experiment name was specified!') + diff --git a/new_impl/cv/third_party/nni_new/tools/nnictl/config_utils.py b/new_impl/cv/third_party/nni_new/tools/nnictl/config_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..78f884e6f08f0883112fe5317e29d4c5735c432a --- /dev/null +++ b/new_impl/cv/third_party/nni_new/tools/nnictl/config_utils.py @@ -0,0 +1,169 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +import sqlite3 +import json_tricks +from .constants import NNI_HOME_DIR +from .common_utils import get_file_lock + +def config_v0_to_v1(config: dict) -> dict: + if 'clusterMetaData' not in config: + return config + elif 'trainingServicePlatform' in config: + import copy + experiment_config = copy.deepcopy(config) + if experiment_config['trainingServicePlatform'] == 'hybrid': + inverse_config = {'hybridConfig': experiment_config['clusterMetaData']['hybrid_config']} + platform_list = inverse_config['hybridConfig']['trainingServicePlatforms'] + for platform in platform_list: + inverse_config.update(_inverse_cluster_metadata(platform, experiment_config['clusterMetaData'])) + experiment_config.update(inverse_config) + else: + inverse_config = _inverse_cluster_metadata(experiment_config['trainingServicePlatform'], experiment_config['clusterMetaData']) + experiment_config.update(inverse_config) + experiment_config.pop('clusterMetaData') + return experiment_config + else: + raise RuntimeError('experiment config key `trainingServicePlatform` not found') + +def _inverse_cluster_metadata(platform: str, metadata_config: list) -> dict: + inverse_config = {} + if platform == 'local': + inverse_config['trial'] = {} + for kv in metadata_config: + if kv['key'] == 'local_config': + inverse_config['localConfig'] = kv['value'] + elif kv['key'] == 'trial_config': + inverse_config['trial'] = kv['value'] + elif platform == 'remote': + for kv in metadata_config: + if kv['key'] == 'machine_list': + inverse_config['machineList'] = kv['value'] + elif kv['key'] == 'trial_config': + inverse_config['trial'] = kv['value'] + elif kv['key'] == 'remote_config': + inverse_config['remoteConfig'] = kv['value'] + elif platform == 'pai': + for kv in metadata_config: + if kv['key'] == 'pai_config': + inverse_config['paiConfig'] = kv['value'] + elif kv['key'] == 'trial_config': + inverse_config['trial'] = kv['value'] + elif platform == 'kubeflow': + for kv in metadata_config: + if kv['key'] == 'kubeflow_config': + inverse_config['kubeflowConfig'] = kv['value'] + elif kv['key'] == 'trial_config': + inverse_config['trial'] = kv['value'] + elif platform == 'frameworkcontroller': + for kv in metadata_config: + if kv['key'] == 'frameworkcontroller_config': + inverse_config['frameworkcontrollerConfig'] = kv['value'] + elif kv['key'] == 'trial_config': + inverse_config['trial'] = kv['value'] + elif platform == 'aml': + for kv in metadata_config: + if kv['key'] == 'aml_config': + inverse_config['amlConfig'] = kv['value'] + elif kv['key'] == 'trial_config': + inverse_config['trial'] = kv['value'] + elif platform == 'adl': + for kv in metadata_config: + if kv['key'] == 'adl_config': + inverse_config['adlConfig'] = kv['value'] + elif kv['key'] == 'trial_config': + inverse_config['trial'] = kv['value'] + else: + raise RuntimeError('training service platform {} not found'.format(platform)) + return inverse_config + +class Config: + '''a util class to load and save config''' + def __init__(self, experiment_id: str, log_dir: str): + self.experiment_id = experiment_id + self.conn = sqlite3.connect(os.path.join(log_dir, experiment_id, 'db', 'nni.sqlite')) + self.refresh_config() + + def refresh_config(self): + '''refresh to get latest config''' + sql = 'select params from ExperimentProfile where id=? order by revision DESC' + args = (self.experiment_id,) + self.config = config_v0_to_v1(json_tricks.loads(self.conn.cursor().execute(sql, args).fetchone()[0])) + + def get_config(self): + '''get a value according to key''' + return self.config + +class Experiments: + '''Maintain experiment list''' + def __init__(self, home_dir=NNI_HOME_DIR): + os.makedirs(home_dir, exist_ok=True) + self.experiment_file = os.path.join(home_dir, '.experiment') + self.lock = get_file_lock(self.experiment_file, stale=2) + with self.lock: + self.experiments = self.read_file() + + def add_experiment(self, expId, port, startTime, platform, experiment_name, endTime='N/A', status='INITIALIZED', + tag=[], pid=None, webuiUrl=[], logDir='', prefixUrl=None): + '''set {key:value} pairs to self.experiment''' + with self.lock: + self.experiments = self.read_file() + self.experiments[expId] = {} + self.experiments[expId]['id'] = expId + self.experiments[expId]['port'] = port + self.experiments[expId]['startTime'] = startTime + self.experiments[expId]['endTime'] = endTime + self.experiments[expId]['status'] = status + self.experiments[expId]['platform'] = platform + self.experiments[expId]['experimentName'] = experiment_name + self.experiments[expId]['tag'] = tag + self.experiments[expId]['pid'] = pid + self.experiments[expId]['webuiUrl'] = webuiUrl + self.experiments[expId]['logDir'] = str(logDir) + self.experiments[expId]['prefixUrl'] = prefixUrl + self.write_file() + + def update_experiment(self, expId, key, value): + '''Update experiment''' + with self.lock: + self.experiments = self.read_file() + if expId not in self.experiments: + return False + if value is None: + self.experiments[expId].pop(key, None) + else: + self.experiments[expId][key] = value + self.write_file() + return True + + def remove_experiment(self, expId): + '''remove an experiment by id''' + with self.lock: + self.experiments = self.read_file() + if expId in self.experiments: + self.experiments.pop(expId) + self.write_file() + + def get_all_experiments(self): + '''return all of experiments''' + return self.experiments + + def write_file(self): + '''save config to local file''' + try: + with open(self.experiment_file, 'w') as file: + json_tricks.dump(self.experiments, file, indent=4) + except IOError as error: + print('Error:', error) + return '' + + def read_file(self): + '''load config from local file''' + if os.path.exists(self.experiment_file): + try: + with open(self.experiment_file, 'r') as file: + return json_tricks.load(file) + except ValueError: + return {} + return {} diff --git a/new_impl/cv/third_party/nni_new/tools/nnictl/constants.py b/new_impl/cv/third_party/nni_new/tools/nnictl/constants.py new file mode 100644 index 0000000000000000000000000000000000000000..120674bc5f3481b6474f85107aa99b81044bdd4f --- /dev/null +++ b/new_impl/cv/third_party/nni_new/tools/nnictl/constants.py @@ -0,0 +1,79 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +from colorama import Fore + +NNI_HOME_DIR = os.path.join(os.path.expanduser('~'), 'nni-experiments') + +ERROR_INFO = 'ERROR: ' +NORMAL_INFO = 'INFO: ' +WARNING_INFO = 'WARNING: ' + +DEFAULT_REST_PORT = 8080 +REST_TIME_OUT = 20 + +EXPERIMENT_SUCCESS_INFO = Fore.GREEN + 'Successfully started experiment!\n' + Fore.RESET + \ + '------------------------------------------------------------------------------------\n' \ + 'The experiment id is %s\n'\ + 'The Web UI urls are: %s\n' \ + '------------------------------------------------------------------------------------\n\n' \ + 'You can use these commands to get more information about the experiment\n' \ + '------------------------------------------------------------------------------------\n' \ + ' commands description\n' \ + '1. nnictl experiment show show the information of experiments\n' \ + '2. nnictl trial ls list all of trial jobs\n' \ + '3. nnictl top monitor the status of running experiments\n' \ + '4. nnictl log stderr show stderr log content\n' \ + '5. nnictl log stdout show stdout log content\n' \ + '6. nnictl stop stop an experiment\n' \ + '7. nnictl trial kill kill a trial job by id\n' \ + '8. nnictl --help get help information about nnictl\n' \ + '------------------------------------------------------------------------------------\n' \ + 'Command reference document https://nni.readthedocs.io/en/latest/Tutorial/Nnictl.html\n' \ + '------------------------------------------------------------------------------------\n' + +LOG_HEADER = '-----------------------------------------------------------------------\n' \ + ' Experiment start time %s\n' \ + '-----------------------------------------------------------------------\n' + +EXPERIMENT_START_FAILED_INFO = 'There is an experiment running in the port %d, please stop it first or set another port!\n' \ + 'You could use \'nnictl stop --port [PORT]\' command to stop an experiment!\nOr you could ' \ + 'use \'nnictl create --config [CONFIG_PATH] --port [PORT]\' to set port!\n' + +EXPERIMENT_INFORMATION_FORMAT = '----------------------------------------------------------------------------------------\n' \ + ' Experiment information\n' \ + '%s\n' \ + '----------------------------------------------------------------------------------------\n' + +EXPERIMENT_DETAIL_FORMAT = 'Id: %s Name: %s Status: %s Port: %s Platform: %s StartTime: %s EndTime: %s\n' + +EXPERIMENT_MONITOR_INFO = 'Id: %s Status: %s Port: %s Platform: %s \n' \ + 'StartTime: %s Duration: %s' + +TRIAL_MONITOR_HEAD = '-------------------------------------------------------------------------------------\n' + \ + '%-15s %-25s %-25s %-15s \n' % ('trialId', 'startTime', 'endTime', 'status') + \ + '-------------------------------------------------------------------------------------' + +TRIAL_MONITOR_CONTENT = '%-15s %-25s %-25s %-15s' + +TRIAL_MONITOR_TAIL = '-------------------------------------------------------------------------------------\n\n\n' + +TUNERS_SUPPORTING_IMPORT_DATA = { + 'TPE', + 'Anneal', + 'GridSearch', + 'MetisTuner', + 'BOHB', + 'SMAC', + 'BatchTuner' +} + +TUNERS_NO_NEED_TO_IMPORT_DATA = { + 'Random', + 'Hyperband' +} + +SCHEMA_TYPE_ERROR = '%s should be %s type!' +SCHEMA_RANGE_ERROR = '%s should be in range of %s!' +SCHEMA_PATH_ERROR = '%s path not exist!' diff --git a/new_impl/cv/third_party/nni_new/tools/nnictl/launcher.py b/new_impl/cv/third_party/nni_new/tools/nnictl/launcher.py new file mode 100644 index 0000000000000000000000000000000000000000..9144bc02678de02739994b8a5c05f7b085c1d9ad --- /dev/null +++ b/new_impl/cv/third_party/nni_new/tools/nnictl/launcher.py @@ -0,0 +1,576 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import json +import os +from pathlib import Path +import sys +import string +import random +import time +import tempfile +import re +from subprocess import Popen, check_call, CalledProcessError, PIPE, STDOUT +from nni.experiment.config import ExperimentConfig, convert +from nni.tools.annotation import expand_annotations, generate_search_space +from nni.tools.package_utils import get_builtin_module_class_name +import nni_node # pylint: disable=import-error +from .launcher_utils import validate_all_content +from .rest_utils import rest_put, rest_post, check_rest_server, check_response +from .url_utils import cluster_metadata_url, experiment_url, get_local_urls, set_prefix_url +from .config_utils import Config, Experiments +from .common_utils import get_yml_content, get_json_content, print_error, print_normal, detect_port, get_user + +from .constants import NNI_HOME_DIR, ERROR_INFO, REST_TIME_OUT, EXPERIMENT_SUCCESS_INFO, LOG_HEADER +from .command_utils import check_output_command, kill_command +from .nnictl_utils import update_experiment + +k8s_training_services = ['kubeflow', 'frameworkcontroller', 'adl'] + +def get_log_path(experiment_id): + '''generate stdout and stderr log path''' + os.makedirs(os.path.join(NNI_HOME_DIR, experiment_id, 'log'), exist_ok=True) + stdout_full_path = os.path.join(NNI_HOME_DIR, experiment_id, 'log', 'nnictl_stdout.log') + stderr_full_path = os.path.join(NNI_HOME_DIR, experiment_id, 'log', 'nnictl_stderr.log') + return stdout_full_path, stderr_full_path + +def print_log_content(config_file_name): + '''print log information''' + stdout_full_path, stderr_full_path = get_log_path(config_file_name) + print_normal(' Stdout:') + print(check_output_command(stdout_full_path)) + print('\n\n') + print_normal(' Stderr:') + print(check_output_command(stderr_full_path)) + +def start_rest_server(port, platform, mode, experiment_id, foreground=False, log_dir=None, log_level=None, url_prefix=None): + '''Run nni manager process''' + if detect_port(port): + print_error('Port %s is used by another process, please reset the port!\n' \ + 'You could use \'nnictl create --help\' to get help information' % port) + exit(1) + + if (platform not in ['local', 'aml']) and detect_port(int(port) + 1): + print_error('%s mode need an additional adjacent port %d, and the port %d is used by another process!\n' \ + 'You could set another port to start experiment!\n' \ + 'You could use \'nnictl create --help\' to get help information' % (platform, (int(port) + 1), (int(port) + 1))) + exit(1) + + print_normal('Starting restful server...') + + entry_dir = nni_node.__path__[0] + if (not entry_dir) or (not os.path.exists(entry_dir)): + print_error('Fail to find nni under python library') + exit(1) + entry_file = os.path.join(entry_dir, 'main.js') + + if sys.platform == 'win32': + node_command = os.path.join(entry_dir, 'node.exe') + else: + node_command = os.path.join(entry_dir, 'node') + cmds = [node_command, '--max-old-space-size=4096', entry_file, '--port', str(port), '--mode', platform, \ + '--experiment_id', experiment_id] + if mode == 'view': + cmds += ['--start_mode', 'resume'] + cmds += ['--readonly', 'true'] + else: + cmds += ['--start_mode', mode] + if log_dir is not None: + cmds += ['--log_dir', log_dir] + if log_level is not None: + cmds += ['--log_level', log_level] + if foreground: + cmds += ['--foreground', 'true'] + if url_prefix: + _validate_prefix_path(url_prefix) + set_prefix_url(url_prefix) + cmds += ['--url_prefix', url_prefix] + + stdout_full_path, stderr_full_path = get_log_path(experiment_id) + with open(stdout_full_path, 'a+') as stdout_file, open(stderr_full_path, 'a+') as stderr_file: + start_time = time.time() + time_now = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(start_time)) + #add time information in the header of log files + log_header = LOG_HEADER % str(time_now) + stdout_file.write(log_header) + stderr_file.write(log_header) + if sys.platform == 'win32': + from subprocess import CREATE_NEW_PROCESS_GROUP + if foreground: + process = Popen(cmds, cwd=entry_dir, stdout=PIPE, stderr=STDOUT, creationflags=CREATE_NEW_PROCESS_GROUP) + else: + process = Popen(cmds, cwd=entry_dir, stdout=stdout_file, stderr=stderr_file, creationflags=CREATE_NEW_PROCESS_GROUP) + else: + if foreground: + process = Popen(cmds, cwd=entry_dir, stdout=PIPE, stderr=PIPE) + else: + process = Popen(cmds, cwd=entry_dir, stdout=stdout_file, stderr=stderr_file) + return process, int(start_time * 1000) + +def set_trial_config(experiment_config, port, config_file_name): + '''set trial configuration''' + request_data = dict() + request_data['trial_config'] = experiment_config['trial'] + response = rest_put(cluster_metadata_url(port), json.dumps(request_data), REST_TIME_OUT) + if check_response(response): + return True + else: + print('Error message is {}'.format(response.text)) + _, stderr_full_path = get_log_path(config_file_name) + if response: + with open(stderr_full_path, 'a+') as fout: + fout.write(json.dumps(json.loads(response.text), indent=4, sort_keys=True, separators=(',', ':'))) + return False + +def set_adl_config(experiment_config, port, config_file_name): + '''set adl configuration''' + adl_config_data = dict() + # hack for supporting v2 config, need refactor + adl_config_data['adl_config'] = {} + response = rest_put(cluster_metadata_url(port), json.dumps(adl_config_data), REST_TIME_OUT) + err_message = None + if not response or not response.status_code == 200: + if response is not None: + err_message = response.text + _, stderr_full_path = get_log_path(config_file_name) + with open(stderr_full_path, 'a+') as fout: + fout.write(json.dumps(json.loads(err_message), indent=4, sort_keys=True, separators=(',', ':'))) + return False, err_message + set_V1_common_config(experiment_config, port, config_file_name) + result, message = setNNIManagerIp(experiment_config, port, config_file_name) + if not result: + return result, message + #set trial_config + return set_trial_config(experiment_config, port, config_file_name), None + +def validate_response(response, config_file_name): + err_message = None + if not response or not response.status_code == 200: + if response is not None: + err_message = response.text + _, stderr_full_path = get_log_path(config_file_name) + with open(stderr_full_path, 'a+') as fout: + fout.write(json.dumps(json.loads(err_message), indent=4, sort_keys=True, separators=(',', ':'))) + print_error('Error:' + err_message) + exit(1) + +# hack to fix v1 version_check and log_collection bug, need refactor +def set_V1_common_config(experiment_config, port, config_file_name): + version_check = True + #debug mode should disable version check + if experiment_config.get('debug') is not None: + version_check = not experiment_config.get('debug') + #validate version check + if experiment_config.get('versionCheck') is not None: + version_check = experiment_config.get('versionCheck') + response = rest_put(cluster_metadata_url(port), json.dumps({'version_check': version_check}), REST_TIME_OUT) + validate_response(response, config_file_name) + if experiment_config.get('logCollection'): + data = json.dumps({'log_collection': experiment_config.get('logCollection')}) + response = rest_put(cluster_metadata_url(port), data, REST_TIME_OUT) + validate_response(response, config_file_name) + +def setNNIManagerIp(experiment_config, port, config_file_name): + '''set nniManagerIp''' + if experiment_config.get('nniManagerIp') is None: + return True, None + ip_config_dict = dict() + ip_config_dict['nni_manager_ip'] = {'nniManagerIp': experiment_config['nniManagerIp']} + response = rest_put(cluster_metadata_url(port), json.dumps(ip_config_dict), REST_TIME_OUT) + err_message = None + if not response or not response.status_code == 200: + if response is not None: + err_message = response.text + _, stderr_full_path = get_log_path(config_file_name) + with open(stderr_full_path, 'a+') as fout: + fout.write(json.dumps(json.loads(err_message), indent=4, sort_keys=True, separators=(',', ':'))) + return False, err_message + return True, None + +def set_kubeflow_config(experiment_config, port, config_file_name): + '''set kubeflow configuration''' + kubeflow_config_data = dict() + kubeflow_config_data['kubeflow_config'] = experiment_config['kubeflowConfig'] + response = rest_put(cluster_metadata_url(port), json.dumps(kubeflow_config_data), REST_TIME_OUT) + err_message = None + if not response or not response.status_code == 200: + if response is not None: + err_message = response.text + _, stderr_full_path = get_log_path(config_file_name) + with open(stderr_full_path, 'a+') as fout: + fout.write(json.dumps(json.loads(err_message), indent=4, sort_keys=True, separators=(',', ':'))) + return False, err_message + set_V1_common_config(experiment_config, port, config_file_name) + result, message = setNNIManagerIp(experiment_config, port, config_file_name) + if not result: + return result, message + #set trial_config + return set_trial_config(experiment_config, port, config_file_name), err_message + +def set_frameworkcontroller_config(experiment_config, port, config_file_name): + '''set kubeflow configuration''' + frameworkcontroller_config_data = dict() + frameworkcontroller_config_data['frameworkcontroller_config'] = experiment_config['frameworkcontrollerConfig'] + response = rest_put(cluster_metadata_url(port), json.dumps(frameworkcontroller_config_data), REST_TIME_OUT) + err_message = None + if not response or not response.status_code == 200: + if response is not None: + err_message = response.text + _, stderr_full_path = get_log_path(config_file_name) + with open(stderr_full_path, 'a+') as fout: + fout.write(json.dumps(json.loads(err_message), indent=4, sort_keys=True, separators=(',', ':'))) + return False, err_message + set_V1_common_config(experiment_config, port, config_file_name) + result, message = setNNIManagerIp(experiment_config, port, config_file_name) + if not result: + return result, message + #set trial_config + return set_trial_config(experiment_config, port, config_file_name), err_message + +def set_shared_storage(experiment_config, port, config_file_name): + if 'sharedStorage' in experiment_config: + data = json.dumps({'shared_storage_config': experiment_config['sharedStorage']}) + response = rest_put(cluster_metadata_url(port), data, REST_TIME_OUT) + err_message = None + if not response or not response.status_code == 200: + if response is not None: + err_message = response.text + _, stderr_full_path = get_log_path(config_file_name) + with open(stderr_full_path, 'a+') as fout: + fout.write(json.dumps(json.loads(err_message), indent=4, sort_keys=True, separators=(',', ':'))) + return False, err_message + return True, None + +def set_experiment_v1(experiment_config, mode, port, config_file_name): + '''Call startExperiment (rest POST /experiment) with yaml file content''' + request_data = dict() + request_data['authorName'] = experiment_config['authorName'] + request_data['experimentName'] = experiment_config['experimentName'] + request_data['trialConcurrency'] = experiment_config['trialConcurrency'] + request_data['maxExecDuration'] = experiment_config['maxExecDuration'] + request_data['maxExperimentDuration'] = str(experiment_config['maxExecDuration']) + 's' + request_data['maxTrialNum'] = experiment_config['maxTrialNum'] + request_data['maxTrialNumber'] = experiment_config['maxTrialNum'] + request_data['searchSpace'] = experiment_config.get('searchSpace') + request_data['trainingServicePlatform'] = experiment_config.get('trainingServicePlatform') + # hack for hotfix, fix config.trainingService undefined error, need refactor + request_data['trainingService'] = {'platform': experiment_config.get('trainingServicePlatform')} + if experiment_config.get('description'): + request_data['description'] = experiment_config['description'] + if experiment_config.get('multiPhase'): + request_data['multiPhase'] = experiment_config.get('multiPhase') + if experiment_config.get('multiThread'): + request_data['multiThread'] = experiment_config.get('multiThread') + if experiment_config.get('nniManagerIp'): + request_data['nniManagerIp'] = experiment_config.get('nniManagerIp') + if experiment_config.get('advisor'): + request_data['advisor'] = experiment_config['advisor'] + if request_data['advisor'].get('gpuNum'): + print_error('gpuNum is deprecated, please use gpuIndices instead.') + if request_data['advisor'].get('gpuIndices') and isinstance(request_data['advisor'].get('gpuIndices'), int): + request_data['advisor']['gpuIndices'] = str(request_data['advisor'].get('gpuIndices')) + else: + request_data['tuner'] = experiment_config['tuner'] + if request_data['tuner'].get('gpuNum'): + print_error('gpuNum is deprecated, please use gpuIndices instead.') + if request_data['tuner'].get('gpuIndices') and isinstance(request_data['tuner'].get('gpuIndices'), int): + request_data['tuner']['gpuIndices'] = str(request_data['tuner'].get('gpuIndices')) + if 'assessor' in experiment_config: + request_data['assessor'] = experiment_config['assessor'] + if request_data['assessor'].get('gpuNum'): + print_error('gpuNum is deprecated, please remove it from your config file.') + #debug mode should disable version check + if experiment_config.get('debug') is not None: + request_data['versionCheck'] = not experiment_config.get('debug') + #validate version check + if experiment_config.get('versionCheck') is not None: + request_data['versionCheck'] = experiment_config.get('versionCheck') + if experiment_config.get('logCollection'): + request_data['logCollection'] = experiment_config.get('logCollection') + request_data['clusterMetaData'] = [] + if experiment_config['trainingServicePlatform'] == 'kubeflow': + request_data['clusterMetaData'].append( + {'key': 'kubeflow_config', 'value': experiment_config['kubeflowConfig']}) + request_data['clusterMetaData'].append( + {'key': 'trial_config', 'value': experiment_config['trial']}) + elif experiment_config['trainingServicePlatform'] == 'frameworkcontroller': + request_data['clusterMetaData'].append( + {'key': 'frameworkcontroller_config', 'value': experiment_config['frameworkcontrollerConfig']}) + request_data['clusterMetaData'].append( + {'key': 'trial_config', 'value': experiment_config['trial']}) + elif experiment_config['trainingServicePlatform'] == 'adl': + request_data['clusterMetaData'].append( + {'key': 'trial_config', 'value': experiment_config['trial']}) + response = rest_post(experiment_url(port), json.dumps(request_data), REST_TIME_OUT, show_error=True) + if check_response(response): + return response + else: + _, stderr_full_path = get_log_path(config_file_name) + if response is not None: + with open(stderr_full_path, 'a+') as fout: + fout.write(json.dumps(json.loads(response.text), indent=4, sort_keys=True, separators=(',', ':'))) + print_error('Setting experiment error, error message is {}'.format(response.text)) + return None + +def set_experiment_v2(experiment_config, mode, port, config_file_name): + '''Call startExperiment (rest POST /experiment) with yaml file content''' + response = rest_post(experiment_url(port), json.dumps(experiment_config), REST_TIME_OUT, show_error=True) + if check_response(response): + return response + else: + _, stderr_full_path = get_log_path(config_file_name) + if response is not None: + with open(stderr_full_path, 'a+') as fout: + fout.write(json.dumps(json.loads(response.text), indent=4, sort_keys=True, separators=(',', ':'))) + print_error('Setting experiment error, error message is {}'.format(response.text)) + return None + +def set_platform_config(platform, experiment_config, port, config_file_name, rest_process): + '''call set_cluster_metadata for specific platform''' + print_normal('Setting {0} config...'.format(platform)) + config_result, err_msg = None, None + if platform == 'adl': + config_result, err_msg = set_adl_config(experiment_config, port, config_file_name) + elif platform == 'kubeflow': + config_result, err_msg = set_kubeflow_config(experiment_config, port, config_file_name) + elif platform == 'frameworkcontroller': + config_result, err_msg = set_frameworkcontroller_config(experiment_config, port, config_file_name) + else: + raise Exception(ERROR_INFO % 'Unsupported platform!') + exit(1) + if config_result: + config_result, err_msg = set_shared_storage(experiment_config, port, config_file_name) + if config_result: + print_normal('Successfully set {0} config!'.format(platform)) + else: + print_error('Failed! Error is: {}'.format(err_msg)) + try: + kill_command(rest_process.pid) + except Exception: + raise Exception(ERROR_INFO % 'Rest server stopped!') + exit(1) + +def launch_experiment(args, experiment_config, mode, experiment_id, config_version): + '''follow steps to start rest server and start experiment''' + # check packages for tuner + package_name, module_name = None, None + if experiment_config.get('tuner') and experiment_config['tuner'].get('builtinTunerName'): + package_name = experiment_config['tuner']['builtinTunerName'] + module_name, _ = get_builtin_module_class_name('tuners', package_name) + elif experiment_config.get('advisor') and experiment_config['advisor'].get('builtinAdvisorName'): + package_name = experiment_config['advisor']['builtinAdvisorName'] + module_name, _ = get_builtin_module_class_name('advisors', package_name) + if package_name and module_name: + try: + stdout_full_path, stderr_full_path = get_log_path(experiment_id) + with open(stdout_full_path, 'a+') as stdout_file, open(stderr_full_path, 'a+') as stderr_file: + check_call([sys.executable, '-c', 'import %s'%(module_name)], stdout=stdout_file, stderr=stderr_file) + except CalledProcessError: + print_error('some errors happen when import package %s.' %(package_name)) + print_log_content(experiment_id) + if package_name in ['SMAC', 'BOHB', 'PPOTuner']: + print_error(f'The dependencies for {package_name} can be installed through pip install nni[{package_name}]') + raise + if config_version == 1: + log_dir = experiment_config['logDir'] if experiment_config.get('logDir') else NNI_HOME_DIR + else: + log_dir = experiment_config['experimentWorkingDirectory'] if experiment_config.get('experimentWorkingDirectory') else NNI_HOME_DIR + log_level = experiment_config['logLevel'] if experiment_config.get('logLevel') else 'info' + #view experiment mode do not need debug function, when view an experiment, there will be no new logs created + foreground = False + if mode != 'view': + foreground = args.foreground + if log_level not in ['trace', 'debug'] and (args.debug or experiment_config.get('debug') is True): + log_level = 'debug' + # start rest server + if config_version == 1: + platform = experiment_config['trainingServicePlatform'] + elif isinstance(experiment_config['trainingService'], list): + platform = 'hybrid' + else: + platform = experiment_config['trainingService']['platform'] + + rest_process, start_time = start_rest_server(args.port, platform, \ + mode, experiment_id, foreground, log_dir, log_level, args.url_prefix) + # save experiment information + Experiments().add_experiment(experiment_id, args.port, start_time, + platform, + experiment_config.get('experimentName', 'N/A') + , pid=rest_process.pid, logDir=log_dir, prefixUrl=args.url_prefix) + # Deal with annotation + if experiment_config.get('useAnnotation'): + path = os.path.join(tempfile.gettempdir(), get_user(), 'nni', 'annotation') + if not os.path.isdir(path): + os.makedirs(path) + path = tempfile.mkdtemp(dir=path) + if config_version == 1: + nas_mode = experiment_config['trial'].get('nasMode', 'classic_mode') + code_dir = expand_annotations(experiment_config['trial']['codeDir'], path, nas_mode=nas_mode) + experiment_config['trial']['codeDir'] = code_dir + else: + code_dir = expand_annotations(experiment_config['trialCodeDirectory'], path) + experiment_config['trialCodeDirectory'] = code_dir + search_space = generate_search_space(code_dir) + experiment_config['searchSpace'] = search_space + assert search_space, ERROR_INFO % 'Generated search space is empty' + elif config_version == 1: + if experiment_config.get('searchSpacePath'): + search_space = get_json_content(experiment_config.get('searchSpacePath')) + experiment_config['searchSpace'] = search_space + else: + experiment_config['searchSpace'] = '' + + # check rest server + running, _ = check_rest_server(args.port) + if running: + print_normal('Successfully started Restful server!') + else: + print_error('Restful server start failed!') + print_log_content(experiment_id) + try: + kill_command(rest_process.pid) + except Exception: + raise Exception(ERROR_INFO % 'Rest server stopped!') + exit(1) + if config_version == 1 and mode != 'view': + # set platform configuration + set_platform_config(experiment_config['trainingServicePlatform'], experiment_config, args.port,\ + experiment_id, rest_process) + + # start a new experiment + print_normal('Starting experiment...') + # set debug configuration + if mode != 'view' and experiment_config.get('debug') is None: + experiment_config['debug'] = args.debug + if config_version == 1: + response = set_experiment_v1(experiment_config, mode, args.port, experiment_id) + else: + response = set_experiment_v2(experiment_config, mode, args.port, experiment_id) + if response: + if experiment_id is None: + experiment_id = json.loads(response.text).get('experiment_id') + else: + print_error('Start experiment failed!') + print_log_content(experiment_id) + try: + kill_command(rest_process.pid) + except Exception: + raise Exception(ERROR_INFO % 'Restful server stopped!') + exit(1) + url_prefix_format = '' if args.url_prefix is None else '/{0}'.format(args.url_prefix) + if experiment_config.get('nniManagerIp'): + web_ui_url_list = ['http://{0}:{1}{2}'.format(experiment_config['nniManagerIp'], str(args.port), url_prefix_format)] + else: + web_ui_url_list = get_local_urls(args.port, url_prefix_format) + Experiments().update_experiment(experiment_id, 'webuiUrl', web_ui_url_list) + + print_normal(EXPERIMENT_SUCCESS_INFO % (experiment_id, ' '.join(web_ui_url_list))) + if mode != 'view' and args.foreground: + try: + while True: + log_content = rest_process.stdout.readline().strip().decode('utf-8') + print(log_content) + except KeyboardInterrupt: + kill_command(rest_process.pid) + print_normal('Stopping experiment...') + +def _validate_v1(config, path): + try: + validate_all_content(config, path) + except Exception as e: + print_error(f'Config V1 validation failed: {repr(e)}') + exit(1) + +def _validate_v2(config, path): + base_path = Path(path).parent + try: + conf = ExperimentConfig(_base_path=base_path, **config) + return conf.json() + except Exception as e: + print_error(f'Config V2 validation failed: {repr(e)}') + +def _validate_prefix_path(path): + assert not path.startswith('/'), 'URL prefix should not start with "/".' + parts = path.split('/') + valid = all(re.match('^[A-Za-z0-9_-]*$', part) for part in parts) + assert valid, 'URL prefix should only contain letter, number, underscore, and hyphen.' + +def create_experiment(args): + '''start a new experiment''' + experiment_id = ''.join(random.sample(string.ascii_letters + string.digits, 8)) + config_path = os.path.abspath(args.config) + if not os.path.exists(config_path): + print_error('Please set correct config path!') + exit(1) + config_yml = get_yml_content(config_path) + + if 'trainingServicePlatform' in config_yml: + _validate_v1(config_yml, config_path) + platform = config_yml['trainingServicePlatform'] + if platform in k8s_training_services: + schema = 1 + config_v1 = config_yml + else: + schema = 2 + config_v2 = convert.to_v2(config_yml).json() + else: + config_v2 = _validate_v2(config_yml, config_path) + schema = 2 + + try: + if schema == 1: + launch_experiment(args, config_v1, 'new', experiment_id, 1) + else: + launch_experiment(args, config_v2, 'new', experiment_id, 2) + except Exception as exception: + restServerPid = Experiments().get_all_experiments().get(experiment_id, {}).get('pid') + if restServerPid: + kill_command(restServerPid) + print_error(exception) + exit(1) + +def manage_stopped_experiment(args, mode): + '''view a stopped experiment''' + update_experiment() + experiments_config = Experiments() + experiments_dict = experiments_config.get_all_experiments() + experiment_id = None + #find the latest stopped experiment + if not args.id: + print_error('Please set experiment id! \nYou could use \'nnictl {0} id\' to {0} a stopped experiment!\n' \ + 'You could use \'nnictl experiment list --all\' to show all experiments!'.format(mode)) + exit(1) + else: + if experiments_dict.get(args.id) is None: + print_error('Id %s not exist!' % args.id) + exit(1) + if experiments_dict[args.id]['status'] != 'STOPPED': + print_error('Only stopped experiments can be {0}ed!'.format(mode)) + exit(1) + experiment_id = args.id + print_normal('{0} experiment {1}...'.format(mode, experiment_id)) + experiment_config = Config(experiment_id, experiments_dict[args.id]['logDir']).get_config() + experiments_config.update_experiment(args.id, 'port', args.port) + args.url_prefix = experiments_dict[args.id]['prefixUrl'] + assert 'trainingService' in experiment_config or 'trainingServicePlatform' in experiment_config + try: + if 'trainingServicePlatform' in experiment_config: + experiment_config['logDir'] = experiments_dict[args.id]['logDir'] + launch_experiment(args, experiment_config, mode, experiment_id, 1) + else: + experiment_config['experimentWorkingDirectory'] = experiments_dict[args.id]['logDir'] + launch_experiment(args, experiment_config, mode, experiment_id, 2) + except Exception as exception: + restServerPid = Experiments().get_all_experiments().get(experiment_id, {}).get('pid') + if restServerPid: + kill_command(restServerPid) + print_error(exception) + exit(1) + +def view_experiment(args): + '''view a stopped experiment''' + manage_stopped_experiment(args, 'view') + +def resume_experiment(args): + '''resume an experiment''' + manage_stopped_experiment(args, 'resume') diff --git a/new_impl/cv/third_party/nni_new/tools/nnictl/launcher_utils.py b/new_impl/cv/third_party/nni_new/tools/nnictl/launcher_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..36033671586f9afdd242d9267a9c8eb076b8b604 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/tools/nnictl/launcher_utils.py @@ -0,0 +1,128 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +from schema import SchemaError +from .config_schema import NNIConfigSchema +from .common_utils import print_normal + +def expand_path(experiment_config, key): + '''Change '~' to user home directory''' + if experiment_config.get(key): + experiment_config[key] = os.path.expanduser(experiment_config[key]) + +def parse_relative_path(root_path, experiment_config, key): + '''Change relative path to absolute path''' + if experiment_config.get(key) and not os.path.isabs(experiment_config.get(key)): + absolute_path = os.path.join(root_path, experiment_config.get(key)) + print_normal('expand %s: %s to %s ' % (key, experiment_config[key], absolute_path)) + experiment_config[key] = absolute_path + +def parse_time(time): + '''Change the time to seconds''' + unit = time[-1] + if unit not in ['s', 'm', 'h', 'd']: + raise SchemaError('the unit of time could only from {s, m, h, d}') + time = time[:-1] + if not time.isdigit(): + raise SchemaError('time format error!') + parse_dict = {'s':1, 'm':60, 'h':3600, 'd':86400} + return int(time) * parse_dict[unit] + +def parse_path(experiment_config, config_path): + '''Parse path in config file''' + expand_path(experiment_config, 'searchSpacePath') + if experiment_config.get('logDir'): + expand_path(experiment_config, 'logDir') + if experiment_config.get('trial'): + expand_path(experiment_config['trial'], 'codeDir') + if experiment_config['trial'].get('authFile'): + expand_path(experiment_config['trial'], 'authFile') + if experiment_config['trial'].get('ps'): + if experiment_config['trial']['ps'].get('privateRegistryAuthPath'): + expand_path(experiment_config['trial']['ps'], 'privateRegistryAuthPath') + if experiment_config['trial'].get('master'): + if experiment_config['trial']['master'].get('privateRegistryAuthPath'): + expand_path(experiment_config['trial']['master'], 'privateRegistryAuthPath') + if experiment_config['trial'].get('worker'): + if experiment_config['trial']['worker'].get('privateRegistryAuthPath'): + expand_path(experiment_config['trial']['worker'], 'privateRegistryAuthPath') + if experiment_config['trial'].get('taskRoles'): + for index in range(len(experiment_config['trial']['taskRoles'])): + if experiment_config['trial']['taskRoles'][index].get('privateRegistryAuthPath'): + expand_path(experiment_config['trial']['taskRoles'][index], 'privateRegistryAuthPath') + if experiment_config.get('tuner'): + expand_path(experiment_config['tuner'], 'codeDir') + if experiment_config.get('assessor'): + expand_path(experiment_config['assessor'], 'codeDir') + if experiment_config.get('advisor'): + expand_path(experiment_config['advisor'], 'codeDir') + if experiment_config.get('machineList'): + for index in range(len(experiment_config['machineList'])): + expand_path(experiment_config['machineList'][index], 'sshKeyPath') + if experiment_config['trial'].get('paiConfigPath'): + expand_path(experiment_config['trial'], 'paiConfigPath') + + # If users use relative path, convert it to absolute path. + root_path = os.path.dirname(config_path) + if experiment_config.get('searchSpacePath'): + parse_relative_path(root_path, experiment_config, 'searchSpacePath') + if experiment_config.get('logDir'): + parse_relative_path(root_path, experiment_config, 'logDir') + if experiment_config.get('trial'): + # In AdaptDL mode, 'codeDir' shouldn't be parsed because it points to the path in the container. + if experiment_config.get('trainingServicePlatform') != 'adl': + parse_relative_path(root_path, experiment_config['trial'], 'codeDir') + if experiment_config['trial'].get('authFile'): + parse_relative_path(root_path, experiment_config['trial'], 'authFile') + if experiment_config['trial'].get('ps'): + if experiment_config['trial']['ps'].get('privateRegistryAuthPath'): + parse_relative_path(root_path, experiment_config['trial']['ps'], 'privateRegistryAuthPath') + if experiment_config['trial'].get('master'): + if experiment_config['trial']['master'].get('privateRegistryAuthPath'): + parse_relative_path(root_path, experiment_config['trial']['master'], 'privateRegistryAuthPath') + if experiment_config['trial'].get('worker'): + if experiment_config['trial']['worker'].get('privateRegistryAuthPath'): + parse_relative_path(root_path, experiment_config['trial']['worker'], 'privateRegistryAuthPath') + if experiment_config['trial'].get('taskRoles'): + for index in range(len(experiment_config['trial']['taskRoles'])): + if experiment_config['trial']['taskRoles'][index].get('privateRegistryAuthPath'): + parse_relative_path(root_path, experiment_config['trial']['taskRoles'][index], 'privateRegistryAuthPath') + if experiment_config.get('tuner'): + parse_relative_path(root_path, experiment_config['tuner'], 'codeDir') + if experiment_config.get('assessor'): + parse_relative_path(root_path, experiment_config['assessor'], 'codeDir') + if experiment_config.get('advisor'): + parse_relative_path(root_path, experiment_config['advisor'], 'codeDir') + if experiment_config.get('machineList'): + for index in range(len(experiment_config['machineList'])): + parse_relative_path(root_path, experiment_config['machineList'][index], 'sshKeyPath') + if experiment_config['trial'].get('paiConfigPath'): + parse_relative_path(root_path, experiment_config['trial'], 'paiConfigPath') + + # For frameworkcontroller a custom configuration path may be specified + if experiment_config.get('frameworkcontrollerConfig'): + if experiment_config['frameworkcontrollerConfig'].get('configPath'): + parse_relative_path(root_path, experiment_config['frameworkcontrollerConfig'], 'configPath') + +def set_default_values(experiment_config): + if experiment_config.get('maxExecDuration') is None: + experiment_config['maxExecDuration'] = '999d' + if experiment_config.get('maxTrialNum') is None: + experiment_config['maxTrialNum'] = 99999 + if experiment_config['trainingServicePlatform'] == 'remote' or \ + experiment_config['trainingServicePlatform'] == 'hybrid' and \ + 'remote' in experiment_config['hybridConfig']['trainingServicePlatforms']: + for index in range(len(experiment_config['machineList'])): + if experiment_config['machineList'][index].get('port') is None: + experiment_config['machineList'][index]['port'] = 22 + +def validate_all_content(experiment_config, config_path): + '''Validate whether experiment_config is valid''' + parse_path(experiment_config, config_path) + set_default_values(experiment_config) + + NNIConfigSchema().validate(experiment_config) + + if 'maxExecDuration' in experiment_config: + experiment_config['maxExecDuration'] = parse_time(experiment_config['maxExecDuration']) diff --git a/new_impl/cv/third_party/nni_new/tools/nnictl/nnictl.py b/new_impl/cv/third_party/nni_new/tools/nnictl/nnictl.py new file mode 100644 index 0000000000000000000000000000000000000000..a5f02c8c9e360a1757ac2008212908f47b3263b8 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/tools/nnictl/nnictl.py @@ -0,0 +1,281 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import argparse +import logging +import os +import pkg_resources +from colorama import init +from .common_utils import print_error +from .launcher import create_experiment, resume_experiment, view_experiment +from .updater import update_searchspace, update_concurrency, update_duration, update_trialnum, import_data +from .nnictl_utils import stop_experiment, trial_ls, trial_kill, list_experiment, experiment_status,\ + log_trial, experiment_clean, platform_clean, experiment_list, \ + monitor_experiment, export_trials_data, trial_codegen, webui_url, \ + get_config, log_stdout, log_stderr, search_space_auto_gen, webui_nas, \ + save_experiment, load_experiment +from .algo_management import algo_reg, algo_unreg, algo_show, algo_list +from .constants import DEFAULT_REST_PORT +from .import ts_management + +init(autoreset=True) + +if os.environ.get('COVERAGE_PROCESS_START'): + import coverage + coverage.process_startup() + +def nni_info(*args): + if args[0].version: + try: + print(pkg_resources.get_distribution('nni').version) + except pkg_resources.ResolutionError: + print_error('Get version failed, please use `pip3 list | grep nni` to check nni version!') + else: + print('please run "nnictl {positional argument} --help" to see nnictl guidance') + +def parse_args(): + logging.getLogger().setLevel(logging.ERROR) + + '''Definite the arguments users need to follow and input''' + parser = argparse.ArgumentParser(prog='nnictl', description='use nnictl command to control nni experiments') + parser.add_argument('--version', '-v', action='store_true') + parser.set_defaults(func=nni_info) + + # create subparsers for args with sub values + subparsers = parser.add_subparsers() + + # parse the command of auto generating search space + parser_start = subparsers.add_parser('ss_gen', help='automatically generate search space file from trial code') + parser_start.add_argument('--trial_command', '-t', required=True, dest='trial_command', help='the command for running trial code') + parser_start.add_argument('--trial_dir', '-d', default='./', dest='trial_dir', help='the directory for running the command') + parser_start.add_argument('--file', '-f', default='nni_auto_gen_search_space.json', dest='file', help='the path of search space file') + parser_start.set_defaults(func=search_space_auto_gen) + + # parse start command + parser_start = subparsers.add_parser('create', help='create a new experiment') + parser_start.add_argument('--config', '-c', required=True, dest='config', help='the path of yaml config file') + parser_start.add_argument('--port', '-p', default=DEFAULT_REST_PORT, dest='port', type=int, help='the port of restful server') + parser_start.add_argument('--debug', '-d', action='store_true', help=' set debug mode') + parser_start.add_argument('--url_prefix', '-u', dest='url_prefix', help=' set prefix url') + parser_start.add_argument('--foreground', '-f', action='store_true', help=' set foreground mode, print log content to terminal') + parser_start.set_defaults(func=create_experiment) + + # parse resume command + parser_resume = subparsers.add_parser('resume', help='resume a new experiment') + parser_resume.add_argument('id', nargs='?', help='The id of the experiment you want to resume') + parser_resume.add_argument('--port', '-p', default=DEFAULT_REST_PORT, dest='port', type=int, help='the port of restful server') + parser_resume.add_argument('--debug', '-d', action='store_true', help=' set debug mode') + parser_resume.add_argument('--foreground', '-f', action='store_true', help=' set foreground mode, print log content to terminal') + parser_resume.set_defaults(func=resume_experiment) + + # parse view command + parser_view = subparsers.add_parser('view', help='view a stopped experiment') + parser_view.add_argument('id', nargs='?', help='The id of the experiment you want to view') + parser_view.add_argument('--port', '-p', default=DEFAULT_REST_PORT, dest='port', type=int, help='the port of restful server') + parser_view.set_defaults(func=view_experiment) + + # parse update command + parser_updater = subparsers.add_parser('update', help='update the experiment') + #add subparsers for parser_updater + parser_updater_subparsers = parser_updater.add_subparsers() + parser_updater_searchspace = parser_updater_subparsers.add_parser('searchspace', help='update searchspace') + parser_updater_searchspace.add_argument('id', nargs='?', help='the id of experiment') + parser_updater_searchspace.add_argument('--filename', '-f', required=True) + parser_updater_searchspace.set_defaults(func=update_searchspace) + parser_updater_concurrency = parser_updater_subparsers.add_parser('concurrency', help='update concurrency') + parser_updater_concurrency.add_argument('id', nargs='?', help='the id of experiment') + parser_updater_concurrency.add_argument('--value', '-v', required=True) + parser_updater_concurrency.set_defaults(func=update_concurrency) + parser_updater_duration = parser_updater_subparsers.add_parser('duration', help='update duration') + parser_updater_duration.add_argument('id', nargs='?', help='the id of experiment') + parser_updater_duration.add_argument('--value', '-v', required=True, help='the unit of time should in {\'s\', \'m\', \'h\', \'d\'}') + parser_updater_duration.set_defaults(func=update_duration) + parser_updater_trialnum = parser_updater_subparsers.add_parser('trialnum', help='update maxtrialnum') + parser_updater_trialnum.add_argument('id', nargs='?', help='the id of experiment') + parser_updater_trialnum.add_argument('--value', '-v', required=True) + parser_updater_trialnum.set_defaults(func=update_trialnum) + + #parse stop command + parser_stop = subparsers.add_parser('stop', help='stop the experiment') + parser_stop.add_argument('id', nargs='?', help='the id of experiment, use \'all\' to stop all running experiments') + parser_stop.add_argument('--port', '-p', dest='port', type=int, help='the port of restful server') + parser_stop.add_argument('--all', '-a', action='store_true', help='stop all of experiments') + parser_stop.set_defaults(func=stop_experiment) + + #parse trial command + parser_trial = subparsers.add_parser('trial', help='get trial information') + #add subparsers for parser_trial + parser_trial_subparsers = parser_trial.add_subparsers() + parser_trial_ls = parser_trial_subparsers.add_parser('ls', help='list trial jobs') + parser_trial_ls.add_argument('id', nargs='?', help='the id of experiment') + parser_trial_ls.add_argument('--head', type=int, help='list the highest experiments on the default metric') + parser_trial_ls.add_argument('--tail', type=int, help='list the lowest experiments on the default metric') + parser_trial_ls.set_defaults(func=trial_ls) + parser_trial_kill = parser_trial_subparsers.add_parser('kill', help='kill trial jobs') + parser_trial_kill.add_argument('id', nargs='?', help='the id of experiment') + parser_trial_kill.add_argument('--trial_id', '-T', required=True, dest='trial_id', help='the id of trial to be killed') + parser_trial_kill.set_defaults(func=trial_kill) + parser_trial_codegen = parser_trial_subparsers.add_parser('codegen', help='generate trial code for a specific trial') + parser_trial_codegen.add_argument('id', nargs='?', help='the id of experiment') + parser_trial_codegen.add_argument('--trial_id', '-T', required=True, dest='trial_id', help='the id of trial to do code generation') + parser_trial_codegen.set_defaults(func=trial_codegen) + + #parse experiment command + parser_experiment = subparsers.add_parser('experiment', help='get experiment information') + #add subparsers for parser_experiment + parser_experiment_subparsers = parser_experiment.add_subparsers() + parser_experiment_show = parser_experiment_subparsers.add_parser('show', help='show the information of experiment') + parser_experiment_show.add_argument('id', nargs='?', help='the id of experiment') + parser_experiment_show.set_defaults(func=list_experiment) + parser_experiment_status = parser_experiment_subparsers.add_parser('status', help='show the status of experiment') + parser_experiment_status.add_argument('id', nargs='?', help='the id of experiment') + parser_experiment_status.set_defaults(func=experiment_status) + parser_experiment_list = parser_experiment_subparsers.add_parser('list', help='list all of running experiment ids') + parser_experiment_list.add_argument('--all', action='store_true', default=False, help='list all of experiments') + parser_experiment_list.set_defaults(func=experiment_list) + parser_experiment_clean = parser_experiment_subparsers.add_parser('delete', help='clean up the experiment data') + parser_experiment_clean.add_argument('id', nargs='?', help='the id of experiment') + parser_experiment_clean.add_argument('--all', action='store_true', default=False, help='delete all of experiments') + parser_experiment_clean.set_defaults(func=experiment_clean) + #import tuning data + parser_import_data = parser_experiment_subparsers.add_parser('import', help='import additional data') + parser_import_data.add_argument('id', nargs='?', help='the id of experiment') + parser_import_data.add_argument('--filename', '-f', required=True) + parser_import_data.set_defaults(func=import_data) + #export trial data + parser_trial_export = parser_experiment_subparsers.add_parser('export', help='export trial job results to csv or json') + parser_trial_export.add_argument('id', nargs='?', help='the id of experiment') + parser_trial_export.add_argument('--type', '-t', choices=['json', 'csv'], required=True, dest='type', help='target file type') + parser_trial_export.add_argument('--filename', '-f', required=True, dest='path', help='target file path') + parser_trial_export.add_argument('--intermediate', '-i', action='store_true', + default=False, help='are intermediate results included') + parser_trial_export.set_defaults(func=export_trials_data) + #save an NNI experiment + parser_save_experiment = parser_experiment_subparsers.add_parser('save', help='save an experiment') + parser_save_experiment.add_argument('id', nargs='?', help='the id of experiment') + parser_save_experiment.add_argument('--path', '-p', required=False, help='the folder path to store nni experiment data, \ + default current working directory') + parser_save_experiment.add_argument('--saveCodeDir', '-s', action='store_true', default=False, help='save codeDir data \ + of the experiment') + parser_save_experiment.set_defaults(func=save_experiment) + #load an NNI experiment + parser_load_experiment = parser_experiment_subparsers.add_parser('load', help='load an experiment') + parser_load_experiment.add_argument('--path', '-p', required=True, help='the path of nni package file') + parser_load_experiment.add_argument('--codeDir', '-c', required=True, help='the path of codeDir for loaded experiment, \ + this path will also put the code in the loaded experiment package') + parser_load_experiment.add_argument('--logDir', '-l', required=False, help='the path of logDir for loaded experiment') + parser_load_experiment.add_argument('--searchSpacePath', '-s', required=False, help='the path of search space file for \ + loaded experiment, this path contains file name. Default in $codeDir/search_space.json') + parser_load_experiment.set_defaults(func=load_experiment) + + #parse platform command + parser_platform = subparsers.add_parser('platform', help='get platform information') + #add subparsers for parser_platform + parser_platform_subparsers = parser_platform.add_subparsers() + parser_platform_clean = parser_platform_subparsers.add_parser('clean', help='clean up the platform data') + parser_platform_clean.add_argument('--config', '-c', required=True, dest='config', help='the path of yaml config file') + parser_platform_clean.set_defaults(func=platform_clean) + + #TODO:finish webui function + #parse board command + parser_webui = subparsers.add_parser('webui', help='get web ui information') + #add subparsers for parser_board + parser_webui_subparsers = parser_webui.add_subparsers() + parser_webui_url = parser_webui_subparsers.add_parser('url', help='show the url of web ui') + parser_webui_url.add_argument('id', nargs='?', help='the id of experiment') + parser_webui_url.set_defaults(func=webui_url) + parser_webui_nas = parser_webui_subparsers.add_parser('nas', help='show nas ui') + parser_webui_nas.add_argument('--port', default=6060, type=int, help='port of nas ui') + parser_webui_nas.add_argument('--logdir', default='.', type=str, help='the logdir where nas ui will read data') + parser_webui_nas.set_defaults(func=webui_nas) + + #parse config command + parser_config = subparsers.add_parser('config', help='get config information') + parser_config_subparsers = parser_config.add_subparsers() + parser_config_show = parser_config_subparsers.add_parser('show', help='show the information of config') + parser_config_show.add_argument('id', nargs='?', help='the id of experiment') + parser_config_show.set_defaults(func=get_config) + + #parse log command + parser_log = subparsers.add_parser('log', help='get log information') + # add subparsers for parser_log + parser_log_subparsers = parser_log.add_subparsers() + parser_log_stdout = parser_log_subparsers.add_parser('stdout', help='get stdout information') + parser_log_stdout.add_argument('id', nargs='?', help='the id of experiment') + parser_log_stdout.add_argument('--tail', '-T', dest='tail', type=int, help='get tail -100 content of stdout') + parser_log_stdout.add_argument('--head', '-H', dest='head', type=int, help='get head -100 content of stdout') + parser_log_stdout.add_argument('--path', action='store_true', default=False, help='get the path of stdout file') + parser_log_stdout.set_defaults(func=log_stdout) + parser_log_stderr = parser_log_subparsers.add_parser('stderr', help='get stderr information') + parser_log_stderr.add_argument('id', nargs='?', help='the id of experiment') + parser_log_stderr.add_argument('--tail', '-T', dest='tail', type=int, help='get tail -100 content of stderr') + parser_log_stderr.add_argument('--head', '-H', dest='head', type=int, help='get head -100 content of stderr') + parser_log_stderr.add_argument('--path', action='store_true', default=False, help='get the path of stderr file') + parser_log_stderr.set_defaults(func=log_stderr) + parser_log_trial = parser_log_subparsers.add_parser('trial', help='get trial log path') + parser_log_trial.add_argument('id', nargs='?', help='the id of experiment') + parser_log_trial.add_argument('--trial_id', '-T', dest='trial_id', help='find trial log path by id') + parser_log_trial.set_defaults(func=log_trial) + + #parse algo command + parser_algo = subparsers.add_parser('algo', help='control nni builtin tuner, assessor and advisor algorithms') + # add subparsers for parser_algo + parser_algo_subparsers = parser_algo.add_subparsers() + parser_algo_reg = parser_algo_subparsers.add_parser( + 'register', + aliases=('reg',), + help='''register algorithms as nni builtin algorithm, for example: + nnictl reg --meta_path + where is the path to a meta data in yml format, + reference the nni document and examples/tuners/customized_tuner example + for the format of the yml file.''' + ) + parser_algo_reg.add_argument('--meta_path', '-m', dest='meta_path', help='path to the meta file', required=True) + parser_algo_reg.set_defaults(func=algo_reg) + + parser_algo_unreg = parser_algo_subparsers.add_parser('unregister', aliases=('unreg',), help='unregister algorithm') + parser_algo_unreg.add_argument('name', nargs=1, help='builtin name of the algorithm') + parser_algo_unreg.set_defaults(func=algo_unreg) + + parser_algo_show = parser_algo_subparsers.add_parser('show', help='show the information of algorithm') + parser_algo_show.add_argument('name', nargs=1, help='builtin name of the algorithm') + parser_algo_show.set_defaults(func=algo_show) + + parser_algo_list = parser_algo_subparsers.add_parser('list', help='list registered algorithms') + parser_algo_list.set_defaults(func=algo_list) + + #parse trainingservice command + parser_ts = subparsers.add_parser('trainingservice', help='control training service') + # add subparsers for parser_ts + parser_ts_subparsers = parser_ts.add_subparsers() + + parser_ts_reg = parser_ts_subparsers.add_parser('register', help='register training service') + parser_ts_reg.add_argument('--package', dest='package', help='package name', required=True) + parser_ts_reg.set_defaults(func=ts_management.register) + + parser_ts_unreg = parser_ts_subparsers.add_parser('unregister', help='unregister training service') + parser_ts_unreg.add_argument('--package', dest='package', help='package name', required=True) + parser_ts_unreg.set_defaults(func=ts_management.unregister) + + parser_ts_list = parser_ts_subparsers.add_parser('list', help='list custom training services') + parser_ts_list.set_defaults(func=ts_management.list_services) + + # To show message that nnictl package command is replaced by nnictl algo, to be remove in the future release. + def show_messsage_for_nnictl_package(args): + print_error('nnictl package command is replaced by nnictl algo, please run nnictl algo -h to show the usage') + + parser_package_subparsers = subparsers.add_parser('package', help='this argument is replaced by algo', prefix_chars='\n') + parser_package_subparsers.add_argument('args', nargs=argparse.REMAINDER) + parser_package_subparsers.set_defaults(func=show_messsage_for_nnictl_package) + + #parse top command + parser_top = subparsers.add_parser('top', help='monitor the experiment') + parser_top.add_argument('--time', '-t', dest='time', type=int, default=3, help='the time interval to update the experiment status, ' \ + 'the unit is second') + parser_top.set_defaults(func=monitor_experiment) + + args = parser.parse_args() + args.func(args) + +if __name__ == '__main__': + parse_args() diff --git a/new_impl/cv/third_party/nni_new/tools/nnictl/nnictl_utils.py b/new_impl/cv/third_party/nni_new/tools/nnictl/nnictl_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7ef0fe7dd7fc35609820fc6d601a270ba1a9554a --- /dev/null +++ b/new_impl/cv/third_party/nni_new/tools/nnictl/nnictl_utils.py @@ -0,0 +1,964 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import csv +import os +import sys +import json +import time +import shutil +import subprocess +from functools import cmp_to_key +import traceback +from datetime import datetime, timezone +from subprocess import Popen +from nni.tools.annotation import expand_annotations +import nni_node # pylint: disable=import-error +from .rest_utils import rest_get, rest_delete, check_rest_server_quick, check_response +from .url_utils import trial_jobs_url, experiment_url, trial_job_id_url, export_data_url, metric_data_url +from .config_utils import Config, Experiments +from .constants import NNI_HOME_DIR, EXPERIMENT_INFORMATION_FORMAT, EXPERIMENT_DETAIL_FORMAT, EXPERIMENT_MONITOR_INFO, \ + TRIAL_MONITOR_HEAD, TRIAL_MONITOR_CONTENT, TRIAL_MONITOR_TAIL, REST_TIME_OUT +from .common_utils import print_normal, print_error, print_warning, detect_process, get_yml_content, generate_temp_dir +from .common_utils import print_green +from .command_utils import check_output_command, kill_command +from .ssh_utils import create_ssh_sftp_client, remove_remote_directory + +def get_experiment_time(port): + '''get the startTime and endTime of an experiment''' + response = rest_get(experiment_url(port), REST_TIME_OUT) + if response and check_response(response): + content = json.loads(response.text) + return content.get('startTime'), content.get('endTime') + return None, None + +def get_experiment_status(port): + '''get the status of an experiment''' + result, response = check_rest_server_quick(port) + if result: + return json.loads(response.text).get('status') + return None + +def update_experiment(): + '''Update the experiment status in config file''' + experiments_config = Experiments() + experiments_dict = experiments_config.get_all_experiments() + if not experiments_dict: + return None + for key in experiments_dict.keys(): + if isinstance(experiments_dict[key], dict): + if experiments_dict[key].get('status') != 'STOPPED': + rest_pid = experiments_dict[key].get('pid') + if not detect_process(rest_pid): + experiments_config.update_experiment(key, 'status', 'STOPPED') + continue + +def check_experiment_id(args, update=True): + '''check if the id is valid + ''' + if update: + update_experiment() + experiments_config = Experiments() + experiments_dict = experiments_config.get_all_experiments() + if not experiments_dict: + print_normal('There is no experiment running...') + return None + if not args.id: + running_experiment_list = [] + for key in experiments_dict.keys(): + if isinstance(experiments_dict[key], dict): + if experiments_dict[key].get('status') != 'STOPPED': + running_experiment_list.append(key) + elif isinstance(experiments_dict[key], list): + # if the config file is old version, remove the configuration from file + experiments_config.remove_experiment(key) + if len(running_experiment_list) > 1: + print_error('There are multiple experiments, please set the experiment id...') + experiment_information = "" + for key in running_experiment_list: + experiment_information += EXPERIMENT_DETAIL_FORMAT % (key, + experiments_dict[key].get('experimentName', 'N/A'), + experiments_dict[key]['status'], + experiments_dict[key].get('port', 'N/A'), + experiments_dict[key].get('platform'), + time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(experiments_dict[key]['startTime'] / 1000)) if isinstance(experiments_dict[key]['startTime'], int) else experiments_dict[key]['startTime'], + time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(experiments_dict[key]['endTime'] / 1000)) if isinstance(experiments_dict[key]['endTime'], int) else experiments_dict[key]['endTime']) + print(EXPERIMENT_INFORMATION_FORMAT % experiment_information) + exit(1) + elif not running_experiment_list: + print_error('There is no experiment running.') + return None + else: + return running_experiment_list[0] + if experiments_dict.get(args.id): + return args.id + else: + print_error('Id not correct.') + return None + +def parse_ids(args): + '''Parse the arguments for nnictl stop + 1.If port is provided and id is not specified, return the id who owns the port + 2.If both port and id are provided, return the id if it owns the port, otherwise fail + 3.If there is an id specified, return the corresponding id + 4.If there is no id specified, and there is an experiment running, return the id, or return Error + 5.If the id matches an experiment, nnictl will return the id. + 6.If the id ends with *, nnictl will match all ids matchs the regular + 7.If the id does not exist but match the prefix of an experiment id, nnictl will return the matched id + 8.If the id does not exist but match multiple prefix of the experiment ids, nnictl will give id information + ''' + update_experiment() + experiments_config = Experiments() + experiments_dict = experiments_config.get_all_experiments() + if not experiments_dict: + print_normal('Experiment is not running...') + return None + result_list = [] + running_experiment_list = [] + for key in experiments_dict.keys(): + if isinstance(experiments_dict[key], dict): + if experiments_dict[key].get('status') != 'STOPPED': + running_experiment_list.append(key) + elif isinstance(experiments_dict[key], list): + # if the config file is old version, remove the configuration from file + experiments_config.remove_experiment(key) + if args.all: + return running_experiment_list + if args.port is not None: + for key in running_experiment_list: + if experiments_dict[key].get('port') == args.port: + result_list.append(key) + if args.id and result_list and args.id != result_list[0]: + print_error('Experiment id and resful server port not match') + exit(1) + elif not args.id: + if len(running_experiment_list) > 1: + print_error('There are multiple experiments, please set the experiment id...') + experiment_information = "" + for key in running_experiment_list: + experiment_information += EXPERIMENT_DETAIL_FORMAT % (key, + experiments_dict[key].get('experimentName', 'N/A'), + experiments_dict[key]['status'], + experiments_dict[key].get('port', 'N/A'), + experiments_dict[key].get('platform'), + time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(experiments_dict[key]['startTime'] / 1000)) if isinstance(experiments_dict[key]['startTime'], int) else experiments_dict[key]['startTime'], + time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(experiments_dict[key]['endTime'] / 1000)) if isinstance(experiments_dict[key]['endTime'], int) else experiments_dict[key]['endTime']) + print(EXPERIMENT_INFORMATION_FORMAT % experiment_information) + exit(1) + else: + result_list = running_experiment_list + elif args.id.endswith('*'): + for expId in running_experiment_list: + if expId.startswith(args.id[:-1]): + result_list.append(expId) + elif args.id in running_experiment_list: + result_list.append(args.id) + else: + for expId in running_experiment_list: + if expId.startswith(args.id): + result_list.append(expId) + if len(result_list) > 1: + print_error(args.id + ' is ambiguous, please choose ' + ' '.join(result_list)) + return None + if not result_list and (args.id or args.port): + print_error('There are no experiments matched, please set correct experiment id or restful server port') + elif not result_list: + print_error('There is no experiment running...') + return result_list + +def get_config_filename(args): + '''get the file name of config file''' + experiment_id = check_experiment_id(args) + if experiment_id is None: + print_error('Please set correct experiment id.') + exit(1) + return experiment_id + +def get_experiment_port(args): + '''get the port of experiment''' + experiment_id = check_experiment_id(args) + if experiment_id is None: + print_error('Please set correct experiment id.') + exit(1) + experiments_config = Experiments() + experiments_dict = experiments_config.get_all_experiments() + return experiments_dict[experiment_id].get('port') + +def convert_time_stamp_to_date(content): + '''Convert time stamp to date time format''' + start_time_stamp = content.get('startTime') + end_time_stamp = content.get('endTime') + if start_time_stamp: + start_time = datetime.fromtimestamp(start_time_stamp // 1000, timezone.utc).astimezone().strftime("%Y/%m/%d %H:%M:%S") + content['startTime'] = str(start_time) + if end_time_stamp: + end_time = datetime.fromtimestamp(end_time_stamp // 1000, timezone.utc).astimezone().strftime("%Y/%m/%d %H:%M:%S") + content['endTime'] = str(end_time) + return content + +def check_rest(args): + '''check if restful server is running''' + experiments_config = Experiments() + experiments_dict = experiments_config.get_all_experiments() + rest_port = experiments_dict.get(get_config_filename(args)).get('port') + running, _ = check_rest_server_quick(rest_port) + if running: + print_normal('Restful server is running...') + else: + print_normal('Restful server is not running...') + return running + +def stop_experiment(args): + '''Stop the experiment which is running''' + if args.id and args.id == 'all': + print_warning('\'nnictl stop all\' is abolished, please use \'nnictl stop --all\' to stop all of experiments!') + exit(1) + experiment_id_list = parse_ids(args) + if experiment_id_list: + for experiment_id in experiment_id_list: + print_normal('Stopping experiment %s' % experiment_id) + experiments_config = Experiments() + experiments_dict = experiments_config.get_all_experiments() + rest_pid = experiments_dict.get(experiment_id).get('pid') + if rest_pid: + kill_command(rest_pid) + print_normal('Stop experiment success.') + +def trial_ls(args): + '''List trial''' + def final_metric_data_cmp(lhs, rhs): + metric_l = json.loads(json.loads(lhs['finalMetricData'][0]['data'])) + metric_r = json.loads(json.loads(rhs['finalMetricData'][0]['data'])) + if isinstance(metric_l, float): + return metric_l - metric_r + elif isinstance(metric_l, dict): + return metric_l['default'] - metric_r['default'] + else: + print_error('Unexpected data format. Please check your data.') + raise ValueError + + if args.head and args.tail: + print_error('Head and tail cannot be set at the same time.') + return + experiments_config = Experiments() + experiments_dict = experiments_config.get_all_experiments() + experiment_id = get_config_filename(args) + rest_port = experiments_dict.get(experiment_id).get('port') + rest_pid = experiments_dict.get(experiment_id).get('pid') + if not detect_process(rest_pid): + print_error('Experiment is not running...') + return + running, response = check_rest_server_quick(rest_port) + if running: + response = rest_get(trial_jobs_url(rest_port), REST_TIME_OUT) + if response and check_response(response): + content = json.loads(response.text) + if args.head: + assert args.head > 0, 'The number of requested data must be greater than 0.' + content = sorted(filter(lambda x: 'finalMetricData' in x, content), + key=cmp_to_key(final_metric_data_cmp), reverse=True)[:args.head] + elif args.tail: + assert args.tail > 0, 'The number of requested data must be greater than 0.' + content = sorted(filter(lambda x: 'finalMetricData' in x, content), + key=cmp_to_key(final_metric_data_cmp))[:args.tail] + for index, value in enumerate(content): + content[index] = convert_time_stamp_to_date(value) + print(json.dumps(content, indent=4, sort_keys=True, separators=(',', ':'))) + return content + else: + print_error('List trial failed...') + else: + print_error('Restful server is not running...') + return None + +def trial_kill(args): + '''List trial''' + experiments_config = Experiments() + experiments_dict = experiments_config.get_all_experiments() + experiment_id = get_config_filename(args) + rest_port = experiments_dict.get(experiment_id).get('port') + rest_pid = experiments_dict.get(experiment_id).get('pid') + if not detect_process(rest_pid): + print_error('Experiment is not running...') + return + running, _ = check_rest_server_quick(rest_port) + if running: + response = rest_delete(trial_job_id_url(rest_port, args.trial_id), REST_TIME_OUT) + if response and check_response(response): + print(response.text) + return True + else: + print_error('Kill trial job failed...') + else: + print_error('Restful server is not running...') + return False + +def trial_codegen(args): + '''Generate code for a specific trial''' + print_warning('Currently, this command is only for nni nas programming interface.') + exp_id = get_config_filename(args) + experiment_config = Config(exp_id, Experiments().get_all_experiments()[exp_id]['logDir']).get_config() + if not experiment_config.get('useAnnotation'): + print_error('The experiment is not using annotation') + exit(1) + code_dir = experiment_config['trial']['codeDir'] + expand_annotations(code_dir, './exp_%s_trial_%s_code'%(exp_id, args.trial_id), exp_id, args.trial_id) + +def list_experiment(args): + '''Get experiment information''' + experiments_config = Experiments() + experiments_dict = experiments_config.get_all_experiments() + experiment_id = get_config_filename(args) + rest_port = experiments_dict.get(experiment_id).get('port') + rest_pid = experiments_dict.get(experiment_id).get('pid') + if not detect_process(rest_pid): + print_error('Experiment is not running...') + return + running, _ = check_rest_server_quick(rest_port) + if running: + response = rest_get(experiment_url(rest_port), REST_TIME_OUT) + if response and check_response(response): + content = convert_time_stamp_to_date(json.loads(response.text)) + print(json.dumps(content, indent=4, sort_keys=True, separators=(',', ':'))) + return content + else: + print_error('List experiment failed...') + else: + print_error('Restful server is not running...') + return None + +def experiment_status(args): + '''Show the status of experiment''' + experiments_config = Experiments() + experiments_dict = experiments_config.get_all_experiments() + rest_port = experiments_dict.get(get_config_filename(args)).get('port') + result, response = check_rest_server_quick(rest_port) + if not result: + print_normal('Restful server is not running...') + else: + print(json.dumps(json.loads(response.text), indent=4, sort_keys=True, separators=(',', ':'))) + return result + +def log_internal(args, filetype): + '''internal function to call get_log_content''' + file_name = get_config_filename(args) + if filetype == 'stdout': + file_full_path = os.path.join(NNI_HOME_DIR, file_name, 'log', 'nnictl_stdout.log') + else: + file_full_path = os.path.join(NNI_HOME_DIR, file_name, 'log', 'nnictl_stderr.log') + print(check_output_command(file_full_path, head=args.head, tail=args.tail)) + +def log_stdout(args): + '''get stdout log''' + log_internal(args, 'stdout') + +def log_stderr(args): + '''get stderr log''' + log_internal(args, 'stderr') + +def log_trial_adl_helper(args, experiment_id): + # adljob_id format should be consistent to the one in "adlTrainingService.ts": + # const adlJobName: string = `nni-exp-${this.experimentId}-trial-${trialJobId}`.toLowerCase(); + adlJobName = "nni-exp-{}-trial-{}".format(experiment_id, args.trial_id).lower() + print_warning('Note that no log will show when trial is pending or done (succeeded or failed). ' + 'You can retry the command.') + print_green('>>> Trial log streaming:') + try: + subprocess.run( + [ + "kubectl", "logs", + "-l", "adaptdl/job=%s" % adlJobName, + "-f" # Follow the stream + ], # TODO: support remaining argument, uncomment the lines in nnictl.py + ) # TODO: emulate tee behaviors, not necessary tho. + except KeyboardInterrupt: + pass + except Exception: + print_error('Error! Please check kubectl:') + traceback.print_exc() + exit(1) + finally: + print_green('<<< [adlJobName:%s]' % adlJobName) + nni_manager_collection_path = os.path.expanduser('~/nni-experiments/%s/trials/%s/stdout_log_collection.log' % + (experiment_id, args.trial_id)) + print_green('>>> (Optional) How to persist the complete trial log locally:') + print( + 'Please ensure `logCollection: http` ' + 'exists in the experiment configuration yaml. ' + 'After trial done, you can check it from the file below: \n %s' + % nni_manager_collection_path + ) + + +def log_trial(args): + ''''get trial log path''' + trial_id_path_dict = {} + trial_id_list = [] + experiments_config = Experiments() + experiments_dict = experiments_config.get_all_experiments() + experiment_id = get_config_filename(args) + rest_port = experiments_dict.get(experiment_id).get('port') + rest_pid = experiments_dict.get(experiment_id).get('pid') + experiment_config = Config(experiment_id, experiments_dict.get(experiment_id).get('logDir')).get_config() + if not detect_process(rest_pid): + print_error('Experiment is not running...') + return + running, response = check_rest_server_quick(rest_port) + if running: + response = rest_get(trial_jobs_url(rest_port), REST_TIME_OUT) + if response and check_response(response): + content = json.loads(response.text) + for trial in content: + trial_id_list.append(trial.get('trialJobId')) + if trial.get('logPath'): + trial_id_path_dict[trial.get('trialJobId')] = trial['logPath'] + else: + print_error('Restful server is not running...') + exit(1) + is_adl = experiment_config.get('trainingServicePlatform') == 'adl' + if is_adl and not args.trial_id: + print_error('Trial ID is required to retrieve the log for adl. Please specify it with "--trial_id".') + exit(1) + if args.trial_id: + if args.trial_id not in trial_id_list: + print_error('Trial id {0} not correct, please check your command!'.format(args.trial_id)) + exit(1) + if is_adl: + log_trial_adl_helper(args, experiment_id) + # adl has its own way to log trial, and it thus returns right after the helper returns + return + if trial_id_path_dict.get(args.trial_id): + print_normal('id:' + args.trial_id + ' path:' + trial_id_path_dict[args.trial_id]) + else: + print_error('Log path is not available yet, please wait...') + exit(1) + else: + print_normal('All of trial log info:') + for key in trial_id_path_dict: + print_normal('id:' + key + ' path:' + trial_id_path_dict[key]) + if not trial_id_path_dict: + print_normal('None') + +def get_config(args): + '''get config info''' + experiment_id = get_config_filename(args) + experiment_config = Config(experiment_id, Experiments().get_all_experiments()[experiment_id]['logDir']).get_config() + print(json.dumps(experiment_config, indent=4)) + +def webui_url(args): + '''show the url of web ui''' + experiment_id = get_config_filename(args) + experiments_dict = Experiments().get_all_experiments() + print_normal('{0} {1}'.format('Web UI url:', ' '.join(experiments_dict[experiment_id].get('webuiUrl')))) + +def webui_nas(args): + '''launch nas ui''' + print_normal('Starting NAS UI...') + try: + entry_dir = nni_node.__path__[0] + entry_file = os.path.join(entry_dir, 'nasui', 'server.js') + if sys.platform == 'win32': + node_command = os.path.join(entry_dir, 'node.exe') + else: + node_command = os.path.join(entry_dir, 'node') + cmds = [node_command, '--max-old-space-size=4096', entry_file, '--port', str(args.port), '--logdir', args.logdir] + subprocess.run(cmds, cwd=entry_dir) + except KeyboardInterrupt: + pass + +def local_clean(directory): + '''clean up local data''' + print_normal('removing folder {0}'.format(directory)) + try: + shutil.rmtree(directory) + except FileNotFoundError: + print_error('{0} does not exist.'.format(directory)) + +def remote_clean(machine_list, experiment_id=None): + '''clean up remote data''' + for machine in machine_list: + passwd = machine.get('passwd') + userName = machine.get('username') + host = machine.get('ip') + port = machine.get('port') + sshKeyPath = machine.get('sshKeyPath') + passphrase = machine.get('passphrase') + if experiment_id: + remote_dir = '/' + '/'.join(['tmp', 'nni-experiments', experiment_id]) + else: + remote_dir = '/' + '/'.join(['tmp', 'nni-experiments']) + sftp = create_ssh_sftp_client(host, port, userName, passwd, sshKeyPath, passphrase) + print_normal('removing folder {0}'.format(host + ':' + str(port) + remote_dir)) + remove_remote_directory(sftp, remote_dir) + +def experiment_clean(args): + '''clean up the experiment data''' + experiment_id_list = [] + experiments_config = Experiments() + experiments_dict = experiments_config.get_all_experiments() + if args.all: + experiment_id_list = list(experiments_dict.keys()) + else: + if args.id is None: + print_error('please set experiment id.') + exit(1) + if args.id not in experiments_dict: + print_error('Cannot find experiment {0}.'.format(args.id)) + exit(1) + experiment_id_list.append(args.id) + while True: + print('INFO: This action will delete experiment {0}, and it\'s not recoverable.'.format(' '.join(experiment_id_list))) + inputs = input('INFO: do you want to continue?[y/N]:') + if not inputs.lower() or inputs.lower() in ['n', 'no']: + exit(0) + elif inputs.lower() not in ['y', 'n', 'yes', 'no']: + print_warning('please input Y or N.') + else: + break + for experiment_id in experiment_id_list: + experiment_id = get_config_filename(args) + experiment_config = Config(experiment_id, Experiments().get_all_experiments()[experiment_id]['logDir']).get_config() + platform = experiment_config.get('trainingServicePlatform') + if platform == 'remote': + machine_list = experiment_config.get('machineList') + remote_clean(machine_list, experiment_id) + elif platform != 'local': + # TODO: support all platforms + print_warning('platform {0} clean up not supported yet.'.format(platform)) + exit(0) + # clean local data + local_base_dir = experiments_config.experiments[experiment_id]['logDir'] + if not local_base_dir: + local_base_dir = NNI_HOME_DIR + local_experiment_dir = os.path.join(local_base_dir, experiment_id) + experiment_folder_name_list = ['checkpoint', 'db', 'log', 'trials'] + for folder_name in experiment_folder_name_list: + local_clean(os.path.join(local_experiment_dir, folder_name)) + if not os.listdir(local_experiment_dir): + local_clean(local_experiment_dir) + print_normal('removing metadata of experiment {0}'.format(experiment_id)) + experiments_config.remove_experiment(experiment_id) + print_normal('Done.') + +def get_platform_dir(config_content): + '''get the dir list to be deleted''' + platform = config_content.get('trainingServicePlatform') + dir_list = [] + if platform == 'remote': + machine_list = config_content.get('machineList') + for machine in machine_list: + host = machine.get('ip') + port = machine.get('port') + dir_list.append(host + ':' + str(port) + '/tmp/nni') + elif platform == 'pai': + host = config_content.get('paiConfig').get('host') + user_name = config_content.get('paiConfig').get('userName') + output_dir = config_content.get('trial').get('outputDir') + dir_list.append('server: {0}, path: {1}/nni'.format(host, user_name)) + if output_dir: + dir_list.append(output_dir) + return dir_list + +def platform_clean(args): + '''clean up the experiment data''' + config_path = os.path.abspath(args.config) + if not os.path.exists(config_path): + print_error('Please set correct config path.') + exit(1) + config_content = get_yml_content(config_path) + platform = config_content.get('trainingServicePlatform') + if platform == 'local': + print_normal('it doesn’t need to clean local platform.') + exit(0) + if platform not in ['remote', 'pai']: + print_normal('platform {0} not supported.'.format(platform)) + exit(0) + update_experiment() + dir_list = get_platform_dir(config_content) + if not dir_list: + print_normal('No folder of NNI caches is found.') + exit(1) + while True: + print_normal('This command will remove below folders of NNI caches. If other users are using experiments' \ + ' on below hosts, it will be broken.') + for value in dir_list: + print(' ' + value) + inputs = input('INFO: do you want to continue?[y/N]:') + if not inputs.lower() or inputs.lower() in ['n', 'no']: + exit(0) + elif inputs.lower() not in ['y', 'n', 'yes', 'no']: + print_warning('please input Y or N.') + else: + break + if platform == 'remote': + machine_list = config_content.get('machineList') + remote_clean(machine_list) + print_normal('Done.') + +def experiment_list(args): + '''get the information of all experiments''' + update_experiment() + experiments_config = Experiments() + experiments_dict = experiments_config.get_all_experiments() + if not experiments_dict: + print_normal('Cannot find experiments.') + exit(1) + experiment_id_list = [] + if args.all: + for key in experiments_dict.keys(): + experiment_id_list.append(key) + else: + for key in experiments_dict.keys(): + if experiments_dict[key]['status'] != 'STOPPED': + experiment_id_list.append(key) + if not experiment_id_list: + print_warning('There is no experiment running...\nYou can use \'nnictl experiment list --all\' to list all experiments.') + experiment_information = "" + for key in experiment_id_list: + experiment_information += EXPERIMENT_DETAIL_FORMAT % (key, + experiments_dict[key].get('experimentName', 'N/A'), + experiments_dict[key]['status'], + experiments_dict[key].get('port', 'N/A'), + experiments_dict[key].get('platform'), + time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(experiments_dict[key]['startTime'] / 1000)) if isinstance(experiments_dict[key]['startTime'], int) else experiments_dict[key]['startTime'], + time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(experiments_dict[key]['endTime'] / 1000)) if isinstance(experiments_dict[key]['endTime'], int) else experiments_dict[key]['endTime']) + print(EXPERIMENT_INFORMATION_FORMAT % experiment_information) + return experiment_id_list + +def get_time_interval(time1, time2): + '''get the interval of two times''' + try: + seconds = int((time2 - time1) / 1000) + #convert seconds to day:hour:minute:second + days = seconds / 86400 + seconds %= 86400 + hours = seconds / 3600 + seconds %= 3600 + minutes = seconds / 60 + seconds %= 60 + return '%dd %dh %dm %ds' % (days, hours, minutes, seconds) + except: + return 'N/A' + +def show_experiment_info(): + '''show experiment information in monitor''' + update_experiment() + experiments_config = Experiments() + experiments_dict = experiments_config.get_all_experiments() + if not experiments_dict: + print('There is no experiment running...') + exit(1) + experiment_id_list = [] + for key in experiments_dict.keys(): + if experiments_dict[key]['status'] != 'STOPPED': + experiment_id_list.append(key) + if not experiment_id_list: + print_warning('There is no experiment running...') + return + for key in experiment_id_list: + print(EXPERIMENT_MONITOR_INFO % (key, experiments_dict[key]['status'], experiments_dict[key]['port'], \ + experiments_dict[key].get('platform'), time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(experiments_dict[key]['startTime'] / 1000)) if isinstance(experiments_dict[key]['startTime'], int) else experiments_dict[key]['startTime'], \ + get_time_interval(experiments_dict[key]['startTime'], experiments_dict[key]['endTime']))) + print(TRIAL_MONITOR_HEAD) + running, response = check_rest_server_quick(experiments_dict[key]['port']) + if running: + response = rest_get(trial_jobs_url(experiments_dict[key]['port']), REST_TIME_OUT) + if response and check_response(response): + content = json.loads(response.text) + for index, value in enumerate(content): + content[index] = convert_time_stamp_to_date(value) + print(TRIAL_MONITOR_CONTENT % (content[index].get('trialJobId'), content[index].get('startTime'), \ + content[index].get('endTime'), content[index].get('status'))) + print(TRIAL_MONITOR_TAIL) + +def set_monitor(auto_exit, time_interval, port=None, pid=None): + '''set the experiment monitor engine''' + while True: + try: + if sys.platform == 'win32': + os.system('cls') + else: + os.system('clear') + update_experiment() + show_experiment_info() + if auto_exit: + status = get_experiment_status(port) + if status in ['DONE', 'ERROR', 'STOPPED']: + print_normal('Experiment status is {0}.'.format(status)) + print_normal('Stopping experiment...') + kill_command(pid) + print_normal('Stop experiment success.') + exit(0) + time.sleep(time_interval) + except KeyboardInterrupt: + if auto_exit: + print_normal('Stopping experiment...') + kill_command(pid) + print_normal('Stop experiment success.') + else: + print_normal('Exiting...') + exit(0) + except Exception as exception: + print_error(exception) + exit(1) + +def monitor_experiment(args): + '''monitor the experiment''' + if args.time <= 0: + print_error('please input a positive integer as time interval, the unit is second.') + exit(1) + set_monitor(False, args.time) + +def export_trials_data(args): + '''export experiment metadata and intermediate results to json or csv + ''' + def groupby_trial_id(intermediate_results): + sorted(intermediate_results, key=lambda x: x['timestamp']) + groupby = dict() + for content in intermediate_results: + groupby.setdefault(content['trialJobId'], []).append(json.loads(content['data'])) + return groupby + + experiments_config = Experiments() + experiments_dict = experiments_config.get_all_experiments() + experiment_id = get_config_filename(args) + rest_port = experiments_dict.get(experiment_id).get('port') + rest_pid = experiments_dict.get(experiment_id).get('pid') + + if not detect_process(rest_pid): + print_error('Experiment is not running...') + return + running, response = check_rest_server_quick(rest_port) + if not running: + print_error('Restful server is not running') + return + response = rest_get(export_data_url(rest_port), 20) + if response is not None and check_response(response): + content = json.loads(response.text) + if args.intermediate: + intermediate_results_response = rest_get(metric_data_url(rest_port), REST_TIME_OUT) + if not intermediate_results_response or not check_response(intermediate_results_response): + print_error('Error getting intermediate results.') + return + intermediate_results = groupby_trial_id(json.loads(intermediate_results_response.text)) + for record in content: + record['intermediate'] = intermediate_results[record['trialJobId']] + if args.type == 'json': + with open(args.path, 'w') as file: + file.write(json.dumps(content)) + elif args.type == 'csv': + trial_records = [] + for record in content: + formated_record = dict() + if args.intermediate: + formated_record['intermediate'] = '[' + ','.join(record['intermediate']) + ']' + record_value = json.loads(record['value']) + if not isinstance(record_value, (float, int)): + formated_record.update({**record['parameter'], **record_value, **{'trialJobId': record['trialJobId']}}) + else: + formated_record.update({**record['parameter'], **{'reward': record_value, 'trialJobId': record['trialJobId']}}) + trial_records.append(formated_record) + if not trial_records: + print_error('No trial results collected! Please check your trial log...') + exit(0) + with open(args.path, 'w', newline='') as file: + writer = csv.DictWriter(file, set.union(*[set(r.keys()) for r in trial_records])) + writer.writeheader() + writer.writerows(trial_records) + else: + print_error('Unknown type: %s' % args.type) + return + else: + print_error('Export failed...') + +def search_space_auto_gen(args): + '''dry run trial code to generate search space file''' + trial_dir = os.path.expanduser(args.trial_dir) + file_path = os.path.expanduser(args.file) + if not os.path.isabs(file_path): + file_path = os.path.join(os.getcwd(), file_path) + assert os.path.exists(trial_dir) + if os.path.exists(file_path): + print_warning('%s already exists, will be overwritten.' % file_path) + print_normal('Dry run to generate search space...') + Popen(args.trial_command, cwd=trial_dir, env=dict(os.environ, NNI_GEN_SEARCH_SPACE=file_path), shell=True).wait() + if not os.path.exists(file_path): + print_warning('Expected search space file \'{}\' generated, but not found.'.format(file_path)) + else: + print_normal('Generate search space done: \'{}\'.'.format(file_path)) + +def save_experiment(args): + '''save experiment data to a zip file''' + experiments_config = Experiments() + experiments_dict = experiments_config.get_all_experiments() + if args.id is None: + print_error('Please set experiment id.') + exit(1) + if args.id not in experiments_dict: + print_error('Cannot find experiment {0}.'.format(args.id)) + exit(1) + if experiments_dict[args.id].get('status') != 'STOPPED': + print_error('Can only save stopped experiment!') + exit(1) + print_normal('Saving...') + experiment_config = Config(args.id, experiments_dict[args.id]['logDir']).get_config() + logDir = os.path.join(experiments_dict[args.id]['logDir'], args.id) + temp_root_dir = generate_temp_dir() + + # Step1. Copy logDir to temp folder + if not os.path.exists(logDir): + print_error('logDir: %s does not exist!' % logDir) + exit(1) + temp_experiment_dir = os.path.join(temp_root_dir, 'experiment') + shutil.copytree(logDir, temp_experiment_dir) + + # Step2. Copy nnictl metadata to temp folder + temp_nnictl_dir = os.path.join(temp_root_dir, 'nnictl') + os.makedirs(temp_nnictl_dir, exist_ok=True) + try: + with open(os.path.join(temp_nnictl_dir, '.experiment'), 'w') as file: + experiments_dict[args.id]['id'] = args.id + json.dump(experiments_dict[args.id], file) + except IOError: + print_error('Write file to %s failed!' % os.path.join(temp_nnictl_dir, '.experiment')) + exit(1) + nnictl_log_dir = os.path.join(NNI_HOME_DIR, args.id, 'log') + shutil.copytree(nnictl_log_dir, os.path.join(temp_nnictl_dir, args.id, 'log')) + + # Step3. Copy code dir + if args.saveCodeDir: + temp_code_dir = os.path.join(temp_root_dir, 'code') + shutil.copytree(experiment_config['trial']['codeDir'], temp_code_dir) + + # Step4. Copy searchSpace file + search_space_path = experiment_config.get('searchSpacePath') + if search_space_path: + if not os.path.exists(search_space_path): + print_warning('search space %s does not exist!' % search_space_path) + else: + temp_search_space_dir = os.path.join(temp_root_dir, 'searchSpace') + os.makedirs(temp_search_space_dir, exist_ok=True) + search_space_name = os.path.basename(search_space_path) + shutil.copyfile(search_space_path, os.path.join(temp_search_space_dir, search_space_name)) + + # Step5. Archive folder + zip_package_name = 'nni_experiment_%s' % args.id + if args.path: + os.makedirs(args.path, exist_ok=True) + zip_package_name = os.path.join(args.path, zip_package_name) + shutil.make_archive(zip_package_name, 'zip', temp_root_dir) + print_normal('Save to %s.zip success!' % zip_package_name) + + # Step5. Cleanup temp data + shutil.rmtree(temp_root_dir) + +def load_experiment(args): + '''load experiment data''' + package_path = os.path.expanduser(args.path) + if not os.path.exists(args.path): + print_error('file path %s does not exist!' % args.path) + exit(1) + if args.searchSpacePath and os.path.isdir(args.searchSpacePath): + print_error('search space path should be a full path with filename, not a directory!') + exit(1) + temp_root_dir = generate_temp_dir() + shutil.unpack_archive(package_path, temp_root_dir) + print_normal('Loading...') + # Step1. Validation + if not os.path.exists(args.codeDir): + print_error('Invalid: codeDir path does not exist!') + exit(1) + if args.logDir: + if not os.path.exists(args.logDir): + print_error('Invalid: logDir path does not exist!') + exit(1) + experiment_temp_dir = os.path.join(temp_root_dir, 'experiment') + if not os.path.exists(os.path.join(experiment_temp_dir, 'db')): + print_error('Invalid archive file: db file does not exist!') + shutil.rmtree(temp_root_dir) + exit(1) + nnictl_temp_dir = os.path.join(temp_root_dir, 'nnictl') + if not os.path.exists(os.path.join(nnictl_temp_dir, '.experiment')): + print_error('Invalid archive file: nnictl metadata file does not exist!') + shutil.rmtree(temp_root_dir) + exit(1) + try: + with open(os.path.join(nnictl_temp_dir, '.experiment'), 'r') as file: + experiment_metadata = json.load(file) + except ValueError as err: + print_error('Invalid nnictl metadata file: %s' % err) + shutil.rmtree(temp_root_dir) + exit(1) + experiments_config = Experiments() + experiments_dict = experiments_config.get_all_experiments() + experiment_id = experiment_metadata.get('id') + if experiment_id in experiments_dict: + print_error('Invalid: experiment id already exist!') + shutil.rmtree(temp_root_dir) + exit(1) + if not os.path.exists(os.path.join(nnictl_temp_dir, experiment_id)): + print_error('Invalid: experiment metadata does not exist!') + shutil.rmtree(temp_root_dir) + exit(1) + + # Step2. Copy nnictl metadata + src_path = os.path.join(nnictl_temp_dir, experiment_id) + dest_path = os.path.join(NNI_HOME_DIR, experiment_id) + if os.path.exists(dest_path): + shutil.rmtree(dest_path) + shutil.copytree(src_path, dest_path) + + # Step3. Copy experiment data + os.rename(os.path.join(temp_root_dir, 'experiment'), os.path.join(temp_root_dir, experiment_id)) + src_path = os.path.join(os.path.join(temp_root_dir, experiment_id)) + experiment_config = Config(experiment_id, temp_root_dir).get_config() + if args.logDir: + logDir = args.logDir + experiment_config['logDir'] = logDir + else: + if experiment_config.get('logDir'): + logDir = experiment_config['logDir'] + else: + logDir = NNI_HOME_DIR + + dest_path = os.path.join(logDir, experiment_id) + if os.path.exists(dest_path): + shutil.rmtree(dest_path) + shutil.copytree(src_path, dest_path) + + # Step4. Copy code dir + codeDir = os.path.expanduser(args.codeDir) + if not os.path.isabs(codeDir): + codeDir = os.path.join(os.getcwd(), codeDir) + print_normal('Expand codeDir to %s' % codeDir) + experiment_config['trial']['codeDir'] = codeDir + archive_code_dir = os.path.join(temp_root_dir, 'code') + if os.path.exists(archive_code_dir): + file_list = os.listdir(archive_code_dir) + for file_name in file_list: + src_path = os.path.join(archive_code_dir, file_name) + target_path = os.path.join(codeDir, file_name) + if os.path.exists(target_path): + print_error('Copy %s failed, %s exist!' % (file_name, target_path)) + continue + if os.path.isdir(src_path): + shutil.copytree(src_path, target_path) + else: + shutil.copy(src_path, target_path) + + # Step5. Create experiment metadata + experiments_config.add_experiment(experiment_id, + experiment_metadata.get('port'), + experiment_metadata.get('startTime'), + experiment_metadata.get('platform'), + experiment_metadata.get('experimentName'), + experiment_metadata.get('endTime'), + experiment_metadata.get('status'), + experiment_metadata.get('tag'), + experiment_metadata.get('pid'), + experiment_metadata.get('webUrl'), + logDir) + print_normal('Load experiment %s succsss!' % experiment_id) + + # Step6. Cleanup temp data + shutil.rmtree(temp_root_dir) diff --git a/new_impl/cv/third_party/nni_new/tools/nnictl/rest_utils.py b/new_impl/cv/third_party/nni_new/tools/nnictl/rest_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..e98c9a839245ca0775406302dd6bf40bf8a2ae35 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/tools/nnictl/rest_utils.py @@ -0,0 +1,77 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import time +import requests +from .url_utils import check_status_url +from .constants import REST_TIME_OUT +from .common_utils import print_error + +def rest_put(url, data, timeout, show_error=False): + '''Call rest put method''' + try: + response = requests.put(url, headers={'Accept': 'application/json', 'Content-Type': 'application/json'},\ + data=data, timeout=timeout) + return response + except Exception as exception: + if show_error: + print_error(exception) + return None + +def rest_post(url, data, timeout, show_error=False): + '''Call rest post method''' + try: + response = requests.post(url, headers={'Accept': 'application/json', 'Content-Type': 'application/json'},\ + data=data, timeout=timeout) + return response + except Exception as exception: + if show_error: + print_error(exception) + return None + +def rest_get(url, timeout, show_error=False): + '''Call rest get method''' + try: + response = requests.get(url, timeout=timeout) + return response + except Exception as exception: + if show_error: + print_error(exception) + return None + +def rest_delete(url, timeout, show_error=False): + '''Call rest delete method''' + try: + response = requests.delete(url, timeout=timeout) + return response + except Exception as exception: + if show_error: + print_error(exception) + return None + +def check_rest_server(rest_port): + '''Check if restful server is ready''' + retry_count = 20 + for _ in range(retry_count): + response = rest_get(check_status_url(rest_port), REST_TIME_OUT) + if response: + if response.status_code == 200: + return True, response + else: + return False, response + else: + time.sleep(1) + return False, response + +def check_rest_server_quick(rest_port): + '''Check if restful server is ready, only check once''' + response = rest_get(check_status_url(rest_port), 5) + if response and response.status_code == 200: + return True, response + return False, None + +def check_response(response): + '''Check if a response is success according to status_code''' + if response and response.status_code == 200: + return True + return False diff --git a/new_impl/cv/third_party/nni_new/tools/nnictl/ssh_utils.py b/new_impl/cv/third_party/nni_new/tools/nnictl/ssh_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..e3f26a8e24c1be67bf86cfa0192481b245c93f97 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/tools/nnictl/ssh_utils.py @@ -0,0 +1,60 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +from .common_utils import print_error +from .command_utils import install_package_command + +def check_environment(): + '''check if paramiko is installed''' + try: + import paramiko + except: + install_package_command('paramiko') + import paramiko + return paramiko + +def copy_remote_directory_to_local(sftp, remote_path, local_path): + '''copy remote directory to local machine''' + try: + os.makedirs(local_path, exist_ok=True) + files = sftp.listdir(remote_path) + for file in files: + remote_full_path = os.path.join(remote_path, file) + local_full_path = os.path.join(local_path, file) + try: + if sftp.listdir(remote_full_path): + copy_remote_directory_to_local(sftp, remote_full_path, local_full_path) + except: + sftp.get(remote_full_path, local_full_path) + except Exception: + pass + +def create_ssh_sftp_client(host_ip, port, username, password, ssh_key_path, passphrase): + '''create ssh client''' + try: + paramiko = check_environment() + conn = paramiko.Transport(host_ip, port) + if ssh_key_path is not None: + ssh_key = paramiko.RSAKey.from_private_key_file(ssh_key_path, password=passphrase) + conn.connect(username=username, pkey=ssh_key) + else: + conn.connect(username=username, password=password) + sftp = paramiko.SFTPClient.from_transport(conn) + return sftp + except Exception as exception: + print_error('Create ssh client error %s\n' % exception) + +def remove_remote_directory(sftp, directory): + '''remove a directory in remote machine''' + try: + files = sftp.listdir(directory) + for file in files: + filepath = '/'.join([directory, file]) + try: + sftp.remove(filepath) + except IOError: + remove_remote_directory(sftp, filepath) + sftp.rmdir(directory) + except IOError as err: + print_error(err) diff --git a/new_impl/cv/third_party/nni_new/tools/nnictl/ts_management.py b/new_impl/cv/third_party/nni_new/tools/nnictl/ts_management.py new file mode 100644 index 0000000000000000000000000000000000000000..151b053591b549734ae37d5da3afa1bed3694a4f --- /dev/null +++ b/new_impl/cv/third_party/nni_new/tools/nnictl/ts_management.py @@ -0,0 +1,77 @@ +import importlib +import json + +from nni.runtime.config import get_config_file +from .common_utils import print_error, print_green + +_builtin_training_services = [ + 'local', + 'remote', + 'openpai', 'pai', + 'aml', + 'kubeflow', + 'frameworkcontroller', + 'adl', +] + +def register(args): + if args.package in _builtin_training_services: + print_error(f'{args.package} is a builtin training service') + return + + try: + module = importlib.import_module(args.package) + except Exception: + print_error(f'Cannot import package {args.package}') + return + + try: + info = module.nni_training_service_info + except Exception: + print_error(f'Cannot read nni_training_service_info from {args.package}') + return + + try: + info.config_class() + except Exception: + print_error('Bad experiment config class') + return + + try: + service_config = { + 'nodeModulePath': str(info.node_module_path), + 'nodeClassName': info.node_class_name, + } + json.dumps(service_config) + except Exception: + print_error('Bad node_module_path or bad node_class_name') + return + + config = _load() + update = args.package in config + + config[args.package] = service_config + _save(config) + + if update: + print_green(f'Sucessfully updated {args.package}') + else: + print_green(f'Sucessfully registered {args.package}') + +def unregister(args): + config = _load() + if args.package not in config: + print_error(f'{args.package} is not a registered training service') + return + config.pop(args.package, None) + _save(config) + print_green(f'Sucessfully unregistered {args.package}') + +def list_services(_): + print('\n'.join(_load().keys())) + +def _load(): + return json.load(get_config_file('training_services.json').open()) + +def _save(config): + json.dump(config, get_config_file('training_services.json').open('w'), indent=4) diff --git a/new_impl/cv/third_party/nni_new/tools/nnictl/updater.py b/new_impl/cv/third_party/nni_new/tools/nnictl/updater.py new file mode 100644 index 0000000000000000000000000000000000000000..e462562349914c222f29597b7946b1017040cf28 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/tools/nnictl/updater.py @@ -0,0 +1,151 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import json +import os +from .rest_utils import rest_put, rest_post, rest_get, check_rest_server_quick, check_response +from .url_utils import experiment_url, import_data_url +from .config_utils import Config, Experiments +from .common_utils import get_json_content, print_normal, print_error, print_warning +from .nnictl_utils import get_experiment_port, get_config_filename, detect_process +from .launcher_utils import parse_time +from .constants import REST_TIME_OUT, TUNERS_SUPPORTING_IMPORT_DATA, TUNERS_NO_NEED_TO_IMPORT_DATA + +def validate_digit(value, start, end): + '''validate if a digit is valid''' + if not str(value).isdigit() or int(value) < start or int(value) > end: + raise ValueError('value (%s) must be a digit from %s to %s' % (value, start, end)) + +def validate_file(path): + '''validate if a file exist''' + if not os.path.exists(path): + raise FileNotFoundError('%s is not a valid file path' % path) + +def validate_dispatcher(args): + '''validate if the dispatcher of the experiment supports importing data''' + experiment_id = get_config_filename(args) + experiment_config = Config(experiment_id, Experiments().get_all_experiments()[experiment_id]['logDir']).get_config() + if experiment_config.get('tuner') and experiment_config['tuner'].get('builtinTunerName'): + dispatcher_name = experiment_config['tuner']['builtinTunerName'] + elif experiment_config.get('advisor') and experiment_config['advisor'].get('builtinAdvisorName'): + dispatcher_name = experiment_config['advisor']['builtinAdvisorName'] + else: # otherwise it should be a customized one + return + if dispatcher_name not in TUNERS_SUPPORTING_IMPORT_DATA: + if dispatcher_name in TUNERS_NO_NEED_TO_IMPORT_DATA: + print_warning("There is no need to import data for %s" % dispatcher_name) + exit(0) + else: + print_error("%s does not support importing addtional data" % dispatcher_name) + exit(1) + +def load_search_space(path): + '''load search space content''' + content = json.dumps(get_json_content(path)) + if not content: + raise ValueError('searchSpace file should not be empty') + return content + +def get_query_type(key): + '''get update query type''' + if key == 'trialConcurrency': + return '?update_type=TRIAL_CONCURRENCY' + if key == 'maxExecDuration': + return '?update_type=MAX_EXEC_DURATION' + if key == 'searchSpace': + return '?update_type=SEARCH_SPACE' + if key == 'maxTrialNum': + return '?update_type=MAX_TRIAL_NUM' + +def update_experiment_profile(args, key, value): + '''call restful server to update experiment profile''' + experiments_config = Experiments() + experiments_dict = experiments_config.get_all_experiments() + rest_port = experiments_dict.get(get_config_filename(args)).get('port') + running, _ = check_rest_server_quick(rest_port) + if running: + response = rest_get(experiment_url(rest_port), REST_TIME_OUT) + if response and check_response(response): + experiment_profile = json.loads(response.text) + experiment_profile['params'][key] = value + response = rest_put(experiment_url(rest_port)+get_query_type(key), json.dumps(experiment_profile), REST_TIME_OUT) + if response and check_response(response): + return response + else: + print_error('Restful server is not running...') + return None + +def update_searchspace(args): + validate_file(args.filename) + content = load_search_space(args.filename) + args.port = get_experiment_port(args) + if args.port is not None: + if update_experiment_profile(args, 'searchSpace', content): + print_normal('Update %s success!' % 'searchSpace') + else: + print_error('Update %s failed!' % 'searchSpace') + + +def update_concurrency(args): + validate_digit(args.value, 1, 1000) + args.port = get_experiment_port(args) + if args.port is not None: + if update_experiment_profile(args, 'trialConcurrency', int(args.value)): + print_normal('Update %s success!' % 'concurrency') + else: + print_error('Update %s failed!' % 'concurrency') + +def update_duration(args): + #parse time, change time unit to seconds + args.value = parse_time(args.value) + args.port = get_experiment_port(args) + if args.port is not None: + if update_experiment_profile(args, 'maxExecDuration', int(args.value)): + print_normal('Update %s success!' % 'duration') + else: + print_error('Update %s failed!' % 'duration') + +def update_trialnum(args): + validate_digit(args.value, 1, 999999999) + if update_experiment_profile(args, 'maxTrialNum', int(args.value)): + print_normal('Update %s success!' % 'trialnum') + else: + print_error('Update %s failed!' % 'trialnum') + +def import_data(args): + '''import additional data to the experiment''' + validate_file(args.filename) + validate_dispatcher(args) + content = load_search_space(args.filename) + + experiments_dict = Experiments().get_all_experiments() + experiment_id = get_config_filename(args) + rest_port = experiments_dict.get(experiment_id).get('port') + rest_pid = experiments_dict.get(experiment_id).get('pid') + if not detect_process(rest_pid): + print_error('Experiment is not running...') + return + running, _ = check_rest_server_quick(rest_port) + if not running: + print_error('Restful server is not running') + return + + args.port = rest_port + if args.port is not None: + if import_data_to_restful_server(args, content): + pass + else: + print_error('Import data failed!') + +def import_data_to_restful_server(args, content): + '''call restful server to import data to the experiment''' + experiments_dict = Experiments().get_all_experiments() + rest_port = experiments_dict.get(get_config_filename(args)).get('port') + running, _ = check_rest_server_quick(rest_port) + if running: + response = rest_post(import_data_url(rest_port), content, REST_TIME_OUT) + if response and check_response(response): + return response + else: + print_error('Restful server is not running...') + return None diff --git a/new_impl/cv/third_party/nni_new/tools/nnictl/url_utils.py b/new_impl/cv/third_party/nni_new/tools/nnictl/url_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..a4f1b807d2d5784e95d2e38ad4b390a1b4306bbe --- /dev/null +++ b/new_impl/cv/third_party/nni_new/tools/nnictl/url_utils.py @@ -0,0 +1,85 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import socket +import psutil + +BASE_URL = 'http://localhost' + +API_ROOT_URL = '/api/v1/nni' + +EXPERIMENT_API = '/experiment' + +CLUSTER_METADATA_API = '/experiment/cluster-metadata' + +IMPORT_DATA_API = '/experiment/import-data' + +CHECK_STATUS_API = '/check-status' + +TRIAL_JOBS_API = '/trial-jobs' + +EXPORT_DATA_API = '/export-data' + +TENSORBOARD_API = '/tensorboard' + +METRIC_DATA_API = '/metric-data' + +def format_url_path(path): + return API_ROOT_URL if path is None else f'/{path}{API_ROOT_URL}' + +def set_prefix_url(prefix_path): + global API_ROOT_URL + API_ROOT_URL = format_url_path(prefix_path) + +def metric_data_url(port): + '''get metric_data url''' + return '{0}:{1}{2}{3}'.format(BASE_URL, port, API_ROOT_URL, METRIC_DATA_API) + +def check_status_url(port): + '''get check_status url''' + return '{0}:{1}{2}{3}'.format(BASE_URL, port, API_ROOT_URL, CHECK_STATUS_API) + + +def cluster_metadata_url(port): + '''get cluster_metadata_url''' + return '{0}:{1}{2}{3}'.format(BASE_URL, port, API_ROOT_URL, CLUSTER_METADATA_API) + + +def import_data_url(port): + '''get import_data_url''' + return '{0}:{1}{2}{3}'.format(BASE_URL, port, API_ROOT_URL, IMPORT_DATA_API) + + +def experiment_url(port): + '''get experiment_url''' + return '{0}:{1}{2}{3}'.format(BASE_URL, port, API_ROOT_URL, EXPERIMENT_API) + + +def trial_jobs_url(port): + '''get trial_jobs url''' + return '{0}:{1}{2}{3}'.format(BASE_URL, port, API_ROOT_URL, TRIAL_JOBS_API) + + +def trial_job_id_url(port, job_id): + '''get trial_jobs with id url''' + return '{0}:{1}{2}{3}/{4}'.format(BASE_URL, port, API_ROOT_URL, TRIAL_JOBS_API, job_id) + + +def export_data_url(port): + '''get export_data url''' + return '{0}:{1}{2}{3}'.format(BASE_URL, port, API_ROOT_URL, EXPORT_DATA_API) + + +def tensorboard_url(port): + '''get tensorboard url''' + return '{0}:{1}{2}{3}'.format(BASE_URL, port, API_ROOT_URL, TENSORBOARD_API) + + +def get_local_urls(port,prefix): + '''get urls of local machine''' + url_list = [] + for _, info in psutil.net_if_addrs().items(): + for addr in info: + if socket.AddressFamily.AF_INET == addr.family: + url_list.append('http://{0}:{1}{2}'.format(addr.address, port, prefix)) + return url_list diff --git a/new_impl/cv/third_party/nni_new/tools/package_utils/__init__.py b/new_impl/cv/third_party/nni_new/tools/package_utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..7f8f4419651eaeddeb4043e8026a6443bff1fc2d --- /dev/null +++ b/new_impl/cv/third_party/nni_new/tools/package_utils/__init__.py @@ -0,0 +1,229 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from collections import defaultdict +import importlib +import os +from pathlib import Path +import sys +import yaml +from nni.runtime.config import get_config_file + +ALGO_TYPES = ['tuners', 'assessors', 'advisors'] + +def get_all_builtin_names(algo_type): + """Get all builtin names of registered algorithms of specified type + + Parameters + ---------- + algo_type: str + can be one of 'tuners', 'assessors' or 'advisors' + + Returns: list of string + ------- + All builtin names of specified type, for example, if algo_type is 'tuners', returns + all builtin tuner names. + """ + assert algo_type in ALGO_TYPES + + return [x['builtinName'] for x in read_registerd_algo_meta()[algo_type]] + + +def get_registered_algo_meta(builtin_name, algo_type=None): + """ Get meta information of registered algorithms. + + Parameters + ---------- + builtin_name: str + builtin name. + algo_type: str | None + can be one of 'tuners', 'assessors', 'advisors' or None + + Returns: dict | None + ------- + Returns meta information of speicified builtin alogorithms, for example: + { + 'classArgsValidator': 'nni.smac_tuner.SMACClassArgsValidator', + 'className': 'nni.smac_tuner.SMACTuner', + 'builtinName': 'SMAC' + } + """ + assert builtin_name is not None + if algo_type: + assert algo_type in ALGO_TYPES + config = read_registerd_algo_meta() + + candidates = [] + if algo_type: + candidates = config[algo_type] + else: + for algo_type in ALGO_TYPES: + candidates.extend(config[algo_type]) + for meta in candidates: + if meta['builtinName'] == builtin_name: + return meta + return None + +def parse_full_class_name(full_class_name): + if not full_class_name: + return None, None + parts = full_class_name.split('.') + module_name, class_name = '.'.join(parts[:-1]), parts[-1] + return module_name, class_name + +def get_builtin_module_class_name(algo_type, builtin_name): + """Get module name and class name of all builtin algorithms + + Parameters + ---------- + algo_type: str + can be one of 'tuners', 'assessors', 'advisors' + builtin_name: str + builtin name. + + Returns: tuple + ------- + tuple of (module name, class name) + """ + assert algo_type in ALGO_TYPES + assert builtin_name is not None + meta = get_registered_algo_meta(builtin_name, algo_type) + if not meta: + return None, None + return parse_full_class_name(meta['className']) + +def create_validator_instance(algo_type, builtin_name): + """Create instance of validator class + + Parameters + ---------- + algo_type: str + can be one of 'tuners', 'assessors', 'advisors' + builtin_name: str + builtin name. + + Returns: object | None + ------- + Returns validator class instance. + If specified validator class does not exist, returns None. + """ + assert algo_type in ALGO_TYPES + assert builtin_name is not None + meta = get_registered_algo_meta(builtin_name, algo_type) + if not meta or 'classArgsValidator' not in meta: + return None + module_name, class_name = parse_full_class_name(meta['classArgsValidator']) + class_module = importlib.import_module(module_name) + class_constructor = getattr(class_module, class_name) + + return class_constructor() + +def create_builtin_class_instance(builtin_name, input_class_args, algo_type): + """Create instance of builtin algorithms + + Parameters + ---------- + builtin_name: str + builtin name. + input_class_args: dict + kwargs for builtin class constructor + algo_type: str + can be one of 'tuners', 'assessors', 'advisors' + + Returns: object + ------- + Returns builtin class instance. + """ + assert algo_type in ALGO_TYPES + if builtin_name not in get_all_builtin_names(algo_type): + raise RuntimeError('Builtin name is not found: {}'.format(builtin_name)) + + def parse_algo_meta(algo_meta, input_class_args): + """ + 1. parse class_name field in meta data into module name and class name, + for example: + parse class_name 'nni.hyperopt_tuner.hyperopt_tuner.HyperoptTuner' in meta data into: + module name: nni.hyperopt_tuner.hyperopt_tuner + class name: HyperoptTuner + 2. merge user specified class args together with builtin class args. + """ + assert algo_meta + module_name, class_name = parse_full_class_name(algo_meta['className']) + + class_args = {} + if 'classArgs' in algo_meta: + class_args = algo_meta['classArgs'] + if input_class_args is not None: + class_args.update(input_class_args) + + return module_name, class_name, class_args + + algo_meta = get_registered_algo_meta(builtin_name, algo_type) + module_name, class_name, class_args = parse_algo_meta(algo_meta, input_class_args) + + if importlib.util.find_spec(module_name) is None: + raise RuntimeError('Builtin module can not be loaded: {}'.format(module_name)) + + class_module = importlib.import_module(module_name) + class_constructor = getattr(class_module, class_name) + + instance = class_constructor(**class_args) + + return instance + +def create_customized_class_instance(class_params): + """Create instance of customized algorithms + + Parameters + ---------- + class_params: dict + class_params should contains following keys: + codeDirectory: code directory + className: qualified class name + classArgs (optional): kwargs pass to class constructor + + Returns: object + ------- + Returns customized class instance. + """ + + code_dir = class_params.get('codeDirectory') + qualified_class_name = class_params.get('className') + class_args = class_params.get('classArgs') + + if code_dir and not os.path.isdir(code_dir): + raise ValueError(f'Directory not found: {code_dir}') + + sys.path.append(code_dir) + module_name, class_name = qualified_class_name.rsplit('.', 1) + class_module = importlib.import_module(module_name) + class_constructor = getattr(class_module, class_name) + + if class_args is None: + class_args = {} + instance = class_constructor(**class_args) + + return instance + +def _using_conda_or_virtual_environment(): + return sys.prefix != sys.base_prefix or os.path.isdir(os.path.join(sys.prefix, 'conda-meta')) + +def get_registered_algo_config_path(): + return str(get_config_file('registered_algorithms.yml')) + +def read_registerd_algo_meta(): + config_file = get_registered_algo_config_path() + if os.path.exists(config_file): + with open(config_file, 'r') as f: + config = yaml.safe_load(f) + else: + config = defaultdict(list) + for t in ALGO_TYPES: + if t not in config: + config[t] = [] + return config + +def write_registered_algo_meta(config): + config_file = get_registered_algo_config_path() + with open(config_file, 'w') as f: + f.write(yaml.safe_dump(dict(config), default_flow_style=False)) diff --git a/new_impl/cv/third_party/nni_new/tools/trial_tool/__init__.py b/new_impl/cv/third_party/nni_new/tools/trial_tool/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/new_impl/cv/third_party/nni_new/tools/trial_tool/aml_channel.py b/new_impl/cv/third_party/nni_new/tools/trial_tool/aml_channel.py new file mode 100644 index 0000000000000000000000000000000000000000..c8e1d7484a427b80b657d1c889c3a1cbe4aafecc --- /dev/null +++ b/new_impl/cv/third_party/nni_new/tools/trial_tool/aml_channel.py @@ -0,0 +1,47 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from azureml.core.run import Run # pylint: disable=import-error +from .base_channel import BaseChannel +from .log_utils import LogType, nni_log + + +class AMLChannel(BaseChannel): + def __init__(self, args): + self.args = args + self.run = Run.get_context() + super(AMLChannel, self).__init__(args) + self.current_message_index = -1 + + def _inner_open(self): + pass + + def _inner_close(self): + pass + + def _inner_send(self, message): + try: + self.run.log('trial_runner', message.decode('utf8')) + except Exception as exception: + nni_log(LogType.Error, 'meet unhandled exception when send message: %s' % exception) + + def _inner_receive(self): + messages = [] + message_dict = self.run.get_metrics() + if 'nni_manager' not in message_dict: + return [] + message_list = message_dict['nni_manager'] + if not message_list: + return messages + if type(message_list) is list: + if self.current_message_index < len(message_list) - 1: + messages = message_list[self.current_message_index + 1 : len(message_list)] + self.current_message_index = len(message_list) - 1 + elif self.current_message_index == -1: + messages = [message_list] + self.current_message_index += 1 + newMessage = [] + for message in messages: + # receive message is string, to get consistent result, encode it here. + newMessage.append(message.encode('utf8')) + return newMessage diff --git a/new_impl/cv/third_party/nni_new/tools/trial_tool/base_channel.py b/new_impl/cv/third_party/nni_new/tools/trial_tool/base_channel.py new file mode 100644 index 0000000000000000000000000000000000000000..b9d3392abc0b15ce7eb90b07df4107902fb65d9d --- /dev/null +++ b/new_impl/cv/third_party/nni_new/tools/trial_tool/base_channel.py @@ -0,0 +1,158 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import json +import threading +import time +from abc import ABC, abstractmethod +from queue import Empty, Queue + +from .log_utils import LogType, nni_log +from .commands import CommandType + +INTERVAL_SECONDS = 0.5 + + +class BaseChannel(ABC): + def __init__(self, args): + self.is_keep_parsed = args.node_count > 1 + self.args = args + self.node_id = self.args.node_id + + @abstractmethod + def _inner_send(self, message): + pass + + @abstractmethod + def _inner_receive(self): + return [] + + @abstractmethod + def _inner_open(self): + pass + + @abstractmethod + def _inner_close(self): + pass + + def open(self): + # initialize receive, send threads. + self.is_running = True + self.receive_queue = Queue() + self.receive_thread = threading.Thread(target=self._receive_loop) + self.receive_thread.start() + self.send_queue = Queue() + self.send_thread = threading.Thread(target=self._send_loop) + self.send_thread.start() + + self._inner_open() + + client_info = { + "isReady": True, + "runnerId": self.args.runner_id, + "expId": self.args.exp_id, + } + nni_log(LogType.Info, 'Channel: send ready information %s' % client_info) + self.send(CommandType.Initialized, client_info) + + def close(self): + self.is_running = False + try: + self._inner_close() + except Exception as err: + # ignore any error on closing + print("error on closing channel: %s" % err) + + def send(self, command, data): + """Send command to Training Service. + command: CommandType object. + data: string payload. + the message is sent synchronized. + """ + data["node"] = self.node_id + data = json.dumps(data) + data = data.encode('utf8') + message = b'%b%014d%b' % (command.value, len(data), data) + self.send_queue.put(message) + + def sent(self): + return self.send_queue.qsize() == 0 + + def received(self): + return self.receive_queue.qsize() > 0 + + def receive(self): + """Receive a command from Training Service. + Returns a tuple of command (CommandType) and payload (str) + """ + command = None + data = None + + try: + command_content = self.receive_queue.get(False) + if command_content is not None: + if (len(command_content) < 16): + # invalid header + nni_log(LogType.Error, 'incorrect command is found, command must be greater than 16 bytes!') + return None, None + header = command_content[:16] + command = CommandType(header[:2]) + length = int(header[2:]) + if (len(command_content)-16 != length): + nni_log(LogType.Error, 'incorrect command length, length {}, actual data length is {}, header {}.' + .format(length, len(command_content)-16, header)) + return None, None + data = command_content[16:16+length] + data = json.loads(data.decode('utf8')) + if self.node_id is None: + nni_log(LogType.Info, 'Received command, header: [%s], data: [%s]' % (header, data)) + else: + nni_log(LogType.Info, 'Received command(%s), header: [%s], data: [%s]' % (self.node_id, header, data)) + except Empty: + # do nothing, if no command received. + pass + except Exception as identifier: + nni_log(LogType.Error, 'meet unhandled exception in base_channel: %s' % identifier) + return command, data + + def _fetch_message(self, buffer, has_new_line=False): + messages = [] + while(len(buffer)) >= 16: + header = buffer[:16] + length = int(header[2:]) + + message_length = length+16 + total_length = message_length + if has_new_line: + total_length += 1 + + # break, if buffer is too short. + if len(buffer) < total_length: + break + data = buffer[16:message_length] + if has_new_line and 10 != buffer[total_length-1]: + nni_log(LogType.Error, 'end of message should be \\n, but got {}'.format(self.in_cache[total_length-1])) + buffer = buffer[total_length:] + messages.append(header + data) + + return messages, buffer + + def _receive_loop(self): + while (self.is_running): + messages = self._inner_receive() + if messages is not None: + for message in messages: + self.receive_queue.put(message) + time.sleep(INTERVAL_SECONDS) + + def _send_loop(self): + while (self.is_running): + message = None + try: + # no sleep, since it's a block call with INTERVAL_SECONDS second timeout + message = self.send_queue.get(True, INTERVAL_SECONDS) + except Empty: + # do nothing, if no command received. + pass + if message is not None: + self._inner_send(message) diff --git a/new_impl/cv/third_party/nni_new/tools/trial_tool/commands.py b/new_impl/cv/third_party/nni_new/tools/trial_tool/commands.py new file mode 100644 index 0000000000000000000000000000000000000000..86b10a2fe9a85d17bdfb4d9ec57b0b6cceb250da --- /dev/null +++ b/new_impl/cv/third_party/nni_new/tools/trial_tool/commands.py @@ -0,0 +1,25 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from enum import Enum + + +class CommandType(Enum): + Initialize = b'IN' + RequestTrialJobs = b'GE' + ReportMetricData = b'ME' + ReportGpuInfo = b'GI' + UpdateSearchSpace = b'SS' + ImportData = b'FD' + AddCustomizedTrialJob = b'AD' + TrialEnd = b'EN' + Terminate = b'TE' + Ping = b'PI' + + Initialized = b'ID' + NewTrialJob = b'TR' + SendTrialJobParameter = b'SP' + NoMoreTrialJobs = b'NO' + KillTrialJob = b'KI' + StdOut = b'SO' + VersionCheck = b'VC' diff --git a/new_impl/cv/third_party/nni_new/tools/trial_tool/constants.py b/new_impl/cv/third_party/nni_new/tools/trial_tool/constants.py new file mode 100644 index 0000000000000000000000000000000000000000..bef401337039d46aa23fc1e5816b58978d07e104 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/tools/trial_tool/constants.py @@ -0,0 +1,24 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os + +API_ROOT_URL = '/api/v1/nni-pai' + +BASE_URL = 'http://{}' + +LOG_DIR = os.environ['NNI_OUTPUT_DIR'] + +NNI_PLATFORM = os.environ['NNI_PLATFORM'] + +STDOUT_FULL_PATH = os.path.join(LOG_DIR, 'stdout') + +STDERR_FULL_PATH = os.path.join(LOG_DIR, 'stderr') + +STDOUT_API = '/stdout' +VERSION_API = '/version' +PARAMETER_META_API = '/parameter-file-meta' +NNI_SYS_DIR = os.environ['NNI_SYS_DIR'] +NNI_TRIAL_JOB_ID = os.environ['NNI_TRIAL_JOB_ID'] +NNI_EXP_ID = os.environ['NNI_EXP_ID'] +MULTI_PHASE = os.environ['MULTI_PHASE'] diff --git a/new_impl/cv/third_party/nni_new/tools/trial_tool/file_channel.py b/new_impl/cv/third_party/nni_new/tools/trial_tool/file_channel.py new file mode 100644 index 0000000000000000000000000000000000000000..9a431d25f7ee3a4eeff973667d345ce2ddada1cb --- /dev/null +++ b/new_impl/cv/third_party/nni_new/tools/trial_tool/file_channel.py @@ -0,0 +1,75 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os + +from .base_channel import BaseChannel + +command_path = "./commands" +runner_commands_file_name_prefix = "runner_commands" +manager_commands_file_name = "manager_commands.txt" + + +class FileChannel(BaseChannel): + + def __init__(self, args): + self.node_id = args.node_id + self.out_file = None + self.in_file = None + self.in_offset = 0 + self.in_cache = b"" + + super(FileChannel, self).__init__(args) + + def _inner_open(self): + pass + + def _inner_close(self): + if self.out_file is not None: + self.out_file.close() + self.out_file = None + if self.in_file is not None: + self.in_file.close() + self.in_file = None + + def _inner_send(self, message): + if self.out_file is None: + if not os.path.exists(command_path): + os.makedirs(command_path, exist_ok=True) + + if self.node_id is None: + file_name = os.path.join(command_path, "%s.txt" % runner_commands_file_name_prefix) + else: + file_name = os.path.join(command_path, "%s_%s.txt" % ( + runner_commands_file_name_prefix, self.node_id)) + self.out_file = open(file_name, "ab") + + self.out_file.write(message) + self.out_file.write(b'\n') + self.out_file.flush() + + def _open_manager_command(self): + full_name = os.path.join(command_path, manager_commands_file_name) + + if self.in_file is not None and self.in_file.closed: + self.in_file = None + + if self.in_file is None and os.path.exists(full_name): + self.in_file = open(full_name, "rb") + self.in_file.seek(self.in_offset) + + def _inner_receive(self): + messages = [] + + if self.in_file is None: + self._open_manager_command() + if self.in_file is not None: + self.in_file.seek(0, os.SEEK_END) + new_offset = self.in_file.tell() + self.in_file.seek(self.in_offset, os.SEEK_SET) + count = new_offset - self.in_offset + if count > 0: + self.in_cache += self.in_file.read(count) + self.in_offset = new_offset + messages, self.in_cache = self._fetch_message(self.in_cache, True) + return messages diff --git a/new_impl/cv/third_party/nni_new/tools/trial_tool/gpu.py b/new_impl/cv/third_party/nni_new/tools/trial_tool/gpu.py new file mode 100644 index 0000000000000000000000000000000000000000..48dab4b182a6ecd08b1108c297d206663ce9bb40 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/tools/trial_tool/gpu.py @@ -0,0 +1,69 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import subprocess +import time +import traceback +from xml.dom import minidom + + +def collect_gpu_usage(node_id): + cmd = 'nvidia-smi -q -x'.split() + info = None + try: + smi_output = subprocess.check_output(cmd) + info = parse_nvidia_smi_result(smi_output) + except Exception: + traceback.print_exc() + info = gen_empty_gpu_metric() + return info + + +def parse_nvidia_smi_result(smi): + try: + output = {} + xmldoc = minidom.parseString(smi) + gpuList = xmldoc.getElementsByTagName('gpu') + output["Timestamp"] = time.asctime(time.localtime()) + output["gpuCount"] = len(gpuList) + output["gpuInfos"] = [] + for gpuIndex, gpu in enumerate(gpuList): + gpuInfo = {} + gpuInfo['index'] = gpuIndex + gpuInfo['gpuUtil'] = gpu.getElementsByTagName('utilization')[0]\ + .getElementsByTagName('gpu_util')[0]\ + .childNodes[0].data.replace("%", "").strip() + gpuInfo['gpuMemUtil'] = gpu.getElementsByTagName('utilization')[0]\ + .getElementsByTagName('memory_util')[0]\ + .childNodes[0].data.replace("%", "").strip() + processes = gpu.getElementsByTagName('processes') + runningProNumber = len(processes[0].getElementsByTagName('process_info')) + gpuInfo['activeProcessNum'] = runningProNumber + + gpuInfo['gpuType'] = gpu.getElementsByTagName('product_name')[0]\ + .childNodes[0].data + memUsage = gpu.getElementsByTagName('fb_memory_usage')[0] + gpuInfo['gpuMemTotal'] = memUsage.getElementsByTagName('total')[0]\ + .childNodes[0].data.replace("MiB", "").strip() + gpuInfo['gpuMemUsed'] = memUsage.getElementsByTagName('used')[0]\ + .childNodes[0].data.replace("MiB", "").strip() + gpuInfo['gpuMemFree'] = memUsage.getElementsByTagName('free')[0]\ + .childNodes[0].data.replace("MiB", "").strip() + + output["gpuInfos"].append(gpuInfo) + except Exception: + traceback.print_exc() + output = {} + return output + + +def gen_empty_gpu_metric(): + try: + output = {} + output["Timestamp"] = time.asctime(time.localtime()) + output["gpuCount"] = 0 + output["gpuInfos"] = [] + except Exception: + traceback.print_exc() + output = {} + return output diff --git a/new_impl/cv/third_party/nni_new/tools/trial_tool/hdfsClientUtility.py b/new_impl/cv/third_party/nni_new/tools/trial_tool/hdfsClientUtility.py new file mode 100644 index 0000000000000000000000000000000000000000..05d4ea0d85675b5c763bf27f29db50b7900e1d03 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/tools/trial_tool/hdfsClientUtility.py @@ -0,0 +1,92 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +import posixpath +from .log_utils import LogType, nni_log + +def copyHdfsDirectoryToLocal(hdfsDirectory, localDirectory, hdfsClient): + '''Copy directory from HDFS to local''' + if not os.path.exists(localDirectory): + os.makedirs(localDirectory) + try: + listing = hdfsClient.list_status(hdfsDirectory) + except Exception as exception: + nni_log(LogType.Error, 'List hdfs directory {0} error: {1}'.format(hdfsDirectory, str(exception))) + raise exception + + for f in listing: + if f.type == 'DIRECTORY': + subHdfsDirectory = posixpath.join(hdfsDirectory, f.pathSuffix) + subLocalDirectory = os.path.join(localDirectory, f.pathSuffix) + copyHdfsDirectoryToLocal(subHdfsDirectory, subLocalDirectory, hdfsClient) + elif f.type == 'FILE': + hdfsFilePath = posixpath.join(hdfsDirectory, f.pathSuffix) + localFilePath = os.path.join(localDirectory, f.pathSuffix) + copyHdfsFileToLocal(hdfsFilePath, localFilePath, hdfsClient) + else: + raise AssertionError('unexpected type {}'.format(f.type)) + +def copyHdfsFileToLocal(hdfsFilePath, localFilePath, hdfsClient, override=True): + '''Copy file from HDFS to local''' + if not hdfsClient.exists(hdfsFilePath): + raise Exception('HDFS file {} does not exist!'.format(hdfsFilePath)) + try: + file_status = hdfsClient.get_file_status(hdfsFilePath) + if file_status.type != 'FILE': + raise Exception('HDFS file path {} is not a file'.format(hdfsFilePath)) + except Exception as exception: + nni_log(LogType.Error, 'Get hdfs file {0} status error: {1}'.format(hdfsFilePath, str(exception))) + raise exception + + if os.path.exists(localFilePath) and override: + os.remove(localFilePath) + try: + hdfsClient.copy_to_local(hdfsFilePath, localFilePath) + except Exception as exception: + nni_log(LogType.Error, 'Copy hdfs file {0} to {1} error: {2}'.format(hdfsFilePath, localFilePath, str(exception))) + raise exception + nni_log(LogType.Info, 'Successfully copied hdfs file {0} to {1}, {2} bytes'.format(hdfsFilePath, localFilePath, file_status.length)) + +def copyDirectoryToHdfs(localDirectory, hdfsDirectory, hdfsClient): + '''Copy directory from local to HDFS''' + if not os.path.exists(localDirectory): + raise Exception('Local Directory does not exist!') + hdfsClient.mkdirs(hdfsDirectory) + result = True + for file in os.listdir(localDirectory): + file_path = os.path.join(localDirectory, file) + if os.path.isdir(file_path): + hdfs_directory = os.path.join(hdfsDirectory, file) + try: + result = result and copyDirectoryToHdfs(file_path, hdfs_directory, hdfsClient) + except Exception as exception: + nni_log(LogType.Error, + 'Copy local directory {0} to hdfs directory {1} error: {2}'.format(file_path, hdfs_directory, str(exception))) + result = False + else: + hdfs_file_path = os.path.join(hdfsDirectory, file) + try: + result = result and copyFileToHdfs(file_path, hdfs_file_path, hdfsClient) + except Exception as exception: + nni_log(LogType.Error, 'Copy local file {0} to hdfs {1} error: {2}'.format(file_path, hdfs_file_path, str(exception))) + result = False + return result + +def copyFileToHdfs(localFilePath, hdfsFilePath, hdfsClient, override=True): + '''Copy a local file to HDFS directory''' + if not os.path.exists(localFilePath): + raise Exception('Local file Path does not exist!') + if os.path.isdir(localFilePath): + raise Exception('localFile should not a directory!') + if hdfsClient.exists(hdfsFilePath): + if override: + hdfsClient.delete(hdfsFilePath) + else: + return False + try: + hdfsClient.copy_from_local(localFilePath, hdfsFilePath) + return True + except Exception as exception: + nni_log(LogType.Error, 'Copy local file {0} to hdfs file {1} error: {2}'.format(localFilePath, hdfsFilePath, str(exception))) + return False diff --git a/new_impl/cv/third_party/nni_new/tools/trial_tool/log_utils.py b/new_impl/cv/third_party/nni_new/tools/trial_tool/log_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..8d5b3d94c0b60d60b978b94d967f092bfa9a1c56 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/tools/trial_tool/log_utils.py @@ -0,0 +1,219 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +import sys +import json +import logging +import logging.handlers +import time +import threading +import re + +from datetime import datetime +from enum import Enum, unique +from logging import StreamHandler + +from queue import Queue + +from .rest_utils import rest_post +from .url_utils import gen_send_stdout_url +from .commands import CommandType + + +@unique +class LogType(Enum): + Trace = 'TRACE' + Debug = 'DEBUG' + Info = 'INFO' + Warning = 'WARNING' + Error = 'ERROR' + Fatal = 'FATAL' + + +@unique +class StdOutputType(Enum): + Stdout = 'stdout', + Stderr = 'stderr' + + +def nni_log(log_type, log_message): + '''Log message into stdout''' + dt = datetime.now() + print('[{0}] {1} {2}'.format(dt, log_type.value, log_message), flush=True) + + +class NNIRestLogHanlder(StreamHandler): + def __init__(self, host, port, tag, trial_id, channel, std_output_type=StdOutputType.Stdout): + StreamHandler.__init__(self) + self.host = host + self.port = port + self.tag = tag + self.std_output_type = std_output_type + self.trial_id = trial_id + self.channel = channel + self.orig_stdout = sys.__stdout__ + self.orig_stderr = sys.__stderr__ + + def emit(self, record): + log_entry = {} + log_entry['tag'] = self.tag + log_entry['stdOutputType'] = self.std_output_type.name + log_entry['msg'] = self.format(record) + + try: + if self.channel is None: + rest_post(gen_send_stdout_url(self.host, self.port), json.dumps(log_entry), 10, True) + else: + if self.trial_id is not None: + log_entry["trial"] = self.trial_id + self.channel.send(CommandType.StdOut, log_entry) + except Exception as e: + self.orig_stderr.write(str(e) + '\n') + self.orig_stderr.flush() + + +class RemoteLogger(object): + """ + NNI remote logger + """ + + def __init__(self, syslog_host, syslog_port, tag, std_output_type, log_collection, trial_id=None, channel=None, log_level=logging.INFO): + ''' + constructor + ''' + logger_name = 'nni_syslog_{}'.format(tag) + # to prevent multiple trial logged in same logger + if trial_id is not None: + logger_name = '{}_{}'.format(logger_name, trial_id) + self.logger = logging.getLogger(logger_name) + self.log_level = log_level + self.logger.setLevel(self.log_level) + self.pipeReader = None + self.handler = NNIRestLogHanlder(syslog_host, syslog_port, tag, trial_id, channel) + self.logger.addHandler(self.handler) + if std_output_type == StdOutputType.Stdout: + self.orig_stdout = sys.__stdout__ + else: + self.orig_stdout = sys.__stderr__ + self.log_collection = log_collection + + def get_pipelog_reader(self): + ''' + Get pipe for remote logger + ''' + self.pipeReader = PipeLogReader(self.logger, self.log_collection, logging.INFO) + return self.pipeReader + + def flush(self): + ''' + Add flush in handler + ''' + for handler in self.logger.handlers: + handler.flush() + + def write(self, buf): + ''' + Write buffer data into logger/stdout + ''' + for line in buf.rstrip().splitlines(): + self.orig_stdout.write(line.rstrip() + '\n') + self.orig_stdout.flush() + try: + self.logger.log(self.log_level, line.rstrip()) + except Exception: + pass + + def close(self): + ''' + Close handlers and resources + ''' + if self.pipeReader is not None: + self.pipeReader.set_process_exit() + for handler in self.logger.handlers: + handler.close() + self.logger.removeHandler(handler) + + +class PipeLogReader(threading.Thread): + """ + The reader thread reads log data from pipe + """ + + def __init__(self, logger, log_collection, log_level=logging.INFO): + """Setup the object with a logger and a loglevel + and start the thread + """ + threading.Thread.__init__(self) + self.queue = Queue() + self.logger = logger + self.daemon = False + self.log_level = log_level + self.fdRead, self.fdWrite = os.pipe() + self.pipeReader = os.fdopen(self.fdRead) + self.orig_stdout = sys.__stdout__ + self._is_read_completed = False + self.process_exit = False + self.log_collection = log_collection + self.log_pattern = re.compile(r'NNISDK_MEb\'.*\'$') + + def _populateQueue(stream, queue): + ''' + Collect lines from 'stream' and put them in 'quque'. + ''' + time.sleep(1) + while True: + cur_process_exit = self.process_exit + try: + line = self.queue.get(True, 5) + try: + self.logger.log(self.log_level, line.rstrip()) + except Exception: + pass + except Exception: + if cur_process_exit == True: + self._is_read_completed = True + break + + self.pip_log_reader_thread = threading.Thread(target=_populateQueue, args=(self.pipeReader, self.queue)) + self.pip_log_reader_thread.daemon = True + self.start() + self.pip_log_reader_thread.start() + + def fileno(self): + """Return the write file descriptor of the pipe + """ + return self.fdWrite + + def run(self): + """Run the thread, logging everything. + If the log_collection is 'none', the log content will not be enqueued + """ + for line in iter(self.pipeReader.readline, ''): + self.orig_stdout.write(line.rstrip() + '\n') + self.orig_stdout.flush() + + if self.log_collection == 'none': + search_result = self.log_pattern.search(line) + if search_result: + metrics = search_result.group(0) + self.queue.put(metrics+'\n') + else: + self.queue.put(line) + + self.pipeReader.close() + + def close(self): + """Close the write end of the pipe. + """ + os.close(self.fdWrite) + + @property + def is_read_completed(self): + """Return if read is completed + """ + return self._is_read_completed + + def set_process_exit(self): + self.process_exit = True + return self.process_exit diff --git a/new_impl/cv/third_party/nni_new/tools/trial_tool/rest_utils.py b/new_impl/cv/third_party/nni_new/tools/trial_tool/rest_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..959209c7470bdb5dfe1ca32af91ba85757dd51e0 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/tools/trial_tool/rest_utils.py @@ -0,0 +1,44 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import requests + +def rest_get(url, timeout): + '''Call rest get method''' + try: + response = requests.get(url, timeout=timeout) + return response + except Exception as e: + print('Get exception {0} when sending http get to url {1}'.format(str(e), url)) + return None + +def rest_post(url, data, timeout, rethrow_exception=False): + '''Call rest post method''' + try: + response = requests.post(url, headers={'Accept': 'application/json', 'Content-Type': 'application/json'},\ + data=data, timeout=timeout) + return response + except Exception as e: + if rethrow_exception is True: + raise + print('Get exception {0} when sending http post to url {1}'.format(str(e), url)) + return None + +def rest_put(url, data, timeout): + '''Call rest put method''' + try: + response = requests.put(url, headers={'Accept': 'application/json', 'Content-Type': 'application/json'},\ + data=data, timeout=timeout) + return response + except Exception as e: + print('Get exception {0} when sending http put to url {1}'.format(str(e), url)) + return None + +def rest_delete(url, timeout): + '''Call rest delete method''' + try: + response = requests.delete(url, timeout=timeout) + return response + except Exception as e: + print('Get exception {0} when sending http delete to url {1}'.format(str(e), url)) + return None diff --git a/new_impl/cv/third_party/nni_new/tools/trial_tool/trial.py b/new_impl/cv/third_party/nni_new/tools/trial_tool/trial.py new file mode 100644 index 0000000000000000000000000000000000000000..1da398d017ecb98e36508a1683179534e68c1185 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/tools/trial_tool/trial.py @@ -0,0 +1,163 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import ctypes +import os +import sys +import shlex +import tarfile +import time +from datetime import datetime +from subprocess import Popen + +import psutil + +from .log_utils import LogType, RemoteLogger, StdOutputType, nni_log +from .commands import CommandType + +trial_output_path_name = ".nni" + + +class Trial: + def __init__(self, args, data): + self.process = None + self.data = data + self.args = args + self.command_channel = args.command_channel + self.trial_syslogger_stdout = None + + global NNI_TRIAL_JOB_ID + self.id = data["trialId"] + if self.id is None: + raise Exception("trial_id is not found in %s" % data) + os.environ['NNI_TRIAL_JOB_ID'] = self.id + NNI_TRIAL_JOB_ID = self.id + + # for multiple nodes. If it's None, it means single node. + self.node_id = args.node_id + if self.node_id is None: + self.name = self.id + else: + self.name = "%s_%s" % (self.id, self.node_id) + + def run(self): + # redirect trial's stdout and stderr to syslog + self.trial_syslogger_stdout = RemoteLogger(self.args.nnimanager_ip, self.args.nnimanager_port, 'trial', StdOutputType.Stdout, + self.args.log_collection, self.id, self.args.command_channel) + + nni_log(LogType.Info, "%s: start to run trial" % self.name) + + trial_working_dir = os.path.realpath(os.path.join(os.curdir, "..", "..", "trials", self.id)) + self.trial_output_dir = os.path.join(trial_working_dir, trial_output_path_name) + trial_code_dir = os.path.join(trial_working_dir, "code") + trial_nnioutput_dir = os.path.join(trial_working_dir, "nnioutput") + + environ = os.environ.copy() + environ['NNI_TRIAL_SEQ_ID'] = str(self.data["sequenceId"]) + environ['NNI_OUTPUT_DIR'] = os.path.join(trial_working_dir, "nnioutput") + environ['NNI_SYS_DIR'] = trial_working_dir + self.working_dir = trial_working_dir + + # prepare code and parameters + prepared_flag_file_name = os.path.join(trial_working_dir, "trial_prepared") + if not os.path.exists(trial_working_dir): + os.makedirs(trial_working_dir, exist_ok=True) + + os.makedirs(self.trial_output_dir, exist_ok=True) + os.makedirs(trial_nnioutput_dir, exist_ok=True) + # prepare code + os.makedirs(trial_code_dir, exist_ok=True) + with tarfile.open(os.path.join("..", "nni-code.tar.gz"), "r:gz") as tar: + tar.extractall(trial_code_dir) + + # save parameters + nni_log(LogType.Info, '%s: saving parameter %s' % (self.name, self.data["parameter"]["value"])) + parameter_file_name = os.path.join(trial_working_dir, "parameter.cfg") + with open(parameter_file_name, "w") as parameter_file: + parameter_file.write(self.data["parameter"]["value"]) + + # ready flag + with open(prepared_flag_file_name, "w") as prepared_flag_file: + prepared_flag_file.write("%s" % (int(datetime.now().timestamp() * 1000))) + + # make sure code prepared by other node. + if self.node_id is not None: + while True: + if os.path.exists(prepared_flag_file_name): + break + time.sleep(0.1) + + trial_command = self.args.trial_command + + gpuIndices = self.data.get("gpuIndices") + if (gpuIndices is not None): + if sys.platform == "win32": + trial_command = 'set CUDA_VISIBLE_DEVICES="%s " && call %s' % (gpuIndices, trial_command) + else: + trial_command = 'CUDA_VISIBLE_DEVICES="%s " %s' % (gpuIndices, trial_command) + + self.log_pipe_stdout = self.trial_syslogger_stdout.get_pipelog_reader() + self.process = Popen(trial_command, shell=True, stdout=self.log_pipe_stdout, + stderr=self.log_pipe_stdout, cwd=trial_code_dir, env=dict(environ)) + nni_log(LogType.Info, '{0}: spawns a subprocess (pid {1}) to run command: {2}'. + format(self.name, self.process.pid, shlex.split(trial_command))) + + def save_parameter_file(self, command_data): + parameters = command_data["parameters"] + file_index = int(parameters["index"]) + if file_index == 0: + parameter_file_name = "parameter.cfg" + else: + parameter_file_name = "parameter_{}.cfg".format(file_index) + parameter_file_name = os.path.join(self.working_dir, parameter_file_name) + with open(parameter_file_name, "w") as parameter_file: + nni_log(LogType.Info, '%s: saving parameter %s' % (self.name, parameters["value"])) + parameter_file.write(parameters["value"]) + + def is_running(self): + if (self.process is None): + return False + + retCode = self.process.poll() + # child worker process exits and all stdout data is read + if retCode is not None and self.log_pipe_stdout.set_process_exit() and self.log_pipe_stdout.is_read_completed == True: + # In Windows, the retCode -1 is 4294967295. It's larger than c_long, and raise OverflowError. + # So covert it to int32. + retCode = ctypes.c_long(retCode).value + nni_log(LogType.Info, '{0}: subprocess terminated. Exit code is {1}.'.format(self.name, retCode)) + + end_time = int(datetime.now().timestamp() * 1000) + end_message = { + "code": retCode, + "time": end_time, + "trial": self.id, + } + self.command_channel.send(CommandType.TrialEnd, end_message) + self.cleanup() + return False + else: + return True + + def kill(self, trial_id=None): + if trial_id == self.id or trial_id is None: + if self.process is not None: + try: + nni_log(LogType.Info, "%s: killing trial" % self.name) + for child in psutil.Process(self.process.pid).children(True): + child.kill() + self.process.kill() + except psutil.NoSuchProcess: + nni_log(LogType.Info, "kill trial %s failed: %s does not exist!" % (trial_id, self.process.pid)) + except Exception as ex: + nni_log(LogType.Error, "kill trial %s failed: %s " % (trial_id, str(ex))) + self.cleanup() + + def cleanup(self): + nni_log(LogType.Info, "%s: clean up trial" % self.name) + self.process = None + if self.log_pipe_stdout is not None: + self.log_pipe_stdout.set_process_exit() + self.log_pipe_stdout = None + if self.trial_syslogger_stdout is not None: + self.trial_syslogger_stdout.close() + self.trial_syslogger_stdout = None diff --git a/new_impl/cv/third_party/nni_new/tools/trial_tool/trial_keeper.py b/new_impl/cv/third_party/nni_new/tools/trial_tool/trial_keeper.py new file mode 100644 index 0000000000000000000000000000000000000000..65ea160a275dcbd168619294bd89411db30f41a6 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/tools/trial_tool/trial_keeper.py @@ -0,0 +1,264 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import argparse +import ctypes +import json +import logging +import os +import re +import shlex +import sys +import threading +import time +from subprocess import Popen + +import pkg_resources +from pyhdfs import HdfsClient + +from .constants import (LOG_DIR, MULTI_PHASE, NNI_EXP_ID, NNI_PLATFORM, + NNI_SYS_DIR, NNI_TRIAL_JOB_ID) +from .hdfsClientUtility import (copyDirectoryToHdfs, copyHdfsDirectoryToLocal, + copyHdfsFileToLocal) +from .log_utils import LogType, RemoteLogger, StdOutputType, nni_log +from .rest_utils import rest_get, rest_post +from .url_utils import gen_parameter_meta_url, gen_send_version_url + +logger = logging.getLogger('trial_keeper') +regular = re.compile('v?(?P[0-9](\.[0-9]){0,1}).*') + +_hdfs_client = None +_trial_process = None + + +def get_hdfs_client(args): + global _hdfs_client + + if _hdfs_client is not None: + return _hdfs_client + # backward compatibility + hdfs_host = None + + if args.hdfs_host: + hdfs_host = args.hdfs_host + elif args.pai_hdfs_host: + hdfs_host = args.pai_hdfs_host + else: + return None + + if hdfs_host is not None and args.nni_hdfs_exp_dir is not None: + try: + if args.webhdfs_path: + _hdfs_client = HdfsClient(hosts='{0}:80'.format(hdfs_host), user_name=args.pai_user_name, + webhdfs_path=args.webhdfs_path, timeout=5) + else: + # backward compatibility + _hdfs_client = HdfsClient(hosts='{0}:{1}'.format(hdfs_host, '50070'), user_name=args.pai_user_name, + timeout=5) + except Exception as e: + nni_log(LogType.Error, 'Create HDFS client error: ' + str(e)) + raise e + return _hdfs_client + + +def main_loop(args): + '''main loop logic for trial keeper''' + global _trial_process + + if not os.path.exists(LOG_DIR): + os.makedirs(LOG_DIR) + + trial_keeper_syslogger = RemoteLogger(args.nnimanager_ip, args.nnimanager_port, 'trial_keeper', + StdOutputType.Stdout, args.log_collection) + # redirect trial keeper's stdout and stderr to syslog + trial_syslogger_stdout = RemoteLogger(args.nnimanager_ip, args.nnimanager_port, 'trial', StdOutputType.Stdout, + args.log_collection) + sys.stdout = sys.stderr = trial_keeper_syslogger + hdfs_output_dir = None + + if args.hdfs_output_dir: + hdfs_output_dir = args.hdfs_output_dir + elif args.pai_hdfs_output_dir: + hdfs_output_dir = args.pai_hdfs_output_dir + + hdfs_client = get_hdfs_client(args) + + if hdfs_client is not None: + copyHdfsDirectoryToLocal(args.nni_hdfs_exp_dir, os.getcwd(), hdfs_client) + + if args.job_id_file: + with open(args.job_id_file, 'w') as job_file: + job_file.write("%d" % os.getpid()) + + # Notice: We don't appoint env, which means subprocess wil inherit current environment and that is expected behavior + log_pipe_stdout = trial_syslogger_stdout.get_pipelog_reader() + if sys.platform == 'win32': + _trial_process = Popen(args.trial_command, shell=True, stdout=log_pipe_stdout, stderr=log_pipe_stdout) + else: + _trial_process = Popen(args.trial_command, shell=True, stdout=log_pipe_stdout, stderr=log_pipe_stdout, preexec_fn=os.setsid) + nni_log(LogType.Info, 'Trial keeper spawns a subprocess (pid {0}) to run command: {1}'.format(_trial_process.pid, + shlex.split( + args.trial_command))) + + while True: + retCode = _trial_process.poll() + # child worker process exits and all stdout data is read + if retCode is not None and log_pipe_stdout.set_process_exit() and log_pipe_stdout.is_read_completed == True: + # In Windows, the retCode -1 is 4294967295. It's larger than c_long, and raise OverflowError. + # So covert it to int32. + retCode = ctypes.c_long(retCode).value + nni_log(LogType.Info, 'subprocess terminated. Exit code is {}. Quit'.format(retCode)) + if hdfs_output_dir is not None: + # Copy local directory to hdfs for OpenPAI + nni_local_output_dir = os.environ['NNI_OUTPUT_DIR'] + try: + if copyDirectoryToHdfs(nni_local_output_dir, hdfs_output_dir, hdfs_client): + nni_log(LogType.Info, + 'copy directory from {0} to {1} success!'.format(nni_local_output_dir, hdfs_output_dir)) + else: + nni_log(LogType.Info, + 'copy directory from {0} to {1} failed!'.format(nni_local_output_dir, hdfs_output_dir)) + except Exception as e: + nni_log(LogType.Error, 'HDFS copy directory got exception: ' + str(e)) + raise e + + # Exit as the retCode of subprocess(trial) + exit(retCode) + break + + time.sleep(2) + + +def trial_keeper_help_info(*args): + print('please run --help to see guidance') + + +def check_version(args): + try: + trial_keeper_version = pkg_resources.get_distribution('nni').version + except pkg_resources.ResolutionError as err: + # package nni does not exist, try nni-tool package + nni_log(LogType.Error, 'Package nni does not exist!') + os._exit(1) + if not args.nni_manager_version: + # skip version check + nni_log(LogType.Warning, 'Skipping version check!') + else: + try: + trial_keeper_version = regular.search(trial_keeper_version).group('version') + nni_log(LogType.Info, 'trial_keeper_version is {0}'.format(trial_keeper_version)) + nni_manager_version = regular.search(args.nni_manager_version).group('version') + nni_log(LogType.Info, 'nni_manager_version is {0}'.format(nni_manager_version)) + log_entry = {} + if trial_keeper_version != nni_manager_version: + nni_log(LogType.Warning, 'Version does not match!') + error_message = 'NNIManager version is {0}, TrialKeeper version is {1}, NNI version does not match!'.format( + nni_manager_version, trial_keeper_version) + log_entry['tag'] = 'VCFail' + log_entry['msg'] = error_message + rest_post(gen_send_version_url(args.nnimanager_ip, args.nnimanager_port), json.dumps(log_entry), 10, + False) + else: + nni_log(LogType.Info, 'Version match!') + log_entry['tag'] = 'VCSuccess' + rest_post(gen_send_version_url(args.nnimanager_ip, args.nnimanager_port), json.dumps(log_entry), 10, + False) + except AttributeError as err: + nni_log(LogType.Error, err) + + +def is_multi_phase(): + return MULTI_PHASE and (MULTI_PHASE in ['True', 'true']) + + +def download_parameter(meta_list, args): + """ + Download parameter file to local working directory. + meta_list format is defined in paiJobRestServer.ts + example meta_list: + [ + {"experimentId":"yWFJarYa","trialId":"UpPkl","filePath":"/chec/nni/experiments/yWFJarYa/trials/UpPkl/parameter_1.cfg"}, + {"experimentId":"yWFJarYa","trialId":"aIUMA","filePath":"/chec/nni/experiments/yWFJarYa/trials/aIUMA/parameter_1.cfg"} + ] + """ + nni_log(LogType.Debug, str(meta_list)) + nni_log(LogType.Debug, + 'NNI_SYS_DIR: {}, trial Id: {}, experiment ID: {}'.format(NNI_SYS_DIR, NNI_TRIAL_JOB_ID, NNI_EXP_ID)) + nni_log(LogType.Debug, 'NNI_SYS_DIR files: {}'.format(os.listdir(NNI_SYS_DIR))) + for meta in meta_list: + if meta['experimentId'] == NNI_EXP_ID and meta['trialId'] == NNI_TRIAL_JOB_ID: + param_fp = os.path.join(NNI_SYS_DIR, os.path.basename(meta['filePath'])) + if not os.path.exists(param_fp): + hdfs_client = get_hdfs_client(args) + copyHdfsFileToLocal(meta['filePath'], param_fp, hdfs_client, override=False) + + +def fetch_parameter_file(args): + class FetchThread(threading.Thread): + def __init__(self, args): + super(FetchThread, self).__init__() + self.args = args + + def run(self): + uri = gen_parameter_meta_url(self.args.nnimanager_ip, self.args.nnimanager_port) + nni_log(LogType.Info, uri) + + while True: + res = rest_get(uri, 10) + nni_log(LogType.Debug, 'status code: {}'.format(res.status_code)) + if res.status_code == 200: + meta_list = res.json() + download_parameter(meta_list, self.args) + else: + nni_log(LogType.Warning, 'rest response: {}'.format(str(res))) + time.sleep(5) + + fetch_file_thread = FetchThread(args) + fetch_file_thread.start() + + +def _set_adaptdl_signal_handler(): + import signal + global _trial_process + def _handler(signum, frame): + nni_log(LogType.Info, "RECEIVED SIGNAL {}".format(signum)) + nni_log(LogType.Debug, "TRIAL PROCESS ID {}".format(_trial_process.pid)) + if _trial_process and (signum == signal.SIGTERM or signum == signal.SIGINT): + os.killpg(os.getpgid(_trial_process.pid), signal.SIGINT) + os.waitpid(_trial_process.pid, 0) + exit(1) + signal.signal(signal.SIGTERM, _handler) + signal.signal(signal.SIGINT, _handler) + + +if __name__ == '__main__': + '''NNI Trial Keeper main function''' + PARSER = argparse.ArgumentParser() + PARSER.set_defaults(func=trial_keeper_help_info) + PARSER.add_argument('--trial_command', type=str, help='Command to launch trial process') + PARSER.add_argument('--nnimanager_ip', type=str, default='localhost', help='NNI manager rest server IP') + PARSER.add_argument('--nnimanager_port', type=str, default='8081', help='NNI manager rest server port') + PARSER.add_argument('--pai_hdfs_output_dir', type=str, help='the output dir of pai_hdfs') # backward compatibility + PARSER.add_argument('--hdfs_output_dir', type=str, help='the output dir of hdfs') + PARSER.add_argument('--pai_hdfs_host', type=str, help='the host of pai_hdfs') # backward compatibility + PARSER.add_argument('--hdfs_host', type=str, help='the host of hdfs') + PARSER.add_argument('--pai_user_name', type=str, help='the username of hdfs') + PARSER.add_argument('--nni_hdfs_exp_dir', type=str, help='nni experiment directory in hdfs') + PARSER.add_argument('--webhdfs_path', type=str, help='the webhdfs path used in webhdfs URL') + PARSER.add_argument('--nni_manager_version', type=str, help='the nni version transmitted from nniManager') + PARSER.add_argument('--log_collection', type=str, help='set the way to collect log in trialkeeper') + PARSER.add_argument('--job_id_file', type=str, help='set job id file for operating and monitoring job.') + args, unknown = PARSER.parse_known_args() + if args.trial_command is None: + exit(1) + check_version(args) + try: + if NNI_PLATFORM == 'adl': + _set_adaptdl_signal_handler() + main_loop(args) + except SystemExit as se: + nni_log(LogType.Info, 'NNI trial keeper exit with code {}'.format(se.code)) + os._exit(se.code) + except Exception as e: + nni_log(LogType.Error, 'Exit trial keeper with code 1 because Exception: {} is catched'.format(str(e))) + os._exit(1) diff --git a/new_impl/cv/third_party/nni_new/tools/trial_tool/trial_runner.py b/new_impl/cv/third_party/nni_new/tools/trial_tool/trial_runner.py new file mode 100644 index 0000000000000000000000000000000000000000..ebc6ee7dad6644e5c77bdd897cdb0d78aded11d3 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/tools/trial_tool/trial_runner.py @@ -0,0 +1,254 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import argparse +import json +import os +import random +import re +import sys +import time +import traceback +from datetime import datetime, timedelta + +import pkg_resources + +from .gpu import collect_gpu_usage + +idle_timeout_seconds = 10 * 60 +gpu_refressh_interval_seconds = 5 +regular = re.compile('v?(?P[0-9](\.[0-9]){0,1}).*') +trial_runner_syslogger = None + + +def main_loop(args): + '''main loop logic for trial runner''' + idle_last_time = datetime.now() + gpu_refresh_last_time = datetime.now() - timedelta(minutes=1) + try: + if args.job_pid_file: + with open(args.job_pid_file, 'w') as job_file: + job_file.write("%d" % os.getpid()) + + trials = dict() + + command_channel = args.command_channel + # command loop + while True: + command_type, command_data = command_channel.receive() + if command_type == CommandType.NewTrialJob: + trial_id = command_data["trialId"] + if trial_id in trials.keys(): + trial = trials[trial_id] + if trial.is_running(): + raise Exception('trial %s is running already, cannot start a new one' % trial.id) + else: + del trials[trial_id] + trial = Trial(args, command_data) + trial.run() + trials[trial_id] = trial + elif command_type == CommandType.KillTrialJob: + trial_id = command_data + if trial_id in trials.keys(): + trial = trials[trial_id] + trial.kill(command_data) + elif command_type == CommandType.SendTrialJobParameter: + trial_id = command_data["trialId"] + if trial_id in trials.keys(): + trial = trials[trial_id] + trial.save_parameter_file(command_data) + elif command_type is not None: + raise Exception("unknown command %s" % command_type) + + trial_list = list(trials.values()) + for trial in trial_list: + if trial is not None and trial.is_running(): + idle_last_time = datetime.now() + else: + del trials[trial.id] + + if (datetime.now() - idle_last_time).seconds > idle_timeout_seconds: + nni_log(LogType.Info, "trial runner is idle more than {0} seconds, so exit.".format( + idle_timeout_seconds)) + break + + if args.enable_gpu_collect and (datetime.now() - gpu_refresh_last_time).seconds > gpu_refressh_interval_seconds: + # collect gpu information + gpu_info = collect_gpu_usage(args.node_id) + command_channel.send(CommandType.ReportGpuInfo, gpu_info) + gpu_refresh_last_time = datetime.now() + time.sleep(0.5) + except Exception as ex: + traceback.print_exc() + raise ex + finally: + nni_log(LogType.Info, "main_loop exits.") + + trial_list = list(trials.values()) + for trial in trial_list: + trial.kill() + del trials[trial.id] + # wait to send commands + for _ in range(10): + if command_channel.sent(): + break + time.sleep(1) + command_channel.close() + + +def trial_runner_help_info(*args): + print('please run --help to see guidance') + + +def check_version(args): + try: + trial_runner_version = pkg_resources.get_distribution('nni').version + except pkg_resources.ResolutionError as err: + # package nni does not exist, try nni-tool package + nni_log(LogType.Error, 'Package nni does not exist!') + os._exit(1) + if not args.nni_manager_version: + # skip version check + nni_log(LogType.Warning, 'Skipping version check!') + else: + try: + command_channel = args.command_channel + trial_runner_version = regular.search(trial_runner_version).group('version') + nni_log(LogType.Info, '{0}: runner_version is {1}'.format(args.node_id, trial_runner_version)) + nni_manager_version = regular.search(args.nni_manager_version).group('version') + nni_log(LogType.Info, '{0}: nni_manager_version is {1}'.format(args.node_id, nni_manager_version)) + log_entry = {} + if trial_runner_version != nni_manager_version: + nni_log(LogType.Warning, '{0}: Version does not match!'.format(args.node_id)) + error_message = '{0}: NNIManager version is {1}, Trial runner version is {2}, NNI version does not match!'.format( + args.node_id, nni_manager_version, trial_runner_version) + log_entry['tag'] = 'VCFail' + log_entry['msg'] = error_message + command_channel.send(CommandType.VersionCheck, log_entry) + while not command_channel.sent(): + time.sleep(1) + else: + nni_log(LogType.Info, '{0}: Version match!'.format(args.node_id)) + log_entry['tag'] = 'VCSuccess' + command_channel.send(CommandType.VersionCheck, log_entry) + except AttributeError as err: + nni_log(LogType.Error, '{0}: {1}'.format(args.node_id, err)) + +if __name__ == '__main__': + + '''NNI Trial Runner main function''' + PARSER = argparse.ArgumentParser() + PARSER.set_defaults(func=trial_runner_help_info) + PARSER.add_argument('--trial_command', type=str, help='Command to launch trial process') + PARSER.add_argument('--nnimanager_ip', type=str, help='NNI manager rest server IP') + PARSER.add_argument('--nnimanager_port', type=str, help='NNI manager rest server port') + PARSER.add_argument('--nni_manager_version', type=str, help='the nni version transmitted from nniManager') + PARSER.add_argument('--log_collection', type=str, help='set the way to collect log in trial runner') + PARSER.add_argument('--node_count', type=int, help='number of nodes, it determines how to consume command and save code file') + PARSER.add_argument('--job_pid_file', type=str, help='save trial runner process pid') + args, unknown = PARSER.parse_known_args() + + setting_file = "settings.json" + if not os.path.exists(setting_file): + setting_file = "../{}".format(setting_file) + if os.path.exists(setting_file): + with open(setting_file, 'r') as fp: + settings = json.load(fp) + print("setting is {}".format(settings)) + else: + print("not found setting file") + + args.exp_id = settings["experimentId"] + args.platform = settings["platform"] + # runner_id is unique runner in experiment + args.runner_id = os.path.basename(os.path.realpath(os.path.curdir)) + args.runner_name = "runner_"+args.runner_id + args.enable_gpu_collect = settings["enableGpuCollector"] + args.command_channel = settings["commandChannel"] + + if args.trial_command is None: + args.trial_command = settings["command"] + if args.nnimanager_ip is None: + args.nnimanager_ip = settings["nniManagerIP"] + if args.nnimanager_port is None: + args.nnimanager_port = settings["nniManagerPort"] + if args.nni_manager_version is None: + args.nni_manager_version = settings["nniManagerVersion"] + if args.log_collection is None: + args.log_collection = settings["logCollection"] + if args.node_count is None: + # default has only one node. + args.node_count = 1 + + os.environ['NNI_OUTPUT_DIR'] = os.curdir + "/nnioutput" + os.environ['NNI_PLATFORM'] = args.platform + os.environ['NNI_SYS_DIR'] = os.curdir + os.environ['NNI_EXP_ID'] = args.exp_id + os.environ['MULTI_PHASE'] = "true" + os.environ['NNI_TRIAL_JOB_ID'] = "runner" + os.environ['REUSE_MODE'] = "true" + + from .log_utils import LogType, RemoteLogger, StdOutputType, nni_log + from .trial import Trial + from .file_channel import FileChannel + from .web_channel import WebChannel + from .commands import CommandType + + is_multi_node = args.node_count > 1 + + if (is_multi_node): + # for multiple nodes, create a file to get a unique id. + while True: + node_id = random.randint(0, 10000) + unique_check_file_name = "node_%s" % (node_id) + if not os.path.exists(unique_check_file_name): + break + with open(unique_check_file_name, "w") as unique_check_file: + unique_check_file.write("%s" % (int(datetime.now().timestamp() * 1000))) + args.node_id = node_id + else: + # node id is unique in the runner + args.node_id = None + + # init command channel + command_channel = None + if args.command_channel == "file": + command_channel = FileChannel(args) + elif args.command_channel == 'aml': + from .aml_channel import AMLChannel + command_channel = AMLChannel(args) + else: + command_channel = WebChannel(args) + command_channel.open() + + nni_log(LogType.Info, "command channel is {}, actual type is {}".format(args.command_channel, type(command_channel))) + args.command_channel = command_channel + + trial_runner_syslogger = RemoteLogger(args.nnimanager_ip, args.nnimanager_port, 'runner', + StdOutputType.Stdout, args.log_collection, args.runner_name, command_channel) + sys.stdout = sys.stderr = trial_runner_syslogger + nni_log(LogType.Info, "{}: merged args is {}".format(args.node_id, args)) + + if args.trial_command is None: + nni_log(LogType.Error, "{}: no command is found.".format(args.node_id)) + os._exit(1) + check_version(args) + try: + main_loop(args) + except SystemExit as se: + nni_log(LogType.Info, '{}: NNI trial runner exit with code {}'.format(args.node_id, se.code)) + + # try best to send latest errors to server + timeout = 10 + while not command_channel.sent() and timeout > 0: + timeout -= 1 + time.sleep(1) + os._exit(se.code) + finally: + if trial_runner_syslogger is not None: + if trial_runner_syslogger.pipeReader is not None: + trial_runner_syslogger.pipeReader.set_process_exit() + trial_runner_syslogger.close() + + # the process doesn't exit even main loop exit. So exit it explictly. + os._exit(0) diff --git a/new_impl/cv/third_party/nni_new/tools/trial_tool/url_utils.py b/new_impl/cv/third_party/nni_new/tools/trial_tool/url_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7942c62fb5b4d8d3734ddf8c23a21b0f379d76ec --- /dev/null +++ b/new_impl/cv/third_party/nni_new/tools/trial_tool/url_utils.py @@ -0,0 +1,19 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .constants import API_ROOT_URL, BASE_URL, STDOUT_API, NNI_TRIAL_JOB_ID, NNI_EXP_ID, VERSION_API, PARAMETER_META_API + + +def gen_send_stdout_url(ip, port): + '''Generate send stdout url''' + return '{0}:{1}{2}{3}/{4}/{5}'.format(BASE_URL.format(ip), port, API_ROOT_URL, STDOUT_API, NNI_EXP_ID, NNI_TRIAL_JOB_ID) + + +def gen_send_version_url(ip, port): + '''Generate send error url''' + return '{0}:{1}{2}{3}/{4}/{5}'.format(BASE_URL.format(ip), port, API_ROOT_URL, VERSION_API, NNI_EXP_ID, NNI_TRIAL_JOB_ID) + + +def gen_parameter_meta_url(ip, port): + '''Generate send error url''' + return '{0}:{1}{2}{3}'.format(BASE_URL.format(ip), port, API_ROOT_URL, PARAMETER_META_API) diff --git a/new_impl/cv/third_party/nni_new/tools/trial_tool/web_channel.py b/new_impl/cv/third_party/nni_new/tools/trial_tool/web_channel.py new file mode 100644 index 0000000000000000000000000000000000000000..87901e1163a5fbcf82a2bb05c701735a346333fc --- /dev/null +++ b/new_impl/cv/third_party/nni_new/tools/trial_tool/web_channel.py @@ -0,0 +1,57 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import asyncio +import os +import websockets + +from .base_channel import BaseChannel +from .log_utils import LogType, nni_log + + +class WebChannel(BaseChannel): + + def __init__(self, args): + self.node_id = args.node_id + self.args = args + self.client = None + self.in_cache = b"" + self.timeout = 10 + + super(WebChannel, self).__init__(args) + + self._event_loop = None + + def _inner_open(self): + url = "ws://{}:{}".format(self.args.nnimanager_ip, self.args.nnimanager_port) + try: + connect = asyncio.wait_for(websockets.connect(url), self.timeout) + self._event_loop = asyncio.get_event_loop() + client = self._event_loop.run_until_complete(connect) + self.client = client + nni_log(LogType.Info, 'WebChannel: connected with info %s' % url) + except asyncio.TimeoutError: + nni_log(LogType.Error, 'connect to %s timeout! Please make sure NNIManagerIP configured correctly, and accessable.' % url) + os._exit(1) + + def _inner_close(self): + if self.client is not None: + self.client.close() + self.client = None + if self._event_loop.is_running(): + self._event_loop.stop() + self._event_loop = None + + def _inner_send(self, message): + loop = asyncio.new_event_loop() + loop.run_until_complete(self.client.send(message)) + + def _inner_receive(self): + messages = [] + if self.client is not None: + received = self._event_loop.run_until_complete(self.client.recv()) + # receive message is string, to get consistent result, encode it here. + self.in_cache += received.encode("utf8") + messages, self.in_cache = self._fetch_message(self.in_cache) + + return messages diff --git a/new_impl/cv/third_party/nni_new/trial.py b/new_impl/cv/third_party/nni_new/trial.py new file mode 100644 index 0000000000000000000000000000000000000000..e85d292b8c1409d4621e347aea2086f86e76f303 --- /dev/null +++ b/new_impl/cv/third_party/nni_new/trial.py @@ -0,0 +1,156 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .utils import to_json +from .runtime.env_vars import trial_env_vars +from .runtime import platform + + +__all__ = [ + 'get_next_parameter', + 'get_current_parameter', + 'report_intermediate_result', + 'report_final_result', + 'get_experiment_id', + 'get_trial_id', + 'get_sequence_id' +] + + +_params = None +_experiment_id = platform.get_experiment_id() +_trial_id = platform.get_trial_id() +_sequence_id = platform.get_sequence_id() + + +def get_next_parameter(): + """ + Get the hyper paremeters generated by tuner. For a multiphase experiment, it returns a new group of hyper + parameters at each call of get_next_parameter. For a non-multiphase (multiPhase is not configured or set to False) + experiment, it returns hyper parameters only on the first call for each trial job, it returns None since second call. + This API should be called only once in each trial job of an experiment which is not specified as multiphase. + + Returns + ------- + dict + A dict object contains the hyper parameters generated by tuner, the keys of the dict are defined in + search space. Returns None if no more hyper parameters can be generated by tuner. + """ + global _params + _params = platform.get_next_parameter() + if _params is None: + return None + return _params['parameters'] + +def get_current_parameter(tag=None): + """ + Get current hyper parameters generated by tuner. It returns the same group of hyper parameters as the last + call of get_next_parameter returns. + + Parameters + ---------- + tag: str + hyper parameter key + """ + global _params + if _params is None: + return None + if tag is None: + return _params['parameters'] + return _params['parameters'][tag] + +def get_experiment_id(): + """ + Get experiment ID. + + Returns + ------- + str + Identifier of current experiment + """ + return _experiment_id + +def get_trial_id(): + """ + Get trial job ID which is string identifier of a trial job, for example 'MoXrp'. In one experiment, each trial + job has an unique string ID. + + Returns + ------- + str + Identifier of current trial job which is calling this API. + """ + return _trial_id + +def get_sequence_id(): + """ + Get trial job sequence nubmer. A sequence number is an integer value assigned to each trial job base on the + order they are submitted, incremental starting from 0. In one experiment, both trial job ID and sequence number + are unique for each trial job, they are of different data types. + + Returns + ------- + int + Sequence number of current trial job which is calling this API. + """ + return _sequence_id + +_intermediate_seq = 0 + + +def overwrite_intermediate_seq(value): + """ + Overwrite intermediate sequence value. + + Parameters + ---------- + value: + int + """ + assert isinstance(value, int) + global _intermediate_seq + _intermediate_seq = value + + +def report_intermediate_result(metric): + """ + Reports intermediate result to NNI. + + Parameters + ---------- + metric: + serializable object. + """ + global _intermediate_seq + assert _params or trial_env_vars.NNI_PLATFORM is None, \ + 'nni.get_next_parameter() needs to be called before report_intermediate_result' + metric = to_json({ + 'parameter_id': _params['parameter_id'] if _params else None, + 'trial_job_id': trial_env_vars.NNI_TRIAL_JOB_ID, + 'type': 'PERIODICAL', + 'sequence': _intermediate_seq, + 'value': to_json(metric) + }) + _intermediate_seq += 1 + platform.send_metric(metric) + +def report_final_result(metric): + """ + Reports final result to NNI. + + Parameters + ---------- + metric: serializable object + Usually (for built-in tuners to work), it should be a number, or + a dict with key "default" (a number), and any other extra keys. + """ + assert _params or trial_env_vars.NNI_PLATFORM is None, \ + 'nni.get_next_parameter() needs to be called before report_final_result' + metric = to_json({ + 'parameter_id': _params['parameter_id'] if _params else None, + 'trial_job_id': trial_env_vars.NNI_TRIAL_JOB_ID, + 'type': 'FINAL', + 'sequence': 0, + 'value': to_json(metric) + }) + platform.send_metric(metric) diff --git a/new_impl/cv/third_party/nni_new/tuner.py b/new_impl/cv/third_party/nni_new/tuner.py new file mode 100644 index 0000000000000000000000000000000000000000..4fbcc011d0c55676e9bc9c7e906d7655bb5badeb --- /dev/null +++ b/new_impl/cv/third_party/nni_new/tuner.py @@ -0,0 +1,223 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +Tuner is an AutoML algorithm, which generates a new configuration for the next try. +A new trial will run with this configuration. + +See :class:`Tuner`' specification and ``docs/en_US/tuners.rst`` for details. +""" + +import logging + +import nni + +from .recoverable import Recoverable + +__all__ = ['Tuner'] + +_logger = logging.getLogger(__name__) + + +class Tuner(Recoverable): + """ + Tuner is an AutoML algorithm, which generates a new configuration for the next try. + A new trial will run with this configuration. + + This is the abstract base class for all tuners. + Tuning algorithms should inherit this class and override :meth:`update_search_space`, :meth:`receive_trial_result`, + as well as :meth:`generate_parameters` or :meth:`generate_multiple_parameters`. + + After initializing, NNI will first call :meth:`update_search_space` to tell tuner the feasible region, + and then call :meth:`generate_parameters` one or more times to request for hyper-parameter configurations. + + The framework will train several models with given configuration. + When one of them is finished, the final accuracy will be reported to :meth:`receive_trial_result`. + And then another configuration will be reqeusted and trained, util the whole experiment finish. + + If a tuner want's to know when a trial ends, it can also override :meth:`trial_end`. + + Tuners use *parameter ID* to track trials. + In tuner context, there is a one-to-one mapping between parameter ID and trial. + When the framework ask tuner to generate hyper-parameters for a new trial, + an ID has already been assigned and can be recorded in :meth:`generate_parameters`. + Later when the trial ends, the ID will be reported to :meth:`trial_end`, + and :meth:`receive_trial_result` if it has a final result. + Parameter IDs are unique integers. + + The type/format of search space and hyper-parameters are not limited, + as long as they are JSON-serializable and in sync with trial code. + For HPO tuners, however, there is a widely shared common interface, + which supports ``choice``, ``randint``, ``uniform``, and so on. + See ``docs/en_US/Tutorial/SearchSpaceSpec.md`` for details of this interface. + + [WIP] For advanced tuners which take advantage of trials' intermediate results, + an ``Advisor`` interface is under development. + + See Also + -------- + Builtin tuners: + :class:`~nni.algorithms.hpo.hyperopt_tuner.hyperopt_tuner.HyperoptTuner` + :class:`~nni.algorithms.hpo.evolution_tuner.evolution_tuner.EvolutionTuner` + :class:`~nni.algorithms.hpo.smac_tuner.SMACTuner` + :class:`~nni.algorithms.hpo.gridsearch_tuner.GridSearchTuner` + :class:`~nni.algorithms.hpo.networkmorphism_tuner.networkmorphism_tuner.NetworkMorphismTuner` + :class:`~nni.algorithms.hpo.metis_tuner.mets_tuner.MetisTuner` + :class:`~nni.algorithms.hpo.ppo_tuner.PPOTuner` + :class:`~nni.algorithms.hpo.gp_tuner.gp_tuner.GPTuner` + """ + + def generate_parameters(self, parameter_id, **kwargs): + """ + Abstract method which provides a set of hyper-parameters. + + This method will get called when the framework is about to launch a new trial, + if user does not override :meth:`generate_multiple_parameters`. + + The return value of this method will be received by trials via :func:`nni.get_next_parameter`. + It should fit in the search space, though the framework will not verify this. + + User code must override either this method or :meth:`generate_multiple_parameters`. + + Parameters + ---------- + parameter_id : int + Unique identifier for requested hyper-parameters. This will later be used in :meth:`receive_trial_result`. + **kwargs + Unstable parameters which should be ignored by normal users. + + Returns + ------- + any + The hyper-parameters, a dict in most cases, but could be any JSON-serializable type when needed. + + Raises + ------ + nni.NoMoreTrialError + If the search space is fully explored, tuner can raise this exception. + """ + # FIXME: some tuners raise NoMoreTrialError when they are waiting for more trial results + # we need to design a new exception for this purpose + raise NotImplementedError('Tuner: generate_parameters not implemented') + + def generate_multiple_parameters(self, parameter_id_list, **kwargs): + """ + Callback method which provides multiple sets of hyper-parameters. + + This method will get called when the framework is about to launch one or more new trials. + + If user does not override this method, it will invoke :meth:`generate_parameters` on each parameter ID. + + See :meth:`generate_parameters` for details. + + User code must override either this method or :meth:`generate_parameters`. + + Parameters + ---------- + parameter_id_list : list of int + Unique identifiers for each set of requested hyper-parameters. + These will later be used in :meth:`receive_trial_result`. + **kwargs + Unstable parameters which should be ignored by normal users. + + Returns + ------- + list + List of hyper-parameters. An empty list indicates there are no more trials. + """ + result = [] + for parameter_id in parameter_id_list: + try: + _logger.debug("generating param for %s", parameter_id) + res = self.generate_parameters(parameter_id, **kwargs) + except nni.NoMoreTrialError: + return result + result.append(res) + return result + + def receive_trial_result(self, parameter_id, parameters, value, **kwargs): + """ + Abstract method invoked when a trial reports its final result. Must override. + + This method only listens to results of algorithm-generated hyper-parameters. + Currently customized trials added from web UI will not report result to this method. + + Parameters + ---------- + parameter_id : int + Unique identifier of used hyper-parameters, same with :meth:`generate_parameters`. + parameters + Hyper-parameters generated by :meth:`generate_parameters`. + value + Result from trial (the return value of :func:`nni.report_final_result`). + **kwargs + Unstable parameters which should be ignored by normal users. + """ + raise NotImplementedError('Tuner: receive_trial_result not implemented') + + def _accept_customized_trials(self, accept=True): + # FIXME: because Tuner is designed as interface, this API should not be here + + # Enable or disable receiving results of user-added hyper-parameters. + # By default `receive_trial_result()` will only receive results of algorithm-generated hyper-parameters. + # If tuners want to receive those of customized parameters as well, they can call this function in `__init__()`. + + # pylint: disable=attribute-defined-outside-init + self._accept_customized = accept + + def trial_end(self, parameter_id, success, **kwargs): + """ + Abstract method invoked when a trial is completed or terminated. Do nothing by default. + + Parameters + ---------- + parameter_id : int + Unique identifier for hyper-parameters used by this trial. + success : bool + True if the trial successfully completed; False if failed or terminated. + **kwargs + Unstable parameters which should be ignored by normal users. + """ + + def update_search_space(self, search_space): + """ + Abstract method for updating the search space. Must override. + + Tuners are advised to support updating search space at run-time. + If a tuner can only set search space once before generating first hyper-parameters, + it should explicitly document this behaviour. + + Parameters + ---------- + search_space + JSON object defined by experiment owner. + """ + raise NotImplementedError('Tuner: update_search_space not implemented') + + def load_checkpoint(self): + """ + Internal API under revising, not recommended for end users. + """ + checkpoin_path = self.get_checkpoint_path() + _logger.info('Load checkpoint ignored by tuner, checkpoint path: %s', checkpoin_path) + + def save_checkpoint(self): + """ + Internal API under revising, not recommended for end users. + """ + checkpoin_path = self.get_checkpoint_path() + _logger.info('Save checkpoint ignored by tuner, checkpoint path: %s', checkpoin_path) + + def import_data(self, data): + """ + Internal API under revising, not recommended for end users. + """ + # Import additional data for tuning + # data: a list of dictionarys, each of which has at least two keys, 'parameter' and 'value' + pass + + def _on_exit(self): + pass + + def _on_error(self): + pass diff --git a/new_impl/cv/third_party/nni_new/utils.py b/new_impl/cv/third_party/nni_new/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..026e0fbc6419ddd744d321010ed4aa0284da34ce --- /dev/null +++ b/new_impl/cv/third_party/nni_new/utils.py @@ -0,0 +1,307 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import copy +import functools +from enum import Enum, unique +import json_tricks +from schema import And + +from . import parameter_expressions + + +to_json = functools.partial(json_tricks.dumps, allow_nan=True) + +@unique +class OptimizeMode(Enum): + """Optimize Mode class + + if OptimizeMode is 'minimize', it means the tuner need to minimize the reward + that received from Trial. + + if OptimizeMode is 'maximize', it means the tuner need to maximize the reward + that received from Trial. + """ + Minimize = 'minimize' + Maximize = 'maximize' + + +class NodeType: + """Node Type class + """ + ROOT = 'root' + TYPE = '_type' + VALUE = '_value' + INDEX = '_index' + NAME = '_name' + + +class MetricType: + """The types of metric data + """ + FINAL = 'FINAL' + PERIODICAL = 'PERIODICAL' + REQUEST_PARAMETER = 'REQUEST_PARAMETER' + + +def split_index(params): + """ + Delete index infromation from params + """ + if isinstance(params, dict): + if NodeType.INDEX in params.keys(): + return split_index(params[NodeType.VALUE]) + result = {} + for key in params: + result[key] = split_index(params[key]) + return result + else: + return params + + +def extract_scalar_reward(value, scalar_key='default'): + """ + Extract scalar reward from trial result. + + Parameters + ---------- + value : int, float, dict + the reported final metric data + scalar_key : str + the key name that indicates the numeric number + + Raises + ------ + RuntimeError + Incorrect final result: the final result should be float/int, + or a dict which has a key named "default" whose value is float/int. + """ + if isinstance(value, (float, int)): + reward = value + elif isinstance(value, dict) and scalar_key in value and isinstance(value[scalar_key], (float, int)): + reward = value[scalar_key] + else: + raise RuntimeError('Incorrect final result: the final result should be float/int, ' \ + 'or a dict which has a key named "default" whose value is float/int.') + return reward + + +def extract_scalar_history(trial_history, scalar_key='default'): + """ + Extract scalar value from a list of intermediate results. + + Parameters + ---------- + trial_history : list + accumulated intermediate results of a trial + scalar_key : str + the key name that indicates the numeric number + + Raises + ------ + RuntimeError + Incorrect final result: the final result should be float/int, + or a dict which has a key named "default" whose value is float/int. + """ + return [extract_scalar_reward(ele, scalar_key) for ele in trial_history] + + +def convert_dict2tuple(value): + """ + convert dict type to tuple to solve unhashable problem. + NOTE: this function will change original data. + """ + if isinstance(value, dict): + for _keys in value: + value[_keys] = convert_dict2tuple(value[_keys]) + return tuple(sorted(value.items())) + return value + + +def json2space(x, oldy=None, name=NodeType.ROOT): + """ + Change search space from json format to hyperopt format + + """ + y = list() + if isinstance(x, dict): + if NodeType.TYPE in x.keys(): + _type = x[NodeType.TYPE] + name = name + '-' + _type + if _type == 'choice': + if oldy is not None: + _index = oldy[NodeType.INDEX] + y += json2space(x[NodeType.VALUE][_index], + oldy[NodeType.VALUE], name=name+'[%d]' % _index) + else: + y += json2space(x[NodeType.VALUE], None, name=name) + y.append(name) + else: + for key in x.keys(): + y += json2space(x[key], oldy[key] if oldy else None, name+"[%s]" % str(key)) + elif isinstance(x, list): + for i, x_i in enumerate(x): + if isinstance(x_i, dict): + if NodeType.NAME not in x_i.keys(): + raise RuntimeError('\'_name\' key is not found in this nested search space.') + y += json2space(x_i, oldy[i] if oldy else None, name + "[%d]" % i) + return y + + +def json2parameter(x, is_rand, random_state, oldy=None, Rand=False, name=NodeType.ROOT): + """ + Json to pramaters. + + """ + if isinstance(x, dict): + if NodeType.TYPE in x.keys(): + _type = x[NodeType.TYPE] + _value = x[NodeType.VALUE] + name = name + '-' + _type + Rand |= is_rand[name] + if Rand is True: + if _type == 'choice': + _index = random_state.randint(len(_value)) + y = { + NodeType.INDEX: _index, + NodeType.VALUE: json2parameter( + x[NodeType.VALUE][_index], + is_rand, + random_state, + None, + Rand, + name=name+"[%d]" % _index + ) + } + else: + y = getattr(parameter_expressions, _type)(*(_value + [random_state])) + else: + y = copy.deepcopy(oldy) + else: + y = dict() + for key in x.keys(): + y[key] = json2parameter( + x[key], + is_rand, + random_state, + oldy[key] if oldy else None, + Rand, + name + "[%s]" % str(key) + ) + elif isinstance(x, list): + y = list() + for i, x_i in enumerate(x): + if isinstance(x_i, dict): + if NodeType.NAME not in x_i.keys(): + raise RuntimeError('\'_name\' key is not found in this nested search space.') + y.append(json2parameter( + x_i, + is_rand, + random_state, + oldy[i] if oldy else None, + Rand, + name + "[%d]" % i + )) + else: + y = copy.deepcopy(x) + return y + +def merge_parameter(base_params, override_params): + """ + Update the parameters in ``base_params`` with ``override_params``. + Can be useful to override parsed command line arguments. + + Parameters + ---------- + base_params : namespace or dict + Base parameters. A key-value mapping. + override_params : dict or None + Parameters to override. Usually the parameters got from ``get_next_parameters()``. + When it is none, nothing will happen. + + Returns + ------- + namespace or dict + The updated ``base_params``. Note that ``base_params`` will be updated inplace. The return value is + only for convenience. + """ + if override_params is None: + return base_params + is_dict = isinstance(base_params, dict) + for k, v in override_params.items(): + if is_dict: + if k not in base_params: + raise ValueError('Key \'%s\' not found in base parameters.' % k) + if type(base_params[k]) != type(v) and base_params[k] is not None: + raise TypeError('Expected \'%s\' in override parameters to have type \'%s\', but found \'%s\'.' % + (k, type(base_params[k]), type(v))) + base_params[k] = v + else: + if not hasattr(base_params, k): + raise ValueError('Key \'%s\' not found in base parameters.' % k) + if type(getattr(base_params, k)) != type(v) and getattr(base_params, k) is not None: + raise TypeError('Expected \'%s\' in override parameters to have type \'%s\', but found \'%s\'.' % + (k, type(getattr(base_params, k)), type(v))) + setattr(base_params, k, v) + return base_params + +class ClassArgsValidator(object): + """ + NNI tuners/assessors/adivisors accept a `classArgs` parameter in experiment configuration file. + This ClassArgsValidator interface is used to validate the classArgs section in exeperiment + configuration file. + """ + def validate_class_args(self, **kwargs): + """ + Validate the classArgs configuration in experiment configuration file. + + Parameters + ---------- + kwargs: dict + kwargs passed to tuner/assessor/advisor constructor + + Raises: + Raise an execption if the kwargs is invalid. + """ + pass + + def choices(self, key, *args): + """ + Utility method to create a scheme to check whether the `key` is one of the `args`. + + Parameters: + ---------- + key: str + key name of the data to be validated + args: list of str + list of the choices + + Returns: Schema + -------- + A scheme to check whether the `key` is one of the `args`. + """ + return And(lambda n: n in args, error='%s should be in [%s]!' % (key, str(args))) + + def range(self, key, keyType, start, end): + """ + Utility method to create a schema to check whether the `key` is in the range of [start, end]. + + Parameters: + ---------- + key: str + key name of the data to be validated + keyType: type + python data type, such as int, float + start: type is specified by keyType + start of the range + end: type is specified by keyType + end of the range + + Returns: Schema + -------- + A scheme to check whether the `key` is in the range of [start, end]. + """ + return And( + And(keyType, error='%s should be %s type!' % (key, keyType.__name__)), + And(lambda n: start <= n <= end, error='%s should be in range of (%s, %s)!' % (key, start, end)) + ) diff --git a/new_impl/cv/utils/__pycache__/baseline_da.cpython-38.pyc b/new_impl/cv/utils/__pycache__/baseline_da.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..193c5b59bd692a2634bb4aba40f476e715ec7944 Binary files /dev/null and b/new_impl/cv/utils/__pycache__/baseline_da.cpython-38.pyc differ diff --git a/new_impl/cv/utils/__pycache__/elasticfm_da.cpython-38.pyc b/new_impl/cv/utils/__pycache__/elasticfm_da.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..55054efacfbeb9ae4d9d7cb865071513d672a1a4 Binary files /dev/null and b/new_impl/cv/utils/__pycache__/elasticfm_da.cpython-38.pyc differ diff --git a/new_impl/cv/utils/baseline_cl.py b/new_impl/cv/utils/baseline_cl.py new file mode 100644 index 0000000000000000000000000000000000000000..8a0d567099d9e596a149aff8b97eaabcd2b37c3e --- /dev/null +++ b/new_impl/cv/utils/baseline_cl.py @@ -0,0 +1,97 @@ +import sys +from utils.dl.common.env import set_random_seed +set_random_seed(1) + +from typing import List +from data.dataloader import build_dataloader +from data import CLScenario, build_cl_scenario +from methods.elasticdnn.api.online_model_v2 import ElasticDNN_OnlineModel + +import torch +import sys +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from utils.dl.common.env import create_tbwriter +import os +from utils.common.log import logger +from utils.common.data_record import write_json +# from methods.shot.shot import OnlineShotModel +from methods.base.alg import BaseAlg +from methods.base.model import BaseModel + + +def baseline_cl(app_name: str, + scenario: CLScenario, + cl_alg: BaseAlg, + cl_alg_hyp: dict, + cl_model: BaseModel, + device, + __entry_file__, + tag=None): + + # involve_fm = settings['involve_fm'] + + # task_name = app_name + # online_model = elasticfm_model + + log_dir = get_res_save_dir(__entry_file__, tag=tag) + tb_writer = create_tbwriter(os.path.join(log_dir, 'tb_log'), False) + res = [] + global_avg_after_acc = 0. + global_iter = 0 + + for task_index, _ in enumerate(scenario.target_tasks_order): + + cur_target_task_name = scenario.target_tasks_order[scenario.cur_task_index] + # if cur_target_domain_name in da_alg_hyp: + # da_alg_hyp = da_alg_hyp[cur_target_domain_name] + # logger.info(f'use dataset-specific hyps') + + # tmp_sd_path = os.path.join(log_dir, 'tmp_sd_model.pt') + # torch.save({'main': sd}, tmp_sd_path) + + # if task_name != 'cls': + # da_model_args = [f'{task_name}/{domain_index}', + # tmp_sd_path, + # device, + # scenario.num_classes] + # else: + # da_model_args = [f'{task_name}/{domain_index}', + # tmp_sd_path, + # device] + # cur_da_model = da_model(*da_model_args) + + cl_metrics, after_cl_model = cl_alg( + {'main': cl_model}, + os.path.join(log_dir, f'{app_name}/{task_index}') + ).run(scenario, cl_alg_hyp) + # os.remove(tmp_sd_path) + + if task_index > 0: + import shutil + shutil.rmtree(os.path.join(log_dir, f'{app_name}/{task_index}/backup_codes')) + + accs = cl_metrics['accs'] + before_acc = accs[0]['acc'] + after_acc = accs[-1]['acc'] + + tb_writer.add_scalars(f'accs/{app_name}', dict(before=before_acc, after=after_acc), task_index) + tb_writer.add_scalar(f'times/{app_name}', cl_metrics['time'], task_index) + + for _acc in accs: + tb_writer.add_scalar('total_acc', _acc['acc'], _acc['iter'] + global_iter) + global_iter += _acc['iter'] + 1 + + scenario.next_task() + + logger.info(f"app: {app_name}, task {task_index}, acc: {before_acc:.4f} -> " + f"{after_acc:.4f} ({cl_metrics['time']:.2f}s)") + + global_avg_after_acc += after_acc + cur_res = cl_metrics + res += [cur_res] + write_json(os.path.join(log_dir, 'res.json'), res, backup=False) + + global_avg_after_acc /= (task_index + 1) + logger.info(f'-----> final metric: {global_avg_after_acc:.4f}') + write_json(os.path.join(log_dir, f'res_{global_avg_after_acc:.4f}.json'), res, backup=False) + \ No newline at end of file diff --git a/new_impl/cv/utils/baseline_da.py b/new_impl/cv/utils/baseline_da.py new file mode 100644 index 0000000000000000000000000000000000000000..6511851e1b9c1d74d9b919483a03be31d3bab7ca --- /dev/null +++ b/new_impl/cv/utils/baseline_da.py @@ -0,0 +1,114 @@ +import sys +from utils.dl.common.env import set_random_seed +set_random_seed(1) + +from typing import List +from data.dataloader import build_dataloader +from data import Scenario +from methods.elasticdnn.api.online_model_v2 import ElasticDNN_OnlineModel + +import torch +import sys +from torch import nn +from methods.elasticdnn.api.model import ElasticDNN_OfflineSegFMModel, ElasticDNN_OfflineSegMDModel +from methods.elasticdnn.api.algs.md_pretraining_wo_fbs import ElasticDNN_MDPretrainingWoFBSAlg +from methods.elasticdnn.model.base import ElasticDNNUtil +from methods.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from methods.elasticdnn.pipeline.offline.fm_to_md.vit import FM_to_MD_ViT_Util +from methods.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from methods.elasticdnn.pipeline.offline.fm_lora.vit import FMLoRA_ViT_Util +from methods.elasticdnn.model.vit import ElasticViTUtil +from utils.common.file import ensure_dir +from utils.dl.common.model import LayerActivation, get_module, get_parameter +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +from utils.dl.common.loss import CrossEntropyLossSoft +import torch.nn.functional as F +from utils.dl.common.env import create_tbwriter +import os +import shutil +from utils.common.log import logger +from utils.common.data_record import write_json +# from methods.shot.shot import OnlineShotModel +from methods.feat_align.main import OnlineFeatAlignModel, FeatAlignAlg +import tqdm +from methods.feat_align.mmd import mmd_rbf +from methods.base.alg import BaseAlg +from methods.base.model import BaseModel + + +def baseline_da(app_name: str, + scenario: Scenario, + da_alg: BaseAlg, + da_alg_hyp: dict, + da_model: BaseModel, + device, + __entry_file__, + tag=None, + collate_fn=None): + + # involve_fm = settings['involve_fm'] + + task_name = app_name + # online_model = elasticfm_model + + log_dir = get_res_save_dir(__entry_file__, tag=tag) + tb_writer = create_tbwriter(os.path.join(log_dir, 'tb_log'), False) + res = [] + global_avg_after_acc = 0. + global_iter = 0 + + for domain_index, _ in enumerate(scenario.target_domains_order): + + cur_target_domain_name = scenario.target_domains_order[scenario.cur_domain_index] + if cur_target_domain_name in da_alg_hyp: + da_alg_hyp = da_alg_hyp[cur_target_domain_name] + logger.info(f'use dataset-specific hyps') + + # tmp_sd_path = os.path.join(log_dir, 'tmp_sd_model.pt') + # torch.save({'main': sd}, tmp_sd_path) + + # if task_name != 'cls': + # da_model_args = [f'{task_name}/{domain_index}', + # tmp_sd_path, + # device, + # scenario.num_classes] + # else: + # da_model_args = [f'{task_name}/{domain_index}', + # tmp_sd_path, + # device] + # cur_da_model = da_model(*da_model_args) + da_metrics, after_da_model = da_alg( + {'main': da_model}, + os.path.join(log_dir, f'{task_name}/{domain_index}') + ).run(scenario, da_alg_hyp, collate_fn=collate_fn) + # os.remove(tmp_sd_path) + + if domain_index > 0: + shutil.rmtree(os.path.join(log_dir, f'{task_name}/{domain_index}/backup_codes')) + + accs = da_metrics['accs'] + before_acc = accs[0]['acc'] + after_acc = accs[-1]['acc'] + + tb_writer.add_scalars(f'accs/{task_name}', dict(before=before_acc, after=after_acc), domain_index) + tb_writer.add_scalar(f'times/{task_name}', da_metrics['time'], domain_index) + + for _acc in accs: + tb_writer.add_scalar('total_acc', _acc['acc'], _acc['iter'] + global_iter) + global_iter += _acc['iter'] + 1 + + scenario.next_domain() + + logger.info(f"task: {task_name}, domain {domain_index}, acc: {before_acc:.4f} -> " + f"{after_acc:.4f} ({da_metrics['time']:.2f}s)") + + global_avg_after_acc += after_acc + cur_res = da_metrics + res += [cur_res] + write_json(os.path.join(log_dir, 'res.json'), res, backup=False) + + global_avg_after_acc /= (domain_index + 1) + logger.info(f'-----> final metric: {global_avg_after_acc:.4f}') + write_json(os.path.join(log_dir, f'res_{global_avg_after_acc:.4f}.json'), res, backup=False) + \ No newline at end of file diff --git a/new_impl/cv/utils/baseline_da_train_compress.py b/new_impl/cv/utils/baseline_da_train_compress.py new file mode 100644 index 0000000000000000000000000000000000000000..28b2092dc54d1e16329e2cec7cf9a113fa082deb --- /dev/null +++ b/new_impl/cv/utils/baseline_da_train_compress.py @@ -0,0 +1,198 @@ +import sys +from utils.dl.common.env import set_random_seed +set_random_seed(1) + +from typing import List +from data.dataloader import build_dataloader +from data import Scenario +from methods.elasticdnn.api.online_model_v2 import ElasticDNN_OnlineModel + +import torch +import sys +from torch import nn +from methods.elasticdnn.api.model import ElasticDNN_OfflineSegFMModel, ElasticDNN_OfflineSegMDModel +from methods.elasticdnn.api.algs.md_pretraining_wo_fbs import ElasticDNN_MDPretrainingWoFBSAlg +from methods.elasticdnn.model.base import ElasticDNNUtil +from methods.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from methods.elasticdnn.pipeline.offline.fm_to_md.vit import FM_to_MD_ViT_Util +from methods.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from methods.elasticdnn.pipeline.offline.fm_lora.vit import FMLoRA_ViT_Util +from methods.elasticdnn.model.vit import ElasticViTUtil +from utils.common.file import ensure_dir +from utils.dl.common.model import LayerActivation, get_module, get_parameter +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +from utils.dl.common.loss import CrossEntropyLossSoft +import torch.nn.functional as F +from utils.dl.common.env import create_tbwriter +import os +import shutil +from utils.common.log import logger +from utils.common.data_record import write_json +# from methods.shot.shot import OnlineShotModel +from methods.feat_align.main import OnlineFeatAlignModel, FeatAlignAlg +import tqdm +from methods.feat_align.mmd import mmd_rbf +from methods.base.alg import BaseAlg +from methods.base.model import BaseModel +from copy import deepcopy +import time + + +def baseline_da(app_name: str, + scenario: Scenario, + da_alg: BaseAlg, + da_alg_hyp: dict, + da_model: BaseModel, + device, + __entry_file__, + tag=None): + + # involve_fm = settings['involve_fm'] + + task_name = app_name + # online_model = elasticfm_model + + log_dir = get_res_save_dir(__entry_file__, tag=tag) + tb_writer = create_tbwriter(os.path.join(log_dir, 'tb_log'), False) + res = [] + global_avg_after_acc = 0. + global_iter = 0 + + for domain_index, _ in enumerate(scenario.target_domains_order): + + cur_target_domain_name = scenario.target_domains_order[scenario.cur_domain_index] + if cur_target_domain_name in da_alg_hyp: + da_alg_hyp = da_alg_hyp[cur_target_domain_name] + logger.info(f'use dataset-specific da_alg_hyp') + + da_metrics, after_da_model = da_alg( + {'main': da_model}, + os.path.join(log_dir, f'{task_name}/{domain_index}') + ).run(scenario, da_alg_hyp) + # os.remove(tmp_sd_path) + + # 前面在当前域上训练，在这里压缩调优？ + # print(da_model.models_dict['main']) + # 进行压缩 + reducing_width_ratio = 8 + samples = torch.rand(1, 3, 224, 224).to(device) + + trained_fm_model = deepcopy(da_model.models_dict['main']) + fm_da_model = deepcopy(da_model) # 保存大模型 + lora_util = FMLoRA_ViT_Util() + lora_absorbed_fm_model = lora_util.absorb_lora_and_recover_net_structure(trained_fm_model, samples) + compressed_fm_model = FM_to_MD_ViT_Util().init_md_from_fm_by_reducing_width_with_perf_test(lora_absorbed_fm_model, reducing_width_ratio, samples) + da_model.models_dict['main'] = compressed_fm_model + + # 进行调优？之前那个da_metrics是FM的结果吧，调优也能得到一个精度结果换成这个？ + + datasets_for_training = scenario.get_online_cur_domain_datasets_for_training() + train_dataset = datasets_for_training[cur_target_domain_name]['train'] + val_dataset = datasets_for_training[cur_target_domain_name]['val'] + datasets_for_inference = scenario.get_online_cur_domain_datasets_for_inference() + test_dataset = datasets_for_inference + + train_loader = iter(build_dataloader(train_dataset, da_alg_hyp['train_batch_size'], da_alg_hyp['num_workers'], True, None)) + test_loader = build_dataloader(test_dataset, da_alg_hyp['val_batch_size'], da_alg_hyp['num_workers'], False, False) + + for p in compressed_fm_model.parameters(): + p.requires_grad = True + da_model.to_train_mode() + + # 'distill_optimizer': 'AdamW', + # 'distill_optimizer_args': {'lr': 1e-4, 'betas': [0.9, 0.999], 'weight_decay': 0.01}, + optimizer = torch.optim.__dict__['AdamW']([ + {'params': da_model.models_dict['main'].parameters(), **{'lr': 1e-4, 'betas': [0.9, 0.999], 'weight_decay': 0.01}} + ]) + if da_alg_hyp['scheduler'] != '': + scheduler = torch.optim.lr_scheduler.__dict__[da_alg_hyp['scheduler']](optimizer, **da_alg_hyp['scheduler_args']) + else: + scheduler = None + + pbar = tqdm.tqdm(range(da_alg_hyp['num_iters']), dynamic_ncols=True) + + accs = [] + total_train_time = 0. + cur_acc = 0. + for iter_index in pbar: + cur_start_time = time.time() + da_model.to_train_mode() + fm_da_model.to_eval_mode() + + x, y = next(train_loader) + if isinstance(x, dict): + for k, v in x.items(): + if isinstance(v, torch.Tensor): + x[k] = v.to(device) + y = y.to(device) + else: + x, y = x.to(device), y.to(device) + + with torch.no_grad(): + fm_output = fm_da_model.infer(x) + md_output = da_model.infer(x) + + distill_criterion = CrossEntropyLossSoft() + total_loss = distill_criterion(md_output, fm_output) + + optimizer.zero_grad() + total_loss.backward() + optimizer.step() + if scheduler is not None: + scheduler.step() + total_train_time += time.time() - cur_start_time + + if (iter_index + 1) % da_alg_hyp['val_freq'] == 0: + from data import split_dataset + cur_md = da_model.models_dict['main'] + md_for_test = deepcopy(da_model.models_dict['main']) + da_model.models_dict['main'] = md_for_test + cur_test_batch_dataset = split_dataset(test_dataset, da_alg_hyp['val_batch_size'], iter_index + 1)[0] + cur_test_batch_dataloader = build_dataloader(cur_test_batch_dataset, da_alg_hyp['train_batch_size'], da_alg_hyp['num_workers'], False, False) + da_model.to_eval_mode() + cur_acc = da_model.get_accuracy(cur_test_batch_dataloader) + accs += [{ + 'iter': iter_index + 1, + 'acc': cur_acc + }] + pbar.set_description(f'loss: {total_loss:.6f}, cur_acc: {cur_acc:.4f}') + + time_usage = total_train_time + da_metrics = { + 'accs': accs, + 'time': time_usage + } + da_model = fm_da_model # 恢复大模型 + + # 蒸馏结束 + + + if domain_index > 0: + shutil.rmtree(os.path.join(log_dir, f'{task_name}/{domain_index}/backup_codes')) + + accs = da_metrics['accs'] + before_acc = accs[0]['acc'] + after_acc = accs[-1]['acc'] + + tb_writer.add_scalars(f'accs/{task_name}', dict(before=before_acc, after=after_acc), domain_index) + tb_writer.add_scalar(f'times/{task_name}', da_metrics['time'], domain_index) + + for _acc in accs: + tb_writer.add_scalar('total_acc', _acc['acc'], _acc['iter'] + global_iter) + global_iter += _acc['iter'] + 1 + + scenario.next_domain() + + logger.info(f"task: {task_name}, domain {domain_index}, acc: {before_acc:.4f} -> " + f"{after_acc:.4f} ({da_metrics['time']:.2f}s)") + + global_avg_after_acc += after_acc + cur_res = da_metrics + res += [cur_res] + write_json(os.path.join(log_dir, 'res.json'), res, backup=False) + + global_avg_after_acc /= (domain_index + 1) + logger.info(f'-----> final metric: {global_avg_after_acc:.4f}') + write_json(os.path.join(log_dir, f'res_{global_avg_after_acc:.4f}.json'), res, backup=False) + \ No newline at end of file diff --git a/new_impl/cv/utils/elasticfm_cl.py b/new_impl/cv/utils/elasticfm_cl.py new file mode 100644 index 0000000000000000000000000000000000000000..45f833c23df8996da14ef484171f9f03e9726866 --- /dev/null +++ b/new_impl/cv/utils/elasticfm_cl.py @@ -0,0 +1,175 @@ +import sys +from utils.dl.common.env import set_random_seed +set_random_seed(1) + +from typing import List +from data.dataloader import build_dataloader +from data import Scenario, CLScenario +from methods.elasticdnn.api.online_model_v2 import ElasticDNN_OnlineModel + +import torch +import sys +from torch import nn +from methods.elasticdnn.api.model import ElasticDNN_OfflineSegFMModel, ElasticDNN_OfflineSegMDModel +from methods.elasticdnn.api.algs.md_pretraining_wo_fbs import ElasticDNN_MDPretrainingWoFBSAlg +from methods.elasticdnn.model.base import ElasticDNNUtil +from methods.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from methods.elasticdnn.pipeline.offline.fm_to_md.vit import FM_to_MD_ViT_Util +from methods.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from methods.elasticdnn.pipeline.offline.fm_lora.vit import FMLoRA_ViT_Util +from methods.elasticdnn.model.vit import ElasticViTUtil +from utils.common.file import ensure_dir +from utils.dl.common.model import LayerActivation, get_module, get_parameter +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +from utils.dl.common.loss import CrossEntropyLossSoft +import torch.nn.functional as F +from utils.dl.common.env import create_tbwriter +import os +from utils.common.log import logger +from utils.common.data_record import write_json +# from methods.shot.shot import OnlineShotModel +from methods.feat_align.main import OnlineFeatAlignModel, FeatAlignAlg +import tqdm +from methods.feat_align.mmd import mmd_rbf +from methods.base.alg import BaseAlg +from methods.base.model import BaseModel +import shutil + + +def elasticfm_cl(apps_name: List[str], + scenarios: List[CLScenario], + elasticfm_models: List[ElasticDNN_OnlineModel], + da_algs: List[BaseAlg], + da_alg_hyps: List[dict], + da_models: List[BaseModel], + device, + settings, + __entry_file__, + tag=None): + + assert len(apps_name) == 1 + + involve_fm = settings['involve_fm'] + + tasks_name = apps_name + online_models = elasticfm_models + + log_dir = get_res_save_dir(__entry_file__, tag=tag) + tb_writer = create_tbwriter(os.path.join(log_dir, 'tb_log'), False) + res = [] + global_avg_after_acc = 0. + global_iter = 0 + + last_da_model_attrs = {} + + for domain_index, _ in enumerate(scenarios[0].target_tasks_order): + avg_before_acc, avg_after_acc = 0., 0. + cur_res = {} + + for task_name, online_model, scenario, da_alg, da_model, da_alg_hyp in zip(tasks_name, online_models, scenarios, da_algs, da_models, da_alg_hyps): + + cur_target_domain_name = scenario.target_tasks_order[scenario.cur_task_index] + if cur_target_domain_name in da_alg_hyp: + da_alg_hyp = da_alg_hyp[cur_target_domain_name] + logger.info(f'use dataset-specific hyps') + + if domain_index == 0: # add last layer + logger.info(f'add num of classes in the last layer') + online_model.add_cls_in_head(scenario.num_tasks_to_be_learn) + + online_model.set_sd_sparsity(da_alg_hyp['sd_sparsity']) + sd, unpruned_indexes_of_layers = online_model.generate_sd_by_target_samples(scenario.get_online_cur_task_samples_for_training(da_alg_hyp['train_batch_size'])) + + tmp_sd_path = os.path.join(log_dir, 'tmp_sd_model.pt') + torch.save({'main': sd}, tmp_sd_path) + + if 'cls' not in task_name and 'pos' not in task_name and 'vqa' not in task_name: + da_model_args = [f'{task_name}/{domain_index}', + tmp_sd_path, + device, + scenario.num_classes] + else: + da_model_args = [f'{task_name}/{domain_index}', + tmp_sd_path, + device] + + da_model_ent = da_model(*da_model_args) + for k, v in last_da_model_attrs.items(): + logger.info(f'set attr of last model: {k}') + setattr(da_model_ent, k, v) + + da_metrics, after_da_model = da_alg( + {'main': da_model_ent}, + os.path.join(log_dir, f'{task_name}/{domain_index}') + ).run(scenario, {_k: _v for _k, _v in da_alg_hyp.items() if _k != 'sd_sparsity'}) + + last_da_model_attrs = {k: getattr(after_da_model['main'], k) for k in dir(after_da_model['main']) if str(k).startswith('_el_data_')} + logger.info(f'collect attrs in last model: {last_da_model_attrs.keys()}') + + os.remove(tmp_sd_path) + if domain_index > 0: + shutil.rmtree(os.path.join(log_dir, f'{task_name}/{domain_index}/backup_codes')) + + + online_model.last_trained_cls_indexes = torch.LongTensor([i for i in range(scenario.get_cur_class_offset(), + scenario.get_cur_class_offset() + scenario.get_cur_num_class())]) + + online_model.sd_feedback_to_md(after_da_model['main'].models_dict['main'], unpruned_indexes_of_layers) + online_model.md_feedback_to_self_fm() + + accs = da_metrics['accs'] + before_acc = accs[0]['acc'] + after_acc = accs[-1]['acc'] + + avg_before_acc += before_acc + avg_after_acc += after_acc + + for _acc in accs: + tb_writer.add_scalar(f'total_acc', _acc['acc'], _acc['iter'] + global_iter) # TODO: bug here + global_iter += _acc['iter'] + 1 + + tb_writer.add_scalars(f'accs/{task_name}', dict(before=before_acc, after=after_acc), domain_index) + tb_writer.add_scalar(f'times/{task_name}', da_metrics['time'], domain_index) + + scenario.next_task() + + logger.info(f"task: {task_name}, domain {domain_index}, acc: {before_acc:.4f} -> " + f"{after_acc:.4f} ({da_metrics['time']:.2f}s)") + cur_res[task_name] = da_metrics + + if involve_fm: + for online_model in online_models: + online_model.aggregate_fms_to_self_fm([m.models_dict['fm'] for m in online_models]) + for online_model in online_models: + online_model.fm_feedback_to_md() + + avg_before_acc /= len(tasks_name) + avg_after_acc /= len(tasks_name) + tb_writer.add_scalars(f'accs/apps_avg', dict(before=avg_before_acc, after=avg_after_acc), domain_index) + logger.info(f"--> domain {domain_index}, avg_acc: {avg_before_acc:.4f} -> " + f"{avg_after_acc:.4f}") + res += [cur_res] + + global_avg_after_acc += avg_after_acc + + write_json(os.path.join(log_dir, 'res.json'), res, backup=False) + + global_avg_after_acc /= (domain_index + 1) + logger.info(f'-----> final metric: {global_avg_after_acc:.4f}') + write_json(os.path.join(log_dir, f'res_{global_avg_after_acc:.4f}.json'), res, backup=False) + + + +def init_online_model(fm_models_dict_path, md_models_dict_path, task_name, __entry_file__): + fm_models = torch.load(fm_models_dict_path) + md_models = torch.load(md_models_dict_path) + + online_models_dict_path = save_models_dict_for_init({ + 'fm': fm_models['main'], + 'md': md_models['main'], + 'sd': None, + 'indexes': md_models['indexes'], + 'bn_stats': md_models['bn_stats'] + }, __entry_file__, task_name) + return online_models_dict_path diff --git a/new_impl/cv/utils/elasticfm_da.py b/new_impl/cv/utils/elasticfm_da.py new file mode 100644 index 0000000000000000000000000000000000000000..ea8fe6095af31d704bfadc35497f47f261176090 --- /dev/null +++ b/new_impl/cv/utils/elasticfm_da.py @@ -0,0 +1,157 @@ +import sys +from utils.dl.common.env import set_random_seed +set_random_seed(1) + +from typing import List +from data.dataloader import build_dataloader +from data import Scenario +from new_impl.cv.elasticdnn.api.online_model_v2 import ElasticDNN_OnlineModel + +import torch +import sys +from torch import nn +from new_impl.cv.elasticdnn.api.model import ElasticDNN_OfflineSegFMModel, ElasticDNN_OfflineSegMDModel +from new_impl.cv.elasticdnn.api.algs.md_pretraining_wo_fbs import ElasticDNN_MDPretrainingWoFBSAlg +from new_impl.cv.elasticdnn.model.base import ElasticDNNUtil +from new_impl.cv.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from new_impl.cv.elasticdnn.pipeline.offline.fm_to_md.vit import FM_to_MD_ViT_Util +from new_impl.cv.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from new_impl.cv.elasticdnn.pipeline.offline.fm_lora.vit import FMLoRA_ViT_Util +from new_impl.cv.elasticdnn.model.vit import ElasticViTUtil +from utils.common.file import ensure_dir +from utils.dl.common.model import LayerActivation, get_module, get_parameter +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +from utils.dl.common.loss import CrossEntropyLossSoft +import torch.nn.functional as F +from utils.dl.common.env import create_tbwriter +import os +from utils.common.log import logger +from utils.common.data_record import write_json +# from methods.shot.shot import OnlineShotModel +from new_impl.cv.feat_align.main import OnlineFeatAlignModel, FeatAlignAlg +import tqdm +from new_impl.cv.feat_align.mmd import mmd_rbf +from new_impl.cv.base.alg import BaseAlg +import shutil +from new_impl.cv.base.model import BaseModel + + +def elasticfm_da(apps_name: List[str], + scenarios: List[Scenario], + elasticfm_models: List[ElasticDNN_OnlineModel], + da_algs: List[BaseAlg], + da_alg_hyps: List[dict], + da_models: List[BaseModel], + device, + settings, + __entry_file__, + tag=None, + collate_fn=None): + + involve_fm = settings['involve_fm'] + + tasks_name = apps_name + online_models = elasticfm_models + + log_dir = get_res_save_dir(__entry_file__, tag=tag) + tb_writer = create_tbwriter(os.path.join(log_dir, 'tb_log'), False) + res = [] + global_avg_after_acc = 0. + global_iter = 0 + + for domain_index, _ in enumerate(scenarios[0].target_domains_order): + avg_before_acc, avg_after_acc = 0., 0. + cur_res = {} + + for task_name, online_model, scenario, da_alg, da_model, da_alg_hyp in zip(tasks_name, online_models, scenarios, da_algs, da_models, da_alg_hyps): + cur_target_domain_name = scenario.target_domains_order[scenario.cur_domain_index] + if cur_target_domain_name in da_alg_hyp: + da_alg_hyp = da_alg_hyp[cur_target_domain_name] + logger.info(f'use dataset-specific hyps') + online_model.set_sd_sparsity(da_alg_hyp['sd_sparsity']) + if 'transform' in da_alg_hyp.keys(): + sd, unpruned_indexes_of_layers = online_model.generate_sd_by_target_samples(scenario.get_online_cur_domain_samples_for_training(da_alg_hyp['train_batch_size'], da_alg_hyp['transform'], collate_fn=collate_fn)) + else: + sd, unpruned_indexes_of_layers = online_model.generate_sd_by_target_samples(scenario.get_online_cur_domain_samples_for_training(da_alg_hyp['train_batch_size'], collate_fn=collate_fn)) + + tmp_sd_path = os.path.join(log_dir, 'tmp_sd_model.pt') + # tmp_sd_path = 'new_impl/cv/glip/object_detection/results/det_online.py/20231127/999998-175207-results/tmp_sd_model.pt' + torch.save({'main': sd}, tmp_sd_path) + + if 'cls' not in task_name and 'pos' not in task_name and 'vqa' not in task_name: + da_model_args = [f'{task_name}/{domain_index}', + tmp_sd_path, + device, + scenario.num_classes] + else: + da_model_args = [f'{task_name}/{domain_index}', + tmp_sd_path, + device] + da_metrics, after_da_model = da_alg( + {'main': da_model(*da_model_args)}, + os.path.join(log_dir, f'{task_name}/{domain_index}') + ).run(scenario, {_k: _v for _k, _v in da_alg_hyp.items() if _k != 'sd_sparsity'}, collate_fn=collate_fn) + os.remove(tmp_sd_path) + + if domain_index > 0: + shutil.rmtree(os.path.join(log_dir, f'{task_name}/{domain_index}/backup_codes')) + + online_model.sd_feedback_to_md(after_da_model['main'].models_dict['main'], unpruned_indexes_of_layers) + online_model.md_feedback_to_self_fm() + #print(online_model.models_dict['sd']) + accs = da_metrics['accs'] + before_acc = accs[0]['acc'] + after_acc = accs[-1]['acc'] + + avg_before_acc += before_acc + avg_after_acc += after_acc + + for _acc in accs: + tb_writer.add_scalar(f'total_acc', _acc['acc'], _acc['iter'] + global_iter) # TODO: bug here + global_iter += _acc['iter'] + 1 + + tb_writer.add_scalars(f'accs/{task_name}', dict(before=before_acc, after=after_acc), domain_index) + tb_writer.add_scalar(f'times/{task_name}', da_metrics['time'], domain_index) + + scenario.next_domain() + + logger.info(f"task: {task_name}, domain {domain_index}, acc: {before_acc:.4f} -> " + f"{after_acc:.4f} ({da_metrics['time']:.2f}s)") + cur_res[task_name] = da_metrics + + if involve_fm: + for online_model in online_models: + online_model.aggregate_fms_to_self_fm([m.models_dict['fm'] for m in online_models]) + for online_model in online_models: + online_model.fm_feedback_to_md() + + avg_before_acc /= len(tasks_name) + avg_after_acc /= len(tasks_name) + tb_writer.add_scalars(f'accs/apps_avg', dict(before=avg_before_acc, after=avg_after_acc), domain_index) + logger.info(f"--> domain {domain_index}, avg_acc: {avg_before_acc:.4f} -> " + f"{avg_after_acc:.4f}") + res += [cur_res] + + global_avg_after_acc += avg_after_acc + + write_json(os.path.join(log_dir, 'res.json'), res, backup=False) + + global_avg_after_acc /= (domain_index + 1) + logger.info(f'-----> final metric: {global_avg_after_acc:.4f}') + write_json(os.path.join(log_dir, f'res_{global_avg_after_acc:.4f}.json'), res, backup=False) + + + +def init_online_model(fm_models_dict_path, md_models_dict_path, task_name, __entry_file__): + fm_models = torch.load(fm_models_dict_path) + md_models = torch.load(md_models_dict_path) + + online_models_dict_path = save_models_dict_for_init({ + 'fm': fm_models['main'], + 'md': md_models['main'], + 'sd': None, + 'indexes': md_models['indexes'], + 'bn_stats': md_models['bn_stats'] + }, __entry_file__, task_name) + return online_models_dict_path diff --git a/new_impl/hugging_face_impl/bert/impl.py b/new_impl/hugging_face_impl/bert/impl.py new file mode 100644 index 0000000000000000000000000000000000000000..d5a1ae17f4767256c33c9181612104fceb39fb98 --- /dev/null +++ b/new_impl/hugging_face_impl/bert/impl.py @@ -0,0 +1,54 @@ +from methods.elasticdnn.hugging_face.user_impl import HuggingFaceModelAPI +from utils.dl.common.model import LayerActivation, get_module +import torch +import torch.nn.functional as F +from torch import nn +import tqdm + + +class BERTHuggingFaceModelAPI(HuggingFaceModelAPI): + def get_feature_hook(self, fm: nn.Module, device) -> LayerActivation: + return LayerActivation(get_module(fm, 'classifier'), True, device) + + def get_task_head_params(self, fm: nn.Module): + head = get_module(fm, 'classifier') + return list(head.parameters()) + + def get_qkv_proj_ff1_ff2_layer_names(self): + return [[f'bert.encoder.layer.{i}.attention.self.query', f'bert.encoder.layer.{i}.attention.self.key', f'bert.encoder.layer.{i}.attention.self.value', \ + f'bert.encoder.layer.{i}.attention.output.dense', \ + f'bert.encoder.layer.{i}.intermediate.dense', f'bert.encoder.layer.{i}.output.dense'] for i in range(12)] + + def get_accuracy(self, fm: nn.Module, test_loader, device, *args, **kwargs): + acc = 0 + sample_num = 0 + + fm.eval() + + with torch.no_grad(): + pbar = tqdm.tqdm(enumerate(test_loader), total=len(test_loader), dynamic_ncols=True, leave=False) + for batch_index, (x, y) in pbar: + for k, v in x.items(): + if isinstance(v, torch.Tensor): + x[k] = v.to(device) + y = y.to(device) + output = self.infer(fm, x) + pred = F.softmax(output, dim=1).argmax(dim=1) + + # print(pred, y) + + correct = torch.eq(pred, y).sum().item() + acc += correct + sample_num += len(y) + + pbar.set_description(f'cur_batch_total: {len(y)}, cur_batch_correct: {correct}, ' + f'cur_batch_acc: {(correct / len(y)):.4f}') + + acc /= sample_num + return acc + + def infer(self, fm: nn.Module, x, *args, **kwargs): + return fm(**x) + + def forward_to_get_task_loss(self, fm: nn.Module, x, y, *args, **kwargs): + return F.cross_entropy(self.infer(fm, x), y) \ No newline at end of file diff --git a/new_impl/hugging_face_impl/bert/offline/fm_lora.py b/new_impl/hugging_face_impl/bert/offline/fm_lora.py new file mode 100644 index 0000000000000000000000000000000000000000..2c99fb04edbba57750a89d8abf6d37f03988e331 --- /dev/null +++ b/new_impl/hugging_face_impl/bert/offline/fm_lora.py @@ -0,0 +1,83 @@ +import torch +from methods.elasticdnn.api.model import ElasticDNN_OfflineSenClsFMModel, ElasticDNN_OfflineSenClsMDModel +from methods.elasticdnn.api.algs.fm_lora import ElasticDNN_FMLoRAAlg +from methods.elasticdnn.model.base import ElasticDNNUtil +from methods.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from methods.elasticdnn.pipeline.offline.fm_lora.bert import FMLoRA_Bert_Util +from methods.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from methods.elasticdnn.pipeline.offline.fm_to_md.vit import FM_to_MD_ViT_Util +from methods.elasticdnn.model.vit import ElasticViTUtil +from utils.dl.common.model import LayerActivation, get_module, get_parameter, set_module +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +from utils.common.log import logger +import torch.nn.functional as F +import sys + + +if __name__ == '__main__': + from new_impl.hugging_face_impl.bert.impl import BERTHuggingFaceModelAPI + api = BERTHuggingFaceModelAPI() + + from utils.dl.common.env import set_random_seed + set_random_seed(1) + + scenario = build_scenario( + source_datasets_name=['HL5Domains-ApexAD2600Progressive', 'HL5Domains-CanonG3', 'HL5Domains-CreativeLabsNomadJukeboxZenXtra40GB', + 'HL5Domains-NikonCoolpix4300', 'HL5Domains-Nokia6610'], + target_datasets_order=['Liu3Domains-Computer'] * 1, # TODO + da_mode='close_set', + data_dirs={ + **{k: f'/data/zql/datasets/nlp_asc_19_domains/dat/absa/Bing5Domains/asc/{k.split("-")[1]}' + for k in ['HL5Domains-ApexAD2600Progressive', 'HL5Domains-CanonG3', 'HL5Domains-CreativeLabsNomadJukeboxZenXtra40GB', + 'HL5Domains-NikonCoolpix4300', 'HL5Domains-Nokia6610']}, + 'Liu3Domains-Computer': '' + }, + ) + + # 2. init model + device = 'cuda' + from dnns.bert import bert_base_sen_cls + model = bert_base_sen_cls(num_classes=scenario.num_classes) + from methods.elasticdnn.pipeline.fm_to_md.bert import BertSelfAttentionPrunable + for block in model.bert.encoder.layer: + set_module(block, 'attention.self', BertSelfAttentionPrunable.init_from_exist_self_attn(block.attention.self)) + + fm_models_dict_path = save_models_dict_for_init({ + 'main': model + }, __file__, 'fm_bert_base_pretrained_with_sen_cls_head') + + # fm_model = ElasticDNN_Bert_OfflineSenClsFMModel('fm', fm_models_dict_path, device) + from methods.elasticdnn.hugging_face.internal_adapter import ElasticDNN_OfflineFMModel_for_HuggingFaceFM + fm_model = ElasticDNN_OfflineFMModel_for_HuggingFaceFM('fm', fm_models_dict_path, device) + fm_model.set_hugging_face_api(api) + + # 3. init alg + models = { + 'fm': fm_model + } + fm_lora_alg = ElasticDNN_FMLoRAAlg(models, get_res_save_dir(__file__, sys.argv[1])) + + + # 4. run alg + from utils.dl.common.lr_scheduler import get_linear_schedule_with_warmup + fm_lora_alg.run(scenario, hyps={ + 'launch_tbboard': False, + + 'samples_size': {'input_ids': torch.tensor([[ 101, 5672, 2033, 2011, 2151, 3793, 2017, 1005, 1040, 2066, 1012, 102]]).to(device), + 'token_type_ids': torch.tensor([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]).to(device), + 'attention_mask': torch.tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]).to(device), 'return_dict': False}, + + 'ab_r': 8, + 'train_batch_size': 8, + 'val_batch_size': 16, + 'num_workers': 16, + 'optimizer': 'AdamW', + 'optimizer_args': {'lr': 1e-4, 'betas': [0.9, 0.999]}, + 'scheduler': 'LambdaLR', + 'scheduler_args': {'lr_lambda': get_linear_schedule_with_warmup(10000, 310000)}, + 'num_iters': 320000, + 'val_freq': 2, + + # 'fm_lora_ckpt_path': 'experiments/elasticdnn/vit_b_16/offline/fm_lora/cls/results/cls.py/20230607/999995-234355-trial/models/fm_best.pt' + }) \ No newline at end of file diff --git a/new_impl/hugging_face_impl/vit/impl.py b/new_impl/hugging_face_impl/vit/impl.py new file mode 100644 index 0000000000000000000000000000000000000000..ea1b366ec1d6fb91bd8f4dc62f313eb9cb6f6109 --- /dev/null +++ b/new_impl/hugging_face_impl/vit/impl.py @@ -0,0 +1,46 @@ +from methods.elasticdnn.hugging_face.user_impl import HuggingFaceModelAPI +from utils.dl.common.model import LayerActivation, get_module +import torch +import torch.nn.functional as F +from torch import nn +import tqdm + + +class ViTHuggingFaceModelAPI(HuggingFaceModelAPI): + def get_feature_hook(self, fm: nn.Module, device) -> LayerActivation: + return LayerActivation(get_module(fm, 'head'), True, device) + + def get_task_head_params(self, fm: nn.Module): + head = get_module(fm, 'head') + return list(head.parameters()) + + def get_qkv_proj_ff1_ff2_layer_names(self): + return [[f'blocks.{i}.attn.qkv', f'blocks.{i}.attn.proj', f'blocks.{i}.mlp.fc1', f'blocks.{i}.mlp.fc2', ] for i in range(12)] + + def get_accuracy(self, fm: nn.Module, test_loader, device, *args, **kwargs): + acc = 0 + sample_num = 0 + + fm.eval() + + with torch.no_grad(): + pbar = tqdm.tqdm(enumerate(test_loader), total=len(test_loader), dynamic_ncols=True, leave=False) + for batch_index, (x, y) in pbar: + x, y = x.to(device), y.to(device) + output = fm(x) + pred = F.softmax(output, dim=1).argmax(dim=1) + correct = torch.eq(pred, y).sum().item() + acc += correct + sample_num += len(y) + + pbar.set_description(f'cur_batch_total: {len(y)}, cur_batch_correct: {correct}, ' + f'cur_batch_acc: {(correct / len(y)):.4f}') + + acc /= sample_num + return acc + + def infer(self, fm: nn.Module, x, *args, **kwargs): + return fm(x) + + def forward_to_get_task_loss(self, fm: nn.Module, x, y, *args, **kwargs): + return F.cross_entropy(self.infer(fm, x), y) \ No newline at end of file diff --git a/new_impl/hugging_face_impl/vit/offline/fm_lora.py b/new_impl/hugging_face_impl/vit/offline/fm_lora.py new file mode 100644 index 0000000000000000000000000000000000000000..5fa9510869766ab033f0121558111568f2b6a9fc --- /dev/null +++ b/new_impl/hugging_face_impl/vit/offline/fm_lora.py @@ -0,0 +1,70 @@ +import torch +from methods.elasticdnn.api.model import ElasticDNN_OfflineClsFMModel, ElasticDNN_OfflineClsMDModel +from methods.elasticdnn.api.algs.fm_lora import ElasticDNN_FMLoRAAlg +from methods.elasticdnn.model.base import ElasticDNNUtil +from methods.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from methods.elasticdnn.pipeline.offline.fm_lora.vit import FMLoRA_ViT_Util +from methods.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from methods.elasticdnn.pipeline.offline.fm_to_md.vit import FM_to_MD_ViT_Util +from methods.elasticdnn.model.vit import ElasticViTUtil +from utils.dl.common.model import LayerActivation, get_module +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +import torch.nn.functional as F + + + + + +if __name__ == '__main__': + from new_impl.hugging_face_impl.vit.impl import ViTHuggingFaceModelAPI + api = ViTHuggingFaceModelAPI() + + # 1. init scenario + scenario = build_scenario( + source_datasets_name=['GTA5Cls', 'SuperviselyPersonCls'], + target_datasets_order=['CityscapesCls', 'BaiduPersonCls'] * 15, + da_mode='close_set', + data_dirs={ + 'GTA5Cls': '/data/zql/datasets/gta5_for_cls_task', + 'SuperviselyPersonCls': '/data/zql/datasets/supervisely_person_for_cls_task', + 'CityscapesCls': '/data/zql/datasets/cityscapes_for_cls_task', + 'BaiduPersonCls': '/data/zql/datasets/baidu_person_for_cls_task' + }, + ) + + # 2. init model + from dnns.vit import vit_b_16 + fm_models_dict_path = save_models_dict_for_init({ + 'main': vit_b_16(pretrained=True, num_classes=scenario.num_classes) + }, __file__, 'fm_vit_b_16_pretrained') + device = 'cuda' + # fm_model = ElasticDNN_ViT_OfflineClsFMModel('fm', fm_models_dict_path, device) + from methods.elasticdnn.hugging_face.internal_adapter import ElasticDNN_OfflineFMModel_for_HuggingFaceFM + fm_model = ElasticDNN_OfflineFMModel_for_HuggingFaceFM('fm', fm_models_dict_path, device) + fm_model.set_hugging_face_api(api) + + # 3. init alg + models = { + 'fm': fm_model + } + import sys + fm_lora_alg = ElasticDNN_FMLoRAAlg(models, get_res_save_dir(__file__, tag=sys.argv[0])) + + # 4. run alg + from utils.dl.common.lr_scheduler import get_linear_schedule_with_warmup + fm_lora_alg.run(scenario, hyps={ + 'launch_tbboard': False, + + 'samples_size': (1, 3, 224, 224), + 'ab_r': 8, + 'train_batch_size': 256, + 'val_batch_size': 512, + 'num_workers': 16, + 'optimizer': 'Adam', + 'optimizer_args': {'lr': 5e-4, 'betas': [0.9, 0.999]}, + 'scheduler': 'LambdaLR', + 'scheduler_args': {'lr_lambda': get_linear_schedule_with_warmup(10000, 70000)}, + 'num_iters': 80000, + 'val_freq': 2 # NOTE: for debug, save the model every 2 iterations + }) \ No newline at end of file diff --git a/new_impl/hugging_face_impl/vit/offline/pretrain_knowledge_base.py b/new_impl/hugging_face_impl/vit/offline/pretrain_knowledge_base.py new file mode 100644 index 0000000000000000000000000000000000000000..88652aec6341af71d7b71e9b1363d73fbe03e996 --- /dev/null +++ b/new_impl/hugging_face_impl/vit/offline/pretrain_knowledge_base.py @@ -0,0 +1,88 @@ +import torch +import sys +from torch import nn +from dnns.vit import make_softmax_prunable +from methods.elasticdnn.api.model import ElasticDNN_OfflineClsFMModel, ElasticDNN_OfflineClsMDModel +from methods.elasticdnn.api.algs.md_pretraining_wo_fbs import ElasticDNN_MDPretrainingWoFBSAlg +from methods.elasticdnn.model.base import ElasticDNNUtil +from methods.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from methods.elasticdnn.pipeline.offline.fm_to_md.vit import FM_to_MD_ViT_Util +from methods.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from methods.elasticdnn.pipeline.offline.fm_lora.vit import FMLoRA_ViT_Util +from methods.elasticdnn.model.vit import ElasticViTUtil +from utils.dl.common.model import LayerActivation, get_module, get_parameter +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +from utils.dl.common.loss import CrossEntropyLossSoft +import torch.nn.functional as F + + + + + +if __name__ == '__main__': + from new_impl.hugging_face_impl.vit.impl import ViTHuggingFaceModelAPI + api = ViTHuggingFaceModelAPI() + + from utils.dl.common.env import set_random_seed + set_random_seed(1) + + # 1. init model + fm_models_dict_path = 'new_impl/hugging_face_impl/vit/offline/results/fm_lora.py/20231203/999991-170453-new_impl/hugging_face_impl/vit/offline/fm_lora.py/models/fm_best.pt' + fm_models_dict = torch.load(fm_models_dict_path) + fm_models_dict_path = save_models_dict_for_init(fm_models_dict, __file__, 'fm_vit_b_16_cls_lora') + md_models_dict_path = save_models_dict_for_init({ + 'main': -1 + }, __file__, 'md_vit_b_16_none') + device = 'cuda' + + # fm_model = ElasticDNN_ViT_OfflineClsFMModel('fm', fm_models_dict_path, device) + # md_model = ElasticDNN_ViT_OfflineClsMDModel('md', md_models_dict_path, device) + + from methods.elasticdnn.hugging_face.internal_adapter import ElasticDNN_OfflineFMModel_for_HuggingFaceFM, ElasticDNN_OfflineMDModel_for_HuggingFaceFM + fm_model = ElasticDNN_OfflineFMModel_for_HuggingFaceFM('fm', fm_models_dict_path, device) + fm_model.set_hugging_face_api(api) + md_model = ElasticDNN_OfflineMDModel_for_HuggingFaceFM('md', md_models_dict_path, device) + md_model.set_hugging_face_api(api) + + # 2. init alg + models = { + 'fm': fm_model, + 'md': md_model + } + fm_to_md_alg = ElasticDNN_MDPretrainingWoFBSAlg(models, get_res_save_dir(__file__, sys.argv[0])) + + # 3. init scenario + scenario = build_scenario( + source_datasets_name=['GTA5Cls', 'SuperviselyPersonCls'], + target_datasets_order=['CityscapesCls', 'BaiduPersonCls'] * 15, + da_mode='close_set', + data_dirs={ + 'GTA5Cls': '/data/zql/datasets/gta5_for_cls_task', + 'SuperviselyPersonCls': '/data/zql/datasets/supervisely_person_for_cls_task', + 'CityscapesCls': '/data/zql/datasets/cityscapes_for_cls_task', + 'BaiduPersonCls': '/data/zql/datasets/baidu_person_for_cls_task' + }, + ) + + from utils.dl.common.lr_scheduler import get_linear_schedule_with_warmup + fm_to_md_alg.run(scenario, hyps={ + 'launch_tbboard': False, + + 'samples_size': (1, 3, 224, 224), + 'generate_md_width_ratio': 4, + + 'train_batch_size': 128, + 'val_batch_size': 512, + 'num_workers': 16, + 'optimizer': 'AdamW', + 'optimizer_args': {'lr': 1e-4, 'betas': [0.9, 0.999], 'weight_decay': 0.01}, + 'scheduler': 'LambdaLR', + 'scheduler_args': {'lr_lambda': get_linear_schedule_with_warmup(10000, 70000)}, + 'num_iters': 80000, + 'val_freq': 2, + 'distill_loss_weight': 1.0 + }) + + # TODO: + # 1. train MD before inserting FBS? \ No newline at end of file diff --git a/new_impl/hugging_face_impl/vit/offline/pretrain_knowledge_base_and_index.py b/new_impl/hugging_face_impl/vit/offline/pretrain_knowledge_base_and_index.py new file mode 100644 index 0000000000000000000000000000000000000000..8bf0125d710d562e43bbfca8e09fdbc6a2f7a7d7 --- /dev/null +++ b/new_impl/hugging_face_impl/vit/offline/pretrain_knowledge_base_and_index.py @@ -0,0 +1,100 @@ +import torch +import sys +from torch import nn +from dnns.vit import make_softmax_prunable +from methods.elasticdnn.api.model import ElasticDNN_OfflineClsFMModel, ElasticDNN_OfflineClsMDModel +# from methods.elasticdnn.api.algs.md_pretraining_w_fbs import ElasticDNN_MDPretrainingWFBSAlg +from methods.elasticdnn.model.base import ElasticDNNUtil +from methods.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from methods.elasticdnn.pipeline.offline.fm_to_md.vit import FM_to_MD_ViT_Util +from methods.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from methods.elasticdnn.pipeline.offline.fm_lora.vit import FMLoRA_ViT_Util +from methods.elasticdnn.model.vit import ElasticViTUtil +from utils.dl.common.model import LayerActivation, get_module, get_parameter +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +from utils.dl.common.loss import CrossEntropyLossSoft +import torch.nn.functional as F + + +if __name__ == '__main__': + from new_impl.hugging_face_impl.vit.impl import ViTHuggingFaceModelAPI + api = ViTHuggingFaceModelAPI() + + from utils.dl.common.env import set_random_seed + set_random_seed(1) + + # 1. init model + fm_models_dict_path = 'new_impl/hugging_face_impl/vit/offline/results/pretrain_knowledge_base.py/20231203/999991-172715-new_impl/hugging_face_impl/vit/offline/pretrain_knowledge_base.py/models/fm_best.pt' + fm_models_dict_path = save_models_dict_for_init(torch.load(fm_models_dict_path), __file__, 'fm_vit_b_16_cls_lora') + pretrained_md_models_dict_path = 'new_impl/hugging_face_impl/vit/offline/results/pretrain_knowledge_base.py/20231203/999991-172715-new_impl/hugging_face_impl/vit/offline/pretrain_knowledge_base.py/models/md_best.pt' + md_models_dict = torch.load(pretrained_md_models_dict_path) + md_models_dict_path = save_models_dict_for_init(md_models_dict, __file__, 'md_vit_b_16_cls_pretrained_wo_fbs') + device = 'cuda' + + from methods.elasticdnn.hugging_face.internal_adapter import ElasticDNN_OfflineFMModel_for_HuggingFaceFM, ElasticDNN_OfflineMDModel_for_HuggingFaceFM + fm_model = ElasticDNN_OfflineFMModel_for_HuggingFaceFM('fm', fm_models_dict_path, device) + fm_model.set_hugging_face_api(api) + md_model = ElasticDNN_OfflineMDModel_for_HuggingFaceFM('md', md_models_dict_path, device) + md_model.set_hugging_face_api(api) + + # 2. init alg + models = { + 'fm': fm_model, + 'md': md_model + } + from methods.elasticdnn.api.algs.md_pretraining_index_v2_train_index_and_md import ElasticDNN_MDPretrainingIndexAlg + + fm_to_md_alg = ElasticDNN_MDPretrainingIndexAlg(models, get_res_save_dir(__file__, sys.argv[0])) + + # 3. init scenario + scenario = build_scenario( + source_datasets_name=['GTA5Cls', 'SuperviselyPersonCls'], + target_datasets_order=['CityscapesCls', 'BaiduPersonCls'] * 15, + da_mode='close_set', + data_dirs={ + 'GTA5Cls': '/data/zql/datasets/gta5_for_cls_task', + 'SuperviselyPersonCls': '/data/zql/datasets/supervisely_person_for_cls_task', + 'CityscapesCls': '/data/zql/datasets/cityscapes_for_cls_task', + 'BaiduPersonCls': '/data/zql/datasets/baidu_person_for_cls_task' + }, + ) + + from utils.dl.common.lr_scheduler import get_linear_schedule_with_warmup + fm_to_md_alg.run(scenario, hyps={ + 'launch_tbboard': False, + + 'samples_size': (1, 3, 224, 224), + + 'FBS_r': 16, + 'FBS_ignore_layers': [], + + 'train_batch_size': 128, + 'val_batch_size': 512, + 'num_workers': 16, + 'optimizer': 'AdamW', + # 'optimizer_args': {'lr': 1e-4, 'betas': [0.9, 0.999], 'weight_decay': 0.01}, + 'indexes_optimizer_args': {'lr': 3e-3, 'betas': [0.9, 0.999], 'weight_decay': 0.1}, + # 'scheduler': 'StepLR', + # 'scheduler_args': {'step_size': 20000, 'gamma': 0.1}, + # 'optimizer': 'AdamW', + # 'optimizer_args': {'lr': 1e-4, 'betas': [0.9, 0.999], 'weight_decay': 0.01}, + 'optimizer_args': {'lr': 1e-5, 'betas': [0.9, 0.999], 'weight_decay': 0.01}, + + 'scheduler': 'LambdaLR', + 'scheduler_args': {'lr_lambda': get_linear_schedule_with_warmup(10000, 70000)}, + + 'max_sparsity': 0.9, + 'min_sparsity': 0.0, + 'num_iters': 60000, + 'val_freq': 2, + 'index_loss_weight': 1e-4, + 'l1_reg_loss_weight': 1e-9, + 'val_num_sparsities': 4, + + 'bn_cal_num_iters': 0, + + 'index_init': 'zero', + 'index_guided_linear_comb_split_size': 512 + }) + \ No newline at end of file diff --git a/new_impl/mm/Vis_bert/QuestionAnswering/blip.py b/new_impl/mm/Vis_bert/QuestionAnswering/blip.py new file mode 100644 index 0000000000000000000000000000000000000000..3b99e81e45ada2f7e121442193ef4b98eda94150 --- /dev/null +++ b/new_impl/mm/Vis_bert/QuestionAnswering/blip.py @@ -0,0 +1,1464 @@ +from transformers import BlipForQuestionAnswering, BlipConfig,BlipModel +import torch +from torch import nn +from abc import ABC, abstractmethod +from copy import deepcopy +from typing import Optional, Union +from einops import rearrange, repeat +from einops.layers.torch import Rearrange +import tqdm + +from utils.dl.common.model import get_model_device, get_model_latency, get_model_size, set_module +from utils.dl.common.model import set_module, get_module, get_super_module +from utils.common.log import logger +from new_impl.cv.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util, LoRA +from transformers.models.blip.modeling_blip import BlipAttention +from transformers.models.blip.modeling_blip_text import BlipTextSelfAttention,BlipTextAttention,BlipTextSelfOutput +from new_impl.cv.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from new_impl.cv.elasticdnn.model.base import Abs, KTakesAll, ElasticDNNUtil, Layer_WrappedWithFBS + +from typing import Optional, Tuple +import math + +def blip(num_classes): + model = BlipForQuestionAnswering.from_pretrained('new_impl/mm/Vis_bert/QuestionAnswering/VisBert_pretrained') + # linear = model.text_decoder.cls.predictions.decoder + # new_linear = nn.Linear(linear.in_features,30524,bias = True) + # set_module(model,'text_decoder.cls.predictions.decoder',new_linear) + return model +# def blip(num_classes): +# model = BlipForQuestionAnswering.from_pretrained('new_impl/mm/Vis_bert/QuestionAnswering/VisBert_pretrained') +# linear = model.text_decoder.cls.predictions.decoder +# new_linear = nn.Linear(linear.in_features,num_classes,bias = True) +# set_module(model,'text_decoder.cls.predictions.decoder',new_linear) +# return model +# class blip(nn.Module): +# def __init__(self,num_classes): +# super(blip,self).__init__() +# self.blip = BlipModel.from_pretrained('new_impl/mm/Vis_bert/QuestionAnswering/VisBert_pretrained') +# self.cls = nn.Linear(768,num_classes*3) + +# def forward(self,**sample): +# output = self.blip(**sample)[-1]#output the last hidden +# output = self.cls(output[1]) +# return output + + +class ToQKV_WrappedWithLoRA(nn.Module): + def __init__(self, fc: nn.Linear, ab_r: int): + super(ToQKV_WrappedWithLoRA, self).__init__() + + self.fc = fc + self.ab = self.create_ab_as_linear(fc.weight.data, ab_r) + + def create_ab_as_linear(self, fc_weight: torch.Tensor, ab_r: int): + res = nn.Sequential( + LoRA(fc_weight.size(1), fc_weight.size(0) // ab_r, bias=False), + LoRA(fc_weight.size(0) // ab_r, fc_weight.size(0), bias=False) + ).to(fc_weight.device) + nn.init.kaiming_uniform_(res[0].weight, a=5 ** 0.5) + nn.init.zeros_(res[1].weight) + return res + + def forward(self, x): + x1 = self.fc(x) + x2 = self.ab(x) + return x1 + x2 + + +class FMLoRA_blip_Util(FMLoRA_Util): + + @torch.no_grad() + def add_lora_ab_to_fm(self, fm: nn.Module, ab_r: int, samples: dict): + fm.eval() + + # print(samples) + for k, v in samples.items(): + if isinstance(v, torch.Tensor): + samples[k] = v.to(get_model_device(fm)) + + o1 = fm.generate(**samples) + #o1 = fm(**samples) + for name, module in fm.named_modules(): + if name.endswith(('query', 'key', 'value')): + set_module(fm, name, ToQKV_WrappedWithLoRA(module, ab_r)) + elif name.endswith('.qkv'): + set_module(fm, name, ToQKV_WrappedWithLoRA(module, ab_r)) + + + o2 = fm.generate(**samples) + #o2 = fm(**samples) + if isinstance(o1, tuple): + o1 = o1[-1] + o2 = o2[-1] + output_diff = ((o1 - o2) ** 2).sum() + assert output_diff < 1e-5 + return fm + + @torch.no_grad() + def absorb_lora_and_recover_net_structure(self, fm: nn.Module, samples: dict): + fm.eval() + # print('absorb lora before') + + for k, v in samples.items(): + if isinstance(v, torch.Tensor): + samples[k] = v.to(get_model_device(fm)) + + o1 = fm.generate(**samples) + + for name, module in fm.named_modules(): + if not isinstance(module, ToQKV_WrappedWithLoRA): + continue + + fc = module.fc + ab = module.ab + + fc.weight.add_(ab[1].weight @ ab[0].weight) + + set_module(fm, name, fc) + + # print('absorb lora after') + o2 = fm.generate(**samples) + + if isinstance(o1, tuple): + o1 = o1[-1] + o2 = o2[-1] + output_diff = ((o1 - o2) ** 2).sum() + assert output_diff < 1e-6, output_diff + + return fm + + +####Here start with Fbs + +class blipTextAttentionPrunable(BlipTextSelfAttention): + def __init__(self,is_cross_attention): + config = BlipConfig.from_pretrained('new_impl/mm/Vis_bert/QuestionAnswering/VisBert_pretrained') + super(blipTextAttentionPrunable,self).__init__(config.text_config,is_cross_attention) + + def save_attn_gradients(self, attn_gradients): + self.attn_gradients = attn_gradients + + def get_attn_gradients(self): + return self.attn_gradients + + def save_attention_map(self, attention_map): + self.attention_map = attention_map + + def get_attention_map(self): + return self.attention_map + + def transpose_for_scores(self, x): + new_x_shape = x.size()[:-1] + (self.num_attention_heads, -1) + x = x.view(*new_x_shape) + return x.permute(0, 2, 1, 3) + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.FloatTensor] = None, + head_mask: Optional[torch.FloatTensor] = None, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + encoder_attention_mask: Optional[torch.FloatTensor] = None, + past_key_value: Optional[Tuple[Tuple[torch.FloatTensor]]] = None, + output_attentions: Optional[bool] = False, + ) -> Tuple[torch.Tensor]: + mixed_query_layer = self.query(hidden_states) + + # If this is instantiated as a cross-attention module, the keys + # and values come from an encoder; the attention mask needs to be + # such that the encoder's padding tokens are not attended to. + is_cross_attention = encoder_hidden_states is not None + + if is_cross_attention: + key_layer = self.transpose_for_scores(self.key(encoder_hidden_states)) + value_layer = self.transpose_for_scores(self.value(encoder_hidden_states)) + attention_mask = encoder_attention_mask + elif past_key_value is not None: + key_layer = self.transpose_for_scores(self.key(hidden_states)) + value_layer = self.transpose_for_scores(self.value(hidden_states)) + key_layer = torch.cat([past_key_value[0], key_layer], dim=2) + value_layer = torch.cat([past_key_value[1], value_layer], dim=2) + else: + key_layer = self.transpose_for_scores(self.key(hidden_states)) + value_layer = self.transpose_for_scores(self.value(hidden_states)) + + query_layer = self.transpose_for_scores(mixed_query_layer) + + past_key_value = (key_layer, value_layer) + + # Take the dot product between "query" and "key" to get the raw attention scores. + attention_scores = torch.matmul(query_layer, key_layer.transpose(-1, -2)) + + if self.position_embedding_type == "relative_key" or self.position_embedding_type == "relative_key_query": + seq_length = hidden_states.size()[1] + position_ids_l = torch.arange(seq_length, dtype=torch.long, device=hidden_states.device).view(-1, 1) + position_ids_r = torch.arange(seq_length, dtype=torch.long, device=hidden_states.device).view(1, -1) + distance = position_ids_l - position_ids_r + positional_embedding = self.distance_embedding(distance + self.max_position_embeddings - 1) + positional_embedding = positional_embedding.to(dtype=query_layer.dtype) # fp16 compatibility + + if self.position_embedding_type == "relative_key": + relative_position_scores = torch.einsum("bhld,lrd->bhlr", query_layer, positional_embedding) + attention_scores = attention_scores + relative_position_scores + elif self.position_embedding_type == "relative_key_query": + relative_position_scores_query = torch.einsum("bhld,lrd->bhlr", query_layer, positional_embedding) + relative_position_scores_key = torch.einsum("bhrd,lrd->bhlr", key_layer, positional_embedding) + attention_scores = attention_scores + relative_position_scores_query + relative_position_scores_key + + attention_scores = attention_scores / math.sqrt(self.attention_head_size) + if attention_mask is not None: + # Apply the attention mask is (precomputed for all layers in BlipTextModel forward() function) + attention_scores = attention_scores + attention_mask.to(attention_scores.device) + + # Normalize the attention scores to probabilities. + attention_probs = nn.Softmax(dim=-1)(attention_scores) + + # This is actually dropping out entire tokens to attend to, which might + # seem a bit unusual, but is taken from the original Transformer paper. + attention_probs_dropped = self.dropout(attention_probs) + + # Mask heads if we want to + if head_mask is not None: + attention_probs_dropped = attention_probs_dropped * head_mask + + context_layer = torch.matmul(attention_probs_dropped, value_layer) + + context_layer = context_layer.permute(0, 2, 1, 3).contiguous() + new_context_layer_shape = context_layer.size()[:-2] + (-1,) + context_layer = context_layer.view(*new_context_layer_shape) + + outputs = (context_layer, attention_probs) if output_attentions else (context_layer,) + + outputs = outputs + (past_key_value,) + return outputs + @staticmethod + def init_from_exist_self_attn(attn: BlipTextSelfAttention,is_cross_attention): + # print(attn) + + res = blipTextAttentionPrunable(is_cross_attention) + + for attr in dir(attn): + # if str(attr) in ['transpose_for_scores'] or str(attr).startswith('_'): + # continue + # if isinstance(getattr(attn, attr), nn.Module): + # print(attr) + + if isinstance(getattr(attn, attr), nn.Module): + try: + # print(attr, 'ok') + setattr(res, attr, getattr(attn, attr)) + + except Exception as e: + print(attr, str(e)) + + + + return res + + + +# class blipSelfTextAttentionPrunable(BlipTextAttention): +# def __init__(self, config, is_cross_attention=False): +# self.self = blipTextAttentionPrunable(config, is_cross_attention) +# self.output = BlipTextSelfOutput(config) +# self.pruned_heads = set() +# super(blipSelfTextAttentionPrunable,self).__init__(config) + +# def prune_heads(self, heads): +# if len(heads) == 0: +# return +# heads, index = find_pruneable_heads_and_indices( +# heads, self.self.num_attention_heads, self.self.attention_head_size, self.pruned_heads +# ) + +# # Prune linear layers +# self.self.query = prune_linear_layer(self.self.query, index) +# self.self.key = prune_linear_layer(self.self.key, index) +# self.self.value = prune_linear_layer(self.self.value, index) +# self.output.dense = prune_linear_layer(self.output.dense, index, dim=1) + +# # Update hyper params and store pruned heads +# self.self.num_attention_heads = self.self.num_attention_heads - len(heads) +# self.self.all_head_size = self.self.attention_head_size * self.self.num_attention_heads +# self.pruned_heads = self.pruned_heads.union(heads) + +# def forward( +# self, +# hidden_states: torch.Tensor, +# attention_mask: Optional[torch.FloatTensor] = None, +# head_mask: Optional[torch.FloatTensor] = None, +# encoder_hidden_states: Optional[torch.FloatTensor] = None, +# encoder_attention_mask: Optional[torch.FloatTensor] = None, +# past_key_value: Optional[Tuple[Tuple[torch.FloatTensor]]] = None, +# output_attentions: Optional[bool] = False, +# ) -> Tuple[torch.Tensor]: +# self_outputs = self.self( +# hidden_states, +# attention_mask, +# head_mask, +# encoder_hidden_states, +# encoder_attention_mask, +# past_key_value, +# output_attentions, +# ) +# attention_output = self.output(self_outputs[0], hidden_states) +# outputs = (attention_output,) + self_outputs[1:] # add attentions if we output them +# return outputs +# @staticmethod +# def init_from_exist_self_attn(attn: BlipTextAttention,config,is_cross_attention): +# # print(attn) + +# res = blipTextAttentionPrunable(config,is_cross_attention) + +# for attr in dir(attn): +# # if str(attr) in ['transpose_for_scores'] or str(attr).startswith('_'): +# # continue +# # if isinstance(getattr(attn, attr), nn.Module): +# # print(attr) + +# if isinstance(getattr(attn, attr), nn.Module): +# try: +# # print(attr, 'ok') +# setattr(res, attr, getattr(attn, attr)) + +# except Exception as e: +# print(attr, str(e)) + + + +# return res + + + + + + +class blipSelfAttentionPrunable(BlipAttention): + def __init__(self): + config = BlipConfig.from_pretrained('new_impl/mm/Vis_bert/QuestionAnswering/VisBert_pretrained') + super(blipSelfAttentionPrunable, self).__init__(config.vision_config) + + def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int): + return tensor.view(bsz, seq_len, self.num_heads, -1).transpose(1, 2).contiguous() + + def forward( + self, + hidden_states: torch.Tensor, + head_mask: Optional[torch.Tensor] = None, + output_attentions: Optional[bool] = False, + ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: + """Input shape: Batch x Time x Channel""" + + bsz, tgt_len, embed_dim = hidden_states.size() + + mixed_qkv = ( + self.qkv(hidden_states) + .reshape(bsz, tgt_len, 3, self.num_heads, -1) + .permute(2, 0, 3, 1, 4) + ) + query_states, key_states, value_states = mixed_qkv[0], mixed_qkv[1], mixed_qkv[2] + + # Take the dot product between "query" and "key" to get the raw attention scores. + attention_scores = torch.matmul(query_states, key_states.transpose(-1, -2)) + + attention_scores = attention_scores * self.scale + + # Normalize the attention scores to probabilities. + attention_probs = nn.functional.softmax(attention_scores, dim=-1) + + # This is actually dropping out entire tokens to attend to, which might + # seem a bit unusual, but is taken from the original Transformer paper. + attention_probs = self.dropout(attention_probs) + + # Mask heads if we want to + if head_mask is not None: + attention_probs = attention_probs * head_mask + + context_layer = torch.matmul(attention_probs, value_states).permute(0, 2, 1, 3) + + new_context_layer_shape = context_layer.size()[:-2] + (-1,) + context_layer = context_layer.reshape(new_context_layer_shape) + + output = self.projection(context_layer) + + outputs = (output, attention_probs) if output_attentions else (output, None) + + return outputs + + @staticmethod + def init_from_exist_self_attn(attn: BlipAttention): + # print(attn) + + res = blipSelfAttentionPrunable() + + for attr in dir(attn): + # if str(attr) in ['transpose_for_scores'] or str(attr).startswith('_'): + # continue + # if isinstance(getattr(attn, attr), nn.Module): + # print(attr) + + if isinstance(getattr(attn, attr), nn.Module): + try: + # print(attr, 'ok') + setattr(res, attr, getattr(attn, attr)) + + except Exception as e: + print(attr, str(e)) + + + + return res + + +class FM_to_MD_blip_Util(FM_to_MD_Util): + def init_md_from_fm_by_reducing_width(self, fm: nn.Module, reducing_width_ratio: int) -> nn.Module: + fm_vis = deepcopy(fm) + config = BlipConfig.from_pretrained('new_impl/mm/Vis_bert/QuestionAnswering/VisBert_pretrained') + # for block in fm_vis.text_encoder.encoder.layer: + # set_module(block, 'attention.self', blipTextAttentionPrunable.init_from_exist_self_attn(block.attention.self,False)) + # for block in fm_vis.text_encoder.encoder.layer: + # set_module(block, 'crossattention.self', blipTextAttentionPrunable.init_from_exist_self_attn(block.crossattention.self,True)) + + for block in fm_vis.text_decoder.bert.encoder.layer: + set_module(block, 'attention.self', blipTextAttentionPrunable.init_from_exist_self_attn(block.attention.self,False)) + for block in fm_vis.text_decoder.bert.encoder.layer: + set_module(block, 'crossattention.self', blipTextAttentionPrunable.init_from_exist_self_attn(block.crossattention.self,True)) + # for block in fm_vis.vision_model.encoder.layers: + # set_module(block,'self_attn',blipSelfAttentionPrunable.init_from_exist_self_attn(block.self_attn)) + def _f(n): + return int(n // reducing_width_ratio) + + # def _rand_indexes(n): + # return torch.randperm(n)[0: int(n // reducing_width_ratio)] + + def l1_max_indexes(p: torch.Tensor, dim=0): + assert dim in [0, 1] + assert p.dim() in [1, 2, 4] + + if dim == 1: + p = p.T + + p_norm = p.abs().contiguous().view(p.size(0), -1).sum(dim=1) + n = p.size(0) + return p_norm.argsort(descending=True)[0: int(n // reducing_width_ratio)].sort()[0] + + for block_i, block in enumerate(fm_vis.text_decoder.bert.encoder.layer): + for k in ['query', 'key', 'value']: + qkv = get_module(block, f'attention.self.{k}') + + new_qkv = nn.Linear(qkv.in_features, _f(qkv.out_features), + qkv.bias is not None, qkv.weight.device) + indexes = l1_max_indexes(qkv.weight.data, 0) + + new_qkv.weight.data.copy_(qkv.weight.data[indexes]) + if qkv.bias is not None: + new_qkv.bias.data.copy_(qkv.bias.data[indexes]) + set_module(block, f'attention.self.{k}', new_qkv) + + proj = get_module(block, f'attention.output.dense') + new_proj = nn.Linear(_f(proj.in_features), proj.out_features, + proj.bias is not None, proj.weight.device) + new_proj.weight.data.copy_(proj.weight.data[:, l1_max_indexes(proj.weight.data, 1)]) + if proj.bias is not None: + new_proj.bias.data.copy_(proj.bias.data) + set_module(block, f'attention.output.dense', new_proj) + + fc1 = get_module(block, f'intermediate.dense') + new_fc1 = nn.Linear(fc1.in_features, _f(fc1.out_features), + fc1.bias is not None, fc1.weight.device) + indexes = l1_max_indexes(fc1.weight.data, 0) + new_fc1.weight.data.copy_(fc1.weight.data[indexes]) + if fc1.bias is not None: + new_fc1.bias.data.copy_(fc1.bias.data[indexes]) + set_module(block, f'intermediate.dense', new_fc1) + + fc2 = get_module(block, f'output.dense') + new_fc2 = nn.Linear(_f(fc2.in_features), fc2.out_features, + fc2.bias is not None, fc2.weight.device) + new_fc2.weight.data.copy_(fc2.weight.data[:, l1_max_indexes(fc2.weight.data, 1)]) + if fc2.bias is not None: + new_fc2.bias.data.copy_(fc2.bias.data) + set_module(block, f'output.dense', new_fc2) + + + for block_i, block in enumerate(fm_vis.text_decoder.bert.encoder.layer): + for k in ['query', 'key', 'value']: + qkv = get_module(block, f'crossattention.self.{k}') + + new_qkv = nn.Linear(qkv.in_features, _f(qkv.out_features), + qkv.bias is not None, qkv.weight.device) + indexes = l1_max_indexes(qkv.weight.data, 0) + + new_qkv.weight.data.copy_(qkv.weight.data[indexes]) + if qkv.bias is not None: + new_qkv.bias.data.copy_(qkv.bias.data[indexes]) + set_module(block, f'crossattention.self.{k}', new_qkv) + + proj = get_module(block, f'crossattention.output.dense') + new_proj = nn.Linear(_f(proj.in_features), proj.out_features, + proj.bias is not None, proj.weight.device) + new_proj.weight.data.copy_(proj.weight.data[:, l1_max_indexes(proj.weight.data, 1)]) + if proj.bias is not None: + new_proj.bias.data.copy_(proj.bias.data) + set_module(block, f'crossattention.output.dense', new_proj) + + + # for block_i, block in enumerate(fm_vis.text_encoder.encoder.layer): + # for k in ['query', 'key', 'value']: + # qkv = get_module(block, f'attention.self.{k}') + + # new_qkv = nn.Linear(qkv.in_features, _f(qkv.out_features), + # qkv.bias is not None, qkv.weight.device) + # indexes = l1_max_indexes(qkv.weight.data, 0) + + # new_qkv.weight.data.copy_(qkv.weight.data[indexes]) + # if qkv.bias is not None: + # new_qkv.bias.data.copy_(qkv.bias.data[indexes]) + # set_module(block, f'attention.self.{k}', new_qkv) + + # proj = get_module(block, f'attention.output.dense') + # new_proj = nn.Linear(_f(proj.in_features), proj.out_features, + # proj.bias is not None, proj.weight.device) + # new_proj.weight.data.copy_(proj.weight.data[:, l1_max_indexes(proj.weight.data, 1)]) + # if proj.bias is not None: + # new_proj.bias.data.copy_(proj.bias.data) + # set_module(block, f'attention.output.dense', new_proj) + + # fc1 = get_module(block, f'intermediate.dense') + # new_fc1 = nn.Linear(fc1.in_features, _f(fc1.out_features), + # fc1.bias is not None, fc1.weight.device) + # indexes = l1_max_indexes(fc1.weight.data, 0) + # new_fc1.weight.data.copy_(fc1.weight.data[indexes]) + # if fc1.bias is not None: + # new_fc1.bias.data.copy_(fc1.bias.data[indexes]) + # set_module(block, f'intermediate.dense', new_fc1) + + # fc2 = get_module(block, f'output.dense') + # new_fc2 = nn.Linear(_f(fc2.in_features), fc2.out_features, + # fc2.bias is not None, fc2.weight.device) + # new_fc2.weight.data.copy_(fc2.weight.data[:, l1_max_indexes(fc2.weight.data, 1)]) + # if fc2.bias is not None: + # new_fc2.bias.data.copy_(fc2.bias.data) + # set_module(block, f'output.dense', new_fc2) + + + # for block_i, block in enumerate(fm_vis.text_encoder.encoder.layer): + # for k in ['query', 'key', 'value']: + # qkv = get_module(block, f'crossattention.self.{k}') + + # new_qkv = nn.Linear(qkv.in_features, _f(qkv.out_features), + # qkv.bias is not None, qkv.weight.device) + # indexes = l1_max_indexes(qkv.weight.data, 0) + + # new_qkv.weight.data.copy_(qkv.weight.data[indexes]) + # if qkv.bias is not None: + # new_qkv.bias.data.copy_(qkv.bias.data[indexes]) + # set_module(block, f'crossattention.self.{k}', new_qkv) + + # proj = get_module(block, f'crossattention.output.dense') + # new_proj = nn.Linear(_f(proj.in_features), proj.out_features, + # proj.bias is not None, proj.weight.device) + # new_proj.weight.data.copy_(proj.weight.data[:, l1_max_indexes(proj.weight.data, 1)]) + # if proj.bias is not None: + # new_proj.bias.data.copy_(proj.bias.data) + # set_module(block, f'crossattention.output.dense', new_proj) + + + + # for block_i, block in enumerate(fm_vis.vision_model.encoder.layers): + # qkv = block.self_attn.qkv + + # new_qkv = nn.Linear(qkv.in_features, _f(qkv.out_features), + # qkv.bias is not None, qkv.weight.device) + # indexes = l1_max_indexes(qkv.weight.data, 0) + + # new_qkv.weight.data.copy_(qkv.weight.data[indexes]) + # if qkv.bias is not None: + # new_qkv.bias.data.copy_(qkv.bias.data[indexes]) + # set_module(fm_vis, f'vision_model.encoder.layers.{block_i}.self_attn.qkv', new_qkv) + + # proj = block.self_attn.projection + # new_proj = nn.Linear(_f(proj.in_features), proj.out_features, + # proj.bias is not None, proj.weight.device) + # new_proj.weight.data.copy_(proj.weight.data[:, l1_max_indexes(proj.weight.data, 1)]) + # if proj.bias is not None: + # new_proj.bias.data.copy_(proj.bias.data) + # set_module(fm_vis, f'vision_model.encoder.layers.{block_i}.self_attn.projection', new_proj) + + # fc1 = block.mlp.fc1 + # new_fc1 = nn.Linear(fc1.in_features, _f(fc1.out_features), + # fc1.bias is not None, fc1.weight.device) + # indexes = l1_max_indexes(fc1.weight.data, 0) + # new_fc1.weight.data.copy_(fc1.weight.data[indexes]) + # if fc1.bias is not None: + # new_fc1.bias.data.copy_(fc1.bias.data[indexes]) + # set_module(fm_vis, f'vision_model.encoder.layers.{block_i}.mlp.fc1', new_fc1) + + # fc2 = block.mlp.fc2 + # new_fc2 = nn.Linear(_f(fc2.in_features), fc2.out_features, + # fc2.bias is not None, fc2.weight.device) + # new_fc2.weight.data.copy_(fc2.weight.data[:, l1_max_indexes(fc2.weight.data, 1)]) + # if fc2.bias is not None: + # new_fc2.bias.data.copy_(fc2.bias.data) + # set_module(fm_vis, f'vision_model.encoder.layers.{block_i}.mlp.fc2', new_fc2) + + return fm_vis + + def init_md_from_fm_by_reducing_width_with_perf_test(self, fm: nn.Module, reducing_width_ratio: int, + samples: torch.Tensor) -> nn.Module: + fm_size = get_model_size(fm, True) + fm_latency = self._get_model_latency(fm, samples, 20, + get_model_device(fm), 20, False) + + master_dnn = self.init_md_from_fm_by_reducing_width(fm, reducing_width_ratio) + master_dnn_size = get_model_size(master_dnn, True) + logger.debug(f'inited master DNN: {master_dnn}') + master_dnn_latency = self._get_model_latency(master_dnn, samples, 20, + get_model_device(master_dnn), 20, False) + + logger.info(f'init master DNN (w/o FBS yet) by reducing foundation model\'s width (by {reducing_width_ratio:d}x)') + logger.info(f'foundation model ({fm_size:.3f}MB, {fm_latency:.4f}s/sample) -> ' + f'master DNN ({master_dnn_size:.3f}MB, {master_dnn_latency:.4f}s/sample)\n' + f'(model size: ↓ {(fm_size / master_dnn_size):.2f}x, ' + f'latency: ↓ {(fm_latency / master_dnn_latency):.2f}x)') + + return master_dnn + + def _get_model_latency(self, model: torch.nn.Module, model_input_size, sample_num: int, + device: str, warmup_sample_num: int, return_detail=False): + import time + + if isinstance(model_input_size, tuple): + dummy_input = torch.rand(model_input_size).to(device) + else: + dummy_input = model_input_size + + model = model.to(device) + model.eval() + + # warm up + with torch.no_grad(): + for _ in range(warmup_sample_num): + model(**dummy_input) + + infer_time_list = [] + + if device == 'cuda' or 'cuda' in str(device): + with torch.no_grad(): + for _ in range(sample_num): + s, e = torch.cuda.Event(enable_timing=True), torch.cuda.Event(enable_timing=True) + s.record() + model(**dummy_input) + e.record() + torch.cuda.synchronize() + cur_model_infer_time = s.elapsed_time(e) / 1000. + infer_time_list += [cur_model_infer_time] + + else: + with torch.no_grad(): + for _ in range(sample_num): + start = time.time() + model(**dummy_input) + cur_model_infer_time = time.time() - start + infer_time_list += [cur_model_infer_time] + + avg_infer_time = sum(infer_time_list) / sample_num + + if return_detail: + return avg_infer_time, infer_time_list + return avg_infer_time + + +####Here starts with index + +class SqueezeLast(nn.Module): + def __init__(self): + super(SqueezeLast, self).__init__() + + def forward(self, x): + return x.squeeze(-1) + + +class ProjConv_WrappedWithFBS(Layer_WrappedWithFBS): + def __init__(self, proj: nn.Conv2d, r): + super(ProjConv_WrappedWithFBS, self).__init__() + + self.proj = proj + + # for conv: (B, C_in, H, W) -> (B, C_in) -> (B, C_out) + # for mlp in ViT: (B, #patches, D: dim of patches embedding) -> (B, D) -> (B, C_out) + self.fbs = nn.Sequential( + Abs(), + nn.AdaptiveAvgPool1d(1), + SqueezeLast(), + nn.Linear(proj.in_channels, proj.out_channels // r), + nn.ReLU(), + nn.Linear(proj.out_channels // r, proj.out_channels), + nn.ReLU() + ) + + nn.init.constant_(self.fbs[6].bias, 1.) + nn.init.kaiming_normal_(self.fbs[6].weight) + + + def forward(self, x): + if self.use_cached_channel_attention and self.cached_channel_attention is not None: + channel_attention = self.cached_channel_attention + else: + self.cached_raw_channel_attention = self.fbs(x) + self.cached_channel_attention = self.k_takes_all(self.cached_raw_channel_attention) + + channel_attention = self.cached_channel_attention + + raw_res = self.proj(x) + + return channel_attention.unsqueeze(1) * raw_res # TODO: + + +class Linear_WrappedWithFBS(Layer_WrappedWithFBS): + def __init__(self, linear: nn.Linear, r): + super(Linear_WrappedWithFBS, self).__init__() + + self.linear = linear + + # for conv: (B, C_in, H, W) -> (B, C_in) -> (B, C_out) + # for mlp in ViT: (B, #patches, D: dim of patches embedding) -> (B, D) -> (B, C_out) + self.fbs = nn.Sequential( + Rearrange('b n d -> b d n'), + Abs(), + nn.AdaptiveAvgPool1d(1), + SqueezeLast(), + nn.Linear(linear.in_features, linear.out_features // r), + nn.ReLU(), + nn.Linear(linear.out_features // r, linear.out_features), + nn.ReLU() + ) + + nn.init.constant_(self.fbs[6].bias, 1.) + nn.init.kaiming_normal_(self.fbs[6].weight) + + + def forward(self, x): + if self.use_cached_channel_attention and self.cached_channel_attention is not None: + channel_attention = self.cached_channel_attention + else: + self.cached_raw_channel_attention = self.fbs(x) + self.cached_channel_attention = self.k_takes_all(self.cached_raw_channel_attention) + + channel_attention = self.cached_channel_attention + + raw_res = self.linear(x) + + return channel_attention.unsqueeze(1) * raw_res + + +class ToQKV_WrappedWithFBS(Layer_WrappedWithFBS): + """ + This regards to_q/to_k/to_v as a whole (in fact it consists of multiple heads) and prunes it. + It seems different channels of different heads are pruned according to the input. + This is different from "removing some head" or "removing the same channels in each head". + """ + def __init__(self, to_qkv: nn.Linear, r): + super(ToQKV_WrappedWithFBS, self).__init__() + + # self.to_qkv = to_qkv + + self.to_qk = nn.Linear(to_qkv.in_features, to_qkv.out_features // 3 * 2, bias=to_qkv.bias is not None) + self.to_v = nn.Linear(to_qkv.in_features, to_qkv.out_features // 3, bias=to_qkv.bias is not None) + self.to_qk.weight.data.copy_(to_qkv.weight.data[0: to_qkv.out_features // 3 * 2]) + if to_qkv.bias is not None: + self.to_qk.bias.data.copy_(to_qkv.bias.data[0: to_qkv.out_features // 3 * 2]) + self.to_v.weight.data.copy_(to_qkv.weight.data[to_qkv.out_features // 3 * 2: ]) + if to_qkv.bias is not None: + self.to_v.bias.data.copy_(to_qkv.bias.data[to_qkv.out_features // 3 * 2: ]) + + self.fbs = nn.Sequential( + Rearrange('b n d -> b d n'), + Abs(), + nn.AdaptiveAvgPool1d(1), + SqueezeLast(), + nn.Linear(to_qkv.in_features, to_qkv.out_features // 3 // r), + nn.ReLU(), + # nn.Linear(to_qkv.out_features // 3 // r, to_qkv.out_features // 3), + nn.Linear(to_qkv.out_features // 3 // r, self.to_v.out_features), + nn.ReLU() + ) + + nn.init.constant_(self.fbs[6].bias, 1.) + nn.init.kaiming_normal_(self.fbs[6].weight) + + def forward(self, x): + if self.use_cached_channel_attention and self.cached_channel_attention is not None: + channel_attention = self.cached_channel_attention + else: + self.cached_raw_channel_attention = self.fbs(x) + + # print() + # for attn in self.cached_raw_channel_attention.chunk(3, dim=1)[0: 1]: + # print(self.cached_raw_channel_attention.size(), attn.size()) + # print(self.k_takes_all.k) + # print(attn[0].nonzero(as_tuple=True)[0].size(), attn[0]) + + self.cached_channel_attention = self.k_takes_all(self.cached_raw_channel_attention) + + + # for attn in self.cached_channel_attention.chunk(3, dim=1)[0: 1]: + # print(self.cached_channel_attention.size(), attn.size()) + # print(self.k_takes_all.k) + # print(attn[0].nonzero(as_tuple=True)[0].size(), attn[0]) + # print() + + channel_attention = self.cached_channel_attention + + qk = self.to_qk(x) + v = channel_attention.unsqueeze(1) * self.to_v(x) + return torch.cat([qk, v], dim=-1) + + # qkv = raw_res.chunk(3, dim = -1) + + # raw_v = qkv[2] + # print('raw_k, raw_v', qkv[0].sum((0, 1))[0: 10], qkv[0].sum((0, 1)).nonzero(as_tuple=True)[0].size(), + # qkv[1].sum((0, 1))[0: 10], qkv[1].sum((0, 1)).nonzero(as_tuple=True)[0].size(),) + # print('raw_v', raw_v.size(), raw_v.sum((0, 1))[0: 10], raw_v.sum((0, 1)).nonzero(as_tuple=True)[0].size()) + + # qkv_attn = channel_attention.chunk(3, dim=-1) + # print('attn', [attn[0][0: 10] for attn in qkv_attn]) + # print(channel_attention.unsqueeze(1).size(), raw_res.size()) + # print('fbs', channel_attention.size(), raw_res.size()) + # return channel_attention.unsqueeze(1) * raw_res + + +class StaticFBS(nn.Module): + def __init__(self, static_channel_attention): + super(StaticFBS, self).__init__() + assert static_channel_attention.dim() == 2 and static_channel_attention.size(0) == 1 + self.static_channel_attention = nn.Parameter(static_channel_attention, requires_grad=False) # (1, dim) + + def forward(self, x): + # print('staticfbs', x, self.static_channel_attention.unsqueeze(1)) + return x * self.static_channel_attention.unsqueeze(1) + + +class ElasticblipUtil(ElasticDNNUtil): + def convert_raw_dnn_to_master_dnn(self, raw_dnn: nn.Module, r: float, ignore_layers=[]): + assert len(ignore_layers) == 0, 'not supported yet' + + raw_vit = deepcopy(raw_dnn) + + # set_module(module, 'patch_embed.proj', ProjConv_WrappedWithFBS(module.patch_embed.proj, r)) + + for name, module in raw_vit.named_modules(): + # if name.endswith('attn'): + # set_module(module, 'qkv', ToQKV_WrappedWithFBS(module.qkv, r)) + if name.endswith('intermediate'): + set_module(module, 'dense', Linear_WrappedWithFBS(module.dense, r)) + elif name.endswith('mlp'): + set_module(module, 'fc1', Linear_WrappedWithFBS(module.fc1, r)) + + return raw_vit + + def set_master_dnn_sparsity(self, master_dnn: nn.Module, sparsity: float): + # for name, module in master_dnn.named_modules(): + # if not name.endswith('attn'): + # continue + + # q_features = module.qkv.to_qk.out_features // 2 + + # if (q_features - int(q_features * sparsity)) % module.num_heads != 0: + # # tune sparsity to ensure #unpruned channel % num_heads == 0 + # # so that the pruning seems to reduce the dim_head of each head + # tuned_sparsity = 1. - int((q_features - int(q_features * sparsity)) / module.num_heads) * module.num_heads / q_features + # logger.debug(f'tune sparsity from {sparsity:.2f} to {tuned_sparsity}') + # sparsity = tuned_sparsity + # break + + return super().set_master_dnn_sparsity(master_dnn, sparsity) + + def select_most_rep_sample(self, master_dnn: nn.Module, samples: torch.Tensor): + # print(samples) + # return samples[0].unsqueeze(0) + res = {k: v[0: 1] for k, v in samples.items()} + return res + + def extract_surrogate_dnn_via_samples(self, master_dnn: nn.Module, samples: torch.Tensor, return_detail=False): + sample = self.select_most_rep_sample(master_dnn, samples) + # assert sample.dim() == 4 and sample.size(0) == 1 + + # print('before') + master_dnn.eval() + self.clear_cached_channel_attention_in_master_dnn(master_dnn) + with torch.no_grad(): + master_dnn_output = master_dnn(**sample) + + # print('after') + + boosted_vit = deepcopy(master_dnn) + + def get_unpruned_indexes_from_channel_attn(channel_attn: torch.Tensor, k): + assert channel_attn.size(0) == 1, 'use A representative sample to generate channel attentions' + + # print('attn_in_unpruned', channel_attn[0][0: 10]) + + res = channel_attn[0].nonzero(as_tuple=True)[0] # should be one-dim + + # res = channel_attn[0].argsort(descending=True)[0: -int(channel_attn.size(1) * k)].sort()[0] + + # g = channel_attn + # k = g.size(1) - int(g.size(1) * k) + # res = g.topk(k, 1)[1][0].sort()[0] + + return res + + unpruned_indexes_of_layers = {} + + # for attn, ff in boosted_vit.transformer.layers: + # for block_i, block in enumerate(boosted_vit.blocks): + for block_i, block in enumerate(boosted_vit.text_encoder.encoder.layer): + # attn = block.attn + # ff = block.mlp + + ff_0 = get_module(block, f'intermediate.dense') + # ff_0_unpruned_indexes = get_unpruned_indexes_from_channel_attn(ff_0.cached_channel_attention, k) + ff_0_pruned_indexes = ff_0.k_takes_all.cached_i[0].sort()[0] + ff_0_unpruned_indexes = torch.LongTensor([ii for ii in range(ff_0.cached_channel_attention.size(1)) if ii not in ff_0_pruned_indexes]) + new_ff_0 = nn.Linear(ff_0.linear.in_features, ff_0_unpruned_indexes.size(0), ff_0.linear.bias is not None) + new_ff_0.weight.data.copy_(ff_0.linear.weight.data[ff_0_unpruned_indexes]) + if ff_0.linear.bias is not None: + new_ff_0.bias.data.copy_(ff_0.linear.bias.data[ff_0_unpruned_indexes]) + set_module(block, 'intermediate.dense', nn.Sequential(new_ff_0, StaticFBS(ff_0.cached_channel_attention[:, ff_0_unpruned_indexes]))) + + ff_1 = get_module(block, f'output.dense') + new_ff_1 = nn.Linear(ff_0_unpruned_indexes.size(0), ff_1.out_features, ff_1.bias is not None) + new_ff_1.weight.data.copy_(ff_1.weight.data[:, ff_0_unpruned_indexes]) + if ff_1.bias is not None: + new_ff_1.bias.data.copy_(ff_1.bias.data) + set_module(block, 'output.dense', new_ff_1) + + unpruned_indexes_of_layers[f'text_encoder.encoder.layer.{block_i}.intermediate.dense.0.weight'] = ff_0_unpruned_indexes + for block_i,block in enumerate(boosted_vit.vision_model.encoder.layers): + + attn = block.self_attn + ff = block.mlp + ff_0 = ff.fc1 + # ff_0_unpruned_indexes = get_unpruned_indexes_from_channel_attn(ff_0.cached_channel_attention, k) + ff_0_pruned_indexes = ff_0.k_takes_all.cached_i[0].sort()[0] + ff_0_unpruned_indexes = torch.LongTensor([ii for ii in range(ff_0.cached_channel_attention.size(1)) if ii not in ff_0_pruned_indexes]) + new_ff_0 = nn.Linear(ff_0.linear.in_features, ff_0_unpruned_indexes.size(0), ff_0.linear.bias is not None) + new_ff_0.weight.data.copy_(ff_0.linear.weight.data[ff_0_unpruned_indexes]) + if ff_0.linear.bias is not None: + new_ff_0.bias.data.copy_(ff_0.linear.bias.data[ff_0_unpruned_indexes]) + set_module(ff, 'fc1', nn.Sequential(new_ff_0, StaticFBS(ff_0.cached_channel_attention[:, ff_0_unpruned_indexes]))) + + ff_1 = ff.fc2 + new_ff_1 = nn.Linear(ff_0_unpruned_indexes.size(0), ff_1.out_features, ff_1.bias is not None) + new_ff_1.weight.data.copy_(ff_1.weight.data[:, ff_0_unpruned_indexes]) + if ff_1.bias is not None: + new_ff_1.bias.data.copy_(ff_1.bias.data) + set_module(ff, 'fc2', new_ff_1) + + unpruned_indexes_of_layers[f'vision_model.encoder.layers.{block_i}.mlp.fc1.0.weight'] = ff_0_unpruned_indexes + + + for block_i, block in enumerate(boosted_vit.text_decoder.bert.encoder.layer): + # attn = block.attn + # ff = block.mlp + + ff_0 = get_module(block, f'intermediate.dense') + # ff_0_unpruned_indexes = get_unpruned_indexes_from_channel_attn(ff_0.cached_channel_attention, k) + ff_0_pruned_indexes = ff_0.k_takes_all.cached_i[0].sort()[0] + ff_0_unpruned_indexes = torch.LongTensor([ii for ii in range(ff_0.cached_channel_attention.size(1)) if ii not in ff_0_pruned_indexes]) + new_ff_0 = nn.Linear(ff_0.linear.in_features, ff_0_unpruned_indexes.size(0), ff_0.linear.bias is not None) + new_ff_0.weight.data.copy_(ff_0.linear.weight.data[ff_0_unpruned_indexes]) + if ff_0.linear.bias is not None: + new_ff_0.bias.data.copy_(ff_0.linear.bias.data[ff_0_unpruned_indexes]) + set_module(block, 'intermediate.dense', nn.Sequential(new_ff_0, StaticFBS(ff_0.cached_channel_attention[:, ff_0_unpruned_indexes]))) + + ff_1 = get_module(block, f'output.dense') + new_ff_1 = nn.Linear(ff_0_unpruned_indexes.size(0), ff_1.out_features, ff_1.bias is not None) + new_ff_1.weight.data.copy_(ff_1.weight.data[:, ff_0_unpruned_indexes]) + if ff_1.bias is not None: + new_ff_1.bias.data.copy_(ff_1.bias.data) + set_module(block, 'output.dense', new_ff_1) + + unpruned_indexes_of_layers[f'text_decoder.bert.encoder.layer.{block_i}.intermediate.dense.0.weight'] = ff_0_unpruned_indexes + surrogate_dnn = boosted_vit + surrogate_dnn.eval() + surrogate_dnn = surrogate_dnn.to(get_model_device(master_dnn)) + # logger.debug(surrogate_dnn) + with torch.no_grad(): + surrogate_dnn_output = surrogate_dnn(**sample) + + output_diff = ((surrogate_dnn_output.logits - master_dnn_output.logits) ** 2).sum() + # assert output_diff < 1e-4, output_diff + logger.info(f'output diff of master and surrogate DNN: {output_diff}') + # logger.debug(f'example output of master/surrogate: {master_dnn_output.sum(0)[0: 10]}, {surrogate_dnn_output.sum(0)[0: 10]}') + # logger.info(f'\nonly prune mlp!!!!\n') + # logger.info(f'\nonly prune mlp!!!!\n') + + if return_detail: + return boosted_vit, unpruned_indexes_of_layers + + return boosted_vit + + def extract_surrogate_dnn_via_samples_with_perf_test(self, master_dnn: nn.Module, samples: torch.Tensor, return_detail=False): + master_dnn_size = get_model_size(master_dnn, True) + master_dnn_latency = self._get_model_latency(master_dnn, samples, 50, + get_model_device(master_dnn), 50, False) + + res = self.extract_surrogate_dnn_via_samples(master_dnn, samples, return_detail) + if not return_detail: + surrogate_dnn = res + else: + surrogate_dnn, unpruned_indexes_of_layers = res + surrogate_dnn_size = get_model_size(surrogate_dnn, True) + surrogate_dnn_latency = self._get_model_latency(master_dnn, samples, 50, + get_model_device(master_dnn), 50, False) + + logger.info(f'master DNN ({master_dnn_size:.3f}MB, {master_dnn_latency:.4f}s/sample) -> ' + f'surrogate DNN ({surrogate_dnn_size:.3f}MB, {surrogate_dnn_latency:.4f}s/sample)\n' + f'(model size: ↓ {(master_dnn_size / surrogate_dnn_size):.2f}x, ' + f'latency: ↓ {(master_dnn_latency / surrogate_dnn_latency):.2f}x)') + + return res + + def _get_model_latency(self, model: torch.nn.Module, model_input_size, sample_num: int, + device: str, warmup_sample_num: int, return_detail=False): + import time + + if isinstance(model_input_size, tuple): + dummy_input = torch.rand(model_input_size).to(device) + else: + dummy_input = model_input_size + + model = model.to(device) + model.eval() + + # warm up + with torch.no_grad(): + for _ in range(warmup_sample_num): + model(**dummy_input) + + infer_time_list = [] + + if device == 'cuda' or 'cuda' in str(device): + with torch.no_grad(): + for _ in range(sample_num): + s, e = torch.cuda.Event(enable_timing=True), torch.cuda.Event(enable_timing=True) + s.record() + model(**dummy_input) + e.record() + torch.cuda.synchronize() + cur_model_infer_time = s.elapsed_time(e) / 1000. + infer_time_list += [cur_model_infer_time] + + else: + with torch.no_grad(): + for _ in range(sample_num): + start = time.time() + model(**dummy_input) + cur_model_infer_time = time.time() - start + infer_time_list += [cur_model_infer_time] + + avg_infer_time = sum(infer_time_list) / sample_num + + if return_detail: + return avg_infer_time, infer_time_list + return avg_infer_time + + + +#####Here starts with online + +from typing import List +from data.dataloader import build_dataloader +# from methods.elasticdnn.api.online_model import ElasticDNN_OnlineModel +from new_impl.cv.elasticdnn.api.online_model_v2 import ElasticDNN_OnlineModel + +import torch +import sys +from torch import nn +from methods.elasticdnn.api.model import ElasticDNN_OfflineSegFMModel, ElasticDNN_OfflineSegMDModel +from methods.elasticdnn.api.algs.md_pretraining_wo_fbs import ElasticDNN_MDPretrainingWoFBSAlg +from methods.elasticdnn.model.base import ElasticDNNUtil +from methods.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from methods.elasticdnn.pipeline.offline.fm_to_md.vit import FM_to_MD_ViT_Util +from methods.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from methods.elasticdnn.pipeline.offline.fm_lora.vit import FMLoRA_ViT_Util +from methods.elasticdnn.model.vilt import ElasticViltUtil +from utils.common.file import ensure_dir +from utils.dl.common.model import LayerActivation, get_module, get_parameter, set_module +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +from utils.dl.common.loss import CrossEntropyLossSoft +import torch.nn.functional as F +from utils.dl.common.env import create_tbwriter +import os +from utils.common.log import logger +from utils.common.data_record import write_json +# from methods.shot.shot import OnlineShotModel +from methods.ewc.ewc_elasticfm import OnlineEWCModel +import tqdm +# from methods.feat_align.mmd import mmd_rbf +from copy import deepcopy + + +class ElasticDNN_VQAOnlineModel(ElasticDNN_OnlineModel): + @torch.no_grad() + def sd_feedback_to_md(self, after_da_sd, unpruned_indexes_of_layers): + self.models_dict['sd'] = after_da_sd + self.before_da_md = deepcopy(self.models_dict['md']) + + logger.info('\n\nsurrogate DNN feedback to master DNN...\n\n') + # one-to-one + + cur_unpruned_indexes = None + cur_unpruned_indexes_name = None + + for p_name, p in self.models_dict['sd'].named_parameters(): + matched_md_param = self.get_md_matched_param_of_sd_param(p_name) + logger.debug(f'if feedback: {p_name}') + if matched_md_param is None: + continue + logger.debug(f'start feedback: {p_name}, {p.size()} -> {matched_md_param.size()}') + # average + # setattr(matched_md_module, matched_md_param_name, (matched_md_param + p) / 2.) + + if p_name in unpruned_indexes_of_layers.keys(): + cur_unpruned_indexes = unpruned_indexes_of_layers[p_name] + cur_unpruned_indexes_name = p_name + + if p.size() != matched_md_param.size(): + logger.debug(f'cur unpruned indexes: {cur_unpruned_indexes_name}, {cur_unpruned_indexes.size()}') + + if p.dim() == 1: # norm + new_p = deepcopy(matched_md_param) + new_p[cur_unpruned_indexes] = p + elif p.dim() == 2: # linear + if p.size(0) < matched_md_param.size(0): # output pruned + new_p = deepcopy(matched_md_param) + new_p[cur_unpruned_indexes] = p + else: # input pruned + new_p = deepcopy(matched_md_param) + new_p[:, cur_unpruned_indexes] = p + p = new_p + + assert p.size() == matched_md_param.size(), f'{p.size()}, {matched_md_param.size()}' + + # if 'head' in p_name: + if False: + continue + # if False: + # self.last_trained_cls_indexes + assert hasattr(self, 'last_trained_cls_indexes') + print(self.last_trained_cls_indexes) + + diff = self._compute_diff(matched_md_param, p) + # matched_md_param[self.last_trained_cls_indexes].copy_(p[self.last_trained_cls_indexes.to(self.device)]) + matched_md_param.copy_(p) + logger.debug(f'SPECIFIC FOR CL HEAD | end feedback: {p_name}, diff: {diff:.6f}') + else: + diff = self._compute_diff(matched_md_param, (matched_md_param + p) / 2.) + matched_md_param.copy_((matched_md_param + p) / 2.) + logger.debug(f'end feedback: {p_name}, diff: {diff:.6f}') + + def add_cls_in_head(self, num_cls): # NOTE: + head: nn.Linear = get_module(self.models_dict['md'], 'cls') + + new_head = nn.Linear(head.in_features, head.out_features + num_cls, head.bias is not None, device=self.device) + + # nn.init.zeros_(new_head.weight.data) + # nn.init.zeros_(new_head.bias.data) + + new_head.weight.data[0: head.out_features] = deepcopy(head.weight.data) + new_head.bias.data[0: head.out_features] = deepcopy(head.bias.data) + set_module(self.models_dict['md'], 'cls', new_head) + set_module(self.models_dict['fm'], 'cls', new_head) + + def get_accuracy(self, test_loader, *args, **kwargs): + acc = 0 + sample_num = 0 + + from methods.elasticdnn.api.model import VQAScore + vqa_score = VQAScore() + + self.to_eval_mode() + + # from transformers import AutoProcessor + # processor = AutoProcessor.from_pretrained("new_impl/mm/Vis_bert/QuestionAnswering/VisBert_pretrained") + # with torch.no_grad(): + # pbar = tqdm.tqdm(enumerate(test_loader), total=len(test_loader), dynamic_ncols=True, leave=False) + # for batch_index, (x, y, t) in pbar: + # for k, v in x.items(): + # if isinstance(v, torch.Tensor): + # x[k] = v.to(self.device) + # if isinstance(y,dict): + # for k, v in y.items(): + # y[k] = v.to(self.device) + # else: + # y = y.to(self.device) + + # output = self.models_dict['main'].generate(**x) + # total = 0 + # idx = 0 + # for i in output: + # val = processor.decode(i, skip_special_tokens=True) + # text = t[idx] + # if val == text: + # total += 1 + # idx += 1 + + # #vqa_score.update(output, y.labels) + # acc = total / (idx+1) + # #pbar.set_description(f'cur_batch_total: {len(y['label'])}, cur_batch_acc: {vqa_score.compute():.4f}') + # pbar.set_description(f'cur_batch_total: {len(y["labels"])}, cur_batch_acc: {acc:.4f}') + # return acc + with torch.no_grad(): + pbar = tqdm.tqdm(enumerate(test_loader), total=len(test_loader), dynamic_ncols=True, leave=False) + for batch_index, (x, y) in pbar: + for k, v in x.items(): + if isinstance(v, torch.Tensor): + x[k] = v.to(self.device) + y = y.to(self.device) + output = self.infer(x) + + vqa_score.update(output, y) + + pbar.set_description(f'cur_batch_total: {len(y)}, cur_batch_acc: {vqa_score.compute():.4f}') + + return float(vqa_score.compute()) + + def get_elastic_dnn_util(self) -> ElasticDNNUtil: + return ElasticblipUtil() + + def get_fm_matched_param_of_md_param(self, md_param_name): + # only between qkv.weight, norm.weight/bias + self_param_name = md_param_name + fm = self.models_dict['fm'] + if any([k in self_param_name for k in ['fbs', 'ab', 'embeddings']]): + return None + + p = get_parameter(self.models_dict['md'], self_param_name) + if p.dim() == 0: + return None + elif p.dim() == 1 and ('LayerNorm' in self_param_name or 'layernorm' in self_param_name) and 'weight' in self_param_name: + return get_parameter(fm, self_param_name) + + # 1. xx.qkv.to_qkv.yy to xx.qkv.qkv.aa and xx.qkv.abs.zz + if ('query' in self_param_name or 'key' in self_param_name or \ + 'value' in self_param_name) and ('weight' in self_param_name): + + ss = self_param_name.split('.') + + fm_qkv_name = '.'.join(ss[0: -1]) + '.fc' + fm_qkv = get_module(fm, fm_qkv_name) + + fm_abs_name = '.'.join(ss[0: -1]) + '.ab' + fm_abs = get_module(fm, fm_abs_name) + + # NOTE: unrecoverable operation! multiply LoRA parameters to allow it being updated in update_fm_param() + # TODO: if fm will be used for inference, _mul_lora_weight will not be applied! + if not hasattr(fm_abs, '_mul_lora_weight'): + logger.debug(f'set _mul_lora_weight in {fm_abs_name}') + setattr(fm_abs, '_mul_lora_weight', + nn.Parameter(fm_abs[1].weight @ fm_abs[0].weight)) + + return torch.cat([ + fm_qkv.weight.data, # task-agnositc params + fm_abs._mul_lora_weight.data # task-specific params (LoRA) + ], dim=0) + + # elif 'to_qkv.bias' in self_param_name: + # ss = self_param_name.split('.') + + # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv.bias' + # return get_parameter(fm, fm_qkv_name) + + elif 'dense' in self_param_name and 'weight' in self_param_name: + fm_param_name = self_param_name.replace('.linear', '') + return get_parameter(fm, fm_param_name) + + # elif 'mlp.fc2' in self_param_name and 'weight' in self_param_name: + # fm_param_name = self_param_name + # return get_parameter(fm, fm_param_name) + + else: + # return get_parameter(fm, self_param_name) + return None + + def update_fm_param(self, md_param_name, cal_new_fm_param_by_md_param): + if not ('query' in md_param_name or 'key' in md_param_name or 'value' in md_param_name): + matched_fm_param_ref = self.get_fm_matched_param_of_md_param(md_param_name) + matched_fm_param_ref.copy_(cal_new_fm_param_by_md_param) + else: + new_fm_attn_weight, new_fm_lora_weight = torch.chunk(cal_new_fm_param_by_md_param, 2, 0) + + ss = md_param_name.split('.') + fm = self.models_dict['fm'] + + # update task-agnostic parameters + fm_qkv_name = '.'.join(ss[0: -1]) + '.fc' + fm_qkv = get_module(fm, fm_qkv_name) + fm_qkv.weight.data.copy_(new_fm_attn_weight) + + # update task-specific parameters + fm_abs_name = '.'.join(ss[0: -1]) + '.ab' + fm_abs = get_module(fm, fm_abs_name) + fm_abs._mul_lora_weight.data.copy_(new_fm_lora_weight) # TODO: this will not be applied in inference! + + def get_md_matched_param_of_fm_param(self, fm_param_name): + return super().get_md_matched_param_of_fm_param(fm_param_name) + + def get_md_matched_param_of_sd_param(self, sd_param_name): + # raise NotImplementedError + + # only between qkv.weight, norm.weight/bias + self_param_name = sd_param_name + md = self.models_dict['md'] + if any([k in self_param_name for k in ['fbs', 'ab', 'embeddings']]): + return None + + p = get_parameter(self.models_dict['sd'], self_param_name) + if p.dim() == 0: + return None + elif p.dim() == 1 and ('LayerNorm' in self_param_name or 'layernorm' in self_param_name) and 'weight' in self_param_name: + return get_parameter(md, self_param_name) + + # 1. xx.qkv.to_qkv.yy to xx.qkv.qkv.aa and xx.qkv.abs.zz + if ('query' in self_param_name or 'key' in self_param_name or \ + 'value' in self_param_name) and ('weight' in self_param_name): + + + return get_parameter(md, self_param_name) # NOTE: no fbs in qkv! + + # elif 'to_qkv.bias' in self_param_name: + # ss = self_param_name.split('.') + + # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv.bias' + # return get_parameter(fm, fm_qkv_name) + + elif 'intermediate.dense.0.weight' in self_param_name: + fm_param_name = '.'.join(self_param_name.split('.')[0: -2]) + '.linear.weight' + return get_parameter(md, fm_param_name) + + elif 'output.dense' in self_param_name and 'weight' in self_param_name: + fm_param_name = self_param_name + return get_parameter(md, fm_param_name) + + else: + # return get_parameter(fm, self_param_name) + return None + + def get_task_head_params(self): + head = get_module(self.models_dict['sd'], 'cls') + return list(head.parameters()) + + + + +from typing import List, Tuple +from data.dataloader import build_dataloader +# from methods.elasticdnn.api.online_model import ElasticDNN_OnlineModel +from methods.elasticdnn.api.online_model_v2 import ElasticDNN_OnlineModel + +import torch +import sys +from torch import nn +from methods.elasticdnn.api.model import ElasticDNN_OfflineSegFMModel, ElasticDNN_OfflineSegMDModel +from methods.elasticdnn.api.algs.md_pretraining_wo_fbs import ElasticDNN_MDPretrainingWoFBSAlg +from methods.elasticdnn.model.base import ElasticDNNUtil +from methods.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from methods.elasticdnn.pipeline.offline.fm_to_md.vit import FM_to_MD_ViT_Util +from methods.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from methods.elasticdnn.pipeline.offline.fm_lora.vit import FMLoRA_ViT_Util +from methods.elasticdnn.model.vit import ElasticViTUtil +from utils.common.file import ensure_dir +from utils.dl.common.model import LayerActivation, LayerActivation2, get_module, get_parameter, set_module +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +from utils.dl.common.loss import CrossEntropyLossSoft +import torch.nn.functional as F +from utils.dl.common.env import create_tbwriter +import os +from utils.common.log import logger +from utils.common.data_record import write_json +# from methods.shot.shot import OnlineShotModel +from methods.feat_align.main import OnlineFeatAlignModel +import tqdm +from methods.feat_align.mmd import mmd_rbf +from copy import deepcopy + + +class VQAOnlineFeatAlignModel(OnlineFeatAlignModel): + def get_trained_params(self): + qkv_and_norm_params = [p for n, p in self.models_dict['main'].named_parameters() if 'query' in n or 'key' in n or 'value' in n or 'dense' in n or 'LayerNorm' in n] + return qkv_and_norm_params + + def get_feature_hook(self): + return LayerActivation(get_module(self.models_dict['main'], 'cls'), False, self.device) + + def forward_to_get_task_loss(self, x, y): + self.to_train_mode() + o = self.infer(x) + return F.binary_cross_entropy_with_logits(o, y) * y.shape[1] + # o = self.model_dict['main'](**x) + # return o.loss + + def get_mmd_loss(self, f1, f2): + return mmd_rbf(f1, f2) + + def infer(self, x, *args, **kwargs): + return self.models_dict['main'](**x) + + def get_accuracy(self, test_loader, *args, **kwargs): + acc = 0 + sample_num = 0 + + from methods.elasticdnn.api.model import VQAScore + vqa_score = VQAScore() + + self.to_eval_mode() + + # from transformers import AutoProcessor + # processor = AutoProcessor.from_pretrained("new_impl/mm/Vis_bert/QuestionAnswering/VisBert_pretrained") + # with torch.no_grad(): + # pbar = tqdm.tqdm(enumerate(test_loader), total=len(test_loader), dynamic_ncols=True, leave=False) + # for batch_index, (x, y, t) in pbar: + # for k, v in x.items(): + # if isinstance(v, torch.Tensor): + # x[k] = v.to(self.device) + # if isinstance(y,dict): + # for k, v in y.items(): + # y[k] = v.to(self.device) + # else: + # y = y.to(self.device) + + # output = self.models_dict['main'].generate(**x) + # total = 0 + # idx = 0 + # for i in output: + # val = processor.decode(i, skip_special_tokens=True) + # text = t[idx] + # if val == text: + # total += 1 + # idx += 1 + + # #vqa_score.update(output, y.labels) + # acc = total / (idx+1) + # #pbar.set_description(f'cur_batch_total: {len(y['label'])}, cur_batch_acc: {vqa_score.compute():.4f}') + # pbar.set_description(f'cur_batch_total: {len(y["labels"])}, cur_batch_acc: {acc:.4f}') + # return acc + with torch.no_grad(): + pbar = tqdm.tqdm(enumerate(test_loader), total=len(test_loader), dynamic_ncols=True, leave=False) + for batch_index, (x, y) in pbar: + for k, v in x.items(): + if isinstance(v, torch.Tensor): + x[k] = v.to(self.device) + y = y.to(self.device) + output = self.infer(x) + + vqa_score.update(output, y) + + pbar.set_description(f'cur_batch_total: {len(y)}, cur_batch_acc: {vqa_score.compute():.4f}') + + return float(vqa_score.compute()) \ No newline at end of file diff --git a/new_impl/mm/Vis_bert/QuestionAnswering/blip_fbs.py b/new_impl/mm/Vis_bert/QuestionAnswering/blip_fbs.py new file mode 100644 index 0000000000000000000000000000000000000000..ccd7a10382ff7282d0ccc65ce73aa3ee3ba114ef --- /dev/null +++ b/new_impl/mm/Vis_bert/QuestionAnswering/blip_fbs.py @@ -0,0 +1,185 @@ +import torch +from new_impl.cv.elasticdnn.api.model import ElasticDNN_OfflineVQAFMModel, ElasticDNN_OfflineVQAMDModel +from new_impl.cv.elasticdnn.api.algs.fm_lora import ElasticDNN_FMLoRAAlg +from new_impl.cv.elasticdnn.model.base import ElasticDNNUtil +from new_impl.cv.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from blip import FMLoRA_blip_Util +from new_impl.cv.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from blip import FM_to_MD_blip_Util +from utils.dl.common.model import LayerActivation, get_module, get_parameter, set_module +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +from utils.common.log import logger +import torch.nn.functional as F +import sys +from torch import nn +from new_impl.cv.elasticdnn.api.algs.md_pretraining_wo_fbs import ElasticDNN_MDPretrainingWoFBSAlg + + + +class ElasticDNN_blip_OfflineVQAFMModel(ElasticDNN_OfflineVQAFMModel): + def generate_md_by_reducing_width(self, reducing_width_ratio, samples: torch.Tensor): + return FM_to_MD_blip_Util().init_md_from_fm_by_reducing_width_with_perf_test(self.models_dict['main'], + reducing_width_ratio, samples) + # raise NotImplementedError + + def get_feature_hook(self) -> LayerActivation: + return LayerActivation(get_module(self.models_dict['main'], 'text_decoder.cls.predictions.decoder'), True, self.device) + + def get_elastic_dnn_util(self) -> ElasticDNNUtil: + raise NotImplementedError + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + self.to_train_mode() + # print(x['input_ids'].size(), x['pixel_values'].size(), ) + o = self.infer(x).logits + + # print(o.size(), y.size(), o, y) + + return F.binary_cross_entropy_with_logits(o, y) * y.shape[1] + + def get_lora_util(self) -> FMLoRA_Util: + return FMLoRA_blip_Util() + + def get_task_head_params(self): + head = get_module(self.models_dict['main'], 'text_decoder.cls.predictions.decoder') + params_name = {k for k, v in head.named_parameters()} + logger.info(f'task head params: {params_name}') + return list(head.parameters()) + + +class ElasticDNN_blip_OfflineVQAMDModel(ElasticDNN_OfflineVQAMDModel): + def get_feature_hook(self) -> LayerActivation: + return LayerActivation(get_module(self.models_dict['main'], 'text_decoder.cls.predictions.decoder'), True, self.device) + + # def forward_to_get_task_loss(self, x, y, *args, **kwargs): + # self.to_train_mode() + # # print(x['input_ids'].size(), x['pixel_values'].size(), ) + # o = self.infer(x) + + # # print(o.size(), y.size(), o, y) + + # return F.binary_cross_entropy_with_logits(o, y) * y.shape[1] + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + self.to_train_mode() + + o = self.models_dict['main'](**y) + return o.loss + + def get_distill_loss(self, student_output, teacher_output): + #print(student_output.shape, teacher_output.shape) + return F.mse_loss(student_output, teacher_output.detach()) + #return F.cross_entropy(student_output, teacher_output.detach()) + + def get_trained_params(self): + return self.models_dict['main'].parameters() + + def get_matched_param_of_fm(self, self_param_name, fm: nn.Module): # TODO: + if any([k in self_param_name for k in ['fbs', 'ab', 'embeddings']]): + return None + + p = get_parameter(self.models_dict['main'], self_param_name) + if p.dim() == 0: + return None + + elif p.dim() == 1 and 'LayerNorm' in self_param_name and 'weight' in self_param_name: + # if self_param_name.startswith('norm'): + # return None + return get_parameter(fm, self_param_name) + + # 1. xx.query.weight -> xx.query.fc.weight and xx.query.ab.0/1 + if ('query' in self_param_name or 'key' in self_param_name or \ + 'value' in self_param_name) and ('weight' in self_param_name): + ss = self_param_name.split('.') + # raise NotImplementedError() # TODO: + fm_qkv_name = '.'.join(ss[0: -1]) + '.fc' + fm_qkv = get_module(fm, fm_qkv_name) + + fm_abs_name = '.'.join(ss[0: -1]) + '.ab' + fm_abs = get_module(fm, fm_abs_name) + + return torch.cat([ + fm_qkv.weight.data, # task-agnositc params + fm_abs[1].weight @ fm_abs[0].weight + ], dim=0) + + elif ('query' in self_param_name or 'key' in self_param_name or \ + 'value' in self_param_name) and ('bias' in self_param_name): + ss = self_param_name.split('.') + + fm_qkv_name = '.'.join(ss[0: -1]) + '.fc.bias' + return get_parameter(fm, fm_qkv_name) + + elif 'intermediate.dense' in self_param_name: + fm_param_name = self_param_name.replace('.linear', '') + return get_parameter(fm, fm_param_name) + + else: + return get_parameter(fm, self_param_name) + + +if __name__ == '__main__': + from utils.dl.common.env import set_random_seed + set_random_seed(1) + + scenario = build_scenario( + source_datasets_name=['VQA_split1'], + target_datasets_order=['VQA_split1_c'] * 1, # TODO + da_mode='close_set', + data_dirs={ + 'VQA_split1': '/data/zql/datasets/vqav2', + 'VQA_split1_c': '/data/zql/datasets/vqav2' + }, + ) + + # 1. init model + fm_models_dict_path = 'new_impl/mm/Vis_bert/QuestionAnswering/results/blip_lora.py/20231018/999999-095006-/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/new_impl/mm/Vis_bert/QuestionAnswering/blip_lora.py/models/fm_best.pt' + fm_models = torch.load(fm_models_dict_path) + fm_models_dict_path = save_models_dict_for_init(fm_models, __file__, 'fm_blip_vqa_lora') + md_models_dict_path = save_models_dict_for_init({ + 'main': -1 + }, __file__, 'md_blip_none') + torch.cuda.set_device(1) + device = 'cuda' + + fm_model = ElasticDNN_blip_OfflineVQAFMModel('fm', fm_models_dict_path, device) + md_model = ElasticDNN_blip_OfflineVQAMDModel('md', md_models_dict_path, device) + + # 2. init alg + models = { + 'fm': fm_model, + 'md': md_model + } + import sys + + # from experiments.elasticdnn.clip.offline.fm_to_md.cls.md_pretraining_wo_fbs import ElasticDNN_MDPretrainingWoFBSAlg + # from methods.elasticdnn.api.algs.md_pretraining_wo_fbs_clip_debug import ElasticDNN_MDPretrainingWoFBSAlg + fm_to_md_alg = ElasticDNN_MDPretrainingWoFBSAlg(models, get_res_save_dir(__file__, sys.argv[0])) + + # sample_dataset = list(scenario.get_offline_datasets().values())[0]['train'] + + sample_dataset = list(scenario.get_offline_datasets().values())[0]['train'] + sample = sample_dataset[0][0] + + for k, v in sample.items(): + sample[k] = v.unsqueeze(0) + + from utils.dl.common.lr_scheduler import get_linear_schedule_with_warmup + fm_to_md_alg.run(scenario, hyps={ + 'launch_tbboard': False, + + 'samples_size': sample, + 'generate_md_width_ratio': 4, + + 'train_batch_size':32, + 'val_batch_size': 512, + 'num_workers': 16, + 'optimizer': 'AdamW', + 'optimizer_args': {'lr': 1e-5, 'betas': [0.9, 0.999], 'weight_decay': 0.01}, + 'scheduler': 'LambdaLR', + 'scheduler_args': {'lr_lambda': get_linear_schedule_with_warmup(10000, 70000)}, + 'num_iters': 80000, + 'val_freq': 1000, + 'distill_loss_weight': 1. + }) + \ No newline at end of file diff --git a/new_impl/mm/Vis_bert/QuestionAnswering/blip_index.py b/new_impl/mm/Vis_bert/QuestionAnswering/blip_index.py new file mode 100644 index 0000000000000000000000000000000000000000..23ac3b6db8ab3c6b5c14aee0c8669484c3dc5fc5 --- /dev/null +++ b/new_impl/mm/Vis_bert/QuestionAnswering/blip_index.py @@ -0,0 +1,225 @@ +import torch +import sys +from torch import nn +from new_impl.cv.elasticdnn.api.model import ElasticDNN_OfflineVQAFMModel, ElasticDNN_OfflineVQAMDModel +from new_impl.cv.elasticdnn.api.algs.md_pretraining_wo_fbs import ElasticDNN_MDPretrainingWoFBSAlg +from new_impl.cv.elasticdnn.model.base import ElasticDNNUtil +from new_impl.cv.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from blip import FM_to_MD_blip_Util +from new_impl.cv.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from blip import FMLoRA_blip_Util +from blip import ElasticblipUtil +from utils.dl.common.model import LayerActivation2, get_module, get_parameter +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +from utils.dl.common.loss import CrossEntropyLossSoft +import torch.nn.functional as F +from utils.common.log import logger + + +class ElasticDNN_blip_OfflineVQAFMModel(ElasticDNN_OfflineVQAFMModel): + def generate_md_by_reducing_width(self, reducing_width_ratio, samples: torch.Tensor): # TODO: + return FM_to_MD_blip_Util().init_md_from_fm_by_reducing_width_with_perf_test(self.models_dict['main'], + reducing_width_ratio, samples) + # raise NotImplementedError + + def get_feature_hook(self) -> LayerActivation2: + return LayerActivation2(get_module(self.models_dict['main'], 'text_decoder.cls.predictions.decoder')) + + def get_elastic_dnn_util(self) -> ElasticDNNUtil: # TODO: + return ElasticblipUtil() + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + self.to_train_mode() + # print(x['input_ids'].size(), x['pixel_values'].size(), ) + #o = self.infer(x) + o = self.models_dict['main'](**y) + # print(o.size(), y.size(), o, y) + + #return F.binary_cross_entropy_with_logits(o, y) * y.shape[1] + return o.loss + def get_lora_util(self) -> FMLoRA_Util: + return FMLoRA_blip_Util() + + def get_task_head_params(self): + head = get_module(self.models_dict['main'], 'text_decoder.cls.predictions.decoder') + params_name = {k for k, v in head.named_parameters()} + logger.info(f'task head params: {params_name}') + return list(head.parameters()) + + +class ElasticDNN_blip_OfflineVQAMDModel(ElasticDNN_OfflineVQAMDModel): + # def __init__(self, name: str, models_dict_path: str, device: str): + # super().__init__(name, models_dict_path, device) + + # self.distill_criterion = CrossEntropyLossSoft() + + def get_feature_hook(self) -> LayerActivation2: + return LayerActivation2(get_module(self.models_dict['main'], 'text_decoder.cls.predictions.decoder')) + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + self.to_train_mode() + # print(x['input_ids'].size(), x['pixel_values'].size(), ) + #o = self.infer(x) + o = self.models_dict['main'](**y) + # print(o.size(), y.size(), o, y) + + #return F.binary_cross_entropy_with_logits(o, y) * y.shape[1] + return o.loss + + def get_distill_loss(self, student_output, teacher_output): + # print(student_output, teacher_output) + return F.mse_loss(student_output, teacher_output.detach()) + + def get_matched_param_of_fm(self, self_param_name, fm: nn.Module): # TODO: + if any([k in self_param_name for k in ['fbs', 'ab', 'embeddings']]): + return None + + p = get_parameter(self.models_dict['main'], self_param_name) + if p.dim() == 0: + return None + + elif p.dim() == 1 and 'LayerNorm' in self_param_name.lower() and 'weight' in self_param_name: + # if self_param_name.startswith('norm'): + # return None + return get_parameter(fm, self_param_name) + elif p.dim() == 1 and 'norm' in self_param_name and 'weight' in self_param_name: + return get_parameter(fm, self_param_name) + + # 1. xx.query.weight -> xx.query.fc.weight and xx.query.ab.0/1 + if ('query' in self_param_name or 'key' in self_param_name or \ + 'value' in self_param_name) and ('weight' in self_param_name): + ss = self_param_name.split('.') + # raise NotImplementedError() # TODO: + fm_qkv_name = '.'.join(ss[0: -1]) + '.fc' + fm_qkv = get_module(fm, fm_qkv_name) + + fm_abs_name = '.'.join(ss[0: -1]) + '.ab' + fm_abs = get_module(fm, fm_abs_name) + + return torch.cat([ + fm_qkv.weight.data, # task-agnositc params + fm_abs[1].weight @ fm_abs[0].weight + ], dim=0) + + elif ('query' in self_param_name or 'key' in self_param_name or \ + 'value' in self_param_name) and ('bias' in self_param_name): + ss = self_param_name.split('.') + + fm_qkv_name = '.'.join(ss[0: -1]) + '.fc.bias' + return get_parameter(fm, fm_qkv_name) + + elif 'intermediate.dense' in self_param_name: + fm_param_name = self_param_name.replace('.linear', '') + return get_parameter(fm, fm_param_name) + + elif 'qkv.weight' in self_param_name: + ss = self_param_name.split('.') + + fm_qkv_name = '.'.join(ss[0: -1]) + '.fc' + fm_qkv = get_module(fm, fm_qkv_name) + + fm_abs_name = '.'.join(ss[0: -1]) + '.ab' + fm_abs = get_module(fm, fm_abs_name) + + return torch.cat([ + fm_qkv.weight.data, # task-agnositc params + fm_abs[1].weight @ fm_abs[0].weight # task-specific params (LoRA) + ], dim=0) + + # elif 'to_qkv.bias' in self_param_name: + # ss = self_param_name.split('.') + + # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv.bias' + # return get_parameter(fm, fm_qkv_name) + + elif 'mlp.fc1' in self_param_name and 'weight' in self_param_name: + fm_param_name = self_param_name.replace('.linear', '') + return get_parameter(fm, fm_param_name) + + elif 'mlp.fc2' in self_param_name and 'weight' in self_param_name: + fm_param_name = self_param_name + res = get_parameter(fm, fm_param_name) + # print('mlp fc2 debug', fm_param_name, res is None) + return res + else: + #return get_parameter(fm, self_param_name) + return None + + + +if __name__ == '__main__': + from utils.dl.common.env import set_random_seed + set_random_seed(1) + + # 3. init scenario + scenario = build_scenario( + source_datasets_name=['VQA_split1'], + target_datasets_order=['VQA_split1_c'] * 1, # TODO + da_mode='close_set', + data_dirs={ + 'VQA_split1': '/data/zql/datasets/vqav2', + 'VQA_split1_c': '/data/zql/datasets/vqav2' + }, + ) + + # 1. init model + # from dnns.deeplabv3.head import modify_forward_head + # modify_forward_head() # TODO: bring a bug + # from dnns.vit import vit_b_16 + fm_models_dict_path = 'new_impl/mm/Vis_bert/QuestionAnswering/results/blip_fbs.py/20231020/999999-162038-/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/new_impl/mm/Vis_bert/QuestionAnswering/blip_fbs.py/models/fm_best.pt' + fm_models = torch.load(fm_models_dict_path) + fm_models_dict_path = save_models_dict_for_init(fm_models, __file__, 'fm_blip_vqa_lora') + md_models_dict_path = save_models_dict_for_init( + torch.load('new_impl/mm/Vis_bert/QuestionAnswering/results/blip_fbs.py/20231020/999999-162038-/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/new_impl/mm/Vis_bert/QuestionAnswering/blip_fbs.py/models/md_best.pt'), + __file__, 'md_blip_vqa_raw_pretrained') + device = 'cuda' + + fm_model = ElasticDNN_blip_OfflineVQAFMModel('fm', fm_models_dict_path, device) + md_model = ElasticDNN_blip_OfflineVQAMDModel('md', md_models_dict_path, device) + + # 2. init alg + models = { + 'fm': fm_model, + 'md': md_model + } + from new_impl.cv.elasticdnn.api.algs.md_pretraining_index_v2_train_index_and_md import ElasticDNN_MDPretrainingIndexAlg + fm_to_md_alg = ElasticDNN_MDPretrainingIndexAlg(models, get_res_save_dir(__file__, sys.argv[0])) + + sample_dataset = list(scenario.get_offline_datasets().values())[0]['train'] + sample = sample_dataset[0][0] + + from utils.dl.common.lr_scheduler import get_linear_schedule_with_warmup + fm_to_md_alg.run(scenario, hyps={ + 'launch_tbboard': False, + + 'samples_size': sample, + + 'FBS_r': 8, + 'FBS_ignore_layers': [], + + 'train_batch_size': 16, + 'val_batch_size': 256, + 'num_workers': 16, + 'optimizer': 'AdamW', + 'optimizer_args': {'lr': 1e-4, 'betas': [0.9, 0.999], 'weight_decay': 0.01}, + 'scheduler': 'LambdaLR', + 'scheduler_args': {'lr_lambda': get_linear_schedule_with_warmup(10000, 70000)}, + + 'indexes_optimizer_args': {'lr': 3e-3, 'momentum': 0.9, 'weight_decay': 5e-4}, + + 'num_iters': 80000, + 'val_freq': 20, + + 'max_sparsity': 0.9, + 'min_sparsity': 0.0, + 'l1_reg_loss_weight': 1e-9, + 'index_loss_weight': 1e-4, + 'val_num_sparsities': 4, + + 'bn_cal_num_iters': 0, + + 'index_init': 'zero', + 'index_guided_linear_comb_split_size': 512 + }) + \ No newline at end of file diff --git a/new_impl/mm/Vis_bert/QuestionAnswering/blip_lora.py b/new_impl/mm/Vis_bert/QuestionAnswering/blip_lora.py new file mode 100644 index 0000000000000000000000000000000000000000..c5d364389d3ca55ceed70395cea2303b0ba3a82e --- /dev/null +++ b/new_impl/mm/Vis_bert/QuestionAnswering/blip_lora.py @@ -0,0 +1,118 @@ +import torch +from new_impl.cv.elasticdnn.api.model import ElasticDNN_OfflineVQAFMModel, ElasticDNN_OfflineVQAMDModel +from new_impl.cv.elasticdnn.api.algs.fm_lora import ElasticDNN_FMLoRAAlg +from new_impl.cv.elasticdnn.model.base import ElasticDNNUtil +from new_impl.cv.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from blip import FMLoRA_blip_Util +from new_impl.cv.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from utils.dl.common.model import LayerActivation, get_module, get_parameter, set_module +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +from utils.common.log import logger +import torch.nn.functional as F +import sys + + +class ElasticDNN_blip_OfflineVQAFMModel(ElasticDNN_OfflineVQAFMModel): + def generate_md_by_reducing_width(self, reducing_width_ratio, samples: torch.Tensor): + # return FM_to_MD_ViT_Util().init_md_from_fm_by_reducing_width_with_perf_test(self.models_dict['main'], + # reducing_width_ratio, samples) + raise NotImplementedError + + def get_feature_hook(self) -> LayerActivation: + return LayerActivation(get_module(self.models_dict['main'], 'text_decoder.cls.predictions.decoder'), True, self.device) + + def get_elastic_dnn_util(self) -> ElasticDNNUtil: + raise NotImplementedError + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + self.to_train_mode() + # print(x['input_ids'].size(), x['pixel_values'].size(), ) + #o = self.infer(x) + o = self.models_dict['main'](**y) + # print(o.size(), y.size(), o, y) + #return F.cross_entropy(o,y) + #return F.binary_cross_entropy_with_logits(o, y) * y.shape[1] + return o.loss + + def get_lora_util(self) -> FMLoRA_Util: + return FMLoRA_blip_Util() + + def get_task_head_params(self): + head = get_module(self.models_dict['main'], 'text_decoder.cls.predictions.decoder') + params_name = {k for k, v in head.named_parameters()} + logger.info(f'task head params: {params_name}') + return list(head.parameters()) + + +class ElasticDNN_blip_OfflineVQAMDModel(ElasticDNN_OfflineVQAMDModel): + def get_feature_hook(self) -> LayerActivation: + return LayerActivation(get_module(self.models_dict['main'], 'text_decoder.cls.predictions.decoder'), True, self.device) + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + self.to_train_mode() + o = self.infer(x) + return F.binary_cross_entropy_with_logits(o, y) * y.shape[1] + + +if __name__ == '__main__': + from utils.dl.common.env import set_random_seed + set_random_seed(1) + + scenario = build_scenario( + source_datasets_name=['VQA_split1'], + target_datasets_order=['VQA_split1_c'] * 1, # TODO + da_mode='close_set', + data_dirs={ + 'VQA_split1': '/data/zql/datasets/vqav2', + 'VQA_split1_c': '/data/zql/datasets/vqav2' + }, + ) + + # 2. init model + torch.cuda.set_device(1) + device = 'cuda' + from transformers import BlipForQuestionAnswering + from blip import blip + model = blip(scenario.num_classes) + + fm_models_dict_path = save_models_dict_for_init({ + 'main': model + }, __file__, 'fm_blip') + + fm_model = ElasticDNN_blip_OfflineVQAFMModel('fm', fm_models_dict_path, device) + + # 3. init alg + models = { + 'fm': fm_model + } + fm_lora_alg = ElasticDNN_FMLoRAAlg(models, get_res_save_dir(__file__, sys.argv[0])) + + + sample_dataset = list(scenario.get_offline_datasets().values())[0]['train'] + sample = sample_dataset[0][0] + + for k, v in sample.items(): + sample[k] = v.unsqueeze(0) + + + # 4. run alg + from utils.dl.common.lr_scheduler import get_linear_schedule_with_warmup + fm_lora_alg.run(scenario, hyps={ + 'launch_tbboard': False, + + 'samples_size': sample, + + 'ab_r':8 , + 'train_batch_size': 64, + 'val_batch_size': 512, + 'num_workers': 16, + 'optimizer': 'AdamW', + 'optimizer_args': {'lr': 1e-4, 'betas': [0.9, 0.999]}, + 'scheduler': 'LambdaLR', + 'scheduler_args': {'lr_lambda': get_linear_schedule_with_warmup(10000, 310000)}, + 'num_iters': 320000, + 'val_freq': 400, + + # 'fm_lora_ckpt_path': 'experiments/elasticdnn/vit_b_16/offline/fm_lora/cls/results/cls.py/20230607/999995-234355-TokenClsial/models/fm_best.pt' + }) \ No newline at end of file diff --git a/new_impl/mm/Vis_bert/QuestionAnswering/blip_online.py b/new_impl/mm/Vis_bert/QuestionAnswering/blip_online.py new file mode 100644 index 0000000000000000000000000000000000000000..a59fb25f35d39e944475a136cd1c1ac37405d7f6 --- /dev/null +++ b/new_impl/mm/Vis_bert/QuestionAnswering/blip_online.py @@ -0,0 +1,96 @@ +from typing import List +from data.dataloader import build_dataloader +# from methods.elasticdnn.api.online_model import ElasticDNN_OnlineModel +from new_impl.cv.elasticdnn.api.online_model_v2 import ElasticDNN_OnlineModel + +import torch +import sys +from torch import nn +from new_impl.cv.elasticdnn.api.model import ElasticDNN_OfflineSegFMModel, ElasticDNN_OfflineSegMDModel +from new_impl.cv.elasticdnn.api.algs.md_pretraining_wo_fbs import ElasticDNN_MDPretrainingWoFBSAlg +from new_impl.cv.elasticdnn.model.base import ElasticDNNUtil +from new_impl.cv.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from new_impl.cv.elasticdnn.pipeline.offline.fm_to_md.vit import FM_to_MD_ViT_Util +from new_impl.cv.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from new_impl.cv.elasticdnn.pipeline.offline.fm_lora.vit import FMLoRA_ViT_Util +from new_impl.cv.elasticdnn.model.vit import ElasticViTUtil +from utils.common.file import ensure_dir +from utils.dl.common.model import LayerActivation, get_module, get_parameter +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +from utils.dl.common.loss import CrossEntropyLossSoft +import torch.nn.functional as F +from utils.dl.common.env import create_tbwriter +import os +from utils.common.log import logger +from utils.common.data_record import write_json +# from methods.shot.shot import OnlineShotModel +from new_impl.cv.feat_align.main import FeatAlignAlg +import tqdm +from new_impl.cv.feat_align.mmd import mmd_rbf +from new_impl.cv.utils.elasticfm_da import init_online_model, elasticfm_da + + +os.environ['TOKENIZERS_PARALLELISM'] = 'false' +torch.cuda.set_device(1) +device = 'cuda' +app_name = 'vqa' +sd_sparsity = 0.8 + +settings = { + 'involve_fm': True +} + +target_datasets = ['VQAv2_split1_c_gaussian_noise', 'VQAv2_split1_c_shot_noise', 'VQAv2_split1_c_impulse_noise', 'VQAv2_split1_c_defocus_blur', 'VQAv2_split1_c_glass_blur', 'VQAv2_split1_c_motion_blur', 'VQAv2_split1_c_zoom_blur', 'VQAv2_split1_c_snow', 'VQAv2_split1_c_frost', 'VQAv2_split1_c_fog', 'VQAv2_split1_c_brightness', 'VQAv2_split1_c_contrast', 'VQAv2_split1_c_elastic_transform', 'VQAv2_split1_c_pixelate', 'VQAv2_split1_c_jpeg_compression', 'VQAv2_split1_c_speckle_noise', 'VQAv2_split1_c_gaussian_blur', 'VQAv2_split1_c_spatter', 'VQAv2_split1_c_saturate'] * 2 +target_datasets = target_datasets[0: 30] +assert len(target_datasets) == 30 + +scenario = build_scenario( + source_datasets_name=['VQA_split1'], + target_datasets_order=['VQA_split1_c'], + da_mode='close_set', + data_dirs={ + k: '/data/zql/datasets/vqav2' for k in ['VQA_split1', 'VQA_split1_c'] + }, +) + +from blip import ElasticDNN_VQAOnlineModel +elasticfm_model = ElasticDNN_VQAOnlineModel('vqa', init_online_model( + '', + '', + 'vqa', __file__ +), device, { + 'md_to_fm_alpha': 0.2, + 'fm_to_md_alpha': 0.2 +}) + +da_alg = FeatAlignAlg +from blip import VQAOnlineFeatAlignModel +da_model = VQAOnlineFeatAlignModel +da_alg_hyp = { + 'train_batch_size': 64, + 'val_batch_size': 256, + 'num_workers': 0, + 'optimizer': 'AdamW', + 'optimizer_args': {'lr': 1e-6, 'betas': [0.9, 0.999], 'weight_decay': 0.01}, + 'scheduler': '', + 'scheduler_args': {}, + 'num_iters': 100, + 'val_freq': 20, + 'feat_align_loss_weight': 1.0, + 'sd_sparsity': 0.7 +} + + +elasticfm_da( + [app_name], + [scenario], + [elasticfm_model], + [da_alg], + [da_alg_hyp], + [da_model], + device, + settings, + __file__, + sys.argv[0] +) diff --git a/new_impl/mm/git/git.py b/new_impl/mm/git/git.py new file mode 100644 index 0000000000000000000000000000000000000000..44040f6314ef2525d2132e5aa2e44f785a4f68f8 --- /dev/null +++ b/new_impl/mm/git/git.py @@ -0,0 +1,115 @@ +from transformers import BlipForQuestionAnswering, BlipConfig,BlipModel, GitModel +import torch +from torch import nn +from abc import ABC, abstractmethod +from copy import deepcopy +from typing import Optional, Union +from einops import rearrange, repeat +from einops.layers.torch import Rearrange +import tqdm + +from utils.dl.common.model import get_model_device, get_model_latency, get_model_size, set_module +from utils.dl.common.model import set_module, get_module, get_super_module +from utils.common.log import logger +from new_impl.cv.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util, LoRA +from transformers.models.blip.modeling_blip import BlipAttention +from transformers.models.blip.modeling_blip_text import BlipTextSelfAttention,BlipTextAttention,BlipTextSelfOutput +from new_impl.cv.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from new_impl.cv.elasticdnn.model.base import Abs, KTakesAll, ElasticDNNUtil, Layer_WrappedWithFBS + +from typing import Optional, Tuple +import math + +class git(nn.Module): + def __init__(self,num_classes): + super(git,self).__init__() + self.git =GitModel.from_pretrained('') + self.cls = nn.Linear(768,num_classes) + + def forward(self,**sample): + output = self.blip(**sample)[-1]#output the last hidden + output = self.cls(output[1]) + return output + +class ToQKV_WrappedWithLoRA(nn.Module): + def __init__(self, fc: nn.Linear, ab_r: int): + super(ToQKV_WrappedWithLoRA, self).__init__() + + self.fc = fc + self.ab = self.create_ab_as_linear(fc.weight.data, ab_r) + + def create_ab_as_linear(self, fc_weight: torch.Tensor, ab_r: int): + res = nn.Sequential( + LoRA(fc_weight.size(1), fc_weight.size(0) // ab_r, bias=False), + LoRA(fc_weight.size(0) // ab_r, fc_weight.size(0), bias=False) + ).to(fc_weight.device) + nn.init.kaiming_uniform_(res[0].weight, a=5 ** 0.5) + nn.init.zeros_(res[1].weight) + return res + + def forward(self, x): + x1 = self.fc(x) + x2 = self.ab(x) + return x1 + x2 + +class FMLoRA_git_Util(FMLoRA_Util): + + @torch.no_grad() + def add_lora_ab_to_fm(self, fm: nn.Module, ab_r: int, samples: dict): + fm.eval() + + # print(samples) + for k, v in samples.items(): + if isinstance(v, torch.Tensor): + samples[k] = v.to(get_model_device(fm)) + + o1 = fm(**samples) + #o1 = fm(**samples) + for name, module in fm.named_modules(): + if name.endswith(('query', 'key', 'value')): + set_module(fm, name, ToQKV_WrappedWithLoRA(module, ab_r)) + elif name.endswith('.qkv'): + set_module(fm, name, ToQKV_WrappedWithLoRA(module, ab_r)) + + + o2 = fm(**samples) + #o2 = fm(**samples) + if isinstance(o1, tuple): + o1 = o1[-1] + o2 = o2[-1] + output_diff = ((o1 - o2) ** 2).sum() + assert output_diff < 1e-5 + return fm + + @torch.no_grad() + def absorb_lora_and_recover_net_structure(self, fm: nn.Module, samples: dict): + fm.eval() + # print('absorb lora before') + + for k, v in samples.items(): + if isinstance(v, torch.Tensor): + samples[k] = v.to(get_model_device(fm)) + + o1 = fm(**samples) + + for name, module in fm.named_modules(): + if not isinstance(module, ToQKV_WrappedWithLoRA): + continue + + fc = module.fc + ab = module.ab + + fc.weight.add_(ab[1].weight @ ab[0].weight) + + set_module(fm, name, fc) + + # print('absorb lora after') + o2 = fm(**samples) + + if isinstance(o1, tuple): + o1 = o1[-1] + o2 = o2[-1] + output_diff = ((o1 - o2) ** 2).sum() + assert output_diff < 1e-6, output_diff + + return fm diff --git a/new_impl/mm/git/git_lora.py b/new_impl/mm/git/git_lora.py new file mode 100644 index 0000000000000000000000000000000000000000..5ee78bab939319f6cb1cb062f93e9417f6db839d --- /dev/null +++ b/new_impl/mm/git/git_lora.py @@ -0,0 +1,118 @@ +import torch +from new_impl.cv.elasticdnn.api.model import ElasticDNN_OfflineVQAFMModel, ElasticDNN_OfflineVQAMDModel +from new_impl.cv.elasticdnn.api.algs.fm_lora import ElasticDNN_FMLoRAAlg +from new_impl.cv.elasticdnn.model.base import ElasticDNNUtil +from new_impl.cv.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from blip import FMLoRA_blip_Util +from new_impl.cv.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from utils.dl.common.model import LayerActivation, get_module, get_parameter, set_module +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +from utils.common.log import logger +import torch.nn.functional as F +import sys + + +class ElasticDNN_blip_OfflineVQAFMModel(ElasticDNN_OfflineVQAFMModel): + def generate_md_by_reducing_width(self, reducing_width_ratio, samples: torch.Tensor): + # return FM_to_MD_ViT_Util().init_md_from_fm_by_reducing_width_with_perf_test(self.models_dict['main'], + # reducing_width_ratio, samples) + raise NotImplementedError + + def get_feature_hook(self) -> LayerActivation: + return LayerActivation(get_module(self.models_dict['main'], 'cls'), True, self.device) + + def get_elastic_dnn_util(self) -> ElasticDNNUtil + raise NotImplementedError + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + self.to_train_mode() + # print(x['input_ids'].size(), x['pixel_values'].size(), ) + o = self.infer(x) + #o = self.models_dict['main'](**y) + # print(o.size(), y.size(), o, y) + #return F.cross_entropy(o,y) + #return F.binary_cross_entropy_with_logits(o, y) * y.shape[1] + return F.binary_cross_entropy_with_logits(o, y) * y.shape[1] + + def get_lora_util(self) -> FMLoRA_Util: + return FMLoRA_git_Util() + + def get_task_head_params(self): + head = get_module(self.models_dict['main'], 'cls') + params_name = {k for k, v in head.named_parameters()} + logger.info(f'task head params: {params_name}') + return list(head.parameters()) + + +class ElasticDNN_blip_OfflineVQAMDModel(ElasticDNN_OfflineVQAMDModel): + def get_feature_hook(self) -> LayerActivation: + return LayerActivation(get_module(self.models_dict['main'], 'cls'), True, self.device) + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + self.to_train_mode() + o = self.infer(x) + return nn.functional.binary_cross_entropy_with_logits(o, y) * y.shape[1] + + +if __name__ == '__main__': + from utils.dl.common.env import set_random_seed + set_random_seed(1) + + scenario = build_scenario( + source_datasets_name=['VQA_split1'], + target_datasets_order=['VQA_split1_c'] * 1, # TODO + da_mode='close_set', + data_dirs={ + 'VQA_split1': '/data/zql/datasets/vqav2', + 'VQA_split1_c': '/data/zql/datasets/vqav2' + }, + ) + + # 2. init model + torch.cuda.set_device(1) + device = 'cuda' + from transformers import BlipForQuestionAnswering + from git import git + model = git(scenario.num_classes) + + fm_models_dict_path = save_models_dict_for_init({ + 'main': model + }, __file__, 'fm_git') + + fm_model = ElasticDNN_blip_OfflineVQAFMModel('fm', fm_models_dict_path, device) + + # 3. init alg + models = { + 'fm': fm_model + } + fm_lora_alg = ElasticDNN_FMLoRAAlg(models, get_res_save_dir(__file__, sys.argv[0])) + + + sample_dataset = list(scenario.get_offline_datasets().values())[0]['train'] + sample = sample_dataset[0][0] + + for k, v in sample.items(): + sample[k] = v.unsqueeze(0) + + + # 4. run alg + from utils.dl.common.lr_scheduler import get_linear_schedule_with_warmup + fm_lora_alg.run(scenario, hyps={ + 'launch_tbboard': False, + + 'samples_size': sample, + + 'ab_r':8 , + 'train_batch_size': 64, + 'val_batch_size': 512, + 'num_workers': 16, + 'optimizer': 'AdamW', + 'optimizer_args': {'lr': 1e-4, 'betas': [0.9, 0.999]}, + 'scheduler': 'LambdaLR', + 'scheduler_args': {'lr_lambda': get_linear_schedule_with_warmup(10000, 310000)}, + 'num_iters': 320000, + 'val_freq': 400, + + # 'fm_lora_ckpt_path': 'experiments/elasticdnn/vit_b_16/offline/fm_lora/cls/results/cls.py/20230607/999995-234355-TokenClsial/models/fm_best.pt' + }) \ No newline at end of file diff --git a/new_impl/nlp/gpt-neo/text_generation/__pycache__/gpt_neo.cpython-38.pyc b/new_impl/nlp/gpt-neo/text_generation/__pycache__/gpt_neo.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ca6a83a14a522c9b45238f986e40f41c1620a07c Binary files /dev/null and b/new_impl/nlp/gpt-neo/text_generation/__pycache__/gpt_neo.cpython-38.pyc differ diff --git a/new_impl/nlp/gpt-neo/text_generation/gen_baseline.py b/new_impl/nlp/gpt-neo/text_generation/gen_baseline.py new file mode 100644 index 0000000000000000000000000000000000000000..39e48cf08c8e8d3096b271dad47cb87966b6a993 --- /dev/null +++ b/new_impl/nlp/gpt-neo/text_generation/gen_baseline.py @@ -0,0 +1,171 @@ +import os +gpt_neo_series_id = '1.3B_ckpt' +os.environ['gpt_neo_series_id'] = gpt_neo_series_id +import torch +import torch.nn as nn +from methods.elasticdnn.api.algs.fm_lora import ElasticDNN_FMLoRAAlg +from methods.elasticdnn.api.algs.md_pretraining_wo_fbs import ElasticDNN_MDPretrainingWoFBSAlg +from methods.elasticdnn.model.base import ElasticDNNUtil +from methods.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from gpt_neo import getTokenizer, ElasticGPTUtil, FMLoRA_GPT_Util, ElasticDNN_OfflineTextGenFMModel, ElasticDNN_OfflineTextGenMDModel, FM_to_MD_GPT_Util, collate_fn +from methods.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from methods.elasticdnn.pipeline.offline.fm_to_md.vit import FM_to_MD_ViT_Util +from methods.elasticdnn.model.vit import ElasticViTUtil +from methods.elasticdnn.api.algs.md_pretraining_index_v2_train_index_and_md import ElasticDNN_MDPretrainingIndexAlg +from utils.dl.common.model import LayerActivation2, get_module, get_parameter +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_gen_scenario +import torch.nn.functional as F +import os +from utils.dl.common.loss import CrossEntropyLossSoft +from new_impl.cv.feat_align.main_gpt_neo import OnlineFeatAlignModel, FeatAlignAlg +import tqdm +from new_impl.cv.feat_align.mmd import mmd_rbf +from new_impl.cv.utils.baseline_da import baseline_da +from new_impl.cv.elasticdnn.api.online_model_v2 import ElasticDNN_OnlineModel +from utils.common.log import logger +import nltk +from nltk.translate.bleu_score import sentence_bleu, corpus_bleu +from nltk.translate.bleu_score import SmoothingFunction +import json + + +os.environ['TOKENIZERS_PARALLELISM'] = 'true' +os.environ['CUDA_LAUNCH_BLOCKING'] = '1' +torch.cuda.set_device(1) +device = 'cuda' +app_name = 'cls' + +scenario = build_gen_scenario( + source_datasets_name=['No_robots'], + target_datasets_order=['Medicine_task', 'Law_task'] * 10, + da_mode='close_set', + data_dirs={ + 'No_robots': '/data/zql/datasets/no_robots', + 'Law_task': '/data/zql/datasets/law_task', + 'Medicine_task': '/data/zql/datasets/medicine_task', + }, + ) + +class TxtgenOnlineFeatAlignModel(OnlineFeatAlignModel): + def get_trained_params(self): # TODO: elastic fm only train a part of params + #qkv_and_norm_params = [p for n, p in self.models_dict['main'].named_parameters() if 'attention.attention.projection_query' in n or 'attention.attention.projection_key' in n or 'attention.attention.projection_value' in n or 'intermediate.dense' in n or 'output.dense' in n] + qkv_and_norm_params = [p for n, p in self.models_dict['main'].named_parameters()] + return qkv_and_norm_params + + def get_feature_hook(self) -> LayerActivation2: + return LayerActivation2(get_module(self.models_dict['main'], 'model.lm_head')) + + def forward_to_get_task_loss(self, x, y): + losses = self.infer(x) + # print(losses) + + return losses + + def get_mmd_loss(self, f1, f2): + common_shape = min(f1.shape[0], f2.shape[0]) + f1 = f1.view(f1.shape[0], -1) + f2 = f2.view(f2.shape[0], -1) + f1 = f1[:common_shape,:] + f2 = f2[:common_shape,:] + return mmd_rbf(f1, f2) + + def infer(self, x, *args, **kwargs): + return self.models_dict['main'](**x) + + def get_accuracy(self, test_loader, *args, **kwargs): + acc = 0 + sample_num = 0 + tokenizer = getTokenizer() + self.to_eval_mode() + pred_txt = [] + true_txt = [] + res = [] + with torch.no_grad(): + pbar = tqdm.tqdm(enumerate(test_loader), total=len(test_loader), dynamic_ncols=True, leave=False) + for batch_index, (x, _) in pbar: + if len(x) == 0: + continue + # if batch_index > 10: + # break + for k, v in x.items(): + if isinstance(v, torch.Tensor): + x[k] = v.to(self.device) + # input_ids = [] + inputlen = x['len'] + y = x['labels'] + x['labels'] = None + outputs = self.models_dict['main'].generate(x, pad_id=tokenizer.pad_token_id) + + for i, op in enumerate(outputs): + op = op.tolist() + op = list(filter(lambda x: x != tokenizer.pad_token_id, op)) + txt = tokenizer.decode(op) + txt = txt.replace(tokenizer.pad_token, "") + res.append(txt) + txt = txt[inputlen[i]:] + pred_txt.append(nltk.word_tokenize(txt)) + for tp in y: + true_txt.append(nltk.word_tokenize(tokenizer.decode(tp).replace(tokenizer.pad_token, ''))) + # pred = F.softmax(output, dim=1).argmax(dim=1) + # correct = torch.eq(pred, y).sum().item() + # acc += correct + sample_num += len(y) + + # pbar.set_description(f'cur_batch_total: {len(y)}, cur_batch_correct: {correct}, ' + # f'cur_batch_acc: {(correct / len(y)):.4f}') + json.dump(res, open("./gpt_generation.json", "w")) + smooth = SmoothingFunction() + score = 0. + for pred, true in zip(pred_txt, true_txt): + score += sentence_bleu([true], pred, weights=(0.25, 0.25, 0.25, 0.25), smoothing_function=smooth.method1) + score /= sample_num + return score + +da_alg = FeatAlignAlg +from utils.dl.common.lr_scheduler import get_linear_schedule_with_warmup +#from new_impl.cv.model import ClsOnlineFeatAlignModel +da_model = TxtgenOnlineFeatAlignModel( + app_name, + 'new_impl/nlp/gpt-neo/text_generation/results/gen_md_wo_fbs.py/20240113/999999-172009/models/md_best.pt', + device +) +da_alg_hyp = { + 'Medicine_task': { + 'train_batch_size': 2, + 'val_batch_size': 1, + 'num_workers': 2, + 'optimizer': 'AdamW', + 'optimizer_args': {'lr': 5e-6, 'betas': [0.9, 0.999], 'weight_decay': 0.01}, + 'scheduler': '', + 'scheduler_args': {}, + 'num_iters': 1000, + 'val_freq': 200, + 'feat_align_loss_weight': 1.0, + }, + 'Law_task': { + 'train_batch_size': 2, + 'val_batch_size': 1, + 'num_workers': 2, + 'optimizer': 'AdamW', + 'optimizer_args': {'lr': 5e-6, 'betas': [0.9, 0.999], 'weight_decay': 0.01}, + 'scheduler': '', + 'scheduler_args': {}, + 'num_iters': 1000, + 'val_freq': 200, + 'feat_align_loss_weight': 1.0, + }, +} + + +baseline_da( + app_name, + scenario, + da_alg, + da_alg_hyp, + da_model, + device, + __file__, + "results", + collate_fn=collate_fn +) \ No newline at end of file diff --git a/new_impl/nlp/gpt-neo/text_generation/gen_lora.py b/new_impl/nlp/gpt-neo/text_generation/gen_lora.py new file mode 100644 index 0000000000000000000000000000000000000000..6ca0a9a48543355526c13849c44b23dc7badcab1 --- /dev/null +++ b/new_impl/nlp/gpt-neo/text_generation/gen_lora.py @@ -0,0 +1,123 @@ +import os +gpt_neo_series_id = '1.3B_ckpt' +os.environ['gpt_neo_series_id'] = gpt_neo_series_id +os.environ['CUDA_LAUNCH_BLOCKING'] = '1' +# os.environ['CUDA_VISIBLE_DEVICES'] = '0, 1' + +import torch +from methods.elasticdnn.api.model import ElasticDNN_OfflineSenClsFMModel, ElasticDNN_OfflineSenClsMDModel +from methods.elasticdnn.api.algs.fm_lora import ElasticDNN_FMLoRAAlg +from methods.elasticdnn.model.base import ElasticDNNUtil +from methods.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from gpt_neo import FMLoRA_GPT_Util, ElasticDNN_OfflineTextGenFMModel, collate_fn +# from methods.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +# from methods.elasticdnn.pipeline.offline.fm_to_md.vit import FM_to_MD_ViT_Util +# from methods.elasticdnn.model.vit import ElasticViTUtil +from utils.dl.common.model import LayerActivation, get_module, get_parameter, set_module +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_gen_scenario +from utils.common.log import logger +import torch.nn.functional as F +import sys + + +class ElasticDNN_GPT_OfflineTextGenFMModel(ElasticDNN_OfflineTextGenFMModel): + def generate_md_by_reducing_width(self, reducing_width_ratio, samples: torch.Tensor): + # return FM_to_MD_ViT_Util().init_md_from_fm_by_reducing_width_with_perf_test(self.models_dict['main'], + # reducing_width_ratio, samples) + raise NotImplementedError + + def get_feature_hook(self) -> LayerActivation: + return LayerActivation(get_module(self.models_dict['main'], 'classifier'), True, self.device) + + def get_elastic_dnn_util(self) -> ElasticDNNUtil: + raise NotImplementedError + + def forward_to_get_task_loss(self, x, y): + self.to_train_mode() + return self.infer(x) + + def get_lora_util(self) -> FMLoRA_Util: + return FMLoRA_GPT_Util() + + def get_task_head_params(self): + head = get_module(self.models_dict['main'], 'model.lm_head') + params_name = {k for k, v in head.named_parameters()} + logger.info(f'task head params: {params_name}') + return list(head.parameters()) + + +class ElasticDNN_ViT_OfflineDetMDModel(ElasticDNN_OfflineSenClsMDModel): + def get_feature_hook(self) -> LayerActivation: + return LayerActivation(get_module(self.models_dict['main'], 'classifier'), True, self.device) + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + self.to_train_mode() + return F.cross_entropy(self.infer(x), y) + + +if __name__ == '__main__': + from utils.dl.common.env import set_random_seed + set_random_seed(1) + torch.cuda.set_device(0) + # torch.cuda.device_count() + # runned + # gpt_neo_series_id = '125m_ckpt' + # gpt_neo_series_id = '1.3B_ckpt' + + + + os.environ['gpt_neo_series_id'] = gpt_neo_series_id + + scenario = build_gen_scenario( + source_datasets_name=['No_robots'], + target_datasets_order=['No_robots'] * 1, # TODO + da_mode='close_set', + data_dirs={ + 'No_robots': f'/data/zql/datasets/no_robots', + }, + ) + + # 2. init model + device = 'cuda' + from gpt_neo import GPTNeoForNLG, getTokenizer + tokenizer = getTokenizer() + model = GPTNeoForNLG(gpt_neo_series_id) + model.model.resize_token_embeddings(len(tokenizer)) + model.model.tie_weights() + + fm_models_dict_path = save_models_dict_for_init({ + 'main': model + }, __file__, 'gpt_neo_pretrained_text_gen') + + fm_model = ElasticDNN_GPT_OfflineTextGenFMModel('fm', fm_models_dict_path, device) + + # 3. init alg + models = { + 'fm': fm_model + } + fm_lora_alg = ElasticDNN_FMLoRAAlg(models, get_res_save_dir(__file__, "results")) + + + # 4. run alg + from utils.dl.common.lr_scheduler import get_linear_schedule_with_warmup + fm_lora_alg.run(scenario, hyps={ + 'launch_tbboard': False, + + 'samples_size': {'input_ids': torch.tensor([[ 101, 5672, 2033, 2011, 2151, 3793, 2017, 1005, 1040, 2066, 1012, 102]]).to(device), + 'token_type_ids': torch.tensor([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]).to(device), + 'attention_mask': torch.tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]).to(device), 'return_dict': False}, + + 'ab_r': 8, + 'train_batch_size': 4, + 'val_batch_size': 1, + 'num_workers': 4, + 'optimizer': 'AdamW', + 'optimizer_args': {'lr': 5e-5, 'betas': [0.9, 0.999]}, + 'scheduler': 'LambdaLR', + 'scheduler_args': {'lr_lambda': get_linear_schedule_with_warmup(10000, 80000)}, + 'num_iters': 80000, + 'val_freq': 1000, + + # 'fm_lora_ckpt_path': 'experiments/elasticdnn/vit_b_16/offline/fm_lora/cls/results/cls.py/20230607/999995-234355-trial/models/fm_best.pt' + }, collate_fn=collate_fn) \ No newline at end of file diff --git a/new_impl/nlp/gpt-neo/text_generation/gen_md_w_fbs_index.py b/new_impl/nlp/gpt-neo/text_generation/gen_md_w_fbs_index.py new file mode 100644 index 0000000000000000000000000000000000000000..205653d029fb054689ba297c5c10da8b4b6491ac --- /dev/null +++ b/new_impl/nlp/gpt-neo/text_generation/gen_md_w_fbs_index.py @@ -0,0 +1,201 @@ +import os +gpt_neo_series_id = '1.3B_ckpt' +os.environ['gpt_neo_series_id'] = gpt_neo_series_id +import torch +import sys +from torch import nn +from gpt_neo import ElasticGPTUtil, FMLoRA_GPT_Util, ElasticDNN_OfflineTextGenFMModel, ElasticDNN_OfflineTextGenMDModel, FM_to_MD_GPT_Util, collate_fn +from methods.elasticdnn.api.algs.md_pretraining_wo_fbs import ElasticDNN_MDPretrainingWoFBSAlg +from methods.elasticdnn.model.base import ElasticDNNUtil +from methods.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from methods.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from utils.dl.common.model import LayerActivation, LayerActivation2, get_module, get_parameter +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_gen_scenario +from utils.dl.common.loss import CrossEntropyLossSoft2 +import torch.nn.functional as F +from utils.common.log import logger + +class ElasticDNN_GPT_OfflineTextGenFMModel(ElasticDNN_OfflineTextGenFMModel): + def generate_md_by_reducing_width(self, reducing_width_ratio, samples: torch.Tensor): + return FM_to_MD_GPT_Util().init_md_from_fm_by_reducing_width_with_perf_test(self.models_dict['main'], + reducing_width_ratio, samples) + + def generate_md_by_reducing_layers(self, layers, samples: torch.Tensor): + return FM_to_MD_GPT_Util().init_md_from_fm_by_reducing_layers_with_perf_test(self.models_dict['main'], + layers, samples) + + def get_feature_hook(self) -> LayerActivation: + return LayerActivation(get_module(self.models_dict['main'], 'model.lm_head'), True, self.device) + + def get_feature_hooks(self, layers=None): + res = {} + res['head'] = LayerActivation(get_module(self.models_dict['main'], 'model.lm_head'), True, self.device) + res['hiddens'] = [] + for block_i, _ in enumerate(self.models_dict['main'].model.transformer.h): + if layers is None or block_i in layers: + res['hiddens'].append(LayerActivation(get_module(self.models_dict['main'], f'model.transformer.h.{block_i}'), True, self.device)) + return res + + def get_elastic_dnn_util(self) -> ElasticDNNUtil: + return ElasticGPTUtil() + + def forward_to_get_task_loss(self, x, y): + self.to_train_mode() + return self.infer(x) + + def get_lora_util(self) -> FMLoRA_Util: + return FMLoRA_GPT_Util() + + def get_task_head_params(self): + head = get_module(self.models_dict['main'], 'model.lm_head') + params_name = {k for k, v in head.named_parameters()} + logger.info(f'task head params: {params_name}') + return list(head.parameters()) + + +class ElasticDNN_GPT_OfflineTextGenMDModel(ElasticDNN_OfflineTextGenMDModel): + def __init__(self, name: str, models_dict_path: str, device: str): + super().__init__(name, models_dict_path, device) + + self.distill_criterion = CrossEntropyLossSoft2() + # self.distill_criterion = nn.KLDivLoss(reduction="batchmean") + self.hidden_criterion = nn.MSELoss() + + def get_feature_hook(self) -> LayerActivation2: + return LayerActivation2(get_module(self.models_dict['main'], 'model.lm_head')) + + def get_feature_hooks(self): + res = {} + res['head'] = LayerActivation2(get_module(self.models_dict['main'], 'model.lm_head')) + res['hiddens'] = [LayerActivation(get_module(self.models_dict['main'], f'model.transformer.h.{block_i}'), True, self.device) for block_i, _ in enumerate(self.models_dict['main'].model.transformer.h)] + return res + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + self.to_train_mode() + return self.infer(x) + + def get_distill_loss(self, student_hook, teacher_hook, loss_mask): + student_output = student_hook['head'].output + teacher_output = teacher_hook['head'].output + + t = 1 + # print(student_output, teacher_output) + loss_logits = self.distill_criterion( + student_output / t, # vocab_size + teacher_output / t, + loss_mask + ) * (t ** 2) + + loss_hid = 0. + # num_layers = 0 + # for stu_hid, tea_hid in zip(student_hook['hiddens'], teacher_hook['hiddens']): + # loss_hid += self.hidden_criterion(stu_hid.output, tea_hid.output) + # num_layers += 1 + # loss_hid /= num_layers + + loss = loss_logits + loss_hid + + return loss + + def get_matched_param_of_fm(self, self_param_name, fm: nn.Module): # TODO: + if any([k in self_param_name for k in ['fbs', 'wte', 'wpe']]): + return None + + # 1. xx.qkv.to_qkv.yy to xx.qkv.qkv.aa and xx.qkv.abs.zz + if 'q_proj' in self_param_name or 'k_proj' in self_param_name or 'v_proj' in self_param_name: + ss = self_param_name.split('.') + # raise NotImplementedError() # TODO: + fm_qkv_name = '.'.join(ss[0: -1]) + '.fc' + fm_qkv = get_module(fm, fm_qkv_name) + + fm_abs_name = '.'.join(ss[0: -1]) + '.ab' + fm_abs = [get_module(fm, fm_abs_name)] + + return torch.cat([ + fm_qkv.weight.data, # task-agnositc params + torch.cat([(_abs[0].weight.T @ _abs[1].weight.T).T for _abs in fm_abs], dim=0) # task-specific params (LoRA) + ], dim=0) + + elif 'mlp.c_fc' in self_param_name: + fm_param_name = self_param_name.replace('.linear', '') + return get_parameter(fm, fm_param_name) + + else: + return None + + + +if __name__ == '__main__': + from utils.dl.common.env import set_random_seed + set_random_seed(1) + + # 3. init scenario + scenario = build_gen_scenario( + source_datasets_name=['No_robots'], + target_datasets_order=['No_robots'] * 1, # TODO + da_mode='close_set', + data_dirs={ + 'No_robots': f'/data/zql/datasets/no_robots', + }, + ) + + # 1. init model + # from dnns.deeplabv3.head import modify_forward_head + # modify_forward_head() # TODO: bring a bug + # from dnns.vit import vit_b_16 + fm_models_dict_path = 'new_impl/nlp/gpt-neo/text_generation/results/gen_lora.py/20231210/999999-205643-results/models/fm_last.pt' + fm_models = torch.load(fm_models_dict_path) + fm_models_dict_path = save_models_dict_for_init(fm_models, __file__, 'fm_gpt_txt_gen_lora') + md_models_dict_path = save_models_dict_for_init( + torch.load('new_impl/nlp/gpt-neo/text_generation/results/gen_md_wo_fbs.py/20231220/999999-225735/models/md_best.pt'), + __file__, 'md_gpt_txt_gen_raw_pretrained') + device = 'cuda' + + fm_model = ElasticDNN_GPT_OfflineTextGenFMModel('fm', fm_models_dict_path, device) + md_model = ElasticDNN_GPT_OfflineTextGenMDModel('md', md_models_dict_path, device) + + # 2. init alg + models = { + 'fm': fm_model, + 'md': md_model + } + from methods.elasticdnn.api.algs.md_pretraining_index_v2_train_index_and_md import ElasticDNN_MDPretrainingIndexAlg + fm_to_md_alg = ElasticDNN_MDPretrainingIndexAlg(models, get_res_save_dir(__file__, 'results')) + + from utils.dl.common.lr_scheduler import get_linear_schedule_with_warmup + fm_to_md_alg.run(scenario, hyps={ + 'launch_tbboard': False, + + 'samples_size': {'input_ids': torch.tensor([[ 101, 5672, 2033, 2011, 2151, 3793, 2017, 1005, 1040, 2066, 1012, 102]]).to(device), + 'token_type_ids': torch.tensor([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]).to(device), + 'attention_mask': torch.tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]).to(device), 'return_dict': False}, + + 'FBS_r': 16, + 'FBS_ignore_layers': [], + + 'train_batch_size': 4, + 'val_batch_size': 1, + 'num_workers': 4, + 'optimizer': 'AdamW', + 'optimizer_args': {'lr': 1e-4, 'betas': [0.9, 0.999]}, + 'scheduler': 'LambdaLR', + 'scheduler_args': {'lr_lambda': get_linear_schedule_with_warmup(1000, 40000)}, + + 'indexes_optimizer_args': {'lr': 3e-3, 'momentum': 0.9, 'weight_decay': 5e-4}, + + 'num_iters': 40000, + 'val_freq': 100, + + 'max_sparsity': 0.9, + 'min_sparsity': 0.3, + 'l1_reg_loss_weight': 1e-9, + 'index_loss_weight': 1e-4, + 'val_num_sparsities': 4, + + 'bn_cal_num_iters': 0, + + 'index_init': 'zero', + 'index_guided_linear_comb_split_size': 128 + }, collate_fn=collate_fn) + \ No newline at end of file diff --git a/new_impl/nlp/gpt-neo/text_generation/gen_md_wo_fbs.py b/new_impl/nlp/gpt-neo/text_generation/gen_md_wo_fbs.py new file mode 100644 index 0000000000000000000000000000000000000000..9dc65362fde1f57a073b6bea9a293b8260fa113b --- /dev/null +++ b/new_impl/nlp/gpt-neo/text_generation/gen_md_wo_fbs.py @@ -0,0 +1,196 @@ +import os +gpt_neo_series_id = '1.3B_ckpt' +os.environ['gpt_neo_series_id'] = gpt_neo_series_id +os.environ['CUDA_LAUNCH_BLOCKING'] = '1' +# os.environ['CUDA_VISIBLE_DEVICES'] = '0, 1' + +import torch +import torch.nn as nn +from methods.elasticdnn.api.model import ElasticDNN_OfflineSenClsFMModel, ElasticDNN_OfflineSenClsMDModel +from methods.elasticdnn.api.algs.md_pretraining_wo_fbs_gpt_neo import ElasticDNN_MDPretrainingWoFBSAlg +from methods.elasticdnn.model.base import ElasticDNNUtil +from methods.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from gpt_neo import FMLoRA_GPT_Util, ElasticDNN_OfflineTextGenFMModel, ElasticDNN_OfflineTextGenMDModel, FM_to_MD_GPT_Util, collate_fn +# from methods.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +# from methods.elasticdnn.pipeline.offline.fm_to_md.vit import FM_to_MD_ViT_Util +# from methods.elasticdnn.model.vit import ElasticViTUtil +from utils.dl.common.model import LayerActivation, LayerActivation2, get_module, get_parameter, set_module +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_gen_scenario +from utils.common.log import logger +from utils.dl.common.loss import CrossEntropyLossSoft2 +import torch.nn.functional as F +import sys + +class ElasticDNN_GPT_OfflineTextGenFMModel(ElasticDNN_OfflineTextGenFMModel): + def generate_md_by_reducing_width(self, reducing_width_ratio, samples: torch.Tensor): + return FM_to_MD_GPT_Util().init_md_from_fm_by_reducing_width_with_perf_test(self.models_dict['main'], + reducing_width_ratio, samples) + + def generate_md_by_reducing_layers(self, layers, samples: torch.Tensor): + return FM_to_MD_GPT_Util().init_md_from_fm_by_reducing_layers_with_perf_test(self.models_dict['main'], + layers, samples) + + def get_feature_hook(self) -> LayerActivation: + return LayerActivation(get_module(self.models_dict['main'], 'model.lm_head'), True, self.device) + + def get_feature_hooks(self, layers=None): + res = {} + res['head'] = LayerActivation(get_module(self.models_dict['main'], 'model.lm_head'), True, self.device) + res['hiddens'] = [] + for block_i, _ in enumerate(self.models_dict['main'].model.transformer.h): + if layers is None or block_i in layers: + res['hiddens'].append(LayerActivation(get_module(self.models_dict['main'], f'model.transformer.h.{block_i}'), True, self.device)) + return res + + def get_elastic_dnn_util(self) -> ElasticDNNUtil: + raise NotImplementedError + + def forward_to_get_task_loss(self, x, y): + self.to_train_mode() + return self.infer(x) + + def get_lora_util(self) -> FMLoRA_Util: + return FMLoRA_GPT_Util() + + def get_task_head_params(self): + head = get_module(self.models_dict['main'], 'model.lm_head') + params_name = {k for k, v in head.named_parameters()} + logger.info(f'task head params: {params_name}') + return list(head.parameters()) + + +class ElasticDNN_GPT_OfflineTextGenMDModel(ElasticDNN_OfflineTextGenMDModel): + def __init__(self, name: str, models_dict_path: str, device: str): + super().__init__(name, models_dict_path, device) + + self.distill_criterion = CrossEntropyLossSoft2() + # self.distill_criterion = nn.KLDivLoss(reduction="batchmean") + self.hidden_criterion = nn.MSELoss() + + def get_feature_hook(self) -> LayerActivation2: + return LayerActivation2(get_module(self.models_dict['main'], 'model.lm_head')) + + def get_feature_hooks(self): + res = {} + res['head'] = LayerActivation2(get_module(self.models_dict['main'], 'model.lm_head')) + res['hiddens'] = [LayerActivation(get_module(self.models_dict['main'], f'model.transformer.h.{block_i}'), True, self.device) for block_i, _ in enumerate(self.models_dict['main'].model.transformer.h)] + return res + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + self.to_train_mode() + return self.infer(x) + + def get_distill_loss(self, student_hook, teacher_hook, loss_mask): + student_output = student_hook['head'].output + teacher_output = teacher_hook['head'].output + + t = 1 + # print(student_output, teacher_output) + loss_logits = self.distill_criterion( + student_output / t, # vocab_size + teacher_output / t, + loss_mask + ) * (t ** 2) + + loss_hid = 0. + # num_layers = 0 + # for stu_hid, tea_hid in zip(student_hook['hiddens'], teacher_hook['hiddens']): + # loss_hid += self.hidden_criterion(stu_hid.output, tea_hid.output) + # num_layers += 1 + # loss_hid /= num_layers + + loss = loss_logits + loss_hid + + return loss + + def get_matched_param_of_fm(self, self_param_name, fm: nn.Module): # TODO: + if any([k in self_param_name for k in ['fbs', 'wte', 'wpe']]): + return None + + # 1. xx.qkv.to_qkv.yy to xx.qkv.qkv.aa and xx.qkv.abs.zz + if 'q_proj' in self_param_name or 'k_proj' in self_param_name or 'v_proj' in self_param_name: + ss = self_param_name.split('.') + raise NotImplementedError() # TODO: + fm_qkv_name = '.'.join(ss[0: -2]) + '.fc' + fm_qkv = get_module(fm, fm_qkv_name) + + fm_abs_name = '.'.join(ss[0: -2]) + '.ab' + fm_abs = get_module(fm, fm_abs_name) + + return torch.cat([ + fm_qkv.weight.data, # task-agnositc params + torch.cat([(_abs[0].weight.T @ _abs[1].weight.T).T for _abs in fm_abs], dim=0) # task-specific params (LoRA) + ], dim=0) + + elif 'to_qkv.bias' in self_param_name: + ss = self_param_name.split('.') + + fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv.bias' + return get_parameter(fm, fm_qkv_name) + + elif 'mlp.c_fc' in self_param_name: + fm_param_name = self_param_name.replace('.linear', '') + return get_parameter(fm, fm_param_name) + + else: + return get_parameter(fm, self_param_name) + + +if __name__ == '__main__': + from utils.dl.common.env import set_random_seed + set_random_seed(1) + torch.cuda.set_device(1) + # 3. init scenario + os.environ['gpt_neo_series_id'] = gpt_neo_series_id + + scenario = build_gen_scenario( + source_datasets_name=['No_robots'], + target_datasets_order=['No_robots'] * 1, # TODO + da_mode='close_set', + data_dirs={ + 'No_robots': f'/data/zql/datasets/no_robots', + }, + ) + + # 1. init model + + fm_models_dict_path = 'new_impl/nlp/gpt-neo/text_generation/results/gen_lora.py/20231210/999999-205643-results/models/fm_last.pt' + fm_models = torch.load(fm_models_dict_path) + fm_models_dict_path = save_models_dict_for_init(fm_models, __file__, 'fm_gpt_txt_gen_lora') + md_models_dict_path = save_models_dict_for_init({ + 'main': -1 + }, __file__, 'md_gpt_none') + device = 'cuda' + + fm_model = ElasticDNN_GPT_OfflineTextGenFMModel('fm', fm_models_dict_path, device) + md_model = ElasticDNN_GPT_OfflineTextGenMDModel('md', md_models_dict_path, device) + + # 2. init alg + models = { + 'fm': fm_model, + 'md': md_model + } + fm_to_md_alg = ElasticDNN_MDPretrainingWoFBSAlg(models, get_res_save_dir(__file__, None)) + + from utils.dl.common.lr_scheduler import get_linear_schedule_with_warmup + fm_to_md_alg.run(scenario, hyps={ + 'launch_tbboard': False, + + 'samples_size': {'input_ids': torch.tensor([[ 101, 5672, 2033, 2011, 2151, 3793, 2017, 1005, 1040, 2066, 1012, 102]]).to(device), + 'token_type_ids': torch.tensor([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]).to(device), + 'attention_mask': torch.tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]).to(device), 'return_dict': False}, + # 'generate_md_layers': [0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 23], + 'generate_md_width_ratio': 8, + 'train_batch_size': 4, + 'val_batch_size': 1, + 'num_workers': 4, + 'optimizer': 'AdamW', + 'optimizer_args': {'lr': 1e-4, 'betas': [0.9, 0.999]}, + 'scheduler': 'LambdaLR', + 'scheduler_args': {'lr_lambda': get_linear_schedule_with_warmup(10000, 320000)}, + 'num_iters': 320000, + 'val_freq': 1000, + 'distill_loss_weight': 0.6 + }, collate_fn=collate_fn) + \ No newline at end of file diff --git a/new_impl/nlp/gpt-neo/text_generation/gen_online.py b/new_impl/nlp/gpt-neo/text_generation/gen_online.py new file mode 100644 index 0000000000000000000000000000000000000000..1140551bd0316efeefa7d377142daf4654e7d072 --- /dev/null +++ b/new_impl/nlp/gpt-neo/text_generation/gen_online.py @@ -0,0 +1,413 @@ +import os +gpt_neo_series_id = '1.3B_ckpt' +os.environ['gpt_neo_series_id'] = gpt_neo_series_id +import torch +import torch.nn as nn +from methods.elasticdnn.api.algs.fm_lora import ElasticDNN_FMLoRAAlg +from methods.elasticdnn.api.algs.md_pretraining_wo_fbs import ElasticDNN_MDPretrainingWoFBSAlg +from methods.elasticdnn.model.base import ElasticDNNUtil +from methods.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from gpt_neo import getTokenizer, ElasticGPTUtil, FMLoRA_GPT_Util, ElasticDNN_OfflineTextGenFMModel, ElasticDNN_OfflineTextGenMDModel, FM_to_MD_GPT_Util, collate_fn +from methods.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from methods.elasticdnn.pipeline.offline.fm_to_md.vit import FM_to_MD_ViT_Util +from methods.elasticdnn.model.vit import ElasticViTUtil +from methods.elasticdnn.api.algs.md_pretraining_index_v2_train_index_and_md import ElasticDNN_MDPretrainingIndexAlg +from utils.dl.common.model import LayerActivation2, get_module, get_parameter +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_gen_scenario +import torch.nn.functional as F +import os +from utils.dl.common.loss import CrossEntropyLossSoft +from new_impl.cv.feat_align.main_gpt_neo import OnlineFeatAlignModel, FeatAlignAlg +import tqdm +from new_impl.cv.feat_align.mmd import mmd_rbf +from new_impl.cv.utils.elasticfm_da import init_online_model, elasticfm_da +from methods.elasticdnn.api.online_model_v2 import ElasticDNN_OnlineModel +from utils.common.log import logger +import nltk +from nltk.translate.bleu_score import sentence_bleu, corpus_bleu +from nltk.translate.bleu_score import SmoothingFunction +import json + +os.environ['TOKENIZERS_PARALLELISM'] = 'true' + +device = 'cuda:1' +app_name = 'cls' +sd_sparsity = 0.8 + +settings = { + 'involve_fm': True +} + +torch.cuda.set_device(1) + +scenario = build_gen_scenario( + source_datasets_name=['No_robots'], + target_datasets_order=['Medicine_task', 'Law_task'] * 10, + da_mode='close_set', + data_dirs={ + 'No_robots': '/data/zql/datasets/no_robots', + 'Law_task': '/data/zql/datasets/law_task', + 'Medicine_task': '/data/zql/datasets/medicine_task', + }, + ) + +class ElasticDNN_TxtgenOnlineModel(ElasticDNN_OnlineModel): + def get_accuracy(self, test_loader, *args, **kwargs): + acc = 0 + sample_num = 0 + tokenizer = getTokenizer() + self.to_eval_mode() + pred_txt = [] + true_txt = [] + res = [] + with torch.no_grad(): + pbar = tqdm.tqdm(enumerate(test_loader), total=len(test_loader), dynamic_ncols=True, leave=False) + for batch_index, (x, _) in pbar: + if len(x) == 0: + continue + # if batch_index > 10: + # break + for k, v in x.items(): + if isinstance(v, torch.Tensor): + x[k] = v.to(self.device) + # input_ids = [] + inputlen = x['len'] + y = x['labels'] + x['labels'] = None + outputs = self.models_dict['main'].generate(x, pad_id=tokenizer.pad_token_id) + + for i, op in enumerate(outputs): + op = op.tolist() + op = list(filter(lambda x: x != tokenizer.pad_token_id, op)) + txt = tokenizer.decode(op) + txt = txt.replace(tokenizer.pad_token, "") + res.append(txt) + txt = txt[inputlen[i]:] + pred_txt.append(nltk.word_tokenize(txt)) + for tp in y: + true_txt.append(nltk.word_tokenize(tokenizer.decode(tp).replace(tokenizer.pad_token, ''))) + # pred = F.softmax(output, dim=1).argmax(dim=1) + # correct = torch.eq(pred, y).sum().item() + # acc += correct + sample_num += len(y) + + # pbar.set_description(f'cur_batch_total: {len(y)}, cur_batch_correct: {correct}, ' + # f'cur_batch_acc: {(correct / len(y)):.4f}') + json.dump(res, open("./gpt_generation.json", "w")) + smooth = SmoothingFunction() + score = 0. + for pred, true in zip(pred_txt, true_txt): + score += sentence_bleu([true], pred, weights=(0.25, 0.25, 0.25, 0.25), smoothing_function=smooth.method1) + score /= sample_num + return score + + def get_elastic_dnn_util(self) -> ElasticDNNUtil: + return ElasticGPTUtil() + + def get_fm_matched_param_of_md_param(self, md_param_name): + # only between qkv.weight, norm.weight/bias + self_param_name = md_param_name + fm = self.models_dict['fm'] + # if any([k in self_param_name for k in ['fbs', 'cls_token', 'pos_embed']]): + # return None + + # p = get_parameter(self.models_dict['md'], self_param_name) + # if p.dim() == 0: + # return None + # elif p.dim() == 1 and 'norm' in self_param_name and 'weight' in self_param_name: + # return get_parameter(fm, self_param_name) + + if any([k in self_param_name for k in ['fbs', 'wte', 'wpe']]): + return None + + p = get_parameter(self.models_dict['md'], self_param_name) + if p.dim() == 0: + return None + # elif p.dim() == 1 and 'layernorm' in self_param_name and 'weight' in self_param_name: + # return get_parameter(fm, self_param_name) + # 1. xx.qkv.to_qkv.yy to xx.qkv.qkv.aa and xx.qkv.abs.zz + # if 'qkv.weight' in self_param_name: + # ss = self_param_name.split('.') + + # fm_qkv_name = '.'.join(ss[0: -1]) + '.qkv' + # fm_qkv = get_module(fm, fm_qkv_name) + + # fm_abs_name = '.'.join(ss[0: -1]) + '.abs' + # fm_abs = get_module(fm, fm_abs_name) + + # # NOTE: unrecoverable operation! multiply LoRA parameters to allow it being updated in update_fm_param() + # # TODO: if fm will be used for inference, _mul_lora_weight will not be applied! + # if not hasattr(fm_abs, '_mul_lora_weight'): + # logger.debug(f'set _mul_lora_weight in {fm_abs_name}') + # setattr(fm_abs, '_mul_lora_weight', + # nn.Parameter(torch.cat([(_abs[0].weight.T @ _abs[1].weight.T).T for _abs in fm_abs], dim=0))) + + # return torch.cat([ + # fm_qkv.weight.data, # task-agnositc params + # fm_abs._mul_lora_weight.data # task-specific params (LoRA) + # ], dim=0) + + # # elif 'to_qkv.bias' in self_param_name: + # # ss = self_param_name.split('.') + + # # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv.bias' + # # return get_parameter(fm, fm_qkv_name) + + # elif 'mlp.fc1' in self_param_name and 'weight' in self_param_name: + # fm_param_name = self_param_name.replace('.linear', '') + # return get_parameter(fm, fm_param_name) + + # elif 'mlp.fc2' in self_param_name and 'weight' in self_param_name: + # fm_param_name = self_param_name + # return get_parameter(fm, fm_param_name) + + # else: + # # return get_parameter(fm, self_param_name) + # return None + if ('q_proj' in self_param_name or 'k_proj' in self_param_name or \ + 'v_proj' in self_param_name) and ('weight' in self_param_name): + ss = self_param_name.split('.') + + fm_qkv_name = '.'.join(ss[0: -1]) + '.fc' + fm_qkv = get_module(fm, fm_qkv_name) + + fm_abs_name = '.'.join(ss[0: -1]) + '.ab' + fm_abs = get_module(fm, fm_abs_name) + + if not hasattr(fm_abs, '_mul_lora_weight'): + logger.debug(f'set _mul_lora_weight in {fm_abs_name}') + setattr(fm_abs, '_mul_lora_weight', + nn.Parameter(fm_abs[1].weight @ fm_abs[0].weight)) + + return torch.cat([ + fm_qkv.weight.data, # task-agnositc params + fm_abs._mul_lora_weight.data # task-specific params (LoRA) + ], dim=0) + + elif ('q_proj' in self_param_name or 'k_proj' in self_param_name or \ + 'v_proj' in self_param_name) and ('bias' in self_param_name): + ss = self_param_name.split('.') + + fm_qkv_name = '.'.join(ss[0: -1]) + '.fc.bias' + return get_parameter(fm, fm_qkv_name) + + elif 'mlp.c_fc' in self_param_name and 'weight' in self_param_name: + fm_param_name = self_param_name.replace('.linear', '') + return get_parameter(fm, fm_param_name) + elif 'mlp.c_fc' in self_param_name and 'bias' in self_param_name: + fm_param_name = self_param_name.replace('.linear', '') + return get_parameter(fm, fm_param_name) + # elif 'mlp.fc2' in self_param_name and 'weight' in self_param_name: + # fm_param_name = self_param_name.replace('.linear', '') + # return get_parameter(fm, fm_param_name) + else: + #return get_parameter(fm, self_param_name) + return None + + + def update_fm_param(self, md_param_name, cal_new_fm_param_by_md_param): + if not (('q_proj' in md_param_name or 'k_proj' in md_param_name or \ + 'v_proj' in md_param_name) and ('weight' in md_param_name)): + matched_fm_param_ref = self.get_fm_matched_param_of_md_param(md_param_name) + matched_fm_param_ref.copy_(cal_new_fm_param_by_md_param) + else: + new_fm_attn_weight, new_fm_lora_weight = torch.chunk(cal_new_fm_param_by_md_param, 2, 0) + ss = md_param_name.split('.') + fm = self.models_dict['fm'] + # update task-agnostic parameters + fm_qkv_name = '.'.join(ss[0: -1]) + '.fc' + fm_qkv = get_module(fm, fm_qkv_name) + fm_qkv.weight.data.copy_(new_fm_attn_weight) + + # update task-specific parameters + fm_abs_name = '.'.join(ss[0: -1]) + '.ab' + fm_abs = get_module(fm, fm_abs_name) + fm_abs._mul_lora_weight.data.copy_(new_fm_lora_weight) # TODO: this will not be applied in inference! + + def get_md_matched_param_of_fm_param(self, fm_param_name): + return super().get_md_matched_param_of_fm_param(fm_param_name) + + def get_md_matched_param_of_sd_param(self, sd_param_name): + # raise NotImplementedError + + # only between qkv.weight, norm.weight/bias + self_param_name = sd_param_name + md = self.models_dict['md'] + if any([k in self_param_name for k in ['fbs', 'wte', 'wpe']]): + return None + + p = get_parameter(self.models_dict['sd'], self_param_name) + if p.dim() == 0: + return None + elif p.dim() == 1 and ('LayerNorm' in self_param_name or 'ln' in self_param_name) and 'weight' in self_param_name: + return get_parameter(md, self_param_name) + + # 1. xx.qkv.to_qkv.yy to xx.qkv.qkv.aa and xx.qkv.abs.zz + if ('q_proj' in self_param_name or 'k_proj' in self_param_name or \ + 'v_proj' in self_param_name) and ('weight' in self_param_name): + return get_parameter(md, self_param_name) # NOTE: no fbs in qkv! + + # elif 'to_qkv.bias' in self_param_name: + # ss = self_param_name.split('.') + + # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv.bias' + # return get_parameter(fm, fm_qkv_name) + + elif 'mlp.c_fc.0.weight' in self_param_name: + fm_param_name = '.'.join(self_param_name.split('.')[0: -2]) + '.linear.weight' + return get_parameter(md, fm_param_name) + + elif 'mlp.c_fc.0.bias' in self_param_name: + fm_param_name = '.'.join(self_param_name.split('.')[0: -2]) + '.linear.bias' + return get_parameter(md, fm_param_name) + + elif 'mlp.c_proj' in self_param_name and 'weight' in self_param_name: + fm_param_name = self_param_name + return get_parameter(md, fm_param_name) + + elif 'static_channel_attention' not in self_param_name: + return get_parameter(md, self_param_name) + # return None + + def get_task_head_params(self): + head = get_module(self.models_dict['sd'], 'classifier') + return list(head.parameters()) + + + +class TxtgenOnlineFeatAlignModel(OnlineFeatAlignModel): + def get_trained_params(self): # TODO: elastic fm only train a part of params + #qkv_and_norm_params = [p for n, p in self.models_dict['main'].named_parameters() if 'attention.attention.projection_query' in n or 'attention.attention.projection_key' in n or 'attention.attention.projection_value' in n or 'intermediate.dense' in n or 'output.dense' in n] + qkv_and_norm_params = [p for n, p in self.models_dict['main'].named_parameters()] + return qkv_and_norm_params + + def get_feature_hook(self) -> LayerActivation2: + return LayerActivation2(get_module(self.models_dict['main'], 'model.lm_head')) + + def forward_to_get_task_loss(self, x, y): + losses = self.infer(x) + # print(losses) + + return losses + + def get_mmd_loss(self, f1, f2): + common_shape = min(f1.shape[0], f2.shape[0]) + f1 = f1.view(f1.shape[0], -1) + f2 = f2.view(f2.shape[0], -1) + f1 = f1[:common_shape,:] + f2 = f2[:common_shape,:] + return mmd_rbf(f1, f2) + + def infer(self, x, *args, **kwargs): + return self.models_dict['main'](**x) + + def get_accuracy(self, test_loader, *args, **kwargs): + acc = 0 + sample_num = 0 + tokenizer = getTokenizer() + self.to_eval_mode() + pred_txt = [] + true_txt = [] + res = [] + with torch.no_grad(): + pbar = tqdm.tqdm(enumerate(test_loader), total=len(test_loader), dynamic_ncols=True, leave=False) + for batch_index, (x, _) in pbar: + if len(x) == 0: + continue + # if batch_index > 10: + # break + for k, v in x.items(): + if isinstance(v, torch.Tensor): + x[k] = v.to(self.device) + # input_ids = [] + inputlen = x['len'] + y = x['labels'] + x['labels'] = None + outputs = self.models_dict['main'].generate(x, pad_id=tokenizer.pad_token_id) + + for i, op in enumerate(outputs): + op = op.tolist() + op = list(filter(lambda x: x != tokenizer.pad_token_id, op)) + txt = tokenizer.decode(op) + txt = txt.replace(tokenizer.pad_token, "") + res.append(txt) + txt = txt[inputlen[i]:] + pred_txt.append(nltk.word_tokenize(txt)) + for tp in y: + true_txt.append(nltk.word_tokenize(tokenizer.decode(tp).replace(tokenizer.pad_token, ''))) + # pred = F.softmax(output, dim=1).argmax(dim=1) + # correct = torch.eq(pred, y).sum().item() + # acc += correct + sample_num += len(y) + + # pbar.set_description(f'cur_batch_total: {len(y)}, cur_batch_correct: {correct}, ' + # f'cur_batch_acc: {(correct / len(y)):.4f}') + json.dump(res, open("./gpt_generation.json", "w")) + smooth = SmoothingFunction() + score = 0. + for pred, true in zip(pred_txt, true_txt): + score += sentence_bleu([true], pred, weights=(0.25, 0.25, 0.25, 0.25), smoothing_function=smooth.method1) + score /= sample_num + return score + + + + +#from new_impl.cv.model import ElasticDNN_ClsOnlineModel +elasticfm_model = ElasticDNN_TxtgenOnlineModel('gen', init_online_model( + 'new_impl/nlp/gpt-neo/text_generation/results/gen_md_w_fbs_index.py/20231222/999995-003118-results/models/fm_best.pt', + 'new_impl/nlp/gpt-neo/text_generation/results/gen_md_w_fbs_index.py/20231222/999995-003118-results/models/md_best.pt', + 'gen', __file__ +), device, { + 'md_to_fm_alpha': 0.01, + 'fm_to_md_alpha': 0.1 +}) + +da_alg = FeatAlignAlg +from utils.dl.common.lr_scheduler import get_linear_schedule_with_warmup +#from new_impl.cv.model import ClsOnlineFeatAlignModel +da_model = TxtgenOnlineFeatAlignModel +da_alg_hyp = { + 'Medicine_task': { + 'train_batch_size': 2, + 'val_batch_size': 1, + 'num_workers': 2, + 'optimizer': 'AdamW', + 'optimizer_args': {'lr': 1e-6, 'betas': [0.9, 0.999], 'weight_decay': 0.01}, + 'scheduler': '', + 'scheduler_args': {}, + 'num_iters': 1000, + 'val_freq': 200, + 'sd_sparsity':0.3, + 'feat_align_loss_weight': 1.0, + }, + 'Law_task': { + 'train_batch_size': 2, + 'val_batch_size': 1, + 'num_workers': 2, + 'optimizer': 'AdamW', + 'optimizer_args': {'lr': 1e-6, 'betas': [0.9, 0.999], 'weight_decay': 0.01}, + 'scheduler': '', + 'scheduler_args': {}, + 'num_iters': 1000, + 'val_freq': 200, + 'sd_sparsity':0.3, + 'feat_align_loss_weight': 1.0, + }, +} + + +elasticfm_da( + [app_name], + [scenario], + [elasticfm_model], + [da_alg], + [da_alg_hyp], + [da_model], + device, + settings, + __file__, + "results", + collate_fn=collate_fn +) \ No newline at end of file diff --git a/new_impl/nlp/gpt-neo/text_generation/gpt_neo.py b/new_impl/nlp/gpt-neo/text_generation/gpt_neo.py new file mode 100644 index 0000000000000000000000000000000000000000..b8afb1c74ae994cf16881d5d5563f9009c936d33 --- /dev/null +++ b/new_impl/nlp/gpt-neo/text_generation/gpt_neo.py @@ -0,0 +1,693 @@ +import torch +from torch import nn +from copy import deepcopy +from abc import ABC, abstractmethod +from methods.elasticdnn.api.model import ElasticDNN_OfflineFMModel, ElasticDNN_OfflineMDModel +from utils.dl.common.model import get_model_device, get_model_latency, get_model_size, set_module, get_module +from utils.common.log import logger +from transformers import GPTNeoForCausalLM +from utils.dl.common.model import set_module +from torch import nn +import torch +from einops import rearrange, repeat +from einops.layers.torch import Rearrange +from utils.common.log import logger +from methods.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from methods.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util, LoRA +from methods.elasticdnn.model.base import Abs, KTakesAll, ElasticDNNUtil, Layer_WrappedWithFBS +import tqdm +from transformers import GPT2Tokenizer +import os +import nltk +from nltk.translate.bleu_score import sentence_bleu, corpus_bleu +from nltk.translate.bleu_score import SmoothingFunction +import json + +def collate_fn(batch): + dict = {} + input_ids = [] + attention_mask = [] + token_type_ids = [] + labels = [] + return_dict = True + lenli = [] + for item in batch: + if len(item) == 1 or len(item['labels']) == 0: + continue + input_ids.append(item['input_ids'].unsqueeze(0)) + if 'attention_mask' in item.keys(): + attention_mask.append(item['attention_mask'].unsqueeze(0)) + if 'token_type_ids' in item.keys(): + token_type_ids.append(item['token_type_ids'].unsqueeze(0)) + labels.append(item['labels'].unsqueeze(0)) + if 'len' in item.keys(): + lenli.append(item['len']) + + dict['return_dict'] = batch[0]['return_dict'] + if len(input_ids) > 0: + dict['input_ids'] = torch.cat(input_ids, dim=0) + else: + return {}, torch.Tensor([0]) + if len(attention_mask) > 0: + dict['attention_mask'] = torch.cat(attention_mask, dim=0) + if len(token_type_ids) > 0: + dict['token_type_ids'] = torch.cat(token_type_ids, dim=0) + dict['labels'] = torch.cat(labels, dim=0) + if len(lenli) > 0: + dict['len'] = lenli + return dict, torch.Tensor([0]) + +class GPTNeoForNLG(nn.Module): + def __init__(self, series_id): + super(GPTNeoForNLG, self).__init__() + + # logger.info(f'init bert for sen cls (using {bert_model_tag})') + self.model = GPTNeoForCausalLM.from_pretrained(f'experiments/elasticdnn/gpt_neo/{series_id}') + self.config = self.model.config + + self.config.pad_token_id = self.config.eos_token_id + + def generate(self, x, pad_id=None): + return self.model.generate(x['input_ids'], max_new_tokens=128, num_beams=3, early_stopping=True, pad_token_id=pad_id) + + def forward(self, **x): + x['return_dict'] = True + + output = self.model(**x) + + if self.training == True: + return output['loss'] + else: + return output['logits'] + +class ToQKV_WrappedWithLoRA(nn.Module): + def __init__(self, fc: nn.Linear, ab_r: int): + super(ToQKV_WrappedWithLoRA, self).__init__() + + self.fc = fc + self.ab = self.create_ab_as_linear(fc.weight.data, ab_r) + + def create_ab_as_linear(self, fc_weight: torch.Tensor, ab_r: int): + res = nn.Sequential( + LoRA(fc_weight.size(1), fc_weight.size(0) // ab_r, bias=False), + LoRA(fc_weight.size(0) // ab_r, fc_weight.size(0), bias=False) + ).to(fc_weight.device) + nn.init.kaiming_uniform_(res[0].weight, a=5 ** 0.5) + nn.init.zeros_(res[1].weight) + return res + + def forward(self, x): + x1 = self.fc(x) + x2 = self.ab(x) + return x1 + x2 + + +class FMLoRA_GPT_Util(FMLoRA_Util): + + @torch.no_grad() + def add_lora_ab_to_fm(self, fm: nn.Module, ab_r: int, samples: dict): + fm.eval() + + o1 = fm(**samples) + + for name, module in fm.named_modules(): + if name.endswith(('k_proj', 'v_proj', 'q_proj')): + set_module(fm, name, ToQKV_WrappedWithLoRA(module, ab_r)) + + o2 = fm(**samples) + + if isinstance(o1, tuple): + o1 = o1[-1] + o2 = o2[-1] + output_diff = ((o1 - o2) ** 2).sum() + assert output_diff < 1e-5 + + return fm + + @torch.no_grad() + def absorb_lora_and_recover_net_structure(self, fm: nn.Module, samples: dict): + fm.eval() + # print('absorb lora before') + o1 = fm(**samples) + + for name, module in fm.named_modules(): + if not isinstance(module, ToQKV_WrappedWithLoRA): + continue + + fc = module.fc + ab = module.ab + + fc.weight.add_(ab[1].weight @ ab[0].weight) + + set_module(fm, name, fc) + + # print('absorb lora after') + o2 = fm(**samples) + + if isinstance(o1, tuple): + o1 = o1[-1] + o2 = o2[-1] + output_diff = ((o1 - o2) ** 2).sum() + # assert output_diff < 1e-6, output_diff + + return fm + +def getTokenizer(): + tokenizer = GPT2Tokenizer.from_pretrained(f'experiments/elasticdnn/gpt_neo/{os.environ["gpt_neo_series_id"]}', padding_side='left') + # tokenizer.pad_token = tokenizer.eos_token + tokenizer.sep_token = tokenizer.eos_token + special_tokens = {"pad_token":"<|pad|>"}#, "sep_token":"<|sep|>", "bos_token":"<|bos|>"} + tokenizer.add_special_tokens(special_tokens) + tokenizer.pad_token = "<|pad|>" + # tokenizer.bos_token = "<|bos|>" + # tokenizer.sep_token = "<|sep|>" + return tokenizer + +class ElasticDNN_OfflineTextGenFMModel(ElasticDNN_OfflineFMModel): + def get_accuracy(self, test_loader, *args, **kwargs): + acc = 0 + sample_num = 0 + tokenizer = getTokenizer() + self.to_eval_mode() + pred_txt = [] + true_txt = [] + res = [] + with torch.no_grad(): + pbar = tqdm.tqdm(enumerate(test_loader), total=len(test_loader), dynamic_ncols=True, leave=False) + for batch_index, (x, _) in pbar: + if len(x) == 0: + continue + # if batch_index > 10: + # break + for k, v in x.items(): + if isinstance(v, torch.Tensor): + x[k] = v.to(self.device) + # input_ids = [] + inputlen = x['len'] + y = x['labels'] + x['labels'] = None + outputs = self.models_dict['main'].generate(x, pad_id=tokenizer.pad_token_id) + + for i, op in enumerate(outputs): + op = op.tolist() + op = list(filter(lambda x: x != tokenizer.pad_token_id, op)) + txt = tokenizer.decode(op) + txt = txt.replace(tokenizer.pad_token, "") + res.append(txt) + txt = txt[inputlen[i]:] + pred_txt.append(nltk.word_tokenize(txt)) + for tp in y: + true_txt.append(nltk.word_tokenize(tokenizer.decode(tp).replace(tokenizer.pad_token, ''))) + # pred = F.softmax(output, dim=1).argmax(dim=1) + # correct = torch.eq(pred, y).sum().item() + # acc += correct + sample_num += len(y) + + # pbar.set_description(f'cur_batch_total: {len(y)}, cur_batch_correct: {correct}, ' + # f'cur_batch_acc: {(correct / len(y)):.4f}') + json.dump(res, open("./gpt_generation.json", "w")) + smooth = SmoothingFunction() + score = 0. + for pred, true in zip(pred_txt, true_txt): + score += sentence_bleu([true], pred, weights=(0.25, 0.25, 0.25, 0.25), smoothing_function=smooth.method1) + score /= sample_num + return score + + def infer(self, x, *args, **kwargs): + return self.models_dict['main'].forward(**x) + +class FM_to_MD_GPT_Util(FM_to_MD_Util): + def init_md_from_fm_by_reducing_width(self, fm: nn.Module, reducing_width_ratio: int, sparsity=0.0) -> nn.Module: + # sparsity: It is mainly used to make a distilled model used in the baseline algorithm, and this parameter can ensure that the model has the same size as the model used in the online algorithm. + fm_vit = deepcopy(fm) + + for block in fm_vit.model.transformer.h: + tmp = get_module(block, 'attn.attention') + tmp.head_dim = max(tmp.head_dim // reducing_width_ratio, 16) + set_module(block, 'attn.attention', tmp) + #ddw + def _f(n): + return int(n // reducing_width_ratio) + + # def _rand_indexes(n): + # return torch.randperm(n)[0: int(n // reducing_width_ratio)] + + def l1_max_indexes_for_qkv(p: torch.Tensor, dim=0): + assert dim in [0, 1] + assert p.dim() in [1, 2, 4] + + if dim == 1: + p = p.T + + p_norm = p.abs().contiguous().view(p.size(0), -1).sum(dim=1) + n = p.size(0) + return p_norm.argsort(descending=True)[0: max(256, int(n // reducing_width_ratio))].sort()[0] + + def l1_max_indexes(p: torch.Tensor, dim=0): + assert dim in [0, 1] + assert p.dim() in [1, 2, 4] + + if dim == 1: + p = p.T + + p_norm = p.abs().contiguous().view(p.size(0), -1).sum(dim=1) + n = p.size(0) + return p_norm.argsort(descending=True)[0: int(n // reducing_width_ratio)].sort()[0] + + def l1_max_indexes_with_sparsity(p: torch.Tensor, dim=0): + assert dim in [0, 1] + assert p.dim() in [1, 2, 4] + + if dim == 1: + p = p.T + + p_norm = p.abs().contiguous().view(p.size(0), -1).sum(dim=1) + n = p.size(0) + return p_norm.argsort(descending=True)[0: int(n // reducing_width_ratio * (1 - sparsity))].sort()[0] + + for block_i, block in enumerate(fm_vit.model.transformer.h): + for k in ['k_proj', 'v_proj', 'q_proj']: + qkv = get_module(block, f'attn.attention.{k}') + + new_qkv = nn.Linear(qkv.in_features, max(256, _f(qkv.out_features)), + qkv.bias is not None, qkv.weight.device) + indexes = l1_max_indexes_for_qkv(qkv.weight.data, 0) + + new_qkv.weight.data.copy_(qkv.weight.data[indexes]) + if qkv.bias is not None: + new_qkv.bias.data.copy_(qkv.bias.data[indexes]) + set_module(block, f'attn.attention.{k}', new_qkv) + + proj = get_module(block, f'attn.attention.out_proj') + new_proj = nn.Linear(max(256, _f(proj.in_features)), proj.out_features, + proj.bias is not None, proj.weight.device) + new_proj.weight.data.copy_(proj.weight.data[:, l1_max_indexes_for_qkv(proj.weight.data, 1)]) + if proj.bias is not None: + new_proj.bias.data.copy_(proj.bias.data) + set_module(block, f'attn.attention.out_proj', new_proj) + + fc1 = get_module(block, f'mlp.c_fc') + new_fc1 = nn.Linear(fc1.in_features, int(_f(fc1.out_features) * (1 - sparsity)), + fc1.bias is not None, fc1.weight.device) + indexes = l1_max_indexes_with_sparsity(fc1.weight.data, 0) + new_fc1.weight.data.copy_(fc1.weight.data[indexes]) + if fc1.bias is not None: + new_fc1.bias.data.copy_(fc1.bias.data[indexes]) + set_module(block, f'mlp.c_fc', new_fc1) + + fc2 = get_module(block, f'mlp.c_proj') + new_fc2 = nn.Linear(int(_f(fc2.in_features) * (1 - sparsity)), fc2.out_features, + fc2.bias is not None, fc2.weight.device) + new_fc2.weight.data.copy_(fc2.weight.data[:, l1_max_indexes_with_sparsity(fc2.weight.data, 1)]) + if fc2.bias is not None: + new_fc2.bias.data.copy_(fc2.bias.data) + set_module(block, f'mlp.c_proj', new_fc2) + + return fm_vit + + def init_md_from_fm_by_reducing_width_with_perf_test(self, fm: nn.Module, reducing_width_ratio: int, + samples: torch.Tensor) -> nn.Module: + fm_size = get_model_size(fm, True) + fm_latency = self._get_model_latency(fm, samples, 20, + get_model_device(fm), 20, False) + + master_dnn = self.init_md_from_fm_by_reducing_width(fm, reducing_width_ratio) + master_dnn_size = get_model_size(master_dnn, True) + logger.debug(f'inited master DNN: {master_dnn}') + master_dnn_latency = self._get_model_latency(master_dnn, samples, 20, + get_model_device(master_dnn), 20, False) + + logger.info(f'init master DNN (w/o FBS yet) by reducing foundation model\'s width (by {reducing_width_ratio:d}x)') + logger.info(f'foundation model ({fm_size:.3f}MB, {fm_latency:.4f}s/sample) -> ' + f'master DNN ({master_dnn_size:.3f}MB, {master_dnn_latency:.4f}s/sample)\n' + f'(model size: ↓ {(fm_size / master_dnn_size):.2f}x, ' + f'latency: ↓ {(fm_latency / master_dnn_latency):.2f}x)') + + return master_dnn + + def init_md_from_fm_by_reducing_layers(self, fm: nn.Module, layers: list) -> nn.Module: + fm_vit = deepcopy(fm) + tmp_h = [] + for block_i, block in enumerate(fm_vit.model.transformer.h): + if block_i in layers: + tmp_h.append(block) + + tmp_h = nn.ModuleList(tmp_h) + set_module(fm_vit, f'model.transformer.h', tmp_h) + + return fm_vit + + def init_md_from_fm_by_reducing_layers_with_perf_test(self, fm: nn.Module, layers: list, + samples: torch.Tensor) -> nn.Module: + fm_size = get_model_size(fm, True) + fm_latency = self._get_model_latency(fm, samples, 20, + get_model_device(fm), 20, False) + + master_dnn = self.init_md_from_fm_by_reducing_layers(fm, layers) + master_dnn_size = get_model_size(master_dnn, True) + logger.debug(f'inited master DNN: {master_dnn}') + master_dnn_latency = self._get_model_latency(master_dnn, samples, 20, + get_model_device(master_dnn), 20, False) + + logger.info(f'init master DNN (w/o FBS yet) by reducing foundation model\'s layers (to {len(layers):d} layers)') + logger.info(f'foundation model ({fm_size:.3f}MB, {fm_latency:.4f}s/sample) -> ' + f'master DNN ({master_dnn_size:.3f}MB, {master_dnn_latency:.4f}s/sample)\n' + f'(model size: ↓ {(fm_size / master_dnn_size):.2f}x, ' + f'latency: ↓ {(fm_latency / master_dnn_latency):.2f}x)') + + return master_dnn + + def _get_model_latency(self, model: torch.nn.Module, model_input_size, sample_num: int, + device: str, warmup_sample_num: int, return_detail=False): + import time + + if isinstance(model_input_size, tuple): + dummy_input = torch.rand(model_input_size).to(device) + else: + dummy_input = model_input_size + + model = model.to(device) + model.eval() + + # warm up + with torch.no_grad(): + for _ in range(warmup_sample_num): + model(**dummy_input) + + infer_time_list = [] + + if device == 'cuda' or 'cuda' in str(device): + with torch.no_grad(): + for _ in range(sample_num): + s, e = torch.cuda.Event(enable_timing=True), torch.cuda.Event(enable_timing=True) + s.record() + model(**dummy_input) + e.record() + torch.cuda.synchronize() + cur_model_infer_time = s.elapsed_time(e) / 1000. + infer_time_list += [cur_model_infer_time] + + else: + with torch.no_grad(): + for _ in range(sample_num): + start = time.time() + model(**dummy_input) + cur_model_infer_time = time.time() - start + infer_time_list += [cur_model_infer_time] + + avg_infer_time = sum(infer_time_list) / sample_num + + if return_detail: + return avg_infer_time, infer_time_list + return avg_infer_time + +class ElasticDNN_OfflineTextGenMDModel(ElasticDNN_OfflineMDModel): + def get_accuracy(self, test_loader, *args, **kwargs): + acc = 0 + sample_num = 0 + tokenizer = getTokenizer() + self.to_eval_mode() + pred_txt = [] + true_txt = [] + res = [] + with torch.no_grad(): + pbar = tqdm.tqdm(enumerate(test_loader), total=len(test_loader), dynamic_ncols=True, leave=False) + for batch_index, (x, _) in pbar: + if len(x) == 0: + continue + # if batch_index > 10: + # break + for k, v in x.items(): + if isinstance(v, torch.Tensor): + x[k] = v.to(self.device) + # input_ids = [] + inputlen = x['len'] + y = x['labels'] + x['labels'] = None + outputs = self.models_dict['main'].generate(x, pad_id=tokenizer.pad_token_id) + + for i, op in enumerate(outputs): + op = op.tolist() + op = list(filter(lambda x: x != tokenizer.pad_token_id, op)) + txt = tokenizer.decode(op) + txt = txt.replace(tokenizer.pad_token, "") + res.append(txt) + txt = txt[inputlen[i]:] + pred_txt.append(nltk.word_tokenize(txt)) + for tp in y: + true_txt.append(nltk.word_tokenize(tokenizer.decode(tp).replace(tokenizer.pad_token, ''))) + # pred = F.softmax(output, dim=1).argmax(dim=1) + # correct = torch.eq(pred, y).sum().item() + # acc += correct + sample_num += len(y) + + # pbar.set_description(f'cur_batch_total: {len(y)}, cur_batch_correct: {correct}, ' + # f'cur_batch_acc: {(correct / len(y)):.4f}') + json.dump(res, open("./gpt_generation.json", "w")) + smooth = SmoothingFunction() + score = 0. + for pred, true in zip(pred_txt, true_txt): + score += sentence_bleu([true], pred, weights=(0.25, 0.25, 0.25, 0.25), smoothing_function=smooth.method1) + score /= sample_num + return score + + def infer(self, x, *args, **kwargs): + return self.models_dict['main'].forward(**x) + +class SqueezeLast(nn.Module): + def __init__(self): + super(SqueezeLast, self).__init__() + + def forward(self, x): + return x.squeeze(-1) + +class StaticFBS(nn.Module): + def __init__(self, static_channel_attention): + super(StaticFBS, self).__init__() + assert static_channel_attention.dim() == 2 and static_channel_attention.size(0) == 1 + self.static_channel_attention = nn.Parameter(static_channel_attention, requires_grad=False) # (1, dim) + + def forward(self, x): + # print('staticfbs', x, self.static_channel_attention.unsqueeze(1)) + return x * self.static_channel_attention.unsqueeze(1) + +class Linear_WrappedWithFBS(Layer_WrappedWithFBS): + def __init__(self, linear: nn.Linear, r): + super(Linear_WrappedWithFBS, self).__init__() + + self.linear = linear + + # for conv: (B, C_in, H, W) -> (B, C_in) -> (B, C_out) + # for mlp in ViT: (B, #patches, D: dim of patches embedding) -> (B, D) -> (B, C_out) + self.fbs = nn.Sequential( + Rearrange('b n d -> b d n'), + Abs(), + nn.AdaptiveAvgPool1d(1), + SqueezeLast(), + nn.Linear(linear.in_features, max(linear.out_features // r, 36)), + nn.ReLU(), + nn.Linear(max(linear.out_features // r, 36), linear.out_features), + nn.ReLU() + ) + + nn.init.constant_(self.fbs[6].bias, 1.) + nn.init.kaiming_normal_(self.fbs[6].weight) + + + def forward(self, x): + if self.use_cached_channel_attention and self.cached_channel_attention is not None: + channel_attention = self.cached_channel_attention + else: + self.cached_raw_channel_attention = self.fbs(x) + self.cached_channel_attention = self.k_takes_all(self.cached_raw_channel_attention) + + channel_attention = self.cached_channel_attention + + raw_res = self.linear(x) + + return channel_attention.unsqueeze(1) * raw_res + +class ElasticGPTUtil(ElasticDNNUtil): + def convert_raw_dnn_to_master_dnn(self, raw_dnn: nn.Module, r: float, ignore_layers=[]): + assert len(ignore_layers) == 0, 'not supported yet' + + raw_vit = deepcopy(raw_dnn) + + # set_module(module, 'patch_embed.proj', ProjConv_WrappedWithFBS(module.patch_embed.proj, r)) + + for name, module in raw_vit.named_modules(): + # if name.endswith('attn'): + # set_module(module, 'qkv', ToQKV_WrappedWithFBS(module.qkv, r)) + if name.endswith('intermediate'): + set_module(module, 'dense', Linear_WrappedWithFBS(module.dense, r)) + elif name.endswith('mlp'): + set_module(module, 'c_fc', Linear_WrappedWithFBS(module.c_fc, r)) + + + return raw_vit + + def set_master_dnn_sparsity(self, master_dnn: nn.Module, sparsity: float): + # for name, module in master_dnn.named_modules(): + # if not name.endswith('attn'): + # continue + + # q_features = module.qkv.to_qk.out_features // 2 + + # if (q_features - int(q_features * sparsity)) % module.num_heads != 0: + # # tune sparsity to ensure #unpruned channel % num_heads == 0 + # # so that the pruning seems to reduce the dim_head of each head + # tuned_sparsity = 1. - int((q_features - int(q_features * sparsity)) / module.num_heads) * module.num_heads / q_features + # logger.debug(f'tune sparsity from {sparsity:.2f} to {tuned_sparsity}') + # sparsity = tuned_sparsity + # break + + return super().set_master_dnn_sparsity(master_dnn, sparsity) + + def select_most_rep_sample(self, master_dnn: nn.Module, samples: torch.Tensor): + # print(samples) + # return samples[0].unsqueeze(0) + ret_di = samples['return_dict'] + del samples['return_dict'] + res = {k: v[0: 1] for k, v in samples.items()} + res['return_dict'] = ret_di + return res + + def extract_surrogate_dnn_via_samples(self, master_dnn: nn.Module, samples: torch.Tensor, return_detail=False): + sample = self.select_most_rep_sample(master_dnn, samples) + # assert sample.dim() == 4 and sample.size(0) == 1 + + # print('before') + master_dnn.eval() + self.clear_cached_channel_attention_in_master_dnn(master_dnn) + with torch.no_grad(): + master_dnn_output = master_dnn(**sample) + + # print('after') + + boosted_vit = deepcopy(master_dnn) + + def get_unpruned_indexes_from_channel_attn(channel_attn: torch.Tensor, k): + assert channel_attn.size(0) == 1, 'use A representative sample to generate channel attentions' + + # print('attn_in_unpruned', channel_attn[0][0: 10]) + + res = channel_attn[0].nonzero(as_tuple=True)[0] # should be one-dim + + # res = channel_attn[0].argsort(descending=True)[0: -int(channel_attn.size(1) * k)].sort()[0] + + # g = channel_attn + # k = g.size(1) - int(g.size(1) * k) + # res = g.topk(k, 1)[1][0].sort()[0] + + return res + + unpruned_indexes_of_layers = {} + + # for attn, ff in boosted_vit.transformer.layers: + # for block_i, block in enumerate(boosted_vit.blocks): + for block_i, block in enumerate(boosted_vit.model.transformer.h): + # attn = block.attn + # ff = block.mlp + + ff_0 = get_module(block, f'mlp.c_fc') + # ff_0_unpruned_indexes = get_unpruned_indexes_from_channel_attn(ff_0.cached_channel_attention, k) + ff_0_pruned_indexes = ff_0.k_takes_all.cached_i[0].sort()[0] + ff_0_unpruned_indexes = torch.LongTensor([ii for ii in range(ff_0.cached_channel_attention.size(1)) if ii not in ff_0_pruned_indexes]) + new_ff_0 = nn.Linear(ff_0.linear.in_features, ff_0_unpruned_indexes.size(0), ff_0.linear.bias is not None) + new_ff_0.weight.data.copy_(ff_0.linear.weight.data[ff_0_unpruned_indexes]) + if ff_0.linear.bias is not None: + new_ff_0.bias.data.copy_(ff_0.linear.bias.data[ff_0_unpruned_indexes]) + set_module(block, 'mlp.c_fc', nn.Sequential(new_ff_0, StaticFBS(ff_0.cached_channel_attention[:, ff_0_unpruned_indexes]))) + + ff_1 = get_module(block, f'mlp.c_proj') + new_ff_1 = nn.Linear(ff_0_unpruned_indexes.size(0), ff_1.out_features, ff_1.bias is not None) + new_ff_1.weight.data.copy_(ff_1.weight.data[:, ff_0_unpruned_indexes]) + if ff_1.bias is not None: + new_ff_1.bias.data.copy_(ff_1.bias.data) + set_module(block, 'mlp.c_proj', new_ff_1) + + unpruned_indexes_of_layers[f'model.transformer.h.{block_i}.mlp.c_fc.0.weight'] = ff_0_unpruned_indexes + + surrogate_dnn = boosted_vit + surrogate_dnn.eval() + surrogate_dnn = surrogate_dnn.to(get_model_device(master_dnn)) + # logger.debug(surrogate_dnn) + with torch.no_grad(): + surrogate_dnn_output = surrogate_dnn(**sample) + + output_diff = ((surrogate_dnn_output - master_dnn_output) ** 2).sum() + # assert output_diff < 1e-4, output_diff + logger.info(f'output diff of master and surrogate DNN: {output_diff}') + logger.debug(f'example output of master/surrogate: {master_dnn_output.sum(0)[0: 10]}, {surrogate_dnn_output.sum(0)[0: 10]}') + # logger.info(f'\nonly prune mlp!!!!\n') + # logger.info(f'\nonly prune mlp!!!!\n') + + if return_detail: + return boosted_vit, unpruned_indexes_of_layers + + return boosted_vit + + def extract_surrogate_dnn_via_samples_with_perf_test(self, master_dnn: nn.Module, samples: torch.Tensor, return_detail=False): + master_dnn_size = get_model_size(master_dnn, True) + master_dnn_latency = self._get_model_latency(master_dnn, samples, 50, + get_model_device(master_dnn), 50, False) + + res = self.extract_surrogate_dnn_via_samples(master_dnn, samples, return_detail) + if not return_detail: + surrogate_dnn = res + else: + surrogate_dnn, unpruned_indexes_of_layers = res + surrogate_dnn_size = get_model_size(surrogate_dnn, True) + surrogate_dnn_latency = self._get_model_latency(master_dnn, samples, 50, + get_model_device(master_dnn), 50, False) + + logger.info(f'master DNN ({master_dnn_size:.3f}MB, {master_dnn_latency:.4f}s/sample) -> ' + f'surrogate DNN ({surrogate_dnn_size:.3f}MB, {surrogate_dnn_latency:.4f}s/sample)\n' + f'(model size: ↓ {(master_dnn_size / surrogate_dnn_size):.2f}x, ' + f'latency: ↓ {(master_dnn_latency / surrogate_dnn_latency):.2f}x)') + + return res + + def _get_model_latency(self, model: torch.nn.Module, model_input_size, sample_num: int, + device: str, warmup_sample_num: int, return_detail=False): + import time + + if isinstance(model_input_size, tuple): + dummy_input = torch.rand(model_input_size).to(device) + else: + dummy_input = model_input_size + + model = model.to(device) + model.eval() + + # warm up + with torch.no_grad(): + for _ in range(warmup_sample_num): + model(**dummy_input) + + infer_time_list = [] + + if device == 'cuda' or 'cuda' in str(device): + with torch.no_grad(): + for _ in range(sample_num): + s, e = torch.cuda.Event(enable_timing=True), torch.cuda.Event(enable_timing=True) + s.record() + model(**dummy_input) + e.record() + torch.cuda.synchronize() + cur_model_infer_time = s.elapsed_time(e) / 1000. + infer_time_list += [cur_model_infer_time] + + else: + with torch.no_grad(): + for _ in range(sample_num): + start = time.time() + model(**dummy_input) + cur_model_infer_time = time.time() - start + infer_time_list += [cur_model_infer_time] + + avg_infer_time = sum(infer_time_list) / sample_num + + if return_detail: + return avg_infer_time, infer_time_list + return avg_infer_time \ No newline at end of file diff --git a/new_impl/nlp/mobilebert/sentiment_classification/bert.py b/new_impl/nlp/mobilebert/sentiment_classification/bert.py new file mode 100644 index 0000000000000000000000000000000000000000..170e690ece42e588515a154dc8db6ba6f7761d62 --- /dev/null +++ b/new_impl/nlp/mobilebert/sentiment_classification/bert.py @@ -0,0 +1,545 @@ +from transformers import AutoModel, AutoConfig +from utils.dl.common.model import set_module +from torch import nn +import torch +from utils.common.log import logger +from copy import deepcopy +from einops.layers.torch import Rearrange + +from methods.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from methods.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util, LoRA +from utils.common.log import logger +from utils.dl.common.model import set_module, get_module, get_super_module +from utils.dl.common.model import get_model_device, get_model_latency, get_model_size +from utils.common.log import logger + +from transformers.models.mobilebert.modeling_mobilebert import MobileBertSelfAttention +from methods.elasticdnn.model.base import Abs, KTakesAll, ElasticDNNUtil, Layer_WrappedWithFBS +from typing import Optional, Tuple +import math +import os + +bert_model_tag = f'{os.path.dirname(__file__)}/mobilebert-uncased' + +class BertForSenCls(nn.Module): + def __init__(self, num_classes): + super(BertForSenCls, self).__init__() + + logger.info(f'init bert for sen cls (using {bert_model_tag})') + self.bert = AutoModel.from_pretrained(bert_model_tag) + self.classifier = nn.Linear(512, num_classes) + + def forward(self, **x): + x['return_dict'] = False + pool_output = self.bert(**x)[-1] + out = self.classifier(pool_output) + + return out + +class BertSelfAttentionPrunable(MobileBertSelfAttention): + def __init__(self): + config = AutoConfig.from_pretrained(bert_model_tag) + super(BertSelfAttentionPrunable, self).__init__(config) + + def transpose_for_scores(self, x): + new_x_shape = x.size()[:-1] + (self.num_attention_heads, -1) + x = x.view(new_x_shape) + return x.permute(0, 2, 1, 3) + + def forward( + self, + query_tensor, + key_tensor, + value_tensor, + attention_mask=None, + head_mask=None, + output_attentions=None, + ): + mixed_query_layer = self.query(query_tensor) + mixed_key_layer = self.key(key_tensor) + mixed_value_layer = self.value(value_tensor) + + query_layer = self.transpose_for_scores(mixed_query_layer) + key_layer = self.transpose_for_scores(mixed_key_layer) + value_layer = self.transpose_for_scores(mixed_value_layer) + + # Take the dot product between "query" and "key" to get the raw attention scores. + attention_scores = torch.matmul(query_layer, key_layer.transpose(-1, -2)) + attention_scores = attention_scores / math.sqrt(self.attention_head_size) + if attention_mask is not None: + # Apply the attention mask is (precomputed for all layers in BertModel forward() function) + attention_scores = attention_scores + attention_mask + # Normalize the attention scores to probabilities. + attention_probs = nn.Softmax(dim=-1)(attention_scores) + # This is actually dropping out entire tokens to attend to, which might + # seem a bit unusual, but is taken from the original Transformer paper. + attention_probs = self.dropout(attention_probs) + # Mask heads if we want to + if head_mask is not None: + attention_probs = attention_probs * head_mask + context_layer = torch.matmul(attention_probs, value_layer) + context_layer = context_layer.permute(0, 2, 1, 3).contiguous() + new_context_layer_shape = context_layer.size()[:-2] + (-1,) + context_layer = context_layer.view(*new_context_layer_shape) + outputs = (context_layer, attention_probs) if output_attentions else (context_layer,) + return outputs + + @staticmethod + def init_from_exist_self_attn(attn: MobileBertSelfAttention): + # print(attn) + + res = BertSelfAttentionPrunable() + + for attr in dir(attn): + # if str(attr) in ['transpose_for_scores'] or str(attr).startswith('_'): + # continue + # if isinstance(getattr(attn, attr), nn.Module): + # print(attr) + + if isinstance(getattr(attn, attr), nn.Module): + try: + # print(attr, 'ok') + setattr(res, attr, getattr(attn, attr)) + + except Exception as e: + print(attr, str(e)) + + + + return res + +class FM_to_MD_Bert_Util(FM_to_MD_Util): + def init_md_from_fm_by_reducing_width(self, fm: nn.Module, reducing_width_ratio: int) -> nn.Module: + fm_vit = deepcopy(fm) + + for block in fm_vit.bert.encoder.layer: + set_module(block, 'attention.self', BertSelfAttentionPrunable.init_from_exist_self_attn(block.attention.self)) + + def _f(n): + return int(n // reducing_width_ratio) + + # def _rand_indexes(n): + # return torch.randperm(n)[0: int(n // reducing_width_ratio)] + + def l1_max_indexes(p: torch.Tensor, dim=0): + assert dim in [0, 1] + assert p.dim() in [1, 2, 4] + + if dim == 1: + p = p.T + + p_norm = p.abs().contiguous().view(p.size(0), -1).sum(dim=1) + n = p.size(0) + return p_norm.argsort(descending=True)[0: int(n // reducing_width_ratio)].sort()[0] + + for block_i, block in enumerate(fm_vit.bert.encoder.layer): + for k in ['query', 'key', 'value']: + qkv = get_module(block, f'attention.self.{k}') + + new_qkv = nn.Linear(qkv.in_features, _f(qkv.out_features), + qkv.bias is not None, qkv.weight.device) + indexes = l1_max_indexes(qkv.weight.data, 0) + + new_qkv.weight.data.copy_(qkv.weight.data[indexes]) + if qkv.bias is not None: + new_qkv.bias.data.copy_(qkv.bias.data[indexes]) + set_module(block, f'attention.self.{k}', new_qkv) + + proj = get_module(block, f'attention.output.dense') + new_proj = nn.Linear(_f(proj.in_features), proj.out_features, + proj.bias is not None, proj.weight.device) + new_proj.weight.data.copy_(proj.weight.data[:, l1_max_indexes(proj.weight.data, 1)]) + if proj.bias is not None: + new_proj.bias.data.copy_(proj.bias.data) + set_module(block, f'attention.output.dense', new_proj) + + fc1 = get_module(block, f'intermediate.dense') + new_fc1 = nn.Linear(fc1.in_features, _f(fc1.out_features), + fc1.bias is not None, fc1.weight.device) + indexes = l1_max_indexes(fc1.weight.data, 0) + new_fc1.weight.data.copy_(fc1.weight.data[indexes]) + if fc1.bias is not None: + new_fc1.bias.data.copy_(fc1.bias.data[indexes]) + set_module(block, f'intermediate.dense', new_fc1) + + fc2 = get_module(block, f'output.dense') + new_fc2 = nn.Linear(_f(fc2.in_features), fc2.out_features, + fc2.bias is not None, fc2.weight.device) + new_fc2.weight.data.copy_(fc2.weight.data[:, l1_max_indexes(fc2.weight.data, 1)]) + if fc2.bias is not None: + new_fc2.bias.data.copy_(fc2.bias.data) + set_module(block, f'output.dense', new_fc2) + + return fm_vit + + def init_md_from_fm_by_reducing_width_with_perf_test(self, fm: nn.Module, reducing_width_ratio: int, + samples: torch.Tensor) -> nn.Module: + fm_size = get_model_size(fm, True) + fm_latency = self._get_model_latency(fm, samples, 20, + get_model_device(fm), 20, False) + + master_dnn = self.init_md_from_fm_by_reducing_width(fm, reducing_width_ratio) + master_dnn_size = get_model_size(master_dnn, True) + logger.debug(f'inited master DNN: {master_dnn}') + master_dnn_latency = self._get_model_latency(master_dnn, samples, 20, + get_model_device(master_dnn), 20, False) + + logger.info(f'init master DNN (w/o FBS yet) by reducing foundation model\'s width (by {reducing_width_ratio:d}x)') + logger.info(f'foundation model ({fm_size:.3f}MB, {fm_latency:.4f}s/sample) -> ' + f'master DNN ({master_dnn_size:.3f}MB, {master_dnn_latency:.4f}s/sample)\n' + f'(model size: ↓ {(fm_size / master_dnn_size):.2f}x, ' + f'latency: ↓ {(fm_latency / master_dnn_latency):.2f}x)') + + return master_dnn + + def _get_model_latency(self, model: torch.nn.Module, model_input_size, sample_num: int, + device: str, warmup_sample_num: int, return_detail=False): + import time + + if isinstance(model_input_size, tuple): + dummy_input = torch.rand(model_input_size).to(device) + else: + dummy_input = model_input_size + + model = model.to(device) + model.eval() + + # warm up + with torch.no_grad(): + for _ in range(warmup_sample_num): + model(**dummy_input) + + infer_time_list = [] + + if device == 'cuda' or 'cuda' in str(device): + with torch.no_grad(): + for _ in range(sample_num): + s, e = torch.cuda.Event(enable_timing=True), torch.cuda.Event(enable_timing=True) + s.record() + model(**dummy_input) + e.record() + torch.cuda.synchronize() + cur_model_infer_time = s.elapsed_time(e) / 1000. + infer_time_list += [cur_model_infer_time] + + else: + with torch.no_grad(): + for _ in range(sample_num): + start = time.time() + model(**dummy_input) + cur_model_infer_time = time.time() - start + infer_time_list += [cur_model_infer_time] + + avg_infer_time = sum(infer_time_list) / sample_num + + if return_detail: + return avg_infer_time, infer_time_list + return avg_infer_time + +class SqueezeLast(nn.Module): + def __init__(self): + super(SqueezeLast, self).__init__() + + def forward(self, x): + return x.squeeze(-1) + +class Linear_WrappedWithFBS(Layer_WrappedWithFBS): + def __init__(self, linear: nn.Linear, r): + super(Linear_WrappedWithFBS, self).__init__() + + self.linear = linear + + # for conv: (B, C_in, H, W) -> (B, C_in) -> (B, C_out) + # for mlp in ViT: (B, #patches, D: dim of patches embedding) -> (B, D) -> (B, C_out) + self.fbs = nn.Sequential( + Rearrange('b n d -> b d n'), + Abs(), + nn.AdaptiveAvgPool1d(1), + SqueezeLast(), + nn.Linear(linear.in_features, linear.out_features // r), + nn.ReLU(), + nn.Linear(linear.out_features // r, linear.out_features), + nn.ReLU() + ) + self.ln = nn.LayerNorm(linear.out_features) + nn.init.constant_(self.fbs[6].bias, 1.) + nn.init.kaiming_normal_(self.fbs[6].weight) + + + def forward(self, x): + if self.use_cached_channel_attention and self.cached_channel_attention is not None: + channel_attention = self.cached_channel_attention + else: + self.cached_raw_channel_attention = self.fbs(x) + self.cached_channel_attention = self.k_takes_all(self.cached_raw_channel_attention) + + channel_attention = self.cached_channel_attention + + raw_res = self.linear(x) + res = channel_attention.unsqueeze(1) * raw_res + res = self.ln(res) + return res + +class ToQKV_WrappedWithLoRA(nn.Module): + def __init__(self, fc: nn.Linear, ab_r: int): + super(ToQKV_WrappedWithLoRA, self).__init__() + + self.fc = fc + self.ab = self.create_ab_as_linear(fc.weight.data, ab_r) + + def create_ab_as_linear(self, fc_weight: torch.Tensor, ab_r: int): + res = nn.Sequential( + LoRA(fc_weight.size(1), fc_weight.size(0) // ab_r, bias=False), + LoRA(fc_weight.size(0) // ab_r, fc_weight.size(0), bias=False) + ).to(fc_weight.device) + nn.init.kaiming_uniform_(res[0].weight, a=5 ** 0.5) + nn.init.zeros_(res[1].weight) + return res + + def forward(self, x): + x1 = self.fc(x) + x2 = self.ab(x) + return x1 + x2 + + +class FMLoRA_Bert_Util(FMLoRA_Util): + + @torch.no_grad() + def add_lora_ab_to_fm(self, fm: nn.Module, ab_r: int, samples: dict): + fm.eval() + + o1 = fm(**samples) + + for name, module in fm.named_modules(): + if name.endswith(('query', 'key', 'value')): + set_module(fm, name, ToQKV_WrappedWithLoRA(module, ab_r)) + + o2 = fm(**samples) + + if isinstance(o1, tuple): + o1 = o1[-1] + o2 = o2[-1] + output_diff = ((o1 - o2) ** 2).sum() + assert output_diff < 1e-5 + + return fm + + @torch.no_grad() + def absorb_lora_and_recover_net_structure(self, fm: nn.Module, samples: dict): + fm.eval() + # print('absorb lora before') + o1 = fm(**samples) + + for name, module in fm.named_modules(): + if not isinstance(module, ToQKV_WrappedWithLoRA): + continue + + fc = module.fc + ab = module.ab + + fc.weight.add_(ab[1].weight @ ab[0].weight) + + set_module(fm, name, fc) + + # print('absorb lora after') + o2 = fm(**samples) + + if isinstance(o1, tuple): + o1 = o1[-1] + o2 = o2[-1] + output_diff = ((o1 - o2) ** 2).sum() + assert output_diff < 1e-6, output_diff + + return fm + +class StaticFBS(nn.Module): + def __init__(self, static_channel_attention): + super(StaticFBS, self).__init__() + assert static_channel_attention.dim() == 2 and static_channel_attention.size(0) == 1 + self.static_channel_attention = nn.Parameter(static_channel_attention, requires_grad=False) # (1, dim) + + def forward(self, x): + # print('staticfbs', x, self.static_channel_attention.unsqueeze(1)) + return x * self.static_channel_attention.unsqueeze(1) + +class ElasticBertUtil(ElasticDNNUtil): + def convert_raw_dnn_to_master_dnn(self, raw_dnn: nn.Module, r: float, ignore_layers=[]): + assert len(ignore_layers) == 0, 'not supported yet' + + raw_vit = deepcopy(raw_dnn) + + # set_module(module, 'patch_embed.proj', ProjConv_WrappedWithFBS(module.patch_embed.proj, r)) + + for name, module in raw_vit.named_modules(): + # if name.endswith('attn'): + # set_module(module, 'qkv', ToQKV_WrappedWithFBS(module.qkv, r)) + if name.endswith('intermediate'): + set_module(module, 'dense', Linear_WrappedWithFBS(module.dense, r)) + + return raw_vit + + def set_master_dnn_sparsity(self, master_dnn: nn.Module, sparsity: float): + # for name, module in master_dnn.named_modules(): + # if not name.endswith('attn'): + # continue + + # q_features = module.qkv.to_qk.out_features // 2 + + # if (q_features - int(q_features * sparsity)) % module.num_heads != 0: + # # tune sparsity to ensure #unpruned channel % num_heads == 0 + # # so that the pruning seems to reduce the dim_head of each head + # tuned_sparsity = 1. - int((q_features - int(q_features * sparsity)) / module.num_heads) * module.num_heads / q_features + # logger.debug(f'tune sparsity from {sparsity:.2f} to {tuned_sparsity}') + # sparsity = tuned_sparsity + # break + + return super().set_master_dnn_sparsity(master_dnn, sparsity) + + def select_most_rep_sample(self, master_dnn: nn.Module, samples: torch.Tensor): + # print(samples) + # return samples[0].unsqueeze(0) + res = {k: v[0: 1] for k, v in samples.items()} + return res + + def extract_surrogate_dnn_via_samples(self, master_dnn: nn.Module, samples: torch.Tensor, return_detail=False): + sample = self.select_most_rep_sample(master_dnn, samples) + # assert sample.dim() == 4 and sample.size(0) == 1 + + # print('before') + master_dnn.eval() + self.clear_cached_channel_attention_in_master_dnn(master_dnn) + with torch.no_grad(): + master_dnn_output = master_dnn(**sample) + + # print('after') + + boosted_vit = deepcopy(master_dnn) + + def get_unpruned_indexes_from_channel_attn(channel_attn: torch.Tensor, k): + assert channel_attn.size(0) == 1, 'use A representative sample to generate channel attentions' + + # print('attn_in_unpruned', channel_attn[0][0: 10]) + + res = channel_attn[0].nonzero(as_tuple=True)[0] # should be one-dim + + # res = channel_attn[0].argsort(descending=True)[0: -int(channel_attn.size(1) * k)].sort()[0] + + # g = channel_attn + # k = g.size(1) - int(g.size(1) * k) + # res = g.topk(k, 1)[1][0].sort()[0] + + return res + + unpruned_indexes_of_layers = {} + + # for attn, ff in boosted_vit.transformer.layers: + # for block_i, block in enumerate(boosted_vit.blocks): + for block_i, block in enumerate(boosted_vit.bert.encoder.layer): + # attn = block.attn + # ff = block.mlp + + ff_0 = get_module(block, f'intermediate.dense') + # ff_0_unpruned_indexes = get_unpruned_indexes_from_channel_attn(ff_0.cached_channel_attention, k) + ff_0_pruned_indexes = ff_0.k_takes_all.cached_i[0].sort()[0] + ff_0_unpruned_indexes = torch.LongTensor([ii for ii in range(ff_0.cached_channel_attention.size(1)) if ii not in ff_0_pruned_indexes]) + new_ff_0 = nn.Linear(ff_0.linear.in_features, ff_0_unpruned_indexes.size(0), ff_0.linear.bias is not None) + new_ff_0.weight.data.copy_(ff_0.linear.weight.data[ff_0_unpruned_indexes]) + if ff_0.linear.bias is not None: + new_ff_0.bias.data.copy_(ff_0.linear.bias.data[ff_0_unpruned_indexes]) + set_module(block, 'intermediate.dense', nn.Sequential(new_ff_0, StaticFBS(ff_0.cached_channel_attention[:, ff_0_unpruned_indexes]))) + + ff_1 = get_module(block, f'output.dense') + new_ff_1 = nn.Linear(ff_0_unpruned_indexes.size(0), ff_1.out_features, ff_1.bias is not None) + new_ff_1.weight.data.copy_(ff_1.weight.data[:, ff_0_unpruned_indexes]) + if ff_1.bias is not None: + new_ff_1.bias.data.copy_(ff_1.bias.data) + set_module(block, 'output.dense', new_ff_1) + + unpruned_indexes_of_layers[f'bert.encoder.layer.{block_i}.intermediate.dense.0.weight'] = ff_0_unpruned_indexes + + surrogate_dnn = boosted_vit + surrogate_dnn.eval() + surrogate_dnn = surrogate_dnn.to(get_model_device(master_dnn)) + # logger.debug(surrogate_dnn) + with torch.no_grad(): + surrogate_dnn_output = surrogate_dnn(**sample) + + output_diff = ((surrogate_dnn_output - master_dnn_output) ** 2).sum() + # assert output_diff < 1e-4, output_diff + logger.info(f'output diff of master and surrogate DNN: {output_diff}') + logger.debug(f'example output of master/surrogate: {master_dnn_output.sum(0)[0: 10]}, {surrogate_dnn_output.sum(0)[0: 10]}') + # logger.info(f'\nonly prune mlp!!!!\n') + # logger.info(f'\nonly prune mlp!!!!\n') + + if return_detail: + return boosted_vit, unpruned_indexes_of_layers + + return boosted_vit + + def extract_surrogate_dnn_via_samples_with_perf_test(self, master_dnn: nn.Module, samples: torch.Tensor, return_detail=False): + master_dnn_size = get_model_size(master_dnn, True) + master_dnn_latency = self._get_model_latency(master_dnn, samples, 50, + get_model_device(master_dnn), 50, False) + + res = self.extract_surrogate_dnn_via_samples(master_dnn, samples, return_detail) + if not return_detail: + surrogate_dnn = res + else: + surrogate_dnn, unpruned_indexes_of_layers = res + surrogate_dnn_size = get_model_size(surrogate_dnn, True) + surrogate_dnn_latency = self._get_model_latency(master_dnn, samples, 50, + get_model_device(master_dnn), 50, False) + + logger.info(f'master DNN ({master_dnn_size:.3f}MB, {master_dnn_latency:.4f}s/sample) -> ' + f'surrogate DNN ({surrogate_dnn_size:.3f}MB, {surrogate_dnn_latency:.4f}s/sample)\n' + f'(model size: ↓ {(master_dnn_size / surrogate_dnn_size):.2f}x, ' + f'latency: ↓ {(master_dnn_latency / surrogate_dnn_latency):.2f}x)') + + return res + + def _get_model_latency(self, model: torch.nn.Module, model_input_size, sample_num: int, + device: str, warmup_sample_num: int, return_detail=False): + import time + + if isinstance(model_input_size, tuple): + dummy_input = torch.rand(model_input_size).to(device) + else: + dummy_input = model_input_size + + model = model.to(device) + model.eval() + + # warm up + with torch.no_grad(): + for _ in range(warmup_sample_num): + model(**dummy_input) + + infer_time_list = [] + + if device == 'cuda' or 'cuda' in str(device): + with torch.no_grad(): + for _ in range(sample_num): + s, e = torch.cuda.Event(enable_timing=True), torch.cuda.Event(enable_timing=True) + s.record() + model(**dummy_input) + e.record() + torch.cuda.synchronize() + cur_model_infer_time = s.elapsed_time(e) / 1000. + infer_time_list += [cur_model_infer_time] + + else: + with torch.no_grad(): + for _ in range(sample_num): + start = time.time() + model(**dummy_input) + cur_model_infer_time = time.time() - start + infer_time_list += [cur_model_infer_time] + + avg_infer_time = sum(infer_time_list) / sample_num + + if return_detail: + return avg_infer_time, infer_time_list + return avg_infer_time + +def bert_base_sen_cls(num_classes): + return BertForSenCls(num_classes) \ No newline at end of file diff --git a/new_impl/nlp/mobilebert/sentiment_classification/cls.py b/new_impl/nlp/mobilebert/sentiment_classification/cls.py new file mode 100644 index 0000000000000000000000000000000000000000..84245cc731690ab665053b240d18cb150bf6ba14 --- /dev/null +++ b/new_impl/nlp/mobilebert/sentiment_classification/cls.py @@ -0,0 +1,123 @@ +import torch +from methods.elasticdnn.api.model import ElasticDNN_OfflineSenClsFMModel, ElasticDNN_OfflineSenClsMDModel +from methods.elasticdnn.api.algs.fm_lora import ElasticDNN_FMLoRAAlg +from methods.elasticdnn.api.algs.md_pretraining_wo_fbs import ElasticDNN_MDPretrainingWoFBSAlg +from methods.elasticdnn.model.base import ElasticDNNUtil +from methods.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from bert import FMLoRA_Bert_Util +from methods.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from bert import FM_to_MD_Bert_Util +from bert import ElasticBertUtil +from utils.dl.common.model import LayerActivation, get_module, get_parameter, set_module +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +from utils.common.log import logger +import torch.nn.functional as F +import sys + +class ElasticDNN_BERT_OfflineClsFMModel(ElasticDNN_OfflineSenClsFMModel): + def generate_md_by_reducing_width(self, reducing_width_ratio, samples: torch.Tensor): + return FM_to_MD_Bert_Util().init_md_from_fm_by_reducing_width_with_perf_test(self.models_dict['main'], + reducing_width_ratio, samples) + + def get_feature_hook(self) -> LayerActivation: + return LayerActivation(get_module(self.models_dict['main'], 'classifier'), True, self.device) + + def get_elastic_dnn_util(self) -> ElasticDNNUtil: + return ElasticBertUtil() + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + self.to_train_mode() + pred = self.infer(x) + + return F.cross_entropy(pred, y) + + def get_lora_util(self) -> FMLoRA_Util: + return FMLoRA_Bert_Util() + + def get_task_head_params(self): + head = get_module(self.models_dict['main'], 'classifier') + params_name = {k for k, v in head.named_parameters()} + logger.info(f'task head params: {params_name}') + return list(head.parameters()) + +class ElasticDNN_BERT_OfflineClsMDModel(ElasticDNN_OfflineSenClsMDModel): + def get_feature_hook(self) -> LayerActivation: + return LayerActivation(get_module(self.models_dict['main'], 'classifier'), True, self.device) + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + self.to_train_mode() + return self.models_dict['main'](x, y)['total_loss'] + +if __name__ == '__main__': + from utils.dl.common.env import set_random_seed + set_random_seed(1) + # 1. init scenario + # scenario = build_scenario( + # source_datasets_name=['WI_Mask'], + # target_datasets_order=['MakeML_Mask'] * 10, + # da_mode='close_set', + # data_dirs={ + # 'COCO2017': '/data/zql/datasets/coco2017', + # 'WI_Mask': '/data/zql/datasets/face_mask/WI/Medical mask/Medical mask/Medical Mask/images', + # 'VOC2012': '/data/datasets/VOCdevkit/VOC2012/JPEGImages', + # 'MakeML_Mask': '/data/zql/datasets/face_mask/make_ml/images' + # }, + # ) + scenario = build_scenario( + source_datasets_name=['HL5Domains-ApexAD2600Progressive', 'HL5Domains-CanonG3', 'HL5Domains-CreativeLabsNomadJukeboxZenXtra40GB', + 'HL5Domains-NikonCoolpix4300'], + target_datasets_order=['HL5Domains-Nokia6610'] * 1, # TODO + da_mode='close_set', + data_dirs={ + **{k: f'/data/zql/datasets/nlp_asc_19_domains/dat/absa/Bing5Domains/asc/{k.split("-")[1]}' + for k in ['HL5Domains-ApexAD2600Progressive', 'HL5Domains-CanonG3', 'HL5Domains-CreativeLabsNomadJukeboxZenXtra40GB', + 'HL5Domains-NikonCoolpix4300', 'HL5Domains-Nokia6610']} + }, + ) + + # 2. init model + device = 'cuda' + from bert import bert_base_sen_cls + cls_model = bert_base_sen_cls(num_classes=scenario.num_classes) + # x = {'input_ids': torch.tensor([[ 101, 5672, 2033, 2011, 2151, 3793, 2017, 1005, 1040, 2066, 1012, 102]]).to(device), + # 'token_type_ids': torch.tensor([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]).to(device), + # 'attention_mask': torch.tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]).to(device), + # 'return_dict': False} + # print(cls_model(x)) + fm_models_dict_path = save_models_dict_for_init({ + 'main': cls_model + }, __file__, 'fm_bert_pretrained_with_cls_head') + + fm_model = ElasticDNN_BERT_OfflineClsFMModel('fm', fm_models_dict_path, device) + # 3. init alg + models = { + 'fm': fm_model + } + fm_lora_alg = ElasticDNN_FMLoRAAlg(models, get_res_save_dir(__file__, 'result')) + + from PIL import ImageFile + ImageFile.LOAD_TRUNCATED_IMAGES = True + + # 4. run alg + from utils.dl.common.lr_scheduler import get_linear_schedule_with_warmup + fm_lora_alg.run(scenario, hyps={ + 'launch_tbboard': False, + + 'samples_size': {'input_ids': torch.tensor([[ 101, 5672, 2033, 2011, 2151, 3793, 2017, 1005, 1040, 2066, 1012, 102]]).to(device), + 'token_type_ids': torch.tensor([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]).to(device), + 'attention_mask': torch.tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]).to(device), 'return_dict': False}, + + 'ab_r': 8, + 'train_batch_size': 8, + 'val_batch_size': 16, + 'num_workers': 16, + 'optimizer': 'AdamW', + 'optimizer_args': {'lr': 1e-4, 'betas': [0.9, 0.999]}, + 'scheduler': 'LambdaLR', + 'scheduler_args': {'lr_lambda': get_linear_schedule_with_warmup(10000, 310000)}, + 'num_iters': 50000, + 'val_freq': 400, + + # 'fm_lora_ckpt_path': 'experiments/elasticdnn/vit_b_16/offline/fm_lora/cls/results/cls.py/20230607/999995-234355-trial/models/fm_best.pt' + }) \ No newline at end of file diff --git a/new_impl/nlp/mobilebert/sentiment_classification/cls_md_w_fbs_index.py b/new_impl/nlp/mobilebert/sentiment_classification/cls_md_w_fbs_index.py new file mode 100644 index 0000000000000000000000000000000000000000..433a07d6be0851209d9e3fc1e7826f81e7ce1f5e --- /dev/null +++ b/new_impl/nlp/mobilebert/sentiment_classification/cls_md_w_fbs_index.py @@ -0,0 +1,189 @@ +import torch +import sys +from torch import nn +from methods.elasticdnn.api.model import ElasticDNN_OfflineSenClsFMModel, ElasticDNN_OfflineSenClsMDModel +from methods.elasticdnn.api.algs.md_pretraining_wo_fbs import ElasticDNN_MDPretrainingWoFBSAlg +from methods.elasticdnn.model.base import ElasticDNNUtil +from methods.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from bert import FM_to_MD_Bert_Util +from methods.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from bert import FMLoRA_Bert_Util +from bert import ElasticBertUtil +from utils.dl.common.model import LayerActivation2, get_module, get_parameter +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +from utils.dl.common.loss import CrossEntropyLossSoft +import torch.nn.functional as F +from utils.common.log import logger + + +class ElasticDNN_Bert_OfflineSenClsFMModel(ElasticDNN_OfflineSenClsFMModel): + def generate_md_by_reducing_width(self, reducing_width_ratio, samples: torch.Tensor): # TODO: + return FM_to_MD_Bert_Util().init_md_from_fm_by_reducing_width_with_perf_test(self.models_dict['main'], + reducing_width_ratio, samples) + # raise NotImplementedError + + def get_feature_hook(self) -> LayerActivation2: + return LayerActivation2(get_module(self.models_dict['main'], 'classifier')) + + def get_elastic_dnn_util(self) -> ElasticDNNUtil: # TODO: + return ElasticBertUtil() + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + self.to_train_mode() + pred = self.infer(x) + out = F.cross_entropy(pred, y) + # print(out.item()) + return out + + def get_lora_util(self) -> FMLoRA_Util: + return FMLoRA_Bert_Util() + + def get_task_head_params(self): + head = get_module(self.models_dict['main'], 'classifier') + params_name = {k for k, v in head.named_parameters()} + logger.info(f'task head params: {params_name}') + return list(head.parameters()) + + +class ElasticDNN_Bert_OfflineSenClsMDModel(ElasticDNN_OfflineSenClsMDModel): + def __init__(self, name: str, models_dict_path: str, device: str): + super().__init__(name, models_dict_path, device) + + self.distill_criterion = CrossEntropyLossSoft() + + def get_feature_hook(self) -> LayerActivation2: + return LayerActivation2(get_module(self.models_dict['main'], 'classifier')) + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + self.to_train_mode() + pred = self.infer(x) + out = F.cross_entropy(pred, y) + # print(out.item()) + return out + + def get_distill_loss(self, student_output, teacher_output): + # print(student_output, teacher_output) + return self.distill_criterion(student_output, teacher_output) + + def get_matched_param_of_fm(self, self_param_name, fm: nn.Module): # TODO: + if any([k in self_param_name for k in ['fbs', 'ab', 'embeddings', 'ln']]): + return None + + p = get_parameter(self.models_dict['main'], self_param_name) + if p.dim() == 0: + return None + + elif p.dim() == 1 and 'LayerNorm' in self_param_name and 'weight' in self_param_name: + # if self_param_name.startswith('norm'): + # return None + return get_parameter(fm, self_param_name) + + # 1. xx.query.weight -> xx.query.fc.weight and xx.query.ab.0/1 + if ('query' in self_param_name or 'key' in self_param_name or \ + 'value' in self_param_name) and ('weight' in self_param_name): + ss = self_param_name.split('.') + # raise NotImplementedError() # TODO: + fm_qkv_name = '.'.join(ss[0: -1]) + '.fc' + fm_qkv = get_module(fm, fm_qkv_name) + + fm_abs_name = '.'.join(ss[0: -1]) + '.ab' + fm_abs = get_module(fm, fm_abs_name) + + return torch.cat([ + fm_qkv.weight.data, # task-agnositc params + fm_abs[1].weight @ fm_abs[0].weight + ], dim=0) + + elif ('query' in self_param_name or 'key' in self_param_name or \ + 'value' in self_param_name) and ('bias' in self_param_name): + ss = self_param_name.split('.') + + fm_qkv_name = '.'.join(ss[0: -1]) + '.fc.bias' + return get_parameter(fm, fm_qkv_name) + + elif 'intermediate.dense' in self_param_name: + fm_param_name = self_param_name.replace('.linear', '') + return get_parameter(fm, fm_param_name) + + else: + return get_parameter(fm, self_param_name) + + + +if __name__ == '__main__': + from utils.dl.common.env import set_random_seed + set_random_seed(1) + + # 3. init scenario + scenario = build_scenario( + source_datasets_name=['HL5Domains-ApexAD2600Progressive', 'HL5Domains-CanonG3', 'HL5Domains-CreativeLabsNomadJukeboxZenXtra40GB', + 'HL5Domains-NikonCoolpix4300'], + target_datasets_order=['HL5Domains-Nokia6610'] * 1, # TODO + da_mode='close_set', + data_dirs={ + **{k: f'/data/zql/datasets/nlp_asc_19_domains/dat/absa/Bing5Domains/asc/{k.split("-")[1]}' + for k in ['HL5Domains-ApexAD2600Progressive', 'HL5Domains-CanonG3', 'HL5Domains-CreativeLabsNomadJukeboxZenXtra40GB', + 'HL5Domains-NikonCoolpix4300', 'HL5Domains-Nokia6610']} + }, + ) + + # 1. init model + # from dnns.deeplabv3.head import modify_forward_head + # modify_forward_head() # TODO: bring a bug + # from dnns.vit import vit_b_16 + fm_models_dict_path = 'new_impl/nlp/mobilebert/sentiment_classification/results/cls_md_wo_fbs.py/20231019/999999-123150/models/fm_best.pt' + fm_models = torch.load(fm_models_dict_path) + fm_models_dict_path = save_models_dict_for_init(fm_models, __file__, 'fm_bert_base_secls_lora') + md_models_dict_path = save_models_dict_for_init( + torch.load('new_impl/nlp/mobilebert/sentiment_classification/results/cls_md_wo_fbs.py/20231019/999999-123150/models/md_best.pt'), + __file__, 'md_bert_base_secls_raw_pretrained') + device = 'cuda' + + fm_model = ElasticDNN_Bert_OfflineSenClsFMModel('fm', fm_models_dict_path, device) + md_model = ElasticDNN_Bert_OfflineSenClsMDModel('md', md_models_dict_path, device) + + # 2. init alg + models = { + 'fm': fm_model, + 'md': md_model + } + from methods.elasticdnn.api.algs.md_pretraining_index_v2_train_index_and_md import ElasticDNN_MDPretrainingIndexAlg + fm_to_md_alg = ElasticDNN_MDPretrainingIndexAlg(models, get_res_save_dir(__file__, 'result')) + + from utils.dl.common.lr_scheduler import get_linear_schedule_with_warmup + fm_to_md_alg.run(scenario, hyps={ + 'launch_tbboard': False, + + 'samples_size': {'input_ids': torch.tensor([[ 101, 5672, 2033, 2011, 2151, 3793, 2017, 1005, 1040, 2066, 1012, 102]]).to(device), + 'token_type_ids': torch.tensor([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]).to(device), + 'attention_mask': torch.tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]).to(device), 'return_dict': False}, + + 'FBS_r': 16, + 'FBS_ignore_layers': [], + + 'train_batch_size': 4, + 'val_batch_size': 8, + 'num_workers': 8, + 'optimizer': 'AdamW', + 'optimizer_args': {'lr': 1e-4, 'betas': [0.9, 0.999]}, + 'scheduler': 'LambdaLR', + 'scheduler_args': {'lr_lambda': get_linear_schedule_with_warmup(10000, 70000)}, + + 'indexes_optimizer_args': {'lr': 3e-3, 'momentum': 0.9, 'weight_decay': 5e-4}, + + 'num_iters': 80000, + 'val_freq': 400, + + 'max_sparsity': 0.9, + 'min_sparsity': 0.0, + 'l1_reg_loss_weight': 1e-9, + 'index_loss_weight': 1e-4, + 'val_num_sparsities': 4, + + 'bn_cal_num_iters': 0, + + 'index_init': 'zero', + 'index_guided_linear_comb_split_size': 512 + }) + \ No newline at end of file diff --git a/new_impl/nlp/mobilebert/sentiment_classification/cls_md_wo_fbs.py b/new_impl/nlp/mobilebert/sentiment_classification/cls_md_wo_fbs.py new file mode 100644 index 0000000000000000000000000000000000000000..8f366aa66fdd02ab43ba88355d87a190ee6d3812 --- /dev/null +++ b/new_impl/nlp/mobilebert/sentiment_classification/cls_md_wo_fbs.py @@ -0,0 +1,154 @@ +import torch +import sys +from torch import nn +from methods.elasticdnn.api.model import ElasticDNN_OfflineSenClsFMModel, ElasticDNN_OfflineSenClsMDModel +from methods.elasticdnn.api.algs.md_pretraining_wo_fbs import ElasticDNN_MDPretrainingWoFBSAlg +from methods.elasticdnn.model.base import ElasticDNNUtil +from methods.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from bert import FMLoRA_Bert_Util +from methods.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from bert import FM_to_MD_Bert_Util +from bert import ElasticBertUtil +from utils.dl.common.model import LayerActivation2, get_module, get_parameter +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +from utils.dl.common.loss import CrossEntropyLossSoft +import torch.nn.functional as F +from utils.common.log import logger + + +class ElasticDNN_Bert_OfflineSenClsFMModel(ElasticDNN_OfflineSenClsFMModel): + def generate_md_by_reducing_width(self, reducing_width_ratio, samples: torch.Tensor): # TODO: + tmp = FM_to_MD_Bert_Util().init_md_from_fm_by_reducing_width_with_perf_test(self.models_dict['main'], + reducing_width_ratio, samples) + return tmp + # raise NotImplementedError + + def get_feature_hook(self) -> LayerActivation2: + return LayerActivation2(get_module(self.models_dict['main'], 'classifier')) + + def get_elastic_dnn_util(self) -> ElasticDNNUtil: # TODO: + raise ElasticBertUtil() + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + self.to_train_mode() + return F.cross_entropy(self.infer(x), y) + + def get_lora_util(self) -> FMLoRA_Util: + return FMLoRA_Bert_Util() + + def get_task_head_params(self): + head = get_module(self.models_dict['main'], 'classifier') + params_name = {k for k, v in head.named_parameters()} + logger.info(f'task head params: {params_name}') + return list(head.parameters()) + + +class ElasticDNN_Bert_OfflineSenClsMDModel(ElasticDNN_OfflineSenClsMDModel): + def __init__(self, name: str, models_dict_path: str, device: str): + super().__init__(name, models_dict_path, device) + + self.distill_criterion = CrossEntropyLossSoft() + + def get_feature_hook(self) -> LayerActivation2: + return LayerActivation2(get_module(self.models_dict['main'], 'classifier')) + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + self.to_train_mode() + return F.cross_entropy(self.infer(x), y) + + def get_distill_loss(self, student_output, teacher_output): + # print(student_output, teacher_output) + return self.distill_criterion(student_output, teacher_output) + + def get_matched_param_of_fm(self, self_param_name, fm: nn.Module): # TODO: + if any([k in self_param_name for k in ['fbs', 'embeddings']]): + return None + + # 1. xx.qkv.to_qkv.yy to xx.qkv.qkv.aa and xx.qkv.abs.zz + if 'query' in self_param_name or 'key' in self_param_name or 'value' in self_param_name: + ss = self_param_name.split('.') + raise NotImplementedError() # TODO: + fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv' + fm_qkv = get_module(fm, fm_qkv_name) + + fm_abs_name = '.'.join(ss[0: -2]) + '.abs' + fm_abs = get_module(fm, fm_abs_name) + + return torch.cat([ + fm_qkv.weight.data, # task-agnositc params + torch.cat([(_abs[0].weight.T @ _abs[1].weight.T).T for _abs in fm_abs], dim=0) # task-specific params (LoRA) + ], dim=0) + + elif 'to_qkv.bias' in self_param_name: + ss = self_param_name.split('.') + + fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv.bias' + return get_parameter(fm, fm_qkv_name) + + elif 'mlp.fc1' in self_param_name: + fm_param_name = self_param_name.replace('.linear', '') + return get_parameter(fm, fm_param_name) + + else: + return get_parameter(fm, self_param_name) + + +if __name__ == '__main__': + from utils.dl.common.env import set_random_seed + set_random_seed(1) + + # 3. init scenario + scenario = build_scenario( + source_datasets_name=['HL5Domains-ApexAD2600Progressive', 'HL5Domains-CanonG3', 'HL5Domains-CreativeLabsNomadJukeboxZenXtra40GB', + 'HL5Domains-NikonCoolpix4300'], + target_datasets_order=['HL5Domains-Nokia6610'] * 1, # TODO + da_mode='close_set', + data_dirs={ + **{k: f'/data/zql/datasets/nlp_asc_19_domains/dat/absa/Bing5Domains/asc/{k.split("-")[1]}' + for k in ['HL5Domains-ApexAD2600Progressive', 'HL5Domains-CanonG3', 'HL5Domains-CreativeLabsNomadJukeboxZenXtra40GB', + 'HL5Domains-NikonCoolpix4300', 'HL5Domains-Nokia6610']} + }, + ) + + # 1. init model + + fm_models_dict_path = 'new_impl/nlp/mobilebert/sentiment_classification/results/cls.py/20231017/999999-215719-result/models/fm_best.pt' + fm_models = torch.load(fm_models_dict_path) + fm_models_dict_path = save_models_dict_for_init(fm_models, __file__, 'fm_bert_cls_lora') + md_models_dict_path = save_models_dict_for_init({ + 'main': -1 + }, __file__, 'md_bert_none') + device = 'cuda' + + fm_model = ElasticDNN_Bert_OfflineSenClsFMModel('fm', fm_models_dict_path, device) + md_model = ElasticDNN_Bert_OfflineSenClsMDModel('md', md_models_dict_path, device) + + # 2. init alg + models = { + 'fm': fm_model, + 'md': md_model + } + fm_to_md_alg = ElasticDNN_MDPretrainingWoFBSAlg(models, get_res_save_dir(__file__, None)) + + from utils.dl.common.lr_scheduler import get_linear_schedule_with_warmup + fm_to_md_alg.run(scenario, hyps={ + 'launch_tbboard': False, + + 'samples_size': {'input_ids': torch.tensor([[ 101, 5672, 2033, 2011, 2151, 3793, 2017, 1005, 1040, 2066, 1012, 102]]).to(device), + 'token_type_ids': torch.tensor([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]).to(device), + 'attention_mask': torch.tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]).to(device), 'return_dict': False}, + 'generate_md_width_ratio': 4, + + 'train_batch_size': 8, + 'val_batch_size': 16, + 'num_workers': 16, + 'optimizer': 'AdamW', + 'optimizer_args': {'lr': 1e-4, 'betas': [0.9, 0.999]}, + 'scheduler': 'LambdaLR', + 'scheduler_args': {'lr_lambda': get_linear_schedule_with_warmup(10000, 70000)}, + 'num_iters': 27000, + 'val_freq': 1000, + 'distill_loss_weight': 1.0 + }) + \ No newline at end of file diff --git a/new_impl/nlp/mobilebert/sentiment_classification/cls_online.py b/new_impl/nlp/mobilebert/sentiment_classification/cls_online.py new file mode 100644 index 0000000000000000000000000000000000000000..4d948c353b0ba1d871f9445a8cbc3b2664f66bc8 --- /dev/null +++ b/new_impl/nlp/mobilebert/sentiment_classification/cls_online.py @@ -0,0 +1,415 @@ +from typing import Any, Dict, List +from schema import Schema +from data import build_scenario, build_cl_scenario, CLScenario, MergedDataset +from data.dataloader import build_dataloader +# from methods.elasticdnn.api.online_model import ElasticDNN_OnlineModel +from methods.elasticdnn.api.online_model_v2 import ElasticDNN_OnlineModel +from methods.elasticdnn.model.base import ElasticDNNUtil +from methods.gem.gem_el_bert import OnlineGEMModel, GEMAlg +from utils.dl.common.model import LayerActivation, get_module, get_parameter, set_module +from copy import deepcopy +from bert import ElasticBertUtil +import torch +import torch.nn.functional as F +import sys +import tqdm +from torch import nn +from utils.common.log import logger + +# from methods.shot.shot import OnlineShotModel +from experiments.utils.elasticfm_cl import init_online_model, elasticfm_cl +# torch.multiprocessing.set_sharing_strategy('file_system') +device = 'cuda' +app_name = 'secls' +sd_sparsity = 0.8 + +class ElasticDNN_SeClsOnlineModel(ElasticDNN_OnlineModel): + + @torch.no_grad() + def sd_feedback_to_md(self, after_da_sd, unpruned_indexes_of_layers): + self.models_dict['sd'] = after_da_sd + self.before_da_md = deepcopy(self.models_dict['md']) + + logger.info('\n\nsurrogate DNN feedback to master DNN...\n\n') + # one-to-one + + cur_unpruned_indexes = None + cur_unpruned_indexes_name = None + + for p_name, p in self.models_dict['sd'].named_parameters(): + matched_md_param = self.get_md_matched_param_of_sd_param(p_name) + logger.debug(f'if feedback: {p_name}') + if matched_md_param is None: + continue + logger.debug(f'start feedback: {p_name}, {p.size()} -> {matched_md_param.size()}') + # average + # setattr(matched_md_module, matched_md_param_name, (matched_md_param + p) / 2.) + + if p_name in unpruned_indexes_of_layers.keys(): + cur_unpruned_indexes = unpruned_indexes_of_layers[p_name] + cur_unpruned_indexes_name = p_name + + if p.size() != matched_md_param.size(): + logger.debug(f'cur unpruned indexes: {cur_unpruned_indexes_name}, {cur_unpruned_indexes.size()}') + + if p.dim() == 1: # norm + new_p = deepcopy(matched_md_param) + new_p[cur_unpruned_indexes] = p + elif p.dim() == 2: # linear + if p.size(0) < matched_md_param.size(0): # output pruned + new_p = deepcopy(matched_md_param) + new_p[cur_unpruned_indexes] = p + else: # input pruned + new_p = deepcopy(matched_md_param) + new_p[:, cur_unpruned_indexes] = p + p = new_p + + assert p.size() == matched_md_param.size(), f'{p.size()}, {matched_md_param.size()}' + + if 'classifier' in p_name: + continue + # if False: + # self.last_trained_cls_indexes + assert hasattr(self, 'last_trained_cls_indexes') + print(self.last_trained_cls_indexes) + + diff = self._compute_diff(matched_md_param, p) + # matched_md_param[self.last_trained_cls_indexes].copy_(p[self.last_trained_cls_indexes.to(self.device)]) + matched_md_param.copy_(p) + logger.debug(f'SPECIFIC FOR CL HEAD | end feedback: {p_name}, diff: {diff:.6f}') + else: + diff = self._compute_diff(matched_md_param, (matched_md_param + p) / 2.) + matched_md_param.copy_((matched_md_param + p) / 2.) + logger.debug(f'end feedback: {p_name}, diff: {diff:.6f}') + + def add_cls_in_head(self, num_cls): + head: nn.Linear = get_module(self.models_dict['md'], 'classifier') + + new_head = nn.Linear(head.in_features, head.out_features + num_cls, head.bias is not None, device=self.device) + + # nn.init.zeros_(new_head.weight.data) + # nn.init.zeros_(new_head.bias.data) + + new_head.weight.data[0: head.out_features] = deepcopy(head.weight.data) + new_head.bias.data[0: head.out_features] = deepcopy(head.bias.data) + set_module(self.models_dict['md'], 'classifier', new_head) + set_module(self.models_dict['fm'], 'classifier', new_head) + + + def get_accuracy(self, test_loader, *args, **kwargs): + acc = 0 + sample_num = 0 + + self.to_eval_mode() + + with torch.no_grad(): + pbar = tqdm.tqdm(enumerate(test_loader), total=len(test_loader), dynamic_ncols=True, leave=False) + for batch_index, (x, y) in pbar: + for k, v in x.items(): + if isinstance(v, torch.Tensor): + x[k] = v.to(self.device) + y = y.to(self.device) + output = self.infer(x) + pred = F.softmax(output, dim=1).argmax(dim=1) + correct = torch.eq(pred, y).sum().item() + acc += correct + sample_num += len(y) + + pbar.set_description(f'cur_batch_total: {len(y)}, cur_batch_correct: {correct}, ' + f'cur_batch_acc: {(correct / len(y)):.4f}') + + acc /= sample_num + return acc + + def get_elastic_dnn_util(self) -> ElasticDNNUtil: + return ElasticBertUtil() + + def get_fm_matched_param_of_md_param(self, md_param_name): + # only between qkv.weight, norm.weight/bias + self_param_name = md_param_name + fm = self.models_dict['fm'] + if any([k in self_param_name for k in ['fbs', 'ab', 'embeddings','ln']]): + return None + + p = get_parameter(self.models_dict['md'], self_param_name) + if p.dim() == 0: + return None + elif p.dim() == 1 and 'LayerNorm' in self_param_name and 'weight' in self_param_name: + return get_parameter(fm, self_param_name) + + # 1. xx.qkv.to_qkv.yy to xx.qkv.qkv.aa and xx.qkv.abs.zz + if ('query' in self_param_name or 'key' in self_param_name or \ + 'value' in self_param_name) and ('weight' in self_param_name): + + ss = self_param_name.split('.') + + fm_qkv_name = '.'.join(ss[0: -1]) + '.fc' + fm_qkv = get_module(fm, fm_qkv_name) + + fm_abs_name = '.'.join(ss[0: -1]) + '.ab' + fm_abs = get_module(fm, fm_abs_name) + + # NOTE: unrecoverable operation! multiply LoRA parameters to allow it being updated in update_fm_param() + # TODO: if fm will be used for inference, _mul_lora_weight will not be applied! + if not hasattr(fm_abs, '_mul_lora_weight'): + logger.debug(f'set _mul_lora_weight in {fm_abs_name}') + setattr(fm_abs, '_mul_lora_weight', + nn.Parameter(fm_abs[1].weight @ fm_abs[0].weight)) + + return torch.cat([ + fm_qkv.weight.data, # task-agnositc params + fm_abs._mul_lora_weight.data # task-specific params (LoRA) + ], dim=0) + + # elif 'to_qkv.bias' in self_param_name: + # ss = self_param_name.split('.') + + # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv.bias' + # return get_parameter(fm, fm_qkv_name) + + elif 'dense' in self_param_name and 'weight' in self_param_name: + fm_param_name = self_param_name.replace('.linear', '') + return get_parameter(fm, fm_param_name) + + # elif 'mlp.fc2' in self_param_name and 'weight' in self_param_name: + # fm_param_name = self_param_name + # return get_parameter(fm, fm_param_name) + + else: + # return get_parameter(fm, self_param_name) + return None + + def update_fm_param(self, md_param_name, cal_new_fm_param_by_md_param): + if not ('query' in md_param_name or 'key' in md_param_name or 'value' in md_param_name): + matched_fm_param_ref = self.get_fm_matched_param_of_md_param(md_param_name) + matched_fm_param_ref.copy_(cal_new_fm_param_by_md_param) + else: + new_fm_attn_weight, new_fm_lora_weight = torch.chunk(cal_new_fm_param_by_md_param, 2, 0) + + ss = md_param_name.split('.') + fm = self.models_dict['fm'] + + # update task-agnostic parameters + fm_qkv_name = '.'.join(ss[0: -1]) + '.fc' + fm_qkv = get_module(fm, fm_qkv_name) + fm_qkv.weight.data.copy_(new_fm_attn_weight) + + # update task-specific parameters + fm_abs_name = '.'.join(ss[0: -1]) + '.ab' + fm_abs = get_module(fm, fm_abs_name) + fm_abs._mul_lora_weight.data.copy_(new_fm_lora_weight) # TODO: this will not be applied in inference! + + def get_md_matched_param_of_fm_param(self, fm_param_name): + return super().get_md_matched_param_of_fm_param(fm_param_name) + + def get_md_matched_param_of_sd_param(self, sd_param_name): + # raise NotImplementedError + + # only between qkv.weight, norm.weight/bias + self_param_name = sd_param_name + md = self.models_dict['md'] + if any([k in self_param_name for k in ['fbs', 'ab', 'embeddings']]): + return None + + p = get_parameter(self.models_dict['sd'], self_param_name) + if p.dim() == 0: + return None + elif p.dim() == 1 and 'LayerNorm' in self_param_name and 'weight' in self_param_name: + return get_parameter(md, self_param_name) + + if 'classifier' in self_param_name: + return get_parameter(md, self_param_name) + + # 1. xx.qkv.to_qkv.yy to xx.qkv.qkv.aa and xx.qkv.abs.zz + if ('query' in self_param_name or 'key' in self_param_name or \ + 'value' in self_param_name) and ('weight' in self_param_name): + + + return get_parameter(md, self_param_name) # NOTE: no fbs in qkv! + + # elif 'to_qkv.bias' in self_param_name: + # ss = self_param_name.split('.') + + # fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv.bias' + # return get_parameter(fm, fm_qkv_name) + + elif 'intermediate.dense.0.weight' in self_param_name: + fm_param_name = '.'.join(self_param_name.split('.')[0: -2]) + '.linear.weight' + return get_parameter(md, fm_param_name) + + elif 'output.dense' in self_param_name and 'weight' in self_param_name: + fm_param_name = self_param_name + return get_parameter(md, fm_param_name) + + else: + # return get_parameter(fm, self_param_name) + return None + + def get_task_head_params(self): + head = get_module(self.models_dict['sd'], 'classifier') + return list(head.parameters()) + +class SeClsOnlineGEMModel(OnlineGEMModel): + def get_trained_params(self): + qkv_and_norm_params = [p for n, p in self.models_dict['main'].named_parameters() if 'query' in n or 'key' in n or 'value' in n or 'dense' in n or 'LayerNorm' in n] + return qkv_and_norm_params + + def forward_to_get_task_loss(self, x, y): + return F.cross_entropy(self.infer(x), y) + + def add_cls_in_head(self, num_cls): + return + + head: nn.Linear = get_module(self.models_dict['main'], 'head') + + new_head = nn.Linear(head.in_features, head.out_features + num_cls, head.bias is not None, device=self.device) + new_head.weight.data[0: head.out_features] = deepcopy(head.weight.data) + new_head.bias.data[0: head.out_features] = deepcopy(head.bias.data) + set_module(self.models_dict['main'], 'head', new_head) + + def infer(self, x, *args, **kwargs): + return self.models_dict['main'](**x) + + def get_accuracy(self, test_loader, *args, **kwargs): + _d = test_loader.dataset + from data import build_dataloader, split_dataset + if _d.__class__.__name__ == '_SplitDataset' and _d.underlying_dataset.__class__.__name__ == 'MergedDataset': # necessary for CL + print('\neval on merged datasets') + + merged_full_dataset = _d.underlying_dataset.datasets + ratio = len(_d.keys) / len(_d.underlying_dataset) + + if int(len(_d) * ratio) == 0: + ratio = 1. + # print(ratio) + # bs = + # test_loaders = [build_dataloader(split_dataset(d, min(max(test_loader.batch_size, int(len(d) * ratio)), len(d)))[0], # TODO: this might be overlapped with train dataset + # min(test_loader.batch_size, int(len(d) * ratio)), + # test_loader.num_workers, False, None) for d in merged_full_dataset] + + test_loaders = [] + for d in merged_full_dataset: + n = int(len(d) * ratio) + if n == 0: + n = len(d) + sub_dataset = split_dataset(d, min(max(test_loader.batch_size, n), len(d)))[0] + loader = build_dataloader(sub_dataset, min(test_loader.batch_size, n), test_loader.num_workers, False, None) + test_loaders += [loader] + + accs = [self.get_accuracy(loader) for loader in test_loaders] + print(accs) + return sum(accs) / len(accs) + + + + + acc = 0 + sample_num = 0 + + self.to_eval_mode() + + with torch.no_grad(): + pbar = tqdm.tqdm(enumerate(test_loader), total=len(test_loader), dynamic_ncols=True, leave=False) + for batch_index, (x, y) in pbar: + for k, v in x.items(): + if isinstance(v, torch.Tensor): + x[k] = v.to(self.device) + y = y.to(self.device) + output = self.infer(x) + pred = F.softmax(output, dim=1).argmax(dim=1) + correct = torch.eq(pred, y).sum().item() + acc += correct + sample_num += len(y) + + # if batch_index == 0: + # print(pred, y) + + pbar.set_description(f'cur_batch_total: {len(y)}, cur_batch_correct: {correct}, ' + f'cur_batch_acc: {(correct / len(y)):.4f}') + + acc /= sample_num + return acc + +settings = { + 'involve_fm': True +} + +scenario = build_scenario( + source_datasets_name=['HL5Domains-ApexAD2600Progressive', 'HL5Domains-CanonG3', 'HL5Domains-CreativeLabsNomadJukeboxZenXtra40GB', + 'HL5Domains-NikonCoolpix4300'], + target_datasets_order=['HL5Domains-Nokia6610','Liu3Domains-Computer', 'Liu3Domains-Router', 'Liu3Domains-Speaker', + 'Ding9Domains-DiaperChamp', 'Ding9Domains-Norton', 'Ding9Domains-LinksysRouter', + 'Ding9Domains-MicroMP3', 'Ding9Domains-Nokia6600', 'Ding9Domains-CanonPowerShotSD500', + 'Ding9Domains-ipod', 'Ding9Domains-HitachiRouter', 'Ding9Domains-CanonS100', + 'SemEval-Laptop', 'SemEval-Rest'] * 2 + ['Liu3Domains-Computer', 'Liu3Domains-Router'], + da_mode='close_set', + data_dirs={ + **{k: f'/data/zql/datasets/nlp_asc_19_domains/dat/absa/Bing5Domains/asc/{k.split("-")[1]}' + for k in ['HL5Domains-ApexAD2600Progressive', 'HL5Domains-CanonG3', 'HL5Domains-CreativeLabsNomadJukeboxZenXtra40GB', + 'HL5Domains-NikonCoolpix4300', 'HL5Domains-Nokia6610']}, + + **{k: f'/data/zql/datasets/nlp_asc_19_domains/dat/absa/Bing3Domains/asc/{k.split("-")[1]}' + for k in ['Liu3Domains-Computer', 'Liu3Domains-Router', 'Liu3Domains-Speaker']}, + + **{k: f'/data/zql/datasets/nlp_asc_19_domains/dat/absa/Bing9Domains/asc/{k.split("-")[1]}' + for k in ['Ding9Domains-DiaperChamp', 'Ding9Domains-Norton', 'Ding9Domains-LinksysRouter', + 'Ding9Domains-MicroMP3', 'Ding9Domains-Nokia6600', 'Ding9Domains-CanonPowerShotSD500', + 'Ding9Domains-ipod', 'Ding9Domains-HitachiRouter', 'Ding9Domains-CanonS100']}, + + **{k: f'/data/zql/datasets/nlp_asc_19_domains/dat/absa/XuSemEval/asc/14/{k.split("-")[1].lower()}' + for k in ['SemEval-Laptop', 'SemEval-Rest']}, + }, +) + +scenario = build_cl_scenario( + da_scenario=scenario, + target_datasets_name=['HL5Domains-Nokia6610'] * 16, + num_classes_per_task=5, + max_num_tasks=16, + data_dirs={ + **{k: f'/data/zql/datasets/nlp_asc_19_domains/dat/absa/Bing5Domains/asc/{k.split("-")[1]}' + for k in ['HL5Domains-Nokia6610']} + } +) + +elasticfm_model = ElasticDNN_SeClsOnlineModel('secls', init_online_model( + 'new_impl/nlp/mobilebert/sentiment_classification/results/cls_md_w_fbs_index.py/20231019/999999-222456-result/models/fm_best.pt', + 'new_impl/nlp/mobilebert/sentiment_classification/results/cls_md_w_fbs_index.py/20231019/999999-222456-result/models/md_best.pt', + 'cls', __file__ +), device, { + 'md_to_fm_alpha': 0.2, + 'fm_to_md_alpha': 0.2 +}) + +da_alg = GEMAlg +da_model = SeClsOnlineGEMModel +da_alg_hyp = { + 'train_batch_size': 4, + 'val_batch_size': 16, + 'num_workers': 4, + 'optimizer': 'AdamW', + 'optimizer_args': {'lr': 1e-4, 'betas': [0.9, 0.999], 'weight_decay': 0.01}, + 'scheduler': '', + 'scheduler_args': {}, + 'num_iters': 100, + 'val_freq': 20, + 'n_memories': 4 , + 'n_inputs': 3 * 224 * 224, + 'margin': 0.5, + 'num_my_iters': 0, + 'sd_sparsity': 0.7 +} + + +elasticfm_cl( + [app_name], + [scenario], + [elasticfm_model], + [da_alg], + [da_alg_hyp], + [da_model], + device, + settings, + __file__, + "results" +) diff --git a/new_impl/nlp/mobilebert/sentiment_classification/mobilebert-uncased/config.json b/new_impl/nlp/mobilebert/sentiment_classification/mobilebert-uncased/config.json new file mode 100644 index 0000000000000000000000000000000000000000..5c8c20f167f638a4db09132d703c368634008fc5 --- /dev/null +++ b/new_impl/nlp/mobilebert/sentiment_classification/mobilebert-uncased/config.json @@ -0,0 +1,31 @@ +{ + "_name_or_path": "google/mobilebert-uncased", + "architectures": [ + "MobileBertForPreTraining" + ], + "attention_probs_dropout_prob": 0.1, + "classifier_activation": false, + "embedding_size": 128, + "hidden_act": "relu", + "hidden_dropout_prob": 0.0, + "hidden_size": 512, + "initializer_range": 0.02, + "intermediate_size": 512, + "intra_bottleneck_size": 128, + "key_query_shared_bottleneck": true, + "layer_norm_eps": 1e-12, + "max_position_embeddings": 512, + "model_type": "mobilebert", + "normalization_type": "no_norm", + "num_attention_heads": 4, + "num_feedforward_networks": 4, + "num_hidden_layers": 24, + "pad_token_id": 0, + "transformers_version": "4.6.0.dev0", + "trigram_input": true, + "true_hidden_size": 128, + "type_vocab_size": 2, + "use_bottleneck": true, + "use_bottleneck_attention": false, + "vocab_size": 30522 +} diff --git a/new_impl/nlp/roberta/sentiment-classification/__pycache__/roberta.cpython-38.pyc b/new_impl/nlp/roberta/sentiment-classification/__pycache__/roberta.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..233214f5c7709263f4cbd74807f19dbbe6420350 Binary files /dev/null and b/new_impl/nlp/roberta/sentiment-classification/__pycache__/roberta.cpython-38.pyc differ diff --git a/new_impl/nlp/roberta/sentiment-classification/cls_baseline.py b/new_impl/nlp/roberta/sentiment-classification/cls_baseline.py new file mode 100644 index 0000000000000000000000000000000000000000..5a55fa0bb8e0a7a06b87ca7e30c5aae04eb2715a --- /dev/null +++ b/new_impl/nlp/roberta/sentiment-classification/cls_baseline.py @@ -0,0 +1,176 @@ +import os +#bert_path should be the path of the roberta-base dir +os.environ['bert_path'] = '/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/new_impl/nlp/roberta/sentiment-classification/roberta-base' +os.environ["TOKENIZERS_PARALLELISM"] = "false" +import torch +import torch.nn as nn +from methods.elasticdnn.api.algs.fm_lora import ElasticDNN_FMLoRAAlg +from methods.elasticdnn.api.algs.md_pretraining_wo_fbs import ElasticDNN_MDPretrainingWoFBSAlg +from methods.elasticdnn.model.base import ElasticDNNUtil +from methods.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from methods.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from methods.elasticdnn.pipeline.offline.fm_to_md.vit import FM_to_MD_ViT_Util +from methods.elasticdnn.model.vit import ElasticViTUtil +from methods.elasticdnn.api.algs.md_pretraining_index_v2_train_index_and_md import ElasticDNN_MDPretrainingIndexAlg +from utils.dl.common.model import LayerActivation2, get_module, get_parameter +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +import torch.nn.functional as F +from utils.dl.common.loss import CrossEntropyLossSoft +from new_impl.cv.feat_align.main import OnlineFeatAlignModel, FeatAlignAlg +import tqdm +from new_impl.cv.feat_align.mmd import mmd_rbf +from new_impl.cv.utils.baseline_da import baseline_da +from methods.elasticdnn.api.online_model_v2 import ElasticDNN_OnlineModel +from utils.common.log import logger +import json +from roberta import FMLoRA_Roberta_Util, RobertaForSenCls, FM_to_MD_Roberta_Util, ElasticRobertaUtil +from copy import deepcopy + +torch.cuda.set_device(1) + +# from methods.shot.shot import OnlineShotModel +from experiments.utils.elasticfm_cl import init_online_model, elasticfm_cl +# torch.multiprocessing.set_sharing_strategy('file_system') +device = 'cuda:1' +app_name = 'secls' + +scenario = build_scenario( + source_datasets_name=['HL5Domains-ApexAD2600Progressive', 'HL5Domains-CanonG3', 'HL5Domains-CreativeLabsNomadJukeboxZenXtra40GB'], + target_datasets_order=['HL5Domains-Nokia6610', 'HL5Domains-NikonCoolpix4300'] * 10, # TODO + da_mode='close_set', + data_dirs={ + **{k: f'/data/zql/datasets/nlp_asc_19_domains/dat/absa/Bing5Domains/asc/{k.split("-")[1]}' + for k in ['HL5Domains-ApexAD2600Progressive', 'HL5Domains-CanonG3', 'HL5Domains-CreativeLabsNomadJukeboxZenXtra40GB', + 'HL5Domains-NikonCoolpix4300', 'HL5Domains-Nokia6610']} + }, + ) + +class SeClsOnlineFeatAlignModel(OnlineFeatAlignModel): + def get_trained_params(self): # TODO: elastic fm only train a part of params + #qkv_and_norm_params = [p for n, p in self.models_dict['main'].named_parameters() if 'attention.attention.projection_query' in n or 'attention.attention.projection_key' in n or 'attention.attention.projection_value' in n or 'intermediate.dense' in n or 'output.dense' in n] + qkv_and_norm_params = [p for n, p in self.models_dict['main'].named_parameters()] + return qkv_and_norm_params + + def get_feature_hook(self) -> LayerActivation2: + return LayerActivation2(get_module(self.models_dict['main'], 'classifier')) + + def forward_to_get_task_loss(self, x, y): + self.to_train_mode() + return F.cross_entropy(self.infer(x), y) + + def get_mmd_loss(self, f1, f2): + common_shape = min(f1.shape[0], f2.shape[0]) + f1 = f1.view(f1.shape[0], -1) + f2 = f2.view(f2.shape[0], -1) + f1 = f1[:common_shape,:] + f2 = f2[:common_shape,:] + return mmd_rbf(f1, f2) + + def infer(self, x, *args, **kwargs): + return self.models_dict['main'](**x) + + def get_accuracy(self, test_loader, *args, **kwargs): + _d = test_loader.dataset + from data import build_dataloader, split_dataset + if _d.__class__.__name__ == '_SplitDataset' and _d.underlying_dataset.__class__.__name__ == 'MergedDataset': # necessary for CL + print('\neval on merged datasets') + + merged_full_dataset = _d.underlying_dataset.datasets + ratio = len(_d.keys) / len(_d.underlying_dataset) + + if int(len(_d) * ratio) == 0: + ratio = 1. + # print(ratio) + # bs = + # test_loaders = [build_dataloader(split_dataset(d, min(max(test_loader.batch_size, int(len(d) * ratio)), len(d)))[0], # TODO: this might be overlapped with train dataset + # min(test_loader.batch_size, int(len(d) * ratio)), + # test_loader.num_workers, False, None) for d in merged_full_dataset] + + test_loaders = [] + for d in merged_full_dataset: + n = int(len(d) * ratio) + if n == 0: + n = len(d) + sub_dataset = split_dataset(d, min(max(test_loader.batch_size, n), len(d)))[0] + loader = build_dataloader(sub_dataset, min(test_loader.batch_size, n), test_loader.num_workers, False, None) + test_loaders += [loader] + + accs = [self.get_accuracy(loader) for loader in test_loaders] + print(accs) + return sum(accs) / len(accs) + + acc = 0 + sample_num = 0 + + self.to_eval_mode() + + with torch.no_grad(): + pbar = tqdm.tqdm(enumerate(test_loader), total=len(test_loader), dynamic_ncols=True, leave=False) + for batch_index, (x, y) in pbar: + for k, v in x.items(): + if isinstance(v, torch.Tensor): + x[k] = v.to(self.device) + y = y.to(self.device) + output = self.infer(x) + pred = F.softmax(output, dim=1).argmax(dim=1) + correct = torch.eq(pred, y).sum().item() + acc += correct + sample_num += len(y) + + # if batch_index == 0: + # print(pred, y) + + pbar.set_description(f'cur_batch_total: {len(y)}, cur_batch_correct: {correct}, ' + f'cur_batch_acc: {(correct / len(y)):.4f}') + + acc /= sample_num + return acc + +da_alg = FeatAlignAlg +from utils.dl.common.lr_scheduler import get_linear_schedule_with_warmup +#from new_impl.cv.model import ClsOnlineFeatAlignModel +da_model = SeClsOnlineFeatAlignModel( + app_name, + 'new_impl/nlp/roberta/sentiment-classification/results/cls_md_wo_fbs.py/20240113/999996-140353/models/md_best.pt', + device +) + +da_alg_hyp = { + 'HL5Domains-Nokia6610': { + 'train_batch_size': 32, + 'val_batch_size': 256, + 'num_workers': 8, + 'optimizer': 'AdamW', + 'optimizer_args': {'lr': 2e-7, 'betas': [0.9, 0.999], 'weight_decay': 0.01}, + 'scheduler': '', + 'scheduler_args': {}, + 'num_iters': 100, + 'val_freq': 20, + 'feat_align_loss_weight': 1.0, + }, + 'HL5Domains-NikonCoolpix4300': { + 'train_batch_size': 32, + 'val_batch_size': 128, + 'num_workers': 8, + 'optimizer': 'AdamW', + 'optimizer_args': {'lr': 2e-7, 'betas': [0.9, 0.999], 'weight_decay': 0.01}, + 'scheduler': '', + 'scheduler_args': {}, + 'num_iters': 100, + 'val_freq': 20, + 'feat_align_loss_weight': 1.0, + }, +} + + +baseline_da( + app_name, + scenario, + da_alg, + da_alg_hyp, + da_model, + device, + __file__, + "results" +) diff --git a/new_impl/nlp/roberta/sentiment-classification/cls_lora.py b/new_impl/nlp/roberta/sentiment-classification/cls_lora.py new file mode 100644 index 0000000000000000000000000000000000000000..afec107822af3d2f0c73cae200a73254b43518d2 --- /dev/null +++ b/new_impl/nlp/roberta/sentiment-classification/cls_lora.py @@ -0,0 +1,98 @@ +import os +#bert_path should be the path of the roberta-base dir +os.environ['bert_path'] = '/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/new_impl/nlp/roberta/sentiment-classification/roberta-base' + +import torch +from methods.elasticdnn.api.model import ElasticDNN_OfflineSenClsFMModel, ElasticDNN_OfflineSenClsMDModel +from methods.elasticdnn.api.algs.fm_lora import ElasticDNN_FMLoRAAlg +from methods.elasticdnn.model.base import ElasticDNNUtil +from methods.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +# from methods.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +# from methods.elasticdnn.pipeline.offline.fm_to_md.vit import FM_to_MD_ViT_Util +# from methods.elasticdnn.model.vit import ElasticViTUtil +from roberta import FMLoRA_Roberta_Util, RobertaForSenCls +from utils.dl.common.model import LayerActivation, get_module, get_parameter, set_module +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +from utils.common.log import logger +import torch.nn.functional as F +import sys + +class ElasticDNN_Roberta_OfflineSenClsFMModel(ElasticDNN_OfflineSenClsFMModel): + def generate_md_by_reducing_width(self, reducing_width_ratio, samples: torch.Tensor): + # return FM_to_MD_ViT_Util().init_md_from_fm_by_reducing_width_with_perf_test(self.models_dict['main'], + # reducing_width_ratio, samples) + raise NotImplementedError + + def get_feature_hook(self) -> LayerActivation: + return LayerActivation(get_module(self.models_dict['main'], 'classifier'), True, self.device) + + def get_elastic_dnn_util(self) -> ElasticDNNUtil: + raise NotImplementedError + + def forward_to_get_task_loss(self, x, y): + self.to_train_mode() + pred = self.infer(x) + return F.cross_entropy(pred, y) + + def get_lora_util(self) -> FMLoRA_Util: + return FMLoRA_Roberta_Util() + + def get_task_head_params(self): + head = get_module(self.models_dict['main'], 'classifier') + params_name = {k for k, v in head.named_parameters()} + logger.info(f'task head params: {params_name}') + return list(head.parameters()) + +if __name__ == '__main__': + from utils.dl.common.env import set_random_seed + set_random_seed(1) + torch.cuda.set_device(1) + + scenario = build_scenario( + source_datasets_name=['HL5Domains-ApexAD2600Progressive', 'HL5Domains-CanonG3', 'HL5Domains-CreativeLabsNomadJukeboxZenXtra40GB'], + target_datasets_order=['HL5Domains-Nokia6610', 'HL5Domains-NikonCoolpix4300'] * 1, # TODO + da_mode='close_set', + data_dirs={ + **{k: f'/data/zql/datasets/nlp_asc_19_domains/dat/absa/Bing5Domains/asc/{k.split("-")[1]}' + for k in ['HL5Domains-ApexAD2600Progressive', 'HL5Domains-CanonG3', 'HL5Domains-CreativeLabsNomadJukeboxZenXtra40GB', + 'HL5Domains-NikonCoolpix4300', 'HL5Domains-Nokia6610']} + }, + ) + + # 2. init model + device = 'cuda' + model = RobertaForSenCls(num_classes=scenario.num_classes) + fm_models_dict_path = save_models_dict_for_init({ + 'main': model + }, __file__, 'roberta_pretrained_sen_cls') + + fm_model = ElasticDNN_Roberta_OfflineSenClsFMModel('fm', fm_models_dict_path, device) + + # 3. init alg + models = { + 'fm': fm_model + } + fm_lora_alg = ElasticDNN_FMLoRAAlg(models, get_res_save_dir(__file__, "results")) + + + # 4. run alg + from utils.dl.common.lr_scheduler import get_linear_schedule_with_warmup + fm_lora_alg.run(scenario, hyps={ + 'launch_tbboard': False, + + 'samples_size': {'input_ids': torch.tensor([[ 101, 5672, 2033, 2011, 2151, 3793, 2017, 1005, 1040, 2066, 1012, 102]]).to(device), + 'token_type_ids': torch.tensor([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]).to(device), + 'attention_mask': torch.tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]).to(device), 'return_dict': False}, + + 'ab_r': 8, + 'train_batch_size': 32, + 'val_batch_size': 128, + 'num_workers': 32, + 'optimizer': 'AdamW', + 'optimizer_args': {'lr': 1e-4, 'betas': [0.9, 0.999]}, + 'scheduler': 'LambdaLR', + 'scheduler_args': {'lr_lambda': get_linear_schedule_with_warmup(10000, 80000)}, + 'num_iters': 80000, + 'val_freq': 1000, + }) \ No newline at end of file diff --git a/new_impl/nlp/roberta/sentiment-classification/cls_md_w_fbs_index.py b/new_impl/nlp/roberta/sentiment-classification/cls_md_w_fbs_index.py new file mode 100644 index 0000000000000000000000000000000000000000..eba5414a52d1f80f385e899a32b29fbf8b558ff5 --- /dev/null +++ b/new_impl/nlp/roberta/sentiment-classification/cls_md_w_fbs_index.py @@ -0,0 +1,174 @@ +import os +#bert_path should be the path of the roberta-base dir +os.environ['bert_path'] = '/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/new_impl/nlp/roberta/sentiment-classification/roberta-base' + +import torch +import sys +from torch import nn +from methods.elasticdnn.api.model import ElasticDNN_OfflineSenClsFMModel, ElasticDNN_OfflineSenClsMDModel +from methods.elasticdnn.api.algs.md_pretraining_wo_fbs import ElasticDNN_MDPretrainingWoFBSAlg +from methods.elasticdnn.model.base import ElasticDNNUtil +from methods.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from roberta import FMLoRA_Roberta_Util, RobertaForSenCls, FM_to_MD_Roberta_Util, ElasticRobertaUtil +from methods.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from utils.dl.common.model import LayerActivation2, get_module, get_parameter +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +from utils.dl.common.loss import CrossEntropyLossSoft +import torch.nn.functional as F +from utils.common.log import logger + +torch.cuda.set_device(1) + +class ElasticDNN_Roberta_OfflineSenClsFMModel(ElasticDNN_OfflineSenClsFMModel): + def generate_md_by_reducing_width(self, reducing_width_ratio, samples: torch.Tensor): # TODO: + tmp = FM_to_MD_Roberta_Util().init_md_from_fm_by_reducing_width_with_perf_test(self.models_dict['main'], + reducing_width_ratio, samples) + return tmp + + def get_feature_hook(self) -> LayerActivation2: + return LayerActivation2(get_module(self.models_dict['main'], 'classifier')) + + def get_elastic_dnn_util(self) -> ElasticDNNUtil: # TODO: + return ElasticRobertaUtil() + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + self.to_train_mode() + return F.cross_entropy(self.infer(x), y) + + def get_lora_util(self) -> FMLoRA_Util: + return FMLoRA_Roberta_Util() + + def get_task_head_params(self): + head = get_module(self.models_dict['main'], 'classifier') + params_name = {k for k, v in head.named_parameters()} + logger.info(f'task head params: {params_name}') + return list(head.parameters()) + + +class ElasticDNN_Roberta_OfflineSenClsMDModel(ElasticDNN_OfflineSenClsMDModel): + def __init__(self, name: str, models_dict_path: str, device: str): + super().__init__(name, models_dict_path, device) + + self.distill_criterion = CrossEntropyLossSoft() + + def get_feature_hook(self) -> LayerActivation2: + return LayerActivation2(get_module(self.models_dict['main'], 'classifier')) + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + self.to_train_mode() + return F.cross_entropy(self.infer(x), y) + + def get_distill_loss(self, student_output, teacher_output): + # print(student_output, teacher_output) + return self.distill_criterion(student_output, teacher_output) + + def get_matched_param_of_fm(self, self_param_name, fm: nn.Module): # TODO: + if any([k in self_param_name for k in ['fbs', 'embeddings', 'ln']]): + return None + + # 1. xx.qkv.to_qkv.yy to xx.qkv.qkv.aa and xx.qkv.abs.zz + if ('query' in self_param_name or 'key' in self_param_name or 'value' in self_param_name) \ + and 'weight' in self_param_name: + ss = self_param_name.split('.') + # raise NotImplementedError() # TODO: + fm_qkv_name = '.'.join(ss[0: -1]) + '.fc' + fm_qkv = get_module(fm, fm_qkv_name) + + fm_abs_name = '.'.join(ss[0: -1]) + '.ab' + fm_abs = [get_module(fm, fm_abs_name)] + + return torch.cat([ + fm_qkv.weight.data, # task-agnositc params + torch.cat([(_abs[0].weight.T @ _abs[1].weight.T).T for _abs in fm_abs], dim=0) # task-specific params (LoRA) + ], dim=0) + + elif ('query' in self_param_name or 'key' in self_param_name or 'value' in self_param_name) \ + and 'bias' in self_param_name: + ss = self_param_name.split('.') + + fm_qkv_name = '.'.join(ss[0: -1]) + '.fc.bias' + return get_parameter(fm, fm_qkv_name) + + elif 'intermediate.dense' in self_param_name: + fm_param_name = self_param_name.replace('.linear', '') + return get_parameter(fm, fm_param_name) + + else: + return get_parameter(fm, self_param_name) + + + +if __name__ == '__main__': + from utils.dl.common.env import set_random_seed + set_random_seed(1) + + + # 3. init scenario + scenario = build_scenario( + source_datasets_name=['HL5Domains-ApexAD2600Progressive', 'HL5Domains-CanonG3', 'HL5Domains-CreativeLabsNomadJukeboxZenXtra40GB'], + target_datasets_order=['HL5Domains-Nokia6610', 'HL5Domains-NikonCoolpix4300'] * 1, # TODO + da_mode='close_set', + data_dirs={ + **{k: f'/data/zql/datasets/nlp_asc_19_domains/dat/absa/Bing5Domains/asc/{k.split("-")[1]}' + for k in ['HL5Domains-ApexAD2600Progressive', 'HL5Domains-CanonG3', 'HL5Domains-CreativeLabsNomadJukeboxZenXtra40GB', + 'HL5Domains-NikonCoolpix4300', 'HL5Domains-Nokia6610']} + }, + ) + + # 1. init model + fm_models_dict_path = 'new_impl/nlp/roberta/sentiment-classification/results/cls_lora.py/20240105/999999-182730-results/models/fm_best.pt' + fm_models = torch.load(fm_models_dict_path) + fm_models_dict_path = save_models_dict_for_init(fm_models, __file__, 'fm_roberta_sen_cls_lora') + md_models_dict_path = save_models_dict_for_init( + torch.load('new_impl/nlp/roberta/sentiment-classification/results/cls_md_wo_fbs.py/20240106/999999-134324/models/md_best.pt', map_location=torch.device('cuda:1')), + __file__, 'md_roberta_sen_cls_raw_pretrained') + device = 'cuda:1' + + fm_model = ElasticDNN_Roberta_OfflineSenClsFMModel('fm', fm_models_dict_path, device) + md_model = ElasticDNN_Roberta_OfflineSenClsMDModel('md', md_models_dict_path, device) + + # 2. init alg + models = { + 'fm': fm_model, + 'md': md_model + } + from methods.elasticdnn.api.algs.md_pretraining_index_v2_train_index_and_md import ElasticDNN_MDPretrainingIndexAlg + fm_to_md_alg = ElasticDNN_MDPretrainingIndexAlg(models, get_res_save_dir(__file__, 'results')) + + from utils.dl.common.lr_scheduler import get_linear_schedule_with_warmup + fm_to_md_alg.run(scenario, hyps={ + 'launch_tbboard': False, + + 'samples_size': {'input_ids': torch.tensor([[ 101, 5672, 2033, 2011, 2151, 3793, 2017, 1005, 1040, 2066, 1012, 102]]).to(device), + 'token_type_ids': torch.tensor([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]).to(device), + 'attention_mask': torch.tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]).to(device), 'return_dict': False}, + + 'FBS_r': 12, + 'FBS_ignore_layers': [], + + 'train_batch_size': 32, + 'val_batch_size': 128, + 'num_workers': 32, + 'optimizer': 'AdamW', + 'optimizer_args': {'lr': 1e-5, 'betas': [0.9, 0.999]}, + 'scheduler': 'LambdaLR', + 'scheduler_args': {'lr_lambda': get_linear_schedule_with_warmup(10000, 70000)}, + + 'indexes_optimizer_args': {'lr': 3e-3, 'momentum': 0.9, 'weight_decay': 5e-4}, + + 'num_iters': 80000, + 'val_freq': 100, + + 'max_sparsity': 0.9, + 'min_sparsity': 0.3, + 'l1_reg_loss_weight': 1e-9, + 'index_loss_weight': 1e-4, + 'val_num_sparsities': 4, + + 'bn_cal_num_iters': 0, + + 'index_init': 'zero', + 'index_guided_linear_comb_split_size': 512 + }) + \ No newline at end of file diff --git a/new_impl/nlp/roberta/sentiment-classification/cls_md_wo_fbs.py b/new_impl/nlp/roberta/sentiment-classification/cls_md_wo_fbs.py new file mode 100644 index 0000000000000000000000000000000000000000..13c3fafe5bbab82dbbf6802d09af849ab86d1ef0 --- /dev/null +++ b/new_impl/nlp/roberta/sentiment-classification/cls_md_wo_fbs.py @@ -0,0 +1,155 @@ +import os +#bert_path should be the path of the roberta-base dir +os.environ['bert_path'] = '/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/new_impl/nlp/roberta/sentiment-classification/roberta-base' + +import torch +import sys +from torch import nn +from methods.elasticdnn.api.model import ElasticDNN_OfflineSenClsFMModel, ElasticDNN_OfflineSenClsMDModel +from methods.elasticdnn.api.algs.md_pretraining_wo_fbs import ElasticDNN_MDPretrainingWoFBSAlg +from methods.elasticdnn.model.base import ElasticDNNUtil +from methods.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from roberta import FMLoRA_Roberta_Util, RobertaForSenCls, FM_to_MD_Roberta_Util +from methods.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from utils.dl.common.model import LayerActivation2, get_module, get_parameter +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +from utils.dl.common.loss import CrossEntropyLossSoft +import torch.nn.functional as F +from utils.common.log import logger + + +class ElasticDNN_Roberta_OfflineSenClsFMModel(ElasticDNN_OfflineSenClsFMModel): + def generate_md_by_reducing_width(self, reducing_width_ratio, samples: torch.Tensor): # TODO: + tmp = FM_to_MD_Roberta_Util().init_md_from_fm_by_reducing_width_with_perf_test(self.models_dict['main'], + reducing_width_ratio, samples) + return tmp + # raise NotImplementedError + + def get_feature_hook(self) -> LayerActivation2: + return LayerActivation2(get_module(self.models_dict['main'], 'classifier')) + + def get_elastic_dnn_util(self) -> ElasticDNNUtil: # TODO: + return None + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + self.to_train_mode() + return F.cross_entropy(self.infer(x), y) + + def get_lora_util(self) -> FMLoRA_Util: + return FMLoRA_Roberta_Util() + + def get_task_head_params(self): + head = get_module(self.models_dict['main'], 'classifier') + params_name = {k for k, v in head.named_parameters()} + logger.info(f'task head params: {params_name}') + return list(head.parameters()) + + +class ElasticDNN_Roberta_OfflineSenClsMDModel(ElasticDNN_OfflineSenClsMDModel): + def __init__(self, name: str, models_dict_path: str, device: str): + super().__init__(name, models_dict_path, device) + + self.distill_criterion = CrossEntropyLossSoft() + + def get_feature_hook(self) -> LayerActivation2: + return LayerActivation2(get_module(self.models_dict['main'], 'classifier')) + + def forward_to_get_task_loss(self, x, y, *args, **kwargs): + self.to_train_mode() + return F.cross_entropy(self.infer(x), y) + + def get_distill_loss(self, student_output, teacher_output): + # print(student_output, teacher_output) + return self.distill_criterion(student_output, teacher_output) + + def get_matched_param_of_fm(self, self_param_name, fm: nn.Module): # TODO: + if any([k in self_param_name for k in ['fbs', 'embeddings']]): + return None + + # 1. xx.qkv.to_qkv.yy to xx.qkv.qkv.aa and xx.qkv.abs.zz + if 'query' in self_param_name or 'key' in self_param_name or 'value' in self_param_name: + ss = self_param_name.split('.') + raise NotImplementedError() # TODO: + fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv' + fm_qkv = get_module(fm, fm_qkv_name) + + fm_abs_name = '.'.join(ss[0: -2]) + '.abs' + fm_abs = get_module(fm, fm_abs_name) + + return torch.cat([ + fm_qkv.weight.data, # task-agnositc params + torch.cat([(_abs[0].weight.T @ _abs[1].weight.T).T for _abs in fm_abs], dim=0) # task-specific params (LoRA) + ], dim=0) + + elif 'to_qkv.bias' in self_param_name: + ss = self_param_name.split('.') + + fm_qkv_name = '.'.join(ss[0: -2]) + '.qkv.bias' + return get_parameter(fm, fm_qkv_name) + + elif 'mlp.fc1' in self_param_name: + fm_param_name = self_param_name.replace('.linear', '') + return get_parameter(fm, fm_param_name) + + else: + return get_parameter(fm, self_param_name) + + +if __name__ == '__main__': + from utils.dl.common.env import set_random_seed + set_random_seed(1) + + # 3. init scenario + scenario = build_scenario( + source_datasets_name=['HL5Domains-ApexAD2600Progressive', 'HL5Domains-CanonG3', 'HL5Domains-CreativeLabsNomadJukeboxZenXtra40GB'], + target_datasets_order=['HL5Domains-Nokia6610', 'HL5Domains-NikonCoolpix4300'] * 1, # TODO + da_mode='close_set', + data_dirs={ + **{k: f'/data/zql/datasets/nlp_asc_19_domains/dat/absa/Bing5Domains/asc/{k.split("-")[1]}' + for k in ['HL5Domains-ApexAD2600Progressive', 'HL5Domains-CanonG3', 'HL5Domains-CreativeLabsNomadJukeboxZenXtra40GB', + 'HL5Domains-NikonCoolpix4300', 'HL5Domains-Nokia6610']} + }, + ) + + # 1. init model + + fm_models_dict_path = 'new_impl/nlp/roberta/sentiment-classification/results/cls_lora.py/20240105/999999-182730-results/models/fm_best.pt' + fm_models = torch.load(fm_models_dict_path) + fm_models_dict_path = save_models_dict_for_init(fm_models, __file__, 'fm_roberta_sen_cls_lora') + md_models_dict_path = save_models_dict_for_init({ + 'main': -1 + }, __file__, 'md_roberta_none') + device = 'cuda' + + fm_model = ElasticDNN_Roberta_OfflineSenClsFMModel('fm', fm_models_dict_path, device) + md_model = ElasticDNN_Roberta_OfflineSenClsMDModel('md', md_models_dict_path, device) + + # 2. init alg + models = { + 'fm': fm_model, + 'md': md_model + } + fm_to_md_alg = ElasticDNN_MDPretrainingWoFBSAlg(models, get_res_save_dir(__file__, None)) + + from utils.dl.common.lr_scheduler import get_linear_schedule_with_warmup + fm_to_md_alg.run(scenario, hyps={ + 'launch_tbboard': False, + + 'samples_size': {'input_ids': torch.tensor([[ 101, 5672, 2033, 2011, 2151, 3793, 2017, 1005, 1040, 2066, 1012, 102]]).to(device), + 'token_type_ids': torch.tensor([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]).to(device), + 'attention_mask': torch.tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]).to(device), 'return_dict': False}, + 'generate_md_width_ratio': 8, + + 'train_batch_size': 32, + 'val_batch_size': 128, + 'num_workers': 32, + 'optimizer': 'AdamW', + 'optimizer_args': {'lr': 1e-4, 'betas': [0.9, 0.999]}, + 'scheduler': 'LambdaLR', + 'scheduler_args': {'lr_lambda': get_linear_schedule_with_warmup(10000, 70000)}, + 'num_iters': 70000, + 'val_freq': 1000, + 'distill_loss_weight': 1.0 + }) + \ No newline at end of file diff --git a/new_impl/nlp/roberta/sentiment-classification/cls_online.py b/new_impl/nlp/roberta/sentiment-classification/cls_online.py new file mode 100644 index 0000000000000000000000000000000000000000..c9919aceaad28ebb49d6492eebbc632f1e4b2c95 --- /dev/null +++ b/new_impl/nlp/roberta/sentiment-classification/cls_online.py @@ -0,0 +1,426 @@ +import os +#bert_path should be the path of the roberta-base dir +os.environ['bert_path'] = '/data/zql/concept-drift-in-edge-projects/UniversalElasticNet/new_impl/nlp/roberta/sentiment-classification/roberta-base' +os.environ["TOKENIZERS_PARALLELISM"] = "false" +import torch +import torch.nn as nn +from methods.elasticdnn.api.algs.fm_lora import ElasticDNN_FMLoRAAlg +from methods.elasticdnn.api.algs.md_pretraining_wo_fbs import ElasticDNN_MDPretrainingWoFBSAlg +from methods.elasticdnn.model.base import ElasticDNNUtil +from methods.elasticdnn.pipeline.offline.fm_lora.base import FMLoRA_Util +from methods.elasticdnn.pipeline.offline.fm_to_md.base import FM_to_MD_Util +from methods.elasticdnn.pipeline.offline.fm_to_md.vit import FM_to_MD_ViT_Util +from methods.elasticdnn.model.vit import ElasticViTUtil +from methods.elasticdnn.api.algs.md_pretraining_index_v2_train_index_and_md import ElasticDNN_MDPretrainingIndexAlg +from utils.dl.common.model import LayerActivation2, get_module, get_parameter +from utils.common.exp import save_models_dict_for_init, get_res_save_dir +from data import build_scenario +import torch.nn.functional as F +from utils.dl.common.loss import CrossEntropyLossSoft +from new_impl.cv.feat_align.main import OnlineFeatAlignModel, FeatAlignAlg +import tqdm +from new_impl.cv.feat_align.mmd import mmd_rbf +from new_impl.cv.utils.elasticfm_da import init_online_model, elasticfm_da +from methods.elasticdnn.api.online_model_v2 import ElasticDNN_OnlineModel +from utils.common.log import logger +import json +from roberta import FMLoRA_Roberta_Util, RobertaForSenCls, FM_to_MD_Roberta_Util, ElasticRobertaUtil +from copy import deepcopy + +torch.cuda.set_device(1) + +# from methods.shot.shot import OnlineShotModel +from experiments.utils.elasticfm_cl import init_online_model, elasticfm_cl +# torch.multiprocessing.set_sharing_strategy('file_system') +device = 'cuda:1' +app_name = 'secls' +sd_sparsity = 0.8 + +settings = { + 'involve_fm': True +} + +scenario = build_scenario( + source_datasets_name=['HL5Domains-ApexAD2600Progressive', 'HL5Domains-CanonG3', 'HL5Domains-CreativeLabsNomadJukeboxZenXtra40GB'], + target_datasets_order=['HL5Domains-Nokia6610', 'HL5Domains-NikonCoolpix4300'] * 10, # TODO + da_mode='close_set', + data_dirs={ + **{k: f'/data/zql/datasets/nlp_asc_19_domains/dat/absa/Bing5Domains/asc/{k.split("-")[1]}' + for k in ['HL5Domains-ApexAD2600Progressive', 'HL5Domains-CanonG3', 'HL5Domains-CreativeLabsNomadJukeboxZenXtra40GB', + 'HL5Domains-NikonCoolpix4300', 'HL5Domains-Nokia6610']} + }, + ) + +class ElasticDNN_SeClsOnlineModel(ElasticDNN_OnlineModel): + + @torch.no_grad() + def sd_feedback_to_md(self, after_da_sd, unpruned_indexes_of_layers): + self.models_dict['sd'] = after_da_sd + self.before_da_md = deepcopy(self.models_dict['md']) + + logger.info('\n\nsurrogate DNN feedback to master DNN...\n\n') + # one-to-one + + cur_unpruned_indexes = None + cur_unpruned_indexes_name = None + + for p_name, p in self.models_dict['sd'].named_parameters(): + matched_md_param = self.get_md_matched_param_of_sd_param(p_name) + logger.debug(f'if feedback: {p_name}') + if matched_md_param is None: + continue + logger.debug(f'start feedback: {p_name}, {p.size()} -> {matched_md_param.size()}') + # average + # setattr(matched_md_module, matched_md_param_name, (matched_md_param + p) / 2.) + + if p_name in unpruned_indexes_of_layers.keys(): + cur_unpruned_indexes = unpruned_indexes_of_layers[p_name] + cur_unpruned_indexes_name = p_name + + if p.size() != matched_md_param.size(): + logger.debug(f'cur unpruned indexes: {cur_unpruned_indexes_name}, {cur_unpruned_indexes.size()}') + + if p.dim() == 1: # norm + new_p = deepcopy(matched_md_param) + new_p[cur_unpruned_indexes] = p + elif p.dim() == 2: # linear + if p.size(0) < matched_md_param.size(0): # output pruned + new_p = deepcopy(matched_md_param) + new_p[cur_unpruned_indexes] = p + else: # input pruned + new_p = deepcopy(matched_md_param) + new_p[:, cur_unpruned_indexes] = p + p = new_p + + assert p.size() == matched_md_param.size(), f'{p.size()}, {matched_md_param.size()}' + + # if 'classifier' in p_name: + # continue + # # if False: + # # self.last_trained_cls_indexes + # assert hasattr(self, 'last_trained_cls_indexes') + # print(self.last_trained_cls_indexes) + + # diff = self._compute_diff(matched_md_param, p) + # # matched_md_param[self.last_trained_cls_indexes].copy_(p[self.last_trained_cls_indexes.to(self.device)]) + # matched_md_param.copy_(p) + # logger.debug(f'SPECIFIC FOR CL HEAD | end feedback: {p_name}, diff: {diff:.6f}') + # else: + diff = self._compute_diff(matched_md_param, (matched_md_param + p) / 2.) + matched_md_param.copy_((matched_md_param + p) / 2.) + logger.debug(f'end feedback: {p_name}, diff: {diff:.6f}') + + def add_cls_in_head(self, num_cls): + head: nn.Linear = get_module(self.models_dict['md'], 'classifier') + + new_head = nn.Linear(head.in_features, head.out_features + num_cls, head.bias is not None, device=self.device) + + # nn.init.zeros_(new_head.weight.data) + # nn.init.zeros_(new_head.bias.data) + + new_head.weight.data[0: head.out_features] = deepcopy(head.weight.data) + new_head.bias.data[0: head.out_features] = deepcopy(head.bias.data) + set_module(self.models_dict['md'], 'classifier', new_head) + set_module(self.models_dict['fm'], 'classifier', new_head) + + + def get_accuracy(self, test_loader, *args, **kwargs): + acc = 0 + sample_num = 0 + + self.to_eval_mode() + + with torch.no_grad(): + pbar = tqdm.tqdm(enumerate(test_loader), total=len(test_loader), dynamic_ncols=True, leave=False) + for batch_index, (x, y) in pbar: + for k, v in x.items(): + if isinstance(v, torch.Tensor): + x[k] = v.to(self.device) + y = y.to(self.device) + output = self.infer(x) + pred = F.softmax(output, dim=1).argmax(dim=1) + correct = torch.eq(pred, y).sum().item() + acc += correct + sample_num += len(y) + + pbar.set_description(f'cur_batch_total: {len(y)}, cur_batch_correct: {correct}, ' + f'cur_batch_acc: {(correct / len(y)):.4f}') + + acc /= sample_num + return acc + + def get_elastic_dnn_util(self) -> ElasticDNNUtil: + return ElasticRobertaUtil() + + def get_fm_matched_param_of_md_param(self, md_param_name): + # only between qkv.weight, norm.weight/bias + self_param_name = md_param_name + fm = self.models_dict['fm'] + if any([k in self_param_name for k in ['fbs', 'ab', 'embeddings','ln']]): + return None + + p = get_parameter(self.models_dict['md'], self_param_name) + if p.dim() == 0: + return None + elif p.dim() == 1 and 'LayerNorm' in self_param_name and 'weight' in self_param_name: + return get_parameter(fm, self_param_name) + + # 1. xx.qkv.to_qkv.yy to xx.qkv.qkv.aa and xx.qkv.abs.zz + if ('query' in self_param_name or 'key' in self_param_name or \ + 'value' in self_param_name) and ('weight' in self_param_name): + + ss = self_param_name.split('.') + + fm_qkv_name = '.'.join(ss[0: -1]) + '.fc' + fm_qkv = get_module(fm, fm_qkv_name) + + fm_abs_name = '.'.join(ss[0: -1]) + '.ab' + fm_abs = get_module(fm, fm_abs_name) + + # NOTE: unrecoverable operation! multiply LoRA parameters to allow it being updated in update_fm_param() + # TODO: if fm will be used for inference, _mul_lora_weight will not be applied! + if not hasattr(fm_abs, '_mul_lora_weight'): + logger.debug(f'set _mul_lora_weight in {fm_abs_name}') + setattr(fm_abs, '_mul_lora_weight', + nn.Parameter(fm_abs[1].weight @ fm_abs[0].weight)) + + return torch.cat([ + fm_qkv.weight.data, # task-agnositc params + fm_abs._mul_lora_weight.data # task-specific params (LoRA) + ], dim=0) + + elif ('query' in self_param_name or 'key' in self_param_name or 'value' in self_param_name) \ + and 'bias' in self_param_name: + ss = self_param_name.split('.') + + fm_qkv_name = '.'.join(ss[0: -1]) + '.fc.bias' + return get_parameter(fm, fm_qkv_name) + + elif 'intermediate.dense' in self_param_name: + fm_param_name = self_param_name.replace('.linear', '') + return get_parameter(fm, fm_param_name) + + else: + return get_parameter(fm, self_param_name) + + def update_fm_param(self, md_param_name, cal_new_fm_param_by_md_param): + if not ('query' in md_param_name or 'key' in md_param_name or 'value' in md_param_name): + matched_fm_param_ref = self.get_fm_matched_param_of_md_param(md_param_name) + matched_fm_param_ref.copy_(cal_new_fm_param_by_md_param) + elif 'bias' in md_param_name: + ss = md_param_name.split('.') + fm = self.models_dict['fm'] + + fm_qkv_name = '.'.join(ss[0: -1]) + '.fc' + fm_qkv = get_module(fm, fm_qkv_name) + + fm_qkv.bias.data.copy_(cal_new_fm_param_by_md_param) + + else: + new_fm_attn_weight, new_fm_lora_weight = torch.chunk(cal_new_fm_param_by_md_param, 2, 0) + + ss = md_param_name.split('.') + fm = self.models_dict['fm'] + + # update task-agnostic parameters + fm_qkv_name = '.'.join(ss[0: -1]) + '.fc' + fm_qkv = get_module(fm, fm_qkv_name) + fm_qkv.weight.data.copy_(new_fm_attn_weight) + + # update task-specific parameters + fm_abs_name = '.'.join(ss[0: -1]) + '.ab' + fm_abs = get_module(fm, fm_abs_name) + fm_abs._mul_lora_weight.data.copy_(new_fm_lora_weight) # TODO: this will not be applied in inference! + + def get_md_matched_param_of_fm_param(self, fm_param_name): + return super().get_md_matched_param_of_fm_param(fm_param_name) + + def get_md_matched_param_of_sd_param(self, sd_param_name): + # raise NotImplementedError + + # only between qkv.weight, norm.weight/bias + self_param_name = sd_param_name + md = self.models_dict['md'] + if any([k in self_param_name for k in ['fbs', 'ab', 'embeddings']]): + return None + + p = get_parameter(self.models_dict['sd'], self_param_name) + if p.dim() == 0: + return None + elif p.dim() == 1 and 'LayerNorm' in self_param_name and 'weight' in self_param_name: + return get_parameter(md, self_param_name) + + if 'classifier' in self_param_name: + return get_parameter(md, self_param_name) + elif 'static_channel_attention' in self_param_name: + return None + + + # 1. xx.qkv.to_qkv.yy to xx.qkv.qkv.aa and xx.qkv.abs.zz + if ('query' in self_param_name or 'key' in self_param_name or \ + 'value' in self_param_name) and ('weight' in self_param_name): + + + return get_parameter(md, self_param_name) # NOTE: no fbs in qkv! + + elif ('query' in self_param_name or 'key' in self_param_name or \ + 'value' in self_param_name) and ('bias' in self_param_name): + + + return get_parameter(md, self_param_name) + elif 'intermediate.dense.0.weight' in self_param_name: + fm_param_name = '.'.join(self_param_name.split('.')[0: -2]) + '.linear.weight' + return get_parameter(md, fm_param_name) + + elif 'intermediate.dense.0.bias' in self_param_name: + fm_param_name = '.'.join(self_param_name.split('.')[0: -2]) + '.linear.bias' + return get_parameter(md, fm_param_name) + + elif 'output.dense' in self_param_name and 'weight' in self_param_name: + fm_param_name = self_param_name + return get_parameter(md, fm_param_name) + + else: + return get_parameter(md, self_param_name) + + def get_task_head_params(self): + head = get_module(self.models_dict['sd'], 'classifier') + return list(head.parameters()) + +class SeClsOnlineFeatAlignModel(OnlineFeatAlignModel): + def get_trained_params(self): # TODO: elastic fm only train a part of params + #qkv_and_norm_params = [p for n, p in self.models_dict['main'].named_parameters() if 'attention.attention.projection_query' in n or 'attention.attention.projection_key' in n or 'attention.attention.projection_value' in n or 'intermediate.dense' in n or 'output.dense' in n] + qkv_and_norm_params = [p for n, p in self.models_dict['main'].named_parameters()] + return qkv_and_norm_params + + def get_feature_hook(self) -> LayerActivation2: + return LayerActivation2(get_module(self.models_dict['main'], 'classifier')) + + def forward_to_get_task_loss(self, x, y): + self.to_train_mode() + return F.cross_entropy(self.infer(x), y) + + def get_mmd_loss(self, f1, f2): + common_shape = min(f1.shape[0], f2.shape[0]) + f1 = f1.view(f1.shape[0], -1) + f2 = f2.view(f2.shape[0], -1) + f1 = f1[:common_shape,:] + f2 = f2[:common_shape,:] + return mmd_rbf(f1, f2) + + def infer(self, x, *args, **kwargs): + return self.models_dict['main'](**x) + + def get_accuracy(self, test_loader, *args, **kwargs): + _d = test_loader.dataset + from data import build_dataloader, split_dataset + if _d.__class__.__name__ == '_SplitDataset' and _d.underlying_dataset.__class__.__name__ == 'MergedDataset': # necessary for CL + print('\neval on merged datasets') + + merged_full_dataset = _d.underlying_dataset.datasets + ratio = len(_d.keys) / len(_d.underlying_dataset) + + if int(len(_d) * ratio) == 0: + ratio = 1. + # print(ratio) + # bs = + # test_loaders = [build_dataloader(split_dataset(d, min(max(test_loader.batch_size, int(len(d) * ratio)), len(d)))[0], # TODO: this might be overlapped with train dataset + # min(test_loader.batch_size, int(len(d) * ratio)), + # test_loader.num_workers, False, None) for d in merged_full_dataset] + + test_loaders = [] + for d in merged_full_dataset: + n = int(len(d) * ratio) + if n == 0: + n = len(d) + sub_dataset = split_dataset(d, min(max(test_loader.batch_size, n), len(d)))[0] + loader = build_dataloader(sub_dataset, min(test_loader.batch_size, n), test_loader.num_workers, False, None) + test_loaders += [loader] + + accs = [self.get_accuracy(loader) for loader in test_loaders] + print(accs) + return sum(accs) / len(accs) + + acc = 0 + sample_num = 0 + + self.to_eval_mode() + + with torch.no_grad(): + pbar = tqdm.tqdm(enumerate(test_loader), total=len(test_loader), dynamic_ncols=True, leave=False) + for batch_index, (x, y) in pbar: + for k, v in x.items(): + if isinstance(v, torch.Tensor): + x[k] = v.to(self.device) + y = y.to(self.device) + output = self.infer(x) + pred = F.softmax(output, dim=1).argmax(dim=1) + correct = torch.eq(pred, y).sum().item() + acc += correct + sample_num += len(y) + + # if batch_index == 0: + # print(pred, y) + + pbar.set_description(f'cur_batch_total: {len(y)}, cur_batch_correct: {correct}, ' + f'cur_batch_acc: {(correct / len(y)):.4f}') + + acc /= sample_num + return acc + +elasticfm_model = ElasticDNN_SeClsOnlineModel('secls', init_online_model( + 'new_impl/nlp/roberta/sentiment-classification/results/cls_md_w_fbs_index.py/20240111/999998-203106-results/models/fm_best.pt', + 'new_impl/nlp/roberta/sentiment-classification/results/cls_md_w_fbs_index.py/20240111/999998-203106-results/models/md_best.pt', + 'cls', __file__ +), device, { + 'md_to_fm_alpha': 0.01, + 'fm_to_md_alpha': 0.1 +}) + +da_alg = FeatAlignAlg +from utils.dl.common.lr_scheduler import get_linear_schedule_with_warmup +#from new_impl.cv.model import ClsOnlineFeatAlignModel +da_model = SeClsOnlineFeatAlignModel + +da_alg_hyp = { + 'HL5Domains-Nokia6610': { + 'train_batch_size': 32, + 'val_batch_size': 256, + 'num_workers': 8, + 'optimizer': 'AdamW', + 'optimizer_args': {'lr': 2e-7, 'betas': [0.9, 0.999], 'weight_decay': 0.01}, + 'scheduler': '', + 'scheduler_args': {}, + 'num_iters': 100, + 'val_freq': 20, + 'sd_sparsity':0.3, + 'feat_align_loss_weight': 1.0, + }, + 'HL5Domains-NikonCoolpix4300': { + 'train_batch_size': 32, + 'val_batch_size': 128, + 'num_workers': 8, + 'optimizer': 'AdamW', + 'optimizer_args': {'lr': 2e-7, 'betas': [0.9, 0.999], 'weight_decay': 0.01}, + 'scheduler': '', + 'scheduler_args': {}, + 'num_iters': 100, + 'val_freq': 20, + 'sd_sparsity':0.3, + 'feat_align_loss_weight': 1.0, + }, +} + + +elasticfm_da( + [app_name], + [scenario], + [elasticfm_model], + [da_alg], + [da_alg_hyp], + [da_model], + device, + settings, + __file__, + "results", +) diff --git a/new_impl/nlp/roberta/sentiment-classification/roberta-base/config.json b/new_impl/nlp/roberta/sentiment-classification/roberta-base/config.json new file mode 100644 index 0000000000000000000000000000000000000000..8db5e7ac5bfc9ec8b613b776009300fe3685d957 --- /dev/null +++ b/new_impl/nlp/roberta/sentiment-classification/roberta-base/config.json @@ -0,0 +1,21 @@ +{ + "architectures": [ + "RobertaForMaskedLM" + ], + "attention_probs_dropout_prob": 0.1, + "bos_token_id": 0, + "eos_token_id": 2, + "hidden_act": "gelu", + "hidden_dropout_prob": 0.1, + "hidden_size": 768, + "initializer_range": 0.02, + "intermediate_size": 3072, + "layer_norm_eps": 1e-05, + "max_position_embeddings": 514, + "model_type": "roberta", + "num_attention_heads": 12, + "num_hidden_layers": 12, + "pad_token_id": 1, + "type_vocab_size": 1, + "vocab_size": 50265 +} diff --git a/new_impl/nlp/roberta/sentiment-classification/roberta-base/dict.txt b/new_impl/nlp/roberta/sentiment-classification/roberta-base/dict.txt new file mode 100644 index 0000000000000000000000000000000000000000..9b4d2de40fa2cbeb2c6128cfd031b9872ef4d054 --- /dev/null +++ b/new_impl/nlp/roberta/sentiment-classification/roberta-base/dict.txt @@ -0,0 +1,50260 @@ +13 850314647 +262 800385005 +11 800251374 +284 432911125 +290 394899794 +286 386139013 +257 357878752 +287 311196488 +12 215156821 +329 155236946 +326 154060431 +319 147178919 +318 142591644 +447 130810923 +338 116498242 +351 114784681 +383 108664122 +373 100357189 +366 93880741 +379 93284459 +340 88803471 +355 85749070 +531 85009762 +247 82642284 +307 77095226 +82 76381845 +416 73380803 +422 71911149 +389 68628918 +423 67243391 +468 64317701 +25 63508661 +357 63001640 +339 61994245 +314 60989470 +465 56381137 +481 55817121 +281 55370942 +428 52404829 +8 49955136 +564 49278190 +407 49022194 +251 48828693 +345 46413707 +250 46095324 +511 42623671 +393 41629710 +484 41252315 +356 40985272 +475 40041980 +508 39889004 +517 36480426 +550 35941594 +587 34803895 +547 34523820 +546 33398226 +553 33091056 +543 32654778 +510 32035371 +663 32028126 +460 31691389 +530 31181535 +503 30862486 +635 30813519 +720 30660454 +607 30374808 +477 29369504 +706 29183313 +526 29041171 +14 28893906 +561 27738361 +470 26738514 +614 25458253 +618 24232023 +717 23994060 +673 23817299 +734 23792701 +625 23376942 +661 23220442 +317 22862326 +674 22516011 +632 22500762 +640 22453472 +621 22170426 +656 21469936 +612 21420897 +83 21318775 +679 21314775 +649 21268970 +851 21092011 +938 20404401 +655 20375026 +554 20334200 +584 20320611 +523 20315428 +644 20012607 +40 19422652 +588 19096246 +64 18759122 +617 18693984 +50 18238046 +26689 18079440 +606 17992787 +812 17864313 +6 17843244 +466 17817361 +534 17796224 +532 17532111 +352 17384084 +1 17279082 +611 17091775 +714 17025679 +30 16939428 +645 16677856 +72 16553037 +76 16327061 +651 15971344 +471 15879338 +783 15823492 +683 15819244 +736 15197650 +887 15053172 +784 14786686 +616 14556795 +705 14539133 +691 14309272 +1115 14176045 +26 14145184 +362 14098304 +464 14083981 +16 14033199 +1411 13989417 +1028 13787695 +878 13752356 +1664 13470232 +78 13378307 +1301 13160863 +703 13034870 +780 12998354 +597 12928745 +749 12878804 +852 12866041 +787 12811365 +810 12810008 +1141 12785466 +832 12685151 +981 12676060 +830 12643489 +770 12491952 +1510 12485834 +278 12398432 +513 12345382 +925 12242439 +880 12187173 +838 12095675 +866 12088892 +572 12004582 +1139 11932599 +502 11810743 +347 11778080 +1016 11554835 +1074 11537640 +775 11416721 +883 11147387 +1230 11002697 +835 10997448 +1135 10975044 +867 10945123 +788 10941163 +670 10932097 +1297 10878979 +785 10826031 +17 10797378 +983 10787263 +843 10673666 +259 10657207 +1941 10592731 +279 10574822 +845 10526221 +1110 10523541 +1363 10476406 +1011 10465302 +1285 10446380 +1201 10423577 +968 10348378 +743 10336013 +772 10297739 +1622 10289758 +766 10280263 +2177 10243413 +1181 10234334 +642 10188179 +276 10110644 +815 10077564 +1088 10066642 +2864 10065012 +1218 10051426 +514 10027697 +991 9981059 +881 9920784 +604 9911573 +922 9903273 +892 9899715 +4 9886396 +311 9824882 +777 9788574 +1910 9785844 +360 9705921 +400 9632736 +467 9594120 +821 9549150 +884 9547397 +760 9515151 +1390 9514008 +836 9399858 +88 9399371 +1306 9350994 +350 9326094 +750 9317800 +739 9296956 +910 9295771 +268 9233925 +406 9191046 +1022 9185932 +583 9178621 +509 9120375 +327 9057596 +718 8963492 +995 8936939 +636 8882717 +399 8811370 +826 8792653 +765 8755010 +1440 8704406 +828 8681852 +1029 8668925 +761 8595175 +260 8550153 +68 8521820 +1026 8495454 +1037 8482028 +20 8478672 +18 8372937 +1499 8372574 +371 8359700 +1644 8353078 +32 8336893 +890 8325716 +1119 8309982 +886 8287585 +263 8275838 +309 8275603 +337 8268937 +84 8260458 +1111 8203580 +994 8202575 +272 8193169 +261 8189103 +767 8122128 +390 8069108 +1375 7988935 +1597 7984150 +989 7938818 +73 7922334 +364 7880395 +1107 7805773 +1992 7800278 +283 7777203 +402 7732480 +3217 7712997 +376 7593883 +1266 7589093 +976 7577523 +1194 7566250 +900 7556294 +727 7550625 +1320 7507098 +292 7495866 +77 7470439 +1282 7450955 +1641 7411391 +1171 7409099 +1114 7363929 +1081 7347040 +15 7312426 +367 7306406 +807 7279240 +1160 7272825 +1936 7262915 +274 7253218 +3431 7220578 +299 7218583 +3635 7152871 +3860 7135265 +71 7117156 +1353 7113713 +1392 7090351 +1204 7074121 +3321 7040732 +1043 7037776 +779 7012062 +370 6995545 +19 6983878 +3583 6974965 +898 6966637 +1864 6959506 +711 6952288 +905 6939061 +520 6938890 +582 6920647 +1364 6908579 +1578 6875577 +1105 6855797 +1295 6829761 +1002 6812464 +1256 6769157 +1966 6727321 +657 6641494 +737 6624794 +1104 6593949 +494 6588847 +2997 6571290 +256 6561739 +7303 6545519 +0 6525880 +89 6511169 +74 6481093 +1812 6451687 +2173 6415053 +1448 6412987 +1524 6397024 +1321 6394244 +1584 6392878 +282 6358319 +81 6351445 +1592 6336754 +1705 6331987 +973 6315804 +1234 6313784 +1748 6306370 +449 6287674 +1318 6264091 +1271 6240689 +34 6237906 +1053 6231567 +1123 6220759 +1165 6216478 +1839 6189790 +306 6174365 +1227 6170682 +271 6158328 +2087 6143528 +804 6141406 +1365 6119704 +790 6119691 +1222 6097096 +1528 6093784 +860 6086478 +1718 6080393 +1755 6062347 +304 6058377 +1367 6044404 +418 6021291 +1178 5961986 +273 5944078 +2258 5884266 +921 5875428 +2368 5843037 +1049 5834912 +1444 5825616 +1550 5810506 +1613 5807417 +1625 5806489 +1933 5804564 +3909 5804009 +1315 5793456 +1263 5781210 +412 5780871 +1294 5756002 +1243 5755617 +440 5703257 +288 5696335 +923 5686348 +33 5683530 +4283 5662615 +1542 5657499 +1466 5655969 +2520 5626252 +1737 5617548 +1239 5613802 +1893 5604800 +3502 5592880 +1231 5592559 +805 5586720 +23 5579916 +1422 5562000 +1957 5554673 +21 5545853 +1223 5537815 +1339 5525740 +1439 5518540 +270 5516122 +22 5511292 +1406 5508724 +1751 5500821 +1497 5473031 +1310 5460960 +2237 5456445 +2254 5424602 +3418 5378471 +1366 5362221 +265 5361225 +1541 5359232 +67 5328572 +1637 5322515 +1903 5319711 +1973 5285283 +2938 5283708 +1057 5281356 +1568 5274805 +321 5273108 +2756 5236169 +1830 5223345 +1770 5222102 +65 5208299 +1244 5204410 +1180 5203669 +2098 5169445 +1730 5168645 +2056 5168496 +3349 5156053 +2055 5138614 +2807 5130949 +1101 5123031 +66 5103752 +1816 5093006 +1400 5076411 +1498 5071579 +1642 5055917 +1989 5044992 +1290 5034040 +2643 5023350 +2097 5022728 +1762 5015331 +44 5015012 +479 5008250 +1775 5005597 +2706 5005225 +1909 4997835 +1866 4990678 +1566 4981122 +1336 4950527 +757 4941775 +2063 4937397 +2648 4929257 +293 4925771 +1464 4911409 +2184 4905422 +75 4894914 +392 4889056 +1487 4877439 +1064 4865397 +24 4854296 +1080 4852641 +569 4850805 +1971 4849650 +1605 4847284 +1182 4846260 +1938 4828745 +857 4805157 +1535 4772487 +285 4766512 +1176 4764682 +966 4760238 +2277 4743915 +764 4731348 +1377 4728044 +1479 4720640 +1539 4714734 +1085 4700915 +1811 4696785 +2274 4657189 +869 4652756 +45 4649060 +1099 4644445 +1394 4638480 +1280 4637662 +3000 4618471 +1577 4618338 +544 4614094 +2805 4608260 +35 4606393 +2351 4602990 +1629 4586377 +1661 4584310 +2003 4567755 +49 4546496 +1478 4546419 +2795 4542206 +2828 4536638 +1248 4526225 +1593 4511113 +69 4502347 +2457 4489997 +1511 4484539 +1881 4472447 +47 4460868 +1708 4455874 +1097 4450969 +1551 4445924 +1660 4433465 +1785 4424736 +1627 4412038 +1445 4401529 +2594 4393865 +1719 4389889 +1649 4380458 +2444 4375390 +4287 4371881 +417 4370421 +716 4365322 +1168 4364296 +1735 4360877 +1621 4331683 +2233 4330036 +3249 4325097 +42 4320291 +1276 4314178 +1829 4314165 +1884 4312560 +38 4306679 +2555 4304404 +2084 4296432 +2151 4287967 +1688 4285173 +2831 4280062 +1342 4276707 +1270 4239023 +555 4236623 +1327 4234882 +2139 4227635 +1467 4219535 +2045 4208129 +2714 4198810 +303 4195216 +1771 4185532 +2901 4181081 +2077 4179380 +1863 4178336 +1965 4175165 +2067 4175032 +1716 4169486 +2651 4155215 +2267 4147302 +2607 4145837 +1964 4133545 +1462 4126396 +1978 4126179 +1972 4121510 +1410 4119035 +1679 4106021 +51 4105433 +1871 4105196 +2406 4102924 +2551 4097217 +2008 4097102 +1853 4093083 +70 4092580 +2293 4087207 +2324 4083845 +43 4083418 +1337 4082770 +1813 4079395 +1695 4077362 +960 4077221 +264 4076617 +2688 4072140 +1183 4070745 +1414 4064159 +1474 4057078 +2282 4053024 +3414 4042308 +1430 4037596 +3035 4031658 +1103 4018655 +2059 4015508 +2080 4011218 +2969 4005397 +1919 4000144 +1969 3997080 +316 3993493 +1459 3984707 +1521 3978369 +37 3969953 +1675 3968119 +3009 3965772 +996 3965252 +1596 3943972 +2263 3941497 +3457 3938197 +1450 3937663 +86 3925608 +2058 3919358 +1636 3917053 +1804 3911665 +1429 3909332 +1757 3906464 +354 3891128 +405 3889614 +3176 3888837 +1877 3882885 +1576 3878715 +2893 3873729 +2252 3872753 +1281 3863057 +1254 3862011 +301 3858937 +1048 3852378 +3203 3851712 +2159 3847206 +1626 3842112 +324 3832573 +1760 3832298 +1169 3818301 +2739 3816048 +1687 3814765 +1595 3811813 +1517 3803324 +2260 3791037 +1693 3787455 +1262 3785788 +2102 3779927 +291 3777762 +1923 3777288 +1700 3776768 +2157 3771717 +1378 3757930 +2732 3755186 +79 3754633 +1854 3745778 +3269 3737875 +1502 3737616 +685 3723444 +4200 3719859 +1865 3719356 +128 3715003 +1402 3710786 +2168 3703789 +1986 3699485 +1867 3696280 +2026 3695382 +1683 3694355 +2961 3691575 +1842 3680139 +929 3678416 +2489 3665779 +1052 3661007 +396 3659796 +3329 3643884 +2669 3638778 +1862 3622381 +3452 3605249 +3794 3602291 +2111 3590156 +2046 3587528 +2957 3581398 +1913 3580361 +1441 3577048 +1241 3568130 +46 3565250 +2811 3562952 +2278 3561460 +1998 3550042 +461 3548528 +1744 3532659 +1975 3526648 +2291 3521569 +3056 3518738 +2904 3518633 +1752 3511388 +1900 3510560 +2626 3506312 +1654 3504513 +385 3502364 +2745 3487380 +2057 3487072 +3136 3485766 +7955 3485171 +4139 3481632 +2415 3480433 +2148 3480049 +1628 3467067 +2071 3466219 +2107 3463315 +940 3460357 +1598 3457691 +258 3455533 +1575 3454361 +2826 3452135 +2716 3449165 +3985 3445703 +85 3445461 +1987 3443747 +3598 3439871 +3352 3431656 +2478 3424520 +333 3421332 +246 3418219 +2620 3415717 +1212 3412196 +2450 3409727 +1247 3405540 +1912 3399689 +36 3395391 +346 3391001 +3426 3380470 +3298 3379545 +3292 3377200 +2250 3371380 +2440 3369691 +3061 3367419 +39 3363104 +978 3353736 +1802 3350527 +2431 3348906 +3071 3340128 +2253 3337972 +2494 3334848 +609 3333865 +2310 3329148 +986 3328812 +2635 3325356 +3437 3320853 +2292 3319741 +2823 3308131 +1588 3303360 +269 3302371 +275 3284415 +60 3282646 +2428 3276582 +1918 3276387 +2615 3273427 +2472 3272517 +1690 3267675 +410 3265844 +2678 3262749 +2106 3260749 +2354 3251238 +2717 3247356 +678 3244526 +1109 3242666 +3334 3241700 +3451 3238451 +320 3236458 +3230 3233294 +3389 3229315 +2166 3227294 +1611 3224985 +1994 3213613 +430 3209260 +2986 3199943 +1790 3194716 +1438 3193856 +4784 3192749 +1781 3170903 +302 3166428 +2227 3162561 +54 3145229 +2693 3138924 +1393 3138049 +2597 3137970 +2482 3137124 +3034 3122439 +1946 3121857 +2863 3119047 +3267 3115876 +2041 3113770 +1743 3107914 +2476 3105231 +388 3102434 +300 3100235 +3186 3098789 +1729 3098376 +2488 3094662 +5018 3092842 +4058 3079283 +2156 3078111 +52 3074167 +3096 3072323 +1468 3071877 +2497 3070835 +2793 3050336 +3427 3047066 +1630 3040837 +3284 3037800 +3624 3034708 +2650 3033943 +2785 3033180 +1807 3027961 +3645 3026379 +2691 3025436 +3106 3024747 +3037 3023165 +3759 3023164 +312 3020879 +1767 3018684 +2526 3018183 +666 3015679 +3139 3012306 +3085 3009667 +2223 3002610 +4041 3002353 +2712 3001744 +1838 2997522 +2048 2983869 +2854 2981556 +2534 2972131 +308 2969299 +2646 2967019 +3016 2965071 +3337 2960427 +3187 2957831 +4912 2956818 +3331 2956176 +1643 2956098 +2722 2953729 +2932 2951114 +2422 2950537 +2399 2948398 +500 2946582 +4039 2945677 +3961 2944538 +2222 2943764 +3078 2943739 +4275 2942029 +1724 2934719 +911 2931322 +3296 2930626 +384 2925764 +2319 2924706 +1238 2912540 +1911 2911206 +53 2910401 +2005 2910213 +2923 2909079 +1303 2908146 +4536 2904452 +2921 2898494 +3530 2896507 +343 2894182 +575 2892577 +3058 2891202 +277 2889780 +323 2886056 +710 2881312 +660 2874230 +1949 2873478 +3250 2868743 +225 2861798 +41 2858852 +1808 2848588 +1021 2846040 +3773 2842914 +7713 2841920 +540 2838877 +2137 2837279 +2750 2836122 +3271 2833311 +2994 2832696 +397 2832081 +2174 2831245 +2630 2825882 +1073 2823768 +378 2822150 +2491 2819311 +403 2817676 +2540 2811122 +2060 2808168 +2214 2807667 +2242 2804699 +3554 2801970 +266 2800975 +3442 2799863 +5544 2795504 +1682 2795443 +1351 2777650 +297 2776601 +3155 2770111 +2050 2768526 +3466 2759754 +1544 2759525 +993 2754965 +3340 2752396 +8591 2751808 +1255 2750444 +1895 2750214 +3015 2746600 +3125 2744902 +3945 2744846 +6426 2744124 +2897 2740354 +1309 2739832 +959 2737933 +2822 2737646 +1368 2733555 +2042 2730078 +374 2728295 +3006 2714274 +2245 2700521 +2928 2694744 +2872 2687504 +4896 2686827 +4297 2685685 +2766 2685288 +444 2682283 +2888 2681984 +1200 2679658 +2975 2678829 +377 2675721 +1988 2675064 +2523 2673705 +1583 2671163 +1024 2667070 +415 2666262 +3576 2658993 +2119 2657291 +2647 2648808 +3227 2648233 +1997 2646862 +4081 2645756 +4094 2645293 +1633 2637801 +1917 2637232 +2276 2635825 +2492 2634522 +1312 2634263 +2839 2633915 +2592 2632902 +3662 2624861 +3224 2624698 +1766 2624083 +3663 2624035 +1745 2621047 +5 2620736 +2300 2619855 +4664 2619338 +3430 2619137 +2130 2618208 +6184 2618030 +3687 2611608 +13130 2607739 +2637 2602497 +2622 2597101 +3700 2596588 +2435 2591941 +2158 2587673 +2279 2584888 +2506 2577787 +3724 2574566 +2950 2573209 +2460 2568568 +2125 2566267 +2861 2562749 +1134 2549917 +5454 2544616 +3751 2536696 +1858 2535706 +2579 2530192 +1826 2529534 +2608 2528860 +2681 2527523 +56 2526960 +3814 2525489 +4332 2524158 +2735 2523828 +3367 2523419 +2272 2516165 +3756 2511014 +2585 2509794 +5041 2503584 +4248 2503218 +2802 2502456 +2180 2500659 +3482 2499158 +3899 2496197 +2666 2495174 +395 2490074 +368 2486179 +1976 2484836 +2773 2481413 +669 2475721 +448 2470404 +1314 2468787 +1175 2466968 +3052 2465830 +3491 2465693 +55 2458697 +305 2457793 +2496 2455621 +2241 2454504 +1210 2453199 +2031 2450641 +3111 2447239 +2568 2446982 +7781 2446876 +1635 2445064 +2582 2444049 +2613 2443100 +3195 2441989 +5079 2432713 +2211 2428055 +3234 2426818 +4037 2426428 +1549 2425595 +5991 2421705 +4495 2417713 +952 2416570 +267 2415840 +2458 2414786 +328 2414598 +3790 2413453 +2641 2411736 +1296 2408790 +3199 2407660 +3072 2407258 +763 2402573 +2742 2402326 +4640 2400825 +1907 2399123 +654 2395466 +2911 2394784 +3931 2392276 +3818 2385503 +4346 2385039 +3119 2384203 +31 2383468 +1492 2381026 +3397 2380456 +3484 2379083 +330 2378895 +4706 2377588 +2251 2376885 +2479 2374155 +3053 2371784 +5939 2368766 +1388 2368606 +1692 2366880 +1908 2363611 +4542 2356967 +3596 2356312 +1122 2352389 +5003 2349430 +2962 2349359 +1607 2349127 +3047 2347321 +2627 2346853 +3025 2342305 +2995 2337450 +2835 2335936 +1004 2333657 +3214 2332768 +2029 2332229 +13440 2330317 +1561 2324920 +3074 2315814 +380 2312070 +515 2311421 +365 2310632 +3382 2306041 +363 2305310 +3160 2304973 +296 2303562 +435 2300111 +3512 2297054 +3747 2295650 +1334 2294588 +4281 2294310 +2614 2292185 +2524 2284943 +3394 2281882 +3095 2281714 +2147 2281124 +2187 2279131 +2855 2275319 +2702 2272741 +3517 2271280 +325 2268773 +3520 2268531 +1065 2264589 +3215 2261296 +4395 2260201 +2652 2259754 +1120 2259547 +648 2258982 +4436 2257188 +2089 2255914 +2209 2255859 +4969 2249342 +3022 2248989 +4530 2248944 +2330 2247322 +3261 2246707 +1532 2245244 +12042 2243318 +2270 2243237 +1657 2239438 +2846 2238923 +3999 2237519 +3845 2237334 +2271 2233955 +2126 2233086 +499 2231417 +1659 2229762 +5193 2228394 +3688 2226618 +1382 2226353 +446 2225805 +818 2224123 +2092 2222423 +3623 2220823 +5373 2220082 +2321 2219693 +5057 2219662 +1526 2219349 +5169 2218821 +2912 2217212 +3220 2216122 +3315 2215806 +2808 2214953 +2365 2214929 +2628 2212471 +3805 2211616 +2121 2211232 +1560 2204752 +3259 2204036 +3240 2203146 +2634 2202115 +3656 2200682 +2683 2199255 +3767 2197871 +2612 2193603 +1138 2190750 +3181 2189669 +4193 2189626 +3162 2186721 +2239 2185546 +2988 2185413 +2589 2185214 +1720 2185135 +2192 2184857 +4387 2184827 +4038 2180149 +4492 2178553 +1249 2178508 +3599 2177234 +3988 2176292 +4519 2175412 +2010 2173733 +3965 2173661 +4149 2170484 +3833 2170048 +3017 2169734 +1100 2168903 +4884 2168582 +349 2167597 +2035 2164690 +4040 2163330 +3407 2162686 +3415 2161099 +680 2159475 +1399 2158929 +4661 2157800 +1228 2155490 +551 2154941 +87 2154822 +4376 2154446 +559 2153394 +2392 2153055 +315 2150386 +2342 2150379 +3936 2150034 +1639 2148662 +2837 2143071 +358 2142250 +5153 2141503 +3793 2139387 +2940 2139279 +3126 2139153 +8358 2138790 +1477 2137623 +2720 2137372 +2138 2135247 +5085 2134103 +3957 2132520 +662 2131237 +3432 2125446 +2663 2125055 +8428 2121327 +2408 2121234 +1051 2121150 +3012 2120925 +3740 2120211 +3362 2118913 +2877 2111347 +820 2109304 +1195 2108223 +528 2107435 +2297 2104723 +1956 2104400 +2990 2101876 +5567 2098776 +62 2098771 +2312 2098237 +1570 2097507 +4086 2094834 +1738 2094385 +3142 2094215 +4505 2092752 +1031 2081290 +797 2079864 +2900 2079818 +1157 2079615 +3277 2079046 +3492 2078225 +1803 2078060 +4466 2077826 +4445 2076000 +5953 2074891 +4172 2072344 +2383 2070424 +3274 2062552 +2405 2061430 +474 2059389 +4539 2058329 +5371 2054600 +2753 2053126 +3377 2051234 +2884 2050465 +313 2048274 +1437 2048095 +1495 2047894 +1044 2047797 +1672 2046882 +4773 2046230 +3128 2045207 +4444 2043458 +439 2043235 +12637 2038256 +5692 2037358 +504 2035309 +3307 2032332 +2323 2031475 +2495 2028884 +4196 2027995 +341 2026813 +4176 2023899 +5834 2020359 +4744 2020302 +2563 2016917 +2882 2013222 +505 2010517 +3707 2009054 +1612 2007764 +4652 2006805 +2687 2006766 +5342 1994135 +3670 1992183 +4375 1990482 +2761 1988748 +4756 1987768 +3403 1986706 +2266 1985660 +3066 1985309 +129 1983048 +4481 1981282 +4354 1979172 +2033 1978125 +4576 1977385 +1943 1973375 +2370 1972674 +2486 1971043 +3090 1969680 +2810 1969527 +5401 1967900 +4381 1967895 +3800 1966998 +641 1966963 +2776 1966928 +3611 1965109 +6567 1965030 +3710 1963705 +803 1963552 +1332 1963452 +1600 1962893 +5442 1959629 +2936 1959617 +2723 1956891 +57 1956274 +2331 1955550 +2728 1955126 +4266 1954189 +3708 1952903 +4155 1951761 +3236 1951553 +2011 1949390 +3893 1944470 +3715 1943834 +3501 1942268 +3641 1942249 +3967 1941157 +5695 1939245 +3946 1939135 +1848 1938862 +2982 1935927 +3081 1935780 +2842 1935540 +3393 1929631 +4409 1927034 +533 1926581 +1208 1925558 +6123 1924601 +3328 1919903 +3073 1919157 +5478 1915260 +3690 1915130 +992 1914387 +3519 1914014 +3677 1913098 +4479 1909848 +5555 1908826 +353 1908475 +2952 1907503 +1374 1906721 +2972 1906704 +3151 1906344 +2298 1905945 +5047 1905773 +2407 1905690 +298 1904852 +80 1903294 +1040 1903278 +4590 1902254 +1833 1899229 +5595 1897899 +4251 1897347 +2610 1895735 +2724 1894339 +3626 1892750 +4244 1890732 +4318 1889199 +4957 1886758 +2775 1885004 +5095 1881967 +4721 1880372 +7198 1879205 +3892 1878736 +1558 1875025 +3884 1874312 +3942 1869131 +3206 1868585 +2499 1868244 +2562 1867987 +1507 1866658 +4838 1862252 +289 1862164 +1645 1859225 +2700 1855499 +1754 1855200 +3772 1851086 +4888 1850666 +4045 1848988 +3867 1847435 +157 1843867 +4493 1842927 +2619 1838354 +4786 1836011 +1398 1834121 +3088 1832988 +4120 1831679 +2695 1831610 +5665 1829988 +3381 1828595 +2427 1828369 +4452 1825877 +4477 1823113 +2974 1821399 +6672 1821145 +1814 1821079 +5466 1821016 +4639 1820572 +1265 1819995 +2314 1818041 +1008 1816897 +437 1815421 +3423 1814262 +3245 1812767 +3650 1811355 +2503 1810984 +1728 1810732 +2116 1810704 +8872 1806001 +2332 1805620 +2968 1804542 +1061 1803718 +4581 1802885 +1525 1802513 +3888 1799212 +2925 1794822 +2727 1794109 +1302 1794082 +4560 1793094 +3114 1791585 +3513 1791504 +709 1790178 +4783 1789889 +4488 1789224 +2832 1789080 +2746 1788885 +5652 1787699 +331 1785106 +4865 1784176 +3739 1782930 +4586 1782099 +3877 1780375 +5059 1780266 +2869 1779960 +4485 1778625 +6130 1774579 +3940 1772616 +4271 1768597 +2985 1768596 +4918 1768371 +5030 1768004 +220 1767607 +280 1766245 +1872 1766073 +3706 1765293 +1545 1759778 +5267 1758636 +3761 1757768 +5491 1757560 +2104 1756508 +7313 1756194 +4054 1755775 +2574 1753800 +3667 1753746 +3651 1753739 +4405 1752238 +571 1751391 +4308 1751021 +5437 1750489 +1944 1749114 +1355 1748662 +4274 1746718 +8278 1743712 +578 1742168 +2694 1741205 +2656 1738365 +2636 1738002 +4513 1737002 +2185 1736027 +5449 1728990 +2813 1728933 +2993 1728096 +2587 1728026 +615 1727522 +4881 1726170 +4776 1724711 +2565 1724643 +2726 1722884 +4999 1718878 +3098 1718690 +3685 1717832 +2987 1717445 +1559 1716265 +10205 1715772 +3486 1713237 +427 1712990 +2583 1712089 +4995 1711510 +3871 1710985 +5134 1710681 +4902 1710660 +4602 1710486 +620 1710125 +5537 1709921 +3170 1709734 +1892 1706520 +3173 1705945 +2073 1704583 +5011 1703887 +2346 1703108 +786 1702332 +4059 1699595 +1844 1697348 +295 1696098 +4473 1694758 +4696 1694438 +4280 1692697 +3241 1692185 +3067 1691819 +5694 1688571 +563 1687198 +4987 1687182 +2000 1686158 +1982 1680689 +4317 1679080 +2800 1678829 +2743 1676341 +3782 1676203 +4136 1675550 +4141 1675316 +3011 1674431 +404 1671943 +4508 1671300 +3848 1671224 +6341 1671141 +1486 1670636 +3338 1670469 +4394 1670223 +3357 1669888 +3226 1669751 +6557 1668766 +1359 1668112 +2081 1664853 +2364 1664159 +3146 1663984 +1257 1663233 +2193 1661698 +3294 1659449 +1148 1658760 +411 1658023 +874 1655097 +2219 1654452 +1143 1654129 +4427 1653889 +5583 1651423 +3954 1651313 +2779 1651054 +3216 1650840 +5396 1650681 +2150 1649743 +3127 1647955 +4373 1645141 +3164 1644786 +1433 1644622 +5201 1644557 +2423 1643559 +2672 1641242 +598 1640066 +1869 1639116 +3926 1637219 +6956 1635903 +4390 1635697 +1485 1635611 +6180 1633463 +4186 1632566 +11033 1632005 +10575 1631124 +5398 1630774 +4152 1630130 +6523 1629814 +2708 1628267 +3371 1627540 +4842 1624633 +3807 1624226 +1415 1621531 +414 1620770 +3386 1620487 +5180 1618946 +4956 1618827 +3033 1618817 +4854 1618654 +1395 1617375 +4418 1616971 +3275 1614077 +496 1613939 +4367 1612988 +3115 1611458 +4138 1611246 +707 1609966 +3941 1609655 +943 1608211 +4689 1605650 +541 1602489 +1220 1600656 +5856 1600558 +3465 1599819 +2014 1599485 +2642 1597905 +361 1597790 +3804 1596713 +4930 1595012 +4656 1594232 +4032 1591856 +2094 1591768 +4486 1590377 +3850 1589189 +3417 1587932 +4068 1587210 +6484 1587158 +3573 1586303 +751 1584930 +5273 1584119 +1001 1582810 +4511 1582438 +1350 1582111 +5682 1581323 +5701 1579275 +3750 1578369 +1215 1578106 +1288 1575560 +6443 1574356 +2456 1574032 +4305 1572178 +6786 1570930 +1722 1567616 +5070 1564834 +2391 1563465 +5229 1561859 +4992 1560941 +5828 1560570 +689 1560524 +1000 1560250 +3469 1559413 +3 1558890 +375 1558286 +4789 1557386 +3288 1557204 +5361 1556430 +829 1555089 +602 1553100 +6502 1552435 +3869 1550400 +482 1550363 +3769 1550026 +7 1548704 +695 1548541 +2221 1548436 +3615 1544415 +1891 1544125 +6186 1543528 +1129 1542997 +3741 1540855 +3412 1539970 +2198 1539205 +6193 1536595 +4009 1535736 +3923 1530812 +1879 1530386 +4137 1528607 +4502 1527678 +2513 1527637 +506 1527207 +5717 1527072 +4796 1526999 +5611 1525342 +4438 1524952 +7324 1524876 +5103 1524376 +3932 1523931 +4983 1523249 +2644 1522609 +3354 1522133 +4890 1521795 +3730 1521634 +42159 1521590 +1077 1521212 +1203 1518704 +5531 1515641 +4606 1515536 +3218 1515150 +6745 1513200 +5006 1513115 +5890 1512226 +5052 1511945 +4970 1510725 +6714 1508429 +4030 1506853 +1245 1505561 +3675 1504974 +4426 1504340 +3375 1503790 +3177 1503516 +382 1502511 +39711 1501599 +336 1500357 +1961 1499155 +3504 1498266 +5298 1496420 +732 1496386 +4752 1496009 +2879 1495099 +4260 1494843 +1041 1494107 +4746 1494014 +2556 1493772 +2074 1493385 +8734 1492850 +4979 1492643 +5510 1489374 +3521 1488249 +3812 1487987 +2989 1487864 +1537 1487614 +4219 1485195 +6686 1483477 +2504 1482854 +3584 1481993 +4568 1481365 +3421 1481356 +4237 1481211 +4814 1480648 +334 1479315 +3439 1479269 +3002 1477903 +1417 1477126 +930 1475368 +322 1474128 +1546 1473723 +4647 1473119 +488 1473092 +4073 1473078 +7024 1472792 +1531 1472346 +1797 1471634 +3625 1471454 +2409 1470321 +2196 1468500 +1663 1466759 +5672 1466160 +3689 1465868 +1981 1465495 +2858 1464549 +5199 1464208 +6916 1463903 +3574 1462903 +525 1461357 +3301 1459805 +3341 1459758 +1805 1458557 +2677 1458359 +6821 1458176 +3443 1458132 +4523 1457686 +1821 1456407 +8064 1455853 +2485 1453972 +7648 1453898 +8549 1453728 +3701 1452597 +344 1452384 +5471 1451483 +434 1451045 +624 1450197 +386 1449819 +719 1449119 +3859 1448585 +3092 1447520 +5166 1447492 +5323 1447286 +4585 1446385 +560 1444813 +4056 1444800 +7530 1442047 +3049 1440630 +5399 1440467 +1810 1439368 +39883 1438860 +4619 1438062 +6047 1437307 +4100 1435643 +4403 1435094 +6233 1434841 +4875 1431868 +2215 1430928 +518 1430530 +6798 1430070 +4379 1429963 +2191 1429101 +5212 1428466 +5926 1426982 +3726 1426439 +5127 1425739 +4608 1425400 +3770 1423320 +684 1423301 +3399 1423143 +549 1422979 +1471 1422620 +1326 1422545 +4034 1422191 +4414 1421600 +5260 1421418 +3285 1421044 +4671 1419983 +4388 1419806 +2329 1419508 +3117 1418665 +2512 1418078 +3614 1417918 +5296 1417278 +4380 1416914 +4691 1415979 +1242 1415603 +1042 1415013 +3094 1413700 +3059 1413465 +4019 1411855 +5658 1410605 +4101 1410072 +14420 1409152 +2576 1408914 +7229 1406591 +6095 1406131 +3729 1402106 +6168 1401550 +5535 1401292 +5348 1399444 +3131 1399224 +3050 1399104 +1108 1399004 +4769 1398370 +516 1395487 +5802 1392661 +671 1392187 +5175 1391611 +12216 1390973 +2141 1390969 +3764 1390483 +5158 1389181 +4504 1386980 +2546 1384925 +4301 1384339 +10767 1383934 +6011 1383767 +1793 1383582 +4809 1380537 +2443 1380319 +372 1379345 +2769 1378891 +3822 1378669 +4065 1378474 +7092 1378135 +576 1377793 +8060 1377625 +5150 1375001 +391 1374575 +6246 1374163 +5618 1372598 +4392 1372508 +442 1371294 +2372 1370549 +7636 1368619 +4734 1367530 +2679 1367016 +4811 1365243 +5068 1363461 +2751 1362395 +4986 1361444 +48 1361374 +7638 1360237 +3064 1359735 +9747 1358672 +5747 1358332 +2947 1357419 +2692 1356699 +3613 1356588 +912 1355680 +359 1353287 +4268 1353229 +10395 1351988 +4650 1350702 +3368 1350513 +939 1350079 +452 1349984 +2465 1349409 +4006 1347932 +5710 1347611 +3420 1346666 +5410 1346311 +557 1346053 +2657 1345538 +3562 1344045 +4646 1343369 +1565 1343288 +791 1342812 +3841 1342808 +3244 1342552 +8301 1340601 +15069 1340247 +6290 1339930 +5780 1339030 +5928 1338936 +2231 1338201 +469 1338045 +5123 1337766 +4046 1337492 +1370 1337438 +5742 1336548 +6027 1336530 +3895 1335611 +463 1334739 +2099 1333298 +2763 1332891 +5094 1332269 +6592 1330873 +5096 1330694 +3434 1330495 +11421 1330487 +11092 1330119 +4133 1329427 +2611 1328981 +1701 1328763 +1703 1326543 +2362 1326290 +4574 1325119 +6136 1323601 +1954 1320496 +6334 1318454 +8464 1316365 +2910 1316215 +5093 1315678 +3158 1315478 +5156 1314587 +4314 1314066 +4341 1314003 +409 1313875 +5054 1312722 +4326 1312569 +3327 1311762 +4587 1310546 +7994 1310530 +3436 1310399 +1634 1309798 +4787 1308208 +3842 1307129 +3336 1303877 +1453 1303867 +527 1303388 +3819 1302492 +4870 1302035 +1711 1301722 +4736 1301545 +3280 1301512 +485 1300709 +3938 1300533 +988 1299815 +1023 1299461 +5197 1298056 +3691 1298022 +3785 1297957 +3265 1296350 +2398 1296348 +6729 1296290 +5033 1295463 +3409 1294973 +5913 1291479 +4654 1290537 +7630 1288623 +3709 1286675 +6376 1286193 +3781 1285807 +7756 1285579 +10171 1285322 +2818 1284996 +1014 1282901 +4097 1282603 +4813 1282045 +3038 1281458 +5341 1281418 +420 1281046 +4955 1280803 +8836 1279703 +1731 1278908 +3636 1278857 +3392 1278431 +7392 1277991 +3194 1276594 +4446 1276547 +4540 1275393 +4099 1274105 +4788 1273253 +4167 1272413 +459 1272249 +1948 1271480 +1640 1270835 +7176 1270571 +4429 1269217 +4042 1269107 +2740 1268901 +5866 1268234 +4737 1267826 +6299 1267725 +8150 1266632 +3099 1265853 +19398 1265684 +1878 1265115 +4893 1265091 +2318 1263652 +5627 1263447 +8909 1263276 +5174 1263166 +2829 1261919 +4688 1261278 +6289 1261136 +4765 1258870 +4525 1258646 +4940 1258077 +433 1257886 +4675 1257299 +3950 1257106 +4334 1255854 +4371 1255469 +4478 1255023 +6459 1254251 +3958 1253900 +5364 1253811 +2327 1253249 +119 1252957 +3264 1252530 +3588 1252368 +1624 1252185 +4885 1251484 +521 1251476 +3607 1251163 +7420 1251114 +3572 1250281 +536 1250114 +6325 1249281 +5140 1249176 +5284 1248712 +1828 1247137 +4978 1245733 +4708 1244437 +6825 1243796 +3968 1241799 +6280 1240647 +6835 1240422 +12052 1240210 +6023 1236586 +1283 1236065 +5804 1235634 +4762 1234227 +2095 1233277 +2447 1232432 +445 1231550 +3734 1231293 +3161 1231023 +3612 1230155 +2638 1229977 +1343 1228431 +6516 1227971 +5137 1227542 +7014 1227326 +6638 1226525 +2275 1225586 +5670 1225366 +15862 1224702 +5426 1222773 +1338 1222222 +4569 1222157 +12499 1221455 +4043 1221352 +2043 1221192 +2581 1221146 +622 1220070 +1547 1219481 +1096 1219401 +5461 1219355 +3657 1219079 +4966 1218169 +1137 1218103 +5366 1217930 +2866 1216988 +3356 1216655 +4641 1215786 +5163 1214894 +1656 1214167 +3825 1213380 +443 1212646 +3863 1212582 +6078 1212500 +2426 1212420 +425 1212037 +647 1211602 +7320 1211287 +1086 1211104 +3568 1210892 +4621 1210058 +5741 1209984 +4763 1209467 +4698 1209089 +2948 1208943 +2461 1208205 +1714 1207871 +7372 1205825 +5689 1205245 +4533 1205189 +3084 1205002 +5112 1204702 +3148 1204416 +4205 1203481 +3933 1202908 +11302 1202825 +7968 1201690 +4410 1200581 +6491 1199580 +6884 1199477 +9027 1199088 +6476 1197258 +2402 1196830 +3272 1196577 +4946 1196366 +1847 1195644 +2068 1195640 +2230 1194751 +5668 1194054 +6841 1193691 +6628 1192538 +5617 1192353 +3704 1192118 +454 1190907 +4113 1190819 +3360 1188180 +3595 1188055 +6388 1186942 +3024 1186481 +5676 1186075 +2616 1185977 +5076 1185573 +6997 1185148 +1573 1184210 +2502 1184197 +3956 1183015 +3306 1182352 +4191 1182121 +2907 1181996 +3758 1181687 +2933 1181593 +12820 1181273 +3505 1181150 +2481 1181067 +4258 1180761 +5339 1179754 +4497 1178907 +4705 1178786 +8424 1178672 +6989 1178672 +7225 1178011 +519 1177463 +5613 1177032 +837 1176942 +4203 1176560 +2736 1176366 +7067 1176283 +726 1175669 +2926 1175583 +5839 1175102 +4028 1175019 +6898 1174845 +46444 1173449 +3518 1172575 +2176 1172215 +3344 1171044 +6682 1170042 +4286 1170039 +4075 1168934 +3925 1168605 +3516 1168434 +7915 1167772 +6314 1167088 +1666 1166890 +6586 1165644 +3896 1164453 +1205 1163886 +4315 1163570 +6717 1163432 +4048 1163167 +6301 1162084 +7281 1161824 +3589 1161761 +1045 1161449 +3335 1160825 +3840 1160664 +4406 1160456 +4330 1160159 +1340 1159444 +871 1158633 +5995 1158628 +3722 1158586 +4618 1158428 +5010 1157604 +4564 1157265 +3190 1156679 +4369 1156412 +4047 1154902 +6342 1154157 +5707 1154075 +4393 1153892 +3355 1153701 +1413 1152770 +6542 1152768 +6447 1152716 +7366 1152389 +5382 1152114 +4683 1151509 +2311 1150108 +2328 1149589 +5179 1149446 +3496 1149070 +4206 1148604 +3748 1146891 +4425 1146252 +5311 1145728 +6702 1145147 +4398 1144278 +3952 1144184 +4795 1143514 +4900 1141559 +6858 1140893 +3348 1140558 +4166 1140194 +7396 1138944 +1277 1138668 +5001 1137748 +3446 1137430 +10662 1136646 +4858 1136361 +3091 1136277 +8783 1135186 +2605 1135155 +3809 1134429 +3251 1134183 +6983 1133029 +4673 1132793 +6025 1131898 +473 1131860 +1582 1131777 +8900 1131533 +5942 1131397 +5448 1131115 +5845 1131004 +3665 1130444 +4153 1129425 +3580 1129156 +6612 1129095 +5394 1129005 +30494 1128381 +421 1128217 +2883 1128149 +6922 1128102 +7044 1128058 +3788 1126787 +591 1126267 +4067 1123860 +4632 1123334 +5871 1122772 +3653 1121714 +6265 1121635 +2061 1121351 +1709 1121162 +3648 1120782 +7172 1120292 +2112 1120153 +2566 1119761 +4165 1119543 +2039 1118103 +4077 1117991 +5213 1117193 +2939 1116899 +9952 1116227 +11214 1115457 +6294 1115417 +6182 1114988 +1236 1114768 +4676 1113847 +14018 1113731 +5818 1113054 +6568 1112793 +3649 1112551 +7945 1111652 +4290 1111506 +11063 1111148 +7478 1110529 +5664 1110127 +2561 1108755 +11419 1108318 +599 1108264 +7055 1107776 +5025 1107245 +4151 1106800 +6699 1106003 +3660 1105998 +5007 1105989 +4753 1105892 +4122 1103755 +3951 1103085 +4819 1103013 +7541 1102865 +6308 1101995 +5081 1101551 +3827 1101551 +6348 1101534 +7025 1101049 +5520 1100898 +4713 1100782 +8406 1099966 +5257 1099287 +3592 1099115 +9689 1098471 +3497 1097861 +4430 1097362 +9912 1097102 +8153 1097019 +6108 1096435 +6154 1096150 +5502 1094500 +7425 1094293 +7127 1093101 +3283 1093034 +4624 1092815 +7000 1092519 +6241 1091967 +17560 1091497 +512 1090181 +5403 1090022 +6081 1088193 +3359 1088146 +3221 1087980 +7082 1087741 +3290 1087438 +1921 1086770 +10169 1085547 +3717 1084848 +4963 1084155 +3487 1083889 +4512 1083842 +781 1083714 +3402 1082905 +1717 1082724 +3297 1082684 +7910 1082114 +6934 1081913 +4914 1080997 +5504 1080481 +7415 1079818 +4441 1079514 +4928 1079417 +1323 1078936 +5764 1078723 +13598 1078487 +5205 1078470 +4747 1078344 +9068 1077920 +18015 1077721 +7008 1077624 +9502 1077051 +3919 1076785 +7939 1076621 +4461 1076561 +6355 1076297 +3026 1074645 +6046 1074406 +453 1073993 +2128 1073157 +3594 1073046 +2149 1072456 +13520 1072257 +5290 1071604 +5675 1070845 +4050 1070287 +5451 1069447 +6643 1069445 +4336 1069141 +431 1068235 +5389 1067211 +1514 1066254 +3424 1065502 +1906 1065274 +5581 1064940 +7504 1064485 +3578 1064180 +3666 1063907 +3744 1063473 +590 1063051 +5370 1062933 +7072 1062318 +9256 1062317 +833 1062017 +5236 1061829 +16462 1060612 +5291 1059433 +4201 1059425 +3777 1059131 +861 1058768 +2356 1058460 +562 1058081 +4831 1056793 +1362 1056283 +11435 1056101 +5637 1055964 +6079 1055924 +495 1055621 +3463 1055372 +1870 1055004 +3597 1054912 +6191 1054492 +4302 1054350 +672 1054156 +6332 1054152 +5538 1054038 +3774 1053925 +4678 1053559 +5852 1052549 +6698 1052045 +694 1052010 +4642 1051966 +2208 1051338 +5252 1050757 +1619 1050376 +4001 1050365 +3970 1050229 +5316 1049898 +6076 1049745 +5170 1049627 +4633 1049406 +6219 1048935 +3318 1048824 +5087 1048697 +5895 1048672 +4499 1048514 +3776 1048447 +5386 1048337 +5203 1047480 +5293 1047160 +4171 1047152 +3159 1047020 +6150 1046461 +8009 1045545 +1268 1044664 +4953 1044482 +5176 1044260 +6272 1043933 +5349 1043837 +7728 1043680 +3953 1043554 +3197 1042977 +6116 1041224 +8886 1040488 +6343 1040295 +5086 1039785 +958 1039594 +831 1039207 +1317 1038938 +893 1037936 +5597 1037832 +9621 1037628 +538 1037073 +501 1036988 +7908 1036807 +6110 1036682 +1899 1036322 +8511 1035818 +5954 1035279 +96 1035250 +6853 1034370 +3350 1033703 +4610 1033561 +2951 1033423 +7124 1033278 +5693 1032803 +7476 1032681 +6288 1031061 +4496 1030906 +4538 1030856 +4771 1029333 +2075 1028886 +2113 1028695 +10390 1028255 +7779 1027567 +507 1027336 +3406 1027246 +7517 1026986 +5395 1026926 +387 1025854 +3252 1025192 +4570 1024225 +4261 1024017 +8092 1023604 +4964 1023023 +2013 1022989 +3918 1022342 +4599 1021950 +5762 1021750 +6995 1021687 +4291 1020561 +5750 1020427 +3586 1020053 +6035 1019853 +4104 1019777 +7802 1018633 +5587 1018578 +1424 1018565 +2344 1017560 +4609 1017207 +4860 1016525 +4365 1016357 +3511 1016197 +4905 1015998 +4025 1015981 +3347 1015669 +7028 1015628 +2078 1015524 +1983 1015352 +3031 1015337 +979 1015312 +9072 1014743 +5122 1014687 +3891 1014111 +5474 1014058 +7880 1013953 +3210 1013783 +568 1013436 +5220 1013406 +4434 1012403 +4697 1011550 +5362 1011498 +3737 1011380 +756 1011159 +5136 1010585 +5242 1010457 +5110 1010415 +5523 1010390 +4908 1010100 +7534 1010030 +3387 1008087 +8063 1007595 +3910 1007470 +701 1007115 +600 1005923 +2425 1005712 +2713 1005668 +4792 1005214 +3806 1004967 +4190 1004836 +798 1004024 +3947 1003824 +1734 1003335 +774 1002962 +49430 1002589 +5859 1001740 +5704 1001278 +4238 1001170 +918 1001111 +4553 1000558 +5031 1000426 +8437 999699 +2246 999417 +4084 998042 +4197 998026 +8545 997343 +9611 996559 +4887 996285 +7865 995676 +4692 995619 +6961 995585 +1404 995563 +4423 995201 +8108 995057 +3450 994841 +4849 994834 +4220 994662 +889 994403 +6914 993625 +3621 993134 +1039 992744 +5938 992199 +715 992013 +1689 991685 +5281 991483 +1482 990745 +4372 990263 +7964 989765 +10123 989614 +3854 989508 +782 988371 +4800 988139 +4439 988074 +7395 987559 +4325 987452 +2448 986794 +1681 986285 +5882 986221 +5891 984558 +6630 984477 +3555 984317 +5733 983370 +6613 982970 +455 982830 +4725 982772 +7943 981571 +2885 981392 +5419 980765 +7374 979270 +4917 978913 +3561 978800 +1078 978520 +4162 978296 +9669 978242 +2815 978236 +1680 978235 +8693 978157 +498 977781 +4422 977129 +2167 976132 +2390 975986 +4931 975803 +4635 975620 +5922 973722 +5115 973595 +4384 973528 +7927 973338 +8031 973073 +424 972909 +896 972209 +1352 972098 +4974 971531 +3716 971410 +5545 971211 +4837 970688 +3105 970602 +3972 969545 +9388 969002 +6151 968990 +5380 968941 +5798 968828 +6848 968442 +5062 968256 +8136 968147 +7291 967583 +3087 967418 +1962 966850 +3894 966452 +10191 966175 +2051 965356 +913 964915 +8047 964868 +4458 964367 +5300 964311 +6994 964187 +3725 962770 +7529 962681 +8121 962545 +5526 961651 +6033 961565 +6466 961557 +4150 961454 +1650 961250 +486 961032 +491 960800 +5745 960709 +8878 959982 +5334 959692 +5045 959526 +7903 959023 +6057 958754 +5749 958063 +8087 958003 +6264 957693 +4004 957171 +9413 956628 +8611 956473 +627 955912 +1219 955395 +3499 954203 +10610 954183 +776 954130 +3404 953636 +7593 952789 +3713 951708 +4694 951700 +5486 951539 +9089 950646 +2389 950463 +10499 950298 +5788 950061 +6926 948694 +4518 948669 +7864 948572 +748 948224 +7606 947933 +1010 947597 +8059 945880 +4571 944428 +16267 944143 +6705 944128 +8674 943230 +4923 942533 +2421 942380 +5556 941699 +4735 941386 +5270 940960 +4202 940146 +5335 937988 +1890 937973 +6088 937822 +4816 937750 +4588 937427 +7452 937286 +1658 937147 +5533 936928 +1616 936623 +7712 935685 +3108 935633 +6205 935469 +7901 935057 +4950 934554 +3619 934220 +419 933976 +3478 933536 +5262 932866 +945 932541 +18840 931723 +5009 930935 +21138 930513 +1772 929479 +4755 929010 +5941 928630 +7133 927505 +6977 927246 +4833 926967 +7018 926786 +6403 926666 +6497 926492 +10247 926122 +6149 925968 +1446 925939 +25370 925871 +5688 925496 +10 925332 +8123 923747 +5989 923482 +1503 922356 +4637 922330 +4634 921904 +12385 921506 +4631 921152 +7628 921120 +413 920921 +7351 919431 +5975 919370 +5091 919110 +7297 918894 +5149 918737 +10330 918368 +12131 918249 +6621 918125 +7403 918086 +7052 917508 +369 917495 +3886 916915 +5125 916460 +2366 916128 +9116 915995 +7137 915749 +4998 915052 +5929 914971 +5585 914915 +3538 914319 +26442 913487 +4710 913429 +7253 913428 +3303 913013 +4457 912882 +40026 912761 +3914 910594 +6793 910482 +6596 910007 +4947 909361 +3122 908576 +7746 908460 +6510 908043 +2890 908025 +2420 907987 +5699 907839 +5402 907697 +8473 907013 +6128 906605 +4451 906397 +2934 906049 +12874 905857 +1677 905839 +5228 904072 +6103 904029 +3835 903727 +4855 903439 +4750 902769 +332 902136 +9475 902096 +9005 901510 +4916 900786 +6464 899911 +3853 899655 +3268 899638 +8602 899334 +2665 899101 +6402 898959 +2744 898743 +4601 898430 +7062 898339 +8785 897821 +567 897616 +10391 897375 +7725 896852 +15320 896741 +5423 896175 +4490 895533 +6416 895290 +5713 895254 +10501 895030 +1432 894640 +9077 894615 +6483 894614 +5586 894579 +4622 894347 +20877 893427 +2079 893154 +623 893020 +4991 892646 +5690 892425 +10618 890849 +12184 890553 +8244 890099 +1780 889484 +1313 889246 +106 888842 +795 888161 +3638 887959 +4353 887887 +10542 887847 +5409 887321 +7546 887173 +2396 886955 +4347 886599 +850 886537 +629 886491 +1128 886222 +6119 886169 +6727 886128 +4236 886028 +3281 885374 +7466 885097 +873 884923 +10830 884441 +4894 884043 +3878 884013 +7389 883818 +2347 883556 +7848 883275 +5223 883182 +3900 883053 +3783 883041 +5982 882775 +5198 882436 +1736 881949 +2393 881816 +2395 881208 +6358 881038 +7683 880486 +8342 880373 +8078 879936 +10371 879935 +8761 879473 +7356 879394 +9604 879106 +6960 878827 +2624 878148 +4684 877919 +3189 877629 +2547 877507 +4920 876005 +5860 875894 +6133 875719 +5230 875515 +3539 875493 +1959 875161 +8292 875055 +6509 874958 +1012 874561 +7611 874327 +6943 874065 +7557 872959 +3977 872862 +4960 872487 +11529 870460 +3632 869034 +2474 868380 +3278 868202 +7261 868159 +5182 867690 +6196 867596 +5322 867298 +5438 867126 +5214 866142 +5836 865387 +7595 864999 +489 864993 +2199 864291 +5017 864182 +3802 864008 +6849 863857 +4929 863148 +5898 862499 +6441 862493 +7118 861461 +12551 861355 +8488 861121 +9141 861100 +1381 860833 +7533 860823 +7458 860382 +10827 860061 +1451 859861 +4615 859080 +4764 859011 +8111 858777 +4324 858591 +7647 858536 +3223 858325 +5055 857740 +4296 857441 +5797 857174 +4351 856996 +5407 856847 +3544 856775 +1795 856495 +1665 856106 +5495 855994 +4257 855834 +8050 855330 +10109 854644 +7022 854557 +7920 854365 +7799 854340 +5608 854336 +5385 854118 +3046 854067 +7194 852548 +16964 852383 +7651 852355 +12379 852095 +4213 851873 +7456 851550 +11695 851095 +7401 851086 +7412 850942 +1586 850652 +5143 850210 +902 849624 +5952 849584 +6200 849045 +11643 848152 +5071 847489 +458 847417 +4547 847319 +6907 847057 +7705 846873 +1341 846615 +10021 846038 +1902 845644 +5479 845479 +14897 844971 +2188 844313 +5455 844072 +4459 843875 +4719 843858 +2959 843654 +595 843545 +3132 843221 +5924 843160 +1232 842710 +10306 842643 +574 842221 +5287 841971 +10728 841811 +441 841635 +7794 841423 +5609 841175 +5867 841140 +9366 840353 +8403 840105 +7585 840023 +6405 840017 +6875 839897 +2834 839591 +4179 839528 +12011 839205 +7433 839201 +4292 839187 +7492 839060 +3851 838918 +747 838741 +4785 838458 +1620 837460 +1710 837330 +4343 836110 +4216 836033 +8185 835422 +4577 835155 +5529 834254 +1475 834211 +8165 834201 +7627 834128 +2836 834105 +5708 834100 +11383 834017 +9138 833943 +9008 832798 +2124 832447 +4168 832285 +6802 831767 +4952 831698 +6716 831649 +7806 831440 +4333 830985 +5295 830546 +4036 830203 +5716 830154 +5615 830060 +3644 829493 +3608 829215 +2238 829204 +1345 828544 +3048 828260 +6142 827480 +10805 827295 +5202 826828 +4620 826416 +7459 825942 +4856 825116 +5811 824590 +4232 824275 +4362 824215 +2937 823809 +8502 823728 +5358 823668 +6155 823643 +8974 823355 +9266 821977 +6215 821613 +573 821459 +4053 820973 +7017 820798 +5911 819328 +3917 819177 +4643 819129 +7868 819089 +9003 818965 +4146 818654 +7463 818425 +6175 818176 +6669 818066 +8366 817761 +6594 817656 +2853 817307 +118 817287 +5264 817284 +7421 817216 +8211 817116 +9692 817105 +6305 816562 +7973 816504 +8810 815781 +7269 815760 +2682 815662 +12516 815187 +7524 814872 +2873 814870 +2154 814832 +2949 814350 +6318 813767 +7244 813673 +497 813572 +10413 813503 +698 813483 +7859 813085 +5043 812590 +4981 812572 +4922 812555 +5004 812273 +8127 812230 +8414 811932 +6240 811897 +12147 811787 +3616 811551 +5776 811486 +9870 811062 +7740 810523 +2091 810014 +2792 809891 +3205 809207 +6622 808813 +5849 808800 +8200 808021 +585 807549 +9470 807304 +9818 807238 +7083 806891 +4082 806605 +4975 806503 +2316 806460 +3551 805935 +6493 805854 +2816 805436 +5350 804945 +4003 804496 +1706 804386 +4695 804360 +8221 804223 +3211 803968 +8708 803949 +5155 803913 +4988 803565 +4874 803237 +10106 802720 +3232 802701 +7732 802397 +5326 802066 +6427 801361 +5631 801279 +6740 800879 +6100 800742 +401 800726 +4803 800624 +6363 800619 +5727 800412 +1127 800086 +9298 800083 +4996 800014 +4106 799519 +487 799506 +4131 799341 +5610 799162 +7121 798834 +6115 798667 +6481 798460 +4829 798303 +3881 797726 +962 797583 +6642 796913 +7139 796371 +4263 795916 +3872 794949 +11957 794566 +4071 794495 +8533 794334 +8919 793766 +8639 793455 +7516 792793 +5873 792586 +6276 792041 +2034 791816 +8796 791720 +5353 791490 +7103 791434 +2548 791227 +8329 791148 +9605 791129 +9953 790868 +8372 790696 +589 790467 +8144 789428 +5221 789411 +7012 789054 +6520 788825 +7831 788498 +844 788331 +1623 788130 +5696 787166 +4645 787122 +7986 787107 +5827 787081 +5445 787003 +5243 786478 +5046 786472 +6330 786159 +5014 786066 +6379 786013 +10636 785808 +917 785162 +3124 784947 +2640 784189 +9725 783553 +9253 783425 +8049 783310 +6538 782938 +5638 782700 +8632 782584 +6807 782551 +721 782040 +16059 781941 +6333 781716 +6074 781638 +5565 781181 +1765 780911 +5810 780630 +3714 780526 +954 780449 +5778 780376 +5789 779996 +1068 779555 +3363 779048 +5593 778906 +3640 778657 +10240 778494 +4289 778292 +5114 778205 +987 778126 +46640 777912 +7413 777318 +7317 776881 +9 776842 +11115 776693 +5021 776583 +5381 776513 +6563 776178 +3753 776103 +2264 776075 +6905 775817 +6936 775626 +6050 775178 +5752 775047 +7064 774442 +4471 774370 +2007 774332 +7099 774232 +10876 773951 +7078 773426 +3963 773066 +6619 772959 +6888 772667 +8536 772525 +6670 772421 +3829 772119 +6198 771881 +14015 770622 +7002 770508 +1018 770301 +4174 770174 +4701 770100 +8395 769730 +2599 769416 +7040 769323 +5612 769037 +7970 769031 +2438 768787 +10749 768289 +8982 768073 +5161 768005 +2453 767894 +7171 767851 +3815 766903 +3996 766842 +5850 766046 +6228 765987 +4899 765434 +7271 765147 +2718 765137 +6235 765125 +8407 765069 +862 765005 +5906 764849 +1187 764634 +8636 764295 +5814 764221 +4096 764076 +1142 764058 +8620 764028 +19809 763246 +6165 763231 +7840 762640 +6614 762319 +11079 761718 +13308 761408 +10799 761174 +1192 761045 +6774 760963 +3683 760869 +2394 760818 +6482 760799 +7823 760780 +1778 760726 +4462 760542 +6131 760457 +6253 760190 +6070 760142 +9436 760093 +5885 759875 +2449 759865 +1556 759355 +2598 759301 +9648 759298 +6692 759148 +8587 759113 +9180 758708 +4889 758705 +6296 758275 +10614 758002 +6623 757773 +1516 757483 +5983 757289 +5963 757079 +9371 756527 +9839 756521 +6190 756459 +457 756319 +7423 756132 +20635 756084 +2451 756028 +8936 755576 +5517 755366 +3001 754873 +2689 754598 +7797 754588 +7411 754253 +5433 753725 +6260 753428 +11618 752903 +7519 752784 +6189 752766 +1931 752571 +11514 752163 +5441 751927 +11356 751848 +6401 751735 +1324 751409 +2339 751243 +5417 750911 +735 750739 +5022 750547 +3908 750126 +3765 750060 +9461 749906 +8282 749744 +778 748051 +3873 747968 +7212 747911 +13423 747877 +6712 746778 +5940 746619 +593 745833 +12787 745585 +8024 744764 +7874 744587 +6204 744070 +5761 743492 +408 743484 +6490 743166 +7800 742528 +5879 742339 +6266 742335 +4320 741518 +2623 741505 +5363 740788 +1017 740736 +7288 740643 +9019 740546 +6870 740253 +6693 740189 +4329 739395 +7404 739189 +4093 738959 +9825 738518 +8857 737741 +10648 737228 +5318 737201 +16175 736827 +6823 736756 +10807 735530 +3535 735395 +2892 735274 +12180 735097 +3023 734685 +5188 734061 +10006 733791 +14549 733714 +33721 733497 +5875 733377 +9087 733374 +5338 732693 +5863 732401 +12785 732355 +4845 732267 +5857 732242 +18293 731947 +4595 731814 +4222 731703 +9500 731638 +9283 731189 +8877 731155 +8470 731144 +6872 730530 +4188 730422 +6225 730269 +5108 730199 +7148 729814 +1581 729749 +10500 729418 +6832 729243 +2920 729192 +6656 728880 +2670 728538 +7312 728480 +4295 728371 +5462 728288 +6339 728140 +3862 727744 +9975 727410 +7188 727311 +5861 727283 +5387 726788 +11761 726787 +634 726665 +5118 726571 +8976 726201 +8838 725568 +744 725461 +342 725294 +7163 725203 +6709 725026 +1146 724457 +9920 724076 +5896 723335 +4781 722493 +4537 722266 +11171 722136 +4433 722080 +8415 722004 +1095 721984 +8036 721541 +8685 720983 +4934 720956 +8879 720855 +8998 719936 +10654 719841 +4760 719610 +6134 719284 +9918 719163 +7897 718765 +11754 718515 +7432 718161 +9951 718152 +5254 717916 +4158 717914 +5186 717844 +20832 717534 +10039 717274 +6626 717236 +1069 716980 +6593 716881 +6085 716690 +9671 716454 +9070 716415 +8663 716353 +10804 715896 +5706 715743 +5359 715340 +27868 715294 +6792 715292 +6741 715229 +9153 715036 +13129 715014 +5044 714833 +6647 714813 +6511 714437 +8100 714414 +13648 714321 +9735 714169 +10964 714150 +22971 713833 +6829 713647 +4249 713298 +5365 712933 +5732 712932 +4143 712823 +5996 712740 +8618 712707 +6073 712606 +5986 712473 +1030 712471 +1461 712265 +5992 712229 +2066 711631 +18841 711614 +9925 711387 +8259 711334 +7205 710271 +11117 710047 +4793 709820 +6478 709698 +3858 709005 +2996 708450 +6430 708208 +4007 707891 +768 707799 +6067 707724 +9320 707652 +1894 707449 +7431 707015 +1509 706866 +14207 706655 +5511 706560 +6553 706382 +4385 706268 +4925 706167 +8096 706036 +6143 705550 +6188 705375 +9764 705084 +13126 704724 +6554 704581 +7394 704389 +5288 704007 +7912 703955 +4483 703878 +8929 703551 +14466 702956 +9071 702931 +8518 702631 +12517 702625 +32290 702438 +8581 702251 +7962 702211 +5719 702082 +5916 701971 +2577 701821 +7328 701292 +3248 700210 +5743 699668 +5457 699547 +6126 699148 +2931 698934 +6323 698857 +7639 698407 +1494 698189 +5846 697622 +2685 697571 +7791 697392 +294 697012 +13100 696907 +5726 696441 +3200 696066 +5946 696036 +7464 695756 +6304 695647 +10504 695601 +7547 695511 +9880 695470 +4572 695347 +14237 695318 +7373 695309 +9477 695162 +6980 695128 +5806 694770 +605 694642 +2998 694435 +7895 694205 +4528 693789 +2749 693766 +5483 693676 +9566 693519 +2580 693316 +10174 693096 +8704 693080 +9909 692938 +21393 692900 +3993 692679 +12262 692396 +10464 692058 +8155 692022 +6817 691879 +2943 691761 +5645 691684 +6576 691662 +8540 691355 +8742 691341 +4116 691092 +5303 690849 +9570 690723 +2380 690285 +11394 690269 +14017 689530 +6809 689244 +6129 688667 +5369 688418 +4345 688195 +5274 688173 +239 687921 +4090 687761 +8072 687681 +5876 687636 +7195 687577 +5667 687373 +1950 686507 +2484 686452 +8179 686263 +6608 686261 +8688 686110 +6279 685683 +10729 685582 +7173 685513 +6687 685094 +6409 684986 +4469 684945 +10604 684560 +6135 684453 +8832 684446 +14708 684416 +1990 684197 +1082 684138 +5566 683609 +5559 683395 +4074 683378 +2256 683376 +2024 683346 +8073 683283 +897 683245 +6173 683136 +7734 682901 +8882 682762 +1855 682313 +5034 682278 +9852 681791 +7624 681768 +6673 681693 +6411 681081 +7364 681064 +4913 681060 +6031 680976 +7344 680890 +6827 680734 +12168 680648 +9591 680619 +7246 680607 +5644 679833 +4069 679756 +7387 679646 +7318 679542 +5456 679522 +1662 679367 +7886 679365 +7977 679346 +14403 679217 +7311 679157 +11947 679007 +7924 678757 +9755 678632 +2436 678453 +947 678119 +7884 678117 +5884 678031 +6662 677968 +7323 677870 +5543 677819 +2919 677816 +10405 677754 +9101 677581 +6093 677515 +7051 677389 +8224 677269 +7337 677160 +2295 677074 +1158 676751 +6844 676610 +5412 676453 +4306 676409 +7224 676369 +7846 676354 +11289 676149 +3237 676053 +4876 675868 +7259 675866 +9754 675784 +7256 675714 +7771 675679 +11534 675527 +8972 675521 +9761 675505 +876 675302 +4159 675287 +22940 674880 +10131 674860 +690 674447 +1904 674392 +1083 673897 +6032 673886 +5101 673784 +3738 672730 +3180 672440 +4391 672381 +10673 672248 +2857 672245 +2001 672175 +9651 671458 +5413 671453 +11133 671373 +8556 671203 +7522 671159 +4270 671110 +3682 670879 +21648 670847 +5933 670801 +11409 670739 +8362 670409 +11287 670223 +7719 670222 +3698 670219 +7016 670090 +613 669934 +5766 669916 +10093 669908 +249 669379 +5157 669251 +4935 668785 +18912 668708 +1589 668686 +6218 668678 +4221 668611 +1533 668559 +6461 668440 +6666 668008 +9679 667710 +5301 667687 +5920 667645 +6041 667255 +10026 667109 +537 666906 +4455 666658 +5787 666453 +6157 666445 +5207 666201 +17846 665807 +8305 665802 +3992 665716 +8592 665686 +2953 665632 +7482 664859 +7485 664609 +3904 664497 +1325 664191 +3955 663800 +3582 663747 +6232 663677 +5858 663380 +6675 663306 +603 663284 +3510 662971 +3198 662873 +6868 661915 +3398 661810 +2662 661713 +5548 661134 +3254 660739 +3435 660236 +3208 660141 +7602 660107 +2944 660028 +6140 660004 +9084 659896 +22346 659867 +8213 659526 +8359 659308 +8987 659019 +4252 658938 +7283 658854 +3188 658518 +11807 658016 +9906 657789 +4088 657734 +7796 657606 +1092 657505 +1431 657491 +9119 657490 +4751 657349 +5539 656930 +8672 656751 +7486 656405 +746 656119 +2705 656036 +6056 655720 +1995 655638 +8589 655599 +10397 655550 +6725 655530 +10242 655020 +7276 654911 +9074 654748 +38913 654696 +7898 654624 +1740 654207 +4327 654145 +7652 653660 +381 653525 +4061 653521 +1671 653489 +1530 653176 +7189 653134 +13430 652994 +4339 652906 +6049 652838 +37707 652638 +825 652479 +7346 652418 +2780 652110 +3113 651930 +5901 651795 +14024 651686 +6555 651683 +9659 651412 +11154 651044 +4836 650886 +10016 650673 +5887 650661 +9118 650642 +7832 650470 +5564 650455 +7334 650225 +10737 650100 +5292 649726 +2984 649605 +5428 649179 +9280 648905 +6952 648709 +920 648620 +19360 648453 +31215 648415 +5894 648264 +5390 648185 +8607 648182 +7030 647962 +5443 647898 +13785 647830 +5603 647672 +5278 647399 +127 647382 +2538 647133 +5330 646394 +7204 645847 +5488 645597 +6082 645495 +9025 645143 +13119 645117 +12673 645054 +5870 644976 +9439 644729 +6531 644334 +5524 643753 +8308 643717 +2049 643595 +3169 643406 +1102 643192 +1416 643014 +4556 642555 +6801 642064 +1452 641982 +4225 641671 +4961 641591 +664 641444 +9899 641281 +7765 641155 +3668 640797 +577 640764 +8345 640747 +5795 640537 +8411 640494 +5621 640471 +6370 640152 +9486 639933 +13089 639843 +5187 639596 +6665 639411 +8335 639410 +8475 639341 +3705 639284 +5302 638828 +9716 638798 +4361 638678 +2047 638657 +10219 638643 +310 638361 +6891 637744 +7011 637600 +3559 637533 +6559 637504 +687 637481 +10071 637167 +5035 637079 +3811 637040 +6292 636820 +7325 636136 +5679 635912 +9992 635639 +5509 635306 +3152 635231 +12074 635104 +9957 634959 +6655 634887 +9016 634863 +7410 634829 +696 634729 +7960 634441 +3876 634388 +2596 633768 +10013 633760 +6163 633090 +7849 632916 +5506 632595 +4727 632496 +2667 632345 +6768 632173 +6616 632141 +11257 632026 +10068 631604 +7123 631579 +7542 631579 +7023 631466 +7720 631250 +6796 631244 +3070 630513 +3256 630499 +2767 630481 +9974 630365 +692 630358 +5549 630267 +6896 630202 +9002 630065 +6923 629920 +15811 629850 +9738 629838 +2891 629691 +4231 629583 +8606 629251 +11823 629198 +9853 628987 +7027 628837 +3991 628805 +2190 628682 +6862 628532 +4939 627964 +14914 627859 +2902 627520 +11615 627385 +4904 627359 +10650 627124 +5279 626755 +7777 626600 +799 626519 +6183 626487 +15113 626346 +1229 626246 +12224 625949 +36826 625284 +2501 625099 +6000 624750 +9326 624573 +8978 624449 +10101 624337 +2514 624302 +11660 624213 +8318 624036 +4562 623978 +1939 623886 +1067 623705 +7981 623554 +8590 623553 +7150 623194 +7216 623063 +6640 623002 +2127 622758 +8237 622309 +4951 622270 +9885 621960 +7830 621930 +6877 621661 +5485 621547 +854 621467 +3342 621167 +5503 621125 +3745 620985 +668 620572 +1850 620506 +7342 620486 +8082 620245 +565 620179 +429 620017 +9739 619828 +4449 619512 +8737 619500 +3182 619289 +10714 618874 +3332 618818 +3901 618659 +3154 618496 +7272 618484 +7953 618134 +5734 617900 +6065 617790 +9847 617491 +5075 617361 +2257 617024 +3865 616847 +4321 616792 +7029 616539 +11897 616236 +7835 616232 +10433 615972 +9617 615869 +5490 615816 +848 615766 +10357 615359 +7810 614764 +8666 614423 +2414 614200 +5501 614089 +6838 613977 +3791 613640 +1505 613292 +11821 612953 +9024 612648 +6298 612576 +8037 612282 +9392 612232 +7444 612155 +11162 612106 +8667 611893 +859 611851 +5514 611662 +7184 611457 +6633 611425 +7625 611246 +6029 611220 +7703 611085 +1167 610976 +1112 610958 +5405 610796 +6097 610555 +5655 610530 +3834 610027 +7077 610008 +8141 609961 +11070 609752 +126 609368 +7697 608871 +3661 608867 +6337 608742 +1653 608691 +8270 608543 +3563 608510 +1173 608225 +8277 608152 +1602 608008 +5527 607857 +8148 607798 +9307 607780 +6087 607464 +3506 607304 +19195 606852 +4962 606712 +5160 606653 +6885 606329 +1522 606222 +4144 606105 +9375 605871 +23997 605838 +7215 605645 +7136 605240 +7696 605204 +5439 604950 +7867 604769 +4417 604529 +6979 604164 +594 603974 +9389 603879 +9730 603658 +11826 603599 +15007 603523 +450 603404 +7784 603393 +12858 603312 +9763 603136 +2560 602954 +8780 602780 +1015 602661 +7219 602594 +11344 602587 +5729 602572 +6550 602536 +5475 602409 +2493 602094 +2305 602075 +10481 601222 +6467 601095 +15855 601040 +11226 600883 +9406 600845 +12104 600838 +3134 600815 +11223 600279 +592 600247 +10273 600115 +10955 599932 +11523 599822 +10329 599458 +6591 599380 +2631 599358 +6536 598751 +9498 597864 +2584 597750 +942 597599 +3312 597138 +6932 596982 +10663 596904 +11582 596881 +1056 596523 +1380 596400 +7406 596161 +10856 595892 +4178 595769 +6918 595606 +7637 595338 +3262 595268 +7309 595160 +8378 595058 +5235 594694 +6373 594645 +7539 594423 +10318 594361 +8720 594360 +10490 594265 +6782 594145 +4627 594120 +4844 594063 +456 594019 +2202 593801 +10512 593750 +1610 593244 +9232 592972 +13837 592565 +6590 592522 +4944 592362 +7174 592237 +3270 591789 +5465 591294 +6958 591198 +7520 590915 +5966 590914 +6004 590867 +6230 590717 +7380 590682 +9273 590639 +7179 590581 +10252 590079 +11232 590075 +10018 589454 +4397 589171 +10553 588862 +11654 588414 +5265 588100 +10307 588072 +7049 587817 +7971 587507 +7370 587415 +2269 587239 +4827 587061 +2178 587040 +28154 586814 +7681 586670 +2777 586647 +704 586590 +2032 586160 +29240 586009 +4303 585944 +6833 585900 +13304 585788 +658 585509 +4259 584968 +11783 584697 +10010 584641 +436 584638 +4044 584624 +5072 584498 +11149 584320 +2999 583791 +6991 583506 +9359 583285 +10319 583073 +8218 583031 +745 582797 +13210 582777 +6147 582612 +8420 582572 +6275 582333 +9784 582289 +2946 582278 +6928 582062 +5016 581857 +5519 581711 +7525 581516 +4638 581509 +28749 581004 +5917 580810 +2367 580376 +9416 580198 +3110 579900 +3837 579873 +10731 579847 +9443 579795 +9279 579534 +12090 579424 +9533 579369 +9046 579350 +3101 579267 +4079 578683 +11763 578236 +15434 577988 +6356 577910 +3944 577882 +10711 577459 +8514 577435 +7441 577388 +524 577286 +34754 577196 +8694 577150 +6518 577063 +8248 577045 +4839 576905 +11343 576880 +8107 576869 +5584 576838 +1211 576832 +5116 576522 +9593 576137 +9512 575883 +5357 575872 +5981 575845 +4091 575497 +14591 575465 +9955 575415 +8090 575406 +6822 575321 +2306 575187 +6755 575130 +7691 575061 +8055 574976 +8215 574691 +7341 574574 +8083 574574 +11687 574439 +6650 574049 +9358 573897 +6834 573740 +3461 573724 +12508 573413 +7192 573384 +15110 573316 +1924 573275 +6496 573211 +794 573111 +5481 573030 +17674 572959 +8547 572682 +6632 572577 +882 572533 +8990 572530 +7043 571991 +2791 571520 +7668 571498 +8732 571494 +4051 571420 +7674 571339 +936 570961 +3721 570900 +3060 570854 +6731 570838 +6317 570838 +11852 570745 +676 570627 +6414 570552 +8848 570361 +3823 570118 +9264 569921 +6840 569894 +4360 569706 +6268 569701 +6776 569555 +7383 569475 +6551 569359 +15319 569196 +8076 569167 +7142 568628 +11416 568580 +5374 568324 +8740 568107 +1455 568100 +6044 567784 +7437 567697 +1443 567507 +6066 567414 +5865 567337 +13104 567326 +4506 567190 +8819 566704 +8528 566670 +6164 566437 +12988 566156 +7093 565967 +3549 565654 +4877 565544 +4312 565286 +6413 565089 +1140 565068 +8774 564935 +4554 564912 +2213 564896 +10808 564748 +5799 564629 +10556 564192 +2430 564111 +253 563468 +8465 563415 +17768 563337 +8962 563249 +3887 563038 +9640 562781 +907 562010 +6886 561513 +7502 561321 +3326 561303 +10607 561199 +480 561037 +1571 560925 +14987 560911 +3755 560808 +522 560722 +8326 560191 +8261 560155 +10043 560098 +451 559924 +8361 559912 +8028 559794 +9594 559778 +545 559407 +31681 559385 +6946 559306 +7108 559275 +2645 559165 +5635 558799 +8302 558731 +6588 558119 +7263 558055 +14935 557913 +9272 557817 +9589 557707 +11745 557593 +3395 557506 +16004 557248 +6861 557017 +8062 557002 +9349 556662 +4031 556646 +8245 556554 +1460 556326 +4579 556319 +5000 556167 +10105 556070 +5215 556063 +10061 556034 +8744 555748 +1435 555400 +8272 555325 +5964 555146 +12767 555106 +9073 554642 +10530 554555 +11663 554385 +3553 554378 +8198 554300 +7034 554151 +2978 553814 +120 553526 +6584 553151 +3861 553102 +1776 553022 +7748 552759 +462 552707 +2123 552637 +1667 552623 +2661 552550 +16009 552434 +9984 552378 +652 552358 +7786 552200 +16407 551865 +11849 551642 +24385 551605 +1357 551592 +9051 551540 +8384 551212 +16974 551193 +12690 551066 +6372 551013 +14100 550991 +22767 550854 +9181 550845 +3980 550594 +3763 550539 +10383 550504 +1501 550432 +9663 550249 +3454 550052 +10764 549946 +11408 549714 +13417 549671 +5238 549218 +4309 549159 +5598 549109 +7175 548618 +3365 548511 +11264 548307 +8759 548096 +1036 547793 +5575 547523 +6099 547457 +1726 547406 +9095 547200 +11016 546805 +6600 546767 +9117 546746 +4724 546694 +4535 546644 +19859 546641 +10085 546192 +11990 545927 +5616 545745 +11867 545469 +558 545369 +12750 545195 +11453 545040 +6007 545014 +4805 544806 +11536 544790 +8381 544636 +12007 544575 +1678 544421 +9354 544408 +5969 544383 +10120 544328 +2004 544322 +9336 544293 +11047 544173 +10656 544121 +9601 543949 +1133 543728 +2771 543554 +948 543517 +7264 543327 +5436 543269 +5253 543261 +2304 543194 +5464 543083 +9935 542754 +13289 542612 +3830 542534 +8826 542457 +12164 542022 +10432 541880 +9691 541784 +5472 541671 +5715 541589 +1799 541522 +8295 541394 +8390 541248 +7384 541184 +4450 541159 +4739 541155 +2945 541039 +9292 541013 +8985 540742 +14039 540629 +4224 540466 +10207 540401 +9687 540209 +14051 540174 +8015 540103 +6393 539847 +6208 539816 +8665 539668 +9176 539559 +2417 539392 +6529 539382 +6678 539231 +1806 538825 +10158 538815 +9308 538759 +9769 538636 +11798 538499 +2686 538256 +30756 538096 +12774 537997 +10202 537811 +20948 537713 +5984 537666 +6034 537532 +5269 537514 +12024 537300 +6016 537023 +3068 536803 +10653 536746 +4544 536720 +47383 536644 +6303 536574 +1670 536539 +9368 536507 +4711 536324 +6688 536289 +11374 536287 +5332 536164 +5415 535772 +686 535624 +12302 535597 +8034 535259 +7298 535243 +5027 534858 +4404 534801 +4993 534397 +4310 534235 +11474 534227 +9179 534109 +3013 533926 +4267 533674 +5947 533656 +6242 533613 +9657 533598 +6971 533518 +8683 533415 +891 533390 +9894 533388 +10278 533319 +1033 533218 +9432 533117 +10935 533048 +955 533045 +6930 532804 +6609 532747 +6216 532571 +863 532480 +10200 532443 +4973 532320 +3388 532276 +8512 532164 +8747 532078 +6754 531928 +1150 531903 +10629 531822 +9634 531776 +6851 531720 +4503 531705 +11520 531460 +2671 530938 +14536 530797 +2040 530459 +4522 530342 +3376 530119 +11366 530088 +8560 529987 +10183 529970 +3913 529729 +27572 529657 +9231 529501 +6006 529484 +12353 529322 +5582 529169 +10936 529025 +11596 528842 +13270 528815 +4778 528726 +1958 528689 +9965 528383 +7996 528093 +5256 527765 +7987 527727 +8724 527617 +12761 527594 +14205 527540 +30101 527445 +2376 527425 +11113 527174 +12108 526987 +7523 526948 +14861 526831 +6769 526760 +10251 526687 +5141 526686 +12056 526659 +11911 526627 +808 526618 +8829 526430 +15142 526406 +7932 526264 +21842 525942 +4269 525894 +11288 525883 +3703 525813 +11661 525338 +1423 525254 +5107 525204 +9038 525200 +7760 525179 +2675 525099 +7214 525047 +7997 524960 +3202 524802 +6611 524696 +1886 524345 +2200 524290 +11565 524211 +9505 524148 +10177 524099 +8391 523969 +4145 523792 +9309 523601 +1098 523601 +28589 523309 +16404 523284 +13663 523201 +11465 523027 +14943 522826 +8227 522727 +728 522579 +10315 522535 +11182 522489 +5711 522363 +9592 522137 +12901 522119 +6842 521979 +7584 521897 +3605 521789 +3459 521714 +7844 521599 +32790 521575 +9489 521286 +6270 521053 +2142 521051 +8203 521013 +5453 521007 +9836 520903 +38387 520845 +5800 520632 +2442 520625 +10763 520466 +5498 520396 +7588 520298 +8584 520208 +1834 520004 +8830 519633 +6440 519510 +3104 519421 +8519 519326 +12980 518862 +10266 518833 +8705 518717 +7507 518708 +3175 518680 +3489 518674 +10290 518362 +5268 518131 +4498 518124 +2845 518039 +7604 518016 +4871 517787 +5640 517709 +2018 517645 +9552 517437 +2025 517381 +7690 517300 +10674 517255 +1084 517164 +1774 516848 +9481 516274 +9558 516120 +11044 516115 +7543 516105 +8054 516100 +14637 516058 +5049 515800 +12989 515724 +6866 515494 +10665 515424 +5340 515194 +5698 515174 +7907 515166 +17329 515143 +4693 515073 +4349 514677 +610 514618 +4958 514534 +9658 514511 +7058 514162 +11279 514146 +5909 513815 +2403 513591 +4273 513532 +5948 513355 +3324 513090 +7622 512788 +4543 512710 +47013 512667 +15994 512605 +16020 512591 +6227 512519 +14167 512493 +122 512466 +6439 512393 +1787 512303 +4810 512061 +6970 511802 +7348 511766 +13419 511721 +3183 511719 +1697 511536 +1346 511242 +6639 511134 +5642 511062 +11372 511039 +7827 511016 +1360 510851 +7841 510829 +1694 510698 +9638 510663 +5967 510625 +6903 510571 +5425 510480 +5931 510406 +856 510365 +16213 510309 +9840 509866 +9114 509800 +6153 509717 +3503 509652 +7663 509651 +2915 509566 +6492 509531 +4859 509528 +3797 508985 +9869 508778 +7498 508579 +7090 508348 +2935 508256 +8061 508150 +5129 508134 +2850 507942 +10346 507712 +8171 507083 +11126 506565 +12313 506541 +8114 506538 +1925 506318 +7265 506203 +10308 506177 +2788 506150 +18355 506070 +6789 505868 +7428 505805 +7739 505672 +335 505540 +4437 505375 +1046 505338 +7505 505231 +1820 505189 +6646 505079 +4847 505035 +7723 504840 +1874 504583 +10769 504401 +6062 504379 +9393 504300 +3609 504287 +8022 504015 +8163 503963 +5662 503954 +4578 503944 +28808 503835 +9733 503821 +4419 503723 +12152 503716 +9584 503534 +7724 503491 +12139 503393 +1225 503219 +12451 503123 +2404 503095 +7161 503020 +38325 502957 +13064 502866 +13959 502619 +16685 502602 +8489 502599 +2801 502372 +8948 502353 +7744 502332 +11301 502329 +18317 502311 +6380 502208 +9554 502166 +10638 502156 +6350 502075 +14395 502028 +733 501896 +11255 501750 +8793 501681 +2704 501584 +7941 501562 +9142 501326 +8766 501173 +7042 501158 +9161 501143 +14316 501138 +1058 501137 +7567 500930 +17215 500926 +7722 500875 +7686 500769 +10009 500732 +8834 500182 +1267 500098 +5194 500074 +12433 499839 +10094 499728 +1153 499598 +10826 499365 +1951 499236 +6378 499083 +10309 498859 +14771 498759 +11563 498613 +7327 498161 +7069 498100 +12274 498050 +9188 497940 +3514 497889 +9859 497683 +4294 497668 +6634 497660 +8616 497651 +14456 497583 +12420 497494 +9262 497418 +1050 497098 +8849 497057 +9337 496804 +8698 496736 +5680 496519 +8714 496340 +6599 496156 +13922 496122 +12783 496080 +13956 496070 +9904 496047 +9469 495805 +9204 495517 +6856 495493 +4282 495458 +8331 495218 +7741 495080 +7876 495033 +13101 494968 +8868 494871 +10102 494813 +6537 494662 +6174 494574 +1006 494209 +11292 493769 +8138 493741 +7417 493306 +7819 493293 +4021 493280 +5069 493031 +1676 492861 +7618 492809 +619 492743 +7111 492488 +13098 492423 +5219 492395 +9549 492153 +1674 492046 +12644 491967 +5657 491932 +9238 491889 +1186 491713 +9247 491667 +6419 491609 +14080 491582 +12506 491546 +7850 491532 +8276 491506 +8319 491476 +7745 491425 +14389 491164 +11725 491143 +6948 491116 +8852 491069 +12557 491017 +2860 490863 +12438 490713 +11938 490511 +17154 490479 +12352 490449 +9209 490055 +6465 489970 +7362 489941 +9456 489939 +3808 489894 +10691 489840 +6118 489617 +3526 489349 +11717 488745 +13553 488681 +570 488430 +8846 488377 +10579 488145 +7667 488138 +9398 488058 +6252 488041 +10816 487804 +9902 487778 +6635 487683 +10986 487574 +12255 487515 +6772 487362 +10409 487277 +4035 487133 +6508 487039 +7578 486957 +38898 486923 +8209 486827 +2301 486497 +8350 486454 +8649 486382 +10226 486188 +7477 486080 +46195 486017 +4761 485991 +6572 485950 +104 485842 +10589 485787 +5368 485706 +11057 485511 +14379 485358 +22306 485315 +10064 485172 +9040 485169 +2439 484771 +1305 484754 +9301 484652 +6295 484648 +2093 484477 +7095 484430 +9465 484269 +8867 484258 +7164 484152 +30772 483866 +8104 483650 +2516 483348 +9559 483026 +4526 482958 +11345 482881 +7632 482880 +6086 482805 +13479 482529 +8018 482355 +4217 482302 +7147 482262 +10277 482204 +12534 482102 +9613 481874 +8922 481792 +7766 481713 +5908 481572 +3260 481497 +2806 481392 +3242 481272 +9103 481197 +6412 481109 +6374 480706 +5816 480672 +2649 480629 +18325 480570 +28244 480558 +4448 480540 +3257 480335 +6947 480290 +12217 480181 +7699 480013 +3433 479977 +9694 479829 +9014 479704 +14200 479703 +6282 479693 +6541 479643 +32009 479575 +8383 479249 +4598 479147 +10209 479146 +9403 479045 +6448 478902 +17692 478863 +7457 478667 +2433 478511 +724 478152 +13045 478149 +7634 478062 +12031 477941 +8565 477889 +6365 477835 +6122 477730 +1569 477700 +8239 477668 +581 477633 +864 477246 +7007 477223 +8971 477114 +10937 477035 +7487 476747 +10596 476592 +11840 476530 +8916 476449 +9276 476256 +8676 476237 +2163 476190 +875 476049 +12996 475823 +4531 475813 +11467 475808 +3771 475740 +5625 475421 +16987 475263 +10704 475232 +36430 474914 +4377 474829 +10503 474740 +10810 474711 +9759 474548 +13189 474527 +17944 474409 +4130 474295 +25650 474213 +2606 474053 +11448 473925 +6159 473703 +7552 473678 +956 473675 +2019 473515 +4304 473492 +13347 473405 +1299 473258 +8507 473017 +10930 472996 +6209 472990 +1590 472948 +6507 472885 +12155 472759 +10401 472745 +12087 472716 +6026 472567 +8847 472500 +5184 472211 +8286 472197 +9544 472194 +6105 472192 +11231 471976 +12696 471900 +9317 471819 +4625 471755 +9327 471505 +7473 470986 +2840 470738 +8197 470623 +14780 470281 +12946 470152 +3813 469939 +12382 469887 +5308 469869 +10382 469852 +17662 469530 +13509 469357 +5650 469084 +7274 468913 +3537 468876 +6512 468810 +8466 468776 +16693 468472 +4767 468256 +8461 468202 +13285 468151 +729 468066 +2737 467943 +6654 467872 +2543 467645 +539 467455 +7169 467379 +21003 467300 +7675 467213 +12815 467106 +14138 466738 +3166 466697 +5739 466627 +6177 466578 +1898 466078 +30251 465915 +7881 465902 +9772 465804 +13312 465503 +19646 465313 +5897 465263 +7560 465238 +8119 465232 +7151 465210 +15752 464964 +4476 464929 +3077 464826 +4941 464756 +10152 464493 +3255 464477 +7930 464418 +8561 464331 +8057 464290 +5073 464075 +13191 463887 +6498 463863 +7357 463530 +9815 463411 +13489 463148 +12369 463029 +6762 462915 +4482 462736 +2350 462731 +6506 462634 +2152 462592 +11622 462312 +8668 462185 +5878 462019 +2852 462006 +16145 461943 +6064 461812 +11772 461652 +3529 461506 +11617 461437 +13662 461340 +2922 461227 +21195 461196 +8733 461097 +8546 461014 +19299 460751 +9131 460711 +7863 460685 +4240 460580 +8889 460543 +25151 460407 +41615 460212 +6322 460155 +7914 460099 +15572 459295 +4580 459246 +12682 459131 +6899 458997 +9722 458947 +5951 458826 +2701 458808 +1512 458735 +10655 458723 +9674 458635 +7285 458614 +13864 458453 +7706 458443 +14463 458408 +11969 458352 +12022 458342 +8030 458206 +5718 458203 +9499 458191 +18091 457686 +10820 457682 +11153 457653 +17189 457496 +7397 457166 +15435 457165 +10404 457133 +1079 457130 +7186 456764 +11831 456653 +10633 456482 +870 456379 +10719 456339 +4089 456215 +10470 456203 +11729 456195 +13178 456192 +3768 456183 +14691 455976 +28 455976 +10976 455911 +4663 455897 +5249 455773 +8442 455449 +5282 455325 +3720 455145 +13356 455082 +11581 454949 +2381 454810 +601 454802 +903 454655 +10099 454543 +16483 454518 +10399 454507 +1252 454428 +11305 454423 +3246 454342 +12605 454177 +15247 453973 +3866 453730 +46481 453702 +4548 453681 +10325 453562 +11566 453362 +6287 453357 +9240 453334 +6244 453271 +4474 453245 +4524 453216 +16519 453141 +1897 453137 +3732 453109 +3885 453072 +15244 452997 +11434 452891 +11100 452784 +949 452773 +10747 452756 +8223 452719 +6125 452444 +8207 452351 +12053 452321 +2870 452256 +10303 452236 +8112 452129 +2044 452086 +11749 451878 +7009 451838 +8098 451813 +11233 451812 +8853 451678 +15302 451647 +7257 451557 +4851 451508 +24247 451387 +14651 451369 +5671 451350 +29690 451218 +5673 450985 +7429 450645 +14526 450579 +1489 450407 +18381 450182 +895 450118 +18279 450047 +7511 449889 +7564 449873 +2539 449697 +10423 449668 +10222 449447 +12427 449438 +8233 449184 +15446 449140 +13659 449125 +7852 449025 +4487 449014 +14486 449007 +9750 448958 +6889 448930 +11189 448906 +11769 448691 +9528 448621 +8339 448582 +7460 448524 +7480 448475 +6645 448428 +16167 448391 +5961 448383 +11468 448323 +10947 448316 +7448 448262 +10831 448231 +7743 448200 +11790 448123 +3864 448020 +9699 447968 +17741 447947 +8551 447942 +10997 447848 +631 447787 +8956 447706 +5831 447623 +741 447620 +10428 447512 +13310 447352 +1273 447226 +8084 447095 +5508 446729 +6951 446617 +6810 446454 +9252 446194 +1304 446147 +11628 446139 +12408 446021 +5600 445908 +3620 445780 +11102 445687 +1817 445635 +15260 445631 +4521 445604 +7891 445507 +9644 445478 +10540 445477 +7623 445150 +4869 445065 +15846 445057 +13965 444997 +6547 444996 +10497 444967 +7698 444957 +7262 444924 +21996 444912 +3565 444847 +7361 444827 +8601 444817 +1090 444803 +9519 444736 +4790 444625 +6867 444594 +17420 444560 +7491 444513 +8190 444458 +1513 444222 +10522 444039 +6674 444008 +7183 443956 +2088 443644 +9207 443623 +11320 443607 +10088 443581 +11136 443530 +4460 443497 +6042 443486 +7716 443421 +13854 443389 +5674 443287 +16581 443281 +9167 443098 +7165 442919 +4687 442895 +4866 442855 +10784 442654 +4732 442633 +865 442626 +5237 442517 +12652 442432 +10075 442326 +3525 442279 +3483 442207 +853 442188 +10834 442053 +10092 442033 +6544 442002 +9212 441858 +8776 441583 +12857 441523 +9588 441423 +13445 441268 +7301 441201 +9670 441165 +9208 441056 +9397 440989 +7393 440970 +30830 440747 +14959 440610 +6286 440540 +9157 440442 +10033 440162 +8313 440089 +7407 440088 +5078 439983 +10297 439973 +3585 439845 +11632 439807 +11088 439726 +8873 439603 +11905 439537 +7050 439349 +12771 439335 +10578 439236 +14006 439225 +9189 439199 +13943 439161 +8242 438925 +27737 438892 +14685 438862 +2841 438853 +8188 438775 +6060 438757 +10657 438692 +8468 438514 +6939 438377 +5224 438336 +3602 438299 +10029 437970 +7475 437917 +21675 437790 +11328 437753 +5258 437656 +793 437502 +2105 437344 +11461 437203 +18335 437071 +6366 437018 +7995 437015 +7506 436901 +10062 436677 +6846 436653 +10159 436578 +10848 436530 +10660 436510 +7369 436436 +7032 436299 +2446 436290 +5066 436196 +7974 435903 +8568 435896 +18936 435627 +10991 435619 +8386 435599 +5628 435553 +3975 435372 +796 435152 +7445 435125 +11307 434750 +4364 434663 +9667 434529 +9629 434332 +6706 434291 +8486 434203 +3981 434045 +8137 433968 +8526 433856 +11780 433721 +12803 433690 +9817 433586 +3020 433537 +10115 433505 +4903 433496 +16077 433356 +10275 433334 +11243 433238 +12088 433209 +9144 433171 +3695 433088 +7472 433029 +9319 432997 +8026 432869 +6697 432716 +9036 432689 +6505 432658 +4911 432291 +28227 432239 +4758 432224 +1845 432086 +10836 431880 +7021 431848 +8296 431812 +12328 431752 +1005 431721 +5923 431708 +13162 431573 +13097 431521 +32825 431122 +10913 431081 +9623 430972 +13682 430905 +25440 430798 +15016 430612 +2696 430394 +1587 430367 +10463 430242 +12612 430177 +10448 430165 +9196 430158 +20241 429817 +3263 429681 +12533 429646 +12577 429472 +3907 429457 +14717 429425 +9041 429321 +10140 429303 +5440 429164 +8225 429150 +7979 429111 +5162 429109 +34979 428999 +8263 428967 +9643 428904 +548 428758 +1698 428731 +2441 428648 +11058 428630 +6575 428307 +8686 428274 +14719 428107 +9980 428077 +9299 428002 +3966 427967 +11431 427858 +626 427844 +7170 427618 +8945 427570 +21703 427336 +12199 427179 +15818 427119 +9524 427061 +94 427057 +10084 427023 +8275 426906 +9803 426806 +4848 426803 +12507 426752 +8093 426716 +5400 426665 +10822 426582 +10544 426474 +8684 426470 +3474 426393 +2824 426292 +1124 426099 +5767 425935 +7763 425894 +2052 425887 +5888 425825 +17894 425817 +6910 425796 +25792 425781 +723 425679 +11668 425517 +8860 425445 +1027 425439 +6454 425414 +6787 425354 +15958 425316 +4712 425219 +23731 425071 +39990 425025 +7822 424954 +7237 424883 +7750 424792 +5469 424745 +13030 424621 +438 424384 +2569 424271 +6795 424188 +10012 424179 +16348 424166 +14359 423890 +3994 423764 +11330 423708 +9889 423613 +7767 423523 +6761 423314 +11237 423255 +9194 423211 +8025 423111 +2787 423038 +10053 423032 +8257 422993 +3838 422979 +8999 422972 +8013 422938 +14886 422812 +3974 422673 +11803 422479 +6277 422372 +10054 422115 +8006 422012 +8482 421985 +5333 421820 +4846 421787 +11123 421771 +13795 421760 +14597 421541 +12268 421468 +9539 421435 +11646 421416 +12848 421415 +10828 421149 +4465 421005 +8312 420878 +1940 420842 +9384 420774 +10343 420682 +38563 420682 +2171 420647 +10907 420307 +14847 420294 +8888 420198 +5496 420132 +4898 419944 +10821 419846 +8754 419807 +18836 419789 +7306 419436 +4480 419357 +13107 419242 +4662 419029 +13301 418988 +7620 418900 +6387 418663 +398 418591 +13316 418591 +5404 418532 +3212 418520 +14683 418447 +14372 418412 +13683 418208 +9753 418195 +4164 418183 +11476 418137 +6869 418119 +8463 418118 +10759 417587 +12478 417567 +14813 417528 +11753 417140 +7772 417075 +15152 417032 +6973 416961 +26840 416803 +2236 416706 +8454 416300 +11228 416181 +18802 416171 +12112 416118 +7600 416058 +14345 416018 +8883 416008 +13620 415903 +3550 415841 +10282 415776 +16127 415765 +9599 415745 +11376 415739 +10074 415692 +731 415656 +7243 415472 +14057 415445 +8157 415366 +5774 415223 +6117 415193 +1117 415145 +13077 415133 +17448 415079 +14424 415006 +5868 414948 +7228 414712 +8564 414687 +1496 414653 +11939 414600 +9137 414451 +18042 414426 +8871 414367 +7527 414259 +9210 414225 +6942 414161 +9284 414093 +11864 413979 +4729 413800 +15336 413797 +29749 413689 +9808 413450 +6986 413448 +6397 413294 +15598 413245 +14819 413174 +13547 413105 +6777 413085 +13804 413005 +13257 412980 +7446 412953 +6273 412946 +12620 412923 +7178 412896 +8953 412856 +10893 412416 +9031 412276 +702 412269 +15338 412154 +7747 412118 +7872 412058 +12769 412014 +11722 411923 +9983 411914 +16256 411890 +10732 411859 +19277 411821 +5883 411797 +11942 411756 +7255 411620 +10568 411592 +22917 411489 +3425 411402 +3494 411372 +18707 411253 +3185 411125 +15733 411004 +10460 410993 +4453 410817 +4401 410718 +7363 410707 +20954 410678 +10270 410579 +8404 410530 +16570 410515 +16863 410211 +11702 410131 +4235 409832 +9001 409794 +12806 409633 +12223 409518 +9396 409503 +13797 409366 +12402 409239 +7526 409230 +12600 409139 +15100 409091 +10681 409050 +10246 408972 +11872 408925 +7579 408648 +9172 408405 +17796 408380 +9115 408352 +21793 408261 +12691 408216 +6364 408060 +6859 408027 +13038 407988 +1632 407958 +12477 407924 +15378 407851 +9352 407688 +3633 407655 +5803 407653 +8646 407621 +7587 407575 +15331 407451 +1758 407253 +877 407147 +11998 407145 +11679 407103 +3601 407096 +4669 407086 +227 407081 +11334 407006 +8131 406946 +1759 406891 +8452 406865 +8997 406822 +9574 406812 +6797 406810 +11336 406762 +8501 406738 +11114 406687 +10311 406649 +10150 406623 +17689 406578 +11300 406421 +9814 406384 +8389 406349 +8271 406333 +7954 406313 +10576 406312 +6565 406306 +7545 406240 +9102 406164 +14328 406013 +2144 405990 +13673 405932 +5337 405901 +8865 405864 +1155 405852 +3719 405528 +9258 405495 +13093 405422 +9011 405319 +6257 405240 +1835 405060 +12871 405052 +5588 404835 +16629 404629 +9799 404597 +15295 404559 +11066 404496 +8716 404329 +12928 404150 +12206 404017 +9635 404014 +7635 403984 +7561 403954 +12388 403633 +13630 403434 +13592 403053 +15896 402919 +20197 402829 +9645 402757 +13393 402755 +12165 402698 +14074 402666 +9332 402472 +12406 402273 +6938 402132 +8471 402079 +15615 401716 +14410 401641 +8020 401607 +9573 401541 +813 401316 +13709 401296 +7267 401278 +4049 401218 +7817 401160 +1125 400938 +6695 400924 +753 400632 +13117 400629 +7467 400617 +7405 400469 +8741 400432 +1347 400368 +10095 400320 +10692 400261 +7558 400226 +9133 400206 +4467 400136 +8814 399952 +1648 399945 +5172 399767 +1723 399704 +5384 399677 +32425 399666 +8130 399658 +26094 399611 +6602 399400 +7501 399051 +21356 399047 +13960 399015 +26119 399002 +1177 399001 +15076 398977 +7965 398937 +14072 398855 +2545 398812 +15033 398769 +10156 398733 +9015 398630 +5550 398523 +3735 398483 +15694 398450 +12318 398428 +9709 398412 +4156 398372 +6937 398056 +13447 397970 +11083 397902 +7612 397887 +677 397838 +4709 397653 +8539 397640 +3405 397576 +15184 397562 +10880 397473 +7718 397294 +1846 397197 +13919 397195 +17199 397161 +12963 397157 +2357 397072 +8168 396992 +13067 396930 +7360 396710 +6625 396663 +12670 396660 +11363 396611 +8644 396544 +1071 396138 +9569 396044 +11103 396035 +16278 396022 +12356 395953 +20790 395715 +12160 395513 +1278 395447 +9985 395426 +6589 395388 +1322 395383 +14327 395356 +10727 395331 +6824 395153 +15301 394895 +9233 394857 +8970 394826 +8147 394812 +11406 394778 +4129 394771 +29657 394731 +13339 394630 +3141 394547 +26220 394385 +9029 394178 +14133 393701 +4313 393607 +10741 393597 +12000 393560 +11038 393455 +7708 393396 +6981 393382 +48539 393366 +12965 393321 +3051 393202 +9697 392989 +2629 392849 +556 392807 +11481 392800 +6595 392750 +1371 392745 +19099 392599 +5602 392531 +7039 392512 +14905 392497 +11118 392472 +18390 392419 +2848 392286 +7982 392117 +1034 392071 +7825 392001 +12930 391952 +8262 391872 +926 391870 +13904 391618 +3528 391502 +6259 391442 +432 391392 +4897 391238 +2797 391028 +21852 390801 +3844 390776 +7075 390638 +11972 390586 +18286 390550 +10914 390522 +12181 390457 +10274 390401 +2411 390339 +11704 390287 +9856 390187 +9268 390171 +12150 390103 +9879 389997 +7471 389987 +9185 389986 +12879 389967 +7455 389884 +1747 389879 +13340 389812 +10127 389663 +10402 389568 +19613 389434 +4285 389340 +2038 389257 +5847 389229 +8097 389217 +10014 389052 +7695 389038 +6964 388887 +11915 388736 +17305 388721 +3069 388564 +6779 388434 +7866 388384 +1789 388251 +12684 388146 +5521 388118 +10595 388088 +6873 387763 +1818 387683 +7054 387518 +10754 387382 +6451 387287 +9655 387128 +8355 387116 +8129 387080 +16393 387050 +13999 386969 +7787 386929 +11276 386759 +5562 386693 +13741 386688 +12285 386638 +6641 386580 +6737 386535 +7187 386445 +2536 386412 +14122 386389 +15682 386380 +6234 386377 +10302 386248 +14115 386219 +16666 386042 +10332 385907 +3964 385839 +6620 385645 +10361 385548 +8902 385496 +10966 385484 +14187 385467 +13633 385375 +17063 385313 +18621 385065 +10617 385042 +12539 384984 +14997 384883 +11983 384868 +5468 384814 +7238 384796 +10721 384572 +15555 384479 +1275 384392 +11432 384301 +2977 384284 +10569 384253 +13359 384110 +11560 384076 +10342 384009 +6569 383954 +19122 383938 +22611 383811 +6036 383687 +8474 383644 +5740 383429 +7692 383359 +586 383295 +1419 383273 +4066 383194 +1538 383149 +3191 382945 +12284 382931 +14620 382918 +9110 382879 +4825 382730 +21199 382531 +8126 382516 +11652 382509 +3889 382433 +4352 382237 +8478 382053 +5422 381988 +6597 381839 +8246 381825 +16052 381819 +2748 381810 +9530 381684 +16119 381666 +6445 381644 +18292 381607 +23370 381562 +10089 381523 +11077 381365 +7226 381330 +13329 381179 +14021 381124 +9758 381123 +11809 381086 +13772 381080 +3036 381064 +15223 381038 +6486 381025 +6814 380938 +13241 380861 +8027 380855 +8884 380820 +2054 380811 +19709 380793 +6504 380605 +9626 380566 +11515 380412 +8628 380336 +13367 380121 +13861 380071 +31362 380053 +1647 380010 +17820 379781 +9166 379560 +6906 379374 +14469 379282 +14227 379210 +3754 379002 +11001 378926 +11526 378867 +8347 378753 +7862 378733 +9281 378662 +17718 378652 +4293 378636 +13302 378542 +6982 378491 +14217 378485 +8494 378459 +4932 378435 +10408 378283 +15796 378222 +10588 378132 +23357 377965 +17462 377860 +8904 377819 +11000 377789 +16738 377598 +17499 377581 +11677 377538 +4936 377530 +1130 377448 +9007 377376 +9900 377348 +7382 377267 +5760 377258 +937 376979 +12252 376780 +20311 376769 +849 376755 +1389 376721 +17715 376614 +10135 376600 +14066 376597 +7026 376387 +132 376386 +6957 376353 +11734 376300 +3258 376225 +6452 376005 +909 375926 +11332 375842 +10386 375819 +10710 375699 +14825 375691 +8088 375684 +12148 375681 +9936 375544 +7270 375490 +11781 375370 +6226 375337 +2437 375301 +9846 375164 +9296 375038 +17027 374977 +2337 374938 +3912 374885 +15162 374882 +8252 374852 +10484 374831 +20251 374829 +2165 374799 +6901 374710 +10243 374664 +13671 374469 +7358 374221 +16120 374131 +3149 374119 +9021 374053 +7110 374044 +11078 373834 +1484 373662 +8069 373610 +10868 373588 +3184 373585 +2754 373555 +1292 373520 +10146 373513 +6912 373432 +7754 373402 +4013 373341 +3792 373296 +9149 373012 +11259 372932 +12159 372917 +4700 372912 +1007 372888 +21538 372840 +4959 372816 +6479 372812 +13226 372725 +7586 372712 +14105 372647 +16720 372515 +5653 372464 +13336 372347 +13612 372275 +4589 372141 +13914 372001 +14433 371923 +1631 371838 +7166 371618 +12906 371602 +8234 371586 +5705 371556 +7875 371528 +3929 371451 +13354 371418 +9677 371286 +9619 371233 +7247 371188 +9511 370981 +8794 370937 +21506 370897 +10058 370891 +10305 370833 +10192 370769 +12665 370754 +16752 370681 +7375 370584 +100 370551 +10145 370550 +12396 370469 +2930 370325 +16440 370303 +6579 370297 +8523 370210 +6450 369913 +2471 369759 +3039 369622 +823 369518 +8673 369498 +28689 369444 +14728 369442 +13019 369391 +14537 369330 +9177 369323 +11846 369307 +9564 369236 +15286 369223 +15725 369135 +8258 369120 +16094 369079 +8217 368989 +12897 368893 +14162 368880 +10918 368753 +7820 368688 +4563 368543 +9807 368432 +9561 368424 +18609 368234 +9075 368089 +9085 368050 +10231 368001 +11138 367959 +12246 367928 +972 367806 +10255 367786 +7906 367661 +8680 367603 +15115 367522 +13048 367461 +11087 367387 +9494 367342 +18071 367262 +964 367230 +12373 367196 +4175 367107 +8659 367082 +2070 367075 +7045 367061 +14510 367048 +10571 367012 +15070 366988 +12654 366942 +3310 366871 +7548 366733 +5977 366666 +12259 366570 +4909 366534 +16640 366491 +3167 366477 +4558 366360 +19610 366257 +11099 366173 +12232 366119 +7509 366023 +9374 365891 +3470 365884 +17127 365533 +8192 365498 +7114 365476 +10380 365341 +16137 365300 +4366 365277 +7946 365216 +7365 365191 +7088 364886 +9480 364726 +8216 364713 +7685 364626 +12932 364617 +2195 364581 +9931 364506 +13844 364477 +11390 364405 +8220 364387 +8253 364234 +14408 364112 +7882 364093 +12263 364016 +7152 363829 +3998 363827 +1087 363800 +2507 363606 +7427 363566 +7521 363514 +10708 363450 +7714 363438 +2248 363431 +15930 363224 +6651 363153 +7733 362858 +6457 362836 +8566 362822 +8498 362813 +1063 362682 +2353 362622 +10040 362443 +14931 361977 +23512 361973 +6766 361959 +5999 361585 +9348 361559 +9376 361481 +7603 361432 +7422 361379 +10184 361347 +5648 361297 +3320 361123 +10017 361097 +14909 361015 +9910 360987 +3824 360918 +6515 360811 +7496 360806 +19303 360766 +8791 360691 +12523 360595 +8993 360575 +5801 360431 +4529 360398 +10965 360256 +229 360226 +9457 360164 +11252 360152 +16346 360033 +14449 359968 +11686 359944 +18334 359855 +14812 359787 +10468 359667 +7998 359571 +9964 359523 +7451 359379 +9241 359237 +6711 359234 +4545 359213 +8287 359148 +816 358897 +10147 358736 +12380 358729 +9949 358648 +1196 358550 +9107 358437 +15795 358413 +5824 358312 +5372 358295 +5124 358289 +9086 358245 +6748 358216 +9168 358064 +6805 357931 +15498 357866 +14679 357853 +14300 357819 +15528 357737 +6069 357670 +40138 357664 +12763 357550 +16051 357498 +12389 357475 +4002 357463 +12069 357303 +6371 357290 +13572 357238 +15342 357172 +9860 357116 +7664 356998 +7950 356936 +10846 356850 +17547 356634 +18475 356600 +13125 356555 +9328 356554 +6543 356451 +2954 356267 +15055 356232 +10833 356213 +26878 356180 +5013 356178 +6953 356175 +5654 356154 +6101 356107 +14641 356073 +9151 356027 +4421 355987 +16535 355894 +5639 355847 +5050 355821 +9938 355713 +11325 355685 +7531 355646 +12981 355592 +7666 355538 +9399 355432 +13415 355422 +7538 355305 +1258 355163 +18545 355158 +2759 355119 +1136 355109 +6617 355081 +13571 355038 +14069 355019 +12437 354981 +17960 354969 +9325 354939 +13708 354915 +6420 354705 +14757 354591 +14919 354480 +9199 354451 +4108 354398 +12949 354382 +6578 354376 +9361 354353 +10718 354244 +13593 354199 +6052 354007 +8436 353994 +9105 353886 +12275 353856 +9121 353405 +11106 353379 +10800 353350 +8598 353325 +2100 353199 +2290 353173 +15080 353027 +11642 353004 +16542 352734 +16111 352650 +15602 352627 +10608 352611 +5313 352603 +17782 352590 +7838 352517 +16445 352493 +6909 352475 +14791 352413 +9857 352193 +17842 352066 +1638 352040 +16712 351998 +2164 351959 +14364 351891 +3287 351887 +8706 351810 +12873 351660 +13075 351610 +22484 351603 +16707 351599 +7381 351585 +4463 351585 +699 351567 +11856 351470 +25851 351312 +10364 351096 +8266 350892 +12492 350873 +14493 350708 +8854 350697 +13346 350683 +10469 350675 +10216 350657 +5147 350609 +4169 350591 +10872 350527 +10366 350409 +12801 350394 +9572 350320 +8149 350232 +6375 350198 +5447 350173 +28141 350103 +18733 350035 +13642 350000 +1144 349908 +10421 349802 +10427 349683 +14377 349622 +17826 349557 +6528 349442 +9877 349409 +4198 349335 +11550 349300 +14840 349249 +16952 349120 +7776 349119 +8914 349098 +15164 349012 +6469 348997 +6206 348967 +7731 348954 +6890 348922 +1348 348722 +10086 348564 +15037 348531 +9345 348509 +10474 348482 +8949 348330 +35327 348207 +13458 348166 +4901 348016 +13853 348002 +10925 347889 +9835 347859 +5414 347837 +3369 347791 +8582 347636 +11682 347617 +31433 347583 +9136 347427 +4702 347410 +9482 347376 +11271 347359 +566 347358 +17334 347227 +13850 347132 +11023 347130 +18263 347068 +10400 347041 +27133 346684 +6462 346645 +11272 346626 +9963 346622 +15198 346518 +7818 346505 +646 346495 +6778 346466 +9775 346459 +13485 346404 +9013 346329 +11946 346288 +10668 346248 +1330 346195 +8517 346056 +6239 345826 +9557 345703 +16978 345593 +8348 345532 +1272 345478 +11238 345461 +18555 345457 +20872 345277 +10359 345271 +17036 345196 +2120 344930 +2280 344773 +8573 344722 +10515 344702 +12020 344681 +10597 344592 +9260 344591 +12047 344581 +10429 344566 +11185 344446 +5892 344360 +5250 344253 +9160 343902 +8458 343880 +7252 343790 +31173 343777 +4022 343555 +10003 343480 +11193 343472 +6562 343348 +13925 343228 +17406 343189 +4730 343165 +11155 343055 +12231 343030 +17014 343022 +16618 342966 +7278 342897 +9211 342896 +5100 342826 +15066 342723 +11594 342721 +12527 342717 +8288 342688 +3664 342668 +9490 342586 +3839 342537 +14429 342519 +1075 342420 +2232 342183 +10980 342178 +2022 342057 +11472 342053 +3979 342036 +15346 341959 +15863 341944 +1691 341900 +15294 341872 +11112 341843 +14569 341817 +8915 341671 +10304 341465 +4843 341417 +6494 341415 +7926 341332 +8515 341225 +12046 341172 +1216 341117 +9473 341068 +9274 341050 +11570 341048 +6659 340992 +22473 340911 +8075 340763 +15393 340693 +14883 340689 +2522 340630 +9960 340607 +8212 340582 +7208 340577 +10594 340450 +21909 340415 +10765 340343 +4583 340262 +8603 340144 +13637 339972 +7138 339832 +9581 339824 +13188 339727 +15689 339721 +8749 339613 +18576 339525 +8425 339432 +10241 339308 +3902 339257 +11937 339211 +8222 339181 +7621 339139 +2518 339112 +11758 339065 +5855 338929 +12173 338908 +12615 338903 +8801 338853 +842 338791 +13142 338723 +5516 338721 +12329 338602 +7388 338534 +26105 338531 +5032 338437 +7795 338417 +8957 338345 +2483 338332 +10038 338262 +3743 338006 +8181 337997 +14952 337748 +5492 337746 +12882 337707 +11648 337676 +8806 337599 +5833 337517 +9780 337469 +4623 337392 +17880 337247 +4745 337244 +12538 337238 +6369 337188 +6758 337119 +15706 337069 +15870 336993 +9400 336894 +5721 336815 +7097 336767 +7869 336739 +20092 336483 +13229 336427 +20486 336410 +12970 336280 +8786 336264 +8440 336084 +12746 336078 +8268 335957 +19034 335955 +20082 335940 +5601 335924 +5210 335873 +17907 335805 +6075 335758 +4835 335634 +2129 335490 +10931 335423 +5607 335409 +11333 335370 +7653 335365 +12400 335354 +11602 335344 +4731 335188 +12575 335173 +1618 335125 +13606 335118 +21285 335017 +12107 334923 +1387 334914 +34033 334900 +14495 334864 +7331 334829 +18740 334712 +3987 334670 +16134 334569 +14860 334478 +1062 334442 +17185 334381 +12497 334367 +19616 334301 +3696 334292 +12772 334266 +11403 334201 +10172 334185 +2183 334078 +17409 334077 +9646 333987 +10941 333957 +4424 333731 +13109 333719 +32214 333692 +9408 333646 +5703 333577 +9901 333570 +7135 333416 +11495 333379 +13953 333300 +10182 333300 +8541 333269 +13833 333263 +9471 333259 +16527 333092 +16533 333048 +1739 333033 +2160 332999 +24520 332927 +17541 332914 +245 332905 +1601 332894 +8989 332859 +16512 332747 +4265 332705 +14576 332575 +3150 332510 +12565 332496 +10439 332445 +31112 332411 +14926 332365 +11450 332296 +1984 332256 +9871 332254 +4277 332226 +11684 332109 +6882 331909 +6733 331893 +11521 331862 +22488 331824 +10839 331793 +1055 331762 +10969 331754 +11177 331679 +18154 331626 +5737 331561 +7801 331482 +14409 331482 +11597 331282 +12023 331207 +9170 331045 +2799 331010 +8124 331002 +11553 330990 +9022 330926 +12827 330921 +10801 330902 +1699 330871 +12880 330544 +12465 330534 +14286 330531 +14783 330519 +7902 330467 +20000 330396 +10750 330328 +8887 330300 +14317 330224 +9620 330208 +13280 330181 +8712 330141 +11584 330100 +5053 329959 +5907 329943 +4415 329942 +10454 329927 +11132 329916 +14858 329894 +15670 329864 +8125 329754 +15422 329750 +12831 329749 +37666 329684 +16433 329663 +3192 329598 +11636 329464 +10193 329407 +11375 329359 +10695 329328 +14388 329322 +8180 329251 +9874 329097 +8920 329091 +4534 328825 +8903 328698 +23964 328655 +17122 328606 +7048 328597 +6211 328591 +8925 328576 +11955 328523 +2378 328502 +14896 328482 +9751 328231 +12578 328224 +1773 328185 +6338 328096 +8839 327999 +10923 327937 +7778 327911 +12617 327835 +9018 327783 +11732 327780 +7563 327739 +6803 327707 +8969 327692 +5552 327602 +3905 327557 +9911 327480 +7119 327458 +36285 327444 +7684 327424 +9616 327313 +11129 327297 +12032 327230 +11490 327080 +7792 327061 +13574 327032 +11398 327007 +8066 326906 +1952 326868 +2710 326703 +13362 326668 +14457 326642 +17339 326625 +12580 326624 +21861 326488 +21157 326473 +34119 326457 +14235 326388 +11819 326384 +4102 326329 +7209 326168 +6195 326143 +10291 326111 +8894 326110 +7330 325951 +5869 325935 +14525 325893 +13120 325862 +6167 325828 +9509 325807 +24382 325742 +7855 325721 +10078 325656 +7112 325654 +5599 325576 +8122 325525 +15105 325514 +15740 325468 +16753 325462 +9517 325427 +10953 325424 +13936 325332 +8862 325273 +3464 325244 +5452 325232 +20983 325102 +17582 325099 +1483 324860 +8035 324825 +8824 324800 +14119 324704 +12766 324689 +9412 324668 +12780 324579 +12521 324471 +9752 324452 +13923 324383 +6001 324264 +13145 324264 +9464 324208 +7336 324205 +15626 324177 +19773 324148 +8701 324135 +8504 324114 +9422 324110 +10457 324084 +5832 324083 +14654 324004 +13777 323947 +5142 323924 +3843 323885 +14264 323868 +8621 323792 +6176 323751 +10538 323625 +38836 323556 +9958 323458 +493 323456 +15406 323421 +10098 323257 +13272 323251 +14882 323077 +18369 323073 +18525 323037 +11278 322890 +24754 322889 +3536 322832 +12542 322786 +7893 322728 +2971 322724 +6949 322711 +12699 322709 +11920 322700 +9813 322679 +17078 322609 +5446 322605 +2778 322596 +13694 322559 +14567 322526 +12049 322315 +23576 322120 +4316 322022 +8699 322006 +9743 321964 +4331 321936 +10843 321863 +19310 321849 +8469 321771 +4121 321673 +12513 321647 +4470 321634 +9431 321433 +5793 321318 +7773 321315 +21279 321289 +8713 321248 +1233 321228 +3731 321181 +17381 321180 +9628 321107 +4386 321100 +9845 321023 +19250 320960 +5064 320841 +5540 320721 +4185 320694 +15469 320670 +7735 320664 +21313 320604 +965 320601 +10529 320574 +12545 320556 +725 320395 +15593 320377 +12641 320315 +3364 320117 +9429 320034 +11009 319918 +11293 319719 +17357 319656 +4105 319633 +2224 319519 +15287 319462 +15929 319083 +5500 319073 +5848 318999 +3826 318962 +7353 318844 +7721 318797 +6495 318768 +16093 318732 +12309 318723 +18672 318712 +20171 318609 +16852 318532 +11169 318478 +9314 318476 +2229 318472 +7329 318393 +7416 318319 +20384 318047 +11004 317965 +17524 317883 +8678 317800 +11540 317774 +659 317752 +5036 317723 +13106 317669 +8787 317543 +13029 317496 +20716 317472 +2821 317076 +13748 317005 +9391 316991 +19332 316987 +19483 316986 +9727 316962 +11658 316942 +12607 316917 +13883 316914 +9155 316768 +2348 316745 +17955 316743 +1373 316719 +7873 316717 +2069 316670 +21105 316623 +8771 316595 +10452 316592 +11756 316513 +17915 316507 +6455 316406 +11824 316389 +22815 316388 +13784 316374 +7424 316364 +6386 316312 +20617 316271 +5557 316215 +20357 316185 +14794 316066 +17352 316046 +28630 315957 +5968 315855 +16428 315835 +6920 315834 +11130 315824 +11545 315711 +2118 315680 +3727 315673 +16182 315559 +15229 315523 +16387 315368 +21933 315289 +11359 315283 +10649 315261 +4659 315233 +14240 315221 +17356 315168 +17192 315122 +8689 315037 +11765 315023 +6251 314974 +12141 314897 +9055 314803 +11845 314792 +13566 314771 +9934 314711 +2741 314681 +27344 314655 +8281 314654 +5874 314594 +9891 314528 +12634 314519 +8782 314510 +21865 314451 +9520 314369 +11768 314347 +10206 314338 +12759 314296 +4255 314282 +35185 314258 +10271 314254 +11926 314202 +1385 314190 +9323 314182 +25841 313849 +13644 313842 +3686 313681 +18726 313573 +10059 313482 +1328 313246 +8365 313222 +7096 313182 +12811 313118 +21877 313081 +16679 313028 +9811 313018 +11882 313016 +5416 312970 +7275 312919 +7613 312904 +4177 312865 +9460 312773 +18848 312722 +14544 312645 +16915 312621 +13901 312583 +7156 312558 +12735 312557 +14850 312525 +9373 312429 +7888 312382 +13449 312351 +29378 312294 +13307 312290 +22026 312233 +17521 312201 +9892 312174 +16476 312147 +1333 312144 +13342 312097 +14598 312036 +18859 312015 +8765 311951 +12228 311876 +5355 311833 +13933 311638 +20741 311606 +9824 311582 +9104 311540 +26138 311538 +13768 311520 +23481 311505 +14600 311385 +18309 311349 +931 311338 +3673 311267 +10651 311194 +6685 311113 +11473 311083 +13276 311034 +8880 310999 +6489 310946 +12695 310937 +4802 310930 +9355 310849 +10938 310762 +4743 310727 +15300 310686 +10768 310656 +9575 310603 +12519 310526 +10215 310423 +13124 310339 +10170 310325 +13902 310286 +13944 310233 +12540 310206 +13317 310177 +12151 310173 +1286 310125 +7125 309987 +11172 309933 +14669 309886 +12926 309859 +7661 309685 +9794 309629 +9585 309568 +14135 309560 +2194 309554 +11294 309522 +2544 309428 +13692 309380 +16248 309370 +17196 309335 +9175 309081 +15261 309005 +1473 308985 +1960 308936 +13456 308907 +10546 308821 +14762 308772 +10591 308714 +11721 308591 +3086 308558 +9363 308490 +13176 308475 +11989 308474 +10543 308427 +23781 308348 +13403 308323 +9099 308280 +11784 308078 +32089 307995 +529 307911 +95 307756 +3939 307654 +16435 307639 +20182 307587 +34847 307510 +10509 307452 +6071 307446 +11500 307410 +10384 307357 +15343 307354 +1963 307209 +9541 307199 +14939 306944 +20014 306924 +11903 306893 +5921 306873 +12723 306865 +3178 306813 +17389 306782 +12705 306774 +12068 306684 +3460 306676 +10070 306651 +11192 306520 +21841 306506 +9017 306491 +20755 306438 +6843 306369 +8670 306349 +17281 306327 +19391 306285 +9450 306242 +16574 305876 +12812 305859 +5067 305834 +7295 305811 +15604 305795 +15454 305645 +9757 305637 +7709 305617 +7443 305616 +2558 305584 +11013 305563 +11161 305551 +8661 305334 +13146 305327 +9294 305325 +6248 305317 +16457 305285 +24505 305278 +9550 305260 +535 305241 +18677 305121 +13832 305112 +5325 305045 +10630 304946 +12983 304915 +20119 304903 +8161 304873 +13840 304833 +12528 304832 +2085 304744 +15664 304720 +885 304692 +22593 304679 +9563 304674 +15392 304568 +9776 304540 +7815 304496 +5331 304477 +5023 304443 +17661 304421 +12344 304409 +14082 304392 +14737 304191 +10134 304135 +19952 304119 +12955 303828 +18505 303801 +7812 303777 +21574 303696 +5209 303694 +11234 303620 +9867 303608 +12751 303483 +12091 303455 +12606 303441 +11142 303322 +9447 303166 +11533 303122 +16168 303103 +7207 303092 +11666 302797 +9842 302745 +11322 302716 +11410 302571 +9567 302570 +18256 302546 +9706 302540 +10494 302536 +14899 302514 +8537 302402 +14101 302392 +5430 302339 +16307 302301 +3325 302118 +13766 302110 +6431 302098 +11626 302087 +1434 301879 +10518 301863 +12669 301838 +10869 301811 +7702 301726 +7258 301624 +7605 301599 +4891 301498 +1504 301484 +11665 301409 +11426 301408 +18220 301381 +13512 301342 +17068 301234 +1942 301225 +3311 301188 +9415 301179 +3820 301175 +11512 301087 +8719 300995 +8182 300908 +17235 300901 +5777 300818 +5794 300789 +7223 300767 +18864 300667 +10443 300646 +24589 300646 +17132 300620 +4340 300534 +7104 300478 +1188 300402 +3019 300388 +6992 300317 +8354 300316 +13609 300300 +4245 300252 +9079 300197 +15199 300105 +7157 300038 +762 299991 +17450 299943 +17044 299810 +16349 299699 +8975 299568 +4806 299541 +2895 299520 +15573 299493 +13116 299369 +10625 299261 +13037 299171 +2419 299150 +7610 299146 +13201 298875 +7599 298787 +3831 298725 +15984 298640 +5171 298514 +2604 298497 +13594 298417 +10287 298407 +10032 298356 +14538 298240 +31061 298197 +17630 298135 +14503 298117 +7808 298087 +4000 298047 +5660 298010 +6213 297993 +14417 297966 +8495 297957 +6245 297936 +29115 297877 +10069 297860 +5932 297831 +847 297818 +15459 297746 +7197 297706 +7937 297624 +6407 297566 +8477 297541 +17490 297510 +7957 297419 +19847 297359 +16298 297176 +16328 297109 +5097 297095 +18708 297085 +2817 297026 +10758 296992 +6941 296987 +17930 296964 +9322 296934 +10299 296918 +7644 296748 +13320 296692 +10927 296691 +11812 296683 +18151 296588 +8650 296576 +11952 296532 +653 296525 +10467 296457 +222 296347 +18985 296336 +17646 296312 +3361 296291 +15508 296281 +10908 296275 +12299 296124 +11589 296065 +4357 296044 +4649 296014 +19912 295919 +10238 295896 +7598 295800 +10902 295794 +824 295763 +2434 295738 +108 295681 +14396 295673 +19566 295671 +16313 295653 +4509 295636 +14616 295610 +17619 295541 +10999 295494 +5755 295491 +8697 295465 +13830 295385 +19045 295195 +21162 295171 +5489 295157 +11191 295115 +15153 295040 +3449 294958 +11150 294898 +18133 294799 +10989 294694 +14770 294585 +11554 294580 +10476 294501 +5002 294469 +3440 294422 +21765 294385 +11791 294325 +15840 294296 +17719 294251 +5812 294223 +25236 294167 +16721 294070 +16888 293941 +868 293915 +12118 293877 +6458 293767 +14533 293678 +10000 293634 +24664 293626 +12019 293496 +1686 293475 +8309 293429 +3055 293333 +13230 293232 +12536 293224 +15369 293223 +8941 293204 +14778 293112 +22535 293108 +11015 293042 +18825 292878 +17159 292877 +8895 292682 +1035 292678 +15372 292655 +14248 292637 +10898 292633 +13973 292544 +11801 292523 +20261 292508 +10609 292453 +11311 292434 +17671 292434 +7768 292322 +26318 292282 +16031 292266 +10948 292244 +10899 292189 +2345 292185 +12671 291955 +11889 291889 +20899 291879 +13183 291782 +5286 291580 +11909 291559 +11637 291497 +6965 291479 +12525 291428 +6170 291277 +4103 291236 +12937 291227 +15523 291170 +16156 291108 +24272 291089 +9507 291083 +11899 291014 +17857 290869 +6521 290825 +4092 290814 +7159 290777 +24320 290770 +12435 290704 +8492 290694 +11932 290634 +11206 290586 +14447 290535 +18961 290530 +48099 290522 +8640 290518 +492 290477 +1418 290394 +19982 290348 +9225 290286 +2261 290270 +11168 290218 +18479 290213 +20499 290165 +11810 290067 +11210 289892 +9329 289865 +15471 289784 +14518 289756 +18020 289680 +14488 289576 +7035 289565 +8662 289554 +11681 289503 +21562 289256 +4863 289185 +4064 289173 +19247 289165 +5195 289164 +16582 289117 +6908 289054 +21866 289013 +12266 288914 +3697 288830 +10233 288812 +12377 288775 +14734 288753 +33683 288737 +771 288733 +2030 288732 +11026 288731 +15317 288722 +6902 288708 +48796 288704 +22386 288678 +21614 288662 +2134 288587 +5765 288576 +13584 288406 +7378 288289 +819 288274 +10129 288254 +11075 288220 +2680 288189 +13831 288175 +18393 288077 +11676 288063 +15383 287991 +4359 287991 +36030 287903 +20852 287857 +5181 287657 +10726 287650 +21323 287633 +10559 287568 +14332 287540 +9555 287519 +9595 287510 +6194 287503 +6539 287493 +11818 287374 +1202 287364 +4319 287305 +18110 287296 +6573 287280 +15891 287257 +9293 287226 +12432 287194 +10030 287189 +18086 287141 +14086 287073 +10762 287047 +13465 287036 +19744 287018 +9387 287010 +14867 286974 +11464 286951 +24003 286921 +8617 286874 +10471 286857 +3112 286748 +12725 286626 +21165 286488 +7319 286337 +12036 286305 +4229 286285 +12537 286255 +11005 286249 +697 286174 +10700 286147 +12548 286127 +7515 286098 +6109 286082 +16015 286066 +8046 286061 +11544 285977 +13855 285908 +11735 285898 +15192 285893 +9653 285800 +10792 285796 +7616 285649 +15556 285556 +17799 285509 +9773 285508 +25287 285465 +14416 285439 +18572 285395 +10358 285292 +9183 285207 +12073 285159 +16754 285095 +8721 285047 +17081 285026 +12593 285023 +9568 284957 +8101 284931 +12002 284929 +8472 284908 +7770 284900 +5077 284807 +28852 284805 +19523 284761 +14219 284693 +15177 284685 +13722 284615 +12734 284575 +13404 284454 +4982 284434 +6751 284376 +10008 284360 +6096 284343 +14999 284301 +10087 284270 +11918 284068 +11699 284065 +10701 283996 +9583 283931 +6291 283902 +15196 283850 +15416 283835 +14521 283648 +13723 283591 +7993 283539 +8559 283442 +7650 283209 +11552 283208 +7929 283037 +18106 283014 +11065 283009 +9793 282858 +10860 282804 +2189 282772 +8717 282753 +2849 282749 +4014 282718 +7900 282712 +7335 282598 +14225 282574 +20228 282555 +12633 282528 +10824 282517 +11455 282474 +552 282383 +11954 282283 +3911 282276 +6757 282252 +8840 282235 +8805 282201 +19375 282196 +8580 282166 +6738 282152 +10906 282124 +13325 282069 +12111 282057 +7193 282053 +13004 281987 +904 281961 +15579 281954 +11003 281952 +17608 281805 +8135 281776 +12308 281762 +5994 281742 +12689 281646 +13678 281637 +3400 281600 +17151 281552 +20176 281468 +6381 281450 +10420 281445 +16105 281417 +6283 281376 +16774 281323 +12865 281321 +9675 281161 +21439 281160 +10073 281140 +15012 281072 +14820 281032 +4420 281012 +13389 280931 +15278 280924 +14707 280864 +11693 280834 +20989 280800 +9061 280631 +7811 280622 +17702 280585 +19712 280517 +18085 280506 +15792 280423 +19747 280412 +8315 280313 +39683 280210 +4679 280163 +14674 280064 +23039 280029 +20707 280003 +21014 279940 +10250 279930 +11936 279904 +15062 279814 +13588 279790 +11975 279759 +16528 279751 +24687 279670 +15616 279652 +11119 279609 +20355 279492 +9458 279484 +2603 279343 +6335 279249 +2340 279229 +17423 279191 +23623 279109 +17325 279054 +3570 278931 +13551 278846 +12995 278839 +18168 278838 +5781 278820 +9986 278790 +4112 278768 +13184 278726 +15890 278588 +18678 278548 +14213 278519 +1979 278509 +16420 278503 +13111 278460 +3846 278452 +5643 278436 +872 278359 +11868 278353 +12688 278278 +14273 278220 +40466 278181 +20700 278103 +14888 277999 +10218 277984 +16940 277960 +2072 277862 +12348 277847 +35946 277842 +16029 277774 +1013 277769 +28840 277738 +2896 277683 +15505 277680 +6446 277613 +8691 277549 +1298 277514 +14752 277419 +5714 277364 +8058 277325 +20894 277293 +16434 277271 +32115 277243 +10667 277221 +5889 277187 +16210 277086 +19788 277074 +3533 277064 +23555 277023 +8177 276841 +10045 276692 +1712 276688 +1335 276680 +20139 276667 +19245 276644 +10652 276628 +12798 276600 +16332 276591 +1702 276565 +13466 276542 +7894 276539 +13750 276477 +28070 276466 +4801 276375 +8758 276319 +6121 276242 +15599 276225 +9216 276173 +18913 276162 +15257 276075 +15397 276033 +12079 275990 +9154 275941 +9066 275923 +10041 275851 +18685 275759 +19362 275648 +5748 275520 +4134 275490 +10968 275417 +8116 275403 +20337 275390 +9676 275296 +14523 275249 +10280 275230 +11829 275119 +9864 275070 +2313 275056 +11010 275033 +1967 274991 +11611 274774 +17836 274745 +7967 274431 +9080 274379 +13891 274378 +11069 274377 +9197 274377 +6487 274363 +48198 274323 +11164 274304 +14725 274301 +12868 274287 +977 274251 +18358 274249 +11072 274222 +8405 273995 +13110 273961 +1436 273939 +9812 273852 +38251 273850 +4976 273834 +37175 273792 +4279 273790 +5756 273779 +10637 273707 +10130 273685 +6080 273601 +12182 273585 +7202 273453 +8850 273393 +1763 273379 +10498 273327 +2387 273318 +14058 273304 +15360 273269 +5572 273260 +6300 273225 +9193 273201 +4873 273182 +18034 273131 +10643 273103 +18966 273048 +13338 273037 +17468 273016 +8942 272969 +10706 272949 +12553 272893 +18353 272878 +15623 272786 +13573 272782 +15901 272749 +9441 272650 +10813 272641 +8735 272558 +6721 272540 +8338 272518 +22905 272496 +9639 272495 +8500 272427 +8099 272380 +9148 272311 +5854 272243 +10204 272231 +20222 272164 +10616 272112 +7036 272099 +12203 272005 +9927 271973 +2528 271969 +14672 271956 +20710 271955 +14481 271954 +13938 271929 +10776 271845 +17588 271844 +36309 271778 +6546 271746 +15296 271737 +9508 271628 +20522 271611 +2172 271603 +8202 271578 +12829 271575 +19800 271532 +26763 271446 +6752 271369 +43577 271245 +11256 271209 +19717 271164 +8095 271162 +11095 271143 +3849 271032 +8235 271029 +738 270906 +11559 270890 +20059 270852 +13408 270845 +3028 270826 +5700 270819 +2382 270815 +8033 270752 +9796 270711 +9707 270670 +15586 270602 +11263 270508 +13032 270474 +3982 270469 +18066 270444 +13059 270444 +14064 270365 +14085 270332 +28145 270328 +12911 270252 +11558 270112 +16239 270107 +15188 270048 +23761 270032 +5826 269980 +11032 269966 +20782 269958 +17838 269931 +13546 269902 +13041 269892 +16831 269852 +14779 269849 +5222 269801 +24711 269796 +36763 269737 +28125 269727 +11462 269718 +17913 269702 +5499 269615 +25335 269614 +14663 269604 +21730 269511 +14912 269493 +8711 269479 +12014 269429 +10436 269409 +14142 269313 +9888 269307 +18513 269211 +12651 269198 +13413 269143 +10261 269129 +1875 269121 +19101 269115 +20259 269074 +6581 268988 +11832 268914 +6224 268884 +11557 268878 +14667 268836 +9713 268811 +4943 268781 +16383 268725 +17704 268673 +20581 268654 +17642 268565 +15348 268558 +8798 268409 +9826 268357 +18772 268355 +13977 268336 +12436 268316 +11357 268262 +9305 268185 +11949 268141 +8509 268137 +10176 268091 +20529 268080 +14260 268054 +16246 268025 +7577 268019 +22635 267980 +12411 267878 +15220 267832 +16902 267772 +15726 267705 +17475 267693 +11997 267649 +9719 267643 +9966 267593 +11706 267584 +27993 267517 +9698 267510 +16769 267493 +12520 267425 +18939 267423 +6998 267352 +13714 267345 +7988 267328 +5432 267300 +9159 267270 +970 267247 +1883 267156 +12597 267130 +969 267116 +7851 267079 +11396 267044 +6319 267022 +7572 267019 +13487 267000 +13586 266983 +25945 266865 +11135 266858 +20426 266817 +22183 266781 +16334 266772 +20325 266757 +21494 266718 +5626 266678 +14033 266677 +2731 266637 +18212 266575 +12601 266521 +8396 266504 +10584 266501 +9829 266467 +5128 266439 +4718 266285 +8283 266232 +3163 266200 +18665 266165 +14043 266065 +2140 266017 +19553 265926 +12724 265897 +12908 265737 +22933 265703 +13990 265584 +15463 265520 +9162 265465 +14517 265285 +12336 265242 +5408 265206 +17167 265200 +15971 265164 +25651 265080 +8727 265079 +19017 265072 +33160 265025 +21147 264980 +16893 264915 +16797 264903 +13684 264899 +9849 264890 +16559 264831 +14718 264765 +12655 264748 +4023 264639 +9947 264581 +16155 264574 +31594 264544 +6104 264519 +15806 264392 +21221 264345 +1397 264227 +4083 264178 +23394 264177 +10083 264160 +33328 264146 +6156 264080 +12939 264080 +9950 264074 +8166 264059 +15030 264027 +32324 263999 +12346 263955 +13474 263950 +11874 263884 +19995 263882 +130 263839 +14634 263819 +5646 263795 +5421 263702 +7294 263697 +17165 263670 +6764 263622 +12503 263616 +6428 263585 +12225 263506 +4853 263495 +24161 263476 +7268 263464 +10829 263388 +8399 263381 +11694 263379 +8341 263349 +10689 263278 +17788 263200 +15832 263001 +10485 262951 +17053 262897 +98 262873 +15503 262806 +10393 262799 +21026 262785 +16089 262736 +7145 262696 +14329 262691 +10988 262667 +4938 262617 +6631 262431 +13094 262414 +4759 262370 +10713 262359 +722 262358 +6607 262294 +10338 262229 +27459 262199 +1985 262192 +8005 262074 +11160 262045 +20065 262043 +2530 262029 +9917 262001 +11107 261971 +5245 261877 +8842 261858 +13244 261844 +1756 261828 +18624 261697 +18496 261651 +13640 261637 +13234 261637 +12460 261630 +13688 261554 +934 261540 +13998 261539 +3949 261532 +15591 261500 +10766 261405 +13463 261377 +3498 261340 +7853 261336 +13657 261333 +16565 261324 +11266 261295 +10050 261267 +17187 261200 +7468 261148 +6570 261101 +9112 261095 +12841 261041 +15465 260960 +19868 260941 +14742 260914 +12080 260898 +12008 260822 +9514 260792 +8968 260774 +18240 260745 +10453 260671 +20614 260614 +12162 260544 +13534 260539 +1207 260515 +19675 260494 +5211 260493 +7532 260462 +27116 260264 +18014 260241 +4272 260231 +12101 260002 +20055 260002 +18184 259947 +21742 259850 +8187 259823 +789 259784 +7073 259779 +11338 259732 +14710 259554 +13820 259548 +16918 259503 +18327 259488 +5354 259482 +1154 259371 +13112 259364 +5411 259218 +17413 259156 +7570 259098 +17028 259065 +3775 259024 +7576 258971 +10495 258913 +6127 258860 +3742 258785 +3524 258778 +29002 258748 +15438 258703 +4948 258702 +19804 258665 +5899 258622 +2197 258614 +6453 258581 +13293 258579 +24880 258460 +9850 258441 +6974 258388 +6488 258313 +13475 258303 +9637 258255 +20633 258235 +20920 258220 +12910 258106 +11326 258084 +15443 258081 +6577 258043 +11323 258035 +8300 258029 +5735 257973 +15415 257934 +16449 257901 +2531 257843 +6894 257785 +6968 257729 +11418 257714 +8449 257532 +7688 257450 +22257 257392 +13134 257380 +8807 257349 +11281 257236 +16076 257119 +14110 257102 +27061 257101 +20430 257077 +6098 257021 +12890 257009 +5779 257002 +6473 256775 +11236 256721 +10403 256692 +3943 256662 +20052 256644 +8441 256628 +18899 256570 +14067 256490 +11576 256482 +10884 256474 +12616 256472 +4111 256447 +16905 256399 +14903 256344 +11195 256341 +7780 256319 +1164 256273 +8967 256253 +3493 256114 +8752 256058 +2541 256032 +10392 256030 +10581 255868 +3378 255861 +1798 255786 +5629 255752 +15191 255708 +11204 255626 +5536 255621 +18473 255525 +28209 255401 +8797 255344 +28086 255308 +5935 255300 +9369 255253 +8781 255237 +14601 255157 +235 255145 +755 255125 +6715 255118 +12457 255082 +3779 255039 +21525 255029 +18193 254999 +4447 254987 +24577 254972 +13069 254847 +12598 254776 +2564 254772 +11274 254713 +9765 254691 +16849 254651 +23425 254644 +22016 254589 +348 254584 +8977 254497 +10190 254449 +20442 254376 +10627 254325 +10417 254323 +13982 254312 +11353 254285 +24538 254246 +13478 254180 +7660 254144 +11131 254100 +14285 254067 +7646 254033 +12201 253999 +11029 253849 +11513 253830 +17218 253821 +1291 253782 +22159 253746 +11847 253742 +11422 253730 +4980 253724 +19817 253712 +9526 253590 +8324 253576 +7439 253554 +3575 253498 +6915 253302 +18406 253209 +14559 253205 +11853 253138 +10724 253101 +15976 253017 +4335 252999 +18489 252895 +19323 252807 +11746 252702 +4117 252677 +16443 252637 +14954 252628 +16028 252566 +15224 252495 +9961 252455 +9300 252455 +840 252429 +5513 252399 +6485 252340 +13016 252335 +10647 252236 +12556 252163 +36103 252155 +5822 252148 +10574 252130 +14159 252092 +14103 251995 +3007 251941 +4868 251923 +11608 251871 +15543 251818 +12627 251743 +2632 251725 +17369 251625 +21605 251579 +23201 251546 +11614 251528 +14636 251438 +14113 251425 +10996 251405 +1059 251391 +17277 251368 +12325 251362 +17676 251307 +15760 251285 +1358 251238 +11925 251213 +35851 251159 +16179 251087 +9506 251020 +9405 251012 +14581 250999 +21714 250981 +15866 250957 +19292 250939 +13778 250936 +2721 250909 +16805 250843 +9609 250738 +2894 250728 +1481 250694 +14984 250681 +12187 250629 +16209 250606 +6667 250598 +12039 250592 +26351 250590 +19464 250589 +3847 250521 +10096 250519 +28143 250431 +11561 250430 +18755 250425 +6831 250402 +5345 250399 +18303 250342 +12744 250205 +17645 250183 +472 250178 +13258 250161 +7889 250156 +13985 250150 +3100 250117 +12399 250031 +11842 250024 +6770 250012 +6028 249984 +19422 249883 +6185 249881 +16095 249875 +14442 249839 +18277 249820 +14008 249716 +4780 249716 +10564 249657 +7935 249650 +6524 249633 +14261 249603 +10942 249535 +8201 249530 +21872 249484 +12387 249481 +5159 249453 +11963 249430 +10852 249408 +20572 249303 +37985 249161 +26225 249133 +1428 249004 +23267 248991 +22347 248928 +542 248860 +28162 248806 +4226 248801 +11675 248800 +8947 248788 +1534 248786 +41224 248756 +9768 248730 +13677 248668 +10844 248660 +15123 248616 +37379 248570 +25672 248531 +9324 248483 +14196 248438 +12097 248435 +19687 248328 +8861 248293 +13279 248278 +13835 248278 +10975 248165 +25682 248143 +8532 248039 +11751 247945 +18524 247934 +19886 247924 +11483 247918 +29393 247899 +3003 247868 +21532 247858 +11886 247855 +9919 247809 +8579 247739 +7508 247717 +5178 247696 +10958 247675 +11760 247663 +8788 247636 +14000 247605 +17104 247525 +11574 247443 +16824 247389 +17551 247323 +2363 247278 +8117 247274 +15970 247271 +15401 247260 +19339 247215 +21336 247052 +5277 246999 +9905 246986 +15461 246929 +1426 246892 +12010 246866 +14665 246863 +7700 246845 +10825 246814 +1384 246762 +26360 246755 +14166 246713 +19786 246710 +1447 246566 +17871 246562 +4170 246466 +12802 246446 +17735 246443 +13863 246416 +9987 246399 +6771 246369 +12037 246276 +9922 246272 +11298 246270 +14071 246245 +12818 246200 +2002 246197 +2899 246184 +12034 246175 +14111 246152 +12287 246149 +18080 246097 +13892 246079 +6102 246042 +18158 246018 +14644 245998 +6158 245996 +12242 245993 +12058 245991 +1837 245897 +700 245844 +8554 245716 +3372 245711 +1287 245705 +1126 245700 +13237 245676 +2906 245653 +4472 245609 +14148 245596 +9715 245545 +11740 245521 +2320 245503 +24782 245428 +9721 245401 +822 245388 +23063 245383 +2416 245248 +44155 245172 +49443 245152 +19522 245137 +712 245074 +22297 245043 +10744 245042 +15395 244980 +9837 244959 +16674 244923 +9220 244912 +5532 244856 +10077 244828 +9686 244823 +14236 244822 +2676 244770 +12132 244758 +9050 244729 +14226 244674 +13157 244611 +17699 244469 +17038 244387 +6893 244356 +13504 244340 +10047 244337 +22867 244314 +13091 244305 +10435 244293 +33672 244254 +15600 244222 +21796 244154 +14971 244148 +20531 244110 +12170 244096 +24721 244062 +37129 244008 +18183 243979 +17559 243908 +15238 243887 +19436 243839 +811 243838 +15006 243778 +15097 243778 +18329 243649 +20536 243618 +4665 243601 +13969 243595 +11342 243536 +579 243534 +15013 243522 +6700 243506 +13524 243505 +7402 243497 +8280 243462 +10301 243456 +21291 243454 +19167 243434 +13472 243353 +20472 243313 +13011 243279 +16238 243267 +17996 243223 +16534 243174 +11811 243163 +1614 243152 +18704 243103 +9217 243076 +5949 243011 +19090 242932 +9923 242910 +12086 242909 +7461 242849 +12700 242795 +18373 242776 +17371 242768 +5563 242766 +47201 242738 +21070 242727 +17634 242709 +10425 242709 +8434 242578 +5305 242536 +13252 242499 +14348 242487 +9246 242471 +15150 242436 +12500 242426 +17528 242424 +18887 242414 +24116 242389 +23488 242387 +7101 242331 +12973 242273 +2617 242131 +7222 242040 +20330 241995 +15713 241800 +16787 241745 +19307 241718 +18884 241713 +14960 241706 +957 241671 +14748 241658 +7102 241557 +7006 241500 +6340 241483 +8946 241477 +17832 241462 +17905 241448 +26244 241403 +15812 241375 +11770 241368 +9661 241361 +17591 241249 +12729 241190 +26058 241190 +6663 241181 +14923 241131 +20131 241093 +21668 241073 +18470 241045 +4852 241036 +24568 241000 +8835 240978 +31259 240907 +13214 240824 +1746 240812 +4597 240806 +11006 240802 +394 240783 +14713 240716 +13535 240664 +14702 240509 +10323 240409 +19025 240405 +15403 240391 +15470 240322 +10682 240296 +10111 240284 +18518 240277 +7160 240260 +15707 240246 +16452 240209 +14178 240201 +16032 240154 +19116 240139 +8113 240113 +13852 240086 +14022 240037 +12591 239957 +14649 239945 +14339 239916 +21474 239915 +17709 239869 +21952 239817 +8264 239800 +17285 239799 +15737 239731 +17205 239671 +12622 239654 +15444 239648 +16316 239634 +2096 239603 +22948 239581 +9988 239554 +15462 239547 +13167 239539 +3989 239496 +14347 239410 +7911 239385 +18950 239341 +16012 239331 +15779 239300 +13245 239217 +10939 239205 +43208 239145 +8044 239110 +32537 239109 +9109 239055 +16468 239053 +9831 239022 +12289 239015 +12473 239008 +18956 238967 +14034 238965 +32390 238949 +1603 238943 +14759 238899 +19568 238873 +11312 238863 +33448 238862 +12602 238848 +10863 238822 +11508 238722 +1721 238695 +15347 238681 +13473 238680 +14443 238591 +32482 238579 +21305 238555 +2412 238509 +24964 238499 +9433 238479 +12778 238475 +5183 238472 +9749 238441 +6707 238429 +13232 238307 +18410 238305 +9977 238236 +8167 238147 +6089 238087 +2262 238000 +23058 237971 +14009 237931 +14218 237929 +8856 237914 +21938 237904 +9865 237862 +23698 237825 +22261 237799 +12066 237794 +19614 237663 +21550 237644 +6503 237632 +21421 237611 +4407 237500 +18728 237407 +12585 237404 +18190 237370 +13913 237323 +6327 237314 +7091 237275 +7484 237239 +14807 237233 +19919 237104 +2240 237027 +12883 237006 +12672 236984 +17245 236981 +18613 236789 +16135 236789 +44510 236747 +35409 236737 +15031 236551 +12013 236527 +15386 236516 +17176 236497 +10815 236496 +10492 236470 +10911 236469 +13537 236392 +4243 236333 +6400 236316 +29007 236291 +7631 236276 +8314 236264 +20765 236214 +23559 236191 +39859 236188 +14123 236160 +113 236154 +5580 236071 +9257 236046 +10670 236036 +15773 236026 +20395 235988 +18889 235975 +3681 235975 +19136 235965 +14552 235898 +8768 235870 +17075 235844 +7031 235821 +18073 235812 +18311 235801 +4822 235768 +25158 235758 +17906 235743 +17016 235723 +17355 235718 +11170 235613 +7129 235589 +23128 235587 +10746 235529 +3960 235490 +19819 235460 +7177 235452 +8996 235421 +16659 235413 +1070 235407 +4157 235389 +8304 235384 +20604 235267 +7834 235243 +10048 235231 +11270 235182 +13701 235151 +16689 235144 +8718 235065 +39999 235029 +15530 235010 +10598 235006 +19477 234921 +22923 234913 +19834 234897 +19603 234886 +8604 234842 +22230 234835 +10312 234832 +13885 234797 +15280 234727 +13754 234708 +15841 234647 +13622 234596 +12412 234580 +7122 234532 +22874 234512 +11546 234510 +12123 234453 +31995 234438 +18623 234425 +10090 234380 +11506 234346 +16295 234317 +28282 234313 +9907 234307 +17030 234286 +15364 234266 +2781 234214 +15273 234174 +9425 234152 +8599 234104 +10912 234073 +19887 234033 +12278 234000 +8779 233995 +928 233978 +22410 233978 +37186 233977 +10552 233962 +17785 233866 +3137 233815 +2286 233801 +12588 233798 +11446 233668 +20209 233640 +11541 233540 +13118 233501 +20994 233463 +7299 233456 +20495 233430 +14615 233395 +12028 233375 +13771 233371 +19940 233078 +10715 233075 +11940 233049 +15744 233012 +14023 233010 +21298 232991 +10755 232962 +12445 232844 +15849 232811 +12161 232793 +13488 232737 +19627 232725 +14303 232713 +9970 232693 +17155 232615 +16293 232590 +18423 232583 +12917 232573 +13225 232560 +16816 232513 +22137 232462 +2550 232439 +39970 232428 +1749 232423 +26053 232383 +7231 232347 +24540 232344 +12452 232280 +20890 232235 +5304 232190 +10861 232125 +5026 232123 +25763 232111 +15375 232100 +20882 232062 +29224 232056 +17051 232023 +6068 231967 +20377 231908 +15868 231882 +21588 231866 +478 231844 +9004 231835 +10025 231767 +3916 231764 +28058 231697 +6324 231683 +18039 231677 +14891 231669 +9009 231622 +9215 231608 +3515 231529 +12316 231500 +10225 231487 +13895 231467 +14790 231452 +1831 231443 +10537 231322 +7571 231276 +19790 231269 +16115 231247 +12990 231210 +7590 231116 +3699 231084 +3041 231078 +12566 231025 +12191 231001 +13613 230993 +8447 230991 +5770 230906 +45811 230849 +1557 230840 +12059 230775 +5120 230749 +6124 230744 +8595 230721 +9718 230711 +12872 230700 +10066 230682 +26280 230666 +14053 230646 +16126 230641 +20815 230603 +20903 230557 +916 230552 +241 230545 +17654 230421 +20202 230418 +5542 230400 +3109 230337 +17364 230305 +3174 230284 +17797 230253 +31561 230216 +13105 230200 +9656 230199 +16639 230184 +11835 230183 +6139 230117 +9082 230099 +14287 230049 +18280 230024 +25670 229987 +7430 229958 +18481 229950 +22671 229936 +11084 229910 +12174 229895 +11073 229858 +13756 229837 +17600 229795 +11447 229737 +8845 229737 +15534 229685 +21576 229658 +3522 229633 +19405 229593 +14381 229582 +18866 229566 +4033 229525 +12075 229520 +22536 229517 +19947 229507 +16280 229473 +6203 229472 +15491 229435 +8563 229365 +11680 229308 +20799 229304 +13705 229298 +18933 229208 +17266 229208 +14462 229158 +12742 229153 +23669 229130 +20382 229080 +10889 229078 +20565 229060 +12439 229004 +5686 228929 +16467 228914 +17542 228834 +11764 228816 +13693 228765 +10823 228749 +15892 228474 +30452 228448 +10472 228390 +16610 228364 +9236 228343 +24174 228330 +17476 228315 +15800 228213 +15981 228148 +7236 228132 +26199 228108 +2684 228107 +3103 228045 +12133 228039 +9672 228007 +4532 228006 +10900 227989 +8924 227958 +8799 227886 +10224 227883 +28986 227838 +10797 227833 +2782 227758 +33007 227718 +12463 227636 +29367 227597 +7221 227558 +12739 227509 +26597 227410 +12914 227402 +20656 227371 +22372 227348 +12954 227344 +4919 227333 +1996 227306 +14516 227304 +19584 227302 +33467 227285 +15549 227265 +14451 227254 +25176 227163 +8513 227110 +9265 227079 +5825 227077 +16916 227064 +8837 227058 +11304 227013 +16525 226927 +13881 226803 +10817 226749 +21513 226745 +48716 226692 +7218 226646 +5019 226635 +38526 226632 +15436 226625 +10175 226595 +8529 226551 +22167 226539 +1520 226496 +8173 226474 +8959 226444 +29411 226444 +10195 226443 +1465 226389 +1025 226359 +2265 226346 +2243 226311 +10592 226276 +12912 226248 +13819 226212 +10972 226199 +13043 226171 +21088 226141 +11620 226139 +22221 226136 +21233 226136 +2575 226078 +7729 226071 +11314 226060 +14845 225997 +5661 225991 +13273 225979 +2385 225975 +21512 225910 +5937 225907 +16330 225870 +4299 225870 +15662 225857 +22750 225820 +11478 225806 +22925 225769 +24033 225758 +13643 225757 +21097 225757 +1162 225739 +17058 225730 +17487 225677 +15077 225668 +6759 225659 +29293 225603 +17807 225584 +6181 225569 +8893 225560 +23547 225538 +17507 225495 +15928 225483 +5164 225472 +117 225463 +17195 225402 +15852 225393 +14431 225373 +19478 225368 +20695 225363 +15318 225341 +17494 225257 +12062 225251 +22908 225210 +15771 225198 +224 225168 +30958 225155 +12686 225055 +13638 225016 +19388 225000 +14703 224981 +16970 224976 +2454 224974 +17690 224952 +20409 224908 +20111 224898 +19765 224893 +11759 224893 +22081 224874 +9316 224841 +16517 224830 +6306 224819 +1540 224790 +13156 224756 +14212 224670 +14554 224670 +9978 224653 +23889 224643 +9701 224604 +10451 224582 +8534 224555 +14900 224517 +6045 224382 +8476 224300 +17691 224299 +4768 224263 +21995 224248 +8901 224228 +15313 224200 +15567 224200 +26040 224190 +36753 224177 +30289 224128 +13053 224119 +8045 224114 +12581 224098 +10532 224031 +14471 224019 +10422 224014 +14129 223973 +10619 223972 +14643 223911 +17217 223859 +18169 223852 +13355 223803 +6742 223772 +4262 223693 +2809 223686 +14430 223638 +10001 223524 +20516 223522 +12974 223505 +16566 223488 +7989 223485 +11445 223472 +13044 223433 +3856 223428 +10349 223409 +14855 223408 +17113 223375 +13388 223352 +14738 223342 +5126 223298 +6684 223218 +13675 223215 +12092 223199 +11380 223127 +14608 223060 +8011 223015 +13453 223013 +3898 222961 +20492 222958 +12300 222874 +16400 222852 +22281 222839 +12218 222818 +15608 222811 +35253 222785 +5074 222748 +13641 222731 +49280 222731 +20769 222727 +6955 222709 +7481 222695 +19502 222671 +1386 222670 +8695 222613 +9344 222608 +237 222484 +21408 222483 +5972 222470 +7196 222462 +17967 222444 +26702 222349 +20940 222340 +16403 222307 +15425 222295 +14256 222285 +17374 222212 +10580 222212 +21776 222204 +13013 222189 +24573 222178 +14109 222164 +3300 222160 +4017 222053 +12330 222052 +13679 222039 +11384 222000 +13477 221999 +8169 221984 +8068 221968 +23647 221922 +9944 221915 +906 221889 +5934 221882 +26603 221847 +10276 221841 +11382 221793 +12971 221790 +3021 221779 +10945 221752 +9221 221735 +23720 221725 +10770 221698 +14305 221674 +23990 221645 +3225 221614 +13720 221600 +13334 221578 +1151 221567 +16378 221516 +1072 221490 +48160 221489 +31033 221484 +1412 221480 +12560 221416 +13670 221410 +14096 221391 +16002 221375 +16732 221374 +3383 221339 +13514 221325 +3481 221313 +8800 221265 +17668 221262 +10690 221256 +12247 221236 +2135 221229 +17716 221203 +3080 221187 +21205 221165 +15895 221164 +9618 221164 +40196 221132 +20541 221101 +9976 221070 +5796 221049 +11436 221034 +7972 221000 +21679 220976 +5060 220952 +18653 220948 +13823 220936 +10745 220919 +18103 220907 +9521 220896 +11492 220856 +10959 220757 +16825 220751 +7080 220723 +17025 220694 +15519 220606 +11873 220606 +15698 220593 +16117 220569 +18921 220522 +2490 220511 +15449 220499 +32777 220459 +12856 220431 +14042 220401 +2535 220336 +999 220320 +8748 220319 +22966 220306 +16615 220206 +30303 220200 +21724 220193 +15679 220149 +35912 220141 +16568 220080 +13995 220065 +23075 220052 +16413 220030 +13351 220009 +5534 220005 +14044 219957 +18675 219945 +6881 219871 +18751 219864 +123 219758 +21791 219714 +15210 219709 +773 219683 +10547 219682 +5757 219678 +15018 219671 +10524 219668 +10933 219637 +18828 219624 +16013 219618 +13133 219536 +10426 219525 +16757 219524 +3903 219524 +15893 219477 +3488 219405 +13528 219373 +3014 219331 +24287 219327 +16798 219323 +19994 219277 +14958 219254 +6865 219234 +24434 219228 +42623 219200 +5231 219155 +9039 219130 +18727 219118 +16937 219109 +27943 219103 +22724 219087 +11085 219087 +23794 219086 +9607 219056 +19660 219029 +4411 218968 +14090 218928 +17337 218915 +22693 218906 +9190 218872 +12297 218858 +5263 218728 +8655 218726 +49258 218675 +3684 218645 +27620 218613 +24712 218557 +9914 218554 +11110 218532 +20893 218523 +17830 218518 +4389 218498 +23400 218425 +11583 218403 +2768 218401 +24537 218396 +7343 218395 +17370 218395 +21406 218351 +15032 218351 +10181 218300 +8231 218226 +12030 218226 +20097 218153 +13428 218146 +18532 218134 +8823 218128 +12846 218114 +9455 218081 +13209 218080 +16310 218054 +20464 218046 +11843 218044 +15203 218041 +10577 218034 +20955 217996 +8548 217983 +10849 217980 +12333 217976 +8630 217966 +18284 217888 +9740 217809 +1221 217776 +7899 217760 +10011 217740 +7626 217665 +13558 217631 +18914 217578 +15937 217549 +29611 217546 +15894 217527 +9030 217526 +18241 217517 +26315 217499 +11511 217481 +23696 217456 +11391 217437 +11067 217404 +1213 217394 +22861 217392 +12834 217369 +10283 217326 +40912 217323 +12716 217304 +14545 217260 +16381 217225 +15207 217189 +4703 217182 +13261 217111 +16050 217102 +17100 217085 +15171 217048 +12639 217029 +10960 217025 +22957 216931 +12950 216859 +18456 216850 +15757 216841 +12704 216839 +12702 216778 +17945 216772 +14342 216745 +23724 216736 +9028 216726 +18901 216711 +3543 216674 +21376 216672 +15734 216671 +3934 216663 +8722 216500 +27987 216482 +20138 216439 +14037 216414 +14491 216413 +2843 216352 +22421 216341 +13578 216333 +16862 216275 +12554 216271 +28537 216255 +5173 216251 +12549 216248 +21623 216176 +17456 216144 +16484 216142 +17659 216113 +17736 216061 +7079 216055 +21913 215938 +14764 215893 +11443 215865 +11776 215842 +31873 215831 +4738 215782 +18501 215701 +12694 215684 +19625 215672 +19003 215654 +6460 215618 +12127 215615 +16160 215607 +12094 215525 +10249 215518 +15045 215504 +10424 215428 +5605 215404 +12082 215401 +17092 215391 +13054 215323 +14288 215244 +9472 215228 +18711 215207 +16538 215140 +15074 215093 +19975 215087 +11816 215077 +28288 215071 +17884 215042 +3937 215038 +12797 214963 +22600 214939 +23475 214902 +5393 214860 +5979 214807 +10802 214804 +1801 214787 +967 214768 +10488 214663 +14263 214634 +10857 214609 +31157 214596 +8905 214546 +4714 214537 +17029 214532 +23368 214500 +15570 214476 +33496 214457 +17445 214346 +13206 214340 +24416 214331 +2487 214279 +18494 214232 +8577 214231 +22028 214210 +3617 214151 +12372 214126 +41805 214121 +13792 214118 +14869 214110 +12713 214089 +11219 214037 +9527 214012 +688 214000 +8627 213979 +3118 213865 +11022 213825 +18136 213766 +16749 213678 +2207 213676 +5167 213671 +12125 213643 +27227 213623 +6857 213550 +12864 213535 +1308 213529 +17397 213499 +23593 213362 +16035 213342 +11917 213329 +9124 213275 +13774 213224 +14183 213191 +22161 213177 +15427 213148 +15233 213144 +12055 213128 +971 213066 +3723 213027 +10640 213027 +13941 213025 +36128 213014 +11091 213006 +17034 212967 +23166 212933 +9370 212901 +10686 212857 +23751 212844 +31777 212803 +19138 212754 +24579 212728 +8858 212692 +5930 212671 +12476 212652 +6580 212651 +6972 212647 +13669 212629 +6383 212600 +18419 212599 +8284 212585 +23015 212571 +19582 212553 +8070 212508 +8855 212503 +9056 212502 +15727 212495 +5276 212469 +14833 212421 +9535 212418 +18152 212403 +9577 212394 +15240 212379 +15607 212375 +24323 212369 +11444 212362 +14973 212361 +17853 212358 +13937 212356 +17313 212347 +6808 212331 +24663 212323 +19033 212285 +11317 212254 +12296 212141 +20365 212058 +14284 212016 +15808 211962 +15955 211952 +16392 211948 +6297 211881 +1856 211881 +14186 211830 +2288 211802 +5837 211713 +2664 211706 +14604 211705 +14901 211645 +28190 211626 +22101 211577 +18330 211516 +19896 211500 +21523 211488 +11392 211463 +2618 211419 +14450 211415 +11904 211401 +11859 211384 +17444 211351 +9343 211266 +2770 211247 +20243 211246 +14659 211228 +9430 211228 +12077 211197 +14732 211181 +15704 211147 +21946 211139 +12269 211069 +11731 211062 +11672 211055 +17872 210994 +15668 210988 +14548 210950 +12969 210928 +1235 210923 +15215 210892 +10909 210879 +48798 210872 +3428 210853 +1523 210844 +12916 210814 +13027 210809 +11728 210779 +29073 210714 +13888 210709 +16846 210705 +9612 210673 +10162 210657 +16085 210643 +14320 210584 +13619 210579 +23194 210418 +19077 210354 +8159 210295 +20118 210278 +24821 210256 +16651 210150 +22637 210124 +2709 210072 +19163 210068 +15254 210067 +44600 210015 +9419 210007 +13988 209990 +15550 209888 +13204 209858 +22280 209856 +17422 209842 +11697 209839 +12936 209801 +11186 209777 +35621 209746 +10164 209721 +18918 209712 +633 209613 +20776 209595 +13737 209449 +31506 209446 +17424 209442 +8226 209393 +24868 209374 +12624 209351 +12511 209343 +2927 209279 +8080 209270 +14740 209255 +5838 209214 +5959 209211 +22548 209176 +13536 209058 +19174 209012 +24144 209012 +18845 208893 +26402 208846 +1156 208837 +12594 208796 +2533 208779 +14358 208763 +36485 208754 +11888 208739 +7232 208733 +23526 208652 +3610 208564 +20178 208484 +43998 208460 +10926 208352 +3458 208345 +13816 208330 +38742 208301 +14081 208278 +11399 208260 +20085 208192 +13961 208187 +12805 208183 +14614 208174 +3883 208152 +1885 208139 +3008 208126 +23639 208079 +17070 208016 +5283 207990 +9341 207938 +5476 207931 +9684 207911 +19661 207893 +11564 207862 +19105 207862 +22847 207837 +16007 207836 +11524 207827 +15144 207818 +21468 207755 +16914 207689 +18404 207670 +12390 207654 +17840 207497 +16008 207494 +45871 207470 +2586 207461 +14628 207455 +18265 207430 +18614 207428 +7081 207359 +8770 207356 +5297 207336 +18798 207306 +15525 207287 +19808 207251 +14506 207241 +20400 207080 +12788 207060 +14741 207043 +41153 207022 +32922 207010 +8160 206987 +1529 206971 +9379 206951 +13721 206944 +10347 206925 +3247 206897 +13387 206828 +19043 206818 +1815 206782 +23894 206771 +16053 206753 +3852 206737 +18075 206716 +9941 206709 +24576 206705 +23174 206693 +12471 206688 +2153 206680 +3304 206669 +18476 206649 +43318 206638 +14293 206636 +4494 206621 +12319 206566 +29533 206556 +12120 206522 +7931 206512 +19284 206500 +1851 206486 +5594 206411 +4990 206402 +3618 206396 +3201 206381 +14401 206362 +19902 206336 +14321 206300 +17322 206286 +11941 206270 +20110 206225 +18222 206211 +17780 206209 +17536 206137 +1118 206127 +11496 206117 +23430 206105 +10932 206102 +17390 206059 +13646 206043 +6653 206006 +16494 205987 +23358 205935 +10438 205877 +8979 205865 +15281 205825 +6302 205821 +11678 205815 +3353 205776 +20819 205742 +13224 205690 +14499 205675 +23554 205671 +28113 205647 +9495 205646 +5576 205637 +16048 205569 +12459 205499 +33597 205435 +7645 205422 +13790 205322 +15542 205315 +46245 205291 +18895 205274 +12244 205269 +20081 205229 +7284 205225 +25869 205181 +24392 205170 +10565 205124 +13611 205121 +6500 205103 +16261 205096 +14147 205064 +6649 205025 +6819 205001 +17272 204957 +8822 204956 +39726 204889 +4400 204876 +9306 204876 +9229 204814 +6747 204801 +12842 204800 +9704 204776 +35747 204774 +9830 204741 +15500 204721 +9930 204720 +14502 204642 +11315 204596 +19732 204574 +9312 204415 +21146 204408 +14496 204374 +11875 204360 +15859 204334 +30917 204299 +8763 204243 +17023 204218 +1779 204140 +13407 204123 +21462 204106 +14802 204105 +18352 204097 +14556 204094 +11378 204029 +17252 203976 +14660 203975 +30102 203960 +18730 203909 +15107 203853 +6091 203824 +6911 203759 +17432 203643 +14393 203587 +19530 203585 +9578 203567 +25890 203504 +13872 203480 +25415 203476 +20533 203450 +19911 203416 +13879 203386 +12579 203381 +3545 203356 +19210 203348 +20163 203348 +14942 203332 +21371 203316 +20091 203230 +10362 203162 +15516 203101 +24836 203043 +14877 203030 +18515 203027 +12985 203014 +11971 202922 +16490 202920 +14241 202830 +9113 202823 +17931 202801 +10862 202692 +21835 202678 +24175 202629 +8906 202566 +22799 202531 +24389 202517 +10620 202510 +23638 202485 +14946 202399 +29602 202361 +6021 202316 +15736 202310 +16078 202283 +1131 202258 +15869 202240 +13181 202236 +1311 202227 +3082 202222 +10920 202218 +24153 202190 +11173 202148 +21997 202148 +17304 202107 +18210 202094 +13803 202072 +7206 202055 +15953 202037 +13549 201990 +3558 201987 +22984 201969 +20953 201925 +13870 201922 +34630 201878 +11152 201875 +20821 201871 +18087 201855 +11815 201831 +17396 201827 +17112 201822 +26283 201750 +7565 201738 +8128 201738 +40055 201718 +22636 201689 +27883 201686 +28047 201679 +22643 201678 +15805 201639 +22207 201629 +15673 201585 +10717 201583 +16637 201556 +15089 201546 +14768 201433 +19304 201430 +25079 201428 +8940 201385 +15499 201358 +13055 201337 +23434 201293 +15035 201291 +22559 201281 +14232 201276 +5840 201267 +13467 201221 +6201 201203 +18319 201201 +18199 201115 +18078 201095 +36458 201045 +13179 201043 +34969 201035 +12555 201016 +24357 201015 +22489 200993 +19635 200941 +12294 200933 +22276 200926 +13205 200911 +31684 200896 +16055 200851 +10664 200848 +1673 200815 +5098 200813 +18322 200775 +5119 200771 +16636 200749 +29394 200726 +13627 200720 +10239 200697 +10716 200671 +8937 200664 +13215 200643 +26914 200631 +18617 200624 +4685 200606 +12822 200605 +5636 200549 +40319 200524 +16783 200520 +21335 200482 +21022 200464 +11299 200458 +27801 200441 +8254 200419 +7676 200363 +8178 200318 +26015 200274 +223 200265 +24527 200249 +2752 200218 +21819 200209 +7958 200165 +15843 200150 +11105 200111 +23524 200102 +10678 200093 +10866 200087 +21284 200061 +13471 200057 +6055 199960 +16736 199936 +18721 199855 +16518 199836 +11635 199796 +1915 199773 +13650 199720 +3828 199718 +13810 199695 +15193 199677 +9916 199675 +10081 199628 +12526 199625 +12647 199611 +25807 199610 +9664 199591 +15314 199455 +16842 199444 +11175 199437 +15531 199413 +34309 199402 +16508 199393 +11104 199386 +7730 199379 +18641 199320 +4173 199281 +8696 199241 +27928 199240 +6603 199220 +10232 199206 +28057 199161 +22553 199146 +26851 199116 +11424 199111 +18757 199088 +5289 199070 +25254 199069 +25125 199065 +14343 198992 +10365 198991 +17936 198991 +27469 198988 +17366 198972 +15154 198972 +4382 198876 +12198 198871 +16759 198834 +14337 198784 +7109 198771 +19740 198641 +17581 198607 +24379 198563 +3816 198537 +9693 198535 +15241 198532 +23817 198507 +18804 198432 +24255 198411 +6107 198349 +17362 198316 +10809 198310 +21116 198310 +5039 198291 +2308 198291 +26088 198236 +18260 198222 +14458 198213 +25389 198156 +21008 198155 +11470 198084 +21859 198070 +11283 198029 +16230 198019 +47809 197994 +13997 197990 +8267 197972 +27011 197967 +12197 197954 +12362 197946 +24131 197924 +10940 197885 +37628 197882 +8811 197822 +14509 197774 +15373 197762 +15515 197717 +16865 197711 +13815 197675 +7813 197661 +12381 197657 +3401 197647 +12273 197626 +21516 197617 +14384 197598 +103 197579 +19135 197563 +18409 197561 +39760 197544 +11634 197538 +20019 197411 +12887 197393 +5956 197370 +24668 197359 +15251 197327 +9342 197322 +27003 197303 +21050 197272 +25759 197268 +25640 197266 +9510 197262 +18204 197255 +9098 197246 +16868 197227 +21507 197214 +15882 197137 +14324 197103 +23827 197101 +9823 197072 +31340 197067 +7326 197066 +10385 196985 +18739 196964 +809 196944 +22319 196940 +19920 196910 +19688 196860 +10478 196853 +11571 196838 +12214 196815 +34058 196793 +16655 196733 +17296 196724 +14611 196714 +14412 196691 +11667 196663 +19413 196620 +8875 196593 +13165 196540 +16734 196518 +24104 196511 +8364 196502 +2326 196500 +5985 196441 +12706 196438 +8208 196390 +5329 196290 +681 196210 +31389 196208 +17135 196207 +25186 196204 +10622 196197 +19843 196177 +9932 196088 +2875 196086 +3798 196083 +9032 196057 +21782 196019 +16083 196002 +8966 195986 +18134 195959 +740 195949 +16118 195920 +17455 195892 +17958 195875 +10326 195842 +40110 195837 +3123 195811 +13035 195796 +12158 195765 +27526 195763 +16551 195691 +24590 195680 +13238 195546 +9340 195533 +19376 195525 +17270 195514 +9048 195472 +13062 195464 +17098 195441 +4968 195439 +13767 195439 +23307 195310 +15221 195299 +5121 195294 +17496 195292 +9513 195226 +9858 195219 +1742 195203 +13910 195163 +10317 195147 +15441 195142 +27606 195130 +28797 195114 +14075 195031 +15819 194979 +17161 194971 +32190 194967 +48063 194956 +2228 194853 +11148 194826 +20569 194815 +10528 194814 +18544 194813 +16384 194776 +11262 194776 +36553 194753 +12838 194747 +12167 194707 +19089 194655 +9442 194643 +15804 194612 +7408 194589 +9714 194544 +13502 194485 +30094 194423 +20758 194418 +7130 194411 +14803 194409 +30683 194325 +3495 194295 +9723 194235 +10375 194199 +19198 194180 +34016 194170 +42897 194156 +16503 194153 +15135 194135 +19964 194121 +24304 194073 +5573 194073 +14307 194006 +12280 194000 +7742 193988 +7617 193977 +10971 193959 +16546 193948 +17628 193922 +22195 193916 +18062 193910 +22516 193894 +34761 193893 +16556 193874 +13212 193838 +13728 193837 +16054 193822 +20494 193814 +8397 193791 +2 193789 +14246 193784 +15872 193776 +11137 193773 +20117 193750 +9096 193748 +12660 193694 +14441 193657 +11607 193643 +26436 193642 +21750 193638 +12547 193620 +22540 193620 +9445 193578 +23941 193545 +30382 193517 +10949 193503 +19665 193474 +12239 193456 +22687 193447 +13975 193440 +22013 193425 +17812 193409 +19639 193409 +10257 193402 +27083 193366 +4378 193331 +25375 193276 +3702 193272 +25014 193255 +1809 193231 +4864 193160 +6561 193114 +15253 193107 +23888 193103 +12564 193103 +12407 193070 +11111 193052 +12847 193023 +97 193021 +8932 193008 +19223 193008 +13324 193005 +9479 192988 +29536 192979 +33888 192911 +11913 192848 +8450 192848 +18006 192801 +4128 192796 +9404 192771 +23502 192733 +9476 192630 +22233 192603 +19984 192576 +4867 192557 +16033 192554 +19579 192541 +22418 192522 +7679 192514 +14161 192475 +12446 192437 +17652 192394 +15039 192387 +13114 192384 +23995 192370 +15283 192312 +16287 192264 +18201 192232 +22099 192198 +22596 192180 +13427 192167 +21299 192163 +5819 192093 +6058 192084 +23445 192073 +23605 192046 +16180 191892 +25715 191891 +19889 191824 +17492 191797 +1458 191777 +12922 191773 +14693 191767 +17375 191734 +13446 191698 +14801 191695 +11202 191687 +16514 191672 +15487 191656 +13791 191645 +15242 191615 +14245 191581 +3801 191536 +12226 191534 +24238 191508 +20103 191476 +2371 191454 +13151 191441 +22578 191429 +23249 191414 +8965 191396 +17578 191380 +12384 191374 +13591 191368 +13395 191318 +5805 191296 +18010 191271 +23078 191243 +7013 191207 +11923 191207 +12934 191173 +11068 191129 +20282 191124 +27290 191101 +20534 191056 +10112 191047 +18024 191030 +6220 191015 +14422 190963 +20154 190925 +24743 190912 +27905 190886 +18153 190854 +12888 190844 +12653 190841 +14765 190815 +13980 190798 +14004 190783 +18652 190783 +8876 190748 +13666 190685 +14052 190683 +16172 190678 +15699 190672 +7536 190620 +7554 190609 +12755 190599 +15426 190597 +4005 190592 +18976 190577 +13454 190558 +20501 190550 +14890 190507 +26355 190499 +14895 190386 +5012 190382 +10143 190362 +23382 190336 +13135 190315 +14143 190311 +8408 190282 +20200 190266 +15010 190236 +18732 190207 +16847 190178 +16109 190116 +24253 190092 +15264 190053 +17392 190030 +30774 190011 +13565 189985 +16499 189968 +20993 189957 +1604 189929 +9779 189922 +10795 189864 +35942 189857 +9386 189853 +25040 189836 +9174 189786 +19690 189778 +12245 189734 +20071 189723 +19574 189707 +18389 189707 +11779 189627 +26691 189621 +18198 189589 +12666 189586 +12509 189521 +10116 189504 +43967 189435 +11795 189420 +13275 189387 +15986 189374 +7057 189357 +11616 189356 +14528 189342 +14852 189333 +7037 189303 +20792 189288 +6660 189274 +13061 189255 +18786 189228 +23344 189217 +21612 189165 +13952 189163 +18394 189160 +14446 189141 +11254 189124 +10733 189118 +20669 189093 +16493 189039 +1408 189035 +17998 188970 +16861 188962 +10688 188950 +9518 188895 +16826 188888 +23040 188881 +17695 188866 +5189 188861 +7707 188860 +19550 188833 +13931 188821 +16929 188812 +11673 188801 +7788 188737 +16431 188706 +17428 188625 +12680 188619 +2964 188531 +8251 188521 +12649 188513 +19126 188486 +13614 188464 +12498 188434 +240 188366 +23960 188355 +16027 188339 +14736 188328 +8557 188257 +14504 188242 +12317 188216 +18928 188187 +13647 188177 +730 188141 +8499 188040 +25752 188019 +25799 188009 +4475 187984 +19705 187920 +242 187919 +9321 187901 +13971 187893 +17949 187866 +26546 187806 +4337 187792 +1574 187732 +16070 187679 +13314 187655 +16290 187605 +6390 187575 +32991 187551 +11080 187546 +6945 187488 +23139 187471 +23788 187464 +9365 187434 +11040 187433 +22381 187427 +18313 187380 +26085 187341 +14982 187323 +8574 187276 +19958 187261 +11414 187206 +1166 187206 +17537 187191 +15381 187178 +13385 187171 +21736 187169 +6530 187142 +7084 187139 +5528 187138 +26856 187128 +10757 187127 +13401 187114 +9991 187065 +12293 187060 +16477 187032 +4484 186968 +8790 186965 +3083 186903 +18033 186884 +17308 186848 +21609 186846 +4527 186814 +17997 186785 +17243 186783 +15114 186781 +15797 186778 +13927 186730 +10373 186727 +10874 186712 +7805 186676 +12698 186634 +14313 186602 +20320 186582 +19271 186577 +7715 186575 +16090 186565 +6236 186562 +13596 186556 +10119 186540 +27792 186538 +14668 186502 +12264 186476 +27837 186457 +32584 186446 +15897 186417 +2814 186411 +1191 186396 +12472 186389 +9173 186320 +20194 186304 +7749 186228 +7239 186223 +12044 186214 +12298 186188 +30851 186186 +10213 186182 +9427 186147 +31547 186096 +26056 186096 +5154 186083 +6548 186073 +1500 186064 +13153 186036 +21740 186013 +12450 186009 +17535 186007 +13187 185997 +15665 185967 +14772 185946 +22445 185931 +39073 185927 +28247 185912 +10671 185884 +13373 185871 +10335 185854 +9969 185840 +14892 185836 +8576 185825 +5614 185778 +5886 185753 +17332 185753 +10566 185745 +20515 185743 +12393 185692 +16300 185686 +26107 185651 +25872 185641 +7582 185618 +11050 185594 +7085 185589 +13266 185586 +10785 185573 +22931 185543 +13052 185522 +16355 185505 +17321 185499 +18563 185409 +6954 185351 +16532 185333 +9034 185322 +8631 185317 +7857 185290 +3642 185273 +13497 185252 +9353 185245 +38926 185244 +21505 185243 +32903 185225 +7785 185218 +12925 185211 +23489 185206 +121 185160 +17914 185159 +20252 185137 +6756 185118 +14333 185097 +15660 185070 +12227 185042 +8210 185040 +14727 185014 +21012 185010 +16968 185009 +30664 184980 +44144 184965 +23088 184951 +7038 184948 +22770 184841 +15980 184824 +18791 184788 +20829 184782 +46011 184765 +13457 184748 +17264 184706 +8363 184673 +5579 184646 +33864 184641 +12134 184598 +28490 184582 +6090 184556 +18088 184536 +30693 184523 +2475 184522 +17901 184511 +12568 184491 +11012 184482 +18530 184461 +21029 184451 +2673 184416 +15453 184413 +23837 184409 +19121 184397 +14995 184396 +14247 184351 +26391 184333 +6648 184215 +17083 184126 +15398 184122 +24154 184099 +17466 184079 +23566 184073 +17057 184041 +1456 184024 +36107 184020 +11441 183972 +12892 183957 +29424 183938 +17728 183935 +17096 183913 +19358 183880 +19592 183868 +12135 183847 +15617 183769 +7947 183698 +22132 183693 +17921 183642 +14930 183631 +16373 183628 +14139 183577 +15339 183563 +20208 183553 +9434 183516 +17948 183511 +15282 183476 +22558 183422 +16245 183386 +20879 183384 +20760 183322 +15112 183317 +19037 183298 +2790 183225 +9334 183222 +12586 183218 +18655 183207 +19823 183187 +21361 183163 +17471 183140 +16633 183140 +35348 183095 +28422 183085 +15329 183035 +21681 183027 +23393 183018 +14365 183016 +26661 182986 +22103 182984 +17188 182968 +6582 182948 +6247 182823 +11128 182782 +14281 182774 +9683 182761 +16821 182757 +18687 182664 +11739 182625 +31996 182584 +11109 182582 +4605 182563 +9033 182516 +9091 182502 +6148 182479 +12365 182357 +16205 182328 +28102 182310 +10703 182278 +2206 182267 +19431 182240 +13020 182239 +19316 182231 +4933 182220 +16828 182198 +3500 182192 +23288 182185 +10491 182150 +15789 182145 +8505 182107 +4886 182055 +11613 181967 +19651 181962 +7266 181950 +12421 181938 +27463 181932 +5092 181911 +26159 181884 +12238 181842 +20754 181817 +13017 181767 +15321 181759 +31415 181722 +17777 181718 +10590 181707 +3557 181705 +22182 181685 +2283 181681 +26182 181679 +15382 181678 +20915 181642 +10281 181615 +11286 181602 +25450 181598 +13081 181592 +24692 181592 +15335 181586 +10245 181585 +35240 181555 +12419 181544 +7379 181476 +12625 181426 +107 181418 +46568 181398 +29911 181382 +11996 181319 +18842 181296 +13197 181295 +26683 181289 +10983 181248 +20143 181240 +9251 181201 +58 181192 +25013 181189 +24269 181184 +8016 181180 +15138 181178 +5246 181147 +10753 181123 +25279 181114 +49188 181107 +22235 181106 +10416 181102 +15962 181083 +858 181077 +10603 181041 +28345 181027 +17517 181012 +13685 181010 +13796 180966 +5962 180927 +10560 180906 +12720 180895 +23443 180844 +14179 180758 +5936 180697 +14774 180686 +13396 180683 +13963 180683 +11806 180666 +21990 180646 +18195 180618 +9111 180605 +3032 180476 +6931 180457 +19029 180423 +26291 180419 +4674 180417 +30096 180414 +12737 180368 +10437 180314 +22937 180290 +21569 180184 +39173 180170 +6477 180162 +8340 180155 +12186 180123 +19784 180123 +12172 180100 +8067 180079 +27686 180044 +22112 180031 +20351 180027 +12386 180017 +24533 180009 +18983 179914 +26013 179907 +2169 179890 +14505 179883 +12057 179881 +19196 179873 +12703 179836 +15692 179684 +10787 179678 +20788 179665 +22382 179642 +14800 179640 +20079 179626 +25496 179602 +23957 179584 +17262 179561 +13076 179553 +7963 179552 +14792 179535 +10431 179516 +19558 179512 +19624 179503 +14444 179480 +1379 179445 +3313 179441 +9385 179437 +9546 179322 +17869 179308 +7308 179274 +21274 179269 +5775 179268 +17675 179252 +7689 179189 +13343 179180 +24797 179178 +29040 179165 +2734 179158 +15638 179153 +13203 179151 +23943 179129 +13718 179096 +24078 179090 +8692 179077 +12051 179067 +42326 179058 +14157 179052 +20884 179044 +16223 179043 +23976 178992 +12200 178959 +16748 178947 +14083 178946 +19055 178945 +24459 178906 +16988 178836 +16219 178833 +48984 178785 +7500 178772 +14297 178751 +9993 178748 +8643 178746 +3010 178732 +22586 178718 +36427 178703 +20158 178700 +18580 178676 +5574 178583 +16110 178569 +14013 178558 +30994 178554 +12157 178553 +28078 178528 +30739 178473 +17349 178470 +16427 178409 +29928 178401 +16522 178396 +16263 178391 +13734 178348 +15914 178347 +12114 178274 +8913 178254 +17774 178239 +17612 178210 +29560 178195 +14283 178152 +20169 178090 +42357 178076 +43461 178071 +22681 178031 +10398 177993 +24004 177926 +7759 177925 +15232 177918 +7440 177896 +18625 177828 +14201 177827 +12714 177772 +13246 177752 +12501 177747 +26809 177746 +1527 177730 +14720 177703 +15271 177600 +15820 177574 +17291 177547 +953 177535 +7980 177516 +16044 177494 +27639 177455 +18012 177379 +7300 177367 +16389 177354 +5783 177341 +15775 177334 +10442 177311 +21248 177251 +15945 177248 +11400 177211 +22490 177209 +16212 177168 +20432 177124 +16625 177111 +12122 177089 +12948 177075 +20312 177066 +26767 177058 +11762 177052 +6679 177034 +36061 177015 +27941 177013 +10388 177008 +8634 177003 +26685 176994 +2259 176811 +21504 176802 +21941 176770 +19051 176713 +15052 176707 +18365 176701 +22311 176694 +23629 176647 +7386 176630 +14645 176620 +19671 176614 +7442 176566 +908 176554 +963 176538 +6708 176535 +13092 176534 +21041 176525 +16584 176498 +2798 176462 +11578 176420 +11303 176408 +7273 176391 +22893 176380 +4614 176339 +6765 176307 +14590 176286 +16176 176282 +12260 176279 +13568 176271 +17986 176270 +13533 176231 +28040 176226 +3978 176223 +9627 176157 +20210 176137 +18996 176125 +21891 176057 +15678 176047 +12532 175994 +15925 175983 +12449 175949 +19925 175941 +16933 175937 +9786 175935 +16699 175919 +9390 175908 +21809 175905 +37108 175883 +12800 175860 +22962 175858 +14841 175811 +13378 175806 +14594 175747 +20353 175712 +11841 175708 +3196 175676 +8260 175670 +9424 175656 +20685 175654 +2473 175652 +12383 175641 +18141 175636 +12070 175606 +17214 175605 +23668 175599 +12878 175578 +6996 175575 +14434 175564 +5117 175538 +19137 175531 +13836 175530 +27748 175492 +24756 175452 +2876 175441 +12006 175423 +18633 175421 +11625 175394 +28111 175300 +17131 175299 +26101 175296 +4757 175293 +16190 175283 +4660 175281 +24499 175281 +13957 175252 +23592 175226 +14532 175208 +6422 175206 +9795 175192 +6320 175173 +21650 175161 +23676 175135 +7764 175042 +9290 174995 +14112 174966 +10904 174950 +10928 174887 +14735 174859 +5728 174841 +14543 174810 +22175 174807 +14530 174738 +1741 174726 +12113 174647 +27616 174644 +13028 174622 +18054 174599 +638 174583 +13182 174582 +15359 174563 +17431 174553 +43850 174526 +16319 174525 +24252 174506 +21198 174471 +20023 174399 +7063 174367 +19562 174366 +22779 174346 +4307 174344 +1769 174327 +14239 174325 +5375 174305 +24454 174256 +8023 174253 +3541 174227 +9088 174178 +12301 174161 +14655 174111 +12462 174104 +25157 174068 +27546 174066 +25995 174064 +19087 174060 +23955 174041 +22102 174021 +13703 174011 +12175 174004 +21204 173982 +17935 173962 +20412 173922 +12083 173901 +14953 173892 +13439 173877 +30175 173873 +11796 173872 +21884 173811 +27911 173805 +29311 173801 +951 173797 +7923 173778 +18526 173761 +12733 173710 +28891 173706 +13121 173684 +1548 173674 +26293 173655 +18632 173633 +29141 173580 +6385 173558 +3694 173557 +15518 173539 +12213 173531 +25225 173478 +10782 173464 +20550 173462 +7227 173435 +817 173425 +8343 173419 +16730 173409 +25147 173394 +33339 173376 +27541 173306 +26203 173301 +31167 173285 +11901 173265 +11623 173194 +14376 173160 +27042 173159 +16079 173133 +7858 173078 +12952 172985 +17110 172967 +9828 172956 +16611 172909 +14788 172908 +21455 172893 +9666 172882 +10725 172849 +14087 172813 +11061 172779 +13139 172764 +12701 172753 +13400 172750 +10645 172749 +26456 172713 +22653 172706 +12824 172703 +7883 172684 +15637 172673 +8508 172668 +3671 172656 +16197 172651 +6359 172639 +25434 172563 +5191 172562 +9915 172552 +1761 172546 +20990 172531 +18520 172530 +7305 172488 +8558 172486 +15345 172477 +19378 172448 +23681 172412 +23936 172403 +14026 172399 +26373 172356 +10194 172322 +11459 172305 +12085 172285 +16336 172271 +13405 172236 +12361 172235 +6061 172209 +3757 172202 +8550 172200 +17222 172176 +6735 172162 +7235 172062 +23975 172027 +16473 172025 +11987 172020 +22098 171992 +17461 171991 +12656 171991 +6763 171989 +46179 171979 +12347 171925 +12470 171923 +8891 171915 +21071 171901 +18870 171890 +1655 171870 +21166 171857 +6444 171853 +1840 171774 +8869 171773 +12784 171753 +28472 171729 +21571 171719 +24763 171628 +12320 171619 +21840 171608 +1927 171529 +18382 171521 +5753 171500 +17290 171498 +44892 171491 +9801 171488 +12584 171458 +101 171438 +7514 171434 +46990 171428 +29913 171428 +20771 171416 +2601 171410 +17460 171395 +21220 171384 +16895 171380 +9127 171376 +15619 171357 +19094 171350 +20455 171331 +33323 171302 +28975 171299 +4774 171292 +20319 171273 +15844 171257 +16686 171115 +27025 171095 +30646 171073 +27607 171054 +9335 171010 +18367 171000 +22569 170994 +33564 170982 +22634 170958 +22300 170944 +18729 170910 +33653 170901 +16657 170899 +7670 170892 +14480 170888 +21862 170874 +17095 170838 +22383 170834 +8991 170821 +22394 170769 +13676 170754 +35659 170660 +38858 170592 +11638 170560 +1240 170543 +26353 170531 +11609 170454 +14690 170433 +12004 170432 +9269 170423 +16146 170371 +27599 170360 +14928 170326 +1930 170295 +7454 170289 +23328 170274 +31546 170245 +5247 170202 +17803 170200 +19258 170198 +5980 170197 +24980 170151 +742 170143 +19057 170127 +4910 170077 +22953 170045 +25585 170029 +21927 170019 +29090 170017 +31159 169975 +7128 169965 +8065 169957 +8833 169947 +27582 169932 +13386 169901 +31346 169899 +11766 169880 +16047 169880 +5978 169862 +16608 169852 +10031 169833 +31078 169816 +32636 169802 +10370 169746 +10675 169728 +47979 169720 +33231 169712 +24684 169670 +16621 169667 +19764 169656 +19988 169648 +29747 169644 +44009 169639 +7074 169638 +9800 169628 +1585 169536 +24073 169523 +21087 169487 +22720 169463 +22434 169431 +25074 169400 +12659 169366 +11733 169362 +35093 169320 +15873 169309 +16602 169292 +14485 169259 +17449 169228 +14234 169204 +30245 169196 +18064 169190 +18735 169178 +12569 169172 +12493 169135 +14648 169118 +25935 169114 +26116 169108 +675 169099 +8306 169097 +2956 169065 +12229 169035 +23076 169019 +11387 168999 +18130 168991 +16930 168979 +12790 168977 +15767 168976 +21110 168952 +31703 168924 +12779 168902 +24485 168902 +29674 168898 +17120 168891 +16359 168887 +19169 168876 +1076 168872 +16894 168838 +10322 168830 +20552 168816 +16324 168804 +26078 168785 +34786 168725 +2006 168691 +11542 168666 +16932 168631 +11968 168561 +18647 168543 +11556 168522 +30384 168493 +899 168471 +13374 168460 +22225 168459 +15881 168449 +11341 168440 +6644 168425 +10712 168414 +21266 168360 +17876 168314 +8610 168313 +22541 168302 +12121 168276 +105 168274 +18053 168229 +11031 168215 +17441 168157 +11640 168079 +1788 168065 +12697 168063 +18079 168041 +14789 168034 +23577 168015 +25686 168011 +14325 168010 +14826 167973 +8808 167968 +7385 167959 +15656 167943 +26406 167933 +3477 167926 +25167 167922 +20441 167896 +9254 167895 +14102 167884 +13580 167830 +22041 167798 +27611 167791 +19711 167781 +29890 167771 +34943 167757 +32149 167722 +24207 167666 +14551 167664 +1929 167661 +5851 167643 +9792 167642 +13207 167637 +17711 167633 +23438 167558 +4095 167545 +18656 167543 +10761 167496 +9310 167481 +31509 167455 +13084 167453 +12823 167449 +12484 167418 +29272 167411 +23780 167399 +19725 167368 +20271 167359 +20386 167350 +8071 167327 +23982 167266 +26451 167221 +16675 167217 +23415 167158 +2508 167108 +17938 167087 +18519 167077 +13717 167065 +9053 167045 +11575 167045 +6723 167044 +5830 167038 +11451 167009 +20237 166982 +13656 166930 +33118 166913 +2929 166913 +34589 166887 +12886 166877 +21011 166858 +20886 166856 +11053 166854 +5225 166852 +13747 166832 +9351 166806 +14676 166779 +16171 166777 +31526 166766 +24031 166749 +26606 166743 +13889 166741 +18679 166738 +11757 166711 +30035 166709 +2114 166669 +22695 166667 +11054 166649 +11442 166649 +15413 166635 +9579 166605 +13239 166595 +13752 166594 +1567 166549 +13699 166525 +17962 166524 +13510 166490 +18387 166475 +1934 166420 +26400 166403 +36759 166372 +15172 166364 +21666 166333 +25733 166308 +24799 166303 +14386 166271 +11463 166237 +8290 166233 +25914 166221 +4207 166202 +11562 166199 +7310 166186 +12576 166158 +12236 166143 +17372 166140 +23270 166122 +10605 166100 +11349 166100 +9726 166095 +24164 166090 +6950 166083 +13267 166071 +19636 166055 +12422 166053 +10217 166036 +14340 166030 +15163 166028 +14048 166027 +14154 165992 +22309 165928 +14266 165928 +34217 165921 +11052 165919 +33399 165910 +3832 165889 +16237 165889 +7287 165873 +22351 165865 +15291 165860 +1928 165844 +17673 165838 +6615 165779 +12844 165748 +47875 165744 +713 165695 +28392 165691 +14678 165625 +16545 165608 +14492 165582 +21402 165530 +22361 165529 +19219 165467 +4596 165457 +12923 165451 +5702 165438 +11820 165426 +1089 165423 +13516 165419 +26190 165371 +25701 165371 +13827 165370 +4328 165308 +34366 165287 +19140 165262 +10320 165222 +11863 165214 +12033 165196 +28523 165172 +31791 165153 +15489 165126 +17274 165125 +15477 165115 +23829 165086 +19290 165064 +21531 165051 +13070 165041 +7710 165021 +11516 165009 +21006 165000 +23832 164975 +9948 164892 +24850 164885 +11101 164882 +27117 164844 +32564 164833 +22771 164818 +8142 164818 +34506 164763 +16192 164663 +13452 164658 +10368 164642 +4342 164624 +23325 164571 +14153 164564 +32136 164562 +19500 164555 +1777 164538 +26216 164529 +21518 164502 +17863 164502 +11662 164492 +14851 164477 +28059 164405 +27035 164382 +12144 164302 +12903 164283 +17564 164257 +13006 164252 +13753 164245 +1768 164242 +9848 164233 +23247 164218 +16788 164197 +12979 164190 +41179 164177 +2217 164136 +9255 164136 +41588 164100 +23413 164093 +14955 164090 +11664 164075 +23429 164074 +2889 164070 +20002 164057 +4630 164034 +13318 164034 +9058 164026 +11543 163998 +13608 163983 +23643 163952 +18067 163942 +4501 163926 +22158 163921 +20404 163910 +21418 163874 +17344 163864 +12444 163801 +901 163789 +29037 163767 +16500 163753 +14093 163735 +29306 163647 +7837 163646 +22223 163631 +13122 163620 +14776 163601 +5589 163591 +9989 163571 +23624 163502 +8274 163471 +16325 163463 +30031 163460 +9330 163450 +839 163430 +24397 163382 +8605 163360 +5988 163329 +11860 163296 +16765 163287 +20527 163284 +3316 163277 +915 163267 +18434 163258 +10165 163223 +14185 163180 +15464 163159 +27274 163156 +20165 163120 +11011 163101 +20390 163090 +15276 163078 +21133 163077 +32694 163019 +19745 163009 +18246 163004 +2181 162993 +8074 162946 +20304 162946 +42057 162941 +11089 162921 +17109 162920 +13799 162841 +18021 162828 +21817 162821 +21270 162821 +11870 162809 +7056 162800 +21167 162709 +19657 162699 +17887 162692 +21023 162683 +12793 162679 +22958 162650 +18862 162610 +18753 162565 +16585 162550 +30843 162549 +19432 162535 +22680 162525 +6895 162511 +18488 162509 +8435 162506 +26301 162478 +46129 162457 +17346 162449 +11094 162410 +23068 162393 +23617 162378 +24232 162378 +17742 162375 +20291 162364 +22260 162360 +22639 162344 +30872 162339 +8431 162339 +15371 162317 +15305 162307 +21896 162293 +26615 162268 +665 162268 +20714 162265 +17368 162256 +20093 162231 +21183 162191 +23200 162167 +19855 162160 +33922 162151 +1615 162141 +33724 162135 +4062 162117 +18570 162085 +7149 162079 +16853 162064 +20742 162037 +25050 162013 +7591 161991 +8369 161974 +28643 161972 +17454 161968 +15472 161941 +31005 161940 +18416 161919 +33911 161884 +10220 161861 +9942 161846 +14089 161843 +12524 161833 +18852 161808 +2218 161795 +21119 161783 +10351 161736 +27847 161736 +11624 161735 +14639 161728 +226 161620 +11736 161616 +20739 161604 +11329 161601 +20352 161563 +17054 161549 +4180 161537 +33385 161528 +25889 161523 +16320 161497 +33193 161491 +31767 161467 +11247 161429 +28759 161424 +14378 161422 +23590 161420 +20393 161410 +15654 161390 +12098 161372 +27986 161357 +18397 161348 +11649 161326 +23070 161293 +25634 161289 +19475 161286 +15916 161274 +13779 161164 +16299 161163 +3984 161154 +19517 161142 +16860 161140 +12546 161125 +8229 161124 +25793 161099 +15332 161092 +14104 161088 +27357 161081 +15099 161076 +16034 161064 +15722 161054 +18618 161004 +21158 160987 +9491 160981 +12096 160973 +18362 160959 +31230 160951 +22171 160939 +15749 160938 +23280 160928 +27180 160917 +19350 160903 +13917 160863 +28686 160840 +22387 160829 +14150 160787 +8950 160765 +7711 160757 +11433 160753 +20447 160753 +21743 160744 +17589 160726 +22830 160718 +15564 160701 +15935 160665 +17086 160655 +11025 160649 +1020 160616 +3915 160590 +21445 160576 +12860 160543 +20653 160539 +22627 160534 +28355 160530 +13727 160517 +13331 160513 +14657 160501 +15293 160493 +12345 160491 +25469 160481 +8859 160468 +21249 160462 +26538 160449 +25684 160447 +5691 160438 +14846 160434 +30427 160423 +5844 160391 +18323 160386 +14318 160373 +25535 160370 +20840 160352 +10526 160335 +16247 160329 +18666 160320 +15850 160286 +15326 160278 +23259 160245 +17260 160245 +24091 160196 +13769 160191 +27540 160149 +12898 160107 +12661 160094 +11657 160079 +5912 160074 +11738 160067 +2913 159992 +8204 159989 +26804 159960 +14595 159957 +14730 159947 +32760 159941 +32078 159907 +13460 159893 +21425 159882 +26618 159858 +32879 159838 +12592 159836 +24832 159811 +15536 159796 +16537 159787 +27828 159774 +18857 159772 +14483 159740 +25054 159712 +9586 159707 +20096 159707 +17079 159702 +14370 159698 +19207 159671 +26824 159630 +33139 159614 +25172 159602 +13686 159592 +40289 159583 +21838 159580 +18128 159571 +14733 159563 +31292 159553 +18139 159513 +44066 159510 +25165 159492 +14371 159474 +17255 159432 +5379 159426 +4119 159418 +15250 159397 +12041 159368 +39463 159338 +14862 159315 +22140 159290 +7769 159278 +8629 159228 +29714 159198 +28855 159198 +14484 159181 +16255 159158 +22945 159156 +9875 159043 +12188 159039 +12877 158992 +33605 158976 +41449 158974 +11212 158974 +32688 158957 +13455 158950 +17638 158947 +8531 158908 +21193 158855 +7220 158842 +3875 158831 +6017 158800 +9724 158796 +13808 158790 +21798 158768 +16569 158740 +8487 158725 +12891 158721 +16169 158668 +50079 158664 +14662 158663 +9565 158660 +26070 158632 +16715 158620 +25982 158612 +14921 158609 +24190 158591 +16304 158587 +16561 158553 +17052 158522 +17590 158510 +2012 158490 +40635 158487 +14193 158454 +16597 158446 +29200 158411 +25768 158388 +16599 158346 +22283 158342 +14190 158305 +17984 158291 +11368 158289 +894 158285 +8726 158253 +13271 158252 +38818 158242 +15909 158213 +15279 158211 +19987 158186 +14029 158156 +2967 158148 +7597 158145 +11282 158119 +30334 158112 +1819 158040 +7211 158026 +8804 158002 +24152 157998 +9863 157974 +15124 157954 +12189 157895 +31507 157885 +10381 157862 +20551 157853 +35293 157847 +8664 157839 +15828 157836 +17114 157813 +15585 157797 +32317 157785 +21141 157733 +29444 157718 +13570 157718 +11352 157674 +933 157613 +13222 157602 +14310 157588 +12929 157586 +18017 157577 +15647 157572 +28278 157566 +14540 157562 +9576 157531 +10599 157529 +11140 157502 +25936 157484 +39637 157468 +5684 157467 +20997 157452 +23115 157437 +18108 157436 +20705 157426 +21457 157387 +16511 157373 +28564 157373 +23420 157353 +13786 157351 +10284 157333 +11723 157333 +24005 157315 +10919 157293 +27874 157280 +16548 157274 +9995 157264 +13141 157253 +15907 157235 +12571 157218 +2247 157218 +28182 157209 +14121 157200 +1552 157195 +7400 157179 +15747 157150 +10188 157144 +13079 157126 +29443 157105 +22605 157085 +14638 157043 +22722 157041 +38061 157031 +19702 156983 +24713 156960 +29600 156910 +27622 156908 +23811 156882 +18612 156867 +29330 156804 +31071 156786 +18340 156752 +26962 156752 +16460 156724 +20126 156706 +5275 156655 +20342 156637 +18259 156614 +888 156611 +15816 156610 +30405 156596 +10742 156594 +26068 156592 +12572 156545 +23980 156530 +18468 156522 +6256 156514 +18504 156499 +4020 156492 +22323 156483 +10685 156482 +18752 156458 +17268 156414 +13788 156414 +5083 156407 +31472 156400 +2908 156375 +26807 156369 +24142 156367 +22571 156353 +11060 156348 +17093 156298 +20642 156289 +19551 156285 +3780 156274 +19495 156268 +800 156267 +20603 156263 +18147 156244 +35151 156227 +18385 156178 +18684 156168 +20601 156162 +8553 156147 +14028 156145 +46279 156133 +16456 156127 +7913 156073 +17066 156071 +12692 156046 +25978 156013 +14173 156010 +25299 156002 +23519 155994 +28335 155959 +20157 155953 +5677 155929 +3971 155902 +12267 155878 +3309 155863 +20933 155816 +24296 155768 +5821 155741 +18236 155729 +17893 155698 +17012 155685 +15902 155658 +20051 155639 +16395 155631 +25688 155605 +11711 155597 +22455 155571 +22095 155561 +18590 155536 +28052 155527 +43856 155456 +25109 155429 +19414 155419 +23546 155417 +25296 155396 +15021 155380 +8624 155360 +34338 155326 +48543 155324 +18264 155290 +2285 155264 +34139 155261 +17920 155238 +25201 155154 +12279 155145 +16075 155111 +14818 155107 +16318 155106 +16397 155078 +37244 155044 +23472 155043 +29220 155029 +11501 155027 +18446 154973 +3319 154963 +18180 154948 +15101 154935 +18105 154933 +16740 154932 +8960 154919 +17616 154899 +28280 154889 +13521 154875 +10583 154855 +7071 154845 +12326 154824 +16024 154804 +21292 154782 +1047 154750 +11188 154727 +18223 154723 +12768 154698 +13391 154683 +11948 154679 +28354 154643 +12105 154624 +24833 154599 +12179 154573 +15399 154535 +17089 154509 +14687 154465 +2374 154450 +10248 154438 +15723 154435 +15621 154410 +630 154398 +14809 154370 +17722 154361 +23424 154352 +14088 154336 +2429 154321 +14275 154316 +19026 154282 +7493 154278 +32948 154265 +3733 154263 +20270 154229 +15830 154223 +19602 154195 +6517 154179 +17989 154171 +46668 154162 +19068 154157 +10285 154148 +30223 154138 +17981 154123 +24545 154116 +8241 154084 +6404 154027 +1185 154019 +4227 154018 +14208 153999 +16811 153998 +18867 153997 +16755 153996 +36431 153984 +25613 153954 +19481 153934 +13435 153913 +34112 153902 +22676 153889 +11096 153830 +18920 153800 +40113 153775 +19447 153772 +32440 153762 +23632 153760 +27052 153749 +11945 153742 +26091 153725 +15718 153715 +31736 153713 +21821 153700 +20153 153693 +1553 153661 +24300 153645 +12395 153620 +11800 153610 +24859 153584 +31318 153521 +17254 153482 +7573 153476 +255 153473 +25230 153472 +24625 153458 +9878 153450 +10114 153435 +21400 153433 +14884 153380 +26918 153347 +12370 153341 +18432 153335 +16459 153330 +26023 153308 +8118 153298 +49355 153294 +25846 153268 +6005 153259 +34910 153259 +6392 153251 +44828 153232 +17530 153226 +20462 153208 +15867 153204 +10548 153174 +18342 153154 +24135 153137 +11335 153129 +15571 153117 +22970 153096 +16507 153049 +14440 153036 +2764 153035 +23533 153022 +8681 153019 +16195 152986 +4507 152962 +8613 152961 +14045 152955 +26326 152952 +4250 152929 +12958 152900 +29825 152864 +19830 152848 +23265 152843 +36863 152777 +31110 152776 +20670 152773 +14580 152733 +13707 152713 +28397 152701 +21820 152700 +14640 152687 +9380 152661 +10635 152624 +14814 152596 +16356 152522 +11246 152502 +40001 152483 +23013 152432 +25316 152418 +27724 152407 +25018 152406 +15628 152383 +35637 152371 +20424 152336 +18269 152305 +13172 152300 +2143 152274 +47633 152263 +3411 152256 +1190 152228 +21827 152218 +16642 152203 +22205 152196 +13148 152180 +4230 152159 +15551 152108 +8002 152108 +15226 152070 +11261 152055 +11705 152032 +20240 152015 +23918 151954 +31625 151951 +21028 151947 +43566 151927 +2341 151908 +19935 151900 +12863 151899 +32958 151881 +15691 151875 +16887 151864 +16963 151863 +14338 151859 +27194 151857 +26909 151855 +19999 151840 +6871 151824 +14593 151815 +26716 151771 +13899 151760 +20753 151700 +36095 151697 +15298 151683 +32496 151606 +2865 151590 +17809 151567 +29797 151566 +36744 151544 +22368 151535 +12271 151498 +19632 151465 +9356 151455 +20583 151415 +17061 151403 +17781 151390 +24968 151389 +41461 151377 +22877 151362 +4906 151313 +13667 151309 +27283 151288 +31764 151280 +18648 151258 +47286 151250 +31918 151179 +16277 151130 +19519 151110 +19931 151099 +21434 151094 +35385 151073 +21808 151068 +4127 151053 +30112 151050 +29402 151046 +17383 151039 +3279 151028 +41727 151001 +34549 150992 +7141 150988 +16001 150978 +12740 150977 +19013 150919 +2317 150903 +13616 150870 +19829 150866 +26544 150859 +23494 150829 +11793 150822 +5678 150789 +21346 150787 +6839 150781 +16612 150779 +2859 150761 +16133 150748 +25462 150743 +25175 150735 +29222 150706 +18266 150702 +15557 150701 +24413 150649 +17378 150591 +14714 150581 +1868 150571 +4189 150541 +2747 150526 +14470 150516 +26960 150510 +12209 150502 +1685 150501 +6880 150485 +18165 150479 +14383 150433 +4561 150430 +12804 150411 +48298 150408 +24766 150396 +22964 150364 +17837 150349 +28792 150347 +4160 150339 +3874 150327 +13113 150326 +9654 150323 +13858 150312 +13476 150291 +13000 150289 +19443 150285 +7856 150248 +20631 150223 +23663 150219 +36878 150211 +4356 150206 +12005 150181 +5061 150174 +17292 150167 +37929 150165 +1174 150149 +7100 150145 +12119 150121 +19934 150114 +25911 150114 +26985 150114 +18551 150112 +10979 150111 +6475 150106 +31063 150073 +23008 150064 +11198 150059 +19131 150055 +20156 150033 +14073 150025 +18140 150022 +14394 150016 +3438 150003 +12027 149972 +20774 149888 +21126 149879 +19097 149868 +10345 149816 +41012 149812 +14438 149801 +18553 149773 +16954 149767 +21423 149719 +32519 149709 +29074 149707 +8784 149698 +2963 149683 +13026 149678 +23052 149660 +28242 149649 +16677 149632 +13599 149629 +36692 149596 +18550 149593 +3351 149580 +3711 149539 +26454 149533 +18916 149510 +23531 149481 +17467 149475 +11207 149458 +11710 149431 +10720 149413 +3629 149411 +18315 149407 +28649 149298 +7490 149297 +43484 149267 +19315 149251 +20561 149234 +12681 149204 +31460 149200 +11707 149196 +19278 149166 +13951 149157 +25452 149155 +2898 149145 +16472 149134 +20276 149129 +16889 149126 +23786 149104 +14799 149093 +13326 149084 +37361 149083 +18499 149066 +29418 149057 +2668 149055 +23902 149051 +12839 149015 +29806 148976 +24380 148923 +28108 148871 +32939 148870 +14497 148862 +5623 148850 +17730 148845 +15635 148837 +44508 148835 +20077 148829 +11499 148813 +14271 148813 +22191 148798 +9737 148798 +16960 148777 +32682 148773 +31630 148764 +35231 148749 +21039 148721 +43841 148710 +31248 148693 +25313 148685 +25043 148683 +21112 148627 +20289 148622 +18811 148620 +16265 148608 +7655 148578 +9045 148574 +15712 148554 +26039 148540 +22264 148525 +13869 148513 +14031 148505 +8392 148502 +34783 148499 +24536 148490 +33110 148480 +21380 148469 +26686 148438 +9695 148399 +15765 148398 +10586 148396 +9834 148374 +14363 148368 +42189 148367 +8297 148330 +42235 148326 +24163 148323 +21511 148301 +13539 148287 +16259 148284 +22841 148280 +34655 148279 +13639 148270 +1873 148220 +236 148220 +28791 148173 +16981 148158 +31002 148157 +19666 148135 +17609 148080 +11527 148078 +16938 148071 +19649 148013 +961 147986 +7673 147977 +15143 147964 +14583 147917 +13645 147910 +12590 147909 +18724 147905 +27502 147901 +23942 147874 +11428 147867 +4740 147865 +27609 147860 +16425 147859 +20731 147848 +29640 147844 +3143 147820 +23852 147767 +10348 147753 +4777 147752 +11930 147739 +16904 147704 +2307 147703 +17498 147652 +12791 147635 +10107 147624 +29751 147606 +16406 147605 +12017 147592 +17625 147562 +9382 147562 +14351 147554 +22244 147533 +5725 147520 +2655 147490 +23655 147473 +21530 147408 +12657 147405 +4879 147395 +22075 147383 +21685 147355 +22786 147242 +21420 147225 +15794 147218 +3577 147175 +9536 147160 +35193 147093 +27348 147076 +17198 147049 +8615 147031 +24037 147017 +6532 147016 +25289 146998 +10897 146965 +156 146955 +39920 146925 +22647 146921 +10148 146909 +10891 146887 +14041 146872 +18143 146867 +15460 146865 +5518 146864 +2917 146864 +17123 146801 +23600 146765 +10582 146703 +23240 146699 +17502 146680 +26943 146669 +7488 146654 +14282 146636 +21161 146625 +22879 146622 +14176 146611 +28250 146596 +18654 146583 +20715 146561 +11056 146533 +16645 146525 +40160 146513 +608 146508 +13770 146488 +16830 146483 +21987 146426 +34162 146377 +14136 146323 +20505 146305 +12741 146284 +13530 146275 +27250 146260 +32489 146242 +9929 146232 +34183 146216 +27775 146212 +21644 146198 +11530 146177 +6309 146169 +19496 146142 +20047 146103 +16521 146087 +22926 146036 +3228 146019 +15108 145976 +27879 145951 +15921 145950 +18818 145948 +11379 145936 +28259 145928 +6051 145927 +27886 145925 +21919 145900 +27771 145884 +19002 145869 +28565 145863 +10229 145861 +19480 145857 +7120 145846 +1901 145839 +16470 145810 +10456 145793 +25464 145789 +11895 145789 +19597 145785 +8769 145767 +24787 145747 +35764 145738 +34116 145734 +36040 145708 +13884 145698 +29285 145693 +29484 145690 +11916 145681 +20244 145661 +26899 145652 +8739 145638 +16231 145626 +11767 145621 +27334 145619 +39823 145614 +24193 145602 +26249 145596 +31774 145592 +12889 145583 +15933 145566 +26739 145565 +48974 145558 +5146 145504 +19383 145482 +22772 145462 +18379 145458 +18695 145447 +20874 145411 +10593 145401 +37720 145373 +19783 145319 +15085 145314 +9297 145310 +17316 145293 +18522 145290 +18785 145276 +63 145276 +6229 145255 +21733 145255 +12479 145254 +23186 145227 +14489 145197 +10752 145179 +39375 145173 +31636 145170 +35597 145170 +39632 145167 +18295 145138 +41924 145119 +14448 145118 +28458 145088 +20671 145086 +10602 145079 +18595 145063 +15190 145062 +19346 145051 +11179 145035 +16216 145026 +15941 144995 +11741 144989 +9261 144984 +18094 144969 +25479 144862 +26136 144855 +27229 144848 +7761 144848 +2973 144807 +17160 144798 +24248 144764 +7200 144739 +22375 144732 +23356 144732 +14361 144729 +21057 144721 +18508 144691 +4817 144670 +9598 144624 +12709 144620 +11108 144598 +6423 144592 +14097 144587 +15822 144585 +14653 144550 +20813 144539 +21541 144521 +26242 144514 +37440 144509 +4114 144509 +13375 144474 +18575 144446 +3222 144446 +10783 144443 +24105 144434 +18283 144432 +10197 144427 +14349 144424 +38813 144413 +39249 144396 +6724 144389 +490 144388 +32452 144361 +7068 144353 +19171 144331 +26154 144327 +22336 144323 +9866 144305 +22108 144292 +6468 144287 +15825 144282 +23702 144255 +6988 144243 +10730 144241 +28414 144241 +17276 144230 +14983 144229 +14864 144214 +26348 144200 +1843 144196 +25357 144180 +14180 144170 +18662 144158 +24067 144156 +14699 144154 +10924 144126 +20162 144107 +22038 144094 +14724 144049 +27633 144028 +8310 143966 +19050 143962 +17411 143945 +2299 143915 +1617 143906 +26372 143898 +17416 143870 +37072 143867 +879 143850 +21624 143840 +6790 143798 +18750 143787 +20672 143775 +3157 143711 +2386 143672 +13143 143671 +21641 143653 +4559 143641 +26134 143618 +15807 143611 +40990 143519 +17206 143512 +12481 143506 +8510 143503 +12495 143490 +10227 143482 +20576 143478 +28379 143414 +15904 143411 +14871 143409 +12153 143373 +24194 143336 +21272 143326 +11700 143323 +19000 143288 +17004 143282 +12683 143277 +6499 143264 +20250 143258 +22543 143237 +17835 143236 +21337 143227 +36888 143176 +8524 143165 +20600 143162 +13186 143159 +5420 143155 +38488 143155 +35295 143153 +26935 143119 +29737 143113 +12290 143106 +6072 143102 +35399 143098 +15050 143097 +19544 143080 +21813 143026 +22371 143025 +13024 143020 +13529 143006 +12719 143001 +8756 142996 +26390 142984 +39099 142967 +13063 142945 +21024 142943 +15455 142914 +6574 142910 +8299 142910 +35136 142909 +22736 142908 +25647 142902 +16352 142896 +18573 142873 +2593 142862 +23312 142838 +36861 142830 +14169 142814 +7409 142806 +9756 142803 +28238 142793 +15949 142780 +24779 142717 +11894 142698 +37911 142687 +5450 142670 +8322 142664 +15133 142663 +22004 142624 +14252 142623 +16291 142622 +19996 142485 +21542 142483 +12237 142454 +22814 142400 +14323 142388 +6048 142372 +24946 142362 +16567 142345 +26046 142335 +12817 142314 +20805 142310 +7251 142296 +33338 142294 +24086 142293 +20461 142280 +2322 142269 +43812 142263 +12312 142254 +9582 142243 +38134 142234 +26907 142226 +16218 142210 +38671 142196 +13002 142196 +4826 142194 +14947 142188 +30836 142187 +10288 142181 +15175 142177 +15743 142174 +8228 142163 +17162 142134 +13005 142118 +31721 142116 +36139 142113 +10128 142107 +9688 142074 +43941 142063 +23773 142057 +14498 142033 +24740 141961 +15644 141960 +30633 141944 +15875 141934 +21114 141929 +13127 141927 +15504 141914 +8373 141897 +18969 141895 +20345 141876 +15234 141835 +8344 141831 +15043 141803 +19877 141769 +13690 141755 +17648 141754 +9285 141750 +12777 141739 +20493 141725 +33876 141723 +20175 141717 +22151 141698 +48828 141689 +25665 141686 +1259 141670 +20698 141670 +14010 141664 +28330 141659 +15349 141650 +5945 141623 +30801 141606 +26414 141595 +17800 141591 +998 141574 +16832 141552 +16492 141540 +27153 141534 +16399 141504 +13444 141490 +19848 141477 +28291 141476 +40253 141467 +21735 141446 +11685 141439 +39304 141433 +17687 141365 +11503 141351 +19534 141326 +18244 141322 +19487 141311 +15985 141281 +31612 141266 +13368 141262 +11290 141261 +10230 141260 +7657 141247 +18213 141238 +14996 141232 +23627 141202 +20075 141173 +5843 141143 +25333 141137 +22862 141121 +10477 141085 +22432 141070 +19425 141064 +20392 141053 +16361 141043 +23407 141014 +40624 141014 +27267 140988 +12258 140965 +18605 140949 +33020 140940 +28553 140933 +20974 140924 +14296 140899 +46935 140878 +2511 140811 +19437 140786 +21107 140783 +18591 140782 +6811 140766 +12243 140763 +1349 140737 +10974 140709 +5151 140698 +12748 140677 +25621 140661 +19263 140656 +16061 140651 +4432 140640 +31658 140637 +40060 140636 +19384 140632 +20814 140631 +25142 140619 +19301 140587 +23198 140585 +14131 140572 +7435 140568 +27515 140552 +15248 140540 +24581 140504 +25657 140498 +8402 140492 +15421 140465 +5720 140454 +17977 140454 +11471 140450 +24851 140420 +25308 140397 +13370 140392 +19479 140354 +22462 140350 +46000 140325 +29759 140315 +12964 140307 +11211 140304 +1197 140274 +20172 140272 +11510 140237 +17087 140207 +17897 140176 +17315 140174 +22064 140170 +37758 140169 +8608 140168 +8105 140165 +27247 140162 +12899 140156 +14436 140154 +20013 140149 +32134 140148 +10998 140138 +23354 140134 +26971 140127 +20327 140124 +26773 140115 +34982 140110 +20286 140083 +19313 140048 +23608 140032 +12505 140012 +27593 139969 +32025 139966 +19347 139957 +13365 139945 +13970 139935 +22452 139930 +8133 139915 +26445 139911 +12967 139876 +15134 139872 +9597 139860 +17107 139855 +19253 139841 +36895 139840 +2703 139835 +12029 139834 +40713 139815 +29269 139809 +17810 139778 +13517 139757 +16543 139737 +16502 139714 +14773 139695 +5351 139685 +12623 139663 +35363 139642 +27501 139623 +4715 139600 +3129 139597 +9245 139595 +9463 139588 +27885 139587 +17965 139583 +19132 139552 +13996 139549 +17282 139532 +15658 139528 +13319 139474 +5958 139469 +22218 139457 +25332 139452 +36779 139435 +22211 139423 +3379 139419 +6921 139416 +9234 139404 +17902 139396 +13288 139395 +23486 139371 +26479 139365 +37135 139360 +18371 139338 +7870 139292 +22597 139285 +26812 139273 +38704 139262 +14170 139260 +41708 139256 +19192 139241 +25003 139237 +2082 139236 +33330 139203 +39818 139192 +34105 139171 +25951 139164 +40020 139161 +21493 139131 +14128 139128 +9303 139119 +8103 139094 +14555 139070 +14156 139032 +22078 139012 +3266 138995 +14680 138963 +41941 138960 +18929 138948 +7938 138938 +17144 138922 +5724 138921 +18764 138920 +18915 138877 +10970 138844 +6255 138824 +19092 138812 +23233 138772 +16526 138749 +18657 138723 +9012 138692 +11580 138689 +16723 138684 +3144 138649 +754 138649 +5005 138632 +13012 138619 +944 138584 +10814 138583 +18560 138535 +17858 138511 +5751 138507 +29467 138495 +10354 138474 +10735 138452 +23019 138452 +29374 138452 +39946 138446 +22954 138429 +20127 138406 +15068 138388 +32644 138345 +31518 138313 +25912 138299 +24349 138293 +19989 138283 +12261 138276 +19494 138260 +28220 138234 +14546 138229 +26508 138228 +13706 138227 +21432 138224 +30093 138195 +18997 138186 +18935 138185 +19009 138150 +9596 138085 +21980 138078 +22301 138067 +21150 138016 +30311 138013 +28395 138002 +1145 137982 +17415 137982 +11892 137980 +48216 137923 +25448 137913 +14482 137889 +18965 137887 +17825 137879 +18523 137867 +11196 137844 +6850 137838 +19036 137832 +8516 137831 +10736 137823 +15686 137814 +8352 137792 +17324 137789 +25204 137788 +30315 137776 +25320 137772 +26874 137734 +13159 137723 +32369 137717 +13817 137699 +34673 137683 +9478 137680 +32646 137661 +21904 137649 +19789 137636 +25091 137583 +16700 137566 +21768 137552 +13066 137528 +23266 137523 +13893 137487 +12016 137481 +12323 137456 +25583 137455 +23300 137453 +15696 137327 +12855 137310 +49219 137299 +8151 137294 +16908 137290 +8439 137277 +6037 137276 +15285 137271 +27265 137223 +14769 137218 +15532 137198 +13763 137185 +30479 137177 +20524 137161 +28356 137159 +8575 137140 +12850 137138 +16045 137125 +27678 137117 +23226 137112 +10696 137107 +16391 137106 +20463 137105 +14743 137089 +15963 137075 +18138 137041 +31335 137033 +33709 137028 +8753 137024 +18310 136991 +8249 136987 +18116 136975 +19228 136973 +24009 136969 +21348 136945 +9997 136892 +35521 136858 +30260 136849 +20902 136836 +22641 136817 +5327 136805 +17073 136787 +13390 136757 +15934 136751 +35109 136719 +25974 136715 +24050 136706 +18903 136670 +37173 136634 +1652 136609 +23940 136579 +4223 136573 +26347 136572 +16162 136554 +16380 136548 +11851 136525 +14937 136515 +19161 136465 +22043 136462 +20849 136448 +31745 136448 +13818 136435 +23956 136431 +47144 136428 +34974 136423 +13801 136409 +22053 136378 +2525 136377 +6210 136367 +31540 136361 +1293 136354 +13993 136347 +14494 136319 +5259 136294 +4818 136274 +17864 136261 +24828 136253 +19674 136251 +25143 136242 +26913 136241 +11412 136220 +3810 136187 +24463 136173 +47962 136161 +18838 136132 +15657 136132 +34891 136078 +18302 136078 +29377 136051 +21349 136043 +28650 136013 +5088 136008 +18250 135991 +21691 135968 +15430 135949 +2654 135947 +44830 135945 +8520 135934 +12429 135919 +19888 135915 +17813 135910 +15827 135906 +22844 135900 +20434 135895 +20201 135887 +15554 135886 +12884 135878 +13859 135821 +827 135794 +27022 135785 +5973 135784 +26721 135779 +20170 135770 +19670 135770 +10892 135733 +15028 135730 +30928 135729 +9537 135713 +16771 135662 +4686 135657 +17899 135653 +7338 135653 +9742 135638 +8874 135624 +25067 135612 +2401 135602 +18644 135580 +4915 135574 +23727 135562 +15201 135534 +5746 135506 +21829 135459 +21040 135442 +17173 135442 +15768 135440 +27139 135425 +23790 135421 +16808 135408 +32722 135364 +31941 135358 +17991 135358 +9996 135356 +9360 135346 +14754 135332 +23049 135312 +18449 135257 +17153 135236 +2470 135234 +12956 135230 +1725 135221 +41107 135218 +16461 135211 +18860 135206 +15651 135184 +19110 135179 +15643 135177 +14149 135109 +16337 135106 +39897 135092 +12693 135089 +26649 135084 +11389 135075 +9145 135074 +26346 135068 +27443 135067 +22798 135063 +8377 135055 +34698 135017 +32250 135009 +24002 135005 +22663 135003 +19696 135001 +27698 134997 +21803 134967 +32108 134952 +18582 134939 +17101 134938 +32641 134906 +16038 134906 +9712 134898 +32462 134885 +15899 134884 +15468 134883 +15374 134876 +12642 134866 +22304 134858 +44133 134834 +15222 134777 +22001 134772 +29760 134753 +24553 134748 +35706 134738 +18715 134725 +9525 134724 +14716 134703 +22089 134687 +46754 134683 +9423 134677 +37118 134677 +25041 134658 +29995 134646 +24585 134646 +7340 134641 +10984 134634 +17387 134623 +32076 134607 +31178 134594 +26542 134575 +3672 134564 +5773 134545 +23006 134542 +24959 134532 +10154 134529 +22534 134525 +14979 134521 +21246 134516 +26376 134513 +12364 134511 +20725 134496 +18412 134495 +22080 134493 +7066 134484 +19412 134469 +7821 134443 +25518 134438 +24023 134427 +13540 134402 +45826 134397 +13843 134350 +17849 134341 +41992 134319 +5434 134314 +16226 134302 +24301 134299 +13742 134277 +17843 134272 +15915 134271 +33693 134260 +31958 134254 +26601 134245 +21500 134191 +33545 134161 +13213 134121 +48922 134117 +17979 134103 +11346 134093 +14723 134054 +18705 134045 +18008 134043 +19416 134036 +33167 134027 +33666 134023 +23043 133985 +8772 133973 +12440 133968 +39419 133931 +19403 133911 +25426 133910 +4454 133866 +22707 133862 +20865 133845 +26778 133839 +36538 133814 +19716 133802 +6362 133794 +32944 133782 +24017 133778 +10659 133777 +20824 133736 +37539 133714 +29771 133710 +6449 133687 +3044 133682 +19497 133674 +18081 133662 +16660 133646 +3448 133633 +18700 133633 +38735 133626 +7594 133617 +14335 133600 +12015 133581 +5460 133580 +20818 133557 +14198 133555 +20001 133546 +10396 133540 +21350 133521 +22864 133484 +33953 133448 +17534 133432 +29638 133397 +18383 133386 +7241 133382 +43630 133346 +25160 133310 +5356 133302 +24534 133271 +21660 133251 +23140 133242 +19600 133234 +17239 133213 +13559 133210 +34944 133196 +17819 133125 +24722 133121 +34509 133118 +7737 133116 +23225 133089 +12957 133079 +19158 133031 +11215 133028 +35004 133010 +8491 132998 +30461 132990 +38599 132981 +17762 132978 +21277 132978 +9587 132973 +4828 132970 +16572 132957 +27803 132945 +20155 132912 +20333 132911 +18805 132907 +1732 132894 +18258 132883 +13601 132867 +17330 132854 +806 132852 +25463 132829 +15475 132816 +24803 132814 +32074 132808 +12078 132797 +16178 132792 +18552 132780 +33362 132778 +23178 132778 +21811 132771 +20030 132746 +34747 132706 +31827 132697 +11881 132687 +7518 132678 +14985 132669 +46800 132651 +22662 132640 +31307 132614 +28231 132609 +14664 132607 +19978 132601 +10794 132574 +32897 132545 +12115 132544 +30758 132541 +27388 132532 +28894 132530 +29066 132508 +18031 132440 +5321 132440 +31156 132418 +25130 132405 +34241 132380 +22255 132338 +18252 132308 +23548 132307 +35300 132300 +26264 132293 +22403 132289 +21358 132287 +27324 132257 +35331 132241 +19623 132232 +3444 132224 +39608 132215 +34995 132211 +7245 132211 +13332 132205 +26383 132203 +19869 132142 +34802 132112 +21419 132108 +12945 132088 +23033 132074 +14352 132073 +20545 132063 +37698 132057 +18132 132029 +13364 132018 +24199 132011 +12249 132005 +21788 132003 +26459 131986 +18585 131981 +14145 131961 +15025 131951 +8094 131944 +5056 131936 +8496 131930 +7951 131926 +32464 131913 +21572 131899 +23890 131893 +31187 131878 +27094 131874 +14775 131870 +16164 131870 +44070 131862 +935 131858 +22892 131842 +33127 131841 +18535 131830 +18045 131822 +17985 131820 +27028 131794 +10990 131793 +233 131755 +1032 131748 +12810 131670 +20295 131661 +7003 131644 +13531 131639 +19283 131633 +19521 131629 +10883 131607 +3102 131603 +23310 131602 +28972 131585 +5165 131579 +17885 131576 +12185 131569 +22688 131533 +23630 131530 +6472 131510 +20857 131502 +10840 131472 +13357 131449 +16131 131445 +19305 131438 +16157 131430 +32188 131412 +19406 131393 +18030 131387 +4016 131369 +16617 131368 +18514 131367 +25953 131334 +13333 131318 +23023 131299 +24315 131296 +15195 131293 +26735 131288 +13793 131276 +26608 131252 +19280 131233 +32737 131230 +5976 131228 +28132 131197 +25623 131179 +27663 131158 +18661 131145 +20152 131143 +15687 131126 +19850 131110 +6783 131085 +13140 131080 +32382 131072 +12809 131070 +1454 131069 +13496 131061 +23342 131057 +28223 131024 +19141 131022 +16924 131003 +15799 130966 +15943 130965 +19069 130964 +38768 130960 +30680 130916 +20230 130906 +22447 130893 +20220 130885 +23641 130885 +18257 130869 +20061 130868 +29755 130867 +13798 130850 +11340 130838 +8379 130835 +9164 130831 +23254 130822 +31592 130815 +31036 130808 +23411 130797 +16746 130783 +19533 130778 +18600 130768 +18192 130748 +5669 130742 +19189 130734 +17419 130704 +18056 130682 +24424 130681 +12854 130680 +19410 130662 +5872 130645 +16784 130638 +13108 130617 +23885 130606 +27779 130603 +14811 130593 +18579 130591 +15495 130558 +20607 130555 +10076 130554 +23366 130553 +18832 130552 +11982 130540 +22126 130538 +30717 130536 +14181 130527 +15064 130516 +31585 130506 +25564 130505 +29872 130490 +26167 130479 +19066 130476 +21922 130471 +8483 130441 +10389 130430 +26338 130402 +8881 130361 +9662 130356 +11817 130350 +16139 130336 +23659 130331 +3291 130328 +10203 130318 +2833 130283 +24310 130243 +29717 130191 +11028 130165 +14760 130155 +10992 130152 +10449 130128 +40518 130127 +13981 130093 +22573 130086 +6197 130086 +37478 130063 +19086 130054 +31311 130029 +49368 130024 +14795 129994 +19386 129991 +16217 129979 +24737 129960 +9543 129951 +45091 129944 +25476 129943 +10978 129933 +5910 129924 +30858 129910 +16043 129863 +21568 129860 +23782 129837 +13256 129807 +8410 129804 +24479 129787 +28236 129781 +34540 129770 +21332 129767 +37415 129704 +12530 129690 +29045 129676 +9052 129673 +19255 129652 +15917 129642 +17261 129624 +24083 129604 +17706 129598 +13597 129594 +30629 129587 +16523 129579 +17767 129570 +34285 129568 +39873 129554 +20623 129536 +17940 129530 +13519 129530 +27583 129529 +15762 129522 +3803 129499 +36401 129483 +30825 129482 +26106 129455 +2960 129446 +28559 129444 +19993 129436 +27209 129405 +6187 129401 +29364 129394 +18577 129378 +22310 129377 +18029 129363 +25556 129363 +32802 129362 +11364 129341 +16958 129340 +17677 129312 +24554 129301 +28150 129243 +37376 129209 +34564 129195 +13825 129189 +4239 129188 +28793 129168 +28319 129154 +10153 129150 +22086 129150 +17317 129150 +14806 129144 +6799 129137 +15577 129112 +23319 129096 +20833 129088 +9410 129056 +13242 129047 +11674 129039 +23704 129017 +11992 129017 +13829 129014 +11216 129009 +29029 128983 +19308 128976 +32408 128968 +11319 128936 +16379 128905 +2871 128865 +16711 128854 +19695 128852 +10214 128837 +10411 128813 +3299 128802 +7087 128781 +15502 128780 +21509 128778 +12469 128777 +14211 128776 +13160 128763 +21076 128759 +30321 128753 +12414 128727 +27918 128698 +30258 128698 +27179 128690 +18873 128688 +16026 128688 +20622 128666 +6429 128658 +15666 128641 +20775 128611 +14722 128607 +20749 128607 +26690 128554 +44723 128548 +18482 128544 +22176 128537 +13794 128537 +34497 128533 +28080 128527 +21320 128520 +14027 128499 +13235 128496 +15275 128492 +21454 128488 +19173 128447 +16976 128441 +15923 128440 +10350 128420 +21543 128354 +17919 128353 +28992 128338 +26617 128329 +14828 128323 +20239 128323 +16257 128293 +14745 128239 +17577 128234 +33487 128231 +6605 128229 +2086 128196 +17795 128182 +11960 128158 +5763 128145 +20691 128128 +9417 128124 +24535 128114 +11848 128102 +29470 128090 +2349 128089 +19832 128082 +14130 128060 +18306 128059 +28931 128054 +14991 128049 +16464 128042 +13906 128036 +13964 128032 +24999 128027 +21608 128023 +22836 128019 +34778 128017 +1224 128012 +8156 127986 +41530 127975 +8255 127963 +30511 127955 +16463 127953 +31802 127951 +20897 127935 +20027 127933 +18447 127917 +15847 127899 +24671 127878 +4161 127872 +20726 127850 +13698 127830 +20159 127826 +7354 127810 +36399 127809 +21120 127800 +11423 127791 +24093 127773 +35482 127769 +13424 127768 +29056 127739 +16967 127728 +3855 127719 +31329 127712 +13123 127695 +31970 127686 +22198 127638 +9449 127632 +20883 127616 +26626 127615 +9437 127588 +24660 127570 +25619 127569 +16481 127568 +16479 127562 +9367 127550 +25615 127547 +14963 127542 +24126 127539 +10838 127539 +20069 127526 +2388 127526 +22821 127519 +19546 127503 +25187 127487 +6437 127487 +29328 127462 +3817 127422 +24841 127416 +11959 127411 +21381 127405 +12685 127401 +29276 127377 +12265 127352 +17435 127348 +15061 127330 +1669 127328 +14292 127320 +17713 127315 +19063 127307 +19157 127281 +8351 127277 +19264 127255 +17757 127245 +11752 127207 +14553 127192 +20188 127170 +30088 127157 +15569 127152 +22424 127141 +7191 127137 +4551 127129 +21257 127127 +31100 127127 +9267 127119 +28181 127109 +14658 127089 +17603 127087 +8851 127082 +11976 127069 +26089 127068 +14976 127059 +3868 127049 +1226 127040 +20116 127039 +16697 127037 +32426 127029 +18282 127021 +8158 127004 +26044 127003 +12176 126996 +17361 126991 +23465 126976 +34283 126916 +24110 126892 +26482 126883 +14798 126866 +36011 126836 +11698 126823 +7736 126779 +28199 126777 +18359 126777 +33157 126772 +18057 126766 +5344 126749 +29852 126727 +8426 126721 +27466 126715 +14460 126712 +3930 126684 +14582 126678 +17865 126674 +8176 126622 +31325 126613 +12128 126609 +39998 126599 +17039 126588 +15325 126572 +15700 126538 +17789 126511 +20229 126490 +27082 126490 +24173 126455 +47417 126452 +21049 126376 +12416 126374 +234 126350 +9608 126318 +10723 126314 +6344 126314 +25693 126308 +27994 126292 +23527 126266 +110 126256 +25438 126255 +20018 126243 +26401 126222 +17009 126214 +35398 126200 +27181 126178 +15394 126166 +22895 126160 +13399 126152 +6152 126070 +33051 126068 +30918 126061 +32300 126058 +9090 126034 +11147 126006 +20573 126005 +22162 125978 +1727 125971 +36872 125968 +31455 125964 +13459 125960 +12431 125944 +23369 125940 +20738 125923 +8992 125913 +31806 125894 +29864 125890 +12959 125889 +17867 125857 +20212 125844 +9126 125812 +21151 125803 +29822 125802 +17769 125762 +14032 125746 +12587 125739 +24872 125735 +27051 125733 +22145 125721 +21329 125709 +2343 125702 +21880 125672 +25026 125659 +15900 125632 +5241 125627 +6434 125614 +23003 125594 +13897 125543 +18051 125534 +13468 125528 +15265 125521 +27165 125510 +28024 125506 +22052 125501 +29111 125497 +26321 125484 +13755 125474 +15778 125466 +11240 125464 +30498 125461 +23754 125450 +23856 125414 +33379 125367 +14165 125365 +12749 125345 +25028 125333 +28313 125331 +16386 125299 +16282 125293 +16949 125290 +36983 125273 +16829 125258 +21631 125254 +4199 125233 +38579 125213 +18794 125199 +7940 125190 +26431 125158 +40396 125151 +18978 125148 +30126 125144 +28830 125126 +17275 125125 +20469 125107 +4666 125106 +16342 125106 +30644 125051 +21555 125045 +10480 125034 +10536 125025 +10756 125025 +12035 125019 +8056 125013 +15972 124988 +18131 124971 +13948 124971 +26728 124956 +14249 124947 +15927 124937 +44130 124928 +46880 124912 +11891 124899 +27120 124857 +17760 124849 +15756 124847 +15458 124842 +15218 124822 +44418 124816 +21899 124814 +16276 124788 +11799 124788 +28274 124781 +15423 124764 +15588 124733 +17626 124723 +35921 124719 +15613 124712 +20062 124697 +29388 124694 +24878 124664 +9150 124653 +5314 124647 +19170 124645 +25005 124640 +13527 124636 +13604 124619 +17604 124610 +4626 124572 +28085 124567 +21801 124559 +42506 124556 +17404 124554 +4716 124547 +11879 124527 +38943 124485 +15524 124457 +14035 124441 +49963 124398 +18502 124394 +34072 124393 +20249 124389 +5299 124386 +15669 124369 +27890 124349 +28293 124329 +18839 124308 +19273 124302 +43204 124276 +21027 124269 +27309 124267 +23692 124252 +18082 124252 +2659 124243 +22727 124243 +1199 124239 +19525 124239 +29897 124224 +22325 124221 +30681 124190 +11782 124180 +22376 124105 +20612 124096 +2373 124096 +27512 124091 +27320 124071 +23953 124058 +16851 124048 +21547 124045 +16408 124037 +19719 123989 +20440 123988 +24185 123978 +16402 123943 +23446 123940 +31832 123926 +25821 123920 +6535 123915 +37236 123894 +26624 123888 +32788 123883 +25408 123870 +36707 123865 +38477 123845 +17021 123842 +3760 123831 +7662 123825 +14962 123814 +30569 123795 +32251 123776 +14787 123775 +21517 123753 +8205 123745 +19874 123734 +14160 123724 +19642 123705 +15408 123694 +14357 123690 +11621 123689 +25522 123687 +26673 123684 +17639 123681 +26849 123656 +5429 123621 +27999 123612 +12852 123612 +27350 123608 +23842 123602 +1214 123579 +30895 123559 +45376 123535 +33461 123520 +21845 123504 +29072 123497 +19183 123465 +10374 123419 +25866 123418 +24405 123406 +31154 123400 +23397 123390 +16003 123386 +19444 123382 +14267 123353 +15259 123346 +19374 123341 +6999 123340 +19330 123337 +11701 123317 +18281 123284 +14476 123278 +30857 123273 +35571 123272 +19451 123265 +16030 123263 +10870 123261 +18403 123215 +17170 123207 +10921 123203 +15837 123185 +29049 123176 +17141 123172 +15368 123168 +16609 123161 +18822 123148 +16524 123143 +38396 123091 +28842 123082 +19249 123080 +34803 123075 +13300 123073 +19595 123057 +15751 123055 +23679 123044 +50196 123042 +27938 123041 +18658 122934 +5391 122903 +23121 122853 +18990 122850 +14816 122833 +12816 122817 +25016 122807 +20268 122806 +33743 122793 +28549 122781 +26640 122767 +34649 122757 +25123 122740 +27557 122730 +13532 122717 +24172 122715 +8199 122667 +18178 122650 +21683 122604 +16690 122592 +35773 122588 +25575 122564 +14276 122559 +17990 122553 +27820 122549 +32907 122517 +22901 122505 +19611 122498 +18044 122478 +15629 122450 +18774 122446 +9156 122434 +27702 122421 +17755 122419 +17064 122398 +23352 122397 +39489 122393 +33099 122372 +3345 122359 +15277 122337 +15821 122332 +28608 122328 +32664 122320 +31045 122298 +24800 122293 +30724 122286 +33391 122275 +16555 122274 +11487 122269 +27922 122268 +29866 122267 +20105 122265 +13589 122260 +11181 122223 +14520 122206 +23933 122204 +21067 122194 +17233 122177 +17333 122168 +28760 122160 +15483 122157 +18207 122151 +19178 122151 +12835 122146 +26896 122141 +14461 122113 +19111 122104 +29431 122093 +25577 122083 +22355 122083 +17105 122065 +12819 122057 +24224 122023 +15772 122005 +21331 121999 +35292 121948 +15000 121946 +16880 121939 +46289 121924 +38718 121920 +18650 121917 +2146 121912 +31574 121909 +23143 121899 +4594 121896 +11844 121885 +24789 121882 +18767 121867 +40418 121845 +29834 121842 +27975 121824 +11914 121808 +26577 121791 +26004 121781 +27862 121776 +18957 121771 +21089 121764 +19015 121750 +14114 121748 +22118 121746 +35825 121727 +22803 121723 +16062 121692 +20963 121687 +27027 121665 +15833 121661 +12840 121654 +43736 121641 +13218 121636 +19188 121623 +14084 121622 +30552 121613 +20841 121595 +7925 121592 +28331 121583 +16289 121565 +28948 121558 +30422 121546 +19518 121542 +16785 121516 +22777 121507 +27604 121474 +22436 121467 +10005 121465 +3079 121458 +25218 121457 +17814 121418 +14144 121412 +10178 121411 +12994 121402 +5813 121392 +17026 121374 +34271 121366 +7210 121366 +22499 121344 +15448 121313 +26380 121311 +16412 121301 +24072 121295 +10133 121291 +14944 121282 +29975 121282 +31770 121280 +24138 121264 +7839 121247 +16997 121228 +4821 121222 +26966 121221 +41770 121215 +4653 121195 +15814 121179 +17442 121169 +7575 121133 +14268 121123 +38779 121115 +15178 121110 +37506 121108 +31107 121078 +10835 121074 +14404 121073 +19684 121048 +12550 121038 +3622 121019 +34273 121016 +15488 121016 +16936 121013 +32054 121011 +24054 121010 +34361 120997 +37822 120990 +3147 120971 +23276 120969 +18157 120957 +10254 120946 +5377 120932 +8702 120927 +20875 120921 +14907 120915 +31004 120888 +2588 120876 +23431 120875 +28212 120873 +41624 120868 +2505 120854 +37213 120848 +15728 120840 +12136 120838 +4830 120827 +23455 120816 +15213 120805 +28323 120804 +20324 120794 +22700 120786 +11230 120775 +31446 120774 +4775 120761 +15957 120760 +24076 120758 +18696 120758 +12518 120749 +28470 120716 +28449 120707 +11244 120691 +15781 120676 +27174 120676 +14425 120666 +15187 120654 +17142 120653 +27655 120646 +16662 120643 +15674 120638 +20952 120631 +15389 120630 +22105 120580 +20606 120580 +9223 120553 +34855 120551 +28275 120508 +22835 120507 +9791 120495 +13503 120475 +3976 120460 +33925 120449 +15709 120440 +20371 120439 +21187 120396 +27470 120386 +14682 120381 +18988 120379 +30197 120370 +11985 120353 +16014 120342 +21900 120323 +16309 120301 +44568 120291 +29583 120286 +10995 120285 +20518 120284 +22249 120281 +17210 120279 +30145 120243 +21404 120234 +5591 120233 +28314 120227 +2537 120221 +19236 120189 +31388 120186 +23732 120184 +14277 120172 +27504 120153 +10337 120141 +18639 120121 +35410 120108 +16224 120100 +45284 120055 +20525 120041 +28382 120031 +16510 120026 +5730 120006 +32612 119981 +49746 119974 +22111 119950 +14712 119947 +43321 119941 +20889 119936 +22250 119924 +14711 119913 +16871 119905 +35686 119871 +19827 119866 +24816 119820 +14199 119818 +10812 119815 +32801 119798 +29875 119795 +24746 119781 +18663 119772 +25876 119743 +33845 119741 +19134 119731 +20246 119692 +15216 119688 +7465 119681 +2758 119676 +10946 119674 +12807 119659 +44083 119659 +28755 119659 +22202 119642 +16220 119634 +29713 119630 +15758 119592 +11397 119582 +31066 119571 +27008 119551 +18101 119524 +21676 119521 +9206 119499 +28822 119497 +38780 119495 +39597 119480 +24215 119476 +18422 119472 +17228 119434 +9484 119428 +12116 119426 +19205 119424 +24462 119423 +35837 119407 +25604 119399 +20444 119399 +15922 119371 +24482 119369 +17698 119363 +31769 119358 +16235 119356 +6249 119355 +19505 119342 +15334 119337 +19059 119312 +19224 119292 +6637 119277 +21638 119272 +40666 119252 +47563 119245 +36677 119243 +15231 119229 +17341 119228 +18413 119221 +19287 119197 +10314 119173 +19992 119162 +28818 119150 +13674 119110 +17318 119107 +42095 119104 +27154 119096 +37518 119076 +20195 119074 +19885 119066 +2462 119063 +14932 119055 +19254 119041 +36552 119014 +11358 118995 +33534 118974 +12163 118957 +27434 118926 +19644 118916 +27511 118900 +17231 118891 +48349 118867 +44337 118827 +3674 118802 +22976 118787 +1319 118776 +15641 118775 +18963 118772 +19047 118771 +31593 118770 +24268 118770 +24120 118766 +24410 118765 +7854 118753 +17658 118737 +12558 118726 +7669 118722 +23634 118718 +12250 118710 +31039 118696 +12711 118695 +24791 118694 +27136 118687 +33445 118674 +17137 118670 +22116 118652 +21372 118651 +24759 118644 +19637 118643 +12314 118624 +4024 118619 +19572 118616 +8106 118612 +21190 118608 +29329 118591 +6749 118583 +5522 118573 +21189 118538 +20727 118524 +35083 118523 +22591 118520 +8336 118518 +18421 118510 +8120 118508 +27478 118507 +27667 118497 +18531 118493 +7975 118489 +18455 118481 +24967 118452 +16622 118441 +16560 118434 +17856 118417 +25048 118416 +17247 118409 +13008 118399 +30169 118392 +27023 118392 +16415 118387 +28237 118383 +25884 118378 +14561 118377 +28623 118376 +1974 118374 +34468 118353 +30997 118352 +17479 118309 +15063 118308 +40188 118300 +12183 118289 +28487 118265 +39692 118254 +10850 118242 +596 118239 +24024 118178 +29010 118175 +26549 118175 +7544 118174 +14622 118141 +21030 118135 +32130 118109 +15964 118100 +10321 118085 +5406 118074 +12799 118067 +9248 118049 +19450 118047 +13353 118038 +20340 118037 +20945 118033 +26562 118027 +24817 118026 +23141 118026 +35232 118004 +20747 118003 +17988 117996 +7001 117976 +26363 117967 +32210 117957 +31842 117951 +29216 117939 +15376 117938 +16727 117927 +22475 117899 +35560 117898 +16101 117871 +14917 117868 +9782 117864 +19837 117850 +16385 117848 +9130 117810 +26768 117788 +997 117779 +6864 117774 +21837 117760 +32055 117755 +36316 117750 +18592 117740 +36606 117732 +6285 117725 +13281 117721 +36345 117709 +4704 117694 +21229 117692 +34174 117672 +3366 117657 +45695 117655 +15129 117648 +46441 117643 +41863 117632 +1555 117625 +16039 117620 +14729 117604 +6161 117600 +32224 117592 +24058 117564 +17326 117563 +15745 117548 +8952 117546 +19235 117537 +17286 117529 +38167 117528 +10601 117522 +10136 117507 +49583 117488 +18821 117476 +46154 117472 +19272 117463 +21343 117461 +27707 117457 +3669 117453 +15766 117428 +34488 117427 +21846 117412 +37851 117408 +26516 117401 +21143 117391 +18725 117376 +34438 117368 +18702 117362 +8162 117343 +25259 117340 +22868 117336 +27818 117332 +18344 117327 +924 117316 +21472 117310 +40439 117306 +16971 117294 +32666 117293 +19125 117285 +31006 117275 +4861 117256 +14968 117255 +18692 117226 +8609 117209 +11854 117197 +30833 117194 +14941 117190 +12924 117181 +16450 117174 +48088 117173 +3305 117171 +15998 117171 +33017 117167 +40858 117164 +32579 117161 +17446 117132 +17636 117129 +21921 117124 +45567 117124 +41445 117094 +17631 117091 +23962 117024 +28367 117022 +14380 117015 +19417 117010 +38029 116996 +708 116994 +28990 116979 +17197 116973 +12368 116961 +23538 116953 +30418 116952 +49611 116944 +20913 116940 +30241 116931 +32451 116924 +31750 116924 +6837 116916 +15228 116889 +21830 116886 +14966 116854 +11377 116852 +24854 116839 +36269 116837 +248 116834 +13939 116827 +39290 116766 +3778 116752 +9771 116747 +22846 116733 +21347 116731 +33457 116727 +18451 116724 +19681 116718 +45790 116681 +13508 116680 +22493 116676 +22598 116671 +4720 116667 +30024 116661 +20928 116656 +21096 116654 +27581 116650 +5758 116642 +12902 116633 +22778 116623 +25894 116617 +15719 116606 +39398 116603 +16807 116602 +27231 116596 +7816 116582 +34108 116573 +30567 116559 +10056 116543 +14715 116519 +14475 116497 +23910 116456 +14834 116456 +30364 116454 +25706 116453 +15753 116444 +28442 116434 +35469 116433 +12310 116425 +15910 116416 +21957 116405 +39808 116388 +16903 116385 +18989 116376 +39487 116376 +14782 116372 +44744 116356 +17180 116354 +18556 116349 +26876 116324 +27436 116313 +25482 116308 +13735 116304 +25394 116294 +26450 116277 +37616 116277 +7643 116276 +10550 116245 +19693 116243 +25852 116220 +1009 116180 +12861 116176 +11043 116174 +17124 116170 +24933 116165 +12921 116149 +18642 116146 +15402 116135 +2469 116096 +28578 116075 +23682 116074 +17939 116040 +17918 116033 +4491 116022 +4399 116019 +22232 116015 +30837 116010 +2296 116003 +19356 115997 +30738 115978 +10934 115971 +15836 115952 +19266 115932 +28571 115926 +23558 115922 +22761 115911 +30217 115908 +28083 115882 +19750 115877 +15418 115869 +7438 115863 +37918 115860 +42442 115851 +22568 115848 +4670 115832 +20537 115831 +26212 115829 +10626 115826 +25722 115824 +24952 115801 +38739 115794 +28172 115756 +22942 115738 +32246 115724 +22698 115711 +26724 115689 +16394 115688 +19860 115671 +22045 115657 +19771 115654 +7286 115652 +31393 115649 +24719 115594 +10606 115590 +11742 115574 +20417 115560 +39641 115541 +30157 115539 +18979 115489 +29784 115471 +22871 115456 +16656 115454 +17412 115448 +20185 115441 +10340 115438 +22152 115437 +22753 115424 +47970 115422 +8332 115410 +20388 115407 +40238 115385 +17136 115384 +19215 115375 +5497 115369 +24630 115363 +36026 115356 +6217 115346 +38307 115331 +12931 115321 +44985 115318 +19240 115308 +8588 115305 +17343 115297 +16965 115297 +21108 115274 +31668 115272 +23423 115269 +44021 115255 +18316 115242 +111 115218 +30017 115213 +19152 115197 +34668 115186 +43449 115178 +14746 115152 +32559 115151 +39199 115140 +19014 115138 +8325 115132 +13348 115122 +28137 115112 +17351 115074 +10961 115046 +24448 115041 +30625 115012 +32473 114997 +27243 114972 +25737 114952 +38988 114930 +33629 114917 +1198 114880 +23041 114878 +33197 114860 +32763 114850 +25732 114849 +24617 114837 +4657 114831 +24237 114822 +18492 114820 +48609 114805 +17726 114802 +19575 114794 +10324 114779 +15009 114778 +10943 114778 +46485 114767 +34609 114761 +24011 114759 +24626 114742 +11393 114718 +14262 114713 +13432 114709 +20517 114709 +18813 114701 +23897 114697 +13813 114693 +24456 114672 +17870 114659 +4820 114652 +15363 114638 +18137 114627 +27239 114619 +37520 114611 +1791 114610 +34022 114584 +15693 114577 +16314 114563 +6696 114550 +22425 114545 +14934 114533 +13068 114533 +37936 114533 +46520 114532 +7510 114530 +19692 114528 +28001 114519 +20858 114465 +15959 114460 +34410 114446 +18429 114436 +21383 114434 +29476 114425 +20830 114419 +36019 114417 +15440 114410 +31140 114376 +13886 114375 +17451 114350 +16728 114339 +19516 114329 +19218 114295 +6347 114288 +6013 114285 +25347 114282 +28960 114280 +44286 114263 +20736 114226 +7758 114219 +25837 114215 +11994 114203 +17293 114196 +9435 114179 +2244 114175 +15186 114155 +18341 114147 +29147 114139 +26803 114113 +12342 114113 +33061 114104 +24829 114104 +18135 114084 +31047 114082 +40560 114068 +21307 114058 +21416 114031 +27046 114011 +24261 114008 +47168 113996 +10344 113973 +9222 113955 +102 113953 +3564 113949 +27026 113948 +26172 113940 +3130 113924 +3063 113907 +24627 113901 +27296 113856 +10157 113854 +11950 113851 +33617 113842 +15589 113838 +31799 113808 +19340 113805 +109 113805 +2573 113794 +47180 113776 +6878 113771 +18077 113739 +17714 113729 +40159 113721 +40344 113718 +38580 113716 +35007 113711 +35053 113685 +10896 113670 +19353 113666 +11339 113666 +47652 113661 +17099 113658 +28785 113653 +25338 113646 +22076 113639 +8085 113636 +9235 113629 +11386 113612 +19344 113612 +17877 113612 +20496 113594 +24682 113590 +32127 113557 +23915 113556 +10462 113553 +2621 113550 +30554 113540 +5133 113528 +13954 113525 +47296 113443 +16726 113424 +10173 113419 +20999 113413 +17584 113387 +35440 113386 +2352 113376 +26619 113372 +14177 113343 +7234 113325 +19289 113294 +25446 113249 +31984 113228 +18682 113217 +22435 113192 +22044 113175 +36952 113170 +20285 113152 +24060 113141 +25736 113139 +30697 113084 +18984 113082 +14568 113068 +24473 113039 +17924 113037 +32376 113032 +33708 113030 +28941 113014 +17298 113013 +13878 113012 +14956 113000 +14542 112997 +29189 112980 +17166 112979 +22199 112966 +23130 112943 +34724 112929 +15683 112912 +17523 112887 +16084 112886 +17655 112871 +17558 112871 +20421 112866 +36477 112850 +17805 112849 +18450 112847 +35401 112845 +20929 112836 +14397 112833 +15181 112827 +12169 112804 +25374 112802 +20970 112801 +5482 112794 +16370 112782 +16550 112781 +21010 112776 +42071 112773 +13196 112755 +16576 112754 +15467 112752 +91 112729 +21966 112720 +29018 112719 +27788 112718 +23107 112696 +6691 112695 +38256 112694 +24610 112693 +10256 112691 +26202 112678 +23950 112673 +1563 112672 +20828 112642 +28587 112640 +16159 112637 +22518 112637 +9467 112608 +23529 112607 +10360 112607 +23948 112603 +9230 112592 +21961 112588 +26630 112585 +12961 112577 +16835 112565 +33726 112551 +28550 112537 +16042 112531 +9732 112521 +24948 112520 +10917 112519 +17238 112508 +15482 112506 +14419 112484 +16573 112478 +20756 112465 +17549 112426 +16294 112420 +18881 112418 +2851 112409 +29758 112392 +16489 112375 +14209 112371 +26585 112361 +40928 112354 +9665 112346 +17033 112337 +43225 112332 +23541 112321 +22694 112320 +25436 112317 +23825 112315 +21998 112298 +8642 112295 +24313 112284 +16840 112281 +29980 112271 +10693 112258 +30426 112255 +21702 112236 +17970 112199 +3027 112194 +24891 112183 +14572 112169 +4722 112149 +3219 112147 +1449 112130 +27496 112098 +22848 112089 +18162 112084 +28375 112054 +20294 112053 +29019 112048 +26736 112047 +5139 112032 +15618 112031 +36067 112030 +26066 112024 +9998 112023 +36518 112019 +29260 112017 +46536 112016 +22268 112007 +12618 111987 +17623 111987 +25750 111980 +30287 111957 +18384 111956 +16057 111956 +33311 111930 +33371 111930 +6606 111906 +21590 111904 +41806 111904 +23718 111903 +29553 111853 +24025 111849 +34687 111840 +27497 111833 +18120 111829 +18098 111817 +13505 111816 +2413 111776 +27906 111774 +18744 111765 +20835 111746 +1800 111745 +18622 111736 +16999 111726 +14047 111710 +26278 111708 +7885 111680 +18981 111679 +37909 111671 +14259 111665 +40098 111653 +22292 111646 +11466 111607 +27473 111603 +15975 111601 +3018 111598 +19612 111585 +18820 111566 +31946 111560 +36178 111544 +14233 111526 +12093 111521 +5336 111514 +13841 111480 +39992 111478 +25694 111476 +49487 111476 +19426 111473 +20449 111469 +1536 111462 +22843 111459 +22807 111449 +21411 111447 +22097 111387 +12510 111385 +20629 111383 +21832 111360 +15185 111321 +9973 111320 +29926 111313 +17888 111306 +25570 111298 +34521 111292 +13762 111274 +38529 111270 +32534 111267 +27119 111259 +18877 111243 +792 111231 +21570 111201 +7738 111178 +31620 111174 +31622 111172 +22581 111154 +2179 111152 +11144 111141 +5040 111121 +16747 111073 +13025 111060 +6010 111050 +17568 111050 +20433 111010 +20577 110983 +19217 110974 +29012 110948 +20562 110917 +18197 110910 +14547 110908 +33098 110894 +35783 110881 +22096 110879 +12821 110869 +16760 110864 +15131 110864 +34560 110857 +27418 110855 +974 110850 +26891 110849 +36986 110848 +36683 110815 +13433 110810 +25813 110795 +24634 110781 +5328 110780 +28604 110768 +19894 110754 +28628 110720 +17508 110714 +24884 110708 +22723 110705 +21001 110697 +4628 110669 +18233 110660 +31617 110625 +23994 110620 +14872 110618 +22537 110592 +16347 110563 +11880 110559 +33847 110559 +20471 110555 +18583 110553 +9678 110546 +13787 110530 +37081 110528 +38805 110507 +15094 110504 +33625 110502 +12397 110501 +34814 110499 +29 110485 +9377 110471 +25814 110470 +37790 110464 +17076 110419 +36199 110412 +32206 110411 +12076 110407 +23687 110402 +8729 110399 +22533 110393 +17565 110375 +25691 110364 +16416 110357 +16292 110346 +28773 110346 +16106 110343 +27147 110342 +13015 110331 +16343 110317 +30744 110312 +15968 110303 +35903 110267 +7934 110261 +243 110257 +30007 110254 +13482 110252 +12256 110244 +23017 110238 +4241 110232 +30379 110211 +20173 110198 +28276 110176 +24065 110172 +19951 110168 +45818 110162 +26420 110158 +18493 110158 +19379 110154 +36852 110123 +17229 110106 +17453 110105 +23707 110102 +12064 110092 +12707 110082 +31244 110069 +28014 110046 +2980 110045 +17556 110043 +1401 110024 +27872 110021 +12648 110011 +10796 110009 +18500 110006 +22414 109992 +30628 109992 +12178 109989 +8423 109985 +15284 109974 +23448 109945 +27206 109927 +22575 109926 +21025 109915 +24264 109903 +23384 109898 +2284 109898 +14479 109885 +29921 109881 +8757 109867 +34753 109867 +27216 109862 +19561 109854 +15731 109841 +31108 109834 +5285 109830 +14206 109802 +20406 109789 +21625 109782 +6162 109780 +27589 109766 +40162 109754 +26853 109753 +33006 109733 +13868 109724 +30215 109723 +17644 109720 +31942 109711 +20794 109705 +8870 109685 +38613 109647 +43105 109641 +21951 109636 +3508 109629 +15610 109610 +29597 109607 +37070 109563 +21823 109545 +14766 109542 +9383 109541 +9708 109539 +16925 109531 +18321 109531 +16251 109514 +22560 109502 +25340 109489 +13071 109479 +32669 109479 +18461 109462 +40532 109458 +34186 109449 +28513 109446 +37921 109420 +8652 109399 +27517 109399 +12430 109375 +14290 109374 +50240 109356 +27058 109344 +25892 109323 +18626 109313 +9178 109304 +36363 109265 +24034 109262 +27333 109250 +35553 109243 +9440 109239 +34699 109239 +49403 109230 +7020 109226 +16073 109215 +17074 109191 +16695 109190 +48583 109183 +47177 109166 +26496 109164 +8567 109162 +16158 109157 +17700 109114 +16058 109112 +27974 109089 +28879 109087 +17163 109079 +25957 109071 +24622 109053 +26049 109044 +17345 109038 +23622 109028 +25949 109027 +27833 109020 +26389 109002 +15169 109002 +25065 108979 +21651 108957 +23804 108946 +14698 108943 +23062 108941 +20387 108930 +15993 108930 +32868 108923 +27835 108908 +20101 108907 +37126 108899 +24649 108881 +23911 108878 +21200 108876 +941 108873 +13050 108861 +21099 108851 +24522 108847 +4984 108838 +15337 108834 +32992 108832 +34553 108827 +5731 108822 +20338 108811 +35345 108795 +37270 108794 +36758 108791 +8459 108790 +17656 108782 +42556 108782 +21707 108777 +41816 108764 +29842 108758 +41759 108758 +15938 108753 +29139 108746 +21068 108743 +16944 108734 +13746 108721 +2136 108712 +21566 108691 +15510 108679 +15533 108678 +26361 108677 +15939 108671 +25826 108655 +43416 108650 +11460 108646 +29761 108646 +18235 108626 +17289 108618 +17342 108612 +20031 108607 +30067 108607 +28245 108589 +29578 108583 +32444 108580 +24141 108576 +23783 108551 +21203 108544 +25203 108539 +45980 108539 +19458 108529 +10957 108528 +29588 108507 +24701 108498 +13695 108489 +8109 108486 +25141 108469 +36451 108460 +13411 108459 +12635 108442 +5558 108421 +15344 108417 +17548 108401 +25539 108399 +17775 108360 +28926 108341 +23103 108333 +45624 108330 +38602 108318 +26905 108304 +31275 108289 +39717 108286 +21673 108284 +10905 108265 +23185 108247 +34479 108242 +12251 108242 +20425 108242 +12177 108231 +21799 108218 +22673 108189 +31988 108187 +23993 108184 +28503 108175 +18427 108171 +32810 108130 +21848 108123 +33624 108112 +11598 108082 +24471 108049 +47604 108032 +18357 108026 +10707 108011 +9122 107999 +17696 107994 +22527 107990 +19647 107989 +15874 107977 +6258 107969 +16019 107967 +23973 107959 +26635 107954 +23887 107919 +13822 107917 +25965 107902 +18693 107889 +35089 107873 +28289 107850 +30770 107844 +10356 107804 +22561 107803 +42735 107795 +13745 107775 +35759 107766 +14992 107755 +28619 107755 +35206 107752 +7614 107745 +9446 107705 +36675 107690 +10327 107670 +41427 107666 +12876 107664 +23086 107651 +19901 107616 +21031 107579 +44321 107574 +33059 107560 +41253 107537 +33673 107536 +30597 107535 +14294 107534 +7250 107531 +27505 107531 +24341 107525 +36700 107516 +28485 107514 +34272 107514 +29336 107476 +20881 107471 +11653 107461 +29575 107452 +6243 107451 +23256 107447 +15529 107429 +21954 107420 +16605 107413 +2375 107413 +18436 107396 +14893 107391 +25523 107378 +7922 107373 +18179 107368 +13493 107366 +35691 107365 +17607 107348 +30401 107338 +1356 107332 +16882 107330 +25247 107317 +29521 107310 +11354 107308 +17942 107301 +23621 107297 +17439 107288 +11659 107272 +28346 107263 +17365 107258 +19658 107251 +44198 107247 +21596 107243 +25427 107235 +15017 107215 +18987 107199 +11401 107183 +10781 107179 +12375 107176 +15561 107168 +39062 107167 +4834 107159 +19038 107153 +21655 107138 +16125 107128 +23404 107127 +7952 107119 +29838 107098 +23514 107074 +22566 107038 +31876 107035 +25477 107014 +17186 107001 +25910 106995 +25824 106991 +25146 106974 +22894 106968 +29058 106953 +3718 106938 +36486 106932 +19113 106932 +24963 106930 +21046 106917 +20349 106912 +8625 106905 +25395 106878 +35188 106861 +34002 106860 +29656 106849 +932 106830 +18787 106830 +28672 106817 +39700 106798 +15058 106798 +31845 106798 +35892 106792 +30081 106760 +18399 106755 +39560 106753 +43262 106752 +13602 106745 +3384 106743 +19748 106725 +20528 106718 +29463 106712 +15645 106680 +17624 106678 +36719 106671 +19439 106659 +42668 106650 +21658 106623 +20236 106603 +12986 106599 +12870 106581 +8079 106576 +39137 106568 +18854 106552 +13664 106543 +12253 106495 +14063 106481 +25136 106471 +16762 106462 +16668 106453 +37578 106452 +29803 106432 +19168 106428 +26005 106408 +21362 106404 +21276 106400 +24271 106381 +26349 106371 +22900 106361 +20657 106360 +17753 106358 +46726 106344 +6113 106338 +24226 106337 +31074 106336 +17927 106333 +28044 106330 +4517 106327 +27687 106320 +19761 106313 +13575 106310 +18174 106282 +29820 106265 +18350 106262 +21723 106250 +37425 106240 +19654 106224 +5261 106223 +33035 106219 +36236 106203 +31333 106199 +43804 106198 +1860 106194 +11813 106179 +5685 106177 +13950 106177 +18418 106170 +16739 106164 +15936 106134 +18778 106127 +35902 106125 +32211 106122 +26815 106121 +27909 106119 +38001 106102 +48245 106097 +10339 106094 +30947 106078 +7842 106055 +24929 106038 +6391 105988 +21430 105973 +16426 105960 +15121 105958 +24128 105950 +17501 105930 +26336 105916 +25422 105910 +44844 105856 +5905 105843 +25485 105839 +2377 105825 +17586 105823 +14175 105822 +25614 105818 +22168 105808 +48560 105805 +42192 105798 +4135 105767 +12918 105742 +31117 105726 +17522 105716 +20732 105712 +16222 105709 +18035 105690 +15290 105633 +32470 105630 +22549 105607 +43013 105596 +18734 105595 +32034 105590 +13860 105581 +31232 105578 +39507 105571 +33787 105569 +5185 105569 +34992 105557 +20487 105544 +18894 105536 +14870 105520 +12941 105491 +29599 105482 +23543 105464 +16836 105460 +28229 105455 +7414 105449 +21615 105449 +36962 105418 +18028 105409 +35468 105406 +39032 105406 +24351 105397 +2220 105387 +39329 105383 +13661 105375 +985 105373 +31610 105352 +16430 105349 +20401 105348 +26038 105347 +21368 105326 +3646 105323 +26474 105320 +6767 105317 +44487 105294 +18782 105285 +17279 105275 +15497 105268 +40707 105266 +30385 105261 +29799 105256 +2557 105241 +1149 105226 +23080 105189 +17554 105179 +20160 105164 +27723 105162 +32586 105160 +10676 105154 +14107 105139 +10982 105131 +27316 105111 +10644 105109 +23365 105106 +1882 105085 +19193 105060 +24057 105059 +35507 105055 +11427 105024 +16065 105020 +18395 105017 +47975 105001 +12583 104995 +1861 104991 +38581 104975 +19589 104971 +19942 104959 +34807 104949 +14739 104948 +30446 104922 +37958 104912 +10510 104907 +6966 104889 +31376 104889 +18474 104873 +22650 104867 +28917 104843 +15857 104825 +12171 104825 +12038 104822 +42555 104819 +36547 104790 +15991 104789 +23030 104785 +19726 104780 +39911 104761 +26779 104752 +27565 104752 +22928 104741 +17184 104739 +16627 104734 +3571 104687 +15494 104677 +27337 104671 +10091 104669 +15989 104665 +26036 104660 +33500 104650 +24690 104606 +20674 104597 +27311 104596 +17783 104590 +23231 104588 +40556 104586 +28163 104581 +13397 104574 +27709 104566 +17621 104565 +30773 104562 +41582 104562 +17592 104532 +28723 104530 +36442 104526 +1264 104523 +16069 104515 +49743 104496 +32743 104485 +27585 104484 +33943 104479 +23229 104478 +30469 104472 +8337 104465 +18247 104461 +25363 104461 +22649 104458 +3969 104437 +23923 104424 +31019 104413 +4967 104390 +17778 104379 +27399 104375 +29209 104367 +13865 104354 +35079 104350 +17544 104349 +8863 104347 +21942 104345 +9703 104340 +12853 104326 +26319 104318 +30190 104315 +23184 104285 +26147 104273 +2903 104262 +20354 104253 +18671 104253 +16800 104229 +14242 104219 +8445 104217 +24824 104211 +33874 104198 +15632 104192 +15122 104183 +22804 104182 +33831 104177 +17587 104144 +31118 104114 +17686 104108 +17348 104103 +19957 104091 +24377 104085 +33828 104081 +16382 104077 +41916 104045 +17172 104043 +1376 104042 +18002 104038 +11978 104035 +20372 104024 +18898 104014 +30884 103943 +18780 103938 +19938 103936 +23151 103928 +15990 103904 +38684 103894 +18645 103870 +30596 103865 +9092 103852 +30105 103835 +47280 103829 +12040 103816 +27428 103816 +27875 103813 +9414 103804 +33442 103802 +18339 103797 +11777 103788 +42263 103767 +43602 103760 +25880 103756 +7793 103754 +19905 103747 +6083 103746 +16049 103738 +22397 103731 +26248 103720 +21553 103708 +16577 103703 +26725 103662 +13629 103660 +23610 103655 +41158 103638 +24263 103636 +24348 103590 +40370 103580 +9729 103568 +9861 103567 +29313 103564 +25756 103550 +24169 103541 +29234 103528 +47827 103506 +13345 103491 +19470 103488 +24411 103488 +17263 103478 +27658 103467 +12089 103457 +28634 103444 +1493 103440 +21160 103432 +24157 103415 +29941 103391 +26476 103389 +23147 103388 +30359 103378 +26540 103363 +11789 103361 +11437 103354 +20346 103346 +14589 103346 +19276 103339 +3339 103311 +37142 103300 +11517 103293 +20070 103275 +5842 103264 +13431 103260 +27811 103244 +45196 103236 +11908 103232 +12728 103220 +26399 103206 +20024 103200 +16194 103196 +31105 103148 +19022 103137 +2175 103129 +17784 103128 +41976 103126 +18507 103123 +9972 103123 +2302 103102 +22307 103102 +12219 103095 +39943 103079 +33089 103052 +36577 103024 +17146 103018 +30341 103009 +42896 103003 +19870 102997 +32681 102991 +29634 102984 +44081 102977 +9529 102948 +32013 102947 +35627 102934 +37568 102931 +27401 102921 +15741 102914 +4015 102903 +8653 102895 +27157 102878 +47214 102866 +13322 102862 +41124 102858 +30679 102858 +39295 102856 +24788 102840 +17175 102840 +27426 102836 +47837 102813 +8583 102810 +21104 102782 +37112 102767 +12281 102746 +33251 102743 +45015 102705 +24267 102704 +29756 102671 +6368 102668 +10151 102659 +25488 102649 +39874 102634 +39393 102632 +30938 102631 +24056 102621 +19471 102619 +9551 102609 +6345 102608 +22063 102603 +42631 102602 +17399 102593 +3870 102589 +33577 102587 +39723 102543 +12270 102529 +15136 102525 +21122 102522 +38938 102520 +2881 102514 +3370 102507 +48286 102505 +2117 102500 +37178 102500 +15268 102482 +33185 102482 +20976 102474 +30789 102472 +34325 102467 +18781 102456 +21007 102453 +10185 102450 +14197 102440 +19560 102427 +36595 102375 +31214 102367 +4591 102346 +31790 102342 +25720 102333 +33998 102322 +25758 102308 +34955 102296 +22783 102275 +33644 102260 +46614 102258 +30537 102232 +23096 102226 +12109 102226 +23264 102218 +38465 102200 +40911 102195 +21310 102194 +44634 102193 +15476 102191 +23458 102181 +29156 102179 +16225 102168 +28648 102167 +33556 102156 +18177 102139 +7332 102129 +25401 102126 +16737 102106 +22197 102104 +20880 102095 +26395 102091 +27612 102084 +26965 102082 +30966 102073 +14666 102066 +27240 102061 +3920 102059 +34612 102023 +29497 102013 +18947 102013 +34492 102003 +32835 101996 +34806 101986 +23602 101982 +20242 101975 +5473 101972 +17169 101970 +8132 101968 +33208 101965 +17754 101957 +16600 101953 +16857 101939 +24532 101910 +49927 101896 +42177 101891 +34308 101891 +22737 101889 +22729 101888 +26890 101883 +24654 101875 +22359 101875 +1163 101864 +11081 101853 +22943 101845 +20431 101843 +32748 101835 +23967 101825 +15777 101819 +23787 101814 +23189 101797 +21545 101792 +39220 101773 +25582 101761 +8562 101754 +17311 101740 +22982 101738 +29188 101725 +18565 101715 +21435 101714 +40897 101711 +19010 101700 +33670 101699 +14366 101688 +34985 101687 +9734 101682 +14289 101662 +17779 101650 +25854 101637 +13929 101621 +28187 101620 +22187 101605 +14243 101605 +7292 101591 +17557 101589 +25927 101575 +17091 101562 +18593 101542 +30164 101540 +30008 101533 +23508 101508 +19166 101497 +18326 101451 +30979 101436 +24365 101436 +33817 101433 +29618 101426 +23708 101424 +16181 101413 +43025 101391 +3042 101389 +31463 101385 +7367 101375 +35321 101374 +35532 101370 +29468 101349 +14108 101344 +25819 101342 +26850 101315 +6883 101315 +24171 101305 +46198 101304 +37686 101297 +11670 101289 +1920 101275 +10296 101271 +25656 101262 +40730 101257 +12311 101237 +31523 101234 +20720 101219 +25217 101212 +18697 101203 +40186 101194 +14563 101183 +16322 101180 +25712 101177 +11651 101167 +22412 101167 +20825 101160 +46055 101145 +14704 101139 +21567 101129 +14835 101127 +20538 101116 +14330 101095 +20078 101079 +47677 101066 +43480 101065 +19095 101037 +11773 101021 +25162 101015 +22551 101012 +19048 101010 +23204 101006 +30933 100994 +22238 100974 +29504 100967 +21664 100965 +20699 100965 +9331 100964 +16896 100963 +24150 100958 +45726 100946 +16092 100944 +22392 100928 +27545 100921 +25510 100913 +24972 100898 +38880 100887 +26166 100877 +49251 100856 +22706 100856 +24658 100850 +23014 100847 +38507 100845 +5768 100834 +29585 100820 +22978 100773 +18902 100769 +8195 100745 +14839 100734 +3116 100728 +17032 100702 +26122 100699 +41673 100697 +15481 100667 +23034 100663 +20793 100660 +17898 100658 +13542 100658 +22669 100657 +31134 100637 +14230 100636 +16953 100622 +40419 100616 +2838 100609 +10447 100600 +18737 100583 +23018 100572 +32017 100554 +13166 100545 +17485 100536 +28893 100535 +12442 100532 +18608 100530 +14924 100523 +27555 100523 +31183 100517 +27196 100509 +35852 100507 +20192 100505 +4726 100505 +3746 100502 +15702 100496 +22498 100492 +19683 100492 +16006 100456 +39929 100453 +29764 100451 +6223 100439 +21501 100433 +43724 100430 +40458 100410 +19369 100402 +28772 100394 +8303 100394 +29545 100385 +45526 100360 +39781 100356 +17085 100353 +30115 100343 +46965 100335 +9785 100334 +9766 100314 +33324 100280 +16046 100274 +5807 100274 +28528 100268 +21098 100265 +38762 100258 +10161 100256 +39268 100256 +36194 100251 +24607 100247 +29170 100235 +28800 100226 +2991 100180 +28515 100159 +26862 100146 +1316 100144 +23796 100138 +11046 100136 +19337 100127 +12608 100126 +7376 100114 +29732 100107 +17567 100098 +20991 100094 +12636 100093 +24130 100092 +45130 100092 +984 100092 +13851 100071 +33452 100071 +39410 100054 +22259 100046 +9967 100038 +30593 100023 +27548 100018 +10680 100008 +39030 100008 +29384 99993 +17724 99992 +25358 99984 +21080 99981 +3857 99981 +32719 99971 +7649 99960 +32787 99958 +36388 99950 +43708 99940 +28635 99937 +32705 99930 +13254 99925 +30776 99923 +35481 99922 +17234 99912 +15119 99899 +34908 99861 +13805 99861 +33609 99851 +22677 99843 +18276 99837 +26183 99827 +15309 99810 +20263 99799 +41405 99793 +18512 99793 +16839 99779 +13545 99766 +19239 99755 +41800 99754 +17862 99754 +1794 99749 +21647 99737 +39964 99736 +23383 99723 +31908 99720 +25999 99720 +11308 99720 +35956 99695 +5105 99693 +26207 99660 +30783 99655 +24450 99653 +27343 99648 +24842 99644 +24347 99634 +18954 99625 +20820 99621 +23461 99614 +29798 99593 +10279 99589 +25605 99582 +19910 99562 +26775 99561 +8597 99558 +14574 99545 +6269 99520 +22321 99518 +14195 99510 +48148 99507 +32098 99474 +23316 99455 +12340 99450 +26786 99446 +17572 99423 +19401 99415 +45395 99405 +2733 99392 +26670 99378 +31826 99377 +18630 99373 +22505 99372 +37388 99371 +17978 99364 +12366 99357 +18297 99352 +5315 99352 +23618 99350 +21235 99343 +13898 99335 +26345 99331 +16236 99327 +39140 99323 +38772 99314 +28038 99308 +16121 99306 +18975 99302 +23309 99263 +12434 99260 +19096 99255 +29297 99240 +31370 99229 +19318 99226 +12403 99216 +29721 99214 +14251 99213 +19268 99200 +34705 99187 +11999 99183 +13526 99183 +16717 99175 +26103 99152 +21883 99148 +19288 99131 +18953 99126 +20039 99125 +31833 99122 +23245 99121 +22692 99118 +29532 99112 +39810 99072 +21810 99069 +17640 99069 +34707 99067 +30489 99055 +33478 99054 +18229 99034 +28651 99027 +34930 99027 +26530 99022 +26555 99020 +43305 99001 +28444 99001 +38204 98999 +24638 98996 +22526 98983 +14796 98983 +19229 98967 +30399 98959 +18460 98956 +23099 98954 +17543 98953 +14627 98946 +27539 98941 +48023 98930 +27725 98918 +10508 98910 +24166 98907 +30654 98895 +27382 98894 +19321 98893 +16123 98889 +22644 98882 +17430 98881 +19605 98878 +27806 98875 +19123 98875 +23207 98824 +25987 98811 +26048 98805 +35801 98804 +11348 98796 +10944 98793 +10117 98787 +15104 98780 +25724 98770 +47804 98760 +37903 98756 +14299 98745 +22896 98739 +26211 98735 +9649 98716 +19604 98712 +14056 98710 +18370 98651 +35434 98644 +238 98630 +47317 98624 +20843 98621 +32987 98612 +34880 98609 +15387 98604 +7190 98579 +32370 98579 +22977 98578 +20611 98567 +33905 98561 +2036 98556 +31057 98553 +43265 98550 +45870 98549 +7217 98549 +16113 98546 +33997 98528 +11251 98527 +42484 98525 +18336 98522 +26409 98518 +19035 98512 +47251 98487 +15548 98478 +3630 98467 +21258 98454 +23730 98450 +16613 98449 +10793 98428 +26443 98414 +14570 98405 +4402 98405 +32364 98397 +30091 98392 +40944 98386 +12277 98362 +20947 98360 +23148 98355 +21135 98351 +42745 98344 +25185 98339 +7991 98331 +10265 98313 +28901 98292 +4593 98291 +33261 98290 +24644 98283 +13626 98282 +18587 98281 +11502 98276 +31437 98267 +26622 98265 +21764 98242 +45408 98214 +32330 98212 +15639 98198 +16777 98196 +27440 98189 +38278 98169 +24395 98167 +25051 98164 +39588 98158 +31596 98157 +32536 98133 +24558 98122 +12765 98117 +28194 98102 +28437 98098 +24955 98087 +44211 98086 +41196 98079 +37536 98069 +21635 98058 +3628 98053 +48931 98053 +15377 98041 +10771 98034 +28118 98030 +32443 98013 +21923 98008 +23916 98006 +25718 98005 +16687 98002 +16215 98001 +46819 97997 +11404 97997 +3445 97990 +15750 97970 +8623 97967 +28892 97960 +19508 97960 +13231 97943 +42883 97942 +10705 97939 +28061 97919 +29014 97915 +14887 97908 +31974 97903 +6855 97903 +17765 97903 +14155 97900 +16614 97893 +26117 97891 +37154 97883 +30890 97880 +26746 97874 +23335 97858 +10022 97838 +32020 97828 +12678 97827 +45810 97821 +28752 97795 +30996 97783 +31949 97779 +12544 97767 +39143 97757 +25967 97745 +16418 97741 +19064 97735 +30612 97693 +31357 97686 +29529 97682 +11927 97679 +17031 97679 +32597 97675 +29357 97674 +10414 97665 +45051 97663 +34827 97662 +40197 97660 +25017 97655 +13555 97649 +18288 97648 +7619 97621 +14612 97621 +12456 97621 +24723 97618 +31661 97616 +5900 97605 +32183 97601 +18026 97599 +42693 97599 +16531 97596 +26520 97586 +29667 97584 +29596 97581 +19723 97575 +27439 97575 +30757 97553 +31473 97552 +26082 97544 +26328 97542 +20908 97540 +37085 97527 +18121 97515 +33140 97514 +15633 97513 +27396 97512 +19556 97510 +22030 97498 +22888 97492 +11014 97487 +20321 97485 +33685 97481 +28744 97463 +34068 97455 +18539 97452 +15663 97451 +16731 97434 +22147 97423 +23949 97421 +28514 97417 +49173 97414 +28193 97385 +22329 97366 +25659 97354 +41035 97343 +26860 97342 +44369 97324 +39452 97317 +12371 97315 +17300 97313 +40449 97290 +20410 97288 +17145 97282 +22710 97279 +40195 97271 +19552 97265 +4246 97248 +21815 97245 +33515 97229 +31558 97228 +22060 97228 +39802 97223 +16885 97204 +43004 97203 +6701 97196 +28076 97186 +25734 97186 +35807 97182 +20571 97179 +19187 97172 +11986 97157 +11822 97156 +22295 97144 +30236 97137 +30140 97126 +14301 97108 +21085 97100 +22870 97092 +28949 97089 +23327 97080 +14927 97073 +24892 97071 +9078 97070 +25779 97053 +17080 97048 +22181 97048 +24805 97047 +48206 97042 +14014 97036 +17694 97032 +28853 97030 +25223 97019 +21092 97015 +8042 97008 +14898 97007 +34681 97005 +18594 97001 +18834 96999 +46499 96989 +18176 96985 +45212 96983 +19922 96982 +44726 96970 +4126 96968 +34755 96962 +24784 96957 +25428 96943 +48460 96934 +40388 96923 +2825 96911 +21686 96907 +32265 96898 +23301 96894 +11862 96892 +23299 96891 +21124 96879 +5106 96872 +38276 96871 +19144 96865 +26197 96858 +17511 96854 +17531 96848 +16312 96845 +15053 96835 +16716 96830 +8421 96817 +18543 96812 +18118 96805 +32615 96803 +18411 96789 +3076 96780 +20888 96775 +19459 96769 +6018 96764 +24495 96759 +22051 96753 +26299 96752 +4604 96742 +23071 96741 +1106 96729 +34327 96727 +38619 96710 +38740 96708 +43237 96702 +46097 96692 +35234 96689 +19468 96678 +8923 96678 +30785 96671 +33750 96670 +24514 96666 +17207 96649 +20420 96646 +19433 96645 +19884 96635 +24350 96635 +6661 96624 +23165 96608 +20254 96601 +33094 96592 +22979 96591 +22623 96590 +23160 96579 +13522 96578 +9203 96556 +24048 96555 +9971 96528 +10780 96521 +34701 96520 +25863 96507 +36231 96504 +24942 96501 +34060 96479 +20040 96463 +14392 96453 +29015 96451 +9696 96441 +30257 96436 +31784 96430 +13543 96429 +19267 96422 +47967 96420 +13826 96420 +36889 96417 +4798 96406 +34440 96405 +25483 96376 +19133 96356 +34780 96356 +41348 96344 +33718 96343 +10378 96333 +17486 96328 +13309 96326 +9192 96311 +21754 96301 +29136 96290 +27278 96290 +26605 96273 +31269 96262 +26843 96260 +29815 96243 +4862 96231 +21219 96219 +20787 96213 +2856 96205 +33130 96196 +28445 96191 +22604 96180 +27340 96178 +15544 96168 +25115 96151 +19165 96145 +10007 96142 +28036 96135 +20408 96116 +2725 96115 +22989 96083 +19143 96079 +27264 96072 +26077 96072 +9632 96053 +4063 96036 +24089 96036 +29239 96032 +23273 96026 +14661 96021 +35415 96012 +3043 96011 +27567 96010 +39447 96010 +9069 95993 +31171 95988 +27353 95985 +32137 95980 +40695 95952 +20750 95941 +40222 95937 +21100 95922 +21558 95913 +32410 95910 +27177 95908 +20056 95890 +31220 95888 +13905 95883 +35274 95871 +22668 95857 +18275 95855 +26594 95835 +25786 95822 +14564 95821 +27308 95810 +24281 95805 +19897 95803 +32490 95796 +14948 95762 +2017 95741 +19713 95740 +24387 95733 +25075 95724 +22474 95724 +18831 95719 +34886 95717 +27425 95714 +13315 95698 +28492 95692 +48395 95686 +30652 95683 +30651 95671 +39361 95668 +41114 95663 +35121 95658 +23867 95651 +31981 95650 +2021 95649 +21662 95645 +16632 95638 +25377 95637 +21461 95633 +28120 95633 +25646 95620 +16588 95617 +36799 95615 +9883 95607 +20925 95592 +18511 95588 +44168 95587 +39672 95576 +21164 95565 +18027 95562 +33972 95558 +31637 95548 +22365 95536 +35336 95531 +8825 95523 +22339 95522 +7608 95521 +24947 95515 +14411 95506 +1307 95505 +16989 95486 +18616 95482 +33902 95473 +25409 95465 +29488 95452 +21552 95448 +7474 95436 +32605 95425 +33147 95421 +15304 95409 +39124 95402 +30970 95402 +33859 95401 +21851 95396 +12589 95395 +25507 95358 +24262 95358 +24014 95356 +46366 95351 +4107 95324 +11491 95315 +17747 95299 +22582 95286 +33405 95282 +37066 95272 +38131 95270 +16264 95267 +16649 95265 +25729 95254 +15174 95230 +16509 95221 +23879 95218 +41688 95215 +29658 95204 +8356 95193 +17917 95190 +33352 95171 +31735 95167 +35983 95166 +8812 95147 +12196 95147 +10837 95146 +9873 95136 +20935 95134 +7469 95114 +14751 95109 +21984 95097 +19018 95075 +32714 95065 +29809 95057 +33777 95057 +25376 95057 +32678 95048 +17182 95037 +46281 95032 +39245 95014 +25303 95012 +37487 94994 +42748 94986 +31341 94966 +19923 94954 +33356 94942 +16917 94935 +1704 94933 +20299 94931 +29782 94925 +24066 94922 +20664 94915 +17973 94910 +10516 94908 +19701 94899 +9395 94892 +9346 94887 +36715 94880 +22912 94880 +21318 94874 +10788 94873 +31656 94867 +21223 94849 +34962 94832 +34462 94830 +12233 94826 +13548 94793 +36636 94775 +12202 94762 +26784 94758 +6421 94752 +27731 94750 +34710 94739 +32540 94736 +3057 94727 +11024 94710 +31565 94694 +49281 94689 +19001 94683 +31678 94677 +6545 94676 +35687 94673 +26316 94633 +21134 94630 +34050 94630 +18023 94626 +44572 94623 +44781 94623 +8743 94596 +7807 94590 +5633 94589 +13587 94580 +36978 94574 +23122 94574 +12341 94564 +36002 94562 +18142 94560 +47678 94555 +13018 94537 +27356 94533 +9937 94525 +682 94499 +21640 94495 +16243 94465 +36503 94462 +47660 94450 +33503 94441 +28303 94435 +4749 94429 +32831 94420 +8530 94420 +26487 94418 +18469 94417 +36251 94414 +29125 94408 +14091 94399 +25494 94399 +25455 94377 +14331 94376 +12248 94351 +6750 94335 +18808 94333 +21490 94320 +22733 94314 +11696 94312 +23149 94305 +17241 94299 +22185 94288 +28814 94273 +26502 94273 +14272 94272 +14700 94259 +24768 94253 +13481 94244 +6014 94234 +30279 94231 +29686 94225 +6703 94216 +17306 94216 +3323 94178 +14515 94178 +46820 94167 +3005 94160 +18364 94150 +29662 94130 +29208 94129 +10036 94128 +26253 94128 +28294 94121 +39728 94102 +18480 94091 +27821 94088 +18420 94085 +9855 94077 +41260 94067 +6897 94063 +32951 94062 +10444 94043 +13180 94043 +16619 94037 +26113 94031 +2064 94029 +16308 94028 +37401 94028 +28094 94019 +39125 94017 +27373 94015 +11775 94014 +22378 94008 +16741 94001 +16630 93994 +33984 93982 +22609 93963 +18948 93957 +13282 93949 +44323 93934 +13464 93932 +12354 93930 +14362 93923 +20328 93920 +25234 93919 +33527 93918 +15889 93906 +21021 93906 +43870 93903 +26310 93900 +12488 93900 +11855 93898 +39039 93895 +16374 93866 +26002 93859 +16744 93856 +14856 93839 +14613 93838 +17500 93826 +15323 93817 +21366 93803 +22626 93795 +15887 93795 +31704 93784 +22529 93784 +44317 93779 +40057 93769 +19985 93767 +801 93764 +27839 93746 +25263 93723 +32253 93722 +23905 93721 +13443 93719 +25281 93709 +24623 93701 +36421 93695 +22007 93687 +46306 93686 +32853 93681 +25170 93666 +15209 93656 +24677 93641 +17575 93638 +11331 93636 +1172 93625 +38188 93622 +31099 93611 +40445 93606 +27321 93601 +24230 93597 +13199 93586 +18320 93570 +18025 93568 +28996 93564 +37033 93563 +15784 93561 +21639 93553 +30828 93548 +30618 93537 +43245 93534 +31201 93532 +19024 93527 +30456 93527 +2905 93518 +8007 93500 +20265 93497 +22125 93482 +1733 93482 +39508 93474 +19844 93473 +21127 93472 +43513 93459 +35689 93452 +37907 93450 +1562 93429 +5152 93413 +33519 93375 +12026 93372 +22085 93372 +19493 93369 +6713 93365 +19504 93352 +24683 93347 +27297 93340 +34906 93338 +33458 93338 +24006 93338 +20066 93336 +23689 93335 +30286 93335 +28499 93334 +13716 93325 +17873 93324 +44502 93309 +4370 93305 +31136 93289 +10981 93284 +38881 93278 +18548 93272 +10295 93266 +50048 93265 +24391 93248 +4489 93237 +11248 93237 +17852 93233 +11902 93222 +6919 93198 +42100 93192 +26102 93178 +20680 93163 +46719 93161 +26581 93156 +20054 93155 +26906 93147 +22385 93113 +8525 93108 +23746 93103 +10212 93102 +30839 93100 +23205 93097 +19294 93086 +24918 93085 +32938 93085 +20640 93076 +667 93075 +3004 93074 +36384 93063 +15255 93058 +30600 93056 +18272 93046 +20588 93041 +29483 93036 +2309 93024 +5132 93015 +40318 92991 +8931 92991 +25620 92989 +44075 92987 +8764 92985 +25305 92983 +25277 92980 +14879 92973 +29986 92951 +16876 92946 +44041 92944 +27571 92918 +10740 92916 +7918 92914 +36516 92907 +26548 92906 +27060 92905 +9532 92905 +23095 92897 +37490 92887 +35493 92884 +14686 92881 +34593 92863 +23057 92846 +46917 92840 +20042 92840 +29070 92838 +7633 92830 +12849 92825 +1490 92822 +45303 92813 +25860 92800 +36832 92793 +26916 92791 +22019 92786 +35591 92785 +31065 92776 +3927 92775 +41164 92760 +10459 92752 +23477 92737 +26057 92730 +29210 92720 +10973 92712 +5232 92711 +24359 92705 +16766 92705 +36812 92702 +17347 92685 +32413 92675 +19201 92674 +11887 92664 +35017 92664 +28646 92661 +10418 92641 +49762 92640 +27197 92636 +19882 92616 +23453 92613 +48607 92600 +19175 92599 +26727 92597 +5102 92590 +25629 92577 +252 92571 +5028 92570 +19845 92565 +6436 92559 +36611 92554 +40021 92550 +23968 92549 +13909 92544 +25339 92539 +11980 92536 +13227 92534 +28170 92528 +26645 92527 +42499 92522 +18043 92508 +16729 92503 +12106 92502 +22299 92483 +25937 92475 +31233 92465 +15568 92444 +17090 92443 +21051 92440 +28591 92440 +6171 92411 +26655 92397 +24584 92386 +14314 92384 +19297 92377 +12896 92376 +24433 92374 +36771 92372 +12862 92368 +21452 92367 +46459 92358 +30870 92352 +7352 92337 +7061 92328 +24325 92322 +22188 92321 +38838 92316 +16446 92309 +23557 92299 +18643 92298 +10916 92283 +35078 92260 +7322 92256 +27822 92251 +24656 92247 +18463 92237 +39661 92227 +9064 92217 +20278 92214 +9496 92212 +34881 92210 +23074 92204 +25449 92191 +19883 92186 +24219 92184 +12394 92182 +3652 92170 +6683 92164 +42630 92161 +22574 92161 +26610 92156 +22106 92156 +18059 92142 +29654 92139 +22206 92133 +30325 92133 +25317 92121 +39309 92115 +10527 92106 +28393 92100 +23056 92099 +42150 92097 +21216 92095 +26800 92087 +17150 92081 +38478 92078 +20399 92072 +8048 92063 +27138 92063 +33892 92057 +7642 92046 +43326 92046 +14573 92043 +29786 92038 +11280 92028 +27662 92026 +35646 92025 +19102 92023 +7928 92018 +41345 92013 +19440 91973 +1093 91965 +7836 91956 +34405 91939 +23239 91929 +35497 91928 +6424 91916 +23217 91909 +19838 91909 +42350 91901 +26334 91895 +35816 91894 +20269 91892 +30718 91879 +37969 91869 +27397 91860 +34077 91858 +14151 91858 +9501 91855 +22423 91839 +11519 91834 +30462 91831 +14068 91822 +24402 91813 +10865 91811 +40132 91800 +48858 91799 +18221 91797 +12417 91795 +19862 91786 +46381 91781 +46260 91774 +6456 91765 +33318 91757 +25116 91754 +5200 91737 +21294 91735 +36904 91733 +20067 91721 +16288 91712 +16170 91709 +36574 91700 +30678 91699 +30437 91688 +20326 91683 +21814 91672 +45963 91662 +32659 91636 +41845 91629 +26938 91628 +20383 91621 +27957 91620 +28662 91619 +32979 91615 +30570 91607 +32002 91591 +26565 91588 +9827 91567 +34915 91566 +30012 91562 +34686 91552 +23084 91543 +17555 91541 +45229 91527 +34696 91525 +21256 91522 +4682 91516 +39303 91511 +34422 91500 +13958 91496 +22219 91493 +46422 91471 +32526 91470 +47775 91463 +6040 91460 +17909 91457 +13216 91449 +12796 91444 +27907 91443 +25173 91430 +42286 91423 +20693 91419 +16866 91415 +22765 91411 +11498 91403 +8677 91396 +8394 91383 +22730 91375 +18783 91374 +40474 91368 +29753 91348 +22889 91344 +24337 91343 +41525 91340 +22949 91334 +31890 91331 +23699 91322 +25971 91308 +26777 91308 +11291 91307 +7368 91305 +10798 91299 +18478 91272 +16641 91269 +12609 91244 +28983 91230 +15824 91227 +12562 91224 +12060 91223 +16369 91222 +23353 91220 +28411 91218 +21661 91211 +24569 91210 +20009 91210 +45371 91202 +25861 91202 +34474 91200 +25984 91189 +21879 91170 +10910 91168 +13834 91159 +19343 91156 +36588 91150 +20037 91146 +33749 91143 +22625 91131 +2509 91128 +34732 91098 +30880 91097 +25725 91070 +14821 91063 +19547 91062 +21674 91042 +31158 91034 +10028 91033 +30043 91027 +14426 91025 +17219 91024 +30046 91023 +25403 91020 +32342 91016 +39265 91015 +43664 91003 +27840 91001 +29359 90985 +19387 90978 +10505 90960 +21621 90959 +33242 90945 +13946 90944 +21915 90942 +24635 90941 +16132 90938 +9702 90936 +29489 90927 +3468 90926 +21649 90913 +20958 90900 +25923 90899 +15992 90897 +35633 90887 +6726 90881 +21971 90869 +29628 90833 +14731 90827 +44025 90826 +26695 90806 +20460 90802 +42185 90800 +35165 90794 +46561 90793 +20744 90788 +29475 90778 +8375 90769 +20905 90760 +11224 90757 +14805 90754 +20445 90749 +23401 90746 +25355 90745 +25154 90744 +11163 90741 +27523 90734 +37387 90731 +40412 90730 +3534 90717 +41346 90716 +25349 90716 +23348 90716 +20060 90705 +6571 90703 +31367 90700 +30926 90696 +24950 90694 +24346 90689 +24489 90677 +36029 90667 +12025 90667 +13058 90666 +9191 90664 +34051 90653 +16809 90649 +19322 90646 +21392 90646 +14701 90631 +17851 90631 +40644 90629 +11146 90629 +32632 90625 +12001 90620 +14874 90605 +47966 90604 +26593 90566 +24484 90565 +40288 90564 +21682 90555 +35671 90544 +39927 90542 +13569 90541 +32832 90540 +28661 90540 +6818 90494 +14843 90492 +28157 90474 +46832 90471 +22776 90466 +11814 90459 +34676 90459 +27325 90455 +35564 90433 +36985 90432 +19446 90431 +17246 90424 +14005 90400 +22332 90391 +12867 90389 +15919 90389 +28534 90385 +3054 90384 +27123 90380 +6179 90369 +15027 90348 +37874 90337 +17512 90325 +19221 90317 +26187 90309 +42284 90307 +12050 90272 +29741 90260 +13992 90260 +44340 90258 +9123 90249 +31429 90243 +9094 90233 +6351 90232 +23167 90231 +31355 90228 +21486 90212 +7659 90210 +21438 90195 +38428 90192 +30231 90185 +35874 90179 +42873 90167 +23403 90166 +19704 90164 +29570 90155 +4808 90144 +28464 90134 +30307 90129 +16603 90115 +29060 90084 +38774 90082 +7230 90081 +27075 90071 +36467 90067 +21429 90066 +20128 90058 +29459 90052 +12782 90047 +30153 90043 +25930 90039 +17738 90032 +27449 90015 +22054 90013 +21611 90009 +30305 90007 +28088 90001 +24497 89973 +18485 89969 +10080 89960 +20618 89956 +15197 89950 +33286 89941 +13083 89940 +15492 89935 +6237 89910 +99 89908 +25331 89905 +27521 89900 +30700 89889 +21094 89876 +30706 89862 +32486 89832 +34584 89809 +24903 89790 +19528 89789 +11912 89787 +24375 89784 +17102 89776 +44452 89773 +19503 89772 +41900 89768 +35896 89762 +16900 89760 +18972 89751 +42052 89745 +1887 89743 +26924 89735 +19185 89734 +20795 89727 +6024 89723 +18952 89721 +43791 89710 +34766 89704 +34138 89697 +38018 89666 +37619 89649 +10235 89647 +33072 89646 +27529 89644 +4464 89642 +28558 89637 +34017 89631 +16405 89630 +17672 89628 +37123 89621 +35047 89619 +29362 89618 +34895 89610 +32982 89591 +14562 89589 +28488 89587 +22756 89582 +37584 89569 +18959 89566 +24599 89549 +29118 89544 +42208 89537 +11567 89483 +28984 89465 +31231 89449 +22129 89445 +39739 89445 +26740 89444 +24303 89429 +36025 89413 +6442 89413 +20038 89409 +42682 89400 +25794 89392 +25056 89364 +19813 89364 +10316 89363 +22020 89356 +26762 89352 +19893 89352 +23503 89344 +11489 89341 +26296 89337 +1217 89335 +19762 89326 +18596 89322 +26387 89322 +26871 89320 +4500 89317 +10977 89305 +40378 89302 +14507 89298 +27259 89297 +9347 89282 +37690 89276 +37090 89264 +27602 89254 +20435 89238 +9744 89232 +33190 89219 +18814 89214 +33886 89204 +18529 89202 +17047 89198 +15176 89195 +15354 89191 +20050 89188 +22554 89182 +20767 89175 +36633 89171 +25107 89167 +28823 89141 +19555 89120 +12599 89115 +37217 89112 +22331 89107 +20687 89097 +36069 89093 +34601 89090 +15735 89076 +20950 89069 +43174 89068 +12130 89067 +31402 89064 +20982 89060 +34281 89046 +44467 89038 +16772 89035 +35141 89033 +14062 89029 +28176 89026 +16186 89018 +13269 89016 +41826 89014 +30883 89010 +23987 89009 +36429 88992 +16439 88987 +10211 88981 +27508 88970 +37940 88969 +28999 88958 +26772 88951 +29816 88945 +38617 88939 +26311 88925 +20923 88924 +39113 88915 +41782 88906 +17669 88900 +24632 88869 +23221 88864 +32414 88862 +27015 88857 +9760 88848 +17000 88841 +28224 88835 +26527 88835 +31920 88827 +8051 88783 +37519 88779 +15180 88745 +26930 88742 +12543 88731 +29383 88705 +26553 88695 +33014 88691 +48812 88689 +19962 88667 +41904 88658 +40974 88658 +27980 88656 +41703 88655 +1179 88655 +48822 88632 +23027 88622 +41005 88616 +10137 88611 +36971 88610 +39045 88607 +43126 88606 +20625 88601 +35692 88599 +13248 88586 +49003 88585 +30848 88577 +38317 88556 +20339 88552 +17682 88545 +31683 88530 +18586 88516 +25407 88513 +36590 88511 +20673 88510 +13217 88506 +24374 88506 +23749 88504 +15212 88500 +17284 88500 +20313 88499 +32241 88490 +17221 88486 +13809 88465 +19434 88465 +38522 88456 +32438 88443 +21431 88423 +31213 88420 +32174 88415 +24685 88383 +15355 88376 +18581 88355 +13380 88347 +12391 88343 +23934 88340 +23714 88328 +16221 88324 +10521 88319 +27624 88319 +28803 88308 +41905 88301 +12204 88288 +25588 88280 +15056 88278 +30722 88274 +21787 88272 +29043 88266 +22987 88259 +49183 88255 +13617 88253 +26816 88252 +7070 88251 +21334 88247 +18628 88212 +3133 88198 +24949 88197 +21082 88190 +4582 88188 +38758 88184 +19585 88179 +44535 88175 +35982 88163 +17480 88160 +20145 88159 +36245 88154 +15165 88151 +115 88146 +2016 88119 +34317 88106 +19186 88084 +7755 88080 +16859 88067 +18631 88056 +15420 88053 +2369 88051 +32601 88050 +27095 88048 +22837 88047 +17707 88046 +26149 88041 +30920 88040 +16064 88036 +9459 88029 +37332 88028 +18992 88026 +10379 88024 +17509 88021 +11203 88021 +16957 88014 +18285 88002 +27146 87994 +32176 87987 +23050 87981 +15801 87967 +39540 87967 +13652 87966 +25208 87947 +42605 87943 +33613 87937 +48955 87926 +11933 87924 +33375 87916 +29505 87903 +31893 87902 +27341 87889 +13848 87889 +16513 87886 +21546 87885 +22486 87883 +26898 87878 +34380 87877 +42065 87875 +28336 87874 +34136 87870 +36718 87867 +22875 87867 +23083 87850 +40766 87848 +23332 87846 +28180 87823 +19020 87823 +37685 87821 +21387 87810 +41137 87797 +20088 87795 +28431 87790 +46287 87790 +19710 87785 +4125 87779 +14356 87778 +3600 87767 +48125 87766 +17650 87762 +28775 87741 +33945 87736 +26973 87730 +30441 87726 +41578 87725 +12461 87723 +25359 87712 +45961 87709 +31618 87691 +44303 87688 +23456 87686 +9538 87672 +40782 87672 +9630 87667 +16790 87642 +14309 87637 +25290 87637 +4072 87629 +26944 87622 +18063 87615 +17299 87603 +21131 87601 +25891 87597 +49657 87597 +34916 87591 +14642 87588 +25845 87571 +38330 87569 +40833 87560 +20137 87551 +15906 87546 +22459 87541 +44824 87541 +45034 87527 +22513 87527 +6676 87523 +35917 87515 +29727 87509 +32099 87508 +5352 87502 +20439 87483 +17059 87474 +22350 87468 +39727 87467 +33742 87456 +29832 87455 +25168 87454 +24160 87451 +32919 87428 +20391 87422 +4029 87420 +23386 87417 +35062 87416 +27699 87413 +15601 87411 +11922 87411 +20466 87406 +14025 87405 +19072 87404 +30784 87397 +1113 87395 +19686 87389 +26671 87386 +8455 87381 +38442 87381 +36443 87363 +20667 87362 +19407 87355 +13483 87353 +41742 87350 +116 87319 +30174 87314 +29195 87309 +40214 87303 +7065 87302 +37756 87299 +27144 87283 +13251 87280 +32733 87276 +20080 87269 +12915 87263 +19979 87254 +25571 87253 +40735 87247 +5216 87243 +30925 87241 +23972 87238 +11979 87218 +24092 87199 +22570 87199 +32552 87193 +42644 87190 +29775 87186 +21929 87183 +33218 87179 +24749 87172 +19098 87166 +21156 87163 +21784 87162 +13277 87156 +23996 87153 +19766 87145 +21226 87139 +28160 87137 +31926 87128 +19907 87119 +30512 87116 +45446 87114 +4629 87109 +23427 87108 +18637 87107 +19878 87105 +8736 87087 +46552 87080 +16345 87068 +25351 87060 +4355 87050 +23505 87043 +41182 87032 +4124 87020 +22338 87016 +19812 87009 +21115 87005 +48855 87005 +31273 87005 +23422 87003 +19390 86988 +27768 86982 +21698 86976 +23116 86962 +20007 86959 +25566 86954 +21385 86952 +19536 86949 +8792 86943 +17968 86920 +8174 86919 +38536 86905 +6063 86901 +20703 86898 +36810 86894 +28126 86890 +16789 86885 +21853 86881 +30188 86871 +39651 86864 +33521 86862 +18182 86794 +10486 86793 +14454 86792 +44311 86779 +22363 86773 +21367 86765 +6944 86763 +6352 86755 +7155 86746 +40902 86745 +22000 86734 +33659 86734 +26157 86731 +17216 86730 +39734 86729 +18620 86726 +25273 86720 +5596 86708 +11743 86698 +29023 86698 +7059 86696 +41078 86692 +47249 86684 +21732 86675 +34899 86673 +28020 86666 +14319 86663 +38403 86650 +40208 86648 +34767 86648 +15974 86641 +16250 86637 +34662 86634 +47089 86629 +38928 86617 +30941 86595 +3374 86590 +21178 86585 +17546 86574 +11074 86569 +22287 86567 +35979 86561 +5681 86552 +9200 86549 +34914 86544 +20256 86537 +20149 86534 +20064 86533 +19506 86528 +38433 86524 +18437 86523 +31239 86522 +17710 86521 +6587 86521 +22034 86519 +35105 86513 +20411 86510 +29025 86498 +10517 86485 +20479 86485 +46950 86480 +14453 86465 +24381 86458 +12913 86443 +16285 86440 +43248 86430 +16810 86427 +31863 86426 +27307 86426 +37956 86416 +19863 86405 +31742 86403 +24211 86401 +19733 86398 +33164 86396 +21854 86393 +11267 86390 +18102 86389 +29413 86385 +24621 86382 +42700 86378 +14848 86378 +2287 86373 +12466 86365 +22999 86360 +23665 86352 +22254 86342 +6039 86338 +50158 86337 +35805 86333 +35593 86330 +21882 86328 +42168 86324 +24406 86311 +15688 86305 +16227 86302 +40669 86301 +9700 86300 +29317 86295 +40233 86280 +31865 86275 +10466 86256 +17200 86255 +19607 86252 +14513 86248 +24659 86243 +19861 86240 +6658 86221 +48297 86217 +3532 86216 +14981 86192 +18710 86180 +24775 86178 +32456 86177 +15111 86175 +35680 86154 +39995 86152 +4565 86145 +30006 86119 +19342 86114 +22010 86111 +47594 86102 +18634 86096 +46187 86094 +33337 86078 +41145 86076 +23171 86067 +23992 86054 +32405 86043 +23978 86038 +23152 86035 +42602 86035 +13991 86026 +13360 86023 +44481 86020 +43586 85995 +30985 85990 +21779 85988 +36647 85984 +32145 85984 +19091 85979 +20723 85970 +10819 85966 +23173 85955 +25124 85952 +12423 85949 +18851 85940 +33664 85931 +16583 85923 +20063 85918 +15029 85918 +41839 85915 +41960 85914 +24998 85901 +17697 85901 +27355 85892 +6313 85891 +44749 85880 +17232 85876 +33514 85867 +16453 85860 +10407 85858 +14350 85856 +22849 85835 +38665 85818 +27973 85810 +28077 85807 +19341 85804 +34682 85799 +44170 85798 +20592 85797 +21079 85791 +47538 85790 +48848 85782 +24404 85771 +37003 85769 +12363 85756 +49315 85752 +32619 85743 +11201 85714 +37006 85709 +6794 85702 +19698 85702 +21391 85696 +39282 85682 +32175 85666 +21375 85666 +13152 85656 +24435 85645 +27618 85634 +17065 85633 +18659 85631 +26374 85628 +20514 85605 +20823 85599 +23077 85598 +27213 85593 +22289 85585 +44359 85571 +25909 85565 +24187 85554 +16638 85537 +1713 85534 +19409 85527 +30496 85519 +24352 85508 +26428 85501 +13259 85501 +17157 85501 +33814 85500 +40777 85492 +21214 85486 +17520 85481 +32716 85466 +22139 85463 +47574 85451 +1300 85447 +22072 85440 +12753 85438 +20584 85429 +43784 85427 +31225 85421 +13920 85414 +26312 85404 +17242 85403 +11438 85403 +14936 85397 +23202 85390 +46185 85375 +33590 85372 +15711 85369 +34041 85365 +33579 85355 +16102 85354 +17545 85354 +42766 85336 +29679 85326 +10223 85325 +38552 85319 +20558 85307 +17395 85297 +39086 85295 +44298 85291 +2360 85279 +42086 85229 +38343 85228 +29949 85222 +32961 85219 +29445 85218 +15587 85208 +19208 85208 +28213 85206 +28099 85203 +41505 85202 +24205 85202 +43596 85194 +15780 85194 +17438 85184 +20416 85168 +13631 85165 +27437 85162 +35663 85144 +39017 85136 +22714 85100 +36976 85098 +18013 85096 +36591 85091 +28438 85079 +24097 85067 +35386 85055 +7877 85048 +31483 85027 +24451 85024 +43002 85016 +23701 85016 +33347 84993 +41200 84983 +18927 84979 +20734 84970 +22959 84955 +18689 84945 +18723 84939 +21822 84921 +31840 84912 +26262 84907 +34233 84900 +38118 84899 +34118 84893 +4954 84883 +30845 84882 +28049 84881 +25044 84875 +36646 84852 +28943 84843 +42502 84843 +15410 84842 +31088 84829 +10475 84828 +26472 84822 +22546 84798 +25243 84792 +20428 84780 +41438 84769 +20089 84735 +20708 84727 +36229 84724 +47430 84724 +28595 84720 +32280 84715 +49220 84709 +10885 84706 +23757 84702 +6169 84700 +41406 84691 +19078 84686 +24694 84679 +26378 84668 +24636 84664 +45265 84637 +2517 84635 +32314 84617 +16635 84612 +41695 84611 +22562 84610 +19856 84609 +18440 84601 +11222 84597 +14529 84591 +20770 84589 +32260 84583 +20132 84581 +42037 84578 +16189 84555 +27876 84552 +20122 84548 +35181 84541 +44113 84540 +19921 84539 +24751 84537 +13144 84535 +23373 84533 +27304 84530 +19565 84526 +8040 84520 +29324 84519 +48880 84511 +35397 84505 +34545 84501 +15611 84497 +38701 84493 +20396 84487 +5029 84481 +14866 84476 +35766 84462 +38623 84462 +16086 84456 +25569 84448 +42649 84440 +33195 84424 +10104 84420 +13713 84404 +24795 84401 +35034 84400 +39503 84396 +32022 84377 +14705 84368 +40993 84365 +19361 84360 +23436 84357 +14584 84352 +25027 84340 +49722 84326 +31067 84323 +40679 84318 +24592 84293 +30263 84286 +33881 84282 +29612 84277 +31812 84264 +26643 84257 +2600 84257 +759 84240 +36418 84236 +13949 84232 +26889 84227 +22712 84221 +15982 84216 +23399 84215 +114 84202 +10734 84196 +43575 84156 +24273 84156 +16812 84155 +31853 84150 +19908 84144 +23799 84141 +19419 84118 +31252 84118 +17598 84115 +35727 84094 +45577 84092 +29144 84089 +40626 84080 +16564 84076 +2204 84075 +32116 84069 +8444 84067 +20559 84066 +5651 84061 +23860 84058 +28064 84049 +28584 84041 +27647 84029 +14557 84024 +16104 84020 +39617 84009 +24167 83997 +3193 83990 +47995 83975 +18606 83975 +11718 83972 +37919 83962 +23218 83959 +42823 83952 +29395 83950 +21128 83946 +17892 83945 +12335 83944 +35329 83941 +45093 83930 +25711 83925 +34113 83915 +16498 83910 +9278 83906 +3093 83901 +24276 83874 +25430 83867 +32924 83857 +17519 83857 +31133 83847 +16082 83843 +34235 83842 +22430 83805 +26403 83802 +17513 83793 +22451 83789 +35464 83787 +37492 83774 +35242 83765 +46335 83757 +27645 83757 +13681 83746 +29314 83746 +28890 83724 +27810 83719 +28486 83717 +35673 83715 +4078 83714 +29777 83710 +41721 83708 +27322 83705 +35338 83703 +36694 83698 +30029 83690 +30090 83688 +39830 83687 +44145 83684 +38115 83678 +7158 83677 +23985 83676 +5634 83673 +7282 83661 +27940 83657 +35670 83649 +40599 83648 +8298 83623 +34005 83610 +20130 83610 +42525 83606 +28454 83553 +43192 83552 +44603 83550 +28910 83539 +20648 83524 +18876 83523 +26356 83521 +12146 83507 +27671 83493 +49484 83488 +8089 83484 +14514 83480 +26720 83479 +30750 83476 +45814 83473 +47989 83463 +21043 83449 +42178 83443 +18388 83431 +4741 83420 +29502 83417 +26925 83403 +18964 83394 +26140 83383 +13158 83378 +39278 83375 +47534 83374 +21415 83366 +18261 83361 +26131 83360 +17048 83359 +24655 83357 +27335 83356 +23851 83353 +13585 83348 +28136 83345 +26083 83343 +21601 83343 +19621 83339 +44024 83333 +1572 83327 +7814 83325 +27251 83321 +34852 83305 +22904 83303 +21715 83295 +18669 83293 +30927 83293 +3441 83288 +24401 83274 +25635 83268 +12349 83254 +17583 83254 +21804 83252 +21726 83251 +23734 83237 +10854 83231 +39751 83228 +19541 83222 +18290 83213 +29104 83211 +43761 83209 +25406 83202 +32189 83184 +23271 83184 +43772 83146 +19499 83145 +4147 83143 +32557 83142 +7140 83136 +8291 83130 +45646 83127 +21182 83112 +25246 83099 +27069 83097 +27170 83096 +21752 83090 +8656 83090 +32620 83075 +8827 83071 +9636 83067 +18227 83050 +23000 83046 +46508 83043 +37242 83042 +16794 83011 +36385 83008 +12881 83001 +32889 82994 +37529 82991 +13014 82987 +17211 82985 +18018 82981 +4677 82980 +9650 82977 +5683 82975 +34927 82961 +25300 82960 +27141 82953 +10019 82944 +49001 82942 +8843 82933 +7371 82932 +8321 82922 +15802 82912 +28300 82907 +18506 82899 +11630 82889 +23648 82887 +37427 82882 +20610 82882 +32365 82876 +9624 82853 +46518 82849 +15473 82831 +11351 82821 +35819 82808 +13469 82804 +42379 82793 +30595 82792 +27347 82787 +16628 82779 +39542 82771 +23485 82756 +28405 82747 +39634 82742 +39151 82740 +3509 82734 +25727 82733 +22665 82730 +15950 82729 +6214 82724 +41835 82718 +8896 82715 +37866 82710 +9999 82701 +25329 82696 +11316 82687 +6192 82680 +25567 82674 +11241 82673 +19601 82670 +18346 82669 +2186 82665 +13740 82653 +20951 82649 +1469 82635 +32267 82625 +27103 82610 +35806 82602 +25023 82588 +30198 82585 +3906 82560 +17202 82555 +13719 82553 +40209 82550 +19408 82538 +20385 82535 +21514 82521 +12254 82513 +44291 82500 +28536 82492 +17082 82488 +9275 82469 +17421 82467 +3075 82450 +45518 82444 +29669 82442 +34968 82425 +33909 82424 +23690 82423 +20227 82418 +21036 82416 +7574 82416 +3253 82415 +15237 82415 +11034 82411 +25764 82394 +15034 82390 +24282 82387 +36546 82373 +35160 82370 +23710 82362 +18598 82356 +47297 82355 +48685 82349 +15353 82346 +20862 82346 +13219 82345 +22426 82344 +27067 82335 +11166 82333 +10585 82314 +27884 82313 +19162 82312 +28804 82309 +29256 82305 +12972 82300 +8802 82297 +33306 82287 +33404 82285 +23390 82282 +24619 82276 +24302 82269 +15539 82264 +29661 82247 +30701 82247 +47618 82247 +21078 82241 +24408 82240 +23677 82232 +20850 82229 +15447 82226 +19472 82220 +14413 82219 +20309 82196 +46879 82190 +41463 82189 +16143 82184 +16196 82183 +30972 82180 +1993 82173 +31440 82162 +41668 82157 +34238 82149 +10561 82128 +43412 82123 +15636 82117 +28146 82117 +11125 82115 +34015 82104 +15831 82104 +23432 82102 +42730 82088 +30047 82085 +23929 82079 +16586 82073 +18660 82067 +18759 82066 +41459 82064 +15292 82051 +18830 82046 +25046 82045 +7233 82042 +38990 82038 +13582 82032 +9226 81980 +21247 81977 +27376 81961 +25227 81957 +42342 81947 +24902 81940 +32887 81940 +36143 81930 +15219 81922 +919 81920 +24566 81914 +20870 81910 +20140 81909 +29586 81901 +22216 81900 +45103 81893 +19486 81878 +21983 81874 +32289 81870 +30809 81868 +37053 81866 +19399 81850 +15983 81845 +22563 81842 +23572 81842 +8139 81836 +9128 81835 +46397 81830 +20293 81801 +19586 81783 +23007 81782 +29961 81764 +43287 81756 +27929 81752 +29543 81746 +23575 81736 +13022 81724 +21761 81723 +13655 81719 +11045 81715 +12142 81710 +42429 81706 +23616 81703 +29103 81702 +7789 81700 +14146 81693 +32660 81672 +30037 81671 +29742 81662 +20121 81661 +19370 81638 +26706 81626 +14588 81618 +30319 81618 +50041 81613 +29707 81611 +19190 81607 +40751 81605 +47189 81603 +28318 81591 +23715 81585 +27638 81584 +33837 81583 +17201 81577 +23038 81572 +31787 81556 +16122 81552 +50083 81547 +29337 81542 +40365 81531 +35271 81528 +45202 81525 +20232 81502 +47630 81502 +18810 81484 +19643 81480 +23091 81478 +5444 81477 +20697 81477 +25126 81475 +22144 81474 +769 81474 +28269 81463 +39043 81461 +20887 81459 +23161 81458 +23026 81451 +39926 81451 +15049 81444 +23501 81440 +40650 81435 +25524 81408 +1646 81400 +15918 81400 +38455 81393 +8367 81384 +10993 81384 +32899 81381 +50133 81375 +3473 81360 +24345 81358 +27893 81353 +12760 81344 +15646 81342 +23670 81337 +14511 81326 +15940 81319 +19755 81319 +10887 81318 +28175 81314 +14439 81312 +30572 81306 +42576 81303 +39534 81290 +12770 81290 +14326 81277 +25087 81271 +13552 81268 +16060 81259 +22092 81247 +35002 81244 +24423 81243 +38180 81235 +29594 81225 +20992 81224 +26320 81221 +21140 81215 +44786 81212 +36882 81212 +26162 81179 +32675 81171 +1515 81170 +30527 81169 +28310 81165 +19397 81164 +33143 81161 +22760 81159 +21773 81149 +22517 81149 +35893 81147 +24103 81137 +34590 81135 +27070 81133 +20582 81129 +28577 81123 +24956 81121 +25314 81114 +36746 81114 +20721 81106 +24930 81104 +18537 81102 +5063 81101 +26022 81100 +49396 81097 +30404 81074 +33728 81070 +3654 81068 +24115 81059 +41434 81057 +34520 81057 +17002 81054 +30666 81041 +31569 81037 +40130 81037 +21586 81032 +43682 81030 +29479 81029 +35603 81026 +25767 81025 +8984 80993 +4754 80989 +19594 80983 +20332 80979 +29702 80975 +23841 80974 +20003 80974 +20033 80972 +25692 80968 +35839 80965 +21289 80960 +7656 80949 +15060 80949 +6094 80930 +37544 80927 +37709 80924 +31451 80915 +25133 80906 +41401 80905 +16129 80903 +7999 80886 +41281 80883 +34086 80870 +39342 80864 +42257 80859 +40263 80859 +44365 80857 +23597 80856 +32720 80854 +21125 80848 +15269 80847 +24862 80845 +26174 80844 +37386 80843 +13175 80826 +21670 80822 +21137 80808 +17926 80808 +36867 80796 +16616 80794 +29379 80791 +30882 80789 +22408 80788 +16329 80788 +22998 80773 +20842 80757 +35877 80753 +22952 80752 +21073 80739 +30400 80735 +40775 80731 +40167 80729 +18405 80727 +17194 80723 +22296 80716 +28016 80706 +25708 80704 +9862 80691 +24523 80691 +9913 80690 +36085 80687 +23333 80685 +29843 80679 +45406 80664 +25664 80645 +18714 80643 +22612 80629 +20973 80628 +18181 80627 +46434 80609 +32153 80608 +30340 80606 +26681 80604 +14415 80598 +25011 80590 +19392 80588 +21293 80583 +24985 80568 +6360 80567 +16022 80562 +24162 80548 +32474 80544 +25073 80536 +34595 80528 +13605 80518 +17660 80516 +25495 80514 +31394 80512 +38572 80512 +29989 80509 +1999 80508 +31723 80501 +25680 80491 +12061 80482 +36831 80480 +21032 80478 +13421 80478 +22393 80463 +23371 80452 +31488 80447 +31164 80442 +26541 80433 +38717 80417 +48823 80416 +44008 80409 +7126 80385 +19981 80378 +24969 80373 +35270 80368 +31530 80362 +48114 80360 +20864 80352 +31867 80351 +30525 80350 +28856 80349 +40592 80346 +33990 80345 +31839 80344 +7833 80334 +47299 80329 +22120 80328 +31241 80322 +45788 80321 +6985 80318 +39261 80308 +21510 80308 +15770 80292 +18788 80285 +30370 80275 +48514 80274 +25827 80268 +24594 80258 +17376 80257 +20224 80253 +10888 80252 +23238 80248 +10163 80244 +17526 80238 +20374 80237 +27386 80229 +24801 80218 +2053 80217 +13915 80213 +26989 80212 +34133 80200 +20586 80200 +38672 80194 +25451 80193 +15671 80185 +23999 80169 +40886 80166 +15565 80166 +5082 80162 +30900 80156 +19769 80150 +19139 80150 +11951 80147 +21885 80143 +23844 80137 +6238 80121 +43083 80114 +30759 80114 +34443 80100 +21268 80099 +38847 80098 +21857 80073 +29900 80068 +10057 80068 +13632 80050 +19799 80048 +2132 80045 +37503 80041 +21824 80040 +25939 80038 +13987 80032 +8484 80023 +45208 80014 +21986 80012 +20678 80012 +11165 79976 +41774 79971 +44126 79970 +38388 79969 +4794 79968 +22726 79963 +8738 79960 +31868 79955 +26734 79954 +20284 79940 +19333 79939 +35618 79934 +28847 79927 +22014 79925 +21622 79923 +29423 79921 +36502 79919 +23974 79918 +29860 79915 +33173 79909 +27220 79907 +29399 79906 +35248 79906 +31314 79894 +10551 79889 +48471 79877 +16652 79863 +48053 79863 +40013 79862 +14625 79857 +21977 79829 +19581 79823 +19061 79817 +21994 79817 +32548 79813 +14003 79811 +38289 79810 +43109 79805 +41147 79756 +14508 79749 +38427 79744 +24703 79729 +19824 79727 +37321 79714 +36975 79713 +26900 79689 +33011 79682 +2527 79680 +32711 79679 +28921 79671 +35457 79666 +26441 79656 +20892 79652 +42124 79649 +47463 79637 +22782 79635 +14842 79622 +33978 79614 +32354 79610 +15883 79607 +16898 79604 +8041 79602 +17106 79601 +20484 79600 +20376 79597 +38562 79592 +21443 79587 +11260 79577 +22652 79575 +32862 79568 +41828 79563 +33126 79558 +25766 79548 +34861 79544 +14432 79544 +25420 79542 +22258 79540 +31796 79537 +22956 79516 +21534 79497 +17987 79492 +34101 79489 +17129 79481 +26020 79469 +18129 79463 +20956 79450 +30794 79439 +1463 79429 +21843 79428 +45868 79418 +2794 79412 +23965 79408 +43323 79407 +13416 79407 +35695 79396 +24270 79394 +5736 79389 +23510 79380 +23744 79372 +29169 79356 +21982 79353 +38245 79352 +45411 79323 +15715 79321 +25893 79316 +11691 79313 +27366 79309 +37259 79303 +24340 79295 +45753 79292 +29396 79292 +28121 79292 +5722 79290 +20717 79278 +39847 79265 +31537 79264 +8152 79251 +22525 79235 +32363 79232 +29827 79230 +2958 79227 +29948 79223 +21148 79222 +30254 79214 +18333 79210 +17335 79206 +38957 79204 +18495 79200 +24563 79199 +23738 79198 +28615 79192 +35530 79192 +43463 79184 +35568 79179 +18360 79168 +31701 79156 +43017 79141 +43676 79130 +34760 79122 +26602 79116 +21834 79101 +18516 79095 +42148 79070 +14354 79062 +14969 79059 +26532 79054 +38911 79053 +21751 79048 +26680 79048 +20006 79046 +50040 79033 +31462 79033 +18564 79029 +43443 79028 +23137 79022 +9908 79020 +20508 79019 +39314 79019 +30356 79017 +27690 79010 +18465 78997 +27902 78989 +27716 78971 +11059 78971 +13871 78948 +48067 78943 +28320 78927 +11518 78916 +26672 78914 +49652 78904 +15738 78895 +20187 78894 +20811 78893 +13448 78892 +43531 78889 +35506 78886 +25072 78882 +6415 78879 +25687 78877 +28716 78870 +5864 78861 +43050 78855 +7976 78850 +23797 78837 +38732 78837 +44047 78835 +5309 78835 +45178 78834 +4510 78827 +44765 78811 +12193 78808 +43512 78806 +22992 78806 +45932 78801 +33860 78799 +15954 78779 +31251 78772 +20555 78770 +47661 78770 +4214 78769 +15774 78769 +12490 78767 +29520 78767 +35289 78758 +48976 78756 +26297 78753 +30885 78744 +21603 78739 +15834 78738 +20114 78736 +10262 78730 +24738 78719 +16376 78717 +34334 78706 +19514 78701 +23540 78695 +10557 78689 +21339 78686 +31143 78680 +3413 78673 +28201 78672 +20585 78672 +2830 78670 +15088 78661 +12993 78660 +16173 78652 +22839 78637 +10473 78636 +36562 78634 +35304 78618 +41999 78610 +26579 78595 +11225 78591 +21939 78585 +38660 78578 +34265 78575 +19070 78573 +30861 78564 +43090 78563 +24496 78562 +19115 78561 +1329 78558 +27538 78547 +8543 78546 +44185 78543 +31896 78543 +10049 78542 +40434 78536 +23302 78532 +20900 78532 +23528 78532 +34481 78531 +36239 78519 +32791 78518 +32687 78515 +9897 78500 +36634 78497 +42103 78487 +33436 78484 +45667 78482 +18561 78480 +26719 78479 +43450 78473 +31258 78462 +20690 78450 +17067 78447 +27930 78444 +29102 78442 +34798 78441 +10562 78431 +20302 78424 +24486 78417 +35829 78416 +47819 78411 +12900 78406 +31176 78396 +16758 78394 +18775 78388 +24353 78386 +35390 78385 +29685 78385 +22661 78381 +38338 78375 +37704 78362 +36520 78332 +9485 78326 +28215 78324 +20681 78323 +16563 78316 +21502 78314 +43991 78311 +17463 78301 +24699 78298 +18931 78293 +11744 78286 +31028 78286 +20394 78279 +27055 78278 +21629 78275 +37932 78270 +17359 78259 +29780 78254 +19257 78246 +15871 78245 +42225 78242 +29354 78232 +17798 78232 +25343 78231 +7809 78230 +30976 78228 +41363 78228 +31145 78217 +24020 78217 +16590 78196 +12138 78196 +37575 78195 +27185 78177 +19127 78171 +28706 78158 +31038 78127 +19262 78125 +7117 78118 +46620 78116 +40176 78116 +19916 78108 +20901 78106 +24101 78101 +23518 78086 +47857 78085 +12595 78083 +31293 78076 +37979 78074 +30658 78056 +27235 78055 +9516 78038 +29547 78031 +30284 78013 +112 78012 +41271 78007 +24059 77989 +18930 77971 +4965 77967 +27848 77965 +12758 77962 +29309 77958 +29946 77948 +27269 77932 +18797 77932 +17743 77928 +31535 77926 +28612 77911 +21678 77906 +40014 77902 +9059 77897 +17908 77890 +12081 77885 +34989 77860 +17503 77857 +10511 77851 +46551 77849 +17929 77844 +11174 77839 +36395 77827 +20196 77820 +32036 77818 +43692 77817 +35221 77806 +22128 77799 +34305 77796 +23285 77795 +6144 77792 +28967 77783 +19634 77777 +17615 77773 +36374 77764 +25000 77762 +17314 77759 +36300 77753 +23703 77749 +28519 77743 +43405 77742 +7959 77740 +11048 77740 +13523 77734 +46216 77727 +19490 77726 +49490 77724 +48971 77721 +23612 77716 +44599 77715 +30036 77715 +12825 77712 +43162 77712 +30227 77693 +23762 77688 +32994 77681 +16388 77676 +23492 77674 +22461 77663 +9452 77659 +6178 77627 +27963 77625 +36805 77620 +21818 77615 +32319 77614 +40154 77613 +28970 77608 +9060 77599 +39592 77597 +13195 77580 +20594 77580 +12894 77579 +30044 77568 +21757 77563 +45056 77562 +44419 77558 +27628 77551 +15558 77546 +6788 77541 +23261 77538 +41272 77538 +34928 77528 +19782 77522 +12350 77518 +18391 77506 +44842 77504 +7905 77503 +31139 77501 +35652 77500 +35999 77499 +25875 77480 +25802 77478 +24519 77476 +8635 77474 +19441 77471 +30350 77468 +43455 77467 +17168 77462 +19997 77459 +8250 77458 +38434 77457 +20873 77449 +27668 77447 +16152 77446 +35169 77421 +36814 77420 +44544 77418 +21218 77409 +24822 77400 +13294 77395 +32450 77387 +25092 77387 +21487 77383 +40460 77380 +39211 77377 +13800 77367 +43932 77358 +18517 77355 +23866 77347 +22286 77340 +16867 77334 +25152 77324 +20553 77320 +22514 77320 +17793 77307 +33783 77303 +14586 77286 +10377 77281 +8918 77270 +25931 77259 +36419 77256 +15592 77256 +29682 77249 +15146 77246 +22444 77239 +4383 77225 +36988 77224 +33444 77218 +48983 77214 +31997 77212 +16644 77206 +11205 77205 +5998 77202 +25210 77200 +20996 77196 +48157 77195 +26415 77194 +11778 77192 +18149 77189 +34003 77187 +23474 77173 +8777 77171 +29518 77166 +17916 77163 +1152 77149 +21790 77145 +16901 77142 +20927 77138 +13541 77116 +15920 77113 +2774 77105 +20367 77099 +22265 77096 +38680 77094 +31121 77094 +39012 77089 +27677 77088 +41373 77083 +48729 77074 +21479 77073 +14998 77069 +24210 77061 +24081 77044 +35129 77042 +21102 77035 +22863 77031 +25352 77021 +43915 77018 +39869 77017 +44202 77010 +4942 77002 +41503 77001 +23470 77000 +27232 76985 +41388 76981 +29901 76961 +38044 76955 +16241 76950 +42868 76948 +19699 76941 +4592 76939 +23085 76930 +24596 76929 +26324 76928 +44289 76912 +17703 76910 +40780 76909 +24927 76889 +44748 76882 +22242 76860 +19372 76851 +19787 76843 +47457 76840 +47435 76836 +15246 76835 +24324 76834 +22420 76830 +12933 76830 +4514 76822 +33940 76814 +17745 76813 +24234 76808 +38203 76807 +23631 76806 +33363 76796 +1750 76791 +37602 76789 +39414 76789 +15173 76788 +28603 76788 +38759 76784 +32539 76783 +10234 76772 +38157 76762 +21322 76762 +28617 76760 +29525 76744 +33654 76743 +17732 76734 +41462 76728 +17923 76722 +22237 76718 +26196 76716 +27805 76710 +26839 76709 +19775 76705 +19509 76704 +37916 76704 +3789 76703 +50199 76702 +41064 76688 +29218 76679 +31368 76651 +32840 76651 +25709 76646 +41324 76644 +21692 76643 +43733 76642 +15490 76622 +20932 76622 +29808 76614 +16803 76606 +38464 76586 +23105 76585 +23255 76574 +48323 76572 +27615 76568 +33668 76568 +18304 76552 +27519 76532 +29350 76528 +47765 76527 +19435 76523 +32400 76522 +35535 76514 +18646 76508 +24384 76508 +33495 76507 +28569 76503 +42008 76499 +8393 76496 +48773 76493 +22471 76488 +25828 76481 +25232 76471 +41239 76469 +26464 76457 +34461 76455 +9728 76448 +31525 76447 +37335 76439 +40520 76426 +38500 76423 +23125 76416 +23306 76411 +21806 76410 +33358 76405 +32228 76401 +34141 76393 +47412 76393 +31797 76392 +13422 76385 +10871 76382 +7762 76380 +16919 76375 +40742 76375 +29039 76370 +25238 76367 +27074 76359 +33254 76355 +15230 76351 +11633 76342 +29523 76337 +24666 76334 +35315 76331 +40811 76328 +23102 76326 +28838 76321 +26010 76317 +11220 76316 +29496 76311 +31369 76308 +44424 76306 +14478 76304 +7116 76304 +38154 76303 +22021 76298 +27484 76298 +16480 76292 +34240 76292 +29116 76292 +26669 76281 +6250 76280 +25315 76280 +12338 76268 +19214 76260 +39855 76257 +28370 76256 +33419 76250 +17484 76243 +22608 76243 +34001 76234 +15788 76229 +29876 76229 +50170 76224 +30269 76220 +34004 76218 +23675 76213 +23913 76210 +23439 76203 +39325 76184 +32878 76174 +25159 76166 +25301 76164 +25098 76160 +12531 76153 +48841 76142 +17637 76140 +31748 76139 +22384 76131 +31049 76122 +17179 76121 +34074 76121 +8432 76118 +33642 76114 +29351 76113 +29137 76111 +35925 76102 +22333 76098 +36411 76068 +20359 76060 +30451 76057 +38294 76054 +30509 76051 +35512 76026 +29902 76024 +8398 76021 +13429 76019 +21171 76018 +29781 76017 +7185 76015 +30278 75999 +3235 75996 +34092 75987 +9631 75975 +45933 75965 +21873 75954 +23500 75953 +49182 75947 +24240 75940 +1786 75938 +28928 75933 +45197 75926 +16855 75925 +2784 75921 +25119 75920 +7307 75919 +8189 75911 +12512 75903 +12727 75899 +35559 75894 +11347 75888 +36468 75879 +18843 75876 +46050 75875 +37392 75874 +48496 75857 +19232 75856 +26758 75854 +35823 75840 +44963 75831 +24007 75816 +33572 75812 +35276 75811 +46065 75804 +30939 75794 +21867 75794 +9302 75791 +46078 75784 +5272 75778 +35833 75769 +16691 75755 +13306 75748 +37298 75744 +11708 75730 +22983 75728 +29340 75721 +8353 75717 +30914 75715 +42954 75712 +22379 75707 +19460 75701 +19906 75698 +28084 75689 +21771 75686 +37480 75684 +13873 75684 +23944 75680 +24960 75675 +27465 75674 +41637 75669 +18694 75658 +30428 75655 +26757 75645 +18205 75638 +20363 75635 +34936 75633 +22485 75627 +15102 75617 +6357 75615 +29960 75600 +49209 75584 +8679 75577 +3546 75576 +30977 75569 +25632 75557 +25833 75549 +19977 75545 +33956 75544 +24444 75531 +24106 75524 +28862 75524 +25847 75505 +25542 75499 +33128 75495 +26982 75493 +32671 75485 +41838 75484 +32819 75476 +48438 75472 +32834 75463 +18926 75459 +23658 75455 +27455 75449 +20919 75449 +22565 75446 +31928 75443 +37641 75432 +30316 75432 +29437 75427 +37788 75419 +39457 75419 +33354 75418 +35425 75415 +26711 75396 +31411 75387 +39900 75380 +43544 75378 +20358 75374 +24419 75373 +28232 75372 +9940 75371 +24557 75366 +25771 75352 +14607 75344 +32027 75329 +5903 75304 +45951 75299 +27225 75295 +22353 75291 +14040 75284 +31726 75270 +38079 75262 +39762 75258 +25662 75255 +28576 75247 +38851 75237 +5168 75233 +21778 75228 +3540 75220 +35213 75213 +27098 75211 +12978 75206 +40768 75203 +30237 75202 +48503 75183 +37036 75164 +19338 75158 +20422 75157 +21694 75142 +36697 75138 +28579 75128 +27513 75113 +46708 75106 +28582 75102 +34254 75099 +34790 75099 +45674 75095 +22193 75095 +33214 75092 +39791 75091 +34064 75074 +32313 75073 +44492 75071 +20995 75066 +17895 75063 +26587 75046 +14857 75043 +13491 75040 +17013 75037 +8612 75035 +34856 75029 +38630 75028 +20090 75026 +32684 75022 +32890 75022 +38146 75021 +13299 75006 +11167 74991 +25950 74981 +22642 74971 +32119 74969 +25012 74955 +40089 74954 +30089 74943 +3995 74942 +18533 74941 +41788 74940 +49224 74936 +1564 74935 +9106 74934 +44692 74924 +18307 74920 +37191 74908 +34550 74892 +32623 74878 +3165 74870 +18599 74862 +16396 74848 +45276 74848 +5048 74839 +24971 74824 +28679 74812 +4132 74812 +43269 74808 +27794 74802 +9893 74773 +32856 74770 +34132 74759 +24601 74750 +29963 74744 +18112 74741 +44585 74739 +35344 74730 +36047 74721 +37445 74712 +17859 74711 +38287 74711 +25555 74710 +25697 74705 +23290 74697 +44745 74696 +33179 74689 +20136 74683 +19274 74679 +23931 74675 +23359 74667 +6199 74664 +25849 74661 +21442 74654 +21491 74645 +26237 74637 +40526 74624 +18452 74624 +38632 74621 +34602 74616 +30529 74615 +26126 74614 +19656 74610 +40667 74599 +17504 74594 +25800 74593 +22377 74589 +29166 74588 +17794 74580 +26430 74573 +39898 74555 +31348 74553 +21196 74551 +5620 74551 +18917 74547 +28510 74546 +5427 74545 +18886 74536 +36200 74534 +40095 74533 +5823 74531 +15648 74530 +13434 74528 +46347 74527 +30921 74525 +16540 74524 +18607 74521 +30371 74515 +29004 74508 +29651 74506 +14134 74479 +19204 74475 +24165 74472 +19084 74468 +1170 74468 +34643 74465 +19866 74452 +15072 74450 +3879 74444 +18812 74444 +48312 74440 +21414 74438 +19234 74436 +21802 74415 +18807 74406 +25379 74399 +34638 74398 +11017 74394 +42247 74389 +48834 74384 +23880 74379 +36602 74360 +38367 74356 +2334 74356 +27669 74355 +14216 74348 +40243 74345 +21428 74345 +50203 74338 +41377 74335 +38083 74332 +42993 74331 +30320 74326 +8265 74316 +21210 74310 +21355 74300 +5606 74291 +40765 74288 +31486 74287 +31605 74283 +20008 74282 +19802 74277 +16562 74277 +26325 74277 +19100 74270 +42799 74267 +41662 74243 +38485 74229 +25810 74228 +40004 74225 +34603 74220 +21345 74218 +13621 74209 +39547 74207 +13480 74192 +24688 74191 +29339 74191 +29295 74172 +35380 74171 +30244 74161 +39836 74161 +11601 74158 +23805 74150 +13876 74150 +3799 74145 +15677 74116 +6053 74114 +32010 74112 +36359 74108 +20011 74103 +41010 74095 +35668 74092 +22056 74087 +6212 74082 +17734 74076 +42471 74073 +47793 74072 +43103 74069 +14836 74066 +23532 74065 +41398 74061 +4883 74055 +31901 74055 +32355 74046 +23822 74039 +37277 74031 +16269 74022 +46391 74015 +41059 74013 +28697 74010 +39979 74008 +8429 74002 +26170 74000 +37755 73997 +30605 73995 +21403 73992 +18816 73991 +21409 73984 +19147 73976 +29044 73968 +38109 73960 +37937 73956 +35985 73955 +46019 73951 +42266 73949 +35049 73945 +37762 73936 +23748 73913 +29692 73907 +45373 73899 +4550 73875 +32416 73858 +42445 73845 +33359 73842 +18858 73839 +2633 73835 +45377 73829 +16108 73829 +30497 73826 +30421 73826 +27096 73821 +24176 73820 +36945 73810 +48043 73804 +30915 73799 +31245 73798 +30288 73795 +11002 73795 +18466 73787 +33425 73777 +29804 73772 +43647 73770 +21020 73763 +42228 73761 +33263 73752 +11475 73752 +19396 73752 +21978 73750 +43159 73723 +32730 73711 +22946 73708 +46583 73704 +20962 73702 +13590 73700 +35373 73684 +27525 73684 +48009 73669 +27700 73666 +48371 73665 +30699 73664 +46210 73664 +17552 73662 +46530 73659 +26148 73653 +33798 73638 +23598 73632 +42603 73623 +35056 73612 +32883 73610 +33125 73606 +19075 73603 +29632 73594 +26145 73592 +29432 73590 +12968 73589 +17506 73586 +27464 73585 +19949 73585 +29831 73584 +35834 73580 +5990 73578 +26127 73576 +18194 73568 +20762 73559 +22816 73552 +42509 73551 +27184 73542 +26188 73540 +30107 73539 +38030 73536 +3233 73532 +34911 73528 +39660 73527 +9979 73519 +27059 73514 +32670 73514 +14291 73511 +8330 73508 +38282 73505 +44271 73500 +19606 73491 +42365 73489 +47882 73487 +2983 73484 +18578 73482 +34679 73479 +42884 73476 +34794 73474 +30302 73469 +39598 73466 +9890 73465 +30492 73458 +13802 73455 +40325 73443 +19269 73441 +29430 73437 +29539 73429 +24994 73416 +15262 73404 +23584 73401 +39644 73400 +44035 73400 +24390 73398 +25531 73394 +16249 73377 +21234 73365 +24924 73362 +41204 73347 +38830 73342 +29131 73339 +5038 73335 +4018 73328 +15606 73321 +44919 73320 +37948 73316 +43280 73312 +26369 73309 +36556 73307 +5791 73306 +46353 73306 +15442 73304 +11650 73293 +24500 73292 +48314 73279 +27953 73272 +8091 73272 +28691 73269 +6326 73264 +17267 73262 +38555 73256 +29264 73251 +38620 73250 +25865 73248 +27390 73238 +21245 73235 +31044 73228 +26269 73226 +4516 73220 +18300 73218 +29696 73214 +19320 73212 +17947 73206 +48863 73205 +20190 73203 +14295 73198 +41595 73180 +25111 73175 +24030 73166 +27857 73160 +45673 73154 +28997 73150 +21278 73150 +28824 73147 +37723 73144 +27637 73139 +27752 73135 +29681 73135 +18435 73121 +3634 73106 +44684 73099 +30859 73096 +26223 73095 +32930 73092 +20665 73091 +27223 73089 +22884 73084 +43931 73077 +15001 73067 +19751 73061 +26570 73061 +29148 73056 +29811 73037 +15563 73029 +12643 73025 +40422 73024 +35134 73022 +36554 73021 +17770 73018 +23361 73007 +30901 73003 +17748 72999 +12529 72998 +36176 72980 +41783 72979 +26256 72979 +2674 72964 +38075 72946 +38253 72945 +8841 72943 +31712 72943 +37791 72938 +15996 72937 +1953 72932 +3542 72927 +15680 72922 +16975 72912 +29768 72912 +47040 72905 +36815 72904 +25924 72903 +22173 72899 +36548 72896 +40997 72886 +33895 72866 +35657 72865 +25311 72845 +9067 72839 +3213 72835 +37664 72832 +49425 72831 +26564 72831 +32256 72817 +38662 72814 +32166 72810 +31623 72802 +37651 72787 +43720 72786 +16147 72777 +10082 72773 +27485 72765 +40744 72757 +28908 72743 +8000 72730 +9717 72718 +18922 72709 +40351 72708 +6281 72696 +20510 72692 +45844 72690 +4823 72684 +14873 72678 +39193 72672 +4396 72658 +29916 72628 +11715 72627 +23476 72620 +20273 72616 +22538 72609 +14617 72601 +46458 72599 +22751 72578 +28753 72578 +21539 72572 +25508 72570 +35480 72570 +49063 72568 +19328 72567 +33989 72558 +8710 72553 +42722 72551 +7824 72542 +36955 72530 +28195 72523 +8400 72507 +26834 72505 +31864 72501 +30984 72480 +32411 72480 +16834 72479 +23389 72458 +34331 72455 +24177 72455 +24430 72447 +15793 72444 +39404 72440 +12603 72437 +41489 72434 +25888 72429 +12322 72428 +9135 72421 +25244 72417 +25070 72415 +22878 72408 +44398 72404 +26017 72400 +32839 72390 +20005 72387 +25625 72364 +29092 72356 +19349 72342 +31606 72321 +11644 72320 +5051 72318 +4098 72313 +22990 72310 +43640 72310 +12786 72308 +8493 72308 +28695 72300 +32359 72299 +21794 72290 +38977 72287 +36884 72287 +12991 72284 +30881 72260 +30711 72255 +37383 72251 +33073 72250 +12947 72241 +19430 72240 +20233 72232 +26014 72231 +44878 72217 +3579 72212 +18454 72211 +25573 72210 +23246 72203 +21033 72202 +40649 72186 +24730 72182 +45011 72178 +29186 72178 +21989 72176 +38681 72168 +6836 72156 +32758 72156 +46373 72155 +48007 72154 +26041 72153 +26958 72151 +17729 72145 +28918 72142 +24139 72140 +20712 72140 +16815 72138 +28307 72086 +20347 72078 +23840 72069 +33587 72055 +17790 72052 +27900 72051 +24107 72049 +22354 72048 +23374 72038 +18562 72036 +17062 72034 +21925 72026 +31285 72024 +17951 72016 +28819 72016 +4374 72011 +4573 72010 +36980 72007 +17514 71996 +46643 71986 +1922 71979 +6690 71978 +43703 71974 +23101 71974 +40498 71967 +11265 71961 +10642 71957 +32955 71942 +26285 71937 +9047 71934 +19104 71933 +13414 71929 +9100 71928 +39072 71921 +31138 71921 +15761 71915 +29292 71915 +27491 71913 +41736 71911 +40892 71907 +26021 71902 +38789 71898 +10051 71896 +19960 71888 +37260 71870 +27315 71863 +23060 71860 +40394 71857 +25082 71849 +17283 71841 +23195 71835 +34286 71835 +49521 71822 +18366 71820 +39016 71820 +46625 71816 +49618 71805 +34269 71799 +39576 71799 +17382 71795 +24901 71787 +47489 71781 +15631 71777 +37717 71774 +28110 71766 +14880 71764 +12195 71755 +24887 71753 +43731 71748 +16921 71740 +49117 71738 +45856 71731 +11258 71724 +22326 71717 +47813 71700 +1980 71685 +30323 71681 +36041 71670 +6106 71670 +31256 71669 +24790 71665 +16305 71650 +40899 71647 +36671 71646 +8821 71639 +48018 71638 +33948 71636 +38809 71632 +26665 71631 +8571 71628 +42920 71621 +15024 71618 +17084 71609 +40846 71601 +27873 71601 +4057 71593 +17972 71585 +30022 71585 +45301 71581 +35018 71576 +24883 71549 +25717 71546 +26073 71546 +37968 71542 +49528 71504 +43816 71495 +11719 71494 +11277 71488 +34659 71487 +46936 71484 +21690 71480 +22441 71474 +46321 71472 +40065 71470 +38297 71469 +32655 71468 +27580 71466 +33807 71442 +21159 71439 +21453 71437 +19826 71429 +25520 71426 +7277 71426 +19062 71420 +39958 71414 +27574 71403 +22349 71388 +13932 71386 +39236 71384 +22721 71365 +23324 71362 +28594 71358 +24429 71354 +42389 71352 +29779 71351 +23693 71343 +23766 71342 +42990 71339 +46502 71337 +27113 71334 +41763 71319 +42620 71318 +32691 71317 +34009 71310 +13085 71295 +7596 71293 +20248 71292 +45749 71290 +24098 71289 +28461 71286 +36679 71281 +38817 71277 +35383 71271 +10890 71269 +21325 71253 +9921 71251 +34214 71246 +11071 71244 +23281 71240 +20542 71240 +48807 71238 +29263 71236 +27942 71233 +28284 71225 +24114 71218 +21636 71218 +38981 71216 +38687 71215 +21042 71211 +49808 71207 +21528 71201 +29365 71200 +18973 71191 +39109 71190 +11973 71187 +19810 71186 +30055 71185 +29499 71183 +28148 71183 +23213 71174 +21213 71149 +4985 71145 +21522 71134 +30886 71129 +39116 71127 +21710 71121 +43792 71116 +33376 71113 +20936 71108 +3373 71094 +27476 71092 +21083 71068 +28763 71062 +36192 71061 +17319 71038 +37399 71036 +17208 71034 +22972 71024 +16036 71024 +37257 71012 +43152 71009 +47712 70998 +37443 70993 +29745 70981 +12885 70978 +33557 70969 +45913 70959 +26657 70959 +33996 70951 +26051 70948 +27457 70943 +10334 70943 +23642 70932 +48912 70930 +37938 70918 +11439 70912 +20102 70906 +12718 70905 +25638 70900 +18348 70896 +2521 70892 +17571 70886 +41584 70886 +44436 70880 +22267 70872 +38286 70867 +12129 70857 +15367 70854 +30879 70846 +27336 70843 +16504 70842 +42373 70836 +24241 70830 +20567 70830 +30054 70825 +24329 70824 +40680 70823 +41227 70820 +35703 70819 +22576 70814 +42281 70809 +23530 70803 +34860 70795 +45606 70782 +40250 70782 +41749 70770 +49194 70768 +37742 70766 +23798 70763 +17601 70762 +10450 70755 +29178 70752 +8368 70747 +26275 70735 +24808 70734 +15065 70731 +35840 70723 +20838 70716 +25747 70709 +2062 70701 +41803 70700 +26251 70693 +25815 70675 +48467 70673 +28517 70672 +22318 70667 +40942 70666 +23158 70655 +40875 70652 +29535 70652 +22855 70635 +9774 70630 +45026 70622 +12674 70621 +29996 70619 +25413 70596 +39953 70596 +34660 70595 +32829 70593 +36126 70588 +39365 70580 +16817 70579 +9868 70578 +22660 70577 +28507 70577 +23388 70576 +26931 70576 +17427 70574 +24600 70568 +37595 70567 +22100 70565 +23791 70561 +14697 70556 +19155 70555 +16506 70537 +36525 70535 +13610 70529 +1206 70521 +27648 70515 +27328 70514 +15649 70511 +25416 70502 +20616 70494 +23809 70485 +2553 70479 +32164 70470 +11420 70470 +19300 70468 +23776 70465 +16315 70464 +19898 70464 +41082 70455 +41396 70443 +32767 70430 +254 70422 +18742 70420 +40764 70418 +19821 70413 +26790 70412 +11493 70412 +24447 70406 +19774 70400 +35301 70391 +990 70390 +37634 70379 +28372 70378 +25740 70374 +21363 70373 +21473 70372 +43235 70368 +39607 70364 +26228 70359 +24216 70351 +27767 70349 +4123 70346 +16252 70339 +48633 70332 +46608 70328 +7447 70321 +17834 70313 +8462 70308 +32121 70306 +25972 70300 +26234 70298 +38201 70296 +38971 70292 +14619 70278 +40814 70275 +21503 70267 +25137 70263 +3581 70263 +24306 70260 +21540 70244 +21000 70244 +12833 70237 +27105 70234 +44406 70234 +22036 70232 +32466 70231 +49404 70224 +22169 70222 +30160 70214 +19225 70193 +21613 70192 +21805 70190 +18763 70186 +4520 70181 +42355 70181 +8939 70179 +1836 70177 +580 70171 +18868 70166 +36233 70163 +12468 70157 +29591 70156 +22364 70156 +7609 70155 +41390 70148 +6513 70148 +28538 70146 +22107 70144 +28546 70143 +27063 70143 +22084 70142 +40011 70134 +26027 70132 +29278 70131 +45181 70127 +24317 70124 +28533 70122 +36782 70115 +22520 70109 +28583 70109 +42746 70097 +26081 70085 +29885 70083 +14675 70079 +35281 70074 +36348 70072 +30840 70067 +29699 70066 +24853 70057 +21296 70056 +4782 70038 +24068 70032 +12482 70030 +31353 70017 +17226 70016 +6556 70011 +1825 70008 +28412 69996 +18070 69988 +24628 69971 +39209 69959 +13856 69952 +34904 69950 +40840 69948 +15932 69944 +30242 69937 +19032 69933 +20234 69924 +20262 69918 +33188 69907 +39136 69891 +21048 69889 +31976 69883 +31481 69881 +18127 69880 +34203 69878 +752 69878 +6610 69871 +3455 69861 +18904 69854 +32461 69854 +40269 69845 +16781 69841 +18892 69828 +18878 69827 +4541 69827 +9487 69827 +6806 69819 +43346 69816 +14353 69798 +25030 69794 +49626 69792 +26838 69791 +45549 69787 +21869 69774 +23683 69763 +30183 69759 +33174 69758 +41979 69756 +12762 69751 +21354 69736 +43047 69736 +16270 69734 +50026 69732 +22632 69732 +14747 69731 +17338 69723 +38643 69717 +32902 69712 +23110 69705 +27549 69700 +9277 69694 +3507 69686 +44805 69681 +30534 69678 +24865 69678 +6784 69672 +36900 69668 +34250 69664 +32205 69664 +28717 69662 +26954 69656 +28509 69654 +42038 69649 +23932 69648 +21719 69646 +12515 69640 +24681 69637 +44416 69624 +28831 69615 +37042 69608 +38982 69601 +28726 69584 +33329 69576 +29454 69562 +1888 69560 +31990 69558 +49798 69544 +32629 69541 +24208 69540 +23845 69536 +36267 69535 +21934 69530 +16823 69514 +6307 69514 +17952 69508 +22898 69507 +38921 69502 +45341 69498 +46579 69491 +26426 69491 +38057 69486 +148 69485 +19571 69469 +35189 69467 +37377 69461 +30822 69455 +17657 69455 +16920 69447 +19238 69442 +43428 69439 +48588 69438 +14258 69432 +20318 69428 +44373 69421 +17384 69421 +22343 69416 +42273 69407 +36073 69393 +44099 69386 +34558 69382 +2182 69372 +44122 69362 +8958 69360 +15987 69355 +36721 69351 +14001 69349 +15236 69341 +32168 69336 +44799 69329 +30259 69321 +45754 69314 +23082 69307 +34344 69301 +43561 69281 +11425 69264 +38700 69258 +37729 69256 +27384 69253 +41071 69239 +29931 69229 +21573 69226 +18907 69225 +25274 69223 +27859 69222 +36389 69219 +31470 69218 +41325 69218 +10573 69208 +38152 69199 +25572 69189 +31291 69186 +11688 69169 +15708 69166 +47847 69162 +39068 69161 +47431 69152 +41555 69145 +47711 69141 +45044 69135 +13099 69132 +33413 69129 +21059 69125 +29650 69123 +17596 69119 +38872 69117 +13223 69117 +32308 69116 +34939 69107 +24657 69092 +31212 69088 +27518 69087 +36425 69083 +45293 69073 +43537 69071 +27361 69066 +42764 69047 +13311 69045 +16544 69045 +37448 69037 +48876 69037 +31288 69035 +35359 69023 +8585 69014 +11898 69008 +10236 69006 +38952 69005 +26861 68998 +30154 68997 +15596 68994 +17149 68989 +38839 68989 +8401 68987 +27330 68980 +8043 68980 +50090 68980 +37013 68972 +16592 68972 +28969 68964 +45294 68963 +5347 68961 +27817 68960 +23563 68958 +28098 68954 +31515 68953 +29849 68952 +30282 68946 +20186 68945 +19828 68944 +4892 68940 +25690 68935 +32268 68934 +16436 68934 +5769 68933 +20912 68927 +17265 68920 +40056 68920 +29055 68908 +19760 68907 +7753 68904 +1935 68903 +7076 68901 +20167 68891 +27432 68890 +46828 68889 +24910 68885 +20547 68875 +3567 68863 +24425 68852 +8654 68852 +23223 68835 +11197 68830 +35421 68824 +8910 68812 +33217 68811 +28493 68809 +45863 68800 +18052 68798 +46535 68794 +47925 68792 +26722 68790 +3089 68790 +28325 68788 +31316 68779 +28206 68778 +40946 68772 +18225 68769 +22328 68767 +41769 68761 +17670 68748 +15595 68747 +33005 68742 +19181 68738 +22091 68727 +33632 68725 +2567 68720 +9641 68707 +9195 68698 +23367 68695 +26963 68694 +42953 68682 +17437 68678 +48757 68677 +48847 68674 +28021 68673 +23873 68672 +42629 68663 +45347 68658 +32448 68654 +43154 68647 +26902 68647 +19792 68633 +34748 68631 +27409 68630 +33533 68623 +36508 68621 +42999 68618 +22094 68611 +42113 68606 +16879 68604 +33423 68602 +26831 68566 +29534 68559 +36498 68554 +14098 68548 +49495 68547 +22200 68542 +32331 68535 +28991 68534 +37558 68531 +44349 68530 +28652 68526 +11020 68522 +18215 68522 +39036 68522 +40770 68514 +44911 68509 +6514 68491 +31123 68490 +32980 68488 +2234 68488 +20866 68486 +38261 68481 +35798 68480 +42574 68479 +46388 68479 +7933 68469 +23498 68452 +40072 68449 +26510 68442 +43623 68440 +17388 68438 +6389 68436 +24257 68423 +38059 68421 +42736 68418 +45936 68413 +26128 68413 +39313 68412 +35323 68403 +35594 68392 +39025 68389 +42094 68389 +17847 68386 +16631 68385 +15328 68375 +32043 68370 +19849 68366 +26490 68365 +22066 68357 +18046 68351 +2303 68337 +9876 68336 +13137 68333 +23823 68332 +15652 68332 +35080 68330 +23323 68327 +43890 68326 +46732 68320 +19103 68319 +27939 68311 +27234 68306 +27391 68301 +29140 68300 +26137 68300 +36333 68299 +8183 68290 +31189 68284 +36576 68283 +16187 68280 +27568 68277 +15538 68276 +44018 68273 +47717 68253 +39905 68252 +24436 68249 +27407 68240 +21591 68238 +40015 68237 +19345 68231 +20226 68230 +21894 68221 +38372 68212 +30151 68212 +31568 68190 +16858 68186 +18664 68180 +35249 68168 +46578 68143 +33915 68142 +34093 68118 +20437 68114 +37946 68102 +40317 68102 +40543 68098 +18741 68094 +21709 68085 +36662 68083 +42705 68059 +8789 68055 +16703 68053 +40696 68051 +49293 68048 +12291 68045 +41971 68045 +47349 68038 +18815 68034 +38511 68033 +49894 68021 +26547 68021 +34536 68018 +28207 68017 +41086 68006 +47822 68005 +1970 67984 +24028 67974 +20934 67971 +36987 67966 +10113 67960 +22951 67949 +22142 67944 +21035 67942 +36353 67938 +20780 67936 +47074 67928 +25705 67927 +21916 67919 +8230 67917 +44050 67910 +21365 67885 +19371 67884 +25240 67878 +26835 67875 +22601 67874 +35432 67872 +16056 67863 +27298 67860 +48711 67849 +23865 67829 +22827 67823 +30974 67809 +27989 67792 +21344 67777 +38992 67776 +36472 67776 +14786 67766 +39026 67764 +31861 67757 +43814 67744 +21792 67733 +18510 67733 +41540 67732 +23490 67731 +40528 67730 +13087 67730 +9580 67729 +26240 67727 +1472 67727 +33601 67721 +47265 67721 +38919 67714 +20986 67710 +13384 67707 +49338 67694 +12828 67690 +30957 67687 +19454 67685 +18278 67654 +33513 67647 +14398 67646 +22594 67641 +11786 67639 +11876 67631 +2965 67624 +21427 67616 +20959 67612 +18376 67612 +26566 67609 +7201 67606 +27878 67604 +35933 67590 +13174 67587 +13839 67580 +4210 67579 +25112 67570 +46533 67567 +15967 67567 +9288 67558 +39154 67553 +39091 67544 +19917 67544 +20474 67543 +23774 67539 +38058 67537 +15299 67536 +19082 67534 +3795 67530 +26329 67512 +30455 67511 +41422 67509 +37016 67505 +32741 67502 +30062 67498 +24362 67497 +19206 67497 +36755 67486 +8457 67486 +26071 67485 +38495 67483 +21769 67476 +40983 67460 +41618 67451 +19325 67443 +14904 67441 +27088 67432 +32952 67427 +36507 67406 +29180 67403 +27947 67402 +30438 67398 +19930 67392 +20307 67391 +34187 67386 +45899 67384 +13033 67380 +26821 67373 +36933 67370 +7199 67368 +36592 67367 +34430 67365 +38080 67336 +3153 67334 +17688 67334 +32404 67333 +49369 67330 +20718 67327 +25327 67321 +43487 67314 +43229 67311 +10044 67310 +31277 67305 +29684 67302 +19903 67297 +22590 67291 +27183 67288 +15754 67287 +19222 67287 +22744 67282 +39046 67281 +38798 67277 +10613 67264 +39690 67260 +29711 67255 +21407 67245 +6207 67241 +19074 67237 +27029 67230 +28889 67226 +28267 67224 +12483 67216 +35335 67215 +49118 67214 +47097 67210 +16796 67207 +4233 67207 +16011 67191 +12458 67184 +22115 67183 +48831 67181 +19040 67180 +5570 67179 +4140 67176 +17983 67174 +35060 67167 +40688 67167 +26573 67160 +16365 67158 +24332 67149 +17756 67147 +35022 67146 +13042 67146 +2205 67137 +35127 67135 +40399 67133 +31147 67131 +24769 67131 +31931 67100 +27252 67100 +19909 67095 +19954 67094 +29630 67093 +141 67090 +19118 67089 +28332 67086 +45269 67082 +13824 67082 +36709 67080 +23594 67072 +27498 67061 +34922 67056 +44421 67043 +43765 67042 +27681 67037 +3314 67021 +31096 67016 +25250 67005 +17759 67001 +18571 66993 +9553 66992 +20179 66988 +29426 66977 +39944 66971 +19939 66965 +13624 66964 +9690 66958 +25772 66950 +5943 66948 +914 66937 +27208 66937 +38707 66931 +17010 66916 +8196 66915 +20868 66914 +42849 66913 +33021 66909 +23959 66906 +16354 66884 +26191 66878 +29595 66874 +9426 66872 +32800 66871 +25192 66866 +20100 66864 +21770 66862 +22050 66851 +24715 66851 +31422 66848 +46574 66843 +28105 66838 +14925 66831 +49858 66830 +15137 66829 +36131 66822 +11825 66818 +24530 66807 +13924 66806 +24716 66805 +26590 66799 +29451 66789 +16326 66788 +20836 66786 +30065 66785 +24796 66784 +7060 66782 +30060 66781 +30849 66779 +17585 66776 +24905 66776 +7680 66766 +6271 66765 +41037 66764 +46355 66764 +44272 66760 +29793 66742 +26500 66741 +8480 66739 +20917 66730 +23311 66727 +43861 66724 +20124 66724 +40563 66706 +41665 66705 +40524 66696 +21777 66695 +21536 66694 +23717 66689 +19609 66688 +46039 66687 +31879 66686 +24507 66676 +26888 66675 +34037 66674 +26810 66672 +40752 66671 +29290 66670 +25721 66667 +10363 66663 +40376 66660 +10419 66658 +34143 66645 +15274 66641 +21549 66633 +22427 66614 +37878 66586 +50236 66581 +42540 66581 +40645 66581 +22231 66577 +42847 66574 +45277 66567 +26432 66561 +24561 66549 +19485 66549 +34245 66545 +23834 66544 +40030 66540 +28459 66539 +7916 66538 +19507 66528 +20975 66527 +16552 66525 +36290 66524 +36254 66523 +45109 66517 +35696 66517 +20796 66516 +23771 66513 +26636 66502 +34639 66497 +4680 66491 +41913 66486 +28233 66484 +45327 66484 +37521 66479 +20048 66470 +25878 66469 +4804 66460 +5792 66455 +49398 66448 +21240 66426 +43757 66408 +27861 66400 +47515 66397 +15417 66396 +29105 66386 +37131 66383 +35459 66374 +4600 66368 +27415 66365 +22256 66364 +12012 66358 +28962 66357 +28225 66354 +15997 66351 +24386 66347 +21225 66346 +24012 66345 +26596 66344 +35449 66338 +47447 66338 +25858 66334 +41514 66330 +42542 66326 +29287 66319 +36398 66313 +32019 66307 +26206 66303 +26928 66297 +44294 66297 +21697 66296 +49767 66295 +33078 66293 +27718 66291 +30466 66290 +23134 66287 +39370 66268 +25433 66267 +25988 66261 +24506 66244 +30555 66234 +20578 66233 +37339 66227 +21718 66226 +24922 66224 +1937 66223 +17402 66221 +20615 66220 +21433 66216 +35032 66210 +23586 66204 +17824 66200 +33171 66198 +37556 66196 +30888 66190 +37499 66186 +21712 66183 +45177 66183 +31846 66165 +13381 66156 +25809 66154 +19088 66145 +19180 66144 +39288 66144 +33769 66140 +24578 66140 +26936 66139 +32826 66138 +41946 66127 +28123 66126 +49783 66126 +24049 66125 +42439 66123 +36524 66121 +37951 66112 +19645 66109 +36031 66104 +23275 66104 +17393 66104 +35466 66102 +18104 66083 +3527 66079 +20729 66079 +15155 66078 +21377 66075 +23570 66072 +33628 66058 +23938 66048 +47699 66043 +14990 66037 +34937 66035 +31453 66030 +19445 66030 +19348 66026 +45346 66016 +15079 66007 +35715 66005 +38548 66000 +38153 65991 +37517 65991 +30166 65984 +45603 65971 +9562 65967 +24777 65964 +25424 65959 +23667 65959 +25103 65957 +40935 65951 +35754 65946 +40701 65945 +27561 65943 +40274 65937 +44107 65934 +20675 65933 +38826 65927 +25318 65925 +26043 65925 +34268 65910 +48041 65908 +48117 65899 +28079 65896 +40633 65893 +31302 65891 +33280 65891 +36696 65891 +39745 65887 +35043 65887 +24291 65887 +36963 65885 +38262 65885 +40691 65883 +29050 65883 +34299 65881 +38111 65877 +23654 65875 +20834 65872 +18159 65871 +17400 65869 +27697 65862 +41611 65849 +46669 65845 +18558 65830 +35724 65825 +27851 65810 +29107 65807 +27262 65806 +13040 65802 +35845 65801 +23263 65798 +38459 65796 +30171 65791 +43774 65789 +20511 65786 +31320 65784 +41223 65776 +16996 65766 +22947 65764 +48309 65762 +40863 65761 +6785 65758 +38222 65757 +6804 65742 +33071 65740 +37276 65739 +33122 65739 +18629 65736 +23818 65735 +27475 65734 +32860 65727 +8622 65724 +23442 65717 +32358 65707 +27474 65707 +23824 65698 +36386 65698 +5577 65688 +10496 65681 +41943 65678 +11454 65675 +35966 65675 +19672 65672 +4807 65671 +13513 65669 +32969 65668 +28756 65657 +33568 65656 +35953 65653 +19244 65645 +29569 65643 +25425 65639 +31757 65634 +18273 65628 +6015 65625 +23009 65625 +22670 65621 +19972 65621 +15424 65619 +14192 65616 +35954 65615 +32445 65608 +29202 65608 +48164 65606 +16980 65604 +42475 65604 +32596 65603 +17118 65600 +44801 65600 +46406 65590 +32735 65589 +28011 65586 +27284 65582 +7583 65578 +43207 65573 +16931 65572 +11991 65561 +19233 65546 +29155 65542 +18298 65537 +33262 65533 +21154 65530 +43741 65528 +15456 65524 +39718 65508 +32569 65507 +15705 65504 +20350 65502 +24587 65498 +38164 65495 +29398 65495 +43393 65493 +24562 65489 +19114 65489 +20668 65477 +30538 65466 +34700 65463 +22049 65448 +34548 65447 +34014 65440 +27182 65432 +41702 65430 +21669 65427 +23372 65425 +35446 65425 +38178 65413 +21144 65412 +17749 65408 +34892 65405 +29232 65404 +43742 65401 +42151 65399 +25834 65394 +34315 65393 +14837 65392 +40685 65390 +16654 65389 +21607 65389 +35297 65378 +21816 65366 +19023 65364 +26852 65347 +27807 65347 +19736 65337 +21850 65336 +18574 65335 +21172 65322 +16833 65304 +31532 65302 +30973 65298 +29168 65274 +32762 65273 +9057 65269 +29450 65265 +8164 65261 +2760 65256 +35485 65254 +19865 65248 +46051 65245 +12984 65244 +43226 65242 +30172 65239 +18955 65239 +6084 65232 +31381 65230 +29614 65225 +31747 65202 +13369 65200 +38863 65198 +46653 65189 +37796 65188 +41140 65183 +12295 65181 +22547 65180 +39479 65177 +1506 65172 +25212 65167 +41902 65161 +39148 65157 +30793 65151 +29235 65150 +1361 65149 +6874 65148 +31001 65148 +2552 65138 +48484 65133 +13202 65132 +29242 65132 +37030 65130 +47154 65121 +21374 65119 +28526 65113 +44119 65112 +37180 65104 +28491 65102 +33368 65100 +44143 65096 +44381 65091 +17890 65088 +31956 65082 +18668 65067 +31781 65053 +18345 65041 +40992 65037 +9647 65032 +42125 65030 +35351 65029 +42498 65016 +9357 65005 +33522 64997 +30510 64996 +29065 64992 +30889 64992 +16966 64975 +26614 64971 +33936 64956 +40807 64948 +27819 64940 +27995 64930 +17301 64924 +31647 64923 +41310 64920 +30713 64919 +1491 64917 +11837 64914 +39360 64913 +24755 64910 +12552 64907 +25547 64904 +9822 64903 +34369 64900 +23981 64897 +34623 64895 +5493 64885 +49042 64883 +32914 64880 +26647 64878 +25241 64876 +19281 64872 +32753 64865 +26139 64864 +44386 64859 +27281 64851 +24739 64846 +44918 64838 +37566 64837 +22406 64833 +29844 64824 +42824 64805 +38063 64802 +14189 64796 +19664 64796 +5226 64782 +45463 64775 +47735 64766 +24780 64765 +22033 64742 +27048 64735 +28010 64734 +20083 64723 +27044 64718 +5458 64715 +37097 64709 +35971 64708 +3308 64701 +46811 64664 +34295 64662 +41896 64661 +27200 64652 +9062 64637 +26016 64631 +21890 64630 +26886 64628 +25980 64626 +33663 64621 +38719 64620 +3948 64618 +22780 64614 +22739 64601 +45556 64598 +44626 64582 +45125 64580 +40308 64579 +27899 64569 +24311 64568 +30375 64567 +32232 64565 +23591 64564 +49872 64553 +4323 64552 +6978 64549 +29382 64548 +19727 64540 +3231 64538 +30449 64533 +48906 64531 +21016 64531 +22248 64528 +16184 64525 +27276 64518 +21643 64514 +48769 64514 +39203 64511 +28530 64511 +23176 64508 +31332 64504 +31290 64502 +28654 64500 +7919 64500 +25397 64494 +28527 64492 +47595 64484 +33052 64481 +26215 64471 +18396 64466 +32607 64461 +21825 64459 +5360 64453 +16669 64450 +44179 64446 +28197 64443 +10286 64436 +19836 64433 +33806 64431 +25293 64419 +48361 64416 +43419 64408 +20108 64400 +38355 64398 +39358 64397 +15202 64381 +30256 64355 +36740 64355 +34998 64354 +36304 64343 +38320 64341 +18971 64336 +33081 64334 +34518 64329 +32083 64325 +30000 64322 +22701 64318 +19818 64306 +37313 64295 +40978 64294 +26730 64290 +24846 64271 +36121 64270 +27558 64261 +36009 64260 +18977 64259 +41926 64244 +26609 64238 +46779 64238 +28387 64238 +21826 64235 +40907 64224 +29671 64224 +24989 64214 +22656 64210 +25592 64206 +20743 64200 +25784 64193 +42392 64193 +24939 64189 +13398 64180 +45399 64171 +14635 64156 +26001 64155 +32029 64152 +48677 64145 +29927 64140 +42288 64135 +48302 64135 +4515 64131 +2101 64129 +38230 64125 +25655 64125 +14076 64121 +28779 64117 +5647 64115 +15217 64114 +36092 64113 +41967 64112 +16161 64107 +16487 64103 +22494 64102 +25063 64100 +33041 64096 +27759 64090 +24488 64083 +28973 64075 +46889 64075 +17515 64063 +31725 64060 +23769 64050 +35675 64047 +20161 64046 +26688 64040 +26112 64036 +28153 64031 +26470 64024 +44210 64015 +34571 64013 +42754 64007 +3921 64003 +21230 63999 +24356 63996 +46949 63996 +49860 63995 +43442 63995 +35128 63992 +2384 63991 +32033 63990 +9182 63984 +16274 63982 +15225 63979 +4070 63978 +31639 63975 +22465 63974 +41651 63973 +9448 63964 +35198 63962 +32504 63961 +45187 63958 +13103 63957 +12960 63950 +38360 63945 +13880 63937 +9804 63930 +11313 63921 +19772 63918 +47592 63915 +29333 63914 +20546 63914 +41073 63912 +37924 63907 +34821 63904 +24916 63903 +35444 63900 +44352 63891 +25076 63890 +4648 63889 +43985 63881 +6719 63881 +7803 63878 +35377 63878 +38523 63877 +38285 63866 +33899 63865 +6929 63864 +40352 63860 +46539 63858 +22303 63842 +36437 63837 +30061 63837 +35756 63836 +18041 63835 +41533 63833 +30987 63821 +22967 63820 +18722 63818 +34744 63812 +22745 63805 +25997 63798 +49577 63798 +17510 63796 +25519 63785 +41892 63784 +41965 63770 +19538 63757 +33702 63754 +25368 63752 +16096 63747 +45732 63743 +11362 63741 +38174 63739 +25776 63730 +36302 63728 +31101 63726 +24643 63725 +25699 63722 +46994 63713 +4636 63712 +47693 63712 +14838 63705 +10300 63703 +20646 63699 +28784 63696 +20634 63694 +22358 63685 +20144 63683 +28477 63682 +11184 63680 +24000 63673 +21261 63668 +32885 63667 +9819 63666 +16773 63664 +29020 63660 +37134 63652 +31477 63638 +46729 63633 +45555 63632 +11866 63631 +15903 63631 +14646 63631 +19130 63628 +18542 63623 +33753 63623 +25963 63620 +17385 63616 +29207 63608 +37461 63607 +19265 63594 +21508 63593 +22160 63584 +26536 63580 +42628 63580 +14254 63575 +43376 63574 +36762 63562 +34796 63561 +38315 63559 +31104 63552 +33861 63550 +18951 63549 +34351 63535 +25268 63532 +45064 63530 +29353 63529 +5659 63528 +35201 63526 +20853 63522 +19811 63520 +37043 63510 +12535 63506 +25976 63503 +30705 63500 +43868 63497 +11176 63484 +31352 63473 +39387 63471 +25493 63470 +45067 63470 +18768 63469 +30186 63465 +32249 63463 +31480 63457 +39011 63446 +22335 63443 +7861 63443 +24793 63441 +14602 63437 +17641 63429 +31498 63427 +25134 63426 +17297 63425 +29494 63420 +16486 63417 +21060 63409 +30663 63408 +35296 63401 +32967 63395 +33228 63392 +21729 63392 +17140 63374 +5317 63374 +30292 63371 +38234 63366 +37286 63360 +10141 63355 +19734 63350 +11804 63348 +19474 63340 +22247 63340 +12944 63322 +24018 63317 +38558 63316 +44936 63315 +33675 63309 +26224 63307 +26732 63300 +18058 63298 +27965 63296 +26684 63290 +27887 63277 +27221 63276 +1193 63271 +33453 63265 +21585 63258 +33578 63258 +40959 63243 +18720 63239 +24995 63234 +27004 63227 +27127 63224 +30647 63222 +20334 63219 +12997 63206 +24231 63200 +35554 63196 +25557 63194 +46989 63192 +48316 63191 +40074 63184 +38606 63184 +10994 63179 +47894 63176 +19442 63173 +15714 63170 +11284 63168 +46411 63167 +38129 63165 +24478 63158 +21255 63158 +49482 63158 +1407 63144 +21388 63141 +24069 63136 +34990 63135 +38646 63131 +25678 63125 +34596 63123 +45189 63122 +34040 63116 +34231 63112 +18905 63097 +11857 63093 +38358 63092 +36806 63091 +28776 63083 +37931 63081 +31892 63075 +29439 63072 +33290 63072 +47005 63070 +26424 63069 +25560 63058 +40430 63056 +48158 63054 +17464 63051 +22113 63047 +35065 63031 +50075 63023 +21667 63021 +20034 63019 +29943 63018 +28092 63017 +2108 63016 +41640 63013 +1132 63012 +20624 63009 +36876 63005 +27305 63002 +20381 63002 +39523 62996 +20478 62995 +49250 62994 +14797 62994 +13060 62992 +44310 62991 +23574 62990 +45476 62967 +26830 62965 +11296 62964 +9362 62963 +32507 62960 +31379 62955 +45033 62949 +23284 62947 +35938 62944 +22131 62944 +29135 62942 +25077 62938 +21578 62927 +25773 62927 +39372 62925 +3990 62921 +43341 62916 +23816 62916 +31091 62911 +33075 62909 +32442 62896 +18248 62881 +38014 62879 +26304 62872 +22352 62867 +8081 62866 +12976 62864 +50231 62857 +24921 62842 +18636 62837 +19846 62837 +41155 62821 +26574 62810 +42923 62806 +20844 62801 +31198 62794 +19650 62783 +17240 62778 +16856 62777 +11555 62777 +18049 62755 +25381 62751 +20043 62746 +36903 62744 +15624 62742 +23853 62741 +43678 62733 +6435 62733 +42952 62723 +14967 62723 +13049 62721 +19052 62720 +31351 62705 +23145 62704 +40596 62682 +27065 62678 +32805 62667 +33745 62661 +23417 62652 +27408 62651 +25071 62648 +35037 62647 +49067 62645 +38534 62633 +30712 62631 +42838 62626 +37629 62624 +25654 62620 +4142 62620 +44760 62618 +15845 62603 +3447 62602 +34809 62595 +7004 62593 +46137 62589 +45329 62587 +42862 62584 +26421 62583 +13696 62574 +15835 62573 +34529 62558 +25544 62533 +42345 62518 +23183 62516 +29987 62514 +25789 62501 +38162 62501 +39854 62496 +26397 62485 +20822 62484 +36839 62474 +31682 62473 +31443 62465 +32135 62459 +47955 62453 +37095 62453 +43097 62450 +34061 62442 +33633 62434 +28340 62429 +24786 62427 +17103 62419 +27327 62417 +40359 62413 +28234 62412 +36997 62408 +7254 62406 +8593 62406 +25580 62406 +31086 62406 +6652 62405 +21965 62398 +2445 62396 +28246 62395 +37075 62394 +31720 62392 +28767 62388 +11458 62385 +18534 62382 +36391 62380 +24648 62377 +31163 62375 +32385 62372 +22679 62367 +42328 62360 +24475 62355 +23741 62350 +36354 62339 +24672 62336 +29817 62332 +33782 62331 +19531 62331 +26218 62328 +17470 62306 +30050 62304 +43415 62300 +16985 62293 +38487 62277 +28364 62268 +47304 62257 +23421 62252 +10458 62249 +20260 62247 +25543 62245 +18869 62239 +36735 62235 +43256 62231 +28574 62228 +25562 62227 +42521 62222 +36768 62221 +230 62217 +30040 62215 +33880 62212 +35104 62207 +21489 62201 +13001 62195 +35256 62193 +24986 62193 +49018 62190 +9338 62183 +27855 62171 +31603 62153 +12721 62149 +34685 62149 +46977 62149 +41969 62142 +37732 62140 +20316 62132 +21888 62131 +43611 62127 +28671 62123 +23180 62123 +38137 62122 +12480 62120 +32628 62118 +35024 62117 +27495 62108 +49145 62102 +30250 62091 +24734 62064 +30913 62064 +34429 62054 +46327 62051 +11531 62033 +13154 62028 +19418 62024 +34475 62022 +38087 62021 +231 62021 +15396 62008 +34199 62003 +1715 61999 +48674 61998 +46409 61995 +30339 61994 +36897 61990 +42261 61989 +42621 61987 +29573 61985 +13377 61984 +31513 61978 +29042 61962 +15605 61953 +48734 61946 +24203 61944 +38725 61937 +10901 61934 +30868 61934 +29772 61921 +18175 61918 +31689 61915 +41804 61914 +25943 61906 +26519 61903 +29837 61901 +15810 61896 +15813 61893 +23117 61892 +17776 61889 +46227 61866 +35357 61862 +35685 61857 +43431 61855 +40129 61852 +40808 61849 +44014 61847 +13096 61846 +42554 61846 +39292 61842 +49002 61837 +22121 61814 +47891 61810 +32925 61807 +26394 61798 +40587 61798 +30721 61796 +15630 61788 +26951 61786 +23336 61766 +32302 61755 +22531 61745 +25781 61738 +23755 61736 +22236 61733 +12043 61731 +36743 61729 +18188 61723 +46072 61722 +39123 61718 +30755 61718 +7829 61705 +41718 61704 +43296 61704 +20010 61702 +34677 61701 +14046 61698 +16991 61697 +37553 61697 +3560 61695 +20676 61692 +42198 61685 +31934 61682 +26879 61678 +15785 61673 +24308 61668 +42356 61666 +22859 61660 +36858 61655 +45402 61650 +31810 61638 +47688 61619 +22703 61619 +17481 61615 +35517 61613 +44410 61596 +35228 61593 +12003 61583 +40349 61581 +29026 61576 +48273 61569 +28218 61562 +38920 61559 +25731 61552 +22655 61546 +32945 61545 +28161 61541 +46611 61534 +45022 61528 +13193 61514 +21056 61506 +47554 61503 +15357 61502 +32757 61495 +31238 61489 +30391 61486 +42518 61482 +24407 61479 +32046 61475 +39244 61451 +24246 61448 +10533 61447 +28082 61441 +35733 61435 +44252 61429 +21868 61424 +31517 61419 +17758 61418 +23236 61418 +12283 61412 +29301 61405 +24925 61402 +45996 61401 +8001 61395 +47727 61390 +36605 61372 +19177 61372 +39094 61368 +26386 61359 +10531 61355 +34166 61355 +32533 61342 +29259 61333 +33917 61333 +33652 61327 +44838 61322 +11977 61317 +41930 61315 +30349 61314 +28357 61311 +34081 61307 +17886 61305 +20266 61299 +6990 61299 +28344 61296 +34264 61290 +29036 61289 +34282 61284 +32202 61283 +14756 61281 +35003 61281 +36246 61280 +19915 61265 +37307 61264 +24312 61257 +32985 61253 +18289 61249 +12332 61245 +17766 61244 +34845 61239 +45369 61233 +30082 61222 +21652 61216 +23695 61211 +33585 61199 +15405 61190 +16081 61188 +15015 61185 +41476 61183 +19590 61181 +23859 61170 +18863 61167 +18490 61161 +11187 61159 +42786 61148 +44843 61148 +27346 61147 +35406 61139 +33867 61134 +44840 61129 +36773 61125 +36432 61123 +50070 61122 +32028 61117 +45562 61116 +12938 61113 +14831 61105 +20942 61104 +13335 61100 +48405 61090 +34226 61088 +8481 61085 +15478 61083 +49748 61083 +21207 61079 +25144 61077 +46286 61073 +31469 61072 +43130 61062 +39822 61061 +25907 61048 +36045 61046 +47561 61037 +22769 61036 +34838 61036 +5424 61034 +38633 61033 +11007 61028 +31083 61025 +20759 61025 +30224 61022 +44914 61021 +22483 61020 +16869 61019 +31788 61019 +19798 61014 +50198 61001 +7355 60988 +21580 60971 +23855 60969 +27135 60966 +48654 60963 +38931 60960 +29194 60955 +40390 60945 +30877 60939 +36405 60929 +42073 60929 +40542 60924 +39772 60914 +30084 60911 +18392 60902 +38822 60895 +46277 60890 +4298 60888 +14049 60881 +32972 60873 +27962 60865 +29716 60851 +11249 60842 +15537 60840 +45005 60835 +26124 60831 +42291 60828 +29461 60826 +26717 60826 +28607 60825 +39161 60824 +36964 60824 +34207 60824 +26457 60816 +5592 60812 +15194 60805 +9158 60805 +21581 60802 +26370 60800 +48937 60796 +18032 60795 +31604 60791 +26340 60789 +1405 60786 +37525 60786 +22320 60783 +38502 60765 +39952 60762 +25738 60756 +32064 60753 +33677 60751 +30873 60749 +22828 60744 +46744 60739 +24217 60738 +29370 60732 +41645 60726 +39656 60717 +48535 60715 +26194 60715 +44288 60714 +23098 60713 +45155 60711 +42226 60701 +46898 60673 +23625 60672 +29862 60671 +48857 60666 +25276 60654 +33681 60648 +28836 60647 +15266 60643 +41852 60638 +13966 60622 +25219 60620 +31600 60617 +31549 60616 +48639 60615 +23089 60611 +47319 60606 +25121 60605 +38554 60603 +29199 60601 +30544 60598 +25221 60595 +17436 60594 +19659 60593 +20022 60593 +28641 60590 +17115 60590 +43599 60589 +15584 60589 +27692 60585 +42183 60566 +15947 60565 +42196 60562 +32392 60560 +25704 60558 +22448 60555 +21932 60549 +24458 60547 +39330 60543 +29537 60537 +32528 60527 +29635 60520 +44997 60518 +45073 60514 +28837 60511 +47802 60499 +46766 60497 +45475 60497 +17020 60484 +14977 60483 +42619 60480 +25906 60478 +30771 60467 +2658 60460 +19814 60460 +47552 60455 +41886 60448 +36106 60444 +47956 60433 +15634 60428 +11577 60425 +20343 60420 +18680 60417 +49984 60413 +28142 60403 +4681 60401 +44790 60400 +40636 60397 +23895 60390 +32745 60388 +46151 60384 +9372 60384 +32874 60383 +32964 60374 +26844 60369 +46193 60365 +18823 60359 +26653 60359 +30942 60355 +37816 60352 +22705 60342 +45194 60327 +44111 60323 +23440 60322 +44358 60319 +27211 60315 +33024 60303 +40917 60302 +35771 60302 +39092 60300 +40769 60294 +19554 60291 +29738 60290 +47403 60284 +30909 60271 +46191 60271 +25546 60261 +32484 60261 +27452 60257 +38965 60250 +13704 60250 +31022 60249 +30063 60249 +46134 60246 +35855 60243 +39829 60238 +4668 60237 +18218 60237 +29542 60233 +37684 60224 +48403 60210 +37470 60207 +12645 60205 +21926 60196 +13764 60195 +16558 60195 +4348 60194 +31939 60194 +27492 60183 +34608 60180 +37159 60179 +48423 60162 +47485 60161 +29556 60161 +39097 60161 +34664 60152 +49200 60151 +37914 60142 +25500 60137 +18549 60126 +28443 60112 +20713 60101 +13894 60095 +25094 60095 +22740 60094 +42441 60086 +43898 60085 +28816 60073 +44949 60068 +45585 60067 +23556 60066 +16983 60058 +28415 60056 +19974 60049 +33511 60047 +44443 60044 +39480 60041 +24250 60038 +30874 60037 +21630 60035 +21831 60035 +18777 60032 +29787 60031 +30393 60023 +38426 60015 +40588 60009 +37240 60004 +2755 59997 +28188 59996 +25996 59996 +29255 59994 +20895 59989 +34146 59983 +41953 59981 +32625 59979 +17248 59979 +5104 59972 +48283 59969 +37391 59969 +37119 59967 +15429 59965 +35020 59965 +30954 59962 +31309 59954 +27782 59938 +6138 59937 +37897 59935 +35122 59932 +26781 59931 +21208 59929 +25639 59924 +39671 59923 +23133 59922 +15120 59920 +33760 59907 +31782 59900 +5080 59899 +44391 59896 +20757 59894 +47148 59891 +35142 59889 +46684 59885 +17792 59884 +32727 59878 +20802 59870 +19679 59863 +36800 59857 +33636 59854 +15716 59853 +2090 59853 +24758 59852 +17806 59850 +32715 59848 +27173 59847 +8490 59846 +27951 59844 +41407 59843 +47919 59843 +20785 59838 +31432 59834 +28214 59832 +30855 59825 +4278 59824 +35647 59823 +32609 59822 +35984 59820 +27312 59812 +11532 59807 +35734 59805 +3752 59802 +22909 59794 +33421 59791 +43923 59788 +22253 59785 +30503 59777 +44525 59772 +34164 59772 +19548 59770 +26258 59768 +16667 59767 +25062 59765 +41161 59755 +17007 59751 +43728 59748 +48269 59743 +7890 59742 +22186 59740 +15627 59733 +20867 59729 +44437 59723 +22017 59722 +31195 59722 +25457 59721 +33775 59719 +44192 59717 +22415 59711 +22631 59708 +42614 59705 +8146 59693 +6564 59688 +22366 59684 +25181 59682 +3692 59681 +11469 59681 +37216 59677 +20403 59676 +36788 59673 +13051 59672 +9777 59668 +39777 59660 +38979 59657 +35101 59653 +31829 59642 +30490 59641 +34311 59634 +33588 59631 +39042 59629 +15086 59629 +42351 59619 +22127 59613 +20446 59613 +32193 59611 +22487 59610 +24696 59607 +23691 59598 +2355 59598 +22470 59594 +30632 59593 +16066 59586 +22994 59578 +18611 59575 +16206 59575 +30720 59574 +19524 59571 +42997 59570 +13759 59568 +44871 59566 +23759 59565 +39909 59558 +21967 59549 +30814 59546 +28416 59542 +15409 59539 +25453 59537 +21444 59536 +41282 59530 +23662 59528 +30447 59521 +42386 59519 +36506 59517 +43084 59516 +31889 59513 +6310 59502 +30860 59487 +43771 59483 +45023 59478 +7019 59472 +44354 59465 +33983 59464 +41790 59457 +6263 59445 +25530 59441 +31321 59441 +32468 59441 +24233 59440 +48130 59438 +19685 59431 +5919 59426 +25904 59423 +29912 59421 +9214 59415 +26412 59413 +31392 59409 +15308 59408 +30030 59404 +24938 59402 +25177 59394 +30113 59390 +39421 59380 +29895 59379 +637 59379 +29731 59377 +25024 59376 +23760 59371 +4924 59370 +47312 59370 +19107 59368 +7978 59362 +26219 59359 +38351 59358 +10015 59354 +49737 59352 +2122 59351 +32814 59351 +9242 59350 +21958 59347 +11086 59347 +35831 59334 +8307 59333 +1425 59327 +36219 59316 +15046 59308 +23752 59307 +18640 59302 +16696 59300 +29491 59298 +21228 59290 +17533 59289 +15625 59288 +42692 59287 +48421 59280 +31831 59278 +23387 59264 +4284 59263 +7896 59262 +24707 59254 +16962 59245 +17963 59238 +34102 59236 +22370 59236 +34365 59235 +44677 59233 +34967 59231 +20476 59229 +21047 59228 +24221 59226 +37604 59224 +10110 59224 +35311 59223 +16331 59221 +24080 59216 +12140 59213 +18287 59204 +28087 59203 +36745 59201 +19933 59201 +46527 59199 +1579 59198 +24873 59187 +11327 59179 +8453 59171 +17280 59162 +38462 59156 +27430 59155 +26029 59149 +32480 59149 +26289 59148 +39720 59144 +25525 59137 +39337 59135 +43675 59128 +37484 59115 +44691 59114 +19943 59106 +32295 59101 +39733 59099 +24544 59095 +43855 59091 +50024 59086 +4288 59084 +4699 59083 +33198 59083 +42541 59070 +46488 59067 +30473 59066 +31922 59065 +28347 59052 +39566 59035 +14255 59033 +42167 59029 +26221 59028 +24314 59027 +34123 59026 +20057 59025 +36866 59020 +14915 59014 +33163 59014 +30519 59013 +13806 59001 +31365 59000 +27091 58999 +29417 58999 +43775 58997 +42290 58994 +27869 58990 +26648 58982 +22025 58979 +37438 58979 +42845 58975 +18756 58971 +46997 58969 +10611 58969 +12494 58967 +40425 58967 +50251 58964 +47686 58963 +20323 58958 +33976 58945 +49486 58945 +37899 58943 +24848 58943 +44660 58932 +15451 58930 +42207 58925 +46089 58925 +30829 58919 +30130 58917 +26855 58915 +33779 58912 +28476 58907 +33009 58904 +34161 58898 +23405 58896 +21185 58889 +41524 58885 +35774 58884 +50089 58880 +43945 58873 +23482 58867 +26241 58853 +24090 58852 +4055 58849 +25751 58830 +21617 58829 +36761 58820 +24889 58820 +42599 58819 +38028 58818 +9809 58816 +47220 58809 +35577 58807 +41556 58807 +27410 58805 +27870 58805 +36608 58803 +24501 58798 +28902 58792 +45083 58791 +38879 58785 +18792 58785 +22579 58780 +22618 58773 +19577 58770 +38223 58764 +41870 58746 +17731 58745 +34818 58744 +25616 58740 +42436 58738 +29237 58738 +35110 58734 +20918 58733 +39459 58725 +28738 58715 +37463 58713 +39812 58709 +46767 58707 +14306 58705 +28783 58701 +20733 58699 +9962 58697 +44236 58696 +23278 58694 +29898 58694 +35209 58689 +34854 58685 +36438 58681 +25804 58680 +46007 58680 +26698 58677 +26875 58672 +22503 58672 +3140 58670 +40894 58669 +16793 58662 +44970 58661 +28844 58652 +34988 58638 +40442 58638 +33269 58635 +15147 58634 +29327 58633 +21695 58622 +20368 58617 +30485 58613 +40934 58611 +40389 58609 +21935 58608 +32141 58606 +31155 58593 +46463 58590 +35374 58589 +41701 58588 +30588 58588 +7556 58582 +35651 58581 +48014 58574 +10951 58573 +22775 58569 +20839 58564 +42994 58559 +41643 58555 +21887 58552 +22217 58545 +39291 58543 +26031 58542 +2455 58537 +25966 58536 +33596 58532 +46123 58530 +45633 58527 +35719 58524 +36830 58521 +41480 58517 +20058 58513 +23303 58513 +31095 58511 +40950 58509 +28317 58508 +18050 58492 +44108 58487 +36642 58468 +19655 58465 +41457 58463 +27562 58462 +33085 58453 +39871 58444 +19296 58443 +30937 58443 +7162 58440 +26877 58436 +23182 58431 +44848 58423 +9228 58421 +21529 58418 +44493 58418 +24040 58414 +36457 58410 +40929 58405 +3639 58404 +26880 58403 +3490 58403 +12282 58393 +30726 58389 +36514 58387 +42293 58387 +42928 58378 +26658 58374 +32215 58370 +10199 58369 +31698 58368 +37824 58366 +25432 58364 +35717 58359 +24490 58347 +20644 58343 +29493 58341 +47328 58335 +26061 58334 +49457 58333 +3796 58323 +14164 58318 +30002 58318 +36586 58317 +26986 58314 +28606 58312 +22040 58310 +43989 58306 +14019 58297 +24421 58293 +7483 58288 +18850 58283 +17627 58279 +35014 58255 +32881 58255 +12137 58253 +39708 58249 +20632 58244 +27010 58234 +49318 58218 +9896 58209 +19924 58190 +33315 58190 +40665 58189 +49537 58186 +28305 58184 +6020 58183 +23694 58181 +50135 58179 +41213 58174 +30275 58170 +44559 58165 +36944 58162 +40264 58158 +30853 58143 +41034 58141 +30051 58128 +25108 58126 +21326 58123 +40424 58118 +47665 58118 +26100 58118 +26186 58117 +33824 58116 +21002 58116 +19567 58105 +19420 58099 +31818 58099 +47928 58097 +31403 58095 +34851 58094 +17259 58085 +23649 58080 +4612 58079 +49622 58074 +27306 58073 +23961 58067 +17213 58060 +23112 58048 +41326 58037 +26668 58033 +32872 58029 +38302 58027 +32333 58025 +27931 58021 +37915 58020 +47689 58020 +45239 58013 +21290 58011 +28429 58011 +16841 58007 +26582 58002 +46451 58002 +32483 58001 +24178 57999 +30607 57998 +38493 57996 +40915 57983 +38099 57982 +48453 57976 +43474 57970 +37051 57967 +24260 57964 +25798 57964 +32095 57962 +34056 57956 +29861 57953 +21554 57950 +23179 57940 +8448 57938 +23810 57932 +36162 57927 +10483 57927 +40977 57926 +44150 57920 +28502 57918 +45587 57910 +21422 57909 +30592 57908 +32613 57895 +13623 57890 +37599 57890 +42127 57890 +26969 57887 +18368 57885 +21265 57877 +27268 57871 +29345 57862 +20709 57862 +45775 57858 +46278 57852 +16969 57849 +16799 57847 +37718 57845 +1859 57835 +39257 57835 +32677 57812 +23258 57812 +36896 57812 +15710 57809 +34177 57809 +14721 57805 +41429 57804 +19591 57799 +30239 57797 +37976 57793 +33871 57788 +35654 57781 +33555 57780 +24725 57779 +20477 57769 +29652 57766 +2966 57764 +37562 57760 +15898 57751 +24228 57750 +13010 57750 +11159 57746 +9851 57740 +48591 57739 +40228 57735 +16475 57733 +34552 57731 +46691 57726 +35589 57719 +37538 57717 +3333 57717 +46905 57712 +24521 57712 +32453 57712 +29361 57708 +36292 57703 +23706 57697 +48500 57690 +24560 57689 +24082 57684 +37356 57680 +34010 57679 +5507 57677 +31449 57675 +36583 57673 +21239 57664 +25260 57662 +5656 57660 +43310 57659 +45031 57659 +39828 57655 +39619 57654 +26192 57654 +7540 57648 +27163 57642 +39722 57639 +8675 57634 +30176 57634 +28451 57630 +42417 57625 +20730 57612 +38175 57608 +31522 57604 +29405 57604 +31421 57592 +38242 57585 +49165 57573 +46507 57569 +30415 57568 +33611 57531 +9493 57527 +30300 57527 +41648 57526 +21606 57519 +30775 57514 +29254 57513 +33285 57511 +29955 57497 +47458 57491 +17701 57484 +28882 57476 +25913 57473 +28885 57472 +26798 57469 +10641 57466 +46368 57462 +49325 57455 +34356 57454 +39616 57454 +28539 57452 +14322 57451 +21169 57445 +25631 57441 +34029 57439 +20450 57438 +41045 57437 +30524 57435 +7956 57430 +43701 57424 +36644 57420 +31265 57420 +40718 57412 +5663 57408 +34034 57406 +17855 57404 +32521 57402 +29151 57392 +45108 57388 +36541 57386 +28506 57372 +37751 57368 +19200 57366 +6540 57363 +13102 57358 +23244 57352 +9956 57349 +20752 57346 +2796 57338 +21960 57327 +21593 57325 +27684 57319 +34954 57316 +28602 57313 +30255 57309 +9503 57289 +92 57268 +35622 57267 +34763 57264 +23603 57258 +39654 57255 +20151 57254 +33265 57249 +47879 57242 +10841 57235 +45496 57232 +24070 57230 +33658 57225 +34262 57220 +28521 57218 +35113 57216 +33835 57211 +32906 57205 +7213 57201 +11417 57197 +14270 57194 +31977 57187 +17950 57178 +38045 57170 +42368 57164 +4799 57161 +26042 57158 +40837 57146 +29265 57144 +14455 57141 +48714 57140 +21976 57126 +23601 57117 +25541 57116 +24494 57115 +15327 57113 +43999 57107 +7640 57103 +35437 57099 +36327 57091 +37352 57086 +10055 57082 +14269 57078 +44978 57077 +44333 57074 +44042 57065 +34735 57059 +31734 57056 +32679 57052 +32132 57050 +38814 57042 +5233 57039 +2131 57037 +23349 57036 +23713 57034 +38935 57024 +7549 57023 +39255 57022 +36205 57015 +23181 57007 +12392 57000 +4611 56997 +47697 56991 +10445 56985 +46884 56981 +30132 56975 +15412 56974 +16899 56965 +7757 56961 +30042 56960 +43349 56955 +35006 56948 +45966 56948 +25004 56945 +14274 56929 +29267 56926 +31027 56918 +23164 56918 +24182 56915 +25292 56902 +24928 56902 +29360 56900 +44343 56899 +42476 56895 +32701 56892 +36995 56889 +34428 56886 +25509 56886 +41430 56885 +2571 56885 +45774 56882 +18911 56874 +14603 56874 +43748 56867 +45247 56860 +29229 56854 +29466 56847 +30761 56839 +44128 56834 +35786 56825 +43379 56824 +24338 56822 +23666 56814 +28980 56814 +24760 56811 +45658 56806 +42627 56806 +20779 56803 +30777 56791 +37610 56790 +38244 56789 +21688 56787 +8467 56787 +47609 56787 +47518 56786 +17539 56785 +23292 56784 +25677 56778 +6331 56778 +12410 56776 +4182 56760 +22580 56753 +23153 56752 +46429 56749 +37439 56748 +2715 56743 +23848 56742 +25565 56740 +27605 56739 +39490 56731 +22156 56730 +18459 56725 +26497 56719 +41898 56715 +25325 56711 +41252 56710 +18967 56709 +44483 56703 +35658 56701 +44019 56697 +35362 56684 +38435 56682 +43479 56681 +47186 56681 +45644 56679 +27762 56677 +37884 56675 +30935 56659 +25309 56658 +6959 56658 +29456 56655 +28639 56646 +31731 56635 +49627 56635 +32774 56628 +10855 56627 +21450 56620 +25266 56612 +30508 56607 +37990 56606 +30179 56602 +14920 56602 +29653 56601 +26718 56596 +48988 56590 +26744 56589 +41778 56586 +37373 56586 +11944 56581 +21584 56570 +37761 56569 +13978 56568 +22395 56567 +21795 56564 +14565 56563 +9610 56556 +45795 56554 +16465 56554 +40421 56554 +38423 56551 +28625 56551 +22595 56550 +45209 56538 +28903 56537 +20938 56528 +6003 56526 +49197 56525 +20467 56516 +26980 56515 +44718 56514 +37318 56513 +26274 56511 +25398 56509 +18111 56505 +26771 56503 +44500 56502 +36791 56501 +15078 56500 +39845 56496 +8443 56486 +45400 56483 +16571 56481 +31395 56477 +36096 56475 +15157 56474 +36656 56459 +20694 56450 +25421 56435 +15170 56430 +24913 56430 +27479 56430 +27592 56429 +9444 56424 +39406 56421 +23097 56418 +11970 56414 +38608 56412 +33340 56409 +43425 56406 +15729 56403 +19197 56400 +15721 56391 +1783 56378 +27719 56373 +43995 56372 +37954 56368 +28109 56358 +35803 56358 +33482 56352 +35337 56350 +32882 56344 +4815 56338 +39034 56337 +47608 56334 +33181 56327 +20806 56323 +21295 56321 +28593 56319 +44067 56314 +16909 56311 +49037 56310 +22986 56305 +29452 56304 +24662 56297 +9682 56294 +47015 56286 +24831 56285 +7349 56285 +36074 56282 +33973 56279 +32736 56278 +22638 56278 +11645 56278 +26379 56277 +12663 56274 +18846 56271 +38398 56269 +42158 56265 +42395 56261 +41120 56258 +49842 56258 +48781 56253 +32254 56248 +47012 56247 +22196 56247 +5944 56245 +21755 56232 +30033 56219 +47164 56219 +20798 56212 +28633 56207 +19363 56204 +39451 56202 +40559 56195 +20807 56192 +20831 56189 +37198 56188 +21783 56184 +33736 56183 +32472 56179 +38583 56177 +28916 56177 +38820 56176 +40088 56171 +30668 56136 +20281 56136 +47586 56130 +18271 56125 +9685 56124 +25769 56120 +11190 56116 +33946 56112 +39630 56109 +32578 56106 +27079 56102 +12632 56092 +11402 56091 +29957 56084 +16199 56073 +29406 56069 +36448 56063 +45942 56060 +26271 56059 +19932 56054 +40870 56054 +27217 56052 +7015 56046 +33600 56044 +33255 56038 +11865 56033 +25280 56033 +26444 56031 +25696 56031 +2065 56019 +45897 56008 +17459 56005 +6008 56004 +11671 56003 +44366 56001 +24041 55998 +1488 55996 +32530 55992 +35923 55988 +11218 55985 +31599 55983 +30080 55976 +21693 55973 +43811 55970 +40745 55968 +33247 55956 +33907 55953 +30069 55944 +11194 55942 +14036 55942 +25466 55940 +36610 55939 +37121 55938 +3391 55932 +21074 55932 +28286 55931 +39965 55927 +26384 55926 +12752 55921 +16814 55920 +22613 55915 +15096 55898 +16670 55896 +25190 55889 +29871 55885 +18603 55880 +27134 55878 +15051 55871 +17878 55869 +43859 55861 +7315 55860 +35326 55852 +11522 55852 +5378 55849 +27364 55837 +27107 55833 +35210 55831 +19311 55830 +18879 55824 +5955 55818 +23523 55816 +26009 55814 +32848 55813 +5015 55811 +26509 55809 +23159 55806 +44250 55802 +3346 55802 +29802 55802 +24369 55802 +40855 55798 +28090 55795 +39205 55793 +26974 55791 +28359 55790 +13984 55790 +37714 55786 +29663 55782 +24993 55780 +20768 55775 +19334 55762 +49112 55756 +37084 55748 +42408 55740 +10570 55739 +27845 55732 +37329 55724 +16927 55723 +23729 55720 +4311 55715 +17622 55707 +42760 55700 +37325 55691 +7293 55688 +20981 55688 +16340 55686 +34632 55682 +36070 55679 +14500 55678 +50066 55675 +25148 55670 +14652 55668 +20436 55668 +25390 55658 +29385 55652 +39670 55647 +24718 55646 +23243 55641 +23208 55638 +26012 55637 +18993 55632 +37514 55622 +46049 55621 +29527 55618 +30038 55617 +21054 55616 +19770 55610 +24984 55608 +11572 55606 +32710 55597 +36512 55593 +23604 55588 +39343 55587 +39150 55583 +41069 55580 +28829 55578 +28034 55573 +28046 55565 +41950 55564 +41328 55563 +24420 55562 +32151 55558 +43653 55551 +25068 55547 +29093 55545 +32560 55544 +38808 55541 +45851 55541 +45982 55540 +43308 55535 +21579 55532 +25036 55520 +34110 55510 +47261 55508 +41764 55506 +10482 55505 +33627 55504 +20937 55497 +6720 55495 +48968 55483 +22228 55479 +20073 55476 +24651 55475 +41046 55474 +43858 55473 +2023 55471 +42221 55463 +19630 55459 +40776 55457 +23678 55455 +25411 55446 +28713 55443 +43042 55438 +22853 55436 +38400 55433 +33025 55431 +49108 55431 +18676 55426 +24588 55426 +35106 55418 +44923 55414 +24720 55411 +30276 55404 +43688 55403 +27675 55399 +12668 55398 +40801 55396 +37708 55386 +33839 55382 +19008 55378 +5547 55375 +25749 55373 +24706 55368 +10333 55361 +18239 55361 +26917 55359 +26452 55350 +33710 55348 +26494 55342 +38927 55341 +26521 55341 +36132 55339 +30423 55339 +30912 55336 +42573 55334 +36989 55319 +1159 55317 +26785 55317 +25179 55316 +6254 55315 +9886 55315 +37322 55315 +41360 55313 +32084 55306 +4215 55303 +21093 55295 +41817 55292 +42431 55288 +35924 55287 +36440 55283 +37254 55279 +36778 55270 +41683 55269 +28203 55268 +21836 55268 +28580 55267 +19569 55260 +33848 55259 +28563 55259 +36358 55240 +39350 55237 +38102 55229 +29981 55228 +34840 55222 +37728 55218 +49375 55217 +12582 55217 +28385 55203 +44385 55201 +9606 55200 +27903 55194 +42334 55191 +34395 55188 +13349 55188 +29089 55184 +24773 55181 +34629 55179 +30020 55178 +38651 55178 +3380 55177 +24051 55176 +35935 55175 +38191 55172 +27224 55154 +17866 55153 +46107 55151 +37206 55147 +37314 55142 +28295 55130 +27414 55129 +39206 55129 +46362 55127 +12613 55119 +24013 55119 +37465 55110 +46849 55105 +46132 55100 +43911 55096 +30204 55092 +44470 55088 +33388 55088 +14854 55084 +43881 55084 +15227 55080 +28450 55073 +44922 55068 +24815 55067 +21478 55067 +49696 55063 +19402 55060 +28273 55059 +24119 55057 +31375 55056 +26646 55054 +30013 55048 +29009 55044 +50106 55032 +22153 55030 +35130 55022 +47545 55019 +49006 55019 +39019 55013 +31878 54992 +37745 54990 +9731 54987 +2786 54984 +1237 54980 +15084 54978 +18908 54970 +37808 54965 +38283 54963 +32327 54961 +36066 54961 +37752 54960 +35860 54960 +40931 54957 +33517 54955 +37768 54955 +21748 54952 +22584 54951 +27362 54948 +49570 54941 +23507 54936 +21192 54934 +34012 54933 +3604 54928 +49407 54927 +39834 54919 +32111 54909 +46921 54896 +35546 54890 +38730 54887 +33291 54885 +14578 54872 +30623 54867 +34905 54864 +13402 54851 +25365 54845 +45747 54842 +41352 54842 +24053 54840 +43997 54836 +33466 54832 +36274 54831 +28423 54820 +27704 54818 +24071 54814 +28616 54814 +5554 54807 +1945 54798 +29914 54797 +41493 54795 +19851 54794 +32502 54786 +31087 54777 +24673 54771 +49247 54770 +27043 54766 +24039 54762 +19980 54751 +30902 54750 +40513 54747 +49195 54742 +44282 54741 +6962 54740 +33446 54738 +42397 54735 +49185 54735 +31501 54730 +35368 54728 +34573 54726 +41454 54719 +16417 54718 +16422 54702 +42239 54701 +38337 54695 +18681 54683 +1442 54682 +45813 54682 +6329 54675 +11965 54674 +41875 54669 +42803 54656 +47692 54656 +14906 54654 +18349 54651 +29579 54648 +7046 54648 +30696 54642 +30229 54641 +37113 54638 +28568 54637 +47470 54636 +29689 54632 +23917 54627 +38031 54626 +44117 54623 +17848 54622 +49873 54615 +15655 54615 +43052 54614 +158 54613 +7106 54607 +6395 54603 +22995 54602 +30522 54599 +30827 54589 +35233 54585 +35573 54579 +32281 54578 +31174 54577 +3787 54571 +34782 54569 +33201 54556 +44864 54556 +31413 54556 +6474 54555 +50219 54539 +22699 54535 +2210 54524 +4163 54521 +30357 54521 +27142 54520 +24646 54519 +28572 54516 +29673 54514 +23135 54512 +32387 54512 +16279 54498 +43251 54489 +6775 54488 +24571 54483 +33799 54482 +22963 54472 +16661 54472 +46116 54471 +38796 54465 +15851 54456 +37305 54454 +27101 54452 +33630 54448 +42857 54446 +43593 54445 +18148 54437 +29128 54422 +27721 54421 +17046 54415 +26210 54413 +20483 54411 +48994 54409 +31884 54408 +16945 54405 +23452 54403 +36232 54402 +19839 54401 +30413 54400 +11227 54394 +37551 54379 +13875 54376 +26637 54372 +13691 54362 +17458 54359 +21626 54349 +26808 54348 +19880 54341 +25806 54340 +10687 54336 +27695 54334 +34958 54334 +42971 54330 +42718 54327 +29850 54320 +20279 54312 +28682 54304 +38964 54284 +42358 54281 +31998 54278 +31531 54265 +29983 54262 +19179 54261 +40153 54257 +33124 54257 +3145 54254 +10845 54253 +25229 54249 +2418 54241 +25288 54238 +13649 54236 +29243 54236 +1684 54235 +27864 54231 +30386 54227 +41758 54226 +17008 54222 +46194 54218 +26819 54216 +17693 54216 +9633 54216 +40457 54213 +12830 54211 +49862 54210 +29704 54209 +13268 54209 +22405 54207 +40514 54205 +6780 54204 +25695 54200 +8600 54185 +42626 54183 +17380 54181 +25360 54178 +15581 54176 +35082 54174 +39585 54163 +30953 54160 +16371 54148 +28849 54147 +16653 54143 +10867 54143 +16491 54137 +38716 54134 +27888 54133 +49637 54129 +19780 54129 +13409 54124 +6718 54118 +8570 54118 +23048 54115 +25501 54100 +37408 54097 +31075 54093 +13436 54090 +39095 54089 +30064 54085 +33584 54078 +20931 54076 +16272 54070 +36188 54070 +42658 54069 +31003 54068 +39543 54067 +47404 54064 +16360 54062 +23375 54058 +25478 54054 +38886 54052 +6800 54049 +28729 54035 +7537 54035 +43968 54033 +45488 54033 +41262 54032 +22819 54032 +25006 54032 +49179 54032 +45159 54028 +42813 54028 +14951 54027 +15385 54026 +20609 54023 +24888 54021 +46346 54011 +25473 54010 +32595 54009 +35605 54006 +22792 54005 +46924 53997 +15003 53994 +49980 53992 +19004 53991 +9037 53986 +40727 53986 +18861 53980 +41314 53971 +22965 53970 +44953 53968 +32807 53966 +30740 53952 +47137 53950 +26903 53934 +27996 53933 +42157 53933 +40595 53933 +30690 53932 +22220 53931 +12496 53928 +5560 53928 +41440 53923 +36847 53922 +24046 53919 +19216 53915 +37264 53915 +38510 53911 +24810 53909 +24124 53906 +16283 53900 +37476 53899 +26143 53896 +45524 53883 +29889 53882 +35606 53878 +14599 53877 +22373 53874 +31465 53866 +31670 53855 +27395 53855 +44348 53848 +23011 53847 +14618 53847 +46081 53846 +34208 53839 +2916 53837 +24785 53828 +47358 53827 +45861 53821 +28285 53818 +40570 53809 +12907 53808 +37079 53803 +42569 53792 +18882 53788 +29729 53785 +18444 53782 +34047 53777 +28746 53774 +35159 53772 +23989 53771 +16702 53767 +34470 53762 +9767 53761 +21875 53758 +36909 53757 +42077 53751 +19641 53747 +40497 53744 +30767 53737 +15106 53732 +30220 53732 +37009 53730 +29154 53726 +13039 53723 +693 53718 +23069 53718 +29959 53714 +42211 53714 +26359 53711 +3655 53710 +30048 53694 +28790 53688 +28701 53687 +42427 53682 +33773 53681 +35196 53676 +42916 53673 +8544 53661 +37063 53660 +26893 53655 +44633 53654 +19156 53654 +35019 53645 +40355 53644 +23408 53643 +21378 53639 +29109 53638 +21252 53638 +16984 53632 +27078 53629 +39712 53628 +24896 53624 +17017 53619 +26788 53616 +38870 53615 +12072 53614 +35730 53613 +34216 53613 +43459 53609 +20247 53600 +19145 53600 +5738 53598 +28023 53598 +11505 53582 +30545 53572 +21949 53572 +22974 53571 +11802 53566 +26600 53561 +28932 53556 +23419 53553 +43022 53552 +26300 53550 +22711 53549 +1932 53546 +32650 53542 +36892 53536 +33846 53534 +34527 53531 +9453 53529 +24343 53528 +2133 53526 +26408 53521 +19482 53516 +40598 53513 +34858 53513 +29944 53509 +23499 53509 +40332 53509 +13653 53506 +30135 53503 +29724 53501 +19648 53498 +49383 53497 +45483 53495 +4344 53489 +26837 53478 +35413 53477 +30630 53469 +25025 53464 +40924 53457 +30686 53455 +34292 53453 +29932 53445 +32693 53445 +40447 53439 +44886 53438 +25607 53437 +47461 53435 +46233 53432 +45680 53429 +45777 53428 +20225 53424 +25805 53422 +42568 53421 +34491 53420 +19708 53419 +39028 53418 +33506 53415 +23304 53414 +27804 53410 +36747 53397 +29219 53395 +28678 53393 +26910 53391 +35583 53388 +38264 53387 +45517 53387 +27926 53385 +7181 53382 +45088 53382 +48854 53378 +28805 53378 +31390 53376 +41027 53375 +22890 53369 +28898 53368 +29576 53368 +44178 53364 +21654 53362 +24764 53356 +49019 53352 +24977 53350 +49134 53349 +14120 53348 +48886 53346 +45263 53346 +29117 53341 +24502 53339 +20107 53335 +37105 53333 +33565 53329 +24691 53326 +21870 53320 +47756 53316 +40377 53314 +37896 53313 +42645 53308 +22910 53306 +38934 53284 +26770 53282 +38039 53278 +12398 53275 +37406 53273 +29481 53269 +10244 53259 +35138 53258 +25596 53250 +22464 53242 +48209 53237 +14344 53233 +46269 53231 +45676 53229 +16149 53216 +15200 53214 +6689 53213 +27433 53210 +26074 53207 +47216 53205 +29762 53200 +15952 53198 +7249 53195 +26306 53194 +26827 53185 +28554 53185 +22495 53185 +2226 53182 +35023 53182 +31645 53181 +23199 53180 +27482 53178 +34599 53178 +20191 53177 +33067 53174 +44404 53171 +34666 53171 +30792 53170 +47974 53170 +16886 53170 +32150 53168 +50151 53164 +34070 53163 +39133 53148 +22577 53145 +43588 53145 +25503 53145 +24556 53143 +26270 53130 +37890 53124 +12514 53120 +17379 53106 +13200 53101 +25610 53099 +34671 53093 +140 53091 +45078 53087 +27654 53083 +21910 53078 +42404 53064 +18232 53057 +28669 53054 +46929 53051 +19329 53049 +25336 53042 +41746 53036 +34304 53034 +43092 53028 +22478 53022 +20465 53014 +32420 53012 +22042 53003 +12067 53001 +41535 53000 +27601 52999 +36364 52989 +39127 52985 +5569 52983 +28436 52975 +13450 52973 +35287 52972 +36601 52968 +28947 52967 +29028 52961 +47714 52961 +21467 52957 +244 52954 +29624 52945 +20845 52944 +34502 52941 +35980 52941 +30392 52936 +28005 52935 +26926 52930 +23536 52926 +48320 52918 +40471 52914 +29030 52908 +44870 52900 +45183 52897 +47499 52897 +2009 52895 +26133 52884 +29883 52883 +31445 52877 +25754 52873 +34579 52856 +43809 52853 +40583 52852 +41074 52852 +36444 52850 +31405 52844 +15087 52843 +43657 52840 +21905 52838 +31363 52837 +48498 52832 +42453 52829 +21129 52825 +39730 52824 +16843 52823 +32104 52822 +16072 52816 +44588 52815 +31541 52815 +12154 52809 +20219 52802 +34383 52800 +16021 52799 +5343 52799 +14355 52789 +39083 52788 +24109 52785 +39569 52775 +20029 52769 +46133 52765 +41731 52761 +12628 52748 +40753 52744 +10489 52743 +24235 52736 +34234 52736 +28518 52736 +22659 52725 +43738 52724 +27360 52720 +33667 52719 +35918 52717 +6826 52717 +30301 52717 +22817 52715 +426 52710 +34260 52700 +35782 52699 +43524 52693 +7289 52692 +26708 52692 +32325 52690 +39465 52687 +44610 52686 +1391 52686 +22624 52685 +32162 52667 +46034 52666 +40847 52662 +42805 52661 +18896 52661 +14400 52660 +37928 52659 +18673 52648 +23146 52640 +27429 52639 +22416 52632 +40212 52625 +33719 52625 +33305 52619 +21616 52619 +27544 52595 +21940 52589 +28251 52588 +9076 52588 +40789 52577 +28007 52574 +42058 52572 +33906 52563 +25138 52560 +33050 52558 +25879 52557 +47535 52556 +39983 52546 +22629 52543 +25304 52537 +40302 52527 +31609 52523 +21937 52520 +14849 52518 +18754 52518 +39435 52515 +48378 52513 +21874 52510 +21696 52510 +32673 52504 +45215 52500 +35510 52499 +45910 52497 +27834 52496 +33731 52489 +35118 52483 +28308 52479 +33757 52476 +30500 52475 +28610 52464 +41559 52463 +24680 52451 +44663 52449 +32932 52449 +17472 52447 +31177 52443 +43934 52431 +16442 52430 +48115 52429 +20348 52427 +40851 52424 +28131 52421 +33364 52420 +21103 52419 +43348 52417 +19184 52416 +27152 52410 +22672 52409 +45059 52406 +40151 52402 +4443 52397 +49270 52393 +36000 52391 +29541 52391 +23444 52390 +13147 52384 +46837 52383 +29622 52383 +23609 52379 +35239 52370 +36317 52362 +2255 52360 +2162 52359 +11275 52357 +49362 52356 +30791 52352 +27406 52346 +31825 52344 +42152 52339 +24982 52336 +22124 52336 +28739 52334 +19545 52332 +32783 52327 +41772 52322 +24813 52321 +31638 52319 +35146 52312 +10624 52311 +20032 52310 +50050 52305 +45867 52302 +48249 52302 +32631 52298 +8506 52297 +33158 52291 +32063 52290 +20370 52286 +19948 52285 +3172 52285 +19473 52284 +49661 52284 +23656 52278 +32262 52270 +23908 52268 +9943 52263 +33093 52257 +21311 52257 +32784 52257 +32554 52256 +37358 52253 +32911 52252 +41077 52248 +31969 52246 +26749 52244 +33348 52244 +37370 52235 +42724 52234 +23127 52234 +41546 52229 +9741 52228 +24869 52222 +29574 52220 +45594 52219 +47905 52217 +41594 52214 +22234 52206 +34180 52205 +24297 52203 +40476 52201 +23871 52200 +14885 52199 +15245 52190 +7086 52188 +24727 52183 +28581 52180 +6845 52180 +21379 52176 +24354 52173 +11042 52171 +30645 52170 +11712 52166 +14656 52159 +40879 52156 +22209 52156 +47318 52152 +46212 52149 +42433 52143 +21600 52139 +11883 52139 +21657 52137 +25928 52133 +36912 52131 +38252 52126 +35025 52123 +22450 52118 +36846 52117 +29617 52111 +42710 52110 +30993 52108 +26339 52103 +48334 52095 +20135 52089 +19058 52087 +47058 52085 +21671 52085 +21972 52077 +41085 52076 +43276 52074 +42236 52065 +36455 52063 +38479 52063 +34762 52062 +31967 52061 +26760 52058 +24653 52055 +24551 52052 +31420 52051 +32598 52048 +13034 52046 +20549 52045 +10743 52042 +16455 52038 +1905 52035 +8172 52034 +21717 52032 +42888 52026 +34172 52023 +13240 52016 +17164 52015 +27124 52015 +9083 52013 +32857 52002 +39732 52001 +25090 52000 +32837 51993 +17310 51987 +27089 51983 +28939 51981 +21095 51980 +35107 51967 +28139 51965 +35865 51963 +43423 51955 +23462 51949 +46584 51949 +26179 51944 +28030 51942 +40084 51942 +5240 51939 +20764 51937 +46595 51936 +31328 51936 +35849 51934 +45415 51932 +35031 51930 +37089 51917 +43867 51914 +25441 51904 +19918 51896 +13660 51895 +27394 51894 +42640 51893 +48576 51891 +24652 51885 +47647 51872 +34537 51870 +20123 51868 +19973 51868 +25049 51866 +38458 51864 +32931 51863 +33332 51860 +30821 51858 +38309 51853 +29853 51842 +44194 51831 +25237 51830 +21440 51828 +40676 51823 +44703 51823 +25209 51822 +19617 51821 +28504 51820 +22440 51819 +11858 51811 +38207 51809 +35690 51803 +22583 51802 +31665 51800 +23395 51795 +38695 51794 +36134 51789 +47983 51788 +30170 51788 +41675 51786 +17550 51784 +40500 51782 +9994 51782 +22521 51779 +34477 51778 +21963 51777 +47972 51777 +37237 51767 +25917 51765 +33473 51765 +22850 51763 +31933 51763 +19835 51763 +37789 51761 +31874 51757 +36780 51755 +1599 51750 +37468 51740 +29366 51740 +36289 51731 +34392 51731 +47290 51722 +33259 51719 +16658 51714 +32384 51713 +17941 51713 +14390 51703 +23904 51703 +19955 51700 +38736 51694 +42609 51694 +33865 51693 +40426 51673 +7005 51663 +11590 51655 +38597 51652 +20418 51644 +14020 51643 +33401 51643 +44956 51638 +32543 51626 +48722 51623 +36426 51621 +31888 51616 +4728 51615 +48946 51612 +19971 51611 +5809 51604 +34426 51599 +19619 51599 +43976 51593 +15288 51591 +31378 51588 +33841 51587 +27073 51583 +45552 51582 +28719 51582 +33275 51581 +23169 51580 +33134 51573 +27978 51572 +18942 51572 +49068 51570 +28178 51569 +33575 51569 +20389 51560 +8140 51560 +39246 51558 +26942 51555 +1991 51554 +12405 51541 +33002 51539 +34358 51531 +46795 51531 +4555 51518 +36283 51518 +31236 51518 +11600 51509 +42729 51506 +9171 51503 +25579 51502 +814 51501 +49408 51493 +13341 51489 +41575 51489 +41698 51484 +23469 51467 +47680 51463 +15612 51463 +27979 51458 +50054 51457 +44459 51457 +37557 51452 +18738 51450 +27795 51448 +8010 51447 +30580 51441 +21174 51434 +45978 51434 +30878 51425 +32283 51425 +44537 51420 +42179 51416 +30059 51408 +29567 51407 +25643 51404 +24976 51403 +26507 51402 +45316 51399 +36775 51395 +47706 51393 +27635 51393 +45744 51392 +17831 51391 +39561 51387 +17993 51384 +13358 51383 +49474 51382 +32379 51378 +40124 51370 +26466 51368 +21386 51368 +13563 51360 +49470 51355 +26666 51353 +31935 51352 +30594 51351 +10455 51345 +31663 51333 +46413 51331 +27001 51329 +45047 51329 +24826 51328 +36224 51313 +27062 51308 +47451 51304 +44882 51300 +36168 51297 +38290 51296 +46184 51291 +36129 51291 +22818 51284 +46892 51280 +18347 51273 +20106 51271 +3983 51270 +25326 51268 +38916 51261 +16724 51260 +46685 51260 +14059 51259 +31822 51256 +25337 51255 +13776 51255 +36464 51245 +45962 51245 +47474 51241 +16961 51238 +10293 51234 +25829 51227 +32959 51227 +44413 51224 +47437 51223 +16240 51217 +21786 51209 +9844 51209 +29754 51205 +45651 51197 +19970 51179 +22134 51177 +9339 51165 +34996 51164 +37319 51155 +24431 51155 +17489 51153 +47225 51146 +20809 51141 +28942 51141 +33733 51140 +30274 51139 +30346 51138 +36856 51134 +19365 51129 +26405 51124 +28053 51104 +29227 51103 +13247 51102 +24555 51100 +25080 51088 +18486 51085 +23983 51085 +24900 51081 +39386 51079 +21526 51079 +25761 51078 +35294 51075 +28621 51071 +30137 51062 +34557 51061 +34501 51061 +41715 51057 +15370 51052 +30436 51050 +31582 51049 +41088 51049 +39774 51047 +50214 51040 +25916 51040 +44322 51029 +38656 51025 +40794 51024 +37194 51021 +27627 51020 +16939 51015 +24931 51014 +29440 51010 +32306 51008 +37803 51007 +34246 51006 +41135 51002 +35949 50997 +29296 50997 +1066 50992 +2738 50991 +36250 50981 +38026 50975 +48133 50974 +40603 50972 +22308 50965 +36433 50954 +17328 50953 +9843 50953 +32090 50951 +5251 50944 +44851 50934 +23044 50932 +28750 50925 +34130 50924 +11141 50919 +28241 50917 +20706 50916 +3590 50913 +24953 50912 +24417 50908 +38000 50905 +44044 50905 +35864 50898 +7146 50894 +27826 50894 +8021 50891 +48694 50888 +48756 50887 +35595 50885 +36312 50870 +25291 50855 +15547 50852 +31459 50850 +44998 50845 +41191 50841 +9219 50826 +16437 50813 +32792 50812 +14878 50805 +7512 50797 +25037 50796 +30158 50787 +23534 50784 +46760 50782 +12215 50775 +40304 50772 +45926 50771 +14535 50770 +48470 50762 +33586 50761 +8715 50761 +34678 50761 +34531 50760 +39636 50758 +149 50753 +14587 50750 +24570 50749 +15701 50748 +48684 50747 +38494 50732 +34933 50731 +45221 50726 +28471 50726 +29097 50723 +41329 50723 +39554 50718 +48664 50712 +32766 50697 +28210 50694 +6519 50692 +7053 50688 +17966 50686 +28895 50684 +39517 50683 +33836 50671 +14753 50663 +35280 50660 +8422 50654 +18424 50653 +7489 50648 +21766 50645 +49554 50637 +34890 50631 +14677 50628 +41598 50626 +24550 50623 +40975 50623 +14250 50622 +22334 50622 +18407 50613 +7860 50598 +26298 50597 +38336 50596 +17139 50591 +34723 50591 +19260 50591 +32244 50586 +19950 50581 +32740 50577 +48506 50573 +37843 50572 +23568 50571 +15513 50571 +27351 50564 +30778 50556 +37671 50554 +21081 50550 +21973 50548 +8927 50545 +23016 50544 +26633 50537 +2519 50537 +2452 50533 +46985 50530 +29176 50521 +11217 50514 +24140 50505 +45683 50504 +26434 50502 +42908 50502 +22881 50500 +37005 50484 +49411 50476 +25762 50476 +27318 50473 +28866 50470 +38236 50468 +23053 50467 +7528 50466 +30862 50464 +45808 50462 +12832 50455 +43943 50448 +44833 50448 +47930 50444 +7983 50440 +46117 50435 +37069 50427 +27150 50422 +12935 50421 +31132 50418 +37315 50417 +29387 50414 +28548 50413 +33091 50413 +36816 50412 +36497 50409 +16142 50409 +28754 50402 +27014 50395 +36494 50392 +46234 50389 +21044 50387 +29480 50380 +43183 50377 +27338 50373 +18833 50371 +31679 50370 +31109 50367 +27168 50366 +22654 50362 +33474 50355 +25047 50354 +30865 50354 +34291 50353 +33343 50348 +26250 50345 +36994 50342 +41379 50341 +26929 50336 +24464 50328 +35515 50322 +26477 50320 +33955 50315 +14633 50314 +47908 50314 +24218 50313 +36628 50313 +37953 50310 +22924 50309 +41494 50309 +28655 50296 +26881 50289 +28135 50289 +43081 50275 +30810 50272 +30959 50261 +20566 50259 +21592 50255 +22008 50255 +27164 50249 +44127 50245 +49939 50241 +44216 50237 +40996 50230 +19291 50228 +22463 50225 +31295 50220 +47528 50218 +14007 50214 +46106 50210 +31354 50208 +26413 50206 +36569 50204 +28993 50199 +21072 50194 +36701 50191 +22969 50191 +32697 50186 +10915 50183 +28806 50178 +1796 50175 +22150 50168 +28560 50164 +16601 50150 +6975 50149 +6315 50149 +17651 50140 +43268 50139 +37930 50136 +20458 50134 +34154 50133 +27460 50129 +43029 50127 +6728 50123 +18217 50117 +28368 50114 +29835 50107 +44209 50105 +29642 50098 +40947 50092 +38625 50091 +48195 50079 +35259 50077 +29000 50077 +22524 50075 +20402 50071 +43070 50067 +10228 50059 +25474 50054 +22960 50054 +26934 50051 +39538 50048 +14298 50043 +23560 50040 +45773 50032 +29358 50031 +3678 50028 +22029 50028 +25505 50024 +36195 50015 +24757 50015 +28498 50006 +9492 50005 +20812 50005 +49523 49984 +39604 49984 +25985 49971 +14414 49966 +27367 49965 +4850 49956 +50224 49950 +29098 49944 +44038 49938 +46008 49937 +32909 49937 +11871 49933 +29997 49933 +7497 49929 +20647 49911 +15520 49911 +34252 49911 +30408 49909 +49653 49906 +34321 49904 +23313 49903 +37197 49902 +34929 49901 +29974 49898 +16704 49893 +27793 49884 +41777 49884 +9132 49873 +34342 49867 +20473 49864 +15951 49864 +18294 49863 +28009 49861 +19149 49861 +19833 49859 +36272 49857 +17003 49853 +17961 49851 +33307 49843 +2145 49840 +27114 49840 +12303 49838 +19730 49834 +35372 49831 +24641 49823 +24987 49820 +24298 49815 +33331 49815 +28002 49811 +19142 49808 +24650 49805 +43581 49805 +26783 49800 +28685 49797 +43543 49791 +30980 49791 +27499 49784 +46864 49783 +29076 49783 +25885 49779 +33941 49777 +22980 49777 +36665 49775 +21139 49763 +29380 49762 +27735 49761 +46562 49759 +30747 49759 +31555 49758 +45711 49758 +28545 49747 +21738 49746 +44654 49734 +36071 49732 +21632 49723 +39916 49723 +45132 49721 +18527 49720 +47479 49718 +39533 49714 +47246 49714 +18231 49708 +13560 49705 +10684 49703 +28389 49700 +5709 49699 +20916 49692 +43178 49690 +36714 49689 +11365 49683 +48064 49679 +44379 49675 +26332 49674 +27736 49672 +2037 49665 +30261 49665 +47725 49661 +30353 49656 +39567 49653 +33019 49650 +12418 49640 +27623 49639 +32984 49638 +23378 49635 +26279 49634 +20781 49629 +32724 49624 +49266 49623 +22603 49622 +25386 49617 +46988 49608 +41471 49598 +15140 49597 +31994 49596 +44579 49595 +45264 49589 +20728 49587 +47681 49584 +35223 49584 +28796 49577 +35454 49574 +36540 49572 +34069 49565 +31247 49565 +30367 49565 +21956 49560 +20596 49555 +25330 49548 +39620 49547 +33381 49545 +27625 49542 +24604 49522 +41987 49519 +7279 49518 +8170 49515 +31186 49508 +49769 49496 +28733 49492 +37648 49490 +43590 49484 +34355 49484 +28774 49483 +14435 49476 +4264 49476 +5786 49473 +27317 49473 +36240 49472 +42687 49470 +25783 49463 +42251 49460 +40973 49454 +44466 49448 +25110 49444 +29765 49442 +34613 49442 +30310 49441 +27997 49432 +36054 49431 +20909 49427 +43244 49427 +32520 49414 +30967 49413 +21212 49412 +38748 49408 +34597 49406 +38085 49403 +25856 49402 +46305 49400 +25439 49400 +10122 49386 +46834 49384 +29894 49383 +10341 49374 +13190 49373 +25743 49369 +45614 49367 +31331 49365 +46882 49365 +16994 49363 +48487 49361 +28887 49360 +23674 49356 +43039 49352 +31250 49350 +44135 49348 +36483 49342 +26467 49341 +12324 49341 +23385 49336 +5494 49330 +43974 49327 +24183 49324 +23059 49319 +22808 49316 +3385 49311 +39159 49311 +36197 49310 +24147 49304 +6924 49302 +8930 49301 +33045 49292 +19429 49288 +38591 49285 +25739 49285 +39476 49283 +29723 49269 +25069 49269 +36270 49268 +38376 49265 +42478 49265 +46855 49263 +42070 49262 +13208 49255 +30834 49251 +1784 49245 +33427 49243 +31719 49240 +14346 49240 +15885 49235 +46903 49229 +12428 49225 +35866 49223 +4872 49218 +13363 49213 +35135 49211 +12604 49209 +41391 49206 +34485 49198 +39735 49197 +24063 49190 +3097 49186 +36682 49185 +43144 49184 +19065 49181 +32344 49177 +25744 49174 +22557 49169 +33168 49166 +17482 49164 +32510 49162 +42815 49158 +25986 49154 +36686 49154 +43106 49152 +21045 49150 +36293 49150 +44188 49148 +17946 49128 +44636 49125 +28324 49121 +15783 49119 +32195 49119 +22716 49115 +18351 49113 +37726 49108 +29778 49103 +31070 49101 +16441 49100 +37731 49095 +45094 49089 +21152 49085 +28381 49076 +28068 49076 +41334 49076 +9044 49068 +16200 49064 +36110 49063 +5089 49060 +26006 49059 +24483 49044 +43400 49042 +24582 49038 +47755 49036 +28596 49032 +27843 49027 +48253 49023 +34205 49021 +50209 49013 +19742 49006 +42734 49005 +32600 49004 +39614 49000 +29373 48996 +29604 48993 +19694 48991 +47170 48989 +45759 48988 +28027 48981 +46930 48975 +23660 48974 +19796 48973 +30574 48968 +41716 48956 +42737 48956 +45588 48956 +26580 48950 +38408 48948 +47978 48944 +21201 48944 +36717 48942 +37632 48938 +14709 48933 +36294 48933 +28704 48933 +12212 48926 +29016 48916 +36212 48912 +46854 48910 +35495 48905 +25702 48904 +2867 48903 +36114 48901 +40343 48900 +34715 48895 +35183 48886 +31015 48884 +21123 48884 +16596 48880 +19112 48876 +30587 48871 +25859 48867 +27631 48852 +25791 48847 +29887 48846 +12920 48846 +16620 48844 +44043 48839 +24470 48838 +49132 48837 +38674 48837 +27441 48831 +23094 48828 +17721 48826 +10903 48825 +31274 48824 +22511 48824 +19620 48819 +21892 48813 +47817 48808 +18651 48808 +27045 48804 +19427 48799 +23334 48799 +31261 48798 +17336 48796 +48199 48791 +9787 48786 +37221 48772 +43579 48768 +36351 48764 +36242 48761 +43127 48760 +14253 48757 +35199 48750 +32772 48748 +25922 48745 +28854 48743 +30930 48743 +32219 48728 +28144 48724 +20198 48721 +19753 48720 +32793 48715 +48039 48715 +26664 48713 +25644 48711 +29708 48709 +26110 48708 +13972 48703 +39397 48694 +38914 48689 +19794 48689 +21680 48676 +37116 48673 +39005 48672 +22073 48657 +31608 48655 +29403 48647 +25242 48646 +33154 48641 +28202 48637 +26535 48633 +35094 48632 +37184 48631 +28952 48630 +35862 48624 +46516 48622 +29091 48620 +33764 48618 +33226 48617 +41220 48613 +4428 48613 +19542 48612 +33541 48610 +38034 48608 +31983 48606 +22057 48597 +22746 48593 +27241 48574 +25533 48572 +4300 48571 +40611 48569 +46395 48566 +18949 48559 +29332 48557 +27630 48555 +37813 48554 +31597 48550 +43274 48548 +25602 48546 +34541 48540 +28025 48538 +24611 48538 +12415 48533 +13262 48531 +25459 48527 +49553 48527 +34874 48527 +46057 48527 +49372 48523 +38866 48509 +24778 48508 +34224 48505 +22437 48495 +45468 48490 +17975 48490 +24256 48482 +29031 48476 +37882 48475 +39163 48473 +40255 48472 +26811 48466 +36450 48465 +1852 48465 +27121 48464 +19367 48463 +19961 48459 +44518 48455 +44655 48454 +40793 48454 +12464 48453 +34384 48452 +34902 48450 +36172 48441 +16339 48441 +29079 48437 +10760 48437 +33469 48435 +27587 48432 +36776 48426 +35208 48425 +19054 48425 +25516 48424 +34078 48422 +41973 48414 +11122 48412 +38173 48400 +4584 48399 +34882 48397 +26277 48391 +34434 48390 +29683 48382 +7752 48377 +45983 48376 +40910 48374 +31848 48373 +39769 48366 +12222 48364 +49752 48364 +23726 48363 +17019 48362 +26366 48359 +37091 48357 +36991 48356 +49851 48341 +22374 48341 +47572 48339 +21309 48338 +23337 48332 +28262 48329 +29246 48326 +49598 48325 +26130 48321 +18690 48315 +38594 48314 +23487 48314 +12573 48312 +31017 48310 +45736 48307 +21273 48305 +8648 48300 +29355 48295 +41089 48293 +7672 48292 +28714 48291 +20661 48276 +34912 48270 +45568 48263 +8671 48262 +44485 48260 +2690 48257 +27912 48256 +45494 48256 +21645 48246 +36559 48242 +46567 48239 +30622 48235 +26557 48229 +44523 48227 +36960 48227 +19867 48223 +10461 48221 +17723 48219 +29549 48218 +44862 48218 +24904 48218 +45142 48216 +23100 48213 +37279 48209 +9887 48209 +46470 48205 +18230 48202 +45511 48202 +48850 48202 +39037 48199 +33119 48197 +24814 48195 +42792 48194 +9042 48189 +39685 48187 +23253 48186 +34276 48182 +43373 48177 +43120 48174 +36193 48173 +5090 48171 +49902 48167 +24894 48161 +36833 48160 +43663 48159 +37248 48150 +28799 48150 +42898 48146 +32656 48145 +39339 48140 +47199 48135 +31068 48133 +35455 48131 +21725 48119 +47916 48119 +41336 48115 +33271 48114 +33977 48108 +35008 48107 +49432 48106 +30383 48106 +24975 48102 +47773 48099 +23506 48097 +33133 48091 +36612 48090 +5306 48090 +36998 48065 +37073 48064 +20206 48062 +11934 48060 +32728 48056 +45425 48054 +16341 48052 +36281 48044 +17363 48035 +35465 48033 +15880 48030 +16357 48025 +21527 48023 +22275 48023 +33502 48015 +13672 48014 +41679 48012 +31986 48012 +28867 48011 +33755 48009 +24776 48006 +8357 48004 +37292 47998 +34746 47994 +37027 47992 +40519 47990 +36279 47980 +6417 47979 +36083 47978 +28456 47977 +31114 47970 +38527 47969 +34567 47968 +39302 47965 +45504 47964 +25600 47960 +32377 47960 +27712 47958 +33424 47957 +10811 47948 +28771 47935 +35262 47924 +26539 47917 +29282 47917 +3643 47915 +39412 47915 +18255 47913 +14750 47910 +38088 47905 +6398 47904 +40470 47903 +45246 47901 +40662 47899 +24504 47899 +41229 47898 +49292 47898 +46160 47896 +35309 47896 +47329 47892 +40453 47890 +26352 47889 +22682 47885 +21262 47885 +39441 47884 +27238 47884 +28378 47879 +14132 47878 +21969 47877 +31533 47876 +33851 47875 +30582 47875 +22342 47874 +15075 47874 +40955 47871 +35332 47870 +27244 47864 +26181 47863 +22832 47858 +43956 47853 +29846 47852 +2697 47844 +20666 47844 +26281 47843 +18268 47843 +9928 47836 +2083 47836 +32350 47834 +37342 47831 +21315 47829 +27132 47825 +15809 47819 +38987 47819 +34375 47811 +47668 47809 +41836 47806 +25284 47803 +17273 47802 +39862 47800 +16311 47791 +1476 47789 +47853 47786 +39027 47783 +48213 47783 +8012 47782 +33063 47765 +27445 47759 +34849 47745 +41970 47743 +42170 47742 +42876 47742 +1161 47739 +36742 47733 +47151 47729 +13115 47725 +18301 47725 +35272 47723 +27219 47708 +3416 47707 +28177 47699 +38738 47699 +29770 47687 +26465 47686 +38383 47685 +22069 47680 +21456 47679 +14174 47674 +23539 47667 +46247 47665 +34459 47663 +22179 47659 +22449 47658 +42791 47654 +39603 47653 +50169 47652 +26667 47651 +44498 47645 +44868 47642 +28876 47640 +26701 47638 +5109 47637 +19327 47637 +31064 47634 +30573 47633 +35905 47625 +45232 47624 +43616 47621 +28217 47618 +48642 47617 +27086 47617 +42992 47611 +42327 47611 +23688 47609 +26282 47598 +36099 47596 +40852 47596 +43766 47594 +33297 47593 +49466 47591 +17209 47585 +43670 47582 +24748 47581 +32353 47571 +48342 47569 +38757 47561 +32905 47561 +32085 47556 +9250 47542 +35729 47542 +37663 47540 +16112 47538 +33694 47537 +31556 47535 +28277 47531 +25104 47526 +38429 47525 +39074 47524 +40404 47523 +46739 47521 +28940 47519 +27329 47518 +23753 47514 +29041 47510 +31267 47506 +35163 47506 +18150 47505 +33159 47503 +24811 47492 +22148 47488 +25342 47487 +38812 47487 +39283 47486 +25808 47483 +15486 47483 +12808 47482 +30103 47481 +19159 47481 +32572 47477 +30430 47475 +25881 47464 +37486 47459 +27506 47446 +28334 47432 +21561 47418 +18801 47417 +30180 47414 +39383 47411 +25587 47410 +28091 47408 +41626 47405 +47228 47403 +29419 47402 +1480 47400 +30363 47399 +33319 47394 +43558 47393 +27946 47391 +16623 47391 +37749 47390 +31271 47390 +18619 47388 +44003 47374 +21458 47373 +29181 47371 +38119 47362 +31807 47362 +10963 47356 +28060 47355 +46922 47354 +37331 47351 +27577 47345 +32884 47333 +22515 47327 +19853 47322 +19354 47319 +40431 47314 +16275 47311 +5915 47310 +44339 47303 +35489 47299 +43398 47294 +35096 47293 +18799 47293 +46577 47288 +29005 47287 +25534 47286 +27657 47281 +20298 47279 +42299 47272 +27613 47271 +29341 47270 +45723 47268 +38908 47263 +37164 47253 +38260 47252 +1251 47243 +47848 47234 +35817 47231 +28453 47230 +28878 47227 +5065 47223 +20214 47217 +13220 47216 +25649 47214 +38186 47210 +46822 47209 +34693 47207 +17827 47207 +19520 47204 +19756 47202 +24249 47201 +32514 47200 +2844 47198 +49416 47195 +36874 47186 +39962 47184 +37800 47183 +30669 47181 +43100 47180 +25671 47175 +15450 47174 +28888 47160 +34079 47156 +19484 47156 +42274 47154 +6278 47154 +37793 47153 +33816 47153 +31693 47149 +47218 47148 +33858 47146 +40689 47137 +21287 47137 +40105 47133 +30571 47133 +32402 47132 +38009 47131 +24294 47129 +22930 47120 +19707 47115 +34941 47114 +42425 47112 +19498 47111 +28609 47109 +34134 47104 +44056 47100 +19976 47094 +17678 47084 +47562 47083 +15310 47073 +16458 47068 +37228 47064 +23756 47063 +15590 47060 +19355 47056 +42906 47056 +45512 47056 +32529 47052 +11593 47052 +31528 47052 +21713 47051 +28880 47045 +20800 47043 +22291 47033 +40511 47032 +48569 47016 +48060 47011 +43799 47003 +20113 47001 +36579 46998 +36748 46996 +34263 46996 +49520 46994 +38686 46986 +35177 46971 +42967 46970 +14336 46967 +45184 46965 +47413 46965 +34789 46965 +45139 46964 +3065 46962 +23921 46961 +28687 46955 +32654 46950 +44738 46946 +35153 46942 +32761 46937 +17647 46935 +46150 46934 +38761 46931 +35355 46921 +12454 46913 +30876 46903 +3552 46902 +27458 46902 +23277 46901 +10465 46898 +35662 46897 +16643 46895 +41794 46895 +39524 46892 +42899 46892 +39164 46887 +36101 46879 +11370 46878 +12221 46873 +32293 46870 +29641 46869 +28424 46863 +28309 46861 +13775 46860 +39767 46856 +29343 46855 +34155 46853 +35932 46852 +46875 46850 +50173 46846 +35426 46843 +7615 46840 +42004 46840 +46447 46837 +28786 46834 +37048 46829 +32122 46829 +29280 46827 +1091 46827 +10709 46824 +42571 46820 +30468 46812 +35526 46807 +28807 46786 +6059 46785 +9313 46784 +43652 46779 +39640 46776 +48568 46764 +5957 46763 +25194 46763 +15432 46761 +50186 46760 +26135 46759 +36360 46755 +11019 46751 +30097 46749 +27952 46749 +32808 46748 +3935 46742 +25584 46738 +47377 46725 +26996 46725 +24134 46725 +24709 46720 +23585 46717 +36865 46715 +31910 46710 +16854 46708 +36093 46703 +48853 46698 +19900 46698 +43104 46694 +31349 46692 +41238 46686 +21521 46678 +49412 46678 +44026 46677 +45332 46677 +14406 46675 +25371 46664 +43257 46655 +33656 46655 +46861 46652 +27201 46649 +24944 46646 +29101 46642 +32343 46636 +17999 46634 +21306 46629 +32294 46629 +49104 46628 +23581 46624 +30875 46614 +48794 46612 +45645 46611 +45516 46610 +32396 46608 +20922 46608 +38406 46606 +11321 46605 +49612 46603 +35441 46600 +49190 46592 +9805 46592 +48701 46584 +30968 46582 +30066 46577 +46340 46575 +31377 46575 +35901 46567 +22208 46565 +21232 46562 +19108 46557 +45542 46557 +21194 46551 +35631 46546 +42580 46546 +29619 46542 +40943 46540 +44614 46537 +26957 46535 +32393 46532 +27377 46527 +47219 46524 +26975 46524 +24415 46523 +19680 46517 +40530 46513 +30811 46500 +30264 46491 +37054 46491 +18011 46489 +23250 46486 +21499 46482 +37818 46472 +22032 46469 +19831 46467 +34498 46463 +38833 46459 +12732 46458 +25710 46456 +29433 46452 +31207 46449 +47272 46447 +34569 46444 +33018 46438 +5530 46432 +18040 46428 +39891 46423 +28033 46423 +47353 46423 +33471 46415 +16678 46414 +35941 46414 +34191 46402 +18970 46402 +15183 46398 +17220 46397 +35180 46394 +13665 46383 +23126 46383 +32556 46382 +23087 46378 +47295 46377 +47196 46366 +31043 46364 +8110 46360 +13494 46358 +40698 46356 +40363 46355 +31168 46353 +35182 46351 +11143 46351 +34935 46346 +34122 46340 +27791 46338 +31172 46333 +40702 46330 +6172 46330 +16682 46322 +49597 46319 +23564 46319 +28590 46318 +31894 46316 +32542 46310 +24299 46303 +23214 46302 +6202 46299 +25598 46299 +13350 46299 +30548 46293 +49825 46288 +40062 46287 +34862 46283 +21163 46283 +20637 46283 +47560 46276 +30381 46274 +11599 46272 +27717 46270 +6120 46269 +21242 46269 +20025 46268 +33431 46263 +23447 46262 +36319 46260 +46980 46258 +32806 46258 +15008 46253 +29950 46251 +33570 46249 +14894 46239 +26761 46237 +19270 46234 +32921 46233 +42818 46228 +7671 46225 +37644 46222 +27254 46222 +47984 46221 +32729 46216 +2909 46212 +40657 46210 +29621 46209 +33713 46208 +14106 46206 +26034 46200 +22204 46196 +14265 46195 +11035 46193 +42437 46189 +44302 46189 +49967 46178 +44301 46177 +22633 46176 +40490 46165 +36930 46164 +8320 46159 +18093 46152 +11528 46151 +37026 46151 +35045 46149 +35370 46147 +30168 46146 +4772 46145 +43747 46137 +41130 46132 +34076 46130 +35088 46124 +16505 46123 +37428 46120 +20638 46115 +43007 46112 +43944 46109 +16258 46106 +22567 46098 +36214 46098 +23807 46093 +46972 46091 +9939 46091 +25384 46088 +29283 46087 +37293 46082 +7296 46079 +26707 46079 +36513 46072 +26173 46070 +32794 46069 +37891 46068 +42886 46058 +28328 46053 +44131 46048 +29397 46040 +42232 46039 +40741 46034 +26000 46034 +12205 46032 +37457 46032 +20964 46031 +19252 46030 +26753 46030 +32479 46029 +39078 46024 +25419 46023 +27642 46023 +45827 46017 +42097 46016 +32773 46015 +24227 46013 +35842 46009 +49300 46004 +32204 46002 +37554 46000 +23589 45991 +45046 45989 +48742 45987 +30355 45987 +34038 45986 +15307 45982 +39784 45980 +39987 45978 +39956 45976 +28388 45976 +22823 45976 +26473 45974 +22024 45969 +48437 45964 +38960 45958 +39716 45952 +31545 45952 +18458 45949 +45344 45949 +33561 45942 +34875 45941 +15861 45939 +32120 45939 +35639 45935 +32769 45930 +26990 45927 +43953 45925 +28954 45923 +38540 45916 +50172 45916 +40033 45912 +31448 45909 +39067 45905 +44769 45904 +29094 45900 +44768 45900 +18638 45900 +47244 45896 +29722 45892 +39960 45887 +46330 45887 +24372 45887 +16761 45865 +28107 45865 +8154 45864 +28031 45863 +36111 45861 +47950 45860 +32894 45859 +13908 45858 +47513 45857 +34151 45856 +27404 45855 +20453 45853 +14214 45848 +20751 45847 +43066 45844 +48901 45838 +20454 45834 +28271 45830 +35988 45827 +17134 45825 +11669 45820 +34243 45820 +47722 45818 +26471 45818 +39131 45806 +28448 45805 +46033 45802 +34316 45800 +20300 45799 +37676 45798 +34195 45795 +7550 45791 +26063 45787 +39420 45776 +30232 45771 +28183 45769 +26641 45765 +44818 45764 +40192 45764 +43176 45763 +36145 45762 +12942 45758 +26463 45756 +45878 45751 +26153 45749 +12192 45748 +37347 45747 +20979 45745 +44846 45740 +24843 45739 +42324 45735 +44220 45734 +28858 45732 +28351 45729 +27151 45728 +35011 45726 +26592 45726 +39235 45724 +27198 45724 +21721 45721 +37975 45715 +29442 45714 +17860 45699 +36383 45699 +23268 45698 +38754 45697 +37900 45696 +48709 45695 +37261 45695 +9926 45687 +45238 45687 +34152 45684 +2397 45683 +26333 45682 +37550 45681 +40966 45675 +31151 45673 +27769 45670 +8687 45664 +30185 45662 +27412 45661 +15787 45653 +44820 45649 +33536 45648 +33900 45647 +38705 45646 +25417 45642 +21767 45642 +22362 45640 +39678 45636 +50220 45628 +35073 45627 +41523 45625 +14065 45624 +24344 45624 +43153 45623 +31749 45622 +44652 45618 +36058 45615 +45413 45615 +39453 45611 +40641 45609 +27126 45601 +40136 45601 +36910 45601 +49628 45598 +32920 45597 +43492 45595 +49323 45589 +18005 45585 +29211 45582 +21762 45581 +49796 45577 +23831 45574 +36308 45571 +27758 45570 +40835 45569 +34486 45557 +37234 45553 +11720 45552 +34885 45548 +31629 45546 +46644 45537 +27253 45531 +26919 45525 +22136 45522 +29924 45521 +31489 45520 +49987 45520 +22564 45510 +29546 45510 +11537 45509 +10415 45508 +19259 45507 +24572 45499 +12304 45497 +40606 45494 +47748 45493 +42329 45492 +33790 45490 +15854 45490 +24170 45487 +45958 45477 +18076 45476 +41740 45473 +23550 45470 +38854 45468 +16183 45461 +41113 45458 +11055 45456 +35940 45455 +36394 45452 +21064 45451 +46544 45449 +9673 45449 +31059 45449 +9548 45448 +36769 45446 +28737 45442 +40307 45440 +23672 45439 +34663 45437 +23162 45434 +38689 45431 +42139 45426 +45200 45424 +40900 45424 +17620 45423 +38878 45422 +47657 45421 +39493 45401 +33182 45401 +19467 45398 +30795 45397 +16260 45391 +32367 45390 +35550 45386 +49341 45385 +40461 45380 +18219 45374 +8638 45371 +26484 45361 +21447 45360 +19941 45353 +10554 45352 +1823 45347 +29369 45345 +24702 45344 +13255 45341 +15098 45339 +29462 45334 +39373 45334 +41529 45333 +36941 45332 +23945 45328 +16010 45325 +50028 45324 +41928 45319 +21396 45318 +27162 45318 +49655 45310 +37222 45308 +26526 45303 +19852 45300 +27742 45296 +33105 45293 +4247 45291 +47410 45289 +37327 45285 +22717 45285 +46958 45282 +41053 45278 +20109 45272 +25514 45268 +6425 45263 +23272 45241 +21856 45239 +27036 45239 +24168 45230 +43939 45229 +15379 45223 +21352 45223 +25903 45223 +26949 45221 +30019 45218 +31130 45216 +41484 45215 +15125 45210 +32993 45207 +25685 45203 +38048 45200 +16911 45199 +45049 45195 +38932 45191 +9143 45188 +35794 45186 +37743 45183 +42186 45178 +33464 45174 +19172 45170 +17594 45164 +30551 45164 +36288 45164 +17133 45162 +27371 45162 +26093 45154 +48335 45154 +24309 45147 +28747 45147 +31991 45146 +38862 45145 +28978 45145 +29062 45144 +26887 45143 +25591 45143 +8145 45141 +29735 45139 +35906 45138 +23426 45132 +42850 45127 +36098 45122 +22035 45122 +35822 45120 +47573 45116 +26692 45113 +2361 45108 +15878 45107 +48687 45101 +24513 45096 +33270 45094 +22458 45092 +5145 45089 +26704 45087 +38378 45077 +26822 45071 +21515 45070 +43533 45069 +25830 45065 +42753 45063 +6940 45049 +28192 45048 +34683 45045 +32123 45042 +10035 45039 +44486 45035 +5042 45034 +29051 45032 +40859 45030 +7580 45025 +40687 45024 +30506 45013 +28959 45012 +42802 45012 +32816 45007 +22826 44996 +9474 44990 +31955 44989 +33187 44989 +49232 44988 +31202 44986 +9781 44978 +39076 44977 +39308 44974 +34965 44971 +24094 44964 +228 44960 +30549 44958 +23175 44953 +35613 44952 +17727 44938 +29985 44937 +11369 44930 +48604 44924 +22210 44923 +47877 44920 +10698 44909 +25797 44905 +20414 44904 +34030 44904 +27666 44903 +40904 44898 +39652 44897 +42828 44896 +27573 44894 +24288 44891 +49498 44886 +20485 44883 +24158 44882 +14558 44881 +17237 44879 +6293 44877 +22245 44876 +39679 44873 +13921 44870 +42490 44862 +38041 44860 +22911 44854 +46690 44854 +39128 44852 +39826 44851 +27109 44840 +23055 44839 +18875 44838 +19678 44838 +41520 44837 +5135 44836 +35584 44835 +30246 44828 +45222 44827 +24143 44827 +39411 44818 +49901 44817 +30559 44817 +48592 44815 +33963 44813 +23191 44808 +45892 44797 +30621 44795 +32154 44793 +25964 44793 +7949 44790 +34019 44787 +9243 44787 +1420 44786 +36703 44785 +42587 44782 +13131 44771 +46825 44767 +26820 44766 +9745 44766 +2498 44761 +42606 44758 +17817 44754 +18487 44752 +29524 44752 +32160 44750 +38189 44750 +39085 44745 +36587 44742 +10858 44739 +42378 44730 +31142 44728 +42495 44724 +19461 44719 +16806 44718 +15452 44716 +35873 44714 +39778 44711 +28801 44710 +26782 44709 +34480 44705 +33224 44703 +1269 44703 +32435 44702 +16364 44699 +36712 44699 +44913 44696 +11318 44691 +23537 44690 +11787 44689 +34903 44688 +21599 44683 +33896 44682 +27746 44679 +30568 44673 +30627 44670 +26576 44668 +43273 44658 +18107 44654 +29805 44652 +35429 44650 +49668 44648 +15411 44640 +47169 44633 +32581 44631 +37324 44631 +33344 44627 +25245 44625 +20084 44614 +33612 44606 +32167 44606 +19251 44605 +23684 44597 +7316 44597 +33732 44594 +23234 44593 +29086 44588 +12998 44588 +37378 44584 +31428 44584 +35212 44582 +47747 44581 +41630 44577 +26524 44575 +30086 44574 +16578 44571 +26978 44571 +47162 44567 +30844 44566 +4717 44565 +46662 44563 +14002 44556 +21259 44556 +9295 44554 +21441 44549 +33097 44546 +25498 44544 +30982 44543 +7921 44543 +27393 44541 +20914 44540 +27829 44539 +48073 44537 +22981 44534 +41909 44533 +232 44532 +34966 44528 +48028 44524 +31103 44522 +31450 44521 +22154 44521 +3924 44512 +17045 44511 +47830 44509 +43171 44509 +29697 44507 +46505 44502 +32049 44502 +21395 44500 +25307 44497 +45760 44496 +26774 44494 +22763 44486 +38094 44484 +34336 44481 +33852 44479 +46598 44478 +35278 44478 +36915 44475 +23454 44472 +6549 44471 +31648 44466 +22047 44464 +47533 44455 +48127 44454 +36582 44453 +38728 44453 +1184 44453 +23051 44443 +33189 44442 +38590 44435 +48965 44433 +32225 44432 +33418 44428 +41545 44427 +47399 44426 +25129 44425 +22194 44423 +9139 44423 +25929 44422 +32954 44421 +21062 44419 +48368 44416 +18206 44413 +15650 44407 +37829 44404 +27326 44404 +10441 44401 +26818 44398 +30416 44397 +1955 44397 +42917 44396 +37748 44394 +27688 44394 +38777 44394 +10487 44393 +41007 44390 +42610 44378 +35889 44373 +4607 44372 +15815 44371 +41218 44363 +28815 44362 +41980 44359 +38760 44357 +26243 44355 +20177 44355 +30004 44351 +24088 44348 +10121 44346 +45998 44341 +21282 44341 +48279 44340 +22651 44339 +45817 44335 +12754 44332 +23341 44328 +29262 44327 +26217 44324 +36619 44321 +18944 44321 +40459 44320 +50232 44317 +32235 44315 +28875 44314 +22790 44313 +46547 44313 +38176 44310 +8086 44307 +38962 44305 +40891 44299 +30796 44295 +25882 44295 +28826 44294 +35075 44292 +29183 44291 +33935 44290 +38868 44289 +16429 44288 +14418 44287 +43285 44280 +2249 44278 +24705 44274 +41985 44271 +25760 44270 +32630 44269 +14280 44268 +7390 44265 +41819 44261 +26422 44260 +22229 44256 +43150 44253 +28417 44252 +37437 44249 +46474 44245 +39835 44244 +42546 44238 +35471 44234 +10832 44232 +29643 44231 +25921 44227 +17042 44226 +48184 44221 +21497 44212 +25393 44210 +38883 44207 +27166 44207 +23109 44206 +14650 44198 +46673 44197 +40199 44184 +8866 44184 +22481 44183 +28994 44182 +20221 44181 +20305 44181 +27892 44179 +35173 44176 +23891 44174 +34290 44173 +28265 44166 +24487 44164 +32662 44164 +19822 44164 +43493 44160 +14922 44159 +18415 44158 +31137 44154 +29244 44153 +31706 44152 +28629 44152 +22068 44145 +35158 44145 +20766 44142 +36880 44140 +22413 44135 +23801 44135 +10539 44131 +13782 44130 +23671 44125 +23467 44124 +43842 44112 +35879 44109 +20969 44104 +40581 44101 +37692 44100 +31487 44096 +42575 44095 +35238 44095 +35245 44093 +36710 44091 +30530 44088 +41954 44085 +44795 44078 +13420 44076 +39167 44074 +36183 44073 +24857 44067 +9218 44066 +41719 44065 +12194 44063 +45830 44062 +38316 44057 +40772 44057 +26826 44049 +32481 44036 +37371 44030 +39857 44029 +33964 44028 +38794 44027 +46339 44023 +38951 44019 +37278 44017 +18938 44012 +40790 44012 +14902 44011 +33879 44009 +32114 44002 +19854 44002 +29962 44001 +41566 43997 +27405 43980 +33082 43978 +48695 43976 +19501 43972 +45024 43971 +19129 43971 +33260 43971 +43644 43970 +46218 43963 +29791 43962 +47041 43962 +32270 43961 +11285 43960 +44490 43956 +4276 43950 +39675 43948 +39438 43943 +39041 43943 +37029 43941 +26364 43936 +44752 43934 +29830 43928 +48918 43924 +21254 43920 +38291 43916 +46593 43910 +12009 43909 +30471 43901 +47600 43900 +18372 43897 +35424 43887 +37316 43881 +47096 43881 +38368 43877 +39917 43875 +25735 43874 +31283 43872 +28944 43871 +2338 43871 +25334 43867 +18761 43862 +22401 43856 +22507 43853 +32913 43850 +22172 43849 +46221 43847 +19700 43846 +27191 43844 +42458 43838 +28433 43838 +36263 43835 +24818 43828 +35501 43825 +19394 43824 +38775 43821 +35968 43820 +40568 43802 +38003 43798 +30247 43788 +24414 43787 +21533 43786 +29623 43783 +45129 43777 +37973 43774 +34624 43773 +38181 43771 +18016 43766 +31906 43758 +15546 43750 +31012 43748 +32240 43744 +49565 43741 +37660 43741 +46496 43736 +7944 43735 +36151 43734 +40045 43730 +49676 43728 +39824 43726 +42998 43725 +23163 43724 +16972 43722 +30365 43718 +16302 43716 +35534 43711 +36184 43711 +8102 43710 +33589 43709 +32776 43708 +48271 43698 +35473 43696 +16912 43694 +27064 43694 +32237 43692 +31210 43686 +27937 43684 +49906 43684 +38299 43677 +13583 43672 +15312 43671 +44180 43669 +41249 43662 +34099 43659 +37965 43652 +32222 43650 +48292 43647 +32813 43645 +28097 43641 +16185 43641 +28748 43636 +28541 43636 +25183 43632 +36324 43631 +36051 43631 +5280 43627 +31229 43626 +46982 43625 +36708 43624 +34221 43622 +33184 43621 +14868 43619 +20784 43609 +36334 43608 +45527 43607 +44200 43601 +15526 43601 +47766 43599 +24027 43598 +33593 43589 +34759 43587 +32018 43585 +19842 43584 +24605 43582 +36622 43577 +32309 43575 +13926 43563 +14421 43561 +12776 43559 +29563 43556 +31804 43553 +32469 43546 +16018 43542 +15559 43541 +36811 43540 +33729 43534 +48660 43533 +30502 43527 +25703 43522 +23457 43519 +5960 43519 +26956 43499 +36266 43496 +26578 43485 +29637 43484 +34740 43484 +24498 43483 +43457 43482 +35222 43480 +45354 43479 +28179 43476 +28765 43469 +40621 43464 +32067 43462 +18701 43462 +39770 43457 +20296 43457 +17841 43455 +18824 43455 +40677 43453 +16698 43452 +37416 43446 +42657 43444 +15092 43443 +41711 43441 +31856 43438 +25088 43436 +33875 43434 +42409 43433 +45789 43431 +16242 43428 +27936 43426 +45262 43425 +16706 43425 +24747 43423 +46301 43422 +48417 43418 +29899 43417 +43350 43416 +22619 43414 +27895 43410 +46427 43408 +17879 43406 +30477 43403 +29956 43397 +50146 43390 +47443 43389 +45235 43385 +34188 43384 +34127 43381 +42438 43381 +38553 43380 +7314 43380 +43715 43380 +30786 43377 +34884 43376 +49235 43376 +44792 43372 +33680 43370 +40856 43364 +41878 43363 +34170 43363 +31165 43361 +17323 43360 +18328 43358 +48257 43358 +27693 43351 +41893 43349 +34020 43343 +33884 43341 +28032 43339 +28403 43335 +29935 43334 +37317 43333 +32956 43329 +39552 43326 +28965 43322 +27773 43320 +34419 43317 +21333 43317 +22388 43311 +29217 43310 +8433 43307 +15597 43302 +18871 43296 +43833 43295 +38132 43288 +27472 43285 +44441 43285 +31182 43282 +44699 43282 +34837 43276 +35254 43274 +12789 43274 +20589 43271 +22141 43268 +26096 43267 +37247 43262 +26263 43259 +22922 43259 +29338 43255 +31669 43255 +44866 43252 +12305 43251 +37871 43250 +28006 43244 +42504 43242 +29482 43235 +48303 43232 +44972 43232 +47977 43222 +25527 43220 +6361 43218 +48614 43217 +18443 43212 +18228 43212 +26067 43212 +45391 43209 +27400 43206 +50058 43205 +26723 43204 +30184 43202 +33805 43197 +38096 43188 +41225 43184 +16890 43181 +31860 43173 +33516 43172 +36062 43170 +17011 43165 +34994 43163 +39673 43155 +39381 43152 +41756 43151 +32311 43150 +34454 43149 +35405 43148 +42841 43147 +45707 43144 +32995 43137 +22411 43133 +27257 43129 +44429 43129 +8333 43128 +31772 43127 +33292 43123 +42749 43121 +47143 43120 +41175 43119 +28003 43117 +38065 43116 +49462 43116 +45691 43112 +22555 43110 +44595 43110 +32721 43109 +45375 43107 +27780 43106 +26481 43105 +45715 43102 +31780 43099 +21684 43096 +43383 43096 +37432 43093 +34397 43093 +47129 43092 +25948 43087 +20791 43084 +48705 43082 +24322 43079 +33177 43076 +40586 43074 +24823 43067 +25674 43063 +7089 43061 +33763 43060 +43175 43053 +42048 43049 +31779 43046 +24339 43044 +17685 43044 +37451 43044 +40221 43043 +11429 43041 +47770 43039 +28680 43034 +18536 43032 +42116 43026 +7248 43023 +23005 43016 +33877 43012 +13789 43012 +38561 43010 +49756 43006 +40958 43006 +23777 43005 +25900 43004 +44268 43004 +28699 43001 +25135 43001 +27827 42996 +31505 42994 +4546 42994 +23875 42990 +50081 42989 +44941 42985 +27921 42970 +17937 42964 +35947 42961 +33619 42959 +44255 42956 +32423 42955 +31588 42954 +33655 42954 +43075 42953 +28168 42952 +37291 42950 +41269 42950 +21701 42949 +24698 42944 +25550 42944 +9401 42937 +37727 42928 +17119 42924 +37493 42919 +42689 42916 +25470 42915 +28384 42914 +46664 42912 +29368 42904 +44243 42900 +29464 42893 +44874 42892 +34087 42891 +26937 42890 +42410 42886 +33802 42883 +9660 42879 +27000 42870 +42694 42869 +13544 42864 +20636 42857 +22391 42854 +34147 42853 +23588 42852 +26731 42848 +40898 42848 +48256 42844 +18809 42842 +28846 42840 +42142 42835 +38091 42834 +26765 42828 +36641 42826 +29225 42825 +44316 42824 +29138 42819 +44717 42816 +36175 42816 +10155 42813 +26084 42813 +17072 42812 +24111 42810 +31866 42808 +40171 42806 +37625 42804 +16262 42804 +48784 42803 +15054 42801 +29877 42801 +50217 42796 +26460 42794 +49241 42794 +40502 42791 +39511 42791 +6583 42786 +24827 42784 +35456 42783 +29063 42783 +30445 42783 +38225 42779 +19248 42776 +31372 42769 +27798 42769 +38504 42768 +48666 42768 +28963 42766 +42020 42764 +24893 42754 +30079 42754 +27617 42753 +31534 42740 +22389 42738 +35581 42737 +35463 42737 +37367 42734 +36557 42731 +24409 42729 +33720 42727 +36589 42724 +21173 42721 +26832 42714 +47971 42712 +7115 42708 +23120 42706 +20977 42702 +29867 42701 +25713 42697 +40913 42690 +7878 42688 +43770 42687 +25669 42686 +21260 42680 +44630 42677 +37343 42676 +43259 42670 +48516 42669 +37040 42668 +34031 42666 +33068 42665 +23065 42665 +16756 42663 +48222 42657 +23355 42657 +23963 42656 +13441 42655 +49076 42653 +3566 42647 +28909 42644 +32024 42642 +23045 42629 +46699 42624 +42890 42623 +36004 42623 +36566 42622 +23297 42621 +22607 42621 +49852 42618 +29407 42618 +19213 42617 +26607 42616 +13169 42606 +47018 42602 +25993 42602 +44756 42599 +16676 42594 +32692 42591 +37245 42590 +27188 42589 +19071 42588 +37784 42586 +39408 42585 +44554 42585 +25156 42581 +29874 42578 +39536 42578 +44932 42570 +40936 42567 +42774 42565 +36630 42564 +40854 42561 +28055 42556 +31464 42554 +44147 42548 +30011 42548 +32415 42547 +10210 42541 +36355 42541 +44524 42539 +34259 42538 +3922 42538 +30253 42534 +1279 42534 +34940 42533 +27877 42532 +1926 42531 +8633 42531 +36808 42529 +22645 42528 +32234 42528 +38900 42525 +29739 42520 +27462 42509 +29718 42504 +42511 42503 +25402 42497 +36706 42493 +22269 42489 +14452 42483 +33833 42482 +29514 42479 +40982 42469 +10369 42469 +47364 42468 +44731 42466 +22732 42465 +37000 42461 +34202 42455 +28731 42452 +5759 42450 +39676 42448 +44945 42448 +3453 42445 +17193 42445 +24195 42441 +38698 42441 +26115 42436 +15956 42435 +8898 42432 +29789 42431 +18084 42430 +27420 42429 +37565 42429 +45510 42425 +48113 42424 +39631 42424 +29404 42421 +45807 42420 +47741 42418 +42852 42417 +18338 42416 +50073 42414 +28158 42411 +28848 42409 +37100 42409 +3548 42407 +20570 42407 +14423 42403 +35505 42402 +39473 42401 +26571 42398 +30481 42395 +37626 42391 +27591 42389 +33992 42388 +37753 42387 +29605 42385 +24278 42380 +28668 42376 +11440 42372 +29648 42371 +44299 42366 +25578 42360 +23064 42358 +40648 42357 +2803 42357 +39935 42351 +30149 42343 +39080 42343 +22337 42333 +44574 42328 +43074 42324 +26814 42317 +27041 42314 +13323 42314 +29615 42308 +22087 42302 +45928 42297 +48233 42296 +48750 42291 +1427 42285 +28874 42278 +33264 42276 +22886 42272 +36167 42261 +39795 42260 +31581 42246 +29376 42245 +31016 42244 +38682 42241 +16708 42237 +37560 42236 +31474 42234 +47507 42227 +23298 42222 +23188 42222 +30852 42221 +34483 42220 +33233 42214 +17708 42214 +46941 42212 +11984 42209 +30507 42207 +27756 42201 +32553 42198 +37589 42197 +23922 42195 +29608 42194 +33863 42191 +22587 42188 +46714 42184 +23262 42183 +27621 42182 +18940 42179 +48120 42178 +16694 42175 +36160 42173 +33599 42168 +49314 42168 +49157 42166 +26511 42160 +37301 42151 +48138 42149 +22852 42147 +10679 42144 +48066 42143 +32947 42139 +48111 42138 +43573 42133 +26118 42132 +28127 42131 +24137 42130 +7418 42123 +49030 42123 +16091 42120 +36189 42119 +33752 42116 +43542 42116 +34702 42114 +34225 42112 +38769 42110 +43014 42105 +41439 42099 +21405 42099 +25862 42091 +44615 42090 +15126 42090 +30850 42086 +37862 42079 +31025 42078 +26050 42078 +44312 42074 +24286 42070 +35765 42067 +47508 42067 +41151 42067 +26554 42066 +37935 42065 +14477 42065 +20848 42065 +37299 42064 +43086 42063 +26195 42063 +31296 42061 +41354 42054 +38999 42049 +35268 42048 +20356 42039 +1344 42033 +13847 42032 +26114 42031 +35969 42026 +50062 42024 +36684 42023 +40544 42022 +14220 42021 +24804 42016 +35927 42015 +22023 42012 +17875 42006 +12491 42002 +14382 42000 +22327 41999 +41957 41998 +28827 41988 +40795 41986 +26741 41982 +39725 41979 +29172 41978 +35576 41976 +33417 41974 +36415 41973 +47438 41973 +7167 41970 +8430 41967 +34731 41967 +28742 41967 +49377 41966 +41437 41961 +42843 41960 +13500 41957 +35250 41957 +37099 41956 +33970 41954 +31987 41953 +35154 41952 +42450 41948 +44341 41944 +44564 41944 +47122 41943 +41773 41936 +24761 41935 +41126 41935 +25460 41934 +46009 41933 +43190 41933 +28870 41916 +28964 41912 +30023 41911 +32433 41910 +33546 41906 +41318 41902 +48268 41899 +24549 41899 +46587 41893 +4658 41878 +28946 41877 +38068 41876 +46742 41876 +36939 41875 +30648 41873 +32726 41869 +28637 41867 +31085 41866 +49829 41861 +30124 41849 +4194 41845 +27118 41844 +43261 41842 +44158 41841 +21155 41839 +23362 41833 +29592 41827 +44775 41825 +643 41823 +46703 41822 +30161 41821 +49431 41819 +42085 41812 +35197 41807 +29503 41805 +29300 41804 +43965 41801 +36783 41797 +31692 41794 +38616 41791 +49774 41791 +47181 41788 +45450 41784 +42209 41781 +44992 41780 +16350 41778 +37876 41773 +49101 41773 +26795 41767 +33914 41764 +25803 41764 +34372 41762 +43520 41759 +42140 41752 +37620 41742 +40815 41737 +30990 41734 +28467 41730 +37872 41728 +22190 41728 +49589 41727 +18155 41725 +29933 41722 +46412 41720 +40181 41719 +21269 41714 +37993 41709 +22697 41707 +23237 41706 +37764 41706 +24368 41701 +37267 41700 +43381 41699 +24907 41699 +47547 41699 +44160 41698 +20443 41697 +28511 41696 +49595 41696 +49088 41693 +31430 41690 +27256 41689 +35981 41683 +47444 41682 +33662 41680 +31060 41678 +14204 41678 +35650 41677 +47095 41675 +30550 41673 +36550 41670 +49744 41667 +6736 41667 +10187 41666 +40448 41665 +5239 41661 +35547 41660 +38421 41659 +5234 41658 +46609 41649 +16163 41647 +29691 41646 +29038 41645 +22883 41645 +25096 41640 +44284 41637 +14579 41628 +24574 41625 +11993 41622 +47952 41619 +41189 41618 +40280 41617 +40690 41615 +23220 41606 +35701 41605 +35205 41604 +45058 41604 +14304 41602 +18683 41602 +31468 41601 +26247 41600 +49839 41596 +17353 41592 +38631 41591 +37429 41590 +42047 41590 +38801 41590 +26204 41589 +11268 41586 +28658 41584 +42123 41579 +16074 41575 +27487 41572 +46175 41572 +25445 41564 +36953 41562 +41273 41559 +43851 41558 +29633 41556 +15763 41554 +30517 41553 +5324 41552 +36034 41551 +12110 41544 +31113 41543 +27368 41542 +23685 41538 +48590 41530 +35698 41527 +1403 41526 +25010 41525 +28279 41521 +33687 41517 +35308 41516 +47500 41515 +49634 41515 +24181 41513 +47100 41508 +35672 41507 +31598 41504 +29145 41504 +21699 41491 +35616 41490 +46361 41489 +49297 41482 +14522 41481 +33878 41481 +20016 41478 +38005 41473 +29473 41473 +34228 41470 +29952 41470 +28778 41469 +29655 41467 +7569 41467 +45800 41462 +43499 41461 +27176 41460 +6816 41454 +37128 41454 +30041 41453 +17223 41449 +46157 41449 +40185 41446 +34824 41446 +32927 41445 +31999 41437 +43157 41437 +46515 41434 +15315 41431 +28703 41425 +31056 41421 +40372 41419 +45318 41414 +32747 41414 +45615 41414 +13290 41400 +25521 41397 +40066 41394 +30144 41393 +48501 41391 +50210 41390 +23619 41390 +18146 41386 +36256 41384 +32910 41382 +40252 41381 +26427 41381 +43902 41378 +28386 41377 +29906 41375 +34592 41371 +19176 41367 +41353 41365 +49739 41364 +26591 41362 +34781 41361 +18897 41349 +41565 41346 +50017 41345 +49994 41342 +32421 41342 +27596 41340 +29187 41337 +36015 41337 +46907 41336 +38019 41334 +32960 41334 +20597 41333 +32946 41318 +31235 41318 +28705 41316 +46073 41308 +41779 41307 +27263 41305 +46264 41303 +32576 41299 +35785 41299 +30674 41299 +24767 41292 +37462 41292 +35784 41287 +27761 41285 +48155 41283 +29517 41282 +47380 41282 +41738 41273 +47579 41271 +37984 41270 +31289 41268 +36213 41268 +27199 41266 +43018 41265 +40960 41262 +49468 41260 +33238 41252 +44763 41246 +13730 41245 +48015 41244 +23800 41239 +36049 41238 +39053 41236 +14945 41236 +43554 41236 +28408 41234 +23815 41220 +15739 41219 +22709 41217 +31685 41217 +27228 41209 +24909 41205 +41860 41197 +31843 41193 +41188 41192 +35525 41189 +15350 41187 +29915 41178 +46946 41176 +24399 41174 +49759 41170 +2992 41169 +14125 41168 +14437 41163 +15365 41162 +31641 41162 +37303 41159 +28069 41159 +37886 41159 +43255 41153 +46013 41152 +17253 41151 +18100 41150 +22749 41149 +28845 41148 +35683 41146 +34797 41137 +35341 41137 +28711 41131 +17561 41129 +29013 41124 +6732 41122 +22985 41112 +37812 41100 +20072 41098 +33826 41097 +7942 41096 +21911 41096 +30963 41094 +40323 41091 +41483 41088 +29206 41086 +41404 41078 +20595 41077 +20827 41075 +35261 41075 +5248 41073 +20509 41072 +5630 41067 +20797 41065 +21437 41060 +24403 41046 +49683 41046 +30649 41046 +42474 41043 +26337 41041 +35951 41041 +37114 41040 +28226 41036 +25675 41034 +34963 41032 +23628 41032 +45261 41025 +38560 41010 +49690 41008 +14988 41003 +37668 41003 +34819 41002 +15995 40999 +47031 40998 +28494 40998 +34173 40992 +34528 40991 +19404 40990 +37489 40987 +26863 40987 +48741 40985 +27205 40985 +44447 40979 +2424 40976 +27481 40973 +12272 40969 +31360 40968 +1594 40962 +34815 40960 +22572 40956 +42549 40955 +30252 40955 +22189 40954 +40018 40952 +24825 40952 +48119 40948 +7144 40943 +40551 40943 +46773 40942 +30125 40941 +25321 40937 +16941 40932 +38413 40932 +46131 40924 +47061 40923 +35097 40917 +25429 40915 +49059 40913 +20724 40912 +31875 40906 +43671 40898 +39271 40887 +23209 40882 +21338 40881 +49536 40878 +42040 40877 +31586 40876 +42618 40876 +39915 40874 +27230 40871 +16232 40868 +9409 40868 +39178 40866 +23700 40866 +39001 40860 +36090 40859 +34150 40858 +28933 40856 +30943 40848 +46896 40845 +24277 40842 +48258 40841 +33631 40835 +3456 40830 +23716 40825 +46450 40824 +31824 40821 +3606 40820 +34006 40819 +27710 40817 +18995 40808 +32752 40805 +40338 40804 +32696 40798 +41597 40785 +40827 40784 +18784 40782 +41837 40780 +29420 40779 +34551 40766 +47342 40762 +48171 40761 +24602 40749 +31193 40749 +40661 40747 +21731 40736 +32213 40734 +18568 40733 +33230 40732 +39047 40730 +18308 40724 +34614 40723 +4440 40723 +43414 40722 +37357 40710 +20133 40708 +43754 40706 +40224 40701 +39262 40700 +34863 40698 +30299 40695 +27969 40689 +10520 40689 +30412 40688 +44146 40684 +46426 40681 +20215 40679 +17477 40678 +38411 40673 +29979 40673 +33657 40671 +44516 40671 +22791 40668 +43188 40661 +17815 40659 +16647 40658 +19752 40656 +23673 40655 +27649 40651 +44473 40650 +28211 40648 +46919 40647 +4949 40647 +23315 40645 +34654 40644 +43726 40642 +28186 40638 +37172 40636 +35629 40634 +22523 40633 +32181 40631 +27017 40630 +33107 40629 +33822 40626 +20816 40625 +29252 40622 +45687 40622 +38193 40618 +33904 40617 +44096 40612 +22117 40611 +45566 40610 +27143 40603 +33595 40601 +36138 40600 +38645 40597 +41246 40593 +27416 40592 +15433 40592 +26350 40591 +31149 40590 +20199 40586 +8039 40584 +37880 40575 +32170 40570 +21286 40569 +19891 40568 +48967 40564 +28239 40556 +33598 40555 +40183 40555 +43933 40547 +37459 40545 +21426 40544 +26506 40544 +39240 40541 +14137 40540 +43288 40534 +29165 40532 +28304 40532 +24136 40527 +16432 40526 +47997 40522 +37001 40522 +24265 40515 +43972 40513 +24096 40506 +28982 40504 +23347 40500 +26897 40497 +18145 40494 +44076 40493 +35797 40490 +12919 40490 +44285 40488 +44620 40484 +29580 40482 +40771 40479 +25270 40479 +28552 40479 +35303 40477 +23142 40459 +43988 40456 +37546 40454 +36190 40452 +27644 40451 +39088 40450 +40773 40447 +35055 40446 +33294 40441 +12306 40439 +40144 40437 +36820 40431 +13989 40429 +9806 40429 +26754 40428 +27776 40428 +39864 40426 +41994 40425 +29130 40418 +17006 40414 +33229 40413 +40617 40411 +20068 40407 +18377 40403 +30745 40401 +24197 40394 +8954 40393 +21898 40391 +17340 40391 +36660 40386 +45443 40386 +35520 40381 +45678 40381 +37885 40381 +1822 40380 +41720 40379 +20924 40378 +16529 40371 +39281 40369 +40906 40345 +50180 40342 +33256 40339 +38308 40329 +12487 40328 +44715 40323 +40986 40323 +19079 40320 +47014 40317 +36691 40309 +39221 40308 +39160 40308 +43003 40307 +22829 40300 +19230 40298 +19564 40292 +35492 40288 +26208 40285 +36489 40283 +17579 40282 +28263 40279 +45378 40274 +48390 40274 +38639 40273 +46902 40273 +28446 40272 +39273 40268 +31564 40268 +31343 40268 +40000 40266 +25319 40265 +30407 40265 +35934 40258 +40048 40255 +42545 40254 +34297 40253 +1396 40249 +44456 40245 +31089 40245 +12234 40236 +13758 40229 +12747 40228 +27774 40228 +22932 40226 +33550 40225 +31975 40223 +40634 40214 +49882 40211 +49952 40208 +32647 40206 +32566 40194 +26995 40192 +31580 40191 +42837 40182 +40614 40181 +35275 40180 +15206 40172 +29972 40168 +48226 40167 +39002 40165 +25447 40165 +28598 40164 +1094 40157 +31031 40157 +16191 40155 +48434 40151 +32094 40150 +13376 40144 +26411 40140 +48759 40139 +29303 40137 +29909 40137 +36672 40136 +25728 40136 +46629 40132 +37341 40125 +23516 40122 +31278 40122 +34725 40120 +41415 40119 +44771 40116 +13260 40112 +23721 40110 +28551 40108 +34644 40108 +37271 40105 +39796 40101 +27192 40099 +20851 40097 +45030 40097 +28715 40097 +27778 40096 +42613 40096 +31454 40091 +40909 40090 +42892 40089 +47149 40089 +31434 40088 +39663 40082 +35939 40080 +30748 40076 +42119 40074 +41903 40074 +13550 40071 +32497 40070 +9545 40066 +21390 40065 +35856 40063 +15786 40060 +34839 40060 +28930 40058 +29776 40056 +17465 40056 +32699 40055 +19706 40054 +22243 40050 +46190 40050 +17288 40042 +32798 40040 +12149 40040 +33950 40033 +46295 40031 +37140 40026 +47624 40026 +41522 40025 +25960 40024 +48367 40022 +36529 40021 +37986 40020 +26977 40020 +35878 40019 +29947 40013 +28103 40013 +45548 40010 +47492 40003 +42858 40002 +41448 40002 +47042 40001 +47140 39987 +49870 39984 +37607 39979 +31792 39970 +28700 39963 +48218 39958 +33039 39958 +29119 39956 +13564 39956 +43200 39953 +21992 39948 +43314 39941 +19638 39941 +29559 39931 +17823 39928 +49290 39927 +12795 39925 +50045 39923 +49866 39916 +24492 39909 +32933 39906 +42371 39904 +13164 39903 +29268 39899 +6746 39896 +32476 39893 +19936 39893 +48456 39893 +38537 39889 +33237 39888 +40678 39885 +25035 39883 +27948 39877 +8890 39877 +36158 39876 +25059 39864 +19815 39864 +48153 39863 +47004 39863 +30238 39861 +45099 39856 +41378 39841 +30820 39839 +36849 39836 +26884 39834 +21037 39832 +21749 39829 +35090 39814 +45419 39812 +29414 39809 +39564 39808 +36326 39801 +42826 39801 +25356 39800 +46379 39796 +14822 39795 +48296 39795 +41532 39791 +43896 39791 +32492 39791 +27753 39790 +48128 39789 +17613 39787 +48904 39786 +49420 39782 +15208 39781 +19231 39779 +17580 39775 +19720 39775 +22865 39773 +29939 39772 +42514 39766 +37215 39765 +40320 39762 +31219 39761 +35582 39761 +35016 39760 +35769 39759 +43424 39758 +9239 39754 +38787 39752 +31869 39746 +25788 39746 +31323 39745 +26533 39740 +39369 39736 +24616 39733 +37987 39731 +45036 39731 +31819 39730 +6382 39727 +37524 39727 +48580 39725 +30495 39725 +36661 39724 +46851 39721 +37464 39717 +34244 39717 +23906 39713 +29767 39711 +21991 39707 +17128 39706 +34025 39705 +20292 39696 +31146 39690 +44221 39679 +28900 39673 +44628 39673 +24676 39672 +14465 39667 +48881 39665 +37738 39664 +20423 39663 +48424 39661 +43278 39661 +34857 39660 +30052 39658 +36906 39657 +33512 39651 +7350 39646 +24274 39645 +37722 39645 +41534 39644 +25944 39644 +42880 39641 +27790 39641 +30304 39640 +31650 39637 +29184 39637 +34417 39636 +13849 39634 +25920 39634 +43113 39625 +36573 39617 +45513 39616 +3736 39616 +26923 39613 +39138 39613 +45763 39610 +36161 39604 +21304 39604 +32110 39602 +37606 39597 +34970 39597 +37683 39585 +3476 39583 +38228 39582 +26019 39581 +30095 39580 +18117 39577 +46041 39577 +12156 39574 +46004 39574 +39510 39573 +35382 39572 +31651 39571 +45121 39568 +32592 39566 +6813 39566 +48022 39565 +35057 39561 +19462 39560 +7153 39559 +36511 39559 +47788 39558 +23021 39555 +36756 39547 +45220 39546 +41824 39545 +13916 39544 +31543 39542 +28567 39541 +24509 39539 +47331 39535 +36637 39534 +22457 39531 +15999 39530 +44803 39530 +33530 39529 +14202 39528 +43359 39528 +41696 39528 +26478 39524 +6892 39524 +26322 39515 +46738 39513 +37623 39512 +28369 39509 +30614 39502 +32683 39495 +30354 39494 +48914 39487 +39798 39485 +13986 39485 +47769 39481 +16097 39479 +38496 39479 +49286 39476 +31794 39471 +44618 39471 +45829 39466 +39593 39461 +15675 39459 +36264 39458 +41891 39449 +8328 39446 +21758 39446 +24936 39437 +42625 39430 +16673 39421 +41410 39418 +27799 39418 +30280 39417 +37952 39416 +34453 39413 +39502 39412 +33919 39409 +41865 39409 +27998 39408 +42402 39406 +25032 39399 +30219 39397 +20104 39395 +15977 39394 +30147 39393 +37149 39393 +10751 39388 +31356 39387 +36012 39380 +3627 39377 +16144 39374 +23392 39364 +33183 39360 +36050 39357 +27172 39351 +28257 39349 +38550 39345 +29253 39338 +28138 39320 +43170 39318 +36068 39318 +42966 39318 +35445 39317 +37810 39316 +22348 39315 +45459 39315 +46035 39312 +26792 39303 +24118 39300 +14473 39299 +45602 39298 +49807 39295 +29132 39292 +20968 39290 +25256 39286 +18060 39283 +30940 39281 +38443 39276 +30948 39268 +32133 39267 +25770 39263 +22950 39256 +30032 39256 +27673 39254 +47405 39250 +40684 39249 +35216 39245 +28447 39240 +12425 39240 +24906 39237 +31209 39234 +35091 39233 +44226 39232 +46637 39219 +27112 39216 +41871 39213 +25061 39213 +46414 39210 +27588 39209 +27218 39207 +30272 39205 +27715 39202 +27289 39202 +41040 39201 +29687 39200 +29449 39200 +43264 39198 +37246 39198 +22800 39197 +24254 39197 +7240 39196 +48026 39193 +43949 39193 +2878 39191 +27171 39188 +20245 39186 +33411 39183 +30501 39183 +26260 39182 +21917 39180 +32117 39172 +47099 39171 +24876 39170 +44708 39169 +25345 39169 +35711 39168 +36519 39165 +27970 39161 +35077 39159 +34328 39158 +47745 39156 +34451 39155 +39832 39152 +24940 39151 +25959 39150 +38648 39141 +17763 39141 +17525 39133 +36332 39131 +11485 39129 +16874 39128 +45546 39126 +27777 39119 +13974 39112 +29965 39111 +18047 39111 +25667 39111 +36339 39109 +31270 39109 +42578 39108 +46878 39108 +48343 39107 +28396 39104 +39865 39104 +37019 39102 +30904 39100 +2336 39100 +39399 39090 +25191 39089 +31111 39080 +26090 39076 +33334 39075 +44058 39073 +44940 39073 +8388 39070 +31427 39063 +29558 39061 +37294 39060 +37949 39057 +21009 39053 +42021 39050 +16409 39045 +40468 39043 +37721 39041 +24870 39039 +36280 39037 +28130 39034 +34741 39033 +40693 39027 +32926 39026 +50191 39025 +44880 39023 +35995 39022 +47750 39021 +32618 39017 +17764 39014 +13278 39009 +44175 39000 +21077 38999 +42201 38998 +34691 38996 +32789 38989 +41209 38985 +36146 38975 +14916 38973 +6311 38970 +50008 38970 +39263 38969 +21492 38968 +42340 38967 +24591 38965 +28520 38962 +28597 38951 +40440 38951 +27345 38949 +23326 38943 +11587 38943 +35184 38933 +28174 38926 +19631 38924 +44697 38924 +45447 38923 +31930 38921 +31557 38918 +34326 38914 +33866 38914 +20044 38912 +27680 38912 +43112 38908 +48049 38907 +25286 38897 +23451 38895 +40273 38893 +22278 38891 +39694 38890 +47959 38890 +49335 38889 +24772 38887 +41884 38886 +44091 38881 +23350 38878 +15720 38876 +33770 38876 +41264 38875 +16414 38873 +24726 38871 +8479 38865 +34611 38862 +40123 38862 +27203 38861 +25707 38859 +46337 38856 +43064 38856 +28936 38855 +36534 38852 +40817 38852 +39110 38851 +21895 38839 +45396 38839 +41342 38837 +44512 38829 +29512 38827 +27917 38827 +21858 38825 +49590 38822 +39477 38821 +20803 38816 +29213 38814 +43942 38807 +17771 38807 +20861 38805 +40905 38799 +11961 38798 +38379 38790 +27564 38790 +48563 38788 +24693 38787 +21153 38784 +28420 38783 +34894 38773 +23092 38771 +49524 38769 +34738 38767 +16099 38767 +37285 38766 +17469 38764 +24279 38760 +48622 38756 +22429 38755 +27009 38754 +47347 38749 +45699 38746 +30110 38740 +17250 38737 +46702 38733 +41370 38730 +48572 38727 +44770 38724 +43929 38714 +35038 38714 +17278 38711 +48274 38710 +6860 38704 +41414 38703 +28825 38699 +42375 38699 +14407 38698 +45429 38693 +41149 38689 +49847 38686 +23036 38685 +10067 38683 +22466 38683 +25785 38674 +34737 38671 +41063 38670 +33849 38667 +41883 38659 +36453 38657 +43032 38653 +35919 38652 +6773 38647 +2466 38644 +15366 38642 +20777 38640 +40268 38636 +40291 38635 +41516 38633 +19044 38633 +35524 38629 +32144 38628 +31805 38627 +38086 38626 +40270 38621 +36336 38621 +33387 38620 +17818 38619 +35800 38617 +34360 38615 +20772 38614 +9821 38611 +46288 38609 +22691 38600 +48422 38597 +25272 38591 +29645 38590 +41018 38589 +20711 38588 +24962 38587 +49510 38587 +47828 38586 +33520 38585 +14368 38577 +32261 38575 +20211 38575 +39198 38573 +46895 38569 +38889 38566 +32391 38562 +44666 38561 +47257 38554 +35710 38554 +47098 38553 +39013 38552 +37274 38544 +23483 38536 +28585 38532 +32109 38530 +46791 38526 +22813 38524 +35870 38516 +25782 38513 +46357 38512 +9271 38512 +45211 38510 +27033 38510 +28976 38498 +45248 38498 +33047 38498 +36380 38494 +47713 38493 +37086 38490 +30624 38490 +38745 38488 +30514 38487 +26611 38487 +2886 38486 +10873 38485 +43351 38484 +41562 38483 +42191 38481 +34841 38481 +45994 38474 +45712 38474 +22921 38473 +41567 38467 +28925 38464 +21577 38461 +21084 38459 +12331 38458 +4187 38456 +29133 38455 +26529 38455 +41013 38451 +48998 38450 +31475 38448 +22657 38445 +29551 38444 +47313 38442 +45769 38424 +38192 38424 +34688 38420 +42240 38414 +18483 38412 +34135 38410 +33488 38403 +31759 38398 +23571 38396 +36946 38395 +41929 38393 +40003 38393 +40878 38387 +34898 38380 +13155 38379 +18160 38378 +2711 38376 +27640 38374 +26485 38370 +41737 38366 +21759 38355 +27486 38353 +38612 38348 +11350 38348 +33591 38347 +28159 38346 +48486 38344 +25823 38344 +46657 38340 +21186 38338 +46312 38332 +29951 38331 +33065 38329 +43504 38328 +22302 38327 +11726 38323 +22622 38322 +35325 38320 +28204 38319 +32680 38318 +23460 38317 +35112 38316 +14964 38314 +34176 38313 +11878 38313 +48406 38312 +40037 38311 +12794 38311 +33048 38310 +30601 38310 +49844 38305 +40964 38304 +31081 38297 +16979 38294 +23402 38292 +17108 38291 +26284 38289 +46852 38285 +39153 38284 +12574 38284 +39180 38282 +27960 38281 +1753 38281 +28627 38280 +40798 38279 +37256 38278 +47982 38274 +45074 38274 +20643 38274 +28588 38272 +2529 38269 +24920 38267 +37624 38267 +9198 38265 +7391 38263 +49187 38262 +23463 38259 +44488 38253 +21797 38246 +26950 38245 +35174 38243 +12940 38229 +34920 38224 +28802 38217 +34641 38216 +44234 38214 +34464 38206 +21985 38199 +24845 38188 +37681 38185 +44767 38185 +38182 38184 +48106 38182 +16913 38182 +17610 38180 +24745 38174 +21610 38172 +30533 38169 +26423 38168 +9289 38166 +34425 38161 +29214 38160 +45342 38154 +14215 38152 +20876 38146 +38425 38145 +31342 38143 +32221 38140 +48566 38137 +32674 38136 +9249 38132 +46730 38132 +32357 38131 +3897 38129 +47022 38124 +20658 38124 +34332 38120 +35427 38120 +26660 38119 +13733 38118 +39580 38113 +32744 38109 +39574 38109 +49400 38108 +37995 38104 +26392 38101 +36919 38101 +37104 38097 +43955 38091 +31572 38090 +14974 38085 +20957 38085 +37630 38085 +2289 38083 +39146 38079 +28140 38078 +39589 38076 +33162 38065 +12409 38065 +29057 38064 +31773 38062 +34869 38059 +10179 38054 +21847 38048 +36539 38044 +24528 38043 +34089 38041 +40496 38041 +47130 38038 +47993 38036 +36817 38036 +38585 38035 +41119 38032 +30376 38031 +46942 38031 +41031 38028 +31359 38025 +48531 38025 +32877 38023 +28306 38023 +3029 38020 +29033 38017 +34948 38016 +44376 38015 +33592 38013 +23580 38012 +38520 38011 +35894 38006 +8014 38001 +39853 37999 +48843 37998 +28302 37993 +16718 37987 +32052 37986 +20911 37986 +47028 37978 +48731 37970 +21382 37967 +44495 37962 +17974 37958 +34298 37958 +6601 37958 +32201 37957 +39015 37956 +29649 37954 +13501 37951 +23437 37948 +32611 37938 +25840 37935 +39348 37935 +29068 37935 +25499 37933 +41861 37932 +27734 37931 +42761 37930 +8637 37929 +30989 37927 +37868 37926 +32547 37921 +18703 37920 +25778 37917 +37333 37911 +37646 37910 +36824 37907 +4435 37901 +39525 37900 +33155 37899 +44151 37890 +45156 37889 +34774 37885 +18069 37882 +35529 37877 +45916 37870 +32337 37866 +36459 37864 +27560 37862 +29538 37862 +30453 37860 +15511 37857 +20413 37857 +21241 37855 +21953 37854 +42320 37853 +26697 37852 +48102 37850 +46783 37847 +49252 37846 +12987 37840 +26983 37839 +16398 37839 +21537 37839 +36016 37837 +17680 37837 +13983 37836 +34441 37836 +37537 37835 +25171 37835 +44390 37829 +37643 37829 +41714 37826 +25870 37822 +24783 37819 +29526 37818 +14191 37818 +34987 37817 +24184 37817 +41320 37813 +18890 37808 +32900 37805 +39556 37800 +43793 37798 +28116 37795 +33508 37787 +32754 37787 +14487 37785 +38209 37779 +45223 37779 +45162 37778 +31018 37777 +20605 37775 +24245 37773 +20497 37773 +24612 37771 +44888 37767 +43503 37766 +49255 37765 +41680 37762 +25404 37762 +42110 37756 +47796 37754 +9652 37754 +22497 37740 +6743 37737 +33178 37735 +44904 37733 +29082 37732 +38853 37730 +23740 37724 +34505 37724 +24180 37715 +20856 37715 +22312 37711 +31227 37707 +20506 37704 +25322 37700 +28727 37699 +40585 37697 +35548 37696 +37611 37694 +45498 37692 +43307 37690 +40714 37688 +35557 37687 +14539 37686 +40249 37684 +17828 37684 +41664 37684 +49874 37684 +37980 37679 +14184 37678 +42101 37673 +26308 37671 +13603 37668 +34872 37666 +45846 37666 +48928 37665 +36575 37664 +28481 37662 +42743 37662 +38272 37662 +22119 37659 +50060 37658 +15803 37651 +18019 37650 +43783 37649 +42241 37647 +21728 37646 +39082 37626 +36234 37622 +40755 37614 +23653 37610 +44766 37607 +43744 37604 +8578 37601 +45054 37601 +31510 37600 +32338 37599 +834 37595 +30194 37591 +40593 37584 +24213 37583 +25149 37579 +48681 37575 +46016 37572 +27140 37567 +33509 37567 +44635 37562 +47825 37562 +38359 37561 +13689 37551 +32298 37549 +45237 37543 +24468 37539 +43709 37534 +9451 37532 +26872 37526 +22913 37525 +18601 37524 +19570 37521 +22163 37517 +34771 37517 +49727 37511 +40919 37510 +38306 37505 +22226 37503 +44507 37500 +33004 37499 +41236 37491 +47416 37490 +35168 37489 +15884 37488 +32156 37485 +21780 37484 +32161 37482 +15552 37479 +32493 37479 +39501 37479 +46313 37477 +13437 37473 +36689 37468 +32209 37464 +32299 37460 +29024 37458 +39114 37454 +26168 37454 +20264 37453 +36237 37450 +35598 37448 +40547 37444 +12905 37430 +38060 37421 +20943 37420 +47739 37419 +5020 37415 +24991 37414 +26987 37412 +34039 37411 +25083 37408 +29270 37408 +22648 37399 +39484 37399 +22991 37397 +36238 37392 +40868 37391 +16284 37386 +5841 37383 +40327 37382 +38127 37380 +35688 37374 +35996 37373 +15042 37371 +22324 37369 +24222 37366 +41268 37365 +24275 37365 +33036 37362 +30692 37362 +32012 37362 +48306 37361 +44827 37356 +26705 37354 +44331 37353 +36322 37351 +29555 37350 +16598 37342 +47055 37339 +33735 37339 +18937 37337 +35761 37335 +26045 37333 +13732 37328 +25648 37325 +38876 37325 +36097 37322 +15445 37321 +47062 37318 +27277 37318 +30788 37316 +11413 37316 +27522 37314 +14577 37310 +34594 37309 +42346 37304 +36478 37303 +42264 37302 +29157 37302 +31300 37302 +36340 37299 +35723 37296 +49789 37293 +7434 37292 +20855 37288 +40410 37287 +44368 37286 +45075 37283 +44049 37282 +28740 37279 +31287 37278 +17529 37277 +17138 37271 +40170 37271 +26448 37263 +30685 37263 +22419 37258 +41846 37255 +33966 37253 +42951 37251 +46675 37251 +26979 37249 +50071 37249 +24839 37248 +34378 37244 +25399 37238 +37088 37238 +27255 37237 +47584 37236 +30499 37235 +10572 37235 +18173 37231 +28647 37231 +3289 37225 +4110 37224 +39813 37220 +16802 37217 +26912 37216 +36349 37214 +43227 37212 +47039 37210 +37306 37209 +10877 37205 +29318 37205 +37901 37204 +5111 37204 +16087 37202 +40953 37199 +29807 37198 +45143 37198 +49417 37193 +36412 37190 +43369 37190 +40152 37176 +21839 37167 +41917 37166 +17618 37151 +19385 37151 +44388 37148 +34401 37143 +49558 37140 +18209 37140 +47704 37139 +32593 37136 +35148 37132 +38439 37126 +43386 37115 +38884 37115 +25095 37111 +28164 37104 +22059 37103 +3276 37102 +36035 37102 +20482 37101 +46043 37100 +47203 37089 +8316 37089 +32817 37087 +35826 37081 +26787 37078 +48629 37077 +27656 37075 +46466 37075 +15876 37064 +38975 37064 +39973 37064 +33492 37060 +43222 37058 +37220 37058 +47566 37049 +42418 37046 +42515 37043 +20599 37042 +34289 37042 +37526 37041 +43497 37041 +36179 37039 +47248 37034 +47767 37032 +19944 37027 +33438 37026 +33840 37025 +12726 37025 +39931 37024 +21633 37020 +40991 37016 +28522 37014 +40796 37012 +25796 37008 +35737 37003 +34494 37001 +22728 36999 +29992 36997 +47494 36996 +34387 36992 +45782 36990 +31897 36990 +31485 36981 +13284 36981 +39207 36980 +12143 36980 +44232 36977 +37873 36975 +46105 36973 +25161 36970 +23211 36968 +39851 36967 +18747 36964 +41315 36964 +32455 36960 +21013 36959 +30638 36957 +13857 36955 +29587 36954 +35052 36953 +43751 36953 +28017 36952 +24678 36948 +18426 36944 +45927 36942 +20053 36940 +21303 36938 +24624 36937 +31243 36936 +15746 36935 +34063 36935 +45161 36932 +14055 36932 +28200 36929 +37064 36926 +25369 36924 +42851 36921 +45043 36915 +46696 36908 +26108 36908 +24603 36907 +8214 36907 +27149 36904 +42252 36900 +19737 36893 +24974 36890 +41039 36886 +42664 36885 +17943 36885 +30332 36885 +5925 36880 +27207 36880 +17024 36878 +26972 36871 +6730 36869 +32008 36867 +20961 36862 +21217 36860 +47631 36849 +43186 36848 +42017 36846 +15826 36843 +5723 36842 +31169 36838 +27057 36838 +37192 36829 +29968 36829 +20427 36826 +18910 36823 +33811 36822 +48179 36822 +27402 36822 +36599 36817 +43203 36811 +32231 36808 +32532 36804 +29349 36804 +16725 36803 +47496 36802 +36860 36795 +38669 36791 +45497 36785 +4234 36784 +23464 36776 +41400 36770 +28071 36761 +16286 36760 +47460 36759 +26086 36753 +36838 36750 +38221 36748 +44257 36741 +16198 36730 +10632 36727 +29598 36725 +32107 36725 +31766 36720 +38051 36719 +39454 36717 +45995 36715 +19689 36713 +45948 36711 +28788 36710 +36844 36708 +2757 36708 +21981 36706 +49230 36704 +22988 36703 +18457 36701 +37409 36700 +22279 36698 +41521 36691 +38722 36691 +36973 36682 +35867 36678 +32626 36677 +44584 36674 +32651 36674 +20459 36673 +47888 36672 +35458 36671 +34716 36669 +19146 36665 +45609 36664 +29763 36659 +43206 36655 +44725 36655 +31503 36650 +40381 36649 +16234 36647 +42926 36646 +41693 36645 +19803 36640 +33862 36639 +21902 36630 +33603 36629 +35356 36625 +49606 36620 +48373 36619 +10410 36617 +49890 36612 +39635 36611 +9710 36610 +38323 36610 +33109 36610 +25285 36609 +11407 36606 +38386 36599 +48261 36598 +47387 36594 +33928 36591 +25622 36591 +17822 36591 +21281 36589 +46525 36588 +25946 36587 +49581 36580 +34834 36579 +10180 36579 +33560 36576 +43562 36576 +20141 36569 +45386 36569 +44275 36564 +44098 36563 +16485 36560 +36901 36559 +7783 36557 +25418 36556 +33266 36551 +35139 36546 +24452 36542 +37087 36538 +24875 36538 +27800 36537 +34711 36537 +28857 36534 +26407 36533 +23020 36533 +29286 36531 +5113 36529 +30734 36525 +24770 36523 +41554 36520 +10879 36518 +28728 36517 +16928 36514 +21202 36503 +32172 36503 +34538 36501 +38961 36499 +36344 36496 +41539 36494 +49414 36493 +27982 36491 +34266 36483 +35218 36480 +32708 36474 +30425 36468 +34503 36467 +30463 36464 +45535 36457 +31029 36450 +37543 36449 +22620 36449 +24970 36448 +37454 36447 +46407 36444 +34109 36442 +27372 36441 +7182 36439 +35353 36438 +39287 36436 +28868 36435 +39338 36433 +29134 36428 +18941 36425 +44094 36418 +25653 36415 +33781 36413 +42639 36412 +11269 36410 +46711 36403 +27563 36401 +9394 36398 +37062 36397 +33138 36395 +29561 36386 +31925 36377 +47339 36364 +25278 36362 +31521 36360 +16770 36356 +38265 36356 +35137 36352 +35714 36346 +22530 36342 +31858 36341 +13862 36341 +47108 36340 +10775 36334 +24943 36333 +41579 36330 +42910 36328 +946 36325 +40007 36319 +45764 36316 +35161 36316 +40140 36308 +40737 36289 +39190 36287 +43115 36287 +13712 36286 +45503 36280 +26829 36280 +48786 36269 +47223 36267 +30991 36264 +9407 36260 +13928 36258 +28907 36255 +37355 36254 +27785 36251 +32698 36248 +32037 36243 +47640 36241 +36958 36238 +19967 36234 +38078 36231 +33015 36231 +16071 36226 +27653 36222 +23565 36222 +46467 36222 +42362 36221 +29429 36220 +47260 36217 +33408 36216 +29801 36215 +23318 36214 +29728 36213 +31738 36212 +22396 36208 +35015 36203 +8003 36198 +47805 36195 +16266 36193 +35899 36184 +46619 36182 +27967 36177 +19795 36173 +45062 36165 +28979 36163 +46548 36163 +16751 36161 +24149 36159 +35028 36155 +27896 36154 +41710 36150 +40857 36147 +38036 36146 +32950 36142 +34581 36135 +28817 36130 +48215 36128 +44967 36127 +41247 36127 +36082 36123 +42447 36116 +46208 36113 +13080 36113 +49777 36112 +11656 36112 +28301 36109 +9534 36108 +35863 36104 +43627 36102 +25382 36102 +37814 36095 +49547 36094 +31188 36089 +31753 36089 +20702 36087 +26915 36082 +46062 36080 +18776 36079 +26492 36077 +31093 36075 +42046 36069 +36666 36067 +38046 36066 +47453 36065 +35519 36064 +40757 36064 +44001 36063 +19583 36062 +41754 36061 +15047 36061 +37115 36061 +6744 36058 +46769 36055 +25998 36053 +48803 36049 +39239 36047 +14078 36047 +16942 36045 +28974 36044 +49162 36042 +40071 36039 +46656 36039 +1608 36037 +28896 36036 +44875 36035 +15640 36035 +24911 36031 +42028 36031 +48776 36031 +38200 36029 +42941 36022 +18055 36021 +42882 36014 +37185 36014 +20930 36012 +33027 36010 +8745 36003 +29008 36002 +47478 36002 +48667 36002 +44611 36000 +34993 35998 +27889 35998 +44823 35994 +42391 35988 +32855 35987 +16233 35985 +43840 35983 +35484 35982 +27760 35976 +33289 35970 +9590 35970 +38304 35969 +19963 35962 +50142 35961 +48357 35955 +40697 35955 +48450 35952 +28953 35946 +48574 35944 +42231 35942 +49879 35942 +38185 35931 +25022 35930 +15127 35929 +22713 35923 +49445 35922 +21360 35921 +29099 35917 +36527 35910 +43960 35909 +49363 35905 +20205 35904 +37223 35903 +32755 35900 +8413 35899 +49772 35896 +45484 35895 +37125 35895 +13313 35894 +47268 35893 +42292 35892 +46292 35887 +35531 35880 +42870 35880 +17230 35878 +27081 35878 +26682 35876 +42972 35876 +2699 35875 +27533 35875 +33499 35873 +24856 35867 +24305 35866 +21227 35865 +41214 35864 +15681 35860 +34232 35858 +42900 35856 +30005 35850 +34080 35849 +42594 35847 +28592 35843 +39226 35843 +44571 35842 +41528 35833 +26993 35830 +35431 35828 +23835 35828 +26598 35820 +31412 35805 +41840 35803 +46658 35802 +14231 35801 +33982 35794 +23881 35794 +15048 35791 +8926 35791 +27040 35789 +31170 35786 +44581 35779 +45457 35779 +39674 35779 +32227 35778 +25153 35774 +2880 35769 +7704 35767 +23468 35762 +23971 35758 +47083 35754 +42517 35751 +17303 35748 +36102 35744 +30741 35743 +47401 35734 +43185 35725 +30727 35722 +32568 35720 +44907 35720 +9982 35713 +40761 35712 +28096 35711 +29976 35704 +39129 35700 +10886 35696 +34923 35692 +42462 35690 +32274 35688 +19085 35678 +19926 35674 +28430 35673 +17287 35666 +49684 35665 +35021 35663 +28297 35658 +43388 35658 +27659 35654 +37571 35653 +26968 35653 +40120 35653 +43131 35653 +49909 35652 +16990 35649 +15484 35647 +49334 35646 +41108 35642 +37413 35642 +41685 35640 +38598 35635 +30615 35635 +27249 35635 +31730 35634 +15204 35633 +12646 35631 +40126 35629 +42927 35628 +46635 35627 +41254 35626 +30528 35620 +39539 35620 +27019 35616 +11794 35615 +41226 35610 +41125 35610 +31814 35609 +34756 35609 +18567 35607 +49009 35603 +47664 35603 +31211 35600 +41854 35597 +28843 35595 +3693 35594 +31716 35591 +21288 35591 +24877 35586 +46242 35583 +44989 35583 +41123 35582 +50176 35577 +44355 35575 +43342 35575 +35545 35574 +23836 35571 +39863 35570 +13911 35570 +18943 35558 +34689 35558 +26268 35552 +45240 35547 +38991 35547 +5148 35544 +46814 35535 +3396 35534 +42370 35530 +28676 35528 +29954 35523 +22240 35522 +30483 35522 +48525 35509 +39081 35508 +29818 35499 +46829 35499 +48202 35493 +42088 35491 +32112 35488 +36079 35487 +48383 35487 +37111 35486 +10612 35485 +9614 35485 +36668 35481 +32549 35481 +37210 35480 +14016 35477 +42423 35477 +22860 35475 +48986 35473 +7094 35473 +27158 35471 +28922 35468 +34253 35460 +38791 35458 +33689 35457 +47868 35448 +40054 35446 +30835 35446 +35960 35441 +31010 35438 +11869 35437 +16017 35432 +46715 35430 +23833 35430 +21495 35430 +22606 35428 +40608 35427 +25086 35427 +25341 35425 +9213 35424 +38227 35424 +40450 35418 +28640 35418 +26273 35416 +20735 35411 +29500 35409 +46631 35407 +24575 35399 +40070 35397 +39384 35396 +31192 35395 +42106 35393 +25540 35390 +16451 35387 +49935 35380 +10722 35379 +46626 35371 +7717 35370 +13383 35370 +33148 35364 +47476 35363 +21170 35360 +38912 35356 +42508 35356 +46909 35350 +38190 35349 +40300 35348 +35704 35345 +43978 35343 +20405 35335 +25437 35334 +24724 35332 +26246 35330 +11488 35329 +18893 35328 +27392 35326 +14312 35325 +46534 35325 +20315 35325 +45735 35322 +24242 35319 +45405 35317 +33083 35316 +28850 35315 +48748 35314 +6704 35312 +47163 35312 +33016 35310 +18795 35307 +39741 35305 +33960 35302 +48221 35300 +32779 35299 +35005 35295 +27189 35294 +11371 35292 +34035 35284 +20987 35280 +43511 35277 +14647 35269 +34690 35268 +48358 35267 +35643 35266 +26313 35264 +34036 35264 +45350 35263 +48420 35263 +46348 35261 +34785 35258 +43958 35257 +30248 35252 +39885 35252 +25089 35251 +15496 35245 +27797 35240 +26276 35239 +6145 35233 +38848 35230 +31102 35227 +32171 35225 +40078 35223 +42825 35219 +49654 35218 +48656 35218 +15148 35216 +29087 35211 +49679 35210 +6812 35205 +40831 35205 +36972 35201 +42741 35189 +40515 35184 +31649 35174 +6760 35169 +40395 35168 +24820 35166 +25511 35165 +10355 35164 +48636 35163 +13295 35163 +34670 35161 +40589 35155 +23042 35154 +30267 35154 +7154 35154 +36446 35144 +40446 35139 +44565 35137 +21019 35128 +33211 35128 +49530 35126 +27339 35122 +32786 35117 +38238 35113 +28048 35113 +42002 35109 +36738 35109 +23144 35107 +38267 35105 +42339 35105 +33416 35105 +32575 35104 +44219 35104 +8928 35102 +25269 35097 +6433 35096 +49575 35089 +40326 35089 +36014 35087 +2762 35080 +29069 35075 +48347 35074 +37733 35074 +25898 35071 +34341 35070 +32653 35065 +43837 35062 +39155 35059 +23912 35058 +17018 35051 +11385 35051 +29204 35045 +29174 35043 +44981 35040 +39496 35037 +42836 35032 +31009 35026 +31566 35026 +32142 35020 +32996 35020 +34658 35017 +20203 35016 +45098 35010 +12315 35005 +12611 35003 +21018 34996 +4733 34995 +36057 34994 +3168 34984 +42607 34983 +25991 34979 +42422 34979 +31947 34976 +46652 34975 +44590 34974 +39797 34972 +40871 34969 +44810 34969 +19203 34962 +26551 34961 +41795 34952 +30802 34949 +44306 34948 +41586 34939 +29096 34936 +36564 34935 +33341 34931 +38006 34931 +41918 34931 +43716 34931 +45315 34930 +38135 34929 +42988 34929 +37041 34928 +29322 34914 +38373 34914 +20810 34913 +40115 34913 +32886 34910 +41699 34908 +45652 34907 +34859 34905 +39389 34901 +21881 34898 +14385 34898 +32143 34897 +45085 34897 +13265 34896 +45017 34895 +39233 34893 +45172 34892 +38966 34888 +27910 34888 +29122 34887 +39740 34880 +30505 34877 +37212 34877 +32185 34869 +16520 34868 +41231 34865 +33121 34865 +34721 34860 +17190 34857 +34730 34856 +41747 34856 +28684 34850 +40294 34847 +26525 34838 +32129 34834 +47421 34829 +38842 34825 +4655 34821 +45407 34816 +24100 34815 +30971 34811 +31166 34808 +46964 34803 +39439 34801 +33475 34799 +42022 34799 +43853 34797 +44806 34796 +26895 34794 +17309 34794 +21928 34793 +28872 34793 +30707 34790 +34278 34788 +47450 34785 +32621 34780 +31476 34778 +29298 34770 +46881 34770 +45502 34767 +38008 34766 +45525 34763 +12792 34761 +46622 34759 +44643 34757 +23123 34753 +26381 34752 +36209 34744 +24750 34742 +23644 34740 +30373 34739 +41855 34738 +6471 34737 +31397 34737 +41478 34736 +26146 34736 +24736 34733 +43189 34733 +21524 34731 +38621 34728 +36932 34727 +29448 34726 +12679 34717 +47193 34711 +49278 34711 +20641 34710 +17171 34709 +46891 34701 +23219 34700 +31135 34696 +39229 34694 +31607 34694 +47027 34690 +19559 34686 +24917 34683 +33607 34679 +35556 34679 +29552 34676 +47495 34674 +46017 34673 +24043 34668 +27676 34664 +31778 34656 +6604 34655 +5459 34649 +43507 34645 +49791 34645 +30805 34643 +29903 34641 +33301 34640 +6406 34638 +35131 34630 +42793 34628 +41186 34627 +38731 34627 +23924 34626 +41157 34625 +49304 34622 +27531 34622 +40760 34620 +8232 34617 +50136 34615 +18072 34615 +39793 34608 +22356 34606 +32051 34603 +9668 34602 +23998 34602 +40716 34601 +36177 34597 +40137 34595 +25536 34594 +44182 34592 +37661 34591 +41431 34590 +25400 34588 +46230 34584 +30199 34583 +39322 34583 +30584 34582 +36135 34580 +49827 34580 +31786 34580 +27159 34579 +12893 34579 +28439 34573 +41654 34564 +49600 34559 +26675 34558 +32016 34558 +32686 34557 +48598 34556 +22409 34556 +38206 34551 +23552 34550 +12621 34549 +29275 34547 +24155 34545 +25822 34541 +30361 34540 +47428 34538 +47610 34538 +46709 34535 +44845 34534 +42767 34533 +45334 34533 +32545 34530 +39460 34529 +42727 34528 +32375 34528 +34484 34522 +40131 34521 +43051 34519 +27920 34516 +35569 34513 +34382 34513 +30780 34506 +26303 34504 +28881 34499 +32378 34493 +36033 34486 +24895 34484 +32759 34479 +21090 34473 +30155 34472 +46549 34472 +45226 34469 +36043 34468 +49817 34468 +37782 34466 +35483 34466 +22754 34465 +42205 34464 +19543 34460 +20341 34457 +48537 34457 +29129 34456 +28782 34455 +5376 34452 +13425 34451 +17684 34448 +13907 34447 +49119 34445 +47434 34442 +50128 34441 +31514 34440 +28362 34428 +47229 34422 +34532 34419 +23496 34419 +48596 34418 +28692 34416 +45918 34414 +30435 34413 +39326 34409 +36006 34402 +14940 34402 +36105 34400 +18667 34398 +33040 34395 +32386 34395 +23338 34392 +40487 34390 +30148 34389 +30616 34385 +34057 34381 +41250 34380 +30338 34373 +36367 34371 +29923 34370 +18007 34369 +21901 34368 +19336 34366 +41321 34366 +23079 34364 +16819 34362 +21897 34362 +39981 34360 +8731 34357 +40309 34352 +24732 34352 +13484 34350 +48985 34350 +21132 34348 +33149 34348 +44474 34347 +34452 34347 +42879 34346 +32591 34345 +21466 34343 +45698 34338 +41393 34335 +29528 34332 +23222 34331 +24665 34329 +24792 34324 +34418 34321 +27988 34317 +17516 34305 +39191 34289 +46848 34288 +39838 34287 +48180 34286 +23573 34286 +34568 34280 +29215 34276 +18554 34275 +33620 34274 +47620 34273 +30749 34271 +30804 34270 +47662 34268 +42566 34261 +41549 34253 +16488 34252 +30121 34251 +32561 34247 +25663 34246 +43527 34243 +36819 34242 +43067 34237 +21328 34233 +33451 34231 +26309 34230 +44995 34227 +27359 34226 +31924 34225 +31667 34223 +30806 34220 +39152 34220 +32781 34219 +48586 34217 +12021 34216 +30560 34214 +9043 34211 +21557 34203 +32820 34198 +45972 34193 +31082 34192 +19453 34188 +27860 34185 +8619 34185 +49243 34184 +49840 34181 +12745 34180 +27641 34180 +15132 34173 +43079 34172 +26864 34170 +32861 34170 +48118 34167 +14127 34166 +45436 34164 +44674 34159 +46913 34159 +36968 34157 +30458 34152 +8134 34149 +35162 34147 +19366 34144 +23874 34139 +33443 34136 +24525 34136 +40830 34124 +26854 34124 +34864 34124 +20146 34122 +17744 34115 +45138 34114 +49708 34113 +28736 34104 +45655 34101 +25312 34099 +25344 34097 +40565 34092 +45241 34086 +32191 34086 +5590 34083 +28065 34079 +40834 34073 +49248 34072 +22544 34071 +15856 34066 +45275 34064 +43802 34061 +30639 34056 +48833 34053 +44244 34050 +40930 34048 +33456 34044 +38347 34041 +33282 34041 +11785 34041 +49792 34034 +45997 34030 +33776 34029 +21482 34029 +48521 34026 +46517 34026 +37407 34020 +34887 34017 +11361 34017 +32292 34016 +49911 34012 +33688 34010 +32192 34010 +38923 34007 +34826 34006 +16884 33999 +25378 33998 +44645 33991 +26235 33991 +19588 33983 +37052 33979 +35408 33978 +47146 33976 +16768 33975 +5131 33974 +47367 33968 +47165 33965 +34011 33964 +40473 33963 +33535 33962 +34194 33959 +43865 33958 +34493 33957 +46196 33952 +38130 33952 +24552 33950 +34449 33949 +41237 33949 +49868 33942 +15479 33940 +10851 33939 +15004 33939 +32118 33936 +19452 33930 +39680 33926 +49699 33922 +36021 33922 +29167 33914 +10661 33914 +49000 33912 +5784 33912 +36531 33902 +37375 33900 +14681 33900 +25899 33899 +46317 33899 +45168 33898 +6501 33898 +14755 33897 +25239 33890 +28484 33889 +48696 33888 +31279 33888 +40121 33887 +45179 33883 +45954 33880 +13867 33879 +24367 33879 +24202 33877 +41160 33873 +15717 33868 +49099 33867 +4212 33865 +47742 33861 +35820 33858 +7774 33856 +27012 33854 +47761 33854 +44505 33853 +39565 33850 +27030 33848 +44580 33848 +40813 33848 +44283 33847 +46390 33844 +33930 33842 +18417 33842 +41475 33835 +45061 33834 +27018 33833 +35176 33832 +42959 33824 +37252 33824 +37970 33822 +30673 33822 +22088 33821 +44240 33821 +34230 33821 +40210 33817 +40494 33812 +49831 33809 +39388 33804 +46524 33795 +47550 33795 +18674 33790 +45365 33786 +8555 33783 +26160 33780 +44428 33780 +41016 33779 +30403 33778 +36449 33772 +21111 33770 +26569 33767 +40949 33765 +37183 33759 +29626 33758 +43332 33758 +42893 33757 +4882 33755 +22996 33752 +41956 33751 +46706 33751 +10034 33748 +39579 33748 +46307 33745 +27696 33744 +47887 33741 +48878 33738 +34192 33731 +30936 33717 +29197 33716 +45939 33712 +20504 33710 +42874 33710 +39993 33709 +43818 33706 +34423 33705 +43553 33703 +38156 33702 +31760 33701 +47866 33689 +32258 33685 +30558 33682 +39642 33680 +32838 33679 +36202 33679 +44005 33677 +37594 33673 +5431 33673 +34396 33664 +31834 33662 +19535 33660 +36535 33654 +30222 33647 +37933 33646 +41350 33645 +19512 33643 +37397 33640 +47810 33639 +9483 33638 +31677 33631 +25839 33628 +27729 33626 +42807 33620 +40237 33620 +36585 33615 +36923 33615 +46646 33605 +16037 33592 +25700 33590 +22239 33583 +21034 33580 +30318 33577 +35086 33577 +38341 33571 +25383 33570 +38050 33570 +27757 33567 +42265 33566 +40722 33565 +38584 33564 +42099 33561 +45306 33560 +4468 33552 +29794 33550 +32551 33549 +39425 33548 +36466 33543 +35640 33540 +41259 33537 +28681 33535 +25353 33533 +30368 33531 +34125 33526 +34925 33525 +45642 33525 +49032 33524 +48977 33518 +19067 33516 +33397 33514 +34846 33514 +34322 33512 +45313 33511 +27503 33509 +42688 33502 +21863 33502 +33046 33498 +40150 33484 +34511 33479 +28460 33476 +33057 33475 +24765 33473 +31914 33473 +37080 33472 +38869 33472 +21944 33470 +34065 33468 +24733 33466 +32709 33465 +23291 33464 +37881 33463 +36223 33462 +34412 33459 +11484 33459 +42444 33458 +33671 33457 +49384 33454 +26028 33453 +41421 33448 +34918 33447 +10772 33446 +32565 33443 +15877 33442 +16973 33442 +42129 33441 +36218 33440 +49142 33438 +28543 33437 +17911 33437 +41751 33436 +47105 33435 +43210 33433 +10646 33432 +45832 33414 +46737 33408 +37353 33405 +36845 33405 +17717 33405 +41001 33405 +33287 33403 +40174 33399 +38090 33398 +9895 33397 +44371 33395 +35883 33391 +38397 33390 +16539 33381 +23379 33372 +45950 33372 +48595 33371 +28115 33369 +36794 33368 +36736 33364 +27385 33364 +35516 33361 +29477 33360 +28977 33358 +32329 33357 +38327 33357 +14596 33356 +40102 33353 +44364 33346 +34514 33345 +33055 33343 +28440 33341 +48519 33338 +24674 33335 +27450 33332 +22262 33330 +48240 33325 +7693 33325 +38123 33324 +27665 33323 +30070 33322 +35872 33322 +28149 33321 +42262 33319 +46961 33315 +47433 33314 +23035 33311 +10881 33308 +43880 33307 +47591 33307 +19393 33307 +15672 33301 +14152 33298 +12327 33297 +46174 33296 +33327 33296 +43165 33292 +33537 33292 +40229 33290 +44735 33287 +41109 33280 +40534 33277 +49364 33277 +42913 33274 +47002 33273 +18254 33270 +10440 33263 +43672 33260 +29606 33260 +19807 33258 +33166 33256 +14970 33253 +49859 33251 +25590 33251 +20218 33250 +42968 33250 +7033 33249 +39976 33248 +37024 33248 +24285 33242 +35225 33241 +35527 33241 +46225 33240 +40508 33237 +49291 33237 +39486 33235 +26677 33233 +30798 33229 +41623 33228 +36825 33225 +29201 33223 +34318 33223 +41117 33219 +43344 33219 +23863 33217 +33304 33216 +28208 33214 +38751 33212 +36893 33209 +44277 33207 +34157 33207 +18934 33204 +42361 33204 +21209 33201 +44619 33195 +29077 33193 +45324 33193 +24133 33191 +20548 33189 +40625 33187 +43852 33186 +32057 33186 +47729 33178 +38677 33177 +16000 33176 +37754 33176 +49838 33175 +31553 33170 +22735 33165 +13438 33163 +19428 33162 +32841 33161 +46382 33161 +23340 33158 +21449 33156 +33626 33155 +35358 33152 +39300 33151 +46401 33149 +13828 33147 +41921 33147 +26097 33146 +40896 33144 +18430 33141 +21484 33141 +37502 33139 +20451 33139 +26531 33138 +28290 33137 +47691 33135 +46226 33131 +13935 33126 +27303 33124 +26230 33124 +32297 33121 +35491 33118 +49910 33115 +29001 33112 +34277 33110 +50100 33109 +41907 33108 +3997 33107 +42255 33104 +8820 33102 +41975 33101 +20087 33096 +26069 33092 +25873 33088 +36032 33087 +47917 33086 +45010 33082 +19596 33082 +33774 33080 +22794 33077 +49178 33077 +24863 33075 +42598 33072 +7843 33070 +41558 33064 +39989 33064 +49349 33063 +39966 33061 +30100 33059 +38348 33054 +43044 33049 +35461 33044 +30799 33043 +41802 33041 +27626 33039 +41879 33038 +26552 33038 +26200 33036 +47645 33036 +27961 33036 +28072 33034 +30290 33032 +29607 33023 +32563 33022 +27706 33021 +30703 33019 +30742 33010 +42647 33006 +14094 33006 +26845 33001 +38486 33000 +39351 32999 +43360 32998 +46243 32998 +44716 32993 +47222 32991 +33679 32989 +44937 32986 +35725 32982 +36638 32982 +2707 32979 +44462 32977 +40995 32977 +31675 32976 +26970 32967 +22340 32967 +40883 32966 +42491 32964 +21222 32960 +30689 32952 +43494 32950 +31722 32945 +45550 32944 +37844 32942 +45779 32939 +38749 32938 +43088 32936 +20028 32934 +32349 32933 +7775 32931 +34115 32929 +43635 32926 +44304 32923 +4192 32923 +42617 32923 +24613 32919 +42146 32919 +30181 32919 +17605 32915 +44712 32915 +31511 32908 +48145 32906 +34394 32906 +34942 32904 +37255 32902 +33754 32897 +33106 32895 +44720 32895 +38042 32895 +29061 32894 +17037 32890 +44861 32887 +28734 32885 +6984 32878 +44377 32872 +45451 32865 +43567 32865 +17204 32865 +43133 32864 +19220 32864 +13896 32862 +38573 32861 +44656 32853 +29584 32853 +33690 32852 +33712 32852 +27913 32852 +43598 32849 +28287 32848 +30896 32845 +42659 32844 +22011 32844 +28562 32843 +36115 32843 +49573 32838 +13451 32833 +18484 32833 +49559 32825 +37792 32824 +28877 32823 +37807 32821 +44142 32820 +47782 32817 +25009 32812 +8017 32810 +42633 32809 +31921 32808 +49331 32807 +17980 32805 +23830 32799 +34868 32796 +19998 32793 +25042 32792 +35858 32792 +37107 32791 +27783 32789 +24595 32784 +40968 32782 +45360 32780 +46392 32776 +43279 32773 +28468 32772 +40416 32770 +15214 32769 +31900 32765 +47958 32764 +17954 32763 +49237 32758 +42200 32751 +24192 32751 +22357 32750 +32409 32750 +36222 32749 +20904 32745 +5893 32745 +31899 32744 +6913 32742 +33390 32740 +41038 32739 +43560 32737 +41122 32732 +35333 32730 +20429 32730 +45041 32729 +9838 32729 +46806 32726 +22284 32724 +37398 32723 +31707 32723 +37202 32722 +41210 32717 +14334 32716 +48743 32715 +42755 32714 +27714 32713 +35108 32712 +31640 32712 +31550 32711 +48829 32711 +5993 32703 +46843 32701 +43107 32698 +13710 32698 +39959 32696 +35634 32692 +26150 32690 +42377 32690 +21560 32689 +39395 32686 +34833 32684 +40597 32683 +32525 32681 +11595 32680 +40017 32679 +6146 32677 +27424 32677 +30429 32677 +26948 32675 +32764 32674 +28327 32670 +36001 32669 +41674 32668 +49725 32666 +8380 32665 +29251 32661 +30766 32660 +11750 32656 +8521 32652 +48329 32647 +42128 32647 +34598 32643 +33504 32642 +32303 32640 +10196 32640 +34465 32636 +28557 32635 +18528 32634 +3293 32630 +45558 32630 +36765 32629 +36187 32629 +4080 32627 +32220 32625 +27781 32620 +36465 32613 +40191 32612 +28221 32602 +46188 32598 +41154 32598 +49245 32597 +29441 32597 +2467 32596 +1696 32595 +46419 32587 +3171 32585 +42859 32579 +35290 32577 +27652 32573 +39430 32573 +33169 32569 +37785 32565 +31084 32565 +36396 32562 +47019 32559 +24700 32558 +10847 32555 +23196 32551 +40501 32548 +39541 32545 +34563 32542 +48715 32541 +36318 32540 +33298 32539 +47320 32539 +34409 32539 +30127 32538 +30995 32537 +38691 32535 +48704 32533 +802 32530 +41418 32529 +16177 32526 +25855 32526 +26231 32525 +35857 32522 +19457 32522 +30123 32521 +35599 32521 +45802 32521 +35084 32519 +30434 32518 +43136 32515 +47948 32514 +47115 32513 +26342 32510 +18746 32509 +50102 32504 +39474 32502 +44193 32497 +27222 32497 +9311 32497 +8899 32494 +2027 32494 +38985 32491 +23364 32486 +8317 32483 +47119 32482 +4416 32477 +37312 32475 +3243 32475 +21181 32468 +42513 32468 +27342 32468 +36155 32468 +36731 32467 +40527 32467 +47820 32464 +46624 32460 +37433 32459 +46959 32451 +32389 32450 +13636 32449 +35669 32446 +15156 32445 +12773 32444 +36841 32444 +35563 32441 +39577 32433 +28062 32427 +49038 32426 +29854 32418 +47008 32417 +34751 32413 +48621 32412 +40525 32411 +27212 32409 +47054 32406 +45120 32403 +21551 32403 +49198 32398 +47619 32397 +14781 32397 +43082 32396 +50185 32395 +26265 32395 +33440 32391 +34275 32385 +47386 32379 +41428 32378 +35959 32377 +45889 32375 +34323 32372 +39599 32370 +41750 32368 +46991 32365 +40536 32363 +41527 32361 +39937 32361 +32282 32356 +35931 32353 +8569 32352 +11714 32346 +39804 32337 +5463 32336 +31496 32333 +33389 32333 +34757 32330 +21184 32330 +32637 32325 +40836 32323 +6863 32321 +46400 32320 +35322 32315 +43091 32309 +27287 32302 +42844 32297 +38515 32293 +24885 32293 +30832 32291 +40535 32290 +35352 32289 +38010 32288 +31883 32285 +32446 32283 +30824 32275 +36947 32275 +48363 32273 +38441 32273 +25818 32259 +36331 32253 +27728 32250 +41961 32248 +30869 32246 +40180 32244 +36855 32239 +37609 32238 +40529 32237 +30733 32237 +29930 32235 +42278 32235 +42414 32235 +24438 32233 +40409 32227 +34357 32224 +18731 32224 +41996 32224 +46345 32221 +40028 32219 +37110 32217 +33604 32217 +38945 32214 +11906 32214 +44262 32213 +42798 32212 +17801 32211 +32850 32211 +48910 32206 +28230 32205 +35880 32205 +35389 32203 +50099 32200 +35054 32196 +47749 32193 +47968 32193 +46945 32191 +28981 32190 +33267 32187 +34791 32176 +34071 32169 +33785 32167 +27453 32167 +11036 32166 +38424 32164 +22696 32164 +38436 32164 +47543 32163 +40699 32158 +46908 32158 +33064 32156 +12307 32156 +8988 32154 +29564 32152 +37700 32146 +36711 32145 +29855 32133 +13330 32132 +30291 32132 +12211 32132 +29813 32130 +20910 32121 +35897 32120 +19825 32119 +18998 32116 +29509 32113 +9381 32113 +45665 32108 +26286 32105 +30819 32104 +34114 32103 +14428 32102 +48455 32099 +12288 32098 +19540 32095 +39713 32092 +37417 32091 +43638 32090 +44553 32089 +46420 32089 +37109 32088 +37679 32087 +23775 32080 +34977 32079 +13625 32077 +19728 32075 +26583 32074 +29236 32073 +32522 32069 +18211 32065 +32638 32063 +36651 32062 +31717 32062 +43508 32058 +32045 32049 +36078 32048 +14070 32047 +42454 32043 +49127 32040 +35314 32038 +41951 32031 +43893 32031 +46967 32031 +36784 32029 +12826 32024 +47029 32022 +38049 32020 +13418 32019 +33326 32017 +36053 32013 +34616 32011 +37479 32010 +17130 32010 +43043 32009 +20535 32009 +36943 32008 +32066 32007 +12048 32006 +48837 31988 +47239 31984 +18627 31978 +42304 31974 +47267 31974 +9615 31967 +44794 31966 +30336 31965 +21175 31960 +13934 31957 +41106 31953 +14832 31953 +31980 31952 +47893 31950 +35164 31948 +44414 31947 +34587 31946 +35026 31943 +32732 31939 +2812 31939 +50160 31938 +42932 31938 +35371 31934 +43684 31933 +16338 31933 +45931 31930 +40296 31927 +21267 31926 +27049 31923 +26226 31921 +31570 31920 +34052 31918 +27977 31911 +20267 31909 +36182 31907 +44446 31906 +50034 31903 +29907 31900 +38444 31898 +6739 31897 +32544 31889 +41894 31888 +31180 31882 +27128 31882 +29695 31881 +34646 31880 +41244 31878 +31850 31878 +29620 31878 +32718 31876 +49285 31871 +19914 31870 +37861 31868 +47653 31867 +40087 31866 +36005 31858 +35033 31857 +22146 31853 +13634 31853 +49458 31850 +16664 31849 +44273 31846 +46173 31844 +50195 31840 +40313 31839 +25838 31838 +40330 31837 +30981 31836 +46461 31836 +32652 31836 +43545 31833 +13760 31832 +27679 31828 +42949 31828 +33435 31827 +48093 31827 +29348 31820 +18061 31817 +28660 31817 +27832 31811 +49264 31807 +22602 31805 +18770 31803 +17953 31802 +22431 31797 +32702 31794 +39248 31792 +44849 31792 +13057 31791 +15574 31788 +50013 31787 +35365 31787 +30440 31784 +27842 31784 +32159 31780 +14315 31779 +30387 31775 +29920 31769 +38074 31768 +44476 31766 +18113 31766 +31190 31764 +36329 31760 +39638 31757 +41809 31756 +38811 31756 +29856 31753 +48475 31748 +34444 31748 +47926 31746 +40068 31745 +46108 31742 +40774 31742 +8746 31729 +18216 31729 +35081 31728 +47107 31728 +23808 31727 +33191 31726 +25765 31724 +38331 31721 +34082 31721 +25553 31720 +34446 31720 +40112 31717 +24642 31717 +2730 31717 +29548 31716 +20540 31710 +28466 31703 +1668 31703 +38781 31703 +18906 31698 +50215 31698 +43068 31697 +23583 31695 +40980 31694 +44773 31693 +41286 31692 +45848 31686 +26742 31685 +11295 31679 +13842 31678 +22787 31678 +21168 31677 +30723 31676 +44776 31676 +40334 31674 +37552 31673 +36369 31673 +23892 31671 +27137 31669 +17249 31669 +31690 31667 +35808 31664 +23061 31661 +12413 31661 +41856 31657 +47259 31653 +47841 31652 +7010 31651 +40032 31644 +1409 31644 +43712 31644 +30092 31643 +38724 31640 +41177 31636 +50225 31632 +48509 31629 +26239 31621 +29075 31619 +30114 31610 +5902 31608 +48723 31599 +39809 31598 +41413 31596 +13658 31595 +49753 31595 +31911 31594 +14541 31593 +33144 31591 +22858 31588 +23876 31587 +35661 31581 +47865 31579 +29474 31578 +47254 31573 +24645 31573 +36544 31572 +26883 31572 +15541 31562 +37719 31562 +41655 31561 +47316 31560 +29516 31560 +26429 31555 +25994 31552 +28483 31550 +42204 31548 +28469 31541 +40869 31539 +28937 31538 +36402 31538 +11509 31537 +15622 31532 +30577 31530 +36959 31529 +30362 31528 +15839 31525 +44889 31523 +30057 31523 +45112 31522 +20587 31518 +25489 31513 +39254 31510 +47734 31507 +47204 31506 +45445 31506 +46070 31504 +35436 31503 +34750 31501 +23410 31499 +42248 31497 +36404 31493 +13595 31490 +35361 31490 +38240 31488 +43195 31486 +36654 31484 +44181 31481 +42193 31480 +33115 31479 +38850 31477 +29540 31475 +26065 31475 +24838 31474 +41631 31470 +27867 31466 +10289 31466 +6734 31465 +31713 31463 +27990 31459 +15829 31457 +43693 31455 +31240 31452 +44346 31447 +24988 31446 +49013 31446 +48973 31445 +23409 31445 +38024 31443 +39844 31442 +29288 31433 +37820 31431 +25480 31428 +34810 31426 +27854 31424 +33544 31422 +38475 31421 +23849 31420 +15560 31419 +40094 31417 +41517 31407 +43609 31402 +27578 31389 +36362 31387 +34144 31387 +30911 31386 +43556 31382 +26290 31375 +38531 31374 +42403 31371 +39955 31371 +37989 31370 +41399 31368 +18856 31366 +13470 31366 +45309 31360 +44205 31360 +35587 31359 +34665 31357 +39514 31354 +42596 31345 +47158 31341 +31345 31340 +44577 31340 +24932 31340 +22788 31338 +37582 31335 +48354 31333 +41062 31333 +48214 31332 +31602 31329 +23491 31328 +32847 31326 +36186 31326 +44722 31325 +36920 31325 +42461 31323 +46377 31322 +42497 31319 +39532 31317 +8004 31316 +19777 31316 +40873 31315 +35592 31314 +44834 31312 +40579 31311 +25405 31307 +32339 31304 +32749 31301 +38361 31299 +6967 31297 +27720 31295 +44678 31293 +27597 31289 +22067 31288 +35853 31288 +41118 31279 +13506 31277 +16454 31268 +22674 31267 +31601 31261 +38890 31261 +40818 31257 +17251 31256 +37137 31255 +48491 31254 +29191 31250 +45852 31248 +43371 31246 +46113 31242 +6231 31241 +48917 31234 +30182 31233 +17111 31233 +2591 31232 +6558 31229 +18758 31229 +11655 31228 +15093 31228 +27689 31226 +43578 31225 +28516 31223 +42153 31222 +39362 31220 +29750 31220 +38940 31214 +32424 31213 +32974 31213 +6820 31213 +10372 31208 +34190 31201 +40064 31193 +483 31186 +19191 31185 +44534 31184 +29498 31172 +35376 31168 +31672 31164 +48172 31156 +7985 31154 +30195 31153 +15407 31153 +48824 31152 +22664 31147 +34209 31146 +40562 31145 +1383 31144 +24774 31143 +38468 31141 +31632 31140 +23930 31135 +47961 31132 +44811 31131 +32624 31130 +40142 31124 +25989 31123 +40939 31122 +9093 31122 +16875 31120 +35552 31117 +12559 31116 +38405 31115 +35462 31112 +24686 31111 +5505 31109 +41536 31109 +35963 31106 +37382 31103 +41116 31102 +30847 31100 +46128 31099 +40750 31098 +50043 31096 +38545 31095 +42394 31095 +33026 31094 +29190 31093 +49309 31080 +40623 31079 +10631 31078 +18126 31074 +39975 31073 +21460 31073 +11569 31072 +37208 31072 +48308 31071 +32035 31068 +44527 31068 +17632 31066 +29571 31066 +26823 31058 +22556 31057 +32471 31056 +10987 31055 +29833 31054 +43325 31054 +31816 31053 +35132 31049 +47587 31048 +11525 31047 +28292 31045 +14585 31043 +5255 31040 +36454 31039 +42285 31037 +45583 31037 +33850 31036 +32075 31035 +31575 31024 +24453 31021 +47667 31020 +33145 31016 +23418 31016 +36441 31015 +47779 31013 +22589 31013 +46124 31011 +46638 31004 +47395 31001 +31306 31001 +42045 30999 +42958 30995 +5568 30994 +27881 30994 +30842 30992 +29171 30989 +46398 30983 +18171 30981 +34952 30976 +44085 30975 +46494 30974 +25594 30973 +27763 30971 +37658 30971 +49982 30968 +38410 30964 +38098 30960 +26375 30959 +23819 30952 +48692 30947 +34726 30944 +47448 30940 +38007 30939 +49556 30938 +45466 30937 +47751 30927 +49344 30926 +31756 30925 +10507 30922 +48044 30920 +38246 30917 +19019 30916 +35665 30914 +33037 30912 +33649 30912 +15419 30905 +36730 30904 +31055 30899 +39877 30898 +37687 30897 +39130 30895 +37959 30889 +21224 30883 +30714 30880 +23553 30876 +42308 30876 +47414 30873 +7562 30873 +37840 30872 +44300 30867 +36869 30866 +47519 30860 +21589 30860 +49591 30858 +41199 30856 +25790 30854 +46627 30853 +48097 30851 +48939 30849 +44245 30847 +32395 30844 +34499 30841 +35522 30841 +8527 30840 +48805 30839 +37967 30839 +43829 30837 +32827 30834 +24280 30833 +8725 30829 +37783 30828 +34107 30827 +43613 30825 +35152 30822 +37302 30822 +25150 30821 +11551 30816 +48669 30814 +33901 30812 +22070 30812 +7694 30810 +20314 30801 +22822 30799 +31916 30798 +21211 30797 +5388 30794 +34800 30789 +46531 30789 +49083 30789 +48454 30785 +46265 30781 +18546 30778 +42768 30777 +39939 30776 +31297 30769 +48020 30767 +45230 30765 +32859 30765 +23680 30765 +46564 30763 +34672 30762 +14744 30760 +37533 30760 +5266 30755 +41653 30754 +43035 30752 +23235 30748 +33058 30748 +50076 30741 +29048 30739 +34406 30739 +17403 30730 +6664 30726 +46394 30726 +22009 30724 +38668 30722 +16955 30721 +38939 30721 +46168 30712 +17786 30706 +44313 30704 +31380 30701 +20283 30701 +44809 30701 +23322 30698 +33686 30691 +40712 30691 +20189 30690 +30797 30688 +45455 30688 +23914 30686 +21988 30679 +45540 30677 +38609 30674 +37501 30673 +45794 30670 +25487 30669 +50153 30663 +26561 30659 +47136 30658 +27944 30657 +40826 30654 +46867 30652 +16775 30650 +35859 30649 +40145 30648 +37421 30647 +36798 30645 +29159 30644 +22796 30640 +29925 30640 +29455 30635 +25323 30634 +38647 30629 +47607 30626 +23210 30621 +21753 30616 +42835 30616 +48241 30613 +29879 30611 +25777 30608 +42732 30599 +40590 30597 +49529 30596 +41184 30594 +26518 30591 +1246 30591 +17891 30590 +14278 30590 +45092 30588 +41442 30587 +39132 30584 +41843 30583 +31504 30582 +42337 30580 +11367 30578 +35776 30578 +41110 30577 +15948 30575 +37482 30575 +29958 30572 +31237 30570 +13761 30569 +36417 30567 +10167 30565 +35601 30562 +13507 30559 +42025 30558 +41332 30557 +30949 30555 +44353 30551 +37300 30550 +44432 30542 +22074 30542 +22477 30541 +45260 30541 +37850 30540 +32989 30539 +16587 30539 +38198 30536 +42833 30535 +9810 30534 +27102 30530 +31591 30530 +48001 30530 +38708 30530 +35175 30526 +11457 30525 +25857 30524 +10535 30524 +44693 30517 +37530 30515 +29331 30514 +27047 30513 +41997 30513 +36140 30505 +45593 30502 +48446 30497 +18462 30496 +37795 30492 +15912 30480 +12489 30479 +41574 30476 +26156 30469 +30111 30462 +45581 30462 +3107 30460 +43983 30460 +35607 30459 +42708 30449 +45370 30448 +31284 30446 +31344 30444 +46981 30442 +39875 30440 +49567 30438 +18119 30435 +37179 30433 +31414 30433 +27233 30432 +11830 30431 +39775 30427 +46125 30424 +39977 30423 +46206 30423 +45292 30422 +45432 30422 +5687 30418 +37144 30416 +30098 30416 +42016 30410 +42160 30401 +30119 30400 +33890 30398 +26613 30397 +29083 30394 +23970 30394 +46418 30387 +18837 30385 +22271 30385 +47131 30383 +8908 30383 +44607 30379 +32642 30379 +48336 30378 +39867 30376 +28683 30376 +42041 30376 +32973 30375 +41395 30374 +40101 30372 +16923 30371 +4791 30369 +43409 30368 +25310 30367 +6935 30364 +18242 30356 +42912 30355 +32039 30354 +30864 30353 +44657 30352 +44714 30351 +50020 30350 +44472 30349 +2170 30349 +40168 30347 +11245 30346 +32077 30341 +38942 30337 +27661 30336 +25253 30335 +32690 30334 +46937 30332 +48644 30329 +42806 30327 +29053 30324 +16591 30324 +41091 30320 +43374 30319 +41968 30316 +44497 30312 +35788 30306 +33393 30306 +13136 30303 +36877 30300 +24099 30300 +50086 30295 +22734 30295 +49023 30290 +39498 30288 +13192 30285 +48575 30281 +45919 30273 +39311 30272 +1470 30270 +44297 30270 +27285 30267 +45204 30266 +30983 30263 +20965 30263 +21236 30261 +20598 30260 +23739 30256 +45374 30248 +38097 30247 +40594 30245 +27431 30239 +17121 30238 +26992 30230 +17478 30230 +18291 30229 +46944 30225 +20985 30225 +10875 30222 +14308 30220 +39659 30219 +43746 30217 +31700 30217 +16390 30217 +38101 30217 +42018 30205 +46694 30202 +46853 30200 +34566 30199 +39293 30192 +27332 30192 +11890 30188 +30955 30188 +47207 30187 +24123 30184 +30285 30183 +48778 30177 +37854 30175 +41187 30172 +49066 30170 +19691 30169 +44016 30168 +39331 30164 +38319 30163 +47791 30158 +39145 30157 +21520 30157 +9421 30157 +42779 30156 +41197 30154 +39469 30153 +37211 30151 +8995 30150 +23232 30147 +34466 30143 +36159 30142 +42715 30138 +6267 30135 +48830 30135 +9006 30133 +37263 30129 +21342 30117 +18343 30113 +26343 30112 +16992 30107 +48045 30106 +41834 30105 +46818 30103 +18597 30102 +26514 30102 +47188 30101 +42551 30101 +8334 30099 +37583 30096 +26024 30095 +46215 30094 +45479 30092 +48870 30091 +40810 30086 +20216 30085 +46454 30082 +36522 30081 +36154 30081 +45709 30080 +13056 30080 +46590 30078 +3120 30076 +49181 30074 +25085 30073 +27085 30071 +38895 30070 +41728 30070 +49259 30070 +49804 30069 +45397 30069 +38268 30066 +34313 30060 +30962 30058 +38596 30057 +34120 30054 +17925 30052 +41056 30051 +47052 30047 +31408 30035 +31338 30033 +40428 30031 +6160 30028 +43452 30025 +40800 30020 +50233 30016 +45404 30011 +28051 30005 +33034 30003 +46801 30002 +43080 29999 +43338 29992 +50156 29991 +35844 29989 +30450 29987 +27210 29987 +39980 29986 +48966 29986 +30737 29986 +41869 29981 +44670 29979 +28821 29976 +19840 29972 +35340 29971 +21747 29970 +30099 29969 +14918 29968 +15352 29965 +28171 29964 +45389 29956 +39052 29954 +48920 29946 +31887 29944 +41881 29944 +33551 29938 +49522 29936 +43662 29932 +25491 29930 +15979 29930 +38771 29929 +26674 29928 +28659 29926 +12018 29921 +49180 29917 +25372 29916 +42242 29914 +37617 29912 +30523 29909 +29284 29906 +48483 29903 +40723 29901 +33394 29898 +38578 29897 +44852 29895 +46336 29894 +45612 29893 +8751 29892 +46776 29892 +49389 29891 +42134 29890 +33706 29890 +33001 29887 +20217 29882 +45598 29876 +41402 29875 +41690 29875 +24983 29870 +33927 29869 +37759 29867 +48733 29866 +30665 29866 +19759 29866 +44222 29859 +45308 29857 +49207 29857 +49350 29857 +35674 29856 +41067 29856 +36890 29851 +38384 29849 +46790 29848 +10023 29848 +43123 29846 +44740 29843 +47973 29840 +35682 29839 +38273 29829 +32081 29828 +16202 29827 +36036 29824 +47091 29807 +33303 29806 +28479 29806 +44522 29804 +19106 29803 +10666 29799 +43466 29799 +38303 29799 +47334 29792 +41044 29783 +47523 29783 +10434 29783 +44924 29775 +33273 29772 +40999 29770 +5987 29769 +43779 29766 +14949 29763 +27660 29762 +42671 29756 +45164 29755 +33868 29755 +27375 29753 +35195 29750 +22741 29748 +13724 29747 +42275 29739 +21627 29737 +45151 29736 +38849 29732 +44109 29731 +42738 29730 +17443 29730 +47980 29729 +28342 29728 +41266 29728 +16801 29727 +36741 29726 +14994 29725 +48105 29710 +35186 29709 +20373 29708 +45967 29708 +44345 29704 +28042 29702 +38723 29694 +33370 29691 +15942 29690 +33820 29688 +29315 29688 +39071 29687 +45045 29686 +44173 29685 +9035 29685 +14464 29679 +34106 29676 +34692 29663 +26631 29661 +39475 29658 +31046 29656 +44637 29655 +43981 29651 +28315 29650 +34404 29650 +41432 29649 +34547 29648 +37205 29647 +19892 29645 +44141 29644 +42023 29642 +33618 29642 +27066 29638 +32871 29637 +30442 29634 +33300 29634 +25164 29629 +48266 29629 +31940 29628 +41070 29625 +47643 29624 +40315 29617 +40077 29614 +38255 29613 +5037 29608 +28540 29607 +36313 29606 +21772 29605 +39189 29598 +50103 29597 +31282 29597 +26842 29597 +39537 29593 +47001 29592 +15388 29592 +36296 29589 +36065 29587 +30457 29581 +44247 29581 +48492 29579 +43309 29579 +44039 29578 +41608 29578 +43247 29576 +23212 29574 +34956 29573 +49471 29573 +28166 29570 +45171 29565 +39894 29565 +46784 29564 +23154 29563 +32272 29561 +36262 29556 +33562 29552 +27468 29551 +33350 29551 +36089 29549 +41621 29549 +37627 29549 +33013 29546 +40293 29543 +23073 29536 +24010 29533 +4254 29529 +20490 29528 +20489 29528 +14222 29527 +10985 29515 +44064 29511 +30808 29510 +25412 29508 +23947 29503 +9945 29501 +43700 29498 +45894 29495 +26847 29494 +41933 29489 +41098 29489 +41004 29489 +18425 29489 +46833 29486 +34709 29484 +45153 29483 +28035 29482 +27984 29480 +14830 29475 +38756 29475 +22456 29473 +41288 29473 +19513 29472 +39423 29472 +31785 29470 +48061 29469 +30335 29468 +31224 29464 +25414 29462 +28311 29461 +15703 29461 +21844 29456 +34986 29456 +48325 29456 +46066 29455 +25871 29453 +42656 29452 +28219 29447 +49996 29447 +8503 29445 +48663 29445 +31125 29440 +39612 29434 +19640 29433 +35000 29433 +42256 29431 +34576 29428 +40469 29426 +34091 29425 +13132 29423 +15527 29421 +34772 29420 +45274 29418 +42176 29416 +26161 29412 +25214 29412 +2820 29409 +33531 29408 +49402 29407 +33043 29407 +48116 29406 +12321 29406 +42706 29405 +40096 29404 +49549 29403 +14550 29402 +9788 29402 +19491 29400 +47135 29399 +3659 29397 +24742 29396 +32148 29390 +36673 29390 +49186 29386 +49305 29384 +17408 29382 +47778 29379 +46166 29378 +16713 29377 +35972 29365 +32858 29364 +36100 29364 +26455 29359 +36724 29358 +28828 29355 +48963 29354 +38126 29353 +24284 29349 +37999 29346 +45255 29340 +36377 29339 +49663 29336 +39846 29336 +36993 29333 +39521 29329 +23657 29328 +33974 29326 +40557 29324 +5817 29322 +45888 29321 +36593 29320 +45622 29318 +22919 29317 +1857 29317 +49419 29314 +1554 29308 +47895 29307 +35752 29306 +44115 29304 +46504 29303 +31444 29302 +29392 29298 +42079 29290 +39858 29289 +41602 29284 +36221 29284 +35518 29281 +22061 29277 +13243 29273 +33407 29272 +41128 29272 +48411 29271 +30923 29270 +49758 29269 +38930 29265 +3762 29262 +22704 29261 +33432 29259 +18428 29253 +42677 29252 +45906 29252 +32770 29251 +20481 29250 +27260 29243 +45981 29243 +27494 29242 +17666 29241 +11395 29240 +39267 29239 +47844 29234 +25127 29233 +35435 29231 +33576 29229 +40297 29228 +32823 29228 +23868 29224 +20407 29224 +44475 29223 +35588 29218 +14172 29218 +37035 29214 +47737 29214 +26354 29213 +26433 29212 +36215 29204 +35229 29204 +37018 29201 +29175 29199 +16067 29198 +35227 29197 +36060 29197 +36153 29193 +20564 29191 +42665 29189 +26726 29188 +47874 29187 +47628 29182 +37590 29182 +42772 29180 +45039 29180 +47327 29178 +37170 29175 +38765 29173 +2028 29172 +28810 29169 +46058 29169 +37290 29168 +30826 29168 +43272 29166 +27108 29163 +31763 29161 +26868 29157 +47565 29154 +22502 29154 +47325 29154 +36042 29147 +44499 29141 +18467 29138 +25532 29138 +9798 29136 +31552 29129 +41307 29128 +42933 29125 +44460 29123 +38614 29120 +47501 29119 +27824 29118 +35821 29118 +33038 29117 +18208 29111 +46563 29109 +41768 29103 +43713 29098 +26079 29098 +38133 29096 +13812 29094 +41550 29090 +42560 29088 +23927 29087 +45105 29084 +41978 29083 +45843 29083 +37207 29082 +47541 29081 +18986 29075 +43365 29074 +35778 29074 +34522 29070 +35354 29069 +42674 29064 +20978 29063 +39471 29063 +25897 29059 +15653 29058 +45174 29057 +38492 29057 +35367 29056 +30460 29052 +33949 29050 +34975 29048 +34843 29047 +31697 29046 +26946 29045 +38349 29044 +24615 29044 +37904 29042 +38888 29040 +48388 29038 +21519 29037 +45087 29035 +45586 29035 +35451 29029 +40582 29028 +31203 29026 +26567 29025 +43033 29020 +39879 29018 +25933 29016 +30557 29016 +44407 29013 +43906 29013 +23067 29008 +45690 29006 +17152 29002 +30704 29001 +40867 29000 +36435 29000 +15695 28998 +36617 28996 +34770 28995 +48095 28994 +31963 28991 +48217 28988 +36022 28988 +45190 28980 +44434 28978 +49452 28977 +30015 28976 +8658 28973 +21734 28973 +41606 28973 +28529 28973 +25681 28968 +1827 28967 +28129 28963 +41301 28961 +8175 28959 +30480 28958 +37281 28957 +37106 28957 +46167 28955 +18191 28955 +39024 28955 +30760 28954 +37585 28954 +8374 28950 +30131 28949 +22277 28949 +44533 28948 +26748 28947 +17667 28945 +45040 28945 +10952 28938 +26793 28936 +31423 28927 +39055 28924 +38763 28924 +44073 28924 +49726 28915 +30083 28910 +20125 28910 +43139 28907 +34267 28904 +2609 28901 +18698 28899 +45312 28899 +44105 28898 +35214 28895 +43798 28894 +33543 28893 +45985 28891 +40284 28890 +36789 28889 +22390 28886 +24283 28886 +47166 28886 +34274 28880 +25848 28879 +35257 28879 +13372 28876 +9854 28871 +37011 28871 +43317 28866 +40227 28864 +14881 28864 +32513 28864 +47289 28863 +30271 28860 +49040 28856 +25350 28852 +29896 28847 +28043 28846 +44885 28844 +26882 28842 +3676 28842 +42464 28841 +36669 28838 +20984 28836 +28184 28836 +42957 28831 +44594 28829 +10336 28829 +34279 28826 +37830 28825 +43286 28823 +27598 28823 +38258 28823 +47422 28821 +11612 28816 +48397 28814 +9010 28810 +33957 28809 +41217 28806 +47940 28805 +25019 28805 +39951 28805 +37402 28802 +39033 28801 +47301 28797 +17426 28796 +38530 28794 +28720 28794 +36667 28793 +48259 28792 +31985 28789 +44624 28788 +37385 28787 +23886 28787 +48707 28784 +39705 28784 +47034 28782 +25002 28781 +41739 28780 +28884 28779 +40738 28779 +32328 28778 +36311 28775 +29973 28764 +43646 28761 +29436 28761 +37061 28760 +38369 28759 +10100 28759 +28912 28755 +35768 28753 +11836 28753 +45501 28752 +31710 28749 +30372 28747 +43141 28747 +25298 28746 +44964 28744 +46283 28743 +29308 28741 +27766 28740 +40658 28739 +23345 28738 +36403 28736 +16114 28735 +34324 28734 +41234 28733 +37209 28731 +23742 28727 +40675 28727 +17540 28724 +43124 28720 +26189 28719 +33818 28718 +30803 28718 +28256 28717 +33472 28715 +15166 28715 +9600 28713 +32990 28705 +48134 28705 +18398 28705 +16438 28703 +13394 28699 +42078 28698 +41042 28689 +8387 28688 +48384 28685 +28371 28684 +15730 28684 +36795 28681 +48993 28681 +31398 28678 +18509 28678 +33645 28675 +33316 28673 +43295 28673 +32266 28671 +49734 28671 +11090 28670 +45805 28670 +36286 28668 +15791 28668 +47183 28668 +45004 28665 +34433 28665 +33903 28663 +33053 28662 +22018 28660 +44732 28658 +21066 28657 +48464 28654 +14696 28653 +29726 28652 +34385 28645 +39269 28642 +35219 28639 +34628 28637 +15059 28636 +22215 28636 +46142 28634 +29416 28633 +30807 28631 +45456 28630 +3986 28622 +49775 28622 +36871 28621 +33639 28620 +35288 28618 +40200 28617 +37657 28615 +19928 28615 +31416 28610 +25388 28607 +42222 28600 +29088 28600 +34561 28598 +29814 28598 +11479 28597 +34831 28596 +49779 28596 +49301 28595 +43181 28589 +7495 28589 +45084 28589 +40075 28587 +37136 28585 +36273 28579 +36720 28578 +12292 28570 +32531 28569 +33355 28569 +49236 28569 +48698 28569 +10272 28568 +39947 28567 +37760 28566 +27971 28562 +43028 28556 +47094 28554 +40479 28552 +16244 28552 +49320 28552 +12455 28548 +22789 28546 +18296 28542 +37243 28541 +29305 28540 +28426 28537 +27610 28531 +7966 28529 +26976 28521 +49465 28521 +27145 28519 +50162 28517 +38901 28516 +49978 28510 +49212 28508 +39519 28508 +29766 28507 +40316 28506 +26075 28506 +35641 28506 +32459 28505 +45858 28504 +34391 28504 +29892 28501 +45055 28501 +50134 28500 +34512 28500 +27310 28499 +39418 28498 +25517 28497 +14054 28496 +29893 28492 +1354 28492 +34415 28491 +42374 28491 +47093 28490 +33741 28490 +29381 28488 +40838 28486 +32658 28482 +26920 28481 +28950 28479 +39185 28478 +39063 28477 +15978 28475 +32199 28475 +45703 28470 +33748 28467 +21544 28466 +42161 28464 +29390 28461 +38121 28459 +25812 28457 +39440 28455 +30343 28454 +31128 28453 +27841 28451 +35095 28445 +45096 28444 +47606 28444 +34515 28443 +2772 28442 +50072 28439 +14972 28438 +35447 28435 +18214 28433 +40802 28432 +30591 28426 +31828 28424 +24394 28423 +39144 28423 +45283 28420 +39843 28415 +47568 28414 +37283 28414 +39924 28408 +44034 28404 +30743 28400 +49435 28396 +28441 28392 +42388 28389 +39202 28388 +49227 28388 +48415 28386 +32187 28384 +49507 28382 +48377 28382 +36821 28371 +30419 28371 +40329 28367 +39238 28367 +46060 28365 +40463 28362 +38702 28362 +34722 28361 +38840 28361 +1121 28361 +45231 28359 +48969 28357 +44264 28356 +93 28353 +49401 28353 +16351 28347 +21979 28345 +43347 28342 +40023 28342 +20045 28341 +42699 28338 +35046 28338 +28927 28334 +11041 28333 +45663 28328 +28674 28326 +35868 28326 +33479 28324 +49871 28323 +25730 28322 +38867 28318 +49164 28316 +29937 28315 +42816 28313 +45414 28313 +4350 28312 +45393 28308 +20544 28302 +28709 28302 +47629 28302 +50042 28301 +38968 28297 +50190 28292 +38081 28291 +39982 28288 +37360 28283 +47266 28282 +45730 28282 +27966 28280 +15545 28276 +49242 28275 +45146 28274 +31729 28271 +39988 28270 +42102 28268 +42212 28267 +31524 28265 +25220 28264 +30465 28263 +35537 28261 +48355 28255 +36371 28253 +18909 28247 +37473 28246 +48352 28235 +22048 28231 +40271 28229 +2970 28219 +14427 28216 +42331 28214 +28270 28214 +34296 28212 +41704 28212 +48929 28206 +20235 28206 +23664 28204 +49957 28204 +31194 28198 +32042 28191 +38570 28190 +37750 28189 +39856 28184 +13021 28184 +49206 28183 +46240 28180 +38158 28179 +41487 28177 +28413 28177 +12619 28177 +49729 28176 +40721 28176 +13406 28172 +49786 28172 +35757 28168 +24633 28163 +33821 28162 +42831 28157 +46559 28156 +36013 28155 +45600 28150 +38404 28150 +34212 28146 +22840 28141 +27451 28140 +27713 28140 +30045 28139 +21741 28137 +44541 28137 +27087 28135 +36123 28134 +33253 28128 +34458 28124 +27934 28123 +49078 28121 +21716 28120 +30120 28119 +49698 28118 +33830 28118 +47890 28118 +16977 28116 +27417 28115 +14986 28114 +26997 28113 +23820 28108 +22907 28108 +43764 28107 +11747 28105 +30191 28102 +39048 28101 +45694 28100 +26165 28100 +24830 28099 +43622 28092 +28531 28091 +26120 28087 +25139 28084 +38773 28084 +49080 28077 +27507 28076 +35117 28072 +39247 28072 +47341 28072 +49308 28071 +37507 28065 +45296 28065 +18779 28065 +46845 28061 +30586 28060 +38589 28059 +33212 28056 +43472 28049 +12738 28046 +37964 28043 +5190 28035 +31048 28035 +48982 28034 +23723 28033 +14785 28031 +48927 28031 +40038 28030 +48706 28024 +20095 28022 +46553 28022 +39645 28019 +23578 28019 +43882 28019 +37419 28011 +24752 28009 +49448 28007 +39349 28006 +34307 28003 +17712 28002 +40272 28002 +42341 28000 +22479 28000 +49399 27997 +20539 27997 +43905 27996 +50065 27996 +14402 27994 +14118 27992 +32030 27992 +41537 27990 +44335 27989 +37734 27989 +18982 27988 +48100 27986 +22768 27984 +26651 27983 +47943 27981 +38622 27979 +32182 27979 +34432 27974 +50077 27971 +48074 27970 +46873 27969 +19027 27968 +40504 27966 +39594 27964 +24472 27964 +40788 27963 +34476 27962 +32695 27958 +18262 27958 +29999 27957 +22795 27956 +47992 27954 +48281 27952 +38476 27951 +35891 27949 +31014 27944 +42723 27943 +39171 27940 +20663 27938 +14534 27937 +26780 27936 +46473 27934 +42935 27934 +34066 27934 +36957 27930 +25021 27929 +41194 27925 +38792 27920 +18712 27919 +24997 27919 +34526 27915 +19549 27913 +47695 27913 +9428 27913 +22532 27909 +48101 27905 +41366 27900 +47884 27898 +31655 27897 +2379 27897 +15182 27896 +25264 27895 +34182 27894 +39606 27894 +29471 27891 +40093 27891 +29998 27890 +49587 27889 +10950 27885 +24961 27883 +22109 27877 +49354 27877 +42775 27876 +50011 27875 +25515 27872 +43845 27871 +48293 27866 +34353 27864 +31903 27864 +36908 27861 +41669 27858 +24965 27858 +45671 27856 +29769 27855 +40444 27851 +37381 27848 +23170 27848 +33378 27842 +25868 27837 +28056 27836 +46927 27836 +47332 27836 +49295 27835 +43534 27834 +36257 27828 +49620 27828 +21264 27827 +28240 27827 +31228 27825 +41851 27823 +37879 27822 +40369 27820 +48238 27819 +23561 27817 +9224 27808 +47832 27805 +22491 27803 +20094 27802 +1916 27800 +48051 27797 +46163 27791 +22476 27787 +48360 27780 +31907 27776 +21634 27775 +21602 27774 +47338 27774 +32407 27774 +17049 27773 +28945 27765 +13495 27764 +26833 27761 +22715 27759 +45743 27758 +18164 27754 +44772 27754 +45381 27753 +29409 27746 +23542 27745 +27349 27742 +32242 27741 +32935 27737 +47382 27734 +19966 27734 +39842 27734 +42218 27734 +8961 27731 +36165 27728 +31862 27725 +32100 27725 +32880 27725 +46647 27725 +27272 27718 +40845 27717 +49887 27713 +27178 27708 +32312 27705 +36495 27702 +42154 27702 +8019 27699 +49428 27697 +43843 27694 +9720 27693 +46778 27692 +36220 27691 +44129 27691 +34642 27689 +27552 27687 +40610 27681 +44477 27680 +31548 27679 +39336 27679 +27242 27678 +19456 27677 +30141 27673 +32417 27671 +42811 27669 +30536 27661 +37927 27659 +39704 27653 +40941 27651 +43914 27651 +15614 27649 +41605 27648 +34669 27646 +31578 27645 +40903 27643 +45437 27642 +21005 27642 +45273 27642 +45291 27638 +46213 27636 +29228 27630 +41717 27630 +49120 27628 +42510 27626 +41368 27626 +31702 27625 +40049 27625 +28532 27624 +38120 27624 +15316 27623 +44071 27623 +36185 27621 +40067 27619 +48797 27619 +43916 27619 +45756 27618 +27527 27616 +42773 27615 +34413 27615 +40933 27607 +33660 27605 +46605 27604 +44653 27604 +46054 27602 +44583 27602 +36277 27596 +20651 27594 +24992 27589 +36063 27588 +31877 27588 +20148 27586 +43071 27581 +38937 27580 +48815 27575 +41889 27575 +24639 27574 +33277 27564 +45541 27563 +27461 27555 +35190 27551 +41991 27546 +38877 27545 +39379 27544 +22212 27542 +46351 27542 +43417 27541 +15073 27540 +47632 27539 +42907 27538 +42260 27536 +41927 27534 +41278 27534 +39613 27533 +21642 27529 +32957 27528 +17896 27527 +26764 27524 +36491 27520 +41337 27517 +37034 27512 +25251 27512 +28781 27511 +47351 27511 +39669 27508 +38690 27508 +23743 27501 +36086 27498 +26213 27492 +45828 27489 +16203 27478 +29095 27478 +23551 27476 +48788 27475 +43552 27475 +39805 27472 +37147 27471 +46229 27470 +28063 27468 +9054 27466 +30406 27466 +39214 27465 +36999 27463 +49976 27463 +37962 27462 +45733 27461 +43110 27461 +8963 27460 +7513 27459 +8376 27458 +11449 27458 +31691 27457 +41015 27456 +48611 27450 +35830 27450 +28478 27449 +33834 27447 +24038 27447 +49824 27439 +47077 27437 +20419 27436 +25366 27436 +27099 27435 +37494 27435 +43886 27429 +40643 27428 +32523 27424 +39390 27424 +46030 27421 +36378 27415 +30695 27412 +49340 27412 +32388 27408 +31733 27407 +41663 27406 +33396 27405 +33771 27405 +34523 27405 +23535 27404 +30730 27399 +48637 27395 +44668 27389 +37532 27388 +28462 27386 +48029 27384 +48690 27383 +47602 27379 +46236 27378 +31841 27377 +48016 27376 +30702 27367 +34059 27365 +44463 27361 +47366 27361 +24481 27354 +40724 27352 +33032 27351 +42878 27347 +47256 27346 +12475 27343 +25741 27342 +38905 27342 +25942 27342 +38056 27342 +26030 27341 +25481 27340 +31196 27339 +6470 27338 +45955 27336 +47483 27335 +26512 27334 +21397 27333 +30919 27326 +32804 27324 +32799 27321 +50139 27315 +28067 27305 +39868 27304 +47024 27301 +30122 27297 +38332 27295 +27516 27293 +46718 27292 +36157 27291 +19226 27291 +44999 27289 +43919 27289 +39783 27287 +36688 27287 +40604 27285 +50202 27284 +6221 27279 +47270 27279 +41760 27278 +38305 27275 +34247 27275 +44011 27273 +29100 27271 +47092 27267 +30225 27267 +1372 27265 +44854 27263 +34044 27262 +20626 27261 +43489 27260 +35660 27260 +44246 27259 +39602 27258 +45831 27239 +44350 27237 +41241 27235 +42701 27235 +15724 27226 +42653 27220 +39699 27217 +30932 27215 +45716 27212 +40027 27211 +8647 27211 +19041 27210 +36653 27204 +46753 27203 +48186 27202 +11082 27199 +48094 27196 +13729 27193 +43356 27191 +23024 27190 +32700 27188 +43299 27188 +37846 27187 +33372 27184 +40706 27184 +35402 27182 +32585 27182 +11486 27178 +44517 27175 +26543 27168 +40134 27167 +21739 27162 +14853 27161 +43501 27159 +42777 27158 +37621 27157 +40187 27157 +48058 27155 +1369 27154 +15823 27153 +22482 27152 +49642 27150 +28638 27149 +38747 27147 +30944 27147 +37238 27146 +33507 27146 +39135 27145 +9820 27144 +32508 27140 +39059 27137 +7592 27136 +38170 27135 +37701 27132 +29032 27130 +33069 27129 +23747 27127 +33844 27124 +41138 27122 +47441 27118 +43621 27117 +31912 27113 +40844 27113 +22213 27111 +34374 27109 +25836 27106 +46688 27103 +42420 27099 +36116 27096 +42162 27096 +13579 27096 +42984 27093 +37847 27090 +38634 27087 +39840 27087 +34249 27085 +40306 27082 +17225 27081 +12570 27078 +46280 27078 +34489 27075 +37268 27072 +42087 27072 +42642 27067 +14965 27066 +36615 27065 +32040 27061 +34482 27061 +11373 27057 +35882 27055 +33882 27055 +47816 27051 +25431 27049 +41820 27045 +47867 27045 +34377 27041 +48673 27039 +40462 27035 +40303 27033 +4997 27033 +36936 27032 +29198 27031 +40922 27028 +16767 27025 +26461 27021 +11619 27015 +37645 27014 +40888 27014 +42467 27013 +47721 27012 +30146 27012 +22689 27010 +18090 27008 +29858 27006 +41479 27006 +34873 27006 +44389 27005 +47856 27004 +49880 27002 +17882 27001 +44658 26999 +42564 26995 +25114 26995 +16986 26995 +33400 26992 +38125 26991 +33008 26976 +21646 26969 +18760 26967 +35502 26965 +39204 26963 +38893 26962 +31662 26960 +21548 26956 +48166 26955 +31499 26949 +39899 26947 +35328 26941 +43650 26941 +27853 26940 +46630 26939 +44536 26939 +10628 26936 +33638 26934 +36517 26933 +44138 26933 +29371 26930 +43587 26929 +49202 26927 +21459 26924 +45560 26922 +46901 26922 +47509 26920 +45768 26920 +4008 26920 +11588 26919 +45613 26919 +36887 26918 +32998 26917 +42238 26913 +44102 26912 +42347 26911 +49924 26909 +47564 26906 +24914 26905 +25627 26903 +25140 26903 +32050 26901 +10694 26899 +38340 26899 +32460 26898 +19150 26898 +24295 26894 +37127 26893 +29326 26887 +20288 26883 +41385 26883 +35722 26880 +39903 26879 +45834 26877 +37673 26877 +43056 26874 +34471 26871 +22824 26871 +43483 26864 +16088 26863 +39277 26860 +36626 26859 +39392 26859 +36072 26855 +6396 26852 +47277 26852 +31339 26851 +42673 26844 +30768 26843 +39750 26843 +8844 26843 +38810 26838 +29744 26834 +23615 26833 +33117 26830 +47655 26830 +39402 26828 +43236 26828 +32373 26822 +45879 26815 +35499 26813 +44545 26813 +29627 26811 +15684 26810 +36243 26810 +19164 26808 +7936 26807 +50166 26805 +44053 26804 +37563 26803 +44068 26800 +6887 26799 +24398 26796 +32152 26794 +48140 26794 +42270 26791 +27916 26789 +35375 26789 +32399 26789 +20457 26784 +47187 26784 +48350 26783 +49680 26782 +29153 26781 +27838 26780 +43548 26776 +35677 26775 +32731 26775 +46090 26773 +48234 26773 +37241 26770 +44947 26768 +27115 26767 +9797 26766 +31020 26765 +7347 26764 +5829 26761 +37547 26761 +31837 26760 +15603 26760 +44426 26758 +45363 26757 +41284 26753 +33558 26752 +7398 26751 +49382 26745 +39954 26733 +21672 26733 +41112 26728 +23515 26727 +34314 26725 +33916 26725 +25726 26724 +61 26723 +24055 26720 +38823 26720 +32712 26717 +37320 26717 +40429 26716 +24229 26714 +46721 26713 +11538 26713 +42894 26709 +9642 26708 +34436 26705 +36375 26703 +49571 26700 +48730 26700 +31160 26698 +41306 26698 +11027 26697 +45922 26694 +40024 26693 +39279 26692 +37004 26685 +15509 26674 +37694 26673 +30504 26673 +34256 26671 +34159 26671 +24855 26671 +31654 26671 +45253 26669 +43232 26669 +48511 26668 +38013 26668 +28152 26668 +36625 26667 +24026 26665 +46139 26665 +38753 26663 +14050 26662 +24798 26660 +50131 26659 +17367 26656 +40215 26653 +38635 26653 +43333 26648 +43606 26648 +46119 26646 +22166 26646 +34424 26638 +47157 26636 +46252 26631 +34801 26631 +35059 26630 +9990 26629 +45233 26628 +43396 26626 +42335 26625 +30992 26623 +46027 26617 +34835 26617 +41243 26616 +38364 26613 +48211 26612 +25113 26612 +43674 26609 +42143 26605 +7892 26603 +15297 26601 +22785 26600 +47510 26595 +10352 26592 +44480 26592 +23722 26588 +35916 26588 +43435 26587 +29081 26587 +34951 26586 +46266 26585 +35124 26584 +39764 26581 +34606 26581 +6680 26580 +29120 26580 +31855 26579 +33552 26579 +12230 26579 +32811 26574 +33988 26573 +29819 26570 +38750 26570 +44782 26567 +36217 26563 +34184 26561 +26756 26558 +36879 26557 +49950 26557 +45643 26557 +22842 26556 +40432 26555 +50206 26551 +31280 26549 +18849 26548 +23113 26543 +49672 26538 +36649 26538 +33812 26536 +38726 26535 +44672 26534 +43891 26534 +46813 26534 +34825 26533 +50212 26532 +45968 26530 +35273 26521 +43973 26521 +35243 26515 +35617 26515 +16705 26513 +43470 26508 +27927 26507 +49296 26507 +36565 26507 +43439 26506 +42934 26504 +20854 26502 +49263 26494 +33542 26492 +22854 26491 +41129 26488 +24209 26487 +34743 26487 +32097 26478 +45625 26469 +32180 26468 +46272 26468 +32073 26468 +19956 26464 +15161 26462 +47224 26461 +44118 26461 +16421 26461 +26885 26458 +18855 26457 +45481 26450 +46705 26449 +48012 26445 +35824 26444 +36328 26444 +22298 26444 +47590 26441 +35246 26441 +48626 26439 +49410 26431 +35871 26427 +35360 26420 +15391 26418 +39600 26415 +49981 26411 +44062 26411 +36039 26402 +35998 26402 +42434 26401 +39035 26398 +36141 26398 +49353 26395 +8775 26393 +36210 26387 +44017 26386 +35419 26385 +24129 26384 +42961 26384 +39978 26380 +44976 26379 +24669 26378 +5950 26373 +30787 26369 +24897 26367 +27155 26366 +35076 26361 +42922 26361 +44853 26359 +26007 26358 +32643 26358 +27258 26356 +45060 26351 +35989 26346 +49376 26343 +31714 26339 +28556 26339 +21327 26337 +40295 26337 +25940 26337 +26988 26337 +42132 26336 +41207 26336 +32128 26335 +133 26334 +27411 26333 +38887 26332 +44832 26329 +46557 26328 +41024 26327 +38568 26323 +49889 26319 +12401 26319 +34450 26317 +33538 26316 +35068 26314 +8818 26313 +26869 26313 +33569 26313 +24446 26312 +7751 26310 +46101 26309 +36905 26308 +48122 26307 +48242 26306 +32746 26305 +41084 26304 +25905 26302 +21781 26301 +49861 26301 +7826 26301 +47696 26295 +20591 26292 +49588 26291 +35804 26291 +45553 26290 +49159 26289 +38151 26288 +44092 26282 +45188 26281 +28915 26280 +15905 26279 +47442 26276 +37151 26274 +43607 26273 +28841 26273 +31026 26272 +44983 26271 +3238 26268 +13635 26268 +22005 26265 +17457 26264 +34339 26264 +23339 26264 +44680 26256 +38714 26255 +45812 26255 +46750 26253 +37580 26252 +41672 26251 +22508 26244 +30390 26238 +46275 26236 +47263 26235 +40715 26233 +12722 26231 +18251 26230 +26550 26227 +42352 26223 +26769 26223 +37513 26223 +26556 26222 +44314 26221 +36373 26219 +45418 26218 +42970 26214 +15151 26211 +13882 26205 +21931 26203 +49631 26202 +41885 26200 +29203 26197 +38784 26196 +31076 26192 +5820 26191 +15838 26190 +15457 26187 +45905 26186 +47436 26186 +47391 26185 +29108 26179 +34140 26178 +34229 26175 +36149 26174 +20415 26166 +31216 26165 +22968 26164 +25396 26162 +24327 26161 +42149 26157 +39340 26154 +39548 26153 +40080 26147 +37939 26146 +35404 26146 +1782 26145 +45499 26144 +33961 26141 +46782 26136 +43936 26135 +11755 26133 +8730 26129 +42679 26127 +46952 26126 +35264 26124 +39761 26118 +15594 26112 +16296 26110 +35126 26109 +40571 26109 +32285 26107 +22246 26107 +30657 26106 +48755 26105 +28399 26103 +40427 26098 +44986 26096 +11178 26096 +40670 26095 +40622 26093 +43330 26092 +49913 26092 +37193 26091 +48152 26089 +11713 26089 +42683 26087 +20375 26087 +31952 26086 +32494 26085 +1580 26085 +47589 26084 +48299 26082 +30410 26078 +20017 26074 +27111 26068 +22138 26067 +45014 26066 +33985 26066 +40016 26066 +40194 26063 +29386 26063 +24125 26062 +8497 26061 +37171 26061 +25328 26060 +42049 26059 +38333 26057 +40086 26057 +1038 26050 +45102 26050 +41021 26048 +43500 26047 +23398 26046 +48916 26044 +42465 26044 +15258 26044 +27814 26043 +49208 26039 +14824 26035 +32361 26032 +46698 26031 +28173 26031 +30162 26030 +48247 26029 +35679 26027 +23768 26023 +48517 26021 +39535 26020 +17969 26019 +33137 26014 +48151 26013 +35586 26012 +28573 26007 +31491 26007 +25055 25998 +40467 25997 +14140 25997 +38603 25994 +42713 25994 +28341 25992 +3637 25991 +19527 25991 +38210 25988 +41193 25988 +41212 25987 +27749 25986 +13940 25984 +46423 25982 +46938 25981 +40567 25980 +49192 25977 +28555 25977 +36460 25977 +37853 25977 +42067 25972 +49596 25963 +18844 25960 +21417 25959 +47964 25959 +39527 25959 +29703 25959 +49455 25959 +49272 25957 +29143 25954 +31471 25952 +19242 25951 +29675 25949 +48077 25949 +19652 25946 +39625 25945 +28155 25945 +45584 25944 +37639 25943 +32997 25940 +27646 25938 +19663 25934 +27898 25933 +9402 25932 +42914 25932 +35074 25930 +34179 25924 +25801 25923 +32056 25923 +49085 25919 +46232 25919 +41283 25916 +25324 25915 +39928 25913 +46069 25909 +35536 25908 +47300 25908 +22743 25907 +30960 25903 +17611 25903 +15431 25903 +27796 25893 +35667 25892 +49673 25891 +48356 25891 +36618 25888 +22224 25886 +32622 25882 +9515 25880 +17071 25878 +20366 25877 +26620 25875 +18540 25874 +40580 25873 +32093 25872 +9165 25870 +22122 25869 +20944 25863 +25132 25863 +44757 25851 +44186 25851 +42801 25850 +43395 25848 +46308 25848 +45471 25846 +41139 25845 +38248 25843 +24958 25841 +22801 25840 +40085 25839 +40335 25834 +38683 25834 +24670 25833 +15659 25831 +40492 25829 +44511 25827 +27446 25823 +9187 25820 +40398 25819 +36410 25809 +11158 25807 +49393 25805 +34626 25803 +1519 25803 +36392 25802 +15361 25800 +45785 25800 +50243 25799 +49761 25796 +34347 25792 +42415 25792 +17995 25791 +43426 25789 +27202 25787 +33933 25783 +38766 25779 +3890 25779 +34830 25779 +21907 25775 +25603 25774 +25816 25774 +23864 25773 +35904 25770 +44100 25765 +22588 25761 +25548 25754 +34752 25752 +49014 25750 +31334 25749 +11724 25739 +30129 25733 +44006 25733 +36597 25726 +39212 25726 +41498 25723 +34565 25722 +39506 25722 +15159 25722 +34938 25718 +29936 25718 +16204 25718 +9304 25716 +22891 25715 +15946 25714 +11120 25711 +26710 25709 +42810 25708 +39624 25707 +49311 25706 +35150 25705 +39901 25703 +27770 25703 +49385 25702 +45957 25702 +36966 25699 +46322 25691 +35987 25689 +21283 25689 +38652 25683 +41687 25680 +27190 25679 +34201 25677 +44938 25676 +39195 25672 +28283 25670 +31011 25669 +20871 25666 +24062 25666 +26733 25662 +34510 25658 +49609 25654 +41465 25653 +42224 25652 +38077 25652 +40615 25649 +30790 25648 +25883 25647 +38944 25647 +48341 25647 +40694 25646 +38393 25646 +40090 25646 +47284 25644 +29757 25641 +43888 25638 +34896 25638 +28702 25636 +38324 25633 +31222 25632 +39646 25619 +36766 25618 +45469 25614 +42716 25613 +43478 25611 +33156 25608 +47406 25605 +10853 25603 +44813 25597 +35412 25595 +26634 25590 +37196 25584 +20521 25578 +40932 25575 +49636 25572 +40358 25570 +37887 25568 +19781 25568 +37982 25565 +47388 25565 +43574 25560 +31406 25559 +44570 25553 +30200 25551 +30433 25549 +23104 25538 +43871 25537 +49310 25535 +7658 25534 +40336 25532 +41041 25532 +40509 25531 +39054 25527 +35042 25526 +32457 25526 +41617 25523 +48961 25519 +37809 25519 +14684 25518 +33540 25518 +43340 25516 +6566 25516 +47059 25514 +42969 25513 +33965 25513 +39272 25512 +45934 25510 +4012 25505 +41168 25502 +30389 25499 +32842 25498 +35342 25497 +49639 25497 +30841 25496 +29478 25496 +39038 25496 +37710 25490 +41799 25487 +42354 25485 +39376 25483 +50112 25482 +38409 25480 +20303 25479 +22616 25477 +27863 25477 +21999 25476 +41613 25474 +4087 25473 +38239 25473 +48071 25471 +48581 25468 +45048 25467 +42325 25466 +28626 25465 +44548 25463 +8194 25463 +50150 25463 +46439 25462 +37161 25461 +11703 25459 +46591 25458 +19715 25458 +35551 25457 +38311 25453 +47455 25452 +22975 25452 +42457 25450 +49175 25448 +30682 25447 +43555 25445 +34784 25433 +24075 25430 +13327 25429 +8803 25428 +46969 25426 +25864 25421 +12966 25417 +28374 25413 +31527 25411 +17060 25404 +29192 25403 +26659 25401 +18003 25400 +44730 25399 +42083 25395 +32102 25394 +35120 25392 +48962 25392 +42865 25391 +37330 25391 +42387 25389 +41083 25388 +38366 25388 +31298 25387 +46465 25384 +33437 25377 +27739 25376 +48243 25374 +47043 25373 +30999 25373 +49265 25369 +22509 25369 +43924 25365 +14221 25364 +47774 25362 +24597 25358 +31054 25358 +46365 25357 +37230 25355 +47872 25353 +44662 25353 +46586 25352 +36023 25351 +36244 25347 +47912 25345 +32912 25345 +42733 25339 +47902 25337 +45480 25335 +47075 25333 +32334 25330 +49441 25326 +14375 25325 +28971 25324 +33022 25318 +40829 25315 +3030 25314 +24316 25314 +24812 25314 +49723 25312 +36087 25306 +45042 25304 +38450 25303 +41014 25300 +37593 25299 +6321 25297 +15071 25296 +27266 25296 +11808 25295 +43737 25295 +41724 25293 +41639 25291 +31260 25290 +24289 25289 +49830 25289 +41593 25283 +39930 25282 +25197 25279 +18402 25279 +37287 25278 +34126 25274 +41087 25273 +32218 25273 +7904 25271 +37572 25270 +40472 25267 +43528 25265 +41131 25265 +13009 25264 +40362 25260 +30563 25260 +48579 25252 +42982 25247 +43928 25246 +34637 25245 +49025 25244 +38328 25243 +7470 25240 +37309 25240 +38229 25232 +28998 25232 +50087 25230 +44993 25229 +25886 25225 +46859 25223 +46603 25218 +38922 25218 +41028 25212 +44777 25211 +43694 25211 +29334 25210 +15014 25209 +20682 25209 +7321 25209 +35965 25206 +40923 25203 +11834 25202 +11591 25200 +23066 25199 +8981 25196 +22876 25195 +36681 25194 +26961 25194 +20049 25192 +46797 25192 +21302 25190 +45837 25188 +48280 25187 +39264 25179 +39710 25175 +31953 25174 +48550 25173 +35310 25171 +22083 25171 +36676 25171 +46172 25170 +43773 25170 +3959 25168 +42203 25167 +49069 25166 +33459 25166 +40169 25165 +39591 25161 +32173 25161 +30566 25156 +47869 25156 +21855 25153 +36509 25152 +47842 25150 +36793 25148 +45714 25147 +46469 25140 +23308 25139 +38541 25138 +38322 25135 +12522 25129 +44737 25124 +36094 25124 +36207 25123 +23466 25121 +36705 25118 +42426 25118 +39006 25115 +49391 25114 +22178 25114 +32000 25113 +28380 25112 +25364 25110 +21720 25108 +38611 25107 +46122 25105 +35305 25103 +43114 25097 +12992 25096 +37635 25096 +8143 25089 +45921 25083 +38915 25083 +23351 25083 +35155 25083 +38150 25079 +41786 25078 +30283 25072 +31037 25071 +43354 25069 +42156 25069 +11121 25063 +47855 25062 +47309 25060 +35907 25057 +30752 25056 +42076 25056 +10072 25055 +25100 25054 +24016 25053 +35267 25053 +48418 25052 +46951 25051 +13814 25050 +46487 25048 +12502 25048 +46040 25048 +25975 25047 +42421 25046 +38446 25045 +8327 25044 +26595 25043 +25045 25040 +24074 25040 +19731 25037 +48606 25031 +38163 25029 +31418 25028 +37145 25027 +36536 25027 +10394 25023 +20679 25023 +24580 25021 +25252 25017 +33206 25017 +32079 25017 +42765 25016 +34980 25016 +39056 25015 +8323 25014 +49897 25012 +44103 25009 +48315 25006 +48149 25005 +49564 25003 +38259 25000 +49805 24999 +47716 24998 +41638 24998 +46029 24994 +33573 24994 +32140 24991 +42925 24990 +33646 24989 +36108 24989 +19128 24988 +18089 24988 +38212 24986 +29991 24984 +14767 24984 +38997 24980 +40540 24979 +38712 24976 +49483 24975 +33580 24974 +49593 24974 +33123 24968 +24151 24966 +47142 24966 +41864 24966 +35709 24963 +47362 24962 +11683 24961 +33342 24959 +48981 24957 +38076 24955 +13095 24953 +33548 24952 +39677 24951 +19608 24949 +38329 24949 +37008 24949 +26222 24948 +47487 24945 +29647 24944 +18096 24940 +23549 24939 +28690 24939 +36080 24938 +17094 24937 +33958 24937 +27186 24932 +41290 24929 +45186 24928 +27106 24925 +46943 24924 +25774 24919 +48990 24918 +33525 24917 +45110 24914 +32436 24913 +21038 24911 +33869 24911 +46594 24910 +40314 24910 +38506 24908 +17593 24908 +14519 24908 +39497 24906 +27034 24906 +33827 24904 +15501 24904 +23129 24903 +20966 24903 +28191 24902 +45741 24901 +49629 24901 +47481 24901 +21912 24900 +41121 24899 +38171 24898 +18123 24893 +36898 24893 +44491 24892 +46592 24891 +27037 24891 +45245 24888 +44496 24887 +35993 24885 +32908 24885 +14374 24884 +43576 24878 +28693 24876 +49619 24875 +47291 24874 +34620 24870 +37098 24867 +43281 24866 +42294 24864 +16549 24861 +46734 24855 +49154 24854 +18318 24854 +45420 24854 +40488 24850 +25755 24846 +38859 24845 +46087 24832 +23567 24829 +39318 24829 +4363 24828 +40862 24826 +39595 24825 +49380 24816 +48921 24812 +38089 24809 +32158 24808 +31384 24804 +41890 24801 +45780 24801 +47007 24794 +22433 24793 +13046 24793 +38524 24792 +38054 24792 +10259 24789 +10149 24784 +39416 24783 +45320 24783 +17570 24777 +28281 24775 +18829 24774 +46291 24773 +34403 24771 +36366 24768 +20012 24767 +43250 24766 +14829 24765 +21370 24762 +31115 24759 +5470 24758 +36864 24755 +38483 24753 +36306 24751 +27314 24750 +33788 24749 +42662 24749 +48348 24748 +47843 24745 +15158 24743 +44689 24743 +10534 24734 +48766 24734 +26959 24732 +41285 24732 +20563 24732 +34559 24728 +47851 24722 +47221 24722 +43161 24720 +18186 24713 +44201 24711 +29438 24701 +2273 24701 +5790 24699 +30443 24696 +49440 24695 +35324 24690 +39468 24689 +49105 24689 +30021 24687 +44951 24685 +10818 24684 +15067 24682 +48651 24682 +42526 24680 +31407 24680 +28045 24679 +48135 24678 +38721 24677 +12276 24677 +47462 24676 +37833 24672 +35247 24666 +42505 24666 +43370 24663 +44841 24660 +35708 24660 +48025 24659 +42446 24656 +24508 24656 +46529 24655 +11235 24655 +39064 24649 +37229 24648 +44089 24648 +15480 24643 +36627 24642 +45731 24641 +24148 24641 +24044 24636 +30671 24634 +38657 24632 +45141 24630 +45629 24628 +39744 24626 +34817 24623 +43603 24623 +41180 24621 +45423 24620 +48430 24619 +30398 24619 +47243 24617 +27808 24616 +19151 24614 +42108 24610 +42137 24604 +43963 24603 +6274 24601 +46273 24600 +33215 24599 +29774 24597 +44088 24597 +30964 24597 +45002 24595 +34625 24595 +29977 24591 +36480 24589 +47390 24588 +35699 24585 +39837 24583 +24467 24583 +47780 24581 +38084 24577 +43486 24576 +14311 24574 +47337 24573 +47352 24572 +29969 24571 +41544 24567 +50147 24554 +37102 24551 +27093 24549 +32321 24549 +37570 24548 +41755 24548 +20719 24546 +35740 24545 +4085 24542 +49599 24541 +20639 24537 +24204 24535 +9205 24529 +34490 24525 +11928 24520 +32226 24517 +42707 24513 +33669 24512 +45578 24509 +27569 24507 +40383 24506 +29810 24506 +42315 24505 +31150 24503 +46042 24503 +37251 24500 +46674 24494 +43658 24494 +41061 24491 +35649 24490 +43151 24490 +24432 24488 +34370 24487 +40791 24486 +39686 24485 +23802 24482 +11964 24482 +29603 24481 +36144 24481 +28770 24478 +39009 24474 +32627 24472 +44514 24469 +47153 24469 +24629 24468 +33088 24468 +33192 24466 +44526 24463 +19879 24462 +29185 24461 +49246 24457 +32062 24452 +30539 24448 +36657 24447 +39158 24445 +40286 24444 +47833 24444 +40812 24444 +33938 24442 +43166 24441 +12626 24441 +48024 24439 +29408 24438 +28151 24430 +28124 24427 +49848 24426 +47208 24421 +7654 24421 +48957 24419 +46282 24419 +33317 24416 +7499 24416 +9291 24410 +37614 24408 +41472 24408 +26714 24408 +1968 24407 +29839 24407 +8921 24406 +46462 24405 +49932 24404 +26797 24403 +33227 24402 +29346 24402 +37466 24402 +27750 24397 +36777 24390 +38172 24388 +36551 24385 +27825 24382 +46580 24379 +45874 24379 +40464 24377 +2602 24376 +38675 24373 +32427 24373 +45185 24371 +30214 24369 +16935 24368 +21321 24365 +49447 24363 +43122 24359 +44022 24359 +49975 24358 +22155 24357 +44958 24353 +36629 24352 +16877 24351 +28390 24350 +45956 24348 +38875 24348 +44631 24340 +43935 24339 +26151 24338 +37542 24335 +41258 24333 +29501 24333 +49919 24328 +36397 24324 +10186 24323 +42744 24318 +22812 24314 +19677 24313 +24631 24313 +19466 24312 +48474 24311 +1284 24310 +31882 24310 +37076 24309 +44966 24306 +29400 24302 +49968 24301 +37068 24301 +38974 24299 +44695 24298 +28343 24294 +35615 24287 +164 24283 +27380 24283 +20507 24282 +49956 24281 +46098 24279 +45607 24279 +34310 24278 +34287 24277 +45343 24277 +34535 24276 +32277 24273 +49895 24271 +43224 24268 +30643 24267 +39682 24266 +40206 24263 +6525 24261 +29506 24260 +40787 24258 +40305 24256 +44979 24255 +38232 24253 +45000 24250 +21597 24249 +30152 24242 +46899 24242 +48244 24241 +30028 24240 +33971 24240 +35970 24239 +48225 24239 +16733 24235 +42651 24235 +32217 24235 +44789 24230 +23640 24229 +29173 24228 +36680 24228 +28722 24218 +18900 24215 +38746 24215 +45478 24215 +27784 24214 +40609 24213 +44681 24205 +43677 24201 +36979 24194 +37992 24194 +39100 24193 +29616 24193 +5880 24189 +44065 24187 +42364 24186 +35403 24184 +27514 24183 +41959 24181 +43138 24177 +6463 24174 +27871 24174 +24899 24174 +45976 24172 +43885 24167 +46074 24167 +47709 24165 +36393 24160 +37169 24156 +31964 24155 +35846 24155 +32562 24153 +21436 24148 +41515 24147 +3593 24147 +31304 24146 +36749 24140 +22845 24138 +33698 24136 +26586 24134 +38909 24134 +49191 24132 +37002 24132 +9531 24128 +45969 24121 +34407 24119 +30546 24118 +47378 24117 +46164 24115 +41132 24109 +45355 24108 +39721 24108 +23779 24104 +42227 24104 +36247 24102 +33220 24101 +39380 24100 +43957 24098 +50127 24097 +19946 24096 +16804 24096 +24809 24096 +8053 24096 +30578 24095 +21179 24095 +37528 24095 +48412 24091 +45016 24089 +22702 24087 +15580 24086 +45477 24086 +25666 24084 +38595 24083 +47840 24080 +48996 24075 +41797 24073 +12757 24070 +48083 24066 +23279 24057 +36760 24056 +4616 24056 +38873 24054 +33981 24050 +40076 24049 +46628 24047 +44822 24047 +46641 24043 +35602 24042 +20980 24041 +42363 24038 +38380 24037 +5561 24037 +39706 24035 +17839 24031 +48787 24029 +5918 24029 +23317 24027 +23495 24027 +1289 24026 +37469 24026 +34583 24023 +23449 24020 +47753 24016 +36570 24013 +36323 24012 +43329 24012 +33102 24009 +44530 24009 +41526 24007 +46323 24007 +22293 24006 +30484 24002 +22214 24002 +34467 24000 +32139 23999 +36434 23999 +31035 23998 +46794 23995 +25492 23994 +22764 23993 +46436 23988 +48036 23982 +27550 23977 +43220 23970 +29457 23965 +12337 23961 +35638 23960 +18068 23958 +29601 23957 +26750 23957 +41367 23955 +37396 23955 +35010 23955 +40739 23954 +48793 23954 +47923 23954 +19400 23946 +23582 23946 +40569 23945 +47953 23945 +37282 23945 +42937 23943 +36298 23942 +49048 23942 +33832 23940 +11604 23939 +45460 23938 +24667 23933 +41656 23931 +45660 23929 +49745 23929 +48196 23928 +49163 23928 +28605 23923 +25189 23919 +21924 23918 +18124 23917 +38339 23917 +43836 23913 +28029 23913 +44093 23913 +30377 23912 +19913 23911 +14527 23905 +13581 23903 +49741 23903 +48643 23902 +38431 23902 +43495 23899 +26679 23897 +49854 23893 +49941 23893 +38989 23892 +47906 23888 +42570 23886 +37682 23885 +25265 23883 +33302 23880 +48055 23878 +5244 23876 +31072 23872 +38976 23871 +42319 23870 +4118 23869 +16795 23867 +47740 23867 +18736 23864 +49904 23862 +21727 23862 +26468 23855 +38547 23854 +47673 23853 +34822 23852 +163 23851 +45176 23849 +39882 23848 +44664 23848 +26255 23847 +18305 23840 +37730 23836 +38525 23832 +48710 23831 +31162 23827 +37998 23821 +31835 23819 +48255 23817 +47944 23817 +26141 23811 +42648 23811 +37431 23810 +35286 23807 +33872 23807 +40125 23807 +45564 23806 +33921 23803 +26650 23802 +46601 23801 +40103 23801 +33665 23800 +43759 23799 +16469 23797 +36907 23796 +41029 23793 +29458 23791 +38107 23791 +21659 23791 +21238 23786 +47873 23785 +40758 23777 +46138 23775 +45326 23775 +31497 23774 +44206 23773 +46396 23771 +49378 23769 +39472 23766 +12124 23764 +8349 23761 +33079 23761 +50117 23759 +26713 23756 +15189 23755 +43788 23752 +41174 23751 +22177 23749 +35978 23748 +29422 23745 +39215 23745 +25962 23744 +29258 23744 +49566 23743 +13866 23740 +23450 23738 +45543 23731 +42305 23730 +44865 23729 +16735 23729 +41564 23728 +38737 23727 +45679 23726 +43469 23721 +42584 23719 +33610 23717 +26404 23712 +40437 23711 +22820 23711 +36462 23710 +48459 23709 +32953 23709 +15576 23705 +16140 23704 +39256 23701 +49225 23699 +33819 23698 +49716 23695 +14163 23694 +5546 23694 +40600 23693 +46324 23686 +29922 23685 +50229 23684 +42795 23682 +38147 23681 +49992 23678 +40887 23673 +4972 23666 +7568 23664 +39058 23663 +41882 23662 +49077 23661 +35092 23661 +31466 23660 +38871 23658 +32583 23658 +23899 23658 +20649 23656 +32672 23655 +49695 23653 +45727 23652 +22090 23650 +43743 23649 +39305 23649 +32703 23646 +40022 23644 +39549 23643 +48675 23643 +39609 23642 +33969 23640 +43149 23638 +39623 23638 +44263 23637 +10292 23633 +14117 23632 +44002 23629 +34085 23624 +44290 23622 +34728 23620 +22802 23616 +27603 23615 +43538 23614 +42206 23610 +42051 23610 +46731 23606 +42537 23605 +45289 23604 +20207 23604 +40198 23599 +39139 23595 +43831 23593 +48112 23591 +40732 23590 +44027 23590 +12448 23589 +38539 23589 +24113 23587 +11097 23583 +50082 23581 +26175 23580 +26295 23579 +36330 23575 +23925 23573 +47132 23570 +47674 23570 +48898 23570 +42155 23570 +3882 23569 +43147 23569 +35066 23568 +49148 23568 +36052 23567 +40660 23567 +10956 23566 +48862 23564 +28688 23564 +37923 23564 +49124 23563 +43711 23562 +16554 23561 +36357 23557 +20530 23551 +47687 23551 +35716 23545 +43964 23545 +39773 23538 +42489 23534 +35910 23533 +42582 23532 +46983 23531 +43980 23530 +34645 23529 +38249 23529 +40354 23524 +32904 23520 +37132 23519 +41081 23517 +20470 23515 +37622 23514 +37071 23510 +44471 23509 +26417 23508 +19080 23508 +47398 23507 +47836 23505 +40321 23495 +48275 23493 +38440 23489 +9186 23489 +44915 23489 +44682 23483 +37225 23476 +33042 23474 +31191 23474 +28081 23472 +41859 23466 +30746 23465 +12866 23465 +32535 23465 +13128 23464 +30106 23462 +19881 23458 +39003 23457 +34131 23448 +27500 23447 +38706 23447 +16595 23444 +35848 23442 +32663 23439 +13283 23437 +43984 23436 +43951 23434 +12610 23433 +48903 23430 +41983 23428 +16837 23423 +20896 23423 +20926 23420 +6830 23419 +38275 23415 +40882 23413 +28435 23413 +30764 23413 +34376 23412 +44551 23412 +39119 23412 +36024 23411 +43011 23410 +43218 23400 +17821 23400 +12814 23397 +43918 23394 +9184 23392 +39252 23389 +38224 23385 +30193 23384 +8038 23383 +46789 23377 +50035 23376 +35085 23374 +39382 23373 +46099 23372 +32477 23365 +15022 23362 +18923 23361 +31823 23360 +38159 23360 +45158 23356 +41634 23356 +28708 23353 +34717 23351 +36891 23350 +24102 23350 +49951 23349 +35366 23344 +43482 23342 +22938 23341 +3928 23337 +42590 23337 +24518 23337 +36914 23334 +10567 23333 +41412 23333 +37308 23332 +11360 23331 +17320 23331 +42608 23330 +43334 23326 +23986 23320 +12975 23320 +49623 23319 +31847 23318 +43234 23314 +39108 23313 +42272 23313 +15540 23312 +26087 23306 +34917 23306 +40893 23301 +41303 23297 +38205 23293 +37364 23287 +30488 23287 +49392 23286 +5058 23282 +38473 23280 +2572 23280 +40455 23280 +24383 23279 +19056 23273 +17015 23272 +37474 23272 +28812 23271 +37249 23269 +45428 23269 +32320 23268 +26789 23265 +8247 23264 +45514 23263 +41447 23262 +46443 23260 +44465 23260 +11953 23259 +47230 23255 +41632 23255 +28260 23253 +41619 23252 +47829 23249 +48389 23249 +37703 23247 +47446 23246 +45244 23246 +47234 23245 +37020 23243 +29629 23243 +26121 23238 +12207 23230 +42297 23228 +26499 23226 +24206 23224 +46844 23224 +36488 23222 +19492 23222 +40740 23220 +45806 23219 +47669 23219 +39997 23213 +29022 23211 +42762 23210 +17116 23206 +11929 23204 +49433 23201 +35697 23199 +18960 23198 +47452 23196 +37021 23193 +42661 23193 +26288 23193 +37515 23185 +38733 23185 +48827 23185 +37825 23182 +10208 23181 +17802 23178 +24393 23178 +48943 23174 +38346 23164 +35990 23163 +45095 23153 +6328 23153 +45317 23152 +34420 23152 +37603 23149 +42936 23147 +38569 23145 +44816 23143 +38575 23142 +42611 23141 +28989 23141 +5130 23141 +13382 23136 +45822 23135 +38402 23134 +45799 23133 +21705 23133 +27595 23130 +27959 23128 +40047 23121 +49317 23118 +47626 23117 +36931 23116 +43921 23115 +33241 23115 +41677 23115 +39409 23114 +42013 23114 +47938 23113 +37153 23113 +46416 23112 +42989 23110 +27280 23109 +38741 23106 +33759 23100 +37823 23100 +39328 23098 +11051 23096 +24637 23091 +36809 23090 +36913 23089 +34578 23089 +46318 23087 +39352 23084 +29688 23080 +47134 23076 +22037 23074 +22914 23073 +26523 23072 +43096 23072 +41775 23072 +26955 23065 +28391 23064 +36859 23064 +40821 23063 +46755 23062 +46565 23060 +43407 23059 +27850 23053 +48520 23051 +162 23050 +32863 23047 +30273 23046 +32126 23044 +5418 23043 +19005 23042 +38022 23041 +41167 23040 +48197 23034 +49846 23033 +30677 23029 +47049 23029 +30816 23029 +36996 23028 +31234 23025 +47910 23022 +41322 23022 +43187 23022 +43132 23018 +35394 23017 +45207 23014 +25456 23013 +13781 23011 +36974 23008 +28675 23008 +27740 23008 +6815 23007 +49473 23006 +22628 23005 +36505 23002 +35549 22999 +48154 22998 +30085 22997 +34776 22996 +47803 22995 +38313 22985 +35467 22983 +47627 22977 +24095 22973 +45657 22971 +40965 22969 +19873 22965 +46408 22962 +37869 22960 +48934 22957 +23433 22957 +45604 22956 +36951 22949 +27413 22948 +29693 22948 +46338 22945 +43366 22944 +23441 22935 +47057 22933 +45631 22931 +39667 22930 +38254 22929 +38472 22927 +35623 22926 +32962 22925 +48577 22922 +26699 22919 +24547 22919 +45191 22917 +39466 22916 +48461 22916 +47047 22916 +45621 22913 +38533 22910 +25066 22910 +47104 22909 +42279 22907 +34427 22906 +29978 22903 +27226 22902 +34251 22894 +11456 22893 +43072 22887 +32725 22882 +35962 22879 +39320 22879 +9282 22875 +8186 22873 +49867 22871 +45534 22868 +35391 22868 +37895 22866 +3209 22864 +46493 22861 +39448 22861 +18745 22858 +41785 22856 +25528 22854 +31705 22853 +33616 22852 +39518 22852 +43156 22849 +43705 22847 +32488 22847 +42449 22841 +38959 22839 +27294 22838 +31218 22837 +47045 22835 +34765 22825 +48542 22824 +43539 22822 +22941 22821 +47112 22819 +40881 22816 +44457 22816 +31452 22815 +31516 22812 +33962 22812 +49820 22811 +34343 22810 +41923 22806 +38831 22806 +36678 22805 +40328 22805 +34416 22805 +45001 22805 +10563 22804 +34574 22804 +39799 22803 +41338 22803 +39545 22802 +37060 22801 +40681 22801 +39495 22801 +25262 22800 +15620 22799 +45974 22799 +22398 22796 +49721 22795 +21464 22789 +46671 22789 +40141 22788 +24717 22781 +48913 22781 +38666 22780 +41649 22776 +48883 22773 +12334 22773 +36729 22773 +33887 22772 +48426 22767 +33697 22767 +48811 22767 +28407 22765 +25257 22764 +44455 22762 +35876 22761 +25184 22761 +46831 22759 +47194 22755 +23569 22753 +40408 22751 +49914 22746 +37737 22740 +29152 22739 +37618 22739 +33808 22737 +30296 22735 +20021 22735 +29910 22733 +30610 22733 +36734 22733 +31751 22731 +45331 22731 +4779 22722 +37711 22720 +41569 22719 +42872 22716 +32082 22709 +41589 22708 +50007 22705 +41093 22702 +37864 22701 +32322 22697 +48817 22697 +6346 22692 +46735 22690 +36723 22684 +41612 22684 +50193 22684 +33104 22684 +15358 22683 +37831 22679 +39563 22679 +46642 22678 +35171 22677 +44361 22670 +33791 22666 +31024 22665 +49326 22665 +39610 22664 +50078 22661 +38967 22653 +40554 22652 +30631 22644 +35369 22640 +48868 22633 +45650 22633 +35885 22632 +39101 22631 +47883 22630 +39709 22629 +34153 22627 +46186 22625 +20120 22623 +30731 22622 +50208 22621 +43485 22621 +41491 22621 +44582 22620 +27427 22617 +37857 22615 +34439 22613 +32060 22610 +40819 22606 +12339 22601 +36699 22599 +33700 22599 +46376 22599 +43570 22598 +37576 22594 +8586 22594 +36785 22577 +2981 22577 +24061 22576 +37893 22571 +42725 22571 +27543 22568 +50105 22568 +49453 22567 +42794 22566 +42050 22561 +19697 22561 +46241 22560 +39985 22558 +11716 22558 +47449 22553 +46199 22552 +46094 22552 +43565 22552 +43337 22551 +22806 22551 +42081 22550 +35291 22549 +40231 22547 +37972 22545 +16344 22543 +24675 22541 +40652 22541 +39512 22538 +45068 22535 +47102 22534 +40149 22534 +27964 22534 +35562 22533 +49177 22527 +41190 22526 +32263 22526 +46446 22525 +30342 22524 +50023 22522 +48345 22522 +49079 22522 +35700 22518 +48559 22518 +33924 22517 +39925 22517 +34185 22516 +37856 22516 +47372 22515 +48324 22514 +48513 22512 +17077 22512 +13654 22511 +47314 22510 +47480 22509 +48872 22507 +31242 22507 +46350 22506 +36260 22502 +36088 22501 +40861 22499 +36561 22496 +41453 22496 +48950 22492 +36118 22485 +33246 22484 +36048 22483 +48683 22483 +35475 22481 +21263 22479 +43101 22477 +21760 22474 +23286 22473 +24244 22471 +39093 22470 +45070 22469 +35544 22467 +25261 22466 +46309 22464 +38422 22462 +16541 22457 +39779 22457 +21595 22455 +45019 22449 +34961 22447 +46001 22445 +49920 22443 +47302 22441 +42069 22434 +37028 22434 +15428 22434 +47356 22433 +18946 22432 +15390 22429 +2976 22426 +50046 22426 +44784 22425 +47684 22423 +48079 22421 +41172 22419 +46141 22419 +8885 22417 +16023 22416 +59 22415 +48419 22413 +48792 22412 +42122 22410 +46682 22409 +16515 22408 +21480 22407 +18332 22405 +9542 22404 +28764 22404 +46389 22395 +42516 22395 +36368 22395 +4413 22392 +5712 22390 +33932 22388 +14673 22384 +46082 22384 +44550 22383 +42398 22380 +22443 22379 +30374 22378 +44641 22377 +38298 22373 +39174 22371 +23607 22368 +35739 22367 +40956 22365 +22539 22364 +30908 22356 +47599 22355 +48370 22354 +42846 22354 +42043 22351 +27489 22348 +32146 22348 +47355 22348 +42163 22347 +48359 22347 +43355 22346 +18588 22345 +26625 22344 +47159 22340 +25954 22339 +32617 22339 +42666 22338 +7143 22338 +13900 22334 +46804 22334 +50021 22333 +34346 22331 +23637 22330 +47067 22330 +30117 22327 +16367 22326 +31144 22321 +40619 22320 +49624 22318 +7917 22316 +32676 22315 +42330 22313 +20498 22308 +36801 22305 +21582 22305 +29680 22300 +44074 22300 +44228 22300 +30206 22297 +16297 22296 +30077 22296 +21706 22292 +40749 22291 +20686 22288 +2914 22286 +41425 22284 +35012 22282 +20737 22281 +41734 22279 +5138 22278 +28860 22265 +44382 22261 +44665 22261 +31992 22259 +50197 22259 +40985 22258 +22104 22256 +47679 22255 +15095 22253 +42061 22252 +26644 22249 +45029 22249 +19797 22247 +49799 22240 +47475 22240 +49801 22239 +39299 22233 +34812 22232 +32867 22230 +38986 22225 +30243 22224 +46836 22224 +42487 22220 +32538 22219 +42111 22218 +47459 22213 +39018 22210 +37334 22209 +10525 22208 +22438 22203 +22006 22202 +30314 22200 +20041 22199 +47324 22198 +48427 22198 +29162 22192 +34907 22189 +3045 22187 +20099 22186 +38371 22183 +40190 22177 +29783 22177 +42695 22174 +41144 22173 +39483 22172 +24358 22171 +47354 22169 +42399 22166 +49647 22160 +32005 22156 +49456 22154 +34053 22153 +33076 22150 +40438 22143 +31206 22140 +14957 22140 +22610 22138 +38549 22134 +48005 22133 +36203 22132 +47557 22131 +27816 22130 +34463 22128 +32751 22127 +49093 22127 +45279 22124 +38374 22123 +33602 22122 +42714 22121 +40499 22119 +50012 22114 +21914 22114 +39949 22114 +47363 22113 +39743 22113 +40163 22112 +40333 22107 +37272 22106 +40762 22104 +34330 22102 +33310 22101 +38356 22100 +45984 22100 +27483 22097 +12756 22095 +35282 22093 +30277 22092 +18238 22092 +13185 22090 +34158 22090 +29581 22090 +1609 22088 +30924 22087 +18793 22086 +49754 22085 +28645 22083 +22899 22080 +37615 22077 +44230 22075 +37636 22073 +31754 22072 +9063 22070 +24867 22067 +22939 22061 +29865 22061 +30846 22057 +46255 22057 +48624 22053 +19011 22052 +48284 22051 +48480 22046 +36530 22044 +48248 22043 +41165 22040 +30694 22039 +42366 22037 +45382 22034 +22404 22032 +11690 22031 +45787 22030 +27021 22021 +45737 22017 +32516 22017 +24326 22016 +42144 22012 +24132 22007 +25174 22007 +35642 22004 +29059 22003 +42947 22003 +13518 22003 +39243 22002 +25392 22000 +49172 21998 +34950 21998 +44106 21997 +42107 21997 +45757 21997 +30863 21997 +36956 21995 +41289 21995 +26178 21994 +47850 21991 +12715 21991 +35973 21987 +15879 21985 +37794 21984 +31800 21980 +3040 21979 +41211 21979 +25545 21979 +45173 21979 +25835 21978 +28911 21977 +24449 21977 +48201 21972 +45314 21970 +6781 21969 +32165 21968 +38903 21961 +48332 21959 +37567 21958 +33794 21957 +17377 21954 +37414 21954 +49990 21950 +45792 21945 +34096 21942 +48632 21941 +29266 21939 +47241 21936 +37735 21934 +31982 21933 +49306 21933 +16557 21932 +37338 21930 +39544 21928 +49329 21926 +19154 21925 +2510 21924 +31891 21922 +32648 21922 +29249 21921 +45971 21918 +49508 21917 +39653 21917 +49514 21916 +21401 21916 +29054 21916 +9556 21915 +47929 21912 +48721 21901 +48512 21899 +19117 21897 +30464 21895 +45572 21892 +35071 21891 +37776 21888 +35735 21887 +41075 21887 +43704 21887 +45012 21886 +40841 21881 +34530 21877 +16138 21875 +43231 21871 +39122 21871 +36620 21871 +38310 21870 +38629 21862 +42380 21860 +36249 21856 +43434 21852 +38352 21852 +41304 21850 +29299 21850 +42712 21849 +49231 21849 +6925 21848 +44569 21847 +17833 21843 +35681 21843 +45213 21842 +17417 21841 +49586 21840 +32430 21836 +47124 21836 +21619 21833 +49706 21833 +47814 21833 +34959 21832 +42534 21829 +46431 21827 +45259 21821 +37747 21820 +26846 21819 +39553 21816 +39004 21816 +33374 21814 +48004 21811 +34546 21811 +2765 21810 +28718 21809 +16353 21808 +29792 21808 +27279 21807 +27379 21802 +43315 21801 +47128 21801 +21280 21800 +5604 21798 +34828 21797 +23206 21795 +40876 21790 +16501 21788 +47909 21786 +38092 21786 +43184 21785 +21243 21785 +35133 21784 +18166 21781 +38703 21780 +49986 21780 +45654 21779 +26302 21777 +47885 21777 +37434 21773 +39918 21773 +47023 21772 +41308 21771 +10878 21768 +44539 21768 +35678 21765 +27629 21765 +34137 21764 +41423 21761 +23203 21760 +39558 21759 +27672 21758 +14933 21757 +48008 21755 +40885 21752 +36314 21752 +39586 21751 +40081 21751 +41580 21748 +49888 21748 +37613 21745 +33398 21744 +7399 21743 +26776 21740 +48949 21739 +27836 21736 +33345 21736 +27398 21735 +35815 21733 +24198 21733 +42905 21733 +17069 21731 +8917 21731 +42704 21731 +38995 21729 +37094 21729 +43134 21729 +44815 21728 +50064 21727 +38456 21726 +44860 21725 +36299 21723 +10198 21722 +41789 21722 +39689 21722 +19895 21722 +18790 21719 +34302 21717 +40275 21717 +36201 21716 +41813 21715 +48070 21714 +4994 21713 +30732 21711 +17746 21706 +43862 21704 +36496 21701 +44903 21700 +29700 21698 +46472 21697 +31383 21694 +35664 21692 +44461 21688 +41287 21687 +35809 21686 +44430 21685 +37777 21682 +6753 21678 +32869 21678 +12909 21678 +28037 21676 +41291 21674 +24047 21673 +36271 21671 +46492 21670 +17148 21669 +48783 21667 +33608 21663 +40234 21662 +45661 21659 +35721 21654 +44593 21653 +40184 21651 +47076 21651 +37804 21651 +27291 21650 +49714 21649 +49010 21643 +36643 21638 +28694 21638 +25155 21637 +41548 21636 +1707 21636 +45409 21634 +24156 21630 +49983 21621 +44251 21621 +35890 21614 +45722 21614 +41660 21613 +24225 21612 +40877 21608 +43509 21605 +47411 21604 +27904 21604 +10139 21598 +19539 21595 +45090 21592 +42780 21589 +41275 21589 +49386 21586 +48091 21583 +16153 21583 +38466 21581 +39771 21580 +32567 21577 +41822 21577 +29052 21576 +26008 21575 +34820 21574 +13036 21573 +46962 21571 +47326 21570 +46960 21563 +40618 21562 +32970 21562 +43536 21559 +16907 21557 +48311 21550 +38783 21550 +39526 21548 +46526 21547 +45551 21546 +49719 21544 +35472 21541 +16005 21540 +47336 21537 +25118 21536 +26330 21536 +46197 21533 +13047 21533 +42529 21532 +26738 21532 +22742 21529 +43666 21523 +20523 21522 +26305 21520 +40823 21513 +32439 21511 +27537 21511 +44149 21509 +39117 21504 +39942 21500 +40405 21498 +43289 21497 +41477 21497 +27891 21497 +41542 21496 +39880 21494 +28957 21493 +46663 21491 +40385 21489 +13874 21486 +30521 21485 +22667 21480 +4442 21479 +40058 21475 +46290 21475 +46762 21473 +18835 21472 +17354 21469 +46724 21465 +46606 21464 +44439 21464 +41577 21462 +48924 21461 +10954 21455 +7041 21454 +37505 21449 +40495 21448 +46661 21444 +47205 21442 +24318 21442 +49555 21442 +32058 21436 +41810 21432 +45649 21431 +33987 21430 +27743 21423 +29840 21423 +37055 21421 +14863 21420 +46298 21418 +39530 21417 +40276 21416 +44857 21415 +7494 21413 +26092 21413 +39801 21406 +33979 21401 +36664 21396 +42194 21395 +43624 21391 +42309 21390 +28073 21388 +40860 21388 +49747 21382 +48649 21381 +38856 21381 +38605 21377 +43780 21376 +29796 21374 +32824 21374 +2281 21373 +43860 21373 +49688 21371 +28402 21368 +46044 21362 +46885 21356 +40128 21355 +16820 21355 +20847 21354 +48176 21352 +26922 21351 +48250 21351 +45025 21350 +37483 21348 +49809 21346 +29733 21345 +45521 21338 +42921 21335 +42055 21334 +18635 21332 +21389 21329 +44994 21328 +20378 21327 +10138 21324 +49184 21322 +46916 21322 +48425 21321 +36376 21319 +40671 21317 +41557 21315 +16692 21312 +48661 21310 +42634 21306 +28041 21301 +46509 21301 +30986 21300 +49276 21299 +44048 21299 +33462 21296 +26261 21293 +49640 21293 +49102 21292 +6618 21288 +45193 21287 +40954 21287 +44137 21286 +43125 21285 +47101 21282 +5771 21281 +33484 21277 +14889 21276 +38576 21275 +41101 21271 +42533 21269 +46293 21266 +31938 21266 +37475 21266 +49692 21264 +18038 21260 +48200 21260 +29067 21259 +40785 21258 +44987 21258 +40889 21256 +39357 21251 +43523 21248 +45993 21247 +19006 21245 +30931 21241 +1116 21237 +30606 21237 +43803 21227 +45505 21225 +13967 21223 +43168 21223 +46254 21223 +36208 21222 +12065 21221 +28677 21221 +49478 21220 +46585 21219 +41292 21216 +39578 21216 +39344 21216 +43293 21212 +42280 21207 +21301 21206 +33621 21206 +41666 21201 +30662 21201 +35875 21201 +31199 21201 +30650 21201 +31120 21200 +17483 21196 +32243 21191 +35418 21188 +43403 21182 +39201 21176 +42960 21173 +22805 21171 +39736 21171 +37304 21171 +32200 21168 +42472 21168 +44281 21168 +13903 21167 +39312 21167 +40051 21166 +4184 21164 +10683 21163 +39251 21157 +32371 21154 +39166 21154 +33883 21153 +32968 21152 +49973 21151 +21101 21148 +41066 21146 +26155 21146 +23709 21141 +41590 21140 +39111 21138 +31495 21137 +41460 21137 +48987 21127 +13821 21127 +38073 21123 +46005 21123 +23155 21122 +47988 21116 +2868 21115 +38906 21115 +48564 21112 +40245 21112 +40173 21103 +31880 21103 +44162 21100 +27403 21100 +134 21099 +34865 21094 +11156 21093 +36981 21089 +43016 21089 +21871 21086 +27559 21086 +45682 21085 +31350 21083 +43874 21082 +39023 21078 +46847 21076 +38505 21074 +46067 21071 +48182 21066 +21687 21066 +28730 21057 +50189 21056 +42875 21054 +34335 21052 +11229 21050 +10922 21050 +37348 21049 +36927 21046 +42726 21046 +46550 21045 +26588 21044 +13065 21040 +13090 21037 +22834 21036 +13082 21035 +42268 21030 +45630 21029 +8451 21029 +47792 21024 +28757 21022 +34312 21022 +49461 21022 +49233 21019 +38407 21018 +16742 21017 +39186 21016 +35140 21014 +20949 21013 +45458 21008 +49979 21004 +37323 21001 +43626 21001 +30420 21001 +32915 20999 +43253 20992 +45136 20991 +40700 20990 +1889 20989 +26123 20988 +33553 20987 +38546 20983 +46271 20981 +5853 20980 +42015 20974 +25283 20974 +39216 20973 +30565 20972 +43820 20971 +44957 20971 +35145 20971 +37472 20971 +43610 20970 +36445 20969 +49381 20968 +47283 20968 +14975 20966 +34178 20966 +32704 20965 +35720 20963 +36902 20961 +34647 20959 +35030 20959 +46612 20958 +5877 20957 +35378 20956 +32975 20954 +35755 20947 +14238 20947 +41166 20946 +46378 20944 +27556 20942 +38846 20941 +18521 20938 +35388 20935 +40116 20935 +36549 20933 +48399 20928 +37531 20927 +39449 20923 +47303 20919 +12999 20917 +25235 20916 +27701 20916 +42005 20914 +46146 20913 +33483 20912 +37117 20907 +39327 20907 +30234 20907 +43633 20902 +18122 20898 +45493 20895 +6933 20893 +7845 20884 +33723 20877 +24418 20874 +39904 20872 +48736 20870 +8660 20868 +26307 20867 +49122 20866 +33449 20863 +30270 20862 +30893 20861 +48726 20861 +45072 20860 +38636 20860 +49282 20859 +42010 20859 +49418 20858 +34359 20856 +37716 20855 +24543 20853 +33284 20852 +40157 20851 +17156 20851 +48809 20847 +49697 20843 +40957 20842 +34742 20840 +42747 20839 +41858 20836 +37769 20835 +22058 20835 +43363 20833 +48680 20831 +5927 20830 +13228 20826 +33856 20826 +37780 20825 +48230 20824 +40375 20819 +48212 20819 +3712 20814 +45719 20809 +26267 20809 +23072 20809 +41043 20808 +33647 20804 +49940 20803 +47287 20798 +3138 20795 +27129 20794 +32599 20788 +22012 20787 +30018 20780 +49024 20775 +39118 20775 +30317 20775 +36337 20773 +28004 20773 +47551 20773 +47904 20770 +43690 20770 +39513 20765 +18448 20763 +9790 20762 +28022 20761 +45325 20759 +33056 20758 +31409 20757 +39940 20757 +25983 20755 +40979 20754 +45485 20753 +45704 20752 +36059 20752 +4552 20746 +34288 20745 +40285 20744 +29878 20744 +35156 20742 +37902 20741 +48263 20741 +29593 20740 +43510 20740 +41195 20739 +49542 20738 +40523 20738 +43221 20737 +2400 20736 +39895 20734 +40182 20732 +49625 20732 +8778 20731 +50032 20729 +32276 20725 +40564 20723 +24441 20722 +28352 20717 +6261 20716 +33797 20715 +34618 20715 +49707 20715 +45929 20713 +4878 20713 +43116 20713 +47009 20712 +28833 20712 +38066 20708 +16946 20707 +16575 20707 +45693 20706 +48052 20704 +34435 20703 +46460 20703 +26817 20700 +22266 20699 +39738 20697 +27068 20694 +13561 20693 +35997 20692 +37806 20691 +45128 20690 +39831 20689 +24661 20687 +47861 20686 +20098 20686 +44004 20680 +44261 20678 +39242 20675 +41148 20674 +11586 20671 +45118 20671 +4832 20671 +40043 20667 +44184 20659 +22797 20659 +25033 20657 +43322 20654 +22454 20653 +33258 20653 +34793 20649 +38220 20648 +40248 20648 +36722 20646 +43996 20645 +48189 20643 +10079 20642 +43540 20641 +47913 20635 +16709 20626 +43193 20622 +37765 20614 +39768 20612 +45028 20607 +47484 20606 +12743 20606 +45632 20606 +13962 20604 +21250 20597 +38082 20597 +28667 20596 +42740 20595 +49561 20587 +15535 20587 +43367 20585 +41364 20585 +43410 20584 +40987 20584 +10672 20583 +49109 20581 +38055 20580 +40179 20578 +42583 20574 +25642 20573 +24753 20570 +34124 20569 +49944 20568 +49351 20563 +49288 20562 +31041 20560 +34456 20556 +41019 20556 +32918 20553 +50000 20552 +37845 20550 +33239 20549 +41344 20546 +19738 20545 +35926 20544 +43391 20542 +36265 20542 +47799 20541 +37349 20539 +45900 20534 +47408 20528 +35474 20526 +42105 20524 +49853 20523 +42885 20521 +41473 20519 +40005 20515 +39179 20514 +46984 20512 +2201 20512 +36424 20512 +12843 20509 +29508 20507 +23909 20504 +39378 20503 +46415 20502 +30691 20502 +47276 20501 +45668 20495 +42175 20493 +41017 20493 +17982 20491 +50213 20491 +48062 20490 +37860 20490 +40175 20489 +23661 20489 +1508 20487 +37224 20484 +14761 20479 +48159 20478 +10294 20477 +44817 20476 +47426 20473 +37258 20473 +41847 20473 +34046 20471 +18009 20470 +38586 20470 +29715 20470 +47702 20469 +43169 20466 +33993 20462 +44218 20460 +41745 20459 +34400 20459 +44925 20458 +32059 20458 +5965 20458 +20456 20448 +39731 20446 +45758 20446 +48866 20445 +39188 20445 +43912 20444 +11062 20440 +34813 20437 +42117 20437 +45421 20436 +33758 20435 +46827 20433 +34739 20431 +34524 20429 +15578 20427 +6019 20426 +40638 20426 +25228 20426 +24008 20424 +37289 20423 +11415 20422 +42778 20422 +47666 20421 +4456 20417 +37680 20416 +29984 20416 +35736 20416 +38053 20415 +45538 20413 +45940 20413 +47897 20412 +37827 20412 +5084 20411 +17665 20411 +24461 20410 +34300 20409 +41495 20409 +23346 20408 +42918 20407 +28613 20406 +31674 20405 +43420 20404 +45590 20403 +46387 20402 +27288 20401 +44927 20400 +47969 20400 +45840 20399 +42215 20396 +21470 20396 +41136 20392 +16366 20388 +43685 20387 +39403 20386 +41302 20384 +46759 20383 +45850 20380 +40147 20379 +24882 20376 +35396 20376 +41232 20373 +30139 20372 +33272 20369 +48429 20369 +43707 20367 +45366 20365 +35200 20365 +37802 20363 +18688 20363 +41143 20362 +19633 20361 +24019 20356 +40486 20356 +47818 20354 +21197 20353 +9832 20347 +35908 20344 +42650 20342 +45089 20342 +28664 20341 +33854 20341 +22274 20340 +38907 20338 +35810 20337 +45720 20336 +26939 20333 +25271 20332 +24334 20332 +19317 20331 +34169 20330 +33367 20329 +48229 20326 +4977 20324 +42819 20321 +31589 20317 +46352 20316 +44576 20312 +50181 20307 +43631 20301 +41486 20301 +48466 20298 +36603 20298 +41295 20298 +40820 20297 +29988 20295 +49125 20294 +46023 20294 +41706 20293 +35964 20293 +45280 20292 +46435 20287 +32238 20286 +37453 20284 +32069 20276 +42199 20275 +45842 20274 +46169 20268 +48294 20266 +31080 20265 +40940 20264 +32739 20262 +37424 20257 +32923 20256 +35828 20256 +20655 20255 +26794 20254 +39315 20254 +36928 20254 +40433 20252 +33967 20246 +5217 20240 +45131 20238 +42213 20237 +21319 20237 +33402 20235 +24531 20235 +45339 20232 +39368 20222 +30813 20221 +21618 20219 +40797 20219 +38885 20215 +16634 20204 +33186 20201 +31634 20199 +50140 20197 +29734 20196 +38628 20195 +48544 20192 +20397 20187 +33112 20184 +13749 20183 +3962 20183 +39096 20181 +45009 20180 +37852 20180 +46147 20178 +46341 20176 +32475 20171 +4575 20170 +42353 20170 +36076 20170 +33549 20167 +24981 20167 +46235 20166 +47106 20165 +49898 20165 +30641 20163 +48909 20161 +17488 20159 +25746 20155 +47623 20154 +36600 20154 +48136 20151 +30192 20145 +18541 20142 +31148 20142 +31633 20136 +29884 20131 +48782 20129 +30518 20125 +15289 20115 +32795 20114 +32738 20113 +38312 20109 +44793 20108 +47957 20103 +23012 20098 +33463 20089 +49089 20089 +49921 20088 +41065 20086 +48380 20083 +49674 20079 +47985 20078 +16714 20077 +42871 20077 +30189 20077 +49644 20075 +43875 20074 +14141 20073 +25990 20073 +39697 20073 +25099 20071 +30969 20069 +26266 20064 +18765 20063 +28358 20063 +49934 20062 +45385 20060 +18156 20060 +37988 20059 +17992 20059 +40035 20051 +44736 20050 +44260 20047 +13515 20045 +40423 20044 +45797 20043 +36870 20042 +49012 20035 +44400 20032 +48386 20028 +33939 20026 +25506 20025 +23928 20024 +42411 20022 +13651 20022 +39120 20021 +45427 20021 +41198 20018 +22528 20017 +48463 20014 +47642 20012 +8416 20009 +21918 20007 +40311 20005 +22809 20002 +25979 20002 +39664 19999 +30087 19998 +36961 19997 +37826 19996 +43697 19995 +3229 19992 +39210 19990 +43057 19984 +15609 19981 +35490 19980 +48295 19977 +27787 19976 +38697 19972 +21860 19971 +40393 19970 +46180 19968 +49244 19964 +36226 19964 +45106 19961 +30396 19959 +49995 19957 +47675 19953 +32362 19947 +18314 19946 +25679 19945 +37601 19944 +47578 19944 +32318 19943 +13628 19943 +42276 19941 +48291 19940 +45401 19938 +13887 19933 +36725 19932 +41600 19930 +44136 19929 +41948 19928 +47069 19926 +33366 19923 +45252 19920 +46636 19911 +37015 19908 +13073 19906 +42428 19904 +38918 19903 +43901 19901 +30322 19900 +39878 19899 +47330 19892 +33199 19890 +46617 19889 +27968 19889 +41938 19887 +41752 19886 +42091 19885 +44123 19885 +39628 19882 +40465 19881 +48057 19881 +44639 19880 +18380 19879 +1060 19878 +45472 19877 +11988 19875 +4971 19874 +17910 19872 +22690 19867 +42939 19864 +47468 19863 +23651 19858 +20112 19858 +42430 19856 +34487 19851 +46201 19845 +34437 19840 +41257 19837 +37775 19833 +50085 19831 +38438 19831 +42074 19827 +38237 19826 +45107 19822 +46314 19820 +49503 19820 +29401 19820 +22157 19819 +25117 19815 +42413 19815 +34792 19812 +20448 19811 +46758 19806 +43600 19805 +33101 19805 +32852 19803 +44908 19800 +37203 19799 +45111 19798 +12658 19791 +25661 19788 +32941 19784 +40573 19783 +39107 19777 +44705 19772 +42250 19772 +49488 19770 +9622 19769 +45721 19762 +40256 19761 +34964 19761 +42601 19761 +43876 19761 +32001 19760 +46792 19757 +38802 19755 +48376 19752 +41722 19750 +27894 19748 +48177 19747 +45113 19746 +40848 19745 +11877 19744 +12235 19744 +36276 19743 +18048 19742 +42790 19741 +48408 19736 +34631 19736 +47601 19734 +15514 19733 +36104 19727 +42769 19726 +44881 19726 +39872 19725 +44617 19724 +36693 19721 +48745 19720 +32316 19720 +38654 19717 +9872 19716 +23172 19712 +43808 19710 +48882 19709 +43580 19707 +44578 19705 +39696 19701 +42962 19698 +30235 19698 +27992 19698 +44928 19697 +43673 19697 +49650 19696 +40654 19695 +45531 19695 +17751 19694 +19007 19692 +41647 19691 +41261 19690 +3121 19689 +16998 19689 +38970 19688 +45288 19686 +32616 19685 +42789 19684 +38070 19683 +46356 19683 +48584 19681 +46438 19679 +23812 19678 +48108 19678 +10803 19678 +47824 19675 +30620 19674 +12359 19674 +41604 19669 +41895 19668 +31184 19667 +27880 19665 +45124 19664 +47070 19661 +46883 19660 +46302 19656 +44667 19652 +23093 19650 +16684 19649 +43432 19646 +44969 19644 +16530 19643 +49097 19643 +6854 19643 +17573 19642 +7992 19637 +31000 19634 +44977 19633 +40682 19631 +28631 19629 +38457 19628 +15507 19626 +49196 19625 +35438 19624 +47576 19619 +26694 19617 +46020 19615 +47308 19615 +41458 19612 +43571 19610 +28761 19608 +40572 19606 +41914 19605 +48739 19605 +20280 19603 +47491 19602 +32928 19601 +25346 19599 +46258 19598 +43451 19592 +47493 19590 +46028 19584 +32454 19583 +10600 19583 +43813 19580 +47482 19574 +31094 19573 +37167 19571 +42690 19569 +42343 19568 +42463 19564 +44363 19556 +28869 19555 +42776 19547 +40417 19545 +33505 19545 +44893 19544 +43026 19544 +49216 19542 +20335 19541 +28254 19539 +40747 19538 +30409 19536 +31551 19536 +6694 19534 +38445 19534 +42115 19533 +30903 19532 +24292 19531 +48030 19530 +42214 19526 +50168 19524 +48638 19523 +20193 19521 +20502 19520 +48110 19519 +15911 19517 +48194 19515 +40659 19515 +43284 19514 +26729 19513 +40620 19511 +35795 19510 +49446 19508 +38843 19508 +18442 19507 +23396 19504 +34694 19504 +26766 19500 +38142 19497 +38828 19495 +33070 19495 +21968 19493 +46061 19493 +44549 19492 +50080 19491 +28106 19487 +13344 19486 +18099 19484 +41381 19483 +26025 19482 +33751 19479 +44399 19479 +22165 19479 +48392 19474 +38793 19474 +42321 19471 +21889 19465 +38117 19459 +43778 19457 +46894 19455 +34431 19454 +46572 19452 +37586 19449 +46632 19449 +29891 19447 +40279 19447 +46417 19447 +48264 19446 +26233 19445 +36580 19444 +38015 19444 +49837 19443 +37522 19443 +28455 19442 +18885 19441 +46333 19437 +43908 19436 +10201 19435 +27300 19433 +31286 19429 +48488 19428 +30765 19426 +49115 19425 +31563 19422 +38314 19420 +35609 19415 +43948 19414 +22135 19412 +31613 19407 +40639 19405 +31972 19404 +47521 19401 +34615 19394 +40972 19394 +38139 19392 +30482 19390 +39912 19389 +37832 19389 +39332 19388 +18584 19388 +42548 19386 +47899 19386 +47498 19385 +38071 19384 +44879 19381 +32340 19380 +1261 19379 +45960 19378 +39147 19373 +22504 19372 +45864 19371 +49123 19371 +46442 19370 +39089 19370 +50161 19369 +38350 19369 +38202 19362 +41599 19361 +38600 19361 +23190 19359 +22550 19359 +38954 19358 +23952 19358 +46871 19356 +47990 19356 +39112 19355 +21722 19355 +42956 19354 +44649 19354 +40756 19352 +34808 19349 +30424 19349 +42032 19348 +11021 19348 +40207 19347 +49064 19346 +35143 19345 +49953 19344 +36275 19344 +38852 19344 +49147 19343 +34026 19341 +39800 19341 +25512 19341 +43006 19341 +4989 19336 +24480 19335 +41456 19335 +21463 19335 +9411 19330 +23645 19328 +23435 19326 +46239 19325 +7462 19324 +47760 19323 +35854 19323 +9378 19322 +18713 19321 +35167 19320 +43719 19319 +45006 19319 +42130 19315 +48856 19313 +48634 19313 +46886 19311 +38177 19310 +47369 19309 +42759 19308 +24945 19307 +31577 19306 +44215 19304 +28864 19303 +41642 19302 +43182 19298 +46925 19296 +38509 19295 +44468 19295 +49319 19292 +48764 19292 +37839 19291 +42336 19291 +31050 19287 +44338 19283 +28114 19281 +46096 19278 +16606 19277 +45734 19276 +25052 19274 +34023 19272 +44031 19269 +37050 19268 +41833 19267 +38116 19266 +36555 19264 +29289 19263 +48092 19261 +18363 19260 +41541 19258 +23522 19258 +47931 19257 +45989 19254 +47683 19253 +49701 19250 +46052 19249 +34621 19246 +33203 19245 +40711 19243 +47900 19243 +47233 19242 +2874 19237 +46501 19232 +49111 19231 +18386 19230 +38062 19230 +46056 19227 +15676 19227 +38375 19226 +45838 19225 +47200 19223 +48724 19223 +39102 19222 +49439 19218 +42075 19216 +18144 19214 +45881 19211 +46872 19211 +29344 19210 +47949 19209 +39573 19207 +41925 19206 +45349 19206 +44896 19203 +38685 19202 +44397 19198 +41169 19197 +21177 19196 +8460 19196 +35123 19194 +48573 19184 +18095 19182 +16956 19180 +1914 19179 +24990 19179 +16786 19176 +23562 19176 +10042 19175 +25029 19175 +34619 19175 +37652 19174 +36813 19172 +36948 19170 +20773 19161 +24412 19159 +20763 19158 +29304 19157 +27319 19156 +38412 19156 +22918 19155 +46109 19154 +40441 19147 +37798 19144 +39405 19140 +48150 19139 +39785 19137 +29114 19137 +38604 19131 +43462 19129 +44408 19124 +45122 19121 +17434 19121 +34200 19120 +46749 19118 +27130 19114 +35509 19113 +49509 19113 +50010 19112 +42407 19112 +45452 19112 +44960 19110 +46868 19108 +45696 19107 +48096 19106 +46270 19105 +27270 19098 +22902 19089 +34393 19089 +10519 19088 +33074 19085 +43864 19082 +42174 19080 +25633 19080 +44560 19078 +48630 19075 +11850 19073 +48524 19071 +36305 19069 +22869 19066 +18719 19064 +43048 19060 +44891 19058 +28871 19058 +47670 19057 +47880 19057 +44513 19053 +14688 19052 +31643 19047 +48260 19047 +33651 19047 +39413 19045 +46263 19044 +33571 19039 +17191 19036 +43009 19036 +44176 19033 +41792 19032 +34779 19031 +28886 19030 +46158 19027 +46993 19026 +48825 19023 +45944 19022 +19741 19019 +41583 19018 +21936 19016 +49845 19013 +48398 19012 +25058 19011 +5467 19008 +36282 19007 +41492 19006 +24512 19005 +48187 19004 +42763 19002 +29274 19001 +16982 19001 +47237 19001 +33325 19001 +50027 18997 +42435 18992 +35230 18989 +46589 18989 +40631 18985 +18749 18983 +26368 18982 +36228 18982 +21663 18979 +4027 18979 +13807 18978 +36499 18978 +36109 18977 +46176 18972 +26491 18968 +39443 18965 +49974 18963 +31457 18963 +46103 18954 +47785 18952 +47016 18951 +27175 18951 +43986 18950 +41072 18949 +34408 18947 +48813 18946 +7290 18945 +47396 18945 +41280 18945 +48645 18944 +30187 18936 +50167 18933 +43339 18933 +38650 18932 +29163 18931 +12650 18931 +40736 18930 +47915 18929 +17181 18928 +39584 18927 +20619 18925 +47305 18924 +44063 18924 +30142 18919 +44214 18918 +10027 18917 +26967 18914 +41279 18912 +48170 18912 +45225 18910 +50222 18905 +22192 18898 +46285 18894 +39324 18893 +37446 18892 +20297 18891 +43119 18888 +13163 18888 +43465 18885 +13297 18883 +41627 18882 +49397 18881 +43445 18881 +37458 18881 +47941 18881 +48246 18878 +8289 18877 +37889 18876 +42562 18875 +33995 18875 +49964 18873 +49052 18862 +49857 18861 +32606 18860 +41874 18856 +44598 18854 +29277 18853 +46623 18852 +41765 18852 +42011 18850 +38169 18849 +27991 18847 +34349 18846 +5312 18842 +46250 18839 +21704 18836 +40251 18831 +26901 18830 +23770 18828 +47350 18821 +27214 18820 +48930 18819 +19735 18817 +35387 18813 +39509 18810 +29410 18809 +47807 18803 +49489 18800 +17040 18794 +48462 18794 +34850 18790 +34411 18788 +34610 18787 +47694 18786 +2203 18785 +40401 18782 +43427 18779 +13264 18779 +25775 18778 +43819 18776 +27636 18773 +46476 18770 +37941 18768 +33086 18768 +41955 18764 +45705 18762 +45441 18758 +43300 18751 +8535 18747 +43854 18745 +44265 18740 +38318 18739 +35087 18736 +18991 18735 +8522 18729 +45857 18727 +37691 18725 +44685 18725 +49499 18724 +27215 18723 +34718 18721 +35950 18719 +43564 18718 +23168 18717 +24085 18717 +19489 18716 +16448 18715 +43822 18715 +48665 18711 +49550 18711 +42145 18708 +10639 18702 +26836 18702 +48697 18696 +39568 18695 +27844 18695 +48772 18694 +47397 18693 +48037 18692 +25367 18692 +39615 18686 +44504 18685 +48442 18683 +43385 18683 +5666 18682 +42675 18680 +43883 18680 +24647 18678 +42271 18676 +41403 18674 +5914 18674 +26892 18674 +48864 18672 +48647 18667 +17566 18666 +14391 18664 +42283 18664 +32031 18662 +21113 18661 +24212 18661 +48381 18658 +37400 18656 +46649 18649 +24360 18645 +34919 18645 +47613 18644 +39550 18641 +33934 18639 +41998 18636 +16424 18628 +32942 18626 +24112 18626 +36241 18625 +26098 18622 +40414 18621 +36818 18620 +48648 18618 +47812 18613 +49130 18607 +9333 18606 +21930 18604 +41050 18602 +26991 18599 +47901 18598 +42443 18596 +30380 18595 +42938 18592 +45791 18592 +34829 18591 +43722 18590 +44372 18589 +36255 18584 +28919 18584 +17271 18581 +23862 18578 +25955 18577 +24121 18576 +42482 18574 +36056 18572 +44675 18570 +40804 18569 +24355 18569 +49816 18569 +42702 18568 +37905 18568 +46743 18567 +36949 18567 +10430 18566 +41060 18561 +8456 18559 +25612 18558 +31728 18557 +44269 18555 +29919 18555 +42405 18554 +16579 18546 +24084 18544 +20150 18544 +34888 18543 +46692 18542 +30297 18541 +31737 18540 +18497 18538 +22062 18535 +48489 18534 +44622 18532 +49720 18532 +23736 18528 +47085 18527 +34712 18524 +35443 18524 +44087 18524 +32501 18516 +19537 18515 +26341 18513 +28547 18510 +32527 18508 +35712 18505 +45639 18502 +41330 18501 +47838 18500 +48719 18500 +25593 18499 +38752 18498 +44955 18496 +27547 18490 +39866 18490 +6900 18490 +22285 18490 +17350 18490 +43930 18489 +32179 18487 +44586 18486 +48808 18482 +9802 18478 +35064 18476 +19424 18473 +50205 18471 +29982 18470 +48861 18470 +49330 18470 +42307 18469 +47037 18465 +44055 18465 +33096 18465 +42412 18464 +35035 18463 +48006 18463 +43389 18462 +34333 18462 +39334 18459 +15026 18458 +46752 18457 +10859 18456 +44753 18455 +43950 18454 +41183 18453 +8032 18452 +24145 18449 +47646 18445 +47275 18444 +13031 18444 +44327 18442 +41815 18441 +46678 18441 +44573 18439 +38226 18434 +42604 18433 +26994 18431 +46296 18430 +29929 18429 +39921 18429 +13607 18421 +42493 18421 +39168 18419 +38902 18416 +46026 18412 +44396 18410 +38187 18410 +37452 18410 +19465 18406 +49150 18402 +43557 18402 +49424 18399 +48578 18393 +33838 18392 +41531 18390 +41962 18390 +16845 18385 +35775 18380 +48305 18380 +46900 18379 +11605 18378 +31439 18378 +27949 18377 +48337 18369 +23789 18368 +46681 18368 +21086 18368 +43191 18367 +31257 18366 +23733 18366 +39492 18364 +43680 18361 +45839 18358 +47963 18356 +34652 18353 +8206 18352 +42739 18347 +41610 18346 +43559 18346 +44295 18341 +41228 18336 +23156 18336 +43446 18336 +26857 18335 +40908 18335 +45943 18335 +6969 18330 +44719 18329 +43521 18329 +38565 18327 +49666 18324 +46093 18323 +45702 18321 +44659 18315 +40350 18313 +45628 18312 +17418 18309 +46860 18307 +30352 18307 +47512 18307 +48618 18306 +43277 18303 +17881 18301 +38588 18300 +34897 18298 +35350 18293 +31989 18291 +36823 18290 +33131 18289 +39776 18289 +50074 18289 +48602 18286 +40348 18281 +45486 18277 +48414 18276 +47488 18275 +21118 18274 +35513 18271 +45214 18270 +16536 18267 +43468 18267 +27789 18267 +43847 18264 +17425 18262 +41850 18261 +12357 18261 +36954 18258 +50029 18255 +40052 18254 +34773 18253 +14124 18252 +43283 18248 +46478 18248 +48603 18244 +45319 18244 +31255 18241 +48740 18240 +43059 18238 +31671 18236 +38451 18236 +11931 18232 +32216 18224 +47389 18218 +44172 18216 +49033 18214 +32007 18214 +22055 18214 +22592 18212 +44847 18211 +40451 18211 +48892 18209 +41051 18207 +28524 18205 +24511 18205 +30613 18203 +40505 18201 +33142 18200 +45515 18198 +48653 18198 +44375 18197 +13337 18196 +44612 18192 +32203 18188 +17407 18187 +38642 18187 +45256 18185 +48767 18185 +4667 18183 +43665 18182 +48156 18180 +30026 18180 +13161 18180 +37699 18179 +42001 18179 +50052 18177 +28089 18176 +20627 18176 +38508 18171 +9680 18171 +29080 18170 +42635 18169 +42955 18169 +35635 18164 +45157 18162 +33923 18160 +38219 18160 +40287 18158 +42996 18155 +36523 18155 +35251 18152 +8417 18152 +19872 18151 +24704 18150 +23108 18145 +43312 18144 +41963 18142 +23428 18135 +46079 18130 +31793 18124 +45891 18123 +31664 18119 +25723 18118 +17791 18117 +44494 18117 +44305 18116 +29670 18116 +31904 18115 +22851 18114 +38161 18108 +19945 18106 +31305 18105 +31715 18103 +45007 18101 +19743 18100 +17725 18095 +49051 18091 +47582 18090 +43172 18088 +34390 18085 +29550 18084 +48075 18084 +44542 18083 +15493 18083 +47570 18081 +44826 18078 +29746 18077 +32106 18074 +19241 18074 +47274 18070 +37363 18070 +38692 18067 +6012 18067 +47859 18061 +29149 18060 +34097 18059 +27935 18054 +24437 18054 +39961 18049 +48441 18044 +23611 18037 +43243 18037 +41263 18034 +38627 18032 +13072 18031 +43563 18028 +48945 18028 +32315 18026 +26486 18023 +28575 18019 +43239 18018 +35914 18016 +48165 18016 +28095 18011 +42187 18007 +16306 18006 +21015 18005 +41596 18004 +42622 18003 +42312 18003 +43668 17999 +44307 17998 +36804 17997 +47240 17993 +49348 17993 +26388 17987 +5307 17986 +38471 17985 +34167 17985 +34706 17982 +48465 17981 +44694 17977 +30867 17976 +36659 17975 +32208 17975 +32512 17973 +43887 17967 +41235 17966 +44114 17966 +45728 17963 +22467 17962 +49641 17960 +20308 17957 +42237 17956 +39134 17954 +39633 17952 +44743 17949 +38179 17947 +41841 17945 +47896 17934 +49900 17929 +49366 17926 +45523 17919 +39601 17918 +12360 17917 +49803 17910 +32449 17909 +19357 17908 +48054 17902 +31007 17899 +37045 17899 +9097 17898 +47612 17893 +14445 17893 +43098 17891 +40143 17891 +39450 17890 +33429 17888 +41601 17886 +35051 17881 +32649 17881 +49915 17880 +40663 17875 +44887 17874 +47111 17871 +31554 17869 +49875 17868 +19319 17868 +17391 17866 +37994 17865 +37836 17863 +20846 17860 +16870 17859 +27524 17858 +4742 17855 +22469 17855 +47726 17851 +41095 17849 +41732 17849 +46618 17848 +49313 17847 +46914 17846 +24802 17846 +27664 17844 +44543 17843 +39417 17837 +38213 17836 +48556 17831 +22272 17831 +34575 17827 +31761 17827 +21176 17827 +49579 17827 +37152 17825 +30541 17822 +24510 17821 +19816 17821 +22993 17820 +37074 17818 +49234 17818 +43656 17817 +41990 17817 +46440 17815 +49254 17815 +48623 17812 +44253 17811 +48090 17810 +48086 17810 +15311 17810 +37057 17809 +30675 17808 +11827 17808 +44203 17802 +22512 17799 +46888 17798 +44943 17789 +44552 17787 +26446 17786 +47965 17783 +13221 17780 +45938 17780 +11535 17780 +10702 17777 +29519 17777 +35298 17773 +17850 17772 +28906 17768 +44661 17766 +37637 17764 +29917 17764 +1764 17764 +34337 17762 +24290 17761 +22407 17760 +49437 17759 +18243 17757 +28905 17756 +45462 17756 +44532 17754 +49436 17753 +20164 17753 +29857 17748 +18819 17747 +10260 17747 +47152 17738 +46046 17738 +48444 17734 +17177 17733 +37828 17730 +24609 17724 +8760 17719 +39647 17715 +41351 17714 +37288 17714 +48947 17714 +20683 17709 +26055 17708 +45114 17707 +11049 17705 +42676 17704 +43739 17702 +31960 17702 +30233 17702 +42188 17698 +50223 17698 +35838 17697 +44897 17695 +23792 17695 +47758 17692 +12875 17691 +44259 17690 +25683 17688 +43805 17686 +37394 17686 +43046 17681 +49782 17681 +44069 17680 +48047 17675 +43490 17672 +24559 17671 +44020 17670 +23977 17670 +27454 17664 +45616 17663 +28863 17661 +41369 17661 +2463 17659 +29126 17658 +45216 17653 +12443 17647 +20630 17643 +29247 17642 +47731 17641 +43732 17638 +41705 17636 +45825 17635 +47044 17635 +40104 17635 +44169 17634 +25652 17634 +23521 17634 +41793 17633 +35379 17633 +32661 17631 +33108 17629 +44060 17628 +8645 17628 +42322 17623 +39121 17621 +33481 17619 +46371 17616 +43336 17615 +47191 17615 +39090 17610 +34128 17603 +47056 17595 +42084 17595 +40392 17594 +41141 17594 +30226 17592 +27369 17588 +43209 17587 +30329 17585 +2216 17584 +31950 17583 +40839 17583 +32870 17580 +37346 17579 +14911 17577 +35731 17576 +11030 17576 +42782 17576 +11018 17576 +40324 17575 +31336 17575 +37779 17575 +43946 17573 +49856 17572 +45738 17571 +43787 17570 +40260 17567 +34684 17566 +44934 17566 +20971 17565 +50183 17563 +46159 17563 +44673 17562 +24035 17561 +28329 17561 +38744 17560 +40201 17558 +45465 17557 +46267 17556 +39887 17553 +26331 17551 +48236 17546 +42258 17546 +10328 17546 +43605 17545 +39572 17541 +44965 17539 +45670 17534 +43027 17534 +46841 17533 +35909 17532 +30074 17531 +50238 17530 +36252 17522 +49075 17517 +45636 17515 +31310 17514 +46665 17514 +44000 17514 +46519 17510 +39570 17507 +38497 17506 +39437 17505 +38896 17504 +35911 17502 +18538 17502 +39816 17499 +46248 17497 +27020 17497 +47184 17490 +20238 17490 +17614 17490 +40301 17490 +35744 17489 +11981 17488 +43589 17485 +19899 17484 +41094 17483 +46887 17483 +35961 17481 +43625 17479 +46432 17476 +48884 17475 +27480 17474 +44121 17474 +6030 17473 +28321 17469 +43529 17469 +35211 17468 +29123 17468 +2862 17463 +26940 17461 +48555 17460 +43063 17460 +38624 17458 +42488 17457 +42964 17452 +33913 17451 +13303 17450 +18999 17447 +2847 17442 +44973 17441 +29882 17440 +25020 17436 +42131 17434 +45080 17434 +38385 17433 +37592 17423 +43240 17423 +44442 17421 +14913 17419 +37696 17416 +29511 17416 +6671 17416 +49602 17416 +26458 17415 +49703 17414 +21594 17414 +45182 17409 +48818 17406 +14606 17403 +37296 17403 +39491 17397 +41761 17396 +45964 17396 +15082 17393 +33441 17391 +45935 17391 +41376 17388 +42581 17388 +46468 17386 +27374 17386 +40647 17385 +41409 17385 +31098 17383 +41047 17382 +36479 17381 +45648 17380 +841 17379 +20745 17379 +44078 17378 +42965 17378 +17005 17376 +13462 17375 +3591 17374 +42980 17373 +49257 17370 +29078 17370 +23037 17369 +13249 17368 +32895 17367 +50165 17366 +45761 17363 +37174 17360 +25213 17357 +49303 17354 +24866 17353 +36227 17352 +39183 17351 +44153 17345 +39923 17344 +4148 17343 +48926 17343 +18431 17343 +46815 17343 +29150 17341 +43292 17341 +47370 17341 +19295 17335 +19211 17334 +46717 17331 +49211 17330 +48327 17329 +5772 17327 +43994 17322 +19285 17321 +34997 17317 +44764 17317 +24064 17316 +49356 17316 +18471 17316 +33554 17316 +43223 17314 +28780 17314 +19676 17314 +49394 17313 +36191 17312 +4566 17311 +43683 17310 +35881 17310 +42494 17306 +26908 17306 +48552 17305 +43146 17300 +41159 17291 +41700 17290 +48228 17290 +42643 17288 +16750 17287 +13287 17286 +11134 17284 +44329 17284 +41561 17283 +45949 17282 +36352 17281 +38513 17277 +24189 17277 +19768 17271 +37078 17271 +42301 17265 +34667 17256 +35945 17254 +33975 17251 +35157 17251 +47776 17251 +49114 17249 +44077 17245 +42141 17242 +9108 17242 +27187 17240 +14993 17240 +46781 17239 +35625 17238 +48959 17237 +48188 17236 +15522 17235 +24729 17232 +48964 17230 +40733 17229 +33635 17226 +47946 17224 +28425 17218 +31042 17216 +22658 17215 +49525 17213 +32178 17211 +36728 17204 +46807 17203 +28599 17201 +20020 17201 +38679 17198 +2570 17197 +40545 17194 +41455 17194 +47127 17192 +40384 17191 +42703 17191 +42563 17187 +30583 17187 +9244 17187 +26245 17184 +48952 17183 +31686 17181 +34211 17174 +44900 17174 +14817 17166 +32186 17166 +37201 17164 +45659 17161 +25435 17159 +39196 17158 +44980 17157 +38208 17155 +44417 17152 +31993 17147 +47544 17145 +48894 17144 +12220 17144 +37496 17141 +38864 17139 +44489 17133 +30010 17132 +26755 17131 +40202 17130 +27919 17130 +47555 17125 +49730 17120 +46091 17120 +20306 17116 +42978 17116 +47871 17116 +39422 17115 +24196 17109 +2110 17108 +37007 17107 +43514 17105 +44249 17104 +43228 17104 +10221 17104 +26632 17102 +38418 17102 +31401 17102 +46095 17102 +43515 17101 +44528 17101 +46953 17101 +32717 17101 +43687 17097 +50006 17096 +23509 17095 +33528 17092 +45765 17089 +11962 17088 +29544 17087 +30009 17085 +3475 17079 +18547 17077 +30516 17077 +49289 17076 +34090 17073 +31951 17072 +45559 17072 +41644 17071 +44877 17069 +31126 17069 +38972 17069 +50055 17061 +35149 17061 +37883 17060 +37460 17060 +17147 17058 +49189 17057 +44729 17055 +44171 17054 +16593 17053 +49053 17046 +31021 17044 +25551 17044 +37354 17043 +33691 17042 +43619 17041 +46112 17039 +30344 17037 +44183 17037 +33980 17033 +36170 17031 +47603 17028 +43413 17026 +46798 17025 +39061 17021 +41827 17020 +7909 17019 +47672 17015 +40108 17011 +49253 17010 +26439 17004 +48528 17003 +49645 16995 +26419 16995 +49532 16993 +44589 16990 +35745 16989 +44950 16988 +13321 16985 +21237 16981 +40262 16980 +26358 16978 +41867 16977 +42520 16972 +36038 16970 +46114 16970 +50019 16969 +47210 16968 +47036 16967 +46571 16967 +16327 16966 +46542 16965 +3471 16963 +21993 16961 +35743 16958 +48840 16956 +32803 16956 +33704 16956 +31741 16956 +36390 16952 +46877 16947 +24079 16945 +34472 16941 +29273 16940 +15252 16940 +46809 16938 +26560 16938 +5744 16937 +45886 16934 +28322 16933 +41267 16931 +20650 16928 +24428 16927 +22545 16923 +33336 16919 +32381 16917 +39106 16914 +42385 16912 +42820 16907 +44701 16906 +31740 16906 +30910 16906 +30173 16905 +38484 16904 +49459 16904 +22123 16903 +24978 16902 +32428 16902 +21628 16901 +17227 16900 +34600 16896 +20704 16895 +14474 16895 +10774 16892 +44509 16891 +45100 16888 +20204 16886 +42856 16886 +42054 16885 +47641 16884 +40277 16884 +44116 16883 +25606 16883 +38947 16878 +49700 16877 +47539 16877 +48431 16876 +41300 16875 +33202 16874 +14038 16873 +40849 16872 +48262 16871 +46803 16870 +45557 16870 +24443 16869 +47156 16868 +50119 16868 +35330 16862 +37649 16862 +48518 16859 +45467 16851 +33430 16851 +38467 16850 +43807 16849 +39528 16846 +45053 16841 +42946 16841 +47898 16838 +49735 16836 +20181 16836 +15128 16833 +27579 16833 +49768 16833 +32707 16831 +45952 16827 +45666 16824 +4797 16819 +16948 16817 +47360 16816 +40193 16815 +44383 16814 +32264 16805 +22251 16800 +50226 16799 +11076 16795 +34478 16795 +35915 16793 +40602 16791 +41784 16790 +33701 16790 +30817 16790 +46503 16789 +40926 16788 +26584 16786 +40673 16785 +45206 16784 +31644 16776 +43723 16775 +46928 16774 +46303 16773 +40575 16773 +40984 16768 +23251 16767 +45862 16766 +37165 16765 +31830 16762 +44384 16758 +46704 16757 +15356 16757 +36594 16753 +43824 16753 +42253 16751 +27365 16750 +45987 16747 +47907 16745 +38353 16742 +21275 16739 +49669 16738 +49787 16736 +20526 16736 +50242 16733 +25574 16732 +20960 16730 +39765 16728 +17117 16725 +48747 16725 +17932 16724 +47424 16723 +46862 16723 +23750 16723 +40366 16722 +46012 16720 +2212 16720 +43140 16720 +42483 16717 +19423 16717 +43952 16714 +31438 16709 +29828 16708 +41134 16708 +47994 16705 +44647 16704 +29248 16703 +50174 16697 +42600 16696 +37512 16693 +43275 16691 +42672 16689 +49500 16688 +39557 16687 +49087 16686 +31615 16682 +17041 16681 +44030 16677 +46405 16676 +37336 16676 +41563 16676 +36924 16673 +17854 16669 +42313 16668 +49822 16663 +45601 16662 +41872 16660 +18769 16659 +49541 16659 +45902 16657 +49685 16656 +46325 16653 +37797 16651 +43021 16650 +40380 16650 +44207 16647 +37689 16645 +31595 16644 +26984 16641 +32196 16637 +23545 16636 +33135 16631 +40435 16631 +23197 16627 +41103 16626 +34823 16625 +49750 16621 +41723 16620 +45372 16619 +39217 16618 +16466 16613 +42024 16607 +38567 16606 +25932 16605 +41934 16605 +44395 16602 +44741 16599 +10264 16599 +47886 16597 +21785 16597 +49481 16589 +39702 16589 +48820 16588 +39984 16586 +33730 16585 +43937 16580 +25660 16579 +47921 16574 +49784 16574 +49512 16569 +29106 16567 +15960 16565 +25595 16560 +47787 16560 +49733 16560 +45126 16558 +20921 16556 +34008 16551 +28566 16549 +27080 16549 +42053 16546 +41922 16541 +45770 16541 +43615 16539 +11603 16536 +22752 16534 +39852 16533 +45310 16533 +31185 16532 +49718 16531 +44931 16527 +2625 16525 +22288 16524 +45692 16524 +48979 16522 +45508 16522 +31624 16518 +48428 16516 +17617 16514 +30378 16509 +48109 16507 +1876 16507 +46410 16506 +20277 16505 +11180 16502 +33947 16498 +34921 16496 +38734 16494 +46761 16494 +43835 16491 +47846 16487 +975 16486 +24474 16485 +27747 16483 +36452 16482 +48409 16482 +38138 16479 +44780 16475 +44869 16474 +49682 16474 +28958 16469 +49878 16465 +30602 16464 +2819 16462 +47616 16457 +46256 16457 +48365 16455 +32478 16453 +48612 16452 +38140 16451 +44134 16448 +39933 16448 +47849 16446 +28601 16443 +43645 16442 +17883 16436 +34129 16434 +45290 16434 +42557 16432 +11337 16432 +34570 16427 +49294 16424 +950 16420 +49592 16418 +49922 16417 +27053 16415 +42691 16410 +50088 16410 +45766 16409 +48679 16409 +26026 16404 +47548 16403 +44671 16402 +19368 16398 +38827 16394 +42480 16391 +23331 16389 +31836 16388 +38257 16388 +23878 16386 +14194 16385 +46918 16382 +40503 16382 +32044 16381 +24695 16378 +46957 16375 +48098 16375 +42853 16369 +6316 16368 +41511 16367 +49287 16367 +20223 16366 +16780 16365 +37841 16364 +34007 16361 +48839 16360 +5904 16360 +44856 16358 +35648 16358 +49949 16356 +42486 16354 +24117 16354 +34675 16351 +47597 16351 +45500 16350 +37688 16350 +38556 16348 +48530 16348 +27786 16348 +40282 16344 +8382 16343 +40218 16342 +20580 16340 +47503 16338 +24366 16336 +19673 16333 +47580 16330 +29319 16327 +41466 16324 +44152 16322 +44921 16322 +9903 16321 +30617 16319 +32796 16318 +27858 16316 +31387 16316 +49936 16316 +14609 16306 +49055 16305 +33204 16305 +26537 16298 +26382 16296 +47728 16296 +27727 16293 +6137 16291 +49971 16288 +43608 16288 +43686 16287 +33320 16286 +17295 16286 +44422 16285 +16650 16284 +48173 16280 +35758 16279 +47247 16275 +45605 16271 +17976 16266 +45571 16262 +46736 16261 +49943 16260 +50123 16251 +44761 16251 +29487 16250 +42459 16249 +47346 16248 +22380 16246 +8703 16245 +48338 16243 +38806 16242 +15130 16240 +49638 16240 +38195 16237 +45925 16231 +5487 16231 +40962 16226 +19381 16225 +42787 16223 +46511 16221 +29788 16220 +46660 16219 +42416 16219 +35504 16218 +46500 16216 +44778 16214 +46192 16214 +49617 16212 +28505 16210 +49327 16209 +31752 16206 +25122 16206 +50234 16205 +8907 16205 +45506 16201 +36150 16200 +37739 16199 +50014 16198 +45877 16197 +41450 16195 +17307 16193 +22082 16192 +27383 16188 +37491 16187 +29193 16187 +12764 16186 +49307 16186 +44698 16185 +36925 16185 +21312 16184 +28751 16183 +11145 16181 +11956 16179 +41984 16178 +6629 16174 +32657 16174 +33908 16172 +35772 16172 +42060 16170 +24526 16167 +44750 16166 +47155 16165 +42572 16165 +48771 16162 +2294 16161 +38017 16157 +31371 16156 +23858 16156 +46839 16147 +35258 16144 +39077 16143 +30636 16138 +49193 16138 +27722 16138 +6534 16137 +43977 16127 +31404 16126 +37166 16126 +3410 16123 +45439 16122 +19335 16121 +32335 16118 +47192 16116 +29665 16115 +37849 16113 +49614 16108 +37577 16108 +927 16104 +30402 16100 +34661 16100 +26398 16099 +42080 16098 +45362 16097 +8236 16096 +21369 16096 +21145 16095 +36885 16092 +48960 16090 +46393 16090 +1274 16089 +44324 16087 +45701 16086 +45224 16086 +30472 16085 +25282 16084 +49717 16083 +47385 16083 +7871 16081 +45050 16080 +44229 16078 +15117 16076 +33894 16076 +11627 16073 +40019 16073 +2542 16072 +38710 16071 +49218 16071 +37059 16068 +9438 16067 +43252 16067 +46556 16067 +48372 16067 +50068 16065 +40299 16065 +48628 16062 +37540 16062 +41650 16059 +15748 16057 +17889 16055 +19806 16054 +37678 16050 +40353 16050 +6377 16043 +43795 16040 +44132 16040 +41299 16040 +41496 16034 +14203 16034 +32587 16032 +48167 16031 +48655 16030 +40521 16028 +45304 16026 +15044 16021 +35869 16015 +45127 16015 +47467 16015 +40225 16014 +41201 16012 +43199 16011 +33422 16009 +45653 16008 +43128 16008 +28361 16006 +31276 16004 +10446 16003 +21300 16000 +45116 15998 +45909 15997 +41936 15995 +45618 15995 +13461 15995 +27456 15994 +49959 15992 +46342 15991 +44125 15988 +49533 15987 +41932 15984 +32038 15983 +48223 15979 +36568 15973 +46756 15971 +49517 15971 +37925 15969 +29873 15967 +40538 15965 +28795 15963 +33175 15960 +37163 15960 +39701 15957 +20579 15954 +31852 15952 +21498 15952 +9134 15945 +46992 15941 +35684 15941 +44427 15940 +44609 15936 +47860 15935 +47374 15929 +17752 15927 +12474 15923 +35976 15921 +26252 15919 +30143 15918 +2161 15916 +43324 15914 +26314 15912 +22149 15909 +31062 15908 +43438 15908 +40382 15907 +32271 15906 +10412 15906 +47202 15895 +48548 15895 +20817 15892 +50192 15891 +6132 15888 +6791 15886 +34893 15885 +49760 15883 +48717 15882 +46018 15882 +29142 15882 +37774 15881 +25673 15877 +49199 15877 +39285 15876 +31124 15874 +26254 15871 +37640 15870 +20183 15869 +20941 15868 +38819 15868 +43758 15866 +37204 15863 +41513 15859 +17538 15856 +36303 15856 +9736 15854 +44796 15851 +48640 15849 +47567 15849 +14808 15849 +26035 15848 +28951 15836 +7302 15836 +43335 15834 +38659 15834 +27323 15830 +34414 15829 +28475 15828 +43568 15827 +49985 15827 +49659 15825 +45570 15820 +39766 15815 +8750 15815 +30576 15810 +43230 15809 +50187 15806 +25842 15806 +12596 15804 +32351 15804 +26676 15801 +44561 15798 +31152 15798 +31821 15797 +32876 15796 +40970 15796 +34320 15793 +42612 15792 +45804 15787 +46484 15785 +44254 15782 +43776 15780 +13538 15774 +31153 15772 +39182 15771 +44040 15770 +38069 15770 +32610 15769 +45573 15769 +39306 15768 +48896 15766 +36148 15764 +41416 15764 +47634 15763 +32434 15759 +26505 15758 +48691 15756 +42834 15753 +38025 15751 +38498 15750 +35217 15750 +15965 15748 +9147 15747 +32412 15747 +22685 15747 +43421 15746 +14405 15744 +47976 15741 +23192 15740 +42911 15739 +10189 15739 +45453 15739 +23857 15738 +24548 15736 +16362 15736 +40340 15735 +23847 15733 +10506 15733 +48081 15727 +30262 15727 +38770 15726 +30397 15726 +38699 15725 +37838 15725 +48554 15724 +28421 15720 +43643 15720 +15562 15720 +35191 15720 +35486 15719 +2554 15718 +37677 15718 +24476 15718 +42796 15718 +44367 15714 +11539 15713 +48129 15712 +39649 15712 +48980 15712 +16482 15710 +44084 15705 +33715 15702 +20967 15700 +46064 15699 +49373 15699 +37667 15699 +48089 15699 +37182 15698 +41327 15698 +23260 15694 +23937 15693 +29522 15690 +43550 15690 +26171 15685 +27381 15683 +43846 15676 +43585 15675 +31492 15675 +20076 15672 +37926 15672 +39848 15671 +42864 15670 +21637 15665 +9954 15663 +24979 15662 +43502 15662 +37488 15661 +24330 15661 +34572 15659 +43148 15658 +29179 15658 +37744 15657 +43498 15656 +44650 15656 +40298 15655 +31973 15655 +48540 15654 +14623 15654 +50182 15652 +41178 15651 +23029 15650 +43959 15648 +31635 15648 +30324 15646 +44095 15646 +49766 15645 +22825 15642 +49961 15639 +21565 15638 +41009 15637 +8831 15636 +35334 15632 +34083 15628 +47876 15628 +13912 15625 +37564 15623 +42027 15622 +32177 15616 +39515 15612 +41003 15609 +32163 15604 +36739 15604 +49156 15600 +34871 15600 +39346 15595 +6917 15594 +40146 15592 +49141 15591 +32614 15590 +34799 15588 +43390 15587 +48891 15587 +19326 15585 +28028 15584 +31583 15581 +49039 15579 +44774 15578 +39208 15578 +37058 15575 +8596 15570 +13274 15569 +5383 15568 +43155 15566 +38795 15566 +30454 15565 +48161 15564 +33523 15562 +40531 15560 +34736 15559 +42991 15558 +4723 15556 +48587 15555 +45801 15553 +39790 15551 +22071 15546 +40784 15543 +44235 15543 +34117 15542 +45217 15541 +28419 15541 +12358 15538 +32304 15537 +40584 15534 +45305 15532 +49423 15529 +47420 15525 +44124 15519 +41607 15518 +43639 15514 +35814 15512 +43863 15510 +13023 15510 +43884 15508 +49469 15505 +47211 15504 +43938 15504 +11943 15503 +26558 15501 +28897 15501 +36645 15495 +45752 15494 +32257 15492 +50069 15486 +4338 15486 +28000 15481 +45257 15476 +50132 15476 +40100 15476 +39817 15472 +43584 15470 +28777 15468 +20488 15468 +41636 15468 +44370 15466 +45097 15465 +2789 15465 +50095 15463 +38194 15463 +26873 15461 +28418 15460 +34931 15457 +32812 15457 +32633 15456 +19718 15456 +46486 15455 +34160 15445 +28261 15444 +49262 15440 +22856 15432 +26703 15428 +47273 15426 +27435 15426 +16063 15416 +3647 15414 +42401 15414 +47038 15413 +46856 15413 +9884 15412 +13286 15409 +48712 15407 +15667 15406 +38956 15402 +48718 15401 +48290 15400 +38279 15398 +44905 15395 +25853 15395 +37966 15389 +46874 15387 +43045 15386 +46693 15386 +46088 15385 +6114 15383 +23883 15383 +43060 15382 +45295 15382 +41949 15381 +50141 15379 +29748 15376 +28249 15372 +48019 15372 +46926 15368 +43927 15364 +18187 15363 +45219 15363 +35958 15361 +27590 15360 +35948 15359 +40400 15356 +38782 15355 +15019 15348 +45575 15345 +44629 15340 +19983 15339 +34972 15329 +15908 15329 +48970 15323 +6054 15323 +21908 15320 +42528 15318 +39807 15316 +43526 15311 +47359 15311 +22480 15309 +39444 15306 +41829 15304 +39957 15303 +41712 15299 +24045 15299 +33412 15298 +31058 15296 +37600 15294 +41572 15292 +12664 15291 +27532 15290 +32934 15287 +35578 15280 +42987 15279 +46209 15274 +45930 15273 +43637 15271 +41830 15270 +39896 15266 +46970 15263 +49594 15262 +45860 15261 +42891 15260 +45781 15259 +35241 15259 +46048 15257 +23412 15253 +26799 15251 +30068 15248 +43547 15246 +41171 15242 +38743 15241 +24422 15241 +33415 15240 +138 15238 +40373 15237 +46725 15235 +47553 15233 +46319 15233 +39748 15230 +46152 15230 +28985 15230 +30058 15226 +43926 15225 +38973 15224 +16778 15218 +28299 15217 +49823 15215 +29469 15215 +19279 15215 +25475 15210 +8311 15210 +33695 15205 +38711 15205 +38503 15202 +40743 15202 +41426 15201 +32487 15201 +42854 15199 +21341 15199 +29421 15198 +38845 15197 +42318 15194 +16688 15191 +43142 15190 +48800 15188 +6092 15186 +31927 15185 +44884 15185 +21556 15183 +48935 15182 +46162 15181 +36757 15180 +44688 15180 +32347 15178 +48307 15177 +19703 15169 +35614 15169 +44638 15162 +46940 15160 +13711 15159 +49713 15159 +49557 15156 +49757 15156 +21394 15154 +44739 15153 +34210 15152 +39950 15149 +21774 15142 +42121 15141 +18773 15141 +43872 15139 +11629 15138 +47922 15136 +10513 15128 +47789 15124 +23136 15123 +40118 15119 +30133 15118 +22599 15118 +49841 15117 +48627 15115 +47141 15109 +19532 15107 +44328 15104 +46560 15101 +27110 15097 +44890 15096 +45438 15096 +8657 15095 +50239 15094 +27751 15089 +42333 15088 +4617 15087 +44045 15086 +39505 15086 +18234 15086 +45134 15084 +30118 15081 +31776 15081 +42863 15077 +31097 15076 +42456 15075 +48597 15072 +48326 15071 +30988 15070 +48143 15070 +49764 15067 +47927 15065 +32775 15064 +46359 15063 +28248 15062 +5177 15060 +38786 15059 +35718 15059 +47920 15057 +24926 15056 +18958 15054 +39341 15052 +47529 15051 +42007 15051 +43030 15050 +24439 15048 +38607 15046 +43760 15043 +43290 15041 +9748 15039 +23737 15036 +49552 15033 +50049 15032 +23587 15032 +30653 15031 +46838 15029 +41443 15028 +44821 15028 +27363 15024 +33897 15022 +44721 15020 +29664 15020 +40616 15016 +35115 15011 +50254 15011 +21061 15011 +18375 15010 +35187 15010 +42632 15009 +49170 15009 +33594 15008 +22500 15008 +42220 15006 +49005 15002 +41587 15000 +7607 14995 +43440 14988 +43592 14986 +47752 14985 +47588 14983 +8412 14982 +41676 14980 +33033 14980 +15011 14980 +18037 14978 +21812 14975 +26589 14975 +49352 14975 +28920 14969 +37801 14967 +42003 14960 +26236 14957 +45338 14953 +33498 14952 +43817 14948 +39289 14943 +27100 14939 +43454 14939 +46607 14933 +41553 14931 +38419 14930 +42589 14928 +49954 14927 +46498 14926 +39870 14925 +24837 14924 +37449 14924 +49544 14921 +25969 14920 +46528 14919 +39780 14914 +39974 14914 +26209 14911 +27976 14910 +45210 14905 +43826 14905 +45580 14903 +17562 14894 +37977 14893 +36640 14888 +45345 14886 +42092 14885 +18826 14885 +43725 14878 +33944 14877 +30348 14877 +42349 14875 +22666 14874 +34218 14874 +5346 14873 +44237 14871 +40046 14868 +48439 14864 +22003 14861 +44802 14860 +37943 14860 +41919 14860 +45924 14859 +15002 14859 +47464 14858 +33842 14855 +38342 14853 +36235 14851 +35284 14851 +22442 14851 +45820 14851 +33136 14849 +33335 14848 +45718 14847 +38216 14843 +40726 14841 +49988 14840 +47736 14840 +49113 14839 +27 14839 +44734 14839 +25641 14833 +10669 14833 +45489 14830 +42068 14828 +47379 14826 +37162 14818 +44929 14818 +45071 14813 +40475 14806 +16683 14805 +16993 14802 +49336 14801 +29112 14796 +25222 14791 +18891 14789 +48732 14788 +47439 14783 +44037 14783 +44521 14782 +34294 14779 +15888 14778 +15091 14777 +19805 14777 +40646 14773 +37218 14772 +48328 14772 +48635 14771 +34984 14768 +49131 14766 +49800 14766 +41812 14765 +42012 14764 +34900 14764 +41255 14759 +22002 14758 +17212 14745 +25558 14743 +14158 14742 +16214 14739 +4937 14737 +32965 14736 +26323 14735 +46300 14734 +21191 14734 +26859 14727 +39162 14724 +43040 14724 +45079 14722 +11828 14719 +48232 14717 +32588 14713 +5024 14710 +45424 14708 +13947 14707 +18361 14705 +12378 14701 +40655 14698 +28205 14698 +27248 14697 +25443 14697 +16363 14697 +15642 14696 +27039 14695 +40008 14695 +36984 14693 +40576 14692 +38196 14686 +42544 14682 +29161 14680 +30709 14675 +41294 14675 +34909 14673 +48192 14664 +49021 14663 +36921 14662 +46857 14661 +48449 14660 +40411 14658 +50148 14656 +40938 14654 +39276 14653 +41543 14653 +50067 14652 +38661 14651 +34733 14647 +45596 14647 +37702 14640 +26752 14639 +34213 14638 +50228 14637 +46910 14637 +19030 14637 +46770 14636 +37284 14635 +45165 14635 +32198 14635 +19042 14631 +50091 14627 +44469 14619 +35423 14617 +47236 14612 +24465 14611 +46821 14611 +25608 14611 +18874 14608 +7701 14606 +43411 14599 +49463 14595 +42812 14590 +49649 14589 +42909 14585 +48768 14585 +47003 14585 +35384 14583 +29677 14580 +48252 14579 +43617 14579 +30128 14575 +48322 14574 +29644 14570 +17740 14568 +35626 14567 +47733 14567 +38168 14566 +10514 14564 +45574 14561 +16016 14559 +47440 14558 +40884 14555 +48775 14552 +30907 14552 +30619 14550 +4651 14549 +33532 14548 +44690 14546 +2942 14544 +31642 14542 +49092 14535 +41058 14534 +43069 14528 +16926 14527 +41347 14526 +49442 14525 +35400 14524 +49395 14523 +14810 14522 +46480 14519 +32286 14519 +27682 14518 +48810 14514 +49047 14511 +20906 14506 +17126 14505 +47176 14504 +31308 14501 +47121 14499 +42770 14497 +45576 14497 +24744 14497 +47053 14491 +48777 14488 +41508 14488 +49656 14487 +46633 14484 +48888 14483 +29823 14479 +30945 14478 +49049 14476 +47517 14474 +26745 14473 +13738 14470 +30056 14470 +32380 14469 +45426 14465 +27650 14464 +50031 14463 +38112 14461 +11221 14460 +16423 14458 +39662 14456 +33809 14454 +8008 14454 +17956 14453 +33322 14452 +44959 14449 +37898 14447 +33060 14445 +49299 14444 +48615 14441 +48751 14440 +21653 14437 +49497 14436 +8542 14430 +41811 14430 +27651 14428 +34329 14428 +44440 14426 +31746 14426 +39259 14426 +39968 14418 +36624 14417 +49421 14416 +48925 14415 +20317 14415 +46385 14414 +43001 14411 +20142 14409 +15439 14404 +25231 14402 +46876 14399 +46869 14399 +47744 14397 +41823 14393 +30975 14393 +32929 14390 +47719 14389 +41981 14389 +42406 14388 +37888 14387 +49477 14385 +49491 14385 +31849 14381 +3973 14380 +29709 14379 +26927 14379 +37535 14378 +33622 14377 +48132 14377 +48000 14375 +47032 14374 +46489 14373 +26904 14370 +45448 14366 +43448 14362 +31385 14362 +39562 14356 +19194 14355 +42298 14355 +26981 14354 +45431 14353 +44344 14353 +37912 14352 +38933 14349 +9778 14348 +46816 14346 +38532 14345 +33281 14344 +37670 14343 +48168 14340 +48072 14339 +9420 14338 +49540 14334 +27831 14331 +36614 14330 +31885 14329 +42681 14329 +45272 14327 +43496 14326 +41670 14326 +49917 14325 +40312 14322 +41467 14322 +20808 14321 +19199 14319 +24460 14317 +43781 14314 +32431 14312 +45620 14307 +46823 14305 +45473 14302 +48479 14301 +34165 14301 +40728 14300 +29562 14297 +38658 14296 +24886 14293 +32041 14286 +41339 14286 +32279 14283 +44438 14282 +34319 14282 +48942 14280 +10962 14277 +50005 14272 +9488 14271 +46457 14269 +48571 14267 +49815 14265 +39029 14264 +48546 14263 +47323 14263 +36152 14262 +24236 14261 +46558 14259 +27125 14256 +22114 14255 +35666 14253 +31870 14252 +22313 14249 +21964 14248 +31709 14248 +28339 14247 +37044 14245 +45895 14241 +21745 14239 +49359 14239 +19757 14237 +41507 14228 +12166 14226 +27271 14226 +43201 14225 +41988 14224 +28363 14223 +30781 14222 +2464 14222 +25820 14220 +30598 14218 +37056 14215 +13680 14215 +46712 14215 +26335 14214 +30561 14214 +21475 14214 +35009 14214 +30637 14205 +47381 14204 +44762 14203 +40967 14199 +37067 14199 +30708 14197 +46751 14196 +45619 14195 +44640 14194 +38512 14191 +32780 14191 +49072 14188 +41276 14184 +35728 14183 +48254 14179 +36774 14177 +21446 14175 +24331 14175 +21308 14174 +40029 14173 +46645 14173 +18789 14170 +40566 14169 +38199 14167 +42259 14165 +35166 14162 +34084 14159 +39941 14154 +45610 14150 +165 14149 +36934 14148 +29725 14146 +43679 14146 +8427 14140 +50221 14133 +38709 14131 +43464 14130 +41844 14127 +39363 14127 +12210 14126 +36439 14123 +42096 14121 +26616 14119 +11324 14118 +49274 14117 +39066 14116 +44676 14116 +12367 14114 +49538 14106 +49298 14105 +46370 14104 +20939 14099 +17957 14099 +45020 14097 +36422 14094 +24220 14093 +17429 14091 +14689 14090 +16607 14088 +6657 14080 +49605 14078 +47525 14076 +27846 14072 +38538 14071 +38917 14070 +4672 14066 +33676 14066 +45491 14065 +45937 14061 +45688 14060 +37569 14056 +27056 14054 +49892 14052 +46456 14052 +8279 14045 +34636 14045 +9560 14044 +32571 14044 +15249 14037 +39687 14037 +42190 14035 +33804 14031 +39963 14027 +39367 14026 +19986 14025 +45388 14023 +43745 14021 +26032 14019 +38033 14015 +18274 14015 +38414 14014 +46223 14014 +46670 14013 +20258 14008 +24996 14006 +44996 14004 +49937 14000 +11748 13999 +47033 13998 +46588 13997 +25832 13996 +47583 13994 +40189 13994 +25468 13993 +28934 13992 +43058 13989 +31119 13986 +39270 13984 +44540 13982 +49810 13981 +50124 13978 +42977 13977 +48887 13976 +44787 13976 +47743 13976 +38489 13972 +16827 13971 +41181 13966 +45412 13965 +32372 13964 +38474 13962 +11547 13961 +37917 13960 +36532 13959 +8933 13955 +34622 13954 +48785 13953 +25202 13953 +45579 13952 +41725 13952 +41011 13948 +23032 13947 +48270 13943 +47577 13941 +49938 13939 +31322 13937 +39141 13936 +31978 13935 +33210 13924 +13088 13919 +16624 13917 +32269 13912 +33640 13908 +36474 13906 +44007 13905 +5551 13905 +37158 13899 +37280 13898 +45323 13898 +48276 13898 +39546 13896 +25968 13896 +30076 13892 +48905 13890 +50107 13890 +45917 13890 +48208 13889 +31268 13880 +35452 13876 +43447 13876 +48013 13873 +9946 13873 +40216 13869 +43163 13869 +26052 13866 +46200 13866 +17055 13865 +45882 13854 +43954 13852 +36091 13851 +47593 13850 +46380 13849 +35192 13849 +30575 13849 +40443 13849 +28988 13843 +31200 13843 +35099 13841 +29971 13831 +50063 13830 +49923 13830 +29182 13829 +10331 13827 +31319 13827 +17244 13819 +22617 13818 +24923 13814 +49271 13814 +39629 13814 +19721 13809 +39319 13809 +10634 13808 +23459 13803 +36829 13802 +34981 13800 +45920 13800 +49916 13798 +34062 13794 +8409 13787 +24860 13787 +37160 13786 +47011 13785 +28296 13780 +10037 13778 +49903 13775 +46863 13772 +32844 13768 +27685 13767 +35920 13766 +35511 13765 +145 13758 +11797 13757 +26628 13754 +46974 13754 +8817 13753 +42817 13752 +37944 13750 +43402 13746 +24608 13743 +37894 13742 +36007 13741 +29160 13733 +43019 13728 +46224 13728 +23901 13726 +10406 13725 +42056 13723 +45328 13722 +40042 13720 +44279 13716 +26435 13715 +34045 13715 +43591 13715 +9711 13714 +32006 13714 +23633 13713 +3408 13712 +32401 13712 +38610 13709 +45945 13709 +23697 13708 +15235 13707 +16779 13706 +50004 13705 +19937 13704 +50115 13704 +11297 13701 +33361 13699 +33414 13695 +40628 13694 +45086 13692 +47197 13687 +42029 13687 +41568 13686 +16872 13686 +15404 13684 +26184 13682 +33150 13681 +38713 13674 +46003 13672 +31399 13672 +47343 13670 +35957 13666 +40036 13664 +31621 13662 +45884 13657 +45821 13657 +48351 13655 +19046 13655 +39079 13652 +32346 13651 +25188 13650 +48802 13645 +38655 13644 +23504 13643 +9026 13640 +41417 13640 +36848 13639 +26515 13635 +44984 13635 +25844 13634 +49223 13632 +43062 13632 +48340 13632 +23215 13629 +45287 13627 +19599 13624 +50018 13623 +33740 13621 +18185 13617 +35789 13616 +27378 13613 +46181 13611 +49090 13610 +8897 13610 +6585 13602 +17928 13602 +43873 13600 +25467 13600 +22748 13599 +45487 13598 +27576 13596 +18817 13596 +28075 13595 +37675 13593 +47079 13592 +37295 13591 +6526 13591 +45152 13590 +43202 13588 +26612 13584 +39529 13584 +8690 13584 +34352 13583 +8219 13581 +48440 13580 +39366 13578 +42369 13573 +41221 13572 +37369 13571 +43832 13568 +4841 13565 +22747 13562 +34844 13561 +43179 13561 +22031 13560 +45359 13560 +39754 13555 +43364 13555 +49240 13554 +49480 13552 +46716 13548 +30922 13548 +24697 13544 +49931 13543 +4408 13542 +39385 13541 +42295 13541 +22143 13540 +46299 13540 +45669 13540 +47549 13539 +2719 13538 +25716 13537 +16864 13536 +32383 13536 +43785 13534 +23636 13531 +11381 13530 +46597 13529 +41547 13527 +16881 13519 +34680 13519 +19243 13516 +26099 13515 +41452 13512 +33696 13512 +48846 13507 +41277 13501 +42093 13498 +32287 13498 +33216 13492 +39481 13491 +36797 13489 +34469 13489 +18968 13488 +44097 13482 +37441 13477 +48078 13476 +46810 13475 +8973 13475 +48207 13474 +21598 13472 +45361 13470 +38535 13470 +14171 13469 +19153 13469 +40754 13467 +36842 13463 +39494 13460 +27237 13457 +48210 13456 +23988 13456 +49818 13455 +41435 13452 +44394 13451 +27054 13450 +46963 13448 +48435 13448 +26064 13447 +22290 13446 +36537 13443 +40379 13443 +30556 13437 +28570 13437 +42244 13437 +41684 13435 +4208 13434 +4195 13431 +38824 13431 +16665 13427 +40040 13427 +28724 13425 +31106 13416 +34533 13415 +30526 13414 +16701 13413 +34088 13413 +26205 13411 +6002 13408 +45180 13408 +45855 13408 +41887 13405 +23579 13402 +20272 13400 +38729 13396 +46171 13385 +47373 13381 +46328 13377 +49928 13375 +49613 13373 +48317 13373 +28272 13371 +40226 13370 +31584 13369 +48374 13360 +49711 13358 +28093 13356 +25057 13356 +10502 13351 +6627 13351 +49020 13351 +47575 13350 +46713 13349 +25742 13349 +19965 13349 +9227 13348 +45330 13341 +36572 13341 +23784 13339 +43378 13334 +30590 13332 +20783 13329 +40601 13329 +23293 13327 +21063 13325 +18706 13324 +39819 13323 +1003 13321 +46038 13318 +49965 13312 +49203 13308 +43473 13307 +47821 13304 +43541 13303 +43129 13301 +48010 13300 +48770 13297 +27705 13295 +18649 13293 +47269 13291 +47138 13290 +42709 13287 +48445 13286 +31961 13285 +30491 13282 +44855 13281 +22282 13278 +47996 13278 +50098 13277 +31479 13275 +16719 13274 +23132 13270 +28665 13268 +48339 13266 +47811 13266 +5310 13264 +42561 13263 +48042 13262 +49345 13262 +147 13262 +30729 13259 +42460 13251 +17493 13245 +48735 13245 +47637 13240 +10004 13239 +33066 13238 +38389 13238 +45440 13236 +37934 13236 +33235 13234 +36635 13232 +43005 13231 +38064 13230 +33146 13229 +36751 13227 +25934 13226 +46826 13224 +18691 13221 +29391 13218 +12441 13215 +25628 13213 +24191 13212 +19767 13209 +7629 13208 +38144 13198 +36802 13198 +11200 13191 +43121 13190 +11497 13188 +31902 13188 +20344 13188 +45991 13185 +16228 13184 +161 13183 +12837 13182 +24951 13179 +42269 13178 +37654 13177 +40050 13177 +41265 13177 +48282 13175 +19298 13174 +40642 13170 +12638 13168 +47870 13167 +26371 13166 +36484 13163 +39684 13160 +25563 13159 +32937 13155 +35655 13154 +8986 13152 +33942 13148 +46053 13147 +15141 13147 +23898 13146 +34973 13145 +45589 13143 +41419 13141 +15553 13136 +2480 13134 +22916 13132 +48050 13132 +4076 13128 +33703 13125 +32374 13122 +32245 13120 +28835 13119 +44835 13119 +41625 13115 +45149 13114 +41464 13113 +48532 13111 +38861 13107 +43699 13103 +37328 13103 +34215 13102 +8944 13101 +23951 13101 +40177 13101 +17705 13100 +22184 13099 +48104 13096 +38437 13096 +48849 13095 +47161 13092 +47777 13092 +31544 13089 +40989 13088 +46689 13088 +33351 13087 +12541 13087 +38416 13085 +39260 13083 +46522 13080 +14360 13077 +43311 13075 +45322 13074 +29730 13067 +45387 13064 +35428 13063 +15303 13061 +24258 13060 +49998 13057 +36335 13056 +50036 13051 +29678 13051 +3631 13048 +30034 13047 +37638 13042 +48646 13042 +38233 13041 +31562 13040 +45470 13039 +48331 13036 +49569 13036 +41944 13035 +23283 13030 +24529 13028 +38280 13028 +38301 13027 +49881 13026 +49989 13024 +46772 13021 +15168 13021 +50137 13020 +36578 13020 +37471 13019 +35072 13018 +41730 13017 +45915 13017 +40779 13017 +19286 13013 +41616 13009 +47757 13007 +47238 13005 +45923 13005 +30395 13005 +42332 13004 +31315 13004 +17178 12995 +19511 12995 +28710 12993 +42512 12993 +32463 12990 +48502 12989 +10065 12986 +43211 12985 +39972 12981 +15160 12980 +23763 12973 +32822 12969 +50096 12969 +50154 12965 +2109 12963 +42531 12963 +17934 12960 +28216 12955 +44621 12954 +36456 12953 +47264 12952 +33382 12948 +19669 12944 +24223 12944 +9504 12942 +42136 12941 +47321 12941 +16680 12935 +26393 12934 +37368 12931 +24819 12928 +34193 12926 +40522 12925 +49635 12925 +44883 12922 +31249 12921 +38788 12921 +42072 12917 +43756 12917 +40039 12916 +36492 12916 +47622 12914 +34605 12909 +17804 12902 +47231 12899 +49966 12899 +38994 12898 +49812 12897 +38996 12897 +46316 12896 +35542 12894 +24159 12892 +7553 12892 +44445 12891 +31872 12888 +13615 12885 +36420 12882 +9571 12880 +41362 12879 +15384 12878 +38559 12871 +40763 12869 +38395 12867 +32845 12865 +48065 12865 +37559 12857 +29310 12856 +30754 12849 +35111 12849 +36206 12846 +40683 12842 +45946 12842 +36315 12840 +23343 12839 +36416 12839 +39371 12837 +5204 12836 +35596 12834 +31567 12830 +25867 12826 +46802 12825 +49776 12825 +44733 12822 +41022 12817 +48689 12815 +45883 12815 +13739 12808 +43306 12806 +35936 12805 +13410 12799 +45013 12792 +47839 12792 +23725 12789 +17569 12788 +43362 12779 +18771 12775 +32307 12774 +49671 12773 +32768 12769 +32397 12768 +41935 12765 +20907 12764 +35285 12764 +11064 12761 +9816 12761 +49479 12759 +20575 12757 +35487 12754 +36413 12753 +18602 12753 +49814 12752 +45203 12752 +16850 12751 +45885 12748 +34883 12743 +37971 12742 +41689 12741 +47407 12741 +33029 12725 +40951 12722 +47048 12722 +39913 12719 +31461 12717 +39485 12711 +14784 12708 +45815 12705 +37138 12704 +19246 12700 +36010 12698 +48385 12692 +27076 12691 +40533 12687 +43970 12686 +23850 12686 +45849 12683 +42538 12682 +45816 12678 +32505 12678 +49681 12677 +46482 12676 +35832 12676 +46475 12670 +11692 12669 +24477 12669 +47536 12668 +5974 12668 +42585 12660 +20654 12658 +22857 12658 +15776 12658 +33200 12658 +2410 12658 +48021 12656 +12708 12652 +13702 12652 +46817 12650 +50114 12649 +30779 12645 +32999 12644 +40731 12641 +48940 12637 +21065 12636 +23606 12633 +38394 12630 +33737 12624 +28787 12623 +22510 12620 +42147 12620 +48076 12619 +45333 12607 +35619 12606 +48619 12602 +50129 12600 +48752 12597 +6993 12596 +36967 12595 +13490 12594 +44241 12591 +23778 12588 +12102 12587 +13918 12583 +47081 12582 +28353 12581 +19758 12573 +28512 12572 +34236 12569 +42904 12568 +38950 12567 +33222 12564 +34054 12561 +35644 12559 +37122 12559 +44605 12555 +27131 12555 +34197 12550 +39049 12548 +34848 12546 +33244 12542 +25297 12541 +34719 12541 +33090 12541 +16995 12540 +19093 12538 +21886 12538 +46596 12537 +11124 12537 +36543 12536 +49268 12534 +44072 12528 +39571 12527 +15521 12526 +16838 12524 +38829 12522 +23216 12521 +34513 12520 +41270 12518 +46774 12516 +49275 12512 +20860 12511 +15690 12509 +41912 12506 +41509 12505 +8052 12505 +50194 12503 +44814 12502 +32898 12501 +13744 12501 +50056 12499 +38047 12496 +48978 12488 +37715 12487 +16722 12487 +32940 12482 +46740 12481 +20519 12481 +25206 12480 +9746 12476 +41173 12475 +40114 12474 +11213 12463 +37997 12461 +43969 12457 +41659 12454 +21488 12454 +49260 12453 +48033 12452 +47914 12452 +22872 12451 +30670 12449 +25484 12446 +38557 12444 +49343 12443 +32278 12443 +33403 12442 +17737 12442 +5397 12441 +31294 12439 +33268 12437 +43629 12434 +48508 12434 +34507 12433 +26999 12431 +23939 12430 +39758 12429 +50230 12427 +40127 12424 +48353 12423 +43794 12421 +39060 12421 +41500 12418 +29110 12418 +28050 12418 +39442 12415 +46723 12412 +31435 12411 +41030 12404 +13138 12403 +26033 12403 +45859 12400 +13838 12398 +30330 12394 +40371 12393 +19904 12392 +32828 12390 +34517 12390 +32765 12388 +37805 12382 +49360 12382 +25630 12377 +40346 12374 +47125 12368 +49732 12367 +22472 12366 +36836 12365 +21535 12365 +29257 12363 +42390 12362 +9023 12361 +41371 12359 +48919 12359 +27594 12358 +37892 12356 +47858 12354 +43669 12354 +28377 12346 +39719 12342 +49169 12341 +12667 12340 +43696 12338 +47425 12337 +44266 12337 +26462 12336 +49886 12336 +49367 12336 +11310 12334 +48059 12334 +9081 12333 +22614 12333 +42800 12325 +38011 12323 +40002 12323 +30688 12323 +39020 12322 +26144 12322 +41857 12321 +27897 12320 +35448 12317 +39889 12313 +46765 12310 +45063 12310 +12447 12310 +32500 12309 +13846 12306 +22762 12303 +12687 12301 +45383 12300 +40357 12298 +48178 12296 +31959 12296 +15969 12294 +37983 12291 +26125 12288 +44895 12287 +41571 12286 +48533 12279 +39176 12279 +44961 12276 +47087 12275 +21604 12275 +48842 12273 +24606 12268 +48401 12266 +33165 12263 +26709 12263 +35777 12263 +45394 12261 +18226 12257 +28743 12255 +36407 12252 +18299 12251 +43849 12248 +39237 12245 +36372 12244 +44029 12240 +11098 12239 +5835 12238 +38345 12236 +45880 12235 +29465 12235 +49449 12234 +41240 12233 +34024 12232 +42616 12232 +45076 12227 +45947 12227 +49153 12227 +36163 12220 +45482 12220 +17599 12218 +25458 12218 +39969 12211 +4253 12210 +49460 12208 +43205 12208 +39250 12206 +43196 12205 +43659 12204 +33100 12204 +43294 12204 +3569 12202 +41469 12202 +31373 12201 +47315 12199 +41105 12199 +38797 12193 +30547 12192 +28913 12191 +49755 12189 +44804 12188 +46433 12187 +28768 12187 +43519 12186 +6394 12186 +34653 12185 +43752 12183 +33857 12180 +29035 12180 +38857 12179 +40165 12179 +49106 12178 +48859 12177 +19968 12176 +47759 12174 +50218 12172 +48287 12169 +32004 12167 +17001 12167 +47392 12165 +45903 12153 +48224 12153 +39794 12149 +49016 12148 +28904 12147 +42019 12144 +44433 12143 +25568 12143 +38980 12141 +49152 12139 +45959 12135 +22936 12132 +44080 12124 +10020 12124 +48819 12122 +46262 12121 +5971 12113 +39294 12112 +20074 12110 +10739 12110 +4549 12107 +50053 12105 +25973 12105 +38832 12102 +28496 12102 +45904 12101 +45965 12097 +40553 12095 +31313 12095 +49526 12095 +48387 12093 +20556 12092 +47051 12092 +46986 12090 +33559 12086 +34055 12083 +32668 12073 +26018 12072 +8538 12072 +39436 12069 +43992 12068 +33793 12067 +42931 12066 +36261 12064 +24890 12063 +25097 12061 +4880 12061 +39356 12057 +48085 12057 +30431 12053 +35364 12051 +27299 12050 +32558 12050 +4567 12048 +32366 12048 +47617 12047 +32893 12046 +17360 12044 +37265 12042 +42217 12040 +48482 12035 +46771 12034 +31616 12032 +49387 12032 +49724 12031 +47110 12027 +48478 12025 +23705 12022 +33674 12021 +34049 12020 +23047 12017 +13379 12016 +28366 12014 +37516 12011 +27882 12010 +25504 12009 +49977 12008 +42500 12005 +17497 12004 +46971 12001 +26806 12000 +27815 12000 +40337 11998 +42756 11996 +31520 11995 +35751 11995 +49662 11994 +41958 11991 +35100 11991 +50057 11987 +49907 11987 +8240 11987 +47556 11985 +31789 11979 +43634 11977 +43345 11976 +48871 11975 +46920 11972 +47311 11970 +22897 11970 +7828 11969 +39715 11967 +30946 11967 +41048 11967 +15267 11964 +48481 11958 +41691 11956 +14670 11955 +34223 11955 +48251 11953 +45970 11953 +49972 11948 +48048 11948 +46987 11945 +45713 11943 +49664 11940 +25526 11938 +43249 11936 +36493 11934 +30887 11934 +43516 11927 +28169 11926 +34627 11926 +41451 11925 +31040 11924 +18438 11911 +37819 11908 +12084 11907 +30735 11903 +37310 11899 +3419 11898 +34591 11897 +48139 11896 +46024 11895 +3317 11895 +21485 11894 +35928 11894 +42277 11892 +26418 11892 +39746 11891 +27331 11890 +49054 11889 +33641 11889 +49667 11886 +50241 11882 +46205 11882 +40122 11880 +49100 11877 +48779 11876 +31943 11873 +16188 11873 +29124 11866 +44962 11862 +48391 11859 +42945 11853 +27535 11852 +44212 11851 +48758 11849 +39626 11849 +49098 11842 +41389 11836 +47771 11836 +41906 11827 +49826 11824 +44224 11821 +43947 11819 +40824 11817 +7132 11815 +41787 11814 +27738 11813 +23471 11813 +26493 11810 +39991 11808 +5206 11806 +48693 11800 +40866 11799 +43920 11797 +49321 11795 +42839 11794 +47615 11793 +48300 11792 +20568 11791 +47878 11790 +35713 11789 +21353 11788 +17597 11788 +28349 11785 +45656 11784 +33279 11783 +28409 11781 +49238 11781 +38217 11780 +43838 11780 +38673 11778 +37963 11777 +34399 11774 +17331 11774 +49740 11767 +33739 11766 +41697 11763 +6904 11758 +48203 11756 +20253 11755 +22201 11750 +44411 11749 +27490 11747 +38664 11744 +48068 11741 +32068 11740 +48668 11739 +33308 11736 +48616 11735 +47465 11732 +41842 11730 +46768 11729 +21244 11725 +41316 11724 +36137 11724 +47854 11722 +44479 11721 +46686 11721 +46575 11719 +47335 11719 +31312 11718 +46246 11714 +31030 11712 +44899 11712 +41796 11709 +42832 11709 +36687 11706 +15517 11703 +23765 11702 +24919 11700 +47998 11700 +48040 11697 +47088 11694 +48989 11690 +20274 11687 +28185 11687 +42919 11682 +41733 11682 +30203 11676 +21559 11675 +29158 11675 +47080 11675 +11250 11672 +6522 11671 +28400 11671 +22615 11668 +34656 11667 +46276 11656 +28117 11653 +11242 11646 +30025 11645 +47801 11644 +41397 11644 +35392 11644 +50122 11643 +50164 11643 +50244 11642 +35044 11642 +42686 11641 +49947 11640 +46793 11631 +27554 11630 +32580 11629 +36136 11628 +40656 11625 +45599 11624 +39806 11623 +12710 11620 +45380 11619 +11355 11619 +42393 11617 +46569 11614 +14228 11612 +44139 11604 +41502 11602 +40578 11601 +30655 11601 +20310 11595 +42552 11593 +35884 11592 +13877 11591 +49835 11590 +36807 11590 +40512 11589 +37549 11589 +46978 11587 +25034 11585 +40843 11584 +47532 11581 +11116 11577 +47103 11570 +29003 11566 +45065 11564 +44325 11564 +23187 11560 +47278 11560 +37598 11559 +45845 11558 +36631 11557 +41497 11557 +36230 11557 +42230 11556 +47520 11555 +27423 11554 +48507 11553 +47073 11553 +43429 11553 +29666 11552 +45873 11552 +27471 11549 +49205 11549 +41801 11548 +33926 11544 +12286 11544 +43821 11542 +3135 11538 +26377 11537 +37038 11535 +36716 11534 +48851 11533 +42827 11532 +45167 11531 +43750 11531 +17574 11525 +25373 11524 +27708 11524 +50211 11524 +37103 11520 +49210 11517 +44700 11517 +171 11513 +42112 11511 +31217 11511 +46404 11510 +44351 11510 +48394 11510 +25952 11510 +16211 11509 +40945 11507 +24127 11506 +32873 11506 +34398 11504 +35237 11499 +46032 11494 +26559 11490 +48032 11489 +47456 11482 +49502 11480 +25038 11480 +27923 11479 +35763 11479 +33428 11478 +17808 11477 +25896 11476 +34028 11473 +31299 11472 +25166 11471 +35204 11466 +46573 11465 +36409 11463 +49633 11460 +40805 11460 +24491 11459 +32589 11459 +36325 11451 +45793 11448 +26528 11448 +14630 11447 +45611 11443 +37947 11441 +42118 11440 +855 11436 +49405 11434 +36113 11432 +43800 11429 +47514 11425 +49103 11425 +37705 11425 +31948 11425 +18196 11424 +40976 11422 +44558 11421 +26894 11417 +40244 11416 +20789 11414 +35027 11411 +42267 11410 +49027 11408 +25877 11407 +49062 11406 +32866 11404 +32723 11401 +40969 11400 +39359 11398 +35417 11397 +44602 11396 +44933 11396 +45847 11395 +50038 11390 +27683 11389 +23525 11389 +46006 11388 +19729 11386 +24370 11386 +22460 11381 +42323 11379 +46244 11374 +16377 11373 +31013 11369 +40510 11363 +8641 11363 +29590 11360 +25908 11355 +40539 11350 +44342 11347 +25093 11346 +14244 11346 +43797 11346 +30891 11345 +46523 11342 +7345 11339 +25981 11337 +33780 11337 +31034 11337 +49962 11336 +30687 11335 +46483 11334 +32604 11329 +47754 11329 +36501 11328 +48932 11325 +17874 11324 +27754 11322 +17829 11319 +44454 11319 +41821 11317 +34388 11317 +33103 11317 +49534 11316 +18092 11315 +28406 11315 +45710 11311 +50120 11311 +8294 11309 +44982 11306 +49689 11305 +45569 11300 +43506 11298 +19331 11297 +20361 11297 +23900 11294 +45069 11292 +47700 11288 +29245 11284 +29625 11282 +25956 11280 +25196 11279 +10313 11275 +34788 11275 +48907 11271 +42039 11268 +47942 11267 +49773 11266 +47933 11265 +43143 11264 +33257 11259 +23517 11258 +42848 11256 +45686 11254 +47710 11252 +42469 11252 +26921 11250 +45057 11250 +44059 11249 +43917 11247 +44990 11247 +46063 11243 +25552 11240 +32236 11239 +47621 11238 +37405 11237 +50253 11233 +45637 11233 +13170 11233 +15988 11232 +31374 11231 +32503 11230 +38263 11228 +27865 11228 +39504 11224 +42044 11223 +31560 11221 +9129 11216 +34163 11214 +41311 11213 +35539 11213 +38295 11213 +41036 11211 +17903 11208 +50138 11202 +37039 11197 +39461 11196 +14011 11191 +46747 11190 +49585 11189 +22422 11187 +10738 11186 +36216 11185 +35543 11183 +17257 11182 +36287 11182 +21069 11181 +41142 11181 +39396 11177 +46375 11176 +24542 11173 +40204 11173 +35694 11173 +30308 11171 +41603 11170 +37837 11166 +11199 11165 +34978 11165 +45623 11165 +45641 11164 +49371 11164 +18827 11160 +8385 11159 +18001 11159 +44315 11158 +43399 11157 +47954 11157 +41877 11154 +38100 11143 +6410 11140 +45536 11139 +27530 11137 +34368 11131 +19739 11128 +33801 11122 +43535 11121 +36639 11118 +11958 11117 +40605 11110 +40916 11108 +49332 11107 +31317 11100 +31366 11092 +12836 11089 +23872 11088 +39522 11086 +12504 11085 +32157 11084 +49166 11080 +46667 11077 +8184 11072 +14467 11066 +8938 11063 +28054 11063 +16375 11061 +18312 11060 +39010 11060 +30411 11053 +39936 11052 +38470 11052 +34714 11051 +40865 11050 +46830 11048 +34878 11047 +49948 11047 +21359 11047 +30366 11044 +31808 11044 +32854 11044 +32124 11043 +29177 11043 +34971 11042 +42523 11032 +37232 11025 +41899 11024 +29425 11023 +46403 11016 +28015 11014 +47235 11013 +32296 11010 +11309 11010 +33999 11007 +35263 11004 +41163 11003 +42400 11002 +35836 10999 +19714 10998 +30196 10996 +38592 10994 +44248 10994 +2268 10990 +30417 10990 +2359 10988 +45052 10986 +47114 10985 +34729 10982 +46540 10982 +24879 10980 +46576 10979 +20364 10979 +16910 10975 +33288 10966 +27122 10966 +2325 10965 +46015 10963 +20475 10960 +39084 10959 +49365 10958 +41692 10948 +40734 10947 +4748 10946 +35753 10942 +36528 10941 +27744 10941 +13618 10939 +38778 10934 +26932 10934 +47123 10933 +46354 10931 +42751 10929 +33219 10928 +29231 10926 +44023 10925 +46161 10918 +29427 10917 +45117 10917 +13554 10912 +36350 10910 +48889 10910 +36225 10909 +5782 10909 +50121 10907 +31079 10906 +31587 10904 +45307 10903 +29490 10901 +37275 10899 +10237 10892 +42944 10889 +7242 10887 +43595 10886 +41102 10886 +48804 10885 +45771 10885 +31657 10884 +36164 10882 +33225 10882 +23177 10875 +42641 10874 +47371 10873 +36075 10870 +28427 10867 +45104 10863 +46687 10862 +19377 10860 +14229 10860 +23635 10860 +32422 10858 +38104 10858 +34768 10857 +16401 10857 +27301 10856 +46259 10855 +43460 10854 +31529 10853 +15270 10850 +23295 10849 +46931 10847 +29084 10844 +46399 10842 +44401 10840 +25559 10837 +21398 10836 +43649 10835 +48017 10833 +13845 10832 +42126 10830 +41646 10824 +36020 10824 +36055 10822 +4154 10819 +25958 10814 +49017 10813 +29375 10812 +24849 10811 +12117 10811 +27204 10810 +25609 10810 +45021 10809 +44101 10803 +42963 10800 +17035 10799 +32499 10798 +32087 10798 +34189 10797 +135 10793 +41384 10793 +15057 10789 +39426 10789 +22164 10787 +33738 10784 +46490 10778 +41920 10776 +30075 10775 +47368 10775 +47126 10774 +45591 10766 +31458 10763 +18464 10763 +28189 10761 +44713 10761 +36873 10759 +46425 10758 +45745 10755 +38432 10754 +23861 10752 +25233 10750 +39428 10749 +39932 10749 +47831 10748 +22774 10743 +26438 10742 +46153 10741 +27741 10736 +14758 10733 +6677 10731 +16474 10728 +43041 10728 +38517 10727 +43632 10726 +23296 10724 +37297 10724 +47707 10719 +9259 10715 +36835 10714 +17959 10708 +7377 10708 +27295 10707 +25825 10707 +46666 10704 +29812 10702 +48239 10698 +46145 10697 +47293 10695 +35422 10695 +18569 10689 +4644 10688 +20380 10687 +12563 10684 +28463 10683 +32896 10680 +41408 10679 +39892 10679 +34745 10678 +41825 10676 +50002 10672 +44412 10668 +31500 10666 +32053 10665 +24188 10665 +41766 10664 +46453 10663 +37032 10661 +46045 10660 +32977 10660 +28434 10658 +38023 10655 +43076 10652 +23882 10650 +41245 10650 +34145 10649 +42536 10642 +23479 10641 +20761 10641 +24266 10640 +17868 10640 +38993 10639 +27354 10638 +48432 10636 +41713 10636 +13687 10635 +36342 10634 +35317 10626 +34421 10625 +16710 10624 +41331 10623 +21469 10619 +34021 10619 +44484 10617 +45416 10616 +43137 10615 +33205 10613 +48594 10612 +49143 10610 +20362 10610 +49610 10602 +30207 10601 +41355 10599 +18962 10598 +36423 10596 +33095 10596 +26176 10593 +38363 10592 +34811 10591 +33455 10584 +1250 10583 +49958 10580 +45357 10579 +28535 10574 +38969 10572 +39353 10569 +29990 10568 +30163 10565 +46835 10564 +25258 10563 +36381 10562 +19148 10558 +32936 10557 +42104 10546 +48821 10544 +13492 10543 +49107 10541 +40292 10537 +15769 10536 +43327 10535 +46720 10532 +41361 10531 +43522 10530 +42303 10528 +42559 10527 +21136 10526 +49519 10523 +42306 10517 +33727 10515 +40178 10513 +46648 10506 +39948 10506 +11631 10506 +42808 10505 +38910 10502 +40961 10502 +30116 10501 +33581 10497 +40686 10497 +28834 10496 +49993 10495 +38678 10493 +48235 10488 +25617 10486 +30823 10477 +27932 10477 +42233 10470 +13725 10469 +42829 10468 +47504 10466 +13168 10459 +29434 10458 +29824 10457 +31519 10457 +34239 10453 +46812 10449 +45776 10441 +5632 10440 +18331 10439 +16025 10436 +47394 10436 +41831 10435 +43382 10435 +47784 10432 +39499 10431 +16317 10427 +11507 10417 +34957 10416 +28586 10412 +47659 10411 +22684 10404 +38670 10404 +49821 10394 +40937 10392 +48992 10391 +20620 10389 +25561 10382 +6636 10380 +27071 10377 +21399 10377 +32026 10375 +13811 10374 +25599 10374 +11008 10373 +11209 10373 +32574 10369 +47281 10368 +40872 10366 +41986 10363 +17602 10360 +34853 10358 +35653 10357 +47310 10356 +34734 10344 +18980 10343 +31226 10340 +43215 10339 +44944 10339 +41133 10337 +12355 10336 +42902 10336 +23920 10334 +35796 10329 +37311 10325 +44876 10325 +31771 10323 +18766 10320 +17398 10320 +4613 10320 +30815 10319 +26480 10317 +3302 10313 +44356 10312 +9120 10312 +25008 10312 +47800 10311 +46865 10310 +35478 10306 +48169 10298 +25198 10293 +8813 10291 +24541 10286 +25015 10286 +30228 10285 +37908 10284 +48601 10284 +15858 10282 +20452 10281 +7984 10281 +14793 10280 +33767 10277 +41910 10276 +41635 10276 +47179 10274 +16672 10271 +34542 10270 +20166 10266 +38449 10265 +46437 10262 +29836 10260 +29127 10258 +29011 10257 +50094 10255 +41319 10254 +44403 10253 +18888 10250 +36827 10245 +27438 10243 +35811 10240 +20692 10240 +32577 10239 +43358 10235 +38357 10234 +29967 10234 +30532 10233 +48915 10232 +29279 10232 +12099 10231 +38490 10230 +47790 10230 +23650 10228 +40148 10227 +32506 10225 +9125 10219 +43372 10215 +25361 10214 +45268 10210 +32011 10207 +26568 10207 +21297 10203 +42697 10201 +23896 10201 +23813 10198 +39284 10196 +29389 10196 +2729 10196 +43173 10195 +44988 10193 +34651 10188 +9140 10186 +43823 10185 +37867 10182 +43031 10181 +35762 10180 +38785 10177 +42302 10177 +41735 10176 +21708 10173 +19653 10169 +30078 10168 +46543 10167 +46364 10164 +26059 10163 +31262 10163 +44420 10163 +31653 10161 +26489 10159 +48676 10158 +43401 10154 +5578 10151 +37555 10150 +32750 10146 +20696 10142 +37442 10139 +16663 10139 +39884 10139 +40630 10139 +17183 10139 +40119 10138 +28673 10137 +32864 10135 +46445 10134 +49765 10131 +48436 10130 +18557 10129 +22903 10129 +42577 10128 +22708 10127 +33450 10126 +2335 10124 +46334 10122 +36120 10118 +44710 10112 +42219 10112 +39040 10108 +47635 10107 +40783 10103 +35048 10101 +47167 10099 +38955 10099 +19448 10098 +44894 10092 +27488 10091 +33243 10090 +42981 10089 +16848 10088 +39648 10087 +43012 10086 +47262 10086 +21130 10086 +46204 10085 +2639 10084 +45725 10082 +38037 10081 +23712 10080 +34555 10080 +49004 10077 +34175 10075 +23406 10074 +26453 10072 +13567 10072 +49710 10071 +16321 10071 +46934 10067 +36400 10065 +14938 10057 +37786 10056 +33747 10056 +35541 10054 +41510 10052 +47375 10051 +26164 10051 +26259 10049 +27160 10047 +49678 10047 +35203 10045 +45638 10045 +168 10045 +40880 10044 +32986 10034 +32498 10029 +32273 10026 +24052 10025 +50130 10022 +36081 10022 +41880 10020 +48002 10017 +34946 10016 +21970 10012 +8626 10011 +31795 10010 +44187 10010 +48816 10008 +18374 10008 +23828 10007 +48174 10007 +32305 10000 +46178 9992 +49738 9991 +22294 9986 +34362 9978 +31494 9968 +45278 9968 +43530 9966 +136 9966 +38166 9965 +45778 9963 +22227 9962 +50022 9958 +38284 9955 +36652 9953 +39698 9952 +50039 9948 +17904 9947 +46654 9946 +19309 9946 +26952 9939 +38803 9938 +49229 9935 +46148 9934 +8951 9934 +29964 9931 +45018 9929 +35114 9929 +45348 9928 +33526 9925 +44280 9924 +48146 9923 +27467 9915 +44362 9915 +26449 9913 +30579 9913 +24937 9906 +37023 9905 +49450 9900 +46372 9899 +31694 9898 +19081 9895 +40006 9891 +36268 9888 +40082 9885 +42109 9881 +18503 9880 +47636 9878 +48220 9877 +34657 9876 +18491 9874 +18354 9871 +44696 9869 +18324 9868 +31628 9867 +50152 9865 +44347 9865 +38266 9864 +39169 9863 +42985 9861 +22920 9861 +48672 9857 +20778 9853 +28995 9853 +11253 9847 +11208 9846 +48708 9838 +41312 9835 +9522 9833 +24614 9830 +38247 9829 +45740 9829 +31281 9826 +40994 9823 +11893 9823 +38924 9822 +45444 9821 +22399 9821 +43061 9821 +16419 9821 +48510 9820 +25586 9817 +38420 9816 +35416 9816 +46805 9815 +41192 9814 +17447 9808 +48832 9807 +14989 9802 +19628 9802 +32447 9801 +40555 9797 +48605 9795 +14876 9794 +41474 9793 +34588 9791 +49429 9791 +49091 9790 +23124 9790 +4557 9789 +23478 9788 +23274 9785 +28008 9784 +44616 9782 +30358 9779 +49675 9776 +48038 9775 +31646 9775 +33369 9773 +5484 9770 +44287 9768 +3467 9765 +35812 9764 +41952 9761 +38516 9756 +40491 9753 +30763 9751 +25843 9751 +49677 9748 +50044 9746 +42698 9745 +4115 9745 +43601 9743 +32836 9739 +42282 9739 +39213 9738 +34960 9737 +26132 9735 +19598 9734 +30470 9732 +31482 9730 +151 9730 +41897 9727 +42593 9722 +35787 9716 +48551 9715 +50111 9708 +35312 9708 +16496 9705 +35226 9705 +48035 9703 +27092 9702 +28026 9701 +49060 9700 +24108 9700 +26488 9694 +32336 9693 +37741 9690 +37534 9687 +42860 9683 +48899 9682 +39729 9680 +7436 9676 +24874 9675 +49576 9675 +24794 9670 +44519 9670 +49204 9669 +11838 9669 +25105 9669 +42595 9665 +38514 9665 +20134 9664 +47644 9661 +27236 9660 +27016 9659 +33113 9659 +49415 9656 +49705 9654 +25689 9654 +23954 9652 +43467 9650 +47698 9647 +19124 9647 +31197 9645 +24762 9645 +27104 9637 +43263 9635 +44906 9635 +49284 9634 +16174 9628 +34348 9628 +46973 9627 +31324 9625 +20628 9622 +24251 9621 +47376 9619 +10132 9619 +35850 9611 +3531 9609 +32976 9606 +24593 9606 +49283 9604 +41866 9604 +32301 9603 +34100 9603 +37450 9596 +49548 9592 +33392 9591 +22738 9590 +26813 9590 +41682 9588 +38004 9587 +49136 9585 +34067 9585 +19352 9584 +18224 9584 +21180 9583 +31303 9583 +4211 9581 +28798 9580 +45066 9580 +37608 9578 +33460 9578 +32582 9577 +43698 9577 +41485 9572 +1651 9569 +11388 9569 +42448 9559 +39432 9557 +13731 9557 +44378 9555 +35827 9555 +20035 9546 +50061 9542 +5553 9541 +33823 9536 +27954 9533 +45973 9533 +34544 9532 +49616 9532 +46449 9531 +5862 9528 +24445 9526 +31337 9524 +42742 9524 +33959 9520 +46906 9519 +45697 9518 +39876 9515 +43352 9515 +650 9513 +24735 9513 +48662 9510 +43304 9507 +31361 9507 +42138 9505 +46149 9504 +21483 9504 +24858 9499 +43620 9494 +22439 9493 +42579 9492 +19389 9491 +43830 9490 +30073 9489 +43909 9489 +44242 9481 +8238 9481 +27812 9480 +34073 9465 +33049 9460 +46170 9458 +41942 9457 +32983 9456 +24427 9454 +28987 9453 +3295 9452 +48141 9451 +10258 9450 +20878 9448 +16154 9438 +31673 9431 +43767 9430 +42667 9429 +24371 9429 +27691 9426 +49584 9425 +14929 9419 +43111 9417 +30515 9417 +47526 9415 +32248 9411 +7687 9407 +8256 9407 +32778 9406 +47986 9395 +15764 9391 +13976 9391 +37447 9383 +32071 9378 +44393 9376 +18880 9371 +19626 9370 +38072 9364 +13955 9361 +47017 9359 +29845 9358 +2653 9355 +49328 9353 +8446 9351 +36969 9349 +41576 9346 +39639 9345 +18762 9343 +49651 9339 +33614 9336 +9789 9327 +42034 9326 +27981 9326 +40454 9321 +26185 9321 +39433 9321 +28600 9321 +44902 9321 +48703 9319 +49269 9316 +18000 9316 +43505 9316 +18378 9308 +43254 9307 +48011 9307 +30177 9306 +48490 9301 +49215 9301 +42591 9299 +42830 9297 +37541 9296 +35732 9295 +9270 9295 +49022 9292 +22344 9289 +34870 9286 +48175 9285 +35567 9284 +31923 9284 +2015 9283 +36650 9282 +49043 9281 +45708 9279 +17294 9278 +12775 9278 +39587 9278 +24781 9277 +36482 9276 +38501 9276 +44858 9274 +38382 9272 +27387 9267 +33312 9267 +50059 9265 +25007 9265 +37974 9264 +50184 9261 +41176 9260 +48795 9258 +34713 9258 +44746 9256 +45836 9255 +50104 9246 +39581 9245 +11579 9243 +46031 9240 +49793 9240 +30611 9232 +36992 9230 +47060 9225 +25031 9224 +34508 9222 +45595 9218 +48237 9216 +33003 9215 +49160 9213 +45896 9211 +48137 9209 +50037 9208 +43476 9207 +35624 9206 +37497 9204 +39297 9202 +9540 9201 +33873 9198 +49171 9197 +39424 9196 +41156 9191 +20520 9190 +23484 9187 +47502 9187 +29071 9181 +41501 9180 +14079 9179 +45742 9177 +39022 9177 +4926 9172 +31699 9171 +10842 9170 +42821 9169 +40803 9169 +46998 9167 +8669 9167 +27493 9166 +38720 9165 +44079 9164 +16745 9161 +47650 9156 +37781 9153 +27302 9150 +46130 9149 +15167 9143 +49955 9141 +43755 9140 +47605 9138 +47786 9138 +28104 9138 +34257 9136 +37037 9134 +16128 9134 +48780 9133 +48865 9130 +38136 9124 +32988 9120 +45461 9110 +44839 9108 +43594 9107 +48936 9105 +43681 9102 +41937 9096 +47845 9094 +48565 9092 +26513 9086 +9163 9084 +46363 9083 +38694 9081 +47415 9080 +29859 9080 +28432 9080 +14626 9077 +31539 9076 +49361 9072 +37219 9071 +48382 9069 +48713 9067 +24620 9065 +22811 9061 +42930 9061 +43135 9060 +38755 9060 +44975 9058 +44357 9057 +45597 9057 +19302 9057 +39104 9055 +47808 9053 +44788 9050 +29196 9049 +41054 9043 +39445 9042 +37950 9039 +32255 9038 +33954 9036 +44164 9034 +30961 9031 +49333 9031 +45266 9031 +44597 9030 +42532 9027 +23150 9026 +48407 9026 +26805 9026 +32809 9023 +1260 9023 +43301 9022 +48671 9020 +34043 9019 +40998 9017 +30104 9015 +28489 9011 +48163 9005 +49084 9003 +24864 8996 +47250 8993 +40918 8992 +40213 8991 +40342 8989 +31962 8988 +50246 8988 +49539 8986 +33510 8984 +28832 8983 +37591 8983 +50201 8982 +26483 8975 +30951 8972 +31571 8972 +34879 8967 +2020 8966 +10002 8964 +35299 8963 +23031 8960 +10124 8957 +12982 8952 +40061 8951 +32021 8949 +40220 8948 +46261 8948 +44909 8946 +49863 8943 +36702 8942 +32233 8940 +14632 8939 +40921 8939 +33395 8934 +7339 8929 +49545 8927 +39627 8926 +29233 8924 +40729 8923 +31131 8919 +30294 8918 +40436 8917 +28741 8916 +38804 8915 +24567 8913 +30978 8907 +35477 8903 +14694 8902 +30369 8900 +45390 8897 +32515 8895 +21975 8892 +27755 8892 +44687 8891 +41170 8887 +41757 8884 +33278 8883 +37130 8881 +27084 8880 +42567 8877 +50255 8875 +18414 8874 +37875 8874 +38825 8873 +35748 8871 +47035 8868 +26080 8862 +14571 8856 +39031 8848 +30053 8847 +48515 8846 +50171 8845 +37561 8841 +46870 8841 +45267 8841 +37326 8839 +24915 8836 +19929 8832 +21410 8831 +50003 8825 +39050 8825 +31223 8824 +44702 8824 +41374 8822 +37587 8821 +22317 8820 +35411 8817 +24565 8816 +21587 8815 +36965 8813 +49034 8811 +42678 8807 +36609 8804 +48826 8802 +35194 8802 +42033 8801 +33563 8796 +23230 8795 +27090 8794 +47999 8793 +47724 8792 +31484 8791 +49161 8787 +44623 8786 +48599 8783 +30309 8782 +33889 8780 +27077 8778 +43343 8775 +30459 8775 +39583 8773 +4109 8772 +35349 8767 +45410 8766 +29530 8765 +29706 8762 +29568 8760 +44751 8759 +43604 8759 +49065 8759 +30333 8758 +17772 8757 +39657 8756 +35224 8754 +31426 8752 +35450 8752 +22731 8750 +23919 8746 +42597 8744 +38693 8742 +48725 8740 +45893 8737 +32971 8735 +25780 8733 +38949 8732 +46071 8727 +29970 8725 +6418 8719 +7887 8719 +48190 8719 +17312 8717 +10806 8715 +49891 8715 +48951 8715 +17994 8714 +7566 8712 +30178 8712 +49693 8710 +39596 8710 +20652 8707 +47344 8701 +1019 8701 +35636 8699 +15860 8698 +25145 8694 +10555 8692 +2924 8692 +19864 8690 +33491 8690 +48879 8690 +26662 8685 +41152 8683 +16335 8681 +48410 8681 +14373 8679 +48126 8675 +43093 8673 +50110 8672 +33493 8666 +49797 8665 +43940 8665 +8651 8657 +48476 8655 +29238 8652 +28348 8649 +44556 8645 +7682 8639 +40254 8638 +38844 8634 +29609 8634 +37092 8629 +39665 8625 +42781 8624 +26743 8621 +31744 8620 +49715 8619 +29363 8618 +46655 8612 +19120 8612 +6262 8611 +45634 8611 +18974 8611 +16763 8611 +32348 8610 +34171 8608 +16130 8607 +48400 8605 +39401 8603 +28809 8601 +49658 8599 +13525 8596 +10376 8593 +26385 8592 +39075 8592 +41686 8591 +47185 8590 +50252 8589 +40963 8587 +40725 8585 +34381 8584 +46068 8584 +17088 8582 +49133 8582 +13366 8580 +35749 8579 +38582 8578 +40552 8578 +46211 8577 +28128 8577 +21700 8576 +21448 8575 +30866 8574 +40217 8573 +40607 8571 +21746 8568 +17394 8563 +35316 8559 +44529 8557 +6022 8553 +6963 8553 +26169 8553 +32943 8551 +38480 8549 +41652 8548 +37746 8547 +41876 8547 +49543 8543 +41023 8542 +40205 8540 +28495 8537 +35116 8537 +30513 8536 +47806 8536 +35898 8534 +12257 8533 +48877 8532 +43877 8529 +36663 8528 +47497 8527 +37366 8526 +33360 8524 +25215 8520 +37877 8520 +30635 8517 +35986 8511 +29790 8510 +38984 8508 +33346 8506 +31803 8505 +39307 8505 +30039 8502 +17971 8501 +16447 8500 +34363 8499 +30634 8498 +14210 8496 +40166 8494 +20543 8492 +37049 8490 +22222 8489 +1331 8483 +36414 8483 +47511 8483 +48897 8481 +37724 8477 +26177 8476 +47862 8475 +20500 8474 +26627 8472 +47072 8467 +44157 8467 +46059 8466 +50097 8464 +4218 8463 +40746 8463 +29321 8463 +27161 8461 +25387 8458 +47178 8457 +46650 8454 +49785 8448 +43180 8447 +41814 8444 +39274 8443 +39464 8440 +36790 8439 +40172 8438 +38093 8438 +46707 8437 +46555 8435 +29908 8430 +44901 8427 +34500 8423 +44939 8421 +14875 8419 +17452 8413 +35994 8412 +19991 8409 +49221 8409 +49261 8409 +23248 8408 +49876 8407 +42246 8405 +48495 8398 +24146 8397 +28636 8396 +50245 8395 +42372 8393 +31092 8393 +28929 8391 +38958 8386 +28501 8386 +48333 8386 +48923 8385 +41762 8384 +12561 8380 +39051 8380 +34094 8379 +39234 8378 +25676 8376 +44863 8376 +39605 8373 +35566 8372 +18800 8370 +21373 8370 +44873 8370 +29694 8366 +46304 8366 +28642 8363 +38108 8362 +35179 8360 +38401 8359 +39782 8357 +32352 8356 +35886 8354 +18924 8354 +32491 8353 +45979 8351 +43164 8349 +43430 8348 +46915 8346 +39014 8342 +32194 8337 +45258 8333 +35125 8331 +38983 8331 +33194 8330 +37372 8329 +13511 8328 +23377 8325 +31330 8324 +29034 8323 +19793 8321 +34836 8321 +44482 8319 +39218 8310 +38334 8305 +40341 8304 +40239 8300 +27005 8298 +32061 8297 +35313 8296 +27442 8293 +38800 8291 +21564 8291 +35610 8290 +47345 8290 +35252 8287 +47245 8283 +23227 8279 +45464 8278 +34075 8275 +34306 8273 +9237 8272 +40828 8268 +41518 8267 +12190 8261 +33141 8259 +30414 8259 +43219 8259 +46537 8256 +41771 8253 +34777 8249 +39400 8247 +39335 8247 +44669 8246 +45561 8242 +31652 8239 +35900 8238 +40482 8234 +46214 8229 +37181 8227 +34648 8227 +41743 8226 +3358 8225 +33240 8223 +37231 8222 +40952 8221 +17050 8221 +44308 8220 +33722 8217 +13250 8216 +42731 8216 +37253 8215 +34220 8213 +25442 8212 +39228 8208 +3485 8202 +35580 8202 +26469 8202 +19261 8201 +35215 8201 +45236 8200 +41694 8198 +42663 8193 +28196 8190 +19668 8187 +27726 8186 +33313 8184 +13743 8183 +40484 8183 +25817 8179 +42344 8170 +26751 8167 +46104 8163 +21893 8163 +28316 8162 +10621 8158 +46513 8156 +16553 8156 +28859 8156 +44258 8155 +45796 8150 +34367 8149 +14706 8145 +7581 8140 +34913 8140 +46207 8139 +30725 8139 +47217 8136 +32821 8135 +6111 8134 +23119 8133 +639 8131 +39914 8128 +23893 8127 +31979 8123 +2076 8121 +39057 8118 +44837 8116 +47685 8111 +42861 8111 +46135 8109 +40407 8107 +36156 8102 +49031 8099 +49044 8095 +4824 8092 +49316 8091 +43710 8088 +38897 8087 +42383 8085 +2459 8082 +16271 8082 +32125 8082 +22838 8078 +33891 8076 +39227 8075 +21188 8075 +42229 8075 +37659 8075 +41420 8074 +42181 8072 +41873 8070 +19749 8070 +44783 8068 +33497 8068 +30476 8068 +3766 8068 +45270 8067 +42950 8065 +48504 8061 +20722 8060 +41798 8059 +25078 8058 +49648 8058 +26841 8051 +45192 8051 +44606 8051 +40708 8051 +17327 8049 +6722 8048 +47477 8045 +44449 8044 +48087 8042 +29631 8041 +29701 8040 +43546 8039 +37596 8035 +42842 8035 +48121 8035 +28398 8034 +28473 8034 +22519 8033 +35498 8032 +39296 8031 +25811 8029 +40612 8027 +43982 8027 +35479 8027 +29938 8026 +35565 8024 +24021 8022 +44380 8021 +15400 8011 +12977 8010 +23946 8007 +45875 8007 +38544 8003 +35943 8003 +39241 8002 +23620 7999 +38447 7998 +11494 7998 +33615 7996 +36950 7990 +22882 7987 +25601 7986 +34983 7986 +30313 7984 +26362 7983 +19053 7982 +20988 7974 +47723 7974 +47306 7973 +25120 7970 +25668 7969 +46284 7968 +16207 7963 +36211 7959 +27866 7957 +18267 7950 +40550 7947 +35612 7946 +24966 7941 +46084 7939 +35888 7938 +36018 7936 +43436 7935 +16411 7932 +8614 7928 +41620 7925 +38676 7923 +13352 7919 +40374 7915 +19119 7915 +33711 7911 +49413 7910 +25938 7910 +35013 7909 +32594 7909 +42338 7905 +24708 7904 +30108 7902 +31141 7893 +44601 7891 +35937 7886 +48589 7880 +18036 7877 +40079 7877 +46014 7875 +48620 7874 +43689 7874 +46680 7867 +43618 7866 +40483 7866 +31762 7864 +35070 7855 +46203 7850 +44800 7846 +18445 7844 +16124 7841 +47298 7840 +34389 7839 +41032 7838 +20147 7836 +39170 7830 +26437 7828 +48699 7828 +36837 7824 +25513 7823 +38882 7821 +20287 7816 +28480 7814 +15512 7813 +45772 7811 +49864 7811 +47409 7805 +43730 7804 +49578 7802 +47889 7792 +38587 7792 +14763 7790 +49158 7789 +32640 7788 +48890 7773 +38978 7769 +37410 7766 +1606 7765 +38638 7763 +8934 7763 +45532 7762 +32901 7760 +49833 7760 +32981 7759 +24598 7755 +33567 7753 +43655 7745 +39301 7743 +45154 7740 +14575 7736 +26654 7736 +34867 7735 +45081 7734 +44791 7734 +29566 7734 +49467 7732 +29261 7727 +29531 7725 +44567 7725 +34447 7725 +29271 7724 +27732 7723 +46842 7720 +35991 7718 +26072 7715 +32155 7713 +21833 7710 +30027 7709 +29821 7702 +38453 7702 +25537 7701 +38293 7700 +17633 7699 +44196 7698 +43397 7698 +90 7697 +42452 7694 +18718 7694 +476 7692 +50250 7690 +32458 7690 +27245 7689 +49302 7685 +36990 7675 +33377 7672 +35929 7670 +49493 7667 +48844 7664 +38454 7661 +33765 7661 +21121 7659 +33952 7659 +48288 7654 +32888 7654 +44061 7653 +42942 7644 +48762 7640 +44503 7640 +41490 7640 +27261 7638 +24200 7636 +37025 7629 +30608 7627 +46616 7625 +31851 7622 +36899 7622 +25211 7621 +35319 7621 +24640 7619 +27273 7616 +37767 7616 +26201 7613 +36077 7613 +43642 7610 +45681 7608 +29888 7604 +40767 7603 +35574 7602 +34695 7601 +15263 7601 +49930 7600 +17373 7599 +41127 7594 +32875 7594 +37957 7593 +48027 7589 +33574 7585 +41628 7583 +19959 7582 +19256 7580 +25039 7576 +11730 7576 +167 7576 +48494 7574 +30394 7567 +2532 7564 +33249 7562 +37548 7561 +37189 7558 +36868 7554 +38288 7553 +34866 7551 +48836 7547 +37362 7542 +50047 7542 +25831 7540 +49632 7540 +47663 7537 +25354 7537 +15023 7535 +34270 7535 +49279 7534 +38767 7533 +48570 7527 +39103 7526 +29676 7523 +42940 7522 +45746 7519 +41274 7517 +49454 7513 +49942 7509 +46075 7509 +41025 7508 +27419 7507 +24328 7507 +32602 7506 +48549 7505 +11157 7501 +14566 7497 +16813 7496 +37384 7492 +26425 7491 +41681 7490 +38002 7487 +39087 7485 +36008 7484 +8767 7479 +10268 7479 +6879 7477 +50207 7476 +39427 7473 +48561 7472 +44644 7470 +26652 7468 +27421 7467 +20998 7466 +12851 7465 +38391 7464 +49121 7464 +37047 7459 +35244 7454 +36840 7450 +38231 7449 +4857 7443 +25977 7441 +31129 7436 +42455 7436 +40548 7434 +39225 7433 +30328 7432 +12951 7431 +21004 7429 +50126 7425 +30659 7424 +49580 7424 +38415 7423 +36295 7422 +31508 7421 +29668 7414 +11935 7407 +15205 7407 +37065 7406 +26003 7404 +39230 7403 +30281 7401 +41356 7400 +38837 7396 +36500 7394 +16229 7389 +34204 7389 +41206 7388 +38141 7386 +10479 7385 +44920 7382 +34554 7381 +46126 7375 +49970 7373 +30306 7373 +42670 7373 +40842 7370 +26696 7370 +2595 7366 +41341 7365 +44742 7362 +47213 7359 +38143 7359 +45509 7354 +13773 7354 +18806 7352 +6976 7352 +41570 7349 +35255 7348 +32441 7347 +30240 7344 +44374 7340 +39197 7334 +46604 7333 +33161 7330 +12631 7327 +35590 7321 +45282 7319 +34586 7313 +21620 7313 +18408 7311 +21357 7310 +34953 7306 +18161 7306 +49686 7300 +14624 7297 +33383 7296 +15330 7295 +38482 7291 +31272 7290 +34926 7287 +35570 7287 +23958 7286 +44563 7284 +26953 7280 +37365 7279 +49869 7277 +31493 7275 +49492 7273 +24042 7273 +48799 7273 +43892 7267 +48451 7265 +37834 7261 +42296 7260 +28732 7259 +17563 7259 +32771 7256 +7641 7254 +48272 7254 +24259 7253 +7426 7252 +15582 7250 +28923 7247 +37404 7246 +29294 7245 +47133 7244 +40506 7242 +32949 7236 +45898 7234 +41079 7233 +20086 7230 +47516 7229 +36284 7228 +45627 7225 +34048 7224 +13498 7223 +40368 7219 +5881 7218 +37504 7216 +44724 7216 +29006 7215 +46728 7213 +42006 7213 +49572 7209 +49712 7206 +23843 7202 +49008 7201 +26747 7199 +42771 7198 +21215 7195 +41931 7194 +19573 7193 +32105 7192 +49379 7189 +20468 7188 +44679 7182 +37821 7176 +40097 7176 +34947 7176 +28039 7175 +44292 7173 +11861 7172 +25294 7171 +7790 7168 +20740 7167 +25106 7161 +35119 7159 +39156 7156 +39478 7155 +38020 7154 +44812 7152 +42466 7148 +37697 7147 +36648 7145 +46251 7145 +24396 7145 +39861 7142 +27447 7141 +18065 7136 +1054 7134 +25306 7131 +44360 7130 +9833 7126 +48593 7125 +31400 7124 +47937 7123 +48582 7122 +25919 7121 +37139 7120 +37740 7119 +20438 7118 +3680 7114 +41208 7112 +33433 7105 +43894 7104 +28721 7102 +30542 7101 +44387 7094 +16950 7091 +46933 7089 +47585 7088 +43404 7087 +36772 7086 +27477 7084 +33010 7083 +32645 7079 +8243 7078 +41488 7077 +24583 7074 +8700 7067 +47294 7066 +42317 7064 +43827 7063 +30769 7063 +50025 7059 +36822 7058 +48631 7054 +17664 7051 +39881 7050 +49144 7049 +48485 7040 +15742 7040 +19746 7039 +29205 7039 +36883 7037 +45853 7034 +17679 7032 +31755 7030 +36881 7027 +32797 7025 +43878 7022 +42550 7022 +37945 7021 +27958 7021 +26629 7018 +41499 7015 +45166 7003 +45336 7001 +39839 6998 +9770 6994 +44054 6993 +41298 6992 +23493 6991 +31968 6990 +44227 6987 +19312 6987 +8191 6983 +43320 6982 +34379 6981 +49516 6981 +36803 6976 +32550 6976 +44759 6975 +27536 6973 +40139 6971 +36526 6970 +31919 6969 +23228 6968 +49095 6968 +40988 6967 +33357 6966 +22621 6966 +43387 6964 +29492 6964 +18994 6963 +46546 6963 +17733 6963 +43441 6960 +49828 6955 +10773 6955 +2315 6955 +46975 6953 +11477 6951 +46683 6950 +42979 6950 +34455 6948 +36834 6946 +7559 6944 +43456 6943 +49138 6942 +24834 6941 +31590 6941 +19615 6939 +31122 6939 +39902 6938 +46102 6937 +44276 6937 +17043 6937 +26964 6935 +36754 6932 +40331 6931 +39200 6930 +46002 6930 +26062 6926 +41744 6924 +14472 6918 +15380 6916 +41633 6916 +37269 6916 +44991 6916 +18202 6912 +44159 6911 +20026 6910 +31419 6910 +49370 6910 +48124 6909 +29307 6908 +24342 6901 +29223 6893 +29221 6891 +33786 6883 +39458 6882 +50188 6882 +38241 6880 +22585 6876 +3836 6870 +29017 6868 +49562 6865 +28018 6862 +41748 6859 +33349 6859 +49802 6859 +33028 6853 +5970 6851 +40853 6847 +48791 6844 +49709 6842 +14629 6836 +13078 6836 +36607 6834 +38027 6832 +50178 6831 +49176 6830 +43437 6830 +42696 6829 +37674 6827 +31053 6814 +42171 6814 +34242 6813 +49670 6812 +30603 6812 +40478 6812 +36843 6811 +22927 6810 +49422 6806 +43740 6805 +12962 6805 +16776 6802 +18453 6795 +40107 6791 +40397 6791 +22955 6788 +47172 6788 +28745 6781 +38035 6781 +13576 6781 +41862 6779 +12629 6779 +46120 6776 +33684 6775 +10387 6773 +19593 6769 +31815 6763 +46025 6760 +33077 6759 +18615 6759 +17505 6756 +46022 6753 +48567 6753 +14950 6753 +36122 6748 +17922 6745 +18743 6744 +28298 6743 +40091 6743 +32916 6740 +43828 6736 +20684 6732 +16444 6731 +26257 6731 +28100 6730 +8077 6728 +48895 6728 +10166 6727 +13263 6726 +41008 6725 +25947 6725 +35608 6724 +46477 6723 +32184 6722 +36894 6719 +46421 6719 +44591 6717 +25385 6715 +40981 6714 +31442 6712 +48084 6712 +43517 6709 +38899 6709 +43572 6709 +24361 6708 +14126 6708 +30642 6708 +31798 6706 +43614 6705 +21206 6705 +26447 6703 +16951 6700 +47781 6700 +24835 6696 +47117 6693 +40360 6691 +43612 6691 +49790 6689 +20613 6683 +50016 6681 +35645 6676 +41901 6675 +32665 6675 +35750 6670 +32851 6669 +46369 6669 +33485 6669 +38998 6666 +40901 6666 +37436 6663 +38688 6663 +23935 6660 +46077 6658 +19576 6654 +33650 6652 +12904 6652 +15732 6646 +2333 6644 +42310 6644 +45038 6643 +28019 6642 +26152 6638 +35802 6636 +44501 6635 +19209 6633 +16764 6632 +40044 6631 +27619 6630 +41242 6629 +19578 6629 +45685 6627 +15145 6622 +23767 6617 +10623 6615 +37870 6614 +10677 6613 +12095 6612 +31116 6611 +39462 6610 +36733 6608 +43777 6608 +10699 6607 +44819 6602 +4812 6600 +13556 6600 +28101 6595 +42503 6595 +139 6594 +37150 6591 +40920 6590 +35835 6585 +47537 6584 +46610 6583 +31945 6582 +29942 6579 +26599 6578 +23806 6578 +16594 6576 +35630 6574 +45271 6573 +49884 6573 +2358 6572 +18400 6571 +44798 6568 +42481 6559 +42685 6557 +50015 6552 +7113 6550 +31857 6550 +32138 6548 +29347 6548 +48765 6547 +46076 6544 +32429 6542 +49728 6540 +33080 6540 +45869 6534 +17649 6533 +42948 6532 +26663 6526 +48529 6526 +35220 6525 +24388 6525 +16873 6521 +47206 6518 +15973 6517 +47918 6517 +25200 6516 +42135 6516 +46613 6516 +10118 6513 +20602 6509 +10126 6508 +49811 6508 +15341 6508 +25423 6507 +41202 6507 +38095 6500 +28465 6498 +48103 6497 +25163 6496 +46374 6496 +42485 6493 +31358 6492 +45119 6492 +29918 6491 +27856 6489 +40322 6489 +13942 6488 +40914 6488 +25636 6486 +19314 6486 +20689 6485 +13194 6482 +33373 6480 +27024 6480 +35742 6478 +7419 6477 +48852 6477 +49918 6473 +23520 6471 +31436 6468 +6668 6467 +27195 6464 +31809 6462 +16943 6454 +5515 6450 +39841 6448 +47050 6448 +36533 6446 +40059 6446 +37835 6443 +33314 6440 +48954 6439 +41251 6438 +47705 6437 +45337 6437 +14302 6436 +7107 6434 +28820 6430 +40481 6427 +2979 6425 +20863 6424 +44478 6423 +26272 6423 +43975 6421 +10786 6421 +49028 6420 +43331 6419 +44910 6418 +16947 6418 +32113 6417 +24469 6417 +17302 6413 +39788 6411 +47524 6410 +8419 6406 +15583 6402 +48413 6401 +44935 6400 +14279 6399 +46311 6398 +40864 6397 +27956 6396 +41767 6396 +46228 6394 +14823 6393 +25941 6390 +30520 6390 +47116 6389 +44213 6384 +35726 6382 +33333 6381 +31627 6369 +25486 6369 +46850 6354 +37340 6353 +23903 6350 +41591 6349 +40387 6347 +42440 6347 +12453 6343 +36604 6342 +15256 6342 +8892 6339 +40230 6339 +49110 6338 +38835 6337 +8371 6334 +4927 6331 +15103 6329 +49096 6327 +46780 6326 +27634 6324 +49126 6322 +3480 6319 +18853 6318 +46315 6318 +20168 6316 +48493 6316 +38452 6310 +7798 6309 +34000 6309 +35746 6309 +38936 6309 +43020 6308 +33283 6308 +36515 6306 +34674 6305 +49770 6304 +45234 6299 +32485 6298 +6432 6296 +47150 6296 +29027 6295 +28663 6293 +40705 6293 +17203 6292 +48379 6289 +15864 6281 +33410 6279 +47065 6274 +42887 6273 +46253 6271 +29880 6270 +6077 6269 +36133 6259 +41614 6254 +26129 6253 +42797 6249 +40339 6249 +48545 6248 +14061 6247 +30014 6246 +33716 6245 +27745 6244 +42243 6239 +42501 6236 +17739 6232 +13968 6232 +33353 6229 +35283 6229 +27608 6228 +42721 6225 +21053 6213 +32223 6211 +48790 6210 +31915 6208 +45565 6206 +46634 6204 +28698 6204 +21117 6201 +7727 6200 +29356 6199 +13499 6197 +34617 6194 +20557 6192 +45783 6188 +48613 6187 +21496 6186 +28066 6185 +44758 6183 +43749 6182 +39431 6181 +45582 6180 +47852 6177 +33276 6175 +33480 6172 +47242 6165 +29291 6161 +35790 6160 +22961 6160 +8273 6155 +46521 6153 +39707 6151 +43099 6145 +21878 6143 +17224 6141 +31688 6139 +43714 6139 +32555 6131 +47285 6129 +28410 6128 +45977 6128 +25302 6127 +34556 6125 +21106 6123 +1457 6123 +15324 6122 +43641 6122 +8269 6119 +17258 6118 +44336 6117 +45841 6114 +14399 6112 +49045 6108 +40825 6106 +7551 6106 +21330 6102 +24881 6101 +45353 6100 +36046 6096 +34104 6094 +29610 6093 +48468 6092 +22079 6089 +26687 6089 +20301 6086 +16136 6084 +31932 6083 +15118 6079 +48659 6078 +31614 6075 +28497 6075 +49999 6072 +17663 6069 +49563 6068 +48814 6067 +45352 6066 +49780 6065 +28735 6065 +45677 6065 +35381 6065 +34539 6062 +46127 6060 +35460 6054 +50237 6048 +31801 6048 +28813 6047 +28873 6047 +20257 6044 +43036 6041 +26504 6040 +32394 6037 +26522 6033 +49621 6033 +41470 6033 +6624 6031 +23966 6030 +17606 6030 +33365 6029 +45751 6029 +16681 6027 +45833 6026 +13253 6026 +3547 6022 +22360 6015 +46428 6015 +30676 6014 +38727 6014 +26563 6013 +46700 6012 +42784 6011 +25961 6010 +13007 6009 +16906 6008 +33170 6005 +6828 6003 +10310 6002 +37235 6001 +41076 6001 +33299 6000 +32072 5999 +42788 5998 +47625 5993 +32103 5993 +48749 5992 +35955 5991 +36787 5990 +43904 5986 +46840 5981 +16604 5979 +49515 5978 +146 5978 +45529 5976 +6384 5973 +42814 5970 +40640 5969 +32323 5968 +48433 5967 +36346 5966 +46786 5965 +47190 5965 +45901 5963 +35503 5960 +30710 5957 +34280 5952 +46231 5952 +39787 5949 +37656 5946 +38344 5945 +31559 5942 +41394 5941 +25199 5941 +12614 5941 +27520 5937 +16547 5936 +43844 5935 +35302 5932 +43108 5932 +43392 5932 +49277 5931 +31695 5931 +50175 5930 +39792 5926 +38618 5921 +28452 5921 +29800 5921 +46080 5919 +47658 5919 +35843 5919 +39456 5917 +46785 5913 +43368 5913 +46297 5911 +30337 5911 +30894 5908 +47427 5908 +28851 5907 +40890 5903 +15437 5903 +47288 5901 +38103 5900 +43990 5893 +30956 5892 +8864 5892 +18716 5891 +15333 5885 +39286 5884 +14182 5883 +50051 5879 +39919 5878 +42881 5874 +46294 5871 +50144 5870 +37393 5864 +38114 5860 +46144 5857 +33494 5854 +40692 5852 +33476 5849 +44278 5846 +37736 5844 +37573 5835 +23028 5835 +24546 5834 +45430 5834 +33380 5833 +27038 5832 +34350 5825 +41581 5823 +48867 5822 +5477 5818 +14490 5815 +45592 5815 +31502 5814 +44968 5811 +38183 5808 +32706 5808 +42254 5807 +49946 5806 +36732 5806 +28482 5805 +23054 5803 +20331 5799 +26517 5795 +45854 5793 +36343 5792 +20659 5791 +29146 5788 +42133 5781 +39429 5778 +30486 5776 +47255 5775 +44199 5773 +46538 5773 +39938 5772 +26801 5771 +49485 5768 +43198 5766 +48789 5765 +37725 5763 +18114 5762 +24954 5762 +39888 5760 +27097 5756 +26604 5753 +8911 5748 +48034 5745 +14369 5744 +13173 5744 +31859 5743 +47173 5739 +30543 5737 +24321 5735 +24840 5735 +23869 5735 +8552 5733 +33245 5732 +10541 5730 +46599 5729 +47471 5729 +36727 5726 +35628 5723 +5218 5721 +28644 5720 +45608 5714 +150 5711 +46219 5710 +47522 5708 +40632 5705 +30854 5700 +46383 5698 +41358 5696 +29515 5693 +48941 5693 +27553 5692 +27072 5684 +42098 5683 +30003 5682 +27915 5682 +12730 5675 +41057 5675 +32345 5672 +26495 5672 +42419 5669 +44566 5669 +48902 5665 +37672 5658 +37942 5658 +24243 5657 +48536 5654 +33853 5652 +27148 5649 +34227 5647 +32978 5645 +17844 5643 +40874 5642 +37865 5642 +37960 5637 +23252 5636 +12895 5636 +22027 5635 +31391 5631 +44270 5628 +18709 5628 +36341 5624 +49687 5624 +39908 5623 +49691 5619 +44596 5615 +9933 5614 +21665 5613 +41667 5613 +37770 5612 +29121 5608 +40653 5605 +20946 5602 +24912 5601 +42014 5596 +18439 5596 +21789 5592 +49494 5588 +49406 5588 +35977 5585 +36064 5580 +8572 5578 +45912 5578 +36147 5575 +30474 5573 +43532 5573 +49081 5572 +45198 5572 +41915 5568 +35320 5566 +4026 5566 +26848 5565 +22915 5564 +46748 5562 +47834 5560 +37418 5560 +42975 5559 +34198 5558 +40925 5558 +47730 5555 +48838 5555 +31205 5555 +9898 5552 +44319 5551 +26621 5550 +31396 5550 +28561 5544 +35738 5539 +29513 5538 +32833 5537 +41317 5536 +47454 5530 +18203 5530 +26416 5529 +13236 5527 +27389 5527 +47772 5527 +25753 5523 +19380 5521 +13074 5520 +39575 5519 +36027 5517 +33815 5516 +26109 5515 +36198 5513 +46710 5511 +16743 5510 +31626 5508 +50177 5508 +25554 5506 +24524 5502 +43925 5500 +46727 5499 +30698 5492 +31971 5491 +13305 5490 +1824 5490 +28861 5489 +42182 5483 +15239 5483 +36347 5483 +17576 5482 +44709 5479 +13930 5478 +34042 5475 +28914 5475 +10789 5474 +47419 5471 +42809 5470 +31441 5469 +34634 5461 +4945 5454 +50248 5452 +8360 5450 +38113 5446 +24333 5446 +47357 5445 +50092 5443 +23010 5440 +47402 5436 +27694 5433 +27193 5433 +29320 5432 +30892 5424 +32635 5424 +33582 5421 +27313 5420 +40164 5418 +47960 5415 +2515 5412 +46639 5411 +46510 5406 +23599 5406 +26287 5405 +35676 5403 +26858 5403 +25901 5403 +46183 5402 +41622 5396 +21149 5391 +13890 5388 +38270 5386 +10894 5385 +35575 5382 +46566 5379 +28956 5379 +3204 5377 +42031 5374 +27031 5374 +20180 5371 +46085 5368 +35393 5368 +8293 5368 +49357 5367 +49239 5367 +43406 5361 +38615 5361 +13149 5359 +40710 5358 +45201 5355 +46452 5353 +30934 5353 +43085 5352 +23416 5351 +30493 5347 +33634 5345 +42036 5343 +22468 5341 +31253 5340 +36204 5339 +14387 5339 +24426 5328 +45530 5326 +44831 5324 +42114 5324 +44225 5322 +48562 5320 +30156 5320 +40759 5319 +49057 5318 +42895 5316 +11921 5308 +29672 5307 +48652 5306 +44531 5306 +34924 5306 +9602 5304 +44604 5303 +9924 5301 +16478 5301 +9350 5300 +40546 5298 +45417 5293 +33092 5292 +50125 5292 +46360 5289 +4358 5288 +25624 5288 +49574 5285 +30230 5282 +28614 5280 +40627 5277 +28122 5274 +28268 5272 +7131 5270 +46897 5270 +38032 5269 +29302 5268 +44515 5267 +23391 5265 +35780 5263 +47783 5263 +44156 5262 +14168 5262 +43651 5260 +36428 5260 +2578 5254 +29945 5251 +27830 5246 +24378 5244 +33855 5244 +25454 5237 +39007 5235 +42547 5232 +45133 5230 +41995 5230 +10615 5229 +45647 5227 +23969 5223 +13233 5221 +46455 5221 +15036 5220 +26357 5219 +34769 5215 +44859 5214 +39407 5210 +36632 5210 +47911 5209 +30345 5209 +32465 5207 +26870 5203 +26828 5198 +35895 5196 +31711 5189 +37855 5187 +47892 5187 +46177 5186 +34442 5182 +43866 5179 +33898 5173 +45300 5166 +50157 5165 +20837 5165 +38948 5164 +3239 5161 +29934 5159 +17495 5158 +48657 5151 +40232 5150 +21656 5150 +23793 5148 +24015 5148 +18925 5145 +30581 5144 +45941 5139 +45285 5137 +33762 5136 +31898 5135 +34877 5134 +49742 5134 +44727 5131 +35781 5131 +24516 5130 +38518 5127 +46911 5120 +24364 5117 +42507 5115 +44296 5114 +29660 5112 +11924 5111 +12485 5111 +11788 5107 +49702 5105 +44562 5104 +45140 5102 +27925 5098 +31659 5097 +7203 5095 +30218 5094 +19016 5093 +39934 5090 +2500 5088 +19763 5086 +17056 5084 +42588 5081 +16107 5081 +40577 5080 +5320 5080 +22077 5078 +33172 5073 +21413 5071 +34371 5071 +44683 5064 +48473 5061 +20231 5059 +9881 5059 +46367 5058 +23314 5057 +23758 5057 +25444 5055 +36487 5054 +39668 5053 +49140 5051 +43054 5049 +15924 5048 +14978 5048 +46384 5048 +12859 5046 +46037 5045 +35913 5044 +49451 5039 +40009 5037 +42042 5035 +36125 5033 +15782 5032 +45547 5031 +27586 5030 +44451 5029 +28653 5024 +46329 5022 +48278 5022 +44086 5022 +27448 5020 +9315 5020 +26802 5019 +18559 5018 +27282 5017 +1880 5016 +39219 5012 +6349 5012 +32091 5011 +44330 5011 +38799 5007 +19160 5006 +15966 5006 +16372 5005 +8029 5002 +16358 5002 +21351 5000 +24515 5000 +46808 4995 +18097 4995 +42901 4993 +31127 4993 +16671 4992 +28839 4990 +26365 4989 +17964 4980 +21800 4977 +49444 4977 +41908 4974 +26737 4973 +43869 4973 +47763 4972 +6560 4971 +15848 4971 +40809 4967 +35346 4966 +19722 4963 +20701 4963 +43260 4962 +19563 4961 +48267 4957 +33825 4956 +45368 4952 +23838 4947 +44032 4945 +37124 4945 +13371 4943 +41296 4941 +39551 4941 +32096 4940 +43661 4939 +26294 4937 +49912 4936 +40480 4935 +26163 4935 +14605 4934 +38715 4933 +14749 4932 +38197 4930 +33054 4929 +48477 4928 +40083 4926 +21906 4926 +31611 4925 +47639 4924 +22880 4923 +22369 4920 +41848 4919 +47020 4919 +33129 4918 +11239 4916 +28935 4914 +25757 4914 +38566 4909 +17595 4908 +48738 4908 +33132 4906 +48948 4904 +43810 4899 +39455 4896 +30589 4895 +10791 4891 +41387 4891 +28258 4888 +28508 4885 +16580 4884 +20891 4884 +22646 4883 +22400 4882 +33734 4881 +35170 4880 +14910 4878 +30016 4876 +17097 4874 +17861 4871 +47466 4869 +29352 4869 +11995 4868 +37120 4866 +3784 4861 +21465 4860 +43418 4857 +46514 4855 +24455 4849 +45101 4845 +23686 4845 +25391 4844 +45403 4843 +21231 4842 +39650 4840 +43762 4835 +39559 4834 +45175 4834 +45243 4832 +31811 4832 +48191 4832 +38821 4827 +6222 4824 +47333 4817 +19109 4816 +46047 4814 +46615 4810 +40281 4810 +13442 4805 +27772 4801 +43660 4798 +30150 4796 +31382 4792 +33248 4790 +41411 4789 +32966 4772 +47025 4769 +17491 4769 +36982 4766 +44231 4764 +35236 4764 +46111 4757 +27566 4749 +15865 4747 +20369 4746 +33744 4743 +29639 4741 +31425 4739 +30001 4736 +38521 4736 +25101 4734 +42995 4733 +46602 4733 +49094 4730 +25169 4730 +33196 4728 +43167 4726 +44120 4725 +42384 4723 +21962 4717 +17143 4715 +8285 4712 +41424 4712 +44239 4711 +41233 4710 +10267 4710 +41446 4706 +24201 4703 +40133 4701 +16040 4699 +14631 4699 +42553 4698 +5192 4698 +1189 4696 +32608 4696 +6009 4695 +4368 4694 +45299 4694 +39555 4694 +18932 4693 +36476 4690 +40347 4690 +19031 4690 +33234 4689 +42223 4687 +41097 4682 +43118 4679 +40850 4679 +9169 4676 +40651 4676 +33518 4672 +46995 4665 +39622 4665 +49475 4665 +30071 4664 +48301 4663 +43377 4661 +43628 4659 +13361 4659 +40258 4652 +43258 4650 +6354 4648 +37712 4643 +39833 4643 +4183 4642 +28228 4637 +36504 4625 +19801 4622 +43471 4621 +20885 4619 +46659 4617 +42711 4616 +43194 4615 +48056 4613 +49438 4612 +45729 4612 +32101 4609 +33766 4608 +22552 4600 +26572 4600 +40402 4599 +20593 4596 +45914 4596 +37214 4596 +34749 4596 +24908 4594 +40799 4591 +23646 4587 +10353 4587 +20574 4587 +35779 4584 +23814 4582 +16254 4577 +36471 4576 +49604 4576 +41205 4576 +22015 4569 +44190 4568 +23821 4562 +32356 4561 +21109 4559 +20898 4559 +48538 4557 +42477 4556 +46155 4554 +29868 4552 +35442 4551 +48131 4549 +34562 4549 +41359 4548 +49788 4545 +33152 4545 +47109 4545 +16497 4544 +50200 4541 +10895 4538 +34149 4537 +19778 4535 +38365 4533 +48860 4532 +41966 4527 +24179 4525 +29507 4523 +33789 4522 +30736 4518 +33951 4515 +26865 4514 +25249 4512 +34293 4511 +19212 4511 +46021 4511 +31718 4510 +20688 4508 +27286 4506 +10103 4504 +40668 4500 +21575 4496 +44425 4496 +30388 4495 +15944 4494 +46824 4493 +33772 4488 +48682 4488 +19871 4486 +48885 4485 +38941 4483 +14116 4480 +22866 4478 +15414 4474 +9523 4473 +36942 4473 +45035 4467 +38110 4466 +35952 4462 +14224 4460 +46722 4459 +44850 4457 +3479 4456 +37273 4454 +19724 4453 +27955 4452 +35414 4450 +37155 4448 +43815 4447 +25461 4442 +46955 4442 +19359 4441 +33406 4439 +39347 4438 +32023 4436 +44538 4432 +7453 4430 +24517 4429 +31871 4427 +33661 4427 +34577 4425 +6038 4424 +48142 4424 +49646 4420 +30535 4418 +7098 4416 +34496 4414 +11306 4414 +46343 4413 +36571 4413 +36320 4410 +49511 4405 +3462 4404 +28266 4402 +29428 4401 +35632 4399 +17440 4399 +18847 4398 +42822 4397 +44642 4394 +33717 4392 +20360 4391 +43008 4391 +24214 4390 +36084 4386 +9625 4383 +50118 4383 +43597 4375 +49217 4375 +29323 4373 +38644 4373 +37773 4373 +47598 4368 +37012 4366 +44191 4366 +38148 4365 +36387 4363 +3749 4362 +17474 4358 +41185 4356 +47473 4347 +35741 4345 +36510 4334 +44166 4331 +17236 4329 +44409 4328 +15038 4327 +23795 4324 +38215 4320 +166 4320 +48003 4312 +36613 4307 +28961 4305 +16368 4304 +40247 4303 +38399 4300 +28712 4296 +50204 4290 +31680 4288 +37188 4287 +26545 4287 +33274 4285 +42432 4285 +45675 4282 +33678 4281 +32086 4278 +20015 4278 +23002 4278 +46217 4273 +39945 4272 +38564 4271 +13994 4270 +35813 4268 +37815 4268 +14459 4267 +38274 4267 +20129 4266 +44613 4262 +28376 4261 +11093 4255 +29720 4253 +48875 4252 +42359 4250 +28074 4242 +49347 4241 +37500 4240 +50109 4233 +18919 4232 +47935 4230 +30540 4229 +8193 4226 +24586 4223 +37766 4218 +23735 4218 +42903 4217 +46775 4217 +36117 4216 +29736 4214 +46733 4213 +39582 4209 +38043 4209 +7969 4208 +33968 4204 +18074 4203 +31895 4202 +32603 4199 +46320 4199 +14621 4196 +33062 4195 +14844 4194 +36911 4191 +3821 4190 +7678 4187 +27358 4187 +27050 4184 +3880 4183 +34795 4181 +38891 4180 +49751 4180 +38499 4172 +5435 4165 +4770 4165 +20826 4164 +8809 4163 +43316 4158 +49749 4155 +49531 4153 +9462 4149 +36781 4149 +32419 4147 +45077 4143 +43790 4135 +10168 4132 +34303 4132 +37351 4130 +28119 4129 +37148 4129 +11709 4128 +9152 4127 +42059 4126 +4256 4126 +43721 4126 +43270 4125 +38038 4124 +26639 4122 +32065 4121 +32495 4121 +49222 4120 +5754 4120 +30221 4120 +37920 4117 +28252 4117 +28474 4117 +45988 4115 +33321 4108 +41020 4108 +28333 4101 +25538 4097 +29453 4092 +44808 4089 +48608 4083 +42197 4083 +22773 4079 +10558 4077 +43654 4076 +29064 4075 +39223 4067 +41982 4067 +48082 4063 +34457 4061 +43094 4059 +38641 4059 +37344 4059 +30998 4058 +31069 4054 +9547 4052 +39275 4051 +25267 4049 +43987 4048 +48625 4045 +21945 4042 +26111 4042 +9783 4039 +10777 4037 +36259 4031 +28147 4031 +40203 4026 +41940 4026 +8594 4023 +50149 4023 +30564 4020 +45367 4020 +30950 4020 +22885 4018 +37655 4017 +38790 4017 +49041 4011 +28769 4011 +46672 4010 +44707 4010 +43525 4003 +44898 4001 +25576 4000 +36558 3996 +1832 3995 +47252 3991 +31775 3988 +49836 3987 +15005 3984 +24036 3984 +43384 3983 +3658 3983 +28428 3981 +15575 3978 +34635 3974 +45242 3974 +131 3972 +41781 3971 +36291 3969 +43233 3968 +33606 3968 +32032 3962 +11183 3962 +47924 3961 +28373 3959 +34703 3956 +40637 3956 +38815 3955 +37250 3953 +32252 3953 +42166 3952 +26317 3951 +19083 3950 +33236 3947 +49061 3946 +48558 3946 +32782 3945 +38640 3944 +19076 3943 +47823 3942 +34764 3938 +36598 3936 +24564 3934 +19275 3934 +22719 3934 +4412 3932 +32634 3931 +36770 3930 +46430 3929 +23772 3927 +34168 3927 +15179 3925 +49899 3924 +26396 3922 +46697 3922 +28789 3919 +1253 3918 +35767 3913 +45205 3912 +35967 3911 +41560 3910 +11452 3910 +44592 3909 +39266 3908 +24941 3907 +22522 3905 +16844 3905 +49128 3904 +48447 3902 +33501 3899 +30952 3899 +43895 3892 +26214 3890 +29230 3888 +7304 3887 +30326 3881 +12103 3881 +45027 3878 +37022 3877 +41671 3877 +39069 3875 +25255 3869 +46100 3868 +38807 3866 +30672 3863 +42592 3858 +47113 3856 +10263 3855 +22683 3855 +42009 3854 +49501 3854 +18401 3853 +41609 3850 +47638 3849 +26180 3847 +6353 3846 +39126 3846 +47365 3842 +44946 3839 +23803 3837 +8816 3836 +38491 3834 +41741 3824 +38855 3824 +18498 3824 +44028 3823 +27933 3819 +15322 3818 +40211 3816 +36616 3816 +45835 3815 +40034 3815 +32147 3814 +33800 3813 +28766 3808 +32169 3808 +25637 3808 +34148 3808 +2783 3804 +47798 3803 +42539 3801 +28338 3801 +31909 3799 +41482 3798 +19622 3798 +2827 3797 +18270 3793 +38874 3793 +45254 3792 +3273 3789 +41026 3789 +28404 3788 +37713 3787 +28611 3787 +38417 3783 +19776 3780 +10024 3778 +36463 3777 +5619 3776 +48754 3775 +32846 3774 +33486 3769 +26344 3769 +8912 3768 +33176 3767 +36850 3766 +45700 3762 +46220 3762 +48330 3758 +41989 3758 +44458 3757 +22170 3755 +41219 3754 +49926 3754 +29486 3749 +49151 3746 +17518 3746 +31929 3746 +44223 3744 +45684 3742 +30871 3741 +49908 3741 +29870 3740 +33796 3739 +26367 3736 +12208 3731 +36258 3730 +47279 3726 +19449 3720 +49933 3718 +23241 3713 +28622 3712 +10367 3711 +35395 3711 +35178 3708 +46799 3708 +10929 3708 +38637 3705 +31696 3702 +21849 3701 +30609 3698 +47071 3697 +48181 3696 +42473 3696 +41104 3694 +17256 3692 +18610 3691 +13757 3690 +49256 3687 +41552 3684 +15466 3678 +12351 3676 +46182 3675 +49560 3674 +37403 3673 +22929 3670 +43087 3664 +35791 3664 +37146 3661 +8935 3660 +34999 3660 +16301 3659 +43782 3657 +39967 3657 +18167 3656 +30265 3655 +18172 3655 +46676 3655 +31954 3651 +39345 3650 +37647 3650 +38663 3646 +32685 3642 +41162 3640 +46358 3633 +41000 3632 +23877 3631 +36674 3631 +42195 3624 +44747 3622 +48547 3622 +49849 3620 +39825 3619 +26037 3618 +26440 3616 +22810 3615 +37510 3614 +28883 3613 +43582 3611 +19754 3611 +23114 3607 +38601 3607 +45037 3602 +37017 3602 +33547 3600 +44435 3597 +47718 3593 +29046 3591 +47815 3591 +45798 3591 +32830 3590 +12145 3589 +28222 3589 +31410 3589 +39849 3587 +38296 3587 +37477 3587 +13751 3586 +40517 3586 +36977 3585 +10046 3583 +26060 3578 +48310 3574 +47348 3574 +42180 3574 +26198 3568 +27614 3567 +44318 3566 +33439 3564 +42840 3563 +41392 3562 +33539 3561 +33714 3560 +19488 3560 +50145 3558 +18717 3557 +24335 3556 +43907 3554 +17386 3554 +35279 3553 +40106 3546 +24741 3545 +36124 3545 +45890 3545 +35620 3544 +23826 3544 +44706 3541 +29085 3541 +47531 3538 +49388 3537 +34445 3530 +19618 3527 +28312 3525 +38481 3523 +42652 3522 +5649 3522 +2660 3518 +41512 3514 +36623 3510 +31417 3510 +17125 3510 +36436 3505 +26656 3504 +25205 3500 +33087 3498 +41055 3498 +46701 3497 +19557 3497 +14815 3495 +32070 3495 +27908 3490 +18337 3490 +4707 3490 +20336 3487 +26638 3487 +44464 3485 +36542 3482 +7555 3476 +43271 3474 +15109 3473 +48319 3470 +37996 3469 +39815 3468 +40025 3467 +39590 3466 +46326 3461 +39531 3459 +23513 3459 +37187 3458 +49139 3457 +39470 3456 +11430 3455 +3390 3454 +49794 3453 +45809 3452 +49026 3452 +5196 3451 +33637 3450 +48686 3449 +40832 3449 +10493 3448 +21677 3448 +26193 3448 +33931 3447 +43319 3447 +35770 3442 +46912 3442 +30331 3437 +20275 3437 +36851 3436 +35347 3436 +47026 3435 +46893 3435 +44267 3433 +29435 3432 +34248 3430 +46143 3430 +37077 3427 +33293 3425 +47322 3422 +47361 3419 +43158 3416 +43160 3413 +37481 3410 +36037 3409 +41090 3409 +41506 3408 +43477 3407 +40709 3406 +38841 3404 +35974 3403 +22492 3403 +18249 3400 +30448 3400 +10587 3398 +47209 3397 +7333 3396 +36028 3396 +26238 3396 +27002 3393 +49155 3393 +49601 3393 +31743 3392 +15340 3391 +5480 3388 +18589 3387 +33784 3384 +42165 3384 +35792 3383 +43212 3383 +32815 3381 +23380 3379 +37010 3377 +44942 3377 +24844 3376 +19364 3373 +11151 3370 +22630 3369 +47227 3368 +33707 3366 +26693 3366 +35202 3366 +10545 3365 +1518 3363 +44754 3361 +26158 3359 +34364 3357 +23480 3357 +19293 3355 +14257 3354 +38321 3347 +21251 3344 +44807 3341 +25064 3340 +50093 3339 +41707 3336 +48289 3329 +30818 3329 +7990 3323 +4228 3322 +27849 3320 +21744 3319 +43806 3316 +11127 3315 +22252 3313 +32341 3312 +42082 3309 +14012 3308 +44954 3304 +34258 3301 +18872 3299 +43216 3296 +44154 3295 +40778 3295 +26501 3294 +28012 3294 +44334 3294 +49945 3293 +44238 3293 +38460 3293 +21075 3291 +20621 3289 +44836 3288 +33409 3285 +49036 3284 +34805 3282 +39658 3281 +48522 3281 +35760 3275 +18604 3274 +33420 3274 +35265 3273 +45755 3267 +33447 3263 +35102 3258 +38040 3253 +33792 3252 +27983 3251 +31073 3250 +46257 3247 +18245 3245 +15759 3243 +40806 3237 +35500 3235 +3322 3234 +31254 3229 +20480 3229 +40781 3228 +41808 3227 +20255 3224 +35420 3223 +35147 3223 +8370 3220 +49893 3212 +41709 3209 +17643 3208 +39070 3207 +48402 3207 +33120 3207 +39232 3207 +20184 3206 +22873 3206 +13600 3203 +40063 3201 +45872 3200 +47118 3200 +48933 3196 +23111 3194 +35058 3190 +35488 3184 +28938 3184 +42976 3183 +19237 3182 +36356 3178 +49608 3177 +26104 3175 +46932 3174 +43303 3174 +49850 3174 +18865 3173 +43979 3173 +44090 3171 +32368 3167 +22935 3167 +49506 3166 +44217 3164 +28525 3162 +44177 3162 +35538 3160 +40310 3157 +30444 3156 +44450 3153 +35433 3152 +18441 3151 +32229 3147 +45160 3144 +47120 3143 +23652 3141 +29485 3140 +30467 3138 +6552 3135 +31008 3134 +46331 3131 +45866 3131 +6876 3130 +41436 3130 +22314 3125 +38542 3125 +32247 3121 +15485 3121 +16891 3119 +48641 3118 +38764 3117 +7047 3113 +35172 3110 +42289 3109 +42184 3108 +23321 3107 +34032 3103 +32963 3102 +36196 3102 +37046 3101 +3062 3100 +49496 3095 +37653 3093 +48650 3091 +13577 3080 +9681 3079 +40135 3078 +16818 3065 +28326 3064 +49116 3064 +48346 3060 +29565 3060 +36567 3058 +3179 3056 +44825 3056 +25850 3055 +31467 3050 +43214 3047 +24400 3047 +44632 3047 +10697 3047 +24852 3044 +19012 3043 +11639 3039 +24728 3038 +37778 3037 +12063 3034 +27914 3034 +29335 3032 +20329 3030 +4921 3030 +36248 3028 +46512 3025 +38128 3024 +45528 3022 +2225 3022 +41080 3022 +39184 3021 +32818 3020 +48975 3013 +12343 3012 +5697 3011 +45169 3010 +7449 3010 +37359 3010 +38012 3010 +42646 3009 +22428 3008 +33209 3007 +37395 3006 +13003 3000 +20859 2999 +35277 2996 +28811 2996 +13668 2995 +46010 2993 +42660 2993 +47445 2991 +41780 2991 +35001 2990 +17527 2986 +35528 2984 +26715 2983 +22305 2981 +16151 2980 +46679 2978 +39986 2978 +36767 2975 +2103 2974 +22417 2974 +36857 2973 +5144 2969 +41911 2968 +35533 2967 +33643 2967 +32275 2964 +41382 2964 +9981 2963 +13086 2962 +19306 2959 +46115 2958 +45311 2952 +23839 2948 +17174 2947 +38052 2942 +28865 2940 +44195 2940 +40792 2934 +47046 2933 +27950 2933 +38106 2932 +31579 2930 +25102 2921 +169 2921 +22678 2920 +39688 2920 +37799 2919 +26941 2918 +36970 2915 +19351 2912 +19857 2908 +30715 2907 +39355 2904 +35494 2902 +23360 2902 +45195 2899 +32467 2898 +41357 2895 +32197 2895 +11549 2894 +37693 2894 +14501 2891 +41111 2890 +22506 2887 +47064 2886 +43834 2884 +31619 2882 +27575 2881 +12467 2879 +39789 2876 +45147 2875 +14095 2875 +5319 2874 +20660 2872 +44779 2872 +48845 2872 +35041 2871 +10864 2871 +25490 2870 +22453 2869 +32418 2866 +38277 2862 +38577 2858 +46332 2857 +16166 2856 +23414 2849 +26327 2845 +40927 2843 +23289 2842 +32570 2841 +49883 2841 +38865 2839 +37597 2838 +11573 2837 +46757 2836 +33746 2834 +38816 2833 +45490 2833 +49885 2833 +29863 2832 +21828 2831 +44012 2829 +36112 2827 +49806 2821 +37422 2819 +48728 2817 +42120 2816 +39354 2815 +27945 2814 +47384 2810 +24503 2810 +19282 2807 +26232 2807 +49046 2805 +16589 2804 +38392 2803 +38354 2801 +3330 2800 +48700 2799 +45522 2798 +30205 2792 +37262 2790 +48107 2788 +9762 2784 +34582 2782 +19580 2780 +24087 2778 +30661 2776 +33870 2776 +38292 2774 +39516 2773 +47558 2772 +34889 2764 +38067 2759 +26623 2758 +43753 2758 +25925 2753 +27169 2749 +25714 2746 +50235 2745 +43217 2744 +37978 2742 +28457 2742 +27292 2742 +43695 2741 +29719 2737 +44608 2730 +37157 2729 +21384 2727 +35029 2727 +35558 2726 +48938 2726 +22367 2725 +40010 2725 +23025 2722 +42173 2719 +41068 2715 +23745 2715 +47145 2712 +44952 2711 +46954 2708 +23269 2708 +38281 2705 +32689 2705 +30487 2705 +14980 2703 +16141 2698 +38904 2694 +31966 2693 +37031 2693 +41203 2690 +49615 2690 +27510 2686 +30753 2684 +18566 2683 +19324 2677 +35470 2677 +1792 2677 +33114 2676 +42522 2674 +39323 2668 +47084 2666 +49763 2665 +17414 2664 +38448 2664 +49513 2664 +20869 2663 +4181 2663 +4209 2662 +47006 2662 +8709 2661 +37177 2655 +17022 2654 +21974 2651 +26142 2650 +21340 2647 +22203 2644 +34111 2640 +40099 2638 +34787 2636 +31077 2635 +28167 2633 +47030 2629 +37612 2628 +47174 2628 +15817 2626 +18699 2626 +48443 2626 +40257 2625 +35841 2625 +37156 2623 +6367 2622 +34727 2622 +40613 2621 +42216 2621 +49427 2617 +30293 2616 +44110 2614 +38269 2614 +16897 2613 +45537 2613 +37200 2612 +38235 2611 +40895 2610 +5541 2609 +49071 2608 +35693 2608 +45824 2606 +45135 2606 +39759 2604 +41807 2603 +11610 2603 +48162 2601 +8723 2595 +33910 2594 +4052 2584 +22887 2584 +49832 2584 +44711 2580 +34095 2577 +31301 2576 +16273 2575 +48147 2570 +37511 2568 +980 2568 +41776 2566 +35476 2563 +21471 2561 +46554 2560 +47469 2555 +19526 2555 +37411 2553 +19875 2549 +41972 2548 +32713 2547 +41256 2547 +1591 2545 +18748 2544 +25180 2543 +43706 2542 +30249 2541 +34876 2540 +23081 2540 +44163 2539 +47506 2539 +46471 2538 +9146 2537 +18796 2532 +16516 2529 +49337 2528 +22755 2528 +43583 2524 +30585 2519 +26759 2514 +4840 2510 +42974 2509 +42064 2508 +4766 2507 +44453 2506 +48658 2505 +31266 2505 +31447 2504 +39994 2501 +50030 2499 +28620 2499 +36379 2495 +38571 2493 +18022 2493 +43691 2492 +33986 2490 +44189 2487 +5525 2485 +34181 2480 +19411 2479 +36321 2474 +49505 2474 +758 2471 +36361 2470 +27528 2468 +32785 2467 +38925 2467 +49228 2467 +19382 2464 +44930 2463 +26011 2462 +31181 2462 +41375 2460 +49342 2458 +45492 2457 +30728 2455 +41033 2454 +42451 2453 +22180 2453 +18109 2452 +46495 2452 +31542 2445 +30599 2444 +47951 2441 +36670 2437 +41993 2437 +36621 2434 +34142 2433 +23157 2430 +36017 2425 +40117 2425 +35887 2424 +50163 2424 +47651 2424 +30216 2421 +48744 2419 +31512 2418 +35040 2416 +35847 2416 +23118 2414 +44046 2409 +48469 2408 +47764 2406 +21317 2405 +7168 2404 +41729 2404 +29510 2402 +49137 2400 +21055 2399 +30553 2396 +39194 2393 +22501 2390 +11792 2390 +38946 2385 +33768 2384 +29904 2383 +42558 2380 +25472 2376 +30312 2376 +48900 2375 +25380 2374 +39175 2373 +49795 2372 +34495 2371 +45454 2369 +20004 2369 +49174 2369 +17410 2368 +45672 2366 +39814 2364 +8707 2363 +30782 2363 +11884 2362 +21689 2362 +23870 2359 +21920 2356 +34804 2352 +22093 2347 +5208 2346 +9065 2345 +43145 2344 +14671 2342 +19515 2340 +13783 2337 +14961 2335 +45689 2335 +31631 2334 +46156 2333 +31838 2332 +29905 2331 +45724 2328 +41372 2325 +34720 2320 +47472 2317 +34932 2316 +36690 2316 +40452 2316 +33583 2314 +45519 2314 +24871 2310 +48505 2309 +43786 2309 +47708 2306 +46479 2306 +46121 2303 +40574 2302 +47010 2299 +37199 2298 +22022 2298 +24319 2296 +40246 2294 +44974 2290 +30347 2288 +45803 2286 +47086 2283 +39724 2283 +28618 2280 +39065 2278 +5641 2276 +30604 2275 +10142 2272 +21476 2272 +23257 2270 +40155 2269 +40259 2268 +28968 2265 +24771 2263 +46136 2261 +41947 2260 +36658 2257 +13557 2252 +27985 2250 +34354 2250 +48499 2249 +27600 2246 +27167 2246 +44448 2245 +8964 2243 +17750 2242 +16495 2239 +22542 2239 +47701 2239 +46947 2237 +49358 2237 +42624 2235 +49226 2234 +28198 2231 +48321 2230 +40485 2230 +41100 2230 +10778 2229 +23979 2228 +34196 2227 +21864 2226 +35705 2226 +43702 2225 +31844 2223 +39890 2221 +33725 2221 +40971 2220 +19858 2218 +36370 2217 +33912 2214 +10967 2210 +159 2209 +25178 2208 +8682 2204 +12731 2204 +31179 2204 +48760 2201 +47720 2200 +45876 2196 +42855 2195 +36127 2195 +34340 2193 +39910 2190 +28365 2187 +22330 2183 +6527 2182 +846 2179 +50249 2178 +17635 2177 +38155 2173 +32212 2170 +37859 2169 +28656 2165 +37633 2163 +45662 2163 +39971 2161 +40541 2161 +28762 2160 +48956 2160 +30856 2159 +49896 2157 +48806 2157 +21412 2155 +48869 2155 +49630 2154 +27703 2151 +32230 2150 +42986 2148 +49214 2145 +49665 2142 +43000 2141 +24861 2141 +31660 2141 +44405 2140 +42719 2140 +48369 2139 +39021 2136 +7879 2135 +49905 2132 +44208 2132 +37195 2131 +3679 2129 +47068 2128 +27275 2127 +42758 2124 +37665 2123 +47835 2120 +20290 2116 +42169 2115 +43049 2111 +45286 2111 +41052 2109 +11774 2107 +48523 2100 +32667 2100 +46237 2100 +45351 2098 +49504 2097 +17269 2092 +48720 2091 +36253 2089 +21017 2089 +42867 2089 +23004 2088 +26047 2088 +32734 2081 +46890 2080 +28624 2079 +36408 2078 +40413 2074 +42530 2074 +33044 2069 +32573 2068 +29250 2066 +45008 2065 +49374 2064 +15931 2062 +45908 2062 +47649 2061 +37910 2058 +19890 2057 +44971 2052 +49050 2050 +48362 2047 +10882 2047 +47063 2045 +49925 2036 +12126 2035 +5785 2034 +41444 2034 +46165 2034 +46582 2029 +40591 2028 +11771 2024 +40053 2022 +47732 2022 +9454 2020 +38667 2019 +36044 2019 +48285 2019 +34460 2019 +33920 2018 +46787 2017 +13715 2016 +25529 2015 +4322 2014 +25224 2013 +9201 2012 +39500 2012 +11139 2008 +14188 2005 +33111 2005 +26575 2000 +41753 1998 +19469 1998 +9959 1996 +48452 1995 +30640 1995 +39377 1994 +23595 1993 +24376 1992 +14092 1991 +43717 1990 +28337 1989 +48183 1987 +30167 1986 +25348 1985 +11405 1984 +41099 1983 +48375 1982 +16791 1982 +22046 1980 +23320 1978 +44010 1977 +28794 1974 +36916 1974 +22759 1974 +21876 1974 +28758 1973 +48046 1972 +16883 1967 +20560 1965 +30626 1965 +16878 1964 +45356 1960 +15798 1960 +31364 1959 +15566 1959 +30475 1958 +19182 1956 +20748 1953 +12376 1952 +13291 1952 +22793 1951 +18883 1950 +40356 1949 +50247 1947 +35523 1945 +46621 1944 +18237 1943 +29881 1941 +37430 1940 +41519 1938 +49434 1936 +39044 1936 +47656 1934 +31327 1933 +21583 1933 +28134 1933 +26947 1933 +39000 1931 +38860 1929 +44015 1923 +45507 1923 +26945 1922 +44797 1921 +28707 1920 +13726 1916 +9418 1914 +29829 1913 +40158 1912 +137 1911 +33000 1910 +4907 1909 +38649 1909 +42381 1908 +49834 1907 +35039 1904 +21058 1902 +32014 1901 +45364 1899 +45163 1899 +36918 1896 +36406 1895 +48617 1894 +5512 1894 +23907 1893 +43037 1890 +19667 1890 +32892 1889 +2559 1886 +38696 1882 +32080 1880 +22833 1877 +21756 1876 +11805 1875 +32259 1872 +35975 1872 +30201 1871 +45249 1867 +13328 1867 +22402 1864 +13292 1864 +36545 1863 +49322 1862 +1147 1857 +34013 1857 +44140 1856 +27809 1856 +35307 1855 +26410 1852 +43857 1849 +42973 1848 +34098 1848 +44867 1847 +19990 1845 +11967 1843 +46386 1843 +23381 1836 +31264 1832 +47292 1829 +42866 1827 +33030 1826 +35540 1819 +20608 1818 +39149 1818 +44625 1817 +46238 1814 +48763 1812 +26292 1811 +35453 1810 +43034 1807 +29886 1806 +24957 1806 +47768 1804 +10063 1804 +35611 1802 +46846 1800 +22130 1799 +37093 1797 +28899 1796 +43789 1795 +12676 1795 +38528 1790 +47160 1788 +33566 1787 +37498 1785 +40558 1785 +45554 1776 +1421 1775 +5294 1775 +17358 1774 +18253 1773 +27444 1772 +44728 1769 +19028 1768 +48344 1763 +12567 1762 +27670 1758 +2235 1757 +42915 1753 +35861 1752 +43475 1751 +25895 1750 +7479 1750 +22973 1742 +43267 1739 +38776 1738 +45170 1737 +50227 1736 +46651 1736 +42929 1734 +16892 1733 +49346 1731 +37133 1729 +37350 1729 +21943 1728 +22766 1726 +4011 1725 +49865 1725 +39621 1725 +35656 1721 +7450 1720 +49213 1719 +49877 1715 +25748 1715 +41115 1713 +29743 1711 +23193 1707 +43422 1705 +31347 1705 +33810 1704 +25275 1703 +27674 1702 +15090 1701 +39157 1697 +33648 1695 +42636 1694 +42063 1694 +20746 1694 +31913 1693 +24807 1691 +14859 1690 +44402 1689 +14908 1689 +40261 1688 +13736 1685 +49168 1683 +40267 1683 +11907 1683 +36655 1681 +39643 1681 +16410 1679 +39316 1677 +49819 1674 +19463 1673 +40507 1673 +31246 1673 +46448 1673 +42316 1671 +31687 1671 +9466 1671 +49603 1670 +7726 1667 +34156 1665 +25471 1663 +15020 1663 +34301 1661 +1947 1661 +36181 1660 +31326 1657 +30266 1656 +43458 1656 +41964 1650 +38469 1650 +44293 1649 +33692 1647 +43380 1643 +26678 1643 +43433 1642 +31758 1641 +47400 1639 +2477 1638 +36307 1638 +48991 1637 +10549 1636 +32332 1634 +16268 1633 +28966 1632 +30656 1632 +47226 1629 +46979 1629 +19373 1628 +35036 1626 +38929 1624 +6312 1624 +42615 1622 +45819 1621 +47863 1620 +47195 1619 +26933 1617 +46695 1615 +27711 1614 +18686 1613 +28394 1613 +33386 1612 +42524 1612 +48205 1608 +32360 1605 +9318 1604 +45626 1603 +48526 1601 +29710 1601 +25874 1601 +14610 1599 +3472 1597 +44309 1594 +39922 1594 +19510 1590 +38381 1590 +34121 1588 +28253 1587 +49660 1586 +29773 1585 +32048 1582 +34219 1582 +45664 1581 +46464 1579 +5271 1578 +48911 1578 +22110 1573 +42720 1572 +34018 1569 +38335 1569 +44704 1569 +15149 1568 +45137 1567 +42172 1566 +35339 1563 +43910 1562 +45250 1560 +13392 1560 +23022 1559 +33012 1559 +48277 1557 +16333 1553 +23764 1553 +19202 1553 +19060 1551 +30049 1550 +43993 1546 +32432 1541 +41657 1541 +12374 1540 +31905 1536 +45032 1536 +43408 1534 +39224 1531 +42717 1531 +44423 1528 +19876 1528 +12054 1524 +13412 1523 +34816 1522 +28383 1518 +30295 1516 +39298 1514 +21142 1513 +30327 1513 +19953 1512 +39850 1511 +21563 1505 +17653 1504 +5099 1502 +48123 1499 +30268 1498 +39181 1497 +45297 1495 +36854 1494 +21947 1494 +32639 1494 +21271 1493 +18200 1493 +45748 1491 +43961 1488 +7847 1488 +48534 1487 +43073 1486 +29826 1485 +49991 1483 +42492 1482 +3603 1481 +40704 1480 +43024 1476 +45739 1475 +35585 1471 +41222 1470 +3728 1468 +30360 1465 +40489 1465 +37455 1464 +29712 1463 +30136 1462 +28112 1462 +21052 1459 +33699 1459 +31676 1457 +44036 1454 +43729 1454 +39310 1453 +43078 1453 +33250 1451 +15926 1446 +44506 1442 +29869 1440 +28156 1440 +44520 1438 +47690 1435 +142 1433 +33223 1432 +48761 1432 +34775 1431 +11900 1431 +39187 1430 +47746 1429 +29316 1428 +42468 1428 +36365 1427 +27823 1424 +30562 1424 +31263 1423 +40031 1422 +3429 1421 +30159 1420 +44415 1418 +11039 1418 +47066 1415 +47090 1415 +47212 1412 +20491 1409 +34402 1408 +5008 1408 +28924 1406 +37706 1404 +26227 1404 +45384 1401 +47991 1401 +11411 1399 +20972 1399 +41049 1399 +19662 1399 +38211 1399 +46858 1397 +21314 1391 +13780 1391 +42360 1391 +33705 1387 +49736 1386 +36563 1386 +44013 1384 +37337 1383 +46923 1379 +49312 1379 +23242 1377 +46763 1375 +31490 1375 +49929 1373 +35944 1370 +11482 1370 +38300 1369 +50155 1366 +32590 1363 +43825 1362 +47881 1360 +46349 1360 +48585 1360 +40156 1359 +47797 1359 +37141 1358 +32326 1355 +41305 1355 +12630 1351 +45767 1350 +34945 1350 +29698 1349 +19438 1348 +25182 1344 +24457 1341 +43444 1340 +23473 1339 +24935 1339 +27422 1338 +49035 1338 +40403 1336 +41818 1334 +35600 1334 +6284 1331 +7665 1331 +36171 1328 +3343 1326 +45887 1325 +40092 1324 +23511 1321 +47215 1320 +23626 1319 +11833 1317 +16080 1317 +49267 1316 +46581 1313 +33756 1308 +45251 1308 +25787 1307 +16041 1304 +27901 1302 +36382 1300 +33885 1300 +37906 1296 +19529 1295 +45907 1294 +37239 1293 +3282 1290 +38834 1290 +29613 1290 +18170 1289 +45533 1288 +31221 1285 +6852 1285 +36698 1285 +19395 1285 +40406 1285 +42519 1285 +23106 1283 +47947 1282 +33180 1281 +46570 1281 +37508 1278 +42300 1277 +27813 1275 +8485 1274 +39391 1274 +45750 1272 +27730 1271 +42680 1270 +45999 1270 +23305 1269 +36695 1267 +47546 1266 +11641 1263 +28401 1263 +50084 1263 +26825 1258 +45302 1254 +14524 1254 +33295 1253 +12100 1253 +38953 1253 +34697 1252 +39222 1252 +38021 1252 +36560 1250 +36180 1249 +3422 1248 +40456 1246 +16100 1245 +38543 1242 +37390 1240 +11037 1238 +25216 1235 +42479 1234 +26866 1234 +48835 1233 +34708 1232 +47393 1231 +43727 1229 +39827 1228 +29557 1227 +40069 1227 +26796 1226 +19455 1226 +44197 1225 +34640 1225 +10748 1224 +2155 1224 +27765 1216 +29241 1215 +43117 1215 +47795 1212 +47738 1211 +49694 1211 +20046 1211 +47864 1209 +46344 1208 +33623 1206 +43763 1204 +14060 1200 +40161 1199 +24539 1196 +16648 1195 +29281 1193 +17912 1193 +22065 1188 +43971 1188 +37863 1186 +35061 1186 +25618 1184 +12927 1183 +18125 1181 +37523 1179 +37101 1179 +41333 1177 +160 1176 +12241 1175 +14804 1175 +17720 1174 +27370 1173 +41248 1170 +17816 1169 +46746 1167 +39520 1165 +38271 1163 +34255 1163 +11727 1162 +33232 1162 +43879 1162 +21950 1162 +39098 1161 +22944 1161 +36750 1161 +10779 1160 +34519 1160 +25918 1158 +45281 1157 +36461 1154 +7503 1153 +36297 1152 +29572 1152 +37650 1150 +7180 1149 +30351 1149 +36764 1147 +23046 1145 +32756 1144 +48472 1143 +29472 1143 +5227 1142 +14865 1140 +12813 1140 +28243 1137 +8943 1137 +37848 1136 +19969 1136 +47527 1135 +34261 1132 +21903 1129 +35260 1127 +14560 1126 +42030 1126 +41592 1124 +47171 1124 +12426 1122 +46956 1122 +12071 1121 +20662 1119 +21711 1117 +36713 1117 +48227 1116 +22781 1115 +30660 1115 +43551 1113 +43328 1111 +41504 1109 +18163 1108 +31431 1106 +48873 1103 +31051 1102 +38184 1102 +40816 1097 +37345 1096 +35103 1095 +34704 1094 +27764 1092 +41096 1088 +47429 1088 +35561 1087 +41468 1086 +49551 1085 +43089 1083 +27570 1082 +47945 1078 +18477 1077 +49146 1077 +6847 1076 +16208 1073 +16934 1072 +16281 1067 +37420 1067 +24239 1065 +50143 1065 +2698 1064 +39786 1063 +37168 1061 +47082 1060 +45340 1059 +38124 1058 +16922 1056 +46310 1054 +11568 1053 +42527 1053 +32131 1053 +29460 1053 +44082 1052 +32849 1052 +18083 1052 +46796 1049 +49056 1049 +13486 1045 +30719 1041 +46966 1039 +28255 1036 +23138 1035 +22273 1035 +12486 1030 +45145 1028 +49070 1027 +37380 1027 +39695 1025 +41661 1024 +47611 1019 +48144 1019 +22345 1018 +44204 1017 +45640 1017 +45115 1017 +42752 1016 +31944 1016 +40674 1015 +41343 1014 +8762 1014 +45148 1013 +44575 1011 +33843 1011 +30667 1010 +43266 1008 +44948 1006 +33252 1005 +26911 1005 +35930 1004 +42654 1003 +42245 1002 +21424 999 +33151 998 +45865 997 +44052 996 +40822 996 +23294 993 +31386 993 +12640 989 +9705 989 +20786 986 +16471 985 +29495 984 +49535 984 +12953 982 +25915 982 +9020 978 +11647 976 +1209 976 +13211 972 +47139 966 +43966 966 +22174 961 +41002 960 +46976 959 +39258 959 +46086 958 +39611 958 +43922 956 +48080 956 +39231 955 +35407 955 +19820 951 +21324 950 +19421 950 +47826 949 +45442 949 +33031 946 +48997 943 +24714 941 +33221 940 +23001 940 +6043 939 +41585 939 +26076 938 +27156 935 +47715 934 +40561 931 +29325 931 +48313 931 +48364 929 +47340 928 +12045 927 +37605 925 +36853 922 +30751 920 +12675 919 +47671 915 +40041 914 +38165 911 +9497 911 +36752 911 +10790 908 +7948 906 +15697 905 +44165 904 +33213 903 +28544 903 +35707 902 +44917 901 +29848 900 +12736 898 +26475 896 +31965 892 +37096 892 +25195 892 +48958 891 +45992 889 +38963 888 +38218 887 +27632 887 +42655 880 +36119 879 +32403 878 +25001 873 +37771 873 +17845 872 +18670 872 +29312 869 +41726 862 +15139 862 +25597 862 +43488 859 +21481 858 +25081 858 +40223 857 +45474 856 +22718 853 +27972 853 +46497 850 +32003 849 +21737 847 +24029 847 +47676 846 +31023 846 +37695 843 +21477 843 +37955 842 +42669 840 +32092 840 +43549 838 +39666 836 +39482 833 +19682 833 +46764 832 +7535 832 +6927 832 +14367 831 +28165 830 +25745 830 +34222 828 +31724 827 +143 824 +42000 823 +48670 823 +17158 821 +11839 820 +33524 817 +39105 816 +25902 815 +29554 815 +41791 815 +22316 814 +44646 813 +32291 809 +49426 805 +37456 803 +41309 803 +43015 803 +38551 800 +42565 799 +2549 797 +44829 796 +9263 796 +38461 795 +43889 795 +35269 794 +29994 794 +36003 794 +31917 793 +22270 791 +12424 791 +46274 791 +33470 790 +33778 789 +46092 788 +46036 785 +41335 784 +31456 784 +21775 783 +15211 783 +15081 782 +3786 781 +29841 781 +16116 780 +43297 777 +43903 776 +34976 772 +19587 772 +4431 772 +40391 771 +49201 771 +21763 769 +21091 768 +45298 767 +45990 766 +48265 763 +37374 760 +29847 760 +42924 759 +39763 756 +7961 755 +41150 753 +37014 753 +30298 751 +30838 750 +144 748 +175 748 +40664 746 +9202 745 +15362 745 +38626 745 +39415 744 +36875 741 +15886 741 +5392 741 +15306 741 +23497 739 +25410 738 +45398 736 +14030 735 +38430 734 +28133 734 +22758 733 +18189 733 +34284 730 +30138 729 +46741 728 +23991 728 +29940 727 +36584 727 +37423 727 +49568 726 +20379 725 +47903 722 +8795 721 +6141 719 +31813 719 +48678 717 +22341 716 +46202 715 +27802 714 +41293 713 +27352 711 +44148 709 +37961 708 +29785 706 +40477 705 +29412 705 +21948 704 +43900 701 +40367 699 +22446 699 +43718 697 +42684 692 +37772 690 +45911 688 +46866 686 +30831 685 +8728 685 +36685 685 +10108 684 +33918 684 +39618 682 +48746 681 +39317 681 +34585 680 +3556 679 +21955 678 +15790 674 +6336 673 +9603 673 +47505 672 +46189 668 +25226 668 +23719 667 +49518 665 +37083 664 +39691 661 +43238 658 +25295 656 +6480 653 +23884 653 +48688 652 +20645 650 +36310 650 +48972 650 +31424 647 +48702 646 +34534 642 +41216 642 +30916 640 +33084 639 +37588 639 +45358 638 +40265 637 +49007 636 +30929 636 +8815 635 +44274 634 +8955 633 +38145 630 +43357 628 +37581 627 +29705 626 +43962 626 +33116 626 +22934 624 +46506 623 +42376 622 +36142 622 +11966 622 +32284 620 +36278 619 +9841 617 +29966 616 +38105 616 +17433 615 +19779 614 +170 613 +25060 613 +7280 612 +36737 612 +29415 612 +24679 611 +22784 611 +47762 610 +982 608 +48416 606 +9468 604 +46083 603 +42348 602 +43023 599 +29047 599 +46532 598 +29582 598 +34237 597 +24442 597 +22263 596 +41146 595 +12712 594 +45617 593 +27643 593 +47078 593 +39893 591 +49704 589 +49339 589 +32288 588 +49969 587 +37176 586 +43913 586 +24032 585 +15474 583 +44557 583 +2468 581 +43055 580 +34580 580 +37190 576 +8773 576 +19841 575 +46118 573 +35266 573 +43291 571 +43491 569 +22906 568 +13700 567 +37266 567 +28657 566 +10523 566 +38593 566 +6710 565 +31175 565 +13562 562 +40717 561 +23854 555 +38574 555 +48553 554 +34758 554 +29740 553 +28235 552 +9882 552 +50101 549 +39747 547 +37527 547 +29953 545 +45228 545 +43095 543 +44627 542 +40748 540 +40948 539 +14077 535 +41092 535 +42314 535 +43636 531 +37763 530 +29212 529 +42757 527 +46777 526 +42249 524 +28350 524 +23224 523 +27032 522 +41215 521 +15083 519 +14726 519 +28360 518 +44051 518 +16148 517 +16792 515 +13298 515 +36338 514 +48393 513 +16150 512 +31854 512 +15661 511 +43648 511 +31817 511 +36470 510 +21316 507 +47581 506 +35430 506 +38463 505 +48774 505 +49135 504 +48801 503 +27246 502 +32015 502 +21364 501 +19785 498 +45520 498 +24122 497 +35067 497 +5622 497 +15961 496 +48557 493 +23287 492 +10144 491 +10097 491 +26867 487 +41888 487 +49409 484 +24336 484 +39488 483 +25248 482 +20398 481 +38892 479 +20322 479 +16626 477 +35508 476 +26791 476 +40290 475 +49546 475 +16253 474 +43768 473 +43197 473 +14099 470 +47418 469 +9287 468 +14692 468 +25053 467 +45227 466 +42164 466 +14512 466 +28632 465 +45986 464 +30134 462 +38519 462 +28955 460 +29636 458 +33489 457 +44057 457 +20036 457 +23376 456 +15272 455 +36166 452 +29577 450 +32541 449 +24373 448 +24689 448 +39737 447 +5571 447 +10160 446 +49582 446 +49015 446 +44755 444 +23711 442 +42804 441 +28013 441 +40672 440 +37143 440 +35439 439 +8346 439 +35050 437 +15116 436 +36447 436 +44926 435 +16323 434 +31768 434 +44174 431 +14777 430 +15853 430 +21451 428 +41340 428 +47232 428 +46999 427 +35799 425 +35555 425 +25131 424 +19927 423 +41349 423 +8983 421 +34991 417 +20213 416 +34949 415 +33682 415 +44332 414 +29589 413 +49771 413 +49129 412 +24186 412 +33803 411 +48893 410 +25645 408 +19415 407 +44547 407 +35235 405 +39321 404 +40786 403 +37389 401 +34650 401 +45717 399 +20590 397 +8994 396 +26054 393 +48031 393 +13697 391 +2955 390 +40266 386 +1977 385 +17405 384 +2115 382 +3207 382 +45082 381 +3156 381 +32517 381 +43102 381 +40537 380 +43375 378 +47932 378 +30109 377 +48737 376 +16165 373 +19629 371 +30072 370 +37485 368 +3286 368 +48995 367 +24493 367 +25465 366 +43848 366 +21253 365 +24440 364 +49607 359 +37227 359 +48541 358 +41573 358 +40386 353 +7601 352 +29851 352 +22133 352 +29659 352 +49011 351 +45218 351 +18004 350 +40720 349 +37467 346 +42783 345 +32398 343 +49960 342 +35306 342 +34345 341 +27007 341 +12404 340 +42543 339 +42638 338 +49167 335 +31208 334 +45784 333 +33296 333 +20512 332 +32742 332 +18356 331 +44587 331 +30165 329 +41433 327 +40364 327 +9049 326 +2918 326 +17683 325 +17681 325 +24077 324 +25611 324 +6166 323 +39681 323 +49855 323 +44256 322 +39333 322 +49073 320 +31937 320 +37509 317 +43077 316 +15842 316 +33384 315 +25497 315 +30965 315 +48497 315 +16193 315 +47987 313 +22725 313 +46140 313 +2804 312 +32524 311 +20532 310 +47939 310 +28542 310 +39192 309 +10125 308 +17773 305 +47423 305 +46268 301 +24363 301 +20513 300 +26498 300 +50179 299 +36469 298 +34543 297 +20115 297 +28696 297 +36786 296 +26712 294 +31090 293 +42062 290 +23131 287 +38326 287 +34373 287 +26095 286 +25626 286 +46424 285 +15506 285 +9000 285 +31936 283 +30800 282 +49058 281 +27551 280 +44648 276 +12240 276 +18803 275 +7589 274 +41641 274 +40109 274 +1896 273 +47000 273 +32518 272 +45379 272 +45635 272 +33529 272 +13765 270 +40420 270 +12845 269 +16201 268 +33465 268 +24001 267 +49249 267 +41853 267 +4010 266 +8418 265 +49813 265 +25581 264 +43481 263 +46788 262 +41629 262 +24710 261 +35144 261 +39907 260 +47021 260 +20801 260 +25128 259 +30812 259 +48304 258 +18115 257 +48404 256 +33829 256 +23846 255 +6112 254 +5997 254 +6987 253 +7677 253 +26503 253 +44161 253 +46491 252 +48231 251 +11910 251 +37669 250 +35702 249 +49464 248 +41323 248 +37233 247 +26998 246 +36828 246 +24806 244 +32207 243 +30762 242 +42637 242 +16103 242 +172 242 +49472 242 +25084 241 +7105 239 +19791 238 +41974 237 +45335 237 +31052 236 +37426 236 +13177 235 +38894 235 +37495 234 +25549 234 +25795 234 +43801 234 +43282 233 +13296 231 +38149 231 +24466 230 +28264 230 +34525 230 +48448 228 +36581 227 +36521 227 +49082 225 +48318 225 +8755 225 +36792 223 +43353 223 +19039 223 +43302 222 +20677 220 +47307 220 +32310 220 +31732 220 +19021 219 +17401 219 +36922 219 +12717 218 +33426 217 +46904 217 +33309 216 +33795 215 +29993 214 +4242 214 +32546 212 +35818 208 +47175 208 +11896 208 +34934 208 +48908 208 +42367 208 +43769 207 +50159 207 +35514 205 +45495 204 +43213 203 +2590 203 +4895 203 +27542 202 +39996 201 +35063 201 +47569 200 +40283 198 +37817 198 +1543 197 +37757 197 +14592 196 +27733 195 +39394 195 +13979 194 +12662 193 +2887 193 +42287 191 +22241 191 +32891 189 +11548 189 +26642 188 +11480 187 +15755 186 +10060 186 +15913 186 +45144 185 +42026 185 +19227 184 +33761 184 +25207 182 +38122 182 +44916 181 +29113 181 +22831 179 +13945 179 +6681 178 +34901 178 +22322 178 +50108 177 +29164 176 +23544 174 +46545 174 +15351 173 +22496 173 +39742 173 +11606 173 +36796 173 +26229 171 +39115 171 +35572 170 +16959 169 +30716 168 +37642 167 +44651 167 +27852 167 +40073 167 +37913 167 +45975 166 +46968 166 +38362 165 +29446 165 +7359 164 +17473 163 +40012 163 +47258 163 +46677 162 +49390 162 +45150 162 +24022 161 +24898 160 +32239 160 +45434 159 +27509 159 +39703 159 +3587 157 +11974 157 +36726 156 +15685 155 +38653 154 +24618 154 +17532 153 +24973 151 +41386 151 +48944 151 +41678 151 +7804 149 +33991 148 +21959 146 +14531 145 +41441 144 +37412 144 +41658 143 +6399 142 +2941 142 +33893 141 +41849 140 +11919 140 +44912 139 +46541 138 +32088 137 +35604 137 +48610 137 +50033 137 +19476 137 +25926 136 +23330 136 +49843 136 +50001 135 +48999 134 +48458 134 +36301 133 +16646 132 +47271 132 +40111 132 +23329 132 +41313 131 +23728 130 +33207 128 +47542 127 +37082 127 +10269 126 +39434 126 +48204 125 +43667 124 +6533 122 +49273 122 +40361 122 +49324 121 +25589 121 +8438 120 +2432 120 +17761 120 +10052 120 +37435 119 +47981 118 +12869 116 +34607 115 +41006 113 +47596 113 +45123 112 +43246 110 +40719 110 +26700 110 +41945 109 +152 109 +16098 109 +6598 109 +47253 107 +46249 107 +29021 106 +45563 106 +47147 105 +45539 105 +35343 105 +43241 105 +50216 105 +43734 105 +35922 104 +29226 104 +44431 104 +221 104 +25970 104 +27293 103 +1841 103 +14468 103 +42311 103 +47559 102 +49643 102 +35069 101 +8115 101 +44233 100 +16303 100 +47282 99 +40629 99 +39008 99 +32509 98 +32917 98 +42396 98 +176 97 +36169 96 +48753 96 +10253 94 +40516 94 +3523 93 +48185 93 +49086 93 +44104 92 +31204 92 +43839 91 +27924 90 +17933 90 +49476 89 +36596 89 +46745 89 +44167 87 +30531 87 +47486 87 +40549 86 +35992 85 +34832 85 +33468 85 +46996 84 +40493 84 +43242 84 +7134 83 +11585 80 +22640 80 +37811 80 +42869 80 +39860 79 +25719 78 +23984 77 +11273 76 +41481 76 +20554 75 +13198 75 +28725 74 +42210 74 +155 73 +48727 73 +35318 72 +41365 71 +32511 70 +40345 70 +31708 69 +44872 69 +41832 69 +9968 69 +28670 69 +30478 67 +10298 67 +11885 66 +11737 66 +17787 66 +41939 65 +41868 65 +39467 65 +41538 64 +44546 63 +20503 62 +47682 61 +4060 61 +16068 60 +48457 59 +36473 59 +45449 59 +50113 58 +15040 58 +46110 58 +47614 58 +50116 57 +43313 57 +18945 57 +29795 57 +37858 56 +26534 55 +18433 55 +39886 55 +33490 54 +40415 54 +36704 53 +44785 53 +43899 51 +32865 51 +47530 51 +39364 50 +42535 49 +25887 49 +37981 49 +49527 49 +33937 48 +36940 48 +22315 47 +48874 47 +4204 47 +23596 47 +31478 47 +42035 46 +9286 46 +38016 46 +11689 46 +42785 45 +46222 45 +33994 45 +47794 45 +43897 45 +42877 45 +48953 44 +44686 44 +37545 44 +45435 43 +34386 42 +7260 42 +13171 42 +23926 41 +24307 40 +31820 40 +44392 39 +48600 38 +41230 38 +38390 38 +13150 37 +154 37 +38243 36 +29447 36 +44326 36 +29646 35 +19073 35 +15041 35 +45433 34 +42382 32 +38377 32 +47540 32 +47936 32 +47432 31 +49149 31 +30432 31 +39280 30 +48366 30 +12943 29 +37922 29 +14695 29 +13426 29 +16782 28 +49997 28 +36926 28 +23613 28 +22675 28 +35098 28 +6408 28 +33153 28 +31739 28 +43518 27 +7782 27 +29752 27 +23363 26 +45823 26 +43053 26 +40236 24 +37787 24 +49029 24 +33023 24 +20804 24 +34103 24 +27013 23 +37991 23 +42943 23 +44033 23 +41380 22 +25698 22 +42750 22 +25362 22 +44555 22 +22039 20 +42983 20 +45762 20 +27006 20 +12677 19 +48219 19 +43394 19 +37662 19 +17553 18 +45422 18 +30439 18 +14341 18 +11592 17 +33929 17 +17629 17 +42889 17 +153 17 +14223 17 +43010 17 +32843 16 +48193 16 +35793 16 +36475 16 +31161 15 +16822 15 +47182 15 +44112 14 +34604 14 +31881 14 +46402 13 +36937 12 +28500 12 +24731 12 +27584 12 +43796 11 +36481 11 +40703 11 +19049 10 +44444 10 +4690 10 +42470 9 +41977 9 +36917 9 +46948 9 +10658 9 +38250 9 +43298 8 +45953 8 +36929 8 +42234 7 +38160 7 +47490 7 +40235 7 +5808 7 +45786 7 +36490 6 +5367 6 +27534 6 +21807 5 +36886 5 +39693 5 +30684 5 +37226 5 +15243 5 +34633 5 +22997 5 +25658 4 +45321 4 +8980 4 +47648 4 +34206 4 +43569 4 +36862 3 +49778 3 +45392 3 +42066 3 +36130 3 +46939 3 +6438 3 +34842 2 +48527 2 +38370 2 +34473 2 +40278 2 +20174 2 +5815 1 +9364 1 +39142 1 +47703 1 +49074 1 +31536 1 +14827 1 +23090 1 +43735 1 +24847 1 +40219 1 +32437 1 +31727 1 +124 0 +125 0 +173 0 +174 0 +177 0 +178 0 +179 0 +180 0 +181 0 +182 0 +183 0 +184 0 +185 0 +186 0 +187 0 +188 0 +189 0 +190 0 +191 0 +192 0 +193 0 +194 0 +195 0 +196 0 +197 0 +198 0 +199 0 +200 0 +201 0 +202 0 +203 0 +204 0 +205 0 +206 0 +207 0 +208 0 +209 0 +210 0 +211 0 +212 0 +213 0 +214 0 +215 0 +216 0 +217 0 +218 0 +219 0 +628 0 +1849 0 +4603 0 +5624 0 +8828 0 +11504 0 +12781 0 +17811 0 +17900 0 +18472 0 +22686 0 +22757 0 +23282 0 +23614 0 +23785 0 +24293 0 +24934 0 +25193 0 +25502 0 +25992 0 +28666 0 +29342 0 +29372 0 +30202 0 +30208 0 +30209 0 +30210 0 +30211 0 +30212 0 +30213 0 +30897 0 +30898 0 +30899 0 +30905 0 +30906 0 +31032 0 +31538 0 +31573 0 +31576 0 +31666 0 +31765 0 +31783 0 +31886 0 +31957 0 +32047 0 +32406 0 +33434 0 +33454 0 +33477 0 +33813 0 +34027 0 +34448 0 +34504 0 +34516 0 +35207 0 +35496 0 +35579 0 +36173 0 +36174 0 +36935 0 +36938 0 +37444 0 +37574 0 +37579 0 +37631 0 +37842 0 +38214 0 +39165 0 +39172 0 +39177 0 +39253 0 +39374 0 +39446 0 +39655 0 +39714 0 +39749 0 +39752 0 +39753 0 +39755 0 +39756 0 +39757 0 +39803 0 +39811 0 +39820 0 +39821 0 +39906 0 +40240 0 +40241 0 +40242 0 +41297 0 +41383 0 +41551 0 +42089 0 +42090 0 +42202 0 +42424 0 +42496 0 +42586 0 +42728 0 +43038 0 +43065 0 +43177 0 +43361 0 +43453 0 +44320 0 +45003 0 +45199 0 +45544 0 +45545 0 +45706 0 +46600 0 +47198 0 +47571 0 +47654 0 +47934 0 +48069 0 +48396 0 +49731 0 +49781 0 +50009 0 +50256 0 +madeupword0000 0 +madeupword0001 0 +madeupword0002 0 diff --git a/new_impl/nlp/roberta/sentiment-classification/roberta-base/merges.txt b/new_impl/nlp/roberta/sentiment-classification/roberta-base/merges.txt new file mode 100644 index 0000000000000000000000000000000000000000..226b0752cac7789c48f0cb3ec53eda48b7be36cc --- /dev/null +++ b/new_impl/nlp/roberta/sentiment-classification/roberta-base/merges.txt @@ -0,0 +1,50001 @@ +#version: 0.2 +Ġ t +Ġ a +h e +i n +r e +o n +Ġt he +e r +Ġ s +a t +Ġ w +Ġ o +e n +Ġ c +i t +i s +a n +o r +e s +Ġ b +e d +Ġ f +in g +Ġ p +o u +Ġa n +a l +a r +Ġt o +Ġ m +Ġo f +Ġ in +Ġ d +Ġ h +Ġan d +i c +a s +l e +Ġt h +i on +o m +l l +en t +Ġ n +Ġ l +s t +Ġ re +v e +Ġ e +r o +l y +Ġb e +Ġ g +Ġ T +c t +Ġ S +i d +o t +Ġ I +u t +e t +Ġ A +Ġ is +Ġ on +i m +a m +o w +a y +a d +s e +Ġth at +Ġ C +i g +Ġf or +a c +Ġ y +v er +u r +Ġ u +l d +Ġs t +Ġ M +' s +Ġ he +Ġ it +at ion +it h +i r +c e +Ġy ou +i l +Ġ B +Ġw h +o l +Ġ P +Ġw ith +Ġ 1 +t er +c h +Ġa s +Ġw e +Ġ ( +n d +i ll +Ġ D +i f +Ġ 2 +a g +er s +k e +Ġ " +Ġ H +e m +Ġc on +Ġ W +Ġ R +he r +Ġw as +Ġ r +o d +Ġ F +u l +at e +Ġa t +r i +p p +o re +ĠT he +Ġs e +u s +Ġp ro +Ġh a +u m +Ġa re +Ġd e +a in +an d +Ġo r +ig h +es t +is t +a b +r om +Ġ N +t h +Ġc om +Ġ G +u n +o p +0 0 +Ġ L +Ġn ot +es s +Ġe x +Ġ v +re s +Ġ E +e w +it y +an t +Ġb y +e l +o s +or t +o c +q u +Ġf rom +Ġha ve +Ġs u +i ve +ou ld +Ġs h +Ġth is +n t +r a +p e +igh t +ar t +m ent +Ġa l +u st +en d +- - +al l +Ġ O +ac k +Ġc h +Ġ le +i es +re d +ar d +â Ģ +ou t +Ġ J +Ġa b +e ar +i v +al ly +ou r +o st +g h +p t +Ġp l +as t +Ġc an +a k +om e +u d +T he +Ġh is +Ġd o +Ġg o +Ġh as +g e +' t +Ġ U +r ou +Ġs a +Ġ j +Ġb ut +Ġw or +Ġa ll +e ct +Ġ k +am e +Ġw ill +o k +Ġw he +Ġthe y +id e +0 1 +f f +ic h +p l +t her +Ġt r +. . +Ġin t +i e +u re +ag e +Ġn e +i al +a p +in e +ic e +Ġm e +Ġo ut +an s +on e +on g +ion s +Ġwh o +Ġ K +Ġu p +Ġthe ir +Ġa d +Ġ 3 +Ġu s +at ed +ou s +Ġm ore +u e +o g +ĠS t +in d +i ke +Ġs o +im e +p er +. " +b er +i z +a ct +Ġon e +Ġsa id +Ġ - +a re +Ġyou r +c c +ĠT h +Ġc l +e p +a ke +ab le +i p +Ġcon t +Ġwh ich +i a +Ġ im +Ġab out +Ġwe re +ver y +u b +Ġh ad +Ġ en +Ġcom p +, " +ĠI n +Ġu n +Ġa g +i re +ac e +a u +ar y +Ġw ould +as s +r y +Ġ âĢ +c l +o ok +e re +s o +Ġ V +ig n +i b +Ġof f +Ġt e +v en +Ġ Y +i le +o se +it e +or m +Ġ2 01 +Ġre s +Ġm an +Ġp er +Ġo ther +or d +ul t +Ġbe en +Ġl ike +as e +an ce +k s +ay s +ow n +en ce +Ġd is +ct ion +Ġan y +Ġa pp +Ġs p +in t +res s +ation s +a il +Ġ 4 +ic al +Ġthe m +Ġhe r +ou nt +ĠC h +Ġa r +Ġ if +Ġthe re +Ġp e +Ġy ear +a v +Ġm y +Ġs ome +Ġwhe n +ou gh +ac h +Ġth an +r u +on d +ic k +Ġo ver +ve l +Ġ qu +Ċ Ċ +Ġs c +re at +re e +ĠI t +ou nd +p ort +Ġal so +Ġp art +f ter +Ġk n +Ġbe c +Ġt ime +en s +Ġ 5 +op le +Ġwh at +Ġn o +d u +m er +an g +Ġn ew +-- -- +Ġg et +or y +it ion +ing s +Ġj ust +Ġint o +Ġ 0 +ent s +o ve +t e +Ġpe ople +Ġp re +Ġit s +Ġre c +Ġt w +i an +ir st +ar k +or s +Ġwor k +ad e +o b +Ġs he +Ġo ur +w n +in k +l ic +Ġ1 9 +ĠH e +is h +nd er +au se +Ġh im +on s +Ġ [ +Ġ ro +f orm +i ld +at es +ver s +Ġon ly +o ll +Ġs pe +c k +e ll +am p +Ġa cc +Ġb l +i ous +ur n +f t +o od +Ġh ow +he d +Ġ ' +Ġa fter +a w +Ġat t +o v +n e +Ġpl ay +er v +ic t +Ġc ould +it t +Ġa m +Ġf irst +Ġ 6 +Ġa ct +Ġ $ +e c +h ing +u al +u ll +Ġcom m +o y +o ld +c es +at er +Ġf e +Ġbe t +w e +if f +Ġtw o +oc k +Ġb ack +) . +id ent +Ġu nder +rou gh +se l +x t +Ġm ay +rou nd +Ġp o +p h +is s +Ġd es +Ġm ost +Ġd id +Ġad d +j ect +Ġin c +f ore +Ġp ol +on t +Ġag ain +cl ud +ter n +Ġkn ow +Ġne ed +Ġcon s +Ġc o +Ġ . +Ġw ant +Ġse e +Ġ 7 +n ing +i ew +ĠTh is +c ed +Ġe ven +Ġin d +t y +ĠW e +at h +Ġthe se +Ġp r +Ġu se +Ġbec ause +Ġf l +n g +Ġn ow +ĠâĢ ĵ +c om +is e +Ġm ake +Ġthe n +ow er +Ġe very +ĠU n +Ġse c +os s +u ch +Ġe m +Ġ = +ĠR e +i ed +r it +Ġin v +le ct +Ġsu pp +at ing +Ġl ook +m an +pe ct +Ġ 8 +ro w +Ġb u +Ġwhe re +if ic +Ġyear s +i ly +Ġd iff +Ġsh ould +Ġre m +T h +I n +Ġe v +d ay +' re +ri b +Ġre l +s s +Ġde f +Ġr ight +Ġs y +) , +l es +00 0 +he n +Ġth rough +ĠT r +_ _ +Ġw ay +Ġd on +Ġ , +Ġ1 0 +as ed +Ġas s +ub lic +Ġre g +ĠA nd +i x +Ġ very +Ġin clud +ot her +Ġim p +ot h +Ġsu b +ĠâĢ Ķ +Ġbe ing +ar g +ĠW h += = +ib le +Ġdo es +an ge +r am +Ġ 9 +er t +p s +it ed +ation al +Ġb r +Ġd own +Ġman y +ak ing +Ġc all +ur ing +it ies +Ġp h +ic s +al s +Ġde c +at ive +en er +Ġbe fore +il ity +Ġwe ll +Ġm uch +ers on +Ġth ose +Ġsu ch +Ġ ke +Ġ end +ĠB ut +as on +t ing +Ġl ong +e f +Ġth ink +y s +Ġbe l +Ġs m +it s +a x +Ġo wn +Ġpro v +Ġs et +if e +ment s +b le +w ard +Ġsh ow +Ġp res +m s +om et +Ġo b +Ġs ay +ĠS h +t s +f ul +Ġe ff +Ġg u +Ġin st +u nd +re n +c ess +Ġ ent +ĠY ou +Ġgo od +Ġst art +in ce +Ġm ade +t t +st em +ol og +u p +Ġ | +um p +Ġhe l +ver n +ul ar +u ally +Ġa c +Ġm on +Ġl ast +Ġ2 00 +1 0 +Ġst ud +u res +ĠA r +sel f +ar s +mer ic +u es +c y +Ġm in +oll ow +Ġc ol +i o +Ġm od +Ġc ount +ĠC om +he s +Ġf in +a ir +i er +âĢ Ķ +re ad +an k +at ch +e ver +Ġst r +Ġpo int +or k +ĠN ew +Ġs ur +o ol +al k +em ent +Ġus ed +ra ct +we en +Ġs ame +ou n +ĠA l +c i +Ġdiff ere +Ġwh ile +---- ---- +Ġg ame +ce pt +Ġs im +.. . +Ġin ter +e k +Ġre port +Ġpro du +Ġst ill +l ed +a h +Ġhe re +Ġwor ld +Ġth ough +Ġn um +ar ch +im es +al e +ĠS e +ĠI f +/ / +ĠL e +Ġre t +Ġre f +Ġtr ans +n er +ut ion +ter s +Ġt ake +ĠC l +Ġcon f +w ay +a ve +Ġgo ing +Ġs l +u g +ĠA meric +Ġspe c +Ġh and +Ġbet ween +ist s +ĠD e +o ot +I t +Ġe ar +Ġagain st +Ġh igh +g an +a z +at her +Ġex p +Ġo p +Ġin s +Ġg r +Ġhel p +Ġre qu +et s +in s +ĠP ro +is m +Ġf ound +l and +at a +us s +am es +Ġp erson +Ġg reat +p r +Ġs ign +ĠA n +' ve +Ġs omet +Ġs er +h ip +Ġr un +Ġ : +Ġt er +ire ct +Ġf ollow +Ġd et +ic es +Ġf ind +1 2 +Ġm em +Ġc r +e red +e x +Ġex t +ut h +en se +c o +Ġte am +v ing +ou se +as h +at t +v ed +Ġsy stem +ĠA s +d er +iv es +m in +Ġle ad +ĠB l +c ent +Ġa round +Ġgo vern +Ġc ur +vel op +an y +Ġc our +al th +ag es +iz e +Ġc ar +od e +Ġl aw +Ġre ad +' m +c on +Ġre al +Ġsupp ort +Ġ1 2 +.. .. +Ġre ally +n ess +Ġf act +Ġd ay +Ġb oth +y ing +Ġs erv +ĠF or +Ġth ree +Ġw om +Ġm ed +od y +ĠThe y +5 0 +Ġex per +t on +Ġe ach +ak es +Ġc he +Ġc re +in es +Ġre p +1 9 +g g +ill ion +Ġg rou +ut e +i k +W e +g et +E R +Ġm et +Ġs ays +o x +Ġd uring +er n +iz ed +a red +Ġf am +ic ally +Ġha pp +ĠI s +Ġch ar +m ed +v ent +Ġg ener +i ent +p le +i et +re nt +1 1 +v es +pt ion +Ġ2 0 +form ation +Ġc or +Ġoff ic +ie ld +Ġto o +is ion +Ġin f +Ġ Z +t he +o ad +Ġp ublic +Ġpro g +r ic +* * +Ġw ar +Ġp ower +v iew +Ġf ew +Ġl oc +Ġdiffere nt +Ġst ate +Ġhe ad +' ll +Ġp oss +Ġst at +re t +ant s +Ġv al +Ġis s +Ġc le +i vers +an c +Ġex pl +Ġan other +Ġ Q +Ġa v +th ing +n ce +W h +Ġch ild +Ġs ince +i red +l ess +Ġl ife +Ġde velop +itt le +Ġde p +Ġp ass +ã ĥ +Ġt urn +or n +Th is +b ers +ro ss +ĠA d +Ġf r +Ġres p +Ġsec ond +o h +Ġ / +Ġdis c +Ġ & +Ġsomet hing +Ġcomp le +Ġ ed +Ġf il +Ġmon th +a j +u c +Ġgovern ment +Ġwith out +Ġle g +Ġd ist +Ġp ut +Ġqu est +an n +Ġpro t +2 0 +Ġne ver +i ence +Ġle vel +Ġar t +Ġth ings +Ġm ight +Ġeff ect +Ġcont ro +Ġc ent +Ġ1 8 +Ġall ow +Ġbel ie +ch ool +ot t +Ġinc re +Ġfe el +Ġres ult +Ġl ot +Ġf un +ot e +Ġt y +ere st +Ġcont in +Ġus ing +Ġb ig +2 01 +Ġas k +Ġb est +Ġ ) +I N +Ġo pp +3 0 +Ġnum ber +in ess +S t +le ase +Ġc a +Ġm ust +Ġd irect +Ġg l +Ġ < +Ġop en +Ġp ost +Ġcom e +Ġse em +ord ing +Ġwe ek +ate ly +it al +Ġe l +ri end +Ġf ar +Ġt ra +in al +Ġp ri +ĠU S +Ġpl ace +Ġfor m +Ġto ld +" : +ain s +at ure +ĠTr ump +Ġst and +Ġ # +id er +ĠF r +Ġne xt +Ġs oc +Ġp ur +Ġle t +Ġl ittle +Ġh um +Ġ i +r on +1 5 +Ġ1 5 +Ġcomm un +Ġm ark +ĠThe re +Ġw r +ĠTh at +Ġin formation +w ays +Ġb us +a pp +Ġinv est +m e +Ġh ard +ain ed +e ad +Ġim port +Ġapp ro +Ġt est +Ġt ri +Ġre st +os ed +Ġf ull +Ġc are +ĠS p +Ġc ase +O N +Ġs k +Ġl ess +Ġ + +Ġpart ic +ĠP l +ab ly +u ck +is hed +ch n +b e +Ġl ist +at or +Ġto p +Ġad v +ĠB e +ru ct +Ġd em +r ation +l ing +g y +re en +g er +Ġh ome +Ġle ft +Ġbet ter +Ġd ata +Ġ1 1 +Ġatt ack +Ġpro ble +l ine +ard s +Ġbe h +r al +ĠH ow +ĠS he +ar ge +Ġ -- +: // +Ġb ro +ĠP h +at s +Ġbu ild +w w +id ed +a im +as es +en cy +Ġm ain +in ed +Ġinclud ing +Ġ { +Ġg ot +Ġint erest +Ġke ep +Ġ X +Ġe as +ain ing +Ġcl ass +âĢ ¦ +ĠN o +Ġv ar +Ġsm all +amp le +A T +Ġ ide +ĠS o +Ġre ce +Ġpol it +Ġm ov +Ġpl an +Ġper cent +iv ing +Ġc amp +Ġp ay +1 4 +s c +is ed +Ġu nt +one y +pl oy +== == +Ġdid n +ĠI nd +el s +ert ain +Ġp os +__ __ +i ver +Ġpro cess +Ġprog ram +if ied +ĠR ep +1 6 +u ro +olog y +at ter +in a +Ġn ame +ĠA ll +Ġf our +Ġret urn +v ious +b s +Ġcall ed +Ġm ove +ĠS c +ir d +Ġgrou p +Ġb re +Ġm en +Ġc ap +t en +e e +Ġd ri +le g +he re +uth or +Ġp at +Ġcur rent +id es +Ġp op +t o +ent ion +Ġal ways +Ġm il +Ġwom en +Ġ1 6 +Ġo ld +iv en +ra ph +ĠO r +r or +ent ly +Ġn ear +ĠE x +re am +s h +Ġ1 4 +Ġf ree +iss ion +st and +ĠC on +al ity +us ed +1 3 +Ġdes ign +Ġch ange +Ġch ang +Ġb o +Ġv is +em ber +Ġb ook +read y +Ġk ill +2 5 +pp ed +Ġa way +Ġab le +Ġcount ry +Ġcon st +ar n +Ġor der +A R +i or +i um +or th +1 8 +ail able +Ġs w +Ġm illion +Ġ1 3 +at ic +t ed +ĠG o +Ġo per +en g +Ġth ing +aj or +con om +ĠCom m +Ġwh y +u red +ur al +Ġs chool +b y +ĠM ar +Ġa ff +Ġd ays +Ġan n +us h +an e +I f +e g +Ġpro f +Ġhe alth +ou th +B ut +ion al +. , +Ġs ol +Ġal ready +Ġ3 0 +Ġchar act +H e +Ġf riend +E S +i ans +ic le +' d +ĠO n +Ġle ast +Ġp rom +Ġd r +Ġh ist +it her +Ġ est +i qu +1 7 +s on +Ġte ll +Ġt alk +oh n +o int +le ction +A N +Ġunt il +au gh +Ġl ater +Ġ ve +Ġv iew +end ing +iv ed +Ġwor d +w are +Ġc ost +Ġen ough +Ġg ive +ĠUn ited +Ġte chn +are nt +O R +Ġp ar +ĠD r +Ġ201 6 +r ist +er ing +Ġ  +Ġl arge +s ide +ac y +cc ess +Ġw in +Ġimport ant +Ġ19 9 +Ġdoes n +Ġ1 7 +Ġbus iness +Ġcle ar +Ġre se +" , +ur y +Ġe qu +as ter +al f +ĠAmeric an +n ect +Ġex pect +ivers ity +Ġo cc +ĠF l +Ġk ind +Ġme an +Ġp ast +Ġde v +Ġb as +le t +ra ft +Ġor gan +Ġde l +Ġper form +Ġst ory +Ġse ason +ĠC ol +Ġcl aim +Ġc ame +Ġwith in +Ġl ine +Ġpro ject +ĠA t +Ġcontro l +end ed +ĠS y +Ġa ir +iz ation +Ġ * +le y +Ġm oney +id d +Y ou +f or +Ġfam ily +Ġm aking +Ġb it +Ġpol ice +Ġhapp en +Ġ vers +on y +u ff +ĠW hen +Ġs it +ide o +l f +is on +Ġsu re +g in +Ġapp ear +Ġl ight +Ġ es +o f +Ġw ater +Ġt imes +n ot +Ġg row +Ġcomp any +ĠT e +ow s +Ġm ar +our ce +i ol +ar m +b r +Ġex ample +Ġcon c +Ġf ore +ĠT o +p ro +E N +ri es +Ġ2 5 +ĠC an +ne y +Ġact ually +Ġe ver +ur ity +ak en +ap s +Ġt ax +Ġm ajor +am a +Ġof ten +er al +Ġhum an +Ġj ob +is ter +Ġav ailable +oc r +en n +a id +iv id +Ġrec ord +? " +Ġs ing +ĠA m +id ence +Ġnew s +st er +Ġe conom +Ġfollow ing +ĠB r +is ing +Ġh our +m ost +um ent +Ġse x +Ġdes c +Ġbec ome +ĠE d +Ġto ok +Ġha ving +Ġprodu ct +a ult +A s +ar ing +Ġme ans +Ġh op +un e +Ġch o +Ġc ertain +Ġn on +Ġde al +2 4 +le ment +oc i +en e +Ġs ide +ĠP r +ĠM ay +Ġre ason +u ed +c hed +ul ation +Ġe lect +Ġoffic ial +Ġposs ible +Ġh old +and s +ot s +Ġc ity +or ies +Ġse ver +Ġchild ren +Ġon ce +Ġact iv +l er +Ġn ight +it ions +ĠJ ohn +a pe +pl ay +Ġd one +Ġl im +Ġwork ing +ĠP res +or ld +e b +ĠC o +Ġb ody +ail s +ut es +ĠM r +Ġwhe ther +Ġa uthor +ro p +Ġpro per +Ġse en +) ; +Ġf ac +ĠS u +Ġcon d +it ing +Ġcour se +Ġ } +-------- -------- +a ign +Ġev ent +Ġen g +Ġp ot +Ġin tern +i am +Ġsh ort +em pt +ã Ĥ +ĠG od +il ar +8 0 +Ġor ig +I S +our n +ab ility +it ive +Ġd am +Ġ1 00 +Ġp ress +Ġdo ing +Ġprot ect +r ing +Ġthough t +Ġquest ion +re w +ĠW ar +Ġsever al +ĠSt ate +Ġg iven +Ġf und +ĠT w +Ġw ent +an ces +w ork +p or +m y +4 0 +Ġar g +art ment +ust om +Ġpol ic +Ġme et +Ġc reat +2 2 +ĠSt ates +Ġg ames +ra w +ut ure +Ġunder stand +ur s +ĠO b +l ish +s y +Ġm akes +Ġw on +ag on +Ġh tt +Ġl ove +ent ial +Ġcomple te +p ar +ĠI m +A L +Ġacc ount + ł +ore d +ver t +Ġ ident +Ġ201 5 +Ġother s +ĠM in +i ber +ver age +The re +ition al +d d +Ġpro b +Ġyou ng +Ġal ong +Ġacc ording +Ġy et +Ġmem bers +ĠWh at +o id +ĠM an +A nd +Ġam ong +a i +Ġem ploy +ĠR es +Ġ > +Ġinv ol +Ġl ow +a f +ĠC ar +Ġh ig +ĠO ne +ĠS ec +in ation +Ġlike ly +Ġan t +ag ed +ĠR uss +Ġb en +Ġre le +F or +b ack +ĠN ot +Ġpres ident +b all +Ġacc ess +ivid ual +ĠD em +ĠE uro +6 0 +Ġkn own +ir l +ĠG r +Ġear ly +u se +iet y +âĢ ĵ +Ġf ight +Ġs ent +Ġto day +Ġmark et +" . +Ġb ased +Ġstr ong +ur ther +Ġde b +m ber +Ġproble m +Ġde ath +Ġsoc ial +im ate +A S +ort un +Ġcamp aign +er y +C h +Ġe y +i ally +Ġm us +w h +p os +Ġ er +Ġsa f +Ġmonth s +ir on +Ġv iol +Ġf ive +Ġst re +Ġplay ers +in c +al d +y ear +a un +Ġsu ccess +Ġpres ent +ere nce +Ġ201 4 +Ġsu gg +Ġpartic ular +Ġtr y +Ġsugg est +ĠCh rist +on es +Ġpri v +2 3 +Ġc rit +Ġl and +Ġloc al +if y +2 9 +Ġa ut +E D +ĠG u +Ġm ult +Ġpolit ical +Ġask ed +Ġfor mer +it ter +ri pt +Ġcl ose +Ġp ract +ĠY ork +Ġget ting +Ġac ross +Ġcom b +Ġbelie ve +Ġ z +Ġto get +Ġtoget her +ĠC ent +ir c +Ġind ividual +ĠM c +2 7 +is k +ĠE ng +Ġf ace +Ġ2 4 +Ġval ue +Ġare a +e v +Ġw rit +ĠPres ident +Ġv ot +Ġke y +Ġm om +p ut +Ġany thing +Ġexper ience +att le +Ġm ind +a ff +om m +Ġf uture +g ed +Ġc ut +Ġto t +it ch +Ġv ideo +Ġinvest ig +Ġn et +ĠM y +r ict +i en +. ) +Ġimp ro +th ough +ward s +Ġcon nect +ĠM ed +sel ves +ens ive +m b +o ber +at ors +A n +Ġ5 0 +Ġre du +res ent +Ġab ove +Ġf re +ĠEuro pe +s w +Ġam ount +ĠA pp +Ġe ither +Ġmil it +Ġan al +Ġf ail +ĠE n +al es +Ġspec ial +Ġbl ack +I T +c her +Ġlook ing +Ġf ire +y n +Ġal most +o on +Ġstud y +Ġm iss +c hes +ro wn +Ġt re +Ġcommun ity +Ġmed ia +Ġf ood +Ġcom es +ĠUn iversity +Ġsing le +Wh at +u ly +Ġh alf +ag ue +h od +ĠRep ublic +Ġstart ed +Ġqu ick +ot o +b ook +Ġiss ue +it or +Ġel se +Ġcons ider +2 6 +ro du +Ġt aken +2 8 +9 9 +ĠW ith +Ġtr ue +Ġw a +Ġtr ad +Ġag o +Ġm ess +ie f +Ġadd ed +o ke +Ġb ad +Ġf av +3 3 +Ġsim ilar +as k +ĠD on +Ġcharact er +ort s +ĠH ouse +Ġreport ed +Ġty pe +v al +i od +ĠHow ever +Ġt arg +Ġent ire +pp ing +Ġhist ory +Ġl ive +ff ic +.... .... +ed eral +Ġtr ying +Ġdisc uss +ĠH ar +ac es +l ished +Ġse lf +os p +re st +Ġro om +el t +Ġf all +ol ution +Ġe t +Ġ x +Ġis n +Ġide a +b o +Ġs ound +ĠD ep +Ġsome one +ci ally +ull y +Ġf oc +Ġob ject +if t +ap er +Ġplay er +Ġr ather +Ġserv ice +as hing +ĠD o +ĠP art +ru g +m on +p ly +Ġm or +Ġnot hing +Ġprov ide +I C +un g +Ġpart y +Ġex ist +Ġm ag +7 0 +Ġr ul +Ġh ouse +Ġbeh ind +Ġhow ever +ĠW orld +Ġs um +Ġapp lic +Ġ ; +Ġfun ction +g r +ĠP ol +Ġfr ont +2 00 +Ġser ies +Ġt em +Ġty p +ill s +Ġo pt +Ġpoint s +Ġbel ow +itt ed +Ġspec ific +Ġ201 7 +um b +Ġr a +Ġpre vious +Ġpre t +re me +Ġc ustom +Ġcour t +ĠM e +Ġre pl +Ġwho le +g o +c er +Ġt reat +ĠA ct +Ġprob ably +Ġle arn +end er +ĠA ss +Ġvers ion +n ow +Ġche ck +ĠC al +R E +min ist +O n +our ces +Ġben ef +Ġd oc +Ġdet er +Ġen c +Ġsu per +Ġadd ress +Ġv ict +Ġ201 3 +Ġme as +t r +Ġf ield +W hen +Ġsign ific +u ge +Ġfe at +Ġcomm on +l oad +Ġbe gin +Ġbr ing +Ġa ction +er man +Ġdesc rib +Ġind ust +Ġwant ed +ri ed +m ing +Ġatt empt +4 5 +f er +Ġd ue +ress ion +# # +Ġsh all +Ġs ix +o o +Ġst ep +Ġp ub +Ġhim self +Ġ2 3 +Ġc op +Ġd est +Ġst op +A C +ib ility +Ġl ab +ic ult +Ġhour s +Ġcre ate +Ġf urther +ĠAmeric a +ĠC ity +Ġd ou +he ad +S T +ĠN orth +c ing +Ġn ational +u le +ĠIn st +Ġt aking +ĠQ u +ir t +Ġre d +Ġrese arch +v iron +ĠG e +Ġbre ak +an a +Ġsp ace +ater ial +Ġrec ent +ĠA b +Ġgener al +Ġh it +Ġper iod +Ġevery thing +ive ly +Ġph ys +Ġsay ing +an ks +Ġc ou +Ġc ult +ac ed +e al +u ation +Ġc oun +l u +Ġinclud e +Ġpos ition +ĠA fter +ĠCan ad +ĠE m +Ġim m +ĠR ed +Ġp ick +Ġcom pl +Ġm atter +re g +e xt +ang u +is c +o le +a ut +Ġcomp et +e ed +f ect +Ġ2 1 +ĠS en +ĠThe se +as ing +Ġcan not +Ġin it +Ġrel ations +ac hed +Ġb ar +Ġ4 0 +ĠT H +Ġ201 2 +Ġv ol +Ġg round +Ġsec urity +Ġup d +il t +3 5 +Ġconc ern +ĠJ ust +Ġwh ite +Ġseem s +ĠH er +pe cially +i ents +Ġann oun +Ġf ig +ight s +Ġst ri +l ike +id s +Ġs us +Ġw atch +Ġ â +Ġw ind +ĠC ont +Ġit self +Ġm ass +A l +y le +iqu e +ĠN ational +Ġab s +Ġp ack +Ġout side +Ġan im +Ġp ain +et er +Ġman ag +du ct +og n +Ġ ] +ĠSe pt +se c +o ff +ĠJ an +Ġf oot +ad es +Ġth ird +Ġm ot +Ġev idence +int on +Ġth reat +a pt +pl es +c le +Ġl o +Ġde cl +Ġit em +med i +Ġrep resent +om b +am er +Ġsignific ant +og raph +s u +Ġc al +i res +00 00 +I D +A M +Ġsim ply +Ġlong er +Ġf ile +O T +c he +S o +ate g +or g +ĠH is +Ġen er +Ġd om +Ġup on +il i +": " +Ġthem selves +Ġcom ing +Ġqu ite +Ġdiff icult +ĠB ar +il ities +re l +end s +c ial +6 4 +Ġwom an +ra p +y r +Ġne cess +ip s +Ġte xt +Ġrequ ire +Ġmilit ary +Ġre view +Ġresp ons +7 5 +Ġsub ject +Ġinst ead +Ġiss ues +Ġg en +" ," +Ġmin utes +Ġwe ap +r ay +am ed +t ime +b l +H ow +Ġc ode +ĠS m +Ġhig her +ĠSt e +r is +Ġp age +Ġstud ents +ĠIn tern +Ġmet hod +ĠA ug +ĠP er +ĠA g +Ġpolic y +ĠS w +Ġex ec +Ġac cept +um e +rib ut +Ġword s +Ġfin al +Ġchang es +ĠDem ocr +Ġfriend s +Ġres pect +Ġe p +Ġcomp an +iv il +Ġdam age +** ** +og le +viron ment +Ġne g +ent al +Ġa p +Ġtot al +iv al +! " +l im +Ġneed s +Ġag re +Ġdevelop ment +Ġa ge +ip le +2 1 +Ġresult s +ĠA f +S h +Ġg un +ĠOb ama +ro ll +Ġ @ +Ġright s +ĠB rit +Ġrun ning +Ġwas n +Ġp ort +Ġr ate +Ġpret ty +Ġtarg et +Ġsa w +Ġc irc +Ġwor ks +ic ro +al t +o ver +ww w +Th at +l ier +Ġevery one +ud e +Ġp ie +idd le +ra el +Ġr ad +Ġbl ock +Ġw alk +T o +ã ģ +n es +ĠA ust +a ul +ro te +ĠS outh +ess ion +op h +Ġshow s +Ġs ite +Ġj o +Ġr isk +cl us +l t +Ġin j +id ing +ĠS pe +Ġch all +ir m +Ġ2 2 +itt ing +st r +Ġh y +L E +ke y +Ġbe gan +at ur +ashing ton +l am +ĠD av +b it +Ġs ize +ĠP ar +3 8 +ourn al +f ace +Ġdec ision +Ġl arg +Ġj ud +re ct +Ġcontin ue +ĠO ct +ove red +ĠI nt +==== ==== +Ġp arent +ĠW ill +Ġeas y +Ġd rug +ang er +Ġs ense +Ġd i +id ay +Ġener gy +ist ic +Ġass oci +ar ter +ob al +e ks +ĠE l +ur ch +Ġg irl +o e +it le +Ġ2 8 +ĠC he +Ġrequ est +Ġso on +Ġh ost +k y +Ġst ates +om es +Ġm aterial +le x +Ġmom ent +Ġan sw +on se +Ġes pecially +Ġn orm +Ġserv ices +p ite +r an +Ġro le +4 4 +) : +Ġc red +C l +____ ____ +Ġm at +Ġl og +ĠCl inton +O U +Ġoff ice +Ġ2 6 +Ġch arg +Ġtr ack +m a +Ġhe art +Ġb all +Ġperson al +Ġbuild ing +n a +s et +b ody +ĠBl ack +Ġincre ase +itt en +Ġneed ed +3 6 +3 2 += " +Ġl ost +Ġbec ame +Ġgrou ps +ĠM us +Ġw rote +ĠP e +Ġpro p +j oy +à © +ĠWh ite +Ġde ad +. ' +Ġhtt p +Ġwe bs +O S +Ġins ide +Ġwr ong +Ġstat ement +Ġ ... +y l +Ġfil m +Ġmus ic +Ġsh are +ific ation +Ġre lease +Ġfor ward +Ġst ay +Ġcomp ut +it te +s er +Ġorig inal +Ġc ard +Ġc and +Ġd iv +at ural +Ġfav or +O M +Ġc ases +us es +Ġse ction +Ġle ave +g ing +ov ed +ĠW ashington +3 9 +ĠG l +Ġrequ ired +act ion +ap an +o or +it er +ĠK ing +Ġcount ries +ĠG erman +ll ing +Ġ2 7 +3 4 +Ġquest ions +Ġpr im +Ġc ell +Ġsh oot +Ġany one +ĠW est +Ġaff ect +ep end +Ġon line +ĠIs rael +ĠSept ember +Ġab ility +Ġcont ent +is es +Ġre ve +Ġl aun +Ġind ic +Ġfor ce +c ast +Ġso ld +av ing +f l +Ġso ft +Ġcompan ies +ce ed +Ġart icle +Ġa ud +Ġre v +Ġed uc +Ġplay ing +0 5 +Ġhe ld +ct or +Ġrele ased +Ġf ederal +3 7 +Ġad minist +Ġinter view +Ġinst all +Ġrece ived +Ġs ource +u k +P h +Ġser ious +Ġcre ated +Ġc ause +Ġim medi +Ġdef in +u el +ĠDep artment +ct ions +ĠC our +ĠN ow +z e +it es +it ution +Ġl ate +Ġspe ak +n ers +Ġleg al +ar i +ĠC or +Ġwe eks +Ġmod el +Ġp red +Ġex act +B C +ĠB y +IN G +os ing +Ġt akes +Ġreg ard +Ġopp ortun +Ġpr ice +Ġ19 8 +ĠA pr +f ully +Ġor d +Ġproble ms +ru ction +h am +ĠC ount +le ge +Ġlead ers +E T +le v +Ġde ep +olog ical +es e +h aps +ĠS ome +Ġp ers +Ġcont ract +Ġrelations hip +s p +ou d +Ġb ase +4 8 +m it +A d +anc ial +Ġcons um +Ġpot ential +Ġl angu +re m +et h +Ġrel ig +ress ed +6 6 +Ġl ink +Ġl ower +ay er +ĠJ une +Ġf em +un t +er c +ur d +Ġcont act +Ġ ill +Ġm other +Ġest ab +h tt +ĠM arch +ĠB ro +ĠCh ina +Ġ2 9 +Ġs qu +Ġprov ided +Ġa verage +as ons +Ġ201 1 +Ġex am +l in +5 5 +n ed +Ġper fect +Ġt ou +al se +u x +Ġbu y +Ġsh ot +Ġcol lect +Ġph ot +Ġplay ed +Ġsur pr +Ġofficial s +Ġsim ple +av y +Ġindust ry +Ġhand s +g round +Ġp ull +Ġr ound +Ġus er +Ġr ange +u ary +Ġpriv ate +op s +e es +Ġw ays +ĠM ich +Ġve h +Ġex cept +Ġter ms +im um +pp er +I ON +ore s +ĠDr agon +ou l +Ġd en +Ġperform ance +Ġb ill +c il +4 7 +Ġen vironment +Ġex c +ad d +Ġwor th +Ġp ict +Ġch ance +Ġ201 8 +b or +Ġspe ed +ict ion +Ġal leg +ĠJ apan +at ory +re et +Ġm atch +ĠI I +Ġst ru +ord er +Ġst e +Ġl iving +Ġst ruct +in o +Ġse par +her n +Ġresp onse +Ġen joy +Ġv ia +A D +um ents +ace book +Ġmem ber +ib r +iz ing +Ġto ol +ĠM on +ĠWh ile +h ood +ĠA ng +ĠD ef +Ġoff er +T r +a ur +Ġturn ed +ĠJ uly +d own +an ced +Ġrec ently +ĠE ar +Ġc e +ĠSt ar +ĠC ong +rough t +Ġbl ood +Ġhop e +Ġcom ment +ain t +Ġar ri +il es +Ġpartic ip +ough t +ri ption +0 8 +4 9 +Ġg ave +Ġse lect +Ġkill ed +sy ch +Ġgo es +i j +Ġc oll +Ġimp act +at ives +ĠS er +0 9 +ĠAug ust +Ġb oy +d e +ĠD es +Ġf elt +U S +Ġexpect ed +Ġim age +ĠM ark +cc ording +o ice +E C +ĠM ag +en ed +h old +ĠP ost +Ġpre vent +N o +Ġinvol ved +Ġey es +Ġquick ly +A t +un k +Ġbeh av +Ġ ur +Ġl ed +c ome +e y +Ġcand id +Ġear lier +Ġfoc us +et y +P ro +led ge +ix ed +ill ed +Ġpop ular +A P +Ġset t +l ight +Ġvar ious +in ks +Ġlevel s +Ġro ad +ell ig +ab les +he l +itte e +ĠG ener +y pe +Ġhe ard +ic les +Ġm is +Ġus ers +ĠS an +Ġimpro ve +Ġf ather +Ġse arch +The y +v il +Ġprof ess +Ġkn ew +Ġl oss +Ġev ents +6 5 +Ġb illion +0 7 +0 2 +ĠNew s +ĠA M +Ġco ver +w here +ens ion +Ġb ott +Ġare as +en ces +op e +ĠTw itter +a el +Ġget s +ĠGo ogle +Ġs n +i ant +Ġv ote +Ġnear ly +Ġinclud ed +Ġrec ogn +z z +m m +al ed +Ġhappen ed +0 4 +Ġh ot +Ġwho se +Ġc ivil +Ġsu ff +o es +it iz +ĠSy ri +Ġresp ond +Ġh on +Ġfeat ures +Ġeconom ic +ĠApr il +r im +Ġtechn ology +Ġo ption +ag ing +Ġpur ch +R e +Ġl at +ch ie +is l +Ġrec omm +u f +Ġtr aining +Ġeffect s +Ġf ast +Ġ201 0 +Ġocc ur +Ġwebs ite +Ġem ail +Ġs ens +e ch +Ġo il +Ġinf lu +Ġcurrent ly +ĠS ch +ĠAd d +Ġgo al +Ġsc ient +Ġcon v +1 00 +em y +Ġdec ided +Ġtra vel +Ġm ention +L L +0 3 +Ġe lection +Ġph one +Ġlook s +Ġsit uation +Ġc y +Ġh or +b ed +ĠCour t +a ily +av es +Ġqu ality +ĠCom p +w ise +Ġt able +Ġst aff +ĠW ind +et t +Ġtri ed +ide red +Ġadd ition +Ġb ox +Ġl ack +ar ily +Ġw ide +Ġm id +Ġbo ard +ys is +Ġant i +h a +Ġd ig +en ing +Ġd ro +C on +6 8 +Ġsl ow +b ased +se qu +Ġp ath +E x +ak er +Ġwork ed +Ġp en +Ġeng ine +Ġlook ed +ĠSu per +ĠS erv +Ġvict im +U n +Ġproper ty +Ġint rodu +Ġexec ut +ĠP M +L e +Ġcol or +ĠM ore +Ġ6 0 +Ġnet work +Ġd ate +c ul +id ge +Ġext ra +3 1 +Ġs le +6 7 +Ġw ond +Ġreport s +j ust +ĠAust ral +Ġcap ital +Ġen s +Ġcomm and +Ġallow ed +Ġpre p +Ġca pt +h ib +Ġnum bers +ch an +Ġf air +m p +om s +Ġre ach +W ith +t ain +Ġbro ad +Ġcou ple +ec ause +ly ing +ĠF eb +Ġsc reen +Ġl ives +Ġpri or +ĠCong ress +A r +Ġappro ach +Ġe mer +ar ies +ĠD is +s erv +ĠN e +Ġbu ilt +c ies +Ġre pe +Ġrul es +for ce +ĠP al +Ġfin ancial +Ġcons idered +ĠCh ar +n ces +ĠI S +Ġb rought +Ġb i +i ers +ĠS im +O P +Ġproduct s +Ġvis it +Ġdoc ument +Ġcon duct +Ġcomplete ly +in ing +ĠCal if +ib ly +Ġwr itten +ĠT V +em ents +Ġd raw +O ne +Ġpub lished +Ġsec ret +r ain +he t +ĠF acebook +ond ay +ĠU p +Ġsex ual +Ġth ous +ĠP at +Ġ ess +Ġstand ard +Ġar m +g es +ect ion +Ġf ell +Ġfore ign +an i +ĠFr iday +Ġreg ular +in ary +Ġincre ased +Ġus ually +Ġdem on +Ġd ark +Ġadd itional +ro l +ĠO f +Ġprodu ction +! ! +und red +Ġintern ational +id ents +ĠF ree +rou p +Ġr ace +Ġm ach +Ġh uge +A ll +le ar +ove mber +Ġto wn +Ġatt ention +ĠO ff +y ond +ĠThe n +f ield +Ġter ror +ra z +ĠB o +Ġmeet ing +ĠP ark +Ġar rest +Ġf ear +Ġa w +ĠV al +or ing +' , +Ġext reme +ar r +Ġwork ers +A fter +Ġ3 1 +n et +am ent +Ġdirect ly +Ġpop ulation +ub e +ĠOct ober +ĠI N +ĠJan uary +5 9 +ĠDav id +Ġc ross +ce mber +ĠF irst +Ġmess age +ir it +Ġn ation +Ġp oll +is ions +Ġansw er +n y +is ode +Ġcar ry +ĠRuss ia +Ġhe ar +eng th +ro y +Ġn atural +in ally +Ġdo g +m itted +Ġtr ade +Ġsub st +Ġmult iple +ĠAf ric +Ġf ans +Ġs ort +Ġgl obal +ic ation +ĠW ed +ar a +Ġa chie +Ġlangu age +ve y +Ġt al +Ġnecess ary +Ġdet ails +Ġs en +ĠS und +ĠRe g +ĠR ec +0 6 +Ġs il +ress ive +Ġmed ical +un ch +orn ia +Ġu nd +f ort +oc ks +ĠM onday +ues day +c raft +7 7 +ur t +Ġ ver +ĠH ill +Ġrece ive +Ġmor ning +es tern +Ġb ank +Ġs at +ir th +ĠH igh +Ġdev ice +ĠTH E +ĠCent er +Ġsaf e +Ġp le +ĠCanad a +Ġsystem s +Ġass ist +Ġsur v +Ġb attle +ĠS oc +vert is +S he +Ġp aper +Ġgrow th +Ġc ast +S c +Ġpl ans +ll ed +Ġpart s +Ġw all +Ġmove ment +Ġpract ice +im ately +Ġdis play +Ġsomet imes +om p +ĠP aul +ĠY es +k ing +5 8 +o ly +Ġs on +Ġav oid +ok es +ĠJ ew +Ġto wards +as c +Ġ // +ĠK ore +Ġtalk ing +Ġcor rect +Ġsp ent +ic ks +i able +e ared +Ġter m +Ġwant s +om ing +Ġ ut +Ġdou b +Ġfor ces +Ġp lease +6 9 +ĠN ovember +at form +ond on +Ġon es +Ġimmedi ately +ĠRuss ian +ĠM et +Ġde g +Ġparent s +C H +ĠAmeric ans +al y +ĠM od +Ġsh own +Ġcond itions +Ġst uff +Ġre b +ĠY our +Ġinclud es +n own +ĠS am +Ġexper ien +m ission +ĠE ven +augh t +Ġannoun ced +ĠRepublic an +Ġdeter min +Ġdescrib ed +ĠCount y +( ) +Ġdo or +Ġchang ed +Ġne igh +ĠH ere +Ġcle an +Ġp an +ĠDe cember +ĠEurope an +ir ing +ap ter +Ġcl ub +ĠT uesday +Ġp aid +ĠN et +Ġattack s +Ġcharact ers +Ġal one +Ġdirect or +d om +Ġ3 5 +Ġl oad +Ġr out +ĠCalif ornia +Ġfin ally +Ġr ac +Ġcont r +Ġexact ly +res h +p ri +ĠIs lam +Ġn ature +Ġcare er +Ġlat est +Ġcon vers +ĠS l +p ose +ci ent +ĠIn c +iv ity +8 8 +ĠA tt +ĠM or +nes day +Ġwe ight +k en +Ġnot e +Ġteam s +Ġ \ +air s +ĠG reen +Ġh undred +on ent +Ġstre ng +Ġcons ist +ic ated +Ġreg ul +Ġl ic +ast ic +Ġt en +urs day +ellig ence +ous ly +ĠU K +B I +Ġcost s +Ġind epend +ĠA P +Ġnorm al +Ġh om +Ġob vious +Ġs we +Ġst ar +Ġread y +ac her +Ġimp lement +g est +Ġs ong +ĠG et +ĠL ab +Ġinterest ing +us ing +Ġg iving +ĠSund ay +Ġet c +Ġm iddle +Ġrem ember +r ight +os ition +ut ions +Ġm ax +4 6 +Ġyour self +Ġdem and +Ġtreat ment +Ġd anger +ĠC ons +Ġgu y +ĠBrit ish +Ġphys ical +Ġrel ated +Ġrem ain +Ġcould n +Ġref er +Ġc itiz +b ox +EN T +bo ard +Ġin n +I G +er o +ĠSt reet +osp ital +ren ch +cher s +Ġst ra +O L +ag er +ĠA N +Ġeas ily +I A +en ge +in y +Ġcl os +ock ed +Ġus es +ĠC oun +I m +u ild +? ? +m ore +Ġan g +Ġwr ite +ol ute +5 7 +Ġlead er +Ġread ing +< / +Ġaut om +est s +4 3 +Ġleg isl +ĠG old +Ġdesign ed +ĠS T +ĠLe g +a res +Ġbe aut +ĠT ex +Ġappear s +Ġstru gg +ĠR om +Ġ 00 +Ġcho ice +Ġparticular ly +ĠF rom +op er +ĠL ondon +ann ed +Ġallow s +ob ile +Ġdiffere nce +âĢ ¢ +ĠV iew +ĠWed nesday +Ġal though +Ġrel ative +Ġapplic ation +ate ver +Ġare n +Ġmy self +Ġim ag +Ġdis e +Ġsoc iety +Ġfre qu +ĠEng lish +Ġpo or +ĠD ay +Ġwrit ing +Ġse ven +Ġstart ing +Ġb ud +Ġpr int +ĠTr ans +uf act +ĠSt ud +n ew +Ġcr im +Ġg ives +Ġco ol +a e +i ance +ĠGener al +Ġthink ing +Ġsa ve +Ġlim ited +ĠPart y +Ġmean ing +p en +ow ers +ĠJ ack +E M +Ġn ice +ru pt +Ġg as +Ġe ight +Ġfe et +Ġeff ort +Ġ ign +ic it +B l +co in +Ġop in +Ġbr ain +Wh ile +he st +ĠTh ursday +Ġwould n +augh ter +Ġtou ch +le ments +Ġstud ies +Ġcent er +c ont +or ge +Ġcomput er +Ġinvestig ation +P l +or ks +Ġ200 8 +Ġincre asing +Ġst ore +Ġcom ments +Ġb al +m en +Ġdo ll +Ġl iber +Ġw ife +Ġlaw s +atur day +it ness +Ġmod ern +ĠS k +Ġadminist ration +Ġopportun ity +Ġs al +Ġpower ful +M y +Ġclaim s +ĠEar th +ord s +Ġt itle +Ġes c +n ame +N ot +om en +Ġbe yond +Ġc amer +Ġse ll +it ute +ear ch +Ġapp l +im ent +4 2 +ĠAr t +Ġun f +Ġviol ence +ur g +ĠE ast +Ġcomp ared +Ġopt ions +Ġthrough out +Ġv s +ig r +. [ +ac hes +7 8 +Ġfil es +F L +E L +ar ian +ĠJ ames +ĠA ir +an ch +Ġdet ail +Ġpie ce +P S +Ġn amed +Ġeduc ation +Ġdri ve +Ġitem s +Ġstud ent +ic ed +: : +ic o +Ġth row +Ġsc ene +Ġcomple x +Ġ200 9 +Ġpre c +ĠB re +7 9 +Ġcon cept +Ġstat us +am ing +Ġd ied +Ġknow ledge +Ġbegin ning +O D +ru ary +Ġcertain ly +Ġgu ys +Ġsl ight +in n +ound s +Ġf ine +Ġf at +ic ations +Ġper haps +ĠA nt +Ġinc ome +Ġhtt ps +Ġmajor ity +port s +st on +Ġgreat er +Ġfe ed +ent ially +Ġsaf ety +Ġun ique +and om +Ġg one +Ġshow ed +Ġhist or +Ġcoun ter +i us +id a +Ġlead ing +i pe +Ġs end +ĠDon ald +er ve +Ġdef ense +ines e +Ġy es +ĠF ire +ĠMus lim +ra q +Ġcontin ued +os h +Ġprov ides +Ġpr ison +ĠP re +Ġhapp y +Ġeconom y +Ġtr ust +ag s +ĠG ame +Ġweap ons +um an +ĠC le +it ation +Ġanal ysis +ĠT imes +Ġsc ience +- > +Ġfig ure +Ġdis app +ent y +Ġsoft ware +Ġu lt +Ġoffic ers +N ew +I s +Ġrem ains +ĠInd ia +Ġp sych +ri ef +Ġc at +es c +Ġob serv +Ġst age +ĠD ark +Ġent er +ch ange +Ġpass ed +Ġdes pite +ĠO ut +Ġmov ie +r s +Ġv oice +m ine +ĠPl ay +Ġto ward +ĠT er +Ġreg ion +Ġval ues +or ters +Ġm ount +Ġoffic er +ĠO ther +b an +Ġh ous +w ood +ro om +I V +ĠS un +se e +ĠO ver +ro g +9 0 +Ġl ay +ĠT ur +a wn +Ġpress ure +ĠS ub +Ġbook s +ed om +ĠS and +A A +ag o +Ġre asons +f ord +Ġactiv ity +U T +N ow +ĠSen ate +ce ll +n ight +Ġcall s +in ter +Ġlet ter +ĠR ob +ĠJ e +Ġcho ose +ĠL aw +G et +B e +Ġro b +Ġtyp es +Ġpl atform +Ġqu arter +R A +ĠT ime +Ġmay be +ĠC r +9 5 +p re +Ġmov ing +Ġl if +Ġgo ld +Ġs om +Ġpat ients +Ġtr uth +ĠK e +ur ance +ant ly +m ar +Ġchar ge +ĠG reat +Ġce le +---------------- ---------------- +Ġro ck +ro id +an cy +Ġcred it +a ud +B y +ĠE very +Ġmov ed +ing er +rib ution +Ġn ames +Ġstra ight +ĠHe alth +ĠW ell +Ġfe ature +Ġr ule +Ġsc he +in ated +ĠMich ael +ber g +4 1 +il ed +b and +Ġcl ick +ĠAng el +on ents +Â Ń +ĠI raq +ĠS aturday +Ġa ware +p art +Ġpat tern +O W +ĠL et +Ġgr ad +ign ed +Ġassoci ated +Ġst yle +n o +i ation +a ith +il ies +Ġst ories +ur ation +Ġindividual s +ĠâĢ ¦ +m iss +ĠAss oci +ish ing +ab y +Ġsum mer +ĠB en +Ġ3 2 +Ġar ch +ut y +ĠTex as +h ol +Ġfull y +Ġm ill +Ġfollow ed +ĠB ill +ĠInd ian +ĠSec ret +ĠB el +ĠFeb ruary +Ġjob s +Ġseem ed +ĠGo vern +i pped +Ġreal ity +Ġl ines +Ġp ark +Ġmeas ure +ĠO ur +I M +Ġbro ther +Ġgrow ing +Ġb an +Ġest im +Ġc ry +ĠS chool +Ġme chan +ĠO F +ĠWind ows +Ġr ates +ĠO h +Ġpos itive +Ġcult ure +ist ics +ic a +Ġh ar +y a +ite ly +i pp +Ġm ap +en cies +ĠWill iam +I I +ak ers +5 6 +ĠM art +ĠR em +Ġal tern +it ude +Ġco ach +row d +D on +Ġk ids +Ġj ournal +Ġcor por +Ġf alse +Ġwe b +Ġsle ep +Ġcont ain +Ġst o +Ġb ed +iver se +ĠR ich +ĠCh inese +Ġp un +Ġme ant +k nown +Ġnot ice +Ġfavor ite +a ven +Ġcond ition +Ġpur pose +) ) +Ġorgan ization +Ġchall eng +Ġman ufact +Ġsus p +ĠA c +Ġcrit ic +un es +uc lear +Ġm er +vent ion +Ġ8 0 +Ġm ist +ĠU s +ĠT or +htt p +ol f +Ġlarg er +Ġadv ant +Ġrese ar +Ġact ions +m l +Ġke pt +Ġa im +, ' +c ol +Ġbenef its +if ying +Ġact ual +ĠIntern ational +Ġveh icle +Ġch ief +Ġeff orts +ĠLe ague +ĠM ost +Ġwa it +Ġad ult +Ġover all +Ġspe ech +Ġhigh ly +Ġfem ale +Ġer ror +Ġeffect ive +5 4 +Ġenc our +w ell +Ġfail ed +Ġcons erv +Ġprogram s +Ġt rou +Ġa head +5 00 +vertis ement +I P +ĠF ound +p ir +Ġ % +Ġcr ime +and er +Ġloc ation +ĠI ran +Ġbehav ior +az ing +Ġr are +Ġem b +Ġca used +Ġsh ip +Ġact ive +Ġcont ribut +Ġg reen +Ġac qu +Ġref lect +ven ue +Ġf irm +Ġb irth +] . +Ġclear ly +Ġem ot +Ġag ency +ri age +Ġmem ory +9 8 +S A +ĠSe e +ac ing +C C +Ġbig gest +Ġr ap +Ġbas ic +Ġb and +e at +Ġsus pect +ĠM ac +Ġ9 0 +m ark +ist an +Ġsp read +am s +k i +as y +ra v +ĠR ober +Ġdemon str +r ated +Ġabs olute +Ġpl aces +Ġim pl +ibr ary +Ġc ards +Ġdest roy +Ġv irt +ve re +Ġapp eared +y an +p oint +Ġbe g +Ġtem per +s pe +ant ed +ear s +ĠD irect +Ġl ength +Ġbl og +am b +Ġint eg +Ġres ources +ac c +if ul +Ġsp ot +Ġfor ced +Ġthous ands +ĠMin ister +Ġqu al +ĠF rench +at ically +Ġgener ally +Ġdr ink +Ġth us +I L +od es +Ġappro pri +ĠRe ad +Ġwh om +Ġey e +Ġcol lege +Ġ4 5 +ire ction +Ġens ure +Ġapp arent +id ers +Ġrelig ious +Ġmin or +ol ic +Ġt ro +ĠWh y +rib ute +m et +Ġprim ary +Ġdevelop ed +Ġpe ace +Ġsk in +st e +av a +Ġbl ue +Ġfam ilies +Ġ ir +Ġapp ly +Ġin form +ĠSm ith +C T +i i +Ġlim it +Ġres ist +........ ........ +um n +Ġconf lic +Ġtw e +ud d +ĠT om +Ġl iter +qu e +b on +Ġha ir +Ġevent ually +Ġp us +Ġhelp ed +Ġag g +or ney +ĠApp le +Ġf it +ĠS ur +Ġpre m +Ġs ales +Ġsecond s +Ġstreng th +Ġfeel ing +¿ ½ +Ġt our +Ġknow s +o om +Ġex erc +Ġsom ew +ï ¿½ +> > +Ġsp okes +Ġide as +Ġreg ist +so ft +ĠD el +ĠP C +Ġpro pos +Ġlaun ch +Ġbott om +T H +ĠP lease +v est +it z +ĠIn ter +Ġsc ript +Ġr at +ar ning +Ġ il +ĠJ er +ĠA re +Ġwh atever +ok en +ci ence +Ġmod e +Ġag ree +Ġs ources +Ġinit ial +Ġrest rict +Ġwond er +us ion +## ## +ĠS il +vil le +Ġb urn +t w +as ion +Ġ £ +Ġn or +u ing +Ġre ached +Ġs un +Ġc ateg +ig ration +Ġc ook +Ġprom ot +Ġm ale +Ġcl imate +Ġf ix +Ġalleg ed +U R +all ed +Ġim ages +C ont +ot a +Ġschool s +i os +Ġd rop +Ġst ream +ĠM o +Ġprevious ly +al ing +Ġp et +Ġdou ble +Ġ( @ +ann el +Ġdef ault +t ies +Ġr ank +ĠD ec +ĠCoun cil +Ġweap on +Ġst ock +Ġanal y +ĠSt r +Ġpict ure +ĠPol ice +f erence +Ġcent ury +Ġcitiz ens +Ġon to +Ġexp and +Ġhe ro +ĠS ol +Ġw ild +Ġupd ate +Ġcustom ers +r ont +d ef +Ġl ik +Ġcrim inal +ĠChrist ian +S P +7 6 +Ġle aving +Ġother wise +ĠD ist +Ġbas is +5 2 +5 3 +ic ip +ĠB er +Ġrecomm end +Ġfl oor +Ġc rowd +ol es +Ġ7 0 +Ġcent ral +ĠE v +Ġd ream +Ġdown load +Ġconf ir +ĠTh om +Ġwind ow +Ġhapp ens +Ġun it +Ġt end +Ġs pl +Ġbec omes +Ġfight ing +Ġpred ict +ĠP ress +ĠP ower +Ġhe avy +ak ed +Ġf an +or ter +ate gy +B A +iz es +Ġsp end +H ere +Ġ200 7 +Ġad op +ĠH am +Ġfoot ball +ĠP ort +od ay +5 1 +amp ions +Ġtrans fer +h t +Ġ3 8 +ter m +ac ity +Ġb ur +] , +tern al +r ig +b ut +Ġthere fore +ĠB ecause +res p +re y +Ġm ission +S ome +Ġnot ed +Ġass um +Ġdise ase +Ġed it +Ġprog ress +r d +ĠB rown +oc al +Ġadd ing +Ġra ised +ĠAn y +Ġt ick +Ġsee ing +ĠPe ople +Ġagre ement +Ġser ver +Ġw at +Ġdeb ate +Ġsupp osed +il ing +Ġlarg est +Ġsuccess ful +ĠP ri +ĠDemocr atic +Ġj ump +ĠSyri a +Ġown ers +Ġoff ers +Ġshoot ing +Ġeff ic +se y +Ġha ven +ver se +te red +ĠL ight +im al +ĠB ig +Ġdef end +Ġbe at +Ġrecord s +% ) +Ġsc en +Ġemploy ees +Ġdev ices +he m +Ġcom mer +ĠM ex +Ġbenef it +ĠPro f +Ġil leg +Ġsur face +ĠAl so +Ġh arm +ing ly +w ide +ĠA lex +Ġsh ut +ĠC ur +Ġl ose +p m +Ġchall enge +se mb +Ġst ation +Ġint elligence +Ġacc ur +ĠFl or +Ġrequ ires +ĠM al +b um +Ġh ospital +Ġsp irit +Ġoff ered +Ġprodu ce +ĠComm un +Ġcreat ing +Ġcr is +s pect +Ġend ed +Ġd aily +Ġvot ers +land s +i as +i h +on a +Ġsm art +ĠOff ice +ĠL ord +ri al +ĠIntern et +Ġcirc um +Ġextreme ly +' . +Ġopin ion +ĠM il +Ġg ain +B S +ĠF in +y p +Ġuse ful +Ġbud get +Ġcom fort +is f +Ġback ground +el ine +Ġep isode +Ġen emy +Ġtri al +Ġestab lish +d ate +ĠC ap +Ġcontin ues +Ġshow ing +ĠUn ion +w ith +Ġpost ed +ĠSy stem +Ġe at +ri an +Ġr ise +ĠGerman y +il s +Ġsign ed +Ġv ill +Ġgr and +m or +ĠEng land +Ġproject s +um ber +Ġconf erence +z a +Ġrespons ible +ĠAr ab +Ġlearn ed +âĢĶ âĢĶ +i pping +ĠGe orge +O C +Ġreturn ed +ĠAustral ia +Ġb rief +Q u +Ġbr and +ill ing +ab led +Ġhig hest +Ġtr ain +ĠComm ission +wh ile +Ġn om +cept ion +Ġm ut +ĠBl ue +Ġinc ident +v ant +8 6 +ĠI D +Ġn uclear +7 4 +ĠL ike +ĠR E +ĠM icro +l i +m ail +Ġcharg es +8 9 +Ġad just +ad o +Ġear th +N A +Ġpr ices +P A +Ġd raft +Ġrun s +Ġcandid ate +ens es +Ġmanag ement +ĠPh il +ĠM iss +Ġte ach +g ram +Ġunderstand ing +a it +ic ago +A dd +ĠE p +sec ut +Ġsepar ate +Ġinst ance +Ġe th +Ġun less +**** **** +ĠF ore +in ate +Ġoper ations +S p +Ġf aith +g ar +ĠCh urch +ron ic +Ġconf ig +os ure +Ġactiv ities +Ġtrad itional +Ġ3 6 +Ġd irection +Ġmach ine +Ġsur round +Ġp ush +un ction +ĠE U +Ġeas ier +Ġarg ument +G B +Ġm icro +Ġsp ending +iz ations +Ġthe ory +ad ow +Ġcall ing +ĠL ast +Ġd er +Ġinflu ence +Ġcomm it +Ġph oto +Ġun c +ist ry +g n +ast e +ack s +Ġdis p +ad y +d o +ĠG ood +Ġ ` +Ġw ish +Ġreve aled +Âł Âł +l ig +Ġen force +ĠComm ittee +Ġche m +Ġmil es +Ġinterest ed +Ġsol ution +ic y +in ct +Ġ- > +ĠD et +Ġrem oved +Ġcomp ar +e ah +Ġpl ant +ĠS ince +Ġachie ve +Ġadvant age +Ġslight ly +b ing +Ġpl aced +u nder +201 5 +ĠM ad +Ġt im +os es +Ġc ru +ĠR ock +Ġmost ly +Ġneg ative +Ġset ting +Ġprodu ced +Ġm ur +Ġconnect ion +ĠM er +Ġdri ver +Ġexecut ive +Ġass ault +Ġb orn +ĠV er +t ained +Ġstruct ure +Ġredu ce +Ġdec ades +Ġd ed +u ke +ĠM any +idd en +Ġle ague +S e +Ġjo in +Ġdis co +Ġd ie +c ks +act ions +Ġass ess +ag n +Ġgo als +our s +I R +Ġsen ior +ill er +m od +ip ment +oc ol +u y +ĠQ ue +Ġpart ies +ir gin +Ġle arning +it able +Ġstre et +Ġcamer a +A pp +Ġsk ills +b re +c ious +Ġcele br +ĠFr anc +Ġexist ing +Ġwill ing +l or +Ġ id +ĠSp ace +Ġcrit ical +ĠL a +ortun ately +Ġser ve +Ġc old +Ġspec ies +T S +Ġanim als +ĠB ay +Ġold er +ĠU nder +est ic +ĠT re +Ġte acher +Ġpre fer +v is +Ġth read +ĠM att +Ġmanag er +ãĥ » +Ġprofess ional +ĠV ol +Ġnot es +The se +ul a +Ġf resh +ent ed +u zz +ed y +clus ion +ĠR el +Ġdoub t +E O +Ġopen ed +ĠB it +Ad vertisement +Ġgu ess +ĠU N +Ġse qu +Ġexpl ain +ott en +Ġatt ract +ak s +Ġstr ing +Ġcont ext +oss ible +ĠRepublic ans +Ġsol id +Ġc ities +Ġask ing +Ġr andom +u ps +ur ies +ar ant +dd en +g l +ĠFlor ida +Ġdep end +ĠSc ott +Ġ3 3 +Ġi T +ic on +Ġmention ed +Ġ2 000 +Ġclaim ed +Ġdefin itely +ul f +Ġc ore +Ġopen ing +ĠCon st +wh ich +ĠT ra +A G +7 2 +Ġbelie ved +ad a +Ġ4 8 +ĠSec urity +yr ight +ĠP et +ĠL ou +Ġhold ing +======== ======== +Ġ ice +Ġb row +Ġauthor ities +h ost +w ord +Ġsc ore +ĠD iv +Ġcell s +Ġtrans l +Ġneigh bor +Ġrem ove +u ct +Ġdist rict +ĠA ccording +Ġwor se +Ġconcern s +Ġpresident ial +Ġpolic ies +ĠH all +7 3 +Ġh us +A Y +Ġ200 6 +ĠJ ud +Ġindepend ent +ĠJust ice +ili ar +pr int +igh ter +Ġprotect ion +z en +Ġsu dden +h ouse +ĠJ es +P R +ĠIn f +Ġb ul +Ġ _ +ĠServ ice +ĠP R +Ġstr ategy +ff ect +Ġgirl s +Ġmiss ing +oy al +ĠTe am +ul ated +Ġd at +Ġpolit ics +ab or +A ccording +Ġspe ll +Ġg raph +ort hern +T C +A b +Ġlab or +is her +Ġk ick +ĠiT unes +Ġstep s +pos es +Ġsmall er +E n +ber t +Ġro ll +Ġresear chers +Ġcl osed +Ġtrans port +Ġlaw y +________ ________ +ĠCh icago +Ġas pect +Ġn one +Ġmar riage +9 6 +Ġe lements +ĠF re +ĠS al +Ġd ram +F C +t op +e qu +Ġhe aring +Ġsupport ed +Ġtest ing +co hol +Ġmass ive +Ġst ick +Ġgu ard +is co +ph one +F rom +How ever +Ġb order +Ġcop y +ograph y +l ist +7 1 +Ġown er +cl ass +ru it +r ate +ĠO nce +Ġdig ital +Ġt ask +ER S +Ġinc red +t es ++ + +ĠFr ance +Ġb reat +ow l +Ġiss ued +ĠW estern +Ġdet ect +Ġpart ners +Ġsh ared +ĠC all +Ġcan cer +ac he +rib e +Ġexpl ained +Ġhe at +{ " +Ġinvest ment +ĠB ook +Ġw ood +Ġtool s +ĠAl though +Ġbelie f +Ġcris is +Ġg e +ĠM P +Ġoper ation +ty pe +~ ~ +g a +Ġcont ains +ant a +Ġexp ress +ĠG roup +ĠJ ournal +k a +Ġam b +ĠUS A +Ġfind ing +Ġfund ing +h ow +Ġestab lished +ide os +Ġdeg ree +Ġdanger ous +ang ing +Ġfre edom +pp ort +out hern +Ġch urch +Ġc atch +ĠTw o +Ġpres ence +ĠGu ard +U p +Ġauthor ity +ĠPro ject +Ġbut ton +Ġcon sequ +Ġval id +Ġwe ak +Ġstart s +Ġref erence +ĠM em +" ) +U N +or age +ĠO pen +Ġcol lection +y m +g ency +Ġbeaut iful +ro s +Ġtell s +Ġwa iting +n el +Ġprov iding +ĠDemocr ats +Ġd aughter +Ġm aster +Ġpur poses +ĠJapan ese +Ġequ al +Ġturn s +Ġdoc uments +Ġwatch ing +R es +Ġr an +201 4 +Ġre ject +ĠKore a +Ġvictim s +Le vel +ere nces +Ġw itness +Ġ3 4 +Ġre form +com ing +Ġocc up +Ġc aught +Ġtra ffic +ad ing +Ġmod els +ar io +Ġserv ed +Ġb atter +u ate +ĠSecret ary +Ġagre ed +Ġtr uly +yn am +ĠR et +Ġun its +ĠRes earch +h and +az ine +ĠM ike +Ġvar iety +ot al +Ġam azing +Ġconfir med +Ġentire ly +Ġpurch ase +Ġe lement +Ġc ash +Ġdeter mine +D e +Ġc ars +ĠW all +â ĸ +Ġview s +Ġdrug s +Ġdep artment +ĠSt ep +u it +Ġ3 9 +as ure +ĠCl ass +Ġc overed +ĠB ank +Ġme re +u ana +Ġmult i +Ġm ix +Ġun like +lev ision +Ġsto pped +Ġs em +ĠG al +ul es +Ġwe l +ĠJohn son +l a +Ġsk ill +Ġbec oming +ri e +Ġappropri ate +f e +ell ow +ĠPro t +ul ate +oc ation +Ġweek end +od ies +Ġsit es +Ġanim al +ĠT im +Ġsc ale +Ġcharg ed +Ġinst ruct +ill a +Ġmethod s +Ġc ert +Ġjud ge +ĠH el +Ġdoll ars +Ġstand ing +ĠS qu +Ġdeb t +l iam +Ġdri ving +ĠS um +ĠEd ition +Ġal bum +and on +I F +ĠU k +6 3 +ad er +Ġcommer cial +es h +ĠGovern ment +Ġdisc overed +Ġout put +ĠHill ary +ĠCar ol +Ġ200 5 +Ġab use +anc ing +Ġsw itch +Ġann ual +T w +Ġst ated +ag ement +in ner +Ġdem ocr +Ġres idents +Ġallow ing +Ġfact ors +od d +Ġf uck +em ies +Ġoccur red +ot i +Ġn orth +ĠP ublic +Ġinj ury +Ġins urance +C L +oll y +ã Ģ +Ġrepe ated +Ġar ms +ang ed +Ġconst ruction +Ġf le +P U +ic ians +Ġfor ms +ĠMc C +ant ic +Ġm ental +p ire +Ġequ ipment +Ġf ant +Ġdiscuss ion +Ġregard ing +k in +ar p +Ġch air +og ue +Ġpro ceed +ĠI d +O ur +Ġmur der +M an +Ġ4 9 +as p +Ġsupp ly +Ġin put +Ġwe alth +liam ent +Ġpro ced +or ial +ĠSt at +ĠN FL +hen s +ĠInst itute +Ġput ting +ourn ament +et ic +Ġloc ated +Ġk id +er ia +r un +Ġpr inc +Ġ ! +go ing +ĠB et +Ġcl ot +Ġtell ing +Ġprop osed +i ot +or ry +Ġfund s +g ment +ĠL ife +Ġb aby +ĠB ack +Ġsp oke +Im age +Ġear n +ĠA T +g u +Ġex change +ĠL in +ov ing +Ġp air +M ore +az on +Ġarrest ed +Ġkill ing +c an +ĠC ard +y d +Ġident ified +Ġm obile +Ġthan ks +ony m +ĠF orm +Ġhundred s +ĠCh ris +ĠC at +Ġtre nd +h at +ĠA v +om an +Ġelect ric +ĠW il +S E +O f +Ġrest aur +ot ed +Ġtr ig +Ġn ine +Ġb omb +Wh y + ¯ +Ġco verage +Ġapp eal +ĠRober t +ĠS up +Ġfin ished +Ġfl ow +Ġdel iver +Ġcal cul +Ġphot os +Ġph il +Ġpie ces +Ġapp re +k es +Ġr ough +D o +Ġpart ner +Ġconcern ed +Ġ3 7 +ĠG en +C ol +ct ors +Ġ= > +st ate +Ġsuggest ed +ĠFor ce +C E +Ġher self +ĠPl an +w orks +o oth +ren cy +Ġcor ner +Ġhus band +Ġintern et +ĠA ut +em s +os en +ĠAt l +g en +Ġbal ance +6 2 +Ġsound s +te xt +Ġar r +ov es +Ġmill ions +Ġrad io +Ġsat isf +ĠD am +M r +G o +S pe +Ġcomb at +r ant +ĠG ree +Ġf uel +Ġdist ance +Ġtest s +Ġdec re +ĠE r +Ġman aged +D S +Ġt it +Ġmeas ures +ĠL iber +Ġatt end +as hed +ĠJ ose +ĠN ight +d it +ĠN ov +ĠE nd +out s +Ġgener ation +Ġadv oc +y th +Ġconvers ation +ĠS ky +act ive +ce l +ri er +ĠFr ank +Ġg ender +Ġcon cent +Ġcar ried +and a +ĠV irgin +Ġarri ved +ic ide +ad ed +Ġfail ure +Ġmin imum +le ts +Ġwor st +Ġkeep ing +Ġint ended +Ġilleg al +Ġsub sc +Ġdetermin ed +Ġtri p +Y es +Ġra ise +Ġ ~ +Ġfeel s +Ġpack age +ĠJ o +h i +201 6 +re al +Ġf ra +Ġsy mb +M e +uck y +p ret +ĠK h +ĠEd it +ĠWe b +em ic +ĠCol or +Ġjust ice +I nt +Ġfar m +ck now +" > +el ess +Ġredu ced +Ġ5 00 +x x +ĠR ad +ĠW ood +Ġcl in +Ġhy p +il er +ur a +k ins +8 5 +6 1 +ĠThe ir +ĠM ary +Ġs an +Ġno vel +ĠWh o +Ġcap acity +Ġimp ossible +Ġpl ays +Ġmin ister +ij uana +ic ate +ĠS et +Ġf ram +Ġ ing +Ġcommun ities +ĠF BI +it a +Ġb on +Ġstr ateg +Ġinterest s +l ock +g ers +m as +ĠAN D +Ġconflic t +Ġrequire ments +Ġs ac +Ġoper ating +in i +rel ated +Ġcomm itted +Ġrelative ly +Ġs outh +¯ ¯ +Ġaff ord +Ġident ity +Ġdec isions +Ġacc used +pl ace +Ġvict ory +o ch +i at +N ame +C om +t ion +ed s +Ġsee k +Ġt ight +ĠIm ages +Ġinit i +Ġhum ans +Ġfam iliar +Ġaud ience +Ġintern al +vent ure +Ġs ides +ĠT O +Ġd im +Ġcon clud +Ġapp oint +Ġenforce ment +ĠJ im +ĠAssoci ation +Ġcircum st +ĠCanad ian +Ġjo ined +Ġdiffere nces +ĠL os +Ġprot est +Ġtw ice +w in +Ġgl ass +ars h +ĠAr my +Ġexp ression +Ġdec ide +Ġplan ning +an ia +Ġhand le +ĠMicro soft +ĠN or +Ġmax imum +ĠRe v +Ġse a +Ġev al +Ġhel ps +re f +Ġb ound +Ġm outh +Ġstand ards +Ġcl im +ĠC amp +ĠF ox +cl es +Ġar my +ĠTe chn +ack ing +x y +S S +Ġ4 2 +Ġbu g +ĠUk rain +ĠM ax +ĠJ ones +ĠSh ow +l o +Ġplan et +Ġ7 5 +Ġwin ning +Ġf aster +Ġspe ct +Ġbro ken +T R +Ġdef ined +Ġhealth y +Ġcompet ition +htt ps +ĠIs land +ĠF e +Ġannoun ce +ĠC up +ĠInst ead +Ġcl ient +Ġposs ibly +se ction +ock et +l ook +Ġfin ish +Ġcre w +Ġres erv +Ġed itor +Ġh ate +Ġs ale +Ġcontro vers +Ġp ages +w ing +Ġnum er +Ġopp osition +Ġ200 4 +Ġref uge +Ġfl ight +Ġap art +ĠL at +A meric +ĠAfric a +Ġapplic ations +ĠPal est +ĠB ur +Ġg ar +ĠSoc ial +Ġup gr +Ġsh ape +Ġspe aking +ans ion +a o +ĠS n +Ġwor ry +ĠBrit ain +P lease +rou d +Ġh un +Ġintrodu ced +Ġd iet +I nd +ĠSec ond +Ġfun ctions +ut s +ĠE ach +ĠJe ff +Ġst ress +Ġaccount s +Ġgu arant +ĠAn n +ed ia +Ġhon est +Ġt ree +ĠAfric an +ĠB ush +} , +Ġs ch +ĠOn ly +Ġf if +ig an +Ġexerc ise +ĠEx p +Ġscient ists +Ġlegisl ation +ĠW ork +ĠS pr +à Ĥ +ĠH uman +Ġ è +Ġsur vey +Ġr ich +ri p +Ġmain tain +Ġfl o +Ġleaders hip +st ream +ĠIslam ic +Ġ 01 +ĠCol lege +Ġmag ic +ĠPr ime +Ġfig ures +201 7 +ind er +x ual +ĠDe ad +Ġabsolute ly +Ġfour th +Ġpresent ed +resp ond +rib le +Ġal cohol +at o +ĠD E +por ary +Ġgr ab +Ġvar i +Ġqu ant +ĠPh oto +Ġpl us +r ick +ar ks +Ġaltern ative +Ġp il +Ġappro x +th at +Ġobject s +ĠR o +ĠAnd roid +Ġsignificant ly +ĠR oad +k ay +R ead +av or +Ġa cknow +ĠH D +ĠS ing +O r +ĠM ont +Ġun s +pro f +Ġneg oti +ĠAr ch +ik i +Ġte levision +ĠJew ish +Ġcomm ittee +Ġmot or +Ġappear ance +Ġs itting +Ġstri ke +ĠD own +com p +ĠH ist +Ġf old +ac ement +ĠLou is +Ġbel ong +ĠâĢ ¢ +Ġm ort +Ġprep ared +Ġ6 4 +ĠM aster +Ġind eed +ĠD en +Ġre nt +T A +our ney +ar c +S u +9 7 +Ġadv ice +Ġchang ing +Ġlist ed +Ġlaun ched +is ation +ĠP eter +is hes +Ġl ived +ĠM el +ĠSup reme +ĠF ederal +Ġ) ; +ruct ure +Ġset s +Ġphil os +u ous +Ġ ł +Ġappl ied +ĠN OT +Ġhous ing +ĠM ount +Ġo dd +Ġsu st +D A +ffic ient +Ġ ? +ol ved +Ġp owers +Ġth r +Ġrem aining +ĠW ater +L C +Ġca uses +ãģ ® +Ġman ner +ad s +Ġsuggest s +Ġend s +stand ing +f ig +ĠD un +id th +Ġg ay +Ġter min +ĠAngel es +M S +Ġscient ific +Ġco al +ap ers +b ar +ĠThom as +Ġsy m +ĠR un +th is +P C +igr ants +Ġmin ute +ĠDist rict +cell ent +Ġle aves +Ġcomple ted +am in +Ġfoc used +Ġmon itor +Ġveh icles +M A +ĠM ass +ĠGr and +Ġaffect ed +itution al +Ġconst ruct +Ġfollow s +Ġt on +re ens +Ġh omes +ĠE xt +ĠLe vel +r ast +ĠI r +Ġel im +Ġlarge ly +ĠJ oe +Ġvot es +all s +Ġbusiness es +ĠFound ation +ĠCent ral +Ġy ards +Ġmaterial s +ul ner +Ġgu ide +Ġclos er +um s +Ġsp orts +ed er +J ust +Ġtax es +8 4 +ĠO ld +Ġdec ade +ol a +Ġv ir +Ġdro pped +Ġdel ay +it ect +Ġsec ure +ste in +le vel +Ġtre ated +Ġfil ed +ain e +Ġv an +Ġm ir +Ġcol umn +ict ed +e per +Ġro t +Ġcons ult +Ġent ry +Ġmar ijuana +ĠD ou +Ġapparent ly +ok ing +clus ive +Ġincre ases +an o +Ġspecific ally +Ġte le +ens ions +Ġrelig ion +ab ilities +Ġfr ame +ĠN ote +ĠLe e +Ġhelp ing +Ġed ge +ost on +Ġorgan izations +à ĥ +ĠB oth +hip s +Ġbig ger +Ġbo ost +ĠSt and +Ġro w +ul s +ab ase +Ġr id +L et +are n +ra ve +Ġst ret +P D +Ġv ision +Ġwe aring +Ġappre ci +Ġa ward +ĠU se +Ġfact or +w ar +ul ations +) ( +Ġg od +Ġter rit +Ġpar am +ast s +8 7 +Ġen emies +ĠG ames +F F +Ġacc ident +W ell +ĠMart in +T ER +Ġat h +ĠHe ll +Ġfor g +Ġve ter +ĠMed ic +f ree +Ġst ars +Ġexp ensive +Ġac ad +ra wn +ĠW he +Ġl ock +Ġform at +Ġsold iers +s m +Ġag ent +Ġrespons ibility +or a +ĠS cience +Ġrap id +Ġt ough +ĠJes us +Ġbelie ves +M L +Ġwe ar +le te +Ãĥ ÃĤ +ĠD ri +Ġcomm ission +ĠB ob +O h +ap ed +Ġwar m +ÃĥÃĤ ÃĥÃĤ +Ġ200 3 +ort ion +Ġhas n +ust er +Ġun ivers +ĠI ll +Ġk ing +olog ies +9 4 +ĠT em +ĠM os +Ġpat ient +ĠMex ico +ce an +ĠDe ath +ĠSand ers +y ou +ĠC ast +ĠComp any +pt y +Ġhappen ing +F P +ĠB attle +Ġb ought +A m +M od +U s +ut ers +ĠC re +ĠTh ose +Ġ4 4 +is er +Ġs oul +ĠT op +ĠHar ry +ĠA w +Ġse at +ff ee +Ġrev olution +Ġ( " +ĠD uring +et te +Ġr ing +Ġoff ensive +Ġreturn s +Ġv ideos +Ġdis cl +Ġfam ous +en ced +ĠS ign +ĠR iver +Ġ3 00 +P M +ĠB us +ĠC H +Ġcandid ates +ard en +Ġpercent age +Ġvis ual +Ġthan k +Ġtrou ble +ner gy +Ġ200 1 +Ġpro ve +ash ion +Ġen h +ĠL ong +U M +Ġconnect ed +Ġposs ibility +O ver +Ġexper t +Ġl ibrary +art s +ĠDirect or +Ġfell ow +9 2 +ir ty +Ġd ry +Ġsign s +ĠL ove +Ġqu iet +f oot +Ġp ure +ĠH un +Ġf illed +ph as +ĠE lect +end ment +ĠEx pl +Ġun able +n s +m o +Ġv ast +ob e +Ġident ify +app ing +ĠCarol ina +g ress +Ġpro te +Ġf ish +Ġcircumst ances +raz y +ĠPh ot +Ġb odies +ĠM ur +Ġdevelop ing +ĠA R +Ġexperien ced +Ġsubst ant +ĠBo ard +es ome +Ġdom estic +Ġcomb ined +ĠP ut +Ġchem ical +ĠCh ild +Ġpo ol +ĠC y +Ġe gg +c ons +st ers +Ġh urt +Ġmark ets +Ġconserv ative +Ġsupp orters +Ġag encies +id el +O b +ur b +Ġ4 3 +ĠDef ense +y e +ĠA p +du le +Ġtemper ature +Ġconduct ed +ĠCh ief +Ġpull ed +Ġf ol +L ast +ont o +os is +V ER +D es +ĠP an +F irst +Ġadv ance +Ġlic ense +r ors +ĠJ on +Ġimag ine +Ġhe ll +Ġf ixed +Ġinc or +os ite +ĠL og +ick en +] : +Ġsurpr ise +h ab +Ġc raft +ol t +ĠJ ul +Ġd ial +Ġrele vant +Ġent ered +Ġlead s +ĠA D +ĠCle an +Ġpict ures +ess or +Ġal t +Ġpay ing +P er +ĠMark et +Ġupd ates +am ily +ĠT ype +ĠH ome +Ġ5 5 +semb ly +rom e +8 3 +Ġgreat est +Ġhe ight +Ġhe av +ain ts +Ġlist en +as er +ĠS H +Ġcap able +ac le +Ġpers pect +in ating +Ġoff ering +ry pt +ĠDe velop +ab in +r c +Ġbr ight +al ty +ar row +Ġsupp l +ind ing +ack ed +gy pt +ĠAn other +p g +ĠVirgin ia +ĠL u +Ġpl anned +Ġp it +Ġswe et +T ype +ĠD i +Ġtyp ically +ĠFranc isco +Ġpro spect +ĠD an +Ġte en +re es +Ġsc hed +Ġh ol +Ġsc r +Ġlot s +l ife +Ġnews p +Ġfor get +ĠN one +ĠM iddle +ĠR yan +ed d +Ġse vere +Ġsu it +ll er +9 3 +Ġcor respond +Ġexpl os +u ations +Ġfl ag +g ame +r id +Ġpr in +ĠD ata +Ġde ploy +ĠEn ter +su it +gh an +ĠM en +Ġthough ts +Ġmat ters +Ġad apt +ĠA ri +Ġf ill +Ġfor th +Ġs am +Ġ4 1 +Ġpay ment +ĠH or +Ġsp ring +du c +Ġl osing +Ġbring ing +F O +al a +Ġdist ribution +he red +b our +ĠIsrael i +om a +Ġcomb ination +Ġpl enty +V E +C an +ĠH aw +Ġper man +ĠSpe cial +Ġto w +Ġsee king +Ġexam ples +Ġclass es +c r +Ġbe er +Ġmov es +ĠI P +ĠK n +Ġpan el +E ven +Ġproper ly +Ġr is +Ġpl ug +Ġestim ated +E very +Ġdef ensive +ag raph +Ġpre gn +Ġinst it +ĠV ict +Ġvol ume +Ġpos itions +Ġl inks +ĠPro gram +ĠWe ek +ag ues +Ġtrans form +k er +ĠC EO +Ġc as +Ġopp onent +Ġtwe et +ĠC ode +Ġsh op +Ġf ly +Ġtal ks +Ġb ag +Ph one +Ġa id +Ġpl ants +Ġ6 5 +Ġatt orney +ar ters +qu est +ĠMag ic +Ġbeg ins +Ġmy ster +Ġenvironment al +Ġst orage +N N +Ġm arg +Ġs ke +Ġmet al +ell y +Ġord ered +Ġrem ained +Ġl oved +Ġprom pt +Ġupd ated +Ġexper ts +Ġwalk ing +Ġan cient +Ġperform ed +AT E +Ġne ither +i ency +Ġmanufact ure +ĠP ak +Ġselect ed +Ġm ine +Ġult imately +Ġexpl an +Ġlab el +ĠServ ices +ribut ed +Tr ump +Ġsy n +ĠU lt +S C +Ġme at +Ġg iant +ĠW ars +ĠO N +Ġad m +Ġinter pret +Ġeven ing +Ġev il +ĠB oston +ĠW ild +Ġ à +ĠBit coin +ĠAm azon +D r +ĠIn formation +Ġobvious ly +Ġadv anced +Ph oto +ol ar +Ġwe ather +Ġsymb ol +Ġso le +Ġpot entially +ost er +Ġorig inally +m un +3 00 +az e +ess ions +Ġde ck +Ġst ood +Ġyou th +ĠB ern +R ep +ĠT est +Ġbas ically +ot ic +Ġinvol ve +ol it +ly n +S ee +Ġair craft +Ġconf irm +E W +Ġmess ages +ĠRich ard +Ġk it +Ġpro hib +Ġv ulner +is ters +Ġexist ence +Ġturn ing +ĠS P +Ġdes ire +Ġfl at +Ġm ent +se ason +ang es +Ġneighbor hood +ĠL ake +AT ION +Ġpoint ed +b ur +Ġinn ov +uc ks +U L +Ġprofess or +Ġexp ressed +A B +ic ious +Ġ200 2 +ĠDe v +Ġs ession +Ġb are +s en +Ġdis s +ĠC ath +ĠP ass +ĠP oint +Ġdo ctor +or row +ail ed +ĠR ub +ĠD C +ĠChar l +p erson +Ġwrit er +igh ters +ure au +Ġob lig +Ġrecord ed +Ġbro ke +Ġord ers +il ty +Ġmot ion +in ity +l aw +ad ium +Ġimm igration +Ġcontr ast +Ġb att +Ġex cellent +Ġtechn ical +am i +Ġt un +Ġcl oud +ĠY ear +ge on +Ġcre ation +Ġstr ange +Ġa uth +Ġfor t +b orn +Ġext ent +ĠT oday +ĠCl ub +Ġr ain +Ġs ample +Ġaccept ed +Ġt act +Ġf ired +ĠS on +Ġstand s +Ġb oot +Ġ4 7 +Ġstat ements +Ġvers ions +Ġse lling +ound ed +Ġ199 0 +Ġwere n +ĠW atch +Ġexper iment +P ost +Ġret ail +ul ed +In st +un te +ãĥ ¼ +Ġdep art +Ġb ond +i very +om pl +Ġre action +ĠSyri an +ĠP ac +app ed +ani el +D P +Ġres olution +Ġre act +Ġappro ved +on om +m ond +ĠO ffic +-- - +Ġrepl ace +Ġt ack +Ġsp ort +Ġch ain +Ġemer gency +r ad +ĠPalest in +Ġ4 6 +Ġautom atically +Ġrout e +Ġp al +Ġb anks +ĠPar is +ĠMed ia +ro ad +ic ing +i xt +ist ed +Ġg rew +Ġco ord +ĠW here +om in +Ġsub s +� � +Ġ ± +Ġcorpor ate +Ġse lection +n oon +ĠRep ort +c s +clud ing +ord ers +anc he +ĠIt s +Ġslow ly +ĠE gypt +ĠA cc +Ġcol le +iqu es +E X +Ġattempt s +ur l +ĠC ross +Ġfind ings +ĠS C +ĠO R +Ġind ex +ens ity +ĠW ay +ĠL and +Ġsh ock +d is +Ġd ynam +Ġc art +m osp +S ince +i est +ĠB oy +Ġst orm +ĠCont in +201 3 +he w +il it +Ġess ential +iqu id +O ther +ive red +Ġreason able +A ct +Ġsub sequ +ĠP ack +ĠF ort +Ġconsider ing +Ġun iversity +l og +Ġmar ried +Ġill ust +ĠTr ue +£ ı +Ġnumer ous +rast ructure +Ġserious ly +Ġrefer red +u a +Ġconsist ent +on na +ĠRe al +ru ption +ci ples +Ġfact s +9 1 +ot es +er g +The n +Ġacc ompl +N ote +Ġre venue +Ġpass ing +Ġm al +e en +ĠY et +Ġg ather +ter day +ew ork +ĠA uthor +P e +Ġopt im +Ġr ub +Ġè £ı +Ġun known +st one +Ġun ion +ol ve +Ġopportun ities +Ġbrow ser +ĠW al +ĠC ost +Ġreport ing +st s +p et +Ġs and +Ġsudden ly +Ġsurpr ising +ĠV R +Ġsomew hat +ĠB as +ult ure +iz z +ĠC D +Ġchalleng es +Ġsett ings +Ġexperien ces +ĠF ull +Ġcan n +Ġrece iving +ES T +Ġj oint +Ġcult ural +Ġa st +8 2 +as tern +ce ived +ĠC ru +Ġb ull +p ired +am m +Ġfac ing +p ower +Ġb oss +ĠH ol +Ġinst r +Ġincreasing ly +Ġsh ift +Ġstre ets +ĠWilliam s +ab b +Ġl ie +Ġl augh +ĠC a +P L +Ġadult s +Ġcustom er +Ġob tained +Ġsupport ing +ht ml +f ire +Ġdetail ed +Ġpick ed +ĠR ight +ld er +E E +st ood +ĠK im +Ġw ire +Ġs ight +Ġdevelop ers +Ġpers ons +Ġs ad +Ġc up +Ġwar ning +Ġboy s +l ong +Ġb ird +f o +Ġw al +Ġobserv ed +Ġz one +iven ess +Ġch annel +c ript +Ġref used +ĠAg ain +Ġsu c +Ġspokes man +ĠRe f +r ite +ou ston +ãĥ ³ +ĠS her +Ġact s +ĠN ame +Ġstrugg le +ar ry +omet imes +Ġdisc rim +H T +Ġcateg ory +Ġreal ize +Ġemploy ee +ĠAf ghan +en ger +Ġgun s +ĠSte ve +ĠM ot +ĠO l +ok ed +Ġth ick +Ġfair ly +ill y +Ġsur ve +ĠM at +we ight +â Ķ +Ġtro ops +Ġag ents +Ġbatter y +Ġmot iv +à ¡ +S ec +d en +o very +L S +Ġfl u +Ġconf ident +ĠO per +Ġem pty +Ġp hen +Ġse ctor +Ġexc ited +Ġrem ote +ap h +o en +Ġdestroy ed +Ġmor al +ĠH P +ĠR on +Ġd ress +ĠB at +Ġl it +ĠM S +Ġa f +H L +r um +is ms +Ġshould n +Ġsym pt +ĠTor onto +het ic +Ġcar bon +Ġinstall ed +Ġviol ent +Ġsol ar +j a +Ġpract ices +Ġr ide +ĠP enn +Ġimpro ved +Ġaud io +Ġbehav i +ĠP S +Ġe ating +D ata +ĠRe view +p ass +cl aim +u ated +ang ers +c hen +Ġproper ties +Ġany where +An other +Ġbl ow +ĠJack son +Ġp roud +Ġplan e +l ines +Ġsqu are +Ġpro of +ans as +Ġtalk ed +m akers +Ġs ister +Ġhold s +Ġres ident +Ġ= = +Ġresist ance +Ġspl it +Ġpro secut +Ġconf idence +res ents +Ġcut s +Ġexcept ion +Ġz ero +Get ty +Ġcop yright +Ġtot ally +orm al +ific ations +ĠAustral ian +Ġs ick +Ġ1 50 +Ġhouse hold +Ġfe es +Ġdri vers +og en +ĠN Y +Ġnecess arily +Ġregul ations +ear ing +s l +Ġperspect ive +c are +ic ial +H is +Ġesc ape +Ġsurpr ised +ĠV an +ur rent +Ġv ac +8 1 +ĠTh us +Ġem phas +ĠCh ampions +ĠI ce +Ġn arr +Ġhead s +Ġca using +b el +f ortunately +ĠM a +Ġtarg ets +ci pl +Ġafter noon +Ġadd s +ĠMay be +ĠF our +ess ed +ple te +Ġus ual +ch o +ing u +Ġwith d +ĠE nergy +ĠE conom +O O +Ġart icles +Ġinj ured +Ġman age +Ġexpl ains +Ġdi agn +R ec +at ures +Ġlink ed +Ġdiscuss ed +Ġexpl o +Ġocc asion +ath an +Ġopp osite +Ġfac es +Ġden ied +ĠK night +Ġn ut +Ġapprox imately +Ġdisapp oint +onym ous +ĠB est +ĠL o +ĠH y +ĠA ff +Ġvot ing +an while +ĠII I +Ġinstit utions +ag ram +ĠD aily +Ġdr ag +Ġnear by +Ġgu ilty +Ġcon ver +P re +s hip +Ġre ward +Ġphilos oph +ĠS S +u gh +Ġapp s +f riend +Ġu pper +Ġad vert +Ġs now +Ġfr ust +Ġour selves +F r +ĠD ie +amp ion +Ġdis miss +Ġc ere +Ġsign al +f rom +Ġ ). +Ġ5 2 +Ġcr imes +it ors +est ival +use um +Ġcoun cil +ĠS aud +M ay +ĠG un +ic ian +et her +Ġsu fficient +ĠH en +so le +Ġhistor ical +ĠF ar +ĠT urn +Ġp in +Ġsuc ceed +m at +ly mp +Ġtrad ition +ĠO k +Ġc ro +Ġdesc ription +al le +Ġsk y +T e +Ġwide ly +Ġw ave +Ġdefin ition +ĠJew s +Ġcy cle +Ġref ere +Ġbr ings +us al +Ġal ive +Ġfrequ ently +Ġint ention +ĠCont rol +l v +y stem +Ġpriv acy +g ent +ren ce +ĠQu est +ĠChrist mas +Ġr ail +Ġco oper +Ġtest ed +ĠC apt +as ks +Ġcomfort able +Ġdel ivered +sc ape +Ġdep th +ĠG OP +Ġwrit es +Ġass ets +Ġsa v +im ents +Ġtrans ition +Ġart ist +ĠL ook +Ġl ob +Ġcomp onents +ar ity +Ġwalk ed +Ġro ot +Ġparticip ants +Ġnot iced +Ġres c +Ġn av +ĠAd minist +d a +ut ral +pl ate +Ġimport ance +Ġass ert +ious ly +c ription +Ġinj uries +ĠChe ck +Ġregist ered +Ġint ent +Ġmiss ed +ograph ic +Ġsent ence +oun ter +Ġassist ance +ev in +Ġdat abase +Ġbuild ings +Ġclass ic +Ġth inks +ĠOh io +P r +ug g +Ġfe e +p an +Ġeffect ively +Ġfac ility +Ġbe ar +Ġch apter +Ġdog s +ĠCol umb +Ġl atter +it ial +Ġad mitted +T V +ĠGe org +Ġpost s +\ \ +Ġlawy er +Ġequ ival +Ġm and +Ġcontro lled +ĠW alk +ĠAnd rew +Ġmen u +am ental +Ġprotect ed +v a +Ġadminist r +or al +Ġre in +ĠS ar +Ġamount s +Ġn ative +ĠM oon +Ġrep resents +Ġab andon +Ġcarry ing +Ġt ank +m ary +Ġdecl ared +T ube +Ġh at +Ġpun ish +el lect +m es +Ġun iverse +ĠR od +ph y +Ġinf rastructure +Ġ5 1 +Ġopp osed +ow nt +c a +ĠM ake +Ġhard ware +Ġco ffee +R el +b al +w orld +ĠS af +ĠSe a +in als +Ġown ed +Ġh all +ers ion +Ġdescrib e +ĠP ot +Ġport ion +Ġat mosp +Ġgovern ments +Ġdep ending +Ġoff ense +Ġtr ick +aw a +ĠL ine +ĠV is +ĠH ard +ĠOr ig +ĠCl ick +Ġdes k +ĠVal ley +ĠS ov +Ġmov ies +Ġrem ark +Ġm ail +Ġcons cious +Ġrul ing +ĠR ights +Ġmed ic +he nt +ĠW omen +> < +Ġrepl aced +ĠP rem +ĠTh anks +Ġre new +ĠB all +if orm +Ġsh ots +C omm +Ġar med +Ġconst ant +Ġt aste +Ġreal ized +Ġbu ff +Ġm o +Ġeffic ient +M ost +or ation +if ies +Ġcommun ication +Ġfl ood +Ġconsequ ences +Ġany way +ig g +ĠG M +ĠTh ank +Ġ iron +Ġev olution +ĠC op +tw itter +Ġ9 5 +Ġrelationship s +ad el +ĠYou ng +Ġpropos al +ay ers +uild ing +ĠH ot +OR E +c os +Ġcoll abor +P G +ax y +Ġknow ing +Ġsupport s +ow ed +Ġcontrol s +Ġmere ly +um er +Ġath let +Ġf ashion +p ath +Ġg ift +Ġer a +AN D +Ġkind s +ĠKore an +Ġleg it +ul ous +Ġess entially +Ġthe rap +n ic +Ġsuff ered +Ġh ur +Ġprom ise +Ġex cess +Ġover w +Ġpr ime +ĠH ouston +er ry +ĠM s +R S +201 2 +Ġst ores +ĠO lymp +Ġj ourney +Al though +S ub +ĠE duc +ĠCh apter +Ġrequest s +Ġconsum ers +Ġt iny +Ġis ol +ĠF air +b a +ĠY OU +Ġcr ash +ce ler +Ġemot ional +Ġgood s +Ġelect ed +Ġmod er +ĠLin ux +Ġbl ocks +Ġis land +ĠSoc iety +Ġelect ions +Ġbroad cast +Ġche ap +Ġn ations +Ġse asons +4 00 +Ġwas te +ĠS at +Ġfield s +em ploy +Ġprof ile +Ġauth ors +AL L +ĠG ra +w est +ĠT y +Ġdeath s +Ġv acc +Ġfor med +Ġd u +Ġon going +ĠMuslim s +el f +ig ure +Ġass ume +ĠUkrain e +w ater +Ġco ast +Ġvot ed +g or +ĠA S +ĠMich igan +az a +ĠAr m +i ro +Ġf lex +as ters +' ' +Ġwel come +ar l +Ġloc ations +ig ation +ĠF il +Ġbu ying +Ġarch itect +Ġhard er +ĠC ub +Ġinter face +Ġrestaur ant +Ġdisco ver +Ġex ceed +Ġfav our +ger y +Ġd uty +Ġp itch +ad or +ĠM ach +b oy +Ġrespond ed +Ġext ended +her s +M any +ra id +if er +ĠIn s +S er +Ġmed ium +s he +ĠS ports +Ġmag azine +ut ation +Ġlim its +ĠG all +Ġex ternal +raz il +Ġyoung er +t le +Ġrem ind +ĠC ON +Ġimmedi ate +Ġh idden +Ġvol unte +Ġsim pl +od cast +Ġph ase +d r +Ġpl ot +Ġexp osure +R I +og rap +v in +an ish +ĠAc ad +ĠEng ine +Ġexp ansion +ĠP ay +Y our +Ġpus hed +ĠE ll +ĠHe ad +Ġmarket ing +ĠA C +k et +Ġh its +Ġg ro +ĠA ge +ĠSc ot +] [ +Ġst im +Ġi Phone +Ī Ĵ +Ġn arrow +ĠGet ty +ĠTur key +Ġperfect ly +Ġen able +ut ch +Ġprec ise +Ġreg ime +Ġsh if +Ġcomp ens +g un +d iv +Ġch osen +ĠK en +An y +Ġtre es +Ġrecomm ended +ĠR en +u able +ĠH T +F ollow +E G +ĠH and +ĠK enn +Ġarg uments +Ġex ists +Ġb ike +ĠCons erv +Ġbre aking +ĠG ar +Ġc razy +Ġvirt ual +ay lor +ix el +Ġ19 80 +Ġper mission +ĠSer ies +Ġconsum er +Ġclose ly +c alled +Ġ5 4 +Ġhop es +Ġar ray +ĠW in +ĠLab our +Ġsp ons +ĠI re +Ġp ow +Ġread ers +Ġemploy ment +Ġcreat ure +Ġresult ing +Ġaccur ate +Ġmom ents +Ġarg ued +Ġp ed +D uring +Ġ5 3 +ĠT al +Ġs ought +Ġsuff ering +Ġ icon +le e +Ġ( $ +al ian + ° +Ġp ra +Ġbon us +( " +k o +Ġact ing +D E +f all +Ġcompar ison +Ġsm ooth +ĠN AS +u pp +ĠJose ph +ep ing +ĠT ake +ĠM id +Ġs ending +f ast +ĠF all +Ġdeal ing +us er +ĠOr gan +C o +Ġatt ached +Ġse es +% . +Ġtyp ical +AR T +Ġfind s +ĠAs ia +um in +ĠC ore +ĠE nt +in ent +u ce +ĠBl ood +ĠN ever +Ġem ails +Ġhigh light +Ġconf ront +at us +ut ed +Ġun us +Ġtop ic +ĠAd am +Ġb le +at i +Ġunder stood +S et +st ruct +T P +Ġm ob +a a +ĠSt art +pect ed +se ll +Ġded icated +ĠC A +u an +Ġsong s +esc ription +Ġte ch +Ġr ape +Ġas ide +Ġgr ant +Ġ5 6 +s ub +Ġarg ue +Ġcont aining +Ġsche dule +Ġliber al +Ġpublic ly +Ġheav ily +ĠU t +in er +ĠS ection +ĠC are +we et +l s +D is +âĶ Ģ +ĠF ollow +B ack +ĠI T +Ġb es +j i +ĠH it +est ed +Ġevery body +ĠSw ed +Ġfem in +Ġfac ilities +Ġcon ven +C omp +ĠO S +c ore +Ġan x +Ġdiv ision +ĠC am +ĠSt an +m ates +Ġexpl ore +pl om +Ġsh ares +pl oad +an es +Ġide al +et ers +ĠB ase +Ġpl astic +Ġdist inct +ĠNet work +ĠSe attle +Ġtrad ing +ens us +int end +Ġex hib +Ġinit ially +ĠF ood +Ġthous and +ĠBus iness +act er +Ġpar agraph +Ġrough ly +Ġw ww +Ġcreat ive +ĠCon f +Ġconsum ption +Ġfil ms +ag an +Ġob tain +Ġt all +Ġt or +Ġacknow led +Ġg rown +al o +K E +Ġ4 00 +end ers +t aining +U G +Ġsu icide +Ġwat ched +ĠL ist +al i +re hens +Ġsurround ing +Ġp ip +Ġf lying +ĠJ ava +ord an +Ġserv ing +in ations +p ost +Ġsh o +A v +Ġj ail +z y +Ġ199 9 +Ġ< / +Ġliter ally +ĠS ir +Ġexp osed +Ġl ies +st ar +Ġb at +Ġear ned +ĠD ig +Ġspec ified +ĠSe ason +Ġdeg rees +Don ald +Ġcent re +Ġsh aring +Ġwin ter +ĠC O +C he +Ġ Î +M P +Ġun w +Ġfew er +ĠM ir +Ġsomew here +ĠK ey +Ġattack ed +ĠK ir +Ġdom ain +Ġstrong er +Ġ9 9 +Ġpen alty +I d +Sc ript +Ġdecl ined +Ġne ck +Ġfra ud +Ġcur rency +Ġr ising +R C +âĢ¦ âĢ¦ +H z +Ġt ab +Ġtal ent +n am +ĠN BA +Ġvill age +Ġleg s +ĠN ext +E d +Ġac id +Ġhy d +8 00 +Ġinvol ving +ĠIm age +ĠBe fore +F l +Ġyes terday +S ource +Ġterror ist +Ġsu p +Ġsy nt +ĠSaud i +Ġw est +Ġr u +b urg +Ġvis ible +Ġstru ck +r ison +Ġaw esome +Ġd rawn +Ġansw ers +ĠG irl +ĠR am +Ġthreat s +Ġdef eat +os it +Ġv ent +atur ally +Americ an +end a +ĠH oly +Ġr um +% , +c ase +ĠHist ory +ĠYou Tube +Ġsit uations +ĠD NA +S te +Ġsa ved +It em +Ġrec ip +olog ist +Ġfac ed +Ġel ig +O nce +ĠL i +u h +Ġmist ake +ĠDiv ision +ĠB ell +Ġsympt oms + ® +Ġdom in +Ġfall ing +Ġend ing +as hes +Ġmat ches +ĠOn line +Ġexplan ation +D ef +red it +Ġany more +ĠT otal +ĠF OR +us hed +Ġlet ters +Ġris ks +ĠO K +Ġreported ly +: \ +Ġpl ate +Ġsubject s +Ġattempt ed +if ier +ian a +Ġunlike ly +ĠTh ough +um a +ĠIn vest +ĠPr in +ic an +ĠD ar +ĠColor ado +au g +Ġve get +a os +ri a +Ġshe l +Ġmark ed +Ġ( ) +Ġsp r +p o +ĠL ink +Ġdef e +ĠJ r +Ġthem e +Ġpass ion +ĠP en +Ġinf o +iz er +Ġsh it +ĠC ivil +ap se +c re +Ġpo ly +Ġcomp onent +ĠChar les +ĠIre land +ĠPro v +Ġdo ctors +Ġgr anted +Ġpain t +Ġhon or +Ġsm oke +Ġpay ments +Ġprim arily +ĠKing dom +r ich +ate ll +Ġde als +Ġsched uled +Ġfund amental +Ġprote in +Ġnewsp aper +Ġcl ients +yth on +ĠD ate +h us +Ġfeed back +Ġstret ch +Ġc ock +Ġhot el +ĠQue en +Ġsu gar +Ġj u +Ġmil k +Ġappro val +ĠL ive +Ġequival ent +ef ully +Ġins ert +z ona +Ġext ension +d ri +J ohn +Ġacc omp +S m +ĠF und +Ġconst antly +Ġ` ` +Ġgener ated +ĠA ction +ĠP sych +ĠT ri +Ġrecogn ize +Ġv ary +ph a +ĠR a +d f +et ch +ĠSov iet +Tw o +Ġpattern s +Ġprof ession +an ing +T ime +ĠL im +Ġcol ors +ĠA z +ĠT R +Ġinf ect +Ġphen omen +Ġshe ll +Al so +Ġput s +Ġdel ivery +Ġbro wn +Ġprocess ing +Ġlight s +ess age +ĠBro ok +ĠA ud +l ation +Ġindust rial +L ike +ĠB razil +rou s +ES S +ĠL uc +Ġsome how +Ġ8 5 +Ġpro port +Ġpolit icians +Ġindic ate +Ġh ole +Ġtechn iques +Ġcompet itive +Ġph r +Ġv o +ist ent +ĠD ream +Ġcamp us +Ġaspect s +Ġhelp ful +Ġsh ield +or se +Ġtrig ger +m al +Ġ5 8 +Ġt ort +Ġperson ally +Ġt ag +Ġkeep s +ĠV ideo +Ġben ch +Ġg ap +a ire +Ġe ast +Ġrec overy +per ial +Ġprof it +ĠM ic +Ġ5 7 +Ġcol on +Ġstrong ly +st yle +Ġalleg ations +h an +Ġrep orters +j o +r ine +arg et +and al +Ġ0 3 +Ġfl ash +tr ans +Ġstr ict +Ġpark ing +ĠPak istan +Ġl i +Ġwe ird +ĠE ric +Ġreg ions +ĠJ un +Ġint ellect +ĠW H +od ing +rib utes +up id +ĠT it +Ġf inger +or ia +Ġe lev +ĠF ield +Ġcon clusion +; ; +Ġfeel ings +Ġext ensive +Ġm ixed +Ġne uro +v y +Ġhar ass +ĠC irc +ou ch +Ġterrit ory +Ġsuccess fully +M ar +Ġing red +Ġoverw hel +Ġl ayer +V iew +Ġall ies +ill ance +ĠTh ree +Ġb unch +Ġnorm ally +Ġnet works +Ġsac r +ĠC IA +b les +Ġch ose +Ġopp onents +Ġregard less +Ġfr anch +Ġpre f +ĠP o +Ġbr idge +ann a +ĠSil ver +Ġw age +p age +ri or +Ġrad ical +ĠL ittle +Ġman ip +Ġsecret ary +Ġg ang +D R +F A +Ġdec ent +ĠSp irit +Ġun cle +ĠDevelop ment +Ġinvest ors +Ġwall s +Ġpub lish +Ġgener ate +iss ions +c ar +Ġprom ote +Ġcut ting +Ġche st +Ġdrink ing +Ġcollect ed +Ġ7 2 +Ġhop ing +Ġem br +gor ith +Ġwar ned +Ġinstruct ions +O G +ĠD id +ĠAg ency +Ġg ear +Ġcritic ism +ĠF urther +Ġut il +ann y +R ed +Ġcoun sel +ĠAs ian +Ġredu ction +p ool +Ġteach ing +Ġdeep ly +i y +Ġestim ates +Ġcho ices +Ġperman ent +in em +ke l +Ġf asc +p se +f ile +ĠL ow +ĠP erson +Ġt ournament +st al +Ġm el +U ST +ĠR ay +az i +V al +Ġcont ained +ĠH olly +Ġw ake +Ġreve al +Ġprocess es +ĠIS IS +Ġ0 9 +Ġbl ind +Ġste el +ĠB ad +Ġcare fully +app y +ro it +Ġg aming +Ġhous es +ĠC oll +Ġtr uck +er m +Ġsc ored +Ġocc as +ret urn +b ound +v ar +Ġsh arp +Ġaf raid +ĠE X +am ber +c ific +Ġsche me +N C +ĠPol it +Ġdecl ine +Ġ199 8 +Ġpus hing +Ġposs ession +Ġpriv ile +Ġteacher s +Ġy ield +H A +ĠDav is +it led +#### #### +Ġr ig +ĠD aniel +ac on +Ġh ide +ut en +Ġcolle agues +Ġprin ciples +Ġl oud +Ġs in +ĠDem on +Ġst one +Ġ0 2 +Ġt aught +Ġter rible +Ġst uck +ĠPol icy +te en +Ġimplement ation +ĠB BC +ĠAP I +Ġwhe el +all as +Ġch ampions +ol ars +play er +Ġrepeated ly +ĠSt ill +Ġlik es +ast y +es ter +ĠCath olic +R L +Ġb ath +Ġno ise +t itle +Ġn orthern +P art +Ġmag n +Ġf ab +ĠAs h +Ġdis pl +Ġtick et +Ġm urd +Ġalong side +ĠMus ic +Ġr iver +ĠSte el +ĠC L +ĠPl ayer +ĠM ult +ow ing +re p +s ize +Ġt ur +ĠGeorg ia +isc al +ra ction +Ġc able +Ġ5 9 +Ġw ins +Ġup coming +Ġsurv ive +Ġins pired +ĠEduc ation +Ġstat istics +ĠF oot +iam i +Ġy ellow +ĠP age +. - +ĠH as +Ġur ban +Ġa x +es sel +\ " +Ġquarter back +Ġreg ister +ĠLab or +Ġab ilities +ĠF amily +Ġvar iable +ĠPr ice +Ġcont em +Ġth in +ĠE qu +d ata +Ġg otten +Ġconst it +Ġas ks +Ġt ail +Ġexc iting +ĠE ffect +ĠSp anish +Ġencour age +ins on +ĠA h +Ġcommit ment +C S +Ġr ally +Ġ: : +Ġsubs id +Ġsp in +Ġcapt ured +201 8 +Ġinn oc +Ġalleged ly +ĠC ome +Ġart ists +ĠN umber +Ġelect ronic +Ġreg ional +ap es +Ġw ra +Ġmy th +pr ise +ĠM iller +ĠC reat +ĠEp isode +b ell +Ġdirect ed +Ġext ract +Ġs orry +Ġv ice +ag ger +ĠSu pport +Ġ6 6 +ĠI ron +Ġwonder ful +Ġg ra +N et +ion e +E ng +Ġsh ips +ik es +ĠK evin +it ar +Ġactiv ists +tr ue +ĠAri zona +ent h +ĠDes pite +ĠS E +Ġha bit +ern el +Ġin qu +Ġab ortion +Ġv oid +Ġexpl icit +Ġeng aged +Ġang ry +Ġr ating +Ġfr ag +b ro +ick ing +d ev +Ġwor ried +Ġob ser +Ġap artment +ĠG T +Ġest ate +ĠConst itution +em on +ĠS now +Ġcount y +Ġdis ag +ĠStep hen +Ġimm igrants +w ind +ĠN ations +Ġfol ks +O ut +Ġg all +Ġtarget ed +Ġst ead +ĠB on +ĠL ib +Ġinform ed +Ġ12 0 +ch ain +idel ines +or ough +Ġdri ven +Ġregular ly +Ġbas ket +Ġprinc iple +oc ument +Ġst un +ib ilities +ĠRom an +ĠAb out +Ġal ert +Ġdemocr acy +Ġrepresent ed +H S +c ers +p arent +Ar t +p ack +Ġdi plom +re ts +ĠN O +Ġcapt ure +ĠAd v +Ħ ¢ +Ġannounce ment +ĠL ear +Ġh ook +Ġpur s +ĠS uch +ĠC amer +Ġrefuge es +ĠV e +P ol +Ġrecogn ized +l ib +Ġhad n +A ss +Ġpil ot +us hing +Ġreturn ing +Ġtra il +ĠSt one +Ġrout ine +Ġcour ts +Ġdes per +Ġfriend ly +ĠIt aly +Ġpl ed +Ġbreat h +Ġstud io +N S +Ġimp ressive +ĠAfghan istan +Ġf ing +Ġd ownt +ink ing +ĠR og +i ary +col or +se x +ar on +Ġf ault +ĠN ick +D own +ĠR ose +ĠS outhern +X X +is odes +L ist +6 00 +Ġout come +er r +Ġelse where +Ġret ire +Ġp ounds +ĠGl obal +Pe ople +Ġcommun ications +Ġlo an +Ġrat io +ĠEm pire +Ġg onna +Ġinv ent +D F +Ġ19 70 +ĠComm on +p at +Ġprom ised +Ġd inner +ĠH om +Ġcreat es +Ġoper ate +ver ty +ĠJ ordan +et ime +Ġsust ain +R eg +Ġincred ible +im a +Ġwar rant +Ġm m +A tt +Ġlaw suit +Ġreview s +it ure +ĠS ource +l ights +ĠF ord +Ġ6 3 +g roup +st ore +Ġfeat ured +Ġfore ver +Ġpo verty +ĠP op +ĠC NN +az z +ab is +ach ing +Ġl aid +ĠSu pp +Ġfil ter +en a +ĠCommun ity +Ġcreat ures +u ction +ĠR oyal +Ġassoci ation +ĠCon nect +ĠBr ad +âĸ Ī +l ers +the re +ĠG i +Ġval uable +AC K +ĠT aylor +Ġl iquid +ĠAtt orney +ĠCar l +ĠF inal +ag a +ĠWil son +B ecause +ĠProf essor +ak a +Ġincred ibly +r ance +! ) +R ef +s k +Ġsol utions +Ġatmosp here +Ġbl ame +um es +ĠN ob +C A +um ps +r ical +ĠPut in +ĠD est +or ic +ĠP A +Ġrespect ively +w an +Ġfif th +â Ħ¢ +ĠC ry +Ġgovern or +res ident +Ġpurch ased +Ġh ack +Ġint ense +ob s +Ġorig in +Ġdef ine +Ġcare ful +** * +Ġshould er +Cl ick +Ġt ied +Ġdest ruction +ou red +Ġno body +Ġh o +ĠEx per +Ġt ip +" ; +Ġtechn ique +Ġj ur +ĠP ok +b ow +Ġleg end +Ġacc ord +Ġbus y +ĠInt el +Ġh ang +ak i +. ] +âĢĶâĢĶ âĢĶâĢĶ +Ġsur gery +Ġrep rodu +Ġun iform +Ġscen es +c ode +Ġ6 2 +l isher +ĠH ave +ph ia +Ġcry pt +Ġrec on +Ġsc ream +Ġadop ted +Ġsc ores +N e +ĠIt alian +in cluding +B O +Ġindic ated +Ġent ertain +G u +T ext +i el +Ġtw enty +Ġeng age +off s +ĠPac ific +Ġsm ile +Ġperson nel +Ġto ler +Ġdo ors +Ġt one +Ġmach ines +Ġent ering +ten ance +C O +ĠJer sey +Ġfore st +Ġhor se +Ġcompl aint +ĠSpr ing +y o +ĠPl us +ed ing +ĠRet urn +qu arters +ial s +c ow +Ġacad emic +Ġf ruit +Ġ199 6 +og ether +Ġw ine +Ġpur su +ĠSte ven +Ġlic ens +Wh o +Ġclot hes +re ction +Ġsqu ad +Ġst able +Ġr aw +z ens +St ar +ut ies +anc er +Ġke ys +ĠM u +Ġcompl icated +ig er +ĠTe xt +Ġabs or +Ġ6 8 +Ġfun ny +Ġrel ief +ĠL ew +ĠC ook +Ġch art +Ġdraw ing +G E +Ġmod ule +ĠB ull +I LL +Ġs alt +0000 0000 +il le +Ġres ource +aw ay +adel phia +ĠB ru +Ġ6 7 +Ġsome body +Ġparticip ate +Ġro se +we red +Ġmus cle +Ġcons ent +Ġcontin uing +ĠGuard ian +ĠOr der +reg on +Ġre ar +Ġprov ision +Ġlik ed +ri ent +Ġb ra +Tr ans +Ġmeet ings +Ġto x +Ġcon vent +Ġaut o +Ġrec ording +ĠSo ft +00 1 +ĠR oll +Ġprogram ming +Ġp ic +Ġprov ed +Ġst ab +ĠA st +Ġca ption +ul ating +ĠAtt ack +Ġnew ly +Ġ199 7 +f r +Ġdis cipl +ĠGree k +Ġed ition +ĠDo es +ĠB ox +if le +ack et +Ġpass es +Ġgu est +Ġac celer +it als +U D +Ġaut hent +ĠR est +ov al +t a +u ine +Ġarm or +ĠT own +Ġcomp at +Ġinc hes +Des pite +Ġass ign +he rent +Ġprep are +ĠM eg +oc key +Ġdep ends +Ġtrack s +w atch +Ġl ists +ĠN orthern +Ġal ter +re c +ĠE astern +Ġcond em +Ġevery where +? ' +Ġaff ili +Ġf ought +": {" +Ġm ac +it arian +Ġsc ope +ĠA L +aw s +ar ms +Ġqu e +Ġenjoy ed +nes ota +Ġagg ressive +ĠSt ory +ĠI V +Ġrec ipe +Ġrare ly +ĠMed ical +val ue +ang el +ay ing +omet hing +Ġsub section +Ġs outhern +Ġfrequ ency +re te +roll ed +ult s +ĠN ic +Ġbeh alf +Ġsequ ence +ab et +Ġcontrovers ial +Ġcomp rom +Ġwork er +Ġmain ly +Ġal gorith +ĠM ajor +or ce +g ender +Ġorgan ized +Ġf ake +Ġconclud ed +ĠE D +ĠEx ec +r age +Ġch ances +ber ry +ĠTr ad +Ġconfig uration +Ġwithd raw +Ġf ro +ud es +ĠBro ther +ĠB rian +Ġtri es +Ġsam ples +Ġb id +ĠGold en +Ġphot ograph +if est +ĠD O +ĠPar liament +******** ******** +R em +Ġcont est +Ġsign ing +p x +ĠZ eal +âĶĢ âĶĢ +E ar +Ġex it +Be fore +ĠCor por +n ull +mon th +Ġrac ial +ott ed +ĠV eg +ĠRe uters +Ġsw ord +ps on +ĠRom ney +a ed +Ġt rib +Ġin ner +Ġprot ocol +ĠB i +ĠM iami +ever al +p ress +Ġsh ipping +ĠAm endment +ĠHow ard +con nect +ĠD isc +ĠJ ac +iam ond +ĠThere fore +s es +ĠPrin cess +ĠUS B +ĠAn th +Ġsurve illance +Ġap olog +Ġ6 1 +ow a +Ġf ulf +j s +Ġl uck +ust ed +Ġ § +n i +Ġant icip +em an +Ġwin ner +Ġsil ver +ll a +ic ity +Ġunus ual +Ġcr ack +Ġt ies +e z +Ġpract ical +Ġprov ince +ĠPl ace +Ġprior ity +IC E +Ġdescrib es +Ġbr anch +F orm +ask a +miss ions +b i +Ġp orn +ĠTur k +Ġent hus +Ġf ighters +Ġ0 8 +ĠDet roit +Ġfound ation +av id +A re +Ġjud gment +cl ing +Ġsol ve +ĠDes ign +W here +hes is +ĠT ro +a fter +Ġne utral +ĠPalestin ian +ĠHolly wood +Ġadv is +ĠN on +y es +ol is +Ġrep utation +Ġsm ell +Ġb read +ĠB ul +ĠBe ach +Ġclaim ing +Ġgen etic +Ġtechn ologies +Ġupgr ade +row s +Ġdevelop er +ĠJ osh +ĠDis ney +erv ed +ip al +Ġun ex +Ġbare ly +t hen +ĠP ub +Ġill ness +et ary +ĠB al +Ġp atch +Ġbut t +Ġst upid +ĠD og +ĠD allas +f ront +ie ce +Ġprot ests +Ġch at +oen ix +Ġw ing +Ġpar liament +Ġ7 7 +ose xual +Ġre nder +pt ions +ĠCo ast +os a +ĠG reg +h op +ĠMan agement +Ġbit coin +Ġrec over +Ġincor por +or ne +ĠUs ing +Ġpre ced +Ġthreat ened +Ġspirit ual +ĠE vent +ĠF red +Ġadvert ising +Ġimprove ments +ĠC ustom +Ġer rors +Ġsens itive +ĠN avy +Ġcre am +L ook +Ġex clusive +Ġcomp rehens +Ġde leg +Ġcon ce +Ġrem em +Ġstruct ures +Ġst ored +N D +Ġ1 000 +U P +ĠB udd +A F +w oman +ĠAcad emy +ð Ł +se a +Ġtem porary +Ab out +es ters +Ġtick ets +Ġposs ess +in ch +o z +Ġl a +Ġcontract s +Ġun p +Ġc ig +ĠK at +ult ural +as m +Ġmount ain +ĠCapt ain +St ep +m aking +ĠSp ain +Ġequ ally +Ġl ands +at ers +Ġreject ed +er a +im m +ri x +C D +Ġtrans action +g ener +less ly +Ġ| | +Ġc os +ĠHen ry +Ġprov isions +Ġg ained +Ġdirect ory +Ġra ising +ĠS ep +ol en +ond er +Ġcon sole +in st +Ġb om +Ġunc ertain +1 50 +ock ing +Ġmeas ured +Ġpl ain +Ġse ats +Ġd ict +S L +af e +Ġest imate +iz on +at hered +Ġcontribut ed +Ġep isodes +omm od +G r +AN T +Ġ6 9 +G ener +Ġ2 50 +vious ly +rog en +Ġterror ism +Ġmove ments +ent le +oun ce +ĠS oul +Ġpre v +ĠT able +act s +ri ors +t ab +Ġsuff er +Ġn erv +Ġmain stream +ĠW olf +Ġfranch ise +b at +Ġdem ands +Ġag enda +Ġdo zen +Ġclin ical +iz ard +ĠO p +t d +Ġvis ited +ĠPer haps +Ġact or +Ġde lic +Ġcont ribute +Ġin ject +ĠE s +ac co +Ġlist ening +Ġcon gress +epend ent +Ġprem ium +Ġ7 6 +ĠIr ish +Ġass igned +ĠPh ys +Ġworld wide +Ġnarr ative +ot ype +m ont +b ase +ĠB owl +ĠAdminist ration +Ġrel ation +ĠE V +C P +Ġco vers +Ġ7 8 +Ġcert ific +Ġgr ass +Ġ0 4 +pir acy +ir a +Ġengine ering +ĠM ars +Ġun employ +ĠFore ign +st ract +Ġv en +Ġst eal +Ġrepl ied +Ġult imate +Ġtit les +d ated +Ġj oy +a us +Ġhy per +ak u +Ġoffic ially +ĠPro duct +Ġdifficult y +per or +Ġresult ed +rib ed +l ink +wh o +~~ ~~ +ĠSpe ed +ĠV iet +W ind +ĠBar ack +Ġrestrict ions +ĠSh are +Ġ199 5 +ition ally +Ġbeaut y +op t +Ġm aps +ĠC R +ĠN ation +ĠCru z +W ill +Ġelectric ity +Ġor g +Ġb urd +Ġviol ation +Ġus age +Ġper mit +ĠCh ron +ĠF ant +Ġn aturally +Ġ0 7 +Ġth rown +ĠAw oken +Ġal ien +ĠHer o +ĠK ent +ĠR ick +ri ke +Ġp ace +}, {" +G L +Ġpo ison +ĠT ower +Ġform al +al ysis +Ġgen uine +Ġk il +a ver +Ġproced ure +ĠPro p +intend o +ĠM ain +as ant +Ġtr ained +G ame +ĠL oad +ĠM A +Ġcru cial +Ġle ts +ĠF R +Ġch ampion +1 01 +ĠCon ference +Ġwrit ers +Ġconnect ions +Ġo kay +ir ms +ĠR and +Ġenc ounter +ĠB uff +Ġachie ved +Ġche cks +isc ons +Ġassist ant +Ġwhen ever +ĠA ccess +ĠU r +b in +Ġcl ock +is p +op her +Ġb orrow +Ġm ad +Ġperson ality +on ly +IS T +ab ama +Ġg ains +Ġcommon ly +Ġter r +Ġhyp ot +Ġre ly +Ġt iss +iscons in +Ġrid ic +f unction +ĠO regon +Ġun com +r ating +el and +ĠN C +Ġm oon +ann on +Ġvulner able +ut ive +³³ ³³ +ĠRad io +Ġw estern +se ct +ĠT ony +Ġocc urs +ĠO s +ĠH on +Ã Ń +Ġv essel +ĠScot land +Ġdiscrim ination +Ġsubsequ ent +st ring +Ġfant asy +ĠSh adow +Ġtest im +W E +it i +r as +Ġbo at +Ġmar ks +Ġord inary +Ġre n +Ġrepresent ative +Ġpet ition +Ġ7 3 +Ġad venture +Ġign ore +ĠPhil adelphia +ĠS av +V P +Ġfact ory +Ġt asks +Ġdep ression +z ed +................ ................ +ĠSt orm +Ġc ogn +Ġelig ible +Ġredu cing +v ia +Ġ0 5 +Ġstri king +Ġdoll ar +h o +O V +Ġinstr ument +Ġphilosoph y +ĠMo ore +ĠA venue +Ġrul ed +ĠFr ont +IN E +ĠM ah +Ġscen ario +ĠNAS A +Ġen orm +Ġdeb ut +Ġte a +T oday +Ġabs ence +S im +Ġh am +le ep +Ġt ables +ĠHe art +M I +K e +re qu +V D +m ap +Ġchair man +Ġp ump +Ġrapid ly +v i +Ġsubstant ial +E P +d es +ch ant +ili pp +ĠS anta +ri ers +anche ster +L oad +ĠC ase +Ġsa ving +Ġ7 4 +ĠA FP +er ning +oun ced +ĠMin nesota +ĠW as +Ġrec ru +Ġassess ment +ĠB ron +U E +Ġdynam ic +Ġf urn +ul ator +Ġprop ag +h igh +Ġacc ommod +Ġst ack +ĠS us +w rit +Ġre ven +ĠGod d +ĠZeal and +ab s +Ġbr ut +Ġper pet +h ot +Ġhard ly +ĠB urn +ãĤ ¹ +Ġst y +Ġtrans actions +Ġg ate +Ġsc reens +Ġsub mitted +Ġ1 01 +Ġlangu ages +ugh t +em en +Ġfall s +Ġc oc +Ĥ ¬ +Ġstri kes +p a +Ġdel iber +ĠI M +Ġrel ax +ann els +ĠSen ator +Ġext rem +Ġ} , +ĠDe b +Ġbe ll +Ġdis order +c ut +Ġi OS +Ġl ocked +Ġem issions +Ġshort ly +" ] +ĠJud ge +ĠS ometimes +Ġr ival +Ġd ust +Ġreach ing +F ile +¯¯ ¯¯ +ino is +ĠJ ason +Ġs atell +are t +Ġst ations +Ġag ric +ĠTechn ology +com es +ĠUn fortunately +ĠChild ren +Ġappl ies +ast ed +Ġan ger +ail ability +ĠDam age +Ġcomp are +ĠStand ard +Ġaim ed +ĠB a +angu age +Ġreg ulation +Ġj ury +Ġair port +Ġse ctions +ĠPr ince +em ed +Ġmedic ine +Ġh itting +Ġsp ark +ol ves +Ġad s +St ate +Ġfood s +Ġrepl acement +Ġch icken +Ġlow est +Ġmind s +Ġinvol ves +u i +Ġarr ang +Ġproced ures +ĠWh ich +ivers ary +Ġb ills +Ġimprove ment +Ġin ev +Ġexpect ations +Ġintellect ual +Ġsp aces +Ġmechan ism +2 50 +bre ak +ĠZ e +ĠT enn +ĠB alt +Ġbar rel +Ġstat ic +man n +Pol ice +Ġt ips +Ġhand ling +c us +od ed +il ton +ir y +Ġjournal ists +our se +Ġcom ic +Ġnom ine +IT Y +Ġvers us +Ġlo op +Ġsur f +ĠInd ust +ĠHun ter +Ġbelief s +is an +Ġset up +Ġbre w +im age +Ġcomput ers +f ol +} ," +ĠMed al +Ġtax p +Ġdisplay ed +Ġg rav +Ġf iscal +M on +ĠMos cow +ĠK ong +ĠCent re +Ġcamer as +ĠMr s +ĠH ay +Ġa ver +ĠK elly +p y +Ġrequire ment +Ġent itled +omb ie +Ġsh adow +ag ic +ĠA k +Ġel ite +Ġdiv ided +Ġhead ing +Ġcop ies +Ġloss es +Ġv it +k ed +ĠB ry +Ġan s +ĠSte am +Ġrep orter +he im +ĠIt em +Ġsuper ior +d on +ere nt +à ¶ +Ġtherap y +Ġpe ak +ĠMod el +Ġl ying +Ġg am +z er +r itten +Ġrespons es +Ġconsider ation +ĠB ible +Ġl oyal +Ġinst ant +Ġp m +ĠFore st +à ¼ +Ġext end +Ġconv icted +Ġfound er +Ġconv in +ĠO ak +che ck +Ġsch olars +p ed +Ġover se +T op +c ount +ĠAr k + · +Ġ0 6 +ĠL A +m d +ĠLat in +im ental +ĠC PU +Ġsubst ance +Ġminor ity +Ġmanufact uring +E r +ocol ate +Ġatt ended +ĠMan ager +r ations +Ġappreci ate +om y +GB T +id ency +B L +Ġguarant ee +pos ition +Ġo cean +clud e +Ġhead ed +Ġt ape +Ġlo ose +Ġlog ic +Ġpro ven +Ġsp ir +Ġad mit +is a +Ġinvestig ate +Ġ199 4 +sy lv +ĠL ost +c est +Ġ7 1 +Ġrequest ed +Ġwind ows +ĠPok é +ĠWith out +M et +Ġbehavi our +Ġread er +Ġh ung +ĠKe ep +Ġro les +Ġimplement ed +Ġbl ank +Ġserv es +ĠJ ay +Ġc ited +ĠF riend +prof it +ap on +Ġrep air +it em +arr ass +Ġcrit ics +ad i +ĠF ather +Ġsh out +Ġf ool +Ġ8 8 +Ġprodu cing +Ġl ib +Ġround s +Ġcirc le +Ġpre par +Ġsub mit +Ġn ic +mor row +ãĥ « +U nder +Ġv ital +ater n +Ġpass word +Ġpublic ation +Ġprom inent +Ġspeak s +Ġb ars +Ġde eper +ĠM ill +port ed +Ġw id +Ġbut ter +Ġsm oking +Ġindic ates +K ey +rop ri +ĠF ile +all ing +ast ing +ĠR us +Ġad j +Ġ7 9 +av al +Ġpres um +bur gh +on ic +Ġf ur +Ġpoll s +ik a +Ġsecond ary +Ġmon ster +ig s +ĠCur rent +E vent +Ġowners hip +end ar +Ġarri ve +ĠT ax +Ġn ull +ĠPri v +Ġth ro +Ġk iss +c at +Ġup set +ang le +it ches +ect or +olog ists +ĠGal axy +Ġcor ruption +Ġh int +ent er +ĠH ospital +Ġgreat ly +Ġbeg un +es y +Ġso il +ĠAnt on +Ġmain tenance +ãĥ © +Ġdo zens +Ġhuman ity +ĠAl abama +Ġr om +w orth +ap ing +sylv ania +l ah +Ġg athered +G A +Ġattack ing +f ound +ĠSqu are +Ġar bit +ict ions +ĠW isconsin +Ġd ance +ĠS aint +arch y +Ġbase ball +Ġcontribut ions +Ġliter ature +Ġex ha +per ty +t est +Ġb ab +Ġcontain er +let ter +Ġfall en +Ġwebs ites +Ġbott le +ĠS ac +Ġbre ast +ĠP L +Ġveter an +Ġinterview s +ĠA le +Ġb anned +eng ers +ĠRev olution +in th +Ġconc erning +IV E +Ġexp enses +ĠMatt hew +ĠColumb ia +d s +ist ance +Ġent ity +.. ." +Ġrel iable +Ġpar alle +ĠChrist ians +Ġopin ions +Ġin du +l ow +Ġcompet e +Ġth orough +Ġemploy ed +Ġestablish ment +ig en +ĠC ro +Ġlawy ers +ĠSt ation +T E +ĠL ind +ĠP ur +it ary +Ġeffic iency +âĢ IJ +ĠL y +Ġm ask +Ġdis aster +Ġag es +ER E +es is +ĠH old +Ġcas ual +b led +Ġen abled +ĠEn vironment +ĠInt elligence +i per +ĠM ap +ĠB E +Ġemer ged +is dom +Ġc abin +Ġregist ration +Ġfing ers +Ġro ster +Ġfram ework +ĠDo ctor +et ts +Ġtransport ation +Ġaware ness +H er +Ġattempt ing +O ff +ĠSt ore +ÃĥÃĤÃĥÃĤ ÃĥÃĤÃĥÃĤ +ĠK now +Ġdef ence +Ġsc an +ĠT en +ĠCh air +ĠP H +ĠAtl anta +Ġfuck ing +Ġans wered +b n +ĠK ar +Ġcateg ories +Ġr ational +Ġc ust +Ġrob ot +Ġcorrect ly +Ġg if +Ġgraph ics +m ic +Ġground s +ĠO pp +i ate +Ġdist ributed +Ġsan ctions +Ġchalleng ing +ut o +Ġingred ients +Ġinv ited +Ġfound ed +ĠRe qu +d ed +Ġb owl +Ġbrother s +ĠH a +I O +Ġw ages +im ore +oc ial +Ġse ed +ative ly +Ġaddress es +ĠI owa +ab eth +Ġatt itude +is d +ch ild +Ġm ole +Ġdisco very +y ard +B r +Ġ8 2 +Ġsuppl ies +ell ing +Ġdist ingu +C R +Ġre cept +Ġ vert +Ġsw im +b ec +d oor +ĠY eah +Ġg al +Ġinter act +ĠE SP +ĠC S +amp s +Ġconvin ced +Ġobject ive +Ġdis h +ĠPhot os +l ad +Ġdownt own +o il +in ction +Ġto morrow +ĠC OM +Ġsurv ival +sh ot +Ġsett lement +C ons +ĠX box +int erest +ĠS M +arg o +en ess +Ġeth nic +b ered +M in +ĠT ok +Ġinc ent +ĠComm and +Ġmain tained +Ġbreak s +br idge +at ar +ag g +ĠF inally +un icip +ĠO nt +le ft +Ġrecogn ition +Ġ* / +ĠP ers +Ġwe lf +Ġaddress ed +ĠK ansas +Ġvir us +Ġwhere as +Ġp apers +ram s +ĠMin istry +Ġple asure +Ġacqu ired +Ġd uration +j pg +Ġcal m +ĠN HL +Ġburn ing +Ġfold er +ick ed +ĠP y +ĠIll inois +Cl ass +ĠGodd ess +Ġperform ing +Ġwelf are +j ar +In ter +Ġl in +Ġenh ance +Ġnot ion +f are +yp es +ĠAre a +Ġcann abis +ĠDie go +f s +ĠM anchester +com m +in ite +Ġcover ing +ĠS ound +Ġ19 60 +Ġ8 4 +e lect +z ing +Ġcitiz en +Ġph ones +Ġr aid +Ġign ored +ĠOb ject +Ġu pload +c ard +Ġmod ified +Ġroom s +ia h +r ange +he ast +ach us +Ġsuggest ing +âĢ ĭ +gr ade +E l +Ġclot hing +Ġr h +ĠH an +un ity +en cing +ĠAust in +sec ution +t ra +d em +ĠQ ual +Ġhe aven +Ġst ages +Ġw edd +pl us +ific ial +ĠIm m +ĠH o +iet ies +Ġphr ase +Ġbr ill +act ory +Ġprov iders +Ġsil ence +Ġa er +ĠA I +ĠAd venture +Ġplatform s +Ġdemonstr ated +Ġinter f +ing ton +Ġr aces +Ġgr ade +ult ane +ĠTh rough +f alse +Ġb ow +ĠA B +Ġfl avor +Ġhistor ic +g ov +Ġcol our +Ġview ed +ĠEm ail +el come +Ġinter vention +Ġd iversity +Ġperiod s +Ġre verse +ĠV ery +Ġqu ote +ĠLe ft +th rough +Ġsc rew +Ġland ing +Ġp ill +Ġw et +Ġprot esters +Ġrepe at +av ed +er k +Ġsal ary +ĠPenn sylvania +St ill +Ġmay or +Ġkit chen +Ġfeat uring +ĠM useum +ĠT ournament +ĠF al +Ġser vers +U C +Ġany body +im g +ĠTr ade +ixt ure +the less +Ġfin ance +Ġcl osing +ĠPat ri +i ac +ab el +Ġ> > +or ous +Ġf irms +sc reen +un a +Ġemb arrass +ul se +Ġlet ting +Ġth rew +ile y +Ġch annels +l an +ĠVeg as +Ġse ar +Ġfant astic +ar re +uzz le +ĠD er +Th ose +Ġsw ing +Ġshe et +ind ex +co ver +og an +Ġvari ables +ĠTe ch +Ġsp oken +ac hel +ĠD a +ĠMount ain +Ġload ed +Ġfoot age +vers ion +Ġun l +ĠPh oenix +Ġthrow ing +Ġf iring +Ġtrack ing +Ġw idth +Ġstrugg ling +ro oms +ot ion +Ġmonth ly +ĠSer ver +Ġegg s +op en +M C +Ġ199 3 +Ġh ired +Ġstay ed +ĠAll en +Ġst ro +Ġ9 8 +st ep +ĠTurk ish +Ġfab ric +ist ing +ĠD om +Ġd ates +Ġpr on +Ġbasket ball +Ġl ucky +ĠArab ia +Ġassum ed +est y +Ġaff airs +Ġgl ad +ĠInd eed +ĠF A +ĠW ord +Ġjo ining +if ice +p read +ir ts +ĠSe lect +Ġpop ulations +aw are +Ġn ose +Ġcompl aints +st art +Ġsc oring +Th anks +Ġmin ing +Ġvisit ors +S H +Ġdam aged +Ġcharacter istics +ĠP ent +D C +Ġ8 3 +ĠS ix +r ates +Ġfl ags +ĠB rew +d og +M ark +// // +Ġexec ution +Ġj oke +ph ones +Ġtestim ony +Ġob st +Q L +ĠC ut +Ġstud ied +ĠN intendo +ick et +ĠN BC +Ġl ad +ĠB ra +ĠM oh +Ġk ernel +Ġoverwhel ming +Ġag ed +Ġapplic able +ĠC ond +Ġroad s +ĠBl ock +m ade +od ge +Ġcomm ands +Ġoff ices +vel and +Ġt ut +Ġrece iver +ĠF ro +Ġsho pping +Ġi P +ĠSt re +ĠA BC +Ġentertain ment +ĠB ow +ort ed +M c +Ġread s +gr ad +ĠCol lect +Ġâ ĪĴ +ĠCap ital +eder ation +Ġemploy er +Ġinvolve ment +Ġanx iety +al ia +Ġro of +ĠAm ong +ĠDemocr at +Ġstat s +ĠV ill +Ġconst itutional +Ġrefer ring +itt y +Ġtack le +out ube +Ġback ed +ĠH ong +ĠBro ad +Ġe le +ĠO tt +Ġ199 2 +h our +achus etts +C al +Ġdefe ated +Ġ8 1 +es p +Ġseem ingly +w as +ĠJ enn +ĠK urd +Ġg ene +Ġdisc ount +R et +EC T +( ); +Ġclub s +Ġs id +ĠM arsh +Che ck +Ġp p +ĠE ag +ides pread +Ġbe ings +F T +Ġintrodu ction +ĠCh ange +AR D +Ġ1 10 +ad ows +ier ce +Ġme al +a uthor +ĠB ang +lah oma +Ġr anks +201 1 +?? ?? +m ax +Ġcoll apse +Ġop ens +Ġe cho +Ġs oph +Ġrac ist +Ġenorm ous +Ġw aves +Ġt ap +Ġcomprehens ive +. -- +ĠR oy +Ġfarm ers +Rel ated +a ired +ron es +ĠC rim +Ġproport ion +Ġdesign s +Ġnegoti ations +Ġvirt ually +ĠBat man +Ġwar n +Ġlegit imate +m ate +Ġcon vention +, , +net ic +ĠS D +Ġconsist ently +Ġcompens ation +Ġpunish ment +Ġy e +Ġt ie +ĠB ureau +ir lf +ĠB u +ĠA ren +ĠPh ilipp +Ġkn ife +Ġmem ories +ĠR oss +Ġang le +Ġ8 6 +ĠTh under +Ġre nd +ĠT our +Ġcount s +s ung +ĠIm p +Ġeduc ational +Ġaccess ible +C OM +Ġd rew +y er +G l +am ine +OR T +O B +I B +m aster +Ġtri als +og y +h ar +ĠTr ust +Ġprefer red +irlf riend +ĠN ev +Ġb in +Ġc ow +P age +Ġsign ature +ĠB L +7 00 +Ġret ired +Ġby tes +Ġneigh b +ĠLeg end +Ġdev ast +Ġsuspect ed +is ons +ĠPoké mon +sc ale +Ġcap abilities +Ġre vel +Ġche ese +d y +igr ant +Ġfail ing +b its +ĠHer oes +ĠG host +ĠS cient +Ġappoint ed +ur i +Ġinst itution +Ġexpand ed +g reg +Ġmonitor ing +Ġp odcast +Ġcoal ition +Ġ9 6 +J o +Ġst olen +ĠS ab +Ġstop s +Ġhol iday +Ġint r +C ar +Bl ack +ĠL GBT +Ġwar ming +ĠAnd erson +Ġ8 9 +Ġprodu cer +M ed +Ġaccur acy +ĠMar vel +iz abeth +ĠPat rick +m ony +Ġmin i +ac les +Ġover t +the y +Ġmembers hip +ĠV en +Ġex ch +Ġrem oval +ĠD ave +T Y +m ad +ĠF ind +Ġad equ +Ġe c +Ġte eth +Ġemot ion +Ġper m +Ġsole ly +d b +Ġextra ord +IG HT +c al +Ġgu idelines +Ġd ying +Ġsusp ended +ĠPrem ier +ĠAnth ony +el ve +Ġd ad +ĠE th +ĠFoot ball +Ġabandon ed +Ġ< < +Ġm arch +Ġhor ror +âĢ¦ " +Ġchild hood +Ġcampaign s +Ġl unch +ĠAl bert +bl ock +âĸĪ âĸĪ +ound ing +Ġb one +or gan +ad ers +ĠFl ash +ĠDri ve +Ġton ight +Ġw ars +ĠF L +Ġform ation +con st +New s +Ġcom pe +or ious +ĠSt aff +Ġdiscuss ions +ĠProt ection +ĠJ am +Ġcrit eria +Ġinstall ation +Ġaccompl ish +iz za +Ġpub lisher +Ġresc ue +ĠT ry +U LL +ĠS om +ĠH op +ore t +th s +ord on +Ġp ocket +ĠIn v +Down load +ĠCr ime +Ġb ene +ĠGu ide +ĠAs sembly +Ġparam eters +I E +ĠAlex ander +Ġconc ert +ĠSc he +Ġsh oes +Ġvis iting +Ġrec all +Ġb ub +Ġr ural +Ġconc rete +ĠR os +N ext +R uss +Ġlo ans +ĠSh ield +Ġtre m +hem at +k g +ĠHar ris +is ition +ĠM ove +ĠF C +Ġf ate +ĠCh o +Ġt ired +Ġprinc ipal +h ist +ien ces +ath y +Ġse vent +Ġm ood +Ġstrateg ic +Ġdise ases +Ġfor um +Ġtem por +Ġhead quarters +P ar +ig e +fl ix +Ġgu itar +Ġ9 4 +On ly +Ġrele ases +ro ph +================ ================ +Ġ6 00 +ĠContin ue +ig ate +ĠC rit +sy stem +Ġdis abled +Ġunex pected +ith ub +Ġuncle ar +ĠE st +Ġcontr ad +Ġstrateg ies +vent ures +Ġpass age +AM E +Ġimpro ving +Ġreve als +Ġdecre ase +ov a +Ġann oy +ĠSh ort +ĠL ibrary +Ġcy ber +n ell +ĠH ur +ĠC B +Ġphot ograp +U I +Ġs ed +G e +Ġ8 7 +Ġd iverse +Ġencour aged +Ġcons piracy +Ġbird s +Ġoper ator +Ġhand ful +Ġclass ified +? ) +Ġdram atic +Ġinvestig ators +it o +Ġw idespread +ĠR oom +-------------------------------- -------------------------------- +Ġcollect ive +Ġjournal ist +St ring +Ġtemper atures +il a +Ġgu id +Ġins pect +Ġmiss ile +ĠMay or +Ġman ual +Ġsim ultane +Ġrat ings +Ġsu ck +Ġ9 7 +Ġunivers al +Ġph arm +Ġdis rupt +ian o +A V +Ġf t +Ġstat ist +old s +ĠWalk er +ph p +Ġunder t +ĠL as +ish op +nt il +res hold +ĠWhe ther +M s +Ġden y +ĠCl oud +Ġprov ider +Ġsurv iv +ĠUp date +h as +Ġmist akes +ch arge +pl ed +r ity +Ġn ode +ĠMass achusetts +ool s +lic ation +Ġf ails +em ale +or i +back s +Ġsh irt +Ġ' ' +ĠN AT +Ġwat ers +els on +Ġe ase +Ġsc ar +Ġcont ents +m ind +Ġcont ribution +Ġsh r +Ġhand ed +Ġst ability +Ġtra ve +E m +Ġmir ror +12 3 +Ġwe igh +Ġf iction +ou ver +ist ant +r ition +ĠF ed +Ġphys ically +Ġst ake +ĠArt icle +ĠAr c +ĠLew is +ĠM ind +Ġdemonstr ate +Ġprof its +v ision +om ic +ol id +Ġbatt les +Ġdri ves +Ġeas tern +ĠS ony +!! ! +ar ation +v ard +ĠG L +port ation +Ġ9 2 +Ġlaw makers +Ġprotect ing +ĠE PA +Ġy eah +Ġsh ame +ol ph +e ven +x it +Ġatt ach +Ġrepresent ing +Ġob s +ĠUt ah +iff s +ĠFre edom +à ³ +A K +Ġinc idents +it age +Ġview ers +c d +Ġm ouse +Ġcl ar +Ġaccord ance +Ġb ot +c or +ĠSum mer +he ld +Ġinnoc ent +Ġiniti ative +ol s +________________ ________________ +Ġsp ots +p ace +Ġconvent ional +Ġcorpor ations +Ġblock ed +H D +at tered +Ġref ers +Ġbu ck +ĠDig ital +12 0 +Ġtop ics +T F +Ä ģ +br id +re ement +Ġunder lying +ĠM ember +Ġinvestig ating +Ġpregn ancy +Ġtouch down +ĠB and +ĠCall er +Ġinst ances +P P +w a +G ood +Ġ199 1 +ĠC old +Ġfear s +Ġrem arks +Ĩ Ĵ +at al +Ġm it +Ġexper iments +i pt +Col or +ind u +Up date +Ġ9 3 +A g +Ġ å +anc ouver +B oth +Ġjud ges +Ob ject +Ġst ere +umb n +Ġparticip ation +ĠSt ars +ĠJ ere +Ġweek ly +ĠB an +Ġconvers ations +ĠP itt +u z +ĠIndian a +ĠK ick +Ġinf ection +Ġhero es +Ġsett led +Ġstri p +Ġh al +Ġd ump +ĠS ci +Ġl es +Ġref erences +ĠU RL +ĠBr idge +Ġwant ing +For ce +Ġex clus +Me anwhile +m n +Ġg entle +m aker +sen al +ĠG ro +ou ri +ĠR ain +ĠAll iance +Ġl ift +el a +S D +ĠCle veland +Ġrank ed +Ġst adium +Ġdead ly +ä ¸ +Ġr iding +ar ia +ĠAr mor +Ġdocument ation +ĠGree ce +ree k +Ġl ens +ĠS a +Ġg ross +ĠE mer +ag ers +ĠD ub +ĠR h +ĠAM D +Ġarri val +Ġdes ert +Ġsupp lement +ĠRes p +Ġkn ee +Ġmarg in +f ont +og g +201 0 +ĠP ir +ĠP rom +iv als +Ġint ake +Ġdifferent ly +ug s +Ġb its +clud ed +Ġsearch ing +ĠD u +um ble +Ġfunction al +ĠBalt imore +ĠC ould +Ġdes ired +Ġcirc uit +ĠL yn +ĠG O +ĠF alse +re pre +' : +alt ies +Ġmin im +Ġdro ve +ĠSh ould +Ġh ip +Ġpro s +Ġut ility +ĠN ature +ĠM ode +P resident +o pp +r at +form ance +Ġconcent ration +Ġf ont +ĠB ud +Ġam id +Ġre vers +ĠM L +B ar +Ġinter action +Ġjur isd +Ġspell s +d ep +f il +Ġcivil ians +ut ter +ĠCo oper +ĠBel ow +Ġent rance +Ġcon vert +Ġcontrovers y +ow ered +Ġcontr ary +Ġar c +ĠExec utive +ĠOffic er +Ġpack ages +Ġprog ressive +w idth +Ġreserv ed +v ol +ĠSam sung +Ġprint ed +Ġcent ers +Ġintrodu ce +ĠKenn edy +Ġodd s +Ġsure ly +Ġindepend ence +Ġpass engers +repre ne +ĠBe h +Ġl oves +ĠESP N +Ġfac ilit +Ġident ical +Ġdo ct +Ġpartners hip +con f +ĠH ide +Ġconf used +ĠC ow +M en +Ġw rest +ĠIraq i +Ġh oles +ĠStud ies +Ġpregn ant +h ard +Ġsign als +I X +Ġpull ing +Ġgrad uate +Ġnomine e +D ate +Ġper mitted +Ġâ Ĥ¬ +ĠOk lahoma +St art +Ġauthor ized +Ġal arm +ĠC os +v an +Ġgener ations +c ular +Ġdr agon +ĠSoft ware +ĠEd ward +Ġcontro ller +S en +ge red +ĠV ik +Ġappro ached +Th ank +Ġcan ce +Ġform ula +ĠSm all +Ġweak ness +Ġr amp +it udes +j ud +Ġbrill iant +Ġacc us +s ource +Ġ8 00 +ĠE vil +S w +Ġhom eless +we ek +i ens +r ics +ĠTh ird +T O +Ġorgan ic +Ġpresent ation +ag h +ĠDown load +v ation +Ġas sembly +or able +hold ers +ĠBern ie +ĠHel p +Ġt ong +ĠF ight +Ġbe ach +B ook +ĠL ic +Ġr ush +ĠR ound +ou p +ĠMar x +Ġcalcul ated +ĠDe vil +ĠSar ah +Ġoccasion ally +Ġbul let +Av ailable +g ate +Ġ9 1 +Ġh osp +Ġprom ises +ĠH IV +ĠSt adium +ĠSt ock +ĠCorpor ation +g age +N G +ĠC redit +Ġs ne +ib l +Ġacc um +s uch +Ġterror ists +Ġconscious ness +ĠZ h +Ġdram a +ool a +pir ation +Ġlab our +ĠN in +Ġut ter +Ġdemocr atic +Ġass ass +il ation +Ġg est +Ġab road +Ġmet ab +Ġs orts +Ġfl av +U B +Ġm g +ĠNot hing +ĠO d +Ġmus ical +200 9 +Ġdro ps +oc ated +ater al +0000 00 +Ġg re +Ġequ ality +Ġburd en +Ġv ig +ĠLe ader +-------- ---- +Ġcere mony +Ġf ighter +Ġact ors +Ġ æ +am an +F i +Ġal ign +put er +Ġe lder +ĠN SA +Ġrepresent ation +ĠOnt ario +IT H +usal em +Ġharass ment +itz er +Ġsy mp +Ġbox es +ĠD R +Ġman ifest +at re +Ġ ^ +Ġd ies +le ton +Ġmiss ions +et he +Ġres olve +Ġfollow ers +Ġas c +Ġk m +l ord +am med +Ġsil ent +ĠAssoci ated +Ġtim ing +Ġprison ers +ĠK ings +ĠF ive +Ġtow er +Ġappro aches +Ġprecise ly +Ġb ureau +ĠM other +ĠI ss +Ġkey board +it ual +Ġfund ed +Ġstay ing +Ġpsych ological +Ġm ile +ĠLe on +ĠBar b +w ill +Ġw ider +ĠAtl antic +Ġt ill +ĠR ome +ro t +Ġaccomp an +Ġfl our +ac o +W orld +ĠExp ress +ĠY u +C or +Ġple ased +part y +Ġpoint ing +Ġinf lation +Ġro y +Ġ ), +ain er +Ġwedd ing +orm on +Ġrequ iring +Ġqual ified +Ġse gment +EN D +Ġs izes +e als +Ġcor rupt +ass ador +Ġcele b +Ġdream s +ĠM ess +Ġcheck ing +ĠV ersion +Ġprep aring +Ġact ively +ĠD iff +Ġl ux +ĠW inter +act eria +ĠN E +Ġdep uty +Ġtrans gender +Ġsum mary +Ġin her +er ies +ch ar +ĠY an +Ġkn ock +ĠP ath +Ġl ip +roll er +Ġimp ression +Ġcelebr ate +Ġsl ide +Ġgu ests +Ġcl ip +F S +Ġsav ings +Ġcapt ain +Ġleg acy +ĠDen ver +Ġw ounded +tab oola +AC T +Ġpurs ue +Ġo xy +Ġ q +Ġsem i +ĠN eed +ĠAff airs +Ġob sc +Ġcheck ed +Ġd ual +C ode +ĠM D +le m +ult y +Ġ © +ĠEl izabeth +Ġcent uries +ard ed +s rc +Ġev ident +enn is +at in +Ġunemploy ment +ĠMar io +Ġint im +Ch rist +Ġbi ological +Ġsold ier +ĠAdd ed +Ġm ath +ĠG il +Ġbi as +Ġd ating +ĠO cean +Ġm ice +M us +h ire +ĠT es +Ser ver +lim ited +S ize +Ġmet ers +Ġrock et +es see +Ġcertific ate +ĠIran ian +AS S +Ġgr id +D ec +Ġro lling +com mun +ĠSwed en +b ury +Ġtiss ue +Ġrac ism +ĠL ocal +Ġmyster y +Ġexam ine +Ġst em +Ġs its +Ġhop ed +ot ing +Ġdial ogue +Ġpers u +W atch +l ay +M AN +Ġch ronic +ĠPort land +mark et +ĠS EC +Ġparalle l +Ġsc andal +Ġcar ries +Ġphenomen on +h uman +ack er +ĠO x +Ġretire ment +tain ment +ov ie +ĠG ear +Ġd uties +Ġdo se +Ġsc roll +M B +in f +Ġsa uce +Ġland scape +red dit +ĠChampions hip +ĠRed dit +al id +Ġco in +Ġover s +Ġpost ing +ab out +Ġf el +and y +Ġb old +Ġfocus ing +e ffect +G R +Ġde emed +Ġrecommend ations +Ġste pped +Ġvot er +ĠDe ep +ĠInst agram +Ġmoder ate +ĠMary land +Ġrestrict ed +ĠM B +ĠCh all +Ġto b +Ġc ir +ĠO cc +ĠE ver +Ġcoll aps +IN FO += - +ĠP ict +ĠAcc ount +n c +Ġo ught +Ġex port +Ġdr unk +( ' +Ġw ise +ĠM ort +ne cess +Ġan cest +ĠInc re +Ġfrequ ent +m ir +Ġinterpret ation +Ġdepend ent +Ġco ins +ĠB ol +V ideo +ĠJust in +Ġfat al +Ġcook ing +Ġconf usion +ip her +Ġcust ody +ĠMor gan +om ach +ĠGovern or +Ġrestaur ants +el ing +Ġacknowled ged +Ġthe r +Ġgen es +ch ing +He y +Ġtact ics +ĠMex ican +Ġv end +Ġhe s +qu er +Ġnot ing +ĠCamer on +Ġtarget ing +ro ck +Ġcred its +Ġemot ions +Ġrepresent atives +new s +Ġlegisl ative +Ġrem oving +Ġtweet ed +ĠCar ter +ĠF ixed +Ġfor cing +Ġspeak er +Ġm ales +ĠViet nam +l ined +Ġconcept s +Ġvo ices +o ir +ĠT rib +W he +ĠJer usalem +ĠS ant +Ġc ul +Ġl ady +ĠHaw ai +Ġar ts +ĠIn n +ĠMach ine +ĠEm peror +Ġsl ot +g ly +ĠPro cess +II I +Ġathlet es +ĠTem ple +ĠRep resent +Ġpres c +Ġt ons +Ġgold en +Ġp unch +ĠG R +iver pool +Ġen act +Ġlob by +Ġm os +Ġpick ing +Ġlif etime +Ġcogn itive +E ach +z o +Ġd ub +Ġcons ists +ol n +Ġf estival +am ous +Ġint ellig +w ords +ĠSm art +Ġde le +Ġl apt +Ġmag ical +ĠS in +b us +ur ities +igh th +ĠRub y +ĠS ure +ol ving +Ġj un +O ST +Ġimp osed +Ġast ron +Ġcor rel +ĠN S +ĠK it +ĠF uture +b urn +Ġimm une +oc us +Ġcour ses +ĠSt ring +Ġle an +Ġg host +Ġout comes +Ġexp ense +Ġevery day +Ġaccept able +A h +Ġequ ipped +Ġor ange +F R +ĠD utch +Th ough +ĠR ank +Q U +ĠRober ts +wh at +re nd +Ġdisapp ear +Ġsp awn +ĠL am +o is +Ġdes erve +Ġmin imal +Ġnerv ous +ĠW ould +Ġro ok +ĠV ancouver +Ġres ign +sh ire +ĠW orks +ĠB uild +Ġafford able +ĠG ary +ĠAren a +Ġh anging +Ġimpl ications +ĠS ong +Ġmain taining +Ġgu ards +C ON +Ġder ived +Ġexecut ed +Ġthe ories +Ġqu oted +ĠAnd re +og a +sel ess +in fo +ĠBel g +Ġt ears +ĠSur v +Ġbirth day +ig ious +im mer +Ġspect rum +Ġarchitect ure +Ġrec ruit +arm a +T able +Ġmon sters +ĠG ov +Ġdest ination +Ġattract ive +Ġf oss +ĠMore over +Ġpres ents +TH E +Ġrep ly +pt on +Ġc um +Ġdel ight +Ġaffect s +Ġdon ations +ĠT oy +ĠH im +M ENT +Ġover come +it ched +ĠFant asy +ĠH at +ĠBe ast +b ott +Ġinvestig ations +R un +Ġhun ting +d i +f und +Ġs essions +est yle +Ġport ray +oid s +Y eah +Ġcommun icate +Ġcom edy +ĠY ang +Ġbel t +ĠMar ine +Ġpredict ed +Pl ay +Ġimportant ly +Ġremark able +Ġelim inate +D avid +Ġb ind +V ID +Ġadvoc ates +ĠG aza +im p +D B +ĠN a +ĠSim ilar +I ES +Ġchar ity +v as +m ath +Ġâ ĸ +ok er +nd um +Ġcap s +ĠH al +2 000 +e an +Ġfle et +Ġrec re +R ight +Ġsleep ing +ij ing +k ind +Ġdesign ated +à ¤ +Ġanim ation +ke e +ĠInt rodu +Ġ/ > +Ġdelay ed +Ġtrem end +Ġcur ious +U se +Ġle ct +d am +Ġinnov ation +ĠPoint s +Ġload ing +Ġdisp ute +ct ic +ird s +ĠB Y +Ġn urs +ĠVal ue +ION S +ĠH um +Ġtem plate +m ers +Ġappear ances +ĠEnter tainment +Ġtransl ation +Ġsa ke +Ġbene ath +Ġin hib +Ġe uro +abet es +Ġstud ying +ĠM as +Ġper ceived +Ġexam ined +Ġe ager +Ġco aches +Ġim per +ch i +Ġprodu ces +" ). +ĠEvery one +Ġm unicip +Ġg irlfriend +Ġh ire +ĠV ice +Ġsu itable +op y +Ġin equ +ĠD uke +f ish +f irst +ĠO bs +Ġinter ior +ĠBru ce +ĠR y +Ġanal ys +Ġconsider able +Ġfore cast +Ġf ert +ors hip +ĠD rug +ĠA LL +: " +th ur +ĠM ail +Ġball ot +Ġinst antly +ĠCh annel +Ġp icks +Ġ198 9 +Ġt ent +ol i +Ġcivil ian +b ling +ell o +b u +Ġin ch +Ġlog o +Ġcooper ation +Ġwal ks +Ġinvest ments +Ġimp rison +ĠF estival +ĠK y +Ġleg ally +Ġg ri +ch arg +S l +Ġthreat ening +du ction +fl ow +Ġdismiss ed +ibr aries +c ap +e le +ĠMc G +ĠHar vard +ĠConserv ative +ĠC BS +p ng +Ġro ots +ĠH aving +umb led +ĠF un +\ / +ĠS earch +ple x +Ġdiscuss ing +Ġcontin u +ĠT ai +ĠW ik +F ree +f it +Ġref use +Ġmanag ing +Ġsy nd +ip edia +w alk +Ġprofession als +Ġguid ance +Ġunivers ities +Ġas semb +unt u +F inally +AS E +ĠAut o +ĠH ad +Ġann iversary +L D +ĠD ur +ĠUlt imate +ih ad +pro duct +Ġtrans it +Ġrest ore +Ġexpl aining +Ġass et +Ġtransfer red +Ġbur st +ap olis +ĠMag azine +ĠC ra +ĠB R +gg ed +ĠH E +M ich +b et +ĠL ady +yl um +erv es +Ġme ets +wh ite +L og +Ġcorrespond ing +Ġins isted +G G +Ġsurround ed +Ġt ens +Ġl ane +Ġco inc +h ome +Ġexist ed +ect ed +ĠDou ble +lam m +Ġske pt +ex p +Ġper ception +ie v +ĠBe ing +o ft +Ġadop t +. : +] ; +Wind ows +Ġsatell ite +AS H +Ġinf ant +d escription +ĠMe anwhile +c m +oc a +ĠT reat +act or +Ġtob acco +ĠN orm +em ption +Ġfl esh +Ġj e +o op +ĠHe aven +Ġbe ating +an im +Ġgather ing +Ġcult iv +G O +ab e +ĠJon athan +ĠSaf ety +Ġbad ly +pro t +Ġcho osing +Ġcontact ed +Ġqu it +Ġdist ur +Ġst ir +Ġto ken +D et +ĠP a +Ġfunction ality +00 3 +s ome +Ġlimit ations +Ġmet h +b uild +con fig +N T +re ll +ble m +ĠM om +Ġveter ans +ĠH u +Ġtrend s +are r +ĠG iven +ĠCa ption +m ay +AS T +Ġwond ering +ĠCl ark +n ormal +Ġsepar ated +Ġdes p +st ic +b rew +Ġrel ating +ĠN ik +ĠF arm +Ġenthus i +g ood +d eb +Ġactiv ist +Ġm art +Ġexplos ion +ĠEconom ic +L ink +Ġins ight +Ġconven ient +Ġcounter part +su pport +ĠV irt +ag en +ĠTenn essee +ĠSim on +ĠA ward +OC K +ĠF igure +Ġoverse as +Ġpr ide +ĠC as +n ote +m g +C urrent +Ġdispl ays +cont ent +Ġtravel ing +Ġhosp itals +ĠFin ancial +ĠP ast +Ġdefend ant +Ġstream ing +m ble +ĠBer lin +uk i +Ġdist ribut +Ġant ib +Ġch ocolate +ĠCast le +Ġinter rupt +ĠR ow +Ġconvers ion +Ġbug s +ĠR ather +li est +L Y +ĠJe an +com mon +ak h +Ġ1 30 +ot ton +ĠDe an +Ġam endment +Ġgame play +ĠWar ren +od a +Ġhigh lights +Ġir re +ĠNAT O +Ġball s +Ġdemand ing +U RE +ĠL uke +F igure +st op +on ia +z one +iz ers +ĠW R +Ġaward ed +Ġregul atory +ĠH art +ĠS N +pl ing +Ġs our +ĠP ixel +us ive +Ġf et +ĠS ent +Ġautom atic +Ġf er +vern ment +ĠKh an +T ON +f ather +Ġextraord inary +th rop +ĠP ython +ĠG PU +Ġsex ually +Ġdesk top +it ivity +ĠAnton io +Ġo rient +Ġe ars +ob by +ous es +vertis ements +Ġmanufacture rs +ic ient +min ute +Ġconv iction +Ġg arden +p ublic +Ġsatisf ied +f old +O K +Ġin hab +ĠTh ink +Ġprogram me +Ġst omach +Ġcoord in +Ġh oly +Ġth reshold +Ġr het +Ġser ial +Ġemploy ers +ĠEvery thing +ra h +Ġb other +Ġbr ands +Val ue +ĠT ed +ĠPlan et +Ġp ink +ĠFurther more +s a +P E +re ck +ĠUS D +ot te +Ġ& & +Ġland ed +g ets +Ġprodu cers +Ġhealth care +Ġdomin ant +Ġdest ro +Ġam ended +ch ron +Ġf its +ĠSy d +ĠAuthor ity +AT CH +Ġfight s +ĠL LC +Ġ-- - +ĠCor p +Ġtox ic +spe cific +ĠC orn +ĠChe l +Ġtele phone +ĠP ant +Ġmyster ious +aun ch +od ox +med ia +Ġwitness es +ag u +Ġquestion ed +ĠBre xit +ĠRem ember +ene z +Ġend orse +iat ric +ĠId ent +Ġridic ulous +1 10 +Ġpr ayer +Ġscient ist +Ġ19 50 +ĠA qu +Ġunder ground +ĠU FC +m are +ĠL ater +w ich +Ġsubsc rib +Ġhost s +Ġer r +Ġgr ants +ant om +Ġsum mon +ear ly +ĠC lear +ĠPr im +Ġsusp ension +Ġguarant eed +app er +Ġr ice +ĠSe an +ĠSh in +Ġrefere ndum +Ġfl ed +r ust +Ġ3 60 +ter y +Ġsh ocked +B R +ĠO il +ĠAll ah +Ġpart ly +Ġign or +Ġtrans mission +Ġhom osexual +ivers al +Ġhop efully +ãĤ ¤ +Ġless on +L eg +Ġ .. +Y et +t able +app ropri +re tt +Ġbo ards +Ġincor rect +Ġb acteria +ar u +am ac +Ġsn ap +.' " +Ġpar ad +t em +he art +Ġav ailability +Ġw isdom +Ġ( + +Ġpri est +ĠÂł ĠÂł +O pen +Ġsp an +Ġparam eter +Ġconv ince +Ġ( %) +r ac +Ġf o +Ġsafe ly +Ġconver ted +ĠOlymp ic +Ġres erve +Ġhe aling +ĠM ine +M ax +Ġin herent +ĠGra ham +Ġinteg rated +D em +Ġpip eline +Ġapp lying +Ġem bed +ĠCharl ie +Ġc ave +200 8 +Ġcons ensus +Ġre wards +P al +ĠHT ML +Ġpopular ity +look ing +ĠSw ord +ĠAr ts +' ) +Ġelect ron +clus ions +Ġinteg rity +Ġexclus ively +Ġgr ace +Ġtort ure +Ġburn ed +tw o +Ġ18 0 +P rodu +Ġent reprene +raph ics +Ġg ym +ric ane +ĠT am +Ġadministr ative +Ġmanufacture r +Ġ vel +ĠN i +Ġisol ated +ĠMedic ine +Ġback up +Ġpromot ing +Ġcommand er +Ġfle e +ĠRus sell +Ġforg otten +ĠMiss ouri +Ġres idence +m ons +Ġrese mb +Ġw and +Ġmeaning ful +P T +Ġb ol +Ġhe lic +Ġwealth y +Ġr ifle +str ong +row ing +pl an +as ury +âĢ¦ . +Ġexpand ing +ĠHam ilton +Ġrece ives +S I +eat ures +ĠAn im +RE E +P ut +Ġbrief ly +ri ve +Ġstim ul +Ġ`` ( +Ġ __ +Ġch ip +Ġha z +Ġpri ze +ĠTh ings +AC E +ul in +d ict +ok u +Ġassoci ate +ock ets +y outube +St ory +ateg ory +Ġm ild +ail ing +ĠY e +O rig +ĠK a +or ig +Ġpropag anda +Ġan onymous +Ġstrugg led +Ġout rage +AT ED +ĠBe ijing +r ary +Ġle ather +Ġworld s +Ġbroad er +12 5 +id al +ĠBet ter +Ġt ear +E xt +Ġpropos als +Ġit er +ĠSqu ad +Ġvol unt +m i +D id +ĠP u +p in +Ġspeak ers +Ġb orders +Ġfig ured += ' +Ġsimultane ously +aed a +Ġcharg ing +Ġur ged +Ġcon j +25 6 +ĠG ordon +mer ce +Ġdocument ary +Sh are +it ol +ON E +ĠG arden +h att +ĠThom pson +ane ous +ap ore +Ġt anks +Ġless ons +tr ack +Ġout standing +Ġvolunte ers +Ġsp ray +Ġmanag ers +l arge +Ġcamp s +Ġart ificial +ĠR u +Ġb ags +th al +Ġcompat ible +ĠBl ade +Ġf ed +Ġarg ues +F I +Ġunf air +Ġcor n +Ġoff set +Ġdirect ions +Ġdisappoint ed +ĠCon vention +Ġview ing +M E +oc ity +Ġtown s +Ġlay ers +Ġro lled +Ġjump ed +Ġatt ribute +Ġun necess +inc oln +Ġsupp ose +ĠNet her +ch a +Ġbur ied +Ġsix th +B en +ress ing +OU R +Ġw ound +Ġcy cl +Ġmechan isms +Ġcongress ional +ĠE lement +Ġagre ements +Ġdec or +Ġclos est +ĠM it +Go ogle +} } +Ġm ixture +Ġflu id +S ign +ĠSch olar +Ġp ist +ask et +ab ling +Ġrac ing +he ro +ri el +ass y +Ġche aper +b en +Ġvert ical +amac are +ĠRead ing +g ments +Ġhelic op +Ġsacr ifice +ay a +p aren +V A +ĠL es +ĠStud io +Ġviol ations +ĠAn na +ac er +é ¾ +ĠR at +ĠBe ck +ĠD ick +ĠA CT +Ġcomp osition +Ġtext ure +ĠO wn +Ġsmart phone +ĠN A +Ġfor b +im port +Ġdef ending +il st +re r +Ġo h +ĠJere my +Ġbank ing +cept ions +Ġrespect ive +/ . +Ġdr inks +ĠW i +Ġb ands +ĠL iverpool +Ġg rip +ĠB uy +Ġopen ly +Ġreview ed +per t +Ġver ify +ĠCo le +ĠW ales +M O +Ġun pre +Ġshel ter +ĠIm perial +Ġgu i +ĠD ak +Ġsuggest ions +Ġexplicit ly +Ġsl ave +Ġblock chain +Ġcompet ing +Ġprom ising +S ON +Ġsoc cer +Ġconst itution +4 29 +Ġdist ract +ĠU ser +es ides +ĠMet hod +ĠTok yo +Ġaccompan ied +Cl ient +s ur +al og +Ġident ification +Ġinv asion +as ma +Ġindust ries +pp ers +Ġsub tle +ĠUn it +n atural +Ġsurv ived +Ġfl aw +ĺ ħ +ĠH oll +Ġdef icit +Ġtut orial +ĠCh ance +Ġarg uing +Ġcontem porary +Ġinteg ration +for ward +Ġt um +it is +Ġh iding +ĠD omin +ĠT an +ĠB uilding +ĠV in +Ġspokes person +ĠNot es +Ġemer ging +Ġprepar ation +Ġpro st +Ġsuspect s +Ġaut onom +D escription +Ġdeal t +ĠP ear +Ġstead y +Ġdecre ased +Ġso vere +ĠCl in +Ġgrad ually +ors es +ĠW AR +S erv +ãĤ ¢ +h r +Ġd irty +ĠB arn +ĠB C +Ġd il +Ġcal endar +Ġcompl iance +Ġch amber +b b +Ġpass enger +ate ful +ĠT itle +ĠSyd ney +ĠG ot +Ġdark ness +Ġdef ect +Ġpack ed +ass ion +Ġgod s +Ġh arsh +IC K +le ans +Ġalgorith m +Ġoxy gen +Ġvis its +Ġbl ade +Ġkil omet +ĠKent ucky +Ġkill er +P ack +enn y +Ġdiv ine +Ġnom ination +be ing +Ġeng ines +Ġc ats +Ġbuff er +ĠPh ill +Ġtra ff +AG E +Ġtong ue +Ġrad iation +ere r +m em +ĠExpl icit +é¾ į +Ġcou ples +Ġphys ics +ĠMc K +Ġpolit ically +aw ks +ĠBl oom +Ġwor ship +e ger +ut er +ĠF O +Ġmat hemat +Ġsent enced +Ġdis k +ĠM arg +Ġ/ * +P I +Ġoption al +Ġbab ies +Ġse eds +ĠScott ish +Ġth y +] ] +ĠHit ler +P H +ng th +Ġrec overed +ing e +Ġpow der +Ġl ips +Ġdesign er +Ġdis orders +Ġcour age +Ġch aos +" },{" +Ġcar rier +b ably +H igh +ĠR T +es ity +l en +Ġrout es +u ating +F il +N OT +w all +s burgh +Ġeng aging +ĠJava Script +ore r +li hood +Ġun ions +ĠF ederation +ĠTes la +Ġcomple tion +ĠT a +Ġprivile ge +ĠOr ange +Ġne ur +paren cy +Ġb ones +Ġtit led +Ġprosecut ors +ĠM E +Ġengine er +ĠUn iverse +ĠH ig +n ie +o ard +Ġheart s +ĠG re +uss ion +Ġmin istry +Ġpen et +ĠN ut +ĠO w +ĠX P +in stein +Ġbul k +S ystem +ic ism +ĠMarket able +Ġpre val +Ġpost er +Ġatt ending +ur able +Ġlicens ed +ĠG h +et ry +ĠTrad able +Ġbl ast +à ¤ +ĠTit an +ell ed +d ie +H ave +ĠFl ame +Ġprof ound +Ġparticip ating +Ġan ime +ĠE ss +Ġspec ify +Ġregard ed +ĠSpe ll +Ġs ons +own ed +Ġm erc +Ġexper imental +land o +h s +ĠDun geon +in os +Ġcomp ly +ĠSystem s +ar th +Ġse ized +l ocal +ĠGirl s +ud o +on ed +ĠF le +Ġconstruct ed +Ġhost ed +Ġsc ared +act ic +ĠIs lands +ĠM ORE +Ġbl ess +Ġblock ing +Ġch ips +Ġev ac +P s +Ġcorpor ation +Ġo x +Ġlight ing +Ġneighb ors +ĠU b +ar o +Ġbe ef +ĠU ber +F acebook +ar med +it ate +ĠR ating +ĠQu ick +Ġoccup ied +Ġaim s +ĠAdd itionally +ĠInt erest +Ġdram atically +Ġhe al +Ġpain ting +Ġengine ers +M M +ĠM ust +Ġquant ity +P aul +Ġearn ings +ĠPost s +st ra +ãĥ¼ ãĥ +Ġst ance +Ġdro pping +sc ript +Ġd ressed +M ake +Ġjust ify +ĠL td +Ġprompt ed +Ġscr ut +Ġspeed s +ĠGi ants +om er +ĠEd itor +Ġdescrib ing +ĠL ie +ment ed +Ġnow here +oc aly +Ġinst ruction +fort able +Ġent ities +Ġc m +ĠN atural +Ġinqu iry +Ġpress ed +iz ont +for ced +Ġra ises +ĠNet flix +ĠS ide +Ġout er +Ġamong st +im s +ows ki +Ġclim b +ne ver +Ġcomb ine +d ing +Ġcomp r +Ġsignific ance +Ġremem bered +ĠNev ada +ĠT el +ĠSc ar +ĠWar riors +ĠJ ane +Ġcou p +b as +Ġtermin al +, - +O H +Ġt ension +Ġw ings +ĠMy ster +�� �� +ĠUn like +val id +viron ments +ĠAl i +Ġn aked +book s +ĠM un +ĠG ulf +Ġd ensity +Ġdim in +Ġdesper ate +Ġpres idency +Ġ198 6 +h y +IN D +Ġun lock +im ens +Ġhand led +ĠE b +Ġdisapp eared +Ġgen re +Ġ198 8 +Ġdetermin ation +St ream +ik o +ap ters +Ġacknow ledge +J an +Ġcapital ism +P at +Ġ20 20 +Ġpain ful +Ġcur ve +Ġbom bs +st orm +ĠMet al +en cer +ĠF ig +ĠA aron +anc hes +Ġins piration +Ġexha ust +t ains +ash i +Ġdesc ript +Ġr itual +ĠChel sea +Ġpromot ion +ĠH ung +ĠW ard +iv a +ĠE T +Ġto ss +all ow +ĠFranc is +D ep +Ġhapp iness +ĠGl ass +Ġbet a +Ġstreng then +N E +o a +Ġbutt ons +ĠMur ray +Ġkick ed +Qu est +ĠT alk +ĠS everal +ĠZ ero +Ġdr one +ul k +Ġc am +ĠM obile +Ġprevent ing +Ġret ro +ĠA x +Ġcru el +Ġflo at +. ), +Ġfil ing +ĠGr ant +ĠB or +Ġr ib +Ġchampions hip +ĠM erc +Ġsty les +Ġc ake +Ġbuild s +ĠS elf +io x +Ġep ic +oy d +B el +ĠSt ew +. ( +ah u +ĠBe yond +Ġout s +Ġsol o +ĠT ree +Ġpres erve +Ġt ub +AR E +ro c +ĠIm pro +ĠW right +Ġbu nd +Ġtr aged +Ġoccas ional +b ian +Sec ond +r ons +Ġinter actions +form ed +s ing +Ġown s +Ġh ockey +Gener al +Ġlog ical +Ġexp end +Ġesc al +ĠGr iff +ĠC rown +ĠRes erve +Ġsto pping +Ġexc use +sec ond +Ġoper ated +Ġre aches +ĠMal ays +Ġpoll ution +ĠBrook lyn +Ġde lete +Ġhas h +Bl ock +ah a +âĢ ³ +Ġsh orter +p iece +> >> +ĠM ormon +t or +Ġpartic les +ĠB art +ry ption +Ġad min +Ġsqu ee +VID IA +Ġcreat or +iam eter +ic ular +N BC +Ġgrab bed +Ġn odd +Ġr ated +Ġrot ation +Ġgr asp +Ġexcess ive +ĠE C +ĠWh it +Ġinvent ory +ault s +ĠF B +Ġe cosystem +Ġbill ions +Ġvent ure +n amed +Ġdef ender +out e +Inst ead +ir able +W ar +Ġassum ption +Ġb ite +Ġearth qu +t ail +sp ace +Ġgif ts +boy s +Ġinev itable +Ġstruct ural +Ġbenef icial +Ġcompe lling +h ole +erv ation +Ġco at +o j +inc arn +ĠY ears +Ġdetermin ing +Ġrhet oric +Ġbound aries +Ġwh ites +A nt +add y +) - +ra ham +eter min +Ġhar vest +ĠCon c +Ġlapt op +ĠM atch +Ġenjoy ing +cc a +oll ar +Ġtri ps +Ġadd iction +ĠS ak +Ġpow ered +Ġc ous +ĠRuss ians +ie re +Ġret rie +qu ality +Ġdiff er +Ġking dom +ĠL aur +ĠCap itol +Ġcon clusions +ĠAl tern +ĠN av +Ġtrans parent +B ER +G roup +ĠCom plete +Ġinf er +Ġint rig +Ġins ane +R O +oph ob +is en +qu al +Mich ael +Ġm useum +ĠP ope +Ġres et +r ative +f ive +Ġagg reg +itte es +osit ory +Ġcar b +ĠRec ord +Ġdec ides +ĠF ix +Ġexcept ions +ĠCommission er +un s +ĠEnvironment al +Ġlegend ary +ist ence +Ġtun nel +k m +Ġins ult +Ġt roll +Ġsh ake +Ġdet ention +qu es +ĠCh rome +ĠF iles +Ġsub t +Ġprospect s +Ġpro l +re nder +pro of +Ġperform ances +St r +Ġh ref +ern ame +Ġachieve ment +Ġf ut +F ull +ĠLe ban +go ogle +ãĥ Ī +amp a +May be +Ġproject ed +ĠE mb +Ġcol leg +Ġa wards +Ġâ Ķ +G old +ĠBl ake +ĠR aj +if ting +Ġp ending +Ġinst inct +Ġdevelop ments +Con nect +ĠM and +ĠW ITH +ĠPhilipp ines +prof ile +Ġalt ogether +ĠB und +ĠT D +oo oo +amp ed +ip h +Ġste am +Ġold est +Ġdet ection +ul pt +Ġ ç +ĠWay ne +200 6 +f a +Ġcir cles +ĠF u +Ġdon ors +appropri ate +ĠDak ota +j amin +Ġmotiv ated +Ġpurch ases +ĠLouis iana +ĠS pl +Ġgl obe +Ġ10 5 +z ip +c all +Ġdepart ments +Ġsustain able +10 5 +ĠO P +if iers +Ġprevent ed +Ġinc omp +ĠComm ander +Ġdom inated +Ġ » +Ġinvest ed +Ġcomplex ity +Ġin cl +Ġens uring +Ġreal m +yn c +ĠInd ependent +r ained +ĠJ en +ĠFl ight +Ġat he +Ġspec ulation +ĠT E +oc ate +t ic +Ġpl aint +her ry +Ġto y +Ġ1 11 +Ġpl ates +st atus +ĠIs a +Ġdev oted +C op +ĠE S +25 5 +ur rency +M ain +Ġsl aves +Ġpe pper +Ġqu otes +Ġce iling +ĠF ish +Ġtrans formation +Ġfra ction +Ġadvant ages +Ġto ile +Ġstun ning +Ġmo ist +bre aking +s i +ĠL ocation +ĠMed ium +Ġtext s +Ġu gly +Ġb io +. âĢĶ +ĠB ased +Ġtr ains +ĠW ing +ĠAn cient +ĠRec ords +ĠH ope +Spe cial +ades h +ob i +[ / +Ġtempor arily +V er +h u +os er +Ġover night +Ġm amm +ĠTre asury +ĠV enezuel +ĠMeg a +Ġt ar +Ġexpect s +bl ack +or ph +\\ \\ +Ġaccept ance +Ġrad ar +s is +Ġjun ior +Ġfram es +Ġobserv ation +ac ies +P ower +ĠAdv anced +M ag +olog ically +ĠMe chan +Ġsent ences +Ġanaly sts +augh ters +force ment +Ġv ague +Ġcl ause +Ġdirect ors +Ġeval uate +Ġcabin et +M att +ĠClass ic +A ng +Ġcl er +ĠB uck +Ġresear cher +Ġ16 0 +Ġpoor ly +Ġexperien cing +ĠP ed +ĠMan hattan +Ġfre ed +Ġthem es +ad vant +Ġn in +Ġpra ise +10 4 +ĠLib ya +b est +Ġtrust ed +Ġce ase +Ġd ign +D irect +Ġbomb ing +Ġm igration +ĠSci ences +Ġmunicip al +ĠA verage +Ġgl ory +Ġreve aling +Ġare na +Ġuncertain ty +Ġbattle field +ia o +G od +Ġc inem +ra pe +el le +ap ons +Ġlist ing +Ġwa ited +Ġsp otted +ke ley +ĠAud io +e or +ard ing +idd ing +ig ma +ĠN eg +Ġl one +Ġ ---- +ex e +d eg +Ġtrans f +Ġwas h +Ġsl avery +Ġexpl oring +ĠW W +ats on +Ġen cl +l ies +ĠC reek +Ġwood en +Man ager +ĠBr and +um my +ĠAr thur +Ġbureau cr +Ġbl end +ar ians +F urther +Ġsupposed ly +Ġwind s +Ġ19 79 +Ġgrav ity +Ġanalys es +ĠTra vel +ĠV eter +Ġd umb +Ġaltern ate +g al +Ġconsum ed +Ġeffect iveness +.' ' +Ġpath s +ond a +L A +ĠStr ong +Ġen ables +Ġesc aped +Ġ" " +Ġ1 12 +Ġ198 3 +Ġsm iled +Ġtend ency +F ire +Ġp ars +ĠR oc +Ġl ake +Ġf itness +ĠA th +ĠH orn +Ġh ier +Ġimp ose +m other +Ġp ension +ic ut +bor ne +ic iary +. _ +ĠS U +Ġpol ar +is y +eng u +itial ized +AT A +w rite +Ġexerc ises +ĠD iamond +ot ypes +Ġharm ful +on z +Ġprint ing +st ory +Ġexpert ise +ĠG er +Ġtraged y +ĠF ly +Ġd ivid +amp ire +st ock +M em +Ġre ign +Ġun ve +Ġam end +ĠProp het +Ġmut ual +ĠF ac +Ġrepl acing +H ar +ĠCirc uit +Ġthro at +ĠSh ot +Ġbatter ies +Ġto ll +Ġaddress ing +ĠMedic aid +Ġp upp +ĠN ar +ol k +Ġequ ity +M R +ĠHis pan +ĠL arge +m id +D ev +Ġexp ed +Ġdem o +ĠMarsh all +erg us +Ġf iber +Ġdiv orce +ĠCre ate +Ġsl ower +ĠPark er +ĠStud ent +ĠTr aining +Ret urn +ĠT ru +Ġc ub +ĠRe ached +Ġpan ic +Ġqu arters +Ġre ct +Ġtreat ing +Ġr ats +ĠChristian ity +ol er +Ġsac red +Ġdecl are +ul ative +et ing +Ġdeliver ing +est one +Ġt el +ĠL arry +Ġmet a +ac cept +art z +ĠRog er +hand ed +Ġhead er +Ġtra pped +ĠCent ury +Ġkn ocked +ĠOx ford +Ġsurviv ors +b ot +Ġdemon stration +Ġd irt +Ġass ists +OM E +ĠD raft +ortun ate +fol io +pe red +ust ers +g t +ĠL ock +Ġjud icial +ver ted +Ġsec ured +out ing +ĠBook s +Ġhost ing +Ġlif ted +l ength +Ġj er +Ġwhe els +ĠR ange +umbn ails +Ġdiagn osis +te ch +ĠStew art +ĠP ract +Ġnation wide +Ġde ar +Ġoblig ations +Ġgrow s +Ġmand atory +Ġsusp icious +! ' +A pr +G reat +Ġmort gage +Ġprosecut or +Ġeditor ial +ĠK r +Ġprocess ed +ung le +Ġflex ibility +Ear lier +ĠC art +ĠS ug +Ġfoc uses +Ġstart up +Ġbre ach +ĠT ob +cy cle +ãĢ Į +ro se +Ġb izarre +ãĢ į +Ġveget ables +$ $ +Ġret reat +osh i +ĠSh op +ĠG round +ĠSt op +ĠHawai i +ĠA y +Per haps +ĠBe aut +uff er +enn a +Ġproduct ivity +F ixed +cont rol +Ġabs ent +ĠCamp aign +G reen +Ġident ifying +Ġreg ret +Ġpromot ed +ĠSe ven +Ġer u +ne ath +aug hed +ĠP in +ĠL iving +C ost +om atic +me ga +ĠN ig +oc y +Ġin box +Ġem pire +Ġhor izont +Ġbr anches +Ġmet aph +Act ive +ed i +ĠFil m +ĠS omething +Ġmod s +inc ial +ĠOrig inal +G en +Ġspir its +Ġear ning +H ist +Ġr iders +Ġsacr ific +M T +ĠV A +ĠS alt +Ġoccup ation +ĠM i +Ġdis g +lic t +Ġn it +Ġn odes +e em +ĠP ier +Ġhat red +ps y +ãĥ ī +Ġthe ater +Ġsophistic ated +Ġdef ended +Ġbes ides +Ġthorough ly +ĠMedic are +Ġbl amed +arent ly +Ġcry ing +F OR +pri v +Ġsing ing +ĠI l +Ġc ute +o ided +olit ical +ĠNe uro +å ¤ +Ġdon ation +ĠEag les +ĠG ive +T om +Ġsubstant ially +ĠLic ense +ĠJ a +Ġg rey +ĠAn imal +ĠE R +ĠU nd +Ġke en +Ġconclud e +ĠMississ ippi +Eng ine +ĠStud ios +P ress +o vers +ll ers +Ġ3 50 +ĠR angers +Ġr ou +ert o +E p +iss a +iv an +Ġse al +ĠReg ist +dis play +Ġwe aken +u um +ĠComm ons +ĠS ay +Ġcult ures +Ġl aughed +Ġsl ip +Ġtreat ments +iz able +m art +ĠR ice +Ġbe ast +Ġob esity +ĠLa ure +ig a +Wh ich +hold er +Ġelder ly +Ġp ays +Ġcompl ained +Ġc rop +Ġpro c +Ġexplos ive +ĠF an +ĠAr senal +A uthor +ef ul +Ġme als +Ġ( - +id ays +Ġimag ination +Ġann ually +Ġm s +as ures +H ead +ik h +m atic +Ġboy friend +ĠCom puter +Ġb ump +Ġsur ge +ĠCra ig +ĠKir k +D el +medi ate +Ġscen arios +ĠM ut +ĠSt ream +Ġcompet itors +Ù Ħ +ĠStan ford +ĠRes ources +az ed +b age +Ġorgan is +ĠRe lease +Ġsepar ately +Ġha bits +Ġmeasure ments +ĠCl ose +Ġaccomp any +Ġg ly +Ġt ang +ĠR ou +Ġplug in +Ġcon vey +ĠChall enge +oot s +j an +Ġcur s +ĠRel ations +ke eper +Ġapproach ing +p ing +Spe aking +Ġarrang ement +ĠV I +are ttes +Ġaffect ing +Ġperm its +b ecause +Ġu seless +ĠH us +!! !! +Ġdestro ying +Un fortunately +Ġfasc inating +S em +Ġelect oral +Ġtrans parency +ĠCh aos +Ġvolunte er +Ġstatist ical +Ġactiv ated +ro x +We b +H E +ĠHamp shire +is ive +M ap +Ġtr ash +ĠLaw rence +st ick +C r +Ġr ings +EX T +Ġoper ational +op es +D oes +ĠEv ans +Ġwitness ed +P ort +Ġlaunch ing +ec onom +w ear +ĠPart icip +um m +cul es +ĠR AM +ĠT un +Ġass ured +Ġb inary +Ġbet ray +Ġexpl oration +ĠF el +Ġad mission +it ated +S y +Ġav oided +ĠSim ulator +Ġcelebr ated +ĠElect ric +¥ ŀ +Ġcl uster +itzer land +he alth +L ine +ĠN ash +at on +Ġsp are +Ġenter prise +ĠD IS +clud es +Ġfl ights +Ġreg ards +ĠÃ Ĺ +h alf +Ġtr ucks +Ġcontact s +Ġunc ons +ĠCl imate +Ġimm ense +N EW +oc c +ect ive +Ġemb od +Ġpat rol +Ġbes ide +Ġv iable +Ġcre ep +Ġtrig gered +ver ning +Ġcompar able +q l +Ġg aining +ass es +Ġ( ); +ĠG rey +ĠM LS +s ized +Ġpros per +" ? +Ġpoll ing +Ġsh ar +ĠR C +Ġfire arm +or ient +Ġf ence +Ġvari ations +g iving +ĠP i +osp el +Ġpled ge +Ġc ure +Ġsp y +Ġviol ated +Ġr ushed +Ġstro ke +ĠBl og +sel s +ĠE c +,' ' +Ġp ale +ĠColl ins +ter ror +ĠCanad ians +Ġt une +Ġlabor atory +Ġn ons +t arian +Ġdis ability +ĠG am +Ġsing er +al g +ĠSen ior +Ġtrad ed +ĠWar rior +Ġinf ring +ĠFrank lin +Ġstr ain +ĠSwed ish +Ġsevent h +ĠB enn +ĠT ell +Ġsynd rome +Ġwond ered +id en +++ ++ +ig o +Ġpur ple +Ġjournal ism +Ġreb el +Ġf u +bl og +Ġinv ite +ren cies +ĠCont act +Is rael +ĠCont ent +Ġche er +Ġbed room +ĠEngine ering +ĠQue ens +Ġd well +ĠPlay Station +ĠD im +ĠCol on +l r +Ġoper ates +Ġmotiv ation +US A +ast ered +C ore +ĠTr uth +ol o +OS E +ĠMem ory +Ġpred ec +Ġan arch +Ġ19 20 +ĠY am +à ¨ +b id +Ġgr ateful +Ġexc itement +Ġtre asure +Ġlong est +ct ive +Ġdes erves +Ġreserv es +Ġcop s +ĠOtt awa +ĠEgypt ian +ank ed +Ġart if +Ġhypot hesis +: / +Ġpurch asing +Ġlove ly +H P +Ġdiv ide +Ġstrict ly +Ġquestion ing +Ġtaxp ayers +ĠJ oy +Ġroll s +ĠHe avy +Ġp orts +Ġmag netic +Ġinf lamm +Ġbr ush +t ics +â ĪĴ +Ġbott les +pp y +Ġp add +ãĤ ¯ +m illion +Ġdevast ating +Ġcomp iled +Ġmed ication +Ġtw elve +ĠPer ry +Sp ace +im b +y our +Ġle aked +ĠT ar +Ġun ity +Ġinfect ed +Ġtravel ed +ID E +ĠMc Donald +t xt +ĠPr inc +Ġinter ven +ĠTai wan +ĠP ow +Ġbe aring +ĠTh read +Ġz ones +iz ards +un ks +Ch apter +ll or +Ġ · +Ġw ounds +Ġdisc retion +Ġsucceed ed +ik ing +Ġicon ic +C all +Ġscreen ing +ĠM is +ict s +Ġmin isters +Ġsepar ation +Pl ayer +Ġb ip +Ġbel oved +Ġcount ing +ĠE ye +ar ound +ing ing +Ġtable t +Ġoff ence +in ance +h ave +ĠInf o +ĠNin ja +Ġprotect ive +ĠC ass +M ac +ĠQual ity +N orth +Ġ ic +ĠCub a +ĠChron icle +ĠPro perty +Ġfast est +ot os +ĠG erm +OW N +Ġbo om +ĠStan ley +ergus on +Ġcle ver +Ġent ers +m ode +ter ior +ĠS ens +Ġlin ear +AR K +Ġcomp aring +Ġpure ly +Ġsaf er +ĠPot ter +Ġc ups +R T +Ġgl uc +Ġatt ributed +Ġdu pl +ĠP ap +Ġprec ious +Ġp a +iction ary +ĠT ig +ĠTo o +ol utions +st an +Ġrob ots +Ġlob b +Ġstat ute +Ġprevent ion +w estern +16 0 +ĠAct ive +ĠMar ia +h al +N one +ell ar +ĠK B +ĠPart ners +ĠSing le +ĠFollow ing +ang o +ac ious +Ġth ou +Ġk g +Ġinflu ential +ĠFriend s +S ur +ain ted +Ġfor ums +Ġst arter +Ġcitizens hip +ĠE lection +on ge +ot ation +os ph +;; ;; +ut ical +p ur +ere n +Ġaccus ations +bit ious +ab bit +ĠOr d +Post ed +ir k +Ġsens itivity +ic he +ĠAm y +ĠF ab +Ġsum mit +Ġped est +Ġrub ber +Ġagric ultural +Ġcan cel +A E +Ġin aug +Ġcont am +Ġfirm ly +i w +st age +ĠK an +Ġt ier +Ġinv ention +Ġtransl ated +ĠR ules +B ox +Tw itter +ID S +Ġp izza +Ġdeb ug +ĠD rop +v s +Ġh orses +b ig +Ġb oring +Ġh ood +ĠMcC ain +at ched +ĠBro s +Ġsk ip +Ġess ay +st at +ĠLeg ends +Ġam munition +au c +Ġshoot er +Ġun h +Ġsuppl ied +Ġgener ic +ĠS K +ib an +yr ics +Ġ25 5 +Ġclim bing +Form er +Ġfl ip +Ġjump ing +Ġfrust ration +ĠTer ry +Ġneighborhood s +Ġmed ian +be an +Ġbr ains +Follow ing +Ġsh aped +Ġdraw s +Ġal tered +J ack +Ġrecip es +Ġsk illed +we alth +ach i +e lection +Ġbehavi ors +de als +ĠU ntil +F e +Ġdecl aration +mar ks +ĠBet ween +cel ona +Ġres on +Ġbub ble +Am ong +Ġim perial +G S +Ġfemin ist +200 5 +ĠK yle +Ġaccount ing +ĠTe le +ĠT yr +Ġconnect ing +Ġre hab +ĠP red +s im +Ġmeant ime +Ġphys ician +M W +ĠCamp bell +ĠBr andon +Ġcontribut ing +ĠR ule +ĠWe ight +ĠN ap +Ġinter active +Ġv ag +Ġhel met +ĠCom b +f our +Ġsh ipped +Ġcomple ting +ĠP D +PD ATE +Ġspread ing +Ġsc ary +erv ing +ĠG as +Ġfr ank +s chool +Ġrom antic +Ġstab il +R ob +Ġaccur ately +Ġac ute +ĠH ann +Ġsymbol s +Ġcivil ization +ĠA W +Ġlight ning +Ġcons iders +Ġven ue +Ġ × +Ġo ven +ĠS F +h is +Ġn u +ĠLear n +Ġpe oples +Ġst d +Ġsle e +Ġs lic +ĠStat istics +Ġcor ners +ĠB aker +Ġ: ) +ment ation +ol ver +Ġlaugh ing +ĠT odd +ond e +ĠH ills +Ġn uts +ĠW oman +pl ane +Ġl iver +ĠIn side +S orry +Ġagre es +Ġfund ament +ĠF isher +Ġa uction +Ġthread s +gl as +ĠBas ic +ĠN at +Ġlack ing +Ġceleb ration +j u +Ġs illy +E uro +Ġt att +ight y +cont rolled +T est +ĠSing h +Ġr age +Ġrh yth +o ffic +ĠPh antom +Ġhead lines +Ġrespond ing +ĠMor ning +Ġvit amin +Ġboot s +ĠS ite +al in +p i +Ġvir al +ĠU C +D ER +ĠSe x +Ġst ocks +c urrent +Ġch urches +ĠR are +ĠMur phy +Ġden ial +ĠG aming +Ġtou g +Ġn ick +Ġm akers +ĠRon ald +Ġgener ous +ĠD oc +ĠMor ris +Ġtransform ed +ĠN ormal +Ġ10 4 +ĠKick starter +ĠUp on +On line +ĠI RS +Ġw rap +Ġl oving +Ġarri ves +ĠD ue +Ġhe ter +ĠM ade +Ġrent al +Ġbelong s +Ġatt orneys +Ġcro ps +Ġmat ched +ul um +ol ine +10 9 +Ġdis par +Ġbuy ers +ĠCam bridge +Ġeth ics +rou ps +Ġjust ified +Ġmarg inal +Ġrespect ed +win ning +Ġnodd ed +ĠSer ge +ĠForm er +C raft +######## ######## +ĠWar ner +Ġd ash +et e +Ġent ert +ĠE scape +out heast +Ġkn ees +ĠB omb +Ġr ug +P ass +Ġatt itudes +go vernment +ĠPri or +Ġqual ities +Ġnot ification +ĠPh one +l ie +Ġanticip ated +ĠCom bat +ĠBar ry +Ġ198 2 +Us ers +on er +Ġcomput ing +ĠConnect icut +Ġless er +Ġpe ers +ĠC u +Ġtechn ically +Ġsub mission +ĠUn iversal +Ġman ually +our ge +Ġrespond ents +ĠB TC +ĠH ost +Ġf are +ĠB ird +Ġrece ipt +al so +Ġj ack +Ġagric ulture +Ġsk ull +Ġ! = +Ġpass ive +ĠC I +Ġsoc ieties +Ġremind ed +Ġinter ference +B uy +Ġâ ľ +g on +Ġscrut iny +ĠW itch +Ġconduct ing +Ġ ãĥ +Ġexch anges +ĠMit chell +Ġinhab it +Ġtw ist +B D +Ġwhere ver +group on +Ġj okes +ĠBen jamin +ĠR andom +fr ame +ĠL ions +Ġhighlight ed +ĠArk ansas +E nt +Ġp ile +Ġpre lim +g s +mind ed +Ġfel ony +ĠG A +ĠL uck +Ġpract ically +ĠB os +Ġact ress +D am +ĠB ou +Ġvis a +Ġembed ded +Ġhy brid +Ġear liest +Ġsoon er +s ocial +ĠH A +Ġste ep +Ġdis advant +Ġexplo it +ĠE gg +ĠUlt ra +Ġnecess ity +L ocal +ie ge +Ġd ated +Ġmass es +Ġsubsc ription +pl ess +Ġan onym +Ġpresum ably +Bl ue +The ir +asket ball +ĠPhil ip +Ġcom ed +load ed +r ane +Ġref lection +Ch ina +Ġext ends +Ġform ing +Ġund ers +200 1 +Ġgr at +Ġconcent rations +Ġins ulin +Ġsec ular +Ġwh ilst +Ġwin ners +Ad vertisements +Ġdeliber ately +ĠWork ing +Ġs ink +et ics +d ale +Ġmand ate +Ġg ram +Ġvac ation +Ġwarn ings +ri pp +ĠTH AT +Ġcomment ary +Ġint u +Ġa est +Ġreason ing +Ġbreak down +ĠZ ombie +Ġ-- > +ĠPolit ical +c ott +Ġthr ust +Ġtechn ological +Ġdec iding +Ġtraff icking +L ong +W elcome +pr ising +ĠCommun ications +Ġend ors +Ġsw ift +Ġmetab ol +co ins +res a +ĠHT TP +Ġen roll +ĠH appy +us r +int age +Ġ[ " +u ably +ĠM aterial +Ġrepe al +Se pt +k h +ĠMod i +Ġunder neath +ĠI L +sh ore +Ġdiagn osed +ace utical +Ġsh ower +au x +ĠSw itch +ĠStre ngth +Ġj ihad +n ational +Ġtra uma +uss y +on i +Ġcons olid +Ġcal ories +ĠF lynn +ag ged +16 8 +ĠP ink +Ġfulf ill +Ġch ains +Ġnot ably +ĠA V +L ife +ĠCh uck +m us +ĠUr ban +ĠH end +Ġdep osit +ĠS ad +Ġaff air +OR K +ie val +ĠF DA +Ġt rop +ĠOver all +Ġvirt ue +Ġsatisf action +au nd +Ġl un +ĠSw itzerland +ĠOper ation +pro cess +Ġsh ook +Ġcount ies +le ased +ĠCharl otte +1 12 +Ġtrans cript +Ġre dd +p ush +ĠHe y +ĠAn alysis +[ " +Ġaltern atives +ard less +Ġele ph +Ġpre jud +ĠLe af +H aving +ĠH ub +Ġexpress ions +ĠVol ume +Ġshock ing +ĠRed s +Ġread ily +Ġplan ets +ad ata +Ġcollaps ed +ĠMad rid +Ġir rit +i pper +ĠEn c +ĠW ire +Ġbu zz +ĠG P +ash a +Ġaccident ally +ur u +Ġfrust rated +ĠS A +Ġhung ry +ĠH uff +Ġlab els +ant o +ĠE P +Ġbar riers +) | +ĠBer keley +ĠJ ets +Ġp airs +ĠL an +J ames +ĠB ear +Ġhum or +ĠLiber ty +Ġmagn itude +Ġag ing +ĠM ason +Ġfriends hip +umb ling +Ġemer ge +Ġnewsp apers +Ġam bitious +ĠRich ards +atern al +Ġ198 1 +Ġcook ies +Ġsc ulpt +Ġpur suit +L ocation +Ġscript s +p c +Ġarrang ements +Ġd iameter +Ġl oses +am ation +Ġl iqu +ĠJ ake +aret te +Ġunderstand s +ĠZ en +v m +Ġappro ve +Ġw ip +Ġult ra +Ġint end +ĠD I +asc ular +Ġst ays +ĠK or +ĠK l +Ġinvest ing +L a +Ġbelie ving +b ad +m outh +Ġtaxp ayer +ãĥ ĥ +ĠQue bec +Ġl ap +ĠSw iss +d rop +Ġdr ain +ir i +et c +ft en +ĠN ex +Ġst raw +Ġscream ing +Ġcount ed +Ġdam aging +Ġamb assador +cent ury +Ġpro x +Ġarrest s +u v +il ateral +ĠCh arg +Ġpresc ribed +Ġindepend ently +Ġf ierce +ĠB aby +Ġb rave +Ġsu its += > +Ġbas eline +ĠR ate +Ġis lands +Ġ( ( +g reen +ix els +Ġname ly +ĠVill age +th an +am y +V ersion +g mail +ential s +ĠS ud +ĠMel bourne +Ġarri ving +Ġquant um +e ff +rop olitan +T ri +Ġfun eral +ĠI R +ÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤ ÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤ +ĠC ob +it ably +Ġt urb +Ġcomb o +Re view +Ġdeploy ment +u ity +ĠB ott +Ġinv isible +Ġrender ing +Ġunl ocked +Ġa qu +ĠVlad imir +Ġp ad +ĠBr ain +ĠLeg acy +dr agon +ĠKurd ish +Ġsound ed +Ġdet ained +ĠD M +g ary +Ġd aughters +Ġdistur bing +uk a +ĠPar ad +Ġt ast +Ġunf ortunate +Ġu l +em in +Ġattend ance +tr l +Ġpar ks +ĠMem orial +ĠAl ice +oth y +gu ard +ĠD ise +ĠSh an +ĠFor um +R ich +Ġshif ted +ue z +Ġl ighter +ĠMag n +Ġc od +S ch +ham mad +P ub +3 50 +ĠP okemon +Ġprot otype +Ġun re +B ase +ĠStud ents +ĠRep ly +ĠCommun ist +Ġg au +ĠTy ler +I Z +Ġparticip ated +Ġsup rem +ĠDet ails +Ġvessel s +ro d +Ġt ribe +ke ep +Ġassum ptions +Ġp ound +Ġcr ude +ĠAv ailable +Ġswim ming +Ġin clusion +Ġadv ances +c ulation +Ġconserv ation +Ġover d +ĠBuff alo +Art icle +ed ge +Ġaw a +ĠMad ison +Ġsid ew +Ġcat ast +ĠK rist +uc le +ĠHigh way +ĠTer ror +Ġactiv ation +Ġuncons cious +ĠSat an +ĠSus an +ill ery +Ġarr anged +i op +Ġrum ors +ur ring +th ink +ĠKe ith +ĠK ind +Ġavoid ing +by n +n ut +ĠSpe aker +r us +n ames +Ġgu ilt +ĠOlymp ics +Ġsa il +ĠM es +lev ant +ĠColumb us +a ft +C ity +S outh +ĠHar vey +ĠP un +S everal +Ġment ally +Ġimp ress +m ount +ĠUb untu +âĢĶâĢĶâĢĶâĢĶ âĢĶâĢĶâĢĶâĢĶ +ĠSuper man +ĠMP s +Ġintent ions +ĠR acing +Ġlike lihood +Ġ2 40 +T otal +Ġto ys +ĠW atson +Ġur ge +L ear +ĠP aper +Ġoccur ring +ĠB eng +ĠC ert +Ġst ones +T im +ĠTw in +z b +ĠD ynam +Ġpolit ician +k ens +ĠEnter prise +UT ERS +Ġab ol +Ġref resh +Ġarbit rary +pe ction +Ġtrou bles +Ġ} ); +t v +Ġpil ots +Ġdist ribute +Ġaud it +Ġp ause +orig inal +Ġr ivals + £ +F ig +T L +ab il +ry ing +L in +ion ed +l on +Ġf ancy +Ġcr ashed +Ġt ract +Ġshe d +Ġcons ume +B ased +down load +in it +Ġvolt age +Int rodu +Ġcondem ned +ĠFin ance +res pect +Ġex cluded +Ġestablish ing +her ic +Ġher itage +Ġspect acular +Ġun st +ĠSnow den +ĠL ane +S an +Ġprotect ions +st ruction +inc inn +Ġmac ro +C ustom +ios ity +Ġes p +Ġfunction ing +Ġm ush +Ġp uzzle +Ġeth ical +M al +Ġgo verning +ĠF erguson +Ġrest ored +Ġst ressed +ĠCoun ter +ĠK as +cl ip +AN S +Ġse iz +U K +by ss +old own +ap i +Ġperman ently +oun ters +W est +Th rough +L ight +at oes +Ġne at +Ġc ord +ure r +Ġsevere ly +ĠA ven +Ġinter rog +Ġtri ple +G iven +N umber +Ġar ise +Ġs her +pl ant +Ġfl ower +ĠC ou +Ġat e +Ġnew er +b ul +Ġmean while +ĠL air +Ġadjust ment +ĠCop yright +Ġd ivers +i ological +Ġgam ers +o at +Ġhistor ically +Ġanal og +Ġlong time +Ġpres cription +ĠM ist +ĠHy per +ĠM aine +ĠDe ity +Ġmulti pl +ĠRe incarn +ĠH yd +ĠP ic +S il +r ants +ĠC ris +. ; +( { +epend ence +Ġrec y +ate ur +Ġqu ad +Ġgl ob +Ġcon ced +te am +Ġcapital ist +ĠL ot +Ġroy al +ĠCy ber +Ġblack s +met ic +ri v +ĠD anny +Ġsp o +ĠR O +Ġanim ated +rypt ed +ĠDep uty +Ġrend ered +F E +Ġstre ak +Ġcloud s +ĠDou g +~~~~ ~~~~ +Ġdisc our +ĠVe h +Ġpsych ology +ĠJ ourney +Ġcry stal +ĠFro st +Ġsuspic ion +Ġrel ate +or us +ĠC rypt +ĠN VIDIA +com ed +ut ing +incinn ati +Ġvulner ability +ost ic +Ġisol ation +Ġcool ing +ĠCoal ition +Ġ1 19 +F our +ĠDe al +Ġâ ī +se mble +ram ent +ĠBar celona +Ġ10 2 +Ġcoc aine +ocaly pse +F eb +ogen ic +Ġmut ation +Ġcrypt oc +ĠK el +ĠG it +a is +Ġs isters +AN K +Ġactiv ate +T er +Ġd read +yl on +Ġprop ri +A ust +ĠDef ault +Ġout door +Ġshe er +ce ive +Ġg ently +Ð ¾ +Pro gram +Ġâ ĨĴ +Ġve gan +ĠCr us +Ġrespons ibilities +ĠH R +OL D +Ġprev ents +Ġst iff +ĠW ere +Ġathlet ic +ĠSc ore +Ġ) : +Ġcolumn s +ĠL oc +av ailable +ĠF ram +ĠS essions +Ġcompan ion +Ġpack s +14 0 +ĠKn ights +Ġf art +Ġstream s +Ġsh ore +Ġapp eals +ĠPer formance +h aul +ĠSt ra +ĠN ag +10 3 +ĠTrans portation +B B +E v +z an +P ublic +Ġtw in +uls ion +M ult +Ġelect ro +Ġstat ue +ation ally +ĠN ort +Ġins pection +/ * +ig ue +Ġcomp assion +ĠT ales +ĠSte in +ĠSc reen +ĠB ug +ĠL ion +g irl +Ġwithdraw al +Ġobject ives +Ġblood y +Ġprelim inary +Ġj acket +Ġdim ensions +ĠC ool +ĠOcc up +Ġw reck +Ġdoub led +ank ing +Ġ19 75 +Ġglass es +ĠW ang +pro v +P ath +connect ed +ĠMult i +ĠNor way +agon ist +Ġfe ared +Ġtouch ing +Ġarg uably +¯¯¯¯ ¯¯¯¯ +ĠNC AA +che m +Ġsp at +ĠW WE +ĠC el +ig ger +Ġattack er +ĠJo in +ob ject +ett a +Ġelim inated +d et +Ġdest ruct +ĠLuc as +ct uary +18 0 +ĠBr ady +ĠBl ues +B ay +au kee +Ġtim eline +Ġdeleg ates +w ritten +uff icient +Ġsh apes +Cop yright +ou ble +serv ice +Ġp ione +Ġcolleg es +Ġrow s +Ġsp ite +Ġassess ed +3 60 +Ġle ase +Ġconfident ial +ck er +ĠMan ning +ĠV oice +Ġse aled +Ġcalcul ate +N O +ĠAss istant +Ġteen ager +ul ent +ather ine +Ġm ock +Ġd iamond +Ġf est +Ġsw itched +Ġres ume +ĠPu erto +Ġl anes +ir ation +ĠSimilar ly +Ġro d +ĠS el +ĠPal ace +ĠLim ited +e ous +Ġvar iant +Ġw ard +Ġ) ) +Sh ow +OO K +A lex +ĠN ep +br is +ĠWik ipedia +Ġexcept ional +Ġman ages +ĠD raw +Ag ain +Ġco pper +ut t +Ġex ports +Ġport folio +Ġelev ated +R ated +ĠOther wise +ĠT act +ĠShe l +ĠT X +" âĢĶ +Ġres ur +ĠW a +ven ant +Ġmon etary +pe ople +E mail +Ġfif ty +ĠS weet +ĠMalays ia +Ġconf using +ĠR io +ud a +uten ant +" ); +Ġpra ised +Ġvol umes +t urn +Ġm ature +Ġnon profit +Ġpassion ate +ĠPriv ate +Ġ10 3 +Ġdesc end +ç ¥ŀ +uff y +head ed +Whe ther +ri en +ze ch +be it +Ġch rom +ĠMc M +Ġd ancing +Ġe leg +ĠNot iced +11 5 +Ġadvoc acy +ENT S +amb ling +ĠMin or +ĠF inn +Ġprior ities +Ġthere of +ĠSt age +ĠRog ers +Ġsubst itute +ĠJ ar +ĠJeff erson +Ġlight ly +10 2 +ĠL isa +u its +ys ical +Ġshif ts +Ġd rones +Ġwork place +Ġres id +ens ed +ah n +Ġpref erences +ser ver +Ġdeb ates +d oc +ĠGod s +Ġhelicop ter +Ġhon our +Ġconsider ably +ed ed +ĠF emale +ĠAn ne +Ġre un +ĠF ace +ĠHall ow +ĠBud get +Ġcondem n +Ġt ender +Pro f +ocr atic +ĠTurn er +ĠAg ric +Ġ19 76 +Ġa pt +d isc +ĠF ighter +ĠA ur +Ġgar bage +in put +ĠK arl +ĠOl iver +ĠL anguage +k n +N on +ĠCl ar +Ġtrad itions +Ġad vertisement +ĠS or +Ġarch ive +Ġvill ages +7 50 +Ġimplement ing +w aukee +Ġdiet ary +Ġswitch ing +Rep ublic +Ġvel ocity +Ġc it +ĠA wards +Ġfin ancing +Ġlast ed +) ] +Ġrem inder +P erson +Ġprec ision +Ġdesign ers +ĠF ried +ĠB order +Ġtr agic +Ġw ield +Ġiniti atives +ĠT ank +w er +Ġjo ins +R o +in ery +Ġar row +Ġgener ating +found er +Ġsear ches +Ġrandom ly +A ccess +Ġb atch +Ġp osed +l at +Ġpursu ing +as a +Ġtest ified +form ing +ĠSh ar +w iki +ĠE ither +S ometimes +Ġsen ators +ĠJohn ny +ĠTal iban +ĠG PS +":" / +ãģ® å +Ġanaly zed +ĠRub io +ĠMove ment +op ard +ii i +St and +f ight +Ġign oring +i ang +ĠG N +so ever +ĠST AT +Ġref using +Ġswe at +Ġb ay +P ORT +ir med +ak y +Ġdis pro +Ġlabel ed +Ġ10 8 +H ello +Ġple asant +ab a +Ġtri umph +Ġab oard +Ġinc om +ĠC row +le tt +Ġfol k +Ġch ase +` ` +ĠBr us +Ġte ens +c ue +Ġter rain +h yd +il ight +OR Y +Su pport +ew s +ll i +rain ts +ĠC and +Ġab used +ach ment +l arg +B as +ĠC ancer +Ġ19 78 +Ġsupp orter +ac cess +ĠTer min +ĠT ampa +ĠAN Y +Ġnew est +ĠCrim inal +ed u +Ġ19 30 +Ġadm its +Ġend e +Ġfail ures +ur ate +ful ness +cy cl +ĠSub ject +Ġinf inite +th ree +W A +p it +ĠInst all +R ad +ili ation +G M +Ġcontin ent +Ġaccommod ate +ĠCl ay +Ġp up +ĠF unction +Ġham mer +ĠAlbert a +Ġrev ised +Ġminor ities +Ġmeasure ment +Con nell +Ġdis able +ĠM ix +In cre +Ġfor k +ĠR osen +Ġimpl ies +umb lr +AN G +Ġprote ins +Ġagg ression +Ġfacilit ate +S N +Ġilleg ally +u er +Ġacad em +Ġp uzz +ĠSh ift +p ay +oll o +Ġaud iences +B uild +Ġno ble +Ġsynt ax +â ĺħ +Ġbe am +ĠB ed +ĠA ld +Ġorig ins +v ideo +Ġ19 77 +ĠAss ault +Ġgar age +Te am +Ġver dict +Ġd war +ĠVirt ual +e vent +Ke ep +Ġsent iment +Ġwild life +sh irt +Ġb urg +Ġrecommend ation +rep resent +Ġgall ery +own ers +Ġsch olar +Ġconven ience +ĠSw ift +Ġconv inc +C ap +Ġwar fare +ĠVis ual +Ġconst itute +Ġab ort +ĠWe ather +ĠLook ing +ĠH em +Ġmart ial +Ġinc oming +et ition +Ġtoler ance +ĠCre ated +Ġfl ows +ĠE lder +Ġsoul s +Ġf oul +ĠP ain +ĠC AN +Ġ2 20 +b c +he nd +Ġgen ius +R eal +ĠW r +omet er +p ad +Ġlim iting +ĠS i +ĠL ore +ĠAd ventures +Ġvar ied +D isc +f in +ĠPerson al +Ch ris +Ġinv ented +Ġd ive +ĠR ise +Ġo z +ĠCom ics +Ġexp ose +ĠRe b +let ters +s ite +im ated +Ġh acking +Ġeduc ated +ĠNob ody +Ġdep ri +Ġincent ive +ãĤ · +Ġovers ight +Ġtrib es +ĠBelg ium +Ġlicens ing +our t +Produ ct +ah l +ĠG em +Ġspecial ist +Ġc ra +ann ers +ĠCor byn +Ġ19 73 +RE AD +Ġsum mar +Ġover look +ĠApp lication +Ġin appropriate +Ġdownload ed +Q ue +ĠB ears +Ġth umb +ĠChar acter +ĠReincarn ated +ĠS id +Ġdemonstr ates +s ky +ĠBloom berg +ĠAr ray +ĠRes ults +ĠFour th +ĠED T +ĠO scar +c end +Ġ10 6 +ĠN ULL +ĠH ERE +m atch +ĠBr un +Ġgluc ose +ie g +eg u +Ġcert ified +Ġrel ie +Ġhuman itarian +Ġpr ayers +K ing +Ġn an +h ou +10 8 +ul u +Ġrenew able +Ġdistingu ish +Ġd ense +ĠV ent +ĠPack age +ĠB oss +Ġedit ors +Ġm igr +T ra +ĠPet ers +ĠAr ctic +200 4 +ĠC ape +Ġloc ally +Ġlast ing +Ġhand y +. ). +P an +ĠR ES +Ind ex +Ġt ensions +Ġformer ly +Ġide ological +Ġsens ors +Ġdeal ers +Ġdef ines +S k +Ġproceed s +Ġpro xy +az ines +ĠB ash +ĠP ad +ĠC raft +eal ous +Ġshe ets +omet ry +J une +cl ock +T T +ĠThe atre +ĠB uzz +Ġch apters +Ġmill enn +Ġd ough +ĠCongress ional +Ġimag ined +av ior +Ġclin ic +Ġ19 45 +Ġhold er +ro ot +oles ter +Ġrest art +B N +ĠHam as +ĠJ ob +Ġor b +Ġr am +Ġdiscl ose +Ġtransl ate +Ġimm igrant +Ġannoy ing +Ġtreat y +an ium +ĠTe a +ĠLeg ion +Ġcrowd s +ĠB ec +ĠA er +oh yd +B ro +Look ing +Ġl bs +Ġagg ress +Ġse am +Ġinter cept +ĠM I +mer cial +act iv +ĠC it +Ġdim ension +Ġconsist ency +Ġr ushing +ĠDou glas +Ġtr im +Inst all +ick er +Ġsh y +10 6 +Ġment ions +pe lled +ĠT ak +c ost +Ġclass room +Ġfort une +dri ven +Ġun le +ĠWhe el +Ġinvest or +ĠM asters +k it +Ġassoci ations +ĠEv olution +op ing +us cript +Ġprov incial +ĠWal ter +av i +S O +Ġun limited +Eng lish +ĠC ards +ĠEb ola +ne red +Ġreven ge +Ġout right +um per +Ġf itting +ĠSol id +Ġform ally +Ġproblem atic +Ġhaz ard +Ġenc ryption +Ġstraight forward +ĠA K +Ġp se +ĠOr b +ĠCh amber +ĠM ak +Cont ents +Ġloyal ty +Ġl yrics +ĠSy m +Ġwel comed +Ġcook ed +Ġmon op +Ġn urse +Ġmis leading +Ġe ternal +Ġshif ting +Ġ+ = +V is +Ġinst itutional +ill ary +Ġp ant +VER T +ĠA CC +ĠEn h +Ġinc on +ĠRE UTERS +Ġdon ated +âĢ¦âĢ¦ âĢ¦âĢ¦ +In tern +Ġexhib it +Ġt ire +ĠR ic +ĠCh ampion +ĠMu hammad +N ING +ĠSoc cer +Ġmob ility +Ġvary ing +ĠM ovie +Ġl ord +o ak +F ield +Ġve ctor +us ions +Ġsc rap +Ġen abling +m ake +T or +. * +| | +ĠWe bsite +ĠN PC +Ġsocial ist +ĠBill y +ĠAdd itional +Ġc argo +Ġfar ms +ĠSo on +ĠPri ze +Ġmid night +Ġ9 00 +se en +ĠSp ot +Ġshe ep +Ġspons ored +ĠH i +ĠJ ump +Ġ19 67 +Micro soft +ĠAg ent +Ġch arts +d ir +Ġadj acent +Ġtr icks +Ġman ga +Ġex agger +/ > +foot ball +ĠF CC +G C +ĠT ier +and ra +OU ND +% ), +Ġfru its +V C +ĠA A +R ober +Ġmid st +â Ĺ +ank a +Ġlegisl ature +ĠNe il +Ġtour ists +" " +ĠWar ning +ĠNever theless +ĠOffic ial +ĠWh atever +Ġm old +Ġdraft ed +Ġsubst ances +Ġbre ed +Ġt ags +ĠT ask +Ġver b +Ġmanufact ured +com ments +ĠPol ish +Pro v +Ġdetermin es +Ob ama +k ers +Ġutter ly +Ġse ct +sc he +ĠG ates +ĠCh ap +Ġal uminum +Ġz ombie +ĠT ouch +ĠU P +Ġsatisf y +Ġpred omin +asc ript +Ġelabor ate +Ġ19 68 +Ġmeas uring +ĠV ari +any ahu +Ġs ir +ul ates +id ges +ick ets +ĠSp encer +T M +oub ted +Ġpre y +Ġinstall ing +ĠC ab +re ed +re ated +Su pp +Ġwr ist +ĠK erry +10 7 +ĠK le +ĠR achel +Ġc otton +ĠA RE +ĠE le +Cont rol +Ġload s +ĠD od +an as +b one +Ġclass ical +ĠReg ional +ĠInt eg +V M +Ġdes ires +Ġaut ism +support ed +ĠM essage +Ġcomp act +writ er +Ġ10 9 +ĠHur ricane +c ision +Ġcy cles +Ġdr ill +Ġcolle ague +Ġm aker +G erman +Ġmist aken +S un +ĠG ay +Ġwhat soever +Ġsell s +ĠA irl +l iv +ĠO ption +Ġsol ved +Ġse ctors +Ġhorizont al +Ġequ ation +ĠSk ill +ĠB io +g ement +ĠSn ap +ĠLeg al +Ġtradem ark +Ġmake up +Ġassemb led +Ġsa ves +ĠHallow een +ĠVer mont +ĠFR OM +Ġfar ming +ĠP odcast +accept able +ĠHig her +Ġas leep +ull ivan +Ġrefere n +ĠLe v +Ġbul lets +ok o +H C +Ġst airs +Ġmain tains +ĠL ower +ĠV i +Ġmar ine +Ġac res +Ġcoordin ator +ĠJ oh +Ġcounterpart s +ĠBrother s +Ġind ict +b ra +Ġch unk +Ġc ents +H ome +ĠMon th +Ġaccording ly +if les +ĠGerm ans +ĠSy n +H ub +Ġey eb +âĶĢâĶĢ âĶĢâĶĢ +Ġr anges +ĠHoll and +ĠRob ot +f c +M ike +Ġpl asma +Ġsw ap +Ġath lete +ĠR ams +,' " +Ġinfect ions +Ġcor rid +Ġv ib +Ġpat ches +Ġtradition ally +Ġrevel ation +Ġswe ep +Ġgl ance +Ġin ex +200 3 +ĠR aw +work ing +os ures +ĠD at +ĠLyn ch +Ġle verage +ĠRe id +Ġcorrel ation +ian ces +av ascript +Ġrep ository +ret ty +Ġ19 72 +24 0 +Ġo un +p ol +ĠRe ed +Ġtact ical +is ite +App le +ĠQu inn +Ġrap ed +ill o +Euro pe +Ġalgorith ms +ĠRod rig +i u +Ġill um +Ġf ame +Ġintrodu cing +Ġdel ays +ĠRaid ers +Ġwh istle +Ġnovel s +ĠRe ally +Ġder iv +Ġpublic ations +ĠNe ither +ĠCom merce +Ġa ston +l anguage +Not es +ĠR oth +ĠF ear +Ġm ate +Ġpar ade +ĠQ B +Ġman eu +ĠC incinnati +m itting +Ġwa ist +ĠR ew +Ġdisc ont +Ð ° +Ġst aring +Ġal ias +Ġsec urities +Ġtoile t +ĠJ edi +Ġun law +v ised +//// //// +] ( +ĠWe iss +Ġpre st +ĠComp an +Ġmem o +ĠGr ace +J uly +ĠEl ite +cent er +ĠSt ay +Ġgal axy +Ġto oth +ĠS ettings +Ġsubject ed +ãĤ ¦ +Ġline back +Ġretail ers +ĠW ant +Ġd angers +A ir +Ġvolunt ary +ew ay +Ġinterpret ed +ot ine +à § +Ġp el +Serv ice +ĠEvent ually +Ġcare ers +Ġthreat en +Ġmem or +ĠBrad ley +anc ies +s n +ĠUn known +N ational +Ġsh adows +ail and +ĠD ash +Every one +izz ard +M arch += ( +Ġpull s +Ġstr anger +Ġback wards +ĠBern ard +imens ional +Ġch ron +Ġtheoret ical +k top +Ġw are +ĠInvest ig +ĠIn iti +ĠOper ations +o ven +oc ide +* / +Ġfl ames +ĠC ash +sh it +Ġc ab +ĠAn aly +ĠSe ah +Ġdefin ing +Ġorder ing +Ġimm un +Ġpers istent +AC H +Russ ian +m ans +Ġh ind +Ġphot ography + © +Ġh ug +Ġ10 7 +ĠH ence +i ots +ude au +Ġsubsid ies +Ġroutine ly +ĠDev ice +it ic +Ġdisg ust +land er +Ġ19 40 +Ġassign ment +ĠB esides +w ick +ĠD ust +us c +struct ed +11 1 +de velop +Ġf ond +Ġinter section +Ġdign ity +Ġcommission er +With out +re ach +Ġcart oon +Ġsc ales +ãĥ Ń +F IG +Ġsurve ys +ĠIndones ia +Ġart work +Ġun ch +Ġcy cling +un ct +au er +or ate +ĠOb viously +Ġcharacter ized +fe ld +Ġaff irm +Ġinn ings +Ġ é +Ġal iens +Ġcl oth +et ooth +ĠC ertain + § +Ġdig est +k now +ĠX L +Ġpredict ions +Ġd in +W AR +Ġafter math +Ex ample +ĠSu ccess +ĠTh r +IG N +Ġmin er +B us +Ġcl arity +heim er +ĠO UT +ĠS end +ĠCirc le +ĠD iet +Ġpron ounced +Ġcreat ors +Ġearthqu ake +atter y +ge ons +Ġo d +Ġlay ing +or p +U lt +pro ject +Ġunder min +Ġsequ el +S am +ĠDark ness +Ġre ception +b ull +Y S +ĠV ir +Ġsequ ences +ĠCo in +Ġout fit +ĠW ait +1 19 +Ġdel ivers +.... .. +Ġbl own +ĠE sc +ĠM ath +per m +ĠU l +Ġgl im +Ġfac ial +Ġgreen house +Ġto kens +/ - +ĠAnn ual +ĠON E +Ġteen age +ĠPhys ical +ĠL ang +ĠC elt +Ġsu ed +ivid ually +Ġpat ience +ch air +reg ular +Ġa ug +in v +ex cept +ĠL il +Ġn est +f d +s um +ĠCh ase +Russ ia +ĠJenn ifer +Ġoff season +Over all +F ore +Ġr iot +A ud +form er +Ġdefend ers +ĠC T +iot ic +rib ly +Ġautom ated +Ġpen is +Ġins ist +Ġdi agram +ĠS QL +ĠG arc +Ġw itch +cl ient +ier ra +am bers +Ġrec ount +f ar +V ery +oster one +Ġappreci ated +ĠPer fect +S ection +Ġd oses +oca ust +Ġcost ly +Ġg rams +ĠSh i +Ġwrest ling +Ġ19 71 +Ġtro phy +Ġn erve +ĠK az +ĠExper ience +Ġpled ged +Ġplay back +Ġcreat ivity +by e +Ġattack ers +Ġhold ers +ĠCo ach +ĠPh D +Ġtransf ers +Ġcol ored +ĠH indu +Ġd rown +Ġlist ened +ĠW A +ias m +P O +Ġappeal ing +Ġdiscl osed +ĠCh icken +ag ging +Ġple aded +Ġnav igation +ĠReturn s +Ġ[ [ +R OR +E A +Ġphotograp her +ĠR ider +ipp ers +Ġsl ice +Ġe rect +Ġhe d +iss ance +ĠVik ings +ur ious +Ġapp et +oubted ly +Ch ild +Ġauthent ic +o os +ĠM aking +Ġannoun cing +Ġb od +Ġmet er +ĠN ine +ĠR ogue +Ġwork force +Ġrenew ed +Ġorganis ations +ac s +P LE +Sh ort +Ġcomp ounds +ĠVis it +Ġen velop +ear th +Ġsupport ive +gg le +ĠBrus sels +ĠGu ild +Cre ate +RE L +Ġaver aged +Ġ19 69 +ri ages +Ġlength y +Ġforg ot +O kay +ĠE rd +Ġdeal er +Ġrec ession +D D +Ġdesper ately +Ġhun ger +Ġst icks +Ġm ph +ĠF aith +Ġintention ally +Ġdem ol +ue ller +ĠS ale +Ġde bris +s pring +Ġle ap +>> >> +Ġcontain ers +se lling +rane an +atter ing +Ġcomment ed +ĠC M +on ut +Ġwood s +es pecially +Ġorgan ize +iv ic +ĠWood s +ang a +s qu +Ġm aj +am on +Ġax is +Ġ19 74 +ĠDen mark +Ġwar rior +ĠP and +Ġout lined +ĠB O +ins ula +z illa +eb ook +Ġd are +Ġsear ched +Ġnav igate +S n +writ ing +Ġun ited +J apan +ĠHe brew +Ġfl ame +Ġrel ies +Ġcatch ing +ĠSh o +Ġimprison ment +Ġp ockets +Ġclos ure +ĠF am +t im +ade qu +Act ivity +Ġrecru iting +ĠW ATCH +ĠArgent ina +d est +Ġapolog ize +or o +Ġlack s +Ġtun ed +ĠGriff in +Ġinf amous +Ġcelebr ity +ss on +Ġ ---------------------------------------------------------------- +ĠIs is +ĠDis play +Ġcred ibility +Ġeconom ies +Ġhead line +ĠCow boys +Ġind ef +Ġl ately +Ġincent ives +but ton +ĠM ob +A ut +Ġres igned +ĠO m +c amp +Ġprof iles +Ġsche mes +olph ins +ay ed +Cl inton +en h +ĠY ahoo +Ġab st +Ġan k +su its +Ġw ished +ĠMar co +udd en +Ġsp here +ĠB ishop +Ġincorpor ated +ĠPl ant +11 4 +Ġh ated +p ic +Ġdon ate +Ġl ined +Ġbe ans +Ġsteal ing +Ġcost ume +Ġsher iff +Ġfor ty +Ġint act +Ġadapt ed +Ġtrave lling +b art +Ġnice ly +Ġdri ed +Ġsc al +os ity +NOT E +ĠB h +ĠBron cos +ĠI gn +Ġint imate +Ġchem istry +Ġopt imal +D eb +ĠGener ation +Ġ] , +ich i +ĠW ii +ĠYOU R +vent ions +W rite +Ġpop ul +un ning +ĠW or +V ol +Ġqu een +head s +K K +Ġanaly ze +op ic +ear chers +Ġd ot +leg raph +ast ically +Ġupgr ades +Ġca res +Ġext ending +Ġfree ze +Ġin ability +Ġorg ans +Ġpret end +Ġout let +11 3 +ol an +ĠM all +ul ing +t alk +Ġexpress ing +ĠAl ways +ĠBe gin +f iles +Ġlic enses +% % +ĠM itt +Ġfil ters +ĠMil waukee +G N +Ġunf old +M o +Ġnut rition +pp o +B o +Ġfound ing +Ġunder mine +Ġeas iest +ĠC zech +ĠM ack +Ġsexual ity +ĠN ixon +W in +ĠAr n +ĠK in +ãĤ £ +ic er +Ġfort un +Ġsurf aces +agh d +Ġcar riers +ĠP ART +ĠT ib +Ġinter val +Ġfrust rating +ĠSh ip +ĠAr med +ff e +Ġbo ats +ĠAb raham +in is +Ġsu ited +th read +i ov +ab ul +ĠVenezuel a +Ġto m +su per +Ġcast le +alth ough +iox ide +ec hes +Ġevolution ary +Ġnegoti ate +Ġconfront ed +Rem ember +Ġ17 0 +S uch +Ġ9 11 +m ult +ĠA byss +ur ry +ke es +spe c +ĠBarb ara +Ġbelong ing +Ġvill ain +ist ani +Ġaccount able +Ġport ions +ĠDe cl +U r +ĠK ate +g re +Ġmag azines +UC K +Ġregul ate +om on +ĠAl most +Ġover view +Ġsc ram +Ġl oot +ĠF itz +Ġcharacter istic +ĠSn ake +s ay +ĠR ico +Ġtra it +ĠJo ined +au cus +Ġadapt ation +ĠAirl ines +Ġarch ae +ĠI de +Ġb ikes +Ġliter ary +Ġinflu ences +ĠUs ed +C reat +Ġple a +ĠDef ence +ĠAss ass +Ġp ond +UL T +) " +Ġeval uated +Ġob taining +Ġdem ographic +Ġvig il +ale y +Ġsp ouse +ĠSeah awks +resp ons +ĠB elt +um atic +Ġr ises +run ner +ĠMichel le +Ġpot ent +r ace +ĠP AC +F ind +olester ol +IS S +ĠIntrodu ced +ress es +ign ment +O s +ĠT u +ĠDe x +ic ides +Ġspark ed +ĠLaur a +ĠBry ant +Ġsm iling +ĠNex us +Ġdefend ants +ĠCat al +Ġdis hes +sh aped +Ġpro long +m t +( $ +ãĢ Ĥ +Ġcalcul ations +ĠS ame +Ġp iv +H H +Ġcance lled +Ġgr in +Ġterrit ories +ist ically +C ome +ĠP arent +Pro ject +Ġneg lig +ĠPriv acy +Ġam mo +LE CT +olute ly +ĠEp ic +Ġmis under +w al +Apr il +m os +path y +ĠC arson +Ġalbum s +ĠE asy +Ġpist ol +< < +Ġ\ ( +t arget +hel p +Ġinter pre +cons cious +ĠH ousing +ĠJ oint +12 7 +Ġbe ers +s cience +ĠFire fox +effect ive +ĠC abin +ĠO kay +ĠApp lic +Ġspace craft +ĠS R +ve t +ĠStr ange +S B +Ġcor ps +iber al +e fficient +Ġpreval ence +Ġeconom ists +11 8 +Th read +ord able +OD E +ĠC ant +=- =- +if iable +ĠA round +Ġpo le +Ġwilling ness +CL A +ĠK id +Ġcomple ment +Ġsc attered +Ġin mates +Ġble eding +e very +Ġque ue +ĠTr ain +Ġh ij +Ġme lee +ple ted +Ġdig it +Ġg em +offic ial +Ġlif ting +Ð µ +Re qu +it utes +Ġpack aging +ĠWork ers +h ran +ĠLeban on +ol esc +Ġpun ished +ĠJ uan +Ġj am +ĠD ocument +Ġm apping +ic ates +Ġinev itably +Ġvan illa +ĠT on +Ġwat ches +Ġle agues +Ġiniti ated +deg ree +port ion +Ġrec alls +Ġru in +Ġm elt +I AN +Ġhe m +Ex p +Ġb aking +ĠCol omb +at ible +Ġrad ius +pl ug +ĠI F +et ically +Ġf ict +H ER +ĠT ap +atin um +Ġin k +Ġco h +ĠW izard +b oth +te x +Ġsp ends +ĠCurrent ly +ĠP it +Ġneur ons +ig nt +Ġr all +Ġbus es +b uilding +Ġadjust ments +Ġc ried +ibl ical +att ed +ĠZ ion +ĠM atter +Ġmed itation +ĠD ennis +Ġour s +ĠT ab +Ġrank ings +ort al +Ġad vers +Ġsur render +ĠG ob +ci um +om as +im eter +Ġmulti player +Ġhero in +Ġoptim istic +Ġindic ator +ĠBr ig +Ġgro cery +Ġapplic ant +ĠRock et +v id +Ex ception +p ent +Ġorgan izing +Ġenc ounters +ĠT OD +Ġjew el +S ave +ĠChrist ie +Ġhe ating +Ġl azy +ĠC P +Ġcous in +Con fig +Ġreg ener +Ġne arest +Ġachie ving +EN S +th row +ĠRich mond +ant le +200 2 +Ġan ten +b ird +13 3 +Ġn arc +r aint +un ny +ĠHispan ic +ourn aments +Ġprop he +ĠTh ailand +ĠT i +Ġinject ion +Ġinher it +rav is +Ġmed i +Ġwho ever +ĠDE BUG +G P +ĠH ud +C ard +p rom +Ġp or +Ġover head +L aw +Ġviol ate +Ġhe ated +Ġdescript ions +Ġachieve ments +ĠBe er +ĠQu ant +W as +Ġe ighth +ĠI v +Ġspecial ized +U PDATE +ĠD elta +P op +J ul +ĠAs k +oph y +Ġnews letters +ĠT ool +Ġg ard +ĠConf eder +ĠGM T +ĠAb bott +Ġimm unity +ĠV M +Is lam +Ġimpl icit +w d +Ġ19 44 +rav ity +omet ric +Ġsurv iving +ur ai +ĠPr ison +Ġr ust +ĠSk etch +Ġbe es +ĠThe ory +Ġmer it +T ex +ch at +Ġm im +Ġpast e +ĠK och +Ġignor ance +ĠSh oot +Ġbas ement +Un ited +ĠAd vis +he ight +Ġf oster +Ġdet ain +in formation +Ġne ural +' ; +Ġprov es +all ery +Ġinv itation +um bers +Ġc attle +Ġbicy cle +z i +Ġconsult ant +Ġap ology +ĠT iger +Ġ12 3 +99 9 +Ġind ividually +r t +ig ion +ĠBrazil ian +Ġdist urb +Ġentreprene urs +Ġfore sts +cer pt +pl ates +p her +clip se +Ġtw itter +Ġac ids +ograph ical +h um +ĠB ald +if ully +Ġcomp iler +ĠD A +Ġdon or +as i +Ġtrib al +l ash +ĠCon fig +Ġapplic ants +Ġsal aries +13 5 +Put in +ĠF ocus +ir s +Ġmisc onduct +ĠH az +Ġeat en +M obile +Mus lim +ĠMar cus +v iol +Ġfavor able +Ġst ub +ad in +ĠH ob +Ġfaith ful +Ġelectron ics +Ġvac uum +w ait +back ed +econom ic +d ist +Ġten ure +Ġsince re +ĠT ogether +ĠW ave +Ġprog ression +Ġden ying +Ġdist ress +br aska +th ird +Ġmix ing +Ġcolon ial +Ġpriv ately +Ġun rest +atern ity +Ġprem ises +ant i +greg ation +Ġlic ence +ĠH ind +ĠSam uel +Ġconvinc ing +ĠA ce +ĠR ust +ĠNet anyahu +Ġhand les +ĠP atch +orient ed +ah o +ĠG onz +Ġhack ers +claim er +Ġcustom s +ĠGr an +f ighters +Ġl uc +Ġman uscript +aren thood +Ġdev il +Ġwar riors +Ġoff enders +Will iam +Ġhol idays +Ġnight mare +Ġle ver +iff erent +St at +Ġexhib ition +put ed +ĠP ure +Ġal pha +Ġenthus iasm +ĠRepresent atives +E AR +ĠT yp +Ġwhe at +ĠAl f +Ġcor rection +Ġev angel +AT T +M iss +Ġs oup +Ġimpl ied +par am +Ġsex y +ĠL ux +Ġrep ublic +p atch +ab lish +Ġic ons +Ġfather s +ĠG ET +ĠCar ib +Ġregul ated +ĠCo hen +ĠBob by +Ġn er +Ġb ent +vent ory +ĠAl ong +ĠE ST +ĠWall ace +Ġmurd ers +r ise +ke ll +ĠCommon wealth +Ġn asty +et a +ĠM IT +Ġadminist ered +Ġgenuine ly +Ed itor +n ick +Ġhyd ro +**************** **************** +ĠB le +Ġfin es +Ġg orge +aus ible +r h +Ġapp le +ment ioned +Ġro pe +ot yp +H R +Ġdisappoint ing +Ġc age +n ik +Ġdoub ts +ĠF REE +print s +ĠM UST +Ġvend ors +ĠIn qu +Ġliber als +Ġcontract or +Ġup side +child ren +Ġtrick y +Ġregul ators +charg ed +l iter +Ġ *** +Ġreb ell +l ang +Ġloc als +Ġphys icians +Ġhe y +ar se +t m +ĠLe x +Ġbehavior al +success ful +F X +Ġbr ick +ov ic +Ġcon form +Ġreview ing +Ġins ights +Ġbi ology +ĠRem ove +ĠExt ra +Ġcomm itting +indu ced +ignt y +ig m +Ġat omic +Comm on +ĠE M +ĠP ere +ĠIt ems +e h +Ġpres erved +ĠH ood +Ġprison er +Ġbankrupt cy +Ġg ren +us hes +Ġexplo itation +Ġsign atures +Ġfin an +] ," +ĠM R +Ġme g +rem lin +Ġmusic ians +Ġselect ing +Ġexam ining +IN K +l ated +H i +Ġart ic +Ġp ets +Ġimp air +ĠM AN +Ġtable ts +in clude +R ange +Ġca ut +Ġlog s +Ġmount ing +Ġun aware +Ġdynam ics +ĠPalest ine +ĠQu arter +ĠPur ple +Ġm a +ĠIm port +Ġcollect ions +ci ation +Ġsuccess or +Ġcl one +Ġaim ing +Ġposs essed +Ġstick ing +Ġsh aking +Ġloc ate +ĠH ockey +T urn +17 0 +Ġfif teen +ĠHar rison +Ġcontinu ously +ĠT C +ĠVal ent +ĠRes cue +Ġby pass +am ount +Ġm ast +Ġprotect s +Ġart istic +Ġsomet ime +Ġsh oe +Ġshout ed +ific ant +et itive +ĠReg ister +ĠJ in +Ġconcent rated +ling ton +on ies +Ġgener ator +yr im +ĠAr men +Ġclear ing +id o +ĠT W +al ph +Ġlad ies +H ard +Ġdial og +Ġinput s +æ ľ +Ġpos es +Ġsl ots +ĠPrem ium +Ġle aks +Ġboss es +Ġ11 3 +c ourse +A cc +ĠNew ton +ĠAust ria +ĠM age +Ġte aches +ab ad +Ġwe ars +Ġc yl +Ġcur se +ĠS ales +ĠW ings +Ġp sy +Ġg aps +ĠIce land +ĠP interest +Ġland lord +Ġdefin itions +ĠK er +Ġsufficient ly +ĠP ence +ĠArch itect +Ġsur pass +Ġ11 4 +Ġsuper hero +ĠDise ase +Ġpri ests +ĠC ulture +Ġdefin itive +Ġsecret ly +ĠD ance +inst all +ch ief +ĠJess ica +W ould +Up dated +Ġlock er +ĠK ay +Ġmem orial +è ¦ +f at +Ġdis gu +Ġflav ors +ĠBase ball +ĠRes istance +Ġk icks +Ġen v +Ġteen agers +D ark +ĠC AR +Ġh alt +ĠL G +ĠGab riel +Ġfe ver +Ġs atur +Ġm all +Ġaffili ate +ĠS leep +ĠSpe cific +ĠV el +Ġj ar +ĠSac red +ĠEd wards +ĠA CL +Ġret ained +ĠG iant +Ġlim itation +in ces +Ġref usal +ĠT ale +ĠBut ler +Ġacc idents +ĠC SS +Ġimport ed +ĠCop y +Î ± +ER T +z el +Ġdiv isions +h ots +ĠAl b +ĠD S +Load er +W ashington +at isf +ĠCreat ive +\ . +ĠAut om +red ict +Ġrecept or +ĠCarl os +Met hod +ok a +Ġmal icious +Ġste pping +, [ +ĠD ad +Ġatt raction +ĠEffect s +ĠPir ate +ĠC er +ĠIndust ry +ĠR ud +Ġchar ter +Ġd ining +Ġins ists +Ġconfig ure +Ġ( # +ĠSim ple +ĠSc roll +UT C +17 5 +ĠK on +Ġmarket place +Ġ ãĤ +Ġref res +Ġg ates +er red +ĠP od +Ġbeh ave +Fr ank +n ode +Ġendors ed +he tt +as ive +ĠHom eland +Ġr ides +ĠLe ave +er ness +Ġflood ing +A FP +Ġris en +Ġcontin ually +Ġun anim +ĠCont ract +ĠP as +Ġgu ided +ĠCh ile +b d +Ġsu cc +pt ic +Ġcomm ittees +ĠL uther +ĠAny one +Ġs ab +12 4 +Ġp ixel +ĠB ak +ĠT ag +ĠBenn ett +En ter +sm all +ĠPresident ial +Ġp ul +Ġcontr ace +arch ive +Ġcoast al +ĠK ids +19 2 +âĢ ² +ick y +ING TON +Ġw olf +ĠSt alin +T ur +id get +am as +ĠUn less +Ġspons or +Ġmor ph +ĠCho ose +Ġrun ner +Ġun bel +Ġm ud +ĠMan a +Ġdub bed +Ġg odd +ure rs +wind ow +Ġrel ied +Ġcelebr ating +os c +Ġ13 5 +Ġlobb ying +Ġincom plete +Ġrestrict ion +Ġinc ap +it us +Ġexpect ation +ĠAp ollo +Ġint ens +Ġsyn c +G H +Ġmanip ulation +B Y +Ġspe ar +Ġbre asts +Ġvol can +il ia +M aterial +Ġform ats +ĠB ast +Ġparliament ary +Ġsn ake +Ġserv ants +ĠTr udeau +ĠGr im +ĠArab ic +ĠSC P +ĠBoy s +st ation +Ġprospect ive +ord e +in itialized +Ġb ored +AB LE +Ġaccess ed +Ġtax i +ĠShe ll +aid en +urs ed +in ates +ĠIns urance +ĠPet e +Sept ember +6 50 +Ġad ventures +ĠCo ver +Ġt ribute +Ġsk etch +Ġem power +Ġ Ø +ĠGl enn +ĠD aw += \" +ĠPolit ics +Ġgu ides +Ġd ioxide +ĠG ore +ĠBr ight +ĠS ierra +Ġval ued +c ond +Ġpo inter +Se lect +Ġrisk y +Ġabsor b +im ages +Ġref uses +Ġbon uses +__ _ +Ġh ilar +ĠF eatures +2 20 +ĠCollect or +F oot +Ġ19 64 +cul us +Ġd awn +Ġwork out +ĠL O +Ġphilosoph ical +ĠSand y +ĠYou th +Ġl iable +A f +bl ue +Ġovert urn +less ness +ĠTrib une +ĠIn g +Ġfact ories +Ġcat ches +Ġpr one +Ġmat rix +Ġlog in +Ġin acc +Ġex ert +s ys +Ġneed le +ĠQ ur +Ġnot ified +ould er +t x +Ġremind s +Ġpublisher s +Ġn ort +Ġg it +Ġfl ies +ĠEm ily +Ġflow ing +ĠAl ien +ĠStr ateg +Ġhard est +Ġmod ification +AP I +ĠM Y +Ġcr ashes +st airs +n umber +Ġur ging +ch annel +ĠFal con +Ġinhabit ants +Ġterr ifying +Ġutil ize +Ġban ner +Ġcig arettes +Ġsens es +ĠHol mes +Ġpract ition +ĠPhill ips +ott o +Ġcomp ile +Mod el +ĠK o +Ġ[ ] +Americ ans +ĠTer ms +Ġmed ications +ĠAn a +Ġfundament ally +ĠNot ice +Ġwe aker +Ġ 0000 +Ġgar lic +Ġout break +Ġeconom ist +ĠB irth +Ġobst acles +ar cer +ĠOr thodox +Ġplace bo +ĠC rew +asp berry +ĠAng els +Ġdis charge +Ġdestruct ive +11 7 +ĠR ising +Ġd airy +l ate +Ġcoll ision +ĠTig ers +ean or +ocument ed +ĠIn valid +Ġd ont +ĠL iter +ĠV a +Ġhyd rogen +Ġvari ants +ĠBrown s +Ġ19 65 +Ġind igenous +Ġtrad es +Ġremain der +Ġswe pt +ĠImp act +Ġred ist +Ġun int +grad uate +ãĥ ķ +ĠW ILL +ãģ® ç +ĠCrit ical +Ġf isher +Ġv icious +Ġrevers ed +Y ear +ĠS ox +Ġshoot ings +Ġfil ming +Ġtouchdown s +ai res +m el +Ġgrand father +Ġaffect ion +ing le +Ġover ly +Add itional +Ġsup reme +ĠGr ad +Ġsport ing +Ġmer cy +ĠBrook s +ount y +Ġperform s +Ġtight ly +Ġdem ons +Ġkill ings +Ġfact ion +ĠNov a +aut s +Ġund oubtedly +ar in +Ġunder way +ra k +Ġl iv +ĠReg ion +Ġbrief ing +s ers +cl oud +ĠM ik +us p +Ġpred iction +az or +Ġport able +ĠG and +Ġpresent ing +Ġ10 80 + » +ush i +ĠSp ark +there um +Ġjust ification +ĠN y +Ġcontract ors +ming ham +ĠSt yle +å ħ +ĠChron icles +ĠPict ure +Ġprov ing +Ġw ives +set t +Ġmole cules +ĠFair y +Ġconsist ing +Ġp ier +al one +in ition +Ġn ucle +j son +Ġg otta +Ġmob il +Ġver bal +ar ium +Ġmon ument +uck ed +Ġ25 6 +T ech +mine craft +ĠTr ack +Ġt ile +Ġcompat ibility +as is +Ġs add +Ġinstruct ed +ĠM ueller +Ġle thal +Ġhorm one +Ġor che +el se +Ġske let +Ġentert aining +Ġminim ize +ag ain +Ġunder go +Ġconst raints +Ġcig arette +ĠIslam ist +Ġtravel s +ĠPant hers +l ings +C are +Ġlaw suits +ur as +Ġcry st +Ġlow ered +Ġaer ial +Ġcomb inations +Ġha un +Ġch a +Ġv ine +Ġquant ities +Ġlink ing +b ank +Ġso y +B ill +ĠAngel a +Ġrecip ient +ĠProt est +Ġs ocket +Ġsolid arity +Ġâ Ĩ +m ill +Ġvar ies +ĠPak istani +Dr agon +Ġun e +Ġhor izon +³³³³ ³³³³ +Ġprov inces +Ġfrank ly +Ġenact ed +not es +[ ' +Ġ19 2 +ocr acy +Ġendorse ment +Ġover time +Tr ue +L ab +lic ted +ĠD NC +Ġbe ats +ĠJam ie +15 2 +ĠIN T +Cont act +Ġaccount ed +h ash +ĠPack ers +p ires +Ġles bian +Ġamend ments +Ġhop eful +ĠFin land +Ġspot light +Ġconfig ured +Ġtrou bled +Ġg aze +ĠCal gary +Ġrel iability +Ġins urg +sw er +b uy +ĠSk in +Ġp ixels +Ġhand gun +Ġpar as +Ġcateg or +ĠE L +ĠRe x +Ind eed +Ġkind a +Ġconj unction +ĠBry an +ĠMan ufact +y ang +Pl us +S QL +ish ment +Ġdom inate +Ġn ail +Ġo ath +Ġeru pt +ĠF ine +it bart +ĠCh ip +ĠAb d +ĠN am +Ġbuy er +Ġdiss ent +Le aks +Cont in +Ġr ider +ĠSome one +Ġill usion +c in +ĠBoe ing +Ġin adequ +ov ation +i ants +Ġreb uild +4 50 +ĠDest iny +S W +ĠT ill +H it +ia z +ĠBang l +acher s +ĠRe form +Ġse gments +Ġsystem atic +d c +ĠConserv atives +Ġport al +h or +ĠDragon bound +Ġdrag ged +om o +Ġthe e +ad vert +ĠRep orts +ĠE t +Ġbarrel s +Aug ust +Ġcompar isons +Ġhe x +Ġan throp +" [ +bor ough +ab i +Ġpict ured +play ing +ĠAdd ress +ĠMir ror +Sm ith +Ġt ires +ĠN PR +AA AA +Ġclass ification +ĠTh an +ĠH arm +ĠR A +Ġreject ion +min ation +Ġr anged +ĠF alls +D I +H ost +ãĤ ´ +ĠEx ample +list ed +th irds +Ġsaf egu +br and +Ġprob able +Can ada +IT ION +ĠQ aeda +Ġch ick +Ġimport s +h it +l oc +W W +Ġble w +Ġany time +Ġwh oles +ik ed +Ġcal culation +cre ate +ĠO ri +Ġupgr aded +Ġapp ar +ut ory +ĠM ol +B rit +ĠJ ong +IN AL +ĠStart ing +Ġd ice +urt le +Ġre lying +cl osure +Ġprof itable +Ġsl aughter +ĠMan ual +c aster +Ġ" $ +Ġfe ather +ĠSim ply +ie ves +Ġdeter ior +ĠPC I +Ġst amp +Ġfl aws +Ġsh ade +ham mer +Ġpass port +Ġcont ing +am el +Ġobser vers +Ġneg lect +ĠR B +ĠBrother hood +Ġskept ical +f amily +us k +Ġemotion ally +â Ļ +ĠBet a +ason able +id ity +ĠM ul +Ġkick ing +ĠC arm +oll ah +VERT IS +ĠAt hen +Ġlad der +ĠBul let +å £ +00 01 +ĠWild life +ĠM ask +ĠN an +R ev +Ġun acceptable +leg al +Ġcrowd ed +ag i +ĠC ox +j e +Ġmor ality +Ġfu els +Ġc ables +Ġman kind +ĠCarib bean +Ġanch or +Ġby te +ĠO ften +ĠO z +Ġcraft ed +Ġhistor ian +ĠW u +Ġtow ers +ĠCitiz ens +Ġhel m +Ġcred entials +Ġsing ular +ĠJes se +Ġtack les +Ġcont empt +Ġa fore +ĠSh adows +Ġn il +Ġur gent +app le +bl ood +Ġv on +Ġoff line +Ġbreat he +Ġj umps +Ġirre levant +ox ic +om al +import ant +J im +Ġgl oves +arm ing +dep th +Ġtal ents +ook ie +ĠS B +Ġpal m +uff s +est a +IG H +Ġcan on +ĠVer izon +ĠP le +Ġcou pled +vel t +Ġfundra ising +ĠGet ting +ĠD LC +Ġmathemat ical +ĠH S +ĠCard inals +te lling +Ġspons ors +Ġ Ï +ĠBull s +op tion +Ġprop ose +Ġmem orable +Ġembr aced +Ġdecl ining +He alth +ed a +Ġ} ; +Ġsp am +m ile +Ġpit cher +ĠE ight +Ġcar ing +ut ic +ro le +Ġair line +ernand ez +ĠAth let +Ġcert ification +ux e +rig er +Ġem pir +Ġsens ation +Ġdis m +Ġb olt +Ġev olve +H ouse +Ġconsult ation +ĠD uty +Ġtou ches +ĠN athan +Ġf aint +h ad +" ( +ĠCons umer +ĠExt reme +Ġ12 7 +ĠHer m +ĠSac rament +iz oph +Ġanx ious +ul ously +Ġsoc ially +ĠU TC +Ġsol ving +ĠLet ter +Hist ory +ed uc +Pr ice +) ); +Ġrel oad +am ic +Ġp ork +Ġdisc ourse +Ġt ournaments +ai ro +ĠK ur +ĠCost a +Ġviol ating +Ġinterf ere +Ġrecre ational +uff le +Ġspe eches +Ġneed ing +Ġremem bers +Ġcred ited +n ia +f ocused +amer a +Ġb ru +um bs +ĠCub an +Ġpreced ing +Ġnons ense +ac ial +Ġsmart phones +ĠSt ories +S ports +ĠEmer gency +oun cing +ef ined +Ġb er +Ġconsult ing +Ġm asters +he astern +." [ +ĠRun ning +Ġsus cept +ĠF eng +Americ a +pr ises +st itial +ĠWeek ly +ĠGreat er +mod ules +if ter +G raphics +ul er +Ġwho lly +Ġsupp ress +Ġconce aled +Ġhapp ily +Ġaccept s +ĠEn joy +Ġr ivers +ĠEx cept +2 25 +ĠN HS +ĠMc Connell +Ġp ussy +fer red +ut able +Ġatt ain +Ġ> = +Ġdepos its +roph ic +Ġnot orious +ĠSh aw +il itation +Ġepid emic +all ic +Ġsmall est +ov ich +Ġaccess ories +per ties +Ġsur plus +ĠMe ch +Ġamb ig +ĠImm igration +Ġch im +ev al +Ġpract icing +ĠMyster y +Ġdom ains +ĠSil icon +app s +Ġkilomet ers +e a +ĠSm ash +Ġwarrant y +Ġn ost +s il +re v +J on +ĠDub lin +Ġtast es +Ġb out +g reat +er ror +Ġsw itches +ĠB apt +D O +ok i +Ġsour ced +pro du +Ġattach ment +ĠIss ue +ĠQuest ion +Jo in +Ġf itted +Ġunlaw ful +^ ^ +ere k +Ġauthent ication +Ġst ole +Ġaccount ability +l abel +S earch +Ġal beit +atic an +fund ed +ĠAdd ing +ĠI Q +Ġsub mar +l it +a que +ĠLear ning +Ġint eger +M aster +ĠCh rom +Ġprem ier +O p +ĠLi u +Ġbl essed +ĠGl obe +ĠResp onse +Ġlegit im +ĠMer kel +Ġdispos al + ´ +Ġgau ge +pe at +Ġindu ced +Ġquestion able +arth y +ĠV it +ĠF eed +U ntil +U t +worth y +R Y +ĠH erald +ĠHam mer +Ġmed al +ĠR ivers +ĠH ack +Ġclar ify +Ġtrack ed +Ġautonom ous +Ġten ant +ĠQ atar +er ie +Ġgr im +ĠMon itor +Ġresist ant +ĠSpe c +ĠWell s +N AS +14 8 +Ġmin ers +iot ics +Ġmiss es +11 6 +g ian +g it +ĠE yes +p res +Ġgrad uated +Ġang el +Ġsyn chron +Ġefficient ly +Ġtrans mitted +H arry +Ġglob ally +EN CE +ĠMont ana +r aged +ĠPre vention +Ġp iss +ĠL l +Ġshe lf +ĠB JP +ĠTest ament +ĠL ate +ik er +ĠH app +ĠJul ian +h all +Ġsp ont +Ġshut down +Ġincons istent +Ġsubscrib ers +Ġske leton +ĠNe braska +Ġins pire +ĠV oid +F eed +Ġang les +ĠSpr ings +Ġbench mark +Ġvacc ines +izoph ren +se xual +uff ed +Ġsh ine +ĠK ath +Ġgest ure +ine a +Ġr ip +Ġopp ression +Ġcons cience +b t +ĠL um +Ġinc idence +ĠF a +w r +Ġmin eral +ĠSp urs +alk y +Ġth under +Ġop io +Be ing +ĠPal m +Ġwas ted +Ġl b +i aries +ĠIniti ative +Ġcur ric +Ġmark er +ĠMc L +Ġext ensions +ĠP v +ĠAr ms +Ġoffer ings +Ġdef enses +Ġvend or +Ġcontrad ict +ĠCol in +Ġredd it +Ġper ipher +12 2 +Ġs ins +E dit +IC T +So ft +ĠSh ah +Ġadministr ator +ĠT rip +Ġporn ography +Ġtu ition +in ence +ĠPro gress +Ġcat alog +Ġsu ite +Ġh ike +Ġreprodu ctive +eng ine +Ġd rought +ĠNo ah +Ġ2 30 +Ġd ude +Ġrelax ed +Ġpart ition +Ġparticip ant +Ġtel esc +Ġfe as +ĠF F +own er +Ġswe eping +Ġl enses +Ġmatch up +ĠRe pl +ourn als +Ġcred ible +Ġgrand mother +Ġther mal +Ġsubscrib ing +Ġident ities +col m +U CT +Ġreluct ant +us ers +ĠC ort +Ġassist ed +OS S +ATION S +IS H +Ġpharm aceutical +ic able +ad ian +ĠSon ic +ĠF ury +ĠM ong +A H +ĠPsych ology +Ġph osph +Ġtreat s +Ń Ķ +Ġstead ily +ĠHell o +Ġrel ates +Ġcl ue +Ex pl +a uth +Ġrev ision +Ġe ld +os ion +Ġbr on +14 4 +ri kes +Ġmin es +Ġblank et +ĠF ail +el ed +ĠIm agine +ĠPl anned +a ic +Re quest +M ad +ĠHor se +ĠEag le +Ġcap ac +15 7 +Ġl ing +ĠN ice +ĠP arenthood +min ster +og s +ens itive +Not hing +Ġcar n +F in +ĠP E +Ġr ifles +ĠL P +S and +Ġgui Active +Ġtour ist +C NN +Ġunve iled +Ġpredec essor +} { +u ber +Ġoff shore +Ġopt ical +ĠR ot +ĠPear l +et on +Ġst ared +Ġfart her +at ility +cont in +ĠG y +ĠF oster +ĠC oc +ri ents +Ġdesign ing +ĠEconom y +ON G +W omen +ĠN ancy +er ver +Ġmas cul +Ġcasual ties +Ġ2 25 +ĠS ullivan +ĠCh oice +Ġa ster +w s +Ġhot els +Ġconsider ations +Ġcou ch +ĠSt rip +ĠG n +Ġmanip ulate +l ied +Ġsynt hetic +Ġassault ed +Ġoff enses +ĠDra ke +Ġim pe +Oct ober +ĠHer itage +h l +ĠBl air +Un like +Ġg rief +Ġ4 50 +Ġopt ed +Ġresign ation +il o +Ġver se +ĠT omb +Ġu pt +Ġa ired +ĠH ook +ĠML B +Ġassum es +out ed +ĠV ers +Ġinfer ior +Ġbund le +ĠD NS +ograp her +Ġmult ip +ĠSoul s +Ġillust rated +Ġtact ic +Ġdress ing +Ġdu o +Con f +Ġrel ent +Ġc ant +Ġscar ce +Ġcand y +ĠC F +Ġaffili ated +Ġspr int +yl an +ĠGarc ia +Ġj unk +Pr int +ex ec +C rit +Ġport rait +ir ies +ĠOF F +Ġdisp utes +W R +L ove +ãģ Ħ +ĠRe yn +Ġh ipp +op ath +Ġflo ors +ĠFe el +Ġwor ries +Ġsett lements +ĠP os +Ġmos que +Ġfin als +Ġcr ushed +ĠPro bably +ĠB ot +ĠM ans +ĠPer iod +Ġsovere ignty +Ġsell er +Ġap ost +Ġam ateur +Ġd orm +Ġconsum ing +Ġarm our +ĠRo ose +Ġint ensive +Ġelim inating +ĠSun ni +ĠAle ppo +j in +Ġadv ise +p al +ĠH alo +Ġdes cent +Ġsimpl er +Ġbo oth +ST R +L ater +ĠC ave +== = +Ġm ol +Ġf ist +Ġshot gun +su pp +Ġrob bery +E ffect +Ġobsc ure +ĠProf essional +Ġemb assy +Ġmilit ant +Ġinc arcer +Ġgener ates +Ġlaun ches +Ġadministr ators +Ġsh aft +Ġcirc ular +Ġfresh man +ĠW es +ĠJo el +ĠD rew +ĠDun can +ĠApp arently +s ight +ĠIntern al +ĠInd ividual +ĠF E +Ġb ore +ĠM t +Ġbroad ly +ĠO ptions +ount ain +ip es +ĠV ideos +20 4 +Ġh ills +Ġsim ulation +Ġdisappoint ment +it an +ĠLabor atory +Ġup ward +Ġbound ary +Ġdark er +h art +Ġdomin ance +C ong +ĠOr acle +ĠL ords +Ġscholars hip +ĠVin cent +ed e +ĠR ah +Ġencour ages +ro v +Ġqu o +Ġprem ise +ĠCris is +ĠHol ocaust +Ġrhyth m +Ġmet ric +cl ub +Ġtransport ed +Ġn od +ĠP ist +Ġancest ors +ĠFred er +th umbnails +ĠC E +ON D +Ph il +ven ge +ĠProduct s +cast le +Ġqual ifying +ĠK aren +VERTIS EMENT +Ġmight y +Ġexplan ations +Ġfix ing +D i +Ġdecl aring +Ġanonym ity +Ġju ven +ĠN ord +ĠDo om +ĠAct ually +O k +ph is +ĠDes ert +Ġ11 6 +I K +ĠF M +Ġinc omes +V EL +ok ers +Ġpe cul +Ġlight weight +g ue +Ġacc ent +Ġincre ment +ĠCh an +Ġcompl aining +ĠB aghd +Ġmidfield er +Ġover haul +Pro cess +ĠH ollow +ĠTit ans +Sm all +man uel +ĠUn ity +ĠEv ents +S ty +Ġdispro portion +n esty +en es +ĠC od +Ġdemonstr ations +ĠCrim son +ĠO H +Ġen rolled +Ġc el +ĠBre tt +Ġa ide +Ġhe els +Ġbroad band +Ġmark ing +Ġw izard +ĠN J +ĠChief s +Ġingred ient +Ġd ug +ĠSh ut +urch ase +end or +Ġfar mer +ĠGold man +12 9 +15 5 +Or der +Ġl ion +i ably +Ġst ain +ar ray +ilit ary +ĠFA Q +Ġexpl oded +ĠMcC arthy +ĠT weet +ĠG reens +ek ing +l n +ens en +Ġmotor cycle +Ġpartic le +Ġch olesterol +B ron +Ġst air +Ġox id +Ġdes irable +ib les +Ġthe or +for cing +Ġpromot ional +ov o +b oot +ĠBon us +raw ling +Ġshort age +ĠP sy +Ġrecru ited +Ġinf ants +Ġtest osterone +Ġded uct +Ġdistinct ive +Ġfirm ware +bu ilt +14 5 +Ġexpl ored +Ġfact ions +Ġv ide +Ġtatt oo +Ġfinan cially +Ġfat igue +Ġproceed ing +const itutional +Ġmis er +Ġch airs +gg ing +ipp le +Ġd ent +Ġdis reg +ç Ķ +st ant +ll o +b ps +aken ing +Ġab normal +ĠE RA +å£ « +ĠH BO +ĠM AR +Ġcon cess +Ġserv ant +Ġas pir +l av +ĠPan el +am o +Ġprec ip +Ġrecord ings +Ġproceed ed +Ġcol ony +ĠT ang +ab lo +Ġstri pped +Le ft +to o +Ġpot atoes +Ġfin est +% ). +Ġc rap +ĠZ ach +ab ases +ĠG oth +Ġbillion aire +w olf +Ġsan ction +S K +Ġlog ged +P o +ey ed +un al +Ġcr icket +Ġarm ies +Ġunc overed +Cl oud +ó n +Ġreb ounds +Ġm es +O per +P ac +Ġnation ally +Ġinsert ed +p ict +Ġgovern ance +Ð ¸ +Ġprivile ges +G ET +Ġfavor ites +im ity +Ġlo ver +the m +em pl +Ġgorge ous +An n +Ġsl ipped +Ġve to +B ob +Ġsl im +u cc +ĠF ame +udden ly +Ġden ies +ĠM aur +Ġdist ances +Ġw anna +t ar +ĠS ER +Ġâ Ī +Ġle mon +at hetic +Ġlit eral +Ġdistingu ished +Ġansw ering +G I +Ġrelig ions +ĠPhil os +ĠL ay +Ġcomp os +ire ments +ĠK os +ine z +roll ing +Ġyoung est +and ise +ĠB orn +Ġalt ar +am ina +ĠB oot +v oc +Ġdig ging +Ġpress ures +Ġl en +26 4 +Ġassass ination +ĠBir mingham +ĠMy th +Ġsovere ign +ĠArt ist +ĠPhot ograph +Ġdep icted +Ġdisp ens +orth y +Ġamb ul +int eg +ĠC ele +ĠTib et +Ġhier archy +Ġc u +Ġpre season +ĠPet erson +Ġcol ours +Ġworry ing +Ġback ers +ĠPal mer +ĠÎ ¼ +Ġcontribut or +Ġhear ings +Ġur ine +Ġ Ù +ourge ois +Sim ilar +ĠZ immer +s omething +ĠUS C +Ġstrength s +ĠF I +Ġlog ging +As ked +ĠTh ai +in qu +ĠW alt +Ġcrew s +it ism +3 01 +Ġshar ply +um ed +Ġred irect +r ators +In f +ĠWe apons +Ġte asp +19 99 +L ive +ĠEs pecially +ĠS ter +ĠVeter ans +Ġint ro +other apy +Ġmal ware +Ġbre eding +Ġmole cular +ĠR oute +ĠCom ment +oc hem +Ġa in +Se ason +Ġlineback er +Ä « +ĠEconom ics +es ar +ĠL ives +ĠEm ma +Ġk in +ĠTer rit +Ġpl anted +ot on +ĠBut ter +ĠSp ons +P ER +Ġdun geon +Ġsymb olic +Ġfil med +Ġdi ets +Ġconclud es +Ġcertain ty +ĠForm at +Ġstr angers +form at +ĠPh ase +Ġcop ied +Ġmet res +ld a +ĠUs ers +Ġdeliber ate +Ġwas hed +ĠL ance +im ation +Ġimpro per +ĠGen esis +ick r +ĠK ush +Ġreal ise +Ġembarrass ing +alk ing +b ucks +Ġver ified +Ġout line +year s +ĠIn come +20 2 +Ġz ombies +F inal +ĠMill enn +Ġmod ifications +ĠV ision +ĠM oses +ver b +iter ranean +ĠJ et +Ġnav al +ĠA gg +Ġur l +Ġvict ories +Ġnon etheless +Ġinj ust +ĠF act +ç ļ +Ġins ufficient +re view +face book +Ġnegoti ating +Ġguarant ees +im en +uten berg +Ġg ambling +Ġcon gr +Load ing +Ġnever theless +Ġpres idents +ĠIndust rial +Ġ11 8 +Ġp oured +ĠT ory +Ġ17 5 +Ġ: = +Sc ott +ange red +T ok +Ġorgan izers +M at +ĠG rowth +Ġad ul +Ġens ures +Ġ11 7 +é¾į å +Ġmass acre +Ġgr ades +be fore +AD VERTISEMENT +ĠSl ow +ĠM MA +âĢĶ " +ĠV atican +Q aeda +Ġo we +66 66 +ĠS orry +ĠGr ass +Ġbackground s +Ġexha usted +Ġcl an +Ġcomprom ised +ĠE lf +ĠIsa ac +ens on +In vest +IF A +Ġinterrupt ed +ãĥī ãĥ© +Ġtw isted +ĠDrag ons +M ode +ĠK remlin +Ġfert il +he res +ph an +ĠN ode +f ed +ĠOr c +Ġunw illing +C ent +Ġprior it +Ġgrad uates +Ġsubject ive +Ġiss uing +ĠL t +Ġview er +Ġw oke +Th us +bro ok +Ġdep ressed +Ġbr acket +ĠG or +ĠFight ing +Ġstri ker +Rep ort +ĠPortug al +Ġne o +w ed +19 9 +Ġflee ing +sh adow +ident ified +US E +Ste am +Ġstret ched +Ġrevel ations +art ed +ĠD w +Ġalign ment +est on +ĠJ ared +S ep +Ġblog s +up date +g om +r isk +Ġcl ash +ĠH our +Ġrun time +Ġunw anted +Ġsc am +Ġr ack +Ġen light +on est +ĠF err +Ġconv ictions +Ġp iano +Ġcirc ulation +ĠW elcome +Ġback lash +ĠW ade +Ġrece ivers +ot ive +J eff +Ġnetwork ing +ĠPre p +ĠExpl orer +Ġlect ure +Ġupload ed +ĠMe at +B LE +ĠNaz is +ĠSy nd +st ud +ro ots +ri ans +Ġportray ed +Ġ ?? +ĠBudd ha +s un +Rober t +ĠCom plex +Ġover see +Ġste alth +T itle +ĠJ obs +ĠK um +Ġappreci ation +ĠM OD +Ġbas ics +Ġcl ips +Ġnurs ing +Ġpropos ition +Ġreal ised +ĠNY C +Ġall ocated +ri um +ar an +ĠPro duction +ĠV ote +Ġsm ugg +Ġhun ter +az er +ĠCh anges +Ġfl uct +y on +Ar ray +Ġk its +W ater +Ġuncom mon +Ġrest ing +ell s +w ould +Ġpurs ued +Ġassert ion +omet own +ĠMos ul +ĠPl atform +io let +Ġshare holders +Ġtra ils +P ay +ĠEn forcement +ty pes +ĠAn onymous +Ġsatisf ying +il ogy +Ġ( ' +w ave +c ity +Ste ve +Ġconfront ation +ĠE ld +C apt +ah an +ht m +ĠC trl +ON S +2 30 +if a +hold ing +Ġdelic ate +Ġj aw +ĠGo ing +or um +S al +Ġd ull +ĠB eth +Ġpr isons +Ġe go +ĠEl sa +avor ite +ĠG ang +ĠN uclear +Ġsp ider +ats u +Ġsam pling +Ġabsor bed +ĠPh arm +iet h +Ġbuck et +ĠRec omm +O F +ĠF actory +AN CE +Ġb acter +H as +ĠObs erv +12 1 +Ġprem iere +De velop +Ġcur rencies +C ast +Ġaccompany ing +ĠNash ville +Ġfat ty +ĠBre nd +Ġloc ks +Ġcent ered +ĠU T +augh s +or ie +ĠAff ordable +v ance +D L +em et +Ġthr one +ĠBlu etooth +Ġn aming +if ts +AD E +Ġcorrect ed +Ġprompt ly +ĠST R +Ġgen ome +Ġcop e +Ġval ley +Ġround ed +ĠK end +al ion +p ers +Ġtour ism +Ġst ark +v l +Ġblow ing +ĠSche dule +st d +Ġunh appy +Ġlit igation +ced es +Ġand roid +Ġinteg ral +ere rs +ud ed +t ax +Ġre iter +ĠMot ors +oci ated +Ġwond ers +ĠAp ost +uck ing +ĠRoose velt +f ram +Ġyield s +Ġconstit utes +aw k +Int erest +Ġinter im +Ġbreak through +ĠC her +Ġpro sec +ĠD j +ĠM T +Res p +ĠP T +Ġs perm +ed it +B T +Lin ux +count ry +le ague +Ġd ick +Ġo ct +Ġinsert ing +Ġsc ra +ĠBrew ing +Ġ19 66 +Ġrun ners +Ġpl un +id y +ĠD ian +Ġdys function +Ġex clusion +Ġdis gr +Ġincorpor ate +Ġrecon c +Ġnom inated +ĠAr cher +d raw +achel or +Ġwrit ings +Ġshall ow +Ġh ast +ĠB MW +ĠR S +Ġth igh +Ġ19 63 +Ġl amb +Ġfav ored +ag le +Ġcool er +ĠH ours +ĠG U +ĠOrig in +Ġglim pse +---------------- ---- +L im +Ġche ek +Ġj ealous +- ' +Ġhar ness +ĠPo ison +Ġdis abilities +ne apolis +Ġout look +Ġnot ify +ĠIndian apolis +Ġab rupt +ns ic +Ġenc rypted +Ġfor fe +reat h +Ġr abb +Ġfound ations +Ġcompl iment +ĠInter view +ĠS we +Ġad olesc +Ġmon itors +ĠSacrament o +Ġtime ly +Ġcontem pl +Ġposition ed +Ġpost ers +ph ies +iov ascular +v oid +ĠFif th +Ġinvestig ative +OU N +Ġinteg rate +ĠIN C +ish a +ibl ings +ĠRe quest +ĠRodrig uez +Ġsl ides +ĠD X +Ġfemin ism +Ġdat as +Ġb end +ir us +ĠNig eria +F ox +Ch ange +Ġair plane +ĠLad en +Ġpublic ity +ixt y +Ġcommit ments +Ġaggreg ate +Ġdisplay ing +ĠAr row +Ġ12 2 +Ġrespect s +and roid +s ix +ĠSh a +Ġrest oration +) \ +W S +oy s +Ġillust rate +with out +12 6 +ĠâĶ Ĥ +Ġpick up +n els +Ġ .... +f ood +ĠF en +) ? +Ġphenomen a +Ġcompan ions +ĠW rite +Ġsp ill +Ġbr idges +ĠUp dated +ĠF o +Ġinsect s +ASH INGTON +Ġsc are +il tr +ĠZh ang +Ġsever ity +Ġind ul +14 9 +ĠCo ffee +Ġnorm s +Ġp ulse +ĠF T +Ġhorr ific +ĠDest roy +ĠJ SON +Ġo live +Ġdiscuss es +R est +E lect +ĠW inn +ĠSurv iv +ĠH ait +S ure +op ed +Ġro oted +ĠS ke +ĠBron ze +Ġl ol +Def ault +Ġcommod ity +red ited +Ġliber tarian +Ġforb idden +Ġgr an +à ¨ +Ġl ag +en z +dri ve +Ġmathemat ics +Ġw ires +Ġcrit ically +Ġcarb ohyd +ĠChance llor +ĠEd die +Ġban ning +ĠF ri +Ġcompl ications +et ric +ĠBangl adesh +Ġband width +St op +ĠOrig inally +Ġhalf way +yn asty +sh ine +Ġt ales +rit ies +av ier +Ġspin ning +ĠWH O +Ġneighbour hood +b ach +Ġcommer ce +ĠS le +B U +Ġentreprene ur +Ġpecul iar +ĠCom ments +f re +3 20 +IC S +Ġimag ery +ĠCan on +ĠElect ronic +sh ort +( ( +D ig +Ġcomm em +u ced +Ġincl ined +ĠSum mon +Ġcl iff +ĠMed iterranean +Ġpo etry +Ġprosper ity +ĠRe ce +Ġp ills +m ember +Ġfin ale +un c +ĠG ig +ä ½ +Ġl od +Ġback ward +- + +ĠFor ward +Ġth ri +s ure +Ġso ap +ĠF X +R ES +ĠSe xual +oul os +Ġfool ish +Ġright eous +Ġco ff +terror ism +ust ain +ot er +Ġab uses +ne xt +Ġab usive +Ġthere after +Ġprohib ition +ĠS UP +Ġd ip +Ġr ipped +Ġinher ited +Ġb ats +st ru +G T +Ġflaw ed +ph abet +Ġf og +do ors +Ġim aging +Ġdig its +ĠHung ary +Ġar rog +Ġteach ings +Ġprotocol s +ĠB anks +à ¸ +p ound +ĠC urt +." ) +. / +Ġex emption +end ix +ĠM ull +Ġimpro ves +ĠG amer +d imensional +I con +ĠMarg aret +St atus +d ates +Ġint ends +Ġdep ict +Ġpark ed +J oe +ĠMar ines +chn ology +! ). +Ġjud ged +Ġwe ights +R ay +Ġapart ments +he ster +Ġrein force +Ġoff ender +occ up +Ġs ore +e pt +ĠPH P +ĠB row +Ġauthor ization +ĠR isk +ĠDel aware +ĠQ U +Ġnot ifications +Ġsun light +Ġex clude +d at +Ġm esh +ĠSud an +Ġbelong ed +Ġsub way +Ġno on +ĠInter ior +ol ics +ĠL akers +Ġc oding +Dis claimer +Cal if +O ld +Ġdis l +???? ? +Ġconfir ms +Ġrecruit ment +Ġhom icide +Cons ider +ĠJeff rey +ft y +} ; +Ġobject ion +do ing +ĠLe o +W ant +Ġgl ow +ĠClar ke +ĠNorm an +Ġver ification +Ġpack et +ĠForm ula +Ġpl ag +es ville +Ġshout ing +Ġo v +ĠR EC +ĠB ub +Ġn inth +Ġener g +Ġvalid ity +Ġup s +j ack +Ġneighbor ing +ĠN ec +ew orks +ĠH ab +are z +Ġsp ine +Ġevent ual +ĠLe aders +ĠC arn +Ġprob ation +Ġrom ance +ms g +ĠMechan ical +ER Y +R ock +Ġpart isan +N ode +ass ets +min ent +Ġforeign ers +Ġtest ify +ĠUs ually +l ords +ĠG ren +ĠPow ell +BI L +Ġs r +Ġadd ict +Ġshell s +Ġs igh +ĠY ale +tern ity +Ġ7 50 +E U +ĠR ifle +Ġpat ron +em a +ĠB annon +an ity +Ġtrop ical +ĠV II +c ross +Every thing +ĠIS O +Ġhum ble +ass ing +ĠF IG +Ġupd ating +ys on +Ġcal cium +Ġcompet ent +Ġste ering +Pro t +ĠS Y +ĠFin als +ĠR ug +15 9 +13 7 +ĠG olf +Ġ12 6 +Ġaccommod ation +ĠHug hes +Ġaest hetic +art isan +ĠTw ilight +Ġpr ince +ĠAgric ulture +ĠDis co +Ġpreced ent +Ġtyp ing +author ized +O ption +ĠA ub +l ishes +ach t +m ag +P eter +ĠU FO +mont on +ĠL ith +Ġa rom +Ġsec uring +Ġconf ined +priv ate +Ġsw ords +Ġmark ers +Ġmetab olic +se lect +ĠCur se +ĠO t +g ressive +Ġinc umb +ĠS aga +Ġpr iced +Ġclear ance +Cont ent +Ġdr illing +Ġnot ices +Ġb ourgeois +Ġv est +Ġcook ie +ĠGuard ians +ry s +in yl +Ġ12 4 +Ġpl ausible +on gh +ĠOd in +Ġconcept ion +ĠY uk +ĠBaghd ad +ĠFl ag +Aust ral +ĠI BM +Ġintern ationally +ĠWiki Leaks +I ED +Ġc yn +Ġcho oses +ĠP ill +Ġcomb ining +Ġrad i +ĠMoh ammed +def ense +atch ing +Sub ject +ic iency +Fr ame +Ġ{ " +Ġche ss +Ġtim er +19 0 +Ġt in +Ġord inance +emet ery +Ġacc using +Ġnotice able +Ġcent res +Ġl id +ĠM ills +img ur +Ġz oom +erg ic +Ġcomp ression +pr im +f ind +Ġsur g +Ġp and +ĠK ee +ĠCh ad +cell ence +oy le +Ġsocial ism +ĠT ravis +ĠM Hz +Ġgu ild +ALL Y +ĠSub scribe +ĠRel ated +Ġoccur rence +itch ing +Ġfict ional +Ġcr ush +ĠE A +c od +m ix +ĠTri ple +Ġretrie ve +Ġstimul us +Ġpsych iat +ĠDo or +Ġhomosexual ity +Ġelement ary +Ġcell ular +id ian +ĠL aun +Ġintrig uing +Ġfo am +ĠB ass +id i +its u +Ġass ure +Ġcongr at +Ġbusiness man +ĠBo ost +cl ose +Ġl ied +Ġsc iences +ĠO mega +ĠG raphics +Ġ< = +sp oken +Ġconnect ivity +S aturday +ĠAven gers +Ġto ggle +Ġank le +Ġnational ist +mod el +ĠP ool +ophob ia +V ar +ĠM ons +ator ies +Ġaggress ively +C lear +For ge +act ers +Ġhed ge +Ġpip es +Ġbl unt +Ġs q +Ġremote ly +W ed +as ers +Ġref riger +Ġt iles +Ġresc ued +Ġcompr ised +ins ky +Ġman if +avan augh +Ġprol ifer +Ġal igned +x ml +Ġtri v +Ġcoord ination +ĠP ER +ĠQu ote +13 4 +b f +ĠS aw +Ġtermin ation +Ġ19 0 +Ġadd itions +Ġtri o +Ġproject ions +Ġpositive ly +Ġin clusive +Ġmem br +19 90 +old er +Ġpract iced +ink le +Ar ch +Ġstar ters +ari us +Ġinter mediate +ĠBen ef +ĠK iller +Ġinter ventions +ĠK il +ĠF lying +In v +Ġprem ature +Ġpsych iatric +Ġind ie +Ġcoll ar +ĠRain bow +af i +Ġdis ruption +ĠFO X +cast ing +Ġmis dem +c ro +Ġw ipe +ard on +Ġb ast +ĠTom my +ĠRepresent ative +Ġbell y +ĠP O +ĠBre itbart +13 2 +Ġmess aging +Sh ould +Ref erences +ĠG RE +ist ical +L P +ĠC av +ĠC razy +Ġintu itive +ke eping +ĠM oss +Ġdiscont in +ĠMod ule +Ġun related +ĠPract ice +ĠTrans port +Ġstatist ically +orn s +Ġs ized +p u +Ġca f +ĠWorld s +ĠRod gers +ĠL un +ĠCom ic +l iving +Ġc ared +Ġclim bed +) { +Ġconsist ed +Ġmed ieval +fol k +Ġh acked +Ġd ire +ĠHerm ione +Ġt ended +ce ans +D aniel +w ent +Ġlegisl ators +Ġred es +g ames +Ġg n +am iliar +Ġ+ + +gg y +th reat +Ġmag net +Ġper ceive +Ġz ip +Ġindict ment +Ġcrit ique +g ard +ĠSaf e +ĠC ream +Ġad vent +ob a +Ġv owed +ous ands +Ġsk i +Ġabort ions +u art +Ġstun ned +Ġadv ancing +Ġlack ed +Ġ\ " +Ġsch izophren +Ġeleg ant +Ġconf erences +Ġcance led +ĠHud son +ĠHop efully +Ġtr ump +Ġfrequ encies +Ġmet eor +ĠJun ior +ĠFle et +ĠMal colm +ĠT ools +Ġ ........ +Ġh obby +ĠEurope ans +Ġ15 00 +ĠInt o +Ġs way +ĠApp ro +ĠCom pl +Comm unity +Ġt ide +ĠSum mit +ä » +Ġinter vals +ĠE ther +Ġhabit at +ĠSteven s +lish ing +ĠDom ain +Ġtrig gers +Ġch asing +Ġchar m +ĠFl ower +it ored +Ġbless ing +Ġtext ures +F ive +Ġliqu or +R P +F IN +Ġ19 62 +C AR +Un known +Ġres il +ĠL ily +Ġabund ance +Ġpredict able +r ar +Ġbull shit +le en +che t +M or +M uch +ä ¹ +Ġemphas ized +Ġcr ust +Ġprim itive +Ġenjoy able +ĠPict ures +Ġteam mate +pl er +ĠT ol +ĠK ane +Ġsummon ed +th y +ram a +ĠH onda +Ġreal izing +Ġquick er +Ġconcent rate +cle ar +Ġ2 10 +ĠErd ogan +ar is +Ġrespond s +ĠB I +Ġelig ibility +Ġpus hes +ĠId aho +Ġagg rav +Ġru ins +ur ations +Ġb ans +Ġan at +sh are +Ġgr ind +h in +um en +Ġut ilities +ĠYan kees +Ġdat abases +ĠD D +Ġdispl aced +Ġdepend encies +Ġstim ulation +h un +h ouses +ĠP retty +ĠRaven s +ĠTOD AY +Ġassoci ates +Ġthe rape +cl ed +Ġde er +Ġrep airs +rent ice +Ġrecept ors +Ġrem ed +ĠC e +Ġmar riages +Ġball ots +ĠSold ier +Ġhilar ious +op l +13 8 +Ġinherent ly +Ġignor ant +Ġb ounce +ĠE aster +REL ATED +ĠCur rency +E V +ãĥ ŀ +ĠLe ad +Ġdece ased +B rien +ĠMus k +J S +Ġmer ge +heart ed +c reat +m itt +m und +ĠâĢ ĭ +ĠB ag +Ġproject ion +Ġj ava +ĠStand ards +ĠLeon ard +Ġcoc onut +ĠPop ulation +Ġtra ject +Ġimp ly +Ġcur iosity +ĠD B +ĠF resh +ĠP or +Ġheav ier +ne ys +gom ery +Ġdes erved +Ġphr ases +ĠG C +Ġye ast +d esc +De ath +Ġreb oot +Ġmet adata +IC AL +Ġrep ay +ĠInd ependence +Ġsubur ban +ical s +Ġat op +Ġall ocation +gener ation +ĠG ram +Ġmoist ure +Ġp ine +ĠLiber als +Ġa ides +Ġund erest +ĠBer ry +Ġcere mon +3 70 +ast rous +ĠPir ates +Ġt ense +ĠIndust ries +ĠApp eals +ĠN ear +Ġè£ı ç +Ġlo vers +ĠC AP +ĠC raw +Ġg iants +Ġeffic acy +E lement +ĠBeh avior +ĠToy ota +Ġint est +P riv +A I +Ġmaneu ver +Ġperfect ion +Ġb ang +p aper +r ill +Ge orge +b order +in ters +ĠS eth +Ġcl ues +ĠLe vi +ĠRe venue +14 7 +Ġv apor +Ġfortun ate +Ġthreat ens +Ġve t +Ġdepend ency +ers ed +art icle +ĠBl izzard +Ġch lor +Ġmin us +ĠB ills +Ġcryptoc urrency +Ġmetabol ism +ter ing +Ġp estic +step s +ĠTre asure +ract ed +ĠConst ant +Ġtem p +13 9 +ĠDet ective +ur ally +Ġrecover ing +Ġcort ex +Ġ14 4 +cl osed +Ġprejud ice +aun ted +Ġstorm s +ĠN OW +Ġmach inery +Add ress +Ġcompe lled +27 0 +Ġdesp air +b ane +Ġveget able +Ġbed s +Lear n +Ġcolor ful +Ġsp ike +Ġmarg ins +Ġsymp athy +Ġworks hop +ĠC BC +S at +Ġburn s +ĠG ender +Ġ12 9 +ĠC able +Ġdeb ts +ĠThe resa +Ġreflect ing +Ġa irst +Ġr im +ram id +Ġweakness es +W rit +ogg le +t i +ĠCh arge +Ġwe ighed +Ġ( . +Ġl aughter +Ġrou ter +ĠDemocr acy +D ear +Ġhas ht +Ġd y +Ġhint s +run ning +Ġfin ishes +ar us +M ass +res ult +asc us +Ġv intage +Ġcon qu +Ġwild ly +ac ist +Ġl ingu +Ġprot agonist +st rom +te enth +ĠSol o +m ac +f illed +Ġre nown +it ives +Ġmot ive +ĠAnt ar +ĠM ann +ĠAd just +Ġrock ets +Ġtrou bling +e i +Ġorgan isms +ass is +Christ ian +Ġ14 5 +ĠH ass +Ġsw all +Ġw ax +ĠSurv ival +V S +ĠM urd +v d +stand ard +Ġdrag ons +Ġacceler ation +r ational +f inal +Ġp aired +ĠE thereum +Ġinterf aces +Ġres ent +Ġartif acts +Å « +are l +Ġcompet itor +ĠNich olas +ĠSur face +c pp +ĠT ot +Ġeconom ically +Ġorgan ised +Ġen forced +in ho +Ġvar ieties +Ġab dom +ĠBa iley +id av +ĠSal v +p aid +Ġalt itude +ess ert +ĠG utenberg +are a +op oulos +Ġprofess ors +igg s +ĠF ate +he y +Ġ3 000 +D ist +Ġtw ins +c ill +ĠM aps +Ġtra ps +Ġwe ed +ĠK iss +Ġy oga +Ġrecip ients +ĠWest minster +Ġpool s +ĠWal mart +18 8 +ĠSchool s +att ack +ĠAR M +par agraph +W arning +j l +Ġself ish +anche z +ĠHe ights +F re +ĠS oph +Ġ -------------------------------- +t ml +33 3 +Ġraid s +Ġsatell ites +KE Y +Ġlast s +Ñ Ĥ +In s +ĠD ame +Ġunp redict +// / +gh ai +Ġart illery +Ġcru ise +Ġg el +ĠCabin et +Ġbl ows +ĠE sp +Ġprox imity +ot he +ĠSk ills +ĠU pper +ob o +ĠN DP +Ġenjoy s +Ġrepe ating +ĠConst ruction +ĠQuest ions +H illary +Ġu int +Ġprocess ors +ĠGib son +ĠMult iple +q a +ĠB om +ĠM iles +vent ional +Ġhur ts +s kin +ĠA IDS +Ġadvis ers +ĠR oot +Ġmethod ology +ĠD ale +Ġdet on +ĠKnow ledge +sequ ently +Ġ12 1 +Ġconnect s +C y +ĠD anger +Ġcontribut ors +ĠB ent +Ġbr ass +ĠGun s +int o +ĠFort une +Ġbro ker +bal ance +Ġlength s +Ġv ic +Ġaver aging +Ġappropri ately +ĠCamer a +Ġsand wich +ĠCD C +Ġcoord inate +Ġnav ig +Ġgood ness +l aim +Ġbra ke +Ġextrem ist +ĠW ake +ĠM end +ĠT iny +ĠC OL +ĠR F +ĠD ual +ĠW ine +C ase +Ġref ined +Ġl amp +L ead +Ġb apt +ĠCar b +ĠS add +ĠMin neapolis +PD F +Ear ly +ĠH idden +I ts +ĠT IME +Ġp ap +Ġcommission ed +ĠF ew +ĠCol ts +ĠB ren +Ġbot hered +Ġlike wise +Ex per +ĠSch w +c ry +n n +ĠM itch +im on +M G +b m +UM P +r ays +Ġregist ry +Ġ2 70 +ach ine +re lla +ant ing +00 000 +Ġru ined +sp ot +Ġt a +Ġmaxim ize +Ġincon ven +D ead +H uman +En abled +ĠMar ie +Ġch ill +ĠParad ise +Ġstar ring +ĠLat ino +ĠProt ocol +ĠE VER +Ġsuppl iers +m essage +ĠBro ck +Ġser um +âĸĪâĸĪ âĸĪâĸĪ +Ġen comp +Ġamb ition +ues e +Ġar rows +And rew +Ġanten na +Ġ19 61 +ĠB ark +Ġb ool +ãĤ ª +ĠSt orage +Ġrail way +Ġtoug her +ĠC ad +Ġwas hing +P y +' ] +em bed +ĠMem phis +ack le +Ġfam ously +ĠF ortunately +ov ies +Ġmind set +Ġsne ak +ĠD h +RA W +ĠSim pson +Ġliv est +Ġland mark +Ġc ement +L ow +Ġthr illed +ĠCour se +in el +Ġch uck +id ate +gl obal +Ġwh it +Ġ � +ad ays +s ki +ĠS V +Ġvir uses +30 6 +ĠResp ons +Ġthe aters +ĠBr anch +ĠGene va +ĠM K +Ġunbel iev +Ġcommun ist +Orig inal +ĠRe ceived +ĠTrans fer +ĠAr g +In put +ĠStr ategy +Ġpal ace +the ning +D ri +Ġsent encing +umbn ail +Ġp ins +re cy +Ġs iblings +Get ting +ĠB U +ĠNorth west +Ġprolong ed +ĠSak ura +C omb +ĠB our +Ġinadequ ate +ĠK ash +Ġus ername +ĠImpro ve +Ġbatt ling +ĠM AC +Ġcurric ulum +Ġs oda +ĠC annon +Ġsens ible +sp ons +De cember +Ġw icked +ĠP engu +Ġdict ators +ĠHe arts +og yn +Ġsimilar ities +ĠSt ats +Ġh ollow +it ations +": [ +Ġh over +ĠList en +s ch +S und +Ġc ad +ĠPar ks +Ġl ur +Ġhy pe +ĠL em +N AME +is ure +Fr iday +Ġshoot s +Ġclos es +Ġd b +ĠR idge +ĠDiff erent +Ġrepl ies +ĠBroad way +op ers +Ġint oler +ĠZe us +akes pe +Ġpropri etary +Ġrequest ing +Ġcontro llers +ĠM IN +im edia +be cca +Ġexp ans +Ġoil s +B ot +ĠCh and +Ġpr inter +Ġto pped +ĠP OL +ĠEar lier +S ocial +av in +Ġdecre ases +ĠSe b +Ġspecific ations +ĠBl ast +ĠK urt +Ġfre el +B rown +Ġdil ig +ro e +ĠPro blem +ĠQu ad +Ġdecent ral +ĠV ector +an ut +Ġplug ins +ĠGreg ory +Ġfuck ed +el ines +ĠAmb assador +t ake +Ġcle ans +ong yang +An onymous +st ro +" } +al ine +ĠO dd +ĠE ug +2 16 +Ġbo il +ĠP owers +Ġnurs es +Ob viously +ĠTechn ical +Ġexceed ed +OR S +Ġextrem ists +Ġtr aces +ex pl +Ġcom r +ĠS ach +) / +Ġm asks +Ġsc i +B on +Ġreg ression +we gian +Ġadvis or +it ures +ĠV o +ex ample +ĠInst ruct +Ġs iege +Ġredu ctions +pt r +Ġstat utory +Ġrem oves +Ġp uck +red its +Ġbe e +Ġsal ad +Ġpromot ions +ĠJosh ua +with standing +ET H +ĠCh a +im us +Ġexpend iture +aun ting +Ġdelight ed +Ġ15 5 +be h +Ġcar pet +ĠSp art +Ġj ungle +l ists +Ġbull ying +ĠNob el +ĠGl en +Ġreferen ced +Ġintrodu ces +se in +Ġcho pped +gl ass +ĠW rest +Ġneutral ity +Ġâ Ļ +Ġinvestig ator +Ġshel ves +Ġun constitutional +Ġreprodu ction +Ġmer chant +m ia +Ġmet rics +Ġexplos ives +ĠSon ia +Ġbod ily +Ġthick ness +Ġpredomin antly +ĠAb ility +Ġmon itored +IC H +Ġ] . +ĠMart inez +Ġvis ibility +Ġqu eries +Ġgen ocide +ĠWar fare +Qu ery +Ġstud ios +Ġemb ry +Ġcorrid or +Ġclean ed +com plete +ĠM H +Ġenroll ment +ING S +Ġimpact ed +Ġdis astrous +ĠY un +ĠCl aire +ĠBas ically +y t +uster ity +Ġindirect ly +w ik +Ġd od +ĠCar r +Ġam p +Ġprohib it +ĠIn itial +ĠR d +ij i +Ġeduc ate +c orn +i ott +ĠBeaut y +Ġdetect ive +ĠCon n +s ince +Ġst agger +Ġob ese +Ġb ree +olog ic +is se +walk er +Ġbl ades +Ġlaw ful +fun c +ĠBeh ind +Ġappet ite +Ġ( * +Ġt ennis +Ġoff spring +Ġj ets +Ġstruct ured +Ġafore mentioned +N ov +Ġsc aling +f ill +Ġst ew +Ġcur b +ĠStep han +ed In +S F +ob ic +é ŃĶ +ou g +ĠM M +Ġgen etically +ope z +13 6 +Ġu mb +anc ers +Ġcoh ort +Ġmerch andise +Ġimp osing +ĠLegisl ature +ĠArch ive +iv ia +ĠN aval +Ġoff ences +Ġmir acle +Ġsn apped +Ġf oes +Ġextensive ly +ĠR af +Ġc ater +ed ience +K it +ĠB in +Ġrecomm ends +ĠC ities +Ġrig id +ĠRE AD +ĠNob le +ĠT ian +Ġcertific ates +ant is +o iler +ĠBudd hist +d id +Ġsurvey ed +Ġdown ward +Ġprint s +ĠMot ion +ron ics +ĠS ans +oss ibly +u ctions +Ġcolon ies +ĠDan ish +un it +Ġsp oil +Ġadvis ory +ber ries +Pl an +Ġspecific ation +op hers +ĠRes ource +Ġsh irts +prising ly +commun ications +Ġtriv ial +Ġmention ing +ise xual +Ġsupp lements +Ġsuper vision +B P +v or +Ġw it +Ġco oldown +Ġplaint iff +ĠReview s +ĠS ri +ĠM int +ĠSug ar +Ġafter ward +ĠPri est +ĠInvest ment +og ene +ĠT aking +Ġstretch ing +Ġinflamm ation +ĠTe hran +Ġl ining +Ġfree zing +ĠEnt ity +Ġins piring +spe cial +pr ice +Ġsu e +ĠP orter +oun ge +ET A +ĠD erek +ĠLu is +u o +ym ph +Ġex terior +ih il +ĠAsh ley +in ator +Ġnut rients +ĠTh rones +Ġfin ances +ĠIn spect +Ġspe cially +ĠRequ ired +ĠP TS +ĠViol ence +oint ed +sh ots +Ġex cerpt +co on +IN S +ĠG ri +Ġrecogn ised +We ek +You ng +Ġv om +is le +ĠCur ry +ĠBudd h +Ġnot ebook +Ġd urable +/ ? +ĠG ad +ĠP upp +Ġforg ive +p ark +Ġpersonal ities +an alysis +cl amation +Ġelev ator +Ġware house +ĠR ole +un n +Ġillust ration +ĠSc an +Ġatmosp heric +Im port +AN C +rict ed +f u +01 0 +Ġar che +Ġreward ed +akespe are +Ġintern ally +ĠR BI +alk er +Ġeleph ant +ow itz +ĠP izza +Ġbip artisan +é s +Ġslow ed +ĠSt ark +Ġover ride +OU S +Ġ3 20 +undred s +ĠDe ck +ĠC ensus +be e +14 6 +ot or +Ġ ip +Ġu b +oc ations +ĠBut ton +r ice +Ġc ripp +ff f +Ġorig inated +Ġoverwhel med +app a +Ġfore most +âĢ ij +ĠL EG +re lease +eat ured +at ches +Ġre ps +Ġl ending +ĠRe ference +ĠCl ient +16 5 +vent h +Com plete +ĠPat rol +Ġsw orn +c am +Ġshut tle +ĠR alph +Ġh ometown +- , +on al +ĠB P +å ı +Ġpersu ade +ĠAlex and +Ġcomb ines +Ġv ivid +ĠL ag +Ġenc oding +Ġsal vation +w en +ĠRec overy +i ya +Un iversity +ĠB iden +Ġbud gets +ĠTex ans +f its +Ġhon ored +Ġp ython +T D +## # +cl one +Ġbl ink +ĠL iquid +Ġunemploy ed +Ġcl ashes +ĠCoun sel +Ġdirect ing +Ġpun ct +ĠFal cons +Ġsh ark +ĠDam ascus +Ġje ans +Ġemb ark +Ġse ize +Ġup wards +2 80 +ĠE z +ĠAny thing +Ġex otic +l ower +ĠCreat or +ĠU m +Ġsubur bs +ber ger +ĠW end +Ġm int +ĠX X +ĠD ro +Ġsuff ers +Ġher b +t ree +Ġfrag ile +Ġflood ed +ĠAl cohol +ole an +ny der +ĠK O +F ram +Ġ13 6 +Ġow ed +ĠMe lee +ĠH ash +Ġwh isk +Ġsu do +r r +Qu ick +app ro +Ġi i +ĠEx amples +he e +Ġpromot es +per ature +k ar +ĠHon or +Ġs odium +ĠL if +ros so +intend ent +Ġcorrespond ent +F ound +sec ret +Ġident ifies +ag ne +Ġl ou +ĠP P +Ġcoinc idence +m ove +Ġmilit ia +Ġinf iltr +ĠPrim ary +Ġpitch ing +ĠI b +ĠGO OD +ãĤ ¸ +ĠW izards +ir al +ĠVen us +R R +ĠâĢ ķ +ĠCase y +Ġsad ly +Ġadm ire +Ġembarrass ed +c b +M el +Ġtub es +Ġbeaut ifully +ĠQueens land +Bel ow +re z +qu et +ple asant +Ġ « +C amp +Ġdec isive +19 98 +ĠL amb +ut ton +h n +ĠJ agu +au nder +ĠC ord +Ġcl erk +Ġca ffe +Ġwip ed +Ġre im +ĠMount ains +Ġimprison ed +Ġdevelop s +ĠP ra +Ġmodel ing +Any one +ance l +ĠS it +Ġshield s +Ġl awn +Ġcard iovascular +Ġdemonstr ating +Ġpar se +ĠIsrael is +Ġeuro s +14 3 +Ġgl orious +ins ki +ec d +Ġcondition ing +Ġhel pless +Ġmicro sc +ĠHar bor +Ġst akes +Ġ2 60 +Ġun equ +ĠFl oyd +Ġd amp +Ġappar atus +ĠLaw s +Ġcoun ters +Ġindu ce +at able +ĠAh med +Ġsl am +N ovember +Ġpers ist +Ġim minent +á n +Ġsh red +Ġph ases +ĠEd monton +ĠArm strong +ĠMe et +ĠK itty +Ñ Ģ +c irc +ĠAd ult +Ġa rose +ĠX en +D an +g ow +Ġsuper f +ĠAd mir +Ġend ure +Ġkey word +yr us +Ġy arn +Ġpath way +ĠHop kins +mid t +Ġcens orship +d ependent +Ġinstruct or +S ources +Ġto e +Ġball oon +N ob +Ġsw ear +ĠCast ro +Ġgl oss +ĠK avanaugh +Ġremark ably +Ph otos +ĠN om +ĠS outheast +y ers +Ġvalid ation +Ġcann on +ĠVict ory +ĠPier re +Ġcaut ious +Aud io +Ġf etch +ĠG ift +ĠH yp +Ġrem edy +Z E +Ġsc ent +Ġbe ard +ĠR ut +- " +Ġpat ents +H y +Ġun just +Ġpot ato +Ġforth coming +Ġche f +ĠR ift +aff e +ĠR OM +ĠL aunch +Ġp ads +ĠNe o +Ġon set +Ġsquee ze +s afe +Ġpref ix +ĠT M +ĠN early +ĠClin ical +ĠM ental +ot iation +ĠUn ic +ant ry +ĠC ir +Ġep it +à ¦ +Ġextract ed +verse ly +ri ad +Ġstr ains +Ġto ps +Ġpo em +ĠRand y +ĠMap le +TH ER +up iter +ĠSS D +ļ é +Ġun con +per ing +Ġsle pt +in ers +Ġunder water +ĠEv idence +g one +20 5 +Ġhistor ians +Ġsynt hesis +Ġf rog +b asketball +Ġvibr ant +Ġsub ord +Ġ3 65 +ĠD ial +Ġcooper ate +HA HA +Ġgreet ed +15 8 +Ġj azz +Ġinto x +ĠWalk ing +Ġsuper visor +ĠF usion +ĠMer cedes +s end +H am +s d +n l +Ġtour s +ĠF IFA +Ġcul p +g d +30 4 +Ġple as +Ġillust rates +ĠColomb ia +Ġhighlight ing +ĠSum mary +Ġexp osing +ĠD ru +Ġir ony +r itional +ĠCar roll +ĠEll is +P ict +ĠR apt +Ġad apter +Ġun m +Ġcor pse +Ġceleb rities +D en +at um +ĠAp ocalypse +ĠW ag +lin ing +Ġhorm ones +R ub +ĠX i +ĠV aults +20 8 +alky rie +inos aur +Ġfeed s +v ity +Ġdefe ating +W ait +Ġemphas ize +ĠSteel ers +yr inth +le ys +ĠWhe never +Current ly +ĠCl ock +Ġcollect ively +any on +ĠJ P +Ġment ality +Ġdownload s +Ġsurround ings +ĠBarn es +Ġflags hip +Ġindic ators +Ġgra pp +Jan uary +ĠElement al +ĠAthen a +ib al +Ġs ights +Ġcap ita +ĠTreat y +Ġvo iced +ĠG az +let te +Ġy a +Ġexp ired +Leg end +H ot +n ature +Ġunst able +Ġ2 80 +à º +Com ment +AL E +Ġquest s +Ġhand ler +n is +Ġvers atile +Ġconce al +enge ance +ĠInter active +Ġobs essed +ĠDog s +Ġcr acked +S ound +s v +ĠD ylan +ro ads +f x +ĠCath olics +ĠH ag +Ġsl ammed +Ġgl owing +s ale +Ġtiss ues +ĠCh i +ne e +Ġc her +s ic +ur rection +Ġb acon +ul atory +) ." +Ġir regular +FOR M +ass ed +Ġintention al +Ġcompens ate +ĠSpe aking +ĠS ets +15 3 +Ġconvent ions +b ands +em ade +Ġe cc +ĠWin ston +ĠAssass in +ĠBelg ian +Ġdepend ence +Ġnic he +Ġb ark +ĠJ azz +Ġdisadvant age +Ġgas oline +Ġ16 5 +çļ Ħ +ess a +mod ule +ang ular +O Y +ĠTreat ment +it as +ol ation +ĠArn old +Ġfe ud +ĠN est +Ġthe atre +ew ater +Ġmin ors +olic y +ĠH aven +div ision +Ġtr unk +F ar +ĠP ull +Ġcapt uring +Ġ18 00 +ĠTe en +Ġex empl +Ġclin ics +ĠB urg +Ġsubst it +Ġpay load +ĠL av +ĠT roy +ĠW itness +Ġfrag ments +Ġpass words +Ġg ospel +ĠG in +Ġten ants +ol ith +S ix +Pre vious +ĠAg es +ĠDar win +Ġbl at +Ġem pathy +sm ith +b ag +ĠE cho +ĠC amb +ĠM add +ĠB oo +Ġred e +ĠBurn ing +Ġsmooth ly +ĠAd rian +ĠV ampire +ĠMon sters +ste am +Sty le +M a +re a +ĠD war +aly st +urs or +Ġelim ination +Ġcrypt o +ch t +ĠE ternal +âĢ¦ ] +ĠS orce +I ll +N ER +Ġu h +Con clusion +w age +Ġresp ir +Ġrem inis +het ical +Ġg y +Ġutil ized +ic idal +Ġ19 00 +Ġhun ters +ĠSw an +ĠRe act +Ġvis itor +ĠThanks giving +30 8 +Post s +Ġh ips +19 97 +om ers +Ġkn ocking +ĠVeh icle +Ġt il +Ġ13 8 +Ġm i +ĠInvest igation +ĠKen ya +Ġcas ino +Ġmot ives +Ġreg ain +re x +Ġweek ends +Ġstab bed +bor o +Ġexplo ited +ĠHA VE +ĠTe levision +c ock +Ġprepar ations +Ġende av +ĠRem ote +ĠM aker +ĠPro du +ĠEv an +Ġinform ational +ĠLouis ville +15 4 +ĠDream s +Ġpl ots +ĠRun ner +Ġhur ting +Ġacad emy +ĠMont gomery +n m +ĠL anc +ĠAl z +2 10 +el ong +Ġretail er +Ġar ising +Ġrebell ion +Ġbl onde +play ed +Ġinstrument al +C ross +Ġret ention +Ġtherape utic +Ġse as +Ġinfant ry +ĠCl int +Ġprompt ing +Ġbit ch +Ġst ems +ĠK ra +Ġthe sis +ĠB og +ru ed +Ġk ings +Ġcl ay +ific ent +ĠY ES +ĠTh ing +ĠCub s +vey ard +els h +in arily +ĠE y +ĠRoll ing +Ġev olving +Ind ia +Ġrecogn izes +Ġgrad uation +is ers +Ġfert ility +ĠMil an +Comm and +Ġbox ing +Ġ19 43 +Ġgl uten +ĠEm ir +Ġid ol +Ġcon ceived +ĠCre ation +Mer it +udd y +uss ions +ĠLie utenant +iet al +Ġunch anged +ĠSc ale +ĠCrime a +ball s +ator ial +Ġdepth s +Ġempir ical +Ġtrans m +Ġuns afe +miss ible +com fort +15 6 +Ġmechan ic +00 2 +l ins +Ġsm oked +P os +Ġslow ing +Ġl av +Tex as +Ġche ating +ĠMet ropolitan +eth yl +Ġdiscover ing +as se +Ġpen cil +ĠPy ongyang +Ġclos et +ĠShe et +ĠEnt ry +ou stic +Ġmy st +er ate +ari at +Ġminer als +Ġmusic ian +ĠP ul +ĠM az +24 9 +Ġper missions +Ġ iv +en ary +ick ers +ĠB ing +he a +en able +Ġgri ev +Ġassert ed +ĠColon el +Ġaff idav +w o +Ġse ated +ĠR ide +Ġpaint ings +ĠP ix +Ġ13 7 +ish i +umb ai +g otten +ĠEar l +Ġin ning +Ġc ensus +Ġtrave lled +ĠCons ult +18 5 +b ind +Ġsimpl icity +Ġoverlook ed +ĠHelp ful +Ġmon key +Ġoverwhelming ly +Bl ood +ĠFl int +ĠJ ama +ĠPres ent +ĠR age +ĠT A +pt ive +Ġturn out +w ald +ĠD olphins +ĠV PN +Ġon ion +Ġcraft ing +m ma +ĠMerc ury +Ġarr ange +Ġalert s +ĠO T +zb ollah +Ġg ases +ĠRichards on +s al +l ar +Ġfro st +Ġlower ing +Ġacc laim +Ġstart ups +ĠG ain +ess ment +Ġguard ian +äº º +ĠP ie +ĠL inks +Ġmer its +Ġaw ake +Ġparent al +Ġexceed s +Ġid le +ĠPil ot +Ġe Bay +ĠAc cept +ipe g +C am +ĠK ot +Ġtrad ers +olit ics +unk er +ĠP ale +os i +an mar +Ġ19 47 +ĠF ell +est ial +it ating +G F +ĠS r +if ted +Ġconnect or +ĠB one +ill es +2 60 +h ma +Ġoverl ap +ĠGit Hub +Ġclean er +ĠBapt ist +ĠW AS +Ġlung s +Ñ ģ +ĠB UT +Ġc ite +Ġpit ched +reat ment +Ġtro phies +ĠN u +38 6 +ĠPr ide +Ġattend ees +[ ] +17 9 +Ġspat ial +Ġpri zes +ĠRel igion +Ġshow case +ĠC ategory +vid ia +T arget +Pro perty +? , +Ġf usion +p ie +ĠU CLA +Ġsound track +Ġprin cess +ĠC aval +sh ould +Ġlim bs +Back ground +Ġlone ly +Ġc ores +ĠT ail +she et +Ġ13 2 +R a +ãĤ « +ĠB olt +Ġbook ed +Ġadmin ister +Ġequ als +w y +Ġobserv ing +ĠBar on +ĠAd obe +Ġv irgin +ĠSocial ist +M ove +gh azi +ĠLind a +2 12 +Ġbre wing +Ġmerch ants +bur se +Ġdiv or +Ġmet als +ĠN er +Ġsum s +ĠEn emy +Ġen vision +Ġgrant ing +ĠH oney +ĠSk yrim +Ġsoc io +gr aded +Ġselect ive +W ASHINGTON +Ġ19 48 +ĠSir ius +ĠG ross +act ivity +ĠI van +Ġfur ious +BS D +ĠPre vious +Ġrespons ive +Ġchar itable +Ġle aning +ĠP ew +Ġviol ates +\\\\ \\\\ +ĠCom ing +w ire +Ġpo et +Ġres olutions +comm and +ĠPortug uese +Ġnick name +Ġde af +Feb ruary +Ġrecogn ise +Ġentire ty +Ġseason al +pl aced +ĠTe legraph +Ġmicro phone +our ing +Ġgr ains +Ġgovern ed +Ġpost p +ĠW aters +in ement +Ġund ocumented +ĠCom cast +Ġf ox +Ġassault s +re on +man y +ĠJen kins +ĠAny way +Ġassess ments +Ġdown s +ĠM ouse +Ġsuper b +k t +ĠD ow +Ġtax ation +4 01 +Ġsm iles +Ġundert aken +Ġex h +Ġenthusi astic +Ġtw ent +Ġgovernment al +Ġautonom y +ĠTechn ologies +ĠCh ain +Ġpreval ent +f b +Ġnic otine +og ram +j ob +Ġawa iting +ĠMen u +Ġdep uties +k ov +ish ops +But ton +ĠShan ghai +Ġdies el +ĠD uck +R yan +ĠPC s +N F +j ury +ent e +Ġinacc urate +edd y +Wh atever +Ġshow c +ĠN ad +od us +et r +Ġplaint iffs +ĠW OR +ĠAss ange +Ġpriv at +Ġpremium s +Ġt am +UR L +Ġel ites +ĠR anger +otten ham +ĠH off +ĠAt hens +Ġdefin ite +Ġs ighed +Ġeven ly +2 11 +ĠAm ber +ak ia +Ġmail ing +Ġcr ashing +ĠConfeder ate +ru gged +W al +ĠDep ths +Ġjuven ile +Ġreact or +Introdu ction +ĠDel uxe +19 95 +ĠS anchez +ĠM ead +iv able +: - +ĠPlan ning +ĠT rap +qu in +ĠProt ect +ve red +In formation +Ġkid ney +inn amon +l as +Ġpolic ing +Ġtoler ate +ĠQ i +Ġbi ased +F ort +ĠK i +s ave +Ġprivile ged +Ġbe asts +ĠGl as +ĠC inem +Ġcome back +Sund ay +Ġext inction +h ops +Ġtrans mit +Ġdoub les +ĠFl at +16 7 +Ġdis puted +Ġinjust ice +f oo +V ict +role um +ĠJul ie +Con text +ĠR arity +iss ue +Comp onent +Ġcounsel ing +an ne +d ark +Ġobject ions +u ilt +Ġg ast +Ġpl ac +Ġun used +ãĥ ĩ +ĠT rial +ĠJ as +hed ral +ob b +Ġtempor al +ĠPR O +ĠN W +ĠAnn iversary +L arge +Ġther m +Ġd avid +Ġsystem ic +ĠSh ir +m ut +ĠNe pt +add ress +Ġscan ning +Ġunderstand able +Ġcan vas +C at +ĠZ oo +Ġang els +L O +ĠStat ement +ĠS ig +ov able +ĠA way +sh aring +ocr ats +st ated +Ġweigh ing +N or +w ild +B ey +Ġaston ishing +ĠReyn olds +Ġop ener +Ġtrain er +Ġsurg ical +p n +Ġadjust ing +whe el +Ġf rown +erv ative +Ġsusp end +With in +te in +Ġobst acle +Ġliber ties +ym es +Ġur anium +ans om +an ol +ub a +ĠL oss +Ġa rous +ĠHend erson +W ow +s pl +c ur +ĠÂ Ń +Ġtheir s +Dam age +Ġdownload ing +Ġdisc ern +ĠSt o +ĠFl a +Ġh ath +ĠA j +Ġun pleasant +Europe an +exp ensive +Ġscreens hot +ĠU V +Ġall ied +ĠPers ian +Ġmonop oly +Ġat om +ĠReds kins +"> < +Ġcan cell +Ġcinem a +13 1 +f air +ĠAlf red +Ġd uck +arg s +22 3 +ĠIS I +Ġsign aling +in ar +Ġlaugh s +Ġfor wards +Ġreck less +Ġlisten ers +at ivity +Ġvast ly +n ant +L ess +ĠHun ting +ĠScient ific +IT ED +Ġkn ight +ĠH TC +us a +t mp +Ġr ude +ĠLegend ary +Ġar ises +B ad +ĠCl aim +pe g +Ġreal ities +Th ink +Ġ ° +Ġro de +Ġstri ve +Ġan ecd +Ġshort s +Ġhypot hes +Ġcoord inated +ĠGand hi +ĠF PS +R ED +Ġsuscept ible +Ġshr ink +ĠCh art +Hel p +Ġ ion +de ep +rib es +ĠK ai +ĠCustom er +Sum mary +Ġc ough +w ife +Ġl end +Ġposition ing +Ġlot tery +ĠC anyon +Ġf ade +Ġbron ze +ĠKenn y +Ġbo asts +ĠEnh anced +rec ord +Ġemer gence +Ġa kin +ĠB ert +it ous +âĸ ij +Ġst ip +Ġexch anged +om ore +als h +Ġreserv oir +Ġstand point +W M +Ġiniti ate +Ġdec ay +Ġbrew ery +Ġter ribly +Ġmort al +lev ard +Ġrev is +N I +el o +Ġconf ess +ĠMS NBC +Ġsub missions +Cont roller +Ġ20 2 +ĠR uth +} ); +ĠAz ure +Ġ ." +20 6 +ĠMarket ing +Ġl aund +ien cies +Ġrenown ed +ĠT rou +ĠN GO +ble ms +Ġterr ified +Ġwar ns +Ġper t +Ġuns ure +4 80 +ale z +ult z +ĠOut side +Ġst yl +ĠUnder ground +Ġp anc +Ġd ictionary +Ġf oe +rim inal +ĠNor wegian +Ġj ailed +Ġm aternal +é e +ĠLu cy +c op +Ch o +Ġuns igned +ĠZe lda +ĠIns ider +ĠContin ued +Ġ13 3 +ĠNar uto +ĠMajor ity +16 9 +ĠW o +ãĤ ĵ +Ġpast or +Ġinform al +Ð ½ +an throp +jo in +ãģ Ĺ +it ational +N P +ĠWrit ing +f n +ĠB ever +19 5 +Ġy elling +Ġdr astically +Ġe ject +Ġne ut +Ġth rive +ĠFre qu +ou x +Ġpossess es +ĠSen ators +ĠD ES +ĠSh akespeare +ĠFran co +ĠL B +uch i +Ġinc arn +Ġfound ers +F unction +Ġbright ness +ĠB T +Ġwh ale +ĠThe ater +m ass +ĠD oll +S omething +Ġecho ed +ĠHe x +c rit +af ia +Ġgodd ess +Ġele ven +ĠPre view +ĠAur ora +Ġ4 01 +uls ive +ĠLog an +in burgh +ĠCent ers +ĠON LY +ĠA id +Ġparad ox +Ġh urd +ĠL C +D ue +c ourt +Ġoff ended +Ġeval uating +ĠMatthew s +Ġto mb +Ġpay roll +Ġextra ction +ĠH ands +if i +Ġsuper natural +ĠCOM M +] = +dog s +Ġ5 12 +ĠMe eting +Rich ard +ĠMax imum +Ġide als +Th ings +m and +ĠReg ardless +Ġhum ili +b uffer +L ittle +ĠD ani +ĠN ak +Ġliber ation +ĠA be +ĠO L +Ġstuff ed +ac a +ind a +raph ic +Ġmos qu +Ġcampaign ing +Ġoccup y +S qu +r ina +ĠW el +ĠV S +Ġphys ic +Ġp uls +r int +oad ed +ET F +ĠArch ives +Ġven ues +h ner +ĠTur bo +Ġl ust +Ġappeal ed +que z +il ib +ĠTim othy +Ġo mn +d ro +Ġobs ession +ĠSav age +19 96 +Gl obal +J es +2 14 +Ġsl iding +Ġdisapp ro +ĠMag ical +Ġvolunt arily +g b +ane y +Ġprop het +ĠRe in +ĠJul ia +ĠW orth +aur us +Ġb ounds +ie u +)) ) +Ġcro re +ĠCitiz en +S ky +Ġcolumn ist +Ġseek ers +ond o +IS A +ĠL ength +Ġnost alg +Ġnew com +Ġdet rim +ent ric +3 75 +ĠG E +Ġaut op +Ġacadem ics +App Data +ĠS hen +Ġid iot +ĠTrans it +Ġteasp oon +W il +K O +ĠCom edy +> , +Ġpop ulated +W D +Ġp igs +ĠO culus +Ġsymp athetic +Ġmar athon +19 8 +Ġseiz ure +s ided +Ġd op +irt ual +L and +ĠFl oor +osa urs +... ] +Ġl os +Ġsubsid iary +E Y +ĠPart s +ĠSt ef +ĠJud iciary +Ġ13 4 +Ġmir rors +Ġk et +t imes +Ġneuro log +Ġc av +ĠGu est +Ġtum or +sc ill +ĠLl oyd +E st +Ġcle arer +Ġstere otypes +Ġd ur +not hing +Red dit +Ġnegoti ated +---------------- -------- +23 5 +Ġfl own +ĠSe oul +ĠRes ident +ĠS CH +Ġdisappear ance +ĠV ince +g rown +Ġgrab s +r il +ĠInf inite +ĠTw enty +Ġpedest rian +Ġjer sey +ĠF ur +ĠInf inity +ĠEll iott +Ġment or +Ġmor ally +Ġob ey +sec ure +iff e +Ġantib iotics +ang led +ĠFre eman +ĠIntrodu ction +J un +Ġm arsh +ic ans +ĠEV ENTS +och ond +W all +icult y +Ġmisdem eanor +Ġl y +Th omas +ĠRes olution +Ġanim ations +ĠD ry +Ġinter course +ĠNew castle +ĠH og +ĠEqu ipment +17 7 +Ġterrit orial +Ġarch ives +20 3 +Fil ter +ĠMun ich +Ġcommand ed +ĠW and +Ġpit ches +ĠCro at +Ġrat ios +ĠM its +Ġaccum ulated +ĠSpecific ally +Ġgentle man +acer b +Ġp enn +Ġa ka +ĠF uk +Ġinterven e +ĠRef uge +ĠAlz heimer +Ġsuccess ion +oh an +d oes +L ord +Ġsepar at +Ġcorrespond ence +Ġsh iny +P rior +Ġs ulf +Ġmiser able +Ġded ication +( ). +Ġspecial ists +Ġdefect s +ĠC ult +ĠX ia +Ġje opard +ĠO re +Ab ility +Ġle ar +Ġamb itions +ĠB MI +ĠArab s +Ġ19 42 +Ġpres ervation +ific ate +Ġash amed +l oss +ĠRest aur +Ġrese mble +Ġen rich +ĠK N +ĠCl an +fl oat +Ġplay able +IT T +Ġharm ony +arr ison +ĠWe instein +w ere +Ġpoison ing +ĠCom put +ĠWord Press +m ajor +ĠVal ve +F an +ĠTh row +ĠRom ans +ĠDep ression +ad os +Ġtort ured +Ġbal ancing +bott om +Ġacqu iring +ĠMon te +ard i +Ġa ura +Ġ# # +ĠStand ing +ĠAtl as +C F +Ġintr ins +ĠBen ghazi +Ġcamp ing +Ġt apped +bl ade +st rous +ĠR abb +ĠW ritten +t ip +ĠNe igh +ster dam +ĠAll ow +ĠHe aling +ĠR hod +n um +Ġcaffe ine +ĠPer cent +Ġbo o +Ġapp les +30 5 +Ġwel coming +Ġappl aud +Ġa usterity + ± +ĠRe ality +ef e +å ® +Ġsu cks +Ġtab s +ĠPay Pal +Ġback pack +Ġgif ted +abul ary +ĠSc out +ir teen +Ġch in +Ġo mitted +Ġnegative ly +Ġaccess ing +ĠE arn +Ġambul ance +Ġhead phones +Ġ20 5 +ĠRef resh +p resident +ĠKit chen +ĠEnt ered +ĠS nyder +00 5 +om ical +Ġborrow ed +ĠN em +Ġav iation +Ġst all +rim ination +Ġuniform s +it ime +ĠSim mons +ener gy +ab lished +y y +qual ified +Ġrall ies +ĠSt uart +fl ight +Ġgang s +r ag +Ġv ault +lu x +ĠCom par +Ġdesign ation +20 9 +ĠJ os +d ollar +z ero +Ġwell s +30 3 +Ġconstitu ents +Ġhe ck +Ġc ows +Ġcommand ers +Ġdifferent ial +ĠC atherine +29 9 +Ġval ve +Ġbr ace +Ġperspect ives +c ert +f act +icular ly +ĠMc N +pl anes +Ġint ric +Ġpe as +ov an +Ġtoss ed +ret ch +ĠL opez +Ġunf amiliar +de ath +ĠA part +ĠCh ang +Ġrelie ved +rop he +Ġair ports +Ġfre ak +ut il +M ill +ĠCh in +ĠOw en +m ale +ĠBro ken +ĠWind s +ro b +r ising +Ġfire fighters +Ġauthor itarian +Ġ14 8 +Bit coin +ex ternal +Ġbrow sers +iche ver +or ian +Ġun b +Ġpo ke +ĠZ ot +M id +ĠPop ular +Ġco vert +Ġcont ributes +Ġ6 50 +Ġcont ention +G ate +Ġcons oles +Ġchrom os +ĠI X +Ġvis ually +ĠE isen +Ġjewel ry +Ġdeleg ation +Ġacceler ate +ĠR iley +Ġsl ope +Ġind oor +it ially +Ġhuge ly +Ġtun nels +Ġfin ed +Ġdirect ive +Ġfore head +ustom ed +Ġsk ate +Mus ic +g as +Ġrecogn izing +am bo +Ġover weight +ĠGr ade +Ù Ĭ +Ġsound ing +Ġlock ing +ĠR EM +St ore +Ġexc av +ĠLike wise +ĠL ights +Ġel bow +ĠSupp ly +w ic +Ġhands ome +19 94 +C oll +Ġadequ ately +ĠAssoci ate +Ġstri ps +Ġcrack down +Ġmar vel +ĠK un +Ġpass ages +@@ @@ +ĠT all +Ġthought ful +names e +Ġprost itution +bus iness +Ġball istic +person al +c ig +iz ational +R ound +ĠÂłĠÂł ĠÂłĠÂł +ĠCole man +Ġadm itting +ĠPl ug +Ġbit coins +ĠSu z +Ġfair ness +Ġsupp lier +Ġcatast rophic +ĠHel en +o qu +M arc +ĠArt icles +g ie +Ġend angered +Ġdest iny +ĠVol t +ol ia +ax is +Ġche at +Ġun ified +IC O +qu ote +30 2 +ĠS ed +Ġsupp ression +Ġanaly zing +Ġsqu at +Ġfig uring +Ġcoordin ates +Ġch unks +Ġ19 46 +Ġsub p +Ġw iki +ĠFor bes +ĠJ upiter +ĠE rik +im er +ĠCom mercial +\ ) +Ġlegitim acy +Ġd ental +ĠMe an +Ġdefic its +5 50 +Orig inally +ĠHor ror +Ġcontam ination +ll ah +Ġconf isc +ĠCl are +T B +ĠF ailed +an ed +Ġrul er +ĠCont roller +Ġfemin ists +F ix +g ay +20 7 +Ġr abbit +Th ird +ownt own +Ġgl ue +Ġvol atile +Ġsh ining +Ġf oll +Ġimp aired +Ġsup ers +æ Ī +Ġcl utch +ļé ĨĴ +Ġpro let +Ġ( ! +Ġy elled +ĠK iev +ĠEr n +ĠSh ock +K B +Ġsit uated +qu ery +ĠN as +Ġan nex +char acter +ĠHol iday +Ġautom ation +ĠJ ill +ĠRem astered +Ġl inem +Ġwild erness +ĠHor izon +ĠGu inea +A Z +Ġmain land +Ġsec recy +LE ASE +Ġp unk +ĠProv ince +( ), +Spe ed +Ġhand ing +ĠSeb ast +S ir +r ase +Ġj ournals +Ġcon gest +ĠT ut +ir rel +Ġschizophren ia +Ġmis ogyn +health y +I ron +Ġreact ed +- $ +25 2 +Ġpl ural +Ġpl um +Ġbarg ain +Ġground ed +f inder +Ġdis se +ĠL az +O OD +Ġat roc +F actory +Ġmin ions +Ġo ri +ĠB rave +ĠP RE +ĠMy anmar +ĠH od +Ġexped ition +Ġexpl ode +ĠCo ord +Ġext r +ĠB rief +ĠAD HD +Ġhard core +feed ing +Ġd ile +ĠF ruit +Ġvacc ination +ĠM ao +osp here +Ġcont ests +- | +Ġf ren +isp here +R om +ĠSh arp +ĠTre nd +Ġdis connect +âĢ¢ âĢ¢ +Ġper secution +Ear th +Ġhealth ier +38 4 +Ġc ob +ĠTr inity +OW S +AN N +Ġspecial ty +Ġg ru +Ġcooper ative +wh y +Start ing +ĠIss ues +st re +ens or +Ġ18 5 +Ad v +! ? +ĠRe vel +em ia +ĠH ulk +Ġcelebr ations +ĠS ou +ra ud +ĠKle in +Ġun real +con text +Ġpartners hips +Ġadop ting +t ical +Ġspl ash +ĠHe zbollah +c ategory +cycl op +xt on +ĠD ot +urd y +t z +Ġenvelop e +ĠN L +â ķ +Ġwhere in +Spe c +18 4 +Ġte lev +al iation +Ġmyth s +å ° +Ġrig orous +Ġcommun icating +Ġobser ver +Ġre he +ĠW ash +Ġapolog ized +ĠT in +Ġexpend itures +work ers +d ocument +Ġhes itate +ĠLen in +Ġunpredict able +Ġrenew al +cl er +ok ia +ĠCON T +Ġpost season +Tok ens +Ġex acerb +Ġbet ting +Ġ14 7 +Ġelev ation +W ood +ĠSol omon +19 4 +00 4 +out put +Ġredu nd +ĠM umbai +Ġp H +Ġreprodu ce +ĠD uration +MA X +Ġb og +C BS +ĠBal ance +ĠS gt +ĠRec ent +Ġc d +Ġpo pped +Ġincomp et +pro p +ay an +g uy +Pac ific +Ġty r +Ġ{ { +ĠMy stic +ĠD ana +Ġmast urb +Ġge ometry +à ¢ +ĠCor rect +Ġtraject ory +Ġdistract ed +Ġf oo +ĠW elsh +L uc +m ith +Ġrug by +Ġrespir atory +Ġtri angle +Ġ2 15 +Ġunder graduate +ĠSuper ior +ch anging +_ - +Ġright ly +Ġrefere e +Ġluc rative +Ġun authorized +Ġresemb les +ĠGN U +ĠDer by +Ġpath ways +ĠL ed +Ġend urance +Ġst int +Ġcollect or +F ast +Ġd ots +Ġnational s +ĠSec urities +Ġwh ip +Par am +Ġlearn s +M agic +Ġdetail ing +m oon +Ġbroadcast ing +Ġb aked +26 5 +hol m +ĠS ah +ĠHus sein +ĠCourt esy +17 4 +Ġ14 6 +Ġge ographic +pe ace +Ġjud ging +ĠS tern +B ur +Ġstory line +G un +ĠSt ick +24 5 +30 7 +ãĤ´ ãĥ³ +ĠAdminist rator +Ġbur nt +Ġp ave +ch oes +Ex ec +Ġcamp uses +Res ult +Ġmut ations +ĠCh arter +Ġcapt ures +Ġcomp ares +Ġbad ge +S cient +Ġer ad +ier y +o i +ett es +ĠE state +Ġst rap +Ġproud ly +Ġf ried +Ġwithd rawn +ĠV oy +ph ony +It ems +ĠP ierce +b ard +Ġann otation +ant on +ill on +Im pro +... ) +Ġhapp ier +---- -- +ad just +Ġstaff ers +Ġactiv ism +Ġper f +Ġal right +N eed +Ġcomm ence +Ġopio id +ĠAm anda +E s +ĠP ars +ĠK aw +W orks +24 8 +Ġind o +t c +end ant +ĠM oto +Ġlegal ization +OT E +Ġtask ed +Ġt sp +ĠACT IONS +16 6 +Ġrefres hing +ĠN R +ĠPere z +Ġinfring ement +S Y +List en +in ning +k u +Ġrot ate +pro gram +ar ah +Des ign +Ġ( £ +Ġst oring +Ġwar rants +Ġjud gement +ĠB rist +us ually +ph oto +ĠR an +ĠP ine +Ġoutrage ous +ĠValent ine +lu ence +ĠEvery body +Al tern +Ġrele vance +Ġtermin ated +Ġd essert +Ġfulf illed +Ġprosecut ed +ĠW ords +Ġm igrant +Ġcultiv ation +ÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤ ÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤ +idel ity +ĠV ern +ĠLog in +Ġmetaph or +ĠT ip +Ġrecru its +ĠP ig +rib ing +Ġenthusi asts +ex per +Ġfright ening +ĠH air +ans on +str ate +Ġh i +He ight +Ġown ing +n one +Ġdis like +Ġkn ives +pher d +Ġloud ly +ĠAP Is +Dis play +ĠL ac +ĠUS S +ab l +ver ages +J ew +Ġ17 2 +ĠHist orical +at oon +ĠPhys ics +in tern +Ġwarm th +Ġto pp +D M +Ġgun man +Ġem peror +od i +ãĥ £ +in atory +ĠR ib +Ġ13 1 +ĠSat urn +ĠSh ining +Ġw aking +Qu otes +Ġcomed ian +en berg + ½ +Ġbelie vers +Ġpaper work +c ustom +Ġle v +Ġl ament +Ġpour ing +22 2 +p olitical +ĠSupp lement +m aid +Ġcruel ty +Ġt read +ys ics +A w +rit es +Ġmod ifier +ĠP osition +Ad am +l b +ub s +Ġimper fect +Ġcl usters +ĠEngine er +ĠC herry +Ġinaug uration +ĠS au +Ġembod iment +ĠUn cle +Ġover r +Ġexplos ions +c ule +ĠPrinc eton +ĠAndre a +Ġincorrect ly +Ġearn est +Ġpil gr +ĠS print +Ġslee ve +Ġhe ars +ĠAm azing +Ġbrow sing +ag in +Ġhom eland +Ġha w +Ġd iving +ist ered +17 8 +Ġbarg aining +ĠArc ade +Ġdeleg ate +ters on +................................ ................................ +ĠJackson ville +27 5 +Ġst agn +Ġad am +ĠSher man +C B +Ġsub urb +ĠFood s +Ġconver ting +ĠAr ist +Ġch ambers +l ove +Ġam ino +ĠG an +Ġmad ness +m c +ĠUS E +def ined +Ġul tr +ind ust +Ġw olves +l ance +Add itionally +Ġcr acks +as ia +ĠRe ason +ĠP ump +Ġaccident al +ĠL aser +ĠR id +Ġinitial ized +ell i +Ġun named +Ġn oun +ĠPass ed +Ġhost age +ĠEth iop +sh irts +Ġun rel +ĠEmb assy +Ġ19 41 +Ġat oms +Ġpur ported +16 4 +ĠF i +Ġgall ons +ĠMon ica +Ġp g +en ment +Ġsort ed +ĠG ospel +Ġhe ights +Ġtr aced +Ġunder going +She ll +Ġs acks +Ġproport ions +Ġhall uc +F ont +ac et +Ġwar mer +ĠIN TER +Ġgrab bing +Pl ug +Ġreal ization +ĠBur ke +Ġen chant +AT ER +ĠSe ed +Ġabund ant +F M +Ġc ivic +V s +is i +Ġv ow +Ġre per +ĠPartners hip +Ġpenet ration +Ġax e +Ġsh attered +ĠZ ombies +Ġv inyl +ĠAl ert +e on +Ġoblig ed +ĠIll ust +ĠPl aza +ĠFront ier +Ġdavid jl +ĠSer ial +ĠH av +ĠNut rition +B i +Ġâĸ Ī +ĠJ ays +lin ux +Ġhur ry +Ġv oy +Ġhop eless +ĠSte alth +Ġ ãģ +ess ors +tt le +b org +ĠSaf ari +f ell +Ġw ary +d ue +ĠAb ove +H a +E LL +Ġnot or +ĠW on +T oo +Ġoccup ations +Ġposs essions +Ġinv iting +Ġpred ators +Ġacceler ated +Ġ15 7 +uter te +ĠC ube +e ast +acc ount +G ive +Ġtrans plant +red ients +id able +Ġscreens hots +ĠG und +ĠF S +Ġtravel ers +Ġsens ory +ĠF iat +ĠRock ets +İ ĭ +_ { +F riend +Ġchar ming +AL S +Ġenjoy ment +m ph +Ġ5 000 +ĠRE G +Ù Ĩ +b ia +Ġcomp ilation +ro st +ĠV P +ĠSch ne +201 9 +Ġcop ying +M ORE +ĠFl ore +f alls +2 15 +t otal +Ġdis ciples +d ouble +Ġexceed ing +Ġsm ashed +Ġconcept ual +ĠRom ania +ĠB rent +ĠI CE +ĠT ou +Ġg rap +Ġn ails +18 9 +ãĥ ĺ +Ġproc ure +e ur +Ġconfir ming +ĠC ec +aw i +ĠEd en +Ġn g +Ġengine ered +at ics +Ġhook ed +Ġdisgust ing +ĠMur der +ãĤ ¿ +L ibrary +Ġ16 8 +Al most +hem atic +Men u +ĠNot re +ĠJ ur +Ġkidn apped +Ġhack er +ĠJ ade +Ġcreep y +Ġdraw ings +ĠSpons or +Ġcycl ists +ĠGob lin +Ġoptim ized +Ġst aged +ĠMc D +bet ween +A ge +en o +S ex +ĠW ide +n ings +av is +Ġincap able +ĠK ob +Ġreward ing +ĠL one +oles cent +Ġcontract ed +Ġstick y +J ose +B all +f est +ĠIn put +ĠRec ently +Ġto mat +squ are +App lication +Ġnit rogen +Ġdupl icate +ĠRec on +ĠD ear +L ondon +Ġint ra +Ġd ock +Ġout reach +ĠM illion +Ġmamm als +am pton +V AL +Ġsn aps +Ġd os +ĠWh ole +ĠRead y +T ry +ĠWinn ipeg +ear ance +Ġinc urred +ren ched +ĠNS W +il ot +rain e +Ġc ube +g ot +Ġrun way +etermin ed +ĠHaw ks +Ġsurviv or +ĠW ish +ĠD in +ĠDE F +ĠV ault +18 7 +Ġmush rooms +Ġcris p +be y +ĠDisco very +Ġdevelopment al +Ġparad igm +Ġcha otic +ĠT su +Ġ3 33 +b ons +Ġbacter ial +Ġcomm its +Ġcos mic +Ġme ga +oc ative +ĠP aint +ophob ic +Ġv ain +Ġcar ved +ĠTh ief +ĠG ul +ows hip +Ġc ites +ĠEd inburgh +Ġdimin ished +Ġacknowled ges +ĠK ills +Ġmic row +ĠHer a +Ġsen iors +Ġwhere by +H op +at ron +Ġun available +ĠN ate +Ġ4 80 +Ġsl ated +ĠRe becca +ĠB attery +Ġgram mar +Ġhead set +Ġcurs or +Ġex cluding +any e +aunder ing +eb in +Ġfeas ible +ĠPub lishing +ĠLab s +ĠCl iff +ĠFerr ari +Ġp ac +vis ible +mark ed +pe ll +Ġpol ite +Ġstagger ing +ĠGal actic +Ġsuper st +Ġpar an +ĠOffic ers +ãĢ ģ +Ġspecific s +ul us +23 9 +ĠP aste +AM P +ĠPan ama +ĠDe lete +angu ard +rest rial +Ġhero ic +ĠD y +ا ÙĦ +Ġincumb ent +Ġcr unch +t ro +Ġsc oop +Ġblog ger +Ġsell ers +ure n +Ġmedic ines +ĠC aps +ĠAnim ation +ox y +Ġout ward +Ġinqu iries +22 9 +Ġpsych ologist +ĠS ask +ev il +Ġcontam inated +ãĤ ¨ +he rence +Ġbrand ed +ĠAbd ul +z h +Ġparagraph s +Ġmin s +Ġcor related +er b +Ġimp art +Ġmil estone +ĠSol utions +ot le +Ġunder cover +Ġmar ched +ĠCharg ers +f ax +ĠSec rets +Ġr uth +we ather +Ġfemin ine +Ġsh am +Ġprest igious +igg ins +Ġs ung +hist ory +ett le +gg ie +Ġout dated +ol and +Ġper ceptions +ĠS ession +ĠDod gers +u j +ĠE ND +D oc +Ġdefic iency +Gr and +ĠJ oker +Ġretro spect +Ġdiagn ostic +Ġharm less +Ġro gue +ĠA val +E qu +Ġtrans c +ĠRoberts on +ĠDep ending +ĠBurn s +iv o +Ġhost ility +F eatures +ĵ ĺ +Ġdis comfort +ĠL CD +spec ified +ĠEx pect +3 40 +Ġimper ative +ĠReg ular +Ch inese +Ġstate wide +Ġsy mm +Ġlo ops +Ġaut umn +N ick +Ġsh aping +Ġqu ot +Ġc herry +ĠCross ref +è¦ ļéĨĴ +Stand ard +he ed +ĠD ell +ĠViet namese +Ġo st +ĠV alkyrie +O A +Ass ad +Ġreb ound +ĠTra ffic +pl aces +æ ĺ +ĠB uc +17 2 +Ġshel ters +Ġins isting +ĠCertain ly +ĠKenn eth +ĠT CP +Ġpen al +ĠRe play +he ard +Ġdial ect +iz a +ĠF Y +it cher +ĠD L +Ġspir al +Ġquarterback s +Ġh ull +Ġgo ogle +Ġto dd +ĠSter ling +ĠPl ate +Ġsp ying +mb ol +ĠReal m +ĠPro ced +ĠCr ash +Ġtermin ate +Ġprotest ing +C enter +gu ided +Ġun cover +Ġboy cott +Ġreal izes +s ound +Ġpret ending +ĠV as +19 80 +Ġfram ed +Ġ13 9 +Ġdesc ended +Ġrehab ilitation +Ġborrow ing +ĠB uch +Ġbl ur +R on +ĠFro zen +en za +Ch ief +ĠP oor +Ġtransl ates +M IN +Ġ2 12 +J ECT +Ġerupt ed +Ġsuccess es +S EC +Ġpl ague +Ġg ems +d oms +Ġstret ches +ĠSp y +Ġstory telling +C redit +ĠP ush +Ġtra ction +Ġin effective +ĠL una +Ġt apes +Ġanaly tics +erc ise +Ġprogram mes +ĠCar bon +Ġbeh old +he avy +ĠConserv ation +ĠF IR +Ġs ack +ter min +ric ks +Ġhous ed +Ġunus ually +I ce +Ġexecut ing +ĠMor oc +ed ay +Ġed itions +Ġsm arter +ĠB A +Ġout law +Ġvan ished +ib a +AL SE +ĠSil va +23 8 +C ould +Ġphilos opher +Ġevac uated +Sec ret +14 2 +Ġvis as +ãĤ ¬ +ĠM alt +ĠClear ly +ĠN iger +ĠC airo +ĠF ist +3 80 +ĠX ML +aut o +it ant +Ġrein forced +Rec ord +ĠSurviv or +G Hz +Ġscrew s +parent s +Ġo ceans +ma res +Ġbra kes +vas ive +Ġhell o +ĠS IM +rim p +Ġo re +ĠArm our +24 7 +Ġterr ific +Ġt ones +14 1 +ĠMin utes +Ep isode +Ġcur ves +Ġinflamm atory +Ġbat ting +ĠBeaut iful +L ay +Ġunp op +v able +Ġr iots +ĠTact ics +b augh +ĠC ock +Ġorg asm +ĠS as +Ġconstruct or +et z +G ov +Ġant agon +Ġthe at +Ġde eds +ha o +c uts +ĠMc Cl +Ġu m +ĠScient ists +Ġgrass roots +ys sey +"] => +Ġsurf aced +Ġsh ades +Ġneighb ours +Ġad vertis +oy a +Ġmer ged +Up on +Ġg ad +Ġanticip ate +Any way +Ġsl ogan +Ġdis respect +I ran +ĠT B +act ed +Ġsubp oen +medi ately +OO OO +Ġwa iver +Ġvulner abilities +ott esville +ĠHuff ington +J osh +ĠD H +M onday +ĠEll en +K now +x on +it ems +22 8 +Ġf ills +ĠN ike +Ġcum ulative +and als +I r +Ġ ì +Ġfr iction +ig ator +Ġsc ans +ĠVi enna +ld om +Ġperform ers +P rim +Ġb idding +M ur +Ġlean ed +ĠPri x +al ks +Ġ[ âĢ¦] +ĠTw itch +ĠDevelop er +ĠG ir +Ġcall back +Ab stract +Ġacc ustomed +Ġfreed oms +ĠP G +ur acy +Ġl ump +is man +,, ,, +19 92 +ĠR ED +Ġwor m +M atch +ĠPl atinum +I J +ĠOwn er +Tri via +com pl +Ġnew born +Ġfant as +O wn +Ġ19 59 +Ġsymp ath +Ġub iqu +Ġoutput s +Ġal lev +Ġpr ag +K evin +Ġfav ors +Ġbur ial +Ġn urt +so lete +c ache +Ġ15 6 +Ġunl ocks +te chn +M aking +Ġcon quer +ad ic +æ ĸ +Ġel f +Ġelect orate +ĠKurd s +ĠSt ack +ĠSam urai +Ġâ ĺħ +Ġ{ } +ĠS aid +ĠFall out +Ġkind ness +ĠCustom s +ĠBou levard +Ġhelicop ters +ot ics +ĠVe get +com ment +Ġcritic ised +Ġpol ished +ĠRem ix +ĠC ultural +Ġrec ons +Ġdo i +at em +Sc reen +Ġbar red +Com ments +ĠGener ally +Ġsl ap +7 20 +V ari +p ine +Ġem pt +Ġh ats +ĠPlay ing +l ab +a verage +form s +ĠC otton +Ġcan s +ĠD ON +ĠSom alia +C rypt +ĠIncre ases +E ver +mod ern +Ġsur geon +3 000 +Ġrandom ized +================================ ================================ +B ern +im pl +ĠC OR +Ġpro claim +th ouse +Ġto es +Ġam ple +Ġpres erving +Ġdis bel +gr and +B esides +Ġsil k +ĠPat tern +h m +Ġenter prises +Ġaffidav it +ĠAdvis ory +Ġadvert ised +ĠRel igious +se ctions +psy ch +ĠField s +aw ays +Ġhasht ag +ĠNight mare +Ġv ampire +Ġfore nsic +rosso ver +n ar +Ġn avy +Ġvac ant +ĠD uel +Ġhall way +Ġface book +ident ally +ĠN RA +Ġm att +Ġhur ricane +ĠKir by +ĠP uzzle +Ġsk irt +ou st +du llah +Ġanal ogy +in ion +Ġtomat oes +ĠN V +ĠPe ak +ĠMe yer +Ġappoint ments +Ġm asc +Ġal ley +re hend +Ġchar ities +Ġund o +Ġdest inations +ĠTest ing +"> " +c ats +* . +Ġgest ures +gener al +Le ague +Ġpack ets +ĠInspect or +ĠBer g +Ġfraud ulent +Ġcritic ize +F un +Ġbl aming +nd ra +Ġsl ash +ĠE ston +Ġpropos ing +Ġwh ales +Ġtherap ist +Ġsub set +Ġle isure +EL D +ĠC VE +ĠAct ivity +Ġcul min +sh op +ĠD AY +is cher +ĠAdmir al +ĠAtt acks +Ġ19 58 +Ġmem oir +Ġfold ed +Ġsex ist +Ġ15 3 +ĠL I +Ġread ings +Ġembarrass ment +ĠEmploy ment +w art +ch in +Ġcontin uation +l ia +Rec ently +Ġd uel +Ġevac uation +ĠKash mir +Ġdis position +ĠR ig +Ġbol ts +Ġins urers +4 67 +M ex +Ġret aliation +Ġmis ery +Ġunre asonable +r aining +I mm +ĠP U +em er +Ġgen ital +ãĤ ³ +ĠC andy +Ġon ions +ĠP att +lin er +Ġconced ed +Ġf a +Ġfor c +ĠH ernandez +ĠGe off +deb ian +ĠTe ams +Ġc ries +Ġhome owners +23 7 +A BC +Ġst itch +Ġstat istic +Ġhead ers +ĠBi ology +Ġmot ors +ĠG EN +ĠL ip +Ġh ates +Ġhe el +S elf +i pl +ED IT +ort ing +Ġann ot +ĠSpe ech +old emort +ĠJ avascript +ĠLe Bron +Ġfoot print +Ġf n +Ġseiz ures +n as +h ide +Ġ19 54 +ĠBe e +ĠDecl aration +ĠKat ie +Ġreserv ations +N R +f emale +Ġsatur ated +Ġb iblical +Ġtroll s +Dev ice +ph otos +Ġdr ums +ãĥīãĥ© ãĤ´ãĥ³ +N ight +f ighter +ĠH ak +ri ber +Ġc ush +Ġdiscipl inary +ba um +ĠG H +ĠSch midt +ilib rium +Ġs ixty +ĠKush ner +ro ts +Ġp und +ĠR ac +Ġspr ings +Ġcon ve +Bus iness +F all +Ġqual ifications +Ġvers es +Ġnarc iss +ĠK oh +ĠW ow +ĠCharl ottesville +ed o +Ġinterrog ation +ĠW ool +36 5 +B rian +Ġâľ ĵ +Ġalleg es +ond s +id ation +ĠJack ie +y u +Ġl akes +Ġworth while +Ġcryst als +ĠJud a +Ġcomp rehend +Ġfl ush +Ġabsor ption +ĠO C +Ġfright ened +ĠCh ocolate +Mart in +Ġbu ys +Ġbu cks +Ġapp ell +ĠChampions hips +Ġlist ener +ĠDef ensive +Ġc z +ud s +ĠM ate +Ġre play +Ġdecor ated +Ġs unk +ĠV IP +ĠAn k +Ġ19 5 +aa aa +Nob ody +ĠMil k +ĠG ur +ĠM k +ĠS ara +Ġse ating +ĠW id +Tr ack +Ġemploy s +Ġgig antic +AP P +ãĤ § +in ventory +Ġtow el +at che +l asting +ĠT L +Ġlat ency +Ġkn e +B er +me aning +Ġup held +Ġplay ground +Ġm ant +S ide +Ġstere o +Ġnorth west +Ġexception ally +Ġr ays +Ġrec urring +D rive +Ġup right +Ġab duct +ĠMar athon +Ġgood bye +Ġal phabet +h p +Ġcourt room +ring ton +ot hing +T ag +Ġdiplom ats +Ġbar bar +ĠAqu a +18 3 +33 33 +Ġmat urity +Ġinst ability +ĠAp ache +Ġ= == +Ġfast ing +ĠGr id +Mod Loader +Ġ15 2 +A bs +ĠOper ating +ett i +Ġacqu aint +Don nell +ĠK em +ĠFor ge +Ġarm ored +M il +Ġphilos ophers +in vest +Pl ayers +â Ī +Ġmy riad +Ġcomr ades +R ot +Ġremember ing +Ġcorrespond s +Ġprogram mers +ĠLyn n +Ġo lig +Ġco herent +yn chron +ĠChem ical +Ġj ugg +p air +post s +E ye +ĠIn ner +Ġsem ester +ott est +ĠEmir ates +ric anes +or ously +m its +ĠW is +Ġd odge +l ocation +Ġf aded +Am azon +ĠPro ceed +ĠIN FO +j ournal +ĠTru ck +T en +Ġ2 17 +Ġstat utes +m obile +ĠT ypes +Rec omm +b uster +pe x +Ġleg ends +Ġhead ache +f aced +ĠWi Fi +if ty +ĠH ER +Ġcirc uits +ER ROR +22 6 +ol in +Ġcyl inder +osp ace +ik ers +P rem +Qu ant +Ġconflic ting +Ġslight est +Ġfor ged +ion age +Step hen +ĠK ub +ĠOpp ortun +ĠHe al +Ġbl o +Ġrul ers +Ġh uh +Ġsubmar ine +f y +ass er +Ġallow ance +ĠKas ich +ĠT as +ĠAustral ians +Forge ModLoader +ĠâĨ ij +ĠMat rix +am ins +Ġ12 00 +ĠAc qu +23 6 +D ocument +ĠBre aking +19 3 +ĠSub st +ĠRoll er +ĠPro perties +ĠN I +t ier +Ġcr ushing +Ġadvoc ating +Further more +keep ers +Ġsex ism +x d +Ġcall er +ĠS ense +chie ve +ĠT F +Ġfuel ed +Ġreminis cent +Ġobs ess +ur st +Ġup hold +ĠF ans +het ics +Ġâ Ĺ +ĠB ath +Ġbe verage +Ġo scill +25 4 +Ġpol es +Ġgrad ual +Ġex ting +ĠS uff +ĠS uddenly +Ġlik ing +Ġ19 49 +un ciation +am ination +ĠO mar +ĠL V +ĠCon sequently +Ġsynt hes +ĠG IF +Ġp ains +Ġinteract ing +u ously +inc re +Ġrum or +ĠScient ology +19 7 +ĠZ ig +Ġspe lling +ĠA SS +Ġexting u +ms on +Ġg h +Ġremark ed +ĠStrateg ic +ĠM ON +å ¥ +g ae +ĠWH AT +E ric +ĠCamp us +Ġmeth ane +Ġimag in +J UST +ĠAl m +X T +i q +ĠR SS +Ġwrong doing +att a +Ġbig ot +Ġdemonstr ators +ĠCal vin +ĠV illa +Ġmembr ane +ĠAw esome +Ġbenef ic +26 8 +Ġmagn ificent +ĠL ots +G reg +ĠBor is +Ġdetain ees +ĠH erman +Ġwhis pered +Ġa we +Prof essor +fund ing +Ġphys iological +ĠDest ruction +Ġlim b +Ġmanip ulated +Ġbub bles +Ġpse ud +Ġhyd ra +ĠBrist ol +Ġst ellar +ĠExp ansion +ĠK ell +ĠInterest ingly +Ġm ans +Ġdrag ging +Ġec ological +ĠF it +Ġg ent +Ġbenef ited +ĠHait i +Ġpoly g +ãĥ İ +Ġ20 30 +Ġpro w +Ġrecon struction +Ġwas t +Ġpsych ic +ĠGree ks +Hand ler +16 2 +ĠP ulse +Ġsol icit +Ġsy s +Ġinflu x +ĠG entle +per cent +Ġprolifer ation +Ġtax able +Ġdisreg ard +Ġesc aping +Ġg inger +Ġwith stand +Ġdevast ated +ĠD ew +ser ies +Ġinject ed +ela ide +Ġturn over +he at +Ļ Ĥ +H appy +ĠSil ent +ãĤ Ń +iv ism +Ġir rational +AM A +Ġre ef +r ub +Ġ16 2 +Ġbank ers +ĠEth ics +v v +Ġcritic isms +K n +18 6 +M ovie +ĠT ories +Ġno od +Ġdist ortion +F alse +od ore +Ġt asty +Res earch +ĠU ID +- ) +Ġdivor ced +ĠM U +ĠHay es +ĠIs n +ian i +ĠH Q +Ġ" # +ign ant +Ġtra umatic +ĠL ing +H un +Ġsab ot +on line +r andom +Ġren amed +ra red +K A +d ead +é t +ĠAss istance +Ġse af +++++ ++++ +Ġse ldom +ĠWeb b +Ġbo olean +u let +Ġref rain +ĠDI Y +ru le +Ġshut ting +Ġutil izing +load ing +ĠPar am +co al +oot er +Ġattract ing +ĠD ol +Ġher s +ag netic +ĠRe ach +im o +Ġdisc arded +ĠP ip +01 5 +ü r +Ġm ug +Im agine +C OL +Ġcurs ed +ĠSh ows +ĠCurt is +ĠSach s +spe aking +ĠV ista +ĠFram ework +ong o +Ġsub reddit +Ġcr us +ĠO val +R ow +g rowing +Ġinstall ment +Ġgl ac +ĠAdv ance +EC K +ĠLGBT Q +LE Y +Ġac et +Ġsuccess ive +ĠNic ole +Ġ19 57 +Qu ote +Ġcircumst ance +ack ets +Ġ14 2 +ort ium +Ġguess ed +ĠFr ame +Ġperpet rators +ĠAv iation +ĠBen ch +Ġhand c +A p +Ġ19 56 +25 9 +r and +Net Message +d in +urt les +h ig +ĠV III +ff iti +ĠSw ords +b ial +Ġkidn apping +dev ice +Ġb arn +ĠEl i +auc as +S end +Con structed +Ġ ½ +Ġneed les +Ġad vertisements +Ġv ou +Ġexhib ited +ĠFort ress +As k +B erry +TY PE +Ġcan cers +ump ing +ĠTerrit ory +Ġpr ud +Ġn as +Ġathe ist +Ġbal ances +ãģ Ł +ĠSh awn +& & +Ġland sc +ĠR GB +Ġpet ty +Ġex cellence +Ġtransl ations +Ġpar cel +ĠChe v +E ast +ĠOut put +im i +Ġamb ient +ĠTh reat +Ġvill ains +Ġ5 50 +IC A +Ġtall er +Ġle aking +c up +Ġpol ish +Ġinfect ious +ĠK C +Ġ@ @ +back ground +Ġbureaucr acy +ĠS ai +un less +it ious +ĠSky pe +At l +ID ENT +00 8 +Ġhyp ocr +Ġpit chers +Ġguess ing +ĠF INAL +Bet ween +Ġvill agers +Ġ25 2 +f ashion +ĠTun is +Be h +ĠEx c +ĠM ID +28 8 +ĠHas kell +19 6 +ĠN OR +Ġspec s +Ġinv ari +Ġgl ut +ĠC ars +Ġimp ulse +Ġhon ors +g el +Ġjurisd ictions +ĠBund le +ul as +Calif ornia +ĠIncre ase +Ġp ear +Ġsing les +Ġc ues +Ġunder went +ĠW S +Ġexagger ated +Ġdub ious +Ġfl ashing +L OG +) ]. +J ournal +t g +V an +ĠI stanbul +ĠIn sp +ĠFrank en +D raw +Ġsad ness +Ġiron ic +ĠF ry +x c +Ġ16 4 +is ch +W ay +ĠProtest ant +h orn +Ġun aff +ĠV iv +ill as +ĠProduct ions +ĠH ogan +Ġper imeter +ĠS isters +Ġspont aneous +Ġdown side +Ġdescend ants +Ġor n +w orm +Japan ese +Ġ19 55 +Ġ15 1 +ĠDo ing +els en +umb les +Ġrad ically +ĠDr um +ĠB ach +Ġli abilities +ĠO B +ĠElement ary +Ġmem e +yn es +Ġfinger print +ĠGr ab +Ġundert ake +Mem bers +ĠRead er +ĠSim s +g od +Ġhypot hetical +s cient +ĠA J +Ġchar ism +Ġad missions +ĠMiss ile +tr ade +Ġexerc ising +ĠBack ground +W ritten +Ġvoc als +whe ther +Ġv i +ĠW inner +Ġl itter +ĠSh ooting +ST EM +ãĤ ¡ +ĠA FL +Ġvari ability +Ġe ats +ĠD PS +b row +Ġeleph ants +Ġstr at +Ġ Å +Ġsett lers +Matt hew +Ġin advert +H I +ĠIM F +ĠGo al +Ġnerv es +John son +ey e +ablish ment +Th ursday +BIL ITY +H ad +am oto +het amine +ep s +Ġmit ochond +Ġcomp ressed +ĠTre vor +ĠAnim als +T ool +L ock +Ġtwe ak +Ġpin ch +Ġcancell ation +P ot +Ġfoc al +ĠAst ron +17 3 +ĠA SC +ĠO THER +umn i +Ġdem ise +d l +Ù ħ +Sem itism +Ġcr acking +Ġcollabor ative +Ġexpl ores +s ql +Ġher bs +Ġconfig urations +m is +ĠRes ult +ace y +ĠSm oke +Ġsan ct +el ia +Ġdeg ener +Ġdeep est +Ġscream ed +Ġn ap +Soft ware +ĠST AR +E F +ĠX in +spons ored +mans hip +23 3 +Ġprim aries +Ġfilter ing +Ġas semble +m il +ĠMy ers +b ows +Ġpun ched +M ic +Ġinnov ations +Ġfun c +and o +Ġfr acking +ĠV ul +о Ð +osh op +ĠIm mun +Ġsett ling +Ġadolesc ents +Ġreb uilding +Ġtransform ing +Ġpar ole +Ġhar bor +Ġbook ing +ot ional +onge vity +ĠY o +b ug +Ġemer ges +ĠMethod s +ĠCh u +P res +ĠDun geons +Ġtra iling +ĠR um +ĠH ugh +å¤ © +ĠE ra +ĠBatt les +Res ults +ĠTr ading +Ġvers a +c ss +ax ies +he et +Ġgre ed +19 89 +Ġgard ens +Ġconting ent +P ark +ĠLeaf s +h ook +ro be +Ġdiplom acy +ĠF uel +ĠInv asion +Ġupgr ading +M ale +Ġe lic +Ġrelent less +ĠCo venant +ap esh +ĠT rop +T y +pro duction +art y +Ġpun ches +ak o +cyclop edia +ĠR abbit +ĠHD MI +Ġ14 1 +Ġf oil +Item Image +ĠF G +Ġimplement ations +ĠP om +ixt ures +Ġaw ait +Ġ3 30 +am us +Ġumb rella +Ġfore see +se par +Ġcircum cision +Ġperipher al +S ay +ĠExper t +In c +Ġwithd rew +ĠAnd ers +f ried +Ġradio active +ĠOp ening +Ġboard ing +ĠN D +Ġover throw +Act iv +W P +ĠAct s +× Ļ +Ġmot ions +v ic +ĠM ighty +ĠDef ender +a er +Ġthank ful +ĠK illing +ĠBr is +mo il +Ġpredict ing +26 6 +ch oice +Ġkill ers +Ġinc ub +ĠChe st +ather ing +Ġpro claimed +fl ower +oss om +umbled ore +ĠCy cling +ĠOccup y +AG ES +P en +ĠY ug +Ġpack aged +Ġheight ened +c ot +st ack +C ond +Ġst amps +m age +Ġpersu aded +Ġens l +ĠCard inal +Ġsol itary +Ġpossess ing +ĠC ork +Ġev id +ĠT ay +Ġbl ues +Ġextrem ism +Ġlun ar +Ġcl own +Te chn +Ġfest ivals +ĠPv P +ĠL ar +Ġconsequ ently +p resent +Ġsom eday +ç İĭ +ĠMet eor +Ġtour ing +c ulture +Ġbe aches +S hip +c ause +ĠFl ood +ãĥ ¯ +Ġpur ity +th ose +Ġem ission +b olt +Ġch ord +ĠScript ure +L u +Ġ$ { +cre ated +Other s +25 8 +Ġelement al +Ġannoy ed +ĠA E +d an +ĠS ag +Res earchers +Ġfair y +âĢĵ âĢĵ +======== ==== +Sm art +GG GG +Ġskelet ons +Ġpup ils +link ed +Ġur gency +en abled +ĠF uck +Ġcoun cill +r ab +U AL +T I +Ġlif es +Ġconf essed +B ug +Ġharm on +ĠCON FIG +ĠNe utral +D ouble +Ġst aple +ĠSH A +Brit ish +ĠSN P +AT OR +oc o +Ġswing ing +ge x +ole on +pl ain +ĠMiss ing +ĠTro phy +v ari +ran ch +Ġ3 01 +4 40 +00000000 00000000 +Ġrest oring +Ġha ul +uc ing +ner g +Ġfut ures +Ġstrateg ist +quest ion +Ġlater al +ĠB ard +Ġs or +ĠRhod es +ĠD owntown +????? - +ĠL it +ĠB ened +Ġco il +st reet +ĠPort al +FI LE +ĠG ru +* , +23 1 +ne um +Ġsuck ed +Ġr apper +Ġtend encies +ĠLaure n +cell aneous +26 7 +Ġbrow se +Ġover c +head er +o ise +Ġbe et +ĠG le +St ay +Ġm um +Ġtyp ed +Ġdiscount s +T alk +ĠO g +ex isting +ĠS ell +u ph +C I +ĠAust rian +ĠW arm +Ġdismiss al +Ġaver ages +c amera +Ġalleg iance +L AN +=" # +Ġcomment ators +ĠSet ting +ĠMid west +Ġpharm ac +ĠEX P +Ġstain less +Ch icago +Ġt an +24 4 +Ġcountry side +ĠV ac +29 5 +Ġpin ned +Ġcr ises +Ġstandard ized +T ask +ĠJ ail +ĠD ocker +col ored +f orth +" }, +Ġpat rons +Ġsp ice +Ġm ourn +ĠM ood +Ġlaund ry +Ġequ ip +ĠM ole +y ll +ĠTH C +n ation +ĠSher lock +Ġiss u +ĠK re +ĠAmeric as +ĠA AA +Ġsystem atically +Ġcont ra +ĠS ally +Ġrational e +Ġcar riage +Ġpe aks +Ġcontrad iction +ens ation +ĠFail ure +Ġpro ps +Ġnames pace +Ġc ove +field s +ãĤ ĭ +Ġw ool +ĠC atch +Ġpresum ed +ĠD iana +r agon +ig i +Ġh amm +Ġst unt +ĠG UI +ĠObserv atory +ĠSh ore +Ġsmell s +ann ah +Ġcock pit +ĠD uterte +8 50 +Ġopp ressed +bre aker +ĠCont ribut +ĠPer u +ĠMons anto +ĠAtt empt +Ġcommand ing +Ġfr idge +ĠR in +ĠChe ss +ual ity +Ġo l +Republic an +ĠGl ory +ĠW IN +.... ... +ag ent +read ing +Ġin h +J ones +Ġcl icks +al an +Ġ[ ]; +ĠMaj esty +ĠC ed +op us +ate l +à ª +AR C +ĠEc uador +ãĥ ł +ĠK uro +Ġritual s +Ġcapt ive +Ġoun ce +Ġdisag reement +Ġsl og +f uel +P et +M ail +Ġexerc ised +Ġsol ic +Ġrain fall +Ġdev otion +ĠAss essment +Ġrob otic +opt ions +ĠR P +ĠFam ilies +ĠFl ames +Ġassign ments +00 7 +aked own +Ġvoc abulary +Re illy +Ġc aval +g ars +Ġsupp ressed +ĠS ET +ĠJohn s +Ġwar p +bro ken +Ġstat ues +Ġadvoc ated +Ġ2 75 +Ġper il +om orph +ĠF emin +per fect +Ġh atch +L ib +5 12 +Ġlif elong +3 13 +Ġche eks +Ġnum bered +ĠM ug +B ody +ra vel +We ight +ĠJ ak +ĠHe ath +Ġkiss ing +ĠJ UST +Ġw aving +u pload +Ġins ider +ĠPro gressive +ĠFil ter +tt a +ĠBe am +Ġviol ently +ip ation +Ġskept icism +Ġ19 18 +ĠAnn ie +ĠS I +Ġgen etics +Ġon board +at l +ĠFried man +ĠB ri +cept ive +Ġpir ate +ĠRep orter +27 8 +Ġmyth ology +Ġe clipse +Ġsk ins +Ġgly ph +ing ham +F iles +C our +w omen +Ġreg imes +Ġphotograp hed +K at +ĠMA X +Offic ials +Ġunexpected ly +Ġimpress ions +F ront +;;;; ;;;; +Ġsuprem acy +Ġs ang +Ġaggrav ated +Ġabrupt ly +ĠS ector +Ġexc uses +Ġcost ing +ide press +St ack +ĠR NA +ob il +Ġghost s +ld on +at ibility +Top ics +Ġreim burse +ĠH M +ĠDe g +Ġth ief +y et +ogen esis +le aning +ĠK ol +ĠB asketball +Ġf i +ĠSee ing +Ġrecy cling +Ġ[ - +Cong ress +Ġlect ures +P sy +Ġne p +Ġm aid +Ġori ented +A X +Ġrespect ful +re ne +fl ush +ĠUn loaded +re quest +gr id +ĠAltern atively +ĠHug o +Ġdec ree +ĠBuddh ism +and um +And roid +ĠCong o +ĠJoy ce +Ġacknowled ging +hes ive +ĠTom orrow +ĠH iro +th ren +ĠM aced +Ġho ax +ĠIncre ased +ĠPr adesh +W ild +____ __ +16 1 +Ġa unt +Ġdistribut ing +ĠT ucker +ĠSS L +ĠW olves +B uilding +ou lt +ĠLu o +ĠY as +ĠSp ir +ĠSh ape +ĠCamb od +ĠIP v +Ġm l +Ġext rad +39 0 +ĠPenn y +d ream +Ġstation ed +opt ional +ew orthy +. +ĠWorks hop +ĠRet ail +ĠAv atar +6 25 +N a +ĠV C +ĠSec ure +M Y +19 88 +oss ip +Ġpro state +Ġund en +Ġg amer +ĠCont ents +ĠWar hammer +ĠSent inel +3 10 +Ġse gregation +ĠF lex +ĠM AY +Ġdr ills +ĠDrug s +Islam ic +Ġsp ur +Ġca fe +Ġimag inary +Ġgu iding +Ġsw ings +ĠThe me +ob y +Ġn ud +Ġbe gging +Ġstr ongh +Ġreject ing +Ġpedest rians +ĠPro spect +R are +s le +Ġconcess ions +ĠConst itutional +Ġbe ams +Ġfib ers +p oon +Ġinstinct s +pro perty +ĠB IG +Sand ers +im ates +Ġco ating +Ġcorps es +ĠTR UE +check ed +Ġ16 6 +A sh +ĠJ S +ĠF iction +Ġcommun al +Ġener getic +oooo oooo +Ġnow adays +IL D +ib o +ĠSU V +R en +Ġdwell ing +Sil ver +Ġt ally +ĠM oving +Ġcow ard +Ġgener als +Ġhorn s +Ġcirc ulated +Ġrob bed +ĠUn limited +Ġharass ed +Ġinhib it +Ġcomp oser +ĠSpot ify +Ġspread s +3 64 +Ġsu icidal +Ġno ises +ĠSt ur +Ġs aga +ĠK ag +is o +Ġtheoret ically +M oney +Ġsimilar ity +Ġslic ed +ut ils +ing es +" - +Ġan th +Ġimp ed +Mod ule +Through out +Ġmen us +comm ittee +and i +ob j +in av +f ired +ĠAb dullah +Ġund ead +Ġfont s +H old +EN G +Ġsustain ability +Ġfl ick +Ġr azor +ĠF est +ĠChar acters +Ġword ing +Ġpopul ist +Ġcritic izing +Ġm use +v ine +Ġcard board +Ġkind ly +Ġfr inge +ĠThe ft +icult ural +Ġgovern ors +Ġ ���� +Ġ16 3 +Ġtime out +ĠA uth +Child ren +A U +Ġred emption +ĠAl ger +Ġ19 14 +Ġw aved +Ġastron auts +og rams +Ġsw amp +ĠFinn ish +Ġcand le +Ġton nes +ut m +Ġr ay +Ġsp un +Ġfear ful +art icles +Ġca us +or ically +ĠRequ ires +ĠG ol +Ġpop e +Ġinaug ural +Ġg le +AD A +ĠIS IL +ĠOff ensive +Ġwatch dog +Ġbal con +ent ity +ĠH oo +Ġgall on +AC C +Ġdoub ling +Ġimpl ication +ĠS ight +Ġdoct r +---- --- +Ġ\ \ +Ġm alt +R oll +Ġâī ¥ +Ġrec ap +add ing +u ces +ĠB end +fig ure +Ġtur key +Ġsoc ietal +ĠT ickets +Ġcommer cially +Ġsp icy +Ġ2 16 +ĠR amp +Ġsuperior ity +à ¯ +ĠTr acker +C arl +ĠC oy +ĠPatri ot +Ġconsult ed +Ġlist ings +Ġsle w +reens hot +ĠG one +Ġ[ ...] +30 9 +Ġh ottest +Ø ± +Ġrock y +ĠD iaz +Ġmass age +Ġpar aly +Ġp ony +A z +Ġcart ridge +ĠN Z +Ġsn ack +ĠLam ar +ple ment +ĠLes lie +Ġm ater +Ġsn ipp +24 6 +Ġjoint ly +ĠBris bane +ĠiP od +Ġpump ing +Ġgo at +ĠSh aron +eal ing +Ġcor on +Ġan omal +rah im +ĠConnect ion +Ġsculpt ure +Ġsched uling +ĠD addy +at hing +Ġeyeb rows +Ġcur ved +Ġsent iments +Ġdraft ing +D rop +( [ +Ġnom inal +ĠLeaders hip +ĠG row +Ġ17 6 +Ġconstruct ive +iv ation +Ġcorrupt ed +ger ald +ĠC ros +ĠChe ster +ĠL ap +ãģ ª +OT H +D ATA +Ġal mond +pro bably +I mp +Ġfe ast +ĠWar craft +F lor +Ġcheck point +Ġtrans cription +Ġ20 4 +Ġtwe aks +Ġrel ieve +S cience +Ġperform er +Z one +Ġtur moil +ig ated +hib it +ĠC afe +the med +Ġflu or +ben ch +Ġde com +ĠU nt +ĠBar rett +ĠF acts +Ġt asting +ĠPTS D +ĠSe al +ĠJuda ism +ĠDynam ic +ĠC ors +V e +ĠM ing +ĠTrans form +v on +ĠDef enders +ĠTact ical +ĠV on +ĠUn ivers +Ġdist orted +ĠB reath +?' " +Ġag on +ĠDead ly +Ġl an +ĠCy cle +orn ed +Ġrel iably +Ġgl or +ĠMon key +ãĥ ¡ +Ġad ren +Ġmicrow ave +ĠAl ban +irc raft +dig it +sm art +ĠD read +¯¯¯¯¯¯¯¯ ¯¯¯¯¯¯¯¯ +{ { +ĠRoc hester +Ġsimpl ified +Ġinf licted +Ġtake over +Ġyour selves +ad itional +Ġmus cular +K S +Ġing en +T ax +ĠFe ature +27 7 +Ġcru c +Ġcr ate +Ġun identified +Ġacclaim ed +ĠM anga +ĠFr ances +ĠNep al +ĠG erald +ĠKu wait +Ġsl ain +ĠHe b +ĠG oku +ãģ® æ +28 6 +M rs +ĠC ody +ĠSan ctuary +01 6 +Ġdism ant +Ġdatas et +ĠH ond +b uck +ĠPat terson +Ġpal ette +ĠG D +ic ol +ĠL odge +Ġplanet ary +ak in +ĠRegist ered +ab we +ĠPeters burg +Ġha iled +ĠP iece +S che +ĠDO J +Ġen umer +18 1 +ĠObs erver +ĠB old +f ounded +com merce +Ġexplo its +ĠF inding +UR N +ĠS ne +ĠAc id +ay ette +ĠVal ues +Ġdr astic +Ġarchitect ural +Ġ" . +× ķ +ump ed +Ġwra pping +Ġwid ow +ĠSl ayer +l ace +on ce +German y +av oid +Ġtem ples +P AR +à ´ +ĠLuc ifer +ĠFl ickr +l ov +for ces +Ġsc outing +Ġlou der +tes y +Ġbefore hand +Ä ĵ +ĠNe on +ĠW ol +ĠTyp ically +ĠPolit ico +-+ -+ +Ġbuild er +Ġder ive +K ill +Ġp oker +Ġambig uous +Ġlif ts +Ġcy t +Ġrib s +ood le +ĠS ounds +h air +ĠSynd rome +t f +Ġproport ional +u id +Ġper taining +ĠKind le +ĠNeg ro +Ġreiter ated +ĠTon ight +oth s +ĠCorn ell +Ġo wing +Ġ20 8 +elf are +oc ating +ĠB irds +Sub scribe +Ġess ays +Ġburd ens +Ġillust rations +ar ious +ER AL +ĠCal cul +Ġx en +ĠLink edIn +ĠJ ung +Ġredes ign +Con nor +29 6 +Ġrevers al +ĠAd elaide +ĠL L +Ġs inking +Ġg um +US H +c apt +ĠGr imm +Ġfoot steps +ĠCB D +isp ers +Ġpro se +Wed nesday +ĠM ovies +ed in +Ġoverturn ed +Ġcontent ious +US B +~~~~~~~~ ~~~~~~~~ +ĠCo pper +Ġpoint less +N V +val ues +olph in +d ain +Ġdepos ited +ĠG W +Ġpreced ed +ĠCl a +ĠGo lem +ĠN im +ĠÎ ² +ĠEngine ers +m iddle +Ġfl att +oper ative +Ġcouncil s +imb abwe +el in +Ġstress ful +ĠL D +Ġres h +l ake +Ġwheel chair +ĠAltern ative +Ġoptim ize +oper ation +Ġpe ek +Ġones elf +ig il +Ġtrans itions +op athy +bl ank +Ġ16 9 +17 1 +________________________________ ________________________________ +Ġl aundering +En c +ĠD EC +Ġwork outs +Ġsp ikes +Ġdin osaurs +Ġdiscrim inatory +P ool +R ather +38 5 +R NA +tes ters +et o +ĠIdent ity +Ġve in +ĠBur ton +Ġarc ade +4 20 +Ult imately +ĠSad ly +à ° +p ill +Ġcub ic +ĠSpect rum +the se +st ates +Ġun official +h awks +ĠEVER Y +Ġrain bow +Ġincarcer ation +and ing +Ġsy ll +ĠEver ton +Ġ17 9 +ĠSer bia +Ġ18 9 +m eter +ĠMic key +Ġant iqu +Ġfact ual +ne ck +ĠN are +n orm +m ust +Ġhigh ways +Ġgl am +Ġdivid ing +ĠSquad ron +ĠMar tha +Ġbirth s +C over +//////// //////// +ĠW ong +Ph ot +ĠA LS +ri o +ĠNon etheless +ĠL emon +Ġ20 6 +ĠE E +Ġderiv ative +ĠWW II +v ote +Ġthere in +Ġsepar ating +44 6 +sy nc +ĠStre ets +Ġr att +Ġmunicip ality +ĠShort ly +Ġmon k +) ," +Ġscr ub +Ġoper atives +Ne ither +Pl ace +ĠLim it +F emale +ĠAct or +Char acter +Ġconstit uted +35 7 +Ġprotest ed +ĠSt raw +ĠHe ight +ild a +ĠTy ph +Ġflood s +Ġcos metic +W AY +pert ure +up on +t ons +ess ing +ĠP ocket +Ġro oft +ĠC aucas +Ġant idepress +Ġincomp atible +EC D +Ġoper a +ĠCont est +Ġgener ators +l ime +Def ense +19 87 +for um +Ġsav age +ĠHung arian +n z +Ġmet allic +Ġex pelled +Ġres idency +Ġdress es +66 6 +ĠC lement +f ires +C ategory +Ġge ek +al is +Ġc emetery +educ ated +Ġc rawl +ĠUn able +ĠT yson +ak is +Ġp ardon +ĠW ra +Ġstrengthen ed +ĠF ors +33 5 +ĠH C +ĠM ond +Ġvisual s +ĠBeat les +ett lement +Ġ ï +g ro +Ġb ash +Ġpo orest +Ġex cel +Ġaspir ations +ĠM unicip +ens ible +Ġceremon ies +Ġintimid ation +ĠCON TR +be ck +ĠK ap +as u +Ġtradem arks +ĠS ew +ĠComp etition +net work +ĠAr ri +ĠT et +Ro aming +W C +D at +Ġso b +Ġpair ing +Ġoverd ose +SA Y +ab er +Ġrev olt +ĠF ah +act ing +e q +est ation +F ight +ĠMar ks +27 3 +Ġ17 8 +R aw +ãģ ĭ +34 9 +bl ocks +Ġver ge +est ine +ĠPod esta +Ġinv asive +Ġprofound ly +ĠA o +e ach +Ġl est +inter pret +Ġshr inking +Ġerr one +Ġche es +ly s +ĠI vy +ĠDirect ory +Ġhint ed +V ICE +Ġcontact ing +ĠG ent +he i +Ġlabel ing +Ġmerc ury +ĠL ite +Ġexp ires +Ġdest abil +rit is +c u +Ġfeather s +Ġste er +Ġprogram med +ĠV ader +Go ing +ĠE lim +Ġy o +ĠMic he +Ġ20 3 +Ġslee ves +Ġb ully +ĠHum ans +36 8 +Ġcomp ress +ĠBan ner +AR S +Ġa while +Ġcal ib +Ġspons orship +ĠDiff iculty +ĠP apers +Ġident ifier +} . +Ġy og +ĠSh ia +Ġclean up +Ġvib e +int rodu +im ming +Austral ia +Ġout lines +ĠY outube +tr ain +ĠM akes +Ġde ported +Ġcent r +ĠD ug +ĠB oulder +ĠBuff y +Ġinj unction +ĠHar ley +ĠG roups +ĠD umbledore +ĠCl ara +Ġ" - +Ġsacrific ed +ep h +Sh adow +ib ling +Ġfreel ance +Ġevident ly +ph al +Ġret ains +M ir +Ġfin ite +d ar +ĠC ous +Ġrep aired +Ġperiod ic +Ġchampions hips +Ġaster oid +bl ind +Ġexpress ly +ĠAst ros +Ġsc aled +Ġge ographical +ĠRap ids +En joy +Ġel astic +ĠMoh amed +Mark et +be gin +Ġdisco vers +Ġtele communications +Ġscan ner +Ġen large +Ġsh arks +Ġpsy chedel +ĠRou ge +Ġsnap shot +is ine +X P +Ġpestic ides +ĠL SD +ĠDist ribution +re ally +Ġde gradation +Ġdisgu ise +Ġbi om +ĠEX T +Ġequ ations +Ġhaz ards +ĠComp ared +) * +Ġvirt ues +Ġeld ers +Ġenh ancing +ĠAc ross +er os +ang ling +Ġcomb ust +ucc i +Ġconc ussion +Ġcontrace ption +ĠK ang +Ġexpress es +Ġa ux +ĠP ione +Ġexhib its +Deb ug +OT AL +ĠAl ready +ĠWheel er +Ġexp ands +? : +Ġreconc iliation +Ġpir ates +Ġpur se +Ġdiscour age +Ġspect acle +R ank +Ġwra ps +ĠTh ought +Ġimp ending +O pp +ĠAng lo +ĠE UR +Ġscrew ed +ret ched +Ġencour agement +mod els +Ġconf use +mm m +ĠVit amin +âĸij âĸij +C ru +Ġkn ights +Ġdisc ard +Ġb ishops +ĠW ear +ĠGar rett +k an +ãĥ Ł +Ġmascul ine +cap ital +ĠA us +Ġfat ally +th anks +ĠA U +ĠG ut +12 00 +Ġ 00000000 +Ġsur rog +ĠBI OS +ra its +ĠWat ts +Ġresur rection +ĠElect oral +ĠT ips +4 000 +Ġnut rient +Ġdepict ing +Ġspr ink +Ġm uff +ĠL IM +ĠS ample +ps c +ib i +gener ated +Ġspec imens +Ġdiss atisf +Ġtail ored +Ġhold ings +ĠMonth ly +ĠE at +po ons +Ġne c +ĠC age +ĠLot us +ĠLan tern +Ġfront ier +Ġp ensions +Ġj oked +ĠHard y +=-=- =-=- +r ade +U ID +Ġr ails +Ġem it +Ġsl ate +Ġsm ug +Ġsp it +ĠCall s +ĠJac obs +f eat +ĠU E +Ġrest ruct +Ġregener ation +Ġenerg ies +ĠCon nor +OH N +ĠChe ese +Ġg er +Ġresur rect +man agement +N W +Ġpres ently +ĠBru ins +M ember +ĠM ang +id an +Ġboost ing +w yn ++ . +requ isite +ĠNY PD +ĠMe gan +ĠCond itions +Ġp ics +nes ium +ĠR ash +Ġ17 4 +ĠD ucks +Ġemb ro +z u +on ian +rel igious +Ġc raz +ĠAC A +ĠZ ucker +EM A +ĠPro s +We apon +ĠKn ox +ĠAr duino +Ġst ove +Ġheaven s +ĠP urchase +Ġher d +Ġfundra iser +Dig ital +5 000 +Ġprop onents +/ âĢĭ +Ġj elly +ĠVis a +Ġmon ks +Ġadvance ment +ĠW er +Ġ18 7 +e us +ert ility +Ġfet al +Ġ19 36 +L o +Ġout fits +Ġstair case +b omb +Ġcustom ized +cl air +T ree +Ġm apped +ĠConsider ing +ĠTor res +Ġmeth yl +Ġapprox imate +Ġdo om +ĠHans en +Ġc rossover +Ġstand alone +ä ¼ +Ġinv ites +Ġgra veyard +Ġh p +Donald Trump +Ġesc ort +G ar +Ġpredec essors +Ġh ay +Ġen zyme +ĠStra ight +vis ors +I ng +ane ously +ĠApp lied +Ġf ec +ĠDur ant +Ġout spoken +or b +Ġz eal +Ġdisgr ace +' ). +ĠChe ng +28 9 +ĠRen a +ĠSu icide +29 4 +Ġout raged +ĠNew man +ĠN vidia +ĠA ber +ĠB ers +Ġrecre ation +Wind ow +ĠD P +x e +Ġped oph +Ġfall out +ambo o +Ġpresent ations +ĠApp s +Ġh tml +3 45 +ĠX XX +Ġrub bing +ĠLe ather +Ġhum idity +se ys +est ablished +ĠUn its +64 6 +Ġrespect able +A uto +Ġthri ving +ĠInn ovation +ang s +Ext ra +reg ulation +29 8 +p ick +Ex amples +ĠC J +Att ack +Ġdr acon +L T +Ġstick er +re rs +Ġsun ny +I ss +reg ulated +d im +ĠAb stract +Ġhus bands +Off ice +om ination +it ars +AN GE +asc al +ĠK ris +ĠInf antry +Ġm alf +ĠA the +ĠR ally +bal anced +................ ........ +OU P +Ġmole cule +met ics +ĠSpl it +ĠInstruct ions +ĠN ights +c ards +Ġt ug +Ġcon e +å Ń +Ġt x +ĠDisc ussion +Ġcatast rophe +pp e +g io +Ġcommun ism +Ġhal ted +ĠGu ant +cle an +ĠSc hed +ĠK anye +Ġw ander +ĠSer iously +Ġ18 8 +enn ial +f ollow +product ive +ĠFl ow +ĠS ail +Ġc raw +Ġsim ulations +or u +ang les +ĠN olan +Ġmen stru +4 70 +Ġ20 7 +aj a +Ġcas ually +board ing +Ġ2 22 +ov y +ĠN umbers +um at +O E +28 7 +ĠCle mson +Ġcert s +Ġsl id +ĠT ribe +Ġto ast +Ġfort unes +Ġf als +ĠComm ittees +Ġg p +Ġf iery +ĠN ets +ĠAn ime +Pack age +ĠComp are +l aughter +in fect +Ġatroc ities +Ġjust ices +Ġins ults +ĠVern on +Ġsh aken +Ġperson a +est amp +36 7 +br ain +Ġexperiment ing +K en +ĠElect ronics +Ġ16 1 +dom ain +Ġgraph ical +b ishop +Ġwho pping +ĠEv angel +Ġadvertis ers +ĠSpe ar +Ġb ids +Ġdestro ys +ut z +Ġunders c +ĠAD D +Ġan ts +ĠC um +ipp les +ĠF ill +Ġgl anced +Ġind icted +ĠE ff +Ġmis con +ĠDes ktop +Ġab ide +ãĥ Ģ +ĠI o +ĠC oul +Ġcaps ule +ĠCh rys +M ON +Ġund es +ĠI RA +Ġc itation +Ġdict ate +ĠNet works +ĠConf lict +ĠSt uff +x a +is ec +ĠChem istry +Ġquarter ly +William s +an an +O pt +ĠAlexand ria +out heastern +ĠSpring field +ĠBlack s +Ġge ography +24 2 +Ġut most +ĠEx xon +ab outs +E VA +ĠEn able +ĠBar r +Ġdisag reed +ĠCy prus +Ġdement ia +Ġlab s +Ġubiqu itous +ĠLO VE +Ġconsolid ated +s r +Ġcream y +ĠTim ber +Reg ardless +ĠCert ificate +Ġ" ... +ogen ous +Capt ain +Ġinsult ing +ĠSor os +ĠInst r +ĠBulgar ia +bet ter +Ġsuck ing +ĠDavid son +at z +Ġcoll ateral +g if +Ġplag ued +ĠC ancel +ĠGard ner +R B +Ġsix teen +Rem ove +ur istic +c ook +R od +Ġcompr ising +f le +) âĢĶ +ĠVik ing +g rowth +agon al +Ġsr f +af ety +m ot +N early +st own +ĠF actor +Ġautom obile +Ġproced ural +m ask +amp ires +Ġdisapp ears +j ab +3 15 +Ġ19 51 +ne eded +Ġd aring +le ader +Ġp odium +Ġun healthy +Ġm und +Ġpy ramid +oc re +Ġkiss ed +Ġdream ed +ĠFant astic +ĠG ly +å Ĭ +Ġgreat ness +Ġsp ices +Ġmet ropolitan +Ġcomp uls +i ets +101 6 +ĠSh am +ĠP yr +fl ies +ĠMid night +Ġswall owed +Ġgen res +ĠL ucky +ĠRew ards +Ġdisp atch +ĠI PA +ĠApp ly +Ġa ven +al ities +3 12 +th ings +Ġ( ). +Ġm ates +ĠS z +ĠC OP +ol ate +O FF +Ġre charge +c aps +ĠYork er +ic one +Ġgal axies +ile aks +D ave +ĠP uzz +ĠCelt ic +ĠA FC +27 6 +ĠS ons +Ġaffirm ative +H or +Ġtutorial s +ĠC ITY +ĠR osa +ĠExt ension +Ser ies +Ġf ats +Ġr ab +l is +Ġun ic +Ġe ve +ĠSp in +Ġadul thood +ty p +Ġsect arian +Ġcheck out +ĠCy cl +S ingle +Ġmart yr +Ġch illing +88 8 +ou fl +Ġ] ; +Ġcongest ion +m k +ĠWhere as +Ġ19 38 +ur rencies +er ion +Ġbo ast +ĠPat ients +Ġch ap +ĠB D +real DonaldTrump +Ġexam ines +h ov +Ġstart ling +ĠBab ylon +w id +om ew +br ance +ĠOd yssey +w ig +Ġtor ch +ĠV ox +ĠMo z +ĠT roll +ĠAn s +Similar ly +ĠF ul +00 6 +Un less +ĠAl one +st ead +ĠPub lisher +r ights +t u +ĠDoes n +Ġprofession ally +Ġcl o +ic z +Ġste als +Ġ á +19 86 +Ġst urdy +ĠJoh ann +Ġmed als +Ġfil ings +ĠFr aser +d one +Ġmult inational +Ġf eder +Ġworth less +Ġp est +Yes terday +ank ind +Ġg ays +Ġb orne +ĠP OS +Pict ure +Ġpercent ages +25 1 +r ame +Ġpot ions +AM D +ĠLeban ese +Ġr ang +ĠL SU +ong s +Ġpen insula +ĠCl ause +AL K +oh a +ĠMac Book +Ġunanim ous +Ġl enders +Ġhang s +Ġfranch ises +ore rs +ĠUp dates +Ġisol ate +and ro +S oon +Ġdisrupt ive +ĠSur ve +Ġst itches +ĠSc orp +ĠDomin ion +Ġsupp lying +Ar g +Ġtur ret +ĠL uk +Ġbr ackets +* ) +ĠRevolution ary +ĠHon est +Ġnot icing +ĠSh annon +Ġafford ed +Ġth a +ĠJan et +! -- +ĠNare ndra +ĠPl ot +H ol +se ver +e enth +Ġobst ruction +Ġ10 24 +st aff +j as +or get +sc enes +l aughs +ĠF argo +cr ime +Ġorche str +Ġde let +ili ary +rie ved +Ġmilit ar +ĠGreen e +âĹ ı +ãģ ¦ +ĠGu ards +Ġunle ashed +ĠWe ber +Ġadjust able +Ġcal iber +Ġmotiv ations +Ġà ł +m Ah +ĠL anka +hand le +Ġp ent +ĠR av +ĠAng ular +ĠK au +umb ing +Ġphil anthrop +Ġde hyd +Ġtox icity +e er +ĠY ORK +w itz +å ¼ +ĠI E +commun ity +ĠA H +Ġret ali +Ġmass ively +ĠDani els +ĠD EL +Ġcar cin +Ur l +Ġrout ing +ĠNPC s +ĠR AF +ry ce +Ġwa ived +ĠGu atem +Every body +Ġco venant +Ġ17 3 +Ġrelax ing +Ġqu art +al most +Ġguard ed +ĠSold iers +ĠPL AY +Ġout going +L AND +Ġre write +ĠM OV +ĠIm per +ĠS olution +Ġphenomen al +Ġl ongevity +Ġimp at +ĠN issan +ir ie +Ġod or +ĠZ ar +ok s +Ġmilit ias +ĠSP EC +Ġtoler ated +ars er +ĠBrad ford ++ , +Ġsur real +s f +Can adian +Ġresemb lance +Ġcarbohyd rate +VI EW +Ġaccess ory +me al +larg est +ieg el +Some one +Ġtoug hest +os o +Ġfun nel +Ġcondemn ation +lu ent +Ġw ired +ĠSun set +Jes us +ĠP ST +ĠP ages +ĠTy coon +ĠP F +Ġselect ions +Ġ ठ+part isan +Ġhigh s +ĠR une +Ġcraft s +le ad +ĠParent s +Ġre claim +ek er +ĠAll ied +ae per +Ġlo oming +Ġbenefic iaries +ĠH ull +Stud ents +Jew ish +d j +Ġp act +tem plate +ĠOffic ials +ĠBay lor +Ġhe mp +Ġyouth s +ĠLevel s +ĠX iao +ĠC hes +Ġende avor +ĠRem oved +Ġhipp ocamp +H ell +ãĤ Ĭ +80 5 +Ġd inosaur +ĠWr ath +ĠIndones ian +Ġcalcul ator +ĠD ictionary +Ġ4 20 +ĠM AG +( _ +! , +t arians +Ġrestrict ing +rac use +Ġweek day +OU NT +Ġsh rugged +leg round +Ġb ald +ĠDo ctors +Ġt outed +ĠMax well +Ġ2 14 +Ġdiplom at +Ġrep ression +Ġconstitu ency +v ice +r anked +ĠNap oleon +g ang +ĠFore ver +t un +Ġbul b +ĠPD T +ĠC isco +V EN +Ġres umed +Ste ven +ĠManit oba +Ġfab ulous +ĠAg ents +19 84 +Ġam using +ĠMyster ies +Ġor thodox +fl oor +Ġquestion naire +Ġpenet rate +Ġfilm makers +ĠUn c +Ġst amped +Ġth irteen +Ġout field +Ġforward ed +Ġapp ra +Ġa ided +t ry +Ġunf ocused +ĠL iz +ĠWend y +ĠSc ene +Ch arg +Ġreject s +Ġleft ist +ĠProv idence +ĠBr id +reg n +Ġprophe cy +ĠL IVE +4 99 +Ġfor ge +ĠF ML +Ġintrins ic +ĠF rog +Ġw ont +ĠH olt +Ġfam ed +CL US +aeper nick +ĠH ate +ĠC ay +Ġregister ing +ort ality +rop y +ocaly ptic +a an +n av +Ġfasc ist +IF IED +Ġimpl icated +ĠRes ort +ĠChand ler +ĠBr ick +P in +ys c +Us age +ĠHel m +us ra +âĺħ âĺħ +ĠAb bas +Ġunanim ously +Ġke eper +Ġadd icted +?? ? +Ġhelm ets +Ġant ioxid +aps ed +80 8 +gi ene +Ġwa its +Ġmin ion +ra ved +ĠP orsche +Ġdream ing +Ġ17 1 +ĠC ain +Ġun for +ass o +ĠConfig uration +k un +hard t +Ġn ested +ĠL DS +L ES +Ġt ying +en os +Ġc ue +ĠMar qu +sk irts +Ġclick ed +Ġexp iration +ĠAccording ly +ĠW C +Ġbless ings +Ġaddict ive +ĠN arr +y x +ĠJagu ars +Ġrent s +ĠS iber +Ġt ipped +ous se +ĠFitz gerald +Ġhier arch +out ine +Ġwa velength +> . +ch id +ĠProcess ing +/ + +r anking +E asy +ĠConst ruct +Ġt et +ins ured +H UD +Ġqu oting +Ġcommun icated +in x +Ġin mate +Ġerect ed +ĠAbs olutely +ĠSure ly +Ġun im +ĠThr one +he id +Ġcl aws +Ġsuper star +ĠL enn +ĠWh is +U k +ab ol +Ġsk et +ĠN iet +Ġper ks +Ġaff inity +Ġopen ings +phas is +Ġdiscrim inate +T ip +v c +Ġgr inding +ĠJenn y +Ġast hma +hol es +ĠHom er +Ġreg isters +ĠGl ad +Ġcre ations +Ġlith ium +Ġappl ause +unt il +Just ice +ĠTur ks +Ġsc andals +Ġb ake +t ank +M ech +ĠMe ans +ĠM aid +Republic ans +is al +wind ows +ĠSant os +Ġveget ation +33 8 +t ri +Ġfl ux +ins ert +Ġclar ified +Ġmort g +ĠCh im +ĠT ort +Ġdiscl aim +met al +ĠAs ide +Ġindu ction +Ġinf l +Ġathe ists +amp h +Ġe ther +ĠV ital +ĠBu ilt +M ind +Ġweapon ry +S ET +Ġ18 6 +ad min +g am +cont ract +af a +Ġderiv atives +Ġsn acks +Ġch urn +E conom +Ġca pped +ĠUnder standing +ĠH ers +ĠI z +Ġd uct +I ENT +augh ty +Ġâľ Ķ +ĠN P +Ġsa iling +In itialized +Ġt ed +Ġreact ors +ĠL omb +Ġcho ke +ĠW orm +Ġadm iration +Ġsw ung +ens ibly +Ġr ash +ĠGo als +ĠImport ant +Sh ot +ĠR as +Ġtrain ers +ĠB un +Work ing +Ġhar med +ĠPand ora +ĠL TE +Ġmush room +ĠCH AR +ĠF ee +ĠM oy +B orn +ol iberal +ĠMart ial +Ġgentle men +Ġling ering +Offic ial +Ġgra ffiti +ĠN ames +D er +Ġqu int +ist rate +aze era +ĠNOT ICE +ĠFlore nce +Ġpay able +Ġdep icts +ĠSpe cies +He art +âĶĢâĶĢâĶĢâĶĢ âĶĢâĶĢâĶĢâĶĢ +Ġencl osed +Incre ases +D aily +ĠL is +Ġenact ment +ĠB acon +ĠSt eele +dem and +Ġ18 3 +Ġmouth s +Ġstr anded +Ġenhance ment +01 1 +ĠWh ats +Ġhe aled +en y +ĠR ab +Ġ3 40 +ĠLab yrinth +ro ach +ĠY osh +ĠCl ippers +Ġconcert s +Intern et +35 5 +Ġstick ers +Ġter med +ĠAx e +Ġgrand parents +Fr ance +ĠCl im +ĠU h +ul ic +Ġthr ill +cent ric +ĠOver view +ĠCond uct +Ġsubstant ive +Ġ18 2 +m ur +Ġstr ay +ĠCo ff +Ġrep etitive +ĠFor gotten +Ġqual ification +ew itness +ĠZ imbabwe +Ġsim ulated +ĠJ D +25 3 +ĠW are +Ġun sc +T imes +Ġsum mons +Ġdis connected +Ġ18 4 +ci us +ĠGu jar +od ka +Ġer ase +ĠTob acco +elect ed +Ġun cont +ĠShe pard +ĠL amp +Ġalert ed +Ġoper ative +arn a +u int +Ġneglig ence +ac ements +Ġsup ra +Ġprev ail +ĠSh ark +Ġbel ts +ãģ « +Ġt ighter +Engine ers +Ġin active +Ġexp onent +ĠWill ie +a ples +Ġhe ir +ĠH its +ian n +ĠS ays +Ġcurrent s +ĠBeng al +Ġar ist +B uffer +Ġbree ze +ĠWes ley +Col a +Ġpron oun +Ġde ed +ĠK ling +Ġof t +Ġinf lict +Ġpun ishing +Ġn m +ik u +OD UCT +01 4 +Ġsubsid y +ĠDE A +ĠHer bert +ĠJ al +B ank +Ġdef erred +Ġship ment +B ott +Ġal le +b earing +HT ML +Off line +Ġ2 13 +Ġscroll ing +Ġsc anned +ĠLib yan +ĠT OP +ch rom +d t +col umn +Psy NetMessage +Z ero +Ġtor so +0 50 +âķ IJ +Ġimp erson +ĠSchw artz +ud ic +Ġpiss ed +ĠS app +25 7 +ĠIS Ps +og l +Ġsuper vised +Ġad olescent +Ġatt ained +ĠDel ivery +ĠB unny +Ġ19 37 +Ġmini ature +Ġo s +Ġ3 70 +60 8 +ĠMour inho +Ġinn ate +Ġtem po +ĠN M +ĠFall en +00 9 +Ġprov ocative +Stream er +ĠBened ict +ĠBol she +Ġt urtle +ĠPC B +ĠEqu al +Direct or +ĠR end +Ġflu ids +Author ities +Ġcous ins +requ ency +ĠNeigh bor +s ets +sh ared +Char les +pass word +Ġg ears +Ġ2 11 +ĠHard ware +ri ka +Ġup stream +H om +Ġdisproportion ately +iv ities +Ġund efined +Ġelect rons +Ġcommem or +Event ually +Ġ> < +Ġir responsible +2 18 +ĠRe leased +ĠO VER +ĠI GN +ĠB read +st ellar +ĠS age +tt ed +dam age +ed ition +ĠPre c +Ġl ime +Ġconf inement +Ġcal orie +we apon +Ġdiff ering +ĠS ina +m ys +am d +Ġintric ate +k k +ĠP AT +ã o +st ones +lin ks +Ġr anch +Sem itic +Ġdifferent iate +ĠS inger +occup ied +Ġfort ress +c md +Ġinter ception +ĠAnk ara +Ġre pt +ĠSol itaire +Ġrem ake +p red +Ġd ared +aut ions +ĠB ACK +Run ning +Ġdebug ging +Ġgraph s +3 99 +ĠNig el +Ġb un +Ġpill ow +Ġprog ressed +fashion ed +Ġob edience +ER N +Ġrehe ars +C ell +t l +S her +Ġher ald +ĠPay ment +ĠC ory +ĠDe pt +Ġrep ent +ĠWe ak +uck land +Ġple asing +Ġshort ages +Ġjur ors +ĠK ab +q qa +Ant i +Ġw ow +ĠRC MP +Ġt sun +ĠS ic +Ġcomp rises +Ġsp ies +Ġprec inct +n u +Ġur ges +Ġtim ed +Ġstrip es +ĠB oots +Ġy en +Adv anced +Ġdisc rete +ĠArch angel +employ ment +D iff +Ġmon uments +Ġ20 9 +work er +Ġ19 6 +ĠI g +utter stock +T PS +J ac +Ġhomeless ness +Ġcomment ator +Ġrac ially +f ing +se ed +E le +ell ation +Ġeth anol +Ġpar ish +ĠD ong +ĠAw akening +Ġdev iation +ĠB earing +ĠTsu k +Ġrec ess +Ġl ymph +ĠCann abis +å ľ +ĠNEW S +Ġd ra +ĠStef an +ĠWr ong +ĠS AM +Ġloose ly +Ġinterpre ter +ĠPl ain +Go vernment +Ġbigot ry +Ġgren ades +ave z +pict ured +Ġmand ated +ĠMon k +ĠPed ro +Ġl ava +27 4 +Ġcyn ical +ĠScroll s +l ocks +M p +Ġcon gregation +orn ings +ph il +ĠI bid +Ġf erv +Ġdisapp earing +Ġarrog ant +sy n +ĠMa ver +ĠSu it +24 1 +Ġab bre +ack ers +P a +ĠY el +Whe never +Ġ23 5 +ĠV ine +ĠAn at +Ġext inct +LE T +Ġexecut able +V ERS +ox ide +D NA +ĠP rel +Ġresent ment +Ġcompr ise +ĠAv iv +Ġinter ceptions +Ġprol ific +IN A +ĠEr in +though t +2 19 +ĠPsychiat ry +un ky +chem ist +H o +ĠMcC oy +Ġbr icks +L os +ri ly +ĠUS SR +Ġr ud +Ġl aud +ĠW ise +ĠEmer ald +Ġrev ived +Ġdam ned +ĠRep air +id em +ct ica +Ġpatri arch +ĠN urs +me g +Ġcheap est +re ements +empt y +ĠCele br +Ġdepri vation +ch anted +ĠTh umbnails +E nergy +ĠEth an +ĠQ ing +Ġopp oses +W IND +v ik +ĠM au +ĠS UB +66 7 +G RE +ĠVol unte +nt on +C ook +å IJ +es que +Ġplum met +Ġsu ing +Ġpron ounce +Ġresist ing +ĠF ishing +ĠTri als +Ġy ell +Ġ3 10 +Ġin duct +Ġpersonal ized +oft en +R eb +EM BER +Ġview point +Ġexist ential +() ) +rem ove +MENT S +l asses +Ġev apor +Ġa isle +met a +Ġreflect ive +Ġentit lement +Ġdev ised +mus ic +asc ade +Ġwind ing +off set +Ġaccess ibility +ke red +Bet ter +ĠJohn ston +th inking +S now +ĠCroat ia +ĠAt omic +27 1 +34 8 +Ġtext book +ĠSix th +Ġ اÙĦ +Ġsl ider +ĠBur ger +b ol +S ync +Ġgrand children +Ġc erv ++ ) +Ġe ternity +Ġtweet ing +Ġspec ulative +Ġpiv otal +ĠW P +ĠT ER +ynam ic +Ġu pl +ĠC ats +per haps +Ġclass mates +Ġblat ant +' - +Ġl akh +ant ine +ĠB org +i om +/ ( +ĠAthlet ic +Ġs ar +OT A +ĠHoff man +Never theless +Ġad orable +Ġspawn ed +Ass ociated +ĠDom estic +Ġimpl ant +ĠLux em +ĠK ens +Ġp umps +ĠS AT +Att ributes +50 9 +av our +Ġcentral ized +ĠT N +Ġfresh ly +ĠA chieve +Ġouts iders +her ty +ĠRe e +ĠT owers +ĠD art +ak able +Ġm p +ĠHeaven ly +Ġr ipe +ĠCarol ine +ry an +Ġclass ics +Ġret iring +Ġ2 28 +Ġa h +Ġdeal ings +Ġpunch ing +ĠChap man +O ptions +max well +vol ume +Ġst al +Ġex ported +ĠQu ite +Ġnumer ical +B urn +F act +ĠKey stone +Ġtrend ing +Ġalter ing +ĠAfric ans +47 8 +ĠM N +ĠKn ock +Ġtempt ation +Ġprest ige +Over view +ĠTrad itional +ĠBah rain +Priv ate +ĠH OU +Ġbar r +ĠT at +C ube +US D +ĠGrand e +ĠG at +ĠFl o +Ġres ides +Ġind ec +vol ent +Ġperpet ual +ub es +Ġworld view +ĠQuant um +Ġfil tered +Ġen su +orget own +ERS ON +ĠM ild +37 9 +OT T +à ¥ +Ġvit amins +Ġrib bon +Ġsincere ly +ĠH in +Ġeight een +Ġcontradict ory +Ġgl aring +Ġexpect ancy +Ġcons pir +Ġmon strous +Ġ3 80 +re ci +Ġhand ic +Ġpump ed +Ġindic ative +Ġr app +Ġav ail +ĠLEG O +ĠMar ijuana +19 85 +ert on +Ġtwent ieth +################ ################ +ĠSw amp +Ġval uation +Ġaffili ates +adjust ed +ĠFac ility +26 2 +Ġenz ymes +itud inal +Ġimp rint +S ite +Ġinstall er +ĠT RA +m ology +lin ear +ĠCollect ive +ig ating +ĠT oken +Ġspec ulated +K N +ĠC ly +or ity +Ġdef er +Ġinspect ors +appro ved +R M +ĠSun s +Ġinform ing +ĠSy racuse +ib li +7 65 +Ġgl ove +Ġauthor ize +âĢ¦âĢ¦âĢ¦âĢ¦ âĢ¦âĢ¦âĢ¦âĢ¦ +ĠCru ise +Ġcontract ing +she ll +IF E +ĠJew el +p ract +ĠPhot oshop +ĠKnow ing +h arm +Ġattract ions +ad an +et us +01 8 +w agen +Al t +Ġmultip ly +Ġequ ilibrium +: { +ĠF ighters +ĠEd gar +Ġfour teen +Go vern +Ġmis use +Ġab using +Ġancest ry +ram er +64 4 +Ġwor ms +Ġthick er +ĠComb ine +Ġpeas ants +Ġv ind +Ġcon quest +Ġm ocked +Ġc innamon +ĠC ald +ĠGall up +Ġavoid ance +Ġincarn ation +ĠStr at +Ġt asted +ent a +ĠN eal +p ared +Ġtermin ology +ject ion +Scient ists +ĠIN S +ĠDe e +Ġdirect ories +R oad +ĠSh ap +br ight +ĠDirect ors +ĠCol umn +Ġb ob +Ġprefer ably +Ġgl itch +f urt +Ġe g +id is +C BC +Ġsur rendered +Ġtest ament +33 6 +ug gest +ĠN il +an other +Ġpat hetic +ĠDon na +Ġ2 18 +ĠA very +Ġwhis key +Ġf ixture +ĠCon quest +Ġbet s +O cc +ĠLe icester +] ." +Ġ) ); +Ġfl ashes +45 6 +Ġmask ed +ge bra +Ġcomput ed +che l +aud er +Ġdefe ats +ĠLiber ation +ĠOs ama +ĠV ive +Ch anges +Ch annel +Ġtar iffs +Ġm age +ĠS ax +Ġinadvert ently +ĠC RE +ĠRe aper +ink y +gr ading +Ġstere otyp +Ġcur l +ĠF ANT +Ġfram eworks +M om +ĠAn ch +Ġflav our +car bon +Ġperm itting +let cher +ĠMo zilla +ĠPark ing +ĠCh amp +Sc roll +Ġmurd erer +Ġrest ed +Ġow es +ĠP oss +AD D +IF F +res olution +ĠMin ing +Ġcompar ative +D im +Ġneighbour ing +ĠA ST +ĠT oxic +Ġbi ases +Ġgun fire +ur ous +ĠMom ent +19 83 +Ġper vasive +tt p +ĠNorm ally +r ir +S arah +ĠAlb any +Ġun sett +ĠS MS +ip ers +l ayer +ĠWh ites +up le +Ġtur bo +ĠLe eds +Ġthat s +ĠMin er +M ER +ĠRe ign +Ġper me +ĠBl itz +Ġ19 34 +Ġintimid ating +t ube +Ġecc entric +ab olic +box es +ĠAssoci ates +v otes +Ġsim ulate +um bo +aster y +Ġship ments +FF FF +an th +Ġseason ed +Ġexperiment ation +âĸ ł +law s +Me et +idd les +ant ics +R ating +IS IS +h ift +Ġfront s +b uf +01 7 +Ġun att +ĠD il +le ases +ĠGard ens +77 7 +t ouch +ve ll +45 8 +Ġ= ==== +s aving +Ġer osion +ĠQu in +Ġearn s +Ġaccomplish ment +ĠWe i +Ġ< [ +____ _ +Ġir rig +ĠT eddy +Ġconqu ered +ĠArm ored +Ġassert s +Ġmanip ulating +r é +Ġtranscript s +G allery +Ġplot ting +Ne il +Ġbetray al +load er +ĠS ul +Ġdispl acement +Ġroy alty +ĠW I +he it +ĠDev ices +alle l +Ġmunicipal ities +Ġcan al +St ars +ĠU AE +Ġ" âĢ¦ +ĠC U +ab ove +Ġreson ance +ĠguiActive Un +add ed +ĠBra ves +ĠI bn +Ġhere by +ĠB RE +Ġshare holder +ĠH ir +ĠJ i +Ġstrange ly +Ġadm ired +Ġpl ight +Ġb achelor +ĠP ole +cipl inary +T ony +ĠArmen ian +Ġun man +ĠZion ist +St age +isco ver +Ġautom otive +Ġs idelines +Ġsl ick +ĠRena issance +ĠF UN +Im ages +ĠH aj +Ġp ing +Ġshort cut +ĠBl vd +ĠLook s +Ġbur sts +Ġcl amp +Ġm ish +Ġsort ing +Ġpatri ot +Ġcorrect ness +ĠScand inav +ĠCaval iers +p ython +az ar +Ġ3 75 +ĠJa une +40 9 +Ġdetrim ental +Ġstab bing +Ġpoison ed +Ġf ountain +oc ent +or st +ĠMar i +Ġr ains +ĠO vers +ĠInst itution +ud get +AM Y +t ale +ĠK R +ĠPr ices +Ġhead aches +Ġlands l +ĠA ura +Bon us +ĠZ hao +ĠH ip +Ġhop s +ĠKurd istan +Ġexplo iting +ry n +Ġhypocr isy +op ening +Ġgun shot +Ġw ed +inter stitial +Inter stitial +Ġam en +Bre aking +Ġmarket ed +W ire +ĠC rowd +Contin ue +ĠK nown +ĠEffect ive +ore an +iz ons +Jose ph +Ġescal ation +us ername +Ġcur tain +AT ES +ĠP AR +ĠM iy +Ġcounter fe +l ene +Ġcont enders +d aily +ĠAs c +ĠPhill ip +most ly +Ġfil ename +he ne +Ġresemb ling +Ġst aging +ĠCh loe +Ġw iring +H on +ĠRen ew +ott age +ĠHy brid +m uch +Ġstro kes +Ġpolicy makers +AP TER +ĠArk ham +pl ot +Ġassist ants +Ġde port +ĠSe ga +Ġinflu enza +ĠC ursed +ĠK obe +Ġskin ny +Prov ider +ĠR ip +Ġincrement al +product s +B F +Ġd ome +ĠC redits +Ġlos ers +int s +ĠBet ty +ĠTal ent +ĠD AM +L v +E ss +Ġd ens +tem p +J udge +od ic +Ġ' ( +UR ES +ets k +V O +Ġretrie ved +Ġarchitect s +Ù ĩ +Ġeth ic +ĠSecond ary +st ocks +ad ia +Ġ3 25 +ĠOp inion +Ġsimultane ous +Ġd izz +ul p +Ġsmugg ling +ipp ery +R andom +f acing +ĠD as +Ġstock p +Ġdiscl osures +po inter +Ġcor al +ĠSe lection +ĠP ike +ival ent +Ġruth less +ĠR im +Ġensu ing +ĠExper iment +Ġcongress man +Ġbelie ver +Ġun specified +ĠM ord +Ġknowledge able +ĠV ERY +T X +Ġstra ps +Ġtur f +apesh ifter +Ġmar ital +Ġfl ock +ãģ Ĩ +26 3 +AM ES +ĠOpp osition +Ġtre asures +ĠG OD +Ġmodel ed +ĠWOR LD +Ġ( [ +ĠUs age +H F +Ġ$ ( +uss ed +Ġpione er +E ight +par se +b read +rit z +ĠMir anda +ĠK ant +++ ) +ore n +Ġprov oked +Ġbre eds +ĠIn cludes +ĠPast ebin +ĠFl ip +J ava +Ġbr ink +Ġrum ored +Ġun seen +Ġgar nered +ĠDef in +al ted +Ġtatt oos +Ġhes itation +is itions +ĠWe aver +ĠReport ing +Ġtherap ies +Ġconsult ants +Ġresid ual +ĠMal i +ĠRom a +i ago +ĠRes idents +ub i +Ġremed ies +Ġadapt ive +ĠAl ive +ĠBar cl +Ġwal lets +c rypt +etermin ation +ĠPel osi +Ġsl ipping +oton in +Ġall iances +pat rick +ir is +Ġor th +ĠPer kins +ĠDe V +ĠG ets +Ġdry ing +ge e +fore st +ĠFor get +ore m +33 9 +Ġvague ly +ĠD ion +ĠP orn +ĠH OW +Ġp neum +Ġrub ble +ĠT aste +enc ia +ĠG el +Ġd st +Ġ24 5 +ĠMoroc co +inf lamm +ĠTw ins +Ġb ots +d aughter +ĠB alk +Ġbre thren +Ġlog os +Ġgo bl +f ps +Ġsub division +Ġp awn +Ġsquee zed +Ġmor ale +ĠD W +' " +Ġkn ot +ook y +Ġdiv isive +Ġboost ed +ch y +ãĥ IJ +if act +Ġnewcom ers +ĠWrest ling +Ġsc outs +w olves +R at +Ġnin eteenth +ĠOs borne +St ats +Ġem powered +Ġpsych opath +ĠO EM +ugg age +ĠP K +ĠMoh ammad +P ak +Ġanarch ists +ĠExt ract +est hes +ĠStock holm +l oo +ĠG raph +Ġdeploy ing +ĠStr anger +ĠM old +Ġstaff er +Ġdiscount ed +uck le +ple ase +ĠLand ing +ÃŃ a +Ġ19 3 +Ġan te +Ġrep etition +Ġ+ /- +Ġpar ody +Ġlive ly +AA A +ĠHor us +Ġp its +ind ers +L OC +ĠVen ice +40 6 +ĠDis cover +â Ĩ +ellect ual +Ġp ens +Ġey el +ig uous +Im pl +Ġj oking +Ġinv al +ĠBel fast +Ġcredit ors +ĠSky walker +ov sky +Ġcease fire +Ġse als +is oft +) ). +ĠFel ix +IT S +Ġt resp +ĠBlock chain +ew are +ĠSch war +en ne +mount ed +ĠBe acon +les h +Ġimmense ly +Ġche ering +Em ploy +sc ene +ish ly +atche wan +ĠNic olas +Ġdr ained +ĠEx it +ĠAz erb +j un +Ġflo ated +u ania +De ep +Ġsuper v +Ġmyst ical +ĠD ollar +ĠApost le +ĠR EL +ĠProv ided +ĠB ucks +ãĥ ´ +cut ting +Ġenhance ments +ĠPengu ins +ĠIsa iah +Ġj erk +ĠW yn +Ġst alled +Ġcryptoc urrencies +ĠR oland +sing le +Ġl umin +ĠF ellow +ĠCap acity +ĠKaz akh +W N +Ġfin anced +38 9 +Ġt id +Ġcoll usion +ĠMy r +î Ģ +Sen ator +Ġped iatric +Ġneat ly +Ġsandwic hes +ĠArchitect ure +Ġt ucked +Ġbalcon y +Ġearthqu akes +qu ire +F uture +Ġhe fty +é Ĺ +Ġspecial izes +Ġstress es +Ġs ender +Ġmisunder standing +Ġep ile +Ġprov oke +ĠCol ors +Ġdis may +uk o +[ _ +58 6 +ne utral +Ġdon ating +ĠRand all +Mult i +Ġconvenient ly +ĠS ung +ĠC oca +Ġt ents +ĠAc celer +Ġpart nered +27 2 +ir ming +ĠB AS +s ometimes +Ġobject ed +ub ric +p osed +LC S +gr ass +Ġattribut able +V IS +Israel i +Ġrepe ats +ĠR M +v ag +ut a +in ous +Ġin ert +ĠMig uel +æ Ń +ĠHawai ian +B oard +Ġart ific +ĠAzerb ai +as io +ĠR ent +A IN +Ġappl iances +Ġnational ity +Ġass hole +ĠN eb +Ġnot ch +h ani +ĠBr ide +Av ailability +Ġintercept ed +Ġcontin ental +Ġsw elling +ĠPers pect +b ies +. < +ith metic +ĠL ara +Ġtempt ing +add r +Ġoversee ing +cl ad +ĠD V +ĠGing rich +Ġm un +ĠApp ropri +Ġalter ations +ĠPat reon +Ġha voc +Ġdiscipl ines +Ġnotor iously +aku ya +ier i +? ). +ĠW ent +Ġsil icon +Ġtre mb +Cont ainer +K nown +Ġmort ar +est e +ick a +Ar thur +ĠPre viously +ĠMart y +Ġsp arse +g ins +Ġin ward +ĠParticip ant +C opy +ĠM isc +Ġantib iotic +ĠRet ro +Ġel usive +Ġass ail +ĠBatt alion +ĠB ought +Ġdimin ish +ĠEuro pa +s ession +ĠDanger ous +ies el +Ġdisbel ief +Ġbl asts +ext reme +ĠBoy d +ĠProject s +ĠGu ys +Ġunder gone +Ġgr ill +ĠDw ight +Ġ19 7 +US ER +Ġfiles ystem +Ġcl ocks +T aylor +Ġwra pper +Ġfold ing +ous and +ĠPhilipp ine +ATION AL +ĠPer th +Ġas hes +Ġaccum ulate +ĠGate way +Sh op +orks hire +H an +ĠBar rel +ĠLe h +ĠX V +Ġwh im +Ġrep o +ĠC G +ĠM am +Ġincorpor ating +Ġbail out +Ġlingu istic +Ġdis integ +C LE +Ġcinem atic +ĠF iber +S yn +il ion +ĠCom pos +c hens +Ġne oc +Ġbo iled +F INE +on o +un cle +ik en +ĠB M +Î ¹ +Ġreceipt s +Ġdisp osed +ĠTh irty +ĠR ough +ĠA BS +Ġnot withstanding +oll en +# $ +Ġunrel iable +Ġbl oom +Ġmedi ocre +Ġtr am +ĠTas man +Ġsh akes +Ġmanifest o +ĠM W +Ġsatisf actory +Ġsh ores +Ġcomput ation +Ġassert ions +orm ons +ar ag +ab it +Dem ocrats +ĠL oot +ĠVol ks +ha ired +Ġgrav itational +S ing +ĠM iz +Ġthro ttle +Ġtyr anny +ĠView s +Ġrob ber +ĠMinor ity +Ġsh rine +sc ope +pur pose +Ġnucle us +our cing +ĠUS DA +ĠD HS +w ra +ĠBow ie +Sc ale +ĠB EL +x i +I ter +Ġ( ), +w right +Ġsail ors +ous ed +NAS A +ĠPro of +ĠMin eral +t oken +ĠF D +R ew +Ġe ll +6 30 +Ġchance llor +ĠG os +Ġamount ed +ĠRec re +ome z +ĠOpt im +ĠOl ive +Ġtrack er +ow ler +ĠUn ique +R oot +Ġmar itime +ĠQur an +ĠAd apt +Ġecosystem s +ĠRe peat +ĠS oy +ĠI MP +Ġgrad uating +and em +P ur +ĠRes et +ĠTr ick +ĠPh illy +ĠT ue +ĠMalays ian +Ġclim ax +Ġb ury +Ġcons pic +ĠSouth ampton +ĠFl owers +Ġesc orted +ĠEduc ational +ĠI RC +Ġbrut ally +e ating +Ġpill ar +ĠS ang +ĠJ ude +ar ling +ĠAm nesty +Ġrem inding +ĠAdminist rative +hes da +Ġfl ashed +ĠP BS +per ate +fe ature +Ġsw ipe +Ġgra ves +oult ry +26 1 +bre aks +ĠGu er +Ġsh rimp +ĠV oting +qu ist +Ġanaly tical +Ġtables poons +ĠS OU +Ġresear ched +Ġdisrupt ed +Ġj our +Ġrepl ica +Ġcart oons +b ians +} ) +c opy +G ot +ou ched +P UT +Ġsw arm +not ations +s aid +Ġreb uilt +Ġcollabor ate +Ġr aging +Ġn ar +Ġdem ographics +ĠD DR +Ġdist rust +oss ier +ĠK ro +Ġpump kin +Ġreg rets +Ġfatal ities +ĠL ens +ĠO le +p d +Ġpupp et +ĠOut look +ĠSt am +O l +F air +U U +Ġre written +Ä ± +Ġfasc inated +Ġve ctors +Ġtrib unal +u ay +ĠM ats +ĠCo ins +[ [ +Ġ18 1 +Ġrend ers +ĠK aepernick +Ġesp ionage +Ġsum m +Ġd itch +Acc ount +Ġspread sheet +Ġmut ant +p ast +40 7 +Ġd ye +Ġinit iation +Ġ4 000 +Ġpunish able +Ġth inner +ĠKh al +Ġinter medi +D un +ĠGoth am +Ġeager ly +Ġvag inal +p owers +V W +ĠWATCH ED +Ġpred ator +ams ung +Ġdispar ity +Ġ[ * +Ġam ph +Ġout skirts +ĠSpir its +Ġskelet al +Ð » +ĠR ear +Ġissu ance +ĠLog ic +re leased +Z Z +ĠB ound +Ent ry +Ġex its +is ol +ĠFound er +Ġw re +ĠGreen land +ĠM MO +t aker +IN C +ãģ ¾ +Ġhour ly +hen ko +Ġfantas ies +Ġdis ob +Ġdemol ition +ãĥ ĭ +Ġen listed +rat ulations +Ġmis guided +Ġens ured +Ġdiscour aged +m ort +Ġfl ank +Ġc ess +Ġreact s +ĠS ere +s ensitive +ĠSer pent +ass ad +Ġ24 7 +Ġcalm ly +b usters +Ġble ed +ĠSt ro +Ġamuse ment +ĠAntar ctica +Ġs cept +ĠG aw +a q +ason ic +Ġsp rawling +n ative +atur ated +ĠBattle field +IV ERS +E B +ĠG ems +ĠNorth western +ĠFil ms +ĠAut omatic +Ġappre hend +ãģ ¨ +Ġgui Name +Ġback end +Ġevid enced +ge ant +01 2 +ĠS iege +Ġexternal To +Ġunfocused Range +ĠguiActiveUn focused +Ġgui Icon +ĠexternalTo EVA +ĠexternalToEVA Only +F ri +ch ard +en aries +Ġchief s +Ġc f +ĠH UD +Ġcorro bor +Ġd B +ĠT aken +ĠPat ricia +ra il +ĠCh arm +ĠLiber tarian +rie ve +Person al +ĠO UR +ger ies +Ġdump ing +Ġneurolog ical +it imate +ĠClint ons +raft ed +ĠM olly +Ġtermin als +reg ister +Ġfl are +Ġenc oded +Ġautop sy +p el +m achine +Ġexempt ions +ĠRoy als +d istance +Ġdraft s +Ġl ame +ĠC unning +Ġsp ouses +ĠMark ets +ĠCar rier +Ġimp lying +ĠY ak +s id +Ġl oser +Ġvigil ant +Ġimpe achment +Ġaug mented +ĠEmploy ees +Ġunint ended +tern ally +ĠW att +Ġrecogn izable +ess im +æ Ŀ +Ġco ated +r ha +Ġlie utenant +ĠLegisl ation +pub lished +44 4 +01 3 +Ġide ally +ĠPass word +Ġsimpl ify +ĠMet a +ĠM RI +Ġple ading +organ ized +hand ler +Ġun ravel +cor rect +Ġ icy +Ġparan oid +Ġpass er +Ġinspect ions +of er +ĠHealth care +28 3 +ĠBr ut +iol a +for ge +ĠMed ieval +MS N +ie vers +ĠProgram ming +å ī +Ġ2 23 +m u +ĠC LE +ug a +Ġsho ppers +Ġinform ative +ĠPl ans +Ġsupplement ation +ĠT ests +ty ard +ocy tes +ĠVeg a +ĠGujar at +erman ent +Ex cept +ĠL OT +all a +ĠC umm +ĠO sw +Ġven om +ĠDeb t +ĠD OWN +Ġreun ion +Ġm uc +ĠRel ief +Ġge op +ĠðŁ ĺ +al ogue +An th +ech o +Ġcor ros +Ġrepl ication +ĠBl azing +ĠD aughter +Ġinf lic +ĠLind sey +Ù Ī +28 4 +Ex it +Ġgl oom +TA IN +Ġundermin ing +Ġadv ising +h idden +Ġover flow +Ġg or +urd ue +Ġe choes +enh agen +Ġimp uls +d rug +c ash +Ġas ync +Ġmir ac +at ts +p unk +Ġpiv ot +ĠLegisl ative +Ġblog gers +ĠCl aw +s burg +d yl +ĠRecomm end +Ġver te +Ġprohib iting +ĠPant her +Jon athan +Ġo min +Ġhate ful +28 1 +ĠOr che +ĠMurd och +down s +Ġas ymm +G ER +Al ways +Ġinform s +ĠW M +ĠP ony +ĠApp endix +ĠAr lington +J am +Ġmedic inal +ĠS lam +IT IES +Ġre aff +ĠR i +F G +S pring +b ool +Ġthigh s +Ġmark ings +ĠRa qqa +ĠL ak +p oll +ts ky +ĠMort y +ĠDef inition +Ġdeb unk +end ered +ĠLe one +a vers +Ġmortg ages +App arently +N ic +ha us +ĠTh ousands +au ld +Ġm ash +sh oot +Ġdi arr +Ġconscious ly +H ero +e as +ĠN aturally +ĠDestroy er +Ġdash board +serv ices +R og +Ġmillenn ials +Ġinv ade +- ( +Ġcomm issions +ĠA uckland +Ġbroadcast s +Ġfront al +Ġcr ank +ĠHist oric +Ġrum ours +CT V +Ġster il +Ġboost er +rock et +ãĤ ¼ +ut sche +ĠP I +Ġ2 33 +ĠProdu cer +ĠAnaly tics +Ġinval uable +Ġunint ention +ĠC Y +Ġscrut in +Ġg igg +Ġeng ulf +Ġprolet ariat +Ġh acks +ĠH ew +ar ak +ĠSl ime +ield ing +ag her +ĠEll iot +Ġtele com +Ġ2 19 +ult an +ĠAr bor +ĠSc outs +B an +Ġlifes pan +Ġbl asp +38 8 +Ġjud iciary +ĠContin ental +ask ing +Mc C +L ED +Ġbag gage +ĠSorce rer +Ġrem nants +ĠGriff ith +ets u +ĠSub aru +ĠPerson ality +des igned +ush ima +agn ar +Ġrec oil +Ġpass ions +\ ": +Ġte e +Ġabol ition +ĠCreat ing +j ac +Ġ19 4 +01 9 +Ġpill ars +ric hed +/ " +t k +Ġlive lihood +Ġro asted +ah on +ĠH utch +ass ert +Ġdivid end +Ġkn it +Ġd aunting +Ġdisturb ance +Ġsh ale +Ġcultiv ated +Ġrefriger ator +L B +ĠN ET +Ġcommercial s +Ġthink ers +45 5 +Ġch op +B road +Ġsuspic ions +Ġtag ged +l ifting +Ġsty lish +ĠShield s +Short ly +Ġt ails +A uth +ST E +ĠG AME +Ġse ism +ĠK is +olog ne +Ġcow ork +Ġforc ibly +Ġthy roid +ĠP B +AN E +mar ried +h orse +Ġpoly mer +ĠCh al +od or +DE BUG +ĠCon text +Ġbl iss +Ġpin point +ĠMat hemat +leg ram +ĠWeek end +Ġlab elled +Ġb art +it les +Ġest rogen +âĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶ âĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶ +" ' +Ġvis ibly +Ġouts ider +aid a +Are a +Ġdisse min +Ġdish onest +ĠCl osed +ĠBullet in +ĠRam sey +sw ord +ĠX I +our ced +S ame +34 6 +ĠRe pe +ĠK ou +c ake +em is +C ache +ĠMe aning +ĠEn light +onom y +Ġmanifest ation +sw orth +J ay +Ġch ore +ö r +D ream +Ġsanction ed +Ġcult urally +ĠA ra +N av +Ġthe ological +Ġstr ut +ĠV O +ĠHand book +Ġconstruct ing +Ġ ¶ +ĠBenef its +ĠPsych ological +s ac +å ¸ +p olicy +ĠMat ters +ĠReport ed +ĠBy te +Ġvit ro +ĠM aiden +Ġl am +ĠJenn ings +Ġgar ment +ĠRut gers +ĠStaff ord +ĠWell ington +Ġinter mitt +Ġn pm +Ġord eal +Ġplug ged +o oming +in ished +fram ework +Ġtim ber +Ġc ass +Ġ8 50 +il ess +ĠRed ux +7 68 +St re +Ġsurpass ed +w hel +Ġparalle ls +Ġve il +ĠG I +ĠR EST +Ġread iness +s ort +Ġmod ifying +ĠSl ate +ru ff +Ġmar ble +Ġinf rared +Ġaud itor +ĠFANT ASY +ĠP overty +ĠS PD +Ġ" ( +K y +RA Y +Ġexecut ions +ĠBever ly +ĠMarx ism +ĠBur st +ĠK ali +est ones +Clear ly +E ll +ãģ § +ĠProceed ings +T oken +IF IC +ñ a +Cent ral +ĠH aley +ĠD rama +Ġform ations +OR N +Book s +Ġdom inating +ĠFly ers +ĠCompan ion +Ġdiscipl ined +ĠYug oslav +ĠSpell s +Ġv engeance +Ġland lords +L en +ĠO gre +ano ia +Ġpier cing +Ġcon greg +Ġscore r +ob ia +Ġnic kel +ĠLear ns +Ġre jo +Ġmaster piece +Fl ash +Ġinhab ited +ĠOpen GL +ĠD ud +ĠI CO +Ġar ter +Ġpl ur +Ġmaster y +Ġlong standing +st ed +Ġw ines +Ġtelev ised +ĠSh rine +ĠBay ern +Ġâ ĵĺ +Ġencl osure +j ohn +Ġprophe ts +ĠRes urrection +ĠOrd ers +Ġun even +r als +Ġd wind +ĠL ah +ĠSl oven +37 8 +Ġins istence +aff le +ĠCl one +Ġhard ship +ĠCongress man +Ġple ad +Ġreview ers +Ġc ured +Ġ19 35 +as ley +f ake +ĠTh inking +yd ia +P ART +ĠD ota +o it +Ġwh ipped +Ġb ouncing +ĠHispan ics +com ings +Ġcann abin +ĠCh ambers +ĠZ ack +Option al +Ġco ats +Ġprow ess +ĠNort on +Ġplain ly +Ġfre ight +Ġinhib ition +Ġcl am +Ġ30 3 +ke f +ale igh +L uke +Ġpsych o +ator ium +M ED +Ġtreat ies +Ġind isc +Ġd c +OP S +Ġresil ient +ĠInter state +Ġsl ack +Ġmund ane +Ġestab lishes +35 9 +Ġstr ained +Ġn ond +S us +Ġcast e +ar ate +ie ving +Ġunfair ly +Ġpars er +on ial +urs ive +V ia +ĠOtt o +ĠAuthor ities +stro ke +K R +ĠMer cy +Ġfurn ished +Ġout set +Ġmet ic +19 82 +olith ic +ĠT ent +og ical +ĠA ircraft +Ġh ides +ĠBec ame +Ġeduc ators +re aching +Ġvol atility +Ġtodd ler +ĠNAS CAR +ĠTw elve +ĠHigh lights +Ġgra pe +Ġspl its +Ġpe asant +Ġre neg +ĠMS I +Tem p +st ars +Ġtre k +ĠHy de +b inding +Ġreal ism +Ġox ide +ĠH os +Ġmount s +Ġbit ing +Ġcollaps ing +Ġpost al +Ġmuse ums +Ġdet ached +Ġrespect ing +Ġmonop ol +Ġwork flow +ĠC ake +Tem plate +ĠOrgan isation +Ġpers istence +36 9 +C oming +B rad +Ġredund ant +ĠG TA +Ġb ending +Ġrev oked +Ġoff ending +Ġfram ing +Ġprint f +Comm un +mem bers +Out side +Ġconst rued +Ġc oded +F ORE +Ġch ast +Ch at +Ind ian +ĠY ard +? !" +ĠP orts +ĠX avier +ĠR ET +' ." +ĠBo at +iv ated +ich t +umer able +D s +ĠDun n +Ġcoff in +Ġsecure ly +ĠRapt ors +ĠB es +Install ation +Ġin ception +ĠHealth y +end ants +Ġpsych ologists +ĠShe ikh +c ultural +ĠBlack Berry +sh ift +F red +oc he +Ġc akes +ĠS EO +ĠG ian +ĠAs ians +og ging +e lement +Ġpund its +ĠV augh +ĠG avin +Ġh itter +Ġdrown ed +Ġch alk +ĠZ ika +Ġmeas les +80 2 +âĢ¦ .. +ĠAW S +] " +Ġdist ort +ĠM ast +Ġantib odies +ĠM ash +Mem ory +ĠUg anda +ĠPro b +Ġvom iting +ĠTurn s +Ġoccup ying +Ġev asion +ĠTher apy +Ġprom o +Ġelect r +Ġblue print +ĠD re +pr iced +ĠDep ot +Ġallev iate +ĠSom ali +m arg +n ine +Ġnostalg ia +ĠShe pherd +Ġcaval ry +Ġtor ped +ĠBlood y +x b +Ġs ank +Ġgo alt +report print +embed reportprint +clone embedreportprint +ĠIn itially +ĠF ischer +Ġnot eworthy +c ern +Ġin efficient +raw download +rawdownload cloneembedreportprint +c ation +ĠD ynasty +l ag +D ES +Ġdistinct ly +ĠEston ia +Ġopen ness +Ġg ossip +ru ck +W idth +ĠIb rahim +Ġpet roleum +Ġav atar +ĠH ed +ath a +ĠHog warts +Ġc aves +67 8 +Ġsafegu ard +ĠM og +iss on +ĠDur ham +sl aught +ĠGrad uate +Ġsub conscious +ĠEx cellent +ĠD um +---- - +Ġp iles +ĠW ORK +ĠG arn +ĠF ol +ĠAT M +Ġavoid s +ĠT ul +Ġble ak +EL Y +iv ist +light ly +P ers +ĠD ob +ĠL S +Ġins anity +Î µ +atal ie +En large +Ġtw ists +Ġfault y +Ġpir acy +Ġimp over +Ġrug ged +ĠF ashion +Ġs ands +' ? +sw ick +Ġn atives +Ġhe n +ĠNo ise +ãĥ Ĺ +Ġg reens +Ġfree zer +Ġd ynasty +ĠFather s +ĠNew ark +Ġarchae ological +Ġo t +ob ar +Ġblock ade +Ġall erg +L V +Ġdeb it +ĠR FC +ĠMil ton +ĠPress ure +Ġwill ingly +Ġdisproportion ate +Ġopp ressive +Ġdiamond s +Ġbelong ings +19 70 +Ġbell s +Ġimperial ism +Ġ2 27 +Ġexpl oding +ĠE clipse +Ġ19 19 +Ġr ant +Ġnom inations +34 7 +Ġpeace fully +ric a +ĠF UCK +Ġvib ration +mal ink +Ġro pes +ĠIv anka +ĠBrew ery +ĠBook er +ĠOw ens +go ers +Serv ices +ĠSn ape +Ġ19 1 +39 5 +Ġ2 99 +just ice +Ġb ri +Ġdisc s +Ġprom inently +Ġvul gar +Ġsk ipping +l ves +Ġtsun ami +37 4 +ĠU rug +ĠE id +rec ated +p hen +Ġfault s +ĠStart ed +9 50 +Ġp i +Ġdetect or +Ġbast ard +Ġvalid ated +Space Engineers +OUR CE +Ġ( ~ +Ġuns ur +Ġaff irmed +Ġfasc ism +Ġres olving +ĠCh avez +ĠC yn +Ġdet ract +L ost +Ġrig ged +Ġhom age +ĠBrun o +55 5 +ec a +Ġpress es +Ġhum our +Ġsp acing +Ġ' / +olk ien +C oun +OP ER +T re +S on +ĠCambod ia +ier re +m ong +o zy +Ġliquid ity +ĠSov iets +ĠFernand o +Ġ2 29 +Ġsl ug +ĠCatal an +elect ric +Ġsc enery +ĠH earth +Ġconst rained +Ġgoal ie +ĠGu idelines +ĠAm mo +ĠPear son +Ġtax ed +Ġfet us +Resp onse +ĠAlex is +th ia +G uy +Ġrecon struct +Ġextrem es +Ġconclud ing +ĠP eg +ook s +Ġded uctions +R ose +Ġground breaking +ĠT arg +ãĥ ģ +ĠRe ve +res ource +Ġmo ons +Ġelectrom agnetic +Ġamid st +ĠVik tor +N ESS +B ACK +Ġcomm ute +ĠAna heim +Ġfluct uations +6 40 +Ġnood les +ĠCop enhagen +ĠT ide +ĠGri zz +ĠS EE +Ġpip elines +Ġsc ars +end o +ag us +ĠE TF +/ # +ĠBec ome +44 8 +Ġvis c +ĠRecomm ended +Ġj umper +Ġcogn ition +Ġassass in +Ġwitness ing +ĠSet up +Ġl ac +v im +IS M +p ages +SS L +35 8 +Ġad ject +indust rial +l ore +cher y +Ġgl itter +Ġc alf +Flor ida +Ġspoil ers +Ġsucceed s +Ġch anting +Ġslog ans +ĠTr acy +Vis it +rol ogy +Ġm ornings +Ġline age +Ġs ip +Ġintense ly +Ġflour ish +ĠSle eping +ĠF em +or por +ĠK lan +ĠDar th +h ack +ĠNi elsen +Ġtum ors +Ġprocure ment +ĠY orkshire +Ġra ided +K Y +An na +Ġ// [ +ĠDis order +ĠMust ang +ĠW en +ĠTry ing +s q +Ġdeliver ies +Ġshut ter +Ġcere bral +Ġbip olar +ĠC N +l ass +j et +Ġdeb ating +> : +Ġe agle +gr ades +ĠD ixon +UG C +M AS +ĠDr aco +ĠMach ines +aff er +Ġem an + ² +pr on +ĠG ym +Ġcompar atively +ĠTrib unal +PR O +Ġle x +Ġfert ile +Ġdep ressing +Ġsuperf icial +ess ential +ĠHun ters +g p +Ġprom inence +L iber +ĠAn cest +ote chnology +Ġm ocking +ĠTra ff +ĸ ļ +Med ium +I raq +Ġpsychiat rist +Quant ity +ĠL ect +Ġno isy +5 20 +G Y +Ġsl apped +ĠM TV +Ġpar a +p ull +Mult iple +as her +Ġn our +ĠSe g +Spe ll +v ous +ord ial +Sen ior +ĠGold berg +ĠPl asma +ne ed +Ġmess enger +ere t +Ġteam ed +Ġliter acy +ĠLe ah +ĠD oyle +Ġem itted +U X +Ġev ade +Ġm aze +Ġwrong ly +ĠL ars +Ġstere otype +Ġpled ges +Ġarom a +ĠM ET +Ġac re +ĠO D +Ġf f +Ġbrew eries +ĠH ilton +und le +ĠK ak +ĠThank fully +ĠCan ucks +in ctions +ĠApp ears +Ġco er +Ġundermin ed +ro vers +And re +Ġbl aze +um ers +Ġfam ine +amp hetamine +ulk an +Am ount +Ġdesper ation +wik ipedia +develop ment +ĠCor inth +uss ia +Jack son +L I +N ative +R s +Oh io +ĠKath leen +F ortunately +Ġattend ant +ĠPre ferred +ĠDid n +ĠV s +M is +Ġrespond ent +Ġb oun +st able +Ġp aved +Ġunex pl +ĠChe ney +L M +ĠC ull +bl own +Ġconfront ing +oc ese +serv ing +W i +ĠLith uania +ann i +Ġst alk +h d +Ġv ener +AP H +ynchron ous +UR R +um ably +hist oric +H alf +H ay +Ġresil ience +spe ction +Ġabandon ing +O bs +ĠDeb bie +Ġgrad ient +ĠPl aint +ĠCan al +AR CH +Ġexpans ive +Ġfun g +Ġb ounced +U nd +Ġprec autions +Ġclar ification +Ġd agger +Ġgri ps +Ġ µ +ĠRiver a +ĠUnd ead +is ites +ĠFIR ST +ñ o +aud i +Ġhost ages +Ġcompl iant +Ġal umni +Se ven +Ġcyber security +e ither +Col lect +Ġinvari ably +ĠS oci +Ġlaw maker +Ġa le +ĠPerson ally +N azi +Ġcustom ization +ĠPro c +ĠSask atchewan +eat uring +Ġsp ared +Ġdiscontin ued +Ġcomput ational +ĠMotor ola +Ġsuprem acist +government al +Ġparad ise +ĠDown ing +ĠNik on +Ġcat alyst +ber ra +Tor onto +8 75 +bet a +ĠMac ron +Ġunreal istic +ve ctor +ĠVeh icles +it iveness +ĠR V +ĠCol bert +s in +o ji +ent in +ĠKr ish +hell o +ff ield +ok y +ĠT ate +Ġmap le +Ġa ids +chem ical +33 4 +n uts +ĠWar p +Ġx x +ĠRob b +umer ous +_- _ +ft ime +ĠV W +Ġw inger +ĠD ome +t ools +ĠP V +ĠGe orgetown +Ġg eared +Ġjihad ists +Ġc p +Ġster oids +M other +cler osis +ĠDR M +nes ia +Ġl inger +Ġimm ersive +ĠC OUN +Ġoutwe igh +ens ual +B and +Ġtransform s +mat ched +ps ons +ĠJud icial +f actor +Ġrefer ral +Ġodd ly +ĠW enger +B ring +ĠB ows +60 2 +IC LE +Ġl ions +ĠAcad emic +ĠTh orn +ĠRa ider +kef eller +St orage +L ower +ĠOr t +ĠEqu ality +AL T +ĠS OC +T ypes +Ġl yn +ĠAss et +co at +TP P +C VE +ĠPione er +app lication +Mod ern +ĠH K +En vironment +Al right +R ain +IP P +ĠShi ite +Ġm ound +ĠAb ilities +cond ition +St aff +Ġcompet ence +ĠM oor +ĠDi ablo +Ġwith held +Ġost ensibly +ĠB rom +Ġms g +Ġden omin +ĠRef erences +ĠF P +Ġplun ged +Ġp amph +m oving +cent ral +Ġdown right +Ġf ading +T al +T yp +ĠTh y +uk es +it he +Ġo ve +Ġbatt led +Ġseaf ood +Ġfig ur +ĠR D +c rop +Ġsqu ads +{ \ +à ¹ +ĠE h +Ġinterview ing +ĠQ in +Ġas piring +PL IC +Ġcla uses +ĠG ast +ĠN ir +Ġl uggage +Ġh ose +Ġsystem d +Ġdesc ending +ĠRev ised +ĠR ails +al ign +70 9 +33 7 +Ġf ug +charg ing +t ags +Ġut er +k ish +WAR NING +49 0 +prof its +Ġvoy age +Ġa ce +ĠV anguard +ĠT anks +ĠM uk +Ġ2 26 +S afe +Ar mor +Ġvolcan ic +Ġwom b +ĠM IL +Ġbegin ner +ĠRec ogn +ĠA AP +PL AY +) ! +Ġdetect ing +c n +Ġbre aches +Bas ically +ĠP ag +ĠMunicip al +ĠInd ie +ĠL af +ĠDis able +ĠOl son +Ġrest rained +Ġrul ings +Ġhum ane +ev ents +ĠCinem a +display Text +ĠH atch +action Date +onna issance +Ġassault ing +ĠL ug +CH AT +Ġvig orous +ĠPer se +Ġintoler ance +ĠSnap chat +ĠSh arks +Ġd ummy +ĠDi agn +ĠGu itar +im eters +40 3 +RE G +A x +Ġsepar ates +ĠMah m +Ġt v +j ah +O OL +C irc +ĠWinds or +uss ian +Ġintu ition +Ġdis dain +ĠDon ovan +Ġ2 21 +E mb +Ġcondem ning +Ġgener osity +zz y +Ġpant ies +ĠPre vent +Action Code +AN A +34 2 +external ActionCode +Ġspec ifying +Ġcryst all +J ere +Ġru pt +ĠApp rentice +Ġprof iling +Ð º +St rike +Ġsid eline +Ġoblig ated +Ġocc ult +Ġbureaucr atic +ant ically +rupt ed +neg ative +ĠEthiop ia +ĠC ivic +Ġins iders +el igible +ĠTV s +ĠB AR +ĠT I +i ologist +ĠA IR +Ġsubstit uted +Ar ab +ĠS aul +ĠY og +p rem +Ġbuild ers +Ġstation ary +Ġdoubt ful +Ġvig orously +Ġthr illing +Ph ysical +ĠCare y +ĠHyd ra +geon ing +ĠS ly +y ton +Ġborrow ers +ĠPark inson +Ġ ë +ĠJama ica +Ġsat ir +Ġinsurg ents +ĠF irm +Ġis ot +ĠK arn +our ning +ak ens +doc s +l ittle +ĠMon aco +CL ASS +Tur key +L y +ĠCon an +ass ic +Ġstar red +ĠPac ers +et ies +Ġt ipping +M oon +ĠR w +s ame +Ġcav ity +Ġgo of +ĠZ o +Sh ock +um mer +Ġemphas izes +Ġreg rett +Ġnovel ty +Ġen vy +ĠPass ive +r w +50 5 +Ġind ifferent +ĠR ica +ĠHim self +ĠFred die +Ġad ip +ä¸ Ģ +Ġbreak out +Ġhur ried +ĠHu ang +ĠD isk +Ġro aming +?????- ?????- +U V +ĠRick y +ĠS igma +Ġmarginal ized +Ġed its +Ġ30 4 +mem ory +Ġspec imen +29 3 +ãģ ¯ +Ġvert ically +Ġaud ition +ĠHe ck +Ġc aster +ĠHold ings +ad al +ĠC ron +ĠL iam +Ġdef lect +P ick +ĠDeb ug +RE F +Ġvers atility +ot hes +class ified +ĠMah ar +ĠH ort +C ounter +st asy +not iced +33 1 +ĠSh im +f uck +ĠB ie +Ġair ing +ĠPro tein +ĠHold ing +Ġspect ators +ili ated +ĠThat cher +n osis +ãĥ¼ ãĥ³ +Te le +B oston +ĠTem pl +st ay +Ġdecl arations +47 9 +Vol ume +ĠDesign er +ĠOver watch +id ae +Ġon wards +Ġn ets +ĠMan ila +part icularly +Ġpolit ic +o other +Ġport raits +Ġpave ment +c ffff +Ġs aints +Ġbegin ners +ES PN +Ġshort comings +âķIJ âķIJ +Ġcom et +ĠOrgan ic +qu el +Ġhospital ized +Bre ak +Ġpe el +dyl ib +asp x +ur ances +ĠT IM +P g +Ġread able +ĠMal ik +Ġm uzzle +Ġbench marks +d al +ĠV acc +ĠH icks +60 9 +ĠB iblical +he ng +Ġover load +ĠCivil ization +Ġimm oral +Ġf ries +ãĤ Ĵ +Ġreprodu ced +Ġform ulation +j ug +ire z +g ear +Ġco ached +Mp Server +ĠS J +ĠK w +In it +d eal +ĠO ro +ĠL oki +ĠSong s +Ġ23 2 +ĠLou ise +asion ally +Ġunc ond +olly wood +Ġprogress ives +ĠEn ough +ĠDo e +Ġwreck age +Ġbr ushed +ĠBase Type +Ġz oning +ish able +het ically +ĠC aucus +ĠH ue +Ġk arma +ĠSport ing +Ġtrad er +Ġseem ing +ĠCapt ure +4 30 +b ish +Ġt unes +Ġindo ors +ĠSp here +ĠD ancing +TER N +Ġno b +ĠG ST +m aps +Ġpe ppers +F it +Ġoverse es +ĠRabb i +ĠR uler +vert ising +off ice +xx x +Ġra ft +Ch anged +Ġtext books +L inks +ĠO mn +ãĢ ij +Ġinconven ience +ĠDon etsk += ~ +Ġimplicit ly +Ġboost s +ĠB ones +ĠBo om +Cour tesy +Ġsens ational +AN Y +Ġgre edy +ed en +Ġinex per +ĠL er +ĠV ale +Ġtight en +ĠE AR +ĠN um +Ġancest or +S ent +ĠH orde +urg ical +all ah +Ġsa p +amb a +ĠSp read +tw itch +Ġgrand son +Ġfract ure +Ġmoder ator +ĠSe venth +ĠRe verse +Ġestim ation +Cho ose +Ġpar ach +Ġbar ric +ãĢ IJ +Ġcomp ass +Ġall ergic +âĢ ķ +OT HER +err illa +Ġw agon +Ġz inc +Ġrub bed +ĠFull er +ĠLuxem bourg +ĠHoo ver +Ġli ar +ĠEven ing +ĠCob b +est eem +Ġselect or +ĠB rawl +is ance +ĠE k +Ġtro op +Ġg uts +ĠApp eal +ĠTibet an +Ġrout ines +ĠM ent +Ġsummar ized +steam apps +Ġtr anqu +Ġ19 29 +or an +ĠAut hent +Ġg maxwell +Ġappre hens +Ġpo ems +Ġsa usage +ĠWeb ster +ur us +Ġthem ed +Ġl ounge +Ġcharg er +Sp oiler +Ġsp illed +h og +ĠSu nder +ĠA in +ĠAng ry +Ġdis qual +ĠFrequ ency +ĠEther net +Ġhel per +Per cent +Ġhorr ifying +Ġa il +ĠAll an +EE E +ĠCross ing +44 9 +Ġh olog +ĠPuzz les +ĠGo es +eren n +60 4 +ãģ ı +ĠRaf ael +Ġatt en +ĠE manuel +Ġup ro +ĠSus p +P sych +ĠTr ainer +ĠN ES +ĠHun ts +bec ue +Ġcounsel or +R ule +Ġtox ins +Ġb anners +r ifice +Ġgreet ing +Ġfren zy +Ġall ocate +Ġ* ) +ex pr +50 3 +ĠCh ick +ĠT orn +Ġconsolid ation +ĠF letcher +sw itch +fr ac +cl ips +ĠMcK in +ĠLun ar +Mon th +IT CH +Ġscholar ly +rap ed +39 8 +Ġ19 10 +Ġe greg +Ġin secure +Ġvict orious +cffff cc +Ġsing led +Ġel ves +ĠW ond +bur st +Ġcam oufl +ĠBL ACK +Ġcondition ed +ç ī +ans wered +Ġcompuls ory +asc ist +Ġpodcast s +ĠFrank furt +bn b +Ġne oliberal +ĠKey board +ĠBel le +w arm +Ġtrust s +Ġins ured +ĠBu cc +us able +60 7 +ĠPl ains +Ġ18 90 +Ġsabot age +Ġlod ged +f elt +Ġg a +ĠN arc +ĠSal em +Ġsevent y +ĠBl ank +p ocket +Ġwhis per +Ġm ating +om ics +ĠSal man +ĠK ad +Ġan gered +Ġcoll isions +Ġextraord inarily +Ġcoerc ion +G host +b irds +è Ģ +k ok +Ġper missible +avor able +Ġpo inters +Ġdiss ip +ac i +Ġtheat rical +ĠCos mic +Ġforget ting +Ġfinal ized +å¤ § +y out +l ibrary +Ġbo oming +ĠBel ieve +ĠTe acher +ĠL iv +ĠGOOD MAN +ĠDomin ican +OR ED +ĠPart ies +Ġprecip itation +ĠSl ot +R oy +ĠComb ined +Ġinteg rating +Ġch rome +Ġintest inal +ĠRe bell +Ġmatch ups +Ġblock buster +ĠLore n +ĠLe vy +Ġpre aching +ĠS ending +ĠPur pose +ra x +f if +Ġauthor itative +ĠP ET +ast ical +Ġdish on +Ġchat ting +Ġ"$ :/ +Connect ion +Ġrecre ate +Ġdel inqu +Ġbro th +ĠD irty +ĠAd min +z man +Ġscholars hips +Ġ25 3 +cont act +als a +7 67 +c reen +abb age +Ġ19 15 +Ġbl ended +Ġal armed +L anguage +35 6 +Ġbl ends +ĠCh anged +W olf +Ġhe pat +Creat ing +Ġper secut +Ġsweet ness +art e +Ġforfe iture +ĠRober to +im pro +N FL +ĠMag net +Det ailed +Ġinsign ificant +ĠPOL IT +ĠBB Q +ĠC PS +Ġse aw +amin er +m L +end if +f inals +Ġ26 5 +u ish +Ġ} ) +ĠPro blems +Ġem blem +Ġserious ness +Ġpars ing +Ġsubst itution +Ġpress ured +Ġrecy cled +ale b +Rub y +Ġprof iciency +Dri ver +ĠW ester +: ' +AF TA +Ġm antle +ĠClay ton +fl ag +Ġpractition er +c overed +ĠSt ruct +add afi +4 25 +ĠTown ship +ĠHyd ro +Lou is +34 3 +Ġcond o +ĠT ao +Ġutil ization +Ġnause a +ĠDem s +rid ges +p ause +Ġform ulas +Ġchall enger +37 6 +Ġdefect ive +ĠRail way +ĠPub Med +Ġyog urt +l bs +ĠNor folk +OP E +ĠMood y +Ġdistribut or +Ġscroll s +Ġextract s +St an +Ġv iability +Ġexp oses +Ġstar vation +ĠStep s +ĠD odd +f ew +ST D +33 2 +Ġclos ures +Ġcomplement ary +ĠS asha +ump y +Ġmon et +Ġartic ulate +ĠDo ct +k iller +Ġsc rim +Ġ2 64 +Ġprost itutes +Ġse vered +Ġattach ments +Ġcool ed +L ev +ĠF alk +f ail +Ġpolic eman +ĠD ag +Ġpray ed +ĠK ernel +Ġcl ut +Ġc ath +Ġan omaly +St orm +em aker +ĠBreak fast +ul i +o ire +J J +h z +Oper ation +ĠS ick +35 4 +ĠGuatem ala +R ate +Ġexp osures +f aces +ĠArch ae +ra f +ĠM ia +Ġ20 25 +Ġop aque +Ġdisgu ised +ĠHead quarters +S ah +Ġp ots +9 78 +ĠM alf +Ġfrown ed +Ġpoison ous +ĠCon vers +ee ks +Ġcr ab +." " +Ġtre ason +Ġr anc +Ġescal ating +Ġwar r +Ġmob s +Ġl amps +ĠSun shine +ĠBrun swick +Ph ones +Ġspe lled +ĠSk ip +Ġ20 50 +Ġ19 11 +ĠPl uto +ĠAm end +Ġme ats +38 7 +Ġst omp +ĠZh ou +ĠLevi athan +ĠHaz ard +ad v +ĠOr well +Ġal oud +Ġb umper +ĠAn arch +ub untu +ĠSer ious +f itting +ĠOption al +ĠCec il +RE AM +Ġser otonin +Ġcultiv ate +ag ogue +} \ +Ġmos ques +ĠSun ny +Ġre active +rev olution +ĠL up +ĠFed ora +Ġdefense man +ĠV ID +ist ine +Ġdrown ing +ĠBroad casting +Ġthr iller +ĠS cy +Ġacceler ating +Ġdirect s +od ied +b ike +d uration +Ġpain fully +R edd +Ġproduct ions +Ġg ag +Ġwh ist +Ġs ock +Ġinf initely +ĠConc ern +ĠCit adel +Ġlie u +Ġcand les +ogene ous +arg er +Ġheaven ly +inflamm atory +Per formance +C s +ruct ose +az aki +Ġp essim +Ġinf erence +Ġpow d +ĠZ oe +Ġpain ts +Ġd azz +pt a +-------- --- +Ġins pir +ĠExper imental +ĠKn ife +reg or +b ors +Ġshow ers +rom eda +Ġs aint +Ġben ign +ĠJ iang +Ġenvision ed +Ġsh roud +IF T +H O +Ġsh uff +ĠI CC +Ġse greg +Ġrevis it +ighth ouse +L i +Ġsub strate +ĠSe as +ĠRew ard +ĠH ep +ĠBr ass +s bm +Ġelim inates +Ġst amina +ĠV AT +ĠLo an +Ġconst raint +Ġappropri ated +Ġp es +ĠA LE +r anging +Ġ40 4 +39 2 +Ġintellectual s +ach u +Ġrestruct uring +ĠLe vin +Ġrun es +Ġdelight ful +Ġcarbohyd rates +ĠMod els +ĠExp o +Ġtransport ing +all oc +Ġring ing +S amsung +Ġscarce ly +ĠURL s +ĠM AS +Ġprot otypes +Ġnarr ator +ĠCPU s +cd n +ĠBart on +Ġdecided ly +ĠSh u +ix ir +oc ious +ĠMy st +N intendo +Ġre use +Ġforg iven +F ew +in ical +n at +Ġseam less +ĠEv a +ĠE VE +ĠJ O +land ers +Ġso fter +neg ie +Ġtrans ient +Ġorb ital +Ġfulf il +ĠK om +Hop efully +Ġdynam ically +ĠHun ger +å Ľ +ĠArmen ia +el man +ber to +Ġp ige +ĠID s +lim it +Ġve ins +Ġso aring +p acks +Gold en +ĠCr ab +ist or +ĠR PM +Ġ$ $ +g ression +Ġjihad ist +Ġgam ble +Ġcare g +Ġinf lated +F ace +ĠFire arms +ĠEm manuel +â Ŀ +Ġsh ocks +gr ab +Ġspl end +ĠHP V +ab ortion +Ab ove +Ent ity +play ers +Ġcomm enced +ul ence +Ġfulfill ment +Ġembod iments +ĠW elfare +Ġha il +Ġ< @ +tt en +Ġcat cher +ĠJ azeera +Ġvolcan o +Ġstabil ize +ĠHand ler +Ġintens ified +ĠAb rams +Ġhum iliation +p aced +60 5 +ĠCent OS +Spe cific +Ġhe ed +ĠC AM +ĠGal ile +D ie +Ġabol ished +ĠThom son +ĠTe achers +ĠW ass +j ong +ĠIS BN +ĠAll ies +sh ake +å · +v ict +How ard +Ġde em +Ġexceed ingly +ĠSmart stocks +ib e +Ġdoor way +Ġcompet ed +ig mat +Ġnational ists +Ġg room +ĠKe en +Ġdispos able +de cl +ĠT olkien +ĠSche me +Ġb iod +Ġav id +ĠEl on +ag ar +ĠT SA +R oman +Ġartific ially +Ġadvis ors +X L +ĠInf erno +36 6 +Ġted ious +ĠPhot ography +ĠCar rie +Ġtro pe +ĠSand ra +Ġdec imal +Que en +ĠGund am +ĠO M +ote ch +N BA +Ġ19 32 +Ġent renched +ĠMar ion +Ġfr aternity +Lab our +Hen ry +Ġlat itude +E ither +Ġenh ances +ĠPot ential +Ġsh ines +id ad +Ġbread th +Ġcapac ities +ĠðŁ ĻĤ +ĠBron x +Ġsex es +Ġdifferent iation +Ġheavy weight +ĠT aj +d ra +Ġmigr ate +Ġexhaust ion +ĠR UN +els ius +ĠCu omo +Ġgu itars +Ġcl ones +ĠSom ew +ĠP ry +------------ - +Ġwarr anted +cy cles +Ġsalv age +Ġdis ks +R ANT +ĠNGO s +ĠMart ian +":[ {" +Ġadd icts +oj ure +il let +Ġamazing ly +art ments +p ixel +ĠGPU s +Lay out +è £ +ĠTam il +ĠBas il +Ġimpart ial +ĠSt ructure +f ork +b ryce +Ġr idge +ĠHamb urg +ri ous +Ġbl itz +cig arettes +Ġcan ned +40 2 +Ġiron ically +Ġcompassion ate +ĠHaw kins +. # +ĠCat hedral +Ġrall ied +in ternal +Ġqu ota +st akes +T EXT +m om +Ġcomple tes +Ġ23 8 +Ġsh rug +ãĥ ij +ĠN inth +Ġrev ise +ĠProv ider +Ġtre acher +Ġqu asi +ĠPR ES +Ġdep osition +Ġconfidential ity +iss ors +Ġim balance +Ġspan ning +Ġang ular +ĠC ul +commun ication +ĠNor a +ĠGen ius +op ter +Ġs acked +Sp ot +Ġfine ly +ĠCH R +28 2 +w aves +Pal est +ĠRo hing +N L +è ¿ +Ġsh itty +ĠSc alia +4 75 +Pro gress +Ġreferen cing +Ġclass rooms +ab ee +Ġs od +hes ion +70 8 +ĠZucker berg +ĠFin ish +ĠScot ia +ĠSav ior +ĠInstall ation +an tha +( - +Ġ30 2 +ĠP unk +Ġcr ater +yout u +Ġro ast +Ġinflu encing +Ġd up +ĠJ R +ĠG rav +Ġstat ure +Ġbath rooms +A side +W iki +me an +ĠZ ak +ĠOn es +ĠN ath +Ġhyper t +Ġcommence ment +C ivil +Ġmoder ately +Ġdistribut ors +Ġbreast feeding +Ġ9 80 +ĠS ik +ĠC ig +ĠAM ER +R IP +ĠCare er +ust ing +Ġmess ed +Ġe h +ĠJ ensen +/ $ +Ġblack mail +Ġconvers ions +Ġscientific ally +Ġmant ra +p aying +Ġiv ory +ĠCour ts +OU GH +aunt let +Ser ial +B row +ĠH undreds +3 23 +Ġpe e +Ġlin ux +Ġsub mer +ĠPrinc ipal +48 5 +ĠD SL +ĠCous ins +Ġdoctr ines +ĠAthlet ics +Ġ3 15 +ĠK arma +Ġatt ent +ur ger +Ġpresc ribe +Ġenc aps +ĠC ame +Ġsecret ive +ĠCr imes +d n +C lean +ĠEgypt ians +ĠCar penter +Ġ ll +H um +ĠMil o +Ġcapital ists +Ġbrief ed +T we +ĠBas in +elve t +M os +Ġplun ge +ĠKa iser +ĠFu j +ill in +Ġsafegu ards +Ġo ste +ĠOpportun ity +ĠM afia +ĠCall ing +ap a +ur ban +br ush +ill ard +c é +int elligence +ĠL ob +ĠDru id +Ġsm oother +Ġfoot ing +Ġmotor ists +arc ity +Ġmascul inity +Ġm ism +Ġabdom inal +ĠTa vern +ĠR oh +Ġesc apes +s igned +Anth ony +Ġsacrific ing +Ġintim acy +Ġan terior +ĠK od +Ġmot if +Ġg raz +Ġvisual ization +Ġguitar ist +ĠTro tsky +m agic +D ar +ĠMor i +Ġw ards +Ġtoile ts +l est +Ġtele port +ĠSund ays +ĠPl at +ET S +Ġe Sports +Pat rick +ĠK atherine +en ko +Ġhas sle +ĠM ick +gg les +Ġh ob +aint ain +Ġair borne +Ġsp ans +Ġch ili +Ġa perture +Ġvolunte ered +ĠInc ident +ĠF res +ĠVeter an +augh tered +ing o +Ġun insured +CL OSE +Ġf use +Ġer otic +Ġadvert ise +ra ising +Text ure +Ġatt ends +ĠRE AL +udd led +Ġsm oot +Ġ30 5 +ĠWill is +Ġbl ond +An alysis +ĠV T +on ica +Ġstrongh old +R F +N M +. >> +Ġprosper ous +Ġbo asted +29 2 +ĠManufact uring +PR ESS +g ren +Ġpharm acy +ĠRoc kefeller +k ai +Ġth umbs +ĠH ut +Ġmother board +Ġguard ians +ĠAl ter +ll ular +Ġsh ack +Ġwise ly +Ġback bone +erv a +Ġsu icides +ĠMcG regor +ij ah +E mer +ĠB rav +Ġdesign ate +P OST +produ ced +Ġcleans ing +irl wind +ex istent +ĠHum ph +ĠPay ne +Ġv ested +Å ¡ +Ġstring ent +ion a +Ġuns ub +Ġsum med +ĠHer cules +sub ject +ĠR agnar +ĠN os +Ġcharacter ization +Ġsav vy +ĠDaw son +ĠCas ino +Ġf ri +ĠBar rier +Ġmis information +Ġins ulation +Ġcorrid ors +Ġair planes +ĠNo ct +ah i +Ġ19 16 +k b +arm ac +Ġsh un +Ġsche ma +Ġhorr ified +Ġ23 9 +aund ers +N B +i ates +er ity +ĠSh ard +Ġr arity +Ġgroup ed +ĠGh ana +again st +ĠBi ological +ĠA ware +ow ell +Ï Ħ +ĠBe au +sh aw +H ack +ĠJul ius +US S +ol son +aun a +c ru +ĠMaur ice +ĠI k +Ġsequ encing +Ġradical s +Ġ( ?, +v irtual +Ġany ways +Ġreper c +Ġhand lers +Ġhes itant +é ĥ +ĠM F +ple mentation +ass ociated +Ġcampaign ed +ĠY ue +ut ations +ĠY oga +Ġsim mer +Ġro ds +Ġmel ody +Ġconv oy +v ideos +Ġscreen ed +N eg +ochem ical +Ġ( )) +Ġultr as +Ġant ip +ĠIsland ers +70 4 +Ġfet ish +Ġridic ulously +ĠK art +Ġmitochond rial +Ġinterf ering +Build er +Ġover fl +Ġac ne +ĠM ud +ĠK err +f lex +ĠPost al +ĠBalt ic +47 7 +ĠPers ons +our age +H B +ĠM use +ĠImm ortal +ĠDri ving +Ġpet itions +Ġsubsc ript +Ġs orce +ĠProcess or +ut on +S ony +Ġph on +Ġr aced +ĠAnth rop +Ġday time +ĠEx ercise +Add ing +Ġeng ages +ĠQual comm +Ġmir acles +Ġmem es +ĠDr ink +ĠOri oles +Ġhair s +ĠPol ar +ath om +Ġsl ippery +ĠR emy +Ġcar amel +ĠY EAR +Ġal k +I gn +a ution +ĠMer lin +ĠC ran +Ġap ologies +Ġ4 10 +Ġout ing +ĠMem ories +app ointed +Ġcount ered +u ld +pos ing +Ġfire wall +ĠW ast +ĠW et +work ed +se ller +Ġrepe aled +ere o +ass uming +BL IC +m ite +ĠCEO s +ĠChap el +ellig ent +________________ ________ +D og +Ġw art +Ġsubsc riber +s ports +Ġbe gged +ĠM V +Ġsem if +eth ical +Ġpre ach +Ġrev ital +Ġpun itive +Ġshort cuts +Ġinstit uted +ĠWars aw +Ġabdom en +ĠK ING +Ġsuper intendent +Ġf ry +ĠGe o +T OR +Ġcontrad ictions +apt ic +Ġlandsc apes +b ugs +Ġcl ust +Ġvol ley +c ribed +Ġt andem +Ġrob es +WH AT +Ġpromot er +Ġel oqu +review ed +ĠD K +ĠPl ato +Ġf ps +T ank +ĠDer rick +Ġpriorit ize +as per +ĠHond uras +ĠCom pleted +ne c +Ġm og +n ir +ĠMay o +DE F +st all +in ness +ĠVolks wagen +Ġprec aution +ĠM ell +i ak +ist ries +Ġ24 8 +Ġoverl apping +Sen ate +ĠEnh ance +res y +rac ial +OR TS +ĠM ormons +Str ong +ĠCo ch +Mex ico +ĠMad uro +Ġj ars +Ġcan e +W ik +oll a +iff erence +Ġphysic ist +ĠMag gie +Ġ28 5 +Ġdep iction +ĠMcL aren +J u +Ġsl ows +Ġcommission ers +ĠWill ow +ĠExpl os +hov ah +Ġtechn ician +Ġhom icides +ĠFl av +ĠTr uman +Ġ100 00 +u ctor +Ġsh ader +News letter +45 7 +Ġre ver +Ġhard ened +Ġwhere abouts +Ġrede velop +Ġcar bs +Ġtra vers +Ġsqu irrel +Ġfoll ower +Ġs ings +50 8 +Ġrabb its +emon ium +Ġdocument ing +Ġmisunder stood +) ' +R ick +gg ies +Ġprem ie +Ġsk ating +Ġpass ports +Ġf ists +aged don +H aw +AC P +0 80 +ĠThough ts +ĠCarl son +Ġpriest hood +h ua +Ġdun geons +ĠLo ans +Ġant is +Ġfamiliar ity +ĠS abb +op al +ĠIn k +st rike +Ġc ram +Ġlegal ized +Ġcu isine +Ġfib re +Tra vel +ĠMon ument +OD Y +eth y +Ġinter state +ĠP UR +em porary +ĠArab ian +develop ed +Ġsadd le +Ġg ithub +ĠOff er +ĠIS P +ro let +ĠSUP ER +ĠDen is +Ġmultipl ier +Ġstir red +Interest ingly +Ġcustom ary +Ġbill ed +he x +Ġmultipl ied +Ġfl ipping +ĠCros by +Ġfundament als +ia e +ĠPlay ed +ĠAt om +am azon +ĠFl am +ee z +activ ated +Ġtables poon +Ġliberal ism +ĠPal in +ĠP atel +N um +ĠT AM +Ġs urn +ĠRel oaded +Ġco ined +" ], +ĠCl ash +ĠAg u +Ġprag matic +ĠActiv ate +Ġ8 02 +Ġtrail ers +Ġsil hou +Ġprob es +Ġcirc us +ĠB ain +ĠLind say +ĠAb bey +Del ivery +Ġconcess ion +Ġgast ro +ĠSpr ite +Ä Ł +and el +Ġg imm +Ġaut obi +ĠT urtle +Ġwonder fully +ĠHar am +ĠWorld wide +ĠHand le +Ġtheor ists +Ġsle ek +ĠZh u +ograph ically +EG A +ĠOwn ers +ath s +ĠAntar ctic +n atal +=" " +fl ags +`` `` +Ġs ul +K h +Ġpot assium +Ġlinem an +Ġcere al +ĠSe asons +Ġ20 22 +Ġmat hematic +Ġastron omers +prof essional +Ġf ares +cknow led +Ġch i +Ġyoung sters +Ġmistaken ly +Ġhem isphere +ĠDiv inity +r one +Ġ" , +r ings +Ġattract s +v ana +å ¹ +C AP +Ġplay list +Ġpor ch +ãģ £ +Ġincorpor ates +Ġso ak +Ġassert ing +ĠTerror ism +ĠP ablo +J a +ces ter +Ġfear ing +ĠPr ayer +Ġescal ated +G W +Ġro be +ĠBright on +ac ists +ĠSym phony +ĠDwar f +ĠPar ade +ĠLe go +Ġinex pl +Ġl ords +le af +RA G +l iber +Ġcig ars +ĠJe hovah +60 6 +WIND OWS +ĠLiber ia +eb us +He avy +Ġl ubric +ĠR W +angu ages +Ġnarrow ed +com puter +ĠE mber +Ġmurder ing +Ġdown stream +ĠT uls +ĠT ables +Top ic +ĠAcc uracy += / +l ost +ĠRe i +Ġprogress es +b ear +Ġestablish ments +Just in +ĠPe ach +ĠG omez +å ¿ +ĠTri angle +Id ent +ĠH ive +Res ources +Ġmix es +ĠAss uming +M u +Ġhyp oc +Ġs ane +ĠW an +id ious +Su ccess +Ġ io +Ang el +Ġdanger ously +ĠCreat ure +W ORK +: [ +ĠKat rina +List ener +M iller +ĠId lib +h ang +Ġcircum vent +h ref +Ġcel estial +ĠWe eks +ĠP ug +ĠDal ton +Ġsubpoen a +uk u +Ġpers isted +pe i +old ing +ĠDoc uments +ĠH ast +ĠC ENT +Ġprim er +Ġsyn onymous +Ġn ib +om bs +Ġnot ation +ĠD ish +ĠAt mosp +Ġforb id +ĠAN G +pat tern +l os +Ġproject iles +b rown +." , +ĠVen om +Ġfierce ly +ub lished +ĠU ran +ĠNic arag +4 10 +ĠC AL +OT OS +ĠMir acle +ĠEn chant +Ġguard ing +app end +Att ach +Ġlevel ed +Ġcond oms +ih ilation +64 9 +Ġnight mares +ĠTHE Y +ĠST ART +ĠK inn +Ġroomm ate +Ġhy giene +o pping +J ob +Ġl vl +ĠV ER +ĠKe eping +ab etic +Ġformat ting +eral a +Ġrev isions +Ġres urg +T el +ĠGood man +35 3 +p od +Ġind isp +ĠTrans lation +Ġg own +ĠM und +Ġc is +Ġby stand +col lect +ĠPun jab +act ively +ĠG amb +te ll +Ġimport ing +g encies +Ġloc om +ĠBr ill +H oly +ĠBer ger +Ġshow down +Ġrespond ers +IL Y +Ġt akedown +le ted +Ġmat tered +Ġpredict ive +Ġover lay +G PU +ĠV ick +Ġconvey ed +T ab +pe er +Sc an +Ġdefensive ly +v ae +Ġappro ving +Ġt iers +ĠV ia +quer ade +ĠSaud is +Ġdemol ished +ĠProp he +Ġmon o +Ġhospital ity +H AM +ĠAri el +M OD +ĠTor ah +Ġbl ah +ĠBel arus +erent ial +ĠT uc +Ġbank er +39 7 +Ġmosqu it +ĠScient ist +ĠMus ical +Ġh ust +Sh ift +Ġtor ment +Ġstand off +E duc +ĠF og +Ġampl ifier +Sh ape +Inst ance +ĠCrit ics +Ġda emon +H ouston +Ġmatt ress +ĠID F +Ġobsc ene +ĠA mer +hett i +Ġcomp iling +35 2 +vere tt +ĠRed uction +ist ration +ĠBl essed +ĠB achelor +3 16 +Ġpr ank +ĠVul can +dd ing +Ġm ourning +ĠQu int +ĠBl aster +test ing +Ġsed iment +>> > +ĠE ternity +ĠWH ERE +ĠM aze +Ġreact ing +ĠAl v +oms day +ĠC RA +Ġtransl ator +Ġbog us +at u +We bsite +oll s +Ġbapt ism +Ġs ibling +ĠAut umn +ve z +ãģ® é +gu ards +Ge org +assad ors +ĠFre ud +Ġcontin ents +ĠReg istry +Bern ie +ĸļ 士 +Ġtoler ant +ĠU W +Ġhor ribly +99 5 +ĠMID I +Ġimpat ient +oc ado +er i +ĠWor st +ĠNor ris +ĠTalk ing +Ġdef ends +ens able +Ġ20 21 +Ġanat omy +L ew +Ġdraw er +ĠCan berra +Ġpatri otic +é¾įå ĸļ士 +ĠAv g +AR M +Ġundis closed +Ġfare well +45 9 +b able +ĠAll ison +OL OG +Ġcon co +t ight +ĠAC PI +ĠM ines +l ich +ĠâĶ ľ +represent ed +200 000 +Ġenthusi ast +OT S +b il +ĠIng redients +Ġinvent or +ĠMy SQL +³³ Âł +ĠAB OUT +with in +Ġm k +B ul +ĠF ake +Ġdracon ian +W a +hel m +ĠTer ran +erv ille +Ġcommon place +SI ZE +Ġ" < +re place +ograph s +ĠSE LECT +inc ible +ĠMost ly +ĠShe ffield +ĠID E +ugg le +Ġcit ations +h urst +ĠUn ix +Ġunle ash +ĠP iper +ĠN ano +Ġsucc umb +Ġreluct ance +Ġ25 00 +ĠMer chant +Ġwire t +Ġcomb os +ĠBirth day +Ġchar coal +ĠU PS +ĠFair fax +Ġdrive way +ĠT ek +ĠP itch +ove re +Ġtechn icians +ĠAct ual +fl ation +ĠF iscal +ĠEm pty +an amo +Ġmag nesium +Ġsl ut +Ġgrow ers +Invest igators +( ): +ĠS atellite +ĠKe ynes +miss ive +l ane +Ġb orough +3 44 +ĠTE AM +ĠBet hesda +C V +h ower +ĠR AD +Ġch ant +ĠR iy +Ġcompos itions +Ġmild ly +Ġmedd ling +Ġag ility +ane ers +5 01 +Ġsyn th +ling er +29 1 +Ġex claimed +Part y +Ġcont amin +ĠMan or +ĠResp ond +Ġpra ising +Ġman ners +fle et +Sum mer +ĠLy nd +ĠDef initely +gr im +Ġbow ling +st ri +ç Ľ +y nt +Ġmand ates +D IV +Ġreconc ile +view s +ĠDam on +vet te +F lo +ĠGreat est +il on +ic ia +Ġportray al +Ġcush ion +50 4 +19 79 +oss al +App lic +sc ription +Ġmit igation +AT S +p ac +Ġer ased +Ġdefic iencies +ĠHolland e +ĠX u +Ġb red +Ġpregn ancies +f emin +Ġem ph +Ġpl anners +Ġout per +utter ing +Ġperpet rator +Ġm otto +ĠEll ison +ĠNE VER +Ġadmitted ly +AR I +ĠAzerbai jan +Ġmill isec +Ġcombust ion +ĠBott le +ĠL und +ĠP s +ĠD ress +Ġfabric ated +Ġbat tered +Ġs idel +ĠNot ting +Fore ign +ĠJer ome +0 20 +ĠAr bit +Ġkn ots +ĠR IGHT +M oving +ãģ Ļ +Ġsur geries +Ġcour thouse +Ġm astered +Ġhover ing +ĠBr an +ĠAl ison +Ġsaf est +m ilitary +Ġbull ied +Ġbar rage +Read er +ES E +ĠGe ographic +T ools +3 14 +ĠGe ek +ro th +gl ers +ĠF IN +Ï ģ +ĠA ston +al tern +48 8 +Ġveter in +G amer +Ġint el +ren ches +Sh ield +Ġam nesty +ĠB har +Ġp iled +Ġhonor able +ĠInst itutes +Ġso aked +Ġcom a +ĠE FF +34 1 +by tes +ĠG mail +le in +ĠCanad iens +m aterial +I l +Ġinstruct ors +ĠK Y +Ġconce ive +ub b +ĠP ossible +Ġeas ing +ĠChrist ina +Ġcar ic +ĠHD R +R OM +Ġsho vel +de lete +Ġp uff +ĠCh anging +Ġseam lessly +Att ribute +Ġacqu isitions +ak ery +ĠE F +Ġaut istic +ĠT akes +ĠPow der +ĠSt ir +5 10 +ĠBub ble +sett ings +ĠF owler +Ġmust ard +Ġmore over +Ġcopyright ed +ĠLED s +15 00 +æ ī +ĠH IS +en f +Ġcust od +ĠH uck +G i +Ġim g +An swer +C t +j ay +ĠInf rastructure +Ġfeder ally +L oc +Ġmicro bes +Ġover run +dd s +ot ent +adi ator +>>>> >>>> +Ġtorn ado +Ġadj ud +Ġintrig ued +Ġs i +ĠRevel ation +pro gress +Ġburgl ary +ĠSai yan +ĠK athy +Ġser pent +ĠAndre as +Ġcomp el +ess ler +ĠPl astic +ĠAd vent +ĠPos itive +ĠQ t +ĠHind us +reg istered +ular ity +Ġrighteous ness +Ġdemon ic +u itive +ĠB DS +ĠGre gg +c ia +ĠCrus ade +ĠSina i +W ARE ++ ( +Ġme ll +Ġder ail +y ards +A st +Ġnotice ably +ĠO ber +R am +Ġun noticed +Ġse q +av age +T s +Ġ6 40 +Ġconced e +Ġ] ) +F ill +Ġcapt ivity +ĠImprove ment +ĠCrus ader +ara oh +M AP +æ Ĺ +Ġstr ide +al ways +F ly +N it +Ġal gae +ĠCook ing +ĠDo ors +Mal ley +Ġpolic emen +ãģ į +Ġastron aut +access ible +49 5 +ĠR AW +cl iffe +udic rous +Ġdep ended +al ach +Ġvent ures +ra ke +Ġt its +ĠH ou +Ġcond om +ormon al +Ġind ent +Ġupload ing +Foot note +Import ant +Ġ27 1 +Ġmind ful +Ġcont ends +C ra +Ġcal ibr +ĠO ECD +plug in +F at +ĠIS S +ĠDynam ics +ans en +68 6 +' ), +Ġsp rite +Ġhand held +ĠH ipp +=~ =~ +Tr ust +Ġsem antics +ĠBund es +ĠRen o +ĠLiter ature +s ense +G ary +ĠA eg +ĠTr in +EE K +Ġcler ic +ĠSS H +Ġch rist +Ġinv ading +ib u +Ġen um +aur a +Ġal lege +ĠInc redible +B BC +Ġth ru +Ġsa iled +Ġem ulate +Ġin security +Ġc rou +Ġaccommod ations +Ġincompet ent +Ġsl ips +ĠEarth qu +s ama +IL LE +Ġi Phones +as aki +Ġby e +Ġar d +Ġext ras +Ġsl aughtered +Ġcrowd funding +res so +Ġfil ib +ĠER ROR +ĠT LS +e gg +ĠIt al +Ġen list +ĠCatal onia +ĠSc ots +Ġser geant +Ġdiss olve +N H +Ġstand ings +ri que +I Q +Ġbenef iciary +Ġaqu arium +You Tube +ĠPower Shell +Ġbright est +ĠWar rant +S old +Writ ing +Ġbegin nings +ĠRes erved +ĠLatin os +head ing +Ġ4 40 +Ġrooft op +AT ING +Ġ3 90 +VP N +G s +k ernel +turn ed +Ġprefer able +Ġturn overs +ĠH els +S a +ĠShin ji +ve h +ĠMOD ULE +V iol +Ġex iting +Ġj ab +ĠVan illa +Ġac ron +ĠG ap +ber n +A k +ĠMc Gu +Ġend lessly +ĠFar age +ĠNo el +V a +M K +Ġbr ute +ĠK ru +ĠES V +ĠOl ivia +âĢ ł +ĠK af +Ġtrust ing +Ġh ots +3 24 +Ġmal aria +Ġj son +Ġp ounding +ort ment +Count ry +Ġpostp oned +Ġunequ iv +? ), +ĠRo oney +udd ing +ĠLe ap +ur rence +sh apeshifter +ĠH AS +os ate +Ġca vern +Ġconserv atism +ĠB AD +Ġmile age +Ġarrest ing +V aults +Ġmix er +Dem ocratic +ĠB enson +Ġauth ored +8 000 +Ġpro active +ĠSpirit ual +t re +Ġincarcer ated +ĠS ort +Ġpe aked +Ġwield ing +re ciation +×Ļ × +P atch +ĠEm my +Ġex qu +tt o +ĠRat io +ĠP icks +ĠG ry +ph ant +Ġf ret +Ġeth n +Ġarch ived +% - +c ases +ĠBl aze +Ġim b +c v +y ss +im ony +Ġcount down +Ġaw akening +ĠTunis ia +ĠRe fer +ĠM J +Ġun natural +ĠCar negie +iz en +ĠN uggets +he ss +Ġev ils +64 7 +Ġintrodu ctory +l oving +ĠMcM ahon +Ġambig uity +L abel +ĠAlm ighty +Ġcolor ing +ĠCl aus +set ting +N ULL +ĠF avorite +ĠS IG +> ( +ĠSh iva +ĠMay er +Ġstorm ed +ĠCo verage +we apons +igh am +Ġun answered +Ġle ve +Ġc oy +c as +b ags +as ured +Se attle +ĠSant orum +ser ious +Ġcourage ous +ĠS oup +Ġconfisc ated +Ġ// / +Ġuncon ventional +Ġmom s +ĠRohing ya +ĠOrche stra +ĠPot ion +Ġdisc redit +ĠF IL +f ixed +ĠDe er +do i +ĠDim ension +Ġbureaucr ats +et een +Ġaction Group +oh m +Ġb umps +ĠUt ility +Ġsubmar ines +ren heit +re search +ĠShap iro +Ġsket ches +Ġde ceptive +ĠV il +es ame +ĠEss entially +Ġramp age +isk y +Ġmut tered +th ritis +Ġ23 6 +f et +b ars +Ġpup il +ĠTh ou +o S +s ong +Ġfract ured +Ġre vert +pict ure +Ġcrit erion +us her +Ġreperc ussions +ĠV intage +ĠSuper intendent +Offic ers +Ġflag ged +Ġbl ames +Ġin verse +ograp hers +Ġmakes hift +Ġdev oid +Ġfoss ils +ĠArist otle +ĠFund s +Ġde pleted +ĠFl u +ĠY uan +Ġw oes +Ġlip id +Ġsit u +requ isites +Ġfurn ish +ĠSam ar +Ġshame ful +Ġadverse ly +Ġad ept +Ġrem orse +Ġmurder ous +uck les +ĠE SL +Ġ3 14 +s ent +Ġred ef +ĠC ache +ĠP urs +ig ans +Ġ4 60 +Ġpres criptions +Ġf res +F uck +ocr ates +Tw enty +ĠWe ird +ĠT oggle +ĠC alled +itiz ens +Ġp oultry +Ġharvest ing +ãĤ¦ ãĤ¹ +Bott om +Ġcaution ed +t n +39 6 +ĠNik ki +Ġeval uations +Ġharass ing +Ġbind ings +ĠMon etary +Ġhit ters +Ġadvers ary +un ts +Ġset back +Ġenc rypt +ĠC ait +Ġl ows +eng es +ĠN orn +Ġbul bs +Ġbott led +ĠVoy ager +3 17 +Ġsp heres +p olitics +Ġsubt ract +Ġsens ations +Ġapp alling +Ġ3 16 +Ġenvironment ally +ĠST EM +Ġpub lishes +5 60 +Ġdilig ence +48 4 +Ġadv ises +Ġpet rol +Ġimag ining +Ġpatrol s +ĠInt eger +ĠAs hes +act us +ĠRad iant +ĠL T +it ability +ht aking +Set ting +Ġnu anced +ĠRe ef +ĠDevelop ers +N i +pie ces +99 0 +Lic ense +Ġlow ers +ĠOtt oman +3 27 +oo o +Ġqu itting +mark ets +Beh ind +Ġbas in +Ġdoc s +an ie +fl ash +ct l +Ġcivil ized +ĠFuk ushima +"] ," +ĠK S +ĠHonest ly +ar at +Ġconstruct s +ĠL ans +ĠD ire +ĠLI KE +ĠTrou ble +Ġwith holding +ĠOb livion +Ġsan ity +any a +Con st +Ġgro cer +ĠC elsius +Ġrecount ed +ĠW ife +B order +ate red +h appy +Ġspo iler +Ġlog ically +H all +Ġsucceed ing +Ġpoly morph +Ġax es +ĠShot gun +ĠS lim +ĠPrin ciples +ĠL eth +art a +Ġsc or +Sc reenshot +Ġrelax ation +#$ #$ +Ġdeter rent +idd y +Ġpower less +Ġles bians +Ġch ords +ĠEd ited +se lected +Ġseparat ists +000 2 +Ġair space +Ġturn around +Ġc unning +P ATH +P oly +Ġbomb ed +Ġt ion +x s +Ġwith hold +Ġw aged +ĠLiber ties +Fl ag +Ġcomfort ing +45 4 +ĠI ris +are rs +Ġr ag +Ġrel ocated +ĠGu arant +Ġstrateg ically +Ġgam ma +uber ty +ĠLock heed +g res +Ġgr illed +ĠLow e +st ats +ĠR ocks +Ġsens ing +Ġrent ing +ĠGe ological +ا Ø +ot rop +Ġse w +Ġimproper ly +48 6 +Ġâĸ ł +Ġstar ving +ĠB j +Disc ussion +3 28 +ĠCom bo +ĠFix es +N AT +Ġstri ving +th ora +Ġharvest ed +ĠP ing +Ġplay ful +Ġaven ues +Ġoccup ational +Ġw akes +ĠCou rier +Ġdrum mer +ĠBrow ser +ĠH outh +it u +Ġapp arel +p aste +Ġhun ted +ĠSecond ly +l ain +X Y +ĠP IN +ic ons +Ġcock tails +Ġs izable +Ġhurd les +est inal +ĠRecre ation +Ġe co +64 8 +ĠD ied +m int +Ġfinger prints +Ġdis pose +ĠBos nia +ts y +22 00 +Ġins pected +ĠF ou +Ġf uss +Ġamb ush +ĠR ak +Ġmanif ested +Pro secut +Ġsuff ice +ren ces +Ġcompens ated +ĠC yrus +Ġgen us +ĠWolver ine +ĠTrend s +Ġh ikes +ĠSe en +Ġen rol +C old +Ġpol itely +ĠSl av +ĠRu pert +Ġey ewitness +ĠAl to +Ġun comp +Ġposter ior +M ust +ĠHer z +Ġprogress ively +Ġ23 4 +Ġind ifference +ĠCunning ham +Ġacadem ia +Ġse wer +Ġast ounding +ĠA ES +r ather +Ġeld est +Ġclim bs +ĠAdd s +Ġout cry +Ġcont ag +ĠH ouses +Ġpe pt +ĠMel ania +interest ed +ĠU CH +ĠR oots +ĠHub bard +ĠT BD +ĠRoman ian +fil ename +St one +ĠIm pl +Ġchromos ome +C le +d x +Ġscram bled +ĠP t +Ġ24 2 +OP LE +Ġtremend ously +St reet +Ġcra ving +Ġbund led +ĠR G +p ipe +Ġinj uring +Ġarc ane +Part icip +ĠHero ic +st y +Ġto pping +ĠTemp est +rent ices +b h +Ġpar anoia +ĠUnic ode +Ġegreg ious +Ġ\ ' +ĠOsw ald +Ġgra vel +ĠSim psons +Ġbl and +ĠGuant anamo +Writ er +lin ers +ĠD ice +J C +Ġpar ity +Ġs ided +Ġ23 7 +ĠPyr rha +at ters +d k +F ine +comp an +Ġform ulated +ĠId ol +il ers +hem oth +ĠF av +Ġintr usion +Ġcar rots +ĠL ayer +ĠH acker +Ġ ---------------- +Ġmoder ation +é ģ +oc oc +Ġcharacter ize +ĠTe resa +Ġsocio economic +Ġper k +ĠParticip ation +tr aining +ĠPaul o +ph ys +Ġtrust worthy +Ġembod ied +ĠMer ch +c urrency +ĠPrior ity +Ġte asing +Ġabsor bing +Ġunf inished +ĠCompar ison +Ġdis ple +writ ers +Ġprofess ions +ĠPengu in +Ġang rily +ĠL INK +68 8 +ĠCor respond +Ġprev ailed +Ġcart el +l p +as ms +ĠRed emption +ĠIslam ists +effect s +d ose +ĠL atter +ĠHal ifax +Ġv as +ĠTop ics +ĠN amed +advert ising +zz a +IC ES +Ġret arded +ach able +ĠPupp et +ĠItem Level +Ġret ract +Ġident ifiable +A aron +ĠB uster +s ol +hel le +as semb +H ope +r anged +B a +ĠP urch +é Ģ +ĠSir i +Ġarri vals +Ġ19 12 +Ġshort ened +Ġ3 12 +Ġdiscrep ancy +ĠTem perature +ĠWal ton +Ġkind erg +p olit +Ġrem ix +Ġconnect ors +ãĥĺ ãĥ© +ĠKazakh stan +dom inated +Ġsu gars +im ble +ĠPan ic +ĠDem and +ĠCol ony +on en +ĠM ER +7 75 +ur ia +aza ar +ĠDeg ree +P ri +Ġsun shine +Ġ25 1 +Ġpsychedel ic +Ġdigit ally +ĠBra un +Ġsh immer +Ġsh ave +ĠTel esc +ĠAst ral +ĠVenezuel an +ĠO G +Ġc rawling +Int eg +ĠFe ather +Ġunfold ing +Ġappropri ation +Ġè£ı è +ĠMob ility +ĠN ey +- . +b ilt +L IN +ĠT ube +ĠCon versely +Ġkey boards +ĠC ao +Ġover th +Ġla ure +>> \ +ĠV iper +ach a +Off set +ĠR aleigh +ĠJ ae +J ordan +j p +Ġtotal itarian +Connect or +Ġobserv es +ĠSpart an +ĠIm mediately +ĠSc al +C ool +Ġt aps +Ġro ar +P ast +Ġch ars +ĠB ender +ĠShe ldon +Ġpain ter +Ġbe acon +ĠCreat ures +Ġdownt urn +Ġh inder +ĠAnd romeda +à Ľ +cc oli +ĠF itness +et rical +Ġutil izes +Ġsen ate +Ġen semble +Ġche ers +T W +Ġaff luent +k il +ry lic +ord ering +Com puter +Ġgru esome +ost ics +ĠUb isoft +ĠKel ley +Ġw rench +Ġbourgeois ie +IB LE +ĠPrest on +w orn +ar ist +reat ing +Ġst ained +ar ine +Ġsl ime +EN N +Ġche sts +Ġground water +ann ot +ĠTr ay +ĠLoc ke +ĠC TR +Ġd udes +ĠEx ternal +ĠDec oder +Ġpar amed +ĠMed line +80 9 +ĠD inner +rup al +g z +ĠG um +ĠDem o +j ee +Ġd h +ber man +arch s +Ġen qu +ĠEp stein +Ġdevast ation +Ġfriends hips +ĠAr d +Ġ23 1 +ĠRub in +ĠDist ance +Ġsp urred +Ġd ossier +Ġover looking +\\\\\\\\ \\\\\\\\ +Fore st +ĠCom es +\ ", +ĠIran ians +Ġf ixtures +L aughs +Ġcur ry +ĠKing ston +Ġsqu ash +Ġcat alogue +Ġabnormal ities +Ġdigest ive +.... ..... +Ġsubord inate +og ly +Ġ24 9 +M iddle +Ġmass ac +Ġburg ers +Ġdown stairs +Ġ19 31 +39 4 +ĠV G +Ġl asers +ĠS ikh +ĠAlex a +der ived +Ġcycl ist +ãģ® éŃĶ +onel iness +!!!! !!!! +Ġbuff s +leg ate +Ġrap ing +Ġrecomm ending +ro red +Ġmult icultural +un ique +Ġbusiness men +Ġune asy +ĠM AP +Ġdisp ersed +cipl ine +J ess +ĠK erala +å § +Ġabst raction +Sur v +U h +Ġprin ters +ij a +ow der +Ġanalog ous +ĠA SP +af er +Ġunfold ed +Ġlevel ing +Ġbre ached +ĠH earing +Ġn at +Ġtransl ating +crit ical +Ġant agonist +ĠYes terday +Ġfuzz y +w ash +m ere +Ġbe wild +ĠM ae +V irgin +ph rase +Ġsign aled +ĠH IGH +Ġprot ester +Ġgar ner +unk nown +Ġk ay +Ġabduct ed +Ġst alking +am n +Ġdes erving +ĠR iv +ĠJ orge +Ġscratch ing +ĠS aving +ip ing +Ġte ase +Ġmission ary +ĠMor row +T IME +P resent +Ġchem otherapy +tern ess +ĠH omes +ĠP urdue +Ġst aunch +ĠWhit ney +ĠTH ERE +Î ¼ +iat us +ĠErn est +ĠDe ploy +Ġcove ted +F ML +ĠDial ogue +Ġex ited +f ruit +Ġner d +":" "," +Ġv ivo +ru ly +4 60 +ĠAm en +rehens ible +Ġâ ĺ +D IR +Ġad herence +Ġche w +ĠCo ke +ĠSerge i +dig ital +ĠNe ck +g ently +enth al +/ ) +Ġwe ary +Ġgu ise +ĠConc ord +ĠOn ion +at cher +Ġb inge +ĠDirect ive +Ġman ned +ans k +Ġill usions +Ġbillion aires +38 3 +oly n +odynam ic +ĠWhe at +ĠA lic +Ġcol oured +ĠN AFTA +ab o +Ġmac ros +ind ependent +s weet +Ġsp ac +ĠK abul +Ġ Ä +em e +Ġdict ated +Ġsh outs += { +Ġr ipping +ĠSh ay +ĠCr icket +direct ed +Ġanalys ed +ĠWAR RANT +ag ons +ĠBlaz ers +Ġche ered +Ġar ithmetic +ĠTan z +37 3 +ĠFl ags +Ġ29 5 +Ġw itches +ĠIn cluded +ĠG ained +ĠBl ades +G am +ĠSam antha +ĠAtl antis +ĠPr att +Ġspo iled +ĠI B +ĠRam irez +Pro bably +re ro +ĠN g +ĠWar lock +t p +Ġover he +Ġadministr ations +Ġt int +Ġreg iment +Ġpist ols +Ġblank ets +Ġep ist +Ġbowl s +Ġhydra ulic +Ġde an +Ġj ung +Ġasc end +70 5 +ĠSant iago +à ® +Ġun avoid +ĠSh aman +re b +Ġstem ming +99 8 +ĠM G +st icks +esthes ia +ER O +Ġmor bid +ĠGr ill +ĠP oe +any l +Ġdele ting +ĠSurve illance +Ġdirect ives +Ġiter ations +ĠR ox +ĠMil ky +F ather +Ġpat ented +44 7 +Ġprec ursor +Ġm aiden +ĠP hen +ĠVe gan +ĠPat ent +K elly +Redd itor +Ġn ods +Ġvent ilation +ĠSchwar z +Ġw izards +Ġomin ous +ĠHe ads +ĠB G +Ġl umber +ĠSp iel +Ġis Enabled +Ġancest ral +ĠSh ips +Ġwrest ler +ph i +Ġy uan +ĠRebell ion +Ġice berg +Ġmag ically +Ġdivers ion +ar ro +yth m +ĠR iders +ĠRob bie +ĠK ara +ĠMain tenance +ĠHer b +Ġhar ms +p acked +ĠFe instein +Ġmarry ing +Ġbl ending +ĠR ates +Ġ18 80 +Ġwr ink +ĠUn ch +ĠTor ch +desc ribed +Ġhuman oid +ilit ating +ĠCon v +ĠFe ld +IGH TS +Ġwhistlebl ower +ort mund +ets y +arre tt +ĠMon o +ĠI ke +ĠC NBC +ĠW AY +ĠMD MA +ĠIndividual s +Ġsupplement al +Ġpower house +ĠSt ru +F ocus +aph ael +ĠCol leg +att i +Z A +Ġp erenn +ĠSign ature +ĠRod ney +Ġcub es +idd led +ĠD ante +ĠIN V +iling ual +ĠC th +Ġso fa +Ġintimid ate +ĠR oe +ĠDi plom +ĠCount ries +ays on +Ġextrad ition +Ġdis abling +ĠCard iff +Ġmemor andum +ĠTr ace +Ġ?? ? +se ctor +ĠRou hani +ĠY ates +ĠFree ze +Ġbl adder +M otor +ĠProm ise +ant asy +Ġforesee able +ĠC ologne +cont ainer +ĠTre es +ĠG ors +ĠSin clair +Ġbar ring +key e +Ġsl ashed +ĠStat istical +é ĩ +Ġâĸ º +All ows +Ġhum ility +Ġdr illed +ĠF urn +44 3 +Ġse wage +Ġhome page +Ġcour tyard +Ġv ile +Ġsubsid iaries +aj o +direct ory +Ġam mon +V ers +charg es +Ġ} } +ĠCh ains +Ġ24 6 +n ob +Ġper cept +Ġg rit +Ġfisher men +ĠIraq is +ĠDIS TR +ĠF ULL +ĠEval uation +g raph +at ial +Ġcooper ating +Ġmel an +Ġenlight ened +Ġal i +t ailed +Ġsal ute +Ġweak est +ĠBull dogs +U A +ĠAll oy +Ġsem en +oc ene +ĠWilliam son +s pr +, âĢĶ +ĠG F +itt ens +Be at +ĠJ unk +iph ate +ĠFarm ers +ĠBit coins +ig ers +d h +ĠL oyal +p ayer +Ġentert ained +Ġpenn ed +Ġcoup on +Que ue +Ġweaken ing +c arry +Ġunderest imate +Ġshoot out +Ġcharism atic +ĠProced ure +Ġprud ent +in ances +Ġric hes +Ġcort ical +Ġstr ides +Ġd rib +ĠOil ers +5 40 +ĠPer form +ĠBang kok +Ġe uth +S ER +Ġsimpl istic +t ops +camp aign +Q uality +Ġimpover ished +ĠEisen hower +Ġaug ment +ĠH arden +Ġinterven ed +Ġlist ens +ĠK ok +Ġs age +Ġrub bish +ĠD ed +Ġm ull +pe lling +Ġvide ot +Produ ction +D J +m iah +Ġadapt ations +Ġmed ically +Ġboard ed +Ġarrog ance +Ġscra pped +Ġopp ress +FORM ATION +Ġj unction +4 15 +EE EE +S kill +Ġsub du +ĠSug gest +ĠP ett +Ġle tt +ĠMan ip +ĠC af +ĠCooper ation +T her +Ġreg ained +¶ æ +ref lect +Ġth ugs +ĠShel by +Ġdict ates +ĠWe iner +ĠH ale +Ġbatt leground +s child +Ġcond ol +h unt +osit ories +Ġacc uses +Fil ename +Ġsh ri +Ġmotiv ate +Ġreflect ions +N ull +ĠL obby +¥ µ +ĠS ATA +ĠBack up +Ñ ĥ +n in +ĠCor rection +Ġju icy +ut ra +ĠP ric +Ġrest raining +ĠAir bnb +ĠAr rest +Ġappropri ations +Ġsl opes +Ġmans laughter +Ġwork ings +ĠH uss +ĠF rey +Le ave +ĠHarm ony +ĠF eder +Ġ4 30 +Ġt rench +Ġglad ly +Ġbull pen +ĠG au +b ones +Ġgro ove +Ġpre text +ã ħĭ +Ġtransm itter +ĠComp onent +Ġunder age +ĠEm pires +T ile +Ġo y +ĠMar vin +ĠC AS +Ġbl oss +Ġrepl icated +ĠMar iners +Marc us +ĠBl ocks +Ġliber ated +Ġbutter fly +Fe el +Ġfer mentation +Ġyou tube +Ġoff end +ĠTer m +res ist +Ġcess ation +Ġinsurg ency +Ġb ir +ĠRa ise +59 5 +Ġhypothes es +50 2 +Ġpl aque +ocr at +Ġjack ets +ĠHuff Post +am ong +Ġconf er +48 7 +ĠL illy +Ġadapt ing +ĠF ay +Ġsh oved +ve c +Ġref ine +Ġg on +Ġgun men +z ai +ĠShut tle +ĠI zan +Ġ19 13 +Ġple thora +· · +Ġ5 10 +Ġp uberty +Ġ24 1 +ĠWe alth +ĠAl ma +ĠM EM +ĠAd ults +C as +pr ison +R ace +Ġwater proof +Ġathlet icism +Ġcapital ize +ĠJu ice +Ġillum inated +ĠP ascal +Ġirrit ation +ĠWitness es +ad le +ĠAst ro +Ġf ax +ĠEl vis +Prim ary +ĠL ich +ĠEl ves +Ġres iding +Ġst umble +3 19 +ĠP KK +Ġadvers aries +D OS +ĠR itual +Ġsm ear +Ġar son +ident al +Ġsc ant +Ġmon archy +Ġhal ftime +Ġresid ue +Ġind ign +ĠSh aun +ĠEl m +aur i +A ff +W ATCH +ĠLy on +hel ps +36 1 +Ġlobby ist +Ġdimin ishing +Ġout breaks +Ġgo ats +f avorite +ĠN ah +son ian +ĠBo oster +Ġsand box +ĠF are +ĠMalt a +Ġatt Rot +ĠM OR +ld e +Ġnavig ating +T ouch +Ġunt rue +ĠDis aster +Ġl udicrous +Pass word +ĠJ FK +blog spot +4 16 +ĠUN DER +ern al +Ġdelay ing +T OP +Ġimpl ants +ĠAV G +ĠH uge +att r +Ġjournal istic +ĠPe yton +ĠI A +R ap +go al +ĠProgram me +Ġsm ashing +w ives +print ln +ĠPl ague +in us +EE P +Ġcru iser +ĠPar ish +umin ium +Ġoccup ants +ĠJ ihad +m op +Ġp int +Ġhe ct +ĠMe cca +direct or +ĠFund ing +ĠM ixed +Ġst ag +T ier +Ġg ust +Ġbright ly +ors i +Ġup hill +R D +Ġles ions +ĠBund y +liv ious +Ġbi ologist +ĠFac ulty +ĠAuthor ization +Ġ24 4 +All ow +ï ¸ +ĠGi ul +Ġpert inent +ot aur +es se +ĠRo of +Ġunman ned +35 1 +ĠSh ak +ĠO rient +Ġend anger +D ir +Ġrepl en +ed ient +Ġtail or +Ġgad gets +Ġaud ible +âĺ Ĩ +N ice +Ġbomb ard +ĠR ape +Ġdef iance +ĠTW O +ĠFilip ino +Ġunaff ected +erv atives +Ġso ared +ĠBol ton +Ġcomprom ising +ĠBrew ers +R AL +ĠA HL +icy cle +Ġv ampires +Ġdi pped +oy er +ĠX III +Ġsidew ays +ĠW aste +ĠD iss +ĠâĶľ âĶĢâĶĢ +$ . +Ġhabit ats +ĠBe ef +tr uth +tr ained +spl it +R us +And y +ĠB ram +RE P +p id +è£ ħ +ĠMut ant +An im +ĠMar ina +Ġfut ile +hig hest +f requency +Ġepile psy +Ġcop ing +Ġconc ise +Ġtr acing +ĠS UN +pan el +ĠSoph ie +ĠCrow ley +ĠAd olf +ĠShoot er +Ġsh aky +ĠI G +ĠL ies +ĠBar ber +p kg +Ġupt ake +Ġpred atory +UL TS +/ ** +Ġintox icated +ĠWest brook +od der +he ment +Ġbas eman +AP D +st orage +ĠFif ty +ed itor +G EN +UT ION +ir ting +Ġse wing +r ift +Ġag ony +ĠS ands +Ġ25 4 +C ash +Ġl odge +Ġp unt +N atural +ĠIde as +Ġerrone ous +ĠSens or +ĠHann ity +Ġ19 21 +Ġm ould +ĠG on +kay a +Ġanonym ously +ĠK EY +Ġsim ulator +W inter +Ġstream ed +50 7 +? ", +Ġte ased +Ġco efficient +Ġwart ime +ĠTH R +' '. +ĠBank ing +mp ire +Ġf andom +Ġl ia +G a +Ġdown hill +Ġinterpre ting +Ind ividual +N orm +Ġjealous y +bit coin +Ġple asures +ĠToy s +ĠChev rolet +ĠAd visor +IZ E +Ġrecept ions +70 6 +C ro +Ġ26 2 +Ġcit rus +ir u +Review er +ject ed +U ES +an z +19 81 +ĠWork er +Ġcompl ied +ores cent +contin ental +T on +ĠPr ism +ĠShe ep +Ġ28 8 +n ox +ĠV og +O rd +Ġreal ms +te k +Ġirrig ation +Ġbicy cles +Ġelectron ically +p oly +t all +() ); +Ġaest hetics +ĠInteg rated +Expl ore +Ġd unk +47 6 +p ain +ĠJac ques +ĠD mit +Fram es +Ġreun ited +Ġhum id +D ro +P olitical +Ġyouth ful +Ġent ails +Ġmosqu ito +36 3 +spe cies +Ġcoord inating +ĠMay hem +ĠMagn us +M ount +Impro ved +ĠST ATE +ATT LE +Ġflow ed +Ġtack led +Ġfashion ed +Ġre organ +iv ari +f inger +Ġreluct antly +et ting +ĠV and +you ng +ĠGar land +Ġpresum ption +Ġamen ities +ĠPle asant +on ential +ĠO xy +Ġmor als +ĠY ah +Read y +Sim on +En h +D emon +Ġcl ich +Mon itor +ĠD U +Ġwel comes +Ġstand out +Ġdread ful +Ġban anas +Ġball oons +h ooting +bas ic +Ġsuff ix +Ġd uly +can o +Ch ain +at os +Ġgeop olitical +Ġ( & +ĠGem ini +ÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤ ÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤ +Ġacqu itted +L uck +prot ect +10 24 +Ġsc arcity +Ġmind fulness +ec ided +D N +pr ime +ĠPres idents +ĠVID EO +Ġ( âĪĴ +add ock +N OR +ĠP ru +p un +ĠL OL +)) )) +ĠL iqu +ĠS AS +Ġsty ling +Ġpunish ments +Ġnum b +Ġasc ertain +ĠRock ies +f lu +Th umbnail +Ġperpet rated +ĠSem i +Ġdis arm +ĠOld er +ĠEx ception +Ġexponent ially +ĠCommun ities +Ġabol ish +ĠPart ner +pt oms +Ġ7 77 +ĠFo ley +ĠC ases +Ġgre ase +ĠReb irth +G round +Ġ; ) +ĠDoct rine +ik ini +Y e +ĠBl ossom +Ġpers ists +b ill +Ġinf usion +Ġbud dies +9 11 +ĠPat ient +Ġdem os +Ġacquaint ance +ĠP aw +at ari +Ġx ml +Ġfasc ination +ĠSer ve +Ï Ĥ +br anded +Ġa z +Return s +Ġover shadow +Ġro am +Ġspeed y +n umbered +hel ial +Ġdisc iple +Ġass urances +g iven +pect ing +ĠN atalie +çĶ ° +Ġmosquit oes +rote in +Ġnumer ic +Ġindepend ents +Ġtrans itional +Ġreaction ary +ĠMech dragon +do ctor +Ġshort est +Ġsequ ential +ĠB ac +ĠAccount s +ãģ Į +ach y +ract ive +ĠReg iment +Ġbreat htaking +ffic iency +ĠB ates +Ġ3 11 +Ġward robe +ft s +ĠBer k +Sim ply +ĠRivers ide +iver ing +ident ial +lu cent +Ġen riched +ĠCon ver +ĠG iving +ãĥ Ļ +Ġlegal ize +ĠF TC +Ġfre aking +M ix +Ġter restrial +es ian +ci ents +W ing +LO AD +Ġled ge +ĠViol ent +ĠMet all +Ġ30 8 +Ġs outheastern +hett o +M eat +Ġslow down +Ġret reated +Jere my +end as +**** * +er ic +Ġre ins +opp able +ĠHuman ity +ear ances +rig an +C amera +Ġwa ivers +s oc +Ġalter ation +trans form +ĠC emetery +50 6 +Ġindef inite +Ġstim ulating +y g +60 3 +ĠS op +Ġdescript ive +Ph ase +ĠEd mund +Ġpneum onia +vent us +A mb +Ġlabor atories +ĠEx clusive +ug ar +W ere +Ġmalf unction +Ġhomosexual s +Ġ---- --- +un i +Ġturb ines +ĠEqu ity +D u +Ġmind ed +ĠR H +ĠBlack hawks +Ġfe ats +Ġ17 00 +re pl +36 2 +lad en +Ġindisp ensable +ly ss +tt i +Ġre el +Ġdiver ted +Ġlik eness +Ġsubscript ions +Ġfing ert +Ġfil thy +dest ruct +d raft +ĠBernard ino +l aunch +Ġper plex +ĠS UM +car b +Ġswe ater +ĠVent ure +ĠJ ag +ĠCele b +ĠV oters +Ġstead fast +Ġathlet ics +ĠHans on +ĠDr ac +Tr acker +Ġcomm end +ĠPres idency +ĠD ID +in formed +Ġweb page +P retty +Ġforce fully +ãĥĥ ãĤ¯ +Ġrel ocation +Ġsat ire +â ī +ĠSunder land +æ Ħ +V oice +???? ???? +Ġinform ant +Ġbow el +ĠUn iform +Ġ ..." +Ġpur ge +Ġpic nic +ĠU mb +ĠU PDATE +ĠSapp hire +ĠSt all +le arn +Ġobject ively +Ġob liter +Ġlooph ole +Ġjour neys +Ġo mission +Pro s +ĠSid ney +pl oma +Ġspray ed +Ġg uru +Ġtra itor +Ġtim et +Ġsn apping +ĠSe vent +urn al +ĠUk ip +Ġb owed +por al +l iberal +R os +Quest ions +i OS +Ġsummar ize +ST AT +Ġ18 50 +ap est +Ġl ender +ĠVari able +br inging +ĠL ORD +, ) +Ġcollaps es +x iety +ĠN ed +Y D +ĠSch a +Ġantib ody +Ġdis band +y re +ill usion +Ġro ver +s hed +ĠHiro sh +cc i +Ġcal am +ĠMort on +P interest +Ġ19 28 +ĠE uras +ord es +Ġf ences +ĠIn ventory +ĠVal encia +ĠU d +ĠT iff +Ġsqu e +Ġqu otation +Ġtroubles ome +er ker +QU EST +ĠKing doms +s outh +Ġle vy +Pr ince +ĠSt ing +Ġnick named +Ġapp e +Ġphot ographic +Ġcorp us +re ference +ĠT rog +U nt +) =( +ĠLat via +Ġactiv ating +Ġlicense e +Ġdispar ities +ĠNews letter +ãĥĥ ãĥĪ +Ġfree ing +ĠJe ep +ĠPer ception +ins k +Ġsil icone +ĠHay den +Le an +ĠSuz uki +ibr arian +66 8 +Ġsp or +Ġcorrel ations +ag hetti +Ġtu ber +ĠIP CC +il us +ĠV u +Ġwealth iest +ĠCarb uncle +an za +Ġfool ed +ĠZ ur +Ġd addy +ran o +il ian +Ġknock out +f man +requ ired +ĠWik ileaks +ĠD uffy +ON T +Ġins ol +ĠObject s +Ġb ou +ĠNord ic +ĠIns ert +sc an +Ġd ancers +Ġid iots +major ity +ĠNev ille +ĠFree BSD +Ġt art +pan ic +69 0 +Ġcoc oa +Ġsam pled +Ġlook up +Ind ust +Ġinject ions +gen re +Ġa u +Ġroad way +Ġgen itals +K ind +ĠEx aminer +ĠY az +F resh +Ġpar alysis +ĠAl uminum +Ġre ap +ok é +Ġsl oppy +ĠTun nel +pos ium +ner y +en ic +Ġher bal +ĠOut er +ĠBuild er +Ġinc ur +Ġide ologies +Ġback ups +cons uming +ĠDet ect +de ck +ĠKN OW +ĠG ret +ĠM IC +Ġtough ness +ĠEx hibit +Ġh ive +L es +ĠSCH OOL +ĠAt ari +ald e +ĠN ull +and estine +m ouse +Ġbrig ade +48 9 +Ġrev ol +ĠLaw son +ĠW ah +op oly +eb ted +ĠS aunders +Ġ3 13 +ĠW inc +Ġtab oo +ĠHel met +Ġw edge +ch ip +ĠT ina +b g +Ġinf uri +r n +Ġanomal ies +ĠSy nc +ĠEx am +ĠComm it +ĠDi ary +ĠALS O +ĠDe bor +omed ical +Ġcomprehens ion +6 55 +Ġempower ing +Ġ ire +Ġju ices +ĠE TH +ĠBox ing +=" / +Ġfacilit ated +p oke +ĠPars ons +ĠMod er +tra vel +Ġcivil izations +Ġliber tarians +Ġrun e +ĠCl arks +at hed +Ġcampaign ers +ĠDis patch +ĠFah renheit +ĠCap com +-------- -- +Ġl ace +Ġdr aining +Ġl iner +ĠArt ificial +é n +t ask +] ). +ĠGM O +ĠOper ator +ord inary +ĠInf luence +ĠU ps +Ġpot ency +uss en +osp ons +ĠSw im +ĠDead line +Un ity +Ġcul inary +Ġenlight enment +Ġwe arer +Ġmin ed +Ġp ly +Ġinc est +ĠDVD s +W alk +B TC +Tr ade +Ġdev al +ib and +ĠOvers ight +Palest inian +Ġd art +Ġm ul +L R +Ġrem ovable +ĠReal ms +ì Ŀ +Ġmisc ar +ĠV ulkan +68 5 +è re +ĠS ap +Ġmer ging +ĠCar ly +che ster +Ġbr isk +Ġlux urious +ĠGener ator +Ġbit terness +Ġed ible +Ġ24 3 +T G +Ġrect angle +With No +bel ow +J enn +Ġdark est +Ġh itch +Ġdos age +Ġsc aven +ĠK eller +ĠIllust rated +Certain ly +ĠMaver icks +Marg inal +Ġdiarr hea +Ġenorm ously +Ġ9 99 +sh r +qu art +Ġadam ant +ĠM ew +Ġren ovation +Ġcerv ical +ĠPercent age +en ers +ĠKim ber +Ġflo ats +Ġde x +ĠW itcher +ĠSwan sea +d m +Ġsal ty +y ellow +Ġca pe +ĠDr ain +ĠPaul a +ĠTol edo +les i +Mag azine +ĠW ick +ĠM n +ĠA ck +ĠR iding +AS ON +Ġhom ophobic +AR P +Ġwand ered +C PU +ood oo +ĠP ipe +Ġtight ening +ĠBut t +3 18 +Ġdesert ed +S ession +Ġfacilit ating +J ump +Ġemer gencies +OW ER +Ġexhaust ive +ĠAF TER +Ġheart beat +ĠLab el +ack y +ĠCert ified +ilt ration +Z e +ĠU tt +Ġ13 00 +Ġpres ume +ĠDis p +Ġsur ged +Ġdoll s +Col umb +Ġchim pan +ĠR azor +Ġt icks +Ġcouncill or +Ġpilgr image +ĠReb els +ĠQ C +ĠA uction +x ia +ik k +b red +Ġinsert ion +Ġco arse +d B +SE E +ĠZ ap +ĠF oo +Ġcontem por +ĠQuarter ly +ot ions +ĠAl chemist +ĠT rey +ĠDu o +S weet +80 4 +ĠGi ov +Ġfun n +N in +h off +Ġram ifications +Ġ19 22 +ĠExper ts +az es +Ġgar ments +ar ial +ĠN ab +Ġ25 7 +ĠV ed +Ġhum orous +ĠPom pe +Ġn ylon +Ġlur king +ĠSerge y +ĠMatt is +Ġmisogyn y +ĠComp onents +ĠWatch ing +ĠF olk +ract ical +B ush +Ġt aped +Ġgroup ing +Ġbe ads +Ġ20 48 +Ġcon du +quer que +Read ing +Ġgriev ances +Ult ra +Ġend point +H ig +ĠSt atic +ĠScar borough +L ua +ĠMess i +a qu +ĠPsy Net +ĠR udd +Ġa venue +v p +J er +Ġsh ady +ĠRes ist +ĠArt emis +Ġcare less +Ġbro kers +Ġtemper ament +Ġ5 20 +T ags +ĠTurn ing +Ġut tered +Ġp edd +Ġimpro vised +Ġ: ( +Ġtab l +Ġpl ains +16 00 +press ure +ĠEss ence +marg in +friend s +ĠRest oration +Ġpoll ut +ĠPok er +ĠAugust ine +ĠC IS +ĠSE AL +or ama +Ġth wart +se ek +Ġp agan + º +cp u +Ġg arn +Ġass ortment +ĠI LCS +t ower +Recomm ended +Ġun born +ĠRandom Redditor +ĠRandomRedditor WithNo +Ġparaly zed +Ġeru ption +Ġinter sect +ĠSt oke +ĠS co +B ind +å ¾ +ĠP NG +ĠNeg ative +ĠNO AA +Le on +Ġall oy +ĠL ama +ĠD iversity +5 75 +Ġunderest imated +ĠSc or +Ġm ural +Ġb usted +so on +l if +Ġnone x +Ġall ergy +ĠUnder world +ĠR ays +ĠBl asio +Ġh rs +ĠD ir +Ġ3 27 +by ter +Ġrepl acements +Ġactiv ates +ri ved +M H +Ġp ans +ĠH I +Ġlong itudinal +Ġnu isance +al er +Ġsw ell +ĠS igned +s ci +ĠIs les +ĠA GA +Ġdef iant +Ġson ic +oc on +K C +ĠA im +t ie +ah ah +Ġm L +D X +Ġb isc +ĠBill board +ĠSY STEM +NE Y +ga ard +Ġdist ressed +former ly +Al an +Ġche fs +Ġopt ics +ĠC omet +ĠAM C +Ġredes igned +irm ation +Ġsight ings +38 2 +3 11 +ĠW B +Ġcont raction +ĠT OTAL +D ual +Ġstart led +Ġunderstand ably +Ġsung lasses +ETH OD +Ġd ocker +Ġsurf ing +ĠH EL +ĠSl ack +ton es +Ġsh alt +Vis ual +49 8 +Dep artment +c ussion +Ġunrest ricted +Ġt ad +Ġre name +employ ed +Ġeduc ating +Ġgrin ned +bed room +ĠActiv ities +ĠV elvet +ĠSW AT +Ġsh uffle +ig or +Ġsatur ation +F inding +c ream +ic ter +Ġv odka +tr acking +te c +Ġfore ground +iest a +Ġve hement +ĠEC B +ĠT ie +E y +Ġt urtles +ĠRail road +ĠKat z +ĠFram es +Ġmen ace +ĠFell owship +ĠEss ential +ugg ish +Ġdri p +ch witz +ĠKy oto +s b +ĠN ina +Param eter +Ġal arms +ĠCl aud +Ġpione ering +Ġchief ly +ĠSc ream +Col lection +Ġthank fully +ĠRonald o +åŃ IJ +st rip +ĠDisney land +com mercial +See ing +S oul +Ġevac uate +Ġc iv +ĠAs he +Ġdiv ides +ĠD agger +rehens ive +Ġber ries +ĠD F +Ġs ushi +Ġplur ality +W I +Ġdisadvant aged +Ġbatt alion +ob iles +45 1 +Ġcl ing +Ġunden iable +ĠL ounge +Ġha unt +p he +Ġquant ify +Ġdiff ered +Ġ[* ] +ĠV iz +c um +sl ave +Ġvide og +Ġqu ar +Ġbund les +ĠAl onso +t ackle +Ġneur onal +Ġlandsl ide +conf irmed +ĠDep th +Ġrenew ables +B ear +ĠMaced onia +Ġjer seys +Ġb unk +ĠSp awn +ĠControl s +ĠBuch anan +Ġrobot ics +Ġemphas izing +ĠTut orial +h yp +ist on +Ġmonument al +æ ° +ĠCar ry +Ġt bsp +en ance +H ill +art hed +Ġro tten +De an +Ġtw isting +Ġgood will +Ġimm ersion +L iving +Ġbr ushes +ĠC GI +ĠAt k +tr aditional +Ġph antom +ĠSt amina +Ġexpans ions +ĠMar in +Ġembark ed +ĠE g +int estinal +ĠPE OPLE +ĠBo oth +ĠApp alach +Ġreleg ated +V T +M IT +Ġmust er +Ġwithdraw ing +Ġmicrosc ope +ĠG athering +ĠC rescent +ĠArgent ine +ĠDec re +ĠDomin ic +Ġbud s +ant age +ĠI on +Ġwid ened +ONS ORED +ĠGl oves +iann opoulos +raz en +fe el +Ġrepay ment +Ġhind sight +ĠRE ALLY +ĠPist ol +ĠBra h +Ġwat ts +Ġsurv ives +Ġfl urry +iss y +Al ert +ĠUrug uay +Ph oenix +S low +ĠG rave +ĠF ir +Ġmanage able +Ġtar iff +ĠU DP +ĠPist ons +ĠNiger ian +Ġstrike outs +Ġcos metics +whel ming +f ab +c ape +pro xy +Ġre think +Ġover coming +sim ple +Ġw oo +Ġdistract ing +ĠSt anton +ĠTuls a +ĠD ock +65 9 +Ġdisc ord +ĠEm acs +ĠV es +ĠR OB +Ġreass uring +Ġcons ortium +Muslim s +3 21 +Ġprompt s +se i +ĠH itch +imp osed +ĠF ool +Ġindisc rim +wr ong +bu querque +D avis +! ] +Ġtim eless +ĠNE ED +Ġpestic ide +Ġrally ing +ĠCal der +Ġå ¤ +Ġx p +ĠUn le +ĠEx port +lu aj +B uff +) [ +Ġsq or +S audi +Ġis tg +Ġindul ge +pro c +Ġdisg usted +Ġcomp ounded +Ġn em +Ġschool ing +ĠC ure +process ing +S ol +Ġpro verb +it ized +ĠAlv arez +Ġscar f +Ġrect angular +re ve +Ġh ormonal +ĠSt ress +itiz en +Ġ4 25 +girl s +ĠNo ir +ĠR app +Ġmar ches +ch urch +ĠUs es +Ġ40 5 +ĠBer m +Ġord inances +ĠJud gment +Charg es +ĠZ in +Ġdust y +Ġstraw berries +Ġper ce +ĠTh ur +ĠDebor ah +net flix +ĠLam bert +Ġam used +ĠGu ang +Y OU +R GB +ĠC CTV +Ġf iat +r ang +Ġf ederation +ĠM ant +ĠB ust +ĠM are +respect ive +ĠM igration +ĠB IT +59 0 +Ġpatriot ism +Ġout lining +reg ion +ĠJos é +Ġbl asting +ĠEz ra +B s +Ġundermin es +ĠSm ooth +Ġcl ashed +rad io +Ġtransition ing +ĠBucc aneers +ĠOw l +Ġplug s +Ġh iatus +ĠPin ball +Ġm ig +ĠNut r +ĠWolf e +Ġinteg ers +Ġor bits +ĠEd win +ĠDirect X +b ite +Ġbl azing +v r +Ed ge +ĠP ID +ex it +ĠCom ed +ĠPath finder +ĠGu id +ĠSign s +ĠZ er +ĠAg enda +Ġreimburse ment +M esh +i Phone +ĠMar cos +ĠS ites +h ate +en burg +Ġs ockets +p end +Bat man +v ir +ĠSH OW +Ġprovision al +con n +ĠDeath s +AT IVE +Pro file +sy m +J A +Ġnin ja +inst alled +id ates +eb ra +ĠOm aha +Ġse izing +ĠBe asts +Ġsal ts +M ission +Gener ally +ĠTr ilogy +he on +leg ates +Ġd ime +Ġf aire +par able +G raph +Ġtotal ing +Ġdiagram s +ĠYan uk +ple t +ĠMe h +Ġmyth ical +ĠStep hens +aut ical +ochem istry +Ġkil ograms +Ġel bows +anc ock +ĠB CE +ĠPr ague +Ġimpro v +ĠDev in +Ġ" \ +par alle +Ġsuprem acists +ĠB illion +Ġreg imen +inn acle +Ġrequ isite +ang an +ĠBur lington +ain ment +ĠObject ive +oms ky +G V +Ġun ilateral +Ġt c +Ġh ires +ment al +Ġinvol untary +Ġtrans pl +ĠASC II + ¨ +Ev ents +Ġdoub ted +ĠKa plan +ĠCour age +ig on +ĠMan aging +ĠT art +Ġfalse hood +ĠV iolet +Ġair s +Ġfertil izer +Brit ain +Ġaqu atic +ou f +W ords +ĠHart ford +Ġeven ings +ĠV engeance +qu ite +G all +ĠP ret +Ġp df +ĠL M +ĠSo chi +ĠInter cept +9 20 +Ġprofit ability +ĠId le +ĠMac Donald +ĠEst ablishment +um sy +Ġgather ings +ĠN aj +Charl ie +Ġas cent +ĠProt ector +Ġal gebra +Ġbi os +for ums +EL S +Introdu ced +Ġ3 35 +Ġastron omy +Cont ribut +ĠPol ic +Pl atform +Ġcontain ment +w rap +Ġcoron ary +ĠJ elly +man ager +Ġheart breaking +c air +ĠChe ro +c gi +Med ical +ĠAccount ability +! !" +oph ile +Ġpsych otic +ĠRest rict +Ġequ itable +iss ues +Ġ19 05 +ĠN ek +c ised +ĠTr acking +Ġo zone +Ġcook er +ros is +Ġre open +Ġinf inity +ĠPharm aceutical +ens ional +Att empt +ĠR ory +Mar co +Ġawa its +H OW +t reated +Ġbol st +Ġreve red +Ġp ods +opp ers +00 10 +Ġampl itude +ric an +SP ONSORED +Ġtrou sers +Ġhal ves +ĠK aine +ĠCut ler +ĠA UTH +Ġsplend id +Ġprevent ive +ĠDud ley +if acts +umin ati +ĠY in +Ġad mon +ĠV ag +Ġin verted +Ġhast ily +ĠH ague +L yn +Ġled ger +Ġastron omical +get ting +Ġcirc a +ĠC ic +ĠTenn is +Lim ited +Ġd ru +ĠBY U +Ġtrave llers +Ġp ane +ĠInt ro +Ġpatient ly +Ġa iding +Ġlo os +ĠT ough +Ġ29 3 +Ġconsum es +Source File +Ġ"" " +Ġbond ing +Ġtil ted +Ġmenstru al +ĠCel estial +UL AR +Plug in +Ġrisk ing +N az +ĠRiy adh +Ġacc redited +Ġsk irm +é Ľ +Ġexam iner +Ġmess ing +Ġnear ing +ĠC hern +ĠBeck ham +Ġsw apped +Ġgo ose +K ay +Ġlo fty +ĠWal let +Ġ[ ' +Ġap ocalypse +Ġb amboo +ĠSP ACE +ĠEl ena +Ġ30 6 +ac ons +Ġtight ened +Ġadolesc ence +Ġrain y +Ġvandal ism +ĠNew town +Ġcon ject +c akes +Ġche ated +Ġmoder ators +par ams +E FF +Ġdece it +ĠST L +ĠTanz ania +ĠR I +Ġ19 23 +ĠEx ile +the l +Ġthe olog +Ġquir ky +ĠIr vine +Ġneed y +or is +U m +K a +Ġmail box +3 22 +Ġb os +ĠPet ra +K ING +Ġenlarg ed +O ften +Ġbad ass +Ġ3 43 +ĠPl aces +ĠC AD +Ġpr istine +Ġinterven ing +d irection +Ġl az +ĠD SM +Ġproject ing +ĠF unk +ag og +pay ment +n ov +Ġch atter +AR B +Ġexam inations +ĠHouse hold +ĠG us +F ord +4 14 +B oss +Ġmy stic +Ġle aps +ĠB av +ul z +b udget +Foot ball +Ġsubsid ized +Ġfirst hand +Ġcoinc ide +oc ular +Con n +ĠColl abor +Ġfool s +am ura +ah ar +r ists +Ġsw ollen +Ġexp ended +ĠP au +s up +Ġsp ar +Ġkey note +s uff +Ġunequ al +Ġprogress ing +str ings +ĠGamer gate +Dis ney +ĠEle ven +om nia +Ġscript ed +Ġear ners +bro ther +ĠEn abled +æ ³ +Ġlar vae +ĠL OC +m ess +Wil son +ĠTem plate +success fully +Ġparam ount +Ġcamoufl age +Ġbind s +ĠQu iet +ĠSh utterstock +r ush +Ġmasc ot +fort une +ĠCol t +ĠBe yon +hab i +Ġha irc +Ġ26 7 +ĠDe us +Ġtw itch +Ġconcent rating +Ġn ipples +c ible +Ġg ir +N Z +M ath +n ih +Requ ired +Ġp onder +ĠS AN +Ġwedd ings +Ġl oneliness +N ES +ĠMah jong +69 5 +add le +ĠGar ner +ĠC OUR +Br idge +Ġsp ree +ĠCald well +Ġbri bery +Ġ���� ���� +plug ins +Ġr acket +Ġchamp agne +vers ible +V ote +Ġmod ifiers +May or +6 80 +Ġassemb lies +ĠS ultan +ĠN ing +ĠLad ies +Ġsulf ur +Ġor bs +Ġ---- - +____ ___ +ĠJournal ism +Ġes ports +Ġl ush +Ġh ue +Ġspect ral +H onest +ãĥ ı +Ġbus hes +Ġrein forcement +Ġre opened +ĠWhe els +ĠM org +rie ving +Ġaux iliary +Ġj Query +ĠB AT +tes que +Ġver tex +p ure +f rey +ãĤ º +d os +Ġty ph +Ġc ull +Ġe q +Ġdec on +Ġtoss ing +Ġdispar ate +ĠBr igham +print f +led ged +Ġsu nd +Ġco zy +Ġhepat itis +per forming +Ġav al +ĠG G +f uture +Ġpet ertodd +ĠKos ovo +Ġmagn ets +Al ready +ĠEd ison +ĠCe res +ĠRA ID +Ġbrill iance +57 6 +Ġder ives +Ġhypert ension +ĠÎ Ķ +Ġlamb da +Ġfl air +Ġmission aries +Ġrap es +ĠSt arter +ĠMon ths +Ġdef y +Ġseism ic +ĠR aphael +Ġeuro zone +65 6 +z sche +Ġscr atched +Ġb ows +ĠLenn on +ĠGa ia +Ġdri pping +f acts +A le +Ġfrog s +ĠBre ast +ogene ity +ĠProsecut or +Ġampl ified +ĠHod g +ĠF n +Th ousands +ĠNI H +ĠMonitor ing +FT WARE +ĠPri ebus +ĠG rowing +hun ter +Ġdiagn ose +ĠM ald +ĠL R +Ġcrown ed +Ġburst ing +Ġdiss olution +j avascript +Ġuseful ness +ĠExec ution +: ( +ĠIv ory +a ah +Ġpersecut ed +viol ence +ist as +ĠCr ate +Ġimpuls es +ĠSp ani +ed es +Hand le +ĠZ erg +think able +Last ly +Ġspont aneously +Ġinconven ient +Ġdismiss ing +Ġpl otted +Ġeight y +Ġ7 37 +r ish +ĠThor nton +ath am +Ġsit com +V en +Rec ipe +t el +l und +Ġcle ars +ĠSas uke +Ġ25 8 +Ġopt ing +Ġen raged +est hetic +ĠA e +uch s +Pre p +Fl ow +Ġrun off +ĠE ating +ĠG iles +ĠAct ing +res ources +ib aba +Ġr pm +Ġske wed +ĠBl anc +ĠS akuya +Ġhot ter +Ġ19 24 +op ian +ck o +Ġcr umbling +Ġcapt ains +ĠAppropri ations +le aders +dro pping +an uts +Ġrevers ing +ĠP ose +ĠS ek +Sc ot +ĠIde a +c ise +ĠSloven ia +Ġ3 17 +Do ctor +Ġcro cod +ald i +Se a +ĠFar rell +Ġmerc enaries +ĠR NC +ĠGu ess +Ġp acing +M achine +Streamer Bot +ĠChar ity +Ġ29 8 +Ġcann ons +ĠTob y +TPP StreamerBot +ĠPass ion +cf g +Th om +Ġbad ges +ĠBern stein +. âĢĵ +ĠP OP +ĠCon j +Ġinitial ization +Ġbiod iversity +D ub +Ġfeud al +Ġdisclaim er +Ġc row +Ġign ition +ar f +S HA +Ġk Hz +h azard +ĠArt ists +oe uv +67 9 +ĠRud y +N ine +ĠRam adan +å ½ +itt o +Ġadren aline +C ert +Ġsmell ed +Ġimp unity +Ġag endas +ĠRe born +ĠCon cent +ĠSe ems +Ġo mega +ĠDust in +Ġback er +ĠSau ce +ĠBoy le +W IN +Ġsp ins +Ġpa uses +u pt +Ġshred ded +Ġstra pped +ĠCor ruption +Ġscr atches +Ġn i +Ġatt ire +ĠS AF +Factory Reloaded +ĠI PS +Ġ( % +Ġsem inar +f ocus +c ivil +Ġ18 60 +int osh +Ġcontin ual +Ġabbre vi +ĠS ok +oc obo +X M +Ġfr antic +Ġunavoid able +Ġar tery +Ġannot ations +b ath +Cl imate +Ġd ors +ĠSl ide +co ord +ĠRel oad +ĠL DL +ĠLove craft +Ġunim agin +Ġresemb led +Ġbarr acks +n p +Ġsurrog ate +Ġcategor ized +ãĤ © +Ġvacc inated +Ġdrain age +Ġind ist +ĠWhats App +Ġ18 70 +oler ance +inv oke +am orph +Ġrecon nect +Ġem anc +Ġblind ness +Ġ12 80 +intern et +c ollar +Ġalt ru +Ġab yss +ĠT RI +65 7 +Ġinf used +HE AD +Ġforest ry +ĠWood y +ĠC i +w i +s am +78 4 +hol iday +Ġmog ul +ĠF ees +ĠD EN +In ternal +ur bed +f usc +at om +ĠIll usion +Ġpoll ed +Ġfl ap +Ġco ax +L GBT +An aly +ĠSect ions +ĠCalif orn +em n +Ġh ither +ĠN IGHT +Ġn ailed +ĠPip eline +39 1 +o of +ĠPr imal +vere nd +Ġsl ashing +Ġret ri +avi our +Ġdepart ing +g il +IS C +Ġmid way +Ġultras ound +Ġbeh aving +ĠT ara +class es +V irtual +ĠColon ial +Ġstri pping +Ġorchestr ated +ĠGra ves +45 2 +ĠIron ically +ĠWrit ers +Ġl ends +ĠMan z +Ġra ven +Ġoxid ative +Ġ26 6 +EL F +act ually +asc ar +D raft +Ġfavour able +Ġhumili ating +Ġf idelity +ĠH of +ĠX uan +49 6 +Ġlay ered +at is +79 0 +Ġpay check +it on +K ar +ĠVM ware +ĠFar mer +Ġserv ic +gl omer +Ġsl ump +ĠFab ric +ĠD OC +est ing +Ġreass ure +Ġph yl +v olt +it ory +R ules +Ġoxid ation +Ġpri zed +Ġmist ress +ĠDj ango +WAR N +å ij +Ġenc ode +ĠFeed back +Ġstupid ity +I an +ĠYugoslav ia +× ¨ +ac l +UT E +19 77 +Ġqual ifies +Ġpuls es +pret ty +Ġfro ze +Ġs s +Iter ator +Ġur gently +Ġm ailed +ĠCh am +Ġsust aining +Ġbas il +Ġpupp ies +il ant +ĠP LEASE +l ap +ace ous +F ear +ĠMaster y +aut omatic +ĠT AG +Ġant im +ag les +47 3 +fram es +Ġwh ispers +ĠWho ever +Ġbra very +ĠUK IP +ract ions +"" " +Ġt ame +Ġpart ed +every thing +CON T +Ġind ebted +Ġadd r +re k +IR ED +Ġem inent +cl inton +Ġo usted +Ġreview er +Ġmelt down +Ġre arr +ĠY ao +the real +aby te +Ġst umbling +Ġbat ches +Ġ25 9 +Ġcontrace ptive +Ġprost itute +ens is +De cl +ĠSt rikes +M ilitary +ĠO ath +v acc +pp ings +05 2 +Ġpart Name +amp ing +Rep orts +K I +CH R +Ġsubt ly +sw ers +Bl ake +us ual +Ġcontest ants +Ġcart ridges +ĠGRE AT +Ġbl ush +ĠâĢ º +47 2 +Ġreason ed +ãĥ ¤ +paralle led +Ġd yn +ag ate +Ġnight ly +å Ĩ +55 6 +Ġsem antic +ĠAdv oc +Ġ !! +Ġdisag rees +ĠB W +V eh +Ġharm ing +Ġembr aces +Ġstri ves +Ġin land +ĠK ard +Ġhe ats +ĠGin ny +ut an +ern aut +yl ene +ĠE lev +J D +Ġh ars +ĠStar r +Ġsk ysc +Ġcollabor ators +Us ually +Ġrev olutions +ĠSTAT S +Ġdism antle +Ġconfident ly +Ġkin etic +Al i +Ġpercent ile +Ġextract ing +ill ian +est ead +Ġphysic ists +ĠMarsh al +Ġfell owship +Ġd ashed +ĠU R +ĠSi oux +ĠComp act +am ide +P ython +ĠLe igh +ĠPharm ac +ist rates +her ical +Ġf ue +ĠE min +Ġ( { +ĠNeighbor hood +Ġdisrupt ing +ĠD up +Ġg land +ĠSe v +ĠMar ian +arg on +ĠD und +Ġ< !-- +Ġstr and +Ġstadium s +z os +Ġpsych osis +ĠR ack +Ġbrilliant ly +ï¸ ı +Ġsubmer ged +ĠInst it +ĠCh ow +Ġc ages +ĠH ats +ĠU rs +Ġdil uted +us at +ien ne +ĠMembers hip +ĠBur k +Ġ ie +Ġarche type +D rug +ult on +ĠSp ock +ĠMcK ay +ĠDep end +F eatured +S oc +19 78 +ĠB ere +Ġrelent lessly +Ġcripp ling +Ġar thritis +çĶ Ł +ĠTrop ical +ĠBul g +ĠCher yl +Ġadm irable +Ġsub title +Over ride +Ġorig inating +ĠC CP +Ġsw ore +ĠSo le +ĠDis orders +3 29 +Ġprocess ion +Ġref urb +Ġimm ersed +requ ently +Ġskept ics +Ġcer amic +m itter +en stein +b elt +ĠT IT +b idden +Ġf ir +m ist +> ] +Ġwe ave +ĠParad ox +Ġentr usted +ĠBarcl ays +Ġnovel ist +og ie +80 6 +Ġnin ety +Ġdisag reements +@@@@ @@@@ +ĠAus chwitz +c ars +ĠL ET +t ub +arant ine +P OS +Ġback story +Ġcheer ful +ĠR ag +ek a +bi ased +Ġinexper ienced +ak ra +ĠW itt +t an +Ġrap ist +Ġplate au +ch al +ĠInqu is +exp ression +Ġc ipher +Ġsh aving +add en +re ly +( \ +ism a +ĠReg ulatory +CH AR +ily n +N VIDIA +G U +Ġmur m +la us +Christ opher +Ġcontract ual +ĠPro xy +ĠJa ime +ĠMethod ist +Ġstew ards +st a +per ia +Ġphys iology +Ġbump ed +Ġf ructose +Austral ian +ĠMet allic +ĠMas querade +ar b +Ġprom ul +Ġdown fall +Ġbut cher +Ġb our +ĠIN FORMATION +ĠB is +pect s +ad ena +Ġcontempl ating +ar oo +cent ered +ĠPe aks +Us ed +Ġmod em +Ġg enders +Ġ8 000 +37 1 +Ġm aternity +ĠR az +Ġrock ing +Ġhandgun s +ĠD ACA +Aut om +ĠN ile +Ġtum ult +ĠBenef it +ĠAppro ach +works hop +ĠLe aving +G er +inst ead +Ġvibr ations +Ġrep ositories +49 7 +ĠA unt +ĠJ ub +ĠExp edition +Al pha +Ġs ans +Ġoverd ue +Ġoverc rowd +Ġlegisl atures +Ġp aternal +ĠLeon ardo +Ġexp ressive +Ġdistract ions +Ġsil enced +tr ust +Ġb iking +Ġ5 60 +Ġpropri et +Ġimp osition +Ġcon glomer +Ġ= ================================================================ +ĠTe aching +ĠY ose +int ensive +T own +Ġtroll ing +ĠGr ac +ĠAS US +Y o +Ġspecial s +ĠNep h +ĠGod zilla +Dat abase +ĠHe gel +Ġ27 2 +19 76 +ĠGl oria +Ġdis emb +ĠInvestig ations +ĠB ane +ag ements +St range +Ġtre asury +ĠPl ays +Ġundes irable +Ġwid ening +Ġverb ally +Ġinf ancy +Ġcut ter +f ml +Ġ21 00 +prot otype +f ine +Ġdec riminal +Ġdysfunction al +Ġbes ie +ĠErn st +z eb +Ġnort heastern +Ġa ust +por ate +ĠMar lins +Ġsegreg ated +ew orld +ĠMa her +Ġtra verse +Ġmon astery +ur gy +G ear +s and +Com pl +ĠE MP +Ġpl ent +ĠMer cer +Ġ27 6 +TA BLE +Config uration +H undreds +Ġpr ic +Ġcollabor ating +ĠPar amount +ĠCumm ings +Ġ( < +Ġrecord er +Ġfl ats +Ġ4 16 +wh ose +Font Size +ĠOr bit +Y R +Ġwr ists +Ġb akery +) } +ĠB ounty +ĠLanc aster +Ġend ings +acc ording +ĠSal am +e asy +75 5 +ĠBur r +ĠBarn ett +onom ous +Un ion +Ġpreced ence +ĠScholars hip +ĠU X +Ġroll out +Ġbo on +al m +ĠCan ter +æ µ +Ġround ing +Ġcl ad +Ġv ap +ĠF eatured +is ations +Ġ5 40 +pol ice +Ġunsett ling +Ġdr ifting +ĠLum ia +ĠObama Care +ĠF avor +Hy per +ĠRoth schild +ĠMil iband +an aly +ĠJul iet +H u +Ġrec alling +a head +69 6 +Ġunf avorable +Ġd ances +O x +Ġleg ality +Ġ40 3 +rom ancer +Ġinqu ire +ĠM oves +\ "> +ĠVari ant +ĠMess iah +ĠL CS +ĠBah á +75 6 +Ġeyeb row +Ġ ¥ +ĠMc F +ĠFort y +M as +Ġpan icked +Ġtransform ations +q q +Ġrev olves +ring e +ĠA i +ax e +Ġon ward +ĠC FR +ĠB are +log in +Ġliqu ids +Ġde comp +second ary +il an +ĠCon vert +ami ya +Ġprosecut ing +Ġâī ¡ +ĠYork ers +ĠByr ne +sl ow +aw ei +J ean +Ġ26 9 +ĠSky dragon +Ġ é +ĠNicarag ua +ĠHuck abee +ĠHigh ly +Ġamph ib +ĠPast or +ĠL ets +Ġbl urred +Ġvisc eral +ĠC BO +Ġcollabor ated +z ig +Leg al +Ġapart heid +Ġbr id +Ġpres et +ĠD ET +ĠAM A +× Ķ +arch ing +auc uses +build er +Ġpo etic +Ġem ulator +ĠMole cular +Ġhon oring +ise um +Ġtract or +ĠCl uster +ĠCal m +ared evil +Ġsidew alks +Ġviol in +Ġgeneral ized +ĠAle c +Ġemb argo +Ġfast ball +ĠHT TPS +ĠL ack +ĠCh ill +ri ver +C hel +ĠSw arm +ĠLev ine +ro ying +L aunch +Ġkick er +Ġadd itive +ĠDe als +W idget +cont aining +Ġescal ate +ĠOP EN +Ġtwe aked +Ġst ash +Ġsp arks +ĠEs sex +ĠE cc +Ġconv ict +Ġblog ging +I ER +ĠH L +Ġmurd erers +75 9 +ĠH ib +Ġde pl +ĠJ ord +S ac +Ġdis sect +ĠHow e +os her +Ġcustom izable +ĠFran z +Ġat ro +Ä ĩ +Ġ000 4 +Ġout post +R oss +Ġglyph osate +ĠHast ings +ĠBE FORE +Ġsh ove +o pped +ĠSc ala +Ġam ulet +an ian +Ġexacerb ated +Ġe ater +47 1 +UM E +Ġpul p +izont al +ĠZ am +ĠAT I +imm une +aby tes +Ġunnecess arily +ĠC AT +ĠAx is +Ġvisual ize +à ī +ĠRad ical +f m +Doc uments +ĠFor rest +Ġcontext ual +ĠSy mbol +Ġtent ative +ĠDO ES +ĠGood s +Ġintermitt ent +} : +medi ated +Ġridic ule +Ġathe ism +Ġpath ogens +ĠM um +Ġre introdu +Ġ30 7 +i HUD +Ġflash light +Ġsw earing +Ġp engu +B u +Ġrot ated +ĠCr ane +Ġ() ); +Ġfashion able +Ġendors ing +46 3 +) [ +Ġingest ion +Ġcook s +Ġ9 50 +ot omy +ĠIm am +Ġk a +Ġte aser +ĠGhost s +ĠãĤ µ +19 69 +Ï ĥ +ub by +Ġconver ter +zan ne +end e +ĠPre par +ĠNic kel +ĠChim era +h im +ĠTyr ann +ĠSabb ath +ĠNich ols +Ġra pt +ih ar +Ġshe lling +Ġillum inate +Ġdent ist +ut or +ĠInteg ration +Ġwh ims +ĠLiter ary +Be aut +Ġp archment +ag ara +Br and +Ġder og +âĢ¦ ) +ĠNor se +Ġunw itting +Ġc uc +Ġborder line +Ġupset ting +Ġrec ourse +Ġd raped +ĠRad ar +Ġcold er +ĠPep si +im inary +], [ +65 8 +V i +ĠF rem +ĠP es +Ġveter inary +ĠT ED +ĠEp idem +n ova +k id +Ġdev out +o ct +j ad +M oh +ĠP AY +Ġge ometric +Ġ3 23 +Ġcircum ference +ich ick +19 75 +ĠY uri +ĠSh all +ĠH over +un in +S pr +Ġg raft +ĠHapp iness +Ġdisadvant ages +att acks +Ġhub s +ĠStar Craft +é ĸ +Ġgall eries +ĠKor ra +Ġgrocer ies +ĠGors uch +Ġrap ists +Ġfun gi +ĠTyph oon +V ector +ĠEm press +b attle +4 68 +Ġparas ite +ĠBom ber +S G +ex ist +ĠP f +Ġun se +Ġsurge ons +B irth +ĠUn sure +ĠPrint ed +ĠBehavior al +ĠA ster +Pak istan +Ġun ethical +Ġs v +ĠIo T +Ġlay outs +P ain +Ġconst ants +ĠL W +ĠB ake +Ġtow els +Ġdeterior ation +ĠBol ivia +Ġblind ed +ĠW arden +ĠMist ress +Ġon stage +Ġcl ans +ĠB EST +19 60 +Ġant ique +Ġrhet orical +ĠPer cy +ĠRw anda +, . +B ruce +Ġtra umat +ĠParliament ary +Ġfoot note +id ia +ĠLear ned +se eking +gen ic +Ġdim ensional +H ide +èĢ ħ +Ġintrig ue +in se +Ġle ases +Ġapp rentices +w ashing +Ġ19 26 +V ILLE +Ġsw oop +s cl +Ġbed rooms +on ics +ĠCr unch +comp atible +Ġincap ac +ĠYemen i +ash tra +z hou +d anger +Ġmanifest ations +ĠDem ons +AA F +Secret ary +ACT ED +L OD +Ġam y +ra per +eth nic +4 17 +Ġpos itives +Ġ27 3 +ĠRefuge es +Ġus b +ĠV ald +odd y +ĠMahm oud +As ia +Ġskull s +ĠEx odus +ĠComp et +ĠL IC +ĠM ansion +ĠA me +Ġconsolid ate +storm s +ont ent +99 6 +Ġcl en +Ġm ummy +fl at +75 8 +ĠV OL +oter ic +n en +ĠMin ute +S ov +Ġfin er +R h +ly cer +Ġreinforce ments +ĠJohann es +ĠGall agher +Ġgym n +S uddenly +Ġext ortion +k r +i ator +T a +Ġhippocamp us +N PR +ĠComput ing +Ġsquare ly +Ġmod elling +ĠFor ums +ĠL isp +ĠKrish na +Ġ3 24 +Ġr ushes +Ġens ued +Ġcre eping +on te +n ai +il ater +ĠHorn ets +Ġob livious +IN ST +55 9 +Ġjeopard y +Ġdistingu ishing +j ured +Ġbeg s +sim ilar +ph ot +5 30 +ĠPark way +Ġs inks +ĠHearth stone +ib ur +ĠBat on +Av oid +Ġd ancer +Ġmag istrate +ary n +Ġdisturb ances +ĠRom ero +Ġpar aph +Ġmis chief +âĸ ĵ +ĠSh aria +Ġur inary +r oute +iv as +f itted +Ġeject ed +ĠAl buquerque +Ġ4 70 +Ġirrit ated +ĠZ ip +ĠB iol +à į +Ġden ounce +Ġbin aries +ĠVer se +Ġopp os +ĠKend rick +ĠG PL +Ġsp ew +ĠEl ijah +ĠE as +Ġdr ifted +so far +Ġannoy ance +ĠB ET +47 4 +ĠSt rongh +it ates +ĠCogn itive +oph one +ĠIdent ification +ocr ine +connect ion +Ġbox er +ĠAS D +ĠAre as +Y ang +t ch +ull ah +Ġdece ive +Comb at +ep isode +cre te +W itness +Ġcondol ences +ht ar +Ġhe als +Ġbuck ets +ĠLA W +B lu +Ġsl ab +ĠOR DER +oc l +att on +ĠSteven son +ĠG inger +ĠFriend ly +ĠVander bilt +sp irit +ig l +ĠReg arding +ĠPR OG +Ġse aling +start ing +Ġcard inal +ĠV ec +ĠBe ir +Ġmillisec onds +we ak +per se +Ġster ile +ĠCont emporary +ĠPh ant +ĠCl o +Ġout p +Ġex iled +Ġ27 7 +Ġself ie +Ġman ic +Ġn ano +ter ms +Alex ander +Ġres olves +Ġmillenn ia +Ġexpl odes +Ġconst ellation +Ġadul tery +m otion +D OC +Ġbroad casters +Ġkinderg arten +ĠMay weather +ĠE co +ich o +Ġ28 7 +l aun +Ġm ute +Ġdisc reet +Ġpres chool +Ġpre empt +De lete +ĠFre ed +P i +H K +Ġblock er +ĠC umber +Ġw rought +d ating +Ġins urer +Ġquot as +Ġpre ached +Ġev iction +ĠReg ina +ĠP ens +Ġsevent een +ĠN ass +D ick +Ġfold s +Ġd otted +ĠA ad +Un iversal +Ġp izz +ĠG uru +Ġso ils +Ġno vice +ĠNe ander +Ġst ool +Ġdeton ated +ĠPik achu +ĠMass ive +IV ER +ĠAb del +Ġsubdu ed +Ġtall est +Ġprec arious +Ġa y +r ification +ĠOb j +c ale +Ġun question +cul osis +ad as +igr ated +D ays +Ġque ens +ĠGaz ette +ĠCol our +ĠBow man +ĠJ J +ï ve +Ġdomin ates +Stud ent +Ġm u +Ġback log +ĠElect ro +Tr uth +48 3 +Ġcond ensed +r ules +ĠCons piracy +Ġacron ym +hand led +ĠMat te +j ri +ĠImp ossible +l ude +cre ation +Ġwar med +ĠSl ave +Ġmis led +Ġfer ment +ĠK ah +ink i +ke leton +cy l +ĠKar in +Hun ter +Reg ister +ĠSur rey +Ġst ares +ĠW idth +ĠN ay +ĠSk i +Ġblack list +uck et +Ġexp ulsion +im et +Ġret weet +vant age +Fe ature +Ġtro opers +Ġhom ers +9 69 +Ġconting ency +ĠW TC +ĠBrew er +fore ign +W are +S olar +Ġund ue +RE C +ulner able +path ic +ĠBo ise +Ġ3 22 +Ġarous ed +ĠY ing +ä¸ į +uel ess +Ġp as +Ġmor p +Ġfl oral +Ex press +ud ging +k B +ĠGr anted +Ø ¯ +ĠMich a +ĠGoth ic +ĠSPEC IAL +ĠRic ardo +F ran +Ġadminister ing +6 20 +por a +Ġ ® +Ġcomprom ises +Ġb itten +Ac cept +Th irty +Ð ² +Ġmater ially +ĠTer r +ig matic +ch ains +Ġdo ve +stad t +Mar vel +FA ULT +Ġwind shield +Ġ3 36 +ad ier +Ġsw apping +Ġflaw less +ĠPred ator +ĠMiche le +Ġprop ulsion +ĠPsych ic +Ġassign ing +Ġfabric ation +Ġbar ley +l ust +Ġtow ering +Ġalter cation +ĠBent ley +Sp here +Ġtun a +ĠClass es +Fre edom +un er +L ady +v oice +Ġcool est +or r +Ġpal p +$ { +Ġhyster ia +ĠMet atron +p ants +Ġspawn ing +Exper ts +ĠInvest ors +ĠAn archy +Ġshr unk +ĠVict im +Ġ28 9 +Ġec stasy +ĠB inding +58 5 +ĠMel ody +57 8 +ot ally +ĠE tsy +lig a +Ġapplaud ed +Ġswe ating +Ġredist ributed +Ġpop corn +Ġsem inal +f ur +ĠNeuro science +R and +ĠO st +ĠMadd en +ĠIncre asing +ĠDaw kins +ĠSub way +Ġar sen +cons erv +B UR +Ġsp iked +ĠLy ft +ĠImper ium +ĠDrop box +Ġfav oured +Ġencomp asses +gh ost +Ġins pires +Ġbur geoning +ĠY oshi +ĠVert ical +ĠAud itor +Ġint ending +Ġfilib uster +Bl oom +f ac +ĠCav s +ign ing +Ġcowork ers +ĠBarb arian +rem ember +FL AG +Ġaudit ory +ason ry +Col lege +Ġmut ed +gem ony +ob in +ĠPsych o +9 68 +Ġlav ish +Ġhierarch ical +ĠDr one +ou k +Ġcripp led +ĠMax im +Sl ot +Ġqu iz +ĠV id +if ling +Ġarchae ologists +Ġabandon ment +d ial +le on +ĠF as +T ed +Ġr aspberry +Ġmaneu vers +Ġbehavi ours +Ġins ure +Ġrem od +Sw itch +h oe +Ġsp aced +Ġafford ability +ĠF ern +not ation +ĠBal anced +Ġoccup ies +en vironment +Ġneck lace +Ġsed an +F U +ĠBrav o +Ġab users +ĠAn ita +met adata +ĠG ithub +ait o +ĠF aster +ĠWass erman +ĠF lesh +Ġth orn +r arily +ĠMer ry +w ine +Ġpopul ace +ĠL ann +Ġrepair ing +Ġpsy che +Ġmod ulation +aw aru +âĢĭ âĢĭ +ari j +Ġdecor ations +Ġapolog ise +ĠG arg +app ly +Ġgive away +ĠFl an +ĠWy att +U ber +Ġauthor ised +ĠMor al +HAHA HAHA +activ ate +Ġtorped o +ĠF AR +Ġam assed +ĠA ram +ark in +ĠVict ims +st ab +Ġo m +ĠE CO +Ġopio ids +Ġpurpose ly +ĠV est +Ġer g +at an +ĠSur gery +Ġcorrect ing +ĠOrt iz +ĠBe et +Ġrev oke +Ġfre eway +ĠH iggins +F ail +ĠFar ms +ĠAT P +h ound +Ġp oking +ĠCommun ists +mon ster +iment ary +Ġunlock ing +Ġunf it +we ed +en ario +at ical +ĠEnlight enment +ĠN G +ĠComp ensation +de en +ĠWid ow +ĠCind y +ĠAfter wards +Ġ6 000 +ikh ail +ag ically +Ġrat ified +Ġcasual ty +H OME +p sey +f ee +Ġspark ling +Ġd é +Ġconcert ed +C atal +Ġcomp lying +ĠA res +ĠD ent +Sh ut +Ġsk im +ad minist +Ġhost ilities +ĠG ins +Ġ6 08 +Ġm uddy +ĠMc Int +ĠDec ay +5 25 +Ġconspic uous +ĠEx posure +Ġresc ind +Ġwear able +Ġ3 28 +our met +ah s +ĠRob ots +Ġe clips +inst ance +ĠRE PORT +ĠApp l +0 30 +ĠSk ies +01 00 +Ġfall acy +S ocket +ĠRece iver +Ġsol ves +ĠButter fly +ĠSho pping +ĠFI RE +65 4 +Med ic +Ġsing ers +ĠNeed less +'' '' +isher s +ĠD ive +58 8 +Ġselect ively +Ġcl umsy +88 9 +Ġpurch aser +ear ned +ard y +Ġbenef iting +eng lish +Ġyield ing +ĠP our +Ġspin ach +Ġdel ve +ĠC rom +6 10 +Ġexport ing +ĠMA KE +Ġ26 3 +Ġg rop +Ġenv oy +ĠInqu iry +ĠLu igi +d ry +ĠT uring +Thumbnail Image +ĠVar iety +Ġfac et +Ġfl uffy +Ġexcerpt s +Ġsh orth +ĠOl sen +CL UD +Ġrel iant +ĠUN C +T our +Ġbat hing +Comp any +Ġglobal ization +P red +ĠMalf oy +Ġh oc +j am +craft ed +ĠBond s +ĠKiss inger +Eng land +Ġorder ly +cat entry +Ġ26 1 +Ġexch anging +ĠInt ent +ĠAmend ments +D OM +Ġst out +³³³³³³³³ ³³³³³³³³ +ĠAir bus +Ġ27 8 +hy de +P oll +Item ThumbnailImage +Ġlooph oles +ĠPill ar +Ġexpl or +St retch +A part +Ġun married +Lim it +ĠTransform ers +Ġintellect ually +unct ure +18 00 +Ġd arn +B razil +Ġleft over +ber us +f red +Mine craft +3 26 +ĠForm s +Ġproof s +ĠDes igned +Ġindex es +ĠSupp ose +EM S +ĠL oving +ĠBon nie +im ating +OT US +Ġconduct or +Ġbehav ed +ĠF ren +Ġsy nerg +Ġmillenn ium +Ġcater ing +ĠL auder +W r +ĠY iannopoulos +ĠAT F +Ġensl aved +Ġawaken ed +D VD +ĠED ITION +ĠConc ert +ĠChall enger +ĠH aku +umer ic +Ġdep recated +ĠSH AR +4 12 +Ġdy stop +Ġtremb ling +Ġdread ed +ĠSp ac +p adding +Re pl +ĠG arrison +M ini +Ġun paralleled +am ar +URR ENT +w reck +c ertain +t al +ĠC LS +app ings +Ġsens ed +Ġf encing +ĠPas o +ĠDes k +Ġsc off +Ġcontem plate +ĠL iga +l iquid +75 7 +Ġapp rentice +ĠUCH IJ +5 70 +ĠTh ousand +ĠIll um +Ġchampion ed +ãĤ Į +Ġelect ors +Ġ3 98 +ĠH ancock +round ed +ĠJ OHN +Ġuns atisf +Ġqual ifier +ĠGad get +EN E +Ġdead liest +ĠPl ants +Ġ ions +Ġacc ents +Ġtwe aking +Ġsh aved +F REE +ĠCh aser +Again st +9 60 +Ġmeth amphetamine +Ġnormal ized +Ġ$ \ +ĠPre cision +ĠGu am +Ġch oked +ĠX II +ĠCast ing +Tor rent +Ġscal p +ĠJagu ar +w it +Ġsem ic +ix ie +ĠG ould +Ġconf ines +N usra +ĠL on +ĠJ ugg +y cle +ĠCod ec +E gypt +Ġrest rain +ĠAl iens +Ġch oking +ĠD unk +ĠBell a +ab c +Ġsl ang +Ġneuro trans +s av +Ġempower ment +â ĨĴ +Ġclim bers +ĠM im +ĠF ra +ros se +Cap ital +ĠCth ulhu +Inter face +Ġprof icient +ĠIN TO +Ġ3 18 +ront al +5 80 +ĠDes pair +K enn +Ġscrim mage +ĠCo at +as ions +Ġwall paper +ĠJ ol +Ġresurg ence +Ġant iv +ĠB alls +² ¾ +Ġbuff ers +Ġsub system +ĠSt ellar +ĠL ung +A IDS +Ġerad icate +Ġblat antly +Ġbehav es +ĠN un +Ġant ics +ex port +DE V +w b +Ġph p +ĠInteg rity +Ġexplore r +Ġrev olving +auth ored +g ans +Ġbas k +Ġas ynchronous +å į +TH ING +69 8 +G ene +ĠR acer +ĠN ico +iss ued +Ġser mon +p ossibly +Ġsize of +Ġentrepreneur ial +ox in +ĠMin erva +Ġpl atoon +n os +ri ks +A UT +ĠAval anche +ĠDes c +ij 士 +ĠP oc +Ġconf erred +Î » +Ġpat ched +F BI +66 2 +Ġfract ures +Ġdetect s +Ġded icate +Ġconstitu ent +Ġcos mos +W T +Ġswe ats +Ġspr ung +b ara +s olid +Ġuns us +Ġbul ky +ĠPhilipp e +ĠFen rir +Ġtherap ists +ore al +^^ ^^ +Ġtotal ed +Ġboo ze +ĠR PC +Prosecut ors +Ġdis eng +ĠSh ared +Ġmotor cycles +Ġinvent ions +Ġlett uce +ĠMer ge +ĠJ C +Ġspiritual ity +ĠWAR NING +Ġunl ucky +ĠT ess +Ġtong ues +ĠD UI +T umblr +Ġle ans +Ġinv aders +Ġcan opy +ĠHur ricanes +ĠB ret +ĠAP PLIC +id ine +ick le +Reg arding +Ġve ggies +Ġe jac +ju ven +F ish +D EM +ĠD ino +Th row +ĠCheck ing +be ard +( & +Ġj ails +Ġh r +trans fer +iv ating +Ġfle ets +ĠIm ag +ĠMc Donnell +Ġsnipp et +Is a +ĠCh att +ĠSt ain +ĠSet FontSize +ĠO y +ĠMathemat ics +49 4 +Ġelectro ly +ĠG ott +ĠBr as +B OOK +ĠF inger +d ump +Ġmut ants +Ġrent als +Ġinter tw +Ġc reek +ail a +Bro ther +ĠDisc ord +pe e +raw ler +Ġcar p +Ġ27 9 +ãĤ· ãĥ£ +rel ations +Ġcontr asts +Col umn +Ġrec onnaissance +Ġun know +Ġl ooting +Ġregul ates +Ġopt imum +ĠChero kee +ĠA ry +Lat est +Ġroad side +Ġd anced +ĠUnic orn +A cknowled +Ġuncont roll +ĠM US +at io +ch ance +ha ven +VAL UE +Ġfavour ites +Ġceremon ial +b inary +pe ed +wood s +EM P +Ġv ascular +Ġcontempl ated +Ġbar ren +ĠL IST +Y ellow +ospons ors +Ġwhisk y +ĠM amm +ĠDeV os +min imum +H ung +44 2 +P ic +ĠSnap dragon +77 6 +Ġcar ving +Ġund ecided +Ġadvantage ous +Ġpal ms +ĠA Q +Ġst arch +L oop +Ġpadd le +Ġfl aming +ĠHor izons +An imation +bo ost +Ġprob abilities +ĠM ish +Ġex odus +ĠEditor ial +Ġfung us +Ġdissent ing +ĠDel icious +rog ram +ĠD yn +d isk +t om +Ġfab rics +ĠC ove +ĠB ans +Ġsoft en +ĠCON S +Ġin eligible +Ġestim ating +ĠLex ington +pract ice +of i +Ġshe dding +ĠN ope +Ġbreat hed +ĠCorinth ians +y ne +ek i +B ull +Ġatt aching +reens hots +Ġanaly se +ĠK appa +Ġuns ustainable +Ġinter pol +ank y +he mer +Ġprot agonists +Ġform atted +ĠBry ce +ĠAch illes +ĠAb edin +sh ock +Ġb um +b os +qu a +ĠW arn +q t +ĠDi abetes +8 64 +ĠIn visible +Ġvan ish +Ġtrans mitting +Ġmur ky +ĠFe i +Ġawa ited +ĠJur assic +umm ies +Ġmen acing +g all +C ath +B uilt +ild o +ĠV otes +Ġon t +Ġmun itions +ĠFre em +ÃŃ n +Ġdec ency +lo pp +ie ved +ĠG ord +Ġun thinkable +ĠNews week +Ġ3 21 +He at +Ġpresent er +ji ang +Ġpl ank +ĠAval on +Ġben z +ĠR out +Ġslam ming +ĠD ai +ou ter +ĠCook ie +ĠAlic ia +ge y +Ġvan ity +Ġow l +á µ +t ested +ĠAw akens +Ġcan v +Ġblind ly +ĠRid ley +ĠEm ails +Requ ires +ĠSer bian +ograp hed +if rame +eter ia +Ġaltern ating +qu iet +Ġsoc iology +ĠUn lock +ĠCommun ism +Ġo ps +Ġatt ribution +Ġab duction +ĠAb ram +Ġsidel ined +ĠB OOK +Ġref ining +ĠFe eling +ĠOs lo +ĠPru itt +r ack +ang ible +Ġcaut iously +ĠM ARK +eed s +M ouse +ĠStep h +ĠP air +S ab +99 7 +ĠBa al +B ec +Ġcomm a +ĠP all +ĠG ael +Ġmisunder stand +ĠP esh +Order able +Ġdis mal +ĠSh iny +% " +Ġreal istically +Ġpat io +ĠG w +ĠVirt ue +Ġexhaust ing +wh atever +oph ys +y ip +4 18 +Ad just +ĠWa iting +ess on +ĠMaz da +ĠDo zens +Ġstream lined +Ġincompet ence +ĠM eth +Ġeth os +ON ES +Ġincent iv +Ġgr itty +ĠBut cher +Head er +Ġexp onential +à Ł +Ġcorrel ate +Ġcons ensual +s ounding +R ing +Orig in +Ġcon clusive +fe et +ac ly +ĠF ernandez +Buy able +Ġd ucks +aunt lets +Ġel ong +Ġ28 6 +Ġsim ul +G as +ĠK irst +Ġprot r +ĠRob o +ĠAo E +op ol +Ġpsych ologically +sp in +ilater ally +ĠCon rad +W ave +44 1 +ĠAd vertisement +ĠHarm on +ĠOri ental +is Special +Ġpresum ptive +Ġw il +ĠK ier +ne a +Ġp pm +Ġhar bour +ĠW ired +comp any +Ġcor oner +atur days +ĠP roud +ĠN EXT +ĠFl ake +val ued +ce iver +Ġfra ught +Ġc asing +Ġrun away +Ġg in +ĠLaure nt +ĠHar lem +ĠCur iosity +qu ished +Ġneuro science +ĠH ulu +Ġborrow er +Ġpetition er +ĠCo oldown +W ARD +Ġinv oking +conf idence +For ward +Ġst s +pop ulation +Delivery Date +Fil m +ĠC ov +quick Ship +quickShip Available +prim ary +isSpecial Orderable +inventory Quantity +channel Availability +BO X +ĠMulti player +ĠJen ner +77 8 +ĠM d +Ġ~ /. +M N +Ġchild ish +Ġantioxid ant +ĠChrom ebook +Ġ27 4 +Ġscreen play +Ġadvent urous +ĠRelations hip +respons ive +ming ton +Ġcorner stone +ĠF ey +F IR +Ġrook ies +ĠF eaturing +Ġorig inate +Ġelectro des +ant es +Ġscript ures +Ġgl ued +Ġdiscont ent +Ġaff licted +lay out +B rave +Ġm osa +ĠQuant ity +ĠH ik +w inner +H ours +Ġent ail +ĠCell s +olog ue +Ġv il +Ġpre acher +Ġdecor ative +d ifferent +Ġprejud ices +ĠSm oking +ĠNotting ham +so Type +Ġrhyth ms +ĠAl ph +bl ast +Ste el +ĠDaniel le +Ġstr ife +Ġrem atch +so DeliveryDate +ĠF ork +t rip +ol ulu +hes es +C G +ĠPOLIT ICO +ost a +ĠDr ift +é¾įå ¥ +é¾įå¥ ij士 +Ġvet ting +ĠJin ping +ĠRec ession +Min or +ĠF raud +enf ranch +Ġconven ed +ĠNA ACP +ĠMill ions +ĠFarm ing +ĠW oo +ĠFl are +rit o +imm igrant +Ġvac ancy +ĠHE AD +ĠV aj +eg al +ĠV igil +Stud y +Ġru ining +Ġr acks +Ġhe ater +ĠRand olph +ĠBr ush +ĠT ir +Ø ¨ +Ġc ov +% ] +Ġrecount s +ĠO PT +ĠM elt +Ġtr uce +Ġcas inos +Ġcrus ade +Ġcarn age +Ġstri pe +ĠK yl +Text ures +Ġ6 98 +Ġpro clamation +Ġgood ies +Ġ........ .. +pro claimed +P olit +Ġtop ical +Ġspecial ize +ĠA min +g m +Ġanch ored +Ġbear ings +s ample +ĠHigh land +ĠAut ism +Ġmerc enary +Ġinterview er +L ER +ĠSom ers +Ġembry o +ĠAss y +Ġ28 1 +ĠEd iting +ĠCh osen +6 60 +Ġp ci +ĠThunder bolt +BI LL +Ġchuck led +jri wal +h of +Ġearth ly +() { +ind ependence +Ġdisp ers +ĠV endor +ĠG areth +Ġp als +P enn +ĠSub mit +ic um +Th u +Ġcl andestine +Ġcann ibal +ĠCl erk +E Stream +gal itarian +âĻ ¥ +g ew +Ġhor rend +ĠL ov +ĠRe action +ocr in +Class ic +Ġecho ing +Ġdiscl osing +ĠIns ight +og un +ĠInc arn +upload s +pp erc +guy en +Ġ19 01 +ĠB ars +68 7 +Ġb ribes +ĠFres no +ur at +ĠRe ese +Ġintr usive +Ġgri pping +ĠBlue print +ĠR asm +un ia +man aged +ĠHeb do +Ġ3 45 +Ġdec oding +Ġpo ets +Ġj aws +ĠF IGHT +am eless +ĠMead ows +ĠHar baugh +Inter view +ĠH osp +ĠB RA +Ġdelet ion +m ob +W alker +ĠMoon light +ĠJ ed +ĠSoph ia +Ġus ur +Ġfortun ately +ĠPut ting +ĠF old +Ġsan itation +Ġpart isans +IS ON +B ow +ĠCON C +ĠRed uced +ĠS utton +Ġtouch screen +Ġembry os +âĢ¢âĢ¢ âĢ¢âĢ¢ +ĠK rug +com bat +ĠPet roleum +Ġam d +ĠCos mos +Ġpresc ribing +Ġconform ity +ours es +Ġplent iful +Ġdis illusion +ĠEc ology +itt al +Ġf anc +Ġassass inated +regn ancy +Ġperenn ial +ĠBul lets +Ġst ale +Ġc ached +ĠJud ith +ĠDise ases +All en +Ġl as +Ġsh ards +ĠSu arez +ĠFriend ship +inter face +ĠSupp orters +add ons +46 2 +ĠIm ran +ĠW im +Ġnew found +ĠM b +An imal +Ġd arling +and e +Ġrh y +ĠTw isted +pos al +yn ski +Var ious +× ľ +ĠK iw +uy omi +Ġwell being +ĠL au +an os +Ġunm ist +Ġmac OS +Ġrest room +ĠOl iv +ĠAir ways +Ġtimet able +9 80 +Ġrad ios +v oy +ias co +Ġcloud y +ĠDraw ing +Any thing +Sy ria +ĠH ert +st aking +Ġun checked +Ġb razen +ĠN RS +69 7 +onom ic +est ablish +Ġl eng +Ġdi agonal +ĠF ior +L air +ĠSt ard +Ġdef icient +jo ining +be am +Ġomn ip +Ġbl ender +Ġsun rise +Mo ore +ĠF ault +ĠCost ume +ĠM ub +Fl ags +an se +Ġpay out +ĠGovern ors +ĠD illon +ĠBan ana +N ar +Ġtra iled +Ġimperial ist +um ann +ats uki +4 35 +ĠRoad s +Ġsl ur +ĠIde ally +Ġt renches +C trl +Ġmir rored +ĠZ el +ĠC rest +Comp at +ĠRoll s +sc rib +ĠTra ils +omet ers +w inter +Ġimm ortality +il ated +Ġcontrad icts +un iversal +ill ions +ĠM ama +opt im +AT URE +Ġge o +et ter +ĠCar lo +4 24 +Ġcanon ical +ĠStrongh old +n ear +Ġperf ume +Ġorche stra +od iac +Ġup he +Ġreign ing +vers ive +Ġc aucuses +ĠD EM +Ġinsult ed +Ġ---- -- +ĠCr ush +Ġroot ing +ĠWra ith +Ġwh ore +Ġto fu +C md +ĠB ree +Ġ$ _ +Ġr ive +ĠAd vertising +Ġw att +ĠH O +Ġpersu asive +ĠParam eters +Ġobserv ational +ĠN CT +ĠMo j +ĠSal on +Ġtr unc +Ġexqu isite +ĠMar a +Ġpo op +ĠAN N +Ex c +ĠWonder ful +ĠT aco +Ġhome owner +ĠSmith sonian +orpor ated +mm mm +Ġlo af +ĠYam ato +ĠInd o +Ġcl inging +á s +Ġimm utable +h ub +Or ange +Ġfingert ips +ĠWood en +ĠK idd +ĠJ PM +ĠDam n +C ow +c odes +48 2 +Ġiniti ating +ĠEl k +ĠCut ting +Ġabsent ee +ĠV ance +ĠLil ith +G UI +Ġobsc ured +Ġdwar ves +ĠCh op +ĠB oko +Val ues +Ġmult imedia +Ġbrew ed +Reg ular +CRIP TION +ĠMort al +Ġa pex +Ġtravel er +Ġbo ils +Ġspray ing +Rep resent +ĠStars hip +4 28 +Ġdisappro val +Ġshadow y +Ġlament ed +ĠRe place +ĠFran ç +67 7 +d or +Ġunst oppable +Ġcoh orts +gy n +ĠClass ics +ĠAm ph +Ġsl uggish +ĠAdd iction +ĠPad res +Ġins cription +Ġin human +min us +ĠJere miah +at ars +Ter ror +ĠT os +ĠSh arma +ast a +c atch +Ġpl umbing +ĠTim bers +Sh ar +H al +ĠO sc +Ġcou pling +hum ans +Ġsp onge +Ġid ols +ĠSp a +ĠAdv ocate +ĠBe ats +lu a +Ġtick ing +Ġload er +ĠG ron +8 10 +Ġstim ulated +Ġside bar +ĠManufact urer +ore And +19 73 +Ġpra ises +ĠFl ores +dis able +ĠElect rical +ra ise +E th +Ġmigr ated +Ġlect urer +K ids +ĠCa vern +Ġk ettle +Ġgly c +ĠMand ela +ĠF ully +å§ « +FIN EST +Ġsquee zing +ĠRy der +amp oo +oreAnd Online +Inst oreAndOnline +Buyable InstoreAndOnline +Ġcommem orate +ĠRamp age +Aust in +ĠSh roud +ĠRu ins +9 15 +ĠK H +Ġwater front +ĠE SC +b aby +ĠC out +ĠEm blem +Ġequival ents +49 2 +Un ique +ĠNiet zsche +brow ser +Ġim itation +ĠWere wolf +ĠKir in +ac as +' ," +Ġà ¾ +Review ed +Ġc unt +Ġvo ic +ĠLen ovo +Ġbond ed +48 1 +Ġinhib itors +Ġendeav ors +ĠHav ana +ĠSt out +ĠJ olly +A ctor +*/ ( +Ġoccur rences +ĠT ens +Incre ased +ĠACT ION +Ġ ãĢĮ +ĠRank ings +ĠB reat +Ġ30 9 +D ou +Ġimpact ing +ĠDuc hess +pre fix +Q B +Ġsummon ing +Ġbest owed +ĠKe pler +ĠPOW ER +c ube +ĠK its +ĠG rip +Ġop ium +Ġrep utable +t oc +ich ael +ĠR ipple +Ġcaf é +ĠZ oom +ĠBur ma +Ġwa ive +Ġst alls +Ġdem eanor +inc erity +Ġfluor ide +ĠSH OULD +Par is +Ġlong ing +Ġpl at +Ġgross ly +Ġbull s +Ġshowc asing +ex pected +ĠG addafi +engine ering +Re peat +ĠK ut +Ġconce ivable +Ġtrim med +osc ope +ĠCand idate +ĠT ears +rol og +Lew is +S UP +Ġroad map +Ġsal iva +Ġtrump et +Jim my +Ġmirac ulous +Ġcolon ization +Ġam put +ĠGN OME +ate ch +D ifferent +ĠE LE +ĠGovern ments +ĠA head +ãħĭ ãħĭ +word press +L IB +ĠIn clude +ĠDor othy +0 45 +ĠColomb ian +Ġle ased +88 4 +Ġde grading +ĠDa isy +i ations +Ġbapt ized +Ġsurn ame +co x +Ġblink ed +ãĥ ¢ +Ġpoll en +Ġder mat +Ġre gex +ĠNich olson +ĠE ater +ç ľ +rad or +Ġnarrow er +Ġhur ricanes +Ġhalluc inations +r idden +ISS ION +ĠFire fly +Ġattain ment +Ġnom inate +Ġav ocado +ĠM eredith +Ġt s +Ġreve rence +Ġe uph +Ġcr ates +ĠT EXT +Ġ4 43 +Ġ3 19 +J SON +iqu ette +Ġshort stop +ic key +Ġpro pelled +Ġap i +ĠTh ieves +77 9 +Ġovers aw +Ġcol i +ĠNic ola +Ġover cl +ik awa +ĠC yr +Ġ38 4 +78 9 +ĠAll ows +10 27 +Det roit +TR Y +set up +ĠSocial ism +Sov iet +s usp +ĠAP R +ĠShut down +Ġal uminium +zb ek +ĠL over +GGGG GGGG +Ġdemocr acies +Ġ19 08 +ĠMer rill +ĠFranco is +gd ala +Ġtraff ickers +ĠT il +ĠGo at +Ġsp ed +ĠRes erv +Ġpro d +55 2 +Ġc ac +ĠUn iv +ĠSch we +Ġsw irling +ĠWild erness +ĠEgg s +Ġsadd ened +Ġarch aic +H yd +Ġexcess ively +B RE +Ġaer ospace +ĠVo ices +Cra ig +Ġign ited +In itially +ĠMc A +Ġhand set +Ġreform ing +Ġfrust rations +ĠDead pool +ĠBel ichick +ract or +ĠRagnar ok +ĠD rupal +ĠApp roximately +19 20 +ĠHub ble +arm or +ĠSar as +ĠJon as +Ġnostalg ic +Ġfeas ibility +Sah aran +Ġorb iting +Ġ9 70 +R u +Ġsh in +ĠInvestig ators +Ġinconsist encies +ĠP AN +B G +Ġgraz ing +Ġdetect ors +ĠStart up +ĠFun ny +ĠNa omi +Consider ing +Ġh og +ut f +ce mic +Ġfort ified +ĠFun ctions +Ġcod ec +nut rition +H at +" ! +micro soft +55 8 +ĠTh in +ĠA CE +Al ias +ĠO PS +p apers +P K +ãĢ İ +Ġimpro bable +N orthern +equ al +Ġlook out +Ġty res +ĠMod ified +ĠK op +Abs olutely +Ġbuild up +sil ver +Ġaud i +Ġgro tesque +ĠSab er +ĠPres byter +ON Y +Ġglac iers +ĠSho als +ĠK ass +ĠH RC +ĠNic ol +ĠL unch +ĠF oss +âĸ Ĵ +AD RA +ĠOne Plus +o ing +ground s +Ġincident al +Ġdatas ets +68 9 +ĠClarks on +Ġassemb ling +ĠCorrect ions +Ġdrink ers +Ġqual ifiers +Ġle ash +Ġunf ounded +ĠH undred +Ġkick off +T i +Ġrecon cil +ĠGr ants +ĠCompl iance +ĠDexter ity +Ġ19 06 +w arn +D allas +Max imum +n ard +av ia +be aut +ens itivity +tr ace +Ġpione ers +ĠF ract +ãĢ ı +Ġpre cept +Ġgloss y +ĠI EEE +Ac ross +Ġ6 80 +S leep +che on +Ġsatir ical +ĠMin otaur +ĠCla ude +Ġr é +ape go +Ġcar rot +ĠSem in +ino a +Ġz o +Ind ependent +Ġdiagn oses +ĠC ue +M AR +Ġrend ition +ĠK ik +Ġpath ology +Ġselect s +Link edIn +Ġass ay +ĠD res +Ġtext ual +post ed +IT AL +ĠM aul +N eal +Ġinter connected +Ġerr atic +ĠVir us +Ġ5 30 +Ġenvironmental ists +ĠP helps +Ġeng agements +ĠIN ST +Ġeconom ical +nox ious +Ġg earing +izz y +Ġfavor ably +ĠMcG ill +T erm +Ġh anged +Ġball park +ĠRe yes +Ġbe ware +ĠP sal +ĠMass acre +q i +Ġin accessible +acly sm +Ġfr ay +ill ac +Ġbitter ly +ĠCert ification +Mich igan +Ġir respective +al ore +Em pty +Ġendorse ments +Ġund et +f g +equ ipped +Ġmerc iless +ĠC ust +Ġimm ature +Ġvou cher +ĠBlack well +Ñ ı +h awk +dis ciplinary +ile e +ĠMak oto +ĠD ude +ãĥĩ ãĤ£ +Y ears +Ġin ver +Ġsh aman +ĠY ong +ip el +ell en +ĠCath y +br ids +Ġs arc +65 1 +N ear +Ġground work +Ġam az +Ġ4 15 +ĠHunting ton +hew s +ĠB ung +Ġarbit rarily +ĠW it +ĠAl berto +Ġdis qualified +best os +46 1 +Ġp c +Ġ28 4 +ro bat +Rob in +Ġh ugs +ĠTrans ition +ĠOcc asionally +Ġ3 26 +ĠWh ilst +ĠLe y +Ġspaces hip +cs v +Ġun successfully +ĠA u +le ck +ĠWing ed +ĠGrizz lies +. � +Ġne arer +ĠSorce ress +ĠInd igo +El se +8 40 +let es +Co ach +Ġup bringing +ĠK es +Ġseparat ist +Ġrac ists +Ġch ained +Ġabst inence +lear ning +Ġrein stated +Ġsymm etry +Ġremind ers +ĠChe vy +Ġm ont +Ġexempl ary +ĠT OR +Z X +Ġqual itative +ĠSt amp +ĠSav annah +ĠRoss i +Ġp aed +Ġdispens aries +ĠWall s +ĠCh ronic +Ġcompliment ary +ĠBeir ut +Ġ+ --- +igs list +Ġcrypt ographic +mas ters +ĠCap itals +Ġmax imal +Ġent ropy +Point s +Ġcombat ants +l ip +ĠGl ob +ĠB MC +ph ase +th ank +HT TP +Ġcomm uter +Ġ\( \ +.. / +ĠReg ener +ĠDO I +ĠActiv ision +Ġsl it +os al +RE M +Ġch ants +Y u +Ke ys +Bre xit +ĠFor ced +Ari zona +Ġsquad ron +IS O +ĠMal one +Ġ3 38 +Ġcontrast ing +Ġt idal +Ġlib el +Ġimpl anted +Ġupro ar +ĠC ater +Ġpropos itions +M anchester +ĠEuro s +it amin +G il +ĠEl ven +ĠSe ek +ĠB ai +Ġredevelop ment +ĠTown s +ĠL ub +! ", +al on +K rist +Ġmeas urable +Ġimagin able +Ġapost les +Y N +7 60 +Ġster oid +Ġspecific ity +ĠL ocated +ĠBeck er +ĠE du +ĠDiet ary +uts ch +ĠMar ilyn +Ġbl ister +ĠM EP +ĠK oz +ĠC MS +y ahoo +ĠCar ney +Ġbo asting +ĠC aleb +By te +read s +ad en +Pro blem +ĠWood ward +S we +S up +ĠK GB +Set up +Ġtac it +Ġret ribution +Ġd ues +ĠM ü +. ? +ä¸ Ń +p ots +Ġcame o +ĠP AL +educ ation +A my +like ly +g ling +Ġconstitution ally +ĠHam m +ĠSpe ak +Ġwid gets +br ate +Ġcra ppy +ĠI ter +Ġanticip ating +ĠB out +P ixel +ĠY ep +ĠLaur ie +Ġh ut +Ġbullet in +ĠSal vation +Ġch ats +ear able +Honest ly +AL TH +onse qu +c ult +isco very +ovy ch +Ġse lves +ĠSat oshi +S ounds +Ġconver gence +ĠRosen berg +19 74 +Ġnas al +Ġfull est +Ġfer ocious +x us +ist e +AM S +Ġlobb ied +Ġso othing +ĠGun n +t oday +0 24 +Ġinspir ational +ĠN BN +p b +g ewater +or ah +all owed +ĠCol iseum +Ġspecial izing +Ġinsane ly +ĠT ape +del ay +Ġt arn +ĠP ound +Ġmel anch +Ġdeploy ments +il and +Ġless en +Ġfur ry +ĠUE FA +Ġblood shed +ĠMe ier +ither ing +Ġhe irs +ĠJ aw +ax ter +ĠPublic ations +Ġal ters +int ention +ĠWinc hester +d etermination +ĠLif etime +th in +Mon ster +7 80 +Ġapprox imation +Ġsuper markets +ĠSecond s +or os +h uge +Ġb ribe +ĠLIM ITED +un ed +Ġmis interpret +ĠIn jury +Ġ3 67 +Ġthreshold s +ĠCarn ival +Ġgastro intestinal +Ġguid eline +Ġde ceived +f eatures +Ġpurported ly +ĠRon nie +ĠNew t +Ġsp acious +as us +Ġsuperhero es +ĠCyn thia +le gged +k amp +ch io +Ġth umbnail +ĠShir ley +ill ation +Ġshe ds +ĠZ y +E PA +Ġdam s +Ġy awn +n ah +ĠPe ggy +ĠE rie +ĠJu ventus +ĠF ountain +r x +don ald +al bum +ĠComp rehensive +Ġc aching +ĠU z +ulner ability +ĠPrinc iple +ĠJ ian +ing ers +cast s +ĠOs iris +ch art +t ile +ĠTiff any +ĠPatt on +ĠWh ip +Ġovers ized +J e +ĠCind erella +ĠB orders +ĠDa esh +M ah +Ġdog ma +Ġcommun ists +v u +Coun cil +Ġfresh water +Ġw ounding +Ġdeb acle +Ġyoung ster +Ġthread ed +ĠB ots +ĠSav ings +ãģ Ĥ +ol ing +oh o +Ġillum ination +M RI +Ġlo osen +tr ump +ag ency +ur ion +Ġmoment arily +ĠCh un +ĠBud apest +ĠAl ley +D isk +Ġaston ished +ĠCon quer +ĠAccount ing +h aving +ĠWe in +ĠAl right +Ġrev olver +Ġdel usion +Ġrelic s +Ġad herent +qu ant +Ġhand made +or io +Ġcomb ating +c oded +Ġquad ru +re th +N ik +ĠTrib al +ĠMyster ious +Ġin hal +ĠWin ning +ĠClass ification +ch anged +Ġun ab +Ġsc orn +icip ated +w l +ond uctor +Ġrein forcing +ĠChild hood +an ova +Ġadventure r +Ġdoctor al +ĠStrateg ies +Ġengulf ed +ĠEnc ounter +Ġl ashes +Crit ical +ric ular +ĠU TF +oci ation +check ing +ĠConsult ing +Run time +per iod +ĠAs gard +Ġdist illed +ĠPas adena +ĠD ying +ĠCOUN TY +Ġgran ite +Ġsm ack +Ġparach ute +ĠS UR +Virgin ia +ĠF urious +78 7 +ĠO kin +Ġcam el +ĠM bps +19 72 +ĠCh ao +ĠC yan +j oice +ef er +ĠW rap +ĠDeb ate +S eg +Ġfore arm +ĠIgn ore +Ġtim estamp +Ġprob ing +ĠNo on +ĠGra il +f en +Ġdorm ant +ĠFirst ly +ĠE ighth +ĠH UN +ĠDes ire +or as +Girl s +ĠDes mond +z ar +am ines +O AD +exec ute +Ġbo obs +ĠAT L +_ ( +Chel sea +Ġmasturb ation +ĠCo C +Ġdestroy er +ĠCh omsky +Ġsc atter +ĠAss ets +79 6 +ĠC argo +Ġrecept ive +ĠSc ope +Ġmarket ers +Ġlaun chers +Ġax le +ĠSE A +se q +ĠM off +f inding +ĠGib bs +Georg ia +extreme ly +N J +Ġlab orers +st als +Ġmed iation +ĠH edge +at own +Ġi od +des pite +v ill +J ane +ex istence +Ġcoinc ided +ĠUt ilities +ĠChe ap +Ġlog istical +Ġcul mination +ĠNic otine +p ak +F older +Ġrod ents +st uff +Ġlaw fully +Ġreper to +io ch +j j +Dial ogue +HH HH +lic tion +Look s +Ġ29 7 +Ġtur rets +ĠAb andon +Ġinc ess +ĠTraff ord +Ġcur led +Ġprefer ring +Ġprivat ization +Ġir resist +ĠP anda +ĠSh ake +ĠMc Gr +ãĥ Ħ +und ers +Ġdiscrim inated +Ġbart ender +I LE +Atl antic +Ġprop ensity +ĠW iz +ĠG im +con ference +Ġrein forces +G h +w agon +Ġe erie +F al +Ġhug ged +rac ist +R IC +F u +Ġf iller +ĠSt ub +Ġeng raved +ĠWrest le +Ġimagin ative +ĠPe er +ĠFact ors +an us +ĠDrac ula +mon itor +Ġrou ters +ib ia +ĠBoo lean +end ale +ĠSl aughter +ĠSh ack +R FC +ĠSpiel berg +S ax +ĠPH OTO +ĠCl over +ĠR ae +Dep ending +ĠMem or +ar am +Ġpier ced +Ġcur tains +v ale +ĠInqu isition +ĠP oke +Ġforecast ing +Ġcompl ains +S ense +ĠHer mes +isc overed +Ġb ible +ĠMor ph +Ġg erm +78 5 +D ON +Ġcon gen +Ġcr ane +ĠD PR +Ġrespect fully +R oom +ĠN aw +ĠDal ai +re ason +ĠAng us +Educ ation +ĠTitan ic +Ë ľ +Ġo val +un ited +Ġthird s +Ġmoist ur +ĠC PC +M iami +Ġtent acles +ĠPol aris +ex c +ex clusive +ĠPra irie +Ġcol ossal +ĠBl end +sur prisingly +ÃŃ s +Ġindo ctr +Ġbas al +ĠMP EG +und o +Spl it +Develop ment +Ġlan tern +19 71 +Ġprov ocation +Ġang uish +ĠB ind +ĠLe ia +duc ers +ipp y +conserv ancy +Ġinitial ize +ĠTw ice +ĠSu k +Ġpred ic +Ġdi ploma +Ġsoc iop +Ing redients +Ġhamm ered +ĠIr ma +Q aida +Ġglim ps +ĠB ian +Ġst acking +Ġf end +gov track +Ġun n +dem ocratic +ig ree +Ġ5 80 +Ġ29 4 +Ġstraw berry +ID ER +Ġcher ished +ĠH ots +Ġinfer red +Ġ8 08 +ĠS ocrates +O regon +ĠR oses +ĠFO IA +Ġins ensitive +Ġ40 8 +Recomm end +ĠSh ine +Ġpain staking +UG E +ĠHell er +ĠEnter prises +I OR +ad j +N RS +L G +Ġalien ated +Ġacknowled gement +ĠA UD +ĠRen eg +Ġvou chers +Ġ9 60 +Ġm oot +ĠDim ensions +Ġc abbage +B right +g at +ĠK lu +Ġlat ent +Ġz e +ĠM eng +Ġdis perse +Ġpand emonium +H Q +Ġvirt uous +ĠLoc ations +ee per +prov ided +Ġse ams +ĠW T +iz o +PR OV +Ġtit anium +Ġrecol lection +Ġcr an +Ġ7 80 +ĠN F +49 1 +64 2 +p acking +59 8 +text ure +Sp ider +fre edom +cipl ed +ĠTAM ADRA +âĻ ¦ +aut hent +ĠW ANT +r ified +Ġr ites +Ġuter us +k iss +Ġâī ¤ +Ġsk illet +Ġdis enfranch +ĠGa al +Comp an +Ġage ing +gu ide +B alt +Ġiter ator +Ġdiscretion ary +t ips +Ġprim ates +ĠTechn ique +ĠPay ments +az el +ĠR OCK +stant ial +0 60 +Ġd mg +ĠJack ets +ĠPlay off +Ġnurs ery +ĠSy mb +art on +Ġannex ation +Color ado +Ġco ils +ĠSh oes +âĦ¢ : +ĠRo z +COM PLE +ĠEve rest +ĠTri umph +J oy +G rid +à ¼ +process or +ĠPros per +ĠSever us +ĠSelect ed +r g +ĠTay yip +St ra +Ġski ing +Ġ? ) +Ġpe g +Tes la +Ġtime frame +Ġmaster mind +ĠN B +scient ific +ĠSh it +gener ic +IN TER +N UM +Ġst roll +ĠEn ix +ĠM MR +ĠE MS +m ovie +Ĥ ª +Ġminim izing +idd ling +Ġilleg itimate +Ġprot otyp +Ġpremature ly +Ġmanual s +obb ies +ĠCass idy +D EC +des ktop +Ġaer os +Ġscreen ings +Ġdeb ilitating +ĠGr ind +nature conservancy +Ġf ades +ter mination +assets adobe +F actor +Ġdefinitive ly +P oké +ap ult +ĠLaf ayette +C orn +ĠCor al +Ġstagn ant +T ue +Ġdissatisf action +G ender +Ġkid neys +ĠG ow +ĠDef eat +ĠAsh ton +Ġcart els +Ġfore closure +ĠExpl ore +stre ngth +ot in +Ġveterin arian +Ġf umble +Ġpar ap +ĠSt rait +r ils +Ġpr ick +ĠBerm uda +ĠAm munition +skin ned +Ġab ound +ĠB raz +Ġshar per +ĠAsc ension +Ġ9 78 +Ġpreview s +Ġcommun ion +ĠX Y +Ġph ony +Ġnewcom er +Ġ3 32 +." ," +Ġredist ribution +Prot ect +ĠSo f +K al +Ġlip stick +w orst +Ġtang led +Ġretrospect ive +int eger +Ġvolunte ering +Ġ19 07 +Ġ -------------------- +ic hen +Ġunve iling +Ġsen seless +Ġfisher ies +\ - +Ġh inges +Ġcalcul us +My th +Ġund efeated +Ġoptim izations +Ġdep ress +Ġbill board +ĠY ad +ĠPy ramid +Is n +I de +Ġleg ion +ĠK ramer +ent anyl +Ġpenet rating +ĠHaw th +ĠPR ODUCT +ĠGer ard +ĠP act +ĠIn cluding +ĠEl ias +ĠEl aine +vis ual +Ġhum ming +Ġcond esc +ĠF asc +ä¸ Ĭ +Ġe galitarian +Ġdev s +ĠD ahl +O ps +D H +ĠB ounce +id ated +ald o +Ġrepublic an +Ġh amb +ĠS ett +ograph ies +CH APTER +Ġtrans sexual +Ġsky rocket +ans wer +Ġmark up +Ø ª +Ġhero ine +Comp are +ĠT av +Be ast +Ġsuccess ors +Ġna ïve +ĠBuck ley +st ress +me at +Ġdownload able +Ġindex ed +Ġsc aff +ĠL ump +ĠHom o +Stud io +In sp +Ġr acked +far ious +ĠPet ty +Ex ternal +Ġ19 09 +W ars +com mit +put ers +Ġun ob +ĠEr r +ĠE G +ĠAl am +ĠSiber ia +ĠAtmosp heric +IS TER +ĠSatan ic +trans lation +ĠL oud +tra umatic +l ique +Ġreson ate +ĠWel ch +Ġspark ing +ĠT OM +t one +Ġout l +Ġhandc uffed +ĠSer ie +8 01 +Ġland marks +ĠRee ves +Ġsoft ened +Ġdazz ling +ĠW anted +month s +Mag ikarp +Ġunt reated +ĠBed ford +M i +ĠDynam o +O re +79 5 +Ġwrong ful +Ġl ured +Ġcort isol +Ġve x +d rawn +ile t +Download ha +ĠF action +Ġlab yrinth +Ġhij acked +w aters +er ick +Ġsuper iors +ĠRow ling +ĠGu inness +Ġt d +99 2 +Ġune arthed +Ġcentr if +Ġsham eless +P od +ĠF ib +Ġ icing +Ġpredict or +Ġ29 2 +fore station +con struct +C and +@ # +Ġag itated +Ġre pr +OV A +Ġkn itting +ĠLim a +Ġf odder +68 4 +ĠPerson a +k l +7 01 +Ġbreak up +á ¸ +Ġapp alled +Ġantidepress ants +ĠSus sex +Har ris +ĠTher mal +ee ee +U pload +Ġg ulf +Ġdoor step +ĠSh ank +L U +ĠM EN +ĠP ond +s orry +Ġmis fortune +n ance +Ġb ona +M ut +Ġde graded +ĠL OG +ĠN ess +an imal +Ġa version +und own +Ġsupplement ed +ĠC ups +Ġ50 4 +Ġdep rive +ĠSpark le +Å Ĥ +ĠMed itation +auth ors +ĠSab an +ĠN aked +air d +ĠMand arin +ĠScript ures +ĠPerson nel +ĠMahar ashtra +Ġ19 03 +ĠP ai +ĠMir age +omb at +Access ory +Ġfrag mented +T ogether +Ġbelie vable +ĠGl adiator +al igned +ĠSl ug +M AT +Ġconvert ible +ĠBour bon +amer on +ĠRe hab +nt ax +Ġpowd ered +pill ar +Ġsm oker +ĠMans on +ĠB F +5 11 +ĠGood ell +ĠD AR +m ud +g art +Ġob edient +ĠTrans mission +ĠDon ation +8 80 +Ġbother ing +Material s +ãĤ ± +dest roy +Ġfore going +Ġanarch ism +ĠK ry +ice ps +Ġl ittered +ĠSch iff +Ġanecd otal +un its +Ġf ian +ĠSt im +ĠS OME +ĠInv aders +Ġbehaviour al +ĠVent ures +Ġsub lime +Ġfru ition +ĠPen alty +Ġcorros ion +¶ ħ +Ġlik ened +Ġbesie ged +ween ey +ĠCre ep +Ġlinem en +mult i +ic ably +ud der +Ġvital ity +Ġshort fall +ĠP ants +ap ist +H idden +ĠDro ps +med ical +Ġpron unciation +ĠN RL +Ġinsight ful +J V +ĠBe ard +ĠCh ou +Ġchar ms +Ġb ins +Ġamb assadors +ĠS aturdays +Ġinhib itor +ĠFr anch +6 01 +', ' +ĠCon or +art ney +ĠX peria +g rave +be es +ĠProtest ants +Ġso aking +ĠM andal +Ġph ased +Ġ6 60 +Ġsc ams +Ġbuzz ing +ĠItal ians +ĠLoren zo +ĠJ A +Ġhes itated +Ġcl iffs +ĠG OT +ingu ishable +Ġk o +Ġinter ruption +Z ip +Lear ning +Ġundersc ores +ĠBl ink +K u +57 9 +ĠAut ob +I RE +Ġwater ing +Ġpast ry +8 20 +Ġvision ary +ĠTempl ar +awa ited +Ġpist on +Ġant id +current ly +Ġp ard +Ġw aging +Ġnob ility +ĠY us +Ġinject ing +f aith +ĠP ASS +å º +Ġret ake +ĠPR OC +Ġcat hedral +b ash +Ġwrest lers +Ġpartner ing +Ġn oses +Ġ3 58 +Trans form +am en +Ġb outs +ĠId eal +ĠConstant in +Ġse p +ĠMon arch +att en +ĠPe oples +mod ified +Ġmor atorium +Ġpen chant +Ġoffensive ly +Ġprox ies +ok ane +ĠTaiwan ese +ĠP oo +ĠH OME +us ional +Ġver bs +ĠO man +vis ory +Ġpersu asion +Ġmult it +Ġsc issors +G ay +ow ay +oph ysical +l us +gn u +Ġap ocalyptic +Ġabsurd ity +Ġplay book +Ġautobi ography +I UM +Ġsne aking +ĠSim ulation +pp s +ell ery +Plan et +Ġright fully +Ġn iece +ĠN EC +ĠIP O +ĠDis closure +lean or +ous y +ST ER +Ġ28 2 +Cru z +Ch all +64 3 +ĠSurv ive +ĠF atal +ĠAm id +ap o +We apons +D EN +7 70 +ĠGreen wald +Ġlin en +al os +Ġpollut ants +ĠPCI e +k at +Ġp aw +ĠK raft +C hem +ĠTermin ator +Ġre incarn +Ġ] [ +ĠSe eds +Ġsilhou ette +ĠSt ores +Ġgro oming +ĠD irection +ĠIs abel +ĠBr idges +ðŁ ij +E ED +ĠM orsi +Ġval ves +ĠRank ed +ĠPh arma +ĠOrgan izations +Ġpenet rated +ĠRod ham +ĠProt oss +Ġove rest +Ġex asper +ĠT J +Ġ 000000 +Ġtrick le +Ġbour bon +WH O +Ġw retched +Ġmicrosc opic +Ġcheck list +Ġad orned +R oyal +Ad minist +ĠRet irement +ĠHig hest +We ather +ile ge +Ġincre ments +ĠC osponsors +Ġmas se +ĠS inn +r f +Ġh ordes +as sembly +75 4 +ĠNat asha +ĠTY PE +ĠGEN ERAL +Ġarr anging +Ġ40 7 +l ator +Ġg lean +Ġdisc redited +Ġclin icians +UN E +Ġachie ves +ĠEm erson +com plex += [ +Ġprincip ally +Ġfra il +p icked +Ġthan king +Ġre cl +ĠL AST +Ġsupp ressing +il ic +Ġantidepress ant +ĠLis bon +Ġth or +Ġsp a +Ġking doms +ĠPear ce +em o +Ġpl ung +Ġdiv est +Ġ ******************************** +b is +osp els +ad r +Sp irit +hall a +P ink +end ez +Ġresurrect ed +esc ape +ĠRosen stein +Ġge ological +Ġnecess ities +Ġcarn iv +ĠE lys +ĠBar ney +Ġ29 6 +dig y +ST ON +D OWN +Ġmil estones +Ġk er +Ġdismant ling +Ġre prim +Ġcross ings +19 45 +Ġpatri archy +Ġblasp hemy +Ġ3 59 +met ry +ĠOb esity +ĠDiff erences +bl ocking +ãĥķ ãĤ¡ +ich ita +ĠSab ha +ph alt +ĠCol o +ual a +effic ients +ĠMed ina +con sole +55 7 +ĠHann ibal +ĠHab it +ĠF ever +Ġthen ce +Ġsyn agogue +Ġessential s +Ġw ink +ĠTr ader +ID A +ĠSp oiler +ĠIceland ic +ĠHay ward +Ġpe ac +Ġmal ice +Ġflash back +Ġth w +Ġlay offs +L iquid +Ġtro oper +Ġh inge +ĠRead ers +Ph ill +ĠB auer +Cre ated +Ġaud its +ac compan +Ġunsus pecting +ier a +6666 6666 +Ġbro ch +Ġapprehend ed +ĠM alk +cer ning +ĠCod ex +O VER +M arsh +ĠD eng +ĠExp ression +Ġdisrespect ful +Ġasc ending +t ests +ĠPlaint iff +ster y +ĠAl ibaba +din and +ĠDem psey +Applic ations +mor al +Ġthrough put +Ġquar rel +Ġm ills +Ġhe mor +ĠC ASE +terror ist +st im +ifest yle +ro zen +CE PT +Ar k +u ci +lect ic +Ġirrit ating +she ets +A y +Ġrede emed +Ġhorn y +ĠTe ach +ĠS ear +dem ocracy +4 65 +ĠRest ore +Ġstand by +ĠP is +iff in +Ġsleep y +Ġextr ater +Ġcompl iments +Fram eworks +Ġinstall s +Ġb anging +sur face +found land +Ġmetaph ysical +Ġ28 3 +oul s +dev ices +Ar gs +ĠSac rifice +ĠMcC orm +es on +Cons ervative +ĠM ikhail +see ing +is ively +ĠRo oms +ĠGener ic +Ġenthusi astically +Ġgri pped +Ġcomed ic +ĠElectric ity +Ġgu errilla +Ġdec oration +ĠPerspect ive +Ġconsult ations +Ġun amb +Ġplag iar +Ġmagic ian +Ġe rection +ĠTour ism +or ied +ro xy +11 00 +T am +Ī è +Î ³ +× ª +ĠPred ators +Nit rome +Ġtelesc opes +project s +Ġun protected +Ġst ocked +ĠEnt reprene +nex pected +Ġwast ewater +V ill +Ġint imately +Ġi Cloud +ĠConst able +Ġspo of +Ġne farious +Ġfin s +Ġcens or +ĠMod es +ĠEs per +ar bon +Ġinter sections +Ġlaud ed +Ġphys i +Ġgener ously +ĠThe Nitrome +ĠTheNitrome Fan +Ġar isen +ĠÙ Ī +Ġg lands +ĠPav ilion +ĠGu pta +Ġuniform ly +Ġr amps +ri et +ĠWH EN +ĠVan essa +Ġrout ed +Ġlim p +ĠC PI +p ter +int uitive +Ġv aping +Ġexperiment ed +ĠOlymp us +ĠAm on +Ġsight ing +Ġinfiltr ate +ĠGentle man +Ġsign ings +ĠMe ow +ĠNav igation +che cks +4 33 +Ġel apsed +ĠBulg arian +esp ie +ĠS OM +d uring +Ġsp ills +anc a +ĠPly mouth +M AL +Ġdomest ically +ĠWater gate +ĠF AM +k illed +ed ited +ĠYour self +Ġsynchron ization +ĠPract ices +ST EP +Ġgen omes +ĠQ R +not ice +Ġloc ating +z in +Ġ3 29 +al cohol +Ġk itten +V o +Ġr inse +Ġgrapp le +ĠSc rew +ĠD ul +A IR +Ġle asing +ĠCaf é +Ġro ses +ĠRes pect +Ġmis lead +Ġperfect ed +Ġnud ity +Ġnon partisan +ĠCons umption +Report ing +Ġnu ances +Ġdeduct ible +ĠSh ots +Ġ3 77 +Ġæ ľ +ano oga +Ben ef +ĠB am +ĠS amp +if ix +Ġgal van +ĠMed als +rad ius +Ġno bles +Ġe aves +igr ate +K T +ĠHar bour +u ers +Ġrisk ed +re q +Ġneuro t +get table +ain a +Rom ney +Ġunder pin +Ġlo ft +ĠSub committee +ĠMong ol +b iz +Ġmanif ests +ass isted +ĠG aga +Ġsy nergy +Ġreligious ly +ĠPre f +ĠG erry +T AG +ĠCho i +4 66 +beh ind +ĠO u +Gold Magikarp +Ġhemor rh +R iver +Ġtend on +Ġinj ure +ĠF iona +Ġp ag +Ġag itation +|| || +ur an +ĠE SA +Ġest eem +Ġdod ging +Ġ4 12 +r ss +Ġce ases +ex cluding +Ġint akes +Ġinsert s +Ġemb old +ĠO ral +up uncture +4 11 +ĠUn ified +ĠDe le +Ġfurn ace +ĠCoy otes +ĠBr ach +L abor +Ġhand shake +Ġbru ises +Gr ade +éĹ ĺ +ĠGram my +ile en +St ates +ĠScandinav ian +ĠKard ash +8 66 +Ġeffort lessly +ĠDI RECT +ĠTH EN +ĠMe i +ert ation +19 68 +Ġgro in +w itch +Requ irements +98 5 +Ġroof s +Ġest ates +ĠH F +Ġha ha +Ġdense ly +ĠO CT +Ġpl astics +Ġincident ally +ĠTr acks +ĠTax es +Ġch anted +Ġforce ful +ĠBie ber +ĠK ahn +K ent +ĠC ot +lic ts +F ed +Ġhide ous +ĠVer d +ĠSynd icate +ĠIl legal +J et +ĠD AV +re asonable +c rew +Ġfundamental ist +Ġtruth ful +ĠJ ing +Ġl il +Ġdown ed +Ġen chanted +ĠPolic ies +ĠMcM aster +ĠH are +ides how +Ġpar ams +en cers +gorith m +Ġallow ances +Ġturb ulent +Ġcomplex ities +ĠK T +Ġ3 37 +ĠGen etic +F UN +D oug +t ick +Ġg igs +ument hal +Ġpatriarch al +Ġcal c +, ... +Ġc out +ĠGu an +Ġpath ological +ĠR ivals +Ġunder rated +Ġflu orescent +ĠJ iu +arna ev +ĠQu an +Ġ4 29 +Ġ ਠ+M ario +Con struct +ĠC itation +ĠR acial +ĠR SA +ĠF idel +Ġ3 95 +Person ally +C ause +à » +rad ical +in en +Ġvehement ly +ĠPap a +Ġintern ship +Ġfl akes +ĠRe ck +Luck ily +B ra +20 20 +rav ings +R N +W onder +Ser iously +Ġre usable +Ġpoll uted +ĠP eng +le igh +ind le +Ġcircuit ry +ĠMad onna +ĠB ART +Res idents +att ribute +Phil adelphia +Cl ub +Ġplan ner +Ġfr antically +Ġfaith fully +ĠTerrit ories +ĠL AT +ĠAnders en +an u +ĠP ARK +ĠS ora +i age +ĠPlay offs +ĠG CC +4 27 +Ġab norm +ĠL ever +Ġdisob edience +As ync +ĠShe a +V ert +Ġsk irts +ĠSaw yer +x p +Ġwors ening +Ġsc apego +ĠAng le +oth al +Ġtro ve +ĠSt y +ĠN guyen +mar ine +ide on +Dep ths +Bl og +ĠIll uminati +Ġtract s +Ġorgan ise +Ġo str +F s +Ġlever aging +ĠD aredevil +as ar +Ġl ang +Ġex termin +urs ions +ĠRom o +ãĤ¤ ãĥĪ +Ġcont ended +Ġencounter ing +ĠTable t +ĠAltern ate +sk ill +Ġswe ets +Ġco hesive +cap acity +Ġrep ud +Ġl izard +ro o +Ġpilgr ims +ĠR uff +ĠInstr ument +ĠLog o +uit ous +E H +Ġsales man +Ġank les +L ed +ĠPat ty +ud os +Own er +Ġdiscrep ancies +k j +M U +Ġuncond itional +Dragon Magazine +i ard +O ak +ĠConvers ation +be er +ĠOs aka +D elta +us ky +Ġsecret ion +Ġpl aza +Ġm ing +Ġde pletion +ĠM ous +ĠI TS +ĠH imal +ĠFle ming +Ġcyt ok +ĠH ick +Ġbat ters +ĠInt ellectual +6 75 +é r +IS ION +ĠQu entin +ĠCh apters +ih adi +Ġco aster +WAY S +ĠL izard +ĠY or +and ering +S kin +ha ust +ab by +Ġportray ing +Ġwield ed +d ash +Ġprop onent +Ġr ipple +Ġgrap hene +Ġfly er +Ġrec urrent +Ġdev ils +Ġwater fall +æĺ ¯ +go o +Text Color +Ġtam pering +IV ES +TR UMP +ĠAb el +ĠS AL +ĠHend ricks +ĠLu cius +b ots +Ġ40 96 +IST ORY +Gu est +ĠN X +in ant +Ben z +ĠLoad ed +ĠCle ver +t reatment +Ġta vern +Ġ3 39 +ĠT NT +ific antly +Tem perature +F el +Ġunder world +ĠJud ges +Ġ< + +Ġst ump +Ġoccup ancy +Ġab er +ĠF inder +) ", +ĠN unes +res et +in et +ect omy +Ġwell ness +ĠP eb +quart ered +and an +Ġneg atives +ĠTh iel +ĠCl ip +ĠL TD +Ġbl ight +Ġreperto ire +K yle +Ġqu er +ĠC es +Ġha pl +98 9 +ĠTh ames +isc opal +Des k +ivari ate +ĠEx cellence +found ation +Ġâ ĩ +X i +Ġmyster iously +esty les +Ġper ish +ĠEng els +ĠDE AD +09 0 +}} } +ĠUn real +Ġrest less +ID ES +orth odox +ĠInter mediate +Ġdin ners +ĠTr out +ĠSe ym +ĠHall s +og ged +Ġtraged ies +Ġdid nt +67 6 +Ġail ments +Ġobserv able +ĠV ide +ad apt +ĠD usk +Ġprofessional ism +ĠPres cott +ĠInd ies +p ox +ĠMe hran +W ide +Ġend emic +ĠPar an +B ird +Ġped als +ĠI U +ĠAdam ant +ĠH urt +Ġcorrel ates +urd en +Ġspons oring +cl imate +ĠUnivers ities +ĠK not +enn es +ĠDam ian +ĠAx el +S port +Ġbar b +ĠS no +sh own +ste en +ud ence +Ġnon violent +Ġhom ophobia +Ġbiom ass +ĠDet ail +Ġsrf N +ĠT une +accompan ied +I ENCE +Al bert +ĠMong o +z x +ĠCer berus +or bit +c ens +Ġsl ay +SH ARE +H Y +Ġb rawl +ĠPro be +Ġnonex istent +ĠClare nce +ĠBlack burn +Ġport als +ĠR ita +ĠRem ain +ĠLe vant +Ġtrick ed +ĠF erry +aver ing +ĠStraw berry +ĠAn swers +Ġhorrend ous +ĠA man +Supp lement +ĠT oad +Ġpe eled +Ġman oeuv +ĠU zbek +mond s +ĠH ector +Ġ40 2 +pe es +fix es +Ġd j +Ġres umes +Ġaccount ant +Ġadvers ity +Ġham pered +ĠL arson +Ġd oping +part s +H ur +Ġbe arded +Ġy r +ĠPlug in +å¥ ³ +Ġ/ ** +rol ley +Ġwaters hed +ĠSub mission +if lower +AS C +Ġcho ir +Ġsculpt ures +m A +incre asing +ai i +Ġsne akers +Ġconfront s +ĠEle phant +ĠEl ixir +Ġrec al +ĠT TL +w idget +ĠW ax +ĠGr ayson +Ġha irst +Ġhumili ated +ĠWAR N +app iness +ĠT TC +F uel +Ġpol io +Ġcomplex es +Ġbab e +ĠX IV +P F +). [ +P arts +Ġ4 35 +M eg +ĠY ards +ĠAL P +Ġy ells +Ġprin ces +Ġbull ies +ĠCapital ism +ex empt +FA Q +ĠSp onge +ĠAl a +Ġpleas antly +Ġbu f +Ġden ote +Ġunp ublished +Ġkne eling +asc a +Ġl apse +al ien +99 4 +Ġrefere es +ĠLaw yers +S anta +Ġpuzz ling +ĠProm etheus +ĠPh araoh +ĠDel ay +Ġfacilit ates +ĠC ES +Ġjew els +Ġbook let +ond ing +Ġpolar ization +ĠMor an +ĠSal ad +ĠS OS +ĠAdv ice +PH OTOS +IC AN +iat ures +ex press +ĠWonder land +ĠC ODE +ĠCL ASS +9 75 +Ġg rep +ĠD iesel +ĠGl ac +! ?" +Ġr m +o ine +disc rimination +ĠN urse +m allow +Ġv ortex +ĠCons ortium +Ġlarge Download +stra ight +augh lin +G rad +Ġpublic ized +ĠW aves +ĠRed d +Ġfest ivities +ĠM ane +ar ov +Ġfleet ing +ĠDr unk +ug en +C ele +Ġchromos omes +ĠD OT +-+-+ -+-+ +Ġbus iest +ĠBe aver +Sy rian +ĠK yr +k as +ĠCross Ref +19 50 +76 01 +Ġrepe aling +ĠWin ners +ĠMac ro +ĠD OD +bl ance +S ort +64 1 +Ġmet re +ĠD irk +Ġgo ggles +Ġdraw backs +Ġcomplain ant +Ġauthor izing +Ġantit rust +oper ated +Ġm ah +Ġexagger ation +Am azing +ĠSer aph +Ġha ze +w ow +Ġextingu ished +Ġcan yon +ĠB osh +Ġv ents +Ġsc rape +Cor rect +4 26 +Ġav g +Dem and +ĠâĪ ¼ +Ġmicrobi ota +"} ]," +ĠSt ev +B io +ĠPlan es +Ġsuggest ive +Ġdec ipher +ĠRefuge e +ĠKe jriwal +ĠGreen peace +Ġdecl ass +ĠSound ers +Ġth o +Ġdec rypt +Ġbr ushing +ĠJane iro +ip op +S i +8 77 +ĠGeoff rey +Ġc pu +ĠHaz el +Ġview points +Ġcris py +ĠNot ification +Ġsold er +ĠMod est +ĠHem isphere +Ġcass ette +in cludes +Ġident ifiers +ĠC ALL +in cent +T odd +ĠSwe ep +Ġ3 34 +b oss +Ġsm ir +gin x +Ġtown ship +Ġg rieving +ĠMos que +Net flix +AS ED +ĠMillenn ials +oc om +19 67 +Ġbold ly +s leep +Ġes che +arij uana +Ġsw irl +ĠPen al +Ġneglig ent +ĠStephen son +K ER +ĠZ oro +ris is +Ġlocal ization +ĠSeym our +ĠAng lic +red itation +prot ection +ĠPa ige +Ġo mit +ĠR ousse +ĠT ub +Ġinv itations +t ty +Ġm oss +ph ysical +C redits +Ġan archy +Ġchild care +Ġl ull +ĠM ek +ĠL anguages +lat est +ĠSan ford +Ġus ability +Ġdiff use +ĠD ATA +Ġsp rites +ĠVeget a +ĠProm otion +ãĥ¼ ãĤ¯ +rict ing +z ee +Tur kish +ĠTD s +pro ven +57 1 +Ġsmug glers +707 10 +Ġreform ed +ĠLo is +Ġun fl +ĠWITH OUT +ĠReturn ing +ann ie +ĠTom as +Fr anc +ĠProf it +ĠSER V +ĠR umble +ik uman +es an +Ġt esters +Ġgad get +Ġbrace let +ĠF SA +comp onent +Ġparamed ics +Ġj an +ĠRem em +ĠSk inner +Ġl ov +ĠQu ake +rom a +Ġfl ask +Pr inc +Ġover power +Ġlod ging +ĠK KK +ret te +Ġabsor bs +w rote +Ġ ," +K ings +ĠH ail +ĠFall ing +xt ap +ĠHel ena +ire ns +L arry +Ġpamph let +ĠC PR +G ro +ĠHirosh ima +Ġhol istic +". [ +Ġdet achment +Ġas pire +Ġcompl icit +ĠGreen wood +Ġresp awn +ĠSt upid +ĠFin ished +f al +b ass +Ġab hor +Ġmock ery +ĠFe ast +VID EO +Ġcon sec +ĠHung ry +P ull +ĠH ust +it ance +? ãĢį +) -- +ĠPar allel +con v +4 69 +ha ar +w ant +P aper +m ins +ĠTor o +ĠTR UMP +ĠR ai +D W +ĠW icked +ĠL ep +Ġfun ky +Ġdetrim ent +ios is +ache v +Ġde grade +im ilation +Ġret ard +Ġfrag mentation +Ġcow boy +ĠY PG +ĠH AL +Parent s +ĠS ieg +ĠStra uss +ĠRub ber +× IJ +Fr ag +Ġp t +Ġoption ally +ĠZ IP +ĠTrans cript +ĠD well +88 2 +M erc +ĠM OT +ãĥ¯ ãĥ³ +Ġhun ts +Ġexec utes +In cludes +Ġacid ic +ĠRespons ibility +ĠD umb +we i +And erson +ĠJas per +ight on +abs olutely +Ad ult +Ġpl under +Mor ning +ĠT ours +ĠD ane +Î º +ĠT EST +ĠG ina +Ġcan ine +aw an +Ġsocial ists +ĠS oda +Ġimp etus +ĠSupplement ary +oli ath +ĠKinn ikuman +mitted ly +second s +Ġorganis ers +Ġdocument aries +Vari able +GRE EN +Ġres orts +Ġbr agging +Ġ3 68 +Art ist +w k +bl ers +Un common +ĠRet rieved +Ġhect ares +Ġtox in +r ank +Ġfaith s +ĠG raphic +Ġve c +ĠL IA +Af rican +Ġard ent +end iary +L ake +ĠD OS +cient ious +ĠOk awaru +ĠAll y +ĠTim eline +D ash +ĠI c +contin ue +Ġt idy +Ġinstinct ively +ĠP ossibly +ĠOut door +ĠWould n +Ġl ich +ĠBr ay +ĠA X +Ġà ī +Ġ+ # +\ ' +Direct ory +ab iding +Ġf eral +ic ative +but t +Ġper verse +S alt +Ġwar ped +Ġnin eteen +Ġcabin ets +Ġsrf Attach +ĠSl oan +Ġpower ing +reg ation +F light +se vere +Ġst ren +Ġc og +ap ache +Ġâ Ŀ +Ġcaf eteria +p aces +ĠGrim oire +uton ium +Ġr aining +Ġcir cling +Ġlineback ers +c redit +Ġrep atri +ĠCam den +lic ense +Ġly ric +Ġdescript or +Ġval leys +Ġre q +Ġback stage +ĠPro hibition +ĠK et +Op ening +S ym +æĸ ¹ +Ġserv ings +Ġoverse en +Ġaster oids +ĠMod s +ĠSpr inger +ĠCont ainer +è » +ĠM ens +Ġmult im +Ġfire fighter +pe c +Ġchlor ine +Ð ¼ +end i +Ġsp aring +Ġpolyg amy +ĠR N +ĠP ell +Ġt igers +Ġflash y +ĠMad ame +S word +Ġpref rontal +Ġpre requisite +uc a +Ġw ifi +Ġmiscon ception +Ġharsh ly +ĠStream ing +ot om +ĠGiul iani +foot ed +Ġtub ing +ind ividual +z ek +n uclear +m ol +Ġright ful +49 3 +Ġspecial ization +Ġpassion ately +ĠVel ocity +ĠAv ailability +T enn +Ġl atch +ĠSome body +Ġhel ium +cl aw +Ġdi pping +XX X +Ġinter personal +7 10 +Ġsub ter +Ġbi ologists +ĠLight ing +Ġopt ic +Ġden im +end on +ĠC orm +Ġ3 41 +ĠC oup +Ġfear less +Ġal ot +ĠCliff ord +ĠRun time +ĠProv ision +up dated +lene ck +Ġneur on +Ġgrad ing +ĠC t +sequ ence +in ia +con cept +Ġro aring +ri val +ĠCaucas ian +Ġmon og +key es +Ġappell ate +Ġlia ison +EStream Frame +ĠPl um +! . +Ġsp herical +Ġper ished +Ġbl ot +Ġben ches +Ġ4 11 +Ġpione ered +Ġhur led +Jenn ifer +ĠYose mite +Ch air +Ġreef s +Ġelect or +ĠAnt hem +65 2 +Ġun install +Ġimp ede +Ġbl inking +Ġgot o +Dec re +A ren +Ġstabil ization +ĠDis abled +ĠYanuk ovych +Ġoutlaw ed +ĠVent ura +ten ess +Ġplant ation +Ġy acht +ĠHu awei +Ġsol vent +Ġgr acious +Ġcur iously +Ġcapac itor +Ġc x +ĠRef lex +Ph ys +ĠC f +pt in +cons ervative +Ġinv ocation +c our +F N +ĠNew ly +H our +As ian +ĠLe ading +ĠAer ospace +An ne +Ġpre natal +Ġdeterior ating +H CR +ĠNorm andy +ol ini +ĠAm bro +9 10 +Ġset backs +ĠT RE +Ġs ig +ĠSc ourge +59 7 +79 8 +Game play +Ġm sec +M X +Ġprice y +ĠL LP +aker u +Ġover arching +ĠB ale +Ġworld ly +Cl ark +Ġscen ic +Ġdisl iked +ĠCont rolled +T ickets +ĠE W +ab ies +ĠPl enty +Non etheless +Ġart isan +Trans fer +ĠF amous +Ġinf ield +ble y +Ġunres olved +ĠML A +ãĤ Ĥ +Cor rection +Ġdemocr at +ĠMore no +ro cal +il ings +Ġsail or +Ġr ife +h ung +Ġtrop es +Ġsn atched +ĠL IN +ĠB ib +ES A +ĠPre v +ĠCam el +run time +Ġob noxious +4 37 +Ġsum mers +Ġunexpl ained +ĠWal ters +cal iber +Ġg ull +ĠEnd urance +ä½ ľ +Ġ3 47 +Ir ish +Ġaer obic +Ġcr amped +ĠHon olulu +à © +us erc +ec ast +AC Y +ĠQu ery +ãĤ¹ ãĥĪ +Bet a +Ġsuscept ibility +ĠSh iv +ĠLim baugh +Ġà ĸ +ĠN XT +ĠM uss +ĠBrit ons +ES CO +EG IN +Ġ% % +Ġsec ession +ĠPat ron +ĠLu a +n aires +ĠJPM organ +us b +ocy te +Ġcouncill ors +ĠLi ang +f arm +Ġnerv ously +Ġattract iveness +ĠK ov +j ump +Pl ot +Ġst ains +ĠStat ue +ĠApost les +he ter +ĠSUP PORT +Ġoverwhel m +Y ES +Ġ29 1 +d ensity +Ġtra pping +M it +Ġf ide +ĠPam ela +atl antic +Dam n +Ġp ts +OP A +Ġserv icing +Ġoverfl owing +ul o +ĠE rit +t icket +light ing +ĠH mm +ãĥ¼ ãĥ« +im oto +Ġchuck le +4 23 +ãģ ķ +sh ape +Ġque ues +Ġanch ors +ãĤ¼ ãĤ¦ãĤ¹ +F er +Ġaw oke +Ġ6 66 +h ands +Ġdiver gence +Ġ50 5 +T ips +Ġdep ot +Ġske w +ĠDel iver +op ot +Ġdiv ul +ĠE B +uns igned +ĠUn i +X box +Ġfor ks +Ġ7 02 +å ¯ +Ġpromot ers +ĠV apor +Ġlev ied +sl ot +Ġpig ment +Ġcyl inders +C RE +Ġsn atch +Ġperpet ually +Ġl icking +ĠFe et +ĠKra ken +ĠHold en +ĠCLS ID +m r +Ġproject or +Ġden otes +Ġchap el +ĠTor rent +b ler +R oute +ĠDef endant +ĠPublisher s +ĠM ales +ĠInn ov +ĠAg ility +rit er +ty mology +st ores +L ind +Ġf olly +ĠZur ich +B le +Ġnurt ure +Ġcoast line +uch in +D omin +Ġfri vol +ĠCons olid +res ults +M J +Ġphyl ogen +Ġha uled +ĠW iley +ĠJess ie +ĠPrep are +ĠE ps +Ġtreasure r +I AS +Ġcolon ists +Ġin und +ĠWW F +ĠCon verted +6 000 +out side +ĠApp earance +ĠRel ic +ĠM ister +s aw +Ġresult ant +Ġadject ive +ĠLaure l +ĠHind i +b da +Pe ace +Ġreb irth +Ġmembr anes +Ġforward ing +Ġcoll ided +ĠCar olyn +K ansas +5 99 +ĠSolid GoldMagikarp +Be ck +Ġstress ing +ĠGo o +ĠCooper ative +Ġf s +ĠAr chie +L iter +ĠK lopp +J erry +Ġfoot wear +War ren +Ġsc ree +h are +Under standing +P ed +Ġanth ology +ĠAnn ounce +M ega +Ġflu ent +Ġbond age +ĠDisc ount +il ial +C art +ĠNight mares +Sh am +ĠB oll +uss ie +H ttp +Atl anta +Ġun recogn +ĠB id +Ġunder grad +Ġforg iving +ĠGl over +AAAA AAAA +4 45 +V G +pa io +kill ers +Ġrespons ibly +Ġmobil ize +Ġeffect ed +ĠL umin +Ġk ale +Ġinfring ing +ann ounced +Ġf itt +b atch +ĠT ackle +ĠL ime +ĠAP P +uke mia +Ġrub y +Ġex oner +ĠCas ual +0 70 +Ġpel vic +Ġautom ate +ĠK ear +ĠCoast al +Ġcre ed +Ġbored om +ĠSt un +ri ott +Ĥ İ +Ġregener ate +Ġcomed ians +ĠOP ER +Sp ons +id ium +on is +L ocated +05 7 +Ġsusp ense +ĠD ating +C ass +Ġneoc ons +ĠShin zo +Ġaw oken +ch rist +ĠMess ages +att led +ĠSpr ay +ĠSp ice +C W +Ġshield ing +ĠG aul +Am id +Ġparam ilitary +Ġmult if +ĠTan ner +il k +Ġgodd amn +g ements +Ġbe friend +m obi +Ġ3 88 +fold er +acc a +Ġins in +g ap +N ev +fif th +Ġpsychiat ry +b anks +TH IS +Ġhar b +ac qu +Ġfac ade +ĠPower Point +80 3 +Ġbl uff +Sh ares +Ġfavor ing +El izabeth +Ãį Ãį +Ġr anger +77 2 +ĠAr che +h ak +ĠGen etics +ĠF EMA +Ġev olves +Ġest e +ĠP ets +ĠM é +ĠInterest ing +ĠCanter bury +ch apter +ĠStar fleet +Sp anish +Ġdraw back +ĠNor wich +9 70 +n orth +ag anda +Ġtransform ative +ram ids +bi ology +ad ay +Ġpropag ation +ĠGam ma +ĠDen ise +ĠCalcul ator +ent imes +ĠB ett +Ġapp endix +ĠHD D +AK ING +Ġst igmat +Ġhol ster +Ġord inarily +Ch ance +ĠCont rary +Ġad hesive +Ġgather s +6 12 +re au +ony ms +ew ays +Ġindu ces +Ġinterchange able +se m +Wh it +Ġtr ance +Ġincorpor ation +ĠExt ras +Fin ancial +Ġawkward ly +ĠStur geon +ĠH Y +Norm ally +ĠEnd ing +ĠAss ist +enc rypted +Ġsub jug +Ġn os +Ġfan atic +C ub +C U +?" . +Ġirre versible +å Ĥ +03 1 +ĠH AR +sp read +ul ia += $ +Sc ope +L ots +Ġlif estyles +ol on +Ġf eds +Ġcongrat ulate +web kit +Ġindist inguishable +ĠSw ing +Ġcommand ments +qu ila +ab ella +m ethyl +ann abin +Ġo vere +Ġlob ster +ĠQU EST +ĠCONT IN +bern atorial +:::: :::: +ĠTra ve +ĠSam oa +AN I +75 2 +Ð ´ +userc ontent +ĠMod erate +y eah +ĠK itt +Ġwe e +Ġstuff ing +ĠInter vention +ĠD ign +Ġware houses +ĠF iji +Ġpel lets +Ġtake away +ĠT ABLE +ĠClass ical +col lection +Ġland fall +ĠMus cle +Ġsett les +ĠAD V +Ġ3 44 +L aura +Ġf ared +ĠPart ial +4 36 +oss ibility +ĠD aly +ĠT arant +ĠFu ji +am l +c ence +55 1 +ĠProced ures +ĠO CD +ĠU D +t in +Q UI +ach o +4 38 +Ġgl itches +Ġenchant ment +Ġcalcul ates +IR O +ĠH ua +alys es +ĠL ift +um o +Ġle apt +Ġhypothes ized +ĠGust av +it ans +VERS ION +æ ł +Rog er +Ġr and +ĠAd apter +Ġ3 31 +ĠPet ition +k ies +M ars +Ġunder cut +ze es +ĠLy ons +ĠDH CP +Miss ing +Ġretire es +Ġins idious +el i +> ) +. ãĢį +Ġfinal ists +ĠA ure +Ġacc user +Ġwas tes +ĠY s +ĠL ori +Ġconstitu encies +Ġsupp er +Ġmay hem +or ange +Ġmis placed +Ġmanager ial +Ġex ce +ĠCL I +Ġprim al +ĠL ent +Cry stal +h over +ĠN TS +end um +Ġd w +ĠAl c +n ostic +Ġpres erves +ĠTs arnaev +Ġtri pled +rel ative +Arc ade +k illing +ĠW EEK +ĠH anna +D ust +Com pleted +ģ « +Ġappro ves +ĠSur f +ĠLuther an +ven ants +Ġrobber ies +we ights +soft ware +at ana +ug al +Ġgrav y +ĠC ance +OLOG Y +ly ak +Ton ight +Ġunve il +Ġ19 04 +ĠMin ion +ent ious +st ice +pack ages +ĠG EAR +Ġg ol +ĠHutch inson +ĠProf ession +ĠG UN +ĠDiff erence +ĠTsuk uyomi +ĠLes bian +6 70 +Ġfug itive +ĠPlan etary +-------------------------------- ------------------------ +Ġacc rued +Ġch icks +Ġsto pp +Ġblock ers +C od +Ġcomment ers +ĠSomew here +ĠPhot ographer +the me +Ġmay oral +w u +Ġanten nas +Ġrev amped +ĠSubject s +it é +im ura +Ġentr ances +liter ally +Ġten ets +ĠO MG +ĠMP H +ĠDon key +ĠOff ense +Ġ" + +Sn ap +ĠAF B +Ġan imate +ĠS od +His panic +Ġinconsist ency +D b +F Y +Ex port +Ġa pe +Ġpear l +ib el +ĠPAC s +Ġ{ \ +Ġact u +ĠHS BC +camp us +Ġpay off +Ġde ities +ĠN ato +ou ple +Ġcens ored +ĠCl ojure +Ġconf ounding +en i +Ġreck on +op he +Ġspot ting +Ġsign ifies +Ġprop el +Ġfest ive +S uggest +Ġpled ging +ĠB erman +Ġrebell ious +Ġovershadow ed +Ġinfiltr ated +j obs +67 2 +Ġscal able +Ġdomin ion +ĠNew foundland +ĠMead ow +Ġpart itions +AM I +Ġsupplement ary +str ument +Ġhair y +Ġperpet uate +Ġnuts hell +ĠPot ato +ĠHob bit +Ġcur ses +Flo at +Ġquiet er +Ġfuel ing +Ġcaps ules +ĠL ust +ĠH aunted +Exec utive +Ġchild birth +G re +Ġrad iant +å İ +Ġm alls +Ġin ept +ĠWarrant y +Ġspect ator +E h +t hens +Ġculmin ating +æ © +ary a +ãĤ ® +ilit arian +ĠOR IG +ĠSp ending +pt ives +ĠS iren +ĠRec ording +ay ne +Ġv im +Ġspr ang +T ang +ĠM FT +mor ning +ĠWe ed +m peg +cess ion +ĠCh ung +7 30 +w arning +56 2 +handed ly +P oor +P olitics +: # +Ġp ian +Ġfec es +ĠDocument ation +Ġban ished +Ġ3 99 +ĠAR C +Ġhe inous +J ake +ĠAm ir +way ne +v re +os henko +Ġnotebook s +Ġfound ational +Ġmarvel ous +ixt ape +Ġwithdraw als +Ġh orde +ĠD habi +is able +ĠK D +Ġcontag ious +ĠD ip +ĠAr rows +Ġpronoun s +Ġmorph ine +ĠB US +68 2 +Ġk osher +fin ished +ĠInstr uments +Ġf used +yd en +ĠSal mon +F ab +aff ected +K EN +C ENT +Dom ain +Ġpoke mon +ĠDr inking +G rowing +ĠInvestig ative +ĠA ether +em i +Ġtabl oid +Ġrep ro +ĠNot withstanding +ĠBers erker +Ġdram as +Ġclich é +Ġb ung +ĠU RI +ĠD os +0 44 +Ġpast ors +Ġl s +Ġac rylic +aun ts +Ed ward +Ġmajor ities +B ang +Ġfield ing +ĠRepl acement +ĠAl chemy +pp ard +ĠRome o +ĠSan ct +ĠLav rov +ib ble +Inst ruct +Ġimp ractical +ĠPlay boy +ce phal +Ġsw aps +Ġk an +ĠThe o +Ġillust rating +Ġdismant led +ĠTrans gender +ĠG uth +UG H +Ġtriumph ant +Ġencomp ass +Ġbook mark +udd in +j er +Ġpred icate +ES H +Ġwhen ce +ĠAB E +Ġnon profits +Se qu +Ġdi abetic +Ġp end +Ġheart felt +sh i +Ġinter acts +ĠTele com +Ġbombard ment +dep ending +ĠLow ry +ĠAd mission +ĠBl ooming +ust ration +ene gger +B rew +Ġmol ten +ĠNer d +P IN +âĸ Ģ +ave ment +Ġtou red +Ġco efficients +ĠTray von +ans son +Ġsand y +t old +fl ows +Ġpop ulous +ĠT inder +ĠBl iss +R achel +Min imum +Ġcontest ant +ĠRed uce +ĠMor se +ĠGrass ley +ĠClick er +Ġexp r +Ġs incerity +Ġmar qu +Ġelic it +ĠPro position +ĠDemon ic +Ġtac os +G reek +Ġpost war +Ġin sofar +ĠP ork +Ġ35 2 +doctor al +walk ing +Ġmid term +ĠSam my +sight ed +ĠTR ANS +ic i +AL D +ĠUS L +ĠF ISA +ĠAm pl +ĠAlex andra +ine lli +Tr ain +Ġsign ify +ĠVers us +Ġob fusc +Ġk h +Ġagg ro +ĠRen ault +Ġ3 48 +5 18 +ox icity +0 22 +ĠTw ist +Ġgoof y +D ynamic +Ġbrief ings +m ight +8 99 +Ġderog atory +T ro +Ġfor ging +ĠKor an +ĠMar ried +ĠBuc s +Ġpal ate +ĠCon version +m able +4 13 +Ġ( _ +Ġs iph +ĠN EO +col lege +Ġmarg inally +Ġfl irt +ĠTra ps +ĠP ace +é »Ĵ +Ġgoalt ender +Ġforb ids +Ġcler ks +ĠT ant +ĠRobb ins +ĠPrint ing +Ġpremie red +Ġmagn ification +ĠT G +ĠR ouse +ĠM ock +odynam ics +Ġpre clude +ism o +ĠPul itzer +Ġaval anche +ĠK odi +rib une +ĠL ena +Elect ric +Ġref inery +Ġend owed +Ġcounsel ors +Ġd olphin +ĠM ith +Ġarm oured +hib ited +Beg in +ĠP W +O il +ĠV or +ĠShar if +ĠFraz ier +est ate +Ġj ams +Pro xy +Ġband its +ĠPresbyter ian +ĠPrem iere +t iny +ĠCru el +Test ing +Ġhom er +ĠV ERS +ĠPro l +ĠDep osit +ĠCoff in +Ġsemin ars +Ġs ql +ĠDef endants +Altern atively +ĠR ats +ç « +ethy st +' > +Ġiss uer +58 9 +Ġch aired +ĠAccess ories +man ent +Ġmar row +ĠPrim ordial +C N +Ġlimit less +ĠCarn age +Ġund rafted +q v +IN ESS +on ew +Ġco hesion +98 7 +Ġne cks +Ġfootball er +ĠG ER +Ġdetect able +ĠSupport ing +ĠCS V +oc ally +k Hz +Ġund e +Ġsh one +Ġbud ding +tra k +Stand ing +ĠStar craft +ĠKem p +Ben ch +Ġthw arted +ĠGround s +ath i +L isa +Dial og +ĠS X +V ision +Ġingen ious +Ù IJ +Ġfost ering +ĠZ a +ĠIn gram +Ġ" @ +N aturally +6 16 +0 35 +ĠF AC +H mm +55 4 +Ġacceler ator +ĠV end +Ġsun screen +Ġtuber culosis +rav iolet +ĠFunction al +ĠEr rors +ed ar +19 66 +ĠSpect re +ĠRec ipes +88 5 +ĠM ankind +L iverpool +Ġ| -- +Ġsubst itutes +ĠX T +w ired +Ġinc o +ĠAf gh +E va +ic c +S ong +K night +Ġdilig ently +ĠBroad cast +A id +Ġaf ar +ĠH MS +aton in +ĠGr ateful +Ġfire place +ĠOm ni +e uro +ĠF RE +ĠSh ib +ĠDig est +t oggle +Ġheads ets +Ġdiff usion +ĠSqu irrel +ĠF N +Ġdark ened +out her +Ġsleep s +ĠX er +gun s +Ġset ups +Ġpars ed +Ġmamm oth +ĠCur ious +g ob +ĠFitz patrick +ĠEm il +im ov +........ ..... +ĠB enny +Second ly +Ġheart y +Ġcons on +st ained +Ġgal actic +cl ave +Ġplummet ed +Ġp ests +Ġsw at +Ġrefer rals +ĠLion el +h oly +Ġunder dog +ĠSl ater +ĠProv ide +ĠAm ar +ress or +å Į +ong a +Ġtim id +Ġp iety +ĠD ek +Ġsur ging +az o +Ġ6 10 +Ġdes ks +ĠSp okane +ĠAn field +Ġwars hips +ĠCob ra +Ġar ming +clus ively +ĠBad ge +ag ascar +ĠPR ESS +ĠMcK enzie +ĠFer dinand +burn ing +Af ee +Ġtyr ann +ĠI w +ĠBo one +100 7 +ĠRe pt +Ċ Âł +Ġcar avan +ĠD ill +ĠBundes liga +Ch uck +Ġheal er +ãĥ¼ãĥ Ĩ +ĠH obby +Ġneg ate +Ġcrit iques +section al +mop olitan +Ġd x +Ġouts ourcing +ĠC ipher +t ap +Sh arp +Ġup beat +Ġhang ar +Ġcru ising +ĠNi agara +Ġ3 42 +ill us +ĠS v +Ġsubt itles +Ġsqu ared +Ġbook store +Ġrevolution aries +ĠCarl ton +ab al +Ut ah +Ġdesp ise +ĠU M +cons ider +aid o +Ġc arts +ĠT urtles +Tr aining +Ġhonor ary + ¢ +Ġtri angles +4 22 +Ġreprint ed +Ġgrace ful +ĠMong olia +Ġdisrupt ions +ĠB oh +Ġ3 49 +Ġdr ains +Ġcons ulate +Ġb ends +Ġm afia +ur on +ĠF ulton +m isc +Ġren al +Ġin action +ck ing +Ġphot ons +Ġbru ised +ĠC odes +og i +Ġn ests +ĠLove ly +ĠLib re +ĠD aryl +Ġ# ## +S ys +. ," +Ġfree zes +est ablishment +and owski +Ġcum bers +ĠSt arg +ĠBom bs +Ġleg ions +Ġhand writing +Ġgr un +ĠC ah +sequ ent +Ġm oth +ĠMS M +Ins ert +F if +Ġmot el +Ġdex ter +ĠB ild +hearted ly +Ġpro pe +ĠText ure +ĠJ unction +ynt hesis +oc ard +ĠVer a +ĠBar th +Ġμ g +Ġl ashed +Ġ35 1 +ĠZ amb +ĠSt aples +ĠCort ex +ĠCork er +Ġcontinu um +ĠWR ITE +unt a +rid or +Ġde ems +0 33 +ĠG OLD +p as +Ġrep ressive +ãĥĨ ãĤ£ +Ġbaff led +Sc ar +Ġc rave +Ġ ______ +Ġentrepreneurs hip +ĠDirector ate +Ġ' [ +Ġv ines +Ġasc ended +ĠGR OUP +ĠGood bye +Ġdo gged +ãĥ´ ãĤ¡ +Man ufact +Ġunimagin able +ri ots +ier rez +Ġrel ativity +ĠCraft ing +ra ught +ud en +c ookie +Ġassass ins +Ġdissatisf ied +ac ci +Ġcondu it +Sp read +ĠR ican +n ice +izz le +Ġsc ares +ĠWH Y +ph ans +5 35 +Ġprot racted +ĠKrist en +5 36 +ĠSc rib +ĠNe h +Ġtwent ies +Ġpredic ament +Ġhandc uffs +Ġfruit ful +ĠU L +ĠLud wig +Ġatt est +ĠBre aker +Ġbi ologically +ĠDeal er +Ġrenov ations +f w +ess en +Al ice +ĠHen ri +Ġun ilaterally +ĠS idd +h ai +ĠSt retch +S ales +Ġcumbers ome +ĠJ avier +Ġtrend y +Ġrot ting +ĠChall enges +Ġscra ps +Ġfac ets +ĠVer onica +ĠVer ge +ĠS ana +Al ien +ĠR ih +Ġrad ial +ect ar +Ġ6 30 +cl i +Mar ie +Ġwild fire +ĠCat o +h ander +Ġwait ress +Ġch ops +ĠS ECTION +Ġblunt ly +ĠCat alog +n ian +stud y +Ġpat rolling +ĠT enth +nex us +ĠN ON +op sy +Ġsc athing +s ie +Ġdeterior ated +V B +Naz is +Ġdep ictions +Ġauthent icated +ĠCon ce +k rit +Ġpromul g +ĠL ONG +U FC +ĠVis itors +ĠRec all +Ġrehab ilit +ĠSL I +Ġglac ier +ĠB ite +Ġ50 3 +Ġvom it +Ġfer mented +ĠKh alid +Ġgrad ed +ĠMag icka +ĠIch igo +power ful +ic ators +75 3 +Ġsh rew +Ġ35 6 +Ġlegal izing +Ġall otted +ĠArch demon +ith ing +igg urat +V OL +Le od +Ġo ily +Ġindu cing +Ġamy gdala +Ġadm ins +ĠAcqu isition +C AN +Ġsche matic +Ġmo an +ĠCamer oon +Ġt ink +Ġmer ry +Ġbutter flies +ĠGo ff +Ġworks pace +ĠCor ona +Ġj avascript +ĠD olphin +ĠCant or +4 64 +to e +AP S +ĠAg ing +Ġpadd ed +ĠZ heng +ĠHe ld +Ġest ranged +Ġ7 70 +. } +ĠDun ham +Ġsm okes +Ġcap itals +und ai +Sh in +ĠFound ing +Ġent itle +Ġcenter piece +D iscover +Ġthere to +al ert +ĠN ou +ĠAnaly st +l c +F H +FI ELD +ĠP OV +gr ay +Ġar cs +ĠH OT +Ġr s +Ġoblig atory +ĠArchitect s +ĠS ven +ĠF EC +0 200 +Christ mas +ĠAlban ia +rat om +58 7 +Ġhard ships +Ġaut os +ĠCharg es +Ġap es +Ġ3 76 +wal let +Ġintox ication +Ġgobl in +Ġ5 70 +++++++++ ++++++++ +ĠYel p +ĠMag netic +ĠBr iggs +R ail +Ġspawn s +ĠW iggins +Ġshowc ased +Ġres orted +ub en +Ġwh ipping +Ġim itate +Ġdigest ion +ĠUS PS +ĠG est +Ġye a +ĠT ight +ind al +ic as +` . +C AST +'' ; +ĠF et +opath ic +In valid +Ġregrett ed +Ġbro ccoli +ĠSc ores +e ve +Ġpost ings +Ġaccum ulating +Ġneed less +elf th +Ġmay ors +Ġsc rib +Ġanecd otes +Ġbot ched +ĠRib bon +ĠConstant ine +i uses +ess es +Ġdev ise +Comp ared +Ġp udding +Ġg arg +Ġev oke +79 7 +Ġdet ox +9 09 +ĠPie ces +ĠMcC artney +Ġmet ast +ĠK rypt +P OR +Ġt ending +ĠMerch ants +Pro of +ĠV arg +ĠPort able +ãĥ¼ãĥĨ ãĤ£ +B rain +25 00 +Ġfol iage +Ø ¹ +Ġment ors +ĠA ires +Ġminimal ist +Ġing ested +ĠTro jan +ĠQ ian +inv olved +0 27 +Ġer oded +RA FT +Ġbl urry +M ob +Ġbuff et +ĠFn atic +ae a +KN OWN +ĠIn it +s afety +en um +ACT ION +ĠCrus her +ĠD ates +Ġ ................ +c alling +ak ov +Ġvent ured +Ġ5 55 +au ga +H art +ĠA ero +M AC +Ġthin ly +Ġar ra +ST ATE +ild e +ĠJac qu +ĠFem ales +Ġthe orem +Ġ3 46 +Ġsmart est +ĠPU BLIC +ĠK ron +ĠB its +ĠV essel +ĠTele phone +Ġdec ap +Ġadj unct +ĠS EN +mer ga +Ġred acted +Ġpre historic +Ġexplan atory +ĠRun s +ĠUtt ar +ĠM anny +ĠAUTH OR +ĠUnle ashed +ĠBow ling +be ans +79 3 +Ġunivers es +Ġsens it +ĠK ung +re peat +ctr l +Ġp aced +Ġfull er +Cl ock +Ġrec omb +ĠF aul +ĠB unker +Ġpool ed +Ġan a +ĠM outh +LL OW +hum ane +Ġbull do +ĠMicha els +f am +Ġwreck ed +Ġport rays +ĠWh ale +ĠH es +Ġguess es +ĠBrow se +ĠL APD +Ġconsequ ential +ĠInn ocent +ĠD RAG +Ġtrans gress +ĠO aks +Ġtri via +ĠRes on +ĠA DS +-- + +ĠT oll +Ġgrasp ing +ĠTHE M +ĠT ags +ĠCon clusion +Ġpract icable +Ġho op +Ġunintention ally +Ġign ite +ĠM ov +ur ized +le hem +Ter min +Ġcolour ful +ĠLin ear +ĠEll ie +G y +Ġman power +Ġj s +Ġem oji +ĠSHAR ES +_ . +0000 7 +Ġsophistic ation +Ġunders core +Ġpract ise +Ġbl ob +op ens +Uk raine +Ke eping +Y C +J R +ult imate +Cl aim +Ġautom obiles +99 3 +ste el +Ġpart ing +ĠL ank +... ? +Ġ38 5 +Ġremem brance +Ġe ased +Ġcov ari +ĠS ind +Effect ive +Ġdisse mination +ĠMo ose +ĠCl apper +br ates +App ly +Ġinv is +Ġwors ened +âĢĶ - +Ġlegisl ator +ĠL ol +ĠRow e +Ġdealers hip +um ar +id ences +Ġinvestig ates +Ġc ascade +Ġbid der +ĠB EN +Iron ically +Ġpres iding +Ġd ing +Ġcontrad icted +Ġshut s +ĠF IX +Ġ3 66 +Dist rict +Ġsin ful +ĠChar isma +o ops +Ġtot ality +Ġrest itution +ĠOpt imus +ĠD ah +Ġcl ueless +urn ed +Ġnut rit +Ġland owners +Ġfl ushed +Ġbroad en +m ie +Ġprint ln +Ġn ig +ĠCorp us +J en +Ġprot o +ĠWik imedia +ĠPal o +C OR +Ġstory lines +Ġevangel icals +ĠDar rell +Ġrot or +ĠH W +sk illed +ery l +Ġbe gg +ĠBl umenthal +Ġwe aving +Ġdown wards +ĠJack et +ĠANG EL +Te chnology +Ġes oteric +alde hyde +Ġfur iously +Ġforeign er +We ak +CH O +ĠH ound +Exper ience +ĠPlay station +ĠM IA +ĠU ng +cl oth +ag all +Ġcal ming +iz ens +St ruct +ĠW itches +ĠCeleb ration +Ġ........ ...... +pt roller +ĠTC U +Ġb unny +ãĥ į +ut orial +Ġup scale +ĠSt a +ĠCol ossus +Ġchlor ide +ĠZ ac +ĠRe asons +ĠBrook ings +ĠWH ITE +][ / +ĠL ose +9 05 +Ġunders ide +ern els +Ġv ape +do zen +upp et +ĠST OP +mat ical +ĠStat ements +hed dar +P AC +Custom er +Ġmem os +ĠP J +end ars +ĠLim its +l augh +Ġstabil ized +ĠALE C +Y A +Up grade +al am +Ġtechn o +Ġan ew +fore seen +Ġcolleg iate +ĠPy ro +ĠD ism +Ġfront line +Ġammon ia +I U +Qu ite +John ny +ass in +G OP +ĠSt yles +ĠSovere ign +acter ial +5 49 +ĠR IP +ĠL ists +Ġ3 64 +ĠRece p +s ocket +ĠByr d +ĠCand le +An cient +Ġappell ant +en forcement +ace a +ans ki +Ġold s +88 6 +Ġsl urs +Ġem pires +Ġbuck le +Ġalien ation +ĠAber deen +Ġunic orn +Ġoverr iding +ĠL X +pp a +Ġdesp ised +ĠB ugs +ĠB ST +S outhern +5 33 +Ġhall mark +ĠPost er +Ġstem med +Ġprincip als +ĠT ECH +ĠSand wich +It aly +Ġche esy +ĠSet TextColor +ĠProt ective +ĠC ohn +J O +apt op +Re ason +Lead er +ĠUnder stand +ĠFr idays +ĠContin uous +Ġcl ipping +ĠR ye +Ġber th +tim er +ann is +re act +Ġbuff alo +ĠPar as +Ġ6 55 +Ġpres ided +ĠSun rise +Ġve ts +Ġcl oves +ĠMcC ull +Stre ngth +G AN +Ġill iter +ĠPric ing +l é +Ġresist or +Ġbr un +ĠSuff olk +Ñ ĭ +ĠL iver +Re leased +Ġwhat s +8 60 +ĠMe asures +Ġden ouncing +ĠRy zen +Ġsou ven +Ġcareg ivers +ch ini +ĠScar lett +Ġt rough +Cong ratulations +Ġtax is +ĠTrad ition +j it +Ġtable top +Ġhither to +Ġdis information +off ensive +h ra +ĠDISTR ICT +Ġcompl icate +chen ko +ĠRecon struction +Ġpalp able +Ġa usp +Ġ4 28 +Ġshowc ases +ĠPublic ation +know ledge +inn on +4 19 +Ġretri eval +and ers +Ġref ute +Ġinqu ired +g ur +Ġneg ativity +Ġcons erve +Ġafter life +Ġpres upp +ĠGill espie +Ġm t +ĠD N +T ap +Ġper pend +ĠS my +does n +Ġsp illing +Ġhyp ers +K ate +® , +ke pt +ĠP owered +Ġj a +ĠK lux +ard e +ab an +Ġ4 44 +Ġflatt ened +ĠImprove ments +urg a +ĠK und +Ġins cribed +Ġfac ult +Ġunpre pared +ĠCons umers +Ġsatisf ies +Ġpul monary +Ġinf iltration +Ġex ternally +Ġcongrat ulations +ag han +Ġair liner +Ġfl ung +Ġfly ers +G D +Ġsnipp ets +Ġrec ursive +Ġmaster ing +L ex +Ġovert ly +v g +Ġluck ily +Ġenc ro +ĠLanc et +ĠAbyss al +function al +Ġs ow +Ġsqu id +Ġnar ration +Ġn aughty +ĠHon our +ĠSpart ans +Ġsh atter +ĠTac oma +ĠCal ories +ĠR aces +Sub mit +Ġpurpose fully +w av +ĠY ok +F est +ĠG err +Met ro +Ġit iner +f amous +Ġ" { +in line +was her +Iss ue +ĠCL IENT +oz o +Vers ions +7 25 +ĠGl ock +Ġshield ed +ĠPC R +ENC Y +ĠWe ld +ĠSim pl +Ġredirect ed +ĠK ham +Ġ( > +Ġlab ou +Ġdi apers +ss l +Ġcell ar +organ isms +ore sc +ĠBer ks +did n +Sh ipping +C hest +Ġund one +Ġmillion aire +Ġc ords +ĠYoung er +appropri ately +Ġsequ els +u ve +ant icipated +Ġle wd +ĠSh irt +ĠDmit ry +V eter +Ġsl aying +ĠY ar +Ġcompl ication +I owa +ĠEric a +ĠBL M +g irlfriend +b odied +6 26 +19 63 +Ġintermedi ary +Ġcons olation +M ask +ĠSi em +ow an +Beg inning +Ġfix me +Ġculmin ated +Ġcon duc +ĠVolunte er +Ġpos itional +Ġgre ets +ĠDefin itions +Ġthink er +Ġingen uity +Ġfresh men +ĠMom ents +Ġ35 7 +ate urs +ĠFed Ex +s g +69 4 +Ġdwind ling +ĠBO X +sel age +Ġt mp +Ġst en +ĠS ut +Ġneighbourhood s +Ġclass mate +f ledged +Ġleft ists +Ġclim ates +ATH ER +ĠScy the +ul iffe +Ġs ag +Ġho pped +ĠF t +ĠE ck +ĠC K +ĠDo omsday +k ids +Ġgas ped +Ġmon iker +ĠL od +ĠC FL +t ions +r ums +fol ios +Ġm d +Ġunc anny +Ġtrans ports +ĠLab rador +Ġrail ways +Ġappl iance +ĠCTR L +æ Ģ +Pop ulation +ĠConfeder acy +Ġunb earable +Ġdors al +ĠIn form +op ted +ĠK ILL +Mar x +Ġhypoc ritical +q us +ĠN umerous +ĠGeorg ian +ĠAmbro se +ĠL och +Ġgu bernatorial +ĠX eon +ĠSupp orts +ens er +ee ly +ĠAven ger +19 65 +Ar my +Ġju xtap +Ġcho pping +ĠSpl ash +ĠS ustainable +ĠFin ch +Ġ18 61 +ict ive +at meal +ĠG ohan +Ġlights aber +ĠG PA +ug u +ĠRE PL +vari able +Ġher pes +Ġdesert s +ac iously +Ġsitu ational +week ly +ob l +Ġtext ile +ĠCorn wall +Ġcontrace ptives +ĠA ke +] - +ä¹ ĭ +: , +ĠW em +ĠB ihar +Ġ' . +Ġbe re +Ġanal ogue +ĠCook ies +Ġtake off +Whe el +Ġmaj estic +Ġcomm uting +0 23 +ĠCor pse +ass ment +min i +Ġgor illa +ĠAl as +ere e +Ġacquaint ances +ĠAd vantage +Ġspirit ually +Ġey ed +pm wiki +ĠE nder +Ġtrans lucent +Ġnight time +ĠIM AGES +5 45 +ĠK amp +ĠFre ak +Ġ ig +Port land +4 32 +ĠM ata +Ġmar ines +Ġh ors +ater asu +ĠAtt ribution +Ġ-------- - +Ġk ins +ĠBEL OW +++ + +Ġre eling +ol ed +Ġcl utter +ĠRel ative +Ġ4 27 +B US +Ġa vert +ĠChe ong +ĠA ble +ĠPry or +Develop er +Ġen cyclopedia +ĠUSA F +ĠG arry +Sp ain +Bl ocks +Ġexp osition +ĠGamer Gate +W OR +Ġstockp ile +Ġclot hed +ĠT one +ĠR ue +t umblr +Ġtreacher ous +Ġf rying +Ñ Į +ĠS ph +Ġrest raints +Ġemb odies +ĠG es +S afety +Ġnegoti ators +min ing +ĠAppalach ian +L OS +ĠJenn a +Ġpass ers +ç ĭ +sn ap +Ġshort en +creat or +Ġinn umerable +uther land +67 4 +ĠW OM +ĠAs cend +ĠArm ory +ĠTrans action +K ick +Ġsuit case +day Name +Ġwaste ful +mar riage +ĠMcC abe +ite ch +ĠO ss +Cl osure +ĠTreasure r +Ġindec ent +ĠD ull +Ġresid ences +19 59 +ĠS ettlement +Ham ilton +Ġself ies +ĠRank ing +ĠBark ley +ĠB ore +ĠW CS +ĠMar itime +ĠH uh +ĠForest ry +Ġcultiv ating +ĠBall ard +Ġg arrison +ĠSD L +9 30 +Ġnas cent +Ġirresist ible +Ġaw fully +\/ \/ +Ġequ ate +Ġanthrop ology +ĠSylv ia +Ġintest ine +Ġinnoc uous +cess ive +ag ra +ĠMet roid +G rant +8 55 +ģ ĸ +Ġ" _ +ãĥĥ ãĥī +Ġappra isal +ĠFred dy +04 6 +Ġ40 6 +Ġ18 30 +Ġd ocking +St atic +Ġp ont +ĠVolt age +ĠSt ead +ĠMort gage +ĠJon ah +Y L +CLASS IFIED +Ġas bestos +nik ov +Ġcoll agen +ĠOrb ital +P ocket +7 99 +Ġhy brids +inc hes +Ġinv oice +und y +Ġinequ alities +T rend +w ashed +B ALL +Ġluc id +ĠComment ary +Ġw itty +Br andon +Ġbru ising +Ġ6 20 +es cent +box ing +P OL +Ġ3 78 +R ect +Ġlic ences +ĠMcG ee +p ressed +D anny +Ġj ammed +ord inate +Ġle th +Ġdistingu ishes +ĠYam aha +IL S +ĠH ume +ĠC ategories +Rober ts +Ch art +Ġbeet le +ĠGra veyard +Ġ($ ) +o ÄŁ +Ġtw ilight +are lla +á ½ +Ġbooth s +ĠH HS +ĠFeld man +Ġexcav ation +Ġphilosoph ies +at ography +ĠGar age +te chnology +Ġunfor gettable +Ġver ifying +Ġsubord inates +E ls +Ġne b +G aming +EN A +ĠAchieve ment +it ters +ĠG abe +Ġd umps +for cer +Ġpo ignant +ĠM BA +ĠHe idi +ime i +Ġm ages +Ġliber ate +Ġcircum cised +ĠMer maid +ĠMat th +t ogether +ĠW ichita +Ġstore front +ĠAd in +V II +Four th +Ġexplore rs +W ER +Not able +Bro ok +m ens +F aith +-------- - +ĠJ ou +¬ ¼ +Ġpine apple +Ġam alg +el n +ark able +ĠãĤµ ãĥ¼ãĥĨãĤ£ +ĠãĤµãĥ¼ãĥĨãĤ£ ãĥ¯ãĥ³ +Ġov arian +ĠE choes +Ġhairc ut +Ġp av +Ġch illed +anas ia +Ġsty led +Ġd ab +ni per +Ġminister ial +ĠD UP +T an +Ġsul ph +ĠD eter +ĠBo hem +od an +Ġeduc ator +â ĵĺ +sp ir +Ch icken +ĠE leanor +Ġqu i +Ġheav iest +Ġgrasp ed +U RA +Ġcro oked +Jess ica +pro blem +Ġpred etermined +Ġman iac +Ġbreath s +ĠLauder dale +Ġh obbies +y z +Cr ime +Ġcharism a +d L +Ġle aping +Ġk ittens +Ang elo +ĠJ ACK +ĠSu zanne +Ġhal ting +ENT ION +Ġswall owing +ĠEarthqu ake +Ġeight eenth +ĠN IC +ĠIN F +ĠCons cious +Ġparticular s +circ le +7 40 +Ġbene volent +Ġ7 47 +Ġ4 90 +Ġr undown +ĠVal erie +ĠB UR +Ġcivil isation +ĠS chn +W B +ot ide +intern ational +Ġj ohn +Ġ19 02 +Ġpe anuts +Ġflav ored +k us +Ġro ared +Ġcut off +é £ +Ġorn ament +Ġarchitect ures +Ġ3 69 +ol or +ĠWild e +ĠC RC +ĠAdjust ed +Ġprov oking +land ish +Ġrational ity +Ġjust ifies +Ġdisp el +Ġa meric +ĠPol es +Ø © +Ġen vis +ĠD oodle +ä½ ¿ +igs aw +auld ron +Techn ical +T een +up hem +ĠX iang +Ġdetract ors +ĠZ i +ĠJournal ists +Ġconduc ive +ĠVolunte ers +Ġs d +Know ing +Ġtrans missions +ĠPL AN +ĠL IB +Ġall uded +Ġob e +Ġd ope +ĠGold stein +Ġwavelength s +ĠDest ination +nd a +ug i +Ġattent ive +ĠLe an +ral tar +Ġman g +mb uds +ak ings +b ender +Ġacc ol +Ġcraw led +N OW +Min nesota +Ġflour ished +ĠZ up +ĠSuper visor +ĠOliv ier +Ex cellent +Ġwid en +D one +Ġw ig +Ġmiscon ceptions +Cor p +W an +Ġvener able +ĠNot ably +ĠKling on +an imate +Bo ost +ĠS AY +miss ing +ibli ography +mel on +Ġpay day +Ø ³ +bo le +Ġve iled +ĠAl phabet +It alian +Ġever lasting +ĠR IS +ĠC ree +rom pt +Ġh ating +Ġgrin ning +Ġge ographically +OS H +Ġwe eping +ĠÂłĠÂłĠÂłĠÂł ĠÂłĠÂłĠÂłĠÂł +Ġimpe cc +Let ter +Ġblo ated +PL A +ĠFe in +Ġper sever +Th under +Ġa ur +ĠR L +Ġpit falls +âĸ º +Ġpredomin ant +Ġ5 25 +7 18 +AP E +7 14 +Ġfarm land +ĠQ iao +Ġv iolet +ĠBah amas +Ġinflic ting +ĠE fficiency +Ġhome brew +Ġundert ook +Ġcur ly +ĠHard ing +man ia +59 6 +Ġtem pered +Ġhar rowing +ĠP ledge +ĠFranken stein +è ª +M otion +Ġpredict ably +ĠExpl osion +oc using +er d +col o +FF ER +Ġback field +ĠV IDE +ue bl +N arr +ĠArg ument +Ġgen omic +Ġbout ique +Ġbatt ed +ĠB inary +Ġg amb +ĠRh ythm +67 3 +Ġa float +ĠOlymp ia +Y ING +Ġend if +is in +Ġwin ters +Ġsc attering +I v +D istance +Ġtr u +ĠCom fort +Ġne xus +Ġair flow +ĠByz antine +p ayers +con i +ĠB etsy +D eal +ĠN ug +ĠContin ent +red ibly +Ġoptim izing +al beit +Ġec static +ĠPro to +ç · +iv ot +âĸ Ħ +em p +rou nder +Ġcl out +ĠI ST +66 3 +ĠDoll ars +ĠD AC +Ġsubsc ribed +Ġrehears al +Ġam ps +ĠSh ang +es m +Ġspr inkle +Ġassail ant +ĠO o +ĠCoin base +T act +Ġret ina +Ġn uns +R ON +att o +Ġj ug +ĠSV G +Ġb ikini +ĠFI LE +ĠFound ers +ep ort +ĠK P +Ġrest ores +ĠTh ick +Ġash ore +Ġappro vals +R ender +M AG +G raham +ĠCort ana +ãĥ³ ãĤ¸ +ss h +or ians +ars ity +ĠInsp ired +u pper +Ġsign alling +Ġreb uke +Ġfl ares +Ġdownt ime +Stud ies +Ġstagn ation +ĠSequ ence +Ġgr unt +Ġass ures +ĠPL A +59 2 +Ġintra ven +d epend +Sus an +ĠManz iel +Man ia +Cont ract +Ġsl ams +Ġcult ured +Ġcred itor +L IST +ĠH UM +ĠChatt anooga +serv ed +Ġclo aked +ĠF TP +p owder +ĠSt ella +uct ive +Ġcheap ly +ĠMU CH +ĠGalile o +Ġsu ites +spe ech +Ġdeliber ations +ĠCh ips +« ĺ +Bal ance +ĠWyn ne +ĠAk ron +Ass et +Ġhon oured +Ġed ged +Like wise +anim ous +ĠW age +ĠEz ek +ad vertisement +ĠRT X +ĠM AD +Ġmigr ating +ĠS QU +Ġ4 75 +Ed ited +Ġshorth and +ĠBas ics +Ġcro tch +ĠEV EN +Ġv m +effic iency +Ġcal ves +ĠF rie +ĠBrill iant +Ġstri kers +Ġrepent ance +Ġarter ies +r l +B ed +h ap +Ġcrypt ography +ĠSab res +Ġ4 14 +vi ks +ih ara +aps es +T alking +Ġintertw ined +Ġdoc ks +Ġalle le +ĠArt ifact +ĠH IM +t orn +ç ķ +Ġop acity +ĠE ly +os uke +Ġn ipple +Ġhand written +ĠV K +ĠChamber lain +ĠLa os +ig raph +g row +Ġtr illions +Ġdescend ant +ĠSail or +as uring +Ġce ilings +ĠWare house +f lying +ĠGl ow +Ġn ont +Ġmiscar riage +Ġrig s +Ġmin istries +Ġelabor ated +Ġdel usional +ĠHum ane +Ġ3 79 +n ets +Ġblack out +add ers +Ġn p +ĠT ire +ro sc +Ġsub div +Ġlink age +Ġchron ological +ĠHER O +Ġres ettlement +ĠVin yl +Ġpast oral +ĠMob il +ĠBar bar +Co oldown +ĠF ritz +c riminal +re pe +Ġbell ig +ĠBre ed +Ġ4 18 +Ġsem blance +ij k +Ġcur tail +Ġclin ch +cont ained +ĠProm pt +ast on +Ġw i +Ġpursu its +5 15 +ĠGl oss +Ġfl ips +Ġcoup ons +Ġcl oning +ĠLike ly +Rem oved +ĠQu artz +r ices +ĠSpe ars +Ġp ious +Ġdep reciation +ĠD are +oun ces +am az +O nt +Ġp innacle +d ocker +0 26 +ĠW yr +ĠPro per +Ë Ī +n il +By tes +Ġseek er +t rial +Ġunf olds +ĠMar se +Ġextravag ant +ĠSurviv ors +RED ACTED +ĠSpeed way +ĠCra igslist +sub mit +ĠGener ations +Ġup holding +Ġblood stream +ĠMiss ions +ĠL awn +Ġlim bo +ene i +H uh +ĠWild cats +pre p +ĠMark us +ĠFor bidden +rit ic +IN O +Ġexhib iting +requ ent +ch uk +Ġhabit ual +ĠComp atibility +Dr ag +RIP T +uj ah +GR OUND +Ġdelinqu ent +Ġburn er +Ġcontempor aries +Ġgimm ick +load s +Ġno zzle +p odcast +ĠW ak +ĠStat en +ĠK uh +ãģ ĵ +inter rupted +Ġinv incible +ĠBurn ett +cig arette +ĠPeb ble +ĠTem porary +ĠMar ino +58 2 +Ġwast eland +ident ly +T x +Ġr ite +ĠPan asonic +ĠM iddles +ĠHort on +ae us +Ġc uring +Ġm ats +Ġadj ourn +Ġfears ome +pe z +bo ats +Ġpro pell +Ġconflic ted +ĠAng er +Ġinsurg ent +K arl +Ġco ales +Ġsouth western +Ġdis su +ĠO vert +******** **** +Ġbox ed +ĠBr une +aa a +Ġgard ening +ĠEng el +tr acks +Ġpur ified +Ġplace holder +ĠL ikes +Ġd an +G ab +Ġe ct +ĠF aw +ĠEl iot +Ġ' , +otrop ic +ĠRu in +hed on +Ġca ul +Ġa ft +ĠCad illac +gh a +ass ian +ud eb +ĠT ick +Ġadjust s +AR GET +5 37 +isc he +ant y +ĠFried rich +ĠBl izz +ĠA OL +Camp aign +Ġmamm al +ĠVe il +ĠK ev +ĠMaur it +ĠDam ien +N ation +E astern +Ġ{ : +Ġ= ================================ +Ġstereotyp ical +Ġatt ic +ĠCy borg +requ ire +Ġaward ing +ĠPap ua +bt n +b ent +B oo +Ġ( = +ĠX ander +ĠSomers et +Ġcatch y +Ġcert ify +STR UCT +Ġit al +Ġt ides +ĠBr ands +G ray +comp etitive +Ġcur ator +ĠD G +omin ium +ĠGM Os +ci ating +ĠCarm en +ow ard +Balt imore +Ġr gb +C u +Ġwip es +spe ll +IT NESS +Ġsummar izes +ĠRe vis +Ġwhistlebl owers +ĠBre ach +Ġcro chet +k os +ews ki +Ġrep et +Ġcrim son +ĠKar achi +read able +dim ension +ĠI gor +ild ed +ĠZ ed +ĠKe ane +ĠCos metic +DE P +Ġretreat ing +ĠU A +ens ical +Ġd usk +ĠDick ens +Ġaren as +ĠPass age +level s +Ġcur v +P ope +Ġch ores +ĠEl ise +ĠComp ass +b ub +Ġmamm alian +ĠSans krit +ĠAN C +ĠCr ack +Q ual +L aun +amp unk +Ġlearn ers +Ġglam orous +Ġfur the +erm ott +c and +Gener ic +Ġnarr ated +Ġdisorder ly +ĠTrans actions +ĠDet ention +ĠR oku +Ä į +Ġunder statement +ĠS aur +ĠRodrig o +ĠAS AP +S in +Ġre joice +Method s +Ġelectro de +Ġworsh ipped +Ġid i +ĠPhys icians +Ġpop up +Ġde ft +ĠRem oval +ĠBu enos +ver bs +Ġfun k +ush a +rict ion +ore a +ĠBang alore +ĠKen obi +zz i +Ġnorm ative +Ġgobl ins +Ġcaf es +ĠUN CLASSIFIED +ĠF ired +S IGN +Ġs clerosis +ĠV oter +ĠSon ny +ĠExt end +ĠEV s +Ar senal +Ġp si +Ġwid est +ĠT us +Ġlo oms +Ġjust ifying +ĠGr anger +è ¯ +Ref er +58 3 +Ġflour ishing +ab re +Ġr ave +ĠCont ra +Ġ18 98 +Add s +Ġf ul +ĠCo oke +some one += # +67 1 +Ġy ak +Ġar te +ĠMis cellaneous +ĠDet ection +ĠCl ancy +â ģ +ass ies +Ġval iant +ĠFemin ist +cor ruption +V el +P ear +Ġsucc inct +Ġquick est +k w +Ġsp itting +ĠL ibraries +åħ ī +ant z +D ad +ĠSpec ifications +rup ulous +and r +RES ULTS +Ġsnow ball +Ġpred is +ĠB axter +ĠNurs ing +ĠCh aff +s we +Ġout age +Ġnest ing +Ġnotor iety +tr igger +on ite +j on +Ġf ou +ook ed +ĠCelebr ity +re ality +Ġfat ig +Ġhug ging +Ġbother s +ĠPan zer +ĠCh andra +fig ured +Ġvol ts +ĠCloud s +Ġfee ble +ĠCur ve +ĠAs us +78 6 +abs or +ĠV ICE +ĠH ess +Ġmanufact ures +Ġgri zz +ĠPower ful +ac id +Ġsub sections +ĠKrug man +ĠAl ps +is u +Ġsequ est +ĠUlt ron +ĠT inker +ĠGo ose +Ġmism atch +Att orney +Ġmorph ology +ĠSix ers +ut tered +ĠE LECT +gr an +Rus sell +ĠG SL +Ġfort night +Ġ. ) +Ġapost le +pr one +el ist +Unt itled +ĠIm plementation +ist ors +Ġtank er +Ġpl ush +Ġattend ants +ĠT ik +ĠGreen wich +ĠY on +ĠSP L +cell s +unt led +S olution +ĠQu é +Ġvac ated +Ġupt ick +ĠMer idian +æ ĥ +ĠDr ill +9 25 +58 4 +Ġrenov ated +ĠKub rick +zy k +Ġl ousy +pp el +ohyd rate +ĠI zzy +lesi astical +CC C +ĠAj ax +Ġad apters +ĠPetra eus +Ġaffirm ation +ĠST OR +le ms +ad oes +ĠConstantin ople +Ġp onies +Ġl ighthouse +Ġadherent s +ĠBre es +omorph ic +Fight ing +Ġpl aster +ĠP VC +ĠOb st +Ġdear ly +ĠTo oth +icks on +Ġsh aming +P lex +A gg +ĠâĢ¦ " +Ġsub reddits +Ġpige on +ĠResident ial +ĠPass ing +Ġl um +ĠP ension +Ġpessim istic +Ġ4 32 +z inski +c ade +0 75 +Ġapolog ised +iy ah +Put ting +Ġgloom y +ĠLy me +=-=-=-=- =-=-=-=- +ĠT ome +ĠPsych iatric +ĠH IT +c ms +ap olog +Ġbreak er +Ġdeep en +Ġtheor ist +ĠHigh lands +Ġb aker +Ġst aples +Ġinterf ered +ĠAb ortion +jo ined +ch u +Ġform ulate +Ġvacc inations +Ġban ter +phe us +Ġoutfield er +ĠM eter +Ġ# #### +Ġ18 95 +Ġnarrow ing +ĠST ORY +f p +ĠC ST +ign ore +Ġproclaim ing +ĠR U +ĠB ALL +yn a +65 3 +Ġpos it +P RE +59 4 +ĠRegist rar +ĠPil grim +ic io +Ġpre tt +Ġlif eless +Ġ__ _ +Ne igh +ĠCh urches +orn o +Ġor cs +Ġkind red +ĠAud it +Ġmillenn ial +ĠPers ia +g ravity +ĠDis ability +ĠD ARK +W s +od on +Ġgrand daughter +ĠBro oke +ĠA DA +ER A +Ġpick ups +ĠWil kinson +ĠSh ards +ĠN K +Ġexp el +ĠKis lyak +Ġj argon +Ġpolar ized +ian e +Pub lisher +Ġreb utt +Ġapprehens ion +ĠK essler +Ġpr ism +F UL +19 64 +ĠL oll +ä ¿ +le thal +Å Ł +Ġg hetto +Ġb oulder +ĠSlow ly +ĠOsc ars +ĠInst ruction +ĠUl tr +ĠM oe +N ich +ĠP ATH +( * +ĠRE LEASE +un ing +rou se +en eg +Ġre imb +ĠDet ected +Do S +Ġster ling +Ġaggreg ation +ĠLone ly +ĠAtt end +hig her +Ġairst rike +ks on +SE LECT +Ġdef lation +ĠHer rera +C ole +rit ch +Ġadvis able +F ax +Ġwork around +Ġp id +mort em +ers en +Ġtyp o +Ġal um +78 2 +ĠJam al +script s +Ġcapt ives +ĠPres ence +ĠLie berman +angel o +Ġalcohol ism +ass i +Ġrec ite +Ġgap ing +Ġbask ets +ĠG ou +Brow ser +ne au +Ġcorrect ive +und a +sc oring +ĠX D +Ġfil ament +Ġdeep ening +ĠStain less +Int eger +Ġbu ggy +Ġten ancy +ĠMub arak +Ġt uple +ĠD roid +ĠS itting +Ġforfe it +ĠRasm ussen +ixt ies +es i +ĠKim mel +Ġmetic ulously +Ġap opt +ĠS eller +08 8 +ec ake +hem atically +T N +Ġmind less +Ġdig s +ĠAcc ord +ons ense +em ing +br ace +Ġe Book +ĠDist ribut +ĠInvest ments +w t +] ), +beh avior +56 3 +Ġbl inding +ĠPro testers +top ia +Ġreb orn +ĠKel vin +ĠDo ver +ĠD airy +ĠOut s +Ġ[ / +Ï Ģ +b p +ĠVan ity +ĠRec ap +ĠHOU SE +ĠF ACE +Ġ4 22 +69 2 +ĠAnt ioch +cook ed +Ġcoll ide +Ġa pr +Ġsle eper +ĠJar vis +Ġalternative ly +ĠLe aves +ĠM aw +Ġantiqu ity +ĠAdin ida +Ġab user +Poké mon +Ġass orted +ĠRev ision +ĠP iano +ĠG ideon +O cean +Ġsal on +Ġbust ling +ogn itive +ĠRah man +Ġwa iter +Ġpres ets +ĠO sh +ĠG HC +oper ator +Ġrept iles +Ġ4 13 +ĠG arr +ĠCh ak +Ġhas hes +Ġfail ings +Ġfolk lore +Ġab l +ĠC ena +ĠMac Arthur +ĠCOUR T +Ġperipher y +app ers +Ġreck oned +ĠInf lu +ĠC ET +Ġ3 72 +ĠDefin itive +ass ault +4 21 +Ġreservoir s +Ġd ives +ĠCo il +DA Q +Ġvivid ly +ĠR J +ĠBel lev +Ġec lectic +ĠShow down +ĠK M +ip ed +reet ings +ĠAs uka +L iberal +ĠÏ Ħ +Ġbystand ers +ĠGood win +uk ong +S it +ĠT rem +Ġcrim inally +ĠCirc us +ch rome +88 7 +Ġnan op +ĠOb i +ĠL OW +o gh +ĠAuth ors +ob yl +Ur ban +Ġt i +ĠWe ir +t rap +ag y +Ġparent heses +Ġout numbered +Ġcounter productive +ĠTob ias +ub is +P arser +ST AR +Ġsyn aptic +ĠG ears +Ġh iber +Ġdebunk ed +Ġex alted +aw atts +H OU +Ch urch +ĠPix ie +ĠU ri +ĠForm ation +ĠPred iction +C EO +Ġthro tt +ĠBrit ann +ĠMad agascar +ë ĭ +Ġbill boards +ĠRPG s +ĠBe es +complete ly +F IL +Ġdoes nt +ĠGreen berg +re ys +Ġsl ing +Ġempt ied +ĠPix ar +ĠDh arma +l uck +ingu ished +Ġend ot +Ġbab ys +05 9 +che st +r ats +Ġr idden +Ġbeet les +Ġillum inating +Ġfict itious +ĠProv incial +Ġ7 68 +Ġshe pherd +ĠR ender +Ġ18 96 +C rew +Ġmold ed +ĠXia omi +ĠSp iral +Ġdel im +Ġorgan ising +Ġho ops +ĠBe i +z hen +Ġfuck in +Ġdec ad +Ġun biased +am my +sw ing +Ġsmugg led +Ġk ios +ĠP ERSON +ĠInquis itor +Ġsnow y +Ġscrap ing +ĠBurg ess +P tr +ag ame +R W +Ġdro id +ĠL ys +ĠCass andra +Jac ob +Ġ35 4 +Ġpast ure +Ġfr anc +ĠScot ch +ĠEnd s +ĠI GF +def inition +Ġhyster ical +ĠBrown e +77 1 +Ġmobil ization +æ ķ +iqu eness +Th or +Ġspear headed +Ġembro iled +Ġconject ure +jud icial +Ch oice +Ġpaper back +P ir +Ġrec overs +ĠSur ge +ĠSh ogun +ĠPed iatrics +ãģ ł +Ġsweep s +ĠLabor atories +ĠP acks +al us +add in +Ġhead lights +g ra +Ev idence +COL OR +Ad min +Ĭ ± +Ġconco ct +s ufficient +Ġun marked +Ġrich ness +Ġdiss ertation +Ġseason ing +Ġg ib +ĠM ages +un ctions +ĠN id +che at +ĠTM Z +c itizens +ĠCatholic ism +n b +Ġdisemb ark +ĠPROG RAM +a ques +Ty ler +Or g +ĠSl ay +ĠN ero +ĠTown send +IN TON +te le +Ġmes mer +9 01 +Ġfire ball +ev idence +aff iliated +ĠFrench man +ĠAugust a +0 21 +Ġs led +Ġre used +ĠImmun ity +Ġwrest le +assemb led +Mar ia +Ġgun shots +ĠBarb ie +Ġcannabin oids +ĠTo ast +ĠK inder +IR D +Ġre juven +Ġg ore +Ġrupt ure +Ġbre aching +ĠCart oon +Ġ4 55 +ĠPale o +6 14 +Ġspe ars +ĠAm es +ab us +Mad ison +GR OUP +Ġab orted +y ah +Ġfel on +Ġcaus ation +Ġprep aid +Ġp itted +op lan +ĠShel ley +ĠRus so +ĠP agan +Ġwill fully +ĠCan aver +und rum +ĠSal ary +ĠAr paio +read er +ĠR ational +ĠOver se +ĠCa uses +Ġ* . +Ġw ob +Ke ith +ĠCons ent +man ac +77 3 +6 23 +Ġfate ful +et imes +Ġspir ited +ĠD ys +Ġhe gemony +Ġboy cot +ĠEn rique +em outh +Ġtim elines +ĠSah ara +ĠRel ax +ĠQuin cy +ĠLess ons +ĠE QU +SE A +N K +ĠCost co +Incre ase +Ġmotiv ating +ĠCh ong +am aru +ĠDiv ide +Ġped igree +ĠTasman ia +ĠPrel ude +L as +9 40 +57 4 +Ġch au +ĠSp iegel +un ic +-- > +ĠPhil ips +ĠKaf ka +Ġuphe aval +Ġsent imental +Ġsa x +ĠAk ira +ser ial +Mat rix +Ġelect ing +Ġcomment er +ĠNeb ula +ple ts +ĠNad u +ĠAd ren +Ġen shr +ĠR AND +fin ancial +ĠCly de +uther ford +Ġsign age +Ġde line +Ġphosph ate +rovers ial +f ascist +ĠV all +ĠBeth lehem +Ġfor s +Ġeng lish +S olid +N ature +Ġv a +ĠGu ests +Ġtant al +Ġauto immune +;;;;;;;; ;;;; +ĠTot ally +ĠO v +Ġdef ences +ĠCoc onut +Ġtranqu il +Ġpl oy +Ġflav ours +ĠFl ask +ãĤ¨ ãĥ« +ĠWest on +ĠVol vo +8 70 +Ġmicro phones +ver bal +R PG +Ġi ii +; } +0 28 +Ġhead lined +Ġprim ed +Ġho ard +ĠSh ad +ĠEN TER +Ġtri angular +Ġcap it +l ik +ĠAn cients +Ġl ash +Ġconv ol +Ġcolon el +en emy +G ra +Ġpub s +ut ters +Ġassign s +ĠPen et +ĠMon strous +ĠBow en +il ver +H aunted +ĠD ing +start ed +pl in +Ġcontamin ants +ĠDO E +ff en +ĠTechn ician +R y +Ġrob bers +Ġhot line +ĠGuard iola +ĠKau fman +row er +ĠDres den +ĠAl pine +E lf +Ġf mt +ĠS ard +urs es +g pu +Un ix +Ġunequiv ocally +ĠCitizens hip +qu ad +m ire +ĠS weeney +B attery +6 15 +Ġpanc akes +Ġo ats +M aps +ĠCont rast +mbuds man +ĠE PS +Ġsub committee +Ġsour cing +Ġs izing +ĠBuff er +ĠMand atory +Ġmoder ates +ĠPattern s +ĠCh ocobo +ĠZ an +ĠSTAT ES +ĠJud ging +ĠIn her +* : +Ġb il +ĠY en +Ġexh ilar +oll ower +z ers +Ġsn ug +max imum +Ġdesp icable +ĠP ACK +ĠAn nex +Ġsarcast ic +Ġlate x +Ġt amp +ĠS ao +b ah +ĠRe verend +ĠChin atown +ĠA UT +d ocumented +ĠGA BA +ĠCan aan +ĠÙ ħ +Ġgovern s +pre v +E sc +ĠEst imates +OS P +Ġendeav our +ĠCl osing +omet ime +every one +Ġwor sen +Ġsc anners +Ġdev iations +ĠRobot ics +ĠCom pton +Ġsorce rer +Ġend ogenous +Ġem ulation +ĠPier cing +ĠA ph +ĠS ocket +Ġb ould +ĠO U +ĠBorder lands +Ġ18 63 +G ordon +ĠW TO +Ġrestrict s +Ġmosa ic +Ġmel odies +ç Ħ +T ar +Ġdis son +ĠProv ides +Ġ ...... +b ek +F IX +Ġbro om +ans hip +Do ctors +Ġner ds +ĠReg ions +na issance +Ġmet e +Ġcre pt +pl ings +Ġgirlfriend s +kn it +ig ent +ow e +Ġus hered +ĠB az +M obil +4 34 +ĠPres ents +orig in +Ġins omnia +ĠA ux +4 39 +ĠCh ili +irs ch +G AME +Ġgest ation +alg ia +rom ising +$ , +c row +ĠIn spection +at omic +Rel ations +J OHN +rom an +ĠClock work +ĠBak r +m one +M ET +Ġthirst y +Ġb c +Ġfacult ies +R um +Ġnu ance +ĠD arius +ple ting +fter s +etch up +Reg istration +ĠK E +R ah +Ġpref erential +ĠL ash +ĠH H +Val id +ĠN AV +Ġstar ve +ĠG ong +z ynski +ĠAct ress +Ġw ik +Ġun accompanied +lv l +Br ide +AD S +ĠCommand o +ĠVaugh n +Wal let +Ġho pping +ĠV ie +Ġcave ats +Ġal as +if led +ab use +66 1 +Ġib n +Ġg ul +Ġrob bing +t il +IL A +Ġmit igating +Ġapt ly +Ġty rant +Ġmid day +ĠGil more +ĠDe cker +Ġ§ § +part ial +Ex actly +Ġphen otype +Ġ[+ ] +ĠP lex +ĠI ps +vers ions +Ġe book +Ġch ic +g ross +":" "},{" +ĠSur prisingly +M organ +Ġresid ues +ĠConf ederation +in feld +Ġl yr +mod erate +Ġperpend icular +V K +Ġsynchron ized +Ġrefres hed +Ġad ore +ĠTor ment +ol ina +Ġ26 00 +Item Tracker +Ġp ies +ĠF AT +ĠR HP +0 48 +ĠRES P +ĠB J +all ows +P and +Ġunw elcome +ĠV oc +ĠBast ard +ĠO W +ĠL AR +ĠHeal er +Environment al +ĠKen yan +ĠTr ance +ĠP ats +Ġali ases +ĠGar field +Ġcampaign er +Ġadvance ments +ĠOkin awa +ĠC oh +ows ky +Ġstar ved +Ġsize able +Ġ: -) +Ġm RNA +Ġsusp ensions +ist ar +Scot land +Pr in +-------------------------------- ---------------- +Ġ50 2 +Ġteasp oons +Ġ10 50 +Ġcoerc ive +ĠMason ic +edd ed +ĠPass enger +Ġl att +Ġbr aces +ĠSt eal +ĠNY T +ĠK ats +ĠCel est +ae z +T u +ĠCoul ter +ðŁ ĺ +Fl ickr +ĠWil mington +ith s +++ ; +Ġv ending +Ġneg ro +ĠPh i +ĠYellow stone +Call back +Ġsh ampoo +ĠSh ades +w at +Ġsuper human +Ġridic uled +Ġhol iest +om bo +Ġintern s +Ġh one +ĠPar agu +UR I +Ġd angling +ãĤ » +so v +ict ional +av ailability +Ġrev ocation +Ġd ow +in ic +ĠTHE IR +Ġis o +Ġout ings +ĠLeth al +Ġ) )) +Ġinacc ur +Ġout landish +Ġan us +let ico +id on +l ol +Ġun regulated +Ġsuccumb ed +Ġc uff +ĠWast eland +let al +Ġsub str +Ġcoff ers +Ġautom akers +ov i +ĠX ue +ĠDayton a +Ġjar ring +Ġf umes +Ġdisband ed +z ik +itt on +Ġstriking ly +Ġsp ores +Ad apter +.) : +ĠLynd on +ival ry +Ġor ally +Ġtumult uous +Ġdisple asure +Ġcon es +or rect +Ġappe ase +Ġder by +ĠTrip oli +ĠAl ess +Ġp oked +ĠGu ilty +v P +En ough +Ġorig inals +6 99 +Ġrabb i +Ġproverb ial +Ġpostp one +el ope +ĠMist y +Ġstaff ed +ĠUn employment +redit ary +Ġdilig ent +re comm +me asures +as in +8 25 +Ġpond s +Ġmm ol +ĠS AR +ĠC ARE +Ġ3 71 +Ġclen ched +ĠCors air +Ġcaric ature +z n +att ach +ĠSch ro +spe ak +p ainted +ĠS uc +ĠE NT +Ġcell ul +ĠP aid +di agn +WH ERE +Ġtext ed +B arn +Ġret racted +ĠRe ferred +S av +Ġup keep +Ġwork places +ĠTok ens +Ġampl ify +cl inical +Ġmult ic +mber g +Ġconvol uted +Reg ion +5 65 +ĠTop ic +Ġsn ail +Ġsal ine +Ġins urrection +ĠPet r +f orts +B AT +ĠNav ajo +Ġrud imentary +ĠLak sh +OND ON +Me asure +Ġtransform er +ĠGodd ard +Ġcoinc ides +ir in +R ex +ĠB ok +qu it +Ġshotgun s +Ġprolet arian +Ġsc orp +ĠAd a +5 14 +Ġsl ander +record ed +Ġemb ell +ris ome +Ġapolog izing +ĠMul cair +ĠGib raltar +Cl a +Ġall ot +ĠAtt ention +Ġ4 33 +le ave +Ġwh ine +ĠIss a +ĠFa ust +ĠBar ron +hen y +Ġvictim ized +J ews +Ġnurt uring +ett el +W inged +ĠSub tle +Ġflavor ful +ĠRep s +eng ed +call back +Ġdirection al +Ġcl asp +ĠDirect ions +plan et +icult ure +Hel per +ic ion +ac ia +Ġç ¥ŀ +Ġsur ges +Ġcan oe +ĠPrem iership +be en +Ġdef ied +ĠTro oper +Ġtrip od +Ġgas p +ĠE uph +ĠAd s +vern ight +high ly +R ole +Ġent angled +ĠZe it +6 18 +ĠRust y +Ġhaven s +ĠVaugh an +HA EL +ĠSER VICE +/ , +Ġstr icken +Ġdel usions +Ġb is +ĠH af +Ġgrat ification +Ġent icing +UN CH +Ad ams +ĠOL ED +ĠBeet le +Ġ18 99 +ĠSO FTWARE +ateg or +V L +ĠTot em +ĠG ators +AT URES +Ġimped ance +Reg istered +ĠC ary +ĠAer ial +on ne +en ium +Ġd red +ĠBe g +Ġconcurrent ly +Ġsuper power +ĠX an +j ew +imes ter +ĠDick inson +âĶ ģ +F la +Ġp ree +ĠRoll ins +© ¶æ +Ġden omination +ĠL ana +5 16 +Ġinc iting +sc ribed +j uries +ĠWond ers +app roximately +Ġsusp ending +Ġmountain ous +ĠL augh +oid al +N s +Det ect +) = +ĠL uthor +ĠSchwarz enegger +ĠMull er +ĠDev i +ec ycle +J ar +6 13 +ĠL ongh +B ah +ĠSP ORTS +n w +Ġref inement +Ġwater ways +Ġd iner +Bl ade +68 3 +F ac +Ġinitial s +Ġro g +Ġparan ormal +B UT +Ġ[ ( +ĠSw anson +ĠM esh +âĸ ¬ +Impro ve +ĠRad iation +ĠEst her +ĠE sk +ĠA ly +ik y +Ġir rad +ĠBuck ingham +Ġref ill +Ġ. _ +Re pe +CON CLUS +Ġdifferent iated +Ġchi rop +ĠAt kins +Pat tern +Ġexc ise +Ġcab al +N SA +ĠST A +ĠS IL +ĠPar aly +Ġr ye +ĠHow ell +ĠCount down +ness es +alys ed +Ġres ize +ãĤ ½ +Ġbudget ary +ĠStr as +w ang +Ġap iece +Ġprecinct s +Ġpe ach +Ġsky line +Ġ35 3 +pop ular +App earances +ĠMechan ics +ĠDev Online +S ullivan +Z en +Ġp u +op olis +5 44 +Ġde form +Ġcounter act +ĠL ange +Ġ4 17 +Con sole +77 4 +Ġnodd ing +Ġpopul ism +Ġhe p +Ġcoun selling +compl iance +U FF +Ġunden iably +Ġrail ing +ĠHor owitz +ĠSim one +ĠBung ie +Ġa k +ĠTal ks +x ff +fl ake +Cr ash +Ġsweat y +Ġban quet +ĠOFF IC +Ġinvent ive +Ġastron omer +ĠStam ford +ĠSc are +ĠGRE EN +olic ited +Ġr usher +Ġcent rist +ight ing +Ġsub class +Ġdis av +Ġdef und +ĠN anto +oci ate +m ast +Ġpac if +Ġm end +e ers +imm igration +ESS ION +Ġnumber ing +Ġlaugh able +ĠEnd ed +v iation +em ark +P itt +Ġmetic ulous +ĠL F +Ġcongrat ulated +ĠBir ch +Ġsway ed +Ġsemif inals +Ġhum ankind +m atter +ĠEqu ip +opa usal +S aid +ĠLay out +Ġvo icing +Ġth ug +Ġporn ographic +I PS +Ġmo aning +Ġgriev ance +Ġconf essions +esc al +TEXT URE +Aut hent +os aurus +P urchase +Ġreleg ation +al ter +ĠÂł Âł +Ġr iddled +Ġo gre +ĠLow ell +Occ up +E at +ĠHy der +ĠAdvis er +Com merce +H unt +ĠOr th +ĠComp etitive +ĠCL A +CD C +Ġsal ads +F le +Ġindustrial ized +` , +ĠO WN +Ġbec k +ĠPart icularly +oub t +Ġm M +ĠHuss ain +ĠChen nai +Ġ9 20 +Ġappoint ing +ĠCull en +,,,, ,,,, +Ġp ores +ver ified +Ġbi ochemical +em ate +Ġcoward ly +ĠHels inki +ĠEthiop ian +S OURCE +ER C +est ro +Ġbi otech +ĠS our +Ġbrew er +Bloom berg +Ġintens ify +Gl ass +an co +ĠF DR +gre SQL +ĠF ires +©¶æ ¥µ +ec o +100 1 +ĠHom eless +Ġinstant aneous +ĠH aste +ig el +D iamond +Ġp aving +Ġland fill +Ġd ads +h oun +: ] +Ġinc endiary +ĠLiving ston +ĠHil bert +ĠChe cks +st yles +in ators +ĠCl ive +ph rine +Ġchimpan zees +Ġp all +ĠJ M +ĠAad haar +ð Ŀ +Ġachie vable +dis abled +P ET +OOOO OOOO +M ot +Ġint angible +Ġbal let +ĠWe bs +ĠEst imated +Effect s +Ġb ailed +Josh ua +Ġturb ulence +Ġoccup ant +ĠDay light +Ġ36 1 +me et +Ġstat ically +Ġon look +Ġk i +il legal +Ġvel vet +Ġdehyd ration +Ġacqu ies +ĠRe z +ak ura +ĠU pton +at ro +Ġincomp rehensible +Ġback door +ĠRh ino +7 27 +Ġmath s +) + +Ġhe resy +Ġd f +ĠRoc he +ĠL ydia +Ġpanc reat +re ply +arre ll +Ġsolicit ation +Ġcirc adian +BI P +Ġfor ay +Ġcrypt ic +iz u +ime o +ĠTom ato +ĠH oms +ex amination +Ġqu arry +ĠVal iant +ĠJer icho +ĠIN CLUD +Ġ18 40 +5 19 +Ġres ists +Ġsnap shots +ĠSp ur +ĠAnt iqu +Log in +Ġbest selling +Ġant ic +ĠS utherland +ãĤ¢ ãĥ« +Ġ~ / +ĠP arm +è ĥ +P ages +int ensity +Ġimm obil +Ġ18 65 +zz o +Ġn ifty +Ġf entanyl +ĠPres ervation +op hen +Ġd arts +ĠD inosaur +po inters +ĠR ite +s uggest +aware ness +ĠSher idan +Ġst ances +Ġsor cery +Ġper jury +ĠNik ola +ie ver +Ġf iance +ĠJordan ian +ĠBall oon +Ġn ab +Ġk b +Ġhuman ities +ĠTan aka +hill ary +Ġconsult ancy +ĠZ ub +Ġrem ission +Ġconf id +CH Q +ĠF ug +Ġimpro vis +Y ep +/ _ +Ġunwilling ness +Ġport folios +05 5 +ĠInstruct or +aim an +Ġclaim ants +M bps +ĠBy e +re ceived +T weet +Ġind emn +ri z +am ara +N at +Ġeval uates +ĠL ur +ep ad +FO X +ĠTh ro +Ġrust y +Ġbed rock +ĠOp rah +J B +Ġmanip ulative +Ġwill ful +Ġrel apse +Ġext ant +The me +S ensor +ĠSt ability +go vern +Ġpo ppy +Ġkn ack +Ġins ulated +ĠT ile +ĠExt rem +Ġunt old +Ġconver ge +Ġref uel +ig roup +Ġdistort ions +Ġrav aged +Ġmechan ically +ĠRe illy +ĠN ose +ĠIncarn ation +ĠBeck y +abb ling +Ġt aco +Ġr ake +Ġmelanch oly +Ġillust rious +ĠDart mouth +Gu ide +ĠR azer +ĠBen z +Ult imate +ĠSur prise +Ġpage ant +off er +Who ever +Ġw iser +Ġchem ist +ĠHE LL +ĠBul k +Ġpl utonium +ĠCO VER +Ö ¼ +f ailed +Ġtire lessly +Ġinf ertility +ĠTr ident +ĠShow time +ĠC iv +V ice +requ ires +itt ance +Ġun controlled +interest ing +56 1 +Ġinnov ate +ateg ic +L ie +ĠS elling +U l +Ġsav ior +ĠT osh +Ġsw ast +P ASS +Ġr ink +Ġcard io +ĠI ro +ud i +Ġv antage +Ġv ans +ĠNi ño ++ = +Ġpropag ate +< ? +Ġmethod ological +204 39 +Ġtrig lycer +Ġing rained +ĠAn notations +arr anted +6 17 +ĠS odium +ĠA AC +techn ical +mult ipl +Ġ3 73 +å ĭ +Ġdec isively +Ġboost ers +Ġdessert s +ĠGren ade +Ġtest ifying +ĠSc ully +ID s +Ġlock down +ĠSc her +ĠR é +ĠWhit man +ĠRams ay +rem ote +Ġh ikers +ĠHy undai +Ġcons cientious +Ġcler ics +ĠSiber ian +ut i +is bury +Ġrel ayed +Ġqu artz +ĠC BI +seek ers +ull a +Ġweld ing +ĠSh al +ble acher +T ai +ĠSam son +Ġt umble +ĠInvest or +Ġsub contract +ĠShin ra +ow icz +j andro +d ad +Ġtermin ating +ĠNe ural +ä» £ +Ġleak age +ĠMid lands +ĠCaucas us +í ķ +c it +ll an +iv ably +ĠAlb ion +Ġ4 57 +Ġregist rations +Ġcomr ade +Ġclip board +0 47 +Ġdiscour aging +ĠO ops +Ad apt +Ġem path +n v +ĠPR OT +ĠDon n +ĠP ax +ĠB ayer +t is +Squ are +Ġfoot prints +part icip +ĠChile an +B rend +ind ucing +M agn +Ġclub house +ĠMagn um +Ġenc amp +ĠEth nic +uch a +ere y +Ġw atered +ĠCal ais +Ġcomplex ion +Ġsect s +Ġren ters +Ġbr as +oÄŁ an +Time out +Man agement +Ġinf ographic +P okemon +Cl ar +Ġloc ality +Ġfl ora +as el +P ont +Ġpop ulate +ĠO ng +Ġsubs istence +Ġa uctions +ĠMcA uliffe +ĠL OOK +br inger +Ġtit an +Ġmanif old +ĠâĹ ı +Ġcalibr ated +Ġcal iphate +ĠSH E +ĠCommission ers +ce ivable +j c +W inner +5 24 +Ġcond one +Other wise +Ġp iling +Ġem body +ĠCrime an +ut ics +ĠEx hibition +Ġ4 26 +e ering +Ġv ying +ĠH UGE +* =- +Ġprin cipled +à ¦ +Ġquir ks +ĠEdit ors +put ing +G ES +ĠF TA +ठ¾ +add on +ĠH AM +ĠFrie za +W oman +. $ +Ġc rib +ĠHer od +Ġtim ers +ĠSp aces +ĠMac intosh +at aka +Ġgl ide +Ġsmell ing +ĠB AL +Ġun su +Ġcond os +Ġbicy cl +ĠRev ival +55 3 +Ġjugg ling +H ug +ĠKardash ian +ĠBalk ans +mult iple +Ġnutrit ious +oc ry +19 00 +Ġinteg rates +Ġad joining +ĠF older +roll ment +ven ient +Ġu ber +y i +Ġwh iff +ĠJu ven +ĠB orough +net te +Ġb ilingual +ĠSp arks +ph thal +man ufact +Ġt outing +ĠPH I +Ke efe +Rew ard +Ġinf all +ĠTem per +typ ically +ĠNik ol +Ġregular s +Ġpseud onym +Ġexhib itions +Ġbl aster +Ġ40 9 +w arming +Ġrever ber +Ġrecip rocal +Ġ6 70 +ip ient +b ett +ĠBe gins +Ġit ching +ĠPh ar +Ass uming +Ġem itting +ĠML G +Ġbirth place +Ġt aunt +ĠL uffy +ĠAm it +Ġcir cled +ĠN ost +enn ett +Ġde forestation +ĠHist orically +ĠEvery day +Ġovert ake +79 2 +Ġn un +ĠLuc ia +Ġaccompan ies +ĠSe eking +ĠTr ash +an ism +R ogue +Ġnorth western +ĠSupplement al +ĠNY U +ĠF RI +ĠSat isf +x es +5 17 +Ġreass ured +Ġspor adic +Ġ7 01 +Ġmed ial +Ġcannabin oid +Ġbarbar ic +Ġep is +ĠExplos ive +ĠD ough +Ġuns olved +Support ed +Ġacknowled gment +sp awn +Ġkit chens +Ġ- = +talk ing +ic ist +ĠPeg asus +ĠPS U +Ġphot on +ĠAuthent ication +R G +@# & +76 2 +ĠCl air +Ġdi aper +Ġbr ist +ĠProsecut ors +ĠJ em +6 28 +ĠEvery where +ĠJean ne +equ ality +ãĥ© ãĥ³ +object s +ĠPel icans +Ġ39 2 +Ġbl u +b ys +ĠA go +Ġinstruction al +Ġdiscrim inating +ĠTR AN +ĠCorn el +ag os +Ġty re +Ġas piration +ĠBrid gewater +": - +! ". +ĠEn s +ĠCoc o +P ie +Ġdet ach +ĠC ouch +Ġphys ique +ĠOccup ations +osc opic +en ough +B uzz +App earance +Y P +Ġrac er +Ġcompl icity +r pm +T oy +Ġinterrupt s +ĠCat alyst +Ġut ilitarian +imp act +Ġsp aghetti +Ġp orous +Ġeste emed +Ġinc iner +ĠI OC +7 48 +Ġesp resso +ĠSm ile +abil ia +6 35 +Ġmathematic ian +Ġ4 24 +ĠK L +ĠH IP +Ġover heard +ĠT ud +ĠT ec +Ġqu izz +Ġfl attering +Ġcon n +âĢ İ +Ġatt aches +ĠR OS +ĠAC S +Ġt cp +ĠSh ame +sk ip +res pected +ĠTrin idad +gr ain +Ġfooth old +ĠUnch arted +ĠJul io +z l +av ored +ĠAn xiety +er rors +ĠCent auri +its ch +D addy +Ġclutch ing +ĠIm plement +ĠGut ierrez +Ġ7 60 +Ġtele portation +end ra +Ġrevers ible +st ros +Ad venture +08 3 +Ġliber ating +Ġas phalt +ĠSp end +AR DS +im sy +PR ES +ĠEmer ging +Ġwild fires +Ġtechn ologically +Ġem its +ĠART ICLE +Ġirregular ities +Ġcher ish +çī Ī +Ġst ink +ĠR ost +Econom ic +Ġcough ing +ĠMcC ann +pro perties +ilant ro +Ġreneg oti +Trans lation +Ġin quest +ĠGra pe +oot ers +gu i +ĠSwords man +ace ae +h itting +Ġr c +Ġexert ed +ĠS AP +it ent +Ġperil ous +Ġobsc urity +Ġassass inate +Ġab original +Ġresc uing +ĠSh attered +lock ing +all ion +Ch anging +ĠHar rington +ĠB ord +ĠAfgh ans +Jam ie +aret z +ĠAugust us +Ġ38 6 +8 30 +Ġj og +ok ingly +Tr igger +ĠH OR +Stat istics +Ġviewers hip +Ġadd itives +h ur +Ġmaxim izing +ĠR ove +ĠLou ie +ĠBuck et +ĠCHR IST +ou sel +Ġstre aks +ir ted +Ġt ert +Ġcolonial ism +Ġbur ying +y k +Cond ition +ĠDPR K +By Id +75 1 +âĹ ¼ +Ġwor risome +Ġvoc ational +sl ice +Ġsa ils +ĠCorrection al +95 4 +Ġt ul +K id +l uster +Ġfam ilial +ĠSp it +ĠEp iscopal +Specific ally +ĠVol cano +run s +q s +Ġve tted +Ġcram med +t rop +here r +Thank fully +Ġper cussion +Ġor anges +Ġround up +Ġ4 99 +x ious +Char acters +ĠZion ism +ĠR ao +ÃĽ ÃĽ +W F +Ġunintention al +ONE Y +Gr ab +Com mercial +Ġglut amate +ĠMcK enna +ru ciating +ning ton +ih u +Ch an +ĠSw ap +Ġleaf lets +Ġfunction ally +er ous +F arm +Ġcal oric +ĠLiter ally +con cert +Ġshe nan +Ġrep aid +ey es +Ġbas hing +ĠG orge +Ġcollabor ations +Ġun account +itch ie +Ġteam work +pp elin +Ġpip ing +Ġmin ced +Ġd iam +ri eg +Ġmasc ara +Ġsuck er +ĠMo ons +App s +ĠPe ck +Ġper v +ĠFl oat +o ley +ĠN ish +im ize +Ġarom atic +u in +end ish +! / +ĠB icycle +ĠAS IC +ile ged +ĠQuad ro +ios yn +Ġlock out +ĠW ink +SP EC +Attempt s +Ġseed ed +red o +ias is +Ġsn ag +ãĥķ ãĤ© +ãĤ ¶ +Ġground ing +Ġrelie ver +Ġfrivol ous +ĠG ifts +ĠF aces +Es pecially +Ġmicrobi ome +im ag +ĠSch l +ĠP les +ĠBle ach +ĠIr win +ĠE aton +ĠDisc iple +Ġmultipl ication +Ġcoer ced +Ġ4 19 +st h +E vil +B omb +Ġex orc +Ġstag gered +L ESS +Ġinert ia +ĠED IT +Ġgo b +Tr aditional +Ġclass y +Lear y +ĠP AGE +yr s +Ġtrans porter +Ġmat ured +Ġhij ab +Ġbi ome +Where as +Ġex termination +ĠT ues +ĠT akeru +ĠAud rey +er ial +ĠAd en +aff les +Ġnarciss istic +ĠB aird +UT F +I re +ĠCon nie +Ch amp +Ġwhis pering +ĠH att +D K +Ġdis infect +Ġdeduct ed +Ġpart ake +Ġdown grade +ĠEs ports +ĠContin uing +Ġdemocr atically +icro bial +itt a +Ġlim estone +Ġexempt ed +ĠFren zy +H erm +7 28 +Ġfled gling +Met a +765 61 +69 3 +% : +w ake +5 26 +ĠDis cipline +Ġvirgin ity +ĠLeg ions +ĠFrank ie +int ent +Ġrest rooms +ĠRou ter +da q +Ġobjection able +âĨ ij +w ark +ĠRah ul +g ain +activ ation +abs olute +ĠAccess ed +Ġ24 00 +ogg les +Ġsecond ly +ĠDEF ENSE +Ġpost age +wra pper +sh arp +7 29 +Ġcommun icates +Ġadd on +ĠMil itia +H ong +Ġsl umped +ĠJP EG +ĠI car +ad ish +68 1 +Ġmaj esty +ĠWolf gang +ĠEl astic +u per +Ġv iz +Ġunconscious ly +ĠST D +ĠS ass +Ġflower ing +ĠHel ic +ĠDra per +ĠAm ateur +Ġman ure +Ġdis ingen +ĠLe i +br ing +9 49 +Ġinhib ited +Ġhead quartered +Ġen igmatic +�� � +Ġred ress +R H +Ġratt led +Ġd iction +l io +ĠT BA +ĠSN AP +C alling +Ġfasc ists +ĠD ove +iew icz +0 36 +Ġco asts +ĠR ect +Ġ) ] +L ot +6 29 +ĠS EM +ĠPeters en +ĠExpl ain +ĠBo ards +ĠBe zos +ĠJ ournals +Ġ20 24 +p arser +Ġmist rust +Ġgr ate +ĠL ocked +bo a +S aint +g aming +Ġvow el +in ately +bl ow +All ah +Ġun matched +Ġb ordering +ĠExp end +n r +Or acle +rou ch +Ġcont iguous +ac us +Ġdist raught +58 1 +Ġanat omical +O X +ap ixel +8 33 +ĠPL US +Ġres usc +Ġab iding +57 3 +Ġvac ancies +Em ily +Ġhyp othal +ĠWer ner +ĠWe e +ĠDJ s +5 13 +Ġwitch craft +Ġac upuncture +ent ary +benef it +Product s +ĠP SP +ĠMP G +ĠJ inn +ĠJ arrett +Ġ4 45 +ĠIm aging +ĠP yth +Fin ish +Ġte x +Ġjuven iles +Ġhero ism +Ġdoubt less +ĠA ki +ĠT end +ĠPatri arch +Ġbit ters +ĠTele communications +it atively +ag na +Ġr g +ĠS OLD +Ġcomp ulsion +ĠN asa +ĠKath ryn +Ġmillion aires +Ġintrins ically +Ġbolst ered +time out +fl o +Ġtut or +p our +Stat ement +Ġ{ * +ĠRud olph +ĠKimber ly +rog ens +adi q +] + +Ġindign ation +Ġfract uring +ĠRe leases +ĠGr ain +pro tein +L ago +Ġvac ations +Ġboot ed +ĠTH REE +ĠH G +oresc ence +Ġt f +Ġso ar +iosyn cr +Ġgl ances +ĠSp oon +ĠJ ury +ĠCow boy +Ġcreat ively +Hig her +Ġsolic itor +Ġhaw k +ac io +89 6 +Ġsuperf lu +Ġbombs hell +ct ure +Ġbroker age +Ġraid ing +Ġf rench +Ġang led +Trans action +ĠGen ocide +u pe +ĠHait ian +57 2 +! : +Ġunwitting ly +iter ator +sc roll +Ġtall ied +Ġbi omedical +ĠC ARD +Ġe uphem +Ġbrain storm +a quin +K o +Mic helle +ĠR unes +ĠBall istic +ud ers +Ġmod esty +ĠiP ads +ĠEzek iel +Y E +Ġstars hip +Ġpower fully +Ġper l +ĠSh ade +ĠQu art +ĠE EG +Ġfisher man +OS ED +ĠTyp ical +df x +Ġmes hes +Ġet ched +worth iness +Ġtopp led +Ġ3 96 +or ius +We iss +Ġmy sql +ĠVal halla +Ù Ĵ +le asing +Ġrec omp +rap nel +S el +04 3 +Ġder ailed +ĠGu ides +IR T +Ġde human +ĠBritt any +" )) +Ġex claim +Ġb alk +Ġ8 40 +CLA IM +int el +L AB +Ġpe gged +Ġast roph +sm oking +Ġrig ging +Ġfix ation +Ġcat apult +ins ide +ĠC ascade +ĠBolshe vik +G aza +Dep th +Ġloud spe +Ġalmond s +me yer +l eness +j en +f resh +Ġunbeat en +ĠSqu id +ĠPres umably +Tim er +B W +Ġro sters +Ġell ipt +ĠHar riet +dat abase +ĠMut ual +ĠComm odore +uk ed +kn ife +ĠCOMM UN +h ya +Ġmel ts +arch ives +Ġrat ification +Ġmultip lying +Ġinter oper +Ġasc ert +w ings +ver ting +ĠScorp ion +ay e +ĠPorts mouth +ĠM TA +n it +iaz ep +Ġqu arantine +Ġslides how +Ġcent imeters +Ġsyn opsis +Ġsp ate +th irst +Ġnom inating +ĠMel vin +Pre view +Ġthro b +Ġgener ational +ĠRad ius +rest ling +put able +aw ar +N ECT +Ġunlaw fully +ĠRevel ations +Wik ipedia +sur v +Ġeye ing +ij n +ĠF W +Ġbr unt +Ġinter stellar +Ġcl itor +ĠCroat ian +ĠCh ic +ev a +ĠDis app +ĠA kin +iner ies +d ust +Interest ed +Ġgen esis +ĠE ucl +ö n +p icking +Ġmut ated +Ġdisappro ve +ĠHD L +Ġ6 25 +Ì ¶ +c ancer +Ġsqu ats +Ġle vers +Disc uss += ] +D ex +ĠVIDE OS +A UD +Ġtrans act +ĠKin ect +ĠK uala +ĠC yp +7 47 +Ġsh attering +Ġarsen ic +ĠInt ake +ĠAngel o +ĠQu it +ĠK he +Ġ18 93 +M aker +0 29 +ĠPain ting +Dis able +9 16 +Ġanal ges +Ġtact ile +Ġprop hes +Ġd iced +ĠTravel s +ĠHe ader +ĠClub s +Ass istant +Ġinc rim +Ġd ips +Ġcruc ifix +ĠShan ahan +ĠInter pret +Ġ40 90 +al ogy +abb a +Ġsimul ac +hus band +S IM +Ġrecy cle +uc er +ed ged +Ġre naissance +ĠBomb ay +Cath olic +ĠL INE +ĠCl othing +re ports +Ġpl aus +Ġd ag +ĠM ace +Z I +Ġintr uder +ĠVeter inary +g ru +Ġsne aky +ĠS ie +ĠC innamon +P OSE +Ġcou rier +ĠC NS +Ġemanc ipation +s it +Ġplay through +ĠFac ilities +v irt +ĠG auntlet +Thom pson +Ġunbeliev ably +Param eters +Ġst itching +ign e +ĠTH ESE +Priv acy +Ġshenan igans +Ġvit ri +ĠVal id +59 1 +Ń · +ĠProt otype +ink a +SC P +ĠT id +è Ī +old ed +Ġindividual ity +Ġbark ing +Ġm ars +ĠW D +Ġ8 20 +Ġt ir +Ġsl apping +Ġdisgr untled +ĠAng ola +ri us +ĠTorn ado +ĠTh urs +Ġcapt cha +Ġang st +ĠP og +ĠAssass ins +ĠAd idas +Ġjoy ful +Ġwh ining +Emer gency +Ġphosph orus +Ġatt rition +oph on +ĠTimber wolves +ĠJ ah +ĠBr inging +ĠW ad +ĠEn sure +oh l +ĠX ie +omm el +c mp +Ġz ipper +Ġrel at +ĠCor ridor +m ilo +T ING +Av g +Ġcro pped +] } +Ġr aged +ĠLump ur +ĠGuer rero +our ke +N ut +Ġoff sets +og lu +dr m +Ġmort als +lat able +Ġdismiss ive +ä¸ ī +Ġthro ats +Ġchips et +ĠSpot light +Catal og +art ist +G b +Ġch illy +Ġst oked +Ġ3 74 +W ard +L atin +Ġf iasco +Ġble ach +Ġb rav +Enh anced +Ġin oc +ĠFior ina +_ > +Ġle ukemia +Ġel uc +Ġannoun cer +ĠLith uan +ĠArm ageddon +å ĩ +Len in +ĠR uk +Ġpe pp +ĠRom antic +ĠP IT +ĠInter stellar +ĠAt kinson +R aid +J s +Go al +C ourse +Ġvan ishing +es ley +ĠR ounds +Els a +59 3 +Ġredund ancy +ĠST AND +Ġprop hetic +Ġhabit able +ry u +Ġfaint ly +M ODE +Ġfl anked +IR C +Aw esome +Ġsp urious +ĠZ ah +ĠMS G +Ġsh ading +Ġmotiv ational +ĠSant ana +ĠS PR +Ġexc ruciating +om ial +ĠM iko +ĠLe opard +A byss +Ġ[ | +d irty +Ġbath s +Ġdem oral +and re +P B +Ġun ification +Ġsac rament +Ġ[ & +Ġpric eless +Ġgel atin +Ġeman ating +ĠAll aah +98 6 +Ġout burst +Ġer as +ĠX VI +ĠSP I +O tt +ĠLaz arus +PL IED +F lying +blog s +W isconsin +R aven +Ġreb ate +Ġcreep s +ĠSp an +ĠPain ter +ĠKir a +ĠAm os +ĠCor vette +Cons umer +ĠRec over +ck i +Ġpes ky +ĠIn vention +Compan ies +Ġchalleng ers +ad emic +ĠUkrain ians +ĠNeuro log +ĠFors aken +Ġent rants +Ġemb attled +Ġdef unct +ĠGlac ier +Ġpo isons +ĠH orses +m akes +ĠD irt +Ġ4 23 +hh h +ĠTrans formation +QUI RE +................ .. +Ġtrave ller +ĠSe xy +ĠK ern +ip olar +Ġransom ware +oooooooo oooooooo +E c +rub y +Prof essional +ĠOut break +arg ument +G rey +ĠFif a +ĠCH O +ĠFOR M +ĠAm trak +- [ +Ġcr adle +Ġantioxid ants +ãģ®å ® +7 36 +ĠNAS L +ĠContribut ions +Ind iana +ĠST EP +C SS +Ġsal ient +Ġall ocations +yr ights +Ġm ashed +ĠCut ter +Sex ual +Ġp ounded +Ġfan base +Ġc asc +ĠTrans parency +Ġanaly tic +ĠSummon er +× ŀ +ĠAD C +det ail +Ġvan quished +Ġcr abs +ar ie +Dest roy +ĠS ack +Ġtrans istor +Al abama +ĠK oen +ĠFisher ies +c one +Ġannex ed +ĠM GM +es a +Ġf aked +ĠCong ratulations +Ġhind ered +Ġcorrection al +ĠI TV +lee ve +Ġin appropriately +lic ks +Ġtresp ass +Ġp aws +Ġnegoti ator +ĠChrist ensen +lim its +ĠDian ne +Ġeleg ance +ĠContract s +an ke +Ob j +Ġvigil ance +Ġcast les +ĠN AD +ĠHol o +Ġemph atically +ĠTit us +ĠServ ing +ĠRich ie +ĠP igs +5 68 +Ġanim osity +ĠAtt ributes +ĠU riel +M Q +my ra +ĠApplic ant +Ġpsychiat rists +ĠV ij +ĠAb by +ag ree +P ush +Ġk Wh +hib a +Ġinc ite +ĠWe asley +ĠTax i +minist ic +hy per +ĠF arn +Ġ6 01 +ĠNation wide +F ake +95 2 +Ġma ize +Ġinteract ed +Ġtransition ed +Ġparas itic +Ġharm onic +Ġdec aying +Ġbas eless +ns ics +Ġtrans pired +Ġabund antly +ĠFore nsic +Ġtread mill +ĠJ av +ab and +Ġssh d +Ġfront man +ĠJak arta +oll er +dro ps +ĠSERV ICES +rompt u +oph ical +h ospital +bled on +6 45 +Ġmid range +ĠEV ENT +cul ated +raw led +Ġper ched +Ġover board +ĠPe el +ĠP wr +ĠCar th +ĠCOM PLE +co e +sh all +Ġdeter rence +M ETHOD +ĠAbs ent +M EN +Ġs ill +ĠLE VEL +Y ork +Ġsin ners +ĠOP EC +ĠN ur +ĠDesign s +se lection +Ġunw orthy +CH A +Ġstreng thens +88 3 +ed ly +Ġslic ing +Ġmal nutrition +Ġfilm making +ĠPol k +ur ated +Ġ4 21 +bre akers +!' " +Ġwet lands +ĠDisc rimination +Ġallow able +Ġste ered +ĠSic ily +S AM +Ġmust ache +Ġm ids +Ġcl ipped +Ġcirc ulate +Ġbr ittle +ĠBuild ings +ra ised +ĠRound up +Ġwealth ier +Ġoverw rite +Ġover powered +ĠGerr ard +s ites +PD ATED +Ġacute ly +ĠGam ble +Ġp im +ĠK us +Typ ically +De ploy +ĠMoroc can +p otion +com be +Ġvigil ante +Ġ36 3 +St ew +ĠB agg +Ġres ided +ĠSp o +Ġrem nant +Ġempt iness +br ainer +Ġout patient +pri ority +Ġle ptin +ĠPay ton +ĠGle aming +ĠS hed +ĠPol o +ĠMormon ism +rest ricted +arl ane +w x +Ġcreat ine +ĠAn on +ĠST UD +ĠJ UL +ĠT ee +5 28 +08 9 +Ġhat ched +Dis patch +ĠCompos ite +Ġ45 1 +p uff +ĠX COM +ĠOr n +ĠTH ANK +END ED +ĠAshe ville +Ġà ľ +Ġman go +ĠS lightly +world ly +ĠW ander +ĠExp and +ĠCh r +M ist +Ġorthodox y +ĠUN ESCO +reg ate +Else where +k ie +ir led +Ġtopp le +Ġadopt ive +ĠLeg s +d ress +ĠS agan +b are +ĠGl ou +Cr unch +Ġhelp ers +Ġchron ically +ĠH uma +1 0000 +Ġaccommod ating +äº Ķ +Ġwrink les +Ġdod ged +four th +Ġpre con +Ġcompress or +ĠK are +Ġev ict +ĠWar wick +im ar +Ġmodern ization +Ġband wagon +Ġref uted +Ġnet ted +ĠNa ples +ĠGen ie +per ors +Ġfield ed +Ġde re +ĠPar ables +le es +Ġtr out +asp ers +Ġn ihil +Ġhapp iest +Ġflo ppy +ĠLo ft +ĠHe ard +Ġun ison +Ġl ug +ĠRed mond +class ic +Supp orters +SH IP +G MT +Ġfue lled +ç IJ +Ġd d +ĠEmin em +Ġ18 97 +NY SE +Ġsecret aries +ĠF IA +ĠCanaver al +F avorite +Ġp omp +Ġdetain ee +ers hip +aim on +i our +ĠA pex +Ġplant ations +am ia +ac ion +R ust +Ġtow ed +ĠTru ly +5 77 +Ġshel tered +r ider +W o +Ġl air +ĠInt elligent +impro ve +m atically +Ġet iquette +ad ra +all o +ĠJun o +any thing +ĠStru ggle +ĠPred ict +ĠGr imes +ĠAMER ICA +ct x +ĠSit uation +W OOD +Ġsol uble +me ier +Ġintoler able +ang ering +Ġun interrupted +Ġtool tip +Ġinterrog ated +Ġgun ned +ĠSne ak +æŃ ¦ +Ġt ether +Ġcr umble +L ens +Ġclust ered +ĠSy l +ĠHas an +Ġdystop ian +w ana +Ġjoy stick +ĠTh ib +amm u +Tom orrow +5 46 +Ġoverc ame +Ġminim ized +cept or +Run ner +ENG TH +ĠBrend a +ĠAchieve ments +Ġtor ches +Ġrapp ort +ĠInvestig ator +ĠHand ling +rel ation +g rey +8 15 +Ġk cal +ĠComm ands +d q +Ġcur ls +Ġbe arer +Ġcyn icism +it ri +ĠUse ful +B ee +D CS +Ġab ras +P ract +BIL ITIES +7 12 +Ġdebug ger +Ġdebt or +ĠL ia +ĠK ers +Ġexacerb ate +ĠSt acy +ĠB land +ĠSc enes +Ġbranch ing +âĸĪâĸĪâĸĪâĸĪ âĸĪâĸĪâĸĪâĸĪ +ape ake +Ġs alsa +Ġmish and +ĠKon ami +ĠN ib +Ġanecd ote +Ġagree able +Ï ī +ĠNath aniel +ĠHe isman +ĠB eware +Ġ18 86 +spect ive +69 1 +5 22 +Ġinhib its +Ġhas hing +Ġ18 89 +å° Ĩ +v ich +P ure +Ġsolid ly +Ġaspir in +im aru +Ġstreet car +ĠU CS +ĠJ udd +Ġflash backs +p ins +Ġ14 40 +ĠUN HCR +ĠSym ptoms +T IT +5 38 +F ra +% ); +Ġo oz +Ġcur few +Ġcal med +Ġparticip ates +Te X +Ġnons ensical +Ġfull back +ĠDe L +mon key +h ari +Ġmetabol ites +Ġloot ed +ĠAL WAYS +ĠB CC +L t +oc het +B one +Ġveto ed +Ġg cc +ĠCL ICK +Ġ18 88 +s af +Ġstiff ness +Ġlow ly +ĠGe h +vers on +ors et +Ġun foreseen +Ġan esthesia +ĠOpt ical +Ġrecon structed +ĠT up +sh ows +NEW S +ĠNewsp aper +ĠA SA +ter a +N umbers +Ġinexpl icable +× ij +Ġhard ness +unt arily +ĠA cer +grad ient +ARD IS +Ġwood land +Ġmetaph ors +ĠWem bley +ĠPa vel +phil is +Ġre writing +Ġpercept ual +Ġ10 70 +worm s +ĠDown s +Ġunsur prisingly +Ġtag ging +fl ame +Ġlit res +Ġboun ces +ĠB abe +sh ut +Ġoverd oses +ĠShe ila +ĠCh au +ĠBl ess +Capt ure +ĠSign ificant +ĠSc ion +Ġ38 9 +ĠMc H +ĠTitan ium +ĠMe al +amed a +ag ents +agg ressive +B illy +76 3 +ĠS aying +DER R +it one +Coll ins +B ound +Ġbol ted +ĠDM CA +95 3 +Ġun iqueness +Ġep igen +un ci +ant am +Ġreck oning +ch airs +OG R +ĠSen egal +Ġ18 62 +re levant +Ġ ¯ +Ġpharm acies +ĠG eral +v ier +Y an +OR PG +Ġrab id +b ending +ĠUN ITED +Ġ4 65 +As sembly +Ġwe ep +Ġbe hest +ĠMother s +ĠJ ace +h id +Ġwh irlwind +ĠUN IVERS +Ġut opian +Ġkidn ap +Ph ilipp +K in +89 3 +Ġlivest ream +ĠM ISS +Ġsub versive +ĠTechn iques +ĠJUST ICE +ĠB ASE +Ġ38 7 +Ġassail ants +ĠHard core +Ġsprink led +ĠP se +é ļ +print ed +ĠH au +OR GE +ĠT OUR +Ġl aced +Ġit ch +G iving +Ġport ed +78 1 +//////////////// //////////////// +bre eding +Ġlog ger +ĠH OL +inn ie +First ly +Ġembry onic +Ġdeleg ated +p ai +O IL +Ġcentr ally +ĠR x +ĠSc outing +D utch +Ġhe reditary +ĠCru iser +s at +5 29 +ĠMar riott +other mal +Ġprohib itions +E arn +ĠSt ab +ĠColleg es +ĠBel ief +st retched +ĠL H +ĠEntity Item +C IA +Ġun rem +Ġlaure ate +Ġdenomin ations +sum mary +h ler +S pect +ĠK laus +ĠBe ans +Ġins ur +ĠPA X +Ġfield er +ĠV et +ĠSp arrow +z ie +ĠS Q +ĠMond ays +ĠOff line +ĠLer ner +ĠExt ensions +Ire land +Ġpatron age +Ġcontrast ed +ĠMan ia +h irt +Mos cow +Ġcondem ns +ĠAn ge +Ġcomp osing +ĠPe pe +ĠP addock +Ġheter ogeneity +Ġide ologically +Ġf ishes +Ġcur sing +ĠR utherford +ĠFlo ating +ĠAm elia +Te a +Syn opsis +Ġstun ts +Ġbe ad +Ġstock ing +ĠM ILL +ob ook +mass ive +\ < +Ġh ump +ĠPref erences +Engine Debug +ge ist +ĠNiet o +ome ver +ish y +eval uate +col onial +Altern ative +ĠGo Pro +ĠV ortex +ĠNET WORK +ans ky +Sec ure +ĠTh rust +Sn ake +Ġparcel s +Ġsam urai +Ġactress es +N ap +M F +ifer ation +Be er +5 23 +ĠI ly +oint ment +P ing +Ġstri ped +ĠMell on +oss ession +Ġneut ron +end ium +Ġa ph +ĠFlav oring +Ġ38 3 +Ġrespons iveness +ĠJ indal +ĠHitch cock +Den ver +ĠDRAG ON +sm anship +ĠDu pl +Ġs ly +Ġweb cam +ĠTw ain +ĠDar ling +ili ate +cons umer +D IT +Ġnames ake +Ġun orthodox +Ġfun er +ĠPL oS +ĠCONTR OL +ozy g +ogl obin +F ACE +ER G +ĠD ia +ĠF iesta +ce le +0 34 +Ġencl ave +âĸ¬ âĸ¬ +on ement +al ist +M and +Ġhome grown +ĠF ancy +Ġconcept ions +ĠCont ains +ure en +Ġreiter ate +Ġme ager +Ġinstall ments +Sp awn +6 27 +Ġphot oc +ĠCab rera +ĠRos enthal +ĠLans ing +is ner +Ġinvest s +ĠUFO s +EX P +Hard ware +Ġtr agically +Ġconced es +ie ft +ch am +bor gh +ĠSch r +ĠMel anie +ĠH oy +Ġvisit ation +Ġid iosyncr +Ġfract ions +Ġfore skin +ob os +Ġpo aching +ĠVI EW +Ġstimul ates +ĠG ork +can on +M IC +ĠNem esis +ĠInd ra +ĠDM V +Ġ5 29 +Ġinspect ing +Ġgrand ma +ĠW hedon +ĠSh ant +ĠP urg +ik an +ĠT eg +ĠCL R +z ac +Vict oria +ĠVer ify +ion ics +Ġpart ying +ĠM ou +col our +Ġtestim onies +l ations +Ġpress uring +hi ro +ac ers +Ġf id +ang ler +ĠCS I +Ġhere after +Ġdiss idents +report ing +iph any +che v +Ġsol itude +Ġl obe +Ġind is +Ġcred ential +re cent +ad ult +ĠNir vana +ĠFranch ise +L ayer +H yp +ĠBerks hire +Ġwill s +t if +Ġtot em +ĠJud ah +rep air +Inst ant +5 48 +Ġemb assies +Ġbott leneck +Ġb ount +Ġtyp ew +ĠAl vin +j ing +im ilar +R ush +Ġbr im +ĠHEL P +A im +] ' +Ġpass ively +Ġbound ed +ĠR ated +Ġcriminal ity +Ġbiom ark +Ġdisp atcher +ĠTow ards +Ġ+ ++ +right eous +f rog +ĠP anc +C arter +0 32 +æ© Ł +Ġult raviolet +ĠLic ensed +ĠT ata +ĠBl essing +ĠG AM +Ġchem ically +ĠSe af +ĠRE LE +ĠMerc enary +capital ist +Ġform ulations +Ġann ihilation +ĠVer b +ĠAr gon +Ġun loaded +Ġmorp hed +Ġconqu ering +back er +I ELD +Ġtheft s +Ġfront runner +ĠRoy ale +ĠFund amental +el ight +C hip +necess ary +ay n +ĠSl ip +Ġ4 48 +cern ed +P ause +Ġshock ingly +ĠAB V +Ġcomp osure +7 33 +ĠMotors port +ah ime +Mur ray +M ach +Ġgr ids +Ġdeb ian +Ġfurther more +Ġdexter ity +ĠCollect ions +os lov +il age +b j +ĠMont eneg +Ġstrut Connector +Ġmassac res +Ġbrief s +fet ched +uv ian +ol ition +Fail ure +emon ic +Ġfl ared +Ġclaim ant +Ġc ures +Ġgive aways +ĠSubst ance +al ions +Ġcr inge +ĠK ul +Ġarist ocracy +ĠUl ster +ol ated +h ousing +ĠM IS +Ġgl ared +ĠWil helm +ne eds +lam bda +build ers +ĠV IS +Ġradi ator +ĠGhost busters +Ġ4 36 +act ual +Ġher ds +ç a +watch ing +Ġcounter ing +Ch arge +Ġchar red +Ġwar heads +Ġiod ine +ĠM acy +04 1 +Ġdepart ures +ĠS ins +Ġdy ed +ĠConcept s +g ado +7 13 +Ġquot ations +Ġg ist +ĠChrist y +Ġant igen +ĠHem p +ĠD rawn +ĠB arg +ez vous +Ġp aternity +Ġar du +ĠAnch orage +ĠR ik +Ġover loaded +ĠUs ername +ĠTam my +ĠN au +ĠCell ular +Ġw aning +Ġrod ent +ĠWor cester +il ts +ĠT ad +Ġdwell ings +Ġbull ish +4 31 +Ġretali ate +Ġmig raine +ĠChev ron +CH ECK +Ġdon key +c rim +SP A +ĠAn alog +Ġmarqu ee +ĠHa as +B ir +ĠGD DR +ĠDownload s +Ġwill power +ĠFor th +ĠRecord ed +Ġimp ossibility +ĠLog ged +ĠFr anks +ĠR att +in itions +Ġclean ers +Ġsore ly +Ġflick ering +ĠEx amination +c atching +allow een +Ms g +Ġdun no +F a +Ġdys ph +c razy +.' '. +Ġmain line +Ġc s +Ġp tr +ĠW ally +ig un +95 1 +ĠBig foot +f ights +Ġretrie ving +J r +Ġdupl ication +ĠExpl an +Ġrel ational +Ġqu aint +Ġbisc uits +Ġad o +Ġsh udder +Ġantid ote +blood ed +ks h +Ġsa uces +Ġrein vest +Ġdispens ary +ĠD iver +Ġ9 000 +stud ent +Ġin separ +esc ap +Ġtodd lers +ĠGP IO +ĠAss ignment +head ers +Ġlack luster +Ġab ack +95 6 +Ġtool bar +7 45 +Ġo ust +Ġcontempl ation +ĠPRES IDENT +Ġ4 58 +==== == +Ġguarantee ing +ĠHe ist +ĠCann es +Ļ ½ +Ġcollabor ator +ĠAm p +Ġg ou +ĠSH ALL +st ories +78 3 +Ġmobil ized +Ġbro od +ĠL U +ĠðŁ ij +Ġref in +ĠAnthrop ology +v ind +ill i +Ġwarrant ies +ĠB abel +Ġsw ath +Ġc aches +Ġantagon ists +art ifacts +Ġhot ly +ĠSt arts +ĠG ö +z ag +!! !!! +Ġsc ourge +Ġcons piring +ru its +re verse +ĠShe en +ĠJes uit +ĠGiov anni +ad ies +Ġbutt ocks +ear cher +ac an +Ġvolley ball +Ġshroud ed +Ġscore board +b ats +ĠI PM +Ġass es +Ġde regulation +ĠTe legram +ĠReb oot +Ġ7 000 +ĠCan ary +Ġk ernels +ĠFranç ois +ĠD uff +ĠP on +ĠLe ica +ĠGar min +Ġor phans +ĠClaud ia +Ġcal endars +ĠLe ilan +ent o +R ocket +Ġbr unch +ĠHaw king +ain ers +Ġsens ibilities +Ġk W +ĠK and +Ġre claimed +Ġinteresting ly +× © +rom y +J M +ĠEnhance ment +b ush +Sk ip +Ġrapp ers +Ġg azing +p edia +ath lon +Rev olution +Ġsn ipers +Ġre verted +Ġconglomer ate +T erry +79 4 +Ġhars her +Ġdes olate +ĠHit man +Comm ission +Ġ( / +âĢ¦ ." +Com par +Ġampl ification +om inated +Ġreg ress +ĠColl ider +Ġinform ants +Ġg azed diff --git a/new_impl/nlp/roberta/sentiment-classification/roberta-base/tokenizer.json b/new_impl/nlp/roberta/sentiment-classification/roberta-base/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..ad0bcbeb288f0d1373d88e0762e66357f55b8311 --- /dev/null +++ b/new_impl/nlp/roberta/sentiment-classification/roberta-base/tokenizer.json @@ -0,0 +1 @@ +{"version":"1.0","truncation":null,"padding":null,"added_tokens":[{"id":0,"special":true,"content":"","single_word":false,"lstrip":false,"rstrip":false,"normalized":true},{"id":1,"special":true,"content":"","single_word":false,"lstrip":false,"rstrip":false,"normalized":true},{"id":2,"special":true,"content":"","single_word":false,"lstrip":false,"rstrip":false,"normalized":true},{"id":3,"special":true,"content":"","single_word":false,"lstrip":false,"rstrip":false,"normalized":true},{"id":50264,"special":true,"content":"","single_word":false,"lstrip":true,"rstrip":false,"normalized":true}],"normalizer":null,"pre_tokenizer":{"type":"ByteLevel","add_prefix_space":false,"trim_offsets":true},"post_processor":{"type":"RobertaProcessing","sep":["",2],"cls":["",0],"trim_offsets":true,"add_prefix_space":false},"decoder":{"type":"ByteLevel","add_prefix_space":true,"trim_offsets":true},"model":{"dropout":null,"unk_token":null,"continuing_subword_prefix":"","end_of_word_suffix":"","fuse_unk":false,"vocab":{"":0,"":1,"":2,"":3,".":4,"Ġthe":5,",":6,"Ġto":7,"Ġand":8,"Ġof":9,"Ġa":10,"Ġin":11,"-":12,"Ġfor":13,"Ġthat":14,"Ġon":15,"Ġis":16,"âĢ":17,"'s":18,"Ġwith":19,"ĠThe":20,"Ġwas":21,"Ġ\"":22,"Ġat":23,"Ġit":24,"Ġas":25,"Ġsaid":26,"Ļ":27,"Ġbe":28,"s":29,"Ġby":30,"Ġfrom":31,"Ġare":32,"Ġhave":33,"Ġhas":34,":":35,"Ġ(":36,"Ġhe":37,"ĠI":38,"Ġhis":39,"Ġwill":40,"Ġan":41,"Ġthis":42,")":43,"ĠâĢ":44,"Ġnot":45,"Ŀ":46,"Ġyou":47,"ľ":48,"Ġtheir":49,"Ġor":50,"Ġthey":51,"Ġwe":52,"Ġbut":53,"Ġwho":54,"Ġmore":55,"Ġhad":56,"Ġbeen":57,"Ġwere":58,"Ġabout":59,",\"":60,"Ġwhich":61,"Ġup":62,"Ġits":63,"Ġcan":64,"Ġone":65,"Ġout":66,"Ġalso":67,"Ġ$":68,"Ġher":69,"Ġall":70,"Ġafter":71,".\"":72,"/":73,"Ġwould":74,"'t":75,"Ġyear":76,"Ġwhen":77,"Ġfirst":78,"Ġshe":79,"Ġtwo":80,"Ġover":81,"Ġpeople":82,"ĠA":83,"Ġour":84,"ĠIt":85,"Ġtime":86,"Ġthan":87,"Ġinto":88,"Ġthere":89,"t":90,"ĠHe":91,"Ġnew":92,"ĠâĢĶ":93,"Ġlast":94,"Ġjust":95,"ĠIn":96,"Ġother":97,"Ġso":98,"Ġwhat":99,"I":100,"Ġlike":101,"a":102,"Ġsome":103,"S":104,"ë":105,"Ġthem":106,"Ġyears":107,"'":108,"Ġdo":109,"Ġyour":110,"Ġ-":111,"Ġ1":112,"\"":113,"Ġif":114,"Ġcould":115,"?":116,"Ġno":117,"i":118,"m":119,"Ġget":120,"ĠU":121,"Ġnow":122,"Ġhim":123,"Ġback":124,"ĠBut":125,"ĠâĢĵ":126,"Ġmy":127,"Ġ'":128,"Ġonly":129,"Ġthree":130,";":131,"Ġ2":132,"The":133,"1":134,"Ġpercent":135,"Ġagainst":136,"Ġbefore":137,"Ġcompany":138,"o":139,"ĠTrump":140,"Ġhow":141,"Ġbecause":142,"Ġany":143,"Ġmost":144,"Ġbeing":145,"Ġmake":146,"Ġwhere":147,"Ġduring":148,"Ġthrough":149,"Ġwhile":150,"000":151,"ĠThis":152,"Ġmillion":153,"ing":154,"Ġ3":155,"Ġmade":156,"Ġwell":157,"Ġ10":158,"Ġdown":159,"Ġoff":160,"Ġsays":161,"Ġme":162,"ĠB":163,"Ġgoing":164,"Ġteam":165,"ĠWe":166,"Ġthose":167,"Ġgovernment":168,"Ġway":169,"We":170,"Ġmany":171,"Ġthen":172,"Ġwork":173,"Ġtold":174,"com":175,"2":176,"Ġgame":177,"ĠAnd":178,"in":179,"year":180,"Ġp":181,"Ġvery":182,"Ġday":183,"Ġhome":184,"Ġtake":185,"Ġweek":186,"Ġsince":187,"ĠNew":188,"Ġmay":189,"Ġeven":190,"Ġseason":191,"Ġsee":192,"Ġ2017":193,"Ġstate":194,"Ġ5":195,"ed":196,"Ġshould":197,"Ġaround":198,"Ġ2018":199,"Ġsecond":200,"Ġus":201,"Ġstill":202,"Ġmuch":203,"Ġ4":204,"Ġgood":205,"Ġthink":206,"%":207,"ĠS":208,"Ġthese":209,"Ġmarket":210,"ĠD":211,"th":212,"Ġgo":213,"'re":214,"Ġsuch":215,"Ġknow":216,"Ġincluding":217,"Ġdon":218,"y":219,"Ġnext":220,"ĠP":221,"Ġdid":222,"Ġunder":223,"Ġsay":224,"en":225,"ĠL":226,"Ġbetween":227,"Ġper":228,"ĠK":229,"ĠC":230,"Ġ6":231,"Ġworld":232,"Ġpart":233,"ĠN":234,"Ġright":235,"Ġwant":236,"Ġfour":237,"),":238,"Ġhigh":239,"Ġneed":240,"re":241,"e":242,"It":243,"Ġhelp":244,"5":245,"3":246,"Ġcountry":247,"ĠR":248,"Ġpolice":249,"A":250,"Ġlong":251,"ĠThey":252,"Ġend":253,"er":254,"ĠT":255,"ĠM":256,"u":257,"Ġboth":258,"Ġhere":259,"an":260,"on":261,"Ġ7":262,"Ġde":263,"ĠShe":264,"Ġbusiness":265,"Ġreport":266,"j":267,"ers":268,"Ġreally":269,"ĠPresident":270,"ar":271,"ĠG":272,"ĠFriday":273,"ĠF":274,"Ġbest":275,"Ġsame":276,"Ġanother":277,"Ġset":278,"old":279,"ĠThat":280,"as":281,"n":282,"Ġcome":283,"Ġfamily":284,"Ġpublic":285,"ĠFor":286,"ĠAs":287,"0":288,"ĠH":289,"Ġ8":290,"Ġ20":291,"Ġfive":292,"es":293,"ĠTuesday":294,"Ġn":295,"ĠThursday":296,"Ġquarter":297,"h":298,"Ġtop":299,"Ġgot":300,"Ġlife":301,"ĠMonday":302,"Ġfound":303,"Ġuse":304,"ĠW":305,"4":306,"ĠWednesday":307,"Ġown":308,"Ġaccording":309,"Ġplay":310,"Ġshow":311,"ĠSt":312,"Ġman":313,"Ġleft":314,"ĠUnited":315,"Ġ12":316,"Ġplace":317,"ĠIf":318,"Ġlot":319,"Ġformer":320,"Ġ0":321,").":322,"Ġsupport":323,"ie":324,"Ġbillion":325,"Ġt":326,"Ġshares":327,"!":328,"z":329,"k":330,"ĠState":331,"Ġpoints":332,"Ġgroup":333,"Ġschool":334,"Ġinformation":335,"Ġ2016":336,"al":337,"r":338,"Ġwin":339,"Ġnews":340,"Ġused":341,"Ġput":342,"Ġcity":343,"ĠJ":344,"ĠThere":345,"Ġnumber":346,"C":347,"'ve":348,"Ġeach":349,"Ġtoo":350,"Ġwon":351,"ly":352,"Ġmonth":353,"is":354,"Ġadded":355,"Ġlook":356,"Ġbetter":357,"Ġevery":358,"Ġ&":359,"Ġdays":360,"Ġ9":361,"Ġtook":362,"Ġnight":363,"Ġe":364,"Ġ11":365,"os":366,"Ġfew":367,"or":368,"ĠNorth":369,"ĠYou":370,"Ġthird":371,"Ġgreat":372,"Ġcalled":373,"ĠOn":374,"Ġpast":375,"Ġcame":376,"Ġmonths":377,"ĠSaturday":378,"Ġ15":379,"Ġbig":380,"ĠE":381,"ĠUS":382,"Ġthings":383,"ĠO":384,"Ġd":385,"Ġstart":386,"B":387,"Ġstock":388,"Ġ30":389,"Ġwomen":390,"ĠSouth":391,"ĠMay":392,"Ġnever":393,"Ġpresident":394,"ĠSunday":395,"Ġwithout":396,"man":397,"8":398,"Ġdidn":399,"Ġlocal":400,"6":401,"Ġsomething":402,"Ġcase":403,"ĠAll":404,"it":405,"7":406,"ĠSo":407,"Ġchildren":408,"Ġaway":409,"Ġlittle":410,"Ġsix":411,"ĠCity":412,"ĠCounty":413,"Ġdata":414,"at":415,"Ġalready":416,"d":417,"Ġmoney":418,"Ġearly":419,"Ġacross":420,"Ġexpected":421,"Ġrun":422,"Ġlater":423,"am":424,"Ġprice":425,"Ġgames":426,"ĠMr":427,"b":428,"Ġmight":429,"Ġdifferent":430,"Ġreported":431,"Ġdeal":432,"Ġmedia":433,"Ġgrowth":434,"Ġcommunity":435,"ĠChina":436,"'m":437,"c":438,"Ġwent":439,"ĠNo":440,"Ġable":441,"Ġmaking":442,"Ġarea":443,"Ġfar":444,"Ġstatement":445,"ĠHouse":446,"Ġworking":447,"M":448,"Ġk":449,"Ġseen":450,"Ġcompanies":451,"Ġtoday":452,"Ġmembers":453,"Ġuntil":454,"Ġfull":455,"Ġagain":456,"Ġhalf":457,"Ġshare":458,"le":459,"Ġalways":460,"Ġcourt":461,"l":462,"and":463,"Ġchange":464,"Ġfind":465,"9":466,"Ġsystem":467,"ĠV":468,"ĠYork":469,"ĠAmerican":470,"Ġhead":471,"Ġplayers":472,"Ġdoes":473,"Ġhealth":474,"Ġm":475,"Ġpower":476,"Ġpoint":477,"Ġhit":478,"Ġ.":479,"Ġ--":480,"Ġfree":481,".,":482,"Ġlead":483,"Ġseveral":484,"Ġrecent":485,"Ġcall":486,"N":487,"Ġlaw":488,"Ġkeep":489,"Ġopen":490,"ĠNews":491,"Ġgive":492,"ia":493,"ĠMarch":494,"D":495,"ĠNational":496,"ĠAt":497,"Ġtimes":498,"Ġfuture":499,"R":500,"Ġ14":501,"ĠJune":502,"Ġofficials":503,"Ġ18":504,"Ġimportant":505,"f":506,"Ġfinal":507,"Ġ13":508,"ĠOne":509,"P":510,"Ġfollowing":511,"Ġcar":512,"Ġleast":513,"Ġwater":514,"Ġevent":515,"Ġline":516,"Ġmove":517,"Ġservices":518,"Ġhaving":519,"ĠWhen":520,"Ġstudents":521,"ĠPolice":522,"el":523,"Ġam":524,"ĠZ":525,"Ġside":526,"Ġstory":527,"Ġdue":528,"Ġmeeting":529,"K":530,"Ġmust":531,"ĠStates":532,"Ġlikely":533,"G":534,"Ġcontinue":535,"Ġago":536,"Ġparty":537,"Ġmajor":538,"Ġindustry":539,"Ġless":540,"30":541,"Ġun":542,"Ġhard":543,"Ġservice":544,"Ġ16":545,"Ġlooking":546,"Ġheld":547,"ve":548,"Ġwhether":549,"ĠJuly":550,"Ġtaken":551,"Ġalong":552,"Ġasked":553,"Ġstarted":554,"Ġbecome":555,"Ġforward":556,"Ġresearch":557,"Ġoffice":558,"Ġpolitical":559,"to":560,"Ġtogether":561,"Ġgetting":562,"Ġplan":563,"Ġ25":564,"T":565,"Ġamong":566,"Ġcoming":567,"Ġdecision":568,"Ġvideo":569,"Ġ2015":570,"g":571,"ĠAfter":572,"Ġsecurity":573,"L":574,"Ġcare":575,"Ġgiven":576,"Ġavailable":577,"âĢĶ":578,"Ġs":579,"ĠWest":580,"'ll":581,"Ġpay":582,"Ġnear":583,"Ġsaying":584,"Ġannounced":585,"Ġprogram":586,"ĠApril":587,"Ġreal":588,"ĠUniversity":589,"ĠWith":590,"AP":591,"Ġsocial":592,"Ġclose":593,"et":594,"Ġcurrent":595,"Ġwhy":596,"F":597,"ĠTo":598,"ĠTwitter":599,"Ġthough":600,"Ġ17":601,"Ġtaking":602,"ĠInc":603,"Ġmen":604,"w":605,"Ġcomes":606,"ley":607,"Ġdoing":608,"Ġprocess":609,"ĠJohn":610,"ch":611,"00":612,"Ġfinancial":613,"Ġlow":614,"Ġenough":615,"ĠWhile":616,"Ġfurther":617,"Ġpost":618,"Ġfeel":619,"st":620,"Ġperson":621,"ĠFacebook":622,"ĠWorld":623,"Ġwithin":624,"ad":625,"Ġdone":626,"the":627,"Ġlate":628,"Ġtax":629,"Ġdoesn":630,"Ġthing":631,"Ġnational":632,"Ġjob":633,"Ġusing":634,"ĠHowever":635,"ic":636,"Ġcampaign":637,"Ġrecord":638,"Ġbehind":639,"://":640,"ĠDepartment":641,"p":642,"Ġothers":643,"ĠJanuary":644,"Ġorder":645,"Ġ[":646,"Ġsales":647,"Ġyet":648,"Ä":649,"Ġsmall":650,"Ġseries":651,"Ġface":652,"ĠWhat":653,"Ġ50":654,"Ġever":655,"Ġearlier":656,"Ġlove":657,"up":658,"Ġrights":659,"ĠAn":660,"ist":661,"Ġmorning":662,"ĠWashington":663,"Ġyoung":664,"Ġlatest":665,"ĠIndia":666,"Ġtrying":667,"Ġfire":668,"Ġled":669,"Ġstrong":670,"Ġreturn":671,"Ġlevel":672,"O":673,"Ġaverage":674,"Ġperiod":675,"Ġexperience":676,"ak":677,"Ġpossible":678,"Ġbelieve":679,"Ġinclude":680,"Ġoil":681,"Ġrecently":682,"Ġonce":683,"Ġknown":684,"Ġlost":685,"Ġsure":686,"us":687,"Ġweeks":688,"Ġfood":689,"Ġreports":690,"Ġrating":691,"ĠMinister":692,"Ġwoman":693,"Ġprovide":694,"Ġproject":695,"Ġissue":696,"Ġlive":697,"10":698,"Ġclear":699,"he":700,"Ġcost":701,"Ġplayed":702,"Ġreleased":703,"Ġcoach":704,"v":705,"Ġ24":706,"Ġseven":707,"Ġplans":708,"Ġdevelopment":709,"ur":710,"ĺ":711,"Ġincrease":712,"This":713,"Ġpolicy":714,"Ġcent":715,"Ġbased":716,"E":717,"il":718,"ĠDecember":719,"Ġglobal":720,"Ġtrade":721,"Ġhours":722,"Ġhigher":723,"Ġgoal":724,"H":725,"ĠAl":726,"Ġ100":727,"Ġminutes":728,"Ġelection":729,"ĠAmerica":730,"Ġrate":731,"ĠCh":732,"Ġ21":733,"...":734,"ĠWhite":735,"Ġdirector":736,"Ġposition":737,"Ġshot":738,"Ġlarge":739,"Ġc":740,"Ġb":741,"]":742,"Ġissues":743,"Ġdeath":744,"Ġbuilding":745,"Ġtotal":746,"Ġoften":747,"Ġv":748,"Ġcountries":749,"Ġhistory":750,"Ġoutside":751,"Ġfederal":752,"Ġ19":753,"Ġfact":754,"ĠHigh":755,"Ġcareer":756,"im":757,"Ġinternational":758,"ĠNovember":759,"Ġfront":760,"Ġkind":761,"Ġkey":762,"ra":763,"ĠSan":764,"Ġshort":765,"Ġname":766,"ĠAccording":767,"Ġcourse":768,"Ġre":769,"Ġwanted":770,"W":771,"ĠSeptember":772,"Ġinterest":773,"Ġrole":774,"Ġresults":775,"Ġeconomic":776,"Ġ2014":777,"Ġchance":778,"ĠOctober":779,"Ġspecial":780,"Ġofficial":781,"Ġneeds":782,"um":783,"Ġl":784,"Ġproducts":785,"Ġnon":786,"Ġ@":787,"ĠBank":788,"Ġahead":789,"Ġhouse":790,"U":791,"Ġboard":792,"Ġold":793,"Ġsaw":794,"Ġlower":795,"ĠEuropean":796,"Ġcontrol":797,"ĠRussia":798,"Ġeight":799,"Ġrelease":800,"Ġpotential":801,"Ġthought":802,"Ġinvestigation":803,"Ġonline":804,"based":805,"Ġtechnology":806,"ĠDonald":807,"id":808,"Ġbody":809,"Ġrisk":810,"ian":811,"Ġcapital":812,"Ġstaff":813,"Ġaction":814,"ĠLeague":815,"Ġplaying":816,"Ġmakes":817,"Ġalmost":818,"Ġperformance":819,"Ġ22":820,"Ġg":821,"Ġfilm":822,"Ġnearly":823,"ĠCenter":824,"Ġvisit":825,"ĠGroup":826,"Ġbank":827,"Ġbit":828,"Ġreceived":829,"ĠAugust":830,"Ġmilitary":831,"ĠHis":832,"ine":833,"Ġchief":834,"ĠSchool":835,"Ġbring":836,"ĠCourt":837,"Ġ(@":838,"Ġmeans":839,"ĠSh":840,"Ġfans":841,"Ġse":842,"Ġ40":843,"20":844,"\".":845,"V":846,"Ġcut":847,"Ġkilled":848,"Ġ#":849,"Ġprices":850,"Ġgave":851,"ĠStreet":852,"ir":853,"ĠY":854,"Ġcurrently":855,"Ġf":856,"ay":857,"ne":858,"te":859,"Ġtry":860,"ĠPark":861,"ĥ":862,"J":863,"Ġquestion":864,"Ġhand":865,"Ġeconomy":866,"Ġinvestors":867,"able":868,"Ġplayer":869,"ĠBy":870,"ĠDavid":871,"Ġloss":872,"ab":873,"Ġbelow":874,"Ġwrote":875,"co":876,"ate":877,"Ġrunning":878,"un":879,"Ġbegan":880,"Ġsingle":881,"Ġfield":882,"Ġ23":883,"Ġleader":884,"Ġw":885,"ĠCalifornia":886,"Ġfourth":887,"Ġactually":888,"Ġlist":889,"ll":890,"Ġcouple":891,"Ġstudy":892,"Ġteams":893,"He":894,"ah":895,"ĠCanada":896,"Ġla":897,"Ġresult":898,"Ġaccess":899,"Ġvote":900,"ĠMore":901,"ĠFebruary":902,"Ġrevenue":903,"Ġoffer":904,"Ġlet":905,"ier":906,"Ġbuy":907,"Ġattack":908,"Ġblack":909,"Ġr":910,"Ġareas":911,"Ġstop":912,"Ġimpact":913,"Ġmatch":914,"Ġinvestment":915,"Ġcustomers":916,"Ġleaders":917,"ies":918,"Ġmember":919,"Ġchild":920,"Ġroad":921,"ul":922,"Ġvalue":923,"Ġshows":924,"ĠDr":925,"ĠDe":926,"ant":927,"ĠLondon":928,"Ġroom":929,"Ġmusic":930,"Ġproduction":931,"Ġanything":932,"Ġfirm":933,"Ġbiggest":934,"Ġair":935,"Ġproblem":936,"Ġgeneral":937,"Ġwasn":938,"Ġi":939,"Ġprivate":940,"Ġespecially":941,"Ġadministration":942,"Ġadditional":943,"ĠCo":944,"Ġopportunity":945,"Ġhold":946,"&":947,"Ġmatter":948,"Ġsenior":949,"Ġclub":950,"Ġsomeone":951,"ĠÃ":952,"ĠEast":953,"Ġ2019":954,".'":955,"Ġneeded":956,"ĠJames":957,"time":958,"Ġhowever":959,"Ġeverything":960,"Ġeveryone":961,"Ġdied":962,"Ġinvolved":963,"Ġfriends":964,"Ġisn":965,"Ġworth":966,"ik":967,"ĠCup":968,"Ġshowed":969,"There":970,"Ġ28":971,"Ġmeet":972,"Ġ26":973,"Ġ27":974,"Y":975,"Ġregion":976,"ĠPress":977,"ĠNow":978,"Ġson":979,"Ġspace":980,"Ġleading":981,"Ġstates":982,"Ġweekend":983,"Ġ£":984,"Ġmother":985,"Ġprevious":986,"ĠUK":987,"ĠMichael":988,"Ġleave":989,"est":990,"em":991,"Ġz":992,"ĠSome":993,"ors":994,"out":995,"15":996,"Ġwar":997,"Ġwebsite":998,"Ġstar":999,"X":1000,"ro":1001,"Ġtarget":1002,"Ġhimself":1003,"Ġturn":1004,"ĠEurope":1005,"Ġworked":1006,"Ġenergy":1007,"Ġscored":1008,"Ġ*":1009,"Ġsoon":1010,"Ġball":1011,"ĠTV":1012,"Ġannual":1013,"Ġ2013":1014,"Ġrace":1015,"ĠInternational":1016,"'d":1017,"ĠMarket":1018,"Ġconference":1019,"io":1020,"Ġo":1021,"Ġchanges":1022,"ig":1023,"Ġofficers":1024,"Ġinside":1025,"Ġform":1026,"Ġpublished":1027,"Ġphone":1028,"Ġco":1029,"Ġlegal":1030,"Ġexecutive":1031,"Ġfight":1032,"ings":1033,"Ġhope":1034,"Ġsummer":1035,"Ġofficer":1036,"Ġfootball":1037,"Ġproperty":1038,"@":1039,"Ġbook":1040,"Ġparents":1041,"Ġcosts":1042,"ac":1043,"Ġmanager":1044,"Ġcreate":1045,"Ġage":1046,"Ġemail":1047,"Ġmarkets":1048,"Ġmain":1049,"Ġhuman":1050,"Ġsent":1051,"Ġmanagement":1052,"ĠDay":1053,"ton":1054,"Ġcash":1055,"Ġfocus":1056,"Ġexpect":1057,"Ġtraining":1058,"Ġbecame":1059,"Ġwhose":1060,"Ġevents":1061,"Ġround":1062,"ĠLe":1063,"Ġfell":1064,"Ġabove":1065,"Ġanalysts":1066,"Ġtalk":1067,"Ġsituation":1068,"ri":1069,"ated":1070,"ke":1071,"Ġwants":1072,"ag":1073,"Ġlives":1074,"om":1075,"Ġal":1076,"Ġdemand":1077,"Ġsafety":1078,"Ġrest":1079,"ĠCouncil":1080,"Ġpersonal":1081,"Ġsite":1082,"ĠRussian":1083,"Ġmid":1084,"Ġnothing":1085,"Ġwhole":1086,"Ġbill":1087,"Ġsold":1088,"ĠBritish":1089,"se":1090,"Ġremain":1091,"12":1092,"Ġforeign":1093,"Ġshooting":1094,"Ġstay":1095,"50":1096,"ang":1097,"Ġhospital":1098,"Ġbad":1099,"Ġaddress":1100,"ĠKorea":1101,"Ġhappened":1102,"Ġcharges":1103,"Ġwhite":1104,"Ġ31":1105,"If":1106,"Ġearnings":1107,"Ġbreak":1108,"Ġlight":1109,"Ġterms":1110,"ĠChinese":1111,"ĠSenate":1112,"ana":1113,"Ġidea":1114,"ap":1115,"of":1116,"Ġnine":1117,"Ġcompared":1118,"Ġbuild":1119,"ard":1120,"In":1121,"Ġsimilar":1122,"Ġgas":1123,"Ġvictory":1124,"Ġ2012":1125,"Ġdebt":1126,"ĠMar":1127,"Ġarrested":1128,"Ġcomment":1129,"Ġincreased":1130,"Ġmedical":1131,"Ġ29":1132,"ĠJan":1133,"Ġgroups":1134,"Ġdespite":1135,"Ġfall":1136,"Ġtell":1137,"Ġworkers":1138,"Ġtown":1139,"é":1140,"Ġwife":1141,"Ġquestions":1142,"Ġcontinued":1143,"Ġheart":1144,"Ġmet":1145,"Ġbrought":1146,"Ġhelped":1147,"ĠCongress":1148,"Ġstep":1149,"Ġfather":1150,"Ġmoment":1151,"Ġproduct":1152,"Ġprobably":1153,"Ġlargest":1154,"Ġvehicle":1155,"ĠEngland":1156,"Ġallow":1157,"Ġstarting":1158,"Ġkids":1159,"Ġincident":1160,"Ġnet":1161,"Ġrates":1162,"ĠRead":1163,"Ġpressure":1164,"Ġincluded":1165,"Ġread":1166,"Ġissued":1167,"ol":1168,"Ġeither":1169,"Ġefforts":1170,"Ġincludes":1171,"ĠRepublican":1172,"ish":1173,"âĢ¦":1174,"Ġgoals":1175,"aj":1176,"Ġen":1177,"x":1178,"Ġraised":1179,"au":1180,"Ġlonger":1181,"ut":1182,"Ġwatch":1183,"ĠTexas":1184,"You":1185,"Ġrange":1186,"nd":1187,"Ġfunds":1188,"Ġremains":1189,"ĠMark":1190,"Ġ60":1191,"Ġque":1192,"sh":1193,"Ġinterview":1194,"Ġrather":1195,"Ġresidents":1196,"Ġgrowing":1197,"Ġpre":1198,"Ġpaid":1199,"Ġcases":1200,"ĠReuters":1201,"Ġdifficult":1202,"Ġsign":1203,"ĠGoogle":1204,"Ġhttps":1205,"ĠPaul":1206,"Ġliving":1207,"day":1208,"ĠQ":1209,"iz":1210,"ĠRed":1211,"Ġland":1212,"They":1213,"ĠRoad":1214,"_":1215,"ĠThese":1216,"Ġview":1217,"Ġagency":1218,"Ġreason":1219,"Ġallowed":1220,"ĠAustralia":1221,"az":1222,"ĠRe":1223,"Ġturned":1224,"11":1225,"Ġnation":1226,"Ġready":1227,"Ġpress":1228,"Ġbudget":1229,"Ġdaily":1230,"ĠChief":1231,"Ġfamilies":1232,"Ġsignificant":1233,"ĠFirst":1234,"Ġthemselves":1235,"Ġj":1236,"Ġruns":1237,"Ġaccused":1238,"Ġtakes":1239,"Ġspent":1240,"Ġvia":1241,"ot":1242,"ina":1243,"25":1244,"land":1245,"Ġexample":1246,"Ġauthorities":1247,"Ġdate":1248,"Ġended":1249,"all":1250,"Reuters":1251,"Ġbusinesses":1252,"ans":1253,"Ġdetails":1254,"Ġground":1255,"Ġpretty":1256,"ĠApple":1257,"ation":1258,"ĠSmith":1259,"ĠCompany":1260,"ĠFlorida":1261,"Ġdrug":1262,"Ġresponse":1263,"one":1264,"Ġeducation":1265,"Ġmean":1266,"Ġleague":1267,"Ġanyone":1268,"Ġminister":1269,"Ġtitle":1270,"Ġadding":1271,"Ġproblems":1272,"Ġopening":1273,"Ġconditions":1274,"Ġred":1275,"Ġdecided":1276,"Å":1277,"Ġposted":1278,"term":1279,"Ġamount":1280,"ĠEU":1281,"Ġsuccess":1282,"Ġevidence":1283,"ĠObama":1284,"Ġaddition":1285,"Ġprovided":1286,"ĠLos":1287,"Ġagreement":1288,"Ġstage":1289,"ens":1290,"Ġrelationship":1291,"ĠGeneral":1292,"Ġsector":1293,"Ġstudent":1294,"ating":1295,"Ġtest":1296,"\",":1297,"Ġwinning":1298,"Ġfelt":1299,"Ġsource":1300,"Z":1301,"Ġseems":1302,"Ġcause":1303,"Ġschools":1304,"Ġdrive":1305,"Ġensure":1306,"Ġhuge":1307,"ĠMy":1308,"ĠHealth":1309,"Ġscene":1310,"Ġgiving":1311,"Ġcenter":1312,"Ġpositive":1313,"Ġyards":1314,"Ġjobs":1315,"Ġaccount":1316,"Ġheard":1317,"Ġquality":1318,"Ġways":1319,"Ġimmediately":1320,"Ġemployees":1321,"are":1322,"Ġpass":1323,"ĠCEO":1324,"Ġreceive":1325,"Ġlooks":1326,"ĠAfrica":1327,"Ġthroughout":1328,"led":1329,"Ġrelated":1330,"Ġsell":1331,"ĠUnion":1332,"ĠPhoto":1333,"ter":1334,"Ġquickly":1335,"ĠHow":1336,"Ġvarious":1337,"Ġreach":1338,"Ġpick":1339,"Ġcharged":1340,"Ġquite":1341,"ent":1342,"q":1343,"ins":1344,"Ġphoto":1345,"Ġunderstand":1346,"ĠâĢ¢":1347,"Ġreached":1348,"Ġtrack":1349,"uk":1350,"Ġeffort":1351,"ville":1352,"Ġcentral":1353,"Ġdaughter":1354,"Ġcontract":1355,"Ġinjury":1356,"Ġopened":1357,"Ġ($":1358,"Ġstraight":1359,"17":1360,"Ġcredit":1361,"ĠIndian":1362,"Ġsexual":1363,"Ġworks":1364,"Ġeasy":1365,"18":1366,"Ġclosed":1367,"Ġh":1368,"Ġhappen":1369,"Ġforce":1370,"ler":1371,"Ġhappy":1372,"Ġshared":1373,"Ġoverall":1374,"Ġmoving":1375,"á":1376,"Ġprojects":1377,"ĠBlack":1378,"Ġconcerns":1379,"Ġclass":1380,"Ġtried":1381,"Ġappeared":1382,"Ġcontent":1383,"ĠDistrict":1384,"Ġterm":1385,"Ġinstead":1386,"ĠOffice":1387,"Ġcontinues":1388,"Ġlevels":1389,"Ġafternoon":1390,"Ġfund":1391,"Ġsale":1392,"Ġdriver":1393,"Ġask":1394,"Ġcannot":1395,"ner":1396,"end":1397,"ĠHere":1398,"field":1399,"Ġstore":1400,"www":1401,"Ġcertain":1402,"Ġself":1403,"Ġdollar":1404,"ĠHer":1405,"Ġpopular":1406,"Ġfollow":1407,"Ġspending":1408,"by":1409,"Ġmoved":1410,"Ġgoes":1411,"Ġcreated":1412,"Ġstand":1413,"Ġoperations":1414,"Ġlooked":1415,"Ġtreatment":1416,"ov":1417,"Ġdistrict":1418,"Ġsigned":1419,"Ġhands":1420,"Ġmodel":1421,"ĠAngeles":1422,"Ġy":1423,"Ġborder":1424,"Ġincome":1425,"ĠLast":1426,"Ġcharge":1427,"Ġdriving":1428,"ĠJapan":1429,"Ġrise":1430,"Ġtalks":1431,"Ġfollowed":1432,"Ġpreviously":1433,"Ġusers":1434,"Ġfunding":1435,"ĠJohnson":1436,"Ġ":1437,"ou":1438,"ai":1439,"Ġnamed":1440,"Ġfriend":1441,"ĠNov":1442,"Ġdefense":1443,"ĠBritain":1444,"Ġentire":1445,"Ġtrading":1446,"Ġfailed":1447,"ĠEl":1448,"Ġclaims":1449,"Ġcomments":1450,"Ġbeat":1451,"ib":1452,"Ġbasis":1453,"ĠJones":1454,"Ġpresent":1455,"ĠBe":1456,"Ġdouble":1457,"Ġrose":1458,"ite":1459,"Ġability":1460,"Ġoriginal":1461,"Ġdead":1462,"ĠCommission":1463,"ĠMe":1464,"Ġcompetition":1465,"Ġ2011":1466,"Ġknew":1467,"Ġmaterial":1468,"av":1469,"ĠFrance":1470,"Ġscore":1471,"Ġsense":1472,"Ġserious":1473,"Ġconfirmed":1474,"Ġanti":1475,"Ġviolence":1476,"Ġimprove":1477,"son":1478,"ó":1479,"ĠAP":1480,"Ġsh":1481,"Ġhost":1482,"ĠMike":1483,"Ġpatients":1484,"ĠNFL":1485,"Ġcrisis":1486,"Ġrevealed":1487,"ach":1488,"ĠPrime":1489,"Ġbuilt":1490,"ĠNot":1491,"Ġrules":1492,"Ġelse":1493,"Ġdepartment":1494,"Ġitself":1495,"ise":1496,"500":1497,"Ġcomplete":1498,"ion":1499,"Ġtrial":1500,"ĠBay":1501,"ĠDec":1502,"Ġattention":1503,"Ġtravel":1504,"ĠCentral":1505,"ry":1506,"Ġagreed":1507,"Ġmind":1508,"ĠMc":1509,"Ġ70":1510,"Ġcontact":1511,"ari":1512,"ĠTimes":1513,"Ġspot":1514,"ĠFrench":1515,"Ġgets":1516,"op":1517,"Ġbrand":1518,"Ġcalls":1519,"Ġbanks":1520,"Ġdesign":1521,"Ġsafe":1522,"Ġoffers":1523,"Ġpractice":1524,"ĠOf":1525,"á":1526,"ling":1527,"Ġtrue":1528,"off":1529,"Ġnumbers":1530,"Ġfun":1531,"Ġlearn":1532,"Ġmultiple":1533,"ĠIs":1534,"res":1535,"als":1536,"Ġcommon":1537,"ized":1538,"Ġchallenge":1539,"Ġcommittee":1540,"ĠOur":1541,"Ġbase":1542,"ani":1543,"ĠAssociation":1544,"ung":1545,"Ġnetwork":1546,"ĠBrown":1547,"Ġapproach":1548,"16":1549,"Ġfinished":1550,"Ġreview":1551,"Ġrequired":1552,"Ġapp":1553,"ĠMan":1554,"ĠâĢ¦":1555,"twitter":1556,"ĠDemocratic":1557,"13":1558,"Ġevening":1559,"ĠTom":1560,"ä":1561,"ĠAssociated":1562,"ĠCanadian":1563,"Ġcollege":1564,"Ġspokesman":1565,"Ġarticle":1566,"Ġtowards":1567,"ĠChicago":1568,"Ġmovie":1569,"14":1570,"ity":1571,"Ġforces":1572,"ĠChris":1573,"ĠDemocrats":1574,"Ġfeatures":1575,"Ġhearing":1576,"ĠX":1577,"ĠAlso":1578,"Ġmessage":1579,"age":1580,"Ġnoted":1581,"ĠSuper":1582,"Ġthousands":1583,"aw":1584,"ĠBill":1585,"ĠAr":1586,"ĠLa":1587,"ip":1588,"Ġ/":1589,"ĠDuring":1590,"Ġnote":1591,".)":1592,"Ġwrong":1593,"if":1594,"Ġpassed":1595,"ĠTwo":1596,"Ġdie":1597,",'":1598,"ĠDon":1599,"ĠGermany":1600,"Ġletter":1601,"Ġdescribed":1602,"ĠIran":1603,"ĠWilliams":1604,"Ġparticularly":1605,"Ġadd":1606,"Ġconversation":1607,"ĠSe":1608,"Ġhighest":1609,"be":1610,"Ġhomes":1611,"Ġsports":1612,"Ġgone":1613,"ĠAd":1614,"Ġel":1615,"Ġopportunities":1616,"Ġwords":1617,"Ġleaving":1618,"ĠChristmas":1619,"As":1620,"ĠGovernment":1621,"Ġsimply":1622,"Ġhusband":1623,"ĠResearch":1624,"ĠMexico":1625,"ates":1626,"ale":1627,"ĠGreen":1628,"$":1629,"od":1630,"ĠHall":1631,"Ġnatural":1632,"Ġoperating":1633,"les":1634,"ations":1635,"ĠKim":1636,"Ġgold":1637,"ok":1638,"Ġprovides":1639,"(":1640,"ell":1641,"Ġbegin":1642,"ĠParty":1643,"back":1644,"ĠAmazon":1645,"19":1646,"Ġmajority":1647,"ĠEven":1648,"Ġcheck":1649,"Ġweather":1650,"Ġorganization":1651,"Ġstories":1652,"ĠCar":1653,"Ġforced":1654,"ĠGeorge":1655,"Ġwalk":1656,"ong":1657,"Ġfiled":1658,"ĠJustice":1659,"Ġlaunched":1660,"Ġoffered":1661,"Ġwww":1662,"Ġconstruction":1663,"ĠBen":1664,"Ġserved":1665,"Ġ...":1666,"Ġparts":1667,"Ġcancer":1668,"Ġguys":1669,"Reporting":1670,"ash":1671,"less":1672,"Ġleadership":1673,"ĠCommittee":1674,"Ġregular":1675,"Ġcouncil":1676,"Ġcars":1677,"ĠDirector":1678,"Ġjudge":1679,"Ġvictims":1680,"ĠDaily":1681,"Ġkept":1682,"Ġeffect":1683,"Ġbeyond":1684,"pm":1685,"Ġtalking":1686,"Ġconsidered":1687,"ore":1688,"ĠAdvertisement":1689,"Ġst":1690,"ED":1691,"Ġmiddle":1692,"Ġraise":1693,"we":1694,"Ġclaimed":1695,"ino":1696,"Ġalleged":1697,"ĠPro":1698,"ĠScott":1699,"ĠOct":1700,"Ġconsider":1701,"ĠShare":1702,"Ġtraffic":1703,"ĠAfrican":1704,"Ġcouldn":1705,"Ġtoward":1706,"Ġsearch":1707,"But":1708,"Ġlaunch":1709,"Ġinjured":1710,"That":1711,"Ġalthough":1712,"Ġactivities":1713,"Ġchanged":1714,"Ġsources":1715,"Ġmissing":1716,"Ġu":1717,"Ġ35":1718,"Ġcover":1719,"ised":1720,"Ġ|":1721,"ow":1722,"ES":1723,"Ġdecades":1724,"ich":1725,"Ġcaused":1726,"Ġelections":1727,"ane":1728,"IS":1729,"Ġfeet":1730,"ĠBar":1731,"Ġversion":1732,"Ġgrow":1733,"Ġvehicles":1734,"Ġoptions":1735,"Ġindividual":1736,"Ġenvironment":1737,"ĠRobert":1738,"ĠValley":1739,"ĠFrom":1740,"per":1741,"ara":1742,"Ġsystems":1743,"Ġprotect":1744,"ĠKing":1745,"Ġinjuries":1746,"Ġfinally":1747,"Ġnuclear":1748,"40":1749,"Ġratio":1750,"Ġgun":1751,"ĠPakistan":1752,"ĠManagement":1753,"ĠAir":1754,"ce":1755,"Ġopposition":1756,"ment":1757,"ick":1758,"Ġpro":1759,"Ġact":1760,"Ġplatform":1761,"Ġlack":1762,"Ġpair":1763,"Ġ500":1764,"Ġcalling":1765,"ary":1766,"Ġprograms":1767,"Ġscheduled":1768,"Ġfast":1769,"Ġjoined":1770,"ĠWar":1771,"ĠEditing":1772,"ĠSince":1773,"ĠRyan":1774,"ĠMac":1775,"ĠBig":1776,"ĠLake":1777,"Ġdigital":1778,"When":1779,"ue":1780,"Ġassets":1781,"Ġseeing":1782,"ĠAct":1783,"Ġpartner":1784,"ĠBoard":1785,"Ġbeginning":1786,"Ġsupply":1787,"Ġmiles":1788,"Ġprison":1789,"ons":1790,"ĠAmericans":1791,"ub":1792,"ĠOr":1793,"me":1794,"Ġbenefits":1795,"Ġbenefit":1796,"Ġmeasures":1797,"Ġhear":1798,"Ġparties":1799,"Ġsuccessful":1800,"ĠJust":1801,"Ġvictim":1802,"Ġblock":1803,"Ġlimited":1804,"Ġtrip":1805,"ĠPeople":1806,"Ġserve":1807,"Ġart":1808,"ism":1809,"Ġwide":1810,"ĠSch":1811,"Ġ80":1812,"ĠThomas":1813,"Ġ90":1814,"Ġstocks":1815,"Ġgirl":1816,"ĠAsia":1817,"Ġseeking":1818,"Ġcertainly":1819,"ĠServices":1820,"ĠCollege":1821,"Ġcommunities":1822,"Ġextra":1823,"Ġ2010":1824,"ness":1825,"Ġholding":1826,"ous":1827,"Ġtough":1828,"ade":1829,"Ġmobile":1830,"Ġowns":1831,"ĠDo":1832,"ĠFire":1833,"Ġspoke":1834,"Ġreturned":1835,"Ġsize":1836,"Ġcriminal":1837,"ĠInstagram":1838,"Ġoffering":1839,"ĠGod":1840,"ĠService":1841,"Ġpage":1842,"her":1843,"Ġdeep":1844,"wood":1845,"Ġcrime":1846,"ĠSports":1847,"ile":1848,"ĠGlobal":1849,"Ġproposed":1850,"ain":1851,"Ġsession":1852,"ĠFederal":1853,"ĠSyria":1854,"Ġch":1855,"Ġthreat":1856,"Ġallegations":1857,"ĠRepublicans":1858,"ĠGerman":1859,"Ġstrategy":1860,"Ġcommercial":1861,"ING":1862,"ĠSecretary":1863,"Q":1864,"Ġreporters":1865,"100":1866,"ĠCapital":1867,"ĠBoth":1868,"ĠPost":1869,"ĠIsrael":1870,"Ġsave":1871,"ts":1872,"ill":1873,"Ġdrop":1874,"Ġreserved":1875,"ĠMany":1876,"Ġavoid":1877,"Ġ200":1878,"iv":1879,"Ġdamage":1880,"Ġcondition":1881,"Ġdropped":1882,"Ġdoor":1883,"Ġplanning":1884,"ire":1885,"Ġcard":1886,"Ġdesigned":1887,"Ġreduce":1888,"AN":1889,"ĠUn":1890,"ford":1891,"ĠThen":1892,"Ġpic":1893,"ĠCopyright":1894,"Ġrain":1895,"ĠMartin":1896,"Ġdomestic":1897,"45":1898,"ge":1899,"Ġmurder":1900,"Ġspeech":1901,"line":1902,"Ġhelping":1903,"Ġplanned":1904,"Ġfeature":1905,"ud":1906,"Ġtype":1907,"ham":1908,"ĠPublic":1909,"ja":1910,"Ġinsurance":1911,"Ġattacks":1912,"ĠCorp":1913,"Ġforecast":1914,"Ġresources":1915,"ma":1916,"?\"":1917,"ĠAm":1918,"ĠSept":1919,"Ġpush":1920,"Ġattorney":1921,"23":1922,"Ġemergency":1923,"Ġwinner":1924,"Ġblood":1925,"Ġnorth":1926,"ĠFeb":1927,"Ġbaby":1928,"Ġfloor":1929,"Ġspend":1930,"Ġex":1931,"Ġdollars":1932,"Ġunit":1933,"ĠHill":1934,"Ġder":1935,"ĠAbout":1936,"Ġalone":1937,"ization":1938,"Ġpresidential":1939,"Ġactivity":1940,"ĠTHE":1941,"ee":1942,"ber":1943,"ĠOther":1944,"Ġowner":1945,"Ġhour":1946,"Ġcities":1947,"Ġanswer":1948,"ide":1949,"Ġfully":1950,"ek":1951,"ists":1952,"Ġcoverage":1953,"Ġvs":1954,"Ġfigure":1955,"Ġpopulation":1956,"org":1957,"Ġsnow":1958,"Ġbecoming":1959,"ĠSam":1960,"ĠCarolina":1961,"Ġjoin":1962,"Ġprofit":1963,"Ġitems":1964,"Ġindex":1965,"Ġanalysis":1966,"Ġtournament":1967,"Ġstake":1968,"Ġperfect":1969,"way":1970,"Ġband":1971,"Ġgirls":1972,"Ġoption":1973,"Ġplays":1974,"oc":1975,"Ġproviding":1976,"ÃŃ":1977,"24":1978,"Ġwouldn":1979,"Ġones":1980,"Ġdeclined":1981,"Ġwritten":1982,"Ġvoters":1983,"Ġcandidate":1984,"Ġsuspect":1985,"Ġpolicies":1986,"Ġpeace":1987,"ast":1988,"Ġparticular":1989,"for":1990,"Ġhopes":1991,"Ġstation":1992,"ĠMost":1993,"Ġspeak":1994,"ĠRiver":1995,"Ġasking":1996,"Ġstatements":1997,"Ġfifth":1998,"ha":1999,"ĠNigeria":2000,"af":2001,"Ġexplained":2002,"Ġbar":2003,"Ġhousing":2004,"ĠSanta":2005,"Ġidentified":2006,"Ġsimple":2007,"Ġcritical":2008,"ĠClub":2009,"ĠSecurity":2010,"ĠLike":2011,"Ġstarts":2012,"art":2013,"Ġstreet":2014,"Ġreality":2015,"Ġheavy":2016,"Ġprogress":2017,"Ġshowing":2018,"Ġchallenges":2019,"Ġban":2020,"Ġcommitted":2021,"35":2022,"»":2023,"Ġdirectly":2024,"Ġaren":2025,"Ġclaim":2026,"ĠWestern":2027,"ind":2028,"Ġgives":2029,"ĠSaudi":2030,"Ġchoice":2031,"ĠTh":2032,"Ġapproved":2033,"Ġlocated":2034,"Ġarrived":2035,"22":2036,"Ġcaught":2037,"Ġprofessional":2038,"Ġmissed":2039,"Ġculture":2040,"ĠYear":2041,"ĠOhio":2042,"ĠLtd":2043,"ĠAnother":2044,"Ġseem":2045,"Ġbelieves":2046,"Ġbelieved":2047,"Ġcharacter":2048,"ĠAug":2049,"red":2050,"Ġfine":2051,"Ġprior":2052,"Ġthinking":2053,"Ġhttp":2054,"Ġ+":2055,"Ġzone":2056,"Ġputting":2057,"Ġcrash":2058,"ĠAustralian":2059,"ĠAb":2060,"Ġfocused":2061,"ĠREUTERS":2062,"ĠFox":2063,"ĠSp":2064,"Ġtraditional":2065,"Ġanalyst":2066,"Ġwait":2067,"IT":2068,"Ġrequest":2069,"ru":2070,"ians":2071,"ize":2072,"Ġfinish":2073,"Ġlaws":2074,"Ġran":2075,"ER":2076,"Ġsouth":2077,"Ġspeed":2078,"Ġmovement":2079,"Ġassault":2080,"Ġexchange":2081,"Ġappear":2082,"ĠSun":2083,"Ġle":2084,"Ġmaybe":2085,"Ġlosing":2086,"Ġsubject":2087,"ive":2088,"mer":2089,"ĠBusiness":2090,"ĠBl":2091,"Ġappears":2092,"Ġadvantage":2093,"ĠLee":2094,"ada":2095,"ĠUnder":2096,"Ġprevent":2097,"Ġrespect":2098,"Ġsex":2099,"Ġcentre":2100,"ĠJoe":2101,"ado":2102,"Ġtable":2103,"Ġequipment":2104,"Ġfair":2105,"Ġtour":2106,"Ġ32":2107,"ĠFinancial":2108,"Ġcounty":2109,"Ġdevices":2110,"Ġcustomer":2111,"Ġinfrastructure":2112,"Ġexpectations":2113,"Ġfacing":2114,"Ġupon":2115,"Ġcross":2116,"ĠOpen":2117,"AL":2118,"Ġquick":2119,"Ġattempt":2120,"Ġcompleted":2121,"Ġfacility":2122,"Ġconfidence":2123,"ĠSupreme":2124,"Ġpiece":2125,"our":2126,"Ġplaces":2127,"Ġsometimes":2128,"Ġpoor":2129,"Ġstorm":2130,"Ġhot":2131,"Ġaffected":2132,"na":2133,"Ġabuse":2134,"ĠMs":2135,"Ġword":2136,"over":2137,"Ġbrother":2138,"Ġnecessary":2139,"Ġeventually":2140,"ĠStar":2141,"Ġsend":2142,"Ġboy":2143,"ĠRs":2144,"Ġremember":2145,"21":2146,"Ġclimate":2147,"Ġcapacity":2148,"Ġresponsible":2149,"ĠMatt":2150,"month":2151,"Ġsuffered":2152,"%.":2153,"og":2154,"ĠPeter":2155,"Ġ,":2156,"Ġfeeling":2157,"ze":2158,"Ġbuying":2159,"oy":2160,"ij":2161,"Ġbought":2162,"Ġactions":2163,"Ġowned":2164,"Ġ___":2165,"Ġphysical":2166,"Ġspecific":2167,"Ġbattle":2168,"ĠEnergy":2169,"Ġpicture":2170,"Ġactive":2171,"Ġindividuals":2172,"Ġguy":2173,"Ġregional":2174,"Ġbond":2175,"ows":2176,"ĠToronto":2177,"Ġrule":2178,"Ġdevelop":2179,"Ġcrowd":2180,"Ġguilty":2181,"Ġfemale":2182,"Ġselling":2183,"ĠFollow":2184,"Ġmyself":2185,"ata":2186,"Ġdevice":2187,"Ġreasons":2188,"Ġrecords":2189,"Ġfighting":2190,"ON":2191,"ities":2192,"ĠHome":2193,"Ġstatus":2194,"Ġplant":2195,"Ġdrugs":2196,"ĠChurch":2197,"Ġcompletely":2198,"Ġdisease":2199,"Ġhighly":2200,"ĠParis":2201,"Ġdecade":2202,"Ġowners":2203,"Ġwall":2204,"Ġcamp":2205,"ĠSteve":2206,"Ġreporting":2207,"Ġearned":2208,"ĠImages":2209,"Ġexisting":2210,"ĠSen":2211,"Ġconcern":2212,"Ġhundreds":2213,"Ġsong":2214,"Ġknows":2215,"Ġunique":2216,"Ġlose":2217,"ĠKh":2218,"Ġapproximately":2219,"Ġhaven":2220,"Ġpark":2221,"Ġindependent":2222,"ĠAlthough":2223,"ĠAndrew":2224,"Ġpaper":2225,"Ġdeveloped":2226,"Ġrising":2227,"Ġdirect":2228,"Ġpurchase":2229,"Ġexactly":2230,"Ġq":2231,"Ġmassive":2232,"Ġbox":2233,"Ġchampion":2234,"ĠClinton":2235,"Ġvoice":2236,"Ġarrest":2237,"ĠKorean":2238,"Ġlearning":2239,"ĠVirginia":2240,"Ġsa":2241,"Ġpar":2242,"Ġchairman":2243,"Ġagencies":2244,"Ġhealthy":2245,"ĠThose":2246,"Ġpowerful":2247,"Ġ45":2248,"Ġdifference":2249,"ĠJackson":2250,"Ġenforcement":2251,"Ġdividend":2252,"qu":2253,"Ġenjoy":2254,"Ġruling":2255,"Ġongoing":2256,"Ġsoftware":2257,"ks":2258,"Ġlocation":2259,"Ġmostly":2260,"Ġcandidates":2261,"men":2262,"Ġbroke":2263,"What":2264,"ĠBr":2265,"Ġ2008":2266,"Ġconsumer":2267,"Ġdiscuss":2268,"Ġdi":2269,"Ġprimary":2270,"ĠEn":2271,"Ġgreen":2272,"Ġconcerned":2273,"Ġimage":2274,"ĠPremier":2275,"ĠMeanwhile":2276,"Ġfired":2277,"ĠBoston":2278,"ann":2279,"Ġcamera":2280,"Ġtraded":2281,"Ġhasn":2282,"Ġexcited":2283,"Ġincreasing":2284,"ĠDespite":2285,"Ġcitizens":2286,"Ġeuro":2287,"Ġreportedly":2288,"Ġminute":2289,"ĠWill":2290,"ĠLLC":2291,"Ġsp":2292,"ĠMichigan":2293,"Ġstopped":2294,"Ġeye":2295,"Ġdenied":2296,"Ġmodern":2297,"ĠWall":2298,"Ġdefinitely":2299,"point":2300,"Ġlines":2301,"Ġpolitics":2302,"Ġhotel":2303,"Ġretail":2304,"Ġstated":2305,"ĠOver":2306,"Ġgrew":2307,"Ġbroadcast":2308,"Ġlegislation":2309,"Ġfresh":2310,"Ġbid":2311,"Ġmanaged":2312,"Ġsociety":2313,"Ġscoring":2314,"ĠGet":2315,"Ġintelligence":2316,"Ġholiday":2317,"Ġgovernor":2318,"Ġestimated":2319,"Ġexperts":2320,"ĠJeff":2321,"Ġstruck":2322,"Ġhits":2323,"Ġcarry":2324,"Ġplaced":2325,"Ġstores":2326,"Ġexpressed":2327,"Ġvalued":2328,"Ġad":2329,"Ġtwice":2330,"ala":2331,"Ġdisplay":2332,"Ġusually":2333,"Ġresponded":2334,"Ġdog":2335,"AS":2336,"ĠFed":2337,"Ġ2009":2338,"Ġdocuments":2339,"Ġnormal":2340,"Ġtrain":2341,"Ġfl":2342,"Ġshown":2343,"ĠEd":2344,"Ġsort":2345,"Ġallegedly":2346,"Ġshots":2347,"ka":2348,"Ġaccounts":2349,"Ġyesterday":2350,"Ġcreating":2351,"Ġchurch":2352,"Ġbus":2353,"Ġaward":2354,"Ġequity":2355,"Ġphotos":2356,"Ġ33":2357,"Ġfiscal":2358,"je":2359,"Ġconsumers":2360,"ĠManchester":2361,"no":2362,"ĠKevin":2363,"Ġgain":2364,"Ġcorporate":2365,"Ġcivil":2366,"ĠMiddle":2367,"ally":2368,"Ġsound":2369,"ĠEnglish":2370,"IC":2371,"Ġwinds":2372,"Ġworst":2373,"ĠGrand":2374,"Ġeffective":2375,"ĠIsland":2376,"Ġdrivers":2377,"Ġfan":2378,"pe":2379,"Ġsides":2380,"ĠGo":2381,"Ġclean":2382,"âĢĵ":2383,"Ġtelevision":2384,"ĠJr":2385,"Ġallows":2386,"My":2387,"Ġgreater":2388,"ance":2389,"Ġdecisions":2390,"Ġrestaurant":2391,"ĠHospital":2392,"ĠTr":2393,"Ġbalance":2394,"Ġmph":2395,"Ġkeeping":2396,"Ġseconds":2397,"Ġweapons":2398,"ert":2399,"Ġpain":2400,"ass":2401,"Ġsteps":2402,"ger":2403,"ĠBrexit":2404,"Ġremaining":2405,"Ġbringing":2406,"ure":2407,"Ġweight":2408,"And":2409,"Ġwriting":2410,"Photo":2411,"ĠChristian":2412,"ob":2413,"Ġsport":2414,"Ġfigures":2415,"Ġtrust":2416,"Ġskills":2417,"Ġseat":2418,"Ġfaces":2419,"ck":2420,"Ġborn":2421,"Ġsuper":2422,"Ġfuel":2423,"Ġdel":2424,"Ġmeant":2425,"ica":2426,"Ġjustice":2427,"Ġspring":2428,"Ġkilling":2429,"Ġnegative":2430,"ĠRichard":2431,"Ġund":2432,"Ġfactors":2433,"Ġsigns":2434,"Ġlearned":2435,"ĠGame":2436,"Ġaudience":2437,"Ġdeliver":2438,"Ġillegal":2439,"Ġblue":2440,"Ġscreen":2441,"Ġremained":2442,"Ġannouncement":2443,"IN":2444,"Ġwaiting":2445,"Ġthanks":2446,"Ġimmigration":2447,"ĠFBI":2448,"Ġwarned":2449,"Ġmeasure":2450,"Ġdraw":2451,"Ġpositions":2452,"Ġdebut":2453,"ĠMedia":2454,"Ġallowing":2455,"air":2456,"hen":2457,"Ġmark":2458,"ys":2459,"Ġprepared":2460,"ĠVegas":2461,"ep":2462,"ice":2463,"2018":2464,"Ġdefensive":2465,"60":2466,"ĠBeach":2467,"Ġpulled":2468,"£":2469,"Ġlawyer":2470,"Ġcast":2471,"Ġsolution":2472,"Ġeyes":2473,"Ġmarketing":2474,"ĠFoundation":2475,"Ġrisks":2476,"ĠToday":2477,"za":2478,"Ġdraft":2479,"Ġice":2480,"26":2481,"ĠHar":2482,"ĠExecutive":2483,"Ġtruck":2484,"ions":2485,"ĠYour":2486,"ĠIreland":2487,"ĠJim":2488,"Ġha":2489,"Ġfear":2490,"Ġ36":2491,"UR":2492,"ĠFord":2493,"Ġwatching":2494,"ien":2495,"Ġstyle":2496,"ĠGood":2497,"Ġwearing":2498,"ĠHouston":2499,"Ġonto":2500,"Ġboost":2501,"Ġapplication":2502,"ĠDan":2503,"Ġspread":2504,"ĠDavis":2505,"Ġstrike":2506,"els":2507,"Ġwind":2508,"Ġinterested":2509,"Ġguard":2510,"Ġmission":2511,"Ġyourself":2512,"Ġoperation":2513,"Ġlarger":2514,"She":2515,"Ġseasons":2516,"28":2517,"27":2518,"Ġrespond":2519,"ci":2520,"ĠCentre":2521,"Our":2522,"Ġnames":2523,"Ġflight":2524,"Ġquarterback":2525,"Ġstandard":2526,"so":2527,"Ġsuggested":2528,"ĠMal":2529,"Ġolder":2530,"ini":2531,"Ġperhaps":2532,"ont":2533,"ĠInstitute":2534,"Ġmillions":2535,"Ġmental":2536,"ÃĤ":2537,"ga":2538,"Ġclients":2539,"Ġplease":2540,"Ġloan":2541,"Ġaware":2542,"ft":2543,"int":2544,"75":2545,"05":2546,"AY":2547,"ĠOut":2548,"Ġhair":2549,"ied":2550,"Ġseemed":2551,"ene":2552,"ty":2553,"NYSE":2554,"Ġoffensive":2555,"Ġtaxes":2556,"Ġinitial":2557,"ren":2558,"Ġseparate":2559,"la":2560,"ĠMiami":2561,"AC":2562,"Ġclearly":2563,"Ġfit":2564,"ĠCoast":2565,"Ġfirms":2566,"Ġpartners":2567,"Ġupcoming":2568,"Ġcold":2569,"Ġproposal":2570,"AT":2571,"Ġshut":2572,"ĠCommunity":2573,"Ġnature":2574,"ĠSal":2575,"Ġbottom":2576,"ting":2577,"ĠClick":2578,"Ġnice":2579,"ets":2580,"Ġhurt":2581,"itt":2582,"ama":2583,"Ġcarried":2584,"ĠCon":2585,"rd":2586,"Ġestate":2587,"ĠLas":2588,"ĠLaw":2589,"ng":2590,"Ġprotection":2591,"Ġproduce":2592,"Ġcurrency":2593,"Ġhappens":2594,"ĠPer":2595,"ney":2596,"ĠLong":2597,"Ġfellow":2598,"Ġcuts":2599,"Ġreading":2600,"ano":2601,"Ġproud":2602,"ost":2603,"ĠUN":2604,"ĠArizona":2605,"AD":2606,"Ġhelps":2607,"Ġwinter":2608,"Ġfinding":2609,"ĠGold":2610,"att":2611,"ĠWhy":2612,"Ġbasketball":2613,"lin":2614,"ĠCan":2615,"ĠBowl":2616,"ial":2617,"ĠAlex":2618,"200":2619,"AM":2620,"Ġpresence":2621,"Ġproduced":2622,"Ġdeveloping":2623,"Ġregarding":2624,"Ġdebate":2625,"Ġvice":2626,"ĠItaly":2627,"Ġsu":2628,"its":2629,"ator":2630,"Ġ34":2631,"Ġcomplex":2632,"Ġpresented":2633,"Ġresearchers":2634,"Ġslow":2635,"ya":2636,"Ġsanctions":2637,"Ġloved":2638,"Ġseek":2639,"Ġresponsibility":2640,"Ġadmitted":2641,"Ġalbum":2642,"Ġsolutions":2643,"Ġfacilities":2644,"ett":2645,"ĠGu":2646,"ĠWell":2647,"Ġlawmakers":2648,"Ġmiss":2649,"ful":2650,"ĠNick":2651,"'.":2652,"Ġfeels":2653,"Ġprime":2654,"Ġknowledge":2655,"Ġdeals":2656,"ĠTaylor":2657,"Ġsurvey":2658,"ĠFrancisco":2659,"Ġjoint":2660,"Ġwhom":2661,"Ġsit":2662,"01":2663,"Ġtr":2664,"Ġorganizations":2665,"ĠAvenue":2666,"ĠTheir":2667,"ĠTim":2668,"Ġrally":2669,"game":2670,"Ġbigger":2671,"Ġlawsuit":2672,"Ġrecorded":2673,"Ġfavorite":2674,"yard":2675,"Ġtransaction":2676,"Ġqu":2677,"oh":2678,"Ġinteresting":2679,"Ġinflation":2680,"ath":2681,"Ġstuff":2682,"Ġindustrial":2683,"ico":2684,"TS":2685,"Ġspeaking":2686,"Ġlosses":2687,"ID":2688,"ĠStadium":2689,"Ġstars":2690,"ĠWomen":2691,"ĠBlue":2692,"Ġwins":2693,"Ġdes":2694,"Ġcompetitive":2695,"ters":2696,"Ġpounds":2697,"Ġdirection":2698,"Ġinnings":2699,"ĠBest":2700,"Ġactor":2701,"Ġdangerous":2702,"Ġrequire":2703,"Ġplus":2704,"Ġsolid":2705,"Ġgeneration":2706,"Ġstrength":2707,"ĠMary":2708,"For":2709,"Ġplenty":2710,"ĠTeam":2711,"Ġinfluence":2712,"Ġfaced":2713,"Ġes":2714,"ĠIslamic":2715,"let":2716,"ĠDevelopment":2717,"Ġpath":2718,"Ġyouth":2719,"Ġcommitment":2720,"Ġbeautiful":2721,"ĠJack":2722,"ort":2723,"Ġten":2724,"Ġattend":2725,"ars":2726,"ón":2727,"Ġviews":2728,"Ġeuros":2729,"Ġauthor":2730,"Ġcore":2731,"Ġsupporters":2732,"ĠiPhone":2733,"Ġfashion":2734,"Ġsmaller":2735,"Ġelected":2736,"Ġuniversity":2737,"Ġpicked":2738,"wa":2739,"Ġordered":2740,"ĠSc":2741,"ĠÅ":2742,"Ġlargely":2743,"+":2744,"ĠAttorney":2745,"Ġpaying":2746,"AR":2747,"Ġconnection":2748,"Ġsetting":2749,"Ġna":2750,"ĠRock":2751,"Ġrecovery":2752,"ew":2753,"Ġserving":2754,"Ġsurprise":2755,"Ġoccurred":2756,"Ġdivision":2757,"Ġtelling":2758,"Ġmargin":2759,"Ġ2020":2760,"Ġsister":2761,"ĠNBA":2762,"Ġvoted":2763,"Ġcon":2764,"By":2765,"Ġ49":2766,"Ġfoot":2767,"ü":2768,"ĠTurkey":2769,"Ġamazing":2770,"Ġcombined":2771,"Ġappearance":2772,"Ġeasily":2773,"DAY":2774,"Ġnotes":2775,"ĠStart":2776,"Ġlanguage":2777,"Ġextremely":2778,"Ġcloudy":2779,"ĠLet":2780,"Ġdelivered":2781,"Ġimproved":2782,"Ġcollection":2783,"ĠPM":2784,"Ġestimates":2785,"Ġboys":2786,"izing":2787,"Ġtext":2788,"Ġcloser":2789,"Ġprotest":2790,"Ġprovince":2791,"Ġshop":2792,"Ġsmart":2793,"de":2794,"ĠSheriff":2795,"EN":2796,"Ġcorner":2797,"Ġpanel":2798,"Ġbooks":2799,"Ġsupported":2800,"Ġmentioned":2801,"ver":2802,"ĠMinistry":2803,"ĠPrince":2804,"ĠUSA":2805,"Ġreceiving":2806,"Ġchoose":2807,"ĠIN":2808,"ĠSpain":2809,"Ġsection":2810,"Ġconsidering":2811,"ĠCor":2812,"Ġwish":2813,"Ġwelcome":2814,"ĠConference":2815,"ere":2816,"ĠOfficer":2817,"Ġhoping":2818,"Ġportfolio":2819,"Ġstandards":2820,"Ġgrand":2821,"ĠReal":2822,"Ġsecure":2823,"ĠCorporation":2824,"ĠRep":2825,"ĠKelly":2826,"Ġstreets":2827,"Ġsitting":2828,"Ġslightly":2829,"ĠInvestment":2830,"99":2831,"ond":2832,"Ġunits":2833,"Ġvotes":2834,"Ġsegment":2835,"Ġchampionship":2836,"Ġsquad":2837,"iting":2838,"ron":2839,"®":2840,"Ġem":2841,"Ġtouch":2842,"Ġ38":2843,"Ġceremony":2844,"Ġdecide":2845,"Ġapproval":2846,"So":2847,"ĠPort":2848,"Ġsub":2849,"Ġsc":2850,"Ġrep":2851,"ĠWeek":2852,"Ġupper":2853,"Ġagree":2854,"ny":2855,"Ġmatches":2856,"ics":2857,"Ġtweeted":2858,"Ġheat":2859,"ĠGreat":2860,"Ġpenalty":2861,"Ġmass":2862,"Ġalongside":2863,"Ġherself":2864,"berg":2865,"Ġscience":2866,"Ġentered":2867,"Ġappeal":2868,"ĠPr":2869,"Ġfile":2870,"che":2871,"ĠReport":2872,"ĠThree":2873,"ĠNorthern":2874,"ĠJordan":2875,"Ġamid":2876,"Ġpace":2877,"Ġjail":2878,"Ġfinance":2879,"ĠYoung":2880,"32":2881,"Ġwilling":2882,"Ġconduct":2883,"ĠPar":2884,"Ġestablished":2885,"Ġreturns":2886,"Ġaid":2887,"Ġinternet":2888,"IA":2889,"29":2890,"Ġmeetings":2891,"Ġwarning":2892,"ĠCl":2893,"Ġcampus":2894,"Most":2895,"ĠFund":2896,"ĠWilliam":2897,"ĠJapanese":2898,"Ġconsensus":2899,"Ġbrain":2900,"!\"":2901,"Ġpoll":2902,"Ġtech":2903,"Ġtrend":2904,"Ġpotentially":2905,"Ġreduced":2906,"ĠShow":2907,"Ġ37":2908,"Ġhappening":2909,"ĠBrazil":2910,"pl":2911,"ĠCal":2912,"Ġcovered":2913,"Ġenter":2914,"TV":2915,"Ġcatch":2916,"foot":2917,"Ġunion":2918,"Ġexpansion":2919,"ĠSingapore":2920,"ĠDetroit":2921,"Ġattended":2922,"ats":2923,"Ġnewspaper":2924,"ĠDivision":2925,"news":2926,"Ġcap":2927,"Ġremoved":2928,"Ġ48":2929,"ĠRoyal":2930,"Ġwindow":2931,"Ġparking":2932,"Ġdark":2933,"Ġstanding":2934,"Ġupdate":2935,"Ġagent":2936,"Ġtransfer":2937,"ĠArmy":2938,"Ġuses":2939,"80":2940,"ĠTe":2941,"Ġintroduced":2942,"Ġmale":2943,"ĠSouthern":2944,"Ġratings":2945,"Ġisland":2946,"ĠMiller":2947,"Ġteachers":2948,"Ġadvice":2949,"Ġfamiliar":2950,"uf":2951,"Ġsought":2952,"Ġpor":2953,"ĠEric":2954,"Ġda":2955,"Ġideas":2956,"uh":2957,"Ġsixth":2958,"Ġtalent":2959,"ĠImage":2960,"ering":2961,"run":2962,"ments":2963,"Ġconducted":2964,"300":2965,"Ġurged":2966,"Ġdiscovered":2967,"Ġpl":2968,"Ġunderstanding":2969,"Ġoffense":2970,"Ġsecretary":2971,"Ġsk":2972,"Ġloans":2973,"ĠGr":2974,"Ġapplications":2975,"Ġcrude":2976,"go":2977,"ĠInstead":2978,"Ġopinion":2979,"Ġdoubt":2980,"ey":2981,"Ġdis":2982,"31":2983,"Ġexperienced":2984,"Ġleg":2985,"ĠCleveland":2986,"ven":2987,"Ġfailure":2988,"market":2989,"ack":2990,"Ġdecline":2991,"Ġchanging":2992,"Ġ300":2993,"Ġdefence":2994,"ĠBrian":2995,"Ġdelivery":2996,"Ġmarried":2997,"Ġdeclared":2998,"Ġpull":2999,"Ġlimit":3000,"ĠMORE":3001,"Ġdefeat":3002,"Ġexpand":3003,"ĠColorado":3004,"ĠRob":3005,"iss":3006,"Ġworse":3007,"Ġperform":3008,"ising":3009,"Ġ2007":3010,"ĠDel":3011,"Ġsurgery":3012,"Ġeasier":3013,"Ġmaintain":3014,"ĠEx":3015,"Ġtied":3016,"Ġeast":3017,"Ġuser":3018,"ola":3019,"Ġprogramme":3020,"Ġmanufacturing":3021,"Ġhitting":3022,"Ġx":3023,"Ġskin":3024,"Ġartist":3025,"Ġtells":3026,"Ġnearby":3027,"ĠDaniel":3028,"ĠPower":3029,"Ġdetermined":3030,"Ġactual":3031,"Ġtreated":3032,"Ġlived":3033,"Ġcomputer":3034,"Ġcool":3035,"oo":3036,"ĠPl":3037,"Ġeffects":3038,"Ġenvironmental":3039,"ĠMorgan":3040,"Ġflow":3041,"Ġachieve":3042,"ĠBell":3043,"Ġtesting":3044,"ĠBob":3045,"Ġwhatever":3046,"ĠBecause":3047,"US":3048,"ĠHollywood":3049,"Ġconflict":3050,"Ġwalking":3051,"ĠJudge":3052,"ĠAlabama":3053,"Ġaircraft":3054,"Ġte":3055,"well":3056,"Ġgoods":3057,"Ġidentify":3058,"Ġassociated":3059,"ĠVer":3060,"ĠEducation":3061,"Ġairport":3062,"IL":3063,"Ġfalling":3064,"Ġgiant":3065,"ĠMa":3066,"ĠMedical":3067,"Ġride":3068,"Ġden":3069,"º":3070,"ĠJose":3071,"Ġwest":3072,"ĠPacific":3073,"Ġvisitors":3074,"ĠWatch":3075,"ĠNations":3076,"Ġgains":3077,"Ġschedule":3078,"34":3079,"ĠExchange":3080,"Ġpayments":3081,"ĠII":3082,"70":3083,"No":3084,"ĠSyrian":3085,"ĠAdam":3086,"Ġne":3087,"Ġpartnership":3088,"Ġbl":3089,"ĠGeorgia":3090,"Ġsites":3091,"Ġmodels":3092,"Ġdegree":3093,"Ġdetermine":3094,"ĠWilson":3095,"Ġcontest":3096,"Ġprofessor":3097,"ĠChelsea":3098,"Ġmeaning":3099,"ĠGames":3100,"ĠTrust":3101,"ĠAsian":3102,"33":3103,"Ġlink":3104,"ĠUp":3105,"Ġholds":3106,"ĠTop":3107,"ĠItalian":3108,"ord":3109,"ĠKansas":3110,"Ġfarmers":3111,"Ġextended":3112,"Ġbirth":3113,"Ġreform":3114,"Ġrelations":3115,"Ġwrite":3116,"Ġsupporting":3117,"55":3118,"ita":3119,"Ġnotice":3120,"ster":3121,"Ġanimals":3122,"ĠJersey":3123,"Ġarm":3124,"ĠForeign":3125,"ĠLife":3126,"Ġtruly":3127,"ĠOnce":3128,"ĠMayor":3129,"ĠFree":3130,"ĠAgency":3131,"ĠWood":3132,"Ġpassing":3133,"DA":3134,"Ġ52":3135,"Ġmoves":3136,"Ġcom":3137,"house":3138,"ĠIts":3139,"Ġmarijuana":3140,"ines":3141,"Ġveteran":3142,"Ġvariety":3143,"ki":3144,"ff":3145,"amb":3146,"Ġlisted":3147,"Ġpushed":3148,"Ġvolume":3149,"Ġincreasingly":3150,"Ġkick":3151,"Ġrock":3152,"ank":3153,"Ġfees":3154,"Ġenable":3155,"Ġimages":3156,"Ġtruth":3157,"Ġministry":3158,"Ġrare":3159,"ĠDallas":3160,"ĠMinnesota":3161,"Ġcontributed":3162,"ĠCharles":3163,"Ġpercentage":3164,"Ġtechnical":3165,"ĠApp":3166,"Ġassistant":3167,"Ġinterests":3168,"Ġimmediate":3169,"38":3170,"ĠTown":3171,"Ġclosing":3172,"ĠAnthony":3173,"Ġsouthern":3174,"ase":3175,"ĠPutin":3176,"ĠForce":3177,"ba":3178,"Ġrefused":3179,"ĠStill":3180,"ix":3181,"ĠCol":3182,"Ġmaterials":3183,"Ġstructure":3184,"Ġdriven":3185,"Ġpatient":3186,"Ġbroken":3187,"Ġradio":3188,"Ġscale":3189,"Ġreplace":3190,"Ġ39":3191,"ĠLand":3192,"Ġdeputy":3193,"und":3194,"Ġcolor":3195,"OS":3196,"Ġroads":3197,"Ġcorruption":3198,"ĠRose":3199,"Ġemployee":3200,"ĠWater":3201,"Ġseats":3202,"Ġwalked":3203,"ec":3204,"Ġcents":3205,"Ġchain":3206,"Ġpayment":3207,"ĠAndroid":3208,"eb":3209,"Ġcommission":3210,"Ġthrow":3211,"Ġcount":3212,"Ġaccident":3213,"Ġexpensive":3214,"ered":3215,"ĠYes":3216,"ĠLouis":3217,"Ġstudies":3218,"Ġinvestigating":3219,"Ġcentury":3220,"Ġdiscussion":3221,"Ġinter":3222,"DAQ":3223,"ĠBefore":3224,"Ġinitially":3225,"*":3226,"Ġinvestments":3227,"Ġmulti":3228,"Ġtight":3229,"Ġconfident":3230,"Ġcounter":3231,"ĠQu":3232,"Ġgovernments":3233,"Ġarmed":3234,"Ġsuit":3235,"Ġrow":3236,"Ġlocations":3237,"Ġepisode":3238,"itch":3239,"Ġyounger":3240,"Ġfestival":3241,"Ġpitch":3242,"ĠOF":3243,"Ġtalked":3244,"ca":3245,"Ġprotests":3246,"Ġtargets":3247,"90":3248,"Ġoriginally":3249,"Ġsinger":3250,"Ġjourney":3251,"ug":3252,"Ġapply":3253,"Ġteacher":3254,"Ġchances":3255,"):":3256,"Ġdeaths":3257,"isation":3258,"ĠStephen":3259,"Ġcode":3260,"ĠChampionship":3261,"ĠJason":3262,"ĠAT":3263,"Ġaccept":3264,"ĠSeries":3265,"Ġvalues":3266,"Ġbed":3267,"ĠHarry":3268,"Ġflat":3269,"Ġtools":3270,"Ġpublicly":3271,"37":3272,"Ġpointed":3273,"ĠGolden":3274,"ps":3275,"Ġunable":3276,"ants":3277,"Ġestimate":3278,"Ġwarm":3279,"Ġbasic":3280,"ern":3281,"Ġraising":3282,"ĠRelated":3283,"Ġultimately":3284,"Ġnorthern":3285,"Ġplane":3286,"ĠVice":3287,"ĠRaj":3288,"ĠJustin":3289,"anc":3290,"Ġbrings":3291,"ĠArt":3292,"OT":3293,"Ġshift":3294,"ĠBBC":3295,"ĠSu":3296,"BS":3297,"Ġbag":3298,"Ġdoctor":3299,"Ġfill":3300,"Ġdowntown":3301,"Ġpossibility":3302,"ĠAg":3303,"Ġest":3304,"44":3305,"Ġstruggling":3306,"Ġlinked":3307,"Ġtickets":3308,"ĠJay":3309,"ĠCall":3310,"Ġstands":3311,"Ġwedding":3312,"Ġresident":3313,"eng":3314,"Ġleads":3315,"Ġadvance":3316,"ĠAtlanta":3317,"Ġtie":3318,"Ġadvanced":3319,"pt":3320,"burg":3321,"ĠEarlier":3322,"ĠSw":3323,"ĠZealand":3324,"Ġexercise":3325,"ĠAM":3326,"Ġaffect":3327,"Ġpossession":3328,"Ġinvolving":3329,"Ġ42":3330,"Ġwriter":3331,"ĠBeijing":3332,"Ġdoctors":3333,"Ġobviously":3334,"Ġer":3335,"ĠOlympic":3336,"Ġ75":3337,"ĠKhan":3338,"ĠFort":3339,"app":3340,"like":3341,"Ġsea":3342,"ock":3343,"Ġmix":3344,"ĠIraq":3345,"ĠMuslim":3346,"ĠFinally":3347,"Ġcontinuing":3348,"Ġpr":3349,"ĠKe":3350,"ĠJoseph":3351,"Ġexpects":3352,"Ġinstitutions":3353,"Ġconservative":3354,"own":3355,"ĠChairman":3356,"Ġreturning":3357,".-":3358,"Ġstood":3359,"Ġvision":3360,"ess":3361,"Ġadults":3362,"Ġyield":3363,"Ġprove":3364,"Ġorders":3365,"Ġdream":3366,"36":3367,"related":3368,"Ġsl":3369,"Ġeverybody":3370,"ui":3371,"Ġrepresents":3372,"Ġdiscussed":3373,"Ġbecomes":3374,"Ġvillage":3375,"CC":3376,"Ġnegotiations":3377,"ĠPhiladelphia":3378,"Ġcelebrate":3379,"Ġfarm":3380,"ç":3381,"Ġregistered":3382,"ĠGovernor":3383,"OL":3384,"ĠMon":3385,"Ġfiling":3386,"04":3387,"SE":3388,"ĠAssembly":3389,"Ġactress":3390,"Ġsi":3391,"Ġthank":3392,"Ġheading":3393,"ĠWho":3394,"Ġfamous":3395,"Ġconsecutive":3396,"Ġmarriage":3397,"ette":3398,"NAS":3399,"acks":3400,"ĠPlease":3401,"ĠDiego":3402,"Ġbaseball":3403,"ĠMoore":3404,"Ġties":3405,"Ġcarrying":3406,"que":3407,"Ġturning":3408,"ĠMcC":3409,"ĠKen":3410,"OR":3411,"ĠStock":3412,"Ġbuildings":3413,"49":3414,"ĠVan":3415,"39":3416,"ĠSeattle":3417,"Ġwild":3418,"Ġcrew":3419,"Ġroute":3420,"ĠTime":3421,"Ġtonight":3422,"Ġmoments":3423,"Ġvideos":3424,"Ġinternal":3425,"ĠLiverpool":3426,"port":3427,"Ġchair":3428,"Ġrival":3429,"ĠScotland":3430,"round":3431,"ith":3432,"Ġbreaking":3433,"Ġvoting":3434,"ically":3435,"Ġproducer":3436,"ĠLove":3437,"Ġremove":3438,"PA":3439,"Ġasset":3440,"Ġrequires":3441,"Ġsigning":3442,"ages":3443,"Ġimpressive":3444,"ĠIrish":3445,"Ġauthority":3446,"Ġruled":3447,"Ġaimed":3448,"Ġcaptain":3449,"AG":3450,"Ġplants":3451,"ĠAnderson":3452,"ĠSpanish":3453,"Ġbanking":3454,"Ġthreats":3455,"Ġsuspended":3456,"Ġtests":3457,"Ġreligious":3458,"Ġelectric":3459,"ĠREAD":3460,"Ġstrategic":3461,"Ġsplit":3462,"ex":3463,"Ġpractices":3464,"ĠIsraeli":3465,"ĠArabia":3466,"ĠMoscow":3467,"Ġfranchise":3468,"Ġcustody":3469,"ĠOld":3470,"Ġrequirements":3471,"Ġquarterly":3472,"Ġcomfortable":3473,"Ġcrimes":3474,"Ġheaded":3475,"Ġnewsletter":3476,"Ġanimal":3477,"Ġregulations":3478,"long":3479,"ĠCNN":3480,"Ġassists":3481,"Ġshopping":3482,"ĠGov":3483,"ĠSecurities":3484,"Ġassistance":3485,"Ġnor":3486,"Ġrelatively":3487,"Ġincreases":3488,"Ġgenerally":3489,"Ġ55":3490,"Ġgained":3491,"Ġ41":3492,"Ġpictures":3493,"gan":3494,"Ġpop":3495,"Ġupdates":3496,"ĠRepublic":3497,"Ġrebounds":3498,"ĠPatrick":3499,"Ġrelief":3500,"Ġacting":3501,"ĠFestival":3502,"Ġ2006":3503,"Ġboss":3504,"Ġtypes":3505,"65":3506,"ĠYet":3507,"Ġpurpose":3508,"ning":3509,"Ġmatters":3510,"Ġcompete":3511,"ball":3512,"ĠRam":3513,"Ġsw":3514,"ĠFollowing":3515,"ĠBush":3516,"Ġtroops":3517,"Ġsupposed":3518,"Ġfreedom":3519,"Ġfeatured":3520,"Ġstorage":3521,"ĠInformation":3522,"ĠHong":3523,"Ġgolf":3524,"Ġagents":3525,"Ġfraud":3526,"Ġminimum":3527,"Ġartists":3528,"Ġeat":3529,"high":3530,"ĠFormer":3531,"ĠKong":3532,"ĠJosh":3533,"ĠDelhi":3534,"Ġshowers":3535,"ĠAcademy":3536,"Ġapartment":3537,"Ġvan":3538,"Ġfish":3539,"oe":3540,"Ġfilms":3541,"ĠBo":3542,"Ġedge":3543,"Ġpossibly":3544,"Ġtweet":3545,"09":3546,"Ġresolution":3547,"jo":3548,"Ġkill":3549,"Ġ44":3550,"Ġcell":3551,"Ġscheme":3552,"Ġth":3553,"Ġbonds":3554,"Ġentry":3555,"Ġsecret":3556,"Ġ43":3557,"Ġending":3558,"Ġweren":3559,"ĠCredit":3560,"ĠLive":3561,"Ġretired":3562,"Ġmachine":3563,"Ġsummit":3564,"Ġsharing":3565,"Ġacquired":3566,"Ġera":3567,"Ġwear":3568,"ical":3569,"07":3570,"Ġexciting":3571,"li":3572,"BC":3573,"ĠSocial":3574,"Ġhistoric":3575,"ĠChe":3576,"ĠLewis":3577,"ira":3578,"Ġstolen":3579,"ĠSpeaking":3580,"Ġsleep":3581,"Ġspokeswoman":3582,"week":3583,"Ġpurchased":3584,"Ġimportance":3585,"EC":3586,"Ġends":3587,"Ġdress":3588,"Ġparliament":3589,"ĠCruz":3590,"Ġcards":3591,"hi":3592,"ĠEmail":3593,"Ġrepresent":3594,"Ġbrands":3595,"ĠSenior":3596,"Ġparticipants":3597,"Ġfly":3598,"Ġidentity":3599,"ĠHam":3600,"ĠSky":3601,"ij":3602,"SA":3603,"Ġpromised":3604,"Ġtrouble":3605,"Ġsuffering":3606,"Ġleaves":3607,"Ġsuggest":3608,"Sh":3609,"Ġbusy":3610,"Ġproperties":3611,"Ġworldwide":3612,"Ġcloud":3613,"ĠSEC":3614,"Ġclosely":3615,"Ġmanage":3616,"Ġnumerous":3617,"Ġbackground":3618,"ĠExpress":3619,"Ġ65":3620,"ĠTony":3621,"ĠMadrid":3622,"ev":3623,"der":3624,"Ġsignificantly":3625,"Ġalternative":3626,"Ġship":3627,"head":3628,"ators":3629,"Ġdinner":3630,"ax":3631,"SC":3632,"Ġcriticism":3633,"ĠMah":3634,"ĠMin":3635,"rie":3636,"ĠTour":3637,"Ġbench":3638,"Ġadds":3639,"Ġseriously":3640,"star":3641,"ĠJournal":3642,"ĠDi":3643,"ali":3644,"Ġsentence":3645,"ĠSeveral":3646,"Ġmayor":3647,"ati":3648,"Ġsuggests":3649,"Ġbehavior":3650,"Ġstronger":3651,"ĠFood":3652,"Ġclient":3653,"not":3654,"ĠPrice":3655,"Ġtargeted":3656,"ĠSingh":3657,"ĠNetwork":3658,"Ġprosecutors":3659,"Ġdirected":3660,"ĠDemocrat":3661,"bl":3662,"ues":3663,"ĠFamily":3664,"Ġconnected":3665,"ĠChampions":3666,"Ġroughly":3667,"Ġabsolutely":3668,"08":3669,"Ġpassengers":3670,"ö":3671,"ĠSpecial":3672,"Ġcoast":3673,"Ġcomplaint":3674,"Ġ400":3675,"ĠEm":3676,"ves":3677,"Ġdogs":3678,"Ġhandle":3679,"Ġotherwise":3680,"Ġsees":3681,"Ġticket":3682,"ĠAward":3683,"All":3684,"Ġtask":3685,"Ġsongs":3686,"ĠAmong":3687,"Ġdedicated":3688,"Ġsteel":3689,"looking":3690,"Ġshortly":3691,"Ġtackle":3692,"ative":3693,"Ġminor":3694,"â":3695,"Ġprovider":3696,"vers":3697,"use":3698,"ives":3699,"Ġtypically":3700,"Ġarms":3701,"ĠAnt":3702,"ĠIS":3703,"Ġjump":3704,"Ġ©":3705,"47":3706,"aff":3707,"Ġmonthly":3708,"ĠMicrosoft":3709,"ĠCBS":3710,"Ġthreatened":3711,"Ġhonor":3712,"ĠMo":3713,"42":3714,"Ġinning":3715,"Ġpool":3716,"Ġhealthcare":3717,"ĠStory":3718,"ĠTennessee":3719,"Ġpromote":3720,"EL":3721,"Ġemotional":3722,"Ġpe":3723,"Ġfactor":3724,"Ġinvestigators":3725,"Ľ":3726,"ĠBack":3727,"ĠProject":3728,"Ġcu":3729,"side":3730,"Ġmessages":3731,"TH":3732,"eg":3733,"Ġexperiences":3734,"Ġcausing":3735,"Ġjoining":3736,"Ġpackage":3737,"Ġbodies":3738,"Ġlots":3739,"ĠHarris":3740,"Ġcl":3741,"ĠInternet":3742,"free":3743,"Ġperformed":3744,"Ġpieces":3745,"buy":3746,"Ġcaption":3747,"Ġweb":3748,"Ġcontracts":3749,"At":3750,"Ġattempted":3751,"Ġunlikely":3752,"Ġclick":3753,"Ġinvest":3754,"IM":3755,"ĠView":3756,"Ġneighborhood":3757,"Ġring":3758,"ĠFour":3759,"ail":3760,"46":3761,"One":3762,"Ġnative":3763,"CH":3764,"OM":3765,"Ġalcohol":3766,"ĠVal":3767,"Ġcharacters":3768,"ĠPat":3769,"Ġpoliticians":3770,"ĠMag":3771,"Ġbegins":3772,"ĠAk":3773,"Ġlos":3774,"Ġpersonnel":3775,"Ġenjoyed":3776,"ĠTechnology":3777,"Ġsun":3778,"ĠIT":3779,"Ġdocument":3780,"Ġdeficit":3781,"Ġcoalition":3782,"Ġmemory":3783,"Ġpushing":3784,"any":3785,"ified":3786,"Ġfounder":3787,"Ġ2000":3788,"2017":3789,"Ġvisited":3790,"ĠThough":3791,"ph":3792,"Ġsoft":3793,"Ġflag":3794,"Ġmom":3795,"inch":3796,"ĠSamsung":3797,"Ġapps":3798,"Ġtouchdown":3799,"ĠCare":3800,"ĠMrs":3801,"Ġredistributed":3802,"Ġencourage":3803,"ched":3804,"Ġtend":3805,"Ġregions":3806,"pp":3807,"IP":3808,"br":3809,"ush":3810,"Ġargued":3811,"Ġjunior":3812,"BA":3813,"Ġsevere":3814,"ĠNIGHT":3815,"Ġdef":3816,"Ġsurrounding":3817,"48":3818,"Ġengine":3819,"Ġfilled":3820,"Ġseventh":3821,"Ġbattery":3822,"ĠAllen":3823,"Ġguidance":3824,"Ġroll":3825,"Ġrural":3826,"Ġexpert":3827,"Ġconvicted":3828,"Ġlikes":3829,"ĠRo":3830,"Ġgrown":3831,"Ġretirement":3832,"Ġintended":3833,"Ġmis":3834,"Ġarmy":3835,"Ġdance":3836,"ĠThank":3837,"Ġent":3838,"Ġoutlook":3839,"Ġpara":3840,"Ġdry":3841,"ĠTO":3842,"era":3843,"Ġwaste":3844,"Ġfaster":3845,"ĠEagles":3846,"TA":3847,"ĠFrank":3848,"Ã":3849,"LE":3850,"ura":3851,"ko":3852,"ao":3853,"Ġdistribution":3854,"Ġimprovement":3855,"Ġplayoff":3856,"Ġacquisition":3857,"ĠCH":3858,"Ġtomorrow":3859,"Ġstruggle":3860,"ĠHuman":3861,"Ġnewly":3862,"oon":3863,"ĠNe":3864,"con":3865,"sc":3866,"Ġunless":3867,"Ġtransition":3868,"ten":3869,"ĠInter":3870,"Ġequal":3871,"Ġrec":3872,"Ġappointed":3873,"Ġwake":3874,"ĠEarth":3875,"ose":3876,"ĠEastern":3877,"Ġsoldiers":3878,"ĠParliament":3879,"Ġsets":3880,"Ġattempts":3881,"ĠIllinois":3882,"Ġrevenues":3883,"ĠWil":3884,"Ġheads":3885,"Ġprepare":3886,"Ġpriority":3887,"PS":3888,"ĠJo":3889,"ĠNBC":3890,"Ġtherefore":3891,"yn":3892,"Ġinitiative":3893,"ct":3894,"Ġcoffee":3895,"ĠFair":3896,"43":3897,"den":3898,"form":3899,"ova":3900,"Ġappropriate":3901,"ĠPlay":3902,"Ġaccepted":3903,"Ġcreative":3904,"Ġfollows":3905,"Ġrescue":3906,"Ġtree":3907,"With":3908,"ĠNetflix":3909,"ĠFootball":3910,"Ġsurprised":3911,"Ġlowest":3912,"800":3913,"amp":3914,"Ġworried":3915,"mar":3916,"ran":3917,"Ġvisiting":3918,"Ġselected":3919,"ĠMusic":3920,"ĠAnn":3921,"Ġexplain":3922,"ging":3923,"Ġwidely":3924,"Ġsquare":3925,"Ġtrends":3926,"Ġimproving":3927,"ĠHead":3928,"ĠQueen":3929,"ĠSociety":3930,"Ġcutting":3931,"ĠGOP":3932,"03":3933,"',":3934,"ET":3935,"ĠDrive":3936,"oll":3937,"ato":3938,"ĠSea":3939,"Ġjury":3940,"ĠRights":3941,"Ġinvestor":3942,"ĠABC":3943,"Ġtool":3944,"ĠAre":3945,"Ġrejected":3946,"Ġemerging":3947,"Ġcounts":3948,"Ġnations":3949,"Ġfalse":3950,"Ġtreat":3951,"va":3952,"Ġweak":3953,"ĠHighway":3954,"down":3955,"Ġstruggled":3956,"ĠMP":3957,"Ġguests":3958,"Ġgender":3959,"Ġhouses":3960,"rit":3961,"ĠWild":3962,"Ġstreak":3963,"uc":3964,"ĠReserve":3965,"ĠRatings":3966,"alt":3967,"Ġgreatest":3968,"Ġlawyers":3969,"Ġreaching":3970,"Ġtemperatures":3971,"To":3972,"Ġoutstanding":3973,"Ġpasses":3974,"Ġfaith":3975,"inc":3976,"Ġcr":3977,"Ġinformed":3978,"oz":3979,"Ġtrees":3980,"Ġsending":3981,"Ġ150":3982,"bo":3983,"Ġwine":3984,"ros":3985,"Ġsuspected":3986,"Ġrepeatedly":3987,"Ġhat":3988,"Ġshape":3989,"ĠWh":3990,"Ġassist":3991,"Ġstress":3992,"Ġfeed":3993,"ark":3994,"ored":3995,"Ġwatched":3996,"Ġincredible":3997,"cl":3998,"nt":3999,"Ġentertainment":4000,"ih":4001,"Ġbeauty":4002,"Ġbi":4003,"ĠLocal":4004,"Ġsat":4005,"41":4006,"Ġbroad":4007,"Ġheavily":4008,"Ġengaged":4009,"Ġspecifically":4010,"ĠMen":4011,"ĠRoss":4012,"Ġ2005":4013,"ST":4014,"95":4015,"Ġdownload":4016,"400":4017,"Ġsentenced":4018,"ĠCatholic":4019,"ĠOklahoma":4020,"Ġthrew":4021,"Ġworry":4022,"Ġimp":4023,"Ġdrove":4024,"Ġcolleagues":4025,"Ġagenda":4026,"64":4027,"ĠEach":4028,"Ġfee":4029,"New":4030,"ium":4031,"Ġspokesperson":4032,"Ġbills":4033,"Ġ47":4034,"ĠAfghanistan":4035,"Ġinvited":4036,"ĠYouTube":4037,"Ġanniversary":4038,"Ġdozen":4039,"ram":4040,"ĠOnly":4041,"Ġemployment":4042,"Getty":4043,"Ġgap":4044,"Ġsweet":4045,"ĠLittle":4046,"Ġinf":4047,"ying":4048,"Ġglass":4049,"Ġclasses":4050,"Ġcoal":4051,"ĠSub":4052,"Ġduty":4053,"CA":4054,"Ġcoaches":4055,"Â":4056,"anna":4057,"ĠSk":4058,"Ġ46":4059,"ison":4060,"ille":4061,"ĠST":4062,"ric":4063,"Ġparticipate":4064,"Ġequ":4065,"Ġrich":4066,"Ġrespectively":4067,"Ġexpenses":4068,"Ġcombination":4069,"right":4070,"Ġshareholders":4071,"Ġturns":4072,"Ġearn":4073,"Ġ51":4074,"ured":4075,"Ġdrink":4076,"ĠKar":4077,"ĠShares":4078,"ĠMid":4079,"ĠGetty":4080,"Ġbridge":4081,"lo":4082,"Ġinspired":4083,"Ġsurface":4084,"Ġgift":4085,"ence":4086,"Ġchallenging":4087,"Ġoffices":4088,"Ġsuspects":4089,"ĠFinance":4090,"Ġab":4091,"bound":4092,"Ġmomentum":4093,"Ġbacked":4094,"Ġparent":4095,"Ġcrucial":4096,"ave":4097,"Ġdealing":4098,"Ġregulatory":4099,"Ġapparently":4100,"ĠMat":4101,"Ġapart":4102,"Ġport":4103,"ole":4104,"Ġbeach":4105,"Ġcultural":4106,"Ġinstitutional":4107,"Ġbeating":4108,"ĠIowa":4109,"ĠAli":4110,"67":4111,"Ġje":4112,"ays":4113,"Ġweekly":4114,"Ġbirthday":4115,"Ġpipeline":4116,"Ġknee":4117,"Ġsolar":4118,"ĠPe":4119,"Ġcategory":4120,"ĠArea":4121,"ky":4122,"ures":4123,"06":4124,"ĠBall":4125,"Ġsemi":4126,"ĠHamilton":4127,"hip":4128,"ĠPh":4129,"ĠNext":4130,"Ġathletes":4131,"ii":4132,"Ġmovies":4133,"han":4134,"net":4135,"Ġplastic":4136,"Ġbehalf":4137,"gen":4138,"Ġfindings":4139,"Ġstretch":4140,"ĠSa":4141,"Ġofficially":4142,"ĠSarah":4143,"Ġprivacy":4144,"ĠMad":4145,"Ġnone":4146,"gh":4147,"On":4148,"Ġdrama":4149,"ĠFl":4150,"ika":4151,"ĠArsenal":4152,"Ġviolent":4153,"UN":4154,"called":4155,"59":4156,"Ġhate":4157,"Ġrelationships":4158,"Ġgranted":4159,"ĠJon":4160,"Ġlisten":4161,"season":4162,"Ġfewer":4163,"GA":4164,"ĠLabour":4165,"Ġremarks":4166,"ĠJonathan":4167,"ĠRos":4168,"sey":4169,"ĠOntario":4170,"ĠThompson":4171,"ĠNight":4172,"Ġranked":4173,"ĠUkraine":4174,"Ġimmigrants":4175,"Ġdegrees":4176,"ĠGe":4177,"Ġlabor":4178,"umb":4179,"ĠYORK":4180,"Ġallies":4181,"sp":4182,"hed":4183,"sw":4184,"Ġtariffs":4185,"SP":4186,"Ġclassic":4187,"Ġawards":4188,"ents":4189,"Ġfix":4190,"Ġsoccer":4191,"Ġconcert":4192,"ust":4193,"Ġadult":4194,"Ġoutput":4195,"Ġmanaging":4196,"02":4197,"Ġpromise":4198,"Ġawareness":4199,"Ġgross":4200,"Ġentering":4201,"Ġpo":4202,"oj":4203,"Ġmetal":4204,"Ġexit":4205,"Ġexcellent":4206,"Ġclubs":4207,"hold":4208,"Ġreplaced":4209,"ĠClass":4210,"Ġscientists":4211,"Ġprimarily":4212,"ĠMer":4213,"ão":4214,"Ġcircumstances":4215,"ades":4216,"Ġsupplies":4217,"aker":4218,"ĠSand":4219,"Ġscandal":4220,"Ġsettlement":4221,"ĠWisconsin":4222,"ĠWarriors":4223,"ĠAustin":4224,"Ġjournalists":4225,"ening":4226,"Ġreflect":4227,"ĠBuy":4228,"ĠAwards":4229,"Ġselection":4230,"ĠBel":4231,"bury":4232,"Ġtechnologies":4233,"%,":4234,"ime":4235,"ĠÄ":4236,"ĠAdministration":4237,"Ġchannel":4238,"Star":4239,"Ġtransport":4240,"Ġawarded":4241,"ena":4242,"Ġmotor":4243,"orn":4244,"kin":4245,"Ġfeaturing":4246,"Ġphones":4247,"ĠAND":4248,"Ġrelevant":4249,"ĠSee":4250,"Ġwinners":4251,"Ġdad":4252,"ĠSource":4253,"ĠCheck":4254,"aut":4255,"ĠFar":4256,"Ġopponents":4257,"Ġoutcome":4258,"Ġdoors":4259,"Ġsuicide":4260,"ima":4261,"Ġjumped":4262,"Ġperspective":4263,"Ġtransportation":4264,"Ġthinks":4265,"ĠMor":4266,"Ġdeadline":4267,"Ġ53":4268,"ĠDeputy":4269,"ery":4270,"Ġdetailed":4271,"uch":4272,"ĠBur":4273,"Ġtrades":4274,"ĠGreg":4275,"Ġzero":4276,"erson":4277,"ĠChildren":4278,"Ġdu":4279,"66":4280,"Ġmixed":4281,"ĠBarack":4282,"54":4283,"Ġterritory":4284,"Ġac":4285,"Ġconcept":4286,"ĠAdd":4287,"Ġourselves":4288,"Ġreaction":4289,"ĠSydney":4290,"ink":4291,"Ġconsistent":4292,"Ġboat":4293,"room":4294,"Ġdozens":4295,"Ġeffectively":4296,"but":4297,"Ġmotion":4298,"Ġalive":4299,"ĠKey":4300,"weight":4301,"Ġexports":4302,"Ġoperate":4303,"Ġregime":4304,"ĠAuthority":4305,"och":4306,"ĠCR":4307,"leg":4308,"Ġforget":4309,"American":4310,"bs":4311,"Ġthoughts":4312,"ĠSign":4313,"ĠPatriots":4314,"Ġbrief":4315,"ĠOregon":4316,"ĠBal":4317,"Ġmine":4318,"Ġciting":4319,"Ġmagazine":4320,"more":4321,"ERS":4322,"ĠBer":4323,"ua":4324,"ox":4325,"ĠMain":4326,"Ġinstance":4327,"tr":4328,"Ġrestaurants":4329,"ora":4330,"Ġharassment":4331,"\",\"":4332,"Ł":4333,"Ġsilver":4334,"ĠMueller":4335,"ĠSenator":4336,"ĠEvery":4337,"Ġfootage":4338,"ms":4339,"Ġopposed":4340,"ĠLink":4341,"Ġver":4342,"Ġpleased":4343,"ame":4344,"ending":4345,"Ġrivals":4346,"ida":4347,"ike":4348,"ta":4349,"ĠCook":4350,"Ġheadquarters":4351,"ear":4352,"Ġaggressive":4353,"Ġcourts":4354,"ĠMuseum":4355,"Ġim":4356,"ĠHoldings":4357,"Ġcommunication":4358,"Ġphase":4359,"yl":4360,"Ġpowers":4361,"Ġproved":4362,"Ġcarbon":4363,"Ġaside":4364,"ĠOlympics":4365,"Ġgathered":4366,"ĠPennsylvania":4367,"Ġsmartphone":4368,"ĠMet":4369,"ĠHurricane":4370,"Ġprotected":4371,"Ġcommunications":4372,"Ġemerged":4373,"Ġaim":4374,"Ġstable":4375,"ides":4376,"GB":4377,"Ġentirely":4378,"Ġmissile":4379,"ĠGen":4380,"Ġunclear":4381,"Ġelectricity":4382,"ology":4383,"away":4384,"Ġlicense":4385,"ĠPittsburgh":4386,"Ġcameras":4387,"Ġmusical":4388,"Ġmanagers":4389,"57":4390,"Ġscores":4391,"Ġprofile":4392,"hel":4393,"¼":4394,"Ġshouldn":4395,"RA":4396,");":4397,"Ġpermanent":4398,"ome":4399,"Ġet":4400,"Ġmar":4401,"Ġfavor":4402,"Ġmaker":4403,"Ġdiscussions":4404,"ory":4405,"Ġsharp":4406,"Ġpleaded":4407,"Ġpassenger":4408,"quarter":4409,"Ġdem":4410,"Ġversus":4411,"Ġmainly":4412,"Ġeighth":4413,"ĠAirport":4414,"ĠCross":4415,"million":4416,"ĠNas":4417,"Ġcited":4418,"56":4419,"Ġyes":4420,"ĠBelow":4421,"arn":4422,"ĠTurkish":4423,"ĠSl":4424,"Ġstepped":4425,"Ġproducers":4426,"Ġovernight":4427,"Ġsounds":4428,"52":4429,"Ġ64":4430,"Ġ54":4431,"58":4432,"ĠClark":4433,"ĠRick":4434,"Ġgr":4435,"ĠMont":4436,"Ġbeer":4437,"une":4438,"Ġreporter":4439,"Ġcharity":4440,"Ġeating":4441,"Ġextend":4442,"Ġguess":4443,"NA":4444,"Ġhedge":4445,"Ġencouraged":4446,"owned":4447,"ĠMel":4448,"ĠKentucky":4449,"ace":4450,"Ġlineup":4451,"Ġhosts":4452,"Ġcapable":4453,"PR":4454,"ĠArts":4455,"Ġcontroversial":4456,"Ġhosted":4457,"ries":4458,"Ġroster":4459,"Ġfixed":4460,"ĠWalker":4461,"ged":4462,"Ġdisaster":4463,"Ġdispute":4464,"ĠDenver":4465,"ĠTrade":4466,"ute":4467,"ese":4468,"cy":4469,"Ġgrant":4470,"ĠMax":4471,"Ġdistance":4472,"isc":4473,"Ġeditor":4474,"ĠDave":4475,"Ġperformances":4476,"Ġlay":4477,"Ġvulnerable":4478,"ĠMurray":4479,"ĠâĤ¬":4480,"Ġmining":4481,"Ġ2004":4482,"level":4483,"ability":4484,"Ġauto":4485,"Ġfake":4486,"Ġattacked":4487,"ona":4488,"ups":4489,"ened":4490,"Ġfallen":4491,"Ġstations":4492,"ĠContact":4493,"itz":4494,"Ġincidents":4495,"Ġcomplaints":4496,"Ġoperates":4497,"Ġrefugees":4498,"Ġessential":4499,"ĠTest":4500,"Ġdemands":4501,"Ġroles":4502,"yr":4503,"Ġacts":4504,"Ġusual":4505,"ring":4506,"Ġhanded":4507,"ĠMatthew":4508,"hour":4509,"Ġindustries":4510,"Ġshoot":4511,"ĠAuthorities":4512,"Ġprobe":4513,"ĠUtah":4514,"ĠRBI":4515,"ĠAD":4516,"Ġprospect":4517,"outs":4518,"ĠUber":4519,"Ġbright":4520,"Ġmention":4521,"Ġsavings":4522,"ĠMiss":4523,"ONDON":4524,"Ġ1990":4525,"arm":4526,"ĠTen":4527,"These":4528,"Ġexplains":4529,"minute":4530,"85":4531,"Ġmaximum":4532,"Ġro":4533,"Ġrookie":4534,"Ġstudio":4535,"ĠCam":4536,"ĠGal":4537,"Ġdefend":4538,"hand":4539,"53":4540,"ĠOil":4541,"Ġserves":4542,"Ġsn":4543,"ios":4544,"ĠDefense":4545,"AB":4546,"Ġhired":4547,"Ġsupports":4548,"Ġpremium":4549,"ef":4550,"Ġfailing":4551,"ĠIndiana":4552,"Ġexp":4553,"Ġobjective":4554,"Ġaffordable":4555,"ĠCom":4556,"ĠThanks":4557,"Ġanywhere":4558,"Ġconfirm":4559,"ited":4560,"Ġrepresenting":4561,"Ġwitness":4562,"69":4563,"Ġclaiming":4564,"Ġviolation":4565,"Ġhistorical":4566,"med":4567,"Ġpreparing":4568,"ĠTech":4569,"Ġposts":4570,"OC":4571,"ĠGraham":4572,"ĠGl":4573,"ĠLions":4574,"ales":4575,"ĠID":4576,"Ġcorrect":4577,"ĠAntonio":4578,"Ġadvertising":4579,"Ġeastern":4580,"OW":4581,"Ġholdings":4582,"Ġpolls":4583,"ĠSH":4584,"Ġexecutives":4585,"ĠJewish":4586,"ĠGary":4587,"Ġprize":4588,"ĠCommissioner":4589,"Ġcells":4590,"ify":4591,"Ġlunch":4592,"Ġdemocracy":4593,"ĠEr":4594,"Ġregularly":4595,"Ġresulted":4596,"ĠAve":4597,"ĠPartners":4598,"Ġrewritten":4599,"Ġlo":4600,"Ġcooperation":4601,"ĠGulf":4602,"Ġsmoke":4603,"ĠMemorial":4604,"Ġwave":4605,"Ġfears":4606,"Ġkid":4607,"ĠGiants":4608,"Ġrecovered":4609,"row":4610,"ĠRadio":4611,"ĠBarcelona":4612,"Ġwonderful":4613,"ĠDow":4614,"Ġstream":4615,"ĠSimon":4616,"Ġdetail":4617,"Ġvolunteers":4618,"ĠInd":4619,"Ġforms":4620,"mann":4621,"ĠRay":4622,"oor":4623,"ĠTake":4624,"Ġrepresented":4625,"het":4626,"Ġblow":4627,"aged":4628,"RE":4629,"ĠMissouri":4630,"Ġcovering":4631,"Ġprofits":4632,"Ġconcluded":4633,"Ġthus":4634,"ĠColumbia":4635,"ode":4636,"ĠZimbabwe":4637,"Ġdisclosed":4638,"Ġlifted":4639,"ĠSean":4640,"ĠHarvey":4641,"ĠPlus":4642,"ces":4643,"ĠGreece":4644,"ĠLady":4645,"Ġdelay":4646,"Ġkitchen":4647,"ĠIndex":4648,"Ġbear":4649,"Ġputs":4650,"new":4651,"88":4652,"ĠAsh":4653,"Å¡":4654,"Ġperforming":4655,"law":4656,"ĠPart":4657,"Ġindicated":4658,"Ġannounce":4659,"Ġcompensation":4660,"Ġka":4661,"ĠScience":4662,"ris":4663,"Ġrecommendations":4664,"ĠSecond":4665,"Ġlights":4666,"Ġtemporary":4667,"urs":4668,"Ġwestern":4669,"stone":4670,"68":4671,"ĠDisney":4672,"Ġplayoffs":4673,"Ġjudges":4674,"Ġengineering":4675,"ĠPen":4676,"ĠPal":4677,"Ġobvious":4678,"ĠBridge":4679,"ĠEnd":4680,"ĠArab":4681,"Ġexcept":4682,"Ġhole":4683,"class":4684,"Ġcauses":4685,"Ġconnect":4686,"ĠAI":4687,"An":4688,"Ġchose":4689,"ĠElizabeth":4690,"min":4691,"Ġproper":4692,"ĠNHL":4693,"Ġraces":4694,"Ġinnovation":4695,"Ġsugar":4696,"600":4697,"ĠModi":4698,"illa":4699,"Ġtrillion":4700,"ĠSar":4701,"ĠAffairs":4702,"Ġimpossible":4703,"Ġguide":4704,"Ġcaptured":4705,"ĠSales":4706,"Ġspecies":4707,"51":4708,"Ġar":4709,"Ġmaster":4710,"Ġstayed":4711,"iro":4712,"ĠEconomic":4713,"Ġvast":4714,"ili":4715,"Ġpet":4716,"ye":4717,"77":4718,"Ġkeeps":4719,"ĠPhil":4720,"ĠEPS":4721,"ĠRegional":4722,"Ġsectors":4723,"Ġdesire":4724,"ĠStanley":4725,"¾":4726,"Ġunknown":4727,"Ġpot":4728,"ĠPR":4729,"Ġknowing":4730,"Ġflying":4731,"ĠTreasury":4732,"iers":4733,"enn":4734,"ably":4735,"Ġsick":4736,"Ġmanner":4737,"Ġmanufacturers":4738,"Ġchampions":4739,"gy":4740,"Part":4741,"ister":4742,"ĠMountain":4743,"Ġimagine":4744,"Ġportion":4745,"ĠCamp":4746,"Ġchemical":4747,"ible":4748,"ĠAnaly":4749,"ĠBureau":4750,"Ġpm":4751,"Ġupdated":4752,"Ġetc":4753,"ĠField":4754,"iles":4755,"Ġobtained":4756,"Ġstick":4757,"Ġcat":4758,"har":4759,"Ġmarked":4760,"Ġmedium":4761,"ĠDes":4762,"People":4763,"Ġwealth":4764,"ores":4765,"ĠBaltimore":4766,"Ġtip":4767,"Ġdismissed":4768,"ĠVictoria":4769,"ĠBrad":4770,"Ch":4771,"Ġ56":4772,"Ġstadium":4773,"eth":4774,"Ġthunder":4775,"Ġtested":4776,"Ġdrawn":4777,"Ġcounsel":4778,"ld":4779,"Ġspirit":4780,"uss":4781,"Ġtheme":4782,"my":4783,"Ġnecessarily":4784,"Ġelements":4785,"Ġcollected":4786,"ĠRes":4787,"ĠMaryland":4788,"ĠEnter":4789,"Ġfounded":4790,"ae":4791,"Ġpilot":4792,"Ġshoulder":4793,"PC":4794,"Ġargument":4795,"Ġyen":4796,"Ġreceiver":4797,"Ġharm":4798,"ĠET":4799,"Ġprotesters":4800,"Ġ72":4801,"ĠAaron":4802,"Ġed":4803,"Ġexpecting":4804,"\":\"":4805,"Ġbike":4806,"Äĩ":4807,"Ġluxury":4808,"half":4809,"ĠBarbara":4810,"Ġfoundation":4811,"Ġill":4812,"Ġsubmitted":4813,"Ġdeeply":4814,"Ġhospitals":4815,"ĠBJP":4816,"Ġshock":4817,"Ġplatforms":4818,"Ġsummary":4819,"ĠWhere":4820,"Ġcelebration":4821,"iff":4822,"Ġveterans":4823,"Ġachieved":4824,"fl":4825,"Ġactivists":4826,"ĠManager":4827,"Ġformal":4828,"Ġformed":4829,"Ġinvestigate":4830,"ĠKyle":4831,"Ġ:":4832,"ĠRa":4833,"ovic":4834,"Ġdrinking":4835,"Ġnetworks":4836,"ĠAlexander":4837,"ĠOs":4838,"Ġ)":4839,"Ġbomb":4840,"Ġrecalled":4841,"ito":4842,"ient":4843,"Ġrepresentatives":4844,"ĠChrist":4845,"ĠWay":4846,"Ġdeadly":4847,"Ġinvesting":4848,"ĠRussell":4849,"Ġconsumption":4850,"Ġharder":4851,"Ġbail":4852,"Ġcritics":4853,"Ġdanger":4854,"Ġdrew":4855,"ĠSol":4856,"Ġcopyright":4857,"ĠHenry":4858,"Ġbuyers":4859,"Ġresidential":4860,"Ġmaintenance":4861,"pr":4862,"Ġmarks":4863,"Ġages":4864,"Ġcovers":4865,"Ġton":4866,"Ġtitles":4867,"ĠPS":4868,"ĠEvans":4869,"Ġmigrants":4870,"Ġflights":4871,"Ġmonitoring":4872,"Ġaddressed":4873,"Ġvital":4874,"Ġcontrolled":4875,"Ġweapon":4876,"Ġinches":4877,"Ġreduction":4878,"Ġurban":4879,"Ġcoaching":4880,"Ġreducing":4881,"ila":4882,"Ġrealize":4883,"Ġmeat":4884,"Ġref":4885,"Ġoverseas":4886,"Ġblame":4887,"Ġterrorist":4888,"Ġstuck":4889,"ĠUs":4890,"esh":4891,"pro":4892,"Ġ58":4893,"ough":4894,"Ġexposure":4895,"ĠAbu":4896,"state":4897,"Ġproviders":4898,"Ġfore":4899,"Ġjet":4900,"bar":4901,"Ġownership":4902,"ret":4903,"Ġupset":4904,"Ġfacts":4905,"Ġpurchasing":4906,"Ġreforms":4907,"Ġriver":4908,"Ġsomebody":4909,"Ġguest":4910,"iy":4911,"Ġauction":4912,"ĠReading":4913,"Ġconsequences":4914,"Ġrepresentative":4915,"Ġappointment":4916,"add":4917,"Ġcollaboration":4918,"ĠTesla":4919,"ĠCohen":4920,"Ġengagement":4921,"Ġspeaks":4922,"EST":4923,"Ġexposed":4924,"Ġmaintained":4925,"rs":4926,"Ġdating":4927,"ĠProgram":4928,"board":4929,"Ġracing":4930,"Ġpension":4931,"ign":4932,"iti":4933,"ĠFive":4934,"Ġextensive":4935,"ĠHa":4936,"ĠPoint":4937,"ĠMexican":4938,"Ġexpanded":4939,"Ġtotally":4940,"Ġinvestigations":4941,"ĠOrleans":4942,"Ġcycle":4943,"ĠESPN":4944,"ifying":4945,"Ġcup":4946,"ĠAz":4947,"ĠInvestors":4948,"Ġengage":4949,"reg":4950,"Ġfought":4951,"Ġterrorism":4952,"Ġblocked":4953,"ĠOK":4954,"Äį":4955,"72":4956,"Ġdestroyed":4957,"«":4958,"Ġstaying":4959,"Ġafford":4960,"Ġappearances":4961,"ĠHills":4962,"Ġcrore":4963,"Ġstrategies":4964,"Ġtips":4965,"ĠSm":4966,"ĠFr":4967,"Ġbanned":4968,"ĠSon":4969,"ask":4970,"Ġlimits":4971,"Ġrecognition":4972,"Ġeligible":4973,"ĠGar":4974,"Ġvolatility":4975,"Ġlaid":4976,"nes":4977,"Ġgrade":4978,"ĠRE":4979,"ĠHart":4980,"Ġ57":4981,"oma":4982,"Ġuncertainty":4983,"Ġrecognized":4984,"ĠPC":4985,"Ġchosen":4986,"uz":4987,"Ġadviser":4988,"una":4989,"Ġassessment":4990,"Ġreveal":4991,"mo":4992,"After":4993,"ĠBro":4994,"ĠOff":4995,"Ġpeak":4996,"Ġreferred":4997,"ĠSC":4998,"Ġ2003":4999,"ification":5000,"Ġshutdown":5001,"ĠOfficials":5002,"ias":5003,"Ġextreme":5004,"Ġflood":5005,"Ġhockey":5006,"Ġwage":5007,"ĠNet":5008,"Ġdamaged":5009,"Ġreplacement":5010,"ĠMaria":5011,"Ġcreation":5012,"Ġguns":5013,"aci":5014,"Ġworker":5015,"do":5016,"Ġviewers":5017,"Ġseed":5018,"sts":5019,"Ġtouchdowns":5020,"Ġmistake":5021,"ray":5022,"ull":5023,"Ġpricing":5024,"Ġstrongly":5025,"Ġaims":5026,"ĠNavy":5027,"ĠEgypt":5028,"ker":5029,"Ġve":5030,"ĠSteven":5031,"Ġres":5032,"ational":5033,"Ġrequests":5034,"Ġemissions":5035,"ĠArena":5036,"uma":5037,"ĠAtlantic":5038,"hr":5039,"ĠAFP":5040,"ĠSquare":5041,"Ġcontribute":5042,"Ġfunction":5043,"Ġdec":5044,"ĠNelson":5045,"89":5046,"Ġreferendum":5047,"ĠPre":5048,"Ġapplied":5049,"ĠGMT":5050,"ĠIranian":5051,"ĠNigerian":5052,"ĠAny":5053,"NG":5054,"Ġacknowledged":5055,"Ġreferring":5056,"Ġventure":5057,"Ġimports":5058,"Ġblog":5059,"Ġfutures":5060,"OU":5061,"ĠUFC":5062,"Ġneither":5063,"Ġextension":5064,"hes":5065,"ĠMed":5066,"76":5067,"Ġsustainable":5068,"ains":5069,"Ġreputation":5070,"ĠVancouver":5071,"Ġbasically":5072,"acy":5073,"Ġsad":5074,"ĠFrancis":5075,"ĠKennedy":5076,"ĠNevada":5077,"ĠLu":5078,"ras":5079,"ĠAv":5080,"Ġrear":5081,"ĠHo":5082,"Ġproperly":5083,"abe":5084,"ĠHotel":5085,"Ġopinions":5086,"under":5087,"ĠStation":5088,"ĠFOR":5089,"ops":5090,"Ġadopted":5091,"ĠSwiss":5092,"ĠCountry":5093,"ĠTer":5094,"ĠAndy":5095,"Me":5096,"ĠCooper":5097,"ĠTigers":5098,"ĠCreek":5099,"Ġgay":5100,"iner":5101,"ĠAN":5102,"Ġbird":5103,"lla":5104,"ĠKate":5105,"ĠPet":5106,"ni":5107,"Ġprospects":5108,"ater":5109,"ites":5110,"Ġescape":5111,"lam":5112,"ake":5113,"Ġ1980":5114,"ĠLag":5115,"Ġsuccessfully":5116,"Ġdistricts":5117,"Ġministers":5118,"aries":5119,"Ġframe":5120,"ĠON":5121,"ĠEuro":5122,"ĠMarkets":5123,"Ġregister":5124,"Ġdefeated":5125,"Ġdevelopments":5126,"Ġninth":5127,"Ġquiet":5128,"Ġgenerated":5129,"Ġvaluable":5130,"Ġrecommended":5131,"ĠTheatre":5132,"ĠCap":5133,"bed":5134,"Ġreference":5135,"Ġease":5136,"oring":5137,"Ġ66":5138,"Ġimprovements":5139,"Ġelsewhere":5140,"ĠHillary":5141,"Ġdefender":5142,"ĠRight":5143,"zy":5144,"Ġcomprehensive":5145,"Ġspotted":5146,"ĠOakland":5147,"ĠOk":5148,"ĠSystem":5149,"ique":5150,"Ġpersons":5151,"Ġexist":5152,"Ġbroader":5153,"Ġclinical":5154,"Ġ2001":5155,"oul":5156,"Ġsecurities":5157,"ghan":5158,"Ġshelter":5159,"ero":5160,"ATED":5161,"Ġhosting":5162,"Ġselect":5163,"ĠKavanaugh":5164,"Ġrestrictions":5165,"osa":5166,"Ġyields":5167,"ĠLA":5168,"Ġ59":5169,"Ġwonder":5170,"Ġabsence":5171,"ür":5172,"ÅĤ":5173,"DP":5174,"Ġelectronic":5175,"Ġillegally":5176,"Ġmicro":5177,"ĠNEW":5178,"Ġhall":5179,"Ġaged":5180,"Ġtemperature":5181,"cast":5182,"atic":5183,"Ġlegacy":5184,"Ġaffairs":5185,"ji":5186,"ĠResources":5187,"Ġgang":5188,"winning":5189,"Ġattending":5190,"aro":5191,"Ġfriendly":5192,"aine":5193,"Ġcannabis":5194,"Ġairline":5195,"Ġnoting":5196,"Ġprofessionals":5197,"ĠFREE":5198,"RC":5199,"Ġfinancing":5200,"Ġindependence":5201,"ved":5202,"Ġresulting":5203,"Ġsteady":5204,"ĠWinter":5205,"uring":5206,"Ġhoped":5207,"98":5208,"Ġpresentation":5209,"aya":5210,"Ġrated":5211,"osh":5212,"ĠAnalysis":5213,"=":5214,"Ġdonations":5215,"IR":5216,"Ġcombat":5217,"ĠHoward":5218,"anda":5219,"79":5220,"Ġinvested":5221,"Ġexpanding":5222,"omb":5223,"ress":5224,"ble":5225,"Ġjournalist":5226,"ĠWoods":5227,"Ġcenters":5228,"ott":5229,"Ġstreaming":5230,"Ġterror":5231,"Ġsustained":5232,"ĠWWE":5233,"pre":5234,"ÅŁ":5235,"ait":5236,"Ġarrival":5237,"Ġresidence":5238,"Ġextent":5239,"Ġarrive":5240,"Ġ2002":5241,"Ġestablish":5242,"74":5243,"ĠArgentina":5244,"ĠDem":5245,"inn":5246,"aud":5247,"ĠNCAA":5248,"Ġquestioned":5249,"Ġballot":5250,"Ġmin":5251,"Ġlandscape":5252,"Ġhorse":5253,"Ġopponent":5254,"iel":5255,"Ġprompted":5256,"atory":5257,"Ġlift":5258,"Ġassociation":5259,"cher":5260,"Ġdefending":5261,"Ġtiny":5262,"Ġpoverty":5263,"ĠSafety":5264,"Ġpetition":5265,"ĠLimited":5266,"ĠCA":5267,"FC":5268,"Ãł":5269,"oni":5270,"Ġmonitor":5271,"ÃŃa":5272,"MA":5273,"Ġanswers":5274,"ĠMitchell":5275,"Ġbo":5276,"ĠShah":5277,"Ġsm":5278,"Ġmedal":5279,"ĠCivil":5280,"Ġrecognize":5281,"key":5282,"Ġpregnant":5283,"Ġspots":5284,"ante":5285,"Ġacademic":5286,"Ġinitiatives":5287,"Ġsecured":5288,"ĠCL":5289,"ils":5290,"Ġanticipated":5291,"Ġinvolvement":5292,"ĠMake":5293,"Ġinsisted":5294,"ĠWales":5295,"Ġclothing":5296,"Ġtracks":5297,"Ġsymptoms":5298,"Ġplate":5299,"ĠNY":5300,"Ġretailers":5301,"ĠPan":5302,"Ġfled":5303,"Ġquoted":5304,"Ġsaved":5305,"ĠCarter":5306,"Ġteaching":5307,"ĠTokyo":5308,"ĠCr":5309,"ĠSix":5310,"ĠPicture":5311,"Ġrecover":5312,"Ġcomedy":5313,"ree":5314,"Ġstrikes":5315,"ĠSanders":5316,"sel":5317,"Ġgraduate":5318,"Ġpending":5319,"St":5320,"Ġwarrant":5321,"Ġhonest":5322,"ĠGM":5323,"Ġnoticed":5324,"ĠGalaxy":5325,"ider":5326,"Ġproposals":5327,"Ġwore":5328,"Ġindeed":5329,"EM":5330,"ĠChannel":5331,"ances":5332,"ĠBrady":5333,"86":5334,"Ġgotten":5335,"Ġthrowing":5336,"ĠLeader":5337,"ĠVideo":5338,"71":5339,"Ġwelcomed":5340,"NEW":5341,"Ġfairly":5342,"Ġpromises":5343,"ĠSilver":5344,"Ġrape":5345,"Ġopener":5346,"ares":5347,"ĠSir":5348,"making":5349,"Ġcur":5350,"Ġrooms":5351,"73":5352,"Ġamounts":5353,"ĠIndustry":5354,"ĠDar":5355,"Ġ62":5356,"ted":5357,"Ġabroad":5358,"ĠMaybe":5359,"Ġreaders":5360,"oke":5361,"Ġpublication":5362,"ĠJean":5363,"Ġoperator":5364,"ĠHaving":5365,"ĠMil":5366,"life":5367,"Ġgenerate":5368,"ĠCraig":5369,"ĠMass":5370,"ĠBh":5371,"Ġrequested":5372,"Ġcrazy":5373,"ĠSpace":5374,"Ġcopy":5375,"Ġexport":5376,"Ġcontext":5377,"Ġbr":5378,"62":5379,"ĠRobinson":5380,"Ġcyber":5381,"ENT":5382,"BI":5383,"arg":5384,"Ġspeaker":5385,"Ġdramatic":5386,"ĠOl":5387,"ĠMill":5388,"Ġtrained":5389,"Ġediting":5390,"Ġsalary":5391,"Ġdirectors":5392,"Ġexplore":5393,"Ġlucky":5394,"Ġprominent":5395,"Ġbrothers":5396,"Ġneck":5397,"icht":5398,"ĠWatson":5399,"born":5400,"Ġproven":5401,"Ġprincipal":5402,"Ġedition":5403,"Ed":5404,"Ġswitch":5405,"maker":5406,"Ġrelative":5407,"mi":5408,"ĠBruce":5409,"ho":5410,"ĠScottish":5411,"water":5412,"ĠSport":5413,"ĠKings":5414,"ĠCollins":5415,"adi":5416,"Ġcelebrated":5417,"Ġclothes":5418,"Ġsunny":5419,"ĠCharlotte":5420,"ees":5421,"Ġscenes":5422,"ĠData":5423,"Ġwounded":5424,"Ġunusual":5425,"Ġrealized":5426,"ĠPlan":5427,"ĠTrans":5428,"ĠFC":5429,"Ġletters":5430,"Ġalerts":5431,"ĠWarren":5432,"DS":5433,"oss":5434,"pping":5435,"Ġsuspension":5436,"Ġbenchmark":5437,"ĠAcc":5438,"Ġalert":5439,"Ġpassion":5440,"ĠEst":5441,"Ġlatter":5442,"Ġstability":5443,"Ġarts":5444,"Ġpursue":5445,"ĠSeason":5446,"Ġfields":5447,"Ġmethod":5448,"63":5449,"Ġfolks":5450,"Ġexclusive":5451,"Ġcrews":5452,"Ġsessions":5453,"ĠMajor":5454,"ĠMount":5455,"Ġmap":5456,"Ġ=":5457,"Ġsituations":5458,"ĠBerlin":5459,"rey":5460,"Ġdates":5461,"Ġsheet":5462,"ĠLo":5463,"Ġfighters":5464,"ĠMart":5465,"Ġatmosphere":5466,"Ġillness":5467,"Ġcompeting":5468,"ĠChristopher":5469,"ĠRoy":5470,"mm":5471,"iano":5472,"Ġge":5473,"ĠRams":5474,"Ġconversations":5475,"ĠPa":5476,"ĠTel":5477,"Ġappreciate":5478,"78":5479,"ĠTotal":5480,"low":5481,"ĠStone":5482,"Ġopposite":5483,"Ġbarrel":5484,"Ġdevelopers":5485,"Ġexpress":5486,"Ġhighs":5487,"which":5488,"par":5489,"ĠVietnam":5490,"Ġblocks":5491,"Ġrecording":5492,"Ġadjusted":5493,"Ġret":5494,"ĠAR":5495,"Ġmilitants":5496,"Ġinnovative":5497,"ĠGhana":5498,"FR":5499,"Ġfantastic":5500,"Ġmortgage":5501,"ando":5502,"ĠLane":5503,"ises":5504,"ĠÂ":5505,"Ġhomeless":5506,"ĠKal":5507,"Ġapproached":5508,"Ġrounds":5509,"Ġmargins":5510,"ament":5511,"ĠMotor":5512,"Ġencouraging":5513,"ÂŃ":5514,"uru":5515,"Ġhandling":5516,"ĠMassachusetts":5517,"Ġplanet":5518,"ĠSpring":5519,"ĠBon":5520,"gu":5521,"Beat":5522,"Ġdrawing":5523,"ĠPhoenix":5524,"very":5525,"aid":5526,"ĠSte":5527,"ĠEntertainment":5528,"ĠRon":5529,"Ġassigned":5530,"ĠSA":5531,"News":5532,"Ġinterviews":5533,"ĠOh":5534,"media":5535,"vel":5536,"Ġpermission":5537,"Ġtransactions":5538,"Ġtraders":5539,"Ġsolo":5540,"Ġprovincial":5541,"Ġsuggesting":5542,"¡":5543,"Ġdiverse":5544,"Ġ67":5545,"Ġranks":5546,"ĠFre":5547,"Ġfavourite":5548,"Ġ63":5549,"Ġdifferences":5550,"Ġtargeting":5551,"Ġactors":5552,"Ġ76":5553,"icated":5554,"Ġcollect":5555,"akes":5556,"war":5557,"Ġcontained":5558,"ches":5559,"Ġlibrary":5560,"Ġsegments":5561,"ĠLine":5562,"ê":5563,"ual":5564,"Ġbags":5565,"Ġfactory":5566,"Ġear":5567,"Ġsomewhat":5568,"Ġrail":5569,"ĠUP":5570,"ula":5571,"ĠNiger":5572,"Ġlas":5573,"Ġimplementation":5574,"Ġemails":5575,"kel":5576,"wing":5577,"Ġadvised":5578,"--":5579,"istic":5580,"Ġdepth":5581,"Ġshoes":5582,"ĠJennifer":5583,"Ġvenue":5584,"Ġcontain":5585,"Ġhighlights":5586,"Ġcapabilities":5587,"Ġprocesses":5588,"Ġtradition":5589,"Ġcontacted":5590,"Ġproducing":5591,"Ġtrail":5592,"rem":5593,"Ġ600":5594,"Ġ68":5595,"AA":5596,"ĠBa":5597,"ĠSuch":5598,"ĠTyler":5599,"ipp":5600,"Ġsurvived":5601,"ami":5602,"ĠContinue":5603,"Ġcapture":5604,"bi":5605,"61":5606,"96":5607,"Ġthreatening":5608,"Ġkeen":5609,"dale":5610,"Ġtrailer":5611,"Ġstages":5612,"ĠGordon":5613,"Ġfinishing":5614,"Ġlegislative":5615,"Ġuseful":5616,"ĠGreek":5617,"ald":5618,"Ġgrounds":5619,"ĠDu":5620,"storms":5621,"ills":5622,"Ġexpense":5623,"Ġdetained":5624,"Today":5625,"Ġdiet":5626,"Ġwood":5627,"ĠCameron":5628,"Ġthrown":5629,"Ġcricket":5630,"Ġideal":5631,"with":5632,"Ġteammates":5633,"ours":5634,"Ġprojected":5635,"Ġpersonally":5636,"ĠBoy":5637,"rom":5638,"ĠPhilippines":5639,"win":5640,"ges":5641,"Ġcounties":5642,"ĠBaker":5643,"Ġprosecutor":5644,"Ġroof":5645,"met":5646,"Ġpartly":5647,"ĠMoon":5648,"eman":5649,"Ġfocusing":5650,"Ġfishing":5651,"than":5652,"ĠJeremy":5653,"ĠBad":5654,"ais":5655,"Ġcontrols":5656,"Ġtonnes":5657,"Ġshall":5658,"Ġ61":5659,"Ġgathering":5660,"ĠERA":5661,"Ġpresidency":5662,"Ġ85":5663,"ĠGas":5664,"Ġscenario":5665,"Ġquarters":5666,"Ġang":5667,"Ġsettled":5668,"ĠCommerce":5669,"Ġanybody":5670,"Ġgarden":5671,"ĠLibrary":5672,"Ġbet":5673,"Ġtopic":5674,"olo":5675,"Ġintense":5676,"87":5677,"Ġlinks":5678,"Ġmed":5679,"ĠAG":5680,"Ġflooding":5681,"ĠMurphy":5682,"PM":5683,"Ġfinds":5684,"Ġsensitive":5685,"pped":5686,"Ġcompletion":5687,"Ġminority":5688,"Ġvon":5689,"Ġstriking":5690,"rich":5691,"Ġbars":5692,"Ġefficient":5693,"Ġcontributions":5694,"Ġvisits":5695,"Ġattract":5696,"ĠMalaysia":5697,"ĠREL":5698,"Ġopens":5699,"Ġessentially":5700,"Ġreasonable":5701,"Ġsentiment":5702,"ĠMelbourne":5703,"Ġfitness":5704,"Ġfrequently":5705,"ĠRangers":5706,"Ġmuseum":5707,"ĠDNA":5708,"Ġcontrast":5709,"ĠAdams":5710,"ĠWin":5711,"Ġfalls":5712,"Ġimposed":5713,"250":5714,"ood":5715,"ĠRio":5716,"Ġchoices":5717,"Ġyellow":5718,"rin":5719,"ben":5720,"ĠStaff":5721,"ĠIndonesia":5722,"Ġcarries":5723,"Ġtourism":5724,"UM":5725,"ĠOrange":5726,"sell":5727,"Ġresolve":5728,"ĠMumbai":5729,"Ġpan":5730,"Ġimplement":5731,"Ġmidfielder":5732,"OP":5733,"Ġtensions":5734,"Ġ800":5735,"ĠLord":5736,"ĠLight":5737,"Ġlies":5738,"és":5739,"Ġparticipation":5740,"Ġtries":5741,"Ġsheriff":5742,"degree":5743,"Ġcongressional":5744,"Ġmode":5745,"Ġregulation":5746,"ĠJacob":5747,"ĠCrown":5748,"Ġbowl":5749,"ĠMississippi":5750,"Ġtheft":5751,"ĠKingdom":5752,"Ġresort":5753,"Ġroyal":5754,"Ġunemployment":5755,"PP":5756,"Ġnomination":5757,"ĠTR":5758,"Ġbehaviour":5759,"bank":5760,"ĠForest":5761,"WASHINGTON":5762,"ĠOthers":5763,"Ġslowly":5764,"Ġmenu":5765,"vo":5766,"ĠSy":5767,"ĠMetro":5768,"ĠLisa":5769,"Ġregistration":5770,"While":5771,"ĠJesus":5772,"Ġ250":5773,"Ġprocessing":5774,"Ġmonetary":5775,"ape":5776,"ener":5777,"ĠSystems":5778,"Ġdisappointed":5779,"Ġprint":5780,"uy":5781,"ħ":5782,"Ġdemanding":5783,"Ġincredibly":5784,"play":5785,"Ġsurveillance":5786,"ĠStandard":5787,"Ġperiods":5788,"Ġwrites":5789,"ĠLuke":5790,"ĠPalestinian":5791,"Ġwalks":5792,"Ġriding":5793,"Ġwaters":5794,"ĠSox":5795,"Ġtraveling":5796,"Ġtap":5797,"Ġorganized":5798,"Ġresource":5799,"Ġangry":5800,"Ġtiming":5801,"Ġempty":5802,"Ġmilk":5803,"Ġtherapy":5804,"ĠBrandon":5805,"mon":5806,"Ġnationwide":5807,"Ġnovel":5808,"ĠStorm":5809,"iet":5810,"ĠBre":5811,"Ġbegun":5812,"Ġdiplomatic":5813,"Ġads":5814,"ĠDC":5815,"ĠOb":5816,"ĠMontreal":5817,"ĠDown":5818,"ĠMilwaukee":5819,"Ġmeal":5820,"ĠPuerto":5821,"ĠMas":5822,"Ġjoy":5823,"Ġdeparture":5824,"ĠWright":5825,"Ġspoken":5826,"style":5827,"ĠAction":5828,"ĠComey":5829,"Ġdelivering":5830,"Ġtoll":5831,"Ġmidnight":5832,"ĠRevenue":5833,"Ġfiring":5834,"Ġstunning":5835,"Ġkicked":5836,"ĠOttawa":5837,"Ġefficiency":5838,"ĠLincoln":5839,"Ġtaste":5840,"ez":5841,"ĠWeather":5842,"ĠMorning":5843,"Ġhadn":5844,"Ġdiversity":5845,"ily":5846,"ĠAy":5847,"Ġargue":5848,"Ġerror":5849,"Ġtaught":5850,"Ġche":5851,"Ġoccasion":5852,"Ġinc":5853,"ĠOrlando":5854,"ĠOnline":5855,"Ġlegs":5856,"ĠNation":5857,"uck":5858,"Ġwidespread":5859,"ĠOcean":5860,"Ġconstantly":5861,"ĠLatin":5862,"Ġcomfort":5863,"Ġrely":5864,"uff":5865,"ĠCard":5866,"aring":5867,"Ġhumans":5868,"ĠThomson":5869,"aka":5870,"BIT":5871,"ĠReview":5872,"po":5873,"ú":5874,"Ġtrucks":5875,"Ġforecasts":5876,"view":5877,"Ġlongtime":5878,"ĠConstitution":5879,"Ġreserves":5880,"bit":5881,"Ġstressed":5882,"Ġcontribution":5883,"Ġchicken":5884,"ĠDE":5885,"Ġfat":5886,"ĠOscar":5887,"Ġcriticized":5888,"Ġtestimony":5889,"Ġapparent":5890,"Ġconstant":5891,"Ġcabinet":5892,"ĠDuke":5893,"Ġaspects":5894,"lic":5895,"ĠVol":5896,"Ġwing":5897,"Ġreb":5898,"ĠSessions":5899,"ĠSmart":5900,"car":5901,"ĠIm":5902,"Ġoperational":5903,"Ġregulators":5904,"ĠJimmy":5905,"eter":5906,"Ġnobody":5907,"ĠMarc":5908,"Ġliterally":5909,"Ġresistance":5910,"ĠKam":5911,"Ġsexually":5912,"Ġ69":5913,"uth":5914,"Ġviewed":5915,"Ġpicks":5916,"Ġdin":5917,"Ġtalented":5918,"Ġtennis":5919,"Ġstrengthen":5920,"Ġgl":5921,"ĠProtection":5922,"Ġinstalled":5923,"ways":5924,"ĠCampbell":5925,"ĠPortland":5926,"Ġintent":5927,"ĠPalace":5928,"Ġsecondary":5929,"Ġlocked":5930,"ĠPA":5931,"Ġlanded":5932,"Ġlength":5933,"Ġboosted":5934,"Ġpurchases":5935,"Ġcommand":5936,"ĠAsked":5937,"Ġspaces":5938,"Ġiconic":5939,"Ġrecommend":5940,"Ġduties":5941,"Ġseized":5942,"Ġdelayed":5943,"FA":5944,"AND":5945,"daq":5946,"Ġhiring":5947,"Ġoccur":5948,"DC":5949,"ĠMus":5950,"Ġag":5951,"Ġhopefully":5952,"ĠPenn":5953,"ards":5954,"Ġstriker":5955,"Ġrent":5956,"ĠTy":5957,"ĠBuffalo":5958,"ĠKy":5959,"Ġhike":5960,"pper":5961,"Ġ120":5962,"Ġop":5963,"Ġwheel":5964,"ĠIan":5965,"Ġchart":5966,"tt":5967,"Ġvolunteer":5968,"IG":5969,"person":5970,"ight":5971,"ĠBook":5972,"unt":5973,"ĠTechnologies":5974,"Now":5975,"Ġfavour":5976,"ĠGh":5977,"ĠQatar":5978,"ĠDutch":5979,"ĠGrant":5980,"ĠBan":5981,"rel":5982,"Ġagreements":5983,"Ġeducational":5984,"worth":5985,"ĠWard":5986,"700":5987,"Ġanymore":5988,"Ġrepair":5989,"Ġoperators":5990,"ĠLi":5991,"ots":5992,"ĠLouisiana":5993,"ĠWhether":5994,"Ġodds":5995,"Ġnoon":5996,"ĠStr":5997,"Ġfail":5998,"iser":5999,"Ġforever":6000,"Ġrecall":6001,"ĠPo":6002,"ĠHot":6003,"Ġdesigner":6004,"ido":6005,"LL":6006,"ĠControl":6007,"Ġsurvive":6008,"iam":6009,"Ġorganisation":6010,"ĠWork":6011,"Ġwider":6012,"Ġtank":6013,"work":6014,"ĠAS":6015,"Ġposting":6016,"Ġsuddenly":6017,"MC":6018,"ĠAL":6019,"ĠProfessor":6020,"ĠCoach":6021,"Ġrushed":6022,"Ġafraid":6023,"Ġactivist":6024,"that":6025,"ĠFilm":6026,"Ġbacking":6027,"Ġhousehold":6028,"Ġsignal":6029,"Ġaccurate":6030,"str":6031,"ĠThread":6032,"ĠBears":6033,"ATION":6034,"ĠAlliance":6035,"ĠMcDonald":6036,"ĠVenezuela":6037,"ogg":6038,"ĠWindows":6039,"makers":6040,"Ġutility":6041,"Ġrapidly":6042,"Ġattractive":6043,"Ġpa":6044,"ĠLarry":6045,"Ġmisconduct":6046,"Ġfreshman":6047,"Ġqualified":6048,"Ġcleared":6049,"Ġcrashed":6050,"Ġparticipating":6051,"Ġpages":6052,"Ġhighlight":6053,"Ġdialogue":6054,"ĠAlberta":6055,"Ġca":6056,"Ġwitnesses":6057,"ables":6058,"Ġfollowers":6059,"Ġensuring":6060,"Ġpromoting":6061,"Ġsearching":6062,"Ġremote":6063,"Ġclash":6064,"Ġfirefighters":6065,"Ġteen":6066,"ĠPlace":6067,"ĠNote":6068,"Ġregardless":6069,"ult":6070,"oney":6071,"ander":6072,"ional":6073,"ining":6074,"Ġdemanded":6075,"ĠCommunications":6076,"Ġconsideration":6077,"TC":6078,"ĠSoutheast":6079,"aga":6080,"ĠGarden":6081,"inger":6082,"ht":6083,"Ġbranch":6084,"Ġmouth":6085,"Ġaudio":6086,"Ġraw":6087,"Ġcoordinator":6088,"Ġexact":6089,"ĠHan":6090,"Ġdelays":6091,"ĠWal":6092,"ĠWells":6093,"Ġng":6094,"Ġhandful":6095,"Ġgirlfriend":6096,"Ġtypical":6097,"ĠWayne":6098,"ĠFranklin":6099,"Ġconstitutional":6100,"ĠChance":6101,"Ġblamed":6102,"rim":6103,"Ġpreliminary":6104,"Ġlie":6105,"da":6106,"ĠCapitol":6107,"Ġroutine":6108,"ĠNASA":6109,"Ġtre":6110,"ĠGolf":6111,"Ġsight":6112,"ĠDer":6113,"Ġreserve":6114,"150":6115,"Ġspeculation":6116,"Ġcompetitors":6117,"ĠMacron":6118,"ony":6119,"Ġovertime":6120,"Ġ71":6121,"Ġdepending":6122,"ĠWarner":6123,"Ġaccusations":6124,"ius":6125,"Ġpredicted":6126,"ĠCharlie":6127,"Ġeverywhere":6128,"Ġcable":6129,"ĠSaint":6130,"ĠRegion":6131,"Ġhero":6132,"ĠEmb":6133,"Ġkinds":6134,"Ġstarter":6135,"Ġsolve":6136,"ĠGuard":6137,"Ġloves":6138,"ĠDouglas":6139,"Ġfunded":6140,"ĠBrent":6141,"ĠAnyone":6142,"Ġsubstantial":6143,"ĠMarine":6144,"ĠMichelle":6145,"Ġcelebrating":6146,"Ġoffset":6147,"Ġbutton":6148,"gg":6149,"Ġmedicine":6150,"uri":6151,"Ġsomewhere":6152,"PD":6153,"Ġmon":6154,"Ġfires":6155,"final":6156,"oth":6157,"ined":6158,"Ġunderway":6159,"Ġmistakes":6160,"Ġgrateful":6161,"Ġcheap":6162,"È":6163,"Ġ95":6164,"Ġviolations":6165,"arr":6166,"Ġsurprising":6167,"Ġob":6168,"ĠNATO":6169,"Ġcontroversy":6170,"ĠSweden":6171,"Ġfuneral":6172,"Ġreviews":6173,"Ġpromotion":6174,"TY":6175,"Ġliberal":6176,"Ġpromising":6177,"ĠSP":6178,"How":6179,"Ġmemories":6180,"Ġbreast":6181,"zi":6182,"ights":6183,"Ġpattern":6184,"Ġoutdoor":6185,"ĠMu":6186,"Ġrush":6187,"ĠTheresa":6188,"ĠPol":6189,"Ġdescribe":6190,"ĠBand":6191,"ĠStewart":6192,"Ġ1999":6193,"ĠRaiders":6194,"mp":6195,"Ġprocedures":6196,"Ġplot":6197,"Ġhire":6198,"used":6199,"Ġ1970":6200,"Ġpicking":6201,"ĠSim":6202,"Ġregard":6203,"inal":6204,"backs":6205,"ĠHard":6206,"ĠLow":6207,"ĠAc":6208,"Is":6209,"Ġguarantee":6210,"ĠGiven":6211,"Ġbeta":6212,"ĠTre":6213,"Ġtrans":6214,"Ġretailer":6215,"Ġpurposes":6216,"ĠHol":6217,"Ġenjoying":6218,"Ġbrown":6219,"ĠPerry":6220,"Ġplea":6221,"MS":6222,"ĠDakota":6223,"ĠParker":6224,"Ġcommit":6225,"ĠLawrence":6226,"ĠMorris":6227,"ended":6228,"Ġvirtual":6229,"ÃĹ":6230,"Ġfruit":6231,"84":6232,"ĠHas":6233,"ishing":6234,"Ġdominated":6235,"ĠFA":6236,"Ġchannels":6237,"Ġunderstood":6238,"Ġcitizen":6239,"Ġchecks":6240,"ĠKenya":6241,"Ġdisabled":6242,"SD":6243,"Ġprotecting":6244,"Ġtweets":6245,"Ġsparked":6246,"ĠCO":6247,"§":6248,"ori":6249,"ĠGDP":6250,"ĠSer":6251,"ĠVisit":6252,"ĠMS":6253,"Ġbarely":6254,"Ġsand":6255,"Ġap":6256,"aging":6257,"Ġrel":6258,"ĠPerhaps":6259,"ĠMourinho":6260,"ĠJets":6261,"Ġdisclosure":6262,"Ġhighlighted":6263,"Ġimplemented":6264,"Ġcompliance":6265,"ĠAB":6266,"ĠAssistant":6267,"ĠCape":6268,"Ġfunny":6269,"Ġleverage":6270,"Ġmachines":6271,"Ġranging":6272,"Ġfastest":6273,"ĠRoberts":6274,"ĠPolicy":6275,"gar":6276,"Ġcollapse":6277,"ĠThrough":6278,"Ġrobbery":6279,"ĠHay":6280,"Ġelite":6281,"ĠDigital":6282,"ĠFun":6283,"ĠAlan":6284,"ement":6285,"Ġmit":6286,"Ġspin":6287,"Ġlistening":6288,"ĠDoug":6289,"ĠSaints":6290,"Ġinterior":6291,"Ġenhance":6292,"ĠCardinals":6293,"ever":6294,"Ġrobust":6295,"Ġinform":6296,"Ġsuffer":6297,"book":6298,"ĠMuslims":6299,"Ġagriculture":6300,"Ġkm":6301,"Ġdivers":6302,"ñ":6303,"ĠReg":6304,"Ġequivalent":6305,"Ġcraft":6306,"Ġsettle":6307,"Ġcontains":6308,"ĠMack":6309,"ĠDis":6310,"ĠFore":6311,"ĠSudan":6312,"ĠMail":6313,"ĠBrooklyn":6314,"izer":6315,"bn":6316,"Ġhundred":6317,"Ġexhibition":6318,"ĠHave":6319,"vin":6320,"Ġcivilians":6321,"ĠCincinnati":6322,"Some":6323,"ĠSE":6324,"Ġbat":6325,"ĠIns":6326,"Ġcalm":6327,"Ġtone":6328,"Ġnormally":6329,"Ġseeks":6330,"ĠAss":6331,"Ġmembership":6332,"Ġannually":6333,"Ġemployers":6334,"CO":6335,"Ġcomplicated":6336,"Ġheadlines":6337,"ĠLabor":6338,"Ġlifestyle":6339,"ĠRen":6340,"ĠRich":6341,"cent":6342,"ude":6343,"Ġawesome":6344,"Ġpaint":6345,"Ġrolling":6346,"Ġwalls":6347,"Ġlab":6348,"Ġtourists":6349,"care":6350,"Ġgear":6351,"izz":6352,"Ġcream":6353,"ĠTro":6354,"ices":6355,"Ġpack":6356,"Ġdiseases":6357,"ĠSpeaker":6358,"ĠOfficers":6359,"Ġsky":6360,"83":6361,"ĠBE":6362,"Ġcategories":6363,"Ġindicate":6364,"Ġru":6365,"ĠSony":6366,"ĠDun":6367,"ocks":6368,"Ġconcrete":6369,"ĠMadison":6370,"ĠSab":6371,"IV":6372,"Ġobserved":6373,"ria":6374,"Ġinterim":6375,"Ġencounter":6376,"ista":6377,"Ġanger":6378,"Ġrapid":6379,"mail":6380,"Ġdestination":6381,"ĩ":6382,"Ġbreaks":6383,"rell":6384,"ĠChase":6385,"Ġattorneys":6386,"Ġrolled":6387,"ĠSprings":6388,"ĠVillage":6389,"TO":6390,"HS":6391,"Ġcampaigns":6392,"ologist":6393,"ĠTax":6394,"ĠIII":6395,"Ġteach":6396,"Ġprovision":6397,"Ġrem":6398,"Ġshirt":6399,"Ġdeployed":6400,"Ġguidelines":6401,"Ġav":6402,"zer":6403,"Ġrushing":6404,"94":6405,"place":6406,"Man":6407,"Ġdivided":6408,"ĠGun":6409,"Ġwindows":6410,"Ġcomponents":6411,"aba":6412,"ĠSwitzerland":6413,"election":6414,"ĠTampa":6415,"ĠAri":6416,"ás":6417,"Ġhighway":6418,"Ġacres":6419,"Ġcrown":6420,"known":6421,"Ġinquiry":6422,"url":6423,"Ġexpertise":6424,"Ġpraised":6425,"yer":6426,"Ġconclusion":6427,"Ġabortion":6428,"Ġlady":6429,"Ġtribute":6430,"Ġunveiled":6431,"Ġbeaten":6432,"TE":6433,"ĠMot":6434,"unk":6435,"Ġtriple":6436,"Ġforcing":6437,"ĠTickets":6438,"uit":6439,"Ġiron":6440,"Ġscientific":6441,"ĠIP":6442,"Ġdiagnosed":6443,"Ġocean":6444,"wide":6445,"ĠCowboys":6446,"LC":6447,"Ġmethods":6448,"ĠFind":6449,"ĠDean":6450,"Ġfundamental":6451,"ĠGill":6452,"Ġfeelings":6453,"IO":6454,"hu":6455,"Ġfeedback":6456,"ote":6457,"Ġduo":6458,"fully":6459,"get":6460,"Ġproof":6461,"story":6462,"Ġlongest":6463,"Ġshops":6464,"ĠJong":6465,"ĠCro":6466,"ĠHawaii":6467,"91":6468,"ĠJake":6469,"ĠSusan":6470,"Ġsubmit":6471,"rav":6472,"Ġmodest":6473,"Ġlit":6474,"Ġattempting":6475,"Ġsits":6476,"Ġaddressing":6477,"93":6478,"ĠBi":6479,"Ġlying":6480,"ĠOrganization":6481,"ĠOak":6482,"oli":6483,"Ġfatal":6484,"Ġmountain":6485,"val":6486,"lu":6487,"ĠMaine":6488,"Ġcharging":6489,"Ġresigned":6490,"illo":6491,"Ġrecommendation":6492,"party":6493,"ĠWeb":6494,"ĠPanthers":6495,"Ġnoise":6496,"ĠBrussels":6497,"awa":6498,"Ġambassador":6499,"Ġaccessible":6500,"ĠCalgary":6501,"idd":6502,"ĠAirlines":6503,"gr":6504,"Ġnu":6505,"roy":6506,"ĠMars":6507,"ĠPoland":6508,"ĠJerry":6509,"ados":6510,"ĠRico":6511,"ĠMir":6512,"ĠFin":6513,"ious":6514,"Ġpacked":6515,"Ġinsider":6516,"President":6517,"ĠBull":6518,"ĠYemen":6519,"ĠConnecticut":6520,"Ġ73":6521,"Ġdepartments":6522,"Ġorganic":6523,"ĠSummer":6524,"ĠBet":6525,"ste":6526,"zo":6527,"rat":6528,"Ġalliance":6529,"Ġintervention":6530,"wan":6531,"ĠOR":6532,"Ġdefined":6533,"ĠÃł":6534,"ĠChiefs":6535,"Ġknocked":6536,"ared":6537,"Ġholes":6538,"Ġpulling":6539,"ĠTodd":6540,"ĠJamie":6541,"ĠSher":6542,"Ġsignature":6543,"ĠSur":6544,"Ġgym":6545,"ĠVladimir":6546,"ĠThailand":6547,"Ġgaming":6548,"Ġsaving":6549,"ceive":6550,"82":6551,"ĠBern":6552,"ĠDid":6553,"Ġhardware":6554,"ished":6555,"Ġconspiracy":6556,"ANS":6557,"ĠIntelligence":6558,"Ġassembly":6559,"Ġ101":6560,"Ġconcise":6561,"ĠManhattan":6562,"Ġbelief":6563,"Ġsurge":6564,"Ġdeserve":6565,"Ġconsistently":6566,"ĠNor":6567,"okes":6568,"ðŁ":6569,"ME":6570,"ĠAsset":6571,"Ġsubstance":6572,"Ġprefer":6573,"Ġburning":6574,"ĠNik":6575,"ook":6576,"ĠPinterest":6577,"Ġboyfriend":6578,"ĠHal":6579,"ĠMerkel":6580,"Ġintroduce":6581,"ĠLinkedIn":6582,"ĠFull":6583,"ĠFarm":6584,"Ġchildhood":6585,"ĠTransportation":6586,"Ġterrible":6587,"du":6588,"Ġintention":6589,"Ġseemingly":6590,"elle":6591,"Ġfoods":6592,"Ġtitled":6593,"Ġdual":6594,"Ġimport":6595,"Ġdeveloper":6596,"UL":6597,"ington":6598,"ĠDelta":6599,"?'":6600,"iness":6601,"Ġquit":6602,"ĠGarcia":6603,"ĠSri":6604,"Ġhip":6605,"ĠBrazilian":6606,"elt":6607,"ively":6608,"Ġstructures":6609,"Ġlabour":6610,"Ġneighbors":6611,"Ġtill":6612,"Ġsoil":6613,"Ġdropping":6614,"Ġnominee":6615,"Ġmeets":6616,"92":6617,"rant":6618,"isa":6619,"Ġluck":6620,"aa":6621,"jet":6622,"ĠTor":6623,"ĠCrime":6624,"Ġlane":6625,"Ġflu":6626,"Ġlaunching":6627,"ĠAutom":6628,"aks":6629,"Ġuniversities":6630,"Ġpollution":6631,"ĠAdvis":6632,"ĠMall":6633,"ls":6634,"Ġdeeper":6635,"Ġrepeated":6636,"Ġmeanwhile":6637,"Ġchip":6638,"Ġoutlets":6639,"Ġliked":6640,"Ġsal":6641,"Ġwelfare":6642,"ago":6643,"Ġmakers":6644,"ving":6645,"fer":6646,"Ġovercome":6647,"mb":6648,"Ġshocked":6649,"akers":6650,"Ġnonprofit":6651,"Ġdonated":6652,"eral":6653,"Ġresume":6654,"Ġlogo":6655,"Ġsubscription":6656,"Ġ74":6657,"ela":6658,"Ġaspect":6659,"html":6660,"Ġsorry":6661,"Ġupgrade":6662,"Ġstance":6663,"Ġfr":6664,"Ġpapers":6665,"Ġattacking":6666,"Ġmeaningful":6667,"81":6668,"ĠWeinstein":6669,"Ġcreates":6670,"Ġhonour":6671,"ĠReply":6672,"oph":6673,"Ġmarch":6674,"Ġsmile":6675,"Ġcomparison":6676,"will":6677,"ĠSanchez":6678,"Ġvoter":6679,"Ġtheory":6680,"Ġequally":6681,"ĠRoger":6682,"Ġperfectly":6683,"Ġlanding":6684,"Ġbillions":6685,"ĠBloomberg":6686,"Ġpermit":6687,"Ġfinals":6688,"Ġracial":6689,"Ġpregnancy":6690,"iled":6691,"ĠFederation":6692,"Ġforest":6693,"Ġtag":6694,"aul":6695,"Ġdrinks":6696,"Ġ(\"":6697,"ĠMobile":6698,"Ġtouched":6699,"Ġclock":6700,"Ġreg":6701,"Ġasylum":6702,"igan":6703,"Ġsenator":6704,"Ġ99":6705,"ĠKumar":6706,"Ġskill":6707,"Ġ1998":6708,"pa":6709,"ĠAf":6710,"Ġmood":6711,"ston":6712,"Ġhang":6713,"ĠMPs":6714,"Please":6715,"ĠEve":6716,"Ġdocumentary":6717,"Ġpersonality":6718,"ĠCast":6719,"Ġdiscount":6720,"bing":6721,"ĠBoeing":6722,"Ġdepend":6723,"Ġcrossing":6724,"EX":6725,"Ġsucceed":6726,"Ġhumanitarian":6727,"ĠMuhammad":6728,"Ġwages":6729,"Ġcolumn":6730,"Ġexternal":6731,"Ġstatistics":6732,"ĠTODAY":6733,"Ġtrips":6734,"Ġta":6735,"Ġpenalties":6736,"Ġwriters":6737,"Ġshipping":6738,"ĠIndians":6739,"Ġsalt":6740,"ĠIndustrial":6741,"ĠYankees":6742,"ĠDen":6743,"Ġrough":6744,"Ġbarrels":6745,"ĠHor":6746,"bert":6747,"ĠDep":6748,"Ġresign":6749,"97":6750,"Ġballs":6751,"ĠJun":6752,"ĠBab":6753,"Ġassociate":6754,"Ġstring":6755,"Ġhub":6756,"Ġorgan":6757,"ĠMarshall":6758,"ĠFIFA":6759,"ĠMun":6760,"ency":6761,"research":6762,"Ġpeers":6763,"Ġtall":6764,"ĠGoldman":6765,"Don":6766,"Ġparade":6767,"Ġparks":6768,"Ġdet":6769,"Ġdisappointing":6770,"Ġreflects":6771,"ĠLakers":6772,"Ġfiles":6773,"Ġrelatives":6774,"ĠUSD":6775,"ĠArticle":6776,"Ġcustom":6777,"ĠCarlos":6778,"Ġtracking":6779,"Ġmaintaining":6780,"ĠCur":6781,"ardo":6782,"ĠSkip":6783,"Ġattitude":6784,"Just":6785,"Ġinstitution":6786,"Ġnarrow":6787,"Ġsnap":6788,"Ġenterprise":6789,"Ġdrives":6790,"Ġ77":6791,"Ġcrop":6792,"Ġvirus":6793,"Ġcelebrity":6794,"Ġeconomies":6795,"ued":6796,"Ġsum":6797,"ĠDubai":6798,"ĠInsurance":6799,"Ĺ":6800,"ury":6801,"ĠUnfortunately":6802,"Ġclosure":6803,"ota":6804,"ĠPhilip":6805,"oms":6806,"Ġinvestigated":6807,"Ġgenerations":6808,"ĠETF":6809,"ĠKeith":6810,"ĠLater":6811,"isk":6812,"Ġpreferred":6813,"Ġdefault":6814,"Ġtowns":6815,"ĠRod":6816,"ĠDie":6817,"Ġintegrated":6818,"Ġacquiring":6819,"Ġvoices":6820,"Ġser":6821,"Ġpresents":6822,"ĠBR":6823,"ĠEmergency":6824,"Ġreligion":6825,"HA":6826,"Ġresponding":6827,"ĠThings":6828,"Ġbeef":6829,"ĠWithout":6830,"urd":6831,"ĠCarl":6832,"Ġadministrative":6833,"ĠWhich":6834,"Ġchallenged":6835,"Ġcooking":6836,"ivid":6837,"ĠFer":6838,"Ġtremendous":6839,"ĠTerry":6840,"iri":6841,"CS":6842,"ĠJunior":6843,"ĠReddit":6844,"Ġtea":6845,"Ġaccounting":6846,"lan":6847,"Ġdetention":6848,"Ġreplied":6849,"SI":6850,"ĠHel":6851,"ns":6852,"ĠProf":6853,"Ġramp":6854,"ĠConservative":6855,"Ġattendance":6856,"Ġspecialist":6857,"ĠFinal":6858,"Ġadvertisement":6859,"Ġacquire":6860,"ĠWhatsApp":6861,"Ġworkforce":6862,"ĠCalif":6863,"Ġspeakers":6864,"ĠEPA":6865,"Ġconviction":6866,"hire":6867,"ĠFisher":6868,"ĠIntel":6869,"Ġbin":6870,"ĠWas":6871,"Ġearth":6872,"vi":6873,"Ġhurricane":6874,"Ġholidays":6875,"Ġassume":6876,"Ġinvolve":6877,"Ġdynamic":6878,"ĠGre":6879,"Ġitem":6880,"Ġpound":6881,"Ġanxiety":6882,"ĠPrint":6883,"rop":6884,"Ġautomatically":6885,"Ġdiscrimination":6886,"ĠLam":6887,"ĠColl":6888,"Ġimpressed":6889,"Ġinvolves":6890,"ĠLes":6891,"ĠTri":6892,"ĠLook":6893,"ĠiOS":6894,"Ġgrab":6895,"ĠAngel":6896,"Ġstops":6897,"ĠPay":6898,"ĠECB":6899,"Ġbunch":6900,"Ġletting":6901,"ele":6902,"ĠAdditionally":6903,"Ġboards":6904,"NC":6905,"Ġtragedy":6906,"Ġpink":6907,"Ġgonna":6908,"ones":6909,"Ġrev":6910,"ĠIndependent":6911,"ĠCambridge":6912,"ĠPence":6913,"Ġprosecution":6914,"Ġdeputies":6915,"ĠAhmed":6916,"Ġlows":6917,"ĠAmy":6918,"ĠBuilding":6919,"mark":6920,"Ġsmooth":6921,"Ġsole":6922,"Ġwanting":6923,"ĠHeart":6924,"Ġobtain":6925,"ĠBus":6926,"Ġexchanges":6927,"friendly":6928,"Ġlabel":6929,"elect":6930,"ĠCompanies":6931,"owing":6932,"ĠCB":6933,"RI":6934,"ĠMaster":6935,"Ġliquid":6936,"ĠDanny":6937,"Ġproceeds":6938,"ĠLaura":6939,"card":6940,"Ġtears":6941,"Ġexploration":6942,"Ġdepression":6943,"ken":6944,"ĠFe":6945,"Ġlending":6946,"ĠYouth":6947,"ality":6948,"NS":6949,"Ġmoon":6950,"ĠTaiwan":6951,"Ġstruggles":6952,"Ġdiscovery":6953,"Ġqualify":6954,"Ġwireless":6955,"alia":6956,"Ġwitnessed":6957,"Ġheight":6958,"ĠGuy":6959,"left":6960,"KE":6961,"Ġfoul":6962,"ĠMohammed":6963,"Ġgrass":6964,"ĠNon":6965,"Ġswim":6966,"Ġbrilliant":6967,"you":6968,"ĠFlynn":6969,"Ġsinging":6970,"eria":6971,"UT":6972,"ĠMcCain":6973,"ĠSep":6974,"ĠWars":6975,"Ġburden":6976,"Ġpas":6977,"Ġabandoned":6978,"Ġint":6979,"ĠTurner":6980,"Ġcollective":6981,"ĠEnvironmental":6982,"ĠStudents":6983,"Ġofferings":6984,"Ġresignation":6985,"Ġexplosion":6986,"ĠKoh":6987,"ager":6988,"Ġthrows":6989,"Ġasks":6990,"light":6991,"Ġanyway":6992,"Ġyard":6993,"Ġcarrier":6994,"Ġwaves":6995,"backed":6996,"TR":6997,"oud":6998,"Ġbreach":6999,"Ġdated":7000,"Ġdressed":7001,"ĠDodgers":7002,"oles":7003,"Ġ78":7004,"Ġreads":7005,"Ġpredict":7006,"ĠJerusalem":7007,"ĠPT":7008,"Ġcrack":7009,"yan":7010,"Ġnights":7011,"eline":7012,"Ġconvinced":7013,"Ġlock":7014,"Ġcarefully":7015,"ĠMercedes":7016,"Ġultimate":7017,"Ġdist":7018,"Ġslight":7019,"ĠEdwards":7020,"Ġswing":7021,"iling":7022,"Ġknife":7023,"ĠNashville":7024,"IF":7025,"inder":7026,"udd":7027,"Ġsenators":7028,"ĠFurther":7029,"ĠXi":7030,"Ġstr":7031,"ĠOd":7032,"days":7033,"Ġcomm":7034,"Ġverdict":7035,"Ġconfirmation":7036,"king":7037,"ĠCS":7038,"Ġadvocates":7039,"Ġpride":7040,"Ġmemorial":7041,"ams":7042,"erman":7043,"Ġteenager":7044,"ĠNeil":7045,"uts":7046,"Ġsoul":7047,"see":7048,"post":7049,"Ġchest":7050,"fire":7051,"ĠLynch":7052,"Ġpeaceful":7053,"OND":7054,"ĠIndustries":7055,"ĠJuan":7056,"Ġrestore":7057,"Ġreliable":7058,"ming":7059,"agan":7060,"Source":7061,"ĠCabinet":7062,"Ġremarkable":7063,"ĠTrudeau":7064,"ĠEs":7065,"Ġintegrity":7066,"ove":7067,"fe":7068,"Ġproceedings":7069,"Ġconnections":7070,"Ġunprecedented":7071,"ĠGlen":7072,"ux":7073,"Ġearning":7074,"Ġingredients":7075,"Ġnominated":7076,"ĠBangladesh":7077,"made":7078,"Ġlessons":7079,"Ġbreakfast":7080,"ĠRelations":7081,"Ġloose":7082,"Al":7083,"Ġupgraded":7084,"ral":7085,"ĠPage":7086,"oto":7087,"ĠQueensland":7088,"Ġprocedure":7089,"ĠSmall":7090,"Ġrespective":7091,"Ġpictured":7092,"ĠBas":7093,"Ġpreparation":7094,"ĠMyanmar":7095,"Ġdonation":7096,"Ġvisible":7097,"iest":7098,"ĠBroadway":7099,"rick":7100,"ĠSchools":7101,"Ġarrests":7102,"ĠJessica":7103,"ĠBengal":7104,"Ġhell":7105,"Ġannouncing":7106,"Ġmail":7107,"ĠMcG":7108,"two":7109,"rest":7110,"OD":7111,"ĠBradley":7112,"Ġdoubled":7113,"Ġpledged":7114,"Ġcomeback":7115,"Ġextraordinary":7116,"Ġslide":7117,"Ġassess":7118,"Ġagricultural":7119,"ĠKay":7120,"Ġvendors":7121,"Ġnarrative":7122,"Ġreviewed":7123,"ĠPass":7124,"Ġinspiration":7125,"ĠHunter":7126,"Ġcalendar":7127,"ĠDiamond":7128,"Ġremoval":7129,"ners":7130,"ĠKap":7131,"Ġconsent":7132,"Ġvisual":7133,"Ġcheese":7134,"ĠTher":7135,"ĠFR":7136,"ĠShanghai":7137,"iah":7138,"ĠCole":7139,"AK":7140,"Ġranking":7141,"Ġcook":7142,"Ġhalftime":7143,"ĠStars":7144,"Ġroutes":7145,"aim":7146,"Ġestablishment":7147,"ĠMug":7148,"Ġsurvivors":7149,"urg":7150,"ĠBrett":7151,"Ġunexpected":7152,"ained":7153,"Ġrarely":7154,"ĠGall":7155,"Ġadvocate":7156,"ĠNad":7157,"Ġ911":7158,"Ġracist":7159,"erer":7160,"ĠRev":7161,"ĠSection":7162,"Ġhelpful":7163,"CT":7164,"agg":7165,"Ġgovernance":7166,"Ġfelony":7167,"Ġoptimistic":7168,"Ġelectoral":7169,"EG":7170,"town":7171,"Ġdaughters":7172,"Ġanswered":7173,"Ġthin":7174,"ĠClassic":7175,"Ġshareholder":7176,"ĠBlake":7177,"ĠFla":7178,"Ġparliamentary":7179,"dy":7180,"Ġcommented":7181,"Ġtri":7182,"Ġglobe":7183,"Ġmandate":7184,"Ġslipped":7185,"ĠTower":7186,"Ġoperated":7187,"gers":7188,"Ġassured":7189,"ĠMartinez":7190,"Ġdesigns":7191,"ĠModel":7192,"Ġstakeholders":7193,"Ġdefended":7194,"Ġseniors":7195,"Ġvacation":7196,"Ġglobally":7197,"ump":7198,"Not":7199,"Ġclip":7200,"Ġarticles":7201,"BR":7202,"km":7203,"ĠFront":7204,"PL":7205,"Ġadoption":7206,"Ġsudden":7207,"Ġframework":7208,"Ġhanging":7209,"gl":7210,"ĠSel":7211,"Ġmoderate":7212,"Ġreverse":7213,"income":7214,"cor":7215,"ĠGB":7216,"Ġphysically":7217,"Ġtransparency":7218,"ĠElectric":7219,"Ġrefugee":7220,"profile":7221,"iva":7222,"ately":7223,"ĠAC":7224,"Ġtransferred":7225,"Ġaffair":7226,"ĠAlaska":7227,"oria":7228,"ĠChange":7229,"Ġrepeat":7230,"Ġscreening":7231,"ender":7232,"ĠCas":7233,"ĠDav":7234,"Ġfocuses":7235,"Ġcommissioner":7236,"Ġupside":7237,"ĠKeep":7238,"ĠBlues":7239,"ently":7240,"Ġaut":7241,"Ġexperiencing":7242,"aman":7243,"Ġapprove":7244,"Ġmile":7245,"Ġcheaper":7246,"ĠWind":7247,"ĠStore":7248,"Ġgrabbed":7249,"Ġsons":7250,"Ġfighter":7251,"Ġum":7252,"ĠBased":7253,"don":7254,"Ġconstitution":7255,"finals":7256,"act":7257,"¢":7258,"Ġmill":7259,"Ġorganisations":7260,"ĠToyota":7261,"Ġyuan":7262,"Ġterrorists":7263,"Ġforth":7264,"Ġavailability":7265,"Ġentrance":7266,"Ġvolumes":7267,"Ġmult":7268,"plus":7269,"ĠColumbus":7270,"ĠSummit":7271,"Ġbabies":7272,"ĠMur":7273,"ĠGray":7274,"ĠChar":7275,"ĠButler":7276,"Ġpose":7277,"ĠNatural":7278,"ĠAtt":7279,"Ġdecrease":7280,"Ġtens":7281,"kt":7282,"Ġminds":7283,"Ġimpacted":7284,"Ġchapter":7285,"ĠOp":7286,"ĠHarrison":7287,"ĠRodriguez":7288,"Ġethnic":7289,"Ġtravelling":7290,"ĠBond":7291,"ader":7292,"core":7293,"Ġgallery":7294,"founder":7295,"ĠVill":7296,"Ġdecent":7297,"ĠHistory":7298,"ĠInt":7299,"ĠNa":7300,"ĠHad":7301,"Ġmainstream":7302,"ĠTs":7303,"Ġbottle":7304,"sen":7305,"Ġrecession":7306,"Ġsophomore":7307,"Ġsilence":7308,"cc":7309,"Ġqualifying":7310,"Ġcomplained":7311,"ĠRad":7312,"Ġactively":7313,"Ġbacks":7314,"ĠMusk":7315,"Ġcareful":7316,"Ġmeals":7317,"ĠDor":7318,"Ġmess":7319,"ĠBelgium":7320,"Ġke":7321,"ĠLopez":7322,"Ġbow":7323,"Ġhelicopter":7324,"was":7325,"Ġstone":7326,"kins":7327,"Ġunlike":7328,"Ġcollision":7329,"ĠAlt":7330,"HP":7331,"ĠMason":7332,"has":7333,"Ġclimbed":7334,"Ġindication":7335,"Ġhotels":7336,"Ġloud":7337,"ĠMilan":7338,"kes":7339,"Ġbadly":7340,"Ġtrials":7341,"Ġimpacts":7342,"ĠJane":7343,"Ġcrossed":7344,"Ġdiscussing":7345,"ĠSM":7346,"Ġpopularity":7347,"ĠWant":7348,"fall":7349,"Ġartificial":7350,"ĠBu":7351,"akh":7352,"Ġdominant":7353,"gov":7354,"Ġpremier":7355,"Ġexecution":7356,"gate":7357,"Ġswimming":7358,"Ġchat":7359,"Ġdevastating":7360,"acking":7361,"Ġreception":7362,"urt":7363,"Ġtheater":7364,"Ġgather":7365,"Ġtear":7366,"uro":7367,"Ġdemocratic":7368,"Ġrebels":7369,"Ġlifetime":7370,"Ġradical":7371,"uan":7372,"Ġtechniques":7373,"ache":7374,"ior":7375,"Ġcamps":7376,"Ġtelephone":7377,"ĠDublin":7378,"ĠBrand":7379,"ĠMarcus":7380,"aun":7381,"ĠRec":7382,"Ġ82":7383,"ban":7384,"Ġsafely":7385,"aku":7386,"aki":7387,"Ġbankruptcy":7388,"FF":7389,"Ġformat":7390,"Ġattached":7391,"ĠFame":7392,"ĠEdward":7393,"Ġmerger":7394,"ĠRepresentatives":7395,"izes":7396,"Ġhidden":7397,"Ġval":7398,"zz":7399,"Ġexcess":7400,"Ġscope":7401,"Ġdivorce":7402,"Ġburn":7403,"Ġrequirement":7404,"BB":7405,"ĠHand":7406,"Ġcons":7407,"Ġrisen":7408,"Ġtwitter":7409,"Ġoffseason":7410,"ĠSometimes":7411,"ĠInf":7412,"ĠAng":7413,"uer":7414,"report":7415,"Ġdreams":7416,"Ġ700":7417,"ips":7418,"ĠDream":7419,"Ġgifts":7420,"Ġsomehow":7421,"ĠTur":7422,"ĠRachel":7423,"can":7424,"Ġlog":7425,"ĠMedicaid":7426,"Ġles":7427,"Ġtired":7428,"ĠArkansas":7429,"Ġliquidity":7430,"ĠPhillips":7431,"ĠBTC":7432,"Ġhide":7433,"Ġpun":7434,"ĠRun":7435,"lyn":7436,"ĠUC":7437,"ĠDesign":7438,"ĠDev":7439,"Ġvaluation":7440,"Ġreveals":7441,"ĠChild":7442,"other":7443,"Ġposed":7444,"lee":7445,"Ġships":7446,"ĠTrue":7447,"Ġdescribes":7448,"Ġrunner":7449,"bro":7450,"Ġankle":7451,"Ġod":7452,"ĠAnnual":7453,"CL":7454,"Ġoverhaul":7455,"ned":7456,"Ġbold":7457,"Ġmo":7458,"ĠFalls":7459,"Ġemployed":7460,"ĠGro":7461,"Ġflash":7462,"ĠTD":7463,"Ġnervous":7464,"Ġintegration":7465,"Ġsmartphones":7466,"Ġmovements":7467,"nie":7468,"ition":7469,"ĠThird":7470,"Ģ":7471,"Ġmetres":7472,"Ġeconomist":7473,"omp":7474,"Ġteens":7475,"Ġeveryday":7476,"Ġinterviewed":7477,"Ġbriefly":7478,"],":7479,"uke":7480,"ĠFOX":7481,"Ġunderlying":7482,"ĠLuc":7483,"Ġcourses":7484,"ss":7485,"amed":7486,"°":7487,"ju":7488,"ĠBanks":7489,"Ġoutfit":7490,"illing":7491,"Ġtrafficking":7492,"Ġurging":7493,"Ġbelt":7494,"Ġrid":7495,"CP":7496,"Ġelderly":7497,"ĠGrowth":7498,"án":7499,"ĠSn":7500,"Ġsurrounded":7501,"Ġsisters":7502,"ĠIslam":7503,"Ġsynd":7504,"ĠCosta":7505,"di":7506,"ĠKl":7507,"Ġmanufacturer":7508,"holders":7509,"Ġelement":7510,"Ġload":7511,"Ġbooked":7512,"Ġaccompanied":7513,"ĠChamber":7514,"Ġbriefing":7515,"Oh":7516,"imi":7517,"ĠDefence":7518,"ĠCurrently":7519,"aking":7520,"Ġhandled":7521,"ĠCD":7522,"ĠBenjamin":7523,"Ġpocket":7524,"ĠKashmir":7525,"Ġlighting":7526,"aps":7527,"Ġ1997":7528,"ech":7529,"Ġaddiction":7530,"Ġbases":7531,"Ġpriorities":7532,"Ġhardly":7533,"ĠQuebec":7534,"ĠEarn":7535,"IES":7536,"ĠZach":7537,"ĠAlong":7538,"MI":7539,"Ġins":7540,"ĠRogers":7541,"ĠKan":7542,"ĠFuture":7543,"Ġtriggered":7544,"ĠUnit":7545,"Ġweighed":7546,"Ġpointing":7547,"Ġchocolate":7548,"ĠBrowns":7549,"ĠISIS":7550,"Ġgoalkeeper":7551,"Ġsaves":7552,"ĠAndre":7553,"burn":7554,"ĠCont":7555,"ĠNetherlands":7556,"Ġpolitically":7557,"ĠAshley":7558,"ĠWhit":7559,"aded":7560,"PH":7561,"Ġborders":7562,"ORE":7563,"Ġally":7564,"Trump":7565,"istan":7566,"ĠHunt":7567,"ĠCancer":7568,"ĠGrace":7569,"ĠTottenham":7570,"Ġ1960":7571,"ĠMarg":7572,"ĠBryan":7573,"ĠAgain":7574,"acing":7575,"Ġarguments":7576,"ĠSouthwest":7577,"Ġvocal":7578,"Ġjudgment":7579,"Ġengaging":7580,"Ġadopt":7581,"Ġrental":7582,"Ġlinebacker":7583,"ĠKardashian":7584,"Ġepisodes":7585,"..":7586,"Ġunt":7587,"Ġvowed":7588,"Ġ79":7589,"ule":7590,"Ġtransit":7591,"Ġoffshore":7592,"Ġsuppliers":7593,"Ġarguing":7594,"Ġsatellite":7595,"ĠLind":7596,"ĠTaliban":7597,"Buy":7598,"ĠCaribbean":7599,"ĠBarry":7600,"Ġauthors":7601,"ĠWolf":7602,"Ġviewing":7603,"ĠCubs":7604,"From":7605,"Ġ%":7606,"Ġcurrencies":7607,"Why":7608,"ĠBroncos":7609,"Ġtrick":7610,"Ġdiesel":7611,"ĠLiberal":7612,"FL":7613,"Ġtopics":7614,"Ġretain":7615,"ĠLiberty":7616,"Ġacquisitions":7617,"ced":7618,"Ġfre":7619,"Ġfleet":7620,"Ġcopper":7621,"ĠPot":7622,"jen":7623,"ĠElliott":7624,"ĠPyongyang":7625,"Ġobject":7626,"ĠUse":7627,"Ġmutual":7628,"MP":7629,"Ġev":7630,"Ġdeny":7631,"ĠEveryone":7632,"lling":7633,"Ġpays":7634,"Ġdrought":7635,"Ġcorn":7636,"Ġworkplace":7637,"rig":7638,"ĠMn":7639,"Ġadvisory":7640,"ĠCat":7641,"Ġchronic":7642,"ĠSteelers":7643,"Ġboxes":7644,"ĠNap":7645,"Ġdemonstrated":7646,"ĠTournament":7647,"Ġsymbol":7648,"ĠAfghan":7649,"ĠTan":7650,"ired":7651,"ĠEv":7652,"ĠConsumer":7653,"Ġmoral":7654,"ĠAdditional":7655,"Ġwebsites":7656,"Ġoccasions":7657,"Ġfate":7658,"Ġpitcher":7659,"Ġtaxpayers":7660,"Ġdeemed":7661,"ĠLibya":7662,"Ġpriced":7663,"Ġdistributed":7664,"ĠForum":7665,"Ġrice":7666,"Ġbloc":7667,"Ġprovisions":7668,"agh":7669,"Ġpen":7670,"Ġattracted":7671,"ĠEdmonton":7672,"Ġthousand":7673,"Ġpainting":7674,"Ġil":7675,"Ġcourtesy":7676,"Ġeliminate":7677,"Ġacc":7678,"Ġmeters":7679,"Ġreflected":7680,"Ġcomponent":7681,"Every":7682,"Ġsells":7683,"Ġfault":7684,"Ġburned":7685,"ĠKirk":7686,"ĠAnna":7687,"Ġappeals":7688,"Ġeggs":7689,"Ġfrequent":7690,"Ġtrigger":7691,"Ġrevised":7692,"ĠAngela":7693,"Ġ81":7694,"Ġsingles":7695,"Ġviral":7696,"Ġworries":7697,"ĠShould":7698,"profit":7699,"Ġraises":7700,"ĠBryant":7701,"ĠProduct":7702,"Ġtenure":7703,"Ġdiabetes":7704,"Ġcolour":7705,"azz":7706,"ĠGirls":7707,"Ġpractical":7708,"Ġblind":7709,"ancing":7710,"pictured":7711,"Ġfinale":7712,"ĠElection":7713,"Ġathletic":7714,"Ġpromoted":7715,"Ġflowers":7716,"Ġtrains":7717,"ario":7718,"Ġsufficient":7719,"IE":7720,"Ġexamples":7721,"Ġshed":7722,"Ġbirds":7723,"Ġchaos":7724,"Ġwound":7725,"Ġrocket":7726,"Ġwet":7727,"Ġsample":7728,"ĠNag":7729,"ĠOliver":7730,"Ġscrutiny":7731,"ĠSeven":7732,"ĠRoman":7733,"ĠFred":7734,"Ġweird":7735,"ĠTam":7736,"ĠSupport":7737,"ĠNathan":7738,"Ġstudying":7739,"Ġintroduction":7740,"Ġtons":7741,"cer":7742,"aus":7743,"ION":7744,"Ġcritic":7745,"ĠAh":7746,"alo":7747,"pur":7748,"Ġstorms":7749,"ĠMission":7750,"Ġcredits":7751,"Ġgrants":7752,"Ġcomp":7753,"Ġhearts":7754,"part":7755,"Ġpin":7756,"Ġsubsequent":7757,"Ġmad":7758,"ĠSacramento":7759,"woman":7760,"from":7761,"Ġoutcomes":7762,"Ġoldest":7763,"Ġdesperate":7764,"ĠTal":7765,"ĠDJ":7766,"ward":7767,"Ġaudiences":7768,"Ġimportantly":7769,"ĠEmily":7770,"sk":7771,"ĠHeat":7772,"ĠType":7773,"ĠPeace":7774,"Ġsuspicious":7775,"aly":7776,"ĠGET":7777,"ĠCAP":7778,"dis":7779,"ĠIraqi":7780,"ĠReed":7781,"Ġstrange":7782,"ĠParent":7783,"900":7784,"Ġglad":7785,"ĠTroy":7786,"ĠShort":7787,"Ġheritage":7788,"Ġarriving":7789,"ingly":7790,"Ġtransformation":7791,"Ġlease":7792,"Ġcollapsed":7793,"cha":7794,"ĠPatrol":7795,"Ġcomputers":7796,"Ġprinciples":7797,"Ġsporting":7798,"ĠHughes":7799,"mile":7800,"ĠCit":7801,"Ġdrilling":7802,"ĠBox":7803,"ÃŁ":7804,"bre":7805,"ĠOverall":7806,"Ġopioid":7807,"Ġdelighted":7808,"Ġhonored":7809,"ĠCold":7810,"Ġunions":7811,"ĠCou":7812,"ĠCircuit":7813,"Ġblast":7814,"sson":7815,"ĠHernandez":7816,"ĠLooking":7817,"Ġlegally":7818,"ĠWalmart":7819,"bridge":7820,"Ġmat":7821,"rad":7822,"ids":7823,"Ġdining":7824,"Ġrebound":7825,"abad":7826,"ĠRom":7827,"Ġimpose":7828,"ĠAlpha":7829,"ĠWeekly":7830,"TER":7831,"ĠJam":7832,"Ġabsolute":7833,"Ġinventory":7834,"ĠBilly":7835,"ĠKaren":7836,"ĠFriends":7837,"ĠCent":7838,"ĠVikings":7839,"ĠMuch":7840,"cell":7841,"ads":7842,"Ġph":7843,"Ġkiller":7844,"ĠMembers":7845,"Ġshooter":7846,"ĠInvestigators":7847,"ĠJoshua":7848,"Ġparticipated":7849,"Ġinnocent":7850,"ĠRichmond":7851,"itor":7852,"ĠDal":7853,"ĠOperator":7854,"Ġmakeup":7855,"Ġconf":7856,"ĠNEWS":7857,"ĠDef":7858,"Ġchase":7859,"ĠCost":7860,"mont":7861,"\":":7862,"Ġarrangements":7863,"stein":7864,"Ġretire":7865,"ĠLuis":7866,"Ġrenewed":7867,"ĠTownship":7868,"Ġchecked":7869,"arts":7870,"ĠCash":7871,"Ġcentres":7872,"chers":7873,"ĠSolutions":7874,"Ġlegend":7875,"ige":7876,"most":7877,"osed":7878,"ĠPor":7879,"Ġpremiere":7880,"FS":7881,"Ġmissiles":7882,"ĠLang":7883,"Ġsing":7884,"best":7885,"Ġtail":7886,"Ġriders":7887,"Picture":7888,"zen":7889,"ĠKent":7890,"Ġtransform":7891,"Ġwildlife":7892,"Ġsmoking":7893,"Ġpreseason":7894,"ĠLucas":7895,"ĠAnne":7896,"owski":7897,"Ġtape":7898,"Ġdisplayed":7899,"Ġforum":7900,"Ġanonymity":7901,"ĠIndianapolis":7902,"hips":7903,"acc":7904,"ĠMoreover":7905,"lers":7906,"area":7907,"ĠIndeed":7908,"Ġconducting":7909,"Ġinfection":7910,"Ġdealt":7911,"OB":7912,"asing":7913,"ĠGaza":7914,"itter":7915,"ĠKa":7916,"Ġhopeful":7917,"ĠSnow":7918,"Ġentitled":7919,"Ġaffecting":7920,"Ġeager":7921,"Ġcircle":7922,"Ġlaugh":7923,"ĠProsecutors":7924,"ĠDur":7925,"Ġbarriers":7926,"ĠPoll":7927,"oun":7928,"ĠPalm":7929,"chi":7930,"Ġsamples":7931,"Ġcompromise":7932,"atter":7933,"Ġenormous":7934,"Ġé":7935,"coming":7936,"ĠPharmaceutical":7937,"Ġrank":7938,"Let":7939,"Ġtransgender":7940,"ĠCloud":7941,"FO":7942,"ĠBor":7943,"Ġbonus":7944,"Ġordinary":7945,"ĠPres":7946,"ĠHIV":7947,"ires":7948,"OSE":7949,"Ġdancing":7950,"ĠHD":7951,"Ġversions":7952,"Ġ88":7953,"rate":7954,"Ġtackles":7955,"Ġknock":7956,"ĠEmma":7957,"Ġmotivated":7958,"ĠBennett":7959,"ĠBurn":7960,"Ġgrid":7961,"Ġembrace":7962,"ĠSpurs":7963,"Ġflows":7964,"ĠGer":7965,"Ġsponsored":7966,"Ġsurvival":7967,"ching":7968,"Ġ1995":7969,"Ġreward":7970,"Ġdepends":7971,"Ġpostseason":7972,"Ġloaded":7973,"Ġneutral":7974,"ĠPop":7975,"BL":7976,"Ġrevolution":7977,"ĠFreedom":7978,"Ġrecovering":7979,"Ġrequiring":7980,"ALL":7981,"ARE":7982,"Ġmini":7983,"lt":7984,"ĠFDA":7985,"Ġcarpet":7986,"ĠPrior":7987,"Ġadmission":7988,"ĠEver":7989,"ĠTribune":7990,"ĠRonaldo":7991,"Ġthick":7992,"Ġlanes":7993,"Ġ84":7994,"ĠMemphis":7995,"Ġopt":7996,"BO":7997,"Ġfaculty":7998,"ĠChad":7999,"ĠSUV":8000,"ĠHen":8001,"Ġeste":8002,"ĠHu":8003,"ĠAgriculture":8004,"store":8005,"ĠDrug":8006,"inter":8007,"Ġ1996":8008,"ident":8009,"Ġbackup":8010,"ĠHonda":8011,"ĠHope":8012,"oes":8013,"ums":8014,"amer":8015,"Ġbreath":8016,"Ġ110":8017,"Ġjoke":8018,"ĠAld":8019,"Ġwondering":8020,"ĠAssad":8021,"ĠRem":8022,"Ġfundraising":8023,"pot":8024,"è":8025,"Ġquestioning":8026,"Ġpent":8027,"ĠMoney":8028,"ĠMedicine":8029,"wick":8030,"ĠKnights":8031,"Ġbatting":8032,"ĠMos":8033,"Ġdesignated":8034,"isse":8035,"Ġspotlight":8036,"Ġlake":8037,"Ġcaution":8038,"Ġinmates":8039,"Ġlap":8040,"CE":8041,"ĠJavascript":8042,"ĠDeutsche":8043,"ĠFargo":8044,"Ġguaranteed":8045,"borough":8046,"Ġfunctions":8047,"ĠElementary":8048,"ĠChuck":8049,"Ġpitched":8050,"ĠKrist":8051,"Ġsteal":8052,"Ġchips":8053,"Ġalarm":8054,"Ġbeloved":8055,"scale":8056,"Ġassaulted":8057,"ĠPentagon":8058,"Ġtemporarily":8059,"Ġ93":8060,"Ġ>":8061,"ĠPortugal":8062,"ti":8063,"HL":8064,"Ġdecreased":8065,"Ġexistence":8066,"Ġisolated":8067,"Ġdeposit":8068,"Ġstudied":8069,"\")":8070,"Ġtrophy":8071,"ĠBrooks":8072,"Ġbattling":8073,"Ġweaker":8074,"ĠPrivate":8075,"ĠAccess":8076,"Ġvirtually":8077,"Ġshortage":8078,"Ġgaining":8079,"Ġbathroom":8080,"TON":8081,"Ġconcerning":8082,"Ġengineer":8083,"Ġbread":8084,"Ġdemonstrate":8085,"ĠDh":8086,"Ġhorses":8087,"Ġintersection":8088,"Ġcolors":8089,"Ġdelegation":8090,"Ġnotable":8091,"Ġwithdrawal":8092,"ĠDennis":8093,"Ġlocally":8094,"Ġcoastal":8095,"Ġcomply":8096,"ĠMoh":8097,"ĠAlbert":8098,"Ġclosest":8099,"ĠCITY":8100,"Ġ83":8101,"Ġcancelled":8102,"ĠðŁ":8103,"Ġsharply":8104,"RS":8105,"Ġproductivity":8106,"Ġbasket":8107,"SS":8108,"Ġadmit":8109,"ool":8110,"ination":8111,"ĠBB":8112,"Ġsur":8113,"ĠSteel":8114,"ĠTed":8115,"ĠPac":8116,"Ġpatterns":8117,"Ġlisting":8118,"Ġreplacing":8119,"ĠPradesh":8120,"Ġroots":8121,"Ġbroker":8122,"ĠWriting":8123,"Ġsued":8124,"Ġorganised":8125,"ĠThanksgiving":8126,"ĠNOT":8127,"Ġjournalism":8128,"uel":8129,"Ġkilometers":8130,"Ġhunt":8131,"berry":8132,"ĠMother":8133,"Ġlegitimate":8134,"Ġinput":8135,"ĠRel":8136,"ĠGuardian":8137,"Ar":8138,"Ġtransported":8139,"Ġbedroom":8140,"ashing":8141,"Ġbats":8142,"Ġcleaning":8143,"Ġwrapped":8144,"Pacific":8145,"Ġfence":8146,"Ġtestified":8147,"Ġ1994":8148,"Ġinterference":8149,"Ġmatching":8150,"Ġexpression":8151,"eta":8152,"ĠSpencer":8153,"Ġstrategist":8154,"who":8155,"Ġvictories":8156,"Ġ2022":8157,"Ġstakes":8158,"Ġbuses":8159,"ĠHousing":8160,"Ġeditorial":8161,"Ġ86":8162,"ĠBishop":8163,"Ġfrustrated":8164,"Ġappearing":8165,"http":8166,"IGHT":8167,"Ġmemo":8168,"Ġinsiders":8169,"Even":8170,"Ġclassroom":8171,"Ġchef":8172,"aining":8173,"].":8174,"ĠMcD":8175,"Ġ87":8176,"ĠPunjab":8177,"Ġancient":8178,"Ġresolved":8179,"Ġdying":8180,"Ġdestruction":8181,"Ġgoverning":8182,"Ġrestructuring":8183,"ĠPick":8184,"Ġmunicipal":8185,"Ġengines":8186,"ĠHudson":8187,"Æ":8188,"Ġrepeal":8189,"standing":8190,"Ġbound":8191,"ĠOS":8192,"ĠCommonwealth":8193,"Ġdescription":8194,"Ġhouseholds":8195,"Ġmal":8196,"Ġstopping":8197,"equ":8198,"Ġregulator":8199,"Ġcontaining":8200,"Ġremoving":8201,"Ġwithdraw":8202,"Ġburied":8203,"Ġlists":8204,"ĠGil":8205,"Ġlowered":8206,"Ġformally":8207,"ĠRound":8208,"asi":8209,"¥":8210,"lett":8211,"Ġprogressive":8212,"ĠFalcons":8213,"ĠRaw":8214,"gun":8215,"Ġcontributing":8216,"Ġhunting":8217,"Ġvalid":8218,"Ġexception":8219,"ĠPlayers":8220,"ĠTra":8221,"Ġracism":8222,"hing":8223,"chen":8224,"Ġdifferently":8225,"Ġchampionships":8226,"ĠEng":8227,"ĠNO":8228,"ĠAuto":8229,"ĠErdogan":8230,"iding":8231,"Ġwarming":8232,"Ġcivilian":8233,"ĠDam":8234,"Ġfantasy":8235,"ĠNav":8236,"itions":8237,"ĠDrew":8238,"ĠNancy":8239,"Ġtrapped":8240,"ĠRussians":8241,"ĠIC":8242,"Ġflexibility":8243,"ular":8244,"Ġviolated":8245,"ipped":8246,"Ġgarage":8247,"ĠDeep":8248,"Ġpraise":8249,"ĠLab":8250,"ĠPlayer":8251,"Ġjudicial":8252,"Ġdonate":8253,"Ġseparated":8254,"Ġreleases":8255,"nik":8256,"Ġexplanation":8257,"aph":8258,"Ġloyal":8259,"Ġstrongest":8260,"ĠShar":8261,"Ġrescued":8262,"Ġambitious":8263,"Ġclimb":8264,"Ġscared":8265,"Ġignored":8266,"cut":8267,"Ġstole":8268,"Ġweakness":8269,"ĠRidge":8270,"oa":8271,"LA":8272,"Ġdep":8273,"ĠPowell":8274,"Do":8275,"Ġprotein":8276,"Ġreiterated":8277,"ĠCox":8278,"aling":8279,"ĠUnlike":8280,"ĠKane":8281,"ĠMcConnell":8282,"Ġshowcase":8283,"Ġuniform":8284,"ower":8285,"Ġdiscover":8286,"stop":8287,"ipper":8288,"Ġtreatments":8289,"Ġgrocery":8290,"Ġsubscribers":8291,"lock":8292,"ple":8293,"Ġflew":8294,"ania":8295,"Ġstepping":8296,"ĠSoviet":8297,"Ġconsultant":8298,"ags":8299,"ĠLim":8300,"Ġ91":8301,"ĠCode":8302,"ports":8303,"box":8304,"Ġlakh":8305,"Ġreminder":8306,"ym":8307,"ĠTravis":8308,"Ġpure":8309,"now":8310,"ĠVR":8311,"Ġachievement":8312,"ĠEmirates":8313,"ĠThunder":8314,"Ġmerely":8315,"ĠCa":8316,"ĠAverage":8317,"ĠDa":8318,"Ġtopped":8319,"ĠCurry":8320,"Ġchemicals":8321,"Ġamendment":8322,"ĠBorder":8323,"ĠBat":8324,"Ġ130":8325,"Ġprogramming":8326,"Ġtele":8327,"ĠKarl":8328,"Ġaveraged":8329,"ĠSpe":8330,"world":8331,"PG":8332,"Ġfights":8333,"ĠPrincess":8334,"ĠCIA":8335,"ĠAbe":8336,"Ġacted":8337,"only":8338,"Ġinsight":8339,"Ġathlete":8340,"ĠTar":8341,"commerce":8342,"Ġaveraging":8343,"cr":8344,"ĠPalestinians":8345,"Well":8346,"Ġbull":8347,"Ġchoosing":8348,"Ġsurely":8349,"ĠSecret":8350,"Ġteammate":8351,"ĠAmendment":8352,"ĠBirmingham":8353,"Ġexcitement":8354,"strong":8355,"ĠSin":8356,"Ġdamages":8357,"rated":8358,"Ġrankings":8359,"Ġconservation":8360,"home":8361,"erm":8362,"ield":8363,"Ġdisorder":8364,"acher":8365,"Ġnaturally":8366,"atur":8367,"Ġpackages":8368,"Ġapproaches":8369,"icks":8370,"ourn":8371,"Ġodd":8372,"Ġshore":8373,"ĠBeing":8374,"Ġmagic":8375,"Ġtourist":8376,"largest":8377,"Ġwhenever":8378,"Ġlenders":8379,"Ġegg":8380,"ĠChair":8381,"Ġlets":8382,"Ġwarnings":8383,"į":8384,"Ġpol":8385,"Ġdrag":8386,"ĠAmb":8387,"ĠCle":8388,"ĠLouisville":8389,"ĠShaw":8390,"lands":8391,"Ġanthem":8392,"ĠTrail":8393,"Ġaccepting":8394,"anger":8395,"good":8396,"ĠBroad":8397,"ĠLebanon":8398,"ĠMillion":8399,"ĠHenderson":8400,"Ġwh":8401,"Ġdust":8402,"Ġ92":8403,"ĠMend":8404,"Ġchecking":8405,"ĠCow":8406,"sized":8407,"Ġautomatic":8408,"Ġcelebrates":8409,"Ġarena":8410,"Ġfinger":8411,"ĠHarvard":8412,"Ġfrustration":8413,"Ġstrict":8414,"Ġpreserve":8415,"Ġsleeping":8416,"Ġconverted":8417,"Ġinsights":8418,"Ġtra":8419,"Ġjailed":8420,"Ġchamber":8421,"Ġtoxic":8422,"ading":8423,"ĠTriple":8424,"grade":8425,"ĠRest":8426,"ĠHoly":8427,"oper":8428,"Ġdesk":8429,"Ġmatchup":8430,"Ġsteep":8431,"ĠGot":8432,"lay":8433,"ĠCab":8434,"aked":8435,"ĠFoster":8436,"Ġrunners":8437,"ĠNA":8438,"Ġdestroy":8439,"Ġsupportive":8440,"ĠRacing":8441,"Ġtrademark":8442,"Ġjacket":8443,"Ġhorror":8444,"ĠAle":8445,"Ġass":8446,"Ġsch":8447,"abb":8448,"Ġplanes":8449,"Ġimpression":8450,"ĠEarly":8451,"ĠPompe":8452,"Ġking":8453,"Ġsilent":8454,"ĠCuba":8455,"Ġmedication":8456,"ences":8457,"list":8458,"ailing":8459,"WA":8460,"ella":8461,"Ġprop":8462,"Ġhalt":8463,"Ġslowing":8464,"ĠFoods":8465,"Ġanonymous":8466,"kh":8467,"Ġtraveled":8468,"Ġcommunicate":8469,"Ġter":8470,"ĠHockey":8471,"ĠRobin":8472,"Ġswept":8473,"Ġclinic":8474,"ration":8475,"len":8476,"Ġau":8477,"Ġcareers":8478,"ĠSound":8479,"Ġaddresses":8480,"China":8481,"ĠSr":8482,"Ġexhibit":8483,"ĠMotors":8484,"ĠIl":8485,"Ġinstall":8486,"ĠOkay":8487,"Ġ>>":8488,"hood":8489,"stand":8490,"Ġaudit":8491,"Ġcake":8492,"Ġflames":8493,"bel":8494,"ĠMust":8495,"ĠManafort":8496,"Ġcommodity":8497,"night":8498,"ĠRoom":8499,"ĠLanka":8500,"Ġcommander":8501,"ln":8502,"Ġdatabase":8503,"ĠSet":8504,"Ġgraduated":8505,"ĠTarget":8506,"Ġoutbreak":8507,"rou":8508,"ĠPope":8509,"ĠEqu":8510,"Ġpolling":8511,"Ġdig":8512,"Ġbrutal":8513,"ĠBarn":8514,"Ġdefinition":8515,"Ġpit":8516,"Ġpickup":8517,"ĠBitcoin":8518,"ĠReid":8519,"Ġloving":8520,"ĠHerald":8521,"ĠCanadians":8522,"Ġneighbor":8523,"Ġdies":8524,"ione":8525,"ĠRef":8526,"big":8527,"Ġguards":8528,"including":8529,"ente":8530,"Ġpartially":8531,"Image":8532,"Ġbulk":8533,"Ġslot":8534,"ĠNorthwest":8535,"ĠBarclays":8536,"Ġairlines":8537,"iver":8538,"isi":8539,"Ġsubsidiary":8540,"Ġcont":8541,"ĠDaniels":8542,"Ġscript":8543,"Ġunfair":8544,"Ġscreens":8545,"Ġprof":8546,"ĠIrma":8547,"Ġ1992":8548,"Ġmandatory":8549,"ĠSant":8550,"Ġsuspicion":8551,"NES":8552,"ĠLauren":8553,"igen":8554,"Ġprevention":8555,"Ġtension":8556,"ema":8557,"Ġtasks":8558,"Ġshake":8559,"Ġexplosive":8560,"Ġaffects":8561,"Ġmum":8562,"ĠDog":8563,"rer":8564,"Ġopted":8565,"Ġtrio":8566,"Ġlesson":8567,"Ġautomotive":8568,"where":8569,"ĠMontgomery":8570,"Ġcouples":8571,"Ġ89":8572,"AF":8573,"Ġinfo":8574,"ĠForm":8575,"Ġspectrum":8576,"Ġbands":8577,"Ġokay":8578,"Ġstroke":8579,"ĠNetanyahu":8580,"Ġwealthy":8581,"ĠAround":8582,"ĠGlenn":8583,"sec":8584,"there":8585,"ickets":8586,"ĠBudget":8587,"ĠBMW":8588,"Ġflagship":8589,"rier":8590,"Ġpodcast":8591,"Ġpursuing":8592,"Ġpos":8593,"ĠIslands":8594,"ĠUrban":8595,"page":8596,"Ġemotions":8597,"ided":8598,"Ġdividends":8599,"Ġboom":8600,"Ġaccusing":8601,"ird":8602,"ĠNam":8603,"ava":8604,"Ġwishes":8605,"ĠNy":8606,"ĠStanford":8607,"Ġcriteria":8608,"ĠJews":8609,"Ġengineers":8610,"Ġaccuracy":8611,"Ġdisplays":8612,"Ġdeserves":8613,"ridge":8614,"omm":8615,"aur":8616,"Ġdramatically":8617,"Ġunity":8618,"speed":8619,"Ġdeclining":8620,"Ġpermits":8621,"ĠKn":8622,"Ġconsulting":8623,"aux":8624,"ATE":8625,"ĠWat":8626,"ĠEditor":8627,"sy":8628,"urn":8629,"ĠUsing":8630,"asc":8631,"ital":8632,"Ġcre":8633,"quality":8634,"Ġce":8635,"Ġenemy":8636,"Ġoffence":8637,"icket":8638,"ĠDick":8639,"ĠTH":8640,"ĠChampionships":8641,"Ġoverwhelming":8642,"rib":8643,"ku":8644,"rap":8645,"Ġhomer":8646,"acion":8647,"member":8648,"erv":8649,"aney":8650,"MB":8651,"eded":8652,"Ġpunishment":8653,"Ġnegotiate":8654,"ĠFile":8655,"stream":8656,"ĠHur":8657,"Ġnose":8658,"ĠFab":8659,"iter":8660,"Ġpainful":8661,"ITY":8662,"eren":8663,"Ġcollecting":8664,"Additional":8665,"Ġentrepreneurs":8666,"bal":8667,"Ġexploring":8668,"Ġguitar":8669,"Ġpartnerships":8670,"Ġfurniture":8671,"Ġauthorized":8672,"Ġeasing":8673,"shirt":8674,"ĠGross":8675,"Ġpolitician":8676,"ĠSimpson":8677,"Ġdrone":8678,"ĠKatie":8679,"Ġprofitability":8680,"ĠNHS":8681,"ĠSierra":8682,"ĠNorway":8683,"ASHINGTON":8684,"ific":8685,"Ġcondemned":8686,"team":8687,"ĠNebraska":8688,"Ġthrilled":8689,"iller":8690,"Ġpatrol":8691,"ĠWR":8692,"orm":8693,"Ġspectacular":8694,"ĠKnight":8695,"ĠTravel":8696,"nam":8697,"Ġmuscle":8698,"ĠRain":8699,"ĠColombia":8700,"Ġnursing":8701,"Ġmigration":8702,"ĠMitch":8703,"Ġreleasing":8704,"ĠBesides":8705,"ĠMul":8706,"Ġheadline":8707,"Ġcontemporary":8708,"Ġdev":8709,"ĠChan":8710,"Ġindicates":8711,"ĠAp":8712,"ĠLt":8713,"ĠMarvel":8714,"Ġremembered":8715,"®":8716,"ĠForces":8717,"ĠColin":8718,"ĠGabriel":8719,"Ġobjects":8720,"ĠRHP":8721,"kar":8722,"ĠKo":8723,"Ġsignals":8724,"Ġinner":8725,"real":8726,"RO":8727,"Ġromantic":8728,"cat":8729,"ĠKel":8730,"Ġgut":8731,"ĠBoys":8732,"Ġyoungest":8733,"ĠCeltics":8734,"Ġslated":8735,"Ġremind":8736,"Ġproductive":8737,"set":8738,"Co":8739,"ĠBailey":8740,"Ġrenewable":8741,"ĠCarson":8742,"ĠDj":8743,"ĠKos":8744,"Ġurge":8745,"Ġfin":8746,"Ġpursuit":8747,"ĠCON":8748,"ĠChapter":8749,"Ġpal":8750,"Ġgate":8751,"ĠPackers":8752,"ĠReports":8753,"ĠRugby":8754,"ĠMasters":8755,"MO":8756,"Ġ98":8757,"Ġcatches":8758,"ĠAgreement":8759,"ĠTillerson":8760,"ĠIce":8761,"Ġrumors":8762,"ĠLeonard":8763,"ĠDolphins":8764,"ĠLP":8765,"top":8766,"ĠCrist":8767,"ĠHon":8768,"Ġblaze":8769,"Ġrhetoric":8770,"ands":8771,"ady":8772,"David":8773,"igh":8774,"Ġbuzz":8775,"ĠStrong":8776,"Ġshocking":8777,"ĠRh":8778,"Ġnegotiating":8779,"Ġtender":8780,"ĠJohnny":8781,"ĠMario":8782,"Ġ97":8783,"ĠHeritage":8784,"Ġexists":8785,"Ġprayers":8786,"Ġlengthy":8787,"Ġsafer":8788,"ĠHalloween":8789,"ĠJared":8790,"ĠConnect":8791,"Ġbump":8792,"Ġstrain":8793,"Ġfilling":8794,"Ġtrauma":8795,"Ġcompleting":8796,"cht":8797,"Ġkillings":8798,"anne":8799,"GE":8800,"ĠRescue":8801,"Ġdealers":8802,"Ġlocals":8803,"ĠVictor":8804,"Ġtragic":8805,"Ġdelivers":8806,"orts":8807,"Ġrugby":8808,"Ġinstallation":8809,"asa":8810,"ĠBart":8811,"Ġjournal":8812,"school":8813,"ĠCome":8814,"ĠVeterans":8815,"Sun":8816,"Ġcrowds":8817,"Ġtransparent":8818,"Ġimplications":8819,"ĠHuawei":8820,"sex":8821,"Ġrallied":8822,"Ġresponses":8823,"Ġdebris":8824,"Ġconvention":8825,"Ġmothers":8826,"BE":8827,"ĠRoute":8828,"Ġrebel":8829,"ĠEmmanuel":8830,"aster":8831,"Ġunderstands":8832,"pound":8833,"ĠCastle":8834,"Ġ2021":8835,"rik":8836,"ĠGR":8837,"Ġconvince":8838,"ault":8839,"Ġpassionate":8840,"ĠSciences":8841,"Ġarrives":8842,"idad":8843,"Ġcelebrities":8844,"ends":8845,"ĠFans":8846,"Ġdish":8847,"ĠCorps":8848,"hat":8849,"Ġemployer":8850,"ĠHy":8851,"Ġpowered":8852,"Ġgrandmother":8853,"ĠFL":8854,"oured":8855,"VE":8856,"ĠInst":8857,"ĠPerez":8858,"Ġtune":8859,"Ġcitizenship":8860,"Ġignore":8861,"Ġdoubles":8862,"IB":8863,"Ġprogrammes":8864,"inda":8865,"Ġentities":8866,"ĠInterior":8867,"Ġprompting":8868,"Ġwire":8869,"Ġtheatre":8870,"%)":8871,"Ġheels":8872,"ĠJu":8873,"Ġdeposits":8874,"Ġtrash":8875,"mond":8876,"she":8877,"iana":8878,"Ġislands":8879,"ĠTommy":8880,"Ġpub":8881,"Ġdiscipline":8882,"ĠSW":8883,"Ġmusicians":8884,"Ġembassy":8885,"ĠQB":8886,"hander":8887,"UES":8888,"ĠFerguson":8889,"Ġblocking":8890,"ahn":8891,"Ġfines":8892,"Ġtactics":8893,"Ġbullet":8894,"Ġequipped":8895,"Ġescaped":8896,"ĠSil":8897,"ĠPack":8898,"ĠAthletic":8899,"ĠMic":8900,"ĠDoes":8901,"ĠCarr":8902,"ĠChargers":8903,"ĠKyl":8904,"Ġzones":8905,"µ":8906,"iki":8907,"Ġgreatly":8908,"ĠMD":8909,"Ġimmigrant":8910,"ĠConstruction":8911,"ĠBorn":8912,"iment":8913,"ĠWade":8914,"Ġvisa":8915,"Ġgenuine":8916,"Ġelectronics":8917,"ĠSat":8918,"Ġsponsors":8919,"ĠMontana":8920,"Ġspell":8921,"ĠSachs":8922,"ĠEt":8923,"Ġfoster":8924,"Ġlocker":8925,"Ġexplaining":8926,"ĠAge":8927,"Ġgunman":8928,"Ġsauce":8929,"Ġcry":8930,"Ġstimulus":8931,"Ġarray":8932,"Ġcompare":8933,"Ġboats":8934,"Ġext":8935,"iders":8936,"ĠAst":8937,"ĠParks":8938,"ester":8939,"Ġ94":8940,"Ġrelating":8941,"Ġvegetables":8942,"Ġaccountable":8943,"Ġhyper":8944,"ĠWim":8945,"Ġnewest":8946,"ĠRome":8947,"ĠChancellor":8948,"CBS":8949,"Ġbusinessman":8950,"ĠDelaware":8951,"Ġlands":8952,"court":8953,"aria":8954,"Ġapproaching":8955,"cker":8956,"ĠSalt":8957,"ĠMak":8958,"Ġtreating":8959,"Ġsubsequently":8960,"ĠEll":8961,"xton":8962,"Ġ180":8963,"Ġdetermination":8964,"ĠSalman":8965,"ĠJoel":8966,"Ġclassified":8967,"Ġspan":8968,"Ġearthquake":8969,"ranked":8970,"Ġ96":8971,"ĠTiger":8972,"Ġadvocacy":8973,"mit":8974,"Ġcolleges":8975,"ĠYeah":8976,"ĠCaptain":8977,"Ġorange":8978,"Ġprojections":8979,"Ġelectrical":8980,"ĠMA":8981,"olog":8982,"ĠNewcastle":8983,"oppers":8984,"Ġrepresentation":8985,"Ġlawsuits":8986,"just":8987,"aced":8988,"ĠRace":8989,"ĠAqu":8990,"ĠBills":8991,"Ġexclusively":8992,"ĠProfile":8993,"Ġhometown":8994,"ĠStan":8995,"Ġstarring":8996,"Ġdeciding":8997,"ĠRating":8998,"ĠMedicare":8999,"ĠTransport":9000,"Ġmystery":9001,"ĠTa":9002,"ĠPad":9003,"ĠSwedish":9004,"ĠCarroll":9005,"about":9006,"Ġtorn":9007,"Ġnurse":9008,"NE":9009,"Ġwaited":9010,"ĠJeffrey":9011,"ĠUntil":9012,"Ġbone":9013,"ĠBobby":9014,"Ġpronounced":9015,"Ġpharmaceutical":9016,"ĠGallery":9017,"ĠMatch":9018,"Ġeconomists":9019,"ĠMarketing":9020,"face":9021,"ĠPetroleum":9022,"ories":9023,"ĠMets":9024,"ĠCore":9025,"billion":9026,"Ġexamination":9027,"ĠPorter":9028,"2016":9029,"Ġgolden":9030,"Ġsem":9031,"ĠDuterte":9032,"ĠJefferson":9033,"ĠTehran":9034,"ĠLeicester":9035,"ĠDA":9036,"Ġadapt":9037,"ĠDame":9038,"ĠRic":9039,"Ġunchanged":9040,"ect":9041,"Ġsections":9042,"kg":9043,"igned":9044,"Ġfilings":9045,"Ġreact":9046,"Ġurgent":9047,"Ġvessels":9048,"Ġspark":9049,"Ġbutter":9050,"ĠCons":9051,"Ġstating":9052,"Ġcorporations":9053,"ĠHus":9054,"Ġdamaging":9055,"raw":9056,"Ġequality":9057,"Two":9058,"ĠMills":9059,"iu":9060,"Ġobligation":9061,"ĠBrook":9062,"arian":9063,"Re":9064,"Ġphotographs":9065,"Ġepic":9066,"ĠStudent":9067,"ĠTherefore":9068,"Ġgod":9069,"ĠFILE":9070,"iqu":9071,"Ġdescribing":9072,"Ġproceed":9073,"Ġcas":9074,"ĠKat":9075,"ĠBra":9076,"Ġadequate":9077,"Ġpassage":9078,"Ġthanked":9079,"USA":9080,"ĠNeither":9081,"ĠLegislature":9082,"Ġfinances":9083,"Ġinst":9084,"ĵ":9085,"ĠAngels":9086,"Ġvet":9087,"ĠDead":9088,"Ex":9089,"Ġkicks":9090,"force":9091,"Ġsoy":9092,"ĠWindsor":9093,"Ġenhanced":9094,"Ġ1993":9095,"ĠCzech":9096,"Ġgradually":9097,"ĠMagic":9098,"Ġshadow":9099,"Ġneighborhoods":9100,"ĠRivers":9101,"Ġrapper":9102,"ĠGirl":9103,"ĠRot":9104,"Ġcrackdown":9105,"fish":9106,"Ġpreventing":9107,"Ġproduces":9108,"ĠMi":9109,"Ġnotified":9110,"Ġunderground":9111,"WE":9112,"Ġadmits":9113,"Ġboxing":9114,"Ġrefer":9115,"Ġcommitments":9116,"ĠWoman":9117,"Ġdenies":9118,"col":9119,"ĠSide":9120,"Ġambulance":9121,"ĠRodgers":9122,"Ġaftermath":9123,"Ġdeck":9124,"irmed":9125,"Ġerrors":9126,"ĠConvention":9127,"Ġcurb":9128,"ĠShop":9129,"ĠThai":9130,"Ġma":9131,"Ġrespected":9132,"ĠMVP":9133,"Ġborrowing":9134,"Ġcruise":9135,"ĠSure":9136,"Ġsentencing":9137,"ĠObamacare":9138,"ĠIr":9139,"ĠSale":9140,"ĠPete":9141,"Ġopenly":9142,"Ġstartup":9143,"rock":9144,"Ġcargo":9145,"Ġtelecom":9146,"ĠDownload":9147,"Ġextending":9148,"ĠCurrent":9149,"Ġcompetitions":9150,"ĠKids":9151,"Ġshy":9152,"ĠKerry":9153,"ĠNever":9154,"ĠDevils":9155,"Ġprim":9156,"Con":9157,"Ġcurve":9158,"Ġassumed":9159,"Ġadjust":9160,"Ġimmune":9161,"UE":9162,"ĠUr":9163,"Ġconventional":9164,"Ġgrandchildren":9165,"ĠBol":9166,"Ad":9167,"ĠMaduro":9168,"fi":9169,"ĠUAE":9170,"ĠOrgan":9171,"Ġindicating":9172,"iem":9173,"ĠAgainst":9174,"ĠAmbassador":9175,"ĠSeoul":9176,"Ġcriminals":9177,"how":9178,"put":9179,"Ġreminded":9180,"Ġparked":9181,"lich":9182,"Ġcontinent":9183,"Ġmatched":9184,"ĠNicole":9185,"Ġgenetic":9186,"Ġhumanity":9187,"ĠTem":9188,"Ġindicator":9189,"Ġvessel":9190,"Ġdefendant":9191,"ĠGriffin":9192,"jan":9193,"Ġvend":9194,"boro":9195,"Ġbrokerage":9196,"ĠFall":9197,"Ġmere":9198,"VILLE":9199,"Ġlasted":9200,"ĠMind":9201,"Ġpatch":9202,"ĠInsider":9203,"ĠComm":9204,"Ġtechnique":9205,"ĠIM":9206,"ĠCavaliers":9207,"Ġshame":9208,"Ġmil":9209,"oot":9210,"irt":9211,"Ġcop":9212,"ĠLeon":9213,"Ġfrozen":9214,"Ġslip":9215,"pton":9216,"Ġpanels":9217,"Ġpitching":9218,"Ġleather":9219,"ĠLogan":9220,"ĠNearly":9221,"urch":9222,"Ġinstructions":9223,"ĠRow":9224,"ĠKurdish":9225,"this":9226,"Ġlegendary":9227,"su":9228,"Ġstabbed":9229,"sters":9230,"Ġteenage":9231,"def":9232,"Ġoversight":9233,"Ġvolatile":9234,"Ġtransmission":9235,"ĠSgt":9236,"ĠIndigenous":9237,"ĠOxford":9238,"ĠCasey":9239,"Ġcor":9240,"Ġsalaries":9241,"Ġsponsor":9242,"Ġprescription":9243,"mat":9244,"ĠLeeds":9245,"ĠPakistani":9246,"Ġevil":9247,"Ġtables":9248,"ĠAbdul":9249,"Ġexpectation":9250,"Ġlegislature":9251,"ĠLin":9252,"¹":9253,"Ġcontractor":9254,"Ġshifting":9255,"Ġgenerous":9256,"ĠEddie":9257,"Ġpuck":9258,"utt":9259,"Ġdubbed":9260,"Ġnowhere":9261,"Ġbetting":9262,"Ġdisclose":9263,"Ĥ":9264,"ĠFashion":9265,"ĠHarper":9266,"handed":9267,"isha":9268,"ĠReds":9269,"Ġachievements":9270,"ume":9271,"Ġshootings":9272,"Ġadvisers":9273,"ĠEaster":9274,"Ġinternationally":9275,"ĠWi":9276,"ĠGandhi":9277,"ĠChristians":9278,"Ġrecruiting":9279,"Ġexperiment":9280,"Ġsol":9281,"Ġdifficulties":9282,"Ġinfluential":9283,"Ġhybrid":9284,"Ġformation":9285,"ĠBoulevard":9286,"Ġflags":9287,"Ġformula":9288,"front":9289,"Ġinclusion":9290,"ĠNone":9291,"ICE":9292,"Ġfilming":9293,"ĠLou":9294,"ĠReynolds":9295,"Ġpump":9296,"Ġexceptional":9297,"ANG":9298,"ĠCorporate":9299,"SAN":9300,"ĠHealthcare":9301,"ĠUkrainian":9302,"aron":9303,"Ġpants":9304,"Ġdrops":9305,"ete":9306,"ĠStudies":9307,"Ġwounds":9308,"END":9309,"Ġshower":9310,"Ġreviewing":9311,"ĠGreater":9312,"Ġ»":9313,"itors":9314,"alled":9315,"Ġsqu":9316,"ĠRonald":9317,"ĠInv":9318,"Ġtougher":9319,"Ġbalanced":9320,"Ġlined":9321,"Ġprinciple":9322,"Ġ1950":9323,"Ġleak":9324,"Be":9325,"Ġcircuit":9326,"Ġunfortunate":9327,"ĠGran":9328,"ĠFish":9329,"Ġfriendship":9330,"asp":9331,"OO":9332,"Ġobligations":9333,"Ġcoup":9334,"OK":9335,"Ġbreakdown":9336,"Ġhook":9337,"Ġresearcher":9338,"inated":9339,"ĠMarie":9340,"ĠGab":9341,"ĠWA":9342,"quez":9343,"General":9344,"ĠSwift":9345,"Ġgust":9346,"ĠCarol":9347,"ĠCentury":9348,"ĠOPEC":9349,"ĠRd":9350,"ĠCop":9351,"Ġsubjects":9352,"ĠComments":9353,"ases":9354,"Ġrelation":9355,"ĠEnvironment":9356,"ı":9357,"Ġgasoline":9358,"ĠLog":9359,"Ġicon":9360,"Ġprofitable":9361,"ĠRetail":9362,"ANC":9363,"Ġappealing":9364,"Ġvillages":9365,"Ġpizza":9366,"Ġmall":9367,"Ġtower":9368,"ĠLinda":9369,"Ġaccomplished":9370,"Ġpod":9371,"Ġleaked":9372,"ĠWed":9373,"Ġmer":9374,"Ġopposing":9375,"!'":9376,"Ġstomach":9377,"Ġrevealing":9378,"Ġho":9379,"DF":9380,"ĠSterling":9381,"Ġsolely":9382,"Ġpres":9383,"ĠCy":9384,"ĠLatest":9385,"ĠPitt":9386,"ĠThink":9387,"Ġcapability":9388,"aled":9389,"Ġexecuted":9390,"alling":9391,"ĠSilva":9392,"Ġrestricted":9393,"Ġdeclaration":9394,"Ġkilometres":9395,"rol":9396,"Ġidentifying":9397,"Ġdonors":9398,"vent":9399,"Ġcostly":9400,"ense":9401,"ĠSeeking":9402,"OURCE":9403,"iving":9404,"Ġplacing":9405,"tech":9406,"Ġbottles":9407,"writer":9408,"ĠSeahawks":9409,"oming":9410,"ĠArthur":9411,"ously":9412,"bin":9413,"ĠVa":9414,"Ġbias":9415,"Ġliability":9416,"ift":9417,"rak":9418,"aves":9419,"Ġcautious":9420,"ĠPrize":9421,"iley":9422,"ĠSharma":9423,"global":9424,"Ġwars":9425,"sm":9426,"ĠRemember":9427,"wind":9428,"ĠRichardson":9429,"ĠSum":9430,"ĠVincent":9431,"ĠRice":9432,"inf":9433,"Ġconsultation":9434,"range":9435,"Ġbacteria":9436,"Ġarchitecture":9437,"Ġpole":9438,"ĠMach":9439,"Ġcattle":9440,"Ġabused":9441,"being":9442,"ĠHERE":9443,"Ġfame":9444,"Ġhearings":9445,"ĠBrit":9446,"Ġjoins":9447,"ĠMcGregor":9448,"Ġoppose":9449,"Ġcheer":9450,"itting":9451,"imes":9452,"Ġusage":9453,"Ġstint":9454,"Ġoutlet":9455,"Ġshoppers":9456,"ĠBaptist":9457,"Ġinappropriate":9458,"ĠALSO":9459,"Ġstealing":9460,"Ġpledge":9461,"ĠRan":9462,"Ġphotographer":9463,"Ġprevented":9464,"Ġ01":9465,"ĠEngineering":9466,"ĠProducts":9467,"Ġuniverse":9468,"ĠMcCarthy":9469,"¿":9470,"graded":9471,"Ġinspection":9472,"Ġind":9473,"Fi":9474,"aren":9475,"Ġprotections":9476,"Ġsorts":9477,"ĠWorks":9478,"Ġbillionaire":9479,"ĠGay":9480,"ĠiPad":9481,"IX":9482,"Ġdefendants":9483,"band":9484,"Ġfarms":9485,"Ġhom":9486,"gal":9487,"iant":9488,"Ġnortheast":9489,"ĠJoint":9490,"Ġcanceled":9491,"Ġtoys":9492,"Ġrein":9493,"ĠTumblr":9494,"pees":9495,"ĠAut":9496,"Police":9497,"Ġaide":9498,"Ġachieving":9499,"Ġmund":9500,"ĠCommercial":9501,"first":9502,"Ġanticipate":9503,"iac":9504,"Ġprobation":9505,"hem":9506,"Ġports":9507,"ĠKer":9508,"Ġsupplier":9509,"ĠFather":9510,"ĠAnti":9511,"ashed":9512,"ĠTable":9513,"bledon":9514,"Ġunf":9515,"ĠRash":9516,"ĠLeBron":9517,"Car":9518,"bu":9519,"ĠDerek":9520,"Ġaccounted":9521,"ĠPri":9522,"nings":9523,"Ġreceives":9524,"lev":9525,"Ġbilateral":9526,"ĠList":9527,"ĠLG":9528,"ĠJazz":9529,"Ġrestored":9530,"Ġbattles":9531,"ials":9532,"Ġoccupied":9533,"Ġrepairs":9534,"Ġradar":9535,"ĠMLB":9536,"ĠNC":9537,"Ġflexible":9538,"ĠCommand":9539,"Ġcoat":9540,"ĠVir":9541,"ĠColts":9542,"ĠBC":9543,"Ġtwin":9544,"Ġprisoners":9545,"Ġslowed":9546,"hop":9547,"ĠInn":9548,"Ġconflicts":9549,"Ġmeasured":9550,"Ġautonomous":9551,"ĠBow":9552,"Ġdisc":9553,"inson":9554,"ĠSche":9555,"aire":9556,"ĠSU":9557,"ĠPeterson":9558,"Ġdrafted":9559,"ĠPelosi":9560,"ĠSoon":9561,"Ġmechanism":9562,"Ġaccountability":9563,"ĠNortheast":9564,"Ġfo":9565,"Ġanalytics":9566,"ĠEverything":9567,"Ġperceived":9568,"bers":9569,"Ġcelebrations":9570,"Ġinstruments":9571,"Ġstrip":9572,"ĠJuventus":9573,"Ġunfortunately":9574,"ĠGA":9575,"Ġwrestling":9576,"Ġstatue":9577,"vis":9578,"five":9579,"Ġmarine":9580,"ĠSamuel":9581,"Ġresponsibilities":9582,"hill":9583,"Ġrecruit":9584,"Ġreferee":9585,"ĠRail":9586,"ĠEagle":9587,"ĠCongressional":9588,"Ġbreathing":9589,"Ġbass":9590,"hit":9591,"Ġspreading":9592,"Ġevacuated":9593,"Ġintellectual":9594,"Ġsovereign":9595,"ocked":9596,"Ġslammed":9597,"Ġformerly":9598,"Ġarch":9599,"Ġdifficulty":9600,"ĠAFC":9601,"ĠFresh":9602,"Ġinvite":9603,"oner":9604,"ĠMich":9605,"Ġpitches":9606,"stock":9607,"Ġinitiated":9608,"ĠKu":9609,"ĠFlorence":9610,"yd":9611,"ĠFast":9612,"Ġmusician":9613,"ĠChile":9614,"anga":9615,"Ġdairy":9616,"Ġcontractors":9617,"ador":9618,"ĠPlanning":9619,"Ġultra":9620,"Ġprayer":9621,"Ġsuggestions":9622,"ĠEk":9623,"Ġrandom":9624,"ĠSullivan":9625,"Ġsensor":9626,"Ġhomicide":9627,"ĠIncome":9628,"Ġsettings":9629,"Ġacknowledge":9630,"ĠStay":9631,"Ġterminal":9632,"Ġ1991":9633,"West":9634,"hard":9635,"arc":9636,"Ġcombine":9637,"Ġprivately":9638,"Ġbarrier":9639,"Ġmedian":9640,"Ġwhereas":9641,"ĠTitans":9642,"Ġincentives":9643,"Ġhistorically":9644,"Ġindictment":9645,"Ġhiding":9646,"ĠPDT":9647,"Ġrebuild":9648,"hol":9649,"Ġpour":9650,"Ġairports":9651,"ĠEdinburgh":9652,"Ġappoint":9653,"ĠJul":9654,"Ġconfusion":9655,"Ġdam":9656,"ork":9657,"Ġcalculated":9658,"Ġhood":9659,"ĠTemple":9660,"ĠYorkshire":9661,"EP":9662,"ented":9663,"Ġapology":9664,"awi":9665,"Ġfacilitate":9666,"ĠSheffield":9667,"Ġrides":9668,"Ġcompelling":9669,"ĠGonzalez":9670,"roll":9671,"ONG":9672,"UP":9673,"ĠAj":9674,"pen":9675,"ĠVar":9676,"ĠIPO":9677,"ĠAnimal":9678,"Ġshifted":9679,"Ġ140":9680,"Ġtobacco":9681,"El":9682,"ild":9683,"Ġuncertain":9684,"Un":9685,"Ġcaps":9686,"Ġrecreational":9687,"ĠTu":9688,"Ġenc":9689,"More":9690,"iko":9691,"ĠEverton":9692,"ĠWalk":9693,"Ġmurdered":9694,"Ġpur":9695,"Ġdivisions":9696,"ivo":9697,"Ġfarming":9698,"Ġcourage":9699,"ped":9700,"Ġcrying":9701,"Ġattributed":9702,"ée":9703,"Ġimplementing":9704,"ĠWang":9705,"Ġspeeds":9706,"alk":9707,"aming":9708,"eries":9709,"Ġavoided":9710,"ĠMessi":9711,"Ġconsiderable":9712,"rt":9713,"Ġinauguration":9714,"ĠPH":9715,"Ġsoldier":9716,"Ġore":9717,"ollywood":9718,"otive":9719,"ĠAuburn":9720,"ĠSav":9721,"ĠPut":9722,"Ġemphasis":9723,"Ġaf":9724,"owed":9725,"Ġdiagnosis":9726,"Ġcart":9727,"Ġassisted":9728,"ĠOrder":9729,"ĠEstate":9730,"Ġintends":9731,"ĠCommon":9732,"Ġadventure":9733,"Ġbeliefs":9734,"Ġlasting":9735,"cel":9736,"Ġdeployment":9737,"tra":9738,"ĠStories":9739,"Ġquote":9740,"Ġfeared":9741,"Ġconvenience":9742,"Ġoptimism":9743,"Ġscientist":9744,"ĠEnterprise":9745,"ĠRex":9746,"ĠFel":9747,"Ġposes":9748,"Ġroot":9749,"Ġevacuation":9750,"Ġpresidents":9751,"ĠRather":9752,"Ġgrave":9753,"ĠHeights":9754,"Ġjumping":9755,"driven":9756,"Ġaluminum":9757,"Ġholders":9758,"Ġboot":9759,"iber":9760,"Ġprecious":9761,"uation":9762,"FP":9763,"uses":9764,"Ġcommentary":9765,"Ġadvances":9766,"ĠNissan":9767,"Ġbronze":9768,"Ġinspire":9769,"Ġstarters":9770,"ĠEvan":9771,"rah":9772,"body":9773,"Ġcrops":9774,"Ġseeds":9775,"Ġharsh":9776,"ĠHomeland":9777,"Ġenabled":9778,"ological":9779,"Ġworkshop":9780,"Ġchains":9781,"amps":9782,"Ġamongst":9783,"ĠBear":9784,"Ġcertified":9785,"ĠJulie":9786,"Ġmountains":9787,"VA":9788,"Ġfed":9789,"Ġbuyer":9790,"ahl":9791,"ĠBos":9792,"ĠCrystal":9793,"Ġquest":9794,"ĠStein":9795,"Ġacceptable":9796,"Ġunbeaten":9797,"iring":9798,"ural":9799,"Ġuncomfortable":9800,"Ġpartial":9801,"Ġsacrifice":9802,"ĠGrande":9803,"Ġarrangement":9804,"Ġpackaging":9805,"screen":9806,"Ġmirror":9807,"Ġsweep":9808,"Ġconnecting":9809,"Ġpanic":9810,"ĠJacksonville":9811,"ĠKremlin":9812,"Ġorigin":9813,"Brien":9814,"Ġnorthwest":9815,"Ġcarriers":9816,"ĠRiley":9817,"Ġaud":9818,"Ġappreciation":9819,"Ġeliminated":9820,"ĠAnalyst":9821,"CR":9822,"Ġfirearm":9823,"Ġaccommodate":9824,"Ġstructural":9825,"Ġappealed":9826,"Ġcharter":9827,"ressing":9828,"Ġalike":9829,"white":9830,"Ġslowdown":9831,"Ġweigh":9832,"ĠPalmer":9833,"ound":9834,"ĠConn":9835,"Ġbranches":9836,"Ġace":9837,"Ġinsists":9838,"yo":9839,"ĠLynn":9840,"ĠCC":9841,"ĠWithin":9842,"Ġcoll":9843,"Ġsustain":9844,"Ġemerge":9845,"ĠBattle":9846,"VER":9847,"Ġaviation":9848,"Ġenables":9849,"ĠProduction":9850,"ĠGrove":9851,"Ġnationally":9852,"ĠBaldwin":9853,"rent":9854,"Ġfirearms":9855,"irm":9856,"Ġconsiders":9857,"ĠCosby":9858,"ĠMcK":9859,"ĠEnt":9860,"Ġincumbent":9861,"iance":9862,"Ġgiants":9863,"Ġkan":9864,"Ġminimal":9865,"ivity":9866,"ĠSay":9867,"ĠNass":9868,"Ġlovely":9869,"ĠFurthermore":9870,"Ġdisplaced":9871,"Ġcontacts":9872,"NY":9873,"Ġtechnological":9874,"ancy":9875,"Ġant":9876,"ope":9877,"ĠFY":9878,"Ġfavorable":9879,"ĠVirgin":9880,"Ġcasual":9881,"ĠLat":9882,"Ġpopulations":9883,"Ġromance":9884,"Ġforgotten":9885,"Ġfleeing":9886,"Ġspecialty":9887,"Ġdrill":9888,"Ġapplying":9889,"Ġcocaine":9890,"rea":9891,"Ġheroin":9892,"Ġsweeping":9893,"ĠMaj":9894,"Ġtroubled":9895,"Ġcolleague":9896,"Ġedged":9897,"omes":9898,"ĠHappy":9899,"´":9900,"Ġmilitant":9901,"boy":9902,"aver":9903,"Yes":9904,"llo":9905,"Ġsupporter":9906,"ĠSubscribe":9907,"ĠBird":9908,"ĠGibson":9909,"Ġhill":9910,"Ġnewspapers":9911,"ĠPHOTO":9912,"Ġouting":9913,"Ġdefine":9914,"Ġann":9915,"Ġrobot":9916,"Ġregret":9917,"ĠCould":9918,"raz":9919,"Ġceiling":9920,"Ġorganizers":9921,"ĠTw":9922,"Ġcriticised":9923,"ĠJoh":9924,"ĠJe":9925,"ĠBulls":9926,"Ġteeth":9927,"ĠRanch":9928,"ĠAndrea":9929,"Ġconservatives":9930,"Ġmag":9931,"vey":9932,"Ġpredecessor":9933,"ĠJPMorgan":9934,"Ġdraws":9935,"umber":9936,"Ġvaccine":9937,"ĠDas":9938,"Ġdisappeared":9939,"ĠIron":9940,"Ġlitigation":9941,"vert":9942,"Ġbelong":9943,"ĠRet":9944,"owers":9945,"rain":9946,"controlled":9947,"ĠKil":9948,"Ġrehab":9949,"ĠAustria":9950,"Ġprivilege":9951,"Ġbounce":9952,"Ġbout":9953,"ĠIslamist":9954,"Ġtaxi":9955,"ody":9956,".'\"":9957,"Ġdos":9958,"shire":9959,"Ġaccidents":9960,"Ġdemonstration":9961,"His":9962,"ĠBO":9963,"ĠICE":9964,"van":9965,"File":9966,"ĠManning":9967,"ounded":9968,"Ġdirections":9969,"lled":9970,"Ġoffences":9971,"Ġlaptop":9972,"ĠUniversal":9973,"Ġmilestone":9974,"ĠNarendra":9975,"Ġnotion":9976,"Ġuns":9977,"ĠLower":9978,"Ġmidfield":9979,"Ġoutper":9980,"trans":9981,"ĠJa":9982,"three":9983,"Adds":9984,"Ġpressures":9985,"Ġprohibited":9986,"Ġutilities":9987,"Ġbes":9988,"ĠReporter":9989,"Ġcommodities":9990,"leton":9991,"Ġslower":9992,"EE":9993,"auer":9994,"Ġtablet":9995,"sl":9996,"iously":9997,"Ġaiming":9998,"eland":9999,"ĠNEXT":10000,"tered":10001,"IVE":10002,"onic":10003,"May":10004,"ĠMilitary":10005,"Mark":10006,"Ġlender":10007,"mate":10008,"Ġaboard":10009,"they":10010,"Ġrespondents":10011,"Ġconversion":10012,"Ġsecuring":10013,"Ġentity":10014,"ĠHarbor":10015,"ĠCu":10016,"Ġcats":10017,"ĠACC":10018,"ĠIbrahim":10019,"GL":10020,"Ġinvitation":10021,"Ġcond":10022,"ĠRecords":10023,"ĠAdrian":10024,"Ġbrave":10025,"Ġmineral":10026,"Ġsooner":10027,"Ġsatisfied":10028,"Ġpets":10029,"Ġnotably":10030,"ı":10031,"Ġmarking":10032,"ĠRO":10033,"ĠHaw":10034,"ĠVis":10035,"Ġmarketplace":10036,"ĠNat":10037,"ĠForward":10038,"ĠLeft":10039,"Ġaggravated":10040,"ĠClose":10041,"acey":10042,"Ġlandmark":10043,"Ġdisruption":10044,"ĠChallenge":10045,"ĠDays":10046,"ĠCoun":10047,"ahan":10048,"Ġaides":10049,"South":10050,"ĠDylan":10051,"ĠRavens":10052,"ĠNature":10053,"lli":10054,"Ġdiplomats":10055,"350":10056,"ĠDrake":10057,"tag":10058,"Ġlicensed":10059,"ĠDenmark":10060,"Ġcancel":10061,"Ġinstant":10062,"DI":10063,"Ġpunch":10064,"ĠJenkins":10065,"Ġstrengthening":10066,"des":10067,"-$":10068,"Ġallegation":10069,"Ġsizes":10070,"iza":10071,"Ġmentally":10072,"ĠResidents":10073,"acked":10074,"Ġsensors":10075,",'\"":10076,"illion":10077,"ĠChampion":10078,"Ġexcessive":10079,"Ġhum":10080,"ĠComp":10081,"rend":10082,"ĠLakes":10083,"Ġburst":10084,"Ġtrainer":10085,"Ġclearing":10086,"ĠSilicon":10087,"Ġ350":10088,"DE":10089,"ĠGates":10090,"ĠHorn":10091,"ests":10092,"ĠCourtesy":10093,"Ġbipartisan":10094,"Ġhabits":10095,"ĠAlexa":10096,"walk":10097,"Ġsnapped":10098,"ĠEight":10099,"itis":10100,"zel":10101,"Ġcustoms":10102,"Ġsouthwest":10103,"Ġvary":10104,"Because":10105,"Ġpayout":10106,"Ġaccelerate":10107,"ĠBarr":10108,"tu":10109,"Ġfined":10110,"cost":10111,"ĠTheater":10112,"ĠCorbyn":10113,"Ġstem":10114,"Ġundermine":10115,".;":10116,"Ġstays":10117,"Ġbreakthrough":10118,"Ġturnover":10119,"hot":10120,"Ġtriumph":10121,"Ġpainted":10122,"ĠWinnipeg":10123,"ĠKas":10124,"ĠStuart":10125,"irk":10126,"Am":10127,"Ġtrusted":10128,"aze":10129,"ĠLate":10130,"Ġaccessories":10131,"Ġmemorable":10132,"ĠFool":10133,"Ġrotation":10134,"ĠBulldogs":10135,"ĠChen":10136,"Ġpoised":10137,"ĠMonte":10138,"ĠClarke":10139,"leading":10140,"Ġvenues":10141,"Ġbeneficial":10142,"ĠLiam":10143,"ĠBrothers":10144,"ĠNeed":10145,"Ġconc":10146,"olly":10147,"ĠJulian":10148,"ogue":10149,"Ġfounding":10150,"Ġsidelines":10151,"Ġdeclare":10152,"ĠMember":10153,"Ġexamine":10154,"abs":10155,"Ġboundaries":10156,"ĠBrisbane":10157,"Ġlaunches":10158,"lor":10159,"ĠGa":10160,"Ġthr":10161,"expected":10162,"wal":10163,"ĠBarnes":10164,"Ġclashes":10165,"content":10166,"ĠClemson":10167,"iger":10168,"Mar":10169,"Ġaccord":10170,"Ġsoutheast":10171,"ģ":10172,"ĠStarbucks":10173,"osing":10174,"Ġseasonal":10175,"icking":10176,"Ġloyalty":10177,"Ġtent":10178,"ĠDy":10179,"Ġevident":10180,"Ġlobby":10181,"Ġtours":10182,"Ġbombing":10183,"uations":10184,"Ġrises":10185,"Ġdemonstrations":10186,"ĠWATCH":10187,"pin":10188,"Ġdeb":10189,"ĠDraft":10190,"rog":10191,"Ġseal":10192,"ĠPerformance":10193,"ĠLGBT":10194,"Ġsed":10195,"Ġgig":10196,"nan":10197,"Ġrainfall":10198,"Ġfabric":10199,"Ġmanages":10200,"Ġlifting":10201,"ĠMagazine":10202,"ĠCriminal":10203,"Ġhikes":10204,"Ġcatching":10205,"Ġ1989":10206,"OG":10207,"Ġdisappointment":10208,"Ġir":10209,"ĠEV":10210,"stown":10211,"pass":10212,"120":10213,"Ġmedals":10214,"ĠSimmons":10215,"Ġinaugural":10216,"ĠCorn":10217,"Ġmotorcycle":10218,"lets":10219,"ĠSkype":10220,"ét":10221,"Ġscary":10222,"opp":10223,"thirds":10224,"ĠMohamed":10225,"Ġteenagers":10226,"ANK":10227,"Ġserver":10228,"Ġouts":10229,"Ġdishes":10230,"four":10231,"dr":10232,"ĠOt":10233,"ĠSandy":10234,"ĠShane":10235,"orters":10236,"SH":10237,"Ġtouching":10238,"ĠNike":10239,"ĠHBO":10240,"driving":10241,"Ġplug":10242,"ĠBaseball":10243,"eling":10244,"hn":10245,"ulate":10246,"eed":10247,"ĠChristine":10248,"ĠGlobe":10249,"Ġethics":10250,"ĠTrevor":10251,"iya":10252,"Ġ360":10253,"Ġawaiting":10254,"Ġcounterpart":10255,"Ġsubsidies":10256,"pointers":10257,"Ġspy":10258,"ILL":10259,"Ġtakeover":10260,"ĠBeyond":10261,"Ġsurprisingly":10262,"TION":10263,"ĠSong":10264,"Ġni":10265,"Ġcommonly":10266,"Ġjack":10267,"Ġsubstitute":10268,"ews":10269,"Ġrecalls":10270,"ĠCommons":10271,"Ġsin":10272,"del":10273,"ĠMod":10274,"Ġpressing":10275,"ĠTelevision":10276,"ĠInside":10277,"ª":10278,"Ġbacklash":10279,"Ġcredible":10280,"ĠJenner":10281,"ĠPu":10282,"ĠStevens":10283,"ĠWE":10284,"Last":10285,"Ġinsurers":10286,"ĠJoin":10287,"bled":10288,"digit":10289,"Ġflooded":10290,"ĠShore":10291,"ĠTrophy":10292,"zing":10293,"ĠImmigration":10294,"Ġsuperior":10295,"IAN":10296,"Ġcasino":10297,"Ġenabling":10298,"Ġmeantime":10299,"Ġperformers":10300,"Ġproportion":10301,"Ġlawmaker":10302,"ĠConf":10303,"Ġconvert":10304,"Ġfarmer":10305,"Ġbu":10306,"ĠGE":10307,"ĠRepresentative":10308,"ĠBannon":10309,"ĠHelp":10310,"PT":10311,"formed":10312,"ĠSuperintendent":10313,"Ġfrustrating":10314,"ĠRegister":10315,"ĠPolitical":10316,"Ġboots":10317,"ĠRu":10318,"ĠSha":10319,"Ġinstrument":10320,"tor":10321,"ĠBelt":10322,"ĠWalsh":10323,"Ġrecipe":10324,"ilt":10325,"ĠClean":10326,"iors":10327,"Ġtwenty":10328,"iler":10329,"nder":10330,"Ġwinger":10331,"Ġwheat":10332,"ĠAviation":10333,"Ġcorrupt":10334,"Ġconnectivity":10335,"ĠVen":10336,"order":10337,"esc":10338,"break":10339,"Ġmetals":10340,"Ġtraditionally":10341,"Ġbell":10342,"Ġviolating":10343,"rough":10344,"Ġintroducing":10345,"Ġguided":10346,"ĠMol":10347,"Ġdesert":10348,"ĠBree":10349,"Le":10350,"ĠZone":10351,"ĠGlass":10352,"ĠEUR":10353,"ĠYahoo":10354,"Ġlaps":10355,"Ġdiffer":10356,"ĠHold":10357,"Ġtimely":10358,"Ġsuccessor":10359,"Ġcomic":10360,"Ġbears":10361,"Ġlicence":10362,"Ġreject":10363,"Ġsophisticated":10364,"Too":10365,"Ġobjectives":10366,"ĠId":10367,"urers":10368,"Ġraid":10369,"COM":10370,"Ġelect":10371,"ĠHampshire":10372,"Ġlens":10373,"Ġdesigners":10374,"Ġpresently":10375,"ĠRCMP":10376,"ĠEgyptian":10377,"ĠWalter":10378,"ĠWallace":10379,"Ġ2025":10380,"utics":10381,"ried":10382,"Ġrefuse":10383,"Ġsiblings":10384,"ĠNothing":10385,"Ġdressing":10386,"Ġnerve":10387,"AST":10388,"Ġuncertainties":10389,"Ġtale":10390,"ĠTalk":10391,"Ġissuing":10392,"shot":10393,"ĠTak":10394,"Ġacid":10395,"ĠNintendo":10396,"Ġwash":10397,"pd":10398,"ĠClaire":10399,"ĠScot":10400,"Ġsuits":10401,"ĠBayern":10402,"gest":10403,"Ġapplicable":10404,"Ġinteraction":10405,"ĠEnforcement":10406,"ĠRohingya":10407,"Ġjan":10408,"Ġunited":10409,"ĠCoalition":10410,"Ġlegislators":10411,"Ġdetectives":10412,"ĠSing":10413,"ĠBetween":10414,"ĠPoly":10415,"pool":10416,"mal":10417,"Ġreply":10418,"Ġschemes":10419,"ĠHolmes":10420,"ĠSenators":10421,"ĠVerizon":10422,"Ġwelcoming":10423,"ĠCricket":10424,"ĠMarco":10425,"ĠYears":10426,"ĠLiving":10427,"Ġcounterparts":10428,"ĠParadise":10429,"ĠTrad":10430,"#":10431,"iw":10432,"ĠSoccer":10433,"umbled":10434,"Ġdeceased":10435,"heim":10436,"Ġevaluation":10437,"Ġwrap":10438,"Ġmild":10439,"aji":10440,"ĠUCLA":10441,"ĠNative":10442,"president":10443,"ĠXbox":10444,"Ġenterprises":10445,"ĠSlam":10446,"oga":10447,"Rock":10448,"piece":10449,"ĠColeman":10450,"Ġcomparable":10451,"uba":10452,"Ġprovinces":10453,"ĠFormula":10454,"ipt":10455,"ô":10456,"Ġtick":10457,"ĠIMF":10458,"anch":10459,"atta":10460,"rew":10461,"However":10462,"LS":10463,"etta":10464,"ĠCustoms":10465,"SU":10466,"Ġpublishing":10467,"Ġinch":10468,"Ġkills":10469,"¤":10470,"ĠSus":10471,"ĠBeth":10472,"Ġsteam":10473,"jpg":10474,"pointer":10475,"Ġturnovers":10476,"Ġpowder":10477,"ĠUSB":10478,"ĠWildlife":10479,"ĠDirect":10480,"atively":10481,"ĠFerrari":10482,"Ġpleasure":10483,"ĠMatthews":10484,"Ġski":10485,"ography":10486,"ĠVermont":10487,"ĠMargaret":10488,"ĠMunich":10489,"Ġlayer":10490,"ĠProperty":10491,"Ġeconomics":10492,"ĠCrew":10493,"UK":10494,"Ġunnecessary":10495,"ĠGlasgow":10496,"Ġsealed":10497,"Ġclarity":10498,"Ġsurplus":10499,"ĠCanyon":10500,"ĠApart":10501,"Ġacceptance":10502,"ĠEllis":10503,"uster":10504,"rid":10505,"ĠHawks":10506,"Ġstatewide":10507,"Ġthreaten":10508,"ĠJail":10509,"Ġinclusive":10510,"Ġmud":10511,"Ġpat":10512,"Ġbitter":10513,"Ġalternatives":10514,"Ġaffiliate":10515,"Ġevaluate":10516,"ĠBaby":10517,"Ġperception":10518,"tim":10519,"Ġrefusing":10520,"Ġgrey":10521,"Ġarguably":10522,"Ġfirmly":10523,"ĠDark":10524,"Ġexcuse":10525,"ĠRaymond":10526,"Ġballots":10527,"inton":10528,"Ġ125":10529,"ĠCatherine":10530,"Ġsacks":10531,"ĠDeb":10532,"Ġworkout":10533,"web":10534,"Ġbatteries":10535,"breaking":10536,"ML":10537,"Ġunacceptable":10538,"ĠValentine":10539,"ĠYOU":10540,"ĠRT":10541,"Ġjurisdiction":10542,"Ġexamined":10543,"strom":10544,"ĠPocket":10545,"Ġcement":10546,"Ġuniversal":10547,"ĠOz":10548,"Ġkit":10549,"Ġchurches":10550,"Ġsuburban":10551,"ĠKushner":10552,"ĠDavidson":10553,"Sports":10554,"email":10555,"Ġrealistic":10556,"Ġintend":10557,"ĠGrey":10558,",''":10559,"Ġscholarship":10560,"Ġphilosophy":10561,"Ġwheels":10562,"Ġmotivation":10563,"eway":10564,"match":10565,"ĠDate":10566,"John":10567,"Ġcontrolling":10568,"750":10569,"aven":10570,"Ġfilmed":10571,"Ġ160":10572,"ĠBrock":10573,"ĠDetails":10574,"Ġlogistics":10575,"Ġassumptions":10576,"ĠStep":10577,"Ġfails":10578,"ĠNotre":10579,"Ġjuice":10580,"Ġcounting":10581,"Ġphotograph":10582,"Ġfortunate":10583,"Ġestablishing":10584,"ĠNJ":10585,"ĠWorkers":10586,"ĠQuinn":10587,"ĠHeather":10588,"Ġtimeline":10589,"Ġimported":10590,"ĠNASCAR":10591,"Ġexercises":10592,"Ġsearched":10593,"ĠRalph":10594,"alf":10595,"Ġgene":10596,"Ġdependent":10597,"én":10598,"iate":10599,"ĠBristol":10600,"Ġhung":10601,"Ġtropical":10602,"Ġintensity":10603,"ĠIdaho":10604,"ĠMull":10605,"Ġsuite":10606,"Ġblockchain":10607,"cz":10608,"ovich":10609,"Ġworn":10610,"ĠLE":10611,"AV":10612,"emi":10613,"Ġidentification":10614,"Ġtunnel":10615,"ĠARE":10616,"ĠArm":10617,"Ġoutrage":10618,"Ġtwist":10619,"uka":10620,"ĠGra":10621,"Ġjets":10622,"ĠThus":10623,"Ġcompound":10624,"Ġfinancially":10625,"2019":10626,"asse":10627,"Ġspare":10628,"ĠNoah":10629,"ĠMade":10630,"ĠMom":10631,"Ġphenomenon":10632,"Ġnurses":10633,"Ġoutlined":10634,"Ġpolit":10635,"ĠCarm":10636,"Ġleagues":10637,"Ġmath":10638,"Ġmodified":10639,"Ġwillingness":10640,"ĠAmanda":10641,"Ġgrandfather":10642,"Of":10643,"DR":10644,"Ġdip":10645,"ĠRAM":10646,"ĠChristie":10647,"Ġargues":10648,"ĠEX":10649,"ĠNine":10650,"ĠScroll":10651,"ĠTHIS":10652,"Pro":10653,"Ġkeys":10654,"Ġprocessor":10655,"Ġscam":10656,"ĠTraining":10657,"Ġhoney":10658,"Ĵ":10659,"Ġfacebook":10660,"ĠLegal":10661,"Ġaging":10662,"Ġspiritual":10663,"ĠHost":10664,"Ġlung":10665,"ĠUSC":10666,"Ġdirt":10667,"Ġfe":10668,"after":10669,"ĠDiana":10670,"Ġounce":10671,"date":10672,"ĠFinals":10673,"Ķ":10674,"Ġthorough":10675,"Ġviable":10676,"Ġanytime":10677,"Ġfost":10678,"orter":10679,"ware":10680,"ĠHolland":10681,"ĠMand":10682,"ĠSend":10683,"2013":10684,"ĠVolkswagen":10685,"Ġsuitable":10686,"ifies":10687,"Ġcomedian":10688,"Ġneighbours":10689,"ĠKnow":10690,"Ġcurious":10691,"ĠTwenty":10692,"ĠPrevention":10693,"ĠStephanie":10694,"Ġpilots":10695,"Ġstored":10696,"Ġdire":10697,"Ġfits":10698,"ision":10699,"ĠShell":10700,"Ġshifts":10701,"Ġpepper":10702,"Ġattendees":10703,"ĠName":10704,"hers":10705,"rip":10706,"Ġwatchdog":10707,"andy":10708,"Ġbio":10709,"Ġpublisher":10710,"powered":10711,"ĠCM":10712,"rian":10713,"ĠRand":10714,"wise":10715,"ĠJesse":10716,"Ġladies":10717,"ĠMetropolitan":10718,"ĠMicro":10719,"Ġkicking":10720,"Ġmeg":10721,"Ġclouds":10722,"Ġtrim":10723,"wear":10724,"ĠML":10725,"Ġconsists":10726,"Ġrig":10727,"Ġhonestly":10728,"GS":10729,"ĠNicholas":10730,"Ġcope":10731,"Ġpublish":10732,"working":10733,"bur":10734,"ĠNar":10735,"olds":10736,"aja":10737,"ĠSad":10738,"Ġclicking":10739,"Ġbids":10740,"ĠZuckerberg":10741,"Ġ900":10742,"Ġexam":10743,"ivers":10744,"Ġpray":10745,"Ġreader":10746,"ĠSeth":10747,"inem":10748,"Ġconfront":10749,"stra":10750,"AW":10751,"ĠGian":10752,"Ġaccordance":10753,"Ġinteract":10754,"ĠSharks":10755,"Ġfireworks":10756,"gment":10757,"illy":10758,"Ġconst":10759,"ARY":10760,"Ġprizes":10761,"Ġshoulders":10762,"Ġaccessed":10763,"Ġecosystem":10764,"Ġlicensing":10765,"La":10766,"Ġdedication":10767,"Ġdé":10768,"Ġyouths":10769,"lem":10770,"Ġtoy":10771,"ĠProm":10772,"ounding":10773,"rod":10774,"Ġ1000":10775,"ishes":10776,"Over":10777,"Ġgaps":10778,"Ġmissions":10779,"Ġrailway":10780,"Day":10781,"orp":10782,"ĠSchumer":10783,"Ġeclipse":10784,"Ġshell":10785,"ĠBY":10786,"Many":10787,"ĠRecord":10788,"Ġdrunk":10789,"ayan":10790,"Ġsuggestion":10791,"Ġdefenders":10792,"ĠNewton":10793,"Ġdisputes":10794,"Ġevolution":10795,"Ġcredibility":10796,"ĠTenn":10797,"Ġplain":10798,"size":10799,"cont":10800,"Ġlone":10801,"Ġfingers":10802,"BUR":10803,"ĠInvestigation":10804,"ĠQualcomm":10805,"var":10806,"Ġcountless":10807,"ĠRebecca":10808,"½":10809,"abi":10810,"Ġreflecting":10811,"ĠTurn":10812,"Ġinteractive":10813,"Ġincentive":10814,"second":10815,"offs":10816,"ĠBerkeley":10817,"ĠTexans":10818,"Ġheated":10819,"Ġscorer":10820,"ĠSharif":10821,"Ġmigrant":10822,"west":10823,"ĠHoliday":10824,"Ġwrist":10825,"Ġchairs":10826,"Ġrecommends":10827,"ĠWildcats":10828,"ĠPed":10829,"ĠQuarter":10830,"ĠIV":10831,"ĠArch":10832,"Ġstandings":10833,"Ġbombs":10834,"Ġcapped":10835,"Can":10836,"Ġcaring":10837,"ĠLah":10838,"lim":10839,"Ġdragged":10840,"ĠBeat":10841,"DB":10842,"Ġaired":10843,"Ġjeans":10844,"action":10845,"Ġgenerating":10846,"ĠGir":10847,"risk":10848,"lon":10849,"stage":10850,"âĤ¬":10851,"earing":10852,"ĠTogether":10853,"Ġreun":10854,"ĠCorey":10855,"ĠBak":10856,"Ġprestigious":10857,"Ġapplicants":10858,"here":10859,"ĠMattis":10860,"Ġridiculous":10861,"ĠLess":10862,"Ġrains":10863,"Ġpresenting":10864,"anti":10865,"Ġdisabilities":10866,"Ġapartments":10867,"storm":10868,"ĠHem":10869,"Ġhabit":10870,"ĠRuth":10871,"ĠNPR":10872,"nut":10873,"Ġappreciated":10874,"Ġseparation":10875,"uda":10876,"Ġminus":10877,"ĠPhotos":10878,"Ġblew":10879,"ĠVoice":10880,"Ġrallies":10881,"Ġfond":10882,"ĠTaking":10883,"yt":10884,"FE":10885,"ĠTory":10886,"ressed":10887,"ĠLy":10888,"Ġrocks":10889,"ĠRah":10890,"Ġelementary":10891,"nis":10892,"ĠPresidential":10893,"Ġnutrition":10894,"Ġbaseman":10895,"Ġsuperstar":10896,"ĠWa":10897,"lar":10898,"Ġstaged":10899,"ĠLearn":10900,"Ġbroadcaster":10901,"Ġboasts":10902,"Ġdoubts":10903,"rum":10904,"Ġbare":10905,"cap":10906,"Ġclimbing":10907,"ĠSelect":10908,"ĠCant":10909,"ĠNord":10910,"ĠBeck":10911,"ĠKad":10912,"ello":10913,"Ġenforce":10914,"ĠZe":10915,"ked":10916,"elly":10917,"ĠLED":10918,"ĠOperations":10919,"ĠLuk":10920,"Ġcertificate":10921,"Ġdeter":10922,"Ġspill":10923,"Ġgrain":10924,"league":10925,"Up":10926,"ĠKid":10927,"using":10928,"ĠJays":10929,"Ġoccasionally":10930,"ĠMI":10931,"yes":10932,"Ġdetect":10933,"Ġpropaganda":10934,"Ġneighboring":10935,"sub":10936,"avan":10937,"ĠAstros":10938,"oti":10939,"threatening":10940,"Ġshorter":10941,"INGS":10942,"Ġfeeding":10943,"Ġelevated":10944,"ĠWenger":10945,"Ġundergo":10946,"Ġpsychological":10947,"Ġautom":10948,"NP":10949,"anks":10950,"ĠNokia":10951,"Ġdrones":10952,"Ġrecognised":10953,"Ġheroes":10954,"agen":10955,"Ġparole":10956,"ĠBah":10957,"Ġhomeowners":10958,"ĠSweet":10959,"Ġinstances":10960,"ĠParish":10961,"ĠSL":10962,"Ġunw":10963,"Ġdelicious":10964,"¯":10965,"ĠInvestments":10966,"ĠPhilippine":10967,"inos":10968,"Ġmes":10969,"Ġbite":10970,"Ġcornerback":10971,"ĠHat":10972,"Ġdeserved":10973,"ologists":10974,"[":10975,"Ġwrongdoing":10976,"ĠTrent":10977,"ĠVe":10978,"ĠDeal":10979,"Mr":10980,"Ġovers":10981,"Ġhonors":10982,"ĠITV":10983,"Ġpayroll":10984,"Ġconfused":10985,"Ġelaborate":10986,"ange":10987,"World":10988,"ĠResort":10989,"ilia":10990,"ĠKr":10991,"Ġconclude":10992,"First":10993,"ĠDR":10994,"Ġpeer":10995,"Ġrunway":10996,"ĠPotter":10997,"cons":10998,"bad":10999,"si":11000,"ĠClimate":11001,"ĠHoll":11002,"Ġweighing":11003,"Ġepidemic":11004,"ĠBible":11005,"Ġhon":11006,"Ġrenew":11007,"Ġgambling":11008,"ĠNationals":11009,"itable":11010,"ĠOutlook":11011,"Ġreactions":11012,"ĠCos":11013,"ĠDana":11014,"India":11015,"ĠAirbus":11016,"power":11017,"watch":11018,"Ġstyles":11019,"Ġordinance":11020,"Ġcam":11021,"Ġinvent":11022,"ĠDurant":11023,"Ġexchanged":11024,"Ġyoga":11025,"ĠMichel":11026,"ĠWyoming":11027,"ĠPhase":11028,"ĠHannah":11029,"Ġtem":11030,"Ġfare":11031,"omer":11032,"Ġtrails":11033,"Ġquietly":11034,"ĠFourth":11035,"Ġwise":11036,"Ġappetite":11037,"Ġpedestrian":11038,"Ġfierce":11039,"hin":11040,"ako":11041,"Ġvacant":11042,"Ġdynamics":11043,"Ġbust":11044,"ĠGT":11045,"century":11046,"Ġpermitted":11047,"Ġfog":11048,"Ġrecruitment":11049,"ĠDue":11050,"Ġbro":11051,"Ġsil":11052,"ĠOpp":11053,"Ġphrase":11054,"ĠChip":11055,"ĠBase":11056,"Ġjazz":11057,"Ġenemies":11058,"Ġremainder":11059,"bles":11060,"Ġ105":11061,"ĠGur":11062,"Ġretiring":11063,"ĠCour":11064,"ĠSi":11065,"Ġinevitable":11066,"ĠAdvisory":11067,"ĠCampaign":11068,"ĠPeninsula":11069,"base":11070,"Ġjustify":11071,"inen":11072,"North":11073,"Ġfreezing":11074,"Ġphotography":11075,"Ġappointments":11076,"ĠTree":11077,"Os":11078,"Ġdivide":11079,"ĠMMA":11080,"Ġdeclines":11081,"ĠAbbott":11082,"ACH":11083,"ĠJah":11084,"Ġspr":11085,"Ġskilled":11086,"ĠTry":11087,"ANT":11088,"ael":11089,"ĠMcN":11090,"Ġtariff":11091,"generation":11092,"ĠMans":11093,"Or":11094,"Ġraped":11095,"Ġdisability":11096,"Ġnominations":11097,"Ġhappiness":11098,"ĠLSU":11099,"ĠInterstate":11100,"ĠDance":11101,"ĠMaking":11102,"Ġbailout":11103,"oro":11104,"ĠObviously":11105,"Ġinbox":11106,"football":11107,"hy":11108,"ĠCase":11109,"Ġentertaining":11110,"Ġhardest":11111,"ĠOpposition":11112,"Ġflip":11113,"ĠPirates":11114,"anu":11115,"ĠKlopp":11116,"Ġballistic":11117,"Ġprinted":11118,"ĠNFC":11119,"UST":11120,"Ġglasses":11121,"Ġrum":11122,"ĠDuncan":11123,"hal":11124,"Ġpreview":11125,"BER":11126,"dec":11127,"Ġsustainability":11128,"Ġaff":11129,"Ġhungry":11130,"service":11131,"avi":11132,"Ġsometime":11133,"Ġmod":11134,"ĠLib":11135,"oko":11136,"Ġfundraiser":11137,"Ġcrowded":11138,"mates":11139,"Ġcreativity":11140,"ĠHell":11141,"Ġtreaty":11142,"ĠSoftware":11143,"ĠRandy":11144,"ĠPolish":11145,"sa":11146,"ardi":11147,"Ġcab":11148,"ĠCamera":11149,"Ġlicenses":11150,"Ġ1988":11151,"Ġcontinuous":11152,"Ġpaired":11153,"Ġtally":11154,"Ġgrip":11155,"cho":11156,"Ġsurged":11157,"Ġpodium":11158,"Ġcontrary":11159,"SL":11160,"ĠResearchers":11161,"cing":11162,"Ġmi":11163,"Ġdisputed":11164,"Ġgrades":11165,"Ġseverely":11166,"ĠMcL":11167,"ondo":11168,"Ġshelters":11169,"Ġdomain":11170,"ĠSwitch":11171,"Ġtestify":11172,"case":11173,"omet":11174,"atch":11175,"ĠAff":11176,"Ġcasting":11177,"berger":11178,"Ġintimate":11179,"erc":11180,"plan":11181,"ĠPast":11182,"ĠUt":11183,"Ġapologized":11184,"ĠDet":11185,"alle":11186,"Ġwhilst":11187,"Ġpel":11188,"Ġexecute":11189,"Ġharmful":11190,"ĠRB":11191,"onda":11192,"ĠFul":11193,"II":11194,"Those":11195,"Ġcryptocurrency":11196,"Ġrealise":11197,"ĠAthens":11198,"ĠApplication":11199,"ORD":11200,"Ġmidst":11201,"ĠSem":11202,"Ġmessaging":11203,"Ġcousin":11204,"ĠMarsh":11205,"ĠAlmost":11206,"uto":11207,"wire":11208,"ĠManaging":11209,"Ġsends":11210,"ĠDerby":11211,"Ġpad":11212,"Ġdevoted":11213,"ĠWorking":11214,"ĠWestminster":11215,"Ġdirty":11216,"ements":11217,"ĠLew":11218,"door":11219,"Ġadvisor":11220,"ival":11221,"Ġsubscribe":11222,"Ġcredited":11223,"Ġpressed":11224,"Ġbrick":11225,"Ġrehabilitation":11226,"Ġ\"[":11227,"erry":11228,"Ġtransformed":11229,"arp":11230,"Ġreceivers":11231,"ĠFan":11232,"ĠKris":11233,"ĠCharlottesville":11234,"Ġste":11235,"Ġconstructed":11236,"Ġbroadly":11237,"ĠBetter":11238,"ĠJanet":11239,"Ġenthusiasm":11240,"ĠIrving":11241,"ĠConst":11242,"Everyone":11243,"agn":11244,"ĠCrawford":11245,"Ġregards":11246,"ĠBurns":11247,"Ġjokes":11248,"erg":11249,"ARD":11250,"apped":11251,"Ġtravelled":11252,"ĠPoor":11253,"ĠHolly":11254,"Ġcontainer":11255,"Ġinfected":11256,"Ġlean":11257,"ĠWould":11258,"Ġmagnitude":11259,"ĠDou":11260,"minded":11261,"Ġpastor":11262,"Ġwherever":11263,"ulation":11264,"Ġ1986":11265,"ĠMegan":11266,"Ġgraphic":11267,"Ġtalents":11268,"Ġkn":11269,"ĠEC":11270,"ĠMcM":11271,"ĠKon":11272,"eni":11273,"ĠEsc":11274,"inas":11275,"ĠNom":11276,"Ġchasing":11277,"arl":11278,"ĠHungary":11279,"Ġmainland":11280,"ĠDist":11281,"utes":11282,"Ġrubber":11283,"iat":11284,"ĠMorrison":11285,"ushing":11286,"iny":11287,"Ġcopies":11288,"ĠFat":11289,"agged":11290,"Ġfloating":11291,"ĠCurtis":11292,"Ġfatally":11293,"ĠManuel":11294,"Ġgraduates":11295,"nar":11296,"ĠKenny":11297,"Ġretreat":11298,"Ġretro":11299,"ĠPierre":11300,"listed":11301,"ĠDale":11302,"ding":11303,"Ġintentions":11304,"Ġsentences":11305,"ĠSere":11306,"Ġinvasion":11307,"Ġpremiums":11308,"ĠGardner":11309,"Ġshipments":11310,"Ġcol":11311,"bell":11312,"ilo":11313,"Ġworthy":11314,"Ġinterceptions":11315,"Ġcomplain":11316,"icle":11317,"ĠTah":11318,"ĠMt":11319,"ĠSyracuse":11320,"Since":11321,"aches":11322,"ĠCand":11323,"Ġinteractions":11324,"ĠShawn":11325,"nc":11326,"Ġtheaters":11327,"ART":11328,"Th":11329,"Ġalter":11330,"aley":11331,"imo":11332,"Ġresponders":11333,"kan":11334,"ĠDarren":11335,"Ġdeliveries":11336,"PI":11337,"125":11338,"Ġlaughing":11339,"ĠPatterson":11340,"Ġinfections":11341,"Ġtur":11342,"130":11343,"Ġhackers":11344,"Ġwarn":11345,"Ġfreeze":11346,"Ġscreaming":11347,"ĠEcho":11348,"ĠDom":11349,"MAN":11350,"ĠJoy":11351,"Ġbeneath":11352,"ĠHalf":11353,"Ġpatent":11354,"Ġugly":11355,"Ġlip":11356,"Ġnominees":11357,"ĠGrade":11358,"Ġinfluenced":11359,"Ġabilities":11360,"Ġlimiting":11361,"Ġsmell":11362,"Ġesc":11363,"ĠBernard":11364,"cs":11365,"ĠMyers":11366,"oted":11367,"Black":11368,"Ġlim":11369,"Ġsworn":11370,"ĠBlair":11371,"anes":11372,"ĠEvent":11373,"Ġmature":11374,"Ġpositioned":11375,"Ġerupted":11376,"grand":11377,"ĠTell":11378,"Ġbackdrop":11379,"Ġyeah":11380,"ĠClear":11381,"Ġsignificance":11382,"Ġpatience":11383,"ĠWing":11384,"Ġhorrible":11385,"Ġdeploy":11386,"ipe":11387,"Ġbitcoin":11388,"Ġcommitting":11389,"Ġdismiss":11390,"ĠBlood":11391,"ĠMeyer":11392,"selling":11393,"Ġregarded":11394,"Ġlottery":11395,"ĠLuther":11396,"Ġpipe":11397,"Ġcro":11398,"ĠANC":11399,"ĠSolar":11400,"Ġsimilarly":11401,"Ġham":11402,"ĠHonor":11403,"tar":11404,"gin":11405,"ĠArmstrong":11406,"Ġbrowser":11407,"agon":11408,"via":11409,"Ġentries":11410,"Ġinfl":11411,"Ġgraduation":11412,"Ġalleges":11413,"ĠLoading":11414,"Ġsuperb":11415,"ially":11416,"Ġadministrator":11417,"uls":11418,"Ġartistic":11419,"ĠANGEL":11420,"ĠBang":11421,"Ġfossil":11422,"¨":11423,"Ġpoly":11424,"ĠGuardiola":11425,"ĠPerth":11426,"Ġeducate":11427,"Cl":11428,"Ġcommittees":11429,"Ġforthcoming":11430,"Ġadjustments":11431,"count":11432,"Ġincoming":11433,"brook":11434,"ĠMinneapolis":11435,"Ġgown":11436,"ĠCroatia":11437,"host":11438,"Ġcompetitor":11439,"Ġlyrics":11440,"Ġbelonging":11441,"ĠFrances":11442,"ĠHaley":11443,"ĠBruins":11444,"Ġmask":11445,"ĠPv":11446,"dollar":11447,"Ġbowling":11448,"Ġjewelry":11449,"ĠJulia":11450,"Ġbroadband":11451,"ĠBhar":11452,"ĠArmed":11453,"vy":11454,"government":11455,"kov":11456,"Ġpremises":11457,"Ġjersey":11458,"Ġapplies":11459,"ĠFreeman":11460,"Ġgrows":11461,"ĠEquity":11462,"Ġmaterially":11463,"Ġfigured":11464,"ience":11465,"Ġmajors":11466,"ĠYe":11467,"ĠHey":11468,"oned":11469,"aping":11470,"Ġtoilet":11471,"ĠConnor":11472,"Ġavoiding":11473,"pos":11474,"Once":11475,"ĠRockets":11476,"ĠSnapchat":11477,"Go":11478,"Ġsolidarity":11479,"ĠAffordable":11480,"Ġdial":11481,"ĠOmar":11482,"xt":11483,"ĠVatican":11484,"anta":11485,"ĠSuperior":11486,"Ġbeaches":11487,"ĠKi":11488,"Ã¥":11489,"KY":11490,"Ġgro":11491,"ĠEmpire":11492,"Ġoccurs":11493,"Ġjoked":11494,"Ġquotes":11495,"ĠSaskatchewan":11496,"pert":11497,"Ġmaintains":11498,"olt":11499,"Ġupgrades":11500,"ĠCho":11501,"ĠAlexis":11502,"ĠHundreds":11503,"ĠBud":11504,"Ġcenturies":11505,"ĠInvestor":11506,"ĠGomez":11507,"Ġconceded":11508,"Ġexpressing":11509,"ĠIBM":11510,"Ġadvancing":11511,"ĠDollar":11512,"jer":11513,"Ġexceed":11514,"author":11515,"rist":11516,"seat":11517,"ĠPrimary":11518,"ĠForbes":11519,"ĠAlzheimer":11520,"Ġdevastated":11521,"Ġawful":11522,"ĠStudio":11523,"Ġbullpen":11524,"Ġmobility":11525,"Ġanalyze":11526,"lie":11527,"AFP":11528,"iche":11529,"ĠRoyals":11530,"Ġcoupled":11531,"Ġdug":11532,"ĠRing":11533,"Ġenvironments":11534,"national":11535,"ĠCongo":11536,"Ġalleging":11537,"wn":11538,"ulating":11539,"Ġur":11540,"Ġreaches":11541,"ĠPine":11542,"Ġthreshold":11543,"Ġtournaments":11544,"Ġheating":11545,"ĠGard":11546,"ĠHamas":11547,"Ġ«":11548,"ĠHolding":11549,"Ġpossibilities":11550,"ĠHassan":11551,"ĠMohammad":11552,"Ġoffenders":11553,"Ġautomated":11554,"Ġrealised":11555,"ouse":11556,"building":11557,"ĠDub":11558,"ĠGeneva":11559,"Ġfacial":11560,"ĠRestaurant":11561,"ĠNg":11562,"Ġtot":11563,"Ġgrace":11564,"ĠCP":11565,"Ġposter":11566,"hart":11567,"ĠNi":11568,"Ġreaff":11569,"Ġprov":11570,"Ġ111":11571,"ĠAid":11572,"Ġscrap":11573,"izers":11574,"ogen":11575,"Ġtissue":11576,"Ġvibrant":11577,"Ġrider":11578,"CD":11579,"ĠKitchen":11580,"Ġgenre":11581,"¬":11582,"depth":11583,"kind":11584,"Ġendorsed":11585,"Ġsimultaneously":11586,"Ġintern":11587,"ĠDrag":11588,"Ġembraced":11589,"Ġcounted":11590,"uj":11591,"ĠOg":11592,"Ġphysician":11593,"ĠIR":11594,"IST":11595,"ĠKir":11596,"Ġhacking":11597,"ĠSources":11598,"astic":11599,"growing":11600,"ĠWake":11601,"Ġhint":11602,"Ġcompiled":11603,"Ġreign":11604,"Ġcinema":11605,"Ġboosting":11606,"Ġaccommodation":11607,"ĠEuropa":11608,"Ġsubsidiaries":11609,"Ġclosures":11610,"ĠBil":11611,"ĠBou":11612,"wh":11613,"ĠAw":11614,"FT":11615,"hole":11616,"ĠNova":11617,"ĠNSW":11618,"Ġrap":11619,"Ġencourages":11620,"GR":11621,"ds":11622,"ĠMuk":11623,"ĠSurvey":11624,"ĠReagan":11625,"oning":11626,"Ġneighbouring":11627,"ĠMcCl":11628,"acht":11629,"Ġfinishes":11630,"ĠEsp":11631,"pat":11632,"Ġdestinations":11633,"ĠWagner":11634,"Ġconfronted":11635,"square":11636,"Ġpie":11637,"brand":11638,"hl":11639,"Ġabsent":11640,"Ġsurf":11641,"Ġrifle":11642,"ĠSS":11643,"ĠDeath":11644,"wich":11645,"Ġbeds":11646,"ĠLock":11647,"ĠAgu":11648,"atives":11649,"jee":11650,"Ġoral":11651,"Ġbudgets":11652,"Ġinspiring":11653,"IONS":11654,"works":11655,"Ġspirits":11656,"Ġcabin":11657,"Ġsatisfaction":11658,"Ġvoluntary":11659,"ĠMunicipal":11660,"Ġdeportation":11661,"ĠWriter":11662,"ĠVI":11663,"VERTISEMENT":11664,"/.":11665,"ĠSouthampton":11666,"aces":11667,"ĠHelen":11668,"ĠHum":11669,"110":11670,"Ġgarbage":11671,"through":11672,"Ġkingdom":11673,"MT":11674,"augh":11675,"Ġbizarre":11676,"ĠStarting":11677,"Ġwooden":11678,"ĠProgress":11679,"iron":11680,"sten":11681,"ĠSergio":11682,"ĠHR":11683,"Ġturnout":11684,"ĠAmericas":11685,"ĠSara":11686,"Ġagrees":11687,"apper":11688,"Ġbra":11689,"Ġrecycling":11690,"oom":11691,"Ġflee":11692,"Ġdistinct":11693,"IAL":11694,"aha":11695,"Ġfever":11696,"ĠPartnership":11697,"ĠYu":11698,"ĠPixel":11699,"ĠBlock":11700,"ĠMelissa":11701,"igg":11702,"Ġdecides":11703,"ĠNorman":11704,"Ġmas":11705,"held":11706,"ĠPD":11707,"Ġsheer":11708,"ĠDim":11709,"ĠCass":11710,"Ġcolumnist":11711,"ĠBros":11712,"Ġturnaround":11713,"ĠValue":11714,"ĠBachelor":11715,"awn":11716,"Ġassignment":11717,"ested":11718,"ĠJudiciary":11719,"Ġdiamond":11720,"Ġmus":11721,"Ġindigenous":11722,"lines":11723,"Ġ1984":11724,"igroup":11725,"ict":11726,"ĠJaguars":11727,"Ġlun":11728,"Ġprofiles":11729,"Ġcomputing":11730,"ĠBelgian":11731,"ĠLloyd":11732,"ĠGoing":11733,"Ġdisp":11734,"Ġ1987":11735,"eder":11736,"ĠVin":11737,"Ġgovern":11738,"Ġblend":11739,"ĠSebastian":11740,"ĠMidwest":11741,"iga":11742,"Ġspl":11743,"Ġtopping":11744,"Ġnetworking":11745,"ĠEmer":11746,"Ġoxygen":11747,"ĠInterest":11748,"ĠMoy":11749,"Ġtrader":11750,"Ġbay":11751,"Ġsticking":11752,"ĠMovement":11753,"Ġbidding":11754,"tax":11755,"Ġacademy":11756,"ĠMO":11757,"ĠSpirit":11758,"Ġhealing":11759,"wen":11760,"ĠPrix":11761,"cal":11762,"ĠOperating":11763,"Ġinstantly":11764,"ĠTonight":11765,"Ġsacked":11766,"Ġautomation":11767,"umps":11768,"ĠNey":11769,"March":11770,"ĠBuck":11771,"Ġconcentration":11772,"Here":11773,"Ġtravelers":11774,"Ġprotective":11775,"ĠMoody":11776,"Ġentrepreneur":11777,"Ġfac":11778,"kowski":11779,"Ġpreparations":11780,"Ġdominate":11781,"Ġspray":11782,"Ġdisturbing":11783,"ĠFraser":11784,"ĠCody":11785,"ashi":11786,"ĠPel":11787,"Ġrisky":11788,"Ġawkward":11789,"ĠVA":11790,"ails":11791,"Ġangle":11792,"Ġundergoing":11793,"Ġalbums":11794,"Ġafterwards":11795,"ĠNaw":11796,"uge":11797,"enter":11798,"ĠSussex":11799,"ĠRecently":11800,"Ġlikelihood":11801,"large":11802,"Ġsnaps":11803,"ibr":11804,"ĠMalcolm":11805,"Ġcru":11806,"Ġaltogether":11807,"Ġsetup":11808,"Ġtorture":11809,"Ġfiber":11810,"Ġquarterbacks":11811,"ĠGetting":11812,"ipping":11813,"ĠNorwegian":11814,"ĠMiles":11815,"ĠArnold":11816,"ĠDisease":11817,"Ġtends":11818,"ife":11819,"ĠCaroline":11820,"Ġnavigate":11821,"Ġbrush":11822,"ĠAssociates":11823,"Ġbath":11824,"ĠCenters":11825,"ĠMC":11826,"Ġtaxpayer":11827,"comp":11828,"Ġaccomplish":11829,"ĠTraffic":11830,"ĠBru":11831,"Ġgreenhouse":11832,"ĠMalaysian":11833,"ĠPur":11834,"ased":11835,"ĠKnicks":11836,"aters":11837,"Ġalt":11838,"ICK":11839,"Ġcalculations":11840,"Ġmindset":11841,"unch":11842,"Ġgu":11843,"Ġsteadily":11844,"Ġfiction":11845,"ĠPap":11846,"forming":11847,"ĠActor":11848,"ĠBerry":11849,"imp":11850,"ĠUpper":11851,"Ġassessed":11852,"Ġlawn":11853,"ĠRoh":11854,"Ġclearance":11855,"funded":11856,"Ġpret":11857,"ĠHom":11858,"VS":11859,"ĠTourism":11860,"ĠRy":11861,"ĠGonz":11862,"ĠStudios":11863,"Ġanchor":11864,"Ġrecognise":11865,"Ġcooperate":11866,"enny":11867,"aza":11868,"ĠMeet":11869,"Ġeventual":11870,"SW":11871,"ĠCounsel":11872,"ĠSave":11873,"Ġlucrative":11874,"Ġslim":11875,"ĠGreens":11876,"Ġchemistry":11877,"ĠSheikh":11878,"Ġbridges":11879,"business":11880,"ĠSaf":11881,"ĠGy":11882,"Ġprotocol":11883,"Ġnephew":11884,"ĠBrands":11885,"ĠCulture":11886,"orship":11887,"Ġ(£":11888,"ĠDell":11889,"astics":11890,"Ġproving":11891,"ĠMann":11892,"aca":11893,"Ġindoor":11894,"ĠUganda":11895,"ĠRomney":11896,"ĠStage":11897,"Ġward":11898,"ĠAmber":11899,"haw":11900,"Ġtw":11901,"Ġbullying":11902,"ĠCAR":11903,"Ġassociates":11904,"ĠHopkins":11905,"Ġsuburb":11906,"Ġaggressively":11907,"Ġpostponed":11908,"Ġbas":11909,"Ġburglary":11910,"ĠFound":11911,"Ġfloors":11912,"Any":11913,"Ġjam":11914,"Ġvisibility":11915,"Ġbenefited":11916,"ĠAud":11917,"aying":11918,"iku":11919,"ĠPas":11920,"ĠGPS":11921,"ĠOwens":11922,"Ġreluctant":11923,"ĠOlivia":11924,"ols":11925,"Ġemotion":11926,"ĠHeavy":11927,"Ġhostile":11928,"Ġfavorites":11929,"Ġfeat":11930,"ĠCord":11931,"ĠGO":11932,"Ġindicted":11933,"idal":11934,"ĠIL":11935,"Ħ":11936,"acer":11937,"ICH":11938,"oda":11939,"Ġrecipients":11940,"Ġtribal":11941,"Ġresist":11942,"ĠCritics":11943,"Ġsang":11944,"ĠMath":11945,"ĠBrighton":11946,"ĠKw":11947,"Ġlimitations":11948,"Ġinterception":11949,"onde":11950,"ĠRobertson":11951,"Ġenjoys":11952,"site":11953,"Ġwings":11954,"ĠCeltic":11955,"Ġrelaxed":11956,"Share":11957,"Ġwarrants":11958,"oco":11959,"Ġcritically":11960,"GC":11961,"Ġcute":11962,"Ġlaying":11963,"itude":11964,"ĠMediterranean":11965,"Ġwatches":11966,"Ġdisagree":11967,"ĠReturn":11968,"ARC":11969,"people":11970,"Ġtwelve":11971,"Ġoverdose":11972,"ĠLot":11973,"ĠFROM":11974,"ĠPeters":11975,"Ġadministrators":11976,"Ġslam":11977,"jar":11978,"OH":11979,"ĠInitiative":11980,"Ġteamed":11981,"ĠMajority":11982,"June":11983,"ĠPlaza":11984,"lake":11985,"Ġglimpse":11986,"Ġrings":11987,"Ġos":11988,"Ġmentor":11989,"have":11990,"Ġlanguages":11991,"Ġuncle":11992,"agu":11993,"ĠWine":11994,"ĠCategory":11995,"ĠIng":11996,"Ġcontests":11997,"ĠRosen":11998,"ĠWhatever":11999,"Ġdenying":12000,"ean":12001,"Ġspec":12002,"Ġgrad":12003,"Ġtenants":12004,"show":12005,"ĠGregory":12006,"Ġcontention":12007,"Ġunanimously":12008,"ĠPin":12009,"fa":12010,"ĠPink":12011,"Ġswitched":12012,"acre":12013,"ĠTrading":12014,"VP":12015,"ĠMaple":12016,"Neill":12017,"Ġdiscounts":12018,"alls":12019,"Ġsounded":12020,"Ġrumours":12021,"ĠCre":12022,"hall":12023,"ĠTele":12024,"Ġthankful":12025,"Ġsurveyed":12026,"UB":12027,"Ġdignity":12028,"Ġnod":12029,"Ġmisleading":12030,"ĠTX":12031,"ĠBurke":12032,"Ġmounting":12033,"Ġskies":12034,"Ġbesides":12035,"ĠGarrett":12036,"tha":12037,"Ġintelligent":12038,"Ġtanks":12039,"apping":12040,"ĠRat":12041,"aint":12042,"Ġentertain":12043,"ĠAbdullah":12044,"Ġsink":12045,"ĠLan":12046,"ĠManufacturing":12047,"NFL":12048,"Ġthemes":12049,"ĠHaven":12050,"ĠDavies":12051,"ĠKerr":12052,"ĠLen":12053,"Ġcourtroom":12054,"Ġfailures":12055,"Ġlately":12056,"ĠElectronics":12057,"Ġgorgeous":12058,"Ġnotification":12059,"Ġ2030":12060,"aved":12061,"Ġdeer":12062,"economic":12063,"ĠStatistics":12064,"Ġconfrontation":12065,"Ġgovernors":12066,"ĠHaram":12067,"ĠLGBTQ":12068,"Ġprocessed":12069,"ĠDuchess":12070,"Ġdowns":12071,"Ġpork":12072,"Ġhumor":12073,"ocese":12074,"Ġneeding":12075,"Ġmidterm":12076,"ĠOval":12077,"Ġcorners":12078,"Ġtablets":12079,"eds":12080,"vere":12081,"Ġattacker":12082,"Paul":12083,"pee":12084,"ĠAlice":12085,"Ġrenowned":12086,"Ġ09":12087,"ocking":12088,"Ġcreditors":12089,"ĠPedro":12090,"ĠPhone":12091,"Ġsurveys":12092,"ĠWelsh":12093,"Ġcow":12094,"Ġbuilds":12095,"Ġ000":12096,"ĠAzerbaijan":12097,"ĠYad":12098,"Ġinfant":12099,"Ġmotorists":12100,"Ġpoorly":12101,"Ġmedications":12102,"Ġstupid":12103,"ĠCastro":12104,"user":12105,"antly":12106,"alty":12107,"ĠCond":12108,"issa":12109,"ĠIvan":12110,"Ġcostume":12111,"Ġ08":12112,"Ġhence":12113,"Ġdangers":12114,"Ġbullish":12115,"Life":12116,"Ġflavor":12117,"ĠCharleston":12118,"Ġbikes":12119,"Ġworkshops":12120,"Ġarranged":12121,"Ġcontender":12122,"Ġsequel":12123,"ĠPlant":12124,"Ġdonor":12125,"Ġfactories":12126,"rict":12127,"ellen":12128,"Ġrobots":12129,"ĠWor":12130,"ĠDirectors":12131,"ĠPeru":12132,"Ġqueen":12133,"ĠTimothy":12134,"ĠToo":12135,"Ġobservers":12136,"Ġears":12137,"Ġbel":12138,"link":12139,"uns":12140,"Ġhomers":12141,"Ġadjacent":12142,"Ġconfidential":12143,"Ġstunned":12144,"iden":12145,"illed":12146,"ESS":12147,"Ġconvenient":12148,"ĠLindsey":12149,"por":12150,"upp":12151,"Ġborrow":12152,"ĠAhmad":12153,"ORT":12154,"Ġrelate":12155,"ĠSelf":12156,"ĠVanguard":12157,"utter":12158,"ĠBranch":12159,"ĠBolton":12160,"bat":12161,"Ġoutright":12162,"fighters":12163,"ĠBed":12164,"Ġpes":12165,"inski":12166,"Ġgunshot":12167,"Ġprinting":12168,"ĠSent":12169,"vern":12170,"Ġharvest":12171,"Ġbubble":12172,"Ġrefund":12173,"Ġfuels":12174,"Ġdive":12175,"Ġdiplomat":12176,"Ġpile":12177,"ĠVery":12178,"rot":12179,"ĠSearch":12180,"ĠJoyce":12181,"ĠPruitt":12182,"ĠLevel":12183,"ĠBP":12184,"ĠLac":12185,"had":12186,"Ġexpenditure":12187,"ĠMadd":12188,"Ġpockets":12189,"ĠClippers":12190,"ĠDear":12191,"ĠGive":12192,"Ġhal":12193,"Ġvertical":12194,"Ġwholesale":12195,"what":12196,"ĠSpringfield":12197,"ayed":12198,"ĠSom":12199,"Ġsecrets":12200,"Ġcharts":12201,"iar":12202,"ibility":12203,"LAND":12204,"Ġbearing":12205,"Ġprom":12206,"Ġtab":12207,"Ġsheets":12208,"ĠGL":12209,"Ġendless":12210,"opening":12211,"ĠOwen":12212,"Ġunderneath":12213,"ĠErik":12214,"ĠDACA":12215,"Ġsteering":12216,"Ġfootprint":12217,"ĠRoma":12218,"ĠDucks":12219,"ĠEllen":12220,"ĠProfessional":12221,"ĠGardens":12222,"Ġgoalie":12223,"Ġshine":12224,"Ġturmoil":12225,"Ġhunger":12226,"ĠâĢĭ":12227,"active":12228,"hey":12229,"Ġblessed":12230,"ason":12231,"oping":12232,"ĠThousands":12233,"Ġdose":12234,"ĠLor":12235,"Ġevolved":12236,"Ġcharities":12237,"ĠPE":12238,"ĠRub":12239,"ws":12240,"Ġmist":12241,"ĠShen":12242,"Ġbiological":12243,"ĠTweet":12244,"Ġcollections":12245,"Ġsubstantially":12246,"inner":12247,"Ġbattled":12248,"ĠCong":12249,"Hold":12250,"wp":12251,"Ġwells":12252,"Ġsake":12253,"Ġunrest":12254,"ĠKurt":12255,"Ġripped":12256,"itation":12257,"Ġneighbourhood":12258,"Ġinv":12259,"Ġcad":12260,"ĠCuban":12261,"ĠWealth":12262,"Ġtuition":12263,"Ġdeclaring":12264,"sch":12265,"orne":12266,"Ġwondered":12267,"ĠChaff":12268,"Ġdealer":12269,"ĠNumber":12270,"Mobile":12271,"Ġscratch":12272,"Ġprepares":12273,"ĠSens":12274,"ĠIstanbul":12275,"ĠPanama":12276,"ĠCay":12277,"Ġallocation":12278,"itutional":12279,"Ġhar":12280,"ĠNazi":12281,"ĠSund":12282,"Ġwarehouse":12283,"Ġbackyard":12284,"ĠIll":12285,"Ġunlawful":12286,"ĠReform":12287,"Ġbasement":12288,"ĠHi":12289,"ĠPictures":12290,"Ġtransfers":12291,"ĠSell":12292,"Ġfluid":12293,"Ġambitions":12294,"wife":12295,"Ġintensive":12296,"Ġsteals":12297,"Ġfestive":12298,"ĠHayes":12299,"Ġrestoration":12300,"Ġbranded":12301,"Journal":12302,"Ġmacro":12303,"Ġconsole":12304,"ĠMelania":12305,"ĠRahul":12306,"Ġdisposal":12307,"Ġcult":12308,"Ġpetrol":12309,"Ġtires":12310,"Ġkidnapping":12311,"Ġ115":12312,"Ġswap":12313,"ĠSud":12314,"Ġblown":12315,"ĠHindu":12316,"ĠBeckham":12317,"ĠGul":12318,"Ġfixture":12319,"Ġwisdom":12320,"Ġmines":12321,"fort":12322,"Ġrivers":12323,"ĠCyber":12324,"Ġtouches":12325,"race":12326,"Ġrelax":12327,"Ġcrashes":12328,"Ġconstituency":12329,"Ġ1979":12330,"Ġbureau":12331,"Ġinterface":12332,"Ġdetected":12333,"ĠBio":12334,"Ġhighlighting":12335,"ames":12336,"Ġcorresponding":12337,"great":12338,"Ġgray":12339,"Ġadvantages":12340,"ĠME":12341,"ĠAbbas":12342,"Ġnaked":12343,"rington":12344,".),":12345,"ĠFace":12346,"third":12347,"Ġtranscript":12348,"ples":12349,"Good":12350,"ĠArctic":12351,"Ġtolerance":12352,"reat":12353,"green":12354,"ĠMik":12355,"Ġoutreach":12356,"Ġrolls":12357,"Ġgen":12358,"Ġsupplied":12359,"Ġguarantees":12360,"aug":12361,"Ġsemif":12362,"ounds":12363,"running":12364,"Ġfitting":12365,"ĠRisk":12366,"iveness":12367,"family":12368,"Ġti":12369,"ĠIsaac":12370,"Ġdump":12371,"ĠPatricia":12372,"Ġpassport":12373,"ĠRhode":12374,"Who":12375,"log":12376,"Ġstat":12377,"Ġrat":12378,"ango":12379,"SB":12380,"ĠMaur":12381,"Ġsmiling":12382,"Ġstrikeouts":12383,"Ġpupils":12384,"Ġcomplications":12385,"ĠAdvanced":12386,"ĠMonetary":12387,"ĠTall":12388,"ĠALL":12389,"Ġcontributor":12390,"ĠAdvertising":12391,"Ġhorrific":12392,"Ġcompeted":12393,"ĠKenneth":12394,"Ġhailed":12395,"Ġbones":12396,"Ġbolster":12397,"ĠBoss":12398,"Ġhospitalized":12399,"ĠTelegraph":12400,"ĠIndependence":12401,"Ġdr":12402,"ĠHang":12403,"Ġdocumented":12404,"Ġsubtle":12405,"invest":12406,"Ġbounced":12407,"ĠMAN":12408,"Ġprofession":12409,"Ń":12410,"Ġexcellence":12411,"ĠInspector":12412,"ĠBL":12413,"Ġdisrupt":12414,"ĠWinston":12415,"ĠCommunist":12416,"ĠSharon":12417,"Ġmechanical":12418,"Ġtreats":12419,"Ġdesperately":12420,"ĠIndy":12421,"ĠGi":12422,"ĠComposite":12423,"ĠHeath":12424,"aser":12425,"ĠCardiff":12426,"ilit":12427,"Ġeased":12428,"Ġprospective":12429,"Ġcommissioned":12430,"Ġtire":12431,"Ġalign":12432,"Ġgesture":12433,"Ġweakened":12434,"URE":12435,"SN":12436,"Ġnationals":12437,"Ġrelies":12438,"ĠIRS":12439,"ĠCount":12440,"Ġmedicines":12441,"Ġcongress":12442,"Ġstranger":12443,"Qu":12444,"lessly":12445,"ĠQueens":12446,"ĠAlleg":12447,"uing":12448,"ĠWy":12449,"ĠMiguel":12450,"idi":12451,"Ġcivic":12452,"ĠPetro":12453,"endo":12454,"Obviously":12455,"Ġreflection":12456,"ĠStop":12457,"ĠFitzgerald":12458,"placed":12459,"shore":12460,"Ġcorrectly":12461,"ĠNE":12462,"amy":12463,"ĠCT":12464,"some":12465,"ĠMb":12466,"oi":12467,"ĠHogan":12468,"ĠInnovation":12469,"ĠVilla":12470,"ĠCAN":12471,"ĠCemetery":12472,"into":12473,"Ġquestionable":12474,"Ġcreator":12475,"rug":12476,"Ġsemifinals":12477,"mission":12478,"Ġcle":12479,"ĠWaters":12480,"ĠNixon":12481,"ĠBT":12482,"Ġassuming":12483,"ĠJer":12484,"ĠClay":12485,"pack":12486,"ĠCool":12487,"may":12488,"Ġdecor":12489,"Ġspike":12490,"ĠSomalia":12491,"ĠKarn":12492,"ĠDamascus":12493,"Shares":12494,"Ġsus":12495,"ĠMoss":12496,"Ġ1985":12497,"Ġsuperintendent":12498,"ĠResults":12499,"Ġspends":12500,"prom":12501,"Ġshipped":12502,"Ġlaundering":12503,"ĠLeslie":12504,"Ġmeteor":12505,"Ġabandon":12506,"Ġdeliberately":12507,"ĠSentinel":12508,"Ġfascinating":12509,"Ġenrollment":12510,"ĠExperts":12511,"ĠSimilarly":12512,"ĠCuomo":12513,"bor":12514,"Ġune":12515,"neutral":12516,"Ġhamstring":12517,"Ġnegotiated":12518,"zes":12519,"ĠLeo":12520,"ĠDoctor":12521,"Ġcurriculum":12522,"ĠFocus":12523,"Ġtravels":12524,"Ġbeverage":12525,"ĠIncluding":12526,"tz":12527,"type":12528,"ĠRange":12529,"Ġfloods":12530,"Ġcoached":12531,"Ġdominance":12532,"letico":12533,"ĠRafael":12534,"Ġpredictions":12535,"Ġprosperity":12536,"ĠCav":12537,"Ġclinics":12538,"ĠBanking":12539,"ĠComing":12540,"ears":12541,"ĠKaepernick":12542,"ĠBlvd":12543,"Ġretained":12544,"isions":12545,"Ġko":12546,"Ġensemble":12547,"Ġprecise":12548,"Ġcompact":12549,"MD":12550,"ĠJet":12551,"ached":12552,"ĠTru":12553,"ĠBass":12554,"ĠIcon":12555,"Ġexcluding":12556,"sur":12557,"Ġconstruct":12558,"Ġvoiced":12559,"pan":12560,"Ġinability":12561,"Ġexc":12562,"Ġmate":12563,"Ġtrailing":12564,"Ġsuccessive":12565,"Ġbets":12566,"Ġgauge":12567,"Ġminorities":12568,"ĠIND":12569,"ĠVel":12570,"ĠGP":12571,"oid":12572,"bon":12573,"Ġpred":12574,"Ġdash":12575,"Ġperformer":12576,"Ġoccasional":12577,"aken":12578,"mes":12579,"America":12580,"Ġliver":12581,"Sp":12582,"Big":12583,"Ġwildfires":12584,"ĠJackie":12585,"ĠLed":12586,"ĠFinland":12587,"Ġjurors":12588,"olic":12589,"urance":12590,"ĠEdge":12591,"open":12592,"Ġscenarios":12593,"Ġglory":12594,"entry":12595,"ĠCoffee":12596,"rep":12597,"ĠChand":12598,"ĠVas":12599,"ĠIslamabad":12600,"Ġbur":12601,"ĠFle":12602,"ĠEdition":12603,"Ġshoe":12604,"ï¸ı":12605,"**":12606,"tle":12607,"ĠEb":12608,"keeping":12609,"ĠBasketball":12610,"ĠVon":12611,"ĠCF":12612,"MENT":12613,"amm":12614,"ĠFernando":12615,"Ġcompares":12616,"ĠDouble":12617,"Ġconvictions":12618,"Ġatop":12619,"Ġcops":12620,"Ġremembers":12621,"Ġlacking":12622,"dom":12623,"itate":12624,"ĠBeauty":12625,"Ġdevelops":12626,"ĠGor":12627,"Ġfunctional":12628,"ĠCOUNTY":12629,"ĠUpon":12630,"Ġsprint":12631,"Ġinjection":12632,"Ġminors":12633,"ĠTamil":12634,"ĠGat":12635,"101":12636,"ety":12637,"Ġdrum":12638,"Ġtasked":12639,"Ġpact":12640,"Ġ170":12641,"MR":12642,"ĠRamos":12643,"Ġcandy":12644,"Sc":12645,"iced":12646,"Ġsupermarket":12647,"Ġworrying":12648,"Ġsellers":12649,"ĠTag":12650,".:":12651,"Ġmixture":12652,"oting":12653,"Bl":12654,"ĠLl":12655,"ĠJal":12656,"ican":12657,"ĠBid":12658,"country":12659,"ĠStrategy":12660,"Ġadverse":12661,"Ġplunged":12662,"ĠMit":12663,"Ġstark":12664,"aton":12665,"Ġbooking":12666,"Tr":12667,"Ġcontainers":12668,"Ġvintage":12669,"ĠPit":12670,"Ġsurfaced":12671,"Ġindependently":12672,"Ġdetection":12673,"ĠBeyon":12674,"Ġcasualties":12675,"Ġstabbing":12676,"oved":12677,"Ġbarred":12678,"Ġthereby":12679,"Ġpartnered":12680,"Ġposing":12681,"ĠShannon":12682,"ĠChapel":12683,"Ġtechnically":12684,"uous":12685,"»":12686,"ometer":12687,"Ġwildfire":12688,"share":12689,"heart":12690,"Ġammunition":12691,"Ġthrive":12692,"ĠStre":12693,"GP":12694,"cé":12695,"ĠMonaco":12696,"goal":12697,"ĠUm":12698,"ĠHSBC":12699,"ĠHilton":12700,"ĠViv":12701,"ĠKell":12702,"Ġdecisive":12703,"Ġmotive":12704,"amo":12705,"feld":12706,"ĠWH":12707,"iry":12708,"ulu":12709,"ĠSchneider":12710,"Ġcampaigning":12711,"Ġseparately":12712,"igo":12713,"ĠED":12714,"ĠRamirez":12715,"Ġmetro":12716,"ĠPatel":12717,"ĠChi":12718,"ĠAudi":12719,"Ġcharacteristics":12720,"Ġrestart":12721,"Ġkeyboard":12722,"ĠSD":12723,"his":12724,"biz":12725,"ĠSoft":12726,"ĠGrammy":12727,"Ġcontested":12728,"Ġweekends":12729,"Ġ112":12730,"Ġcycling":12731,"Ġhealthier":12732,"ija":12733,"Ġheader":12734,"Ġemploy":12735,"İ":12736,"Ġshortages":12737,"ĠAsk":12738,"ĠIvanka":12739,"Ġpartisan":12740,"Ġflowing":12741,"Ġcave":12742,"ENS":12743,"Ġups":12744,"read":12745,"ouch":12746,"Ġ102":12747,"Ġforming":12748,"bot":12749,"bie":12750,"Ġenrolled":12751,"Ġconcussion":12752,"Ġaffidavit":12753,"Ġmysterious":12754,"uries":12755,"ĠMang":12756,"Ġauthentic":12757,"Ġmetrics":12758,"ĠTwins":12759,"Ġprep":12760,"IJ":12761,"Ġdesired":12762,"ĠDiv":12763,"wall":12764,"ĠTab":12765,"Ġcompet":12766,"Ġrelied":12767,"Ġinequality":12768,"Ġmanual":12769,"ĠBucks":12770,"agging":12771,"Ġcorporation":12772,"Ġbanner":12773,"Ġgraphics":12774,"Ġaccurately":12775,"ĠMeeting":12776,"Ġconsult":12777,"ser":12778,"Ġprotesting":12779,"Ġhurting":12780,"omed":12781,"tes":12782,"Ġrode":12783,"Ġstartups":12784,"Ġhanding":12785,"ĠNest":12786,"Ġconsistency":12787,"anned":12788,"dem":12789,"ĠLyon":12790,"ĠCompetition":12791,"Ġtricky":12792,"Ġcos":12793,"ĠBengals":12794,"arry":12795,"Ġunderwent":12796,"ĠKit":12797,"à":12798,"uploads":12799,"Ġskate":12800,"Ġ''":12801,"Ġjun":12802,"ĠContent":12803,"focused":12804,"lat":12805,"ĠExp":12806,"ought":12807,"Ġnightmare":12808,"ĠExpect":12809,"Ġprecisely":12810,"ĠMonica":12811,"Ġlobbying":12812,"ĠChester":12813,"ĠInvest":12814,"Former":12815,"Ġimminent":12816,"ĠNL":12817,"Ġcomparing":12818,"ĠChes":12819,"ede":12820,"ĠNobel":12821,"mers":12822,"ĠKin":12823,"ĠBoko":12824,"ount":12825,"Ġthoroughly":12826,"Ġscattered":12827,"sharing":12828,"markets":12829,"ĠMis":12830,"Ġambition":12831,"Ġpreference":12832,"Ġeffectiveness":12833,"rio":12834,"Ġheavyweight":12835,"Ġovert":12836,"anya":12837,"ĠKanye":12838,"ishi":12839,"Ġrewards":12840,"uled":12841,"bach":12842,"Ġemphasized":12843,"Ġapologize":12844,"ĠRecent":12845,"!!":12846,"Ġanimated":12847,"ĠExxon":12848,"Ġfruits":12849,"Ġstripped":12850,"fold":12851,"ĠIndonesian":12852,"ller":12853,"Ġdementia":12854,"Ġkidney":12855,"Ġhalted":12856,"years":12857,"Ġconcerts":12858,"Ġrefers":12859,"ĠFri":12860,"Your":12861,"irl":12862,"Ġleap":12863,"jud":12864,"ĠHugh":12865,"ĠFO":12866,"Ġsore":12867,"Ġkil":12868,"ĠMate":12869,"cci":12870,"Ġsetback":12871,"Ġtightening":12872,"keeper":12873,"ĠAlbany":12874,"Ġpolicymakers":12875,"Ġdisorders":12876,"ĠCBC":12877,"ĠDiaz":12878,"Ġmaps":12879,"Ġroutinely":12880,"Ġverify":12881,"Ġbash":12882,"ĠJinping":12883,"Ġdisasters":12884,"ĠMonroe":12885,"ĠLouise":12886,"JP":12887,"ĠNevertheless":12888,"Ġconcessions":12889,"ĠPog":12890,"going":12891,"ĠFifth":12892,"ĠJill":12893,"ICT":12894,"ĠFM":12895,"ĠSugar":12896,"ĠBarb":12897,"Ġmidway":12898,"Ġtin":12899,"ĠPic":12900,"ĠPL":12901,"Ġleaks":12902,"Ġgrief":12903,"Ġtattoo":12904,"`":12905,"Ġment":12906,"ĠNu":12907,"Ġmarry":12908,"Ġdiving":12909,"Ġ1982":12910,"Ġcoin":12911,"ĠPoc":12912,"Ġstarred":12913,"ĠRiverside":12914,"Ġsidelined":12915,"Ġminers":12916,"STON":12917,"Ġbelongs":12918,"ĠSantos":12919,"ĠTechnical":12920,"aco":12921,"Ġadvise":12922,"Ġstreams":12923,"Ġcooler":12924,"ĠHE":12925,"Ġordering":12926,"ĠTask":12927,"ĠACT":12928,"ĠAnton":12929,"Ġcertification":12930,"ĠLeafs":12931,"ĠTS":12932,"ĠSerbia":12933,"azi":12934,"inks":12935,"ĠEST":12936,"Ġrelay":12937,"°":12938,"Ġdisappearance":12939,"ĠRomania":12940,"Ġoven":12941,"Ġowed":12942,"ĠStrip":12943,"ulated":12944,"UC":12945,"ITE":12946,"bling":12947,"Then":12948,"ppy":12949,"Ġunlimited":12950,"Ġcalories":12951,"Ġmerchandise":12952,"Ġblonde":12953,"ĠSpicer":12954,"performing":12955,"Ġimpl":12956,"Ġplates":12957,"Ġmosque":12958,"Ġdemon":12959,"Ġought":12960,"Ġdumped":12961,"Ġtracked":12962,"even":12963,"Ġstabil":12964,"imet":12965,"ĠLiga":12966,"ugh":12967,"ther":12968,"agar":12969,"Ġarchitect":12970,"Ġallocated":12971,"ĠJoey":12972,"Ġmarathon":12973,"master":12974,"ĠBert":12975,"Ġast":12976,"ĠEbola":12977,"ĠConservation":12978,"nic":12979,"Ġparallel":12980,"Ġinmate":12981,"Ġlocate":12982,"Ġdistribute":12983,"guard":12984,"Ġtackling":12985,"ential":12986,"Ġvi":12987,"Ġcups":12988,"Ġrhythm":12989,"Ġendured":12990,"ĠHub":12991,"ois":12992,"ĠLiberals":12993,"ĠRedskins":12994,"ĠEP":12995,"ĠKnox":12996,"fr":12997,"Ġmassacre":12998,"oka":12999,"Ġcompl":13000,"raft":13001,"ĠPublished":13002,"Ġattraction":13003,"ĠStephens":13004,"ility":13005,"ĠPul":13006,"ĠCapt":13007,"Ġexploded":13008,"Ġexceeded":13009,"lying":13010,"Ġcal":13011,"Mart":13012,"Ġpaintings":13013,"inate":13014,"ĠBrendan":13015,"Ġfortune":13016,"onductor":13017,"Ġphysicians":13018,"ĠStudy":13019,"ĠBul":13020,"ĠModern":13021,"HD":13022,"ĠBour":13023,"Ġtying":13024,"Ġ1967":13025,"Ġlighter":13026,"Ġtoss":13027,"inspired":13028,"Ġgreeted":13029,"Ġcycl":13030,"Ġverified":13031,"Ġmerit":13032,"sign":13033,"lder":13034,"Ġdebts":13035,"ĠSnyder":13036,"Ġamendments":13037,"Ġindicators":13038,"ĠDortmund":13039,"then":13040,"ĠListen":13041,"ĠFB":13042,"ref":13043,"ĠIoT":13044,"ĠBrewers":13045,"ĠLeadership":13046,"ĠNicolas":13047,"ĠBody":13048,"Ġsam":13049,"ĠAdvisor":13050,"Ġcord":13051,"Ġabuses":13052,"ĠPortuguese":13053,"Ġflown":13054,"VR":13055,"Ġconsumed":13056,"Ġreass":13057,"Ġalien":13058,"Ġrivalry":13059,"ĠREPORT":13060,"ĠRush":13061,"Ġdirecting":13062,"Ġsearches":13063,"ĠHP":13064,"ĠRoll":13065,"ĠFay":13066,"ĠClare":13067,"Ġhaul":13068,"Ġriot":13069,"Ġsettlements":13070,"Ġnorm":13071,"Ġaccelerated":13072,"ĠLok":13073,"Ġclever":13074,"Ġhyd":13075,"Ġstats":13076,"ĠHull":13077,"kers":13078,"Ġbuys":13079,"uter":13080,"Ġfue":13081,"https":13082,"UD":13083,"Ġisolation":13084,"Ġsuspend":13085,"ĠRules":13086,"ĠCircle":13087,"ĠHopefully":13088,"played":13089,"âĢ³":13090,"ĠPRE":13091,"sim":13092,"edd":13093,"ĠProperties":13094,"Ġbeans":13095,"Ġrevive":13096,"ĠBir":13097,"oug":13098,"Ġmob":13099,"Ġshowdown":13100,"iman":13101,"Ġpap":13102,"Ġvol":13103,"wu":13104,"Ġdiver":13105,"Ġpill":13106,"ĠMarlins":13107,"ĠLamar":13108,"Ġpersistent":13109,"Ġcondolences":13110,"ĠThor":13111,"Ab":13112,"Ġimpress":13113,"ĠRaptors":13114,"Ġreferences":13115,"Ġstiff":13116,"ĠBash":13117,"eding":13118,"Ġmurders":13119,"ĠGene":13120,"ĠManila":13121,"Ġbrokers":13122,"Ms":13123,"start":13124,"ĠDhabi":13125,"etz":13126,"Ġsubmission":13127,"ĠSchmidt":13128,"ĠPersonal":13129,"ĠBeverly":13130,"ĠMovie":13131,"ĠLamb":13132,"Ġplacement":13133,"Ġfolk":13134,"Ġfrequency":13135,"Ġplanted":13136,"Ġtwins":13137,"prov":13138,"rec":13139,"Ġpermanently":13140,"Ġcoordination":13141,"ĠCart":13142,"Ġobstacles":13143,"Ġliterature":13144,"Ġtu":13145,"Ġchill":13146,"ĠReserved":13147,"Ġlovers":13148,"ĠOutside":13149,"Ġslideshow":13150,"ĠGru":13151,"Ġty":13152,"Ġsalad":13153,"Ġlaboratory":13154,"ĠHolt":13155,"Ġ103":13156,"urb":13157,"ĠOrganisation":13158,"ĠAndrews":13159,"Ġrecipient":13160,"arch":13161,"Ġbleeding":13162,"ĠPand":13163,"Ġoverturned":13164,"Ġlistened":13165,"Ġclause":13166,"Ġnationalist":13167,"Ġresumed":13168,"ĠCout":13169,"ĠPride":13170,"Ġlayers":13171,"ĠBella":13172,"Ġreversed":13173,"Ġpriest":13174,"ĠFX":13175,"Ġalbeit":13176,"Ġhalfway":13177,"Ġcotton":13178,"ĠCarey":13179,"ĠTE":13180,"OCK":13181,"Ġbuck":13182,"ributes":13183,"ea":13184,"Ġfancy":13185,"ĠBuc":13186,"Ġbans":13187,"uters":13188,"Ġliabilities":13189,"ĠSou":13190,"ĠBernie":13191,"Ġintervene":13192,"food":13193,"ĠNDP":13194,"Ġinsist":13195,"Ġcontracted":13196,"hawk":13197,"),\"":13198,"ĠDawn":13199,"Ġmol":13200,"Ġcommissioners":13201,"Ġstranded":13202,"Ġoverwhelmed":13203,"Ġrecipes":13204,"Ġva":13205,"Ġrad":13206,"Ġscare":13207,"rez":13208,"Ġeliminating":13209,"Ġresc":13210,"ĠBreak":13211,"chn":13212,"Ġdelight":13213,"iot":13214,"Ġfreely":13215,"TI":13216,"ĠBluetooth":13217,"ĠMonth":13218,"ĠFlor":13219,"ĠFreddie":13220,"Ġtrailed":13221,"Ġinvestigative":13222,"Ġimposing":13223,"Ġattracting":13224,"awk":13225,"ĠSherman":13226,"Ġsucceeded":13227,"Ġvent":13228,"Ġreconciliation":13229,"ĠCel":13230,"ĠThroughout":13231,"ĠDowntown":13232,"ĠBrother":13233,"Ġtraditions":13234,"Ġmir":13235,"Ġstamp":13236,"tery":13237,"etti":13238,"isch":13239,"tic":13240,"Ġbanning":13241,"loss":13242,"ĠSpeedway":13243,"Ġstalled":13244,"ĠEN":13245,"ASH":13246,"thing":13247,"ĠAppeals":13248,"rac":13249,"Ġdistress":13250,"ĠConservatives":13251,"ĠPremium":13252,"usa":13253,"Ġslump":13254,"imm":13255,"ĠSupp":13256,"ĠWong":13257,"Ġdistant":13258,"Ġ104":13259,"Ġtide":13260,"ĠNorfolk":13261,"ĠYang":13262,"Ġsmashed":13263,"ĠBarrett":13264,"inho":13265,"Ġrobbed":13266,"ĠFarmers":13267,"filled":13268,"BT":13269,"Ġautumn":13270,"Ġtemple":13271,"ĠJacobs":13272,"Ġprecipitation":13273,"ĠHours":13274,"ĠFlight":13275,"Ġbeside":13276,"ĠOre":13277,"!)":13278,"ĠTurnbull":13279,"Ġpig":13280,"Ġcooling":13281,"Ġservers":13282,"oriented":13283,"Ġlocks":13284,"ĠSears":13285,"aving":13286,"ĠQuick":13287,"ĠGlob":13288,"ĠMining":13289,"Ġhorizon":13290,"arians":13291,"ĠOm":13292,"writing":13293,"Ġbelieving":13294,"Ġbon":13295,"Ġmounted":13296,"Ġpunt":13297,"ucci":13298,"uzz":13299,"cul":13300,"Ġkiss":13301,"ĠOnt":13302,"ĠCyprus":13303,"Ġrelying":13304,"Ġpiano":13305,"Ġcure":13306,"Ġcontinuously":13307,"ĠNobody":13308,"ĠBund":13309,"osis":13310,"ĠAurora":13311,"ĠBach":13312,"ĠKendall":13313,"Ġechoed":13314,"iable":13315,"Ġconscious":13316,"Ġmonster":13317,"omo":13318,"proof":13319,"ĠNate":13320,"Ġfilmmaker":13321,"ĠNaj":13322,"Ġvendor":13323,"ĠFoot":13324,"ĠChang":13325,"ĠFest":13326,"Ġselfie":13327,"Ġenters":13328,"ĠConor":13329,"ĠMosul":13330,"ĠWHAT":13331,"Ġwa":13332,"ĠGamb":13333,"osta":13334,"Ġcautioned":13335,"ĠTucker":13336,"ĠAirways":13337,"Ġvisitor":13338,"Ġ·":13339,"ĠRevolution":13340,"aching":13341,"Ġearliest":13342,"ĠQuality":13343,"Ġshorts":13344,"ube":13345,"ĠOperation":13346,"ĠSabha":13347,"Ġstrengths":13348,"ikes":13349,"Ġsexy":13350,"Ġrot":13351,"ibles":13352,"Ġcolours":13353,"THE":13354,"ailed":13355,"Ġwoke":13356,"ĠEmbassy":13357,"Ġinfamous":13358,"rov":13359,"State":13360,"âĢ¦.":13361,"Ġpond":13362,"Ġcapt":13363,"fore":13364,"De":13365,"Ġedited":13366,"self":13367,"Hey":13368,"Ġportrait":13369,"ĠManufact":13370,"ĠStand":13371,"Ġcontenders":13372,"':":13373,"acker":13374,"Ġwithdrawn":13375,"ĠBraves":13376,"ĠHosp":13377,"changing":13378,"ĠBag":13379,"Ġadjustment":13380,"ĠCousins":13381,"ĠAAP":13382,"Ġfi":13383,"Ġoutdoors":13384,"Ġlacked":13385,"BM":13386,"ĠWHO":13387,"ĠPST":13388,"ĠLuck":13389,"Ġassisting":13390,"ĠGround":13391,"ĠTeen":13392,"ĠOle":13393,"Ġembarrassing":13394,"ĠWalt":13395,"ĠVision":13396,"ĠFal":13397,"ĠZoo":13398,"ĠWorth":13399,"ĠFloyd":13400,"ĠGujarat":13401,"Ġtipped":13402,"Ġfam":13403,"ĠDad":13404,"Ġworship":13405,"Ġtyre":13406,"Ġrebuilding":13407,"Ġqualities":13408,"ĠLives":13409,"Ġbeats":13410,"Ġ450":13411,"Ġexisted":13412,"ĠGeorg":13413,"Ġpoured":13414,"rows":13415,"ĠOx":13416,"ĠSid":13417,"Ġmac":13418,"Ġteaches":13419,"ĠEli":13420,"alla":13421,"Ġdownside":13422,"ĠBend":13423,"non":13424,"ĠArmenia":13425,"Ġcultures":13426,"ĠMae":13427,"Ġduration":13428,"ĠAthletics":13429,"Ġjuvenile":13430,"Ġlid":13431,"Ġbankers":13432,"Ġoverview":13433,"wy":13434,"Ġorbit":13435,"Vs":13436,"because":13437,"Ps":13438,"ĠFran":13439,"Ġtouring":13440,"Ġwary":13441,"Ġ106":13442,"Ġlaser":13443,"ĠVij":13444,"âĦ¢":13445,"Ġsurrender":13446,"press":13447,"rees":13448,"NO":13449,"ĠShortly":13450,"ĠKor":13451,"edu":13452,"Ġhatred":13453,"Ġtee":13454,"Ġfamously":13455,"Ġkeeper":13456,"ND":13457,"Ġreduces":13458,"HC":13459,"Ġhay":13460,"Ġunnamed":13461,"ĠTes":13462,"Ġattackers":13463,"ĠFew":13464,"ĠRichards":13465,"Ġ1968":13466,"Ġspeeches":13467,"Ġcybersecurity":13468,"ĠInfrastructure":13469,"Ġ07":13470,"ENCE":13471,"uties":13472,"Ġanxious":13473,"ĠGang":13474,"Ġannouncements":13475,"lette":13476,"oret":13477,"ĠRockies":13478,"ĠEmployees":13479,"ĠThrones":13480,"Ġhugely":13481,"Ġclin":13482,"ĠHob":13483,"Ġfraction":13484,"ĠOfficial":13485,"ĠMariners":13486,"ĠElse":13487,"Ġsanctuary":13488,"ĠPhotograph":13489,"Ġreopen":13490,"lf":13491,"hm":13492,"vest":13493,"Ġspeeding":13494,"Ġtooth":13495,"ĠShi":13496,"ĠTitle":13497,"ĠMes":13498,"ĠJobs":13499,"fair":13500,"ĠDanish":13501,"ĠMalik":13502,"Ġlaughed":13503,"Ġnavy":13504,"ĠActress":13505,"ĠWilliamson":13506,"overs":13507,"Ġreckless":13508,"Ġjo":13509,"otic":13510,"Ġassaulting":13511,"Ġpri":13512,"ĠPi":13513,"Ġlesser":13514,"Ġtit":13515,"Ġdat":13516,"Ġnail":13517,"ĠMarathon":13518,"ĠGren":13519,"ĠDol":13520,"Ġjointly":13521,"Ġamended":13522,"mine":13523,"ĠBashar":13524,"ĠHyundai":13525,"Ġuncovered":13526,"Ġeducated":13527,"atti":13528,"pres":13529,"ĠBRE":13530,"Ġya":13531,"Bank":13532,"odd":13533,"lit":13534,"ĠLinks":13535,"Ġswitching":13536,"itte":13537,"ĠSind":13538,"erved":13539,"Ġ**":13540,"Ġpositively":13541,"Ġfrankly":13542,"Ġrevenge":13543,"ĠTrinity":13544,"ĠCDC":13545,"Ġthreatens":13546,"Ġhammer":13547,"NET":13548,"ĠMut":13549,"Ġsy":13550,"Ġunidentified":13551,"icken":13552,"Ġdrills":13553,"Ġtense":13554,"Ġforeigners":13555,"OST":13556,"Ġethical":13557,"ĠDurham":13558,"ĠQual":13559,"Ġterritories":13560,"Ġid":13561,"hor":13562,"enders":13563,"Mc":13564,"OV":13565,"percent":13566,"Ġdom":13567,"Ġupward":13568,"Ġamb":13569,"Ġvisas":13570,"zan":13571,"Ãĥ":13572,"Ġundocumented":13573,"Ġsuburbs":13574,"Ġhydro":13575,"ĠJob":13576,"ĠAdelaide":13577,"oya":13578,"ĠSR":13579,"ĠMick":13580,"Ġconsolidation":13581,"Ġemotionally":13582,"ĠHop":13583,"Her":13584,"Ġloses":13585,"ĠMoto":13586,"eled":13587,"Ġregulated":13588,"ental":13589,"Ġencountered":13590,"Ġhop":13591,"ĠTrafford":13592,"Ġsticks":13593,"Ġveto":13594,"Ġexpose":13595,"Ġstretched":13596,"fin":13597,"inance":13598,"chair":13599,"ĠGareth":13600,"ĠPil":13601,"ĠHammond":13602,"Ġserial":13603,"omy":13604,"Ġcellphone":13605,"ĠClara":13606,"Ġreacted":13607,"ĠNic":13608,"ĠHomes":13609,"ĠBroadcasting":13610,"ĠFut":13611,"ĠSupply":13612,"assing":13613,"ĠNewman":13614,"Ġcharitable":13615,"ĠClayton":13616,"Ġsovereignty":13617,"Ġconvincing":13618,"ĠPrincipal":13619,"ĠHigher":13620,"ĠCut":13621,"ĠCarrie":13622,"ĠSpot":13623,"Sometimes":13624,"ĠJar":13625,"ĠConsider":13626,"ieu":13627,"Ġrefinery":13628,"Ġbloody":13629,"wheel":13630,"Ġcryptocurrencies":13631,"Fund":13632,"ĠSunderland":13633,"ĠEvents":13634,"âĢĭ":13635,"Ġaccidentally":13636,"deep":13637,"Ġfranc":13638,"bec":13639,"ĠHartford":13640,"Ġstellar":13641,"wright":13642,"kick":13643,"UG":13644,"ĠBeast":13645,"Ġrefusal":13646,"ĠRoberto":13647,"ĠDixon":13648,"ĠDiane":13649,"name":13650,"asts":13651,"ĠCharter":13652,"Ġfueled":13653,"Ġcontents":13654,"Ġaccessing":13655,"Ġtroubles":13656,"Ġtops":13657,"Ġdebuted":13658,"icating":13659,"Ġinvestigator":13660,"Ġsubscribing":13661,"Ġcoordinated":13662,"ĠFil":13663,"six":13664,"teen":13665,"Ġwithdrew":13666,"ĠGilbert":13667,"Ġ1983":13668,"arsity":13669,"Ġimagination":13670,"Ġhandgun":13671,"ĠAlibaba":13672,"Ġbug":13673,"Ġ107":13674,"ĠCOMP":13675,"ĠSomething":13676,"Ġreliability":13677,"ĠFCC":13678,"ĠFowler":13679,"Ġsingled":13680,"nom":13681,"Ġknocking":13682,"Ġmeddling":13683,"Ġdetermining":13684,"reports":13685,"Ġshade":13686,"ĠSN":13687,"anto":13688,"Ġcomplaining":13689,"ĠNan":13690,"WS":13691,"Ġyoungsters":13692,"Il":13693,"ĠKaw":13694,"ĠProp":13695,"ĠCell":13696,"ĠHurricanes":13697,"Ġpublicity":13698,"ĠXin":13699,"rial":13700,"ICO":13701,"Ġsupervision":13702,"ĠSpotify":13703,"ĠNewport":13704,"Ġprince":13705,"anche":13706,"Ġsubscriber":13707,"ĠVic":13708,"ACT":13709,"ĠRaf":13710,"ĠActing":13711,"Ġcollusion":13712,"pet":13713,"isl":13714,"Ġcommerce":13715,"Health":13716,"ĠAbraham":13717,"pri":13718,"Ġlightweight":13719,"Ġinsurer":13720,"Like":13721,"Ġhelmet":13722,"Ġevac":13723,"look":13724,"ĠNaval":13725,"160":13726,"ĠFleet":13727,"vol":13728,"Ġexpired":13729,"ĠKlein":13730,"ĠEmmy":13731,"ABLE":13732,"ĠMorocco":13733,"ĠTrip":13734,"uted":13735,"Ġnos":13736,"ĠVista":13737,"mas":13738,"ĠRocky":13739,"ĠFlint":13740,"enberg":13741,"ĠBrow":13742,"Ġsignatures":13743,"Ġpolar":13744,"ajo":13745,"Ġendorsement":13746,"Ġreservations":13747,"LIN":13748,"anny":13749,"elli":13750,"last":13751,"Ġoversee":13752,"cm":13753,"ĠOilers":13754,"Are":13755,"Ġjudiciary":13756,"onte":13757,"ĠTrack":13758,"Ġsupervisor":13759,"erk":13760,"isher":13761,"Ġintact":13762,"Ġslid":13763,"icals":13764,"paid":13765,"ĠMAR":13766,"lement":13767,"ĠLiu":13768,"ĠLarge":13769,"ĠWings":13770,"pect":13771,"ĠRum":13772,"Ġanalyzed":13773,"Ġemploys":13774,"arte":13775,"ims":13776,"ĠEventually":13777,"Ġaffiliated":13778,"Ġhospitality":13779,"ĠSprint":13780,"Ġresolutions":13781,"Ġliquor":13782,"ĠNAFTA":13783,"ANY":13784,"Ġradiation":13785,"ĠProv":13786,"Ġpause":13787,"ĠTMZ":13788,"Ġelbow":13789,"Ġresilience":13790,"ĠParents":13791,"mus":13792,"ĠSafe":13793,"Ġinterpretation":13794,"Ġraced":13795,"IND":13796,"KR":13797,"Ġhinted":13798,"ĠErin":13799,"ĠBahrain":13800,"Ġcredentials":13801,"eless":13802,"Ġprocurement":13803,"ĠWebb":13804,"ĠLowe":13805,"ĠNak":13806,"ĠLearning":13807,"zh":13808,"Ġdipped":13809,"ĠSuite":13810,"Ġmisdemeanor":13811,"ALE":13812,"Ġstrengthened":13813,"ĠSophie":13814,"Ġconfirms":13815,"Ġrac":13816,"gey":13817,"Ġshootout":13818,"Ġble":13819,"Ġcircles":13820,"ĠChef":13821,"Ġcomprised":13822,"ĠSantiago":13823,"Ġfeud":13824,"beat":13825,"Ġstaffers":13826,"Ġacute":13827,"ski":13828,"Ġpolled":13829,"ĠKur":13830,"ĠJen":13831,"ĠUltimately":13832,"anded":13833,"ĠHoney":13834,"Ġannounces":13835,"Ġamateur":13836,"around":13837,"Ġfunctioning":13838,"group":13839,"ĠSqu":13840,"Where":13841,"Ġvoid":13842,"ĠSandra":13843,"isers":13844,"Ġhelicopters":13845,"ĠGym":13846,"ĠWol":13847,"mouth":13848,"Ġsubjected":13849,"ici":13850,"ually":13851,"ĠWash":13852,"ĠLindsay":13853,"ĠVers":13854,"Ġjumps":13855,"Ġneglect":13856,"ĠKuwait":13857,"fund":13858,"ĭ":13859,"ather":13860,"lly":13861,"ei":13862,"Although":13863,".''":13864,"Ġunhappy":13865,"Ġpills":13866,"Ġmagical":13867,"Ġdro":13868,"Ġinviting":13869,"ĠJohnston":13870,"oving":13871,"450":13872,"ĠMerc":13873,"Ġadmitting":13874,"Ġinsisting":13875,"ĠCru":13876,"ĠResource":13877,"oir":13878,"Ġcomplexity":13879,"ĠRoth":13880,"ĠCher":13881,"July":13882,"raf":13883,"Ġaggregate":13884,"Ġhelm":13885,"uclear":13886,"olan":13887,"Ġoffenses":13888,"ĠWolves":13889,"ĠFu":13890,"ĠPierce":13891,"Ġemailed":13892,"ĠStra":13893,"Ġpedestrians":13894,"ĠER":13895,"ĠConway":13896,"Ġblowing":13897,"CLOSE":13898,"hab":13899,"ĠGreene":13900,"Ġconfessed":13901,"ĠTorres":13902,"ĠHolocaust":13903,"Ġrepay":13904,"Ġdemonstrates":13905,"ĠPool":13906,"gent":13907,"Ġdeleted":13908,"Ġ$$":13909,"ĠSO":13910,"Ġdri":13911,"ĠNeg":13912,"ĠVP":13913,"ĠPF":13914,"ĠPrep":13915,"Ġorganizing":13916,"icker":13917,"Ġmanufactured":13918,"enson":13919,"adas":13920,"Ġwines":13921,"Ġmachinery":13922,"Ġspecialists":13923,"ĠDetective":13924,"ĠDL":13925,"Op":13926,"Ġquicker":13927,"ĠPenguins":13928,"Engine":13929,"zone":13930,"Ġsequence":13931,"ĠLost":13932,"Ġwarmer":13933,"ĠEthiopia":13934,"Ġaffirmed":13935,"fest":13936,"resses":13937,"Ġsoap":13938,"Ġbooth":13939,"Ġnotorious":13940,"amin":13941,"Ġpursued":13942,"ĠCer":13943,"ĠSB":13944,"Ġlivestock":13945,"Ġtrace":13946,"Ġrespects":13947,"arden":13948,"April":13949,"Ġ128":13950,"ĠSaid":13951,"ennial":13952,"Ġnamely":13953,"ĠBot":13954,"Ġ108":13955,"ĠLem":13956,"nell":13957,"Ġconfirming":13958,"Ġlogged":13959,"Ġprofound":13960,"elo":13961,"ĠChambers":13962,"RT":13963,"Ġnewer":13964,"Ġsideline":13965,"ĠCardinal":13966,"este":13967,"Ġnarrowly":13968,"Ġcompromised":13969,"Ġpolicing":13970,"Ġporn":13971,"Ġarc":13972,"Ġlearnt":13973,"INE":13974,"step":13975,"ĠDomin":13976,"Ġwaist":13977,"Ġboycott":13978,"mitted":13979,"iffs":13980,"ground":13981,"ĠMaterials":13982,"Ġceasefire":13983,"Right":13984,"ĠZen":13985,"estyle":13986,"Thank":13987,"ĠOnePlus":13988,"ĠMLS":13989,"Ġconstituents":13990,"oster":13991,"ĠProsecutor":13992,"Ġpriorit":13993,"ĠDebbie":13994,"ĠExpand":13995,"uv":13996,"Ġintegrate":13997,"Ġimmun":13998,"Ġdisciplinary":13999,"ĠImm":14000,"Ġja":14001,"Ġgardens":14002,"ĠHim":14003,"obe":14004,"Ġhitter":14005,"Ġbullets":14006,"Ġevolving":14007,"ĠScientists":14008,"Michael":14009,"ĠDO":14010,"Ġunbelievable":14011,"Ġlooming":14012,"Ġdownturn":14013,"Ġmentality":14014,"Ġreopened":14015,"Ġash":14016,"ĠChapman":14017,"Ġloop":14018,"ĠUT":14019,"ĠTier":14020,"Ġunaware":14021,"Ġgratitude":14022,"Ġperforms":14023,"olk":14024,"Ġ\"(":14025,"Ġlacks":14026,"Ġinstructed":14027,"ĠRecreation":14028,"sample":14029,"Ġrequesting":14030,"Canada":14031,"Ġsupposedly":14032,"ĠHardy":14033,"Ġholder":14034,"change":14035,"ĠDominic":14036,"ĠXavier":14037,"Ġlig":14038,"Ġcandid":14039,"ĠRab":14040,"Ġconferences":14041,"ĠBurton":14042,"Dr":14043,"Ġmunicipalities":14044,"Ġcrushed":14045,"Ġseekers":14046,"ĠCitizens":14047,"Ġheightened":14048,"ĠCasino":14049,"Ġdesktop":14050,"Ġwhoever":14051,"ĠImpact":14052,"Ġcocktail":14053,"Ġphilanthrop":14054,"ĠSAN":14055,"ĠPreston":14056,"Ġobesity":14057,"Ġrestrict":14058,"ĠKab":14059,"ĠProvidence":14060,"Ġscar":14061,"ĠChart":14062,"Ġbosses":14063,"ĠRate":14064,"Ġsav":14065,"pay":14066,"Ġtransplant":14067,"ĠNoble":14068,"child":14069,"Ġconclusions":14070,"FI":14071,"Ġsack":14072,"Ġexperimental":14073,"holder":14074,"oca":14075,"herty":14076,"ĠMT":14077,"Ġcatcher":14078,"LY":14079,"Ġgrams":14080,"reet":14081,"Ġadaptation":14082,"Ġhumble":14083,"Ġbot":14084,"Ġidentical":14085,"ication":14086,"ifer":14087,"ĠCrow":14088,"Ġregain":14089,"ĠLightning":14090,"Ġkg":14091,"Ġcomposed":14092,"Ġcorrespondent":14093,"Ġreunion":14094,"Ġobserve":14095,"Ġcomprising":14096,"Ġimpeachment":14097,"Ġresh":14098,"Ġlemon":14099,"ĠSnap":14100,"Ġproprietary":14101,"een":14102,"ourt":14103,"Ġdetective":14104,"Ġlabels":14105,"Ġcorridor":14106,"ĠClinic":14107,"Ġarra":14108,"ĠPearl":14109,"Ġinformal":14110,"ĠUnd":14111,"ĠVenezuelan":14112,"Ġpeninsula":14113,"Ġdefeating":14114,"Ġsyndrome":14115,"iere":14116,"Ġspite":14117,"bag":14118,"aran":14119,"Ġspecialized":14120,"ĠAA":14121,"ĠLyn":14122,"Ġinstrumental":14123,"Smith":14124,"Ġpivotal":14125,"Ġnightclub":14126,"ĠCob":14127,"Ġcolorful":14128,"Ġartwork":14129,"Ġ1981":14130,"Ġdawn":14131,"erville":14132,"uated":14133,"ief":14134,"Ġlinking":14135,"ĠOw":14136,"Ġappreci":14137,"Ġreductions":14138,"elling":14139,"Ġsalmon":14140,"bb":14141,"ĠPhillip":14142,"yle":14143,"Ġassure":14144,"Ġdiscretion":14145,"Ġefficiently":14146,"ĠMau":14147,"abil":14148,"Ġintentionally":14149,"Ġactivated":14150,"Ġimmense":14151,"ĠStrategic":14152,"Ġcheating":14153,"ĠTrend":14154,"ĠSamantha":14155,"Ġcomple":14156,"Ġhack":14157,"ĠSerie":14158,"ĠText":14159,"Ġstylish":14160,"ĠFaith":14161,"ĠGST":14162,"Ġexterior":14163,"Ġblessing":14164,"Ġblanket":14165,"Ġcooked":14166,"Ġretaliation":14167,"Ġtro":14168,"Ġshelves":14169,"rose":14170,"ĠGram":14171,"Ġsho":14172,"ĠArgentine":14173,"Ġclerk":14174,"specific":14175,"Ġagreeing":14176,"Ġstandout":14177,"black":14178,"Ġtrending":14179,"Ġviolate":14180,"Get":14181,"ño":14182,"ĠOpt":14183,"ĠFrankfurt":14184,"ĠFranco":14185,"eness":14186,"Ġlining":14187,"Ġzoo":14188,"oil":14189,"lia":14190,"rab":14191,"Ġorganize":14192,"Ġwoods":14193,"Ġscan":14194,"Ġurgency":14195,"Ġoccurring":14196,"Ġreliance":14197,"Ġconcepts":14198,"Ġeligibility":14199,"0000":14200,"ĠBrief":14201,"Ġabusive":14202,"ĠBench":14203,"Ġrub":14204,"ĠDil":14205,"Ġmount":14206,"Ġmaturity":14207,"ĠNut":14208,"nee":14209,"enc":14210,"Ġgunfire":14211,"ĠKill":14212,"Ġgates":14213,"Ġflower":14214,"iol":14215,"Ġshaped":14216,"Ġundoubtedly":14217,"Ġbackgrounds":14218,"ĠComplex":14219,"\":{\"":14220,"Ġnaming":14221,"Ġmonument":14222,"Ġoh":14223,"Ġembedded":14224,"Ġbang":14225,"ĠKro":14226,"Ġaggression":14227,"ĠMits":14228,"During":14229,"ĠEp":14230,"iners":14231,"ĠAnaheim":14232,"Ġrom":14233,"Ġoutgoing":14234,"Ġfulfill":14235,"Ġreminds":14236,"Ġren":14237,"à¤":14238,"ĠSue":14239,"Ġrefresh":14240,"Ġlif":14241,"Ġfil":14242,"ĠLead":14243,"Ġregulate":14244,"ĠTeachers":14245,"Ġclarify":14246,"obs":14247,"Ġblasted":14248,"ĠAx":14249,"Ġflavors":14250,"Ġmega":14251,"Ġhurdles":14252,"Ġinspector":14253,"ĠSalvador":14254,"Ġprescribed":14255,"Ġrenovation":14256,"OUR":14257,"Ġutil":14258,"ĠBradford":14259,"Ġwasted":14260,"Ġlineman":14261,"Ġpalm":14262,"icate":14263,"Ġoverseeing":14264,"otted":14265,"ĠRapids":14266,"Ġjustified":14267,"aby":14268,"Ġextends":14269,"Ġoath":14270,"bow":14271,"ĠRivera":14272,"Jan":14273,"ĠImran":14274,"Ġforests":14275,"ĠShel":14276,"ĠBrun":14277,"Ġaerial":14278,"ĠNOW":14279,"PAR":14280,"Ġbeverages":14281,"ettel":14282,"Ġfragile":14283,"Ġcodes":14284,"Į":14285,"abel":14286,"Watch":14287,"road":14288,"Ġdismissal":14289,"ĠRosa":14290,"Ġcrunch":14291,"²":14292,"Ġinnovations":14293,"Ġhabitat":14294,"Ġforefront":14295,"ĠKoch":14296,"ĠChevrolet":14297,"Ġwheelchair":14298,"Ġconsiderably":14299,"Ġexpenditures":14300,"Ġtexts":14301,"Ġprompt":14302,"Ġskating":14303,"Ġpetroleum":14304,"ĠICC":14305,"Ġvit":14306,"fit":14307,"Ġprolonged":14308,"ĠLucy":14309,"Ġcho":14310,"Ġrocked":14311,"ĠBrom":14312,"Ġfreed":14313,"Ġyours":14314,"ĠEden":14315,"Ġmonitored":14316,"asted":14317,"Ġoversees":14318,"ieri":14319,"Ġideology":14320,"ĠFine":14321,"tering":14322,"Top":14323,"Ġdamp":14324,"uta":14325,"Ġlethal":14326,"Ġpurple":14327,"udge":14328,"ĠChemical":14329,"ĠPetersburg":14330,"Ġwarns":14331,"Ġcollectively":14332,"Ġâ":14333,"Ġplaintiffs":14334,"ĠBoris":14335,"Ġsheep":14336,"oves":14337,"ĠAuthor":14338,"Ġcampuses":14339,"Ġdestroying":14340,"Ġgloves":14341,"Ġcease":14342,"Ġdelegates":14343,"Ġpreceded":14344,"realDonaldTrump":14345,"Ġforwards":14346,"erton":14347,"ĠBuzzFeed":14348,"Ġoccupation":14349,"ĠLegion":14350,"Ġstir":14351,"Ġshale":14352,"Ġterrific":14353,"Ġnewborn":14354,"Ġstandoff":14355,"OWN":14356,"Ġmuscles":14357,"ĠHerman":14358,"ĠLiz":14359,"ĠExperience":14360,"ĠSuccess":14361,"ĠHispanic":14362,"ĠCCTV":14363,"Ġcomplement":14364,"ĠBing":14365,"Ġprem":14366,"ĠJohannes":14367,"Ġdent":14368,"itar":14369,"ĠHein":14370,"ĠNicola":14371,"Ġconcludes":14372,"ĠKhal":14373,"Ġparish":14374,"Ġshaking":14375,"ĠSchw":14376,"mod":14377,"ĠLil":14378,"ña":14379,"ĠBog":14380,"ĠFight":14381,"Ġgre":14382,"Ġfel":14383,"Ġheal":14384,"err":14385,"TM":14386,"airo":14387,"health":14388,"Ġswings":14389,"Ġtier":14390,"anka":14391,"ribune":14392,"emouth":14393,"ĠBloom":14394,"Ġowing":14395,"Tech":14396,"Ġdough":14397,"Ġbatch":14398,"ĠLion":14399,"ĠZamb":14400,"Ġcrashing":14401,"ĠXL":14402,"ppers":14403,"ĠDoctors":14404,"ĠSor":14405,"video":14406,"Ġcigarettes":14407,"ĠBoxing":14408,"Ġconstitute":14409,"Ġconcentrate":14410,"ĠArmenian":14411,"Ġsemester":14412,"position":14413,"emic":14414,"ĠNYC":14415,"ĠCampus":14416,"Ġalternate":14417,"Ġexped":14418,"Ġpublishers":14419,"2015":14420,"Ġunanimous":14421,"ĠPrevious":14422,"Ġwellness":14423,"ĠCreative":14424,"edy":14425,"AGE":14426,"ĠCavs":14427,"Ġ1978":14428,"Ġfu":14429,"ĠTata":14430,"ĠChoice":14431,"Ġwoes":14432,"ĠCable":14433,"Ġ~":14434,"ĠGem":14435,"Ġconsolidated":14436,"ĠManitoba":14437,"Cloud":14438,"Ġrounded":14439,"ĠVentura":14440,"Ġshark":14441,"Ġdresses":14442,"Ġtraction":14443,"eda":14444,"Ġdiv":14445,"Ġdental":14446,"Wh":14447,"ĠGig":14448,"ĠBoyd":14449,"ĠTransit":14450,"Ġtelevised":14451,"SON":14452,"ĠVince":14453,"Ġcloses":14454,"apt":14455,"ĠWheeler":14456,"ĠTyson":14457,"Ġforensic":14458,"Ġpunished":14459,"Ġseas":14460,"Ġnavigation":14461,"Ġprecedent":14462,"Ġextremist":14463,"Ġcomposite":14464,"PO":14465,"Ġsurvivor":14466,"ĠVale":14467,"gars":14468,"HT":14469,"ĠRiyadh":14470,"Ġrevival":14471,"ĠPayne":14472,"Ġcollaborative":14473,"ĠCustomers":14474,"ĠPf":14475,"Ġproves":14476,"erve":14477,"Ġelev":14478,"ĠPaper":14479,"Ġchore":14480,"Ġthriller":14481,"Ġstraw":14482,"cock":14483,"Gu":14484,"Ġaligned":14485,"ĠChronicle":14486,"Ġshouting":14487,"Ġ1976":14488,"Ġlightning":14489,"Ġworlds":14490,"ĠOpening":14491,"enton":14492,"ĠAna":14493,"ĠGol":14494,"ĠTechn":14495,"lis":14496,"Ġorientation":14497,"ĠArri":14498,"ĠPG":14499,"ross":14500,"Ġsank":14501,"LOS":14502,"ĠAllison":14503,"Ġsmiles":14504,"USD":14505,"Ġkits":14506,"Bar":14507,"ĠBri":14508,"Ġounces":14509,"ĠNielsen":14510,"eno":14511,"Ġ109":14512,"Ġnorms":14513,"Ġskip":14514,"180":14515,"Ġmonitors":14516,"2012":14517,"Ġincorporate":14518,"Ġmechanisms":14519,"ĠHack":14520,"ĠBomb":14521,"ĠGavin":14522,"ĠNatalie":14523,"Ġdiscusses":14524,"Ġassembled":14525,"Ġcognitive":14526,"owner":14527,"Ġgenuinely":14528,"Ġdisappear":14529,"ĠAK":14530,"Ġstal":14531,"Ġsoup":14532,"ĠFinn":14533,"Ġcares":14534,"Ġfinest":14535,"Ġtuned":14536,"ende":14537,"ĠStefan":14538,"Ġaccompanying":14539,"î":14540,"Maybe":14541,"Ġoffender":14542,"TT":14543,"Ġ212":14544,"Ġvolleyball":14545,"needed":14546,"Ġquo":14547,"Ġdim":14548,"ĠHistorical":14549,"ĠLance":14550,"gmail":14551,"ĠGate":14552,"Ġdemonstrators":14553,"Ġdy":14554,"cia":14555,"ĠSteele":14556,"ĠJoan":14557,"ĠKerala":14558,"KA":14559,"ĠElectoral":14560,"Ġpaths":14561,"ø":14562,"Ne":14563,"Ġaccepts":14564,"Ġlowering":14565,"Ġportions":14566,"ĠValencia":14567,"Ġfestivals":14568,"Ġgeneric":14569,"usk":14570,"ĠVernon":14571,"ĠOrioles":14572,"Ġrenewal":14573,"Ġbelonged":14574,"Ġbreathe":14575,"Ġ220":14576,"Ġrecruited":14577,"Ġlogic":14578,"Ġrecreation":14579,"Ġverbal":14580,"ĠHaz":14581,"double":14582,"Ġfavourites":14583,"Ġfundamentals":14584,"ĠSoc":14585,"360":14586,"SO":14587,"Ġalerted":14588,"Ġbriefed":14589,"ĠBruno":14590,"Ġseating":14591,"Ġfreight":14592,"ĠAmer":14593,"Ġwished":14594,"table":14595,"growth":14596,"ĠWent":14597,"Ġhilarious":14598,"Ġthroat":14599,"bet":14600,"gon":14601,"Ġample":14602,"hee":14603,"ĠHood":14604,"ĠIceland":14605,"ĠAnkara":14606,"iang":14607,"Ġpracticing":14608,"azer":14609,"Ġleaf":14610,"Ġhottest":14611,"Ġmarginal":14612,"Ġrevelations":14613,"ĠPrices":14614,"ĠLar":14615,"times":14616,"Ġhandles":14617,"ĠNaz":14618,"Ġinstitute":14619,"Ġtranslate":14620,"ĠJP":14621,"Ġsoared":14622,"Ġconsume":14623,"ĠTap":14624,"ĠCelebrity":14625,"ĠMayweather":14626,"ĠOracle":14627,"Ġmor":14628,"ANA":14629,"Ġpaperwork":14630,"aste":14631,"Ġdil":14632,"Ġdecorated":14633,"Ġpromotional":14634,"ĠMerrill":14635,"Ġappliances":14636,"ĠCOP":14637,"Ġlips":14638,"ĠBrennan":14639,"ĠMile":14640,"ĠNetworks":14641,"ĠComment":14642,"ĠIb":14643,"ĠAgg":14644,"IDE":14645,"Ġinitiate":14646,"Ġknockout":14647,"Ġbargain":14648,"Ġaccordingly":14649,"bee":14650,"ĠGerald":14651,"Ġproblematic":14652,"Ġtrap":14653,"Ġfinalists":14654,"addy":14655,"would":14656,"Ġstrictly":14657,"ĠRamsey":14658,"Ġdownward":14659,"Ġextract":14660,"Ġfamed":14661,"ĠOUT":14662,"Ġinduct":14663,"ĠAuckland":14664,"Ġpoetry":14665,"mos":14666,"ĠGuinea":14667,"management":14668,"ohan":14669,"ĠGuide":14670,"aily":14671,"umping":14672,"Ġenacted":14673,"ĠEye":14674,"vision":14675,"umi":14676,"aped":14677,"Ġbicycle":14678,"ĠHouth":14679,"ĠNAS":14680,"Ġtapped":14681,"wer":14682,"otti":14683,"EA":14684,"Ġsurprises":14685,"ĠUpdate":14686,"ĠPun":14687,"ĠMiz":14688,"ĠOro":14689,"Ġcostumes":14690,"title":14691,"Ġsurviving":14692,"According":14693,"themed":14694,"ĠPeoples":14695,"Se":14696,"Ġassociations":14697,"hett":14698,"Time":14699,"Ġessay":14700,"Ġmu":14701,"ĠScore":14702,"ĠSpani":14703,"ĠSEE":14704,"Ġmales":14705,"Ġrage":14706,"EU":14707,"ĠYellow":14708,"rupt":14709,"Ġapparel":14710,"Ġsweat":14711,"Ġnearest":14712,"zman":14713,"Ġanticipation":14714,"Ġinjuring":14715,"Ġousted":14716,"chan":14717,"ĠAlert":14718,"Ġber":14719,"atal":14720,"Com":14721,"Ġ04":14722,"Ġafterward":14723,"edge":14724,"ĠBooker":14725,"lex":14726,"ĠWhole":14727,"Ġtoughest":14728,"ĠMaharashtra":14729,"lier":14730,"ĠTennis":14731,"Ġhandy":14732,"ĠMetal":14733,"ĠiTunes":14734,"ĠDiscovery":14735,"Ġcompassion":14736,"ĠLIVE":14737,"Ġeconomically":14738,"Ġendangered":14739,"GO":14740,"Ġmound":14741,"word":14742,"ĠTouch":14743,"ogo":14744,"Ġincomes":14745,"when":14746,"ĠAside":14747,"Ġscandals":14748,"Ġfunctionality":14749,"ĠAer":14750,"Ġcouncils":14751,"Ġdenial":14752,"140":14753,"Ġimplied":14754,"Ġoutfits":14755,"Ġsuited":14756,"Ġ1973":14757,"ĠPizza":14758,"Ġdebates":14759,"record":14760,"Ġhype":14761,"ĠRus":14762,"ĠRobbie":14763,"Ġtouted":14764,"ĠSharp":14765,"Ġbeings":14766,"Ġslavery":14767,"encies":14768,"ĠRooney":14769,"Ġnan":14770,"Ġraids":14771,"Ġinstructor":14772,"Market":14773,"Ġshook":14774,"Ġdeliberate":14775,"ĠNorthwestern":14776,"ĠEss":14777,"Ġwhatsoever":14778,"ĠConfederate":14779,"YS":14780,"ĠCameroon":14781,"ĠFlip":14782,"Yeah":14783,"Ġwashing":14784,"mand":14785,"ĠLex":14786,"Ġissuance":14787,"Ġniche":14788,"Ġfold":14789,"ĠWendy":14790,"Ġhy":14791,"Ġbucket":14792,"ĠVW":14793,"ĠCairo":14794,"ĠSK":14795,"ĠKang":14796,"Ġintake":14797,"Ġhills":14798,"anz":14799,"©":14800,"ugu":14801,"ĠFortunately":14802,"ĠMarqu":14803,"Ġimprisonment":14804,"oking":14805,"Ġdistributors":14806,"zie":14807,"Ġstip":14808,"ĠWire":14809,"Ġcouncillors":14810,"Ġsue":14811,"ĠRegardless":14812,"ĠEnc":14813,"Ġbaking":14814,"ĠVenture":14815,"Ġintriguing":14816,"Ġupheld":14817,"ĠActive":14818,"Ġgenes":14819,"ĠDawson":14820,"ĠPreviously":14821,"ĠRac":14822,"Ġmetric":14823,"Files":14824,"ĠiPhones":14825,"ĠWelcome":14826,"Ġburns":14827,"ĠScreen":14828,"ashes":14829,"ĠApr":14830,"Ġtheories":14831,"san":14832,"ĠRenault":14833,"ĠSinger":14834,"Ġfounders":14835,"Russian":14836,"ĠBelfast":14837,"Ġimagined":14838,"ĠPlanet":14839,"ĠCatalan":14840,"ĠRochester":14841,"Ġevolve":14842,"ĠOT":14843,"Ġpassword":14844,"Ġhomelessness":14845,"Ġbacklog":14846,"Ġpresenter":14847,"Ġfal":14848,"ISH":14849,"ĠEM":14850,"icked":14851,"Ġunlock":14852,"city":14853,"Ġnegotiation":14854,"Ġdancers":14855,"dan":14856,"ĠCOL":14857,"VC":14858,"boat":14859,"Ġoverly":14860,"deal":14861,"lander":14862,"Ġdiss":14863,"ICS":14864,"Ġfifty":14865,"Ġowe":14866,"Ġprisons":14867,"ifications":14868,"wo":14869,"ĠAu":14870,"Ġapiece":14871,"ĠCourtney":14872,"Ġ1975":14873,"Ġsurpass":14874,"Ġidentities":14875,"Ġintegral":14876,"Ġdocumentation":14877,"Ġelegant":14878,"ĠIg":14879,"Ġdear":14880,"Ġ113":14881,"ĠGupta":14882,"Ġcontentious":14883,"rish":14884,"Ġclues":14885,"Ġadditions":14886,"Ġep":14887,"rus":14888,"Ġcentered":14889,"ĠPhillies":14890,"father":14891,"Ġborough":14892,"Ġbuttons":14893,"Ġdeported":14894,"ĠREC":14895,"ĠAlready":14896,"eh":14897,"hur":14898,"Ġupbeat":14899,"omen":14900,"Ġdetailing":14901,"Ġwr":14902,"Ġvaried":14903,"ĠEconomics":14904,"Ġensures":14905,"ĠCivic":14906,"Ġunpaid":14907,"sold":14908,"ĠHil":14909,"ĠMult":14910,"ĠRising":14911,"ĠMini":14912,"Ġneuro":14913,"Ġpenal":14914,"Ġneighbour":14915,"ĠChavez":14916,"Ġjew":14917,"ĠVIP":14918,"Connor":14919,"ĠTalking":14920,"Ġcorrection":14921,"Ġstandpoint":14922,"roads":14923,"ĠWool":14924,"Ġverification":14925,"Ġmic":14926,"olf":14927,"Ġexemption":14928,"Ġfilter":14929,"Ġballoon":14930,"leases":14931,"ician":14932,"ĠSpr":14933,"Ġtoe":14934,"Ġunconstitutional":14935,"Ġmanslaughter":14936,"Ġtossed":14937,"ĠMeg":14938,"ATIONS":14939,"ACK":14940,"ĠRouge":14941,"ĠHansen":14942,"ĠHook":14943,"Out":14944,"ĠHorse":14945,"ĠBath":14946,"ĠAlways":14947,"Ġincorporated":14948,"Ġconjunction":14949,"ĠFit":14950,"Ġexamining":14951,"Ġwallet":14952,"Ġensured":14953,"Ġacclaimed":14954,"ippers":14955,"Ġbeneficiaries":14956,"Ġunexpectedly":14957,"Ġexploit":14958,"ĠWillie":14959,"Ġcomb":14960,"ĠWalton":14961,"rica":14962,"icky":14963,"Ġate":14964,"ĠPadres":14965,"Ġrib":14966,"Ġsnacks":14967,"ĠFernandez":14968,"ĠMachine":14969,"ction":14970,"Ġillnesses":14971,"ĠHoffman":14972,"ĠSpaceX":14973,"Ġju":14974,"Ġswift":14975,"Ġembark":14976,"ĠRailway":14977,"Ġmeasuring":14978,"agers":14979,"arsh":14980,"Ġessence":14981,"angle":14982,"Ġolive":14983,"ĠCommander":14984,"iggs":14985,"Ġrewarded":14986,"Ġdispatched":14987,"Ġplayground":14988,"½":14989,"ĠProgramme":14990,"Ġstudios":14991,"Ġskeptical":14992,"ĠOlymp":14993,"ĠKeys":14994,"ĠSunshine":14995,"amba":14996,"ĠDonna":14997,"Ġlightly":14998,"Ġobtaining":14999,"Ġpoisoning":15000,"Ġaz":15001,"Ġ1972":15002,"Ġunconscious":15003,"ECT":15004,"Ġlied":15005,"ĠKaz":15006,"Ġ06":15007,"ĠMoving":15008,"Ġnum":15009,"oral":15010,"Ġassessments":15011,"Ġscholarships":15012,"Ġevacuate":15013,"ĠSunni":15014,"Ġquake":15015,"Ġfort":15016,"ques":15017,"ĠAlonso":15018,"Ġthread":15019,"Ġsqueeze":15020,"arat":15021,"oly":15022,"ĠAlphabet":15023,"uting":15024,"icio":15025,"ĠRetirement":15026,"ither":15027,"Ġasleep":15028,"Ġpairs":15029,"Ġmanufacture":15030,"ĠHazard":15031,"Ġsidewalk":15032,"Ġwears":15033,"ĠCraft":15034,"emen":15035,"ieth":15036,"Ġbypass":15037,"ĠLancaster":15038,"Ġflour":15039,"charge":15040,"ĠCLICK":15041,"Ġpotatoes":15042,"ĠKarachi":15043,"Ġvalley":15044,"Ġsights":15045,"Ġfallout":15046,"ords":15047,"BN":15048,"Ġsunshine":15049,"Ġundertaken":15050,"Ġcontestants":15051,"Ġaccomplishments":15052,"Ġconditioning":15053,"Ġcel":15054,"ĠHalifax":15055,"Ġaccent":15056,"***":15057,"Ġpitchers":15058,"Ġadopting":15059,"Ġjustices":15060,"Ġrip":15061,"ince":15062,"Ġelimination":15063,"Ġaerospace":15064,"ĠBeer":15065,"ĠBasin":15066,"Ġunwanted":15067,"goers":15068,"isco":15069,"ĠTwin":15070,"ĠDesert":15071,"rix":15072,"Ġdarkness":15073,"ĠDunn":15074,"City":15075,"pop":15076,"Ġ1969":15077,"ataka":15078,"Ġtal":15079,"Ġautism":15080,"ĠMcLaren":15081,"ĠUEFA":15082,"Ġclassrooms":15083,"ĠLeave":15084,"Americans":15085,"las":15086,"Ġqui":15087,"Ġundefeated":15088,"otto":15089,"ĠNRA":15090,"ĠPorsche":15091,"Ġnuts":15092,"oys":15093,"ĠMethodist":15094,"Ġatt":15095,"Ġtweeting":15096,"children":15097,"eller":15098,"Ġinquiries":15099,"Ġmillennials":15100,"ĠWembley":15101,"INS":15102,"Ġautopsy":15103,"ĠElon":15104,"ĠHicks":15105,"ugg":15106,"Ġwreck":15107,"ĠComcast":15108,"Ġstones":15109,"public":15110,"ĠKem":15111,"bedroom":15112,"ļ":15113,"itated":15114,"Ġsemic":15115,"uman":15116,"Cal":15117,"ANN":15118,"ĠGaz":15119,"Ġundisclosed":15120,"ĠPlanned":15121,"ĠYale":15122,"ĠIST":15123,"lies":15124,"ĠStanding":15125,"Ġrelieved":15126,"EO":15127,"Ġgraduating":15128,"park":15129,"ĠâĢķ":15130,"Ġpensions":15131,"rave":15132,"ĠWonder":15133,"AZ":15134,"Ġcosting":15135,"Ġeditors":15136,"Ġtotaled":15137,"Ġspacecraft":15138,"meter":15139,"Ġ02":15140,"ĠNikki":15141,"sworth":15142,"ĠCrit":15143,"asha":15144,"Ġknees":15145,"Ġhats":15146,"uity":15147,"ĠPanther":15148,"Ġtan":15149,"ĠBuzz":15150,"ĠGlad":15151,"ĠPleasant":15152,"SM":15153,"Ġtricks":15154,"Ġplac":15155,"ĠDanielle":15156,"Ġours":15157,"Ġwashed":15158,"haven":15159,"Ġdrain":15160,"ĠUttar":15161,"Ġapple":15162,"Ġjunk":15163,"Ġturkey":15164,"ĠDug":15165,"Ġdiplomacy":15166,"Ġempire":15167,"Ġpinch":15168,"Ġferry":15169,"ĠDustin":15170,"Ġ03":15171,"Ġelder":15172,"Everything":15173,"ĠProgressive":15174,"ution":15175,"VI":15176,"dam":15177,"Ġlever":15178,"ĠAustralians":15179,"Ġconsequence":15180,"itan":15181,"Ġcondemn":15182,"Ġneg":15183,"ĠOverview":15184,"Ġsuccesses":15185,"Ġprobable":15186,"ĠMirror":15187,"mor":15188,"verse":15189,"Ġevaluating":15190,"ĠBes":15191,"Ġimm":15192,"Ġharness":15193,"Ġresilient":15194,"ĠBuild":15195,"Ġstraightforward":15196,"ADE":15197,"Ġgrandparents":15198,"Ġmarched":15199,"ĠKiev":15200,"Ġchiefs":15201,"oha":15202,"Ġvest":15203,"kn":15204,"enda":15205,"ĠSev":15206,"Ġbatters":15207,"ĠJos":15208,"ĠQue":15209,"ĠCourse":15210,"ĠCorner":15211,"ĠMess":15212,"Ġmourn":15213,"keepers":15214,"ĠRegina":15215,"Everybody":15216,"Ġtrajectory":15217,"Ġdefenseman":15218,"ĠArticles":15219,"Ġspur":15220,"ĠPhD":15221,"Ġpipes":15222,"Ġduck":15223,"Ġcombining":15224,"ĠHit":15225,"ĠGeorgetown":15226,"ĠBee":15227,"Cor":15228,"Ġcomposition":15229,"Ġconnects":15230,"ĠMARK":15231,"taker":15232,"Ġcertainty":15233,"Ġhefty":15234,"ĠHezbollah":15235,"ĠShip":15236,"Ġmalicious":15237,"AI":15238,"Ġbits":15239,"Ġstyl":15240,"Ġimpaired":15241,"ĠCBI":15242,"Despite":15243,"othe":15244,"ĠRyder":15245,"ĠAlf":15246,"ifa":15247,"Ind":15248,"Ġblaming":15249,"ĠToledo":15250,"EW":15251,"ĠEssex":15252,"iated":15253,"ĠAberdeen":15254,"ANCE":15255,"Ġpossess":15256,"Ġsuperhero":15257,"Ġoverhead":15258,"quet":15259,"ĠRicky":15260,"Ġdock":15261,"ĠTelecom":15262,"Ġshelf":15263,"³":15264,"Ġmaritime":15265,"Ġportrayed":15266,"ĠYesterday":15267,"Ġcollided":15268,"Ġcookies":15269,"ĠCul":15270,"Ġindexes":15271,"Ġnaval":15272,"oval":15273,"105":15274,"ĠWeber":15275,"chief":15276,"arma":15277,"ĠRey":15278,"Ġauditor":15279,"ĠMarion":15280,"ĠMartha":15281,"ĠSally":15282,"Ġsedan":15283,"ĠAlison":15284,"nce":15285,"Es":15286,"ĠParade":15287,"Ġpharmacy":15288,"ĠKre":15289,"loe":15290,"cks":15291,"Ġmitigate":15292,"Ġdesigning":15293,"Ġ2024":15294,"Ġportable":15295,"Ġimproves":15296,"ĠAMD":15297,"Ġexcluded":15298,"CON":15299,"ĠOscars":15300,"Ġfixtures":15301,"comb":15302,"ĠBerg":15303,"Ġbother":15304,"Ġboring":15305,"Ġobservation":15306,"ĠCad":15307,"Ġrecordings":15308,"ĠCultural":15309,"Ġweaken":15310,"Ġaccuse":15311,"ĠAbd":15312,"abor":15313,"115":15314,"uffle":15315,"Ġhighways":15316,"atham":15317,"empt":15318,"ĠDeer":15319,"ĠEDT":15320,"ĠWait":15321,"athan":15322,"Ġaccumulated":15323,"Ġguilt":15324,"Ġexempt":15325,"Ġdiluted":15326,"ĠJamal":15327,"Ġshit":15328,"cross":15329,"Ġeve":15330,"Ġshirts":15331,"Ġsatisfy":15332,"ĠPaulo":15333,"AH":15334,"sic":15335,"ĠChloe":15336,"ĠCities":15337,"ĠSwansea":15338,"Ġprecision":15339,"ĠTracy":15340,"ping":15341,"Ġcontinually":15342,"Ġdemographic":15343,"Ġcliff":15344,"Ġjaw":15345,"isted":15346,"ĠDevelop":15347,"ĠAJ":15348,"Ġaisle":15349,"ĠLionel":15350,"Ġpredominantly":15351,"Ġmel":15352,"Ġlifelong":15353,"hs":15354,"Ġshouted":15355,"lad":15356,"Ġdest":15357,"Ġpacks":15358,"ĠKath":15359,"ĠCruise":15360,"fired":15361,"oder":15362,"hua":15363,"Ġgoodbye":15364,"Ġinterfere":15365,"eca":15366,"Ġré":15367,"atum":15368,"itas":15369,"ĠLodge":15370,"ĠWald":15371,"Ġmidday":15372,"umble":15373,"asting":15374,"©":15375,"ĠLeg":15376,"ĠNepal":15377,"Ġchased":15378,"idge":15379,"Ġconv":15380,"Ġfraudulent":15381,"Ġopera":15382,"Ġshr":15383,"ĠUniverse":15384,"ĠJerome":15385,"Ġ1977":15386,"ĠDancing":15387,"ĠRS":15388,"±":15389,"eks":15390,"Ġchic":15391,"Ġpunish":15392,"Ġpropose":15393,"arin":15394,"ĠChop":15395,"ĠAhead":15396,"ĠGallagher":15397,"ĠBangkok":15398,"ĠShelby":15399,"ĠNS":15400,"Ġcheek":15401,"onia":15402,"Ġrelegation":15403,"ĠHind":15404,"ĠCory":15405,"Ġfingerprint":15406,"Ġstrive":15407,"Ġmm":15408,"igs":15409,"Ġholy":15410,"Ġfavored":15411,"ĠSomeone":15412,"ĠLatino":15413,"ĠPatt":15414,"Ġchallenger":15415,"ĠCotton":15416,"Sw":15417,"itten":15418,"ĠXI":15419,"ĠStat":15420,"ĠDIS":15421,"Ġautomakers":15422,"Ġevaluated":15423,"ĠArc":15424,"Ġpersuade":15425,"Af":15426,"Ġreunited":15427,"Ġabs":15428,"Ġbride":15429,"Ġpurely":15430,"uce":15431,"uded":15432,"Ġsettling":15433,"Ġlodged":15434,"Ġfixing":15435,"Ġsuccession":15436,"ĠAlfred":15437,"ĠAlvarez":15438,"mac":15439,"ĠFont":15440,"Ġcontra":15441,"affle":15442,"Ġcopied":15443,"Ġmasses":15444,"ĠElections":15445,"ĠThan":15446,"Ġsoaring":15447,"jay":15448,"Ġsuing":15449,"Ġconcentrated":15450,"Ġconvey":15451,"Ġ240":15452,"gs":15453,"ĠNeal":15454,"Ġnasty":15455,"ĠLB":15456,"odi":15457,"ĠSergei":15458,"Ġthumb":15459,"Ġservants":15460,"Ġrevelation":15461,"Ġdischarge":15462,"ĠBright":15463,"ĠBent":15464,"ĠChrysler":15465,"mill":15466,"ĠImagine":15467,"Ġreceptions":15468,"Ġpersonalities":15469,"Ġsilly":15470,"ĠLoc":15471,"ĠZero":15472,"HI":15473,"rice":15474,"Ġgar":15475,"far":15476,"enh":15477,"ĠBiden":15478,"ĠEntreprene":15479,"Ġassumption":15480,"Ġnicely":15481,"ĠEither":15482,"|":15483,"ĠNW":15484,"ĠKens":15485,"ĠNolan":15486,"Ġowning":15487,"atures":15488,"ĠPastor":15489,"ĠRegistration":15490,"Ġexperiments":15491,"Ġassurance":15492,"Ġhashtag":15493,"oint":15494,"ĠBin":15495,"Ġqualification":15496,"center":15497,"Ġausterity":15498,"ĠPers":15499,"Ġscoop":15500,"Ġpros":15501,"ĠFields":15502,"Ġfur":15503,"ĠJas":15504,"Ġplanting":15505,"security":15506,"ĠTrain":15507,"ĠKathy":15508,"demand":15509,"ĠLev":15510,"Ġtut":15511,"tier":15512,"QU":15513,"Ġexploitation":15514,"Ġignoring":15515,"ĠSex":15516,"Ġadapted":15517,"Ġdisastrous":15518,"Ġempower":15519,"Ġcreators":15520,"ĠLay":15521,"ĠDragon":15522,"ĠWyn":15523,"Ġ1974":15524,"acious":15525,"performance":15526,"ĠTiffany":15527,"isting":15528,"Ġindividually":15529,"ĠLeading":15530,"ĠSask":15531,"Ġcatastrophic":15532,"Ġpunched":15533,"ĠVienna":15534,"Ġsurgical":15535,"Gr":15536,"odo":15537,"Ġgem":15538,"ĠMinority":15539,"Ġmice":15540,"ĠHistoric":15541,"ĠKot":15542,"caster":15543,"Ġsuff":15544,"journal":15545,"Ġpresumably":15546,"ĠBit":15547,"inary":15548,"Ġbre":15549,"Ġenhancing":15550,"Ġgru":15551,"ĠRunning":15552,"hardt":15553,"Ġtroubling":15554,"Ġpumps":15555,"ĠProspect":15556,"etic":15557,"Ġmartial":15558,"Ġcouncillor":15559,"atra":15560,"ths":15561,"ĠSark":15562,"ĠChamp":15563,"scoring":15564,"ĠWel":15565,"rup":15566,"Ġterrifying":15567,"ĠCatch":15568,"Ġinspections":15569,"Ġpornography":15570,"bra":15571,"ĠKeeping":15572,"Ġbanker":15573,"angers":15574,"ĠCrimea":15575,"ĠDisclosure":15576,"iba":15577,"Ġturf":15578,"Ġschedules":15579,"ĠJorge":15580,"ĠAcross":15581,"Ġsolving":15582,"Ġsensation":15583,"ĠWW":15584,"cial":15585,"atz":15586,"Ġlion":15587,"Ġcertificates":15588,"itive":15589,"ĠWes":15590,"ĠPrison":15591,"ĠPlayStation":15592,"duty":15593,"Ġvariable":15594,"Ġstrangers":15595,"istrates":15596,"vs":15597,"Ġreigning":15598,"Ġsliding":15599,"ĠShin":15600,"Ġtelecommunications":15601,"Ġinstalling":15602,"Ġrecogn":15603,"Ġsubway":15604,"too":15605,"ĠMcKin":15606,"ĠStoke":15607,"Ġsensitivity":15608,"bas":15609,"Ġsan":15610,"Ġ(-":15611,"ĠSuarez":15612,"Ġaverages":15613,"ammu":15614,"ĠFen":15615,"Ġrefined":15616,"outh":15617,"Ġcob":15618,"ĠLaz":15619,"essa":15620,"Ġpositioning":15621,"Three":15622,"Ġoils":15623,"Ġassaults":15624,"Ġcompanion":15625,"ĠFlash":15626,"ĠMam":15627,"ĠTill":15628,"Ġblues":15629,"ĠJae":15630,"ĠPier":15631,"Ġbedrooms":15632,"ĠHawkins":15633,"ĠCornell":15634,"Ġanswering":15635,"Ġsec":15636,"Ġrecognizes":15637,"Red":15638,"ĠJamaica":15639,"Ġinsurgents":15640,"Ġbrace":15641,"Ġra":15642,"ĠTai":15643,"ocation":15644,"ignment":15645,"Ġreasonably":15646,"inating":15647,"Ġbonuses":15648,"Ġsandwich":15649,"Ġinadequate":15650,"Ġdelicate":15651,"Ġadorable":15652,"Ġpalace":15653,"Ġsmallest":15654,"Ġpractically":15655,"ĠCrosby":15656,"Ġlevy":15657,"Ġlend":15658,"boards":15659,"shaped":15660,"Ġvulnerability":15661,"ĠKelley":15662,"Ġsponsorship":15663,"ract":15664,"Ġslew":15665,"Ġfederation":15666,"ĠLal":15667,"acies":15668,"ĠFamilies":15669,"Ġproposing":15670,"Ġhyp":15671,"elected":15672,"inkle":15673,"ĠSays":15674,"ĠApollo":15675,"ĠWis":15676,"imer":15677,"Ġcombines":15678,"Ġtim":15679,"ĠQuestion":15680,"Ġborrowers":15681,"Ġswiftly":15682,"ĠMagn":15683,"Ġheadphones":15684,"Russia":15685,"Ġtongue":15686,"Ġbye":15687,"nn":15688,"Ġseller":15689,"ĠWord":15690,"Tom":15691,"ĠDevin":15692,"ĠSurrey":15693,"Ġquad":15694,"Ġcourthouse":15695,"gi":15696,"ĠGrill":15697,">":15698,"Ġrational":15699,"ĠFlames":15700,"ĠCham":15701,"Ġvacuum":15702,"ĠRays":15703,"Ġescalating":15704,"Ġouter":15705,"Ġstretches":15706,"ĠSpeed":15707,"Ġnegatively":15708,"Ġabsorb":15709,"ĠAustrian":15710,"Ġslice":15711,"ĠDiet":15712,"Ġbun":15713,"Ġtactical":15714,"ĠCBD":15715,"Ġedges":15716,"Ġnest":15717,"Ġstrained":15718,"ulates":15719,"ĠTina":15720,"Net":15721,"ķ":15722,"ĠGos":15723,"God":15724,"White":15725,"Ġproudly":15726,"usion":15727,"ĠArlington":15728,"ĠNear":15729,"ĠMaxwell":15730,"Ġbomber":15731,"Ġcared":15732,"Ġapprovals":15733,"Ġexams":15734,"ĠEconomy":15735,"Ġposters":15736,"ĠHampton":15737,"ĠPere":15738,"ĠContract":15739,"Ġhoused":15740,"Ġinstruction":15741,"ĠJess":15742,"Ġacre":15743,"Ġcongestion":15744,"ĠGener":15745,"Ġdioxide":15746,"Ġvar":15747,"ĠAlexandria":15748,"ĠSpider":15749,"Ġcoins":15750,"Ġ225":15751,"Ġterritorial":15752,"ĠSPD":15753,"Ġfloat":15754,"null":15755,"Ġcalculate":15756,"ĠDin":15757,"eto":15758,"Ġcows":15759,"Ġpunct":15760,"Ġexpire":15761,"Ġkidnapped":15762,"Ġcou":15763,"Ġattitudes":15764,"ĠLeh":15765,"ĠHero":15766,"ĠKabul":15767,"Ġcubic":15768,"Ġdigits":15769,"ĠRES":15770,"Ġpipelines":15771,"icide":15772,"ĠSingle":15773,"Ġhurts":15774,"ĠMaz":15775,"ĠPak":15776,"Ġslate":15777,"Ġmultimedia":15778,"ADA":15779,"Mexico":15780,"ĠRelease":15781,"chard":15782,"Ġgarlic":15783,"ĠFletcher":15784,"Ġaforementioned":15785,"Ġ05":15786,"ĠParkway":15787,"Ġfirefighter":15788,"Ġcounseling":15789,"utions":15790,"Cap":15791,"Ġconsultants":15792,"ĠMeh":15793,"ouring":15794,"ĠDI":15795,"mic":15796,"phones":15797,"Ġencounters":15798,"ĠHapp":15799,"Ġcartoon":15800,"flight":15801,"Ġundertake":15802,"ĠHans":15803,"Ġplunge":15804,"ĠParenthood":15805,"Ġkickoff":15806,"ĠCelsius":15807,"ĠRas":15808,"ĠDund":15809,"ounce":15810,"Ġpurse":15811,"Ġmortality":15812,"Ġbrains":15813,"Ġconglomerate":15814,"ĠObserver":15815,"ĠSector":15816,"ĠApparently":15817,"Ġblank":15818,"iston":15819,"Ġweighs":15820,"gro":15821,"ĠPaw":15822,"ĠCOM":15823,"ĠPurdue":15824,"Ġnetted":15825,"ĠLinux":15826,"Mike":15827,"Ġfaithful":15828,"Ġmagazines":15829,"Ġheadquartered":15830,"ĠIps":15831,"Ġindications":15832,"Look":15833,"ĠElite":15834,"Ġsupreme":15835,"Ġchunk":15836,"ĠSz":15837,"ĠVine":15838,"rise":15839,"ĠYas":15840,"general":15841,"ĠOpera":15842,"Ġpriests":15843,"Assad":15844,"Ġaunt":15845,"Ġwhopping":15846,"enzie":15847,"Ġvegan":15848,"Ġinflux":15849,"ĠConsult":15850,"Ġwaiver":15851,"Having":15852,"inning":15853,"Ġproximity":15854,"Ġclassical":15855,"ĠIslanders":15856,"Ġadvertisers":15857,"ĠCe":15858,"ĠSochi":15859,"Ġmemoir":15860,"ĠPlaying":15861,"yers":15862,"Ġstud":15863,"Ġobservations":15864,"Ġadmire":15865,"Ġhiking":15866,"Ġbatter":15867,"Ġconfusing":15868,"Ġprecaution":15869,"kil":15870,"clusive":15871,"opoulos":15872,"ĠWestbrook":15873,"ĠTanzania":15874,"ĠCedar":15875,"usted":15876,"Ġdestructive":15877,"ĠIndies":15878,"osi":15879,"ĠAmid":15880,"Ġintercepted":15881,"Ġpartnering":15882,"Ġsubstances":15883,"ĠSuns":15884,"Ġpromotes":15885,"bird":15886,"Gen":15887,"aper":15888,"ĠEy":15889,"Ġterrain":15890,"Ġ1930":15891,"zon":15892,"Ġbreed":15893,"broken":15894,"uchin":15895,"ĠPrim":15896,"ĠRoland":15897,"Ġfitted":15898,"Ġprotects":15899,"Ġ114":15900,"RP":15901,"Ġdisrupted":15902,"ĠBaylor":15903,"oren":15904,"ĠKeen":15905,"Ġmansion":15906,"Ġgrassroots":15907,"ĠVictory":15908,"Ġbarn":15909,"Ġdepreciation":15910,"oped":15911,"immer":15912,"Ġgarnered":15913,"ĠLip":15914,"ĠTob":15915,"Ġcreatures":15916,"ooter":15917,"Ġconsortium":15918,"obi":15919,"ĠMonster":15920,"arks":15921,"turn":15922,"Ġsketch":15923,"Ġpredicting":15924,"Ġminimize":15925,"ĠEthan":15926,"anson":15927,"ĠAdjusted":15928,"ĠHornets":15929,"ĠNZ":15930,"ĠKathleen":15931,"ĠKier":15932,"ĠMercury":15933,"Ġghost":15934,"Ġhaw":15935,"ĠDemand":15936,"ĠCollection":15937,"ĠFortune":15938,"Ġcruel":15939,"Ġfurious":15940,"ĠKun":15941,"ĠSalem":15942,"Ġunsuccessful":15943,"ĠLomb":15944,"ĠFury":15945,"ahi":15946,"Ġenthusiastic":15947,"Ġsurgeries":15948,"ACE":15949,"Ġroller":15950,"ĠStamford":15951,"Being":15952,"Dec":15953,"check":15954,"Ġaffection":15955,"Ġgifted":15956,"Ġenerg":15957,"Ġvarying":15958,"ĠCharl":15959,"Ġsolved":15960,"ĠNV":15961,"Ġlaptops":15962,"Ġkindness":15963,"mart":15964,"ĠPenny":15965,"Ġ116":15966,"ĠFeder":15967,"ĠCisco":15968,"Ġeducators":15969,"Ġminim":15970,"Ġgangs":15971,"Ġfestivities":15972,"ĠOriginal":15973,"yre":15974,"rying":15975,"Ġtighter":15976,"ĠMalta":15977,"Ġshield":15978,"interest":15979,"Ġbuoy":15980,"Ġsupplement":15981,"ĠSof":15982,"Ġok":15983,"Ġprosecuted":15984,"Ġinterventions":15985,"Ġseize":15986,"Ġcaravan":15987,"ĠCarlson":15988,"ĠEnterprises":15989,"ĠChristina":15990,"ĠWellington":15991,"Ġaltered":15992,"TP":15993,"Ġexpresses":15994,"Ġcomfortably":15995,"Ġstaffing":15996,"afa":15997,"itu":15998,"saving":15999,"Ġinflammation":16000,"hatt":16001,"ĠMiranda":16002,"icious":16003,"Ġgrabbing":16004,"ĠANY":16005,"Ġobjections":16006,"Ġdot":16007,"cle":16008,"Ġrelates":16009,"Ġtribe":16010,"Ġboarding":16011,"ĠEpisode":16012,"ĠEnjoy":16013,"arding":16014,"Ġathletics":16015,"Ġflies":16016,"Ġmortgages":16017,"ruct":16018,"Ġink":16019,"ĠKC":16020,"ĠSecondary":16021,"Ġfer":16022,"ĠQaeda":16023,"OA":16024,"Frank":16025,"track":16026,"ĠChandler":16027,"Ġenv":16028,"ĠLeaders":16029,"ĠKemp":16030,"Ġunsafe":16031,"sponsored":16032,"San":16033,"ĠUsers":16034,"PE":16035,"ĠAccount":16036,"otta":16037,"ĠMix":16038,"ĠCindy":16039,"En":16040,"Ġ175":16041,"Ġoverlooked":16042,"Ġpublications":16043,"Ġrewarding":16044,"Ġexplicit":16045,"Ġnotch":16046,"Ġspecifics":16047,"Ġdesignation":16048,"ĠAppeal":16049,"Ġcontingent":16050,"Ġcage":16051,"ĠKol":16052,"ĠJohns":16053,"ĠReach":16054,"ĠTin":16055,"ĠAfricans":16056,"Ġprec":16057,"ĠRural":16058,"ĠDw":16059,"Ġuphold":16060,"Ġsuffers":16061,"Ġweed":16062,"inst":16063,"Ġcancellation":16064,"ĠShaun":16065,"Ġleve":16066,"Ġdivisive":16067,"Ġhel":16068,"Ġfatigue":16069,"ĠSchwartz":16070,"ĠKirst":16071,"Ġarise":16072,"Ġgrandson":16073,"ĠLawson":16074,"Ġcollaborate":16075,"Ġparticipant":16076,"ĠBryce":16077,"Ġinfield":16078,"mid":16079,"Ġut":16080,"Ġnotices":16081,"Ġsneak":16082,"ĠPAR":16083,"Chris":16084,"Ġutilize":16085,"ĠByron":16086,"ĠZhang":16087,"PF":16088,"Ġoverwhelmingly":16089,"Ġvegetable":16090,"Ġabsurd":16091,"ĠChem":16092,"etime":16093,"Ġenvoy":16094,"Ġlover":16095,"length":16096,"Ġrevolutionary":16097,"ĠYam":16098,"Ġshutting":16099,"mt":16100,"super":16101,"ĠToby":16102,"ĠCoca":16103,"Ġproposition":16104,"Ġembracing":16105,"Ġversatile":16106,"ĠWalking":16107,"Ġillicit":16108,"Ġnude":16109,"Ġunpredictable":16110,"take":16111,"Ġgotta":16112,"ĠXiaomi":16113,"Ġinstit":16114,"ĠPep":16115,"ĠPearson":16116,"Ġrejection":16117,"stead":16118,"Ġmut":16119,"Ġoutspoken":16120,"ĠBaghdad":16121,"ĠFly":16122,"Ġwholly":16123,"ĠRM":16124,"ĠFa":16125,"Ġcleaner":16126,"frey":16127,"ĠHab":16128,"ĠLiber":16129,"Ġwhereabouts":16130,"Ġchefs":16131,"Ġalumni":16132,"Ġstopp":16133,"dd":16134,"forward":16135,"rast":16136,"ĠNash":16137,"ĠCort":16138,"Ġpotent":16139,"Ġmold":16140,"Ġdistinctive":16141,"chip":16142,"ĠBrunswick":16143,"Ġpopulist":16144,"Ġplagued":16145,"eka":16146,"ĠIOC":16147,"ugs":16148,"ĠDob":16149,"Ġmagn":16150,"asser":16151,"hew":16152,"Ġcapturing":16153,"oos":16154,"Ġcrystal":16155,"Ġalarming":16156,"Ġ135":16157,"iating":16158,"Ġnap":16159,"umar":16160,"ĠExpl":16161,"Ġupgrading":16162,"Ġdecl":16163,"Ġoverturn":16164,"ARK":16165,"linked":16166,"ĠContinued":16167,"Ġslumped":16168,"ĠGaga":16169,"iful":16170,"ĠPosted":16171,"ĠRecommended":16172,"Ġsnake":16173,"Ġexplosives":16174,"Ġhind":16175,"Ġcontempt":16176,"Ġmock":16177,"NBA":16178,"Ġstall":16179,"Ġorganisers":16180,"Ġingredient":16181,"Ġblockbuster":16182,"ĠStream":16183,"ĠLeah":16184,"Pic":16185,"Ġventures":16186,"oman":16187,"Ġweakening":16188,"Ġmaximize":16189,"Ġdigging":16190,"uez":16191,"Ġdistinction":16192,"ĠMali":16193,"Ġcontaminated":16194,"Ġhij":16195,"Ġcrafts":16196,"Fl":16197,"Ġcloset":16198,"ĠRapp":16199,"Ġtowers":16200,"Ġamenities":16201,"Ġopioids":16202,"Ġcontend":16203,"load":16204,"ĠJol":16205,"ĠBooks":16206,"Ġsim":16207,"Ġthrilling":16208,"Ġmeter":16209,"ĠMultiple":16210,"Ġarbitration":16211,"Ġcracked":16212,"Pl":16213,"Ġphotographers":16214,"Te":16215,"ĠSidd":16216,"Ġexplored":16217,"170":16218,"Ġpleasant":16219,"ĠCapitals":16220,"ĠRi":16221,"ĠRandall":16222,"overed":16223,"Ġchar":16224,"ĠEverybody":16225,"ĠPolitics":16226,"Ġmoisture":16227,"Ġthriving":16228,"ĠScotia":16229,"arded":16230,"imb":16231,"ĠFantasy":16232,"Ġcemetery":16233,"ĠPath":16234,"eur":16235,"ĠSec":16236,"ĠPlatform":16237,"Ġdeparted":16238,"ĠVIDEO":16239,"ĠPant":16240,"ĠSyn":16241,"Ġ230":16242,"bleacher":16243,"live":16244,"Ġprob":16245,"Ġgymn":16246,"Ġjudged":16247,"orns":16248,"Ġstemming":16249,"umbling":16250,"ĠHew":16251,"ĠCheryl":16252,"Ġconsciousness":16253,"cos":16254,"ĠTate":16255,"CNN":16256,"Ġrecognizing":16257,"meg":16258,"Ġpant":16259,"ulk":16260,"MM":16261,"ĠPrescott":16262,"ĠMarcel":16263,"anas":16264,"Ġhappier":16265,"mag":16266,"ĠLov":16267,"Ġspreads":16268,"ĠSample":16269,"Ġpopped":16270,"HR":16271,"ĠMitt":16272,"Ġ00":16273,"Ġlabeled":16274,"Ġaspirations":16275,"?)":16276,"Ġloads":16277,"ĠBritt":16278,"hurst":16279,"ĠTeams":16280,"Ġextremists":16281,"ĠClement":16282,"lings":16283,"shirts":16284,"cheon":16285,"ĠDEL":16286,"ĠLocation":16287,"Ġpresentations":16288,"ĠFalcon":16289,"Ġtoddler":16290,"kl":16291,"Ġprone":16292,"Ġcommemor":16293,"ĠStanton":16294,"201":16295,"Ġranges":16296,"Ġfielder":16297,"Ġattends":16298,"rade":16299,"Ġproactive":16300,"Ġhostage":16301,"ĠGriffith":16302,"ockey":16303,"ĠAdding":16304,"ĠAFL":16305,"gas":16306,"istics":16307,"Ġsurgeon":16308,"Ġtsunami":16309,"2014":16310,"Ġconstraints":16311,"cu":16312,"Ġsurrendered":16313,"azed":16314,"ĠAirbnb":16315,"650":16316,"zed":16317,"Ġinjustice":16318,"dog":16319,"full":16320,"ĠHear":16321,"Ġsprawling":16322,"Ġhomeland":16323,"ĠSG":16324,"anced":16325,"Ġpools":16326,"ĠCE":16327,"Ġbeers":16328,"AE":16329,"ĠJac":16330,"Ġrecurring":16331,"Writing":16332,"Ġgenius":16333,"ĠFrost":16334,"Ġgrounded":16335,"Ġallege":16336,"lessness":16337,"Ġjumper":16338,"Ġvicious":16339,"Ġsecretly":16340,"Ġhacked":16341,"ĠAmsterdam":16342,"ibu":16343,"Ġ1971":16344,"ĠRosenstein":16345,"nick":16346,"arge":16347,"Ġladder":16348,"elled":16349,"Ġsatellites":16350,"Ġassassination":16351,"ĠDepot":16352,"built":16353,"Ġunrelated":16354,"maid":16355,"ĠDod":16356,"ĠVanderbilt":16357,"Ġboundary":16358,"ĠStafford":16359,"ĠBry":16360,"Ġtribunal":16361,"Ġoutings":16362,"Ġquantity":16363,"imming":16364,"ĠBlacks":16365,"Br":16366,"eri":16367,"uffed":16368,"Ġexplicitly":16369,"ĠBieber":16370,"AKING":16371,"Ġphotographed":16372,"ĠPolit":16373,"Ġpremature":16374,"hered":16375,"ĠVi":16376,"Ġmarsh":16377,"casters":16378,"ĠKra":16379,"Ġdried":16380,"Ġcafe":16381,"eting":16382,"Ġshaping":16383,"aram":16384,"orf":16385,"Ġrichest":16386,"Ġhurricanes":16387,"Ġcommands":16388,"Gl":16389,"anth":16390,"Ġstunt":16391,"Ġyearly":16392,"Ġdefeats":16393,"Ġconsultancy":16394,"call":16395,"Ġlag":16396,"adh":16397,"ĠPalestine":16398,"Ġcustomized":16399,"ĠScar":16400,"ĠWesley":16401,"ready":16402,"Ġpersist":16403,"Ġpacking":16404,"ono":16405,"Ġdischarged":16406,"Ġpouring":16407,"sburg":16408,"Ġreconsider":16409,"ĠMethod":16410,"enez":16411,"cill":16412,"Ġsecular":16413,"pers":16414,"Ġple":16415,"ELS":16416,"ĠMine":16417,"Ġpushes":16418,"Us":16419,"Ġframes":16420,"ĠNets":16421,"ĠSiem":16422,"ĠHitler":16423,"kill":16424,"Ġrented":16425,"Ġcharm":16426,"Ġpulls":16427,"ĠTide":16428,"Ġinsufficient":16429,"itted":16430,"Care":16431,"iera":16432,"Ġcouch":16433,"aders":16434,"ext":16435,"ĠCitizen":16436,"Ġlogical":16437,"ĠMeadows":16438,"ĠDenis":16439,"ĠDrivers":16440,"Ġrepublic":16441,"Ġadvising":16442,"Ġparamedics":16443,"insky":16444,"illard":16445,"encia":16446,"Ġkh":16447,"Ġrh":16448,"Ġfinalized":16449,"Ġreins":16450,"ĠFarrell":16451,"Ġsteer":16452,"Ġproxy":16453,"unes":16454,"ĠSoul":16455,"ĠCopper":16456,"ĠKenyan":16457,"amped":16458,"conference":16459,"sted":16460,"ĠLon":16461,"Ġreplay":16462,"ĠBle":16463,"Ġvibe":16464,"Ġportfolios":16465,"sea":16466,"Ġbeautifully":16467,"Ġairs":16468,"ĠRap":16469,"ĠKatrina":16470,"Ġberth":16471,"gold":16472,"ĠIsaiah":16473,"iques":16474,"elson":16475,"Ġrelentless":16476,"ĠHighland":16477,"ĠPhilippe":16478,"ĠFol":16479,"Ġenduring":16480,"enz":16481,"Ġaer":16482,"icing":16483,"ĠHTC":16484,"Ġdoping":16485,"ĠAlb":16486,"Ġsom":16487,"icia":16488,"Ġcoroner":16489,"Ġdamn":16490,"Ġ119":16491,"Ġwiped":16492,"ĠAuditor":16493,"hern":16494,"ĠJew":16495,"endra":16496,"osp":16497,"ĠRory":16498,"Ġshapes":16499,"ĠPablo":16500,"Ġforemost":16501,"ĠHos":16502,"ĠCunningham":16503,"145":16504,"ĠRecovery":16505,"!!!":16506,"western":16507,"Ġimaging":16508,"ĠRookie":16509,"ĠMTV":16510,"Ġunc":16511,"ĠSporting":16512,"Ġpatrons":16513,"ĠCoverage":16514,"ĠObservatory":16515,"Ġfishermen":16516,"ĠProvince":16517,"ĠAston":16518,"ĠOsh":16519,"ĠWeekend":16520,"Ġrecruits":16521,"Ġdensity":16522,"FM":16523,"ĠGorsuch":16524,"ĠErie":16525,"lining":16526,"Ġshowcased":16527,"ĠRubio":16528,"Ġchaotic":16529,"Ġattractions":16530,"Ġhug":16531,"ĠHerbert":16532,"ĠRespond":16533,"Ġhappily":16534,"Ġtor":16535,"ĠOTHER":16536,"runner":16537,"ĠShakespeare":16538,"Ġstretching":16539,"ĠJudy":16540,"wyn":16541,"ĠCafe":16542,"Ġgreens":16543,"ĠHend":16544,"Ġglam":16545,"iation":16546,"ĠKingston":16547,"Ġincremental":16548,"Live":16549,"ĠBraun":16550,"USS":16551,"reb":16552,"Ġimperative":16553,"Ġsympathy":16554,"Ġrefuge":16555,"Ġadministered":16556,"rance":16557,"ĠLiberia":16558,"Ġmobil":16559,"heads":16560,"Ġinevitably":16561,"ĠEugene":16562,"ĠBerkshire":16563,"ĠHarbour":16564,"ĠTrends":16565,"TB":16566,"Ġdeficits":16567,"Ġlistings":16568,"Ġreadings":16569,"Ġtumor":16570,"Ġoffic":16571,"opy":16572,"Ġdistracted":16573,"Ġappropriately":16574,"ĠWillis":16575,"Ġskirt":16576,"ĠTea":16577,"Ġshades":16578,"Ġbargaining":16579,"Ġretention":16580,"ĠConcert":16581,"ĠMeteor":16582,"ĠCustom":16583,"Ġinputs":16584,"ĠSah":16585,"enta":16586,"Love":16587,"ĠBurg":16588,"ĠCynthia":16589,"ĠMoses":16590,"ubb":16591,"Ġpeoples":16592,"dh":16593,"ĠFro":16594,"bean":16595,"Ġcigarette":16596,"tta":16597,"umm":16598,"Ġphenomenal":16599,"Ġyelling":16600,"Ġinaug":16601,"Ġconven":16602,"ĠGore":16603,"request":16604,"Ġcolonial":16605,"ĠAleppo":16606,"Ġdemolition":16607,"Ġamounted":16608,"Ġstaggering":16609,"Ġclips":16610,"Ġinconsistent":16611,"ĠMilton":16612,"ĠWireless":16613,"ĠReno":16614,"ĠPerkins":16615,"Ġunusually":16616,"Ġmemor":16617,"Ġhectares":16618,"Ġlat":16619,"central":16620,"ĠDig":16621,"ĠMarina":16622,"ĠPartner":16623,"daily":16624,"your":16625,"Reilly":16626,"Ġpope":16627,"phy":16628,"Ġassessing":16629,"ĠRodrigo":16630,"wi":16631,"Ġcompatible":16632,"imate":16633,"Ġgentle":16634,"ĠRhodes":16635,"Brexit":16636,"ieve":16637,"Ġbreaches":16638,"Ġchopped":16639,"Ġcancers":16640,"VEL":16641,"Ġsluggish":16642,"ĠUltra":16643,"ĠUl":16644,"Ġcrises":16645,"ONE":16646,"ĠEquipment":16647,"Ġcater":16648,"Ġadjourn":16649,"Ġreadily":16650,"ĠRolling":16651,"ĠBott":16652,"inel":16653,"ĠRule":16654,"Ġgrind":16655,"ĠHussain":16656,"ussie":16657,"Ġdepressed":16658,"ĠImperial":16659,"ongo":16660,"Ġuniforms":16661,"Ġ117":16662,"Ġchambers":16663,"ĠDum":16664,"ifi":16665,"ĠBetty":16666,"ĠTA":16667,"Ġpromotions":16668,"itary":16669,"Ġcried":16670,"Ġbranding":16671,"ĠBahamas":16672,"ĠDat":16673,"Ġantibiotics":16674,"ĠAus":16675,"Ġumbrella":16676,"Ġgradual":16677,"Ġaltercation":16678,"Ġlure":16679,"ĠJakarta":16680,"Ġunified":16681,"chin":16682,"ettes":16683,"ĠRwanda":16684,"ulations":16685,"Ġbrink":16686,"Ġbroadcasting":16687,"ĠArtist":16688,"Ġrecon":16689,"Ġaqu":16690,"ĠServ":16691,"999":16692,"ĠParticipants":16693,"ĠVentures":16694,"fight":16695,"Ġactivism":16696,"Ġstructured":16697,"Ġportal":16698,"Ġtendency":16699,"ĠAssociate":16700,"Ġcalf":16701,"ĠOrd":16702,"ĠTi":16703,"ĠFrancois":16704,"uary":16705,"ĠVik":16706,"urchase":16707,"Ġfried":16708,"Ġbooming":16709,"Ġparticles":16710,"amas":16711,"INA":16712,"Super":16713,"supp":16714,"urring":16715,"ĠWatts":16716,"affer":16717,"ĠDEC":16718,"Ġroadway":16719,"border":16720,"Ġsequ":16721,"entially":16722,"ieg":16723,"Ġcamping":16724,"Ġ750":16725,"Ġcycles":16726,"ĠReese":16727,"ĠFellow":16728,"isters":16729,"ĠVehicle":16730,"kies":16731,"ĠJonas":16732,"Ġfoundations":16733,"ĠNigel":16734,"Ġstab":16735,"Ġcongressman":16736,"ĠWichita":16737,"antes":16738,"Ġprogression":16739,"Ġditch":16740,"lik":16741,"Ġsid":16742,"Ġele":16743,"ĠMund":16744,"Ġstairs":16745,"lete":16746,"Ġlingering":16747,"Ġsadly":16748,"Ġay":16749,"Em":16750,"Ġdeadliest":16751,"soon":16752,"Ġtangible":16753,"Ġabusing":16754,"Ġcomprises":16755,"vil":16756,"ĠBun":16757,"Ġdoubling":16758,"Ġcommun":16759,"Ġslogan":16760,"Ġloading":16761,"Ġshallow":16762,"Ġattributes":16763,"Che":16764,"Ġcheering":16765,"Ġrefuses":16766,"cam":16767,"bes":16768,"hon":16769,"ĠSpartans":16770,"cept":16771,"ĠComputer":16772,"ĠCanberra":16773,"ĠWARNING":16774,"Ġstuffed":16775,"block":16776,"ĠJennings":16777,"ĠAU":16778,"atin":16779,"Ġom":16780,"Ġbachelor":16781,"Ġprediction":16782,"ĠWinner":16783,"agne":16784,"Ġrob":16785,"ĠKatherine":16786,"Ġli":16787,"ĠHumph":16788,"ĠPEOPLE":16789,"IRO":16790,"Cola":16791,"Ġguitarist":16792,"isen":16793,"ĠHighlights":16794,"Ġwelcomes":16795,"Ġprisoner":16796,"Ġpsychology":16797,"Ġextradition":16798,"Ġrou":16799,"ĠLund":16800,"Ġthoughtful":16801,"RY":16802,"orman":16803,"Alex":16804,"Ġlaughter":16805,"Ġfumble":16806,"Ġsynthetic":16807,"Ġdigit":16808,"ĠRoc":16809,"ĠFactory":16810,"ellery":16811,"ishment":16812,"ilar":16813,"ĠEarl":16814,"ĠSutton":16815,"ĠJur":16816,"ĠAllan":16817,"ĠKoreans":16818,"uki":16819,"Ġculinary":16820,"PU":16821,"Stock":16822,"stars":16823,"ĠDayton":16824,"beck":16825,"Ġinstability":16826,"ĠBring":16827,"Ġbreeding":16828,"Ġmiracle":16829,"bons":16830,"Ġdonating":16831,"ĠKick":16832,"ĠSag":16833,"afi":16834,"Ġharassed":16835,"asm":16836,"Their":16837,"inity":16838,"Ġacademics":16839,"Ġstatute":16840,"ĠAmit":16841,"Ġpressured":16842,"east":16843,"\"),":16844,"iso":16845,"220":16846,"Ġairplane":16847,"ĠMcCabe":16848,"ctions":16849,"ĠMesa":16850,"Ġsensational":16851,"ĠFE":16852,"ĠNeigh":16853,"Ġbribery":16854,"Ġflaws":16855,"Ġfemales":16856,"Ġmisses":16857,"ĠColor":16858,"ĠVietnamese":16859,"ĠMental":16860,"Unfortunately":16861,"ĠPont":16862,"Ġ1940":16863,"dry":16864,"ĠGazette":16865,"ĠAns":16866,"Ġwhistle":16867,"Ġsymbolic":16868,"Ġpossessions":16869,"ĠDriver":16870,"Ġbracket":16871,"ĠReign":16872,"oji":16873,"Ġoct":16874,"Ġtube":16875,"ĠFelix":16876,"Ġtranslated":16877,"Ġpromptly":16878,"ĠErnest":16879,"arth":16880,"Ġdumb":16881,"Ġinfluences":16882,"taking":16883,"Ġprivat":16884,"erers":16885,"Ġmalware":16886,"Ġpredictable":16887,"Ġtighten":16888,"Ġheights":16889,"Ġfairness":16890,"facing":16891,"Ġrematch":16892,"Ġpoet":16893,"Ġfundamentally":16894,"Ġcoveted":16895,"Ġlivelihood":16896,"ĠABOUT":16897,"Ġsourced":16898,"Ġdeferred":16899,"Ġslashed":16900,"ĠSchultz":16901,"Ġtriggering":16902,"ĠShiv":16903,"Ġlithium":16904,"ahead":16905,"Ġleisure":16906,"Ġbackpack":16907,"ilateral":16908,"ĠNuclear":16909,"ĠLeone":16910,"ĠNice":16911,"Ġenthusiasts":16912,"September":16913,"Ġenroll":16914,"ĠWear":16915,"erey":16916,"angs":16917,"such":16918,"Ġunpopular":16919,"Ġdisciplined":16920,"Ġshrinking":16921,"ĠBrewing":16922,"ĠReally":16923,"Ġdirective":16924,"175":16925,"Ġnotifications":16926,"Ġfortunes":16927,"ĠHour":16928,"ĠGan":16929,"ĠChurchill":16930,"ĠDodge":16931,"ĠJeep":16932,"Ġsour":16933,"Ġderived":16934,"Ġft":16935,"riv":16936,"Ġlaundry":16937,"Ġfentanyl":16938,"ĠSioux":16939,"achi":16940,"workers":16941,"Ġworkload":16942,"rooms":16943,"ĠQU":16944,"ĠTruth":16945,"Ġdefenses":16946,"Ġdunk":16947,"IJ":16948,"Ġderby":16949,"ĠMotion":16950,"ĠMayo":16951,"ĠIke":16952,"Ġpreferences":16953,"Ġped":16954,"elman":16955,"moon":16956,"Ġshoots":16957,"ĠNoel":16958,"Ġmilit":16959,"ĠCambodia":16960,"ĠMLA":16961,"Ġhonoured":16962,"fast":16963,"Ġalgorithms":16964,"Ġstormed":16965,"NT":16966,"Benz":16967,"Ġvaccines":16968,"Ġmarching":16969,"Ġ118":16970,"ĠWilmington":16971,"GM":16972,"coin":16973,"Ġunderwater":16974,"ĠClearly":16975,"Ġorgans":16976,"mir":16977,"Ġdenounced":16978,"pless":16979,"imal":16980,"ĠKom":16981,"Ġfatalities":16982,"Ġyoungster":16983,"Ġthirty":16984,"Ġinternally":16985,"222":16986,"Ġdemonstrating":16987,"Ġbusiest":16988,"Ġperpetrators":16989,"Ġstun":16990,"Both":16991,"ĠMcCoy":16992,"gn":16993,"ĠDalton":16994,"ĠDAY":16995,"Ġsacred":16996,"Ġconsuming":16997,"Ġ(+":16998,"ĠPioneer":16999,"ĠApplications":17000,"ĠBolt":17001,"ĠBarkley":17002,"ĠExpo":17003,"ĠLore":17004,"ĠPrivacy":17005,"ĠHarley":17006,"Ġtractor":17007,"Ġtenth":17008,"ĠHaiti":17009,"ÃŃn":17010,"ĠTVs":17011,"ĠCathedral":17012,"Ġunite":17013,"Ġbinding":17014,"oks":17015,"ĠJenny":17016,"Ġcaller":17017,"ĠIngram":17018,"ĠPrairie":17019,"Ġrunoff":17020,"Ġasserted":17021,"icit":17022,"ĠSie":17023,"102":17024,"ĠMB":17025,"Ġobstruction":17026,"Ġgroom":17027,"Ġtolerate":17028,"Ġcans":17029,"forth":17030,"Ġvillain":17031,"Ġdefining":17032,"ĠFrenchman":17033,"otte":17034,"Ġcontr":17035,"clock":17036,"onder":17037,"Ġprolific":17038,"ĠElectronic":17039,"ĠSak":17040,"annie":17041,"ASS":17042,"Ġmultinational":17043,"Associated":17044,"IZ":17045,"ĠBelle":17046,"Ġmand":17047,"asis":17048,"Mac":17049,"Ġpretend":17050,"ĠCommunication":17051,"Ġheartbreaking":17052,"ĠShepherd":17053,"ĠBIG":17054,"mph":17055,"ĠShield":17056,"ĠLiv":17057,"ĠStatus":17058,"Ġbikini":17059,"Ġranch":17060,"Ġpeacefully":17061,"ITCH":17062,"bourne":17063,"ĠVariety":17064,"Ġstationed":17065,"Ġhed":17066,"Ġexhausted":17067,"Ġsurpassed":17068,"Ġcatalyst":17069,"Ġsmuggling":17070,"uating":17071,"Ġ123":17072,"Ġdup":17073,"ĠSul":17074,"conf":17075,"jit":17076,"Ġmaiden":17077,"asta":17078,"ĠCalvin":17079,"borne":17080,"Ġgrim":17081,"Ġtort":17082,"cott":17083,"olas":17084,"NR":17085,"Ġbreakout":17086,"ĠHun":17087,"ĠGuatemala":17088,"Ġhistorian":17089,"ĠLawyers":17090,"ĠDisplay":17091,"Ġobstruct":17092,"ĠOsborne":17093,"Ġtherapies":17094,"ĠAub":17095,"Ġinjunction":17096,"stroke":17097,"Ġseafood":17098,"Ġhazardous":17099,"ĠWolver":17100,"ĠViolence":17101,"ĠBillion":17102,"ĠLetter":17103,"ĠWorldwide":17104,"Real":17105,"Ġexpires":17106,"Ġflawed":17107,"European":17108,"Ġrigorous":17109,"ĠSimilar":17110,"ĠSurface":17111,"ĠEF":17112,"mys":17113,"ĠFunds":17114,"ographer":17115,"Ġtribes":17116,"Ġspouse":17117,"Ġunsure":17118,"aways":17119,"Ġtrainers":17120,"arie":17121,"ĠZar":17122,"ĠComedy":17123,"ĠLit":17124,"ĠNoon":17125,"Ġgallon":17126,"Ġconsulate":17127,"ĠBras":17128,"iology":17129,"onies":17130,"ĠBelichick":17131,"ĠRoot":17132,"ĠLux":17133,"ĠSed":17134,"ĠTos":17135,"Ġinherited":17136,"tw":17137,"Ġdeaf":17138,"Ġdriveway":17139,"jah":17140,"ĠScientific":17141,"ĠNottingham":17142,"both":17143,"awan":17144,"Ġnut":17145,"ĠLebanese":17146,"ĠAAA":17147,"ĠSuzuki":17148,"ĠBU":17149,"ells":17150,"Ġspecify":17151,"ĠNotes":17152,"Ġvoluntarily":17153,"ĠMolly":17154,"Ġoutskirts":17155,"Ġbehaviors":17156,"Ġmilitia":17157,"Ġsplash":17158,"Ġpersonalized":17159,"ĠFiat":17160,"ĠKind":17161,"ĠTruck":17162,"py":17163,"ĠWIN":17164,"dist":17165,"itational":17166,"APP":17167,"ĠPelicans":17168,"ĠGam":17169,"mel":17170,"Ġmandated":17171,"Ġbalances":17172,"ĠWizards":17173,"iary":17174,"ĠAvailable":17175,"Ġkay":17176,"jin":17177,"eyed":17178,"Ġsterling":17179,"Ġconcealed":17180,"ĠFedEx":17181,"ĠPO":17182,"ĠJacqu":17183,"anted":17184,"eme":17185,"ĠDefensive":17186,"manship":17187,"Ġreliever":17188,"Ġshortstop":17189,"Ġphot":17190,"ĠGain":17191,"ĠConcern":17192,"due":17193,"Ġalgorithm":17194,"fell":17195,"ĠMountains":17196,"icians":17197,"Ġhonoring":17198,"Ġuploaded":17199,"Ġtore":17200,"GH":17201,"orde":17202,"ĠCoin":17203,"ĠAven":17204,"Ġliterary":17205,"Before":17206,"Ġtactic":17207,"Ġsocially":17208,"ĠSik":17209,"Ġthermal":17210,"Ġhor":17211,"price":17212,"Ġrooted":17213,"arrow":17214,"Ġcirculating":17215,"Ġlaughs":17216,"ĠLines":17217,"lig":17218,"Ġjudgement":17219,"....":17220,"Ġsewer":17221,"Ġdancer":17222,"ĠPens":17223,"Ġsig":17224,"ische":17225,"wives":17226,"Ġgran":17227,"ĠBron":17228,"ĠHyde":17229,"yards":17230,"Ġcandidacy":17231,"Ġhey":17232,"Ġcontributors":17233,"ĠUpdated":17234,"Ġ190":17235,"Ġhalls":17236,"Ġemphas":17237,"ĠCherry":17238,"Ġrim":17239,"Ġbilled":17240,"Ġbaked":17241,"ĠPopular":17242,"lb":17243,"Ġgravity":17244,"Under":17245,"Ġreservation":17246,"organ":17247,"ĠPict":17248,"ĠWhitney":17249,"Ġonboard":17250,"NEY":17251,"ĠBreaking":17252,"Ġflagged":17253,"rar":17254,"ĠBasic":17255,"ĠDomestic":17256,"ĠPent":17257,"Ġvigilant":17258,"Ġzoning":17259,"Fire":17260,"Ġcorrected":17261,"isbury":17262,"ĠLaure":17263,"ĠDevon":17264,"print":17265,"ĠTopics":17266,"ĠFuel":17267,"Ġcirculation":17268,"ĠPratt":17269,"Ġskiing":17270,"Ġtornado":17271,"dep":17272,"ĠUnless":17273,"ifting":17274,"Ġfool":17275,"should":17276,"Ġinspectors":17277,"Ġprotested":17278,"Ġba":17279,"ussia":17280,"Ġspun":17281,"grass":17282,"phone":17283,"Ġpotato":17284,"ĠBehind":17285,"cil":17286,"Ġconcession":17287,"Ġapplause":17288,"ĠChin":17289,"Ġceremonies":17290,"pit":17291,"Ġtraumatic":17292,"Ġbasics":17293,"Ġparameters":17294,"ĠMoz":17295,"ĠAIDS":17296,"Ph":17297,"Ġjudging":17298,"Ġlecture":17299,"Ġmunicipality":17300,"Ġcardiac":17301,"ogan":17302,"pir":17303,"could":17304,"Channel":17305,"Ġshattered":17306,"ĠAV":17307,"continental":17308,"chie":17309,"ibi":17310,"ĠOy":17311,"Mon":17312,"ĠCN":17313,"WC":17314,"Ġdistributor":17315,"ĠSavannah":17316,"Ġcleaned":17317,"ĠFlores":17318,"Ġembarrassed":17319,"Ġclay":17320,"Ġvolcano":17321,"Ġstressful":17322,"Ġsummoned":17323,"ĠSeg":17324,"Ġstatistical":17325,"ĠShak":17326,"Ġadequately":17327,"worthy":17328,"fighting":17329,"alan":17330,"Ġnecessity":17331,"Ġresidency":17332,"Ġsober":17333,"arius":17334,"ĠTaj":17335,"mount":17336,"wards":17337,"Ġaesthetic":17338,"Coin":17339,"ĠDew":17340,"were":17341,"SK":17342,"Ġpowerhouse":17343,"Ġcleanup":17344,"ĠWITH":17345,"ĠHers":17346,"ĠRao":17347,"ĠFlyers":17348,"Ġdominating":17349,"issued":17350,"ĠMcGr":17351,"Ġinsurgency":17352,"Ġburial":17353,"ĠPlains":17354,"ensive":17355,"ĠPresent":17356,"Mo":17357,"Ġnerves":17358,"Ġsmoothly":17359,"staff":17360,"Ġrestoring":17361,"ĠGeneration":17362,"Ġcommuters":17363,"ĠLegend":17364,"ĠGad":17365,"lied":17366,"Ġissuer":17367,"ĠDozens":17368,"Ġphases":17369,"ĠWu":17370,"ĠTunisia":17371,"ĠPacers":17372,"Ġdur":17373,"ĠIG":17374,"annon":17375,"sided":17376,"Ġvo":17377,"ĠNI":17378,"Ġvitamin":17379,"Ġsoc":17380,"Ġimmunity":17381,"Ġgenerates":17382,"ĠMcGu":17383,"Ġexplores":17384,"Ġassistants":17385,"Ġstems":17386,"ushed":17387,"ĠZak":17388,"ĠOwners":17389,"Ġvariant":17390,"ardy":17391,"ĠNewark":17392,"ĠCatalonia":17393,"Ġautonomy":17394,"Ġgreet":17395,"Ġawait":17396,"ĠLuckily":17397,"ĠTicket":17398,"ĠSTOR":17399,"asy":17400,"Ġincorrect":17401,"Ġconsisting":17402,"Ġperspectives":17403,"ĠQuint":17404,"Ġtotaling":17405,"Ġnortheastern":17406,"Ġcharacterized":17407,"Ġsurfaces":17408,"nation":17409,"Ġprevents":17410,"ĠSho":17411,"Ġelectorate":17412,"Ġshortfall":17413,"chy":17414,"aws":17415,"ĠAddress":17416,"Ġdefensively":17417,"quel":17418,"chester":17419,"Ġterr":17420,"ahu":17421,"lined":17422,"ĠNev":17423,"unn":17424,"Def":17425,"pc":17426,"ĠSig":17427,"Ġnonetheless":17428,"ĠSundays":17429,"ĠBAS":17430,"Ġpolicemen":17431,"ĠGoal":17432,"apa":17433,"Ġrope":17434,"Ġoutage":17435,"ĠPaso":17436,"Ġsadness":17437,"ĠGrowing":17438,"ĠKyr":17439,"Ġale":17440,"ĠBreitbart":17441,"ĠVia":17442,"ĠBrig":17443,"idence":17444,"Ġ145":17445,"quire":17446,"Ġdistraction":17447,"ĠOdd":17448,"ĠSimply":17449,"ĠNin":17450,"Ġcompetent":17451,"ded":17452,"iper":17453,"ĠKaty":17454,"ĠSolomon":17455,"Ġfeeds":17456,"ĠMort":17457,"ĠRica":17458,"affe":17459,"Ġcooperating":17460,"Ġarrivals":17461,"Ġdelete":17462,"ĠAth":17463,"Ġtrustees":17464,"Ġtub":17465,"Ġsaga":17466,"otes":17467,"ĠCJ":17468,"Ġexited":17469,"stakes":17470,"Ġinflu":17471,"2000":17472,"ĠDonovan":17473,"ĠNur":17474,"Ġoutline":17475,"Ġaudition":17476,"oked":17477,"ĠJag":17478,"money":17479,"Ġcardiovascular":17480,"song":17481,"ĠOften":17482,"ĠGoff":17483,"ĠOaks":17484,"Will":17485,"acon":17486,"Ġ?":17487,"Har":17488,"ĠLambert":17489,"atoon":17490,"ĠAF":17491,"ĠMavericks":17492,"nia":17493,"ĠChennai":17494,"\"},\"":17495,"Ġpairing":17496,"mad":17497,"ause":17498,"ĠRide":17499,"111":17500,"ĠFallon":17501,"ĠHyder":17502,"ĠPiper":17503,"Ġfilmmakers":17504,"icon":17505,"ĠBeau":17506,"Ġbutt":17507,"lot":17508,"Ġrifles":17509,"Ġsunglasses":17510,"ĠTRA":17511,"Ġmagnetic":17512,"arty":17513,"ĠYo":17514,"ĠWeight":17515,"?!":17516,"ether":17517,"Ġaspir":17518,"Ġhunters":17519,"Ġcontamination":17520,"Ben":17521,"political":17522,"],\"":17523,"ĠBever":17524,"Ġmonuments":17525,"won":17526,"auc":17527,"Ġexpressions":17528,"Ġlakes":17529,"iao":17530,"abin":17531,"Ġpleading":17532,"Ġdiscounted":17533,"Ġdisappoint":17534,"ĠTW":17535,"craft":17536,"Ġsocieties":17537,"ĠAugusta":17538,"Ġbott":17539,"Ġmarker":17540,"ĠWrestling":17541,"CBC":17542,"athy":17543,"ĠAZ":17544,"Ġfabulous":17545,"valued":17546,"Ġoptical":17547,"Ġshaken":17548,"OSS":17549,"ĠImp":17550,"ĠAUD":17551,"inals":17552,"Ġrevital":17553,"Ġcontroller":17554,"Ġgrasp":17555,"uling":17556,"ĠFrederick":17557,"ague":17558,"bull":17559,"ĠLadies":17560,"Ġdisruptive":17561,"Ġbenefiting":17562,"Ġverge":17563,"ĠDak":17564,"Ġgrabs":17565,"ĠPAC":17566,"GN":17567,"ĠMcMahon":17568,"rob":17569,"ĠEspecially":17570,"ĠChrome":17571,"ĠBundesliga":17572,"104":17573,"Ġliberty":17574,"ĠSF":17575,"Ġvarieties":17576,"East":17577,"Ġgrowers":17578,"Ġsocialist":17579,"Ġunemployed":17580,"AMI":17581,"Ġtotals":17582,"ĠGib":17583,"Ġdefect":17584,"ĠOrtiz":17585,"ĠPerfect":17586,"Ġpraying":17587,"ISS":17588,"Ġul":17589,"Ġthrust":17590,"osc":17591,"ĠOtherwise":17592,"Ġobsessed":17593,"Ġ650":17594,"ĠWebsite":17595,"Ġspectators":17596,"ĠScout":17597,"ĠBoone":17598,"ĠDillon":17599,"Ġabortions":17600,"lect":17601,"utz":17602,"Ġvillagers":17603,"Ġaccelerating":17604,"Ġslap":17605,"Ġvague":17606,"Ġjurisdictions":17607,"League":17608,"ĠUruguay":17609,"Ġobstacle":17610,"Ġmanufactures":17611,"Ġcampaigned":17612,"ĠAdvance":17613,"ĠNort":17614,"emer":17615,"Ġ1964":17616,"Ġirre":17617,"Ġprog":17618,"ĠFeatured":17619,"Ġcommute":17620,"Ġhandset":17621,"akis":17622,"ĠArs":17623,"tail":17624,"iker":17625,"Ġcrafted":17626,"Ġupl":17627,"ĠMarcos":17628,"Looking":17629,"Ġseated":17630,"ĠBoat":17631,"Ġreadiness":17632,"ĠLLP":17633,"otechnology":17634,"facebook":17635,"ĠScouts":17636,"ĠEar":17637,"ĠAdv":17638,"ĠDemocracy":17639,"NI":17640,"oci":17641,"ĠSnapdragon":17642,"Saturday":17643,"ĠPra":17644,"ĠCoastal":17645,"ĠVoters":17646,"ĠLeigh":17647,"ohn":17648,"orry":17649,"Ġtechnicians":17650,"armed":17651,"Ġshrink":17652,"Ġspinning":17653,"agram":17654,"320":17655,"liner":17656,"ĠContest":17657,"ĠCountries":17658,"Ġfarewell":17659,"ĠCW":17660,"aris":17661,"Ġstorytelling":17662,"Ġpasser":17663,"Ġsailing":17664,"control":17665,"Ġdissent":17666,"ĠRih":17667,"Ġedit":17668,"Ġspoilers":17669,"itched":17670,"ĠBentley":17671,"Ġcant":17672,"mn":17673,"ĠMacy":17674,"Ġindefinitely":17675,"Ġvill":17676,"Ġmeth":17677,"ĠEL":17678,"Ġoptional":17679,"Ġremark":17680,"ĠVanessa":17681,"ã":17682,"Ġmasks":17683,"ĠProvincial":17684,"Ġculprit":17685,"ĠTol":17686,"Ġsnack":17687,"ĠInfinity":17688,"ĠPub":17689,"Ġbrakes":17690,"Ġclar":17691,"Ġinception":17692,"love":17693,"Ġwonders":17694,"Ġforged":17695,"ĠCEOs":17696,"Ġspecifications":17697,"irst":17698,"ension":17699,"ĠMarin":17700,"det":17701,"Ġordeal":17702,"ĠFeed":17703,"December":17704,"Ġstrokes":17705,"fect":17706,"orial":17707,"Ġshowcasing":17708,"Ġstack":17709,"UAL":17710,"ĠAlexandra":17711,"Ġpoison":17712,"ĠFry":17713,"ĠCars":17714,"Ġprototype":17715,"ĠUSDA":17716,"ĠIF":17717,"flows":17718,"Ġtailored":17719,"ĠGear":17720,"Ġmyth":17721,"Ġplatinum":17722,"seven":17723,"founded":17724,"encing":17725,"ĠTip":17726,"ĠMald":17727,"Ġgeopolitical":17728,"112":17729,"Ġenqu":17730,"ĠNR":17731,"ĠNadu":17732,"leen":17733,"ĠTat":17734,"Ġcolon":17735,"ĠSize":17736,"Ġvis":17737,"Ġbere":17738,"ĠAnnie":17739,"ĠWatkins":17740,"Ġpumping":17741,"cur":17742,"ĠBates":17743,"Ġslug":17744,"miss":17745,"Ġforecasting":17746,"source":17747,"Ġacknowledges":17748,"Ġprosecute":17749,"Ġtestament":17750,"Ġcum":17751,"ems":17752,"Ġsocks":17753,"ĠSame":17754,"Ġcompetitiveness":17755,"Ġdefinitive":17756,"Ġintensified":17757,"Ġsatisfying":17758,"Ġphysics":17759,"ĠHarden":17760,"Ġsubsidy":17761,"Men":17762,"ĠPaddock":17763,"Ġworkouts":17764,"ĠSaw":17765,"Ġcrisp":17766,"ĠBezos":17767,"ĠVote":17768,"Ġguiding":17769,"anged":17770,"Ġstaple":17771,"ŀ":17772,"ules":17773,"ĠAvengers":17774,"Ġoptim":17775,"ĠBuffett":17776,"Ġtimetable":17777,"oust":17778,"HE":17779,"ĠGrab":17780,"Have":17781,"cca":17782,"Ġwaived":17783,"Ġretaining":17784,"Ġaber":17785,"Ġoffline":17786,"Ġvigil":17787,"books":17788,"ĠRein":17789,"Ġacknowledging":17790,"ĠDoyle":17791,"Ġproteins":17792,"Ġmixing":17793,"ĠAlcohol":17794,"ĠJD":17795,"Ġsyn":17796,"Ġthieves":17797,"Ġhomemade":17798,"Ġfeminist":17799,"ĠRoosevelt":17800,"ĠCoal":17801,"Ġwishing":17802,"ĠSIGN":17803,"ĠLad":17804,"Ġempathy":17805,"ĠBrooke":17806,"ĠMash":17807,"inations":17808,"''":17809,"ulators":17810,"Ġdrastically":17811,"Ġfloral":17812,"ĠGuild":17813,"Ġundercover":17814,"ĠLaboratory":17815,"ĠRank":17816,"Ġrestraining":17817,"Ġparagraph":17818,"Ġpersona":17819,"ĠEmployment":17820,"ogs":17821,"ĠGw":17822,"ĠMedal":17823,"Ġwildly":17824,"fare":17825,"ĠCNBC":17826,"photo":17827,"Ġtransforming":17828,"Ġtermination":17829,"still":17830,"INT":17831,"Ġbal":17832,"ĠEconom":17833,"ĠLarson":17834,"Ġheck":17835,"Ġquantitative":17836,"Ġemergence":17837,"esta":17838,"Ġknot":17839,"Ġwhale":17840,"ĠðŁĺ":17841,"Ġperimeter":17842,"Ġempowerment":17843,"Ġmg":17844,"Ġrents":17845,"Ġrefreshing":17846,"Ġleasing":17847,"Ġpatents":17848,"andi":17849,"Ġfathers":17850,"Ġunse":17851,"Ġprocessors":17852,"Down":17853,"Ġreversal":17854,"veh":17855,"andal":17856,"ĠKov":17857,"Blue":17858,"Ġspecializes":17859,"Link":17860,"ĠConsidering":17861,"ĠEdmund":17862,"Ġneo":17863,"agger":17864,"rg":17865,"Ġseverity":17866,"Ġcour":17867,"RL":17868,"ĠTeresa":17869,"Ġgallons":17870,"Ġacquitted":17871,"Ġaccompl":17872,"Ġcracks":17873,"Ġsciences":17874,"Club":17875,"Ġpredicts":17876,"ĠVu":17877,"Ġhints":17878,"ĠZack":17879,"Ġrefurb":17880,"Ġdestabil":17881,"ĠSamar":17882,"ĠInfo":17883,"fs":17884,"Ġratios":17885,"Ġinherent":17886,"ĠContinental":17887,"Ġtreasure":17888,"Ġcaucus":17889,"Ġenact":17890,"orporated":17891,"ineries":17892,"Ġtastes":17893,"main":17894,"Ġsq":17895,"ickson":17896,"corruption":17897,"ulture":17898,"ĠGoodman":17899,"ĠLing":17900,"ĠSup":17901,"Ġexposing":17902,"immers":17903,"Ġresponds":17904,"heimer":17905,"Air":17906,"ĠFigures":17907,"Ġlongstanding":17908,"ĠAnalytics":17909,"Ġenforced":17910,"Ġnickname":17911,"Ġclinch":17912,"ĠCarpenter":17913,"ĠPharma":17914,"Ġconstructive":17915,"Ġgel":17916,"ĠSham":17917,"ĠTOP":17918,"ĠDerrick":17919,"ör":17920,"birds":17921,"ĠTong":17922,"ĠBatman":17923,"ĠRouhani":17924,"ĠOlive":17925,"ĠRiv":17926,"Ġdessert":17927,"Ġguides":17928,"Ġsag":17929,"Ġchemotherapy":17930,"Ġslept":17931,"ĠFranc":17932,"ĠDunk":17933,"writers":17934,"ĠÃĹ":17935,"Ġ401":17936,"Ġoutfielder":17937,"ĠHamburg":17938,"izu":17939,"Ġscr":17940,"Ġcomparisons":17941,"Ġwhites":17942,"Ġtraits":17943,"Ġcollateral":17944,"LEY":17945,"ideshow":17946,"Ġstatutory":17947,"Ġruin":17948,"Ġsituated":17949,"tem":17950,"Ġinject":17951,"rage":17952,"550":17953,"Ġfactions":17954,"ĠNaomi":17955,"cutting":17956,"Ġcommunicating":17957,"Ġrailroad":17958,"Ġsparking":17959,"Ġrespiratory":17960,"ĠWebster":17961,"ĠCarbon":17962,"Ġundertaking":17963,"Ġcomposer":17964,"ĠFigure":17965,"Ġspecified":17966,"Video":17967,"uber":17968,"Ġsexuality":17969,"lected":17970,"ĠBurger":17971,"ĠCards":17972,"SR":17973,"ĠLie":17974,"Ġrecount":17975,"Ġexceeding":17976,"Ġquoting":17977,"ĠJama":17978,"ĠVictorian":17979,"Ġsway":17980,"ĠGes":17981,"ĠSI":17982,"ĠKazakhstan":17983,"Ġaccusation":17984,"etr":17985,"Ah":17986,"Ġproc":17987,"Ġlamb":17988,"ĠMorales":17989,"ĠLily":17990,"Ġderail":17991,"Ġcontributes":17992,"iddle":17993,"ĠConcord":17994,"Ġelectr":17995,"Ġequip":17996,"Ġquantum":17997,"Ġthereafter":17998,"Ġarrange":17999,"Ġraided":18000,"ĠMove":18001,"ĠSang":18002,"ĠGaming":18003,"Ġbiology":18004,"ĠAmnesty":18005,"Ġdemise":18006,"ĠBarton":18007,"Ġqualifier":18008,"ANI":18009,"Ġundersc":18010,"Ġroyalty":18011,"ĠINC":18012,"Ġsne":18013,"ariat":18014,"ĠWan":18015,"Ġcluster":18016,"quin":18017,"Ġwhales":18018,"ĠFear":18019,"ĠBrew":18020,"Ġdeport":18021,"airs":18022,"Ġcensus":18023,"OUS":18024,"Ġrespectful":18025,"bone":18026,"Ġwaivers":18027,"friend":18028,"Ġsystemic":18029,"ĠDion":18030,"James":18031,"ĠAdmission":18032,"Ġstigma":18033,"ĠTIME":18034,"Ġunderpin":18035,"ĠWitnesses":18036,"Ġdigs":18037,"Ġgenocide":18038,"Ġstaging":18039,"rolled":18040,"Ġspecially":18041,"oop":18042,"Ġbaseline":18043,"ĠRF":18044,"avis":18045,"Ġvocals":18046,"COL":18047,"LD":18048,"Ġimpending":18049,"ĠCaldwell":18050,"Ġaluminium":18051,"Ġstra":18052,"ĠTayyip":18053,"Ġadmissions":18054,"falls":18055,"Ġrealizing":18056,"oen":18057,"ĠRV":18058,"ĠMog":18059,"Ġadvocating":18060,"ĠPepper":18061,"lived":18062,"ĠWick":18063,"Facebook":18064,"ĠSpect":18065,"Ġshout":18066,"Ġfractured":18067,"vet":18068,"Ġ1966":18069,"Ġcompensate":18070,"ĠVolume":18071,"Ġcategor":18072,"ĠHuntington":18073,"Free":18074,"OUGH":18075,"local":18076,"Sch":18077,"uti":18078,"Ġburger":18079,"Ġbush":18080,"Ġimpacting":18081,"Ġfrost":18082,"tti":18083,"ĠFresno":18084,"onz":18085,"shaw":18086,"ĠLibyan":18087,"Ġassert":18088,"ĠLegacy":18089,"ĠIE":18090,"ĠKinder":18091,"ĠHorizon":18092,"Ġtum":18093,"Ġsignaled":18094,"ĠFors":18095,"Ġspeedy":18096,"rang":18097,"ĠFT":18098,"Ġselecting":18099,"Ġpale":18100,"WD":18101,"Ġprobability":18102,"OUND":18103,"istrate":18104,"Ġsens":18105,"ocating":18106,"Ġinterpret":18107,"Ġpuzzle":18108,"Ġinland":18109,"Ġmanipulation":18110,"Sal":18111,"Ġfulfilling":18112,"ĠMcMaster":18113,"Make":18114,"jun":18115,"giving":18116,"ĠNiagara":18117,"Ġscholars":18118,"ALT":18119,"ĠSteam":18120,"omin":18121,"ĠSau":18122,"ĠDowning":18123,"Ġgy":18124,"ĠTit":18125,"ĠLav":18126,"ĠPepsi":18127,"Ġdumping":18128,"ĠDetect":18129,"ĠTDs":18130,"ĠKob":18131,"ĠSY":18132,"Ġpioneer":18133,"Ġ_":18134,"Ġclarified":18135,"ĠTests":18136,"opic":18137,"ĠMN":18138,"ĠBowman":18139,"umin":18140,"Ġwidow":18141,"Ġrallying":18142,"ĠPull":18143,"Ġprojection":18144,"Ġescalation":18145,"Ġlibraries":18146,"ĠFounder":18147,"ĠHugo":18148,"ĠStyle":18149,"Ġfreelance":18150,"Ġlisteners":18151,"Ġdiscovering":18152,"ĠPlans":18153,"Ġfranchises":18154,"ĠPam":18155,"Ġfarther":18156,"UI":18157,"opers":18158,"103":18159,"ublished":18160,"keys":18161,"aky":18162,"Ġinnov":18163,"¦":18164,"ĠDrum":18165,"Ġwraps":18166,"ĠCongressman":18167,"ĠVenus":18168,"fake":18169,"ĠBronx":18170,"ĠDinner":18171,"faced":18172,"Ġbackward":18173,"inge":18174,"Ġarsenal":18175,"ĠAce":18176,"uden":18177,"fre":18178,"Ġspa":18179,"ĠSaunders":18180,"ĠMatter":18181,"ĠSpons":18182,"Ġconsultations":18183,"ĠRuss":18184,"Ġsculpture":18185,"Ġuncommon":18186,"Nov":18187,"pg":18188,"otherapy":18189,"Ġgol":18190,"ĠBlazers":18191,"Ġadvises":18192,"ĠRegulatory":18193,"ĠBoyle":18194,"Äģ":18195,"Ġcuisine":18196,"Ġencouragement":18197,"yp":18198,"eny":18199,"ĠOrchestra":18200,"ĠChicken":18201,"Ġ1965":18202,"ĠPret":18203,"ĠCooperation":18204,"ĠDevices":18205,"ĠRodney":18206,"ĠHonduras":18207,"ĠEgg":18208,"Ġchurn":18209,"Ġclutch":18210,"ĠBernstein":18211,"Ġain":18212,"Ġformidable":18213,"ĠFacility":18214,"Ġpag":18215,"mons":18216,"bol":18217,"Ġliteracy":18218,"Ġsubmissions":18219,"ĠHulu":18220,"ĠConstitutional":18221,"ĠIsh":18222,"ĠPaula":18223,"olve":18224,"Ġabundance":18225,"ĠAla":18226,"ĠEcuador":18227,"Ġreconstruction":18228,"Ġcrush":18229,"reek":18230,"ĠÂŃ":18231,"ibo":18232,"Ġpracticed":18233,"Ġpac":18234,"rett":18235,"Ġpasta":18236,"Ġresp":18237,"ĠFlag":18238,"pal":18239,"Ġcommenting":18240,"Ġrecap":18241,"âĢĶâĢĶ":18242,"ĠToy":18243,"ĠMeredith":18244,"Ġreceipt":18245,"Ġseparating":18246,"ĠMap":18247,"Ġmogul":18248,"ĠBurlington":18249,"Ġger":18250,"Ġcoordinate":18251,"grad":18252,"Ġescalated":18253,"Ġproceeded":18254,"turned":18255,"Ġupt":18256,"hum":18257,"ĠWere":18258,"Whether":18259,"Ġenjoyable":18260,"energy":18261,"Ġprohibit":18262,"Ġhurdle":18263,"Ġdivorced":18264,"Ġcommentator":18265,"GT":18266,"ATH":18267,"Ġtravellers":18268,"Ġpopulated":18269,"ĠVo":18270,"ĠRebels":18271,"Ġspurred":18272,"Ġideological":18273,"Ġelephant":18274,"keyes":18275,"Pat":18276,"Ġlinger":18277,"Ġreps":18278,"Ġcocktails":18279,"ĠKristen":18280,"istically":18281,"Ġgunmen":18282,"Ġ1920":18283,"Ġquart":18284,"National":18285,"Ġexceptions":18286,"kat":18287,"priced":18288,"ĠHarold":18289,"ĠPistons":18290,"Ġcompounds":18291,"Ġmouse":18292,"Ġexhibits":18293,"ĠBurk":18294,"Ġclassmates":18295,"Ġcirculated":18296,"Ġattributable":18297,"ĠBaton":18298,"Ġorganizer":18299,"Ġdurable":18300,"Ġsingers":18301,"ĠOman":18302,"Ġhydrogen":18303,"Ġslash":18304,"Ġaccidental":18305,"ĠAbrams":18306,"KS":18307,"itty":18308,"Ġrust":18309,"Ġselections":18310,"porting":18311,"ĠEmanuel":18312,"XX":18313,"ĠThornton":18314,"Ġcolumns":18315,"Ġsentiments":18316,"fun":18317,"Ġplight":18318,"ĠSister":18319,"ĠMaggie":18320,"hya":18321,"Daniel":18322,"Ġplung":18323,"orio":18324,"ĠYorker":18325,"ĠSaturdays":18326,"Ġloc":18327,"aye":18328,"illon":18329,"ĠConsulting":18330,"pled":18331,"ĠZin":18332,"ĠFarms":18333,"ĠGiuliani":18334,"ĠMIN":18335,"ĠHanson":18336,"ĠComplete":18337,"ourke":18338,"oche":18339,"ĠJord":18340,"Ġprofessors":18341,"ĠWILL":18342,"ĠCron":18343,"Ġdorm":18344,"Ġcracking":18345,"tur":18346,"ORS":18347,"Ant":18348,"Ġdeduction":18349,"ĠSIM":18350,"igue":18351,"ĠValent":18352,"ĠEthereum":18353,"ĠSunny":18354,"ĠExtra":18355,"ivan":18356,"ĠFo":18357,"Ġleases":18358,"ibe":18359,"Ġ1800":18360,"Ġslapped":18361,"emaker":18362,"Ġfa":18363,"rien":18364,"ĠPeriod":18365,"ĠES":18366,"ĠBlu":18367,"Ġpreserving":18368,"Ġsmarter":18369,"mans":18370,"Ġgest":18371,"zu":18372,"nu":18373,"Ġdivest":18374,"roc":18375,"ĠFlood":18376,"Given":18377,"ĠNorton":18378,"Ġgranting":18379,"Ġdealings":18380,"Ġgeographic":18381,"esa":18382,"Ġcub":18383,"Ġcriticizing":18384,"ĠCub":18385,"Ġsurroundings":18386,"ĠInternal":18387,"Ġsle":18388,"Ġcrushing":18389,"ĠPP":18390,"izations":18391,"ĠAbdel":18392,"Joe":18393,"ĠVisitors":18394,"ĠCarly":18395,"INGTON":18396,"ĠGC":18397,"ĠWB":18398,"Ġgently":18399,"·":18400,"though":18401,"ĠAlto":18402,"Ġresting":18403,"ĠPerson":18404,"ĠTon":18405,"Ġbore":18406,"ĠClar":18407,"Ġmot":18408,"Ġbathrooms":18409,"ĠTypically":18410,"Ġdisconnect":18411,"Ġtightly":18412,"ĠHarvest":18413,"ĠHed":18414,"ĠGermans":18415,"atar":18416,"Ġkeynote":18417,"Ġimproper":18418,"fil":18419,"Ġintens":18420,"iev":18421,"Ġmedi":18422,"Ġtenant":18423,"Ġfootsteps":18424,"uli":18425,"Ġlegalization":18426,"106":18427,"ĠLexington":18428,"folio":18429,"Ġ½":18430,"ĠRita":18431,"Ġbattered":18432,"inka":18433,"ĠJavaScript":18434,"ĠMusical":18435,"ĠTalent":18436,"Ġlounge":18437,"Ġintimidation":18438,"ikh":18439,"ĠFam":18440,"Ġtherapeutic":18441,"Ġbalancing":18442,"Ġrocky":18443,"liners":18444,"ĠPredators":18445,"Ġregistering":18446,"Ġdiligence":18447,"ĠRover":18448,"ĠDot":18449,"Ġterminated":18450,"ĠEdu":18451,"Ġcharming":18452,"ĠPLAY":18453,"ĠFact":18454,"ĠCi":18455,").\"":18456,"ĠWrestle":18457,"hun":18458,"Ġopenings":18459,"Ġfou":18460,"Ġ126":18461,"spe":18462,"ĠAW":18463,"Ġbud":18464,"ĠTemper":18465,"ĠOrthodox":18466,"Ġprogressed":18467,"tre":18468,"Ġtasting":18469,"Ġscrutin":18470,"ĠLima":18471,"Ġlayout":18472,"Ġlitter":18473,"ijk":18474,"ĠParkinson":18475,"ĠAnfield":18476,"Ġdevelopmental":18477,"Ġheaven":18478,"ĠWoodward":18479,"index":18480,"Ġpistol":18481,"Ġreson":18482,"ĠWS":18483,"Ġemb":18484,"ĠLap":18485,"ĠPle":18486,"lington":18487,"ĠSit":18488,"Ġabruptly":18489,"ĠSenegal":18490,"ĠYates":18491,"aceutical":18492,"ĠJak":18493,"ĠHastings":18494,"iste":18495,"ĠDB":18496,"ĠAgent":18497,"Ġpreservation":18498,"ĠLank":18499,"ĠSuffolk":18500,"Ġboo":18501,"essed":18502,"Ġempowering":18503,"enne":18504,"Ġrecycled":18505,"Ġstrateg":18506,"Ġbrake":18507,"135":18508,"ĠStef":18509,"ĠFlake":18510,"ĠGregg":18511,"ĠRent":18512,"Ġinstallment":18513,"FW":18514,"ĠCran":18515,"obo":18516,"ml":18517,"ĠJade":18518,"Ġaccuses":18519,"ĠNvidia":18520,"Ġburg":18521,"High":18522,"Ġbothered":18523,"ĠBenn":18524,"Ġinterrupted":18525,"Ġtrek":18526,"Ġserv":18527,"Ġpatron":18528,"Ġdictator":18529,"owa":18530,"jad":18531,"ĠTulsa":18532,"Ġboil":18533,"Ġdisplaying":18534,"Ġcinem":18535,"awaited":18536,"¸":18537,"Ġreacts":18538,"ĠDee":18539,"ĠGron":18540,"igation":18541,"Ġservic":18542,"capt":18543,"Ġinsane":18544,"ĠVeteran":18545,"umen":18546,"End":18547,"ĠCream":18548,"Ġextremism":18549,"ĠMalone":18550,"Col":18551,"Ġsafeguard":18552,"Ġtomatoes":18553,"die":18554,"Ġchamp":18555,"zero":18556,"ĠPRES":18557,"Ġchoir":18558,"Ġpediatric":18559,"Ġprivileged":18560,"Ġdownstream":18561,"Business":18562,"ĠFighting":18563,"atable":18564,"Ġsums":18565,"Ġinsult":18566,"arten":18567,"ĠWikiLeaks":18568,"Ġpads":18569,"Ġretali":18570,"ĠHunts":18571,"Ġindie":18572,"ĠShields":18573,"ĠMortgage":18574,"oses":18575,"ampton":18576,"ĠVideos":18577,"ĠPER":18578,"itionally":18579,"ĠKimmel":18580,"sum":18581,"trade":18582,"acity":18583,"marked":18584,"ĠAngus":18585,"Ġtemper":18586,"Ġseizure":18587,"Ġfictional":18588,"utton":18589,"eva":18590,"Rs":18591,"Ġintra":18592,"ĠRequest":18593,"ppe":18594,"ĠeBay":18595,"ĠUSS":18596,"Ġ1500":18597,"Ġpossessing":18598,"Ġbacon":18599,"ĠSexual":18600,"ĠBuff":18601,"Ġslaughter":18602,"Ġjur":18603,"zhou":18604,"suit":18605,"ĠCha":18606,"ĠBuk":18607,"crime":18608,"ĠEasy":18609,"ĠChain":18610,"aq":18611,"ĠPall":18612,"flation":18613,"225":18614,"oup":18615,"109":18616,"ĠMcKenzie":18617,"Ġclearer":18618,"ĠDogs":18619,"oration":18620,"Ġsubs":18621,"Follow":18622,"ĠShirley":18623,"Ġadjusting":18624,"ĠEFF":18625,"Ġflipped":18626,"Ġconform":18627,"ĠLaurent":18628,"Ġcircular":18629,"ĠNOR":18630,"Ġmort":18631,"Ġtexture":18632,"avour":18633,"Ġflex":18634,"ĠHedge":18635,"ðŁĺ":18636,"Ġtrophies":18637,"ĠINV":18638,"Ġboast":18639,"ĠTyr":18640,"ĠNichols":18641,"ĠSpa":18642,"Ġcheered":18643,"Ġprey":18644,"reach":18645,"Ġbreached":18646,"ĠRegions":18647,"ĠLyft":18648,"ĠTul":18649,"ĠKore":18650,"Ġendure":18651,"ĠCover":18652,"\").":18653,"ĠSavage":18654,"ère":18655,"reens":18656,"Ġnic":18657,"sector":18658,"Ġweaknesses":18659,"Ġreboot":18660,"Ġ210":18661,"Ġimagery":18662,"ĠFrem":18663,"Ġclue":18664,"ĠLars":18665,"Ġfaction":18666,"hetic":18667,"Ġallied":18668,"ĠMarvin":18669,"Ġmethodology":18670,"ĠTN":18671,"Ġutter":18672,"Ġ270":18673,"ĠVolvo":18674,"oline":18675,"ĠACLU":18676,"Ġindirect":18677,"Ġminer":18678,"ĠBale":18679,"ĠStrange":18680,"ĠFuller":18681,"Ġexpelled":18682,"ĠTropical":18683,"Ġremotely":18684,"ĠTIM":18685,"Ġinnocence":18686,"Ġconfined":18687,"Ġfares":18688,"Ġprevalent":18689,"Ġdesp":18690,"House":18691,"azar":18692,"Ġgestures":18693,"ĠCES":18694,"ĠDM":18695,"eal":18696,"ĠÐ":18697,"Ġburnt":18698,"Ġframed":18699,"ĠDani":18700,"Ġhol":18701,"ĠCannes":18702,"ĠHayden":18703,"Ġwardrobe":18704,"ĠAssange":18705,"ĠSamp":18706,"bay":18707,"sky":18708,"ĠHence":18709,"ĠGrizzlies":18710,"rates":18711,"laws":18712,"ĠMandela":18713,"ĠHoover":18714,"rics":18715,"charged":18716,"Ġexclude":18717,"Ġpassive":18718,"Ġcontinuation":18719,"Ġblunt":18720,"Ġvac":18721,"ĠEmerging":18722,"rench":18723,"tv":18724,"ĠHollow":18725,"ĠOC":18726,"Ġadvisors":18727,"Ġrendered":18728,"ĠBernardino":18729,"ĠSupporters":18730,"ronic":18731,"Ġchancellor":18732,"Ġ1963":18733,"Ġuranium":18734,"Ġak":18735,"ĠOptions":18736,"ermott":18737,"ĠBerger":18738,"ibia":18739,"Ġexplosions":18740,"Ġimpairment":18741,"Ġhail":18742,"Ġalley":18743,"Ġcruelty":18744,"ĠClarence":18745,"Ġvariations":18746,"Ġrealm":18747,"Ġrenovations":18748,"ĠNorwich":18749,"Ġbelongings":18750,"Ġmerchants":18751,"ĠMinisters":18752,"ĠDodd":18753,"Ġviewer":18754,"Ġneutrality":18755,"quer":18756,"ĠPrinceton":18757,"dead":18758,"arest":18759,"GET":18760,"ĠCanadiens":18761,"ĠIgn":18762,"clear":18763,"Mal":18764,"ĠBridges":18765,"ĠHayward":18766,"Ġremarked":18767,"ingle":18768,"Ġsob":18769,"Ġdepart":18770,"beans":18771,"Ġpreserved":18772,"ĠFairfax":18773,"Ġforgot":18774,"ĠBeh":18775,"Rob":18776,"Ġcooperative":18777,"ullah":18778,"Ġmates":18779,"Ġrang":18780,"Ġthigh":18781,"Ġabducted":18782,"Ġchaired":18783,"ĠHearts":18784,"Ġidentifies":18785,"ĠBuckingham":18786,"ijn":18787,"ĠJab":18788,"Ġclashed":18789,"feed":18790,"sites":18791,"ĠCareer":18792,"exp":18793,"ĠBuccaneers":18794,"scape":18795,"Ġupdating":18796,"Ġintentional":18797,"ĠGuam":18798,"ĠBreakfast":18799,"ĠHag":18800,"Media":18801,"Ġtapping":18802,"Ġpics":18803,"Ġeaten":18804,"Ġpremise":18805,"Kim":18806,"ĠStorage":18807,"Ġextensively":18808,"Ġoutrageous":18809,"ĠSadly":18810,"Global":18811,"¢":18812,"leaning":18813,"CM":18814,"Ġeasiest":18815,"ument":18816,"Ġ122":18817,"Ġdaunting":18818,"ISE":18819,"Ġsunset":18820,"Ġreset":18821,"Ġbent":18822,"Trust":18823,"ĠCaleb":18824,"ĠRut":18825,"ĠBast":18826,"ETS":18827,"iencies":18828,"Ġpu":18829,"ature":18830,"Ġrealities":18831,"omi":18832,"Ġsoda":18833,"Ġunveil":18834,"ĠGoldberg":18835,"opes":18836,"Ġuprising":18837,"ĠMR":18838,"Ġendorse":18839,"Ġsail":18840,"Ġconverting":18841,"Ġglamorous":18842,"ĠHollande":18843,"108":18844,"isky":18845,"Ġcushion":18846,"240":18847,"Ġadventures":18848,"Ġantitrust":18849,"ĠStockholm":18850,"pace":18851,"ĠVald":18852,"ĠTransfer":18853,"ERT":18854,"ĠMcInt":18855,"Ġsurging":18856,"ogn":18857,"Ġlauded":18858,"ĠZam":18859,"ĠRough":18860,"TOR":18861,"Ġwed":18862,"Ġorigins":18863,"ĠEld":18864,"oso":18865,"Ġsupplying":18866,"ĠPetty":18867,"ĠTwe":18868,"ĠDenise":18869,"ĠBec":18870,"Ġbehave":18871,"Ġ121":18872,"estone":18873,"ĠBoulder":18874,"ĠBlackhawks":18875,"ĠWyatt":18876,"Ġfiguring":18877,"ĠDeborah":18878,"agi":18879,"significant":18880,"Ġasthma":18881,"Ġmessy":18882,"mpire":18883,"Ġax":18884,"Ġaspiring":18885,"ĠNH":18886,"ĠGina":18887,"heavy":18888,"ĠVick":18889,"ÃŃs":18890,"something":18891,"Ġbodily":18892,"Ġunauthorized":18893,"ĠActually":18894,"ĠOH":18895,"Ġmicrophone":18896,"allah":18897,"Ġrampant":18898,"Ġrelocated":18899,"Ġwidening":18900,"ĠCait":18901,"nel":18902,"ĠBlackBerry":18903,"Ġprofessionally":18904,"ĠInterestingly":18905,"Ġbarbecue":18906,"Ġresisting":18907,"ĠNunes":18908,"disc":18909,"Ġgroundbreaking":18910,"orable":18911,"ĠRegulation":18912,"Ġborrowed":18913,"Ġleaking":18914,"Ġlengths":18915,"Ġunveiling":18916,"houses":18917,"Ġ155":18918,"ĠBillboard":18919,"icion":18920,"Times":18921,"ĠZoe":18922,"ĠAbby":18923,"bus":18924,"ĠMinutes":18925,"ributed":18926,"Ġparap":18927,"Ġfertil":18928,"ABC":18929,"ĠIsle":18930,"Ġtherapist":18931,"Ġgubernatorial":18932,"ĠAust":18933,"ĠLoan":18934,"Bo":18935,"ĠNRL":18936,"rag":18937,"Clear":18938,"Ġrevision":18939,"Ġflesh":18940,"BD":18941,"iji":18942,"Ġproductions":18943,"Ġcoconut":18944,"ĠMcCorm":18945,"ĠDash":18946,"Ġgeography":18947,"hearted":18948,"Ġarson":18949,"Ġgoaltender":18950,"Ġbelly":18951,"Ġqualifications":18952,"ĠActiv":18953,"Ġhooked":18954,"ĠHungarian":18955,"Ġprotocols":18956,"inking":18957,"Ġfronts":18958,"ĠKuala":18959,"ĠToys":18960,"ĠFitness":18961,"Ġwarfare":18962,"Ġoutp":18963,"ĠQuestions":18964,"Ġwel":18965,"ĠShan":18966,"ĠMorton":18967,"ĠRomero":18968,"Ġglance":18969,"ĠTay":18970,"Ġsneakers":18971,"ĠSymphony":18972,"Ġinspect":18973,"enna":18974,"Nobody":18975,"Ġscrapped":18976,"ĠDeVos":18977,"ĠDominican":18978,"Ġplanets":18979,"anova":18980,"Ġnotify":18981,"Ġincurred":18982,"Ġunders":18983,"Ġdetainees":18984,"ĠMarriott":18985,"electric":18986,"ĠKes":18987,"union":18988,"ĠWatt":18989,"ATING":18990,"Ġslipping":18991,"Ġraft":18992,"Ġresisted":18993,"Ġcred":18994,"tern":18995,"Ġflurry":18996,"Line":18997,"Ġconsulted":18998,"Ġanalyzing":18999,"107":19000,"ĠWide":19001,"¶":19002,"human":19003,"ĠFEMA":19004,"Ġsmash":19005,"Ġcorps":19006,"Ġbarric":19007,"Ġcollar":19008,"ĠTB":19009,"without":19010,"ĠCanucks":19011,"Ġneedle":19012,"ĠSidney":19013,"ĠLauderdale":19014,"Ġglove":19015,"ilee":19016,"pic":19017,"Ġbenef":19018,"ĠHydro":19019,"ĠDisc":19020,"ĠArg":19021,"Ġtermin":19022,"Ġsympath":19023,"Ġpest":19024,"ĠCoff":19025,"Ġadvancement":19026,"social":19027,"pol":19028,"ĠEmails":19029,"Ġstacked":19030,"ibly":19031,"ĠAlbion":19032,"Ġfist":19033,"hero":19034,"ĠMarian":19035,"asia":19036,"Ġtownship":19037,"Ġslick":19038,"Ġmodeling":19039,"achers":19040,"ĠArgent":19041,"ĠSUN":19042,"arde":19043,"Ġpinned":19044,"Ġhitters":19045,"Ġdare":19046,"ictions":19047,"arily":19048,"Ġsting":19049,"Ġprimaries":19050,"appointed":19051,"Ġformats":19052,"Ġglitter":19053,"Ġpatches":19054,"Ġstrategically":19055,"Ġaka":19056,"Ġyielded":19057,"BY":19058,"Ġjeopard":19059,"ĠVand":19060,"Ġcrowned":19061,"Ġoccupants":19062,"Ġtanker":19063,"ĠVisa":19064,"Great":19065,"Ġseasoned":19066,"ĠAviv":19067,"Ġfiery":19068,"Ġderivatives":19069,"Ġdiverted":19070,"Ġacqu":19071,"Ġsandwiches":19072,"ĠLorenzo":19073,"Ġpardon":19074,"ĠBarber":19075,"ĠAgricultural":19076,"ĠPhilly":19077,"Ġregrets":19078,"ĠMillions":19079,"ĠFrazier":19080,"Ġtreasury":19081,"ĠKenn":19082,"Ġdestined":19083,"olved":19084,"Back":19085,"leader":19086,"lyss":19087,"ĠReyes":19088,"001":19089,"bags":19090,"ĠStandards":19091,"ĠExcellence":19092,"ĠMaid":19093,"ĠAnthem":19094,"FIELD":19095,"Ġrevived":19096,"ĠQuad":19097,"Ġdistinguished":19098,"Ġweighted":19099,"Ġritual":19100,"Ġinvites":19101,"wana":19102,"iture":19103,"ĠCI":19104,"ĠMAY":19105,"Ġunfairly":19106,"ĠKP":19107,"ĠMidlands":19108,"Ġmint":19109,"uers":19110,"Ġcatalog":19111,"arant":19112,"Ġlosers":19113,"Ġscheduling":19114,"esar":19115,"Ġtransferring":19116,"Ġbankrupt":19117,"Ġmethamphetamine":19118,"ĠEsk":19119,"ĠTreatment":19120,"ĠResponse":19121,"Ġhomework":19122,"ĠBald":19123,"Ġembarrassment":19124,"Ġpoorest":19125,"ĠPlatinum":19126,"ĠFac":19127,"Ġunleashed":19128,"Ġbrighter":19129,"002":19130,"Ġdisl":19131,"ĠLowry":19132,"ived":19133,"ĠDemon":19134,"ĠNonetheless":19135,"arro":19136,"ĠCONT":19137,"ifted":19138,"ĠFreder":19139,"isson":19140,"Ġrout":19141,"ARA":19142,"Ġswinging":19143,"Oct":19144,"Ġliable":19145,"Ġleaning":19146,"Ġlungs":19147,"380":19148,"ĠProcess":19149,"ĠCov":19150,"terrorism":19151,"Ġresistant":19152,"Ġpumped":19153,"Ġtripled":19154,"Semitism":19155,"ĠMia":19156,"Ġpenetration":19157,"ĠLutheran":19158,"BU":19159,"odes":19160,"Ġspanning":19161,"utch":19162,"Trans":19163,"ĠVolunteers":19164,"Ġpathway":19165,"Ġinfectious":19166,"Ġdrastic":19167,"ĠEngineers":19168,"Ġprincess":19169,"acts":19170,"usting":19171,"utive":19172,"achel":19173,"DO":19174,"Ġpave":19175,"ĠHerrera":19176,"Ġnearing":19177,"help":19178,"Ġembarked":19179,"Ġmodes":19180,"ĠDriving":19181,"Ġopting":19182,"Best":19183,"Ġbehavioral":19184,"Ġcables":19185,"App":19186,"otion":19187,"ĠExt":19188,"ĠSinclair":19189,"ĠInsp":19190,"Ġsinking":19191,"Next":19192,"ĠLumpur":19193,"ĠShadow":19194,"Donald":19195,"itals":19196,"Ġmentions":19197,"floor":19198,"Ġconsiderations":19199,"ĠSquad":19200,"ĠPlate":19201,"dos":19202,"Friday":19203,"Hopefully":19204,"arre":19205,"Ġalum":19206,"\":\"/":19207,"Ġfet":19208,"anza":19209,"Ġdign":19210,"ĠNguyen":19211,"ĠRutgers":19212,"ĠSew":19213,"Ġfilters":19214,"ofi":19215,"Ġunavailable":19216,"ranking":19217,"Ġrefining":19218,"ĠUNC":19219,"Ġmax":19220,"yll":19221,"Ġhandsome":19222,"Ġutterly":19223,"See":19224,"ĠStores":19225,"Ke":19226,"ĠAdvoc":19227,"ordon":19228,"umbles":19229,"Ġbugs":19230,"olar":19231,"ĠCork":19232,"Ġtoken":19233,"Ġauthorization":19234,"Ġconscience":19235,"Ġrepl":19236,"edi":19237,"owitz":19238,"iven":19239,"Ġlieu":19240,"Ġlifts":19241,"Lean":19242,"Ġmagnificent":19243,"ĠFilms":19244,"onents":19245,"Ġ***":19246,"Green":19247,"ĠAdvocate":19248,"ĠArrow":19249,"Ġblows":19250,"Ġexploited":19251,"fly":19252,"ĠAmar":19253,"ĠNOTICE":19254,"Ġsincere":19255,"found":19256,"ĠRud":19257,"Ġcy":19258,"ĠHeidi":19259,"Ġempowered":19260,"Ġweakest":19261,"ĠKru":19262,"Credit":19263,"aunted":19264,"Ġexotic":19265,"aning":19266,"Ġaw":19267,"ĠMulti":19268,"Ġanimation":19269,"850":19270,"ĠCounter":19271,"ĠNit":19272,"alli":19273,"Ġcapitalize":19274,"Ġexecuting":19275,"Ġdescent":19276,"ovi":19277,"ĠKimberly":19278,"headed":19279,"Ġmentioning":19280,")-":19281,"ĠSpecifically":19282,"ayette":19283,"ihad":19284,"ĠIss":19285,"Ġdisagreed":19286,"ĠKum":19287,"Ġurges":19288,"Ġpermitting":19289,"Ġpy":19290,"isp":19291,"Ġhygiene":19292,"Ġmourning":19293,"Ġcyclists":19294,"cats":19295,"FER":19296,"cycl":19297,"Ġnewcomers":19298,"Ġplead":19299,"Ġmend":19300,"secret":19301,"fan":19302,"Ġtranslates":19303,"unit":19304,"ĠTank":19305,"drive":19306,"ĠSite":19307,"Ġacceleration":19308,"ĠEnrique":19309,"ĠElaine":19310,"Ġstaring":19311,"Ġbackwards":19312,"Ġot":19313,"Ġvot":19314,"ĠHK":19315,"Ġfian":19316,"ĠLockheed":19317,"Ġmanifest":19318,"ĠZurich":19319,"pad":19320,"ĠRav":19321,"flow":19322,"Ġmoms":19323,"ĠSolid":19324,"ĠReady":19325,"aughlin":19326,"Ġreminding":19327,"ĠCOR":19328,"Ġoptimal":19329,"ĠCrisis":19330,"Ġcholesterol":19331,"ĠGerard":19332,"Ġfest":19333,"Ġsanction":19334,"Ġdragging":19335,"inent":19336,"ĠBravo":19337,"Ġamend":19338,"aval":19339,"Ġpoem":19340,"Ġinvasive":19341,"Ġlandsc":19342,"leigh":19343,"Ġheadache":19344,"ĠMuse":19345,"ĠTurning":19346,"girl":19347,"cess":19348,"Ġfalsely":19349,"Ġplaintiff":19350,"Ġheavier":19351,"Ġrumored":19352,"Ġeleven":19353,"ĠConsumers":19354,"ĠOriginally":19355,"ĠStatement":19356,"bors":19357,"Ġrevoked":19358,"ĠOmaha":19359,"Fox":19360,"ĠKle":19361,"Ġvault":19362,"Ġoutdated":19363,"umes":19364,"ĠArk":19365,"Ġapologised":19366,"Ġrockets":19367,"ĠMarines":19368,"Ġcaptures":19369,"ĠMW":19370,"ĠWalters":19371,"ĠFactor":19372,"Ġensuing":19373,"ĠSession":19374,"oons":19375,"Ġ132":19376,"gt":19377,"ĠPoints":19378,"Ġexhaust":19379,"ĠOsaka":19380,"heed":19381,"Ġhandic":19382,"amber":19383,"inging":19384,"Ġll":19385,"Ġescorted":19386,"Ġfloated":19387,"Ġmerge":19388,"Ġcompliment":19389,"ĠVC":19390,"Ġinsulin":19391,"ĠDebt":19392,"ça":19393,"Ġpens":19394,"Ġassertion":19395,"Ġredevelopment":19396,"moderate":19397,"Ġleftist":19398,"ĠBA":19399,"Ġherd":19400,"Ġinsecurity":19401,"liter":19402,"Ġcommence":19403,"ĠCaucus":19404,"Ġnovels":19405,"ĠChevron":19406,"Ġerosion":19407,"ĠNicholson":19408,"ĠRoof":19409,"ĠVolunteer":19410,"Ġcompelled":19411,"Ġcongratulated":19412,"ĠPanel":19413,"Ġov":19414,"idelity":19415,"Ġspect":19416,"Ġbee":19417,"ĠAssistance":19418,"Ġterrified":19419,"iew":19420,"Ġweekday":19421,"ĠHiggins":19422,"special":19423,"ubs":19424,"anton":19425,"Ġbribes":19426,"Ġneat":19427,"ĠCliff":19428,"Ġdisqualified":19429,"ĠND":19430,"Ġvers":19431,"andra":19432,"Ġgraft":19433,"value":19434,"Ġportray":19435,"Ġdaytime":19436,"ksh":19437,"Ġconsist":19438,"Ġhonesty":19439,"ĠTimber":19440,"ĠNich":19441,"Ġinvented":19442,"ĠBuch":19443,"Ġskull":19444,"Ġtags":19445,"Ġ124":19446,"ighth":19447,"Ġrelaxing":19448,"Online":19449,"Ġsanctioned":19450,"Sport":19451,"ĠCove":19452,"Ġcomics":19453,"MW":19454,"AMA":19455,"mother":19456,"Home":19457,"ĠCustomer":19458,"Ġstrides":19459,"ĠWins":19460,"Ġrollout":19461,"ĠWeaver":19462,"Ġshuttle":19463,"Ġsteak":19464,"Ġglorious":19465,"ĠToll":19466,"Ġtrustee":19467,"Ġinstallations":19468,"ĠOpportunity":19469,"Ġoper":19470,"horse":19471,"Ġaided":19472,"irus":19473,"Ġsleek":19474,"Ġyelled":19475,"ĠSocialist":19476,"Ġapplaud":19477,"ĠWah":19478,"Ġdevote":19479,"Ġdh":19480,"Ġarchitectural":19481,"ĠMAC":19482,"centric":19483,"ĠSense":19484,"illas":19485,"ĠArchbishop":19486,"glass":19487,"Ġallowance":19488,"Ġbundle":19489,"andon":19490,"eight":19491,"ĠKare":19492,"haus":19493,"ĠAndreas":19494,"Ġdoll":19495,"RAM":19496,"Ġvolunteering":19497,"ĠRaleigh":19498,"Ġbees":19499,"Ġnickel":19500,"Ġgenerosity":19501,"Ġhomeowner":19502,"ĠLieutenant":19503,"Ġlandfall":19504,"ĠRenew":19505,"ĠGiving":19506,"ĠContribut":19507,"aret":19508,"ulf":19509,"Ġreinforce":19510,"ĠSalv":19511,"ĠVenice":19512,"Ġfreedoms":19513,"ĠTools":19514,"Ġ1962":19515,"ĠWarm":19516,"majority":19517,"Ġpleas":19518,"oding":19519,"plant":19520,"Ġtow":19521,"ĠBlanc":19522,"ĠPipeline":19523,"ĠMoor":19524,"Ġrefrain":19525,"ĠExplore":19526,"language":19527,"cers":19528,"ĠWT":19529,"sent":19530,"ĠNun":19531,"Ġplastics":19532,"acas":19533,"Ġdisruptions":19534,"Ġdiscomfort":19535,"enko":19536,"Ġimprisoned":19537,"Copyright":19538,"Ġmyriad":19539,"Ġparenting":19540,"Ġspree":19541,"NBC":19542,"Ġonion":19543,"ĠIsraelis":19544,"ĠRA":19545,"Ġrelocate":19546,"113":19547,"ĠHir":19548,"ĠDre":19549,"ĠDry":19550,"ĠONE":19551,"ĠAdministrator":19552,"Ġprints":19553,"ĠGret":19554,"Ġundergraduate":19555,"ĠLif":19556,"avers":19557,"ĠCarney":19558,"Ġapex":19559,"Ġlenses":19560,"Ġliberals":19561,"gb":19562,"ĠWhereas":19563,"Ġcountryside":19564,"amine":19565,"ĠTerminal":19566,"Ġintr":19567,"ĠTrey":19568,"ALS":19569,"Ġcontinental":19570,"Ġselfies":19571,"FILE":19572,"ĠUnity":19573,"Ġauthoritarian":19574,"Ġoriginated":19575,"ĠExcept":19576,"yna":19577,"Ġmonet":19578,"Ġundermining":19579,"ĠGS":19580,"pi":19581,"iq":19582,"Ġslides":19583,"ĠSummary":19584,"Ġpains":19585,"cluding":19586,"Ġequation":19587,"locked":19588,"Ġfraternity":19589,"Ġwithstand":19590,"Ġdevastation":19591,"Ġdemo":19592,"late":19593,"Ġpunches":19594,"Ġgeared":19595,"nen":19596,"ĠBowie":19597,"attle":19598,"Ġpolitic":19599,"ĠGle":19600,"mented":19601,"ĠCoordinator":19602,"Ġupwards":19603,"ĠMega":19604,"angled":19605,"Ġengineered":19606,"Ġluggage":19607,"ĠWen":19608,"ĠSergeant":19609,"Ġkindergarten":19610,"ĠPortsmouth":19611,"uddin":19612,"ket":19613,"oba":19614,"Ġoscill":19615,"esse":19616,"ĠOlson":19617,"ĠBorough":19618,"Ġsupplements":19619,"ĠEvening":19620,"ANE":19621,"Ġlava":19622,"Ġgearing":19623,"setting":19624,"urgical":19625,"asty":19626,"ĠDaytona":19627,"Ġbrewery":19628,"Ġpledges":19629,"rounder":19630,"ulous":19631,"ĠHancock":19632,"rex":19633,"Ġram":19634,"Ġproceeding":19635,"ĠMurdoch":19636,"Ġdowngrade":19637,"Ġstatues":19638,"Ġdebated":19639,"ĠSleep":19640,"Ġ144":19641,"ĠRuby":19642,"ĠFi":19643,"123":19644,"ĠArabic":19645,"Ġlasts":19646,"ĠIvy":19647,"ĠWid":19648,"rown":19649,"stick":19650,"?'\"":19651,"ĠSTEM":19652,"Ġsensible":19653,"htar":19654,"Ġharbor":19655,"Ġcra":19656,"ĠAlbum":19657,"ĠCarnival":19658,"Ġimplies":19659,"agement":19660,"ĠInitially":19661,"Ġchooses":19662,"Jeff":19663,"ĠHig":19664,"Ġtam":19665,"Ġlump":19666,"ucks":19667,"Ġrepatri":19668,"ĠMercy":19669,"zza":19670,"Ġ365":19671,"ĠRicardo":19672,"ogram":19673,"Ġundergone":19674,"system":19675,"Ġtel":19676,"ĠKee":19677,"ully":19678,"istas":19679,"Ġgrains":19680,"ĠTomorrow":19681,"ĠRC":19682,"ĠTurk":19683,"Ġfreshmen":19684,"ĠAway":19685,"ĠSach":19686,"ĠUltimate":19687,"Ġoffensively":19688,"ismo":19689,"Ġteaser":19690,"ĠJud":19691,"Ġlegitimacy":19692,"opt":19693,"ĠCobb":19694,"Ġrejecting":19695,"ĠSolo":19696,"ĠArcher":19697,"Ġsoutheastern":19698,"ĠPlain":19699,"ĠLoss":19700,"Ġminerals":19701,"ĠMari":19702,"Ġscrambling":19703,"ĠPeak":19704,"Ġhavoc":19705,"rings":19706,"Ġunofficial":19707,"ĠHaj":19708,"director":19709,"ĠCanal":19710,"ĠNSA":19711,"ĠEaton":19712,"ĠPART":19713,"ĠCommissioners":19714,"Ġwellbeing":19715,"resa":19716,"Ġunderstandable":19717,"dates":19718,"ĠSorry":19719,"Ġastonishing":19720,"Ġrevise":19721,"ĠEc":19722,"ĠLack":19723,"endi":19724,"endale":19725,"also":19726,"Ġcolder":19727,"Ġheel":19728,"Ġcellular":19729,"Conn":19730,"ĠThur":19731,"Ġmassage":19732,"olla":19733,"clus":19734,"Ġtoilets":19735,"ĠCelebr":19736,"Ġtackled":19737,"Ġchorus":19738,"ETA":19739,"anca":19740,"ĠOLED":19741,"Ġpunk":19742,"ĠBrain":19743,"ĠNuggets":19744,"Ġseamless":19745,"make":19746,"atted":19747,"ĠRog":19748,"ĠPatch":19749,"Ġruined":19750,"Ins":19751,"Ġconsolidate":19752,"Ġgospel":19753,"ĠCaption":19754,"Ġoverweight":19755,"Ġscreened":19756,"ĠKraft":19757,"ĠBain":19758,"breaker":19759,"ĠFeinstein":19760,"ĠDoc":19761,"Ġdeepest":19762,"ĠOL":19763,"Ġtunes":19764,"Ġrightly":19765,"ĠLanc":19766,"ĠBrotherhood":19767,"Ġpoultry":19768,"ĠPure":19769,"Ġstimulate":19770,"Ġdiscourse":19771,"ĠStark":19772,"Ġmuseums":19773,"ention":19774,"Ġtaxation":19775,"ĠAkron":19776,"ayer":19777,"ĠKirby":19778,"farm":19779,"oser":19780,"Ġcommend":19781,"Ġunarmed":19782,"ensions":19783,"Ġsuperst":19784,"Ġoceans":19785,"Ġmisuse":19786,"LO":19787,"ĠByrne":19788,"ĠMaritime":19789,"Ġdense":19790,"Ġexcuses":19791,"Ġsuppose":19792,"ĠMarks":19793,"Ġrainy":19794,"Ġreplicate":19795,"Ġboutique":19796,"ĠRenaissance":19797,"jas":19798,"icted":19799,"Ġreferenced":19800,"ĠTir":19801,"ĠHatch":19802,"ĠCry":19803,"ĠPayPal":19804,"Ġfulfil":19805,"ĠHawaiian":19806,"come":19807,"ĠThirty":19808,"Ġ260":19809,"ĠYak":19810,"Ġangles":19811,"Ġlandlord":19812,"Ġlavish":19813,"Women":19814,"ĠNT":19815,"Ġreinforced":19816,"Ġprevail":19817,"ĠCommunities":19818,"Ġfootwear":19819,"Ġassurances":19820,"Ġlb":19821,"Ġairing":19822,"Ġresorts":19823,"ĠFiji":19824,"ĠShay":19825,"Ġprevailing":19826,"many":19827,"Ġimpe":19828,"ĠDul":19829,"Ġsymbols":19830,"zb":19831,"ĠCere":19832,"Ġapplauded":19833,"Ġsoundtrack":19834,"Ġdrunken":19835,"ĠEuropeans":19836,"Ġherds":19837,"moving":19838,"WR":19839,"ĠHindi":19840,"Ġwaking":19841,"Jo":19842,"Andrew":19843,"rosse":19844,"ĠLegislative":19845,"Ġdisgrace":19846,"Nothing":19847,"ĠBulgaria":19848,"Ġhumidity":19849,"Ġtranslation":19850,"Ġmeasurements":19851,"Ġvying":19852,"ĠBrid":19853,"Max":19854,"Ġdir":19855,"unci":19856,"Ġdefines":19857,"Ġperfection":19858,"ancers":19859,"Matt":19860,"ĠShinzo":19861,"ĠPresidents":19862,"Ġginger":19863,"onna":19864,"existing":19865,"rika":19866,"enced":19867,"ĠBray":19868,"Ġgall":19869,"Ġdisrespect":19870,"ĠCumber":19871,"Ġcontestant":19872,"ucky":19873,"anticipated":19874,"abled":19875,"LLOW":19876,"Bel":19877,"ĠKear":19878,"Ġstoryline":19879,"Ġrigs":19880,"ĠScots":19881,"ĠChap":19882,"ĠThankfully":19883,"Ġcommunist":19884,"ĠAdviser":19885,"Ġregist":19886,"Ġannoying":19887,"ĠDVD":19888,"Ġethic":19889,"ĠFilipino":19890,"ĠAdidas":19891,"Ġbilling":19892,"Ġalleviate":19893,"Ġsmoked":19894,"Ġhazard":19895,"EV":19896,"Ag":19897,"baum":19898,"Ġdoses":19899,"Ġoutcry":19900,"Ġinclined":19901,"Ġpsychologist":19902,"itzer":19903,"January":19904,"Ġmornings":19905,"aught":19906,"Ġsurreal":19907,"ĠCannon":19908,"avy":19909,"ĠCris":19910,"cf":19911,"Ġinterpreted":19912,"Ġpersecution":19913,"vation":19914,"Ġupfront":19915,"ĠWaste":19916,"Ġmills":19917,"Ġbombings":19918,"ĠHeaven":19919,"ĠFlat":19920,"Ġboxer":19921,"Ġavenues":19922,"Invest":19923,"ĠZika":19924,"Ġbackstage":19925,"idas":19926,"eston":19927,"ead":19928,"Ġbishops":19929,"Ġrender":19930,"Ġfootballer":19931,"Ġspilled":19932,"Only":19933,"Ġsaddened":19934,"ĠAbove":19935,"inator":19936,"tro":19937,"onen":19938,"ĠAMC":19939,"Ġstringent":19940,"Ġfooting":19941,"ĠGhost":19942,"Ġtexting":19943,"ĠCPI":19944,"ĠUW":19945,"Ġaccol":19946,"iries":19947,"ĠFlex":19948,"ĠCarolyn":19949,"Andre":19950,"Ġsiege":19951,"Muslim":19952,"Ġautomobile":19953,"reci":19954,"Ġdean":19955,"atre":19956,"Ġwax":19957,"Ġwo":19958,"ĠDuffy":19959,"Ġfiance":19960,"Ġfib":19961,"Ġeagle":19962,"ĠCatal":19963,"Ġinfants":19964,"Ġsubmitting":19965,"Ġdownhill":19966,"Ġstaffer":19967,"ĠLights":19968,"Ġeater":19969,"ĠCaliforn":19970,"Ġsupervisors":19971,"ĠPy":19972,"Ġcondemnation":19973,"Ġsci":19974,"Ġhated":19975,"Ġtil":19976,"ĠLavrov":19977,"Ġsab":19978,"Ġmotors":19979,"Ġlogging":19980,"ĠOwn":19981,"Ġpi":19982,"Ġrepeating":19983,"ĠDOJ":19984,"enary":19985,"ĠChow":19986,"fat":19987,"Ġbalcony":19988,"orie":19989,"NING":19990,"ĠUnified":19991,"Neil":19992,"Bill":19993,"ĠSims":19994,"uten":19995,"LV":19996,"ĠEMS":19997,"Ġsip":19998,"Ġreplaces":19999,"ichi":20000,"ĠFig":20001,"ĠCharity":20002,"Ġpeek":20003,"Ġrack":20004,"Ġcousins":20005,"Ġresolving":20006,"Ġthrone":20007,"ĠEngine":20008,"ĠChak":20009,"Ġlamented":20010,"Ġwipe":20011,"Ġnutrients":20012,"ĠChat":20013,"AMP":20014,"ĠOprah":20015,"uming":20016,"serving":20017,"Ġfir":20018,"Ġlandlords":20019,"neck":20020,"Ġupload":20021,"Ġunspecified":20022,"Ġicy":20023,"´":20024,"Ġze":20025,"Ġprohibits":20026,"ĠFI":20027,"Res":20028,"ĠEff":20029,"hell":20030,"umbo":20031,"Ġreceipts":20032,"Ġoperatives":20033,"stant":20034,"Ġwives":20035,"ĠCinema":20036,"Ġnegligence":20037,"Ġgases":20038,"ĠLau":20039,"Ġbrew":20040,"August":20041,"never":20042,"Ġpenned":20043,"Ġincomplete":20044,"ĠZh":20045,"esi":20046,"Ġranged":20047,"apolis":20048,"Ġwithdrawing":20049,"ĠLevi":20050,"ĠLevy":20051,"ĠDaly":20052,"Ġdelaying":20053,"ĠMSNBC":20054,"ĠCyrus":20055,"ĠNutrition":20056,"NN":20057,"Ġwinding":20058,"Ġglow":20059,"ĠMY":20060,"Ġgoodwill":20061,"ĠMON":20062,"Ġslots":20063,"ĠNina":20064,"ĠFIR":20065,"ĠLTE":20066,"ĠInnov":20067,"dev":20068,"ctic":20069,"Ġanalyses":20070,"ĠBangalore":20071,"Ġtales":20072,"Ġovercame":20073,"ĠThurs":20074,"Ġcherry":20075,"ĠNou":20076,"ĠFlowers":20077,"1000":20078,"updated":20079,"rieve":20080,"ĠBeautiful":20081,"iak":20082,"Ġplayback":20083,"Ġheadset":20084,"Ġashamed":20085,"Min":20086,"Ġadm":20087,"ĠLucky":20088,"ĠTucson":20089,"Ġentirety":20090,"ranging":20091,"ĠVance":20092,"kered":20093,"image":20094,"ĠGord":20095,"War":20096,"Ġsimilarities":20097,"dig":20098,"ĠJude":20099,"Ġlonely":20100,"hra":20101,"ĠStaples":20102,"ĠACA":20103,"Ġmeasurement":20104,"Ġcooper":20105,"ATER":20106,"ĠMeng":20107,"Ġbarring":20108,"190":20109,"ĠBatt":20110,"Ġreproductive":20111,"ĠRowe":20112,"Ġsubsid":20113,"Ġslogans":20114,"ugar":20115,"ĠKeller":20116,"ingham":20117,"fuel":20118,"Ġhid":20119,"afe":20120,"Ġindul":20121,"cash":20122,"Ġstressing":20123,"ĠMIT":20124,"Ġtrump":20125,"ancer":20126,"ĠPes":20127,"ĠMint":20128,"Ġcrossover":20129,"ĠWeiss":20130,"ĠElvis":20131,"ĠPermanent":20132,"ĠKhalid":20133,"Ġunjust":20134,"Ġexceptionally":20135,"Ġfut":20136,"Ġavid":20137,"ĠEthics":20138,"Ġutilized":20139,"Ġfeasibility":20140,"Ġcatering":20141,"Press":20142,"wayne":20143,"October":20144,"Ġfavors":20145,"Ġobsession":20146,"Ġmelt":20147,"Ġmug":20148,"ĠMK":20149,"Ġapples":20150,"Ġvine":20151,"cliffe":20152,"Ġgrat":20153,"Ġspells":20154,"ounced":20155,"Ġdecree":20156,"issy":20157,"Team":20158,"Ġdeploying":20159,"Feb":20160,"Ġmiserable":20161,"Ġwat":20162,"ĠBust":20163,"ĠNorris":20164,"ĠTimberwolves":20165,"Ġangered":20166,"ĠArn":20167,"oft":20168,"rome":20169,"Ġadvertisements":20170,"onal":20171,"Ġnun":20172,"Ġtorque":20173,"Ġslave":20174,"Ġnonsense":20175,"Ġcoy":20176,"Ġcites":20177,"Game":20178,"Ġarchitects":20179,"playing":20180,"Ġgener":20181,"Ġsocio":20182,"Ġmeditation":20183,"Ġforgive":20184,"Ġsmiled":20185,"%),":20186,"Ġpers":20187,"ĠSoph":20188,"Ġoccupy":20189,"atton":20190,"Ġwitnessing":20191,"Ġapologise":20192,"Ġpredecessors":20193,"ĠCassidy":20194,"Ġtallied":20195,"NER":20196,"Ġtract":20197,"ĠHolder":20198,"ĠPav":20199,"Ġjackets":20200,"Mel":20201,"raud":20202,"Ġexercising":20203,"ĠChung":20204,"ĠAmin":20205,"athi":20206,"ĠMem":20207,"Ġracked":20208,"Ġcarved":20209,"ĠMickey":20210,"ĠLafayette":20211,"Ġgrill":20212,"ĠINFORMATION":20213,"usc":20214,"ĠPromotion":20215,"yson":20216,"istry":20217,"Ġfulfilled":20218,"Ġrestraint":20219,"Ġpopping":20220,"ĠSlater":20221,"Ġmercy":20222,"aden":20223,"Ġsubmarine":20224,"ĠBowling":20225,"dogs":20226,"ĠSwe":20227,"Ġnoticeable":20228,"Ġbis":20229,"ĠPremiership":20230,"Ġspat":20231,"ĠTow":20232,"ĠWand":20233,"Ġmechanics":20234,"while":20235,"ĠBenson":20236,"Ġmolecules":20237,"Ġcrosses":20238,"Ġrecalling":20239,"ĠCertainly":20240,"HAM":20241,"Ġsever":20242,"ĠRudy":20243,"ĠDUI":20244,"OLD":20245,"ĠTobacco":20246,"Ġsubdued":20247,"Ġquota":20248,"TF":20249,"Ġflats":20250,"Ġemphasize":20251,"Ġbelts":20252,"ĠOpinion":20253,"Ġpiled":20254,"ĠSpark":20255,"ĠElias":20256,"Ġclassification":20257,"ĠHands":20258,"ĠCV":20259,"Ġtoast":20260,"Ġcandle":20261,"atching":20262,"short":20263,"ĠDup":20264,"Ġult":20265,"bats":20266,"Ġmarketers":20267,"ĠAvery":20268,"ĠColbert":20269,"ĠIk":20270,"ĠVac":20271,"ĠJackets":20272,"Ġmerits":20273,"eli":20274,"PORT":20275,"Ġelevator":20276,"irming":20277,"effective":20278,"Ġgroceries":20279,"Ġhi":20280,"ĠINTER":20281,"ĠSAP":20282,"ĠNYPD":20283,"ĠKY":20284,"Ġangel":20285,"Ġspectacle":20286,"ré":20287,"ĠRoche":20288,"Ġinsects":20289,"Ġcommenced":20290,"ĠFoley":20291,"Ġdarker":20292,"ĠUg":20293,"ĠMostly":20294,"Ġtermed":20295,"uci":20296,"ĠExec":20297,"ĠBrittany":20298,"Ġharmony":20299,"Ġadvocated":20300,"Ġparcel":20301,"ĠHots":20302,"Ġmonarch":20303,"ĠSiri":20304,"odge":20305,"ĠPag":20306,"Ġprogressing":20307,"grounds":20308,"Ġonstage":20309,"Ġwarmth":20310,"ĠWon":20311,"Ġviolates":20312,"ĠSaudis":20313,"Ġbumper":20314,"Ġpatrols":20315,"ĠBarron":20316,"Ġindoors":20317,"Ġtar":20318,"Each":20319,"Val":20320,"Ġapplicant":20321,"ĠCater":20322,"Ġclassics":20323,"ĠThreat":20324,"Ġwrapping":20325,"ĠIdlib":20326,"anking":20327,"Did":20328,"adia":20329,"ĠRig":20330,"ĠBram":20331,"ĠLaurie":20332,"ĠHair":20333,"ĠCannabis":20334,"Ġdaylight":20335,"ĠNorm":20336,"ĠRip":20337,"sin":20338,"unta":20339,"Pass":20340,"ĠAcad":20341,"ĠCummings":20342,"Ġtheirs":20343,"ĠDistribution":20344,"especially":20345,"Ġgrilled":20346,"Ġaffiliates":20347,"ĠVander":20348,"ĠCath":20349,"ĠProductions":20350,"ĠTrek":20351,"230":20352,"Ġcasinos":20353,"ĠCain":20354,"atu":20355,"idget":20356,"ĠWinds":20357,"Ġunanswered":20358,"Ġintercept":20359,"ĠMarty":20360,"Ġrefin":20361,"Ġlieutenant":20362,"cas":20363,"Chief":20364,"average":20365,"ilot":20366,"Ġscrimmage":20367,"ĠMud":20368,"speaking":20369,"ĠFranken":20370,"ĠTories":20371,"Ġabstract":20372,"awar":20373,"ĠTerms":20374,"dal":20375,"ĠFur":20376,"Ġhumour":20377,"rh":20378,"Ġsitu":20379,"aed":20380,"ĠFIN":20381,"Ġtranscripts":20382,"approved":20383,"ĠParsons":20384,"Ġpigs":20385,"Ġrepayment":20386,"ĠARM":20387,"ĠElliot":20388,"ĠLevine":20389,"Ġtagged":20390,"pun":20391,"ĠDwight":20392,"Ġconfiguration":20393,"sis":20394,"ĠAdult":20395,"Ġearthquakes":20396,"Ġcreature":20397,"ĠMRI":20398,"Ġmach":20399,"Ġprescriptions":20400,"cover":20401,"Ġministries":20402,"Ġinaccurate":20403,"ĠLabs":20404,"ĠMGM":20405,"Ġtomato":20406,"Ġeng":20407,"Ġopposes":20408,"owan":20409,"Ġmapping":20410,"Ġconsum":20411,"online":20412,"eters":20413,"code":20414,"Aug":20415,"Point":20416,"branded":20417,"pling":20418,"ĠCalder":20419,"Oper":20420,"ĠMiddles":20421,"Ġchampagne":20422,"ĠTues":20423,"Ġsampling":20424,"Ġenergetic":20425,"rano":20426,"ĠStyles":20427,"Ġneglected":20428,"ĠDamon":20429,"Ġendanger":20430,"Ġsouthwestern":20431,"ĠATM":20432,"ĠDuck":20433,"engers":20434,"Ġdan":20435,"yth":20436,"Ġbou":20437,"ĠDecl":20438,"Gold":20439,"Ġprojecting":20440,"Google":20441,"ĠHussein":20442,"Ġaccomplishment":20443,"itarian":20444,"Ġgossip":20445,"ĠRai":20446,"ril":20447,"ĠSke":20448,"Ġpsychiatric":20449,"ĠMacBook":20450,"ĠAdobe":20451,"ĠHodg":20452,"Ġaccompany":20453,"Ġadvertised":20454,"Ġreminiscent":20455,"Ġgeographical":20456,"Ġconvertible":20457,"IK":20458,"CTV":20459,"Ġcommunal":20460,"Ġchim":20461,"Ġselfish":20462,"Ġdrilled":20463,"Ġtortured":20464,"Ġblacks":20465,"noon":20466,"Ġmanifesto":20467,"ĠRichie":20468,"acco":20469,"Im":20470,"Ġdebit":20471,"ĠSNP":20472,"perfect":20473,"gard":20474,"ĠRatio":20475,"Ġstubborn":20476,"Ġaccumulation":20477,"Ġcongregation":20478,"Ġkissing":20479,"Ġkillers":20480,"ĠAbbey":20481,"von":20482,"ĠFuj":20483,"ĠIsabel":20484,"NB":20485,"ĠNish":20486,"ĠJulius":20487,"ĠZimmer":20488,"Ġuncover":20489,"dar":20490,"isle":20491,"ĠCompar":20492,"Ġcounselor":20493,"ĠSok":20494,"ĠCumm":20495,"ĠHip":20496,"Ġurgently":20497,"Ġrentals":20498,"Ġapproving":20499,"Ġirrigation":20500,"Ġprostate":20501,"ĠJudicial":20502,"ĠSubmit":20503,"ĠTanner":20504,"attack":20505,"emb":20506,"Ġreclaim":20507,"Ġec":20508,"Ġbrutality":20509,"Ġcommanding":20510,"Ġreasoning":20511,"Roy":20512,"ĠElect":20513,"ĠMobil":20514,"anding":20515,"Ġmirrors":20516,"Israel":20517,"Ġpavement":20518,"Ġoverdue":20519,"ĠMd":20520,"street":20521,"Ġthrill":20522,"pora":20523,"azon":20524,"Ġbrewing":20525,"enge":20526,"ĠDisaster":20527,"Ġbuilder":20528,"ods":20529,"utsch":20530,"Ġterminals":20531,"ĠBaird":20532,"enburg":20533,"Ġhast":20534,"Ġbrass":20535,"Ġparental":20536,"enture":20537,"ĠConduct":20538,"Ġexpands":20539,"luck":20540,"mur":20541,"ĠBj":20542,"Ġadministrations":20543,"ĠOlivier":20544,"oux":20545,"Ġnarrowed":20546,"winner":20547,"Ġmakeshift":20548,"ĠVAT":20549,"ĠJavier":20550,"-,":20551,"Ġsystematic":20552,"Ġenforcing":20553,"emin":20554,"ĠAudio":20555,"United":20556,"gener":20557,"ĠKara":20558,"ivas":20559,"ĠPretty":20560,"ĠLob":20561,"Ġpetitions":20562,"ĠMercer":20563,"ampa":20564,"product":20565,"Ġdistributing":20566,"Ġtunnels":20567,"Ġcondo":20568,"ĠRSS":20569,"ĠCarlo":20570,"Ġpumpkin":20571,"Ġsto":20572,"Ġassumes":20573,"oway":20574,"hiba":20575,"lection":20576,"Ġgam":20577,"ĠAires":20578,"Ġtransmitted":20579,"Ġtrousers":20580,"Ġcheers":20581,"ĠJensen":20582,"Ġemer":20583,"Ġsimpler":20584,"Ġcolored":20585,"ĠSustainable":20586,"Ġinstruct":20587,"Ġpoles":20588,"Ġsupervised":20589,"Ġinteg":20590,"ĠMoreno":20591,"boarding":20592,"igrant":20593,"ĠYoga":20594,"Ġenvironmentally":20595,"Ġsacrifices":20596,"Ġshores":20597,"Ġ127":20598,"Ġestranged":20599,"Ġintoxicated":20600,"Ġemergencies":20601,"ĠKosovo":20602,"yang":20603,"Ġfastball":20604,"Ġpackaged":20605,"LAN":20606,"Ġhurry":20607,"ĠManny":20608,"Ġporch":20609,"Ġcuriosity":20610,"ĠKend":20611,"thouse":20612,"ĠTou":20613,"mun":20614,"Ġwaving":20615,"Ġpasswords":20616,"ĠSwan":20617,"Ġprefers":20618,"ĠCorrections":20619,"aic":20620,"Ġejected":20621,"Ġdossier":20622,"ĠChal":20623,"Ġfacto":20624,"Ġspine":20625,"leck":20626,"Ġrestriction":20627,"Ġdisagreement":20628,"grown":20629,"ĠEdgar":20630,"Ġquantities":20631,"ĠRapid":20632,"Ġpals":20633,"Ġspared":20634,"Ġremarkably":20635,"ructure":20636,"Ġbackers":20637,"ĠGoals":20638,"cles":20639,"rolling":20640,"ĠBlasio":20641,"Ġorchestra":20642,"ologies":20643,"ĠRise":20644,"Power":20645,"Ġuptick":20646,"atha":20647,"ĠMob":20648,"Ġshotgun":20649,"downs":20650,"ĠBorg":20651,"Ġmorale":20652,"Call":20653,"wave":20654,"ĠDuc":20655,"Ġunwilling":20656,"oad":20657,"Ġbusinessmen":20658,"Ġrefriger":20659,"Ġgamers":20660,"Ġcele":20661,"Ġprecip":20662,"Ġrenegoti":20663,"OY":20664,"ĠPharm":20665,"Ġresponsive":20666,"Ġservant":20667,"eye":20668,"Ġraping":20669,"vas":20670,"Ġgroin":20671,"ĠMelvin":20672,"ĠKurds":20673,"Ġstricter":20674,"ĠMum":20675,"ients":20676,"Ġstandalone":20677,"Ġforums":20678,"Ġcommemorate":20679,"Far":20680,"ĠTelegram":20681,"Ġscreenings":20682,"ĠLeonardo":20683,"ighton":20684,"ĠDOWN":20685,"Ġmodule":20686,"Ġremedy":20687,"Ġ280":20688,"Su":20689,"ĠBecker":20690,"ĠGast":20691,"prem":20692,"ĠInto":20693,"oyle":20694,"114":20695,"Ġadhere":20696,"Report":20697,"ĠJaneiro":20698,"ĠKry":20699,"Pakistan":20700,"Ġrobotic":20701,"ande":20702,"Ġoverlooking":20703,"ĠTreaty":20704,"Ġrect":20705,"yne":20706,"Ġbattlefield":20707,"ĠGeoff":20708,"Ġearns":20709,"ĠMiner":20710,"Ġteased":20711,"Ġexemptions":20712,"Ġvacancy":20713,"oku":20714,"Ġvulnerabilities":20715,"ĠRou":20716,"Ġobserv":20717,"Ġoverlook":20718,"Ġcorrespond":20719,"Ġtheatrical":20720,"Ġrobotics":20721,"ĠCompl":20722,"ĠPasadena":20723,"laden":20724,"Ġvastly":20725,"olit":20726,"Ġjustification":20727,"Ġtampering":20728,"ĠSutherland":20729,"ĠMens":20730,"Ġinvisible":20731,"uren":20732,"ĠAshton":20733,"owl":20734,"Ġdisqual":20735,"ĠEva":20736,"Ġfriction":20737,"ĠIrvine":20738,"Ġaliens":20739,"ĠPension":20740,"ĠAssets":20741,"ĠBenedict":20742,"ittal":20743,"Ġsword":20744,"Ġunderwear":20745,"ĠFarmer":20746,"Ġtimber":20747,"Ġdependence":20748,"ĠTang":20749,"Ġ165":20750,"ĠNazis":20751,"Ġpunching":20752,"ĠGloria":20753,"usat":20754,"Ġluxurious":20755,"chuk":20756,"ĠCot":20757,"Ġregained":20758,"Ġreassure":20759,"Ġhello":20760,"Ġante":20761,"Ġnegotiators":20762,"Add":20763,"paced":20764,"ér":20765,"Ġdemolished":20766,"Ann":20767,"joy":20768,"ĠJenna":20769,"Apple":20770,"Ġdisturbance":20771,"Ġcommissions":20772,"ĠPolitico":20773,"along":20774,"Ġnem":20775,"Ġauctions":20776,"ruck":20777,"ĠOD":20778,"ofer":20779,"Play":20780,"Ġcarn":20781,"vez":20782,"Ġtents":20783,"Ġcongratulate":20784,"ĠLiquid":20785,"ĠCoyotes":20786,"uku":20787,"ĠAllah":20788,"Ġbend":20789,"Ġcanvas":20790,"ĠClifford":20791,"Ġvolunteered":20792,"Luc":20793,"bp":20794,"ĠCensus":20795,"ĠShot":20796,"Ġanonymously":20797,"ĠAnglo":20798,"ĠBayer":20799,"ĠAber":20800,"ĠCorrectional":20801,"Ġhardship":20802,"ĠBuenos":20803,"ĠDaw":20804,"Ġbaskets":20805,"Ġupstairs":20806,"Ġmindful":20807,"ĠLCD":20808,"ĠBlackburn":20809,"ĠHale":20810,"477":20811,"Ġcircus":20812,"ĠDragons":20813,"Ġrubble":20814,"rb":20815,"Ġheadaches":20816,"aunt":20817,"itus":20818,"Ġscaled":20819,"ĠComic":20820,"asio":20821,"ĠNordic":20822,"Per":20823,"Ġbombers":20824,"ilitation":20825,"Ġindirectly":20826,"ĠHod":20827,"andan":20828,"operation":20829,"Ġpuppy":20830,"ĠMats":20831,"Ġstewards":20832,"roup":20833,"Ġmemorandum":20834,"Ġpatio":20835,"const":20836,"ĠBold":20837,"ĠKaiser":20838,"Following":20839,"Ġcompat":20840,"Ġsidewalks":20841,"ĠFitzpatrick":20842,"Ġsunlight":20843,"ĠLever":20844,"ĠBecky":20845,"icles":20846,"ĠProbably":20847,"Ġgarner":20848,"ĠTomas":20849,"Ġblankets":20850,"uga":20851,"jiang":20852,"Ġrevel":20853,"ĠHutch":20854,"llers":20855,"Ġtrimmed":20856,"ĠSTR":20857,"ĠKR":20858,"ĠPike":20859,"ĠASS":20860,"Bay":20861,"Ġdiagnostic":20862,"ĠSteph":20863,"Ġtoured":20864,"ĠAvoid":20865,"vic":20866,"Without":20867,"ĠClinical":20868,"Ġblo":20869,"undo":20870,"ĠBoise":20871,"Ġspeculated":20872,"ĠProt":20873,"vention":20874,"Ġscholar":20875,"ĠSta":20876,"Featured":20877,"ĠPrev":20878,"Ġpenny":20879,"ĠHath":20880,"rawn":20881,"Ġrenovated":20882,"ĠFried":20883,"itol":20884,"uddle":20885,"Ġinquest":20886,"Ġmetropolitan":20887,"lights":20888,"Ġtempo":20889,"onom":20890,"ĠImport":20891,"Asia":20892,"Ġowes":20893,"Ġmagistrate":20894,"ĠFriedman":20895,"Ġcontacting":20896,"Ġstrains":20897,"Ġhomage":20898,"Ġlent":20899,"ception":20900,"git":20901,"Ġlively":20902,"Ġscra":20903,"WW":20904,"ön":20905,"rill":20906,"Jack":20907,"ĠShank":20908,"iani":20909,"Ġdecreasing":20910,"MON":20911,"ĠSupervisor":20912,"ĠCats":20913,"ĠFusion":20914,"Ġracially":20915,"ĠTara":20916,"ĠPurchase":20917,"ĠRally":20918,"ĠGraph":20919,"ĠHello":20920,"hest":20921,"ĠVarg":20922,"Ġdrowned":20923,"ĠThu":20924,"ĠWet":20925,"ĠEug":20926,"Ġrainbow":20927,"Ġtelev":20928,"ĠAmir":20929,"Based":20930,"Ġcookie":20931,"uding":20932,"Ġcontracting":20933,"Ġobjected":20934,"Ġfork":20935,"acent":20936,"ĠTil":20937,"ĠLilly":20938,"ĠEur":20939,"Ġhormone":20940,"Ġnails":20941,"ĠFischer":20942,"Ġpier":20943,"EMENT":20944,"Ġeruption":20945,"visory":20946,"Ġspeculate":20947,"apan":20948,"ĠJub":20949,"ĠHuckabee":20950,"string":20951,"stay":20952,"Ġsustaining":20953,"VM":20954,"Ġpriv":20955,"Ġclos":20956,"Ġdownloaded":20957,"ĠIv":20958,"Ġfinanced":20959,"ĠSao":20960,"ĠEverett":20961,"rene":20962,"ĠWo":20963,"ĠPiet":20964,"Ġengulfed":20965,"Ġexiting":20966,"uni":20967,"horn":20968,"Ġgrav":20969,"ection":20970,"Ġdrainage":20971,"Ġfuelled":20972,"Ġorganizational":20973,"bike":20974,"ĠAreas":20975,"Ġpoliceman":20976,"ĠFirm":20977,"ĠSlide":20978,"Ġrand":20979,"ĠJedi":20980,"Ge":20981,"really":20982,"Manchester":20983,"ĠWise":20984,"parent":20985,"Ġlad":20986,"Ġurine":20987,"ĠColombian":20988,"geon":20989,"Ġ1961":20990,"Mania":20991,"Ġgraph":20992,"Ġcod":20993,"fred":20994,"Ġeffic":20995,"ĠGateway":20996,"asket":20997,"Ġdiminished":20998,"Mass":20999,"Ġ205":21000,"Long":21001,"Ġgranddaughter":21002,"Ġshining":21003,"Semitic":21004,"Ġarising":21005,"Ġ330":21006,"ĠDU":21007,"ĠZah":21008,"Ġexclusion":21009,"ĠClaus":21010,"Ġven":21011,"oine":21012,"ĠAPI":21013,"reve":21014,"Ġmilitias":21015,"Ġfro":21016,"Ġwaved":21017,"ĠLuxembourg":21018,"Ġdiamonds":21019,"Ġstabilize":21020,"Ġqueue":21021,"ĠSponsor":21022,"Ġeldest":21023,"ĠLud":21024,"Ġwasting":21025,"Ġdimension":21026,"Ġmotorcycles":21027,"ucker":21028,"ĠTav":21029,"Ġsupremacy":21030,"Take":21031,"ĠCPU":21032,"cup":21033,"Ġdisregard":21034,"Ġenvelope":21035,"ĠCah":21036,"Ġproposes":21037,"ĠMaurice":21038,"Ġhobby":21039,"Ġharmon":21040,"Ġribbon":21041,"ĠOrigin":21042,"Ġbuilders":21043,"Ġconj":21044,"Ġcert":21045,"eat":21046,"ĠStern":21047,"ulia":21048,"vals":21049,"cling":21050,"Ġprovocative":21051,"Ġsofter":21052,"Ġ1948":21053,"Ġremod":21054,"ĠSob":21055,"Ġmaxim":21056,"Ġblueprint":21057,"oit":21058,"ĠGarner":21059,"Ġfibre":21060,"search":21061,"ĠWrite":21062,"270":21063,"Ġclergy":21064,"ĠPalo":21065,"obile":21066,"Mad":21067,"Ġclown":21068,"Ġtraced":21069,"280":21070,"ĠAlberto":21071,"Ġdrums":21072,"ĠFridays":21073,"ĠStrat":21074,"stated":21075,"ĠStevenson":21076,"Pr":21077,"Ġboasted":21078,"ĠBrees":21079,"ĠDonn":21080,"ĠMaya":21081,"Ġrelieve":21082,"Ġ1080":21083,"Ġcheapest":21084,"Ġuniquely":21085,"Ġjungle":21086,"Ġprevalence":21087,"Ġoutfield":21088,"ĠMaps":21089,"Ġaccustomed":21090,"pac":21091,"Ġcombinations":21092,"ĠSoros":21093,"stad":21094,"Ġket":21095,"Ġdisgusting":21096,"ĠOFF":21097,"irs":21098,"Ġbiased":21099,"Ġpaved":21100,"iked":21101,"utterstock":21102,"ocal":21103,"Ġsurround":21104,"ĠGuang":21105,"Ġspear":21106,"ĠBellev":21107,"ortun":21108,"Rec":21109,"acho":21110,"Ġfrightening":21111,"Ġtyres":21112,"normal":21113,"ĠYan":21114,"ĠWarsaw":21115,"ĠBod":21116,"ourse":21117,"199":21118,"Ver":21119,"erent":21120,"Ġsparkling":21121,"Ġchanting":21122,"Ġ1945":21123,"Ġturbo":21124,"Ġhazards":21125,"IRE":21126,"ĠRonnie":21127,"Ġsplitting":21128,"ĠMatte":21129,"roph":21130,"Ġtended":21131,"Ġvandalism":21132,"alis":21133,"SY":21134,"Ġoversaw":21135,"Happy":21136,"ĠTC":21137,"275":21138,"Ġeco":21139,"ĠKers":21140,"Ġextensions":21141,"ĠFlan":21142,"ĠCena":21143,"ĠDowns":21144,"Ġdrummer":21145,"Ġawaited":21146,"ĠACL":21147,"Ġlegends":21148,"ĠRollins":21149,"hend":21150,"Ġdeparting":21151,"Ġtha":21152,"Ġunre":21153,".(":21154,"Ġfaded":21155,"Ġretirees":21156,"vid":21157,"Ġentrants":21158,"ĠStella":21159,"arer":21160,"Ġteaspoon":21161,"ĠSheridan":21162,"irc":21163,"ĠRelief":21164,"ĠButt":21165,"Ġris":21166,"Ġundermined":21167,"Ġsunk":21168,"Sam":21169,"kamp":21170,"riot":21171,"rating":21172,"Ġclubhouse":21173,"Ġpeaked":21174,"ĠSki":21175,"Ġairstrikes":21176,"Ġconce":21177,"ĠCPR":21178,"Ġesp":21179,"ĠWave":21180,"ĠColiseum":21181,"outheastern":21182,"Ġtrou":21183,"Ġfeather":21184,"ĠSoy":21185,"ĠBihar":21186,"Ġintervened":21187,"mits":21188,"colored":21189,"330":21190,"Ġprocession":21191,"apeake":21192,"ité":21193,"riel":21194,"Ġmart":21195,"afer":21196,"ĠGuests":21197,"ĠPie":21198,"Ġshiny":21199,"ĠSixers":21200,"ĠRoads":21201,"Ġkicker":21202,"ĠCrimes":21203,"Ġfrontier":21204,"ansen":21205,"November":21206,"smith":21207,"ĠLaun":21208,"fried":21209,"weet":21210,"ĠGrass":21211,"Ġsanitation":21212,"ĠEat":21213,"ĠParts":21214,"ĠTun":21215,"amar":21216,"ĠJupiter":21217,"ĠFS":21218,"Ġunsc":21219,"ĠDone":21220,"Ġleveraging":21221,"Ġtucked":21222,"Ġineffective":21223,"Ġriots":21224,"wei":21225,"ĠAttend":21226,"Ġpertaining":21227,"amen":21228,"monds":21229,"Ġmism":21230,"serious":21231,"ĠViol":21232,"rous":21233,"Ġ129":21234,"uebl":21235,"umption":21236,"tri":21237,"ĠWedding":21238,"Ġtroopers":21239,"ĠTHR":21240,"olving":21241,"leys":21242,"Med":21243,"Ġseparatists":21244,"Ġimper":21245,"ĠFrontier":21246,"Ġwhit":21247,"ĠMutual":21248,"Ġrested":21249,"Ġunhealthy":21250,"gang":21251,"Ġresearching":21252,"ĠColonel":21253,"Ġaffordability":21254,"ĠRegarding":21255,"ĠWend":21256,"ĠMellon":21257,"Ġplots":21258,"Ġcanal":21259,"PER":21260,"ĠShopping":21261,"etry":21262,"Ġoccurrence":21263,"Ġgraves":21264,"BF":21265,"ĠKau":21266,"indust":21267,"Ġbeard":21268,"uate":21269,"ĠProdu":21270,"ĠSomali":21271,"ishers":21272,"ĠFell":21273,"ĠHutchinson":21274,"Ġhust":21275,"Ġillustration":21276,"Ġ//":21277,"Ġsharks":21278,"Ġcoincidence":21279,"Ġremake":21280,"Ġmural":21281,"course":21282,"ĠSultan":21283,"arse":21284,"Ġwhip":21285,"ĠPodcast":21286,"Ġtightened":21287,"Ġdenim":21288,"Ġlandfill":21289,"future":21290,"Ġsuperv":21291,"Hand":21292,"Ġpraising":21293,"ĠEly":21294,"ĠGust":21295,"ĠMayer":21296,"Ġorphan":21297,"Ġrepaired":21298,"ĠPir":21299,"Ġspiral":21300,"husband":21301,"ienne":21302,"iatric":21303,"Ġmarriages":21304,"Ġhorn":21305,"plain":21306,"ĠLum":21307,"ession":21308,"ĠFeatures":21309,"Ġbreakup":21310,"Ġentrepreneurship":21311,"rina":21312,"Ġembargo":21313,"Ġcapitalism":21314,"ĠMinor":21315,"Ġpromo":21316,"Ġexcel":21317,"Japan":21318,"Ġworsening":21319,"Ġstumbled":21320,"Ġpins":21321,"Ġswipe":21322,"Ġexile":21323,"Ġseparatist":21324,"ĠBian":21325,"Ġrelocation":21326,"Ġcommanders":21327,"Ġdowned":21328,"Ġblogger":21329,"packed":21330,"ĠSchn":21331,"Ġwaterfront":21332,"ĠYus":21333,"Ġnegotiator":21334,"Ġfavourable":21335,"Iran":21336,"oulder":21337,"Ġcance":21338,"Ġvind":21339,"angel":21340,"Ġauthenticity":21341,"Ġtowel":21342,"bul":21343,"ĠNeville":21344,"ĠBuddhist":21345,"fields":21346,"uly":21347,"Ġniece":21348,"Ġcorrections":21349,"Ġassignments":21350,"ĠSchl":21351,"Ġharmed":21352,"375":21353,"Ġwounding":21354,"ĠPosition":21355,"Ġsupermarkets":21356,"Ġdisclosures":21357,"Ġ185":21358,"esp":21359,"ĠMcCull":21360,"ĠMale":21361,"Ġsailors":21362,"mis":21363,"ĠSophia":21364,"Ġunfolded":21365,"owell":21366,"ĠScarborough":21367,"Ġentrepreneurial":21368,"118":21369,"ogy":21370,"ĠLikewise":21371,"Ġswung":21372,"Ġdrawings":21373,"Ġdrafting":21374,"ĠSimple":21375,"ĠFilip":21376,"arf":21377,"Ġfade":21378,"Ġmerged":21379,"ĠLeaf":21380,"sun":21381,"Ġflame":21382,"Ġindices":21383,"ĠCreate":21384,"ittle":21385,"ĠWer":21386,"ĠMond":21387,"Ġoz":21388,"ĠSmoke":21389,"Ġreplies":21390,"ĠDH":21391,"Ġjud":21392,"ĠFalk":21393,"Ġ---":21394,"Ġconstitutes":21395,"Ġtheat":21396,"119":21397,"Ġintermediate":21398,"vill":21399,"ĠGow":21400,"ĠHut":21401,"ł":21402,"155":21403,"ĠLocated":21404,"ĠDoor":21405,"Ġsliced":21406,"aru":21407,"Ġtearing":21408,"defense":21409,"oyer":21410,"Ġprodu":21411,"Ġseminar":21412,"asso":21413,"Ġpeaks":21414,"Ġconceal":21415,"Ġcrypto":21416,"Ġsetbacks":21417,"ĠAlicia":21418,"ĠFAA":21419,"Ġcontinuity":21420,"Ġcatastrophe":21421,"Ġbeg":21422,"Ġscales":21423,"apixel":21424,"Ġsalon":21425,"Ste":21426,"Ġlesbian":21427,"Ġanticip":21428,"Ġutilization":21429,"Ġchickens":21430,"Ġspinal":21431,"ĠJuliet":21432,"ĠFas":21433,"prising":21434,"ĠSalvation":21435,"Ġ138":21436,"Ġutilizing":21437,"âĢ¢":21438,"ĠMessenger":21439,"Ġrebellion":21440,"ĠAlexand":21441,"Ġinsect":21442,"Ġribs":21443,"ĠBild":21444,"Ġmonopoly":21445,"Queen":21446,"ĠNaples":21447,"Ġ133":21448,"Ġhourly":21449,"Ġego":21450,"Ġpencil":21451,"ĠPew":21452,"Ġdesirable":21453,"vant":21454,"ĠLAT":21455,"Ġperpet":21456,"lish":21457,"Ġ201":21458,"Ġdistances":21459,"Ġdistressed":21460,"Work":21461,"Ġtattoos":21462,"Ġstereotypes":21463,"istent":21464,"ĠCoral":21465,"fo":21466,"Ġpayable":21467,"Ġakin":21468,"ĠLis":21469,"ĠFinding":21470,"Ġsusceptible":21471,"ĠKiw":21472,"Ġforgiveness":21473,"ĠMoment":21474,"ĠDmitry":21475,"Ġrenov":21476,"Ġquint":21477,"ĠWaterloo":21478,"ĠReality":21479,"Ġstray":21480,"ĠBeaver":21481,"Ġbites":21482,"Ġelusive":21483,"Ġvirtue":21484,"Ġgadgets":21485,"Ġlandslide":21486,"ĠHealthy":21487,"Ġpits":21488,"Donnell":21489,"Ġirony":21490,"uct":21491,"Ġpractitioners":21492,"Ġreck":21493,"governmental":21494,"Ġatomic":21495,"Ġmotiv":21496,"Ġpolic":21497,"Ġcommunicated":21498,"ĠHS":21499,"Ġcriticize":21500,"Ġsynerg":21501,"Del":21502,"ĠRoe":21503,"Ġinspirational":21504,"ĠWarning":21505,"pel":21506,"Ġnevertheless":21507,"Ġdespair":21508,"Ġ(.":21509,"Ġfearing":21510,"Ġgrop":21511,"tree":21512,"Ġtrusts":21513,"Ġinterviewing":21514,"amic":21515,"Ġscor":21516,"ject":21517,"Another":21518,"pose":21519,"Ġdepicted":21520,"ĠPhotography":21521,"ĠLenovo":21522,"ĠEpic":21523,"ĠBoot":21524,"GI":21525,"enses":21526,"Class":21527,"arity":21528,"Ġservicing":21529,"ĠHann":21530,"Ġawe":21531,"Ġoverdoses":21532,"ĠFinnish":21533,"Ġpav":21534,"ĠPCs":21535,"SEC":21536,"ĠStro":21537,"Ġattracts":21538,"Ġapprehended":21539,"128":21540,"Ġunstable":21541,"ĠOutdoor":21542,"Ġcloth":21543,"ĠUlster":21544,"Ġvisually":21545,"Ġsculpt":21546,"Ġsufficiently":21547,"ĠKendrick":21548,"Ġengages":21549,"Ġknives":21550,"ĠGut":21551,"Ġarbit":21552,"osition":21553,"Ġemoji":21554,"Ġpinpoint":21555,"Ġremembering":21556,"rence":21557,"ĠVish":21558,"Ġimproperly":21559,"Ġranc":21560,"Ġupstream":21561,"Ġcheckpoint":21562,"Ġrash":21563,"eson":21564,"Ġtoes":21565,"260":21566,"Ġinvalid":21567,"Ġonions":21568,"Ġlashed":21569,"ĠDong":21570,"Ġprovisional":21571,"ĠFern":21572,"Ġirresponsible":21573,"actively":21574,"ĠKnown":21575,"Ġben":21576,"ĠBlank":21577,"Ġactresses":21578,"paying":21579,"Ġsyrup":21580,"isman":21581,"Ġeducating":21582,"Sunday":21583,"ifiable":21584,"Post":21585,"Ġcalculation":21586,"Ġhesitate":21587,"ĠIncreasing":21588,"Ġreeling":21589,"ĠDairy":21590,"ensing":21591,"Ġmaternity":21592,"Ø":21593,"./":21594,"ĠElm":21595,"Ġweddings":21596,"ĠYard":21597,"117":21598,"ĠRocket":21599,"OF":21600,"Ġtreasurer":21601,"Ġrattled":21602,"ĠDrop":21603,"arel":21604,"ĠFulton":21605,"ĠGiant":21606,"ĠFloor":21607,"Jet":21608,"ikk":21609,"ĠBucs":21610,"ostics":21611,"reme":21612,"ĠRouse":21613,"Ġdeliber":21614,"ĠEle":21615,"Ġconducts":21616,"ĠBlog":21617,"connected":21618,"Ġprayed":21619,"Ġcolourful":21620,"Ġaugmented":21621,"Ġbatted":21622,"Ġrelevance":21623,"ĠRomanian":21624,"acqu":21625,"ĠChel":21626,"ĠClo":21627,"ĠGraves":21628,"Ġchees":21629,"ĠGibbs":21630,"CLE":21631,"Ġfertility":21632,"Ġambul":21633,"Ġspecs":21634,"ĠIRA":21635,"ĠBooth":21636,"ithe":21637,"ĠPlayoff":21638,"ammed":21639,"Ġcollaborating":21640,"Ġlunar":21641,"Ġconfronting":21642,"Ġattribute":21643,"King":21644,"riz":21645,"Ġcasualty":21646,"acia":21647,"waters":21648,"Ġpaving":21649,"Ġcaregivers":21650,"nor":21651,"Ġreacting":21652,"ĠHash":21653,"Ġsqueezed":21654,"Ġexert":21655,"ĠMichele":21656,"ĠConc":21657,"ĠHep":21658,"Ġsewage":21659,"wart":21660,"GY":21661,"Ġdiscourage":21662,"ĠFir":21663,"Ġtextile":21664,"ĠSpice":21665,"ĠFah":21666,"Ġcomplainant":21667,"Ġinstinct":21668,"camp":21669,"ĠEdison":21670,"ĠVIDEOS":21671,"LM":21672,"ĠSands":21673,"About":21674,"Ġdisk":21675,"brid":21676,"Ġmuted":21677,"ACC":21678,"Ġwre":21679,"event":21680,"Ġicons":21681,"Express":21682,"udes":21683,"ĠBeatles":21684,"color":21685,"ĠHaas":21686,"ĠWolfe":21687,"ĠYOUR":21688,"Ġaccessibility":21689,"ĠCornwall":21690,"Ġing":21691,"Ġatrocities":21692,"weather":21693,"ĠDominion":21694,"ĠMIL":21695,"ĠLara":21696,"Ġunravel":21697,"Ġmaneuver":21698,"Ġfoam":21699,"ribe":21700,"CI":21701,"Ġcandles":21702,"acs":21703,")(":21704,"coon":21705,"ĠPurple":21706,"ĠGovernors":21707,"ĠKeystone":21708,"ĠYuk":21709,"file":21710,"Ġviol":21711,"gery":21712,"370":21713,"train":21714,"Ġgunshots":21715,"olin":21716,"Ġviruses":21717,"ĠTex":21718,"hours":21719,"Ġprev":21720,"ĠRid":21721,"ected":21722,"ĠVog":21723,"riers":21724,"Ġmurdering":21725,"ĠIz":21726,"Ġdeliberations":21727,"arming":21728,"unda":21729,"Ġrink":21730,"ĠDrugs":21731,"idered":21732,"Ġforge":21733,"Ġexpansive":21734,"VIEW":21735,"ĠBots":21736,"Ġswitches":21737,"KO":21738,"atten":21739,"Ġvariants":21740,"ĠVirtual":21741,"ĠCoch":21742,"yon":21743,"ĠKai":21744,"Ġbullied":21745,"iday":21746,"version":21747,"Ġlib":21748,"ĠCec":21749,"igated":21750,"ĠTRUMP":21751,"ĠPod":21752,"Ġtoppled":21753,"Ġeyeing":21754,"ĠPatients":21755,"techn":21756,"Ġhampered":21757,"Ġavert":21758,"ĠScheme":21759,"ĠCorm":21760,"Ġpony":21761,"Ġzoom":21762,"abo":21763,"Ġsleeves":21764,"lane":21765,"ĠLester":21766,"ĠDane":21767,"Ġcough":21768,"Ġsignings":21769,"HER":21770,"Ġsibling":21771,"Ġredemption":21772,"Ġstockp":21773,"ĠAlgeria":21774,"Ġpadd":21775,"ĠBrenda":21776,"uchi":21777,"Ġtransporting":21778,"Ġspeculative":21779,"ĠSek":21780,"abal":21781,"Ġshipment":21782,"oker":21783,"Ġwarranty":21784,"atan":21785,"Ġblister":21786,"ĠCelebration":21787,"Ġwal":21788,"Ġlac":21789,"Ġprioritize":21790,"ression":21791,"BP":21792,"Ġcollaborated":21793,"ĠNewsletter":21794,"ĠDamian":21795,"ĠResidential":21796,"Ġgra":21797,"Ġfeasible":21798,"ĠCrest":21799,"ĠBean":21800,"ĠSturgeon":21801,"ĠTale":21802,"ĠContin":21803,"ĠMush":21804,"Ġrocking":21805,"ĠMane":21806,"ĠHumane":21807,"resistant":21808,"ĠFra":21809,"highest":21810,"fts":21811,"Ġamassed":21812,"ĠPavilion":21813,"ĠSkin":21814,"Ġunfold":21815,"Ġresur":21816,"ĠPET":21817,"model":21818,"Ġemploying":21819,"Ġrude":21820,"Ġirrelevant":21821,"angu":21822,"Page":21823,"PN":21824,"igator":21825,"ĠReb":21826,"ĠArrest":21827,"ĠGund":21828,"Ġmalls":21829,"zhen":21830,"wed":21831,"Ġdaring":21832,"Ġfactual":21833,"ĠGent":21834,"Ġinforming":21835,"ĠStri":21836,"ĠLounge":21837,".]":21838,"ĠTribunal":21839,"ĠMoines":21840,"Ġshadows":21841,"generated":21842,"fulness":21843,"Ġheartfelt":21844,"ĠLivingston":21845,"ĠClerk":21846,"Ġnationalism":21847,"ĠMiche":21848,"balls":21849,"anos":21850,"agle":21851,"Ġprejudice":21852,"Ġevenly":21853,"Ġswearing":21854,"Ġexits":21855,"Ġcondemning":21856,"Ġvanilla":21857,"club":21858,"ĠFunding":21859,"ĠDover":21860,"Ġhots":21861,"Ġfres":21862,"Ġgoodness":21863,"ĠMcKay":21864,"Ġbulls":21865,"avia":21866,"129":21867,"Ġ1947":21868,"Ġdefamation":21869,"ĠMoran":21870,"irms":21871,"ĠFitz":21872,"ĠRossi":21873,"urated":21874,"Ġvariation":21875,"ĠBauer":21876,"ĠSchro":21877,"Ġcolony":21878,"ĠParliamentary":21879,"ikan":21880,"Ġstirring":21881,"ĠSheldon":21882,"Ġaccessory":21883,"ĠUtilities":21884,"Ġnab":21885,"Ġpract":21886,"Ġherein":21887,"ĠRole":21888,"ĠMant":21889,"Ġpharm":21890,"Ġ215":21891,"ĠNGO":21892,"ĠAnything":21893,"ĠMacedonia":21894,"Ġbree":21895,"ĠWTO":21896,"Chicago":21897,"ĠProtect":21898,"quarters":21899,"ĠGrassley":21900,"ĠInteractive":21901,"ĠInterview":21902,"Ġ550":21903,"Ġastronauts":21904,"Ġfreak":21905,"ĠIntegrated":21906,"Ġindict":21907,"Ġgenerators":21908,"acio":21909,"Kevin":21910,"Ġvaccination":21911,"Ġblockade":21912,"ĠSons":21913,"Ġcapita":21914,"ĠAnita":21915,"ĠExport":21916,"ĠNex":21917,"ĠAram":21918,"Ġzinc":21919,"Ġrevamped":21920,"Ġselective":21921,"Ġmanipulate":21922,"ĠBedford":21923,"ĠBattery":21924,"Ġqualifiers":21925,"lean":21926,"Ġscrew":21927,"film":21928,"ror":21929,"ĠEllison":21930,"ombo":21931,"ĠOst":21932,"165":21933,"Ġslaves":21934,"ĠPayton":21935,"Ġbarg":21936,"Ġrugged":21937,"ĠWinn":21938,"ĠHammer":21939,"ĠUPS":21940,"Euro":21941,"Ġunfamiliar":21942,"Ġdistract":21943,"Ġbuffer":21944,"ledge":21945,"Ġtrunk":21946,"Ġ320":21947,"122":21948,"Ġdilemma":21949,"Ġpra":21950,"Ġutmost":21951,"Ġcampaigners":21952,"icular":21953,"eful":21954,"�":21955,"ĠHQ":21956,"neau":21957,"Ġsir":21958,"test":21959,"Company":21960,"Ġrescind":21961,"ardon":21962,"MG":21963,"Gov":21964,"ĠRaz":21965,"Ġrod":21966,"fed":21967,"Ġpsych":21968,"Ġunin":21969,"ĠArbor":21970,"Ġnewcomer":21971,"ĠEdwin":21972,"raising":21973,"quist":21974,"Ġdiscoveries":21975,"Steve":21976,"Ġscramble":21977,"js":21978,"Ġacoustic":21979,"Ġdeterioration":21980,"Ġobserving":21981,"ĠWinning":21982,"ĠSaban":21983,"idy":21984,"Ġoverd":21985,"Ġscouting":21986,"Ġpunitive":21987,"ĠShelter":21988,"Ġmocked":21989,"Ġdreamed":21990,"Ġinvaluable":21991,"LP":21992,"standard":21993,"Ġrecounted":21994,"ĠSabres":21995,"points":21996,"Ġfringe":21997,"ĠBarker":21998,"alian":21999,"ĠPROV":22000,"Ġcartel":22001,"Ġovercrowd":22002,"tain":22003,"Year":22004,"ĠWelfare":22005,"ĠChr":22006,"Ġintroduces":22007,"ĠDoing":22008,"ĠGlover":22009,"Ġdeteriorating":22010,"Par":22011,"Ġattendant":22012,"ĠMold":22013,"ĠFlying":22014,"ovan":22015,"Ġoptimize":22016,"Ġchapters":22017,"Ġdull":22018,"gay":22019,"ĠATP":22020,"ĠKah":22021,"ainer":22022,"feet":22023,"Ġjoking":22024,"Ġdisadvantage":22025,"Rep":22026,"Ġtwisted":22027,"Ġslain":22028,"Ġcomprise":22029,"Ġrestricting":22030,"Ġdispos":22031,"Ġshaky":22032,"Ġembattled":22033,"owe":22034,"conscious":22035,"oken":22036,"Ġmistaken":22037,"ĠDra":22038,"Ġreservoir":22039,"Ġspate":22040,"Scott":22041,"avor":22042,"Ġqual":22043,"amel":22044,"hunt":22045,"ĠChevy":22046,"Ġclaw":22047,"Ġwitch":22048,"ĠZimmerman":22049,"arium":22050,"Ġrubbish":22051,"Ġstrings":22052,"Ġdoc":22053,"Ġplaque":22054,"ĠCyr":22055,"Ġflourish":22056,"Ġworthwhile":22057,"Ġbanners":22058,"ĠLemon":22059,"ĠRainbow":22060,"Ġconsisted":22061,"ĠHOW":22062,"Ñ":22063,"Ġblogs":22064,"CLUS":22065,"eely":22066,"Ġbeast":22067,"ĠMai":22068,"Ġhostility":22069,"eros":22070,"Ġforeseeable":22071,"ĠCorker":22072,"ĠWEEK":22073,"visors":22074,"ressive":22075,"ĠViktor":22076,"Ġbureaucracy":22077,"Ġ256":22078,"ĠFeel":22079,"ĠAdventure":22080,"Ġefficacy":22081,"ĠInstitution":22082,"ĠHarbaugh":22083,"ĠPractice":22084,"ĠChristianity":22085,"Thanks":22086,"Ġfridge":22087,"idel":22088,"Ġeff":22089,"Ġvein":22090,"terms":22091,"Ġignorance":22092,"Ġscream":22093,"Ġwit":22094,"ĠRousse":22095,"ĠWillow":22096,"Ġhallway":22097,"former":22098,"Ġshooters":22099,"ĠReporting":22100,"Ġgal":22101,"Ġsavvy":22102,"rand":22103,"Ġremed":22104,"ĠBaron":22105,"inar":22106,"Ġseizures":22107,"ĠThorn":22108,"ĠProtesters":22109,"ĠRevolutionary":22110,"think":22111,"ĠCabrera":22112,"Four":22113,"ĠRudd":22114,"Ġprost":22115,"ĠBottom":22116,"Port":22117,"nas":22118,"ifax":22119,"Wire":22120,"Ġtokens":22121,"antis":22122,"ĠSOU":22123,"ĠMilk":22124,"asters":22125,"Ġshrimp":22126,"Ġcakes":22127,"blue":22128,"ifty":22129,"View":22130,"adium":22131,"fen":22132,"zyk":22133,"ĠEmil":22134,"Ġdismay":22135,"Ġtilt":22136,"aska":22137,"Young":22138,"Ġpredators":22139,"Ġovershadowed":22140,"mitt":22141,"ĠSemin":22142,"ĠSchiff":22143,"ĠClarkson":22144,"212":22145,"210":22146,"Ġvanished":22147,"Ġmesh":22148,"ĠBurnett":22149,"ĠMent":22150,"ĠBlind":22151,"ĠPatriot":22152,"ĠVil":22153,"Ġflick":22154,"ĠTowns":22155,"ĠWhites":22156,"Ġspice":22157,"ĠMode":22158,"Ġnominate":22159,"Ġwrest":22160,"ĠAshes":22161,"Ġrows":22162,"ĠClint":22163,"Ġgentleman":22164,"utan":22165,"athlon":22166,"ĠIntermediate":22167,"hews":22168,"Ġoffended":22169,"ĠPaige":22170,"ĠFinch":22171,"ĠAboriginal":22172,"positive":22173,"Stop":22174,"Ġrenting":22175,"Ġ[âĢ¦]":22176,"ĠHert":22177,"Ġvegetation":22178,"apes":22179,"ĠCanon":22180,"appa":22181,"Ġabst":22182,"ĠKatz":22183,"Ġsurfing":22184,"aghan":22185,"ĠPresidency":22186,"Ġscaling":22187,"ĠSas":22188,"Ġpeanut":22189,"Ġrecommending":22190,"cious":22191,"endez":22192,"eker":22193,"ĠKamp":22194,"Ġsitcom":22195,"Ġcrust":22196,"women":22197,"ĠJes":22198,"ĠWhe":22199,"ĠWarwick":22200,"Ġepit":22201,"ĠAlc":22202,"Ġdictate":22203,"ĠSPORTS":22204,"ĠLanguage":22205,"Ġindicative":22206,"ĠMacDonald":22207,"Ġreorgan":22208,"Ġ`":22209,"ARS":22210,"Ġliberation":22211,"Ġbless":22212,"Ġreflective":22213,"Ġà¤":22214,"Ġdesires":22215,"ĠHank":22216,"ĠLaunch":22217,"Ġrotating":22218,"ĠStones":22219,"Ġcoordinating":22220,"ĠZeit":22221,"Ġskepticism":22222,"ĠAlam":22223,"ĠTrout":22224,"ĠSMS":22225,"ĠCrescent":22226,"ĠTeacher":22227,"Ġfury":22228,"Ġeyebrows":22229,"onga":22230,"ĠPilot":22231,"ĠRutherford":22232,"Ġinterstate":22233,"established":22234,"Ġbaggage":22235,"Ġ131":22236,"riks":22237,"mil":22238,"Ġneon":22239,"Ġqueer":22240,"ourced":22241,"ĠKash":22242,"ĠEleven":22243,"illes":22244,"ĠOpportun":22245,"Ġstre":22246,"Washington":22247,"ĠDifferent":22248,"Ġexempl":22249,"Ġboarded":22250,"Ġrogue":22251,"ĠDNC":22252,"rone":22253,"Ġreversing":22254,"nine":22255,"ĠIvory":22256,"itating":22257,"uve":22258,"Ġfracture":22259,"255":22260,"ĠAssessment":22261,"Ġsubjective":22262,"Ġfluct":22263,"ĠJaguar":22264,"Ġstride":22265,"Ġreapp":22266,"ĠGrow":22267,"against":22268,"ĠMedina":22269,"scenes":22270,"ĠNieto":22271,"Ġsou":22272,"ĠFleming":22273,"Ġnarcotics":22274,"ĠBere":22275,"ĠBub":22276,"ĠAck":22277,"Ġvinyl":22278,"ĠCopy":22279,"ĠGarland":22280,"ĠDuty":22281,"Ġinn":22282,"Ġmerchant":22283,"Ġactivate":22284,"Ġglowing":22285,"ettle":22286,"ĠBran":22287,"Ġsilk":22288,"anco":22289,"TL":22290,"ĠFurn":22291,"Ġwithheld":22292,"Ġpulse":22293,"ĠGU":22294,"BUS":22295,"ĠHyper":22296,"Ġpicnic":22297,"Ġpositives":22298,"ĠParamount":22299,"Ġ737":22300,"Ġenlisted":22301,"ĠValerie":22302,"false":22303,"ĠChocolate":22304,"ĠSTAR":22305,"Ġdescended":22306,"Ġtasty":22307,"ĠDaesh":22308,"ĠNed":22309,"Ġcomplimentary":22310,"Ġdepicting":22311,"ĠHavana":22312,"college":22313,"Ġtraces":22314,"Ġundue":22315,"ĠSisters":22316,"aum":22317,"ĠCourier":22318,"ĠOng":22319,"ĠSparks":22320,"ongs":22321,"ĠYong":22322,"URR":22323,"los":22324,"Ġhorsepower":22325,"confidence":22326,"ĠPett":22327,"ĠMeasure":22328,"Ġmarches":22329,"zig":22330,"ĠTOR":22331,"Ġexported":22332,"ĠRak":22333,"ĠInvestigations":22334,"Ġterminate":22335,"ĠTian":22336,"Ġmasters":22337,"ĠDS":22338,"Ġoutraged":22339,"ĠCups":22340,"ĠWeir":22341,"exec":22342,"Ġjourneys":22343,"Ġabide":22344,"Ġavail":22345,"ĠStreets":22346,"Ġfixes":22347,"Ġcocoa":22348,"Ġabundant":22349,"Ġhubs":22350,"mort":22351,"Ġrobberies":22352,"ĠBark":22353,"Ġprecautions":22354,"Ġhammered":22355,"ometric":22356,"mith":22357,"ĠMcCann":22358,"ĠJaw":22359,"ĠQuest":22360,"ĠMcF":22361,"Ġlob":22362,"Ġlegalized":22363,"Ġquirky":22364,"Ġtrailers":22365,"ĠIndividual":22366,"Ġcumulative":22367,"Ġenlarge":22368,"Ġconvoy":22369,"olen":22370,"got":22371,"landers":22372,"Ġscanner":22373,"Ġscans":22374,"ĠEg":22375,"prof":22376,"Ġhosp":22377,"ĠColo":22378,"Ġerr":22379,"Ġdeval":22380,"ĠUsually":22381,"Ġbul":22382,"ummy":22383,"Ġtandem":22384,"occupied":22385,"Ġmandates":22386,"ĠSwim":22387,"121":22388,"ussed":22389,"EF":22390,"Ġfries":22391,"Until":22392,"rc":22393,"Ġbadge":22394,"Ġstrips":22395,"Ġmagnet":22396,"Ġarchive":22397,"stan":22398,"ĠDeadline":22399,"Ġdisposable":22400,"Ġbob":22401,"Ġnorthwestern":22402,"Jul":22403,"ĠSAL":22404,"Ġinfluencing":22405,"Ġdevil":22406,"ĠEllie":22407,"cms":22408,"ingo":22409,"888":22410,"Ġcosmetic":22411,"Also":22412,"Ġyacht":22413,"Ġlazy":22414,"Ġmerc":22415,"Ġabsorbed":22416,"harm":22417,"116":22418,"Ġsubpoena":22419,"Ġcounters":22420,"ĠLori":22421,"Ġrandomly":22422,"nea":22423,"waves":22424,"Ġrelie":22425,"ĠKiss":22426,"Ġchassis":22427,"Ġbakery":22428,"Images":22429,"ĠHolden":22430,"Ġamazed":22431,"Ġalignment":22432,"ĠPowers":22433,"Ġlabelled":22434,"Ġstaunch":22435,"Ġsignaling":22436,"Ġsenate":22437,"Ġunconventional":22438,"ĠAlternative":22439,"Ġambassadors":22440,"ĠVPN":22441,"atics":22442,"Ġmosquito":22443,"ĠScholarship":22444,"Ġhelpless":22445,"alone":22446,"ZA":22447,"chel":22448,"Ġconstituencies":22449,"ĠCafé":22450,"Ġhatch":22451,"ĠRupert":22452,"Ġrendering":22453,"Ġreinstated":22454,"Ġinterval":22455,"Texas":22456,"ĠAHL":22457,"February":22458,"review":22459,"Ġgle":22460,"Ġfals":22461,"Ġmarkers":22462,"Ġgovernmental":22463,"ĠPos":22464,"Ġarose":22465,"every":22466,"Ġrulings":22467,"obar":22468,"Govern":22469,"gren":22470,"isan":22471,"Ġmarketed":22472,"Click":22473,"Ġord":22474,"Ġballoons":22475,"asers":22476,"ĠHorton":22477,"pub":22478,"ĠAerospace":22479,"Ġflank":22480,"Ġmolecular":22481,"bour":22482,"nuts":22483,"Ġalliances":22484,"Ġbenchmarks":22485,"ocate":22486,"stadt":22487,"ĠGoodwin":22488,"lap":22489,"ĠFactors":22490,"Never":22491,"ĠNem":22492,"Ġroadside":22493,"orth":22494,"Ġexhibited":22495,"ĠPearce":22496,"ĠOlsen":22497,"Ġpostal":22498,"ĠLiberation":22499,"reen":22500,"mary":22501,"Ġropes":22502,"Ġlarg":22503,"Ġgob":22504,"boys":22505,"ĠSax":22506,"Ġreimbursement":22507,"ĠVie":22508,"ĠCatholics":22509,"ĠMartial":22510,"Ġpremiered":22511,"Ġawaits":22512,"ĠUnderstanding":22513,"ĠBelarus":22514,"ĠVor":22515,"ogi":22516,"iaz":22517,"Ġvictorious":22518,"Ġancestors":22519,"Ġwreckage":22520,"Ġoppression":22521,"ĠChildhood":22522,"Ġwidth":22523,"ĠPlymouth":22524,"ĠFifty":22525,"Ġoccupancy":22526,"etts":22527,"ĠFiscal":22528,"lifting":22529,"ĠTraditional":22530,"Ġnostalgia":22531,"Law":22532,"Ġlays":22533,"Ġarresting":22534,"Ġanticipating":22535,"Ġinsults":22536,"ĠExtension":22537,"Ġgenerator":22538,"ummer":22539,"Ġageing":22540,"Ġbouncing":22541,"ember":22542,"ĠWAR":22543,"ĠNico":22544,"ĠWow":22545,"ĠRaven":22546,"flower":22547,"ĠCrim":22548,"bh":22549,"Ġundo":22550,"Ġburgers":22551,"roud":22552,"ĠAtkinson":22553,"ĠYEAR":22554,"Ġpoorer":22555,"ICA":22556,"ĠSchedule":22557,"Ġstronghold":22558,"ĠMillennium":22559,"Ġ###":22560,"ilda":22561,"ĠGH":22562,"Ġupscale":22563,"aldi":22564,"ĠResolution":22565,"Ġswelling":22566,"Ġgrieving":22567,"ĠNile":22568,"ĠTig":22569,"ERY":22570,"ooth":22571,"BALL":22572,"Ġballet":22573,"Ġbucks":22574,"ĠUV":22575,"akin":22576,"Ġchilling":22577,"Ġdatabases":22578,"ĠGD":22579,"section":22580,"Ġhires":22581,"Ġmul":22582,"Ġsen":22583,"ĠTownsend":22584,"Ġinspected":22585,"ilic":22586,"Ġdiscriminatory":22587,"fol":22588,"Ġalcoholic":22589,"ĠHoff":22590,"Carl":22591,"Ġvicinity":22592,"lein":22593,"ĠEco":22594,"ĠGovern":22595,"Ġsecrecy":22596,"aned":22597,"ĠDUP":22598,"Ġ570":22599,"Ġsow":22600,"Ġstalls":22601,"Ġinsulting":22602,"ĠDT":22603,"Ġinforms":22604,"fitting":22605,"ĠDepending":22606,"ĠMelanie":22607,"ĠThom":22608,"path":22609,"Ġadmired":22610,"Peter":22611,"idents":22612,"ielding":22613,"ĠShanahan":22614,"TD":22615,"Things":22616,"sn":22617,"Ġconstituted":22618,"Ġ137":22619,"Ġderailed":22620,"ĠBonnie":22621,"Ġgraffiti":22622,"Ġearnest":22623,"Ġcompliant":22624,"blown":22625,"Ġalle":22626,"prise":22627,"Ġfocal":22628,"Ġgentlemen":22629,"ĠTalks":22630,"Ġpassports":22631,"Ġdeprived":22632,"Ġdude":22633,"ĠNath":22634,"Ġgoverned":22635,"Ġsac":22636,"Ġcastle":22637,"qv":22638,"Ġtolerated":22639,"ĠSci":22640,"close":22641,"ĠDynamics":22642,"Ġflashing":22643,"yk":22644,"ĠConsolid":22645,"Ġinherently":22646,"ĠForrest":22647,"Gene":22648,"Public":22649,"Ġloser":22650,"runners":22651,"Ġprudent":22652,"Ġpioneering":22653,"ĠHowe":22654,"ĠButter":22655,"ĠArabian":22656,"acha":22657,"ĠBBQ":22658,"ĠMineral":22659,"Ġdestiny":22660,"Ġretrieve":22661,"ĠBav":22662,"reth":22663,"oby":22664,"ĠGrid":22665,"Ġgrievances":22666,"ĠTips":22667,"Ġadamant":22668,"Ġdiets":22669,"Ġmilestones":22670,"Ġcollects":22671,"ĠLaboratories":22672,"ĠWC":22673,"Ġpostp":22674,"Ġdams":22675,"ĠOEM":22676,"Ġrumor":22677,"Ġlocking":22678,"Ġemission":22679,"Ġqueries":22680,"Jones":22681,"Ġlang":22682,"ĠAcqu":22683,"ĠMedium":22684,"ĠTreasurer":22685,"Sept":22686,"FB":22687,"Ġintegrating":22688,"Ġbolstered":22689,"Ġincorporating":22690,"encers":22691,"Ġirregularities":22692,"Ġnom":22693,"iod":22694,"ĠAi":22695,"Ġsor":22696,"anked":22697,"Ġrehears":22698,"fig":22699,"ĠBug":22700,"hoff":22701,"Ġtrooper":22702,"Ġgalaxy":22703,"amon":22704,"ĠAtlas":22705,"Ġsolicit":22706,"Ġsings":22707,"ĠInstructions":22708,"ĠMig":22709,"thinking":22710,"ĠCostco":22711,"Ġbreasts":22712,"Ġportraits":22713,"ĠCock":22714,"Ġsubscriptions":22715,"Ġpine":22716,"Ġhaunted":22717,"ĠMED":22718,"eer":22719,"ega":22720,"ĠZa":22721,"ENN":22722,"ĠWinners":22723,"aith":22724,"safe":22725,"Ġ143":22726,"ĠWeston":22727,"ĠLansing":22728,"ĠLaurel":22729,"ocrat":22730,"ograph":22731,"Ġmatchups":22732,"ĠFriend":22733,"Ġdigest":22734,"Ġdimensions":22735,"azing":22736,"Ġtipping":22737,"Ġenrich":22738,"gart":22739,"argo":22740,"Ġoutbreaks":22741,"Ġsalvage":22742,"ĠErica":22743,"Ġmodules":22744,"ĠPDF":22745,"ĠGoods":22746,"oots":22747,"2011":22748,"Ġinterrupt":22749,"Ġradi":22750,"ĠSimone":22751,"vell":22752,"ĠSV":22753,"extremely":22754,"Ġstadiums":22755,"ĠRox":22756,"Ġconflicting":22757,"Ġyouthful":22758,"ĠUM":22759,"series":22760,"Ġded":22761,"Ġfielding":22762,"Pre":22763,"itled":22764,"Ġstreamed":22765,"Ġapprentices":22766,"ĠAlec":22767,"ĠGap":22768,"ĠPrem":22769,"Ġleased":22770,"Ġdeepening":22771,"Ġbounds":22772,"Ġrethink":22773,"ĠVoting":22774,"ĠScha":22775,"blood":22776,"ĠReeves":22777,"Ġbells":22778,"Ġcollector":22779,"ĠCrimson":22780,"ĠWheat":22781,"207":22782,"ĠHB":22783,"ĠBCC":22784,"Ġsync":22785,"ĠAnders":22786,"Ġthanking":22787,"Ġlayoffs":22788,"Ġfoolish":22789,"Ġcustod":22790,"Ġelephants":22791,"Ġcorrelation":22792,"ĠHarding":22793,"ĠGPU":22794,"ĠBarnett":22795,"Ġol":22796,"Ġalarms":22797,"Ġfluctuations":22798,"shop":22799,"Ġcommentators":22800,"ĠAlpine":22801,"Ġmur":22802,"Ġbiotech":22803,"Ġunlocked":22804,"ouri":22805,"roe":22806,"ĠPayment":22807,"ĠPOL":22808,"ĠGuest":22809,"Ġphrases":22810,"ĠBuilt":22811,"erves":22812,"Ġnutritional":22813,"205":22814,"ourage":22815,"Related":22816,"Come":22817,"ĠSAT":22818,"Ġgatherings":22819,"Ġsquads":22820,"Ġorganising":22821,"Ġministerial":22822,"Ġkilomet":22823,"ĠJump":22824,"ĠStrength":22825,"ĠFerr":22826,"Ġillustrated":22827,"ĠOber":22828,"Ġextrad":22829,"Ġlimitation":22830,"idis":22831,"ĠMonths":22832,"ifts":22833,"Ġmotives":22834,"Ġmaternal":22835,"Ġbait":22836,"Ġadversity":22837,"Twitter":22838,"ĠUni":22839,"Ġgrappling":22840,"Ġbowls":22841,"ĠHib":22842,"ĠCopenhagen":22843,"Ġsergeant":22844,"Ġintro":22845,"Ġscrambled":22846,"ĠExc":22847,"Ġshowcases":22848,"Ġplotting":22849,"Ġsym":22850,"ĠNah":22851,"berries":22852,"itching":22853,"conn":22854,"istle":22855,"ĠBeginning":22856,"asley":22857,"ĠMeadow":22858,"ĠCra":22859,"Ġsupremacist":22860,"Ġsweats":22861,"production":22862,"innon":22863,"ovo":22864,"Ġscept":22865,"Ġdrowning":22866,"ĠEh":22867,"Ġdecorations":22868,"Ġsympathetic":22869,"raction":22870,"Ġ195":22871,"ripp":22872,"ĠNotice":22873,"charging":22874,"ĠDIY":22875,"ĠJin":22876,"Ġskinny":22877,"Ġmaj":22878,"Ġwhisk":22879,"Ġcongreg":22880,"RAL":22881,"Ġvolley":22882,"Ġestablishments":22883,"Ġcite":22884,"Miss":22885,"Int":22886,"iola":22887,"ĠBare":22888,"KING":22889,"ools":22890,"private":22891,"Ġflaw":22892,"Ġwires":22893,"Ġideals":22894,"oub":22895,"Ġ\"'":22896,"ĠCompet":22897,"ĠStatements":22898,"ĠHDR":22899,"rm":22900,"Ġbegging":22901,"uffs":22902,"Ġdispatch":22903,"Ġskipped":22904,"Ġlabs":22905,"hawks":22906,"Ġexpl":22907,"Ġpatriotic":22908,"ussions":22909,"Ġportrayal":22910,"ĠBudapest":22911,"ĠCod":22912,"Ġextingu":22913,"smart":22914,"Ġburdens":22915,"ĠDrama":22916,"Ġaltitude":22917,"Ġpursuant":22918,"à¥":22919,"atari":22920,"cot":22921,"Ġhotline":22922,"ooters":22923,"ĠRolls":22924,"Ġjeopardy":22925,"oids":22926,"Ġpageant":22927,"149":22928,"Ġdistinguish":22929,"support":22930,"ĠHighlands":22931,"ĠErnst":22932,"ĠHole":22933,"pering":22934,"ĠHasan":22935,"Ġrece":22936,"Ġirregular":22937,"Ġdisturbed":22938,"Ġcoupon":22939,"ĠElijah":22940,"oise":22941,"Ġfriendships":22942,"girlfriend":22943,"Ġrampage":22944,"arers":22945,"Ġdispens":22946,"assion":22947,"Ġtentative":22948,"ĠExploration":22949,"fashioned":22950,"ĠInstit":22951,"Ġthemed":22952,"ĠKurdistan":22953,"ĠCAL":22954,"ĠSweeney":22955,"Ġransom":22956,"Ġstamps":22957,"ĠSchwe":22958,"ĠLucia":22959,"124":22960,"omore":22961,"Ġmotivate":22962,"ĠWorcester":22963,"wald":22964,"CAR":22965,"iken":22966,"andro":22967,"ffic":22968,"ĠRehab":22969,"Ġgrou":22970,"Ġcontrollers":22971,"ĠHai":22972,"nz":22973,"Ġartillery":22974,"ĠMish":22975,"Ġregistry":22976,"Ġfrontman":22977,"ĠCharg":22978,"orneys":22979,"ĠPRESS":22980,"Ġperceptions":22981,"ĠMcGee":22982,"AU":22983,"mg":22984,"Off":22985,"ĠNGOs":22986,"chemical":22987,"Ġbrun":22988,"ĠHav":22989,"Ġlace":22990,"Ġ202":22991,"Ġdefer":22992,"Ġinjected":22993,"Ġgluten":22994,"ĠRin":22995,"ĠAvalanche":22996,"Ġcorpor":22997,"ĠPamela":22998,"Ġfills":22999,"ĠReve":23000,"ĠMonument":23001,"Ġnationalists":23002,"ĠIQ":23003,"adden":23004,"ĠLoop":23005,"Ġ134":23006,"Reg":23007,"click":23008,"bush":23009,"ĠKub":23010,"ipes":23011,"Ġtoggle":23012,"ĠRae":23013,"Ġburgl":23014,"Ġholistic":23015,"ronics":23016,"Ġprominence":23017,"jack":23018,"Ġfinan":23019,"icates":23020,"Ġvel":23021,"important":23022,"Thursday":23023,"chet":23024,"Ġrefunds":23025,"ĠElder":23026,"ĠOwner":23027,"Ġtakeaway":23028,"Pe":23029,"ĠToro":23030,"Tim":23031,"fix":23032,"before":23033,"ĠMotorola":23034,"Ġlev":23035,"Term":23036,"ĠSne":23037,"Ġmisinformation":23038,"ĠSinai":23039,"Ġnitrogen":23040,"Ġ203":23041,"Ġescaping":23042,"Ġjunction":23043,"ĠSantana":23044,"ĠYemeni":23045,"Ġwhipped":23046,"ĠStephenson":23047,"Ġattire":23048,"ĠBard":23049,"atically":23050,"ĠFaul":23051,"ĠSym":23052,"resh":23053,"ĠMG":23054,"Sub":23055,"ĠCarmen":23056,"Ġig":23057,"ĠSanford":23058,"ĠYa":23059,"cycle":23060,"Ġencryption":23061,"ĠScal":23062,"ĠChest":23063,"ĠMadonna":23064,"agin":23065,"ĠDHS":23066,"ĠCed":23067,"YR":23068,"Ġtruce":23069,"ĠBike":23070,"Ġfoes":23071,"ĠSlovakia":23072,"adal":23073,"Rain":23074,"OPE":23075,"Ġlockdown":23076,"Ġunilateral":23077,"Ġoverseen":23078,"Ġblames":23079,"Ġbarrage":23080,"aan":23081,"uds":23082,"ĠRust":23083,"ĠHC":23084,"cox":23085,"ĠAllied":23086,"ĠJosé":23087,"pected":23088,"Ġunp":23089,"Ġsomeday":23090,"Ġdeductions":23091,"icial":23092,"ĠPRO":23093,"ĠIntern":23094,"Ġhemp":23095,"Ġkilograms":23096,"Ġnets":23097,"ĠBACK":23098,"early":23099,"outed":23100,"Ġrelegated":23101,"Ġ1958":23102,"ĠMustang":23103,"Ġgamble":23104,"Ġprostitution":23105,"ĠPapa":23106,"Ġinexpensive":23107,"GHz":23108,"Ġjerseys":23109,"Ġmisery":23110,"VIS":23111,"ĠRAW":23112,"Ġthri":23113,"Ġaffiliation":23114,"small":23115,"Ġflashed":23116,"Ġcoastline":23117,"Ġgard":23118,"Ġsv":23119,"Ġwaits":23120,"itton":23121,"London":23122,"Ġaccus":23123,"ĠCharge":23124,"Ġincub":23125,"Ġwanna":23126,"ĠAwareness":23127,"abies":23128,"ĠUh":23129,"Ġpersuaded":23130,"ĠThames":23131,"Ġcurated":23132,"Ī":23133,"Ġbrutally":23134,"Ġrooftop":23135,"Ġoy":23136,"Ġ1900":23137,"bery":23138,"Ġuphill":23139,"Ġinteracting":23140,"Ġchilly":23141,"ERE":23142,"Ġcapsule":23143,"ĠSaul":23144,"ocker":23145,"Ġdeserving":23146,"ĠBowen":23147,"ĠReaders":23148,"ĠWriters":23149,"Ġartifacts":23150,"ĠRanger":23151,"reau":23152,"Ġimperson":23153,"Ġhears":23154,"ĠMaher":23155,"neg":23156,"Ġmantra":23157,"Ġmull":23158,"Ġelders":23159,"ĠAmtrak":23160,"Ġspouses":23161,"ĠHak":23162,"Ġopenness":23163,"Ġprevailed":23164,"Ġfortnight":23165,"Pal":23166,"ride":23167,"Ġillustrate":23168,"dominated":23169,"trust":23170,"ī":23171,"ĠFemale":23172,"ĠSlim":23173,"Ġdesc":23174,"ĠKathryn":23175,"Ġdeepen":23176,"TAIN":23177,"eredith":23178,"Ġchanted":23179,"ĠHector":23180,"bread":23181,"ĠIsa":23182,"Ġvolcanic":23183,"Ġah":23184,"owners":23185,"aquin":23186,"Ġmelting":23187,"Ġpreschool":23188,"ocus":23189,"ĠMast":23190,"ĠMyr":23191,"Ġsuppress":23192,"Ġversatility":23193,"ĠNEC":23194,"Ġhoax":23195,"Ġmutually":23196,"ĠNeb":23197,"ĠWheel":23198,"kit":23199,"abl":23200,"again":23201,"ĠSonny":23202,"rift":23203,"Ġsweater":23204,"Ġinund":23205,"ĠTaco":23206,"ĠBout":23207,"Ġnonprofits":23208,"Ġmodify":23209,"Ġprofessionalism":23210,"ĠGould":23211,"ĠGuerrero":23212,"Ġterribly":23213,"ĠBenz":23214,"Ġcountered":23215,"Ġbean":23216,"ĠPhelps":23217,"Ġprowess":23218,"bc":23219,"Ġfeast":23220,"Ġ5000":23221,"Ġrevisit":23222,"Ġchin":23223,"agent":23224,"Ġtones":23225,"Ġextraction":23226,"ĠPosts":23227,"oin":23228,"Ġattain":23229,"Ġgardening":23230,"earned":23231,"ĠOtto":23232,"player":23233,"Ġscams":23234,"ĠHonolulu":23235,"ĠAppro":23236,"ĠHIGH":23237,"Ġdwell":23238,"Islam":23239,"leaders":23240,"Ġlegisl":23241,"expl":23242,"ĠChoi":23243,"Ġfrenzy":23244,"Ġcommercially":23245,"Ġlbs":23246,"Ġgateway":23247,"ĠAndersen":23248,"emia":23249,"lez":23250,"Ġresidences":23251,"office":23252,"ĠHelsinki":23253,"olia":23254,"Ġwolf":23255,"Ġstyling":23256,"ĠJunction":23257,"ĠPeyton":23258,"udo":23259,"ĠDorothy":23260,"Ġfreshly":23261,"ĠJulio":23262,"ĠSunset":23263,"ĠMadden":23264,"Ġissu":23265,"Ġsounding":23266,"sports":23267,"Ġmassively":23268,"ĠRahman":23269,"Ġpresided":23270,"Instead":23271,"Ġ136":23272,"ĠHowell":23273,"beit":23274,"Ġprosperous":23275,"Ġwrongly":23276,"ĠRaqqa":23277,"ĠCes":23278,"Ġbuddy":23279,"Ġchatting":23280,"Ġfencing":23281,"Ġtant":23282,"ocated":23283,"ALK":23284,"Ġsnapping":23285,"euro":23286,"Ryan":23287,"ĠRecogn":23288,"ucked":23289,"Ġpurported":23290,"ĠCann":23291,"Ġintimidating":23292,"Ġrulers":23293,"ĠMarse":23294,"Art":23295,"ĠAadhaar":23296,"Ġvows":23297,"Ġhunter":23298,"ourmet":23299,"ĠVarious":23300,"2009":23301,"anie":23302,"Ġcompassionate":23303,"ĠParking":23304,"Ġmalaria":23305,"Ġamnesty":23306,"Ġworsened":23307,"ĠTitan":23308,"Ġcrossings":23309,"drug":23310,"Ġaddicted":23311,"Ġremorse":23312,"ĠDestiny":23313,"Dear":23314,"Ġhur":23315,"Ġimplicated":23316,"Ġplayful":23317,"Ġripe":23318,"Ġsizable":23319,"Ġcrab":23320,"Ġliqu":23321,"Ġdrib":23322,"Ġcontraction":23323,"cro":23324,"ĠGus":23325,"Ġdoomed":23326,"Ġmog":23327,"ĠMonitor":23328,"Count":23329,"Ġsadd":23330,"Ġwrestler":23331,"Ġrestraints":23332,"Ġraging":23333,"185":23334,"Ġtapes":23335,"Ġmitigation":23336,"ocratic":23337,"Ġvib":23338,"ĠSnowden":23339,"aldo":23340,"Ġweights":23341,"Ġ1959":23342,"ucc":23343,"ĠCoc":23344,"Log":23345,"ĠStev":23346,"Ġdealership":23347,"Ġtrademarks":23348,"iru":23349,"Ġbeneficiary":23350,"Ġlegislator":23351,"Ġdeadlines":23352,"Ġcosmetics":23353,"ĠTammy":23354,"ĠCombined":23355,"Ġeducator":23356,"athon":23357,"Ġcombo":23358,"fu":23359,"appropriate":23360,"nington":23361,"ĠLiberties":23362,"missions":23363,"opard":23364,"ĠMondays":23365,"Ġfetch":23366,"Ġhers":23367,"jon":23368,"ukes":23369,"zek":23370,"Ġvetting":23371,"yet":23372,"Ġfacilitating":23373,"ĠStras":23374,"character":23375,"ĠHeads":23376,"Ġclim":23377,"ĠAlbuquerque":23378,"Ġbind":23379,"Ġconcluding":23380,"ĠBasically":23381,"rail":23382,"ĠTCU":23383,"ĠDepression":23384,"Ġhem":23385,"ĠHue":23386,"Ġpand":23387,"Ġscoreboard":23388,"Av":23389,"Ġidol":23390,"compl":23391,"Ġredesign":23392,"ĠJarrett":23393,"Ġfavoured":23394,"ĠINS":23395,"Ġpropelled":23396,"Ġevasion":23397,"Ġwidened":23398,"Ġwastewater":23399,"nard":23400,"responsive":23401,"Ġdemographics":23402,"engine":23403,"ĠBrewer":23404,"ĠBaxter":23405,"ront":23406,"ĠColon":23407,"Ġpromoter":23408,"Ġgenres":23409,"ovsky":23410,"build":23411,"urate":23412,"ĠCohn":23413,"design":23414,"Ġturbulent":23415,"Ġcurtain":23416,"310":23417,"ĠLamp":23418,"ĠBonds":23419,"church":23420,"Ġdeterrent":23421,"Ġdictatorship":23422,"acement":23423,"haul":23424,"Ġspir":23425,"Ġconceived":23426,"Ġstern":23427,"sit":23428,"Ġsingular":23429,"ĠYog":23430,"Ġconditional":23431,"Ġide":23432,"lund":23433,"Ġautop":23434,"ĠBEST":23435,"ĠJed":23436,"Ġrationale":23437,"Ġalarmed":23438,"Ġshovel":23439,"ĠProb":23440,"ĠMao":23441,"ĠBurgess":23442,"Ġ1953":23443,"above":23444,"ĠManson":23445,"Ġdismal":23446,"ĠFrankie":23447,"Ġtempted":23448,"Ġunderdog":23449,"ribing":23450,"ENCY":23451,"ĠDele":23452,"Las":23453,"places":23454,"Ġnotoriously":23455,"ĠAkin":23456,"Ġglut":23457,"Ġseamlessly":23458,"Ġrecess":23459,"written":23460,"ĠTJ":23461,"occ":23462,"ĠTerritory":23463,"ĠAIR":23464,"ĠDiagn":23465,"Ġvacancies":23466,"Ġcultivation":23467,"ĠAless":23468,"Ġrenamed":23469,"ĠMahmoud":23470,"bright":23471,"Ġvisibly":23472,"Ġnas":23473,"erred":23474,"ĠCarn":23475,"Ġtriggers":23476,"Ġpunishing":23477,"Ġluc":23478,"ĠBett":23479,"Ġbeam":23480,"ĠCheng":23481,"aina":23482,"Ġdetermines":23483,"ĠGerry":23484,"Ġshocks":23485,"Ġstainless":23486,"Ġdefects":23487,"ĠCinem":23488,"Ġtorrent":23489,"Ġresurgence":23490,"Ġcoral":23491,"Ġblitz":23492,"ĠGel":23493,"Ġstemmed":23494,"gur":23495,"Ġlymph":23496,"zzo":23497,"Ġspearheaded":23498,"Ġlicences":23499,"';":23500,"Ġarbitrary":23501,"ĠUzbek":23502,"Ġthief":23503,"reaching":23504,"Ġcand":23505,"ĠEA":23506,"ĠParaly":23507,"ĠEmerson":23508,"ĠSergey":23509,"ĠScher":23510,"ĠWr":23511,"rowing":23512,"Ġ3000":23513,"Ġmighty":23514,"elight":23515,"mAh":23516,"Ġcelebr":23517,"ĠConclusion":23518,"ĠCathy":23519,"Ġpolished":23520,"uddled":23521,"ewski":23522,"Ġfucking":23523,"Ġinterfering":23524,"Ġlandscapes":23525,"Ġfearful":23526,"ĠDetention":23527,"%).":23528,"ĠTT":23529,"Ġbleak":23530,"Ġindebted":23531,"Ġcheat":23532,"Ġconsolation":23533,"ĠPace":23534,"raine":23535,"Ġhonorary":23536,"420":23537,"Ġtechnician":23538,"ĠComprehensive":23539,"Ġfences":23540,"Ġwearable":23541,"ĠMarilyn":23542,"stru":23543,"Ġdrained":23544,"ĠGibraltar":23545,"lag":23546,"Ġdisorderly":23547,"Ġproclaimed":23548,"Ġcapacities":23549,"Ġretains":23550,"ĠVid":23551,"oshi":23552,"ĠEid":23553,"Ġanalytical":23554,"ominium":23555,"ĠExaminer":23556,"ĠNAACP":23557,"ocol":23558,"rev":23559,"ĠRim":23560,"ĠWoody":23561,"ĠMcKenna":23562,"ĠLennon":23563,"ĠEmploy":23564,"Fort":23565,"psy":23566,"Ġsphere":23567,"oday":23568,"ĠChick":23569,"ĠCompared":23570,"ĠIranians":23571,"ĠAccountability":23572,"itchie":23573,"ĠDickinson":23574,"Ġflock":23575,"Ġeclips":23576,"Ġnat":23577,"anke":23578,"ĠNeighborhood":23579,"Ġ141":23580,"Ġscarce":23581,"Ġcreations":23582,"lists":23583,"Ġuseless":23584,"Ġcriticisms":23585,"Ġruler":23586,"ĠHick":23587,"arya":23588,"worker":23589,"alam":23590,"Angelo":23591,"otle":23592,"Ġnewsletters":23593,"Ġerected":23594,"Ġzip":23595,"ĠBirthday":23596,"Ġdogged":23597,"Ġdanced":23598,"Ġconfession":23599,"Ġvomiting":23600,"ickers":23601,"Ġfox":23602,"Ġdeduct":23603,"Ġstresses":23604,"poll":23605,"ĠRadar":23606,"Ġengagements":23607,"Ġexaminer":23608,"Ġopportun":23609,"Ġlongevity":23610,"Ġbanana":23611,"carbon":23612,"uo":23613,"ĠLT":23614,"Ġsynagogue":23615,"Ġblackmail":23616,"INK":23617,"Ġfle":23618,"ĠGutierrez":23619,"Ġracket":23620,"Ġevenings":23621,"Ġdietary":23622,"ĠKok":23623,"Ġfaulty":23624,"Ġabandoning":23625,"ĠFlow":23626,"quest":23627,"estead":23628,"Ġbir":23629,"Ġsuicidal":23630,"ĠGift":23631,"ĠMissing":23632,"ĠMazda":23633,"ĠRib":23634,"ĠJourney":23635,"Ġconcede":23636,"Ġbrushed":23637,"Tw":23638,"andowski":23639,"ĠYun":23640,"Bride":23641,"zai":23642,"awatts":23643,"Ġcha":23644,"Ġspans":23645,"SF":23646,"Ġshells":23647,"planned":23648,"ĠGeographic":23649,"ĠVent":23650,"Ġfav":23651,"Ġinterrogation":23652,"Ġvaries":23653,"ĠPlat":23654,"operative":23655,"avid":23656,"Ġgreatness":23657,"ĠStrait":23658,"ĠSelling":23659,"Ġlawful":23660,"Ġlyn":23661,"Ġfunnel":23662,"Ġpundits":23663,"ties":23664,"Ġpneumonia":23665,"Ġcommencement":23666,"Ġbrisk":23667,"fires":23668,"ĠHTML":23669,"ĠSevent":23670,"Ġhistor":23671,"Ġ147":23672,"olls":23673,"Ġpian":23674,"Little":23675,"Ġcommercials":23676,"Ġdeteriorated":23677,"Ġbasin":23678,"Ġprohibition":23679,"Ġrestrictive":23680,"Ġtom":23681,"ĠPulse":23682,"vale":23683,"Ġmim":23684,"ĠLyons":23685,"ĠTrinidad":23686,"data":23687,"195":23688,"ĠPain":23689,"vor":23690,"ĠDirectorate":23691,"Wow":23692,"essential":23693,"Ġemerges":23694,"ĠDoors":23695,"Ġunde":23696,"Ġarchives":23697,"ĠIX":23698,"ĠAman":23699,"oric":23700,"ĠOper":23701,"nothing":23702,"Ġ142":23703,"igr":23704,"rust":23705,"ĠBYU":23706,"ĠBom":23707,"Ġrift":23708,"ĠAbs":23709,"ĠJenn":23710,"Ġrookies":23711,"hoe":23712,"Ġunderage":23713,"eden":23714,"Ġroasted":23715,"Ġenrol":23716,"Ġerased":23717,"Ġfreeway":23718,"Sil":23719,"Ġplanner":23720,"Ġconfess":23721,"ĠDual":23722,"ĠHeadquarters":23723,"bottom":23724,"Ġstatistic":23725,"ĠPush":23726,"Ġanim":23727,"ITT":23728,"Ġexecutions":23729,"Hub":23730,"ĠStick":23731,"Ġobscure":23732,"oven":23733,"Ġcoats":23734,"unc":23735,"Morning":23736,"Ġnit":23737,"mie":23738,"Ġcurves":23739,"gew":23740,"ĠAnniversary":23741,"members":23742,"ĠAbsolutely":23743,"Ġapt":23744,"otional":23745,"ĠGin":23746,"izo":23747,"Ġpretending":23748,"arak":23749,"Ġorganise":23750,"Ġroyalties":23751,"ĠCamden":23752,"Ġsausage":23753,"Inst":23754,"Ġchalk":23755,"ĠSurf":23756,"ĠSunrise":23757,"Ġmoder":23758,"aido":23759,"loving":23760,"lus":23761,"Ġoblig":23762,"Ġmotions":23763,"Ġclarification":23764,"ĠOM":23765,"Ġbishop":23766,"Ġexhibitions":23767,"ĠRifle":23768,"ĠPhot":23769,"ĠHM":23770,"ATIONAL":23771,"Ġwid":23772,"Ġreside":23773,"ĠPV":23774,"OOK":23775,"ĠTue":23776,"Ġ1200":23777,"Ġ1957":23778,"Ġespionage":23779,"ĠAPPLIC":23780,"Ġblasts":23781,"fter":23782,"Ġimmensely":23783,"ĠLots":23784,"Ġinflammatory":23785,"anging":23786,"Ġtumultuous":23787,"identified":23788,"Ġstead":23789,"ĠAch":23790,"Ãī":23791,"Ġbub":23792,"hler":23793,"olution":23794,"Ġshun":23795,"Ġnull":23796,"Ġunused":23797,"ĠObs":23798,"Ġinsol":23799,"ĠAttack":23800,"ertain":23801,"Ġdefiant":23802,"Through":23803,"ĠArmour":23804,"Ġsimulation":23805,"UCK":23806,"Ġinfluenza":23807,"Ġonset":23808,"Ġbored":23809,"Ġsouls":23810,"Ġreferees":23811,"Ġcollaborations":23812,"ĠLer":23813,"Ġcreepy":23814,"Ġanaly":23815,"ĠEffect":23816,"orting":23817,"Card":23818,"Ġdice":23819,"Ġharvesting":23820,"235":23821,"sty":23822,"ĠMcCartney":23823,"Ġsalute":23824,"UMP":23825,"Ġherb":23826,"ĠAbuse":23827,"ĠRamadan":23828,"Ġsuck":23829,"trained":23830,"ĠPhysical":23831,"iren":23832,"anches":23833,"erie":23834,"Ġhangs":23835,"Ġcataly":23836,"Ġintuitive":23837,"assi":23838,"Ġtechn":23839,"Ġjugg":23840,"Ġgameplay":23841,"Ġapolog":23842,"Ġfifteen":23843,"Ġgalleries":23844,"Ġoutlines":23845,"patient":23846,"ĠPotential":23847,"Ġethnicity":23848,"Ġharbour":23849,"Ġoverthrow":23850,"ĠLung":23851,"Ġwarehouses":23852,"ĠMonitoring":23853,"Ġmentors":23854,"Ġsized":23855,"Ġenvisioned":23856,"Ġgin":23857,"DT":23858,"Ġpropel":23859,"ĠKul":23860,"ference":23861,"estic":23862,"ĠLego":23863,"Ġdinners":23864,"ĠMoe":23865,"designed":23866,"ĠSusp":23867,"ĠBrick":23868,"qua":23869,"IDS":23870,"ĠBam":23871,"athe":23872,"Ġslices":23873,"Ġbottled":23874,"thy":23875,"producing":23876,"ĠTerror":23877,"professional":23878,"ĠKis":23879,"erto":23880,"ĠVehicles":23881,"Ġbeforehand":23882,"Ġdetrimental":23883,"weights":23884,"Ġallowances":23885,"Williams":23886,"ĠSyrians":23887,"ĠSto":23888,"Ġcozy":23889,"reditation":23890,"ensen":23891,"ĠSard":23892,"Ġroy":23893,"ooting":23894,"ĠReserv":23895,"ominated":23896,"emate":23897,"ĠTot":23898,"ĠCarnegie":23899,"ĠThib":23900,"ĠMarshal":23901,"Ġ152":23902,"Ġmayors":23903,"inery":23904,"ĠFiona":23905,"ĠCadillac":23906,"ivated":23907,"Ġeagerly":23908,"ĠOffensive":23909,"Ġastronaut":23910,"ĠVital":23911,"Ġcane":23912,"Ġquitting":23913,"ĠLone":23914,"Ġcensorship":23915,"ĠWelch":23916,"ĠUd":23917,"Ġmarquee":23918,"ĠDip":23919,"Ġwhereby":23920,"Ġtiger":23921,"gem":23922,"Ġconserv":23923,"Ġpresumed":23924,"ĠEntry":23925,"ffer":23926,"ĠProceed":23927,"Ġbrawl":23928,"ĠJaime":23929,"Ġecho":23930,"Ġadvancements":23931,"Ġtransitional":23932,"erick":23933,"Ġbully":23934,"anan":23935,"Ġreinvent":23936,"ĠLetters":23937,"Ġbricks":23938,"ĠSmy":23939,"Ġtowering":23940,"gging":23941,"299":23942,"orian":23943,"dimensional":23944,"ĠForty":23945,"ĠSinn":23946,"ushi":23947,"ĠSurveillance":23948,"enabled":23949,"ĠMous":23950,"ĠVive":23951,"Marcus":23952,"Ġvom":23953,"Ġcreek":23954,"Ġlime":23955,"Ġseismic":23956,"ĠFork":23957,"Ġembroiled":23958,"marks":23959,"Ġherald":23960,"ĠSonia":23961,"âĢ¦\"":23962,"wired":23963,"Ġobliged":23964,"ĠProjects":23965,"lde":23966,"ĠRiders":23967,"Ġovercoming":23968,"Mail":23969,"ĠLawn":23970,"ĠHawk":23971,"figure":23972,"ĠWritten":23973,"Ġens":23974,"Ġspacious":23975,"target":23976,"ĠRecep":23977,"ĠSAM":23978,"Ġentertained":23979,"Ġignited":23980,"ĠCENT":23981,"ogenic":23982,"Ġunatt":23983,"Ġexceeds":23984,"Ġ--------------------------------":23985,"Ġpillars":23986,"ĠBorders":23987,"ickey":23988,"Ġextinction":23989,"Ġviability":23990,"Ġtumors":23991,"ĠWilkinson":23992,"ĠKEY":23993,"Ġbins":23994,"ĠReported":23995,"Sm":23996,"ĠExclusive":23997,"ĠChilean":23998,"info":23999,"Ġwilderness":24000,"did":24001,"absolutely":24002,"pillar":24003,"Ġelites":24004,"ĠPreview":24005,"ixie":24006,"Mont":24007,"ribut":24008,"dream":24009,"Ġplanners":24010,"ĠSomerset":24011,"Ġenvis":24012,"ĠStall":24013,"Ġelevate":24014,"ographies":24015,"rama":24016,"Ha":24017,"Ġamidst":24018,"oho":24019,"Ġrejects":24020,"Jim":24021,"Ġmarginally":24022,"Ġusher":24023,"arez":24024,"ĠHawth":24025,"Ġsprink":24026,"ĠOffer":24027,"Ġanchored":24028,"ucking":24029,"ĠGarn":24030,"ĠConserv":24031,"Ġsocietal":24032,"Ġbrowsing":24033,"Ġbidder":24034,"burgh":24035,"ĠRunner":24036,"Ġtrendy":24037,"verts":24038,"imposed":24039,"ĠPatton":24040,"lements":24041,"Ġspicy":24042,"Ġswe":24043,"ĠStrike":24044,"Ġclam":24045,"ĠYankee":24046,"ĠKT":24047,"ĠGreenwood":24048,"ĠWays":24049,"Ġ2050":24050,"Ġattach":24051,"ĠShim":24052,"Ġmeltdown":24053,"Ġassemble":24054,"ĠUPDATE":24055,"Ġscout":24056,"Brown":24057,"ĠKobe":24058,"Ġpostpone":24059,"liness":24060,"allo":24061,"rief":24062,"ĠGerm":24063,"ĠFD":24064,"ĠReggie":24065,"ĠUnivers":24066,"ĠShepard":24067,"Ġcancell":24068,"ĠRomeo":24069,"ĠWarrior":24070,"ench":24071,"ifier":24072,"Ġprivileges":24073,"Ġsenses":24074,"Ġimpoverished":24075,"ĠPostal":24076,"encer":24077,"ĠConrad":24078,"Ġprinter":24079,"Ġinflicted":24080,"ĠGamble":24081,"ĠHeroes":24082,"132":24083,"Ġrevisions":24084,"Ġunsuccessfully":24085,"ĠHeisman":24086,"Ġstamped":24087,"inding":24088,"ĠLuna":24089,"Ġreinvest":24090,"ducers":24091,"ĠPassword":24092,"Leod":24093,"Ġcompounded":24094,"',\"":24095,"ogging":24096,"Ġprobing":24097,"ĠPBS":24098,"ĠMU":24099,"ĠWhenever":24100,"Ġsped":24101,"ĠCompetitive":24102,"isans":24103,"opa":24104,"Ġcleric":24105,"Ġvivid":24106,"à¸":24107,"126":24108,"Ġinconvenience":24109,"udi":24110,"Ġimmersive":24111,"Ġdiversion":24112,"Ġlogs":24113,"Ġspying":24114,"inct":24115,"Ġlitres":24116,"Ġmetallic":24117,"identally":24118,"FX":24119,"Ġloudly":24120,"Ġnursery":24121,"Ġcollectors":24122,"ĠKart":24123,"Ġescalate":24124,"Ġringing":24125,"Ġprocedural":24126,"Ġdisrupting":24127,"ĠEthiopian":24128,"ĠCFL":24129,"Ġillustrates":24130,"Ġperks":24131,"official":24132,"325":24133,"Ġmillennial":24134,"Ġbreadth":24135,"Ġmelted":24136,"Ġ850":24137,"ĠBake":24138,"donald":24139,"ĠGrac":24140,"Ġseeded":24141,"ĠDiscount":24142,"idates":24143,"Ġdrift":24144,"Ġcaptive":24145,"Ġseriousness":24146,"Ġrepercussions":24147,"Ġdisciplines":24148,"Ġthesis":24149,"Ġsleeve":24150,"ses":24151,"Monday":24152,"Ġthwart":24153,"ĠLic":24154,"Ġquadru":24155,"ĠPresbyterian":24156,"Ġreactors":24157,"ĠSuzanne":24158,"ewater":24159,"Ġlam":24160,"Ġbreastfeeding":24161,"Ġrats":24162,"ĠArtists":24163,"Ġdomestically":24164,"Ġdecom":24165,"ĠArms":24166,"basketball":24167,"Ġscrub":24168,"ĠTeddy":24169,"beh":24170,"ĠBetsy":24171,"ĠNursing":24172,"Ġdescriptions":24173,"127":24174,"gil":24175,"itional":24176,"Ġchampioned":24177,"ĠCalling":24178,"Ġrealization":24179,"ĠBuddy":24180,"hou":24181,"ĠDire":24182,"ĠHuff":24183,"Ġlipstick":24184,"Ray":24185,"Ġflare":24186,"belt":24187,"Ġbrightest":24188,"Ġmalfunction":24189,"ĠManor":24190,"Ġsaturated":24191,"rays":24192,"ĠDW":24193,"ixed":24194,"ĠSlovenia":24195,"seen":24196,"ĠCause":24197,"arios":24198,"ASE":24199,"Ġrend":24200,"ĠTBA":24201,"Ġlecturer":24202,"attering":24203,"Ġaffluent":24204,"CEO":24205,"Ġbreathtaking":24206,"ĠGiles":24207,"irth":24208,"ĠPhilips":24209,"Ġposture":24210,"ĠTSA":24211,"heit":24212,"Ġmenace":24213,"ricks":24214,"ĠAden":24215,"ĠReich":24216,"iggle":24217,"ĠShutterstock":24218,"Ġcourageous":24219,"edia":24220,"Staff":24221,"Ġdivert":24222,"ĠCir":24223,"Ġguessing":24224,"apers":24225,"ĠBritons":24226,"lé":24227,"Ġconvened":24228,"ĠSerbian":24229,"Ġricher":24230,"Ġcock":24231,"Ġdeposited":24232,"company":24233,"Ġdelic":24234,"sensitive":24235,"tank":24236,"ĠPatty":24237,"mia":24238,"onomous":24239,"cn":24240,"Ġclamp":24241,"ĠAcademic":24242,"Ġprosecuting":24243,"ĠTransparency":24244,"Ġdeflation":24245,"Ġdashboard":24246,"ĠDress":24247,"Ġlin":24248,"mu":24249,"ĠGoodell":24250,"Ġlav":24251,"ĠTwelve":24252,"Ġflavour":24253,"Ġfiercely":24254,"Ġbloom":24255,"ĠHaf":24256,"ĠGrad":24257,"LET":24258,"ĠSeeing":24259,"oxide":24260,"Ġmenus":24261,"char":24262,"adoes":24263,"combe":24264,"Street":24265,"ĠRidley":24266,"Ġdepicts":24267,"ĠPred":24268,"ÑĢ":24269,"British":24270,"Ġbumps":24271,"Ġlamp":24272,"ĠDesmond":24273,"ĠPB":24274,"Ġfrag":24275,"tin":24276,"ĠSharing":24277,"Ġdesperation":24278,"Ġcommuter":24279,"igrants":24280,"ĠShapiro":24281,"Ġkinda":24282,"Ġimpartial":24283,"ĠJewel":24284,"Ġcongratulations":24285,"Ġcompost":24286,"Ġadmiration":24287,"Ġpaycheck":24288,"ĠAnonymous":24289,"enger":24290,"Mer":24291,"ĠGospel":24292,"ĠEth":24293,"ĠMH":24294,"Ġfem":24295,"ĠTrial":24296,"Ġdepths":24297,"ĠApplied":24298,"Ġgrit":24299,"Ġerase":24300,"sid":24301,"comm":24302,"}":24303,"Ġretreated":24304,"Ġanalysed":24305,"ĠRegular":24306,"ĠPesh":24307,"ICAL":24308,"pei":24309,"ĠReilly":24310,"ĠTrib":24311,"Ġbooths":24312,"Ġdrank":24313,"Ġcoma":24314,"Ġharvested":24315,"ĠCHAR":24316,"Ġbutterfly":24317,"Ġsailed":24318,"ĠDrink":24319,"eping":24320,"ATCH":24321,"ĠLegends":24322,"Ġinsured":24323,"Ġwholes":24324,"ĠBis":24325,"ĠShea":24326,"ighter":24327,"Ġsnakes":24328,"ĠGunn":24329,"ĠPoss":24330,"Ġdispar":24331,"Ġbombshell":24332,"Ġscanning":24333,"340":24334,"choice":24335,"cool":24336,"\"âĢĶ":24337,"ĠTheo":24338,"rine":24339,"ĠJacques":24340,"Ġdisadvantaged":24341,"Ġparamount":24342,"igate":24343,"stat":24344,"anski":24345,"Ġoutsourcing":24346,"Ġpopulous":24347,"Ġbinge":24348,"ĠOrganic":24349,"urban":24350,"Ġyogurt":24351,"Ġretweet":24352,"osen":24353,"cially":24354,"215":24355,"Ġeditions":24356,"Ġburgeoning":24357,"efully":24358,"ĠThousand":24359,"Ġreplacements":24360,"ĠAmazing":24361,"rator":24362,"icy":24363,"Ġintensify":24364,"Sen":24365,"ĠQuincy":24366,"powers":24367,"ĠAur":24368,"ĠZion":24369,"stal":24370,"Ġpillar":24371,"ĠErit":24372,"ĠPerform":24373,"aston":24374,"Eric":24375,"Ġunh":24376,"IFF":24377,"950":24378,"ĠEngineer":24379,"ĠLands":24380,"Ġdubious":24381,"fy":24382,"ĠWI":24383,"ĠSv":24384,"ĠHendricks":24385,"ĠKod":24386,"Ġoutlining":24387,"ĠCorrespond":24388,"amus":24389,"worst":24390,"arter":24391,"coni":24392,"Ġhierarchy":24393,"ĠTHAT":24394,"Ġexce":24395,"Ġrailways":24396,"Ġmasked":24397,"lene":24398,"Ġoutset":24399,"Ġavalanche":24400,"Ġnicknamed":24401,"Ġ702":24402,"Lee":24403,"Ġ139":24404,"ĠSixth":24405,"365":24406,"nda":24407,"Ġaccountant":24408,"Ġobese":24409,"Ġgrape":24410,"Ġimpunity":24411,"ĠYorkers":24412,"Ġguardian":24413,"icity":24414,"Ġcentrist":24415,"Ġwaterways":24416,"ursed":24417,"Ġhopeless":24418,"header":24419,"Ġtack":24420,"Ġric":24421,"umn":24422,"Ġvalve":24423,"Ġtread":24424,"ĠCST":24425,"Ġhepatitis":24426,"ctor":24427,"ĠRED":24428,"Ġsolitary":24429,"NW":24430,"Ġceremonial":24431,"Ġfoe":24432,"Ġling":24433,"Jason":24434,"ĠLisbon":24435,"Ġ1955":24436,"ĠHeller":24437,"Ġkin":24438,"essen":24439,"Ġturbines":24440,"shi":24441,"Ġlodge":24442,"Ġveterinary":24443,"ĠBoll":24444,"ĠConfederation":24445,"ĠJournalists":24446,"Ġtug":24447,"ĠStarr":24448,"Ġpiles":24449,"Way":24450,"adel":24451,"orean":24452,"Ġoft":24453,"Ġshortcomings":24454,"ĠSheila":24455,"Ġbackbone":24456,"III":24457,"ĠDarwin":24458,"ĠTunis":24459,"Ġsuspicions":24460,"Ġdisagreements":24461,"Ġ247":24462,"illery":24463,"'\"":24464,"Ġsegregation":24465,"ohl":24466,"Ġinstincts":24467,"ĠPoo":24468,"nih":24469,"parency":24470,"uddy":24471,"esting":24472,"asses":24473,"ĠIntroduction":24474,"ĠSirius":24475,"Local":24476,"orous":24477,"Ġrehearsal":24478,"Ġdemol":24479,"Ġtraffickers":24480,"Ġupsetting":24481,"Ġheir":24482,"death":24483,"ĠMoments":24484,"Los":24485,"Ġatmospheric":24486,"aints":24487,"ĠDianne":24488,"Ġlikewise":24489,"ĠMing":24490,"auga":24491,"Ġfirsthand":24492,"Ġnarratives":24493,"ĠAstron":24494,"ĠExtreme":24495,"Ġhorns":24496,"ĠSana":24497,"Ġrecapt":24498,"ĠMist":24499,"ĠRandolph":24500,"connect":24501,"Ġindecent":24502,"Ġforty":24503,"Ġjihadists":24504,"azes":24505,"Ġdread":24506,"Ġgrapes":24507,"Ġremoves":24508,"Ġscreamed":24509,"ĠCrus":24510,"ikers":24511,"Ġsnapshot":24512,"ĠCalls":24513,"Cons":24514,"Ġlettuce":24515,"ĠPig":24516,"urable":24517,"jured":24518,"ILY":24519,"ĠJessie":24520,".).":24521,"Pay":24522,"Tra":24523,"----------------":24524,"ĠUnits":24525,"ĠPlayboy":24526,"Ġarthritis":24527,"Ġafforded":24528,"insk":24529,"ĠFake":24530,"ĠLies":24531,"ĠBaltic":24532,"oyal":24533,"ĠVest":24534,"Ġrusher":24535,"Ġincorporates":24536,"ĠMM":24537,"ĠDru":24538,"ĠWare":24539,"ĠSammy":24540,"ĠGob":24541,"ĠRuk":24542,"Ġ146":24543,"ĠCrowd":24544,"Ġduel":24545,"irts":24546,"Ġsourcing":24547,"hp":24548,"ĠJava":24549,"bred":24550,"ĠRefer":24551,"Ġuninsured":24552,"Ġslope":24553,"256":24554,"Ġregulating":24555,"Ġfundra":24556,"Ġinserted":24557,"ĠNickel":24558,"ĠConsumption":24559,"ĠRomo":24560,"Atlantic":24561,"Ġenclave":24562,"Ġpegged":24563,"Ġdirects":24564,"mbudsman":24565,"ĠDES":24566,"Ob":24567,"Ġlimbs":24568,"Ġbury":24569,"ILA":24570,"Ġstew":24571,"Ġbreeze":24572,"Ġabrupt":24573,"ĠGott":24574,"ĠClaude":24575,"Ġgenetically":24576,"Ġrigid":24577,"ĠDudley":24578,"ĠNer":24579,"registered":24580,"Ġentrenched":24581,"Ġextortion":24582,"ĠNurs":24583,"Ġcontingency":24584,"etter":24585,"Ġrejo":24586,"Ġprotagonist":24587,"Ġcounselling":24588,"ĠVit":24589,"aware":24590,"ĠMonsanto":24591,"GG":24592,"Ġincarcerated":24593,"Ġabduction":24594,"Ġreferencing":24595,"Germany":24596,"uates":24597,"reck":24598,"Ġtram":24599,"Ġchron":24600,"Ġmish":24601,"ĠVes":24602,"ĠTire":24603,"Ġvandal":24604,"ĠCrazy":24605,"ĠLifetime":24606,"ĠSpectrum":24607,"celer":24608,"Ġmotto":24609,"hang":24610,"Ġblade":24611,"gel":24612,"Ġbiography":24613,"Ġallegiance":24614,"hod":24615,"hap":24616,"ptic":24617,"acle":24618,"ĠBlade":24619,"ĠBoh":24620,"Ġ149":24621,"Ġchang":24622,"Ġcanned":24623,"Ġfacilitated":24624,"actor":24625,"iologist":24626,"Ġrebuilt":24627,"Ġawake":24628,"Ġmayoral":24629,"ĠEuros":24630,"Ġdangerously":24631,"MK":24632,"Ġreplica":24633,"Ġcoinc":24634,"blog":24635,"ĠEra":24636,"Ġrelinqu":24637,"quite":24638,"ondon":24639,"rosso":24640,"tun":24641,"Ġtouchscreen":24642,"Ġpops":24643,"ousing":24644,"efficient":24645,"Ġ148":24646,"Ġconced":24647,"although":24648,"Ġ1956":24649,"Ġmortar":24650,"ĠCave":24651,"ĠJung":24652,"urer":24653,"Ġillusion":24654,"ĠBerman":24655,"intend":24656,"Ġcoping":24657,"Dem":24658,"tion":24659,"estation":24660,"ĠSounds":24661,"Ġnavigating":24662,"Ġsperm":24663,"Ġreligions":24664,"Ġfol":24665,"Ġheroic":24666,"FD":24667,"Ġhesitant":24668,"asure":24669,"Ġredeem":24670,"Adam":24671,"Ġfireplace":24672,"vertis":24673,"ĠSung":24674,"290":24675,"iland":24676,"ĠUpdates":24677,"OTUS":24678,"ĠPTSD":24679,"Ġhelmets":24680,"\"?":24681,"Ġslashing":24682,"Ġscouts":24683,"Ġspelling":24684,"ĠInitial":24685,"draw":24686,"Ġchallengers":24687,"Ġsupremacists":24688,"Ġpilgrims":24689,"Ġasc":24690,"ĠFill":24691,"ĠPau":24692,"Ġjewel":24693,"ĠMalt":24694,"icip":24695,"Ġinhabitants":24696,"Ġmetre":24697,"ahar":24698,"Comp":24699,"atches":24700,"inv":24701,"Ġcyclist":24702,"ĠQC":24703,"Ġmanually":24704,"ĠAnchorage":24705,"Ġdiscarded":24706,"Ġconsolid":24707,"Ġnavig":24708,"ĠAnimals":24709,"ĠPole":24710,"esson":24711,"Ġ1954":24712,"Ġsorted":24713,"Ġmadness":24714,"ĠBrigade":24715,"ĠGenesis":24716,"Ġdismissing":24717,"ĠPanasonic":24718,"Ġdizz":24719,"ĠEducational":24720,"ĠKO":24721,"ĠPill":24722,"ĠGIF":24723,"Ġbol":24724,"Ġwards":24725,"Ġcontroversies":24726,"Chinese":24727,"Ġantics":24728,"Ġreliant":24729,"ĠMoff":24730,"Ġethanol":24731,"Ġtorch":24732,"rights":24733,"ĠHabit":24734,"arton":24735,"rera":24736,"ĠSasha":24737,"abella":24738,"Ġproliferation":24739,"Ġsincerely":24740,"communication":24741,"ĠNay":24742,"ĠChattanooga":24743,"ounces":24744,"ĠNXT":24745,"ĠEmir":24746,"Ġmanipulated":24747,"Ġharassing":24748,"wat":24749,"Ġbouts":24750,"Book":24751,"Ġhovering":24752,"ĠScan":24753,"ship":24754,"ĠAngola":24755,"ĠLC":24756,"Ġruins":24757,"Ġsexist":24758,"zar":24759,"Ġpledging":24760,"ober":24761,"Ġembold":24762,"Ġobjection":24763,"Ġboasting":24764,"MIN":24765,"Ġherbs":24766,"Ġgears":24767,"ĠIc":24768,"stre":24769,"him":24770,"Ġhomicides":24771,"cki":24772,"castle":24773,"counter":24774,"ĠCAS":24775,"ĠReasons":24776,"ĠDeclaration":24777,"Ġsimplify":24778,"Ġfared":24779,"Ġescort":24780,"Ġkidn":24781,"ĠHamm":24782,"Ġnailed":24783,"Ġaccommodations":24784,"Ġmodifications":24785,"rible":24786,"Ġwool":24787,"EDIT":24788,"2010":24789,"Ġauthentication":24790,"Ġgoat":24791,"hom":24792,"Ġfederally":24793,"ĠRath":24794,"Ġspiked":24795,"Ġmisrepresent":24796,"Ġavenue":24797,"Ġbroadcasts":24798,"ĠEstonia":24799,"ennes":24800,"ĠMare":24801,"ption":24802,"ĠKag":24803,"Ġcircumstance":24804,"orrow":24805,"isons":24806,"ĠCollabor":24807,"Ġstroll":24808,"ĠCPS":24809,"soft":24810,"iral":24811,"apo":24812,"usky":24813,"poke":24814,"Ġwoo":24815,"ĠElena":24816,"ĠLastly":24817,"Ġlinemen":24818,"Canadian":24819,"ĠAnyway":24820,"Ġsubstantive":24821,"ĠCurt":24822,"Ġard":24823,"ĠYosh":24824,"ĠBuchanan":24825,"Ġrevolving":24826,"Ġspecials":24827,"Ġshrine":24828,"Ġlumber":24829,"Ġorchestrated":24830,"kie":24831,"azy":24832,"Ġexpiration":24833,"ĠDaryl":24834,"ĠPatri":24835,"better":24836,"2020":24837,"ĠFav":24838,"ĠOP":24839,"OTT":24840,"Ġflush":24841,"ĠSikh":24842,"Ġecosystems":24843,"ĠBET":24844,"eared":24845,"audio":24846,"ĠFahrenheit":24847,"police":24848,"Ġincarceration":24849,"Ġerupt":24850,"ĠDamien":24851,"ĠHague":24852,"ulz":24853,"ĠAgents":24854,"ĠBanner":24855,"Ġconductor":24856,"ĠAjax":24857,"arson":24858,"Ġrests":24859,"Ġeurozone":24860,"Ġfelon":24861,"Ġcurator":24862,"morning":24863,"Ġevidenced":24864,"ĠNeh":24865,"Ġmattress":24866,"Ġtast":24867,"Ġfueling":24868,"ĠOccup":24869,"Ġbake":24870,"ĠZac":24871,"meaning":24872,"Ill":24873,"ĠHau":24874,"ĠLaden":24875,"Ġbald":24876,"Mary":24877,"oky":24878,"atri":24879,"Ġtracker":24880,"OTA":24881,"catching":24882,"ĠUnderground":24883,"ĠHuffPost":24884,"ĠAtkins":24885,"oglu":24886,"Ġauthorised":24887,"Ġroutines":24888,"ĠHof":24889,"veland":24890,"Ġlangu":24891,"Ġprot":24892,"ĠHyd":24893,"integ":24894,"Ġbravery":24895,"Ġviolin":24896,"Ġdelightful":24897,"Ġticks":24898,"iton":24899,"Ġreap":24900,"Ġoversized":24901,"ĠPitch":24902,"Ġprized":24903,"Ġfusion":24904,"fact":24905,"acting":24906,"Ġfullback":24907,"Ġpolite":24908,"Ġswear":24909,"Ġconfiscated":24910,"ĠStud":24911,"Ġfielded":24912,"rito":24913,"covered":24914,"financial":24915,"bill":24916,"HK":24917,"OTOS":24918,"loaded":24919,"Ġmarble":24920,"ĠDiplom":24921,".âĢĶ":24922,"Ġeats":24923,"Ġbackfield":24924,"Ġtimeframe":24925,"Ġvegetarian":24926,"Ġswaps":24927,"ĠMines":24928,"igor":24929,"ĠLenn":24930,"ĠDP":24931,"ordered":24932,"ĠShark":24933,"Ġquant":24934,"erence":24935,"Ġashes":24936,"ĠBuckley":24937,"ophobia":24938,"Ġwarranted":24939,"Rose":24940,"Ġunreasonable":24941,"ĠJav":24942,"Ġpalette":24943,"Ġjoints":24944,"Ġadvent":24945,"Ġnoteworthy":24946,"ĠNicol":24947,"ĠChristensen":24948,"Ġplummeted":24949,"ayers":24950,"Ġdefends":24951,"Ġcontended":24952,"ĠCongratulations":24953,"kish":24954,"ĠHannity":24955,"Ġgroundwater":24956,"ĠKramer":24957,"Ġerect":24958,"Ġappet":24959,"ĠKardash":24960,"Ġexacerbated":24961,"Ġexplanations":24962,"vious":24963,"eport":24964,"---":24965,"icism":24966,"ĠNatasha":24967,"ĠGeoffrey":24968,"estro":24969,"Article":24970,"Ġincidence":24971,"Ġprovoked":24972,"elf":24973,"Ġinsistence":24974,"ĠOUR":24975,"Ġfertilizer":24976,"Ġstickers":24977,"ĠGators":24978,"ĠLanding":24979,"ĠDON":24980,"sta":24981,"ĠRobbins":24982,"Ġpixels":24983,"ĠHoy":24984,"imated":24985,"ĠÃī":24986,"â":24987,"Ġsimpl":24988,"Other":24989,"245":24990,"Ġforcibly":24991,"'.\"":24992,"Ġsmashing":24993,"Ġmosquitoes":24994,"Ġpaints":24995,"Ġdebating":24996,"enty":24997,"ĠIB":24998,"leaf":24999,"ĠDah":25000,"Ġreferral":25001,"pired":25002,"Ġbrunch":25003,"gie":25004,"Ġvict":25005,"ribute":25006,"Ġbloggers":25007,"Ġgum":25008,"ĠAdmiral":25009,"France":25010,"ĠPK":25011,"ĠSaturn":25012,"Ġinflated":25013,"WAR":25014,"Ġscenic":25015,"usal":25016,"their":25017,"Ġcontends":25018,"Ġpathways":25019,"inis":25020,"Ġawarding":25021,"Ġmisled":25022,"Ġeternal":25023,"Ġexaminations":25024,"Ġpoker":25025,"Ġsafest":25026,"Ġchildcare":25027,"aday":25028,"Ġpreceding":25029,"ĠCollective":25030,"Ġrespectable":25031,"ographical":25032,"Ġoak":25033,"00000":25034,"ĠCorridor":25035,"oran":25036,"133":25037,"Ġmushrooms":25038,"gaard":25039,"ĠOmega":25040,"ĠNaturally":25041,"anim":25042,"Ġcaptains":25043,"Ġtang":25044,"Ġlobbyists":25045,"ĠSug":25046,"Ġsucc":25047,"249":25048,"ENG":25049,"134":25050,"Ġsolic":25051,"ĠAdded":25052,"ĠSuicide":25053,"ĠFULL":25054,"ĠStrauss":25055,"ĠDiesel":25056,"Ġtempting":25057,"acist":25058,"ĠDelivery":25059,"Ġquiz":25060,"ĠPARK":25061,"Ġcollisions":25062,"Ġrestrained":25063,"purpose":25064,"ĠChanges":25065,"Ġabsentee":25066,"Ġprobes":25067,"hib":25068,"Ġcul":25069,"Ġpetty":25070,"Ġnecess":25071,"Ġcues":25072,"OME":25073,"Ġinadvertently":25074,"urity":25075,"ĠStuff":25076,"FG":25077,"Ġwrestlers":25078,"Ġpaste":25079,"ĠRoku":25080,"Ġcardboard":25081,"aires":25082,"Ġvariables":25083,"ĠSaras":25084,"ĠFif":25085,"Ġinvests":25086,"ĠDiscover":25087,"ĠFix":25088,"Thomas":25089,"ĠLunch":25090,"lv":25091,"camera":25092,"Step":25093,"Ġresumes":25094,"ĠSacred":25095,"ĠShooting":25096,"Ġnoble":25097,"Ġslopes":25098,"Ġont":25099,"Ġtwists":25100,"Very":25101,"Ġbigotry":25102,"ĠTib":25103,"Ġmos":25104,"Ġwarrior":25105,"Ġbroadcasters":25106,"Ġubiquitous":25107,"ameda":25108,"Ġchess":25109,"Special":25110,"Ġconver":25111,"Ġdeleg":25112,"endant":25113,"Ġfoil":25114,"Ġlush":25115,"Ġtaxed":25116,"Mag":25117,"ahs":25118,"Ġtablespoons":25119,"scription":25120,"clamation":25121,"ĠCertain":25122,"ĠDiversity":25123,"Ġhairst":25124,"ĠBrewery":25125,"Ġshedding":25126,"Cla":25127,"Ġpenis":25128,"ĠMurder":25129,"Park":25130,"uner":25131,"iments":25132,"ĠOVER":25133,"hus":25134,"Ġtabloid":25135,"Chart":25136,"Ġvouchers":25137,"ĠCoord":25138,"Ġmethane":25139,"ĠFisheries":25140,"ĠKham":25141,"includes":25142,"ĠSuperman":25143,"ensed":25144,"isure":25145,"Amazon":25146,"Ġvacated":25147,"heet":25148,"Ġroast":25149,"Ġlegalize":25150,"ĠTut":25151,"Ġsignage":25152,"init":25153,"Ġthefts":25154,"202":25155,"Ġstatic":25156,"Ġchants":25157,"Bob":25158,"Ġdiscretionary":25159,"Ġendurance":25160,"Ġcollegiate":25161,"Ġcorridors":25162,"Ġslack":25163,"ĠLash":25164,"Az":25165,"Series":25166,"Ġnonpartisan":25167,"ĠMcGill":25168,"Ġuneven":25169,"ulsive":25170,"eu":25171,"Ġpil":25172,"Ġfisheries":25173,"Ġonslaught":25174,"fiction":25175,"holding":25176,"Ġcheated":25177,"Ġtraumat":25178,"lasting":25179,"Ġmultitude":25180,"ĠThr":25181,"ĠBreast":25182,"Ġ1600":25183,"ĠMatth":25184,"Ġdiminish":25185,"ĠFTC":25186,"Ġgram":25187,"ĠResident":25188,"Ġfading":25189,"Ġmarginalized":25190,"ĠLite":25191,"ĠCarlton":25192,"Ġerad":25193,"Welcome":25194,"ĠFaw":25195,"iddy":25196,"Ġparticip":25197,"Ġcz":25198,"Ġtexted":25199,"Ġsuites":25200,"ĠForever":25201,"Ġrendition":25202,"rait":25203,"ĠPrague":25204,"Ġsponsoring":25205,"Ġcompos":25206,"ĠBeacon":25207,"144":25208,"Ġpupil":25209,"Ġintricate":25210,"Ġathleticism":25211,"Ġoptimization":25212,"Ġloot":25213,"polit":25214,"ĠOtt":25215,"Whatever":25216,"uno":25217,"ĠConstable":25218,"esville":25219,"Ġlookout":25220,"ĠAircraft":25221,"Ġspo":25222,"Ġcorrobor":25223,"Ġhiatus":25224,"ĠKnowing":25225,"ĠHamp":25226,"Ġspe":25227,"Ġstoring":25228,"Ġshakes":25229,"uran":25230,"Ġsickness":25231,"Ġliber":25232,"ĠAdministrative":25233,"Ġpleasing":25234,"ĠEqual":25235,"ĠConversation":25236,"Ġalgae":25237,"Ġlobbyist":25238,"ĠHelena":25239,"ptions":25240,"Ġfaire":25241,"ĠGone":25242,"ĠWiggins":25243,"Robert":25244,"Ġlistens":25245,"ĠDaisy":25246,"Ġsticky":25247,"sale":25248,"ĠMarijuana":25249,"ĠSSD":25250,"ĠTool":25251,"once":25252,"ĠHarmon":25253,"mobile":25254,"Ġdetain":25255,"Money":25256,"Ġflawless":25257,"forced":25258,"Ġguru":25259,"Ġairspace":25260,"ĠArchie":25261,"ĠGender":25262,"ĠMeat":25263,"abilities":25264,"ĠBD":25265,"Open":25266,"Ġoutsider":25267,"issue":25268,"Ġlearns":25269,"natural":25270,"Ġvinegar":25271,"ĠSUB":25272,"ĠRecon":25273,"blers":25274,"Ġsniff":25275,"Ġsuppression":25276,"Ġsaf":25277,"urger":25278,"Ġbunker":25279,"asaki":25280,"ĠSpartan":25281,"ĠTok":25282,"Ġrav":25283,"Ġfoc":25284,"Sean":25285,"etric":25286,"Ġballpark":25287,"ĠHerb":25288,"ĠBM":25289,"ĠPublishing":25290,"Ġroadmap":25291,"pered":25292,"Ġpredator":25293,"ĠBlockchain":25294,"Ġvalidity":25295,"ĠGlou":25296,"ĠYamaha":25297,"Ġadop":25298,"Ġswamp":25299,"Ġcomplied":25300,"Ky":25301,"Greg":25302,"casts":25303,"john":25304,"ĠBosnia":25305,"Ġcinematic":25306,"ĠTavern":25307,"Ġfrustrations":25308,"eryl":25309,"Ġfairy":25310,"UNCH":25311,"ĠTus":25312,"Corp":25313,"ĠNug":25314,"closed":25315,"Ġexercised":25316,"urden":25317,"Ġdigitally":25318,"137":25319,"ĠVictims":25320,"Ġreluctance":25321,"ELL":25322,"ĠTribe":25323,"chall":25324,"Ġwhiskey":25325,"ogl":25326,"Ġmater":25327,"ĠBac":25328,"Ġapartheid":25329,"ĠMBA":25330,"mot":25331,"ĠIre":25332,"®,":25333,"ĠChic":25334,"Ġtimed":25335,"ĠDome":25336,"efer":25337,"Ġobserver":25338,"unky":25339,"ĠKant":25340,"Ġundrafted":25341,"Ġsimplicity":25342,"onds":25343,"Ġstoked":25344,"Ġ1949":25345,"Ġransomware":25346,"ĠPow":25347,"ĠAngelo":25348,"ĠAmbrose":25349,"adjusted":25350,"Guard":25351,"138":25352,"ĠKaplan":25353,"stri":25354,"Ġcries":25355,"NF":25356,"atro":25357,"Ġavocado":25358,"illian":25359,"Ġsculptures":25360,"Ġelevation":25361,"Ġinspires":25362,"Ġgenerals":25363,"arb":25364,"chell":25365,"ĠJournalism":25366,"ĠHybrid":25367,"ĠCaller":25368,"vec":25369,"Lu":25370,"Ġresemble":25371,"bys":25372,"erving":25373,"antz":25374,"Ġwiden":25375,"vised":25376,"Ev":25377,"Ġdiagn":25378,"ĠMakes":25379,"Ġcer":25380,"ĠPats":25381,"single":25382,"sche":25383,"struct":25384,"Ġdissolved":25385,"Ġtimeout":25386,"Ġenhancement":25387,"CF":25388,"Ġindust":25389,"ĠDed":25390,"ĠZo":25391,"CB":25392,"Ġpesticides":25393,"ĠRubin":25394,"George":25395,"opal":25396,"Ġmotel":25397,"critical":25398,"Ġcollapsing":25399,"ĠShal":25400,"tex":25401,"Ġcomplementary":25402,"Ġoust":25403,"ĠFlu":25404,"Ġexporting":25405,"Ġdifferential":25406,"north":25407,"ĠFG":25408,"Ġspoon":25409,"sha":25410,"Ġdismantle":25411,"elta":25412,"Ġjar":25413,"space":25414,"Smart":25415,"mere":25416,"Ð":25417,"ĠGillespie":25418,"Lo":25419,"ĠMead":25420,"capacity":25421,"ĠIssue":25422,"050":25423,"ĠVall":25424,"Ġdisgr":25425,"Ġmeme":25426,"Ġpard":25427,"Ġcompensated":25428,"ĠKet":25429,"major":25430,"ĠBren":25431,"Ġheed":25432,"131":25433,"Ġcm":25434,"Ġdazzling":25435,"ĠCheese":25436,"Ġmonumental":25437,"Ġyielding":25438,"Read":25439,"Ġgrinding":25440,"Ang":25441,"Ġdefiance":25442,"Ġintimidated":25443,"Ġ310":25444,"Ġoutsiders":25445,"houn":25446,"Ma":25447,"ĸ":25448,"ĠForget":25449,"ĠSans":25450,"Ġunfolding":25451,"ĠSap":25452,"ĠLak":25453,"Ġsectarian":25454,"ĠDaddy":25455,"oxy":25456,"hitting":25457,"Ġdetectors":25458,"ĠRee":25459,"Ġbroaden":25460,"Ġslaying":25461,"Ġsuspending":25462,"Ġinvestig":25463,"Tuesday":25464,"Ġantibiotic":25465,"ĠShiite":25466,"igi":25467,"ĠExternal":25468,"ĠPhotographer":25469,"Ġerratic":25470,"NJ":25471,"ĠDock":25472,"Ġoutweigh":25473,"rants":25474,"Ġlobster":25475,"Ġreactor":25476,"Ġunrealistic":25477,"ĠAudrey":25478,"ĠYor":25479,"Anyone":25480,"Ġfraught":25481,"е":25482,"ĠWester":25483,"fc":25484,"ĠDunham":25485,"ĠLug":25486,"allow":25487,"139":25488,"Ġparity":25489,"Ġhorizontal":25490,"ijuana":25491,"Ġcivilization":25492,"ĠGins":25493,"Ġsmokers":25494,"ĠDiabetes":25495,"Five":25496,"ĠDG":25497,"Ġunderscores":25498,"Ġelabor":25499,"ĠLub":25500,"ĠDevil":25501,"Ġ154":25502,"ĠGuarant":25503,"ĠPandora":25504,"Ġexcav":25505,"Ġaccuser":25506,"Ġrevolt":25507,"Ġinstructors":25508,"Ġire":25509,"ographic":25510,"ĠCLE":25511,"Ġexpedition":25512,"ould":25513,"Ġstriving":25514,"south":25515,"onis":25516,"ĠSwed":25517,"MY":25518,"ĠLevin":25519,"Ġcarp":25520,"ĠArchitects":25521,"Ġ{":25522,"Ġcovert":25523,"Ġcooled":25524,"ĠStaten":25525,"Ġspecializing":25526,"ĠHazel":25527,"Ġlen":25528,"ighty":25529,"Ġbrilliantly":25530,"Phil":25531,"Ġlament":25532,"Australia":25533,"203":25534,"Ġticking":25535,"Ġadjud":25536,"Ġroommate":25537,"ĠSheet":25538,"capital":25539,"167":25540,"Ġendeavor":25541,"Ġaver":25542,"Ġdues":25543,"ĠCycl":25544,"oried":25545,"Va":25546,"loading":25547,"Ġpremie":25548,"Ġregimes":25549,"ĠAly":25550,"Ġperennial":25551,"Ġconsoles":25552,"Ġironic":25553,"ichael":25554,"Ġvigorously":25555,"Ġtransmit":25556,"gary":25557,"eking":25558,"Ġjails":25559,"ĠEpiscopal":25560,"eddy":25561,"Ġidle":25562,"Ġsafeguards":25563,"Ġdwindling":25564,"NOR":25565,"torn":25566,"ĠEvangel":25567,"ĠPlastic":25568,"ĠTerm":25569,"Ġforwarded":25570,"avage":25571,"Ġrefrigerator":25572,"arna":25573,"ĠGuinness":25574,"ĠCandy":25575,"Ġbotched":25576,"seller":25577,"Ġpul":25578,"grades":25579,"oshenko":25580,"earth":25581,"nette":25582,"Ġtraps":25583,"Ġtarn":25584,"Ġmilitar":25585,"ĠAriel":25586,"Ġtubes":25587,"ulo":25588,"Water":25589,"edin":25590,"Ġmarvel":25591,"chenko":25592,"ĠElk":25593,"spect":25594,"coe":25595,"ĠIllustrated":25596,"Ġruthless":25597,"etermined":25598,"Ġdys":25599,"Ġbreaching":25600,"gee":25601,"Nick":25602,"Ġcruiser":25603,"Ġciv":25604,"Ġdou":25605,"Ġ;":25606,"deb":25607,"ĠAsheville":25608,"Ġbiting":25609,"Ġyo":25610,"Courtesy":25611,"Ġroses":25612,"ĠConsequently":25613,"Ġrevis":25614,"Ġconfinement":25615,"next":25616,"produced":25617,"Ġmoratorium":25618,"Ġkne":25619,"eties":25620,"Ġplethora":25621,"Ġceleb":25622,"FIN":25623,"Ġdepartures":25624,"ĠWynne":25625,"abilia":25626,"ĠCourts":25627,"olis":25628,"Ġcereal":25629,"Ġblended":25630,"333":25631,"ĠLun":25632,"Ġrepe":25633,"Ġmathematics":25634,"Ġpharmacies":25635,"Center":25636,"Ġwhist":25637,"pine":25638,"Ġperm":25639,"Ġcustomary":25640,"Ġhormones":25641,"Ġcleansing":25642,"Ġconfidentiality":25643,"Ġmascot":25644,"Ġslippery":25645,"Ġmediation":25646,"Ġpodcasts":25647,"Ġcoating":25648,"Ġconveyed":25649,"Ġgir":25650,"ĠNurse":25651,"DM":25652,"Ġlured":25653,"orted":25654,"Ġolig":25655,"ritz":25656,"ĠINF":25657,"Ġtirelessly":25658,"Ġdoorstep":25659,"Ġtomb":25660,"Ġwithholding":25661,"irling":25662,"Ġhog":25663,"Ġ156":25664,"Ġgau":25665,"chem":25666,"raid":25667,"Ġtrolls":25668,"Ġ182":25669,"ĠColumb":25670,"Ġtissues":25671,"Ġnaive":25672,"Ġlect":25673,"Central":25674,"Sign":25675,"168":25676,"Ġbribe":25677,"ĠDoll":25678,"ĠTripoli":25679,"Ġfunk":25680,"Ġplaza":25681,"Ġmechanic":25682,"mem":25683,"Ġmonkey":25684,"grid":25685,"Ġtainted":25686,"ĠNicaragua":25687,"pelling":25688,"ĠXia":25689,"ammers":25690,"Ġorth":25691,"ICAN":25692,"Ġrant":25693,"Ġdiary":25694,"ĠHarrington":25695,"Ġimply":25696,"Qaeda":25697,"Ġworsen":25698,"Ġcrafting":25699,"ĠShir":25700,"Ġcoincided":25701,"Ġsnatched":25702,"ileen":25703,"sei":25704,"Ġsurgeons":25705,"directed":25706,"Ġcompulsory":25707,"Ġnowadays":25708,"ĠLI":25709,"ĠRebel":25710,"Ġlions":25711,"ĠJR":25712,"scar":25713,"ĠRespons":25714,"Ġscroll":25715,"ĠErd":25716,"iety":25717,"\";":25718,"ĠBone":25719,"ĠRumble":25720,"ĠKS":25721,"ĠLaur":25722,"kell":25723,"ĠBirds":25724,"agic":25725,"Ġsimmer":25726,"Ġrunaway":25727,"Ġ162":25728,"auna":25729,"Ġdialog":25730,"Ġlouder":25731,"esque":25732,"RR":25733,"Ġbloss":25734,"Ġcaliber":25735,"nery":25736,"Ġhauled":25737,"Ġbacterial":25738,"ĠVanity":25739,"ĠPrograms":25740,"omew":25741,"ĠMama":25742,"Ġarr":25743,"Ġdod":25744,"ĠJarvis":25745,"ĠFIRST":25746,"Ġinjections":25747,"ĠBallard":25748,"Ġmedically":25749,"angan":25750,"ĠNewfoundland":25751,"Ġfracking":25752,"Ġbast":25753,"outing":25754,"Ġmercury":25755,"Ġwatershed":25756,"ĠAmateur":25757,"Ġ153":25758,"escal":25759,"Ġpainter":25760,"creat":25761,"Ġperceive":25762,"Ġgent":25763,"attacks":25764,"worked":25765,"Ġimporting":25766,"Indian":25767,"Ġconvict":25768,"clad":25769,"Ġbudding":25770,"Ġambient":25771,"ĠWitness":25772,"letes":25773,"Ġbuffet":25774,"Ġneedles":25775,"Ġcoding":25776,"Ġchoke":25777,"Ġcorrespondence":25778,"Ġgods":25779,"Ġdances":25780,"Ġsteadfast":25781,"cert":25782,"Ġroaming":25783,"between":25784,"weak":25785,"Jer":25786,"jandro":25787,"Ġdiscouraged":25788,"Ġfruition":25789,"ĠØ":25790,"ĠKop":25791,"ULL":25792,"efe":25793,"imble":25794,"obb":25795,"ulla":25796,"Ġaccredited":25797,"Ġlectures":25798,"bil":25799,"why":25800,"Ġgreeting":25801,"ĠBoost":25802,"Ġmailed":25803,"Ġtroop":25804,"Ġfrig":25805,"Ġrese":25806,"Ġscratched":25807,"Stars":25808,"ĠRailroad":25809,"ĠIdol":25810,"Ġsuccumbed":25811,"ĠWeeks":25812,"ffe":25813,"Ġjihadist":25814,"ITION":25815,"Ġthreads":25816,"ĠGenerally":25817,"Ġmedieval":25818,"Ġquotas":25819,"ĠFerry":25820,"rique":25821,"Ġprod":25822,"ĠEduc":25823,"rive":25824,"Ġensued":25825,"Cy":25826,"Ġinfring":25827,"Ġprank":25828,"Ġfrontline":25829,"Ġcompletes":25830,"upe":25831,"Ġmanageable":25832,"Ġpoems":25833,"otten":25834,"igne":25835,"threat":25836,"ĠDri":25837,"ĠLINK":25838,"Calif":25839,"ĠDos":25840,"ulent":25841,"Ġaids":25842,"Ġslips":25843,"umped":25844,"Ġstyled":25845,"Ġdisproportionately":25846,"ĠDish":25847,"ĠUncle":25848,"andel":25849,"Ġrecharge":25850,"rators":25851,"ĠSPR":25852,"Ġguarded":25853,"ĠGreatest":25854,"ĠSkills":25855,"ĠNob":25856,"ĠDesk":25857,"ĠCros":25858,"Ġwrit":25859,"Ġquery":25860,"ORTS":25861,"Ġbundled":25862,"Ġgib":25863,"Ġeth":25864,"iesta":25865,"Ġevade":25866,"dict":25867,"straight":25868,"Met":25869,"present":25870,"Ġdiff":25871,"Ġdere":25872,"ĠSpl":25873,"Ġrepr":25874,"ĠBeard":25875,"Ġvain":25876,"Ġappointing":25877,"ĠVisual":25878,"caps":25879,"gado":25880,"ĠRican":25881,"ĠPose":25882,"endor":25883,"Ġ222":25884,"ĠLear":25885,"Ġconstructing":25886,"Dan":25887,"ĠSpears":25888,"ĠTherapy":25889,"pta":25890,"Ġrehabilit":25891,"Ġrisked":25892,"ĠGuer":25893,"HF":25894,"Ġ301":25895,"Ġliking":25896,"Ġmodular":25897,"eree":25898,"ĠMAT":25899,"ĠHomeless":25900,"Ġstove":25901,"erd":25902,"hash":25903,"ĠAchilles":25904,"ĠBeta":25905,"Ġincl":25906,"Ġgunned":25907,"ĠCrab":25908,"ĠMara":25909,"Ġinvaded":25910,"ulatory":25911,"ATA":25912,"angering":25913,"onso":25914,"Ġallocate":25915,"Ġgarment":25916,"itudes":25917,"ĠHuang":25918,"Ġstaples":25919,"ĠAlban":25920,"Ġtrough":25921,"Ġupright":25922,"tie":25923,"Ġexploits":25924,"ĠVaughan":25925,"ĠDarrell":25926,"Ġassortment":25927,"ĠChill":25928,"Ġlearners":25929,"aqu":25930,"Ġexplode":25931,"ĠChong":25932,"bt":25933,"opl":25934,"Ġaltern":25935,"Ġ151":25936,"fur":25937,"ULT":25938,"HOU":25939,"ĠMemory":25940,"Ġboosts":25941,"ynes":25942,"priv":25943,"Ġtimeless":25944,"Ġcurtail":25945,"ĠCary":25946,"ĠHud":25947,"Ġexclus":25948,"Ġ275":25949,"Ġfry":25950,"ĠVera":25951,"Ġdefied":25952,"ĠDust":25953,"Ġenvision":25954,"ĠPhilipp":25955,"Ġenhancements":25956,"ĠLIB":25957,"ggy":25958,"ĠAzure":25959,"esis":25960,"Ġcharismatic":25961,"Ġcoincide":25962,"inged":25963,"ĠChoose":25964,"Ġsizeable":25965,"136":25966,"Ġpronounce":25967,"ĠPositive":25968,"Ġideally":25969,"Ġechoes":25970,"Ġcottage":25971,"Ġencrypted":25972,"Prime":25973,"Ġá":25974,"Ġflashes":25975,"Group":25976,"Ġ501":25977,"heat":25978,"atility":25979,"ĠTesting":25980,"pex":25981,"WT":25982,"154":25983,"annah":25984,"Ġcompromising":25985,"Ġinactive":25986,"Ġdisparity":25987,"Ġgruesome":25988,"ĠFeather":25989,"ĠMandal":25990,"Ġthereof":25991,"ĠProducer":25992,"Ġprofiling":25993,"Ġlogistical":25994,"Ġcornerstone":25995,"ĠClaudia":25996,"Congress":25997,"ĠDill":25998,"ophone":25999,"Ġcameo":26000,"ĠCutler":26001,"Ġcraz":26002,"throw":26003,"ĠKasich":26004,"Ġexploiting":26005,"ĠSeas":26006,"agles":26007,"ĠGeological":26008,"ĠStub":26009,"ĠUps":26010,"MER":26011,"Ġmem":26012,"itution":26013,"Ġunderstandably":26014,"Ġcontractual":26015,"warming":26016,"qi":26017,"Sky":26018,"whelming":26019,"Ġcurse":26020,"ĠAren":26021,"Ġ265":26022,"ĠGree":26023,"Ġpresiding":26024,"Works":26025,"stones":26026,"Ġappalling":26027,"plex":26028,"dj":26029,"aunting":26030,"Ġimag":26031,"Ġsexism":26032,"ĠVert":26033,"ĠRag":26034,"ĠBliss":26035,"posium":26036,"div":26037,"Ġexperimenting":26038,"Ass":26039,"Lago":26040,"worthiness":26041,"ĠBerk":26042,"ĠDisneyland":26043,"Ġexaggerated":26044,"iliation":26045,"ĠFP":26046,"Ġprincipals":26047,"Miami":26048,"ropri":26049,"PLE":26050,"iona":26051,"ĠPokemon":26052,"apse":26053,"Ġbubbles":26054,"INC":26055,"ĠCaps":26056,"ĠBrowne":26057,"sing":26058,"Ġcafé":26059,"Ġceilings":26060,"frame":26061,"ĠIrwin":26062,"ATS":26063,"dated":26064,"Ġprotester":26065,"Ġtaps":26066,"ĠOslo":26067,"Ù":26068,"Ġconcentrations":26069,"Ġdistributions":26070,"Ġglucose":26071,"ĠRudolph":26072,"Ġtowels":26073,"Ġâĸº":26074,"Ġneighbourhoods":26075,"Ġinduction":26076,"Ġglaring":26077,"Ġannexation":26078,"Ġunsustainable":26079,"ĠTend":26080,"Ġthumbs":26081,"iegel":26082,"cript":26083,"gor":26084,"closure":26085,"thought":26086,"Ġpaddle":26087,"Ġemulate":26088,"Ġdiameter":26089,"Ġtailor":26090,"ĠCorpor":26091,"icable":26092,"ĠPrin":26093,"Ġadminister":26094,"ĠJudd":26095,"ĠColleg":26096,"aund":26097,"ĠPond":26098,"ĠNOTE":26099,"Ġcombating":26100,"Ġinvention":26101,"ĠOculus":26102,"ĠRepl":26103,"iscal":26104,"Ġtrilogy":26105,"anian":26106,"ATT":26107,"ĠCoke":26108,"DL":26109,"ĠLup":26110,"living":26111,"Ġadvertise":26112,"ĠConnie":26113,"amping":26114,"Ġsung":26115,"ORY":26116,"ĠTet":26117,"Ġsplits":26118,"Ġreconnect":26119,"Ġlou":26120,"mut":26121,"ulator":26122,"Ġstrap":26123,"Ġswallow":26124,"rote":26125,"Ġexec":26126,"ffen":26127,"ĠCombine":26128,"ĠTreat":26129,"Ġsorrow":26130,"ĠNotably":26131,"ĠSever":26132,"rette":26133,"Ġwherein":26134,"Ġtransitioning":26135,"Ġtrout":26136,"Ġcockpit":26137,"Ġcrawl":26138,"Ġferv":26139,"Ġliquids":26140,"Ġtsp":26141,"atell":26142,"Ġmeasles":26143,"Ġjug":26144,"Ac":26145,"ĠKD":26146,"ĠMoose":26147,"Ġvans":26148,"chain":26149,"ĠPapua":26150,"plet":26151,"Wednesday":26152,"lynn":26153,"chery":26154,"budget":26155,"Tony":26156,"ĠBacon":26157,"Ġstirred":26158,"ĠSpecialist":26159,"Ġcounterfeit":26160,"а":26161,"Ġdifferentiate":26162,"Ġmuscular":26163,"ĠTheodore":26164,"Ġlooms":26165,"ĠXX":26166,"ottage":26167,"Ġbenches":26168,"ĠMunicip":26169,"Po":26170,"ĠHeck":26171,"Ġscars":26172,"ĠNim":26173,"ÙĬ":26174,"ĠIngredients":26175,"Ġecological":26176,"ĠAWS":26177,"Ġdispose":26178,"Ġmattered":26179,"Ġ720":26180,"Ġpatriotism":26181,"ĠGrind":26182,"Ġcurved":26183,"opia":26184,"ĠLiqu":26185,"Ġevangelical":26186,"tto":26187,"ĠMaterial":26188,"ĠShowtime":26189,"ĠBS":26190,"Ġcheckpoints":26191,"Ġcrippling":26192,"ĠBalance":26193,"stress":26194,"bearing":26195,"Ġ216":26196,"ĠGuards":26197,"Ġlinebackers":26198,"Ġoffending":26199,"Ġsands":26200,"umbnail":26201,"atorial":26202,"Ġliberties":26203,"ĠGW":26204,"ĠPulitzer":26205,"ĠAlvin":26206,"ĠFAC":26207,"ĠStrategies":26208,"Ġreiter":26209,"ĠRestaur":26210,"ĠLithuania":26211,"ĠSwanson":26212,"terror":26213,"ĠMaurit":26214,"Ġparadise":26215,"zzle":26216,"owment":26217,"ĠWP":26218,"Ġsodium":26219,"Ġfuturistic":26220,"Ġdots":26221,"Anthony":26222,"Though":26223,"Ġstripes":26224,"Ġorig":26225,"ultz":26226,"Ġ340":26227,"KK":26228,"umer":26229,"ivery":26230,"Ġplacebo":26231,"Ġdemocrat":26232,"Ġsubmerged":26233,"ĠHidden":26234,"pieces":26235,"Ġasteroid":26236,"ĠGraphic":26237,"Ġadvert":26238,"sil":26239,"Ġdreaming":26240,"Ġnationality":26241,"Ġfostering":26242,"daughter":26243,"ĠSavings":26244,"Ġmischief":26245,"ĠClair":26246,"ĠBundy":26247,"Ġblatant":26248,"Ġtabs":26249,"qa":26250,"severe":26251,"attered":26252,"Ġgreed":26253,"Ġresembles":26254,"Ġnominal":26255,"Ġineligible":26256,"wealth":26257,"fax":26258,"payers":26259,"Ġdisplacement":26260,"itute":26261,"Ġunpleasant":26262,"ĠPom":26263,"lif":26264,"edo":26265,"ĠNP":26266,"Inter":26267,"Ġcohort":26268,"ĠStacy":26269,"ĠDai":26270,"Ġhistories":26271,"alin":26272,"273":26273,"Ġdram":26274,"ĠKand":26275,"Ġexpectancy":26276,"ansson":26277,"Ġlimbo":26278,"ĠPolar":26279,"Ġdivine":26280,"oused":26281,"Ġshel":26282,"ĠProblem":26283,"achment":26284,"Ġâĸł":26285,"shoot":26286,"antam":26287,"ĠHerz":26288,"Ġ157":26289,"Ġpreventive":26290,"keye":26291,"Sing":26292,"Ġcharacteristic":26293,"Ġcasually":26294,"ĠTaiwanese":26295,"md":26296,"ĠHubbard":26297,"imon":26298,"Ġsect":26299,"148":26300,"Ġmartyr":26301,"stud":26302,"Ġcongrat":26303,"ĠSWAT":26304,"ĠTheory":26305,"INAL":26306,"opping":26307,"ply":26308,"ĠKindle":26309,"uu":26310,"ĠLith":26311,"kaya":26312,"ĠActivity":26313,"uously":26314,"ĠJeb":26315,"tell":26316,"ĠSpin":26317,"ĠExplorer":26318,"Ġfolded":26319,"ĠCanterbury":26320,"ĠStur":26321,"Ġminiature":26322,"Ġmultif":26323,"ĠPressure":26324,"angling":26325,"ĠOverse":26326,"Ġresides":26327,"Ġimpressions":26328,"Ġauthored":26329,"265":26330,"Ġallergies":26331,"143":26332,"ĠJi":26333,"Ġsticker":26334,"ĠAccord":26335,"Ġcaste":26336,"Ġseparates":26337,"ĠFein":26338,"Daily":26339,"179":26340,"ĠScores":26341,"ĠAuction":26342,"hea":26343,"Ġdisclosing":26344,"ĠTacoma":26345,"Ġverse":26346,"ĠBeg":26347,"Ġfabrics":26348,"aez":26349,"Ġattachment":26350,"isy":26351,"Christ":26352,"Ġaddictive":26353,"Ġvir":26354,"Week":26355,"ĠPlum":26356,"croft":26357,"itivity":26358,"ĠExhibition":26359,"Ġbruised":26360,"Ġmimic":26361,"rers":26362,"Ġanal":26363,"Ġunintended":26364,"Ġpall":26365,"atts":26366,"ĠWarn":26367,"Ġslows":26368,"WH":26369,"Ġembro":26370,"nec":26371,"Ġ168":26372,"285":26373,"ologic":26374,"Ġhob":26375,"ĠPeel":26376,"Mill":26377,"eps":26378,"Ġrobbers":26379,"ĠDahl":26380,"semble":26381,"omics":26382,"toe":26383,"ĠLoch":26384,"Ġreproduction":26385,"ĠCullen":26386,"Ġimplants":26387,"Ġwow":26388,"ĠSTATE":26389,"vt":26390,"Ġdepleted":26391,"Ġbreweries":26392,"Ġhateful":26393,"Ġgast":26394,"Ġhollow":26395,"Ġradically":26396,"ographed":26397,"ĠFog":26398,"onian":26399,"ĠSequ":26400,"Ġdisrespectful":26401,"Dis":26402,"ĠExper":26403,"pron":26404,"ĠAmelia":26405,"ĠSage":26406,"bath":26407,"Ġtransformative":26408,"Ġtremendously":26409,"Ġpillow":26410,"ĠNormal":26411,"Cont":26412,"ĠMedic":26413,"educated":26414,"Ġredesigned":26415,"Ġkneeling":26416,"Ġinh":26417,"Ġroofs":26418,"Ġhandmade":26419,"Ġprotracted":26420,"ĠIsn":26421,"ĠCapacity":26422,"Ġsquash":26423,"ĠVega":26424,"Ġfats":26425,"ĠCertified":26426,"ointed":26427,"Ġpricey":26428,"ĠBasil":26429,"Ġfreezer":26430,"Ġscent":26431,"Ġpizz":26432,"ĠArd":26433,"Ġdistractions":26434,"Ġviolently":26435,"ĠHess":26436,"Ġfunc":26437,"Ġundert":26438,"Ġrejuven":26439,"Ġdisbelief":26440,"cluded":26441,"named":26442,"ĠFailure":26443,"kus":26444,"Ġhostages":26445,"ĠSahara":26446,"Ġ1944":26447,"Leary":26448,"ĠPrel":26449,"enza":26450,"ĠAlly":26451,"ĠKak":26452,"Ġcounselors":26453,"ĠGale":26454,"ĠHok":26455,"ĠSold":26456,"Ġhacker":26457,"Ġhun":26458,"Ġbung":26459,"Ġdeclares":26460,"Ġinfringement":26461,"OOD":26462,"Ġdoub":26463,"jam":26464,"Ġallergy":26465,"ĠShipping":26466,"Ġmedic":26467,"Ġaccommod":26468,"Ġdocumenting":26469,"Ġcompanions":26470,"Ġmodelling":26471,"Ġcarriage":26472,"ĠCherokee":26473,"Ġtresp":26474,"Ġtaxable":26475,"ĠActivities":26476,"ĠCrane":26477,"bots":26478,"ĠRusso":26479,"Ġstocked":26480,"ervation":26481,"Ġcoffin":26482,"aign":26483,"guards":26484,"Ġonwards":26485,"Ġfrank":26486,".*":26487,"unic":26488,"Ġcens":26489,"enic":26490,"ruit":26491,"rained":26492,"Ġadapting":26493,"aments":26494,"Ġstagnant":26495,"azaar":26496,"ĠHarlem":26497,"Ġ158":26498,"ysis":26499,"Ġbraking":26500,"Ġdipping":26501,"Ġclan":26502,"ĠShu":26503,"Ġprops":26504,"qualified":26505,"Ġmistakenly":26506,"ĠStalin":26507,"Ġaddicts":26508,"ĠCALL":26509,"ropolis":26510,"aten":26511,"pec":26512,"ĠDro":26513,"ĠFellowship":26514,"ĠSupporting":26515,"loc":26516,"uben":26517,"499":26518,"Bro":26519,"Ġpots":26520,"Ġchunks":26521,"wr":26522,"ĠColonial":26523,"ĠArchitecture":26524,"Ġconstrained":26525,"Ġenvelop":26526,"ĠIronically":26527,"aban":26528,"Ġapparatus":26529,"Ġcue":26530,"Ġborne":26531,"ĠRoz":26532,"ilton":26533,"Ġtheoretical":26534,"ĠWatching":26535,"Ġfuck":26536,"ĠSilk":26537,"ĠSTE":26538,"bler":26539,"ĠPOST":26540,"ĠUpton":26541,"Ġsummons":26542,"ĠCum":26543,"ĠKL":26544,"Ġrelaxation":26545,"ĠDuff":26546,"Ġincumb":26547,"ĠRedd":26548,"Ġstature":26549,"Ġcanv":26550,"added":26551,"Ġremedies":26552,"ĠISO":26553,"ĠDecker":26554,"Ġafloat":26555,"Ġstartling":26556,"ĠBethlehem":26557,"Ġrealizes":26558,"find":26559,"ĠAra":26560,"Ġphased":26561,"arov":26562,"Ġhalting":26563,"ĠWindow":26564,"Ġdentist":26565,"Ġtumble":26566,"Ġvalidation":26567,"Ġcarve":26568,"ĠIPS":26569,"Ġirrit":26570,"ĠEssential":26571,"Ġfluids":26572,"rons":26573,"Ġimplant":26574,"Ġnuisance":26575,"ĠShelley":26576,"ĠGemini":26577,"Ġpharmac":26578,"iction":26579,"Ġtaped":26580,"ĠGovernments":26581,"ruly":26582,"Ġscant":26583,"Ġprominently":26584,"Ġreim":26585,"unning":26586,"arted":26587,"ĠMatters":26588,"Ġ1918":26589,"ĠPros":26590,"atel":26591,"ĠBattalion":26592,"onduct":26593,"talk":26594,"ĠTinder":26595,"ĠInstant":26596,"ĠKern":26597,"Ġbuckets":26598,"ĠGroups":26599,"Ġmetaphor":26600,"cloud":26601,"ĠString":26602,"Ohio":26603,"Ġcaffeine":26604,"Old":26605,"Ġdefinite":26606,"ĠNikola":26607,"ĠLords":26608,"icol":26609,")?":26610,"Ġenjoyment":26611,"Ġfamine":26612,"Ġdefinitions":26613,"ĠJem":26614,"Check":26615,"Ġaiding":26616,"ĠMé":26617,"Ġrenewables":26618,"Ġsightings":26619,"footed":26620,"Box":26621,"Ġgoats":26622,"Ġshack":26623,"AX":26624,"ĠMonk":26625,"ĠGraduate":26626,"Ġmeats":26627,"handle":26628,"147":26629,"rys":26630,"Ġunsub":26631,"Pont":26632,"uble":26633,"440":26634,"Ġeyel":26635,"thro":26636,"Ġcreep":26637,"^^^^":26638,"Ġpopcorn":26639,"Ġcompression":26640,"sal":26641,"ouf":26642,"Ġrepairing":26643,"Think":26644,"Ġdoubtful":26645,"ĠLooks":26646,"Ġtaller":26647,"Ġsul":26648,"sf":26649,"give":26650,"ĠGau":26651,"Ġrevered":26652,"EMBER":26653,"Ġsloppy":26654,"ersen":26655,"Ġvitamins":26656,"ĠImprovement":26657,"Ġprogresses":26658,"Ġdiploma":26659,"semb":26660,"ustain":26661,"Ġchant":26662,"Ġbumped":26663,"Ġsabotage":26664,"nant":26665,"Ġrabbit":26666,"Ġdividing":26667,"ĠDefender":26668,"Ġlik":26669,"Ġirrespective":26670,"cade":26671,"ĠSter":26672,"touch":26673,"EMA":26674,"Ġparted":26675,"ĠBAR":26676,"hung":26677,"Ġannoyed":26678,"Ġhinder":26679,"Ġexamines":26680,"oan":26681,"ĠBoe":26682,"Ġaggreg":26683,"ĠChu":26684,"ĠUCS":26685,"IGHTS":26686,"pez":26687,"ĠUNESCO":26688,"Ġwindshield":26689,"Martin":26690,"Ġwithhold":26691,"does":26692,"Ġbruising":26693,"Ġdeterior":26694,"bourg":26695,"ĠTowers":26696,"JD":26697,"England":26698,"Ġequivalents":26699,"Ġrazor":26700,"Ġreassuring":26701,"Ġident":26702,"Ġ208":26703,"reath":26704,"ceans":26705,"Ġpatrolling":26706,"eve":26707,"pots":26708,"itative":26709,"Ġsided":26710,"Ġsofa":26711,"Ġunborn":26712,"Ġaug":26713,"Ġperpetual":26714,"effect":26715,"represented":26716,"Ġrails":26717,"ĠSummers":26718,"ĠMOR":26719,"ĠSlow":26720,"ĠExpert":26721,"Ġshameful":26722,"Ġaudits":26723,"Sl":26724,"ĠBurr":26725,"adow":26726,"ĠWAY":26727,"anic":26728,"ĠIslamists":26729,"ĠStranger":26730,"pse":26731,"amaz":26732,"ĠPeggy":26733,"ĠSeventh":26734,"Ġscreenplay":26735,"ĠGriff":26736,"Ireland":26737,"142":26738,"Ġneural":26739,"ĠFernand":26740,"ainment":26741,"ĠMigration":26742,"ureen":26743,"ĠSCH":26744,"Sullivan":26745,"ĠWag":26746,"ĠREG":26747,"Ġ420":26748,"inky":26749,"ĠNewspaper":26750,"School":26751,"Ok":26752,"ĠKrishna":26753,"Ġ480":26754,"erald":26755,"Ġskipping":26756,"Ġharrowing":26757,"158":26758,"rogen":26759,"Ġbetrayal":26760,"Ġculmination":26761,"ĠCirc":26762,"Ġ211":26763,"stro":26764,"ĠTrace":26765,"Ġheaviest":26766,"td":26767,"ĠHenri":26768,"epend":26769,"RB":26770,"arella":26771,"umbai":26772,"Ġcrem":26773,"ĠDistribut":26774,"ruff":26775,"Ġscreams":26776,"Ġscathing":26777,"girls":26778,"Ġtiles":26779,"ĠEvil":26780,"usp":26781,"Ġknowledgeable":26782,"Ġrestitution":26783,"ĠWiFi":26784,"Ġitiner":26785,"exper":26786,"oris":26787,"ĠPokémon":26788,"iane":26789,"produ":26790,"ĠAchievement":26791,"Ġbrunt":26792,"ĠSurgery":26793,"Ġpragmatic":26794,"Ber":26795,"ĠKejriwal":26796,"cus":26797,"Ġconsensual":26798,"acet":26799,"ĠSecondly":26800,"Ġdivul":26801,"uca":26802,"Ġbusted":26803,"emies":26804,"ĠMou":26805,"Ġ217":26806,"Ġexcludes":26807,"ĠSamoa":26808,"Ġlofty":26809,"ĠSic":26810,"ĠRemem":26811,"dn":26812,"Ġeradicate":26813,"Ġpies":26814,"Ġscenery":26815,"ATTLE":26816,"ĠWAS":26817,"Ġinnovate":26818,"ĠEverest":26819,"Ġsynonymous":26820,"izen":26821,"Ġeuth":26822,"ĠFIA":26823,"ITIES":26824,"ĠSuddenly":26825,"Ġforay":26826,"pell":26827,"ÄŁ":26828,"licensed":26829,"Ġfra":26830,"Ġblasting":26831,"autical":26832,"ĠBlizzard":26833,"orer":26834,"Ġchili":26835,"ĠSylvia":26836,"except":26837,"tec":26838,"ĠResistance":26839,"young":26840,"usions":26841,"iotic":26842,"ĠDreams":26843,"ĠArchives":26844,"Ġunleash":26845,"ĠPract":26846,"Ġlikened":26847,"Ġga":26848,"Ġdisappearing":26849,"Ġunnoticed":26850,"Ġfrightened":26851,"arms":26852,"ĠCAD":26853,"Ġcoloured":26854,"ĠSigns":26855,"oing":26856,"Ġvodka":26857,"ruption":26858,"otions":26859,"isal":26860,"ĠBecome":26861,"Ġswoop":26862,"reating":26863,"Ġchoking":26864,"Ġunforgettable":26865,"258":26866,"packs":26867,"345":26868,"ĠAutumn":26869,"Ġther":26870,"399":26871,"ĠFaculty":26872,"Ġ1933":26873,"ĠNormally":26874,"orge":26875,"ĠTess":26876,"ĠChrom":26877,"Ġscripts":26878,"Ġbiking":26879,"Act":26880,"Ġgrazing":26881,"ĠLabrador":26882,"ĠLey":26883,"Ġwandering":26884,"Ġfend":26885,"ĠPolk":26886,"ĠKeane":26887,"ĠBeef":26888,"elope":26889,"ĠApproximately":26890,"Ġ1952":26891,"personal":26892,"Ġhistorians":26893,"ĠMcDonnell":26894,"must":26895,"LES":26896,"iking":26897,"Ġtherm":26898,"Ġhumane":26899,"Ġcrowdfunding":26900,"ĠBenefits":26901,"Land":26902,"Ġanalog":26903,"agency":26904,"ĠCrowley":26905,"Ġbirths":26906,"Ġobj":26907,"Ġfren":26908,"ĠSalmon":26909,"bies":26910,"Ġreve":26911,"216":26912,"Ġbetrayed":26913,"Ġinduced":26914,"acles":26915,"Ġtrad":26916,"Ġforgiven":26917,"Ġearners":26918,"208":26919,"Ġxen":26920,"Ġunle":26921,"Ġnecklace":26922,"Ġgravel":26923,"Ġsalads":26924,"Ġgrooming":26925,"California":26926,"Ġpossessed":26927,"Ġproclamation":26928,"Ġsequences":26929,"ream":26930,"FOX":26931,"arkin":26932,"ĠTRAN":26933,"Ġpurs":26934,"ĠLoans":26935,"Ġsacrificed":26936,"Ġiceberg":26937,"Phill":26938,"Ġgalvan":26939,"Ġsmugglers":26940,"formation":26941,"onson":26942,"ĠVaughn":26943,"Ġdoctrine":26944,"ĠEyes":26945,"Ġunmanned":26946,"states":26947,"Ġdetermin":26948,"almost":26949,"Ġeviction":26950,"Ġtid":26951,"ARR":26952,"Ġcooks":26953,"Bad":26954,"ĠCamb":26955,"Ġlinear":26956,"229":26957,"ĠCooke":26958,"ĠPurch":26959,"join":26960,"ĠCult":26961,"ĠRefugee":26962,"Ġslamming":26963,"ĠðŁij":26964,"Ġpedal":26965,"ĠVeronica":26966,"Ġlandowners":26967,"ĠYel":26968,"ĠWorkshop":26969,"antic":26970,"Ġdysfunction":26971,"Ġ229":26972,"Ġculturally":26973,"Ġinfuri":26974,"ĠEck":26975,"sem":26976,"Ġwired":26977,"ĠWerner":26978,"lov":26979,"ĠJasper":26980,"Ġvehemently":26981,"ĠSpy":26982,"lift":26983,"ĠNab":26984,"ĠPound":26985,"ĠHanna":26986,"Ġleveled":26987,"WOOD":26988,"tm":26989,"ĠKitt":26990,"Ġconve":26991,"nat":26992,"Ġjog":26993,"IVER":26994,"Ġmemes":26995,"Ġseaw":26996,"ector":26997,"Ġsprayed":26998,"Ġvaccinated":26999,"Europe":27000,"Ġmustard":27001,"ĠMahm":27002,"Ġ214":27003,"Research":27004,"iminary":27005,"Ġconcerted":27006,"Detroit":27007,"Ġkios":27008,"Ġplummet":27009,"Ġvisuals":27010,"247":27011,"Ġ228":27012,"development":27013,"ĠPascal":27014,"acial":27015,"ĠSeasons":27016,"ĠTL":27017,"480":27018,"ĠReader":27019,"Ġexpulsion":27020,"Ġchoked":27021,"Ġdevotion":27022,"ĠSTAT":27023,"urred":27024,"Ġfascinated":27025,"Ġstealth":27026,"NL":27027,"Ġbooster":27028,"Kat":27029,"ĠPriebus":27030,"Ġaux":27031,"ĠHate":27032,"ĠThing":27033,"Ġabnormal":27034,"Ġcalmly":27035,"Ġdedicate":27036,"cause":27037,"Ġisolate":27038,"ĠPai":27039,"Ġsuspensions":27040,"Ġpoisoned":27041,"ission":27042,"Ġprohibiting":27043,"353":27044,"banks":27045,"Ġkissed":27046,"ĠBegin":27047,"atis":27048,"LI":27049,"Ġshaft":27050,"ĠGuth":27051,"ĠBoo":27052,"Ġcinnamon":27053,"Ġverbally":27054,"ĠRabbi":27055,"Ġmonsters":27056,"done":27057,"ĠClyde":27058,"Ġspar":27059,"ĠCage":27060,"ĠPersons":27061,"305":27062,"ĠMons":27063,"Ġjealous":27064,"Ġswirling":27065,"know":27066,"Ġprote":27067,"Ġcruising":27068,"Ġduly":27069,"Ġchapel":27070,"Ġgroove":27071,"bps":27072,"ĠKelvin":27073,"iom":27074,"aer":27075,"bomb":27076,"Christian":27077,"Ġgigs":27078,"+.":27079,"ĠWei":27080,"Ġfarmland":27081,"otally":27082,"Ġequitable":27083,"ĠCBO":27084,"chool":27085,"amara":27086,"Ġwealthiest":27087,"ĠMeans":27088,"Ġ235":27089,"ĠUk":27090,"steps":27091,"raham":27092,"nerg":27093,"Ġclad":27094,"Ġsled":27095,"ĠMorrow":27096,"152":27097,"ĠRece":27098,"Ġplausible":27099,"Ġbisexual":27100,"artments":27101,"Ġveh":27102,"ĠLoft":27103,"bly":27104,"ĠCONC":27105,"automatic":27106,"Ġmasterpiece":27107,"ĠSpringer":27108,"Ġtendencies":27109,"Ro":27110,"Ġresentment":27111,"Ġadversely":27112,"Ġbandwidth":27113,"ĠDAV":27114,"Ġtun":27115,"Ġpuppies":27116,"ĠBundes":27117,"ĠHort":27118,"ĠGarfield":27119,"Ġenlist":27120,"Ġmont":27121,"gd":27122,"Ġrooting":27123,"Dream":27124,"Ġfulfillment":27125,"chal":27126,"182":27127,"prop":27128,"159":27129,"Ġcourtyard":27130,"iard":27131,"ĠSle":27132,"Ġoperative":27133,"Ġpublishes":27134,"ĠProposition":27135,"Ġcritique":27136,"Ġredist":27137,"wang":27138,"ĠNep":27139,"DD":27140,"Ġbonding":27141,"141":27142,"ĠAssault":27143,"-'":27144,"Ġlodging":27145,"itters":27146,"cigarettes":27147,"Ġ__":27148,"ĠLaf":27149,"GF":27150,"ĠAnat":27151,"ĠStephan":27152,"214":27153,"ĠKass":27154,"Ġviz":27155,"Ġpiling":27156,"Ġfugitive":27157,"ĠCurrency":27158,"ĠCrypto":27159,"Ġfaux":27160,"ĠPing":27161,"ĠLia":27162,"igl":27163,"Ġadversaries":27164,"ĠYPG":27165,"ĠComb":27166,"ĠYar":27167,"heny":27168,"Ġoverhe":27169,"Fest":27170,"emy":27171,"Ever":27172,"Ġ370":27173,"Ġsecretive":27174,"ĠSEN":27175,"ĠMEM":27176,"PRESS":27177,"ĠBirth":27178,"kos":27179,"Ġprecarious":27180,"irting":27181,"ĠUI":27182,"Ġoccupying":27183,"olute":27184,"Ġperiodic":27185,"eon":27186,"iens":27187,"ĠRH":27188,"Win":27189,"Ġplaybook":27190,"Ġexodus":27191,"ĠSkinner":27192,"Ġorderly":27193,"ĠVed":27194,"ouses":27195,"Ġescal":27196,"Ġbenign":27197,"Ġbots":27198,"ĠWhis":27199,"Ġappra":27200,"FOR":27201,"ĠChromebook":27202,"_____":27203,"990":27204,"athed":27205,"Ġspirited":27206,"illi":27207,"Ġbicycles":27208,"orse":27209,"ifestyle":27210,"orno":27211,"ĠDept":27212,"JA":27213,"Ġnausea":27214,"Ġpervasive":27215,"velop":27216,"commun":27217,"ĠUniversities":27218,"Ġremnants":27219,"Ġdisarm":27220,"ĠBoots":27221,"Ġprin":27222,"...\"":27223,"quila":27224,"Ġcautiously":27225,"uper":27226,"onto":27227,"din":27228,"Ġvelocity":27229,"Ġconspiring":27230,"ĠMX":27231,"Ġemphasizing":27232,"Ġâĸ":27233,"ĠStam":27234,"Ġspices":27235,"Ġairplanes":27236,"uty":27237,"culture":27238,"ĠPetr":27239,"Ġglor":27240,"ĠExcel":27241,"ĠSpeech":27242,"Ġharmless":27243,"ĠPend":27244,"ĠCrossing":27245,"ĠDocument":27246,"Ġramifications":27247,"ĠCroatian":27248,"ĠKiller":27249,"Ġmultim":27250,"Ġdiscontinued":27251,"Ġcherished":27252,"ĠMaker":27253,"aspers":27254,"ĠBlooming":27255,"ĠMata":27256,"offic":27257,"Ġsettlers":27258,"ĠPlenty":27259,"ĠInstitutes":27260,"ĠArpaio":27261,"Pool":27262,"ĠSubst":27263,"Ġ380":27264,"Ġdecidedly":27265,"ollah":27266,"Den":27267,"ĠJiang":27268,"ĠAmos":27269,"Grand":27270,"ĠTurns":27271,"meyer":27272,"Ġconducive":27273,"Ġpoignant":27274,"abortion":27275,"Ġnotebook":27276,"Ġshelling":27277,"common":27278,"ĠPavel":27279,"Ġhumid":27280,"Ġinappropriately":27281,"????":27282,"Ġsoar":27283,"Ġdynasty":27284,"Ġresearched":27285,"ĠYon":27286,"Ġmaple":27287,"Ġwedge":27288,"mass":27289,"ĠTM":27290,"USE":27291,"eln":27292,"Ġgloss":27293,"rigan":27294,"steen":27295,"ĠDeV":27296,"Ġdebacle":27297,"Christmas":27298,"Ġtweaks":27299,"grab":27300,"Ġprofoundly":27301,"Ġcampaigner":27302,"ĠSeal":27303,"Ġiteration":27304,"Ġsigh":27305,"Ġunfounded":27306,"Ġframing":27307,"Ġrecognizable":27308,"Ġseizing":27309,"legal":27310,"Ġproportions":27311,"omers":27312,"rek":27313,"Ġscreenshot":27314,"itsu":27315,"ĠOG":27316,"ĠYing":27317,"ĠMississ":27318,"295":27319,"Ġlandsl":27320,"Ġpsychiatrist":27321,"sov":27322,"arine":27323,"Ju":27324,"Ġflo":27325,"apple":27326,"hof":27327,"wig":27328,"ĠENT":27329,"Ġenthusiast":27330,"Such":27331,"ĠArtificial":27332,"happy":27333,"oton":27334,"ĠFram":27335,"ĠRemove":27336,"Ġsmear":27337,"Ġjer":27338,"Ġtopp":27339,"Ġimbalance":27340,"ĠWords":27341,"Ġcoffers":27342,"olina":27343,"Ġrigged":27344,"uction":27345,"idding":27346,"Ġdispensaries":27347,"Ġdermat":27348,"Ġshutter":27349,"idental":27350,"Ġcontinu":27351,"Ġhumility":27352,"Ġbulbs":27353,"Ġ207":27354,"lass":27355,"ĠBeirut":27356,"ĠUlt":27357,"urry":27358,"NEWS":27359,"Ġfeminine":27360,"Ġsimulated":27361,"Ġcharger":27362,"mom":27363,"ĠCreed":27364,"Ġwolves":27365,"essions":27366,"created":27367,"ifiers":27368,"Ġdissemin":27369,"ĠDarling":27370,"umann":27371,"Ġmarrying":27372,"Ġshred":27373,"avin":27374,"Ġbudgetary":27375,"Ġmedicinal":27376,"ulin":27377,"seys":27378,"agues":27379,"Ġextracted":27380,"ĠFlower":27381,"Ġcontinents":27382,"ĠWish":27383,"Ġdivides":27384,"ĠDing":27385,"Ġinsulation":27386,"respect":27387,"ĠABS":27388,"Ġreconcile":27389,"keep":27390,"ILD":27391,"Ġgenome":27392,"Ġ410":27393,"ĠSweep":27394,"Ġharass":27395,"Ġfrantic":27396,"ĠEE":27397,"dad":27398,"Ġaperture":27399,"rought":27400,"Ġhugs":27401,"Ġdrying":27402,"Ġoverrun":27403,"Space":27404,"Ġperiodically":27405,"Ġbrightness":27406,"atched":27407,"kee":27408,"ĠITS":27409,"ĠSpokane":27410,"ĠSeaf":27411,"Ġdesks":27412,"ĠEisen":27413,"ĠOPS":27414,"Ġcider":27415,"Ġacceler":27416,"ĠAthlet":27417,"2008":27418,"ĠGuid":27419,"ĠManip":27420,"Ġmould":27421,"Ġmisguided":27422,"Ġbrow":27423,"Ġmanagerial":27424,"Ġhugged":27425,"Ġfurnish":27426,"ĠHarmony":27427,"ĠHebrew":27428,"Ġtyph":27429,"Ġdecreases":27430,"Ġimpetus":27431,"Ġcontagious":27432,"Ġunch":27433,"209":27434,"Ġswell":27435,"ĠHuffington":27436,"Ġpubs":27437,"Ġadequ":27438,"amoto":27439,"rir":27440,"Ġpristine":27441,"Ġanx":27442,"ĠSecure":27443,"Ġenrichment":27444,"ĠVAL":27445,"Ġsummed":27446,"Ġconfidently":27447,"ĠProfit":27448,"ĠFrog":27449,"ĠLena":27450,"ĠFUN":27451,"Ġbruises":27452,"Ġuproar":27453,"coll":27454,"ĠImpro":27455,"Ġflair":27456,"146":27457,"ĠBrend":27458,"Ġ166":27459,"Ġenhances":27460,"ĠDent":27461,"Ġdegener":27462,"Ġproponents":27463,"ĠInspired":27464,"Ġramps":27465,"Ġwisely":27466,"Western":27467,"Ġtart":27468,"Ġsteered":27469,"Ġtreason":27470,"dropping":27471,"Ġtransc":27472,"ĠScarlett":27473,"ĠEzekiel":27474,"Ġpivot":27475,"esame":27476,"Show":27477,"Ġdiscontent":27478,"ĠJudith":27479,"ĠPutting":27480,"Ġblessings":27481,"Ġhardcore":27482,"Ġtray":27483,"Ġdiscern":27484,"oley":27485,"ouk":27486,"Ġwil":27487,"Ġintolerance":27488,"157":27489,"ĠRelative":27490,"ĠLynd":27491,"Ġwhistleblower":27492,"Ġincon":27493,"ĠTao":27494,"Ġindefinite":27495,"Ġguardians":27496,"Ġagon":27497,"ĠInstruments":27498,"Ġexistential":27499,"AAF":27500,"vind":27501,"Ġbrazen":27502,"condition":27503,"Ġratified":27504,"fam":27505,"ĠHin":27506,"ĠMichaels":27507,"204":27508,"ĠKats":27509,"ITS":27510,"ISON":27511,"prone":27512,"Ġboiling":27513,"Ġprolong":27514,"Ġnoticing":27515,"resident":27516,"brance":27517,"ĠFolk":27518,"Ġdesserts":27519,"uton":27520,"Web":27521,"ĠLongh":27522,"ĠReef":27523,"Going":27524,"ĠCarb":27525,"Sur":27526,"complete":27527,"ĠSloan":27528,"ĠClubs":27529,"ĠSadd":27530,"Ġshrugged":27531,"Ġedible":27532,"ĠTyp":27533,"thal":27534,"ĠRocks":27535,"ĠClive":27536,"Ġkidding":27537,"ĠCrom":27538,"ĠTurks":27539,"ĠWak":27540,"Ġeyewitness":27541,"ĠHass":27542,"collar":27543,"Ġsucceeding":27544,"Ġinsert":27545,"Ġ224":27546,"ĠBret":27547,"Ġneurological":27548,"Ġrewrite":27549,"imil":27550,"ultimate":27551,"ĠJeremiah":27552,"Ġliaison":27553,"Ġpedd":27554,"direct":27555,"ĠYi":27556,"ĠMAD":27557,"ĠOrion":27558,"oyd":27559,"ĠLOC":27560,"release":27561,"Ġinvestigates":27562,"ĠApache":27563,"û":27564,"ĠVend":27565,"Ġcynical":27566,"ĠHelm":27567,"ĠMovies":27568,"tops":27569,"Ġsinister":27570,"Ġunparalleled":27571,"Ġspikes":27572,"Ġoverlap":27573,"enstein":27574,"Ġhypocrisy":27575,"Plus":27576,"Ġexpansions":27577,"Ġvow":27578,"Ġdetonated":27579,"Ġfellowship":27580,"Ġsolicitor":27581,"ĠNewtown":27582,"mony":27583,"ĠLod":27584,"ĠDevelopers":27585,"ateg":27586,"ibus":27587,"Ġcrumbling":27588,"ĠWein":27589,"ĠKlan":27590,"gio":27591,"ĠPhys":27592,"ĠAntarctica":27593,"368":27594,"Ġseam":27595,"Ġautomobiles":27596,"ĠTEAM":27597,"bern":27598,"Ġmanic":27599,"Ġsanct":27600,"Ġequals":27601,"Est":27602,"Ġincentiv":27603,"ĠHawking":27604,"nin":27605,"Ġresonate":27606,"bid":27607,"Ġtelescope":27608,"endon":27609,"ĠVacc":27610,"Ġregretted":27611,"Ġ1300":27612,"ĠForestry":27613,"BOOK":27614,"Ġgroundwork":27615,"Ġessays":27616,"ĠIndo":27617,"Pierre":27618,"ĠChau":27619,"Ġapologies":27620,"killers":27621,"ĠMoroccan":27622,"0001":27623,"336":27624,"Ra":27625,"Ġparcels":27626,"Ġleaned":27627,"Ġthankfully":27628,"ĠSplit":27629,"Ġlobbied":27630,"ĠDegree":27631,"Ġrisking":27632,"assy":27633,"Ġsupplemental":27634,"little":27635,"Ġeclectic":27636,"Ġ206":27637,"ealing":27638,"206":27639,"Ġrepo":27640,"Ġhose":27641,"ayn":27642,"lux":27643,"Ġbeliever":27644,"')":27645,"ĠHide":27646,"vance":27647,"ĠEinstein":27648,"Ġdepos":27649,"Ġfray":27650,"Ġki":27651,"Ġinternship":27652,"ĠHou":27653,"Vis":27654,"Ġstare":27655,"ĠBreed":27656,"option":27657,"Ġvisionary":27658,"Ġmins":27659,"Ġbitten":27660,"ancies":27661,"ĠShake":27662,"Ġtemplate":27663,"Ġliner":27664,"Ġmuster":27665,"appro":27666,"ĠMubarak":27667,"esty":27668,"mong":27669,"actory":27670,"Ġheadphone":27671,"ĠPrec":27672,"Ġwaive":27673,"Ron":27674,"ĠHearing":27675,"Ġimperfect":27676,"Ġsealing":27677,"Ġlocating":27678,"Ġculminated":27679,"chio":27680,"channel":27681,"lust":27682,"ĠLowell":27683,"woods":27684,"Ġsoak":27685,"Ġforbidden":27686,"Ġdetached":27687,"unct":27688,"ĠHunger":27689,"ĠPatient":27690,"ĠPolo":27691,"Saharan":27692,"Jon":27693,"athered":27694,"ĠSignal":27695,"Six":27696,"Ġstatistically":27697,"ITH":27698,"artment":27699,"ĠCU":27700,"Ġhates":27701,"qual":27702,"Ġcapitalist":27703,"ATES":27704,"ĠDesc":27705,"Ġhandcuffed":27706,"Ġindulge":27707,"ĠReligious":27708,"German":27709,"housing":27710,"Ġdismantling":27711,"Ġconventions":27712,"dain":27713,"chairs":27714,"Ġloos":27715,"Ġknowingly":27716,"Var":27717,"Ġhusbands":27718,"eez":27719,"asion":27720,"ĠIssa":27721,"Ġswollen":27722,"Ġ1946":27723,"Ġheadlined":27724,"Chelsea":27725,"Ġignorant":27726,"Ġperipheral":27727,"Note":27728,"Ġaxe":27729,"Ġnicotine":27730,"ĠSanctuary":27731,"Ġ1917":27732,"Ġwithdrawals":27733,"uits":27734,"Hot":27735,"Ġreimburse":27736,"probably":27737,"ĠAdapt":27738,"industrial":27739,"answer":27740,"orus":27741,"ĠMell":27742,"Talk":27743,"Ġcontemplating":27744,"omas":27745,"Ġtaxis":27746,"Ġencompasses":27747,"rations":27748,"ĠLatvia":27749,"Ġhumiliating":27750,"Ġloft":27751,"tight":27752,"rium":27753,"Ġlogin":27754,"ĠBulletin":27755,"Ġturtles":27756,"EAR":27757,"349":27758,"Radio":27759,"ĠBord":27760,"151":27761,"kk":27762,"pocket":27763,"Ġdove":27764,"348":27765,"Ġtemptation":27766,"ĠCoy":27767,"those":27768,"ĠDest":27769,"ishly":27770,"rn":27771,"Ġmammals":27772,"ĠTub":27773,"arial":27774,"ĠPersian":27775,"Ġdaddy":27776,"Zen":27777,"Ġps":27778,"Ġ]":27779,"Field":27780,"adiq":27781,"Ġmeaningless":27782,"Ġprimer":27783,"Ġ1942":27784,"Ġ!":27785,"625":27786,"Ġfashionable":27787,"ĠTheft":27788,"ĠHAVE":27789,"christ":27790,"Ġperil":27791,"Ġrepealing":27792,"Ġbuff":27793,"Ġodor":27794,"Ġstalking":27795,"ĠDems":27796,"iences":27797,"Ġunilaterally":27798,"odies":27799,"ĠQuite":27800,"Ġbloodshed":27801,"Ġinfect":27802,"Ġreminders":27803,"Ġchop":27804,"Ġevapor":27805,"877":27806,"Ġhorrified":27807,"ĠFruit":27808,"rams":27809,"Ġinsecure":27810,"cester":27811,"ĠNationwide":27812,"Ġmocking":27813,"Ret":27814,"Ġcomplying":27815,"sav":27816,"Ġali":27817,"Family":27818,"Ĩ":27819,"Ġdishonest":27820,"Ġincorrectly":27821,"LOAD":27822,"ĠGand":27823,"ourcing":27824,"obby":27825,"ĠPetersen":27826,"Something":27827,"Ġravaged":27828,"limited":27829,"Ġrituals":27830,"ĠKnowledge":27831,"ĠUtility":27832,"Ġdoom":27833,"Ġsheds":27834,"ĠGael":27835,"ĠMillennials":27836,"ĠMonthly":27837,"Ġdomination":27838,"Ġrapport":27839,"spot":27840,"ĠPrest":27841,"ĠHA":27842,"ushes":27843,"Ġtact":27844,"Richard":27845,"Ġgritty":27846,"Does":27847,"ĠTNT":27848,"Ġdownfall":27849,"Wood":27850,"ĠPrediction":27851,"ĠPour":27852,"ĠFraud":27853,"ĠSyndrome":27854,"166":27855,"Ġliteral":27856,"Ġaddict":27857,"ĠLoud":27858,"hens":27859,"ĠAccounts":27860,"distance":27861,"Ġclassmate":27862,"Ġsalv":27863,"Ġunlucky":27864,"Ġpartying":27865,"ĠKou":27866,"ĠSNAP":27867,"%-":27868,"Ġdelegate":27869,"Ġstrikers":27870,"ĠSlate":27871,"Ġarticulate":27872,"390":27873,"Ġinqu":27874,"Ġdiscredit":27875,"ĠPriv":27876,"ploy":27877,"ĠMarketplace":27878,"ĠTune":27879,"visor":27880,"Ġwrestle":27881,"Ġkindly":27882,"ĠCollect":27883,"Ġcirc":27884,"ĠRemain":27885,"Ġ192":27886,"contin":27887,"Ġ325":27888,"Ġsevered":27889,"isations":27890,"Ġmuddy":27891,"Ġtaxing":27892,"ĠRepresent":27893,"ĠSty":27894,"rology":27895,"ĠJudges":27896,"ĠBronze":27897,"ĠApplic":27898,"Ġarrow":27899,"consuming":27900,"ĠFeaturing":27901,"Ġspies":27902,"Ġnoises":27903,"ĠColony":27904,"lost":27905,"Ġopp":27906,"Ġdeem":27907,"ĠGarc":27908,"icent":27909,"ptroller":27910,"liest":27911,"Ġoutward":27912,"ĠUser":27913,"Ġintimidate":27914,"156":27915,"Ġjab":27916,"ANGE":27917,"Jay":27918,"ĠPoverty":27919,"ACA":27920,"Ġrife":27921,"Ġfaint":27922,"ĠAcceler":27923,"tall":27924,"ĠUNITED":27925,"ĠFighter":27926,"ĠGilmore":27927,"Ġsod":27928,"amura":27929,"Ġpredictive":27930,"Ġpolish":27931,"ĠDD":27932,"Ġfabricated":27933,"ĠDag":27934,"Ġfatty":27935,"Ġplague":27936,"Ġexhib":27937,"ĠAdvent":27938,"Ġ1941":27939,"ERSON":27940,"initely":27941,"Ġloneliness":27942,"ĠEquality":27943,"Ġuntrue":27944,"Ġonlook":27945,"Ġfragmented":27946,"ruce":27947,"Ġdistrust":27948,"Ġscal":27949,"ĠCors":27950,"Ġrobbing":27951,"cultural":27952,"clusion":27953,"ĠObi":27954,"sels":27955,"ĠEvidence":27956,"ĠSac":27957,"Ġfragments":27958,"Ġflipping":27959,"ĠRabbit":27960,"Ġdisproportionate":27961,"ĠCreat":27962,"Ġlabeling":27963,"ĠGri":27964,"Ġ161":27965,"ĠEditors":27966,"holm":27967,"adr":27968,"Ĭ":27969,"tailed":27970,"Ġrenters":27971,"Ġnoodles":27972,"Ġcompetence":27973,"Ġpanc":27974,"uration":27975,"Ġacids":27976,"Ġconfid":27977,"rival":27978,"AAA":27979,"kson":27980,"Ġrecreate":27981,"153":27982,"Ġ164":27983,"ĠOlympia":27984,"ĠUnlimited":27985,"ĠShock":27986,"ĠTeaching":27987,"ĠHouses":27988,"resso":27989,"ĠMaw":27990,"Ġreplen":27991,"Ġprotestors":27992,"bey":27993,"Ġsurve":27994,"Ġemphasizes":27995,"223":27996,"ĠEsther":27997,"ĠNikol":27998,"Ġprosecutions":27999,"ĠFreed":28000,"Ġposs":28001,"OTE":28002,"ĠPrayer":28003,"Ġsquarely":28004,"Ġtir":28005,"adv":28006,"Ġbogus":28007,"Ġwrongful":28008,"Ġembell":28009,"Ġseldom":28010,"Ġpossesses":28011,"Er":28012,"ĠAlternatively":28013,"Ġinstituted":28014,"rr":28015,"Ġvocational":28016,"eval":28017,"ĠComics":28018,"Ġstumbling":28019,"335":28020,"Ġdragon":28021,"vine":28022,"services":28023,"Ġcrit":28024,"irens":28025,"Ġlayered":28026,"orb":28027,"Ġdominates":28028,"ĠMarx":28029,"period":28030,"avering":28031,"Ġbrigade":28032,"Ġchem":28033,"ĠEvolution":28034,"ĠSuk":28035,"Ġ209":28036,"ĠMalk":28037,"Ġtallest":28038,"recogn":28039,"ĠCraw":28040,"Ġell":28041,"ĠCaesar":28042,"php":28043,"ĠSurvivors":28044,"sd":28045,"itsch":28046,"ambo":28047,"Ġashore":28048,"acular":28049,"rost":28050,"Ġmurderer":28051,"Ġcasts":28052,"ĠEconomist":28053,"ĠWeapons":28054,"Ġnostalgic":28055,"Skip":28056,"REAM":28057,"Pa":28058,"Ġjournals":28059,"ĠSitting":28060,"Union":28061,"Att":28062,"ĠMaxim":28063,"Ġpurportedly":28064,"Ġrespecting":28065,"ĠMAX":28066,"seed":28067,"Ġjuicy":28068,"ĠGallup":28069,"Ġmileage":28070,"adier":28071,"Ġbod":28072,"DER":28073,"Ġsummers":28074,"icult":28075,"ipl":28076,"ĠDeng":28077,"Ġsmells":28078,"Ġivory":28079,"Ġ255":28080,"Id":28081,"DEN":28082,"Ġ159":28083,"Due":28084,"ĠLighting":28085,"ĠSurely":28086,"Ġsund":28087,"ĠKessler":28088,"immigrant":28089,"Ġtragedies":28090,"ĠOxy":28091,"ĠFixed":28092,"ĠBalk":28093,"Ġoriented":28094,"pher":28095,"Ġkitchens":28096,"Ġhips":28097,"Ġtweak":28098,"Ġtuna":28099,"ĠCla":28100,"Ġdislike":28101,"ussy":28102,"Ġoutnumbered":28103,"Ġplumbing":28104,"Ġcogn":28105,"ĠThrow":28106,"ĠTER":28107,"urally":28108,"ĠMurd":28109,"Ġcreamy":28110,"Ġresiding":28111,"otics":28112,"Ġfingerprints":28113,"!,":28114,"Ġpaused":28115,"ĠMilo":28116,"Ġhomosexuality":28117,"Ġresponsibly":28118,"iop":28119,"UCT":28120,"Ġsucceeds":28121,"ĠCRE":28122,"ĠThatcher":28123,"Ġcurrents":28124,"Ġarises":28125,"Ġwaterproof":28126,"Ġamp":28127,"ĠClaims":28128,"177":28129,"Ġsubpoen":28130,"Ġvig":28131,"ĠNeuro":28132,"Ġblur":28133,"ĠPaint":28134,"campus":28135,"Ġtoughness":28136,"ĠButton":28137,"Neal":28138,"ĠDEN":28139,"ĠNir":28140,"ĠAxel":28141,"EEP":28142,"Ġpint":28143,"Ġagile":28144,"odor":28145,"Ġessentials":28146,"ĠMov":28147,"ĠVenezuel":28148,"Ġexchanging":28149,"ĠNegative":28150,"Mil":28151,"Key":28152,"Ġbuzzing":28153,"ĠStew":28154,"Ġrebuke":28155,"Ġdepl":28156,"ĠKoz":28157,"Ġ163":28158,"Ġshines":28159,"NZ":28160,"Ġcarnage":28161,"cases":28162,"Ġwarmed":28163,"ĠGreenwich":28164,"College":28165,"Ġneedy":28166,"301":28167,"ĠMü":28168,"culation":28169,"Ġ440":28170,"425":28171,"atories":28172,"Ġsatisfactory":28173,"ĠFib":28174,"ĠElim":28175,"developed":28176,"Ġvacations":28177,"Ġpeculiar":28178,"Ġvets":28179,"onest":28180,"ĠPug":28181,"Ġlifestyles":28182,"zzi":28183,"Ġprovoke":28184,"bah":28185,"arger":28186,"ĠVirt":28187,"Sales":28188,"annel":28189,"ĠMeth":28190,"ivating":28191,"Ġrevoke":28192,"ĠAgenda":28193,"ĠIch":28194,"Ġsensit":28195,"ĠAzerbai":28196,"ĠBombay":28197,"Ġuncon":28198,"river":28199,"Ġapr":28200,"actic":28201,"ĠSubaru":28202,"Ġbanquet":28203,"Ġcontradict":28204,"tek":28205,"Football":28206,"igent":28207,"Ġreintrodu":28208,"ĠInsight":28209,"Ġsystematically":28210,"Ġboun":28211,"ĠFishing":28212,"Ġstri":28213,"ĠOB":28214,"Ġstair":28215,"Wall":28216,"ĠAllow":28217,"Ġcaramel":28218,"169":28219,"Ġcafes":28220,"Ġcalcium":28221,"Ġ169":28222,"Ġportraying":28223,"Ġdiscriminate":28224,"Ġunrestricted":28225,"Ġmant":28226,"Ġscarcity":28227,"Ġfeminism":28228,"ĠJJ":28229,"ĠOversight":28230,"ĠCue":28231,"Ġinexperienced":28232,"Ġdrafts":28233,"Ġ1939":28234,"nm":28235,"forest":28236,"ĠHonour":28237,"Ġceramic":28238,"Ġdownstairs":28239,"Ġboon":28240,"Ġmorality":28241,"Ġhorrifying":28242,"Rad":28243,"justice":28244,"Ġmosques":28245,"Ġcurfew":28246,"Ġsurrogate":28247,"Ġreimb":28248,"enth":28249,"pressure":28250,"beam":28251,"Ġwhirlwind":28252,"ĠRecession":28253,"ĠTours":28254,"Ġclusters":28255,"ĠQuant":28256,"Jonathan":28257,"project":28258,"Ġ777":28259,"ĠNOAA":28260,"abis":28261,"Ġdeficiencies":28262,"Ġsuicides":28263,"Ġfoothold":28264,"ĠYah":28265,"imeter":28266,"URN":28267,"Ġcultivate":28268,"Ġnoisy":28269,"Ġ1951":28270,"Ġpressuring":28271,"ĠDeals":28272,"ĠProphet":28273,"ĠWikipedia":28274,"INESS":28275,"ĠShine":28276,"ĠCalled":28277,"ĠSole":28278,"ĠZhou":28279,"Ġasphalt":28280,"armac":28281,"ĠScorp":28282,"ĠUnknown":28283,"ĠPAT":28284,"Heart":28285,"Ġguessed":28286,"Ġsushi":28287,"Ġheartbeat":28288,"Ġconcent":28289,"eret":28290,"plin":28291,"Ġweeds":28292,"Ġbombed":28293,"ĠTerrorism":28294,"Rich":28295,"Ġblades":28296,"Ġhaunt":28297,"Ġstorefront":28298,"Ġthwarted":28299,"access":28300,"ĠLydia":28301,"LINE":28302,"Ġpregnancies":28303,"Ġripping":28304,"ĠBelieve":28305,"spoken":28306,"inian":28307,"sed":28308,"ĠBrass":28309,"econom":28310,"current":28311,"Ġvoc":28312,"Ġmodeled":28313,"Ġpeppers":28314,"otech":28315,"ĠOption":28316,"Connell":28317,"isel":28318,"Ġcompel":28319,"Ġjuveniles":28320,"ĠNET":28321,"ĠEXP":28322,"Ġparadigm":28323,"Des":28324,"Ġ204":28325,"employed":28326,"Ġdurability":28327,"Ġ245":28328,"Ġbillionaires":28329,"violent":28330,"ĠCooperative":28331,"TOP":28332,"ĠGarry":28333,"ĠSoldiers":28334,"Ġdared":28335,"Ġvoucher":28336,"Ġblends":28337,"gue":28338,"Ġadventurous":28339,"Ġorganisms":28340,"Ġgaze":28341,"Ġcrap":28342,"Coach":28343,"omon":28344,"ĠWheels":28345,"ĠGrayson":28346,"Ġrecy":28347,"grave":28348,"Ġallergic":28349,"Ġreef":28350,"Ġbeginnings":28351,"ĠRuff":28352,"Ġclout":28353,"structed":28354,"315":28355,"ĠGeorgian":28356,"say":28357,"Ġsprings":28358,"ĠAsus":28359,"Ġrepaid":28360,"ĠGuys":28361,"ticket":28362,"Ġunb":28363,"ĠCertificate":28364,"ĠSTORY":28365,"cin":28366,"Ġpassions":28367,"Ġmediocre":28368,"Ġlackluster":28369,"vernight":28370,"kids":28371,"ĠWife":28372,"politics":28373,"ĠHimal":28374,"oddy":28375,"ensus":28376,"ĠGustav":28377,"binding":28378,"ĠIndividuals":28379,"Ġmaize":28380,"Ġhoop":28381,"ĠChanging":28382,"Ġlessen":28383,"Ġarranging":28384,"ĠFukushima":28385,"ĠTrying":28386,"ĠMage":28387,"Ġskeleton":28388,"ĠTec":28389,"289":28390,"Ġrecl":28391,"ĠFIL":28392,"Gs":28393,"ĠOdyssey":28394,"ĠProcessing":28395,"ilion":28396,"Ġsubsidized":28397,"Ġabdomen":28398,"Ġanalyse":28399,"music":28400,"clean":28401,"Ġunfinished":28402,"Ġdownloads":28403,"Ġmorally":28404,"Ġ218":28405,"Ġtrib":28406,"Keep":28407,"ĠSER":28408,"FY":28409,"Ġaust":28410,"Ġdiscovers":28411,"ĠGROUP":28412,"ĠMachines":28413,"Ġeroded":28414,"Ġominous":28415,"Ġbrightly":28416,"IME":28417,"Ġwicked":28418,"ĠTrou":28419,"Ġvisions":28420,"Kay":28421,"reported":28422,"Ġbog":28423,"ĠQuin":28424,"ĠSigma":28425,"urned":28426,"ixon":28427,"Ġharming":28428,"Ġcheckout":28429,"inet":28430,"much":28431,"Ġcherish":28432,"ĠByrd":28433,"ĠSamson":28434,"WP":28435,"orders":28436,"boa":28437,"Ġbron":28438,"oki":28439,"ĠRR":28440,"Ġsuitcase":28441,"Ġfeathers":28442,"ĠChristy":28443,"Islamic":28444,"Ġamusement":28445,"ĠISS":28446,"intensive":28447,"Qaida":28448,"Ġneurons":28449,"Ġwagon":28450,"ĠTek":28451,"Ġdolls":28452,"ĠShoot":28453,"Ġunderestimate":28454,"Ġstreamlined":28455,"Ġfractures":28456,"Ġcathedral":28457,"Ġeliminates":28458,"helle":28459,"Ġcitrus":28460,"risis":28461,"Ġimpecc":28462,"istries":28463,"ĠHog":28464,"vote":28465,"pas":28466,"Ġassign":28467,"ĠSongs":28468,"ĠMiracle":28469,"kas":28470,"zynski":28471,"Ġcrane":28472,"Ġadulthood":28473,"ĠBenefit":28474,"ĠGrimes":28475,"Ġpayday":28476,"ablished":28477,"Ġcenterpiece":28478,"Ġhassle":28479,"ĠAppalachian":28480,"follow":28481,"Ġ290":28482,"ĠRL":28483,"ĠDoe":28484,"Ġacclaim":28485,"Ġlevied":28486,"Ġtossing":28487,"Ġcarrots":28488,"ĠDarius":28489,"161":28490,"Ġoffspring":28491,"ĠJury":28492,"ĠTPP":28493,"CAP":28494,"Ġenvironmentalists":28495,"Ġrays":28496,"267":28497,"Ser":28498,"Ġcaptivity":28499,"Ġappellate":28500,"ĠElectricity":28501,"ĠEnough":28502,"232":28503,"Ġfisher":28504,"Ġbrilliance":28505,"Ġpraises":28506,"aunch":28507,"Ġsolicitation":28508,"Ġadolescent":28509,"Ġinferior":28510,"checks":28511,"Set":28512,"Ġmutations":28513,"ĠLatinos":28514,"ĠLicense":28515,"ĠAme":28516,"hirt":28517,"ĠChun":28518,"Ġdeeds":28519,"ldon":28520,"Ġmammoth":28521,"Ġturtle":28522,"rule":28523,"Ken":28524,"Ġvoyage":28525,"gram":28526,"Ġconquer":28527,"Ġretaliate":28528,"ĠPJ":28529,"ĠViking":28530,"Ġsafegu":28531,"ordinary":28532,"ĠArbit":28533,"ĠDigest":28534,"Die":28535,"Ġbureaucratic":28536,"Ġhonorable":28537,"Ġcafeteria":28538,"ĠRAF":28539,"ĠPlaces":28540,"ĠKlu":28541,"Cam":28542,"ĠBiology":28543,"ĠCycling":28544,"imore":28545,"Ġstripping":28546,"Ġwarriors":28547,"Ġbursting":28548,"Ġlapse":28549,"Ġversa":28550,"Ġclicked":28551,"ogh":28552,"Ġ\"âĢ¦":28553,"Ġdiligently":28554,"ĠMiy":28555,"ĠCorpus":28556,"Ġredef":28557,"Ġ176":28558,"ĠInstrument":28559,"ĠOECD":28560,"Ġstro":28561,"Ġmicrowave":28562,"Santa":28563,"Ġpars":28564,"Social":28565,"iffe":28566,"itability":28567,"Equ":28568,"Ġnud":28569,"legged":28570,"ĠTud":28571,"lav":28572,"Ġinterpreter":28573,"alcohol":28574,"Ġimposition":28575,"Ġdwelling":28576,"Ġ1400":28577,"].\"":28578,"ĠIw":28579,"RM":28580,"Ġ555":28581,"Ġparalyzed":28582,"mind":28583,"rans":28584,"adin":28585,"French":28586,"Ġliar":28587,"Represent":28588,"Ġstrapped":28589,"orate":28590,"Ġrigging":28591,"Ġinterrog":28592,"Ġsparse":28593,"ento":28594,"ĠThem":28595,"Ġbaseless":28596,"Ġbuildup":28597,"Ġundecided":28598,"isms":28599,"Ġabduct":28600,"Ġflowed":28601,"Ġprestige":28602,"Ġhacks":28603,"Ġpanicked":28604,"Cast":28605,"ĠKrish":28606,"umat":28607,"Ġantique":28608,"Ġbitters":28609,"Ġentitlement":28610,"Ġstandby":28611,"Ten":28612,"said":28613,"ĠConditions":28614,"events":28615,"Ġobey":28616,"Ġshortest":28617,"etting":28618,"Ġconcentrating":28619,"ĠNeeds":28620,"234":28621,"Ġintrigued":28622,"enting":28623,"ĠXen":28624,"ĠAlger":28625,"seekers":28626,"anish":28627,"Ġ172":28628,"âĢij":28629,"Ġsilicon":28630,"Ġstandardized":28631,"ĠFountain":28632,"essel":28633,"Ġapproves":28634,"Ġsucked":28635,"gone":28636,"ĠBriggs":28637,"brother":28638,"Ġartisan":28639,"ĠContinuing":28640,"vir":28641,"Ġsubmarines":28642,"ĠInk":28643,"program":28644,"ĠNexus":28645,"ĠCoco":28646,"Ġconceptual":28647,"Ġmatt":28648,"aughters":28649,"Ġbaths":28650,"Ġbeaut":28651,"ĠEmerald":28652,"ĠParties":28653,"248":28654,"completely":28655,"esan":28656,"Ġdiarrhea":28657,"Ġ1100":28658,"borg":28659,"ĠBroken":28660,"Ġreiterate":28661,"Ġsorting":28662,"ONS":28663,"Ġ177":28664,"Ġadmin":28665,"ĠMandatory":28666,"Ġsymptom":28667,"Ġpaced":28668,"Remember":28669,"Ġabdominal":28670,"Ġswapped":28671,"Ġtransitions":28672,"IFA":28673,"pretty":28674,"ĠJC":28675,"Ġallotted":28676,"ĠShows":28677,"Arthur":28678,"Ġsoften":28679,"dozen":28680,"Mah":28681,"Ġextinguished":28682,"Ġreelection":28683,"Ġdeployments":28684,"Ġsturdy":28685,"Ġdownright":28686,"Ġjams":28687,"ĠOptim":28688,"Ġhumiliation":28689,"cd":28690,"Ġbunk":28691,"sie":28692,"NAT":28693,"ilies":28694,"Ġimplying":28695,"Ġ<":28696,"Ġhomepage":28697,"242":28698,"Ġey":28699,"Ġdict":28700,"Ġslender":28701,"Ġforehead":28702,"ĠCecil":28703,"Ġshrunk":28704,"ĠExit":28705,"Ġexpressly":28706,"Ġseals":28707,"ĠThiel":28708,"umni":28709,"Ġdamning":28710,"ĠVS":28711,"ulum":28712,"BBC":28713,"URES":28714,"Ġinhal":28715,"Ġfont":28716,"Ġworkplaces":28717,"ĠPUBLIC":28718,"ĠHorror":28719,"Bs":28720,"arta":28721,"ĠBread":28722,"Ġstret":28723,"Ġethos":28724,"Ġstabilized":28725,"Ġconvers":28726,"ĠInqu":28727,"Ġjudgments":28728,"ĠContemporary":28729,"221":28730,"Ġzombie":28731,"VD":28732,"Ġmisunderstanding":28733,"Ġspam":28734,"ĠPapers":28735,"Ġcrocod":28736,"ENA":28737,"ĠJuven":28738,"ĠAbram":28739,"Ġbursts":28740,"atto":28741,"Ġturbulence":28742,"tty":28743,"sexual":28744,"Ġwaning":28745,"community":28746,"Government":28747,"Ġtranspl":28748,"??":28749,"Getting":28750,"ĠRare":28751,"prime":28752,"Ġlooting":28753,"Ġvalidate":28754,"ĠCreating":28755,"ĠCorruption":28756,"Ġspit":28757,"ĠFavorite":28758,"Kar":28759,"Ġadaptive":28760,"ĠART":28761,"Ġtorso":28762,"ĠIdent":28763,"Ġsubdivision":28764,"azo":28765,"Ġconsequently":28766,"Ġrotate":28767,"ĠWit":28768,"Ġestab":28769,"managed":28770,"ĠBound":28771,"Ġskim":28772,"198":28773,"ĠCorona":28774,"ĠâĿ":28775,"Ġwording":28776,"buck":28777,"iph":28778,"patrick":28779,"Help":28780,"flying":28781,"Ġracer":28782,"Ġfisherman":28783,"____":28784,"ackers":28785,"Ġpersisted":28786,"Ġmyths":28787,"Ġgarn":28788,"ologue":28789,"ĠApprentice":28790,"Ġhereby":28791,"Ġvulgar":28792,"ĠGinger":28793,"Ġtrait":28794,"ĠIdea":28795,"Ġfigur":28796,"ĠSchwarzenegger":28797,"ĠSafari":28798,"178":28799,"ĠAsians":28800,"775":28801,"ĠTriangle":28802,"Ġdemons":28803,"ĠOv":28804,"Ġanime":28805,"Broad":28806,"Ġmolecule":28807,"Ġdeposition":28808,"Ġbiodiversity":28809,"modern":28810,"Ġwallets":28811,"NH":28812,"planes":28813,"rats":28814,"ĠSeed":28815,"Ġ174":28816,"umed":28817,"Ġtouting":28818,"gre":28819,"ĠSEAL":28820,"Ġperpetrator":28821,"ĠGerrard":28822,"Ġallocations":28823,"Ġworsh":28824,"payment":28825,"bett":28826,"ĠIssues":28827,"ennis":28828,"eering":28829,"ĠMV":28830,"yi":28831,"hak":28832,"Ġ167":28833,"Ġorchestr":28834,"224":28835,"Ġsup":28836,"Ġleukemia":28837,"osures":28838,"575":28839,"Ġnoticeably":28840,"Ġparamilitary":28841,"ĠTHERE":28842,"Ġwaged":28843,"igrated":28844,"Ġdocumentaries":28845,"Ġsenseless":28846,"Ġbark":28847,"Ġgenetics":28848,"ĠAlbania":28849,"ĠCrypt":28850,"ĠSEO":28851,"Ġnightly":28852,"Ġfaults":28853,"279":28854,"ĠFerdinand":28855,"ĠSylv":28856,"Ġcalam":28857,"ĠMuller":28858,"ĠSpielberg":28859,"Boy":28860,"ĠUrs":28861,"Ġrug":28862,"Ġcolonies":28863,"ĠFunk":28864,"Ġlyric":28865,"ĠATT":28866,"anni":28867,"ĠNB":28868,"Ġthorn":28869,"Ġpertinent":28870,"188":28871,"Ġpartic":28872,"Head":28873,"Pad":28874,"Palestinian":28875,"ĠBarg":28876,"anical":28877,"beaut":28878,"onge":28879,"Ġgigantic":28880,"travel":28881,"Ġdownloading":28882,"Contin":28883,"whe":28884,"plane":28885,"Wil":28886,"IDA":28887,"Ele":28888,"ĠPAL":28889,"Ġbeams":28890,"ĠProud":28891,"ramer":28892,"Ġindependents":28893,"Ġtranslator":28894,"ĠBrah":28895,"ĠTrooper":28896,"aylor":28897,"pson":28898,"Ġguise":28899,"Ġdiffering":28900,"Ġtopple":28901,"ichen":28902,"ĠSeymour":28903,"deg":28904,"ĠMixed":28905,"Ġinvoluntary":28906,"Ġcountdown":28907,"ĠNarc":28908,"ĠAdults":28909,"Ġcoaster":28910,"Ġ342":28911,"ĠAcquisition":28912,"mone":28913,"Ġpenchant":28914,"Brian":28915,"Gh":28916,"Pres":28917,"enei":28918,"Ġreefs":28919,"ĠMaver":28920,"Ġdevised":28921,"ĠIMP":28922,"vict":28923,"Ġagility":28924,"ĠPayments":28925,"respected":28926,"Ġtuning":28927,"ĠFACE":28928,"actions":28929,"Ġyell":28930,"ĠLeaving":28931,"Ġsnowy":28932,"Saudi":28933,"Ġformations":28934,"Ġairborne":28935,"Ġdeed":28936,"ooks":28937,"Ġnamesake":28938,"Ġpunishable":28939,"Ġagg":28940,"oths":28941,"ĠFamous":28942,"ĠDeposit":28943,"Ġinduce":28944,"189":28945,"Ġhesitation":28946,"ĠBrowse":28947,"ople":28948,"reys":28949,"henko":28950,"Ġsecretaries":28951,"Ġintersections":28952,"Ġdiminishing":28953,"ints":28954,"Ġ1934":28955,"ĠInvestigative":28956,"ĠMexicans":28957,"ĠMahar":28958,"ibur":28959,"Ġstocking":28960,"gross":28961,"Ġasbestos":28962,"Ġagitation":28963,"ĠBST":28964,"Overall":28965,"Ġheats":28966,"ĠSpan":28967,"Ġimped":28968,"Ġtrusting":28969,"Pet":28970,"Ġegregious":28971,"Ġcomedians":28972,"zin":28973,"WIN":28974,"Ġchats":28975,"Ġexploding":28976,"ĠTort":28977,"Ġembraces":28978,"Ġneut":28979,"verson":28980,"ouncing":28981,"ĠFiber":28982,"Ġbaker":28983,"Ġunstoppable":28984,"ĠDial":28985,"cars":28986,"Marc":28987,"164":28988,"volt":28989,"Ġceased":28990,"EFF":28991,"Ġpromoters":28992,"Ġcircuits":28993,"Ġexcise":28994,"Ġseminars":28995,"ĠTiny":28996,"ĠImportant":28997,"ĠTup":28998,"Ġoutburst":28999,"ĠSOC":29000,"ĠWWII":29001,"Ġmerging":29002,"highly":29003,"ĠGmail":29004,"ozy":29005,"ĠKB":29006,"Ġlaboratories":29007,"knit":29008,"ĠClosed":29009,"Ġsurrounds":29010,"ĠVet":29011,"Ġcere":29012,"vard":29013,"ĠDeadpool":29014,"text":29015,"Ġinfusion":29016,"Ġcuc":29017,"ĠAtl":29018,"Ġbustling":29019,"ĠSettings":29020,"Ġ193":29021,"ryan":29022,"184":29023,"186":29024,"Ġswat":29025,"rane":29026,"Ġepidem":29027,"lando":29028,"Ġtestifying":29029,"Ġmoistur":29030,"ĠTens":29031,"Ġexemplary":29032,"ĠPump":29033,"Ġforcefully":29034,"ĠFare":29035,"Ġcomplicate":29036,"Fe":29037,"Di":29038,"ĠThy":29039,"Ġcompartment":29040,"ĠFiesta":29041,"Would":29042,"fitted":29043,"Ġcull":29044,"Ġcomedic":29045,"cyl":29046,"Ġwhichever":29047,"stic":29048,"Ġ213":29049,"Ġspills":29050,"Ġplasma":29051,"Ġdisguise":29052,"ĠCompass":29053,"ĠImmun":29054,"Ġscarf":29055,"Ġdisperse":29056,"Ġreckon":29057,"ĠTaste":29058,"root":29059,"ĠGAME":29060,"xx":29061,"Ġhomophobic":29062,"Ġdimin":29063,"/#":29064,"Ġ178":29065,"Ġgems":29066,"lio":29067,"informed":29068,"ample":29069,"XT":29070,"Ġrepression":29071,"ĠTakes":29072,"Ġhabitats":29073,"Ġmountainous":29074,"ĠMcH":29075,"ENC":29076,"Mobil":29077,"Ġreel":29078,"ĠTI":29079,"Ġauthorize":29080,"ĠAccept":29081,"ĠMetall":29082,"CCC":29083,"Ġwetlands":29084,"ĠWitch":29085,"heading":29086,"Ġintervals":29087,"ĠWitt":29088,"hene":29089,"Ġcomforting":29090,"ollen":29091,"ERN":29092,"ooky":29093,"etch":29094,"Ġassailant":29095,"announced":29096,"elin":29097,"plate":29098,"920":29099,"eating":29100,"induced":29101,"ĠIgor":29102,"ĠAmph":29103,"Ġpatented":29104,"posing":29105,"Ġextraordinarily":29106,"Ġfearless":29107,"mortem":29108,"ĠDraw":29109,"ĠRend":29110,"Son":29111,"ridden":29112,"ĠAdvantage":29113,"Ġ305":29114,"Ġroared":29115,"Str":29116,"Ġradioactive":29117,"Ġslur":29118,"ĠRear":29119,"affles":29120,"ĠPon":29121,"Ġost":29122,"umbs":29123,"ĠSlack":29124,"athom":29125,"baby":29126,"213":29127,"ĠSpending":29128,"ĠAccordingly":29129,"Ġclocks":29130,"archs":29131,"Ġsmugg":29132,"Ġmastermind":29133,"ĠKlaus":29134,"alpha":29135,"Ġspoiled":29136,"264":29137,"Pod":29138,"Ġflared":29139,"Ġcomposure":29140,"ĠCAM":29141,"Ġrestruct":29142,"Ġtasted":29143,"ĠKimber":29144,"Ġupheaval":29145,"CHAR":29146,"ĠGeo":29147,"itations":29148,"Ġbegged":29149,"UX":29150,"Authorities":29151,"ĠEngel":29152,"ĠHOME":29153,"Ġratt":29154,"Ġquickest":29155,"475":29156,"ĠSting":29157,"ĠICO":29158,"yu":29159,"Ġdefy":29160,"Prince":29161,"cards":29162,"Ġovertake":29163,"Ġretrieved":29164,"ĠNavajo":29165,"Ġpastry":29166,"ĠLange":29167,"Ġentrusted":29168,"ĠCull":29169,"aler":29170,"Ġdinosaurs":29171,"Ġbragging":29172,"ĠAlley":29173,"meier":29174,"ĠAssuming":29175,"Ġana":29176,"omatic":29177,"Brend":29178,"acted":29179,"Ġexhaustive":29180,"Ġunfit":29181,"Several":29182,"gap":29183,"Ġtet":29184,"228":29185,"Sk":29186,"302":29187,"Ġdeflect":29188,"Ġ179":29189,"226":29190,"Ġadorned":29191,"ĠSpread":29192,"Ġthirds":29193,"ĠSemi":29194,"Ġdescend":29195,"Ġaccumulate":29196,"Ġflavours":29197,"Ġinvoked":29198,"ĠAnge":29199,"Ġprofess":29200,"unks":29201,"ĠKickstarter":29202,"ENTS":29203,"ĠRw":29204,"Ġchatter":29205,"ĠPOS":29206,"Ġcollaborators":29207,"ĠEW":29208,"ĠMarkus":29209,"Ġimpair":29210,"Ġbolt":29211,"Ġglue":29212,"Ġloosely":29213,"ĠSUM":29214,"Ġhydraulic":29215,"Ġpredatory":29216,"Charles":29217,"cond":29218,"Ġspawned":29219,"Fr":29220,"174":29221,"Ġtame":29222,"Ġaggrav":29223,"Ġchrist":29224,"true":29225,"ivable":29226,"Ġhen":29227,"ĠKut":29228,"Ġskyrocket":29229,"Ġeg":29230,"Ġveterinarian":29231,"ĠStats":29232,"Kit":29233,"Ġbiologist":29234,"Spe":29235,"Ġantenna":29236,"Ġsust":29237,"fill":29238,"Ġpayload":29239,"227":29240,"Ġlivestream":29241,"ORN":29242,"ĠAbel":29243,"Ġdeception":29244,"ussen":29245,"Britain":29246,"partisan":29247,"Ġbrowse":29248,"Ġmelan":29249,"172":29250,"ĠNumerous":29251,"ĠMansion":29252,"Ġassailants":29253,"£":29254,"olerance":29255,"Ġdirectives":29256,"ĠInteg":29257,"zers":29258,"Ġduct":29259,"ĠHonestly":29260,"ĠImmediately":29261,"ixty":29262,"Ġdiagnose":29263,"Ġimplication":29264,"ĠiPads":29265,"testers":29266,"riots":29267,"Ġrespons":29268,"XP":29269,"pes":29270,"875":29271,"Ġ199":29272,"ĠPoe":29273,"303":29274,"Ġailments":29275,"ĠCarrier":29276,"Ġeject":29277,"Ġrestroom":29278,"Drive":29279,"manufact":29280,"Ġcompens":29281,"Ġglossy":29282,"Ġrecovers":29283,"Ġthinner":29284,"Ġdescendants":29285,"antle":29286,"Beaut":29287,"competitive":29288,"ĠRobotics":29289,"Ġpretext":29290,"233":29291,"Ġflanked":29292,"ĠâĻ":29293,"Ġguts":29294,"Ġwee":29295,"Ġaccents":29296,"mc":29297,"Ġgrapp":29298,"ĠNathaniel":29299,"ĠMikhail":29300,"Ġobligated":29301,"Ġmanoeuv":29302,"Ġechoing":29303,"Ġ189":29304,"ĠDevice":29305,"isd":29306,"Ġloopholes":29307,"Ġbehold":29308,"ĠMerry":29309,"Ġfunn":29310,"Ġnuanced":29311,"667":29312,"ELY":29313,"ĠTasmania":29314,"ĠSaddam":29315,"Ġquizz":29316,"military":29317,"cient":29318,"Ġoutlaw":29319,"ĠAudit":29320,"ĠBoom":29321,"Ġcrim":29322,"asured":29323,"ĠApps":29324,"ĠKush":29325,"onica":29326,"Ġamput":29327,"signed":29328,"ĠMEN":29329,"ĠRosenberg":29330,"Ġvide":29331,"ĠDirection":29332,"Ġfountain":29333,"TW":29334,"ĠCARE":29335,"Ġreassured":29336,"Food":29337,"Ġdepressing":29338,"ĠWhilst":29339,"reatment":29340,"Ġspelled":29341,"Ġhipp":29342,"ĠPeach":29343,"hound":29344,"Harry":29345,"Ġcatalogue":29346,"ĠCommun":29347,"Ġnurture":29348,"rush":29349,"ĠPopulation":29350,"ĠNTS":29351,"ĠElectrical":29352,"rounded":29353,"Ġblending":29354,"Ġ223":29355,"alities":29356,"ilation":29357,"eas":29358,"estate":29359,"Ġnarrowing":29360,"ĠTreasure":29361,"192":29362,"Ġwhims":29363,"Ġrobber":29364,"Ġsoaked":29365,"nian":29366,"Ġcongest":29367,"ĠYosemite":29368,"notes":29369,"icer":29370,"ĠGuardians":29371,"ĠFrozen":29372,"Ġ187":29373,"Ġhandcuffs":29374,"Someone":29375,"Ġenshr":29376,"gency":29377,"ĠCube":29378,"Ġprinters":29379,"Ġundercut":29380,"ĠSolution":29381,"rosis":29382,"ĠHumanity":29383,"Ġsucks":29384,"ĠSick":29385,"Tax":29386,"Ġtablespoon":29387,"ĠTrin":29388,"ĠArchive":29389,"Mom":29390,"ĠSAY":29391,"Ġdrifting":29392,"ĠFarage":29393,"Ġforging":29394,"WM":29395,"ĠEleanor":29396,"USH":29397,"Ġemph":29398,"Ġcareless":29399,"Ġspew":29400,"Ġinsensitive":29401,"Ġawhile":29402,"Ġcit":29403,"opened":29404,"ĠFem":29405,"Ġvapor":29406,"Ġdownt":29407,"ylene":29408,"Ġclut":29409,"Ġculp":29410,"1990":29411,"Ġdisgruntled":29412,"Students":29413,"uttering":29414,"gyn":29415,"vre":29416,"Ġrapes":29417,"division":29418,"ĠCalendar":29419,"tal":29420,"icts":29421,"caliber":29422,"ĠFighters":29423,"ĠUnc":29424,"163":29425,"ĠRogue":29426,"Ġregistrations":29427,"Ġundermines":29428,"ĠPunch":29429,"Ġdramas":29430,"176":29431,"Ġslider":29432,"ĠFlore":29433,"ر":29434,"Ġbru":29435,"inelli":29436,"Ġdisparities":29437,"ا":29438,"Ġreferrals":29439,"ĠCharges":29440,"Ġbreeds":29441,"ĠMEP":29442,"288":29443,"Ġmouths":29444,"Ġsideways":29445,"Ġbelievers":29446,"ppard":29447,"Ġhotter":29448,"Ġunderestimated":29449,"Ġjelly":29450,"525":29451,"ĠCMS":29452,"ĠWeiner":29453,"Ġguarding":29454,"Ġampl":29455,"ĠKidd":29456,"UF":29457,"orient":29458,"max":29459,"Ash":29460,"Ġwander":29461,"Ġ..........":29462,"ĠDempsey":29463,"ĠToken":29464,"chat":29465,"Justin":29466,"equipped":29467,"ĠBI":29468,"Ġsins":29469,"Ġnond":29470,"ursion":29471,"Ġcoc":29472,"Ġmailing":29473,"ĠArchitect":29474,"Ġhaunting":29475,"Ġpont":29476,"Ġascertain":29477,"Ġwig":29478,"Ġskysc":29479,"Ġarg":29480,"ĠItalians":29481,"/?":29482,"Ġ----------------------------------------------------------------":29483,"ĠPrecision":29484,"EPA":29485,"Ġhotly":29486,"Ġcircumvent":29487,"ĠEcc":29488,"Ġmerch":29489,"akov":29490,"Ġunab":29491,"heres":29492,"Ġsubcommittee":29493,"ĠDiscuss":29494,"ĠChallenger":29495,"crafted":29496,"Ġcanine":29497,"osphere":29498,"Ġspider":29499,"Ġteachings":29500,"atos":29501,"Ġuniversally":29502,"Ġturbine":29503,"ĠLO":29504,"ĠMAG":29505,"Ġpassers":29506,"Ġroundup":29507,"Ġdenounce":29508,"ĠSpiegel":29509,"until":29510,"Ġshaved":29511,"Ġdisdain":29512,"Nazi":29513,"Ġnewfound":29514,"Ġspontaneous":29515,"Ġmash":29516,"ĠDispatch":29517,"Ġsunrise":29518,"ogged":29519,"Ġfuss":29520,"Ġeas":29521,"acci":29522,"ĠTarg":29523,"Ġhash":29524,"lict":29525,"Ġmisc":29526,"ĠSched":29527,"guy":29528,"linger":29529,"warm":29530,"ipel":29531,"ĠGork":29532,"Ġdispatcher":29533,"Ġ315":29534,"Ġfinely":29535,"Ġreliably":29536,"Ġrupt":29537,"Ġnegligent":29538,"Ġendorsements":29539,"ĠOrient":29540,"Ġelectro":29541,"haired":29542,"Ġphysique":29543,"wine":29544,"Ġadolescents":29545,"Ġ184":29546,"alth":29547,"Ġvalidated":29548,"izzard":29549,"ĠPeck":29550,"Ġemblem":29551,"status":29552,"ĠJungle":29553,"orius":29554,"Ġeccentric":29555,"Ġfolding":29556,"poor":29557,"ĠTHC":29558,"appers":29559,"Ġscripted":29560,"239":29561,"ĠPreferred":29562,"digital":29563,"Ġsharper":29564,"Ġportrays":29565,"rative":29566,"238":29567,"Ġ183":29568,"Ġuneasy":29569,"ĠRI":29570,"Ġvil":29571,"171":29572,"Ġspoil":29573,"ĠPricing":29574,"ĠHardware":29575,"Ġ188":29576,"Ġhorrendous":29577,"Ġostensibly":29578,"nah":29579,"Ġgadget":29580,"ADS":29581,"coat":29582,"Ġexhausting":29583,"Ġdraining":29584,"arate":29585,"ĠBulgarian":29586,"emo":29587,"Ġhier":29588,"Ġguitars":29589,"ieties":29590,"assed":29591,"ĠYaz":29592,"Ġaggress":29593,"ĠBG":29594,"vik":29595,"Ġneatly":29596,"Ġpixel":29597,"Ġintimacy":29598,"ĠRug":29599,"Ġ512":29600,"Ġnarrated":29601,"Ġmast":29602,"ĠNos":29603,"ĠHung":29604,"reciation":29605,"ĠChandra":29606,"Ġbios":29607,"ĠEnded":29608,"lique":29609,"ĠCambod":29610,"Ġworrisome":29611,"ĠEQ":29612,"Ġnovelist":29613,"ĠDynamic":29614,"ĠMIC":29615,"Ġdisposed":29616,"Ġbrackets":29617,"Ġhaircut":29618,"ĠLana":29619,"Ġlull":29620,"Ġbillboard":29621,"ĠReverend":29622,"ĠNAV":29623,"borgh":29624,"Ġadrenaline":29625,"Ġseeming":29626,"ĠPCB":29627,"ĠBridgewater":29628,"Ġsquirrel":29629,"262":29630,"write":29631,"Ġstabilization":29632,"wild":29633,"Ġsecession":29634,"Ġpacket":29635,"AMES":29636,"licted":29637,"Ġmalnutrition":29638,"claimed":29639,"Ġcharred":29640,"Ġtragically":29641,"Published":29642,"Ġrepealed":29643,"ĠSawyer":29644,"ĠMormon":29645,"resolution":29646,"ĠSaud":29647,"Henry":29648,"Ġdiscontin":29649,"Ġsnag":29650,"danger":29651,"Ġmixes":29652,"Ġupbringing":29653,"Ġlimb":29654,"ĠFantastic":29655,"Sim":29656,"ĠAugustine":29657,"ĠGreeks":29658,"cod":29659,"ĠHistorically":29660,"mire":29661,"register":29662,"ĠKund":29663,"Ġdebilitating":29664,"Chat":29665,"ĠTau":29666,"ï":29667,"lower":29668,"pie":29669,"Ġ430":29670,"Ġnascent":29671,"Ġ375":29672,"Ġbum":29673,"WI":29674,"Netflix":29675,"whether":29676,"Ġdearly":29677,"eff":29678,"PRES":29679,"Ġlandmarks":29680,"Ġculminating":29681,"Ġmigrate":29682,"balanced":29683,"Ġregulars":29684,"Ġmodification":29685,"Ġdips":29686,"ĠRedmond":29687,"ationally":29688,"atsu":29689,"Ġphilosophical":29690,"Ġtyping":29691,"Ġunreal":29692,"Ġboiled":29693,"Ġblight":29694,"Ġdru":29695,"ĠGaddafi":29696,"Ġnour":29697,"Ġsequential":29698,"Ġaugment":29699,"ĠEuras":29700,"ĠWiley":29701,"endar":29702,"Ġacronym":29703,"esteem":29704,"ĠMajesty":29705,"Ġgrips":29706,"Ġobsolete":29707,"nos":29708,"Made":29709,"ogie":29710,"ĠLiver":29711,"ĠDonetsk":29712,"Ġdynam":29713,"tel":29714,"bring":29715,"Ġknit":29716,"Ġfirepower":29717,"Ġprepaid":29718,"ĠRaphael":29719,"Ġsensing":29720,"720":29721,"WN":29722,"Nor":29723,"puted":29724,"Ġbureaucrats":29725,"ĠAdjust":29726,"Ġintensely":29727,"Ġsunscreen":29728,"Ho":29729,"ĠYelp":29730,"ĠPU":29731,"ĠSerge":29732,"ĠCyp":29733,"ELF":29734,"ĠGuns":29735,"Ġteamwork":29736,"ĠBib":29737,"ĠMaintenance":29738,"perate":29739,"Ġwiping":29740,"Ġcharcoal":29741,"ordan":29742,"International":29743,"Ġbehaving":29744,"Ġsoftened":29745,"ĠIncreased":29746,"Ġunfl":29747,"470":29748,"Ġinformative":29749,"Ġnovelty":29750,"Ġavoidance":29751,"Ġteasing":29752,"matic":29753,"Ġmaid":29754,"ĠPell":29755,"Ġcounterterrorism":29756,"ĠGabe":29757,"ications":29758,"ĠConnection":29759,"ĠInquiry":29760,"isin":29761,"orama":29762,"Ġcorpse":29763,"Ġpractitioner":29764,"itto":29765,"UA":29766,"Ġforestry":29767,"Ġlic":29768,"Ġrevolves":29769,"Ġcalculating":29770,"Ġpuppet":29771,"ulously":29772,"ĠPebble":29773,"Dep":29774,"Ġupholding":29775,"Ġcarving":29776,"Ġwartime":29777,"Ġenvy":29778,"Ġencro":29779,"ĠPunk":29780,"ĠAdminist":29781,"ucha":29782,"Ġbattleground":29783,"Ġlol":29784,"uable":29785,"Ġunheard":29786,"ĠSpur":29787,"phony":29788,"Ġcarc":29789,"ĠSut":29790,"Ġpollutants":29791,"Cr":29792,"Ġvigorous":29793,"355":29794,"ĠMarriage":29795,"Ġstaffed":29796,"fecture":29797,"ĠArabs":29798,"supported":29799,"Ġmanpower":29800,"ĠSatellite":29801,"None":29802,"Ġqueues":29803,"Ġinsightful":29804,"Ġinterchange":29805,"Rel":29806,"Ġsolemn":29807,"Ġsmuggled":29808,"upt":29809,"Ġ171":29810,"Ġparallels":29811,"intelligence":29812,"punk":29813,"Ġrecycle":29814,"Ġdecorative":29815,"Ġshar":29816,"arrell":29817,"iances":29818,"ĠBolivia":29819,"Ġstrengthens":29820,"430":29821,"Ġhardships":29822,"Ġsignalling":29823,"Ġunthinkable":29824,"READ":29825,"Ġtad":29826,"picked":29827,"Ġarmor":29828,"Ġcores":29829,"ĠMatrix":29830,"Ġdj":29831,"Ġevolutionary":29832,"ĠBermuda":29833,"OE":29834,"organized":29835,"Ġrelentlessly":29836,"sol":29837,"ĠMamm":29838,"Ġpounding":29839,"Weather":29840,"Ġrab":29841,"Ġsweets":29842,"funding":29843,"ĠHUD":29844,"ĠSoldier":29845,"reed":29846,"released":29847,"Ġcontainment":29848,"alid":29849,"ĠNikon":29850,"Ġcervical":29851,"Ġign":29852,"Ġalias":29853,"Ġoptimized":29854,"Ġasserting":29855,"ĠAFTER":29856,"Ġflatt":29857,"Ġdinosaur":29858,"ĠRefugees":29859,"ĠAnch":29860,"Ġadjustable":29861,"Ġroaring":29862,"Ġpilgrimage":29863,"Ġcowboy":29864,"Ġentails":29865,"ractions":29866,"EY":29867,"undy":29868,"ĠKuh":29869,"inges":29870,"ĠTerra":29871,"ĠEscape":29872,"Ġrundown":29873,"Ġstriped":29874,"KN":29875,"ocations":29876,"IDENT":29877,"IGH":29878,"Ġavoids":29879,"Moh":29880,"ĠLS":29881,"lbs":29882,"ĠAttempt":29883,"Ġtriangle":29884,"Ġclimax":29885,"Ġhp":29886,"Ġallot":29887,"learning":29888,"ĠJFK":29889,"Justice":29890,"OUT":29891,"ĠHER":29892,"ĠLect":29893,"Ġtrench":29894,"edar":29895,"Ġreservoirs":29896,"uid":29897,"rf":29898,"162":29899,"Ġinterfered":29900,"Ġemit":29901,"these":29902,"444":29903,"ĠLeather":29904,"essing":29905,"ĠEighth":29906,"uckle":29907,"Breaking":29908,"Ġunresolved":29909,"Ġgoose":29910,"252":29911,"platform":29912,"atus":29913,"Ġcomplexion":29914,"ĠBUS":29915,"Ġstruct":29916,"middle":29917,"Sat":29918,"ĠWHERE":29919,"LB":29920,"redible":29921,"vered":29922,"Louis":29923,"ĠBaz":29924,"Eye":29925,"safety":29926,"Ġhypothetical":29927,"Ġbowel":29928,"Ġuntouched":29929,"312":29930,"ĠPric":29931,"Ġastounding":29932,"meet":29933,"Aaron":29934,"ĠWoo":29935,"236":29936,"ĠShape":29937,"Ġdrifted":29938,"Ġtile":29939,"ĠGrim":29940,"Ġundeniable":29941,"Ġ..":29942,"Ġradius":29943,"Ġovarian":29944,"ĠSeriously":29945,"verning":29946,"Ġassertions":29947,"oxic":29948,"231":29949,"ĠViz":29950,"Jackson":29951,"ĠSno":29952,"Ġboycot":29953,"okingly":29954,"ousse":29955,"proclaimed":29956,"Ġblazing":29957,"Ġinefficient":29958,"Ġfig":29959,"Ġbooze":29960,"259":29961,"agus":29962,"statement":29963,"Ġlocom":29964,"Ġtacos":29965,"Ġmemos":29966,"gender":29967,"ĠOrt":29968,"263":29969,"Ġintervening":29970,"Soc":29971,"University":29972,"ĠPis":29973,"ĠReturns":29974,"ĠPAN":29975,"Ġultrasound":29976,"Ġcoherent":29977,"tracking":29978,"rieved":29979,"383":29980,"Ġqualitative":29981,"uld":29982,"ĠGiovanni":29983,"Ġstorylines":29984,"Ġdarkest":29985,"Ġvelvet":29986,"RIP":29987,"Ġcompatibility":29988,"Ġtroll":29989,"CN":29990,"Found":29991,"ĠOu":29992,"Ġtease":29993,"Ġvested":29994,"Ġprovocation":29995,"Ġimprovised":29996,"Ġactivation":29997,"unte":29998,"ĠMonteneg":29999,"ĠJOHN":30000,"ĠReact":30001,"Ġpolluted":30002,"217":30003,"Ġmushroom":30004,"Ġdisconnected":30005,"ĠVoices":30006,"asu":30007,"Ġsensory":30008,"REE":30009,"Ġmonarchy":30010,"Ġ173":30011,"doing":30012,"involved":30013,"ĠJonah":30014,"Ġtoxins":30015,"Ġtv":30016,"Ġacademia":30017,"IQ":30018,"Mor":30019,"ĠStraight":30020,"ĠRN":30021,"ĠâĹı":30022,"Ġpear":30023,"187":30024,"Ġendeavors":30025,"ĠTurbo":30026,"Ġducks":30027,"ĠRamsay":30028,"Ġoutpatient":30029,"Ġcomprehend":30030,"UNE":30031,"Ġbriefings":30032,"total":30033,"Ġmigr":30034,"always":30035,"Ġmoot":30036,"ĠRider":30037,"Ġbiblical":30038,"Form":30039,"Ġcurry":30040,"Ġexquisite":30041,"385":30042,"244":30043,"Ġattendants":30044,"Ġcabinets":30045,"nton":30046,"Baby":30047,"Honestly":30048,"ĠFIRE":30049,"211":30050,"itech":30051,"ĠProsper":30052,"Ġchops":30053,"odic":30054,"Rod":30055,"job":30056,"orset":30057,"ĠAry":30058,"obic":30059,"ĠNil":30060,"isable":30061,"Ġorche":30062,"Ġtrivial":30063,"ĠZy":30064,"ĠXP":30065,"Ġendorsing":30066,"ĠLIM":30067,"adish":30068,"237":30069,"ĠLaws":30070,"heid":30071,"ĠSignature":30072,"ĠVern":30073,"ĠBland":30074,"ansk":30075,"Ġrepository":30076,"ĠPetra":30077,"Enter":30078,"Ġtruths":30079,"Ġbordering":30080,"Ġpenn":30081,"Ġsimplified":30082,"zn":30083,"ĠCree":30084,"Ġ181":30085,"Hi":30086,"ĠGreenberg":30087,"Ġprematurely":30088,"ĠSass":30089,"Ġwrecked":30090,"Ġheinous":30091,"415":30092,"Turn":30093,"zl":30094,"amental":30095,"ĠBraz":30096,"fing":30097,"ĠAngle":30098,"ĠPhantom":30099,"agra":30100,"ĠShack":30101,"Ġhomegrown":30102,"Ġalright":30103,"AME":30104,"ĠKN":30105,"Ġclicks":30106,"Ġmanned":30107,"ĠScope":30108,"Ġextras":30109,"Ġclinicians":30110,"321":30111,"African":30112,"Ġjuices":30113,"Ġrefere":30114,"****":30115,"ambling":30116,"since":30117,"Ġvoic":30118,"QB":30119,"ĠAtmospheric":30120,"Mat":30121,"Ġperpetrated":30122,"ĠSteps":30123,"Fit":30124,"Ġsilenced":30125,"Ġbonded":30126,"Ġquantify":30127,"Houston":30128,"ocracy":30129,"Ġfreeing":30130,"pipe":30131,"corn":30132,"rones":30133,"ooked":30134,"ĠSuz":30135,"Ġunaccount":30136,"196":30137,"Ġlogos":30138,"ĠFurious":30139,"ĠSpart":30140,"urst":30141,"itri":30142,"ĠZub":30143,"ĠActual":30144,"Ġslee":30145,"Ġgag":30146,"Ġmetabolism":30147,"ĠDesigned":30148,"Ġpedigree":30149,"Ġcoolest":30150,"âĿ":30151,"iuses":30152,"ĠYellowstone":30153,"Ġinformant":30154,"Ġushered":30155,"ĠGarg":30156,"thel":30157,"Hop":30158,"Ġrepetitive":30159,"flag":30160,"Ġunmarked":30161,"ĠBrave":30162,"Ġincur":30163,"reading":30164,"ppel":30165,"lah":30166,"ateurs":30167,"286":30168,"ĠAtomic":30169,"Ġappliance":30170,")'":30171,"traditional":30172,"Ġdads":30173,"Ġregimen":30174,"Ġinfrared":30175,"Ġdotted":30176,"Ġtails":30177,"Ġhorrors":30178,"uments":30179,"Ġdub":30180,"lighting":30181,"Ġunearthed":30182,"assisted":30183,"ĠSpiel":30184,"trial":30185,"Ġpersever":30186,"MAX":30187,"Ġicing":30188,"Energy":30189,"Ġ1943":30190,"move":30191,"Error":30192,"Ġliter":30193,"ĠCly":30194,"Ari":30195,"Ġgranite":30196,"Ġcropped":30197,"ĠRD":30198,"ĠREM":30199,"TX":30200,"Ġdispleasure":30201,"ĠComfort":30202,"Ġunsettling":30203,"Ġscratching":30204,"866":30205,"eton":30206,"560":30207,"Ġcommonplace":30208,"Ġreproduced":30209,"ggie":30210,"Ġschooling":30211,"Ġreprim":30212,"Ġdarling":30213,"huge":30214,"ĠDante":30215,"cp":30216,"heastern":30217,"Ġeduc":30218,"Digital":30219,"Ġwrath":30220,"Ġwatering":30221,"ĠTail":30222,"Ġdegradation":30223,"530":30224,"usive":30225,"ĠXu":30226,"ĠAH":30227,"Ġclassy":30228,"ĠSET":30229,"Ġcriminally":30230,"dependent":30231,"ĠAlps":30232,"Ġnotwithstanding":30233,"Ġfamiliarity":30234,"ĠAPP":30235,"aurus":30236,"gments":30237,"Mid":30238,"Ġepilepsy":30239,"Ġresemblance":30240,"brush":30241,"Ġ333":30242,"Ġliberated":30243,"ĠBeng":30244,"ĠLans":30245,"Ġtraff":30246,"ihu":30247,"establish":30248,"Ġcort":30249,"Rick":30250,"Ġplugged":30251,"onement":30252,"ĠAccounting":30253,"Ġreconstruct":30254,"Pop":30255,"Ġincapable":30256,"aho":30257,"ĠDexter":30258,"Ġpitted":30259,"Ġbathing":30260,"Ġdun":30261,"Ġexplor":30262,"ĠMidnight":30263,"Ġactiv":30264,"iann":30265,"likely":30266,"acons":30267,"owicz":30268,"Ġnegativity":30269,"Ġfreel":30270,"ewitness":30271,"Ġinj":30272,"Stephen":30273,"Ġshredded":30274,"Ġprepar":30275,"Script":30276,"Ġcorrectional":30277,"Ġcommits":30278,"hai":30279,"activity":30280,"Imp":30281,"Ġstumble":30282,"Ġcache":30283,"ĠPromise":30284,"Ġprecinct":30285,"Ġmulticultural":30286,"Ġsubstitutes":30287,"Ġshortened":30288,"ovable":30289,"Ġfasting":30290,"Ġinfused":30291,"Ġbulldo":30292,"alm":30293,"Ġadjoining":30294,"Ġmultiplayer":30295,"ĠAlien":30296,"Ġpund":30297,"ethyl":30298,"Ġbliss":30299,"ĠDecision":30300,"Ġbab":30301,"Ġangrily":30302,"another":30303,"oled":30304,"ainted":30305,"ĠPriest":30306,"Ġdraped":30307,"ĠPersonally":30308,"Ġstomp":30309,"ĠWolfgang":30310,"Ġoste":30311,"itches":30312,"Ġhoops":30313,"ĠJO":30314,"Ġsche":30315,"ĠZan":30316,"Ġcleans":30317,"Ġclimbs":30318,"Ġelectronically":30319,"243":30320,"ocy":30321,"gall":30322,"ĠREAL":30323,"Ġmurky":30324,"Ġmodernization":30325,"tub":30326,"Really":30327,"Ġlax":30328,"Ġdoubted":30329,"yden":30330,"ĠPrevent":30331,"UTERS":30332,"Ġoverride":30333,"ĠSAF":30334,"Ġcoun":30335,"Ġexcerpts":30336,"Ġmotivations":30337,"Ġdecency":30338,"Ġastronomers":30339,"orical":30340,"Ġaltering":30341,"Ġ232":30342,"described":30343,"omic":30344,"Ġexh":30345,"Ġknocks":30346,"ĠRiot":30347,"ĠPurs":30348,"equal":30349,"pleting":30350,"llan":30351,"ĠSOL":30352,"iator":30353,"ILE":30354,"ĠWM":30355,"Ġdefences":30356,"Ġforearm":30357,"Toronto":30358,"526":30359,"Ġacne":30360,"Ġthirteen":30361,"itiz":30362,"akable":30363,"charges":30364,"Ġinaction":30365,"Ġbred":30366,"Ġdeficiency":30367,"Ġintrigue":30368,"opoly":30369,"ĠCamer":30370,"ĠMelt":30371,"Ġunlawfully":30372,"Ġpenetrate":30373,"ĠUsed":30374,"ĠDirty":30375,"Ġexcerpt":30376,"ĠYen":30377,"ĠCARD":30378,"Ġcher":30379,"ĠChallenges":30380,"ieves":30381,"Ġambush":30382,"Data":30383,"eeks":30384,"Ġgiveaway":30385,"Ġpawn":30386,"Ġtransf":30387,"renched":30388,"Ġmoderately":30389,"Ġnumbered":30390,"ĠIntegrity":30391,"ĠHOU":30392,"ĠHDMI":30393,"Royal":30394,"LT":30395,"ĠDirk":30396,"izon":30397,"Ġ227":30398,"Ġdisagrees":30399,"ĠNinth":30400,"Ġincrement":30401,"ĠGlory":30402,"suff":30403,"Ġartery":30404,"ĠEmployee":30405,"bum":30406,"ĠEditorial":30407,"Kh":30408,"ĠPremiere":30409,"ĠWeld":30410,"ĠIncluded":30411,"Ġmathematical":30412,"Ġexponentially":30413,"Ġhandwritten":30414,"ĠMAS":30415,"Ġindiscrim":30416,"Ġnutrient":30417,"ĠSelection":30418,"Ġ219":30419,"hyd":30420,"Ġdeton":30421,"æ":30422,"dark":30423,"ĠFidel":30424,"Ġmonkeys":30425,"Ġnutritious":30426,"Ġheadlights":30427,"oller":30428,"piring":30429,"ĠDefenders":30430,"Ġdrown":30431,"elong":30432,"Ġfloats":30433,"graduate":30434,"Ġprosper":30435,"ĠNamed":30436,"ĠEating":30437,"ECK":30438,"establishment":30439,"XM":30440,"Ġsoaking":30441,"278":30442,"Ġlistener":30443,"Ġsimultaneous":30444,"olutions":30445,"payer":30446,"Ġcustomize":30447,"ĠROCK":30448,"Ġaltar":30449,"ĠExercise":30450,"anky":30451,"ĠProfession":30452,"sever":30453,"ĠMerchant":30454,"RF":30455,"ĠCombat":30456,"Ġlegality":30457,"fledged":30458,"Ġdiapers":30459,"lves":30460,"Ġlur":30461,"Ġignores":30462,"ĠProtocol":30463,"Ġrepresentations":30464,"ĠBlumenthal":30465,"ĠLime":30466,"romptu":30467,"Ġbesieged":30468,"dl":30469,"Ġsighting":30470,"ĠParm":30471,"ĠServer":30472,"ĠBenghazi":30473,"estival":30474,"Ġplaylist":30475,"ĠUng":30476,"ĠQuantum":30477,"Ġcompromises":30478,"ĠSurvivor":30479,"ĠMobility":30480,"Ġbounty":30481,"ophers":30482,"ISA":30483,"need":30484,"uese":30485,"Ġorn":30486,"218":30487,"Ġ530":30488,"Ġbuddies":30489,"Ġagendas":30490,"ĠFeldman":30491,"ĠÃĸ":30492,"ĠBMC":30493,"ĠServe":30494,"Ent":30495,"ĠKH":30496,"ĠINT":30497,"Ġlittered":30498,"Ġvisitation":30499,"mist":30500,"Ġdupl":30501,"Ġrouted":30502,"ĠAmount":30503,"Dev":30504,"ĠConv":30505,"Ġslams":30506,"ĠVeterinary":30507,"bold":30508,"Ġ186":30509,"ĠDOT":30510,"builder":30511,"Ġdecay":30512,"ĠHemp":30513,"pelled":30514,"Ġmankind":30515,"Tonight":30516,"Ġeffortlessly":30517,"ĠBUT":30518,"Ġhostilities":30519,"formerly":30520,"alon":30521,"ĠCrash":30522,"humane":30523,"Ġmayhem":30524,"ĠBudd":30525,"Ġdisinformation":30526,"Ġ226":30527,"Ġprototypes":30528,"__":30529,"IVERS":30530,"izzy":30531,"ĠMight":30532,"ĠPip":30533,"pour":30534,"INO":30535,"ĠLL":30536,"Ġwiret":30537,"Ġresorted":30538,"ĠTanaka":30539,"ĠDOES":30540,"Earlier":30541,"HO":30542,"Ġmoniker":30543,"ĠFang":30544,"ĠHua":30545,"bered":30546,"adding":30547,"194":30548,"STR":30549,".\")":30550,"cop":30551,"ĠFlags":30552,"ĠColleges":30553,"ĠUz":30554,"Ġsparks":30555,"Ġparadox":30556,"Marie":30557,"Strong":30558,"Ġstrawberry":30559,"Ġnurturing":30560,"Ġfax":30561,"Tor":30562,"killer":30563,"burse":30564,"Ġattachments":30565,"Ġpup":30566,"Ġexhaustion":30567,"Ġwhisky":30568,"isu":30569,"ologically":30570,"iership":30571,"Ġlamps":30572,"Ġshuff":30573,"Ġcentralized":30574,"ĠNeedless":30575,"Ġgrenade":30576,"Ġrouter":30577,"Ġoptics":30578,"ivering":30579,"Ġpioneers":30580,"ĠHug":30581,"Ġhandguns":30582,"010":30583,"Ġbailed":30584,"uana":30585,"197":30586,"Ġdistorted":30587,"ĠEssentially":30588,"ĠSilent":30589,"Ġcomparative":30590,"Music":30591,"ĠMUS":30592,"Bur":30593,"ĠComet":30594,"ĠWinchester":30595,"IGN":30596,"Mod":30597,"ĠCandidate":30598,"Ġdysfunctional":30599,"ĠCeleb":30600,"Ġhitch":30601,"api":30602,"Ġidiot":30603,"Ġunsupported":30604,"gat":30605,"inker":30606,"Ġredevelop":30607,"Ġdwind":30608,"Ġforgetting":30609,"ĠRost":30610,"Ġremembrance":30611,"Na":30612,"mopolitan":30613,"Ġberries":30614,"Ġmarital":30615,"Vol":30616,"ĠClosing":30617,"ĠHindus":30618,"itism":30619,"Ġrover":30620,"Ġmysteries":30621,"ĠNig":30622,"ucing":30623,"Ġfabrication":30624,"Ġgarments":30625,"Ġwield":30626,"ĠCompton":30627,"357":30628,"Ġoxide":30629,"chron":30630,"ĠThought":30631,"Ġcomed":30632,"ĠEpstein":30633,"ĠBART":30634,"orative":30635,"ĠKahn":30636,"adan":30637,"APH":30638,"cum":30639,"Ġloophole":30640,"ĠGoPro":30641,"osit":30642,"Ġspecification":30643,"ĠAPR":30644,"Ġdrains":30645,"Ġconserve":30646,"ĠMorse":30647,"Ġcalorie":30648,"ĠCheney":30649,"station":30650,"Ġevangel":30651,"Ġspraying":30652,"lections":30653,"Ġenclosure":30654,"Ġcommanded":30655,"ĠOrganizations":30656,"Ġimb":30657,"mins":30658,"ĠTobias":30659,"Ve":30660,"ĠNau":30661,"183":30662,"ĠGuantanamo":30663,"173":30664,"Ġrequisite":30665,"Ġderivative":30666,"Ġpopulism":30667,"Ġcultivated":30668,"lord":30669,"uler":30670,"ĠDEA":30671,"inally":30672,"Ġdemonstr":30673,"trip":30674,"ĠFirefox":30675,"246":30676,"confirmed":30677,"Anne":30678,"Ġtamp":30679,"ĠHousehold":30680,"amous":30681,"Meet":30682,"Ġdashed":30683,"pire":30684,"Ġinex":30685,"Ġloosen":30686,"272":30687,"famous":30688,"ĠHeard":30689,"Ġhindsight":30690,"Ġdepot":30691,"ĠCutting":30692,"ĠMouse":30693,"Ġgeological":30694,"number":30695,"OUN":30696,".,\"":30697,"Ġmoderation":30698,"ĠUNHCR":30699,"Ġdomains":30700,"eco":30701,"Ġcrater":30702,"Ġ510":30703,"kid":30704,"Ġcylinders":30705,"ĠClasses":30706,"Kn":30707,"Ġcarcin":30708,"ĠHunting":30709,"irit":30710,"ARP":30711,"anting":30712,"ĠMarino":30713,"ĠRESP":30714,"ifle":30715,"Ġ239":30716,"fman":30717,"Ġtheoretically":30718,"Ġdistraught":30719,"Ġstaircase":30720,"Ġexpel":30721,"Ġlord":30722,"Ġbehaviours":30723,"Ġprescribing":30724,"ographs":30725,"ĠNewly":30726,"Ġpatiently":30727,"Ġskyline":30728,"udos":30729,"Ġrepertoire":30730,"Ġhover":30731,"mint":30732,"Ġclears":30733,"Ġkale":30734,"ĠSco":30735,"ĠCoulter":30736,"Ġpancreat":30737,"pu":30738,"995":30739,"Ġincompetent":30740,"2007":30741,"Ġgripping":30742,"enable":30743,"Ġreinforcing":30744,"ĠFee":30745,"education":30746,"ĠKuro":30747,"Ġbowed":30748,"Ġshave":30749,"ĠMean":30750,"xi":30751,"Ġinciting":30752,"atters":30753,"Ġecstatic":30754,"hog":30755,"Ġclauses":30756,"Ġsubt":30757,"Ġbehaved":30758,"tains":30759,"Liverpool":30760,"Ġstrives":30761,"ĠKev":30762,"ĠFramework":30763,"defined":30764,"Ġrecounts":30765,"array":30766,"tips":30767,"Ġartificially":30768,"fits":30769,"Clearly":30770,"mediate":30771,"Ġunseen":30772,"Ġthugs":30773,"ĠLent":30774,"Ġ1938":30775,"Ġgenital":30776,"ĠSonic":30777,"ĠWarehouse":30778,"pler":30779,"Ġunm":30780,"Ġpackets":30781,"ĠMET":30782,"ealous":30783,"ographers":30784,"Ġlabou":30785,"Core":30786,"+,":30787,"parable":30788,"Ġstrat":30789,"Ġinvitations":30790,"Ġsouven":30791,"Ġbillboards":30792,"ĠRegulations":30793,"Ġdwarf":30794,"Ġtoler":30795,"Ġprose":30796,"Ġestates":30797,"Ġmetabolic":30798,"ĠSuff":30799,"ĠFirstly":30800,"Ġpolio":30801,"Ġchick":30802,"ĠDaughter":30803,"Ġsubstant":30804,"ĠIdentity":30805,"umbers":30806,"ĠFacts":30807,"Ġfrust":30808,"Ġdissip":30809,"ĠDeck":30810,"Hy":30811,"ĠBirch":30812,"Ġhurled":30813,"democracy":30814,"nered":30815,"eper":30816,"Ġcerebral":30817,"181":30818,"Ġhalves":30819,"abit":30820,"balance":30821,"ĠTibet":30822,"Ġhandheld":30823,"ĠDough":30824,"Ġprogrammed":30825,"hw":30826,"Ġoutlawed":30827,"ĠSerious":30828,"Ġironically":30829,"Ġmanipulating":30830,")\"":30831,"juries":30832,"Ġfragrance":30833,"crete":30834,"ĠHHS":30835,"cience":30836,"Ġcosmic":30837,"Ġforeclosure":30838,"Ġpercentages":30839,"Bus":30840,"Ġenticing":30841,"extra":30842,"ĠShy":30843,"ĠÂ¥":30844,"Ġheadsets":30845,"imensional":30846,"Ġlux":30847,"Ġresidual":30848,"Ġmantle":30849,"ĠSJ":30850,"ĠPeaks":30851,"ĠFinger":30852,"Ġunfolds":30853,"anity":30854,"Ġresettlement":30855,"ĠWeak":30856,"ĠBeen":30857,"Ġ198":30858,"Ġangels":30859,"ĠFarn":30860,"peace":30861,"Ġcapac":30862,"Ġhue":30863,"Ġlust":30864,"traumatic":30865,"laun":30866,"Ġstrawberries":30867,"Ġherbal":30868,"Ġconversions":30869,"ĠHeld":30870,"Ġprescribe":30871,"Its":30872,"ĠDartmouth":30873,"Ġfashioned":30874,"460":30875,"BLE":30876,"international":30877,"Ġlumin":30878,"Ġplantation":30879,"ilde":30880,"490":30881,"Ġeuph":30882,"Ġdisgust":30883,"Ġaspire":30884,"medical":30885,"Ġsocialism":30886,"Ġdissolve":30887,"Wal":30888,"Ġadmittedly":30889,"Ġsewing":30890,"ĠAcer":30891,"Ġtul":30892,"Ġfacilit":30893,"Ġgrandma":30894,"ĠFeeling":30895,"Ġobst":30896,"ĠFranz":30897,"ĠPalin":30898,"ĠIncrease":30899,"gets":30900,"ĠImam":30901,"âĢİ":30902,"Ġcoincides":30903,"urrence":30904,"Ġlifes":30905,"Lab":30906,"Ham":30907,"angelo":30908,"Wild":30909,"Ġvetoed":30910,"Ġventilation":30911,"olid":30912,"Summer":30913,"Ġfacade":30914,"neys":30915,"ĠWOM":30916,"ĠBenny":30917,"ĠMarried":30918,"squ":30919,"ĠReflect":30920,"return":30921,"elia":30922,"olding":30923,"Ġrefine":30924,"ĠMadness":30925,"innacle":30926,"posts":30927,"287":30928,"fruit":30929,"274":30930,"icator":30931,"ĠVoy":30932,"Ġunsett":30933,"Ġfant":30934,"Ġtreaties":30935,"Ġcrystals":30936,"Ġhijacked":30937,"words":30938,"ĠReleased":30939,"Save":30940,"Ġcannon":30941,"Ġanomaly":30942,"Ġbeacon":30943,"Ġcrippled":30944,"Ġbundles":30945,"Ġuntreated":30946,"Ġhappiest":30947,"Ġgalaxies":30948,"Ġoccupational":30949,"416":30950,"Dar":30951,"Ġcrank":30952,"Ġappropriation":30953,"asking":30954,"mens":30955,"Ġdetector":30956,"Ġskewed":30957,"Ġpoke":30958,"254":30959,"Ġhypertension":30960,"apolog":30961,"Ġevaluations":30962,"blocks":30963,"Ġpow":30964,"GEN":30965,"Ġscalp":30966,"Ġarrogant":30967,"AIDS":30968,"ority":30969,"Ġredirect":30970,"Ġderogatory":30971,"Ġlateral":30972,"495":30973,"rolley":30974,"brew":30975,"Ġbabys":30976,"Ġmuff":30977,"ĠRequ":30978,"Ġdime":30979,"Ġwonderfully":30980,"Ġtreasures":30981,"ĠNES":30982,"Ġponds":30983,"Ġimpulse":30984,"Ġdetecting":30985,"Ġgrin":30986,"Ġbrid":30987,"Ġshoved":30988,"Ġpurge":30989,"irteen":30990,"OTHER":30991,"ÙĦ":30992,"irsch":30993,"ĠOcc":30994,"193":30995,"Ġfodder":30996,"wrote":30997,"meric":30998,"posal":30999,"Ġwinters":31000,"ĠJuice":31001,"hub":31002,"Ġcontrasting":31003,"Brazil":31004,"Ġflashy":31005,"uffer":31006,"technology":31007,"Children":31008,"Ġcatapult":31009,"owsky":31010,"ĠEclipse":31011,"abeth":31012,"ĠParticip":31013,"Ġlaud":31014,"ĠQuiet":31015,"Ġsimulations":31016,"Ġsacrificing":31017,"Ġpreaching":31018,"Ġvoicing":31019,"itizen":31020,"Ġgn":31021,"Ġsans":31022,"Ġ285":31023,"ĠRobot":31024,"Ġ1936":31025,"Ġsham":31026,"ĠKislyak":31027,"ĠGCC":31028,"tale":31029,"ĠShades":31030,"Ġsediment":31031,"Ġconveniently":31032,"Give":31033,"mounted":31034,"Ġpeel":31035,"Jun":31036,"ĠEisenhower":31037,"Ġdiplom":31038,"ĠPreservation":31039,"Ġaffirm":31040,"Ġtaboo":31041,"ĠGarr":31042,"ĠApply":31043,"prim":31044,"Ġausp":31045,"Ġtextbook":31046,"Ġforfeit":31047,"icides":31048,"Ġundis":31049,"DJ":31050,"Ġ\"...":31051,"ĠXperia":31052,"Ġfurry":31053,"Australian":31054,"Ġpreach":31055,"Ġparamed":31056,"Ġ196":31057,"agos":31058,"ĠRIP":31059,"Ġ408":31060,"ĠQuarterly":31061,"ĠQuentin":31062,"Ġdeft":31063,"ĠVlad":31064,"massive":31065,"apore":31066,"Ġquestionnaire":31067,"secution":31068,"ĠTunnel":31069,"ĠAssist":31070,"BILITY":31071,"everything":31072,"vich":31073,"Ġcomparatively":31074,"heng":31075,"ETH":31076,"ĠiPod":31077,"Ġinsurgent":31078,"Ġtestosterone":31079,"191":31080,"Ġmoons":31081,"Ġgripped":31082,"Ġstrang":31083,"pects":31084,"ĠSERVICE":31085,"Ġnumb":31086,"Ġmeasurable":31087,"Ġdismantled":31088,"Ġdepict":31089,"Ġretake":31090,"Light":31091,"Ġaquatic":31092,"useum":31093,"judicial":31094,"Ġ****":31095,"Ġrosters":31096,"certain":31097,"Ġhypothesis":31098,"2002":31099,"Snow":31100,"Ġpounded":31101,"ĠZel":31102,"ĠTrem":31103,"iversity":31104,"219":31105,"Jen":31106,"ĠAdventures":31107,"Ġcylinder":31108,"Ġbanging":31109,"Ġbalk":31110,"analy":31111,"ĠHust":31112,"ookie":31113,"ĠReturning":31114,"Ġpods":31115,"analysis":31116,"ĠTruman":31117,"Ġorg":31118,"Ġsar":31119,"Ġdred":31120,"ĠTelecommunications":31121,"ĠSven":31122,"carry":31123,"ĠLOVE":31124,"Ġparting":31125,"asar":31126,"utations":31127,"itic":31128,"Ġactu":31129,"Ġbananas":31130,"ĠNights":31131,"410":31132,"Still":31133,"Ġtweaked":31134,"went":31135,"Ġtoddlers":31136,"irted":31137,"Ġpaed":31138,"ĠWink":31139,"Ġviewpoint":31140,"ĠHelic":31141,"Ġhandshake":31142,"Ġpoaching":31143,"Ġrounding":31144,"268":31145,"ĠNVIDIA":31146,"Ġsquat":31147,"Ġtowed":31148,"Ġhandler":31149,"Ġconspir":31150,"Ġadditionally":31151,"CENT":31152,"ĠÃľ":31153,"article":31154,"ĠTough":31155,"NM":31156,"Rem":31157,"Ġstunts":31158,"ILS":31159,"ĠLM":31160,"Connect":31161,"ĠParagu":31162,"Ġcomplexities":31163,"Ġhugging":31164,"Ġabolish":31165,"ricting":31166,"ĠItems":31167,"Ġtemples":31168,"ĠSeat":31169,"ĠRubber":31170,"Ġindic":31171,"ĠVitamin":31172,"Ġcitations":31173,"Ġarmored":31174,"---------------":31175,"ĠNeo":31176,"ippy":31177,"Que":31178,"Ġrag":31179,"Ġlov":31180,"630":31181,"Ġadept":31182,"orbit":31183,"253":31184,"412":31185,"Ġbutterflies":31186,"Ġoutl":31187,"ĠCycle":31188,"Ġaesthetics":31189,"ĠTwitch":31190,"405":31191,"factor":31192,"ðŁij":31193,"ĠCircus":31194,"Posted":31195,"Ġintroductory":31196,"ĠStack":31197,"atoes":31198,"Ġfurn":31199,"ĠHond":31200,"Ġbipolar":31201,"ĠAging":31202,"inches":31203,"Ġincompetence":31204,"Ġaloud":31205,"Imagine":31206,"Ġsepar":31207,"Ġmanip":31208,"ophobic":31209,"inion":31210,"bek":31211,"Ġquer":31212,"ĠArmen":31213,"Ġhumorous":31214,"Ġmundane":31215,"Ġapologizing":31216,"Ġpioneered":31217,"Ġ303":31218,"282":31219,"Ġcalming":31220,"orious":31221,"760":31222,"Ġstitches":31223,"Ġthrottle":31224,"Ġspinach":31225,"urities":31226,"ĠCologne":31227,"Ġripple":31228,"Cs":31229,"Cent":31230,"Should":31231,"Ġaffinity":31232,"amount":31233,"ĠMISS":31234,"Ġsage":31235,"Ġamusing":31236,"Ġsnatch":31237,"clair":31238,"ĠGuess":31239,"bench":31240,"ĠMoj":31241,"nuclear":31242,"Ġfid":31243,"ĠVM":31244,"ĠGN":31245,"brainer":31246,"Ġcurled":31247,"Ġbushes":31248,"icably":31249,"Ġcreeping":31250,"Ġveil":31251,"ĠALS":31252,"ESPN":31253,"ulsion":31254,"ĠGTX":31255,"ĠANN":31256,"Ġcomplicit":31257,"assault":31258,"IOR":31259,"Ġpolymer":31260,"Ġestimating":31261,"277":31262,"alog":31263,"Ġglimps":31264,"Ġreinforces":31265,"Ġtextbooks":31266,"Ġdictated":31267,"ĠReyn":31268,"latable":31269,"ĠOrth":31270,"520":31271,"Ġtrickle":31272,"ĠWrong":31273,".[":31274,"ĠDesigner":31275,"304":31276,"ĠInner":31277,"Ġrave":31278,"ppa":31279,"ĠGim":31280,"Ġswath":31281,"Ġcarts":31282,"atlantic":31283,"Ġpersists":31284,"ĠDeveloper":31285,"Ġgoodies":31286,"isive":31287,"Inf":31288,"ĠSaving":31289,"loop":31290,"tions":31291,"Ġabusers":31292,"Ġclot":31293,"Ġmesmer":31294,"Ġdeg":31295,"Ġskirts":31296,"257":31297,"Ġunreliable":31298,"ĠCOMM":31299,"Ġ194":31300,"Ġfledgling":31301,"administ":31302,"Israeli":31303,"ĠBarbie":31304,"ĠJeanne":31305,"Ġgenerously":31306,"ĠStruct":31307,"ĠZap":31308,"Ġvetted":31309,"ĠViolet":31310,"Ġ),":31311,"Ġembarrass":31312,"bang":31313,"ĠProvider":31314,"getting":31315,"alg":31316,"Ġunconditional":31317,"ĠHulk":31318,"ĠWad":31319,"utation":31320,"Ġpointless":31321,"Ġdeprivation":31322,"Ġstarving":31323,"ĠImpossible":31324,"ĠStir":31325,"Ġknack":31326,"anse":31327,"Ġsecurely":31328,"Ġply":31329,"395":31330,"Pack":31331,"liv":31332,"Ġridden":31333,"alks":31334,"308":31335,"male":31336,"Ġbitterly":31337,"Ġirrational":31338,"Members":31339,"ported":31340,"qq":31341,"ractor":31342,"Ġinflict":31343,"ĠBoehner":31344,"Ġthickness":31345,"Ġdome":31346,"ĠInflu":31347,"Ġheap":31348,"Ġmirrored":31349,"Ġconstituent":31350,"Ġfertile":31351,"Ġvaping":31352,"266":31353,"riages":31354,"Ġembassies":31355,"Ġpersu":31356,"ĠMacArthur":31357,"issions":31358,"Main":31359,"aths":31360,"onne":31361,"circ":31362,"Ġsweating":31363,"quartered":31364,"Ġsax":31365,"Ġ540":31366,"Ġreputable":31367,"Ġsatire":31368,"Ġpastors":31369,"ventional":31370,"Mic":31371,"female":31372,"Ġpity":31373,"appropri":31374,"voc":31375,"hei":31376,"Ġimperial":31377,"Ġcorrective":31378,"Ġresent":31379,"Ġtempered":31380,"Ġdiffers":31381,"Hamilton":31382,"Ġsaddle":31383,"Ġgrenades":31384,"ĠQuart":31385,"onymous":31386,"til":31387,"Ġdepiction":31388,"Ġdisreg":31389,"Ġpetitioner":31390,"Ġfret":31391,"ĠEns":31392,"Emer":31393,"540":31394,"opathy":31395,"vertisements":31396,"Ġsketches":31397,"venth":31398,"Ġautomate":31399,"Ġjihad":31400,"iping":31401,"Ġtert":31402,"ĠSop":31403,"ships":31404,"Ġdeceptive":31405,"ĠPryor":31406,"ĠGorge":31407,"ĠMeridian":31408,"rero":31409,"affected":31410,"Ġlame":31411,"660":31412,"rub":31413,"Hello":31414,"ĠNumbers":31415,"269":31416,"Ġmarg":31417,"Fran":31418,"640":31419,"Ġcath":31420,"winter":31421,"ĠMosque":31422,"Ġreckoning":31423,"ĠImaging":31424,"Ġmutation":31425,"ĠMild":31426,"Ġkidnap":31427,"Ġnav":31428,"Ġferocious":31429,"Ġdusty":31430,"Cele":31431,"ĠFoss":31432,"Ġregrett":31433,"lymp":31434,"Ġcoli":31435,"Ġstereo":31436,"Ġforesee":31437,"alties":31438,"Ġresusc":31439,"Full":31440,"wash":31441,"ĠINST":31442,"ĠPars":31443,"Ġcoated":31444,"ĠHT":31445,"Ġdiscord":31446,"Ġreforming":31447,"CAN":31448,"Ġblink":31449,"Ġlubric":31450,"Ġmishand":31451,"ensible":31452,"existent":31453,"secondary":31454,"ĠDoesn":31455,"terrorist":31456,"Ġriff":31457,"custom":31458,"ĠDET":31459,"Ġreusable":31460,"ĠCRA":31461,"ĠScalia":31462,"Ġaccelerator":31463,"Ġpropag":31464,"ĠMID":31465,"ework":31466,"Ġlooted":31467,"oscope":31468,"eners":31469,"ruction":31470,"Ġbarr":31471,"Ġviewership":31472,"Ġlends":31473,"obil":31474,"ĠRoots":31475,"ĠCame":31476,"ibel":31477,"Ġglobalization":31478,"lab":31479,"information":31480,"Ġcoordin":31481,"Ġglitch":31482,"Ġworms":31483,"Ġslurs":31484,"Ġcontemplated":31485,"ĠPenal":31486,"Ġ191":31487,"Ġ221":31488,"Ġexposes":31489,"Ġ248":31490,"ĠASP":31491,"Ġdependency":31492,"urga":31493,"pdf":31494,"Ġvibr":31495,"clone":31496,"ossible":31497,"ĠUtt":31498,"serv":31499,"ĠLevant":31500,"maybe":31501,"MU":31502,"ĠLunar":31503,"Ġbystanders":31504,"Ġcapitals":31505,"Ġpreacher":31506,"thin":31507,"Ġunderscore":31508,"Ġ('":31509,"Ġmedd":31510,"Ġautobiography":31511,"Ġpersistence":31512,"Ġarming":31513,"Ġappalled":31514,"Ġcontradictory":31515,"Ġreciproc":31516,"Ġtakedown":31517,"tan":31518,"Ġnecessities":31519,"itans":31520,"ĠAlas":31521,"Ġsegregated":31522,"ĠResponsibility":31523,"ĠSHOW":31524,"ISIS":31525,"Ġpengu":31526,"Ġumb":31527,"ĠHO":31528,"HB":31529,"ĠChou":31530,"Ġalluded":31531,"Ġharms":31532,"bara":31533,"ĠWOR":31534,"Sorry":31535,"Ġstarvation":31536,"Ġspilling":31537,"Ġcarb":31538,"annis":31539,"ĠGarrison":31540,"Ġmillionaire":31541,"ifling":31542,"ĠCancel":31543,"Ġimprint":31544,"Ġborrower":31545,"455":31546,"ĠCic":31547,"Ġexposures":31548,"dest":31549,"Ġunn":31550,"Ġ802":31551,"Ġadherence":31552,"prints":31553,"Ġweary":31554,"Ġwaging":31555,"Ġ1937":31556,"ĠKepler":31557,"%;":31558,"Ġdefective":31559,"ĠReps":31560,"ĠGranted":31561,"Ġdisco":31562,"ĠRanking":31563,"erno":31564,"Ġarchaeological":31565,"sq":31566,"Ġcapit":31567,"Ġfleets":31568,"Ġinventor":31569,"iffin":31570,"Ġspotting":31571,"ĠSHARES":31572,"309":31573,"Hard":31574,"save":31575,"241":31576,"ĠThinking":31577,"XY":31578,"Ġhavens":31579,"Ġmessed":31580,"crop":31581,"Ġperme":31582,"Ġtimelines":31583,"ĠGarage":31584,"Ġplateau":31585,"together":31586,"fox":31587,"Ġfailings":31588,"ĠTight":31589,"ĠPhysics":31590,"ĠScholars":31591,"Ġpans":31592,"Fall":31593,"Ġhull":31594,"GER":31595,"Ġbourbon":31596,"ceived":31597,"Ġsteroids":31598,"Ġhamb":31599,"Ġinterpretations":31600,"Ġcush":31601,"Chair":31602,"Ġinformational":31603,"aryn":31604,"Ġwoven":31605,"Ġamen":31606,"Bre":31607,"Ġrefreshed":31608,"York":31609,"ĠBlast":31610,"Editor":31611,"Ġmotivating":31612,"ĠReason":31613,"Florida":31614,"Ġdreaded":31615,"Ġstationary":31616,"Ġbil":31617,"doors":31618,"Ġslightest":31619,"Ġcombustion":31620,"Ġfascination":31621,"Ġstraps":31622,"scribed":31623,"Ġexhibiting":31624,"Ġsimplest":31625,"Gar":31626,"Ġprogressives":31627,"claim":31628,"ocket":31629,"Ġexoner":31630,"ĠNETWORK":31631,"Brad":31632,"Ġ197":31633,"Ġnightmares":31634,"Ġillust":31635,"among":31636,"ĠGreenpeace":31637,"Ġoval":31638,"Ġblocker":31639,"3000":31640,"ĠMemor":31641,"Ġmids":31642,"Ġconfuse":31643,"YN":31644,"cow":31645,"Ġdispensary":31646,"telling":31647,"Ġentail":31648,"Ġneurolog":31649,"Ġbroth":31650,"Ġpron":31651,"ĠAnswer":31652,"thank":31653,"Ġintersect":31654,"Ġclinging":31655,"ĠKilling":31656,"Ġcohesion":31657,"Ġcategorized":31658,"Ġtangled":31659,"ĠASC":31660,"Arsenal":31661,"ĠAutomatic":31662,"580":31663,"sac":31664,"Ġshady":31665,"consumer":31666,"hetically":31667,"NV":31668,"Ġoverl":31669,"holes":31670,"ĠDonation":31671,"tera":31672,"score":31673,"library":31674,"Ġsmoother":31675,"Ġcoasts":31676,"Ġintercourse":31677,"Ġunfavorable":31678,"erb":31679,"Hel":31680,"Ġbiases":31681,"Ġinheritance":31682,"Ġsuppressed":31683,"ĠRecommend":31684,"iculture":31685,"ighting":31686,"inguished":31687,"idences":31688,"operated":31689,"Ġhors":31690,"Ġshrug":31691,"aila":31692,"ĠConsortium":31693,"Ġveins":31694,"uria":31695,"ĠSmithsonian":31696,"ĠAX":31697,")âĢĶ":31698,"given":31699,"JC":31700,"Ġreneg":31701,"Ġprincip":31702,"Ġextinct":31703,"Golden":31704,"ASON":31705,"Ġstatutes":31706,"292":31707,"ĠGOOD":31708,"ĠGreenland":31709,"ĠRasmussen":31710,"ATHER":31711,"Ġdeserted":31712,"ĠHitchcock":31713,"Ġqualifies":31714,"Ġdreadful":31715,"Ġsupers":31716,"Ġtendon":31717,"oter":31718,"ĠFate":31719,"Ġrestrooms":31720,"igating":31721,"Sher":31722,"Name":31723,"orph":31724,"ĠCritical":31725,"rox":31726,"Ġdefunct":31727,"Ġcanoe":31728,"Ġbiscuits":31729,"Ġwomb":31730,"808":31731,"istar":31732,"Ġroar":31733,"aundering":31734,"iewicz":31735,"ĠNM":31736,"ĠChamberlain":31737,"Ġ233":31738,"ĠCoat":31739,"Ġ999":31740,"aft":31741,"Ġlurking":31742,"ĠPist":31743,"Ġfollower":31744,"Ġcareg":31745,"ÙĨ":31746,"ĠThin":31747,"ZZ":31748,"ĠGI":31749,"ĠVintage":31750,"Ġpainstaking":31751,"Ġgloom":31752,"Ġtbsp":31753,"Ġwhim":31754,"ĠMask":31755,"rugged":31756,"Ġwritings":31757,"stantial":31758,"luence":31759,"ordable":31760,"akia":31761,"Ġassassinated":31762,"Wind":31763,"Ġdemeanor":31764,"Night":31765,"rape":31766,"ĠBringing":31767,"Ġshields":31768,"ĠAntarctic":31769,"Ġfruitful":31770,"ĠBuster":31771,"ĠLois":31772,"Ġ302":31773,"Style":31774,"ĠRIS":31775,"Ġdissatisfaction":31776,"ulp":31777,"ĠLaser":31778,"Ġdisposition":31779,"ĠAnk":31780,"Ġabsorbing":31781,"276":31782,"Ġvolcan":31783,"Ġleftover":31784,"yah":31785,"ĠVaj":31786,"Ġunsolved":31787,"oland":31788,"Ġstained":31789,"Ġpathetic":31790,"ylan":31791,"Ġknots":31792,"immigration":31793,"ieving":31794,"Coming":31795,"Commerce":31796,"ĠHurt":31797,"drawn":31798,"Ġaxis":31799,"Ġdye":31800,"ĠNora":31801,"ĠPortal":31802,"Ġsuspense":31803,"ĠExactly":31804,"Ġpowering":31805,"ĠClock":31806,"Ġdrawer":31807,"ĠSpike":31808,"Ġhallmark":31809,"aber":31810,"ĠTrainer":31811,"UV":31812,"Ġredundant":31813,"Tour":31814,"Ġdesignate":31815,"Ġredress":31816,"ĠUb":31817,"cake":31818,"oded":31819,"Ġkings":31820,"iates":31821,"Ġcoupons":31822,"Ġextremes":31823,"Elect":31824,"Ġcitation":31825,"Ġdirectory":31826,"Ġtranspired":31827,"cele":31828,"gence":31829,"5000":31830,"ostic":31831,"Ġraining":31832,"ĠSight":31833,"videos":31834,"phthal":31835,"llor":31836,"Ġappraisal":31837,"Ġdetox":31838,"Ġelecting":31839,"Ġordinances":31840,"Ġlifespan":31841,"Ref":31842,"Ġilluminated":31843,"Ġforfe":31844,"Making":31845,"ĠWorst":31846,"ĠTP":31847,"Ġfullest":31848,"ĠISIL":31849,"ĠRates":31850,"Ġyeast":31851,"sett":31852,"ĠYok":31853,"innie":31854,"edition":31855,"ĠGoldstein":31856,"Ġunaff":31857,"god":31858,"Ġzo":31859,"rums":31860,"Ġopaque":31861,"ĠHist":31862,"Yesterday":31863,"AMS":31864,"aband":31865,"005":31866,"illary":31867,"ĠSplash":31868,"Ġaccrued":31869,"Ell":31870,"Ġnominating":31871,"ĠBroadcast":31872,"ĠWhip":31873,"ARM":31874,"Ġunnecessarily":31875,"brown":31876,"429":31877,"ansky":31878,"Ġextravagant":31879,"Malley":31880,"wage":31881,"Ġexempted":31882,"Ġtypo":31883,"Ġesports":31884,"ĠStru":31885,"ĠPython":31886,"Ġsaint":31887,"ĠCSI":31888,"ĠPowder":31889,"Ġdisguised":31890,"ĠSubway":31891,"Ġprecursor":31892,"ĠWizard":31893,"Johnson":31894,"icas":31895,"Ġdefaults":31896,"!).":31897,"ebra":31898,"jected":31899,"Ġunaccompanied":31900,"HH":31901,"Ġproced":31902,"clinical":31903,"Ġmitigating":31904,"ĠSoup":31905,"ĠFunny":31906,"344":31907,"Hall":31908,"Ġscalable":31909,"Ġshimmer":31910,"Ġunderstatement":31911,"zeb":31912,"icus":31913,"Ġretract":31914,"IDER":31915,"ieft":31916,"iii":31917,"ĠEmperor":31918,"Ġvoltage":31919,"343":31920,"Rest":31921,"ĠButcher":31922,"Ġlaced":31923,"Ġsalty":31924,"Ġfourteen":31925,"Ġoxy":31926,"Ġraged":31927,"Ġforg":31928,"Ġcaveat":31929,"Ġponder":31930,"process":31931,"Ġghosts":31932,"ĠGoose":31933,"didn":31934,"stood":31935,"amation":31936,"Ġvillains":31937,"contract":31938,"Ġbooted":31939,"ĠDidn":31940,"ĠSalon":31941,"Ġlewd":31942,"ĠFritz":31943,"Ġorganis":31944,"Ġpuzzles":31945,"ĠRX":31946,"Ġcurtains":31947,"ĠPackage":31948,"Ġrebate":31949,"Ġspokes":31950,"Ġoccupant":31951,"Ġfooled":31952,"appy":31953,"Ġyourselves":31954,"Ġmaths":31955,"Ġ630":31956,"bos":31957,"ĠHeb":31958,"APS":31959,"Ġbulletin":31960,"Ġpests":31961,"Ġlum":31962,"ĠHAS":31963,"users":31964,"idated":31965,"Ġpalpable":31966,"ĠFeature":31967,"ĠPKK":31968,"Ġdetriment":31969,"Ġbamboo":31970,"Ġimmersed":31971,"ĠDud":31972,"Ġion":31973,"icc":31974,"ĠIris":31975,"ĠBeats":31976,"Ġimprobable":31977,"Ġfuner":31978,"Ġsprung":31979,"ĠLieberman":31980,"ĠSTA":31981,"venge":31982,"Ġtreacherous":31983,"Ġpreced":31984,"Ġsniper":31985,"ĠGOLD":31986,"ĠSUR":31987,"Nic":31988,"ĠROB":31989,"Camp":31990,"Ġhooks":31991,"oling":31992,"Ġbolst":31993,"339":31994,"heter":31995,"Ġbracelet":31996,"Ġbreat":31997,"307":31998,"ĠTrader":31999,"ĠPixar":32000,"hist":32001,"Ġmenacing":32002,"Ġgrizz":32003,"294":32004,"Ġillustrious":32005,"Ġtransact":32006,"Ġspoiler":32007,"ĠWORK":32008,"Road":32009,"Ġblackout":32010,"Ġencomp":32011,"proven":32012,"ĠFriendship":32013,"Ġentrances":32014,"Ġprofessions":32015,"Ġinsin":32016,"Ġrecorder":32017,"Ġformulation":32018,"govern":32019,"Ġpainfully":32020,"ĠRepe":32021,"eeds":32022,"cru":32023,"ĠDir":32024,"Ġtriumphant":32025,"Ġignition":32026,"xy":32027,"Ġintrusion":32028,"ĠEAR":32029,"RES":32030,"Ġration":32031,"ĠTaken":32032,"Ġcages":32033,"Ġpeg":32034,"Ġcommem":32035,"680":32036,"ĠRite":32037,"Ġfolder":32038,"Ġvertically":32039,"Ġcheeks":32040,"pick":32041,"Ġcrispy":32042,"Ġsqueezing":32043,"ĠBene":32044,"ĠTrailer":32045,"ĠKM":32046,"acceptable":32047,"ĠSetting":32048,"Ġsupernatural":32049,"ĠEz":32050,"Ġvenom":32051,"ĠFrey":32052,"Ġpulp":32053,"Had":32054,"centered":32055,"metics":32056,"Kent":32057,"ĠDOI":32058,"kr":32059,"ĠWHEN":32060,"Ġtakeoff":32061,"isf":32062,"uko":32063,"Ġquasi":32064,"Ġveggies":32065,"Ġpesticide":32066,"Ġstimulating":32067,"Ġacknowledgement":32068,"Ġattained":32069,"ĠBackground":32070,"281":32071,"317":32072,"ĠTrees":32073,"Ġdetractors":32074,"Ġannouncer":32075,"Ġjoyful":32076,"ĠElf":32077,"istration":32078,"phi":32079,"Ġprogressively":32080,"mini":32081,"Ġcontraception":32082,"asca":32083,"ishops":32084,"Ġmisunderstood":32085,"Ġinitiating":32086,"ĠConversely":32087,"338":32088,"080":32089,"idation":32090,"ĠGoes":32091,"Ġimprov":32092,"Ġswapping":32093,"Vict":32094,"Ġdevoid":32095,"fighter":32096,"ĠMori":32097,"Ġvoy":32098,"ĠElev":32099,"ĠAim":32100,"Ġtrustworthy":32101,"Leg":32102,"675":32103,"ĠPossible":32104,"Crunch":32105,"ĠRings":32106,"Ġphony":32107,"Ġbladder":32108,"ĠChall":32109,"Spot":32110,"oak":32111,"Was":32112,"ĠFAM":32113,"ĠAGA":32114,"ĠFifa":32115,"Ġenclosed":32116,"Ġanthrop":32117,"faith":32118,"ĠAux":32119,"Ġgracious":32120,"roller":32121,"Ġdowntime":32122,"swing":32123,"Ġcamouflage":32124,"ĠCosts":32125,"Ġliv":32126,"ricular":32127,"ĠUran":32128,"Ġdisapproval":32129,"Ġpropriet":32130,"bits":32131,"Ġmafia":32132,"ĠSCHOOL":32133,"ĠPrepar":32134,"button":32135,"Almost":32136,"Ġpastoral":32137,"ĠDove":32138,"Hol":32139,"Ġimposes":32140,"ĠDram":32141,"lys":32142,"ĠSAS":32143,"Ġwiring":32144,"271":32145,"ĠModels":32146,"Ġoutpost":32147,"etics":32148,"Ġinsulted":32149,"ĠMongolia":32150,"Ġoverth":32151,"Haw":32152,"ĠHomer":32153,"itta":32154,"raining":32155,"Ġevidently":32156,"raphic":32157,"impact":32158,"Ġfranch":32159,"Ġ2100":32160,"Ġapproximate":32161,"Ġcartoons":32162,"Ġbackups":32163,"umbing":32164,"Ġforceful":32165,"ĠShad":32166,"Ġsurges":32167,"Ġperf":32168,"Ġdele":32169,"Ġquieter":32170,"ĠHorowitz":32171,"ĠDX":32172,"anners":32173,"ĠNinja":32174,"ĠScript":32175,"ĠElise":32176,"collect":32177,"Ġgrading":32178,"ĠBethesda":32179,"Kids":32180,"ĠTelephone":32181,"Ġpreferring":32182,"Ġreconcil":32183,"Ġmango":32184,"ĠHail":32185,"ĠCitizenship":32186,"Master":32187,"cular":32188,"Ġstuffing":32189,"ĠAlive":32190,"ALLY":32191,"Ġchi":32192,"ĠDynam":32193,"ĠRosenthal":32194,"Ġpurity":32195,"Ġtemp":32196,"ĠHAL":32197,"employ":32198,"Ġplentiful":32199,"ĠComed":32200,"Ġstacks":32201,"ĠHuge":32202,"ĠOlder":32203,"Ġsclerosis":32204,"ONY":32205,"Ġfilmmaking":32206,"chance":32207,"Cry":32208,"Ġworkflow":32209,"ĠPersonnel":32210,"awed":32211,"ĠColumn":32212,"Ġuncomp":32213,"Ġdiscriminated":32214,"Ġpts":32215,"Ġallev":32216,"ĠKinn":32217,"meal":32218,"Ġnovice":32219,"Ġcrest":32220,"Ġhearty":32221,"Ġlowers":32222,"inqu":32223,"ĠPlayoffs":32224,"ĠHyp":32225,"Ġautos":32226,"Ġindec":32227,"Ġnighttime":32228,"Ġreflex":32229,"306":32230,"disciplinary":32231,"ophe":32232,"contact":32233,"Ġachievable":32234,"Ġslab":32235,"ĠMessage":32236,"ĠVMware":32237,"ĠDia":32238,"REG":32239,"Ġconfisc":32240,"ĠMechan":32241,"Ġphenomena":32242,"Ġsequencing":32243,"Ġshaming":32244,"Ġcompilation":32245,"ĠAges":32246,"Ġmastered":32247,"Ġagony":32248,"Ġrestrain":32249,"ĠLyme":32250,"Which":32251,"ĠBarney":32252,"ĠConcept":32253,"Ġsuperheroes":32254,"ĠPsychology":32255,"Ġreminis":32256,"violence":32257,"Lead":32258,"Da":32259,"VEN":32260,"ERC":32261,"ĠVoter":32262,"Ġbetray":32263,"Ġsavage":32264,"driver":32265,"IFT":32266,"Chain":32267,"angler":32268,"'-":32269,"lain":32270,"ĠRatt":32271,"bis":32272,"iverse":32273,"Ġdensely":32274,"Ġuncom":32275,"Ġunsuspecting":32276,"Ġstimulation":32277,"diff":32278,"Ġskins":32279,"ĠRiding":32280,"ategic":32281,"ĠUnderstand":32282,"occup":32283,"ĠCooking":32284,"Ġschizophrenia":32285,"ĠKoen":32286,"Ġcomrades":32287,"HY":32288,"Ġfab":32289,"ĠRowling":32290,"Allen":32291,"ĠJUL":32292,"Ġembryos":32293,"UU":32294,"ĠCAT":32295,"Ġtidy":32296,"finger":32297,"ĠCake":32298,"Ġrightfully":32299,"religious":32300,"Ġ407":32301,"Gal":32302,"408":32303,"Ġgrievance":32304,"Ġswallowed":32305,"251":32306,"283":32307,"ĠBarcl":32308,"opter":32309,"Ġpedoph":32310,"Ġcured":32311,"Ġestablishes":32312,"increasing":32313,"tics":32314,"articles":32315,"Ġunethical":32316,"authored":32317,"Ġanchors":32318,"ĠContra":32319,"Ġventured":32320,"ĠCoh":32321,"Ġpuff":32322,"heddar":32323,"Ġomission":32324,"Ġdich":32325,"ceed":32326,"Ġscares":32327,"Ġdoctoral":32328,"293":32329,"ĠUnt":32330,"Ġdop":32331,"ĠInjury":32332,"ificantly":32333,"ĠRift":32334,"ĠOrders":32335,"Ġmobilize":32336,"particularly":32337,"Ġchilled":32338,"Reports":32339,"redibly":32340,"ĠGuru":32341,"Ġvalleys":32342,"Ġtextures":32343,"Ġreuse":32344,"roit":32345,"unts":32346,"Ġirreversible":32347,"Ġwarships":32348,"Ġpus":32349,"Ġpeeled":32350,"Ġthirst":32351,"Ġgrapple":32352,"busters":32353,"Ġnort":32354,"ĠDates":32355,"Safe":32356,"Ġbirthplace":32357,"hemoth":32358,"Ġvile":32359,"Ġ306":32360,"Ram":32361,"activated":32362,"ĠAero":32363,"Ġbutcher":32364,"ĠKnock":32365,"Ġdisturb":32366,"Ġtotality":32367,"tted":32368,"Ġlegit":32369,"cking":32370,"nikov":32371,"Ġfavoring":32372,"lang":32373,"Ġrightful":32374,"orum":32375,"!!!!":32376,"ĠMinute":32377,"Ġpostings":32378,"Java":32379,"510":32380,"Ġmicrobes":32381,"Ġsixteen":32382,"entimes":32383,"Ġbulb":32384,"Ġgoalt":32385,"Ġhumiliated":32386,"ansom":32387,"roach":32388,"Ġgrouping":32389,"hari":32390,"Ġcler":32391,"Ġstared":32392,"ĠSymptoms":32393,"Ġbasil":32394,"Whenever":32395,"ĠWhoever":32396,"Oil":32397,"ĠJericho":32398,"ĠAlm":32399,"Pol":32400,"Hur":32401,"Ġupro":32402,"ĠSpo":32403,"hammer":32404,"Mur":32405,"ĠTorch":32406,"Ġfrequencies":32407,"ĠExpansion":32408,"Ġparalysis":32409,"igon":32410,"ĠSail":32411,"Ġsilently":32412,"Ġrevolver":32413,"Ġstockpile":32414,"Ġpessimistic":32415,"ESA":32416,"Ġdisclaim":32417,"Ġdemocracies":32418,"ĠTales":32419,"ĠAngry":32420,"ĠWhitman":32421,"ĠOri":32422,"Ġtransitioned":32423,"behind":32424,"ĠLAN":32425,"Ġcav":32426,"ĠJazeera":32427,"KC":32428,"ĠInspect":32429,"irty":32430,"ĠAin":32431,"ĠOrig":32432,"Ġobscene":32433,"Ġdormant":32434,"Ġharb":32435,"ĠWiz":32436,"ĠAdolf":32437,"Ġvic":32438,"Ġdenouncing":32439,"Ġye":32440,"aques":32441,"Ġomn":32442,"Ġassemblies":32443,"nosis":32444,"Ġadmon":32445,"Ġanguish":32446,"Ġvag":32447,"YE":32448,"ĠMacro":32449,"Ġrubbing":32450,"Ġreplicated":32451,"Moon":32452,"ĠGuitar":32453,"Ġcentimeters":32454,"amily":32455,"ĠAmes":32456,"Ġchlorine":32457,"Perhaps":32458,"Ġpartisans":32459,"soc":32460,"Ġvagina":32461,"Ġtrove":32462,"ĠYES":32463,"Ġtherapists":32464,"Ġnods":32465,"Ġhanged":32466,"Ġridge":32467,"Ġhaz":32468,"ĠmacOS":32469,"Ġske":32470,"ĠShia":32471,"Ġsteril":32472,"Ġalmond":32473,"ĠRockefeller":32474,"Ġintrinsic":32475,"Certainly":32476,"Ġsublime":32477,"Earn":32478,"abet":32479,"Ġframeworks":32480,"ogical":32481,"ilst":32482,"ipal":32483,"Ġrescuing":32484,"ĠWatergate":32485,"Ġ231":32486,"ĠNano":32487,"ighthouse":32488,"olph":32489,"Ġ312":32490,"Ġhealed":32491,"ĠTomb":32492,"Ġsubst":32493,"Ġsulph":32494,"ĠNewsp":32495,"ĠLama":32496,"venue":32497,"387":32498,"productive":32499,"ĠNEED":32500,"minus":32501,"ĠPages":32502,"cand":32503,"ĠClover":32504,"ĠForensic":32505,"ryn":32506,"ogle":32507,"ocr":32508,"Ġvaccinations":32509,"cies":32510,"ĠMek":32511,"Ġunaffected":32512,"Ġfetal":32513,"ĠDino":32514,"Ġhemisphere":32515,"Ġfroze":32516,"ĠPeg":32517,"Ġmicroscope":32518,"Ġmoderates":32519,"ĠGEN":32520,"ĠHawai":32521,"Ġstagn":32522,"Absolutely":32523,"practice":32524,"IBLE":32525,"cture":32526,"ĠAshe":32527,"Ġcondoms":32528,"Ġpoked":32529,"training":32530,"Ġintermedi":32531,"347":32532,"Ġcardinal":32533,"ĠSpoon":32534,"Ġsupp":32535,"Ġpreviews":32536,"Service":32537,"ĠBeam":32538,"Ġtranscend":32539,"Fresh":32540,"Sure":32541,"Ġ4000":32542,"idential":32543,"ĠCoinbase":32544,"Ġworkings":32545,"ĠPI":32546,"Ġpassionately":32547,"Ġdecisively":32548,"ĠInspection":32549,"Ġinvoke":32550,"Ġstain":32551,"Ġcleaners":32552,"Ġregulates":32553,"Ġshone":32554,"ĠEVERY":32555,"istance":32556,"map":32557,"Ġredu":32558,"Ġoccupies":32559,"Ġprocure":32560,"acket":32561,"roman":32562,"Ġilleg":32563,"Ġleaps":32564,"yond":32565,"Ġyarn":32566,"ĠLTD":32567,"ĠCONTR":32568,"ĠRestoration":32569,"ĠCDs":32570,"Ġdrinkers":32571,"ĠJordanian":32572,"Ġabl":32573,"Ġdisparate":32574,"Ġprimed":32575,"ĠFirearms":32576,"artz":32577,"Ġindispensable":32578,"Ter":32579,"Ġfright":32580,"Ġmarkedly":32581,"Ġroam":32582,"ĠJurassic":32583,"Ġfeder":32584,"Ġpepp":32585,"ĠDV":32586,"Ġpancakes":32587,"sweet":32588,"Ġunmatched":32589,"Ġassembling":32590,"Ultimately":32591,"Ġendeavour":32592,"Ġluckily":32593,"Ġbitch":32594,"Ġelegance":32595,"eers":32596,"drop":32597,"credit":32598,"Ġscourge":32599,"ĠMinimum":32600,"Ġimpatient":32601,"Ġhunted":32602,"ĠGoddard":32603,"Kal":32604,"Ġmined":32605,"Ġcalves":32606,"Ġ234":32607,"Ġplank":32608,"Ġinjecting":32609,"ĠKaufman":32610,"ĠCompliance":32611,"tone":32612,"Ġ345":32613,"Ġdazz":32614,"ĠClarks":32615,"Ġcomprehens":32616,"Ġpist":32617,"Ġrhythms":32618,"Ġreserv":32619,"337":32620,"ĠIDF":32621,"Ġshouts":32622,"midt":32623,"323":32624,"Ġsoothing":32625,"Ġadministr":32626,"Ġgloomy":32627,"Ġfutile":32628,"ĠProhibition":32629,"upon":32630,"ĠAnglic":32631,"seeking":32632,"Ġdodge":32633,"Ds":32634,"ĠGrants":32635,"editor":32636,"ĠInquis":32637,"Ġ1929":32638,"decl":32639,"ĠPorts":32640,"ĠCure":32641,"ĠDPRK":32642,"oct":32643,"Ġvocabulary":32644,"Ġcling":32645,"298":32646,"Ġpeac":32647,"Ġantibodies":32648,"dor":32649,"ĠWorse":32650,"Ġsmelled":32651,"Ġleash":32652,"MED":32653,"Ġdisinteg":32654,"Ġtruthful":32655,"Ġsalesman":32656,"Ġsquares":32657,"susp":32658,"Ġcraving":32659,"Ġwizard":32660,"moral":32661,"ĠQué":32662,"Anything":32663,"Ġfalsehood":32664,"ARI":32665,"Ġcoworkers":32666,"Ġthy":32667,"outher":32668,"Ġbrushing":32669,"ĠProtest":32670,"ĠMF":32671,"abba":32672,"lead":32673,"ĠExhibit":32674,"Ga":32675,"ĠFranks":32676,"Ġdictates":32677,"illegal":32678,"Ġrelayed":32679,"Ġploy":32680,"ĠاÙĦ":32681,"ĠDocuments":32682,"Ġtint":32683,"ĠYuan":32684,"Ġdepended":32685,"Mir":32686,"ĠIntrodu":32687,"Ġrecourse":32688,"oqu":32689,"ĠTED":32690,"Ġdifferentiated":32691,"ĠWalls":32692,"Ġsentimental":32693,"Ġantis":32694,"retion":32695,"comes":32696,"ĠWORLD":32697,"Ġcoax":32698,"ĠTatt":32699,"ĠGingrich":32700,"2006":32701,"ĠBrut":32702,"Second":32703,"posed":32704,"shots":32705,"Ġ313":32706,"idian":32707,"alking":32708,"Ġdens":32709,"Ġgif":32710,"akings":32711,"Ġkeywords":32712,"Ġchast":32713,"Ġadversary":32714,"Ġnick":32715,"iasis":32716,"ĠLegisl":32717,"Ġcoff":32718,"ĠOriental":32719,"ĠMorg":32720,"ĠHAR":32721,"Ġlegalizing":32722,"Ġbanter":32723,"ĠTart":32724,"ĠTRI":32725,"Ġantagon":32726,"ĠGF":32727,"oler":32728,"ĠUFO":32729,"Therefore":32730,"ĠOsama":32731,"ĠStructure":32732,"apps":32733,"Ġpee":32734,"ĠSomehow":32735,"ĠOverwatch":32736,"ĠCasual":32737,"Ġdishon":32738,"SEE":32739,"ctive":32740,"andering":32741,"ĠTransformation":32742,"Andy":32743,"ĠFever":32744,"Ġspectator":32745,"Ġlash":32746,"Ġprotector":32747,"apy":32748,"Ġexhilar":32749,"aroo":32750,"Ġmamm":32751,"Ġbystand":32752,"acky":32753,"Ġdigestive":32754,"Ġamplified":32755,"Ġalpha":32756,"continue":32757,"Low":32758,"Ġdisgusted":32759,"356":32760,"script":32761,"Ġgenerational":32762,"ĠPassenger":32763,"sight":32764,"Ġcout":32765,"Ġhone":32766,"ulse":32767,"Ġignite":32768,"284":32769,"gow":32770,"Ġbinary":32771,"Ġincess":32772,"Review":32773,"607":32774,"ĠSurprise":32775,"Ġirritation":32776,"ĠBarth":32777,"ĠGum":32778,"Ġvideot":32779,"ĠFres":32780,"asons":32781,"Ġcollaborator":32782,"fal":32783,"ĠGon":32784,"Ġsettles":32785,"regular":32786,"Ġmiscarriage":32787,"cube":32788,"Ġsubord":32789,"ĠRegistered":32790,"Ġnotions":32791,"zzy":32792,"Ġrevert":32793,"OFF":32794,"Ġhasht":32795,"ĠPNG":32796,"Ġunimaginable":32797,"builders":32798,"Taylor":32799,"ĠPAY":32800,"Ġ).":32801,"Ġ238":32802,"ĠLAST":32803,"MAS":32804,"Ġillustrations":32805,"Ġparody":32806,"Ġdispersed":32807,"ĠRoses":32808,"Ġestimation":32809,"ĠGets":32810,"Patrick":32811,"CHA":32812,"Ġmisdem":32813,"agate":32814,"alter":32815,"Ġgeo":32816,"Ġenormously":32817,"Ġarrogance":32818,"Ġpert":32819,"Ġmeta":32820,"ĠJuno":32821,"iov":32822,"imov":32823,"Ġchores":32824,"acan":32825,"Paris":32826,"313":32827,"Lewis":32828,"Ġwillingly":32829,"ERA":32830,"Ġencaps":32831,"ilk":32832,"Ġnodes":32833,"Ġenzyme":32834,"want":32835,"Ġtolerant":32836,"Ġcondos":32837,"Ġasserts":32838,"Ġcanon":32839,"Ġscanned":32840,"bishop":32841,"Ġperched":32842,"util":32843,"ĠBonus":32844,"create":32845,"ĠFuk":32846,"Ġmotif":32847,"Ġcontemplate":32848,"ĠBEN":32849,"imir":32850,"Ġacadem":32851,"uvian":32852,"ĠIdeas":32853,"ĠCY":32854,"Ġants":32855,"Ġprostitutes":32856,"2005":32857,"Spring":32858,"ĠBarrel":32859,"ĠAunt":32860,"ĠLudwig":32861,"ĠHerm":32862,"PRO":32863,"obiles":32864,"rack":32865,"STER":32866,"ucket":32867,"Ġmun":32868,"Ġ419":32869,"ICES":32870,"Ġcardio":32871,"Ġtrenches":32872,"Nation":32873,"yahoo":32874,"Ġburd":32875,"Ġnost":32876,"Ġappropriations":32877,"ĠChili":32878,"Josh":32879,"GW":32880,"Ġoppressed":32881,"ĠBEFORE":32882,"Ġmurderous":32883,"Pen":32884,"achable":32885,"Ġrive":32886,"Ġculmin":32887,"Ġdefin":32888,"ĠMord":32889,"idate":32890,"ĠChim":32891,"ource":32892,"ĠElectro":32893,"orthy":32894,"Ġcalendars":32895,"regation":32896,"Ġretrospect":32897,"ĠTribal":32898,"ĠHes":32899,"Ġcran":32900,"Ġcreditor":32901,"Ġfibers":32902,"note":32903,"idays":32904,"ĠSebast":32905,"ĠKitty":32906,"Ġplainly":32907,"ĠLAPD":32908,"Ġtrumpet":32909,"ĠAppropriations":32910,"Hill":32911,"ĠVeget":32912,"296":32913,"lated":32914,"othes":32915,"ibrarian":32916,"Listen":32917,"nex":32918,"WHO":32919,"Ġshampoo":32920,"Ġclaimants":32921,"Ġisol":32922,"Ġunchecked":32923,"Ġmov":32924,"umo":32925,"ĠLens":32926,"Ġdiscreet":32927,"Ġrespectfully":32928,"Ġreclaimed":32929,"ĠHatt":32930,"thus":32931,"ĠFlo":32932,"Ġsumm":32933,"phas":32934,"ĠHaitian":32935,"Ġstrife":32936,"Ġabound":32937,"verted":32938,"Ġpatronage":32939,"449":32940,"Ġprelim":32941,"ĠZhu":32942,"ĠRevel":32943,"adic":32944,"Ġminded":32945,"ĠStability":32946,"Ġresembling":32947,"Ġvending":32948,"ischer":32949,"Ġkisses":32950,"Ġsuperiority":32951,"Ġinfinite":32952,"ISC":32953,"880":32954,"Ġappease":32955,"VO":32956,"404":32957,"ECH":32958,"gam":32959,"River":32960,"metal":32961,"determination":32962,"Cook":32963,"Ġbuds":32964,"Ġ(%)":32965,"ĠCreated":32966,"Ġstrut":32967,"Ġ425":32968,"Ġverte":32969,"ĠOrb":32970,"Ġweaving":32971,"261":32972,"Ġflyers":32973,"spons":32974,"ĠCovenant":32975,"570":32976,"Ġintangible":32977,"ĠBJ":32978,"ĠStead":32979,"ĠBrune":32980,"pain":32981,"independent":32982,"Ball":32983,"witch":32984,"ĠIon":32985,"Ġpupp":32986,"Cash":32987,"ĠConvert":32988,"Ġimpede":32989,"broad":32990,"onew":32991,"Ġsynergy":32992,"Ġcoined":32993,"620":32994,"ivalent":32995,"ĠInfect":32996,"ĠAqua":32997,"Together":32998,"ĠChemistry":32999,"ĠURL":33000,"ampion":33001,"Ġdeclarations":33002,"Ġaffirmative":33003,"umper":33004,"ĠTarant":33005,"Ġstereotype":33006,"Ġbookstore":33007,"incre":33008,"Ġchipset":33009,"Ġangst":33010,"Jose":33011,"laus":33012,"Ġheater":33013,"ipers":33014,"Ġeminent":33015,"hook":33016,"sticks":33017,"ĠCoul":33018,"Ġmildly":33019,"SG":33020,"Ġworm":33021,"Ġdisable":33022,"Ġperfume":33023,"ISTER":33024,"Ġgathers":33025,"ĠLotus":33026,"hyp":33027,"actus":33028,"Ġdistinctly":33029,"fifth":33030,"!),":33031,"ĠCrunch":33032,"Ġcohesive":33033,"Ġfortunately":33034,"Ġninety":33035,"Ġcartels":33036,"empl":33037,"Direct":33038,"Ġcommuting":33039,"ĠSX":33040,"ractive":33041,"Ġtranslating":33042,"ĠAQ":33043,"Ġslay":33044,"abuse":33045,"ĠProc":33046,"ĠCantor":33047,"ĠTas":33048,"Sir":33049,"Thom":33050,"ĠCHRIST":33051,"Ġreceptive":33052,"ĠCornel":33053,"Arab":33054,"Ġgrammar":33055,"Ġhandlers":33056,"Ġalloy":33057,"Ġthinly":33058,"adem":33059,"Ġproponent":33060,"ĠPVC":33061,"Ġstump":33062,"tom":33063,"rets":33064,"iciency":33065,"780":33066,"Ġ311":33067,"ĠClapper":33068,"ITAL":33069,"Ùħ":33070,"Ġnarrator":33071,"Ġblond":33072,"Ġintermittent":33073,"Ġcollabor":33074,"646":33075,"Ġmetast":33076,"Ġregeneration":33077,"ĠLegendary":33078,"Ġgenitals":33079,"Ġbartender":33080,"atson":33081,"Okay":33082,"Ġpassages":33083,"Ġsubstituted":33084,"orr":33085,"ALTH":33086,"Ġartic":33087,"Ġascent":33088,"Ġmatured":33089,"Ġterminology":33090,"served":33091,"ĠDeliver":33092,"Ġattic":33093,"anges":33094,"Ġrenaissance":33095,"Ġbleed":33096,"claimer":33097,"onse":33098,"Sec":33099,"Ġparticle":33100,"aneous":33101,"ateur":33102,"Ġzeal":33103,"ĠPets":33104,"Working":33105,"ĠRespect":33106,"Ġsermon":33107,"ĠProvided":33108,"Ġfilibuster":33109,"Ġabolished":33110,"reviewed":33111,"cription":33112,"Ġrevers":33113,"atered":33114,"435":33115,"Ġwhe":33116,"ometown":33117,"UFC":33118,"products":33119,"Winter":33120,"Ġ304":33121,"Ġsporadic":33122,"orough":33123,"EB":33124,"ĠAgric":33125,"ĠMTA":33126,"wic":33127,"Ġpowerless":33128,"Ġcarrot":33129,"ww":33130,"Ġabsorption":33131,"ĠTyphoon":33132,"Turkey":33133,"Ġproclaim":33134,"Ġhikers":33135,"Ġpractise":33136,"/$":33137,"Ġfingertips":33138,"Ġbaff":33139,"vu":33140,"Ġans":33141,"plug":33142,"Ġacquaintance":33143,"itement":33144,"ihar":33145,"Ġreluctantly":33146,"Ġforc":33147,"Ġguarant":33148,"ĠWanted":33149,"Walk":33150,"addle":33151,"unders":33152,"Fred":33153,"Ġtides":33154,"ĠBai":33155,"Ġcountering":33156,"raper":33157,"ursions":33158,"ĠFlav":33159,"pared":33160,"raised":33161,"Ñı":33162,"ĠDiff":33163,"Ġreload":33164,"ourses":33165,"ĠBurning":33166,"Ġwand":33167,"Ġledger":33168,"Ġcoughing":33169,"ĠLoren":33170,"Nazis":33171,"Ġcompile":33172,"Eight":33173,"icultural":33174,"yy":33175,"Ġ1932":33176,"Run":33177,"AIN":33178,"Ġattractiveness":33179,"ĠOmn":33180,"Ġconfer":33181,"compliance":33182,"Ġembed":33183,"Steven":33184,"2001":33185,"Ġdecre":33186,"Ġprompts":33187,"ĠHare":33188,"Ġleaping":33189,"Ġslaughtered":33190,"Ġforfeiture":33191,"342":33192,"Charl":33193,"CDC":33194,"ographically":33195,"Ġduplicate":33196,"Ġdistracting":33197,"examination":33198,"Ġpeas":33199,"Ġcatchy":33200,"Ġdives":33201,"ĠAda":33202,"Hay":33203,"Ġenthusiastically":33204,"Ġfunky":33205,"kay":33206,"EVA":33207,"Ġpsychologists":33208,"Ġancestry":33209,"iyah":33210,"ifter":33211,"nob":33212,"518":33213,"rouse":33214,"Ġchord":33215,"Ġcone":33216,"Ġbarracks":33217,"ĠRoyale":33218,"ĠIntegration":33219,"Ġtrolling":33220,"ĠSynt":33221,"andals":33222,"ĠGrain":33223,"ĠNeck":33224,"618":33225,"Ġrapist":33226,"pins":33227,"Ġwitty":33228,"Ġdehydration":33229,"arlane":33230,"Ġimmoral":33231,"Ġaccum":33232,"ĠMcAuliffe":33233,"slow":33234,"Ġinjust":33235,"Ġ1700":33236,"Ġcarbs":33237,"Ġintel":33238,"Non":33239,"isks":33240,"Tre":33241,"Ġinterviewer":33242,"sam":33243,"Ġdelve":33244,"Ġadmirable":33245,"ĠROM":33246,"ĠHispanics":33247,"Ġimpart":33248,"Ġunderrated":33249,"Ġvictimized":33250,"ĠPsych":33251,"ppings":33252,"Ġ610":33253,"pole":33254,"Ġdiner":33255,"ĠScale":33256,"Ġunforeseen":33257,"surprisingly":33258,"opus":33259,"ĠCOURT":33260,"Ġjuggling":33261,"ĠFacilities":33262,"Aid":33263,"ĠHPV":33264,"Ġcrawling":33265,"flu":33266,"etary":33267,"ĠHarriet":33268,"329":33269,"ĠSod":33270,"ĠBiological":33271,"birth":33272,"ribed":33273,"Ġpulses":33274,"396":33275,"eways":33276,"ĠAlma":33277,"nov":33278,"015":33279,"ricane":33280,"agna":33281,"Ak":33282,"ĠClaim":33283,"Ġpref":33284,"Ġinterfaces":33285,"ĠADHD":33286,"604":33287,"ZE":33288,"venture":33289,"Ġascend":33290,"ĠGou":33291,"Ġpriceless":33292,"redo":33293,"kw":33294,"Conf":33295,"Ġmah":33296,"Ġpoets":33297,"Ġstalk":33298,"Ġencamp":33299,"Ġhopped":33300,"Ġmelody":33301,"JECT":33302,"eming":33303,"Ġbewild":33304,"aternal":33305,"uchs":33306,"dit":33307,"ĠTransmission":33308,"Lake":33309,"Ġatoms":33310,"ĠThoughts":33311,"ilts":33312,"volume":33313,"Ġsocioeconomic":33314,"atisf":33315,"Ġnarr":33316,"zinski":33317,"ymes":33318,"episode":33319,"Ġinherit":33320,"Ġintending":33321,"Ġarenas":33322,"uras":33323,"burning":33324,"334":33325,"teenth":33326,"Ġsophistication":33327,"Ġscreenshots":33328,"Ġautistic":33329,"lip":33330,"paper":33331,"Ġmonopol":33332,"799":33333,"forms":33334,"ocrats":33335,"Ġpineapple":33336,"Ġbegs":33337,"Ġpersecuted":33338,"Ġsubscribed":33339,"Ġelic":33340,"ĠPRESIDENT":33341,"297":33342,"Ġpreferential":33343,"Ġpyramid":33344,"Ġconvergence":33345,"Ġwob":33346,"Project":33347,"ĠAluminum":33348,"ĠJPM":33349,"ĠBAT":33350,"Ġdolphins":33351,"018":33352,"healthy":33353,"ĠCG":33354,"ĠEffective":33355,"worm":33356,"ĠEas":33357,"olicited":33358,"ĠUSE":33359,"ĠCaval":33360,"Ġswirl":33361,"Ġspaghetti":33362,"Ġinward":33363,"Republican":33364,"Ġpublicized":33365,"Ġeconomical":33366,"Ġsalsa":33367,"ĠTitanic":33368,"dot":33369,"Ġcontro":33370,"ĠBangl":33371,"iban":33372,"ĠKlux":33373,"Ġhinges":33374,"610":33375,"Ġvalves":33376,"profits":33377,"Wonder":33378,"Ġorient":33379,"Ġsque":33380,"Ġprivatization":33381,"Obama":33382,"Thousands":33383,"ĠTasman":33384,"Ġmaze":33385,"eem":33386,"Ġsurvives":33387,"istant":33388,"Ġenriched":33389,"Ġencl":33390,"Ġcompliments":33391,"ĠShoes":33392,"Ġinsanity":33393,"consider":33394,"agog":33395,"Ġbaffled":33396,"Ġ°":33397,"ĠWordPress":33398,"qus":33399,"usual":33400,"stall":33401,"Deb":33402,"ĠRothschild":33403,"Ġesche":33404,"Ġsoph":33405,"Ġambiguous":33406,"negative":33407,"Ġdiscouraging":33408,"Alexander":33409,"319":33410,"Ġsummon":33411,"ipation":33412,"000000":33413,"Ġminimalist":33414,"Ġenraged":33415,"777":33416,"Ġplanetary":33417,"Ġthroughput":33418,"Ġtemperament":33419,"ĠNIC":33420,"ileged":33421,"minster":33422,"ĠPLEASE":33423,"Ġexagger":33424,"ĠDescription":33425,"Ġagitated":33426,"Ġimmortal":33427,"Ġrenders":33428,"Ġcharisma":33429,"sequ":33430,"Ġmajorities":33431,"Ġfreaking":33432,"ĠAdvice":33433,"Ġembodies":33434,"stable":33435,"Ġcustomization":33436,"started":33437,"ĠAutism":33438,"Ġparticipates":33439,"ĠUTC":33440,"Marco":33441,"Ġoddly":33442,"Ġantiqu":33443,"ĠPear":33444,"ĠFey":33445,"Ġcertify":33446,"Ġdisillusion":33447,"ĠPhysicians":33448,"obl":33449,"855":33450,"Ġelim":33451,"Ġ335":33452,"Ol":33453,"ĠSear":33454,"Ġnuances":33455,"past":33456,"Sa":33457,"ĠSlov":33458,"Ġfiltered":33459,"Ġanalogy":33460,"Ġformulate":33461,"Ġarmies":33462,"Ġpuls":33463,"fters":33464,"ilipp":33465,"ĠHOT":33466,"485":33467,"ĠAfghans":33468,"Ġtopical":33469,"ĠBunny":33470,"seeing":33471,"Ġeloqu":33472,"Ġkidneys":33473,"ĠDEM":33474,"pent":33475,"Ġhus":33476,"stores":33477,"ĠProtestant":33478,"Comm":33479,"label":33480,"Kings":33481,"ĠPurpose":33482,"âĢ¦..":33483,"Ġaccumulating":33484,"calling":33485,"Ġgiveaways":33486,"Ġpredicament":33487,"Ġtyp":33488,"Ġtraveler":33489,"003":33490,"impro":33491,"fac":33492,"Ġmapped":33493,"itious":33494,"Ġmasculinity":33495,"Ġtantal":33496,"ĠDJs":33497,"Ġviewpoints":33498,"Burn":33499,"ĠWii":33500,"pak":33501,"ĠEB":33502,"Ġhinge":33503,"Ġfacets":33504,"Ġphotographic":33505,"Ġcompiling":33506,"Ġdecks":33507,"Ġarticulated":33508,"Federal":33509,"crim":33510,"llah":33511,"Ġfiasco":33512,"ĠLIST":33513,"oute":33514,"ĠDraper":33515,"ĠLaos":33516,"Ġclimbers":33517,"raph":33518,"ĠDek":33519,"WAY":33520,"Ġgreets":33521,"Ġoppressive":33522,"otor":33523,"otiation":33524,"\":[":33525,"Record":33526,"mining":33527,"Town":33528,"Ġfavorably":33529,"ĠYoutube":33530,"William":33531,"Ġlan":33532,"âĢ²":33533,"ĠSpec":33534,"Ġtranquil":33535,"ĠClient":33536,"oln":33537,"celona":33538,"Ġrealistically":33539,"Ġmisplaced":33540,"ĠBie":33541,"bye":33542,"Yo":33543,"465":33544,"ĠMadagascar":33545,"oplan":33546,"arist":33547,"Ġconfines":33548,"Ġï":33549,"awks":33550,"Ġpiracy":33551,"Ġunwelcome":33552,"Intel":33553,"Ġparanoid":33554,"CLAIM":33555,"Ġblush":33556,"united":33557,"Ġmotivational":33558,"ĠVII":33559,"Ġdiabetic":33560,"Ġantiv":33561,"Ġdissect":33562,"Ġbestselling":33563,"Ġfluffy":33564,"ĠRemote":33565,"Ġvert":33566,"Correct":33567,"Ġcolossal":33568,"Ġcontrasts":33569,"Ġcirca":33570,"ĠDamage":33571,"Ġunrel":33572,"Ġdiscrepancy":33573,"ĠCIS":33574,"ĠCLASS":33575,"ilty":33576,"Ġsynopsis":33577,"emed":33578,"cakes":33579,"ibal":33580,"inea":33581,"ienced":33582,"Ġimplicit":33583,"ĠLOOK":33584,"Ġsilhouette":33585,"affiliated":33586,"ĠHalo":33587,"377":33588,"Ġlyr":33589,"ĠVide":33590,"herent":33591,"Ġbadges":33592,"plays":33593,"orea":33594,"Ġjammed":33595,"cancer":33596,"ĠYep":33597,"racted":33598,"ĠDisability":33599,"Ġfooth":33600,"friends":33601,"Ġbloated":33602,"Bet":33603,"ĠAntioch":33604,"Ġintrodu":33605,"Ġannexed":33606,"ivism":33607,"ĠFlickr":33608,"pants":33609,"Ġinterruption":33610,"645":33611,"ĠIly":33612,"ĠOss":33613,"ĠAMA":33614,"Ġpolitely":33615,"Ġnatives":33616,"Ġrushes":33617,"enges":33618,"ĠHarm":33619,"Ġdestroyer":33620,"ĠEstimates":33621,"Ġtransforms":33622,"Ġinvariably":33623,"Ġcac":33624,"iency":33625,"599":33626,"Ġconstitutionally":33627,"Ġrappers":33628,"ĠSettlement":33629,"icz":33630,"Ġhardened":33631,"citizens":33632,"Ġcircling":33633,"Ġtrapping":33634,"Ġguaranteeing":33635,"690":33636,"agher":33637,"Ġarcade":33638,"Ġfanc":33639,"Ġslapping":33640,"OPS":33641,"Ġmasse":33642,"Ġpudding":33643,"Jac":33644,"ĠGraphics":33645,"Ġuptake":33646,"?,":33647,"Fair":33648,"ĠSatan":33649,"uffy":33650,"ĠGuatem":33651,"ĠTransaction":33652,"Ġunlocking":33653,"ĠLINE":33654,"Ġapprehens":33655,"Ġglean":33656,"291":33657,"Ġexacerbate":33658,"ĠTrave":33659,"ĠTrop":33660,"Supp":33661,"Ġqueens":33662,"cart":33663,"Ġscrolling":33664,"Ġox":33665,"cone":33666,"Matthew":33667,"ĠDIRECT":33668,"Ġbacker":33669,"Ġthyroid":33670,"Sarah":33671,"ĠEDIT":33672,"ĠActivision":33673,"352":33674,"Ġreinforcements":33675,"Ġding":33676,"Ġplush":33677,"Ġpeanuts":33678,"ĠFant":33679,"ĠPediatrics":33680,"Ġaccommodating":33681,"ĠPractices":33682,"Answer":33683,"racial":33684,"ĠConstant":33685,"740":33686,"strength":33687,"apist":33688,"Ġsynthes":33689,"ĠLeap":33690,"ĠFabric":33691,"Ġbrainstorm":33692,"obia":33693,"Ġconception":33694,"Ġtuberculosis":33695,"Ġmajestic":33696,"ĠTitus":33697,"ĠTee":33698,"Ġlikeness":33699,"ĠSEA":33700,"lite":33701,"Ġ950":33702,"sufficient":33703,"Ġtrem":33704,"Ġharshly":33705,"Ġredacted":33706,"Ġwelding":33707,"Ġperplex":33708,"Ġpoetic":33709,"Ġinsignificant":33710,"Ġware":33711,"Ġwandered":33712,"Ġmete":33713,"ĠSTART":33714,"Ġweaponry":33715,"opsy":33716,"shadow":33717,"Ġobsc":33718,"hare":33719,"ĠOPEN":33720,"Ġdiligent":33721,"Girls":33722,"Ġinitials":33723,"Start":33724,"ĠBrookings":33725,"ombs":33726,"Ġlashes":33727,"essor":33728,"Ġgravy":33729,"ĠUbuntu":33730,"Tree":33731,"Ġ435":33732,"Ġcellar":33733,"Ġaquarium":33734,"ĠPodesta":33735,"361":33736,"ĠController":33737,"Ġeru":33738,"reasonable":33739,"Ġpermissions":33740,"725":33741,"Ġadministering":33742,"Ġflirt":33743,"Ġfleeting":33744,"asive":33745,"Ġsubcontract":33746,"Ġfascist":33747,"Ġcabbage":33748,"science":33749,"Ġboiler":33750,"ioned":33751,"Ġintegrates":33752,"Ġresidue":33753,"KEY":33754,"Ġwi":33755,"Ġsquared":33756,"Unless":33757,"Ġmute":33758,"ĠTuc":33759,"Ġverb":33760,"Gary":33761,"Ġexperimentation":33762,"fee":33763,"chini":33764,"Ġmarrow":33765,"ĠBalt":33766,"Ġnodded":33767,"tn":33768,"Ġmissionary":33769,"OTO":33770,"Ġoptimum":33771,"555":33772,"Ġwhipping":33773,"aunts":33774,"ĠScene":33775,"Ġcharacterize":33776,"Ġretrospective":33777,"Ġutilizes":33778,"Ġhastily":33779,"older":33780,"ĠPW":33781,"Ġsleepy":33782,"020":33783,"ĠAcid":33784,"Ġridiculously":33785,"Ġgigg":33786,"649":33787,"Ġcrus":33788,"ĠShame":33789,"ĠTorn":33790,"finding":33791,"IPS":33792,"Ġplat":33793,"ometers":33794,"Ġamphib":33795,"ellow":33796,"ĠSpecies":33797,"commercial":33798,"Ġvirgin":33799,"Ġdarn":33800,"Ġsorely":33801,"Ġrespondent":33802,"Ġray":33803,"ĠCONS":33804,"Ġunequivocally":33805,"server":33806,"Ġdrip":33807,"ĠRazor":33808,"Ban":33809,"ĠHMS":33810,"Ġhijab":33811,"ĠMuss":33812,"Ġsandy":33813,"Ġaversion":33814,"Ġoverarching":33815,"Ġultr":33816,"ĠIraqis":33817,"Ġuninterrupted":33818,"Ġrouting":33819,"Ġundone":33820,"independence":33821,"gra":33822,"ysics":33823,"inflammatory":33824,"cussion":33825,"ĠDefinitely":33826,"Ġelastic":33827,"peer":33828,"ĠGiov":33829,"ĠMandarin":33830,"Ġscratches":33831,"Ġphysicist":33832,"Ġbestowed":33833,"usually":33834,"OULD":33835,"igration":33836,"Human":33837,"Dead":33838,"osph":33839,"bott":33840,"doctoral":33841,"Ġbending":33842,"Ġconfigurations":33843,"psych":33844,"db":33845,"ĠUD":33846,"Ġarteries":33847,"orically":33848,"Ġblasphemy":33849,"jj":33850,"checking":33851,"adian":33852,"IRD":33853,"ĠDialogue":33854,"Ġshielded":33855,"ĠVox":33856,"Dave":33857,"Ġturb":33858,"ĠMassive":33859,"ĠBMI":33860,"ĠNF":33861,"uced":33862,"ickle":33863,"ishable":33864,"Ġembody":33865,"ÙĪ":33866,"Senior":33867,"ĠResult":33868,"try":33869,"egu":33870,"401":33871,"ĠLoyal":33872,"Ġperilous":33873,"Ġdissu":33874,"Ġmythology":33875,"ĠWax":33876,"Jesus":33877,"ĠMotorsport":33878,"Ġadvis":33879,"ĠAki":33880,"ISM":33881,"tested":33882,"Ġplag":33883,"Ġriches":33884,"ĠOCT":33885,"ĠLocke":33886,"BG":33887,"Ġ460":33888,"rawl":33889,"ĠTermin":33890,"Ġ295":33891,"Ġchopping":33892,"KT":33893,"Ġconverts":33894,"Ask":33895,"alse":33896,"ĠKeynes":33897,"Ġrefuted":33898,"Ġrabbits":33899,"Ġbilingual":33900,"urse":33901,"ĠSalad":33902,"odiac":33903,"Ġsolidly":33904,"Dam":33905,"Ġpp":33906,"rities":33907,"Rah":33908,"itness":33909,"Ġsixty":33910,"332":33911,"cold":33912,"Ġhindered":33913,"Ġclipped":33914,"Ġreceptor":33915,"ĠHoms":33916,"Ġdusk":33917,"Ġarchae":33918,"LR":33919,"Ġrods":33920,"Ġ257":33921,"ĠSith":33922,"ĠPumpkin":33923,"ellation":33924,"ĠWD":33925,"Ġdecriminal":33926,"Ġusable":33927,"Ġcheerful":33928,"ĠInform":33929,"Ġbrushes":33930,"vier":33931,"ĠBrush":33932,"590":33933,"boost":33934,"guided":33935,"ĠMJ":33936,"Ġsatirical":33937,"ortion":33938,"efficiency":33939,"Ġstrands":33940,"ĠWilde":33941,"Ġreproduce":33942,"verage":33943,"Ġlug":33944,"Ġhist":33945,"offer":33946,"Ġcollapses":33947,"Ġclerks":33948,"Ġairstrike":33949,"IPP":33950,"iscover":33951,"Ġnefarious":33952,"Ġstripe":33953,"Ġbona":33954,"ocon":33955,"Ġpunishments":33956,"ITED":33957,"ĠAltern":33958,"testing":33959,"Ġeerie":33960,"erous":33961,"Ġcaves":33962,"Ġcondemns":33963,"ĠDropbox":33964,"inese":33965,"axis":33966,"ĠRegistry":33967,"ĠMong":33968,"Ġbullies":33969,"Ġdocks":33970,"ĠAlter":33971,"rella":33972,"446":33973,"ĠDare":33974,"Ġvirtues":33975,"Ġdont":33976,"Value":33977,"ENE":33978,"received":33979,"Ġseaf":33980,"476":33981,"ilon":33982,"ĠKits":33983,"Ġrarity":33984,"Ġnurt":33985,"skin":33986,"ĠUL":33987,"ĠRegiment":33988,"terior":33989,"hate":33990,"ĠEstimated":33991,"ĠSilence":33992,"Ġorganism":33993,"ĠSigned":33994,"ĠIA":33995,"bite":33996,"Ġthicker":33997,"Ġeyeb":33998,"Ġjournalistic":33999,"ĠDisp":34000,"margin":34001,"Dri":34002,"Ġcomplexes":34003,"Ġimaginary":34004,"Ġrefuel":34005,"Ġmeticulous":34006,"Dub":34007,"Ġhaze":34008,"860":34009,"Ġproverbial":34010,"Ġozone":34011,"cale":34012,"resent":34013,"Ġdiscrete":34014,"boats":34015,"Ġ343":34016,"ĠRET":34017,"Ġsailor":34018,"hair":34019,"gear":34020,"Ġmalt":34021,"Ġpeach":34022,"ĠRabb":34023,"699":34024,"318":34025,"ĠVerge":34026,"Fin":34027,"ĠMighty":34028,"ierce":34029,"403":34030,"Ġdisenfranch":34031,"bass":34032,"nice":34033,"Ġsinks":34034,"ĠLaugh":34035,"367":34036,"ĠZur":34037,"Ġtravers":34038,"ĠMystery":34039,"onsense":34040,"ĠMonarch":34041,"Ġleapt":34042,"ergy":34043,"porate":34044,"display":34045,"ilet":34046,"Ġendemic":34047,"Bern":34048,"Ġpulmonary":34049,"Ġbroch":34050,"ĠManziel":34051,"Lyn":34052,"Repe":34053,"lda":34054,"hands":34055,"Ġtroublesome":34056,"Jordan":34057,"UTION":34058,"ĠALP":34059,"ĠLEG":34060,"Ġreconnaissance":34061,"ĠRNA":34062,"letters":34063,"ĠYounger":34064,"ĠLW":34065,"ĠSensor":34066,"388":34067,"Ġwielding":34068,"spr":34069,"Ġancestral":34070,"331":34071,"OTH":34072,"ĠAxis":34073,"irement":34074,"ĠCompact":34075,"voice":34076,"Ġpercussion":34077,"Ġendeav":34078,"Kate":34079,"ĠJACK":34080,"ĠMagnus":34081,"Ġinterconnected":34082,"ĠTraff":34083,"demon":34084,"Ġardent":34085,"ĠSomers":34086,"andum":34087,"346":34088,"heartedly":34089,"ayne":34090,"Design":34091,"melon":34092,"ĠCarib":34093,"Ġ1935":34094,"intention":34095,"cape":34096,"cend":34097,"organic":34098,"373":34099,"ĠRevival":34100,"ĠBLACK":34101,"Ġaspiration":34102,"yellow":34103,"bodied":34104,"Ġcrave":34105,"ĠIntelligent":34106,"ĠUnique":34107,"tab":34108,"386":34109,"ĠNess":34110,"Official":34111,"Stay":34112,"Ġcreat":34113,"iliary":34114,"rified":34115,"ĠPok":34116,"Ġabolition":34117,"Ka":34118,"ĠCourage":34119,"ĠDickens":34120,"rophic":34121,"ĠFAR":34122,"Ġfurnished":34123,".âĢĵ":34124,"rete":34125,"Ġvaginal":34126,"hner":34127,"ĠLONG":34128,"imates":34129,"ĠLiter":34130,"ĠMeasures":34131,"ĠBelg":34132,"\"-":34133,"ĠRaider":34134,"enario":34135,"rification":34136,"ĠFISA":34137,"ĠStab":34138,"Ġnar":34139,"mund":34140,"Tenn":34141,"Ġwakes":34142,"Ġcharg":34143,"okers":34144,"assment":34145,"Ġsiph":34146,"Ġludicrous":34147,"670":34148,"Ġcompositions":34149,"Ġpinnacle":34150,"ĠRankings":34151,"ĠTelescope":34152,"secure":34153,"Ġib":34154,"Ġaptly":34155,"paste":34156,"ĠJUST":34157,"RD":34158,"herry":34159,"sung":34160,"Ġmig":34161,"naires":34162,"Ġmigrated":34163,"Base":34164,"Ġamazingly":34165,"Ġunregulated":34166,"published":34167,"ĠPIT":34168,"ĠMissile":34169,"extreme":34170,"ĠAlone":34171,"skilled":34172,"ĠRamp":34173,"Ġcamer":34174,"Ġflyer":34175,"Ġbrewers":34176,"ĠReference":34177,"ĠMOV":34178,"ĠLep":34179,"Ġentitle":34180,"ivals":34181,"ĠPIN":34182,"Ġbatches":34183,"Ġunexplained":34184,"Ġenergies":34185,"Ġblurred":34186,"enged":34187,"orig":34188,"WF":34189,"olves":34190,"ĠPicks":34191,"ĠTwice":34192,"arranted":34193,"Ġmembrane":34194,"ĠMoonlight":34195,"Ġsulfur":34196,"Ġpurposely":34197,"Ġfumes":34198,"Ġ(#":34199,"onics":34200,"ivities":34201,"rollers":34202,"Ġflattering":34203,"felt":34204,"Ġintoxication":34205,"Bridge":34206,"ĠFallout":34207,"Ġcreatively":34208,"Ġpsychologically":34209,"Ġdespicable":34210,"gae":34211,"820":34212,"VERS":34213,"Ġtidal":34214,"Ġcarbohydrates":34215,"strip":34216,"Ġgravitational":34217,"Ġfeds":34218,"ĠZhao":34219,"legates":34220,"Ġ307":34221,"String":34222,"ĠRepair":34223,"Ġ1928":34224,"orses":34225,"atography":34226,"Boston":34227,"Ġasymm":34228,"ĠSomebody":34229,"Van":34230,"ĠSovereign":34231,"Ġnotoriety":34232,"Ġsimulate":34233,"ĠDiscussion":34234,"ĠTransition":34235,"Ġcopying":34236,"antage":34237,"ĠRodrig":34238,"Ġindifference":34239,"Ġ580":34240,"Ġastronomical":34241,"Ġscrews":34242,"840":34243,"inates":34244,"ĠStreaming":34245,"Ġentit":34246,"ĠLiterature":34247,"369":34248,"805":34249,"OTS":34250,"о":34251,"img":34252,"inness":34253,"Ġreverber":34254,"Ġpartition":34255,"Short":34256,"Ġmoist":34257,"Ġspoof":34258,"ĠDesire":34259,"orce":34260,"Ġcrammed":34261,"Ġunfor":34262,"Pan":34263,"ingen":34264,"Ġrelat":34265,"Mother":34266,"ĠGn":34267,"altern":34268,"Ġresurg":34269,"Ġcramped":34270,"ĠCitadel":34271,"Ġlaureate":34272,"Ġanalys":34273,"Ġnuns":34274,"ĠTie":34275,"activ":34276,"ĠSurprisingly":34277,"ĠProtective":34278,"ĠRedemption":34279,"Ġendlessly":34280,"Ġfists":34281,"spl":34282,"ĠKron":34283,"ĠExamples":34284,"Especially":34285,"Ġprejud":34286,"ĠSchwar":34287,"Ġ237":34288,"ĠPlants":34289,"ĠUNDER":34290,"Ġlasers":34291,"Ġsher":34292,"Ġgoddess":34293,"Ġwipes":34294,"409":34295,"ĠGTA":34296,"Ġhybrids":34297,"rowd":34298,"ĠMILL":34299,"ĠNUM":34300,"ĠGeek":34301,"ĠTWO":34302,"ĠTimbers":34303,"Ġresembled":34304,"ĠGRE":34305,"Bring":34306,"Ġcompressed":34307,"ĠOral":34308,"379":34309,"Ġwrench":34310,"LCS":34311,"Ġhomosexual":34312,"Kelly":34313,"Ġhump":34314,"ĠSicily":34315,"Ġperished":34316,"aos":34317,"doesn":34318,"scrib":34319,"Charlie":34320,"Ġshuffle":34321,"372":34322,"cedented":34323,"402":34324,"Ġtiers":34325,"Ġinteracted":34326,"ĠHG":34327,"ĠJere":34328,"ĠBRA":34329,"ĠDOC":34330,"things":34331,"Ġfaiths":34332,"Ġgirlfriends":34333,"Ġfortified":34334,"develop":34335,"ĠKus":34336,"iability":34337,"rase":34338,"iotics":34339,"ĠChern":34340,"boxes":34341,"abol":34342,"idan":34343,"emon":34344,"ĠJudaism":34345,"ĠSituation":34346,"ĠGrimm":34347,"Ġgou":34348,"ĠVictim":34349,"backer":34350,"Ġanimosity":34351,"ĠHorizons":34352,"ĠKazakh":34353,"Ġgrossly":34354,"ĠTac":34355,"yg":34356,"366":34357,"Ġcheaply":34358,"Ġformulated":34359,"ĠDangerous":34360,"offensive":34361,"Ġsauces":34362,"Ġkeyboards":34363,"666":34364,"Ġcanopy":34365,"Inc":34366,"astered":34367,"iesel":34368,"Ġadv":34369,"currency":34370,"Ġscapego":34371,"plings":34372,"ĠBDS":34373,"Ġstrangely":34374,"today":34375,"ĠEgyptians":34376,"Ġcoron":34377,"often":34378,"ĠTransformers":34379,"ĠAfterwards":34380,"reated":34381,"Ġpoisonous":34382,"Ġgeographically":34383,"Ġmell":34384,"Cross":34385,"Ġdeductible":34386,"ĠZionist":34387,"Ġcutter":34388,"ĠRP":34389,"ĠImag":34390,"Ġoverflow":34391,"358":34392,"ĠADD":34393,"bones":34394,"Ġflattened":34395,"ĠGREEN":34396,"Ġlaure":34397,"haps":34398,"ĠCellular":34399,"kens":34400,"363":34401,"ĠSmash":34402,"ĠSpeak":34403,"ĠMaiden":34404,"Ġgreedy":34405,"ĠManit":34406,"Ġfacet":34407,"ĠGPA":34408,"Ġracks":34409,"popular":34410,"322":34411,"ĠBars":34412,"avement":34413,"359":34414,"Ġpomp":34415,"Ġregisters":34416,"Fs":34417,"ĠLoving":34418,"ĠTaxi":34419,"concert":34420,"ĠArchae":34421,"Ġcurls":34422,"ĠSpit":34423,"ĠLIFE":34424,"Ġinvade":34425,"rolog":34426,"wreck":34427,"Ġconflicted":34428,"Ġ970":34429,"Ġexiled":34430,"Ġchew":34431,"udging":34432,"Ġexper":34433,"ĠFt":34434,"rius":34435,"ĠXer":34436,"~":34437,"Ġbandwagon":34438,"Fore":34439,"Cat":34440,"Ġoverflowing":34441,"Ġradios":34442,"Much":34443,"Ġfacilitates":34444,"ĠCaf":34445,"ĠQing":34446,"Use":34447,"Ġmang":34448,"Ġpissed":34449,"ĠOuter":34450,"within":34451,"ĠSchr":34452,"ĠSherlock":34453,"Ġ336":34454,"Ġcasc":34455,"chens":34456,"incent":34457,"Ġcultivating":34458,"ampions":34459,"Ġwasteful":34460,"adays":34461,"sets":34462,"ĠLF":34463,"watching":34464,"Ġabandonment":34465,"ĠJesuit":34466,"Ġlegislatures":34467,"regnancy":34468,"ĠColt":34469,"Ġinterns":34470,"Ġundertook":34471,"ĠIPA":34472,"ĠInstall":34473,"nsics":34474,"washer":34475,"Ġbeginners":34476,"ĠDiseases":34477,"Ġlimp":34478,"ĠESA":34479,"Basically":34480,"Ġprud":34481,"LED":34482,"Ġgrease":34483,"ousel":34484,"Ġrotten":34485,"ĠCele":34486,"facts":34487,"ĠLouie":34488,"ĠISI":34489,"481":34490,"Ġsett":34491,"Ġtoug":34492,"ĠReck":34493,"OUNT":34494,"ĠFou":34495,"Ġinhibitor":34496,"gru":34497,"bane":34498,"1980":34499,"ĠPanc":34500,"Ġsuperficial":34501,"Ġauthoritative":34502,"ĠVOL":34503,"790":34504,"Ġcrusade":34505,"airy":34506,"Ġemphatically":34507,"Ġflourishing":34508,"Ġ416":34509,"Ġheroine":34510,"inx":34511,"Ġanch":34512,"stretched":34513,"ĠRegener":34514,"ĠAncient":34515,"evaluate":34516,"Ġantibody":34517,"ĠEston":34518,"ĠAeg":34519,"Ġboldly":34520,"TN":34521,"ĠPercentage":34522,"Ġ747":34523,"Ġrapt":34524,"ĠEdited":34525,"Earth":34526,"phal":34527,"ĠXXX":34528,"arling":34529,"ĠReligion":34530,"Ġ503":34531,"forces":34532,"Ġendpoint":34533,"Miller":34534,"Ba":34535,"Ġdisappears":34536,"andre":34537,"Ġconnector":34538,"407":34539,"ĠTOUR":34540,"aura":34541,"ĠRazer":34542,"UPDATE":34543,"Ġcalib":34544,"original":34545,"ĠMonkey":34546,"Ir":34547,"Ġexacerb":34548,"killing":34549,"Ġforb":34550,"native":34551,"Ġpoking":34552,"Ġveiled":34553,"mails":34554,"Ġalphabet":34555,"Ġawkwardly":34556,"ĠNames":34557,"Ġspiders":34558,"ĠParam":34559,"ĠColour":34560,"Ġunification":34561,"ĠPione":34562,"Ġoffend":34563,"Ġscoff":34564,"ĠSAR":34565,"ĠBuildings":34566,"edes":34567,"ĠAke":34568,"Ġfirmware":34569,"Madison":34570,"policy":34571,"ĠComputing":34572,"ĠRW":34573,"Ġfluent":34574,"Ġdece":34575,"Ġswore":34576,"Ġrestaur":34577,"Ġpresses":34578,"ophon":34579,"Ġphilosopher":34580,"ften":34581,"Ġintruder":34582,"Ġleng":34583,"ĠCowboy":34584,"cled":34585,"Ġmeticulously":34586,"ĠPair":34587,"ĠEND":34588,"Ġcapsules":34589,"Ġauxiliary":34590,"Ġverses":34591,"Ġsheltered":34592,"Ġexplorer":34593,"ĠWolverine":34594,"auts":34595,"Ġinhibitors":34596,"ĠPeng":34597,"ĠValve":34598,"imar":34599,"Ġchuck":34600,"ĠRecording":34601,"Ġardu":34602,"Test":34603,"Ġinterven":34604,"Ġchrome":34605,"months":34606,"tap":34607,"ĠManz":34608,"format":34609,"ĠBalkans":34610,"Ġannex":34611,"uder":34612,"ĠAAC":34613,"Ġdisturbances":34614,"354":34615,"asms":34616,"ĠTad":34617,"puting":34618,"Ġfateful":34619,"imen":34620,"Ġaudi":34621,"ĠNewsweek":34622,"Around":34623,"Ġretribution":34624,"Ġsugars":34625,"Ġescapes":34626,"Ġlegitim":34627,"ĠProof":34628,"Ġmisogyn":34629,"cit":34630,"Ġclutching":34631,"exist":34632,"Ġrevol":34633,"Ġdiscs":34634,"discrimination":34635,"Ġstout":34636,"aline":34637,"ĠRandom":34638,"364":34639,"Ġapprehension":34640,"Ġmockery":34641,"Ġfossils":34642,"ĠStress":34643,"Ġbenefic":34644,"exc":34645,"lude":34646,"Small":34647,"Ġgh":34648,"Ġobserves":34649,"ĠSUP":34650,"Ġbrewer":34651,"ĠESP":34652,"Ġomitted":34653,"multiple":34654,"Ġminimizing":34655,"Ġtaco":34656,"Ġindifferent":34657,"medi":34658,"available":34659,"Ġ252":34660,"Ġsanity":34661,"ĠCookie":34662,"mostly":34663,"near":34664,"NASA":34665,"Ġlowly":34666,"seless":34667,"Ġobsess":34668,"itous":34669,"Dispatch":34670,"Ġcanyon":34671,"Ġbriefs":34672,"Say":34673,"ĠNato":34674,"ĠSpend":34675,"Ġ242":34676,"ĠEthernet":34677,"Ġmatte":34678,"ĠStim":34679,"hetics":34680,"Ġflourished":34681,"389":34682,"ĠMcA":34683,"695":34684,"Ġoverr":34685,"Ġtorment":34686,"Ġpirate":34687,"ĠJohann":34688,"roversial":34689,"ĠUnemployment":34690,"breakers":34691,"ĠMessages":34692,"tones":34693,"Ġtagging":34694,"Ġfrog":34695,"Jewish":34696,"Ġmessenger":34697,"Ġexasper":34698,"ernaut":34699,"Ġnarrower":34700,"ĠCatalyst":34701,"ĠSecrets":34702,"Ġadj":34703,"ĠFug":34704,"Ġaura":34705,"Ġtherape":34706,"mber":34707,"Ġcaliphate":34708,"Ġretreating":34709,"ĠComput":34710,"Ġburying":34711,"Ġail":34712,"Ġgriev":34713,"lins":34714,"825":34715,"tten":34716,"ifully":34717,"ĠTrials":34718,"igma":34719,"Ġ1914":34720,"Ġcoordinates":34721,"ocusing":34722,"ĠFeng":34723,"ĠWhale":34724,"Ġshorten":34725,"Ġcorrectness":34726,"evil":34727,"network":34728,"Ġreactive":34729,"assuming":34730,"ĠLaksh":34731,"games":34732,"Ġruining":34733,"excluding":34734,"annels":34735,"º":34736,"Ġrubbed":34737,"aleb":34738,"flex":34739,"iped":34740,"ĠLimit":34741,"allowed":34742,"ĠDMV":34743,"ĠLD":34744,"Ġstamina":34745,"conduct":34746,"Ġmislead":34747,"lib":34748,"ĠEminem":34749,"Ġpayoff":34750,"Ġkernel":34751,"Ġsweeps":34752,"Ġsonic":34753,"ĠKodi":34754,"unique":34755,"Ġsurrog":34756,"Michigan":34757,"Ġattest":34758,"Ġdummy":34759,"ĠStellar":34760,"ĠSquadron":34761,"ĠHait":34762,"ĠSpirits":34763,"605":34764,"ĠHemisphere":34765,"legram":34766,"ĠRack":34767,"opol":34768,"Ġfreshwater":34769,"cession":34770,"Ġabort":34771,"ĠLOG":34772,"Ġfuzzy":34773,"Ġcrystall":34774,"illation":34775,"ĠFreddy":34776,"Ġsalvation":34777,"Ġjuxtap":34778,"weekly":34779,"usha":34780,"456":34781,"Ġ660":34782,"ĠGlacier":34783,"Ġnegatives":34784,"Ġillegitimate":34785,"ĠProtein":34786,"Moore":34787,"Der":34788,"Ġinfancy":34789,"Again":34790,"ALD":34791,"Leon":34792,"ĠIdeally":34793,"fresh":34794,"730":34795,"Ġgamb":34796,"Ġscrewed":34797,"wow":34798,"Ġembodied":34799,"ĠCinderella":34800,"341":34801,"ĠPiano":34802,"Ġbroccoli":34803,"Ġmats":34804,"ĠZheng":34805,"cream":34806,"anut":34807,"ĠZig":34808,"Columb":34809,"ĠTibetan":34810,"Death":34811,"Ġstren":34812,"ĠVertical":34813,"Ġratification":34814,"Ġprincipally":34815,"ELD":34816,"Ġforbid":34817,"Ġamalg":34818,"blind":34819,"auri":34820,"stery":34821,"Ġbarley":34822,"FBI":34823,"ĠHex":34824,"925":34825,"Domin":34826,"oat":34827,"Ġswayed":34828,"ĠKKK":34829,"ĠTaxes":34830,"Ġker":34831,"eeper":34832,"ĠAwakens":34833,"ĠPix":34834,"ĠKING":34835,"dc":34836,"Ren":34837,"Ġlegitimately":34838,"ĠTriumph":34839,"ĠSites":34840,"ĠSai":34841,"tl":34842,"painted":34843,"ĠWaiting":34844,"starting":34845,"parents":34846,"ĠDuo":34847,"eele":34848,"upper":34849,"ĠInvestig":34850,"Ġeighteen":34851,"Ġcorrelated":34852,"ĠCascade":34853,"acca":34854,"ĠAlph":34855,"ĠPolic":34856,"ĠEVs":34857,"Ġworthless":34858,"ĠIndust":34859,"auld":34860,"ĠYiannopoulos":34861,"ĠEzra":34862,"Ġmorphed":34863,"Ġoriginating":34864,"mania":34865,"Ġsparing":34866,"Ġextrem":34867,"cre":34868,"ults":34869,"mare":34870,"classified":34871,"Ġparachute":34872,"Ġmistrust":34873,"ONT":34874,"Mind":34875,"Ġthru":34876,"707":34877,"ĠTwain":34878,"Ġmelodies":34879,"ĠDanger":34880,"ĠDPS":34881,"Ġderive":34882,"Ġdissolution":34883,"Ġchildbirth":34884,"Ġ415":34885,"fork":34886,"solid":34887,"loads":34888,"ĠCGI":34889,"378":34890,"ĠShed":34891,"Face":34892,"Ġcomet":34893,"iceps":34894,"ĠReduction":34895,"Fly":34896,"jp":34897,"ĠAnimation":34898,"Luke":34899,"Ġabiding":34900,"Ġdevise":34901,"ĠAe":34902,"Ġflux":34903,"Ġbras":34904,"Ġfracturing":34905,"Ġinventive":34906,"ĠGranger":34907,"Ġsap":34908,"inducing":34909,"Ġreviewers":34910,"Officers":34911,"ĠWHY":34912,"Ġamplify":34913,"Ġentr":34914,"Ġslit":34915,"457":34916,"Ġreformed":34917,"ĠPhi":34918,"Ġtempt":34919,"Ġcontradiction":34920,"585":34921,"ĠMaced":34922,"371":34923,"kinson":34924,"robe":34925,"ĠHunters":34926,"astern":34927,"criminal":34928,"jew":34929,"Ġdecentralized":34930,"bands":34931,"Ġavatar":34932,"ĠBarrier":34933,"Ġcharacterization":34934,"student":34935,"Ġgays":34936,"Ġspecialize":34937,"ĠJudging":34938,"Ġinitiation":34939,"Ġshove":34940,"Ġpirates":34941,"Ġfictitious":34942,"ĠPoker":34943,"ĠElsa":34944,"ĠTECH":34945,"handedly":34946,"Ġglued":34947,"Ġclinically":34948,"Ġinaccessible":34949,"Ġderegulation":34950,"Ġprohib":34951,"Ġdangling":34952,"Ġnoses":34953,"Ġstash":34954,"اØ":34955,"ESH":34956,"Ġmonstrous":34957,"Ġcrept":34958,"ĠCharm":34959,"Ġbeh":34960,"Ġshuts":34961,"Ġ236":34962,"imedia":34963,"445":34964,"Du":34965,"Ġafar":34966,"ĠRout":34967,"Ġflares":34968,"Utah":34969,"Ġ808":34970,"Ġjewels":34971,"2004":34972,"Ġrecal":34973,"Gas":34974,"ĠExcellent":34975,"Ġpitfalls":34976,"ĠDrawing":34977,"viously":34978,"angered":34979,"changes":34980,"Ġpasture":34981,"talking":34982,"Ġinequ":34983,"Ġbicycl":34984,"Cost":34985,"423":34986,"bard":34987,"Ġanterior":34988,"ecast":34989,"CHR":34990,"397":34991,"masters":34992,"706":34993,"ĠFinish":34994,"Yet":34995,"study":34996,"ĠCogn":34997,"Ġloaf":34998,"Ġspatial":34999,"ĠParad":35000,"batch":35001,"Ġvents":35002,"Ġspins":35003,"ĠAddiction":35004,"Ġcondone":35005,"Ġproble":35006,"English":35007,"ĠRomans":35008,"ĠSaying":35009,"ĠKling":35010,"Universal":35011,"ivist":35012,"Ġskirm":35013,"Ġ2500":35014,"Ġ263":35015,"aired":35016,"ĠMartian":35017,"ĠCompensation":35018,"lation":35019,"ĠSalam":35020,"LGBT":35021,"ĠDart":35022,"strike":35023,"vasive":35024,"ILLE":35025,"Ġimaginative":35026,"ĠEuph":35027,"Financial":35028,"Ġholog":35029,"orah":35030,"crit":35031,"ĠOswald":35032,"512":35033,"ĠUri":35034,"Ġdiscrepancies":35035,"Ġbeads":35036,"ĠShots":35037,"Mem":35038,"Ġhunts":35039,"Ġsubtly":35040,"Ġ470":35041,"ĠVigil":35042,"Ġsew":35043,"ĠBurma":35044,"igm":35045,"ighed":35046,"swe":35047,"Ġ251":35048,"Ġdeceit":35049,"Ġphysi":35050,"iflower":35051,"ĠCert":35052,"Ġchewing":35053,"rax":35054,"ĠMER":35055,"icient":35056,"Les":35057,"Ġ390":35058,"Ġperjury":35059,"Ġfiltering":35060,"770":35061,"Ġpoppy":35062,"Ġbland":35063,"ĠNasa":35064,"Ġorbiting":35065,"ĠRipple":35066,"otal":35067,"ĠRyu":35068,"ĠShap":35069,"ĠJian":35070,"Ġpiv":35071,"ĠNeptune":35072,"rary":35073,"Ġunavoidable":35074,"Ġguideline":35075,"Ġwaterfall":35076,"inators":35077,"ĠLogic":35078,"ĠPlug":35079,"role":35080,"Ġalterations":35081,"ĠSett":35082,"ĠFeld":35083,"Ġfreezes":35084,"Ġbedrock":35085,"ĠVIEW":35086,"ovation":35087,"Ġneedless":35088,"ĠIU":35089,"ignant":35090,"ĠConfeder":35091,"316":35092,"fine":35093,"Ġjars":35094,"gotten":35095,"Bron":35096,"Ġmindfulness":35097,"imating":35098,"Ġhysteria":35099,"Ġhurried":35100,"Ġinfantry":35101,"ĠNYU":35102,"tags":35103,"Penn":35104,"Ġtracing":35105,"ĠSwing":35106,"ĠIo":35107,"Ġreckoned":35108,"ĠRecall":35109,"ĠVersion":35110,"314":35111,"Ġecology":35112,"Ġarmoured":35113,"Ġresonance":35114,"970":35115,"Ġvigilance":35116,"Ġrede":35117,"ĠBohem":35118,"Ġchau":35119,"ĠDevi":35120,"Ġtru":35121,"))":35122,"Put":35123,"Ġflavored":35124,"ĠClown":35125,"Senate":35126,"ĠScandinavian":35127,"mable":35128,"Residents":35129,"ĠFranchise":35130,"Ġprecincts":35131,"Prem":35132,"ĠNeutral":35133,"coal":35134,"Ġdelinqu":35135,"Mus":35136,"UME":35137,"Ġtedious":35138,"roots":35139,"ĠCondition":35140,"ĠIntercept":35141,"017":35142,"itives":35143,"Ġdefinitively":35144,"Ġobliter":35145,"Ġclandestine":35146,"Ġstagnation":35147,"Ġblindness":35148,"abiding":35149,"Ġremix":35150,"feeding":35151,"Ġunrecogn":35152,"2003":35153,"960":35154,"381":35155,"Ġbulky":35156,"xia":35157,"ivered":35158,"inic":35159,"ĠSoci":35160,"ĠYards":35161,"Ġhides":35162,"Film":35163,"Ġtestim":35164,"Ġblacklist":35165,"Deep":35166,"Standard":35167,"ĠClash":35168,"Ġriddled":35169,"Ġdiseng":35170,"ĠTRE":35171,"ĠIDs":35172,"Ġmigrating":35173,"protect":35174,"Ġgraded":35175,"Ġvaguely":35176,"ĠCharacter":35177,"382":35178,"ĠMOD":35179,"Eng":35180,"Ġmobilized":35181,"Ġsincerity":35182,"Ġ317":35183,"sighted":35184,"ownt":35185,"ĠâĢİ":35186,"umpy":35187,"Ġitching":35188,"ĠVerd":35189,"cook":35190,"Ġsimulator":35191,"players":35192,"Early":35193,"infeld":35194,"Ġmaximizing":35195,"Philipp":35196,"ĠPhotoshop":35197,"Ġdestroys":35198,"Ġbefriend":35199,"Ġfilthy":35200,"ĠIncident":35201,"gha":35202,"Ġcomplicity":35203,"Ġmessing":35204,"YA":35205,"ĠNegro":35206,"adows":35207,"374":35208,"Ġpip":35209,"cean":35210,"Ġ1924":35211,"Sent":35212,"represent":35213,"Ġdeems":35214,"ĠRue":35215,"Ġtitanium":35216,"Ġmanners":35217,"âĢ¦âĢ¦":35218,"bare":35219,"Ġusur":35220,"mma":35221,"ĠPanda":35222,"ulus":35223,"ĠSlav":35224,"324":35225,"ĠMole":35226,"^":35227,"micro":35228,"foreign":35229,"lest":35230,"ocular":35231,"ĠUniv":35232,"ĠFrag":35233,"Ġshepherd":35234,"Ġelectron":35235,"ĠFSA":35236,"Ġunl":35237,"dose":35238,"Ġimmersion":35239,"ĠDeL":35240,"Ġbiomedical":35241,"Anna":35242,"Ġskillet":35243,"Ġrecre":35244,"Ġtrillions":35245,"voy":35246,"Ġnormalized":35247,"radio":35248,"cue":35249,"urbed":35250,"Ġthinkers":35251,"328":35252,"327":35253,"ĠForge":35254,"505":35255,"Ġunbearable":35256,"olini":35257,"Ġdisinfect":35258,"Ġshaving":35259,"Ġtoxicity":35260,"453":35261,"Ġheterosexual":35262,"Baltimore":35263,"Ġstool":35264,"lr":35265,"ĠMk":35266,"Ġantidote":35267,"Dark":35268,"810":35269,"Ġirritated":35270,"ĠSUPPORT":35271,"Chance":35272,"bent":35273,"ĠZelda":35274,"ĠPenguin":35275,"ifled":35276,"Ġarte":35277,"705":35278,"Ġcondol":35279,"izza":35280,"ĠCK":35281,"Ġprojector":35282,"ravings":35283,"Ġ1919":35284,"Ġburner":35285,"ĠSchwarz":35286,"Oregon":35287,"Ġridicule":35288,"Ġinstructional":35289,"Ġ\"#":35290,"ĠDign":35291,"Ġkitten":35292,"Ġconstit":35293,"iration":35294,"Speed":35295,"ecycle":35296,"ĠFalse":35297,"ĠDealer":35298,"Could":35299,"655":35300,"outside":35301,"Ġworldview":35302,"Ġ246":35303,"Ġspitting":35304,"595":35305,"MN":35306,"ĠComes":35307,"ingu":35308,"Ġenzymes":35309,"Ġcompass":35310,"Ġexclaimed":35311,"ĠMalays":35312,"Ġ1916":35313,"Ġcoloring":35314,"Ġrepeats":35315,"Ġsoils":35316,"Ġtrivia":35317,"ĠIsles":35318,"Const":35319,"ĠFiction":35320,"665":35321,"Ġcriminality":35322,"ĠZi":35323,"384":35324,"ĠWilderness":35325,"ĠCanary":35326,"ĠVs":35327,"и":35328,"ĠAPIs":35329,"Ġbehest":35330,"Ġeb":35331,"ĠHipp":35332,"Ġpreempt":35333,"Ġevoke":35334,"Ġinept":35335,"tele":35336,"447":35337,"ĠGarmin":35338,"Ġpursuits":35339,"351":35340,"Ġcliché":35341,"ĠJihad":35342,"Ġ308":35343,"ĠSnake":35344,"ĠAnnounce":35345,"Nearly":35346,"!'\"":35347,"Ġ1927":35348,"saw":35349,"Ġabhor":35350,"Plan":35351,"rawled":35352,"ĠRiy":35353,"ensor":35354,"Fal":35355,"quick":35356,"odynamic":35357,"Ġsubstitution":35358,"Ġprovoking":35359,"Operation":35360,"rupulous":35361,"Ġsweetness":35362,"folk":35363,"ĠDefault":35364,"Ġstarved":35365,"ĠPrinting":35366,"urious":35367,"ĠTracker":35368,"them":35369,"Ġleth":35370,"Ġemptied":35371,"Ġfootprints":35372,"ilian":35373,"Ġbattalion":35374,"Ġprophet":35375,"Ġrailing":35376,"Ġhect":35377,"rouch":35378,"lees":35379,"Ġideologies":35380,"Ġ254":35381,"ĠGods":35382,"ĠAvalon":35383,"Ġfrontrunner":35384,"ĠPork":35385,"ĠPipe":35386,"Ġscaven":35387,"Ġming":35388,"Ġerg":35389,"Ġ520":35390,"Ġhatched":35391,"asant":35392,"ĠHI":35393,"Ġpend":35394,"Ġ288":35395,"Prom":35396,"achev":35397,"ĠEcology":35398,"enforcement":35399,"467":35400,"dule":35401,"Ġrealism":35402,"ĠTypes":35403,"USB":35404,"utra":35405,"ĠHiroshima":35406,"Ġcontradicted":35407,"393":35408,"ĠDSL":35409,"Ġtherein":35410,"ĠReconstruction":35411,"Ġ243":35412,"irled":35413,"479":35414,"ĠWhats":35415,"Currently":35416,"ĠPOWER":35417,"ĠHiro":35418,"ĠBreath":35419,"ĠYourself":35420,"Ġlantern":35421,"376":35422,"É":35423,"ĠHumans":35424,"Lady":35425,"Ġdissemination":35426,"ecake":35427,"ĠChao":35428,"flat":35429,"Ġinspecting":35430,"stration":35431,"Ġidentifiable":35432,"CV":35433,"ĠLobby":35434,"function":35435,"Roll":35436,"DIV":35437,"Tell":35438,"Ġfasc":35439,"ĠAOL":35440,"HM":35441,"Keefe":35442,"Ġporous":35443,"Ġsmoot":35444,"existence":35445,"ĠDeg":35446,"Ġdivor":35447,"isner":35448,"allas":35449,"Bloomberg":35450,"Ġdictators":35451,"ĠGeh":35452,"Ġsilicone":35453,"Ġdab":35454,"Ġmashed":35455,"Ġpric":35456,"might":35457,"ĠBLM":35458,"Ġpatriarch":35459,"Microsoft":35460,"ĠAds":35461,"Ġcoronary":35462,"ĠContrary":35463,"Ġdra":35464,"ĠStarted":35465,"Ġbuckle":35466,"lear":35467,"accept":35468,"Within":35469,"bd":35470,"interested":35471,"bia":35472,"POR":35473,"motion":35474,"ĠFounders":35475,"ĠCassandra":35476,"ĠPassion":35477,"Ġbehavioural":35478,"ĠHealing":35479,"Ġmarkings":35480,"Ġsnowball":35481,"Ġridiculed":35482,"phase":35483,"Ġunto":35484,"aque":35485,"uggets":35486,"Ġfrantically":35487,"Ġcoward":35488,"Ġinconvenient":35489,"Taking":35490,"Afee":35491,"Ġtwisting":35492,"930":35493,"ĠSieg":35494,"ĠGit":35495,"Ġcurs":35496,"ĠGlas":35497,"ĠSignificant":35498,"Ġachieves":35499,"Ġpreferably":35500,"Ġcondensed":35501,"Ġfetus":35502,"Ġunivers":35503,"Ġpse":35504,"Access":35505,"Ġintertwined":35506,"been":35507,"quit":35508,"ĠLEGO":35509,"Ġimagining":35510,"454":35511,"Ġplains":35512,"sequently":35513,"pull":35514,"Fast":35515,"Pot":35516,"yles":35517,"AIR":35518,"Ġblatantly":35519,"eki":35520,"ilated":35521,"ĠMembership":35522,"Ġ262":35523,"Ġ}":35524,"Ġexcavation":35525,"Ġethn":35526,"addin":35527,"Ġfoundational":35528,"ceptions":35529,"ĠViet":35530,"exempt":35531,"Ġmicrophones":35532,"Ġ244":35533,"778":35534,"Ġdwar":35535,"attery":35536,"502":35537,"ĠKik":35538,"Ġinspir":35539,"ĠMaximum":35540,"Ġvengeance":35541,"Ġetched":35542,"outine":35543,"552":35544,"Ġunicorn":35545,"gged":35546,".�":35547,"ĠBlackwell":35548,"ĠStatue":35549,"Ġdissidents":35550,"ĠKaine":35551,"Ġdeforestation":35552,"ĠScholar":35553,"Ġpleasantly":35554,"ÑĤ":35555,"398":35556,"ĠRUN":35557,"arent":35558,"Ġundeniably":35559,"Ġtechnologically":35560,"Ġconsciously":35561,"ĠEther":35562,"Ġproportional":35563,"Ġlaund":35564,"ĠRye":35565,"Ġambiguity":35566,"Ġunmist":35567,"Terror":35568,"ciplinary":35569,"ĠImproved":35570,"hesis":35571,"Ġcooker":35572,"elsen":35573,"Ġguerrilla":35574,"opped":35575,"ATURE":35576,"Ġrequ":35577,"Ġunprepared":35578,"Ġcamel":35579,"Ġfitt":35580,"Sex":35581,"edged":35582,"Ġrecurrent":35583,"ctuary":35584,"ĠCompare":35585,"ĠServing":35586,"Tri":35587,"Ġtransient":35588,"ĠBees":35589,"Ġcovenant":35590,"Ġfantasies":35591,"Ġespresso":35592,"draft":35593,"baugh":35594,"Ġdemocratically":35595,"ĠBans":35596,"ĠManual":35597,"ĠTurtle":35598,"ennett":35599,"achy":35600,"ĠClim":35601,"Ġdescending":35602,"Ġprow":35603,"Ġinconsistencies":35604,"Player":35605,"Ġoblivious":35606,"ĠWonderland":35607,"nav":35608,"aughter":35609,"Ġlod":35610,"Ġ403":35611,"ĠPolaris":35612,"ĠLeia":35613,"ĠInfantry":35614,"Sy":35615,"ĠMeter":35616,"Ġautoimmune":35617,"Ġdiagnoses":35618,"Ġtrespass":35619,"011":35620,"wrong":35621,"ĠGREAT":35622,"Ġtelescopes":35623,"shows":35624,"Pac":35625,"olation":35626,"Ġclerics":35627,"Ġdissenting":35628,"406":35629,"Ġetiquette":35630,"Ġdeterrence":35631,"765":35632,"Ġove":35633,"Has":35634,"Pak":35635,"ा":35636,"ĠNec":35637,"Ġsociology":35638,"witz":35639,"Ġkittens":35640,"Ġcontinual":35641,"Ġoverlapping":35642,"Ġmonks":35643,"ĠMechanical":35644,"Captain":35645,"ocial":35646,"ĠFalling":35647,"ĠCorrection":35648,"ĠTrouble":35649,"Ġslog":35650,"Ġ253":35651,"Ġemanating":35652,"Ġwidest":35653,"PROV":35654,"Japanese":35655,"urat":35656,"Ġboxed":35657,"ĠCases":35658,"Ġjarring":35659,"Fix":35660,"'?":35661,"ĠStrateg":35662,"Republic":35663,"ovy":35664,"362":35665,"ĠMothers":35666,"Ġstreaks":35667,"Ġlocalized":35668,"ĠONLY":35669,"Ġeh":35670,"ĠObject":35671,"Ġstub":35672,"Fre":35673,"ĠScarlet":35674,"Ġmultip":35675,"ĠMaul":35676,"ĠProblems":35677,"cest":35678,"Ġmortal":35679,"Ġarche":35680,"ulet":35681,"Ġfuller":35682,"ĠGER":35683,"Si":35684,"mr":35685,"ĠPowerful":35686,"boxing":35687,"ĠPeer":35688,"Jean":35689,"ĠTF":35690,"Ġplural":35691,"optim":35692,"Jimmy":35693,"ĠFriendly":35694,"Mex":35695,"Ġdepri":35696,"PK":35697,"Ġwaitress":35698,"eph":35699,"arrass":35700,"ikawa":35701,"feel":35702,"Finally":35703,"fourth":35704,"394":35705,"conom":35706,"VT":35707,"Ġeleg":35708,"ivot":35709,"Ġharsher":35710,"ĠPepe":35711,"ĠImpl":35712,"Ġankles":35713,"idity":35714,"ĠPrepare":35715,"Rather":35716,"Ġconservatism":35717,"Ġunquestion":35718,"ribution":35719,"ĠPatent":35720,"ĠDeluxe":35721,"ĠAE":35722,"007":35723,"Ġprag":35724,"bg":35725,"Ġpalate":35726,"Ġintric":35727,"ossom":35728,"Ġspac":35729,"ĠSpotlight":35730,"Seven":35731,"amacare":35732,"ĠGotham":35733,"Ġencompass":35734,"Ġnicer":35735,"ĠLauder":35736,"Ġscaff":35737,"worn":35738,"442":35739,"Ġpropri":35740,"443":35741,"ĠCompos":35742,"ĠIniti":35743,"inth":35744,"Ġrehe":35745,"Prov":35746,"Ġgri":35747,"ossip":35748,"ĠModest":35749,"quiet":35750,"Ġwealthier":35751,"Ġ241":35752,"icum":35753,"Ġcommunism":35754,"Ġhelpers":35755,"Ġbellig":35756,"Ġ405":35757,"uttered":35758,"Ġbitterness":35759,"nl":35760,"474":35761,"Ġvitality":35762,"blank":35763,"ĠLeth":35764,"PAC":35765,"326":35766,"ĠNapoleon":35767,"Ġ299":35768,"ĠReviews":35769,"ĠSect":35770,"Ġstrongh":35771,"ĠTube":35772,"Ġwoodland":35773,"Ġhumming":35774,"411":35775,"Alpha":35776,"Ġundet":35777,"Ġmounts":35778,"Officials":35779,"igning":35780,"830":35781,"ĠStamp":35782,"ubby":35783,"424":35784,"Ġoutlandish":35785,"Ġjerk":35786,"Ġradiant":35787,"Ġcubes":35788,"Director":35789,"Ġatro":35790,"vous":35791,"Sab":35792,"Ġpretended":35793,"Ġ620":35794,"975":35795,"Sham":35796,"Ġpotassium":35797,"ĠAttention":35798,"gly":35799,"opens":35800,"ĠWorker":35801,"porter":35802,"Ġsplendid":35803,"embed":35804,"Je":35805,"ĠMeal":35806,"Ġsurname":35807,"Usually":35808,"Ġtimer":35809,"Ġweave":35810,"irin":35811,"ĠGenetics":35812,"ensual":35813,"Ġmerry":35814,"Ġapprehend":35815,"utsche":35816,"strate":35817,"Ġsupplementary":35818,"ĠRoundup":35819,"upid":35820,"Ġmiraculous":35821,"ĠHUN":35822,"Ġglaciers":35823,"weed":35824,"ĠSuggest":35825,"XL":35826,"authors":35827,"Ġbarking":35828,"ĠUKIP":35829,"leased":35830,"ĠRAD":35831,"Ġfide":35832,"Ġphen":35833,"Ġscanners":35834,"Parents":35835,"ĠBlaze":35836,"Ġtweaking":35837,"Ġelaborated":35838,"Ġsusp":35839,"iscovered":35840,"Ġthighs":35841,"Ġradicals":35842,"ULTS":35843,"aggressive":35844,"endants":35845,"Hon":35846,"Ġcorrecting":35847,"391":35848,"pps":35849,"ĠTerritories":35850,"Ġconferred":35851,"crazy":35852,"utor":35853,"ĠSurvival":35854,"Ġbrowsers":35855,"ĠConflict":35856,"pn":35857,"Ġdeprive":35858,"riage":35859,"ilan":35860,"à¦":35861,"949":35862,"Congratulations":35863,"radical":35864,"ĠHits":35865,"powerful":35866,"Ġcrypt":35867,"745":35868,"ĠRegistrar":35869,"ophile":35870,"ĠElement":35871,"cooked":35872,"ĠTwilight":35873,"Ġdemos":35874,"IER":35875,"Ġstricken":35876,"Magic":35877,"abby":35878,"ĠSack":35879,"ĠShrine":35880,"Nev":35881,"Probably":35882,"ĠWisdom":35883,"ulpt":35884,"opher":35885,"Ġcolonel":35886,"atl":35887,"Tem":35888,"kun":35889,"ĠIndie":35890,"Putin":35891,"jection":35892,"areth":35893,"ĠBullet":35894,"Ġsmartest":35895,"ĠEsper":35896,"Ġproficiency":35897,"Ġcessation":35898,"Ġmars":35899,"ĠDATA":35900,"sup":35901,"Ġostr":35902,"Jane":35903,"Ġpathogens":35904,"hd":35905,"ĠNK":35906,"Ġhorribly":35907,"regulated":35908,"Ġesteemed":35909,"ĠChinatown":35910,"Ġvibration":35911,"Ġoverboard":35912,"ĠRhod":35913,"Ġfeces":35914,"otation":35915,"Ġcryptic":35916,"Bal":35917,"OPER":35918,"Ġaffirmation":35919,"Ġmenstrual":35920,"Ġuntold":35921,"Ġanecdotes":35922,"ĠHOUSE":35923,"Ġcape":35924,"311":35925,"ittance":35926,"ĠRemy":35927,"ĠWaves":35928,"ĠCOVER":35929,"ordinate":35930,"Ġrestricts":35931,"Samsung":35932,"Ġplantations":35933,"olver":35934,"Better":35935,"ĠExplos":35936,"Ġnasal":35937,"ĠSyri":35938,"ĠPerl":35939,"Ġlatency":35940,"othermal":35941,"Sweet":35942,"ĠRyzen":35943,"ĠYuri":35944,"Ġsmack":35945,"Ġcrow":35946,"aniel":35947,"iological":35948,"Ġmonk":35949,"Ġtutorial":35950,"ĠAure":35951,"Ġcliffs":35952,"ameron":35953,"umers":35954,"ĠMour":35955,"Ġunorthodox":35956,"Ġgulf":35957,"Ġintrusive":35958,"ĠVIII":35959,"ĠFF":35960,"Ġenlarged":35961,"Ġspheres":35962,"ĠCheap":35963,"ĠAmend":35964,"Ġ::":35965,"Ġpacing":35966,"ĠStartup":35967,"ĠDating":35968,"racist":35969,"ĠDivine":35970,"Ġpollen":35971,"ĠMeaning":35972,"ĠLei":35973,"ĠMOT":35974,"ĠARC":35975,"legate":35976,"Ġbrav":35977,"Ross":35978,"redit":35979,"414":35980,"ringe":35981,"perhaps":35982,"SPA":35983,"Southern":35984,"Front":35985,"undrum":35986,"Ġassorted":35987,"ĠDawkins":35988,"ĠWrap":35989,"Ġconsequential":35990,"ĠFuji":35991,"458":35992,"Ġunst":35993,"Bon":35994,"acter":35995,"Trade":35996,"ingers":35997,"ĠClin":35998,"Ġstimul":35999,"arah":36000,"inois":36001,"urdy":36002,"Ġobsessive":36003,"Zone":36004,"Ġprimitive":36005,"unctions":36006,"Ġadapter":36007,"Ġassures":36008,"Daddy":36009,"Ġunsatisf":36010,"441":36011,"Ġ1910":36012,"Ġsecondly":36013,"truth":36014,"RED":36015,"040":36016,"Pope":36017,"venants":36018,"Ġestim":36019,"Ġhemorrh":36020,"Ġexcruciating":36021,"459":36022,"Ġboils":36023,"ieved":36024,"Storm":36025,"Ġmanifestation":36026,"Ġinsulated":36027,"fb":36028,"Ġclassify":36029,"Mbps":36030,"Ġinclination":36031,"Ġaur":36032,"Ġpolarized":36033,"Ġoccupations":36034,"Secretary":36035,"Ġcustomizable":36036,"scribe":36037,"Ġadjunct":36038,"Ġ1922":36039,"rived":36040,"ocative":36041,"Friends":36042,"Oak":36043,"Ġpsyche":36044,"Ġwrinkles":36045,"anthrop":36046,"Ġcoercion":36047,"enos":36048,"Ġvariability":36049,"hma":36050,"phot":36051,"ĠXander":36052,"ĠDiss":36053,"Ġtigers":36054,"ahoo":36055,"focus":36056,"rical":36057,"grow":36058,"Ġseminal":36059,"Ġdisciples":36060,"Cas":36061,"Hundreds":36062,"Ġscissors":36063,"correct":36064,"Ġfascism":36065,"imoto":36066,"Ġnudity":36067,"charg":36068,"Ġrusty":36069,"ĠLyndon":36070,"Ġanomalies":36071,"onial":36072,"ĠiCloud":36073,"Ġannoy":36074,"Ġdistortion":36075,"Lou":36076,"ĠGiul":36077,"eyes":36078,"870":36079,"uum":36080,"ĠUltr":36081,"Action":36082,"cigarette":36083,"igators":36084,"kj":36085,"Ġ323":36086,"uine":36087,"Score":36088,"Ġmans":36089,"Security":36090,"Ġarom":36091,"ĠBoards":36092,"Ġwrists":36093,"602":36094,"Ġastronomy":36095,"Ġresin":36096,"width":36097,")/":36098,"Ġconcurrent":36099,"unless":36100,"606":36101,"ĠMagnet":36102,"Ġauthorizing":36103,"ĠJunk":36104,"atical":36105,"Ġauthent":36106,"zac":36107,"413":36108,"ĠGrape":36109,"Ġcircled":36110,"Ġooz":36111,"Ġvisceral":36112,"ointment":36113,"Ġincendiary":36114,"ĠBourbon":36115,"Ġgimmick":36116,"vette":36117,"Stan":36118,"Ġdetachment":36119,"488":36120,"Ġmisogyny":36121,"Ġenlight":36122,"utic":36123,"Ġinquire":36124,"ĠBEL":36125,"ascular":36126,"ĠWasserman":36127,"Dallas":36128,"Ġconstellation":36129,"Ġdystopian":36130,"504":36131,"ĠOptical":36132,"Ġsilhou":36133,"Girl":36134,"ĠGong":36135,"ĠHighest":36136,"????????":36137,"Sav":36138,"ocity":36139,"leted":36140,"Ġattrition":36141,"ĠExpedition":36142,"ĠKilled":36143,"501":36144,"ONES":36145,"dat":36146,"Ġglyphosate":36147,"Ġplugs":36148,"Ġlact":36149,"Fla":36150,"fps":36151,"riger":36152,"Ġparagraphs":36153,"Ġinnate":36154,"ĠFoo":36155,"aternity":36156,"ĠGry":36157,"Ġoneself":36158,"642":36159,"Iowa":36160,"oodle":36161,"ĠCoconut":36162,"ĠChess":36163,"ommel":36164,"Ġmagnesium":36165,"Ġairliner":36166,"Ġexceedingly":36167,"ĠCreator":36168,"YouTube":36169,"Ġsleeper":36170,"Ġlonging":36171,"ĠPercy":36172,"Ġmatrix":36173,"Ġâľ":36174,"Ġbarren":36175,"Mrs":36176,"Ġinvading":36177,"Ġincom":36178,"Ġemperor":36179,"Ġip":36180,"irie":36181,"Ġpredictably":36182,"ĠBless":36183,"Ġsuperpower":36184,":-":36185,"Ġpropensity":36186,"easy":36187,"educ":36188,"ĠPolly":36189,"Ġcumbersome":36190,"Ġcollide":36191,"016":36192,"Ġtransports":36193,"Ġscraps":36194,"below":36195,"Ġhairs":36196,"mentation":36197,"Ġevolves":36198,"ĠFallen":36199,"Ġunsurprisingly":36200,"Ġcuff":36201,"Ġ249":36202,"mental":36203,"ĠCamel":36204,"Ġ337":36205,"Clinton":36206,"Ġdecad":36207,"ĠSTEP":36208,"ĠTestament":36209,"Ġirresistible":36210,"ĠACE":36211,"Ġhamm":36212,"ĠTerr":36213,"Ġcaul":36214,"iggins":36215,"Ġproficient":36216,"resp":36217,"Ġheirs":36218,"Ġ321":36219,"dress":36220,"ĠClothing":36221,"Ġ560":36222,"Ġ264":36223,"ĠRobb":36224,"Ġfrail":36225,"Ġoptimizing":36226,"615":36227,"ĠRefuge":36228,"rowth":36229,"washing":36230,"Ġgenders":36231,"indu":36232,"ĠNAT":36233,"Ġleans":36234,"Ġeyed":36235,"Ġhilar":36236,"vice":36237,"wolf":36238,"Ġfatig":36239,"ococ":36240,"ĠCarry":36241,"Community":36242,"Clark":36243,"itably":36244,"sv":36245,"448":36246,"Ġnumer":36247,"Ġ1925":36248,"ĠBehavioral":36249,"ĠScream":36250,"Ġgeek":36251,"rake":36252,"ĠTTC":36253,"Ġadditives":36254,"ĠBye":36255,"ylon":36256,"Ġfoliage":36257,"ateral":36258,"rapnel":36259,"Science":36260,"Ġrecollection":36261,"thening":36262,"ĠUbisoft":36263,"ĠLur":36264,"ĠOkinawa":36265,"ĠProvision":36266,"ferred":36267,"ĠGrounds":36268,"Ġhops":36269,"aterial":36270,"Ġacad":36271,"Ġengulf":36272,"ĠApex":36273,"frequency":36274,"relations":36275,"ĠCorvette":36276,"ĠRepeat":36277,"Ġanew":36278,"Ġhes":36279,"ĠLair":36280,"ĠPSP":36281,"foundation":36282,"Band":36283,"ĠPublisher":36284,"Ġreciprocal":36285,"Ġ287":36286,"Ġpir":36287,"Adams":36288,"Ġprostitute":36289,"ĠMecca":36290,"ectomy":36291,"Ġskew":36292,"ĠLol":36293,"Voice":36294,"ĠCalais":36295,"ISION":36296,"rue":36297,"Ġgaping":36298,"prot":36299,"Ġ6000":36300,"Ġtilted":36301,"Ġgoofy":36302,"Stand":36303,"Ġfellows":36304,"Ġcurly":36305,"ĠPOW":36306,"Ġlore":36307,"Ġinhabited":36308,"ĠIdentification":36309,"Metro":36310,"Ġdispel":36311,"Ġinvoking":36312,"Ġdeleting":36313,"Ġstigmat":36314,"ĠDalai":36315,"Ġequate":36316,"Ġmascara":36317,"endered":36318,"ĠNYT":36319,"ĠCommittees":36320,"rians":36321,"ĠOlympus":36322,"ĠQR":36323,"ĠDrinking":36324,"Ġbatt":36325,"andr":36326,"computer":36327,"Senator":36328,"ĠTwist":36329,"ĠNoise":36330,"Ġcheesy":36331,"Ġ1931":36332,"Ġtyranny":36333,"Ġnegligible":36334,"ĠBok":36335,"Ġwebpage":36336,"ĠHEAD":36337,"ĠNovel":36338,"Ġquarry":36339,"Ġexpressive":36340,"Ġforgiving":36341,"Among":36342,"asin":36343,"ĠSuc":36344,"Democrats":36345,"795":36346,"Ġaback":36347,"¨":36348,"ĠNeon":36349,"392":36350,"ĠRNC":36351,"ĠPROC":36352,"sein":36353,"Ros":36354,"Ġemot":36355,"ĠASA":36356,"ĠSeb":36357,"ĠExtended":36358,"atern":36359,"Ġpsychedelic":36360,"Fil":36361,"ĠOrwell":36362,"ĠSOS":36363,"Ġconceive":36364,"Ġhobbies":36365,"Ġspecimens":36366,"ĠTEXT":36367,"sometimes":36368,"Mario":36369,"orpor":36370,"ĠTemporary":36371,"Ġapocalypse":36372,"Ġcounterproductive":36373,"ĠQUEST":36374,"ĠCargo":36375,"Amb":36376,"Ġoptic":36377,"groups":36378,"Ġparanoia":36379,".?":36380,"sounding":36381,"mediately":36382,"System":36383,"ubi":36384,"Ġuttered":36385,"Ġgraphs":36386,"âĢĭâĢĭ":36387,"Ġscientifically":36388,"Ġbluntly":36389,"Ġhopping":36390,"Fun":36391,"ĠSUPER":36392,"Ġrobe":36393,"VB":36394,"ĠQuote":36395,"Ġincarnation":36396,"Ġtreadmill":36397,"Ġ1915":36398,"Ġbart":36399,"669":36400,"Ġhoc":36401,"Ġ309":36402,"Ġimprovis":36403,"Ġhut":36404,"Ġmixer":36405,"ĠCt":36406,"span":36407,"Ġwatered":36408,"Ġpatriot":36409,"Ġdehyd":36410,"laughs":36411,"ĠFancy":36412,"ĠVoc":36413,"Ġintellect":36414,"ĠTid":36415,"Ġnesting":36416,"Tel":36417,"Ġ()":36418,"letter":36419,"ĠSeems":36420,"Ops":36421,"ĠContents":36422,"ript":36423,"hani":36424,"Ġrecru":36425,"Ġpickups":36426,"repair":36427,"Throughout":36428,"bear":36429,"Ġconquered":36430,"656":36431,"Ġmalf":36432,"Ġordained":36433,"755":36434,"ĠReprodu":36435,"brain":36436,"ĠOuts":36437,"ĠWage":36438,"Ru":36439,"________":36440,"ĠLAW":36441,"ĠWass":36442,"Ġcomplication":36443,"Fri":36444,"Ġregener":36445,"Wait":36446,"577":36447,"Ġmisconception":36448,"Ġbombardment":36449,"Ġunloaded":36450,"Ġdictionary":36451,"IU":36452,"025":36453,"etically":36454,"ĠNarr":36455,"repe":36456,"Ġassigning":36457,"Rail":36458,"Ġnotebooks":36459,"Ġingest":36460,"Ġrpm":36461,"Ġalienated":36462,"ĠCredits":36463,"Ġindis":36464,"ĠGathering":36465,"aration":36466,"-+-+-+-+":36467,"Ġori":36468,"Ġsr":36469,"ndra":36470,"Ġlibertarian":36471,"Ġcoerced":36472,"ording":36473,"Ġtranqu":36474,"Ġelbows":36475,"549":36476,"Ġping":36477,"ĠRELE":36478,"ĠYanuk":36479,"Ġmaneuvers":36480,"ĠTrojan":36481,"IFIED":36482,"ĠViolent":36483,"è":36484,"Ġlest":36485,"Ġarrows":36486,"frog":36487,"anty":36488,"WB":36489,"ĠSeen":36490,"648":36491,"Ġclutter":36492,"ĠBender":36493,"Ġpessim":36494,"ĠTeg":36495,"Asian":36496,"IFIC":36497,"Ġexponential":36498,"Ġsponge":36499,"rite":36500,"ĠDAM":36501,"Ġtacit":36502,"ĠZoom":36503,"Ġolds":36504,"Ġonward":36505,"ĠSandwich":36506,"missible":36507,"isol":36508,"940":36509,"Ġinciner":36510,"ĠTrick":36511,"Ġawakening":36512,"Ġdart":36513,"ĠCouch":36514,"respons":36515,"ĠElephant":36516,"ĠPluto":36517,"ĠTags":36518,"itcher":36519,"644":36520,"702":36521,"Ġelectrons":36522,"ĠMyth":36523,"ĠAad":36524,"Danny":36525,"Ġcraw":36526,"ĠCertification":36527,"Ġtending":36528,"Ġpellets":36529,"Ġamused":36530,"ĠAuschwitz":36531,"ĠAppl":36532,"iris":36533,"ashion":36534,"walking":36535,"Ġabnorm":36536,"Cro":36537,"?:":36538,"ĠIcelandic":36539,"ĠAvailability":36540,"Ġcann":36541,"Opt":36542,"buster":36543,"ĠQuartz":36544,"Executive":36545,"tracks":36546,"igel":36547,"MIT":36548,"ĠTracking":36549,"Ġconditioned":36550,"Ġsampled":36551,"ĠGenius":36552,"Ġsubstit":36553,"ĠSiberia":36554,"Ġfrequ":36555,"historic":36556,"okin":36557,"OWS":36558,"1500":36559,"warts":36560,"ĠEtsy":36561,"licks":36562,"ĠSmooth":36563,"unity":36564,"515":36565,"Ġperk":36566,"aida":36567,"forts":36568,"ĠUA":36569,"RIC":36570,"Spain":36571,"ĠWired":36572,"cuts":36573,"Ġfurnace":36574,"ĠTOTAL":36575,"ĠTables":36576,"662":36577,"Fab":36578,"Ġquaint":36579,"ĠWorlds":36580,"ĠCabin":36581,"atche":36582,"List":36583,"ĠVO":36584,"Ġkeyword":36585,"Ġ258":36586,"Farm":36587,"timer":36588,"ĠVolt":36589,"Build":36590,"pressed":36591,"*,":36592,"Ġ324":36593,"aiman":36594,"TING":36595,"Ġsneaking":36596,"cery":36597,"Ġcrib":36598,"ĠIllust":36599,"later":36600,"Ġcompar":36601,"Ġpropulsion":36602,"647":36603,"ĠTrails":36604,"Ġperiphery":36605,"steel":36606,"Ġvividly":36607,"ĠConver":36608,"eatured":36609,"427":36610,"463":36611,"Ġapprox":36612,"spin":36613,"Ġconfigured":36614,"inside":36615,"razy":36616,"account":36617,"anye":36618,"riend":36619,"Ġbows":36620,"809":36621,"ĠDEF":36622,"ĠRez":36623,"Fans":36624,"ĠDF":36625,"Ġstains":36626,"ĠAtom":36627,"ĠConce":36628,"ĠTOM":36629,"ĠELECT":36630,"Ġdisappro":36631,"019":36632,"afia":36633,"ĠTemperature":36634,"Ġextracts":36635,"fab":36636,"Ġunsur":36637,"Ġseasoning":36638,"Ty":36639,"KB":36640,"Ġposit":36641,"Ġlocality":36642,"1200":36643,"cour":36644,"izons":36645,"hh":36646,"506":36647,"ĠDLC":36648,"iago":36649,"Ġcorpses":36650,"iddling":36651,"Mayor":36652,"Ġsimplistic":36653,"Ġlibel":36654,"Ġalmonds":36655,"Ġswast":36656,"Change":36657,"ĠJoker":36658,"MAR":36659,"ĠScully":36660,"Ġmailbox":36661,"VIDEO":36662,"ĠKyoto":36663,"esley":36664,"ĠIncredible":36665,"youtube":36666,"Ġinequalities":36667,"Ġbolts":36668,"Ġbothering":36669,"Ġattentive":36670,"ĠSparrow":36671,"Ġdiaper":36672,"Ġfanbase":36673,"Ġuncont":36674,"Ap":36675,"ĠQi":36676,"Price":36677,"471":36678,"Ġpearl":36679,"wid":36680,"899":36681,"ĠPony":36682,"casting":36683,"Ġinhabit":36684,"Ġunve":36685,"Ġinsur":36686,"ĠWee":36687,"658":36688,"Ġeffected":36689,"gger":36690,"Ġinstallments":36691,"imilar":36692,"FU":36693,"Ġinfertility":36694,"climate":36695,"HEAD":36696,"fashion":36697,"ĠTHEY":36698,"jc":36699,"Ġsatisf":36700,"ĠGuidelines":36701,"Ġinsure":36702,"ĠRSA":36703,"Ġvirt":36704,"Ġinterpre":36705,"Joshua":36706,"ĠShut":36707,"Ġtestimonies":36708,"Ñģ":36709,"untary":36710,"417":36711,"Ġbeck":36712,"ĠMilky":36713,"ç":36714,"Ġsequels":36715,"Ġ281":36716,"ĠRibbon":36717,"Ġroomm":36718,"Ġsynchron":36719,"452":36720,"Ġ1926":36721,"Ġhawk":36722,"ĠDisorder":36723,"Ġbackstory":36724,"ĠNum":36725,"Ġoverheard":36726,"technical":36727,"Jud":36728,"aii":36729,"Ġdecon":36730,"ĠRape":36731,"ĠWarrant":36732,"Ġpoop":36733,"spir":36734,"Country":36735,"Ġweld":36736,"Ġabuser":36737,"Ġ------":36738,"material":36739,"Ġpreserves":36740,"spring":36741,"Ġpuzzled":36742,"ĠDebate":36743,"Joseph":36744,"Ġ272":36745,"Blood":36746,"antry":36747,"Ġconverge":36748,"Ġimaginable":36749,"oward":36750,"545":36751,"Ġfug":36752,"Vision":36753,"075":36754,"Ġadoptive":36755,"Ġunknow":36756,"Stream":36757,"Ġaffili":36758,"ĠPUR":36759,"ĠWally":36760,"Ġgamer":36761,"Ġfart":36762,"stice":36763,"Ġcongen":36764,"н":36765,"685":36766,"orst":36767,"ĠATF":36768,"Ġml":36769,"ĠMozilla":36770,"Ġcalmed":36771,"bage":36772,"ĠVault":36773,"arkable":36774,"ĠGuan":36775,"Ġclueless":36776,"umatic":36777,"Ġshameless":36778,"Ġpreached":36779,"Ġmisconceptions":36780,"Ġanthology":36781,"Ġbiomass":36782,"ĠPs":36783,"tails":36784,"Ġexcessively":36785,"Ġextr":36786,"Davis":36787,"Ġgrounding":36788,"Ġshortcuts":36789,"ĠShift":36790,"ĠRew":36791,"ĠIllum":36792,"Ġincite":36793,"sense":36794,"ĠScouting":36795,"otos":36796,"respond":36797,"Ġbeware":36798,"gran":36799,"ĠXV":36800,"JM":36801,"ĠSounders":36802,"Ġ276":36803,"Ġshockingly":36804,"Ġgastrointestinal":36805,"erences":36806,"df":36807,"ĠNG":36808,"Ġdiscredited":36809,"Ġdemoral":36810,"Ġgladly":36811,"Tal":36812,"ĠPredator":36813,"708":36814,"Ġdoi":36815,"Ġdecentral":36816,"illin":36817,"printed":36818,"Ġinflicting":36819,"ribes":36820,"Ġsupper":36821,"abc":36822,"Ġgraz":36823,"980":36824,"Bull":36825,"Ġmillionaires":36826,"Ġvanity":36827,"imony":36828,"Ġbiologists":36829,"Ġalternating":36830,"Ġsleeps":36831,"Force":36832,"ĠPrinc":36833,"ĠTransgender":36834,"Ġ314":36835,"ĠProvide":36836,"enthal":36837,"Ġplum":36838,"Ġresurrect":36839,"CW":36840,"Ġinjure":36841,"ĠPerspective":36842,"ĠBei":36843,"Ġrestless":36844,"aciously":36845,"Ġchlor":36846,"catch":36847,"ĠLuigi":36848,"Ġinconsistency":36849,"Ġwhiff":36850,"Arizona":36851,"ustration":36852,"ĠRaid":36853,"ĠDemons":36854,"ĠVita":36855,":\"":36856,"Ġmigraine":36857,"ĠHamb":36858,"Ġwidget":36859,"451":36860,"Ġrandomized":36861,"etchup":36862,"ĠParticularly":36863,"Ġdiced":36864,"Ġperfected":36865,"roid":36866,"710":36867,"Ġreflections":36868,"Ġantioxidants":36869,"ĠLabel":36870,"Ġ326":36871,"igious":36872,"ĠEucl":36873,"608":36874,"Ġstrand":36875,"ĠDirt":36876,"ĠLift":36877,"suits":36878,"ĠControls":36879,"RAW":36880,"Ġcowardly":36881,"ĠUmb":36882,"Growing":36883,"mington":36884,"Ġ339":36885,"ĠCommit":36886,"Ġnonviolent":36887,"Ġcontaminants":36888,"Ġacrylic":36889,"ĠMAP":36890,"Ġ269":36891,"Ġdegrading":36892,"Ġmiracles":36893,"ĠEstablishment":36894,"despite":36895,"cry":36896,"Ġpauses":36897,"Ġmythical":36898,"Ġtwenties":36899,"Actually":36900,"phan":36901,"recorded":36902,"Ġunwillingness":36903,"engineering":36904,"avored":36905,"Ġdevout":36906,"item":36907,"Ġbunny":36908,"ĠMerchants":36909,"Ġconsumes":36910,"508":36911,"Ġlex":36912,"ĠClause":36913,"Ġchecklist":36914,"Sus":36915,"uther":36916,".#":36917,"Bit":36918,"uay":36919,"bf":36920,"Ġpopulace":36921,"Ġ316":36922,"Ġcombust":36923,"Ġnano":36924,"Ġpopul":36925,"Indust":36926,"Ġcapitalists":36927,"ĠFiles":36928,"Bang":36929,"Ġkosher":36930,"atile":36931,"Ġincrim":36932,"OVER":36933,"Ġmelee":36934,"ymph":36935,"ĠPupp":36936,"evin":36937,"ĠMolecular":36938,"Ġmisinterpret":36939,"vc":36940,"olithic":36941,"ĠSimpsons":36942,"Ġshrew":36943,"Ġselectively":36944,"ĠDrain":36945,"mittedly":36946,"conservative":36947,"True":36948,"Using":36949,"562":36950,"apon":36951,"Ġapprentice":36952,"Mas":36953,"ĠBattlefield":36954,"Ġfing":36955,"Ġconcoct":36956,"ĠVIS":36957,"ĠHuss":36958,"Ġdetects":36959,"ĠFriedrich":36960,"Ġlatitude":36961,"Custom":36962,"ĠÙ":36963,"ĠBones":36964,"whose":36965,"Ġredirected":36966,"aligned":36967,"ĠNeighbor":36968,"ĠAmen":36969,"ĠMarble":36970,"Beyond":36971,"Ġbiomark":36972,"Ġerroneous":36973,"Atlanta":36974,"Ġmasturb":36975,"ĠAssoci":36976,"Albert":36977,"Ġcigar":36978,"ĠFraz":36979,"ethe":36980,"skinned":36981,"Ford":36982,"throp":36983,"Acc":36984,"Ġtricked":36985,"Ġoverwhelm":36986,"Ġimplements":36987,"ĠGeForce":36988,"Ġbounces":36989,"Ġmoderator":36990,"910":36991,"ĠButterfly":36992,"ĠIllegal":36993,"ĠSubject":36994,"RET":36995,"ĠFreeze":36996,"ĠNewt":36997,"Ġuterus":36998,"696":36999,"Ġ267":37000,"tk":37001,"Ġdodged":37002,"liam":37003,"Ġparasite":37004,"obal":37005,"ĠHubble":37006,"Ġtheology":37007,"âĢĶ\"":37008,"height":37009,"Ale":37010,"employment":37011,"ĠWallet":37012,"cessive":37013,"Ġ404":37014,"Ġsimilarity":37015,"zens":37016,"Ġdumps":37017,"Ġdepress":37018,"Ġlifeless":37019,"535":37020,"oard":37021,"Scotland":37022,"Ġbelievable":37023,"Ġcalculator":37024,"ĠNaked":37025,"Ġremission":37026,"Ġoranges":37027,"ĠSections":37028,"Ġentangled":37029,"Ġuncanny":37030,"Ġteaspoons":37031,"vr":37032,"ĠPorn":37033,"Organ":37034,"Ġbund":37035,"Doug":37036,"ĠGHz":37037,"Major":37038,"abus":37039,"Bell":37040,"avier":37041,"Ġimplanted":37042,"RON":37043,"Fle":37044,"462":37045,"509":37046,"Ġgoggles":37047,"Ġmanuscript":37048,"NOT":37049,"ĠCanaveral":37050,"ĠDID":37051,"Season":37052,"HAEL":37053,"Edge":37054,"appiness":37055,"DIS":37056,"Ġplotted":37057,"Ġwrought":37058,"Ġquarantine":37059,"Ġrearr":37060,"itage":37061,"Ġsocket":37062,"Ġbrig":37063,"Ġunbelievably":37064,"abytes":37065,"TG":37066,"Ġ444":37067,"ĠOffic":37068,"Ġacquaintances":37069,"ĠComparison":37070,"Nine":37071,"ĠFeast":37072,"758":37073,"YC":37074,"Ġfiner":37075,"ĠStrawberry":37076,"Ġeternity":37077,"liament":37078,"urrency":37079,"ĠCortana":37080,"ĠSabbath":37081,"Ġsprinkle":37082,"unker":37083,"ĠUE":37084,"flies":37085,"Ġblender":37086,"Ġacutely":37087,"emark":37088,"ĠAffect":37089,"Politics":37090,"Ġsane":37091,"Ġcorrosion":37092,"Ġspirituality":37093,"Ġredeemed":37094,"Ġingrained":37095,"manager":37096,"joined":37097,"ĠDumb":37098,"ĠHeight":37099,"Ġseventeen":37100,"Ġ640":37101,"Ġreviewer":37102,"Ġwallpaper":37103,"Ġnurs":37104,"Ġsubset":37105,"703":37106,"Ġsymbolism":37107,"Ġdudes":37108,"Ġmismatch":37109,"gans":37110,"please":37111,"ĠKE":37112,"Ġatom":37113,"004":37114,"ionic":37115,"Ġservings":37116,"Ġproxies":37117,"Ġtranscription":37118,"yx":37119,"bowl":37120,"iscovery":37121,"ĠScotch":37122,"brace":37123,"riter":37124,"ĠDesktop":37125,"Ġlimestone":37126,"æ":37127,"Neg":37128,"013":37129,"Ġformulas":37130,"Ġeval":37131,"Ġzombies":37132,"GU":37133,"ĠHermes":37134,"Ġbrist":37135,"Mand":37136,"Ġmastery":37137,"Ġgoverns":37138,"Ġconstrued":37139,"region":37140,"Ġemitted":37141,"Vice":37142,"060":37143,"Jennifer":37144,"mol":37145,"Ġjealousy":37146,"Ġingenuity":37147,"bug":37148,"olitical":37149,"Ġperce":37150,"ĠSapp":37151,"dim":37152,"utral":37153,"Ġinterrogated":37154,"Gate":37155,"Ġamber":37156,"911":37157,"ĠEveryday":37158,"ĠDDR":37159,"ĠBlades":37160,"Ġnifty":37161,"Ġmurderers":37162,"Ġpresumption":37163,"Pitt":37164,"Div":37165,"ĠDestination":37166,"having":37167,"Ġprolifer":37168,"Ġbreaker":37169,"ĠBW":37170,"Ġcourier":37171,"Try":37172,"ĠBUR":37173,"itized":37174,"Ġcompress":37175,"Ġrepetition":37176,"ĠTik":37177,"Ġdivergence":37178,"Ġcube":37179,"everyone":37180,"ĠPoles":37181,"418":37182,"ĠHighly":37183,"468":37184,"Jeremy":37185,"Ġcontradictions":37186,"Ġmanure":37187,"Sad":37188,"pletion":37189,"626":37190,"Ġ279":37191,"Ġfrivolous":37192,"ĠCanaan":37193,"olor":37194,"Ġincapac":37195,"ĠGentle":37196,"Ġinsomnia":37197,"ĠJing":37198,"688":37199,"ĠViews":37200,"Ġsyll":37201,"486":37202,"antom":37203,"Ġcog":37204,"aintain":37205,"ĠDVDs":37206,"Ġ318":37207,"archy":37208,"Ġreprodu":37209,"Ġconcedes":37210,"Brook":37211,"Ġinterpreting":37212,"Ġextracting":37213,"Ġess":37214,"uning":37215,"ĠMathematics":37216,"iably":37217,"Ġmultit":37218,"ĠActs":37219,"iliated":37220,"Foreign":37221,"Ġflaming":37222,"ĠCoup":37223,"Ġglitches":37224,"Ġdifferentiation":37225,"ihadi":37226,"ĠDrone":37227,"Ġincompatible":37228,"asher":37229,"documented":37230,"agons":37231,"wark":37232,"Ġshielding":37233,"ĠCorrect":37234,"romising":37235,"uned":37236,"Ġconduit":37237,"ĠDiablo":37238,"Ġbeginner":37239,"Ġarchived":37240,"smanship":37241,"ĠTBD":37242,"digy":37243,"Ġ322":37244,"Ġ268":37245,"ĠTears":37246,"ĠPriority":37247,"Italy":37248,"Ġ^":37249,"annot":37250,"different":37251,"Joy":37252,"Ġbreathed":37253,"heon":37254,"Ġracists":37255,"Ġvascular":37256,"Between":37257,"etition":37258,"ĠLikely":37259,"icans":37260,"529":37261,"ĠMonsters":37262,"agy":37263,"Orange":37264,"hide":37265,"SIM":37266,"Ġdeceive":37267,"ĠDAR":37268,"Ġshattering":37269,"Ġow":37270,"peak":37271,"Ġpreferable":37272,"Ġpiping":37273,"ĠLEDs":37274,"ĠCOMMUN":37275,"ĠConstruct":37276,"008":37277,"Ġdissatisfied":37278,"ĠKNOW":37279,"ĠFrame":37280,"ĠToast":37281,"Ġadore":37282,"history":37283,"Soviet":37284,"reporting":37285,"Ġ266":37286,"pract":37287,"ĠSauce":37288,"686":37289,"ievers":37290,"ĠDomain":37291,"ousand":37292,"768":37293,"Cos":37294,"609":37295,"432":37296,"Ġtransl":37297,"oof":37298,"Ġ292":37299,"Turkish":37300,"ĠPOLIT":37301,"Harris":37302,"bj":37303,"Ġrodents":37304,"556":37305,"Ġintellectuals":37306,"Ġinteroper":37307,"ixt":37308,"Ġunbiased":37309,"itia":37310,"Ġ504":37311,"Ġbuttocks":37312,"ĠFlam":37313,"Ġchrom":37314,"Ġ259":37315,"shock":37316,"ĠRJ":37317,"ĠLich":37318,"422":37319,"Ġcondom":37320,"phen":37321,"Ġvigilante":37322,"Ġowl":37323,"Ġdwellings":37324,"Ġarchaeologists":37325,"Ġ680":37326,"RAY":37327,"Ġ1921":37328,"Ġ625":37329,"ĠPLAN":37330,"alde":37331,"030":37332,"abbling":37333,"Wave":37334,"Ni":37335,"Ġfurthe":37336,"JS":37337,"Ġpsycho":37338,"ĠFrançois":37339,"Ġundergrad":37340,"Ġsuccessors":37341,"Ġpadded":37342,"introdu":37343,"Ġreasoned":37344,"Ġvas":37345,"creen":37346,"onsequ":37347,"starter":37348,"Court":37349,"ĠHIS":37350,"Ġplaster":37351,"Ġranger":37352,"Ġ298":37353,"esters":37354,"Ġglare":37355,"ype":37356,"Ġcompute":37357,"Ali":37358,"mallow":37359,"Ġmasculine":37360,"ĠExamination":37361,"improve":37362,"Ġdeclass":37363,"Ġdecoration":37364,"ĠFIG":37365,"abre":37366,"Ġstale":37367,"abling":37368,"ĠRusty":37369,"ĠASAP":37370,"Ġadjusts":37371,"Ġbluff":37372,"density":37373,"Ġdisse":37374,"Ġcensor":37375,"ervatives":37376,"Ġkettle":37377,"Ġskeptics":37378,"fd":37379,"Imm":37380,"461":37381,"Ġadvantageous":37382,"419":37383,"ĠPresents":37384,"482":37385,"ĠRewards":37386,"Ġovershadow":37387,"Alabama":37388,"ĠCPC":37389,"Ġsock":37390,"ĠChurches":37391,"hidden":37392,"Ġcringe":37393,"ĠHOR":37394,"PB":37395,"Pretty":37396,"Hong":37397,"?),":37398,"687":37399,"Ġgrocer":37400,"472":37401,"565":37402,"itent":37403,"Ġpartake":37404,"wait":37405,"usters":37406,"Ġcones":37407,"Ġconcurrently":37408,"Ġlevers":37409,"Ġaroma":37410,"ĠDrill":37411,"498":37412,"804":37413,"ithering":37414,"Ġ355":37415,"Ġlegion":37416,"Ġvitri":37417,"Ġcondu":37418,"Angel":37419,"OWER":37420,"Ġ{*":37421,"Simon":37422,"Ġsynthesis":37423,"ĠContainer":37424,"sheet":37425,"Bi":37426,"ĠRaspberry":37427,"Ġ328":37428,"anders":37429,"ĠBlossom":37430,"ĠFINAL":37431,"acid":37432,"Ġborderline":37433,"Aut":37434,"Ġoriginate":37435,"Ġtransm":37436,"Ġbuffalo":37437,"atial":37438,"ĠCraigslist":37439,"Ġcredential":37440,"Ġdisbanded":37441,"Ġunprotected":37442,"ĠZer":37443,"waukee":37444,"diagn":37445,"1999":37446,"doc":37447,"ellig":37448,"Ġwarheads":37449,"ĠADS":37450,"verified":37451,"ĠHAM":37452,"785":37453,"Cu":37454,"Ġenorm":37455,"ĠSkill":37456,"\\":37457,"Ġbashing":37458,"Ġloudspe":37459,"during":37460,"Ġdebunked":37461,"adequ":37462,"Ġuh":37463,"Feed":37464,"ificial":37465,"pred":37466,"ĠPassing":37467,"Kyle":37468,"enance":37469,"ĠMex":37470,"itect":37471,"Ġcavern":37472,"Ġtrop":37473,"ĠEliot":37474,"753":37475,"Ġencountering":37476,"Ġsulf":37477,"Always":37478,"ĠGest":37479,"Ġadditive":37480,"Ġ278":37481,"Ġloops":37482,"liberal":37483,"urion":37484,"ĠRefresh":37485,"ĠDynasty":37486,"Ġsweaty":37487,"Ġsails":37488,"protection":37489,"ĠRooms":37490,"ĠEXT":37491,"few":37492,"ĠPaid":37493,"Ġ377":37494,"Ġcolonialism":37495,"Ġchuckle":37496,"Ġarmour":37497,"Ġsoftly":37498,"661":37499,"Building":37500,"ĠAMER":37501,"Ġbabe":37502,"Ġshif":37503,"Sem":37504,"Ġdisembark":37505,"ĠSubstance":37506,"Stone":37507,"Ġdialect":37508,"ĠAph":37509,"Ġspreadsheet":37510,"ierra":37511,"Ġlineage":37512,"ĠCust":37513,"ĠBabe":37514,"Ġwra":37515,"ĠMafia":37516,"Ġflakes":37517,"ĠEVER":37518,"cong":37519,"ĠCreation":37520,"loo":37521,"ĠAmpl":37522,"ĠSpectre":37523,"012":37524,"geons":37525,"Ġswarm":37526,"ĠPale":37527,"ĠSeek":37528,"itures":37529,"Ġarri":37530,"Ġredistribution":37531,"campaign":37532,"ĠAbility":37533,"579":37534,"ournament":37535,"locks":37536,"Ġnests":37537,"ĠConstantine":37538,"Ġwhisper":37539,"Ġshrouded":37540,"changed":37541,"ĠEnhanced":37542,"Ġ920":37543,"Ġglob":37544,"Tam":37545,"Ġoutwe":37546,"Ġilliter":37547,"Ġsurg":37548,"Nap":37549,"ĠAerial":37550,"iferation":37551,"Egypt":37552,"ERO":37553,"Ġantip":37554,"environment":37555,"machine":37556,"Ġrupture":37557,"treatment":37558,"internal":37559,"Ġinfiltrate":37560,"Ġgratification":37561,"Uber":37562,"Ġunequal":37563,"Ġflav":37564,"Lord":37565,"tein":37566,"ĠLOT":37567,"Ġbullshit":37568,"Ġoriginals":37569,"Ġminced":37570,"Ġmultiply":37571,"ayson":37572,"Ġrecomm":37573,"Ġreceptors":37574,"Ġflashlight":37575,"Ġinhuman":37576,"Future":37577,"Ġpuzzling":37578,"Ġrouters":37579,"Ġuncontroll":37580,"responsible":37581,"Ġcellul":37582,"ĠTablet":37583,"Ġbolted":37584,"Ġpermissible":37585,"adra":37586,"picture":37587,"ODY":37588,"BRE":37589,"Iraq":37590,"Total":37591,"rising":37592,"Ġ273":37593,"nv":37594,"Ġ327":37595,"alysed":37596,"infect":37597,"Ġ1912":37598,"ĠVT":37599,"ĠLazarus":37600,"ictive":37601,"Bu":37602,"ĠNEVER":37603,"ĠCODE":37604,"ĠModified":37605,"fetched":37606,"ĠTrap":37607,"mob":37608,"Ġupkeep":37609,"WARD":37610,"Ġbrewed":37611,"Ġsaliva":37612,"Ġ1923":37613,"Ġsteroid":37614,"rather":37615,"ĠVER":37616,"Ġcontextual":37617,"Ont":37618,"ĠLSD":37619,"agine":37620,"Ġaudible":37621,"ĠMeta":37622,"erek":37623,"aults":37624,"ĠOttoman":37625,"ĠIncludes":37626,"Ġocc":37627,"678":37628,"ipple":37629,"Ġcontrasted":37630,"014":37631,"ĠLenin":37632,"Ġomega":37633,"885":37634,"civil":37635,"Ġoverload":37636,"},\"":37637,"Ġprogrammers":37638,"Ġgeometry":37639,"?).":37640,"shift":37641,"ĠClancy":37642,"nr":37643,"verb":37644,"Ġ760":37645,"Ġstaggered":37646,"Playing":37647,"ĠSmile":37648,"Ġcomplains":37649,"ĠSloven":37650,"Ġdisobedience":37651,"creator":37652,"Ġly":37653,"incoln":37654,"emp":37655,"Ġcrate":37656,"ĠPledge":37657,"ĠGPUs":37658,"protected":37659,"Vo":37660,"medium":37661,"Ġacet":37662,"603":37663,"478":37664,"469":37665,"Further":37666,"Ġsensed":37667,"Lock":37668,"Ġcrabs":37669,"ĠChains":37670,"ĠNEO":37671,"Ġexperimented":37672,"ĠRhythm":37673,"802":37674,"Ġhormonal":37675,"491":37676,"ĠMedian":37677,"Ġevaluates":37678,"ippi":37679,"Ġremovable":37680,"Ġvector":37681,"ilant":37682,"TERN":37683,"Ġpurch":37684,"ĠBind":37685,"athering":37686,"Ġcords":37687,"Lib":37688,"Ġdamned":37689,"orc":37690,"ĠEverywhere":37691,"Ġgorilla":37692,"ystem":37693,"fail":37694,"Ġecstasy":37695,"allion":37696,"Sea":37697,"Ġuploading":37698,"ĠSpecific":37699,"Ġreinforcement":37700,"cerned":37701,"ĠDollars":37702,"Twenty":37703,"OX":37704,"ADD":37705,"Ġbraces":37706,"Ġraven":37707,"Ġ1890":37708,"Ġcirculate":37709,"udden":37710,"Disney":37711,"ĠNope":37712,"ĠBagg":37713,"ĠBuddha":37714,"rael":37715,"urus":37716,"ĠKarma":37717,"Ġcurl":37718,"Ġflips":37719,"Ġbearer":37720,"Ġmisunderstand":37721,"Ġabras":37722,"ĠAssassin":37723,"Fact":37724,"Ġinterf":37725,"Ġvantage":37726,"ĠGenocide":37727,"Ġdeducted":37728,"Sep":37729,"McC":37730,"Jessica":37731,"ĠBackup":37732,"Ian":37733,"urnal":37734,"Ġlaborers":37735,"438":37736,"ĠContinuous":37737,"ĠNBN":37738,"Cool":37739,"mitting":37740,"ĠNormandy":37741,"Ġpurchaser":37742,"Ġacquainted":37743,"Ġblogging":37744,"route":37745,"marine":37746,"Ġstartled":37747,"6000":37748,"ĠRadical":37749,"kiss":37750,"ĠBlitz":37751,"express":37752,"Ġ601":37753,"hent":37754,"Ġtink":37755,"pires":37756,"launch":37757,"sg":37758,"ĠEffects":37759,"Ġstiffness":37760,"ĠAllies":37761,"Ġthirsty":37762,"Ġmyst":37763,"Ġlogger":37764,"Ġstances":37765,"ĠEvaluation":37766,"090":37767,"Ġproclaiming":37768,"Ġhypocritical":37769,"496":37770,"Ġcaus":37771,"ĠKappa":37772,"ĠLann":37773,"ĠScientist":37774,"Ġempath":37775,"etrical":37776,"lege":37777,"Hom":37778,"Aud":37779,"ĠColors":37780,"ĠStraw":37781,"each":37782,"ĠPatron":37783,"Ġnuance":37784,"send":37785,"ourney":37786,"ĠPhen":37787,"Ġamino":37788,"ĠSeconds":37789,"Sn":37790,"ĠCiv":37791,"Ġconglomer":37792,"Ġ411":37793,"versely":37794,"487":37795,"prises":37796,"Ġ277":37797,"necessary":37798,"Ġdope":37799,"Late":37800,"Ġrake":37801,"ĠBrigham":37802,"ogun":37803,"ĠSTATES":37804,"ĠGaal":37805,"Ġintellig":37806,"Ġglacier":37807,"destruct":37808,"ĠZucker":37809,"484":37810,"Ġ332":37811,"ĠArist":37812,"Ġprotagonists":37813,"Ġgraveyard":37814,"names":37815,"ĠPax":37816,"Ġthresholds":37817,"Seeing":37818,"Ġmunitions":37819,"Ġcontradicts":37820,"684":37821,"Ġ529":37822,"ĠConcent":37823,"ĠBlessed":37824,"Hz":37825,"Ġinhibit":37826,"Ġshenanigans":37827,"ĠSpear":37828,"Ġoverlay":37829,"ritis":37830,"ilus":37831,"Ġvariance":37832,"Ġoverpower":37833,"viol":37834,"erning":37835,"Ġpolarization":37836,"aito":37837,"GV":37838,"493":37839,"Keeping":37840,"Ġpaternity":37841,"ĠHappiness":37842,"oops":37843,"sb":37844,"xit":37845,"ophysical":37846,"Ġconclusive":37847,"Arch":37848,"Ġmiser":37849,"Ġsuffice":37850,"ĠStout":37851,"Ġhrs":37852,"643":37853,"Ġprincipled":37854,"azine":37855,"atorium":37856,"ĠFairy":37857,"Ġinfiltrated":37858,"ĠHier":37859,"ĠMIA":37860,"inders":37861,"Ġrebutt":37862,"Ġxx":37863,"Ġfeats":37864,"izzle":37865,"Ġ780":37866,"668":37867,"Ġrepressive":37868,"ĠYugoslavia":37869,"sole":37870,"704":37871,"ĠRPG":37872,"ĠTroll":37873,"packing":37874,"ĠDatabase":37875,"ĠVelvet":37876,"ĠRELEASE":37877,"ablish":37878,"smoking":37879,"ĠBottle":37880,"ĠFully":37881,"ĠLean":37882,"Ġobjectively":37883,"ĠFounding":37884,"ĠClassics":37885,"Ġmosaic":37886,"473":37887,"Ġrooft":37888,"Ġcentrally":37889,"Ġdismissive":37890,"Ġparasites":37891,"009":37892,"Ġcursed":37893,"Ġvex":37894,"Ġeconom":37895,"ĠBore":37896,"enery":37897,"ĠFundamental":37898,"ĠOmni":37899,"489":37900,"714":37901,"Ġforegoing":37902,"Ġfragment":37903,"oros":37904,"070":37905,"ĠFaust":37906,"Ġsucking":37907,"Ġnode":37908,"Ġrighteous":37909,"ĠPowered":37910,"426":37911,"HQ":37912,"Ġchronically":37913,"ĠBAL":37914,"Ġprest":37915,"Ġrapists":37916,"ĠRelationship":37917,"ĠCHR":37918,"Ġlinen":37919,"Ġnumerical":37920,"oters":37921,"Ġiterations":37922,"ttes":37923,"ĠENTER":37924,"Ġrabbi":37925,"Ġhoard":37926,"Ġmerciless":37927,"Ġrobes":37928,"ĠSpray":37929,"Ġadvers":37930,"ilantro":37931,"483":37932,"Ġfungus":37933,"Ġalcoholism":37934,"anasia":37935,"ĠCruiser":37936,"Ġmorals":37937,"cision":37938,"measures":37939,"Ġsabot":37940,"Ġrecol":37941,"ĠSaur":37942,"ĠError":37943,"Ġmysteriously":37944,"sle":37945,"Ġfeminists":37946,"д":37947,"ackle":37948,"ĠMarxist":37949,"Ġselves":37950,"Ġdoorway":37951,"Ġdiscard":37952,"Ġbandits":37953,"ĠDive":37954,"ameless":37955,"TRY":37956,"Ġgull":37957,"Ġrepublican":37958,"sr":37959,"ĠDynamo":37960,"Ġembryo":37961,"MENTS":37962,"ĠLOW":37963,"Ġ319":37964,"Ġgly":37965,"Ġcowork":37966,"Coll":37967,"Ġcris":37968,"ĠBanana":37969,"reality":37970,"Ġmobilization":37971,"unal":37972,"Updated":37973,"Crew":37974,"ĠGideon":37975,"Ġvines":37976,"Ġknitting":37977,"Ġdag":37978,"ĠSurv":37979,"Ġvacc":37980,"Ġimpulses":37981,"Northern":37982,"Ġnanop":37983,"allows":37984,"UTH":37985,"Ġflashbacks":37986,"alsa":37987,"Ġ282":37988,"Ġtransmissions":37989,"ĠAlmighty":37990,"Office":37991,"ĠBride":37992,"ĠBeasts":37993,"othy":37994,"ĠClouds":37995,"ĠDyn":37996,"ĠJolly":37997,"District":37998,"Ġveget":37999,"Ġantit":38000,"ĠSmoking":38001,"hess":38002,"Ġcompose":38003,"Ġreligiously":38004,"ĠHY":38005,"Ġfluorescent":38006,"rame":38007,"ĠMeier":38008,"ĠSQ":38009,"benefit":38010,"Thirty":38011,"559":38012,"ĠCance":38013,"586":38014,"Ġgrouped":38015,"Ġphys":38016,"Ġrebellious":38017,"ĠBASE":38018,"chid":38019,"582":38020,"ĠLessons":38021,"ĠWonderful":38022,"ODE":38023,"uctions":38024,"Ġbarbaric":38025,"rahim":38026,"635":38027,"Ġcloves":38028,"ĠNIH":38029,"ossession":38030,"Employ":38031,"Ġliberate":38032,"Gro":38033,"Ġmagician":38034,"ountain":38035,"FORM":38036,"533":38037,"Ġunpredict":38038,"rity":38039,"Ġfaked":38040,"plets":38041,"ppelin":38042,"Living":38043,"Ġnearer":38044,"Ġsuperiors":38045,"Ur":38046,"Ġheroism":38047,"Ġbearded":38048,"006":38049,"Cole":38050,"1970":38051,"Ġsill":38052,"ĠReduce":38053,"OLOG":38054,"onel":38055,"Billy":38056,"ĠPainter":38057,"ansas":38058,"Ġintermediary":38059,"trump":38060,"ĠMith":38061,"otom":38062,"434":38063,"Ġterrit":38064,"Wa":38065,"Ġsuprem":38066,"Rh":38067,"liction":38068,"ĠDEAD":38069,"Ġbothers":38070,"503":38071,"Ġfrogs":38072,"Ġsprinkled":38073,"Ġnil":38074,"628":38075,"Private":38076,"ĠKGB":38077,"Ġoverriding":38078,"Ġdeceived":38079,"698":38080,"idium":38081,"Ġseeker":38082,"Final":38083,"Ġsubconscious":38084,"Ġwom":38085,"Ġcass":38086,"Ġchicks":38087,"Ġverifying":38088,"ective":38089,"inia":38090,"ĠDetection":38091,"MH":38092,"fortable":38093,"ĠISPs":38094,"Ġcrumble":38095,"ĠRecap":38096,"598":38097,"ummies":38098,"export":38099,"Irish":38100,"Ġlil":38101,"ĠRapt":38102,"ĠRIGHT":38103,"Ġanecdotal":38104,"Ġpiercing":38105,"deck":38106,"Liber":38107,"Books":38108,"Ġassassin":38109,"Tur":38110,"revolution":38111,"ĠSheep":38112,"ĠPublishers":38113,"EMS":38114,"iosis":38115,"finder":38116,"ĠCuriosity":38117,"ARB":38118,"ĠConvers":38119,"IVES":38120,"clave":38121,"ĠChaos":38122,"ĠMim":38123,"ĠCostume":38124,"Ġtwe":38125,"Ġintim":38126,"757":38127,"berto":38128,"Ġ261":38129,"VPN":38130,"cribed":38131,"ĠVerb":38132,"cb":38133,"Ġaxle":38134,"Ġsandwic":38135,"Ice":38136,"ĠThermal":38137,"654":38138,"709":38139,"ĠPact":38140,"ĠEnsure":38141,"izable":38142,"497":38143,"Ġbloodstream":38144,"Aw":38145,"Ġleakage":38146,"Ġalleg":38147,"ĠMelody":38148,"681":38149,"Austin":38150,"428":38151,"Ġsummarized":38152,"ĠDefendants":38153,"ĠVader":38154,"Ê":38155,"Ġ1880":38156,"Ġassemb":38157,"YOU":38158,"GREEN":38159,"jury":38160,"4000":38161,"Ġvenerable":38162,"Ġcomputational":38163,"Ġperpetuate":38164,"Ġtorpedo":38165,"Ġaborted":38166,"Ġrhetorical":38167,"ĠOvert":38168,"Ġacknowledgment":38169,"essment":38170,"ĠIGN":38171,"ĠSheen":38172,"571":38173,"Ġcontag":38174,"Ġcultiv":38175,"Ġspawn":38176,"mess":38177,"Dur":38178,"Ġvortex":38179,"ixties":38180,"ĠBlow":38181,"Sum":38182,"Åį":38183,"Rom":38184,"ĠRadeon":38185,"Fed":38186,"Ġameric":38187,"ĠAnth":38188,"Ġantic":38189,"Ġfortress":38190,"Cold":38191,"ĠPredict":38192,"Fake":38193,"Ġilluminate":38194,"Find":38195,"Ġintellectually":38196,"Ġgon":38197,"alker":38198,"Ġinvoice":38199,"IELD":38200,"Ġfools":38201,"ĠEnding":38202,"-(":38203,"Ġalk":38204,"ĠControlled":38205,"Ġpurposefully":38206,"ĠChronic":38207,"Ġrele":38208,"ĠOps":38209,"Party":38210,"ethnic":38211,"ĠSpecifications":38212,"ffee":38213,"ĠTeach":38214,"ulas":38215,"Ġenslaved":38216,"onomy":38217,"Ġtenets":38218,"Ġammonia":38219,"Ġ1913":38220,"Ġdripping":38221,"612":38222,"659":38223,"ĠSagan":38224,"Ġinaccur":38225,"Ġabol":38226,"ĠLIKE":38227,"Ġvisualization":38228,"learn":38229,"anon":38230,"cipline":38231,"Ġadaptations":38232,"Ġwaiter":38233,"nergy":38234,"507":38235,"ĠDK":38236,"YD":38237,"Ġpedest":38238,"Sense":38239,"ĠObst":38240,"Ġresurrection":38241,"ĠSPECIAL":38242,"Unlike":38243,"Ġlia":38244,"Ġpersuasive":38245,"iatrics":38246,"ONEY":38247,"esthetic":38248,"494":38249,"zik":38250,"Ġfract":38251,"ĠOutput":38252,"ĠBers":38253,"rozen":38254,"ĠRevis":38255,"Ġdraconian":38256,"Words":38257,"asions":38258,"ĠClintons":38259,"CU":38260,"History":38261,"Ġtwilight":38262,"iform":38263,"Ġdispl":38264,"progress":38265,"ĠIO":38266,"Ġcannibal":38267,"Michelle":38268,"Ġnerv":38269,"Ġcontexts":38270,"ĠHorses":38271,"Ġanatomy":38272,"ĠLegislation":38273,"ĠBloody":38274,"Ġunwittingly":38275,"Ġinquired":38276,"ĠZip":38277,"ĠDesigns":38278,"Ġirritating":38279,"Ġunison":38280,"ĠRG":38281,"aviour":38282,"Ġpseudo":38283,"ĠVenom":38284,"Ġobscured":38285,"Ġner":38286,"uked":38287,"ORGE":38288,"Ġmomentarily":38289,"olyn":38290,"Syrian":38291,"Ġmicroscopic":38292,"Ġmistress":38293,"Less":38294,"Ġawoke":38295,"Ġtutor":38296,"esome":38297,"ollar":38298,"egg":38299,"UTE":38300,"Buzz":38301,"Ġattainment":38302,"Ġdiscriminating":38303,"::":38304,"Ġ525":38305,"azard":38306,"ĠBrist":38307,"oras":38308,"Ġveterin":38309,"jing":38310,"idon":38311,"ĠAustral":38312,"arious":38313,"ĠGrav":38314,"anol":38315,"ĠQuran":38316,"Ġbleach":38317,"588":38318,"ĠOsw":38319,"Ġdiffered":38320,"typ":38321,"ĠSIL":38322,"failed":38323,"436":38324,"Ġpalms":38325,"ĠFail":38326,"idespread":38327,"Ġchap":38328,"ĠIMAGES":38329,"ACP":38330,"matched":38331,"Ġjaws":38332,"MHz":38333,"Nik":38334,"ĠHume":38335,"OSH":38336,"Ġpresume":38337,"secut":38338,"ĠDied":38339,"ĠBreat":38340,"gins":38341,"prison":38342,"ĠUR":38343,"ĠROS":38344,"isitions":38345,"Ġpelvic":38346,"exclusive":38347,"522":38348,"689":38349,"FN":38350,"Ġener":38351,"Ġdispers":38352,"Ġcohorts":38353,"shut":38354,"ĠLoad":38355,"needs":38356,"azaki":38357,"inoa":38358,"Inside":38359,"usra":38360,"ighters":38361,"Ġ271":38362,"Ġsubordinate":38363,"ĠHOL":38364,"ĠGlow":38365,"Ġincred":38366,"ĠMadame":38367,"Ġoats":38368,"Ġdeviation":38369,"ĠApproach":38370,"Ġnarc":38371,"bart":38372,"bole":38373,"ĠSHE":38374,"effects":38375,"ĠADA":38376,"Ġmuse":38377,"Squ":38378,"Ġneuroscience":38379,"ĠValues":38380,"engu":38381,"Ġdosage":38382,"Ġwhispers":38383,"Ġnaughty":38384,"ĠFarming":38385,"Recently":38386,"Ġrelapse":38387,"rentice":38388,"UGH":38389,"Ġdarkened":38390,"appings":38391,"ĠSlaughter":38392,"ĠAnim":38393,"Ġovertly":38394,"poses":38395,"Ġdeficient":38396,"Ġnecks":38397,"Iron":38398,"Ġphysiological":38399,"ĠLiang":38400,"Ġlear":38401,"Ġcelestial":38402,"Ġpistols":38403,"Ġeyebrow":38404,"915":38405,"ratch":38406,"cephal":38407,"ĠPSU":38408,"Ġphotograp":38409,"ĠGaul":38410,"Ġuncontrolled":38411,"ĠJoined":38412,"652":38413,"itory":38414,"Ġ274":38415,"GAN":38416,"imester":38417,"essional":38418,"Ø©":38419,"Ġuncons":38420,"THER":38421,"Ġpaternal":38422,"Zero":38423,"ugen":38424,"538":38425,"Ġende":38426,"Ġ505":38427,"movie":38428,"Lind":38429,"Ġscorn":38430,"ulty":38431,"Ġpesky":38432,"Ġ8000":38433,"677":38434,"Ġhomophobia":38435,"ranch":38436,"Ġnarciss":38437,"ĠVoyager":38438,"ĠHELP":38439,"528":38440,"edly":38441,"Ġdetract":38442,"Hope":38443,"787":38444,"ĠMerlin":38445,"Ġgrids":38446,"KI":38447,"Mu":38448,"ĠSelected":38449,"select":38450,"ĠModer":38451,"ĠFeet":38452,"Ġrename":38453,"intensity":38454,"Wilson":38455,"Ġ414":38456,"leave":38457,"Ready":38458,"intuitive":38459,"Ġmeager":38460,"Franc":38461,"DH":38462,"Ġrhy":38463,"ĠPillar":38464,"ĠDOE":38465,"minist":38466,"ĠGrave":38467,"isible":38468,"Ess":38469,"Ġempt":38470,"Ġpatched":38471,"ĠAbortion":38472,"rals":38473,"Ġdow":38474,"Ġcrawled":38475,"igrate":38476,"Virginia":38477,"Ġconting":38478,"Ġorphans":38479,"ĠCrimean":38480,"Ġdyn":38481,"Ġshadowy":38482,"sound":38483,"ailable":38484,"Ġ293":38485,"vm":38486,"Ġaccompanies":38487,"Meanwhile":38488,"JR":38489,"ĠDirections":38490,"Ġadolescence":38491,"Ġpenetrated":38492,"bars":38493,"Rev":38494,"Ta":38495,"ĠSkywalker":38496,"ĠFires":38497,"concept":38498,"ĠSIG":38499,"554":38500,"currently":38501,"Ġ----------------":38502,"ĠWHITE":38503,"767":38504,"rors":38505,"PDF":38506,"Ġcasing":38507,"673":38508,"Ġdisapprove":38509,"1800":38510,"ĠWeed":38511,"Ġinhib":38512,"Ġmorbid":38513,"433":38514,"Ġawfully":38515,"Ts":38516,"Maria":38517,"Ġillusions":38518,"Ġtotalitarian":38519,"ollo":38520,"Ġsuppl":38521,"Ġsarc":38522,"ĠRGB":38523,"Ġlauncher":38524,"Ġbadass":38525,"ĠSyd":38526,"Ġscrape":38527,"ĠCLA":38528,"Ġcircum":38529,"657":38530,"Ġnucleus":38531,"ĠUkip":38532,"Ġmodem":38533,"ĠJou":38534,"adders":38535,"Ġwiser":38536,"thereal":38537,"Ġdemocr":38538,"ĠInvalid":38539,"Mine":38540,"Ġmanifested":38541,"meat":38542,"MORE":38543,"Larry":38544,"acements":38545,"Ġspecimen":38546,"results":38547,"Ġswallowing":38548,"Ġpigeon":38549,"tons":38550,"ĠLose":38551,"Ġquartz":38552,"Ġintraven":38553,"Ġ412":38554,"alyst":38555,"Ġengraved":38556,"client":38557,"ĠADV":38558,"ĠShared":38559,"Ġrites":38560,"Ġhysterical":38561,"ĠHUM":38562,"Cow":38563,"orously":38564,"Ġpleasures":38565,"democratic":38566,"Ġamph":38567,"Ġnib":38568,"rieg":38569,"Ġcalculates":38570,"Ġfrying":38571,"favorite":38572,"Ġantim":38573,"ĠDoom":38574,"monitor":38575,"Want":38576,"Ġtemplates":38577,"558":38578,"iever":38579,"Photos":38580,",,":38581,"ĠSync":38582,"Ġconfronts":38583,"kept":38584,"dt":38585,"ĠERROR":38586,"ETF":38587,"578":38588,"Ġspor":38589,"718":38590,"ivation":38591,"ĠHaskell":38592,"Ca":38593,"Ġdick":38594,"Ġcivilized":38595,"Ġblah":38596,"enough":38597,"Ġoccup":38598,"Ġ334":38599,"antically":38600,"584":38601,"ĠDolphin":38602,"ĠStarts":38603,"Ġfanatic":38604,"ت":38605,"imag":38606,"Ġmicrobial":38607,"freedom":38608,"cult":38609,"wra":38610,"Ġ423":38611,"RIPT":38612,"601":38613,"BTC":38614,"atmeal":38615,"653":38616,"agogue":38617,"Ġderives":38618,"Wolf":38619,"466":38620,"Susan":38621,"ĠPassage":38622,"ARDS":38623,"Guy":38624,"Council":38625,"Ġerotic":38626,"pure":38627,"ĠMemories":38628,"ĠWikileaks":38629,"elines":38630,"Ġanth":38631,"Capital":38632,"807":38633,"ĠEggs":38634,"cv":38635,"ctors":38636,"Ġshatter":38637,"Ġesteem":38638,"vity":38639,"ĠVulcan":38640,"effic":38641,"ĠBELOW":38642,"Ġplatoon":38643,"Commun":38644,"oustic":38645,"Amy":38646,"Freedom":38647,"ppo":38648,"Ja":38649,"ĠConan":38650,"Ġinsepar":38651,"scene":38652,"Ġurinary":38653,"gain":38654,"Hillary":38655,"ĠTAM":38656,"Hist":38657,"Ġmechan":38658,"ĠRobots":38659,"Leader":38660,"Ġcartridges":38661,"Ġwhistleblowers":38662,"ĠSPL":38663,"Labour":38664,"unction":38665,"Ġfaithfully":38666,"Ġcoarse":38667,"Ġsynth":38668,"ĠLV":38669,"Ġjustifying":38670,"439":38671,"Victoria":38672,"ĠProceedings":38673,"alogy":38674,"Ġmorph":38675,"Ġcove":38676,"Ġlaughable":38677,"ECA":38678,"Ġ670":38679,"aturated":38680,"ĠSouls":38681,"ĠSleeping":38682,"Ly":38683,"ĠRetro":38684,"Ġastroph":38685,"Ġseism":38686,"atherine":38687,"ĠHercules":38688,"Ġfuse":38689,"ĠHL":38690,"Ġunintentionally":38691,"ĠRé":38692,"iery":38693,"Ġconco":38694,"Ġeras":38695,"recent":38696,"Ġlaunchers":38697,"ĠVolcano":38698,"ĠJace":38699,"Ġterminating":38700,"ĠIde":38701,"zee":38702,"asonic":38703,"itone":38704,"Ġnutshell":38705,"Ġbip":38706,"dies":38707,"Ġ286":38708,"Ġnood":38709,"ĠFathers":38710,"alys":38711,"Ġtheor":38712,"???":38713,"548":38714,"674":38715,"efined":38716,"806":38717,"âĻ":38718,"697":38719,"Ġdecap":38720,"ĠFN":38721,"Ġbureaucr":38722,"ĠGoat":38723,"ĠShang":38724,"Ġsemin":38725,"Ġthroats":38726,"Ġmoth":38727,"herer":38728,"Democratic":38729,"ixtures":38730,"impl":38731,"ĠLogo":38732,"ortunate":38733,"Ġclumsy":38734,"Ġinnocuous":38735,"ĠBlend":38736,"abulary":38737,"ĠFaces":38738,"Ġpornographic":38739,"px":38740,"Information":38741,"Ġfluoride":38742,"Ġatroc":38743,"Ġdelta":38744,"whatever":38745,"ossier":38746,"ĠNoir":38747,"ĠYao":38748,"551":38749,"undred":38750,"Ġmillennium":38751,"Ġferal":38752,"Ġconvinc":38753,"cano":38754,"imsy":38755,"angles":38756,"Ġsterile":38757,"ĠMenu":38758,"779":38759,"ĠCrack":38760,"Ġabundantly":38761,"ĠmL":38762,"Ġinfiltration":38763,"ĠDefinition":38764,"733":38765,"oubt":38766,"Ġorbital":38767,"Ġpiss":38768,"Ġbeet":38769,"679":38770,"Ġcounteract":38771,"ĠALE":38772,"ulative":38773,"crew":38774,"Ġliberating":38775,"ĠDull":38776,"Speaking":38777,"Sadly":38778,"Ġmisfortune":38779,"Ġdolphin":38780,"557":38781,"Ġbould":38782,"ĠTorah":38783,"ĠConfederacy":38784,"421":38785,"Ġorbits":38786,"ocused":38787,"beer":38788,"Rand":38789,"ĠORIG":38790,"Ġmuc":38791,"LER":38792,"ĠMisty":38793,"Ġinexpl":38794,"Ġreptiles":38795,"Ġaven":38796,"blocking":38797,"ĠPASS":38798,"Ġarisen":38799,"ĠMock":38800,"Ġops":38801,"Ġshin":38802,"524":38803,"Ġdigestion":38804,"Soft":38805,"irect":38806,"POL":38807,"ĠSpell":38808,"Level":38809,"Ġhex":38810,"Ġbitcoins":38811,"ĠHungry":38812,"VL":38813,"ĠRealm":38814,"RELATED":38815,"Delta":38816,"Pri":38817,"Ġrejoice":38818,"ĠLatter":38819,"LG":38820,"Ġstupidity":38821,"Ġdonkey":38822,"nova":38823,"Vill":38824,"Ġdecomp":38825,"Ġexternally":38826,"Ġsequest":38827,"815":38828,"Ġshortcut":38829,"riminal":38830,"Hun":38831,"EH":38832,"Ġregiment":38833,"Case":38834,"definition":38835,"Ġappendix":38836,"ĠPlayed":38837,"associated":38838,"izens":38839,"ĠVag":38840,"Ġflung":38841,"Ġfru":38842,"Ġcoil":38843,"________________________":38844,"Ġselects":38845,"Ġsolves":38846,"aea":38847,"985":38848,"Tomorrow":38849,"Ġsear":38850,"APE":38851,"492":38852,"Ġenlightened":38853,"Ġnonexistent":38854,"ĠPotato":38855,"Ghost":38856,"Ġrichness":38857,"ĠKarin":38858,"Ġfamilial":38859,"ĠJA":38860,"Regardless":38861,"Ġepis":38862,"GD":38863,"Ġinsanely":38864,"ĠPhill":38865,"Block":38866,"Finding":38867,"omal":38868,"Ġdecipher":38869,"ĠSwap":38870,"derived":38871,"ĠOFFIC":38872,"Support":38873,"Ġnylon":38874,"Ġexaggeration":38875,"Ġevangelicals":38876,"Ġbearings":38877,"587":38878,"Ġlocale":38879,"Ġpowerfully":38880,"Ġappropriated":38881,"itates":38882,"irlfriend":38883,"cule":38884,"ĠSomewhere":38885,"747":38886,"ĠInteresting":38887,"464":38888,"Ġelong":38889,"Ġdegrade":38890,"rafted":38891,"Ġtutorials":38892,"905":38893,"ĠIntervention":38894,"Ġuniqueness":38895,"Ġ284":38896,"Ġexplorers":38897,"Ġnucle":38898,"ĠMillenn":38899,"511":38900,"ĠReneg":38901,"Ġexecut":38902,"urai":38903,"leon":38904,"Ġdeserts":38905,"ĠCig":38906,"Ġsuggestive":38907,"instead":38908,"Ġlousy":38909,"Ġenigmatic":38910,"594":38911,"Know":38912,"rollment":38913,"ipher":38914,"Ġhumanities":38915,"Ġmodifying":38916,".....":38917,"Ġdegraded":38918,"Ġsuppressing":38919,"Ġeman":38920,"abouts":38921,"functional":38922,"ĠOU":38923,"ĠRelax":38924,"786":38925,"esses":38926,"ĠLogin":38927,"spec":38928,"ĠWWF":38929,"Ġ364":38930,"ĠIsis":38931,"Wisconsin":38932,"Ġequival":38933,"ĠCollector":38934,"ibilities":38935,"malink":38936,"acea":38937,"Ġchained":38938,"Ġarist":38939,"Ġdisadvantages":38940,"ĠBrus":38941,"limits":38942,"ĠDmit":38943,"544":38944,"ĠRecipe":38945,"Ġhabitual":38946,".):":38947,"ĠPRODUCT":38948,"772":38949,"Ġrept":38950,"Ġpathology":38951,"Ġresurrected":38952,"uders":38953,"Ġlingu":38954,"Ġdenomination":38955,"Ġfirewall":38956,"scient":38957,"Ġvaliant":38958,"Kansas":38959,"516":38960,"Ġcontemporaries":38961,"Roman":38962,"Ġaccompan":38963,"Ġantennas":38964,"ĠXan":38965,"Ġelectromagnetic":38966,"ĠNek":38967,"alien":38968,"indle":38969,"Ġgraphene":38970,"Ġgraceful":38971,"syn":38972,"ĠBosh":38973,"Ġ1908":38974,"Ġsuccumb":38975,"Technology":38976,"Ġtoxin":38977,"myra":38978,"essert":38979,"Hell":38980,"Gil":38981,"Ġdiarr":38982,"imeters":38983,"Ġexplo":38984,"Ġgeometric":38985,"ĠNavigation":38986,"cern":38987,"Ġprogrammer":38988,"oÄŁan":38989,"Ġdodging":38990,"ĠLU":38991,"573":38992,"inters":38993,"Ġserum":38994,"Ġuber":38995,"Ġmanga":38996,"762":38997,"ĠOccasionally":38998,"437":38999,"ĠTheme":39000,"Ġimmature":39001,"Ġactivating":39002,"ĠTruly":39003,"د":39004,"osion":39005,"Age":39006,"TIME":39007,"Silver":39008,"sand":39009,"ulnerable":39010,"Ġcram":39011,"Large":39012,"ĠAnger":39013,"icators":39014,"431":39015,"ĠHonest":39016,"zip":39017,"Ġdism":39018,"Ġfades":39019,"ĠPik":39020,"Ast":39021,"sequent":39022,"Ġunsigned":39023,"xious":39024,"creation":39025,"Ġ395":39026,"ottenham":39027,"Ġundesirable":39028,"ugal":39029,"ĠDivide":39030,"lp":39031,"563":39032,"ĠPOP":39033,"ĠCET":39034,"session":39035,"Ġoccurrences":39036,"chu":39037,"ĠACS":39038,"ĠProsecut":39039,"Ġhypnot":39040,"rely":39041,"ERG":39042,"Ven":39043,"Republicans":39044,"inez":39045,"ĠImplementation":39046,"Ġsprang":39047,"Ġobs":39048,"Defense":39049,"Ġunexpl":39050,"ĠPAGE":39051,"ĠTent":39052,"ĠNeurolog":39053,"Ġintuition":39054,"759":39055,"Ġterrestrial":39056,"Ġmorphine":39057,"Ġ.\"":39058,"ĠHydra":39059,"651":39060,"Ġneoliberal":39061,"683":39062,"Ġabnormalities":39063,"quant":39064,"Ġmonastery":39065,"jac":39066,"ĠReaction":39067,"Ġcontraceptive":39068,"ĠBalls":39069,"Ġapost":39070,"676":39071,"ĠHELL":39072,"approximately":39073,"Ġvibrations":39074,"COR":39075,"ĠCPUs":39076,"Ġcontin":39077,"Ġsemblance":39078,"Ġshorth":39079,"tip":39080,"ĠChips":39081,"makes":39082,"Ġprett":39083,"Ġconspicuous":39084,"ĠAmp":39085,"Ġvisualize":39086,"Hu":39087,"sorry":39088,"nai":39089,"ĠArcade":39090,"rimination":39091,"obin":39092,"Ġvampire":39093,"773":39094,"ĠCaucasus":39095,"Medic":39096,"ĠGitHub":39097,"ĠWicked":39098,"ĠFet":39099,"Krist":39100,"998":39101,"Ġfrontal":39102,"Ġ283":39103,"ndum":39104,"Ġidols":39105,"ĠMSG":39106,"ĠShuttle":39107,"ĠTowards":39108,"Ġsaturation":39109,"Ġ®":39110,"Ġcradle":39111,"eteen":39112,"Ġprejudices":39113,"separ":39114,"ĠSoda":39115,"ynam":39116,"Ġnause":39117,"Ġpenetrating":39118,"ĠVampire":39119,"Ġmole":39120,"Ġgoogle":39121,"earance":39122,"583":39123,"Ġdomin":39124,"727":39125,"Kind":39126,"Ġcust":39127,"manuel":39128,"ĠAstro":39129,"Roger":39130,"JO":39131,"killed":39132,"ĠDisapp":39133,"833":39134,"ĠEQU":39135,"Ġprecedence":39136,"mberg":39137,"641":39138,"ĠRoller":39139,"Ġspecifying":39140,"035":39141,"phil":39142,"Ġpowdered":39143,"Ġblot":39144,"Ġdeline":39145,"Bruce":39146,"536":39147,"Ġpim":39148,"leasing":39149,"vacc":39150,"RN":39151,"Ġspacing":39152,"Ġhangar":39153,"ĠPlot":39154,"537":39155,"legraph":39156,"596":39157,"Ġpolyg":39158,"doi":39159,"ĠNerd":39160,"installed":39161,"ĠSeeds":39162,"ĠPlays":39163,"ĠRomance":39164,"layer":39165,"Ġunsu":39166,"Ġcurric":39167,"Mi":39168,"restrial":39169,"ĠNiño":39170,"ĠProper":39171,"Ġpores":39172,"Giving":39173,"aeus":39174,"Middle":39175,"liber":39176,"Ġcombatants":39177,"ĠBulk":39178,"Ġ502":39179,"Ġstru":39180,"ĠLonely":39181,"Companies":39182,"inence":39183,"Autom":39184,"Ġfearsome":39185,"Ġsummar":39186,"Ġrotated":39187,"ĠPLA":39188,"ĠFAT":39189,"572":39190,"ĠSkies":39191,"iour":39192,"Ġintimately":39193,"amera":39194,"Ġ475":39195,"623":39196,"Ġirrig":39197,"Ġboosters":39198,"Ġtransmitting":39199,"DOWN":39200,"ĠAble":39201,"Ġfuriously":39202,"spirit":39203,"Ġgrun":39204,"Ġbible":39205,"ĠAdmir":39206,"Ġ§":39207,"ĠRaise":39208,"Ġflowering":39209,"uxe":39210,"ravis":39211,"urther":39212,"ĠScientology":39213,"pathy":39214,"Ġruth":39215,"Ġtempor":39216,"Ġwhispered":39217,"ogly":39218,"coord":39219,"chlor":39220,"processing":39221,"iott":39222,"ĠTY":39223,"wik":39224,"abolic":39225,"ĠUnable":39226,"ĠLiterary":39227,"ĠpH":39228,"Eastern":39229,"Craig":39230,"Fear":39231,"Ġinventions":39232,"ĠNost":39233,"Ġafflicted":39234,"ĠSwamp":39235,"INST":39236,"Jerry":39237,"Ġprope":39238,"ĠLancet":39239,"Ġrefres":39240,"ĠPrinciples":39241,"ĠLys":39242,"ERAL":39243,"addock":39244,"Ġcynicism":39245,"Ġmassacres":39246,"roo":39247,"Ġcollagen":39248,"Johnny":39249,"Keith":39250,"Italian":39251,"553":39252,"Dad":39253,"Neither":39254,"cler":39255,"ilers":39256,"Ġassass":39257,"Travel":39258,"672":39259,"Ġeaves":39260,"ATOR":39261,"Ġoily":39262,"581":39263,"ateful":39264,"728":39265,"Ġchiefly":39266,"tical":39267,"enes":39268,"ĠWouldn":39269,"ĠJacket":39270,"ĠSuit":39271,"Ġindustrialized":39272,"ĠNose":39273,"ĠSECTION":39274,"Ġredd":39275,"Ġcavity":39276,"Ġconn":39277,"Shield":39278,"Ġtongues":39279,"Ġsuccinct":39280,"views":39281,"ĠMUST":39282,"oliath":39283,"Ġlimitless":39284,"Ġapocalyptic":39285,"ĠAtlantis":39286,"DNA":39287,"ilded":39288,"ĠDresden":39289,"nit":39290,"Ġsubdiv":39291,"gressive":39292,"701":39293,"hops":39294,"alist":39295,"Ġunintentional":39296,"Ġpsychic":39297,"Ġcontrovers":39298,"Ġforeground":39299,"Ġnaïve":39300,"Ġfolders":39301,"icist":39302,"Ġdrawbacks":39303,"ĠToxic":39304,"ophy":39305,"ĠMasonic":39306,"Ġcis":39307,"olated":39308,"Ġdepletion":39309,"Rap":39310,"692":39311,"Ġinver":39312,"ĠFAQ":39313,"Ġmeanings":39314,"Ġbisc":39315,"ĠRage":39316,"Ġresear":39317,"Ep":39318,"Ġunbeat":39319,"ĠComponents":39320,"bub":39321,"ĠInterface":39322,"Isa":39323,"ĠArgon":39324,"Ġdenomin":39325,"Ġmammal":39326,"519":39327,"Ġsizing":39328,"imbabwe":39329,"ĠReplacement":39330,"Georgia":39331,"ĠParticipation":39332,"Ġmelts":39333,"Ġfemin":39334,"514":39335,"Ġseams":39336,"513":39337,"ĠGaw":39338,"Ġbrood":39339,"Mit":39340,"Ġannoyance":39341,"Ġequilibrium":39342,"Ġpatri":39343,"Ġ338":39344,"561":39345,"mentioned":39346,"ĠVotes":39347,"Ġintoler":39348,"Ġstrikingly":39349,"Ġ352":39350,"Ġskeletal":39351,"616":39352,"isition":39353,"Ġfluor":39354,"provided":39355,"517":39356,"Ġclimates":39357,"Ġsensibilities":39358,"ĠFrequ":39359,"onite":39360,"Kenn":39361,"Ġmagnets":39362,"assis":39363,"Ġprerequisite":39364,"Ġ>>>":39365,"Ġscree":39366,"google":39367,"ĠMirage":39368,"Ġevict":39369,"Peace":39370,"Ġmissionaries":39371,"617":39372,"748":39373,"rient":39374,"ĠSTATS":39375,"Bird":39376,"ĠShiva":39377,"ĠBlessing":39378,"Ġredundancy":39379,"Ġphotoc":39380,"ĠOnes":39381,"754":39382,"alert":39383,"urous":39384,"Ġfolklore":39385,"ĠIdeal":39386,"sheets":39387,"according":39388,"Hor":39389,"Cle":39390,"ĠEdit":39391,"671":39392,"olitics":39393,"ĠESC":39394,"Ġparaly":39395,"Ġorgasm":39396,"speak":39397,"ð":39398,"Ġsneaky":39399,"Ġswords":39400,"Ġfandom":39401,"776":39402,"ĠScandinav":39403,"Ġdarts":39404,"546":39405,"cerpt":39406,"ĠGifts":39407,"Ġmagically":39408,"phys":39409,"Laughs":39410,"ĠSour":39411,"ources":39412,"789":39413,"ĠEps":39414,"ository":39415,"uality":39416,"literally":39417,"Ġheavens":39418,"FUL":39419,"Ġie":39420,"ĠISP":39421,"Ġwink":39422,"Ġweeping":39423,"Ġdocking":39424,"ACY":39425,"iece":39426,"Ġsignifies":39427,"guns":39428,"Sac":39429,"Leave":39430,"imation":39431,"Ġunex":39432,"uctive":39433,"ĠFees":39434,"ĠPortable":39435,"ĠInvestigator":39436,"pill":39437,"rehensible":39438,"Ġpotency":39439,"803":39440,"Ġembodiment":39441,"overty":39442,"shine":39443,"REL":39444,"ĠMPH":39445,"ĠPatriarch":39446,"Ġaspirin":39447,"Ġrinse":39448,"Ġinher":39449,"ograms":39450,"ĠTHREE":39451,"qt":39452,"ipples":39453,"Ġdehuman":39454,"Ġslander":39455,"Ġflora":39456,"brow":39457,"Ġblindly":39458,"ectar":39459,"endish":39460,"Ġpigment":39461,"cellent":39462,"Ġyells":39463,"ĠLust":39464,"ĠAttacks":39465,"ĠSyndicate":39466,"otin":39467,"gress":39468,"reenshot":39469,"picking":39470,"Ġacupuncture":39471,"images":39472,"glas":39473,"ĠPolicies":39474,"Ġintestinal":39475,"1998":39476,"ULE":39477,"runs":39478,"ĠNing":39479,"ĠAsuka":39480,"ĠSkull":39481,"Motor":39482,"Ġdefund":39483,"Ġattaching":39484,"ĠBAD":39485,"Ġquarrel":39486,"Child":39487,"Dog":39488,"issan":39489,"irmation":39490,"Ġinline":39491,"ĠLover":39492,"Ġcyan":39493,"entary":39494,"awareness":39495,"Ġtraveller":39496,"âĢIJ":39497,"Ġbeasts":39498,"Ġboobs":39499,"ĠDeadly":39500,"Ġplutonium":39501,"ĠIntellectual":39502,"Jam":39503,"Ġconsec":39504,"663":39505,"ĠVegan":39506,"Ġ331":39507,"uron":39508,"ĠHEL":39509,"reements":39510,"Ġclone":39511,"Ġoutputs":39512,"oult":39513,"ĠDOM":39514,"ĠNX":39515,"Ze":39516,"909":39517,"brate":39518,"arations":39519,"ĠJindal":39520,"Ġbooklet":39521,"amide":39522,"Ġscraping":39523,"Sol":39524,"Date":39525,"796":39526,"Ġfulf":39527,"Ġskeletons":39528,"Ġsaints":39529,"ĠCurious":39530,"Han":39531,"Ġrepud":39532,"osity":39533,"ĠGravity":39534,"Ġmetadata":39535,"Focus":39536,"Ġthrott":39537,"ĠProgramming":39538,"Break":39539,"erver":39540,"Ġknight":39541,"yrs":39542,"Ġ376":39543,"sat":39544,"auto":39545,"Ġbroom":39546,"Ġnerd":39547,"Political":39548,"022":39549,"-------------":39550,"oulos":39551,"Ġrelic":39552,"Ġenactment":39553,"rious":39554,"ĠUniform":39555,"Teen":39556,"Colorado":39557,"055":39558,"Ġangled":39559,"bolt":39560,"ĠNeander":39561,"ĠDism":39562,"thanks":39563,"Polit":39564,"ersion":39565,"dro":39566,"install":39567,"Jake":39568,"hz":39569,"Ġ770":39570,"ĠCommodore":39571,"lahoma":39572,"Ġshri":39573,"Ġ....":39574,"Ġ7000":39575,"scope":39576,"Ġgenesis":39577,"Ġresided":39578,"ĠRivals":39579,"Ġsarcastic":39580,"Ġelicit":39581,"Ġmultiplied":39582,"uitous":39583,"Ġoppress":39584,"ĠPROT":39585,"Ġperpetually":39586,"ĠAdds":39587,"Ġbuffers":39588,"Ġmush":39589,"Ġ354":39590,"Ġpresc":39591,"ĠKung":39592,"682":39593,"Education":39594,"Ġpled":39595,"bsp":39596,"Ġconfessions":39597,"Ġrevocation":39598,"Micro":39599,"ĠHobby":39600,"ĠFatal":39601,"STAR":39602,"Ġworkspace":39603,"Ġtransformations":39604,"Ġportals":39605,"orned":39606,"figured":39607,"Ġlinguistic":39608,"pperc":39609,"ergus":39610,"Fel":39611,"ĠIntent":39612,"Ġ289":39613,"Ġdelinquent":39614,"Ġhandwriting":39615,"Ġvap":39616,"576":39617,"redited":39618,"736":39619,"Ġpsychiatry":39620,"GMT":39621,"Ġdisingen":39622,"Ġcrou":39623,"801":39624,"Ġmalice":39625,"itutes":39626,"ĠTiff":39627,"Ġstink":39628,"574":39629,"Story":39630,"Modern":39631,"ĠGly":39632,"Jamie":39633,"Ġadvertis":39634,"Ġhiber":39635,"Ġinfiltr":39636,"Ġelector":39637,"rovers":39638,"ĠFist":39639,"peed":39640,"ĠClassical":39641,"592":39642,"Ġconscientious":39643,"Surv":39644,"Text":39645,"ĠDrunk":39646,"Ġsupplemented":39647,"THIS":39648,"Ġtimid":39649,"Ġstacking":39650,"rites":39651,"Ġrebirth":39652,"Ġbalcon":39653,"Ġyawn":39654,"rosc":39655,"axy":39656,"Hart":39657,"ĠOPER":39658,"996":39659,"Ġrabid":39660,"ĠTick":39661,"Ġgrinning":39662,"elfth":39663,"045":39664,"Ġjustifies":39665,"ĠPirate":39666,"ĠSalary":39667,"Ġmirac":39668,"613":39669,"inately":39670,"ĠLIN":39671,"Ġinadequ":39672,"NPR":39673,"iddled":39674,"storage":39675,"Ġseventy":39676,"onet":39677,"Ġgastro":39678,"FIR":39679,"Ġrodent":39680,"629":39681,"ĠInclude":39682,"ĠCategories":39683,"ĠLiterally":39684,"Ġpree":39685,"aunder":39686,"ĠLOL":39687,"694":39688,"Ġindef":39689,"Ped":39690,"Ġmenstru":39691,"Ġcensored":39692,"Ġconfigure":39693,"Ġoverest":39694,"igenous":39695,"Ġrectangular":39696,"ĠMIS":39697,"ĠMub":39698,"Ġwitches":39699,"izards":39700,"Ġobnoxious":39701,"ĠLoll":39702,"ĠSEM":39703,"Ġspiritually":39704,"Ġcoer":39705,"Ġmodesty":39706,"butt":39707,"Ġedits":39708,"ĠShall":39709,"sburgh":39710,"Ġ1911":39711,"Rex":39712,"manent":39713,"ĠLithuan":39714,"Ġpointers":39715,"ativity":39716,"retch":39717,"Ġcascade":39718,"ĠRagnarok":39719,"ĠPainting":39720,"ĠATL":39721,"Born":39722,"Ġpadding":39723,"whel":39724,"Ġgrotesque":39725,"Ġtheorists":39726,"forcer":39727,"ĠJinn":39728,"Ġrenal":39729,"jamin":39730,"ĠFEC":39731,".\"\"":39732,"redict":39733,"Ġoppos":39734,"opted":39735,"Sel":39736,"ipment":39737,"752":39738,"792":39739,"Pur":39740,"Ġvolt":39741,"Ġflap":39742,"ĠCASE":39743,"Ġdyed":39744,"orers":39745,"becca":39746,",.":39747,"ifice":39748,"ubes":39749,"Ġyr":39750,"DW":39751,"Ġalteration":39752,"ĠSimpl":39753,"Ġunequiv":39754,"756":39755,"Dou":39756,"Ġplunder":39757,"Ġcommons":39758,"Ġstag":39759,"ĠZeal":39760,"avanaugh":39761,"Self":39762,"none":39763,"EGIN":39764,"Ġflashback":39765,"VAL":39766,"Gab":39767,"ĠCapture":39768,"ĠBrilliant":39769,"ĠDisk":39770,"ĠMood":39771,"Ġhaun":39772,"Ġrotting":39773,"ĠCobra":39774,"Ġpsychopath":39775,"Ġhelper":39776,"Starting":39777,"ĠOrbit":39778,"Ġcaf":39779,"Half":39780,"Volume":39781,"aptop":39782,"ĠSaga":39783,"azor":39784,"593":39785,"774":39786,"ĠCaucasian":39787,"compan":39788,"ĠVERY":39789,"GES":39790,"Ġvomit":39791,"Ġdispro":39792,"ĠMechanics":39793,"Ġ385":39794,"Ġmystical":39795,"AFTA":39796,"Ġbacter":39797,"availability":39798,"Ġhairc":39799,"ĠVec":39800,"rypt":39801,"Ġmanipulative":39802,"shell":39803,"ĠWeird":39804,"jab":39805,"ĠByr":39806,"Bow":39807,"uin":39808,"Ġquot":39809,"MX":39810,"Ġ960":39811,"ĠSharia":39812,"ĠWeapon":39813,"ĠPowerPoint":39814,"Ġstitching":39815,"Ġconstraint":39816,"âľ":39817,"ulic":39818,"597":39819,"omedical":39820,"ĠSupplemental":39821,"ĠSurve":39822,"ĠSubcommittee":39823,"ĠDarkness":39824,"Ġpython":39825,"LU":39826,"Ġ402":39827,"ĠQuan":39828,"ĠModerate":39829,"clusively":39830,"Ġextrap":39831,"Ġlatt":39832,"ĠSTUD":39833,"oslav":39834,"Ġsymb":39835,"battle":39836,"flash":39837,"ĠDeploy":39838,"Ġmicrobiome":39839,"Ġingested":39840,"Ġdistort":39841,"Ġassimil":39842,"Ġmobs":39843,"illet":39844,"Gre":39845,"Ġ294":39846,"Ġforbids":39847,"ĠEfficiency":39848,"ĠClan":39849,"763":39850,"Ġdragons":39851,"States":39852,"ĠMAKE":39853,"ĠBOOK":39854,"ĠRuns":39855,"ĠUX":39856,"EED":39857,"Whoever":39858,"ionics":39859,"worldly":39860,"ĠMermaid":39861,"Ġbenz":39862,"Info":39863,"523":39864,"Ġbiod":39865,"ĠPoison":39866,"ceivable":39867,"Services":39868,"ATIVE":39869,"ĠItem":39870,"Ġdisav":39871,"Ġheter":39872,"Ġasteroids":39873,"ĠWooden":39874,"Ġelectroly":39875,"assadors":39876,"nance":39877,"reflect":39878,"Ġattent":39879,"iphany":39880,"Ġspaceship":39881,"Ġbegg":39882,"algia":39883,"Ax":39884,"Ġidiosyncr":39885,"Ġinserting":39886,"ĠCSS":39887,"ĠLET":39888,"ĠStrikes":39889,"ossibly":39890,"Exp":39891,"Opp":39892,"dden":39893,"Ġplayable":39894,"ĠJM":39895,"Ġlawfully":39896,"ĠBlink":39897,"Ġ413":39898,"Ġoverpowered":39899,"Ġcommenter":39900,"Track":39901,"Ġmethyl":39902,"Ġfermented":39903,"Ġinvaders":39904,"ĠMoves":39905,"Ġcommunicates":39906,"rint":39907,"ĠTray":39908,"jug":39909,"Ġsuperf":39910,"ochet":39911,"ĠJelly":39912,"Ġestrogen":39913,"Dom":39914,"mix":39915,"Gun":39916,"ochemistry":39917,"952":39918,"Ġovere":39919,"ĠPlaintiff":39920,"ĠPilgrim":39921,"ĠSERVICES":39922,"ĠExpend":39923,"ĠFRE":39924,"Ġsmelling":39925,"ĠSpaces":39926,"bris":39927,"Mission":39928,"Ġarter":39929,"Ġautonom":39930,"Lisa":39931,"ĠPercent":39932,"NK":39933,"ĠLimits":39934,"Ġ356":39935,"Recent":39936,"ĠSiberian":39937,"etermin":39938,"nets":39939,"ĠSword":39940,"essee":39941,"Ùĩ":39942,"icycle":39943,"Ġparas":39944,"Ġrud":39945,"Ġscrib":39946,"Ġ1860":39947,"Shop":39948,"orld":39949,"Ġpept":39950,"ENSE":39951,"Ġanimations":39952,"ership":39953,"Search":39954,"ĠUSSR":39955,"washed":39956,"Ġpromulg":39957,"Ġdetainee":39958,"Ġunderest":39959,"ĠAppropri":39960,"Left":39961,"Update":39962,"Wallet":39963,"idently":39964,"ĠBicycle":39965,"Ġgorge":39966,"abyte":39967,"ĠMinecraft":39968,"rike":39969,"997":39970,"Tesla":39971,"Often":39972,"ĠTHESE":39973,"Ġregression":39974,"Hen":39975,"Ġsnippets":39976,"irds":39977,"Ġprinces":39978,"Ġwastes":39979,"ĠWond":39980,"itimate":39981,"ĠMongol":39982,"ĠkW":39983,"Ġidiots":39984,"Ġforeigner":39985,"Upon":39986,"Ġbackdoor":39987,"umph":39988,"ĠSquirrel":39989,"Ġtyped":39990,"Ġblockers":39991,"Vote":39992,"ĠPossibly":39993,"geist":39994,"ĠTRANS":39995,"Ġtitan":39996,"VG":39997,"Ġmicrobi":39998,"Ġinteracts":39999,"Ġmasc":40000,"Ġfinite":40001,"Ġcutoff":40002,"ornings":40003,"Ġprototyp":40004,"Ġcompan":40005,"mology":40006,"ĠBOX":40007,"Cre":40008,"Bot":40009,"grading":40010,"PET":40011,"Ġinsidious":40012,"ĠFranch":40013,"orians":40014,"ĠAUT":40015,"ĠCrush":40016,"589":40017,"question":40018,"anguard":40019,"Ġabsurdity":40020,"?\",":40021,"Hum":40022,"Ġliberalism":40023,"Ġpostwar":40024,"Gener":40025,"Personally":40026,"889":40027,"Bul":40028,"Ġlighthouse":40029,"Ġ291":40030,"VK":40031,"ĠExposure":40032,"Ġsubtract":40033,"ometime":40034,"arbon":40035,"ĠThieves":40036,"anus":40037,"ĠLibertarian":40038,"Raw":40039,"Ġsolvent":40040,"Ġcorros":40041,"Ġsignific":40042,"Ġscholarly":40043,"024":40044,"Ġfetish":40045,"Ġlarvae":40046,"Ġcatast":40047,"Ġtraitor":40048,"ijing":40049,"Demand":40050,"math":40051,"Ġconceivable":40052,"either":40053,"acl":40054,"ĠArrows":40055,"627":40056,"ĠFrankenstein":40057,"entious":40058,"Ġimitation":40059,"amn":40060,"ĠSTOP":40061,"Ġcripp":40062,"zag":40063,"ĠZed":40064,"797":40065,"Along":40066,"Ġwont":40067,"Ġfolds":40068,"Shar":40069,"ĠCommentary":40070,"ĠLibraries":40071,"ĠThunderbolt":40072,"itud":40073,"Toy":40074,"Ġincidentally":40075,"ĠResp":40076,"Ġordinarily":40077,"Ġvanish":40078,"acterial":40079,"Minnesota":40080,"rank":40081,"614":40082,"ĠExam":40083,"Got":40084,"Ġsnipers":40085,"ETHOD":40086,"dirty":40087,"igsaw":40088,"Obs":40089,"ĠAuthors":40090,"Ġillustrating":40091,"782":40092,"864":40093,"Ġblinded":40094,"transfer":40095,"Ġspawning":40096,"ĠDiary":40097,"ĠDNS":40098,"CG":40099,"someone":40100,"Ġcruc":40101,"Morgan":40102,"Learn":40103,"API":40104,"toc":40105,"STAT":40106,"ĠFlame":40107,"aganda":40108,"ĠBenef":40109,"stuff":40110,"SEA":40111,"Ġincest":40112,"Normally":40113,"ĠRU":40114,"Ġarsenic":40115,"isine":40116,"ĠTG":40117,"Type":40118,"regn":40119,"Cass":40120,"Touch":40121,"Site":40122,"Ġpict":40123,"Ġcorrupted":40124,"729":40125,"Ġnineteen":40126,"Ġparaph":40127,"Ġtavern":40128,"Ġretard":40129,"ĠKaf":40130,"Ġcolleg":40131,"bucks":40132,"imum":40133,"ĠCandle":40134,"ĠMisc":40135,"ĠAwesome":40136,"edited":40137,"ĠDN":40138,"otomy":40139,"Ġdisclaimer":40140,"798":40141,"ĠGoodbye":40142,"ucle":40143,"atom":40144,"Judge":40145,"cipl":40146,"Ġinexplicable":40147,"iddler":40148,"781":40149,"Ġempirical":40150,"Veter":40151,"Ġascert":40152,"Ġaest":40153,"Ġlaz":40154,"binary":40155,"Ġ358":40156,"contained":40157,"Ġmultipl":40158,"ocado":40159,"Ġdelusional":40160,"Ġaeros":40161,"udence":40162,"Ġjargon":40163,"estine":40164,"Ġarbitrarily":40165,"Ġprick":40166,"BACK":40167,"amines":40168,"Mess":40169,"Knowing":40170,"ublic":40171,"ĠWarfare":40172,"Ġsignify":40173,"Ġfragmentation":40174,"Tex":40175,"Ġnin":40176,"Ġdise":40177,"882":40178,"hospital":40179,"volent":40180,"Need":40181,"Ġinfer":40182,"Sony":40183,"783":40184,"YING":40185,"Ġinfinity":40186,"ĠFortress":40187,"Ġmustache":40188,"Ġcorresponds":40189,"DX":40190,"Ġunmarried":40191,"ĠCruel":40192,"Ġ1901":40193,"Ġappropri":40194,"ZI":40195,"Ġphosph":40196,"901":40197,"IFE":40198,"Ġ347":40199,"Ġconvoluted":40200,"ĠApost":40201,"htm":40202,"Ġilluminating":40203,"568":40204,"Ġassassinate":40205,"Ġparam":40206,"Ġimpractical":40207,"cedes":40208,"ĠProcedure":40209,"ĠMouth":40210,"Battle":40211,"Ġ451":40212,"Sand":40213,"Ġcontamin":40214,"Hour":40215,"Cell":40216,"BIL":40217,"Ġprecon":40218,"ĠScor":40219,"Ġconfig":40220,"ĠMuscle":40221,"Ġhive":40222,"Ġunderworld":40223,"plement":40224,"Ġpostage":40225,"Ġinterpersonal":40226,"Ġpierced":40227,"Ġcharms":40228,"oscopic":40229,"ASC":40230,"ĠDex":40231,"render":40232,"png":40233,"Ġcritiques":40234,"992":40235,"ĠVinyl":40236,"Bear":40237,"idia":40238,"ĠTemp":40239,"Ġcyn":40240,"ĠBCE":40241,"Ġpatriarchal":40242,"Ġantagonist":40243,"ĠGMO":40244,"Ġunnatural":40245,"Race":40246,"imeo":40247,"ĠUkrainians":40248,"Train":40249,"Ġ329":40250,"ritten":40251,"igil":40252,"Lin":40253,"alus":40254,"*****":40255,"olded":40256,"ĠPegasus":40257,"Bas":40258,"photos":40259,"Ġ820":40260,"Ġsquadron":40261,"ESE":40262,"Ġ373":40263,"Uk":40264,"Lost":40265,"Store":40266,"ĠScenes":40267,"JJ":40268,"Ġlick":40269,"Tyler":40270,"cius":40271,"lishing":40272,"ocl":40273,"Ġassoci":40274,"ensitivity":40275,"entanyl":40276,"Rum":40277,"Ġ443":40278,"onding":40279,"Ġpedals":40280,"ĠPsychological":40281,"Ġthro":40282,"Network":40283,"591":40284,"Pick":40285,"Ġchords":40286,"ĠHound":40287,"entials":40288,"faces":40289,"ĠYin":40290,"ugi":40291,"bows":40292,"ĠForms":40293,"886":40294,"Ox":40295,"Ġ351":40296,"Ġmating":40297,"Ġchirop":40298,"916":40299,"Ġexpend":40300,"Ġusefulness":40301,"Marvel":40302,"ĠStretch":40303,"omez":40304,"ĠJS":40305,"Hal":40306,"fle":40307,"ĠCountdown":40308,"ĠLH":40309,"assian":40310,"vd":40311,"ĠTranscript":40312,"ĠExtrem":40313,"idine":40314,"ustainable":40315,"ederal":40316,"ĠOwl":40317,"Ġcreed":40318,"ĠGrateful":40319,"Ġprenatal":40320,"________________________________":40321,"ĠElements":40322,"âĢ¦)":40323,"nesia":40324,"ARGET":40325,"Ġboredom":40326,"Ġdepictions":40327,"verbal":40328,"ĠeSports":40329,"Laura":40330,"ilage":40331,"ĠGalactic":40332,"Investigators":40333,"Ġscattering":40334,"instein":40335,"ĠExperiment":40336,"ĠRecre":40337,"Ġregul":40338,"Ġrelent":40339,"STE":40340,"Ġslicing":40341,"igans":40342,"raped":40343,"ĠDeter":40344,"Ġsmoker":40345,"ĠWikimedia":40346,"pages":40347,"Ted":40348,"713":40349,"Ġpuberty":40350,"Ġhars":40351,"ĠStarter":40352,"patch":40353,"leeve":40354,"Ġ346":40355,"ĠAccessories":40356,"ventions":40357,"ĠSTAND":40358,"ĠUrug":40359,"ĠOccupy":40360,"Ġbinds":40361,"ĠBubble":40362,"Ġincorporation":40363,"Ġstereotypical":40364,"Ġgor":40365,"987":40366,"Ġevils":40367,"tower":40368,"Ġastronomer":40369,"Ble":40370,"ĠNid":40371,"ĠWidow":40372,"Ġpaw":40373,"Ġinnoc":40374,"ĠOWN":40375,"Ġtofu":40376,"drops":40377,"ĠEval":40378,"693":40379,"Collins":40380,"penter":40381,"ĠNib":40382,"Ġsmokes":40383,"Ġ1850":40384,"Ġtechno":40385,"oooo":40386,"ĠUnic":40387,"ĠKirin":40388,"\":[\"":40389,"Ġincrements":40390,"989":40391,"oodoo":40392,"ĠCyborg":40393,"Ġcures":40394,"ĠOW":40395,"ĠAnnex":40396,"behavior":40397,"/-":40398,"Ġbuggy":40399,"onent":40400,"Bey":40401,"Ġsummarize":40402,"putable":40403,"Ġfri":40404,"Gi":40405,"urances":40406,"ĠAppalach":40407,"Ġhegemony":40408,"ĠOrigins":40409,"Ġconnectors":40410,"ĠAST":40411,"object":40412,"ĠSlay":40413,"Arm":40414,"oston":40415,"ĠEVEN":40416,"Ġprophecy":40417,"Bright":40418,"ĠVector":40419,"Marg":40420,"omical":40421,"Holy":40422,"ĠRPM":40423,"ĠReceiver":40424,"Ġtracts":40425,"boss":40426,"Ġblurry":40427,"aspx":40428,"DES":40429,"Ġcess":40430,"ĠAster":40431,"anything":40432,"levard":40433,"unciation":40434,"jong":40435,"Ġiv":40436,"Common":40437,"ĠDistance":40438,"imus":40439,"outheast":40440,"Ġcir":40441,"ĠCato":40442,"Ġinscribed":40443,"ersed":40444,"Ġanarchy":40445,"Ġplagiar":40446,"Ġthug":40447,"Actor":40448,"ĠTant":40449,"Researchers":40450,"remember":40451,"Ġitch":40452,"Ġrefill":40453,"Ġsucker":40454,"ĠWANT":40455,"RAG":40456,"rencies":40457,"ĠTape":40458,"Ġattaches":40459,"nb":40460,"Tan":40461,"Ġappend":40462,"Ġalas":40463,"951":40464,"panel":40465,"Climate":40466,"icrobial":40467,"Brandon":40468,"ĠFreud":40469,"Ġfungi":40470,"Ġcommenters":40471,"ĠDelicious":40472,"Ġhitherto":40473,"conv":40474,"Ġchemist":40475,"Ġdenominations":40476,"ĠBehavior":40477,"comed":40478,"ĠLantern":40479,"ĠFloating":40480,"magic":40481,"ĠBarbar":40482,"bender":40483,"iliar":40484,"unny":40485,"Ġretracted":40486,"atars":40487,"ĠLovely":40488,"Ġinfinitely":40489,"Ġhumili":40490,"Ġinterestingly":40491,"Ġmunicip":40492,"ĠPanic":40493,"Ġcomprehension":40494,"ĠMassacre":40495,"Ġpersuasion":40496,"enf":40497,"Ġcoded":40498,"higher":40499,"chart":40500,"umbered":40501,"ĠIndigo":40502,"Ġthinker":40503,"Ġgoof":40504,"ĠPetition":40505,"fascist":40506,"absor":40507,"Ġassay":40508,"ĠClassification":40509,"Ġhalluc":40510,"speech":40511,"issues":40512,"Ġinexper":40513,"ĠLibre":40514,"Ġsling":40515,"zech":40516,"Ġpouch":40517,"ĠOffense":40518,"ĠHF":40519,"Fight":40520,"026":40521,"ĠTrident":40522,"fm":40523,"Ġintox":40524,"Ġ465":40525,"colonial":40526,"ovies":40527,"794":40528,"Techn":40529,"undreds":40530,"Ġchildish":40531,"arenthood":40532,"ĠShade":40533,"Host":40534,"Ġdirectional":40535,"reader":40536,"rimp":40537,"ĠEater":40538,"prep":40539,"Ġmeas":40540,"Ġlatch":40541,"inant":40542,"nels":40543,"finished":40544,"application":40545,"Board":40546,"Ġfiller":40547,"ivably":40548,"CAST":40549,"Ġstereotyp":40550,"Ġwarranties":40551,"ĠProbe":40552,"Ġspontaneously":40553,"Ġtropes":40554,"Meg":40555,"ĠHandling":40556,"hemer":40557,"986":40558,"ĠSly":40559,"plates":40560,"Ġmolten":40561,"ĠHIT":40562,"strings":40563,"Ġcentrif":40564,"ĠENG":40565,"Indeed":40566,"Ġ429":40567,"Ġsly":40568,"Ġ490":40569,"Ġhordes":40570,"boot":40571,"691":40572,"ihara":40573,"Ġsubversive":40574,"Russell":40575,"aceous":40576,"wk":40577,"Ġreverence":40578,"Ġingenious":40579,"holiday":40580,"eligible":40581,"ĠTactical":40582,"978":40583,"herence":40584,"Ġgimm":40585,"Ġarchaic":40586,"Ġadam":40587,"Ġ297":40588,"Father":40589,"ĠLerner":40590,"Ġhesitated":40591,"Safety":40592,"Ġawakened":40593,"ueller":40594,"Ġextrater":40595,"Ġmummy":40596,"ĠBuddhism":40597,"Ġ359":40598,"Ġlegions":40599,"Ġprehistoric":40600,"ancouver":40601,"Ġmelancholy":40602,"ĠEnemy":40603,"ĠSyl":40604,"ĠRobo":40605,"verting":40606,"ĠBullets":40607,"essler":40608,"Ġmarvelous":40609,"ĠBened":40610,"Ġsavior":40611,"omever":40612,"Bee":40613,"Ġrapp":40614,"Ġpredomin":40615,"ĠScripture":40616,"Ġsnapshots":40617,"Ġunrem":40618,"Ġsquid":40619,"ĠBuddh":40620,"ĠSantorum":40621,"Internet":40622,"avoid":40623,"Ġunamb":40624,"Ġ296":40625,"Ġnexus":40626,"Ġinterchangeable":40627,"ockets":40628,"Ġfoll":40629,"ĠOPT":40630,"023":40631,"²":40632,"Ġhereditary":40633,"Ġvape":40634,"=\"":40635,"1996":40636,"س":40637,"Emergency":40638,"Ġneb":40639,"Ġisot":40640,"Ġdiam":40641,"stairs":40642,"ĠAppendix":40643,"venient":40644,"Ġinvol":40645,"Ġtheorist":40646,"Ġconqu":40647,"Mich":40648,"ĠSort":40649,"antasy":40650,"dating":40651,"771":40652,"Ġape":40653,"Ġindemn":40654,"ween":40655,"Games":40656,"ascal":40657,"Muslims":40658,"Ġleaflets":40659,"Ġtraverse":40660,"Ġtransgress":40661,"Ġflushed":40662,"893":40663,"lasses":40664,"obos":40665,"ooming":40666,"Ġtou":40667,"mast":40668,"âģ":40669,"751":40670,"Either":40671,"Ġgrate":40672,"urgy":40673,"Ġendowed":40674,"ĠRasm":40675,"Nat":40676,"odka":40677,"olon":40678,"iants":40679,"Ġsensations":40680,"Ġsituational":40681,"pox":40682,"Figure":40683,"Ġslime":40684,"Ġ421":40685,"ollow":40686,"Ġanesthesia":40687,"adult":40688,"ĠPiece":40689,"994":40690,"ĠAnalog":40691,"Iv":40692,"flo":40693,"Ġdomest":40694,"Ġcabal":40695,"Ġgarg":40696,"Ġrabb":40697,"REC":40698,"ISTORY":40699,"Friend":40700,"Ġancestor":40701,"ĠLets":40702,"Ġelf":40703,"Ġlobb":40704,"ĠAdren":40705,"silver":40706,"astical":40707,"Ġstitch":40708,"028":40709,"Hug":40710,"Ġmoss":40711,"ompl":40712,"Ġunob":40713,"883":40714,"Ġcortex":40715,"olutely":40716,"052":40717,"Seattle":40718,"restling":40719,"endment":40720,"Ġ366":40721,"ventus":40722,"ĠRated":40723,"ĠClever":40724,"Ġcloak":40725,"phrase":40726,"flake":40727,"Ġphilosophies":40728,"784":40729,"Ġskulls":40730,"wake":40731,"oru":40732,"ĠACTION":40733,"Ġcomprom":40734,"ĠManufacturer":40735,"ĠImprove":40736,"Ns":40737,"ĠRevenge":40738,"lords":40739,"Ġ417":40740,"iddles":40741,"Ġcondesc":40742,"tiny":40743,"Ġchloride":40744,"greg":40745,"ĠREST":40746,"subject":40747,"Ġundes":40748,"ftime":40749,"Ġbottleneck":40750,"ĠZombie":40751,"Ġhabitable":40752,"Ġcigars":40753,"Ġenlarg":40754,"icester":40755,"ðĿ":40756,"regulation":40757,"arters":40758,"Ġformulations":40759,"Ġadhesive":40760,"Ġ344":40761,"pod":40762,"etitive":40763,"Ġcontinuum":40764,"aghd":40765,"Ġ701":40766,"Ġdisband":40767,"Tu":40768,"Ġcivilisation":40769,"ĠPCI":40770,"Ġcrooked":40771,"ammy":40772,"Ġbrim":40773,"Jr":40774,"ĠBunker":40775,"plot":40776,"Ġwielded":40777,"Ġcaricature":40778,"ĠInfinite":40779,"piracy":40780,"aretz":40781,"Ġstares":40782,"incinnati":40783,"agents":40784,"ĠObamaCare":40785,"asuring":40786,"ansion":40787,"Ġastonished":40788,"iovascular":40789,"Bio":40790,"Ġadvisable":40791,"Ġsender":40792,"887":40793,"Led":40794,"DN":40795,"Ġaggregation":40796,"ĠInnocent":40797,"ĠTransactions":40798,"worms":40799,"ĠWorm":40800,"Ġ363":40801,"ĠBiblical":40802,"rared":40803,"Ġgazing":40804,"chant":40805,"Ġsubordinates":40806,"1600":40807,"actually":40808,"olition":40809,"ĠRTX":40810,"ĠPyramid":40811,"alph":40812,"ĠFPS":40813,"Ġerrone":40814,"ĠLR":40815,"Scientists":40816,"Ġincons":40817,"Ġbrittle":40818,"027":40819,"ĠBowser":40820,"Rub":40821,"links":40822,"ĠWik":40823,"ussion":40824,"Marsh":40825,"resents":40826,"Clean":40827,"Ġbrute":40828,"ĠInventory":40829,"1100":40830,"ĠATK":40831,"793":40832,"Ġcaveats":40833,"ĠKnot":40834,"IRT":40835,"ĠCanad":40836,"isma":40837,"entin":40838,"Own":40839,"Ġ455":40840,"Ġlesions":40841,"ĠAres":40842,"ĠKali":40843,"Ġpaws":40844,"Auto":40845,"Ġdiscrim":40846,"044":40847,"ĠCOUN":40848,"Ġ1905":40849,"Ġexperien":40850,"Ġ406":40851,"achelor":40852,"Ġscarcely":40853,"Ġsynchronized":40854,"Rat":40855,"Blake":40856,"Ġrewriting":40857,"Ġcannons":40858,"stem":40859,"Apparently":40860,"Ġleveling":40861,"?]":40862,"Ġfins":40863,"ĠTone":40864,"ogether":40865,"Sound":40866,"Ġmicrosc":40867,"ĠAsylum":40868,"Ġindividuality":40869,"Ġ432":40870,"lease":40871,"Chuck":40872,"Ġhating":40873,"Ġleftists":40874,"ĠPersonality":40875,"ĠBundle":40876,"Dutch":40877,"Ġtransformer":40878,"iami":40879,"ĠTradition":40880,"ĠRecipes":40881,"Ġdiscour":40882,"Viol":40883,"Ext":40884,"ĠOliv":40885,"ashington":40886,"Ġmillennia":40887,"Ġpsychiatrists":40888,"ĠTrilogy":40889,"inction":40890,"Ġdisliked":40891,"088":40892,"954":40893,"Ġoverloaded":40894,"Ġopium":40895,"acus":40896,"resources":40897,"mud":40898,"ometry":40899,"Hit":40900,"Ġguild":40901,"Ġabyss":40902,"884":40903,"ensity":40904,"ĠDifference":40905,"Electric":40906,"authent":40907,"Ġdownloadable":40908,"ellar":40909,"ĠSavior":40910,"ĠFRI":40911,"Ġ445":40912,"Ġincidental":40913,"Ġanalogue":40914,"ounters":40915,"ĠBuilder":40916,"Ġnarration":40917,"ategor":40918,"raise":40919,"Ġindoctr":40920,"Aren":40921,"Ġbaptism":40922,"Ġobe":40923,"Ġtubing":40924,"apsed":40925,"Fortunately":40926,"gered":40927,"Pict":40928,"Ġmastering":40929,"ĠHIM":40930,"ĠObesity":40931,"Ġornament":40932,"advant":40933,"ĠCous":40934,"032":40935,"cells":40936,"Ġpreclude":40937,"Ġanecdote":40938,"Ġpatriarchy":40939,"ĠSending":40940,"Pie":40941,"Ġdepressive":40942,"ĠEnds":40943,"712":40944,"zos":40945,"icka":40946,"Ġ1906":40947,"Anti":40948,"vana":40949,"ĠRestrict":40950,"Ġprotr":40951,"Ġusername":40952,"Ġparach":40953,"1997":40954,"imental":40955,"rower":40956,"carb":40957,"033":40958,"Ġobligatory":40959,"Ġwillful":40960,"Ġsnail":40961,"json":40962,"izarre":40963,"Ġmiscar":40964,"Ġdopamine":40965,"л":40966,"Ġapplic":40967,"Ġnervously":40968,"YY":40969,"alez":40970,"ĠSoviets":40971,"ĠMister":40972,"Ġcrates":40973,"Ġheavenly":40974,"Ġdoct":40975,"048":40976,"Ġ2400":40977,"ivia":40978,"adies":40979,"Phone":40980,"asks":40981,"Ġperenn":40982,"Ġcomposing":40983,"Ġraiding":40984,"requent":40985,"ibli":40986,"ĠFeedback":40987,"cellaneous":40988,"ĠContracts":40989,"ĠCasting":40990,"vim":40991,"Cut":40992,"Ġabbrevi":40993,"Ġintest":40994,"ricted":40995,"969":40996,"nostic":40997,"Ġinverted":40998,"ĠEG":40999,"aiden":41000,"ĠClaud":41001,"ĠiP":41002,"urized":41003,"Emily":41004,"Ġ353":41005,"Ġ((":41006,"ammad":41007,"Reb":41008,"plom":41009,"YES":41010,"connection":41011,"ĠWra":41012,"ĠMerch":41013,"Ġether":41014,"Elizabeth":41015,"Chip":41016,"relevant":41017,"URA":41018,"Ġantioxidant":41019,"ĠChron":41020,"Ġtheological":41021,"HCR":41022,"ruits":41023,"Body":41024,"enezuel":41025,"Few":41026,"adder":41027,"Ġinducing":41028,"ĠDarth":41029,"Ġimplicitly":41030,"Ġoverfl":41031,"Ġrelics":41032,"Must":41033,"ĠAnswers":41034,"Ġretina":41035,"ĠSlowly":41036,"ĠShib":41037,"software":41038,"Ġ\"\"":41039,"hack":41040,"Apart":41041,"told":41042,"Ger":41043,"Civil":41044,"problem":41045,"Ġslang":41046,"Ġtactile":41047,"Ġtabl":41048,"ĠAscension":41049,"Ġhumankind":41050,"Howard":41051,"rescent":41052,"ĠReleases":41053,"arijuana":41054,"Christopher":41055,"ĠWarden":41056,"blogspot":41057,"ĠVari":41058,"idency":41059,"ĠHandler":41060,"Round":41061,"MJ":41062,"Ġrhyth":41063,"Tai":41064,"terson":41065,"Ġ,\"":41066,"portation":41067,"ĠOrbital":41068,"Ġfantas":41069,"Ġattribut":41070,"Ġdiagram":41071,"atech":41072,"1992":41073,"ibl":41074,"Woman":41075,"ternally":41076,"Days":41077,"Ġdebunk":41078,"ĠPhant":41079,"ĠOath":41080,"sharp":41081,"Ġclaws":41082,"Lots":41083,"Incre":41084,"Aff":41085,"hooting":41086,"rect":41087,"Ġaltru":41088,"Ġwors":41089,"Ġtho":41090,"Ġ349":41091,"clusions":41092,"Ġpseudonym":41093,"Bec":41094,"Ġphosphorus":41095,"ivic":41096,"Ġ348":41097,"otent":41098,"Ġub":41099,"Ġcoales":41100,"regate":41101,"Ġ1870":41102,"Ġglide":41103,"treated":41104,"ĠSymb":41105,"Ġenchant":41106,"Besides":41107,"stocks":41108,"Ġ388":41109,"--------------":41110,"interpret":41111,"ouple":41112,"Ġdrawback":41113,"ĠRevised":41114,"Ġanat":41115,"Ġpsychosis":41116,"ب":41117,"Ġdiffuse":41118,"Ġaffidav":41119,"elve":41120,"amination":41121,"ĠTackle":41122,"hunter":41123,"env":41124,"Ġchests":41125,"Ġsubter":41126,"Ġconquest":41127,"Ġfidelity":41128,"Ġinfringing":41129,"opathic":41130,"ĠGrip":41131,"ĠKeyboard":41132,"Ġobjectionable":41133,"Ġmetabol":41134,"ĠGö":41135,"Room":41136,"...)":41137,"KEN":41138,"assic":41139,"Ġgeop":41140,"Tro":41141,"Ġcursing":41142,"Ġdile":41143,"Ġultraviolet":41144,"inarily":41145,"Ġdistilled":41146,"sect":41147,"ĠShooter":41148,"uckles":41149,"Ġdistortions":41150,"Map":41151,"Doctor":41152,"Ġinstalls":41153,"oire":41154,"Ġstarch":41155,"ociation":41156,"Lev":41157,"Ġscripture":41158,"Ġsalient":41159,"ilitating":41160,"wb":41161,"ĠSov":41162,"ĠDamn":41163,"Grey":41164,"Ġ980":41165,"Ġjung":41166,"Ġlicking":41167,"029":41168,"ĠDian":41169,"ĠBabylon":41170,"к":41171,"ĠRomantic":41172,"Ġguesses":41173,"ĠFren":41174,"Generally":41175,"ultural":41176,"istence":41177,"Ġiniti":41178,"Ġ341":41179,"ĠSlave":41180,"ultan":41181,"ĠTrash":41182,"ĠEmpty":41183,"ĠHundred":41184,"ĠDirective":41185,"Anderson":41186,"Advertisement":41187,"RH":41188,"ĠOo":41189,"ĠHik":41190,"peg":41191,"Sup":41192,"ĠXT":41193,"Ġencrypt":41194,"selage":41195,"ĠThrone":41196,"Ġconsecut":41197,"Li":41198,"ĠVirus":41199,"ĠCookies":41200,"SHIP":41201,"Ġflavorful":41202,"odynamics":41203,"animal":41204,"spread":41205,"ĠIPCC":41206,"jobs":41207,"ernand":41208,"ĠHaunted":41209,"Ġintolerable":41210,"ĠLAR":41211,"ixtape":41212,"Ġneur":41213,"Ġcausal":41214,"ĠPsychiatry":41215,"ĠVim":41216,"Ġgenomic":41217,"duration":41218,"ĠUsername":41219,"ategy":41220,"Ġunic":41221,"ĠKILL":41222,"blooded":41223,"Ġcaucuses":41224,"ĠPOLITICO":41225,"Spanish":41226,"Ġobedience":41227,"Ġinconven":41228,"MAT":41229,"Ġbends":41230,"ĠImprovements":41231,"Ġrelig":41232,"ĠForth":41233,"ĠLumia":41234,"uces":41235,"Ġunim":41236,"ĠStatistical":41237,"kb":41238,"auntlet":41239,"ĠDisco":41240,"ĠInstruction":41241,"ooo":41242,"ĠDictionary":41243,"culated":41244,"Adv":41245,"ĠAvatar":41246,"ictional":41247,"Ġcentr":41248,"ifles":41249,"orks":41250,"skill":41251,"Ġlatex":41252,"ĠPagan":41253,"Ġdevast":41254,"Ġprol":41255,"896":41256,"Product":41257,"968":41258,"Ġfrench":41259,"083":41260,"ĠCluster":41261,"cloth":41262,"ĠFilter":41263,"ĠDisorders":41264,"etimes":41265,"Ġinstinctively":41266,"ĠBritann":41267,"Ġaft":41268,"ĠVict":41269,"Ġâĺħ":41270,"Ġperverse":41271,"Ġcontraceptives":41272,"ĠHannibal":41273,"escap":41274,"ĠApostle":41275,"ĠXiao":41276,"ĠMagnum":41277,"Ġphosphate":41278,"Ġ399":41279,"utable":41280,"Ġsten":41281,"Ġwearer":41282,"Ġsmug":41283,"ĠInfluence":41284,"Ġ384":41285,"Truth":41286,"struction":41287,"Ġmaniac":41288,"ĠMagnetic":41289,"ousands":41290,"Ġsemen":41291,"dir":41292,"ĠTornado":41293,"Ġexplos":41294,"1995":41295,"Xi":41296,"Steel":41297,"057":41298,"Barn":41299,"Fan":41300,"ĠChatt":41301,"Chem":41302,"ĠFold":41303,"bees":41304,"1080":41305,"ĠMaze":41306,"ierre":41307,"oeuv":41308,"Cand":41309,"odium":41310,"mmm":41311,"ereo":41312,"Ġreactionary":41313,"Ġacidic":41314,"ĠRemoval":41315,"Ġnont":41316,"031":41317,"ĠTerminator":41318,"ĠVendor":41319,"enemy":41320,"Ġreconstructed":41321,"ĠGalileo":41322,"Ġtesters":41323,"albeit":41324,"uminium":41325,"Ġrite":41326,"ĠInput":41327,"committee":41328,"Ġjour":41329,"gements":41330,"Ġgerm":41331,"Dick":41332,"ĠRequirements":41333,"omsday":41334,"Î":41335,"ISSION":41336,"Ġmolded":41337,"Ġrye":41338,"Attorney":41339,"population":41340,"Ġrepet":41341,"Sync":41342,"breaks":41343,"Ġbanished":41344,"Ġraspberry":41345,"Ġammo":41346,"Ġorthodox":41347,"Ġwebcam":41348,"ĠAsc":41349,"vl":41350,"1989":41351,"Ġdiscipl":41352,"Ġmoreover":41353,"Ġexplodes":41354,"1960":41355,"Ġpropositions":41356,"Protect":41357,"Ġsexes":41358,"physical":41359,"ĠAthena":41360,"ocent":41361,"ĠGothic":41362,"ĠRacial":41363,"istani":41364,"Ġhelium":41365,"ĠPresumably":41366,"Ġperman":41367,"becue":41368,"ĠHW":41369,"rued":41370,"ĠCNS":41371,"DEP":41372,"ĠManifest":41373,"2500":41374,"ĠMyst":41375,"Economic":41376,"Prot":41377,"Ġledge":41378,"Ġimitate":41379,"ĠTotally":41380,"ĠBeaut":41381,"OIL":41382,"Ġ1440":41383,"Moscow":41384,"ĠSets":41385,"merga":41386,"Ġlesbians":41387,"Walker":41388,"Move":41389,"ĠSOM":41390,"ĠPsy":41391,"strument":41392,"Ġiter":41393,"ĠTosh":41394,"oola":41395,"ĠAntiqu":41396,"ĠShining":41397,"Ġobservational":41398,"VW":41399,"rophe":41400,"034":41401,"Ġcontiguous":41402,"Ġstarve":41403,"sure":41404,"Ġnegate":41405,"Ġmindless":41406,"tf":41407,"Ġdownwards":41408,"046":41409,"riors":41410,"Ġreverted":41411,"ĠAthe":41412,"Bra":41413,"eah":41414,"Rachel":41415,"Hung":41416,"Join":41417,"ĠRaces":41418,"Ġmutant":41419,"Ġuncond":41420,"Ġusability":41421,"NESS":41422,"haust":41423,"036":41424,"Ġobscurity":41425,"Ġimperialism":41426,"Ġemitting":41427,"Ġideologically":41428,"ĠIro":41429,"erva":41430,"ĠIzzy":41431,"ĠLevels":41432,"onym":41433,"ĠConspiracy":41434,"ĠSapphire":41435,"Ul":41436,"Ġhuh":41437,"ochem":41438,"Ġbehaves":41439,"ĠMesh":41440,"Ark":41441,"Ġvec":41442,"ĠActions":41443,"Ġdistinguishing":41444,"ĠTsarnaev":41445,"ĠEndurance":41446,"ederation":41447,"itant":41448,"Ġstreetcar":41449,"041":41450,"ĠAval":41451,"ĠCompanion":41452,"ĠCartoon":41453,"Ġcalculus":41454,"993":41455,"eq":41456,"ĠVanilla":41457,"MAC":41458,"wolves":41459,"fg":41460,"Ġfermentation":41461,"Ġinformants":41462,"Ġsudo":41463,"Ġperipher":41464,"Ġindign":41465,"parts":41466,"detail":41467,"femin":41468,"blade":41469,"Ġinserts":41470,"Ġoffsets":41471,"Ġantidepressants":41472,"Ġphr":41473,"Ġresultant":41474,"biology":41475,"Ġacquies":41476,"UFF":41477,"****************":41478,"ĠPenalty":41479,"Ġrever":41480,"heric":41481,"ĠShadows":41482,"command":41483,"Ġreprint":41484,"089":41485,"empty":41486,"ĠTAG":41487,"stim":41488,"FK":41489,"Ġkins":41490,"uggle":41491,"imura":41492,"wit":41493,"Kill":41494,"Beck":41495,"Ocean":41496,"Ġlabyrinth":41497,"ĠNorse":41498,"IENCE":41499,"Ġ+++":41500,"DoS":41501,"gm":41502,"Ġbarbar":41503,"ĠCeres":41504,"Ġhashing":41505,"eworthy":41506,"Ġrecite":41507,"Ġelectrodes":41508,"Ġconformity":41509,"response":41510,"olate":41511,"Ġ357":41512,"Snap":41513,"Crime":41514,"Ġpointer":41515,"ĠTIT":41516,"Ġdistinctions":41517,"Ġ427":41518,"ĠÙĪ":41519,"abases":41520,"Mars":41521,"ĠSpiritual":41522,"Ġimpuls":41523,"Philadelphia":41524,"1994":41525,"Ġcunning":41526,"Ġfram":41527,"Ġinco":41528,"Ġomnip":41529,"imize":41530,"ervative":41531,"Gy":41532,"Drug":41533,"Ġcarniv":41534,"ĠSailor":41535,"download":41536,"ĠBeetle":41537,"ĠEarthqu":41538,"izontal":41539,"Alan":41540,"Nice":41541,"Prior":41542,"MAG":41543,"Ġautobi":41544,"ĠBrill":41545,"Ġpredominant":41546,"ĠMessiah":41547,"REM":41548,"ĠSlip":41549,"ĠWebs":41550,"ademic":41551,"<":41552,"ĠVessel":41553,"vari":41554,"Code":41555,"Ġbeetle":41556,"projects":41557,"BAT":41558,"Ġpsychotic":41559,"Ġunderside":41560,"Ġrefute":41561,"Considering":41562,"kees":41563,"wd":41564,"priority":41565,"Ġtwentieth":41566,"Ġatheist":41567,"amina":41568,"Ġeuphem":41569,"Ġtripod":41570,"ĠTrayvon":41571,"ĠNON":41572,"2200":41573,"ĠNPC":41574,"ependence":41575,"ĠMHz":41576,"ĠBung":41577,"Ġpane":41578,"Ġaboriginal":41579,"ĠPLUS":41580,"igers":41581,"ĠSexy":41582,"MF":41583,"Chall":41584,"Ay":41585,"ilingual":41586,"adj":41587,"Ġfrown":41588,"successful":41589,"stack":41590,"Ġic":41591,"ĠSeah":41592,"Ġconsequ":41593,"bugs":41594,"ĠScand":41595,"ĠCurve":41596,"Nob":41597,"ĠHoo":41598,"ĠKissinger":41599,"ĠTimeline":41600,"Ġmt":41601,"Description":41602,"YP":41603,"ĠInstallation":41604,"levision":41605,"Ġanthropology":41606,"itzerland":41607,"iaries":41608,"kward":41609,"robat":41610,"Ġcarbohydrate":41611,"Phot":41612,"оÐ":41613,"ĠSQL":41614,"Disc":41615,"Ġdataset":41616,"ynski":41617,"Ġfiat":41618,"ĠDres":41619,"ĠFavor":41620,"ĠHalls":41621,"Alt":41622,"PART":41623,"Spider":41624,"Ġdisabling":41625,"RG":41626,"Ward":41627,"aturation":41628,"Ġwillfully":41629,"Ġlockout":41630,"ĠShutdown":41631,"956":41632,"Ġcommunists":41633,"Against":41634,"Ore":41635,"ĠRik":41636,"ĠASD":41637,"ĠOnion":41638,"Ġparticulars":41639,"Analy":41640,"checked":41641,"selected":41642,"romy":41643,"ĠAkira":41644,"Ġcongr":41645,"Choice":41646,"Ġbos":41647,"organisms":41648,"Ġfrowned":41649,"Tok":41650,"Bir":41651,"ĠScrib":41652,"Ġrealms":41653,"Ġcoercive":41654,"1993":41655,"021":41656,"âĢĵâĢĵ":41657,"athetic":41658,"rior":41659,"Ġfolly":41660,"ĠAMERICA":41661,"Ġcassette":41662,"953":41663,"Ġabsorbs":41664,"043":41665,"quad":41666,"''.":41667,"ĠExtract":41668,"Ġ424":41669,"Whit":41670,"Dun":41671,"Ġexerted":41672,"Ġbrethren":41673,"ĠChronicles":41674,"eric":41675,"Mot":41676,"Ġendings":41677,"piration":41678,"Ġpredetermined":41679,"ĠAirl":41680,"Ġgasp":41681,"Ġ367":41682,"Ġexclaim":41683,"cation":41684,"sort":41685,"idden":41686,"missive":41687,"ع":41688,"oice":41689,"same":41690,"Ott":41691,"Ġscatter":41692,"Flight":41693,"ĠTOD":41694,"Stra":41695,"amia":41696,"IZE":41697,"Ġcompressor":41698,"ixels":41699,"lethal":41700,"ĠExperimental":41701,"Ing":41702,"knife":41703,"Ġvanishing":41704,"ĠRequired":41705,"Stat":41706,"ĠPlex":41707,"spection":41708,"ĠBakr":41709,"Amazing":41710,"Ġbreaths":41711,"rots":41712,"OSP":41713,"Ġ840":41714,"Wars":41715,"OGR":41716,"Ġ372":41717,"ĠKhe":41718,"inous":41719,"lightly":41720,"ĠRounds":41721,"Ġrefinement":41722,"property":41723,"Ġmetaph":41724,"oultry":41725,"istor":41726,"Ġintestine":41727,"eus":41728,"ĠWilhelm":41729,"ĠBane":41730,"emption":41731,"oubtedly":41732,"ĠVirtue":41733,"'),":41734,"Ħ¢":41735,"Ġappar":41736,"ĠTranslation":41737,"Quite":41738,"Ġphysicists":41739,"Ġpriesthood":41740,"Ġallowable":41741,"Saint":41742,"OSED":41743,"bind":41744,"Ġtorches":41745,"osexual":41746,"Cruz":41747,"ertility":41748,"ĠAES":41749,"Ġascended":41750,"Ġmuzzle":41751,"Ġelectors":41752,"ĠKrug":41753,"Ġcc":41754,"classic":41755,"ĠMace":41756,"Å«":41757,"ĠâĢ¦\"":41758,"ĠTEST":41759,"gomery":41760,"Person":41761,"Ġtranslations":41762,"ĠDys":41763,"ĠConsent":41764,"Ġ361":41765,"alos":41766,"Ġallerg":41767,"ĠWast":41768,"ĠChecks":41769,"cerning":41770,"Ġlizard":41771,"Ġrevolutions":41772,"Ġtether":41773,"Ġminimized":41774,"ĠReverse":41775,"itely":41776,"iguous":41777,"athing":41778,"Flow":41779,"Moving":41780,"Ġ409":41781,"047":41782,"Ġsnug":41783,"Nich":41784,"Ġcartridge":41785,"YL":41786,"Ġforwarding":41787,"umerous":41788,"ĠAbedin":41789,"iolet":41790,"tick":41791,"ĠTransform":41792,"Grant":41793,"Ġsubtitles":41794,"ĠEmin":41795,"ghost":41796,"ĠKurd":41797,"Ġfireball":41798,"compatible":41799,"Ġprojectiles":41800,"amorph":41801,"ĠSatisf":41802,"Ġquirks":41803,"Ġrecept":41804,"spective":41805,"Ġgraphical":41806,"ĠPicard":41807,"ĠAuthent":41808,"ĠSponge":41809,"Army":41810,"ĠLumin":41811,"ĠSOME":41812,"Ġsolitude":41813,"ĠSHOULD":41814,"ĠFasc":41815,"opez":41816,"types":41817,"gallery":41818,"OLOGY":41819,"shake":41820,"Ġ369":41821,"Ġreused":41822,"Ġ378":41823,"Ġexorc":41824,"Ġdocs":41825,"Yu":41826,"ĠGOD":41827,"ocrine":41828,"location":41829,"fif":41830,"Grid":41831,"Ġpowd":41832,"Ġ'[":41833,"Ġposterior":41834,"Thompson":41835,"Table":41836,"oslov":41837,"ĠGoddess":41838,"odon":41839,"ĠSTD":41840,"Ġresponsiveness":41841,"stab":41842,"absolute":41843,"Enough":41844,"ĠEssence":41845,"ĠUpgrade":41846,"hematically":41847,"Subscribe":41848,"alsh":41849,"repl":41850,"Ġselector":41851,"ĠLength":41852,"Ġtemporal":41853,"Tele":41854,"ocalyptic":41855,"ĠDeaths":41856,"rl":41857,"Target":41858,"ĠOrn":41859,"ongh":41860,"Ġ1909":41861,"Quest":41862,"Place":41863,"ĠDisabled":41864,"Ġascending":41865,"giene":41866,"ĠMSI":41867,"ivil":41868,"Ġcaval":41869,"Ġintermitt":41870,"Ġsalts":41871,"Apr":41872,"059":41873,"ĠKeeper":41874,"emis":41875,"ĠEternal":41876,"SER":41877,"estones":41878,"Ġrudimentary":41879,"Ġpooled":41880,"ĠAlright":41881,"Ġdiagrams":41882,"ydia":41883,"Jacob":41884,"Ġarchitectures":41885,"ĠUSPS":41886,"Ġfootnote":41887,"ĠBrav":41888,"ĠLeopard":41889,"Ġvirtuous":41890,"ploma":41891,"ĠHIP":41892,"Ġhorizontally":41893,"olith":41894,"Prop":41895,"ĠApocalypse":41896,"Syria":41897,"ĠShowdown":41898,"constitutional":41899,"Independent":41900,"ĠMiliband":41901,"ĠTracks":41902,"adle":41903,"ĠESL":41904,"ĠFIGHT":41905,"Ġjohn":41906,"é":41907,"benef":41908,"eware":41909,"ĠTABLE":41910,"ĠVeg":41911,"ainers":41912,"Ġresolves":41913,"Warren":41914,"ĠRanked":41915,"possibly":41916,"bian":41917,"simple":41918,"Ġuniformly":41919,"ĠSlash":41920,"otton":41921,"ĠAbsent":41922,"agically":41923,"ĠPieces":41924,"Station":41925,"ĠBeware":41926,"ĠDiscrimination":41927,"Ġponies":41928,"Import":41929,"utory":41930,"ĠParas":41931,"Phoenix":41932,"Lat":41933,"UTC":41934,"push":41935,"astically":41936,"urrent":41937,"untarily":41938,"Ġparanormal":41939,"Ġglanced":41940,"Ġmanifestations":41941,"ĠNeuroscience":41942,"irgin":41943,"ROM":41944,"Ġ($)":41945,"Ġ379":41946,"missing":41947,"Ġmercenaries":41948,"Ġenumer":41949,"ĠShant":41950,"Ws":41951,"wered":41952,"Ġbuffs":41953,"ultane":41954,"ĠRohing":41955,"igger":41956,"Ring":41957,"Ġmanifests":41958,"Fat":41959,"ĠReduced":41960,"ĠMinerva":41961,"uart":41962,"ĠArmory":41963,"orange":41964,"igible":41965,"Ġphysiology":41966,"Ut":41967,"Ġparchment":41968,"ĠFired":41969,"trap":41970,"oggle":41971,"mson":41972,"ĠPoster":41973,"Ġbount":41974,"import":41975,"maximum":41976,"Ġ422":41977,"ĠFemin":41978,"Ġnodding":41979,"Ġinscription":41980,"Results":41981,"GRE":41982,"icative":41983,"Ġcognition":41984,"Ġions":41985,"ĠBite":41986,"Ġneutron":41987,"Ġduplication":41988,"ĠZIP":41989,"ĠQuit":41990,"Ġgrasping":41991,"ĠDaylight":41992,"Ġlayouts":41993,"CLA":41994,"reason":41995,"ĠHuh":41996,"Ġpige":41997,"ĠBomber":41998,"Produ":41999,"Ġgland":42000,"ĠAbsolute":42001,"writ":42002,"Ġmassac":42003,"Ġfixation":42004,"device":42005,"yz":42006,"ĠGOT":42007,"ĠDying":42008,"adjust":42009,"grain":42010,"Ġdeform":42011,"Ġtypew":42012,"Ġdagger":42013,"ĠTuring":42014,"ĠBucc":42015,"Heavy":42016,"Ġcommod":42017,"files":42018,"ogeneous":42019,"roth":42020,"Buff":42021,"Ġbookmark":42022,"porary":42023,"Medical":42024,"Um":42025,"Ġtranslucent":42026,"ĠAnxiety":42027,"ĠCorinthians":42028,"optional":42029,"PUT":42030,"Ġcrucifix":42031,"alloween":42032,"ĠVK":42033,"Ġblu":42034,"ĠCorinth":42035,"Mount":42036,"Ġmembranes":42037,"particip":42038,"Ġextraord":42039,"Ġstimulated":42040,"leneck":42041,"Ġspecifies":42042,"Sin":42043,"lash":42044,"Edited":42045,"Ġfused":42046,"Nin":42047,"ĠBungie":42048,"ĠTooth":42049,"WATCH":42050,"Nav":42051,"Initially":42052,"+)":42053,"ĠAncest":42054,"Ġtransmitter":42055,"ĠVolks":42056,"ezvous":42057,"ĠNirvana":42058,"ĠCald":42059,"font":42060,"Und":42061,"remlin":42062,"ichever":42063,"ĠHeal":42064,"shall":42065,"Ġattribution":42066,"authorized":42067,"ĠINTO":42068,"acteria":42069,"ĠTsu":42070,"ĠPlane":42071,"iphate":42072,"igraph":42073,"chev":42074,"Ġinverse":42075,"ifest":42076,"Players":42077,"!!\"":42078,"ĠContrast":42079,"1984":42080,"Ġsevent":42081,"colour":42082,"ĠRational":42083,"virtual":42084,"Ġfec":42085,"ĠETH":42086,"ĠPru":42087,"Õ":42088,"asma":42089,"Cur":42090,"Ġassigns":42091,"Ġridic":42092,"Todd":42093,"ulton":42094,"ĠDefendant":42095,"opsis":42096,"Ġpercentile":42097,"shr":42098,"wagen":42099,"Ġ368":42100,"SIGN":42101,"Screen":42102,"reprene":42103,"Ġerection":42104,"ĠFreak":42105,"ĠStard":42106,"stained":42107,"Ġcla":42108,"fet":42109,"ramids":42110,"QL":42111,"avorable":42112,"ĠTCP":42113,"nown":42114,"ulence":42115,"similar":42116,"Ġlinkage":42117,"ercise":42118,"Path":42119,"LECT":42120,"ĠCollections":42121,"ĠModule":42122,"Ġcs":42123,"Current":42124,"Ġmono":42125,"ĠAlv":42126,"ĠDude":42127,"Ġhypers":42128,"Ġ2600":42129,"surface":42130,"Ġpredictor":42131,"ĠColomb":42132,"Prof":42133,"anqu":42134,"natal":42135,"Ġadultery":42136,"ĠGenerations":42137,"clerosis":42138,"Ġ371":42139,"Ġenlightenment":42140,"onomic":42141,"Ġsatir":42142,"ĠBasics":42143,"Graham":42144,"ĠRove":42145,"Ġadul":42146,"Shut":42147,"ocious":42148,"Ġhandc":42149,"BW":42150,"ĠCognitive":42151,"visible":42152,"Ġinev":42153,"Ġ978":42154,"ĠSupported":42155,"Ġarrays":42156,"Ġalienation":42157,"Weight":42158,"ĠkWh":42159,"Ġwarped":42160,"Ġ386":42161,"lance":42162,"Ġherpes":42163,"ĠPHP":42164,"Ġclaimant":42165,"uitive":42166,"Ġpussy":42167,"Ġcorpus":42168,"ĠAo":42169,"Qual":42170,"ĠXVI":42171,"requ":42172,"Ġsympt":42173,"mination":42174,"Ġhairy":42175,"ĠBattles":42176,"owntown":42177,"Roberts":42178,"Ġnec":42179,"ablo":42180,"AMD":42181,"internet":42182,"Tar":42183,"direction":42184,"ouston":42185,"ĠGlock":42186,"ĠYanukovych":42187,"ogens":42188,"rogram":42189,"otype":42190,"ĠPt":42191,"tenance":42192,"Ġaromatic":42193,"oxin":42194,"Vert":42195,"Ġsociop":42196,"cible":42197,"Db":42198,"________________":42199,"Third":42200,"ĠShips":42201,"!.":42202,"expensive":42203,"WOR":42204,"primary":42205,"Ġ666":42206,"Ġdecaying":42207,"Ġclustered":42208,"Ġbeetles":42209,"ĠHogwarts":42210,"Ġheaders":42211,"ĠJudah":42212,"Ġscen":42213,"Ġcosmos":42214,"ĠGenetic":42215,"blems":42216,"Ġfeeble":42217,"NOW":42218,"NSA":42219,"Ġadminist":42220,"ĠDocker":42221,"portion":42222,"gression":42223,"Ġ1904":42224,"heard":42225,"Ġinhab":42226,"ĠLeaves":42227,"Ġcortisol":42228,"atinum":42229,"unknown":42230,"ĠObserv":42231,"ĠPhilosophy":42232,"Ide":42233,"Ġcopyrighted":42234,"surv":42235,"ĠLocations":42236,"Ġglands":42237,"ĠKnife":42238,"ĠEmber":42239,"ĠUnicorn":42240,"Ġhaste":42241,"Ġkinderg":42242,"ĠTerrit":42243,"ĠKoran":42244,"Ġaval":42245,"addon":42246,"ĠNero":42247,"\"]":42248,"Ġ392":42249,"comfort":42250,"Ġclothed":42251,"ashtra":42252,"mode":42253,"Ġ??":42254,"!\",":42255,"Ġknob":42256,"EMP":42257,"norm":42258,"ĠAgo":42259,"RECT":42260,"Denver":42261,"Ġ1907":42262,"ĠBombs":42263,"Sche":42264,"Ġtriangular":42265,"Ġperv":42266,"rises":42267,"Jes":42268,"Ġcalibration":42269,"Ġts":42270,"Same":42271,"ĠAxe":42272,"ĠMei":42273,"multi":42274,"Ġexerc":42275,"orney":42276,"Ware":42277,"abul":42278,"ĠFior":42279,"Eventually":42280,"ĠGrizz":42281,"Past":42282,"married":42283,"Ġscram":42284,"ĠCache":42285,"posure":42286,"Ġheav":42287,"ĠShirt":42288,"powder":42289,"complex":42290,"Doc":42291,"arus":42292,"Pi":42293,"Ġcurv":42294,"ĠTopic":42295,"Ġ.)":42296,"Ġwills":42297,"philis":42298,"gui":42299,"leground":42300,"Eth":42301,"Strike":42302,"Kid":42303,"Ġdelegated":42304,"Soon":42305,"Ġwast":42306,"gage":42307,"Ġprosecut":42308,"Ġ374":42309,"opolis":42310,"chest":42311,"ensation":42312,"Ġredes":42313,"Ġpresum":42314,"Portland":42315,"Ġannihil":42316,"yssey":42317,"Ġforks":42318,"Ġvitro":42319,"walker":42320,"ĠPsal":42321,"ĠStealth":42322,"Quick":42323,"ĠBaghd":42324,"ĠDrift":42325,"//":42326,"Ġinvincible":42327,"ĠGAM":42328,"Ġcastles":42329,"Ġbondage":42330,"ĠBalloon":42331,"Amid":42332,"individual":42333,"tis":42334,"ĠGuides":42335,"xe":42336,"Cong":42337,"URI":42338,"ĠHH":42339,"PHOTOS":42340,"ĠASIC":42341,"burst":42342,"ahon":42343,"ĠFIX":42344,"ilib":42345,"Ġ457":42346,"ĠLogged":42347,"à¹":42348,"Creat":42349,"inatory":42350,"column":42351,"ĠAugustus":42352,"suggest":42353,"pret":42354,"ĠParan":42355,"Ġsubsistence":42356,"wx":42357,"×":42358,"aleigh":42359,"dash":42360,"ĠMana":42361,"Ko":42362,"opausal":42363,"Ġbene":42364,"ĠSabb":42365,"ĠGhosts":42366,"Ġ1830":42367,"ĠHats":42368,"ĠHive":42369,"Perfect":42370,"Ġsocialists":42371,"Ġtumult":42372,"EGA":42373,"ĠNAME":42374,"Android":42375,"assembled":42376,"phis":42377,"Stage":42378,"Char":42379,"Double":42380,"Ġinsign":42381,"IED":42382,"perial":42383,"ĠEMP":42384,"mx":42385,"Ġskept":42386,"Ġwifi":42387,"Ġparad":42388,"ĠFrequency":42389,"Dist":42390,"nil":42391,"iots":42392,"å":42393,"Message":42394,"Furthermore":42395,"Ġhideous":42396,"ĠLDL":42397,"ĠFault":42398,"ĠDimensions":42399,"ĠImplement":42400,"fram":42401,"Ġamaz":42402,"ĠIndones":42403,"ĠTile":42404,"Ġlar":42405,"gc":42406,"Ġcorrelate":42407,"Ġensl":42408,"mite":42409,"Ġhomosexuals":42410,"Ġagric":42411,"8000":42412,"Ġcuring":42413,"rament":42414,"Ġrecons":42415,"ocene":42416,"ENTION":42417,"Ġcommunion":42418,"ĠFunction":42419,"iple":42420,"Ġredund":42421,"Ġcalibrated":42422,"Ġcontribut":42423,"ĠHuck":42424,"limit":42425,"ĠFedora":42426,"ĠTsuk":42427,"brates":42428,"Ġ1903":42429,"ozo":42430,"visual":42431,"ĠDiscipline":42432,"chains":42433,"ĠOCD":42434,"Ġexpended":42435,"0002":42436,"Ġsty":42437,"ĠNightmare":42438,"ĠReplace":42439,"ounty":42440,"fn":42441,"1900":42442,"ĠEpidem":42443,"ĠFW":42444,"Ġgul":42445,"ĠTomato":42446,"ĠPerse":42447,"wl":42448,"ĠFormation":42449,"Scan":42450,"cosystem":42451,"Brand":42452,"Ġ398":42453,"Ġcaptives":42454,"Ġ×":42455,"ESCO":42456,"ĠEnder":42457,"lesh":42458,"ĠAscend":42459,"poly":42460,"eous":42461,"Ġhyster":42462,"Murray":42463,"phe":42464,"Ġradiator":42465,"esthes":42466,"Ġopin":42467,"Ġconspic":42468,"intosh":42469,"Ġwitchcraft":42470,"ĠCFR":42471,"ussian":42472,"escent":42473,"locking":42474,"Ġnonsensical":42475,"uala":42476,"ĠSerial":42477,"1991":42478,"ĠCalm":42479,"containing":42480,"Ġstimulates":42481,"Ġ448":42482,"Pir":42483,"ĠâĨĴ":42484,"ĠDiver":42485,"Ġmanuscripts":42486,"ĠGaia":42487,"Ñĥ":42488,"Learning":42489,"Ġnipple":42490,"reads":42491,"Ġandroid":42492,"ĠMeditation":42493,"Ġincomprehensible":42494,"edded":42495,"Ġdescendant":42496,"ĠMorty":42497,"Luckily":42498,"ARCH":42499,"ausible":42500,"Dig":42501,"shared":42502,"ĠClip":42503,"Ġtrope":42504,"Ġnarcissistic":42505,"ventures":42506,"Ġcuriously":42507,"ĠCosmos":42508,"Aust":42509,"Lay":42510,"ĠShard":42511,"ĠRecorded":42512,"Ġ458":42513,"........":42514,"Ġperish":42515,"ĠExample":42516,"luent":42517,"Ġapes":42518,"ĠHitch":42519,"Ġholiest":42520,"Ġamplifier":42521,"minent":42522,"xxxxxxxx":42523,"inite":42524,"Ġgenomes":42525,"ĠGuilty":42526,"mult":42527,"Ġorc":42528,"Ġnipples":42529,"Side":42530,"Ġlogically":42531,"Ġdatasets":42532,"ĠTitanium":42533,"Ġrotor":42534,"undle":42535,"handled":42536,"nexpected":42537,"Ġdw":42538,"Ġdiagonal":42539,"ĠAnimated":42540,"Ġnumbering":42541,"Forest":42542,"ĠâĨ":42543,"Prin":42544,"Ġchemically":42545,"ĠGithub":42546,"Ġaph":42547,"ĠFaster":42548,"ĠTinker":42549,"ikini":42550,"Dest":42551,"dri":42552,"Manufact":42553,"isance":42554,"Return":42555,"Alert":42556,"elcome":42557,"ĠMMR":42558,"Ġresid":42559,"ĠLIC":42560,"Ġspecificity":42561,"zanne":42562,"Ġanyways":42563,"Ġ426":42564,"Scot":42565,"astery":42566,"Via":42567,"ĠBlocks":42568,"Ġactivates":42569,"Ġabstinence":42570,"Ġchronological":42571,"Soul":42572,"ĠSchne":42573,"Ġwatts":42574,"AUT":42575,"Ġcalcul":42576,"Simply":42577,"Emb":42578,"ceptive":42579,"ĠCatholicism":42580,"obook":42581,"ĠBits":42582,"ĠMbps":42583,"Ġindignation":42584,"Ġshorthand":42585,"Active":42586,"ĠLimbaugh":42587,"ĠCapcom":42588,"adesh":42589,"Ġclipping":42590,"ĠInstructor":42591,"Secret":42592,"___":42593,"Fer":42594,"rawling":42595,"ĠReward":42596,"Ġweep":42597,"Ġmotherboard":42598,"Above":42599,"metry":42600,"ĠPTS":42601,"Ġbombard":42602,"abetes":42603,".--":42604,"Lens":42605,"Comb":42606,"basic":42607,"ĠREALLY":42608,"Later":42609,"Ġ383":42610,"Ġpositional":42611,"olesc":42612,"Ġcrotch":42613,"ĠMDMA":42614,"requently":42615,"ĠPants":42616,"Ġ433":42617,"uctor":42618,"Ġillumination":42619,"ĠÙħ":42620,"ocrin":42621,"Ġpamph":42622,"atio":42623,"etc":42624,"Ġrestores":42625,"ĠProtector":42626,"Develop":42627,"ĠMew":42628,"trop":42629,"ĠSlayer":42630,"Ti":42631,"ĠNotwithstanding":42632,"Match":42633,"LIST":42634,"IDES":42635,"ĠThick":42636,"Ġdisks":42637,"Kin":42638,"Ġghetto":42639,"ĠObjects":42640,"Ġprism":42641,"ĠNether":42642,"Ġvul":42643,"iky":42644,"]:":42645,"ĠDetail":42646,"Ġfucked":42647,"!?":42648,"anium":42649,"Ġlords":42650,"ilities":42651,"ĠEthnic":42652,"static":42653,"$$":42654,"evidence":42655,"Ġmainline":42656,"Ġpeasant":42657,"ĠEnhance":42658,"ĠForced":42659,"virt":42660,"Ġii":42661,"Ġsymm":42662,"Ġconverter":42663,"ularity":42664,"Ġrepent":42665,"num":42666,"ĠScrew":42667,"ĠFTA":42668,"Ġmarines":42669,"hetto":42670,"blow":42671,"Ġado":42672,"ĠTypical":42673,"Ġoverw":42674,"ĠBerm":42675,"keley":42676,"Song":42677,"hao":42678,"valid":42679,"EXT":42680,"ĠProvides":42681,"âĺħâĺħ":42682,"ĠOdin":42683,"Shot":42684,"Ġgamma":42685,"Princ":42686,"asonry":42687,"ĠAccuracy":42688,"Ġcriterion":42689,"Ġdescriptive":42690,"Gall":42691,"gray":42692,"ĠCalcul":42693,"Ġaxes":42694,"ĠCommunists":42695,"ĠRebellion":42696,"Success":42697,"tg":42698,"Ġâĺ":42699,"Ġmultiplier":42700,"ravity":42701,"Thus":42702,"URL":42703,"Ġalternatively":42704,"duction":42705,"Ġsarcast":42706,"ĠCarth":42707,"ĠUSL":42708,"ĠInvisible":42709,"larg":42710,"pleted":42711,"pathic":42712,"Additionally":42713,"ĠCao":42714,"Ġlatent":42715,"ĠSurge":42716,"MEN":42717,"communications":42718,"ĠArray":42719,"Pink":42720,"commit":42721,"isodes":42722,"earcher":42723,"Ukraine":42724,"ĠAnthrop":42725,"incial":42726,"Ġquotations":42727,"adena":42728,"Ġwhining":42729,"Ġretri":42730,"ĠAssass":42731,"elligent":42732,"ĠPERSON":42733,"Py":42734,"Send":42735,"ĠâĪĴ":42736,"DON":42737,"Ġwatt":42738,"description":42739,"POS":42740,"Ġrepro":42741,"destroy":42742,"icidal":42743,"Ġmidrange":42744,"Ġinfographic":42745,"interesting":42746,"category":42747,"Flash":42748,"ĠInvasion":42749,"ĠExodus":42750,"restricted":42751,"Ġinference":42752,"dding":42753,"mingham":42754,"Ġcircumst":42755,"Wi":42756,"ĠHast":42757,"Ġsubjug":42758,"Ġwhispering":42759,"-.":42760,"Ġadren":42761,"ĠPattern":42762,"BOX":42763,"ĠEnhancement":42764,"Exc":42765,"ĠBucket":42766,"ĠGUN":42767,"deen":42768,"ĠHomo":42769,"1985":42770,"Ġclo":42771,"Ġsnippet":42772,"Ġ1896":42773,"TPP":42774,"Seg":42775,"success":42776,";\"":42777,"ĠMUCH":42778,"Author":42779,"Ġreplication":42780,"Ġhallucinations":42781,"Inv":42782,"ĠAware":42783,"ĠViper":42784,"kai":42785,"frames":42786,"ĠTHANK":42787,"ĠSHA":42788,"wordpress":42789,"Ġbc":42790,"CIA":42791,"arrison":42792,"Ġalloc":42793,"ĠAlz":42794,"letcher":42795,"ĠDaredevil":42796,"iversary":42797,"Ġmanuals":42798,"Catholic":42799,"feat":42800,"Ġkinetic":42801,"JB":42802,"yeah":42803,"ĠLDS":42804,"Ġppm":42805,"ĠADC":42806,"pring":42807,"cence":42808,"Ġclasp":42809,"Ġsetups":42810,"Ġdeity":42811,"ĠIndra":42812,"ĠWander":42813,"Ġantib":42814,"Otherwise":42815,"ombie":42816,"Bitcoin":42817,"ipop":42818,"expression":42819,"Animal":42820,"ĠResurrection":42821,"ĠMoral":42822,"ĠSDK":42823,"Ġwretched":42824,"ogenous":42825,"species":42826,"Ġchuckled":42827,"Thor":42828,"Ġ428":42829,"avery":42830,"ĠPry":42831,"asures":42832,"ĠErn":42833,"apor":42834,"Ġinnumerable":42835,"Ġbaptized":42836,"ĠExplosive":42837,"Ġelves":42838,"idges":42839,"ĠParadox":42840,"Close":42841,"aldehyde":42842,"construct":42843,"Ġvirginity":42844,"Poll":42845,"assin":42846,"Doctors":42847,"Pos":42848,"NECT":42849,"Moreover":42850,"Commercial":42851,"cknowled":42852,"1988":42853,"Ġquotation":42854,"marriage":42855,"ĠBapt":42856,"ĠSina":42857,"ĠGloves":42858,"gian":42859,"Ġconfounding":42860,"URRENT":42861,"Dean":42862,"Brew":42863,"thur":42864,"pty":42865,"immune":42866,"ĠSQU":42867,"Ġcounterfe":42868,"rider":42869,"Ġinferred":42870,"ĠDimension":42871,"ĠToad":42872,"Ġafterlife":42873,"ĠHERO":42874,"Indiana":42875,"seek":42876,"Ġdistinguishes":42877,"ĠQur":42878,"ĠMethods":42879,"combat":42880,"Ġcateg":42881,"ĠStruggle":42882,"teness":42883,"liquid":42884,"Ġblinking":42885,"ĠCONTIN":42886,"iae":42887,"Ġaerobic":42888,"Ġstrugg":42889,"Ġegalitarian":42890,"hello":42891,"orrect":42892,"ĠAbandon":42893,"Ġferment":42894,"Area":42895,"idem":42896,"ĠMania":42897,"Ġjs":42898,"ĠBALL":42899,"Running":42900,"Ġregenerate":42901,"iquid":42902,"Uh":42903,"Crystal":42904,"ĠItal":42905,"ĠHeavenly":42906,"в":42907,"CRIPTION":42908,"Consumer":42909,"dust":42910,"amiliar":42911,"ĠRhino":42912,"Rocket":42913,"Ġreversible":42914,"kok":42915,"ĠSketch":42916,"Ġshotguns":42917,"apses":42918,"Ġdetach":42919,"ĠCells":42920,"artist":42921,"rily":42922,"ĠRestore":42923,"Scar":42924,"Ġevid":42925,"Ġspaced":42926,"ĠContributions":42927,"Ġ418":42928,"ĠMystic":42929,"Ġobfusc":42930,"Russ":42931,"wings":42932,"Pear":42933,"osite":42934,"Nusra":42935,"urations":42936,"ovie":42937,"icago":42938,"ĠConcepts":42939,"Ġstimuli":42940,"Ġaroused":42941,"aughty":42942,"Talking":42943,"ĠPrompt":42944,"Across":42945,"ĠPlaint":42946,"Ġbranching":42947,"Thankfully":42948,"Original":42949,"Esc":42950,"ĠTechnician":42951,"fleet":42952,"usher":42953,"Mos":42954,"livion":42955,"oenix":42956,"Ġhr":42957,"ibble":42958,"Ġindent":42959,"ĠFinished":42960,"Department":42961,"ĠINFO":42962,"Movie":42963,"++":42964,"THING":42965,"Ġtimers":42966,"rocket":42967,"Natural":42968,"lime":42969,"Ġangular":42970,"osure":42971,"Ġdynamically":42972,"Ġpacif":42973,"ĠProcessor":42974,"Ġdisgu":42975,"Ġmoderators":42976,"Ġceases":42977,"Ġinertia":42978,"Ġpaperback":42979,"yton":42980,"ĠHuma":42981,"Ġprohibitions":42982,"Ġgestation":42983,"Bomb":42984,"termin":42985,"Ġcaric":42986,"oS":42987,"tc":42988,"Cop":42989,"raved":42990,"Ġeighty":42991,"ĠEnable":42992,"Ġimplementations":42993,"Ġconquering":42994,"ĠFinder":42995,"window":42996,"Gra":42997,"Ġfonts":42998,"laughter":42999,"Ġcolonization":43000,"ĠDOD":43001,")!":43002,",)":43003,"ĠGeral":43004,"ĠSpoiler":43005,"ĠComponent":43006,"Ġgist":43007,"hiro":43008,"Ġlicens":43009,"nesses":43010,"Ġkarma":43011,"?\".":43012,"OPA":43013,"Ġsquats":43014,"ĠRAND":43015,"Ġorally":43016,"document":43017,"olars":43018,"Ġpresumptive":43019,"Pers":43020,"OAD":43021,"ufficient":43022,"LESS":43023,"Hidden":43024,"ORK":43025,"xs":43026,"Ġmathematician":43027,"ĠGloss":43028,"Ġannihilation":43029,"Ġmanifold":43030,"Ry":43031,"Thunder":43032,"Yan":43033,"Activ":43034,"Ġworldly":43035,"TED":43036,"marg":43037,"ĠStun":43038,"ryce":43039,"ĠVG":43040,"Isn":43041,"ĠCyn":43042,"Expl":43043,"IRED":43044,"Ġcompr":43045,"Ġindisc":43046,"Boss":43047,"()":43048,"berman":43049,"ĠBegins":43050,"ujah":43051,"ornia":43052,"hetical":43053,"Ġcivilizations":43054,"Ġfundamentalist":43055,"strap":43056,"Forward":43057,"ettlement":43058,"Ġprophetic":43059,"glers":43060,"bending":43061,"Terry":43062,"Ġidi":43063,"Ġtrunc":43064,"Ġcreeps":43065,"intel":43066,"switch":43067,"ailand":43068,"Ġinstaller":43069,"GOP":43070,"Ġ499":43071,"ĠParallel":43072,"Cru":43073,"Ġ\"@":43074,"Ġ396":43075,"ĠUnlock":43076,"Raven":43077,"Corn":43078,"Ġcircadian":43079,"Ġ********************************":43080,"iliate":43081,"ĠFunctional":43082,"Ġpronouns":43083,"ĠSatoshi":43084,"Ġstim":43085,"Gay":43086,"Iss":43087,"ĠThief":43088,"atellite":43089,"Ġshards":43090,"Ġphil":43091,"protein":43092,"Ġalters":43093,"Poor":43094,"Typically":43095,"KER":43096,"ociate":43097,"Ġemits":43098,"recy":43099,"Ġmechanically":43100,"Ġ...\"":43101,"nature":43102,"sys":43103,"ysc":43104,"Ġwavelengths":43105,"pattern":43106,"insured":43107,"Ġparasitic":43108,"ĠLCS":43109,"ĠPACs":43110,"Ġheals":43111,"ĠCCP":43112,"ĠHacker":43113,"Ġpsy":43114,"ĠBeans":43115,"Ġdemonic":43116,"JV":43117,"Ġatmosp":43118,"equality":43119,"Ġairst":43120,"Ġincarn":43121,"ynthesis":43122,"Ġequations":43123,"tch":43124,"ĠHUGE":43125,"ĠChanged":43126,"itatively":43127,"Job":43128,"gaming":43129,"Ġ1899":43130,"ĠMorsi":43131,"Ġconjecture":43132,"riad":43133,"Ġprimates":43134,"ĠArtemis":43135,"ĠThro":43136,"Ġbiologically":43137,"Church":43138,"topia":43139,"recomm":43140,"Ġgradient":43141,"Ġful":43142,"Ġbastard":43143,"CHO":43144,"IUM":43145,"sleep":43146,"Construction":43147,"raints":43148,"vable":43149,"ionage":43150,"Ġcomrade":43151,"Ġpopulate":43152,"Ġnerds":43153,"ĠXie":43154,"result":43155,"ĠImper":43156,"Ġpamphlet":43157,"Ku":43158,"Ġbackend":43159,"ificent":43160,"etus":43161,"Ġdisson":43162,"config":43163,"Ġsuc":43164,"Ġwavelength":43165,"external":43166,"owder":43167,"Ġpredis":43168,"eenth":43169,"Det":43170,"andem":43171,"Ġ1865":43172,"ĠDefeat":43173,"Individual":43174,"Ġretrieving":43175,"stories":43176,"Ġdesolate":43177,"Ġlett":43178,"Ġunpublished":43179,"Ġpassively":43180,"Ġdissertation":43181,"raits":43182,"abee":43183,"ĠResist":43184,"Robin":43185,"Ġbenevolent":43186,"blast":43187,"Offic":43188,"snap":43189,"vernment":43190,"Ġextermin":43191,"wt":43192,"bitious":43193,"hibited":43194,"Insp":43195,"posted":43196,"ĠYugoslav":43197,"rational":43198,"adapt":43199,"ĠAtari":43200,"Ġplugin":43201,"oglobin":43202,"efeated":43203,"ĠHRC":43204,"cko":43205,"ilver":43206,"ĠDestruction":43207,"gewater":43208,"ĠRadiation":43209,"Ġimprison":43210,"origin":43211,"antine":43212,"ĠPublication":43213,"Ġhealer":43214,"istered":43215,"ĠTHEIR":43216,"hazard":43217,"Contract":43218,"Ġmediated":43219,"Ġindexed":43220,"ĠSYSTEM":43221,"Labor":43222,"Blade":43223,"Ġyog":43224,"Champ":43225,"Gordon":43226,"IAS":43227,"Ġnineteenth":43228,"animous":43229,"begin":43230,"ĠHolo":43231,"Planet":43232,"udding":43233,"default":43234,"ĠOMG":43235,"Ġwond":43236,"wm":43237,"pend":43238,"Extreme":43239,"Ġinterstellar":43240,"ASED":43241,"ĠBerks":43242,"Ġprimal":43243,"Foot":43244,"Ġinadvert":43245,"amboo":43246,"ĠLeica":43247,"Events":43248,"ĠPigs":43249,"RAFT":43250,"ï":43251,"ĠGentleman":43252,"Multiple":43253,"ĠPsychiatric":43254,"Ġdespise":43255,"ĠZionism":43256,"ĠSSL":43257,"shit":43258,"Ġthreaded":43259,"Ġartifact":43260,"Ġmitochondrial":43261,"ĠLayer":43262,"inus":43263,"podcast":43264,"Ġawaken":43265,"Management":43266,"Ġdelusions":43267,"grey":43268,"Ġpseud":43269,"agonal":43270,"ĠHirosh":43271,"Georg":43272,"Dragon":43273,"Stack":43274,"ohm":43275,"Ġvener":43276,"Row":43277,"Ġsandbox":43278,"Ġblinding":43279,"razen":43280,"Ġ389":43281,"Ġcrappy":43282,"Ġlith":43283,"antha":43284,"Ġplurality":43285,"ĠDAC":43286,"inently":43287,"intage":43288,"Ġ1902":43289,"ĠDepend":43290,"Ġelapsed":43291,"==":43292,"ĠGenie":43293,"Bush":43294,"ĠPlanetary":43295,"Bah":43296,"ĠKira":43297,"emn":43298,"Month":43299,"allic":43300,"coded":43301,"VOL":43302,"Ġ[...]":43303,"ĠRampage":43304,"Ġ(*":43305,"Production":43306,"licts":43307,"Ġinoc":43308,"Cour":43309,"Ġspurious":43310,"Ġultras":43311,"ggles":43312,"Ġdelusion":43313,"ĠRacer":43314,"ĠPrism":43315,"FH":43316,"uppet":43317,"Ġcultured":43318,"Ġ436":43319,"aneously":43320,"اÙĦ":43321,"ĠMissions":43322,"monton":43323,"criptions":43324,"ificate":43325,"Cause":43326,"Ġ1898":43327,"ocaust":43328,"Ġbri":43329,"ĠShoals":43330,"ommod":43331,"alted":43332,"ogenesis":43333,"warn":43334,"illus":43335,"vv":43336,"Ġcontam":43337,"ĠLesbian":43338,"Ġcavalry":43339,"ĠPresence":43340,"rehens":43341,"tool":43342,"accessible":43343,"Ġ(~":43344,"ĠLicensed":43345,"Ġprophets":43346,"Ġboulder":43347,"mean":43348,"akura":43349,"Ġunres":43350,"ĠCinnamon":43351,"Leaks":43352,"........................":43353,"Contact":43354,"Ġassassins":43355,"ĠGreenwald":43356,"dk":43357,"amazon":43358,"Ġagreeable":43359,"ernandez":43360,"Easy":43361,"PLA":43362,"ĠBigfoot":43363,"Ġconvent":43364,"Ġempires":43365,"Ġ387":43366,"Ġgrasped":43367,"Ġruby":43368,"Ġreconc":43369,"Warning":43370,"atem":43371,"Ġretrieval":43372,"ĠFDR":43373,"ĠReaper":43374,"orem":43375,"ĠLuo":43376,"hig":43377,"ĠArmor":43378,"tp":43379,"ĠInterpret":43380,"Conservative":43381,"ĠSodium":43382,"Ġbead":43383,"Ġpropagate":43384,"claw":43385,"href":43386,"ĠPaste":43387,"Ġomit":43388,"Boost":43389,"Diamond":43390,"goo":43391,"Ġanomal":43392,"ĠDISTRICT":43393,"Greek":43394,"warning":43395,"Ġdespised":43396,"Karl":43397,"AGES":43398,"Ġserotonin":43399,"ESSION":43400,"_______":43401,"ĠCollider":43402,"auldron":43403,"Ġsquee":43404,"Control":43405,"ffield":43406,"cycles":43407,"Legal":43408,"xa":43409,"minimum":43410,"ĠGeneric":43411,"Circ":43412,"·":43413,"Behind":43414,"guide":43415,"Ground":43416,"roying":43417,"ĠGrail":43418,"Ġthee":43419,"Ġ9000":43420,"Batman":43421,"Brother":43422,"Ġnons":43423,"RW":43424,"saf":43425,"ĠCroat":43426,"tainment":43427,"sci":43428,"Ye":43429,"Range":43430,"Ey":43431,"perature":43432,"ĠDracula":43433,"oreal":43434,"Fighting":43435,"Ġreleg":43436,"Ġcoupling":43437,"Tracker":43438,"tyard":43439,"Mut":43440,"Military":43441,"lamm":43442,"ittens":43443,"ĠCRC":43444,"ĠXiang":43445,"Ġorthodoxy":43446,"ĠGoth":43447,"Ġalgorith":43448,"ĠAthen":43449,"Ġtyrann":43450,"ĠTorrent":43451,"IDs":43452,"ĠGENERAL":43453,"ĠASUS":43454,"rastructure":43455,"Faith":43456,"models":43457,"rentices":43458,"ĠCurse":43459,"Ġcalibr":43460,"attled":43461,"monary":43462,"Ġpenet":43463,"aclysm":43464,"album":43465,"Ġremnant":43466,"Ġfung":43467,"itiveness":43468,"thodox":43469,"Ġunlocks":43470,"Ġprobabilities":43471,"Ġster":43472,"Ġscrim":43473,"Ġanalytic":43474,"Urban":43475,"âĢĶâĢĶâĢĶâĢĶ":43476,"Craft":43477,"Ġbrut":43478,"1986":43479,"Section":43480,"raged":43481,"arij":43482,"Hero":43483,"ĠHebdo":43484,"ĠEmpress":43485,"Ġvivo":43486,"ĠPublications":43487,"Ġcannabinoids":43488,"arrett":43489,"Ġbounded":43490,"Ġquests":43491,"Ġomin":43492,"ĠRuler":43493,"ĠYue":43494,"ridges":43495,"Ġpeasants":43496,"ĠAlloy":43497,"Desk":43498,"ULAR":43499,"Ġthor":43500,"ĠOvers":43501,"ĠTome":43502,"mk":43503,"Ġ1050":43504,"Ġshroud":43505,"Ġdistribut":43506,"weapons":43507,"ĠAuthorization":43508,"ĠPoke":43509,"ĠAlternate":43510,"scan":43511,"artisan":43512,"ĠGems":43513,"ĠForums":43514,"atonin":43515,"viron":43516,"Rog":43517,"duct":43518,"Ġtabletop":43519,"crow":43520,"/)":43521,"ĠStainless":43522,"ottest":43523,"Ġreborn":43524,"anchez":43525,"cium":43526,"ĠNicarag":43527,"elfare":43528,"Ġupd":43529,"ritic":43530,"bm":43531,"Ġ608":43532,"ĠSlightly":43533,"ĠDrops":43534,"ISO":43535,"ĠiT":43536,"xiety":43537,"ĠGawker":43538,"omination":43539,"ĠReached":43540,"Student":43541,"Drop":43542,"MET":43543,"ĠKubrick":43544,"1950":43545,"ĠTuls":43546,"Ġcomputed":43547,"depending":43548,"ĠCosmetic":43549,"udget":43550,"Lex":43551,"icut":43552,"ĠDepth":43553,"Ġ1893":43554,"ahah":43555,"Ġath":43556,"fights":43557,"thia":43558,"Ġoccult":43559,"Wheel":43560,"ĠSega":43561,"Ġtheolog":43562,"reement":43563,")--":43564,"Ġunus":43565,"ĠGamma":43566,"Looks":43567,"Ġellipt":43568,"Ġairflow":43569,"ĠHimself":43570,"Ġpagan":43571,"ĠRei":43572,"Ġpilgr":43573,"ĠSubmission":43574,"Region":43575,"Ġinsertion":43576,"Ġsket":43577,"Ġsatisfies":43578,"ĠPixie":43579,"Ġcontempl":43580,"abbit":43581,"ĠReplay":43582,"ĠGalile":43583,"ĠGodzilla":43584,"Ġarithmetic":43585,"iasm":43586,"1987":43587,"ĠFeminist":43588,"Liter":43589,"ĠDisable":43590,"ouble":43591,"essors":43592,"Ġfors":43593,"Ġensu":43594,"Putting":43595,"ĠMSM":43596,"Cond":43597,"emade":43598,"Ġindistinguishable":43599,"Magn":43600,"Ġms":43601,"MAL":43602,"ĠBF":43603,"dm":43604,"iltration":43605,"irection":43606,"ĠSpir":43607,"Gb":43608,"ĠIbn":43609,"Abs":43610,"imens":43611,"RNA":43612,"============":43613,"Ġ655":43614,"ĠConversion":43615,"imilation":43616,"igion":43617,"ĠSomew":43618,"mL":43619,"Border":43620,"Ë":43621,"Factor":43622,"Number":43623,"Ġejac":43624,"Cho":43625,"Ġrighteousness":43626,"ĠPATH":43627,"ĠElys":43628,"ouched":43629,"Ġmultic":43630,"Ġfaculties":43631,"ĠEarthquake":43632,"ĠReferences":43633,"ensitive":43634,"Ġimpat":43635,"Ġ................":43636,"buff":43637,"Ġ1895":43638,"colo":43639,"Vi":43640,"Ġubiqu":43641,"ĠChev":43642,"Fish":43643,"ĠBlueprint":43644,"CHQ":43645,"Ġlinem":43646,"ĠFlavor":43647,"Ġcrimson":43648,"ĠAbstract":43649,"arette":43650,"plete":43651,"ranean":43652,"Dash":43653,"Ġdimensional":43654,"Cub":43655,"ttle":43656,"ĠDSM":43657,"Ġinstantaneous":43658,"esy":43659,"Ġepoch":43660,"Brit":43661,"ĠÎ":43662,"ECD":43663,"Ġwarp":43664,"obyl":43665,"ubric":43666,"Ġutilitarian":43667,"Ġsummarizes":43668,"letal":43669,"Ord":43670,"opath":43671,"tained":43672,"ghai":43673,"Ġwhis":43674,"insert":43675,"Ġphon":43676,"rils":43677,"Ġearthly":43678,"ĠAlic":43679,"ĠPCIe":43680,"Ġfurthermore":43681,"ocard":43682,"Ġuter":43683,"ĠAdmin":43684,"ographics":43685,"ĠConstantin":43686,"gravity":43687,"iPhone":43688,"Ġwasteland":43689,"Ġfps":43690,"Tip":43691,"Ġmurm":43692,"paces":43693,"ĠSamurai":43694,"ĠFOIA":43695,"ĠRadiant":43696,"ĠUnreal":43697,"Ġmicrow":43698,"usterity":43699,"zyme":43700,"itbart":43701,"metadata":43702,"Dat":43703,"ĠMoons":43704,"ĠProtestants":43705,"ungle":43706,"Ġvideog":43707,"pid":43708,"Ġdisple":43709,"aucus":43710,"Ġcoils":43711,"ĠDwar":43712,"fixed":43713,"Alice":43714,"Ġgarrison":43715,"ĠVelocity":43716,"ĠJehovah":43717,"Ġfascists":43718,"ĠCHO":43719,"jl":43720,"Ġmetaphors":43721,"ĠSiege":43722,"scientific":43723,"Ä«":43724,"Slow":43725,"hex":43726,"ĠBlaz":43727,"mediated":43728,"esthesia":43729,"ĠAvg":43730,"Ġbelie":43731,"Carter":43732,"Ġexposition":43733,"azeera":43734,"dial":43735,"Ġbask":43736,"Scale":43737,"Ġdisob":43738,"Ġgore":43739,"Ġhypocr":43740,"Ġphantom":43741,"ĠSynd":43742,"BLIC":43743,"pter":43744,"ĠScorpion":43745,"eor":43746,"ĠRecover":43747,"Ġsummoning":43748,"Ġorb":43749,"jump":43750,"Ġ768":43751,"ĠEnix":43752,"Spons":43753,",...":43754,"Wide":43755,"Ġparse":43756,"Ġdebtor":43757,"Ġpathological":43758,"Ġserpent":43759,"ĠFranç":43760,"reetings":43761,"Ġdeletion":43762,"Ġvolunt":43763,"ĠNotification":43764,"liga":43765,"Disk":43766,"Account":43767,"1979":43768,"Ġsymmetry":43769,"ĠBearing":43770,"ĠABV":43771,"ĠORDER":43772,"rpm":43773,"ĠFuck":43774,"?!\"":43775,"mask":43776,"Grade":43777,"neath":43778,"ocom":43779,"Detect":43780,"ryption":43781,"ĠAura":43782,"Ġinert":43783,"PLAY":43784,"gres":43785,"INTON":43786,"Deal":43787,"fficient":43788,"ĠVoid":43789,"gement":43790,"Ġscorp":43791,"Ġreincarn":43792,"ĠVapor":43793,"Ġ1840":43794,"Yellow":43795,"......":43796,"Ġparameter":43797,"ĠDISTR":43798,"ĠForgotten":43799,"Eat":43800,"izational":43801,"Witness":43802,"ĠDupl":43803,"Ġdogma":43804,"Ġzipper":43805,"ĠZeus":43806,"mage":43807,"ormal":43808,"Ġ\".":43809,"Ġecc":43810,"ĠSlot":43811,"ĠRegist":43812,"Others":43813,"VID":43814,"Windows":43815,"Ġshitty":43816,"ĠLethal":43817,"Monster":43818,"ĠExpression":43819,"tx":43820,"ythm":43821,"Were":43822,"ivalry":43823,"atcher":43824,"ĠFormat":43825,"ĠPlasma":43826,"Phys":43827,"laugh":43828,"Fu":43829,"java":43830,"roma":43831,"ĠIncreases":43832,"Ġlicensee":43833,"Ġmystic":43834,"Ġproto":43835,"ĠLoki":43836,"forcing":43837,"hots":43838,"Ġ->":43839,"Outside":43840,"ĠEndless":43841,"Ġachie":43842,"ĠTurtles":43843,"Ġconvin":43844,"JUST":43845,"Ġimmobil":43846,"ĠCauses":43847,"Ġclich":43848,"xes":43849,"ffiti":43850,"Ġhypot":43851,"Bat":43852,"Ġbigot":43853,"Personal":43854,"ĠPharmac":43855,"Lot":43856,"VERT":43857,"Ġbapt":43858,"idelines":43859,"Ġprox":43860,"MAP":43861,"Spirit":43862,"ĠSlug":43863,"Ġebook":43864,"eches":43865,"ĠAndromeda":43866,"Ġceremon":43867,"1975":43868,"PRE":43869,"Ġasshole":43870,"linear":43871,"Nevertheless":43872,"Ġwillpower":43873,"azel":43874,"Fif":43875,"andise":43876,"Ġextravag":43877,"ĠBuffy":43878,"Ġcorrelations":43879,"ptr":43880,"Progress":43881,"shape":43882,"ĠSymbol":43883,"arag":43884,"ĠContext":43885,"ucer":43886,"1983":43887,"ĠMyster":43888,"Pain":43889,"Login":43890,"mbol":43891,"codes":43892,"RANT":43893,"Ġoverse":43894,"opot":43895,"STEM":43896,"enser":43897,"ĠCosmic":43898,"Spl":43899,"ritional":43900,"ĠPharaoh":43901,"ĠRemix":43902,"xon":43903,"ĠXII":43904,"Ġunman":43905,"Ġimmedi":43906,"Ġmonog":43907,"ĠLX":43908,"Ġabstraction":43909,"ocolate":43910,"ĠDonkey":43911,"Ġ!!":43912,"ĠLIA":43913,"shed":43914,"rules":43915,"Ġcalc":43916,"ĠAutob":43917,"anmar":43918,"eworks":43919,"notations":43920,"Ġtenancy":43921,"ĠPetraeus":43922,"dp":43923,"amphetamine":43924,"ĠCortex":43925,"rw":43926,"Ġprojectile":43927,"Ġintrinsically":43928,"Route":43929,"Ġnegoti":43930,"anuts":43931,"Analysis":43932,"redits":43933,"ĠGG":43934,"thread":43935,"ĠChosen":43936,"Years":43937,"otyp":43938,"ĠNCT":43939,"udic":43940,"ochemical":43941,"Neigh":43942,"Ġfishes":43943,"ĠFloat":43944,"Print":43945,"okia":43946,"Ġbarb":43947,"quote":43948,"Lew":43949,"Ġannoun":43950,"istors":43951,"Reading":43952,"ACTION":43953,"Ġintakes":43954,"ĠBeet":43955,"matter":43956,"Swe":43957,"Ther":43958,"Ġtyrant":43959,"ĠPsycho":43960,"ĠDestroy":43961,"Ġesoteric":43962,"Ġbiom":43963,"idious":43964,"Merc":43965,"hran":43966,"ĠBaal":43967,"seconds":43968,"Ġsuperhuman":43969,"ancel":43970,"Ġworshipped":43971,"Ġwebs":43972,"Ġviolet":43973,"ĠMetallic":43974,"eday":43975,"ordering":43976,"Nut":43977,"Ġconstructs":43978,"olescent":43979,"Unit":43980,"otypes":43981,"Ġembryonic":43982,"perm":43983,"Nature":43984,"ĠDecre":43985,"levant":43986,"Ġss":43987,"+(":43988,"ĠDoctrine":43989,"puters":43990,"Ġsaline":43991,"orsche":43992,"1111":43993,"values":43994,"Ġutopian":43995,"ĠBooster":43996,"Technical":43997,"ì":43998,"ĠLIMITED":43999,"nir":44000,"Ġclones":44001,"Performance":44002,"aple":44003,"Ġshudder":44004,"Ġcontempor":44005,"lator":44006,"ĠOops":44007,"Ġammon":44008,"Ġdavid":44009,"Ġbom":44010,"bish":44011,"Ġdetectable":44012,"Ġmultiplying":44013,"Ġreddit":44014,"Prim":44015,"Ġmedial":44016,"Ġsubstrate":44017,"ĠSanskrit":44018,"Spect":44019,"ĠMagical":44020,"Ġarcane":44021,"align":44022,"Ġ1861":44023,"Ġneocons":44024,"Ì":44025,"ĠBounty":44026,"ĠContinent":44027,"Ġhurd":44028,"alions":44029,"Ġgeneralized":44030,"ĠInsect":44031,"Ġsimul":44032,"actual":44033,"advert":44034,"ukong":44035,"Resp":44036,"ĠWarcraft":44037,"Hunter":44038,"hyper":44039,"ĠBreach":44040,"ught":44041,"Ġcomputation":44042,"react":44043,"Feel":44044,"ĠCheong":44045,"Ġslut":44046,"Ġgalactic":44047,"Ġtaunt":44048,"Enjoy":44049,"Ġreprinted":44050,"Word":44051,"ĠHandbook":44052,"amins":44053,"exit":44054,"Wo":44055,"Ġadherents":44056,"Counter":44057,"ĠNode":44058,"ĠTwisted":44059,"Ġgrinned":44060,"universal":44061,"ĠAmon":44062,"Ġaster":44063,"ĠEquip":44064,"!\".":44065,"Ġanalogous":44066,"rients":44067,"alky":44068,"ĠQian":44069,"Ġspont":44070,"docs":44071,"Ġcontemplation":44072,"Ġrevolutionaries":44073,"Ġpreset":44074,"ĠAmendments":44075,"Ġexecutes":44076,"ĠDuration":44077,"Ġcompulsion":44078,"Ġstagger":44079,"ynamic":44080,"blem":44081,"];":44082,"Higher":44083,"Balt":44084,"heast":44085,"Ġcorp":44086,"awei":44087,"Motion":44088,"Mis":44089,"Ġadventurer":44090,"eger":44091,"Ġarsen":44092,"ĠVoltage":44093,"ĠEVENTS":44094,"Salt":44095,"issance":44096,"DK":44097,"Ship":44098,"Ġunwitting":44099,"Ton":44100,"ĠPROGRAM":44101,"Ġtentacles":44102,"erness":44103,"thirst":44104,"Fig":44105,"fty":44106,"ĠTolkien":44107,"Sleep":44108,"ĠExplain":44109,"Pub":44110,"ĠBounce":44111,"ĠDemo":44112,"Ġ1897":44113,"ĠSPI":44114,"intern":44115,"********":44116,"ĠKills":44117,"ĠZombies":44118,"Single":44119,"ratom":44120,"ĠClaw":44121,"hid":44122,"asel":44123,"Shock":44124,"erential":44125,"Ġupgr":44126,"holy":44127,"Ġ\\":44128,"aghetti":44129,"Ġthence":44130,"genic":44131,"papers":44132,"1982":44133,"ravel":44134,"ĠUNIVERS":44135,"Charge":44136,"ĠDelay":44137,"ibrary":44138,"ĠHDD":44139,"olson":44140,"Ġenchanted":44141,"Wr":44142,"graph":44143,"Ġcorro":44144,"ept":44145,"etsu":44146,"ĠQin":44147,"Û":44148,"Ġantidepressant":44149,"ĠCerberus":44150,"Ġappe":44151,"ĠDEFENSE":44152,"Ġdysph":44153,"split":44154,"zilla":44155,"attr":44156,"Clar":44157,"Äĵ":44158,"hov":44159,"IRC":44160,"hibition":44161,"'/":44162,"ĠURLs":44163,"Draft":44164,"Prep":44165,"ĠLanguages":44166,"ĠTravels":44167,"ceiver":44168,"aturally":44169,"pair":44170,"ĠALWAYS":44171,"aaaa":44172,"ĠTenth":44173,"ĠNAD":44174,"Serv":44175,"ĠUID":44176,"cens":44177,"ĠLearned":44178,"Ġtraject":44179,"Ġmoaning":44180,"ĠNare":44181,"Ġingen":44182,"Ġsurn":44183,"Ġfloppy":44184,"breeding":44185,"uph":44186,"rossover":44187,"Understanding":44188,"Glass":44189,"Ġruntime":44190,"gp":44191,"Ġâľĵ":44192,"Ġcyt":44193,"bley":44194,"agall":44195,"Ġunworthy":44196,"otine":44197,"Ġchromosome":44198,"utters":44199,"Ġµ":44200,"Ġexpans":44201,"Ġdement":44202,"Ġinsurrection":44203,"Ġsurviv":44204,"genre":44205,"ospital":44206,"ĠPlato":44207,"ĠTrigger":44208,"selection":44209,"ilege":44210,"Ġsegreg":44211,"itizens":44212,"ĠRAID":44213,"Pure":44214,"hetti":44215,"ĠFailed":44216,"ĠCharacters":44217,"ĠCreep":44218,"akra":44219,"Ec":44220,"ĠAristotle":44221,"Lim":44222,"error":44223,"yrus":44224,"umably":44225,">>":44226,"Ġtsun":44227,"knowledge":44228,"Cert":44229,"bable":44230,"hesion":44231,"ĠProcedures":44232,"Ġmarkup":44233,"ideo":44234,"Ġrhet":44235,"ĠChapters":44236,"ĠChecking":44237,"mega":44238,"Ġphotons":44239,"required":44240,"Unknown":44241,"ĠDrawn":44242,"Ġvari":44243,"EEK":44244,"Ġcompuls":44245,"Ġcloning":44246,"ccoli":44247,"Ġ1070":44248,"Ġkindred":44249,"Ġdiscl":44250,"ĠCind":44251,"Collect":44252,"Ġchromosomes":44253,"phant":44254,"ĠKafka":44255,"Ġeverlasting":44256,"Ġmercenary":44257,"ĠHmm":44258,"----":44259,"riber":44260,"Ġdoubtless":44261,"Ġsusceptibility":44262,"beta":44263,"notice":44264,"Ġcrochet":44265,"Ġrespir":44266,"Ġphilosophers":44267,"ĠExtras":44268,"Ġseparat":44269,"shown":44270,"iblings":44271,"Hispanic":44272,"copy":44273,"Tang":44274,"Knight":44275,"Ġpursu":44276,"ĠAnime":44277,"Ġlipid":44278,"ggies":44279,"levels":44280,"phalt":44281,"ĠCompleted":44282,"bral":44283,"Ġcerv":44284,"ĠAfric":44285,"ĠPhar":44286,"Color":44287,"ogene":44288,"ĠCompan":44289,"memory":44290,"Dust":44291,"ĠXIV":44292,"ĠConsole":44293,"').":44294,"Ġ1888":44295,"byn":44296,"Ġpolygamy":44297,"Auth":44298,"BUT":44299,"istine":44300,"Ġsacr":44301,"Ġabsor":44302,"ijah":44303,"ĠNeural":44304,"olester":44305,"ql":44306,"Already":44307,"Creating":44308,"ĠStarg":44309,"ĠPhilos":44310,"Consider":44311,"Ġrepositories":44312,"cludes":44313,"ĠBuffer":44314,"ĠPerspect":44315,"Ġcomput":44316,"Stew":44317,"iamond":44318,"ĠJudgment":44319,"OVA":44320,"angible":44321,"Ġoxid":44322,"Ġepigen":44323,"Ġsidel":44324,"ĠEag":44325,"devices":44326,"icone":44327,"1920":44328,"atism":44329,"beard":44330,"ĠGujar":44331,"ĠPlaystation":44332,"Ġglances":44333,"ĠCOMPLE":44334,"VERTIS":44335,"ukemia":44336,"Edit":44337,"Tickets":44338,"Square":44339,"ĠSerpent":44340,"Ġtransporter":44341,"MQ":44342,"ĠMongo":44343,"1967":44344,"ibaba":44345,"Ġtimet":44346,"sylvania":44347,"Latin":44348,"osaurs":44349,"Ġhumanoid":44350,"Ġcannabinoid":44351,"Ġdisciple":44352,"Psych":44353,"Ġimpro":44354,"Ġmc":44355,"Raid":44356,"Letter":44357,"ificant":44358,"ĠPortug":44359,"ĠFreem":44360,"Ġappell":44361,"ĠMushroom":44362,"Ġclans":44363,"Ġsinful":44364,"Ġingestion":44365,"ĠDirectory":44366,"abetic":44367,"Ġantigen":44368,"Ġimagin":44369,"mitter":44370,"!!!!!":44371,"ĠDPR":44372,"leness":44373,"\":\"\",\"":44374,"ĠAUTHOR":44375,"Ġgrunt":44376,"Ġflickering":44377,"Cath":44378,"asury":44379,"Ġnozzle":44380,"Secure":44381,"Stre":44382,"ĠBIT":44383,"Ġdeviations":44384,"Professor":44385,"bilt":44386,"ĠConscious":44387,"Ġinterrupts":44388,"ĠMormons":44389,"ĠCutter":44390,"Bed":44391,"ipient":44392,"ĠGhostbusters":44393,"Cart":44394,"endas":44395,"ĠExecution":44396,"ycle":44397,"Ġwedd":44398,"Sold":44399,"Ġvanquished":44400,"Regarding":44401,"Depending":44402,"']":44403,"atron":44404,"oidal":44405,"Cube":44406,"Studio":44407,":/":44408,"ĠExplosion":44409,"activate":44410,"pport":44411,"fuck":44412,"Whe":44413,"Ġsmir":44414,"Ġwidgets":44415,"urses":44416,"izard":44417,")*":44418,"icho":44419,"ĠVersus":44420,"ĠIntroduced":44421,"osaurus":44422,"1977":44423,"forum":44424,"Gray":44425,"Program":44426,"righteous":44427,"endum":44428,"ĠScare":44429,"Ġresists":44430,"*)":44431,"ĠCombo":44432,"Ġsockets":44433,"Ġaston":44434,"LAB":44435,"Ġmutated":44436,"eworld":44437,"DEF":44438,"Trend":44439,"âĢĶ-":44440,"Ġpropagation":44441,"Ġemancipation":44442,"collection":44443,"ĠDifferences":44444,"Tweet":44445,"Ġmajesty":44446,")...":44447,"sylv":44448,"Ġadapters":44449,"Ġmilliseconds":44450,"Jews":44451,"ĠPatreon":44452,"phasis":44453,"ĠHTTP":44454,"onnaissance":44455,"ENDED":44456,"ĠIntro":44457,"qs":44458,"Ġsuperflu":44459,"*.":44460,"Ġminions":44461,"ĠStupid":44462,"Ġspecialization":44463,"ĠPikachu":44464,"Ġappellant":44465,"Training":44466,"circle":44467,"Interest":44468,"Ġfallacy":44469,"ĠDinosaur":44470,"ĠTHEM":44471,"Ġdirectories":44472,"Ġmasturbation":44473,"ĠStain":44474,"1978":44475,"odied":44476,"Ġexqu":44477,"ĠRats":44478,"swick":44479,"Ġemptiness":44480,"ĠXeon":44481,"Ġthereto":44482,"ĠEngels":44483,"ĠSupplement":44484,"Chan":44485,"Ġundead":44486,"ĠNoct":44487,"erest":44488,"ĠQuery":44489,"ĠSOLD":44490,"thritis":44491,"ĠEncounter":44492,"Ġvectors":44493,"Econom":44494,"Rogue":44495,"Ġgelatin":44496,"Rot":44497,"Flickr":44498,"Ġcaching":44499,"Ġloader":44500,"ĠELE":44501,"Ġcamoufl":44502,"Commission":44503,"Ġ1886":44504,"Ġcombos":44505,"ĠAwakening":44506,"Ġfeudal":44507,"Ġasses":44508,"ASY":44509,"atalie":44510,"Ġpanties":44511,"ĠMono":44512,"selves":44513,"Download":44514,"Ġvampires":44515,"------":44516,"ishop":44517,"User":44518,"Ġimperialist":44519,"ĠGOODMAN":44520,"1973":44521,"Vel":44522,"Struct":44523,"ĠUFOs":44524,"drivers":44525,"ĠOptional":44526,"uably":44527,"ĠPrinciple":44528,"verett":44529,"taining":44530,"Ġ1889":44531,"ĠCommunism":44532,"auder":44533,"Keys":44534,"lore":44535,"ĠMedieval":44536,"Hyd":44537,"weapon":44538,"Register":44539,"ĠHighlander":44540,"ĠRFC":44541,"Demon":44542,"ardless":44543,"ĠOrche":44544,"Kick":44545,"pixel":44546,"address":44547,"OUP":44548,"Brain":44549,"ĠMorph":44550,"bash":44551,"ĠANG":44552,"ĠIdle":44553,"ĠLucifer":44554,"Ġcorrelates":44555,"Ġgazed":44556,"colm":44557,"ĠKard":44558,"Solar":44559,"ĠVariable":44560,"ĠPACK":44561,"Ġfuzz":44562,"Ġanonym":44563,"ĠECO":44564,"feature":44565,"ĠEsports":44566,"ĠAnthropology":44567,"cise":44568,"manac":44569,"ĠSupports":44570,"rists":44571,"Quant":44572,"istical":44573,"çļĦ":44574,"Ġdexterity":44575,"monster":44576,"ordial":44577,"Mob":44578,"DEC":44579,"ĠConj":44580,"entric":44581,"1981":44582,"ECTION":44583,"ietal":44584,"ĠUses":44585,"ĠArmageddon":44586,"ĠCapitalism":44587,"Ub":44588,"iazep":44589,"helps":44590,"ouls":44591,"grim":44592,"ĠEthiop":44593,"tesy":44594,"Ġclipboard":44595,"Ġchimpanzees":44596,"PLIC":44597,"Sexual":44598,"wallet":44599,"ĠRect":44600,"ocytes":44601,"ĠHels":44602,"lace":44603,"Damn":44604,"Ġblasp":44605,"ildo":44606,"ĠRober":44607,"APD":44608,"ĠWCS":44609,"ippery":44610,"ellectual":44611,"Ġ$(":44612,"Ġuniverses":44613,"Ġholster":44614,"Ġshading":44615,"Ġinflic":44616,"else":44617,"ĠShiny":44618,"ĠAVG":44619,"Lower":44620,"ĠMayhem":44621,"Originally":44622,"Crypt":44623,"SHARE":44624,"ĠBeir":44625,"!:":44626,"Ġrepentance":44627,"WHAT":44628,".......":44629,"Ġauditory":44630,"aaa":44631,"ĠLoot":44632,"ciples":44633,"Ġcontem":44634,"Ġphoton":44635,"æľ":44636,"omach":44637,"ĠWhedon":44638,"ĠValid":44639,"asonable":44640,"pha":44641,"assad":44642,"ĠPse":44643,"Heat":44644,"Ġplugins":44645,"Ġclenched":44646,"ĠAmeric":44647,"transform":44648,"ĠEnh":44649,"agnetic":44650,"usalem":44651,"sych":44652,"Wed":44653,"replace":44654,"ĠKinect":44655,"shield":44656,"Sax":44657,"ividually":44658,"Ġfunctionally":44659,"Ġ:)":44660,"typically":44661,"Opening":44662,"Fa":44663,"ĠSELECT":44664,"Ġsamurai":44665,"Ġhorde":44666,"entle":44667,"sth":44668,"Changes":44669,"Pin":44670,"ithing":44671,"illance":44672,"ĠEmblem":44673,"ĠMicha":44674,"crypt":44675,"ĠObjective":44676,"ophys":44677,"Ġavg":44678,"poon":44679,"Ġreadable":44680,"ĠRx":44681,"allel":44682,"Sit":44683,"gom":44684,"ureau":44685,"ĠDoodle":44686,"Ġdungeon":44687,"($":44688,"Nintendo":44689,"\"],\"":44690,"Notes":44691,"Grab":44692,"Prosecutors":44693,"Advanced":44694,"Ġ1862":44695,"ĠVeter":44696,"Ġjurisd":44697,"ĠLauncher":44698,"Catal":44699,"udder":44700,"Ġresidues":44701,"Ġregress":44702,"ĠConquer":44703,"osal":44704,"ĠDice":44705,"************":44706,"braska":44707,"ipolar":44708,"Ġathe":44709,"bringing":44710,"Suddenly":44711,"ĠIEEE":44712,"verbs":44713,"Ġdelet":44714,"ipeg":44715,"Previous":44716,"]\"":44717,"Ġsidebar":44718,"illac":44719,"Property":44720,"α":44721,"REP":44722,"Ġauthenticated":44723,"gypt":44724,"uilding":44725,"ĠGing":44726,"Ġwart":44727,"Birth":44728,"Ġobedient":44729,"ĠXuan":44730,"ĠTYPE":44731,"Ġinhibits":44732,"1972":44733,"humans":44734,"IENT":44735,"Ġyoutube":44736,"Shortly":44737,"ophen":44738,"ĠWinc":44739,"ĠWrit":44740,"AUD":44741,"ĠHobbit":44742,"emphasis":44743,"ĠWonders":44744,"Ġtwitch":44745,"ĠProphe":44746,"Berry":44747,"ĠGinny":44748,"ĠBurst":44749,"ĠGenerator":44750,"Ġepile":44751,"ĠBalanced":44752,"GPU":44753,"maps":44754,"Ġneurotrans":44755,"ĠIRC":44756,"Ġ\"$":44757,"Create":44758,"Particip":44759,"ĠMarxism":44760,"Ġthou":44761,"ĠMortal":44762,"Ġ�":44763,"Ġninja":44764,"inburgh":44765,"Ġappro":44766,"ĠPistol":44767,"Jar":44768,"Ġprophes":44769,"classes":44770,"Ġanarchist":44771,"Ġextant":44772,"message":44773,"itaire":44774,"Ġ1863":44775,"ĠProl":44776,"Ġpropell":44777,"Ġimpossibility":44778,"Ġpropos":44779,"itamin":44780,"Rating":44781,"olphin":44782,"Ġmitochond":44783,"versions":44784,"Liberal":44785,"ishy":44786,"Ġspherical":44787,"ĠSurvive":44788,"FREE":44789,"rawler":44790,"Metal":44791,"ĠStarship":44792,"Ġ=================================================================":44793,"ĠDharma":44794,"ĠSeller":44795,"Ġwrapper":44796,"Experience":44797,"Integ":44798,"Customer":44799,"hammad":44800,"Ġunanim":44801,"Jenn":44802,"Ġschizophren":44803,"agree":44804,"ĠEVENT":44805,"Shell":44806,"Ġfractions":44807,"1968":44808,"Ġextermination":44809,"ĠSniper":44810,"Ġpronoun":44811,"ĠHitman":44812,"xp":44813,"resource":44814,"WIND":44815,"Ġhierarchical":44816,"Ġted":44817,"Changing":44818,"Ġplaus":44819,"Transform":44820,"Ġbicy":44821,"imentary":44822,"Fuck":44823,"Mini":44824,"Ġoverc":44825,"ĠOptimus":44826,"outer":44827,"helial":44828,"akening":44829,"fx":44830,"Ġnig":44831,"Ġ+/-":44832,"ĠVICE":44833,"Ġnm":44834,"1976":44835,"ĠRitual":44836,"ĠTyrann":44837,"Ġscriptures":44838,"inical":44839,"ĠNull":44840,"ourgeois":44841,"dra":44842,"Ġpious":44843,"Ġneuron":44844,"Ġcolonists":44845,"ĠNebula":44846,"apply":44847,"Sah":44848,"Marx":44849,"Ġhypotheses":44850,"notation":44851,"acists":44852,"Math":44853,"Manager":44854,"Library":44855,"audi":44856,"Ġmp":44857,"ergic":44858,"Ġwizards":44859,"fw":44860,"DVD":44861,"ĠScala":44862,"Different":44863,"ampoo":44864,"ĠDread":44865,"abbage":44866,"Rus":44867,"ĠDumbledore":44868,"keleton":44869,"elsh":44870,"esian":44871,"ĠCorsair":44872,"Tier":44873,"ĠCelest":44874,"Ġnoun":44875,"Ġlucid":44876,"requisites":44877,"Ġgenus":44878,"Event":44879,"1974":44880,"ĠSatanic":44881,"iox":44882,"ĠHandle":44883,"ĠDestroyer":44884,"Ġinvocation":44885,"ĠXD":44886,"modified":44887,"Gam":44888,"ĠRPC":44889,"Ġsubsystem":44890,"Compared":44891,"odan":44892,"ĠPassive":44893,"ĠHelmet":44894,"nutrition":44895,"riction":44896,"HOW":44897,"Jess":44898,"Ġpiston":44899,"imately":44900,"Ġhypoc":44901,"ĠCelestial":44902,"MRI":44903,"Ġcompiler":44904,"ĠBadge":44905,"ĠRevelation":44906,"Ġintrig":44907,"Grad":44908,"ĠSPACE":44909,"Poly":44910,"ĠVul":44911,"Ġtrembling":44912,"Ġindepend":44913,"doctor":44914,"Certain":44915,"emet":44916,"Password":44917,"Ġgasped":44918,"Ġpronunciation":44919,"Fuel":44920,"ĠSPEC":44921,"assets":44922,"Extra":44923,"Ġformatting":44924,"Ġmods":44925,"\"!":44926,"akedown":44927,"Ġcircuitry":44928,"ĠTRUE":44929,"ĠVeil":44930,"Ġsighed":44931,"Charg":44932,"eals":44933,"Ġworkaround":44934,"Ġank":44935,"ĠScrolls":44936,"Ġdiffusion":44937,"Ġamps":44938,"ĠTempest":44939,"adata":44940,"Ġphenomen":44941,"Ġ???":44942,"Ġpopup":44943,"Ġinhibition":44944,"Ġaliases":44945,"erity":44946,"agraph":44947,"Jew":44948,"Ġbec":44949,"Classic":44950,"comment":44951,"usable":44952,"rodu":44953,"ĠEnlightenment":44954,"Ġinvis":44955,"Ġbiochemical":44956,"latest":44957,"ĠGMOs":44958,"ĠSocialism":44959,"Ġpollut":44960,"Ġeluc":44961,"Js":44962,"orthern":44963,"PDATED":44964,"alyses":44965,"Experts":44966,"Blog":44967,"ĠDemocr":44968,"etooth":44969,"pause":44970,"âĢ¢âĢ¢":44971,"ĠShinji":44972,"Ġdystop":44973,"Sources":44974,"ĠBrach":44975,"np":44976,"ĠXY":44977,"Ġneurot":44978,"assembly":44979,"Ġbourgeois":44980,"ĠReson":44981,"ĠIDE":44982,"Ġrecoil":44983,"raq":44984,"ĠAvenger":44985,"Paper":44986,"UTF":44987,"ĠWrest":44988,"ĠSimulation":44989,"elaide":44990,"ĠDMCA":44991,"utm":44992,"1963":44993,"Ġarcs":44994,"Ġmaximal":44995,"Ġcyl":44996,"Ġphilosoph":44997,"enium":44998,"Ġrelativity":44999,"ĠMacintosh":45000,"Ġpneum":45001,"LOC":45002,"Ġgoddamn":45003,"SHA":45004,"Ġlocalization":45005,"ĠPHI":45006,"Ġhierarch":45007,"Ġatheists":45008,"±":45009,"Luck":45010,"ĠJugg":45011,"options":45012,"alore":45013,"Edward":45014,"Monitor":45015,"Ġneoc":45016,"numbered":45017,"Arc":45018,"ĠCodes":45019,"ĠHallow":45020,"olitan":45021,"sections":45022,"ĠEzek":45023,"Ġamy":45024,"task":45025,"ĠCLS":45026,"ĠValkyrie":45027,"Ġcircumference":45028,"amac":45029,"ĠNotting":45030,"Ġproverb":45031,"Spec":45032,"Ġelemental":45033,"ĠBitcoins":45034,"Except":45035,"Release":45036,"ADVERTISEMENT":45037,"Complete":45038,"phrine":45039,"Ġspores":45040,"random":45041,"neum":45042,"trigger":45043,"ocide":45044,"Ġlongitudinal":45045,"isec":45046,"peat":45047,"Ġprecept":45048,"Wing":45049,"ĠâĹ":45050,"otropic":45051,"mouse":45052,"ĠWitcher":45053,"ĠAppearance":45054,"ROR":45055,"Ġ||":45056,"aird":45057,"Blu":45058,"Ġincomp":45059,"ĠFirefly":45060,"update":45061,"Loc":45062,"Ġnihil":45063,"hesive":45064,"Quality":45065,"youtu":45066,"Seriously":45067,"Ġannot":45068,"ĠCoins":45069,"Visit":45070,"lc":45071,"----------":45072,"Ġdiction":45073,"Ġafore":45074,"Ġimmortality":45075,"ĠForbidden":45076,"Allah":45077,"ĠPartial":45078,"ĠGears":45079,"Ġtrance":45080,"Hat":45081,"irez":45082,"ĠSATA":45083,"Ġelectrode":45084,"ĠLinear":45085,"rikes":45086,"Ġderiv":45087,"ĠXue":45088,"Fine":45089,"ĠIgnore":45090,"desc":45091,"DOM":45092,"Simple":45093,"orescence":45094,"Previously":45095,"Ġcircumcision":45096,"Sphere":45097,"Ġrenown":45098,"SET":45099,"ilight":45100,"ĠByzantine":45101,"EXP":45102,"Ġwhine":45103,"Missing":45104,"Lt":45105,"Guide":45106,"Ġhippocampus":45107,"Ġwip":45108,"yrights":45109,"Ġsubmer":45110,"Maker":45111,"Switch":45112,"Ġspectral":45113,"nect":45114,"Ãį":45115,"Ġreven":45116,"WER":45117,"Adding":45118,"ĠCONTROL":45119,"asper":45120,"0000000":45121,"ynt":45122,"annabin":45123,"ĠAliens":45124,"ĠPCR":45125,"asketball":45126,"ricia":45127,"ĠUnch":45128,"Tap":45129,"Ġpracticable":45130,"ĠUsage":45131,"Ġsoluble":45132,"Scroll":45133,"Random":45134,"Ġmoan":45135,"ĠPuppet":45136,"Dim":45137,"Attack":45138,"Ġspears":45139,"Ġrectangle":45140,"Ġamuse":45141,"ĠDoct":45142,"reon":45143,"ĠReset":45144,"vag":45145,"unin":45146,"ĠBris":45147,"ĠSwarm":45148,"Model":45149,"Standing":45150,"Ġdenotes":45151,"{":45152,"ĠLizard":45153,"nesty":45154,"Ġwor":45155,"Ġamplification":45156,"ĠInferno":45157,"Cover":45158,"SAM":45159,"respective":45160,"Shift":45161,"Ġlibertarians":45162,"Runner":45163,"ĠRevelations":45164,"Spr":45165,"ĠCrusader":45166,"Ġcaffe":45167,"Patch":45168,"stros":45169,"ĠImmortal":45170,"Ġinsofar":45171,"itance":45172,"ĠValhalla":45173,"Ġradial":45174,"Beast":45175,"sync":45176,"Ġ--------":45177,"ĠPathfinder":45178,"iless":45179,"operator":45180,"Choose":45181,"Ġdecode":45182,"Ġvou":45183,"ĠMutant":45184,"ĠCVE":45185,"Female":45186,"Ġoxidation":45187,"inational":45188,"dB":45189,"Scope":45190,"Wan":45191,"ĠBought":45192,"ĠDietary":45193,"rotein":45194,"Present":45195,"aukee":45196,"Ġtotem":45197,"Ġsatur":45198,"wagon":45199,"Builder":45200,"ĠBulg":45201,"Ġsects":45202,"Flo":45203,"ombat":45204,"ĠHermione":45205,"aughs":45206,"Ġhydra":45207,"paren":45208,"ë":45209,"Whereas":45210,"tsky":45211,"Ġchall":45212,"WORK":45213,"opian":45214,"rican":45215,"vati":45216,"ĠHTTPS":45217,"Ġwrink":45218,"Ġthrob":45219,"habi":45220,"Ġiodine":45221,"omorph":45222,"ĠScion":45223,"Hunt":45224,"Written":45225,"iosity":45226,"ĠBrowser":45227,"Ġsinners":45228,"culosis":45229,"Ġunconsciously":45230,"0100":45231,"Ġanarchists":45232,"Pull":45233,"FFER":45234,"Ġpandemonium":45235,"matically":45236,"Rush":45237,"Ġpurified":45238,"ĠCyan":45239,"ĠDifficulty":45240,"«":45241,"Aside":45242,"oggles":45243,"untu":45244,"iege":45245,"iberal":45246,"ĠCOUR":45247,"eteenth":45248,"weeney":45249,"biased":45250,"ĠDecay":45251,"quart":45252,"alysis":45253,"Ġstere":45254,"ellect":45255,"Ġkernels":45256,"juven":45257,"ĠJPEG":45258,"indal":45259,"topic":45260,"Ġidentifier":45261,"åı":45262,"Ġepid":45263,"1969":45264,"Ġpoisons":45265,"sym":45266,"mop":45267,"LOCK":45268,"axe":45269,"cohol":45270,"ctory":45271,"Ġadject":45272,"Skin":45273,"ĠFract":45274,"ĠSHAR":45275,"echo":45276,"thood":45277,"Ġencoding":45278,"Ġrelational":45279,"Len":45280,"Bone":45281,"agara":45282,"uggish":45283,"ĠTanks":45284,"Stats":45285,"lihood":45286,"Mult":45287,"Graph":45288,"ĠCannot":45289,"ĠSpac":45290,"handler":45291,"ĠShit":45292,"Ġmorp":45293,"controller":45294,"udeau":45295,"Screenshot":45296,"Development":45297,"Gear":45298,"Ġtong":45299,"ĠColossus":45300,"rylic":45301,"STRUCT":45302,"capitalist":45303,"Ġsupplementation":45304,"Parts":45305,"pb":45306,"oppy":45307,"pite":45308,"processor":45309,"Ġexplanatory":45310,"Environmental":45311,"Compl":45312,"Gaming":45313,"arently":45314,"Ġconcess":45315,"Ġathlet":45316,"forestation":45317,"orsi":45318,"igmat":45319,"Ġencoded":45320,"misc":45321,"Ġproofs":45322,"ĠRevision":45323,"Ġmathematic":45324,"Ġconstitu":45325,"fficiency":45326,"Ġlightsaber":45327,"gz":45328,"erate":45329,"ournals":45330,"Comment":45331,"Ġpercept":45332,".\"[":45333,"ĠTechniques":45334,"coins":45335,"Shape":45336,"venant":45337,"ĠPrinted":45338,"Native":45339,"ĠGors":45340,"pecting":45341,"ĠDuel":45342,"Ġadmins":45343,"Flor":45344,"ĠDeus":45345,"cham":45346,"ĠRails":45347,"ceptor":45348,"naire":45349,"ĠSquid":45350,"ĠWarranty":45351,"SPEC":45352,"ensis":45353,"FUN":45354,"stellar":45355,"Select":45356,"llular":45357,"arget":45358,"ĠUncharted":45359,"Details":45360,"rison":45361,"Ġsyntax":45362,"chanted":45363,"Ġ-----":45364,"Ġthats":45365,"Registration":45366,"ĠSaber":45367,"ethical":45368,"Ġcryptography":45369,"atown":45370,"Ġdependencies":45371,"nw":45372,"Ġvehement":45373,"Ġrationality":45374,"ĠThou":45375,"Ġ----":45376,"rador":45377,"Ġenh":45378,"ĠCrate":45379,"STATE":45380,"/(":45381,"Ġdelim":45382,"CEPT":45383,"monkey":45384,"pai":45385,"uracy":45386,"Ġmortals":45387,"Sanders":45388,"ĠSeraph":45389,"-\"":45390,"1945":45391,"endix":45392,":'":45393,"ĠLegs":45394,"Exper":45395,"ĠKrypt":45396,"clinton":45397,"Ġuphe":45398,"Vers":45399,"Similarly":45400,"ressor":45401,"leans":45402,"LOG":45403,"cific":45404,"Ġ].":45405,"-)":45406,"resist":45407,"Pred":45408,"Latest":45409,"ilyn":45410,"Ġblob":45411,"Ġdevils":45412,"ĠIllusion":45413,"erella":45414,"Ġyak":45415,"method":45416,"Ġ698":45417,"Shadow":45418,"velt":45419,"Ġsomet":45420,"xc":45421,"Ġtriangles":45422,"netic":45423,"Calling":45424,"ĠDRM":45425,"Ġtriglycer":45426,"Ġinhibited":45427,"Ġnep":45428,"Ġalgebra":45429,"ascar":45430,"laim":45431,"Ġappl":45432,"1971":45433,"Bernie":45434,"Eh":45435,"Ġundefined":45436,"âĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶ":45437,"Sys":45438,"ournaments":45439,"Solid":45440,"Ġhep":45441,"ĠMales":45442,"Agent":45443,"Ġpsychedel":45444,"Wik":45445,"Ġdoctrines":45446,"rection":45447,"Compare":45448,"âĺ":45449,"Ġcertific":45450,"Ġsubstr":45451,"ĠCitation":45452,"ĠAFB":45453,"ĠBecame":45454,"Ġaristocracy":45455,"aryl":45456,"Ġanatomical":45457,"ocumented":45458,"ĠAssy":45459,"ĠFORM":45460,"Traditional":45461,"azines":45462,"Content":45463,"furt":45464,"Ġscripting":45465,"Ġcloaked":45466,"Ġunint":45467,"ĠCivilization":45468,"Desktop":45469,"ĠRagnar":45470,"Ġcurses":45471,"Ġobservable":45472,"ĠSpock":45473,"ĠPyr":45474,"Ġelectrom":45475,"ĠLump":45476,"oresc":45477,"ĠAttribution":45478,"egal":45479,"achusetts":45480,"Ġmarqu":45481,"âĻ¦":45482,"Ġcursor":45483,"ascist":45484,"1966":45485,"edit":45486,"lisher":45487,"ocyte":45488,"Writer":45489,"BILITIES":45490,"ĠUpload":45491,"Ġtreacher":45492,"Ġrecomb":45493,"Ġknights":45494,"Ġimmutable":45495,"ĠPly":45496,"Ġatten":45497,"ĠPassed":45498,"Flying":45499,"icipated":45500,"querade":45501,"ĠZot":45502,"CRE":45503,"ĠCursed":45504,"ickr":45505,"ĠDroid":45506,"thereum":45507,"Ġadjective":45508,"DIT":45509,"Ġtob":45510,"Ġinit":45511,"ĠPenet":45512,"Ġignor":45513,"Ġexalted":45514,"ĠDwell":45515,"assemb":45516,"Ġsentient":45517,"Ġ``":45518,"ĠGoo":45519,"Professional":45520,"othing":45521,"rupted":45522,"olics":45523,"ĠSetup":45524,"Thu":45525,"Campaign":45526,"Secondly":45527,"clipse":45528,"hibit":45529,"amate":45530,"SUP":45531,"ĠSuppose":45532,"submit":45533,"ĠDebian":45534,"Ġantid":45535,"Ġentert":45536,"ysical":45537,"ĠGladiator":45538,"ĠSTL":45539,"ĠBugs":45540,"ĠMech":45541,"ĠCoffin":45542,"itored":45543,"ICLE":45544,"Mist":45545,"Ġinfall":45546,"votes":45547,"actly":45548,"Occ":45549,"ĠConquest":45550,"alach":45551,"Ġintertw":45552,"reverse":45553,"amiya":45554,"icularly":45555,"edom":45556,"ĠLuxem":45557,"Fra":45558,"urrencies":45559,"Ġnobility":45560,"Tab":45561,"Beer":45562,"Ġ10000":45563,"Ġincor":45564,"Ġmelanch":45565,"Depth":45566,"Firstly":45567,"usr":45568,"ĠWiki":45569,"hhhh":45570,"ĠProxy":45571,"Ġantagonists":45572,"Ġtransistor":45573,"ĠRelic":45574,"ĠPrometheus":45575,"Ġ1280":45576,"Coun":45577,"ĠMedals":45578,"stats":45579,"Assembly":45580,"inished":45581,"cemic":45582,"Ġadventurers":45583,"Ġcd":45584,"Supporters":45585,"ĠYs":45586,"])":45587,"Ġneglig":45588,"Request":45589,"Ġwhore":45590,"Ġovercl":45591,"_-":45592,"partial":45593,"amd":45594,"Ġfructose":45595,"Ġdivid":45596,"Administ":45597,"amples":45598,"Boo":45599,"akery":45600,"owered":45601,"hester":45602,"Links":45603,"GROUND":45604,"ethy":45605,"Ġincarcer":45606,"Ġincap":45607,"Drag":45608,"ĠElastic":45609,"âĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶ":45610,"Ultra":45611,"AAAA":45612,"Order":45613,"ĠMysteries":45614,"Ġcanonical":45615,"Ign":45616,"Ġanimate":45617,"wegian":45618,"ggle":45619,"Hash":45620,"Arg":45621,"verty":45622,"Ġanalges":45623,"ouver":45624,"ittees":45625,"ĠAsgard":45626,"______":45627,"Mix":45628,"1964":45629,"Rate":45630,"Ġarousal":45631,"pheus":45632,"undai":45633,"hetamine":45634,"ĠMysterious":45635,"Alright":45636,"ĠHerod":45637,"riott":45638,"ĠAnarchy":45639,"ĠArche":45640,"Question":45641,"Chapter":45642,"Token":45643,"ĠSphere":45644,"Ġinduces":45645,"Audio":45646,"Normal":45647,"Ġprophe":45648,"ĠValiant":45649,"Tag":45650,"Relations":45651,"Ġblinked":45652,"onyms":45653,"ĠVortex":45654,"Ġdb":45655,"emonic":45656,"Phase":45657,"Ġkingdoms":45658,"Twe":45659,"ĠLORD":45660,"plementation":45661,"ĠConstantinople":45662,"helm":45663,"ĠFlesh":45664,"Ġthumbnail":45665,"ledged":45666,"ĠPROG":45667,"Ġdisbel":45668,"ĠLikes":45669,"ĠGamer":45670,"renches":45671,"hattan":45672,"Index":45673,"pecially":45674,"ĠJiu":45675,"Ġwhats":45676,"erion":45677,"xf":45678,"ĠPerception":45679,"Alien":45680,"Capt":45681,"ãĢĤ":45682,"joining":45683,"nesium":45684,"ĠSocrates":45685,"Icon":45686,"animate":45687,"ocalypse":45688,"ĠTactics":45689,"assador":45690,"Veh":45691,"src":45692,",-":45693,"Ġvisc":45694,"ĠDiscord":45695,"initial":45696,"atana":45697,"Size":45698,"Claim":45699,"ffect":45700,"iciary":45701,"Ġturret":45702,"reset":45703,"Ï":45704,"wrap":45705,"ulnerability":45706,"ĠInsert":45707,"Ġirrad":45708,"ognitive":45709,"clips":45710,"uncle":45711,"chemy":45712,"ottesville":45713,"Write":45714,"earances":45715,"1965":45716,"MIC":45717,"Ġmanag":45718,"Ġtelesc":45719,"Termin":45720,"Guest":45721,"Ġdenote":45722,"Failure":45723,"ograp":45724,"âĢķ":45725,"Ġscrolls":45726,"ĠArmored":45727,"Ġrecomp":45728,"Ġplaceholder":45729,"ĠISBN":45730,"ĠBelief":45731,"emporary":45732,"Asset":45733,"arcer":45734,"haar":45735,"assium":45736,"%:":45737,"ernal":45738,"ĠLv":45739,"atible":45740,"Pand":45741,"oubted":45742,"Lie":45743,"bial":45744,"STEP":45745,"Ġpresets":45746,"Ġstatist":45747,"Sund":45748,"reshold":45749,"endium":45750,"\");":45751,"Software":45752,"Ġbasal":45753,"ĠYose":45754,"Ġmortg":45755,"ocry":45756,"Ġsubreddit":45757,"omorphic":45758,"ĠLoaded":45759,"berra":45760,"vg":45761,"orkshire":45762,"ĠChrys":45763,"Repeat":45764,"ĠSimulator":45765,"rx":45766,"gex":45767,"Linux":45768,"ĠInstruct":45769,"irable":45770,"Ġmosquit":45771,"ĠManga":45772,"iOS":45773,"Ġsynt":45774,"Ġclitor":45775,"Ġlobe":45776,"ĠDelete":45777,"CVE":45778,"fortunately":45779,"Enc":45780,"vertising":45781,"Ġanten":45782,"Ġfif":45783,"Study":45784,"prev":45785,"ossus":45786,"Nar":45787,"Decl":45788,"erala":45789,"ĠPrototype":45790,"UGE":45791,"1001":45792,"Ġ---------":45793,"deals":45794,"odcast":45795,"TPS":45796,"Ġcodec":45797,"ittee":45798,"isexual":45799,"ĠBreaker":45800,"menu":45801,"ĠURI":45802,"('":45803,"ĠFiorina":45804,"ĠApostles":45805,"ĠWitches":45806,"raint":45807,"addafi":45808,"ersive":45809,"yrim":45810,"Ġmosa":45811,"Ġrog":45812,"Ear":45813,"âĺħ":45814,"Ġcaloric":45815,"matical":45816,"yrics":45817,"ĠKrugman":45818,"axter":45819,"1016":45820,"Ġsep":45821,"ĠExtend":45822,"ropolitan":45823,"thren":45824,"ologne":45825,"atomic":45826,"Naturally":45827,"Pros":45828,"gencies":45829,"akens":45830,"Male":45831,"Ġcausation":45832,"omnia":45833,"Comments":45834,"eeee":45835,"iquette":45836,"Ġcytok":45837,"ename":45838,"details":45839,"Ġdestruct":45840,"leep":45841,"ĠCavern":45842,"ĠInvention":45843,"ueless":45844,"Ġsubsection":45845,"outhern":45846,"metic":45847,"blogs":45848,"ĠPacks":45849,"ĠArduino":45850,"hhh":45851,"elligence":45852,"imity":45853,"ĠUltron":45854,"astrous":45855,"Ġbiome":45856,"ĠHover":45857,"Ġprivile":45858,"igham":45859,"apest":45860,"ĠYoshi":45861,"Artist":45862,".\",":45863,"gamer":45864,"Virgin":45865,"Tea":45866,"ĠDoomsday":45867,"ĠðŁĻĤ":45868,"terday":45869,"ĠCommando":45870,"ĠAchieve":45871,"chrom":45872,"Ġcryptographic":45873,"Ġrebell":45874,"Specifically":45875,"âĢ¦âĢ¦âĢ¦âĢ¦":45876,"ĠEternity":45877,"Ġemulation":45878,"ĠSERV":45879,"ĠMiscellaneous":45880,"ĠParticipant":45881,"duc":45882,"vp":45883,"ĠSparkle":45884,"ategories":45885,"Ġdecrypt":45886,"ĠGNOME":45887,"activation":45888,"Ġanarch":45889,"owler":45890,"adiator":45891,"itars":45892,"ĠTHEN":45893,")\",":45894,"åħ":45895,"Ġembod":45896,"vae":45897,"âĺĨ":45898,"Member":45899,"Ġrm":45900,"nyder":45901,"ĠLeviathan":45902,"Gaza":45903,"erenn":45904,"Chicken":45905,"ĠDefinitive":45906,"ĠBolshe":45907,"ĠJagu":45908,"gorith":45909,"loader":45910,"exe":45911,".........":45912,"ĠReceived":45913,"ĠProto":45914,"ĠLocked":45915,"Posts":45916,"ankind":45917,"Clock":45918,"ĠCLI":45919,"Throw":45920,"dL":45921,"epad":45922,"ĠAtmosp":45923,"Ġmk":45924,"ĠSteal":45925,"uple":45926,"reference":45927,"ĠGNU":45928,"adelphia":45929,"scripts":45930,"ilaterally":45931,"ĠMods":45932,"odus":45933,"ignty":45934,"REF":45935,"Ġhypothesized":45936,"issors":45937,"Ġanus":45938,"HUD":45939,"rices":45940,"Draw":45941,"Computer":45942,"Below":45943,"uthor":45944,"ĠTact":45945,"=$":45946,"00000000":45947,"Ġcaut":45948,"Sharp":45949,"depend":45950,"Ġtatt":45951,"Goal":45952,"Sounds":45953,"zona":45954,"anyon":45955,"ricanes":45956,"ĠUSAF":45957,"Jump":45958,"Bottom":45959,"etermination":45960,"ĠPles":45961,"Ġhypothes":45962,"Reference":45963,"Ġswall":45964,"Ġmaneu":45965,"rifice":45966,"ĠVeh":45967,"Ġtex":45968,"geoning":45969,"ĠâľĶ":45970,"Mach":45971,"eanor":45972,"%);":45973,"archives":45974,"Ġencyclopedia":45975,"ĠPreferences":45976,"damage":45977,"Done":45978,"Ġcoefficient":45979,"ĠCreatures":45980,"Ġital":45981,"ivari":45982,"Revolution":45983,"Ġnob":45984,"Diff":45985,"Ġabbre":45986,"Writ":45987,"ĠDOS":45988,"redd":45989,"Ġsplend":45990,"orest":45991,"flame":45992,"Ġdevs":45993,"Ġ==":45994,"ĠPuzzle":45995,"Ġgit":45996,"MOD":45997,"ĠArgument":45998,"ĠAbyss":45999,"Studies":46000,"ophob":46001,"uild":46002,"scill":46003,"fp":46004,"Ġplur":46005,"Delete":46006,"ĠFALSE":46007,"FIL":46008,"Ġmicrobiota":46009,"ĠIPv":46010,"Stud":46011,"ortal":46012,"ĠDivinity":46013,"ounter":46014,"ä¸":46015,"Naz":46016,"stals":46017,"ihilation":46018,"Ġpersecut":46019,"ĠPlanes":46020,"viation":46021,"Driver":46022,"ĠEEG":46023,"Unity":46024,"Premium":46025,"ĠSiren":46026,"ĠPaleo":46027,"earchers":46028,"Pract":46029,"Ö":46030,"VII":46031,"mosp":46032,"Ġidentifiers":46033,"Near":46034,"achu":46035,"Apps":46036,"tackle":46037,"COLOR":46038,"Ġperpendicular":46039,"viks":46040,"ecided":46041,"ĠDota":46042,"icons":46043,"Ġpsi":46044,"Brave":46045,"Ġunimagin":46046,"ĠATI":46047,"OOL":46048,"Gender":46049,"ĠSwords":46050,"oples":46051,"Rank":46052,"olphins":46053,"Ġdeities":46054,"ĠXIII":46055,"м":46056,"ĠKraken":46057,"ĠLEVEL":46058,"stasy":46059,"ĠBabel":46060,"Hours":46061,"Avoid":46062,"Mech":46063,"Multi":46064,"Ġect":46065,"Occup":46066,"panic":46067,"Ġmutants":46068,"Evidence":46069,"Tips":46070,"Ġvolts":46071,"Exit":46072,"xb":46073,"planet":46074,"avez":46075,"features":46076,")]":46077,"lol":46078,"ĠNeph":46079,"ĠSanct":46080,"Ġimpover":46081,"................................":46082,"Sty":46083,"Email":46084,"Torrent":46085,"Ġgluc":46086,"ĠSins":46087,"ĠIncarn":46088,"ĠWITHOUT":46089,"ĠPanzer":46090,"ĠAssignment":46091,"versible":46092,"Strange":46093,"ITNESS":46094,"incible":46095,"ZX":46096,"ĠMySQL":46097,"Ġconson":46098,"Ġoxidative":46099,"Machine":46100,"Impro":46101,"Parent":46102,"ĠMetroid":46103,"Educ":46104,"Ġdismant":46105,"dx":46106,"ĠPersona":46107,"ĠHDL":46108,"Americ":46109,"Users":46110,"Ġeighteenth":46111,"WARNING":46112,"ĠLists":46113,"ĠCanter":46114,"ĠTrotsky":46115,"Ġhaha":46116,"]'":46117,"ĠEncyclopedia":46118,"admin":46119,"ĠACTIONS":46120,"idav":46121,"ο":46122,"ĠFTP":46123,"Ġquar":46124,"ongyang":46125,"âĢ¦âĢ¦âĢ¦âĢ¦âĢ¦âĢ¦âĢ¦âĢ¦":46126,"Ġsynchronization":46127,"DEM":46128,"riched":46129,"Ġnegro":46130,"Bench":46131,"Ġfilament":46132,"Ġdecoding":46133,"obj":46134,"Ġjoystick":46135,"Decre":46136,"ĠBolshevik":46137,"Virtual":46138,"ĠSacrament":46139,"xd":46140,"BILL":46141,"-+-+":46142,"¶":46143,"anchester":46144,"Pokemon":46145,"Ġslic":46146,"iameter":46147,"errilla":46148,"Exactly":46149,"\"'":46150,"getic":46151,"3333":46152,"solete":46153,"Ġincorpor":46154,"Ġio":46155,"------------":46156,"Ġantiquity":46157,"ATURES":46158,"Policy":46159,"oppable":46160,"Ġ=>":46161,"ODUCT":46162,"otide":46163,"Ú":46164,"Ġnormative":46165,"Fac":46166,"Ġshaman":46167,"element":46168,"Plex":46169,"INTER":46170,"etsk":46171,"ĠGauntlet":46172,"ĠBIOS":46173,"×ķ":46174,"riet":46175,"Rew":46176,"uristic":46177,"urches":46178,"ĠChomsky":46179,"ixir":46180,"package":46181,"Owner":46182,"Ġschematic":46183,"Assistant":46184,"Ġemanc":46185,"Ġarchetype":46186,"Initial":46187,"intent":46188,"Ġfilib":46189,"ispers":46190,"Flag":46191,"Tank":46192,"Ġinsurg":46193,"Ġapproximation":46194,"Ġsemantic":46195,"Ġsubtitle":46196,"Font":46197,"Ġintimid":46198,"Ġhath":46199,"tools":46200,"gob":46201,"Process":46202,"slave":46203,"ĠJUSTICE":46204,"âĻ¥":46205,"ĠHardcore":46206,"Discover":46207,"Ġexch":46208,"ptive":46209,"units":46210,"ĠDjango":46211,"itudinal":46212,"Ġpc":46213,"akespeare":46214,"ospace":46215,"Ġhorny":46216,"auth":46217,"ĠSkyrim":46218,"ENGTH":46219,"perors":46220,"ĠVulkan":46221,"Ġchimpan":46222,"Ġremem":46223,"Ġopacity":46224,"Ġ:(":46225,"ushima":46226,"Ġawoken":46227,"Ġsacrament":46228,"Beginning":46229,"escape":46230,"Anim":46231,"Ġadvant":46232,"ĠRequires":46233,"output":46234,"Ġdroid":46235,"Yep":46236,"rieving":46237,"Ġpt":46238,"ĠShotgun":46239,"ĠOsiris":46240,"disabled":46241,"ĠRadius":46242,"Medium":46243,"ĠScient":46244,"ĠRept":46245,"ymm":46246,"Ġcp":46247,"ĠLabyrinth":46248,"poral":46249,"Ġ'(":46250,"Hack":46251,"ĠTechnique":46252,"/,":46253,"Ġambig":46254,"Basic":46255,"Ġretrie":46256,"VICE":46257,"BIP":46258,"ragon":46259,"phies":46260,"uminum":46261,"ĠFei":46262,"lesi":46263,"Ġsemantics":46264,"ĠHz":46265,"ĠUnderworld":46266,"Ġendot":46267,"olesterol":46268,"ourning":46269,"Ġcaches":46270,"ĠYug":46271,"Legendary":46272,"ĠDocumentation":46273,"ĠSpiral":46274,"ĠClone":46275,"bnb":46276,"ĠâĶ":46277,"ustom":46278,"Mp":46279,"gettable":46280,"agonist":46281,"Ġneuronal":46282,"culus":46283,"enum":46284,"cules":46285,"Ġmuttered":46286,"ctica":46287,"necess":46288,"ĠSubtle":46289,"Ġsolder":46290,"Environment":46291,"oneliness":46292,"orage":46293,"âĢ¦.\"":46294,"nesota":46295,"agements":46296,"Ùİ":46297,"WHERE":46298,"ĠGDDR":46299,"Scient":46300,"ĠMulcair":46301,"ĠRena":46302,"________________________________________________________________":46303,"antics":46304,"Ġtorped":46305,"Brow":46306,"ossal":46307,"Category":46308,"Regular":46309,"remote":46310,"ãģ":46311,"ĠCoil":46312,"ritch":46313,"specified":46314,"Average":46315,"Ġfingert":46316,"entity":46317,"atibility":46318,"ampunk":46319,"ĠScriptures":46320,"Ġunequ":46321,"arettes":46322,"arching":46323,"Ġastron":46324,"Ġnumeric":46325,"ĠeBook":46326,"remove":46327,"onday":46328,"Ġmetaphysical":46329,"ĠGoku":46330,"Element":46331,"ĠRuin":46332,"Norm":46333,"Ġtox":46334,"puff":46335,"Ġharmonic":46336,"ĠAgility":46337,"ĠHearthstone":46338,"Ġmana":46339,"Points":46340,"Ġconduc":46341,"ĠPersia":46342,"-----":46343,"license":46344,"Application":46345,"assert":46346,"Reader":46347,"ĠSacrifice":46348,"float":46349,"inctions":46350,"byter":46351,"Ġfundament":46352,"\"âĢ¦":46353,"Fourth":46354,"Effective":46355,"ĠMeow":46356,"ĠErrors":46357,"ĠIcar":46358,"ĠMMO":46359,"Ġapostles":46360,"Ġfaintly":46361,"component":46362,"bably":46363,"uggage":46364,"ĠMPG":46365,"krit":46366,"container":46367,"ixture":46368,"ĠPOV":46369,"izabeth":46370,"onut":46371,"isdom":46372,"trace":46373,"ĠSDL":46374,"Interestingly":46375,"ĠExplan":46376,"lesiastical":46377,"ternal":46378,"Bug":46379,"Ġmetabolites":46380,"geries":46381,"Ġsupra":46382,"ĠMakoto":46383,"orget":46384,"racuse":46385,"][":46386,"ĠPrelude":46387,"peria":46388,"tube":46389,"ĠCatalog":46390,"ĠGoblin":46391,"QUEST":46392,"ĠINCLUD":46393,"ĠVERS":46394,"erguson":46395,"Ġcommandments":46396,"ĠUDP":46397,"itle":46398,"ι":46399,"domain":46400,"roximately":46401,"ĠTLS":46402,"ongevity":46403,"Ġmodulation":46404,"Ġdidnt":46405,"ĠCalories":46406,"Applications":46407,"ormon":46408,"Ġsd":46409,"dullah":46410,"Ġcous":46411,"ĠDARK":46412,"clip":46413,"ĠPsychiat":46414,"ĠTanz":46415,"ĠCharisma":46416,"ĠMerge":46417,"ĠKDE":46418,"requires":46419,"urdue":46420,"Ġdecimal":46421,"Ġâī¥":46422,"ĠAuth":46423,"ebted":46424,"ĠTempl":46425,"ĠâĢº":46426,"Ultimate":46427,"Ġmammalian":46428,"advertising":46429,"Ġdominion":46430,"Ġacron":46431,"ĠWem":46432,"ĠHeist":46433,"oiler":46434,"FLAG":46435,"ovember":46436,"Syn":46437,"Ġgodd":46438,"ĠPyth":46439,"Ġglyc":46440,"ĠHelpful":46441,"Ġgad":46442,"chedel":46443,"Similar":46444,"Ġ¶":46445,"Ġnp":46446,"ĠREPL":46447,"Fill":46448,"ĠSunder":46449,"etsy":46450,"ĠPAX":46451,"ĠFemales":46452,"ĠKingdoms":46453,"Ġwhistlebl":46454,"Hide":46455,"serial":46456,"ĠEnemies":46457,"ĠPeb":46458,"Ġpiety":46459,"ifact":46460,"esity":46461,"bsite":46462,"esides":46463,"Ġported":46464,"Ġamygdala":46465,"ĠGerr":46466,"afety":46467,"Ġadip":46468,"(\"":46469,"Ġcf":46470,"Ġurl":46471,"unia":46472,"icro":46473,"Austral":46474,"ĠConfig":46475,"accompanied":46476,"isite":46477,"Ġtextual":46478,"\">":46479,"Ġanecd":46480,"Ġ\",":46481,"angular":46482,"ĠUnicode":46483,"Proof":46484,"Ġmultiplication":46485,"Address":46486,"Ġbytes":46487,"lems":46488,"uterte":46489,"Episode":46490,"oshop":46491,"ritical":46492,"Adjust":46493,"argument":46494,"\\'":46495,"Rober":46496,"pection":46497,"Agg":46498,"äº":46499,"interrupted":46500,"ĠDebor":46501,"Ġlair":46502,"Various":46503,"isively":46504,"ĠStatic":46505,"ohyd":46506,"ĠEchoes":46507,"UID":46508,"raught":46509,"Bott":46510,"Ġapostle":46511,"ĠCentauri":46512,"oxicity":46513,"ibling":46514,"Ġparalle":46515,"inav":46516,"Crit":46517,"ĠTyph":46518,"Ġhig":46519,"ĠEDITION":46520,"Ġcoord":46521,"uish":46522,"sectional":46523,"inki":46524,"Title":46525,"anyahu":46526,"osterone":46527,"Ġdesper":46528,"ribly":46529,"Legend":46530,"afort":46531,"Org":46532,"Ġempir":46533,"ĠQuake":46534,"SSL":46535,"ioxide":46536,"åľ":46537,"Ġenz":46538,"urtle":46539,"BSD":46540,"Rust":46541,"ospels":46542,"Rare":46543,"Ġpartitions":46544,"Ġheresy":46545,"overy":46546,"Ġmonop":46547,"Pixel":46548,"odder":46549,"Option":46550,"withstanding":46551,"Transfer":46552,"Ġarrog":46553,"skip":46554,"ĠSSH":46555,"ĠSph":46556,"Ġcallback":46557,"PIN":46558,"Ġpdf":46559,"Ġplaint":46560,"cipled":46561,"reenshots":46562,"Ġparsing":46563,"::::::::":46564,"ioxid":46565,"Ġhereafter":46566,"ĠFunctions":46567,"ĠBulgar":46568,"Ġintu":46569,"DOC":46570,"Location":46571,"Hyper":46572,"ageddon":46573,"Evil":46574,"illions":46575,"Introduction":46576,"Physical":46577,"ĠLayout":46578,"âķ":46579,"------------------------":46580,"ĠRodham":46581,"ĠPatterns":46582,"Delivery":46583,"Ġdistur":46584,"ĠVolunte":46585,"ĠGUI":46586,"Ġclen":46587,"Ġinacc":46588,"ĠBallistic":46589,"ĠSprite":46590,"Privacy":46591,"theme":46592,"dump":46593,"ĠByte":46594,"ĠIncre":46595,"apult":46596,"ĠWrath":46597,"ensibly":46598,"NOTE":46599,"ounge":46600,"ustomed":46601,"ochond":46602,"ĠQt":46603,"Primary":46604,"Ġsidew":46605,"Root":46606,"gregation":46607,"SQL":46608,"ĠSOFTWARE":46609,"Gallery":46610,"ĠDungeon":46611,"ĠVengeance":46612,"->":46613,"steam":46614,"Ġfrivol":46615,"Ġpid":46616,"filter":46617,"Ġfacult":46618,"doms":46619,"Tool":46620,"1959":46621,"Ġprefix":46622,"Ġcomma":46623,"relative":46624,"Ġformatted":46625,"appropriately":46626,"Ġmd":46627,"xxx":46628,"ĠAuthentication":46629,"ĠWTC":46630,"Ġvulner":46631,"reditary":46632,"Steam":46633,"Tx":46634,"ĠGHC":46635,"Increased":46636,"forcement":46637,"ĠGuant":46638,"bernatorial":46639,"Entry":46640,"ĠWarp":46641,"ĠCreature":46642,"ĠAmmunition":46643,"Ġclust":46644,"ĠInher":46645,"Ġunbel":46646,"RGB":46647,"ĠMankind":46648,"ĠPlague":46649,"Ġ=================================":46650,"psc":46651,"Intern":46652,"tml":46653,"ĠCrusade":46654,"inflamm":46655,"Storage":46656,"token":46657,"inse":46658,"False":46659,"Adult":46660,"Pokémon":46661,"PLIED":46662,"Ġglac":46663,"ĠDwarf":46664,"sequence":46665,"Ġmagnification":46666,"ĠIlluminati":46667,"hedral":46668,"param":46669,"regon":46670,".\",\"":46671,"Eva":46672,"igree":46673,"Object":46674,"Ġoptimizations":46675,"uador":46676,"mmmm":46677,"ullivan":46678,"Ġ[\"":46679,"ĠDusk":46680,"Ġtrig":46681,"Ġiss":46682,"Ġhypert":46683,"Ġperspect":46684,"Ġassum":46685,":,":46686,"Ġinterpol":46687,"Asked":46688,"Boot":46689,"LIB":46690,"Loading":46691,"Ident":46692,"upuncture":46693,"ioch":46694,"Ġprefrontal":46695,"delay":46696,"ĠPoké":46697,"bestos":46698,"overe":46699,"Elf":46700,"eteria":46701,"ĠSneak":46702,"bians":46703,"ĠARTICLE":46704,"Xbox":46705,"encrypted":46706,"ync":46707,"ĠNietzsche":46708,"Nonetheless":46709,"Ġ±":46710,"ĠPrimal":46711,"ĠFlare":46712,"Ġconflic":46713,"ĠRune":46714,"Tes":46715,"cellence":46716,"Mega":46717,"ĠEntity":46718,"chrome":46719,"iatures":46720,"Ġuninstall":46721,"Winner":46722,"aimon":46723,"Ġhomebrew":46724,"Ruby":46725,"araoh":46726,"itime":46727,"Ġpotion":46728,"ĠAllows":46729,"ogyn":46730,"osuke":46731,"Limited":46732,"Ġmacros":46733,"ERROR":46734,"gling":46735,"Ġtodd":46736,"repre":46737,"ĠSakura":46738,"erker":46739,"items":46740,"FIG":46741,"ĠUnle":46742,"Ġhardness":46743,"Split":46744,"Ġarous":46745,"ocally":46746,"Ġì":46747,"ĠEVE":46748,"pleasant":46749,"ihil":46750,"ĠRouter":46751,"ĠLucius":46752,"readable":46753,"Ġtremb":46754,"Dro":46755,"Ġblaster":46756,"Ġbourgeoisie":46757,"NUM":46758,"Alternative":46759,"flags":46760,"GAME":46761,"ebook":46762,"ĠIPM":46763,"Ġcorrel":46764,"Setting":46765,"Frame":46766,"Ġatheism":46767,"Interested":46768,"Liquid":46769,"stanbul":46770,"Lv":46771,"Ġtits":46772,"Ġdc":46773,"×Ļ×":46774,"Ġdoctr":46775,"background":46776,"tsy":46777,"ĠCtrl":46778,"ĠCompatibility":46779,"idae":46780,"example":46781,"perture":46782,"Ġguid":46783,"ĠWinged":46784,"Command":46785,"ridor":46786,"bool":46787,"comments":46788,"ĠImmunity":46789,"Nit":46790,"Statement":46791,"Ġmanif":46792,"ĠIntake":46793,"Bloom":46794,"txt":46795,"context":46796,"input":46797,"achus":46798,"proc":46799,"Ñĭ":46800,"Ġdisemb":46801,"ospons":46802,"utical":46803,"ĠRender":46804,"Ironically":46805,"ursday":46806,"ĠExile":46807,"lishes":46808,"iets":46809,"orescent":46810,"cair":46811,"ĠSubjects":46812,"ĠDungeons":46813,"Ġiii":46814,"neapolis":46815,"ĠBlaster":46816,"Ġphp":46817,"ORED":46818,"ĠSLI":46819,"Ġelig":46820,"ĠIdentified":46821,"ĠBrawl":46822,"bytes":46823,"ĠCTR":46824,"Ġsched":46825,"Assuming":46826,"Bound":46827,"ĠMathemat":46828,"razil":46829,"ĠAstral":46830,"mble":46831,"untled":46832,"Ġmech":46833,"ĠDagger":46834,"ĠUseful":46835,"nesday":46836,"tarians":46837,"AMY":46838,"Camera":46839,"node":46840,"pict":46841,"ginx":46842,"Ġyea":46843,">>>>>>>>":46844,"paragraph":46845,"ĠSupplementary":46846,"9999":46847,"ĠAlchemist":46848,"uzzle":46849,"igun":46850,"ĠCalculator":46851,"ĠApplicant":46852,"hift":46853,"ĠGPL":46854,"Ġencode":46855,"Crash":46856,"ĠNutr":46857,"kHz":46858,"TABLE":46859,"intestinal":46860,"andom":46861,"archive":46862,"Ëľ":46863,"Registered":46864,"Questions":46865,"Remote":46866,"ethyst":46867,"Ġgren":46868,"ĠTexture":46869,"Ġseiz":46870,"Anyway":46871,"ĠVariant":46872,"ê":46873,"Adapt":46874,"ittered":46875,"meta":46876,"ambers":46877,"ĠRuins":46878,"ĠChimera":46879,"password":46880,"ĠReboot":46881,"Ġcaster":46882,"Ġamplitude":46883,"Position":46884,"Ġnotation":46885,"Ġsecretion":46886,"Excellent":46887,"delete":46888,"aminer":46889,"ä»":46890,"Exec":46891,"ĠKenobi":46892,"Interview":46893,"ontent":46894,"ospel":46895,"Ġtuber":46896,"CONT":46897,"roups":46898,"Ġemulator":46899,"Ġjava":46900,"0200":46901,"Ġnested":46902,"Ġfert":46903,")).":46904,"Dex":46905,"ĠSora":46906,"Ġpotions":46907,"ĠAnon":46908,"aah":46909,"Ġdunno":46910,"Ġμ":46911,"Ġmethodological":46912,"itles":46913,"phia":46914,"Beg":46915,"Rules":46916,"ĠXML":46917,"Ġflask":46918,"ĠShogun":46919,"Ġ2048":46920,"atchewan":46921,"Ġfuckin":46922,"Built":46923,"Ġbour":46924,"Ġdisag":46925,"yss":46926,"ĠÏ":46927,"Spoiler":46928,"Wiki":46929,"Ġmorphology":46930,"Ġendors":46931,"Ġdungeons":46932,"dragon":46933,")),":46934,"Ġhous":46935,"Ġoverwhel":46936,"SAY":46937,"abwe":46938,"--------------------------------":46939,"Ġepist":46940,"Ġpalp":46941,"ĠExtensions":46942,"ĠMistress":46943,"ĠUkrain":46944,"================":46945,"edience":46946,"abama":46947,"ĠLua":46948,"ĠOffline":46949,"ĠKonami":46950,"unicip":46951,"ĠMachina":46952,"Specific":46953,"Ġpresupp":46954,"ĠGEAR":46955,"rition":46956,"rences":46957,"successfully":46958,"Ġ1024":46959,"Platform":46960,"}}":46961,"clude":46962,"roxy":46963,"Ġpromot":46964,"ĠAdapter":46965,"rocal":46966,"ĠMasquerade":46967,"Panel":46968,"Language":46969,"elsius":46970,"Push":46971,"abase":46972,"ĠdB":46973,"argon":46974,"ĠRemoved":46975,"amph":46976,"ĠWyr":46977,"Ġindisp":46978,"ĠOkin":46979,"aepernick":46980,"moil":46981,"Continue":46982,"00007":46983,"ĠJournals":46984,"TAG":46985,"ĠRemastered":46986,"Ġsymp":46987,"methyl":46988,"Overview":46989,"umeric":46990,"ĠCodex":46991,".$":46992,"ranged":46993,"Sym":46994,"ĠVerse":46995,"ĠEnabled":46996,"ĠFUCK":46997,"ĠHearth":46998,"Ġbrill":46999,"ĠChaser":47000,"Beh":47001,"ĠAlchemy":47002,"Oracle":47003,"roleum":47004,"ĠVoldemort":47005,"();":47006,"Ġcollaps":47007,"Visual":47008,"ĠAngular":47009,"ĠOsc":47010,"ichita":47011,"Ġcig":47012,"Ġtoolbar":47013,"ĠEnlight":47014,"ÑĮ":47015,"ε":47016,"aliation":47017,"ĠLovecraft":47018,"jri":47019,"ĠInterstellar":47020,"Ġdebugging":47021,"Ġparentheses":47022,"ĠInit":47023,"Located":47024,"Weak":47025,"ĠPvP":47026,"ĠCloak":47027,"uture":47028,"iths":47029,"asionally":47030,"FACE":47031,"Introdu":47032,"');":47033,"slot":47034,"aturday":47035,"ĠNiet":47036,"Ġpuzz":47037,"!!!!!!!!":47038,"folios":47039,"Ç":47040,"Ġverbs":47041,"ĠFrames":47042,"ĠAmbro":47043,"Ġmillisec":47044,"ĠRebell":47045,"ylum":47046,"PASS":47047,"ĠConfiguration":47048,"μ":47049,"brids":47050,"vantage":47051,"Ġ['":47052,"ĠScy":47053,"Benef":47054,"gradation":47055,"ĠOrc":47056,"Resources":47057,"Awesome":47058,"ĠMilitia":47059,"POST":47060,"Ġbinaries":47061,"Mode":47062,"Ġkb":47063,"ĠWARRANT":47064,"hemy":47065,"Desc":47066,"alion":47067,"Ġwiki":47068,"Ġcommer":47069,"Serial":47070,"ĠUncommon":47071,"ignore":47072,"Ġconstructor":47073,"ctl":47074,"Ġ):":47075,"ĠVerify":47076,"Notice":47077,"ĠRPGs":47078,"uckland":47079,"Ġincre":47080,"Pinterest":47081,"ĠDefinitions":47082,"iband":47083,"Ġtd":47084,"Ġsubscrib":47085,"Shin":47086,"ĠGadget":47087,"Document":47088,"å®":47089,"Requ":47090,"QUIRE":47091,"ĠQuadro":47092,"ĠUnix":47093,"Enlarge":47094,"thens":47095,"\"...":47096,"gebra":47097,"pload":47098,"alogue":47099,"vironments":47100,"Strength":47101,"ĠPID":47102,"ĠInvaders":47103,"HOME":47104,"Atl":47105,"ĠBlizz":47106,"ĠWidth":47107,"ĠOpenGL":47108,"zx":47109,"$,":47110,"Ġå":47111,"cig":47112,"lectic":47113,"relation":47114,"Ġfeas":47115,"undown":47116,"Said":47117,"ν":47118,"��":47119,"english":47120,"ĠTokens":47121,"ĠALEC":47122,"OOOO":47123,"isconsin":47124,"Ġconstants":47125,"ĠTemplar":47126,"Accept":47127,"Ġmascul":47128,"enegger":47129,"ampires":47130,"Rated":47131,"lua":47132,"ucl":47133,"ĠSequence":47134,"ĠNRS":47135,"STD":47136,"Cra":47137,"autions":47138,"ĠKernel":47139,"oleon":47140,"htaking":47141,"ancial":47142,"Pages":47143,"orthodox":47144,"ropy":47145,"EEE":47146,"Ġtranssexual":47147,"?????":47148,"Ġsurpr":47149,"arthy":47150,"ĠPsychic":47151,"Ġdorsal":47152,"cember":47153,"joice":47154,"/+":47155,"verend":47156,"uint":47157,"Ġderog":47158,"Subject":47159,"hemat":47160,"!]":47161,"Ġ);":47162,"Ġmeshes":47163,"Ġreperc":47164,"ĠTerran":47165,"åĪ":47166,"Load":47167,"å¹":47168,"ikarp":47169,"rompt":47170,"Ġgoblins":47171,"ĠShattered":47172,"tests":47173,"Spread":47174,"ĠNaruto":47175,"Ġpredic":47176,"Hyp":47177,"ĠArkham":47178,"ĠNASL":47179,"Material":47180,"Rule":47181,"raviolet":47182,"ĠKlingon":47183,"Memory":47184,"acers":47185,"Known":47186,"Important":47187,"Ġα":47188,"Ġtraged":47189,"Ġshalt":47190,"Ġiso":47191,"ĠJSON":47192,"Instant":47193,"Ġpg":47194,"Ġexponent":47195,"formance":47196,"bitcoin":47197,"DOS":47198,"cheat":47199,"Ġrook":47200,"ĠBiol":47201,"noticed":47202,"Ġtwent":47203,"ĠRedux":47204,"ĠBorderlands":47205,"Supported":47206,"TRUMP":47207,"Ġturrets":47208,"include":47209,"Effect":47210,"Ġdisg":47211,"ophical":47212,"ĠFaction":47213,"wiki":47214,"Ġsrc":47215,"Laun":47216,"TIT":47217,"Ġorbs":47218,"Ġincompet":47219,"Ġdescriptor":47220,"ĠTrog":47221,"Contribut":47222,"ĠGodd":47223,"inances":47224,"Ult":47225,"lyak":47226,"âĢ¢âĢ¢âĢ¢âĢ¢":47227,"stitial":47228,"essim":47229,"Graphics":47230,"ubis":47231,"Ġegreg":47232,"DEV":47233,"Ġannotations":47234,"Yang":47235,"ĠDruid":47236,"ĠInquisition":47237,"ohydrate":47238,"Critical":47239,"æĸ":47240,"Sample":47241,"ĠPref":47242,"ĠUnleashed":47243,"ĠAccessed":47244,"Ġconceptions":47245,"Minor":47246,"pard":47247,"prus":47248,"Factory":47249,"thinkable":47250,"Ġexecutable":47251,"chapter":47252,"inyl":47253,"Display":47254,"ilater":47255,"Released":47256,"ĠDirectX":47257,"aneers":47258,"Ġ______":47259,"ĠHilbert":47260,"Options":47261,"Ġsorcery":47262,"esm":47263,"ÏĦ":47264,"Ġdescript":47265,"ĠTycoon":47266,"psons":47267,"Ġcov":47268,"Launch":47269,"ogeneity":47270,"Ġsacrific":47271,"ADRA":47272,"netflix":47273,"flix":47274,"usage":47275,"properties":47276,"attach":47277,"req":47278,"Resource":47279,"requisite":47280,"1007":47281,"ĠMIDI":47282,"ĠZoro":47283,"Tue":47284,"hower":47285,"dds":47286,"ynasty":47287,"headers":47288,"Ġdisproportion":47289,"omaly":47290,"Ġvim":47291,"inces":47292,"edient":47293,"ĠWraith":47294,"ilibrium":47295,"Hig":47296,"ĠFrie":47297,"Meat":47298,"ldom":47299,"KNOWN":47300,"orgetown":47301,"Improve":47302,"10000":47303,"Ġretarded":47304,"Disclaimer":47305,"Ġunfocused":47306,"ĠUnsure":47307,"ĠElixir":47308,"idth":47309,"atural":47310,"ĠErr":47311,"Critics":47312,"ĠBows":47313,"ifferent":47314,"proxy":47315,"Lic":47316,"aucas":47317,"rolet":47318,"ĠCoC":47319,"Ġdoesnt":47320,"phabet":47321,"Version":47322,"Ġhepat":47323,"gif":47324,"izophren":47325,"ãĥ»":47326,"ĠGutenberg":47327,"β":47328,"phans":47329,"Scene":47330,"Ġaccomp":47331,"ilings":47332,"rypted":47333,"aceae":47334,"arantine":47335,"heses":47336,"iasco":47337,"lopp":47338,"ĠGSL":47339,"disk":47340,"ãĢģ":47341,"0010":47342,"ĠOutbreak":47343,"Column":47344,"odox":47345,"atform":47346,"ĠThrust":47347,"ĠSVG":47348,"Enhanced":47349,"¯":47350,"Tools":47351,"rogens":47352,"xus":47353,"Available":47354,"zbollah":47355,"è¡":47356,"osate":47357,"usb":47358,"ordes":47359,"Matrix":47360,"ĠBlazing":47361,"ascus":47362,"ĠSovere":47363,"hement":47364,"*:":47365,"amaru":47366,"Ġparsed":47367,"Bonus":47368,"otrop":47369,"spell":47370,"ancock":47371,"ĠEnchant":47372,"vP":47373,"ĠReferred":47374,"Ġalot":47375,"ĠRuntime":47376,"ĠFn":47377,"CPU":47378,"ĠNicotine":47379,"External":47380,"ĠNightmares":47381,"Ġentropy":47382,"kB":47383,"ĠRealms":47384,"Ġ##":47385,"Ġsubmar":47386,"ĠSlime":47387,"itual":47388,"ĠBastard":47389,"Ġacknowled":47390,"Magazine":47391,"rendered":47392,"ircraft":47393,"CSS":47394,"Numbers":47395,"Pg":47396,"utenant":47397,"ĠPalest":47398,"ĠRoose":47399,"udicrous":47400,"anooga":47401,"Unt":47402,"Ġcapacitor":47403,"Ġschema":47404,"hematic":47405,"ĠPinball":47406,"endars":47407,"Ġ===":47408,"nsic":47409,"ipedia":47410,"Ġchromos":47411,"ĠmRNA":47412,"Ct":47413,"ĠPaladin":47414,"sonian":47415,"Ġæ":47416,"ajor":47417,"repeat":47418,"ortex":47419,"ĠHeroic":47420,"ĠHera":47421,"ociated":47422,"Ġdebug":47423,"osher":47424,"upiter":47425,"_.":47426,"Ġsys":47427,"ĠDownloads":47428,"','":47429,"Adventure":47430,"FORE":47431,"ocument":47432,"arning":47433,"Ġmiscon":47434,"vidia":47435,"Cod":47436,"ibraries":47437,"buffer":47438,"cdn":47439,"ĠModes":47440,"tarian":47441,"ĠPyro":47442,"ĠFixes":47443,"ĠâĪ":47444,"ĠCf":47445,"Testing":47446,"Byte":47447,"nants":47448,"oufl":47449,"ĠCipher":47450,"Aim":47451,"ĠAfgh":47452,"ĠStarCraft":47453,"intendent":47454,"akespe":47455,"Apply":47456,">>>":47457,"Lenin":47458,"ĠShaman":47459,"%\"":47460,"ĠFrenzy":47461,"illusion":47462,"===":47463,"Website":47464,"Allow":47465,"ĠBinary":47466,"ensable":47467,"ĠEmpires":47468,"Ġpromul":47469,"ormonal":47470,"ileaks":47471,"ĠAmmo":47472,"assies":47473,"atican":47474,"avior":47475,"ĠIter":47476,"1024":47477,"uesday":47478,"ĠAppears":47479,"achine":47480,"Problem":47481,"ousy":47482,"ramid":47483,"nox":47484,"··":47485,"omething":47486,"ĠPurg":47487,"artney":47488,"Ġ0000":47489,"psey":47490,"Ġglutamate":47491,"ĠActivate":47492,"Repl":47493,"Priv":47494,"cyclop":47495,"ĠHispan":47496,"atsuki":47497,"Likewise":47498,"JOHN":47499,"POSE":47500,"pherd":47501,"schild":47502,"Ġsuffix":47503,"åIJ":47504,"Ġoptionally":47505,"ĠRecomm":47506,"ĠSpawn":47507,"ARDIS":47508,"Ġinconsist":47509,"Ġenglish":47510,"Beta":47511,"ĠContains":47512,"uddenly":47513,"Ġls":47514,"Dynamic":47515,"åĽ":47516,"Ġ{{":47517,"dq":47518,"Hmm":47519,"oliberal":47520,"ĠCarnage":47521,"ĠRebirth":47522,"incerity":47523,"Ġproletariat":47524,"ĠCrafting":47525,"Explore":47526,"Ġeld":47527,"ĠAnarch":47528,"Ġ(>":47529,"ĠClockwork":47530,"ĠProced":47531,"APTER":47532,"ĠSorcerer":47533,"âĶ":47534,"ĠSnape":47535,"elist":47536,"Balance":47537,"Tube":47538,"Ġ--------------------":47539,"Ġnostalg":47540,"ACTED":47541,"ĠVID":47542,"soever":47543,"ignt":47544,"Ġhypothal":47545,"ĠObj":47546,"igure":47547,"ĠElves":47548,"gorithm":47549,"Romney":47550,"idable":47551,"renheit":47552,"aptic":47553,"Ġnonex":47554,"Profile":47555,"Ġscient":47556,"ĠAchievements":47557,"ĠReload":47558,"Products":47559,"ampire":47560,"pread":47561,"ĠYamato":47562,"Thread":47563,"ĠFML":47564,"ĠForsaken":47565,"Statistics":47566,"Ġ([":47567,"utsu":47568,"nces":47569,"...?":47570,"upload":47571,"Typ":47572,"ĠReflex":47573,"Dial":47574,"Ġspawns":47575,"Server":47576,"Ġacquaint":47577,"iterranean":47578,"='":47579,"Device":47580,"ר":47581,"ocaly":47582,"Remove":47583,"Ġ=====":47584,"Ġabdom":47585,"ideos":47586,"Dual":47587,"Fax":47588,"Ġbesie":47589,"ĠAdin":47590,"Ġdescrib":47591,"Ġiod":47592,"Limit":47593,"aunders":47594,"ĠAssassins":47595,"xxxx":47596,"ulner":47597,"Shipping":47598,"Item":47599,"fortune":47600,"Ġcipher":47601,"mA":47602,"acerb":47603,"ebus":47604,"Ġmodifiers":47605,"Added":47606,"prisingly":47607,"Dir":47608,"ĠArchangel":47609,"umbnails":47610,"Huh":47611,"ĠWARN":47612,"Role":47613,"usional":47614,"Ġcortical":47615,"ĠSCP":47616,"ĠException":47617,"ĠWarhammer":47618,")))":47619,"](":47620,"Ġsynaptic":47621,"Ġcached":47622,"archment":47623,"Ġtarg":47624,"Filter":47625,"ĠHades":47626,"Ġprinc":47627,"halla":47628,"ptoms":47629,"Ïģ":47630,"ructose":47631,"termination":47632,"Ġcompe":47633,"define":47634,"Ġprosec":47635,"require":47636,"ĠCorpse":47637,"Abstract":47638,"********************************":47639,"Used":47640,"ĠIbid":47641,"trak":47642,"ä¸Ń":47643,"ĠGABA":47644,"åĬ":47645,"ĠHegel":47646,"Jere":47647,"odore":47648,"í":47649,"namese":47650,"Origin":47651,"ĠMastery":47652,"gerald":47653,"Charges":47654,"--------------------":47655,"Forge":47656,"comings":47657,"åį":47658,"Ġ(&":47659,"Ġgrap":47660,"Mask":47661,"ĠGundam":47662,"generic":47663,"ĠMalf":47664,"raphics":47665,"Internal":47666,"ourge":47667,"Ġirresist":47668,"sterdam":47669,"Ġendogenous":47670,"Export":47671,"Ġë":47672,"poons":47673,"Ġabund":47674,"ĠQuantity":47675,"Issue":47676,"âĪĴ":47677,"cknow":47678,"Anonymous":47679,"ĠDRAG":47680,"Wikipedia":47681,"Ġsubdu":47682,"iverpool":47683,"apesh":47684,"Ability":47685,"ĠCentOS":47686,"iseum":47687,"lycer":47688,"Untitled":47689,"Ġlineback":47690,"Ġtomat":47691,"byte":47692,"tile":47693,"linux":47694,"Palest":47695,"canon":47696,"FAULT":47697,"ĠkHz":47698,"Ġhelic":47699,"ĠIGF":47700,"WARE":47701,"Feature":47702,"ĠGraveyard":47703,"ĠNemesis":47704,"akuya":47705,"inement":47706,"Ġwhence":47707,"ractical":47708,"Ping":47709,"tesque":47710,"scroll":47711,"espie":47712,"Ġasynchronous":47713,"ocre":47714,"Measure":47715,"morph":47716,"std":47717,"Settings":47718,"Course":47719,"Ġ],":47720,"Ïĥ":47721,"Documents":47722,"estern":47723,"Ġtf":47724,"Ġcircumcised":47725,"geant":47726,"Ġconject":47727,"ĠFolder":47728,"outube":47729,"ĠMedline":47730,"Status":47731,"ctr":47732,"anoia":47733,"ĠPowerShell":47734,"Chel":47735,"Loop":47736,"Ġresize":47737,"aphael":47738,"workshop":47739,"velength":47740,"hover":47741,"flush":47742,"Ġβ":47743,"Task":47744,"pedia":47745,"ptin":47746,"bidden":47747,"windows":47748,"ĠCaucas":47749,"aml":47750,"isoft":47751,"Ġrs":47752,"cgi":47753,"urrection":47754,"miah":47755,"ÏĤ":47756,"Ġplaythrough":47757,"Reddit":47758,"׾":47759,"Ġannotation":47760,"Ġnobles":47761,"seq":47762,"mares":47763,"Ġwik":47764,"foreseen":47765,"RPG":47766,"Ġreper":47767,"aredevil":47768,"arcity":47769,"/\"":47770,"Ġ});":47771,"Ġdiscont":47772,"ĠBinding":47773,"answered":47774,"Mesh":47775,"ĠMPEG":47776,"Ġperceptual":47777,"OTAL":47778,"ursive":47779,"ãģĦ":47780,"Ġplun":47781,"onential":47782,"ãĤ":47783,"ĠReloaded":47784,"iscopal":47785,"ĠDespair":47786,"FIX":47787,"Ġheterogeneity":47788,",[":47789,"ichick":47790,"DCS":47791,"Ġcooldown":47792,"................":47793,"Ġsomew":47794,"Battery":47795,"stract":47796,"Attempt":47797,"allery":47798,"ĠNept":47799,"Ġtac":47800,"ĠElemental":47801,"Function":47802,"Ġbindings":47803,"versive":47804,"ĠWarlock":47805,"Response":47806,"ĠNPCs":47807,"ollower":47808,"ĠReborn":47809,"Ġphenotype":47810,"uscript":47811,"Ġpecul":47812,"!/":47813,"Unique":47814,"ĠFreeBSD":47815,"ĠChero":47816,"Ġcolle":47817,"gently":47818,"Empty":47819,"rss":47820,"Ġdd":47821,"forge":47822,"ĠTraps":47823,"×Ķ":47824,"iblical":47825,"---------":47826,"uminati":47827,"login":47828,"asus":47829,"xual":47830,"ĠMiko":47831,"ĠDrac":47832,"ssh":47833,"Submit":47834,"ĠMultiplayer":47835,"leanor":47836,"Orig":47837,"anism":47838,"peror":47839,"ĠESV":47840,"Ġencour":47841,"å°":47842,"ĠPLoS":47843,"ĠCrusher":47844,"ocrates":47845,"ynchronous":47846,"§":47847,"ĠLuffy":47848,"Lastly":47849,"Ġdiffere":47850,"okane":47851,"Enh":47852,"ursor":47853,"Ġapopt":47854,"ĠTotem":47855,"ä½":47856,"Honest":47857,"xml":47858,"Created":47859,"Ġteleport":47860,"NRS":47861,"ccess":47862,"ilitary":47863,"ackets":47864,"Ġenchantment":47865,"ĠCunning":47866,"ortmund":47867,"Altern":47868,"Alternatively":47869,"ĠLuthor":47870,"Publisher":47871,"GBT":47872,"çĶ":47873,"Activity":47874,"Ġleptin":47875,"æĪ":47876,"ĠStarfleet":47877,"å¸":47878,"oooooooo":47879,"Ġlawy":47880,"Frag":47881,"ת":47882,"yright":47883,"cookie":47884,"Finish":47885,"wikipedia":47886,"ĠAbilities":47887,"interface":47888,"Ġglared":47889,"Engineers":47890,"ĠAtk":47891,"oteric":47892,"Ġbyte":47893,"ossibility":47894,"Label":47895,"ĠCSV":47896,"Ġè":47897,"ĠOblivion":47898,"android":47899,"rehensive":47900,"ĠCommands":47901,"clud":47902,"ĠTutorial":47903,"retched":47904,"irlwind":47905,"conserv":47906,"ministic":47907,"void":47908,"ernels":47909,"alias":47910,"ĠDraco":47911,"desktop":47912,"ĠMormonism":47913,"oÄŁ":47914,"kef":47915,"Ġtimestamp":47916,"WAYS":47917,"ãģĹ":47918,"\"(":47919,"eneg":47920,"CHAT":47921,"Ġnpm":47922,"ĠGrenade":47923,"rongh":47924,"dinand":47925,"Definition":47926,"ĠInteger":47927,"Ġmodifier":47928,"Ġdex":47929,"ĠParameters":47930,"andestine":47931,"ĠSHALL":47932,"Purchase":47933,"enaries":47934,"Ġstarship":47935,"Armor":47936,"Skill":47937,"Ġlookup":47938,"verages":47939,"Minimum":47940,"ĠBleach":47941,"Ġdf":47942,"inosaur":47943,"ixel":47944,"Zip":47945,"temp":47946,"ruby":47947,"Fram":47948,"sword":47949,"Minecraft":47950,"strous":47951,"Client":47952,"ĠBarbarian":47953,"æĹ":47954,"USER":47955,"ĠMehran":47956,"axies":47957,"ermanent":47958,"ĠHeader":47959,"ablishment":47960,"hyde":47961,"Snake":47962,"ĠTelesc":47963,"Pocket":47964,"Ġ........":47965,"Destroy":47966,"Method":47967,"ĠZup":47968,"olulu":47969,"Ġunemploy":47970,"Temp":47971,"ĠExplicit":47972,"人":47973,"cache":47974,"innamon":47975,"Ġunavoid":47976,"Summary":47977,"Ġappre":47978,"Ġtaxp":47979,"XXX":47980,"ieval":47981,"ĠSummon":47982,"å¤":47983,"Lear":47984,"ibliography":47985,"CLASS":47986,"dimension":47987,"ĠHorde":47988,"Ġfilesystem":47989,"ĠQiao":47990,"obbies":47991,"DIR":47992,"Ġimpedance":47993,"éĩ":47994,"Names":47995,"ĠDrupal":47996,"Applic":47997,"imei":47998,"ynchron":47999,"Ire":48000,"ĠMinion":48001,"ĠHaste":48002,"ä¿":48003,"Ġ(=":48004,"LinkedIn":48005,"Maps":48006,"ifacts":48007,"Damage":48008,"odynam":48009,"ĠShroud":48010,"Ancient":48011,"enhagen":48012,"Tact":48013,"anship":48014,"aturdays":48015,"ãģ«":48016,"ikhail":48017,"ãģ®":48018,"framework":48019,"lication":48020,"âĢ¦]":48021,"Plug":48022,"ĠLilith":48023,"browser":48024,"offset":48025,"ĠJuda":48026,"ciating":48027,"console":48028,"Ġ=================":48029,"._":48030,"ĠPuzz":48031,"OPLE":48032,"erial":48033,"OHN":48034,"ĠGolem":48035,"ierrez":48036,"Ġ},":48037,"inition":48038,"insula":48039,"ĠEntered":48040,"greSQL":48041,"ĠFlask":48042,"ĠXCOM":48043,"fixes":48044,"ĠWeasley":48045,"arser":48046,"Ġrc":48047,"microsoft":48048,"HHHH":48049,"INFO":48050,"rehend":48051,"Ġpolymorph":48052,"Button":48053,"âī":48054,"QUI":48055,"twitch":48056,"jriwal":48057,"ĠSaiyan":48058,"Ġadherent":48059,"acters":48060,"arthed":48061,"âĢł":48062,"Ġfoss":48063,"ã":48064,"Quote":48065,"ependent":48066,"Ġhorr":48067,"UGC":48068,"Weiss":48069,"styles":48070,"advertisement":48071,"Credits":48072,"Lua":48073,"ĠUCH":48074,"Ġhorrend":48075,"Ġminion":48076,">,":48077,"ãĥ³":48078,"Ġinclud":48079,"Compar":48080,"Ġ[]":48081,"Ġ(<":48082,"Phones":48083,"paralleled":48084,"HTML":48085,"Ġ(%":48086,"raltar":48087,"Ġamd":48088,"Maximum":48089,"ĠSolitaire":48090,"SCP":48091,"ĠVaugh":48092,"ĠCLR":48093,"database":48094,"module":48095,"̶":48096,"Capture":48097,"Window":48098,"ubuntu":48099,"Includes":48100,"ĠUriel":48101,"ORPG":48102,"κ":48103,"âĪ":48104,"ä¸Ģ":48105,"Ġdexter":48106,"ĠGlac":48107,"slice":48108,"HAHAHAHA":48109,"\\\"":48110,"lations":48111,"ÙIJ":48112,"ĠAUTH":48113,"earch":48114,"ĠSocket":48115,"Character":48116,"Sort":48117,"Ġindist":48118,"/_":48119,"ĠAntar":48120,"ifix":48121,"Ġlich":48122,"variable":48123,"_(":48124,"Ġgui":48125,"Herm":48126,"elvet":48127,"è¯":48128,"Developer":48129,"Ġkcal":48130,"ciation":48131,"Transaction":48132,"Ġdocker":48133,"###":48134,"ĠVegeta":48135,"Result":48136,"ocamp":48137,"aughtered":48138,"Increase":48139,"aples":48140,"iannopoulos":48141,"zbek":48142,"estyles":48143,"emonium":48144,"è¿":48145,"ĠFANT":48146,"Reason":48147,"Elsewhere":48148,"\"\"":48149,"ĠArtifact":48150,"Authent":48151,"herical":48152,"Ġmembr":48153,"socket":48154,"Elsa":48155,"Condition":48156,"Ġlapt":48157,"Ġsorcerer":48158,"Layer":48159,"apters":48160,"Ġveter":48161,"Myth":48162,"ensical":48163,"ÏĢ":48164,"noxious":48165,"Ġunpre":48166,"Flags":48167,"OOOOOOOO":48168,"Ġincent":48169,"Combat":48170,"Session":48171,"Ġteleportation":48172,"éĢ":48173,"ortment":48174,"Admin":48175,"Fixed":48176,"×Ļ":48177,"Ġconfir":48178,"ãģŁ":48179,"morrow":48180,"osponsors":48181,"\\/":48182,"ictionary":48183,"Num":48184,"Ġquir":48185,"åº":48186,"à¨":48187,"Ġ<<":48188,"Attempts":48189,"ãģ§":48190,"λ":48191,"Features":48192,"XXXX":48193,"Ġinflamm":48194,"VERSION":48195,"ortality":48196,"spawn":48197,"ratulations":48198,"Ġcharism":48199,"Ġ&&":48200,"Dialogue":48201,"luster":48202,"<<":48203,"args":48204,"redients":48205,"Ġpredicate":48206,"qqa":48207,"etheus":48208,"Ġ(!":48209,"Ġshowc":48210,"cmd":48211,"bringer":48212,"Ġcoh":48213,"Input":48214,"ĠFANTASY":48215,"Ġfict":48216,"Blocks":48217,"Install":48218,"vector":48219,"umblr":48220,"agnar":48221,"Array":48222,"Ġembry":48223,"Ġtheoret":48224,"Ġhref":48225,"irrel":48226,"irements":48227,"iations":48228,"Ġ(/":48229,"Thumbnail":48230,"Ġhashes":48231,"^^":48232,"Copy":48233,"Ġeq":48234,"translation":48235,"Favorite":48236,"Fail":48237,"Ġogre":48238,"isites":48239,"Merit":48240,"ãģ¦":48241,"DATA":48242,"rarily":48243,"igmatic":48244,"Sequ":48245,"Els":48246,"ãģª":48247,"lehem":48248,"requency":48249,"aughed":48250,"Ġdistingu":48251,"Ġartific":48252,"Ġdwarves":48253,"Í":48254,"resy":48255,"~~":48256,"sofar":48257,"ideon":48258,"ozyg":48259,"EEEE":48260,"ĠMelee":48261,"大":48262,"tumblr":48263,"ssl":48264,"Wra":48265,"ONSORED":48266,"Ġvowel":48267,"},":48268,"Vari":48269,"cientious":48270,"Node":48271,"Ġsorce":48272,"========":48273,"perse":48274,"Detailed":48275,"isphere":48276,"Background":48277,"ĺħ":48278,"Redd":48279,"ìĿ":48280,"ãģ¨":48281,"ĠCTRL":48282,"Ġç":48283,"iculty":48284,"ername":48285,"Ġns":48286,"Deploy":48287,"Ġhapp":48288,"Ġ///":48289,"Begin":48290,"Ġgp":48291,"$.":48292,"Output":48293,"Suggest":48294,"×IJ":48295,"ĠToggle":48296,"Ġnutrit":48297,"Ġ\\\"":48298,"Ġpreval":48299,"Ġsubreddits":48300,"Menu":48301,"Amount":48302,"ĠWasteland":48303,"Ġsprites":48304,"Ġshader":48305,"Ġ;)":48306,"NAME":48307,"CLUD":48308,"Ġgoblin":48309,"Refer":48310,"ÙĴ":48311,"á¹":48312,"Improved":48313,"endiary":48314,"Ġassail":48315,"chieve":48316,"reply":48317,"Ġcontrad":48318,"cients":48319,"GROUP":48320,"Controller":48321,"omsky":48322,"chemist":48323,"packages":48324,"ombies":48325,"scl":48326,"Ġibn":48327,"çĽ":48328,":(":48329,"ĠMinotaur":48330,"niper":48331,"====":48332,"Ġsubsc":48333,"è¦":48334,"Ġinteger":48335,"Ġ\"-":48336,"Ġtheorem":48337,"utenberg":48338,"Trigger":48339,"github":48340,"ä¼":48341,"##":48342,"xtap":48343,"oké":48344,"ilial":48345,"idepress":48346,":\\":48347,"Param":48348,"Correction":48349,"ïve":48350,"Chest":48351,"ש":48352,"ĠÏĦ":48353,"Ġrespawn":48354,"Ġrall":48355,"Ġcreatine":48356,"umsy":48357,"ĠTemplate":48358,"foo":48359,"query":48360,"Ġmanufact":48361,"Hardware":48362,"iframe":48363,"Ġ-------":48364,"Ġrecip":48365,"ĠAttributes":48366,"Ġforeskin":48367,"ãĤĭ":48368,"ãĥĦ":48369,"uania":48370,"................................................................":48371,"Ġphylogen":48372,"eaturing":48373,"Ġsprite":48374,"Ġinvari":48375,"DonaldTrump":48376,"({":48377,"ĠMalfoy":48378,"Gamer":48379,"ĠPlugin":48380,"γ":48381,"Query":48382,"ĠPuzzles":48383,"inventory":48384,"trl":48385,"Insert":48386,"Ġawa":48387,"ĠWerewolf":48388,"Ġhorizont":48389,"×ŀ":48390,"Ġcunt":48391,"]]":48392,"ĠByz":48393,"Mouse":48394,"Ġ[[":48395,"ĠCthulhu":48396,"ĠDRAGON":48397,"Default":48398,"ĠPresbyter":48399,"Ġff":48400,"Ġorcs":48401,"Construct":48402,"ĠDebug":48403,"Ġ*/":48404,"×ij":48405,"Ġembr":48406,"License":48407,"css":48408,"incinn":48409,"Prosecut":48410,"Ġsugg":48411,"å¾":48412,"ĠUndead":48413,"æĿ":48414,"Ġfs":48415,"Ġthw":48416,"Vector":48417,"åĮ":48418,"settings":48419,"å¯":48420,"Ġssh":48421,"ĠConverted":48422,"ãĤĴ":48423,"risome":48424,"Ġagre":48425,"Collection":48426,"cmp":48427,"puter":48428,"alloc":48429,"Ġé":48430,"ascade":48431,"ĠSpells":48432,"Ġ:-)":48433,"Haunted":48434,"Ġadolesc":48435,"FORMATION":48436,"ĠImperium":48437,"ãĥ¼":48438,"Supplement":48439,"Render":48440,"Theme":48441,"ĠTorment":48442,"([":48443,"ëĭ":48444,"Ġhtml":48445,"Ġjuven":48446,"ĠSiber":48447,"Ġdaemon":48448,"ivariate":48449,"objects":48450,"negie":48451,"Ġindu":48452,"landish":48453,"Meta":48454,"Impl":48455,"Ġglyph":48456,"Ġ-->":48457,"Ġstreng":48458,"agascar":48459,"guyen":48460,"((":48461,")[":48462,"ĠNorn":48463,"Ġhippocamp":48464,"Ġ¯":48465,"îĢ":48466,"Connection":48467,"PATH":48468,"mbuds":48469,"ĠShards":48470,"Ġadvoc":48471,"Ġsimulac":48472,"âĸij":48473,"!?\"":48474,"ĠPotion":48475,"Ġamulet":48476,"ĠFnatic":48477,"Ġcryptoc":48478,"wav":48479,"radius":48480,"pkg":48481,"ĠMFT":48482,"æĢ":48483,"Ġtoile":48484,"Items":48485,"ifference":48486,"errors":48487,"ĠCelt":48488,"Ġunpop":48489,"ilogy":48490,"6666":48491,"hesda":48492,"Instruct":48493,"å·":48494,"Materials":48495,"ettings":48496,"Percent":48497,"Ġresistor":48498,"tymology":48499,"Ġdeprecated":48500,"Ġgrep":48501,"ĠWRITE":48502,"Ġtriv":48503,"Ġscrut":48504,"[/":48505,"anyl":48506,"skirts":48507,"MSN":48508,"ĠCodec":48509,"ecd":48510,"Anth":48511,"){":48512,"%]":48513,"veyard":48514,"aspberry":48515,"ãĢ":48516,"Reward":48517,"rha":48518,"Stretch":48519,"]-":48520,"Prev":48521,"Context":48522,"Ġlinux":48523,"HAHA":48524,"perties":48525,"ĠVIDE":48526,"Domain":48527,"Ġmurd":48528,"ĠLegions":48529,"apache":48530,"æŃ":48531,"Pause":48532,"Temperature":48533,"ufact":48534,"igslist":48535,"ĠRetrieved":48536,"èª":48537,"ãģĮ":48538,"Ingredients":48539,"ruary":48540,"dyl":48541,"Alias":48542,"ĠÎĶ":48543,"Ġinval":48544,"amsung":48545,"!--":48546,"olean":48547,"æī":48548,"ãģ¯":48549,"Ġcoefficients":48550,"ĠDHCP":48551,"âĨĴ":48552,"utonium":48553,":[":48554,"âĹ":48555,"cli":48556,"Container":48557,"å¼":48558,"nexus":48559,"SOURCE":48560,"Ò":48561,"=/":48562,"Ġmysql":48563,"ĠGained":48564,"Ġ/*":48565,"uncture":48566,"Ġstatically":48567,"âĸł":48568,"æĺ¯":48569,"æ°":48570,"estamp":48571,"Cache":48572,"ulkan":48573,"staking":48574,"apter":48575,"ãģ¾":48576,"Ġμg":48577,"Ġtremend":48578,"ĠPiercing":48579,"naissance":48580,"ĠHealer":48581,"Enabled":48582,"éģ":48583,"âĸ":48584,"ĠThumbnails":48585,"Ġhither":48586,"Format":48587,"utherland":48588,"íķ":48589,"Ġdestro":48590,"fff":48591,"execute":48592,"msg":48593,"romancer":48594,"ĠCanaver":48595,"ĠVaults":48596,"oided":48597,"iage":48598,"Ġimg":48599,"summary":48600,"]);":48601,"ĠABE":48602,"ĠGamergate":48603,"utherford":48604,"Ġoverwrite":48605,"enment":48606,"æķ":48607,"Ġsystemd":48608,"tif":48609,"]).":48610,"ãĤ¤":48611,"Widget":48612,"======":48613,"(-":48614,"Ġ\"+":48615,"ĠIncarnation":48616,"æĥ":48617,"���":48618,"GUI":48619,"èĥ":48620,"forums":48621,"Ġrunes":48622,"Ġâī¤":48623,"Ġdefic":48624,"Distance":48625,"directory":48626,"ĠHorus":48627,"iltr":48628,"ortium":48629,"Ġ./":48630,"bda":48631,"owship":48632,"ĠâĨij":48633,"}.":48634,"åĩ":48635,"1027":48636,"Weapons":48637,"lucent":48638,"Ġauth":48639,";;":48640,"Recommended":48641,"Ġsurv":48642,"Ġvm":48643,"ĠStronghold":48644,"Ġparan":48645,"ĠTrance":48646,"æĺ":48647,"Ġsovere":48648,"Ġcorrid":48649,"ĠPwr":48650,"Ġ[/":48651,"Ġseq":48652,"Population":48653,"Ġ[];":48654,"Ġreferen":48655,"ĠInstr":48656,"ĠStamina":48657,"kernel":48658,"Python":48659,"-+":48660,"Ġallele":48661,"éĽ":48662,"isode":48663,"ä¸į":48664,"otonin":48665,"modules":48666,"Notable":48667,"Spell":48668,"\\\\":48669,"Pref":48670,"Ġdatas":48671,"setup":48672,"Ġhapl":48673,"Height":48674,"åĭ":48675,"ãģ£":48676,"]),":48677,"Handle":48678,"umenthal":48679,"Package":48680,"Ġenthus":48681,"Ġunsus":48682,"Narr":48683,"Examples":48684,"FAQ":48685,"REDACTED":48686,"Ġnotor":48687,"Enable":48688,"Pattern":48689,"aeda":48690,">.":48691,"CHECK":48692,"Ġ����":48693,"Ġ'.":48694,"Ġãĥ":48695,"append":48696,"����":48697,"gemony":48698,"terness":48699,"ĠHaku":48700,"NVIDIA":48701,"queue":48702,"Bind":48703,"Ġneigh":48704,"armor":48705,"retty":48706,"LOD":48707,"plugins":48708,"Ġ/>":48709,"TYPE":48710,"Ġ4096":48711,"-------":48712,"Preview":48713,"FML":48714,"Ġproletarian":48715,"zees":48716,"enfranch":48717,"ãģĨ":48718,"Ctrl":48719,"Module":48720,"ĠSurviv":48721,"ĠStarcraft":48722,"rored":48723,"reddit":48724,"Ġrul":48725,"Ġtx":48726,"Ġmage":48727,"Sword":48728,"Ġ~/":48729,"Effects":48730,"éļ":48731,"ä¹":48732,"Sensor":48733,"Solution":48734,"ãģĻ":48735,"Arcade":48736,"Ġpredec":48737,"Values":48738,"Length":48739,"Ġfortun":48740,"ttp":48741,"\"[":48742,"tmp":48743,"ĠBerserker":48744,"åĨ":48745,"ositories":48746,"Ġcouncill":48747,"ffff":48748,"));":48749,"Recipe":48750,"ĠASCII":48751,"âĦ¢:":48752,"ä":48753,"Ġhorm":48754,"=>":48755,"sers":48756,"ãģĭ":48757,"Recommend":48758,"['":48759,"agame":48760,"Animation":48761,"aucuses":48762,"Discussion":48763,"Ġhelicop":48764,"å¿":48765,"Float":48766,"Component":48767,"instance":48768,"Ġfoo":48769,"localhost":48770,"=-":48771,"Offset":48772,"Psy":48773,"ĠGohan":48774,"buquerque":48775,"Ġdefe":48776,"chwitz":48777,"parse":48778,"Ġdors":48779,"Ġspons":48780,"Ġasync":48781,"agonists":48782,"Ġindo":48783,".>>":48784,"ĠDisciple":48785,"Ġfilename":48786,"rency":48787,"ĠDise":48788,"Ġ\"/":48789,"template":48790,"ãĤ¹":48791,"swers":48792,"Ġ++":48793,"Ġ[(":48794,"thora":48795,"ĠDepths":48796,"livious":48797,"Ġdisadvant":48798,"foundland":48799,"Upload":48800,"Ġ§§":48801,"Ġsophistic":48802,";}":48803,"izont":48804,"\"}":48805,"estial":48806,"Ranked":48807,"ĠOccupations":48808,"LEASE":48809,"ĠOgre":48810,"folder":48811,"Plot":48812,"farious":48813,"Ġsuscept":48814,"Types":48815,"Discuss":48816,"Ġ'/":48817,"æµ":48818,"earable":48819,"æ³":48820,"Tile":48821,"iatus":48822,"åŃ":48823,"Ġreperto":48824,"Helper":48825,"Returns":48826,"ä¸Ĭ":48827,"imaru":48828,"Ġreq":48829,"Ġdissatisf":48830,"multipl":48831,"}{":48832,"-[":48833,"itial":48834,"*/":48835,"Config":48836,"Example":48837,"ĠjQuery":48838,"Mods":48839,"ĠGPIO":48840,"Ġlaun":48841,"layout":48842,"cised":48843,"Ġ......":48844,"+++":48845,"prototype":48846,"Exception":48847,"Ġsubsections":48848,"Ġresemb":48849,"Ġâĩ":48850,"ĠPubMed":48851,"username":48852,"Ġaggro":48853,"éĥ":48854,"Ġ};":48855,"ĠMages":48856,"ryu":48857,"apons":48858,"Optional":48859,"ĠAncients":48860,"ãĤĬ":48861,"Quotes":48862,"oaded":48863,"Ġsuspic":48864,"inline":48865,"omial":48866,"ĠMahjong":48867,"auntlets":48868,"Ġanarchism":48869,"Ġsubclass":48870,"ĠMLG":48871,"...]":48872,"Dialog":48873,"uphem":48874,"Ġrecursive":48875,"7601":48876,"frac":48877,"Else":48878,"ĠSeverus":48879,"},{\"":48880,"ĠCLIENT":48881,"Ġjavascript":48882,"sama":48883,"ĠLearns":48884,"ãĤĤ":48885,"Upgrade":48886,"Listener":48887,"Ġsnipp":48888,"Ġrune":48889,"ĠTTL":48890,"ertation":48891,"olicy":48892,"=\"\"":48893,"«ĺ":48894,"Ġexpr":48895,"ovych":48896,"Ġãģ":48897,"_-_":48898,"munition":48899,"////":48900,"func":48901,">>>>":48902,"Provider":48903,"Ïī":48904,"BUG":48905,"Ġ[-":48906,"Ġarrang":48907,"merce":48908,"ãĥ":48909,"incarn":48910,"Valid":48911,"ĠAether":48912,"ãĤĵ":48913,"ĠUTF":48914,"ĠMonstrous":48915,"ãĤĮ":48916,"hedon":48917,"áµ":48918,":#":48919,"ĠFrieza":48920,"padding":48921,"Reviewer":48922,"Ġpsychiat":48923,"yrinth":48924,"ĠâĶĤ":48925,"hillary":48926,"Static":48927,"Newsletter":48928,"Avg":48929,"Ġfn":48930,"Topic":48931,"choes":48932,"Ġnewsp":48933,"á¸":48934,"Ġ[+":48935,"~~~~~~~~~~~~~~~~":48936,":]":48937,"apego":48938,"buf":48939,"Translation":48940,"ById":48941,"Ġmmol":48942,"ãĥ¼ãĥ":48943,"å½":48944,"ãĤī":48945,"Ġparser":48946,"ãĥª":48947,"`,":48948,"Lair":48949,")}":48950,"ypes":48951,"adobe":48952,"Ġancest":48953,"ernel":48954,"ĠNULL":48955,"ç«":48956,"anguages":48957,"Increases":48958,"æĦ":48959,"utorial":48960,"ithmetic":48961,"dll":48962,"ĠArcane":48963,"çī":48964,"Ġtc":48965,"urtles":48966,"èĪ":48967,"Bytes":48968,"Slot":48969,"ĠBahá":48970,"Weapon":48971,"widget":48972,"querque":48973,"Ġembodiments":48974,"å¥":48975,"WARN":48976,"swer":48977,"thumbnails":48978,"FFFF":48979,"inguishable":48980,"Ġâī":48981,"Ġ${":48982,"AAAAAAAA":48983,"Conclusion":48984,"ĻĤ":48985,"disable":48986,"Rect":48987,"Ġsubp":48988,"Ġ().":48989,"ĠDetected":48990,"èĢ":48991,"[]":48992,"Ġcoerc":48993,"ĠmM":48994,"recated":48995,"fusc":48996,"ĠSorce":48997,"çĶŁ":48998,").[":48999,"Ġ})":49000,"mobi":49001,"yip":49002,"Acknowled":49003,"ternity":49004,"iqueness":49005,"ython":49006,"><":49007,"Ġstd":49008,"Url":49009,"Ġnamespace":49010,"Ġtion":49011,"oother":49012,"Ó":49013,"Ġhemor":49014,"Ġrg":49015,"ventory":49016,"ãĤ¢":49017,"anamo":49018,"Socket":49019,"Topics":49020,"apeshifter":49021,"gnu":49022,"Ġdetrim":49023,"`.":49024,"romeda":49025,"çIJ":49026,"Ġlambda":49027,"Compan":49028,"Variable":49029,"Ġusb":49030,"ĠAdamant":49031,"ournal":49032,"Ġcovari":49033,"ãĥ©":49034,"éĸ":49035,"åİ":49036,"otaur":49037,"Ġ(),":49038,"Marginal":49039,"ãģı":49040,"Ġphysic":49041,"adeon":49042,"RESULTS":49043,"200000":49044,"ãģį":49045,"udeb":49046,"ãģĵ":49047,"COMPLE":49048,"Ġmsg":49049,"ghazi":49050,"/*":49051,"ĠDeity":49052,"Ġdisapp":49053,"Availability":49054,"Ġillum":49055,"à©":49056,"ptives":49057,",âĢĶ":49058,"chnology":49059,"Ġaccur":49060,"Ġapi":49061,"Obj":49062,"ãĤ«":49063,"ãĤ¸":49064,"ä¹ĭ":49065,"ËĪ":49066,"Ġtcp":49067,"Required":49068,".<":49069,"\".[":49070,"Ġ~/.":49071,"Ġobser":49072,"RFC":49073,"Ġintegers":49074,"åī":49075,"Installation":49076,"Ô":49077,"ó":49078,"csv":49079,"ãĥ«":49080,"ĠNoticed":49081,"âĸĵ":49082,"Tumblr":49083,"Reply":49084,"||":49085,"Ġconclud":49086,"Ġ))":49087,"ebin":49088,"sql":49089,"Closure":49090,"++++":49091,"],[":49092,"âĹı":49093,"Ġprolet":49094,"Ġ>=":49095,"estinal":49096,"Ġ[*":49097,"ĠInquisitor":49098,"Ġcmd":49099,"FINE":49100,"CRIP":49101,"Ġvertex":49102,"TeX":49103,"///":49104,"Ö¼":49105,"iscons":49106,"Ġmyster":49107,"Changed":49108,"timeout":49109,"irtual":49110,"Methods":49111,"Ġcerts":49112,"texture":49113,"Roaming":49114,"Proxy":49115,"Override":49116,"éĹ":49117,"utf":49118,"python":49119,"ĠRarity":49120,"ilitarian":49121,"çľ":49122,"().":49123,"æł":49124,"Ġbuf":49125,"åij":49126,"çķ":49127,"Ġ*.":49128,"umerable":49129,"~~~~":49130,"å¦":49131,"Ġsimultane":49132,"Ġjson":49133,"Requires":49134,"Ġperl":49135,"Interface":49136,"rupal":49137,":":49242,"itialized":49243,"HTTP":49244,"Trivia":49245,"Sov":49246,"wrapper":49247,"={":49248,"ĠAzerb":49249,"aeper":49250,"Ġneighb":49251,"initions":49252,"Ġsts":49253,"ĠSasuke":49254,"#$":49255,"uliffe":49256,"æĸ¹":49257,"++++++++++++++++":49258,"ĠElven":49259,"ãģĤ":49260,"Ġartif":49261,"Folder":49262,"Ġà¨":49263,"åĤ":49264,"Ġphyl":49265,"uggest":49266,"blance":49267,"ãģł":49268,"Requirements":49269,"Usage":49270,"Ġinitialized":49271,"ãģ®æ":49272,"conservancy":49273,"ĠReincarn":49274,")|":49275,"Ġantioxid":49276,"ĠClicker":49277,"Ġunlaw":49278,"Ġ\\(":49279,"ãĥĪ":49280,"Ġ[*]":49281,"Characters":49282,"////////":49283,"ãĢIJ":49284,"ãĤ·":49285,"webkit":49286,"ãĢij":49287,"Ġxp":49288,"alkyrie":49289,"Console":49290,"());":49291,"ĠKorra":49292,"\"))":49293,"oooooooooooooooo":49294,"Timer":49295,"////////////////":49296,"yout":49297,"engeance":49298,"emetery":49299,"Ġmages":49300,"mods":49301,"Null":49302,"Ġphilos":49303,"ascript":49304,"Ġaddon":49305,"ĠâĸĪ":49306,"emale":49307,"----------------------------------------------------------------":49308,"Ġ\\\\":49309,"=[":49310,"ĠParables":49311,"ãĥĨ":49312,"VALUE":49313,"Ġ@@":49314,"Ġuint":49315,"${":49316,"cpp":49317,"%%":49318,"Ġ(âĪĴ":49319,"utils":49320,"prefix":49321,"å°Ĩ":49322,"ãĥŃ":49323,"Completed":49324,"Ġgoto":49325,"ãĤ¯":49326,"Winged":49327,"perty":49328,"[\"":49329,"ãĥİ":49330,"ĠScythe":49331,"Ġæľ":49332,"Ġ!=":49333,"Buffer":49334,"docker":49335,"ĠWATCHED":49336,"èĢħ":49337,"())":49338,"Ġdst":49339,"SIZE":49340,"ĠDemonic":49341,"Ġresil":49342,"ãĤ¿":49343,"Ġpione":49344,"cpu":49345,"++)":49346,"TEXT":49347,"Ġdiscrep":49348,"debian":49349,"quished":49350,"Ġacknow":49351,"Ġtrave":49352,"Ġgcc":49353,"Catalog":49354,"ctrl":49355,"ĠMoroc":49356,"Ġcpu":49357,"Ġ];":49358,"ĠSorceress":49359,"Introduced":49360,"Frames":49361,"Ġcondem":49362,"¶æ":49363,"~~~~~~~~":49364,"ĠEmacs":49365,"][/":49366,"Ġglim":49367,"Init":49368,"ĠPrimordial":49369,"ãĥĥ":49370,"Ġ+=":49371,"Ġblat":49372,"à¼":49373,"------------------------------------------------":49374,"gpu":49375,"ãĥĥãĥĪ":49376,"Ġxml":49377,"Ġboolean":49378,"References":49379,"Ġ?)":49380,"Ġsatell":49381,"Queue":49382,"Ġpestic":49383,"Ġ}}":49384,"Attribute":49385,"Ġdx":49386,"ĠDefin":49387,"Synopsis":49388,"..................":49389,"ãĥ¬":49390,"plugin":49391,"Disable":49392,"0000000000000000":49393,")\\":49394,"ĠIchigo":49395,"println":49396,"rontal":49397,"Setup":49398,"Ġ��������":49399,"å§":49400,"âĸº":49401,"ĠPengu":49402,"ailability":49403,"Duration":49404,"Timeout":49405,"ãĢĮ":49406,"Ġbehav":49407,"Reviewed":49408,"Ġtoget":49409,"\\.":49410,"lished":49411,"Ġthous":49412,"Ġperpend":49413,"ecause":49414,"Layout":49415,"è»":49416,"ĠDexterity":49417,"unsigned":49418,"+=":49419,"[[":49420,"ĠRunes":49421,"ãĤ¦":49422,"};":49423,"})":49424,"FTWARE":49425,"ength":49426,"milo":49427,"duino":49428,"天":49429,"ĠClojure":49430,"ļé":49431,"ãĥ¥":49432,"gradient":49433,"Ġ\"\"\"":49434,"âĨij":49435,"@#":49436,"JSON":49437,"Ġproport":49438,"addr":49439,"});":49440,"ãĥIJ":49441,"ä¸ī":49442,"Ġtmp":49443,"å£":49444,"../":49445,"zsche":49446,"ĠâĪ¼":49447,"Entity":49448,"æ©Ł":49449,"ĠâĶľâĶĢâĶĢ":49450,"filename":49451,"{{":49452,"@@":49453,"ĠSeym":49454,"Ġ/**":49455,"ĠSummoner":49456,"Quantity":49457,"ç·":49458,"Attach":49459,"Ġbool":49460,"Texture":49461,"Ġopio":49462,".}":49463,"ãĥĭ":49464,"integer":49465,"Ġregex":49466,"Ġnomine":49467,"ription":49468,"ãģ®ç":49469,"ãĥķ":49470,"Ġsubparagraph":49471,"GGGG":49472,"Ġexplan":49473,"Header":49474,"Spawn":49475,"toggle":49476,"²¾":49477,"Abyss":49478,"expr":49479,"ĠZerg":49480,"ĠGrimoire":49481,"Contents":49482,"Instance":49483,"cyclopedia":49484,"ãĥĹ":49485,"ĠTakeru":49486,"=(":49487,"代":49488,"\\)":49489,"Ġrgb":49490,"htt":49491,"bryce":49492,"Ġlivest":49493,"ĠAnnotations":49494,"âĶĢâĶĢâĶĢâĶĢâĶĢâĶĢâĶĢâĶĢ":49495,"berus":49496,"ntil":49497,"Ġskelet":49498,"callback":49499,"åħī":49500,"Joined":49501,"ãĤª":49502,"Ġargs":49503,"artifacts":49504,"Ġå¤":49505,"ÃĽ":49506,"ãĥŀ":49507,"Streamer":49508,"}\"":49509,"Ġunden":49510,"ãĥģ":49511,"Īè":49512,"ãĥ£":49513,"Ġ0004":49514,"Ġ\\'":49515,"ãĤ°":49516,"ĠCONFIG":49517,"Ġ#####":49518,"``":49519,"anguage":49520,"Ġ*)":49521,"Template":49522,"MODE":49523,"Ġ00000000":49524,"'';":49525,">":49625,"Ġlvl":49626,"Footnote":49627,"Iter":49628,"####":49629,"ãĥij":49630,"ĠCarbuncle":49631,"Ġ[+]":49632,"Ġmathemat":49633,"Allows":49634,"Ġ4090":49635,"Async":49636,"ģ«":49637,"Ļ½":49638,"))))":49639,"á½":49640,"Ġcx":49641,"Ġansw":49642,"{\"":49643,"ãĥŁ":49644,"addons":49645,"Filename":49646,"Appearances":49647,"ĠãĢĮ":49648,"Ġaddr":49649,"Ġcharact":49650,"glomer":49651,"Advertisements":49652,"Ġdracon":49653,"ĠFenrir":49654,"Ġ();":49655,"ĠCitiz":49656,"acebook":49657,"Ġparams":49658,"]=":49659,"Ġsubscript":49660,"Ġentreprene":49661,"tnc":49662,"iversal":49663,"Ġmillenn":49664,"ithub":49665,"/>":49666,"Ġ\"{":49667,"Frameworks":49668,"avorite":49669,"Ġ])":49670,"Constructed":49671,"fml":49672,"ãĥį":49673,"################################":49674,"-|":49675,"¥ŀ":49676,"Ġwithd":49677,"ĠCth":49678,"AppData":49679,"Msg":49680,":{":49681,"ãĤ¨":49682,"Ġtuple":49683,"ç¥ŀ":49684,"Ġintrins":49685,"ĠCooldown":49686,"ategory":49687,"^{":49688,"ãĥĬ":49689,"''''":49690,"çĶ°":49691,"ĠDEBUG":49692,"Ġcannabin":49693,"ocobo":49694,"Invalid":49695,"ãĥĢ":49696,"Compat":49697,"Ġ({":49698,"Removed":49699,"Ġconvol":49700,"}:":49701,"interstitial":49702,"Ġ\"":49721,"initialized":49722,"Ġexting":49723,"Poké":49724,"Parameters":49725,"¶ħ":49726,"########":49727,"NULL":49728,"ãĥĩ":49729,"groupon":49730,"\\-":49731,"ãĥı":49732,"ãĤ±":49733,"Ġsubsequ":49734,"ccording":49735,"ĠMODULE":49736,"ĠProtoss":49737,"\"},{\"":49738,"Ġ..............":49739,"Integer":49740,"endif":49741,"ãĥĻ":49742,"parser":49743,"lambda":49744,"Ġcarbohyd":49745,"ĠUnloaded":49746,"_{":49747,"âĸ¬âĸ¬":49748,"Ġdebian":49749,"]}":49750,"ãĤ¶":49751,"Parameter":49752,"ãĤ£":49753,"ãĤ»":49754,"Ġ$_":49755,"İĭ":49756,"Ġiterator":49757,"ãĤ¬":49758,"WINDOWS":49759,"CONCLUS":49760,"Ġ\"\\":49761,"umbn":49762,"(&":49763,"ãĥ©ãĥ³":49764,"usercontent":49765,"ometimes":49766,"METHOD":49767,"ãĥ¢":49768,"potion":49769,"ãĥ¯":49770,"everal":49771,"Ġweap":49772,"minecraft":49773,"================================":49774,"printf":49775,"ĠShinra":49776,"Ġreluct":49777,"\\\",":49778,"Runtime":49779,"xff":49780,"ĠAbyssal":49781,"akeru":49782,"Ġ\\(\\":49783,"\"/>":49784,"efficients":49785,"Ü":49786,"avascript":49787,"Ġbehavi":49788,"++;":49789,"=#":49790,"Attributes":49791,"âĵĺ":49792,"lvl":49793,"¬¼":49794,"/**":49795,"Gameplay":49796,"ĠLeilan":49797,">)":49798,"=\"/":49799,"Ġ));":49800,"ãĥĨãĤ£":49801,"ġ":49802,".":49836,"DEBUG":49837,"âĶģ":49838,"ãĢı":49839,"WithNo":49840,"Redditor":49841,"ĠâĶľ":49842,"Ġfmt":49843,"ãĢİ":49844,"Ġmsec":49845,"ĪĴ":49846,"eatures":49847,"itially":49848,"\"\"\"":49849,"ãĥ¼ãĤ¯":49850,"Textures":49851,"\"},":49852,"\"><":49858,"||||":49859,"ß":49860,"iterator":49861,"è£ħ":49862,"Ĥª":49863,"ojure":49864,"ãħĭãħĭ":49865,"ãĥ¼ãĥ³":49866,"Ġprintln":49867,"Ġ][":49868,"âĸĪâĸĪ":49869,"âķIJ":49870,"\\\":":49871,"senal":49872,"é¾į":49873,"é¾":49874,"Ġcryst":49875,"ãĥķãĤ¡":49876,"ĠCosponsors":49877,"ãĤ·ãĥ£":49878,"Magikarp":49879,"ĠMagicka":49880,"âĸĪâĸĪâĸĪâĸĪ":49881,",,,,,,,,":49882,"vertisement":49883,"âĶĢâĶĢâĶĢâĶĢ":49884,"ãĥķãĤ©":49885,"luaj":49886,"CLASSIFIED":49887,".''.":49888,"byss":49889,"Ġ{:":49890,"ĠNanto":49891,"Ġptr":49892,"Ġ%%":49893,"Ġteasp":49894,"[_":49895,"ãĥ¤":49896,"ħĭ":49897,"ŃĶ":49898,"Ġpci":49899,"Ġ\"<":49900,"GGGGGGGG":49901,"æĪ¦":49902,"--+":49903,"ãĤ®":49904,"Ġ())":49905,"âĸ¬":49906,"Ġsizeof":49907,"}}}":49908,";;;;;;;;":49909,">]":49910,"âĸĪâĸĪâĸĪâĸĪâĸĪâĸĪâĸĪâĸĪ":49911,"Vaults":49912,"Ġistg":49913,"Ġnewcom":49914,"=]":49915,"¿½":49916,"ĵĺ":49917,"{\\":49918,"Args":49919,"Ġexha":49920,"(\\":49921,"Ġunnecess":49922,"\"}],\"":49923,"ĠUNCLASSIFIED":49924,">(":49925,"ãĤ¢ãĥ«":49926,"æ©":49927,"70710":49928,"Ń·":49929,"ãĥ¼ãĥĨãĤ£":49930,"ĠSakuya":49931,"ãĥĥãĥī":49932,"ĠPyrrha":49933,"escription":49934,"VIDIA":49935,"================================================================":49936,"Ġlooph":49937,"=~":49938,"Ġcumbers":49939,"Ġ)]":49940,"govtrack":49941,"ĠãĤµ":49942,"Ġsubur":49943,"Þ":49944,"Ġâī¡":49945,"Interstitial":49946,"ãĥ¼ãĥĨ":49947,"Ġgobl":49948,"ãĥīãĥ©":49949,"oldown":49950,"ģĸ":49951,"Depths":49952,"Ġ());":49953,"Ġ._":49954,"20439":49955,"Ġç¥ŀ":49956,"ãģ®å®":49957,"ãĤ¼":49958,"Ġ$\\":49959,"âĹ¼":49960,"Ġencount":49961,"Ġ": 48457, "Ġstreng": 48458, "agascar": 48459, "guyen": 48460, "((": 48461, ")[": 48462, "ĠNorn": 48463, "Ġhippocamp": 48464, "Ġ¯": 48465, "îĢ": 48466, "Connection": 48467, "PATH": 48468, "mbuds": 48469, "ĠShards": 48470, "Ġadvoc": 48471, "Ġsimulac": 48472, "âĸij": 48473, "!?\"": 48474, "ĠPotion": 48475, "Ġamulet": 48476, "ĠFnatic": 48477, "Ġcryptoc": 48478, "wav": 48479, "radius": 48480, "pkg": 48481, "ĠMFT": 48482, "æĢ": 48483, "Ġtoile": 48484, "Items": 48485, "ifference": 48486, "errors": 48487, "ĠCelt": 48488, "Ġunpop": 48489, "ilogy": 48490, "6666": 48491, "hesda": 48492, "Instruct": 48493, "å·": 48494, "Materials": 48495, "ettings": 48496, "Percent": 48497, "Ġresistor": 48498, "tymology": 48499, "Ġdeprecated": 48500, "Ġgrep": 48501, "ĠWRITE": 48502, "Ġtriv": 48503, "Ġscrut": 48504, "[/": 48505, "anyl": 48506, "skirts": 48507, "MSN": 48508, "ĠCodec": 48509, "ecd": 48510, "Anth": 48511, "){": 48512, "%]": 48513, "veyard": 48514, "aspberry": 48515, "ãĢ": 48516, "Reward": 48517, "rha": 48518, "Stretch": 48519, "]-": 48520, "Prev": 48521, "Context": 48522, "Ġlinux": 48523, "HAHA": 48524, "perties": 48525, "ĠVIDE": 48526, "Domain": 48527, "Ġmurd": 48528, "ĠLegions": 48529, "apache": 48530, "æŃ": 48531, "Pause": 48532, "Temperature": 48533, "ufact": 48534, "igslist": 48535, "ĠRetrieved": 48536, "èª": 48537, "ãģĮ": 48538, "Ingredients": 48539, "ruary": 48540, "dyl": 48541, "Alias": 48542, "ĠÎĶ": 48543, "Ġinval": 48544, "amsung": 48545, "!--": 48546, "olean": 48547, "æī": 48548, "ãģ¯": 48549, "Ġcoefficients": 48550, "ĠDHCP": 48551, "âĨĴ": 48552, "utonium": 48553, ":[": 48554, "âĹ": 48555, "cli": 48556, "Container": 48557, "å¼": 48558, "nexus": 48559, "SOURCE": 48560, "Ò": 48561, "=/": 48562, "Ġmysql": 48563, "ĠGained": 48564, "Ġ/*": 48565, "uncture": 48566, "Ġstatically": 48567, "âĸł": 48568, "æĺ¯": 48569, "æ°": 48570, "estamp": 48571, "Cache": 48572, "ulkan": 48573, "staking": 48574, "apter": 48575, "ãģ¾": 48576, "Ġμg": 48577, "Ġtremend": 48578, "ĠPiercing": 48579, "naissance": 48580, "ĠHealer": 48581, "Enabled": 48582, "éģ": 48583, "âĸ": 48584, "ĠThumbnails": 48585, "Ġhither": 48586, "Format": 48587, "utherland": 48588, "íķ": 48589, "Ġdestro": 48590, "fff": 48591, "execute": 48592, "msg": 48593, "romancer": 48594, "ĠCanaver": 48595, "ĠVaults": 48596, "oided": 48597, "iage": 48598, "Ġimg": 48599, "summary": 48600, "]);": 48601, "ĠABE": 48602, "ĠGamergate": 48603, "utherford": 48604, "Ġoverwrite": 48605, "enment": 48606, "æķ": 48607, "Ġsystemd": 48608, "tif": 48609, "]).": 48610, "ãĤ¤": 48611, "Widget": 48612, "======": 48613, "(-": 48614, "Ġ\"+": 48615, "ĠIncarnation": 48616, "æĥ": 48617, "���": 48618, "GUI": 48619, "èĥ": 48620, "forums": 48621, "Ġrunes": 48622, "Ġâī¤": 48623, "Ġdefic": 48624, "Distance": 48625, "directory": 48626, "ĠHorus": 48627, "iltr": 48628, "ortium": 48629, "Ġ./": 48630, "bda": 48631, "owship": 48632, "ĠâĨij": 48633, "}.": 48634, "åĩ": 48635, "1027": 48636, "Weapons": 48637, "lucent": 48638, "Ġauth": 48639, ";;": 48640, "Recommended": 48641, "Ġsurv": 48642, "Ġvm": 48643, "ĠStronghold": 48644, "Ġparan": 48645, "ĠTrance": 48646, "æĺ": 48647, "Ġsovere": 48648, "Ġcorrid": 48649, "ĠPwr": 48650, "Ġ[/": 48651, "Ġseq": 48652, "Population": 48653, "Ġ[];": 48654, "Ġreferen": 48655, "ĠInstr": 48656, "ĠStamina": 48657, "kernel": 48658, "Python": 48659, "-+": 48660, "Ġallele": 48661, "éĽ": 48662, "isode": 48663, "ä¸į": 48664, "otonin": 48665, "modules": 48666, "Notable": 48667, "Spell": 48668, "\\\\": 48669, "Pref": 48670, "Ġdatas": 48671, "setup": 48672, "Ġhapl": 48673, "Height": 48674, "åĭ": 48675, "ãģ£": 48676, "]),": 48677, "Handle": 48678, "umenthal": 48679, "Package": 48680, "Ġenthus": 48681, "Ġunsus": 48682, "Narr": 48683, "Examples": 48684, "FAQ": 48685, "REDACTED": 48686, "Ġnotor": 48687, "Enable": 48688, "Pattern": 48689, "aeda": 48690, ">.": 48691, "CHECK": 48692, "Ġ����": 48693, "Ġ'.": 48694, "Ġãĥ": 48695, "append": 48696, "����": 48697, "gemony": 48698, "terness": 48699, "ĠHaku": 48700, "NVIDIA": 48701, "queue": 48702, "Bind": 48703, "Ġneigh": 48704, "armor": 48705, "retty": 48706, "LOD": 48707, "plugins": 48708, "Ġ/>": 48709, "TYPE": 48710, "Ġ4096": 48711, "-------": 48712, "Preview": 48713, "FML": 48714, "Ġproletarian": 48715, "zees": 48716, "enfranch": 48717, "ãģĨ": 48718, "Ctrl": 48719, "Module": 48720, "ĠSurviv": 48721, "ĠStarcraft": 48722, "rored": 48723, "reddit": 48724, "Ġrul": 48725, "Ġtx": 48726, "Ġmage": 48727, "Sword": 48728, "Ġ~/": 48729, "Effects": 48730, "éļ": 48731, "ä¹": 48732, "Sensor": 48733, "Solution": 48734, "ãģĻ": 48735, "Arcade": 48736, "Ġpredec": 48737, "Values": 48738, "Length": 48739, "Ġfortun": 48740, "ttp": 48741, "\"[": 48742, "tmp": 48743, "ĠBerserker": 48744, "åĨ": 48745, "ositories": 48746, "Ġcouncill": 48747, "ffff": 48748, "));": 48749, "Recipe": 48750, "ĠASCII": 48751, "âĦ¢:": 48752, "ä": 48753, "Ġhorm": 48754, "=>": 48755, "sers": 48756, "ãģĭ": 48757, "Recommend": 48758, "['": 48759, "agame": 48760, "Animation": 48761, "aucuses": 48762, "Discussion": 48763, "Ġhelicop": 48764, "å¿": 48765, "Float": 48766, "Component": 48767, "instance": 48768, "Ġfoo": 48769, "localhost": 48770, "=-": 48771, "Offset": 48772, "Psy": 48773, "ĠGohan": 48774, "buquerque": 48775, "Ġdefe": 48776, "chwitz": 48777, "parse": 48778, "Ġdors": 48779, "Ġspons": 48780, "Ġasync": 48781, "agonists": 48782, "Ġindo": 48783, ".>>": 48784, "ĠDisciple": 48785, "Ġfilename": 48786, "rency": 48787, "ĠDise": 48788, "Ġ\"/": 48789, "template": 48790, "ãĤ¹": 48791, "swers": 48792, "Ġ++": 48793, "Ġ[(": 48794, "thora": 48795, "ĠDepths": 48796, "livious": 48797, "Ġdisadvant": 48798, "foundland": 48799, "Upload": 48800, "Ġ§§": 48801, "Ġsophistic": 48802, ";}": 48803, "izont": 48804, "\"}": 48805, "estial": 48806, "Ranked": 48807, "ĠOccupations": 48808, "LEASE": 48809, "ĠOgre": 48810, "folder": 48811, "Plot": 48812, "farious": 48813, "Ġsuscept": 48814, "Types": 48815, "Discuss": 48816, "Ġ'/": 48817, "æµ": 48818, "earable": 48819, "æ³": 48820, "Tile": 48821, "iatus": 48822, "åŃ": 48823, "Ġreperto": 48824, "Helper": 48825, "Returns": 48826, "ä¸Ĭ": 48827, "imaru": 48828, "Ġreq": 48829, "Ġdissatisf": 48830, "multipl": 48831, "}{": 48832, "-[": 48833, "itial": 48834, "*/": 48835, "Config": 48836, "Example": 48837, "ĠjQuery": 48838, "Mods": 48839, "ĠGPIO": 48840, "Ġlaun": 48841, "layout": 48842, "cised": 48843, "Ġ......": 48844, "+++": 48845, "prototype": 48846, "Exception": 48847, "Ġsubsections": 48848, "Ġresemb": 48849, "Ġâĩ": 48850, "ĠPubMed": 48851, "username": 48852, "Ġaggro": 48853, "éĥ": 48854, "Ġ};": 48855, "ĠMages": 48856, "ryu": 48857, "apons": 48858, "Optional": 48859, "ĠAncients": 48860, "ãĤĬ": 48861, "Quotes": 48862, "oaded": 48863, "Ġsuspic": 48864, "inline": 48865, "omial": 48866, "ĠMahjong": 48867, "auntlets": 48868, "Ġanarchism": 48869, "Ġsubclass": 48870, "ĠMLG": 48871, "...]": 48872, "Dialog": 48873, "uphem": 48874, "Ġrecursive": 48875, "7601": 48876, "frac": 48877, "Else": 48878, "ĠSeverus": 48879, "},{\"": 48880, "ĠCLIENT": 48881, "Ġjavascript": 48882, "sama": 48883, "ĠLearns": 48884, "ãĤĤ": 48885, "Upgrade": 48886, "Listener": 48887, "Ġsnipp": 48888, "Ġrune": 48889, "ĠTTL": 48890, "ertation": 48891, "olicy": 48892, "=\"\"": 48893, "«ĺ": 48894, "Ġexpr": 48895, "ovych": 48896, "Ġãģ": 48897, "_-_": 48898, "munition": 48899, "////": 48900, "func": 48901, ">>>>": 48902, "Provider": 48903, "Ïī": 48904, "BUG": 48905, "Ġ[-": 48906, "Ġarrang": 48907, "merce": 48908, "ãĥ": 48909, "incarn": 48910, "Valid": 48911, "ĠAether": 48912, "ãĤĵ": 48913, "ĠUTF": 48914, "ĠMonstrous": 48915, "ãĤĮ": 48916, "hedon": 48917, "áµ": 48918, ":#": 48919, "ĠFrieza": 48920, "padding": 48921, "Reviewer": 48922, "Ġpsychiat": 48923, "yrinth": 48924, "ĠâĶĤ": 48925, "hillary": 48926, "Static": 48927, "Newsletter": 48928, "Avg": 48929, "Ġfn": 48930, "Topic": 48931, "choes": 48932, "Ġnewsp": 48933, "á¸": 48934, "Ġ[+": 48935, "~~~~~~~~~~~~~~~~": 48936, ":]": 48937, "apego": 48938, "buf": 48939, "Translation": 48940, "ById": 48941, "Ġmmol": 48942, "ãĥ¼ãĥ": 48943, "å½": 48944, "ãĤī": 48945, "Ġparser": 48946, "ãĥª": 48947, "`,": 48948, "Lair": 48949, ")}": 48950, "ypes": 48951, "adobe": 48952, "Ġancest": 48953, "ernel": 48954, "ĠNULL": 48955, "ç«": 48956, "anguages": 48957, "Increases": 48958, "æĦ": 48959, "utorial": 48960, "ithmetic": 48961, "dll": 48962, "ĠArcane": 48963, "çī": 48964, "Ġtc": 48965, "urtles": 48966, "èĪ": 48967, "Bytes": 48968, "Slot": 48969, "ĠBahá": 48970, "Weapon": 48971, "widget": 48972, "querque": 48973, "Ġembodiments": 48974, "å¥": 48975, "WARN": 48976, "swer": 48977, "thumbnails": 48978, "FFFF": 48979, "inguishable": 48980, "Ġâī": 48981, "Ġ${": 48982, "AAAAAAAA": 48983, "Conclusion": 48984, "ĻĤ": 48985, "disable": 48986, "Rect": 48987, "Ġsubp": 48988, "Ġ().": 48989, "ĠDetected": 48990, "èĢ": 48991, "[]": 48992, "Ġcoerc": 48993, "ĠmM": 48994, "recated": 48995, "fusc": 48996, "ĠSorce": 48997, "çĶŁ": 48998, ").[": 48999, "Ġ})": 49000, "mobi": 49001, "yip": 49002, "Acknowled": 49003, "ternity": 49004, "iqueness": 49005, "ython": 49006, "><": 49007, "Ġstd": 49008, "Url": 49009, "Ġnamespace": 49010, "Ġtion": 49011, "oother": 49012, "Ó": 49013, "Ġhemor": 49014, "Ġrg": 49015, "ventory": 49016, "ãĤ¢": 49017, "anamo": 49018, "Socket": 49019, "Topics": 49020, "apeshifter": 49021, "gnu": 49022, "Ġdetrim": 49023, "`.": 49024, "romeda": 49025, "çIJ": 49026, "Ġlambda": 49027, "Compan": 49028, "Variable": 49029, "Ġusb": 49030, "ĠAdamant": 49031, "ournal": 49032, "Ġcovari": 49033, "ãĥ©": 49034, "éĸ": 49035, "åİ": 49036, "otaur": 49037, "Ġ(),": 49038, "Marginal": 49039, "ãģı": 49040, "Ġphysic": 49041, "adeon": 49042, "RESULTS": 49043, "200000": 49044, "ãģį": 49045, "udeb": 49046, "ãģĵ": 49047, "COMPLE": 49048, "Ġmsg": 49049, "ghazi": 49050, "/*": 49051, "ĠDeity": 49052, "Ġdisapp": 49053, "Availability": 49054, "Ġillum": 49055, "à©": 49056, "ptives": 49057, ",âĢĶ": 49058, "chnology": 49059, "Ġaccur": 49060, "Ġapi": 49061, "Obj": 49062, "ãĤ«": 49063, "ãĤ¸": 49064, "ä¹ĭ": 49065, "ËĪ": 49066, "Ġtcp": 49067, "Required": 49068, ".<": 49069, "\".[": 49070, "Ġ~/.": 49071, "Ġobser": 49072, "RFC": 49073, "Ġintegers": 49074, "åī": 49075, "Installation": 49076, "Ô": 49077, "ó": 49078, "csv": 49079, "ãĥ«": 49080, "ĠNoticed": 49081, "âĸĵ": 49082, "Tumblr": 49083, "Reply": 49084, "||": 49085, "Ġconclud": 49086, "Ġ))": 49087, "ebin": 49088, "sql": 49089, "Closure": 49090, "++++": 49091, "],[": 49092, "âĹı": 49093, "Ġprolet": 49094, "Ġ>=": 49095, "estinal": 49096, "Ġ[*": 49097, "ĠInquisitor": 49098, "Ġcmd": 49099, "FINE": 49100, "CRIP": 49101, "Ġvertex": 49102, "TeX": 49103, "///": 49104, "Ö¼": 49105, "iscons": 49106, "Ġmyster": 49107, "Changed": 49108, "timeout": 49109, "irtual": 49110, "Methods": 49111, "Ġcerts": 49112, "texture": 49113, "Roaming": 49114, "Proxy": 49115, "Override": 49116, "éĹ": 49117, "utf": 49118, "python": 49119, "ĠRarity": 49120, "ilitarian": 49121, "çľ": 49122, "().": 49123, "æł": 49124, "Ġbuf": 49125, "åij": 49126, "çķ": 49127, "Ġ*.": 49128, "umerable": 49129, "~~~~": 49130, "å¦": 49131, "Ġsimultane": 49132, "Ġjson": 49133, "Requires": 49134, "Ġperl": 49135, "Interface": 49136, "rupal": 49137, ":": 49242, "itialized": 49243, "HTTP": 49244, "Trivia": 49245, "Sov": 49246, "wrapper": 49247, "={": 49248, "ĠAzerb": 49249, "aeper": 49250, "Ġneighb": 49251, "initions": 49252, "Ġsts": 49253, "ĠSasuke": 49254, "#$": 49255, "uliffe": 49256, "æĸ¹": 49257, "++++++++++++++++": 49258, "ĠElven": 49259, "ãģĤ": 49260, "Ġartif": 49261, "Folder": 49262, "Ġà¨": 49263, "åĤ": 49264, "Ġphyl": 49265, "uggest": 49266, "blance": 49267, "ãģł": 49268, "Requirements": 49269, "Usage": 49270, "Ġinitialized": 49271, "ãģ®æ": 49272, "conservancy": 49273, "ĠReincarn": 49274, ")|": 49275, "Ġantioxid": 49276, "ĠClicker": 49277, "Ġunlaw": 49278, "Ġ\\(": 49279, "ãĥĪ": 49280, "Ġ[*]": 49281, "Characters": 49282, "////////": 49283, "ãĢIJ": 49284, "ãĤ·": 49285, "webkit": 49286, "ãĢij": 49287, "Ġxp": 49288, "alkyrie": 49289, "Console": 49290, "());": 49291, "ĠKorra": 49292, "\"))": 49293, "oooooooooooooooo": 49294, "Timer": 49295, "////////////////": 49296, "yout": 49297, "engeance": 49298, "emetery": 49299, "Ġmages": 49300, "mods": 49301, "Null": 49302, "Ġphilos": 49303, "ascript": 49304, "Ġaddon": 49305, "ĠâĸĪ": 49306, "emale": 49307, "----------------------------------------------------------------": 49308, "Ġ\\\\": 49309, "=[": 49310, "ĠParables": 49311, "ãĥĨ": 49312, "VALUE": 49313, "Ġ@@": 49314, "Ġuint": 49315, "${": 49316, "cpp": 49317, "%%": 49318, "Ġ(âĪĴ": 49319, "utils": 49320, "prefix": 49321, "å°Ĩ": 49322, "ãĥŃ": 49323, "Completed": 49324, "Ġgoto": 49325, "ãĤ¯": 49326, "Winged": 49327, "perty": 49328, "[\"": 49329, "ãĥİ": 49330, "ĠScythe": 49331, "Ġæľ": 49332, "Ġ!=": 49333, "Buffer": 49334, "docker": 49335, "ĠWATCHED": 49336, "èĢħ": 49337, "())": 49338, "Ġdst": 49339, "SIZE": 49340, "ĠDemonic": 49341, "Ġresil": 49342, "ãĤ¿": 49343, "Ġpione": 49344, "cpu": 49345, "++)": 49346, "TEXT": 49347, "Ġdiscrep": 49348, "debian": 49349, "quished": 49350, "Ġacknow": 49351, "Ġtrave": 49352, "Ġgcc": 49353, "Catalog": 49354, "ctrl": 49355, "ĠMoroc": 49356, "Ġcpu": 49357, "Ġ];": 49358, "ĠSorceress": 49359, "Introduced": 49360, "Frames": 49361, "Ġcondem": 49362, "¶æ": 49363, "~~~~~~~~": 49364, "ĠEmacs": 49365, "][/": 49366, "Ġglim": 49367, "Init": 49368, "ĠPrimordial": 49369, "ãĥĥ": 49370, "Ġ+=": 49371, "Ġblat": 49372, "à¼": 49373, "------------------------------------------------": 49374, "gpu": 49375, "ãĥĥãĥĪ": 49376, "Ġxml": 49377, "Ġboolean": 49378, "References": 49379, "Ġ?)": 49380, "Ġsatell": 49381, "Queue": 49382, "Ġpestic": 49383, "Ġ}}": 49384, "Attribute": 49385, "Ġdx": 49386, "ĠDefin": 49387, "Synopsis": 49388, "..................": 49389, "ãĥ¬": 49390, "plugin": 49391, "Disable": 49392, "0000000000000000": 49393, ")\\": 49394, "ĠIchigo": 49395, "println": 49396, "rontal": 49397, "Setup": 49398, "Ġ��������": 49399, "å§": 49400, "âĸº": 49401, "ĠPengu": 49402, "ailability": 49403, "Duration": 49404, "Timeout": 49405, "ãĢĮ": 49406, "Ġbehav": 49407, "Reviewed": 49408, "Ġtoget": 49409, "\\.": 49410, "lished": 49411, "Ġthous": 49412, "Ġperpend": 49413, "ecause": 49414, "Layout": 49415, "è»": 49416, "ĠDexterity": 49417, "unsigned": 49418, "+=": 49419, "[[": 49420, "ĠRunes": 49421, "ãĤ¦": 49422, "};": 49423, "})": 49424, "FTWARE": 49425, "ength": 49426, "milo": 49427, "duino": 49428, "天": 49429, "ĠClojure": 49430, "ļé": 49431, "ãĥ¥": 49432, "gradient": 49433, "Ġ\"\"\"": 49434, "âĨij": 49435, "@#": 49436, "JSON": 49437, "Ġproport": 49438, "addr": 49439, "});": 49440, "ãĥIJ": 49441, "ä¸ī": 49442, "Ġtmp": 49443, "å£": 49444, "../": 49445, "zsche": 49446, "ĠâĪ¼": 49447, "Entity": 49448, "æ©Ł": 49449, "ĠâĶľâĶĢâĶĢ": 49450, "filename": 49451, "{{": 49452, "@@": 49453, "ĠSeym": 49454, "Ġ/**": 49455, "ĠSummoner": 49456, "Quantity": 49457, "ç·": 49458, "Attach": 49459, "Ġbool": 49460, "Texture": 49461, "Ġopio": 49462, ".}": 49463, "ãĥĭ": 49464, "integer": 49465, "Ġregex": 49466, "Ġnomine": 49467, "ription": 49468, "ãģ®ç": 49469, "ãĥķ": 49470, "Ġsubparagraph": 49471, "GGGG": 49472, "Ġexplan": 49473, "Header": 49474, "Spawn": 49475, "toggle": 49476, "²¾": 49477, "Abyss": 49478, "expr": 49479, "ĠZerg": 49480, "ĠGrimoire": 49481, "Contents": 49482, "Instance": 49483, "cyclopedia": 49484, "ãĥĹ": 49485, "ĠTakeru": 49486, "=(": 49487, "代": 49488, "\\)": 49489, "Ġrgb": 49490, "htt": 49491, "bryce": 49492, "Ġlivest": 49493, "ĠAnnotations": 49494, "âĶĢâĶĢâĶĢâĶĢâĶĢâĶĢâĶĢâĶĢ": 49495, "berus": 49496, "ntil": 49497, "Ġskelet": 49498, "callback": 49499, "åħī": 49500, "Joined": 49501, "ãĤª": 49502, "Ġargs": 49503, "artifacts": 49504, "Ġå¤": 49505, "ÃĽ": 49506, "ãĥŀ": 49507, "Streamer": 49508, "}\"": 49509, "Ġunden": 49510, "ãĥģ": 49511, "Īè": 49512, "ãĥ£": 49513, "Ġ0004": 49514, "Ġ\\'": 49515, "ãĤ°": 49516, "ĠCONFIG": 49517, "Ġ#####": 49518, "``": 49519, "anguage": 49520, "Ġ*)": 49521, "Template": 49522, "MODE": 49523, "Ġ00000000": 49524, "'';": 49525, ">": 49625, "Ġlvl": 49626, "Footnote": 49627, "Iter": 49628, "####": 49629, "ãĥij": 49630, "ĠCarbuncle": 49631, "Ġ[+]": 49632, "Ġmathemat": 49633, "Allows": 49634, "Ġ4090": 49635, "Async": 49636, "ģ«": 49637, "Ļ½": 49638, "))))": 49639, "á½": 49640, "Ġcx": 49641, "Ġansw": 49642, "{\"": 49643, "ãĥŁ": 49644, "addons": 49645, "Filename": 49646, "Appearances": 49647, "ĠãĢĮ": 49648, "Ġaddr": 49649, "Ġcharact": 49650, "glomer": 49651, "Advertisements": 49652, "Ġdracon": 49653, "ĠFenrir": 49654, "Ġ();": 49655, "ĠCitiz": 49656, "acebook": 49657, "Ġparams": 49658, "]=": 49659, "Ġsubscript": 49660, "Ġentreprene": 49661, "tnc": 49662, "iversal": 49663, "Ġmillenn": 49664, "ithub": 49665, "/>": 49666, "Ġ\"{": 49667, "Frameworks": 49668, "avorite": 49669, "Ġ])": 49670, "Constructed": 49671, "fml": 49672, "ãĥį": 49673, "################################": 49674, "-|": 49675, "¥ŀ": 49676, "Ġwithd": 49677, "ĠCth": 49678, "AppData": 49679, "Msg": 49680, ":{": 49681, "ãĤ¨": 49682, "Ġtuple": 49683, "ç¥ŀ": 49684, "Ġintrins": 49685, "ĠCooldown": 49686, "ategory": 49687, "^{": 49688, "ãĥĬ": 49689, "''''": 49690, "çĶ°": 49691, "ĠDEBUG": 49692, "Ġcannabin": 49693, "ocobo": 49694, "Invalid": 49695, "ãĥĢ": 49696, "Compat": 49697, "Ġ({": 49698, "Removed": 49699, "Ġconvol": 49700, "}:": 49701, "interstitial": 49702, "Ġ\"": 49721, "initialized": 49722, "Ġexting": 49723, "Poké": 49724, "Parameters": 49725, "¶ħ": 49726, "########": 49727, "NULL": 49728, "ãĥĩ": 49729, "groupon": 49730, "\\-": 49731, "ãĥı": 49732, "ãĤ±": 49733, "Ġsubsequ": 49734, "ccording": 49735, "ĠMODULE": 49736, "ĠProtoss": 49737, "\"},{\"": 49738, "Ġ..............": 49739, "Integer": 49740, "endif": 49741, "ãĥĻ": 49742, "parser": 49743, "lambda": 49744, "Ġcarbohyd": 49745, "ĠUnloaded": 49746, "_{": 49747, "âĸ¬âĸ¬": 49748, "Ġdebian": 49749, "]}": 49750, "ãĤ¶": 49751, "Parameter": 49752, "ãĤ£": 49753, "ãĤ»": 49754, "Ġ$_": 49755, "İĭ": 49756, "Ġiterator": 49757, "ãĤ¬": 49758, "WINDOWS": 49759, "CONCLUS": 49760, "Ġ\"\\": 49761, "umbn": 49762, "(&": 49763, "ãĥ©ãĥ³": 49764, "usercontent": 49765, "ometimes": 49766, "METHOD": 49767, "ãĥ¢": 49768, "potion": 49769, "ãĥ¯": 49770, "everal": 49771, "Ġweap": 49772, "minecraft": 49773, "================================": 49774, "printf": 49775, "ĠShinra": 49776, "Ġreluct": 49777, "\\\",": 49778, "Runtime": 49779, "xff": 49780, "ĠAbyssal": 49781, "akeru": 49782, "Ġ\\(\\": 49783, "\"/>": 49784, "efficients": 49785, "Ü": 49786, "avascript": 49787, "Ġbehavi": 49788, "++;": 49789, "=#": 49790, "Attributes": 49791, "âĵĺ": 49792, "lvl": 49793, "¬¼": 49794, "/**": 49795, "Gameplay": 49796, "ĠLeilan": 49797, ">)": 49798, "=\"/": 49799, "Ġ));": 49800, "ãĥĨãĤ£": 49801, "ġ": 49802, ".": 49836, "DEBUG": 49837, "âĶģ": 49838, "ãĢı": 49839, "WithNo": 49840, "Redditor": 49841, "ĠâĶľ": 49842, "Ġfmt": 49843, "ãĢİ": 49844, "Ġmsec": 49845, "ĪĴ": 49846, "eatures": 49847, "itially": 49848, "\"\"\"": 49849, "ãĥ¼ãĤ¯": 49850, "Textures": 49851, "\"},": 49852, "\"><": 49858, "||||": 49859, "ß": 49860, "iterator": 49861, "è£ħ": 49862, "Ĥª": 49863, "ojure": 49864, "ãħĭãħĭ": 49865, "ãĥ¼ãĥ³": 49866, "Ġprintln": 49867, "Ġ][": 49868, "âĸĪâĸĪ": 49869, "âķIJ": 49870, "\\\":": 49871, "senal": 49872, "é¾į": 49873, "é¾": 49874, "Ġcryst": 49875, "ãĥķãĤ¡": 49876, "ĠCosponsors": 49877, "ãĤ·ãĥ£": 49878, "Magikarp": 49879, "ĠMagicka": 49880, "âĸĪâĸĪâĸĪâĸĪ": 49881, ",,,,,,,,": 49882, "vertisement": 49883, "âĶĢâĶĢâĶĢâĶĢ": 49884, "ãĥķãĤ©": 49885, "luaj": 49886, "CLASSIFIED": 49887, ".''.": 49888, "byss": 49889, "Ġ{:": 49890, "ĠNanto": 49891, "Ġptr": 49892, "Ġ%%": 49893, "Ġteasp": 49894, "[_": 49895, "ãĥ¤": 49896, "ħĭ": 49897, "ŃĶ": 49898, "Ġpci": 49899, "Ġ\"<": 49900, "GGGGGGGG": 49901, "æĪ¦": 49902, "--+": 49903, "ãĤ®": 49904, "Ġ())": 49905, "âĸ¬": 49906, "Ġsizeof": 49907, "}}}": 49908, ";;;;;;;;": 49909, ">]": 49910, "âĸĪâĸĪâĸĪâĸĪâĸĪâĸĪâĸĪâĸĪ": 49911, "Vaults": 49912, "Ġistg": 49913, "Ġnewcom": 49914, "=]": 49915, "¿½": 49916, "ĵĺ": 49917, "{\\": 49918, "Args": 49919, "Ġexha": 49920, "(\\": 49921, "Ġunnecess": 49922, "\"}],\"": 49923, "ĠUNCLASSIFIED": 49924, ">(": 49925, "ãĤ¢ãĥ«": 49926, "æ©": 49927, "70710": 49928, "Ń·": 49929, "ãĥ¼ãĥĨãĤ£": 49930, "ĠSakuya": 49931, "ãĥĥãĥī": 49932, "ĠPyrrha": 49933, "escription": 49934, "VIDIA": 49935, "================================================================": 49936, "Ġlooph": 49937, "=~": 49938, "Ġcumbers": 49939, "Ġ)]": 49940, "govtrack": 49941, "ĠãĤµ": 49942, "Ġsubur": 49943, "Þ": 49944, "Ġâī¡": 49945, "Interstitial": 49946, "ãĥ¼ãĥĨ": 49947, "Ġgobl": 49948, "ãĥīãĥ©": 49949, "oldown": 49950, "ģĸ": 49951, "Depths": 49952, "Ġ());": 49953, "Ġ._": 49954, "20439": 49955, "Ġç¥ŀ": 49956, "ãģ®å®": 49957, "ãĤ¼": 49958, "Ġ$\\": 49959, "âĹ¼": 49960, "Ġencount": 49961, "Ġ (0): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False) + (1): BasicBlock( + ... + ) + ... + ) + ... + ) + + Args: + model (torch.nn.Module): A PyTorch model. + module_name (str): Module name. + module (torch.nn.Module): Target module which will be set into :attr:`model`. + """ + super_module = get_super_module(model, module_name) + setattr(super_module, module_name.split('.')[-1], module) + + +def get_ith_layer(model: torch.nn.Module, i: int): + """Get i-th layer in a PyTorch model. + + Example: + >>> from torchvision.models import vgg16 + >>> model = vgg16() + >>> get_ith_layer(model, 5) + Conv2d(64, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) + + Args: + model (torch.nn.Module): A PyTorch model. + i (int): Index of target layer. + + Returns: + torch.nn.Module: i-th layer in :attr:`model`. + """ + j = 0 + for module in model.modules(): + if len(list(module.children())) > 0: + continue + if j == i: + return module + j += 1 + return None + + +def get_ith_layer_name(model: torch.nn.Module, i: int): + """Get the name of i-th layer in a PyTorch model. + + Example: + >>> from torchvision.models import vgg16 + >>> model = vgg16() + >>> get_ith_layer_name(model, 5) + 'features.5' + + Args: + model (torch.nn.Module): A PyTorch model. + i (int): Index of target layer. + + Returns: + str: The name of i-th layer in :attr:`model`. + """ + j = 0 + for name, module in model.named_modules(): + if len(list(module.children())) > 0: + continue + if j == i: + return name + j += 1 + return None + + +def set_ith_layer(model: torch.nn.Module, i: int, layer: torch.nn.Module): + """Set i-th layer in a PyTorch model. + + Example: + >>> from torchvision.models import vgg16 + >>> model = vgg16() + >>> model + VGG( + (features): Sequential( + (0): Conv2d(3, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) + (1): ReLU(inplace=True) + (2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) + ... + ) + ... + ) + >>> set_ith_layer(model, 2, torch.nn.Conv2d(64, 128, 3)) + VGG( + (features): Sequential( + (0): Conv2d(3, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) + (1): ReLU(inplace=True) + --> (2): Conv2d(64, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) + ... + ) + ... + ) + + Args: + model (torch.nn.Module): A PyTorch model. + i (int): Index of target layer. + layer (torch.nn.Module): The layer which will be set into :attr:`model`. + """ + j = 0 + for name, module in model.named_modules(): + if len(list(module.children())) > 0: + continue + if j == i: + set_module(model, name, layer) + return + j += 1 + + +def get_all_specific_type_layers_name(model: torch.nn.Module, types: Tuple[Type[torch.nn.Module]]): + """Get names of all layers which are give types in a PyTorch model. (e.g. `Conv2d`, `Linear`) + + Example: + >>> from torchvision.models import vgg16 + >>> model = vgg16() + >>> get_all_specific_type_layers_name(model, (torch.nn.Conv2d)) + ['features.0', 'features.2', 'features.5', ...] + + Args: + model (torch.nn.Module): A PyTorch model. + types (Tuple[Type[torch.nn.Module]]): Target types, e.g. `(e.g. torch.nn.Conv2d, torch.nn.Linear)` + + Returns: + List[str]: Names of all layers which are give types. + """ + res = [] + for name, m in model.named_modules(): + if isinstance(m, types): + res += [name] + return res + + +class LayerActivation: + """Collect the input and output of a middle module of a PyTorch model during inference. + + Layer is a wide concept in this class. A module (e.g. ResBlock in ResNet) can be also regarded as a "layer". + + Example: + >>> from torchvision.models import vgg16 + >>> model = vgg16() + >>> # collect the input and output of 5th layer in VGG16 + >>> layer_activation = LayerActivation(get_ith_layer(model, 5), 'cuda') + >>> model(torch.rand((1, 3, 224, 224))) + >>> layer_activation.input + tensor([[...]]) + >>> layer_activation.output + tensor([[...]]) + >>> layer_activation.remove() + """ + def __init__(self, layer: torch.nn.Module, detach: bool, device: str): + """Register forward hook on corresponding layer. + + Args: + layer (torch.nn.Module): Target layer. + device (str): Where the collected data is located. + """ + self.hook = layer.register_forward_hook(self._hook_fn) + self.detach = detach + self.device = device + self.input: torch.Tensor = None + self.output: torch.Tensor = None + self.layer = layer + + def __str__(self): + return '- ' + str(self.layer) + + def _hook_fn(self, module, input, output): + # TODO: input or output may be a tuple + if isinstance(input, tuple): + self.input = input[0].to(self.device) + else: + self.input = input.to(self.device) + + if isinstance(output, tuple): + self.output = output[0].to(self.device) + else: + self.output = output.to(self.device) + + if self.detach: + self.input = self.input.detach() + self.output = self.output.detach() + + def remove(self): + """Remove the hook in the model to avoid performance effect. + Use this after using the collected data. + """ + self.hook.remove() + + +class LayerActivation2: + """Collect the input and output of a middle module of a PyTorch model during inference. + + Layer is a wide concept in this class. A module (e.g. ResBlock in ResNet) can be also regarded as a "layer". + + Example: + >>> from torchvision.models import vgg16 + >>> model = vgg16() + >>> # collect the input and output of 5th layer in VGG16 + >>> layer_activation = LayerActivation(get_ith_layer(model, 5), 'cuda') + >>> model(torch.rand((1, 3, 224, 224))) + >>> layer_activation.input + tensor([[...]]) + >>> layer_activation.output + tensor([[...]]) + >>> layer_activation.remove() + """ + def __init__(self, layer: torch.nn.Module): + """Register forward hook on corresponding layer. + + Args: + layer (torch.nn.Module): Target layer. + device (str): Where the collected data is located. + """ + assert layer is not None + + self.hook = layer.register_forward_hook(self._hook_fn) + self.input: torch.Tensor = None + self.output: torch.Tensor = None + self.layer = layer + + def __str__(self): + return '- ' + str(self.layer) + + def _hook_fn(self, module, input, output): + self.input = input + self.output = output + + def remove(self): + """Remove the hook in the model to avoid performance effect. + Use this after using the collected data. + """ + self.hook.remove() + +class LayerActivation3: + """Collect the input and output of a middle module of a PyTorch model during inference. + + Layer is a wide concept in this class. A module (e.g. ResBlock in ResNet) can be also regarded as a "layer". + + Example: + >>> from torchvision.models import vgg16 + >>> model = vgg16() + >>> # collect the input and output of 5th layer in VGG16 + >>> layer_activation = LayerActivation(get_ith_layer(model, 5), 'cuda') + >>> model(torch.rand((1, 3, 224, 224))) + >>> layer_activation.input + tensor([[...]]) + >>> layer_activation.output + tensor([[...]]) + >>> layer_activation.remove() + """ + def __init__(self, layer: torch.nn.Module, detach: bool, device: str): + """Register forward hook on corresponding layer. + + Args: + layer (torch.nn.Module): Target layer. + device (str): Where the collected data is located. + """ + self.hook = layer.register_forward_hook(self._hook_fn) + self.detach = detach + self.device = device + self.input: torch.Tensor = None + self.output: torch.Tensor = None + self.layer = layer + + def __str__(self): + return '- ' + str(self.layer) + + def _hook_fn(self, module, input, output): + # TODO: input or output may be a tuple + self.input = input + self.output = output + + # if self.detach: + # self.input = self.input.detach() + # self.output = self.output.detach() + + def remove(self): + """Remove the hook in the model to avoid performance effect. + Use this after using the collected data. + """ + self.hook.remove() + +class LayerActivationWrapper: + """A wrapper of :attr:`LayerActivation` which has the same API, but broaden the concept "layer". + Now a series of layers can be regarded as "hyper-layer" in this class. + + Example: + >>> from torchvision.models import vgg16 + >>> model = vgg16() + >>> # collect the input of 5th layer, and output of 7th layer in VGG16 + >>> # i.e. regard 5th~7th layer as a whole module, + >>> # and collect the input and output of this module + >>> layer_activation = LayerActivationWrapper([ + LayerActivation(get_ith_layer(model, 5), 'cuda'), + LayerActivation(get_ith_layer(model, 6), 'cuda') + LayerActivation(get_ith_layer(model, 7), 'cuda') + ]) + >>> model(torch.rand((1, 3, 224, 224))) + >>> layer_activation.input + tensor([[...]]) + >>> layer_activation.output + tensor([[...]]) + >>> layer_activation.remove() + """ + def __init__(self, las: List[LayerActivation]): + """ + Args: + las (List[LayerActivation]): The layer activations of a series of layers. + """ + self.las = las + + def __str__(self): + return '\n'.join([str(la) for la in self.las]) + + @property + def input(self): + """Get the collected input data of first layer. + + Returns: + torch.Tensor: Collected input data of first layer. + """ + return self.las[0].input + + @property + def output(self): + """Get the collected input data of last layer. + + Returns: + torch.Tensor: Collected input data of last layer. + """ + return self.las[-1].output + + def remove(self): + """Remove all hooks in the model to avoid performance effect. + Use this after using the collected data. + """ + [la.remove() for la in self.las] + + +class TimeProfiler: + """ (NOT VERIFIED. DON'T USE ME) + """ + def __init__(self, layer: torch.nn, device): + self.before_infer_hook = layer.register_forward_pre_hook(self.before_hook_fn) + self.after_infer_hook = layer.register_forward_hook(self.after_hook_fn) + + self.device = device + self.infer_time = None + self._start_time = None + + if self.device != 'cpu': + self.s, self.e = torch.cuda.Event(enable_timing=True), torch.cuda.Event(enable_timing=True) + + def before_hook_fn(self, module, input): + if self.device == 'cpu': + self._start_time = time.time() + else: + self.s.record() + + def after_hook_fn(self, module, input, output): + if self.device == 'cpu': + self.infer_time = time.time() - self._start_time + else: + self.e.record() + torch.cuda.synchronize() + self.infer_time = self.s.elapsed_time(self.e) / 1000. + + def remove(self): + self.before_infer_hook.remove() + self.after_infer_hook.remove() + + +class TimeProfilerWrapper: + """ (NOT VERIFIED. DON'T USE ME) + """ + def __init__(self, tps: List[TimeProfiler]): + self.tps = tps + + @property + def infer_time(self): + return sum([tp.infer_time for tp in self.tps]) + + def remove(self): + [tp.remove() for tp in self.tps] \ No newline at end of file diff --git a/utils/dl/common/pruning.py b/utils/dl/common/pruning.py new file mode 100644 index 0000000000000000000000000000000000000000..8eb8257f7a390506da469336efbf20cceb81c2de --- /dev/null +++ b/utils/dl/common/pruning.py @@ -0,0 +1,80 @@ +import copy +import os +import sys +from tabnanny import verbose +from typing import List, Optional, Tuple +import torch + +from ...third_party.nni_new.algorithms.compression.pytorch.pruning import L1FilterPruner +from ...third_party.nni_new.compression.pytorch.speedup import ModelSpeedup +from ...common.others import get_cur_time_str + + +def _prune_module(model, pruner, model_input_size, device, verbose=False, need_return_mask=False): + pruner.compress() + pid = os.getpid() + timestamp = get_cur_time_str() + tmp_model_path = './tmp_weight-{}-{}.pth'.format(pid, timestamp) + tmp_mask_path = './tmp_mask-{}-{}.pth'.format(pid, timestamp) + pruner.export_model(model_path=tmp_model_path, mask_path=tmp_mask_path) + os.remove(tmp_model_path) + + # speed up + dummy_input = torch.rand(model_input_size).to(device) + pruned_model = model + pruned_model.eval() + model_speedup = ModelSpeedup(pruned_model, dummy_input, tmp_mask_path, device) + fixed_mask = model_speedup.speedup_model() + + if not need_return_mask: + os.remove(tmp_mask_path) + return pruned_model + else: + mask = fixed_mask + os.remove(tmp_mask_path) + return pruned_model, mask + + +def l1_prune_model(model: torch.nn.Module, pruned_layers_name: Optional[List[str]], sparsity: float, + model_input_size: Tuple[int], device: str, verbose=False, need_return_mask=False, dep_aware=False): + """Get the pruned model via L1 Filter Pruning. + + Reference: + Li H, Kadav A, Durdanovic I, et al. Pruning filters for efficient convnets[J]. arXiv preprint arXiv:1608.08710, 2016. + + Args: + model (torch.nn.Module): A PyTorch model. + pruned_layers_name (Optional[List[str]]): Which layers will be pruned. If it's `None`, all layers will be pruned. + sparsity (float): Target sparsity. The pruned model is smaller if sparsity is higher. + model_input_size (Tuple[int]): Typically be `(1, 3, 32, 32)` or `(1, 3, 224, 224)`. + device (str): Typically be 'cpu' or 'cuda'. + verbose (bool, optional): Whether to output the verbose log. Defaults to False. (BUG TO FIX) + need_return_mask (bool, optional): Return the fine-grained mask generated by NNI framework for debug. Defaults to False. + dep_aware (bool, optional): Refers to the argument `dependency_aware` in NNI framework. Defaults to False. + + Returns: + torch.nn.Module: Pruned model. + """ + model = copy.deepcopy(model).to(device) + if sparsity == 0: + return model + pruned_model = copy.deepcopy(model).to(device) + + # generate mask + model.eval() + if pruned_layers_name is not None: + config_list = [{ + 'op_types': ['Conv2d', 'ConvTranspose2d'], + 'op_names': pruned_layers_name, + 'sparsity': sparsity + }] + else: + config_list = [{ + 'op_types': ['Conv2d', 'ConvTranspose2d'], + 'sparsity': sparsity + }] + + pruner = L1FilterPruner(model, config_list, dependency_aware=dep_aware, + dummy_input=torch.rand(model_input_size).to(device) if dep_aware else None) + pruned_model = _prune_module(pruned_model, pruner, model_input_size, device, verbose, need_return_mask) + return pruned_model diff --git a/utils/third_party/nni/__init__.py b/utils/third_party/nni/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..0630571ae698850bb0a6948f49797b4bdc9f879c --- /dev/null +++ b/utils/third_party/nni/__init__.py @@ -0,0 +1,20 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +__version__ = '999.0.0-developing' + +from .runtime.env_vars import dispatcher_env_vars +from .utils import ClassArgsValidator + +if dispatcher_env_vars.SDK_PROCESS != 'dispatcher': + from .trial import * + from .smartparam import * + from .common.nas_utils import training_update + +class NoMoreTrialError(Exception): + def __init__(self, ErrorInfo): + super().__init__(self) + self.errorinfo = ErrorInfo + + def __str__(self): + return self.errorinfo diff --git a/utils/third_party/nni/__main__.py b/utils/third_party/nni/__main__.py new file mode 100644 index 0000000000000000000000000000000000000000..e3f982f42fe51450c934f4a03bb15c12f7979318 --- /dev/null +++ b/utils/third_party/nni/__main__.py @@ -0,0 +1,111 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +import argparse +import logging +import json +import base64 + +from .runtime.common import enable_multi_thread, enable_multi_phase +from .runtime.msg_dispatcher import MsgDispatcher +from .tools.package_utils import create_builtin_class_instance, create_customized_class_instance + +logger = logging.getLogger('nni.main') +logger.debug('START') + +if os.environ.get('COVERAGE_PROCESS_START'): + import coverage + coverage.process_startup() + + +def main(): + parser = argparse.ArgumentParser(description='Dispatcher command line parser') + parser.add_argument('--exp_params', type=str, required=True) + args, _ = parser.parse_known_args() + + exp_params_decode = base64.b64decode(args.exp_params).decode('utf-8') + logger.debug('decoded exp_params: [%s]', exp_params_decode) + exp_params = json.loads(exp_params_decode) + logger.debug('exp_params json obj: [%s]', json.dumps(exp_params, indent=4)) + + if exp_params.get('multiThread'): + enable_multi_thread() + if exp_params.get('multiPhase'): + enable_multi_phase() + + if exp_params.get('advisor') is not None: + # advisor is enabled and starts to run + _run_advisor(exp_params) + else: + # tuner (and assessor) is enabled and starts to run + assert exp_params.get('tuner') is not None + tuner = _create_tuner(exp_params) + if exp_params.get('assessor') is not None: + assessor = _create_assessor(exp_params) + else: + assessor = None + dispatcher = MsgDispatcher(tuner, assessor) + + try: + dispatcher.run() + tuner._on_exit() + if assessor is not None: + assessor._on_exit() + except Exception as exception: + logger.exception(exception) + tuner._on_error() + if assessor is not None: + assessor._on_error() + raise + + +def _run_advisor(exp_params): + if exp_params.get('advisor').get('builtinAdvisorName'): + dispatcher = create_builtin_class_instance( + exp_params.get('advisor').get('builtinAdvisorName'), + exp_params.get('advisor').get('classArgs'), + 'advisors') + else: + dispatcher = create_customized_class_instance(exp_params.get('advisor')) + if dispatcher is None: + raise AssertionError('Failed to create Advisor instance') + try: + dispatcher.run() + except Exception as exception: + logger.exception(exception) + raise + + +def _create_tuner(exp_params): + if exp_params.get('tuner').get('builtinTunerName'): + tuner = create_builtin_class_instance( + exp_params.get('tuner').get('builtinTunerName'), + exp_params.get('tuner').get('classArgs'), + 'tuners') + else: + tuner = create_customized_class_instance(exp_params.get('tuner')) + if tuner is None: + raise AssertionError('Failed to create Tuner instance') + return tuner + + +def _create_assessor(exp_params): + if exp_params.get('assessor').get('builtinAssessorName'): + assessor = create_builtin_class_instance( + exp_params.get('assessor').get('builtinAssessorName'), + exp_params.get('assessor').get('classArgs'), + 'assessors') + else: + assessor = create_customized_class_instance(exp_params.get('assessor')) + if assessor is None: + raise AssertionError('Failed to create Assessor instance') + return assessor + + +if __name__ == '__main__': + try: + main() + except Exception as exception: + logger.exception(exception) + raise diff --git a/utils/third_party/nni/algorithms/__init__.py b/utils/third_party/nni/algorithms/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/utils/third_party/nni/algorithms/compression/__init__.py b/utils/third_party/nni/algorithms/compression/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/utils/third_party/nni/algorithms/compression/pytorch/__init__.py b/utils/third_party/nni/algorithms/compression/pytorch/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/utils/third_party/nni/algorithms/compression/pytorch/pruning/__init__.py b/utils/third_party/nni/algorithms/compression/pytorch/pruning/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..dfed2bf142cd0cc66e825ce13549a5f8cd68f499 --- /dev/null +++ b/utils/third_party/nni/algorithms/compression/pytorch/pruning/__init__.py @@ -0,0 +1,15 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .finegrained_pruning import * +from .structured_pruning import * +from .apply_compression import apply_compression_results +from .one_shot import * +from .agp import * +# from .lottery_ticket import LotteryTicketPruner +# from .simulated_annealing_pruner import SimulatedAnnealingPruner +# from .net_adapt_pruner import NetAdaptPruner +# from .admm_pruner import ADMMPruner +# from .auto_compress_pruner import AutoCompressPruner +# from .sensitivity_pruner import SensitivityPruner +# from .amc import AMCPruner diff --git a/utils/third_party/nni/algorithms/compression/pytorch/pruning/admm_pruner.py b/utils/third_party/nni/algorithms/compression/pytorch/pruning/admm_pruner.py new file mode 100644 index 0000000000000000000000000000000000000000..f65f1405e16fd6f4ec942e4ea2781ec9fc82529e --- /dev/null +++ b/utils/third_party/nni/algorithms/compression/pytorch/pruning/admm_pruner.py @@ -0,0 +1,190 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +import torch +from schema import And, Optional + +from nni.compression.pytorch.utils.config_validation import CompressorSchema +from .constants import MASKER_DICT +from .one_shot import OneshotPruner + + +_logger = logging.getLogger(__name__) + + +class ADMMPruner(OneshotPruner): + """ + A Pytorch implementation of ADMM Pruner algorithm. + + Parameters + ---------- + model : torch.nn.Module + Model to be pruned. + config_list : list + List on pruning configs. + trainer : function + Function used for the first subproblem. + Users should write this function as a normal function to train the Pytorch model + and include `model, optimizer, criterion, epoch, callback` as function arguments. + Here `callback` acts as an L2 regulizer as presented in the formula (7) of the original paper. + The logic of `callback` is implemented inside the Pruner, + users are just required to insert `callback()` between `loss.backward()` and `optimizer.step()`. + Example:: + + def trainer(model, criterion, optimizer, epoch, callback): + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + train_loader = ... + model.train() + for batch_idx, (data, target) in enumerate(train_loader): + data, target = data.to(device), target.to(device) + optimizer.zero_grad() + output = model(data) + loss = criterion(output, target) + loss.backward() + # callback should be inserted between loss.backward() and optimizer.step() + if callback: + callback() + optimizer.step() + num_iterations : int + Total number of iterations. + training_epochs : int + Training epochs of the first subproblem. + row : float + Penalty parameters for ADMM training. + base_algo : str + Base pruning algorithm. `level`, `l1` or `l2`, by default `l1`. Given the sparsity distribution among the ops, + the assigned `base_algo` is used to decide which filters/channels/weights to prune. + + """ + + def __init__(self, model, config_list, trainer, num_iterations=30, training_epochs=5, row=1e-4, base_algo='l1'): + self._base_algo = base_algo + + super().__init__(model, config_list) + + self._trainer = trainer + self._num_iterations = num_iterations + self._training_epochs = training_epochs + self._row = row + + self.set_wrappers_attribute("if_calculated", False) + self.masker = MASKER_DICT[self._base_algo](self.bound_model, self) + + def validate_config(self, model, config_list): + """ + Parameters + ---------- + model : torch.nn.Module + Model to be pruned + config_list : list + List on pruning configs + """ + + if self._base_algo == 'level': + schema = CompressorSchema([{ + 'sparsity': And(float, lambda n: 0 < n < 1), + Optional('op_types'): [str], + Optional('op_names'): [str], + }], model, _logger) + elif self._base_algo in ['l1', 'l2']: + schema = CompressorSchema([{ + 'sparsity': And(float, lambda n: 0 < n < 1), + 'op_types': ['Conv2d'], + Optional('op_names'): [str] + }], model, _logger) + + schema.validate(config_list) + + def _projection(self, weight, sparsity): + ''' + Return the Euclidean projection of the weight matrix according to the pruning mode. + + Parameters + ---------- + weight : tensor + original matrix + sparsity : float + the ratio of parameters which need to be set to zero + + Returns + ------- + tensor + the projected matrix + ''' + w_abs = weight.abs() + if self._base_algo == 'level': + k = int(weight.numel() * sparsity) + if k == 0: + mask_weight = torch.ones(weight.shape).type_as(weight) + else: + threshold = torch.topk(w_abs.view(-1), k, largest=False)[0].max() + mask_weight = torch.gt(w_abs, threshold).type_as(weight) + elif self._base_algo in ['l1', 'l2']: + filters = weight.size(0) + num_prune = int(filters * sparsity) + if filters < 2 or num_prune < 1: + mask_weight = torch.ones(weight.size()).type_as(weight).detach() + else: + w_abs_structured = w_abs.view(filters, -1).sum(dim=1) + threshold = torch.topk(w_abs_structured.view(-1), num_prune, largest=False)[0].max() + mask_weight = torch.gt(w_abs_structured, threshold)[:, None, None, None].expand_as(weight).type_as(weight) + + return weight.data.mul(mask_weight) + + def compress(self): + """ + Compress the model with ADMM. + + Returns + ------- + torch.nn.Module + model with specified modules compressed. + """ + _logger.info('Starting ADMM Compression...') + + # initiaze Z, U + # Z_i^0 = W_i^0 + # U_i^0 = 0 + Z = [] + U = [] + for wrapper in self.get_modules_wrapper(): + z = wrapper.module.weight.data + Z.append(z) + U.append(torch.zeros_like(z)) + + optimizer = torch.optim.Adam( + self.bound_model.parameters(), lr=1e-3, weight_decay=5e-5) + + # Loss = cross_entropy + l2 regulization + \Sum_{i=1}^N \row_i ||W_i - Z_i^k + U_i^k||^2 + criterion = torch.nn.CrossEntropyLoss() + + # callback function to do additonal optimization, refer to the deriatives of Formula (7) + def callback(): + for i, wrapper in enumerate(self.get_modules_wrapper()): + wrapper.module.weight.data -= self._row * \ + (wrapper.module.weight.data - Z[i] + U[i]) + + # optimization iteration + for k in range(self._num_iterations): + _logger.info('ADMM iteration : %d', k) + + # step 1: optimize W with AdamOptimizer + for epoch in range(self._training_epochs): + self._trainer(self.bound_model, optimizer=optimizer, + criterion=criterion, epoch=epoch, callback=callback) + + # step 2: update Z, U + # Z_i^{k+1} = projection(W_i^{k+1} + U_i^k) + # U_i^{k+1} = U^k + W_i^{k+1} - Z_i^{k+1} + for i, wrapper in enumerate(self.get_modules_wrapper()): + z = wrapper.module.weight.data + U[i] + Z[i] = self._projection(z, wrapper.config['sparsity']) + U[i] = U[i] + wrapper.module.weight.data - Z[i] + + # apply prune + self.update_mask() + + _logger.info('Compression finished.') + + return self.bound_model diff --git a/utils/third_party/nni/algorithms/compression/pytorch/pruning/agp.py b/utils/third_party/nni/algorithms/compression/pytorch/pruning/agp.py new file mode 100644 index 0000000000000000000000000000000000000000..a1ee5df069d146a6389d3f7f52c61a192d5fd865 --- /dev/null +++ b/utils/third_party/nni/algorithms/compression/pytorch/pruning/agp.py @@ -0,0 +1,151 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +An automated gradual pruning algorithm that prunes the smallest magnitude +weights to achieve a preset level of network sparsity. +Michael Zhu and Suyog Gupta, "To prune, or not to prune: exploring the +efficacy of pruning for model compression", 2017 NIPS Workshop on Machine +Learning of Phones and other Consumer Devices. +""" + +import logging +import torch +from schema import And, Optional +from .constants import MASKER_DICT +from .....compression.pytorch.utils.config_validation import CompressorSchema +from .....compression.pytorch.compressor import Pruner + +__all__ = ['AGPPruner'] + +logger = logging.getLogger('torch pruner') + +class AGPPruner(Pruner): + """ + Parameters + ---------- + model : torch.nn.Module + Model to be pruned. + config_list : listlist + Supported keys: + - initial_sparsity: This is to specify the sparsity when compressor starts to compress. + - final_sparsity: This is to specify the sparsity when compressor finishes to compress. + - start_epoch: This is to specify the epoch number when compressor starts to compress, default start from epoch 0. + - end_epoch: This is to specify the epoch number when compressor finishes to compress. + - frequency: This is to specify every *frequency* number epochs compressor compress once, default frequency=1. + optimizer: torch.optim.Optimizer + Optimizer used to train model. + pruning_algorithm: str + Algorithms being used to prune model, + choose from `['level', 'slim', 'l1', 'l2', 'fpgm', 'taylorfo', 'apoz', 'mean_activation']`, by default `level` + """ + + def __init__(self, model, config_list, optimizer, pruning_algorithm='level'): + super().__init__(model, config_list, optimizer) + assert isinstance(optimizer, torch.optim.Optimizer), "AGP pruner is an iterative pruner, please pass optimizer of the model to it" + self.masker = MASKER_DICT[pruning_algorithm](model, self) + + self.now_epoch = 0 + self.set_wrappers_attribute("if_calculated", False) + + def validate_config(self, model, config_list): + """ + Parameters + ---------- + model : torch.nn.Module + Model to be pruned + config_list : list + List on pruning configs + """ + schema = CompressorSchema([{ + 'initial_sparsity': And(float, lambda n: 0 <= n <= 1), + 'final_sparsity': And(float, lambda n: 0 <= n <= 1), + 'start_epoch': And(int, lambda n: n >= 0), + 'end_epoch': And(int, lambda n: n >= 0), + 'frequency': And(int, lambda n: n > 0), + Optional('op_types'): [str], + Optional('op_names'): [str] + }], model, logger) + + schema.validate(config_list) + + def calc_mask(self, wrapper, wrapper_idx=None): + """ + Calculate the mask of given layer. + Scale factors with the smallest absolute value in the BN layer are masked. + Parameters + ---------- + wrapper : Module + the layer to instrument the compression operation + wrapper_idx: int + index of this wrapper in pruner's all wrappers + Returns + ------- + dict | None + Dictionary for storing masks, keys of the dict: + 'weight_mask': weight mask tensor + 'bias_mask': bias mask tensor (optional) + """ + + config = wrapper.config + + start_epoch = config.get('start_epoch', 0) + freq = config.get('frequency', 1) + + if wrapper.if_calculated: + return None + if not (self.now_epoch >= start_epoch and (self.now_epoch - start_epoch) % freq == 0): + return None + + target_sparsity = self.compute_target_sparsity(config) + new_mask = self.masker.calc_mask(sparsity=target_sparsity, wrapper=wrapper, wrapper_idx=wrapper_idx) + if new_mask is not None: + wrapper.if_calculated = True + + return new_mask + + def compute_target_sparsity(self, config): + """ + Calculate the sparsity for pruning + Parameters + ---------- + config : dict + Layer's pruning config + Returns + ------- + float + Target sparsity to be pruned + """ + + end_epoch = config.get('end_epoch', 1) + start_epoch = config.get('start_epoch', 0) + freq = config.get('frequency', 1) + final_sparsity = config.get('final_sparsity', 0) + initial_sparsity = config.get('initial_sparsity', 0) + if end_epoch <= start_epoch or initial_sparsity >= final_sparsity: + logger.warning('your end epoch <= start epoch or initial_sparsity >= final_sparsity') + return final_sparsity + + if end_epoch <= self.now_epoch: + return final_sparsity + + span = ((end_epoch - start_epoch - 1) // freq) * freq + assert span > 0 + target_sparsity = (final_sparsity + + (initial_sparsity - final_sparsity) * + (1.0 - ((self.now_epoch - start_epoch) / span)) ** 3) + return target_sparsity + + def update_epoch(self, epoch): + """ + Update epoch + Parameters + ---------- + epoch : int + current training epoch + """ + + if epoch > 0: + self.now_epoch = epoch + for wrapper in self.get_modules_wrapper(): + wrapper.if_calculated = False diff --git a/utils/third_party/nni/algorithms/compression/pytorch/pruning/amc/__init__.py b/utils/third_party/nni/algorithms/compression/pytorch/pruning/amc/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..3c89a879c6af42ed372f38f3cc11ea1eb9eb6870 --- /dev/null +++ b/utils/third_party/nni/algorithms/compression/pytorch/pruning/amc/__init__.py @@ -0,0 +1,4 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .amc_pruner import AMCPruner diff --git a/utils/third_party/nni/algorithms/compression/pytorch/pruning/amc/amc_pruner.py b/utils/third_party/nni/algorithms/compression/pytorch/pruning/amc/amc_pruner.py new file mode 100644 index 0000000000000000000000000000000000000000..c2d12429d5fb5d82131b5239ef1992ef100feef6 --- /dev/null +++ b/utils/third_party/nni/algorithms/compression/pytorch/pruning/amc/amc_pruner.py @@ -0,0 +1,296 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +import logging +from copy import deepcopy +from argparse import Namespace +import numpy as np +import torch +from torch.utils.tensorboard import SummaryWriter + +from nni.compression.pytorch.compressor import Pruner +from .channel_pruning_env import ChannelPruningEnv +from .lib.agent import DDPG +from .lib.utils import get_output_folder + +torch.backends.cudnn.deterministic = True + +_logger = logging.getLogger(__name__) + +class AMCPruner(Pruner): + """ + A pytorch implementation of AMC: AutoML for Model Compression and Acceleration on Mobile Devices. + (https://arxiv.org/pdf/1802.03494.pdf) + + Parameters: + model: nn.Module + The model to be pruned. + config_list: list + Configuration list to configure layer pruning. + Supported keys: + - op_types: operation type to be pruned + - op_names: operation name to be pruned + evaluator: function + function to evaluate the pruned model. + The prototype of the function: + >>> def evaluator(val_loader, model): + >>> ... + >>> return acc + val_loader: torch.utils.data.DataLoader + Data loader of validation dataset. + suffix: str + suffix to help you remember what experiment you ran. Default: None. + + # parameters for pruning environment + model_type: str + model type to prune, currently 'mobilenet' and 'mobilenetv2' are supported. Default: mobilenet + flops_ratio: float + preserve flops ratio. Default: 0.5 + lbound: float + minimum weight preserve ratio for each layer. Default: 0.2 + rbound: float + maximum weight preserve ratio for each layer. Default: 1.0 + reward: function + reward function type: + - acc_reward: accuracy * 0.01 + - acc_flops_reward: - (100 - accuracy) * 0.01 * np.log(flops) + Default: acc_reward + # parameters for channel pruning + n_calibration_batches: int + number of batches to extract layer information. Default: 60 + n_points_per_layer: int + number of feature points per layer. Default: 10 + channel_round: int + round channel to multiple of channel_round. Default: 8 + + # parameters for ddpg agent + hidden1: int + hidden num of first fully connect layer. Default: 300 + hidden2: int + hidden num of second fully connect layer. Default: 300 + lr_c: float + learning rate for critic. Default: 1e-3 + lr_a: float + learning rate for actor. Default: 1e-4 + warmup: int + number of episodes without training but only filling the replay memory. During warmup episodes, + random actions ares used for pruning. Default: 100 + discount: float + next Q value discount for deep Q value target. Default: 0.99 + bsize: int + minibatch size for training DDPG agent. Default: 64 + rmsize: int + memory size for each layer. Default: 100 + window_length: int + replay buffer window length. Default: 1 + tau: float + moving average for target network being used by soft_update. Default: 0.99 + # noise + init_delta: float + initial variance of truncated normal distribution + delta_decay: float + delta decay during exploration + + # parameters for training ddpg agent + max_episode_length: int + maximum episode length + output_dir: str + output directory to save log files and model files. Default: ./logs + debug: boolean + debug mode + train_episode: int + train iters each timestep. Default: 800 + epsilon: int + linear decay of exploration policy. Default: 50000 + seed: int + random seed to set for reproduce experiment. Default: None + """ + + def __init__( + self, + model, + config_list, + evaluator, + val_loader, + suffix=None, + model_type='mobilenet', + dataset='cifar10', + flops_ratio=0.5, + lbound=0.2, + rbound=1., + reward='acc_reward', + n_calibration_batches=60, + n_points_per_layer=10, + channel_round=8, + hidden1=300, + hidden2=300, + lr_c=1e-3, + lr_a=1e-4, + warmup=100, + discount=1., + bsize=64, + rmsize=100, + window_length=1, + tau=0.01, + init_delta=0.5, + delta_decay=0.99, + max_episode_length=1e9, + output_dir='./logs', + debug=False, + train_episode=800, + epsilon=50000, + seed=None): + + self.val_loader = val_loader + self.evaluator = evaluator + + if seed is not None: + np.random.seed(seed) + torch.manual_seed(seed) + torch.cuda.manual_seed(seed) + + checkpoint = deepcopy(model.state_dict()) + + super().__init__(model, config_list, optimizer=None) + + # build folder and logs + base_folder_name = '{}_{}_r{}_search'.format(model_type, dataset, flops_ratio) + if suffix is not None: + self.output_dir = os.path.join(output_dir, base_folder_name + '-' + suffix) + else: + self.output_dir = get_output_folder(output_dir, base_folder_name) + + self.env_args = Namespace( + model_type=model_type, + preserve_ratio=flops_ratio, + lbound=lbound, + rbound=rbound, + reward=reward, + n_calibration_batches=n_calibration_batches, + n_points_per_layer=n_points_per_layer, + channel_round=channel_round, + output=self.output_dir + ) + self.env = ChannelPruningEnv( + self, evaluator, val_loader, checkpoint, args=self.env_args) + _logger.info('=> Saving logs to %s', self.output_dir) + self.tfwriter = SummaryWriter(log_dir=self.output_dir) + self.text_writer = open(os.path.join(self.output_dir, 'log.txt'), 'w') + _logger.info('=> Output path: %s...', self.output_dir) + + nb_states = self.env.layer_embedding.shape[1] + nb_actions = 1 # just 1 action here + + rmsize = rmsize * len(self.env.prunable_idx) # for each layer + _logger.info('** Actual replay buffer size: %d', rmsize) + + self.ddpg_args = Namespace( + hidden1=hidden1, + hidden2=hidden2, + lr_c=lr_c, + lr_a=lr_a, + warmup=warmup, + discount=discount, + bsize=bsize, + rmsize=rmsize, + window_length=window_length, + tau=tau, + init_delta=init_delta, + delta_decay=delta_decay, + max_episode_length=max_episode_length, + debug=debug, + train_episode=train_episode, + epsilon=epsilon + ) + self.agent = DDPG(nb_states, nb_actions, self.ddpg_args) + + + def compress(self): + self.train(self.ddpg_args.train_episode, self.agent, self.env, self.output_dir) + + def train(self, num_episode, agent, env, output_dir): + agent.is_training = True + step = episode = episode_steps = 0 + episode_reward = 0. + observation = None + T = [] # trajectory + while episode < num_episode: # counting based on episode + # reset if it is the start of episode + if observation is None: + observation = deepcopy(env.reset()) + agent.reset(observation) + + # agent pick action ... + if episode <= self.ddpg_args.warmup: + action = agent.random_action() + # action = sample_from_truncated_normal_distribution(lower=0., upper=1., mu=env.preserve_ratio, sigma=0.5) + else: + action = agent.select_action(observation, episode=episode) + + # env response with next_observation, reward, terminate_info + observation2, reward, done, info = env.step(action) + + T.append([reward, deepcopy(observation), deepcopy(observation2), action, done]) + + # fix-length, never reach here + # if max_episode_length and episode_steps >= max_episode_length - 1: + # done = True + + # [optional] save intermideate model + if num_episode / 3 <= 1 or episode % int(num_episode / 3) == 0: + agent.save_model(output_dir) + + # update + step += 1 + episode_steps += 1 + episode_reward += reward + observation = deepcopy(observation2) + + if done: # end of episode + _logger.info( + '#%d: episode_reward: %.4f acc: %.4f, ratio: %.4f', + episode, episode_reward, + info['accuracy'], + info['compress_ratio'] + ) + self.text_writer.write( + '#{}: episode_reward:{:.4f} acc: {:.4f}, ratio: {:.4f}\n'.format( + episode, episode_reward, + info['accuracy'], + info['compress_ratio'] + ) + ) + final_reward = T[-1][0] + # print('final_reward: {}'.format(final_reward)) + # agent observe and update policy + for _, s_t, s_t1, a_t, done in T: + agent.observe(final_reward, s_t, s_t1, a_t, done) + if episode > self.ddpg_args.warmup: + agent.update_policy() + + #agent.memory.append( + # observation, + # agent.select_action(observation, episode=episode), + # 0., False + #) + + # reset + observation = None + episode_steps = 0 + episode_reward = 0. + episode += 1 + T = [] + + self.tfwriter.add_scalar('reward/last', final_reward, episode) + self.tfwriter.add_scalar('reward/best', env.best_reward, episode) + self.tfwriter.add_scalar('info/accuracy', info['accuracy'], episode) + self.tfwriter.add_scalar('info/compress_ratio', info['compress_ratio'], episode) + self.tfwriter.add_text('info/best_policy', str(env.best_strategy), episode) + # record the preserve rate for each layer + for i, preserve_rate in enumerate(env.strategy): + self.tfwriter.add_scalar('preserve_rate/{}'.format(i), preserve_rate, episode) + + self.text_writer.write('best reward: {}\n'.format(env.best_reward)) + self.text_writer.write('best policy: {}\n'.format(env.best_strategy)) + self.text_writer.close() diff --git a/utils/third_party/nni/algorithms/compression/pytorch/pruning/amc/channel_pruning_env.py b/utils/third_party/nni/algorithms/compression/pytorch/pruning/amc/channel_pruning_env.py new file mode 100644 index 0000000000000000000000000000000000000000..443daf7efb2f0fe7b909ab0330bb7a3264d59225 --- /dev/null +++ b/utils/third_party/nni/algorithms/compression/pytorch/pruning/amc/channel_pruning_env.py @@ -0,0 +1,543 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +import logging +import time +import math +import copy +import numpy as np +import torch +import torch.nn as nn + +from nni.compression.pytorch.compressor import PrunerModuleWrapper +from .. import AMCWeightMasker + +_logger = logging.getLogger(__name__) + +# for pruning +def acc_reward(net, acc, flops): + return acc * 0.01 + + +def acc_flops_reward(net, acc, flops): + error = (100 - acc) * 0.01 + return -error * np.log(flops) + + +class ChannelPruningEnv: + """ + Env for channel pruning search. + This class is used to prune model using specified pruner. It prunes one layer when + step() is called. When the last layer is pruned, it evaluate the pruned model using + evaluator, and use the returned value of evaluator as reward of the episode. + + Usage: + env = ChannelPruningEnv(pruner, evaluator, val_loader, checkpoint, env_args) + episode = 0 + T = [] + while episode < num_episode: + action = agent.select_action(observation) + observation2, reward, done, info = env.step(action) + T.append([reward, deepcopy(observation), deepcopy(observation2), action, done]) + + if done: # end of episode, last layer pruned + episode += 1 + # train agent with episode data + for _, s_t, s_t1, a_t, done in T: + agent.observe(final_reward, s_t, s_t1, a_t, done) + agent.update_policy() + T = [] + + Attributes: + prunable_idx: layer indices for pruable layers, the index values are the index + of list(self.model.modules()). Pruable layers are pointwise Conv2d layers and Linear + layers. + buffer_idx: layer indices for buffer layers which refers the depthwise layers. + Each depthwise layer is always followd by a pointwise layer for both mobilenet and + mobilenetv2. The depthwise layer's filters are pruned when its next pointwise layer's + corresponding input channels are pruned. + shared_idx: layer indices for layers which share input. + For example: [[1,4], [8, 10, 15]] means layer 1 and 4 share same input, and layer + 8, 10 and 15 share another input. + layer_embedding: embeddings for each prunable layers, the embedding is used as + observation for DDPG agent. + layer_info_dict: flops and number of parameters of each layer. + min_strategy_dict: key is layer index, value is a tuple, the first value is the minimum + action of input channel, the second value is the minimum action value of output channel. + strategy_dict: key is layer index, value is a tuple, the first value is the action of input + channel, the second value is the action of output channel. + + Parameters: + pruner: Pruner + NNI Pruner instance used to prune model. + evaluator: function + function to evaluate the pruned model. + The prototype of the function: + >>> def evaluator(val_loader, model): + >>> ... + >>> return acc + val_loader: torch.utils.data.DataLoader + Data loader of validation dataset. + checkpoint: dict + checkpoint of the model to be pruned. It is used to reset model at beginning of each + episode. + args: + A Namespace object containing following arguments: + model_type: str + model type to prune, currently 'mobilenet' and 'mobilenetv2' are supported. + flops_ratio: float + preserve flops ratio. + lbound: float + minimum weight preserve ratio for each layer. + rbound: float + maximum weight preserve ratio for each layer. + reward: function + reward function type + + # parameters for channel pruning + n_calibration_batches: int + number of batches to extract layer information. + n_points_per_layer: int + number of feature points per layer. + channel_round: int + round channel to multiple of channel_round. + + """ + def __init__(self, pruner, evaluator, val_loader, checkpoint, args): + self.pruner = pruner + self.model = pruner.bound_model + self.checkpoint = checkpoint + self.batch_size = val_loader.batch_size + self.preserve_ratio = args.preserve_ratio + self.channel_prune_masker = AMCWeightMasker(self.model, self.pruner, args.channel_round) + + # options from args + self.args = args + self.lbound = args.lbound + self.rbound = args.rbound + + self.n_calibration_batches = args.n_calibration_batches + self.n_points_per_layer = args.n_points_per_layer + self.channel_round = args.channel_round + + # sanity check + assert self.preserve_ratio > self.lbound, 'Error! You can not achieve preserve_ratio smaller than lbound!' + + # prepare data + self._val_loader = val_loader + self._validate = evaluator + + # build indexs + self._build_index() + self.n_prunable_layer = len(self.prunable_idx) + + # extract information for preparing + self._extract_layer_information() + + # build embedding (static part) + self._build_state_embedding() + + # build reward + self.reset() # restore weight + self.org_acc = self._validate(self._val_loader, self.model) + _logger.info('=> original acc: %.3f', self.org_acc) + self.org_model_size = sum(self.wsize_list) + _logger.info('=> original weight size: %.4f M param', self.org_model_size * 1. / 1e6) + self.org_flops = sum(self.flops_list) + _logger.info('=> FLOPs:') + _logger.info([self.layer_info_dict[idx]['flops']/1e6 for idx in sorted(self.layer_info_dict.keys())]) + _logger.info('=> original FLOPs: %.4f M', self.org_flops * 1. / 1e6) + + self.expected_preserve_computation = self.preserve_ratio * self.org_flops + + self.reward = eval(args.reward) + + self.best_reward = -math.inf + self.best_strategy = None + self.best_d_prime_list = None + self.best_masks = None + + self.org_w_size = sum(self.wsize_list) + + def step(self, action): + # Pseudo prune and get the corresponding statistics. The real pruning happens till the end of all pseudo pruning + if self.visited[self.cur_ind]: + action = self.strategy_dict[self.prunable_idx[self.cur_ind]][0] + preserve_idx = self.index_buffer[self.cur_ind] + else: + action = self._action_wall(action) # percentage to preserve + preserve_idx = None + # prune and update action + action, d_prime, preserve_idx = self.prune_kernel(self.prunable_idx[self.cur_ind], action, preserve_idx) + if not self.visited[self.cur_ind]: + for group in self.shared_idx: + if self.cur_ind in group: # set the shared ones + for g_idx in group: + self.strategy_dict[self.prunable_idx[g_idx]][0] = action + self.strategy_dict[self.prunable_idx[g_idx - 1]][1] = action + self.visited[g_idx] = True + self.index_buffer[g_idx] = preserve_idx.copy() + + self.strategy.append(action) # save action to strategy + self.d_prime_list.append(d_prime) + + self.strategy_dict[self.prunable_idx[self.cur_ind]][0] = action + if self.cur_ind > 0: + self.strategy_dict[self.prunable_idx[self.cur_ind - 1]][1] = action + + # all the actions are made + if self._is_final_layer(): + assert len(self.strategy) == len(self.prunable_idx) + current_flops = self._cur_flops() + acc_t1 = time.time() + acc = self._validate(self._val_loader, self.model) + acc_t2 = time.time() + self.val_time = acc_t2 - acc_t1 + compress_ratio = current_flops * 1. / self.org_flops + info_set = {'compress_ratio': compress_ratio, 'accuracy': acc, 'strategy': self.strategy.copy()} + reward = self.reward(self, acc, current_flops) + + if reward > self.best_reward: + self.best_reward = reward + self.best_strategy = self.strategy.copy() + self.best_d_prime_list = self.d_prime_list.copy() + best_model = os.path.join(self.args.output, 'best_model.pth') + best_mask = os.path.join(self.args.output, 'best_mask.pth') + self.pruner.export_model(model_path=best_model, mask_path=best_mask) + _logger.info('New best reward: %.4f, acc: %.4f, compress: %.4f', self.best_reward, acc, compress_ratio) + _logger.info('New best policy: %s', self.best_strategy) + _logger.info('New best d primes: %s', self.best_d_prime_list) + obs = self.layer_embedding[self.cur_ind, :].copy() # actually the same as the last state + done = True + return obs, reward, done, info_set + + info_set = None + reward = 0 + done = False + self.visited[self.cur_ind] = True # set to visited + self.cur_ind += 1 # the index of next layer + # build next state (in-place modify) + self.layer_embedding[self.cur_ind][-3] = self._cur_reduced() * 1. / self.org_flops # reduced + self.layer_embedding[self.cur_ind][-2] = sum(self.flops_list[self.cur_ind + 1:]) * 1. / self.org_flops # rest + self.layer_embedding[self.cur_ind][-1] = self.strategy[-1] # last action + obs = self.layer_embedding[self.cur_ind, :].copy() + + return obs, reward, done, info_set + + def reset(self): + # restore env by loading the checkpoint + self.pruner.reset(self.checkpoint) + self.cur_ind = 0 + self.strategy = [] # pruning strategy + self.d_prime_list = [] + self.strategy_dict = copy.deepcopy(self.min_strategy_dict) + # reset layer embeddings + self.layer_embedding[:, -1] = 1. + self.layer_embedding[:, -2] = 0. + self.layer_embedding[:, -3] = 0. + obs = self.layer_embedding[0].copy() + obs[-2] = sum(self.wsize_list[1:]) * 1. / sum(self.wsize_list) + self.extract_time = 0 + self.fit_time = 0 + self.val_time = 0 + # for share index + self.visited = [False] * len(self.prunable_idx) + self.index_buffer = {} + return obs + + def prune_kernel(self, op_idx, preserve_ratio, preserve_idx=None): + m_list = list(self.model.modules()) + op = m_list[op_idx] + assert (0. < preserve_ratio <= 1.) + assert type(op) == PrunerModuleWrapper + if preserve_ratio == 1: # do not prune + if (preserve_idx is None) or (len(preserve_idx) == op.module.weight.size(1)): + return 1., op.module.weight.size(1), None # should be a full index + op.input_feat = self.layer_info_dict[op_idx]['input_feat'] + op.output_feat = self.layer_info_dict[op_idx]['output_feat'] + + masks = self.channel_prune_masker.calc_mask(sparsity=1-preserve_ratio, wrapper=op, preserve_idx=preserve_idx) + m = masks['weight_mask'].cpu().data + if type(op.module) == nn.Conv2d: + d_prime = (m.sum((0, 2, 3)) > 0).sum().item() + preserve_idx = np.nonzero((m.sum((0, 2, 3)) > 0).numpy())[0] + else: + assert type(op.module) == nn.Linear + d_prime = (m.sum(1) > 0).sum().item() + preserve_idx = np.nonzero((m.sum(1) > 0).numpy())[0] + + op.weight_mask = masks['weight_mask'] + if hasattr(op.module, 'bias') and op.module.bias is not None and 'bias_mask' in masks: + op.bias_mask = masks['bias_mask'] + + action = (m == 1).sum().item() / m.numel() + return action, d_prime, preserve_idx + + def _is_final_layer(self): + return self.cur_ind == len(self.prunable_idx) - 1 + + def _action_wall(self, action): + """ + Limit the action generated by DDPG for this layer by two constraints: + 1. The total flops must meet the flops reduce target. + For example: the original flops of entire model is 1000, target flops ratio is 0.5, target flops + is 1000*0.5 = 500. The reduced flops of other layers is 400, so the remaining flops quota is 500-400=100, + if the total original flops of this layer is 250, then the maximum ratio is 100/250 = 0.4. So the + action of this layer can not be greater than 0.4. + 2. The action must be greater than lbound which is stored in self.strategy_dict. + """ + assert len(self.strategy) == self.cur_ind + + action = float(action) + action = np.clip(action, 0, 1) + + other_comp = 0 + this_comp = 0 + for i, idx in enumerate(self.prunable_idx): + flop = self.layer_info_dict[idx]['flops'] + buffer_flop = self._get_buffer_flops(idx) + + if i == self.cur_ind - 1: # TODO: add other member in the set + this_comp += flop * self.strategy_dict[idx][0] + # add buffer (but not influenced by ratio) + other_comp += buffer_flop * self.strategy_dict[idx][0] + elif i == self.cur_ind: + this_comp += flop * self.strategy_dict[idx][1] + # also add buffer here (influenced by ratio) + this_comp += buffer_flop + else: + other_comp += flop * self.strategy_dict[idx][0] * self.strategy_dict[idx][1] + # add buffer + other_comp += buffer_flop * self.strategy_dict[idx][0] # only consider input reduction + + self.expected_min_preserve = other_comp + this_comp * action + max_preserve_ratio = (self.expected_preserve_computation - other_comp) * 1. / this_comp + + action = np.minimum(action, max_preserve_ratio) + action = np.maximum(action, self.strategy_dict[self.prunable_idx[self.cur_ind]][0]) # impossible (should be) + + return action + + def _get_buffer_flops(self, idx): + buffer_idx = self.buffer_dict[idx] + buffer_flop = sum([self.layer_info_dict[_]['flops'] for _ in buffer_idx]) + return buffer_flop + + def _cur_flops(self): + flops = 0 + for idx in self.prunable_idx: + c, n = self.strategy_dict[idx] # input, output pruning ratio + flops += self.layer_info_dict[idx]['flops'] * c * n + # add buffer computation + flops += self._get_buffer_flops(idx) * c # only related to input channel reduction + return flops + + def _cur_reduced(self): + # return the reduced weight + reduced = self.org_flops - self._cur_flops() + return reduced + + def _build_index(self): + """ + Build following information/data for later pruning: + self.prunable_idx: layer indices for pruable layers, the index values are the index + of list(self.model.modules()). Pruable layers are pointwise Conv2d layers and Linear + layers. + self.prunable_ops: prunable modules + self.buffer_idx: layer indices for buffer layers which refers the depthwise layers. + Each depthwise layer is always followd by a pointwise layer for both mobilenet and + mobilenetv2. The depthwise layer's filters are pruned when its next pointwise layer's + corresponding input channels are pruned. + self.shared_idx: layer indices for layers which share input. + For example: [[1,4], [8, 10, 15]] means layer 1 and 4 share same input, and layer + 8, 10 and 15 share another input. + self.org_channels: number of input channels for each layer + self.min_strategy_dict: key is layer index, value is a tuple, the first value is the minimum + action of input channel, the second value is the minimum action value of output channel. + self.strategy_dict: same as self.min_strategy_dict, but it will be updated later. + """ + self.prunable_idx = [] + self.prunable_ops = [] + self.layer_type_dict = {} + self.strategy_dict = {} + self.buffer_dict = {} + this_buffer_list = [] + self.org_channels = [] + # build index and the min strategy dict + for i, m in enumerate(self.model.modules()): + if isinstance(m, PrunerModuleWrapper): + m = m.module + if type(m) == nn.Conv2d and m.groups == m.in_channels: # depth-wise conv, buffer + this_buffer_list.append(i) + else: # really prunable + self.prunable_idx.append(i) + self.prunable_ops.append(m) + self.layer_type_dict[i] = type(m) + self.buffer_dict[i] = this_buffer_list + this_buffer_list = [] # empty + self.org_channels.append(m.in_channels if type(m) == nn.Conv2d else m.in_features) + + self.strategy_dict[i] = [self.lbound, self.lbound] + + self.strategy_dict[self.prunable_idx[0]][0] = 1 # modify the input + self.strategy_dict[self.prunable_idx[-1]][1] = 1 # modify the output + + self.shared_idx = [] + if self.args.model_type == 'mobilenetv2': # TODO: to be tested! Share index for residual connection + connected_idx = [4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32] # to be partitioned + last_ch = -1 + share_group = None + for c_idx in connected_idx: + if self.prunable_ops[c_idx].in_channels != last_ch: # new group + last_ch = self.prunable_ops[c_idx].in_channels + if share_group is not None: + self.shared_idx.append(share_group) + share_group = [c_idx] + else: # same group + share_group.append(c_idx) + self.shared_idx.append(share_group) + _logger.info('=> Conv layers to share channels: %s', self.shared_idx) + + self.min_strategy_dict = copy.deepcopy(self.strategy_dict) + + self.buffer_idx = [] + for _, v in self.buffer_dict.items(): + self.buffer_idx += v + + _logger.info('=> Prunable layer idx: %s', self.prunable_idx) + _logger.info('=> Buffer layer idx: %s', self.buffer_idx) + _logger.info('=> Shared idx: %s', self.shared_idx) + _logger.info('=> Initial min strategy dict: %s', self.min_strategy_dict) + + # added for supporting residual connections during pruning + self.visited = [False] * len(self.prunable_idx) + self.index_buffer = {} + + def _extract_layer_information(self): + m_list = list(self.model.modules()) + + self.data_saver = [] + self.layer_info_dict = dict() + self.wsize_list = [] + self.flops_list = [] + + from .lib.utils import measure_layer_for_pruning + + # extend the forward fn to record layer info + def new_forward(m): + def lambda_forward(x): + m.input_feat = x.clone() + #TODO replace this flops counter with nni.compression.torch.utils.counter.count_flops_params + measure_layer_for_pruning(m, x) + y = m.old_forward(x) + m.output_feat = y.clone() + return y + + return lambda_forward + + device = None + for idx in self.prunable_idx + self.buffer_idx: # get all + m = m_list[idx] + m.old_forward = m.forward + m.forward = new_forward(m) + if device is None and type(m) == PrunerModuleWrapper: + device = m.module.weight.device + + # now let the image flow + _logger.info('=> Extracting information...') + with torch.no_grad(): + for i_b, (inputs, target) in enumerate(self._val_loader): # use image from train set + if i_b == self.n_calibration_batches: + break + self.data_saver.append((inputs.clone(), target.clone())) + input_var = torch.autograd.Variable(inputs).to(device) + + # inference and collect stats + _ = self.model(input_var) + + if i_b == 0: # first batch + for idx in self.prunable_idx + self.buffer_idx: + self.layer_info_dict[idx] = dict() + self.layer_info_dict[idx]['params'] = m_list[idx].params + self.layer_info_dict[idx]['flops'] = m_list[idx].flops + self.wsize_list.append(m_list[idx].params) + self.flops_list.append(m_list[idx].flops) + _logger.info('flops: %s', self.flops_list) + for idx in self.prunable_idx: + f_in_np = m_list[idx].input_feat.data.cpu().numpy() + f_out_np = m_list[idx].output_feat.data.cpu().numpy() + if len(f_in_np.shape) == 4: # conv + if self.prunable_idx.index(idx) == 0: # first conv + f_in2save, f_out2save = None, None + elif m_list[idx].module.weight.size(3) > 1: # normal conv + f_in2save, f_out2save = f_in_np, f_out_np + else: # 1x1 conv + # assert f_out_np.shape[2] == f_in_np.shape[2] # now support k=3 + randx = np.random.randint(0, f_out_np.shape[2] - 0, self.n_points_per_layer) + randy = np.random.randint(0, f_out_np.shape[3] - 0, self.n_points_per_layer) + # input: [N, C, H, W] + self.layer_info_dict[idx][(i_b, 'randx')] = randx.copy() + self.layer_info_dict[idx][(i_b, 'randy')] = randy.copy() + + f_in2save = f_in_np[:, :, randx, randy].copy().transpose(0, 2, 1)\ + .reshape(self.batch_size * self.n_points_per_layer, -1) + + f_out2save = f_out_np[:, :, randx, randy].copy().transpose(0, 2, 1) \ + .reshape(self.batch_size * self.n_points_per_layer, -1) + else: + assert len(f_in_np.shape) == 2 + f_in2save = f_in_np.copy() + f_out2save = f_out_np.copy() + if 'input_feat' not in self.layer_info_dict[idx]: + self.layer_info_dict[idx]['input_feat'] = f_in2save + self.layer_info_dict[idx]['output_feat'] = f_out2save + else: + self.layer_info_dict[idx]['input_feat'] = np.vstack( + (self.layer_info_dict[idx]['input_feat'], f_in2save)) + self.layer_info_dict[idx]['output_feat'] = np.vstack( + (self.layer_info_dict[idx]['output_feat'], f_out2save)) + + def _build_state_embedding(self): + # build the static part of the state embedding + _logger.info('Building state embedding...') + layer_embedding = [] + module_list = list(self.model.modules()) + for i, ind in enumerate(self.prunable_idx): + m = module_list[ind].module + this_state = [] + if type(m) == nn.Conv2d: + this_state.append(i) # index + this_state.append(0) # layer type, 0 for conv + this_state.append(m.in_channels) # in channels + this_state.append(m.out_channels) # out channels + this_state.append(m.stride[0]) # stride + this_state.append(m.kernel_size[0]) # kernel size + this_state.append(np.prod(m.weight.size())) # weight size + elif type(m) == nn.Linear: + this_state.append(i) # index + this_state.append(1) # layer type, 1 for fc + this_state.append(m.in_features) # in channels + this_state.append(m.out_features) # out channels + this_state.append(0) # stride + this_state.append(1) # kernel size + this_state.append(np.prod(m.weight.size())) # weight size + + # this 3 features need to be changed later + this_state.append(0.) # reduced + this_state.append(0.) # rest + this_state.append(1.) # a_{t-1} + layer_embedding.append(np.array(this_state)) + + # normalize the state + layer_embedding = np.array(layer_embedding, 'float') + _logger.info('=> shape of embedding (n_layer * n_dim): %s', layer_embedding.shape) + assert len(layer_embedding.shape) == 2, layer_embedding.shape + for i in range(layer_embedding.shape[1]): + fmin = min(layer_embedding[:, i]) + fmax = max(layer_embedding[:, i]) + if fmax - fmin > 0: + layer_embedding[:, i] = (layer_embedding[:, i] - fmin) / (fmax - fmin) + + self.layer_embedding = layer_embedding + diff --git a/utils/third_party/nni/algorithms/compression/pytorch/pruning/amc/lib/__init__.py b/utils/third_party/nni/algorithms/compression/pytorch/pruning/amc/lib/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/utils/third_party/nni/algorithms/compression/pytorch/pruning/amc/lib/agent.py b/utils/third_party/nni/algorithms/compression/pytorch/pruning/amc/lib/agent.py new file mode 100644 index 0000000000000000000000000000000000000000..fe066301b8b5e4325f92a1b98885ae547a7ecee3 --- /dev/null +++ b/utils/third_party/nni/algorithms/compression/pytorch/pruning/amc/lib/agent.py @@ -0,0 +1,232 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import numpy as np + +import torch +import torch.nn as nn +from torch.optim import Adam + +from .memory import SequentialMemory +from .utils import to_numpy, to_tensor + +criterion = nn.MSELoss() +USE_CUDA = torch.cuda.is_available() + + +class Actor(nn.Module): + def __init__(self, nb_states, nb_actions, hidden1=400, hidden2=300): + super(Actor, self).__init__() + self.fc1 = nn.Linear(nb_states, hidden1) + self.fc2 = nn.Linear(hidden1, hidden2) + self.fc3 = nn.Linear(hidden2, nb_actions) + self.relu = nn.ReLU() + self.sigmoid = nn.Sigmoid() + + def forward(self, x): + out = self.fc1(x) + out = self.relu(out) + out = self.fc2(out) + out = self.relu(out) + out = self.fc3(out) + out = self.sigmoid(out) + return out + + +class Critic(nn.Module): + def __init__(self, nb_states, nb_actions, hidden1=400, hidden2=300): + super(Critic, self).__init__() + self.fc11 = nn.Linear(nb_states, hidden1) + self.fc12 = nn.Linear(nb_actions, hidden1) + self.fc2 = nn.Linear(hidden1, hidden2) + self.fc3 = nn.Linear(hidden2, 1) + self.relu = nn.ReLU() + + def forward(self, xs): + x, a = xs + out = self.fc11(x) + self.fc12(a) + out = self.relu(out) + out = self.fc2(out) + out = self.relu(out) + out = self.fc3(out) + return out + + +class DDPG(object): + def __init__(self, nb_states, nb_actions, args): + + self.nb_states = nb_states + self.nb_actions = nb_actions + + # Create Actor and Critic Network + net_cfg = { + 'hidden1': args.hidden1, + 'hidden2': args.hidden2, + # 'init_w': args.init_w + } + self.actor = Actor(self.nb_states, self.nb_actions, **net_cfg) + self.actor_target = Actor(self.nb_states, self.nb_actions, **net_cfg) + self.actor_optim = Adam(self.actor.parameters(), lr=args.lr_a) + + self.critic = Critic(self.nb_states, self.nb_actions, **net_cfg) + self.critic_target = Critic(self.nb_states, self.nb_actions, **net_cfg) + self.critic_optim = Adam(self.critic.parameters(), lr=args.lr_c) + + self.hard_update(self.actor_target, self.actor) # Make sure target is with the same weight + self.hard_update(self.critic_target, self.critic) + + # Create replay buffer + self.memory = SequentialMemory(limit=args.rmsize, window_length=args.window_length) + # self.random_process = OrnsteinUhlenbeckProcess(size=nb_actions, theta=args.ou_theta, mu=args.ou_mu, + # sigma=args.ou_sigma) + + # Hyper-parameters + self.batch_size = args.bsize + self.tau = args.tau + self.discount = args.discount + self.depsilon = 1.0 / args.epsilon + self.lbound = 0. # args.lbound + self.rbound = 1. # args.rbound + + # noise + self.init_delta = args.init_delta + self.delta_decay = args.delta_decay + self.warmup = args.warmup + + # + self.epsilon = 1.0 + # self.s_t = None # Most recent state + # self.a_t = None # Most recent action + self.is_training = True + + # + if USE_CUDA: self.cuda() + + # moving average baseline + self.moving_average = None + self.moving_alpha = 0.5 # based on batch, so small + + def update_policy(self): + # Sample batch + state_batch, action_batch, reward_batch, \ + next_state_batch, terminal_batch = self.memory.sample_and_split(self.batch_size) + + # normalize the reward + batch_mean_reward = np.mean(reward_batch) + if self.moving_average is None: + self.moving_average = batch_mean_reward + else: + self.moving_average += self.moving_alpha * (batch_mean_reward - self.moving_average) + reward_batch -= self.moving_average + # if reward_batch.std() > 0: + # reward_batch /= reward_batch.std() + + # Prepare for the target q batch + with torch.no_grad(): + next_q_values = self.critic_target([ + to_tensor(next_state_batch), + self.actor_target(to_tensor(next_state_batch)), + ]) + + target_q_batch = to_tensor(reward_batch) + \ + self.discount * to_tensor(terminal_batch.astype(np.float)) * next_q_values + + # Critic update + self.critic.zero_grad() + + q_batch = self.critic([to_tensor(state_batch), to_tensor(action_batch)]) + + value_loss = criterion(q_batch, target_q_batch) + value_loss.backward() + self.critic_optim.step() + + # Actor update + self.actor.zero_grad() + + policy_loss = -self.critic([ # pylint: disable=all + to_tensor(state_batch), + self.actor(to_tensor(state_batch)) + ]) + + policy_loss = policy_loss.mean() + policy_loss.backward() + self.actor_optim.step() + + # Target update + self.soft_update(self.actor_target, self.actor) + self.soft_update(self.critic_target, self.critic) + + def eval(self): + self.actor.eval() + self.actor_target.eval() + self.critic.eval() + self.critic_target.eval() + + def cuda(self): + self.actor.cuda() + self.actor_target.cuda() + self.critic.cuda() + self.critic_target.cuda() + + def observe(self, r_t, s_t, s_t1, a_t, done): + if self.is_training: + self.memory.append(s_t, a_t, r_t, done) # save to memory + # self.s_t = s_t1 + + def random_action(self): + action = np.random.uniform(self.lbound, self.rbound, self.nb_actions) + # self.a_t = action + return action + + def select_action(self, s_t, episode): + # assert episode >= self.warmup, 'Episode: {} warmup: {}'.format(episode, self.warmup) + action = to_numpy(self.actor(to_tensor(np.array(s_t).reshape(1, -1)))).squeeze(0) + delta = self.init_delta * (self.delta_decay ** (episode - self.warmup)) + # action += self.is_training * max(self.epsilon, 0) * self.random_process.sample() + action = self.sample_from_truncated_normal_distribution(lower=self.lbound, upper=self.rbound, mu=action, sigma=delta) + action = np.clip(action, self.lbound, self.rbound) + + # self.a_t = action + return action + + def reset(self, obs): + pass + # self.s_t = obs + # self.random_process.reset_states() + + def load_weights(self, output): + if output is None: return + + self.actor.load_state_dict( + torch.load('{}/actor.pkl'.format(output)) + ) + + self.critic.load_state_dict( + torch.load('{}/critic.pkl'.format(output)) + ) + + def save_model(self, output): + torch.save( + self.actor.state_dict(), + '{}/actor.pkl'.format(output) + ) + torch.save( + self.critic.state_dict(), + '{}/critic.pkl'.format(output) + ) + + def soft_update(self, target, source): + for target_param, param in zip(target.parameters(), source.parameters()): + target_param.data.copy_( + target_param.data * (1.0 - self.tau) + param.data * self.tau + ) + + def hard_update(self, target, source): + for target_param, param in zip(target.parameters(), source.parameters()): + target_param.data.copy_(param.data) + + def sample_from_truncated_normal_distribution(self, lower, upper, mu, sigma, size=1): + from scipy import stats + return stats.truncnorm.rvs((lower-mu)/sigma, (upper-mu)/sigma, loc=mu, scale=sigma, size=size) + + diff --git a/utils/third_party/nni/algorithms/compression/pytorch/pruning/amc/lib/memory.py b/utils/third_party/nni/algorithms/compression/pytorch/pruning/amc/lib/memory.py new file mode 100644 index 0000000000000000000000000000000000000000..57bbcfceb86a20092968c9dc75a618221e119174 --- /dev/null +++ b/utils/third_party/nni/algorithms/compression/pytorch/pruning/amc/lib/memory.py @@ -0,0 +1,227 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from __future__ import absolute_import +from collections import deque, namedtuple +import warnings +import random + +import numpy as np + +# [reference] https://github.com/matthiasplappert/keras-rl/blob/master/rl/memory.py + +# This is to be understood as a transition: Given `state0`, performing `action` +# yields `reward` and results in `state1`, which might be `terminal`. +Experience = namedtuple('Experience', 'state0, action, reward, state1, terminal1') + + +def sample_batch_indexes(low, high, size): + if high - low >= size: + # We have enough data. Draw without replacement, that is each index is unique in the + # batch. We cannot use `np.random.choice` here because it is horribly inefficient as + # the memory grows. See https://github.com/numpy/numpy/issues/2764 for a discussion. + # `random.sample` does the same thing (drawing without replacement) and is way faster. + r = range(low, high) + batch_idxs = random.sample(r, size) + else: + # Not enough data. Help ourselves with sampling from the range, but the same index + # can occur multiple times. This is not good and should be avoided by picking a + # large enough warm-up phase. + warnings.warn( + 'Not enough entries to sample without replacement. ' + 'Consider increasing your warm-up phase to avoid oversampling!') + batch_idxs = np.random.random_integers(low, high - 1, size=size) + assert len(batch_idxs) == size + return batch_idxs + + +class RingBuffer(object): + def __init__(self, maxlen): + self.maxlen = maxlen + self.start = 0 + self.length = 0 + self.data = [None for _ in range(maxlen)] + + def __len__(self): + return self.length + + def __getitem__(self, idx): + if idx < 0 or idx >= self.length: + raise KeyError() + return self.data[(self.start + idx) % self.maxlen] + + def append(self, v): + if self.length < self.maxlen: + # We have space, simply increase the length. + self.length += 1 + elif self.length == self.maxlen: + # No space, "remove" the first item. + self.start = (self.start + 1) % self.maxlen + else: + # This should never happen. + raise RuntimeError() + self.data[(self.start + self.length - 1) % self.maxlen] = v + + +def zeroed_observation(observation): + if hasattr(observation, 'shape'): + return np.zeros(observation.shape) + elif hasattr(observation, '__iter__'): + out = [] + for x in observation: + out.append(zeroed_observation(x)) + return out + else: + return 0. + + +class Memory(object): + def __init__(self, window_length, ignore_episode_boundaries=False): + self.window_length = window_length + self.ignore_episode_boundaries = ignore_episode_boundaries + + self.recent_observations = deque(maxlen=window_length) + self.recent_terminals = deque(maxlen=window_length) + + def sample(self, batch_size, batch_idxs=None): + raise NotImplementedError() + + def append(self, observation, action, reward, terminal, training=True): + self.recent_observations.append(observation) + self.recent_terminals.append(terminal) + + def get_recent_state(self, current_observation): + # This code is slightly complicated by the fact that subsequent observations might be + # from different episodes. We ensure that an experience never spans multiple episodes. + # This is probably not that important in practice but it seems cleaner. + state = [current_observation] + idx = len(self.recent_observations) - 1 + for offset in range(0, self.window_length - 1): + current_idx = idx - offset + current_terminal = self.recent_terminals[current_idx - 1] if current_idx - 1 >= 0 else False + if current_idx < 0 or (not self.ignore_episode_boundaries and current_terminal): + # The previously handled observation was terminal, don't add the current one. + # Otherwise we would leak into a different episode. + break + state.insert(0, self.recent_observations[current_idx]) + while len(state) < self.window_length: + state.insert(0, zeroed_observation(state[0])) + return state + + def get_config(self): + config = { + 'window_length': self.window_length, + 'ignore_episode_boundaries': self.ignore_episode_boundaries, + } + return config + + +class SequentialMemory(Memory): + def __init__(self, limit, **kwargs): + super(SequentialMemory, self).__init__(**kwargs) + + self.limit = limit + + # Do not use deque to implement the memory. This data structure may seem convenient but + # it is way too slow on random access. Instead, we use our own ring buffer implementation. + self.actions = RingBuffer(limit) + self.rewards = RingBuffer(limit) + self.terminals = RingBuffer(limit) + self.observations = RingBuffer(limit) + + def sample(self, batch_size, batch_idxs=None): + if batch_idxs is None: + # Draw random indexes such that we have at least a single entry before each + # index. + batch_idxs = sample_batch_indexes(0, self.nb_entries - 1, size=batch_size) + batch_idxs = np.array(batch_idxs) + 1 + assert np.min(batch_idxs) >= 1 + assert np.max(batch_idxs) < self.nb_entries + assert len(batch_idxs) == batch_size + + # Create experiences + experiences = [] + for idx in batch_idxs: + terminal0 = self.terminals[idx - 2] if idx >= 2 else False + while terminal0: + # Skip this transition because the environment was reset here. Select a new, random + # transition and use this instead. This may cause the batch to contain the same + # transition twice. + idx = sample_batch_indexes(1, self.nb_entries, size=1)[0] + terminal0 = self.terminals[idx - 2] if idx >= 2 else False + assert 1 <= idx < self.nb_entries + + # This code is slightly complicated by the fact that subsequent observations might be + # from different episodes. We ensure that an experience never spans multiple episodes. + # This is probably not that important in practice but it seems cleaner. + state0 = [self.observations[idx - 1]] + for offset in range(0, self.window_length - 1): + current_idx = idx - 2 - offset + current_terminal = self.terminals[current_idx - 1] if current_idx - 1 > 0 else False + if current_idx < 0 or (not self.ignore_episode_boundaries and current_terminal): + # The previously handled observation was terminal, don't add the current one. + # Otherwise we would leak into a different episode. + break + state0.insert(0, self.observations[current_idx]) + while len(state0) < self.window_length: + state0.insert(0, zeroed_observation(state0[0])) + action = self.actions[idx - 1] + reward = self.rewards[idx - 1] + terminal1 = self.terminals[idx - 1] + + # Okay, now we need to create the follow-up state. This is state0 shifted on timestep + # to the right. Again, we need to be careful to not include an observation from the next + # episode if the last state is terminal. + state1 = [np.copy(x) for x in state0[1:]] + state1.append(self.observations[idx]) + + assert len(state0) == self.window_length + assert len(state1) == len(state0) + experiences.append(Experience(state0=state0, action=action, reward=reward, + state1=state1, terminal1=terminal1)) + assert len(experiences) == batch_size + return experiences + + def sample_and_split(self, batch_size, batch_idxs=None): + experiences = self.sample(batch_size, batch_idxs) + + state0_batch = [] + reward_batch = [] + action_batch = [] + terminal1_batch = [] + state1_batch = [] + for e in experiences: + state0_batch.append(e.state0) + state1_batch.append(e.state1) + reward_batch.append(e.reward) + action_batch.append(e.action) + terminal1_batch.append(0. if e.terminal1 else 1.) + + # Prepare and validate parameters. + state0_batch = np.array(state0_batch, 'double').reshape(batch_size, -1) + state1_batch = np.array(state1_batch, 'double').reshape(batch_size, -1) + terminal1_batch = np.array(terminal1_batch, 'double').reshape(batch_size, -1) + reward_batch = np.array(reward_batch, 'double').reshape(batch_size, -1) + action_batch = np.array(action_batch, 'double').reshape(batch_size, -1) + + return state0_batch, action_batch, reward_batch, state1_batch, terminal1_batch + + def append(self, observation, action, reward, terminal, training=True): + super(SequentialMemory, self).append(observation, action, reward, terminal, training=training) + + # This needs to be understood as follows: in `observation`, take `action`, obtain `reward` + # and weather the next state is `terminal` or not. + if training: + self.observations.append(observation) + self.actions.append(action) + self.rewards.append(reward) + self.terminals.append(terminal) + + @property + def nb_entries(self): + return len(self.observations) + + def get_config(self): + config = super(SequentialMemory, self).get_config() + config['limit'] = self.limit + return config diff --git a/utils/third_party/nni/algorithms/compression/pytorch/pruning/amc/lib/net_measure.py b/utils/third_party/nni/algorithms/compression/pytorch/pruning/amc/lib/net_measure.py new file mode 100644 index 0000000000000000000000000000000000000000..2c9e815116288c11de98b4061b4dbfc3df029a7e --- /dev/null +++ b/utils/third_party/nni/algorithms/compression/pytorch/pruning/amc/lib/net_measure.py @@ -0,0 +1,123 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import torch + +# [reference] https://github.com/ShichenLiu/CondenseNet/blob/master/utils.py + + +def get_num_gen(gen): + return sum(1 for _ in gen) + + +def is_leaf(model): + return get_num_gen(model.children()) == 0 + + +def get_layer_info(layer): + layer_str = str(layer) + type_name = layer_str[:layer_str.find('(')].strip() + return type_name + + +def get_layer_param(model): + import operator + import functools + + return sum([functools.reduce(operator.mul, i.size(), 1) for i in model.parameters()]) + +count_ops = 0 +count_params = 0 + +def measure_layer(layer, x): + global count_ops, count_params + delta_ops = 0 + delta_params = 0 + multi_add = 1 + type_name = get_layer_info(layer) + + # ops_conv + if type_name in ['Conv2d']: + out_h = int((x.size()[2] + 2 * layer.padding[0] - layer.kernel_size[0]) / + layer.stride[0] + 1) + out_w = int((x.size()[3] + 2 * layer.padding[1] - layer.kernel_size[1]) / + layer.stride[1] + 1) + delta_ops = layer.in_channels * layer.out_channels * layer.kernel_size[0] * \ + layer.kernel_size[1] * out_h * out_w / layer.groups * multi_add + delta_params = get_layer_param(layer) + + # ops_nonlinearity + elif type_name in ['ReLU']: + delta_ops = x.numel() / x.size(0) + delta_params = get_layer_param(layer) + + # ops_pooling + elif type_name in ['AvgPool2d']: + in_w = x.size()[2] + kernel_ops = layer.kernel_size * layer.kernel_size + out_w = int((in_w + 2 * layer.padding - layer.kernel_size) / layer.stride + 1) + out_h = int((in_w + 2 * layer.padding - layer.kernel_size) / layer.stride + 1) + delta_ops = x.size()[1] * out_w * out_h * kernel_ops + delta_params = get_layer_param(layer) + + elif type_name in ['AdaptiveAvgPool2d']: + delta_ops = x.size()[1] * x.size()[2] * x.size()[3] + delta_params = get_layer_param(layer) + + # ops_linear + elif type_name in ['Linear']: + weight_ops = layer.weight.numel() * multi_add + bias_ops = layer.bias.numel() + delta_ops = weight_ops + bias_ops + delta_params = get_layer_param(layer) + + # ops_nothing + elif type_name in ['BatchNorm2d', 'Dropout2d', 'DropChannel', 'Dropout']: + delta_params = get_layer_param(layer) + + # unknown layer type + else: + delta_params = get_layer_param(layer) + + count_ops += delta_ops + count_params += delta_params + + return + + +def measure_model(model, H, W, device): + global count_ops, count_params + count_ops = 0 + count_params = 0 + data = torch.zeros(2, 3, H, W).to(device) + + def should_measure(x): + return is_leaf(x) + + def modify_forward(model): + for child in model.children(): + if should_measure(child): + def new_forward(m): + def lambda_forward(x): + measure_layer(m, x) + return m.old_forward(x) + return lambda_forward + child.old_forward = child.forward + child.forward = new_forward(child) + else: + modify_forward(child) + + def restore_forward(model): + for child in model.children(): + # leaf node + if is_leaf(child) and hasattr(child, 'old_forward'): + child.forward = child.old_forward + child.old_forward = None + else: + restore_forward(child) + + modify_forward(model) + model.forward(data) + restore_forward(model) + + return count_ops, count_params diff --git a/utils/third_party/nni/algorithms/compression/pytorch/pruning/amc/lib/utils.py b/utils/third_party/nni/algorithms/compression/pytorch/pruning/amc/lib/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..f875e8d7d9b363d848aa2424b4f06ddaeb1dc1a6 --- /dev/null +++ b/utils/third_party/nni/algorithms/compression/pytorch/pruning/amc/lib/utils.py @@ -0,0 +1,113 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +import torch + +class TextLogger(object): + """Write log immediately to the disk""" + def __init__(self, filepath): + self.f = open(filepath, 'w') + self.fid = self.f.fileno() + self.filepath = filepath + + def close(self): + self.f.close() + + def write(self, content): + self.f.write(content) + self.f.flush() + os.fsync(self.fid) + + def write_buf(self, content): + self.f.write(content) + + def print_and_write(self, content): + print(content) + self.write(content+'\n') + +def to_numpy(var): + use_cuda = torch.cuda.is_available() + return var.cpu().data.numpy() if use_cuda else var.data.numpy() + + +def to_tensor(ndarray, requires_grad=False): # return a float tensor by default + tensor = torch.from_numpy(ndarray).float() # by default does not require grad + if requires_grad: + tensor.requires_grad_() + return tensor.cuda() if torch.cuda.is_available() else tensor + + +def measure_layer_for_pruning(wrapper, x): + def get_layer_type(layer): + layer_str = str(layer) + return layer_str[:layer_str.find('(')].strip() + + def get_layer_param(model): + import operator + import functools + + return sum([functools.reduce(operator.mul, i.size(), 1) for i in model.parameters()]) + + multi_add = 1 + layer = wrapper.module + type_name = get_layer_type(layer) + + # ops_conv + if type_name in ['Conv2d']: + out_h = int((x.size()[2] + 2 * layer.padding[0] - layer.kernel_size[0]) / + layer.stride[0] + 1) + out_w = int((x.size()[3] + 2 * layer.padding[1] - layer.kernel_size[1]) / + layer.stride[1] + 1) + wrapper.flops = layer.in_channels * layer.out_channels * layer.kernel_size[0] * \ + layer.kernel_size[1] * out_h * out_w / layer.groups * multi_add + wrapper.params = get_layer_param(layer) + # ops_linear + elif type_name in ['Linear']: + weight_ops = layer.weight.numel() * multi_add + bias_ops = layer.bias.numel() + wrapper.flops = weight_ops + bias_ops + wrapper.params = get_layer_param(layer) + return + + +def least_square_sklearn(X, Y): + from sklearn.linear_model import LinearRegression + reg = LinearRegression(fit_intercept=False) + reg.fit(X, Y) + return reg.coef_ + + +def get_output_folder(parent_dir, env_name): + """Return save folder. + Assumes folders in the parent_dir have suffix -run{run + number}. Finds the highest run number and sets the output folder + to that number + 1. This is just convenient so that if you run the + same script multiple times tensorboard can plot all of the results + on the same plots with different names. + Parameters + ---------- + parent_dir: str + Path of the directory containing all experiment runs. + Returns + ------- + parent_dir/run_dir + Path to this run's save directory. + """ + os.makedirs(parent_dir, exist_ok=True) + experiment_id = 0 + for folder_name in os.listdir(parent_dir): + if not os.path.isdir(os.path.join(parent_dir, folder_name)): + continue + try: + folder_name = int(folder_name.split('-run')[-1]) + if folder_name > experiment_id: + experiment_id = folder_name + except: + pass + experiment_id += 1 + + parent_dir = os.path.join(parent_dir, env_name) + parent_dir = parent_dir + '-run{}'.format(experiment_id) + os.makedirs(parent_dir, exist_ok=True) + return parent_dir diff --git a/utils/third_party/nni/algorithms/compression/pytorch/pruning/apply_compression.py b/utils/third_party/nni/algorithms/compression/pytorch/pruning/apply_compression.py new file mode 100644 index 0000000000000000000000000000000000000000..8e6b023f5b90b8483e21bd0bc575b19b4a4df023 --- /dev/null +++ b/utils/third_party/nni/algorithms/compression/pytorch/pruning/apply_compression.py @@ -0,0 +1,29 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +import torch + +logger = logging.getLogger('torch apply compression') + +def apply_compression_results(model, masks_file, map_location=None): + """ + Apply the masks from ```masks_file``` to the model + Note: this API is for inference, because it simply multiplies weights with + corresponding masks when this API is called. + + Parameters + ---------- + model : torch.nn.Module + The model to be compressed + masks_file : str + The path of the mask file + map_location : str + the device on which masks are placed, same to map_location in ```torch.load``` + """ + masks = torch.load(masks_file, map_location) + for name, module in model.named_modules(): + if name in masks: + module.weight.data = module.weight.data.mul_(masks[name]['weight']) + if hasattr(module, 'bias') and module.bias is not None and 'bias' in masks[name]: + module.bias.data = module.bias.data.mul_(masks[name]['bias']) \ No newline at end of file diff --git a/utils/third_party/nni/algorithms/compression/pytorch/pruning/auto_compress_pruner.py b/utils/third_party/nni/algorithms/compression/pytorch/pruning/auto_compress_pruner.py new file mode 100644 index 0000000000000000000000000000000000000000..d52c6ec42da02cf84e0dd87c138f30376404682d --- /dev/null +++ b/utils/third_party/nni/algorithms/compression/pytorch/pruning/auto_compress_pruner.py @@ -0,0 +1,251 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +import os +import copy +import torch +from schema import And, Optional + +from nni.utils import OptimizeMode +from nni.compression.pytorch import ModelSpeedup + +from nni.compression.pytorch.compressor import Pruner +from nni.compression.pytorch.utils.config_validation import CompressorSchema +from .simulated_annealing_pruner import SimulatedAnnealingPruner +from .admm_pruner import ADMMPruner + + +_logger = logging.getLogger(__name__) + + +class AutoCompressPruner(Pruner): + """ + A Pytorch implementation of AutoCompress pruning algorithm. + + Parameters + ---------- + model : pytorch model + The model to be pruned. + config_list : list + Supported keys: + - sparsity : The target overall sparsity. + - op_types : The operation type to prune. + trainer : function + Function used for the first subproblem of ADMM Pruner. + Users should write this function as a normal function to train the Pytorch model + and include `model, optimizer, criterion, epoch, callback` as function arguments. + Here `callback` acts as an L2 regulizer as presented in the formula (7) of the original paper. + The logic of `callback` is implemented inside the Pruner, + users are just required to insert `callback()` between `loss.backward()` and `optimizer.step()`. + Example:: + + def trainer(model, criterion, optimizer, epoch, callback): + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + train_loader = ... + model.train() + for batch_idx, (data, target) in enumerate(train_loader): + data, target = data.to(device), target.to(device) + optimizer.zero_grad() + output = model(data) + loss = criterion(output, target) + loss.backward() + # callback should be inserted between loss.backward() and optimizer.step() + if callback: + callback() + optimizer.step() + evaluator : function + function to evaluate the pruned model. + This function should include `model` as the only parameter, and returns a scalar value. + Example:: + + def evaluator(model): + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + val_loader = ... + model.eval() + correct = 0 + with torch.no_grad(): + for data, target in val_loader: + data, target = data.to(device), target.to(device) + output = model(data) + # get the index of the max log-probability + pred = output.argmax(dim=1, keepdim=True) + correct += pred.eq(target.view_as(pred)).sum().item() + accuracy = correct / len(val_loader.dataset) + return accuracy + dummy_input : pytorch tensor + The dummy input for ```jit.trace```, users should put it on right device before pass in. + num_iterations : int + Number of overall iterations. + optimize_mode : str + optimize mode, `maximize` or `minimize`, by default `maximize`. + base_algo : str + Base pruning algorithm. `level`, `l1` or `l2`, by default `l1`. Given the sparsity distribution among the ops, + the assigned `base_algo` is used to decide which filters/channels/weights to prune. + start_temperature : float + Start temperature of the simulated annealing process. + stop_temperature : float + Stop temperature of the simulated annealing process. + cool_down_rate : float + Cool down rate of the temperature. + perturbation_magnitude : float + Initial perturbation magnitude to the sparsities. The magnitude decreases with current temperature. + admm_num_iterations : int + Number of iterations of ADMM Pruner. + admm_training_epochs : int + Training epochs of the first optimization subproblem of ADMMPruner. + row : float + Penalty parameters for ADMM training. + experiment_data_dir : string + PATH to store temporary experiment data. + """ + + def __init__(self, model, config_list, trainer, evaluator, dummy_input, + num_iterations=3, optimize_mode='maximize', base_algo='l1', + # SimulatedAnnealing related + start_temperature=100, stop_temperature=20, cool_down_rate=0.9, perturbation_magnitude=0.35, + # ADMM related + admm_num_iterations=30, admm_training_epochs=5, row=1e-4, + experiment_data_dir='./'): + # original model + self._model_to_prune = model + self._base_algo = base_algo + + self._trainer = trainer + self._evaluator = evaluator + self._dummy_input = dummy_input + self._num_iterations = num_iterations + self._optimize_mode = OptimizeMode(optimize_mode) + + # hyper parameters for SA algorithm + self._start_temperature = start_temperature + self._stop_temperature = stop_temperature + self._cool_down_rate = cool_down_rate + self._perturbation_magnitude = perturbation_magnitude + + # hyper parameters for ADMM algorithm + self._admm_num_iterations = admm_num_iterations + self._admm_training_epochs = admm_training_epochs + self._row = row + + # overall pruning rate + self._sparsity = config_list[0]['sparsity'] + + self._experiment_data_dir = experiment_data_dir + if not os.path.exists(self._experiment_data_dir): + os.makedirs(self._experiment_data_dir) + + def validate_config(self, model, config_list): + """ + Parameters + ---------- + model : torch.nn.Module + Model to be pruned + config_list : list + List on pruning configs + """ + + if self._base_algo == 'level': + schema = CompressorSchema([{ + 'sparsity': And(float, lambda n: 0 < n < 1), + Optional('op_types'): [str], + Optional('op_names'): [str], + }], model, _logger) + elif self._base_algo in ['l1', 'l2']: + schema = CompressorSchema([{ + 'sparsity': And(float, lambda n: 0 < n < 1), + 'op_types': ['Conv2d'], + Optional('op_names'): [str] + }], model, _logger) + + schema.validate(config_list) + + def calc_mask(self, wrapper, **kwargs): + return None + + def compress(self): + """ + Compress the model with AutoCompress. + + Returns + ------- + torch.nn.Module + model with specified modules compressed. + """ + _logger.info('Starting AutoCompress pruning...') + + sparsity_each_round = 1 - pow(1-self._sparsity, 1/self._num_iterations) + + for i in range(self._num_iterations): + _logger.info('Pruning iteration: %d', i) + _logger.info('Target sparsity this round: %s', + 1-pow(1-sparsity_each_round, i+1)) + + # SimulatedAnnealingPruner + _logger.info( + 'Generating sparsities with SimulatedAnnealingPruner...') + SApruner = SimulatedAnnealingPruner( + model=copy.deepcopy(self._model_to_prune), + config_list=[ + {"sparsity": sparsity_each_round, "op_types": ['Conv2d']}], + evaluator=self._evaluator, + optimize_mode=self._optimize_mode, + base_algo=self._base_algo, + start_temperature=self._start_temperature, + stop_temperature=self._stop_temperature, + cool_down_rate=self._cool_down_rate, + perturbation_magnitude=self._perturbation_magnitude, + experiment_data_dir=self._experiment_data_dir) + config_list = SApruner.compress(return_config_list=True) + _logger.info("Generated config_list : %s", config_list) + + # ADMMPruner + _logger.info('Performing structured pruning with ADMMPruner...') + ADMMpruner = ADMMPruner( + model=copy.deepcopy(self._model_to_prune), + config_list=config_list, + trainer=self._trainer, + num_iterations=self._admm_num_iterations, + training_epochs=self._admm_training_epochs, + row=self._row, + base_algo=self._base_algo) + ADMMpruner.compress() + + ADMMpruner.export_model(os.path.join(self._experiment_data_dir, 'model_admm_masked.pth'), os.path.join( + self._experiment_data_dir, 'mask.pth')) + + # use speed up to prune the model before next iteration, because SimulatedAnnealingPruner & ADMMPruner don't take masked models + self._model_to_prune.load_state_dict(torch.load(os.path.join( + self._experiment_data_dir, 'model_admm_masked.pth'))) + + masks_file = os.path.join(self._experiment_data_dir, 'mask.pth') + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + + _logger.info('Speeding up models...') + m_speedup = ModelSpeedup(self._model_to_prune, self._dummy_input, masks_file, device) + m_speedup.speedup_model() + + evaluation_result = self._evaluator(self._model_to_prune) + _logger.info('Evaluation result of the pruned model in iteration %d: %s', i, evaluation_result) + + _logger.info('----------Compression finished--------------') + + os.remove(os.path.join(self._experiment_data_dir, 'model_admm_masked.pth')) + os.remove(os.path.join(self._experiment_data_dir, 'mask.pth')) + + return self._model_to_prune + + def export_model(self, model_path, mask_path=None, onnx_path=None, input_shape=None, device=None): + _logger.info("AutoCompressPruner export directly the pruned model without mask") + + torch.save(self._model_to_prune.state_dict(), model_path) + _logger.info('Model state_dict saved to %s', model_path) + + if onnx_path is not None: + assert input_shape is not None, 'input_shape must be specified to export onnx model' + # input info needed + if device is None: + device = torch.device('cpu') + input_data = torch.Tensor(*input_shape) + torch.onnx.export(self._model_to_prune, input_data.to(device), onnx_path) + _logger.info('Model in onnx with input shape %s saved to %s', input_data.shape, onnx_path) diff --git a/utils/third_party/nni/algorithms/compression/pytorch/pruning/constants.py b/utils/third_party/nni/algorithms/compression/pytorch/pruning/constants.py new file mode 100644 index 0000000000000000000000000000000000000000..1670384c470ca30fcb2fc5f88bd3658d14010e93 --- /dev/null +++ b/utils/third_party/nni/algorithms/compression/pytorch/pruning/constants.py @@ -0,0 +1,25 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + + +from . import LevelPrunerMasker, SlimPrunerMasker, L1FilterPrunerMasker, \ + L2FilterPrunerMasker, FPGMPrunerMasker, TaylorFOWeightFilterPrunerMasker, \ + ActivationAPoZRankFilterPrunerMasker, ActivationMeanRankFilterPrunerMasker, \ + TRRPrunerMasker, HRankPrunerMasker, PFPMasker + +MASKER_DICT = { + 'level': LevelPrunerMasker, + 'slim': SlimPrunerMasker, + 'l1': L1FilterPrunerMasker, + 'l2': L2FilterPrunerMasker, + 'fpgm': FPGMPrunerMasker, + 'taylorfo': TaylorFOWeightFilterPrunerMasker, + 'apoz': ActivationAPoZRankFilterPrunerMasker, + 'mean_activation': ActivationMeanRankFilterPrunerMasker, + + # implemented by queyu, 2020/11/23 + 'trr': TRRPrunerMasker, + # implemented by queyu, 2021/6/10 + 'hrank': HRankPrunerMasker, + 'pfp': PFPMasker +} diff --git a/utils/third_party/nni/algorithms/compression/pytorch/pruning/constants_pruner.py b/utils/third_party/nni/algorithms/compression/pytorch/pruning/constants_pruner.py new file mode 100644 index 0000000000000000000000000000000000000000..9448d32d8c1080336339eccc6abe207f105dc8d2 --- /dev/null +++ b/utils/third_party/nni/algorithms/compression/pytorch/pruning/constants_pruner.py @@ -0,0 +1,12 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + + +from .one_shot import LevelPruner, L1FilterPruner, L2FilterPruner, TRRFilterPruner + +PRUNER_DICT = { + 'level': LevelPruner, + 'l1': L1FilterPruner, + 'l2': L2FilterPruner, + 'trr': TRRFilterPruner +} diff --git a/utils/third_party/nni/algorithms/compression/pytorch/pruning/finegrained_pruning.py b/utils/third_party/nni/algorithms/compression/pytorch/pruning/finegrained_pruning.py new file mode 100644 index 0000000000000000000000000000000000000000..f4aa174233e977eed7b59c6ba65136526a738603 --- /dev/null +++ b/utils/third_party/nni/algorithms/compression/pytorch/pruning/finegrained_pruning.py @@ -0,0 +1,31 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +import torch +from .weight_masker import WeightMasker + +__all__ = ['LevelPrunerMasker'] + +logger = logging.getLogger('torch pruner') + + +class LevelPrunerMasker(WeightMasker): + """ + Prune to an exact pruning level specification + """ + + def calc_mask(self, sparsity, wrapper, wrapper_idx=None): + weight = wrapper.module.weight.data.clone() + if wrapper.weight_mask is not None: + # apply base mask for iterative pruning + weight = weight * wrapper.weight_mask + + w_abs = weight.abs() + k = int(weight.numel() * sparsity) + if k == 0: + return {'weight_mask': torch.ones(weight.shape).type_as(weight)} + threshold = torch.topk(w_abs.view(-1), k, largest=False)[0].max() + mask_weight = torch.gt(w_abs, threshold).type_as(weight) + mask = {'weight_mask': mask_weight} + return mask diff --git a/utils/third_party/nni/algorithms/compression/pytorch/pruning/lottery_ticket.py b/utils/third_party/nni/algorithms/compression/pytorch/pruning/lottery_ticket.py new file mode 100644 index 0000000000000000000000000000000000000000..b45343457a63e132ea56b82eedb21865da7b5e23 --- /dev/null +++ b/utils/third_party/nni/algorithms/compression/pytorch/pruning/lottery_ticket.py @@ -0,0 +1,146 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import copy +import logging +import torch +from schema import And, Optional +from .....compression.pytorch.utils.config_validation import CompressorSchema +from .....compression.pytorch.compressor import Pruner +from .finegrained_pruning import LevelPrunerMasker + +logger = logging.getLogger('torch pruner') + +class LotteryTicketPruner(Pruner): + """ + Parameters + ---------- + model : pytorch model + The model to be pruned + config_list : list + Supported keys: + - prune_iterations : The number of rounds for the iterative pruning. + - sparsity : The final sparsity when the compression is done. + optimizer : pytorch optimizer + The optimizer for the model + lr_scheduler : pytorch lr scheduler + The lr scheduler for the model if used + reset_weights : bool + Whether reset weights and optimizer at the beginning of each round. + """ + def __init__(self, model, config_list, optimizer=None, lr_scheduler=None, reset_weights=True): + # save init weights and optimizer + self.reset_weights = reset_weights + if self.reset_weights: + self._model = model + self._optimizer = optimizer + self._model_state = copy.deepcopy(model.state_dict()) + self._optimizer_state = copy.deepcopy(optimizer.state_dict()) + self._lr_scheduler = lr_scheduler + if lr_scheduler is not None: + self._scheduler_state = copy.deepcopy(lr_scheduler.state_dict()) + + super().__init__(model, config_list, optimizer) + self.curr_prune_iteration = None + self.prune_iterations = config_list[0]['prune_iterations'] + self.masker = LevelPrunerMasker(model, self) + + def validate_config(self, model, config_list): + """ + Parameters + ---------- + model : torch.nn.Module + Model to be pruned + config_list : list + Supported keys: + - prune_iterations : The number of rounds for the iterative pruning. + - sparsity : The final sparsity when the compression is done. + """ + schema = CompressorSchema([{ + 'sparsity': And(float, lambda n: 0 < n < 1), + 'prune_iterations': And(int, lambda n: n > 0), + Optional('op_types'): [str], + Optional('op_names'): [str] + }], model, logger) + + schema.validate(config_list) + assert len(set([x['prune_iterations'] for x in config_list])) == 1, 'The values of prune_iterations must be equal in your config' + + def _calc_sparsity(self, sparsity): + keep_ratio_once = (1 - sparsity) ** (1 / self.prune_iterations) + curr_keep_ratio = keep_ratio_once ** self.curr_prune_iteration + return max(1 - curr_keep_ratio, 0) + + def _calc_mask(self, wrapper, sparsity): + weight = wrapper.module.weight.data + if self.curr_prune_iteration == 0: + mask = {'weight_mask': torch.ones(weight.shape).type_as(weight)} + else: + curr_sparsity = self._calc_sparsity(sparsity) + mask = self.masker.calc_mask(sparsity=curr_sparsity, wrapper=wrapper) + return mask + + def calc_mask(self, wrapper, **kwargs): + """ + Generate mask for the given ``weight``. + + Parameters + ---------- + wrapper : Module + The layer to be pruned + + Returns + ------- + tensor + The mask for this weight, it is ```None``` because this pruner + calculates and assigns masks in ```prune_iteration_start```, + no need to do anything in this function. + """ + return None + + def get_prune_iterations(self): + """ + Return the range for iterations. + In the first prune iteration, masks are all one, thus, add one more iteration + + Returns + ------- + list + A list for pruning iterations + """ + return range(self.prune_iterations + 1) + + def prune_iteration_start(self): + """ + Control the pruning procedure on updated epoch number. + Should be called at the beginning of the epoch. + """ + if self.curr_prune_iteration is None: + self.curr_prune_iteration = 0 + else: + self.curr_prune_iteration += 1 + assert self.curr_prune_iteration < self.prune_iterations + 1, 'Exceed the configured prune_iterations' + + modules_wrapper = self.get_modules_wrapper() + modules_to_compress = self.get_modules_to_compress() + for layer, config in modules_to_compress: + module_wrapper = None + for wrapper in modules_wrapper: + if wrapper.name == layer.name: + module_wrapper = wrapper + break + assert module_wrapper is not None + + sparsity = config.get('sparsity') + mask = self._calc_mask(module_wrapper, sparsity) + # TODO: directly use weight_mask is not good + module_wrapper.weight_mask = mask['weight_mask'] + # there is no mask for bias + + # reinit weights back to original after new masks are generated + if self.reset_weights: + # should use this member function to reset model weights + self.load_model_state_dict(self._model_state) + self._optimizer.load_state_dict(self._optimizer_state) + if self._lr_scheduler is not None: + self._lr_scheduler.load_state_dict(self._scheduler_state) diff --git a/utils/third_party/nni/algorithms/compression/pytorch/pruning/net_adapt_pruner.py b/utils/third_party/nni/algorithms/compression/pytorch/pruning/net_adapt_pruner.py new file mode 100644 index 0000000000000000000000000000000000000000..6f55234d5b91059c670e40ecf3cd991f2627c12d --- /dev/null +++ b/utils/third_party/nni/algorithms/compression/pytorch/pruning/net_adapt_pruner.py @@ -0,0 +1,351 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +import os +import copy +import json +import torch +from schema import And, Optional + +from nni.utils import OptimizeMode + +from nni.compression.pytorch.compressor import Pruner +from nni.compression.pytorch.utils.config_validation import CompressorSchema +from nni.compression.pytorch.utils.num_param_counter import get_total_num_weights +from .constants_pruner import PRUNER_DICT + + +_logger = logging.getLogger(__name__) + + +class NetAdaptPruner(Pruner): + """ + A Pytorch implementation of NetAdapt compression algorithm. + + Parameters + ---------- + model : pytorch model + The model to be pruned. + config_list : list + Supported keys: + - sparsity : The target overall sparsity. + - op_types : The operation type to prune. + short_term_fine_tuner : function + function to short-term fine tune the masked model. + This function should include `model` as the only parameter, + and fine tune the model for a short term after each pruning iteration. + Example:: + + def short_term_fine_tuner(model, epoch=3): + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + train_loader = ... + criterion = torch.nn.CrossEntropyLoss() + optimizer = torch.optim.SGD(model.parameters(), lr=0.01) + model.train() + for _ in range(epoch): + for batch_idx, (data, target) in enumerate(train_loader): + data, target = data.to(device), target.to(device) + optimizer.zero_grad() + output = model(data) + loss = criterion(output, target) + loss.backward() + optimizer.step() + evaluator : function + function to evaluate the masked model. + This function should include `model` as the only parameter, and returns a scalar value. + Example:: + + def evaluator(model): + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + val_loader = ... + model.eval() + correct = 0 + with torch.no_grad(): + for data, target in val_loader: + data, target = data.to(device), target.to(device) + output = model(data) + # get the index of the max log-probability + pred = output.argmax(dim=1, keepdim=True) + correct += pred.eq(target.view_as(pred)).sum().item() + accuracy = correct / len(val_loader.dataset) + return accuracy + optimize_mode : str + optimize mode, `maximize` or `minimize`, by default `maximize`. + base_algo : str + Base pruning algorithm. `level`, `l1` or `l2`, by default `l1`. Given the sparsity distribution among the ops, + the assigned `base_algo` is used to decide which filters/channels/weights to prune. + sparsity_per_iteration : float + sparsity to prune in each iteration. + experiment_data_dir : str + PATH to save experiment data, + including the config_list generated for the base pruning algorithm and the performance of the pruned model. + """ + + def __init__(self, model, config_list, short_term_fine_tuner, evaluator, + optimize_mode='maximize', base_algo='l1', sparsity_per_iteration=0.05, experiment_data_dir='./'): + # models used for iterative pruning and evaluation + self._model_to_prune = copy.deepcopy(model) + self._base_algo = base_algo + + super().__init__(model, config_list) + + self._short_term_fine_tuner = short_term_fine_tuner + self._evaluator = evaluator + self._optimize_mode = OptimizeMode(optimize_mode) + + # hyper parameters for NetAdapt algorithm + self._sparsity_per_iteration = sparsity_per_iteration + + # overall pruning rate + self._sparsity = config_list[0]['sparsity'] + + # config_list + self._config_list_generated = [] + + self._experiment_data_dir = experiment_data_dir + if not os.path.exists(self._experiment_data_dir): + os.makedirs(self._experiment_data_dir) + + self._tmp_model_path = os.path.join(self._experiment_data_dir, 'tmp_model.pth') + + def validate_config(self, model, config_list): + """ + Parameters + ---------- + model : torch.nn.Module + Model to be pruned + config_list : list + List on pruning configs + """ + + if self._base_algo == 'level': + schema = CompressorSchema([{ + 'sparsity': And(float, lambda n: 0 < n < 1), + Optional('op_types'): [str], + Optional('op_names'): [str], + }], model, _logger) + elif self._base_algo in ['l1', 'l2']: + schema = CompressorSchema([{ + 'sparsity': And(float, lambda n: 0 < n < 1), + 'op_types': ['Conv2d'], + Optional('op_names'): [str] + }], model, _logger) + + schema.validate(config_list) + + def calc_mask(self, wrapper, **kwargs): + return None + + def _update_config_list(self, config_list, op_name, sparsity): + ''' + update sparsity of op_name in config_list + ''' + config_list_updated = copy.deepcopy(config_list) + + for idx, item in enumerate(config_list): + if op_name in item['op_names']: + config_list_updated[idx]['sparsity'] = sparsity + return config_list_updated + + # if op_name is not in self._config_list_generated, create a new json item + if self._base_algo in ['l1', 'l2']: + config_list_updated.append( + {'sparsity': sparsity, 'op_types': ['Conv2d'], 'op_names': [op_name]}) + elif self._base_algo == 'level': + config_list_updated.append( + {'sparsity': sparsity, 'op_names': [op_name]}) + + return config_list_updated + + def _get_op_num_weights_remained(self, op_name, module): + ''' + Get the number of weights remained after channel pruning with current sparsity + + Returns + ------- + int + remained number of weights of the op + ''' + + # if op is wrapped by the pruner + for wrapper in self.get_modules_wrapper(): + if wrapper.name == op_name: + return wrapper.weight_mask.sum().item() + + # if op is not wrapped by the pruner + return module.weight.data.numel() + + def _get_op_sparsity(self, op_name): + for config in self._config_list_generated: + if 'op_names' in config and op_name in config['op_names']: + return config['sparsity'] + return 0 + + def _calc_num_related_weights(self, op_name): + ''' + Calculate total number weights of the op and the next op, applicable only for models without dependencies among ops + + Parameters + ---------- + op_name : str + + Returns + ------- + int + total number of all the realted (current and the next) op weights + ''' + num_weights = 0 + flag_found = False + previous_name = None + previous_module = None + + for name, module in self._model_to_prune.named_modules(): + if not flag_found and name != op_name and type(module).__name__ in ['Conv2d', 'Linear']: + previous_name = name + previous_module = module + if not flag_found and name == op_name: + _logger.debug("original module found: %s", name) + num_weights = module.weight.data.numel() + + # consider related pruning in this op caused by previous op's pruning + if previous_module: + sparsity_previous_op = self._get_op_sparsity(previous_name) + if sparsity_previous_op: + _logger.debug( + "decrease op's weights by %s due to previous op %s's pruning...", sparsity_previous_op, previous_name) + num_weights *= (1-sparsity_previous_op) + + flag_found = True + continue + if flag_found and type(module).__name__ in ['Conv2d', 'Linear']: + _logger.debug("related module found: %s", name) + # channel/filter pruning crossing is considered here, so only the num_weights after channel pruning is valuable + num_weights += self._get_op_num_weights_remained(name, module) + break + + _logger.debug("num related weights of op %s : %d", op_name, num_weights) + + return num_weights + + def compress(self): + """ + Compress the model. + + Returns + ------- + torch.nn.Module + model with specified modules compressed. + """ + _logger.info('Starting NetAdapt Compression...') + + pruning_iteration = 0 + current_sparsity = 0 + delta_num_weights_per_iteration = \ + int(get_total_num_weights(self._model_to_prune, ['Conv2d', 'Linear']) * self._sparsity_per_iteration) + + # stop condition + while current_sparsity < self._sparsity: + _logger.info('Pruning iteration: %d', pruning_iteration) + + # calculate target sparsity of this iteration + target_sparsity = current_sparsity + self._sparsity_per_iteration + + # variable to store the info of the best layer found in this iteration + best_op = {} + + for wrapper in self.get_modules_wrapper(): + _logger.debug("op name : %s", wrapper.name) + _logger.debug("op weights : %d", wrapper.weight_mask.numel()) + _logger.debug("op left weights : %d", wrapper.weight_mask.sum().item()) + + current_op_sparsity = 1 - wrapper.weight_mask.sum().item() / wrapper.weight_mask.numel() + _logger.debug("current op sparsity : %s", current_op_sparsity) + + # sparsity that this layer needs to prune to satisfy the requirement + target_op_sparsity = current_op_sparsity + delta_num_weights_per_iteration / self._calc_num_related_weights(wrapper.name) + + if target_op_sparsity >= 1: + _logger.info('Layer %s has no enough weights (remained) to prune', wrapper.name) + continue + + config_list = self._update_config_list(self._config_list_generated, wrapper.name, target_op_sparsity) + _logger.debug("config_list used : %s", config_list) + + pruner = PRUNER_DICT[self._base_algo](copy.deepcopy(self._model_to_prune), config_list) + model_masked = pruner.compress() + + # Short-term fine tune the pruned model + self._short_term_fine_tuner(model_masked) + + performance = self._evaluator(model_masked) + _logger.info("Layer : %s, evaluation result after short-term fine tuning : %s", wrapper.name, performance) + + if not best_op \ + or (self._optimize_mode is OptimizeMode.Maximize and performance > best_op['performance']) \ + or (self._optimize_mode is OptimizeMode.Minimize and performance < best_op['performance']): + _logger.debug("updating best layer to %s...", wrapper.name) + # find weight mask of this layer + for w in pruner.get_modules_wrapper(): + if w.name == wrapper.name: + masks = {'weight_mask': w.weight_mask, + 'bias_mask': w.bias_mask} + break + best_op = { + 'op_name': wrapper.name, + 'sparsity': target_op_sparsity, + 'performance': performance, + 'masks': masks + } + + # save model weights + pruner.export_model(self._tmp_model_path) + + if not best_op: + # decrease pruning step + self._sparsity_per_iteration *= 0.5 + _logger.info("No more layers to prune, decrease pruning step to %s", self._sparsity_per_iteration) + continue + + # Pick the best layer to prune, update iterative information + # update config_list + self._config_list_generated = self._update_config_list( + self._config_list_generated, best_op['op_name'], best_op['sparsity']) + + # update weights parameters + self._model_to_prune.load_state_dict(torch.load(self._tmp_model_path)) + + # update mask of the chosen op + for wrapper in self.get_modules_wrapper(): + if wrapper.name == best_op['op_name']: + for k in best_op['masks']: + setattr(wrapper, k, best_op['masks'][k]) + break + + current_sparsity = target_sparsity + _logger.info('Pruning iteration %d finished, current sparsity: %s', pruning_iteration, current_sparsity) + _logger.info('Layer %s seleted with sparsity %s, performance after pruning & short term fine-tuning : %s', + best_op['op_name'], best_op['sparsity'], best_op['performance']) + pruning_iteration += 1 + + self._final_performance = best_op['performance'] + + # load weights parameters + self.load_model_state_dict(torch.load(self._tmp_model_path)) + os.remove(self._tmp_model_path) + + _logger.info('----------Compression finished--------------') + _logger.info('config_list generated: %s', self._config_list_generated) + _logger.info("Performance after pruning: %s", self._final_performance) + _logger.info("Masked sparsity: %.6f", current_sparsity) + + # save best config found and best performance + with open(os.path.join(self._experiment_data_dir, 'search_result.json'), 'w') as jsonfile: + json.dump({ + 'performance': self._final_performance, + 'config_list': json.dumps(self._config_list_generated) + }, jsonfile) + + _logger.info('search history and result saved to foler : %s', self._experiment_data_dir) + + return self.bound_model diff --git a/utils/third_party/nni/algorithms/compression/pytorch/pruning/one_shot.py b/utils/third_party/nni/algorithms/compression/pytorch/pruning/one_shot.py new file mode 100644 index 0000000000000000000000000000000000000000..f2470d81ba2ca2245bc9ab11aa996d3030af130e --- /dev/null +++ b/utils/third_party/nni/algorithms/compression/pytorch/pruning/one_shot.py @@ -0,0 +1,489 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +from schema import And, Or, Optional, SchemaError +from .....common.graph_utils import TorchModuleGraph +from .....compression.pytorch.utils.shape_dependency import ChannelDependency, GroupDependency +from .constants import MASKER_DICT +from .....compression.pytorch.utils.config_validation import CompressorSchema +from .....compression.pytorch.compressor import Pruner + + +__all__ = ['LevelPruner', 'SlimPruner', 'L1FilterPruner', 'L2FilterPruner', 'FPGMPruner', + 'TaylorFOWeightFilterPruner', 'ActivationAPoZRankFilterPruner', 'ActivationMeanRankFilterPruner', + 'TRRFilterPruner', 'HRankFilterPruner', 'PFPPruner'] + +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) + + +class OneshotPruner(Pruner): + """ + Prune model to an exact pruning level for one time. + """ + + def __init__(self, model, config_list, pruning_algorithm='level', optimizer=None, **algo_kwargs): + """ + Parameters + ---------- + model : torch.nn.Module + Model to be pruned + config_list : list + List on pruning configs + pruning_algorithm: str + algorithms being used to prune model + optimizer: torch.optim.Optimizer + Optimizer used to train model + algo_kwargs: dict + Additional parameters passed to pruning algorithm masker class + """ + + super().__init__(model, config_list, optimizer) + self.set_wrappers_attribute("if_calculated", False) + self.masker = MASKER_DICT[pruning_algorithm]( + model, self, **algo_kwargs) + + def validate_config(self, model, config_list): + """ + Parameters + ---------- + model : torch.nn.Module + Model to be pruned + config_list : list + List on pruning configs + """ + schema = CompressorSchema([{ + 'sparsity': And(float, lambda n: 0 < n < 1), + Optional('op_types'): [str], + Optional('op_names'): [str] + }], model, logger) + + schema.validate(config_list) + + def calc_mask(self, wrapper, wrapper_idx=None): + """ + Calculate the mask of given layer + Parameters + ---------- + wrapper : Module + the module to instrument the compression operation + wrapper_idx: int + index of this wrapper in pruner's all wrappers + Returns + ------- + dict + dictionary for storing masks, keys of the dict: + 'weight_mask': weight mask tensor + 'bias_mask': bias mask tensor (optional) + """ + if wrapper.if_calculated: + return None + + sparsity = wrapper.config['sparsity'] + if not wrapper.if_calculated: + masks = self.masker.calc_mask( + sparsity=sparsity, wrapper=wrapper, wrapper_idx=wrapper_idx) + + # masker.calc_mask returns None means calc_mask is not calculated sucessfully, can try later + if masks is not None: + wrapper.if_calculated = True + return masks + else: + return None + + +class LevelPruner(OneshotPruner): + """ + Parameters + ---------- + model : torch.nn.Module + Model to be pruned + config_list : list + Supported keys: + - sparsity : This is to specify the sparsity operations to be compressed to. + - op_types : Operation types to prune. + optimizer: torch.optim.Optimizer + Optimizer used to train model + """ + + def __init__(self, model, config_list, optimizer=None): + super().__init__(model, config_list, pruning_algorithm='level', optimizer=optimizer) + + +class SlimPruner(OneshotPruner): + """ + Parameters + ---------- + model : torch.nn.Module + Model to be pruned + config_list : list + Supported keys: + - sparsity : This is to specify the sparsity operations to be compressed to. + - op_types : Only BatchNorm2d is supported in Slim Pruner. + optimizer: torch.optim.Optimizer + Optimizer used to train model + """ + + def __init__(self, model, config_list, optimizer=None): + super().__init__(model, config_list, pruning_algorithm='slim', optimizer=optimizer) + + def validate_config(self, model, config_list): + schema = CompressorSchema([{ + 'sparsity': And(float, lambda n: 0 < n < 1), + 'op_types': ['BatchNorm2d'], + Optional('op_names'): [str] + }], model, logger) + + schema.validate(config_list) + + if len(config_list) > 1: + logger.warning('Slim pruner only supports 1 configuration') + + +class _StructuredFilterPruner(OneshotPruner): + """ + _StructuredFilterPruner has two ways to calculate the masks + for conv layers. In the normal way, the _StructuredFilterPruner + will calculate the mask of each layer separately. For example, each + conv layer determine which filters should be pruned according to its L1 + norm. In constrast, in the dependency-aware way, the layers that in a + dependency group will be pruned jointly and these layers will be forced + to prune the same channels. + """ + + def __init__(self, model, config_list, pruning_algorithm, optimizer=None, dependency_aware=False, dummy_input=None, **algo_kwargs): + super().__init__(model, config_list, pruning_algorithm=pruning_algorithm, + optimizer=optimizer, **algo_kwargs) + self.dependency_aware = dependency_aware + # set the dependency-aware switch for the masker + self.masker.dependency_aware = dependency_aware + self.dummy_input = dummy_input + if self.dependency_aware: + errmsg = "When dependency_aware is set, the dummy_input should not be None" + assert self.dummy_input is not None, errmsg + # Get the TorchModuleGraph of the target model + # to trace the model, we need to unwrap the wrappers + self._unwrap_model() + self.graph = TorchModuleGraph(model, dummy_input) + self._wrap_model() + self.channel_depen = ChannelDependency( + traced_model=self.graph.trace) + self.group_depen = GroupDependency(traced_model=self.graph.trace) + self.channel_depen = self.channel_depen.dependency_sets + self.channel_depen = { + name: sets for sets in self.channel_depen for name in sets} + self.group_depen = self.group_depen.dependency_sets + + def update_mask(self): + if not self.dependency_aware: + # if we use the normal way to update the mask, + # then call the update_mask of the father class + super(_StructuredFilterPruner, self).update_mask() + else: + # if we update the mask in a dependency-aware way + # then we call _dependency_update_mask + self._dependency_update_mask() + + def validate_config(self, model, config_list): + schema = CompressorSchema([{ + Optional('sparsity'): And(float, lambda n: 0 < n < 1), + Optional('op_types'): Or(['Conv2d'], ['Conv2d', 'ConvTranspose2d']), + Optional('op_names'): [str], + Optional('exclude'): bool + }], model, logger) + + schema.validate(config_list) + for config in config_list: + if 'exclude' not in config and 'sparsity' not in config: + raise SchemaError('Either sparisty or exclude must be specified!') + + def _dependency_calc_mask(self, wrappers, channel_dsets, wrappers_idx=None): + """ + calculate the masks for the conv layers in the same + channel dependecy set. All the layers passed in have + the same number of channels. + + Parameters + ---------- + wrappers: list + The list of the wrappers that in the same channel dependency + set. + wrappers_idx: list + The list of the indexes of wrapppers. + Returns + ------- + masks: dict + A dict object that contains the masks of the layers in this + dependency group, the key is the name of the convolutional layers. + """ + # The number of the groups for each conv layers + # Note that, this number may be different from its + # original number of groups of filters. + groups = [self.group_depen[_w.name] for _w in wrappers] + sparsities = [_w.config['sparsity'] for _w in wrappers] + masks = self.masker.calc_mask( + sparsities, wrappers, wrappers_idx, channel_dsets=channel_dsets, groups=groups) + if masks is not None: + # if masks is None, then the mask calculation fails. + # for example, in activation related maskers, we should + # pass enough batches of data to the model, so that the + # masks can be calculated successfully. + for _w in wrappers: + _w.if_calculated = True + return masks + + def _dependency_update_mask(self): + """ + In the original update_mask, the wraper of each layer will update its + own mask according to the sparsity specified in the config_list. However, in + the _dependency_update_mask, we may prune several layers at the same + time according the sparsities and the channel/group dependencies. + """ + name2wrapper = {x.name: x for x in self.get_modules_wrapper()} + wrapper2index = {x: i for i, x in enumerate(self.get_modules_wrapper())} + for wrapper in self.get_modules_wrapper(): + if wrapper.if_calculated: + continue + # find all the conv layers that have channel dependecy with this layer + # and prune all these layers at the same time. + _names = [x for x in self.channel_depen[wrapper.name]] + logger.info('Pruning the dependent layers: %s', ','.join(_names)) + _wrappers = [name2wrapper[name] + for name in _names if name in name2wrapper] + _wrapper_idxes = [wrapper2index[_w] for _w in _wrappers] + + masks = self._dependency_calc_mask( + _wrappers, _names, wrappers_idx=_wrapper_idxes) + if masks is not None: + for layer in masks: + for mask_type in masks[layer]: + assert hasattr( + name2wrapper[layer], mask_type), "there is no attribute '%s' in wrapper on %s" % (mask_type, layer) + setattr(name2wrapper[layer], mask_type, masks[layer][mask_type]) + + +class L1FilterPruner(_StructuredFilterPruner): + """ + Parameters + ---------- + model : torch.nn.Module + Model to be pruned + config_list : list + Supported keys: + - sparsity : This is to specify the sparsity operations to be compressed to. + - op_types : Only Conv2d is supported in L1FilterPruner. + optimizer: torch.optim.Optimizer + Optimizer used to train model + dependency_aware: bool + If prune the model in a dependency-aware way. If it is `True`, this pruner will + prune the model according to the l2-norm of weights and the channel-dependency or + group-dependency of the model. In this way, the pruner will force the conv layers + that have dependencies to prune the same channels, so the speedup module can better + harvest the speed benefit from the pruned model. Note that, if this flag is set True + , the dummy_input cannot be None, because the pruner needs a dummy input to trace the + dependency between the conv layers. + dummy_input : torch.Tensor + The dummy input to analyze the topology constraints. Note that, the dummy_input + should on the same device with the model. + """ + + def __init__(self, model, config_list, optimizer=None, dependency_aware=False, dummy_input=None): + super().__init__(model, config_list, pruning_algorithm='l1', optimizer=optimizer, + dependency_aware=dependency_aware, dummy_input=dummy_input) + + +class TRRFilterPruner(_StructuredFilterPruner): + def __init__(self, model, config_list, optimizer=None, dependency_aware=False, train_loader=None, device='cuda'): + super().__init__(model, config_list, pruning_algorithm='trr', optimizer=optimizer, + dependency_aware=dependency_aware, train_loader=train_loader, device=device) + + +class L2FilterPruner(_StructuredFilterPruner): + """ + Parameters + ---------- + model : torch.nn.Module + Model to be pruned + config_list : list + Supported keys: + - sparsity : This is to specify the sparsity operations to be compressed to. + - op_types : Only Conv2d is supported in L2FilterPruner. + optimizer: torch.optim.Optimizer + Optimizer used to train model + dependency_aware: bool + If prune the model in a dependency-aware way. If it is `True`, this pruner will + prune the model according to the l2-norm of weights and the channel-dependency or + group-dependency of the model. In this way, the pruner will force the conv layers + that have dependencies to prune the same channels, so the speedup module can better + harvest the speed benefit from the pruned model. Note that, if this flag is set True + , the dummy_input cannot be None, because the pruner needs a dummy input to trace the + dependency between the conv layers. + dummy_input : torch.Tensor + The dummy input to analyze the topology constraints. Note that, the dummy_input + should on the same device with the model. + """ + + def __init__(self, model, config_list, optimizer=None, dependency_aware=False, dummy_input=None): + super().__init__(model, config_list, pruning_algorithm='l2', optimizer=optimizer, + dependency_aware=dependency_aware, dummy_input=dummy_input) + + +class FPGMPruner(_StructuredFilterPruner): + """ + Parameters + ---------- + model : torch.nn.Module + Model to be pruned + config_list : list + Supported keys: + - sparsity : This is to specify the sparsity operations to be compressed to. + - op_types : Only Conv2d is supported in FPGM Pruner. + optimizer: torch.optim.Optimizer + Optimizer used to train model + dependency_aware: bool + If prune the model in a dependency-aware way. If it is `True`, this pruner will + prune the model according to the l2-norm of weights and the channel-dependency or + group-dependency of the model. In this way, the pruner will force the conv layers + that have dependencies to prune the same channels, so the speedup module can better + harvest the speed benefit from the pruned model. Note that, if this flag is set True + , the dummy_input cannot be None, because the pruner needs a dummy input to trace the + dependency between the conv layers. + dummy_input : torch.Tensor + The dummy input to analyze the topology constraints. Note that, the dummy_input + should on the same device with the model. + """ + + def __init__(self, model, config_list, optimizer=None, dependency_aware=False, dummy_input=None): + super().__init__(model, config_list, pruning_algorithm='fpgm', + dependency_aware=dependency_aware, dummy_input=dummy_input, optimizer=optimizer) + + +class TaylorFOWeightFilterPruner(_StructuredFilterPruner): + """ + Parameters + ---------- + model : torch.nn.Module + Model to be pruned + config_list : list + Supported keys: + - sparsity : How much percentage of convolutional filters are to be pruned. + - op_types : Currently only Conv2d is supported in TaylorFOWeightFilterPruner. + optimizer: torch.optim.Optimizer + Optimizer used to train model + statistics_batch_num: int + The number of batches to statistic the activation. + dependency_aware: bool + If prune the model in a dependency-aware way. If it is `True`, this pruner will + prune the model according to the l2-norm of weights and the channel-dependency or + group-dependency of the model. In this way, the pruner will force the conv layers + that have dependencies to prune the same channels, so the speedup module can better + harvest the speed benefit from the pruned model. Note that, if this flag is set True + , the dummy_input cannot be None, because the pruner needs a dummy input to trace the + dependency between the conv layers. + dummy_input : torch.Tensor + The dummy input to analyze the topology constraints. Note that, the dummy_input + should on the same device with the model. + + """ + + def __init__(self, model, config_list, optimizer=None, statistics_batch_num=1, + dependency_aware=False, dummy_input=None): + super().__init__(model, config_list, pruning_algorithm='taylorfo', + dependency_aware=dependency_aware, dummy_input=dummy_input, + optimizer=optimizer, statistics_batch_num=statistics_batch_num) + + +class ActivationAPoZRankFilterPruner(_StructuredFilterPruner): + """ + Parameters + ---------- + model : torch.nn.Module + Model to be pruned + config_list : list + Supported keys: + - sparsity : How much percentage of convolutional filters are to be pruned. + - op_types : Only Conv2d is supported in ActivationAPoZRankFilterPruner. + optimizer: torch.optim.Optimizer + Optimizer used to train model + activation: str + The activation type. + statistics_batch_num: int + The number of batches to statistic the activation. + dependency_aware: bool + If prune the model in a dependency-aware way. If it is `True`, this pruner will + prune the model according to the l2-norm of weights and the channel-dependency or + group-dependency of the model. In this way, the pruner will force the conv layers + that have dependencies to prune the same channels, so the speedup module can better + harvest the speed benefit from the pruned model. Note that, if this flag is set True + , the dummy_input cannot be None, because the pruner needs a dummy input to trace the + dependency between the conv layers. + dummy_input : torch.Tensor + The dummy input to analyze the topology constraints. Note that, the dummy_input + should on the same device with the model. + + """ + + def __init__(self, model, config_list, optimizer=None, activation='relu', + statistics_batch_num=1, dependency_aware=False, dummy_input=None): + super().__init__(model, config_list, pruning_algorithm='apoz', optimizer=optimizer, + dependency_aware=dependency_aware, dummy_input=dummy_input, + activation=activation, statistics_batch_num=statistics_batch_num) + + +class ActivationMeanRankFilterPruner(_StructuredFilterPruner): + """ + Parameters + ---------- + model : torch.nn.Module + Model to be pruned + config_list : list + Supported keys: + - sparsity : How much percentage of convolutional filters are to be pruned. + - op_types : Only Conv2d is supported in ActivationMeanRankFilterPruner. + optimizer: torch.optim.Optimizer + Optimizer used to train model. + activation: str + The activation type. + statistics_batch_num: int + The number of batches to statistic the activation. + dependency_aware: bool + If prune the model in a dependency-aware way. If it is `True`, this pruner will + prune the model according to the l2-norm of weights and the channel-dependency or + group-dependency of the model. In this way, the pruner will force the conv layers + that have dependencies to prune the same channels, so the speedup module can better + harvest the speed benefit from the pruned model. Note that, if this flag is set True + , the dummy_input cannot be None, because the pruner needs a dummy input to trace the + dependency between the conv layers. + dummy_input : torch.Tensor + The dummy input to analyze the topology constraints. Note that, the dummy_input + should on the same device with the model. + """ + + def __init__(self, model, config_list, optimizer=None, activation='relu', + statistics_batch_num=1, dependency_aware=False, dummy_input=None): + super().__init__(model, config_list, pruning_algorithm='mean_activation', optimizer=optimizer, + dependency_aware=dependency_aware, dummy_input=dummy_input, + activation=activation, statistics_batch_num=statistics_batch_num) + + +# implemented by queyu, 2021/6/10 +class HRankFilterPruner(_StructuredFilterPruner): + def __init__(self, model, config_list, optimizer=None, activation='relu', + statistics_batch_num=1, dependency_aware=False, dummy_input=None): + super().__init__(model, config_list, pruning_algorithm='hrank', optimizer=optimizer, + dependency_aware=dependency_aware, dummy_input=dummy_input, + activation=activation, statistics_batch_num=statistics_batch_num) + + +# implemented by queyu, 2021/6/10 +class PFPPruner(_StructuredFilterPruner): + def __init__(self, model, config_list, optimizer=None, activation='relu', + statistics_batch_num=1, dependency_aware=False, dummy_input=None): + super().__init__(model, config_list, pruning_algorithm='pfp', optimizer=optimizer, + dependency_aware=dependency_aware, dummy_input=dummy_input, + activation=activation, statistics_batch_num=statistics_batch_num) + + self._unwrap_model() + self.graph = TorchModuleGraph(model, dummy_input) + self._wrap_model() diff --git a/utils/third_party/nni/algorithms/compression/pytorch/pruning/sensitivity_pruner.py b/utils/third_party/nni/algorithms/compression/pytorch/pruning/sensitivity_pruner.py new file mode 100644 index 0000000000000000000000000000000000000000..037e7efc5e9edcd49757ce053bd3b0c1333ae19f --- /dev/null +++ b/utils/third_party/nni/algorithms/compression/pytorch/pruning/sensitivity_pruner.py @@ -0,0 +1,413 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. +import os +import csv +import copy +import json +import logging +import torch + +from schema import And, Optional +from nni.compression.pytorch.compressor import Pruner +from nni.compression.pytorch.utils.config_validation import CompressorSchema +from .constants_pruner import PRUNER_DICT +from nni.compression.pytorch.utils.sensitivity_analysis import SensitivityAnalysis + + +MAX_PRUNE_RATIO_PER_ITER = 0.95 + +_logger = logging.getLogger('Sensitivity_Pruner') +_logger.setLevel(logging.INFO) + +class SensitivityPruner(Pruner): + """ + This function prune the model based on the sensitivity + for each layer. + + Parameters + ---------- + model: torch.nn.Module + model to be compressed + evaluator: function + validation function for the model. This function should return the accuracy + of the validation dataset. The input parameters of evaluator can be specified + in the parameter `eval_args` and 'eval_kwargs' of the compress function if needed. + Example: + >>> def evaluator(model): + >>> device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + >>> val_loader = ... + >>> model.eval() + >>> correct = 0 + >>> with torch.no_grad(): + >>> for data, target in val_loader: + >>> data, target = data.to(device), target.to(device) + >>> output = model(data) + >>> # get the index of the max log-probability + >>> pred = output.argmax(dim=1, keepdim=True) + >>> correct += pred.eq(target.view_as(pred)).sum().item() + >>> accuracy = correct / len(val_loader.dataset) + >>> return accuracy + finetuner: function + finetune function for the model. This parameter is not essential, if is not None, + the sensitivity pruner will finetune the model after pruning in each iteration. + The input parameters of finetuner can be specified in the parameter of compress + called `finetune_args` and `finetune_kwargs` if needed. + Example: + >>> def finetuner(model, epoch=3): + >>> device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + >>> train_loader = ... + >>> criterion = torch.nn.CrossEntropyLoss() + >>> optimizer = torch.optim.SGD(model.parameters(), lr=0.01) + >>> model.train() + >>> for _ in range(epoch): + >>> for _, (data, target) in enumerate(train_loader): + >>> data, target = data.to(device), target.to(device) + >>> optimizer.zero_grad() + >>> output = model(data) + >>> loss = criterion(output, target) + >>> loss.backward() + >>> optimizer.step() + base_algo: str + base pruning algorithm. `level`, `l1` or `l2`, by default `l1`. + sparsity_proportion_calc: function + This function generate the sparsity proportion between the conv layers according to the + sensitivity analysis results. We provide a default function to quantify the sparsity + proportion according to the sensitivity analysis results. Users can also customize + this function according to their needs. The input of this function is a dict, + for example : {'conv1' : {0.1: 0.9, 0.2 : 0.8}, 'conv2' : {0.1: 0.9, 0.2 : 0.8}}, + in which, 'conv1' and is the name of the conv layer, and 0.1:0.9 means when the + sparsity of conv1 is 0.1 (10%), the model's val accuracy equals to 0.9. + sparsity_per_iter: float + The sparsity of the model that the pruner try to prune in each iteration. + acc_drop_threshold : float + The hyperparameter used to quantifiy the sensitivity for each layer. + checkpoint_dir: str + The dir path to save the checkpoints during the pruning. + """ + + def __init__(self, model, config_list, evaluator, + finetuner=None, base_algo='l1', sparsity_proportion_calc=None, + sparsity_per_iter=0.1, acc_drop_threshold=0.05, checkpoint_dir=None): + + self.base_algo = base_algo + self.model = model + super(SensitivityPruner, self).__init__(model, config_list) + # unwrap the model + self._unwrap_model() + _logger.debug(str(self.model)) + self.evaluator = evaluator + self.finetuner = finetuner + self.analyzer = SensitivityAnalysis( + self.model, self.evaluator, prune_type=base_algo, \ + early_stop_mode='dropped', early_stop_value=acc_drop_threshold) + # Get the original accuracy of the pretrained model + self.ori_acc = None + # Copy the original weights before pruning + self.ori_state_dict = copy.deepcopy(self.model.state_dict()) + self.sensitivities = {} + # Save the weight count for each layer + self.weight_count = {} + self.weight_sum = 0 + # Map the layer name to the layer module + self.named_module = {} + + self.Pruner = PRUNER_DICT[self.base_algo] + # Count the total weight count of the model + for name, submodule in self.model.named_modules(): + self.named_module[name] = submodule + if name in self.analyzer.target_layer: + # Currently, only count the weights in the conv layers + # else the fully connected layer (which contains + # the most weights) may make the pruner prune the + # model too hard + # if hasattr(submodule, 'weight'): # Count all the weights of the model + self.weight_count[name] = submodule.weight.data.numel() + self.weight_sum += self.weight_count[name] + # function to generate the sparsity proportion between the conv layers + if sparsity_proportion_calc is None: + self.sparsity_proportion_calc = self._max_prune_ratio + else: + self.sparsity_proportion_calc = sparsity_proportion_calc + # The ratio of remained weights is 1.0 at the begining + self.remained_ratio = 1.0 + self.sparsity_per_iter = sparsity_per_iter + self.acc_drop_threshold = acc_drop_threshold + self.checkpoint_dir = checkpoint_dir + + def validate_config(self, model, config_list): + """ + Parameters + ---------- + model : torch.nn.module + Model to be pruned + config_list : list + List on pruning configs + """ + + if self.base_algo == 'level': + schema = CompressorSchema([{ + 'sparsity': And(float, lambda n: 0 < n < 1), + Optional('op_types'): [str], + Optional('op_names'): [str], + }], model, _logger) + elif self.base_algo in ['l1', 'l2']: + schema = CompressorSchema([{ + 'sparsity': And(float, lambda n: 0 < n < 1), + 'op_types': ['Conv2d'], + Optional('op_names'): [str] + }], model, _logger) + + schema.validate(config_list) + + def load_sensitivity(self, filepath): + """ + load the sensitivity results exported by the sensitivity analyzer + """ + assert os.path.exists(filepath) + with open(filepath, 'r') as csvf: + csv_r = csv.reader(csvf) + header = next(csv_r) + sparsities = [float(x) for x in header[1:]] + sensitivities = {} + for row in csv_r: + layername = row[0] + accuracies = [float(x) for x in row[1:]] + sensitivities[layername] = {} + for i, accuracy in enumerate(accuracies): + sensitivities[layername][sparsities[i]] = accuracy + return sensitivities + + def _max_prune_ratio(self, ori_acc, threshold, sensitivities): + """ + Find the maximum prune ratio for a single layer whose accuracy + drop is lower than the threshold. + + Parameters + ---------- + ori_acc: float + Original accuracy + threshold: float + Accuracy drop threshold + sensitivities: dict + The dict object that stores the sensitivity results for each layer. + For example: {'conv1' : {0.1: 0.9, 0.2 : 0.8}} + Returns + ------- + max_ratios: dict + return the maximum prune ratio for each layer. For example: + {'conv1':0.1, 'conv2':0.2} + """ + max_ratio = {} + for layer in sensitivities: + prune_ratios = sorted(sensitivities[layer].keys()) + last_ratio = 0 + for ratio in prune_ratios: + last_ratio = ratio + cur_acc = sensitivities[layer][ratio] + if cur_acc + threshold < ori_acc: + break + max_ratio[layer] = last_ratio + return max_ratio + + def normalize(self, ratios, target_pruned): + """ + Normalize the prune ratio of each layer according to the + total already pruned ratio and the final target total pruning + ratio + + Parameters + ---------- + ratios: + Dict object that save the prune ratio for each layer + target_pruned: + The amount of the weights expected to be pruned in this + iteration + + Returns + ------- + new_ratios: + return the normalized prune ratios for each layer. + + """ + w_sum = 0 + _Max = 0 + for layername, ratio in ratios.items(): + wcount = self.weight_count[layername] + w_sum += ratio * wcount * \ + (1-self.analyzer.already_pruned[layername]) + target_count = self.weight_sum * target_pruned + for layername in ratios: + ratios[layername] = ratios[layername] * target_count / w_sum + _Max = max(_Max, ratios[layername]) + # Cannot Prune too much in a single iteration + # If a layer's prune ratio is larger than the + # MAX_PRUNE_RATIO_PER_ITER we rescal all prune + # ratios under this threshold + if _Max > MAX_PRUNE_RATIO_PER_ITER: + + for layername in ratios: + ratios[layername] = ratios[layername] * \ + MAX_PRUNE_RATIO_PER_ITER / _Max + return ratios + + def create_cfg(self, ratios): + """ + Generate the cfg_list for the pruner according to the prune ratios. + + Parameters + --------- + ratios: + For example: {'conv1' : 0.2} + + Returns + ------- + cfg_list: + For example: [{'sparsity':0.2, 'op_names':['conv1'], 'op_types':['Conv2d']}] + """ + cfg_list = [] + for layername in ratios: + prune_ratio = ratios[layername] + remain = 1 - self.analyzer.already_pruned[layername] + sparsity = remain * prune_ratio + \ + self.analyzer.already_pruned[layername] + if sparsity > 0: + # Pruner does not allow the prune ratio to be zero + cfg = {'sparsity': sparsity, 'op_names': [ + layername], 'op_types': ['Conv2d']} + cfg_list.append(cfg) + return cfg_list + + def current_sparsity(self): + """ + The sparsity of the weight. + """ + pruned_weight = 0 + for layer_name in self.analyzer.already_pruned: + w_count = self.weight_count[layer_name] + prune_ratio = self.analyzer.already_pruned[layer_name] + pruned_weight += w_count * prune_ratio + return pruned_weight / self.weight_sum + + def compress(self, eval_args=None, eval_kwargs=None, + finetune_args=None, finetune_kwargs=None, resume_sensitivity=None): + """ + This function iteratively prune the model according to the results of + the sensitivity analysis. + + Parameters + ---------- + eval_args: list + eval_kwargs: list& dict + Parameters for the val_funtion, the val_function will be called like + evaluator(*eval_args, **eval_kwargs) + finetune_args: list + finetune_kwargs: dict + Parameters for the finetuner function if needed. + resume_sensitivity: + resume the sensitivity results from this file. + """ + # pylint suggest not use the empty list and dict + # as the default input parameter + if not eval_args: + eval_args = [] + if not eval_kwargs: + eval_kwargs = {} + if not finetune_args: + finetune_args = [] + if not finetune_kwargs: + finetune_kwargs = {} + if self.ori_acc is None: + self.ori_acc = self.evaluator(*eval_args, **eval_kwargs) + assert isinstance(self.ori_acc, float) or isinstance(self.ori_acc, int) + if not resume_sensitivity: + self.sensitivities = self.analyzer.analysis( + val_args=eval_args, val_kwargs=eval_kwargs) + else: + self.sensitivities = self.load_sensitivity(resume_sensitivity) + self.analyzer.sensitivities = self.sensitivities + # the final target sparsity of the model + target_ratio = 1 - self.config_list[0]['sparsity'] + cur_ratio = self.remained_ratio + ori_acc = self.ori_acc + iteration_count = 0 + if self.checkpoint_dir is not None: + os.makedirs(self.checkpoint_dir, exist_ok=True) + modules_wrapper_final = None + while cur_ratio > target_ratio: + iteration_count += 1 + # Each round have three steps: + # 1) Get the current sensitivity for each layer(the sensitivity + # of each layer may change during the pruning) + # 2) Prune each layer according the sensitivies + # 3) finetune the model + _logger.info('Current base accuracy %f', ori_acc) + _logger.info('Remained %f weights', cur_ratio) + # determine the sparsity proportion between different + # layers according to the sensitivity result + proportion = self.sparsity_proportion_calc( + ori_acc, self.acc_drop_threshold, self.sensitivities) + + new_pruneratio = self.normalize(proportion, self.sparsity_per_iter) + cfg_list = self.create_cfg(new_pruneratio) + if not cfg_list: + _logger.error('The threshold is too small, please set a larger threshold') + return self.model + _logger.debug('Pruner Config: %s', str(cfg_list)) + cfg_str = ['%s:%.3f'%(cfg['op_names'][0], cfg['sparsity']) for cfg in cfg_list] + _logger.info('Current Sparsities: %s', ','.join(cfg_str)) + + pruner = self.Pruner(self.model, cfg_list) + pruner.compress() + pruned_acc = self.evaluator(*eval_args, **eval_kwargs) + _logger.info('Accuracy after pruning: %f', pruned_acc) + finetune_acc = pruned_acc + if self.finetuner is not None: + # if the finetune function is None, then skip the finetune + self.finetuner(*finetune_args, **finetune_kwargs) + finetune_acc = self.evaluator(*eval_args, **eval_kwargs) + _logger.info('Accuracy after finetune: %f', finetune_acc) + ori_acc = finetune_acc + # unwrap the pruner + pruner._unwrap_model() + # update the already prune ratio of each layer befor the new + # sensitivity analysis + for layer_cfg in cfg_list: + name = layer_cfg['op_names'][0] + sparsity = layer_cfg['sparsity'] + self.analyzer.already_pruned[name] = sparsity + # update the cur_ratio + cur_ratio = 1 - self.current_sparsity() + modules_wrapper_final = pruner.get_modules_wrapper() + del pruner + _logger.info('Currently remained weights: %f', cur_ratio) + + if self.checkpoint_dir is not None: + checkpoint_name = 'Iter_%d_finetune_acc_%.5f_sparsity_%.4f' % ( + iteration_count, finetune_acc, cur_ratio) + checkpoint_path = os.path.join( + self.checkpoint_dir, '%s.pth' % checkpoint_name) + cfg_path = os.path.join( + self.checkpoint_dir, '%s_pruner.json' % checkpoint_name) + sensitivity_path = os.path.join( + self.checkpoint_dir, '%s_sensitivity.csv' % checkpoint_name) + torch.save(self.model.state_dict(), checkpoint_path) + with open(cfg_path, 'w') as jf: + json.dump(cfg_list, jf) + self.analyzer.export(sensitivity_path) + + if cur_ratio > target_ratio: + # If this is the last prune iteration, skip the time-consuming + # sensitivity analysis + + self.analyzer.load_state_dict(self.model.state_dict()) + self.sensitivities = self.analyzer.analysis( + val_args=eval_args, val_kwargs=eval_kwargs) + + _logger.info('After Pruning: %.2f weights remains', cur_ratio) + self.modules_wrapper = modules_wrapper_final + + self._wrap_model() + return self.model + + def calc_mask(self, wrapper, **kwargs): + return None diff --git a/utils/third_party/nni/algorithms/compression/pytorch/pruning/simulated_annealing_pruner.py b/utils/third_party/nni/algorithms/compression/pytorch/pruning/simulated_annealing_pruner.py new file mode 100644 index 0000000000000000000000000000000000000000..50f193ba9e3951758c59ee7ef90af1633e4674c4 --- /dev/null +++ b/utils/third_party/nni/algorithms/compression/pytorch/pruning/simulated_annealing_pruner.py @@ -0,0 +1,357 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +import os +import math +import copy +import csv +import json +import numpy as np +from schema import And, Optional + +from nni.utils import OptimizeMode + +from .....compression.pytorch.compressor import Pruner +from .....compression.pytorch.utils.config_validation import CompressorSchema +from .constants_pruner import PRUNER_DICT + + +_logger = logging.getLogger(__name__) + + +class SimulatedAnnealingPruner(Pruner): + """ + A Pytorch implementation of Simulated Annealing compression algorithm. + + Parameters + ---------- + model : pytorch model + The model to be pruned. + config_list : list + Supported keys: + - sparsity : The target overall sparsity. + - op_types : The operation type to prune. + evaluator : function + Function to evaluate the pruned model. + This function should include `model` as the only parameter, and returns a scalar value. + Example:: + + def evaluator(model): + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + val_loader = ... + model.eval() + correct = 0 + with torch.no_grad(): + for data, target in val_loader: + data, target = data.to(device), target.to(device) + output = model(data) + # get the index of the max log-probability + pred = output.argmax(dim=1, keepdim=True) + correct += pred.eq(target.view_as(pred)).sum().item() + accuracy = correct / len(val_loader.dataset) + return accuracy + optimize_mode : str + Optimize mode, `maximize` or `minimize`, by default `maximize`. + base_algo : str + Base pruning algorithm. `level`, `l1` or `l2`, by default `l1`. Given the sparsity distribution among the ops, + the assigned `base_algo` is used to decide which filters/channels/weights to prune. + start_temperature : float + Start temperature of the simulated annealing process. + stop_temperature : float + Stop temperature of the simulated annealing process. + cool_down_rate : float + Cool down rate of the temperature. + perturbation_magnitude : float + Initial perturbation magnitude to the sparsities. The magnitude decreases with current temperature. + experiment_data_dir : string + PATH to save experiment data, + including the config_list generated for the base pruning algorithm, the performance of the pruned model and the pruning history. + + """ + + def __init__(self, model, config_list, evaluator, optimize_mode='maximize', base_algo='l1', + start_temperature=100, stop_temperature=20, cool_down_rate=0.9, perturbation_magnitude=0.35, experiment_data_dir='./'): + # original model + self._model_to_prune = copy.deepcopy(model) + self._base_algo = base_algo + + super().__init__(model, config_list) + + self._evaluator = evaluator + self._optimize_mode = OptimizeMode(optimize_mode) + + # hyper parameters for SA algorithm + self._start_temperature = start_temperature + self._current_temperature = start_temperature + self._stop_temperature = stop_temperature + self._cool_down_rate = cool_down_rate + self._perturbation_magnitude = perturbation_magnitude + + # overall pruning rate + self._sparsity = config_list[0]['sparsity'] + # pruning rates of the layers + self._sparsities = None + + # init current performance & best performance + self._current_performance = -np.inf + self._best_performance = -np.inf + self._best_config_list = [] + + self._search_history = [] + + self._experiment_data_dir = experiment_data_dir + if not os.path.exists(self._experiment_data_dir): + os.makedirs(self._experiment_data_dir) + + def validate_config(self, model, config_list): + """ + Parameters + ---------- + model : torch.nn.Module + Model to be pruned + config_list : list + List on pruning configs + """ + + if self._base_algo == 'level': + schema = CompressorSchema([{ + 'sparsity': And(float, lambda n: 0 < n < 1), + Optional('op_types'): [str], + Optional('op_names'): [str], + }], model, _logger) + elif self._base_algo in ['l1', 'l2']: + schema = CompressorSchema([{ + 'sparsity': And(float, lambda n: 0 < n < 1), + 'op_types': ['Conv2d'], + Optional('op_names'): [str] + }], model, _logger) + + schema.validate(config_list) + + def _sparsities_2_config_list(self, sparsities): + ''' + convert sparsities vector into config_list for LevelPruner or L1FilterPruner + + Parameters + ---------- + sparsities : list + list of sparsities + + Returns + ------- + list of dict + config_list for LevelPruner or L1FilterPruner + ''' + config_list = [] + + sparsities = sorted(sparsities) + self.modules_wrapper = sorted( + self.modules_wrapper, key=lambda wrapper: wrapper.module.weight.data.numel()) + + # a layer with more weights will have no less pruning rate + for idx, wrapper in enumerate(self.get_modules_wrapper()): + # L1Filter Pruner requires to specify op_types + if self._base_algo in ['l1', 'l2']: + config_list.append( + {'sparsity': sparsities[idx], 'op_types': ['Conv2d'], 'op_names': [wrapper.name]}) + elif self._base_algo == 'level': + config_list.append( + {'sparsity': sparsities[idx], 'op_names': [wrapper.name]}) + + config_list = [val for val in config_list if not math.isclose(val['sparsity'], 0, abs_tol=1e-6)] + + return config_list + + def _rescale_sparsities(self, sparsities, target_sparsity): + ''' + Rescale the sparsities list to satisfy the target overall sparsity + + Parameters + ---------- + sparsities : list + + target_sparsity : float + the target overall sparsity + + Returns + ------- + list + the rescaled sparsities + ''' + num_weights = [] + for wrapper in self.get_modules_wrapper(): + num_weights.append(wrapper.module.weight.data.numel()) + + num_weights = sorted(num_weights) + sparsities = sorted(sparsities) + + total_weights = 0 + total_weights_pruned = 0 + + # calculate the scale + for idx, num_weight in enumerate(num_weights): + total_weights += num_weight + total_weights_pruned += int(num_weight*sparsities[idx]) + if total_weights_pruned == 0: + return None + scale = target_sparsity / (total_weights_pruned/total_weights) + + # rescale the sparsities + sparsities = np.asarray(sparsities)*scale + + return sparsities + + def _init_sparsities(self): + ''' + Generate a sorted sparsities vector + ''' + # repeatedly generate a distribution until satisfies the overall sparsity requirement + _logger.info('Gererating sparsities...') + while True: + sparsities = sorted(np.random.uniform( + 0, 1, len(self.get_modules_wrapper()))) + + sparsities = self._rescale_sparsities( + sparsities, target_sparsity=self._sparsity) + + if sparsities is not None and sparsities[0] >= 0 and sparsities[-1] < 1: + _logger.info('Initial sparsities generated : %s', sparsities) + self._sparsities = sparsities + break + + def _generate_perturbations(self): + ''' + Generate perturbation to the current sparsities distribution. + + Returns: + -------- + list + perturbated sparsities + ''' + _logger.info("Gererating perturbations to the current sparsities...") + + # decrease magnitude with current temperature + magnitude = self._current_temperature / \ + self._start_temperature * self._perturbation_magnitude + _logger.info('current perturation magnitude:%s', magnitude) + + while True: + perturbation = np.random.uniform(-magnitude, magnitude, len(self.get_modules_wrapper())) + sparsities = np.clip(0, self._sparsities + perturbation, None) + _logger.debug("sparsities before rescalling:%s", sparsities) + + sparsities = self._rescale_sparsities(sparsities, target_sparsity=self._sparsity) + _logger.debug("sparsities after rescalling:%s", sparsities) + + if sparsities is not None and sparsities[0] >= 0 and sparsities[-1] < 1: + _logger.info("Sparsities perturbated:%s", sparsities) + return sparsities + + def calc_mask(self, wrapper, **kwargs): + return None + + def compress(self, return_config_list=False): + """ + Compress the model with Simulated Annealing. + + Returns + ------- + torch.nn.Module + model with specified modules compressed. + """ + _logger.info('Starting Simulated Annealing Compression...') + + # initiaze a randomized action + pruning_iteration = 0 + self._init_sparsities() + + # stop condition + self._current_temperature = self._start_temperature + while self._current_temperature > self._stop_temperature: + _logger.info('Pruning iteration: %d', pruning_iteration) + _logger.info('Current temperature: %d, Stop temperature: %d', + self._current_temperature, self._stop_temperature) + while True: + # generate perturbation + sparsities_perturbated = self._generate_perturbations() + config_list = self._sparsities_2_config_list( + sparsities_perturbated) + _logger.info( + "config_list for Pruner generated: %s", config_list) + + # fast evaluation + pruner = PRUNER_DICT[self._base_algo](copy.deepcopy(self._model_to_prune), config_list) + model_masked = pruner.compress() + evaluation_result = self._evaluator(model_masked) + + self._search_history.append( + {'sparsity': self._sparsity, 'performance': evaluation_result, 'config_list': config_list}) + + if self._optimize_mode is OptimizeMode.Minimize: + evaluation_result *= -1 + + # if better evaluation result, then accept the perturbation + if evaluation_result > self._current_performance: + self._current_performance = evaluation_result + self._sparsities = sparsities_perturbated + + # save best performance and best params + if evaluation_result > self._best_performance: + _logger.info('updating best model...') + self._best_performance = evaluation_result + self._best_config_list = config_list + + # save the overall best masked model + self.bound_model = model_masked + # the ops with sparsity 0 are not included in this modules_wrapper + modules_wrapper_final = pruner.get_modules_wrapper() + break + # if not, accept with probability e^(-deltaE/current_temperature) + else: + delta_E = np.abs(evaluation_result - + self._current_performance) + probability = math.exp(-1 * delta_E / + self._current_temperature) + if np.random.uniform(0, 1) < probability: + self._current_performance = evaluation_result + self._sparsities = sparsities_perturbated + break + + # cool down + self._current_temperature *= self._cool_down_rate + pruning_iteration += 1 + + _logger.info('----------Compression finished--------------') + _logger.info('Best performance: %s', self._best_performance) + _logger.info('config_list found : %s', + self._best_config_list) + + # save search history + with open(os.path.join(self._experiment_data_dir, 'search_history.csv'), 'w') as csvfile: + writer = csv.DictWriter(csvfile, fieldnames=['sparsity', 'performance', 'config_list']) + writer.writeheader() + for item in self._search_history: + writer.writerow({'sparsity': item['sparsity'], 'performance': item['performance'], 'config_list': json.dumps( + item['config_list'])}) + + # save best config found and best performance + if self._optimize_mode is OptimizeMode.Minimize: + self._best_performance *= -1 + with open(os.path.join(self._experiment_data_dir, 'search_result.json'), 'w+') as jsonfile: + json.dump({ + 'performance': self._best_performance, + 'config_list': json.dumps(self._best_config_list) + }, jsonfile) + + _logger.info('search history and result saved to foler : %s', + self._experiment_data_dir) + + if return_config_list: + return self._best_config_list + + # This should be done only at the final stage, + # because the modules_wrapper with all the ops are used during the annealing process + self.modules_wrapper = modules_wrapper_final + + return self.bound_model diff --git a/utils/third_party/nni/algorithms/compression/pytorch/pruning/structured_pruning.py b/utils/third_party/nni/algorithms/compression/pytorch/pruning/structured_pruning.py new file mode 100644 index 0000000000000000000000000000000000000000..d1f65bc00e2c50405a245404e79b0a584734940b --- /dev/null +++ b/utils/third_party/nni/algorithms/compression/pytorch/pruning/structured_pruning.py @@ -0,0 +1,1272 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +import math +from zedl.dl.common.model import get_module +from zedl.third_party.nni.common.graph_utils import TorchModuleGraph +import numpy as np +import torch +import random +from .weight_masker import WeightMasker + +# from util.model import LayerActivation + +__all__ = ['L1FilterPrunerMasker', 'L2FilterPrunerMasker', 'FPGMPrunerMasker', + 'TaylorFOWeightFilterPrunerMasker', 'ActivationAPoZRankFilterPrunerMasker', + 'ActivationMeanRankFilterPrunerMasker', 'SlimPrunerMasker', 'AMCWeightMasker', + 'TRRPrunerMasker', 'HRankPrunerMasker', 'PFPMasker'] + +logger = logging.getLogger('torch filter pruners') + + +class StructuredWeightMasker(WeightMasker): + """ + A structured pruning masker base class that prunes convolutional layer filters. + + Parameters + ---------- + model: nn.Module + model to be pruned + pruner: Pruner + A Pruner instance used to prune the model + preserve_round: int + after pruning, preserve filters/channels round to `preserve_round`, for example: + for a Conv2d layer, output channel is 32, sparsity is 0.2, if preserve_round is + 1 (no preserve round), then there will be int(32 * 0.2) = 6 filters pruned, and + 32 - 6 = 26 filters are preserved. If preserve_round is 4, preserved filters will + be round up to 28 (which can be divided by 4) and only 4 filters are pruned. + + """ + + def __init__(self, model, pruner, preserve_round=1, dependency_aware=False): + self.model = model + self.pruner = pruner + self.preserve_round = preserve_round + self.dependency_aware = dependency_aware + + def calc_mask(self, sparsity, wrapper, wrapper_idx=None, **depen_kwargs): + """ + calculate the mask for `wrapper`. + Parameters + ---------- + sparsity: float/list of float + The target sparsity of the wrapper. If we calculate the mask in + the normal way, then sparsity is a float number. In contrast, if + we calculate the mask in the dependency-aware way, sparsity is a + list of float numbers, each float number corressponds to a sparsity + of a layer. + wrapper: PrunerModuleWrapper/list of PrunerModuleWrappers + The wrapper of the target layer. If we calculate the mask in the normal + way, then `wrapper` is an instance of PrunerModuleWrapper, else `wrapper` + is a list of PrunerModuleWrapper. + wrapper_idx: int/list of int + The index of the wrapper. + depen_kwargs: dict + The kw_args for the dependency-aware mode. + """ + if not self.dependency_aware: + # calculate the mask in the normal way, each layer calculate its + # own mask separately + return self._normal_calc_mask(sparsity, wrapper, wrapper_idx) + else: + # if the dependency_aware switch is on, then calculate the mask + # in the dependency-aware way + return self._dependency_calc_mask(sparsity, wrapper, wrapper_idx, **depen_kwargs) + + def _get_current_state(self, sparsity, wrapper, wrapper_idx=None): + """ + Some pruner may prune the layers in a iterative way. In each pruning iteration, + we may get the current state of this wrapper/layer, and continue to prune this layer + based on the current state. This function is to get the current pruning state of the + target wrapper/layer. + Parameters + ---------- + sparsity: float + pruning ratio, preserved weight ratio is `1 - sparsity` + wrapper: PrunerModuleWrapper + layer wrapper of this layer + wrapper_idx: int + index of this wrapper in pruner's all wrappers + Returns + ------- + base_mask: dict + dict object that stores the mask of this wrapper in this iteration, if it is the + first iteration, then we create a new mask with all ones. If there is already a + mask in this wrapper, then we return the existing mask. + weight: tensor + the current weight of this layer + num_prune: int + how many filters we should prune + """ + msg = 'module type {} is not supported!'.format(wrapper.type) + assert wrapper.type == 'Conv2d' or wrapper.type == 'ConvTranspose2d', msg + weight = wrapper.module.weight.data + bias = None + if hasattr(wrapper.module, 'bias') and wrapper.module.bias is not None: + bias = wrapper.module.bias.data + + if wrapper.weight_mask is None: + mask_weight = torch.ones(weight.size()).type_as(weight).detach() + else: + mask_weight = wrapper.weight_mask.clone() + if bias is not None: + if wrapper.bias_mask is None: + mask_bias = torch.ones(bias.size()).type_as(bias).detach() + else: + mask_bias = wrapper.bias_mask.clone() + else: + mask_bias = None + mask = {'weight_mask': mask_weight, 'bias_mask': mask_bias} + + if isinstance(wrapper.module, torch.nn.Conv2d): + num_total = weight.size(0) + elif isinstance(wrapper.module, torch.nn.ConvTranspose2d): + num_total = weight.size(1) + + num_prune = int(num_total * sparsity) + if self.preserve_round > 1: + num_preserve = num_total - num_prune + num_preserve = int( + math.ceil(num_preserve * 1. / self.preserve_round) * self.preserve_round) + if num_preserve > num_total: + num_preserve = int(math.floor( + num_total * 1. / self.preserve_round) * self.preserve_round) + num_prune = num_total - num_preserve + # weight*mask_weight: apply base mask for iterative pruning + return mask, weight * mask_weight, num_prune + + def _normal_calc_mask(self, sparsity, wrapper, wrapper_idx=None): + """ + Calculate the mask of given layer. + Parameters + ---------- + sparsity: float + pruning ratio, preserved weight ratio is `1 - sparsity` + wrapper: PrunerModuleWrapper + layer wrapper of this layer + wrapper_idx: int + index of this wrapper in pruner's all wrappers + Returns + ------- + dict + dictionary for storing masks, keys of the dict: + 'weight_mask': weight mask tensor + 'bias_mask': bias mask tensor (optional) + """ + mask, weight, num_prune = self._get_current_state( + sparsity, wrapper, wrapper_idx) + num_total = weight.size(0) + if num_total < 2 or num_prune < 1: + return mask + + return self.get_mask(mask, weight, num_prune, wrapper, wrapper_idx) + + def _common_channel_to_prune(self, sparsities, wrappers, wrappers_idx, channel_dsets, groups): + """ + Calculate the common channels should be pruned by all the layers in this group. + This function is for filter pruning of Conv layers. if want to support the dependency-aware + mode for others ops, you need to inherit this class and overwrite `_common_channel_to_prune`. + + Parameters + ---------- + sparsities : list + List of float that specify the sparsity for each conv layer. + wrappers : list + List of wrappers + groups : list + The number of the filter groups of each layer. + wrappers_idx : list + The indexes of the wrappers + """ + # sparsity configs for each wrapper + # sparsities = [_w.config['sparsity'] for _w in wrappers] + # check the type of the input wrappers + for _w in wrappers: + msg = 'module type {} is not supported!'.format(_w.type) + assert _w.type == 'Conv2d', msg + # Among the dependent layers, the layer with smallest + # sparsity determines the final benefit of the speedup + # module. To better harvest the speed benefit, we need + # to ensure that these dependent layers have at least + # `min_sparsity` pruned channel are the same. + if len(channel_dsets) == len(wrappers): + # all the layers in the dependency sets are pruned + min_sparsity = min(sparsities) + else: + # not all the layers in the dependency set + # are pruned + min_sparsity = 0 + # donnot prune the channels that we cannot harvest the speed from + sparsities = [min_sparsity] * len(sparsities) + # find the max number of the filter groups of the dependent + # layers. The group constraint of this dependency set is decided + # by the layer with the max groups. + + # should use the least common multiple for all the groups + # the max_group is lower than the channel_count, because + # the number of the filter is always divisible by the number of the group + max_group = np.lcm.reduce(groups) + channel_count = wrappers[0].module.weight.data.size(0) + device = wrappers[0].module.weight.device + channel_sum = torch.zeros(channel_count).to(device) + for _w, _w_idx in zip(wrappers, wrappers_idx): + # calculate the L1/L2 sum for all channels + c_sum = self.get_channel_sum(_w, _w_idx) + + if c_sum is None: + # if the channel sum cannot be calculated + # now, return None + return None + channel_sum += c_sum + + # prune the same `min_sparsity` channels based on channel_sum + # for all the layers in the channel sparsity + target_pruned = int(channel_count * min_sparsity) + # pruned_per_group may be zero, for example dw conv + pruned_per_group = int(target_pruned / max_group) + group_step = int(channel_count / max_group) + + channel_masks = [] + for gid in range(max_group): + _start = gid * group_step + _end = (gid + 1) * group_step + if pruned_per_group > 0: + threshold = torch.topk( + channel_sum[_start: _end], pruned_per_group, largest=False)[0].max() + group_mask = torch.gt(channel_sum[_start:_end], threshold) + else: + group_mask = torch.ones(group_step).to(device) + channel_masks.append(group_mask) + channel_masks = torch.cat(channel_masks, dim=0) + pruned_channel_index = ( + channel_masks == False).nonzero().squeeze(1).tolist() + logger.info('Prune the %s channels for all dependent', + ','.join([str(x) for x in pruned_channel_index])) + return channel_masks + + def _dependency_calc_mask(self, sparsities, wrappers, wrappers_idx, channel_dsets, groups): + """ + Calculate the masks for the layers in the same dependency sets. + Similar to the traditional original calc_mask, _dependency_calc_mask + will prune the target layers based on the L1/L2 norm of the weights. + However, StructuredWeightMasker prunes the filter completely based on the + L1/L2 norm of each filter. In contrast, _dependency_calc_mask + will try to satisfy the channel/group dependency(see nni.compression.torch. + utils.shape_dependency for details). Specifically, _dependency_calc_mask + will try to prune the same channels for the layers that have channel dependency. + In addition, this mask calculator will also ensure that the number of filters + pruned in each group is the same(meet the group dependency). + + Parameters + ---------- + sparsities : list + List of float that specify the sparsity for each conv layer. + wrappers : list + List of wrappers + groups : list + The number of the filter groups of each layer. + wrappers_idx : list + The indexes of the wrappers + """ + channel_masks = self._common_channel_to_prune( + sparsities, wrappers, wrappers_idx, channel_dsets, groups) + # calculate the mask for each layer based on channel_masks, first + # every layer will prune the same channels masked in channel_masks. + # If the sparsity of a layers is larger than min_sparsity, then it + # will continue prune sparsity - min_sparsity channels to meet the sparsity + # config. + masks = {} + for _pos, _w in enumerate(wrappers): + _w_idx = wrappers_idx[_pos] + sparsity = sparsities[_pos] + name = _w.name + + # _tmp_mask = self._normal_calc_mask( + # sparsity, _w, _w_idx, channel_masks) + base_mask, current_weight, num_prune = self._get_current_state( + sparsity, _w, _w_idx) + num_total = current_weight.size(0) + if num_total < 2 or num_prune < 1: + return base_mask + _tmp_mask = self.get_mask( + base_mask, current_weight, num_prune, _w, _w_idx, channel_masks) + + if _tmp_mask is None: + # if the mask calculation fails + return None + masks[name] = _tmp_mask + return masks + + def get_mask(self, base_mask, weight, num_prune, wrapper, wrapper_idx, channel_masks=None): + """ + Calculate the mask of given layer. + Parameters + ---------- + base_mask: dict + The basic mask with the same shape of weight, all item in the basic mask is 1. + weight: tensor + the module weight to be pruned + num_prune: int + Num of filters to prune + wrapper: PrunerModuleWrapper + layer wrapper of this layer + wrapper_idx: int + index of this wrapper in pruner's all wrappers + channel_masks: Tensor + If mask some channels for this layer in advance. In the dependency-aware + mode, before calculating the masks for each layer, we will calculate a common + mask for all the layers in the dependency set. For the pruners that doesnot + support dependency-aware mode, they can just ignore this parameter. + Returns + ------- + dict + dictionary for storing masks + """ + raise NotImplementedError( + '{} get_mask is not implemented'.format(self.__class__.__name__)) + + def get_channel_sum(self, wrapper, wrapper_idx): + """ + Calculate the importance weight for each channel. If want to support the + dependency-aware mode for this one-shot pruner, this function must be + implemented. + Parameters + ---------- + wrapper: PrunerModuleWrapper + layer wrapper of this layer + wrapper_idx: int + index of this wrapper in pruner's all wrappers + Returns + ------- + tensor + Tensor that indicates the importance of each channel + """ + raise NotImplementedError( + '{} get_channel_sum is not implemented'.format(self.__class__.__name__)) + + +class L1FilterPrunerMasker(StructuredWeightMasker): + """ + A structured pruning algorithm that prunes the filters of smallest magnitude + weights sum in the convolution layers to achieve a preset level of network sparsity. + Hao Li, Asim Kadav, Igor Durdanovic, Hanan Samet and Hans Peter Graf, + "PRUNING FILTERS FOR EFFICIENT CONVNETS", 2017 ICLR + https://arxiv.org/abs/1608.08710 + """ + + def get_mask(self, base_mask, weight, num_prune, wrapper, wrapper_idx, channel_masks=None): + if isinstance(wrapper.module, torch.nn.Conv2d): + # get the l1-norm sum for each filter + w_abs_structured = self.get_channel_sum(wrapper, wrapper_idx) + if channel_masks is not None: + # if we need to mask some channels in advance + w_abs_structured = w_abs_structured * channel_masks + threshold = torch.topk(w_abs_structured.view(-1), + num_prune, largest=False)[0].max() + mask_weight = torch.gt(w_abs_structured, threshold)[ + :, None, None, None].expand_as(weight).type_as(weight) + mask_bias = torch.gt(w_abs_structured, threshold).type_as( + weight).detach() if base_mask['bias_mask'] is not None else None + + return {'weight_mask': mask_weight.detach(), 'bias_mask': mask_bias} + elif isinstance(wrapper.module, torch.nn.ConvTranspose2d): + # get the l1-norm sum for each filter + w_abs_structured = self.get_channel_sum(wrapper, wrapper_idx) + if channel_masks is not None: + # if we need to mask some channels in advance + w_abs_structured = w_abs_structured * channel_masks + threshold = torch.topk(w_abs_structured.view(-1), + num_prune, largest=False)[0].max() + mask_weight = torch.gt(w_abs_structured, threshold)[ + None, :, None, None].expand_as(weight).type_as(weight) + mask_bias = torch.gt(w_abs_structured, threshold).type_as( + weight).detach() if base_mask['bias_mask'] is not None else None + + return {'weight_mask': mask_weight.detach(), 'bias_mask': mask_bias} + + def get_channel_sum(self, wrapper, wrapper_idx): + if isinstance(wrapper.module, torch.nn.Conv2d): + weight = wrapper.module.weight.data + filters = weight.shape[0] + w_abs = weight.abs() + w_abs_structured = w_abs.view(filters, -1).sum(dim=1) + return w_abs_structured + elif isinstance(wrapper.module, torch.nn.ConvTranspose2d): + weight = wrapper.module.weight.data + filters = weight.shape[1] + w_abs = weight.abs() + w_abs_structured = w_abs.view(filters, -1).sum(dim=1) + return w_abs_structured + + +class L2FilterPrunerMasker(StructuredWeightMasker): + """ + A structured pruning algorithm that prunes the filters with the + smallest L2 norm of the weights. + """ + + def get_mask(self, base_mask, weight, num_prune, wrapper, wrapper_idx, channel_masks=None): + # get the l2-norm sum for each filter + w_l2_norm = self.get_channel_sum(wrapper, wrapper_idx) + if channel_masks is not None: + # if we need to mask some channels in advance + w_l2_norm = w_l2_norm * channel_masks + threshold = torch.topk( + w_l2_norm.view(-1), num_prune, largest=False)[0].max() + mask_weight = torch.gt(w_l2_norm, threshold)[ + :, None, None, None].expand_as(weight).type_as(weight) + mask_bias = torch.gt(w_l2_norm, threshold).type_as( + weight).detach() if base_mask['bias_mask'] is not None else None + + return {'weight_mask': mask_weight.detach(), 'bias_mask': mask_bias} + + def get_channel_sum(self, wrapper, wrapper_idx): + weight = wrapper.module.weight.data + filters = weight.shape[0] + w = weight.view(filters, -1) + w_l2_norm = torch.sqrt((w ** 2).sum(dim=1)) + return w_l2_norm + + +class TRRPrunerMasker(StructuredWeightMasker): + """ + A filter pruner via Triplet Response Residual (TRR). + B. Fang, X. Zeng, and M. Zhang, + “NestDNN: Resource-aware multi-tenant on-device deep learning for continuous mobile vision,” + in Proc. ACM Mobicom, 2018, pp. 115–127. + + Implemented by queyu, 2020/11/13 + """ + + def __init__(self, model, pruner, preserve_round=1, dependency_aware=False, train_loader=None, device='cuda'): + super().__init__(model, pruner, preserve_round, dependency_aware) + + assert train_loader is not None, 'TRR needs the training dataset' + self.train_loader = train_loader + self.device = device + + self.model.to(device) + + def get_mask(self, base_mask, weight, num_prune, wrapper, wrapper_idx, channel_masks=None): + trrs = self.get_channel_sum(wrapper, wrapper_idx) + if channel_masks is not None: + # if we need to mask some channels in advance + trrs = trrs * channel_masks + + threshold = torch.topk( + trrs, num_prune, largest=False)[0].max() + mask_weight = torch.gt(trrs, threshold)[ + :, None, None, None].expand_as(weight).type_as(weight) + mask_bias = torch.gt(trrs, threshold).type_as( + weight).detach() if base_mask['bias_mask'] is not None else None + + return {'weight_mask': mask_weight.detach(), 'bias_mask': mask_bias} + + def get_channel_sum(self, wrapper, wrapper_idx): + # add hook in the target module + module_activation = LayerActivation(wrapper.module, '', 'cpu') + self.model.eval() + + filter_num = wrapper.module.weight.data.size()[0] + trrs = torch.zeros(filter_num) + data_num = len(self.train_loader.dataset) + cur_data_num = 0 + triplets_num = 0 + + with torch.no_grad(): + for data, target in self.train_loader: + cur_data_num += data.size()[0] + # if cur_data_num > data_num * 0.2: + # continue + + data, target = data.to(self.device), target.to(self.device) + self.model(data) + + # construct {anc, pos, neg} from data and target + # compute TRR from data in module_activation.output + triplets = self._constrcut_trr_triplets(data, target) + triplets_num += len(triplets) + trrs += self._compute_trr(triplets, module_activation.output) + + module_activation.remove() + return trrs + + def _constrcut_trr_triplets(self, data, target): + # return the indexes, not the data + + # randomly select the anc in data + anc = random.randint(0, data.size()[0] - 1) + anc_label = target[anc] + + triplets = [] + poses = [] + negs = [] + + # find the pos, which has the same class with anc + for i, (item, label) in enumerate(zip(data, target)): + if i == anc: + continue + # pos + if label == anc_label: + poses += [i] + # neg + else: + negs += [i] + + # balance the pos and neg + # min_len = min(len(poses), len(negs)) + # poses = poses[0: min_len + 1] + # negs = negs[0: min_len + 1] + + for pos in poses: + for neg in negs: + triplets += [(anc, pos, neg)] + + return triplets + + def _compute_trr(self, triplets, feature_map): + def l2_norm(data): + return (data ** 2).sum(dim=(1, 2)) + + trr = torch.zeros(feature_map.size()[1]) + for triplet in triplets: + anc, pos, neg = triplet + a = l2_norm(feature_map[anc] - feature_map[neg]) + b = l2_norm(feature_map[anc] - feature_map[pos]) + trr_item = a - b + trr += trr_item + + return trr + + +class FPGMPrunerMasker(StructuredWeightMasker): + """ + A filter pruner via geometric median. + "Filter Pruning via Geometric Median for Deep Convolutional Neural Networks Acceleration", + https://arxiv.org/pdf/1811.00250.pdf + """ + + def get_mask(self, base_mask, weight, num_prune, wrapper, wrapper_idx, channel_masks=None): + min_gm_idx = self._get_min_gm_kernel_idx( + num_prune, wrapper, wrapper_idx, channel_masks) + for idx in min_gm_idx: + base_mask['weight_mask'][idx] = 0. + if base_mask['bias_mask'] is not None: + base_mask['bias_mask'][idx] = 0. + return base_mask + + def _get_min_gm_kernel_idx(self, num_prune, wrapper, wrapper_idx, channel_masks): + channel_dist = self.get_channel_sum(wrapper, wrapper_idx) + if channel_masks is not None: + channel_dist = channel_dist * channel_masks + dist_list = [(channel_dist[i], i) + for i in range(channel_dist.size(0))] + min_gm_kernels = sorted(dist_list, key=lambda x: x[0])[:num_prune] + return [x[1] for x in min_gm_kernels] + + def _get_distance_sum(self, weight, out_idx): + """ + Calculate the total distance between a specified filter (by out_idex and in_idx) and + all other filters. + Parameters + ---------- + weight: Tensor + convolutional filter weight + out_idx: int + output channel index of specified filter, this method calculates the total distance + between this specified filter and all other filters. + Returns + ------- + float32 + The total distance + """ + logger.debug('weight size: %s', weight.size()) + assert len(weight.size()) in [3, 4], 'unsupported weight shape' + + w = weight.view(weight.size(0), -1) + anchor_w = w[out_idx].unsqueeze(0).expand(w.size(0), w.size(1)) + x = w - anchor_w + x = (x * x).sum(-1) + x = torch.sqrt(x) + return x.sum() + + def get_channel_sum(self, wrapper, wrapper_idx): + weight = wrapper.module.weight.data + assert len(weight.size()) in [3, 4] + dist_list = [] + for out_i in range(weight.size(0)): + dist_sum = self._get_distance_sum(weight, out_i) + dist_list.append(dist_sum) + return torch.Tensor(dist_list).to(weight.device) + + +class TaylorFOWeightFilterPrunerMasker(StructuredWeightMasker): + """ + A structured pruning algorithm that prunes the filters with the smallest + importance approximations based on the first order taylor expansion on the weight. + Molchanov, Pavlo and Mallya, Arun and Tyree, Stephen and Frosio, Iuri and Kautz, Jan, + "Importance Estimation for Neural Network Pruning", CVPR 2019. + http://jankautz.com/publications/Importance4NNPruning_CVPR19.pdf + """ + + def __init__(self, model, pruner, statistics_batch_num=1): + super().__init__(model, pruner) + self.pruner.statistics_batch_num = statistics_batch_num + self.pruner.set_wrappers_attribute("contribution", None) + self.pruner.iterations = 0 + self.pruner.patch_optimizer(self.calc_contributions) + + def get_mask(self, base_mask, weight, num_prune, wrapper, wrapper_idx, channel_masks=None): + if isinstance(wrapper.module, torch.nn.Conv2d): + channel_contribution = self.get_channel_sum(wrapper, wrapper_idx) + if channel_contribution is None: + # iteration is not enough + return None + if channel_masks is not None: + channel_contribution = channel_contribution * channel_masks + prune_indices = torch.argsort(channel_contribution)[:num_prune] + for idx in prune_indices: + base_mask['weight_mask'][idx] = 0. + if base_mask['bias_mask'] is not None: + base_mask['bias_mask'][idx] = 0. + return base_mask + elif isinstance(wrapper.module, torch.nn.ConvTranspose2d): + channel_contribution = self.get_channel_sum(wrapper, wrapper_idx) + if channel_contribution is None: + # iteration is not enough + return None + if channel_masks is not None: + channel_contribution = channel_contribution * channel_masks + prune_indices = torch.argsort(channel_contribution)[:num_prune] + for idx in prune_indices: + base_mask['weight_mask'][:, idx] = 0. + if base_mask['bias_mask'] is not None: + base_mask['bias_mask'][idx] = 0. + return base_mask + + def calc_contributions(self): + """ + Calculate the estimated importance of filters as a sum of individual contribution + based on the first order taylor expansion. + """ + if self.pruner.iterations >= self.pruner.statistics_batch_num: + return + for wrapper in self.pruner.get_modules_wrapper(): + if isinstance(wrapper.module, torch.nn.Conv2d): + filters = wrapper.module.weight.size(0) + contribution = ( + wrapper.module.weight*wrapper.module.weight.grad).data.pow(2).view(filters, -1).sum(dim=1) + if wrapper.contribution is None: + wrapper.contribution = contribution + else: + wrapper.contribution += contribution + elif isinstance(wrapper.module, torch.nn.ConvTranspose2d): + filters = wrapper.module.weight.size(1) + contribution = ( + wrapper.module.weight*wrapper.module.weight.grad).data.pow(2).view(filters, -1).sum(dim=1) + if wrapper.contribution is None: + wrapper.contribution = contribution + else: + wrapper.contribution += contribution + + self.pruner.iterations += 1 + + def get_channel_sum(self, wrapper, wrapper_idx): + if self.pruner.iterations < self.pruner.statistics_batch_num: + return None + if wrapper.contribution is None: + return None + return wrapper.contribution + + +class ActivationFilterPrunerMasker(StructuredWeightMasker): + def __init__(self, model, pruner, statistics_batch_num=1, activation='relu'): + super().__init__(model, pruner) + self.statistics_batch_num = statistics_batch_num + self.pruner.hook_id = self._add_activation_collector(self.pruner) + + assert activation in ['relu', 'relu6', None] + if activation == 'relu': + self.pruner.activation = torch.nn.functional.relu + elif activation == 'relu6': + self.pruner.activation = torch.nn.functional.relu6 + else: + self.pruner.activation = None + + def _add_activation_collector(self, pruner): + def collector(collected_activation): + def hook(module_, input_, output): + if pruner.activation is not None: + collected_activation.append( + pruner.activation(output.detach().cpu())) + else: + collected_activation.append( + output.detach().cpu()) + return hook + pruner.collected_activation = {} + pruner._fwd_hook_id += 1 + pruner._fwd_hook_handles[pruner._fwd_hook_id] = [] + + for wrapper_idx, wrapper in enumerate(pruner.get_modules_wrapper()): + pruner.collected_activation[wrapper_idx] = [] + handle = wrapper.register_forward_hook( + collector(pruner.collected_activation[wrapper_idx])) + + pruner._fwd_hook_handles[pruner._fwd_hook_id].append(handle) + return pruner._fwd_hook_id + + +class ActivationAPoZRankFilterPrunerMasker(ActivationFilterPrunerMasker): + """ + A structured pruning algorithm that prunes the filters with the + smallest APoZ(average percentage of zeros) of output activations. + Hengyuan Hu, Rui Peng, Yu-Wing Tai and Chi-Keung Tang, + "Network Trimming: A Data-Driven Neuron Pruning Approach towards Efficient Deep Architectures", ICLR 2016. + https://arxiv.org/abs/1607.03250 + """ + + def get_mask(self, base_mask, weight, num_prune, wrapper, wrapper_idx, channel_masks=None): + apoz = self.get_channel_sum(wrapper, wrapper_idx) + if apoz is None: + # the collected activations are not enough + return None + if channel_masks is not None: + apoz = apoz * channel_masks + + prune_indices = torch.argsort(apoz)[:num_prune] + for idx in prune_indices: + base_mask['weight_mask'][idx] = 0. + if base_mask['bias_mask'] is not None: + base_mask['bias_mask'][idx] = 0. + + if self.pruner.hook_id in self.pruner._fwd_hook_handles: + self.pruner.remove_activation_collector(self.pruner.hook_id) + + return base_mask + + def _calc_apoz(self, activations): + """ + Calculate APoZ(average percentage of zeros) of activations. + + Parameters + ---------- + activations : list + Layer's output activations + + Returns + ------- + torch.Tensor + Filter's APoZ(average percentage of zeros) of the activations + """ + activations = torch.cat(activations, 0) + _eq_zero = torch.eq(activations, torch.zeros_like(activations)) + _apoz = torch.sum(_eq_zero, dim=(0, 2, 3), dtype=torch.float64) / \ + torch.numel(_eq_zero[:, 0, :, :]) + return torch.ones_like(_apoz) - _apoz + + def get_channel_sum(self, wrapper, wrapper_idx): + assert wrapper_idx is not None + activations = self.pruner.collected_activation[wrapper_idx] + if len(activations) < self.statistics_batch_num: + # collected activations is not enough + return None + return self._calc_apoz(activations).to(wrapper.module.weight.device) + + +class ActivationMeanRankFilterPrunerMasker(ActivationFilterPrunerMasker): + """ + A structured pruning algorithm that prunes the filters with the + smallest mean value of output activations. + Pavlo Molchanov, Stephen Tyree, Tero Karras, Timo Aila and Jan Kautz, + "Pruning Convolutional Neural Networks for Resource Efficient Inference", ICLR 2017. + https://arxiv.org/abs/1611.06440 + """ + + def get_mask(self, base_mask, weight, num_prune, wrapper, wrapper_idx, channel_masks=None): + + mean_activation = self.get_channel_sum(wrapper, wrapper_idx) + if mean_activation is None: + # the collected activation is not enough + return None + if channel_masks is not None: + mean_activation = mean_activation * channel_masks + + prune_indices = torch.argsort(mean_activation)[:num_prune] + for idx in prune_indices: + base_mask['weight_mask'][idx] = 0. + if base_mask['bias_mask'] is not None: + base_mask['bias_mask'][idx] = 0. + # if len(activations) < self.statistics_batch_num, the code + # cannot reach here + if self.pruner.hook_id in self.pruner._fwd_hook_handles: + self.pruner.remove_activation_collector(self.pruner.hook_id) + + return base_mask + + def _cal_mean_activation(self, activations): + """ + Calculate mean value of activations. + + Parameters + ---------- + activations : list + Layer's output activations + + Returns + ------- + torch.Tensor + Filter's mean value of the output activations + """ + activations = torch.cat(activations, 0) + mean_activation = torch.mean(activations, dim=(0, 2, 3)) + return mean_activation + + def get_channel_sum(self, wrapper, wrapper_idx): + assert wrapper_idx is not None + activations = self.pruner.collected_activation[wrapper_idx] + if len(activations) < self.statistics_batch_num: + return None + # the memory overhead here is acceptable, because only + # the mean_activation tensor returned by _cal_mean_activation + # is transfer to gpu. + return self._cal_mean_activation(activations).to(wrapper.module.weight.device) + + +class SlimPrunerMasker(WeightMasker): + """ + A structured pruning algorithm that prunes channels by pruning the weights of BN layers. + Zhuang Liu, Jianguo Li, Zhiqiang Shen, Gao Huang, Shoumeng Yan and Changshui Zhang + "Learning Efficient Convolutional Networks through Network Slimming", 2017 ICCV + https://arxiv.org/pdf/1708.06519.pdf + """ + + def __init__(self, model, pruner, **kwargs): + super().__init__(model, pruner) + weight_list = [] + for (layer, _) in pruner.get_modules_to_compress(): + weight_list.append(layer.module.weight.data.abs().clone()) + all_bn_weights = torch.cat(weight_list) + k = int(all_bn_weights.shape[0] * pruner.config_list[0]['sparsity']) + self.global_threshold = torch.topk( + all_bn_weights.view(-1), k, largest=False)[0].max() + + def calc_mask(self, sparsity, wrapper, wrapper_idx=None): + assert wrapper.type == 'BatchNorm2d', 'SlimPruner only supports 2d batch normalization layer pruning' + weight = wrapper.module.weight.data.clone() + if wrapper.weight_mask is not None: + # apply base mask for iterative pruning + weight = weight * wrapper.weight_mask + + base_mask = torch.ones(weight.size()).type_as(weight).detach() + mask = {'weight_mask': base_mask.detach( + ), 'bias_mask': base_mask.clone().detach()} + filters = weight.size(0) + num_prune = int(filters * sparsity) + if filters >= 2 and num_prune >= 1: + w_abs = weight.abs() + mask_weight = torch.gt( + w_abs, self.global_threshold).type_as(weight) + mask_bias = mask_weight.clone() + mask = {'weight_mask': mask_weight.detach( + ), 'bias_mask': mask_bias.detach()} + return mask + + +def least_square_sklearn(X, Y): + from sklearn.linear_model import LinearRegression + reg = LinearRegression(fit_intercept=False) + reg.fit(X, Y) + return reg.coef_ + + +class AMCWeightMasker(WeightMasker): + """ + Weight maskser class for AMC pruner. Currently, AMCPruner only supports pruning kernel + size 1x1 pointwise Conv2d layer. Before using this class to prune kernels, AMCPruner + collected input and output feature maps for each layer, the features maps are flattened + and save into wrapper.input_feat and wrapper.output_feat. + + Parameters + ---------- + model: nn.Module + model to be pruned + pruner: Pruner + A Pruner instance used to prune the model + preserve_round: int + after pruning, preserve filters/channels round to `preserve_round`, for example: + for a Conv2d layer, output channel is 32, sparsity is 0.2, if preserve_round is + 1 (no preserve round), then there will be int(32 * 0.2) = 6 filters pruned, and + 32 - 6 = 26 filters are preserved. If preserve_round is 4, preserved filters will + be round up to 28 (which can be divided by 4) and only 4 filters are pruned. + """ + + def __init__(self, model, pruner, preserve_round=1): + self.model = model + self.pruner = pruner + self.preserve_round = preserve_round + + def calc_mask(self, sparsity, wrapper, wrapper_idx=None, preserve_idx=None): + """ + Calculate the mask of given layer. + Parameters + ---------- + sparsity: float + pruning ratio, preserved weight ratio is `1 - sparsity` + wrapper: PrunerModuleWrapper + layer wrapper of this layer + wrapper_idx: int + index of this wrapper in pruner's all wrappers + Returns + ------- + dict + dictionary for storing masks, keys of the dict: + 'weight_mask': weight mask tensor + 'bias_mask': bias mask tensor (optional) + """ + msg = 'module type {} is not supported!'.format(wrapper.type) + assert wrapper.type in ['Conv2d', 'Linear'], msg + weight = wrapper.module.weight.data + bias = None + if hasattr(wrapper.module, 'bias') and wrapper.module.bias is not None: + bias = wrapper.module.bias.data + + if wrapper.weight_mask is None: + mask_weight = torch.ones(weight.size()).type_as(weight).detach() + else: + mask_weight = wrapper.weight_mask.clone() + if bias is not None: + if wrapper.bias_mask is None: + mask_bias = torch.ones(bias.size()).type_as(bias).detach() + else: + mask_bias = wrapper.bias_mask.clone() + else: + mask_bias = None + mask = {'weight_mask': mask_weight, 'bias_mask': mask_bias} + + num_total = weight.size(1) + num_prune = int(num_total * sparsity) + if self.preserve_round > 1: + num_preserve = num_total - num_prune + num_preserve = int( + math.ceil(num_preserve * 1. / self.preserve_round) * self.preserve_round) + if num_preserve > num_total: + num_preserve = num_total + num_prune = num_total - num_preserve + + if (num_total < 2 or num_prune < 1) and preserve_idx is None: + return mask + + return self.get_mask(mask, weight, num_preserve, wrapper, wrapper_idx, preserve_idx) + + def get_mask(self, base_mask, weight, num_preserve, wrapper, wrapper_idx, preserve_idx): + w = weight.data.cpu().numpy() + if wrapper.type == 'Linear': + w = w[:, :, None, None] + + if preserve_idx is None: + importance = np.abs(w).sum((0, 2, 3)) + # sum magnitude along C_in, sort descend + sorted_idx = np.argsort(-importance) + d_prime = num_preserve + preserve_idx = sorted_idx[:d_prime] # to preserve index + else: + d_prime = len(preserve_idx) + + assert len(preserve_idx) == d_prime + mask = np.zeros(w.shape[1], bool) + mask[preserve_idx] = True + + # reconstruct, X, Y <= [N, C] + X, Y = wrapper.input_feat, wrapper.output_feat + masked_X = X[:, mask] + if w.shape[2] == 1: # 1x1 conv or fc + rec_weight = least_square_sklearn(X=masked_X, Y=Y) + rec_weight = rec_weight.reshape(-1, 1, 1, d_prime) # (C_out, K_h, K_w, C_in') + rec_weight = np.transpose(rec_weight, (0, 3, 1, 2)) # (C_out, C_in', K_h, K_w) + + rec_weight_pad = np.zeros_like(w) + # pylint: disable=all + rec_weight_pad[:, mask, :, :] = rec_weight + rec_weight = rec_weight_pad + + if wrapper.type == 'Linear': + rec_weight = rec_weight.squeeze() + assert len(rec_weight.shape) == 2 + + # now assign + wrapper.module.weight.data = torch.from_numpy(rec_weight).to(weight.device) + + mask_weight = torch.zeros_like(weight) + if wrapper.type == 'Linear': + mask_weight[:, preserve_idx] = 1. + if base_mask['bias_mask'] is not None and wrapper.module.bias is not None: + mask_bias = torch.ones_like(wrapper.module.bias) + else: + mask_weight[:, preserve_idx, :, :] = 1. + mask_bias = None + + return {'weight_mask': mask_weight.detach(), 'bias_mask': mask_bias} + + +# implemented by queyu, 2021/6/10 +class HRankPrunerMasker(ActivationFilterPrunerMasker): + + def get_mask(self, base_mask, weight, num_prune, wrapper, wrapper_idx, channel_masks=None): + if isinstance(wrapper.module, torch.nn.Conv2d): + apoz = self.get_channel_sum(wrapper, wrapper_idx) + if apoz is None: + # the collected activations are not enough + return None + if channel_masks is not None: + apoz = apoz * channel_masks + + prune_indices = torch.argsort(apoz)[:num_prune] + for idx in prune_indices: + base_mask['weight_mask'][idx] = 0. + if base_mask['bias_mask'] is not None: + base_mask['bias_mask'][idx] = 0. + + if self.pruner.hook_id in self.pruner._fwd_hook_handles: + self.pruner.remove_activation_collector(self.pruner.hook_id) + + return base_mask + + elif isinstance(wrapper.module, torch.nn.ConvTranspose2d): + apoz = self.get_channel_sum(wrapper, wrapper_idx) + if apoz is None: + # the collected activations are not enough + return None + if channel_masks is not None: + apoz = apoz * channel_masks + + prune_indices = torch.argsort(apoz)[:num_prune] + for idx in prune_indices: + base_mask['weight_mask'][:, idx] = 0. + if base_mask['bias_mask'] is not None: + base_mask['bias_mask'][idx] = 0. + + if self.pruner.hook_id in self.pruner._fwd_hook_handles: + self.pruner.remove_activation_collector(self.pruner.hook_id) + + return base_mask + + def _calc_hrank(self, activations, wrapper): + if isinstance(wrapper.module, torch.nn.Conv2d): + activations = torch.cat(activations, 0) + hrank_stats = torch.zeros((activations.size()[0], activations.size()[1])) + + for activation_index in range(activations.size()[0]): + activation = activations[activation_index] + + for feature_map_index in range(activation.size()[0]): + feature_map = activation[feature_map_index] + + rank = torch.matrix_rank(feature_map) + hrank_stats[activation_index][feature_map_index] = rank + + # print(hrank_stats[0], hrank_stats[1], hrank_stats[2]) + hrank_stats = torch.mean(hrank_stats, dim=0) + # print(hrank_stats) + # raise NotImplementedError() + return hrank_stats + + elif isinstance(wrapper.module, torch.nn.ConvTranspose2d): + activations = torch.cat(activations, 0) + hrank_stats = torch.zeros((activations.size()[0], activations.size()[1])) + + for activation_index in range(activations.size()[0]): + activation = activations[activation_index] + + for feature_map_index in range(activation.size()[0]): + feature_map = activation[feature_map_index] + + rank = torch.matrix_rank(feature_map) + hrank_stats[activation_index][feature_map_index] = rank + + # print(hrank_stats[0], hrank_stats[1], hrank_stats[2]) + hrank_stats = torch.mean(hrank_stats, dim=0) + # print(hrank_stats) + # raise NotImplementedError() + return hrank_stats + + def get_channel_sum(self, wrapper, wrapper_idx): + assert wrapper_idx is not None + activations = self.pruner.collected_activation[wrapper_idx] + if len(activations) < self.statistics_batch_num: + # collected activations is not enough + return None + return self._calc_hrank(activations, wrapper).to(wrapper.module.weight.device) + + +# implemented by queyu, 2021/6/10 +class PFPMasker(ActivationFilterPrunerMasker): + + def get_mask(self, base_mask, weight, num_prune, wrapper, wrapper_idx, channel_masks=None): + if isinstance(wrapper.module, torch.nn.Conv2d): + apoz = self.get_channel_sum(wrapper, wrapper_idx) + if apoz is None: + # the collected activations are not enough + return None + if channel_masks is not None: + apoz = apoz * channel_masks + + prune_indices = torch.argsort(apoz)[:num_prune] + for idx in prune_indices: + base_mask['weight_mask'][idx] = 0. + if base_mask['bias_mask'] is not None: + base_mask['bias_mask'][idx] = 0. + + if self.pruner.hook_id in self.pruner._fwd_hook_handles: + self.pruner.remove_activation_collector(self.pruner.hook_id) + + return base_mask + elif isinstance(wrapper.module, torch.nn.ConvTranspose2d): + apoz = self.get_channel_sum(wrapper, wrapper_idx) + if apoz is None: + # the collected activations are not enough + return None + if channel_masks is not None: + apoz = apoz * channel_masks + + prune_indices = torch.argsort(apoz)[:num_prune] + for idx in prune_indices: + base_mask['weight_mask'][:, idx] = 0. + if base_mask['bias_mask'] is not None: + base_mask['bias_mask'][idx] = 0. + + if self.pruner.hook_id in self.pruner._fwd_hook_handles: + self.pruner.remove_activation_collector(self.pruner.hook_id) + + return base_mask + + def _calc_filter_sen(self, activations, wrapper): + if isinstance(wrapper.module, torch.nn.Conv2d): + activations = torch.cat(activations, 0) + + # find next conv layer + cur_layer_name = '' + self.pruner._unwrap_model() + for name, module in self.model.named_modules(): + if module == wrapper.module: + cur_layer_name = name + break + if cur_layer_name == '': + raise NotImplementedError() + + print(cur_layer_name) + next_conv_layer = None + while True: + successors = self.pruner.graph.find_successors(cur_layer_name) + if len(successors) == 0: + break + + next_layer_name = successors[0] + next_layer = get_module(self.model, next_layer_name) + if isinstance(next_layer, torch.nn.Conv2d): + next_conv_layer = next_layer + break + + cur_layer_name = next_layer_name + + self.pruner._wrap_model() + + next_conv_layer_weight = next_conv_layer.weight.data + unfold_next_conv_layer_weight = next_conv_layer_weight.view((next_conv_layer_weight.size()[0], -1)) + unfold_activations = torch.nn.functional.unfold( + activations, + kernel_size=next_conv_layer.kernel_size, + stride=next_conv_layer.stride, + padding=next_conv_layer.padding, + dilation=next_conv_layer.dilation + ).to(next_conv_layer_weight.device) + + res = torch.zeros((activations.size()[0], next_conv_layer_weight.size()[1])) + + + print(unfold_next_conv_layer_weight.size(), unfold_activations.size()) + + for i in range(unfold_next_conv_layer_weight.size()[0]): + t = unfold_next_conv_layer_weight[i].unsqueeze(0).unsqueeze(-1) * unfold_activations + t = torch.max(t, dim=-1)[0] + t = t.view((activations.size()[0], next_conv_layer_weight.size()[1], -1)) + t = torch.max(t, dim=-1)[0] + t = t.cpu() + res = torch.max(res, t) + + res = torch.max(res, dim=0)[0] + return res + + elif isinstance(wrapper.module, torch.nn.ConvTranspose2d): + activations = torch.cat(activations, 0) + + # find next conv layer + cur_layer_name = '' + self.pruner._unwrap_model() + for name, module in self.model.named_modules(): + if module == wrapper.module: + cur_layer_name = name + break + if cur_layer_name == '': + raise NotImplementedError() + + print(cur_layer_name) + next_conv_layer = None + while True: + successors = self.pruner.graph.find_successors(cur_layer_name) + if len(successors) == 0: + break + + next_layer_name = successors[0] + next_layer = get_module(self.model, next_layer_name) + if isinstance(next_layer, torch.nn.Conv2d): + next_conv_layer = next_layer + break + + cur_layer_name = next_layer_name + + self.pruner._wrap_model() + + next_conv_layer_weight = next_conv_layer.weight.data + unfold_next_conv_layer_weight = next_conv_layer_weight.view((next_conv_layer_weight.size()[0], -1)) + unfold_activations = torch.nn.functional.unfold( + activations, + kernel_size=next_conv_layer.kernel_size, + stride=next_conv_layer.stride, + padding=next_conv_layer.padding, + dilation=next_conv_layer.dilation + ).to(next_conv_layer_weight.device) + + # res = torch.zeros((activations.size()[0], next_conv_layer_weight.size()[1])) + + # for i in range(unfold_next_conv_layer_weight.size()[0]): + # t = unfold_next_conv_layer_weight[i].unsqueeze(0).unsqueeze(-1) * unfold_activations + # t = torch.max(t, dim=-1)[0] + # t = t.view((activations.size()[0], next_conv_layer_weight.size()[1], -1)) + # t = torch.max(t, dim=-1)[0] + # t = t.cpu() + # res = torch.max(res, t) + + # res = torch.max(res, dim=0)[0] + + res = torch.zeros((activations.size()[0], next_conv_layer_weight.size()[0])) + + for i in range(unfold_next_conv_layer_weight.size()[1]): + print(unfold_next_conv_layer_weight.size(), unfold_activations.size()) + t = unfold_next_conv_layer_weight[:, i].unsqueeze(1).unsqueeze(-1) * unfold_activations + t = torch.max(t, dim=-1)[0] + t = t.view((activations.size()[0], next_conv_layer_weight.size()[0], -1)) + t = torch.max(t, dim=-1)[0] + t = t.cpu() + res = torch.max(res, t) + + res = torch.max(res, dim=0)[0] + return res + + def get_channel_sum(self, wrapper, wrapper_idx): + assert wrapper_idx is not None + activations = self.pruner.collected_activation[wrapper_idx] + if len(activations) < self.statistics_batch_num: + # collected activations is not enough + return None + return self._calc_filter_sen(activations, wrapper).to(wrapper.module.weight.device) \ No newline at end of file diff --git a/utils/third_party/nni/algorithms/compression/pytorch/pruning/weight_masker.py b/utils/third_party/nni/algorithms/compression/pytorch/pruning/weight_masker.py new file mode 100644 index 0000000000000000000000000000000000000000..aec8444ced37bccfa5f05e0a9f81b2843c2a35e4 --- /dev/null +++ b/utils/third_party/nni/algorithms/compression/pytorch/pruning/weight_masker.py @@ -0,0 +1,28 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +class WeightMasker(object): + def __init__(self, model, pruner, **kwargs): + self.model = model + self.pruner = pruner + + def calc_mask(self, sparsity, wrapper, wrapper_idx=None): + """ + Calculate the mask of given layer. + Parameters + ---------- + sparsity: float + pruning ratio, preserved weight ratio is `1 - sparsity` + wrapper: PrunerModuleWrapper + layer wrapper of this layer + wrapper_idx: int + index of this wrapper in pruner's all wrappers + Returns + ------- + dict + dictionary for storing masks, keys of the dict: + 'weight_mask': weight mask tensor + 'bias_mask': bias mask tensor (optional) + """ + + raise NotImplementedError('{} calc_mask is not implemented'.format(self.__class__.__name__)) diff --git a/utils/third_party/nni/algorithms/compression/pytorch/quantization/__init__.py b/utils/third_party/nni/algorithms/compression/pytorch/quantization/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..0e728dcd3f0817a529eb6801f3ab935de6751348 --- /dev/null +++ b/utils/third_party/nni/algorithms/compression/pytorch/quantization/__init__.py @@ -0,0 +1,4 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .quantizers import * diff --git a/utils/third_party/nni/algorithms/compression/pytorch/quantization/quantizers.py b/utils/third_party/nni/algorithms/compression/pytorch/quantization/quantizers.py new file mode 100644 index 0000000000000000000000000000000000000000..689b3c56b2262512f32b4018d40fe6b9fb0f0ed6 --- /dev/null +++ b/utils/third_party/nni/algorithms/compression/pytorch/quantization/quantizers.py @@ -0,0 +1,382 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +import copy +import torch +from schema import Schema, And, Or, Optional +from nni.compression.pytorch.utils.config_validation import CompressorSchema +from nni.compression.pytorch.compressor import Quantizer, QuantGrad, QuantType + +__all__ = ['NaiveQuantizer', 'QAT_Quantizer', 'DoReFaQuantizer', 'BNNQuantizer'] + +logger = logging.getLogger(__name__) + + +class NaiveQuantizer(Quantizer): + """quantize weight to 8 bits + """ + + def __init__(self, model, config_list, optimizer=None): + super().__init__(model, config_list, optimizer) + self.layer_scale = {} + + def validate_config(self, model, config_list): + schema = CompressorSchema([{ + Optional('quant_types'): ['weight'], + Optional('quant_bits'): Or(8, {'weight': 8}), + Optional('op_types'): [str], + Optional('op_names'): [str] + }], model, logger) + + schema.validate(config_list) + + def quantize_weight(self, wrapper, **kwargs): + weight = copy.deepcopy(wrapper.module.old_weight.data) + new_scale = weight.abs().max() / 127 + scale = max(self.layer_scale.get(wrapper.name, 0), new_scale) + self.layer_scale[wrapper.name] = scale + orig_type = weight.type() # TODO: user layer + weight = weight.div(scale).type(torch.int8).type(orig_type).mul(scale) + wrapper.module.weight = weight + return weight + +def update_ema(biased_ema, value, decay, step): + """ + calculate biased stat and unbiased stat in each step using exponential moving average method + + Parameters + ---------- + biased_ema : float + previous stat value + value : float + current stat value + decay : float + the weight of previous stat value, larger means smoother curve + step : int + current step + + Returns + ------- + float, float + """ + biased_ema = biased_ema * decay + (1 - decay) * value + unbiased_ema = biased_ema / (1 - decay ** step) # Bias correction + return biased_ema, unbiased_ema + + +def update_quantization_param(bits, rmin, rmax): + """ + calculate the `zero_point` and `scale`. + + Parameters + ---------- + bits : int + quantization bits length + rmin : float + min value of real value + rmax : float + max value of real value + + Returns + ------- + float, float + """ + # extend the [min, max] interval to ensure that it contains 0. + # Otherwise, we would not meet the requirement that 0 be an exactly + # representable value. + rmin = min(rmin, 0) + rmax = max(rmax, 0) + + # the min and max quantized values, as floating-point values + qmin = 0 + qmax = (1 << bits) - 1 + # First determine the scale. + scale = (rmax - rmin) / (qmax - qmin) + + # Zero-point computation. + initial_zero_point = qmin - rmin / scale + + # Now we need to nudge the zero point to be an integer + nudged_zero_point = 0 + if initial_zero_point < qmin: + nudged_zero_point = qmin + elif initial_zero_point > qmax: + nudged_zero_point = qmax + else: + nudged_zero_point = torch.round(initial_zero_point) + + return scale, nudged_zero_point + + +def get_bits_length(config, quant_type): + if isinstance(config["quant_bits"], int): + return config["quant_bits"] + else: + return config["quant_bits"].get(quant_type) + + +class QAT_Quantizer(Quantizer): + """Quantizer defined in: + Quantization and Training of Neural Networks for Efficient Integer-Arithmetic-Only Inference + http://openaccess.thecvf.com/content_cvpr_2018/papers/Jacob_Quantization_and_Training_CVPR_2018_paper.pdf + """ + + def __init__(self, model, config_list, optimizer=None): + """ + Parameters + ---------- + layer : LayerInfo + the layer to quantize + config_list : list of dict + list of configurations for quantization + supported keys for dict: + - quant_types : list of string + type of quantization you want to apply, currently support 'weight', 'input', 'output' + - quant_bits : int or dict of {str : int} + bits length of quantization, key is the quantization type, value is the length, eg. {'weight', 8}, + when the type is int, all quantization types share same bits length + - quant_start_step : int + disable quantization until model are run by certain number of steps, this allows the network to enter a more stable + state where activation quantization ranges do not exclude a significant fraction of values, default value is 0 + - op_types : list of string + types of nn.module you want to apply quantization, eg. 'Conv2d' + """ + super().__init__(model, config_list, optimizer) + self.steps = 1 + modules_to_compress = self.get_modules_to_compress() + for layer, config in modules_to_compress: + layer.module.register_buffer("zero_point", None) + layer.module.register_buffer("scale", None) + if "output" in config.get("quant_types", []): + layer.module.register_buffer('ema_decay', torch.Tensor([0.99])) + layer.module.register_buffer('tracked_min_biased', torch.zeros(1)) + layer.module.register_buffer('tracked_min', torch.zeros(1)) + layer.module.register_buffer('tracked_max_biased', torch.zeros(1)) + layer.module.register_buffer('tracked_max', torch.zeros(1)) + + def validate_config(self, model, config_list): + """ + Parameters + ---------- + model : torch.nn.Module + Model to be pruned + config_list : list of dict + List of configurations + """ + schema = CompressorSchema([{ + Optional('quant_types'): Schema([lambda x: x in ['weight', 'output']]), + Optional('quant_bits'): Or(And(int, lambda n: 0 < n < 32), Schema({ + Optional('weight'): And(int, lambda n: 0 < n < 32), + Optional('output'): And(int, lambda n: 0 < n < 32), + })), + Optional('quant_start_step'): And(int, lambda n: n >= 0), + Optional('op_types'): [str], + Optional('op_names'): [str] + }], model, logger) + + schema.validate(config_list) + + def _quantize(self, bits, op, real_val): + """ + quantize real value. + + Parameters + ---------- + bits : int + quantization bits length + op : torch.nn.Module + target module + real_val : float + real value to be quantized + + Returns + ------- + float + """ + transformed_val = op.zero_point + real_val / op.scale + qmin = 0 + qmax = (1 << bits) - 1 + clamped_val = torch.clamp(transformed_val, qmin, qmax) + quantized_val = torch.round(clamped_val) + return quantized_val + + def _dequantize(self, op, quantized_val): + """ + dequantize quantized value. + Because we simulate quantization in training process, all the computations still happen as float point computations, which means we + first quantize tensors then dequantize them. For more details, please refer to the paper. + + Parameters + ---------- + op : torch.nn.Module + target module + quantized_val : float + quantized_val value to be dequantized + + Returns + ------- + float + """ + real_val = op.scale * (quantized_val - op.zero_point) + return real_val + + def quantize_weight(self, wrapper, **kwargs): + config = wrapper.config + module = wrapper.module + weight = copy.deepcopy(wrapper.module.old_weight.data) + weight_bits = get_bits_length(config, 'weight') + quant_start_step = config.get('quant_start_step', 0) + assert weight_bits >= 1, "quant bits length should be at least 1" + + if quant_start_step > self.steps: + return weight + + # if bias exists, quantize bias to uint32 + if hasattr(wrapper.module, 'bias') and wrapper.module.bias is not None: + bias = wrapper.module.bias.data + bias_bits = 32 + rmin, rmax = torch.min(bias), torch.max(bias) + module.scale, module.zero_point = update_quantization_param(bias_bits, rmin, rmax) + bias = self._quantize(bias_bits, module, bias) + bias = self._dequantize(module, bias) + wrapper.module.bias.data = bias + + + # quantize weight + rmin, rmax = torch.min(weight), torch.max(weight) + module.scale, module.zero_point = update_quantization_param(weight_bits, rmin, rmax) + weight = self._quantize(weight_bits, module, weight) + weight = self._dequantize(module, weight) + wrapper.module.weight = weight + return weight + + def quantize_output(self, output, wrapper, **kwargs): + config = wrapper.config + module = wrapper.module + output_bits = get_bits_length(config, 'output') + quant_start_step = config.get('quant_start_step', 0) + assert output_bits >= 1, "quant bits length should be at least 1" + + if quant_start_step > self.steps: + return output + + current_min, current_max = torch.min(output), torch.max(output) + module.tracked_min_biased, module.tracked_min = update_ema(module.tracked_min_biased, current_min, + module.ema_decay, self.steps) + module.tracked_max_biased, module.tracked_max = update_ema(module.tracked_max_biased, current_max, + module.ema_decay, self.steps) + module.scale, module.zero_point = update_quantization_param(output_bits, module.tracked_min, module.tracked_max) + out = self._quantize(output_bits, module, output) + out = self._dequantize(module, out) + return out + + def fold_bn(self, config, **kwargs): + # TODO simulate folded weight + pass + + def step_with_optimizer(self): + """ + override `compressor` `step` method, quantization only happens after certain number of steps + """ + self.steps += 1 + + +class DoReFaQuantizer(Quantizer): + """Quantizer using the DoReFa scheme, as defined in: + Zhou et al., DoReFa-Net: Training Low Bitwidth Convolutional Neural Networks with Low Bitwidth Gradients + (https://arxiv.org/abs/1606.06160) + """ + + def __init__(self, model, config_list, optimizer=None): + super().__init__(model, config_list, optimizer) + + def validate_config(self, model, config_list): + """ + Parameters + ---------- + model : torch.nn.Module + Model to be pruned + config_list : list of dict + List of configurations + """ + schema = CompressorSchema([{ + Optional('quant_types'): Schema([lambda x: x in ['weight']]), + Optional('quant_bits'): Or(And(int, lambda n: 0 < n < 32), Schema({ + Optional('weight'): And(int, lambda n: 0 < n < 32) + })), + Optional('op_types'): [str], + Optional('op_names'): [str] + }], model, logger) + + schema.validate(config_list) + + def quantize_weight(self, wrapper, **kwargs): + weight = copy.deepcopy(wrapper.module.old_weight.data) + weight_bits = get_bits_length(wrapper.config, 'weight') + weight = weight.tanh() + weight = weight / (2 * weight.abs().max()) + 0.5 + weight = self.quantize(weight, weight_bits) + weight = 2 * weight - 1 + wrapper.module.weight = weight + # wrapper.module.weight.data = weight + return weight + + def quantize(self, input_ri, q_bits): + scale = pow(2, q_bits) - 1 + output = torch.round(input_ri * scale) / scale + return output + + +class ClipGrad(QuantGrad): + @staticmethod + def quant_backward(tensor, grad_output, quant_type): + if quant_type == QuantType.QUANT_OUTPUT: + grad_output[torch.abs(tensor) > 1] = 0 + return grad_output + + +class BNNQuantizer(Quantizer): + """Binarized Neural Networks, as defined in: + Binarized Neural Networks: Training Deep Neural Networks with Weights and Activations Constrained to +1 or -1 + (https://arxiv.org/abs/1602.02830) + """ + + def __init__(self, model, config_list, optimizer=None): + super().__init__(model, config_list, optimizer) + self.quant_grad = ClipGrad + + def validate_config(self, model, config_list): + """ + Parameters + ---------- + model : torch.nn.Module + Model to be pruned + config_list : list of dict + List of configurations + """ + schema = CompressorSchema([{ + Optional('quant_types'): Schema([lambda x: x in ['weight', 'output']]), + Optional('quant_bits'): Or(And(int, lambda n: 0 < n < 32), Schema({ + Optional('weight'): And(int, lambda n: 0 < n < 32), + Optional('output'): And(int, lambda n: 0 < n < 32), + })), + Optional('op_types'): [str], + Optional('op_names'): [str] + }], model, logger) + + schema.validate(config_list) + + def quantize_weight(self, wrapper, **kwargs): + weight = copy.deepcopy(wrapper.module.old_weight.data) + weight = torch.sign(weight) + # remove zeros + weight[weight == 0] = 1 + wrapper.module.weight = weight + return weight + + def quantize_output(self, output, wrapper, **kwargs): + out = torch.sign(output) + # remove zeros + out[out == 0] = 1 + return out diff --git a/utils/third_party/nni/algorithms/compression/tensorflow/__init__.py b/utils/third_party/nni/algorithms/compression/tensorflow/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/utils/third_party/nni/algorithms/compression/tensorflow/pruning/__init__.py b/utils/third_party/nni/algorithms/compression/tensorflow/pruning/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..f8ac8ea9b95a732cb581f24c7eebd8ba5739a3c6 --- /dev/null +++ b/utils/third_party/nni/algorithms/compression/tensorflow/pruning/__init__.py @@ -0,0 +1 @@ +from .one_shot import * diff --git a/utils/third_party/nni/algorithms/compression/tensorflow/pruning/one_shot.py b/utils/third_party/nni/algorithms/compression/tensorflow/pruning/one_shot.py new file mode 100644 index 0000000000000000000000000000000000000000..76dc1915ce1609dd083738f09a4973b66a63c47f --- /dev/null +++ b/utils/third_party/nni/algorithms/compression/tensorflow/pruning/one_shot.py @@ -0,0 +1,71 @@ +import tensorflow as tf + +from nni.compression.tensorflow import Pruner + +__all__ = [ + 'OneshotPruner', + 'LevelPruner', +] + +class OneshotPruner(Pruner): + def __init__(self, model, config_list, pruning_algorithm='level', **algo_kwargs): + super().__init__(model, config_list) + self.set_wrappers_attribute('calculated', False) + self.masker = MASKER_DICT[pruning_algorithm](model, self, **algo_kwargs) + + def validate_config(self, model, config_list): + pass # TODO + + def calc_masks(self, wrapper, wrapper_idx=None): + if wrapper.calculated: + return None + sparsity = wrapper.config['sparsity'] + masks = self.masker.calc_masks(sparsity, wrapper, wrapper_idx) + if masks is not None: + wrapper.calculated = True + return masks + + +class LevelPruner(OneshotPruner): + def __init__(self, model, config_list): + super().__init__(model, config_list, pruning_algorithm='level') + + +class WeightMasker: + def __init__(self, model, pruner, **kwargs): + self.model = model + self.pruner = pruner + + def calc_masks(self, sparsity, wrapper, wrapper_idx=None): + raise NotImplementedError() + + +class LevelPrunerMasker(WeightMasker): + def calc_masks(self, sparsity, wrapper, wrapper_idx=None): + masks = {} + for weight_variable in wrapper.layer.weights: + if 'bias' in weight_variable.name: + continue + + num_prune = int(tf.size(weight_variable).numpy() * sparsity) + if num_prune == 0: + continue + + weight = weight_variable.read_value() + if wrapper.masks.get(weight_variable.name) is not None: + weight = tf.math.multiply(weight, wrapper.masks[weight_variable.name]) + + w_abs = tf.math.abs(weight) + k = tf.size(weight) - num_prune + topk = tf.math.top_k(tf.reshape(w_abs, [-1]), k)[0] + if tf.size(topk) == 0: + mask = tf.zeros_like(weight) + else: + mask = tf.math.greater_equal(w_abs, topk[-1]) + masks[weight_variable.name] = tf.cast(mask, weight.dtype) + return masks + + +MASKER_DICT = { + 'level': LevelPrunerMasker, +} diff --git a/utils/third_party/nni/algorithms/feature_engineering/__init__.py b/utils/third_party/nni/algorithms/feature_engineering/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/utils/third_party/nni/algorithms/feature_engineering/gbdt_selector/__init__.py b/utils/third_party/nni/algorithms/feature_engineering/gbdt_selector/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..72970ab8565e11e62bae1242a4a8a00c06993cf7 --- /dev/null +++ b/utils/third_party/nni/algorithms/feature_engineering/gbdt_selector/__init__.py @@ -0,0 +1 @@ +from .gbdt_selector import GBDTSelector \ No newline at end of file diff --git a/utils/third_party/nni/algorithms/feature_engineering/gbdt_selector/gbdt_selector.py b/utils/third_party/nni/algorithms/feature_engineering/gbdt_selector/gbdt_selector.py new file mode 100644 index 0000000000000000000000000000000000000000..9ee09c25a38f1dca27fb4ab068bbceeaec69a3bd --- /dev/null +++ b/utils/third_party/nni/algorithms/feature_engineering/gbdt_selector/gbdt_selector.py @@ -0,0 +1,115 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# MIT License +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and +# associated documentation files (the "Software"), to deal in the Software without restriction, +# including without limitation the rights to use, copy, modify, merge, publish, distribute, +# sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or +# substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT +# NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT +# OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# ================================================================================================== + +""" +gbdt_selector.py including: + class GBDTSelector +""" + +import random +from sklearn.model_selection import train_test_split + +# pylint: disable=E0401 +import lightgbm as lgb + +from nni.feature_engineering.feature_selector import FeatureSelector + + +class GBDTSelector(FeatureSelector): + + def __init__(self, **kwargs): + self.selected_features_ = None + self.X = None + self.y = None + self.feature_importance = None + self.lgb_params = None + self.eval_ratio = None + self.early_stopping_rounds = None + self.importance_type = None + self.num_boost_round = None + self.model = None + + + def fit(self, X, y, **kwargs): + """ + Fit the training data to FeatureSelector + + Paramters + --------- + X : array-like numpy matrix + The training input samples, which shape is [n_samples, n_features]. + y : array-like numpy matrix + The target values (class labels in classification, real numbers in + regression). Which shape is [n_samples]. + lgb_params : dict + Parameters of lightgbm + eval_ratio : float + The ratio of data size. It's used for split the eval data and train data from self.X. + early_stopping_rounds : int + The early stopping setting in lightgbm. + importance_type : str + Supporting type is 'gain' or 'split'. + num_boost_round : int + num_boost_round in lightgbm. + """ + assert kwargs['lgb_params'] + assert kwargs['eval_ratio'] + assert kwargs['early_stopping_rounds'] + assert kwargs['importance_type'] + assert kwargs['num_boost_round'] + + self.X = X + self.y = y + self.lgb_params = kwargs['lgb_params'] + self.eval_ratio = kwargs['eval_ratio'] + self.early_stopping_rounds = kwargs['early_stopping_rounds'] + self.importance_type = kwargs['importance_type'] + self.num_boost_round = kwargs['num_boost_round'] + + X_train, X_test, y_train, y_test = train_test_split(self.X, + self.y, + test_size=self.eval_ratio, + random_state=random.seed(41)) + lgb_train = lgb.Dataset(X_train, y_train) + lgb_eval = lgb.Dataset(X_test, y_test, reference=lgb_train) + + self.model = lgb.train(self.lgb_params, + lgb_train, + num_boost_round=self.num_boost_round, + valid_sets=lgb_eval, + early_stopping_rounds=self.early_stopping_rounds) + + self.feature_importance = self.model.feature_importance(self.importance_type) + + + def get_selected_features(self, topk): + """ + Fit the training data to FeatureSelector + + Returns + ------- + list : + Return the index of imprtant feature. + """ + assert topk > 0 + + self.selected_features_ = self.feature_importance.argsort()[-topk:][::-1] + + return self.selected_features_ diff --git a/utils/third_party/nni/algorithms/feature_engineering/gbdt_selector/requirements.txt b/utils/third_party/nni/algorithms/feature_engineering/gbdt_selector/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..b73c05e89d156249906203b67c2e9f786d613738 --- /dev/null +++ b/utils/third_party/nni/algorithms/feature_engineering/gbdt_selector/requirements.txt @@ -0,0 +1 @@ +lightgbm \ No newline at end of file diff --git a/utils/third_party/nni/algorithms/feature_engineering/gradient_selector/__init__.py b/utils/third_party/nni/algorithms/feature_engineering/gradient_selector/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a43cb7578dcb38cbccc33fa11ab6bafc0a71b1fd --- /dev/null +++ b/utils/third_party/nni/algorithms/feature_engineering/gradient_selector/__init__.py @@ -0,0 +1 @@ +from .gradient_selector import FeatureGradientSelector \ No newline at end of file diff --git a/utils/third_party/nni/algorithms/feature_engineering/gradient_selector/constants.py b/utils/third_party/nni/algorithms/feature_engineering/gradient_selector/constants.py new file mode 100644 index 0000000000000000000000000000000000000000..0f70e2043af51bcf35c7514a81889203a9017ccb --- /dev/null +++ b/utils/third_party/nni/algorithms/feature_engineering/gradient_selector/constants.py @@ -0,0 +1,100 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# MIT License +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and +# associated documentation files (the "Software"), to deal in the Software without restriction, +# including without limitation the rights to use, copy, modify, merge, publish, distribute, +# sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or +# substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT +# NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT +# OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# ================================================================================================== + + +import numpy as np + + +class StorageLevel: + DISK = 'disk' + SPARSE = 'sparse' + DENSE = 'dense' + + +class DataFormat: + SVM = 'svm' + NUMPY = 'numpy' + ALL_FORMATS = [SVM, NUMPY] + + +class Preprocess: + """ + center the data to mean 0 and create unit variance + center the data to mean 0 + """ + ZSCORE = 'zscore' + CENTER = 'center' + + +class Device: + CUDA = 'cuda' + CPU = 'cpu' + + +class Checkpoint: + MODEL = 'model_state_dict' + OPT = 'optimizer_state_dict' + RNG = 'torch_rng_state' + + +class NanError(ValueError): + pass + + +class Initialization: + ZERO = 'zero' + ON = 'on' + OFF = 'off' + ON_HIGH = 'onhigh' + OFF_HIGH = 'offhigh' + SKLEARN = 'sklearn' + RANDOM = 'random' + VALUE_DICT = {ZERO: 0, + ON: 1, + OFF: -1, + ON_HIGH: 5, + OFF_HIGH: -1, + SKLEARN: None, + RANDOM: None} + + +class Coefficients: + """" + coefficients for sublinear estimator were computed running the sublinear + paper's authors' code + """ + SLE = {1: np.array([0.60355337]), + 2: np.array([1.52705001, -0.34841729]), + 3: np.array([2.90254224, -1.87216745, 0.]), + 4: np.array([4.63445685, -5.19936195, 0., 1.50391676]), + 5: np.array([6.92948049, -14.12216211, 9.4475009, 0., -1.21093546]), + 6: np.array([9.54431082, -28.09414643, 31.84703652, -11.18763791, -1.14175281, 0.]), + 7: np.array([12.54505041, -49.64891525, 79.78828031, -46.72250909, 0., 0., 5.02973646]), + 8: np.array([16.03550163, -84.286182, 196.86078756, -215.36747071, 92.63961263, 0., 0., -4.86280869]), + 9: np.array([19.86409184, -130.76801006, 390.95349861, -570.09210416, 354.77764899, 0., -73.84234865, 0., 10.09148767]), + 10: np.array([2.41117752e+01, -1.94946061e+02, 7.34214614e+02, -1.42851995e+03, 1.41567410e+03, \ + -5.81738134e+02, 0., 0., 3.11664751e+01, 1.05018365e+00]), + 11: np.array([28.75280839, -279.22576729, 1280.46325445, -3104.47148101, 3990.6092248, -2300.29413333, \ + 0., 427.35289033, 0., 0., -42.17587475]), + 12: np.array([33.85141912, -391.4229382, 2184.97827882, -6716.28280208, 11879.75233977, -11739.97267239, \ + 5384.94542245, 0., -674.23291712, 0., 0., 39.37456439])} + + +EPSILON = 1e-8 diff --git a/utils/third_party/nni/algorithms/feature_engineering/gradient_selector/fginitialize.py b/utils/third_party/nni/algorithms/feature_engineering/gradient_selector/fginitialize.py new file mode 100644 index 0000000000000000000000000000000000000000..6fe28ea5ee5494bc63a1dede2867ed3d2ec9f9a6 --- /dev/null +++ b/utils/third_party/nni/algorithms/feature_engineering/gradient_selector/fginitialize.py @@ -0,0 +1,611 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# MIT License +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and +# associated documentation files (the "Software"), to deal in the Software without restriction, +# including without limitation the rights to use, copy, modify, merge, publish, distribute, +# sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or +# substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT +# NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT +# OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# ================================================================================================== + + +import os +import pickle +import sys +import time + +import numpy as np +import scipy.sparse +from sklearn.datasets import load_svmlight_file + +import torch +from torch.utils.data import DataLoader, Dataset +# pylint: disable=E0611 +from torch.utils.data.dataloader import _SingleProcessDataLoaderIter, _MultiProcessingDataLoaderIter, _utils + +from . import constants +from . import syssettings + +torch.set_default_tensor_type(syssettings.torch.tensortype) +sparsetensor = syssettings.torch.sparse.tensortype + +BYTESPERREAL = 8. +BYTESPERGB = 1024. ** 3 + + +class PrepareData(Dataset): + + def __init__(self, + path_data=None, + data_format=constants.DataFormat.NUMPY, + D=None, N=None, + classification=True, + ordinal=False, + balanced=True, + preprocess=None, + n_to_estimate=None, + MAXMEMGB=syssettings.MAXMEMGB, + set_params=True, + path_mappings=None, + X=None, + y=None, + verbose=0, + n_classes=None, + device=constants.Device.CPU): + """ + Dataset class with helpful features and functions for being included in a dataloader + and managing memory usage. + can read following formats: + svm: svm light format (sklearn.datasets.load_svmlight_file) + numpy: Pass X and y as numpy or sparse arrays + + assumes + 1. if classification, y is in {-1, 1} or continuous and 0 indexed + 2. y can fit into memory + 3. consecutive calls to __getitem__() have consecutive idx values + + notes: + 1. this implementation is not careful wrt/ precise memory reqts. for + example, being able to store one dense row in memory is necessary, + but not sufficient. + 2. for y with 4.2 billion elements, 31.3 GB of memory is necessary + @ 8 bytes/scalar. Use partial fit to avoid loading the entire dataset + at once + 3. disk_size always refer to size of complete data file, even after + a split(). + + + Parameters + ---------- + path_data : str + Path to load data from + data_format : str + File ending for path data. + "numpy" is the default when passing in X and y + D : int + Number of features. + N : int + Number of rows. + classification : bool + If True, problem is classification, else regression. + ordinal: bool + If True, problem is ordinal classification. Requires classification to be True. + balanced : bool + If true, each class is weighted equally in optimization, otherwise + weighted is done via support of each class. Requires classification to be True. + prerocess : str + 'zscore' which refers to centering and normalizing data to unit variance or + 'center' which only centers the data to 0 mean + n_to_estimate : int + Number of rows of data to estimate + MAXMEMGB : float + Maximum allowable size for a minibatch + set_params : bool + Whether or not to determine the statistics of the dataset + path_mappings : str + Used when streaming from disk + X : array-like + Shape = [n_samples, n_features] + The training input samples. + y : array-like + Shape = [n_samples] + The target values (class labels in classification, real numbers in + regression). + verbose : int + Controls the verbosity when fitting. Set to 0 for no printing + 1 or higher for printing every verbose number of gradient steps. + device : str + 'cpu' to run on CPU and 'cuda' to run on GPU. Runs much faster on GPU + n_classes : int + number of classes + """ + + self.path_data = path_data + if self.path_data: + self.disk_size = os.path.getsize(path_data) + else: + assert X is not None, 'X must be specified if no path data' + self.disk_size = X.nbytes if not scipy.sparse.issparse( + X) else X.data.nbytes + assert data_format in constants.DataFormat.ALL_FORMATS, 'Format must in {0}.'.format( + ", ".join(constants.DataFormat.ALL_FORMATS)) + self.format = data_format + self.classification = classification + self.ordinal = ordinal + self.balanced = balanced + self.MAXMEMGB = MAXMEMGB + self.preprocess = preprocess + self.set_params = set_params + self.verbose = verbose + self.n_classes = n_classes + self.device = device + + self.path_data_stats = None + + if D is None: + assert self.disk_size / BYTESPERGB <= self.MAXMEMGB, \ + 'Cannot load data into memory. Supply D.' + + if self.format == constants.DataFormat.SVM: + self.X, self.y = load_svmlight_file(path_data) + elif self.format == constants.DataFormat.NUMPY: + assert X is not None, 'X must be specified in numpy mode' + assert y is not None, 'y must be specified in numpy mode' + self.X = X + self.y = y + if self.n_classes is None: + self.n_classes = np.unique(y).shape[0] + elif self.classification: + assert self.n_classes >= np.unique(y).shape[0], \ + 'n_classes given must be greater than or equal to the number of classes in y' + else: + raise NotImplementedError + self.y = torch.as_tensor(self.y, dtype=torch.get_default_dtype()) + + self.N, self.D = self.X.shape + + # assumes X was returned as a sparse array + self.storage_level = (constants.StorageLevel.SPARSE + if scipy.sparse.issparse(self.X) + else constants.StorageLevel.DENSE) + + else: + assert N is not None, 'Supply N.' + self.N, self.D = N, D + + # assume sparse matrix cannot fit into memory + self.storage_level = constants.StorageLevel.DISK + + self.dense_size_gb = self.get_dense_size() + + # check dense size + self.set_dense_X() + + self.max_rows = int(self.MAXMEMGB * BYTESPERGB / BYTESPERREAL / self.D) + assert self.max_rows, \ + 'Cannot fit one dense row into %d GB memory.' % self.MAXMEMGB + self.max_rows = self.max_batch_size() + sys.stdout.flush() + + if n_to_estimate is None: + self.n_to_estimate = self.max_batch_size() + else: + assert n_to_estimate <= self.N, 'n_to_estimate must be <= N.' + self.n_to_estimate = n_to_estimate + + # initialize disk loader + if self.storage_level == constants.StorageLevel.DISK and self.set_params: + if self.format == constants.DataFormat.SVM: + raise NotImplementedError( + 'Please use partial fit to train on datasets that do not fit in memory') + else: + raise NotImplementedError + + # TODO: use a passed-in RNG here + self.ix_statistics = np.random.permutation(self.N)[:self.n_to_estimate] + self.n_features = self.D + if self.set_params: + if self.verbose: + print('Finding data statistics...', end='') + sys.stdout.flush() + Xmn, sv1, Xsd, ymn, ysd = self.compute_data_stats() + self.set_data_stats(Xmn, sv1, Xsd, ymn, ysd) + if self.verbose: + print() + self.set_return_raw(False) + else: + self.set_return_raw(True) + + self.set_return_np(False) + + # this needs to occur after setting preprocessing params + if (self.storage_level == constants.StorageLevel.DISK and + self.format == constants.DataFormat.SVM and self.set_params): + self.loader.batchsize = 1 + + def get_dense_size(self): + return self.N * self.D * BYTESPERREAL / BYTESPERGB + + def set_dense_X(self): + if self.storage_level != constants.StorageLevel.DISK: + if self.dense_size_gb <= self.MAXMEMGB: + if self.storage_level == constants.StorageLevel.SPARSE: + self.X = self.X.toarray() + self.X = torch.as_tensor( + self.X, dtype=torch.get_default_dtype()) + self.storage_level = constants.StorageLevel.DENSE + + def set_return_np(self, boolean): + + self.return_np = boolean + + def set_return_raw(self, boolean): + + self.return_raw = boolean + + def save_data_stats(self, path_data_stats): + """ + Dumps dataset statistics to pickle file. + """ + + data_stats = { + 'Xmn': self.Xmn, + 'sv1': self.sv1, + 'Xsd': self.Xsd, + 'ymn': self.ymn, + 'ysd': self.ysd, + 'ix_statistics': self.ix_statistics, + } + pickle.dump(data_stats, open(path_data_stats, 'wb')) + + def load_data_stats(self, path_data_stats): + + stats = pickle.load(open(path_data_stats, 'rb')) + self.path_data_stats = path_data_stats + + self.set_data_stats(np.asarray(stats['Xmn']), stats['sv1'], + stats['Xsd'], stats['ymn'], stats['ysd']) + + if self.storage_level == constants.StorageLevel.DISK and hasattr( + self, 'path_mappings'): + if 'ix_statistics' in stats: + self.ix_statistics = stats['ix_statistics'] + else: + self.ix_statistics = range(self.N) + + self.set_return_raw(False) + + def reset(self): + """ + Resets the dataloader. Only implemented for disk StorageLevel. + """ + + if self.storage_level == constants.StorageLevel.DENSE: + pass + elif self.storage_level == constants.StorageLevel.SPARSE: + pass + elif self.storage_level == constants.StorageLevel.DISK: + if self.format == constants.DataFormat.SVM: + self.loader.reset() + else: + raise NotImplementedError + + def todense(self): + + assert hasattr(self, 'Xmn'), 'Set preprocess params first.' + assert len(self) <= self.max_batch_size( + ), 'N must be <= max_batch_size().' + + with torch.no_grad(): + dense, _ = self.split(range(len(self))) + Braw = self.return_raw + Bnp = self.return_np + self.set_return_raw(True) + self.set_return_np(True) + dense.X, dense.y = [], [] + + def f_Xy(X, y): + dense.X.append(X) + dense.y.append(y) + self.apply(f_Xy=f_Xy) + dense.X = dense.X[-1] + dense.y = dense.y[-1] + self.set_return_raw(Braw) + self.set_return_np(Bnp) + dense.storage_level = constants.StorageLevel.DENSE + + return dense + + def split(self, ix): + + assert hasattr(self, 'Xmn'), 'Run set_preprocess_params() first.' + + first = type(self)( + self.path_data, + self.format, + self.D, + N=len(ix), + classification=self.classification, + preprocess=self.preprocess, + n_to_estimate=None, + MAXMEMGB=self.MAXMEMGB, + set_params=False) + second = type(self)( + self.path_data, + self.format, + self.D, + N=self.N - len(ix), + classification=self.classification, + preprocess=self.preprocess, + n_to_estimate=None, + MAXMEMGB=self.MAXMEMGB, + set_params=False) + + first.storage_level = self.storage_level + second.storage_level = self.storage_level + + # copy preprocess params + if not self.classification: + first.ymn = self.ymn + second.ymn = self.ymn + first.ysd = self.ysd + second.ysd = self.ysd + + first.Xmn = self.Xmn + second.Xmn = self.Xmn + first.sv1 = self.sv1 + second.sv1 = self.sv1 + + if self.storage_level == constants.StorageLevel.DISK: + if self.format == constants.DataFormat.SVM: + first.Xsd = self.Xsd + second.Xsd = self.Xsd + else: + raise NotImplementedError + + # initialize data structures + if self.storage_level == constants.StorageLevel.DISK: + if self.format == constants.DataFormat.SVM: + raise NotImplementedError + raise NotImplementedError + elif self.storage_level in [constants.StorageLevel.SPARSE, + constants.StorageLevel.DENSE]: + first.X, first.y = self.X[ix], self.y[ix] + ixsec = list(set(range(self.N)).difference(set(ix))) + second.X, second.y = self.X[ixsec], self.y[ixsec] + + return first, second + + @staticmethod + def sparse_std(X, X_mean): + """ + Calculate the column wise standard deviations of a sparse matrix. + """ + X_copy = X.copy() + X_copy.data **= 2 # square non zero elements + E_x_squared = np.array(X_copy.mean(axis=0)).ravel() + Xsd = np.sqrt(E_x_squared - X_mean**2) + return Xsd + + def compute_data_stats(self): + """ + 1. computes/estimates feature means + 2. if preprocess == 'zscore', computes/estimates feature standard devs + 3. if not classification, computes/estimates target mean/standard dev + 4. estimates largest singular value of data matrix + """ + t = time.time() + X, y = self.X[self.ix_statistics], self.y[self.ix_statistics] + preprocess = self.preprocess + classification = self.classification + + Xmn = (X.mean(dim=0) + if not scipy.sparse.issparse(X) + else np.array(X.mean(axis=0)).ravel()) + + if preprocess == constants.Preprocess.ZSCORE: + Xsd = (X.std(dim=0) + if not scipy.sparse.issparse(X) + else PrepareData.sparse_std(X, Xmn)) + Xsd[Xsd == 0] = 1. + else: + Xsd = 1. + + if preprocess is not None and preprocess: + if preprocess == constants.Preprocess.ZSCORE: + Xc = (X - Xmn) / Xsd + else: + Xc = X - Xmn + else: + Xc = X - Xmn + + sv1 = scipy.sparse.linalg.svds(Xc / ( + torch.sqrt(torch.prod(torch.as_tensor(y.size(), dtype=torch.get_default_dtype()))) + if not scipy.sparse.issparse(X) else y.numpy().size), + k=1, + which='LM', + return_singular_vectors=False) + # avoid runaway sv1 + sv1 = np.array([min(np.finfo(np.float32).max, + sv1[0])]) + + if not classification: + ymn = y.mean() + ysd = y.std() + else: + # TODO: set these, for each class? + ymn = 0. + ysd = 1. + if self.verbose: + print(" computing data statistics took: ", time.time() - t) + + return Xmn, sv1, Xsd, ymn, ysd + + + def set_data_stats(self, Xmn, sv1, Xsd=1., ymn=0., ysd=1.): + """ + Saves dataset stats to self to be used for preprocessing. + """ + + self.Xmn = torch.as_tensor( + Xmn, dtype=torch.get_default_dtype()).to(self.device) + self.sv1 = torch.as_tensor( + sv1, dtype=torch.get_default_dtype()).to(self.device) + self.Xsd = torch.as_tensor( + Xsd, dtype=torch.get_default_dtype()).to(self.device) + self.ymn = torch.as_tensor( + ymn, dtype=torch.get_default_dtype()).to(self.device) + self.ysd = torch.as_tensor( + ysd, dtype=torch.get_default_dtype()).to(self.device) + + + def apply_preprocess(self, X, y): + """ + Faster on gpu device, while dataloading takes up a large portion of the time. + """ + + with torch.no_grad(): + if not self.classification: + y = (y.reshape((-1, 1)) - self.ymn) / self.ysd + else: + y = y.reshape((-1, 1)) + X = (X - self.Xmn) / self.sv1 + + if self.preprocess == constants.Preprocess.ZSCORE: + X /= self.Xsd + + return X, y + + + def max_batch_size(self): + """ + Return the maximum batchsize for the dataset. + """ + + return int(np.min([self.max_rows, self.N])) + + + def apply(self, ix_rows=None, ix_cols=None, f_Xy=None): + + if f_Xy is None: + return + + if ix_rows is None: + ix_rows = range(self.N) + + if ix_cols is None: + ix_cols = range(self.n_features) + + f_Xy((self.X[ix_rows, ix_cols] + if not self.storage_level == constants.StorageLevel.SPARSE + else self.X[ix_rows, ix_cols].toarray()), self.y[ix_rows]) + + + def get_dense_data(self, ix_cols=None, ix_rows=None): + + if ix_cols is None: + ix_cols = range(self.n_features) + + X = [np.zeros((0, len(ix_cols)))] + y = [np.zeros((0, 1))] + Bnp = self.return_np + + def f_Xy(Xb, yb, n): + X[-1] = np.concatenate((X[-1], Xb), axis=0) + y[-1] = np.concatenate((y[-1], yb), axis=0) + self.apply(f_Xy=f_Xy, ix_rows=ix_rows, ix_cols=ix_cols) + self.set_return_np(Bnp) + + return X[-1], y[-1] + + + def __len__(self): + + return self.N + + + def getXy(self, idx): + + if self.storage_level == constants.StorageLevel.DENSE: + X, y = self.X[idx], self.y[idx] + elif self.storage_level == constants.StorageLevel.SPARSE: + # assume subset can fit into memory even if whole matrix cant + X, y = self.X[idx].toarray(), self.y[idx] + else: + raise NotImplementedError + + return X, y + + + def __getitem__(self, idx): + + with torch.no_grad(): + X, y = self.getXy(idx) + X = X.toarray() if scipy.sparse.issparse(X) else X + + X = torch.as_tensor( + X, dtype=torch.get_default_dtype()).to(self.device) + y = torch.as_tensor( + y, dtype=torch.get_default_dtype()).to(self.device) + + if not self.return_raw: + X, y = self.apply_preprocess(X, y) + + if self.classification and ( + self.n_classes is None or self.n_classes == 2): + y[y == 0] = -1 + + if self.return_np: + if constants.Device.CPU not in self.device: + X = X.cpu() + y = y.cpu() + X = X.numpy() + y = y.numpy() + return X, y + + return X, y + + +class ChunkDataLoader(DataLoader): + """ + DataLoader class used to more quickly load a batch of indices at once. + """ + + def __iter__(self): + return _ChunkDataLoaderIter(self) + + +class _ChunkDataLoaderIter: + """ + DataLoaderIter class used to more quickly load a batch of indices at once. + """ + def __init__(self, dataloader): + if dataloader.num_workers == 0: + self.iter = _SingleProcessDataLoaderIter(dataloader) + else: + self.iter = _MultiProcessingDataLoaderIter(dataloader) + + def __next__(self): + # only chunk that is edited from base + if self.iter._num_workers == 0: # same-process loading + indices = next(self.iter._sampler_iter) # may raise StopIteration + if len(indices) > 1: + batch = self.iter._dataset[np.array(indices)] + else: + batch = self.iter._collate_fn([self.iter._dataset[i] for i in indices]) + + if self.iter._pin_memory: + batch = _utils.pin_memory.pin_memory_batch(batch) + return batch + else: + return next(self.iter) diff --git a/utils/third_party/nni/algorithms/feature_engineering/gradient_selector/fgtrain.py b/utils/third_party/nni/algorithms/feature_engineering/gradient_selector/fgtrain.py new file mode 100644 index 0000000000000000000000000000000000000000..377d72691613b93911e2bbc164e3c677ac31f574 --- /dev/null +++ b/utils/third_party/nni/algorithms/feature_engineering/gradient_selector/fgtrain.py @@ -0,0 +1,228 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# MIT License +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and +# associated documentation files (the "Software"), to deal in the Software without restriction, +# including without limitation the rights to use, copy, modify, merge, publish, distribute, +# sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or +# substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT +# NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT +# OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# ================================================================================================== + + +import time + +import numpy as np +import torch +from sklearn.feature_selection import SelectKBest, \ + f_classif, mutual_info_classif, f_regression, mutual_info_regression + +from . import constants +from . import syssettings +from .learnability import Solver +from .utils import EMA + +torch.set_default_tensor_type(syssettings.torch.tensortype) + + +def get_optim_f_stop(maxiter, maxtime, dftol_stop, freltol_stop, + minibatch=True): + """ + Check stopping conditions. + """ + + discount_factor = 1. / 3 + + total_t = [0.] + df_store = [np.nan] + it_store = [0] + relchange_store = [np.nan] + f_ma = EMA(discount_factor=discount_factor) + df_ma = EMA(discount_factor=discount_factor) + + def f_stop(f0, v0, it, t): + + flag_stop = False + + total_t[-1] += t + g = f0.x.grad.clone().cpu().detach() + df = g.abs().max().numpy().squeeze() + v = v0.clone().cpu().detach() + f = v.numpy().squeeze() + + if it >= maxiter: + flag_stop = True + + elif total_t[-1] >= maxtime: + flag_stop = True + + f_ma.update(f) + df_ma.update(df) + rel_change = f_ma.relchange() + + if ((not minibatch) and (df < dftol_stop)) \ + or (minibatch and (df_ma() < dftol_stop)): + flag_stop = True + + if rel_change < freltol_stop: + flag_stop = True + + if not minibatch: + df_store[-1] = df + else: + df_store[-1] = df_ma() + relchange_store[-1] = rel_change + it_store[-1] = it + + return flag_stop + + return f_stop, {'t': total_t, 'it': it_store, 'df': df_store, + 'relchange': relchange_store} + + +def get_init(data_train, init_type='on', rng=np.random.RandomState(0), prev_score=None): + """ + Initialize the 'x' variable with different settings + """ + + D = data_train.n_features + value_off = constants.Initialization.VALUE_DICT[ + constants.Initialization.OFF] + value_on = constants.Initialization.VALUE_DICT[ + constants.Initialization.ON] + + if prev_score is not None: + x0 = prev_score + elif not isinstance(init_type, str): + x0 = value_off * np.ones(D) + x0[init_type] = value_on + elif init_type.startswith(constants.Initialization.RANDOM): + d = int(init_type.replace(constants.Initialization.RANDOM, '')) + x0 = value_off * np.ones(D) + x0[rng.permutation(D)[:d]] = value_on + elif init_type == constants.Initialization.SKLEARN: + B = data_train.return_raw + X, y = data_train.get_dense_data() + data_train.set_return_raw(B) + ix = train_sk_dense(init_type, X, y, data_train.classification) + x0 = value_off * np.ones(D) + x0[ix] = value_on + elif init_type in constants.Initialization.VALUE_DICT: + x0 = constants.Initialization.VALUE_DICT[init_type] * np.ones(D) + else: + raise NotImplementedError( + 'init_type {0} not supported yet'.format(init_type)) + # pylint: disable=E1102 + return torch.tensor(x0.reshape((-1, 1)), + dtype=torch.get_default_dtype()) + + +def get_checkpoint(S, stop_conds, rng=None, get_state=True): + """ + Save the necessary information into a dictionary + """ + + m = {} + m['ninitfeats'] = S.ninitfeats + m['x0'] = S.x0 + x = S.x.clone().cpu().detach() + m['feats'] = np.where(x.numpy() >= 0)[0] + m.update({k: v[0] for k, v in stop_conds.items()}) + if get_state: + m.update({constants.Checkpoint.MODEL: S.state_dict(), + constants.Checkpoint.OPT: S.opt_train.state_dict(), + constants.Checkpoint.RNG: torch.get_rng_state(), + }) + if rng: + m.update({'rng_state': rng.get_state()}) + + return m + + +def _train(data_train, Nminibatch, order, C, rng, lr_train, debug, maxiter, + maxtime, init, dftol_stop, freltol_stop, dn_log, accum_steps, + path_save, shuffle, device=constants.Device.CPU, + verbose=1, + prev_checkpoint=None, + groups=None, + soft_groups=None): + """ + Main training loop. + """ + + t_init = time.time() + + x0 = get_init(data_train, init, rng) + if isinstance(init, str) and init == constants.Initialization.ZERO: + ninitfeats = -1 + else: + ninitfeats = np.where(x0.detach().numpy() > 0)[0].size + + S = Solver(data_train, order, + Nminibatch=Nminibatch, x0=x0, C=C, + ftransform=lambda x: torch.sigmoid(2 * x), + get_train_opt=lambda p: torch.optim.Adam(p, lr_train), + rng=rng, + accum_steps=accum_steps, + shuffle=shuffle, + groups=groups, + soft_groups=soft_groups, + device=device, + verbose=verbose) + S = S.to(device) + + S.ninitfeats = ninitfeats + S.x0 = x0 + + if prev_checkpoint: + S.load_state_dict(prev_checkpoint[constants.Checkpoint.MODEL]) + S.opt_train.load_state_dict(prev_checkpoint[constants.Checkpoint.OPT]) + torch.set_rng_state(prev_checkpoint[constants.Checkpoint.RNG]) + + minibatch = S.Ntrain != S.Nminibatch + + f_stop, stop_conds = get_optim_f_stop(maxiter, maxtime, dftol_stop, + freltol_stop, minibatch=minibatch) + + if debug: + pass + else: + f_callback = None + stop_conds['t'][-1] = time.time() - t_init + + S.train(f_stop=f_stop, f_callback=f_callback) + + return get_checkpoint(S, stop_conds, rng), S + + +def train_sk_dense(ty, X, y, classification): + if classification: + if ty.startswith('skf'): + d = int(ty.replace('skf', '')) + f_sk = f_classif + elif ty.startswith('skmi'): + d = int(ty.replace('skmi', '')) + f_sk = mutual_info_classif + else: + if ty.startswith('skf'): + d = int(ty.replace('skf', '')) + f_sk = f_regression + elif ty.startswith('skmi'): + d = int(ty.replace('skmi', '')) + f_sk = mutual_info_regression + t = time.time() + clf = SelectKBest(f_sk, k=d) + clf.fit_transform(X, y.squeeze()) + ix = np.argsort(-clf.scores_) + ix = ix[np.where(np.invert(np.isnan(clf.scores_[ix])))[0]][:d] + t = time.time() - t + return {'feats': ix, 't': t} diff --git a/utils/third_party/nni/algorithms/feature_engineering/gradient_selector/gradient_selector.py b/utils/third_party/nni/algorithms/feature_engineering/gradient_selector/gradient_selector.py new file mode 100644 index 0000000000000000000000000000000000000000..f7cb69f627442cb8b749f792cecfa0fea7526e1b --- /dev/null +++ b/utils/third_party/nni/algorithms/feature_engineering/gradient_selector/gradient_selector.py @@ -0,0 +1,631 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# MIT License +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and +# associated documentation files (the "Software"), to deal in the Software without restriction, +# including without limitation the rights to use, copy, modify, merge, publish, distribute, +# sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or +# substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT +# NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT +# OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# ================================================================================================== + +import time + +import numpy as np +import pandas as pd + +from sklearn.base import BaseEstimator +from sklearn.feature_selection import SelectorMixin +from sklearn.utils.validation import check_is_fitted + +import torch + +from nni.feature_engineering.feature_selector import FeatureSelector +from . import constants +from .fginitialize import PrepareData +from .fgtrain import _train + + +class FeatureGradientSelector(FeatureSelector, BaseEstimator, SelectorMixin): + def __init__(self, + order=4, + penalty=1, + n_features=None, + max_features=None, + learning_rate=1e-1, + init='zero', + n_epochs=1, + shuffle=True, + batch_size=1000, + target_batch_size=1000, + max_time=np.inf, + classification=True, + ordinal=False, + balanced=True, + preprocess='zscore', + soft_grouping=False, + verbose=0, + device='cpu'): + """ + FeatureGradientSelector is a class that selects features for a machine + learning model using a gradient based search. + + Parameters + ---------- + order : int + What order of interactions to include. Higher orders + may be more accurate but increase the run time. 12 is the maximum allowed order. + penatly : int + Constant that multiplies the regularization term. + n_features: int + If None, will automatically choose number of features based on search. + Otherwise, number of top features to select. + max_features : int + If not None, will use the 'elbow method' to determine the number of features + with max_features as the upper limit. + learning_rate : float + init : str + How to initialize the vector of scores. 'zero' is the default. + Options: {'zero', 'on', 'off', 'onhigh', 'offhigh', 'sklearn'} + n_epochs : int + number of epochs to run + shuffle : bool + Shuffle "rows" prior to an epoch. + batch_size : int + Nnumber of "rows" to process at a time + target_batch_size : int + Number of "rows" to accumulate gradients over. + Useful when many rows will not fit into memory but are needed for accurate estimation. + classification : bool + If True, problem is classification, else regression. + ordinal : bool + If True, problem is ordinal classification. Requires classification to be True. + balanced : bool + If true, each class is weighted equally in optimization, otherwise + weighted is done via support of each class. Requires classification to be True. + prerocess : str + 'zscore' which refers to centering and normalizing data to unit variance or + 'center' which only centers the data to 0 mean + soft_grouping : bool + if True, groups represent features that come from the same source. + Used to encourage sparsity of groups and features within groups. + verbose : int + Controls the verbosity when fitting. Set to 0 for no printing + 1 or higher for printing every verbose number of gradient steps. + device : str + 'cpu' to run on CPU and 'cuda' to run on GPU. Runs much faster on GPU + """ + assert order <= 12 and order >= 1, 'order must be an integer between 1 and 12, inclusive' + assert n_features is None or max_features is None, \ + 'only specify one of n_features and max_features at a time' + + self.order = order + self.penalty = penalty + self.n_features = n_features + self.max_features = max_features + self.learning_rate = learning_rate + self.init = init + self.n_epochs = n_epochs + self.shuffle = shuffle + self.batch_size = batch_size + self.target_batch_size = target_batch_size + self.max_time = max_time + self.dftol_stop = -1 + self.freltol_stop = -1 + self.classification = classification + self.ordinal = ordinal + self.balanced = balanced + self.preprocess = preprocess + self.soft_grouping = soft_grouping + self.verbose = verbose + self.device = device + + self.model_ = None + self.scores_ = None + self._prev_checkpoint = None + self._data_train = None + + def partial_fit(self, X, y, + n_classes=None, + groups=None): + """ + Select Features via a gradient based search on (X, y) on the given samples. + Can be called repeatedly with different X and y to handle streaming datasets. + + Parameters + ---------- + X : array-like + Shape = [n_samples, n_features] + The training input samples. + y : array-like + Shape = [n_samples] + The target values (class labels in classification, real numbers in + regression). + n_classes : int + Number of classes + Classes across all calls to partial_fit. + Can be obtained by via `np.unique(y_all).shape[0]`, where y_all is the + target vector of the entire dataset. + This argument is expected for the first call to partial_fit, + otherwise will assume all classes are present in the batch of y given. + It will be ignored in the subsequent calls. + Note that y doesn't need to contain all labels in `classes`. + groups : array-like + Optional, shape = [n_features] + Groups of columns that must be selected as a unit + e.g. [0, 0, 1, 2] specifies the first two columns are part of a group. + This argument is expected for the first call to partial_fit, + otherwise will assume all classes are present in the batch of y given. + It will be ignored in the subsequent calls. + """ + try: + self._partial_fit(X, y, n_classes=n_classes, groups=groups) + except constants.NanError: + if hasattr(self, '_prev_checkpoint'): + # if it's already done some batches successfully just ignore it + print('failed fitting this batch, loss was nan') + else: + # if this is the first batch, reset and try with doubles + if self.verbose: + print('Loss was nan, trying with Doubles') + self._reset() + torch.set_default_tensor_type(torch.DoubleTensor) + self._partial_fit(X, y, n_classes=n_classes, groups=groups) + + return self + + def _partial_fit(self, X, y, n_classes=None, groups=None): + """ + Private function for partial_fit to enable trying floats before doubles. + """ + # pass in X and y in chunks + if hasattr(self, '_data_train'): + # just overwrite the X and y from the new chunk but make them tensors + # keep dataset stats from previous + self._data_train.X = X.values if isinstance(X, pd.DataFrame) else X + self._data_train.N, self._data_train.D = self._data_train.X.shape + self._data_train.dense_size_gb = self._data_train.get_dense_size() + self._data_train.set_dense_X() + + self._data_train.y = y.values if isinstance(y, pd.Series) else y + self._data_train.y = torch.as_tensor( + y, dtype=torch.get_default_dtype()) + else: + data_train = self._prepare_data(X, y, n_classes=n_classes) + self._data_train = data_train + + batch_size, _, accum_steps, max_iter = self._set_batch_size( + self._data_train) + + rng = None # not used + debug = 0 # {0,1} print messages and do other stuff? + dn_logs = None # tensorboard logs; only specify if debug=1 + path_save = None # intermediate models saves; only specify if debug=1 + m, solver = _train(self._data_train, + batch_size, + self.order, + self.penalty, + rng, + self.learning_rate, + debug, + max_iter, + self.max_time, + self.init, + self.dftol_stop, + self.freltol_stop, + dn_logs, + accum_steps, + path_save, + self.shuffle, + device=self.device, + verbose=self.verbose, + prev_checkpoint=self._prev_checkpoint if hasattr( + self, '_prev_checkpoint') else None, + groups=groups if not self.soft_grouping else None, + soft_groups=groups if self.soft_grouping else None) + + self._prev_checkpoint = m + self._process_results(m, solver, X, groups=groups) + return self + + def fit(self, X, y, + groups=None): + """ + Select Features via a gradient based search on (X, y). + + Parameters + ---------- + X : array-like + Shape = [n_samples, n_features] + The training input samples. + y : array-like + Shape = [n_samples] + The target values (class labels in classification, real numbers in + regression). + groups : array-like + Optional, shape = [n_features] + Groups of columns that must be selected as a unit + e.g. [0, 0, 1, 2] specifies the first two columns are part of a group. + """ + try: + self._fit(X, y, groups=groups) + except constants.NanError: + if self.verbose: + print('Loss was nan, trying with Doubles') + torch.set_default_tensor_type(torch.DoubleTensor) + self._fit(X, y, groups=groups) + return self + + def get_selected_features(self): + return self.selected_features_ + + def _prepare_data(self, X, y, n_classes=None): + """ + Returns a PrepareData object. + """ + return PrepareData(X=X.values if isinstance(X, pd.DataFrame) else X, + y=y.values if isinstance(y, pd.Series) else y, + data_format=constants.DataFormat.NUMPY, + classification=int(self.classification), + ordinal=self.ordinal, + balanced=self.balanced, + preprocess=self.preprocess, + verbose=self.verbose, + device=self.device, + n_classes=n_classes) + + def _fit(self, X, y, groups=None): + """ + Private function for fit to enable trying floats before doubles. + """ + data_train = self._prepare_data(X, y) + + batch_size, _, accum_steps, max_iter = self._set_batch_size( + data_train) + + rng = None # not used + debug = 0 # {0,1} print messages and log to tensorboard + dn_logs = None # tensorboard logs; only specify if debug=1 + path_save = None # intermediate models saves; only specify if debug=1 + m, solver = _train(data_train, + batch_size, + self.order, + self.penalty, + rng, + self.learning_rate, + debug, + max_iter, + self.max_time, + self.init, + self.dftol_stop, + self.freltol_stop, + dn_logs, + accum_steps, + path_save, + self.shuffle, + device=self.device, + verbose=self.verbose, + groups=groups if not self.soft_grouping else None, + soft_groups=groups if self.soft_grouping else None) + + self._process_results(m, solver, X, groups=groups) + return self + + def _process_torch_scores(self, scores): + """ + Convert scores into flat numpy arrays. + """ + if constants.Device.CUDA in scores.device.type: + scores = scores.cpu() + return scores.numpy().ravel() + + def _set_batch_size(self, data_train): + """ + Ensures that batch_size is less than the number of rows. + """ + batch_size = min(self.batch_size, data_train.N) + target_batch_size = min(max( + self.batch_size, self.target_batch_size), data_train.N) + accum_steps = max(int(np.ceil(target_batch_size / self.batch_size)), 1) + max_iter = self.n_epochs * (data_train.N // batch_size) + return batch_size, target_batch_size, accum_steps, max_iter + + def _process_results(self, m, solver, X, groups=None): + """ + Process the results of a run into something suitable for transform(). + """ + self.scores_ = self._process_torch_scores( + torch.sigmoid(m[constants.Checkpoint.MODEL]['x'] * 2)) + if self.max_features: + self.max_features = min([self.max_features, self.scores_.shape[0]]) + n_features = self._recommend_number_features(solver) + self.set_n_features(n_features, groups=groups) + elif self.n_features: + self.set_n_features(self.n_features, groups=groups) + else: + self.selected_features_ = m['feats'] + + # subtract elapsed time from max_time + self.max_time -= m['t'] + + self.model_ = m + + return self + + def transform(self, X): + """ + Returns selected features from X. + + Paramters + --------- + X: array-like + Shape = [n_samples, n_features] + The training input samples. + """ + + self._get_support_mask() + if self.selected_features_.shape[0] == 0: + raise ValueError( + 'No Features selected, consider lowering the penalty or specifying n_features') + return (X.iloc[:, self.selected_features_] + if isinstance(X, pd.DataFrame) + else X[:, self.selected_features_]) + + def get_support(self, indices=False): + """ + Get a mask, or integer index, of the features selected. + + Parameters + ---------- + indices : bool + Default False + If True, the return value will be an array of integers, rather than a boolean mask. + + Returns + ------- + list : + returns support: An index that selects the retained features from a feature vector. + If indices is False, this is a boolean array of shape [# input features], + in which an element is True iff its corresponding feature is selected for retention. + If indices is True, this is an integer array of shape [# output features] whose values + are indices into the input feature vector. + """ + self._get_support_mask() + if indices: + return self.selected_features_ + + mask = np.zeros_like(self.scores_, dtype=bool) + # pylint: disable=E1137 + mask[self.selected_features_] = True + return mask + + def inverse_transform(self, X): + """ + Returns transformed X to the original number of column. + This operation is lossy and all columns not in the transformed data + will be returned as columns of 0s. + """ + self._get_support_mask() + X_new = np.zeros((X.shape[0], self.scores_.shape[0])) + X_new[self.selected_features_] = X + return X_new + + def get_params(self, deep=True): + """ + Get parameters for this estimator. + """ + params = self.__dict__ + params = {key: val for (key, val) in params.items() + if not key.endswith('_')} + return params + + def set_params(self, **params): + """ + Set the parameters of this estimator. + """ + for param in params: + if hasattr(self, param): + setattr(self, param, params[param]) + return self + + def fit_transform(self, X, y): + """ + Select features and then return X with the selected features. + + Parameters + ---------- + X : array-like + Shape = [n_samples, n_features] + The training input samples. + y : array-like + Shape = [n_samples] + The target values (class labels in classification, real numbers in + regression). + """ + self.fit(X, y) + return self.transform(X) + + def _get_support_mask(self): + """ + Check if it is fitted. + """ + check_is_fitted(self, 'scores_') + + def _generate_scores(self, solver, xsub, ysub, step_size, feature_order): + """ + Generate forward passes to determine the number of features when max_features is set. + """ + scores = [] + for i in np.arange(1, self.max_features + 1, step_size): + # optimization possible since xsub is growing? + i = int(np.ceil(i)) + # pylint: disable=E1102 + score = solver.f_train(torch.tensor(np.ones(i), + dtype=torch.get_default_dtype() + ).unsqueeze(1).to(self.device), + xsub[:, feature_order[:i]], + ysub) + if constants.Device.CUDA in score.device.type: + score = score.cpu() + # score.numpy()[0][0] + scores.append(score) + return scores + + def set_n_features(self, n, groups=None): + """ + Set the number of features to return after fitting. + """ + self._get_support_mask() + self.n_features = n + return self._set_top_features(groups=groups) + + def _set_top_features(self, groups=None): + """ + Set the selected features after a run. + + With groups, ensures that if any member of a group is selected, all members are selected + """ + self._get_support_mask() + assert self.n_features <= self.scores_.shape[0], \ + 'n_features must be less than or equal to the number of columns in X' + # pylint: disable=E1130 + self.selected_features_ = np.argpartition( + self.scores_, -self.n_features)[-self.n_features:] + if groups is not None and not self.soft_grouping: + selected_feature_set = set(self.selected_features_.tolist()) + for _ in np.unique(groups): + group_members = np.where(groups == groups)[0].tolist() + if selected_feature_set.intersection(group_members): + selected_feature_set.update(group_members) + self.selected_features_ = np.array(list(selected_feature_set)) + self.selected_features_ = np.sort(self.selected_features_) + return self + + def set_top_percentile(self, percentile, groups=None): + """ + Set the percentile of features to return after fitting. + """ + self._get_support_mask() + assert percentile <= 1 and percentile >= 0, \ + 'percentile must between 0 and 1 inclusive' + self.n_features = int(self.scores_.shape[0] * percentile) + return self._set_top_features(groups=groups) + + def _recommend_number_features(self, solver, max_time=None): + """ + Get the recommended number of features by doing forward passes when max_features is set. + """ + max_time = max_time if max_time else self.max_time + if max_time < 0: + max_time = 60 # allow 1 minute extra if we already spent max_time + MAX_FORWARD_PASS = 200 + MAX_FULL_BATCHES = 3 # the forward passes can take longer than the fitting + # if we allow a full epoch of data to be included. By only doing 3 full batches at most + # we get enough accuracy without increasing the time too much. This + # constant may not be optimal + accum_steps = solver.accum_steps + step_size = max(self.max_features / MAX_FORWARD_PASS, 1) + # pylint: disable=E1130 + feature_order = np.argsort(-self.scores_) # note the negative + t = time.time() + + dataloader_iterator = iter(solver.ds_train) + full_scores = [] + # keep_going = True + with torch.no_grad(): + # might want to only consider a batch valid if there are at least + # two classes + for _ in range(accum_steps * MAX_FULL_BATCHES): + scores = [] + try: + xsub, ysub = next(dataloader_iterator) + except StopIteration: + # done with epoch, don't do more than one epoch + break + except Exception as e: + print(e) + break + if max_time and time.time() - t > max_time: + if self.verbose: + print( + "Stoppinn forward passes because they reached max_time: ", + max_time) + if not full_scores: + # no forward passes worked, return half of max_features + return self.max_features // 2 + break + if solver.multiclass: + for target_class in range(solver.n_classes): + ysub_binary = solver.transform_y_into_binary( + ysub, target_class) + scaling_value = solver._get_scaling_value( + ysub, target_class) + if not solver._skip_y_forward(ysub_binary): + scores = self._generate_scores( + solver, xsub, ysub_binary, step_size, feature_order) + # one row will represent one class that is present in the data + # all classes are weighted equally + full_scores.append( + [score * scaling_value for score in scores]) + else: + if not solver._skip_y_forward(ysub): + scores = self._generate_scores( + solver, xsub, ysub, step_size, feature_order) + full_scores.append(scores) + best_index = FeatureGradientSelector._find_best_index_elbow( + full_scores) + if self.verbose: + print("Forward passes took: ", time.time() - t) + # account for step size and off by one (n_features is 1 indexed, not 0 + # ) + return int( + np.ceil( + np.arange( + 1, + self.max_features + + 1, + step_size))[best_index]) + + @staticmethod + def _find_best_index_elbow(full_scores): + """ + Finds the point on the curve that maximizes distance from the line determined by the endpoints. + """ + scores = pd.DataFrame(full_scores).mean(0).values.tolist() + first_point = np.array([0, scores[0]]) + last_point = np.array([len(scores) - 1, scores[-1]]) + elbow_metric = [] + for i in range(len(scores)): + elbow_metric.append( + FeatureGradientSelector._distance_to_line( + first_point, last_point, np.array([i, scores[i]]))) + return np.argmax(elbow_metric) + + @staticmethod + def _distance_to_line(start_point, end_point, new_point): + """ + Calculates the shortest distance from new_point to the line determined by start_point and end_point. + """ + # for calculating elbow method + return np.cross(new_point - start_point, + end_point - start_point) / np.linalg.norm( + end_point - start_point) + + def _reset(self): + """ + Reset the estimator by deleting all private and fit parameters. + """ + params = self.__dict__ + for key, _ in params.items(): + if key.endswith('_') or key.startswith('_'): + delattr(self, key) + return self diff --git a/utils/third_party/nni/algorithms/feature_engineering/gradient_selector/learnability.py b/utils/third_party/nni/algorithms/feature_engineering/gradient_selector/learnability.py new file mode 100644 index 0000000000000000000000000000000000000000..3a0ab4b39e7bef15101c793f3ec522395c05cd94 --- /dev/null +++ b/utils/third_party/nni/algorithms/feature_engineering/gradient_selector/learnability.py @@ -0,0 +1,534 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# MIT License +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and +# associated documentation files (the "Software"), to deal in the Software without restriction, +# including without limitation the rights to use, copy, modify, merge, publish, distribute, +# sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or +# substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT +# NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT +# OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# ================================================================================================== + +import time + +import numpy as np +import scipy.special +import torch +import torch.nn as nn + +from . import constants +from . import syssettings +from .fginitialize import ChunkDataLoader + +torch.set_default_tensor_type(syssettings.torch.tensortype) +sparsetensor = syssettings.torch.sparse.tensortype + + +def def_train_opt(p): + """ + Return the default optimizer. + """ + return torch.optim.Adam(p, 1e-1, amsgrad=False) + + +def revcumsum(U): + """ + Reverse cumulative sum for faster performance. + """ + return U.flip(dims=[0]).cumsum(dim=0).flip(dims=[0]) + + +def triudr(X, r): + + Zr = torch.zeros_like(X, requires_grad=False) + U = X * r + Zr[:-1] = X[:-1] * revcumsum(U)[1:] + + return Zr + + +def triudl(X, l): + + Zl = torch.zeros_like(X, requires_grad=False) + U = X * l + Zl[1:] = X[1:] * (U.cumsum(dim=0)[:-1]) + + return Zl + + +class ramp(torch.autograd.Function): + """ + Ensures input is between 0 and 1 + """ + + @staticmethod + def forward(ctx, input_data): + ctx.save_for_backward(input_data) + return input_data.clamp(min=0, max=1) + + + @staticmethod + def backward(ctx, grad_output): + input_data, = ctx.saved_tensors + grad_input = grad_output.clone() + grad_input[input_data < 0] = 1e-2 + grad_input[input_data > 1] = -1e-2 + return grad_input + + +class safesqrt(torch.autograd.Function): + """ + Square root without dividing by 0. + """ + @staticmethod + def forward(ctx, input_data): + o = input_data.sqrt() + ctx.save_for_backward(input_data, o) + return o + + + @staticmethod + def backward(ctx, grad_output): + _, o = ctx.saved_tensors + grad_input = grad_output.clone() + grad_input *= 0.5 / (o + constants.EPSILON) + return grad_input + + +class LearnabilityMB(nn.Module): + """ + Calculates the learnability of a set of features. + mini-batch version w/ "left" and "right" multiplies + """ + + + def __init__(self, Nminibatch, D, coeff, groups=None, binary=False, + device=constants.Device.CPU): + super(LearnabilityMB, self).__init__() + + a = coeff / scipy.special.binom(Nminibatch, np.arange(coeff.size) + 2) + self.order = a.size + # pylint: disable=E1102 + self.a = torch.tensor(a, dtype=torch.get_default_dtype(), requires_grad=False) + self.binary = binary + + self.a = self.a.to(device) + + + def ret_val(self, z): + """ + Get the return value based on z. + """ + + if not self.binary: + return 1 - z + + else: + return 0.5 * (1 - safesqrt.apply(ramp.apply(z))) + + + def forward(self, s, X, y): + + l = y.clone() + r = y.clone() + z = 0 + + for i in range(self.order): + if i % 2 == 0: + Z = triudr(X, r) + r = torch.mm(Z, s) + else: + Z = triudl(X, l) + l = torch.mm(Z, s) + if self.a[i] != 0: + # same the computation if a[i] is 0 + p = torch.mm(l.t(), r) + z += self.a[i] * p + return self.ret_val(z) + + +class Solver(nn.Module): + """ + Class that performs the main optimization. + Keeps track of the current x and iterates through data to learn x given the penalty and order. + """ + + def __init__(self, + PreparedData, + order, + Nminibatch=None, + groups=None, + soft_groups=None, + x0=None, + C=1, + ftransform=torch.sigmoid, + get_train_opt=def_train_opt, + accum_steps=1, + rng=np.random.RandomState(0), + max_norm_clip=1., + shuffle=True, + device=constants.Device.CPU, + verbose=1): + """ + + Parameters + ---------- + PreparedData : Dataset of PrepareData class + order : int + What order of interactions to include. Higher orders + may be more accurate but increase the run time. 12 is the maximum allowed order. + Nminibatch : int + Number of rows in a mini batch + groups : array-like + Optional, shape = [n_features] + Groups of columns that must be selected as a unit + e.g. [0, 0, 1, 2] specifies the first two columns are part of a group. + soft_groups : array-like + optional, shape = [n_features] + Groups of columns come from the same source + Used to encourage sparsity of number of sources selected + e.g. [0, 0, 1, 2] specifies the first two columns are part of a group. + x0 : torch.tensor + Optional, initialization of x. + C : float + Penalty parameter. + get_train_opt : function + Function that returns a pytorch optimizer, Adam is the default + accum_steps : int + Number of steps + rng : random state + max_norm_clip : float + Maximum allowable size of the gradient + shuffle : bool + Whether or not to shuffle data within the dataloader + order : int + What order of interactions to include. Higher orders + may be more accurate but increase the run time. 12 is the maximum allowed order. + penalty : int + Constant that multiplies the regularization term. + ftransform : function + Function to transform the x. sigmoid is the default. + device : str + 'cpu' to run on CPU and 'cuda' to run on GPU. Runs much faster on GPU + verbose : int + Controls the verbosity when fitting. Set to 0 for no printing + 1 or higher for printing every verbose number of gradient steps. + """ + super(Solver, self).__init__() + + self.Ntrain, self.D = PreparedData.N, PreparedData.n_features + if groups is not None: + # pylint: disable=E1102 + groups = torch.tensor(groups, dtype=torch.long) + self.groups = groups + else: + self.groups = None + if soft_groups is not None: + # pylint: disable=E1102 + soft_groups = torch.tensor(soft_groups, dtype=torch.long) + self.soft_D = torch.unique(soft_groups).size()[0] + else: + self.soft_D = None + self.soft_groups = soft_groups + + if Nminibatch is None: + Nminibatch = self.Ntrain + else: + if Nminibatch > self.Ntrain: + print('Minibatch larger than sample size.' + + (' Reducing from %d to %d.' + % (Nminibatch, self.Ntrain))) + Nminibatch = self.Ntrain + if Nminibatch > PreparedData.max_rows: + print('Minibatch larger than mem-allowed.' + + (' Reducing from %d to %d.' % (Nminibatch, + PreparedData.max_rows))) + Nminibatch = int(np.min([Nminibatch, PreparedData.max_rows])) + self.Nminibatch = Nminibatch + self.accum_steps = accum_steps + + if x0 is None: + x0 = torch.zeros(self.D, 1, dtype=torch.get_default_dtype()) + self.ftransform = ftransform + self.x = nn.Parameter(x0) + self.max_norm = max_norm_clip + + self.device = device + self.verbose = verbose + + self.multiclass = PreparedData.classification and PreparedData.n_classes and PreparedData.n_classes > 2 + if self.multiclass: + self.n_classes = PreparedData.n_classes + else: + self.n_classes = None + # whether to treat all classes equally + self.balanced = PreparedData.balanced + self.ordinal = PreparedData.ordinal + + if (hasattr(PreparedData, 'mappings') + or PreparedData.storage_level == 'disk'): + num_workers = PreparedData.num_workers + elif PreparedData.storage_level == constants.StorageLevel.DENSE: + num_workers = 0 + else: + num_workers = 0 + + if constants.Device.CUDA in device: + pin_memory = False + else: + pin_memory = False + + if num_workers == 0: + timeout = 0 + else: + timeout = 60 + + self.ds_train = ChunkDataLoader( + PreparedData, + batch_size=self.Nminibatch, + shuffle=shuffle, + drop_last=True, + num_workers=num_workers, + pin_memory=pin_memory, + timeout=timeout) + self.f_train = LearnabilityMB(self.Nminibatch, self.D, + constants.Coefficients.SLE[order], + self.groups, + binary=PreparedData.classification, + device=self.device) + self.opt_train = get_train_opt(torch.nn.ParameterList([self.x])) + self.it = 0 + self.iters_per_epoch = int(np.ceil(len(self.ds_train.dataset) + / self.ds_train.batch_size)) + self.f_train = self.f_train.to(device) + # pylint: disable=E1102 + self.w = torch.tensor( + C / (C + 1), + dtype=torch.get_default_dtype(), requires_grad=False) + self.w = self.w.to(device) + + + def penalty(self, s): + """ + Calculate L1 Penalty. + """ + to_return = torch.sum(s) / self.D + if self.soft_groups is not None: + # if soft_groups, there is an additional penalty for using more + # groups + s_grouped = torch.zeros(self.soft_D, 1, + dtype=torch.get_default_dtype(), + device=self.device) + for group in torch.unique(self.soft_groups): + # groups should be indexed 0 to n_group - 1 + # TODO: consider other functions here + s_grouped[group] = s[self.soft_groups == group].max() + # each component of the penalty contributes .5 + # TODO: could make this a user given parameter + to_return = (to_return + torch.sum(s_grouped) / self.soft_D) * .5 + return to_return + + + def forward_and_backward(self, s, xsub, ysub, retain_graph=False): + """ + Completes the forward operation and computes gradients for learnability and penalty. + """ + f_train = self.f_train(s, xsub, ysub) + pen = self.penalty(s).unsqueeze(0).unsqueeze(0) + # pylint: disable=E1102 + grad_outputs = torch.tensor([[1]], dtype=torch.get_default_dtype(), + device=self.device) + g1, = torch.autograd.grad([f_train], [self.x], grad_outputs, + retain_graph=True) + # pylint: disable=E1102 + grad_outputs = torch.tensor([[1]], dtype=torch.get_default_dtype(), + device=self.device) + g2, = torch.autograd.grad([pen], [self.x], grad_outputs, + retain_graph=retain_graph) + return f_train, pen, g1, g2 + + + def combine_gradient(self, g1, g2): + """ + Combine gradients from learnability and penalty + + Parameters + ---------- + g1 : array-like + gradient from learnability + g2 : array-like + gradient from penalty + """ + to_return = ((1 - self.w) * g1 + self.w * g2) / self.accum_steps + if self.groups is not None: + # each column will get a gradient + # but we can only up or down groups, so the gradient for the group + # should be the average of the gradients of the columns + to_return_grouped = torch.zeros_like(self.x) + for group in torch.unique(self.groups): + to_return_grouped[self.groups == + group] = to_return[self.groups == group].mean() + to_return = to_return_grouped + return to_return + + + def combine_loss(self, f_train, pen): + """ + Combine the learnability and L1 penalty. + """ + return ((1 - self.w) * f_train.detach() + self.w * pen.detach()) \ + / self.accum_steps + + + def transform_y_into_binary(self, ysub, target_class): + """ + Transforms multiclass classification problems into a binary classification problem. + """ + with torch.no_grad(): + ysub_binary = torch.zeros_like(ysub) + if self.ordinal: + # turn ordinal problems into n-1 classifications of is this + # example less than rank k + if target_class == 0: + return None + + ysub_binary[ysub >= target_class] = 1 + ysub_binary[ysub < target_class] = -1 + else: + # turn multiclass problems into n binary classifications + ysub_binary[ysub == target_class] = 1 + ysub_binary[ysub != target_class] = -1 + return ysub_binary + + + def _get_scaling_value(self, ysub, target_class): + """ + Returns the weight given to a class for multiclass classification. + """ + if self.balanced: + if self.ordinal: + return 1 / (torch.unique(ysub).size()[0] - 1) + + return 1 / torch.unique(ysub).size()[0] + else: + if self.ordinal: + this_class_proportion = torch.mean(ysub >= target_class) + normalizing_constant = 0 + for i in range(1, self.n_classes): + normalizing_constant += torch.mean(ysub >= i) + return this_class_proportion / normalizing_constant + else: + return torch.mean(ysub == target_class) + + + def _skip_y_forward(self, y): + """ + Returns boolean of whether to skip the currrent y if there is nothing to be learned from it. + """ + if y is None: + return True + elif torch.unique(y).size()[0] < 2: + return True + else: + return False + + + def train(self, f_callback=None, f_stop=None): + """ + Trains the estimator to determine which features to include. + + Parameters + ---------- + f_callback : function + Function that performs a callback + f_stop: function + Function that tells you when to stop + """ + + t = time.time() + h = torch.zeros([1, 1], dtype=torch.get_default_dtype()) + h = h.to(self.device) + # h_complete is so when we divide by the number of classes + # we only do that for that minibatch if accumulating + h_complete = h.clone() + flag_stop = False + dataloader_iterator = iter(self.ds_train) + self.x.grad = torch.zeros_like(self.x) + while not flag_stop: + try: + xsub, ysub = next(dataloader_iterator) + except StopIteration: + dataloader_iterator = iter(self.ds_train) + xsub, ysub = next(dataloader_iterator) + try: + s = self.ftransform(self.x) + s = s.to(self.device) + if self.multiclass: + # accumulate gradients over each class, classes range from + # 0 to n_classes - 1 + #num_classes_batch = torch.unique(ysub).size()[0] + for target_class in range(self.n_classes): + ysub_binary = self.transform_y_into_binary( + ysub, target_class) + if self._skip_y_forward(ysub_binary): + continue + # should should skip if target class is not included + # but that changes what we divide by + scaling_value = self._get_scaling_value( + ysub, target_class) + f_train, pen, g1, g2 = self.forward_and_backward( + s, xsub, ysub_binary, retain_graph=True) + self.x.grad += self.combine_gradient( + g1, g2) * scaling_value + h += self.combine_loss(f_train, + pen) * scaling_value + else: + if not self._skip_y_forward(ysub): + f_train, pen, g1, g2 = self.forward_and_backward( + s, xsub, ysub) + self.x.grad += self.combine_gradient(g1, g2) + h += self.combine_loss(f_train, pen) + else: + continue + h_complete += h + self.it += 1 + if torch.isnan(h): + raise constants.NanError( + 'Loss is nan, something may be misconfigured') + if self.it % self.accum_steps == 0: + torch.nn.utils.clip_grad_norm_( + torch.nn.ParameterList([self.x]), + max_norm=self.max_norm) + self.opt_train.step() + + t = time.time() - t + if f_stop is not None: + flag_stop = f_stop(self, h, self.it, t) + + if f_callback is not None: + f_callback(self, h, self.it, t) + elif self.verbose and (self.it // self.accum_steps) % self.verbose == 0: + epoch = int(self.it / self.iters_per_epoch) + print( + '[Minibatch: %6d/ Epoch: %3d/ t: %3.3f s] Loss: %0.3f' % + (self.it, epoch, t, h_complete / self.accum_steps)) + + if flag_stop: + break + + self.opt_train.zero_grad() + h = 0 + h_complete = 0 + t = time.time() + except KeyboardInterrupt: + flag_stop = True + break diff --git a/utils/third_party/nni/algorithms/feature_engineering/gradient_selector/requirements.txt b/utils/third_party/nni/algorithms/feature_engineering/gradient_selector/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..7e2873b558c30216e17b584570b7383623b2931c --- /dev/null +++ b/utils/third_party/nni/algorithms/feature_engineering/gradient_selector/requirements.txt @@ -0,0 +1,4 @@ +numpy==1.14.3 +scikit-learn>=0.23.2 +scipy==1.1.0 +torch==1.1.0 diff --git a/utils/third_party/nni/algorithms/feature_engineering/gradient_selector/syssettings.py b/utils/third_party/nni/algorithms/feature_engineering/gradient_selector/syssettings.py new file mode 100644 index 0000000000000000000000000000000000000000..df864b316601464a9f35a7b463933b1f05a9fe3f --- /dev/null +++ b/utils/third_party/nni/algorithms/feature_engineering/gradient_selector/syssettings.py @@ -0,0 +1,29 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# MIT License +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and +# associated documentation files (the "Software"), to deal in the Software without restriction, +# including without limitation the rights to use, copy, modify, merge, publish, distribute, +# sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or +# substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT +# NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT +# OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# ================================================================================================== + + +import torch + +# pytorch +torch.tensortype = torch.FloatTensor +torch.sparse.tensortype = torch.sparse.FloatTensor + +# mem +MAXMEMGB = 10 diff --git a/utils/third_party/nni/algorithms/feature_engineering/gradient_selector/utils.py b/utils/third_party/nni/algorithms/feature_engineering/gradient_selector/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..0ab9b09a25cfb328586c39f00ab9546388b4f8d4 --- /dev/null +++ b/utils/third_party/nni/algorithms/feature_engineering/gradient_selector/utils.py @@ -0,0 +1,78 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# MIT License +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and +# associated documentation files (the "Software"), to deal in the Software without restriction, +# including without limitation the rights to use, copy, modify, merge, publish, distribute, +# sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or +# substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT +# NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT +# OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# ================================================================================================== + + +import numpy as np + +class EMA(): + """ + maintains an exponential moving average + """ + + def __init__(self, f=np.nan, discount_factor=0.1, valid_after=None, + n_iters_relchange=3): + + self.f_ma = [f] + self.fs = [f] + self.gamma = discount_factor + self.rel_change = [np.nan] + if valid_after is None: + self.valid_after = int(1/discount_factor) + else: + self.valid_after = valid_after + self.n_iters_relchange = n_iters_relchange + self.initialized = False + + def reset(self, f): + + self.f_ma = [f] + self.fs = [f] + self.rel_change = [np.nan] + self.initialized = True + + def relchange(self): + + if self.num_updates() > np.max([self.valid_after, + self.n_iters_relchange]): + return np.max(self.rel_change[-self.n_iters_relchange:]) + else: + return np.nan + + def update(self, f_new): + + if not self.initialized: + self.reset(f_new) + else: + self.fs.append(f_new) + self.f_ma.append(self.f_ma[-1]*(1-self.gamma) + self.gamma*f_new) + if self.num_updates() > self.valid_after: + self.rel_change.append(np.abs((self.f_ma[-1]-self.f_ma[-2]) + / self.f_ma[-2])) + + def num_updates(self): + + return len(self.f_ma) + + def __call__(self): + + if self.num_updates() > self.valid_after: + return self.f_ma[-1] + else: + return np.nan diff --git a/utils/third_party/nni/algorithms/hpo/__init__.py b/utils/third_party/nni/algorithms/hpo/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/utils/third_party/nni/algorithms/hpo/batch_tuner/__init__.py b/utils/third_party/nni/algorithms/hpo/batch_tuner/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/utils/third_party/nni/algorithms/hpo/batch_tuner/batch_tuner.py b/utils/third_party/nni/algorithms/hpo/batch_tuner/batch_tuner.py new file mode 100644 index 0000000000000000000000000000000000000000..4f73fce9453161a8f30783f915873ba8e5bb8f31 --- /dev/null +++ b/utils/third_party/nni/algorithms/hpo/batch_tuner/batch_tuner.py @@ -0,0 +1,131 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +batch_tuner.py including: + class BatchTuner +""" + +import logging + +import nni +from nni.tuner import Tuner + +TYPE = '_type' +CHOICE = 'choice' +VALUE = '_value' + +LOGGER = logging.getLogger('batch_tuner_AutoML') + +class BatchTuner(Tuner): + """ + BatchTuner is tuner will running all the configure that user want to run batchly. + + Examples + -------- + The search space only be accepted like: + + :: + + {'combine_params': + { '_type': 'choice', + '_value': '[{...}, {...}, {...}]', + } + } + + """ + + def __init__(self): + self._count = -1 + self._values = [] + + def is_valid(self, search_space): + """ + Check the search space is valid: only contains 'choice' type + + Parameters + ---------- + search_space : dict + + Returns + ------- + None or list + If valid, return candidate values; else return None. + """ + if not len(search_space) == 1: + raise RuntimeError('BatchTuner only supprt one combined-paramreters key.') + + for param in search_space: + param_type = search_space[param][TYPE] + if not param_type == CHOICE: + raise RuntimeError('BatchTuner only supprt \ + one combined-paramreters type is choice.') + + if isinstance(search_space[param][VALUE], list): + return search_space[param][VALUE] + + raise RuntimeError('The combined-paramreters \ + value in BatchTuner is not a list.') + return None + + def update_search_space(self, search_space): + """Update the search space + + Parameters + ---------- + search_space : dict + """ + self._values = self.is_valid(search_space) + + def generate_parameters(self, parameter_id, **kwargs): + """Returns a dict of trial (hyper-)parameters, as a serializable object. + + Parameters + ---------- + parameter_id : int + + Returns + ------- + dict + A candidate parameter group. + """ + self._count += 1 + if self._count > len(self._values) - 1: + raise nni.NoMoreTrialError('no more parameters now.') + return self._values[self._count] + + def receive_trial_result(self, parameter_id, parameters, value, **kwargs): + pass + + def import_data(self, data): + """Import additional data for tuning + + Parameters + ---------- + data: + a list of dictionarys, each of which has at least two keys, 'parameter' and 'value' + """ + if not self._values: + LOGGER.info("Search space has not been initialized, skip this data import") + return + + self._values = self._values[(self._count+1):] + self._count = -1 + + _completed_num = 0 + for trial_info in data: + LOGGER .info("Importing data, current processing \ + progress %s / %s", _completed_num, len(data)) + # simply validate data format + assert "parameter" in trial_info + _params = trial_info["parameter"] + assert "value" in trial_info + _value = trial_info['value'] + if not _value: + LOGGER.info("Useless trial data, value is %s, skip this trial data.", _value) + continue + _completed_num += 1 + if _params in self._values: + self._values.remove(_params) + LOGGER .info("Successfully import data to batch tuner, \ + total data: %d, imported data: %d.", len(data), _completed_num) diff --git a/utils/third_party/nni/algorithms/hpo/bohb_advisor/__init__.py b/utils/third_party/nni/algorithms/hpo/bohb_advisor/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/utils/third_party/nni/algorithms/hpo/bohb_advisor/bohb_advisor.py b/utils/third_party/nni/algorithms/hpo/bohb_advisor/bohb_advisor.py new file mode 100644 index 0000000000000000000000000000000000000000..73687abc5c71145e95d8623b64ed82284c9063e3 --- /dev/null +++ b/utils/third_party/nni/algorithms/hpo/bohb_advisor/bohb_advisor.py @@ -0,0 +1,676 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +''' +bohb_advisor.py +''' + +import sys +import math +import logging +import json_tricks +from schema import Schema, Optional +import ConfigSpace as CS +import ConfigSpace.hyperparameters as CSH + +from nni import ClassArgsValidator +from nni.runtime.protocol import CommandType, send +from nni.runtime.msg_dispatcher_base import MsgDispatcherBase +from nni.utils import OptimizeMode, MetricType, extract_scalar_reward +from nni.runtime.common import multi_phase_enabled + +from .config_generator import CG_BOHB + +logger = logging.getLogger('BOHB_Advisor') + +_next_parameter_id = 0 +_KEY = 'TRIAL_BUDGET' +_epsilon = 1e-6 + + +def create_parameter_id(): + """Create an id + + Returns + ------- + int + parameter id + """ + global _next_parameter_id + _next_parameter_id += 1 + return _next_parameter_id - 1 + + +def create_bracket_parameter_id(brackets_id, brackets_curr_decay, increased_id=-1): + """Create a full id for a specific bracket's hyperparameter configuration + + Parameters + ---------- + brackets_id: int + brackets id + brackets_curr_decay: int + brackets curr decay + increased_id: int + increased id + Returns + ------- + int + params id + """ + if increased_id == -1: + increased_id = str(create_parameter_id()) + params_id = '_'.join([str(brackets_id), + str(brackets_curr_decay), + increased_id]) + return params_id + + +class Bracket: + """ + A bracket in BOHB, all the information of a bracket is managed by + an instance of this class. + + Parameters + ---------- + s: int + The current Successive Halving iteration index. + s_max: int + total number of Successive Halving iterations + eta: float + In each iteration, a complete run of sequential halving is executed. In it, + after evaluating each configuration on the same subset size, only a fraction of + 1/eta of them 'advances' to the next round. + max_budget : float + The largest budget to consider. Needs to be larger than min_budget! + The budgets will be geometrically distributed + :math:`a^2 + b^2 = c^2 \\sim \\eta^k` for :math:`k\\in [0, 1, ... , num\\_subsets - 1]`. + optimize_mode: str + optimize mode, 'maximize' or 'minimize' + """ + def __init__(self, s, s_max, eta, max_budget, optimize_mode): + self.s = s + self.s_max = s_max + self.eta = eta + self.max_budget = max_budget + self.optimize_mode = OptimizeMode(optimize_mode) + + self.n = math.ceil((s_max + 1) * eta**s / (s + 1) - _epsilon) + self.r = max_budget / eta**s + self.i = 0 + self.hyper_configs = [] # [ {id: params}, {}, ... ] + self.configs_perf = [] # [ {id: [seq, acc]}, {}, ... ] + self.num_configs_to_run = [] # [ n, n, n, ... ] + self.num_finished_configs = [] # [ n, n, n, ... ] + self.no_more_trial = False + + def is_completed(self): + """check whether this bracket has sent out all the hyperparameter configurations""" + return self.no_more_trial + + def get_n_r(self): + """return the values of n and r for the next round""" + return math.floor(self.n / self.eta**self.i + _epsilon), math.floor(self.r * self.eta**self.i +_epsilon) + + def increase_i(self): + """i means the ith round. Increase i by 1""" + self.i += 1 + + def set_config_perf(self, i, parameter_id, seq, value): + """update trial's latest result with its sequence number, e.g., epoch number or batch number + + Parameters + ---------- + i: int + the ith round + parameter_id: int + the id of the trial/parameter + seq: int + sequence number, e.g., epoch number or batch number + value: int + latest result with sequence number seq + + Returns + ------- + None + """ + if parameter_id in self.configs_perf[i]: + if self.configs_perf[i][parameter_id][0] < seq: + self.configs_perf[i][parameter_id] = [seq, value] + else: + self.configs_perf[i][parameter_id] = [seq, value] + + def inform_trial_end(self, i): + """If the trial is finished and the corresponding round (i.e., i) has all its trials finished, + it will choose the top k trials for the next round (i.e., i+1) + + Parameters + ---------- + i: int + the ith round + + Returns + ------- + new trial or None: + If we have generated new trials after this trial end, we will return a new trial parameters. + Otherwise, we will return None. + """ + global _KEY + self.num_finished_configs[i] += 1 + logger.debug('bracket id: %d, round: %d %d, finished: %d, all: %d', + self.s, self.i, i, self.num_finished_configs[i], self.num_configs_to_run[i]) + if self.num_finished_configs[i] >= self.num_configs_to_run[i] and self.no_more_trial is False: + # choose candidate configs from finished configs to run in the next round + assert self.i == i + 1 + # finish this bracket + if self.i > self.s: + self.no_more_trial = True + return None + this_round_perf = self.configs_perf[i] + if self.optimize_mode is OptimizeMode.Maximize: + sorted_perf = sorted(this_round_perf.items( + ), key=lambda kv: kv[1][1], reverse=True) # reverse + else: + sorted_perf = sorted( + this_round_perf.items(), key=lambda kv: kv[1][1]) + logger.debug( + 'bracket %s next round %s, sorted hyper configs: %s', self.s, self.i, sorted_perf) + next_n, next_r = self.get_n_r() + logger.debug('bracket %s next round %s, next_n=%d, next_r=%d', + self.s, self.i, next_n, next_r) + hyper_configs = dict() + for k in range(next_n): + params_id = sorted_perf[k][0] + params = self.hyper_configs[i][params_id] + params[_KEY] = next_r # modify r + # generate new id + increased_id = params_id.split('_')[-1] + new_id = create_bracket_parameter_id( + self.s, self.i, increased_id) + hyper_configs[new_id] = params + self._record_hyper_configs(hyper_configs) + return [[key, value] for key, value in hyper_configs.items()] + return None + + def get_hyperparameter_configurations(self, num, r, config_generator): + """generate num hyperparameter configurations from search space using Bayesian optimization + + Parameters + ---------- + num: int + the number of hyperparameter configurations + + Returns + ------- + list + a list of hyperparameter configurations. Format: [[key1, value1], [key2, value2], ...] + """ + global _KEY + assert self.i == 0 + hyperparameter_configs = dict() + for _ in range(num): + params_id = create_bracket_parameter_id(self.s, self.i) + params = config_generator.get_config(r) + params[_KEY] = r + hyperparameter_configs[params_id] = params + self._record_hyper_configs(hyperparameter_configs) + return [[key, value] for key, value in hyperparameter_configs.items()] + + def _record_hyper_configs(self, hyper_configs): + """after generating one round of hyperconfigs, this function records the generated hyperconfigs, + creates a dict to record the performance when those hyperconifgs are running, set the number of finished configs + in this round to be 0, and increase the round number. + + Parameters + ---------- + hyper_configs: list + the generated hyperconfigs + """ + self.hyper_configs.append(hyper_configs) + self.configs_perf.append(dict()) + self.num_finished_configs.append(0) + self.num_configs_to_run.append(len(hyper_configs)) + self.increase_i() + +class BOHBClassArgsValidator(ClassArgsValidator): + def validate_class_args(self, **kwargs): + Schema({ + 'optimize_mode': self.choices('optimize_mode', 'maximize', 'minimize'), + Optional('min_budget'): self.range('min_budget', int, 0, 9999), + Optional('max_budget'): self.range('max_budget', int, 0, 9999), + Optional('eta'): self.range('eta', int, 0, 9999), + Optional('min_points_in_model'): self.range('min_points_in_model', int, 0, 9999), + Optional('top_n_percent'): self.range('top_n_percent', int, 1, 99), + Optional('num_samples'): self.range('num_samples', int, 1, 9999), + Optional('random_fraction'): self.range('random_fraction', float, 0, 9999), + Optional('bandwidth_factor'): self.range('bandwidth_factor', float, 0, 9999), + Optional('min_bandwidth'): self.range('min_bandwidth', float, 0, 9999), + }).validate(kwargs) + +class BOHB(MsgDispatcherBase): + """ + BOHB performs robust and efficient hyperparameter optimization + at scale by combining the speed of Hyperband searches with the + guidance and guarantees of convergence of Bayesian Optimization. + Instead of sampling new configurations at random, BOHB uses + kernel density estimators to select promising candidates. + + Parameters + ---------- + optimize_mode: str + optimize mode, 'maximize' or 'minimize' + min_budget: float + The smallest budget to consider. Needs to be positive! + max_budget: float + The largest budget to consider. Needs to be larger than min_budget! + The budgets will be geometrically distributed + :math:`a^2 + b^2 = c^2 \\sim \\eta^k` for :math:`k\\in [0, 1, ... , num\\_subsets - 1]`. + eta: int + In each iteration, a complete run of sequential halving is executed. In it, + after evaluating each configuration on the same subset size, only a fraction of + 1/eta of them 'advances' to the next round. + Must be greater or equal to 2. + min_points_in_model: int + number of observations to start building a KDE. Default 'None' means + dim+1, the bare minimum. + top_n_percent: int + percentage ( between 1 and 99, default 15) of the observations that are considered good. + num_samples: int + number of samples to optimize EI (default 64) + random_fraction: float + fraction of purely random configurations that are sampled from the + prior without the model. + bandwidth_factor: float + to encourage diversity, the points proposed to optimize EI, are sampled + from a 'widened' KDE where the bandwidth is multiplied by this factor (default: 3) + min_bandwidth: float + to keep diversity, even when all (good) samples have the same value for one of the parameters, + a minimum bandwidth (Default: 1e-3) is used instead of zero. + """ + + def __init__(self, + optimize_mode='maximize', + min_budget=1, + max_budget=3, + eta=3, + min_points_in_model=None, + top_n_percent=15, + num_samples=64, + random_fraction=1/3, + bandwidth_factor=3, + min_bandwidth=1e-3): + super(BOHB, self).__init__() + self.optimize_mode = OptimizeMode(optimize_mode) + self.min_budget = min_budget + self.max_budget = max_budget + self.eta = eta + self.min_points_in_model = min_points_in_model + self.top_n_percent = top_n_percent + self.num_samples = num_samples + self.random_fraction = random_fraction + self.bandwidth_factor = bandwidth_factor + self.min_bandwidth = min_bandwidth + + # all the configs waiting for run + self.generated_hyper_configs = [] + # all the completed configs + self.completed_hyper_configs = [] + + self.s_max = math.floor( + math.log(self.max_budget / self.min_budget, self.eta) + _epsilon) + # current bracket(s) number + self.curr_s = self.s_max + # In this case, tuner increases self.credit to issue a trial config sometime later. + self.credit = 0 + self.brackets = dict() + self.search_space = None + # [key, value] = [parameter_id, parameter] + self.parameters = dict() + + # config generator + self.cg = None + + # record the latest parameter_id of the trial job trial_job_id. + # if there is no running parameter_id, self.job_id_para_id_map[trial_job_id] == None + # new trial job is added to this dict and finished trial job is removed from it. + self.job_id_para_id_map = dict() + # record the unsatisfied parameter request from trial jobs + self.unsatisfied_jobs = [] + + def handle_initialize(self, data): + """Initialize Tuner, including creating Bayesian optimization-based parametric models + and search space formations + + Parameters + ---------- + data: search space + search space of this experiment + + Raises + ------ + ValueError + Error: Search space is None + """ + logger.info('start to handle_initialize') + # convert search space jason to ConfigSpace + self.handle_update_search_space(data) + + # generate BOHB config_generator using Bayesian optimization + if self.search_space: + self.cg = CG_BOHB(configspace=self.search_space, + min_points_in_model=self.min_points_in_model, + top_n_percent=self.top_n_percent, + num_samples=self.num_samples, + random_fraction=self.random_fraction, + bandwidth_factor=self.bandwidth_factor, + min_bandwidth=self.min_bandwidth) + else: + raise ValueError('Error: Search space is None') + # generate first brackets + self.generate_new_bracket() + send(CommandType.Initialized, '') + + def generate_new_bracket(self): + """generate a new bracket""" + logger.debug( + 'start to create a new SuccessiveHalving iteration, self.curr_s=%d', self.curr_s) + if self.curr_s < 0: + logger.info("s < 0, Finish this round of Hyperband in BOHB. Generate new round") + self.curr_s = self.s_max + self.brackets[self.curr_s] = Bracket( + s=self.curr_s, s_max=self.s_max, eta=self.eta, + max_budget=self.max_budget, optimize_mode=self.optimize_mode + ) + next_n, next_r = self.brackets[self.curr_s].get_n_r() + logger.debug( + 'new SuccessiveHalving iteration, next_n=%d, next_r=%d', next_n, next_r) + # rewrite with TPE + generated_hyper_configs = self.brackets[self.curr_s].get_hyperparameter_configurations( + next_n, next_r, self.cg) + self.generated_hyper_configs = generated_hyper_configs.copy() + + def handle_request_trial_jobs(self, data): + """recerive the number of request and generate trials + + Parameters + ---------- + data: int + number of trial jobs that nni manager ask to generate + """ + # Receive new request + self.credit += data + + for _ in range(self.credit): + self._request_one_trial_job() + + def _get_one_trial_job(self): + """get one trial job, i.e., one hyperparameter configuration. + + If this function is called, Command will be sent by BOHB: + a. If there is a parameter need to run, will return "NewTrialJob" with a dict: + { + 'parameter_id': id of new hyperparameter + 'parameter_source': 'algorithm' + 'parameters': value of new hyperparameter + } + b. If BOHB don't have parameter waiting, will return "NoMoreTrialJobs" with + { + 'parameter_id': '-1_0_0', + 'parameter_source': 'algorithm', + 'parameters': '' + } + """ + if not self.generated_hyper_configs: + ret = { + 'parameter_id': '-1_0_0', + 'parameter_source': 'algorithm', + 'parameters': '' + } + send(CommandType.NoMoreTrialJobs, json_tricks.dumps(ret)) + return None + assert self.generated_hyper_configs + params = self.generated_hyper_configs.pop(0) + ret = { + 'parameter_id': params[0], + 'parameter_source': 'algorithm', + 'parameters': params[1] + } + self.parameters[params[0]] = params[1] + return ret + + def _request_one_trial_job(self): + """get one trial job, i.e., one hyperparameter configuration. + + If this function is called, Command will be sent by BOHB: + a. If there is a parameter need to run, will return "NewTrialJob" with a dict: + { + 'parameter_id': id of new hyperparameter + 'parameter_source': 'algorithm' + 'parameters': value of new hyperparameter + } + b. If BOHB don't have parameter waiting, will return "NoMoreTrialJobs" with + { + 'parameter_id': '-1_0_0', + 'parameter_source': 'algorithm', + 'parameters': '' + } + """ + ret = self._get_one_trial_job() + if ret is not None: + send(CommandType.NewTrialJob, json_tricks.dumps(ret)) + self.credit -= 1 + + def handle_update_search_space(self, data): + """change json format to ConfigSpace format dict -> configspace + + Parameters + ---------- + data: JSON object + search space of this experiment + """ + search_space = data + cs = CS.ConfigurationSpace() + for var in search_space: + _type = str(search_space[var]["_type"]) + if _type == 'choice': + cs.add_hyperparameter(CSH.CategoricalHyperparameter( + var, choices=search_space[var]["_value"])) + elif _type == 'randint': + cs.add_hyperparameter(CSH.UniformIntegerHyperparameter( + var, lower=search_space[var]["_value"][0], upper=search_space[var]["_value"][1] - 1)) + elif _type == 'uniform': + cs.add_hyperparameter(CSH.UniformFloatHyperparameter( + var, lower=search_space[var]["_value"][0], upper=search_space[var]["_value"][1])) + elif _type == 'quniform': + cs.add_hyperparameter(CSH.UniformFloatHyperparameter( + var, lower=search_space[var]["_value"][0], upper=search_space[var]["_value"][1], + q=search_space[var]["_value"][2])) + elif _type == 'loguniform': + cs.add_hyperparameter(CSH.UniformFloatHyperparameter( + var, lower=search_space[var]["_value"][0], upper=search_space[var]["_value"][1], + log=True)) + elif _type == 'qloguniform': + cs.add_hyperparameter(CSH.UniformFloatHyperparameter( + var, lower=search_space[var]["_value"][0], upper=search_space[var]["_value"][1], + q=search_space[var]["_value"][2], log=True)) + elif _type == 'normal': + cs.add_hyperparameter(CSH.NormalFloatHyperparameter( + var, mu=search_space[var]["_value"][1], sigma=search_space[var]["_value"][2])) + elif _type == 'qnormal': + cs.add_hyperparameter(CSH.NormalFloatHyperparameter( + var, mu=search_space[var]["_value"][1], sigma=search_space[var]["_value"][2], + q=search_space[var]["_value"][3])) + elif _type == 'lognormal': + cs.add_hyperparameter(CSH.NormalFloatHyperparameter( + var, mu=search_space[var]["_value"][1], sigma=search_space[var]["_value"][2], + log=True)) + elif _type == 'qlognormal': + cs.add_hyperparameter(CSH.NormalFloatHyperparameter( + var, mu=search_space[var]["_value"][1], sigma=search_space[var]["_value"][2], + q=search_space[var]["_value"][3], log=True)) + else: + raise ValueError( + 'unrecognized type in search_space, type is {}'.format(_type)) + + self.search_space = cs + + def handle_trial_end(self, data): + """receive the information of trial end and generate next configuaration. + + Parameters + ---------- + data: dict() + it has three keys: trial_job_id, event, hyper_params + trial_job_id: the id generated by training service + event: the job's state + hyper_params: the hyperparameters (a string) generated and returned by tuner + """ + logger.debug('Tuner handle trial end, result is %s', data) + hyper_params = json_tricks.loads(data['hyper_params']) + self._handle_trial_end(hyper_params['parameter_id']) + if data['trial_job_id'] in self.job_id_para_id_map: + del self.job_id_para_id_map[data['trial_job_id']] + + def _send_new_trial(self): + while self.unsatisfied_jobs: + ret = self._get_one_trial_job() + if ret is None: + break + one_unsatisfied = self.unsatisfied_jobs.pop(0) + ret['trial_job_id'] = one_unsatisfied['trial_job_id'] + ret['parameter_index'] = one_unsatisfied['parameter_index'] + # update parameter_id in self.job_id_para_id_map + self.job_id_para_id_map[ret['trial_job_id']] = ret['parameter_id'] + send(CommandType.SendTrialJobParameter, json_tricks.dumps(ret)) + for _ in range(self.credit): + self._request_one_trial_job() + + def _handle_trial_end(self, parameter_id): + s, i, _ = parameter_id.split('_') + hyper_configs = self.brackets[int(s)].inform_trial_end(int(i)) + + if hyper_configs is not None: + logger.debug( + 'bracket %s next round %s, hyper_configs: %s', s, i, hyper_configs) + self.generated_hyper_configs = self.generated_hyper_configs + hyper_configs + # Finish this bracket and generate a new bracket + elif self.brackets[int(s)].no_more_trial: + self.curr_s -= 1 + self.generate_new_bracket() + self._send_new_trial() + + def handle_report_metric_data(self, data): + """reveice the metric data and update Bayesian optimization with final result + + Parameters + ---------- + data: + it is an object which has keys 'parameter_id', 'value', 'trial_job_id', 'type', 'sequence'. + + Raises + ------ + ValueError + Data type not supported + """ + logger.debug('handle report metric data = %s', data) + if 'value' in data: + data['value'] = json_tricks.loads(data['value']) + if data['type'] == MetricType.REQUEST_PARAMETER: + assert multi_phase_enabled() + assert data['trial_job_id'] is not None + assert data['parameter_index'] is not None + assert data['trial_job_id'] in self.job_id_para_id_map + self._handle_trial_end(self.job_id_para_id_map[data['trial_job_id']]) + ret = self._get_one_trial_job() + if ret is None: + self.unsatisfied_jobs.append({'trial_job_id': data['trial_job_id'], 'parameter_index': data['parameter_index']}) + else: + ret['trial_job_id'] = data['trial_job_id'] + ret['parameter_index'] = data['parameter_index'] + # update parameter_id in self.job_id_para_id_map + self.job_id_para_id_map[data['trial_job_id']] = ret['parameter_id'] + send(CommandType.SendTrialJobParameter, json_tricks.dumps(ret)) + else: + assert 'value' in data + value = extract_scalar_reward(data['value']) + if self.optimize_mode is OptimizeMode.Maximize: + reward = -value + else: + reward = value + assert 'parameter_id' in data + s, i, _ = data['parameter_id'].split('_') + logger.debug('bracket id = %s, metrics value = %s, type = %s', s, value, data['type']) + s = int(s) + + # add to self.job_id_para_id_map here, + # because when the first parameter_id is created, trial_job_id is not known yet. + if data['trial_job_id'] in self.job_id_para_id_map: + assert self.job_id_para_id_map[data['trial_job_id']] == data['parameter_id'] + else: + self.job_id_para_id_map[data['trial_job_id']] = data['parameter_id'] + + assert 'type' in data + if data['type'] == MetricType.FINAL: + # and PERIODICAL metric are independent, thus, not comparable. + assert 'sequence' in data + self.brackets[s].set_config_perf( + int(i), data['parameter_id'], sys.maxsize, value) + self.completed_hyper_configs.append(data) + + _parameters = self.parameters[data['parameter_id']] + _parameters.pop(_KEY) + # update BO with loss, max_s budget, hyperparameters + self.cg.new_result(loss=reward, budget=data['sequence'], parameters=_parameters, update_model=True) + elif data['type'] == MetricType.PERIODICAL: + self.brackets[s].set_config_perf( + int(i), data['parameter_id'], data['sequence'], value) + else: + raise ValueError( + 'Data type not supported: {}'.format(data['type'])) + + def handle_add_customized_trial(self, data): + pass + + def handle_import_data(self, data): + """Import additional data for tuning + + Parameters + ---------- + data: + a list of dictionarys, each of which has at least two keys, 'parameter' and 'value' + + Raises + ------ + AssertionError + data doesn't have required key 'parameter' and 'value' + """ + for entry in data: + entry['value'] = json_tricks.loads(entry['value']) + _completed_num = 0 + for trial_info in data: + logger.info("Importing data, current processing progress %s / %s", _completed_num, len(data)) + _completed_num += 1 + assert "parameter" in trial_info + _params = trial_info["parameter"] + assert "value" in trial_info + _value = trial_info['value'] + if not _value: + logger.info("Useless trial data, value is %s, skip this trial data.", _value) + continue + _value = extract_scalar_reward(_value) + budget_exist_flag = False + barely_params = dict() + for keys in _params: + if keys == _KEY: + _budget = _params[keys] + budget_exist_flag = True + else: + barely_params[keys] = _params[keys] + if not budget_exist_flag: + _budget = self.max_budget + logger.info("Set \"TRIAL_BUDGET\" value to %s (max budget)", self.max_budget) + if self.optimize_mode is OptimizeMode.Maximize: + reward = -_value + else: + reward = _value + self.cg.new_result(loss=reward, budget=_budget, parameters=barely_params, update_model=True) + logger.info("Successfully import tuning data to BOHB advisor.") diff --git a/utils/third_party/nni/algorithms/hpo/bohb_advisor/config_generator.py b/utils/third_party/nni/algorithms/hpo/bohb_advisor/config_generator.py new file mode 100644 index 0000000000000000000000000000000000000000..c6a13c6b35ba3b2db9bbbf8618d3b9a3d9240909 --- /dev/null +++ b/utils/third_party/nni/algorithms/hpo/bohb_advisor/config_generator.py @@ -0,0 +1,344 @@ +# BSD 3-Clause License +# Copyright (c) 2017-2018, ML4AAD +# All rights reserved. + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. + +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. + +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import logging + +import ConfigSpace +import ConfigSpace.hyperparameters +import ConfigSpace.util +import numpy as np +import scipy.stats as sps +import statsmodels.api as sm + +logger = logging.getLogger('BOHB_Advisor') + +class CG_BOHB: + def __init__(self, configspace, min_points_in_model=None, + top_n_percent=15, num_samples=64, random_fraction=1/3, + bandwidth_factor=3, min_bandwidth=1e-3): + """Fits for each given budget a kernel density estimator on the best N percent of the + evaluated configurations on this budget. + + + Parameters: + ----------- + configspace: ConfigSpace + Configuration space object + top_n_percent: int + Determines the percentile of configurations that will be used as training data + for the kernel density estimator, e.g if set to 10 the 10% best configurations will be considered + for training. + min_points_in_model: int + minimum number of datapoints needed to fit a model + num_samples: int + number of samples drawn to optimize EI via sampling + random_fraction: float + fraction of random configurations returned + bandwidth_factor: float + widens the bandwidth for contiuous parameters for proposed points to optimize EI + min_bandwidth: float + to keep diversity, even when all (good) samples have the same value for one of the parameters, + a minimum bandwidth (Default: 1e-3) is used instead of zero. + """ + self.top_n_percent = top_n_percent + self.configspace = configspace + self.bw_factor = bandwidth_factor + self.min_bandwidth = min_bandwidth + + self.min_points_in_model = min_points_in_model + if min_points_in_model is None: + self.min_points_in_model = len(self.configspace.get_hyperparameters())+1 + + if self.min_points_in_model < len(self.configspace.get_hyperparameters())+1: + logger.warning('Invalid min_points_in_model value. Setting it to %i', len(self.configspace.get_hyperparameters()) + 1) + self.min_points_in_model = len(self.configspace.get_hyperparameters()) + 1 + + self.num_samples = num_samples + self.random_fraction = random_fraction + + hps = self.configspace.get_hyperparameters() + + self.kde_vartypes = "" + self.vartypes = [] + + for h in hps: + if hasattr(h, 'choices'): + self.kde_vartypes += 'u' + self.vartypes += [len(h.choices)] + else: + self.kde_vartypes += 'c' + self.vartypes += [0] + + self.vartypes = np.array(self.vartypes, dtype=int) + + # store precomputed probs for the categorical parameters + self.cat_probs = [] + + self.configs = dict() + self.losses = dict() + self.good_config_rankings = dict() + self.kde_models = dict() + + def largest_budget_with_model(self): + if not self.kde_models: + return -float('inf') + return max(self.kde_models.keys()) + + def sample_from_largest_budget(self, info_dict): + """We opted for a single multidimensional KDE compared to the + hierarchy of one-dimensional KDEs used in TPE. The dimensional is + seperated by budget. This function sample a configuration from + largest budget. Firstly we sample "num_samples" configurations, + then prefer one with the largest l(x)/g(x). + + Parameters: + ----------- + info_dict: dict + record the information of this configuration + + Returns + ------- + dict: + new configuration named sample + dict: + info_dict, record the information of this configuration + """ + best = np.inf + best_vector = None + + budget = max(self.kde_models.keys()) + + l = self.kde_models[budget]['good'].pdf + g = self.kde_models[budget]['bad'].pdf + + minimize_me = lambda x: max(1e-32, g(x))/max(l(x), 1e-32) + + kde_good = self.kde_models[budget]['good'] + kde_bad = self.kde_models[budget]['bad'] + + for i in range(self.num_samples): + idx = np.random.randint(0, len(kde_good.data)) + datum = kde_good.data[idx] + vector = [] + + for m, bw, t in zip(datum, kde_good.bw, self.vartypes): + + bw = max(bw, self.min_bandwidth) + if t == 0: + bw = self.bw_factor*bw + vector.append(sps.truncnorm.rvs(-m/bw, (1-m)/bw, loc=m, scale=bw)) + else: + if np.random.rand() < (1-bw): + vector.append(int(m)) + else: + vector.append(np.random.randint(t)) + val = minimize_me(vector) + + if not np.isfinite(val): + logger.warning('sampled vector: %s has EI value %s', vector, val) + logger.warning("data in the KDEs:\n%s\n%s", kde_good.data, kde_bad.data) + logger.warning("bandwidth of the KDEs:\n%s\n%s", kde_good.bw, kde_bad.bw) + logger.warning("l(x) = %s", l(vector)) + logger.warning("g(x) = %s", g(vector)) + + # right now, this happens because a KDE does not contain all values for a categorical parameter + # this cannot be fixed with the statsmodels KDE, so for now, we are just going to evaluate this one + # if the good_kde has a finite value, i.e. there is no config with that value in the bad kde, + # so it shouldn't be terrible. + if np.isfinite(l(vector)): + best_vector = vector + break + + if val < best: + best = val + best_vector = vector + + if best_vector is None: + logger.debug("Sampling based optimization with %i samples failed -> using random configuration", self.num_samples) + sample = self.configspace.sample_configuration().get_dictionary() + info_dict['model_based_pick'] = False + + else: + logger.debug('best_vector: %s, %s, %s, %s', best_vector, best, l(best_vector), g(best_vector)) + for i, _ in enumerate(best_vector): + hp = self.configspace.get_hyperparameter(self.configspace.get_hyperparameter_by_idx(i)) + if isinstance(hp, ConfigSpace.hyperparameters.CategoricalHyperparameter): + best_vector[i] = int(np.rint(best_vector[i])) + sample = ConfigSpace.Configuration(self.configspace, vector=best_vector).get_dictionary() + + sample = ConfigSpace.util.deactivate_inactive_hyperparameters( + configuration_space=self.configspace, + configuration=sample) + info_dict['model_based_pick'] = True + + return sample, info_dict + + def get_config(self, budget): + """Function to sample a new configuration + This function is called inside BOHB to query a new configuration + + Parameters: + ----------- + budget: float + the budget for which this configuration is scheduled + + Returns + ------- + config + return a valid configuration with parameters and budget + """ + logger.debug('start sampling a new configuration.') + sample = None + info_dict = {} + + # If no model is available, sample from prior + # also mix in a fraction of random configs + if not self.kde_models.keys() or np.random.rand() < self.random_fraction: + sample = self.configspace.sample_configuration() + info_dict['model_based_pick'] = False + + if sample is None: + sample, info_dict = self.sample_from_largest_budget(info_dict) + + sample = ConfigSpace.util.deactivate_inactive_hyperparameters( + configuration_space=self.configspace, + configuration=sample.get_dictionary() + ).get_dictionary() + + logger.debug('done sampling a new configuration.') + sample['TRIAL_BUDGET'] = budget + return sample + + def impute_conditional_data(self, array): + return_array = np.zeros(array.shape) + for i in range(array.shape[0]): + datum = np.copy(array[i]) + nan_indices = np.argwhere(np.isnan(datum)).flatten() + while np.any(nan_indices): + nan_idx = nan_indices[0] + valid_indices = np.argwhere(np.isfinite(array[:, nan_idx])).flatten() + if valid_indices: + # pick one of them at random and overwrite all NaN values + row_idx = np.random.choice(valid_indices) + datum[nan_indices] = array[row_idx, nan_indices] + else: + # no good point in the data has this value activated, so fill it with a valid but random value + t = self.vartypes[nan_idx] + if t == 0: + datum[nan_idx] = np.random.rand() + else: + datum[nan_idx] = np.random.randint(t) + nan_indices = np.argwhere(np.isnan(datum)).flatten() + return_array[i, :] = datum + return return_array + + def new_result(self, loss, budget, parameters, update_model=True): + """ + Function to register finished runs. Every time a run has finished, this function should be called + to register it with the loss. + + Parameters: + ----------- + loss: float + the loss of the parameters + budget: float + the budget of the parameters + parameters: dict + the parameters of this trial + update_model: bool + whether use this parameter to update BP model + + Returns + ------- + None + """ + if loss is None: + # One could skip crashed results, but we decided + # assign a +inf loss and count them as bad configurations + loss = np.inf + + if budget not in self.configs.keys(): + self.configs[budget] = [] + self.losses[budget] = [] + + # skip model building if we already have a bigger model + if max(list(self.kde_models.keys()) + [-np.inf]) > budget: + return + + # We want to get a numerical representation of the configuration in the original space + conf = ConfigSpace.Configuration(self.configspace, parameters) + self.configs[budget].append(conf.get_array()) + self.losses[budget].append(loss) + + # skip model building: + # a) if not enough points are available + if len(self.configs[budget]) <= self.min_points_in_model - 1: + logger.debug("Only %i run(s) for budget %f available, need more than %s \ + -> can't build model!", len(self.configs[budget]), budget, self.min_points_in_model+1) + return + # b) during warnm starting when we feed previous results in and only update once + if not update_model: + return + + train_configs = np.array(self.configs[budget]) + train_losses = np.array(self.losses[budget]) + + n_good = max(self.min_points_in_model, (self.top_n_percent * train_configs.shape[0])//100) + n_bad = max(self.min_points_in_model, ((100-self.top_n_percent)*train_configs.shape[0])//100) + + # Refit KDE for the current budget + idx = np.argsort(train_losses) + + train_data_good = self.impute_conditional_data(train_configs[idx[:n_good]]) + train_data_bad = self.impute_conditional_data(train_configs[idx[n_good:n_good+n_bad]]) + + if train_data_good.shape[0] <= train_data_good.shape[1]: + return + if train_data_bad.shape[0] <= train_data_bad.shape[1]: + return + + #more expensive crossvalidation method + #bw_estimation = 'cv_ls' + # quick rule of thumb + bw_estimation = 'normal_reference' + + bad_kde = sm.nonparametric.KDEMultivariate(data=train_data_bad, var_type=self.kde_vartypes, bw=bw_estimation) + good_kde = sm.nonparametric.KDEMultivariate(data=train_data_good, var_type=self.kde_vartypes, bw=bw_estimation) + + bad_kde.bw = np.clip(bad_kde.bw, self.min_bandwidth, None) + good_kde.bw = np.clip(good_kde.bw, self.min_bandwidth, None) + + self.kde_models[budget] = { + 'good': good_kde, + 'bad' : bad_kde + } + + # update probs for the categorical parameters for later sampling + logger.debug('done building a new model for budget %f based on %i/%i split\nBest loss for this budget:%f\n', + budget, n_good, n_bad, np.min(train_losses)) diff --git a/utils/third_party/nni/algorithms/hpo/bohb_advisor/requirements.txt b/utils/third_party/nni/algorithms/hpo/bohb_advisor/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..4c11edf7f230848d2def263005f08fcc6733596a --- /dev/null +++ b/utils/third_party/nni/algorithms/hpo/bohb_advisor/requirements.txt @@ -0,0 +1,2 @@ +ConfigSpace==0.4.7 +statsmodels==0.10.0 \ No newline at end of file diff --git a/utils/third_party/nni/algorithms/hpo/curvefitting_assessor/__init__.py b/utils/third_party/nni/algorithms/hpo/curvefitting_assessor/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..664d87fa8870c4535cbe364f49a4e25082d68589 --- /dev/null +++ b/utils/third_party/nni/algorithms/hpo/curvefitting_assessor/__init__.py @@ -0,0 +1,4 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .curvefitting_assessor import CurvefittingAssessor diff --git a/utils/third_party/nni/algorithms/hpo/curvefitting_assessor/curvefitting_assessor.py b/utils/third_party/nni/algorithms/hpo/curvefitting_assessor/curvefitting_assessor.py new file mode 100644 index 0000000000000000000000000000000000000000..885886e89b83712e7eded05fd89598a9e2232b5a --- /dev/null +++ b/utils/third_party/nni/algorithms/hpo/curvefitting_assessor/curvefitting_assessor.py @@ -0,0 +1,135 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +import datetime +from schema import Schema, Optional + +from nni import ClassArgsValidator +from nni.assessor import Assessor, AssessResult +from nni.utils import extract_scalar_history +from .model_factory import CurveModel + +logger = logging.getLogger('curvefitting_Assessor') + +class CurvefittingClassArgsValidator(ClassArgsValidator): + def validate_class_args(self, **kwargs): + Schema({ + 'epoch_num': self.range('epoch_num', int, 0, 9999), + Optional('start_step'): self.range('start_step', int, 0, 9999), + Optional('threshold'): self.range('threshold', float, 0, 9999), + Optional('gap'): self.range('gap', int, 1, 9999), + }).validate(kwargs) + +class CurvefittingAssessor(Assessor): + """CurvefittingAssessor uses learning curve fitting algorithm to predict the learning curve performance in the future. + It stops a pending trial X at step S if the trial's forecast result at target step is convergence and lower than the + best performance in the history. + + Parameters + ---------- + epoch_num : int + The total number of epoch + start_step : int + only after receiving start_step number of reported intermediate results + threshold : float + The threshold that we decide to early stop the worse performance curve. + """ + + def __init__(self, epoch_num=20, start_step=6, threshold=0.95, gap=1): + if start_step <= 0: + logger.warning('It\'s recommended to set start_step to a positive number') + # Record the target position we predict + self.target_pos = epoch_num + # Start forecasting when historical data reaches start step + self.start_step = start_step + # Record the compared threshold + self.threshold = threshold + # Record the number of gap + self.gap = gap + # Record the number of intermediate result in the lastest judgment + self.last_judgment_num = dict() + # Record the best performance + self.set_best_performance = False + self.completed_best_performance = None + self.trial_history = [] + logger.info('Successfully initials the curvefitting assessor') + + def trial_end(self, trial_job_id, success): + """update the best performance of completed trial job + + Parameters + ---------- + trial_job_id : int + trial job id + success : bool + True if succssfully finish the experiment, False otherwise + """ + if success: + if self.set_best_performance: + self.completed_best_performance = max(self.completed_best_performance, self.trial_history[-1]) + else: + self.set_best_performance = True + self.completed_best_performance = self.trial_history[-1] + logger.info('Updated complted best performance, trial job id: %s', trial_job_id) + else: + logger.info('No need to update, trial job id: %s', trial_job_id) + + def assess_trial(self, trial_job_id, trial_history): + """assess whether a trial should be early stop by curve fitting algorithm + + Parameters + ---------- + trial_job_id : int + trial job id + trial_history : list + The history performance matrix of each trial + + Returns + ------- + bool + AssessResult.Good or AssessResult.Bad + + Raises + ------ + Exception + unrecognize exception in curvefitting_assessor + """ + scalar_trial_history = extract_scalar_history(trial_history) + self.trial_history = scalar_trial_history + if not self.set_best_performance: + return AssessResult.Good + curr_step = len(scalar_trial_history) + if curr_step < self.start_step: + return AssessResult.Good + + if trial_job_id in self.last_judgment_num.keys() and curr_step - self.last_judgment_num[trial_job_id] < self.gap: + return AssessResult.Good + self.last_judgment_num[trial_job_id] = curr_step + + try: + start_time = datetime.datetime.now() + # Predict the final result + curvemodel = CurveModel(self.target_pos) + predict_y = curvemodel.predict(scalar_trial_history) + log_message = "Prediction done. Trial job id = {}, Predict value = {}".format(trial_job_id, predict_y) + if predict_y is None: + logger.info('%s, wait for more information to predict precisely', log_message) + return AssessResult.Good + else: + logger.info(log_message) + standard_performance = self.completed_best_performance * self.threshold + + end_time = datetime.datetime.now() + if (end_time - start_time).seconds > 60: + logger.warning( + 'Curve Fitting Assessor Runtime Exceeds 60s, Trial Id = %s Trial History = %s', + trial_job_id, self.trial_history + ) + + if predict_y > standard_performance: + return AssessResult.Good + return AssessResult.Bad + + except Exception as exception: + logger.exception('unrecognize exception in curvefitting_assessor %s', exception) diff --git a/utils/third_party/nni/algorithms/hpo/curvefitting_assessor/curvefunctions.py b/utils/third_party/nni/algorithms/hpo/curvefitting_assessor/curvefunctions.py new file mode 100644 index 0000000000000000000000000000000000000000..c7bfc3b9d817df38689d24ff317e1543ac0f88e9 --- /dev/null +++ b/utils/third_party/nni/algorithms/hpo/curvefitting_assessor/curvefunctions.py @@ -0,0 +1,296 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +A family of functions used by CurvefittingAssessor +""" + +import numpy as np + +all_models = {} +model_para = {} +model_para_num = {} + +curve_combination_models = ['vap', 'pow3', 'linear', 'logx_linear', 'dr_hill_zero_background', 'log_power', 'pow4', 'mmf', + 'exp4', 'ilog2', 'weibull', 'janoschek'] + + +def vap(x, a, b, c): + """Vapor pressure model + + Parameters + ---------- + x : int + a : float + b : float + c : float + + Returns + ------- + float + np.exp(a+b/x+c*np.log(x)) + """ + return np.exp(a+b/x+c*np.log(x)) + + +all_models['vap'] = vap +model_para['vap'] = [-0.622028, -0.470050, 0.042322] +model_para_num['vap'] = 3 + + +def pow3(x, c, a, alpha): + """pow3 + + Parameters + ---------- + x : int + c : float + a : float + alpha : float + + Returns + ------- + float + c - a * x**(-alpha) + """ + return c - a * x**(-alpha) + + +all_models['pow3'] = pow3 +model_para['pow3'] = [0.84, 0.52, 0.01] +model_para_num['pow3'] = 3 + + +def linear(x, a, b): + """linear + + Parameters + ---------- + x : int + a : float + b : float + + Returns + ------- + float + a*x + b + """ + return a*x + b + + +all_models['linear'] = linear +model_para['linear'] = [1., 0] +model_para_num['linear'] = 2 + + +def logx_linear(x, a, b): + """logx linear + + Parameters + ---------- + x : int + a : float + b : float + + Returns + ------- + float + a * np.log(x) + b + """ + x = np.log(x) + return a*x + b + + +all_models['logx_linear'] = logx_linear +model_para['logx_linear'] = [0.378106, 0.046506] +model_para_num['logx_linear'] = 2 + + +def dr_hill_zero_background(x, theta, eta, kappa): + """dr hill zero background + + Parameters + ---------- + x : int + theta : float + eta : float + kappa : float + + Returns + ------- + float + (theta* x**eta) / (kappa**eta + x**eta) + """ + return (theta * x**eta) / (kappa**eta + x**eta) + + +all_models['dr_hill_zero_background'] = dr_hill_zero_background +model_para['dr_hill_zero_background'] = [0.772320, 0.586449, 2.460843] +model_para_num['dr_hill_zero_background'] = 3 + + +def log_power(x, a, b, c): + """"logistic power + + Parameters + ---------- + x : int + a : float + b : float + c : float + + Returns + ------- + float + a/(1.+(x/np.exp(b))**c) + """ + return a/(1.+(x/np.exp(b))**c) + + +all_models['log_power'] = log_power +model_para['log_power'] = [0.77, 2.98, -0.51] +model_para_num['log_power'] = 3 + + +def pow4(x, alpha, a, b, c): + """pow4 + + Parameters + ---------- + x : int + alpha : float + a : float + b : float + c : float + + Returns + ------- + float + c - (a*x+b)**-alpha + """ + return c - (a*x+b)**-alpha + + +all_models['pow4'] = pow4 +model_para['pow4'] = [0.1, 200, 0., 0.8] +model_para_num['pow4'] = 4 + + +def mmf(x, alpha, beta, kappa, delta): + """Morgan-Mercer-Flodin + http://www.pisces-conservation.com/growthhelp/index.html?morgan_mercer_floden.htm + + Parameters + ---------- + x : int + alpha : float + beta : float + kappa : float + delta : float + + Returns + ------- + float + alpha - (alpha - beta) / (1. + (kappa * x)**delta) + """ + return alpha - (alpha - beta) / (1. + (kappa * x)**delta) + + +all_models['mmf'] = mmf +model_para['mmf'] = [0.7, 0.1, 0.01, 5] +model_para_num['mmf'] = 4 + + +def exp4(x, c, a, b, alpha): + """exp4 + + Parameters + ---------- + x : int + c : float + a : float + b : float + alpha : float + + Returns + ------- + float + c - np.exp(-a*(x**alpha)+b) + """ + return c - np.exp(-a*(x**alpha)+b) + + +all_models['exp4'] = exp4 +model_para['exp4'] = [0.7, 0.8, -0.8, 0.3] +model_para_num['exp4'] = 4 + + +def ilog2(x, c, a): + """ilog2 + + Parameters + ---------- + x : int + c : float + a : float + + Returns + ------- + float + c - a / np.log(x) + """ + return c - a / np.log(x) + + +all_models['ilog2'] = ilog2 +model_para['ilog2'] = [0.78, 0.43] +model_para_num['ilog2'] = 2 + + +def weibull(x, alpha, beta, kappa, delta): + """Weibull model + http://www.pisces-conservation.com/growthhelp/index.html?morgan_mercer_floden.htm + + Parameters + ---------- + x : int + alpha : float + beta : float + kappa : float + delta : float + + Returns + ------- + float + alpha - (alpha - beta) * np.exp(-(kappa * x)**delta) + """ + return alpha - (alpha - beta) * np.exp(-(kappa * x)**delta) + + +all_models['weibull'] = weibull +model_para['weibull'] = [0.7, 0.1, 0.01, 1] +model_para_num['weibull'] = 4 + + +def janoschek(x, a, beta, k, delta): + """http://www.pisces-conservation.com/growthhelp/janoschek.htm + + Parameters + ---------- + x : int + a : float + beta : float + k : float + delta : float + + Returns + ------- + float + a - (a - beta) * np.exp(-k*x**delta) + """ + return a - (a - beta) * np.exp(-k*x**delta) + + +all_models['janoschek'] = janoschek +model_para['janoschek'] = [0.73, 0.07, 0.355, 0.46] +model_para_num['janoschek'] = 4 diff --git a/utils/third_party/nni/algorithms/hpo/curvefitting_assessor/model_factory.py b/utils/third_party/nni/algorithms/hpo/curvefitting_assessor/model_factory.py new file mode 100644 index 0000000000000000000000000000000000000000..e6a6ada9976293d98a44d1d763f787a7c7c9bea1 --- /dev/null +++ b/utils/third_party/nni/algorithms/hpo/curvefitting_assessor/model_factory.py @@ -0,0 +1,330 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +import numpy as np +from scipy import optimize +from .curvefunctions import * # pylint: disable=wildcard-import,unused-wildcard-import + +# Number of curve functions we prepared, more details can be found in "curvefunctions.py" +NUM_OF_FUNCTIONS = 12 +# Number of simulation time when we do MCMC sampling +NUM_OF_SIMULATION_TIME = 20 +# Number of samples we select when we do MCMC sampling +NUM_OF_INSTANCE = 10 +# The step size of each noise when we do MCMC sampling +STEP_SIZE = 0.0005 +# Number of least fitting function, if effective function is lower than this number, we will ask for more information +LEAST_FITTED_FUNCTION = 4 + +logger = logging.getLogger('curvefitting_Assessor') + +class CurveModel: + """Build a Curve Model to predict the performance + + Algorithm: https://github.com/Microsoft/nni/blob/master/src/sdk/pynni/nni/curvefitting_assessor/README.md + + Parameters + ---------- + target_pos : int + The point we need to predict + """ + def __init__(self, target_pos): + self.target_pos = target_pos + self.trial_history = [] + self.point_num = 0 + self.effective_model = [] + self.effective_model_num = 0 + self.weight_samples = [] + + def fit_theta(self): + """use least squares to fit all default curves parameter seperately + + Returns + ------- + None + """ + x = range(1, self.point_num + 1) + y = self.trial_history + for i in range(NUM_OF_FUNCTIONS): + model = curve_combination_models[i] + try: + # The maximum number of iterations to fit is 100*(N+1), where N is the number of elements in `x0`. + if model_para_num[model] == 2: + a, b = optimize.curve_fit(all_models[model], x, y)[0] + model_para[model][0] = a + model_para[model][1] = b + elif model_para_num[model] == 3: + a, b, c = optimize.curve_fit(all_models[model], x, y)[0] + model_para[model][0] = a + model_para[model][1] = b + model_para[model][2] = c + elif model_para_num[model] == 4: + a, b, c, d = optimize.curve_fit(all_models[model], x, y)[0] + model_para[model][0] = a + model_para[model][1] = b + model_para[model][2] = c + model_para[model][3] = d + except (RuntimeError, FloatingPointError, OverflowError, ZeroDivisionError): + # Ignore exceptions caused by numerical calculations + pass + except Exception as exception: + logger.critical("Exceptions in fit_theta: %s", exception) + + def filter_curve(self): + """filter the poor performing curve + + Returns + ------- + None + """ + avg = np.sum(self.trial_history) / self.point_num + standard = avg * avg * self.point_num + predict_data = [] + tmp_model = [] + for i in range(NUM_OF_FUNCTIONS): + var = 0 + model = curve_combination_models[i] + for j in range(1, self.point_num + 1): + y = self.predict_y(model, j) + var += (y - self.trial_history[j - 1]) * (y - self.trial_history[j - 1]) + if var < standard: + predict_data.append(y) + tmp_model.append(curve_combination_models[i]) + median = np.median(predict_data) + std = np.std(predict_data) + for model in tmp_model: + y = self.predict_y(model, self.target_pos) + epsilon = self.point_num / 10 * std + if y < median + epsilon and y > median - epsilon: + self.effective_model.append(model) + self.effective_model_num = len(self.effective_model) + logger.info('List of effective model: %s', self.effective_model) + + def predict_y(self, model, pos): + """return the predict y of 'model' when epoch = pos + + Parameters + ---------- + model : string + name of the curve function model + pos : int + the epoch number of the position you want to predict + + Returns + ------- + int + The expected matrix at pos + """ + if model_para_num[model] == 2: + y = all_models[model](pos, model_para[model][0], model_para[model][1]) + elif model_para_num[model] == 3: + y = all_models[model](pos, model_para[model][0], model_para[model][1], model_para[model][2]) + elif model_para_num[model] == 4: + y = all_models[model](pos, model_para[model][0], model_para[model][1], model_para[model][2], model_para[model][3]) + return y + + def f_comb(self, pos, sample): + """return the value of the f_comb when epoch = pos + + Parameters + ---------- + pos : int + the epoch number of the position you want to predict + sample : list + sample is a (1 * NUM_OF_FUNCTIONS) matrix, representing{w1, w2, ... wk} + + Returns + ------- + int + The expected matrix at pos with all the active function's prediction + """ + ret = 0 + for i in range(self.effective_model_num): + model = self.effective_model[i] + y = self.predict_y(model, pos) + ret += sample[i] * y + return ret + + def normalize_weight(self, samples): + """normalize weight + + Parameters + ---------- + samples : list + a collection of sample, it's a (NUM_OF_INSTANCE * NUM_OF_FUNCTIONS) matrix, + representing{{w11, w12, ..., w1k}, {w21, w22, ... w2k}, ...{wk1, wk2,..., wkk}} + + Returns + ------- + list + samples after normalize weight + """ + for i in range(NUM_OF_INSTANCE): + total = 0 + for j in range(self.effective_model_num): + total += samples[i][j] + for j in range(self.effective_model_num): + samples[i][j] /= total + return samples + + def sigma_sq(self, sample): + """returns the value of sigma square, given the weight's sample + + Parameters + ---------- + sample : list + sample is a (1 * NUM_OF_FUNCTIONS) matrix, representing{w1, w2, ... wk} + + Returns + ------- + float + the value of sigma square, given the weight's sample + """ + ret = 0 + for i in range(1, self.point_num + 1): + temp = self.trial_history[i - 1] - self.f_comb(i, sample) + ret += temp * temp + return 1.0 * ret / self.point_num + + def normal_distribution(self, pos, sample): + """returns the value of normal distribution, given the weight's sample and target position + + Parameters + ---------- + pos : int + the epoch number of the position you want to predict + sample : list + sample is a (1 * NUM_OF_FUNCTIONS) matrix, representing{w1, w2, ... wk} + + Returns + ------- + float + the value of normal distribution + """ + curr_sigma_sq = self.sigma_sq(sample) + delta = self.trial_history[pos - 1] - self.f_comb(pos, sample) + return np.exp(np.square(delta) / (-2.0 * curr_sigma_sq)) / np.sqrt(2 * np.pi * np.sqrt(curr_sigma_sq)) + + def likelihood(self, samples): + """likelihood + + Parameters + ---------- + sample : list + sample is a (1 * NUM_OF_FUNCTIONS) matrix, representing{w1, w2, ... wk} + + Returns + ------- + float + likelihood + """ + ret = np.ones(NUM_OF_INSTANCE) + for i in range(NUM_OF_INSTANCE): + for j in range(1, self.point_num + 1): + ret[i] *= self.normal_distribution(j, samples[i]) + return ret + + def prior(self, samples): + """priori distribution + + Parameters + ---------- + samples : list + a collection of sample, it's a (NUM_OF_INSTANCE * NUM_OF_FUNCTIONS) matrix, + representing{{w11, w12, ..., w1k}, {w21, w22, ... w2k}, ...{wk1, wk2,..., wkk}} + + Returns + ------- + float + priori distribution + """ + ret = np.ones(NUM_OF_INSTANCE) + for i in range(NUM_OF_INSTANCE): + for j in range(self.effective_model_num): + if not samples[i][j] > 0: + ret[i] = 0 + if self.f_comb(1, samples[i]) >= self.f_comb(self.target_pos, samples[i]): + ret[i] = 0 + return ret + + def target_distribution(self, samples): + """posterior probability + + Parameters + ---------- + samples : list + a collection of sample, it's a (NUM_OF_INSTANCE * NUM_OF_FUNCTIONS) matrix, + representing{{w11, w12, ..., w1k}, {w21, w22, ... w2k}, ...{wk1, wk2,..., wkk}} + + Returns + ------- + float + posterior probability + """ + curr_likelihood = self.likelihood(samples) + curr_prior = self.prior(samples) + ret = np.ones(NUM_OF_INSTANCE) + for i in range(NUM_OF_INSTANCE): + ret[i] = curr_likelihood[i] * curr_prior[i] + return ret + + def mcmc_sampling(self): + """Adjust the weight of each function using mcmc sampling. + The initial value of each weight is evenly distribute. + Brief introduction: + (1)Definition of sample: + Sample is a (1 * NUM_OF_FUNCTIONS) matrix, representing{w1, w2, ... wk} + (2)Definition of samples: + Samples is a collection of sample, it's a (NUM_OF_INSTANCE * NUM_OF_FUNCTIONS) matrix, + representing{{w11, w12, ..., w1k}, {w21, w22, ... w2k}, ...{wk1, wk2,..., wkk}} + (3)Definition of model: + Model is the function we chose right now. Such as: 'wap', 'weibull'. + (4)Definition of pos: + Pos is the position we want to predict, corresponds to the value of epoch. + + Returns + ------- + None + """ + init_weight = np.ones((self.effective_model_num), dtype=np.float) / self.effective_model_num + self.weight_samples = np.broadcast_to(init_weight, (NUM_OF_INSTANCE, self.effective_model_num)) + for _ in range(NUM_OF_SIMULATION_TIME): + # sample new value from Q(i, j) + new_values = np.random.randn(NUM_OF_INSTANCE, self.effective_model_num) * STEP_SIZE + self.weight_samples + new_values = self.normalize_weight(new_values) + # compute alpha(i, j) = min{1, P(j)Q(j, i)/P(i)Q(i, j)} + alpha = np.minimum(1, self.target_distribution(new_values) / self.target_distribution(self.weight_samples)) + # sample u + u = np.random.rand(NUM_OF_INSTANCE) + # new value + change_value_flag = (u < alpha).astype(np.int) + for j in range(NUM_OF_INSTANCE): + new_values[j] = self.weight_samples[j] * (1 - change_value_flag[j]) + new_values[j] * change_value_flag[j] + self.weight_samples = new_values + + def predict(self, trial_history): + """predict the value of target position + + Parameters + ---------- + trial_history : list + The history performance matrix of each trial. + + Returns + ------- + float + expected final result performance of this hyperparameter config + """ + self.trial_history = trial_history + self.point_num = len(trial_history) + self.fit_theta() + self.filter_curve() + if self.effective_model_num < LEAST_FITTED_FUNCTION: + # different curve's predictions are too scattered, requires more information + return None + self.mcmc_sampling() + ret = 0 + for i in range(NUM_OF_INSTANCE): + ret += self.f_comb(self.target_pos, self.weight_samples[i]) + return ret / NUM_OF_INSTANCE diff --git a/utils/third_party/nni/algorithms/hpo/evolution_tuner/__init__.py b/utils/third_party/nni/algorithms/hpo/evolution_tuner/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/utils/third_party/nni/algorithms/hpo/evolution_tuner/evolution_tuner.py b/utils/third_party/nni/algorithms/hpo/evolution_tuner/evolution_tuner.py new file mode 100644 index 0000000000000000000000000000000000000000..9277dcca3caac30134a0b787e4ed23e48295c855 --- /dev/null +++ b/utils/third_party/nni/algorithms/hpo/evolution_tuner/evolution_tuner.py @@ -0,0 +1,283 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +evolution_tuner.py +""" + +import copy +import random +import logging + +from collections import deque +import numpy as np +from schema import Schema, Optional + +import nni +from nni import ClassArgsValidator +from nni.tuner import Tuner +from nni.utils import OptimizeMode, extract_scalar_reward, split_index, json2parameter, json2space + +logger = logging.getLogger(__name__) + +class Individual: + """ + Indicidual class to store the indv info. + + Attributes + ---------- + config : str + Search space. + info : str + The str to save information of individual. + result : float + The final metric of a individual. + """ + + def __init__(self, config=None, info=None, result=None): + """ + Parameters + ---------- + config : str + A config to represent a group of parameters. + info : str + result : float + save_dir : str + """ + self.config = config + self.result = result + self.info = info + + def __str__(self): + return "info: " + str(self.info) + \ + ", config :" + str(self.config) + ", result: " + str(self.result) + +class EvolutionClassArgsValidator(ClassArgsValidator): + def validate_class_args(self, **kwargs): + Schema({ + 'optimize_mode': self.choices('optimize_mode', 'maximize', 'minimize'), + Optional('population_size'): self.range('population_size', int, 0, 99999), + }).validate(kwargs) + +class EvolutionTuner(Tuner): + """ + EvolutionTuner is tuner using navie evolution algorithm. + """ + + def __init__(self, optimize_mode="maximize", population_size=32): + """ + Parameters + ---------- + optimize_mode : str, default 'maximize' + population_size : int + initial population size. The larger population size, + the better evolution performance. + """ + self.optimize_mode = OptimizeMode(optimize_mode) + self.population_size = population_size + + self.searchspace_json = None + self.running_trials = {} + self.num_running_trials = 0 + self.random_state = None + self.population = None + self.space = None + self.credit = 0 # record the unsatisfied trial requests + self.send_trial_callback = None + self.param_ids = deque() + + def update_search_space(self, search_space): + """ + Update search space. + + Search_space contains the information that user pre-defined. + + Parameters + ---------- + search_space : dict + """ + self.searchspace_json = search_space + self.space = json2space(self.searchspace_json) + + self.random_state = np.random.RandomState() + self.population = [] + + for _ in range(self.population_size): + self._random_generate_individual() + + def trial_end(self, parameter_id, success, **kwargs): + """ + To deal with trial failure. If a trial fails, + random generate the parameters and add into the population. + Parameters + ---------- + parameter_id : int + Unique identifier for hyper-parameters used by this trial. + success : bool + True if the trial successfully completed; False if failed or terminated. + **kwargs + Not used + """ + self.num_running_trials -= 1 + logger.info('trial (%d) end', parameter_id) + + if not success: + self.running_trials.pop(parameter_id) + self._random_generate_individual() + + if self.credit > 1: + param_id = self.param_ids.popleft() + config = self._generate_individual(param_id) + logger.debug('Send new trial (%d, %s) for reducing credit', param_id, config) + self.send_trial_callback(param_id, config) + self.credit -= 1 + self.num_running_trials += 1 + + def generate_multiple_parameters(self, parameter_id_list, **kwargs): + """ + Returns multiple sets of trial (hyper-)parameters, as iterable of serializable objects. + Parameters + ---------- + parameter_id_list : list of int + Unique identifiers for each set of requested hyper-parameters. + **kwargs + Not used + Returns + ------- + list + A list of newly generated configurations + """ + + result = [] + if 'st_callback' in kwargs: + self.send_trial_callback = kwargs['st_callback'] + else: + logger.warning('Send trial callback is not found in kwargs. Evolution tuner might not work properly.') + for parameter_id in parameter_id_list: + had_exception = False + try: + logger.debug("generating param for %s", parameter_id) + res = self.generate_parameters(parameter_id, **kwargs) + self.num_running_trials += 1 + except nni.NoMoreTrialError: + had_exception = True + if not had_exception: + result.append(res) + return result + + def _random_generate_individual(self): + is_rand = dict() + for item in self.space: + is_rand[item] = True + + config = json2parameter(self.searchspace_json, is_rand, self.random_state) + self.population.append(Individual(config=config)) + + def _generate_individual(self, parameter_id): + """ + This function will generate the config for a trial. + If at the first generation, randomly generates individuals to satisfy self.population_size. + Otherwise, random choose a pair of individuals and compare their fitnesses. + The worst of the pair will be removed. Copy the best of the pair and mutate it to generate a new individual. + + Parameters + ---------- + parameter_id : int + + Returns + ------- + dict + A group of candaidte parameters that evolution tuner generated. + """ + pos = -1 + + for i in range(len(self.population)): + if self.population[i].result is None: + pos = i + break + + if pos != -1: + indiv = copy.deepcopy(self.population[pos]) + self.population.pop(pos) + else: + random.shuffle(self.population) + # avoid only 1 individual has result + if len(self.population) > 1 and self.population[0].result < self.population[1].result: + self.population[0] = self.population[1] + + # mutation on the worse individual + space = json2space(self.searchspace_json, + self.population[0].config) + is_rand = dict() + mutation_pos = space[random.randint(0, len(space)-1)] + + for i in range(len(self.space)): + is_rand[self.space[i]] = (self.space[i] == mutation_pos) + config = json2parameter( + self.searchspace_json, is_rand, self.random_state, self.population[0].config) + + if len(self.population) > 1: + self.population.pop(1) + + indiv = Individual(config=config) + + # remove "_index" from config and save params-id + self.running_trials[parameter_id] = indiv + config = split_index(indiv.config) + return config + + + def generate_parameters(self, parameter_id, **kwargs): + """ + This function will returns a dict of trial (hyper-)parameters. + If no trial configration for now, self.credit plus 1 to send the config later + + Parameters + ---------- + parameter_id : int + + Returns + ------- + dict + One newly generated configuration. + """ + if not self.population: + raise RuntimeError('The population is empty') + + if self.num_running_trials >= self.population_size: + logger.warning("No enough trial config, population_size is suggested to be larger than trialConcurrency") + self.credit += 1 + self.param_ids.append(parameter_id) + raise nni.NoMoreTrialError('no more parameters now.') + + return self._generate_individual(parameter_id) + + def receive_trial_result(self, parameter_id, parameters, value, **kwargs): + """ + Record the result from a trial + + Parameters + ---------- + parameter_id : int + parameters : dict + value : dict/float + if value is dict, it should have "default" key. + value is final metrics of the trial. + """ + reward = extract_scalar_reward(value) + + if parameter_id not in self.running_trials: + raise RuntimeError('Received parameter_id %s not in running_trials.', parameter_id) + + # restore the paramsters contains "_index" + config = self.running_trials[parameter_id].config + self.running_trials.pop(parameter_id) + + if self.optimize_mode == OptimizeMode.Minimize: + reward = -reward + + indiv = Individual(config=config, result=reward) + self.population.append(indiv) + + def import_data(self, data): + pass diff --git a/utils/third_party/nni/algorithms/hpo/gp_tuner/__init__.py b/utils/third_party/nni/algorithms/hpo/gp_tuner/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/utils/third_party/nni/algorithms/hpo/gp_tuner/gp_tuner.py b/utils/third_party/nni/algorithms/hpo/gp_tuner/gp_tuner.py new file mode 100644 index 0000000000000000000000000000000000000000..c4e6e9a89cc6258457f647f80d24464969282357 --- /dev/null +++ b/utils/third_party/nni/algorithms/hpo/gp_tuner/gp_tuner.py @@ -0,0 +1,181 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +GPTuner is a Bayesian Optimization method where Gaussian Process is used for modeling loss functions. + +See :class:`GPTuner` for details. +""" + +import warnings +import logging +import numpy as np +from schema import Schema, Optional + +from sklearn.gaussian_process.kernels import Matern +from sklearn.gaussian_process import GaussianProcessRegressor + +from nni import ClassArgsValidator +from nni.tuner import Tuner +from nni.utils import OptimizeMode, extract_scalar_reward + +from .target_space import TargetSpace +from .util import UtilityFunction, acq_max + +logger = logging.getLogger("GP_Tuner_AutoML") + +class GPClassArgsValidator(ClassArgsValidator): + def validate_class_args(self, **kwargs): + Schema({ + Optional('optimize_mode'): self.choices('optimize_mode', 'maximize', 'minimize'), + Optional('utility'): self.choices('utility', 'ei', 'ucb', 'poi'), + Optional('kappa'): float, + Optional('xi'): float, + Optional('nu'): float, + Optional('alpha'): float, + Optional('cold_start_num'): int, + Optional('selection_num_warm_up'): int, + Optional('selection_num_starting_points'): int, + }).validate(kwargs) + +class GPTuner(Tuner): + """ + GPTuner is a Bayesian Optimization method where Gaussian Process is used for modeling loss functions. + + Parameters + ---------- + optimize_mode : str + optimize mode, 'maximize' or 'minimize', by default 'maximize' + utility : str + utility function (also called 'acquisition funcition') to use, which can be 'ei', 'ucb' or 'poi'. By default 'ei'. + kappa : float + value used by utility function 'ucb'. The bigger kappa is, the more the tuner will be exploratory. By default 5. + xi : float + used by utility function 'ei' and 'poi'. The bigger xi is, the more the tuner will be exploratory. By default 0. + nu : float + used to specify Matern kernel. The smaller nu, the less smooth the approximated function is. By default 2.5. + alpha : float + Used to specify Gaussian Process Regressor. Larger values correspond to increased noise level in the observations. + By default 1e-6. + cold_start_num : int + Number of random exploration to perform before Gaussian Process. By default 10. + selection_num_warm_up : int + Number of random points to evaluate for getting the point which maximizes the acquisition function. By default 100000 + selection_num_starting_points : int + Number of times to run L-BFGS-B from a random starting point after the warmup. By default 250. + """ + + def __init__(self, optimize_mode="maximize", utility='ei', kappa=5, xi=0, nu=2.5, alpha=1e-6, cold_start_num=10, + selection_num_warm_up=100000, selection_num_starting_points=250): + self._optimize_mode = OptimizeMode(optimize_mode) + + # utility function related + self._utility = utility + self._kappa = kappa + self._xi = xi + + # target space + self._space = None + + self._random_state = np.random.RandomState() + + # nu, alpha are GPR related params + self._gp = GaussianProcessRegressor( + kernel=Matern(nu=nu), + alpha=alpha, + normalize_y=True, + n_restarts_optimizer=25, + random_state=self._random_state + ) + # num of random evaluations before GPR + self._cold_start_num = cold_start_num + + # params for acq_max + self._selection_num_warm_up = selection_num_warm_up + self._selection_num_starting_points = selection_num_starting_points + + # num of imported data + self._supplement_data_num = 0 + + def update_search_space(self, search_space): + """ + Update the self.bounds and self.types by the search_space.json file. + + Override of the abstract method in :class:`~nni.tuner.Tuner`. + """ + self._space = TargetSpace(search_space, self._random_state) + + def generate_parameters(self, parameter_id, **kwargs): + """ + Method which provides one set of hyper-parameters. + If the number of trial result is lower than cold_start_number, GPTuner will first randomly generate some parameters. + Otherwise, choose the parameters by the Gussian Process Model. + + Override of the abstract method in :class:`~nni.tuner.Tuner`. + """ + if self._space.len() < self._cold_start_num: + results = self._space.random_sample() + else: + # Sklearn's GP throws a large number of warnings at times, but + # we don't really need to see them here. + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + self._gp.fit(self._space.params, self._space.target) + + util = UtilityFunction( + kind=self._utility, kappa=self._kappa, xi=self._xi) + + results = acq_max( + f_acq=util.utility, + gp=self._gp, + y_max=self._space.target.max(), + bounds=self._space.bounds, + space=self._space, + num_warmup=self._selection_num_warm_up, + num_starting_points=self._selection_num_starting_points + ) + + results = self._space.array_to_params(results) + logger.info("Generate paramageters:\n %s", results) + return results + + def receive_trial_result(self, parameter_id, parameters, value, **kwargs): + """ + Method invoked when a trial reports its final result. + + Override of the abstract method in :class:`~nni.tuner.Tuner`. + """ + value = extract_scalar_reward(value) + if self._optimize_mode == OptimizeMode.Minimize: + value = -value + + logger.info("Received trial result.") + logger.info("value :%s", value) + logger.info("parameter : %s", parameters) + self._space.register(parameters, value) + + def import_data(self, data): + """ + Import additional data for tuning. + + Override of the abstract method in :class:`~nni.tuner.Tuner`. + """ + _completed_num = 0 + for trial_info in data: + logger.info( + "Importing data, current processing progress %s / %s", _completed_num, len(data)) + _completed_num += 1 + assert "parameter" in trial_info + _params = trial_info["parameter"] + assert "value" in trial_info + _value = trial_info['value'] + if not _value: + logger.info( + "Useless trial data, value is %s, skip this trial data.", _value) + continue + self._supplement_data_num += 1 + _parameter_id = '_'.join( + ["ImportData", str(self._supplement_data_num)]) + self.receive_trial_result( + parameter_id=_parameter_id, parameters=_params, value=_value) + logger.info("Successfully import data to GP tuner.") diff --git a/utils/third_party/nni/algorithms/hpo/gp_tuner/target_space.py b/utils/third_party/nni/algorithms/hpo/gp_tuner/target_space.py new file mode 100644 index 0000000000000000000000000000000000000000..7ee52c0e9969a90f931bd9a9e43b194f354d81c4 --- /dev/null +++ b/utils/third_party/nni/algorithms/hpo/gp_tuner/target_space.py @@ -0,0 +1,295 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +Tool class to hold the param-space coordinates (X) and target values (Y). +""" + +import numpy as np +import nni.parameter_expressions as parameter_expressions + + +def _hashable(params): + """ + Transform list params to tuple format. Ensure that an point is hashable by a python dict. + + Parameters + ---------- + params : numpy array + array format of parameters + + Returns + ------- + tuple + tuple format of parameters + """ + return tuple(map(float, params)) + + +class TargetSpace(): + """ + Holds the param-space coordinates (X) and target values (Y) + + Parameters + ---------- + pbounds : dict + Dictionary with parameters names and legal values. + + random_state : int, RandomState, or None + optionally specify a seed for a random number generator, by default None. + """ + + def __init__(self, pbounds, random_state=None): + self._random_state = random_state + + # Get the name of the parameters + self._keys = sorted(pbounds) + + # Create an array with parameters bounds + self._bounds = np.array( + [item[1] for item in sorted(pbounds.items(), key=lambda x: x[0])] + ) + + # check values type + for _bound in self._bounds: + if _bound['_type'] == 'choice': + try: + [float(val) for val in _bound['_value']] + except ValueError: + raise ValueError("GP Tuner supports only numerical values") + + # preallocated memory for X and Y points + self._params = np.empty(shape=(0, self.dim)) + self._target = np.empty(shape=(0)) + + # keep track of unique points we have seen so far + self._cache = {} + + def __contains__(self, params): + """ + check if a parameter is already registered + + Parameters + ---------- + params : numpy array + + Returns + ------- + bool + True if the parameter is already registered, else false + """ + return _hashable(params) in self._cache + + def len(self): + """ + length of registered params and targets + + Returns + ------- + int + """ + assert len(self._params) == len(self._target) + return len(self._target) + + @property + def params(self): + """ + registered parameters + + Returns + ------- + numpy array + """ + return self._params + + @property + def target(self): + """ + registered target values + + Returns + ------- + numpy array + """ + return self._target + + @property + def dim(self): + """ + dimension of parameters + + Returns + ------- + int + """ + return len(self._keys) + + @property + def keys(self): + """ + keys of parameters + + Returns + ------- + numpy array + """ + return self._keys + + @property + def bounds(self): + """ + bounds of parameters + + Returns + ------- + numpy array + """ + return self._bounds + + def params_to_array(self, params): + """ + dict to array + + Parameters + ---------- + params : dict + dict format of parameters + + Returns + ------- + numpy array + array format of parameters + """ + try: + assert set(params) == set(self.keys) + except AssertionError: + raise ValueError( + "Parameters' keys ({}) do ".format(sorted(params)) + + "not match the expected set of keys ({}).".format(self.keys) + ) + return np.asarray([params[key] for key in self.keys]) + + def array_to_params(self, x): + """ + array to dict + + maintain int type if the paramters is defined as int in search_space.json + Parameters + ---------- + x : numpy array + array format of parameters + + Returns + ------- + dict + dict format of parameters + """ + try: + assert len(x) == len(self.keys) + except AssertionError: + raise ValueError( + "Size of array ({}) is different than the ".format(len(x)) + + "expected number of parameters ({}).".format(self.dim) + ) + + params = {} + for i, _bound in enumerate(self._bounds): + if _bound['_type'] == 'choice' and all(isinstance(val, int) for val in _bound['_value']): + params.update({self.keys[i]: int(x[i])}) + elif _bound['_type'] in ['randint']: + params.update({self.keys[i]: int(x[i])}) + else: + params.update({self.keys[i]: x[i]}) + + return params + + def register(self, params, target): + """ + Append a point and its target value to the known data. + + Parameters + ---------- + params : dict + parameters + + target : float + target function value + """ + + x = self.params_to_array(params) + if x in self: + print('Data point {} is not unique'.format(x)) + + # Insert data into unique dictionary + self._cache[_hashable(x.ravel())] = target + + self._params = np.concatenate([self._params, x.reshape(1, -1)]) + self._target = np.concatenate([self._target, [target]]) + + def random_sample(self): + """ + Creates a random point within the bounds of the space. + + Returns + ------- + numpy array + one groupe of parameter + """ + params = np.empty(self.dim) + for col, _bound in enumerate(self._bounds): + if _bound['_type'] == 'choice': + params[col] = parameter_expressions.choice( + _bound['_value'], self._random_state) + elif _bound['_type'] == 'randint': + params[col] = self._random_state.randint( + _bound['_value'][0], _bound['_value'][1], size=1) + elif _bound['_type'] == 'uniform': + params[col] = parameter_expressions.uniform( + _bound['_value'][0], _bound['_value'][1], self._random_state) + elif _bound['_type'] == 'quniform': + params[col] = parameter_expressions.quniform( + _bound['_value'][0], _bound['_value'][1], _bound['_value'][2], self._random_state) + elif _bound['_type'] == 'loguniform': + params[col] = parameter_expressions.loguniform( + _bound['_value'][0], _bound['_value'][1], self._random_state) + elif _bound['_type'] == 'qloguniform': + params[col] = parameter_expressions.qloguniform( + _bound['_value'][0], _bound['_value'][1], _bound['_value'][2], self._random_state) + + return params + + def max(self): + """ + Get maximum target value found and its corresponding parameters. + + Returns + ------- + dict + target value and parameters, empty dict if nothing registered + """ + try: + res = { + 'target': self.target.max(), + 'params': dict( + zip(self.keys, self.params[self.target.argmax()]) + ) + } + except ValueError: + res = {} + return res + + def res(self): + """ + Get all target values found and corresponding parameters. + + Returns + ------- + list + a list of target values and their corresponding parameters + """ + params = [dict(zip(self.keys, p)) for p in self.params] + + return [ + {"target": target, "params": param} + for target, param in zip(self.target, params) + ] diff --git a/utils/third_party/nni/algorithms/hpo/gp_tuner/util.py b/utils/third_party/nni/algorithms/hpo/gp_tuner/util.py new file mode 100644 index 0000000000000000000000000000000000000000..6926f988997a1e0b1d0bd6286dd7f091c38fc8da --- /dev/null +++ b/utils/third_party/nni/algorithms/hpo/gp_tuner/util.py @@ -0,0 +1,235 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +utility functions and classes for GPTuner +""" + +import warnings +import numpy as np +from scipy.stats import norm +from scipy.optimize import minimize + + +def _match_val_type(vals, bounds): + """ + Update values in the array, to match their corresponding type, make sure the value is legal. + + Parameters + ---------- + vals : numpy array + values of parameters + bounds : numpy array + list of dictionary which stores parameters names and legal values. + + Returns + ------- + vals_new : list + The closest legal value to the original value + """ + vals_new = [] + + for i, bound in enumerate(bounds): + _type = bound['_type'] + if _type == "choice": + # Find the closest integer in the array, vals_bounds + # pylint: disable=cell-var-from-loop + vals_new.append(min(bound['_value'], key=lambda x: abs(x - vals[i]))) + elif _type in ['quniform', 'randint']: + vals_new.append(np.around(vals[i])) + else: + vals_new.append(vals[i]) + + return vals_new + + +def acq_max(f_acq, gp, y_max, bounds, space, num_warmup, num_starting_points): + """ + A function to find the maximum of the acquisition function + + It uses a combination of random sampling (cheap) and the 'L-BFGS-B' + optimization method. First by sampling ``num_warmup`` points at random, + and then running L-BFGS-B from ``num_starting_points`` random starting points. + + Parameters + ---------- + f_acq : UtilityFunction.utility + The acquisition function object that return its point-wise value. + + gp : GaussianProcessRegressor + A gaussian process fitted to the relevant data. + + y_max : float + The current maximum known value of the target function. + + bounds : numpy array + The variables bounds to limit the search of the acq max. + + num_warmup : int + number of times to randomly sample the aquisition function + + num_starting_points : int + number of times to run scipy.minimize + + Returns + ------- + numpy array + The parameter which achieves max of the acquisition function. + """ + + # Warm up with random points + x_tries = [space.random_sample() + for _ in range(int(num_warmup))] + ys = f_acq(x_tries, gp=gp, y_max=y_max) + x_max = x_tries[ys.argmax()] + max_acq = ys.max() + + + # Explore the parameter space more throughly + x_seeds = [space.random_sample() for _ in range(int(num_starting_points))] + + bounds_minmax = np.array( + [[bound['_value'][0], bound['_value'][-1]] for bound in bounds]) + + for x_try in x_seeds: + # Find the minimum of minus the acquisition function + res = minimize(lambda x: -f_acq(x.reshape(1, -1), gp=gp, y_max=y_max), + x_try.reshape(1, -1), + bounds=bounds_minmax, + method="L-BFGS-B") + + # See if success + if not res.success: + continue + + # Store it if better than previous minimum(maximum). + if max_acq is None or -res.fun[0] >= max_acq: + x_max = _match_val_type(res.x, bounds) + max_acq = -res.fun[0] + + # Clip output to make sure it lies within the bounds. Due to floating + # point technicalities this is not always the case. + return np.clip(x_max, bounds_minmax[:, 0], bounds_minmax[:, 1]) + + +class UtilityFunction(): + """ + A class to compute different acquisition function values. + + Parameters + ---------- + kind : string + specification of utility function to use + kappa : float + parameter usedd for 'ucb' acquisition function + xi : float + parameter usedd for 'ei' and 'poi' acquisition function + """ + + def __init__(self, kind, kappa, xi): + self._kappa = kappa + self._xi = xi + + if kind not in ['ucb', 'ei', 'poi']: + err = "The utility function " \ + "{} has not been implemented, " \ + "please choose one of ucb, ei, or poi.".format(kind) + raise NotImplementedError(err) + self._kind = kind + + def utility(self, x, gp, y_max): + """ + return utility function + + Parameters + ---------- + x : numpy array + parameters + gp : GaussianProcessRegressor + y_max : float + maximum target value observed so far + + Returns + ------- + function + return corresponding function, return None if parameter is illegal + """ + if self._kind == 'ucb': + return self._ucb(x, gp, self._kappa) + if self._kind == 'ei': + return self._ei(x, gp, y_max, self._xi) + if self._kind == 'poi': + return self._poi(x, gp, y_max, self._xi) + return None + + @staticmethod + def _ucb(x, gp, kappa): + """ + Upper Confidence Bound (UCB) utility function + + Parameters + ---------- + x : numpy array + parameters + gp : GaussianProcessRegressor + kappa : float + + Returns + ------- + float + """ + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + mean, std = gp.predict(x, return_std=True) + + return mean + kappa * std + + @staticmethod + def _ei(x, gp, y_max, xi): + """ + Expected Improvement (EI) utility function + + Parameters + ---------- + x : numpy array + parameters + gp : GaussianProcessRegressor + y_max : float + maximum target value observed so far + xi : float + + Returns + ------- + float + """ + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + mean, std = gp.predict(x, return_std=True) + + z = (mean - y_max - xi)/std + return (mean - y_max - xi) * norm.cdf(z) + std * norm.pdf(z) + + @staticmethod + def _poi(x, gp, y_max, xi): + """ + Possibility Of Improvement (POI) utility function + + Parameters + ---------- + x : numpy array + parameters + gp : GaussianProcessRegressor + y_max : float + maximum target value observed so far + xi : float + + Returns + ------- + float + """ + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + mean, std = gp.predict(x, return_std=True) + + z = (mean - y_max - xi)/std + return norm.cdf(z) diff --git a/utils/third_party/nni/algorithms/hpo/gridsearch_tuner/__init__.py b/utils/third_party/nni/algorithms/hpo/gridsearch_tuner/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..2d97629a4c1a62ea79b489a7ce6bba927d6d4f83 --- /dev/null +++ b/utils/third_party/nni/algorithms/hpo/gridsearch_tuner/__init__.py @@ -0,0 +1,4 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .gridsearch_tuner import GridSearchTuner diff --git a/utils/third_party/nni/algorithms/hpo/gridsearch_tuner/gridsearch_tuner.py b/utils/third_party/nni/algorithms/hpo/gridsearch_tuner/gridsearch_tuner.py new file mode 100644 index 0000000000000000000000000000000000000000..8a9ab0a4eda1c390d0266f8a6b4d4e9007e70a4b --- /dev/null +++ b/utils/third_party/nni/algorithms/hpo/gridsearch_tuner/gridsearch_tuner.py @@ -0,0 +1,208 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +gridsearch_tuner.py including: + class GridSearchTuner +""" + +import copy +import logging +import numpy as np + +import nni +from nni.tuner import Tuner +from nni.utils import convert_dict2tuple + +TYPE = '_type' +CHOICE = 'choice' +VALUE = '_value' + +logger = logging.getLogger('grid_search_AutoML') + +class GridSearchTuner(Tuner): + """ + GridSearchTuner will search all the possible configures that the user define in the searchSpace. + The only acceptable types of search space are ``choice``, ``quniform``, ``randint`` + + Type ``choice`` will select one of the options. Note that it can also be nested. + + Type ``quniform`` will receive three values [``low``, ``high``, ``q``], + where [``low``, ``high``] specifies a range and ``q`` specifies the interval. + It will be sampled in a way that the first sampled value is ``low``, + and each of the following values is 'interval' larger than the value in front of it. + + Type ``randint`` gives all possible intergers in range[``low``, ``high``). Note that ``high`` is not included. + """ + + def __init__(self): + self.count = -1 + self.expanded_search_space = [] + self.supplement_data = dict() + + def _json2parameter(self, ss_spec): + """ + Generate all possible configs for hyperparameters from hyperparameter space. + + Parameters + ---------- + ss_spec : dict or list + Hyperparameter space or the ``_value`` of a hyperparameter + + Returns + ------- + list or dict + All the candidate choices of hyperparameters. for a hyperparameter, chosen_params + is a list. for multiple hyperparameters (e.g., search space), chosen_params is a dict. + """ + if isinstance(ss_spec, dict): + if '_type' in ss_spec.keys(): + _type = ss_spec['_type'] + _value = ss_spec['_value'] + chosen_params = list() + if _type == 'choice': + for value in _value: + choice = self._json2parameter(value) + if isinstance(choice, list): + chosen_params.extend(choice) + else: + chosen_params.append(choice) + elif _type == 'quniform': + chosen_params = self._parse_quniform(_value) + elif _type == 'randint': + chosen_params = self._parse_randint(_value) + else: + raise RuntimeError("Not supported type: %s" % _type) + else: + chosen_params = dict() + for key in ss_spec.keys(): + chosen_params[key] = self._json2parameter(ss_spec[key]) + return self._expand_parameters(chosen_params) + elif isinstance(ss_spec, list): + chosen_params = list() + for subspec in ss_spec[1:]: + choice = self._json2parameter(subspec) + if isinstance(choice, list): + chosen_params.extend(choice) + else: + chosen_params.append(choice) + chosen_params = list(map(lambda v: {ss_spec[0]: v}, chosen_params)) + else: + chosen_params = copy.deepcopy(ss_spec) + return chosen_params + + def _parse_quniform(self, param_value): + """ + Parse type of quniform parameter and return a list + """ + low, high, q = param_value[0], param_value[1], param_value[2] + return np.clip(np.arange(np.round(low/q), np.round(high/q)+1) * q, low, high) + + def _parse_randint(self, param_value): + """ + Parse type of randint parameter and return a list + """ + if param_value[0] >= param_value[1]: + raise ValueError("Randint should contain at least 1 candidate, but [%s, %s) contains none.", + param_value[0], param_value[1]) + return np.arange(param_value[0], param_value[1]).tolist() + + def _expand_parameters(self, para): + """ + Enumerate all possible combinations of all parameters + + Parameters + ---------- + para : dict + {key1: [v11, v12, ...], key2: [v21, v22, ...], ...} + + Returns + ------- + dict + {{key1: v11, key2: v21, ...}, {key1: v11, key2: v22, ...}, ...} + """ + if len(para) == 1: + for key, values in para.items(): + return list(map(lambda v: {key: v}, values)) + + key = list(para)[0] + values = para.pop(key) + rest_para = self._expand_parameters(para) + ret_para = list() + for val in values: + for config in rest_para: + config[key] = val + ret_para.append(copy.deepcopy(config)) + return ret_para + + def update_search_space(self, search_space): + """ + Check if the search space is valid and expand it: support only ``choice``, ``quniform``, ``randint``. + + Parameters + ---------- + search_space : dict + The format could be referred to search space spec (https://nni.readthedocs.io/en/latest/Tutorial/SearchSpaceSpec.html). + """ + self.expanded_search_space = self._json2parameter(search_space) + + def generate_parameters(self, parameter_id, **kwargs): + """ + Generate parameters for one trial. + + Parameters + ---------- + parameter_id : int + The id for the generated hyperparameter + **kwargs + Not used + + Returns + ------- + dict + One configuration from the expanded search space. + + Raises + ------ + NoMoreTrialError + If all the configurations has been sent, raise :class:`~nni.NoMoreTrialError`. + """ + self.count += 1 + while self.count <= len(self.expanded_search_space) - 1: + _params_tuple = convert_dict2tuple(self.expanded_search_space[self.count]) + if _params_tuple in self.supplement_data: + self.count += 1 + else: + return self.expanded_search_space[self.count] + raise nni.NoMoreTrialError('no more parameters now.') + + def receive_trial_result(self, parameter_id, parameters, value, **kwargs): + """ + Receive a trial's final performance result reported through :func:`~nni.report_final_result` by the trial. + GridSearchTuner does not need trial's results. + """ + pass + + def import_data(self, data): + """ + Import additional data for tuning + + Parameters + ---------- + list + A list of dictionarys, each of which has at least two keys, ``parameter`` and ``value`` + """ + _completed_num = 0 + for trial_info in data: + logger.info("Importing data, current processing progress %s / %s", _completed_num, len(data)) + _completed_num += 1 + assert "parameter" in trial_info + _params = trial_info["parameter"] + assert "value" in trial_info + _value = trial_info['value'] + if not _value: + logger.info("Useless trial data, value is %s, skip this trial data.", _value) + continue + _params_tuple = convert_dict2tuple(_params) + self.supplement_data[_params_tuple] = True + logger.info("Successfully import data to grid search tuner.") diff --git a/utils/third_party/nni/algorithms/hpo/hyperband_advisor/__init__.py b/utils/third_party/nni/algorithms/hpo/hyperband_advisor/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/utils/third_party/nni/algorithms/hpo/hyperband_advisor/hyperband_advisor.py b/utils/third_party/nni/algorithms/hpo/hyperband_advisor/hyperband_advisor.py new file mode 100644 index 0000000000000000000000000000000000000000..72e2c28e1a283cb50742c04e4a2e0c1ca41ee8d8 --- /dev/null +++ b/utils/third_party/nni/algorithms/hpo/hyperband_advisor/hyperband_advisor.py @@ -0,0 +1,434 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +hyperband_advisor.py +""" + +import copy +import logging +import math +import sys + +import json_tricks +import numpy as np +from schema import Schema, Optional + +from nni import ClassArgsValidator +from nni.runtime.common import multi_phase_enabled +from nni.runtime.msg_dispatcher_base import MsgDispatcherBase +from nni.runtime.protocol import CommandType, send +from nni.utils import NodeType, OptimizeMode, MetricType, extract_scalar_reward +from nni import parameter_expressions + +_logger = logging.getLogger(__name__) + +_next_parameter_id = 0 +_KEY = 'TRIAL_BUDGET' +_epsilon = 1e-6 + + +def create_parameter_id(): + """Create an id + + Returns + ------- + int + parameter id + """ + global _next_parameter_id + _next_parameter_id += 1 + return _next_parameter_id - 1 + + +def create_bracket_parameter_id(brackets_id, brackets_curr_decay, increased_id=-1): + """Create a full id for a specific bracket's hyperparameter configuration + + Parameters + ---------- + brackets_id: int + brackets id + brackets_curr_decay: + brackets curr decay + increased_id: int + increased id + + Returns + ------- + int + params id + """ + if increased_id == -1: + increased_id = str(create_parameter_id()) + params_id = '_'.join([str(brackets_id), + str(brackets_curr_decay), + increased_id]) + return params_id + + +def json2parameter(ss_spec, random_state): + """Randomly generate values for hyperparameters from hyperparameter space i.e., x. + + Parameters + ---------- + ss_spec: + hyperparameter space + random_state: + random operator to generate random values + + Returns + ------- + Parameter: + Parameters in this experiment + """ + if isinstance(ss_spec, dict): + if NodeType.TYPE in ss_spec.keys(): + _type = ss_spec[NodeType.TYPE] + _value = ss_spec[NodeType.VALUE] + if _type == 'choice': + _index = random_state.randint(len(_value)) + chosen_params = json2parameter(ss_spec[NodeType.VALUE][_index], random_state) + else: + chosen_params = getattr(parameter_expressions, _type)(*(_value + [random_state])) + else: + chosen_params = dict() + for key in ss_spec.keys(): + chosen_params[key] = json2parameter(ss_spec[key], random_state) + elif isinstance(ss_spec, list): + chosen_params = list() + for _, subspec in enumerate(ss_spec): + chosen_params.append(json2parameter(subspec, random_state)) + else: + chosen_params = copy.deepcopy(ss_spec) + return chosen_params + + +class Bracket(): + """A bracket in Hyperband, all the information of a bracket is managed by an instance of this class + + Parameters + ---------- + s: int + The current SH iteration index. + s_max: int + total number of SH iterations + eta: float + In each iteration, a complete run of sequential halving is executed. In it, + after evaluating each configuration on the same subset size, only a fraction of + 1/eta of them 'advances' to the next round. + R: + the budget associated with each stage + optimize_mode: str + optimize mode, 'maximize' or 'minimize' + """ + + def __init__(self, s, s_max, eta, R, optimize_mode): + self.bracket_id = s + self.s_max = s_max + self.eta = eta + self.n = math.ceil((s_max + 1) * (eta ** s) / (s + 1) - _epsilon) + self.r = R / eta ** s + self.i = 0 + self.hyper_configs = [] # [ {id: params}, {}, ... ] + self.configs_perf = [] # [ {id: [seq, acc]}, {}, ... ] + self.num_configs_to_run = [] # [ n, n, n, ... ] + self.num_finished_configs = [] # [ n, n, n, ... ] + self.optimize_mode = OptimizeMode(optimize_mode) + self.no_more_trial = False + + def is_completed(self): + """check whether this bracket has sent out all the hyperparameter configurations""" + return self.no_more_trial + + def get_n_r(self): + """return the values of n and r for the next round""" + return math.floor(self.n / self.eta ** self.i + _epsilon), math.floor(self.r * self.eta ** self.i + _epsilon) + + def increase_i(self): + """i means the ith round. Increase i by 1""" + self.i += 1 + if self.i > self.bracket_id: + self.no_more_trial = True + + def set_config_perf(self, i, parameter_id, seq, value): + """update trial's latest result with its sequence number, e.g., epoch number or batch number + + Parameters + ---------- + i: int + the ith round + parameter_id: int + the id of the trial/parameter + seq: int + sequence number, e.g., epoch number or batch number + value: int + latest result with sequence number seq + + Returns + ------- + None + """ + if parameter_id in self.configs_perf[i]: + if self.configs_perf[i][parameter_id][0] < seq: + self.configs_perf[i][parameter_id] = [seq, value] + else: + self.configs_perf[i][parameter_id] = [seq, value] + + def inform_trial_end(self, i): + """If the trial is finished and the corresponding round (i.e., i) has all its trials finished, + it will choose the top k trials for the next round (i.e., i+1) + + Parameters + ---------- + i: int + the ith round + """ + global _KEY + self.num_finished_configs[i] += 1 + _logger.debug('bracket id: %d, round: %d %d, finished: %d, all: %d', self.bracket_id, self.i, i, + self.num_finished_configs[i], self.num_configs_to_run[i]) + if self.num_finished_configs[i] >= self.num_configs_to_run[i] \ + and self.no_more_trial is False: + # choose candidate configs from finished configs to run in the next round + assert self.i == i + 1 + this_round_perf = self.configs_perf[i] + if self.optimize_mode is OptimizeMode.Maximize: + sorted_perf = sorted(this_round_perf.items(), key=lambda kv: kv[1][1], reverse=True) # reverse + else: + sorted_perf = sorted(this_round_perf.items(), key=lambda kv: kv[1][1]) + _logger.debug('bracket %s next round %s, sorted hyper configs: %s', self.bracket_id, self.i, sorted_perf) + next_n, next_r = self.get_n_r() + _logger.debug('bracket %s next round %s, next_n=%d, next_r=%d', self.bracket_id, self.i, next_n, next_r) + hyper_configs = dict() + for k in range(next_n): + params_id = sorted_perf[k][0] + params = self.hyper_configs[i][params_id] + params[_KEY] = next_r # modify r + # generate new id + increased_id = params_id.split('_')[-1] + new_id = create_bracket_parameter_id(self.bracket_id, self.i, increased_id) + hyper_configs[new_id] = params + self._record_hyper_configs(hyper_configs) + return [[key, value] for key, value in hyper_configs.items()] + return None + + def get_hyperparameter_configurations(self, num, r, searchspace_json, random_state): + """Randomly generate num hyperparameter configurations from search space + + Parameters + ---------- + num: int + the number of hyperparameter configurations + + Returns + ------- + list + a list of hyperparameter configurations. Format: [[key1, value1], [key2, value2], ...] + """ + global _KEY + assert self.i == 0 + hyperparameter_configs = dict() + for _ in range(num): + params_id = create_bracket_parameter_id(self.bracket_id, self.i) + params = json2parameter(searchspace_json, random_state) + params[_KEY] = r + hyperparameter_configs[params_id] = params + self._record_hyper_configs(hyperparameter_configs) + return [[key, value] for key, value in hyperparameter_configs.items()] + + def _record_hyper_configs(self, hyper_configs): + """after generating one round of hyperconfigs, this function records the generated hyperconfigs, + creates a dict to record the performance when those hyperconifgs are running, set the number of finished configs + in this round to be 0, and increase the round number. + + Parameters + ---------- + hyper_configs: list + the generated hyperconfigs + """ + self.hyper_configs.append(hyper_configs) + self.configs_perf.append(dict()) + self.num_finished_configs.append(0) + self.num_configs_to_run.append(len(hyper_configs)) + self.increase_i() + +class HyperbandClassArgsValidator(ClassArgsValidator): + def validate_class_args(self, **kwargs): + Schema({ + 'optimize_mode': self.choices('optimize_mode', 'maximize', 'minimize'), + Optional('R'): int, + Optional('eta'): int + }).validate(kwargs) + +class Hyperband(MsgDispatcherBase): + """Hyperband inherit from MsgDispatcherBase rather than Tuner, because it integrates both tuner's functions and assessor's functions. + This is an implementation that could fully leverage available resources, i.e., high parallelism. + A single execution of Hyperband takes a finite budget of (s_max + 1)B. + + Parameters + ---------- + R: int + the maximum amount of resource that can be allocated to a single configuration + eta: int + the variable that controls the proportion of configurations discarded in each round of SuccessiveHalving + optimize_mode: str + optimize mode, 'maximize' or 'minimize' + """ + + def __init__(self, R=60, eta=3, optimize_mode='maximize'): + """B = (s_max + 1)R""" + super(Hyperband, self).__init__() + self.R = R + self.eta = eta + self.brackets = dict() # dict of Bracket + self.generated_hyper_configs = [] # all the configs waiting for run + self.completed_hyper_configs = [] # all the completed configs + self.s_max = math.floor(math.log(self.R, self.eta) + _epsilon) + self.curr_s = self.s_max + + self.searchspace_json = None + self.random_state = None + self.optimize_mode = OptimizeMode(optimize_mode) + + # This is for the case that nnimanager requests trial config, but tuner cannot provide immediately. + # In this case, tuner increases self.credit to issue a trial config sometime later. + self.credit = 0 + + # record the latest parameter_id of the trial job trial_job_id. + # if there is no running parameter_id, self.job_id_para_id_map[trial_job_id] == None + # new trial job is added to this dict and finished trial job is removed from it. + self.job_id_para_id_map = dict() + + def handle_initialize(self, data): + """callback for initializing the advisor + Parameters + ---------- + data: dict + search space + """ + self.handle_update_search_space(data) + send(CommandType.Initialized, '') + + def handle_request_trial_jobs(self, data): + """ + Parameters + ---------- + data: int + number of trial jobs + """ + for _ in range(data): + ret = self._get_one_trial_job() + send(CommandType.NewTrialJob, json_tricks.dumps(ret)) + + def _get_one_trial_job(self): + """get one trial job, i.e., one hyperparameter configuration.""" + if not self.generated_hyper_configs: + if self.curr_s < 0: + self.curr_s = self.s_max + _logger.debug('create a new bracket, self.curr_s=%d', self.curr_s) + self.brackets[self.curr_s] = Bracket(self.curr_s, self.s_max, self.eta, self.R, self.optimize_mode) + next_n, next_r = self.brackets[self.curr_s].get_n_r() + _logger.debug('new bracket, next_n=%d, next_r=%d', next_n, next_r) + assert self.searchspace_json is not None and self.random_state is not None + generated_hyper_configs = self.brackets[self.curr_s].get_hyperparameter_configurations(next_n, next_r, + self.searchspace_json, + self.random_state) + self.generated_hyper_configs = generated_hyper_configs.copy() + self.curr_s -= 1 + + assert self.generated_hyper_configs + params = self.generated_hyper_configs.pop(0) + ret = { + 'parameter_id': params[0], + 'parameter_source': 'algorithm', + 'parameters': params[1] + } + return ret + + def handle_update_search_space(self, data): + """data: JSON object, which is search space + """ + self.searchspace_json = data + self.random_state = np.random.RandomState() + + def _handle_trial_end(self, parameter_id): + """ + Parameters + ---------- + parameter_id: parameter id of the finished config + """ + bracket_id, i, _ = parameter_id.split('_') + hyper_configs = self.brackets[int(bracket_id)].inform_trial_end(int(i)) + if hyper_configs is not None: + _logger.debug('bracket %s next round %s, hyper_configs: %s', bracket_id, i, hyper_configs) + self.generated_hyper_configs = self.generated_hyper_configs + hyper_configs + + def handle_trial_end(self, data): + """ + Parameters + ---------- + data: dict() + it has three keys: trial_job_id, event, hyper_params + trial_job_id: the id generated by training service + event: the job's state + hyper_params: the hyperparameters (a string) generated and returned by tuner + """ + hyper_params = json_tricks.loads(data['hyper_params']) + self._handle_trial_end(hyper_params['parameter_id']) + if data['trial_job_id'] in self.job_id_para_id_map: + del self.job_id_para_id_map[data['trial_job_id']] + + def handle_report_metric_data(self, data): + """ + Parameters + ---------- + data: + it is an object which has keys 'parameter_id', 'value', 'trial_job_id', 'type', 'sequence'. + + Raises + ------ + ValueError + Data type not supported + """ + if 'value' in data: + data['value'] = json_tricks.loads(data['value']) + if data['type'] == MetricType.REQUEST_PARAMETER: + assert multi_phase_enabled() + assert data['trial_job_id'] is not None + assert data['parameter_index'] is not None + assert data['trial_job_id'] in self.job_id_para_id_map + self._handle_trial_end(self.job_id_para_id_map[data['trial_job_id']]) + ret = self._get_one_trial_job() + if data['trial_job_id'] is not None: + ret['trial_job_id'] = data['trial_job_id'] + if data['parameter_index'] is not None: + ret['parameter_index'] = data['parameter_index'] + self.job_id_para_id_map[data['trial_job_id']] = ret['parameter_id'] + send(CommandType.SendTrialJobParameter, json_tricks.dumps(ret)) + else: + value = extract_scalar_reward(data['value']) + bracket_id, i, _ = data['parameter_id'].split('_') + bracket_id = int(bracket_id) + + # add to self.job_id_para_id_map here, + # because when the first parameter_id is created, trial_job_id is not known yet. + if data['trial_job_id'] in self.job_id_para_id_map: + assert self.job_id_para_id_map[data['trial_job_id']] == data['parameter_id'] + else: + self.job_id_para_id_map[data['trial_job_id']] = data['parameter_id'] + + if data['type'] == MetricType.FINAL: + # sys.maxsize indicates this value is from FINAL metric data, because data['sequence'] from FINAL metric + # and PERIODICAL metric are independent, thus, not comparable. + self.brackets[bracket_id].set_config_perf(int(i), data['parameter_id'], sys.maxsize, value) + self.completed_hyper_configs.append(data) + elif data['type'] == MetricType.PERIODICAL: + self.brackets[bracket_id].set_config_perf(int(i), data['parameter_id'], data['sequence'], value) + else: + raise ValueError('Data type not supported: {}'.format(data['type'])) + + def handle_add_customized_trial(self, data): + pass + + def handle_import_data(self, data): + pass diff --git a/utils/third_party/nni/algorithms/hpo/hyperband_advisor/requirements.txt b/utils/third_party/nni/algorithms/hpo/hyperband_advisor/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/utils/third_party/nni/algorithms/hpo/hyperopt_tuner/__init__.py b/utils/third_party/nni/algorithms/hpo/hyperopt_tuner/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/utils/third_party/nni/algorithms/hpo/hyperopt_tuner/hyperopt_tuner.py b/utils/third_party/nni/algorithms/hpo/hyperopt_tuner/hyperopt_tuner.py new file mode 100644 index 0000000000000000000000000000000000000000..fbe9cda13faac67c060089f68d35048a4109a789 --- /dev/null +++ b/utils/third_party/nni/algorithms/hpo/hyperopt_tuner/hyperopt_tuner.py @@ -0,0 +1,499 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +hyperopt_tuner.py +""" + +import copy +import logging + +import hyperopt as hp +import numpy as np +from schema import Optional, Schema +from nni import ClassArgsValidator +from nni.tuner import Tuner +from nni.utils import NodeType, OptimizeMode, extract_scalar_reward, split_index + +logger = logging.getLogger('hyperopt_AutoML') + + +def json2space(in_x, name=NodeType.ROOT): + """ + Change json to search space in hyperopt. + + Parameters + ---------- + in_x : dict/list/str/int/float + The part of json. + name : str + name could be NodeType.ROOT, NodeType.TYPE, NodeType.VALUE or NodeType.INDEX, NodeType.NAME. + """ + out_y = copy.deepcopy(in_x) + if isinstance(in_x, dict): + if NodeType.TYPE in in_x.keys(): + _type = in_x[NodeType.TYPE] + name = name + '-' + _type + _value = json2space(in_x[NodeType.VALUE], name=name) + if _type == 'choice': + out_y = hp.hp.choice(name, _value) + elif _type == 'randint': + out_y = hp.hp.randint(name, _value[1] - _value[0]) + else: + if _type in ['loguniform', 'qloguniform']: + _value[:2] = np.log(_value[:2]) + out_y = getattr(hp.hp, _type)(name, *_value) + else: + out_y = dict() + for key in in_x.keys(): + out_y[key] = json2space(in_x[key], name + '[%s]' % str(key)) + elif isinstance(in_x, list): + out_y = list() + for i, x_i in enumerate(in_x): + if isinstance(x_i, dict): + if NodeType.NAME not in x_i.keys(): + raise RuntimeError( + '\'_name\' key is not found in this nested search space.' + ) + out_y.append(json2space(x_i, name + '[%d]' % i)) + return out_y + + +def json2parameter(in_x, parameter, name=NodeType.ROOT): + """ + Change json to parameters. + """ + out_y = copy.deepcopy(in_x) + if isinstance(in_x, dict): + if NodeType.TYPE in in_x.keys(): + _type = in_x[NodeType.TYPE] + name = name + '-' + _type + if _type == 'choice': + _index = parameter[name] + out_y = { + NodeType.INDEX: + _index, + NodeType.VALUE: + json2parameter(in_x[NodeType.VALUE][_index], + parameter, + name=name + '[%d]' % _index) + } + else: + if _type in ['quniform', 'qloguniform']: + out_y = np.clip(parameter[name], in_x[NodeType.VALUE][0], in_x[NodeType.VALUE][1]) + elif _type == 'randint': + out_y = parameter[name] + in_x[NodeType.VALUE][0] + else: + out_y = parameter[name] + else: + out_y = dict() + for key in in_x.keys(): + out_y[key] = json2parameter(in_x[key], parameter, + name + '[%s]' % str(key)) + elif isinstance(in_x, list): + out_y = list() + for i, x_i in enumerate(in_x): + if isinstance(x_i, dict): + if NodeType.NAME not in x_i.keys(): + raise RuntimeError( + '\'_name\' key is not found in this nested search space.' + ) + out_y.append(json2parameter(x_i, parameter, name + '[%d]' % i)) + return out_y + + +def json2vals(in_x, vals, out_y, name=NodeType.ROOT): + if isinstance(in_x, dict): + if NodeType.TYPE in in_x.keys(): + _type = in_x[NodeType.TYPE] + name = name + '-' + _type + + try: + out_y[name] = vals[NodeType.INDEX] + # TODO - catch exact Exception + except Exception: + out_y[name] = vals + + if _type == 'choice': + _index = vals[NodeType.INDEX] + json2vals(in_x[NodeType.VALUE][_index], + vals[NodeType.VALUE], + out_y, + name=name + '[%d]' % _index) + if _type == 'randint': + out_y[name] -= in_x[NodeType.VALUE][0] + else: + for key in in_x.keys(): + json2vals(in_x[key], vals[key], out_y, + name + '[%s]' % str(key)) + elif isinstance(in_x, list): + for i, temp in enumerate(in_x): + # nested json + if isinstance(temp, dict): + if NodeType.NAME not in temp.keys(): + raise RuntimeError( + '\'_name\' key is not found in this nested search space.' + ) + else: + json2vals(temp, vals[i], out_y, name + '[%d]' % i) + else: + json2vals(temp, vals[i], out_y, name + '[%d]' % i) + + +def _add_index(in_x, parameter): + """ + change parameters in NNI format to parameters in hyperopt format(This function also support nested dict.). + For example, receive parameters like: + {'dropout_rate': 0.8, 'conv_size': 3, 'hidden_size': 512} + Will change to format in hyperopt, like: + {'dropout_rate': 0.8, 'conv_size': {'_index': 1, '_value': 3}, 'hidden_size': {'_index': 1, '_value': 512}} + """ + if NodeType.TYPE not in in_x: # if at the top level + out_y = dict() + for key, value in parameter.items(): + out_y[key] = _add_index(in_x[key], value) + return out_y + elif isinstance(in_x, dict): + value_type = in_x[NodeType.TYPE] + value_format = in_x[NodeType.VALUE] + if value_type == "choice": + choice_name = parameter[0] if isinstance(parameter, + list) else parameter + for pos, item in enumerate( + value_format): # here value_format is a list + if isinstance( + item, + list): # this format is ["choice_key", format_dict] + choice_key = item[0] + choice_value_format = item[1] + if choice_key == choice_name: + return { + NodeType.INDEX: pos, + NodeType.VALUE: [ + choice_name, + _add_index(choice_value_format, parameter[1]) + ] + } + elif choice_name == item: + return {NodeType.INDEX: pos, NodeType.VALUE: item} + else: + return parameter + return None # note: this is not written by original author, feel free to modify if you think it's incorrect + +class HyperoptClassArgsValidator(ClassArgsValidator): + def validate_class_args(self, **kwargs): + Schema({ + Optional('optimize_mode'): self.choices('optimize_mode', 'maximize', 'minimize'), + Optional('parallel_optimize'): bool, + Optional('constant_liar_type'): self.choices('constant_liar_type', 'min', 'max', 'mean') + }).validate(kwargs) + +class HyperoptTuner(Tuner): + """ + HyperoptTuner is a tuner which using hyperopt algorithm. + """ + + def __init__(self, algorithm_name, optimize_mode='minimize', + parallel_optimize=False, constant_liar_type='min'): + """ + Parameters + ---------- + algorithm_name : str + algorithm_name includes "tpe", "random_search" and anneal". + optimize_mode : str + parallel_optimize : bool + More detail could reference: docs/en_US/Tuner/HyperoptTuner.md + constant_liar_type : str + constant_liar_type including "min", "max" and "mean" + More detail could reference: docs/en_US/Tuner/HyperoptTuner.md + """ + self.algorithm_name = algorithm_name + self.optimize_mode = OptimizeMode(optimize_mode) + self.json = None + self.total_data = {} + self.rval = None + self.supplement_data_num = 0 + + self.parallel = parallel_optimize + if self.parallel: + self.CL_rval = None + self.constant_liar_type = constant_liar_type + self.running_data = [] + self.optimal_y = None + + def _choose_tuner(self, algorithm_name): + """ + Parameters + ---------- + algorithm_name : str + algorithm_name includes "tpe", "random_search" and anneal" + """ + if algorithm_name == 'tpe': + return hp.tpe.suggest + if algorithm_name == 'random_search': + return hp.rand.suggest + if algorithm_name == 'anneal': + return hp.anneal.suggest + raise RuntimeError('Not support tuner algorithm in hyperopt.') + + def update_search_space(self, search_space): + """ + Update search space definition in tuner by search_space in parameters. + + Will called when first setup experiemnt or update search space in WebUI. + + Parameters + ---------- + search_space : dict + """ + self.json = search_space + + search_space_instance = json2space(self.json) + rstate = np.random.RandomState() + trials = hp.Trials() + domain = hp.Domain(None, + search_space_instance, + pass_expr_memo_ctrl=None) + algorithm = self._choose_tuner(self.algorithm_name) + self.rval = hp.FMinIter(algorithm, + domain, + trials, + max_evals=-1, + rstate=rstate, + verbose=0) + self.rval.catch_eval_exceptions = False + + def generate_parameters(self, parameter_id, **kwargs): + """ + Returns a set of trial (hyper-)parameters, as a serializable object. + + Parameters + ---------- + parameter_id : int + + Returns + ------- + params : dict + """ + total_params = self.get_suggestion(random_search=False) + # avoid generating same parameter with concurrent trials because hyperopt doesn't support parallel mode + if total_params in self.total_data.values(): + # but it can cause duplicate parameter rarely + total_params = self.get_suggestion(random_search=True) + self.total_data[parameter_id] = total_params + + if self.parallel: + self.running_data.append(parameter_id) + + params = split_index(total_params) + return params + + def receive_trial_result(self, parameter_id, parameters, value, **kwargs): + """ + Record an observation of the objective function + + Parameters + ---------- + parameter_id : int + parameters : dict + value : dict/float + if value is dict, it should have "default" key. + value is final metrics of the trial. + """ + reward = extract_scalar_reward(value) + # restore the paramsters contains '_index' + if parameter_id not in self.total_data: + raise RuntimeError('Received parameter_id not in total_data.') + params = self.total_data[parameter_id] + + # code for parallel + if self.parallel: + constant_liar = kwargs.get('constant_liar', False) + + if constant_liar: + rval = self.CL_rval + else: + rval = self.rval + # ignore duplicated reported final result (due to aware of intermedate result) + if parameter_id not in self.running_data: + logger.info("Received duplicated final result with parameter id: %s", parameter_id) + return + self.running_data.remove(parameter_id) + + # update the reward of optimal_y + if self.optimal_y is None: + if self.constant_liar_type == 'mean': + self.optimal_y = [reward, 1] + else: + self.optimal_y = reward + else: + if self.constant_liar_type == 'mean': + _sum = self.optimal_y[0] + reward + _number = self.optimal_y[1] + 1 + self.optimal_y = [_sum, _number] + elif self.constant_liar_type == 'min': + self.optimal_y = min(self.optimal_y, reward) + elif self.constant_liar_type == 'max': + self.optimal_y = max(self.optimal_y, reward) + logger.debug("Update optimal_y with reward, optimal_y = %s", self.optimal_y) + else: + rval = self.rval + + + if self.optimize_mode is OptimizeMode.Maximize: + reward = -reward + + domain = rval.domain + trials = rval.trials + + new_id = len(trials) + + rval_specs = [None] + rval_results = [domain.new_result()] + rval_miscs = [dict(tid=new_id, cmd=domain.cmd, workdir=domain.workdir)] + + vals = params + idxs = dict() + + out_y = dict() + json2vals(self.json, vals, out_y) + vals = out_y + for key in domain.params: + if key in [NodeType.VALUE, NodeType.INDEX]: + continue + if key not in vals or vals[key] is None or vals[key] == []: + idxs[key] = vals[key] = [] + else: + idxs[key] = [new_id] + vals[key] = [vals[key]] + + self.miscs_update_idxs_vals(rval_miscs, + idxs, + vals, + idxs_map={new_id: new_id}, + assert_all_vals_used=False) + + trial = trials.new_trial_docs([new_id], rval_specs, rval_results, + rval_miscs)[0] + trial['result'] = {'loss': reward, 'status': 'ok'} + trial['state'] = hp.JOB_STATE_DONE + trials.insert_trial_docs([trial]) + trials.refresh() + + def miscs_update_idxs_vals(self, + miscs, + idxs, + vals, + assert_all_vals_used=True, + idxs_map=None): + """ + Unpack the idxs-vals format into the list of dictionaries that is + `misc`. + + Parameters + ---------- + idxs_map : dict + idxs_map is a dictionary of id->id mappings so that the misc['idxs'] can + contain different numbers than the idxs argument. + """ + if idxs_map is None: + idxs_map = {} + + assert set(idxs.keys()) == set(vals.keys()) + + misc_by_id = {m['tid']: m for m in miscs} + for m in miscs: + m['idxs'] = {key: [] for key in idxs} + m['vals'] = {key: [] for key in idxs} + + for key in idxs: + assert len(idxs[key]) == len(vals[key]) + for tid, val in zip(idxs[key], vals[key]): + tid = idxs_map.get(tid, tid) + if assert_all_vals_used or tid in misc_by_id: + misc_by_id[tid]['idxs'][key] = [tid] + misc_by_id[tid]['vals'][key] = [val] + + def get_suggestion(self, random_search=False): + """ + get suggestion from hyperopt + + Parameters + ---------- + random_search : bool + flag to indicate random search or not (default: {False}) + + Returns + ---------- + total_params : dict + parameter suggestion + """ + if self.parallel and len(self.total_data) > 20 and self.running_data and self.optimal_y is not None: + self.CL_rval = copy.deepcopy(self.rval) + if self.constant_liar_type == 'mean': + _constant_liar_y = self.optimal_y[0] / self.optimal_y[1] + else: + _constant_liar_y = self.optimal_y + for _parameter_id in self.running_data: + self.receive_trial_result(parameter_id=_parameter_id, parameters=None, value=_constant_liar_y, constant_liar=True) + rval = self.CL_rval + + random_state = np.random.randint(2**31 - 1) + else: + rval = self.rval + random_state = rval.rstate.randint(2**31 - 1) + + trials = rval.trials + algorithm = rval.algo + new_ids = rval.trials.new_trial_ids(1) + rval.trials.refresh() + + if random_search: + new_trials = hp.rand.suggest(new_ids, rval.domain, trials, + random_state) + else: + new_trials = algorithm(new_ids, rval.domain, trials, random_state) + rval.trials.refresh() + vals = new_trials[0]['misc']['vals'] + parameter = dict() + for key in vals: + try: + parameter[key] = vals[key][0].item() + except (KeyError, IndexError): + parameter[key] = None + + # remove '_index' from json2parameter and save params-id + total_params = json2parameter(self.json, parameter) + return total_params + + def import_data(self, data): + """ + Import additional data for tuning + + Parameters + ---------- + data: + a list of dictionarys, each of which has at least two keys, 'parameter' and 'value' + """ + _completed_num = 0 + for trial_info in data: + logger.info("Importing data, current processing progress %s / %s", _completed_num, len(data)) + _completed_num += 1 + if self.algorithm_name == 'random_search': + return + assert "parameter" in trial_info + _params = trial_info["parameter"] + assert "value" in trial_info + _value = trial_info['value'] + if not _value: + logger.info("Useless trial data, value is %s, skip this trial data.", _value) + continue + self.supplement_data_num += 1 + _parameter_id = '_'.join( + ["ImportData", str(self.supplement_data_num)]) + self.total_data[_parameter_id] = _add_index(in_x=self.json, + parameter=_params) + self.receive_trial_result(parameter_id=_parameter_id, + parameters=_params, + value=_value) + logger.info("Successfully import data to TPE/Anneal tuner.") diff --git a/utils/third_party/nni/algorithms/hpo/medianstop_assessor/__init__.py b/utils/third_party/nni/algorithms/hpo/medianstop_assessor/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..47668dc26d43b777e001271defcca3c02a195071 --- /dev/null +++ b/utils/third_party/nni/algorithms/hpo/medianstop_assessor/__init__.py @@ -0,0 +1,4 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .medianstop_assessor import MedianstopAssessor diff --git a/utils/third_party/nni/algorithms/hpo/medianstop_assessor/medianstop_assessor.py b/utils/third_party/nni/algorithms/hpo/medianstop_assessor/medianstop_assessor.py new file mode 100644 index 0000000000000000000000000000000000000000..56eb82a3c98e0ce65efaa794fa6ebea8ee989566 --- /dev/null +++ b/utils/third_party/nni/algorithms/hpo/medianstop_assessor/medianstop_assessor.py @@ -0,0 +1,125 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +from schema import Schema, Optional + +from nni import ClassArgsValidator +from nni.assessor import Assessor, AssessResult +from nni.utils import extract_scalar_history + +logger = logging.getLogger('medianstop_Assessor') + +class MedianstopClassArgsValidator(ClassArgsValidator): + def validate_class_args(self, **kwargs): + Schema({ + Optional('optimize_mode'): self.choices('optimize_mode', 'maximize', 'minimize'), + Optional('start_step'): self.range('start_step', int, 0, 9999), + }).validate(kwargs) + +class MedianstopAssessor(Assessor): + """MedianstopAssessor is The median stopping rule stops a pending trial X at step S + if the trial’s best objective value by step S is strictly worse than the median value + of the running averages of all completed trials’ objectives reported up to step S + + Parameters + ---------- + optimize_mode : str + optimize mode, 'maximize' or 'minimize' + start_step : int + only after receiving start_step number of reported intermediate results + """ + def __init__(self, optimize_mode='maximize', start_step=0): + self._start_step = start_step + self._running_history = dict() + self._completed_avg_history = dict() + if optimize_mode == 'maximize': + self._high_better = True + elif optimize_mode == 'minimize': + self._high_better = False + else: + self._high_better = True + logger.warning('unrecognized optimize_mode %s', optimize_mode) + + def _update_data(self, trial_job_id, trial_history): + """update data + + Parameters + ---------- + trial_job_id : int + trial job id + trial_history : list + The history performance matrix of each trial + """ + if trial_job_id not in self._running_history: + self._running_history[trial_job_id] = [] + self._running_history[trial_job_id].extend(trial_history[len(self._running_history[trial_job_id]):]) + + def trial_end(self, trial_job_id, success): + """trial_end + + Parameters + ---------- + trial_job_id : int + trial job id + success : bool + True if succssfully finish the experiment, False otherwise + """ + if trial_job_id in self._running_history: + if success: + cnt = 0 + history_sum = 0 + self._completed_avg_history[trial_job_id] = [] + for each in self._running_history[trial_job_id]: + cnt += 1 + history_sum += each + self._completed_avg_history[trial_job_id].append(history_sum / cnt) + self._running_history.pop(trial_job_id) + else: + logger.warning('trial_end: trial_job_id does not exist in running_history') + + def assess_trial(self, trial_job_id, trial_history): + """assess_trial + + Parameters + ---------- + trial_job_id : int + trial job id + trial_history : list + The history performance matrix of each trial + + Returns + ------- + bool + AssessResult.Good or AssessResult.Bad + + Raises + ------ + Exception + unrecognize exception in medianstop_assessor + """ + curr_step = len(trial_history) + if curr_step < self._start_step: + return AssessResult.Good + + scalar_trial_history = extract_scalar_history(trial_history) + self._update_data(trial_job_id, scalar_trial_history) + if self._high_better: + best_history = max(scalar_trial_history) + else: + best_history = min(scalar_trial_history) + + avg_array = [] + for id_ in self._completed_avg_history: + if len(self._completed_avg_history[id_]) >= curr_step: + avg_array.append(self._completed_avg_history[id_][curr_step - 1]) + if avg_array: + avg_array.sort() + if self._high_better: + median = avg_array[(len(avg_array)-1) // 2] + return AssessResult.Bad if best_history < median else AssessResult.Good + else: + median = avg_array[len(avg_array) // 2] + return AssessResult.Bad if best_history > median else AssessResult.Good + else: + return AssessResult.Good diff --git a/utils/third_party/nni/algorithms/hpo/medianstop_assessor/test.py b/utils/third_party/nni/algorithms/hpo/medianstop_assessor/test.py new file mode 100644 index 0000000000000000000000000000000000000000..8c7d6d927b6b514f7f0117d4424ccceaca92ae6c --- /dev/null +++ b/utils/third_party/nni/algorithms/hpo/medianstop_assessor/test.py @@ -0,0 +1,54 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import argparse +import logging +import random + +from .medianstop_assessor import MedianstopAssessor +from nni.assessor import AssessResult + +logger = logging.getLogger('nni.contrib.medianstop_assessor') +logger.debug('START') + + +def test(): + ''' + tests. + ''' + parser = argparse.ArgumentParser(description='parse command line parameters.') + parser.add_argument('--start_from', type=int, default=10, dest='start_step', + help='Assessing each trial from the step start_step.') + parser.add_argument('--optimize_mode', type=str, default='maximize', + help='Select optimize mode for Tuner: minimize or maximize.') + FLAGS, _ = parser.parse_known_args() + + lcs = [[1,1,1,1,1,1,1,1,1,1], + [2,2,2,2,2,2,2,2,2,2], + [3,3,3,3,3,3,3,3,3,3], + [4,4,4,4,4,4,4,4,4,4]] + #lcs = [[1,1,1,1,1,1,1,1,1,1], + # [1,1,1,1,1,1,1,1,1,1], + # [1,1,1,1,1,1,1,1,1,1]] + + assessor = MedianstopAssessor(FLAGS.optimize_mode, FLAGS.start_step) + for i in range(len(lcs)): + #lc = [] + to_complete = True + for k in range(len(lcs[0])): + #d = random.randint(i*100+0, i*100+100) + #lc.append(d) + ret = assessor.assess_trial(i, lcs[i][:k+1]) + print('result: %d', ret) + if ret == AssessResult.Bad: + assessor.trial_end(i, False) + to_complete = False + break + if to_complete: + assessor.trial_end(i, True) + +try: + test() +except Exception as exception: + logger.exception(exception) + raise diff --git a/utils/third_party/nni/algorithms/hpo/metis_tuner/Regression_GMM/CreateModel.py b/utils/third_party/nni/algorithms/hpo/metis_tuner/Regression_GMM/CreateModel.py new file mode 100644 index 0000000000000000000000000000000000000000..4846c9a67d1d4579456e5c28ce416b9e274566b4 --- /dev/null +++ b/utils/third_party/nni/algorithms/hpo/metis_tuner/Regression_GMM/CreateModel.py @@ -0,0 +1,44 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +import sys +from operator import itemgetter + +import sklearn.mixture as mm + +sys.path.insert(1, os.path.join(sys.path[0], '..')) + + +def create_model(samples_x, samples_y_aggregation, percentage_goodbatch=0.34): + ''' + Create the Gaussian Mixture Model + ''' + samples = [samples_x[i] + [samples_y_aggregation[i]] + for i in range(0, len(samples_x))] + + # Sorts so that we can get the top samples + samples = sorted(samples, key=itemgetter(-1)) + samples_goodbatch_size = int(len(samples) * percentage_goodbatch) + samples_goodbatch = samples[0:samples_goodbatch_size] + samples_badbatch = samples[samples_goodbatch_size:] + + samples_x_goodbatch = [sample_goodbatch[0:-1] + for sample_goodbatch in samples_goodbatch] + #samples_y_goodbatch = [sample_goodbatch[-1] for sample_goodbatch in samples_goodbatch] + samples_x_badbatch = [sample_badbatch[0:-1] + for sample_badbatch in samples_badbatch] + + # === Trains GMM clustering models === # + #sys.stderr.write("[%s] Train GMM's GMM model\n" % (os.path.basename(__file__))) + bgmm_goodbatch = mm.BayesianGaussianMixture( + n_components=max(1, samples_goodbatch_size - 1)) + bad_n_components = max(1, len(samples_x) - samples_goodbatch_size - 1) + bgmm_badbatch = mm.BayesianGaussianMixture(n_components=bad_n_components) + bgmm_goodbatch.fit(samples_x_goodbatch) + bgmm_badbatch.fit(samples_x_badbatch) + + model = {} + model['clusteringmodel_good'] = bgmm_goodbatch + model['clusteringmodel_bad'] = bgmm_badbatch + return model diff --git a/utils/third_party/nni/algorithms/hpo/metis_tuner/Regression_GMM/Selection.py b/utils/third_party/nni/algorithms/hpo/metis_tuner/Regression_GMM/Selection.py new file mode 100644 index 0000000000000000000000000000000000000000..0bca96647d4642afc7d46c8a259f1634b4c41d81 --- /dev/null +++ b/utils/third_party/nni/algorithms/hpo/metis_tuner/Regression_GMM/Selection.py @@ -0,0 +1,93 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +import random +import sys + +from .. import lib_acquisition_function +from .. import lib_constraint_summation + +sys.path.insert(1, os.path.join(sys.path[0], '..')) + + +CONSTRAINT_LOWERBOUND = None +CONSTRAINT_UPPERBOUND = None +CONSTRAINT_PARAMS_IDX = [] + + +def _ratio_scores(parameters_value, clusteringmodel_gmm_good, + clusteringmodel_gmm_bad): + ''' + The ratio is smaller the better + ''' + ratio = clusteringmodel_gmm_good.score( + [parameters_value]) / clusteringmodel_gmm_bad.score([parameters_value]) + sigma = 0 + return ratio, sigma + + +def selection_r(x_bounds, + x_types, + clusteringmodel_gmm_good, + clusteringmodel_gmm_bad, + num_starting_points=100, + minimize_constraints_fun=None): + ''' + Select using different types. + ''' + minimize_starting_points = clusteringmodel_gmm_good.sample(n_samples=num_starting_points) + + outputs = selection(x_bounds, x_types, + clusteringmodel_gmm_good, + clusteringmodel_gmm_bad, + minimize_starting_points[0], + minimize_constraints_fun) + + return outputs + + +def selection(x_bounds, + x_types, + clusteringmodel_gmm_good, + clusteringmodel_gmm_bad, + minimize_starting_points, + minimize_constraints_fun=None): + ''' + Select the lowest mu value + ''' + results = lib_acquisition_function.next_hyperparameter_lowest_mu( + _ratio_scores, [clusteringmodel_gmm_good, clusteringmodel_gmm_bad], + x_bounds, x_types, minimize_starting_points, + minimize_constraints_fun=minimize_constraints_fun) + + return results + + +def _rand_with_constraints(x_bounds, x_types): + ''' + Random generate the variable with constraints + ''' + outputs = None + x_bounds_withconstraints = [x_bounds[i] for i in CONSTRAINT_PARAMS_IDX] + x_types_withconstraints = [x_types[i] for i in CONSTRAINT_PARAMS_IDX] + x_val_withconstraints = lib_constraint_summation.rand(x_bounds_withconstraints, + x_types_withconstraints, + CONSTRAINT_LOWERBOUND, + CONSTRAINT_UPPERBOUND) + if x_val_withconstraints is not None: + outputs = [None] * len(x_bounds) + for i, _ in enumerate(CONSTRAINT_PARAMS_IDX): + outputs[CONSTRAINT_PARAMS_IDX[i]] = x_val_withconstraints[i] + for i, _ in enumerate(outputs): + if outputs[i] is None: + outputs[i] = random.randint(x_bounds[i][0], x_bounds[i][1]) + return outputs + + +def _minimize_constraints_fun_summation(x): + ''' + Minimize constraints fun summation + ''' + summation = sum([x[i] for i in CONSTRAINT_PARAMS_IDX]) + return CONSTRAINT_UPPERBOUND >= summation >= CONSTRAINT_LOWERBOUND diff --git a/utils/third_party/nni/algorithms/hpo/metis_tuner/Regression_GMM/__init__.py b/utils/third_party/nni/algorithms/hpo/metis_tuner/Regression_GMM/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/utils/third_party/nni/algorithms/hpo/metis_tuner/Regression_GP/CreateModel.py b/utils/third_party/nni/algorithms/hpo/metis_tuner/Regression_GP/CreateModel.py new file mode 100644 index 0000000000000000000000000000000000000000..98d1f22ce3c62c0491f91138e8c1df7ebd5e65ed --- /dev/null +++ b/utils/third_party/nni/algorithms/hpo/metis_tuner/Regression_GP/CreateModel.py @@ -0,0 +1,35 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +import sys +import numpy + +import sklearn.gaussian_process as gp + +sys.path.insert(1, os.path.join(sys.path[0], '..')) + + +def create_model(samples_x, samples_y_aggregation, + n_restarts_optimizer=250, is_white_kernel=False): + ''' + Trains GP regression model + ''' + kernel = gp.kernels.ConstantKernel(constant_value=1, + constant_value_bounds=(1e-12, 1e12)) * \ + gp.kernels.Matern(nu=1.5) + if is_white_kernel is True: + kernel += gp.kernels.WhiteKernel(noise_level=1, noise_level_bounds=(1e-12, 1e12)) + regressor = gp.GaussianProcessRegressor(kernel=kernel, + n_restarts_optimizer=n_restarts_optimizer, + normalize_y=True, + alpha=1e-10) + regressor.fit(numpy.array(samples_x), numpy.array(samples_y_aggregation)) + + model = {} + model['model'] = regressor + model['kernel_prior'] = str(kernel) + model['kernel_posterior'] = str(regressor.kernel_) + model['model_loglikelihood'] = regressor.log_marginal_likelihood(regressor.kernel_.theta) + + return model diff --git a/utils/third_party/nni/algorithms/hpo/metis_tuner/Regression_GP/OutlierDetection.py b/utils/third_party/nni/algorithms/hpo/metis_tuner/Regression_GP/OutlierDetection.py new file mode 100644 index 0000000000000000000000000000000000000000..f07a93dd3e80a8fd6f9b0e8ceb3e87a5338e915d --- /dev/null +++ b/utils/third_party/nni/algorithms/hpo/metis_tuner/Regression_GP/OutlierDetection.py @@ -0,0 +1,87 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +OutlierDectection.py +""" + +import os +import sys +from multiprocessing.dummy import Pool as ThreadPool + +from . import CreateModel as gp_create_model +from . import Prediction as gp_prediction + +sys.path.insert(1, os.path.join(sys.path[0], '..')) + + +def _outlierDetection_threaded(inputs): + """ + Detect the outlier + """ + [samples_idx, samples_x, samples_y_aggregation] = inputs + sys.stderr.write("[%s] DEBUG: Evaluating %dth of %d samples\n" + % (os.path.basename(__file__), samples_idx + 1, len(samples_x))) + outlier = None + + # Create a diagnostic regression model which removes the sample that we + # want to evaluate + diagnostic_regressor_gp = gp_create_model.create_model( + samples_x[0:samples_idx] + samples_x[samples_idx + 1:], + samples_y_aggregation[0:samples_idx] + samples_y_aggregation[samples_idx + 1:]) + mu, sigma = gp_prediction.predict( + samples_x[samples_idx], diagnostic_regressor_gp['model']) + + # 2.33 is the z-score for 98% confidence level + if abs(samples_y_aggregation[samples_idx] - mu) > (2.33 * sigma): + outlier = {"samples_idx": samples_idx, + "expected_mu": mu, + "expected_sigma": sigma, + "difference": abs(samples_y_aggregation[samples_idx] - mu) - (2.33 * sigma)} + return outlier + + +def outlierDetection_threaded(samples_x, samples_y_aggregation): + """ + Use Multi-thread to detect the outlier + """ + outliers = [] + + threads_inputs = [[samples_idx, samples_x, samples_y_aggregation] + for samples_idx in range(0, len(samples_x))] + threads_pool = ThreadPool(min(4, len(threads_inputs))) + threads_results = threads_pool.map( + _outlierDetection_threaded, threads_inputs) + threads_pool.close() + threads_pool.join() + + for threads_result in threads_results: + if threads_result is not None: + outliers.append(threads_result) + else: + print("Error: threads_result is None.") + + outliers = outliers if outliers else None + return outliers + + +def outlierDetection(samples_x, samples_y_aggregation): + outliers = [] + for samples_idx, _ in enumerate(samples_x): + #sys.stderr.write("[%s] DEBUG: Evaluating %d of %d samples\n" + # \ % (os.path.basename(__file__), samples_idx + 1, len(samples_x))) + diagnostic_regressor_gp = gp_create_model.create_model(\ + samples_x[0:samples_idx] + samples_x[samples_idx + 1:],\ + samples_y_aggregation[0:samples_idx] + samples_y_aggregation[samples_idx + 1:]) + mu, sigma = gp_prediction.predict(samples_x[samples_idx], + diagnostic_regressor_gp['model']) + # 2.33 is the z-score for 98% confidence level + if abs(samples_y_aggregation[samples_idx] - mu) > (2.33 * sigma): + outliers.append({"samples_idx": samples_idx, + "expected_mu": mu, + "expected_sigma": sigma, + "difference": \ + abs(samples_y_aggregation[samples_idx] - mu) - (2.33 * sigma)}) + + outliers = outliers if outliers else None + return outliers diff --git a/utils/third_party/nni/algorithms/hpo/metis_tuner/Regression_GP/Prediction.py b/utils/third_party/nni/algorithms/hpo/metis_tuner/Regression_GP/Prediction.py new file mode 100644 index 0000000000000000000000000000000000000000..08a32f0e63c86a6fd7dddad558f96c708916e41e --- /dev/null +++ b/utils/third_party/nni/algorithms/hpo/metis_tuner/Regression_GP/Prediction.py @@ -0,0 +1,19 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +import sys + +import numpy + +sys.path.insert(1, os.path.join(sys.path[0], '..')) + + +def predict(parameters_value, regressor_gp): + ''' + Predict by Gaussian Process Model + ''' + parameters_value = numpy.array(parameters_value).reshape(-1, len(parameters_value)) + mu, sigma = regressor_gp.predict(parameters_value, return_std=True) + + return mu[0], sigma[0] diff --git a/utils/third_party/nni/algorithms/hpo/metis_tuner/Regression_GP/Selection.py b/utils/third_party/nni/algorithms/hpo/metis_tuner/Regression_GP/Selection.py new file mode 100644 index 0000000000000000000000000000000000000000..30a41deec17eec6cd6ddf187f1a23e84a38f57e4 --- /dev/null +++ b/utils/third_party/nni/algorithms/hpo/metis_tuner/Regression_GP/Selection.py @@ -0,0 +1,97 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +import random +import sys + +from .. import lib_acquisition_function +from .. import lib_constraint_summation +from .. import lib_data +from . import Prediction as gp_prediction + +sys.path.insert(1, os.path.join(sys.path[0], '..')) + +CONSTRAINT_LOWERBOUND = None +CONSTRAINT_UPPERBOUND = None +CONSTRAINT_PARAMS_IDX = [] + + +def selection_r(acquisition_function, + samples_y_aggregation, + x_bounds, + x_types, + regressor_gp, + num_starting_points=100, + minimize_constraints_fun=None): + ''' + Selecte R value + ''' + minimize_starting_points = [lib_data.rand(x_bounds, x_types) \ + for i in range(0, num_starting_points)] + outputs = selection(acquisition_function, samples_y_aggregation, + x_bounds, x_types, regressor_gp, + minimize_starting_points, + minimize_constraints_fun=minimize_constraints_fun) + + return outputs + +def selection(acquisition_function, + samples_y_aggregation, + x_bounds, x_types, + regressor_gp, + minimize_starting_points, + minimize_constraints_fun=None): + ''' + selection + ''' + outputs = None + + sys.stderr.write("[%s] Exercise \"%s\" acquisition function\n" \ + % (os.path.basename(__file__), acquisition_function)) + + if acquisition_function == "ei": + outputs = lib_acquisition_function.next_hyperparameter_expected_improvement(\ + gp_prediction.predict, [regressor_gp], x_bounds, x_types, \ + samples_y_aggregation, minimize_starting_points, \ + minimize_constraints_fun=minimize_constraints_fun) + elif acquisition_function == "lc": + outputs = lib_acquisition_function.next_hyperparameter_lowest_confidence(\ + gp_prediction.predict, [regressor_gp], x_bounds, x_types,\ + minimize_starting_points, minimize_constraints_fun=minimize_constraints_fun) + elif acquisition_function == "lm": + outputs = lib_acquisition_function.next_hyperparameter_lowest_mu(\ + gp_prediction.predict, [regressor_gp], x_bounds, x_types,\ + minimize_starting_points, minimize_constraints_fun=minimize_constraints_fun) + return outputs + +def _rand_with_constraints(x_bounds, x_types): + ''' + Random generate with constraints + ''' + outputs = None + + x_bounds_withconstraints = [x_bounds[i] for i in CONSTRAINT_PARAMS_IDX] + x_types_withconstraints = [x_types[i] for i in CONSTRAINT_PARAMS_IDX] + x_val_withconstraints = lib_constraint_summation.rand(x_bounds_withconstraints, + x_types_withconstraints, + CONSTRAINT_LOWERBOUND, + CONSTRAINT_UPPERBOUND) + if x_val_withconstraints is not None: + outputs = [None] * len(x_bounds) + + for i, _ in enumerate(CONSTRAINT_PARAMS_IDX): + outputs[CONSTRAINT_PARAMS_IDX[i]] = x_val_withconstraints[i] + + for i, _ in enumerate(outputs): + if outputs[i] is None: + outputs[i] = random.randint(x_bounds[i][0], x_bounds[i][1]) + return outputs + + +def _minimize_constraints_fun_summation(x): + ''' + Minimize the constraints fun summation + ''' + summation = sum([x[i] for i in CONSTRAINT_PARAMS_IDX]) + return CONSTRAINT_UPPERBOUND >= summation >= CONSTRAINT_LOWERBOUND diff --git a/utils/third_party/nni/algorithms/hpo/metis_tuner/Regression_GP/__init__.py b/utils/third_party/nni/algorithms/hpo/metis_tuner/Regression_GP/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/utils/third_party/nni/algorithms/hpo/metis_tuner/__init__.py b/utils/third_party/nni/algorithms/hpo/metis_tuner/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/utils/third_party/nni/algorithms/hpo/metis_tuner/lib_acquisition_function.py b/utils/third_party/nni/algorithms/hpo/metis_tuner/lib_acquisition_function.py new file mode 100644 index 0000000000000000000000000000000000000000..f1b1edfe0165b61c18753dfa823bc2bc10aa8d05 --- /dev/null +++ b/utils/third_party/nni/algorithms/hpo/metis_tuner/lib_acquisition_function.py @@ -0,0 +1,197 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +lib_acquisition_function.py +""" + +import sys +import numpy + +from scipy.stats import norm +from scipy.optimize import minimize + +from . import lib_data + + +def next_hyperparameter_expected_improvement(fun_prediction, + fun_prediction_args, + x_bounds, x_types, + samples_y_aggregation, + minimize_starting_points, + minimize_constraints_fun=None): + """ + "Expected Improvement" acquisition function + """ + best_x = None + best_acquisition_value = None + x_bounds_minmax = [[i[0], i[-1]] for i in x_bounds] + x_bounds_minmax = numpy.array(x_bounds_minmax) + + for starting_point in numpy.array(minimize_starting_points): + res = minimize(fun=_expected_improvement, + x0=starting_point.reshape(1, -1), + bounds=x_bounds_minmax, + method="L-BFGS-B", + args=(fun_prediction, + fun_prediction_args, + x_bounds, + x_types, + samples_y_aggregation, + minimize_constraints_fun)) + + if (best_acquisition_value is None) or \ + (res.fun < best_acquisition_value): + res.x = numpy.ndarray.tolist(res.x) + res.x = lib_data.match_val_type(res.x, x_bounds, x_types) + if (minimize_constraints_fun is None) or \ + (minimize_constraints_fun(res.x) is True): + best_acquisition_value = res.fun + best_x = res.x + + outputs = None + if best_x is not None: + mu, sigma = fun_prediction(best_x, *fun_prediction_args) + outputs = {'hyperparameter': best_x, 'expected_mu': mu, + 'expected_sigma': sigma, 'acquisition_func': "ei"} + + return outputs + + +def _expected_improvement(x, fun_prediction, fun_prediction_args, + x_bounds, x_types, samples_y_aggregation, + minimize_constraints_fun): + # This is only for step-wise optimization + x = lib_data.match_val_type(x, x_bounds, x_types) + + expected_improvement = sys.maxsize + if (minimize_constraints_fun is None) or ( + minimize_constraints_fun(x) is True): + mu, sigma = fun_prediction(x, *fun_prediction_args) + + loss_optimum = min(samples_y_aggregation) + scaling_factor = -1 + + # In case sigma equals zero + with numpy.errstate(divide="ignore"): + Z = scaling_factor * (mu - loss_optimum) / sigma + expected_improvement = scaling_factor * (mu - loss_optimum) * \ + norm.cdf(Z) + sigma * norm.pdf(Z) + expected_improvement = 0.0 if sigma == 0.0 else expected_improvement + + # We want expected_improvement to be as large as possible + # (i.e., as small as possible for minimize(...)) + expected_improvement = -1 * expected_improvement + return expected_improvement + + +def next_hyperparameter_lowest_confidence(fun_prediction, + fun_prediction_args, + x_bounds, x_types, + minimize_starting_points, + minimize_constraints_fun=None): + """ + "Lowest Confidence" acquisition function + """ + best_x = None + best_acquisition_value = None + x_bounds_minmax = [[i[0], i[-1]] for i in x_bounds] + x_bounds_minmax = numpy.array(x_bounds_minmax) + + for starting_point in numpy.array(minimize_starting_points): + res = minimize(fun=_lowest_confidence, + x0=starting_point.reshape(1, -1), + bounds=x_bounds_minmax, + method="L-BFGS-B", + args=(fun_prediction, + fun_prediction_args, + x_bounds, + x_types, + minimize_constraints_fun)) + + if (best_acquisition_value) is None or ( + res.fun < best_acquisition_value): + res.x = numpy.ndarray.tolist(res.x) + res.x = lib_data.match_val_type(res.x, x_bounds, x_types) + if (minimize_constraints_fun is None) or ( + minimize_constraints_fun(res.x) is True): + best_acquisition_value = res.fun + best_x = res.x + + outputs = None + if best_x is not None: + mu, sigma = fun_prediction(best_x, *fun_prediction_args) + outputs = {'hyperparameter': best_x, 'expected_mu': mu, + 'expected_sigma': sigma, 'acquisition_func': "lc"} + return outputs + + +def _lowest_confidence(x, fun_prediction, fun_prediction_args, + x_bounds, x_types, minimize_constraints_fun): + # This is only for step-wise optimization + x = lib_data.match_val_type(x, x_bounds, x_types) + + ci = sys.maxsize + if (minimize_constraints_fun is None) or ( + minimize_constraints_fun(x) is True): + mu, sigma = fun_prediction(x, *fun_prediction_args) + ci = (sigma * 1.96 * 2) / mu + # We want ci to be as large as possible + # (i.e., as small as possible for minimize(...), + # because this would mean lowest confidence + ci = -1 * ci + + return ci + + +def next_hyperparameter_lowest_mu(fun_prediction, + fun_prediction_args, + x_bounds, x_types, + minimize_starting_points, + minimize_constraints_fun=None): + """ + "Lowest Mu" acquisition function + """ + best_x = None + best_acquisition_value = None + x_bounds_minmax = [[i[0], i[-1]] for i in x_bounds] + x_bounds_minmax = numpy.array(x_bounds_minmax) + + for starting_point in numpy.array(minimize_starting_points): + res = minimize(fun=_lowest_mu, + x0=starting_point.reshape(1, -1), + bounds=x_bounds_minmax, + method="L-BFGS-B", + args=(fun_prediction, fun_prediction_args, + x_bounds, x_types, minimize_constraints_fun)) + + if (best_acquisition_value is None) or ( + res.fun < best_acquisition_value): + res.x = numpy.ndarray.tolist(res.x) + res.x = lib_data.match_val_type(res.x, x_bounds, x_types) + if (minimize_constraints_fun is None) or ( + minimize_constraints_fun(res.x) is True): + best_acquisition_value = res.fun + best_x = res.x + + outputs = None + if best_x is not None: + mu, sigma = fun_prediction(best_x, *fun_prediction_args) + outputs = {'hyperparameter': best_x, 'expected_mu': mu, + 'expected_sigma': sigma, 'acquisition_func': "lm"} + return outputs + + +def _lowest_mu(x, fun_prediction, fun_prediction_args, + x_bounds, x_types, minimize_constraints_fun): + """ + Calculate the lowest mu + """ + # This is only for step-wise optimization + x = lib_data.match_val_type(x, x_bounds, x_types) + + mu = sys.maxsize + if (minimize_constraints_fun is None) or ( + minimize_constraints_fun(x) is True): + mu, _ = fun_prediction(x, *fun_prediction_args) + return mu diff --git a/utils/third_party/nni/algorithms/hpo/metis_tuner/lib_constraint_summation.py b/utils/third_party/nni/algorithms/hpo/metis_tuner/lib_constraint_summation.py new file mode 100644 index 0000000000000000000000000000000000000000..948be5b3ca3dcace3ae673ce9e12667c093ca7a4 --- /dev/null +++ b/utils/third_party/nni/algorithms/hpo/metis_tuner/lib_constraint_summation.py @@ -0,0 +1,108 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +lib_constraint_summation.py +""" + +import math +import random + +from operator import itemgetter + + +def check_feasibility(x_bounds, lowerbound, upperbound): + ''' + This can have false positives. + For examples, parameters can only be 0 or 5, and the summation constraint is between 6 and 7. + ''' + # x_bounds should be sorted, so even for "discrete_int" type, + # the smallest and the largest number should the first and the last element + x_bounds_lowerbound = sum([x_bound[0] for x_bound in x_bounds]) + x_bounds_upperbound = sum([x_bound[-1] for x_bound in x_bounds]) + + # return ((x_bounds_lowerbound <= lowerbound) and (x_bounds_upperbound >= lowerbound)) or \ + # ((x_bounds_lowerbound <= upperbound) and (x_bounds_upperbound >= upperbound)) + return (x_bounds_lowerbound <= lowerbound <= x_bounds_upperbound) or \ + (x_bounds_lowerbound <= upperbound <= x_bounds_upperbound) + + +def rand(x_bounds, x_types, lowerbound, upperbound, max_retries=100): + ''' + Key idea is that we try to move towards upperbound, by randomly choose one + value for each parameter. However, for the last parameter, + we need to make sure that its value can help us get above lowerbound + ''' + outputs = None + + if check_feasibility(x_bounds, lowerbound, upperbound) is True: + # Order parameters by their range size. We want the smallest range first, + # because the corresponding parameter has less numbers to choose from + x_idx_sorted = [] + for i, _ in enumerate(x_bounds): + if x_types[i] == "discrete_int": + x_idx_sorted.append([i, len(x_bounds[i])]) + elif (x_types[i] == "range_int") or (x_types[i] == "range_continuous"): + x_idx_sorted.append( + [i, math.floor(x_bounds[i][1] - x_bounds[i][0])]) + x_idx_sorted = sorted(x_idx_sorted, key=itemgetter(1)) + + for _ in range(max_retries): + budget_allocated = 0 + outputs = [None] * len(x_bounds) + + for i, _ in enumerate(x_idx_sorted): + x_idx = x_idx_sorted[i][0] + # The amount of unallocated space that we have + budget_max = upperbound - budget_allocated + # NOT the Last x that we need to assign a random number + if i < (len(x_idx_sorted) - 1): + if x_bounds[x_idx][0] <= budget_max: + if x_types[x_idx] == "discrete_int": + # Note the valid integer + temp = [] + for j in x_bounds[x_idx]: + if j <= budget_max: + temp.append(j) + # Randomly pick a number from the integer array + if temp: + outputs[x_idx] = temp[random.randint( + 0, len(temp) - 1)] + + elif (x_types[x_idx] == "range_int") or \ + (x_types[x_idx] == "range_continuous"): + outputs[x_idx] = random.randint( + x_bounds[x_idx][0], min(x_bounds[x_idx][-1], budget_max)) + + else: + # The last x that we need to assign a random number + randint_lowerbound = lowerbound - budget_allocated + randint_lowerbound = 0 if randint_lowerbound < 0 else randint_lowerbound + + # This check: + # is our smallest possible value going to overflow the available budget space, + # and is our largest possible value going to underflow the + # lower bound + if (x_bounds[x_idx][0] <= budget_max) and \ + (x_bounds[x_idx][-1] >= randint_lowerbound): + if x_types[x_idx] == "discrete_int": + temp = [] + for j in x_bounds[x_idx]: + # if (j <= budget_max) and (j >= + # randint_lowerbound): + if randint_lowerbound <= j <= budget_max: + temp.append(j) + if temp: + outputs[x_idx] = temp[random.randint( + 0, len(temp) - 1)] + elif (x_types[x_idx] == "range_int") or \ + (x_types[x_idx] == "range_continuous"): + outputs[x_idx] = random.randint( + randint_lowerbound, min( + x_bounds[x_idx][1], budget_max)) + if outputs[x_idx] is None: + break + budget_allocated += outputs[x_idx] + if None not in outputs: + break + return outputs diff --git a/utils/third_party/nni/algorithms/hpo/metis_tuner/lib_data.py b/utils/third_party/nni/algorithms/hpo/metis_tuner/lib_data.py new file mode 100644 index 0000000000000000000000000000000000000000..cc3f059514c8c30312acbd0c18ec000803cbe0ad --- /dev/null +++ b/utils/third_party/nni/algorithms/hpo/metis_tuner/lib_data.py @@ -0,0 +1,50 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import math +import random + + +def match_val_type(vals, vals_bounds, vals_types): + ''' + Update values in the array, to match their corresponding type + ''' + vals_new = [] + + for i, _ in enumerate(vals_types): + if vals_types[i] == "discrete_int": + # Find the closest integer in the array, vals_bounds + # pylint: disable=cell-var-from-loop + vals_new.append(min(vals_bounds[i], key=lambda x: abs(x - vals[i]))) + elif vals_types[i] == "range_int": + # Round down to the nearest integer + vals_new.append(math.floor(vals[i])) + elif vals_types[i] == "range_continuous": + # Don't do any processing for continous numbers + vals_new.append(vals[i]) + else: + return None + + return vals_new + + +def rand(x_bounds, x_types): + ''' + Random generate variable value within their bounds + ''' + outputs = [] + + for i, _ in enumerate(x_bounds): + if x_types[i] == "discrete_int": + temp = x_bounds[i][random.randint(0, len(x_bounds[i]) - 1)] + outputs.append(temp) + elif x_types[i] == "range_int": + temp = random.randint(x_bounds[i][0], x_bounds[i][1] - 1) + outputs.append(temp) + elif x_types[i] == "range_continuous": + temp = random.uniform(x_bounds[i][0], x_bounds[i][1]) + outputs.append(temp) + else: + return None + + return outputs diff --git a/utils/third_party/nni/algorithms/hpo/metis_tuner/metis_tuner.py b/utils/third_party/nni/algorithms/hpo/metis_tuner/metis_tuner.py new file mode 100644 index 0000000000000000000000000000000000000000..1a0670f5e517d3ce1cdf775a2590daea7366f4d9 --- /dev/null +++ b/utils/third_party/nni/algorithms/hpo/metis_tuner/metis_tuner.py @@ -0,0 +1,650 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +metis_tuner.py +""" + +import copy +import logging +import random +import statistics +import warnings +from multiprocessing.dummy import Pool as ThreadPool +import numpy as np +from schema import Schema, Optional + +from nni import ClassArgsValidator +from . import lib_constraint_summation +from . import lib_data +from .Regression_GMM import CreateModel as gmm_create_model +from .Regression_GMM import Selection as gmm_selection +from .Regression_GP import CreateModel as gp_create_model +from .Regression_GP import OutlierDetection as gp_outlier_detection +from .Regression_GP import Prediction as gp_prediction +from .Regression_GP import Selection as gp_selection +from nni.tuner import Tuner +from nni.utils import OptimizeMode, extract_scalar_reward + +logger = logging.getLogger("Metis_Tuner_AutoML") + +NONE_TYPE = '' +CONSTRAINT_LOWERBOUND = None +CONSTRAINT_UPPERBOUND = None +CONSTRAINT_PARAMS_IDX = [] + +class MetisClassArgsValidator(ClassArgsValidator): + def validate_class_args(self, **kwargs): + Schema({ + Optional('optimize_mode'): self.choices('optimize_mode', 'maximize', 'minimize'), + Optional('no_resampling'): bool, + Optional('no_candidates'): bool, + Optional('selection_num_starting_points'): int, + Optional('cold_start_num'): int, + }).validate(kwargs) + +class MetisTuner(Tuner): + """ + Metis Tuner + + More algorithm information you could reference here: + https://www.microsoft.com/en-us/research/publication/metis-robustly-tuning-tail-latencies-cloud-systems/ + + Attributes + ---------- + optimize_mode : str + optimize_mode is a string that including two mode "maximize" and "minimize" + + no_resampling : bool + True or False. + Should Metis consider re-sampling as part of the search strategy? + If you are confident that the training dataset is noise-free, + then you do not need re-sampling. + + no_candidates : bool + True or False. + Should Metis suggest parameters for the next benchmark? + If you do not plan to do more benchmarks, + Metis can skip this step. + + selection_num_starting_points : int + How many times Metis should try to find the global optimal in the search space? + The higher the number, the longer it takes to output the solution. + + cold_start_num : int + Metis need some trial result to get cold start. + when the number of trial result is less than + cold_start_num, Metis will randomly sample hyper-parameter for trial. + + exploration_probability: float + The probability of Metis to select parameter from exploration instead of exploitation. + """ + + def __init__( + self, + optimize_mode="maximize", + no_resampling=True, + no_candidates=False, + selection_num_starting_points=600, + cold_start_num=10, + exploration_probability=0.9): + """ + Parameters + ---------- + optimize_mode : str + optimize_mode is a string that including two mode "maximize" and "minimize" + + no_resampling : bool + True or False. + Should Metis consider re-sampling as part of the search strategy? + If you are confident that the training dataset is noise-free, + then you do not need re-sampling. + + no_candidates : bool + True or False. + Should Metis suggest parameters for the next benchmark? + If you do not plan to do more benchmarks, + Metis can skip this step. + + selection_num_starting_points : int + How many times Metis should try to find the global optimal in the search space? + The higher the number, the longer it takes to output the solution. + + cold_start_num : int + Metis need some trial result to get cold start. + when the number of trial result is less than + cold_start_num, Metis will randomly sample hyper-parameter for trial. + + exploration_probability : float + The probability of Metis to select parameter from exploration instead of exploitation. + + x_bounds : list + The constration of parameters. + + x_types : list + The type of parameters. + """ + + self.samples_x = [] + self.samples_y = [] + self.samples_y_aggregation = [] + self.total_data = [] + self.space = None + self.no_resampling = no_resampling + self.no_candidates = no_candidates + self.optimize_mode = OptimizeMode(optimize_mode) + self.key_order = [] + self.cold_start_num = cold_start_num + self.selection_num_starting_points = selection_num_starting_points + self.exploration_probability = exploration_probability + self.minimize_constraints_fun = None + self.minimize_starting_points = None + self.supplement_data_num = 0 + self.x_bounds = [] + self.x_types = [] + + + def update_search_space(self, search_space): + """ + Update the self.x_bounds and self.x_types by the search_space.json + + Parameters + ---------- + search_space : dict + """ + self.x_bounds = [[] for i in range(len(search_space))] + self.x_types = [NONE_TYPE for i in range(len(search_space))] + + for key in search_space: + self.key_order.append(key) + + key_type = {} + if isinstance(search_space, dict): + for key in search_space: + key_type = search_space[key]['_type'] + key_range = search_space[key]['_value'] + idx = self.key_order.index(key) + if key_type == 'quniform': + if key_range[2] == 1 and key_range[0].is_integer( + ) and key_range[1].is_integer(): + self.x_bounds[idx] = [key_range[0], key_range[1] + 1] + self.x_types[idx] = 'range_int' + else: + low, high, q = key_range + bounds = np.clip( + np.arange( + np.round( + low / q), + np.round( + high / q) + 1) * q, + low, + high) + self.x_bounds[idx] = bounds + self.x_types[idx] = 'discrete_int' + elif key_type == 'randint': + self.x_bounds[idx] = [key_range[0], key_range[1]] + self.x_types[idx] = 'range_int' + elif key_type == 'uniform': + self.x_bounds[idx] = [key_range[0], key_range[1]] + self.x_types[idx] = 'range_continuous' + elif key_type == 'choice': + self.x_bounds[idx] = key_range + + for key_value in key_range: + if not isinstance(key_value, (int, float)): + raise RuntimeError( + "Metis Tuner only support numerical choice.") + + self.x_types[idx] = 'discrete_int' + else: + logger.info( + "Metis Tuner doesn't support this kind of variable: %s", + str(key_type)) + raise RuntimeError( + "Metis Tuner doesn't support this kind of variable: %s" % + str(key_type)) + else: + logger.info("The format of search space is not a dict.") + raise RuntimeError("The format of search space is not a dict.") + + self.minimize_starting_points = _rand_init( + self.x_bounds, self.x_types, self.selection_num_starting_points) + + + def _pack_output(self, init_parameter): + """ + Pack the output + + Parameters + ---------- + init_parameter : dict + + Returns + ------- + output : dict + """ + output = {} + for i, param in enumerate(init_parameter): + output[self.key_order[i]] = param + + return output + + + def generate_parameters(self, parameter_id, **kwargs): + """ + Generate next parameter for trial + + If the number of trial result is lower than cold start number, + metis will first random generate some parameters. + Otherwise, metis will choose the parameters by + the Gussian Process Model and the Gussian Mixture Model. + + Parameters + ---------- + parameter_id : int + + Returns + ------- + result : dict + """ + if len(self.samples_x) < self.cold_start_num: + init_parameter = _rand_init(self.x_bounds, self.x_types, 1)[0] + results = self._pack_output(init_parameter) + else: + self.minimize_starting_points = _rand_init( + self.x_bounds, self.x_types, self.selection_num_starting_points) + results = self._selection( + self.samples_x, + self.samples_y_aggregation, + self.samples_y, + self.x_bounds, + self.x_types, + threshold_samplessize_resampling=( + None if self.no_resampling is True else 50), + no_candidates=self.no_candidates, + minimize_starting_points=self.minimize_starting_points, + minimize_constraints_fun=self.minimize_constraints_fun) + + logger.info("Generate paramageters: \n%s", str(results)) + return results + + + def receive_trial_result(self, parameter_id, parameters, value, **kwargs): + """ + Tuner receive result from trial. + + Parameters + ---------- + parameter_id : int + The id of parameters, generated by nni manager. + parameters : dict + A group of parameters that trial has tried. + value : dict/float + if value is dict, it should have "default" key. + """ + value = extract_scalar_reward(value) + if self.optimize_mode == OptimizeMode.Maximize: + value = -value + + logger.info("Received trial result.") + logger.info("value is : %s", str(value)) + logger.info("parameter is : %s", str(parameters)) + + # parse parameter to sample_x + sample_x = [0 for i in range(len(self.key_order))] + for key in parameters: + idx = self.key_order.index(key) + sample_x[idx] = parameters[key] + + # parse value to sample_y + temp_y = [] + if sample_x in self.samples_x: + idx = self.samples_x.index(sample_x) + temp_y = self.samples_y[idx] + temp_y.append(value) + self.samples_y[idx] = temp_y + + # calculate y aggregation + median = get_median(temp_y) + self.samples_y_aggregation[idx] = [median] + else: + self.samples_x.append(sample_x) + self.samples_y.append([value]) + + # calculate y aggregation + self.samples_y_aggregation.append([value]) + + + def _selection( + self, + samples_x, + samples_y_aggregation, + samples_y, + x_bounds, + x_types, + max_resampling_per_x=3, + threshold_samplessize_exploitation=12, + threshold_samplessize_resampling=50, + no_candidates=False, + minimize_starting_points=None, + minimize_constraints_fun=None): + + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + + next_candidate = None + candidates = [] + samples_size_all = sum([len(i) for i in samples_y]) + samples_size_unique = len(samples_y) + + # ===== STEP 1: Compute the current optimum ===== + gp_model = gp_create_model.create_model( + samples_x, samples_y_aggregation) + lm_current = gp_selection.selection( + "lm", + samples_y_aggregation, + x_bounds, + x_types, + gp_model['model'], + minimize_starting_points, + minimize_constraints_fun=minimize_constraints_fun) + if not lm_current: + return None + logger.info({ + 'hyperparameter': lm_current['hyperparameter'], + 'expected_mu': lm_current['expected_mu'], + 'expected_sigma': lm_current['expected_sigma'], + 'reason': "exploitation_gp" + }) + + if no_candidates is False: + # ===== STEP 2: Get recommended configurations for exploration ==== + results_exploration = gp_selection.selection( + "lc", + samples_y_aggregation, + x_bounds, + x_types, + gp_model['model'], + minimize_starting_points, + minimize_constraints_fun=minimize_constraints_fun) + + if results_exploration is not None: + if _num_past_samples(results_exploration['hyperparameter'], samples_x, samples_y) == 0: + temp_candidate = { + 'hyperparameter': results_exploration['hyperparameter'], + 'expected_mu': results_exploration['expected_mu'], + 'expected_sigma': results_exploration['expected_sigma'], + 'reason': "exploration" + } + candidates.append(temp_candidate) + + logger.info("DEBUG: 1 exploration candidate selected\n") + logger.info(temp_candidate) + else: + logger.info("DEBUG: No suitable exploration candidates were") + + # ===== STEP 3: Get recommended configurations for exploitation === + if samples_size_all >= threshold_samplessize_exploitation: + logger.info("Getting candidates for exploitation...\n") + try: + gmm = gmm_create_model.create_model( + samples_x, samples_y_aggregation) + + if ("discrete_int" in x_types) or ("range_int" in x_types): + results_exploitation = gmm_selection.selection( + x_bounds, + x_types, + gmm['clusteringmodel_good'], + gmm['clusteringmodel_bad'], + minimize_starting_points, + minimize_constraints_fun=minimize_constraints_fun) + else: + # If all parameters are of "range_continuous", + # let's use GMM to generate random starting points + results_exploitation = gmm_selection.selection_r( + x_bounds, + x_types, + gmm['clusteringmodel_good'], + gmm['clusteringmodel_bad'], + num_starting_points=self.selection_num_starting_points, + minimize_constraints_fun=minimize_constraints_fun) + + if results_exploitation is not None: + if _num_past_samples(results_exploitation['hyperparameter'], samples_x, samples_y) == 0: + temp_expected_mu, temp_expected_sigma = \ + gp_prediction.predict(results_exploitation['hyperparameter'], gp_model['model']) + temp_candidate = { + 'hyperparameter': results_exploitation['hyperparameter'], + 'expected_mu': temp_expected_mu, + 'expected_sigma': temp_expected_sigma, + 'reason': "exploitation_gmm" + } + candidates.append(temp_candidate) + + logger.info( + "DEBUG: 1 exploitation_gmm candidate selected\n") + logger.info(temp_candidate) + else: + logger.info( + "DEBUG: No suitable exploitation_gmm candidates were found\n") + + except ValueError as exception: + # The exception: ValueError: Fitting the mixture model failed + # because some components have ill-defined empirical covariance + # (for instance caused by singleton or collapsed samples). + # Try to decrease the number of components, or increase + # reg_covar. + logger.info( + "DEBUG: No suitable exploitation_gmm \ + candidates were found due to exception.") + logger.info(exception) + + # ===== STEP 4: Get a list of outliers ===== + if (threshold_samplessize_resampling is not None) and \ + (samples_size_unique >= threshold_samplessize_resampling): + logger.info("Getting candidates for re-sampling...\n") + results_outliers = gp_outlier_detection.outlierDetection_threaded( + samples_x, samples_y_aggregation) + + if results_outliers is not None: + for results_outlier in results_outliers: # pylint: disable=not-an-iterable + if _num_past_samples(samples_x[results_outlier['samples_idx']], samples_x, samples_y) < max_resampling_per_x: + temp_candidate = {'hyperparameter': samples_x[results_outlier['samples_idx']],\ + 'expected_mu': results_outlier['expected_mu'],\ + 'expected_sigma': results_outlier['expected_sigma'],\ + 'reason': "resampling"} + candidates.append(temp_candidate) + logger.info("DEBUG: %d re-sampling candidates selected\n") + logger.info(temp_candidate) + else: + logger.info( + "DEBUG: No suitable resampling candidates were found\n") + + if candidates: + # ===== STEP 5: Compute the information gain of each candidate + logger.info( + "Evaluating information gain of %d candidates...\n") + next_improvement = 0 + + threads_inputs = [[ + candidate, samples_x, samples_y, x_bounds, x_types, + minimize_constraints_fun, minimize_starting_points + ] for candidate in candidates] + threads_pool = ThreadPool(4) + # Evaluate what would happen if we actually sample each + # candidate + threads_results = threads_pool.map( + _calculate_lowest_mu_threaded, threads_inputs) + threads_pool.close() + threads_pool.join() + + for threads_result in threads_results: + if threads_result['expected_lowest_mu'] < lm_current['expected_mu']: + # Information gain + temp_improvement = threads_result['expected_lowest_mu'] - \ + lm_current['expected_mu'] + + if next_improvement > temp_improvement: + next_improvement = temp_improvement + next_candidate = threads_result['candidate'] + else: + # ===== STEP 6: If we have no candidates, randomly pick one === + logger.info( + "DEBUG: No candidates from exploration, exploitation,\ + and resampling. We will random a candidate for next_candidate\n" + ) + + next_candidate = _rand_with_constraints( + x_bounds, + x_types) if minimize_starting_points is None else minimize_starting_points[0] + next_candidate = lib_data.match_val_type( + next_candidate, x_bounds, x_types) + expected_mu, expected_sigma = gp_prediction.predict( + next_candidate, gp_model['model']) + next_candidate = { + 'hyperparameter': next_candidate, + 'reason': "random", + 'expected_mu': expected_mu, + 'expected_sigma': expected_sigma} + + # STEP 7: If current optimal hyperparameter occurs in the history + # or exploration probability is less than the threshold, take next + # config as exploration step + outputs = self._pack_output(lm_current['hyperparameter']) + ap = random.uniform(0, 1) + if outputs in self.total_data or ap <= self.exploration_probability: + if next_candidate is not None: + outputs = self._pack_output(next_candidate['hyperparameter']) + else: + random_parameter = _rand_init(x_bounds, x_types, 1)[0] + outputs = self._pack_output(random_parameter) + self.total_data.append(outputs) + return outputs + + def import_data(self, data): + """ + Import additional data for tuning + + Parameters + ---------- + data : a list of dict + each of which has at least two keys: 'parameter' and 'value'. + """ + _completed_num = 0 + for trial_info in data: + logger.info("Importing data, current processing progress %s / %s", _completed_num, len(data)) + _completed_num += 1 + assert "parameter" in trial_info + _params = trial_info["parameter"] + assert "value" in trial_info + _value = trial_info['value'] + if not _value: + logger.info("Useless trial data, value is %s, skip this trial data.", _value) + continue + self.supplement_data_num += 1 + _parameter_id = '_'.join( + ["ImportData", str(self.supplement_data_num)]) + self.total_data.append(_params) + self.receive_trial_result( + parameter_id=_parameter_id, + parameters=_params, + value=_value) + logger.info("Successfully import data to metis tuner.") + + +def _rand_with_constraints(x_bounds, x_types): + outputs = None + x_bounds_withconstraints = [x_bounds[i] for i in CONSTRAINT_PARAMS_IDX] + x_types_withconstraints = [x_types[i] for i in CONSTRAINT_PARAMS_IDX] + + x_val_withconstraints = lib_constraint_summation.rand( + x_bounds_withconstraints, + x_types_withconstraints, + CONSTRAINT_LOWERBOUND, + CONSTRAINT_UPPERBOUND) + if not x_val_withconstraints: + outputs = [None] * len(x_bounds) + + for i, _ in enumerate(CONSTRAINT_PARAMS_IDX): + outputs[CONSTRAINT_PARAMS_IDX[i]] = x_val_withconstraints[i] + + for i, output in enumerate(outputs): + if not output: + outputs[i] = random.randint(x_bounds[i][0], x_bounds[i][1]) + return outputs + + +def _calculate_lowest_mu_threaded(inputs): + [candidate, samples_x, samples_y, x_bounds, x_types, + minimize_constraints_fun, minimize_starting_points] = inputs + + outputs = {"candidate": candidate, "expected_lowest_mu": None} + + for expected_mu in [ + candidate['expected_mu'] + + 1.96 * + candidate['expected_sigma'], + candidate['expected_mu'] - + 1.96 * + candidate['expected_sigma']]: + temp_samples_x = copy.deepcopy(samples_x) + temp_samples_y = copy.deepcopy(samples_y) + + try: + idx = temp_samples_x.index(candidate['hyperparameter']) + # This handles the case of re-sampling a potential outlier + temp_samples_y[idx].append(expected_mu) + except ValueError: + temp_samples_x.append(candidate['hyperparameter']) + temp_samples_y.append([expected_mu]) + + # Aggregates multiple observation of the sample sampling points + temp_y_aggregation = [statistics.median( + temp_sample_y) for temp_sample_y in temp_samples_y] + temp_gp = gp_create_model.create_model( + temp_samples_x, temp_y_aggregation) + temp_results = gp_selection.selection( + "lm", + temp_y_aggregation, + x_bounds, + x_types, + temp_gp['model'], + minimize_starting_points, + minimize_constraints_fun=minimize_constraints_fun) + + if outputs["expected_lowest_mu"] is None \ + or outputs["expected_lowest_mu"] > temp_results['expected_mu']: + outputs["expected_lowest_mu"] = temp_results['expected_mu'] + + return outputs + + +def _num_past_samples(x, samples_x, samples_y): + try: + idx = samples_x.index(x) + return len(samples_y[idx]) + except ValueError: + logger.info("x not in sample_x") + return 0 + + +def _rand_init(x_bounds, x_types, selection_num_starting_points): + ''' + Random sample some init seed within bounds. + ''' + return [lib_data.rand(x_bounds, x_types) for i + in range(0, selection_num_starting_points)] + + +def get_median(temp_list): + """ + Return median + """ + num = len(temp_list) + temp_list.sort() + print(temp_list) + if num % 2 == 0: + median = (temp_list[int(num / 2)] + temp_list[int(num / 2) - 1]) / 2 + else: + median = temp_list[int(num / 2)] + return median diff --git a/utils/third_party/nni/algorithms/hpo/metis_tuner/requirments.txt b/utils/third_party/nni/algorithms/hpo/metis_tuner/requirments.txt new file mode 100644 index 0000000000000000000000000000000000000000..3dfc2232a18c19a544eb3fb5b2f132e185958f98 --- /dev/null +++ b/utils/third_party/nni/algorithms/hpo/metis_tuner/requirments.txt @@ -0,0 +1 @@ +scikit-learn>=0.23.2 diff --git a/utils/third_party/nni/algorithms/hpo/networkmorphism_tuner/__init__.py b/utils/third_party/nni/algorithms/hpo/networkmorphism_tuner/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/utils/third_party/nni/algorithms/hpo/networkmorphism_tuner/bayesian.py b/utils/third_party/nni/algorithms/hpo/networkmorphism_tuner/bayesian.py new file mode 100644 index 0000000000000000000000000000000000000000..54c1996dc7708568a7065325d24a8b83fdb5a40f --- /dev/null +++ b/utils/third_party/nni/algorithms/hpo/networkmorphism_tuner/bayesian.py @@ -0,0 +1,482 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import math +import random +from copy import deepcopy +from functools import total_ordering +from queue import PriorityQueue + +import numpy as np +from scipy.linalg import LinAlgError, cho_solve, cholesky, solve_triangular +from scipy.optimize import linear_sum_assignment +from sklearn.metrics.pairwise import rbf_kernel + +from nni.utils import OptimizeMode +from .graph_transformer import transform +from .utils import Constant +from .layers import is_layer + + +def layer_distance(a, b): + """The distance between two layers.""" + # pylint: disable=unidiomatic-typecheck + if not isinstance(a, type(b)): + return 1.0 + if is_layer(a, "Conv"): + att_diff = [ + (a.filters, b.filters), + (a.kernel_size, b.kernel_size), + (a.stride, b.stride), + ] + return attribute_difference(att_diff) + if is_layer(a, "Pooling"): + att_diff = [ + (a.padding, b.padding), + (a.kernel_size, b.kernel_size), + (a.stride, b.stride), + ] + return attribute_difference(att_diff) + return 0.0 + + +def attribute_difference(att_diff): + ''' The attribute distance. + ''' + + ret = 0 + for a_value, b_value in att_diff: + if max(a_value, b_value) == 0: + ret += 0 + else: + ret += abs(a_value - b_value) * 1.0 / max(a_value, b_value) + return ret * 1.0 / len(att_diff) + + +def layers_distance(list_a, list_b): + """The distance between the layers of two neural networks.""" + len_a = len(list_a) + len_b = len(list_b) + f = np.zeros((len_a + 1, len_b + 1)) + f[-1][-1] = 0 + for i in range(-1, len_a): + f[i][-1] = i + 1 + for j in range(-1, len_b): + f[-1][j] = j + 1 + for i in range(len_a): + for j in range(len_b): + f[i][j] = min( + f[i][j - 1] + 1, + f[i - 1][j] + 1, + f[i - 1][j - 1] + layer_distance(list_a[i], list_b[j]), + ) + return f[len_a - 1][len_b - 1] + + +def skip_connection_distance(a, b): + """The distance between two skip-connections.""" + if a[2] != b[2]: + return 1.0 + len_a = abs(a[1] - a[0]) + len_b = abs(b[1] - b[0]) + return (abs(a[0] - b[0]) + abs(len_a - len_b)) / \ + (max(a[0], b[0]) + max(len_a, len_b)) + + +def skip_connections_distance(list_a, list_b): + """The distance between the skip-connections of two neural networks.""" + distance_matrix = np.zeros((len(list_a), len(list_b))) + for i, a in enumerate(list_a): + for j, b in enumerate(list_b): + distance_matrix[i][j] = skip_connection_distance(a, b) + return distance_matrix[linear_sum_assignment(distance_matrix)].sum() + abs( + len(list_a) - len(list_b) + ) + + +def edit_distance(x, y): + """The distance between two neural networks. + Args: + x: An instance of NetworkDescriptor. + y: An instance of NetworkDescriptor + Returns: + The edit-distance between x and y. + """ + + ret = layers_distance(x.layers, y.layers) + ret += Constant.KERNEL_LAMBDA * skip_connections_distance( + x.skip_connections, y.skip_connections + ) + return ret + + +class IncrementalGaussianProcess: + """Gaussian process regressor. + Attributes: + alpha: A hyperparameter. + """ + + def __init__(self): + self.alpha = 1e-10 + self._distance_matrix = None + self._x = None + self._y = None + self._first_fitted = False + self._l_matrix = None + self._alpha_vector = None + + @property + def kernel_matrix(self): + ''' Kernel matric. + ''' + return self._distance_matrix + + def fit(self, train_x, train_y): + """ Fit the regressor with more data. + Args: + train_x: A list of NetworkDescriptor. + train_y: A list of metric values. + """ + if self.first_fitted: + self.incremental_fit(train_x, train_y) + else: + self.first_fit(train_x, train_y) + + def incremental_fit(self, train_x, train_y): + """ Incrementally fit the regressor. """ + if not self._first_fitted: + raise ValueError( + "The first_fit function needs to be called first.") + + train_x, train_y = np.array(train_x), np.array(train_y) + + # Incrementally compute K + up_right_k = edit_distance_matrix(self._x, train_x) + down_left_k = np.transpose(up_right_k) + down_right_k = edit_distance_matrix(train_x) + up_k = np.concatenate((self._distance_matrix, up_right_k), axis=1) + down_k = np.concatenate((down_left_k, down_right_k), axis=1) + temp_distance_matrix = np.concatenate((up_k, down_k), axis=0) + k_matrix = bourgain_embedding_matrix(temp_distance_matrix) + diagonal = np.diag_indices_from(k_matrix) + diagonal = (diagonal[0][-len(train_x):], diagonal[1][-len(train_x):]) + k_matrix[diagonal] += self.alpha + + try: + self._l_matrix = cholesky(k_matrix, lower=True) # Line 2 + except LinAlgError: + return self + + self._x = np.concatenate((self._x, train_x), axis=0) + self._y = np.concatenate((self._y, train_y), axis=0) + self._distance_matrix = temp_distance_matrix + + self._alpha_vector = cho_solve( + (self._l_matrix, True), self._y) # Line 3 + + return self + + @property + def first_fitted(self): + ''' if it is firsr fitted + ''' + return self._first_fitted + + def first_fit(self, train_x, train_y): + """ Fit the regressor for the first time. """ + train_x, train_y = np.array(train_x), np.array(train_y) + + self._x = np.copy(train_x) + self._y = np.copy(train_y) + + self._distance_matrix = edit_distance_matrix(self._x) + k_matrix = bourgain_embedding_matrix(self._distance_matrix) + k_matrix[np.diag_indices_from(k_matrix)] += self.alpha + + self._l_matrix = cholesky(k_matrix, lower=True) # Line 2 + + self._alpha_vector = cho_solve( + (self._l_matrix, True), self._y) # Line 3 + + self._first_fitted = True + return self + + def predict(self, train_x): + """Predict the result. + Args: + train_x: A list of NetworkDescriptor. + Returns: + y_mean: The predicted mean. + y_std: The predicted standard deviation. + """ + k_trans = np.exp(-np.power(edit_distance_matrix(train_x, self._x), 2)) + y_mean = k_trans.dot(self._alpha_vector) # Line 4 (y_mean = f_star) + + # compute inverse K_inv of K based on its Cholesky + # decomposition L and its inverse L_inv + l_inv = solve_triangular( + self._l_matrix.T, np.eye( + self._l_matrix.shape[0])) + k_inv = l_inv.dot(l_inv.T) + # Compute variance of predictive distribution + y_var = np.ones(len(train_x), dtype=np.float) + y_var -= np.einsum("ij,ij->i", np.dot(k_trans, k_inv), k_trans) + + # Check if any of the variances is negative because of + # numerical issues. If yes: set the variance to 0. + y_var_negative = y_var < 0 + if np.any(y_var_negative): + y_var[y_var_negative] = 0.0 + return y_mean, np.sqrt(y_var) + + +def edit_distance_matrix(train_x, train_y=None): + """Calculate the edit distance. + Args: + train_x: A list of neural architectures. + train_y: A list of neural architectures. + Returns: + An edit-distance matrix. + """ + if train_y is None: + ret = np.zeros((train_x.shape[0], train_x.shape[0])) + for x_index, x in enumerate(train_x): + for y_index, y in enumerate(train_x): + if x_index == y_index: + ret[x_index][y_index] = 0 + elif x_index < y_index: + ret[x_index][y_index] = edit_distance(x, y) + else: + ret[x_index][y_index] = ret[y_index][x_index] + return ret + ret = np.zeros((train_x.shape[0], train_y.shape[0])) + for x_index, x in enumerate(train_x): + for y_index, y in enumerate(train_y): + ret[x_index][y_index] = edit_distance(x, y) + return ret + + +def vector_distance(a, b): + """The Euclidean distance between two vectors.""" + a = np.array(a) + b = np.array(b) + return np.linalg.norm(a - b) + + +def bourgain_embedding_matrix(distance_matrix): + """Use Bourgain algorithm to embed the neural architectures based on their edit-distance. + Args: + distance_matrix: A matrix of edit-distances. + Returns: + A matrix of distances after embedding. + """ + distance_matrix = np.array(distance_matrix) + n = len(distance_matrix) + if n == 1: + return distance_matrix + np.random.seed(123) + distort_elements = [] + r = range(n) + k = int(math.ceil(math.log(n) / math.log(2) - 1)) + t = int(math.ceil(math.log(n))) + counter = 0 + for i in range(0, k + 1): + for t in range(t): + s = np.random.choice(r, 2 ** i) + for j in r: + d = min([distance_matrix[j][s] for s in s]) + counter += len(s) + if i == 0 and t == 0: + distort_elements.append([d]) + else: + distort_elements[j].append(d) + return rbf_kernel(distort_elements, distort_elements) + + +class BayesianOptimizer: + """ A Bayesian optimizer for neural architectures. + Attributes: + searcher: The Searcher which is calling the Bayesian optimizer. + t_min: The minimum temperature for simulated annealing. + metric: An instance of the Metric subclasses. + gpr: A GaussianProcessRegressor for bayesian optimization. + beta: The beta in acquisition function. (refer to our paper) + search_tree: The network morphism search tree. + """ + + def __init__(self, searcher, t_min, optimizemode, beta=None): + self.searcher = searcher + self.t_min = t_min + self.optimizemode = optimizemode + self.gpr = IncrementalGaussianProcess() + self.beta = beta if beta is not None else Constant.BETA + self.search_tree = SearchTree() + + def fit(self, x_queue, y_queue): + """ Fit the optimizer with new architectures and performances. + Args: + x_queue: A list of NetworkDescriptor. + y_queue: A list of metric values. + """ + self.gpr.fit(x_queue, y_queue) + + def generate(self, descriptors): + """Generate new architecture. + Args: + descriptors: All the searched neural architectures. + Returns: + graph: An instance of Graph. A morphed neural network with weights. + father_id: The father node ID in the search tree. + """ + model_ids = self.search_tree.adj_list.keys() + + target_graph = None + father_id = None + descriptors = deepcopy(descriptors) + elem_class = Elem + if self.optimizemode is OptimizeMode.Maximize: + elem_class = ReverseElem + + # Initialize the priority queue. + pq = PriorityQueue() + temp_list = [] + for model_id in model_ids: + metric_value = self.searcher.get_metric_value_by_id(model_id) + temp_list.append((metric_value, model_id)) + temp_list = sorted(temp_list) + for metric_value, model_id in temp_list: + graph = self.searcher.load_model_by_id(model_id) + graph.clear_operation_history() + graph.clear_weights() + pq.put(elem_class(metric_value, model_id, graph)) + + t = 1.0 + t_min = self.t_min + alpha = 0.9 + opt_acq = self._get_init_opt_acq_value() + while not pq.empty() and t > t_min: + elem = pq.get() + if self.optimizemode is OptimizeMode.Maximize: + temp_exp = min((elem.metric_value - opt_acq) / t, 1.0) + else: + temp_exp = min((opt_acq - elem.metric_value) / t, 1.0) + ap = math.exp(temp_exp) + if ap >= random.uniform(0, 1): + for temp_graph in transform(elem.graph): + if contain(descriptors, temp_graph.extract_descriptor()): + continue + + temp_acq_value = self.acq(temp_graph) + pq.put( + elem_class( + temp_acq_value, + elem.father_id, + temp_graph)) + descriptors.append(temp_graph.extract_descriptor()) + if self._accept_new_acq_value(opt_acq, temp_acq_value): + opt_acq = temp_acq_value + father_id = elem.father_id + target_graph = deepcopy(temp_graph) + t *= alpha + + # Did not found a not duplicated architecture + if father_id is None: + return None, None + nm_graph = self.searcher.load_model_by_id(father_id) + for args in target_graph.operation_history: + getattr(nm_graph, args[0])(*list(args[1:])) + return nm_graph, father_id + + def acq(self, graph): + ''' estimate the value of generated graph + ''' + mean, std = self.gpr.predict(np.array([graph.extract_descriptor()])) + if self.optimizemode is OptimizeMode.Maximize: + return mean + self.beta * std + return mean - self.beta * std + + def _get_init_opt_acq_value(self): + if self.optimizemode is OptimizeMode.Maximize: + return -np.inf + return np.inf + + def _accept_new_acq_value(self, opt_acq, temp_acq_value): + if temp_acq_value > opt_acq and self.optimizemode is OptimizeMode.Maximize: + return True + if temp_acq_value < opt_acq and not self.optimizemode is OptimizeMode.Maximize: + return True + return False + + def add_child(self, father_id, model_id): + ''' add child to the search tree + Arguments: + father_id {int} -- father id + model_id {int} -- model id + ''' + + self.search_tree.add_child(father_id, model_id) + + +@total_ordering +class Elem: + """Elements to be sorted according to metric value.""" + + def __init__(self, metric_value, father_id, graph): + self.father_id = father_id + self.graph = graph + self.metric_value = metric_value + + def __eq__(self, other): + return self.metric_value == other.metric_value + + def __lt__(self, other): + return self.metric_value < other.metric_value + + +class ReverseElem(Elem): + """Elements to be reversely sorted according to metric value.""" + + def __lt__(self, other): + return self.metric_value > other.metric_value + + +def contain(descriptors, target_descriptor): + """Check if the target descriptor is in the descriptors.""" + for descriptor in descriptors: + if edit_distance(descriptor, target_descriptor) < 1e-5: + return True + return False + + +class SearchTree: + """The network morphism search tree.""" + + def __init__(self): + self.root = None + self.adj_list = {} + + def add_child(self, u, v): + ''' add child to search tree itself. + Arguments: + u {int} -- father id + v {int} -- child id + ''' + + if u == -1: + self.root = v + self.adj_list[v] = [] + return + if v not in self.adj_list[u]: + self.adj_list[u].append(v) + if v not in self.adj_list: + self.adj_list[v] = [] + + def get_dict(self, u=None): + """ A recursive function to return the content of the tree in a dict.""" + if u is None: + return self.get_dict(self.root) + children = [] + for v in self.adj_list[u]: + children.append(self.get_dict(v)) + ret = {"name": u, "children": children} + return ret diff --git a/utils/third_party/nni/algorithms/hpo/networkmorphism_tuner/graph.py b/utils/third_party/nni/algorithms/hpo/networkmorphism_tuner/graph.py new file mode 100644 index 0000000000000000000000000000000000000000..9c96b6c2f07088aceb0cb029de02cb4b2f16ae32 --- /dev/null +++ b/utils/third_party/nni/algorithms/hpo/networkmorphism_tuner/graph.py @@ -0,0 +1,995 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import json +from collections.abc import Iterable +from copy import deepcopy, copy +from queue import Queue + +import numpy as np +import torch + +from .layer_transformer import ( + add_noise, + wider_bn, + wider_next_conv, + wider_next_dense, + wider_pre_conv, + wider_pre_dense, + init_dense_weight, + init_conv_weight, + init_bn_weight, +) +from .layers import ( + StubAdd, + StubConcatenate, + StubReLU, + get_batch_norm_class, + get_conv_class, + is_layer, + layer_width, + set_keras_weight_to_stub, + set_stub_weight_to_keras, + set_stub_weight_to_torch, + set_torch_weight_to_stub, + to_real_keras_layer, + layer_description_extractor, + layer_description_builder, +) +from .utils import Constant + + +class NetworkDescriptor: + """A class describing the neural architecture for neural network kernel. + It only record the width of convolutional and dense layers, and the skip-connection types and positions. + """ + + CONCAT_CONNECT = "concat" + ADD_CONNECT = "add" + + def __init__(self): + self.skip_connections = [] + self.layers = [] + + @property + def n_layers(self): + return len(self.layers) + + def add_skip_connection(self, u, v, connection_type): + """ Add a skip-connection to the descriptor. + Args: + u: Number of convolutional layers before the starting point. + v: Number of convolutional layers before the ending point. + connection_type: Must be either CONCAT_CONNECT or ADD_CONNECT. + """ + if connection_type not in [self.CONCAT_CONNECT, self.ADD_CONNECT]: + raise ValueError( + "connection_type should be NetworkDescriptor.CONCAT_CONNECT " + "or NetworkDescriptor.ADD_CONNECT." + ) + self.skip_connections.append((u, v, connection_type)) + + def to_json(self): + ''' NetworkDescriptor to json representation + ''' + + skip_list = [] + for u, v, connection_type in self.skip_connections: + skip_list.append({"from": u, "to": v, "type": connection_type}) + return {"node_list": self.layers, "skip_list": skip_list} + + def add_layer(self, layer): + ''' add one layer + ''' + + self.layers.append(layer) + + +class Node: + """A class for intermediate output tensor (node) in the Graph. + Attributes: + shape: A tuple describing the shape of the tensor. + """ + + def __init__(self, shape): + self.shape = shape + + +class Graph: + """A class representing the neural architecture graph of a model. + Graph extracts the neural architecture graph from a model. + Each node in the graph is a intermediate tensor between layers. + Each layer is an edge in the graph. + Notably, multiple edges may refer to the same layer. + (e.g. Add layer is adding two tensor into one tensor. So it is related to two edges.) + Attributes: + weighted: A boolean of whether the weights and biases in the neural network + should be included in the graph. + input_shape: A tuple of integers, which does not include the batch axis. + node_list: A list of integers. The indices of the list are the identifiers. + layer_list: A list of stub layers. The indices of the list are the identifiers. + node_to_id: A dict instance mapping from node integers to their identifiers. + layer_to_id: A dict instance mapping from stub layers to their identifiers. + layer_id_to_input_node_ids: A dict instance mapping from layer identifiers + to their input nodes identifiers. + layer_id_to_output_node_ids: A dict instance mapping from layer identifiers + to their output nodes identifiers. + adj_list: A two dimensional list. The adjacency list of the graph. The first dimension is + identified by tensor identifiers. In each edge list, the elements are two-element tuples + of (tensor identifier, layer identifier). + reverse_adj_list: A reverse adjacent list in the same format as adj_list. + operation_history: A list saving all the network morphism operations. + vis: A dictionary of temporary storage for whether an local operation has been done + during the network morphism. + """ + + def __init__(self, input_shape, weighted=True): + """Initializer for Graph. + """ + self.input_shape = input_shape + self.weighted = weighted + self.node_list = [] + self.layer_list = [] + # node id start with 0 + self.node_to_id = {} + self.layer_to_id = {} + self.layer_id_to_input_node_ids = {} + self.layer_id_to_output_node_ids = {} + self.adj_list = {} + self.reverse_adj_list = {} + self.operation_history = [] + self.n_dim = len(input_shape) - 1 + self.conv = get_conv_class(self.n_dim) + self.batch_norm = get_batch_norm_class(self.n_dim) + + self.vis = None + self._add_node(Node(input_shape)) + + def add_layer(self, layer, input_node_id): + """Add a layer to the Graph. + Args: + layer: An instance of the subclasses of StubLayer in layers.py. + input_node_id: An integer. The ID of the input node of the layer. + Returns: + output_node_id: An integer. The ID of the output node of the layer. + """ + if isinstance(input_node_id, Iterable): + layer.input = list(map(lambda x: self.node_list[x], input_node_id)) + output_node_id = self._add_node(Node(layer.output_shape)) + for node_id in input_node_id: + self._add_edge(layer, node_id, output_node_id) + + else: + layer.input = self.node_list[input_node_id] + output_node_id = self._add_node(Node(layer.output_shape)) + self._add_edge(layer, input_node_id, output_node_id) + + layer.output = self.node_list[output_node_id] + return output_node_id + + def clear_operation_history(self): + self.operation_history = [] + + @property + def n_nodes(self): + """Return the number of nodes in the model.""" + return len(self.node_list) + + @property + def n_layers(self): + """Return the number of layers in the model.""" + return len(self.layer_list) + + def _add_node(self, node): + """Add a new node to node_list and give the node an ID. + Args: + node: An instance of Node. + Returns: + node_id: An integer. + """ + node_id = len(self.node_list) + self.node_to_id[node] = node_id + self.node_list.append(node) + self.adj_list[node_id] = [] + self.reverse_adj_list[node_id] = [] + return node_id + + def _add_edge(self, layer, input_id, output_id): + """Add a new layer to the graph. The nodes should be created in advance.""" + + if layer in self.layer_to_id: + layer_id = self.layer_to_id[layer] + if input_id not in self.layer_id_to_input_node_ids[layer_id]: + self.layer_id_to_input_node_ids[layer_id].append(input_id) + if output_id not in self.layer_id_to_output_node_ids[layer_id]: + self.layer_id_to_output_node_ids[layer_id].append(output_id) + else: + layer_id = len(self.layer_list) + self.layer_list.append(layer) + self.layer_to_id[layer] = layer_id + self.layer_id_to_input_node_ids[layer_id] = [input_id] + self.layer_id_to_output_node_ids[layer_id] = [output_id] + + self.adj_list[input_id].append((output_id, layer_id)) + self.reverse_adj_list[output_id].append((input_id, layer_id)) + + def _redirect_edge(self, u_id, v_id, new_v_id): + """Redirect the layer to a new node. + Change the edge originally from `u_id` to `v_id` into an edge from `u_id` to `new_v_id` + while keeping all other property of the edge the same. + """ + layer_id = None + for index, edge_tuple in enumerate(self.adj_list[u_id]): + if edge_tuple[0] == v_id: + layer_id = edge_tuple[1] + self.adj_list[u_id][index] = (new_v_id, layer_id) + self.layer_list[layer_id].output = self.node_list[new_v_id] + break + + for index, edge_tuple in enumerate(self.reverse_adj_list[v_id]): + if edge_tuple[0] == u_id: + layer_id = edge_tuple[1] + self.reverse_adj_list[v_id].remove(edge_tuple) + break + self.reverse_adj_list[new_v_id].append((u_id, layer_id)) + for index, value in enumerate( + self.layer_id_to_output_node_ids[layer_id]): + if value == v_id: + self.layer_id_to_output_node_ids[layer_id][index] = new_v_id + break + + def _replace_layer(self, layer_id, new_layer): + """Replace the layer with a new layer.""" + old_layer = self.layer_list[layer_id] + new_layer.input = old_layer.input + new_layer.output = old_layer.output + new_layer.output.shape = new_layer.output_shape + self.layer_list[layer_id] = new_layer + self.layer_to_id[new_layer] = layer_id + self.layer_to_id.pop(old_layer) + + @property + def topological_order(self): + """Return the topological order of the node IDs from the input node to the output node.""" + q = Queue() + in_degree = {} + for i in range(self.n_nodes): + in_degree[i] = 0 + for u in range(self.n_nodes): + for v, _ in self.adj_list[u]: + in_degree[v] += 1 + for i in range(self.n_nodes): + if in_degree[i] == 0: + q.put(i) + + order_list = [] + while not q.empty(): + u = q.get() + order_list.append(u) + for v, _ in self.adj_list[u]: + in_degree[v] -= 1 + if in_degree[v] == 0: + q.put(v) + return order_list + + def _get_pooling_layers(self, start_node_id, end_node_id): + """Given two node IDs, return all the pooling layers between them.""" + layer_list = [] + node_list = [start_node_id] + assert self._depth_first_search(end_node_id, layer_list, node_list) + ret = [] + for layer_id in layer_list: + layer = self.layer_list[layer_id] + if is_layer(layer, "Pooling"): + ret.append(layer) + elif is_layer(layer, "Conv") and layer.stride != 1: + ret.append(layer) + return ret + + def _depth_first_search(self, target_id, layer_id_list, node_list): + """Search for all the layers and nodes down the path. + A recursive function to search all the layers and nodes between the node in the node_list + and the node with target_id.""" + assert len(node_list) <= self.n_nodes + u = node_list[-1] + if u == target_id: + return True + + for v, layer_id in self.adj_list[u]: + layer_id_list.append(layer_id) + node_list.append(v) + if self._depth_first_search(target_id, layer_id_list, node_list): + return True + layer_id_list.pop() + node_list.pop() + + return False + + def _search(self, u, start_dim, total_dim, n_add): + """Search the graph for all the layers to be widened caused by an operation. + It is an recursive function with duplication check to avoid deadlock. + It searches from a starting node u until the corresponding layers has been widened. + Args: + u: The starting node ID. + start_dim: The position to insert the additional dimensions. + total_dim: The total number of dimensions the layer has before widening. + n_add: The number of dimensions to add. + """ + if (u, start_dim, total_dim, n_add) in self.vis: + return + self.vis[(u, start_dim, total_dim, n_add)] = True + for v, layer_id in self.adj_list[u]: + layer = self.layer_list[layer_id] + + if is_layer(layer, "Conv"): + new_layer = wider_next_conv( + layer, start_dim, total_dim, n_add, self.weighted + ) + self._replace_layer(layer_id, new_layer) + + elif is_layer(layer, "Dense"): + new_layer = wider_next_dense( + layer, start_dim, total_dim, n_add, self.weighted + ) + self._replace_layer(layer_id, new_layer) + + elif is_layer(layer, "BatchNormalization"): + new_layer = wider_bn( + layer, start_dim, total_dim, n_add, self.weighted) + self._replace_layer(layer_id, new_layer) + self._search(v, start_dim, total_dim, n_add) + + elif is_layer(layer, "Concatenate"): + if self.layer_id_to_input_node_ids[layer_id][1] == u: + # u is on the right of the concat + # next_start_dim += next_total_dim - total_dim + left_dim = self._upper_layer_width( + self.layer_id_to_input_node_ids[layer_id][0] + ) + next_start_dim = start_dim + left_dim + next_total_dim = total_dim + left_dim + else: + next_start_dim = start_dim + next_total_dim = total_dim + self._upper_layer_width( + self.layer_id_to_input_node_ids[layer_id][1] + ) + self._search(v, next_start_dim, next_total_dim, n_add) + + else: + self._search(v, start_dim, total_dim, n_add) + + for v, layer_id in self.reverse_adj_list[u]: + layer = self.layer_list[layer_id] + if is_layer(layer, "Conv"): + new_layer = wider_pre_conv(layer, n_add, self.weighted) + self._replace_layer(layer_id, new_layer) + elif is_layer(layer, "Dense"): + new_layer = wider_pre_dense(layer, n_add, self.weighted) + self._replace_layer(layer_id, new_layer) + elif is_layer(layer, "Concatenate"): + continue + else: + self._search(v, start_dim, total_dim, n_add) + + def _upper_layer_width(self, u): + for v, layer_id in self.reverse_adj_list[u]: + layer = self.layer_list[layer_id] + if is_layer(layer, "Conv") or is_layer(layer, "Dense"): + return layer_width(layer) + elif is_layer(layer, "Concatenate"): + a = self.layer_id_to_input_node_ids[layer_id][0] + b = self.layer_id_to_input_node_ids[layer_id][1] + return self._upper_layer_width(a) + self._upper_layer_width(b) + else: + return self._upper_layer_width(v) + return self.node_list[0].shape[-1] + + def to_deeper_model(self, target_id, new_layer): + """Insert a relu-conv-bn block after the target block. + Args: + target_id: A convolutional layer ID. The new block should be inserted after the block. + new_layer: An instance of StubLayer subclasses. + """ + self.operation_history.append( + ("to_deeper_model", target_id, new_layer)) + input_id = self.layer_id_to_input_node_ids[target_id][0] + output_id = self.layer_id_to_output_node_ids[target_id][0] + if self.weighted: + if is_layer(new_layer, "Dense"): + init_dense_weight(new_layer) + elif is_layer(new_layer, "Conv"): + init_conv_weight(new_layer) + elif is_layer(new_layer, "BatchNormalization"): + init_bn_weight(new_layer) + + self._insert_new_layers([new_layer], input_id, output_id) + + def to_wider_model(self, pre_layer_id, n_add): + """Widen the last dimension of the output of the pre_layer. + Args: + pre_layer_id: The ID of a convolutional layer or dense layer. + n_add: The number of dimensions to add. + """ + self.operation_history.append(("to_wider_model", pre_layer_id, n_add)) + pre_layer = self.layer_list[pre_layer_id] + output_id = self.layer_id_to_output_node_ids[pre_layer_id][0] + dim = layer_width(pre_layer) + self.vis = {} + self._search(output_id, dim, dim, n_add) + # Update the tensor shapes. + for u in self.topological_order: + for v, layer_id in self.adj_list[u]: + self.node_list[v].shape = self.layer_list[layer_id].output_shape + + def _insert_new_layers(self, new_layers, start_node_id, end_node_id): + """Insert the new_layers after the node with start_node_id.""" + new_node_id = self._add_node(deepcopy(self.node_list[end_node_id])) + temp_output_id = new_node_id + for layer in new_layers[:-1]: + temp_output_id = self.add_layer(layer, temp_output_id) + + self._add_edge(new_layers[-1], temp_output_id, end_node_id) + new_layers[-1].input = self.node_list[temp_output_id] + new_layers[-1].output = self.node_list[end_node_id] + self._redirect_edge(start_node_id, end_node_id, new_node_id) + + def _block_end_node(self, layer_id, block_size): + ret = self.layer_id_to_output_node_ids[layer_id][0] + for _ in range(block_size - 2): + ret = self.adj_list[ret][0][0] + return ret + + def _dense_block_end_node(self, layer_id): + return self.layer_id_to_input_node_ids[layer_id][0] + + def _conv_block_end_node(self, layer_id): + """Get the input node ID of the last layer in the block by layer ID. + Return the input node ID of the last layer in the convolutional block. + Args: + layer_id: the convolutional layer ID. + """ + return self._block_end_node(layer_id, Constant.CONV_BLOCK_DISTANCE) + + def to_add_skip_model(self, start_id, end_id): + """Add a weighted add skip-connection from after start node to end node. + Args: + start_id: The convolutional layer ID, after which to start the skip-connection. + end_id: The convolutional layer ID, after which to end the skip-connection. + """ + self.operation_history.append(("to_add_skip_model", start_id, end_id)) + filters_end = self.layer_list[end_id].output.shape[-1] + filters_start = self.layer_list[start_id].output.shape[-1] + start_node_id = self.layer_id_to_output_node_ids[start_id][0] + + pre_end_node_id = self.layer_id_to_input_node_ids[end_id][0] + end_node_id = self.layer_id_to_output_node_ids[end_id][0] + + skip_output_id = self._insert_pooling_layer_chain( + start_node_id, end_node_id) + + # Add the conv layer + new_conv_layer = get_conv_class( + self.n_dim)( + filters_start, + filters_end, + 1) + skip_output_id = self.add_layer(new_conv_layer, skip_output_id) + + # Add the add layer. + add_input_node_id = self._add_node( + deepcopy(self.node_list[end_node_id])) + add_layer = StubAdd() + + self._redirect_edge(pre_end_node_id, end_node_id, add_input_node_id) + self._add_edge(add_layer, add_input_node_id, end_node_id) + self._add_edge(add_layer, skip_output_id, end_node_id) + add_layer.input = [ + self.node_list[add_input_node_id], + self.node_list[skip_output_id], + ] + add_layer.output = self.node_list[end_node_id] + self.node_list[end_node_id].shape = add_layer.output_shape + + # Set weights to the additional conv layer. + if self.weighted: + filter_shape = (1,) * self.n_dim + weights = np.zeros((filters_end, filters_start) + filter_shape) + bias = np.zeros(filters_end) + new_conv_layer.set_weights( + (add_noise(weights, np.array([0, 1])), add_noise( + bias, np.array([0, 1]))) + ) + + def to_concat_skip_model(self, start_id, end_id): + """Add a weighted add concatenate connection from after start node to end node. + Args: + start_id: The convolutional layer ID, after which to start the skip-connection. + end_id: The convolutional layer ID, after which to end the skip-connection. + """ + self.operation_history.append( + ("to_concat_skip_model", start_id, end_id)) + filters_end = self.layer_list[end_id].output.shape[-1] + filters_start = self.layer_list[start_id].output.shape[-1] + start_node_id = self.layer_id_to_output_node_ids[start_id][0] + + pre_end_node_id = self.layer_id_to_input_node_ids[end_id][0] + end_node_id = self.layer_id_to_output_node_ids[end_id][0] + + skip_output_id = self._insert_pooling_layer_chain( + start_node_id, end_node_id) + + concat_input_node_id = self._add_node( + deepcopy(self.node_list[end_node_id])) + self._redirect_edge(pre_end_node_id, end_node_id, concat_input_node_id) + + concat_layer = StubConcatenate() + concat_layer.input = [ + self.node_list[concat_input_node_id], + self.node_list[skip_output_id], + ] + concat_output_node_id = self._add_node(Node(concat_layer.output_shape)) + self._add_edge( + concat_layer, + concat_input_node_id, + concat_output_node_id) + self._add_edge(concat_layer, skip_output_id, concat_output_node_id) + concat_layer.output = self.node_list[concat_output_node_id] + self.node_list[concat_output_node_id].shape = concat_layer.output_shape + + # Add the concatenate layer. + new_conv_layer = get_conv_class(self.n_dim)( + filters_start + filters_end, filters_end, 1 + ) + self._add_edge(new_conv_layer, concat_output_node_id, end_node_id) + new_conv_layer.input = self.node_list[concat_output_node_id] + new_conv_layer.output = self.node_list[end_node_id] + self.node_list[end_node_id].shape = new_conv_layer.output_shape + + if self.weighted: + filter_shape = (1,) * self.n_dim + weights = np.zeros((filters_end, filters_end) + filter_shape) + for i in range(filters_end): + filter_weight = np.zeros((filters_end,) + filter_shape) + center_index = (i,) + (0,) * self.n_dim + filter_weight[center_index] = 1 + weights[i, ...] = filter_weight + weights = np.concatenate( + (weights, np.zeros((filters_end, filters_start) + filter_shape)), axis=1 + ) + bias = np.zeros(filters_end) + new_conv_layer.set_weights( + (add_noise(weights, np.array([0, 1])), add_noise( + bias, np.array([0, 1]))) + ) + + def _insert_pooling_layer_chain(self, start_node_id, end_node_id): + skip_output_id = start_node_id + for layer in self._get_pooling_layers(start_node_id, end_node_id): + new_layer = deepcopy(layer) + if is_layer(new_layer, "Conv"): + filters = self.node_list[start_node_id].shape[-1] + new_layer = get_conv_class(self.n_dim)( + filters, filters, 1, layer.stride) + if self.weighted: + init_conv_weight(new_layer) + else: + new_layer = deepcopy(layer) + skip_output_id = self.add_layer(new_layer, skip_output_id) + skip_output_id = self.add_layer(StubReLU(), skip_output_id) + return skip_output_id + + def extract_descriptor(self): + """Extract the the description of the Graph as an instance of NetworkDescriptor.""" + main_chain = self.get_main_chain() + index_in_main_chain = {} + for index, u in enumerate(main_chain): + index_in_main_chain[u] = index + + ret = NetworkDescriptor() + for u in main_chain: + for v, layer_id in self.adj_list[u]: + if v not in index_in_main_chain: + continue + layer = self.layer_list[layer_id] + copied_layer = copy(layer) + copied_layer.weights = None + ret.add_layer(deepcopy(copied_layer)) + + for u in index_in_main_chain: + for v, layer_id in self.adj_list[u]: + if v not in index_in_main_chain: + temp_u = u + temp_v = v + temp_layer_id = layer_id + skip_type = None + while not ( + temp_v in index_in_main_chain and temp_u in index_in_main_chain): + if is_layer( + self.layer_list[temp_layer_id], "Concatenate"): + skip_type = NetworkDescriptor.CONCAT_CONNECT + if is_layer(self.layer_list[temp_layer_id], "Add"): + skip_type = NetworkDescriptor.ADD_CONNECT + temp_u = temp_v + temp_v, temp_layer_id = self.adj_list[temp_v][0] + ret.add_skip_connection( + index_in_main_chain[u], index_in_main_chain[temp_u], skip_type + ) + + elif index_in_main_chain[v] - index_in_main_chain[u] != 1: + skip_type = None + if is_layer(self.layer_list[layer_id], "Concatenate"): + skip_type = NetworkDescriptor.CONCAT_CONNECT + if is_layer(self.layer_list[layer_id], "Add"): + skip_type = NetworkDescriptor.ADD_CONNECT + ret.add_skip_connection( + index_in_main_chain[u], index_in_main_chain[v], skip_type + ) + + return ret + + def clear_weights(self): + ''' clear weights of the graph + ''' + self.weighted = False + for layer in self.layer_list: + layer.weights = None + + def produce_torch_model(self): + """Build a new Torch model based on the current graph.""" + return TorchModel(self) + + def produce_keras_model(self): + """Build a new keras model based on the current graph.""" + return KerasModel(self).model + + def produce_onnx_model(self): + """Build a new ONNX model based on the current graph.""" + return ONNXModel(self) + + def parsing_onnx_model(self, onnx_model): + '''to do in the future to use the onnx model + ''' + return ONNXModel(onnx_model) + + def produce_json_model(self): + """Build a new Json model based on the current graph.""" + return JSONModel(self).data + + @classmethod + def parsing_json_model(cls, json_model): + '''build a graph from json + ''' + return json_to_graph(json_model) + + def _layer_ids_in_order(self, layer_ids): + node_id_to_order_index = {} + for index, node_id in enumerate(self.topological_order): + node_id_to_order_index[node_id] = index + return sorted( + layer_ids, + key=lambda layer_id: node_id_to_order_index[ + self.layer_id_to_output_node_ids[layer_id][0] + ], + ) + + def _layer_ids_by_type(self, type_str): + return list( + filter( + lambda layer_id: is_layer(self.layer_list[layer_id], type_str), + range(self.n_layers), + ) + ) + + def get_main_chain_layers(self): + """Return a list of layer IDs in the main chain.""" + main_chain = self.get_main_chain() + ret = [] + for u in main_chain: + for v, layer_id in self.adj_list[u]: + if v in main_chain and u in main_chain: + ret.append(layer_id) + return ret + + def _conv_layer_ids_in_order(self): + return list( + filter( + lambda layer_id: is_layer(self.layer_list[layer_id], "Conv"), + self.get_main_chain_layers(), + ) + ) + + def _dense_layer_ids_in_order(self): + return self._layer_ids_in_order(self._layer_ids_by_type("Dense")) + + def deep_layer_ids(self): + ret = [] + for layer_id in self.get_main_chain_layers(): + layer = self.layer_list[layer_id] + if is_layer(layer, "GlobalAveragePooling"): + break + if is_layer(layer, "Add") or is_layer(layer, "Concatenate"): + continue + ret.append(layer_id) + return ret + + def wide_layer_ids(self): + return ( + self._conv_layer_ids_in_order( + )[:-1] + self._dense_layer_ids_in_order()[:-1] + ) + + def skip_connection_layer_ids(self): + return self.deep_layer_ids()[:-1] + + def size(self): + return sum(list(map(lambda x: x.size(), self.layer_list))) + + def get_main_chain(self): + """Returns the main chain node ID list.""" + pre_node = {} + distance = {} + for i in range(self.n_nodes): + distance[i] = 0 + pre_node[i] = i + for i in range(self.n_nodes - 1): + for u in range(self.n_nodes): + for v, _ in self.adj_list[u]: + if distance[u] + 1 > distance[v]: + distance[v] = distance[u] + 1 + pre_node[v] = u + temp_id = 0 + for i in range(self.n_nodes): + if distance[i] > distance[temp_id]: + temp_id = i + ret = [] + for i in range(self.n_nodes + 5): + ret.append(temp_id) + if pre_node[temp_id] == temp_id: + break + temp_id = pre_node[temp_id] + assert temp_id == pre_node[temp_id] + ret.reverse() + return ret + + +class TorchModel(torch.nn.Module): + """A neural network class using pytorch constructed from an instance of Graph.""" + + def __init__(self, graph): + super(TorchModel, self).__init__() + self.graph = graph + self.layers = [] + for layer in graph.layer_list: + self.layers.append(layer.to_real_layer()) + if graph.weighted: + for index, layer in enumerate(self.layers): + set_stub_weight_to_torch(self.graph.layer_list[index], layer) + for index, layer in enumerate(self.layers): + self.add_module(str(index), layer) + + def forward(self, input_tensor): + topo_node_list = self.graph.topological_order + output_id = topo_node_list[-1] + input_id = topo_node_list[0] + + node_list = deepcopy(self.graph.node_list) + node_list[input_id] = input_tensor + + for v in topo_node_list: + for u, layer_id in self.graph.reverse_adj_list[v]: + layer = self.graph.layer_list[layer_id] + torch_layer = self.layers[layer_id] + + if isinstance(layer, (StubAdd, StubConcatenate)): + edge_input_tensor = list( + map( + lambda x: node_list[x], + self.graph.layer_id_to_input_node_ids[layer_id], + ) + ) + else: + edge_input_tensor = node_list[u] + + temp_tensor = torch_layer(edge_input_tensor) + node_list[v] = temp_tensor + return node_list[output_id] + + def set_weight_to_graph(self): + self.graph.weighted = True + for index, layer in enumerate(self.layers): + set_torch_weight_to_stub(layer, self.graph.layer_list[index]) + + +class KerasModel: + def __init__(self, graph): + import keras + + self.graph = graph + self.layers = [] + for layer in graph.layer_list: + self.layers.append(to_real_keras_layer(layer)) + + # Construct the keras graph. + # Input + topo_node_list = self.graph.topological_order + output_id = topo_node_list[-1] + input_id = topo_node_list[0] + input_tensor = keras.layers.Input( + shape=graph.node_list[input_id].shape) + + node_list = deepcopy(self.graph.node_list) + node_list[input_id] = input_tensor + + # Output + for v in topo_node_list: + for u, layer_id in self.graph.reverse_adj_list[v]: + layer = self.graph.layer_list[layer_id] + keras_layer = self.layers[layer_id] + + if isinstance(layer, (StubAdd, StubConcatenate)): + edge_input_tensor = list( + map( + lambda x: node_list[x], + self.graph.layer_id_to_input_node_ids[layer_id], + ) + ) + else: + edge_input_tensor = node_list[u] + + temp_tensor = keras_layer(edge_input_tensor) + node_list[v] = temp_tensor + + output_tensor = node_list[output_id] + output_tensor = keras.layers.Activation("softmax", name="activation_add")( + output_tensor + ) + self.model = keras.models.Model( + inputs=input_tensor, outputs=output_tensor) + + if graph.weighted: + for index, layer in enumerate(self.layers): + set_stub_weight_to_keras(self.graph.layer_list[index], layer) + + def set_weight_to_graph(self): + self.graph.weighted = True + for index, layer in enumerate(self.layers): + set_keras_weight_to_stub(layer, self.graph.layer_list[index]) + + +class ONNXModel: + # to do in the future using onnx ir + def __init__(self, graph): + pass + + +class JSONModel: + def __init__(self, graph): + data = dict() + node_list = list() + layer_list = list() + operation_history = list() + + data["input_shape"] = graph.input_shape + vis = graph.vis + data["vis"] = list(vis.keys()) if vis is not None else None + data["weighted"] = graph.weighted + + for item in graph.operation_history: + if item[0] == "to_deeper_model": + operation_history.append( + [ + item[0], + item[1], + layer_description_extractor(item[2], graph.node_to_id), + ] + ) + else: + operation_history.append(item) + data["operation_history"] = operation_history + data["layer_id_to_input_node_ids"] = graph.layer_id_to_input_node_ids + data["layer_id_to_output_node_ids"] = graph.layer_id_to_output_node_ids + data["adj_list"] = graph.adj_list + data["reverse_adj_list"] = graph.reverse_adj_list + + for node in graph.node_list: + node_id = graph.node_to_id[node] + node_information = node.shape + node_list.append((node_id, node_information)) + + for layer_id, item in enumerate(graph.layer_list): + layer = graph.layer_list[layer_id] + layer_information = layer_description_extractor( + layer, graph.node_to_id) + layer_list.append((layer_id, layer_information)) + + data["node_list"] = node_list + data["layer_list"] = layer_list + + self.data = data + + +def graph_to_onnx(graph, onnx_model_path): + import onnx + # to do in the future using onnx ir + onnx_out = graph.produce_onnx_model() + onnx.save(onnx_out, onnx_model_path) + return onnx_out + + +def onnx_to_graph(onnx_model, input_shape): + # to do in the future using onnx ir + graph = Graph(input_shape, False) + graph.parsing_onnx_model(onnx_model) + return graph + + +def graph_to_json(graph, json_model_path): + json_out = graph.produce_json_model() + with open(json_model_path, "w") as fout: + json.dump(json_out, fout) + json_out = json.dumps(json_out) + return json_out + + +def json_to_graph(json_model: str): + json_model = json.loads(json_model) + # restore graph data from json data + input_shape = tuple(json_model["input_shape"]) + node_list = list() + node_to_id = dict() + id_to_node = dict() + layer_list = list() + layer_to_id = dict() + operation_history = list() + graph = Graph(input_shape, False) + + graph.input_shape = input_shape + vis = json_model["vis"] + graph.vis = { + tuple(item): True for item in vis} if vis is not None else None + graph.weighted = json_model["weighted"] + layer_id_to_input_node_ids = json_model["layer_id_to_input_node_ids"] + graph.layer_id_to_input_node_ids = { + int(k): v for k, v in layer_id_to_input_node_ids.items() + } + layer_id_to_output_node_ids = json_model["layer_id_to_output_node_ids"] + graph.layer_id_to_output_node_ids = { + int(k): v for k, v in layer_id_to_output_node_ids.items() + } + adj_list = {} + for k, v in json_model["adj_list"].items(): + adj_list[int(k)] = [tuple(i) for i in v] + graph.adj_list = adj_list + reverse_adj_list = {} + for k, v in json_model["reverse_adj_list"].items(): + reverse_adj_list[int(k)] = [tuple(i) for i in v] + graph.reverse_adj_list = reverse_adj_list + + for item in json_model["node_list"]: + new_node = Node(tuple(item[1])) + node_id = item[0] + node_list.append(new_node) + node_to_id[new_node] = node_id + id_to_node[node_id] = new_node + + for item in json_model["operation_history"]: + if item[0] == "to_deeper_model": + operation_history.append( + (item[0], item[1], layer_description_builder(item[2], id_to_node)) + ) + else: + operation_history.append(item) + graph.operation_history = operation_history + + for item in json_model["layer_list"]: + new_layer = layer_description_builder(item[1], id_to_node) + layer_id = int(item[0]) + layer_list.append(new_layer) + layer_to_id[new_layer] = layer_id + + graph.node_list = node_list + graph.node_to_id = node_to_id + graph.layer_list = layer_list + graph.layer_to_id = layer_to_id + + return graph diff --git a/utils/third_party/nni/algorithms/hpo/networkmorphism_tuner/graph_transformer.py b/utils/third_party/nni/algorithms/hpo/networkmorphism_tuner/graph_transformer.py new file mode 100644 index 0000000000000000000000000000000000000000..b03112d63bcc50f5c42e4203dfd4adb70a061119 --- /dev/null +++ b/utils/third_party/nni/algorithms/hpo/networkmorphism_tuner/graph_transformer.py @@ -0,0 +1,167 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from copy import deepcopy + +from random import randrange, sample + +from .graph import NetworkDescriptor +from .layers import ( + StubDense, + StubReLU, + get_batch_norm_class, + get_conv_class, + get_dropout_class, + get_pooling_class, + is_layer, +) +from .utils import Constant + + +def to_wider_graph(graph): + ''' wider graph + ''' + weighted_layer_ids = graph.wide_layer_ids() + weighted_layer_ids = list( + filter( + lambda x: graph.layer_list[x].output.shape[-1], weighted_layer_ids) + ) + wider_layers = sample(weighted_layer_ids, 1) + + for layer_id in wider_layers: + layer = graph.layer_list[layer_id] + if is_layer(layer, "Conv"): + n_add = layer.filters + else: + n_add = layer.units + + graph.to_wider_model(layer_id, n_add) + return graph + + +def to_skip_connection_graph(graph): + ''' skip connection graph + ''' + # The last conv layer cannot be widen since wider operator cannot be done + # over the two sides of flatten. + weighted_layer_ids = graph.skip_connection_layer_ids() + valid_connection = [] + for skip_type in sorted( + [NetworkDescriptor.ADD_CONNECT, NetworkDescriptor.CONCAT_CONNECT]): + for index_a in range(len(weighted_layer_ids)): + for index_b in range(len(weighted_layer_ids))[index_a + 1:]: + valid_connection.append((index_a, index_b, skip_type)) + + if not valid_connection: + return graph + for index_a, index_b, skip_type in sample(valid_connection, 1): + a_id = weighted_layer_ids[index_a] + b_id = weighted_layer_ids[index_b] + if skip_type == NetworkDescriptor.ADD_CONNECT: + graph.to_add_skip_model(a_id, b_id) + else: + graph.to_concat_skip_model(a_id, b_id) + return graph + + +def create_new_layer(layer, n_dim): + ''' create new layer for the graph + ''' + + input_shape = layer.output.shape + dense_deeper_classes = [StubDense, get_dropout_class(n_dim), StubReLU] + conv_deeper_classes = [ + get_conv_class(n_dim), + get_batch_norm_class(n_dim), + StubReLU] + if is_layer(layer, "ReLU"): + conv_deeper_classes = [ + get_conv_class(n_dim), + get_batch_norm_class(n_dim)] + dense_deeper_classes = [StubDense, get_dropout_class(n_dim)] + elif is_layer(layer, "Dropout"): + dense_deeper_classes = [StubDense, StubReLU] + elif is_layer(layer, "BatchNormalization"): + conv_deeper_classes = [get_conv_class(n_dim), StubReLU] + + layer_class = None + if len(input_shape) == 1: + # It is in the dense layer part. + layer_class = sample(dense_deeper_classes, 1)[0] + else: + # It is in the conv layer part. + layer_class = sample(conv_deeper_classes, 1)[0] + + if layer_class == StubDense: + new_layer = StubDense(input_shape[0], input_shape[0]) + + elif layer_class == get_dropout_class(n_dim): + new_layer = layer_class(Constant.DENSE_DROPOUT_RATE) + + elif layer_class == get_conv_class(n_dim): + new_layer = layer_class( + input_shape[-1], input_shape[-1], sample((1, 3, 5), 1)[0], stride=1 + ) + + elif layer_class == get_batch_norm_class(n_dim): + new_layer = layer_class(input_shape[-1]) + + elif layer_class == get_pooling_class(n_dim): + new_layer = layer_class(sample((1, 3, 5), 1)[0]) + + else: + new_layer = layer_class() + + return new_layer + + +def to_deeper_graph(graph): + ''' deeper graph + ''' + + weighted_layer_ids = graph.deep_layer_ids() + if len(weighted_layer_ids) >= Constant.MAX_LAYERS: + return None + + deeper_layer_ids = sample(weighted_layer_ids, 1) + + for layer_id in deeper_layer_ids: + layer = graph.layer_list[layer_id] + new_layer = create_new_layer(layer, graph.n_dim) + graph.to_deeper_model(layer_id, new_layer) + return graph + + +def legal_graph(graph): + '''judge if a graph is legal or not. + ''' + + descriptor = graph.extract_descriptor() + skips = descriptor.skip_connections + if len(skips) != len(set(skips)): + return False + return True + + +def transform(graph): + '''core transform function for graph. + ''' + + graphs = [] + for _ in range(Constant.N_NEIGHBOURS * 2): + random_num = randrange(3) + temp_graph = None + if random_num == 0: + temp_graph = to_deeper_graph(deepcopy(graph)) + elif random_num == 1: + temp_graph = to_wider_graph(deepcopy(graph)) + elif random_num == 2: + temp_graph = to_skip_connection_graph(deepcopy(graph)) + + if temp_graph is not None and temp_graph.size() <= Constant.MAX_MODEL_SIZE: + graphs.append(temp_graph) + + if len(graphs) >= Constant.N_NEIGHBOURS: + break + + return graphs diff --git a/utils/third_party/nni/algorithms/hpo/networkmorphism_tuner/layer_transformer.py b/utils/third_party/nni/algorithms/hpo/networkmorphism_tuner/layer_transformer.py new file mode 100644 index 0000000000000000000000000000000000000000..6ffd1b20fb0e3b4ef0a6d2e54a02fb7cfa7cfc42 --- /dev/null +++ b/utils/third_party/nni/algorithms/hpo/networkmorphism_tuner/layer_transformer.py @@ -0,0 +1,264 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import numpy as np + +from .layers import ( + StubDense, + StubReLU, + get_batch_norm_class, + get_conv_class, + get_n_dim, +) + +NOISE_RATIO = 1e-4 + + +def deeper_conv_block(conv_layer, kernel_size, weighted=True): + '''deeper conv layer. + ''' + n_dim = get_n_dim(conv_layer) + filter_shape = (kernel_size,) * 2 + n_filters = conv_layer.filters + weight = np.zeros((n_filters, n_filters) + filter_shape) + center = tuple(map(lambda x: int((x - 1) / 2), filter_shape)) + for i in range(n_filters): + filter_weight = np.zeros((n_filters,) + filter_shape) + index = (i,) + center + filter_weight[index] = 1 + weight[i, ...] = filter_weight + bias = np.zeros(n_filters) + new_conv_layer = get_conv_class(n_dim)( + conv_layer.filters, n_filters, kernel_size=kernel_size + ) + bn = get_batch_norm_class(n_dim)(n_filters) + + if weighted: + new_conv_layer.set_weights( + (add_noise(weight, np.array([0, 1])), + add_noise(bias, np.array([0, 1]))) + ) + new_weights = [ + add_noise(np.ones(n_filters, dtype=np.float32), np.array([0, 1])), + add_noise(np.zeros(n_filters, dtype=np.float32), np.array([0, 1])), + add_noise(np.zeros(n_filters, dtype=np.float32), np.array([0, 1])), + add_noise(np.ones(n_filters, dtype=np.float32), np.array([0, 1])), + ] + bn.set_weights(new_weights) + + return [StubReLU(), new_conv_layer, bn] + + +def dense_to_deeper_block(dense_layer, weighted=True): + '''deeper dense layer. + ''' + units = dense_layer.units + weight = np.eye(units) + bias = np.zeros(units) + new_dense_layer = StubDense(units, units) + if weighted: + new_dense_layer.set_weights( + (add_noise(weight, np.array([0, 1])), + add_noise(bias, np.array([0, 1]))) + ) + return [StubReLU(), new_dense_layer] + + +def wider_pre_dense(layer, n_add, weighted=True): + '''wider previous dense layer. + ''' + if not weighted: + return StubDense(layer.input_units, layer.units + n_add) + + n_units2 = layer.units + + teacher_w, teacher_b = layer.get_weights() + rand = np.random.randint(n_units2, size=n_add) + student_w = teacher_w.copy() + student_b = teacher_b.copy() + + # target layer update (i) + for i in range(n_add): + teacher_index = rand[i] + new_weight = teacher_w[teacher_index, :] + new_weight = new_weight[np.newaxis, :] + student_w = np.concatenate( + (student_w, add_noise(new_weight, student_w)), axis=0) + student_b = np.append( + student_b, add_noise( + teacher_b[teacher_index], student_b)) + + new_pre_layer = StubDense(layer.input_units, n_units2 + n_add) + new_pre_layer.set_weights((student_w, student_b)) + + return new_pre_layer + + +def wider_pre_conv(layer, n_add_filters, weighted=True): + '''wider previous conv layer. + ''' + n_dim = get_n_dim(layer) + if not weighted: + return get_conv_class(n_dim)( + layer.input_channel, + layer.filters + n_add_filters, + kernel_size=layer.kernel_size, + ) + + n_pre_filters = layer.filters + rand = np.random.randint(n_pre_filters, size=n_add_filters) + teacher_w, teacher_b = layer.get_weights() + + student_w = teacher_w.copy() + student_b = teacher_b.copy() + # target layer update (i) + for i, _ in enumerate(rand): + teacher_index = rand[i] + new_weight = teacher_w[teacher_index, ...] + new_weight = new_weight[np.newaxis, ...] + student_w = np.concatenate((student_w, new_weight), axis=0) + student_b = np.append(student_b, teacher_b[teacher_index]) + new_pre_layer = get_conv_class(n_dim)( + layer.input_channel, n_pre_filters + n_add_filters, layer.kernel_size + ) + new_pre_layer.set_weights( + (add_noise(student_w, teacher_w), add_noise(student_b, teacher_b)) + ) + return new_pre_layer + + +def wider_next_conv(layer, start_dim, total_dim, n_add, weighted=True): + '''wider next conv layer. + ''' + n_dim = get_n_dim(layer) + if not weighted: + return get_conv_class(n_dim)(layer.input_channel + n_add, + layer.filters, + kernel_size=layer.kernel_size, + stride=layer.stride) + n_filters = layer.filters + teacher_w, teacher_b = layer.get_weights() + + new_weight_shape = list(teacher_w.shape) + new_weight_shape[1] = n_add + new_weight = np.zeros(tuple(new_weight_shape)) + + student_w = np.concatenate((teacher_w[:, :start_dim, ...].copy(), + add_noise(new_weight, teacher_w), + teacher_w[:, start_dim:total_dim, ...].copy()), axis=1) + new_layer = get_conv_class(n_dim)(layer.input_channel + n_add, + n_filters, + kernel_size=layer.kernel_size, + stride=layer.stride) + new_layer.set_weights((student_w, teacher_b)) + return new_layer + + +def wider_bn(layer, start_dim, total_dim, n_add, weighted=True): + '''wider batch norm layer. + ''' + n_dim = get_n_dim(layer) + if not weighted: + return get_batch_norm_class(n_dim)(layer.num_features + n_add) + + weights = layer.get_weights() + + new_weights = [ + add_noise(np.ones(n_add, dtype=np.float32), np.array([0, 1])), + add_noise(np.zeros(n_add, dtype=np.float32), np.array([0, 1])), + add_noise(np.zeros(n_add, dtype=np.float32), np.array([0, 1])), + add_noise(np.ones(n_add, dtype=np.float32), np.array([0, 1])), + ] + + student_w = tuple() + for weight, new_weight in zip(weights, new_weights): + temp_w = weight.copy() + temp_w = np.concatenate( + (temp_w[:start_dim], new_weight, temp_w[start_dim:total_dim]) + ) + student_w += (temp_w,) + new_layer = get_batch_norm_class(n_dim)(layer.num_features + n_add) + new_layer.set_weights(student_w) + return new_layer + + +def wider_next_dense(layer, start_dim, total_dim, n_add, weighted=True): + '''wider next dense layer. + ''' + if not weighted: + return StubDense(layer.input_units + n_add, layer.units) + teacher_w, teacher_b = layer.get_weights() + student_w = teacher_w.copy() + n_units_each_channel = int(teacher_w.shape[1] / total_dim) + + new_weight = np.zeros((teacher_w.shape[0], n_add * n_units_each_channel)) + student_w = np.concatenate( + ( + student_w[:, : start_dim * n_units_each_channel], + add_noise(new_weight, student_w), + student_w[ + :, start_dim * n_units_each_channel: total_dim * n_units_each_channel + ], + ), + axis=1, + ) + + new_layer = StubDense(layer.input_units + n_add, layer.units) + new_layer.set_weights((student_w, teacher_b)) + return new_layer + + +def add_noise(weights, other_weights): + '''add noise to the layer. + ''' + w_range = np.ptp(other_weights.flatten()) + noise_range = NOISE_RATIO * w_range + noise = np.random.uniform(-noise_range / 2.0, + noise_range / 2.0, weights.shape) + return np.add(noise, weights) + + +def init_dense_weight(layer): + '''initilize dense layer weight. + ''' + units = layer.units + weight = np.eye(units) + bias = np.zeros(units) + layer.set_weights( + (add_noise(weight, np.array([0, 1])), + add_noise(bias, np.array([0, 1]))) + ) + + +def init_conv_weight(layer): + '''initilize conv layer weight. + ''' + n_filters = layer.filters + filter_shape = (layer.kernel_size,) * get_n_dim(layer) + weight = np.zeros((n_filters, n_filters) + filter_shape) + + center = tuple(map(lambda x: int((x - 1) / 2), filter_shape)) + for i in range(n_filters): + filter_weight = np.zeros((n_filters,) + filter_shape) + index = (i,) + center + filter_weight[index] = 1 + weight[i, ...] = filter_weight + bias = np.zeros(n_filters) + + layer.set_weights( + (add_noise(weight, np.array([0, 1])), + add_noise(bias, np.array([0, 1]))) + ) + + +def init_bn_weight(layer): + '''initilize batch norm layer weight. + ''' + n_filters = layer.num_features + new_weights = [ + add_noise(np.ones(n_filters, dtype=np.float32), np.array([0, 1])), + add_noise(np.zeros(n_filters, dtype=np.float32), np.array([0, 1])), + add_noise(np.zeros(n_filters, dtype=np.float32), np.array([0, 1])), + add_noise(np.ones(n_filters, dtype=np.float32), np.array([0, 1])), + ] + layer.set_weights(new_weights) diff --git a/utils/third_party/nni/algorithms/hpo/networkmorphism_tuner/layers.py b/utils/third_party/nni/algorithms/hpo/networkmorphism_tuner/layers.py new file mode 100644 index 0000000000000000000000000000000000000000..a96c87b7801fe2687f61dfd62fdd13019ecd2ee0 --- /dev/null +++ b/utils/third_party/nni/algorithms/hpo/networkmorphism_tuner/layers.py @@ -0,0 +1,862 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from abc import abstractmethod +from collections.abc import Iterable + +import torch +from torch import nn +from torch.nn import functional +from .utils import Constant + + +class AvgPool(nn.Module): + """ + AvgPool Module. + """ + + def __init__(self): + super().__init__() + + @abstractmethod + def forward(self, input_tensor): + pass + + +class GlobalAvgPool1d(AvgPool): + """ + GlobalAvgPool1d Module. + """ + + def forward(self, input_tensor): + return functional.avg_pool1d(input_tensor, input_tensor.size()[2:]).view( + input_tensor.size()[:2] + ) + + +class GlobalAvgPool2d(AvgPool): + """ + GlobalAvgPool2d Module. + """ + + def forward(self, input_tensor): + return functional.avg_pool2d(input_tensor, input_tensor.size()[2:]).view( + input_tensor.size()[:2] + ) + + +class GlobalAvgPool3d(AvgPool): + """ + GlobalAvgPool3d Module. + """ + + def forward(self, input_tensor): + return functional.avg_pool3d(input_tensor, input_tensor.size()[2:]).view( + input_tensor.size()[:2] + ) + + +class StubLayer: + """ + StubLayer Module. Base Module. + """ + + def __init__(self, input_node=None, output_node=None): + self.input = input_node + self.output = output_node + self.weights = None + + def build(self, shape): + """ + build shape. + """ + + def set_weights(self, weights): + """ + set weights. + """ + self.weights = weights + + def import_weights(self, torch_layer): + """ + import weights. + """ + + def import_weights_keras(self, keras_layer): + """ + import weights from keras layer. + """ + + def export_weights(self, torch_layer): + """ + export weights. + """ + + def export_weights_keras(self, keras_layer): + """ + export weights to keras layer. + """ + + def get_weights(self): + """ + get weights. + """ + return self.weights + + def size(self): + """ + size(). + """ + return 0 + + @property + def output_shape(self): + """ + output shape. + """ + return self.input.shape + + def to_real_layer(self): + """ + to real layer. + """ + + def __str__(self): + """ + str() function to print. + """ + return type(self).__name__[4:] + + +class StubWeightBiasLayer(StubLayer): + """ + StubWeightBiasLayer Module to set the bias. + """ + + def import_weights(self, torch_layer): + self.set_weights( + (torch_layer.weight.data.cpu().numpy(), + torch_layer.bias.data.cpu().numpy()) + ) + + def import_weights_keras(self, keras_layer): + self.set_weights(keras_layer.get_weights()) + + def export_weights(self, torch_layer): + torch_layer.weight.data = torch.Tensor(self.weights[0]) + torch_layer.bias.data = torch.Tensor(self.weights[1]) + + def export_weights_keras(self, keras_layer): + keras_layer.set_weights(self.weights) + + +class StubBatchNormalization(StubWeightBiasLayer): + """ + StubBatchNormalization Module. Batch Norm. + """ + + def __init__(self, num_features, input_node=None, output_node=None): + super().__init__(input_node, output_node) + self.num_features = num_features + + def import_weights(self, torch_layer): + self.set_weights( + ( + torch_layer.weight.data.cpu().numpy(), + torch_layer.bias.data.cpu().numpy(), + torch_layer.running_mean.cpu().numpy(), + torch_layer.running_var.cpu().numpy(), + ) + ) + + def export_weights(self, torch_layer): + torch_layer.weight.data = torch.Tensor(self.weights[0]) + torch_layer.bias.data = torch.Tensor(self.weights[1]) + torch_layer.running_mean = torch.Tensor(self.weights[2]) + torch_layer.running_var = torch.Tensor(self.weights[3]) + + def size(self): + return self.num_features * 4 + + @abstractmethod + def to_real_layer(self): + pass + + +class StubBatchNormalization1d(StubBatchNormalization): + """ + StubBatchNormalization1d Module. + """ + + def to_real_layer(self): + return torch.nn.BatchNorm1d(self.num_features) + + +class StubBatchNormalization2d(StubBatchNormalization): + """ + StubBatchNormalization2d Module. + """ + + def to_real_layer(self): + return torch.nn.BatchNorm2d(self.num_features) + + +class StubBatchNormalization3d(StubBatchNormalization): + """ + StubBatchNormalization3d Module. + """ + + def to_real_layer(self): + return torch.nn.BatchNorm3d(self.num_features) + + +class StubDense(StubWeightBiasLayer): + """ + StubDense Module. Linear. + """ + + def __init__(self, input_units, units, input_node=None, output_node=None): + super().__init__(input_node, output_node) + self.input_units = input_units + self.units = units + + @property + def output_shape(self): + return (self.units,) + + def import_weights_keras(self, keras_layer): + self.set_weights( + (keras_layer.get_weights()[0].T, + keras_layer.get_weights()[1])) + + def export_weights_keras(self, keras_layer): + keras_layer.set_weights((self.weights[0].T, self.weights[1])) + + def size(self): + return self.input_units * self.units + self.units + + def to_real_layer(self): + return torch.nn.Linear(self.input_units, self.units) + + +class StubConv(StubWeightBiasLayer): + """ + StubConv Module. Conv. + """ + + def __init__(self, input_channel, filters, kernel_size, + stride=1, input_node=None, output_node=None): + super().__init__(input_node, output_node) + self.input_channel = input_channel + self.filters = filters + self.kernel_size = kernel_size + self.stride = stride + self.padding = int(self.kernel_size / 2) + + @property + def output_shape(self): + ret = list(self.input.shape[:-1]) + for index, dim in enumerate(ret): + ret[index] = ( + int((dim + 2 * self.padding - self.kernel_size) / self.stride) + 1 + ) + ret = ret + [self.filters] + return tuple(ret) + + def import_weights_keras(self, keras_layer): + self.set_weights( + (keras_layer.get_weights()[0].T, + keras_layer.get_weights()[1])) + + def export_weights_keras(self, keras_layer): + keras_layer.set_weights((self.weights[0].T, self.weights[1])) + + def size(self): + return (self.input_channel * self.kernel_size * + self.kernel_size + 1) * self.filters + + @abstractmethod + def to_real_layer(self): + pass + + def __str__(self): + return ( + super().__str__() + + "(" + + ", ".join( + str(item) + for item in [ + self.input_channel, + self.filters, + self.kernel_size, + self.stride, + ] + ) + + ")" + ) + + +class StubConv1d(StubConv): + """ + StubConv1d Module. + """ + + def to_real_layer(self): + return torch.nn.Conv1d( + self.input_channel, + self.filters, + self.kernel_size, + stride=self.stride, + padding=self.padding, + ) + + +class StubConv2d(StubConv): + """ + StubConv2d Module. + """ + + def to_real_layer(self): + return torch.nn.Conv2d( + self.input_channel, + self.filters, + self.kernel_size, + stride=self.stride, + padding=self.padding, + ) + + +class StubConv3d(StubConv): + """ + StubConv3d Module. + """ + + def to_real_layer(self): + return torch.nn.Conv3d( + self.input_channel, + self.filters, + self.kernel_size, + stride=self.stride, + padding=self.padding, + ) + + +class StubAggregateLayer(StubLayer): + """ + StubAggregateLayer Module. + """ + + def __init__(self, input_nodes=None, output_node=None): + if input_nodes is None: + input_nodes = [] + super().__init__(input_nodes, output_node) + + +class StubConcatenate(StubAggregateLayer): + """StubConcatenate Module. + """ + @property + def output_shape(self): + ret = 0 + for current_input in self.input: + ret += current_input.shape[-1] + ret = self.input[0].shape[:-1] + (ret,) + return ret + + def to_real_layer(self): + return TorchConcatenate() + + +class StubAdd(StubAggregateLayer): + """ + StubAdd Module. + """ + @property + def output_shape(self): + return self.input[0].shape + + def to_real_layer(self): + return TorchAdd() + + +class StubFlatten(StubLayer): + """ + StubFlatten Module. + """ + @property + def output_shape(self): + ret = 1 + for dim in self.input.shape: + ret *= dim + return (ret,) + + def to_real_layer(self): + return TorchFlatten() + + +class StubReLU(StubLayer): + """ + StubReLU Module. + """ + + def to_real_layer(self): + return torch.nn.ReLU() + + +class StubSoftmax(StubLayer): + """ + StubSoftmax Module. + """ + + def to_real_layer(self): + return torch.nn.LogSoftmax(dim=1) + + +class StubDropout(StubLayer): + """ + StubDropout Module. + """ + + def __init__(self, rate, input_node=None, output_node=None): + super().__init__(input_node, output_node) + self.rate = rate + + @abstractmethod + def to_real_layer(self): + pass + + +class StubDropout1d(StubDropout): + """ + StubDropout1d Module. + """ + + def to_real_layer(self): + return torch.nn.Dropout(self.rate) + + +class StubDropout2d(StubDropout): + """ + StubDropout2d Module. + """ + + def to_real_layer(self): + return torch.nn.Dropout2d(self.rate) + + +class StubDropout3d(StubDropout): + """ + StubDropout3d Module. + """ + + def to_real_layer(self): + return torch.nn.Dropout3d(self.rate) + + +class StubInput(StubLayer): + """ + StubInput Module. + """ + + def __init__(self, input_node=None, output_node=None): + super().__init__(input_node, output_node) + + +class StubPooling(StubLayer): + """ + StubPooling Module. + """ + + def __init__(self, + kernel_size=None, + stride=None, + padding=0, + input_node=None, + output_node=None): + super().__init__(input_node, output_node) + self.kernel_size = ( + kernel_size if kernel_size is not None else Constant.POOLING_KERNEL_SIZE + ) + self.stride = stride if stride is not None else self.kernel_size + self.padding = padding + + @property + def output_shape(self): + ret = tuple() + for dim in self.input.shape[:-1]: + ret = ret + (max(int(dim / self.kernel_size), 1),) + ret = ret + (self.input.shape[-1],) + return ret + + @abstractmethod + def to_real_layer(self): + pass + + +class StubPooling1d(StubPooling): + """ + StubPooling1d Module. + """ + + def to_real_layer(self): + return torch.nn.MaxPool1d(self.kernel_size, stride=self.stride) + + +class StubPooling2d(StubPooling): + """ + StubPooling2d Module. + """ + + def to_real_layer(self): + return torch.nn.MaxPool2d(self.kernel_size, stride=self.stride) + + +class StubPooling3d(StubPooling): + """ + StubPooling3d Module. + """ + + def to_real_layer(self): + return torch.nn.MaxPool3d(self.kernel_size, stride=self.stride) + + +class StubGlobalPooling(StubLayer): + """ + StubGlobalPooling Module. + """ + + def __init__(self, input_node=None, output_node=None): + super().__init__(input_node, output_node) + + @property + def output_shape(self): + return (self.input.shape[-1],) + + @abstractmethod + def to_real_layer(self): + pass + + +class StubGlobalPooling1d(StubGlobalPooling): + """ + StubGlobalPooling1d Module. + """ + + def to_real_layer(self): + return GlobalAvgPool1d() + + +class StubGlobalPooling2d(StubGlobalPooling): + """ + StubGlobalPooling2d Module. + """ + + def to_real_layer(self): + return GlobalAvgPool2d() + + +class StubGlobalPooling3d(StubGlobalPooling): + """ + StubGlobalPooling3d Module. + """ + + def to_real_layer(self): + return GlobalAvgPool3d() + + +class TorchConcatenate(nn.Module): + """ + TorchConcatenate Module. + """ + + def forward(self, input_list): + return torch.cat(input_list, dim=1) + + +class TorchAdd(nn.Module): + """ + TorchAdd Module. + """ + + def forward(self, input_list): + return input_list[0] + input_list[1] + + +class TorchFlatten(nn.Module): + """ + TorchFlatten Module. + """ + + def forward(self, input_tensor): + return input_tensor.view(input_tensor.size(0), -1) + + +def keras_dropout(layer, rate): + """ + Keras dropout layer. + """ + + from keras import layers + + input_dim = len(layer.input.shape) + if input_dim == 2: + return layers.SpatialDropout1D(rate) + elif input_dim == 3: + return layers.SpatialDropout2D(rate) + elif input_dim == 4: + return layers.SpatialDropout3D(rate) + else: + return layers.Dropout(rate) + + +def to_real_keras_layer(layer): + """ + Real keras layer. + """ + from keras import layers + + if is_layer(layer, "Dense"): + return layers.Dense(layer.units, input_shape=(layer.input_units,)) + if is_layer(layer, "Conv"): + return layers.Conv2D( + layer.filters, + layer.kernel_size, + input_shape=layer.input.shape, + padding="same", + ) # padding + if is_layer(layer, "Pooling"): + return layers.MaxPool2D(2) + if is_layer(layer, "BatchNormalization"): + return layers.BatchNormalization(input_shape=layer.input.shape) + if is_layer(layer, "Concatenate"): + return layers.Concatenate() + if is_layer(layer, "Add"): + return layers.Add() + if is_layer(layer, "Dropout"): + return keras_dropout(layer, layer.rate) + if is_layer(layer, "ReLU"): + return layers.Activation("relu") + if is_layer(layer, "Softmax"): + return layers.Activation("softmax") + if is_layer(layer, "Flatten"): + return layers.Flatten() + if is_layer(layer, "GlobalAveragePooling"): + return layers.GlobalAveragePooling2D() + return None # note: this is not written by original author, feel free to modify if you think it's incorrect + + +def is_layer(layer, layer_type): + """ + Judge the layer type. + + Returns + ------- + bool + boolean -- True or False + """ + + if layer_type == "Input": + return isinstance(layer, StubInput) + elif layer_type == "Conv": + return isinstance(layer, StubConv) + elif layer_type == "Dense": + return isinstance(layer, (StubDense,)) + elif layer_type == "BatchNormalization": + return isinstance(layer, (StubBatchNormalization,)) + elif layer_type == "Concatenate": + return isinstance(layer, (StubConcatenate,)) + elif layer_type == "Add": + return isinstance(layer, (StubAdd,)) + elif layer_type == "Pooling": + return isinstance(layer, StubPooling) + elif layer_type == "Dropout": + return isinstance(layer, (StubDropout,)) + elif layer_type == "Softmax": + return isinstance(layer, (StubSoftmax,)) + elif layer_type == "ReLU": + return isinstance(layer, (StubReLU,)) + elif layer_type == "Flatten": + return isinstance(layer, (StubFlatten,)) + elif layer_type == "GlobalAveragePooling": + return isinstance(layer, StubGlobalPooling) + return None # note: this is not written by original author, feel free to modify if you think it's incorrect + + +def layer_description_extractor(layer, node_to_id): + """ + Get layer description. + """ + + layer_input = layer.input + layer_output = layer.output + if layer_input is not None: + if isinstance(layer_input, Iterable): + layer_input = list(map(lambda x: node_to_id[x], layer_input)) + else: + layer_input = node_to_id[layer_input] + + if layer_output is not None: + layer_output = node_to_id[layer_output] + + if isinstance(layer, StubConv): + return ( + type(layer).__name__, + layer_input, + layer_output, + layer.input_channel, + layer.filters, + layer.kernel_size, + layer.stride, + layer.padding, + ) + elif isinstance(layer, (StubDense,)): + return [ + type(layer).__name__, + layer_input, + layer_output, + layer.input_units, + layer.units, + ] + elif isinstance(layer, (StubBatchNormalization,)): + return (type(layer).__name__, layer_input, + layer_output, layer.num_features) + elif isinstance(layer, (StubDropout,)): + return (type(layer).__name__, layer_input, layer_output, layer.rate) + elif isinstance(layer, StubPooling): + return ( + type(layer).__name__, + layer_input, + layer_output, + layer.kernel_size, + layer.stride, + layer.padding, + ) + else: + return (type(layer).__name__, layer_input, layer_output) + + +def layer_description_builder(layer_information, id_to_node): + """build layer from description. + """ + layer_type = layer_information[0] + + layer_input_ids = layer_information[1] + if isinstance(layer_input_ids, Iterable): + layer_input = list(map(lambda x: id_to_node[x], layer_input_ids)) + else: + layer_input = id_to_node[layer_input_ids] + layer_output = id_to_node[layer_information[2]] + if layer_type.startswith("StubConv"): + input_channel = layer_information[3] + filters = layer_information[4] + kernel_size = layer_information[5] + stride = layer_information[6] + return globals()[layer_type]( + input_channel, filters, kernel_size, stride, layer_input, layer_output + ) + elif layer_type.startswith("StubDense"): + input_units = layer_information[3] + units = layer_information[4] + return globals()[layer_type](input_units, units, layer_input, layer_output) + elif layer_type.startswith("StubBatchNormalization"): + num_features = layer_information[3] + return globals()[layer_type](num_features, layer_input, layer_output) + elif layer_type.startswith("StubDropout"): + rate = layer_information[3] + return globals()[layer_type](rate, layer_input, layer_output) + elif layer_type.startswith("StubPooling"): + kernel_size = layer_information[3] + stride = layer_information[4] + padding = layer_information[5] + return globals()[layer_type](kernel_size, stride, padding, layer_input, layer_output) + else: + return globals()[layer_type](layer_input, layer_output) + + +def layer_width(layer): + """ + Get layer width. + """ + + if is_layer(layer, "Dense"): + return layer.units + if is_layer(layer, "Conv"): + return layer.filters + raise TypeError("The layer should be either Dense or Conv layer.") + + +def set_torch_weight_to_stub(torch_layer, stub_layer): + stub_layer.import_weights(torch_layer) + + +def set_keras_weight_to_stub(keras_layer, stub_layer): + stub_layer.import_weights_keras(keras_layer) + + +def set_stub_weight_to_torch(stub_layer, torch_layer): + stub_layer.export_weights(torch_layer) + + +def set_stub_weight_to_keras(stub_layer, keras_layer): + stub_layer.export_weights_keras(keras_layer) + + +def get_conv_class(n_dim): + conv_class_list = [StubConv1d, StubConv2d, StubConv3d] + return conv_class_list[n_dim - 1] + + +def get_dropout_class(n_dim): + dropout_class_list = [StubDropout1d, StubDropout2d, StubDropout3d] + return dropout_class_list[n_dim - 1] + + +def get_global_avg_pooling_class(n_dim): + global_avg_pooling_class_list = [ + StubGlobalPooling1d, + StubGlobalPooling2d, + StubGlobalPooling3d, + ] + return global_avg_pooling_class_list[n_dim - 1] + + +def get_pooling_class(n_dim): + pooling_class_list = [StubPooling1d, StubPooling2d, StubPooling3d] + return pooling_class_list[n_dim - 1] + + +def get_batch_norm_class(n_dim): + batch_norm_class_list = [ + StubBatchNormalization1d, + StubBatchNormalization2d, + StubBatchNormalization3d, + ] + return batch_norm_class_list[n_dim - 1] + + +def get_n_dim(layer): + if isinstance(layer, ( + StubConv1d, + StubDropout1d, + StubGlobalPooling1d, + StubPooling1d, + StubBatchNormalization1d, + )): + return 1 + if isinstance(layer, ( + StubConv2d, + StubDropout2d, + StubGlobalPooling2d, + StubPooling2d, + StubBatchNormalization2d, + )): + return 2 + if isinstance(layer, ( + StubConv3d, + StubDropout3d, + StubGlobalPooling3d, + StubPooling3d, + StubBatchNormalization3d, + )): + return 3 + return -1 diff --git a/utils/third_party/nni/algorithms/hpo/networkmorphism_tuner/networkmorphism_tuner.py b/utils/third_party/nni/algorithms/hpo/networkmorphism_tuner/networkmorphism_tuner.py new file mode 100644 index 0000000000000000000000000000000000000000..6a73cad3c6aa345a4e075b571b731417b1bc1615 --- /dev/null +++ b/utils/third_party/nni/algorithms/hpo/networkmorphism_tuner/networkmorphism_tuner.py @@ -0,0 +1,328 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +networkmorphsim_tuner.py +""" + +import logging +import os +from schema import Optional, Schema +from nni.tuner import Tuner +from nni.utils import OptimizeMode, extract_scalar_reward +from .bayesian import BayesianOptimizer +from .nn import CnnGenerator, MlpGenerator +from .utils import Constant +from .graph import graph_to_json, json_to_graph +from nni import ClassArgsValidator + +logger = logging.getLogger("NetworkMorphism_AutoML") + +class NetworkMorphismClassArgsValidator(ClassArgsValidator): + def validate_class_args(self, **kwargs): + Schema({ + Optional('optimize_mode'): self.choices('optimize_mode', 'maximize', 'minimize'), + Optional('task'): self.choices('task', 'cv', 'nlp', 'common'), + Optional('input_width'): int, + Optional('input_channel'): int, + Optional('n_output_node'): int + }).validate(kwargs) + +class NetworkMorphismTuner(Tuner): + """ + NetworkMorphismTuner is a tuner which using network morphism techniques. + + Attributes + ---------- + n_classes : int + The class number or output node number (default: ``10``) + input_shape : tuple + A tuple including: (input_width, input_width, input_channel) + t_min : float + The minimum temperature for simulated annealing. (default: ``Constant.T_MIN``) + beta : float + The beta in acquisition function. (default: ``Constant.BETA``) + algorithm_name : str + algorithm name used in the network morphism (default: ``"Bayesian"``) + optimize_mode : str + optimize mode "minimize" or "maximize" (default: ``"minimize"``) + verbose : bool + verbose to print the log (default: ``True``) + bo : BayesianOptimizer + The optimizer used in networkmorphsim tuner. + max_model_size : int + max model size to the graph (default: ``Constant.MAX_MODEL_SIZE``) + default_model_len : int + default model length (default: ``Constant.MODEL_LEN``) + default_model_width : int + default model width (default: ``Constant.MODEL_WIDTH``) + search_space : dict + """ + + def __init__( + self, + task="cv", + input_width=32, + input_channel=3, + n_output_node=10, + algorithm_name="Bayesian", + optimize_mode="maximize", + path="model_path", + verbose=True, + beta=Constant.BETA, + t_min=Constant.T_MIN, + max_model_size=Constant.MAX_MODEL_SIZE, + default_model_len=Constant.MODEL_LEN, + default_model_width=Constant.MODEL_WIDTH, + ): + """ + initilizer of the NetworkMorphismTuner. + """ + + if not os.path.exists(path): + os.makedirs(path) + self.path = os.path.join(os.getcwd(), path) + if task == "cv": + self.generators = [CnnGenerator] + elif task == "common": + self.generators = [MlpGenerator] + else: + raise NotImplementedError( + '{} task not supported in List ["cv","common"]') + + self.n_classes = n_output_node + self.input_shape = (input_width, input_width, input_channel) + + self.t_min = t_min + self.beta = beta + self.algorithm_name = algorithm_name + self.optimize_mode = OptimizeMode(optimize_mode) + self.json = None + self.total_data = {} + self.verbose = verbose + self.model_count = 0 + + self.bo = BayesianOptimizer( + self, self.t_min, self.optimize_mode, self.beta) + self.training_queue = [] + self.descriptors = [] + self.history = [] + + self.max_model_size = max_model_size + self.default_model_len = default_model_len + self.default_model_width = default_model_width + + self.search_space = dict() + + + def update_search_space(self, search_space): + """ + Update search space definition in tuner by search_space in neural architecture. + """ + self.search_space = search_space + + def generate_parameters(self, parameter_id, **kwargs): + """ + Returns a set of trial neural architecture, as a serializable object. + + Parameters + ---------- + parameter_id : int + """ + if not self.history: + self.init_search() + + new_father_id = None + generated_graph = None + if not self.training_queue: + new_father_id, generated_graph = self.generate() + new_model_id = self.model_count + self.model_count += 1 + self.training_queue.append( + (generated_graph, new_father_id, new_model_id)) + self.descriptors.append(generated_graph.extract_descriptor()) + + graph, father_id, model_id = self.training_queue.pop(0) + + # from graph to json + json_model_path = os.path.join(self.path, str(model_id) + ".json") + json_out = graph_to_json(graph, json_model_path) + self.total_data[parameter_id] = (json_out, father_id, model_id) + + return json_out + + def receive_trial_result(self, parameter_id, parameters, value, **kwargs): + """ + Record an observation of the objective function. + + Parameters + ---------- + parameter_id : int + the id of a group of paramters that generated by nni manager. + parameters : dict + A group of parameters. + value : dict/float + if value is dict, it should have "default" key. + """ + reward = extract_scalar_reward(value) + + if parameter_id not in self.total_data: + raise RuntimeError("Received parameter_id not in total_data.") + + (_, father_id, model_id) = self.total_data[parameter_id] + + graph = self.bo.searcher.load_model_by_id(model_id) + + # to use the value and graph + self.add_model(reward, model_id) + self.update(father_id, graph, reward, model_id) + + + def init_search(self): + """ + Call the generators to generate the initial architectures for the search. + """ + if self.verbose: + logger.info("Initializing search.") + for generator in self.generators: + graph = generator(self.n_classes, self.input_shape).generate( + self.default_model_len, self.default_model_width + ) + model_id = self.model_count + self.model_count += 1 + self.training_queue.append((graph, -1, model_id)) + self.descriptors.append(graph.extract_descriptor()) + + if self.verbose: + logger.info("Initialization finished.") + + + def generate(self): + """ + Generate the next neural architecture. + + Returns + ------- + other_info : any object + Anything to be saved in the training queue together with the architecture. + generated_graph : Graph + An instance of Graph. + """ + generated_graph, new_father_id = self.bo.generate(self.descriptors) + if new_father_id is None: + new_father_id = 0 + generated_graph = self.generators[0]( + self.n_classes, self.input_shape + ).generate(self.default_model_len, self.default_model_width) + + return new_father_id, generated_graph + + def update(self, other_info, graph, metric_value, model_id): + """ + Update the controller with evaluation result of a neural architecture. + + Parameters + ---------- + other_info: any object + In our case it is the father ID in the search tree. + graph: Graph + An instance of Graph. The trained neural architecture. + metric_value: float + The final evaluated metric value. + model_id: int + """ + father_id = other_info + self.bo.fit([graph.extract_descriptor()], [metric_value]) + self.bo.add_child(father_id, model_id) + + def add_model(self, metric_value, model_id): + """ + Add model to the history, x_queue and y_queue + + Parameters + ---------- + metric_value : float + graph : dict + model_id : int + + Returns + ------- + model : dict + """ + if self.verbose: + logger.info("Saving model.") + + # Update best_model text file + ret = {"model_id": model_id, "metric_value": metric_value} + self.history.append(ret) + if model_id == self.get_best_model_id(): + file = open(os.path.join(self.path, "best_model.txt"), "w") + file.write("best model: " + str(model_id)) + file.close() + return ret + + + def get_best_model_id(self): + """ + Get the best model_id from history using the metric value + """ + + if self.optimize_mode is OptimizeMode.Maximize: + return max(self.history, key=lambda x: x["metric_value"])[ + "model_id"] + return min(self.history, key=lambda x: x["metric_value"])["model_id"] + + + def load_model_by_id(self, model_id): + """ + Get the model by model_id + + Parameters + ---------- + model_id : int + model index + + Returns + ------- + load_model : Graph + the model graph representation + """ + + with open(os.path.join(self.path, str(model_id) + ".json")) as fin: + json_str = fin.read().replace("\n", "") + + load_model = json_to_graph(json_str) + return load_model + + def load_best_model(self): + """ + Get the best model by model id + + Returns + ------- + load_model : Graph + the model graph representation + """ + return self.load_model_by_id(self.get_best_model_id()) + + def get_metric_value_by_id(self, model_id): + """ + Get the model metric valud by its model_id + + Parameters + ---------- + model_id : int + model index + + Returns + ------- + float + the model metric + """ + for item in self.history: + if item["model_id"] == model_id: + return item["metric_value"] + return None + + def import_data(self, data): + pass diff --git a/utils/third_party/nni/algorithms/hpo/networkmorphism_tuner/nn.py b/utils/third_party/nni/algorithms/hpo/networkmorphism_tuner/nn.py new file mode 100644 index 0000000000000000000000000000000000000000..9e0072f9b39e3a68b865ba850157fe1080791983 --- /dev/null +++ b/utils/third_party/nni/algorithms/hpo/networkmorphism_tuner/nn.py @@ -0,0 +1,166 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from abc import abstractmethod + +from .graph import Graph +from .layers import (StubDense, StubDropout1d, + StubReLU, get_batch_norm_class, + get_conv_class, + get_dropout_class, + get_global_avg_pooling_class, + get_pooling_class) +from .utils import Constant + + +class NetworkGenerator: + """The base class for generating a network. + It can be used to generate a CNN or Multi-Layer Perceptron. + Attributes: + n_output_node: Number of output nodes in the network. + input_shape: A tuple to represent the input shape. + """ + + def __init__(self, n_output_node, input_shape): + self.n_output_node = n_output_node + self.input_shape = input_shape + + @abstractmethod + def generate(self, model_len, model_width): + pass + + +class CnnGenerator(NetworkGenerator): + """A class to generate CNN. + Attributes: + n_dim: `len(self.input_shape) - 1` + conv: A class that represents `(n_dim-1)` dimensional convolution. + dropout: A class that represents `(n_dim-1)` dimensional dropout. + global_avg_pooling: A class that represents `(n_dim-1)` dimensional Global Average Pooling. + pooling: A class that represents `(n_dim-1)` dimensional pooling. + batch_norm: A class that represents `(n_dim-1)` dimensional batch normalization. + """ + + def __init__(self, n_output_node, input_shape): + super(CnnGenerator, self).__init__(n_output_node, input_shape) + self.n_dim = len(self.input_shape) - 1 + if len(self.input_shape) > 4: + raise ValueError("The input dimension is too high.") + if len(self.input_shape) < 2: + raise ValueError("The input dimension is too low.") + self.conv = get_conv_class(self.n_dim) + self.dropout = get_dropout_class(self.n_dim) + self.global_avg_pooling = get_global_avg_pooling_class(self.n_dim) + self.pooling = get_pooling_class(self.n_dim) + self.batch_norm = get_batch_norm_class(self.n_dim) + + def generate(self, model_len=None, model_width=None): + """Generates a CNN. + Args: + model_len: An integer. Number of convolutional layers. + model_width: An integer. Number of filters for the convolutional layers. + Returns: + An instance of the class Graph. Represents the neural architecture graph of the generated model. + """ + + if model_len is None: + model_len = Constant.MODEL_LEN + if model_width is None: + model_width = Constant.MODEL_WIDTH + pooling_len = int(model_len / 4) + graph = Graph(self.input_shape, False) + temp_input_channel = self.input_shape[-1] + output_node_id = 0 + stride = 1 + for i in range(model_len): + output_node_id = graph.add_layer(StubReLU(), output_node_id) + output_node_id = graph.add_layer( + self.batch_norm( + graph.node_list[output_node_id].shape[-1]), output_node_id + ) + output_node_id = graph.add_layer( + self.conv( + temp_input_channel, + model_width, + kernel_size=3, + stride=stride), + output_node_id, + ) + temp_input_channel = model_width + if pooling_len == 0 or ( + (i + 1) % pooling_len == 0 and i != model_len - 1): + output_node_id = graph.add_layer( + self.pooling(), output_node_id) + + output_node_id = graph.add_layer( + self.global_avg_pooling(), output_node_id) + output_node_id = graph.add_layer( + self.dropout(Constant.CONV_DROPOUT_RATE), output_node_id + ) + output_node_id = graph.add_layer( + StubDense(graph.node_list[output_node_id].shape[0], model_width), + output_node_id, + ) + output_node_id = graph.add_layer(StubReLU(), output_node_id) + graph.add_layer( + StubDense( + model_width, + self.n_output_node), + output_node_id) + return graph + + +class MlpGenerator(NetworkGenerator): + """A class to generate Multi-Layer Perceptron. + """ + + def __init__(self, n_output_node, input_shape): + """Initialize the instance. + Args: + n_output_node: An integer. Number of output nodes in the network. + input_shape: A tuple. Input shape of the network. If it is 1D, ensure the value is appended by a comma + in the tuple. + """ + super(MlpGenerator, self).__init__(n_output_node, input_shape) + if len(self.input_shape) > 1: + raise ValueError("The input dimension is too high.") + + def generate(self, model_len=None, model_width=None): + """Generates a Multi-Layer Perceptron. + Args: + model_len: An integer. Number of hidden layers. + model_width: An integer or a list of integers of length `model_len`. If it is a list, it represents the + number of nodes in each hidden layer. If it is an integer, all hidden layers have nodes equal to this + value. + Returns: + An instance of the class Graph. Represents the neural architecture graph of the generated model. + """ + if model_len is None: + model_len = Constant.MODEL_LEN + if model_width is None: + model_width = Constant.MODEL_WIDTH + if isinstance(model_width, list) and not len(model_width) == model_len: + raise ValueError( + "The length of 'model_width' does not match 'model_len'") + elif isinstance(model_width, int): + model_width = [model_width] * model_len + + graph = Graph(self.input_shape, False) + output_node_id = 0 + n_nodes_prev_layer = self.input_shape[0] + for width in model_width: + output_node_id = graph.add_layer( + StubDense(n_nodes_prev_layer, width), output_node_id + ) + output_node_id = graph.add_layer( + StubDropout1d(Constant.MLP_DROPOUT_RATE), output_node_id + ) + output_node_id = graph.add_layer(StubReLU(), output_node_id) + n_nodes_prev_layer = width + + graph.add_layer( + StubDense( + n_nodes_prev_layer, + self.n_output_node), + output_node_id) + return graph diff --git a/utils/third_party/nni/algorithms/hpo/networkmorphism_tuner/utils.py b/utils/third_party/nni/algorithms/hpo/networkmorphism_tuner/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..0634e7f578bdeb505ce984be971beb3978c78a44 --- /dev/null +++ b/utils/third_party/nni/algorithms/hpo/networkmorphism_tuner/utils.py @@ -0,0 +1,23 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + + +class Constant: + '''Constant for the Tuner. + ''' + MAX_LAYERS = 100 + N_NEIGHBOURS = 8 + MAX_MODEL_SIZE = 1 << 24 + KERNEL_LAMBDA = 1.0 + BETA = 2.576 + MLP_MODEL_LEN = 3 + MLP_MODEL_WIDTH = 5 + MODEL_LEN = 3 + MODEL_WIDTH = 64 + POOLING_KERNEL_SIZE = 2 + DENSE_DROPOUT_RATE = 0.5 + CONV_DROPOUT_RATE = 0.25 + MLP_DROPOUT_RATE = 0.25 + CONV_BLOCK_DISTANCE = 2 + BATCH_SIZE = 128 + T_MIN = 0.0001 diff --git a/utils/third_party/nni/algorithms/hpo/pbt_tuner/__init__.py b/utils/third_party/nni/algorithms/hpo/pbt_tuner/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/utils/third_party/nni/algorithms/hpo/pbt_tuner/pbt_tuner.py b/utils/third_party/nni/algorithms/hpo/pbt_tuner/pbt_tuner.py new file mode 100644 index 0000000000000000000000000000000000000000..507c519a2a880ca99833433a6f61278e308a469f --- /dev/null +++ b/utils/third_party/nni/algorithms/hpo/pbt_tuner/pbt_tuner.py @@ -0,0 +1,456 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import copy +import logging +import os +import random +import numpy as np +from schema import Schema, Optional + +import nni +from nni import ClassArgsValidator +import nni.parameter_expressions +from nni.tuner import Tuner +from nni.utils import OptimizeMode, extract_scalar_reward, split_index, json2parameter, json2space + + +logger = logging.getLogger('pbt_tuner_AutoML') + + +def perturbation(hyperparameter_type, value, resample_probablity, uv, ub, lv, lb, random_state): + """ + Perturbation for hyperparameters + + Parameters + ---------- + hyperparameter_type : str + type of hyperparameter + value : list + parameters for sampling hyperparameter + resample_probability : float + probability for resampling + uv : float/int + upper value after perturbation + ub : float/int + upper bound + lv : float/int + lower value after perturbation + lb : float/int + lower bound + random_state : RandomState + random state + """ + if random.random() < resample_probablity: + if hyperparameter_type == "choice": + return value.index(nni.parameter_expressions.choice(value, random_state)) + else: + return getattr(nni.parameter_expressions, hyperparameter_type)(*(value + [random_state])) + else: + if random.random() > 0.5: + return min(uv, ub) + else: + return max(lv, lb) + + +def exploit_and_explore(bot_trial_info, top_trial_info, factor, resample_probability, epoch, search_space): + """ + Replace checkpoint of bot_trial with top, and perturb hyperparameters + + Parameters + ---------- + bot_trial_info : TrialInfo + bottom model whose parameters should be replaced + top_trial_info : TrialInfo + better model + factor : float + factor for perturbation + resample_probability : float + probability for resampling + epoch : int + step of PBTTuner + search_space : dict + search_space to keep perturbed hyperparameters in range + """ + bot_checkpoint_dir = bot_trial_info.checkpoint_dir + top_hyper_parameters = top_trial_info.hyper_parameters + hyper_parameters = copy.deepcopy(top_hyper_parameters) + random_state = np.random.RandomState() + hyper_parameters['load_checkpoint_dir'] = hyper_parameters['save_checkpoint_dir'] + hyper_parameters['save_checkpoint_dir'] = os.path.join(bot_checkpoint_dir, str(epoch)) + for key in hyper_parameters.keys(): + hyper_parameter = hyper_parameters[key] + if key == 'load_checkpoint_dir' or key == 'save_checkpoint_dir': + continue + elif search_space[key]["_type"] == "choice": + choices = search_space[key]["_value"] + ub, uv = len(choices) - 1, choices.index(hyper_parameter) + 1 + lb, lv = 0, choices.index(hyper_parameter) - 1 + elif search_space[key]["_type"] == "randint": + lb, ub = search_space[key]["_value"][:2] + ub -= 1 + uv = hyper_parameter + 1 + lv = hyper_parameter - 1 + elif search_space[key]["_type"] == "uniform": + lb, ub = search_space[key]["_value"][:2] + perturb = (ub - lb) * factor + uv = hyper_parameter + perturb + lv = hyper_parameter - perturb + elif search_space[key]["_type"] == "quniform": + lb, ub, q = search_space[key]["_value"][:3] + multi = round(hyper_parameter / q) + uv = (multi + 1) * q + lv = (multi - 1) * q + elif search_space[key]["_type"] == "loguniform": + lb, ub = search_space[key]["_value"][:2] + perturb = (np.log(ub) - np.log(lb)) * factor + uv = np.exp(min(np.log(hyper_parameter) + perturb, np.log(ub))) + lv = np.exp(max(np.log(hyper_parameter) - perturb, np.log(lb))) + elif search_space[key]["_type"] == "qloguniform": + lb, ub, q = search_space[key]["_value"][:3] + multi = round(hyper_parameter / q) + uv = (multi + 1) * q + lv = (multi - 1) * q + elif search_space[key]["_type"] == "normal": + sigma = search_space[key]["_value"][1] + perturb = sigma * factor + uv = ub = hyper_parameter + perturb + lv = lb = hyper_parameter - perturb + elif search_space[key]["_type"] == "qnormal": + q = search_space[key]["_value"][2] + uv = ub = hyper_parameter + q + lv = lb = hyper_parameter - q + elif search_space[key]["_type"] == "lognormal": + sigma = search_space[key]["_value"][1] + perturb = sigma * factor + uv = ub = np.exp(np.log(hyper_parameter) + perturb) + lv = lb = np.exp(np.log(hyper_parameter) - perturb) + elif search_space[key]["_type"] == "qlognormal": + q = search_space[key]["_value"][2] + uv = ub = hyper_parameter + q + lv, lb = hyper_parameter - q, 1E-10 + else: + logger.warning("Illegal type to perturb: %s", search_space[key]["_type"]) + continue + + if search_space[key]["_type"] == "choice": + idx = perturbation(search_space[key]["_type"], search_space[key]["_value"], + resample_probability, uv, ub, lv, lb, random_state) + hyper_parameters[key] = choices[idx] + else: + hyper_parameters[key] = perturbation(search_space[key]["_type"], search_space[key]["_value"], + resample_probability, uv, ub, lv, lb, random_state) + bot_trial_info.hyper_parameters = hyper_parameters + bot_trial_info.clean_id() + + +class TrialInfo: + """ + Information of each trial, refresh for each epoch + + """ + + def __init__(self, checkpoint_dir=None, hyper_parameters=None, parameter_id=None, score=None): + self.checkpoint_dir = checkpoint_dir + self.hyper_parameters = hyper_parameters + self.parameter_id = parameter_id + self.score = score + + def clean_id(self): + self.parameter_id = None + +class PBTClassArgsValidator(ClassArgsValidator): + def validate_class_args(self, **kwargs): + Schema({ + 'optimize_mode': self.choices('optimize_mode', 'maximize', 'minimize'), + Optional('all_checkpoint_dir'): str, + Optional('population_size'): self.range('population_size', int, 0, 99999), + Optional('factors'): float, + Optional('fraction'): float, + }).validate(kwargs) + +class PBTTuner(Tuner): + def __init__(self, optimize_mode="maximize", all_checkpoint_dir=None, population_size=10, factor=0.2, + resample_probability=0.25, fraction=0.2): + """ + Initialization + + Parameters + ---------- + optimize_mode : str + maximize or minimize + all_checkpoint_dir : str + directory to store training model checkpoint + population_size : int + number of trials for each epoch + factor : float + factor for perturbation + resample_probability : float + probability for resampling + fraction : float + fraction for selecting bottom and top trials + """ + self.optimize_mode = OptimizeMode(optimize_mode) + if all_checkpoint_dir is None: + all_checkpoint_dir = os.getenv('NNI_CHECKPOINT_DIRECTORY') + logger.info("Checkpoint dir is set to %s by default.", all_checkpoint_dir) + self.all_checkpoint_dir = all_checkpoint_dir + self.population_size = population_size + self.factor = factor + self.resample_probability = resample_probability + self.fraction = fraction + # defined in trial code + #self.perturbation_interval = perturbation_interval + + self.population = None + self.pos = -1 + self.param_ids = [] + self.running = {} + self.finished = [] + self.credit = 0 + self.finished_trials = 0 + self.epoch = 0 + + self.searchspace_json = None + self.space = None + + self.send_trial_callback = None + + logger.info('PBT tuner initialization') + + def update_search_space(self, search_space): + """ + Get search space + + Parameters + ---------- + search_space : dict + Search space + """ + logger.info('Update search space %s', search_space) + self.searchspace_json = search_space + self.space = json2space(self.searchspace_json) + + self.random_state = np.random.RandomState() + self.population = [] + is_rand = dict() + + for item in self.space: + is_rand[item] = True + + for i in range(self.population_size): + hyper_parameters = json2parameter( + self.searchspace_json, is_rand, self.random_state) + hyper_parameters = split_index(hyper_parameters) + checkpoint_dir = os.path.join(self.all_checkpoint_dir, str(i)) + hyper_parameters['load_checkpoint_dir'] = os.path.join(checkpoint_dir, str(self.epoch)) + hyper_parameters['save_checkpoint_dir'] = os.path.join(checkpoint_dir, str(self.epoch)) + self.population.append(TrialInfo(checkpoint_dir=checkpoint_dir, hyper_parameters=hyper_parameters)) + + def generate_multiple_parameters(self, parameter_id_list, **kwargs): + """ + Returns multiple sets of trial (hyper-)parameters, as iterable of serializable objects. + + Parameters + ---------- + parameter_id_list : list of int + Unique identifiers for each set of requested hyper-parameters. + These will later be used in :meth:`receive_trial_result`. + **kwargs + Used for send_trial_callback. + + Returns + ------- + list + A list of newly generated configurations + """ + result = [] + self.send_trial_callback = kwargs['st_callback'] + for parameter_id in parameter_id_list: + had_exception = False + try: + logger.debug("generating param for %s", parameter_id) + res = self.generate_parameters(parameter_id, **kwargs) + except nni.NoMoreTrialError: + had_exception = True + if not had_exception: + result.append(res) + return result + + def generate_parameters(self, parameter_id, **kwargs): + """ + Generate parameters, if no trial configration for now, self.credit plus 1 to send the config later + + Parameters + ---------- + parameter_id : int + Unique identifier for requested hyper-parameters. + This will later be used in :meth:`receive_trial_result`. + **kwargs + Not used + + Returns + ------- + dict + One newly generated configuration + + """ + if self.pos == self.population_size - 1: + logger.debug('Credit added by one in parameters request') + self.credit += 1 + self.param_ids.append(parameter_id) + raise nni.NoMoreTrialError('No more parameters now.') + self.pos += 1 + trial_info = self.population[self.pos] + trial_info.parameter_id = parameter_id + self.running[parameter_id] = trial_info + logger.info('Generate parameter : %s', trial_info.hyper_parameters) + return trial_info.hyper_parameters + + def _proceed_next_epoch(self): + """ + """ + logger.info('Proceeding to next epoch') + self.epoch += 1 + self.population = [] + self.pos = -1 + self.running = {} + #exploit and explore + reverse = True if self.optimize_mode == OptimizeMode.Maximize else False + self.finished = sorted(self.finished, key=lambda x: x.score, reverse=reverse) + cutoff = int(np.ceil(self.fraction * len(self.finished))) + tops = self.finished[:cutoff] + bottoms = self.finished[self.finished_trials - cutoff:] + for bottom in bottoms: + top = np.random.choice(tops) + exploit_and_explore(bottom, top, self.factor, self.resample_probability, self.epoch, self.searchspace_json) + for trial in self.finished: + if trial not in bottoms: + trial.clean_id() + trial.hyper_parameters['load_checkpoint_dir'] = trial.hyper_parameters['save_checkpoint_dir'] + trial.hyper_parameters['save_checkpoint_dir'] = os.path.join(trial.checkpoint_dir, str(self.epoch)) + self.finished_trials = 0 + for _ in range(self.population_size): + trial_info = self.finished.pop() + self.population.append(trial_info) + while self.credit > 0 and self.pos + 1 < len(self.population): + self.credit -= 1 + self.pos += 1 + parameter_id = self.param_ids.pop() + trial_info = self.population[self.pos] + trial_info.parameter_id = parameter_id + self.running[parameter_id] = trial_info + self.send_trial_callback(parameter_id, trial_info.hyper_parameters) + + def receive_trial_result(self, parameter_id, parameters, value, **kwargs): + """ + Receive trial's result. if the number of finished trials equals ``self.population_size``, start the next epoch to + train the model. + + Parameters + ---------- + parameter_id : int + Unique identifier of used hyper-parameters, same with :meth:`generate_parameters`. + parameters : dict + Hyper-parameters generated by :meth:`generate_parameters`. + value : dict + Result from trial (the return value of :func:`nni.report_final_result`). + """ + logger.info('Get one trial result, id = %d, value = %s', parameter_id, value) + value = extract_scalar_reward(value) + trial_info = self.running.pop(parameter_id, None) + trial_info.score = value + self.finished.append(trial_info) + self.finished_trials += 1 + if self.finished_trials == self.population_size: + self._proceed_next_epoch() + + def trial_end(self, parameter_id, success, **kwargs): + """ + Deal with trial failure + + Parameters + ---------- + parameter_id : int + Unique identifier for hyper-parameters used by this trial. + success : bool + True if the trial successfully completed; False if failed or terminated. + **kwargs + Unstable parameters which should be ignored by normal users. + """ + if success: + return + if self.optimize_mode == OptimizeMode.Minimize: + value = float('inf') + else: + value = float('-inf') + trial_info = self.running.pop(parameter_id, None) + trial_info.score = value + self.finished.append(trial_info) + self.finished_trials += 1 + if self.finished_trials == self.population_size: + self._proceed_next_epoch() + + def import_data(self, data): + """ + Parameters + ---------- + data : json obj + imported data records + + Returns + ------- + int + the start epoch number after data imported, only used for unittest + """ + if self.running: + logger.warning("Do not support importing data in the middle of experiment") + return + # the following is for experiment resume + _completed_num = 0 + epoch_data_dict = {} + for trial_info in data: + logger.info("Process data record %s / %s", _completed_num, len(data)) + _completed_num += 1 + # simply validate data format + _params = trial_info["parameter"] + _value = trial_info['value'] + # assign fake value for failed trials + if not _value: + logger.info("Useless trial data, value is %s, skip this trial data.", _value) + _value = float('inf') if self.optimize_mode == OptimizeMode.Minimize else float('-inf') + _value = extract_scalar_reward(_value) + if 'save_checkpoint_dir' not in _params: + logger.warning("Invalid data record: save_checkpoint_dir is missing, abandon data import.") + return + epoch_num = int(os.path.basename(_params['save_checkpoint_dir'])) + if epoch_num not in epoch_data_dict: + epoch_data_dict[epoch_num] = [] + epoch_data_dict[epoch_num].append((_params, _value)) + if not epoch_data_dict: + logger.warning("No valid epochs, abandon data import.") + return + # figure out start epoch for resume + max_epoch_num = max(epoch_data_dict, key=int) + if len(epoch_data_dict[max_epoch_num]) < self.population_size: + max_epoch_num -= 1 + # If there is no a single complete round, no data to import, start from scratch + if max_epoch_num < 0: + logger.warning("No completed epoch, abandon data import.") + return + assert len(epoch_data_dict[max_epoch_num]) == self.population_size + # check existence of trial save checkpoint dir + for params, _ in epoch_data_dict[max_epoch_num]: + if not os.path.isdir(params['save_checkpoint_dir']): + logger.warning("save_checkpoint_dir %s does not exist, data will not be resumed", params['save_checkpoint_dir']) + return + # resume data + self.epoch = max_epoch_num + self.finished_trials = self.population_size + for params, value in epoch_data_dict[max_epoch_num]: + checkpoint_dir = os.path.dirname(params['save_checkpoint_dir']) + self.finished.append(TrialInfo(checkpoint_dir=checkpoint_dir, hyper_parameters=params, score=value)) + self._proceed_next_epoch() + logger.info("Successfully import data to PBT tuner, total data: %d, imported data: %d.", len(data), self.population_size) + logger.info("Start from epoch %d ...", self.epoch) + return self.epoch # return for test diff --git a/utils/third_party/nni/algorithms/hpo/ppo_tuner/__init__.py b/utils/third_party/nni/algorithms/hpo/ppo_tuner/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..ada7e57c23d21f92063544430c6b0d41b05a6680 --- /dev/null +++ b/utils/third_party/nni/algorithms/hpo/ppo_tuner/__init__.py @@ -0,0 +1 @@ +from .ppo_tuner import PPOTuner diff --git a/utils/third_party/nni/algorithms/hpo/ppo_tuner/distri.py b/utils/third_party/nni/algorithms/hpo/ppo_tuner/distri.py new file mode 100644 index 0000000000000000000000000000000000000000..8a2a5ed20c3db6086c74339428fd25e0cf806f57 --- /dev/null +++ b/utils/third_party/nni/algorithms/hpo/ppo_tuner/distri.py @@ -0,0 +1,183 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +functions for sampling from hidden state +""" + +import tensorflow.compat.v1 as tf +tf.disable_v2_behavior() + +from .util import fc + + +class Pd: + """ + A particular probability distribution + """ + def flatparam(self): + raise NotImplementedError + def mode(self): + raise NotImplementedError + def neglogp(self, x): + # Usually it's easier to define the negative logprob + raise NotImplementedError + def kl(self, other): + raise NotImplementedError + def entropy(self): + raise NotImplementedError + def sample(self): + raise NotImplementedError + def logp(self, x): + return - self.neglogp(x) + def get_shape(self): + return self.flatparam().shape + @property + def shape(self): + return self.get_shape() + def __getitem__(self, idx): + return self.__class__(self.flatparam()[idx]) + +class PdType: + """ + Parametrized family of probability distributions + """ + def pdclass(self): + raise NotImplementedError + def pdfromflat(self, flat, mask, nsteps, size, is_act_model): + return self.pdclass()(flat, mask, nsteps, size, is_act_model) + def pdfromlatent(self, latent_vector, init_scale, init_bias): + raise NotImplementedError + def param_shape(self): + raise NotImplementedError + def sample_shape(self): + raise NotImplementedError + def sample_dtype(self): + raise NotImplementedError + + def param_placeholder(self, prepend_shape, name=None): + return tf.placeholder(dtype=tf.float32, shape=prepend_shape+self.param_shape(), name=name) + def sample_placeholder(self, prepend_shape, name=None): + return tf.placeholder(dtype=self.sample_dtype(), shape=prepend_shape+self.sample_shape(), name=name) + +class CategoricalPd(Pd): + """ + Categorical probability distribution + """ + def __init__(self, logits, mask_npinf, nsteps, size, is_act_model): + self.logits = logits + self.mask_npinf = mask_npinf + self.nsteps = nsteps + self.size = size + self.is_act_model = is_act_model + def flatparam(self): + return self.logits + def mode(self): + return tf.argmax(self.logits, axis=-1) + + @property + def mean(self): + return tf.nn.softmax(self.logits) + def neglogp(self, x): + """ + return tf.nn.sparse_softmax_cross_entropy_with_logits(logits=self.logits, labels=x) + Note: we can't use sparse_softmax_cross_entropy_with_logits because + the implementation does not allow second-order derivatives... + """ + if x.dtype in {tf.uint8, tf.int32, tf.int64}: + # one-hot encoding + x_shape_list = x.shape.as_list() + logits_shape_list = self.logits.get_shape().as_list()[:-1] + for xs, ls in zip(x_shape_list, logits_shape_list): + if xs is not None and ls is not None: + assert xs == ls, 'shape mismatch: {} in x vs {} in logits'.format(xs, ls) + + x = tf.one_hot(x, self.logits.get_shape().as_list()[-1]) + else: + # already encoded + assert x.shape.as_list() == self.logits.shape.as_list() + + return tf.nn.softmax_cross_entropy_with_logits_v2( + logits=self.logits, + labels=x) + + def kl(self, other): + """kl""" + a0 = self.logits - tf.reduce_max(self.logits, axis=-1, keepdims=True) + a1 = other.logits - tf.reduce_max(other.logits, axis=-1, keepdims=True) + ea0 = tf.exp(a0) + ea1 = tf.exp(a1) + z0 = tf.reduce_sum(ea0, axis=-1, keepdims=True) + z1 = tf.reduce_sum(ea1, axis=-1, keepdims=True) + p0 = ea0 / z0 + return tf.reduce_sum(p0 * (a0 - tf.log(z0) - a1 + tf.log(z1)), axis=-1) + + def entropy(self): + """compute entropy""" + a0 = self.logits - tf.reduce_max(self.logits, axis=-1, keepdims=True) + ea0 = tf.exp(a0) + z0 = tf.reduce_sum(ea0, axis=-1, keepdims=True) + p0 = ea0 / z0 + return tf.reduce_sum(p0 * (tf.log(z0) - a0), axis=-1) + + def sample(self): + """sample from logits""" + if not self.is_act_model: + re_res = tf.reshape(self.logits, [-1, self.nsteps, self.size]) + masked_res = tf.math.add(re_res, self.mask_npinf) + re_masked_res = tf.reshape(masked_res, [-1, self.size]) + + u = tf.random_uniform(tf.shape(re_masked_res), dtype=self.logits.dtype) + return tf.argmax(re_masked_res - tf.log(-1*tf.log(u)), axis=-1) + else: + u = tf.random_uniform(tf.shape(self.logits), dtype=self.logits.dtype) + return tf.argmax(self.logits - tf.log(-1*tf.log(u)), axis=-1) + + @classmethod + def fromflat(cls, flat): + return cls(flat) # pylint: disable=no-value-for-parameter + +class CategoricalPdType(PdType): + """ + To create CategoricalPd + """ + def __init__(self, ncat, nsteps, np_mask, is_act_model): + self.ncat = ncat + self.nsteps = nsteps + self.np_mask = np_mask + self.is_act_model = is_act_model + def pdclass(self): + return CategoricalPd + + def pdfromlatent(self, latent_vector, init_scale=1.0, init_bias=0.0): + """add fc and create CategoricalPd""" + pdparam, mask, mask_npinf = _matching_fc(latent_vector, 'pi', self.ncat, self.nsteps, + init_scale=init_scale, init_bias=init_bias, + np_mask=self.np_mask, is_act_model=self.is_act_model) + return self.pdfromflat(pdparam, mask_npinf, self.nsteps, self.ncat, self.is_act_model), pdparam, mask, mask_npinf + + def param_shape(self): + return [self.ncat] + def sample_shape(self): + return [] + def sample_dtype(self): + return tf.int32 + +def _matching_fc(tensor, name, size, nsteps, init_scale, init_bias, np_mask, is_act_model): + """ + Add fc op, and add mask op when not in action mode + """ + if tensor.shape[-1] == size: + assert False + return tensor + else: + mask = tf.get_variable("act_mask", dtype=tf.float32, initializer=np_mask[0], trainable=False) + mask_npinf = tf.get_variable("act_mask_npinf", dtype=tf.float32, initializer=np_mask[1], trainable=False) + res = fc(tensor, name, size, init_scale=init_scale, init_bias=init_bias) + if not is_act_model: + re_res = tf.reshape(res, [-1, nsteps, size]) + masked_res = tf.math.multiply(re_res, mask) + re_masked_res = tf.reshape(masked_res, [-1, size]) + return re_masked_res, mask, mask_npinf + else: + return res, mask, mask_npinf diff --git a/utils/third_party/nni/algorithms/hpo/ppo_tuner/model.py b/utils/third_party/nni/algorithms/hpo/ppo_tuner/model.py new file mode 100644 index 0000000000000000000000000000000000000000..c6a8479c6d0d06fcf0f7c4b8a64a62ab968988aa --- /dev/null +++ b/utils/third_party/nni/algorithms/hpo/ppo_tuner/model.py @@ -0,0 +1,152 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +the main model of policy/value network +""" + +import tensorflow.compat.v1 as tf +tf.disable_v2_behavior() + +from .util import initialize, get_session + +class Model: + """ + We use this object to : + __init__: + - Creates the step_model + - Creates the train_model + + train(): + - Make the training part (feedforward and retropropagation of gradients) + + save/load(): + - Save load the model + """ + def __init__(self, *, policy, nbatch_act, nbatch_train, + nsteps, ent_coef, vf_coef, max_grad_norm, microbatch_size=None, np_mask=None): + self.sess = sess = get_session() + + with tf.variable_scope('ppo2_model', reuse=tf.AUTO_REUSE): + # CREATE OUR TWO MODELS + # act_model that is used for sampling + act_model = policy(nbatch_act, 1, sess, np_mask=np_mask, is_act_model=True) + + # Train model for training + if microbatch_size is None: + train_model = policy(nbatch_train, nsteps, sess, np_mask=np_mask, is_act_model=False) + else: + train_model = policy(microbatch_size, nsteps, sess, np_mask=np_mask, is_act_model=False) + + # CREATE THE PLACEHOLDERS + self.A = A = train_model.pdtype.sample_placeholder([None]) + self.ADV = ADV = tf.placeholder(tf.float32, [None]) + self.R = R = tf.placeholder(tf.float32, [None]) + # Keep track of old actor + self.OLDNEGLOGPAC = OLDNEGLOGPAC = tf.placeholder(tf.float32, [None]) + # Keep track of old critic + self.OLDVPRED = OLDVPRED = tf.placeholder(tf.float32, [None]) + self.LR = LR = tf.placeholder(tf.float32, []) + # Cliprange + self.CLIPRANGE = CLIPRANGE = tf.placeholder(tf.float32, []) + + neglogpac = train_model.pd.neglogp(A) + + # Calculate the entropy + # Entropy is used to improve exploration by limiting the premature convergence to suboptimal policy. + entropy = tf.reduce_mean(train_model.pd.entropy()) + + # CALCULATE THE LOSS + # Total loss = Policy gradient loss - entropy * entropy coefficient + Value coefficient * value loss + + # Clip the value to reduce variability during Critic training + # Get the predicted value + vpred = train_model.vf + vpredclipped = OLDVPRED + tf.clip_by_value(train_model.vf - OLDVPRED, - CLIPRANGE, CLIPRANGE) + # Unclipped value + vf_losses1 = tf.square(vpred - R) + # Clipped value + vf_losses2 = tf.square(vpredclipped - R) + + vf_loss = .5 * tf.reduce_mean(tf.maximum(vf_losses1, vf_losses2)) + + # Calculate ratio (pi current policy / pi old policy) + ratio = tf.exp(OLDNEGLOGPAC - neglogpac) + + # Defining Loss = - J is equivalent to max J + pg_losses = -ADV * ratio + + pg_losses2 = -ADV * tf.clip_by_value(ratio, 1.0 - CLIPRANGE, 1.0 + CLIPRANGE) + + # Final PG loss + pg_loss = tf.reduce_mean(tf.maximum(pg_losses, pg_losses2)) + approxkl = .5 * tf.reduce_mean(tf.square(neglogpac - OLDNEGLOGPAC)) + clipfrac = tf.reduce_mean(tf.to_float(tf.greater(tf.abs(ratio - 1.0), CLIPRANGE))) + + # Total loss + loss = pg_loss - entropy * ent_coef + vf_loss * vf_coef + + # UPDATE THE PARAMETERS USING LOSS + # 1. Get the model parameters + params = tf.trainable_variables('ppo2_model') + # 2. Build our trainer + self.trainer = tf.train.AdamOptimizer(learning_rate=LR, epsilon=1e-5) + # 3. Calculate the gradients + grads_and_var = self.trainer.compute_gradients(loss, params) + grads, var = zip(*grads_and_var) + + if max_grad_norm is not None: + # Clip the gradients (normalize) + grads, _grad_norm = tf.clip_by_global_norm(grads, max_grad_norm) + grads_and_var = list(zip(grads, var)) + # zip aggregate each gradient with parameters associated + # For instance zip(ABCD, xyza) => Ax, By, Cz, Da + + self.grads = grads + self.var = var + self._train_op = self.trainer.apply_gradients(grads_and_var) + self.loss_names = ['policy_loss', 'value_loss', 'policy_entropy', 'approxkl', 'clipfrac'] + self.stats_list = [pg_loss, vf_loss, entropy, approxkl, clipfrac] + + + self.train_model = train_model + self.act_model = act_model + self.step = act_model.step + self.value = act_model.value + self.initial_state = act_model.initial_state + + initialize() + + def train(self, lr, cliprange, obs, returns, masks, actions, values, neglogpacs, states=None): + """ + Train the model. + Here we calculate advantage A(s,a) = R + yV(s') - V(s) + + Returns + ------- + obj + = R + yV(s') + """ + advs = returns - values + + # Normalize the advantages + advs = (advs - advs.mean()) / (advs.std() + 1e-8) + + td_map = { + self.train_model.X : obs, + self.A : actions, + self.ADV : advs, + self.R : returns, + self.LR : lr, + self.CLIPRANGE : cliprange, + self.OLDNEGLOGPAC : neglogpacs, + self.OLDVPRED : values + } + if states is not None: + td_map[self.train_model.S] = states + td_map[self.train_model.M] = masks + + return self.sess.run( + self.stats_list + [self._train_op], + td_map + )[:-1] diff --git a/utils/third_party/nni/algorithms/hpo/ppo_tuner/policy.py b/utils/third_party/nni/algorithms/hpo/ppo_tuner/policy.py new file mode 100644 index 0000000000000000000000000000000000000000..a35e514eaef36562c26c6a99e8224b4339ae768c --- /dev/null +++ b/utils/third_party/nni/algorithms/hpo/ppo_tuner/policy.py @@ -0,0 +1,230 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +build policy/value network from model +""" + +import tensorflow.compat.v1 as tf +tf.disable_v2_behavior() + +from .distri import CategoricalPdType +from .util import lstm_model, fc, observation_placeholder, adjust_shape + + +class PolicyWithValue: + """ + Encapsulates fields and methods for RL policy and value function estimation with shared parameters + """ + + def __init__(self, env, observations, latent, estimate_q=False, vf_latent=None, sess=None, np_mask=None, is_act_model=False, **tensors): + """ + Parameters + ---------- + env : obj + RL environment + observations : tensorflow placeholder + Tensorflow placeholder in which the observations will be fed + latent : tensor + Latent state from which policy distribution parameters should be inferred + vf_latent : tensor + Latent state from which value function should be inferred (if None, then latent is used) + sess : tensorflow session + Tensorflow session to run calculations in (if None, default session is used) + **tensors + Tensorflow tensors for additional attributes such as state or mask + """ + + self.X = observations + self.state = tf.constant([]) + self.initial_state = None + self.__dict__.update(tensors) + + vf_latent = vf_latent if vf_latent is not None else latent + + vf_latent = tf.layers.flatten(vf_latent) + latent = tf.layers.flatten(latent) + + # Based on the action space, will select what probability distribution type + self.np_mask = np_mask + self.pdtype = CategoricalPdType(env.action_space.n, env.nsteps, np_mask, is_act_model) + + self.act_latent = latent + self.nh = env.action_space.n + + self.pd, self.pi, self.mask, self.mask_npinf = self.pdtype.pdfromlatent(latent, init_scale=0.01) + + # Take an action + self.action = self.pd.sample() + + # Calculate the neg log of our probability + self.neglogp = self.pd.neglogp(self.action) + self.sess = sess or tf.get_default_session() + + assert estimate_q is False + self.vf = fc(vf_latent, 'vf', 1) + self.vf = self.vf[:, 0] + + if is_act_model: + self._build_model_for_step() + + def _evaluate(self, variables, observation, **extra_feed): + sess = self.sess + feed_dict = {self.X: adjust_shape(self.X, observation)} + for inpt_name, data in extra_feed.items(): + if inpt_name in self.__dict__.keys(): + inpt = self.__dict__[inpt_name] + if isinstance(inpt, tf.Tensor) and inpt._op.type == 'Placeholder': + feed_dict[inpt] = adjust_shape(inpt, data) + + return sess.run(variables, feed_dict) + + def _build_model_for_step(self): + # multiply with weight and apply mask on self.act_latent to generate + self.act_step = step = tf.placeholder(shape=(), dtype=tf.int64, name='act_step') + with tf.variable_scope('pi', reuse=tf.AUTO_REUSE): + from .util import ortho_init + nin = self.act_latent.get_shape()[1].value + w = tf.get_variable("w", [nin, self.nh], initializer=ortho_init(0.01)) + b = tf.get_variable("b", [self.nh], initializer=tf.constant_initializer(0.0)) + logits = tf.matmul(self.act_latent, w)+b + piece = tf.slice(self.mask, [step, 0], [1, self.nh]) + re_piece = tf.reshape(piece, [-1]) + masked_logits = tf.math.multiply(logits, re_piece) + + npinf_piece = tf.slice(self.mask_npinf, [step, 0], [1, self.nh]) + re_npinf_piece = tf.reshape(npinf_piece, [-1]) + + def sample(logits, mask_npinf): + new_logits = tf.math.add(logits, mask_npinf) + u = tf.random_uniform(tf.shape(new_logits), dtype=logits.dtype) + return tf.argmax(new_logits - tf.log(-1*tf.log(u)), axis=-1) + + def neglogp(logits, x): + # return tf.nn.sparse_softmax_cross_entropy_with_logits(logits=self.logits, labels=x) + # Note: we can't use sparse_softmax_cross_entropy_with_logits because + # the implementation does not allow second-order derivatives... + if x.dtype in {tf.uint8, tf.int32, tf.int64}: + # one-hot encoding + x_shape_list = x.shape.as_list() + logits_shape_list = logits.get_shape().as_list()[:-1] + for xs, ls in zip(x_shape_list, logits_shape_list): + if xs is not None and ls is not None: + assert xs == ls, 'shape mismatch: {} in x vs {} in logits'.format(xs, ls) + + x = tf.one_hot(x, logits.get_shape().as_list()[-1]) + else: + # already encoded + assert x.shape.as_list() == logits.shape.as_list() + + return tf.nn.softmax_cross_entropy_with_logits_v2( + logits=logits, + labels=x) + + self.act_action = sample(masked_logits, re_npinf_piece) + self.act_neglogp = neglogp(masked_logits, self.act_action) + + + def step(self, step, observation, **extra_feed): + """ + Compute next action(s) given the observation(s) + + Parameters + ---------- + observation : np array + Observation data (either single or a batch) + **extra_feed + Additional data such as state or mask (names of the arguments should match the ones in constructor, see __init__) + + Returns + ------- + (action, value estimate, next state, negative log likelihood of the action under current policy parameters) tuple + """ + extra_feed['act_step'] = step + a, v, state, neglogp = self._evaluate([self.act_action, self.vf, self.state, self.act_neglogp], observation, **extra_feed) + if state.size == 0: + state = None + return a, v, state, neglogp + + def value(self, ob, *args, **kwargs): + """ + Compute value estimate(s) given the observation(s) + + Parameters + ---------- + observation : np array + Observation data (either single or a batch) + **extra_feed + Additional data such as state or mask (names of the arguments should match the ones in constructor, see __init__) + + Returns + ------- + Value estimate + """ + return self._evaluate(self.vf, ob, *args, **kwargs) + + +def build_lstm_policy(model_config, value_network=None, estimate_q=False, **policy_kwargs): + """ + Build lstm policy and value network, they share the same lstm network. + the parameters all use their default values. + + Parameter + --------- + model_config : obj + Configurations of the model + value_network : obj + The network for value function + estimate_q : bool + Whether to estimate ``q`` + **policy_kwargs + The kwargs for policy network, i.e., lstm model + + Returns + ------- + func + The policy network + """ + policy_network = lstm_model(**policy_kwargs) + + def policy_fn(nbatch=None, nsteps=None, sess=None, observ_placeholder=None, np_mask=None, is_act_model=False): + ob_space = model_config.observation_space + + X = observ_placeholder if observ_placeholder is not None else observation_placeholder(ob_space, batch_size=nbatch) + + extra_tensors = {} + + # encode_observation is not necessary anymore as we use embedding_lookup + encoded_x = X + + with tf.variable_scope('pi', reuse=tf.AUTO_REUSE): + policy_latent = policy_network(encoded_x, 1, model_config.observation_space.n) + if isinstance(policy_latent, tuple): + policy_latent, recurrent_tensors = policy_latent + + if recurrent_tensors is not None: + # recurrent architecture, need a few more steps + nenv = nbatch // nsteps + assert nenv > 0, 'Bad input for recurrent policy: batch size {} smaller than nsteps {}'.format(nbatch, nsteps) + policy_latent, recurrent_tensors = policy_network(encoded_x, nenv, model_config.observation_space.n) + extra_tensors.update(recurrent_tensors) + + _v_net = value_network + + assert _v_net is None or _v_net == 'shared' + vf_latent = policy_latent + + policy = PolicyWithValue( + env=model_config, + observations=X, + latent=policy_latent, + vf_latent=vf_latent, + sess=sess, + estimate_q=estimate_q, + np_mask=np_mask, + is_act_model=is_act_model, + **extra_tensors + ) + return policy + + return policy_fn diff --git a/utils/third_party/nni/algorithms/hpo/ppo_tuner/ppo_tuner.py b/utils/third_party/nni/algorithms/hpo/ppo_tuner/ppo_tuner.py new file mode 100644 index 0000000000000000000000000000000000000000..33b62d600e3b6bb788edab7db53a1223a846b2eb --- /dev/null +++ b/utils/third_party/nni/algorithms/hpo/ppo_tuner/ppo_tuner.py @@ -0,0 +1,654 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +ppo_tuner.py including: + class PPOTuner +""" + +import copy +import logging +import numpy as np +from gym import spaces +from schema import Schema, Optional + +import nni +from nni import ClassArgsValidator +from nni.tuner import Tuner +from nni.utils import OptimizeMode, extract_scalar_reward + +from .model import Model +from .util import set_global_seeds +from .policy import build_lstm_policy + + +logger = logging.getLogger('ppo_tuner_AutoML') + +def _constfn(val): + """ + Wrap as function + """ + def f(_): + return val + return f + + +class ModelConfig: + """ + Configurations of the PPO model + """ + def __init__(self): + self.observation_space = None + self.action_space = None + self.num_envs = 0 + self.nsteps = 0 + + self.ent_coef = 0.0 + self.lr = 3e-4 + self.vf_coef = 0.5 + self.max_grad_norm = 0.5 + self.gamma = 0.99 + self.lam = 0.95 + self.cliprange = 0.2 + self.embedding_size = None # the embedding is for each action + + self.noptepochs = 4 # number of training epochs per update + self.total_timesteps = 5000 # number of timesteps (i.e. number of actions taken in the environment) + self.nminibatches = 4 # number of training minibatches per update. For recurrent policies, + # should be smaller or equal than number of environments run in parallel. + +class TrialsInfo: + """ + Informations of each trial from one model inference + """ + def __init__(self, obs, actions, values, neglogpacs, dones, last_value, inf_batch_size): + self.iter = 0 + self.obs = obs + self.actions = actions + self.values = values + self.neglogpacs = neglogpacs + self.dones = dones + self.last_value = last_value + + self.rewards = None + self.returns = None + + self.inf_batch_size = inf_batch_size + #self.states = None + + def get_next(self): + """ + Get actions of the next trial + """ + if self.iter >= self.inf_batch_size: + return None, None + actions = [] + for step in self.actions: + actions.append(step[self.iter]) + self.iter += 1 + return self.iter - 1, actions + + def update_rewards(self, rewards, returns): + """ + After the trial is finished, reward and return of this trial is updated + """ + self.rewards = rewards + self.returns = returns + + def convert_shape(self): + """ + Convert shape + """ + def sf01(arr): + """ + swap and then flatten axes 0 and 1 + """ + s = arr.shape + return arr.swapaxes(0, 1).reshape(s[0] * s[1], *s[2:]) + self.obs = sf01(self.obs) + self.returns = sf01(self.returns) + self.dones = sf01(self.dones) + self.actions = sf01(self.actions) + self.values = sf01(self.values) + self.neglogpacs = sf01(self.neglogpacs) + + +class PPOModel: + """ + PPO Model + """ + def __init__(self, model_config, mask): + self.model_config = model_config + self.states = None # initial state of lstm in policy/value network + self.nupdates = None # the number of func train is invoked, used to tune lr and cliprange + self.cur_update = 1 # record the current update + self.np_mask = mask # record the mask of each action within one trial + + set_global_seeds(None) + assert isinstance(self.model_config.lr, float) + self.lr = _constfn(self.model_config.lr) + assert isinstance(self.model_config.cliprange, float) + self.cliprange = _constfn(self.model_config.cliprange) + + # build lstm policy network, value share the same network + policy = build_lstm_policy(model_config) + + # Get the nb of env + nenvs = model_config.num_envs + + # Calculate the batch_size + self.nbatch = nbatch = nenvs * model_config.nsteps # num of record per update + nbatch_train = nbatch // model_config.nminibatches # get batch size + # self.nupdates is used to tune lr and cliprange + self.nupdates = self.model_config.total_timesteps // self.nbatch + + # Instantiate the model object (that creates act_model and train_model) + self.model = Model(policy=policy, nbatch_act=nenvs, nbatch_train=nbatch_train, + nsteps=model_config.nsteps, ent_coef=model_config.ent_coef, vf_coef=model_config.vf_coef, + max_grad_norm=model_config.max_grad_norm, np_mask=self.np_mask) + + self.states = self.model.initial_state + + logger.info('=== finished PPOModel initialization') + + def inference(self, num): + """ + Generate actions along with related info from policy network. + observation is the action of the last step. + + Parameters + ---------- + num: int + The number of trials to generate + + Returns + ------- + mb_obs : list + Observation of the ``num`` configurations + mb_actions : list + Actions of the ``num`` configurations + mb_values : list + Values from the value function of the ``num`` configurations + mb_neglogpacs : list + ``neglogp`` of the ``num`` configurations + mb_dones : list + To show whether the play is done, always ``True`` + last_values : tensorflow tensor + The last values of the ``num`` configurations, got with session run + """ + # Here, we init the lists that will contain the mb of experiences + mb_obs, mb_actions, mb_values, mb_dones, mb_neglogpacs = [], [], [], [], [] + # initial observation + # use the (n+1)th embedding to represent the first step action + first_step_ob = self.model_config.action_space.n + obs = [first_step_ob for _ in range(num)] + dones = [True for _ in range(num)] + states = self.states + # For n in range number of steps + for cur_step in range(self.model_config.nsteps): + # Given observations, get action value and neglopacs + # We already have self.obs because Runner superclass run self.obs[:] = env.reset() on init + actions, values, states, neglogpacs = self.model.step(cur_step, obs, S=states, M=dones) + mb_obs.append(obs.copy()) + mb_actions.append(actions) + mb_values.append(values) + mb_neglogpacs.append(neglogpacs) + mb_dones.append(dones) + + # Take actions in env and look the results + # Infos contains a ton of useful informations + obs[:] = actions + if cur_step == self.model_config.nsteps - 1: + dones = [True for _ in range(num)] + else: + dones = [False for _ in range(num)] + + #batch of steps to batch of rollouts + np_obs = np.asarray(obs) + mb_obs = np.asarray(mb_obs, dtype=np_obs.dtype) + mb_actions = np.asarray(mb_actions) + mb_values = np.asarray(mb_values, dtype=np.float32) + mb_neglogpacs = np.asarray(mb_neglogpacs, dtype=np.float32) + mb_dones = np.asarray(mb_dones, dtype=np.bool) + last_values = self.model.value(np_obs, S=states, M=dones) + + return mb_obs, mb_actions, mb_values, mb_neglogpacs, mb_dones, last_values + + def compute_rewards(self, trials_info, trials_result): + """ + Compute the rewards of the trials in trials_info based on trials_result, + and update the rewards in trials_info + + Parameters + ---------- + trials_info : TrialsInfo + Info of the generated trials + trials_result : list + Final results (e.g., acc) of the generated trials + """ + mb_rewards = np.asarray([trials_result for _ in trials_info.actions], dtype=np.float32) + # discount/bootstrap off value fn + mb_returns = np.zeros_like(mb_rewards) + mb_advs = np.zeros_like(mb_rewards) + lastgaelam = 0 + last_dones = np.asarray([True for _ in trials_result], dtype=np.bool) # ugly + for t in reversed(range(self.model_config.nsteps)): + if t == self.model_config.nsteps - 1: + nextnonterminal = 1.0 - last_dones + nextvalues = trials_info.last_value + else: + nextnonterminal = 1.0 - trials_info.dones[t+1] + nextvalues = trials_info.values[t+1] + delta = mb_rewards[t] + self.model_config.gamma * nextvalues * nextnonterminal - trials_info.values[t] + lastgaelam = delta + self.model_config.gamma * self.model_config.lam * nextnonterminal * lastgaelam + mb_advs[t] = lastgaelam # pylint: disable=unsupported-assignment-operation + mb_returns = mb_advs + trials_info.values + + trials_info.update_rewards(mb_rewards, mb_returns) + trials_info.convert_shape() + + def train(self, trials_info, nenvs): + """ + Train the policy/value network using trials_info + + Parameters + ---------- + trials_info : TrialsInfo + Complete info of the generated trials from the previous inference + nenvs : int + The batch size of the (previous) inference + """ + # keep frac decay for future optimization + if self.cur_update <= self.nupdates: + frac = 1.0 - (self.cur_update - 1.0) / self.nupdates + else: + logger.warning('current update (self.cur_update) %d has exceeded total updates (self.nupdates) %d', + self.cur_update, self.nupdates) + frac = 1.0 - (self.nupdates - 1.0) / self.nupdates + lrnow = self.lr(frac) + cliprangenow = self.cliprange(frac) + self.cur_update += 1 + + states = self.states + + assert states is not None # recurrent version + assert nenvs % self.model_config.nminibatches == 0 + envsperbatch = nenvs // self.model_config.nminibatches + envinds = np.arange(nenvs) + flatinds = np.arange(nenvs * self.model_config.nsteps).reshape(nenvs, self.model_config.nsteps) + for _ in range(self.model_config.noptepochs): + np.random.shuffle(envinds) + for start in range(0, nenvs, envsperbatch): + end = start + envsperbatch + mbenvinds = envinds[start:end] + mbflatinds = flatinds[mbenvinds].ravel() + slices = (arr[mbflatinds] for arr in (trials_info.obs, trials_info.returns, trials_info.dones, + trials_info.actions, trials_info.values, trials_info.neglogpacs)) + mbstates = states[mbenvinds] + self.model.train(lrnow, cliprangenow, *slices, mbstates) + +class PPOClassArgsValidator(ClassArgsValidator): + def validate_class_args(self, **kwargs): + Schema({ + 'optimize_mode': self.choices('optimize_mode', 'maximize', 'minimize'), + Optional('trials_per_update'): self.range('trials_per_update', int, 0, 99999), + Optional('epochs_per_update'): self.range('epochs_per_update', int, 0, 99999), + Optional('minibatch_size'): self.range('minibatch_size', int, 0, 99999), + Optional('ent_coef'): float, + Optional('lr'): float, + Optional('vf_coef'): float, + Optional('max_grad_norm'): float, + Optional('gamma'): float, + Optional('lam'): float, + Optional('cliprange'): float, + }).validate(kwargs) + +class PPOTuner(Tuner): + """ + PPOTuner, the implementation inherits the main logic of the implementation + [ppo2 from openai](https://github.com/openai/baselines/tree/master/baselines/ppo2), and is adapted for NAS scenario. + It uses ``lstm`` for its policy network and value network, policy and value share the same network. + """ + + def __init__(self, optimize_mode, trials_per_update=20, epochs_per_update=4, minibatch_size=4, + ent_coef=0.0, lr=3e-4, vf_coef=0.5, max_grad_norm=0.5, gamma=0.99, lam=0.95, cliprange=0.2): + """ + Initialization, PPO model is not initialized here as search space is not received yet. + + Parameters + ---------- + optimize_mode : str + maximize or minimize + trials_per_update : int + Number of trials to have for each model update + epochs_per_update : int + Number of epochs to run for each model update + minibatch_size : int + Minibatch size (number of trials) for the update + ent_coef : float + Policy entropy coefficient in the optimization objective + lr : float + Learning rate of the model (lstm network), constant + vf_coef : float + Value function loss coefficient in the optimization objective + max_grad_norm : float + Gradient norm clipping coefficient + gamma : float + Discounting factor + lam : float + Advantage estimation discounting factor (lambda in the paper) + cliprange : float + Cliprange in the PPO algorithm, constant + """ + self.optimize_mode = OptimizeMode(optimize_mode) + self.model_config = ModelConfig() + self.model = None + self.search_space = None + self.running_trials = {} # key: parameter_id, value: actions/states/etc. + self.inf_batch_size = trials_per_update # number of trials to generate in one inference + self.first_inf = True # indicate whether it is the first time to inference new trials + self.trials_result = [None for _ in range(self.inf_batch_size)] # results of finished trials + + self.credit = 0 # record the unsatisfied trial requests + self.param_ids = [] + self.finished_trials = 0 + self.chosen_arch_template = {} + + self.actions_spaces = None + self.actions_to_config = None + self.full_act_space = None + self.trials_info = None + + self.all_trials = {} # used to dedup the same trial, key: config, value: final result + + self.model_config.num_envs = self.inf_batch_size + self.model_config.noptepochs = epochs_per_update + self.model_config.nminibatches = minibatch_size + + self.send_trial_callback = None + logger.info('Finished PPOTuner initialization') + + def _process_nas_space(self, search_space): + actions_spaces = [] + actions_to_config = [] + for key, val in search_space.items(): + if val['_type'] == 'layer_choice': + actions_to_config.append((key, 'layer_choice')) + actions_spaces.append(val['_value']) + self.chosen_arch_template[key] = None + elif val['_type'] == 'input_choice': + candidates = val['_value']['candidates'] + n_chosen = val['_value']['n_chosen'] + if n_chosen not in [0, 1, [0, 1]]: + raise ValueError('Optional_input_size can only be 0, 1, or [0, 1], but the pecified one is %s' + % (n_chosen)) + if isinstance(n_chosen, list): + actions_to_config.append((key, 'input_choice')) + # FIXME: risk, candidates might also have None + actions_spaces.append(['None', *candidates]) + self.chosen_arch_template[key] = None + elif n_chosen == 1: + actions_to_config.append((key, 'input_choice')) + actions_spaces.append(candidates) + self.chosen_arch_template[key] = None + elif n_chosen == 0: + self.chosen_arch_template[key] = [] + else: + raise ValueError('Unsupported search space type: %s' % (val['_type'])) + + # calculate observation space + dedup = {} + for step in actions_spaces: + for action in step: + dedup[action] = 1 + full_act_space = [act for act, _ in dedup.items()] + assert len(full_act_space) == len(dedup) + observation_space = len(full_act_space) + nsteps = len(actions_spaces) + + return actions_spaces, actions_to_config, full_act_space, observation_space, nsteps + + def _generate_action_mask(self): + """ + Different step could have different action space. to deal with this case, we merge all the + possible actions into one action space, and use mask to indicate available actions for each step + """ + two_masks = [] + + mask = [] + for acts in self.actions_spaces: + one_mask = [0 for _ in range(len(self.full_act_space))] + for act in acts: + idx = self.full_act_space.index(act) + one_mask[idx] = 1 + mask.append(one_mask) + two_masks.append(mask) + + mask = [] + for acts in self.actions_spaces: + one_mask = [-np.inf for _ in range(len(self.full_act_space))] + for act in acts: + idx = self.full_act_space.index(act) + one_mask[idx] = 0 + mask.append(one_mask) + two_masks.append(mask) + + return np.asarray(two_masks, dtype=np.float32) + + def update_search_space(self, search_space): + """ + Get search space, currently the space only includes that for NAS + + Parameters + ---------- + search_space : dict + Search space for NAS + the format could be referred to search space spec (https://nni.readthedocs.io/en/latest/Tutorial/SearchSpaceSpec.html). + """ + logger.info('update search space %s', search_space) + assert self.search_space is None + self.search_space = search_space + + assert self.model_config.observation_space is None + assert self.model_config.action_space is None + + self.actions_spaces, self.actions_to_config, self.full_act_space, obs_space, nsteps = self._process_nas_space(search_space) + + self.model_config.observation_space = spaces.Discrete(obs_space) + self.model_config.action_space = spaces.Discrete(obs_space) + self.model_config.nsteps = nsteps + + # generate mask in numpy + mask = self._generate_action_mask() + + assert self.model is None + self.model = PPOModel(self.model_config, mask) + + def _actions_to_config(self, actions): + """ + Given actions, to generate the corresponding trial configuration + """ + chosen_arch = copy.deepcopy(self.chosen_arch_template) + for cnt, act in enumerate(actions): + act_name = self.full_act_space[act] + (_key, _type) = self.actions_to_config[cnt] + if _type == 'input_choice': + if act_name == 'None': + chosen_arch[_key] = {'_value': [], '_idx': []} + else: + candidates = self.search_space[_key]['_value']['candidates'] + idx = candidates.index(act_name) + chosen_arch[_key] = {'_value': [act_name], '_idx': [idx]} + elif _type == 'layer_choice': + idx = self.search_space[_key]['_value'].index(act_name) + chosen_arch[_key] = {'_value': act_name, '_idx': idx} + else: + raise ValueError('unrecognized key: {0}'.format(_type)) + return chosen_arch + + def generate_multiple_parameters(self, parameter_id_list, **kwargs): + """ + Returns multiple sets of trial (hyper-)parameters, as iterable of serializable objects. + + Parameters + ---------- + parameter_id_list : list of int + Unique identifiers for each set of requested hyper-parameters. + These will later be used in :meth:`receive_trial_result`. + **kwargs + Not used + + Returns + ------- + list + A list of newly generated configurations + """ + result = [] + self.send_trial_callback = kwargs['st_callback'] + for parameter_id in parameter_id_list: + had_exception = False + try: + logger.debug("generating param for %s", parameter_id) + res = self.generate_parameters(parameter_id, **kwargs) + except nni.NoMoreTrialError: + had_exception = True + if not had_exception: + result.append(res) + return result + + def generate_parameters(self, parameter_id, **kwargs): + """ + Generate parameters, if no trial configration for now, self.credit plus 1 to send the config later + + Parameters + ---------- + parameter_id : int + Unique identifier for requested hyper-parameters. + This will later be used in :meth:`receive_trial_result`. + **kwargs + Not used + + Returns + ------- + dict + One newly generated configuration + + """ + if self.first_inf: + self.trials_result = [None for _ in range(self.inf_batch_size)] + mb_obs, mb_actions, mb_values, mb_neglogpacs, mb_dones, last_values = self.model.inference(self.inf_batch_size) + self.trials_info = TrialsInfo(mb_obs, mb_actions, mb_values, mb_neglogpacs, + mb_dones, last_values, self.inf_batch_size) + self.first_inf = False + + trial_info_idx, actions = self.trials_info.get_next() + if trial_info_idx is None: + logger.debug('Credit added by one in parameters request') + self.credit += 1 + self.param_ids.append(parameter_id) + raise nni.NoMoreTrialError('no more parameters now.') + + self.running_trials[parameter_id] = trial_info_idx + new_config = self._actions_to_config(actions) + return new_config + + def _next_round_inference(self): + """ + Run a inference to generate next batch of configurations + """ + logger.debug('Start next round inference...') + self.finished_trials = 0 + self.model.compute_rewards(self.trials_info, self.trials_result) + self.model.train(self.trials_info, self.inf_batch_size) + self.running_trials = {} + # generate new trials + self.trials_result = [None for _ in range(self.inf_batch_size)] + mb_obs, mb_actions, mb_values, mb_neglogpacs, mb_dones, last_values = self.model.inference(self.inf_batch_size) + self.trials_info = TrialsInfo(mb_obs, mb_actions, + mb_values, mb_neglogpacs, + mb_dones, last_values, + self.inf_batch_size) + logger.debug('Next round inference complete.') + # check credit and submit new trials + for _ in range(self.credit): + trial_info_idx, actions = self.trials_info.get_next() + if trial_info_idx is None: + logger.warning('No enough trial config, trials_per_update is suggested to be larger than trialConcurrency') + break + assert self.param_ids + param_id = self.param_ids.pop() + self.running_trials[param_id] = trial_info_idx + new_config = self._actions_to_config(actions) + self.send_trial_callback(param_id, new_config) + self.credit -= 1 + logger.debug('Send new trial (%d, %s) for reducing credit', param_id, new_config) + + def receive_trial_result(self, parameter_id, parameters, value, **kwargs): + """ + Receive trial's result. if the number of finished trials equals self.inf_batch_size, start the next update to + train the model. + + Parameters + ---------- + parameter_id : int + Unique identifier of used hyper-parameters, same with :meth:`generate_parameters`. + parameters : dict + Hyper-parameters generated by :meth:`generate_parameters`. + value : dict + Result from trial (the return value of :func:`nni.report_final_result`). + """ + trial_info_idx = self.running_trials.pop(parameter_id, None) + assert trial_info_idx is not None + + value = extract_scalar_reward(value) + if self.optimize_mode == OptimizeMode.Minimize: + value = -value + + self.trials_result[trial_info_idx] = value + self.finished_trials += 1 + + logger.debug('receive_trial_result, parameter_id %d, trial_info_idx %d, finished_trials %d, inf_batch_size %d', + parameter_id, trial_info_idx, self.finished_trials, self.inf_batch_size) + if self.finished_trials == self.inf_batch_size: + logger.debug('Start next round inference in receive_trial_result') + self._next_round_inference() + + def trial_end(self, parameter_id, success, **kwargs): + """ + To deal with trial failure. If a trial fails, it is popped out from ``self.running_trials``, + and the final result of this trial is assigned with the average of the finished trials. + + Parameters + ---------- + parameter_id : int + Unique identifier for hyper-parameters used by this trial. + success : bool + True if the trial successfully completed; False if failed or terminated. + **kwargs + Not used + """ + if not success: + if parameter_id not in self.running_trials: + logger.warning('The trial is failed, but self.running_trial does not have this trial') + return + trial_info_idx = self.running_trials.pop(parameter_id, None) + assert trial_info_idx is not None + # use mean of finished trials as the result of this failed trial + values = [val for val in self.trials_result if val is not None] + logger.warning('In trial_end, values: %s', values) + self.trials_result[trial_info_idx] = (sum(values) / len(values)) if values else 0 + self.finished_trials += 1 + if self.finished_trials == self.inf_batch_size: + logger.debug('Start next round inference in trial_end') + self._next_round_inference() + + def import_data(self, data): + """ + Import additional data for tuning, not supported yet. + + Parameters + ---------- + data : list + A list of dictionarys, each of which has at least two keys, ``parameter`` and ``value`` + """ + logger.warning('PPOTuner cannot leverage imported data.') diff --git a/utils/third_party/nni/algorithms/hpo/ppo_tuner/requirements.txt b/utils/third_party/nni/algorithms/hpo/ppo_tuner/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..a58dfb12f2fb8505cb47c2c88434bdded4553d47 --- /dev/null +++ b/utils/third_party/nni/algorithms/hpo/ppo_tuner/requirements.txt @@ -0,0 +1,2 @@ +enum34 +gym diff --git a/utils/third_party/nni/algorithms/hpo/ppo_tuner/util.py b/utils/third_party/nni/algorithms/hpo/ppo_tuner/util.py new file mode 100644 index 0000000000000000000000000000000000000000..605292de4002f114e935cb24a87c96921940e959 --- /dev/null +++ b/utils/third_party/nni/algorithms/hpo/ppo_tuner/util.py @@ -0,0 +1,260 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +util functions +""" + +import os +import random +import multiprocessing +import numpy as np +import tensorflow.compat.v1 as tf +tf.disable_v2_behavior() +from gym.spaces import Discrete, Box, MultiDiscrete + +def set_global_seeds(i): + """set global seeds""" + rank = 0 + myseed = i + 1000 * rank if i is not None else None + tf.set_random_seed(myseed) + np.random.seed(myseed) + random.seed(myseed) + +def batch_to_seq(h, nbatch, nsteps, flat=False): + """convert from batch to sequence""" + if flat: + h = tf.reshape(h, [nbatch, nsteps]) + else: + h = tf.reshape(h, [nbatch, nsteps, -1]) + return [tf.squeeze(v, [1]) for v in tf.split(axis=1, num_or_size_splits=nsteps, value=h)] + +def seq_to_batch(h, flat=False): + """convert from sequence to batch""" + shape = h[0].get_shape().as_list() + if not flat: + assert len(shape) > 1 + nh = h[0].get_shape()[-1].value + return tf.reshape(tf.concat(axis=1, values=h), [-1, nh]) + else: + return tf.reshape(tf.stack(values=h, axis=1), [-1]) + +def lstm(xs, ms, s, scope, nh, init_scale=1.0): + """lstm cell""" + _, nin = [v.value for v in xs[0].get_shape()] # the first is nbatch + with tf.variable_scope(scope): + wx = tf.get_variable("wx", [nin, nh*4], initializer=ortho_init(init_scale)) + wh = tf.get_variable("wh", [nh, nh*4], initializer=ortho_init(init_scale)) + b = tf.get_variable("b", [nh*4], initializer=tf.constant_initializer(0.0)) + + c, h = tf.split(axis=1, num_or_size_splits=2, value=s) + for idx, (x, m) in enumerate(zip(xs, ms)): + c = c*(1-m) + h = h*(1-m) + z = tf.matmul(x, wx) + tf.matmul(h, wh) + b + i, f, o, u = tf.split(axis=1, num_or_size_splits=4, value=z) + i = tf.nn.sigmoid(i) + f = tf.nn.sigmoid(f) + o = tf.nn.sigmoid(o) + u = tf.tanh(u) + c = f*c + i*u + h = o*tf.tanh(c) + xs[idx] = h + s = tf.concat(axis=1, values=[c, h]) + return xs, s + +def lstm_model(nlstm=128, layer_norm=False): + """ + Builds LSTM (Long-Short Term Memory) network to be used in a policy. + Note that the resulting function returns not only the output of the LSTM + (i.e. hidden state of lstm for each step in the sequence), but also a dictionary + with auxiliary tensors to be set as policy attributes. + + Specifically, + S is a placeholder to feed current state (LSTM state has to be managed outside policy) + M is a placeholder for the mask (used to mask out observations after the end of the episode, but can be used for other purposes too) + initial_state is a numpy array containing initial lstm state (usually zeros) + state is the output LSTM state (to be fed into S at the next call) + + + An example of usage of lstm-based policy can be found here: common/tests/test_doc_examples.py/test_lstm_example + + Parameters + ---------- + nlstm : int + LSTM hidden state size + layer_norm : bool + if True, layer-normalized version of LSTM is used + + Returns + ------- + function that builds LSTM with a given input tensor / placeholder + """ + + def network_fn(X, nenv=1, obs_size=-1): + with tf.variable_scope("emb", reuse=tf.AUTO_REUSE): + w_emb = tf.get_variable("w_emb", [obs_size+1, 32]) + X = tf.nn.embedding_lookup(w_emb, X) + + nbatch = X.shape[0] + nsteps = nbatch // nenv + + h = tf.layers.flatten(X) + + M = tf.placeholder(tf.float32, [nbatch]) #mask (done t-1) + S = tf.placeholder(tf.float32, [nenv, 2*nlstm]) #states + + xs = batch_to_seq(h, nenv, nsteps) + ms = batch_to_seq(M, nenv, nsteps) + + assert not layer_norm + h5, snew = lstm(xs, ms, S, scope='lstm', nh=nlstm) + + h = seq_to_batch(h5) + initial_state = np.zeros(S.shape.as_list(), dtype=float) + + return h, {'S':S, 'M':M, 'state':snew, 'initial_state':initial_state} + + return network_fn + +def ortho_init(scale=1.0): + """init approach""" + def _ortho_init(shape, dtype, partition_info=None): + #lasagne ortho init for tf + shape = tuple(shape) + if len(shape) == 2: + flat_shape = shape + elif len(shape) == 4: # assumes NHWC + flat_shape = (np.prod(shape[:-1]), shape[-1]) + else: + raise NotImplementedError + a = np.random.normal(0.0, 1.0, flat_shape) + u, _, v = np.linalg.svd(a, full_matrices=False) + q = u if u.shape == flat_shape else v # pick the one with the correct shape + q = q.reshape(shape) + return (scale * q[:shape[0], :shape[1]]).astype(np.float32) + return _ortho_init + +def fc(x, scope, nh, *, init_scale=1.0, init_bias=0.0): + """fully connected op""" + with tf.variable_scope(scope): + nin = x.get_shape()[1].value + w = tf.get_variable("w", [nin, nh], initializer=ortho_init(init_scale)) + b = tf.get_variable("b", [nh], initializer=tf.constant_initializer(init_bias)) + return tf.matmul(x, w)+b + +def _check_shape(placeholder_shape, data_shape): + """ + check if two shapes are compatible (i.e. differ only by dimensions of size 1, or by the batch dimension) + """ + + return True + +# ================================================================ +# Shape adjustment for feeding into tf placeholders +# ================================================================ +def adjust_shape(placeholder, data): + """ + adjust shape of the data to the shape of the placeholder if possible. + If shape is incompatible, AssertionError is thrown + + Parameters + ---------- + placeholder + tensorflow input placeholder + data + input data to be (potentially) reshaped to be fed into placeholder + + Returns + ------- + reshaped data + """ + if not isinstance(data, np.ndarray) and not isinstance(data, list): + return data + if isinstance(data, list): + data = np.array(data) + + placeholder_shape = [x or -1 for x in placeholder.shape.as_list()] + + assert _check_shape(placeholder_shape, data.shape), \ + 'Shape of data {} is not compatible with shape of the placeholder {}'.format(data.shape, placeholder_shape) + + return np.reshape(data, placeholder_shape) + +# ================================================================ +# Global session +# ================================================================ + +def get_session(config=None): + """Get default session or create one with a given config""" + sess = tf.get_default_session() + if sess is None: + sess = make_session(config=config, make_default=True) + return sess + +def make_session(config=None, num_cpu=None, make_default=False, graph=None): + """Returns a session that will use CPU's only""" + if num_cpu is None: + num_cpu = int(os.getenv('RCALL_NUM_CPU', multiprocessing.cpu_count())) + if config is None: + config = tf.ConfigProto( + allow_soft_placement=True, + inter_op_parallelism_threads=num_cpu, + intra_op_parallelism_threads=num_cpu) + config.gpu_options.allow_growth = True + + if make_default: + return tf.InteractiveSession(config=config, graph=graph) + else: + return tf.Session(config=config, graph=graph) + +ALREADY_INITIALIZED = set() + +def initialize(): + """Initialize all the uninitialized variables in the global scope.""" + new_variables = set(tf.global_variables()) - ALREADY_INITIALIZED + get_session().run(tf.variables_initializer(new_variables)) + + ALREADY_INITIALIZED.update(new_variables) + +def observation_placeholder(ob_space, batch_size=None, name='Ob'): + """ + Create placeholder to feed observations into of the size appropriate to the observation space + + Parameters + ---------- + ob_space : gym.Space + observation space + batch_size : int + size of the batch to be fed into input. Can be left None in most cases. + name : str + name of the placeholder + + Returns + ------- + tensorflow placeholder tensor + """ + + assert isinstance(ob_space, (Discrete, Box, MultiDiscrete)), \ + 'Can only deal with Discrete and Box observation spaces for now' + + dtype = ob_space.dtype + if dtype == np.int8: + dtype = np.uint8 + + return tf.placeholder(shape=(batch_size,) + ob_space.shape, dtype=dtype, name=name) + +def explained_variance(ypred, y): + """ + Computes fraction of variance that ypred explains about y. + Returns 1 - Var[y-ypred] / Var[y] + + interpretation: + ev=0 => might as well have predicted zero + ev=1 => perfect prediction + ev<0 => worse than just predicting zero + + """ + assert y.ndim == 1 and ypred.ndim == 1 + vary = np.var(y) + return np.nan if vary == 0 else 1 - np.var(y-ypred)/vary diff --git a/utils/third_party/nni/algorithms/hpo/regularized_evolution_tuner/__init__.py b/utils/third_party/nni/algorithms/hpo/regularized_evolution_tuner/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..befa4e446b9af62ff87abae8bf35134d5eecb68a --- /dev/null +++ b/utils/third_party/nni/algorithms/hpo/regularized_evolution_tuner/__init__.py @@ -0,0 +1 @@ +from .regularized_evolution_tuner import RegularizedEvolutionTuner diff --git a/utils/third_party/nni/algorithms/hpo/regularized_evolution_tuner/regularized_evolution_tuner.py b/utils/third_party/nni/algorithms/hpo/regularized_evolution_tuner/regularized_evolution_tuner.py new file mode 100644 index 0000000000000000000000000000000000000000..ac756c33885b8ad150e862c6eb315136fb0d4b71 --- /dev/null +++ b/utils/third_party/nni/algorithms/hpo/regularized_evolution_tuner/regularized_evolution_tuner.py @@ -0,0 +1,172 @@ +import copy +import logging +import random +from collections import deque + +from schema import Schema, Optional +import nni +from nni.tuner import Tuner +from nni import ClassArgsValidator +from nni.utils import OptimizeMode, extract_scalar_reward + +logger = logging.getLogger(__name__) + + +class FinishedIndividual: + def __init__(self, parameter_id, parameters, result): + """ + Parameters + ---------- + parameter_id: int + the index of the parameter + parameters : dict + chosen architecture and parameters + result : float + final metric of the chosen one + """ + self.parameter_id = parameter_id + self.parameters = parameters + self.result = result + + +class EvolutionClassArgsValidator(ClassArgsValidator): + def validate_class_args(self, **kwargs): + Schema({ + 'optimize_mode': self.choices('optimize_mode', 'maximize', 'minimize'), + Optional('population_size'): self.range('population_size', int, 0, 99999), + Optional('sample_size'): self.range('sample_size', int, 0, 9999), + }).validate(kwargs) + + +class RegularizedEvolutionTuner(Tuner): + """ + RegularizedEvolutionTuner is tuner using Evolution NAS Tuner. + See ``Regularized Evolution for Image Classifier Architecture Search`` for details. + + Parameters + --- + optimize_mode: str + whether to maximize metric or not. default: 'maximize' + population_size: int + the maximum number of kept models + sample_size: int + the number of models chosen from population each time when evolution + """ + def __init__(self, optimize_mode="maximize", population_size=100, sample_size=25): + super(RegularizedEvolutionTuner, self).__init__() + self.optimize_mode = OptimizeMode(optimize_mode) + self.population_size = population_size + self.sample_size = sample_size + self.initial_population = deque() + self.population = deque() + self.history = {} + self.search_space = None + self._from_initial = {} # whether the parameter is from initial population + + def generate_parameters(self, parameter_id, **kwargs): + """ + This function will returns a dict of trial (hyper-)parameters, as a serializable object. + + Parameters + --- + parameter_id: int + the index of current set of parameters + """ + if self.initial_population: + arch = self.initial_population.popleft() + self.history[parameter_id] = arch + self._from_initial[parameter_id] = True + return arch + elif self.population: + sample = [] + while len(sample) < self.sample_size: + sample.append(random.choice(list(self.population))) + + candidate = max(sample, key=lambda x: x.result) + arch = self._mutate_model(candidate) + self.history[parameter_id] = arch + self._from_initial[parameter_id] = False + return arch + else: + raise nni.NoMoreTrialError + + def receive_trial_result(self, parameter_id, parameters, value, **kwargs): + """ + Record the result from a trial + + Parameters + ---------- + parameter_id : int + parameters : dict + value : dict/float + if value is dict, it should have "default" key. + value is final metrics of the trial. + """ + reward = extract_scalar_reward(value) + if parameter_id not in self.history: + raise RuntimeError('Received parameter_id not in total_data.') + params = self.history[parameter_id] + + if self.optimize_mode == OptimizeMode.Minimize: + reward = -reward + + self.population.append(FinishedIndividual(parameter_id, params, reward)) + if len(self.population) > self.population_size: + self.population.popleft() + + def update_search_space(self, search_space): + """ + Update search space. + Search_space contains the information that user pre-defined. + + Parameters + ---------- + search_space : dict + """ + logger.info('update search space %s', search_space) + assert self.search_space is None + self.search_space = search_space + + for _, val in search_space.items(): + if val['_type'] != 'layer_choice' and val['_type'] != 'input_choice': + raise ValueError('Unsupported search space type: %s' % (val['_type'])) + + self._generate_initial_population() + + def trial_end(self, parameter_id, success, **kwargs): + if not success: + del self.history[parameter_id] + if self._from_initial[parameter_id]: + self.initial_population.append(self._random_model()) + del self._from_initial[parameter_id] + + def _mutate(self, key, individual): + mutate_val = self.search_space[key] + if mutate_val['_type'] == 'layer_choice': + idx = random.randint(0, len(mutate_val['_value']) - 1) + individual[key] = {'_value': mutate_val['_value'][idx], '_idx': idx} + elif mutate_val['_type'] == 'input_choice': + candidates = mutate_val['_value']['candidates'] + n_chosen = mutate_val['_value']['n_chosen'] + idxs = [random.randint(0, len(candidates) - 1) for _ in range(n_chosen)] + vals = [candidates[k] for k in idxs] + individual[key] = {'_value': vals, '_idx': idxs} + else: + raise KeyError + + def _random_model(self): + individual = {} + for key in self.search_space.keys(): + self._mutate(key, individual) + return individual + + def _mutate_model(self, model): + new_individual = copy.deepcopy(model.parameters) + mutate_key = random.choice(list(new_individual.keys())) + self._mutate(mutate_key, new_individual) + return new_individual + + def _generate_initial_population(self): + while len(self.initial_population) < self.population_size: + self.initial_population.append(self._random_model()) + logger.info('init population done.') diff --git a/utils/third_party/nni/algorithms/hpo/smac_tuner/__init__.py b/utils/third_party/nni/algorithms/hpo/smac_tuner/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..0b59d5ed1627bc2f27daedb880de62c3952af60c --- /dev/null +++ b/utils/third_party/nni/algorithms/hpo/smac_tuner/__init__.py @@ -0,0 +1,4 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .smac_tuner import SMACTuner diff --git a/utils/third_party/nni/algorithms/hpo/smac_tuner/convert_ss_to_scenario.py b/utils/third_party/nni/algorithms/hpo/smac_tuner/convert_ss_to_scenario.py new file mode 100644 index 0000000000000000000000000000000000000000..ef817c8cf4e10ad08185910b95bd4adc909b2dd3 --- /dev/null +++ b/utils/third_party/nni/algorithms/hpo/smac_tuner/convert_ss_to_scenario.py @@ -0,0 +1,202 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import json + +import numpy as np + + +def get_json_content(file_path): + """ + Load json file content + + Parameters + ---------- + file_path: + path to the file + + Raises + ------ + TypeError + Error with the file path + """ + try: + with open(file_path, 'r') as file: + return json.load(file) + except TypeError as err: + print('Error: ', err) + return None + + +def generate_pcs(nni_search_space_content): + """ + Generate the Parameter Configuration Space (PCS) which defines the + legal ranges of the parameters to be optimized and their default values. + Generally, the format is: + # parameter_name categorical {value_1, ..., value_N} [default value] + # parameter_name ordinal {value_1, ..., value_N} [default value] + # parameter_name integer [min_value, max_value] [default value] + # parameter_name integer [min_value, max_value] [default value] log + # parameter_name real [min_value, max_value] [default value] + # parameter_name real [min_value, max_value] [default value] log + Reference: https://automl.github.io/SMAC3/stable/options.html + + Parameters + ---------- + nni_search_space_content: search_space + The search space in this experiment in nni + + Returns + ------- + Parameter Configuration Space (PCS) + the legal ranges of the parameters to be optimized and their default values + + Raises + ------ + RuntimeError + unsupported type or value error or incorrect search space + """ + categorical_dict = {} + search_space = nni_search_space_content + + def dump_categorical(fd, key, categories): + choice_len = len(categories) + if key in categorical_dict: + raise RuntimeError( + '%s has already existed, please make sure search space has no duplicate key.' % key) + categorical_dict[key] = search_space[key]['_value'] + fd.write('%s categorical {%s} [0]\n' % (key, ','.join(map(str, range(choice_len))))) + + with open('param_config_space.pcs', 'w') as pcs_fd: + if isinstance(search_space, dict): + for key in search_space.keys(): + if isinstance(search_space[key], dict): + try: + if search_space[key]['_type'] == 'choice': + dump_categorical(pcs_fd, key, search_space[key]['_value']) + elif search_space[key]['_type'] == 'randint': + lower, upper = search_space[key]['_value'] + if lower + 1 == upper: + dump_categorical(pcs_fd, key, [lower]) + else: + pcs_fd.write('%s integer [%d, %d] [%d]\n' % (key, lower, upper - 1, lower)) + elif search_space[key]['_type'] == 'uniform': + low, high = search_space[key]['_value'] + if low == high: + dump_categorical(pcs_fd, key, [low]) + else: + pcs_fd.write('%s real [%s, %s] [%s]\n' % (key, low, high, low)) + elif search_space[key]['_type'] == 'loguniform': + # use np.round here to ensure that the rounded default value is in the range, + # which will be rounded in configure_space package + low, high = list(np.round(np.log(search_space[key]['_value']), 10)) + if low == high: + dump_categorical(pcs_fd, key, [search_space[key]['_value'][0]]) + else: + pcs_fd.write('%s real [%s, %s] [%s]\n' % (key, low, high, low)) + elif search_space[key]['_type'] == 'quniform': + low, high, q = search_space[key]['_value'][0:3] + vals = np.clip(np.arange(np.round(low / q), np.round(high / q) + 1) * q, low, high).tolist() + pcs_fd.write('%s ordinal {%s} [%s]\n' % ( + key, + json.dumps(vals)[1:-1], + json.dumps(vals[0]))) + else: + raise RuntimeError('unsupported _type %s' % search_space[key]['_type']) + except: + raise RuntimeError('_type or _value error.') + else: + raise RuntimeError('incorrect search space.') + return categorical_dict + return None + + +def generate_scenario(ss_content): + """ + Generate the scenario. The scenario-object (smac.scenario.scenario.Scenario) is used to configure SMAC and + can be constructed either by providing an actual scenario-object, or by specifing the options in a scenario file. + Reference: https://automl.github.io/SMAC3/stable/options.html + The format of the scenario file is one option per line: + OPTION1 = VALUE1 + OPTION2 = VALUE2 + ... + Parameters + ---------- + abort_on_first_run_crash: bool + If true, SMAC will abort if the first run of the target algorithm crashes. Default: True, + because trials reported to nni tuner would always in success state + algo: function + Specifies the target algorithm call that SMAC will optimize. Interpreted as a bash-command. + Not required by tuner, but required by nni's training service for running trials + always_race_default: + Race new incumbents always against default configuration + cost_for_crash: + Defines the cost-value for crashed runs on scenarios with quality as run-obj. Default: 2147483647.0. + Trials reported to nni tuner would always in success state + cutoff_time: + Maximum runtime, after which the target algorithm is cancelled. `Required if *run_obj* is runtime` + deterministic: bool + If true, the optimization process will be repeatable. + execdir: + Specifies the path to the execution-directory. Default: . + Trials are executed by nni's training service + feature_file: + Specifies the file with the instance-features. + No features specified or feature file is not supported + initial_incumbent: + DEFAULT is the default from the PCS. Default: DEFAULT. Must be from: [‘DEFAULT’, ‘RANDOM’]. + input_psmac_dirs: + For parallel SMAC, multiple output-directories are used. + Parallelism is supported by nni + instance_file: + Specifies the file with the training-instances. Not supported + intensification_percentage: + The fraction of time to be used on intensification (versus choice of next Configurations). Default: 0.5. + Not supported, trials are controlled by nni's training service and kill be assessor + maxR: int + Maximum number of calls per configuration. Default: 2000. + memory_limit: + Maximum available memory the target algorithm can occupy before being cancelled. + minR: int + Minimum number of calls per configuration. Default: 1. + output_dir: + Specifies the output-directory for all emerging files, such as logging and results. + Default: smac3-output_2018-01-22_15:05:56_807070. + overall_obj: + PARX, where X is an integer defining the penalty imposed on timeouts (i.e. runtimes that exceed the cutoff-time). + Timeout is not supported + paramfile: + Specifies the path to the PCS-file. + run_obj: + Defines what metric to optimize. When optimizing runtime, cutoff_time is required as well. + Must be from: [‘runtime’, ‘quality’]. + runcount_limit: int + Maximum number of algorithm-calls during optimization. Default: inf. + Use default because this is controlled by nni + shared_model: + Whether to run SMAC in parallel mode. Parallelism is supported by nni + test_instance_file: + Specifies the file with the test-instances. Instance is not supported + tuner-timeout: + Maximum amount of CPU-time used for optimization. Not supported + wallclock_limit: int + Maximum amount of wallclock-time used for optimization. Default: inf. + Use default because this is controlled by nni + + Returns + ------- + Scenario: + The scenario-object (smac.scenario.scenario.Scenario) is used to configure SMAC and can be constructed + either by providing an actual scenario-object, or by specifing the options in a scenario file + """ + with open('scenario.txt', 'w') as sce_fd: + sce_fd.write('deterministic = 0\n') + # sce_fd.write('output_dir = \n') + sce_fd.write('paramfile = param_config_space.pcs\n') + sce_fd.write('run_obj = quality\n') + + return generate_pcs(ss_content) + + +if __name__ == '__main__': + generate_scenario('search_space.json') diff --git a/utils/third_party/nni/algorithms/hpo/smac_tuner/requirements.txt b/utils/third_party/nni/algorithms/hpo/smac_tuner/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..a3027fb6fe5ca4b4725ee5a3db8ec4e53a996166 --- /dev/null +++ b/utils/third_party/nni/algorithms/hpo/smac_tuner/requirements.txt @@ -0,0 +1,2 @@ +git+https://github.com/QuanluZhang/ConfigSpace.git +git+https://github.com/QuanluZhang/SMAC3.git diff --git a/utils/third_party/nni/algorithms/hpo/smac_tuner/smac_tuner.py b/utils/third_party/nni/algorithms/hpo/smac_tuner/smac_tuner.py new file mode 100644 index 0000000000000000000000000000000000000000..fdaed49d8f2480516ec1e1e62fc43eabdd574049 --- /dev/null +++ b/utils/third_party/nni/algorithms/hpo/smac_tuner/smac_tuner.py @@ -0,0 +1,346 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +smac_tuner.py +""" + +import logging +import sys + +import numpy as np +from schema import Schema, Optional + +from smac.facade.epils_facade import EPILS +from smac.facade.roar_facade import ROAR +from smac.facade.smac_facade import SMAC +from smac.scenario.scenario import Scenario +from smac.utils.io.cmd_reader import CMDReader + +from ConfigSpaceNNI import Configuration + +import nni +from nni import ClassArgsValidator +from nni.tuner import Tuner +from nni.utils import OptimizeMode, extract_scalar_reward + +from .convert_ss_to_scenario import generate_scenario + +logger = logging.getLogger('smac_AutoML') + +class SMACClassArgsValidator(ClassArgsValidator): + def validate_class_args(self, **kwargs): + Schema({ + 'optimize_mode': self.choices('optimize_mode', 'maximize', 'minimize'), + Optional('config_dedup'): bool + }).validate(kwargs) + +class SMACTuner(Tuner): + """ + This is a wrapper of [SMAC](https://github.com/automl/SMAC3) following NNI tuner interface. + It only supports ``SMAC`` mode, and does not support the multiple instances of SMAC3 (i.e., + the same configuration is run multiple times). + """ + def __init__(self, optimize_mode="maximize", config_dedup=False): + """ + Parameters + ---------- + optimize_mode : str + Optimize mode, 'maximize' or 'minimize', by default 'maximize' + config_dedup : bool + If True, the tuner will not generate a configuration that has been already generated. + If False, a configuration may be generated twice, but it is rare for relatively large search space. + """ + self.logger = logger + self.optimize_mode = OptimizeMode(optimize_mode) + self.total_data = {} + self.optimizer = None + self.smbo_solver = None + self.first_one = True + self.update_ss_done = False + self.loguniform_key = set() + self.categorical_dict = {} + self.cs = None + self.dedup = config_dedup + + def _main_cli(self): + """ + Main function of SMAC for CLI interface. Some initializations of the wrapped SMAC are done + in this function. + + Returns + ------- + obj + The object of the SMAC optimizer + """ + self.logger.info("SMAC call: %s", " ".join(sys.argv)) + + cmd_reader = CMDReader() + args, _ = cmd_reader.read_cmd() + + root_logger = logging.getLogger() + root_logger.setLevel(args.verbose_level) + logger_handler = logging.StreamHandler(stream=sys.stdout) + if root_logger.level >= logging.INFO: + formatter = logging.Formatter("%(levelname)s:\t%(message)s") + else: + formatter = logging.Formatter( + "%(asctime)s:%(levelname)s:%(name)s:%(message)s", + "%Y-%m-%d %H:%M:%S") + logger_handler.setFormatter(formatter) + root_logger.addHandler(logger_handler) + # remove default handler + root_logger.removeHandler(root_logger.handlers[0]) + + # Create defaults + rh = None + initial_configs = None + stats = None + incumbent = None + + # Create scenario-object + scen = Scenario(args.scenario_file, []) + self.cs = scen.cs + + if args.mode == "SMAC": + optimizer = SMAC( + scenario=scen, + rng=np.random.RandomState(args.seed), + runhistory=rh, + initial_configurations=initial_configs, + stats=stats, + restore_incumbent=incumbent, + run_id=args.seed) + elif args.mode == "ROAR": + optimizer = ROAR( + scenario=scen, + rng=np.random.RandomState(args.seed), + runhistory=rh, + initial_configurations=initial_configs, + run_id=args.seed) + elif args.mode == "EPILS": + optimizer = EPILS( + scenario=scen, + rng=np.random.RandomState(args.seed), + runhistory=rh, + initial_configurations=initial_configs, + run_id=args.seed) + else: + optimizer = None + + return optimizer + + def update_search_space(self, search_space): + """ + Convert search_space to the format that ``SMAC3`` could recognize, thus, not all the search space types + are supported. In this function, we also do the initialization of `SMAC3`, i.e., calling ``self._main_cli``. + + NOTE: updating search space during experiment running is not supported. + + Parameters + ---------- + search_space : dict + The format could be referred to search space spec (https://nni.readthedocs.io/en/latest/Tutorial/SearchSpaceSpec.html). + """ + self.logger.info('update search space in SMAC.') + if not self.update_ss_done: + self.categorical_dict = generate_scenario(search_space) + if self.categorical_dict is None: + raise RuntimeError('categorical dict is not correctly returned after parsing search space.') + # TODO: this is ugly, we put all the initialization work in this method, because initialization relies + # on search space, also because update_search_space is called at the beginning. + self.optimizer = self._main_cli() + self.smbo_solver = self.optimizer.solver + self.loguniform_key = {key for key in search_space.keys() if search_space[key]['_type'] == 'loguniform'} + self.update_ss_done = True + else: + self.logger.warning('update search space is not supported.') + + def receive_trial_result(self, parameter_id, parameters, value, **kwargs): + """ + Receive a trial's final performance result reported through :func:``nni.report_final_result`` by the trial. + GridSearchTuner does not need trial's results. + + Parameters + ---------- + parameter_id : int + Unique identifier of used hyper-parameters, same with :meth:`generate_parameters`. + parameters : dict + Hyper-parameters generated by :meth:`generate_parameters`. + value : dict + Result from trial (the return value of :func:`nni.report_final_result`). + + Raises + ------ + RuntimeError + Received parameter id not in ``self.total_data`` + """ + reward = extract_scalar_reward(value) + if self.optimize_mode is OptimizeMode.Maximize: + reward = -reward + + if parameter_id not in self.total_data: + raise RuntimeError('Received parameter_id not in total_data.') + if self.first_one: + self.smbo_solver.nni_smac_receive_first_run(self.total_data[parameter_id], reward) + self.first_one = False + else: + self.smbo_solver.nni_smac_receive_runs(self.total_data[parameter_id], reward) + + def param_postprocess(self, challenger_dict): + """ + Postprocessing for a set of hyperparameters includes: + 1. Convert the values of type ``loguniform`` back to their initial range. + 2. Convert ``categorical``: categorical values in search space are changed to list of numbers before, + those original values will be changed back in this function. + + Parameters + ---------- + challenger_dict : dict + challenger dict + + Returns + ------- + dict + dict which stores copy of challengers + """ + converted_dict = {} + for key, value in challenger_dict.items(): + # convert to loguniform + if key in self.loguniform_key: + converted_dict[key] = np.exp(challenger_dict[key]) + # convert categorical back to original value + elif key in self.categorical_dict: + idx = challenger_dict[key] + converted_dict[key] = self.categorical_dict[key][idx] + else: + converted_dict[key] = value + return converted_dict + + def generate_parameters(self, parameter_id, **kwargs): + """ + Generate one instance of hyperparameters (i.e., one configuration). + Get one from SMAC3's ``challengers``. + + Parameters + ---------- + parameter_id : int + Unique identifier for requested hyper-parameters. This will later be used in :meth:`receive_trial_result`. + **kwargs + Not used + + Returns + ------- + dict + One newly generated configuration + """ + if self.first_one: + init_challenger = self.smbo_solver.nni_smac_start() + self.total_data[parameter_id] = init_challenger + return self.param_postprocess(init_challenger.get_dictionary()) + else: + challengers = self.smbo_solver.nni_smac_request_challengers() + challengers_empty = True + for challenger in challengers: + challengers_empty = False + if self.dedup: + match = [v for k, v in self.total_data.items() \ + if v.get_dictionary() == challenger.get_dictionary()] + if match: + continue + self.total_data[parameter_id] = challenger + return self.param_postprocess(challenger.get_dictionary()) + assert challengers_empty is False, 'The case that challengers is empty is not handled.' + self.logger.info('In generate_parameters: No more new parameters.') + raise nni.NoMoreTrialError('No more new parameters.') + + def generate_multiple_parameters(self, parameter_id_list, **kwargs): + """ + Generate mutiple instances of hyperparameters. If it is a first request, + retrieve the instances from initial challengers. While if it is not, request + new challengers and retrieve instances from the requested challengers. + + Parameters + ---------- + parameter_id_list: list of int + Unique identifiers for each set of requested hyper-parameters. + These will later be used in :meth:`receive_trial_result`. + **kwargs + Not used + + Returns + ------- + list + a list of newly generated configurations + """ + if self.first_one: + params = [] + for one_id in parameter_id_list: + init_challenger = self.smbo_solver.nni_smac_start() + self.total_data[one_id] = init_challenger + params.append(self.param_postprocess(init_challenger.get_dictionary())) + else: + challengers = self.smbo_solver.nni_smac_request_challengers() + cnt = 0 + params = [] + for challenger in challengers: + if cnt >= len(parameter_id_list): + break + if self.dedup: + match = [v for k, v in self.total_data.items() \ + if v.get_dictionary() == challenger.get_dictionary()] + if match: + continue + self.total_data[parameter_id_list[cnt]] = challenger + params.append(self.param_postprocess(challenger.get_dictionary())) + cnt += 1 + if self.dedup and not params: + self.logger.info('In generate_multiple_parameters: No more new parameters.') + return params + + def import_data(self, data): + """ + Import additional data for tuning. + + Parameters + ---------- + data : list of dict + Each of which has at least two keys, ``parameter`` and ``value``. + """ + _completed_num = 0 + for trial_info in data: + self.logger.info("Importing data, current processing progress %s / %s", _completed_num, len(data)) + # simply validate data format + assert "parameter" in trial_info + _params = trial_info["parameter"] + assert "value" in trial_info + _value = trial_info['value'] + if not _value: + self.logger.info("Useless trial data, value is %s, skip this trial data.", _value) + continue + _value = extract_scalar_reward(_value) + # convert the keys in loguniform and categorical types + valid_entry = True + for key, value in _params.items(): + if key in self.loguniform_key: + _params[key] = np.log(value) + elif key in self.categorical_dict: + if value in self.categorical_dict[key]: + _params[key] = self.categorical_dict[key].index(value) + else: + self.logger.info("The value %s of key %s is not in search space.", str(value), key) + valid_entry = False + break + if not valid_entry: + continue + # start import this data entry + _completed_num += 1 + config = Configuration(self.cs, values=_params) + if self.optimize_mode is OptimizeMode.Maximize: + _value = -_value + if self.first_one: + self.smbo_solver.nni_smac_receive_first_run(config, _value) + self.first_one = False + else: + self.smbo_solver.nni_smac_receive_runs(config, _value) + self.logger.info("Successfully import data to smac tuner, total data: %d, imported data: %d.", len(data), _completed_num) diff --git a/utils/third_party/nni/algorithms/nas/__init__.py b/utils/third_party/nni/algorithms/nas/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/utils/third_party/nni/algorithms/nas/pytorch/__init__.py b/utils/third_party/nni/algorithms/nas/pytorch/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/utils/third_party/nni/algorithms/nas/pytorch/cdarts/__init__.py b/utils/third_party/nni/algorithms/nas/pytorch/cdarts/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..2d0092784646cef640f40eaeca2df1fe26bd3d4c --- /dev/null +++ b/utils/third_party/nni/algorithms/nas/pytorch/cdarts/__init__.py @@ -0,0 +1,5 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .mutator import RegularizedDartsMutator, RegularizedMutatorParallel, DartsDiscreteMutator +from .trainer import CdartsTrainer \ No newline at end of file diff --git a/utils/third_party/nni/algorithms/nas/pytorch/cdarts/mutator.py b/utils/third_party/nni/algorithms/nas/pytorch/cdarts/mutator.py new file mode 100644 index 0000000000000000000000000000000000000000..85fe9e1a1c1133d32383c8174d0fa5bdfb92f39a --- /dev/null +++ b/utils/third_party/nni/algorithms/nas/pytorch/cdarts/mutator.py @@ -0,0 +1,143 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import torch + +from apex.parallel import DistributedDataParallel # pylint: disable=import-error +from nni.algorithms.nas.pytorch.darts import DartsMutator # pylint: disable=wrong-import-order +from nni.nas.pytorch.mutables import LayerChoice # pylint: disable=wrong-import-order +from nni.nas.pytorch.mutator import Mutator # pylint: disable=wrong-import-order + + +class RegularizedDartsMutator(DartsMutator): + """ + This is :class:`~nni.algorithms.nas.pytorch.darts.DartsMutator` basically, with two differences. + + 1. Choices can be cut (bypassed). This is done by ``cut_choices``. Cutted choices will not be used in + forward pass and thus consumes no memory. + + 2. Regularization on choices, to prevent the mutator from overfitting on some choices. + """ + + def reset(self): + """ + Warnings + -------- + Renamed :func:`~reset_with_loss` to return regularization loss on reset. + """ + raise ValueError("You should probably call `reset_with_loss`.") + + def cut_choices(self, cut_num=2): + """ + Cut the choices with the smallest weights. + ``cut_num`` should be the accumulative number of cutting, e.g., if first time cutting + is 2, the second time should be 4 to cut another two. + + Parameters + ---------- + cut_num : int + Number of choices to cut, so far. + + Warnings + -------- + Though the parameters are set to :math:`-\infty` to be bypassed, they will still receive gradient of 0, + which introduced ``nan`` problem when calling ``optimizer.step()``. To solve this issue, a simple way is to + reset nan to :math:`-\infty` each time after the parameters are updated. + """ + # `cut_choices` is implemented but not used in current implementation of CdartsTrainer + for mutable in self.mutables: + if isinstance(mutable, LayerChoice): + _, idx = torch.topk(-self.choices[mutable.key], cut_num) + with torch.no_grad(): + for i in idx: + self.choices[mutable.key][i] = -float("inf") + + def reset_with_loss(self): + """ + Resample and return loss. If loss is 0, to avoid device issue, it will return ``None``. + + Currently loss penalty are proportional to the L1-norm of parameters corresponding + to modules if their type name contains certain substrings. These substrings include: ``poolwithoutbn``, + ``identity``, ``dilconv``. + """ + self._cache, reg_loss = self.sample_search() + return reg_loss + + def sample_search(self): + result = super().sample_search() + loss = [] + for mutable in self.mutables: + if isinstance(mutable, LayerChoice): + def need_reg(choice): + return any(t in str(type(choice)).lower() for t in ["poolwithoutbn", "identity", "dilconv"]) + + for i, choice in enumerate(mutable.choices): + if need_reg(choice): + norm = torch.abs(self.choices[mutable.key][i]) + if norm < 1E10: + loss.append(norm) + if not loss: + return result, None + return result, sum(loss) + + def export(self, logger=None): + """ + Export an architecture with logger. Genotype will be printed with logger. + + Returns + ------- + dict + A mapping from mutable keys to decisions. + """ + result = self.sample_final() + if hasattr(self.model, "plot_genotype") and logger is not None: + genotypes = self.model.plot_genotype(result, logger) + return result, genotypes + + +class RegularizedMutatorParallel(DistributedDataParallel): + """ + Parallelize :class:`~RegularizedDartsMutator`. + + This makes :func:`~RegularizedDartsMutator.reset_with_loss` method parallelized, + also allowing :func:`~RegularizedDartsMutator.cut_choices` and :func:`~RegularizedDartsMutator.export` + to be easily accessible. + """ + def reset_with_loss(self): + """ + Parallelized :func:`~RegularizedDartsMutator.reset_with_loss`. + """ + result = self.module.reset_with_loss() + self.callback_queued = False + return result + + def cut_choices(self, *args, **kwargs): + """ + Parallelized :func:`~RegularizedDartsMutator.cut_choices`. + """ + self.module.cut_choices(*args, **kwargs) + + def export(self, logger): + """ + Parallelized :func:`~RegularizedDartsMutator.export`. + """ + return self.module.export(logger) + + +class DartsDiscreteMutator(Mutator): + """ + A mutator that applies the final sampling result of a parent mutator on another model to train. + + Parameters + ---------- + model : nn.Module + The model to apply the mutator. + parent_mutator : Mutator + The mutator that provides ``sample_final`` method, that will be called to get the architecture. + """ + def __init__(self, model, parent_mutator): + super().__init__(model) + self.__dict__["parent_mutator"] = parent_mutator # avoid parameters to be included + + def sample_search(self): + return self.parent_mutator.sample_final() diff --git a/utils/third_party/nni/algorithms/nas/pytorch/cdarts/trainer.py b/utils/third_party/nni/algorithms/nas/pytorch/cdarts/trainer.py new file mode 100644 index 0000000000000000000000000000000000000000..1a5174216fc4ffdc224e9cd6bd1483322005fe01 --- /dev/null +++ b/utils/third_party/nni/algorithms/nas/pytorch/cdarts/trainer.py @@ -0,0 +1,275 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import json +import logging +import os + +import torch +import torch.nn as nn +import torch.nn.functional as F +import apex # pylint: disable=import-error +from apex.parallel import DistributedDataParallel # pylint: disable=import-error +from .mutator import RegularizedDartsMutator, RegularizedMutatorParallel, DartsDiscreteMutator # pylint: disable=wrong-import-order +from nni.nas.pytorch.utils import AverageMeterGroup # pylint: disable=wrong-import-order + +from .utils import CyclicIterator, TorchTensorEncoder, accuracy, reduce_metrics + +PHASE_SMALL = "small" +PHASE_LARGE = "large" + + +class InteractiveKLLoss(nn.Module): + def __init__(self, temperature): + super().__init__() + self.temperature = temperature + # self.kl_loss = nn.KLDivLoss(reduction = 'batchmean') + self.kl_loss = nn.KLDivLoss() + + def forward(self, student, teacher): + return self.kl_loss(F.log_softmax(student / self.temperature, dim=1), + F.softmax(teacher / self.temperature, dim=1)) + + +class CdartsTrainer(object): + """ + CDARTS trainer. + + Parameters + ---------- + model_small : nn.Module + PyTorch model to be trained. This is the search network of CDARTS. + model_large : nn.Module + PyTorch model to be trained. This is the evaluation network of CDARTS. + criterion : callable + Receives logits and ground truth label, return a loss tensor, e.g., ``nn.CrossEntropyLoss()``. + loaders : list of torch.utils.data.DataLoader + List of train data and valid data loaders, for training weights and architecture weights respectively. + samplers : list of torch.utils.data.Sampler + List of train data and valid data samplers. This can be PyTorch standard samplers if not distributed. + In distributed mode, sampler needs to have ``set_epoch`` method. Refer to data utils in CDARTS example for details. + logger : logging.Logger + The logger for logging. Will use nni logger by default (if logger is ``None``). + regular_coeff : float + The coefficient of regular loss. + regular_ratio : float + The ratio of regular loss. + warmup_epochs : int + The epochs to warmup the search network + fix_head : bool + ``True`` if fixing the paramters of auxiliary heads, else unfix the paramters of auxiliary heads. + epochs : int + Number of epochs planned for training. + steps_per_epoch : int + Steps of one epoch. + loss_alpha : float + The loss coefficient. + loss_T : float + The loss coefficient. + distributed : bool + ``True`` if using distributed training, else non-distributed training. + log_frequency : int + Step count per logging. + grad_clip : float + Gradient clipping for weights. + interactive_type : string + ``kl`` or ``smoothl1``. + output_path : string + Log storage path. + w_lr : float + Learning rate of the search network parameters. + w_momentum : float + Momentum of the search and the evaluation network. + w_weight_decay : float + The weight decay the search and the evaluation network parameters. + alpha_lr : float + Learning rate of the architecture parameters. + alpha_weight_decay : float + The weight decay the architecture parameters. + nasnet_lr : float + Learning rate of the evaluation network parameters. + local_rank : int + The number of thread. + share_module : bool + ``True`` if sharing the stem and auxiliary heads, else not sharing these modules. + """ + def __init__(self, model_small, model_large, criterion, loaders, samplers, logger=None, + regular_coeff=5, regular_ratio=0.2, warmup_epochs=2, fix_head=True, + epochs=32, steps_per_epoch=None, loss_alpha=2, loss_T=2, distributed=True, + log_frequency=10, grad_clip=5.0, interactive_type='kl', output_path='./outputs', + w_lr=0.2, w_momentum=0.9, w_weight_decay=3e-4, alpha_lr=0.2, alpha_weight_decay=1e-4, + nasnet_lr=0.2, local_rank=0, share_module=True): + if logger is None: + logger = logging.getLogger(__name__) + train_loader, valid_loader = loaders + train_sampler, valid_sampler = samplers + self.train_loader = CyclicIterator(train_loader, train_sampler, distributed) + self.valid_loader = CyclicIterator(valid_loader, valid_sampler, distributed) + + self.regular_coeff = regular_coeff + self.regular_ratio = regular_ratio + self.warmup_epochs = warmup_epochs + self.fix_head = fix_head + self.epochs = epochs + self.steps_per_epoch = steps_per_epoch + if self.steps_per_epoch is None: + self.steps_per_epoch = min(len(self.train_loader), len(self.valid_loader)) + self.loss_alpha = loss_alpha + self.grad_clip = grad_clip + if interactive_type == "kl": + self.interactive_loss = InteractiveKLLoss(loss_T) + elif interactive_type == "smoothl1": + self.interactive_loss = nn.SmoothL1Loss() + self.loss_T = loss_T + self.distributed = distributed + self.log_frequency = log_frequency + self.main_proc = not distributed or local_rank == 0 + + self.logger = logger + self.checkpoint_dir = output_path + if self.main_proc: + os.makedirs(self.checkpoint_dir, exist_ok=True) + if distributed: + torch.distributed.barrier() + + self.model_small = model_small + self.model_large = model_large + if self.fix_head: + for param in self.model_small.aux_head.parameters(): + param.requires_grad = False + for param in self.model_large.aux_head.parameters(): + param.requires_grad = False + + self.mutator_small = RegularizedDartsMutator(self.model_small).cuda() + self.mutator_large = DartsDiscreteMutator(self.model_large, self.mutator_small).cuda() + self.criterion = criterion + + self.optimizer_small = torch.optim.SGD(self.model_small.parameters(), w_lr, + momentum=w_momentum, weight_decay=w_weight_decay) + self.optimizer_large = torch.optim.SGD(self.model_large.parameters(), nasnet_lr, + momentum=w_momentum, weight_decay=w_weight_decay) + self.optimizer_alpha = torch.optim.Adam(self.mutator_small.parameters(), alpha_lr, + betas=(0.5, 0.999), weight_decay=alpha_weight_decay) + + if distributed: + apex.parallel.convert_syncbn_model(self.model_small) + apex.parallel.convert_syncbn_model(self.model_large) + self.model_small = DistributedDataParallel(self.model_small, delay_allreduce=True) + self.model_large = DistributedDataParallel(self.model_large, delay_allreduce=True) + self.mutator_small = RegularizedMutatorParallel(self.mutator_small, delay_allreduce=True) + if share_module: + self.model_small.callback_queued = True + self.model_large.callback_queued = True + # mutator large never gets optimized, so do not need parallelized + + def _warmup(self, phase, epoch): + assert phase in [PHASE_SMALL, PHASE_LARGE] + if phase == PHASE_SMALL: + model, optimizer = self.model_small, self.optimizer_small + elif phase == PHASE_LARGE: + model, optimizer = self.model_large, self.optimizer_large + model.train() + meters = AverageMeterGroup() + for step in range(self.steps_per_epoch): + x, y = next(self.train_loader) + x, y = x.cuda(), y.cuda() + + optimizer.zero_grad() + logits_main, _ = model(x) + loss = self.criterion(logits_main, y) + loss.backward() + + self._clip_grad_norm(model) + optimizer.step() + prec1, prec5 = accuracy(logits_main, y, topk=(1, 5)) + metrics = {"prec1": prec1, "prec5": prec5, "loss": loss} + metrics = reduce_metrics(metrics, self.distributed) + meters.update(metrics) + if self.main_proc and (step % self.log_frequency == 0 or step + 1 == self.steps_per_epoch): + self.logger.info("Epoch [%d/%d] Step [%d/%d] (%s) %s", epoch + 1, self.epochs, + step + 1, self.steps_per_epoch, phase, meters) + + def _clip_grad_norm(self, model): + if isinstance(model, DistributedDataParallel): + nn.utils.clip_grad_norm_(model.module.parameters(), self.grad_clip) + else: + nn.utils.clip_grad_norm_(model.parameters(), self.grad_clip) + + def _reset_nan(self, parameters): + with torch.no_grad(): + for param in parameters: + for i, p in enumerate(param): + if p != p: # equivalent to `isnan(p)` + param[i] = float("-inf") + + def _joint_train(self, epoch): + self.model_large.train() + self.model_small.train() + meters = AverageMeterGroup() + for step in range(self.steps_per_epoch): + trn_x, trn_y = next(self.train_loader) + val_x, val_y = next(self.valid_loader) + trn_x, trn_y = trn_x.cuda(), trn_y.cuda() + val_x, val_y = val_x.cuda(), val_y.cuda() + + # step 1. optimize architecture + self.optimizer_alpha.zero_grad() + self.optimizer_large.zero_grad() + reg_decay = max(self.regular_coeff * (1 - float(epoch - self.warmup_epochs) / ( + (self.epochs - self.warmup_epochs) * self.regular_ratio)), 0) + loss_regular = self.mutator_small.reset_with_loss() + if loss_regular: + loss_regular *= reg_decay + logits_search, emsemble_logits_search = self.model_small(val_x) + logits_main, emsemble_logits_main = self.model_large(val_x) + loss_cls = (self.criterion(logits_search, val_y) + self.criterion(logits_main, val_y)) / self.loss_alpha + loss_interactive = self.interactive_loss(emsemble_logits_search, emsemble_logits_main) * (self.loss_T ** 2) * self.loss_alpha + loss = loss_cls + loss_interactive + loss_regular + loss.backward() + self._clip_grad_norm(self.model_large) + self.optimizer_large.step() + self.optimizer_alpha.step() + # NOTE: need to call here `self._reset_nan(self.mutator_small.parameters())` if `cut_choices` + + # step 2. optimize op weights + self.optimizer_small.zero_grad() + with torch.no_grad(): + # resample architecture since parameters have been changed + self.mutator_small.reset_with_loss() + logits_search_train, _ = self.model_small(trn_x) + loss_weight = self.criterion(logits_search_train, trn_y) + loss_weight.backward() + self._clip_grad_norm(self.model_small) + self.optimizer_small.step() + + metrics = {"loss_cls": loss_cls, "loss_interactive": loss_interactive, + "loss_regular": loss_regular, "loss_weight": loss_weight} + metrics = reduce_metrics(metrics, self.distributed) + meters.update(metrics) + + if self.main_proc and (step % self.log_frequency == 0 or step + 1 == self.steps_per_epoch): + self.logger.info("Epoch [%d/%d] Step [%d/%d] (joint) %s", epoch + 1, self.epochs, + step + 1, self.steps_per_epoch, meters) + + def train(self): + for epoch in range(self.epochs): + if epoch < self.warmup_epochs: + with torch.no_grad(): # otherwise grads will be retained on the architecture params + self.mutator_small.reset_with_loss() + self._warmup(PHASE_SMALL, epoch) + else: + with torch.no_grad(): + self.mutator_large.reset() + self._warmup(PHASE_LARGE, epoch) + self._joint_train(epoch) + + self.export(os.path.join(self.checkpoint_dir, "epoch_{:02d}.json".format(epoch)), + os.path.join(self.checkpoint_dir, "epoch_{:02d}.genotypes".format(epoch))) + + def export(self, file, genotype_file): + if self.main_proc: + mutator_export, genotypes = self.mutator_small.export(self.logger) + with open(file, "w") as f: + json.dump(mutator_export, f, indent=2, sort_keys=True, cls=TorchTensorEncoder) + with open(genotype_file, "w") as f: + f.write(str(genotypes)) diff --git a/utils/third_party/nni/algorithms/nas/pytorch/cdarts/utils.py b/utils/third_party/nni/algorithms/nas/pytorch/cdarts/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..780f6fdc0eecdd2fb00593bad1ef17ba2d8f7d39 --- /dev/null +++ b/utils/third_party/nni/algorithms/nas/pytorch/cdarts/utils.py @@ -0,0 +1,76 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import json +import os + +import torch +import torch.distributed as dist + + +class CyclicIterator: + def __init__(self, loader, sampler, distributed): + self.loader = loader + self.sampler = sampler + self.epoch = 0 + self.distributed = distributed + self._next_epoch() + + def _next_epoch(self): + if self.distributed: + self.sampler.set_epoch(self.epoch) + self.iterator = iter(self.loader) + self.epoch += 1 + + def __len__(self): + return len(self.loader) + + def __iter__(self): + return self + + def __next__(self): + try: + return next(self.iterator) + except StopIteration: + self._next_epoch() + return next(self.iterator) + + +class TorchTensorEncoder(json.JSONEncoder): + def default(self, o): # pylint: disable=method-hidden + if isinstance(o, torch.Tensor): + return o.tolist() + return super().default(o) + + +def accuracy(output, target, topk=(1,)): + """ Computes the precision@k for the specified values of k """ + maxk = max(topk) + batch_size = target.size(0) + + _, pred = output.topk(maxk, 1, True, True) + pred = pred.t() + # one-hot case + if target.ndimension() > 1: + target = target.max(1)[1] + + correct = pred.eq(target.view(1, -1).expand_as(pred)) + + res = [] + for k in topk: + correct_k = correct[:k].view(-1).float().sum(0) + res.append(correct_k.mul_(1.0 / batch_size)) + return res + + +def reduce_tensor(tensor): + rt = tensor.clone() + dist.all_reduce(rt, op=dist.ReduceOp.SUM) + rt /= float(os.environ["WORLD_SIZE"]) + return rt + + +def reduce_metrics(metrics, distributed=False): + if distributed: + return {k: reduce_tensor(v).item() for k, v in metrics.items()} + return {k: v.item() for k, v in metrics.items()} diff --git a/utils/third_party/nni/algorithms/nas/pytorch/classic_nas/__init__.py b/utils/third_party/nni/algorithms/nas/pytorch/classic_nas/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..ec3f5a4894a0460d4a0582a0d6cd43af9bed77e2 --- /dev/null +++ b/utils/third_party/nni/algorithms/nas/pytorch/classic_nas/__init__.py @@ -0,0 +1,4 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .mutator import get_and_apply_next_architecture diff --git a/utils/third_party/nni/algorithms/nas/pytorch/classic_nas/mutator.py b/utils/third_party/nni/algorithms/nas/pytorch/classic_nas/mutator.py new file mode 100644 index 0000000000000000000000000000000000000000..7254a8b0b4a34b5913dfc71d11f7364b86fc04b8 --- /dev/null +++ b/utils/third_party/nni/algorithms/nas/pytorch/classic_nas/mutator.py @@ -0,0 +1,221 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import json +import logging +import os +import sys + +import torch + +import nni +from nni.runtime.env_vars import trial_env_vars +from nni.nas.pytorch.mutables import LayerChoice, InputChoice, MutableScope +from nni.nas.pytorch.mutator import Mutator + +logger = logging.getLogger(__name__) + +NNI_GEN_SEARCH_SPACE = "NNI_GEN_SEARCH_SPACE" +LAYER_CHOICE = "layer_choice" +INPUT_CHOICE = "input_choice" + + +def get_and_apply_next_architecture(model): + """ + Wrapper of :class:`~nni.nas.pytorch.classic_nas.mutator.ClassicMutator` to make it more meaningful, + similar to ``get_next_parameter`` for HPO. + + It will generate search space based on ``model``. + If env ``NNI_GEN_SEARCH_SPACE`` exists, this is in dry run mode for + generating search space for the experiment. + If not, there are still two mode, one is nni experiment mode where users + use ``nnictl`` to start an experiment. The other is standalone mode + where users directly run the trial command, this mode chooses the first + one(s) for each LayerChoice and InputChoice. + + Parameters + ---------- + model : nn.Module + User's model with search space (e.g., LayerChoice, InputChoice) embedded in it. + """ + ClassicMutator(model) + + +class ClassicMutator(Mutator): + """ + This mutator is to apply the architecture chosen from tuner. + It implements the forward function of LayerChoice and InputChoice, + to only activate the chosen ones. + + Parameters + ---------- + model : nn.Module + User's model with search space (e.g., LayerChoice, InputChoice) embedded in it. + """ + + def __init__(self, model): + super(ClassicMutator, self).__init__(model) + self._chosen_arch = {} + self._search_space = self._generate_search_space() + if NNI_GEN_SEARCH_SPACE in os.environ: + # dry run for only generating search space + self._dump_search_space(os.environ[NNI_GEN_SEARCH_SPACE]) + sys.exit(0) + + if trial_env_vars.NNI_PLATFORM is None: + logger.warning("This is in standalone mode, the chosen are the first one(s).") + self._chosen_arch = self._standalone_generate_chosen() + else: + # get chosen arch from tuner + self._chosen_arch = nni.get_next_parameter() + if self._chosen_arch is None: + if trial_env_vars.NNI_PLATFORM == "unittest": + # happens if NNI_PLATFORM is intentionally set, e.g., in UT + logger.warning("`NNI_PLATFORM` is set but `param` is None. Falling back to standalone mode.") + self._chosen_arch = self._standalone_generate_chosen() + else: + raise RuntimeError("Chosen architecture is None. This may be a platform error.") + self.reset() + + def _sample_layer_choice(self, mutable, idx, value, search_space_item): + """ + Convert layer choice to tensor representation. + + Parameters + ---------- + mutable : Mutable + idx : int + Number `idx` of list will be selected. + value : str + The verbose representation of the selected value. + search_space_item : list + The list for corresponding search space. + """ + # doesn't support multihot for layer choice yet + onehot_list = [False] * len(mutable) + assert 0 <= idx < len(mutable) and search_space_item[idx] == value, \ + "Index '{}' in search space '{}' is not '{}'".format(idx, search_space_item, value) + onehot_list[idx] = True + return torch.tensor(onehot_list, dtype=torch.bool) # pylint: disable=not-callable + + def _sample_input_choice(self, mutable, idx, value, search_space_item): + """ + Convert input choice to tensor representation. + + Parameters + ---------- + mutable : Mutable + idx : int + Number `idx` of list will be selected. + value : str + The verbose representation of the selected value. + search_space_item : list + The list for corresponding search space. + """ + candidate_repr = search_space_item["candidates"] + multihot_list = [False] * mutable.n_candidates + for i, v in zip(idx, value): + assert 0 <= i < mutable.n_candidates and candidate_repr[i] == v, \ + "Index '{}' in search space '{}' is not '{}'".format(i, candidate_repr, v) + assert not multihot_list[i], "'{}' is selected twice in '{}', which is not allowed.".format(i, idx) + multihot_list[i] = True + return torch.tensor(multihot_list, dtype=torch.bool) # pylint: disable=not-callable + + def sample_search(self): + """ + See :meth:`sample_final`. + """ + return self.sample_final() + + def sample_final(self): + """ + Convert the chosen arch and apply it on model. + """ + assert set(self._chosen_arch.keys()) == set(self._search_space.keys()), \ + "Unmatched keys, expected keys '{}' from search space, found '{}'.".format(self._search_space.keys(), + self._chosen_arch.keys()) + result = dict() + for mutable in self.mutables: + if isinstance(mutable, (LayerChoice, InputChoice)): + assert mutable.key in self._chosen_arch, \ + "Expected '{}' in chosen arch, but not found.".format(mutable.key) + data = self._chosen_arch[mutable.key] + assert isinstance(data, dict) and "_value" in data and "_idx" in data, \ + "'{}' is not a valid choice.".format(data) + if isinstance(mutable, LayerChoice): + result[mutable.key] = self._sample_layer_choice(mutable, data["_idx"], data["_value"], + self._search_space[mutable.key]["_value"]) + elif isinstance(mutable, InputChoice): + result[mutable.key] = self._sample_input_choice(mutable, data["_idx"], data["_value"], + self._search_space[mutable.key]["_value"]) + elif isinstance(mutable, MutableScope): + logger.info("Mutable scope '%s' is skipped during parsing choices.", mutable.key) + else: + raise TypeError("Unsupported mutable type: '%s'." % type(mutable)) + return result + + def _standalone_generate_chosen(self): + """ + Generate the chosen architecture for standalone mode, + i.e., choose the first one(s) for LayerChoice and InputChoice. + :: + { key_name: {"_value": "conv1", + "_idx": 0} } + { key_name: {"_value": ["in1"], + "_idx": [0]} } + Returns + ------- + dict + the chosen architecture + """ + chosen_arch = {} + for key, val in self._search_space.items(): + if val["_type"] == LAYER_CHOICE: + choices = val["_value"] + chosen_arch[key] = {"_value": choices[0], "_idx": 0} + elif val["_type"] == INPUT_CHOICE: + choices = val["_value"]["candidates"] + n_chosen = val["_value"]["n_chosen"] + if n_chosen is None: + n_chosen = len(choices) + chosen_arch[key] = {"_value": choices[:n_chosen], "_idx": list(range(n_chosen))} + else: + raise ValueError("Unknown key '%s' and value '%s'." % (key, val)) + return chosen_arch + + def _generate_search_space(self): + """ + Generate search space from mutables. + Here is the search space format: + :: + { key_name: {"_type": "layer_choice", + "_value": ["conv1", "conv2"]} } + { key_name: {"_type": "input_choice", + "_value": {"candidates": ["in1", "in2"], + "n_chosen": 1}} } + Returns + ------- + dict + the generated search space + """ + search_space = {} + for mutable in self.mutables: + # for now we only generate flattened search space + if isinstance(mutable, LayerChoice): + key = mutable.key + val = mutable.names + search_space[key] = {"_type": LAYER_CHOICE, "_value": val} + elif isinstance(mutable, InputChoice): + key = mutable.key + search_space[key] = {"_type": INPUT_CHOICE, + "_value": {"candidates": mutable.choose_from, + "n_chosen": mutable.n_chosen}} + elif isinstance(mutable, MutableScope): + logger.info("Mutable scope '%s' is skipped during generating search space.", mutable.key) + else: + raise TypeError("Unsupported mutable type: '%s'." % type(mutable)) + return search_space + + def _dump_search_space(self, file_path): + with open(file_path, "w") as ss_file: + json.dump(self._search_space, ss_file, sort_keys=True, indent=2) diff --git a/utils/third_party/nni/algorithms/nas/pytorch/darts/__init__.py b/utils/third_party/nni/algorithms/nas/pytorch/darts/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..1a22790fb90b37591dd58f590a3b528b78b8257e --- /dev/null +++ b/utils/third_party/nni/algorithms/nas/pytorch/darts/__init__.py @@ -0,0 +1,5 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .mutator import DartsMutator +from .trainer import DartsTrainer diff --git a/utils/third_party/nni/algorithms/nas/pytorch/darts/mutator.py b/utils/third_party/nni/algorithms/nas/pytorch/darts/mutator.py new file mode 100644 index 0000000000000000000000000000000000000000..a4c3898a9b531ea160ea64a17d2e345f6b67d040 --- /dev/null +++ b/utils/third_party/nni/algorithms/nas/pytorch/darts/mutator.py @@ -0,0 +1,85 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from nni.nas.pytorch.mutator import Mutator +from nni.nas.pytorch.mutables import LayerChoice, InputChoice + +_logger = logging.getLogger(__name__) + + +class DartsMutator(Mutator): + """ + Connects the model in a DARTS (differentiable) way. + + An extra connection is automatically inserted for each LayerChoice, when this connection is selected, there is no + op on this LayerChoice (namely a ``ZeroOp``), in which case, every element in the exported choice list is ``false`` + (not chosen). + + All input choice will be fully connected in the search phase. On exporting, the input choice will choose inputs based + on keys in ``choose_from``. If the keys were to be keys of LayerChoices, the top logit of the corresponding LayerChoice + will join the competition of input choice to compete against other logits. Otherwise, the logit will be assumed 0. + + It's possible to cut branches by setting parameter ``choices`` in a particular position to ``-inf``. After softmax, the + value would be 0. Framework will ignore 0 values and not connect. Note that the gradient on the ``-inf`` location will + be 0. Since manipulations with ``-inf`` will be ``nan``, you need to handle the gradient update phase carefully. + + Attributes + ---------- + choices: ParameterDict + dict that maps keys of LayerChoices to weighted-connection float tensors. + """ + def __init__(self, model): + super().__init__(model) + self.choices = nn.ParameterDict() + for mutable in self.mutables: + if isinstance(mutable, LayerChoice): + self.choices[mutable.key] = nn.Parameter(1.0E-3 * torch.randn(mutable.length + 1)) + + def device(self): + for v in self.choices.values(): + return v.device + + def sample_search(self): + result = dict() + for mutable in self.mutables: + if isinstance(mutable, LayerChoice): + result[mutable.key] = F.softmax(self.choices[mutable.key], dim=-1)[:-1] + elif isinstance(mutable, InputChoice): + result[mutable.key] = torch.ones(mutable.n_candidates, dtype=torch.bool, device=self.device()) + return result + + def sample_final(self): + result = dict() + edges_max = dict() + for mutable in self.mutables: + if isinstance(mutable, LayerChoice): + max_val, index = torch.max(F.softmax(self.choices[mutable.key], dim=-1)[:-1], 0) + edges_max[mutable.key] = max_val + result[mutable.key] = F.one_hot(index, num_classes=len(mutable)).view(-1).bool() + for mutable in self.mutables: + if isinstance(mutable, InputChoice): + if mutable.n_chosen is not None: + weights = [] + for src_key in mutable.choose_from: + if src_key not in edges_max: + _logger.warning("InputChoice.NO_KEY in '%s' is weighted 0 when selecting inputs.", mutable.key) + weights.append(edges_max.get(src_key, 0.)) + weights = torch.tensor(weights) # pylint: disable=not-callable + _, topk_edge_indices = torch.topk(weights, mutable.n_chosen) + selected_multihot = [] + for i, src_key in enumerate(mutable.choose_from): + if i not in topk_edge_indices and src_key in result: + # If an edge is never selected, there is no need to calculate any op on this edge. + # This is to eliminate redundant calculation. + result[src_key] = torch.zeros_like(result[src_key]) + selected_multihot.append(i in topk_edge_indices) + result[mutable.key] = torch.tensor(selected_multihot, dtype=torch.bool, device=self.device()) # pylint: disable=not-callable + else: + result[mutable.key] = torch.ones(mutable.n_candidates, dtype=torch.bool, device=self.device()) # pylint: disable=not-callable + return result diff --git a/utils/third_party/nni/algorithms/nas/pytorch/darts/trainer.py b/utils/third_party/nni/algorithms/nas/pytorch/darts/trainer.py new file mode 100644 index 0000000000000000000000000000000000000000..3873035415d8bdbe5f0f73b2f2fa958d36b470a5 --- /dev/null +++ b/utils/third_party/nni/algorithms/nas/pytorch/darts/trainer.py @@ -0,0 +1,214 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import copy +import logging + +import torch +import torch.nn as nn +from nni.nas.pytorch.trainer import Trainer +from nni.nas.pytorch.utils import AverageMeterGroup + +from .mutator import DartsMutator + +logger = logging.getLogger(__name__) + + +class DartsTrainer(Trainer): + """ + DARTS trainer. + + Parameters + ---------- + model : nn.Module + PyTorch model to be trained. + loss : callable + Receives logits and ground truth label, return a loss tensor. + metrics : callable + Receives logits and ground truth label, return a dict of metrics. + optimizer : Optimizer + The optimizer used for optimizing the model. + num_epochs : int + Number of epochs planned for training. + dataset_train : Dataset + Dataset for training. Will be split for training weights and architecture weights. + dataset_valid : Dataset + Dataset for testing. + mutator : DartsMutator + Use in case of customizing your own DartsMutator. By default will instantiate a DartsMutator. + batch_size : int + Batch size. + workers : int + Workers for data loading. + device : torch.device + ``torch.device("cpu")`` or ``torch.device("cuda")``. + log_frequency : int + Step count per logging. + callbacks : list of Callback + list of callbacks to trigger at events. + arc_learning_rate : float + Learning rate of architecture parameters. + unrolled : float + ``True`` if using second order optimization, else first order optimization. + """ + def __init__(self, model, loss, metrics, + optimizer, num_epochs, dataset_train, dataset_valid, + mutator=None, batch_size=64, workers=4, device=None, log_frequency=None, + callbacks=None, arc_learning_rate=3.0E-4, unrolled=False): + super().__init__(model, mutator if mutator is not None else DartsMutator(model), + loss, metrics, optimizer, num_epochs, dataset_train, dataset_valid, + batch_size, workers, device, log_frequency, callbacks) + + self.ctrl_optim = torch.optim.Adam(self.mutator.parameters(), arc_learning_rate, betas=(0.5, 0.999), + weight_decay=1.0E-3) + self.unrolled = unrolled + + n_train = len(self.dataset_train) + split = n_train // 2 + indices = list(range(n_train)) + train_sampler = torch.utils.data.sampler.SubsetRandomSampler(indices[:split]) + valid_sampler = torch.utils.data.sampler.SubsetRandomSampler(indices[split:]) + self.train_loader = torch.utils.data.DataLoader(self.dataset_train, + batch_size=batch_size, + sampler=train_sampler, + num_workers=workers) + self.valid_loader = torch.utils.data.DataLoader(self.dataset_train, + batch_size=batch_size, + sampler=valid_sampler, + num_workers=workers) + self.test_loader = torch.utils.data.DataLoader(self.dataset_valid, + batch_size=batch_size, + num_workers=workers) + + def train_one_epoch(self, epoch): + self.model.train() + self.mutator.train() + meters = AverageMeterGroup() + for step, ((trn_X, trn_y), (val_X, val_y)) in enumerate(zip(self.train_loader, self.valid_loader)): + trn_X, trn_y = trn_X.to(self.device), trn_y.to(self.device) + val_X, val_y = val_X.to(self.device), val_y.to(self.device) + + # phase 1. architecture step + self.ctrl_optim.zero_grad() + if self.unrolled: + self._unrolled_backward(trn_X, trn_y, val_X, val_y) + else: + self._backward(val_X, val_y) + self.ctrl_optim.step() + + # phase 2: child network step + self.optimizer.zero_grad() + logits, loss = self._logits_and_loss(trn_X, trn_y) + loss.backward() + nn.utils.clip_grad_norm_(self.model.parameters(), 5.) # gradient clipping + self.optimizer.step() + + metrics = self.metrics(logits, trn_y) + metrics["loss"] = loss.item() + meters.update(metrics) + if self.log_frequency is not None and step % self.log_frequency == 0: + logger.info("Epoch [%s/%s] Step [%s/%s] %s", epoch + 1, + self.num_epochs, step + 1, len(self.train_loader), meters) + + def validate_one_epoch(self, epoch): + self.model.eval() + self.mutator.eval() + meters = AverageMeterGroup() + with torch.no_grad(): + self.mutator.reset() + for step, (X, y) in enumerate(self.test_loader): + X, y = X.to(self.device), y.to(self.device) + logits = self.model(X) + metrics = self.metrics(logits, y) + meters.update(metrics) + if self.log_frequency is not None and step % self.log_frequency == 0: + logger.info("Epoch [%s/%s] Step [%s/%s] %s", epoch + 1, + self.num_epochs, step + 1, len(self.test_loader), meters) + + def _logits_and_loss(self, X, y): + self.mutator.reset() + logits = self.model(X) + loss = self.loss(logits, y) + self._write_graph_status() + return logits, loss + + def _backward(self, val_X, val_y): + """ + Simple backward with gradient descent + """ + _, loss = self._logits_and_loss(val_X, val_y) + loss.backward() + + def _unrolled_backward(self, trn_X, trn_y, val_X, val_y): + """ + Compute unrolled loss and backward its gradients + """ + backup_params = copy.deepcopy(tuple(self.model.parameters())) + + # do virtual step on training data + lr = self.optimizer.param_groups[0]["lr"] + momentum = self.optimizer.param_groups[0]["momentum"] + weight_decay = self.optimizer.param_groups[0]["weight_decay"] + self._compute_virtual_model(trn_X, trn_y, lr, momentum, weight_decay) + + # calculate unrolled loss on validation data + # keep gradients for model here for compute hessian + _, loss = self._logits_and_loss(val_X, val_y) + w_model, w_ctrl = tuple(self.model.parameters()), tuple(self.mutator.parameters()) + w_grads = torch.autograd.grad(loss, w_model + w_ctrl) + d_model, d_ctrl = w_grads[:len(w_model)], w_grads[len(w_model):] + + # compute hessian and final gradients + hessian = self._compute_hessian(backup_params, d_model, trn_X, trn_y) + with torch.no_grad(): + for param, d, h in zip(w_ctrl, d_ctrl, hessian): + # gradient = dalpha - lr * hessian + param.grad = d - lr * h + + # restore weights + self._restore_weights(backup_params) + + def _compute_virtual_model(self, X, y, lr, momentum, weight_decay): + """ + Compute unrolled weights w` + """ + # don't need zero_grad, using autograd to calculate gradients + _, loss = self._logits_and_loss(X, y) + gradients = torch.autograd.grad(loss, self.model.parameters()) + with torch.no_grad(): + for w, g in zip(self.model.parameters(), gradients): + m = self.optimizer.state[w].get("momentum_buffer", 0.) + w = w - lr * (momentum * m + g + weight_decay * w) + + def _restore_weights(self, backup_params): + with torch.no_grad(): + for param, backup in zip(self.model.parameters(), backup_params): + param.copy_(backup) + + def _compute_hessian(self, backup_params, dw, trn_X, trn_y): + """ + dw = dw` { L_val(w`, alpha) } + w+ = w + eps * dw + w- = w - eps * dw + hessian = (dalpha { L_trn(w+, alpha) } - dalpha { L_trn(w-, alpha) }) / (2*eps) + eps = 0.01 / ||dw|| + """ + self._restore_weights(backup_params) + norm = torch.cat([w.view(-1) for w in dw]).norm() + eps = 0.01 / norm + if norm < 1E-8: + logger.warning("In computing hessian, norm is smaller than 1E-8, cause eps to be %.6f.", norm.item()) + + dalphas = [] + for e in [eps, -2. * eps]: + # w+ = w + eps*dw`, w- = w - eps*dw` + with torch.no_grad(): + for p, d in zip(self.model.parameters(), dw): + p += e * d + + _, loss = self._logits_and_loss(trn_X, trn_y) + dalphas.append(torch.autograd.grad(loss, self.mutator.parameters())) + + dalpha_pos, dalpha_neg = dalphas # dalpha { L_trn(w+) }, # dalpha { L_trn(w-) } + hessian = [(p - n) / 2. * eps for p, n in zip(dalpha_pos, dalpha_neg)] + return hessian diff --git a/utils/third_party/nni/algorithms/nas/pytorch/enas/__init__.py b/utils/third_party/nni/algorithms/nas/pytorch/enas/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d3372836ebba2387ade58161218aad731433e46b --- /dev/null +++ b/utils/third_party/nni/algorithms/nas/pytorch/enas/__init__.py @@ -0,0 +1,5 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .mutator import EnasMutator +from .trainer import EnasTrainer diff --git a/utils/third_party/nni/algorithms/nas/pytorch/enas/mutator.py b/utils/third_party/nni/algorithms/nas/pytorch/enas/mutator.py new file mode 100644 index 0000000000000000000000000000000000000000..7fdba26b99bf72551293fdb77ff61beb974bba8f --- /dev/null +++ b/utils/third_party/nni/algorithms/nas/pytorch/enas/mutator.py @@ -0,0 +1,197 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from nni.nas.pytorch.mutator import Mutator +from nni.nas.pytorch.mutables import LayerChoice, InputChoice, MutableScope + + +class StackedLSTMCell(nn.Module): + def __init__(self, layers, size, bias): + super().__init__() + self.lstm_num_layers = layers + self.lstm_modules = nn.ModuleList([nn.LSTMCell(size, size, bias=bias) + for _ in range(self.lstm_num_layers)]) + + def forward(self, inputs, hidden): + prev_h, prev_c = hidden + next_h, next_c = [], [] + for i, m in enumerate(self.lstm_modules): + curr_h, curr_c = m(inputs, (prev_h[i], prev_c[i])) + next_c.append(curr_c) + next_h.append(curr_h) + # current implementation only supports batch size equals 1, + # but the algorithm does not necessarily have this limitation + inputs = curr_h[-1].view(1, -1) + return next_h, next_c + + +class EnasMutator(Mutator): + """ + A mutator that mutates the graph with RL. + + Parameters + ---------- + model : nn.Module + PyTorch model. + lstm_size : int + Controller LSTM hidden units. + lstm_num_layers : int + Number of layers for stacked LSTM. + tanh_constant : float + Logits will be equal to ``tanh_constant * tanh(logits)``. Don't use ``tanh`` if this value is ``None``. + cell_exit_extra_step : bool + If true, RL controller will perform an extra step at the exit of each MutableScope, dump the hidden state + and mark it as the hidden state of this MutableScope. This is to align with the original implementation of paper. + skip_target : float + Target probability that skipconnect will appear. + temperature : float + Temperature constant that divides the logits. + branch_bias : float + Manual bias applied to make some operations more likely to be chosen. + Currently this is implemented with a hardcoded match rule that aligns with original repo. + If a mutable has a ``reduce`` in its key, all its op choices + that contains `conv` in their typename will receive a bias of ``+self.branch_bias`` initially; while others + receive a bias of ``-self.branch_bias``. + entropy_reduction : str + Can be one of ``sum`` and ``mean``. How the entropy of multi-input-choice is reduced. + """ + + def __init__(self, model, lstm_size=64, lstm_num_layers=1, tanh_constant=1.5, cell_exit_extra_step=False, + skip_target=0.4, temperature=None, branch_bias=0.25, entropy_reduction="sum"): + super().__init__(model) + self.lstm_size = lstm_size + self.lstm_num_layers = lstm_num_layers + self.tanh_constant = tanh_constant + self.temperature = temperature + self.cell_exit_extra_step = cell_exit_extra_step + self.skip_target = skip_target + self.branch_bias = branch_bias + + self.lstm = StackedLSTMCell(self.lstm_num_layers, self.lstm_size, False) + self.attn_anchor = nn.Linear(self.lstm_size, self.lstm_size, bias=False) + self.attn_query = nn.Linear(self.lstm_size, self.lstm_size, bias=False) + self.v_attn = nn.Linear(self.lstm_size, 1, bias=False) + self.g_emb = nn.Parameter(torch.randn(1, self.lstm_size) * 0.1) + self.skip_targets = nn.Parameter(torch.tensor([1.0 - self.skip_target, self.skip_target]), requires_grad=False) # pylint: disable=not-callable + assert entropy_reduction in ["sum", "mean"], "Entropy reduction must be one of sum and mean." + self.entropy_reduction = torch.sum if entropy_reduction == "sum" else torch.mean + self.cross_entropy_loss = nn.CrossEntropyLoss(reduction="none") + self.bias_dict = nn.ParameterDict() + + self.max_layer_choice = 0 + for mutable in self.mutables: + if isinstance(mutable, LayerChoice): + if self.max_layer_choice == 0: + self.max_layer_choice = len(mutable) + assert self.max_layer_choice == len(mutable), \ + "ENAS mutator requires all layer choice have the same number of candidates." + # We are judging by keys and module types to add biases to layer choices. Needs refactor. + if "reduce" in mutable.key: + def is_conv(choice): + return "conv" in str(type(choice)).lower() + bias = torch.tensor([self.branch_bias if is_conv(choice) else -self.branch_bias # pylint: disable=not-callable + for choice in mutable]) + self.bias_dict[mutable.key] = nn.Parameter(bias, requires_grad=False) + + self.embedding = nn.Embedding(self.max_layer_choice + 1, self.lstm_size) + self.soft = nn.Linear(self.lstm_size, self.max_layer_choice, bias=False) + + def sample_search(self): + self._initialize() + self._sample(self.mutables) + return self._choices + + def sample_final(self): + return self.sample_search() + + def _sample(self, tree): + mutable = tree.mutable + if isinstance(mutable, LayerChoice) and mutable.key not in self._choices: + self._choices[mutable.key] = self._sample_layer_choice(mutable) + elif isinstance(mutable, InputChoice) and mutable.key not in self._choices: + self._choices[mutable.key] = self._sample_input_choice(mutable) + for child in tree.children: + self._sample(child) + if isinstance(mutable, MutableScope) and mutable.key not in self._anchors_hid: + if self.cell_exit_extra_step: + self._lstm_next_step() + self._mark_anchor(mutable.key) + + def _initialize(self): + self._choices = dict() + self._anchors_hid = dict() + self._inputs = self.g_emb.data + self._c = [torch.zeros((1, self.lstm_size), + dtype=self._inputs.dtype, + device=self._inputs.device) for _ in range(self.lstm_num_layers)] + self._h = [torch.zeros((1, self.lstm_size), + dtype=self._inputs.dtype, + device=self._inputs.device) for _ in range(self.lstm_num_layers)] + self.sample_log_prob = 0 + self.sample_entropy = 0 + self.sample_skip_penalty = 0 + + def _lstm_next_step(self): + self._h, self._c = self.lstm(self._inputs, (self._h, self._c)) + + def _mark_anchor(self, key): + self._anchors_hid[key] = self._h[-1] + + def _sample_layer_choice(self, mutable): + self._lstm_next_step() + logit = self.soft(self._h[-1]) + if self.temperature is not None: + logit /= self.temperature + if self.tanh_constant is not None: + logit = self.tanh_constant * torch.tanh(logit) + if mutable.key in self.bias_dict: + logit += self.bias_dict[mutable.key] + branch_id = torch.multinomial(F.softmax(logit, dim=-1), 1).view(-1) + log_prob = self.cross_entropy_loss(logit, branch_id) + self.sample_log_prob += self.entropy_reduction(log_prob) + entropy = (log_prob * torch.exp(-log_prob)).detach() # pylint: disable=invalid-unary-operand-type + self.sample_entropy += self.entropy_reduction(entropy) + self._inputs = self.embedding(branch_id) + return F.one_hot(branch_id, num_classes=self.max_layer_choice).bool().view(-1) + + def _sample_input_choice(self, mutable): + query, anchors = [], [] + for label in mutable.choose_from: + if label not in self._anchors_hid: + self._lstm_next_step() + self._mark_anchor(label) # empty loop, fill not found + query.append(self.attn_anchor(self._anchors_hid[label])) + anchors.append(self._anchors_hid[label]) + query = torch.cat(query, 0) + query = torch.tanh(query + self.attn_query(self._h[-1])) + query = self.v_attn(query) + if self.temperature is not None: + query /= self.temperature + if self.tanh_constant is not None: + query = self.tanh_constant * torch.tanh(query) + + if mutable.n_chosen is None: + logit = torch.cat([-query, query], 1) # pylint: disable=invalid-unary-operand-type + + skip = torch.multinomial(F.softmax(logit, dim=-1), 1).view(-1) + skip_prob = torch.sigmoid(logit) + kl = torch.sum(skip_prob * torch.log(skip_prob / self.skip_targets)) + self.sample_skip_penalty += kl + log_prob = self.cross_entropy_loss(logit, skip) + self._inputs = (torch.matmul(skip.float(), torch.cat(anchors, 0)) / (1. + torch.sum(skip))).unsqueeze(0) + else: + assert mutable.n_chosen == 1, "Input choice must select exactly one or any in ENAS." + logit = query.view(1, -1) + index = torch.multinomial(F.softmax(logit, dim=-1), 1).view(-1) + skip = F.one_hot(index, num_classes=mutable.n_candidates).view(-1) + log_prob = self.cross_entropy_loss(logit, index) + self._inputs = anchors[index.item()] + + self.sample_log_prob += self.entropy_reduction(log_prob) + entropy = (log_prob * torch.exp(-log_prob)).detach() # pylint: disable=invalid-unary-operand-type + self.sample_entropy += self.entropy_reduction(entropy) + return skip.bool() diff --git a/utils/third_party/nni/algorithms/nas/pytorch/enas/trainer.py b/utils/third_party/nni/algorithms/nas/pytorch/enas/trainer.py new file mode 100644 index 0000000000000000000000000000000000000000..5e7a966580a107ceb6b9fe88f888dafefb8b69e7 --- /dev/null +++ b/utils/third_party/nni/algorithms/nas/pytorch/enas/trainer.py @@ -0,0 +1,209 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +from itertools import cycle + +import torch +import torch.nn as nn +import torch.optim as optim + +from nni.nas.pytorch.trainer import Trainer +from nni.nas.pytorch.utils import AverageMeterGroup, to_device +from .mutator import EnasMutator + +logger = logging.getLogger(__name__) + + +class EnasTrainer(Trainer): + """ + ENAS trainer. + + Parameters + ---------- + model : nn.Module + PyTorch model to be trained. + loss : callable + Receives logits and ground truth label, return a loss tensor. + metrics : callable + Receives logits and ground truth label, return a dict of metrics. + reward_function : callable + Receives logits and ground truth label, return a tensor, which will be feeded to RL controller as reward. + optimizer : Optimizer + The optimizer used for optimizing the model. + num_epochs : int + Number of epochs planned for training. + dataset_train : Dataset + Dataset for training. Will be split for training weights and architecture weights. + dataset_valid : Dataset + Dataset for testing. + mutator : EnasMutator + Use when customizing your own mutator or a mutator with customized parameters. + batch_size : int + Batch size. + workers : int + Workers for data loading. + device : torch.device + ``torch.device("cpu")`` or ``torch.device("cuda")``. + log_frequency : int + Step count per logging. + callbacks : list of Callback + list of callbacks to trigger at events. + entropy_weight : float + Weight of sample entropy loss. + skip_weight : float + Weight of skip penalty loss. + baseline_decay : float + Decay factor of baseline. New baseline will be equal to ``baseline_decay * baseline_old + reward * (1 - baseline_decay)``. + child_steps : int + How many mini-batches for model training per epoch. + mutator_lr : float + Learning rate for RL controller. + mutator_steps_aggregate : int + Number of steps that will be aggregated into one mini-batch for RL controller. + mutator_steps : int + Number of mini-batches for each epoch of RL controller learning. + aux_weight : float + Weight of auxiliary head loss. ``aux_weight * aux_loss`` will be added to total loss. + test_arc_per_epoch : int + How many architectures are chosen for direct test after each epoch. + """ + def __init__(self, model, loss, metrics, reward_function, + optimizer, num_epochs, dataset_train, dataset_valid, + mutator=None, batch_size=64, workers=4, device=None, log_frequency=None, callbacks=None, + entropy_weight=0.0001, skip_weight=0.8, baseline_decay=0.999, child_steps=500, + mutator_lr=0.00035, mutator_steps_aggregate=20, mutator_steps=50, aux_weight=0.4, + test_arc_per_epoch=1): + super().__init__(model, mutator if mutator is not None else EnasMutator(model), + loss, metrics, optimizer, num_epochs, dataset_train, dataset_valid, + batch_size, workers, device, log_frequency, callbacks) + self.reward_function = reward_function + self.mutator_optim = optim.Adam(self.mutator.parameters(), lr=mutator_lr) + self.batch_size = batch_size + self.workers = workers + + self.entropy_weight = entropy_weight + self.skip_weight = skip_weight + self.baseline_decay = baseline_decay + self.baseline = 0. + self.mutator_steps_aggregate = mutator_steps_aggregate + self.mutator_steps = mutator_steps + self.child_steps = child_steps + self.aux_weight = aux_weight + self.test_arc_per_epoch = test_arc_per_epoch + + self.init_dataloader() + + def init_dataloader(self): + n_train = len(self.dataset_train) + split = n_train // 10 + indices = list(range(n_train)) + train_sampler = torch.utils.data.sampler.SubsetRandomSampler(indices[:-split]) + valid_sampler = torch.utils.data.sampler.SubsetRandomSampler(indices[-split:]) + self.train_loader = torch.utils.data.DataLoader(self.dataset_train, + batch_size=self.batch_size, + sampler=train_sampler, + num_workers=self.workers) + self.valid_loader = torch.utils.data.DataLoader(self.dataset_train, + batch_size=self.batch_size, + sampler=valid_sampler, + num_workers=self.workers) + self.test_loader = torch.utils.data.DataLoader(self.dataset_valid, + batch_size=self.batch_size, + num_workers=self.workers) + self.train_loader = cycle(self.train_loader) + self.valid_loader = cycle(self.valid_loader) + + def train_one_epoch(self, epoch): + # Sample model and train + self.model.train() + self.mutator.eval() + meters = AverageMeterGroup() + for step in range(1, self.child_steps + 1): + x, y = next(self.train_loader) + x, y = to_device(x, self.device), to_device(y, self.device) + self.optimizer.zero_grad() + + with torch.no_grad(): + self.mutator.reset() + self._write_graph_status() + logits = self.model(x) + + if isinstance(logits, tuple): + logits, aux_logits = logits + aux_loss = self.loss(aux_logits, y) + else: + aux_loss = 0. + metrics = self.metrics(logits, y) + loss = self.loss(logits, y) + loss = loss + self.aux_weight * aux_loss + loss.backward() + nn.utils.clip_grad_norm_(self.model.parameters(), 5.) + self.optimizer.step() + metrics["loss"] = loss.item() + meters.update(metrics) + + if self.log_frequency is not None and step % self.log_frequency == 0: + logger.info("Model Epoch [%d/%d] Step [%d/%d] %s", epoch + 1, + self.num_epochs, step, self.child_steps, meters) + + # Train sampler (mutator) + self.model.eval() + self.mutator.train() + meters = AverageMeterGroup() + for mutator_step in range(1, self.mutator_steps + 1): + self.mutator_optim.zero_grad() + for step in range(1, self.mutator_steps_aggregate + 1): + x, y = next(self.valid_loader) + x, y = to_device(x, self.device), to_device(y, self.device) + + self.mutator.reset() + with torch.no_grad(): + logits = self.model(x) + self._write_graph_status() + metrics = self.metrics(logits, y) + reward = self.reward_function(logits, y) + if self.entropy_weight: + reward += self.entropy_weight * self.mutator.sample_entropy.item() + self.baseline = self.baseline * self.baseline_decay + reward * (1 - self.baseline_decay) + loss = self.mutator.sample_log_prob * (reward - self.baseline) + if self.skip_weight: + loss += self.skip_weight * self.mutator.sample_skip_penalty + metrics["reward"] = reward + metrics["loss"] = loss.item() + metrics["ent"] = self.mutator.sample_entropy.item() + metrics["log_prob"] = self.mutator.sample_log_prob.item() + metrics["baseline"] = self.baseline + metrics["skip"] = self.mutator.sample_skip_penalty + + loss /= self.mutator_steps_aggregate + loss.backward() + meters.update(metrics) + + cur_step = step + (mutator_step - 1) * self.mutator_steps_aggregate + if self.log_frequency is not None and cur_step % self.log_frequency == 0: + logger.info("RL Epoch [%d/%d] Step [%d/%d] [%d/%d] %s", epoch + 1, self.num_epochs, + mutator_step, self.mutator_steps, step, self.mutator_steps_aggregate, + meters) + + nn.utils.clip_grad_norm_(self.mutator.parameters(), 5.) + self.mutator_optim.step() + + def validate_one_epoch(self, epoch): + with torch.no_grad(): + for arc_id in range(self.test_arc_per_epoch): + meters = AverageMeterGroup() + for x, y in self.test_loader: + x, y = to_device(x, self.device), to_device(y, self.device) + self.mutator.reset() + logits = self.model(x) + if isinstance(logits, tuple): + logits, _ = logits + metrics = self.metrics(logits, y) + loss = self.loss(logits, y) + metrics["loss"] = loss.item() + meters.update(metrics) + + logger.info("Test Epoch [%d/%d] Arc [%d/%d] Summary %s", + epoch + 1, self.num_epochs, arc_id + 1, self.test_arc_per_epoch, + meters.summary()) diff --git a/utils/third_party/nni/algorithms/nas/pytorch/pdarts/__init__.py b/utils/third_party/nni/algorithms/nas/pytorch/pdarts/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d1d17764ba159b35bcc38efa82a2a30dc2366b76 --- /dev/null +++ b/utils/third_party/nni/algorithms/nas/pytorch/pdarts/__init__.py @@ -0,0 +1,4 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .trainer import PdartsTrainer diff --git a/utils/third_party/nni/algorithms/nas/pytorch/pdarts/mutator.py b/utils/third_party/nni/algorithms/nas/pytorch/pdarts/mutator.py new file mode 100644 index 0000000000000000000000000000000000000000..09ad51c5e471b4acae51d513335682328924b90a --- /dev/null +++ b/utils/third_party/nni/algorithms/nas/pytorch/pdarts/mutator.py @@ -0,0 +1,93 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import copy + +import numpy as np +import torch +from torch import nn + +from nni.algorithms.nas.pytorch.darts import DartsMutator +from nni.nas.pytorch.mutables import LayerChoice + + +class PdartsMutator(DartsMutator): + """ + It works with PdartsTrainer to calculate ops weights, + and drop weights in different PDARTS epochs. + """ + + def __init__(self, model, pdarts_epoch_index, pdarts_num_to_drop, switches={}): + self.pdarts_epoch_index = pdarts_epoch_index + self.pdarts_num_to_drop = pdarts_num_to_drop + if switches is None: + self.switches = {} + else: + self.switches = switches + + super(PdartsMutator, self).__init__(model) + + # this loop go through mutables with different keys, + # it's mainly to update length of choices. + for mutable in self.mutables: + if isinstance(mutable, LayerChoice): + + switches = self.switches.get(mutable.key, [True for j in range(len(mutable))]) + choices = self.choices[mutable.key] + + operations_count = np.sum(switches) + # +1 and -1 are caused by zero operation in darts network + # the zero operation is not in choices list in network, but its weight are in, + # so it needs one more weights and switch for zero. + self.choices[mutable.key] = nn.Parameter(1.0E-3 * torch.randn(operations_count + 1)) + self.switches[mutable.key] = switches + + # update LayerChoice instances in model, + # it's physically remove dropped choices operations. + for module in self.model.modules(): + if isinstance(module, LayerChoice): + switches = self.switches.get(module.key) + choices = self.choices[module.key] + if len(module) > len(choices): + # from last to first, so that it won't effect previous indexes after removed one. + for index in range(len(switches)-1, -1, -1): + if switches[index] == False: + del module[index] + assert len(module) <= len(choices), "Failed to remove dropped choices." + + def export(self): + # Cannot rely on super().export() because P-DARTS has deleted some of the choices and has misaligned length. + results = super().sample_final() + for mutable in self.mutables: + if isinstance(mutable, LayerChoice): + # As some operations are dropped physically, + # so it needs to fill back false to track dropped operations. + trained_result = results[mutable.key] + trained_index = 0 + switches = self.switches[mutable.key] + result = torch.Tensor(switches).bool() + for index in range(len(result)): + if result[index]: + result[index] = trained_result[trained_index] + trained_index += 1 + results[mutable.key] = result + return results + + def drop_paths(self): + """ + This method is called when a PDARTS epoch is finished. + It prepares switches for next epoch. + candidate operations with False switch will be doppped in next epoch. + """ + all_switches = copy.deepcopy(self.switches) + for key in all_switches: + switches = all_switches[key] + idxs = [] + for j in range(len(switches)): + if switches[j]: + idxs.append(j) + sorted_weights = self.choices[key].data.cpu().numpy()[:-1] + drop = np.argsort(sorted_weights)[:self.pdarts_num_to_drop[self.pdarts_epoch_index]] + for idx in drop: + switches[idxs[idx]] = False + return all_switches diff --git a/utils/third_party/nni/algorithms/nas/pytorch/pdarts/trainer.py b/utils/third_party/nni/algorithms/nas/pytorch/pdarts/trainer.py new file mode 100644 index 0000000000000000000000000000000000000000..7f23a6e222731ae6edc5c034f77eafaaf71b4c5e --- /dev/null +++ b/utils/third_party/nni/algorithms/nas/pytorch/pdarts/trainer.py @@ -0,0 +1,86 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import json +import logging + +from nni.nas.pytorch.callbacks import LRSchedulerCallback +from nni.algorithms.nas.pytorch.darts import DartsTrainer +from nni.nas.pytorch.trainer import BaseTrainer, TorchTensorEncoder + +from .mutator import PdartsMutator + +logger = logging.getLogger(__name__) + + +class PdartsTrainer(BaseTrainer): + """ + This trainer implements the PDARTS algorithm. + PDARTS bases on DARTS algorithm, and provides a network growth approach to find deeper and better network. + This class relies on pdarts_num_layers and pdarts_num_to_drop parameters to control how network grows. + pdarts_num_layers means how many layers more than first epoch. + pdarts_num_to_drop means how many candidate operations should be dropped in each epoch. + So that the grew network can in similar size. + """ + + def __init__(self, model_creator, init_layers, metrics, + num_epochs, dataset_train, dataset_valid, + pdarts_num_layers=[0, 6, 12], pdarts_num_to_drop=[3, 2, 1], + mutator=None, batch_size=64, workers=4, device=None, log_frequency=None, callbacks=None, unrolled=False): + super(PdartsTrainer, self).__init__() + self.model_creator = model_creator + self.init_layers = init_layers + self.pdarts_num_layers = pdarts_num_layers + self.pdarts_num_to_drop = pdarts_num_to_drop + self.pdarts_epoch = len(pdarts_num_to_drop) + self.darts_parameters = { + "metrics": metrics, + "num_epochs": num_epochs, + "dataset_train": dataset_train, + "dataset_valid": dataset_valid, + "batch_size": batch_size, + "workers": workers, + "device": device, + "log_frequency": log_frequency, + "unrolled": unrolled + } + self.callbacks = callbacks if callbacks is not None else [] + + def train(self): + + switches = None + for epoch in range(self.pdarts_epoch): + + layers = self.init_layers+self.pdarts_num_layers[epoch] + model, criterion, optim, lr_scheduler = self.model_creator(layers) + self.mutator = PdartsMutator(model, epoch, self.pdarts_num_to_drop, switches) + + for callback in self.callbacks: + callback.build(model, self.mutator, self) + callback.on_epoch_begin(epoch) + + darts_callbacks = [] + if lr_scheduler is not None: + darts_callbacks.append(LRSchedulerCallback(lr_scheduler)) + + self.trainer = DartsTrainer(model, mutator=self.mutator, loss=criterion, optimizer=optim, + callbacks=darts_callbacks, **self.darts_parameters) + logger.info("start pdarts training epoch %s...", epoch) + + self.trainer.train() + + switches = self.mutator.drop_paths() + + for callback in self.callbacks: + callback.on_epoch_end(epoch) + + def validate(self): + self.trainer.validate() + + def export(self, file): + mutator_export = self.mutator.export() + with open(file, "w") as f: + json.dump(mutator_export, f, indent=2, sort_keys=True, cls=TorchTensorEncoder) + + def checkpoint(self): + raise NotImplementedError("Not implemented yet") diff --git a/utils/third_party/nni/algorithms/nas/pytorch/proxylessnas/__init__.py b/utils/third_party/nni/algorithms/nas/pytorch/proxylessnas/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..26feedba7d553c32d61ea9139620a68fca7c12d0 --- /dev/null +++ b/utils/third_party/nni/algorithms/nas/pytorch/proxylessnas/__init__.py @@ -0,0 +1,2 @@ +from .mutator import ProxylessNasMutator +from .trainer import ProxylessNasTrainer diff --git a/utils/third_party/nni/algorithms/nas/pytorch/proxylessnas/mutator.py b/utils/third_party/nni/algorithms/nas/pytorch/proxylessnas/mutator.py new file mode 100644 index 0000000000000000000000000000000000000000..881a6b44038d5d6f2c97b2b106036954401a9cd3 --- /dev/null +++ b/utils/third_party/nni/algorithms/nas/pytorch/proxylessnas/mutator.py @@ -0,0 +1,478 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import math +import torch +from torch import nn as nn +from torch.nn import functional as F +import numpy as np + +from nni.nas.pytorch.base_mutator import BaseMutator +from nni.nas.pytorch.mutables import LayerChoice +from .utils import detach_variable + +class ArchGradientFunction(torch.autograd.Function): + @staticmethod + def forward(ctx, x, binary_gates, run_func, backward_func): + ctx.run_func = run_func + ctx.backward_func = backward_func + + detached_x = detach_variable(x) + with torch.enable_grad(): + output = run_func(detached_x) + ctx.save_for_backward(detached_x, output) + return output.data + + @staticmethod + def backward(ctx, grad_output): + detached_x, output = ctx.saved_tensors + + grad_x = torch.autograd.grad(output, detached_x, grad_output, only_inputs=True) + # compute gradients w.r.t. binary_gates + binary_grads = ctx.backward_func(detached_x.data, output.data, grad_output.data) + + return grad_x[0], binary_grads, None, None + +class MixedOp(nn.Module): + """ + This class is to instantiate and manage info of one LayerChoice. + It includes architecture weights, binary weights, and member functions + operating the weights. + + forward_mode: + forward/backward mode for LayerChoice: None, two, full, and full_v2. + For training architecture weights, we use full_v2 by default, and for training + model weights, we use None. + """ + forward_mode = None + def __init__(self, mutable): + """ + Parameters + ---------- + mutable : LayerChoice + A LayerChoice in user model + """ + super(MixedOp, self).__init__() + self.ap_path_alpha = nn.Parameter(torch.Tensor(len(mutable))) + self.ap_path_wb = nn.Parameter(torch.Tensor(len(mutable))) + self.ap_path_alpha.requires_grad = False + self.ap_path_wb.requires_grad = False + self.active_index = [0] + self.inactive_index = None + self.log_prob = None + self.current_prob_over_ops = None + self.n_choices = len(mutable) + + def get_ap_path_alpha(self): + return self.ap_path_alpha + + def to_requires_grad(self): + self.ap_path_alpha.requires_grad = True + self.ap_path_wb.requires_grad = True + + def to_disable_grad(self): + self.ap_path_alpha.requires_grad = False + self.ap_path_wb.requires_grad = False + + def forward(self, mutable, x): + """ + Define forward of LayerChoice. For 'full_v2', backward is also defined. + The 'two' mode is explained in section 3.2.1 in the paper. + The 'full_v2' mode is explained in Appendix D in the paper. + + Parameters + ---------- + mutable : LayerChoice + this layer's mutable + x : tensor + inputs of this layer, only support one input + + Returns + ------- + output: tensor + output of this layer + """ + if MixedOp.forward_mode == 'full' or MixedOp.forward_mode == 'two': + output = 0 + for _i in self.active_index: + oi = self.candidate_ops[_i](x) + output = output + self.ap_path_wb[_i] * oi + for _i in self.inactive_index: + oi = self.candidate_ops[_i](x) + output = output + self.ap_path_wb[_i] * oi.detach() + elif MixedOp.forward_mode == 'full_v2': + def run_function(key, candidate_ops, active_id): + def forward(_x): + return candidate_ops[active_id](_x) + return forward + + def backward_function(key, candidate_ops, active_id, binary_gates): + def backward(_x, _output, grad_output): + binary_grads = torch.zeros_like(binary_gates.data) + with torch.no_grad(): + for k in range(len(candidate_ops)): + if k != active_id: + out_k = candidate_ops[k](_x.data) + else: + out_k = _output.data + grad_k = torch.sum(out_k * grad_output) + binary_grads[k] = grad_k + return binary_grads + return backward + output = ArchGradientFunction.apply( + x, self.ap_path_wb, run_function(mutable.key, list(mutable), self.active_index[0]), + backward_function(mutable.key, list(mutable), self.active_index[0], self.ap_path_wb)) + else: + output = self.active_op(mutable)(x) + return output + + @property + def probs_over_ops(self): + """ + Apply softmax on alpha to generate probability distribution + + Returns + ------- + pytorch tensor + probability distribution + """ + probs = F.softmax(self.ap_path_alpha, dim=0) # softmax to probability + return probs + + @property + def chosen_index(self): + """ + choose the op with max prob + + Returns + ------- + int + index of the chosen one + numpy.float32 + prob of the chosen one + """ + probs = self.probs_over_ops.data.cpu().numpy() + index = int(np.argmax(probs)) + return index, probs[index] + + def active_op(self, mutable): + """ + assume only one path is active + + Returns + ------- + PyTorch module + the chosen operation + """ + return mutable[self.active_index[0]] + + @property + def active_op_index(self): + """ + return active op's index, the active op is sampled + + Returns + ------- + int + index of the active op + """ + return self.active_index[0] + + def set_chosen_op_active(self): + """ + set chosen index, active and inactive indexes + """ + chosen_idx, _ = self.chosen_index + self.active_index = [chosen_idx] + self.inactive_index = [_i for _i in range(0, chosen_idx)] + \ + [_i for _i in range(chosen_idx + 1, self.n_choices)] + + def binarize(self, mutable): + """ + Sample based on alpha, and set binary weights accordingly. + ap_path_wb is set in this function, which is called binarize. + + Parameters + ---------- + mutable : LayerChoice + this layer's mutable + """ + self.log_prob = None + # reset binary gates + self.ap_path_wb.data.zero_() + probs = self.probs_over_ops + if MixedOp.forward_mode == 'two': + # sample two ops according to probs + sample_op = torch.multinomial(probs.data, 2, replacement=False) + probs_slice = F.softmax(torch.stack([ + self.ap_path_alpha[idx] for idx in sample_op + ]), dim=0) + self.current_prob_over_ops = torch.zeros_like(probs) + for i, idx in enumerate(sample_op): + self.current_prob_over_ops[idx] = probs_slice[i] + # choose one to be active and the other to be inactive according to probs_slice + c = torch.multinomial(probs_slice.data, 1)[0] # 0 or 1 + active_op = sample_op[c].item() + inactive_op = sample_op[1-c].item() + self.active_index = [active_op] + self.inactive_index = [inactive_op] + # set binary gate + self.ap_path_wb.data[active_op] = 1.0 + else: + sample = torch.multinomial(probs, 1)[0].item() + self.active_index = [sample] + self.inactive_index = [_i for _i in range(0, sample)] + \ + [_i for _i in range(sample + 1, len(mutable))] + self.log_prob = torch.log(probs[sample]) + self.current_prob_over_ops = probs + self.ap_path_wb.data[sample] = 1.0 + # avoid over-regularization + for choice in mutable: + for _, param in choice.named_parameters(): + param.grad = None + + @staticmethod + def delta_ij(i, j): + if i == j: + return 1 + else: + return 0 + + def set_arch_param_grad(self, mutable): + """ + Calculate alpha gradient for this LayerChoice. + It is calculated using gradient of binary gate, probs of ops. + """ + binary_grads = self.ap_path_wb.grad.data + if self.active_op(mutable).is_zero_layer(): + self.ap_path_alpha.grad = None + return + if self.ap_path_alpha.grad is None: + self.ap_path_alpha.grad = torch.zeros_like(self.ap_path_alpha.data) + if MixedOp.forward_mode == 'two': + involved_idx = self.active_index + self.inactive_index + probs_slice = F.softmax(torch.stack([ + self.ap_path_alpha[idx] for idx in involved_idx + ]), dim=0).data + for i in range(2): + for j in range(2): + origin_i = involved_idx[i] + origin_j = involved_idx[j] + self.ap_path_alpha.grad.data[origin_i] += \ + binary_grads[origin_j] * probs_slice[j] * (MixedOp.delta_ij(i, j) - probs_slice[i]) + for _i, idx in enumerate(self.active_index): + self.active_index[_i] = (idx, self.ap_path_alpha.data[idx].item()) + for _i, idx in enumerate(self.inactive_index): + self.inactive_index[_i] = (idx, self.ap_path_alpha.data[idx].item()) + else: + probs = self.probs_over_ops.data + for i in range(self.n_choices): + for j in range(self.n_choices): + self.ap_path_alpha.grad.data[i] += binary_grads[j] * probs[j] * (MixedOp.delta_ij(i, j) - probs[i]) + return + + def rescale_updated_arch_param(self): + """ + rescale architecture weights for the 'two' mode. + """ + if not isinstance(self.active_index[0], tuple): + assert self.active_op.is_zero_layer() + return + involved_idx = [idx for idx, _ in (self.active_index + self.inactive_index)] + old_alphas = [alpha for _, alpha in (self.active_index + self.inactive_index)] + new_alphas = [self.ap_path_alpha.data[idx] for idx in involved_idx] + + offset = math.log( + sum([math.exp(alpha) for alpha in new_alphas]) / sum([math.exp(alpha) for alpha in old_alphas]) + ) + + for idx in involved_idx: + self.ap_path_alpha.data[idx] -= offset + + +class ProxylessNasMutator(BaseMutator): + """ + This mutator initializes and operates all the LayerChoices of the input model. + It is for the corresponding trainer to control the training process of LayerChoices, + coordinating with whole training process. + """ + def __init__(self, model): + """ + Init a MixedOp instance for each mutable i.e., LayerChoice. + And register the instantiated MixedOp in corresponding LayerChoice. + If does not register it in LayerChoice, DataParallel does not work then, + because architecture weights are not included in the DataParallel model. + When MixedOPs are registered, we use ```requires_grad``` to control + whether calculate gradients of architecture weights. + + Parameters + ---------- + model : pytorch model + The model that users want to tune, it includes search space defined with nni nas apis + """ + super(ProxylessNasMutator, self).__init__(model) + self._unused_modules = None + self.mutable_list = [] + for mutable in self.undedup_mutables: + self.mutable_list.append(mutable) + mutable.registered_module = MixedOp(mutable) + + def on_forward_layer_choice(self, mutable, *args, **kwargs): + """ + Callback of layer choice forward. This function defines the forward + logic of the input mutable. So mutable is only interface, its real + implementation is defined in mutator. + + Parameters + ---------- + mutable: LayerChoice + forward logic of this input mutable + args: list of torch.Tensor + inputs of this mutable + kwargs: dict + inputs of this mutable + + Returns + ------- + torch.Tensor + output of this mutable, i.e., LayerChoice + int + index of the chosen op + """ + # FIXME: return mask, to be consistent with other algorithms + idx = mutable.registered_module.active_op_index + return mutable.registered_module(mutable, *args, **kwargs), idx + + def reset_binary_gates(self): + """ + For each LayerChoice, binarize binary weights + based on alpha to only activate one op. + It traverses all the mutables in the model to do this. + """ + for mutable in self.undedup_mutables: + mutable.registered_module.binarize(mutable) + + def set_chosen_op_active(self): + """ + For each LayerChoice, set the op with highest alpha as the chosen op. + Usually used for validation. + """ + for mutable in self.undedup_mutables: + mutable.registered_module.set_chosen_op_active() + + def num_arch_params(self): + """ + The number of mutables, i.e., LayerChoice + + Returns + ------- + int + the number of LayerChoice in user model + """ + return len(self.mutable_list) + + def set_arch_param_grad(self): + """ + For each LayerChoice, calculate gradients for architecture weights, i.e., alpha + """ + for mutable in self.undedup_mutables: + mutable.registered_module.set_arch_param_grad(mutable) + + def get_architecture_parameters(self): + """ + Get all the architecture parameters. + + yield + ----- + PyTorch Parameter + Return ap_path_alpha of the traversed mutable + """ + for mutable in self.undedup_mutables: + yield mutable.registered_module.get_ap_path_alpha() + + def change_forward_mode(self, mode): + """ + Update forward mode of MixedOps, as training architecture weights and + model weights use different forward modes. + """ + MixedOp.forward_mode = mode + + def get_forward_mode(self): + """ + Get forward mode of MixedOp + + Returns + ------- + string + the current forward mode of MixedOp + """ + return MixedOp.forward_mode + + def rescale_updated_arch_param(self): + """ + Rescale architecture weights in 'two' mode. + """ + for mutable in self.undedup_mutables: + mutable.registered_module.rescale_updated_arch_param() + + def unused_modules_off(self): + """ + Remove unused modules for each mutables. + The removed modules are kept in ```self._unused_modules``` for resume later. + """ + self._unused_modules = [] + for mutable in self.undedup_mutables: + mixed_op = mutable.registered_module + unused = {} + if self.get_forward_mode() in ['full', 'two', 'full_v2']: + involved_index = mixed_op.active_index + mixed_op.inactive_index + else: + involved_index = mixed_op.active_index + for i in range(mixed_op.n_choices): + if i not in involved_index: + unused[i] = mutable[i] + mutable[i] = None + self._unused_modules.append(unused) + + def unused_modules_back(self): + """ + Resume the removed modules back. + """ + if self._unused_modules is None: + return + for m, unused in zip(self.mutable_list, self._unused_modules): + for i in unused: + m[i] = unused[i] + self._unused_modules = None + + def arch_requires_grad(self): + """ + Make architecture weights require gradient + """ + for mutable in self.undedup_mutables: + mutable.registered_module.to_requires_grad() + + def arch_disable_grad(self): + """ + Disable gradient of architecture weights, i.e., does not + calcuate gradient for them. + """ + for mutable in self.undedup_mutables: + mutable.registered_module.to_disable_grad() + + def sample_final(self): + """ + Generate the final chosen architecture. + + Returns + ------- + dict + the choice of each mutable, i.e., LayerChoice + """ + result = dict() + for mutable in self.undedup_mutables: + assert isinstance(mutable, LayerChoice) + index, _ = mutable.registered_module.chosen_index + # pylint: disable=not-callable + result[mutable.key] = F.one_hot(torch.tensor(index), num_classes=len(mutable)).view(-1).bool() + return result diff --git a/utils/third_party/nni/algorithms/nas/pytorch/proxylessnas/trainer.py b/utils/third_party/nni/algorithms/nas/pytorch/proxylessnas/trainer.py new file mode 100644 index 0000000000000000000000000000000000000000..d9c86a6a9f098792a4731db32dd140ce3708ea8f --- /dev/null +++ b/utils/third_party/nni/algorithms/nas/pytorch/proxylessnas/trainer.py @@ -0,0 +1,500 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import math +import time +import json +import logging + +import torch +from torch import nn as nn + +from nni.nas.pytorch.base_trainer import BaseTrainer +from nni.nas.pytorch.trainer import TorchTensorEncoder +from nni.nas.pytorch.utils import AverageMeter +from .mutator import ProxylessNasMutator +from .utils import cross_entropy_with_label_smoothing, accuracy + +logger = logging.getLogger(__name__) + +class ProxylessNasTrainer(BaseTrainer): + def __init__(self, model, model_optim, device, + train_loader, valid_loader, label_smoothing=0.1, + n_epochs=120, init_lr=0.025, binary_mode='full_v2', + arch_init_type='normal', arch_init_ratio=1e-3, + arch_optim_lr=1e-3, arch_weight_decay=0, + grad_update_arch_param_every=5, grad_update_steps=1, + warmup=True, warmup_epochs=25, + arch_valid_frequency=1, + load_ckpt=False, ckpt_path=None, arch_path=None): + """ + Parameters + ---------- + model : pytorch model + the user model, which has mutables + model_optim : pytorch optimizer + the user defined optimizer + device : pytorch device + the devices to train/search the model + train_loader : pytorch data loader + data loader for the training set + valid_loader : pytorch data loader + data loader for the validation set + label_smoothing : float + for label smoothing + n_epochs : int + number of epochs to train/search + init_lr : float + init learning rate for training the model + binary_mode : str + the forward/backward mode for the binary weights in mutator + arch_init_type : str + the way to init architecture parameters + arch_init_ratio : float + the ratio to init architecture parameters + arch_optim_lr : float + learning rate of the architecture parameters optimizer + arch_weight_decay : float + weight decay of the architecture parameters optimizer + grad_update_arch_param_every : int + update architecture weights every this number of minibatches + grad_update_steps : int + during each update of architecture weights, the number of steps to train + warmup : bool + whether to do warmup + warmup_epochs : int + the number of epochs to do during warmup + arch_valid_frequency : int + frequency of printing validation result + load_ckpt : bool + whether load checkpoint + ckpt_path : str + checkpoint path, if load_ckpt is True, ckpt_path cannot be None + arch_path : str + the path to store chosen architecture + """ + self.model = model + self.model_optim = model_optim + self.train_loader = train_loader + self.valid_loader = valid_loader + self.device = device + self.n_epochs = n_epochs + self.init_lr = init_lr + self.warmup = warmup + self.warmup_epochs = warmup_epochs + self.arch_valid_frequency = arch_valid_frequency + self.label_smoothing = label_smoothing + + self.train_batch_size = train_loader.batch_sampler.batch_size + self.valid_batch_size = valid_loader.batch_sampler.batch_size + # update architecture parameters every this number of minibatches + self.grad_update_arch_param_every = grad_update_arch_param_every + # the number of steps per architecture parameter update + self.grad_update_steps = grad_update_steps + self.binary_mode = binary_mode + + self.load_ckpt = load_ckpt + self.ckpt_path = ckpt_path + self.arch_path = arch_path + + # init mutator + self.mutator = ProxylessNasMutator(model) + + # DataParallel should be put behind the init of mutator + self.model = torch.nn.DataParallel(self.model) + self.model.to(self.device) + + # iter of valid dataset for training architecture weights + self._valid_iter = None + # init architecture weights + self._init_arch_params(arch_init_type, arch_init_ratio) + # build architecture optimizer + self.arch_optimizer = torch.optim.Adam(self.mutator.get_architecture_parameters(), + arch_optim_lr, + weight_decay=arch_weight_decay, + betas=(0, 0.999), + eps=1e-8) + + self.criterion = nn.CrossEntropyLoss() + self.warmup_curr_epoch = 0 + self.train_curr_epoch = 0 + + def _init_arch_params(self, init_type='normal', init_ratio=1e-3): + """ + Initialize architecture weights + """ + for param in self.mutator.get_architecture_parameters(): + if init_type == 'normal': + param.data.normal_(0, init_ratio) + elif init_type == 'uniform': + param.data.uniform_(-init_ratio, init_ratio) + else: + raise NotImplementedError + + def _validate(self): + """ + Do validation. During validation, LayerChoices use the chosen active op. + + Returns + ------- + float, float, float + average loss, average top1 accuracy, average top5 accuracy + """ + self.valid_loader.batch_sampler.batch_size = self.valid_batch_size + self.valid_loader.batch_sampler.drop_last = False + + self.mutator.set_chosen_op_active() + # remove unused modules to save memory + self.mutator.unused_modules_off() + # test on validation set under train mode + self.model.train() + batch_time = AverageMeter('batch_time') + losses = AverageMeter('losses') + top1 = AverageMeter('top1') + top5 = AverageMeter('top5') + end = time.time() + with torch.no_grad(): + for i, (images, labels) in enumerate(self.valid_loader): + images, labels = images.to(self.device), labels.to(self.device) + output = self.model(images) + loss = self.criterion(output, labels) + acc1, acc5 = accuracy(output, labels, topk=(1, 5)) + losses.update(loss, images.size(0)) + top1.update(acc1[0], images.size(0)) + top5.update(acc5[0], images.size(0)) + # measure elapsed time + batch_time.update(time.time() - end) + end = time.time() + + if i % 10 == 0 or i + 1 == len(self.valid_loader): + test_log = 'Valid' + ': [{0}/{1}]\t'\ + 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'\ + 'Loss {loss.val:.4f} ({loss.avg:.4f})\t'\ + 'Top-1 acc {top1.val:.3f} ({top1.avg:.3f})'.\ + format(i, len(self.valid_loader) - 1, batch_time=batch_time, loss=losses, top1=top1) + # return top5: + test_log += '\tTop-5 acc {top5.val:.3f} ({top5.avg:.3f})'.format(top5=top5) + logger.info(test_log) + self.mutator.unused_modules_back() + return losses.avg, top1.avg, top5.avg + + def _warm_up(self): + """ + Warm up the model, during warm up, architecture weights are not trained. + """ + lr_max = 0.05 + data_loader = self.train_loader + nBatch = len(data_loader) + T_total = self.warmup_epochs * nBatch # total num of batches + + for epoch in range(self.warmup_curr_epoch, self.warmup_epochs): + logger.info('\n--------Warmup epoch: %d--------\n', epoch + 1) + batch_time = AverageMeter('batch_time') + data_time = AverageMeter('data_time') + losses = AverageMeter('losses') + top1 = AverageMeter('top1') + top5 = AverageMeter('top5') + # switch to train mode + self.model.train() + + end = time.time() + logger.info('warm_up epoch: %d', epoch) + for i, (images, labels) in enumerate(data_loader): + data_time.update(time.time() - end) + # lr + T_cur = epoch * nBatch + i + warmup_lr = 0.5 * lr_max * (1 + math.cos(math.pi * T_cur / T_total)) + for param_group in self.model_optim.param_groups: + param_group['lr'] = warmup_lr + images, labels = images.to(self.device), labels.to(self.device) + # compute output + self.mutator.reset_binary_gates() # random sample binary gates + self.mutator.unused_modules_off() # remove unused module for speedup + output = self.model(images) + if self.label_smoothing > 0: + loss = cross_entropy_with_label_smoothing(output, labels, self.label_smoothing) + else: + loss = self.criterion(output, labels) + # measure accuracy and record loss + acc1, acc5 = accuracy(output, labels, topk=(1, 5)) + losses.update(loss, images.size(0)) + top1.update(acc1[0], images.size(0)) + top5.update(acc5[0], images.size(0)) + # compute gradient and do SGD step + self.model.zero_grad() + loss.backward() + self.model_optim.step() + # unused modules back + self.mutator.unused_modules_back() + # measure elapsed time + batch_time.update(time.time() - end) + end = time.time() + + if i % 10 == 0 or i + 1 == nBatch: + batch_log = 'Warmup Train [{0}][{1}/{2}]\t' \ + 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' \ + 'Data {data_time.val:.3f} ({data_time.avg:.3f})\t' \ + 'Loss {losses.val:.4f} ({losses.avg:.4f})\t' \ + 'Top-1 acc {top1.val:.3f} ({top1.avg:.3f})\t' \ + 'Top-5 acc {top5.val:.3f} ({top5.avg:.3f})\tlr {lr:.5f}'. \ + format(epoch + 1, i, nBatch - 1, batch_time=batch_time, data_time=data_time, + losses=losses, top1=top1, top5=top5, lr=warmup_lr) + logger.info(batch_log) + val_loss, val_top1, val_top5 = self._validate() + val_log = 'Warmup Valid [{0}/{1}]\tloss {2:.3f}\ttop-1 acc {3:.3f}\ttop-5 acc {4:.3f}\t' \ + 'Train top-1 {top1.avg:.3f}\ttop-5 {top5.avg:.3f}M'. \ + format(epoch + 1, self.warmup_epochs, val_loss, val_top1, val_top5, top1=top1, top5=top5) + logger.info(val_log) + self.save_checkpoint() + self.warmup_curr_epoch += 1 + + def _get_update_schedule(self, nBatch): + """ + Generate schedule for training architecture weights. Key means after which minibatch + to update architecture weights, value means how many steps for the update. + + Parameters + ---------- + nBatch : int + the total number of minibatches in one epoch + + Returns + ------- + dict + the schedule for updating architecture weights + """ + schedule = {} + for i in range(nBatch): + if (i + 1) % self.grad_update_arch_param_every == 0: + schedule[i] = self.grad_update_steps + return schedule + + def _calc_learning_rate(self, epoch, batch=0, nBatch=None): + """ + Update learning rate. + """ + T_total = self.n_epochs * nBatch + T_cur = epoch * nBatch + batch + lr = 0.5 * self.init_lr * (1 + math.cos(math.pi * T_cur / T_total)) + return lr + + def _adjust_learning_rate(self, optimizer, epoch, batch=0, nBatch=None): + """ + Adjust learning of a given optimizer and return the new learning rate + + Parameters + ---------- + optimizer : pytorch optimizer + the used optimizer + epoch : int + the current epoch number + batch : int + the current minibatch + nBatch : int + the total number of minibatches in one epoch + + Returns + ------- + float + the adjusted learning rate + """ + new_lr = self._calc_learning_rate(epoch, batch, nBatch) + for param_group in optimizer.param_groups: + param_group['lr'] = new_lr + return new_lr + + def _train(self): + """ + Train the model, it trains model weights and architecute weights. + Architecture weights are trained according to the schedule. + Before updating architecture weights, ```requires_grad``` is enabled. + Then, it is disabled after the updating, in order not to update + architecture weights when training model weights. + """ + nBatch = len(self.train_loader) + arch_param_num = self.mutator.num_arch_params() + binary_gates_num = self.mutator.num_arch_params() + logger.info('#arch_params: %d\t#binary_gates: %d', arch_param_num, binary_gates_num) + + update_schedule = self._get_update_schedule(nBatch) + + for epoch in range(self.train_curr_epoch, self.n_epochs): + logger.info('\n--------Train epoch: %d--------\n', epoch + 1) + batch_time = AverageMeter('batch_time') + data_time = AverageMeter('data_time') + losses = AverageMeter('losses') + top1 = AverageMeter('top1') + top5 = AverageMeter('top5') + # switch to train mode + self.model.train() + + end = time.time() + for i, (images, labels) in enumerate(self.train_loader): + data_time.update(time.time() - end) + lr = self._adjust_learning_rate(self.model_optim, epoch, batch=i, nBatch=nBatch) + # train weight parameters + images, labels = images.to(self.device), labels.to(self.device) + self.mutator.reset_binary_gates() + self.mutator.unused_modules_off() + output = self.model(images) + if self.label_smoothing > 0: + loss = cross_entropy_with_label_smoothing(output, labels, self.label_smoothing) + else: + loss = self.criterion(output, labels) + acc1, acc5 = accuracy(output, labels, topk=(1, 5)) + losses.update(loss, images.size(0)) + top1.update(acc1[0], images.size(0)) + top5.update(acc5[0], images.size(0)) + self.model.zero_grad() + loss.backward() + self.model_optim.step() + self.mutator.unused_modules_back() + if epoch > 0: + for _ in range(update_schedule.get(i, 0)): + start_time = time.time() + # GradientArchSearchConfig + self.mutator.arch_requires_grad() + arch_loss, exp_value = self._gradient_step() + self.mutator.arch_disable_grad() + used_time = time.time() - start_time + log_str = 'Architecture [%d-%d]\t Time %.4f\t Loss %.4f\t null %s' % \ + (epoch + 1, i, used_time, arch_loss, exp_value) + logger.info(log_str) + batch_time.update(time.time() - end) + end = time.time() + # training log + if i % 10 == 0 or i + 1 == nBatch: + batch_log = 'Train [{0}][{1}/{2}]\t' \ + 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' \ + 'Data Time {data_time.val:.3f} ({data_time.avg:.3f})\t' \ + 'Loss {losses.val:.4f} ({losses.avg:.4f})\t' \ + 'Top-1 acc {top1.val:.3f} ({top1.avg:.3f})\t' \ + 'Top-5 acc {top5.val:.3f} ({top5.avg:.3f})\tlr {lr:.5f}'. \ + format(epoch + 1, i, nBatch - 1, batch_time=batch_time, data_time=data_time, + losses=losses, top1=top1, top5=top5, lr=lr) + logger.info(batch_log) + # validate + if (epoch + 1) % self.arch_valid_frequency == 0: + val_loss, val_top1, val_top5 = self._validate() + val_log = 'Valid [{0}]\tloss {1:.3f}\ttop-1 acc {2:.3f} \ttop-5 acc {3:.3f}\t' \ + 'Train top-1 {top1.avg:.3f}\ttop-5 {top5.avg:.3f}'. \ + format(epoch + 1, val_loss, val_top1, val_top5, top1=top1, top5=top5) + logger.info(val_log) + self.save_checkpoint() + self.train_curr_epoch += 1 + + def _valid_next_batch(self): + """ + Get next one minibatch from validation set + + Returns + ------- + (tensor, tensor) + the tuple of images and labels + """ + if self._valid_iter is None: + self._valid_iter = iter(self.valid_loader) + try: + data = next(self._valid_iter) + except StopIteration: + self._valid_iter = iter(self.valid_loader) + data = next(self._valid_iter) + return data + + def _gradient_step(self): + """ + This gradient step is for updating architecture weights. + Mutator is intensively used in this function to operate on + architecture weights. + + Returns + ------- + float, None + loss of the model, None + """ + # use the same batch size as train batch size for architecture weights + self.valid_loader.batch_sampler.batch_size = self.train_batch_size + self.valid_loader.batch_sampler.drop_last = True + self.model.train() + self.mutator.change_forward_mode(self.binary_mode) + time1 = time.time() # time + # sample a batch of data from validation set + images, labels = self._valid_next_batch() + images, labels = images.to(self.device), labels.to(self.device) + time2 = time.time() # time + self.mutator.reset_binary_gates() + self.mutator.unused_modules_off() + output = self.model(images) + time3 = time.time() + ce_loss = self.criterion(output, labels) + expected_value = None + loss = ce_loss + self.model.zero_grad() + loss.backward() + self.mutator.set_arch_param_grad() + self.arch_optimizer.step() + if self.mutator.get_forward_mode() == 'two': + self.mutator.rescale_updated_arch_param() + self.mutator.unused_modules_back() + self.mutator.change_forward_mode(None) + time4 = time.time() + logger.info('(%.4f, %.4f, %.4f)', time2 - time1, time3 - time2, time4 - time3) + return loss.data.item(), expected_value.item() if expected_value is not None else None + + def save_checkpoint(self): + """ + Save checkpoint of the whole model. Saving model weights and architecture weights in + ```ckpt_path```, and saving currently chosen architecture in ```arch_path```. + """ + if self.ckpt_path: + state = { + 'warmup_curr_epoch': self.warmup_curr_epoch, + 'train_curr_epoch': self.train_curr_epoch, + 'model': self.model.state_dict(), + 'optim': self.model_optim.state_dict(), + 'arch_optim': self.arch_optimizer.state_dict() + } + torch.save(state, self.ckpt_path) + if self.arch_path: + self.export(self.arch_path) + + def load_checkpoint(self): + """ + Load the checkpoint from ```ckpt_path```. + """ + assert self.ckpt_path is not None, "If load_ckpt is not None, ckpt_path should not be None" + ckpt = torch.load(self.ckpt_path) + self.warmup_curr_epoch = ckpt['warmup_curr_epoch'] + self.train_curr_epoch = ckpt['train_curr_epoch'] + self.model.load_state_dict(ckpt['model']) + self.model_optim.load_state_dict(ckpt['optim']) + self.arch_optimizer.load_state_dict(ckpt['arch_optim']) + + def train(self): + """ + Train the whole model. + """ + if self.load_ckpt: + self.load_checkpoint() + if self.warmup: + self._warm_up() + self._train() + + def export(self, file_name): + """ + Export the chosen architecture into a file + + Parameters + ---------- + file_name : str + the file that stores exported chosen architecture + """ + exported_arch = self.mutator.sample_final() + with open(file_name, 'w') as f: + json.dump(exported_arch, f, indent=2, sort_keys=True, cls=TorchTensorEncoder) + + def validate(self): + raise NotImplementedError + + def checkpoint(self): + raise NotImplementedError diff --git a/utils/third_party/nni/algorithms/nas/pytorch/proxylessnas/utils.py b/utils/third_party/nni/algorithms/nas/pytorch/proxylessnas/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..b703810d3b703a33bd6bbe8422c557a1f669146b --- /dev/null +++ b/utils/third_party/nni/algorithms/nas/pytorch/proxylessnas/utils.py @@ -0,0 +1,78 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import torch +import torch.nn as nn + +def detach_variable(inputs): + """ + Detach variables + + Parameters + ---------- + inputs : pytorch tensors + pytorch tensors + """ + if isinstance(inputs, tuple): + return tuple([detach_variable(x) for x in inputs]) + else: + x = inputs.detach() + x.requires_grad = inputs.requires_grad + return x + +def cross_entropy_with_label_smoothing(pred, target, label_smoothing=0.1): + """ + Parameters + ---------- + pred : pytorch tensor + predicted value + target : pytorch tensor + label + label_smoothing : float + the degree of label smoothing + + Returns + ------- + pytorch tensor + cross entropy + """ + logsoftmax = nn.LogSoftmax() + n_classes = pred.size(1) + # convert to one-hot + target = torch.unsqueeze(target, 1) + soft_target = torch.zeros_like(pred) + soft_target.scatter_(1, target, 1) + # label smoothing + soft_target = soft_target * (1 - label_smoothing) + label_smoothing / n_classes + return torch.mean(torch.sum(- soft_target * logsoftmax(pred), 1)) + +def accuracy(output, target, topk=(1,)): + """ + Computes the precision@k for the specified values of k + + Parameters + ---------- + output : pytorch tensor + output, e.g., predicted value + target : pytorch tensor + label + topk : tuple + specify top1 and top5 + + Returns + ------- + list + accuracy of top1 and top5 + """ + maxk = max(topk) + batch_size = target.size(0) + + _, pred = output.topk(maxk, 1, True, True) + pred = pred.t() + correct = pred.eq(target.view(1, -1).expand_as(pred)) + + res = [] + for k in topk: + correct_k = correct[:k].view(-1).float().sum(0, keepdim=True) + res.append(correct_k.mul_(100.0 / batch_size)) + return res diff --git a/utils/third_party/nni/algorithms/nas/pytorch/random/__init__.py b/utils/third_party/nni/algorithms/nas/pytorch/random/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..8a52b8e030549b08ed636c47559c4a144c337446 --- /dev/null +++ b/utils/third_party/nni/algorithms/nas/pytorch/random/__init__.py @@ -0,0 +1 @@ +from .mutator import RandomMutator \ No newline at end of file diff --git a/utils/third_party/nni/algorithms/nas/pytorch/random/mutator.py b/utils/third_party/nni/algorithms/nas/pytorch/random/mutator.py new file mode 100644 index 0000000000000000000000000000000000000000..f302db56c0cdbd611f648342cc26760d344958c4 --- /dev/null +++ b/utils/third_party/nni/algorithms/nas/pytorch/random/mutator.py @@ -0,0 +1,36 @@ +import torch +import torch.nn.functional as F + +from nni.nas.pytorch.mutator import Mutator +from nni.nas.pytorch.mutables import LayerChoice, InputChoice + + +class RandomMutator(Mutator): + """ + Random mutator that samples a random candidate in the search space each time ``reset()``. + It uses random function in PyTorch, so users can set seed in PyTorch to ensure deterministic behavior. + """ + + def sample_search(self): + """ + Sample a random candidate. + """ + result = dict() + for mutable in self.mutables: + if isinstance(mutable, LayerChoice): + gen_index = torch.randint(high=len(mutable), size=(1, )) + result[mutable.key] = F.one_hot(gen_index, num_classes=len(mutable)).view(-1).bool() + elif isinstance(mutable, InputChoice): + if mutable.n_chosen is None: + result[mutable.key] = torch.randint(high=2, size=(mutable.n_candidates,)).view(-1).bool() + else: + perm = torch.randperm(mutable.n_candidates) + mask = [i in perm[:mutable.n_chosen] for i in range(mutable.n_candidates)] + result[mutable.key] = torch.tensor(mask, dtype=torch.bool) # pylint: disable=not-callable + return result + + def sample_final(self): + """ + Same as :meth:`sample_search`. + """ + return self.sample_search() diff --git a/utils/third_party/nni/algorithms/nas/pytorch/spos/__init__.py b/utils/third_party/nni/algorithms/nas/pytorch/spos/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..ed432b0845154c4745aa82c8e6a8bad4290237aa --- /dev/null +++ b/utils/third_party/nni/algorithms/nas/pytorch/spos/__init__.py @@ -0,0 +1,6 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .evolution import SPOSEvolution +from .mutator import SPOSSupernetTrainingMutator +from .trainer import SPOSSupernetTrainer diff --git a/utils/third_party/nni/algorithms/nas/pytorch/spos/evolution.py b/utils/third_party/nni/algorithms/nas/pytorch/spos/evolution.py new file mode 100644 index 0000000000000000000000000000000000000000..bd099e276ebfd4aa68215dbbd25e6443a45b9dd1 --- /dev/null +++ b/utils/third_party/nni/algorithms/nas/pytorch/spos/evolution.py @@ -0,0 +1,223 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import json +import logging +import os +import re +from collections import deque + +import numpy as np +from nni.tuner import Tuner +from nni.algorithms.nas.pytorch.classic_nas.mutator import LAYER_CHOICE, INPUT_CHOICE + + +_logger = logging.getLogger(__name__) + + +class SPOSEvolution(Tuner): + """ + SPOS evolution tuner. + + Parameters + ---------- + max_epochs : int + Maximum number of epochs to run. + num_select : int + Number of survival candidates of each epoch. + num_population : int + Number of candidates at the start of each epoch. If candidates generated by + crossover and mutation are not enough, the rest will be filled with random + candidates. + m_prob : float + The probability of mutation. + num_crossover : int + Number of candidates generated by crossover in each epoch. + num_mutation : int + Number of candidates generated by mutation in each epoch. + """ + + def __init__(self, max_epochs=20, num_select=10, num_population=50, m_prob=0.1, + num_crossover=25, num_mutation=25): + assert num_population >= num_select + self.max_epochs = max_epochs + self.num_select = num_select + self.num_population = num_population + self.m_prob = m_prob + self.num_crossover = num_crossover + self.num_mutation = num_mutation + self.epoch = 0 + self.candidates = [] + self.search_space = None + self.random_state = np.random.RandomState(0) + + # async status + self._to_evaluate_queue = deque() + self._sending_parameter_queue = deque() + self._pending_result_ids = set() + self._reward_dict = dict() + self._id2candidate = dict() + self._st_callback = None + + def update_search_space(self, search_space): + """ + Handle the initialization/update event of search space. + """ + self._search_space = search_space + self._next_round() + + def _next_round(self): + _logger.info("Epoch %d, generating...", self.epoch) + if self.epoch == 0: + self._get_random_population() + self.export_results(self.candidates) + else: + best_candidates = self._select_top_candidates() + self.export_results(best_candidates) + if self.epoch >= self.max_epochs: + return + self.candidates = self._get_mutation(best_candidates) + self._get_crossover(best_candidates) + self._get_random_population() + self.epoch += 1 + + def _random_candidate(self): + chosen_arch = dict() + for key, val in self._search_space.items(): + if val["_type"] == LAYER_CHOICE: + choices = val["_value"] + index = self.random_state.randint(len(choices)) + chosen_arch[key] = {"_value": choices[index], "_idx": index} + elif val["_type"] == INPUT_CHOICE: + raise NotImplementedError("Input choice is not implemented yet.") + return chosen_arch + + def _add_to_evaluate_queue(self, cand): + _logger.info("Generate candidate %s, adding to eval queue.", self._get_architecture_repr(cand)) + self._reward_dict[self._hashcode(cand)] = 0. + self._to_evaluate_queue.append(cand) + + def _get_random_population(self): + while len(self.candidates) < self.num_population: + cand = self._random_candidate() + if self._is_legal(cand): + _logger.info("Random candidate generated.") + self._add_to_evaluate_queue(cand) + self.candidates.append(cand) + + def _get_crossover(self, best): + result = [] + for _ in range(10 * self.num_crossover): + cand_p1 = best[self.random_state.randint(len(best))] + cand_p2 = best[self.random_state.randint(len(best))] + assert cand_p1.keys() == cand_p2.keys() + cand = {k: cand_p1[k] if self.random_state.randint(2) == 0 else cand_p2[k] + for k in cand_p1.keys()} + if self._is_legal(cand): + result.append(cand) + self._add_to_evaluate_queue(cand) + if len(result) >= self.num_crossover: + break + _logger.info("Found %d architectures with crossover.", len(result)) + return result + + def _get_mutation(self, best): + result = [] + for _ in range(10 * self.num_mutation): + cand = best[self.random_state.randint(len(best))].copy() + mutation_sample = np.random.random_sample(len(cand)) + for s, k in zip(mutation_sample, cand): + if s < self.m_prob: + choices = self._search_space[k]["_value"] + index = self.random_state.randint(len(choices)) + cand[k] = {"_value": choices[index], "_idx": index} + if self._is_legal(cand): + result.append(cand) + self._add_to_evaluate_queue(cand) + if len(result) >= self.num_mutation: + break + _logger.info("Found %d architectures with mutation.", len(result)) + return result + + def _get_architecture_repr(self, cand): + return re.sub(r"\".*?\": \{\"_idx\": (\d+), \"_value\": \".*?\"\}", r"\1", + self._hashcode(cand)) + + def _is_legal(self, cand): + if self._hashcode(cand) in self._reward_dict: + return False + return True + + def _select_top_candidates(self): + reward_query = lambda cand: self._reward_dict[self._hashcode(cand)] + _logger.info("All candidate rewards: %s", list(map(reward_query, self.candidates))) + result = sorted(self.candidates, key=reward_query, reverse=True)[:self.num_select] + _logger.info("Best candidate rewards: %s", list(map(reward_query, result))) + return result + + @staticmethod + def _hashcode(d): + return json.dumps(d, sort_keys=True) + + def _bind_and_send_parameters(self): + """ + There are two types of resources: parameter ids and candidates. This function is called at + necessary times to bind these resources to send new trials with st_callback. + """ + result = [] + while self._sending_parameter_queue and self._to_evaluate_queue: + parameter_id = self._sending_parameter_queue.popleft() + parameters = self._to_evaluate_queue.popleft() + self._id2candidate[parameter_id] = parameters + result.append(parameters) + self._pending_result_ids.add(parameter_id) + self._st_callback(parameter_id, parameters) + _logger.info("Send parameter [%d] %s.", parameter_id, self._get_architecture_repr(parameters)) + return result + + def generate_multiple_parameters(self, parameter_id_list, **kwargs): + """ + Callback function necessary to implement a tuner. This will put more parameter ids into the + parameter id queue. + """ + if "st_callback" in kwargs and self._st_callback is None: + self._st_callback = kwargs["st_callback"] + for parameter_id in parameter_id_list: + self._sending_parameter_queue.append(parameter_id) + self._bind_and_send_parameters() + return [] # always not use this. might induce problem of over-sending + + def receive_trial_result(self, parameter_id, parameters, value, **kwargs): + """ + Callback function. Receive a trial result. + """ + _logger.info("Candidate %d, reported reward %f", parameter_id, value) + self._reward_dict[self._hashcode(self._id2candidate[parameter_id])] = value + + def trial_end(self, parameter_id, success, **kwargs): + """ + Callback function when a trial is ended and resource is released. + """ + self._pending_result_ids.remove(parameter_id) + if not self._pending_result_ids and not self._to_evaluate_queue: + # a new epoch now + self._next_round() + assert self._st_callback is not None + self._bind_and_send_parameters() + + def export_results(self, result): + """ + Export a number of candidates to `checkpoints` dir. + + Parameters + ---------- + result : dict + Chosen architectures to be exported. + """ + os.makedirs("checkpoints", exist_ok=True) + for i, cand in enumerate(result): + converted = dict() + for cand_key, cand_val in cand.items(): + onehot = [k == cand_val["_idx"] for k in range(len(self._search_space[cand_key]["_value"]))] + converted[cand_key] = onehot + with open(os.path.join("checkpoints", "%03d_%03d.json" % (self.epoch, i)), "w") as fp: + json.dump(converted, fp) diff --git a/utils/third_party/nni/algorithms/nas/pytorch/spos/mutator.py b/utils/third_party/nni/algorithms/nas/pytorch/spos/mutator.py new file mode 100644 index 0000000000000000000000000000000000000000..1a803cb2e820b0df2dd8b04b3d387af68d3d2680 --- /dev/null +++ b/utils/third_party/nni/algorithms/nas/pytorch/spos/mutator.py @@ -0,0 +1,66 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging + +import numpy as np +from nni.algorithms.nas.pytorch.random import RandomMutator + +_logger = logging.getLogger(__name__) + + +class SPOSSupernetTrainingMutator(RandomMutator): + """ + A random mutator with flops limit. + + Parameters + ---------- + model : nn.Module + PyTorch model. + flops_func : callable + Callable that takes a candidate from `sample_search` and returns its candidate. When `flops_func` + is None, functions related to flops will be deactivated. + flops_lb : number + Lower bound of flops. + flops_ub : number + Upper bound of flops. + flops_bin_num : number + Number of bins divided for the interval of flops to ensure the uniformity. Bigger number will be more + uniform, but the sampling will be slower. + flops_sample_timeout : int + Maximum number of attempts to sample before giving up and use a random candidate. + """ + def __init__(self, model, flops_func=None, flops_lb=None, flops_ub=None, + flops_bin_num=7, flops_sample_timeout=500): + + super().__init__(model) + self._flops_func = flops_func + if self._flops_func is not None: + self._flops_bin_num = flops_bin_num + self._flops_bins = [flops_lb + (flops_ub - flops_lb) / flops_bin_num * i for i in range(flops_bin_num + 1)] + self._flops_sample_timeout = flops_sample_timeout + + def sample_search(self): + """ + Sample a candidate for training. When `flops_func` is not None, candidates will be sampled uniformly + relative to flops. + + Returns + ------- + dict + """ + if self._flops_func is not None: + for times in range(self._flops_sample_timeout): + idx = np.random.randint(self._flops_bin_num) + cand = super().sample_search() + if self._flops_bins[idx] <= self._flops_func(cand) <= self._flops_bins[idx + 1]: + _logger.debug("Sampled candidate flops %f in %d times.", cand, times) + return cand + _logger.warning("Failed to sample a flops-valid candidate within %d tries.", self._flops_sample_timeout) + return super().sample_search() + + def sample_final(self): + """ + Implement only to suffice the interface of Mutator. + """ + return self.sample_search() diff --git a/utils/third_party/nni/algorithms/nas/pytorch/spos/trainer.py b/utils/third_party/nni/algorithms/nas/pytorch/spos/trainer.py new file mode 100644 index 0000000000000000000000000000000000000000..c0ca933991688de973b4f739558c4c3ac0410ef4 --- /dev/null +++ b/utils/third_party/nni/algorithms/nas/pytorch/spos/trainer.py @@ -0,0 +1,95 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging + +import torch +from nni.nas.pytorch.trainer import Trainer +from nni.nas.pytorch.utils import AverageMeterGroup + +from .mutator import SPOSSupernetTrainingMutator + +logger = logging.getLogger(__name__) + + +class SPOSSupernetTrainer(Trainer): + """ + This trainer trains a supernet that can be used for evolution search. + + Parameters + ---------- + model : nn.Module + Model with mutables. + mutator : Mutator + A mutator object that has been initialized with the model. + loss : callable + Called with logits and targets. Returns a loss tensor. + metrics : callable + Returns a dict that maps metrics keys to metrics data. + optimizer : Optimizer + Optimizer that optimizes the model. + num_epochs : int + Number of epochs of training. + train_loader : iterable + Data loader of training. Raise ``StopIteration`` when one epoch is exhausted. + dataset_valid : iterable + Data loader of validation. Raise ``StopIteration`` when one epoch is exhausted. + batch_size : int + Batch size. + workers: int + Number of threads for data preprocessing. Not used for this trainer. Maybe removed in future. + device : torch.device + Device object. Either ``torch.device("cuda")`` or ``torch.device("cpu")``. When ``None``, trainer will + automatic detects GPU and selects GPU first. + log_frequency : int + Number of mini-batches to log metrics. + callbacks : list of Callback + Callbacks to plug into the trainer. See Callbacks. + """ + + def __init__(self, model, loss, metrics, + optimizer, num_epochs, train_loader, valid_loader, + mutator=None, batch_size=64, workers=4, device=None, log_frequency=None, + callbacks=None): + assert torch.cuda.is_available() + super().__init__(model, mutator if mutator is not None else SPOSSupernetTrainingMutator(model), + loss, metrics, optimizer, num_epochs, None, None, + batch_size, workers, device, log_frequency, callbacks) + + self.train_loader = train_loader + self.valid_loader = valid_loader + + def train_one_epoch(self, epoch): + self.model.train() + meters = AverageMeterGroup() + for step, (x, y) in enumerate(self.train_loader): + x, y = x.to(self.device), y.to(self.device) + self.optimizer.zero_grad() + self.mutator.reset() + logits = self.model(x) + loss = self.loss(logits, y) + loss.backward() + self.optimizer.step() + + metrics = self.metrics(logits, y) + metrics["loss"] = loss.item() + meters.update(metrics) + if self.log_frequency is not None and step % self.log_frequency == 0: + logger.info("Epoch [%s/%s] Step [%s/%s] %s", epoch + 1, + self.num_epochs, step + 1, len(self.train_loader), meters) + + def validate_one_epoch(self, epoch): + self.model.eval() + meters = AverageMeterGroup() + with torch.no_grad(): + for step, (x, y) in enumerate(self.valid_loader): + x, y = x.to(self.device), y.to(self.device) + self.mutator.reset() + logits = self.model(x) + loss = self.loss(logits, y) + metrics = self.metrics(logits, y) + metrics["loss"] = loss.item() + meters.update(metrics) + if self.log_frequency is not None and step % self.log_frequency == 0: + logger.info("Epoch [%s/%s] Validation Step [%s/%s] %s", epoch + 1, + self.num_epochs, step + 1, len(self.valid_loader), meters) diff --git a/utils/third_party/nni/algorithms/nas/tensorflow/__init__.py b/utils/third_party/nni/algorithms/nas/tensorflow/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/utils/third_party/nni/algorithms/nas/tensorflow/classic_nas/__init__.py b/utils/third_party/nni/algorithms/nas/tensorflow/classic_nas/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..ec3f5a4894a0460d4a0582a0d6cd43af9bed77e2 --- /dev/null +++ b/utils/third_party/nni/algorithms/nas/tensorflow/classic_nas/__init__.py @@ -0,0 +1,4 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .mutator import get_and_apply_next_architecture diff --git a/utils/third_party/nni/algorithms/nas/tensorflow/classic_nas/mutator.py b/utils/third_party/nni/algorithms/nas/tensorflow/classic_nas/mutator.py new file mode 100644 index 0000000000000000000000000000000000000000..fad4987fed55bc9a1460089ee0fd3b92a63ee83f --- /dev/null +++ b/utils/third_party/nni/algorithms/nas/tensorflow/classic_nas/mutator.py @@ -0,0 +1,215 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import json +import logging +import os +import sys + +import tensorflow as tf + +import nni +from nni.runtime.env_vars import trial_env_vars +from nni.nas.tensorflow.mutables import LayerChoice, InputChoice, MutableScope +from nni.nas.tensorflow.mutator import Mutator + +logger = logging.getLogger(__name__) + +NNI_GEN_SEARCH_SPACE = "NNI_GEN_SEARCH_SPACE" +LAYER_CHOICE = "layer_choice" +INPUT_CHOICE = "input_choice" + + +def get_and_apply_next_architecture(model): + """ + Wrapper of :class:`~nni.nas.tensorflow.classic_nas.mutator.ClassicMutator` to make it more meaningful, + similar to ``get_next_parameter`` for HPO. + Tt will generate search space based on ``model``. + If env ``NNI_GEN_SEARCH_SPACE`` exists, this is in dry run mode for + generating search space for the experiment. + If not, there are still two mode, one is nni experiment mode where users + use ``nnictl`` to start an experiment. The other is standalone mode + where users directly run the trial command, this mode chooses the first + one(s) for each LayerChoice and InputChoice. + Parameters + ---------- + model : nn.Module + User's model with search space (e.g., LayerChoice, InputChoice) embedded in it. + """ + ClassicMutator(model) + + +class ClassicMutator(Mutator): + """ + This mutator is to apply the architecture chosen from tuner. + It implements the forward function of LayerChoice and InputChoice, + to only activate the chosen ones. + Parameters + ---------- + model : nn.Module + User's model with search space (e.g., LayerChoice, InputChoice) embedded in it. + """ + + def __init__(self, model): + super(ClassicMutator, self).__init__(model) + self._chosen_arch = {} + self._search_space = self._generate_search_space() + if NNI_GEN_SEARCH_SPACE in os.environ: + # dry run for only generating search space + self._dump_search_space(os.environ[NNI_GEN_SEARCH_SPACE]) + sys.exit(0) + + if trial_env_vars.NNI_PLATFORM is None: + logger.warning("This is in standalone mode, the chosen are the first one(s).") + self._chosen_arch = self._standalone_generate_chosen() + else: + # get chosen arch from tuner + self._chosen_arch = nni.get_next_parameter() + if self._chosen_arch is None: + if trial_env_vars.NNI_PLATFORM == "unittest": + # happens if NNI_PLATFORM is intentionally set, e.g., in UT + logger.warning("`NNI_PLATFORM` is set but `param` is None. Falling back to standalone mode.") + self._chosen_arch = self._standalone_generate_chosen() + else: + raise RuntimeError("Chosen architecture is None. This may be a platform error.") + self.reset() + + def _sample_layer_choice(self, mutable, idx, value, search_space_item): + """ + Convert layer choice to tensor representation. + Parameters + ---------- + mutable : Mutable + idx : int + Number `idx` of list will be selected. + value : str + The verbose representation of the selected value. + search_space_item : list + The list for corresponding search space. + """ + # doesn't support multihot for layer choice yet + assert 0 <= idx < len(mutable) and search_space_item[idx] == value, \ + "Index '{}' in search space '{}' is not '{}'".format(idx, search_space_item, value) + mask = tf.one_hot(idx, len(mutable)) + return tf.cast(tf.reshape(mask, [-1]), tf.bool) + + def _sample_input_choice(self, mutable, idx, value, search_space_item): + """ + Convert input choice to tensor representation. + Parameters + ---------- + mutable : Mutable + idx : int + Number `idx` of list will be selected. + value : str + The verbose representation of the selected value. + search_space_item : list + The list for corresponding search space. + """ + candidate_repr = search_space_item["candidates"] + multihot_list = [False] * mutable.n_candidates + for i, v in zip(idx, value): + assert 0 <= i < mutable.n_candidates and candidate_repr[i] == v, \ + "Index '{}' in search space '{}' is not '{}'".format(i, candidate_repr, v) + assert not multihot_list[i], "'{}' is selected twice in '{}', which is not allowed.".format(i, idx) + multihot_list[i] = True + return tf.cast(multihot_list, tf.bool) # pylint: disable=not-callable + + def sample_search(self): + """ + See :meth:`sample_final`. + """ + return self.sample_final() + + def sample_final(self): + """ + Convert the chosen arch and apply it on model. + """ + assert set(self._chosen_arch.keys()) == set(self._search_space.keys()), \ + "Unmatched keys, expected keys '{}' from search space, found '{}'.".format(self._search_space.keys(), + self._chosen_arch.keys()) + result = dict() + for mutable in self.mutables: + if isinstance(mutable, (LayerChoice, InputChoice)): + assert mutable.key in self._chosen_arch, \ + "Expected '{}' in chosen arch, but not found.".format(mutable.key) + data = self._chosen_arch[mutable.key] + assert isinstance(data, dict) and "_value" in data and "_idx" in data, \ + "'{}' is not a valid choice.".format(data) + if isinstance(mutable, LayerChoice): + result[mutable.key] = self._sample_layer_choice(mutable, data["_idx"], data["_value"], + self._search_space[mutable.key]["_value"]) + elif isinstance(mutable, InputChoice): + result[mutable.key] = self._sample_input_choice(mutable, data["_idx"], data["_value"], + self._search_space[mutable.key]["_value"]) + elif isinstance(mutable, MutableScope): + logger.info("Mutable scope '%s' is skipped during parsing choices.", mutable.key) + else: + raise TypeError("Unsupported mutable type: '%s'." % type(mutable)) + return result + + def _standalone_generate_chosen(self): + """ + Generate the chosen architecture for standalone mode, + i.e., choose the first one(s) for LayerChoice and InputChoice. + :: + { key_name: {"_value": "conv1", + "_idx": 0} } + { key_name: {"_value": ["in1"], + "_idx": [0]} } + Returns + ------- + dict + the chosen architecture + """ + chosen_arch = {} + for key, val in self._search_space.items(): + if val["_type"] == LAYER_CHOICE: + choices = val["_value"] + chosen_arch[key] = {"_value": choices[0], "_idx": 0} + elif val["_type"] == INPUT_CHOICE: + choices = val["_value"]["candidates"] + n_chosen = val["_value"]["n_chosen"] + if n_chosen is None: + n_chosen = len(choices) + chosen_arch[key] = {"_value": choices[:n_chosen], "_idx": list(range(n_chosen))} + else: + raise ValueError("Unknown key '%s' and value '%s'." % (key, val)) + return chosen_arch + + def _generate_search_space(self): + """ + Generate search space from mutables. + Here is the search space format: + :: + { key_name: {"_type": "layer_choice", + "_value": ["conv1", "conv2"]} } + { key_name: {"_type": "input_choice", + "_value": {"candidates": ["in1", "in2"], + "n_chosen": 1}} } + Returns + ------- + dict + the generated search space + """ + search_space = {} + for mutable in self.mutables: + # for now we only generate flattened search space + if isinstance(mutable, LayerChoice): + key = mutable.key + val = mutable.names + search_space[key] = {"_type": LAYER_CHOICE, "_value": val} + elif isinstance(mutable, InputChoice): + key = mutable.key + search_space[key] = {"_type": INPUT_CHOICE, + "_value": {"candidates": mutable.choose_from, + "n_chosen": mutable.n_chosen}} + elif isinstance(mutable, MutableScope): + logger.info("Mutable scope '%s' is skipped during generating search space.", mutable.key) + else: + raise TypeError("Unsupported mutable type: '%s'." % type(mutable)) + return search_space + + def _dump_search_space(self, file_path): + with open(file_path, "w") as ss_file: + json.dump(self._search_space, ss_file, sort_keys=True, indent=2) diff --git a/utils/third_party/nni/algorithms/nas/tensorflow/enas/__init__.py b/utils/third_party/nni/algorithms/nas/tensorflow/enas/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d3372836ebba2387ade58161218aad731433e46b --- /dev/null +++ b/utils/third_party/nni/algorithms/nas/tensorflow/enas/__init__.py @@ -0,0 +1,5 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .mutator import EnasMutator +from .trainer import EnasTrainer diff --git a/utils/third_party/nni/algorithms/nas/tensorflow/enas/mutator.py b/utils/third_party/nni/algorithms/nas/tensorflow/enas/mutator.py new file mode 100644 index 0000000000000000000000000000000000000000..de43195fa2e6eef6b8c1991cf8d56998d7abc4fb --- /dev/null +++ b/utils/third_party/nni/algorithms/nas/tensorflow/enas/mutator.py @@ -0,0 +1,160 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import tensorflow as tf +from tensorflow.keras.layers import Dense, Embedding, LSTMCell, RNN +from tensorflow.keras.losses import SparseCategoricalCrossentropy, Reduction + +from nni.nas.tensorflow.mutator import Mutator +from nni.nas.tensorflow.mutables import LayerChoice, InputChoice, MutableScope + + +class EnasMutator(Mutator): + def __init__(self, model, + lstm_size=64, + lstm_num_layers=1, + tanh_constant=1.5, + cell_exit_extra_step=False, + skip_target=0.4, + temperature=None, + branch_bias=0.25, + entropy_reduction='sum'): + super().__init__(model) + self.tanh_constant = tanh_constant + self.temperature = temperature + self.cell_exit_extra_step = cell_exit_extra_step + + cells = [LSTMCell(units=lstm_size, use_bias=False) for _ in range(lstm_num_layers)] + self.lstm = RNN(cells, stateful=True) + self.g_emb = tf.random.normal((1, 1, lstm_size)) * 0.1 + self.skip_targets = tf.constant([1.0 - skip_target, skip_target]) + + self.max_layer_choice = 0 + self.bias_dict = {} + for mutable in self.mutables: + if isinstance(mutable, LayerChoice): + if self.max_layer_choice == 0: + self.max_layer_choice = len(mutable) + assert self.max_layer_choice == len(mutable), \ + "ENAS mutator requires all layer choice have the same number of candidates." + if 'reduce' in mutable.key: + bias = [] + for choice in mutable.choices: + if 'conv' in str(type(choice)).lower(): + bias.append(branch_bias) + else: + bias.append(-branch_bias) + self.bias_dict[mutable.key] = tf.constant(bias) + + # exposed for trainer + self.sample_log_prob = 0 + self.sample_entropy = 0 + self.sample_skip_penalty = 0 + + # internal nn layers + self.embedding = Embedding(self.max_layer_choice + 1, lstm_size) + self.soft = Dense(self.max_layer_choice, use_bias=False) + self.attn_anchor = Dense(lstm_size, use_bias=False) + self.attn_query = Dense(lstm_size, use_bias=False) + self.v_attn = Dense(1, use_bias=False) + assert entropy_reduction in ['sum', 'mean'], 'Entropy reduction must be one of sum and mean.' + self.entropy_reduction = tf.reduce_sum if entropy_reduction == 'sum' else tf.reduce_mean + self.cross_entropy_loss = SparseCategoricalCrossentropy(from_logits=True, reduction=Reduction.NONE) + + self._first_sample = True + + def sample_search(self): + self._initialize() + self._sample(self.mutables) + self._first_sample = False + return self._choices + + def sample_final(self): + return self.sample_search() + + def _sample(self, tree): + mutable = tree.mutable + if isinstance(mutable, LayerChoice) and mutable.key not in self._choices: + self._choices[mutable.key] = self._sample_layer_choice(mutable) + elif isinstance(mutable, InputChoice) and mutable.key not in self._choices: + self._choices[mutable.key] = self._sample_input_choice(mutable) + for child in tree.children: + self._sample(child) + if self.cell_exit_extra_step and isinstance(mutable, MutableScope) and mutable.key not in self._anchors_hid: + self._anchors_hid[mutable.key] = self.lstm(self._inputs, 1) + + def _initialize(self): + self._choices = {} + self._anchors_hid = {} + self._inputs = self.g_emb + # seems the `input_shape` parameter of RNN does not work + # workaround it by omitting `reset_states` for first run + if not self._first_sample: + self.lstm.reset_states() + self.sample_log_prob = 0 + self.sample_entropy = 0 + self.sample_skip_penalty = 0 + + def _sample_layer_choice(self, mutable): + logit = self.soft(self.lstm(self._inputs)) + if self.temperature is not None: + logit /= self.temperature + if self.tanh_constant is not None: + logit = self.tanh_constant * tf.tanh(logit) + if mutable.key in self.bias_dict: + logit += self.bias_dict[mutable.key] + softmax_logit = tf.math.log(tf.nn.softmax(logit, axis=-1)) + branch_id = tf.reshape(tf.random.categorical(softmax_logit, num_samples=1), [1]) + log_prob = self.cross_entropy_loss(branch_id, logit) + self.sample_log_prob += self.entropy_reduction(log_prob) + entropy = log_prob * tf.math.exp(-log_prob) + self.sample_entropy += self.entropy_reduction(entropy) + self._inputs = tf.reshape(self.embedding(branch_id), [1, 1, -1]) + mask = tf.one_hot(branch_id, self.max_layer_choice) + return tf.cast(tf.reshape(mask, [-1]), tf.bool) + + def _sample_input_choice(self, mutable): + query, anchors = [], [] + for label in mutable.choose_from: + if label not in self._anchors_hid: + self._anchors_hid[label] = self.lstm(self._inputs) + query.append(self.attn_anchor(self._anchors_hid[label])) + anchors.append(self._anchors_hid[label]) + query = tf.concat(query, axis=0) + query = tf.tanh(query + self.attn_query(anchors[-1])) + query = self.v_attn(query) + + if self.temperature is not None: + query /= self.temperature + if self.tanh_constant is not None: + query = self.tanh_constant * tf.tanh(query) + + if mutable.n_chosen is None: + logit = tf.concat([-query, query], axis=1) + softmax_logit = tf.math.log(tf.nn.softmax(logit, axis=-1)) + skip = tf.reshape(tf.random.categorical(softmax_logit, num_samples=1), [-1]) + skip_prob = tf.math.sigmoid(logit) + kl = tf.reduce_sum(skip_prob * tf.math.log(skip_prob / self.skip_targets)) + self.sample_skip_penalty += kl + log_prob = self.cross_entropy_loss(skip, logit) + + skip = tf.cast(skip, tf.float32) + inputs = tf.tensordot(skip, tf.concat(anchors, 0), 1) / (1. + tf.reduce_sum(skip)) + self._inputs = tf.reshape(inputs, [1, 1, -1]) + + else: + assert mutable.n_chosen == 1, "Input choice must select exactly one or any in ENAS." + logit = tf.reshape(query, [1, -1]) + softmax_logit = tf.math.log(tf.nn.softmax(logit, axis=-1)) + index = tf.reshape(tf.random.categorical(softmax_logit, num_samples=1), [-1]) + skip = tf.reshape(tf.one_hot(index, mutable.n_candidates), [-1]) + # when the size is 1, tf does not accept tensor here, complaining the shape is wrong + # but using a numpy array seems fine + log_prob = self.cross_entropy_loss(logit, query.numpy()) + self._inputs = tf.reshape(anchors[index.numpy()[0]], [1, 1, -1]) + + self.sample_log_prob += self.entropy_reduction(log_prob) + entropy = log_prob * tf.exp(-log_prob) + self.sample_entropy += self.entropy_reduction(entropy) + assert len(skip) == mutable.n_candidates, (skip, mutable.n_candidates, mutable.n_chosen) + return tf.cast(skip, tf.bool) diff --git a/utils/third_party/nni/algorithms/nas/tensorflow/enas/trainer.py b/utils/third_party/nni/algorithms/nas/tensorflow/enas/trainer.py new file mode 100644 index 0000000000000000000000000000000000000000..c20f20ab1a68b2c97e6bf27be0f3567016995f1d --- /dev/null +++ b/utils/third_party/nni/algorithms/nas/tensorflow/enas/trainer.py @@ -0,0 +1,203 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging + +import tensorflow as tf +from tensorflow.keras.optimizers import Adam + +from nni.nas.tensorflow.utils import AverageMeterGroup, fill_zero_grads + +from .mutator import EnasMutator + +logger = logging.getLogger(__name__) + + +class EnasTrainer: + def __init__( + self, + model, + loss, + metrics, + reward_function, + optimizer, + batch_size, + num_epochs, + dataset_train, + dataset_valid, + log_frequency=100, + entropy_weight=0.0001, + skip_weight=0.8, + baseline_decay=0.999, + child_steps=500, + mutator_lr=0.00035, + mutator_steps=50, + mutator_steps_aggregate=20, + aux_weight=0.4, + test_arc_per_epoch=1, + ): + self.model = model + self.loss = loss + self.metrics = metrics + self.reward_function = reward_function + self.optimizer = optimizer + self.batch_size = batch_size + self.num_epochs = num_epochs + + x, y = dataset_train + split = int(len(x) * 0.9) + self.train_set = tf.data.Dataset.from_tensor_slices((x[:split], y[:split])) + self.valid_set = tf.data.Dataset.from_tensor_slices((x[split:], y[split:])) + self.test_set = tf.data.Dataset.from_tensor_slices(dataset_valid) + + self.log_frequency = log_frequency + self.entropy_weight = entropy_weight + self.skip_weight = skip_weight + self.baseline_decay = baseline_decay + self.child_steps = child_steps + self.mutator_lr = mutator_lr + self.mutator_steps = mutator_steps + self.mutator_steps_aggregate = mutator_steps_aggregate + self.aux_weight = aux_weight + self.test_arc_per_epoch = test_arc_per_epoch + + self.mutator = EnasMutator(model) + self.mutator_optim = Adam(learning_rate=self.mutator_lr) + + self.baseline = 0.0 + + def train(self, validate=True): + for epoch in range(self.num_epochs): + logger.info("Epoch %d Training", epoch + 1) + self.train_one_epoch(epoch) + logger.info("Epoch %d Validating", epoch + 1) + self.validate_one_epoch(epoch) + + def validate(self): + self.validate_one_epoch(-1) + + def train_one_epoch(self, epoch): + train_loader, valid_loader = self._create_train_loader() + + # Sample model and train + meters = AverageMeterGroup() + + for step in range(1, self.child_steps + 1): + x, y = next(train_loader) + self.mutator.reset() + + with tf.GradientTape() as tape: + logits = self.model(x, training=True) + if isinstance(logits, tuple): + logits, aux_logits = logits + aux_loss = self.loss(aux_logits, y) + else: + aux_loss = 0.0 + metrics = self.metrics(y, logits) + loss = self.loss(y, logits) + self.aux_weight * aux_loss + + grads = tape.gradient(loss, self.model.trainable_weights) + grads = fill_zero_grads(grads, self.model.trainable_weights) + grads, _ = tf.clip_by_global_norm(grads, 5.0) + self.optimizer.apply_gradients(zip(grads, self.model.trainable_weights)) + + metrics["loss"] = tf.reduce_mean(loss).numpy() + meters.update(metrics) + + if self.log_frequency and step % self.log_frequency == 0: + logger.info( + "Model Epoch [%d/%d] Step [%d/%d] %s", + epoch + 1, + self.num_epochs, + step, + self.child_steps, + meters, + ) + + # Train sampler (mutator) + meters = AverageMeterGroup() + for mutator_step in range(1, self.mutator_steps + 1): + grads_list = [] + for step in range(1, self.mutator_steps_aggregate + 1): + with tf.GradientTape() as tape: + x, y = next(valid_loader) + self.mutator.reset() + + logits = self.model(x, training=False) + metrics = self.metrics(y, logits) + reward = ( + self.reward_function(y, logits) + + self.entropy_weight * self.mutator.sample_entropy + ) + self.baseline = self.baseline * self.baseline_decay + reward * ( + 1 - self.baseline_decay + ) + loss = self.mutator.sample_log_prob * (reward - self.baseline) + loss += self.skip_weight * self.mutator.sample_skip_penalty + + meters.update( + { + "reward": reward, + "loss": tf.reduce_mean(loss).numpy(), + "ent": self.mutator.sample_entropy.numpy(), + "log_prob": self.mutator.sample_log_prob.numpy(), + "baseline": self.baseline, + "skip": self.mutator.sample_skip_penalty, + } + ) + + cur_step = step + (mutator_step - 1) * self.mutator_steps_aggregate + if self.log_frequency and cur_step % self.log_frequency == 0: + logger.info( + "RL Epoch [%d/%d] Step [%d/%d] [%d/%d] %s", + epoch + 1, + self.num_epochs, + mutator_step, + self.mutator_steps, + step, + self.mutator_steps_aggregate, + meters, + ) + + grads = tape.gradient(loss, self.mutator.trainable_weights) + grads = fill_zero_grads(grads, self.mutator.trainable_weights) + grads_list.append(grads) + total_grads = [ + tf.math.add_n(weight_grads) for weight_grads in zip(*grads_list) + ] + total_grads, _ = tf.clip_by_global_norm(total_grads, 5.0) + self.mutator_optim.apply_gradients( + zip(total_grads, self.mutator.trainable_weights) + ) + + def validate_one_epoch(self, epoch): + test_loader = self._create_validate_loader() + + for arc_id in range(self.test_arc_per_epoch): + meters = AverageMeterGroup() + for x, y in test_loader: + self.mutator.reset() + logits = self.model(x, training=False) + if isinstance(logits, tuple): + logits, _ = logits + metrics = self.metrics(y, logits) + loss = self.loss(y, logits) + metrics["loss"] = tf.reduce_mean(loss).numpy() + meters.update(metrics) + + logger.info( + "Test Epoch [%d/%d] Arc [%d/%d] Summary %s", + epoch + 1, + self.num_epochs, + arc_id + 1, + self.test_arc_per_epoch, + meters.summary(), + ) + + def _create_train_loader(self): + train_set = self.train_set.shuffle(1000000).repeat().batch(self.batch_size) + test_set = self.valid_set.shuffle(1000000).repeat().batch(self.batch_size) + return iter(train_set), iter(test_set) + + def _create_validate_loader(self): + return iter(self.test_set.shuffle(1000000).batch(self.batch_size)) diff --git a/utils/third_party/nni/assessor.py b/utils/third_party/nni/assessor.py new file mode 100644 index 0000000000000000000000000000000000000000..b70995cbad6b479d8462967fa17c0f4b7fef8ce9 --- /dev/null +++ b/utils/third_party/nni/assessor.py @@ -0,0 +1,124 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +Assessor analyzes trial's intermediate results (e.g., periodically evaluated accuracy on test dataset) +to tell whether this trial can be early stopped or not. + +See :class:`Assessor`' specification and ``docs/en_US/assessors.rst`` for details. +""" + +from enum import Enum +import logging + +from .recoverable import Recoverable + +__all__ = ['AssessResult', 'Assessor'] + +_logger = logging.getLogger(__name__) + + +class AssessResult(Enum): + """ + Enum class for :meth:`Assessor.assess_trial` return value. + """ + + Good = True + """The trial works well.""" + + Bad = False + """The trial works poorly and should be early stopped.""" + + +class Assessor(Recoverable): + """ + Assessor analyzes trial's intermediate results (e.g., periodically evaluated accuracy on test dataset) + to tell whether this trial can be early stopped or not. + + This is the abstract base class for all assessors. + Early stopping algorithms should inherit this class and override :meth:`assess_trial` method, + which receives intermediate results from trials and give an assessing result. + + If :meth:`assess_trial` returns :obj:`AssessResult.Bad` for a trial, + it hints NNI framework that the trial is likely to result in a poor final accuracy, + and therefore should be killed to save resource. + + If an accessor want's to be notified when a trial ends, it can also override :meth:`trial_end`. + + To write a new assessor, you can reference :class:`~nni.medianstop_assessor.MedianstopAssessor`'s code as an example. + + See Also + -------- + Builtin assessors: + :class:`~nni.algorithms.hpo.medianstop_assessor.MedianstopAssessor` + :class:`~nni.algorithms.hpo.curvefitting_assessor.CurvefittingAssessor` + """ + + def assess_trial(self, trial_job_id, trial_history): + """ + Abstract method for determining whether a trial should be killed. Must override. + + The NNI framework has little guarantee on ``trial_history``. + This method is not guaranteed to be invoked for each time ``trial_history`` get updated. + It is also possible that a trial's history keeps updating after receiving a bad result. + And if the trial failed and retried, ``trial_history`` may be inconsistent with its previous value. + + The only guarantee is that ``trial_history`` is always growing. + It will not be empty and will always be longer than previous value. + + This is an example of how :meth:`assess_trial` get invoked sequentially: + + :: + + trial_job_id | trial_history | return value + ------------ | --------------- | ------------ + Trial_A | [1.0, 2.0] | Good + Trial_B | [1.5, 1.3] | Bad + Trial_B | [1.5, 1.3, 1.9] | Good + Trial_A | [0.9, 1.8, 2.3] | Good + + Parameters + ---------- + trial_job_id : str + Unique identifier of the trial. + trial_history : list + Intermediate results of this trial. The element type is decided by trial code. + + Returns + ------- + AssessResult + :obj:`AssessResult.Good` or :obj:`AssessResult.Bad`. + """ + raise NotImplementedError('Assessor: assess_trial not implemented') + + def trial_end(self, trial_job_id, success): + """ + Abstract method invoked when a trial is completed or terminated. Do nothing by default. + + Parameters + ---------- + trial_job_id : str + Unique identifier of the trial. + success : bool + True if the trial successfully completed; False if failed or terminated. + """ + + def load_checkpoint(self): + """ + Internal API under revising, not recommended for end users. + """ + checkpoin_path = self.get_checkpoint_path() + _logger.info('Load checkpoint ignored by assessor, checkpoint path: %s', checkpoin_path) + + def save_checkpoint(self): + """ + Internal API under revising, not recommended for end users. + """ + checkpoin_path = self.get_checkpoint_path() + _logger.info('Save checkpoint ignored by assessor, checkpoint path: %s', checkpoin_path) + + def _on_exit(self): + pass + + def _on_error(self): + pass diff --git a/utils/third_party/nni/common/__init__.py b/utils/third_party/nni/common/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/utils/third_party/nni/common/graph_utils.py b/utils/third_party/nni/common/graph_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..0037afba9f11864213e10e9ab187e8647b36d425 --- /dev/null +++ b/utils/third_party/nni/common/graph_utils.py @@ -0,0 +1,789 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + + +import logging +import queue +import re +from collections import defaultdict +import torch +from torch.utils.tensorboard._pytorch_graph import NodePy, NodePyIO, NodePyOP, GraphPy +CLASSTYPE_KIND = 'ClassType' +GETATTR_KIND = 'prim::GetAttr' +CAT_KIND = 'aten::cat' +LIST_CONSTRUCT_KIND = 'prim::ListConstruct' +LIST_UNPACK_KIND = 'prim::ListUnpack' +TUPLE_CONSTRUCT_KIND = 'prim::TupleConstruct' +TUPLE_UNPACK_KIND = 'prim::TupleUnpack' + +_logger = logging.getLogger(__name__) + + +def build_module_graph(model, dummy_input): + return TorchModuleGraph(model, dummy_input) + + +def build_graph(model, dummy_input, verbose=False): + g = TorchProtoGraph(model, dummy_input, verbose) + return g.graph_def, g.stepstats + + +def parse_traced_name(module_name): + prefix = 'TracedModule[' + suffix = ']' + if module_name.startswith(prefix) and module_name.endswith(suffix): + module_name = module_name[len(prefix):-len(suffix)] + return module_name + + +class TorchGraph: + """ + This class is to extract pytorch model topology graph by tracing + """ + + def __init__(self, model=None, dummy_input=None, traced_model=None): + """ + Parameters + ---------- + model : pytorch model + The model user wants to speed up + dummy_input : pytorch tensor + The dummy input for ```jit.trace```, users should put it on right device before pass in + traced_model : torch._C.torch.jit.TopLevelTracedModule + An alredy traced model, if traced_model is not None, then TorchGraph will build the graph + based on this traced model and won't trace the model again. + """ + assert torch.__version__ >= '1.3.1' + # check if the input is legal + if traced_model is not None: + assert isinstance(traced_model, torch.jit.TopLevelTracedModule) + self.trace = traced_model + # it's ok if the graph is already unpacked + torch._C._jit_pass_inline(self.trace.graph) + elif model is not None and dummy_input is not None: + self.bound_model = model + self._trace(model, dummy_input) + else: + raise Exception( + 'Please provide model & dummy_input or the traced_model as inputs') + + def _trace(self, model, dummy_input): + with torch.onnx.set_training(model, False): + # import torch.jit + self.trace = torch.jit.trace(model, dummy_input, check_trace=False) + torch._C._jit_pass_inline(self.trace.graph) + + +class TorchProtoGraph(TorchGraph): + """ + Generates model graph for pytorch models in protobuf, this implementation + is borrowed from pytorch v1.4.0, and fixed following issues: + https://github.com/pytorch/pytorch/issues/33691 + https://github.com/pytorch/pytorch/issues/33670 + + """ + + def __init__(self, model, dummy_input, verbose=False): + super().__init__(model, dummy_input) + + from tensorboard.compat.proto.config_pb2 import RunMetadata + from tensorboard.compat.proto.graph_pb2 import GraphDef + from tensorboard.compat.proto.step_stats_pb2 import StepStats, DeviceStepStats + from tensorboard.compat.proto.versions_pb2 import VersionDef + + list_of_nodes = self.parse(self.trace.graph, self.trace, dummy_input) + if verbose: + print(self.trace.graph) + self.stepstats = RunMetadata(step_stats=StepStats( + dev_stats=[DeviceStepStats(device="/device:CPU:0")])) + self.graph_def = GraphDef( + node=list_of_nodes, versions=VersionDef(producer=22)) + + def parse(self, graph, trace, args=None, omit_useless_nodes=True): + """This method parses an optimized PyTorch model graph and produces + a list of nodes and node stats for eventual conversion to TensorBoard + protobuf format. + + Args: + graph (PyTorch module): The model graph to be parsed. + trace (PyTorch JIT TracedModule): The model trace to be parsed. + args (tuple): input tensor[s] for the model. + omit_useless_nodes (boolean): Whether to remove nodes from the graph. + """ + nodes_py = GraphPy() + for node in graph.inputs(): + if omit_useless_nodes: + if not node.uses(): # number of user of the node (= number of outputs/ fanout) + continue + + if node.type().kind() != CLASSTYPE_KIND: + nodes_py.append(NodePyIO(node, 'input')) + + attr_to_scope = dict() + + def node_to_name(d): + return str(d).split(":")[0].strip() + for node in graph.nodes(): + if node.kind() == GETATTR_KIND: + attr_name = node.s('name') + node_name = node_to_name(node) + parent = node.input().node() + # If the parent node is not the top-level "self" node + if parent.kind() == GETATTR_KIND: + parent_scope = attr_to_scope[node_to_name(parent)] + attr_scope = parent_scope.split('/')[-1] + attr_to_scope[node_name] = '{}/{}.{}'.format( + parent_scope, attr_scope, attr_name) + else: + attr_to_scope[node_name] = '__module.{}'.format(attr_name) + # We don't need classtype nodes; scope will provide this information + if node.output().type().kind() != CLASSTYPE_KIND: + node_py = NodePyOP(node) + node_py.scopeName = attr_to_scope[node_name] + nodes_py.append(node_py) + else: + nodes_py.append(NodePyOP(node)) + + # Create sink nodes for output ops + for i, node in enumerate(graph.outputs()): + node_py = NodePyIO(node, 'output') + node_py.debugName = "output.{}".format(i + 1) + node_py.inputs = [node.debugName()] + nodes_py.append(node_py) + + alias_to_name = dict() + base_name = parse_traced_name(trace._name) + for name, module in trace.named_modules(prefix='__module'): + mod_name = parse_traced_name(module._name) + attr_name = name.split('.')[-1] + alias_to_name[name] = '{}[{}]'.format(mod_name, attr_name) + + for node in nodes_py.nodes_op: + module_aliases = node.scopeName.split('/')[-1].split('.') + module_name = '' + for i, alias in enumerate(module_aliases): + if i == 0: + module_name = alias + node.scopeName = base_name + else: + module_name += '.' + alias + node.scopeName += '/' + \ + (alias_to_name[module_name] + if module_name in alias_to_name else alias) + + nodes_py.populate_namespace_from_OP_to_IO() + return nodes_py.to_proto() + + +class NodePyGroup(NodePy): + """ + This class is used to represent a graph node which consists of multiple jit traced nodes. In a pytorch trace graph, + there are multiple nodes are traced for one torch.nn.Module object, we group them together to form a single node to + represent the torch.nn.Module object. We also group some functional call trace nodes together to form a new node. + """ + + def __init__(self, name, unique_name, node_type, op_type, node_cpps, inputs=None, outputs=None, key_node=None): + """ + Parameters: + ----------- + name: str + node name, such as `conv1`, `backbone.classifier` + unique_name: str + A global unique name for current node. Due to some modules, + such as relu, may be reused several times, so the scopename + is not suitable as the global unique identifier, so we add a + unique_name for each node as the global unique identifier. + We should use the unique_name to traverset the module graph. + node_type: str + `module` or `func` + op_type: str + operation type, such as `Conv2d`, `aten::view` + node_cpps: list of torch._C.Node + jit trace nodes which are included in this new node + inputs: list of str + All the inputs of this node, each element is debugName of one input + outputs: list of str + All the outputs of this node, each element is debugName of one output + key_node: torch._C.Node + The key node of this NodePyGroup. + """ + super(NodePyGroup, self).__init__(name, []) + self.node_cpps = node_cpps + self.name = name + self.unique_name = unique_name + self.op_type = op_type + self.type = node_type + self.nodes = [] + self.auxiliary = None + self.add_nodes(node_cpps) + self.inputs = inputs + self.outputs = outputs + # The core node in this NodePyGroup + self.key_node = key_node + + def add_nodes(self, node_cpps): + for node_cpp in node_cpps: + nodepy = NodePyOP(node_cpp) + nodepy.name = node_cpp.scopeName() + '_' + node_cpp.kind() + self.nodes.append(nodepy) + + def sub_node_names(self): + return [x.name for x in self.nodes] + + def __repr__(self): + return 'name: {}, type: {}, op_type: {}, sub_nodes: {}, inputs: {}, outputs: {}, aux: {}'.format( + self.name, self.type, self.op_type, self.sub_node_names(), + self.inputs, self.outputs, self.auxiliary + ) + + +class TorchModuleGraph(TorchGraph): + """ + Generates model graph, each node is created from single or multiple jit trace nodes. + """ + + def __init__(self, model=None, dummy_input=None, traced_model=None): + super().__init__(model, dummy_input, traced_model) + self.global_count = 0 + self.name_to_node, self.input_to_node, self.output_to_node = self._build_graph() + self._extract_auxiliary_info() + + def _expand_key_func_node(self, node, nodes, input_to_node, output_to_node, + module_type): + """ + For trace graph nodes, some nodes are not in modules, these nodes are usually generated by + the functions directly called in module ```forward```. For such nodes, some of them are + trivial op which are label by ```prim::```, some of them are not such ops which is call + non-prim ops. This function is to merge neighbor prim ops to a non-prim op, to construct + a node. + + Parameters + ---------- + node : trace graph node + The non-prim node to expand + nodes : list of trace graph node + All the trace graph nodes within the same scope as the non-prim node + input_to_node : dict + key: input name, value: a node that uses this input + output_to_node : dict + key: output name, value: a node that generates this output + module_type : str + can be 'module' or 'func' + + Returns + ------- + node + the expanded non-prim node + """ + # TODO: scope name could be empty + node_name = '.'.join([self._get_module_name( + node.scopeName()), node.kind(), str(self.global_count)]) + unique_name = node_name + _logger.debug("expand non-prim node, node name: %s", node_name) + self.global_count += 1 + op_type = node.kind() + node_group = [node] + inputs = list() + outputs = list() + node_queue = queue.Queue() + node_queue.put(node) + while not node_queue.empty(): + curr_node = node_queue.get() + for _input in curr_node.inputs(): + input_name = _input.debugName() + if input_name in output_to_node and output_to_node[input_name] in nodes: + predecessor_node = output_to_node[input_name] + if not self._is_key_func(predecessor_node): + node_group.append(predecessor_node) + node_queue.put(predecessor_node) + else: + inputs.append(input_name) + else: + inputs.append(input_name) + for output in node.outputs(): + outputs.append(output.debugName()) + nodepy = NodePyGroup(node_name, unique_name, module_type, op_type, + node_group, inputs=inputs, outputs=outputs, key_node=node) + return nodepy + + def _expand_module_node(self, node, node_name, unique_name, op_type, nodes, + input_to_node, output_to_node, module_type): + """ + merge the adjacent nodes of the module. The difference between the + _expand_module_node and _expand_non_prim_node is that, the _expand_non_prim_node + only merge the prim:: nodes into the aten:: node, in contrast,the _expand_module_node + will merge all adjacent nodes into a same nodepy group. + + Parameters + ---------- + node : trace graph node + The non-prim node to expand + node_name : str + specify the node_name for NodePyGroup + unique_name : str + unique_name for the NodePyGroup + op_type : str + specify the op_type for the NodePyGroup + nodes : list of trace graph node + All the trace graph nodes within the same scope as the non-prim node + input_to_node : dict + key: input name, value: a node that uses this input + output_to_node : dict + key: output name, value: a node that generates this output + module_type : str + can be 'module' or 'func' + Returns + ------- + node + the expanded non-prim node + + """ + _logger.debug("expand module node, node name: %s", node_name) + self.global_count += 1 + if not op_type: + op_type = node.kind() + node_group = [node] + inputs = list() + outputs = list() + node_queue = queue.Queue() + node_queue.put(node) + visited = {node} + while not node_queue.empty(): + curr_node = node_queue.get() + for _input in curr_node.inputs(): + input_name = _input.debugName() + if input_name in output_to_node and output_to_node[input_name] in nodes: + predecessor_node = output_to_node[input_name] + if predecessor_node not in visited: + node_group.append(predecessor_node) + node_queue.put(predecessor_node) + visited.add(predecessor_node) + else: + inputs.append(input_name) + for _output in curr_node.outputs(): + output_name = _output.debugName() + if output_name in input_to_node and input_to_node[output_name] in nodes: + successor_node = input_to_node[output_name] + if successor_node not in visited: + node_group.append(successor_node) + node_queue.put(successor_node) + visited.add(successor_node) + else: + outputs.append(output_name) + + nodepy = NodePyGroup(node_name, unique_name, module_type, op_type, + node_group, inputs=inputs, outputs=outputs) + return nodepy + + def _extract_cat_info(self, node_group, cpp_node): + """ + Extract the detail information of the cat operation, + such the order of the input tensor, the shape of each + input tensor, the output shape, and the cat dimension. + + Parameters + ---------- + node_group : NodePyGroup + cpp_node: torch._C.Node + It should be ```aten::cat``` node + + Returns + ------- + dict + Include auxiliary information for the cat operation. + This dict objec has four keys: 'cat_dim', 'out_shape', + 'in_order' and 'in_shape'. cat_dim is the dimension of + the cat operation to concat the input tensors. out_shape + is the shape of the output tensor of the cat operation. + in_order is an ordered list which contains the corresponding + parent operaion nodes of the input tensors. in_shape is also + an ordered list that contains the input shapes of the input + tensor. + """ + # only suport the cat operation + assert cpp_node.kind() == CAT_KIND + cat_info = {} + # get the shape of the output tensor + t_output = cpp_node.output() + out_shape = t_output.type().sizes() + cat_info['out_shape'] = out_shape + # get the cat dimension + inputs = cpp_node.inputs() + cat_dim = list(inputs)[1].toIValue() + cat_info['cat_dim'] = cat_dim + # get the order of the input tensors + # To get the order of the input tensors, we need + # to be aware of the topology of the model, which + # means we should extract the auxiliary information + # after the build_index function. + input_order = [] + list_construct_cpp = list(cpp_node.inputs())[0].node() + input_tensors = list(list_construct_cpp.inputs()) + for _tensor in input_tensors: + debug_name = _tensor.debugName() + input_order.append(self.output_to_node[debug_name].unique_name) + cat_info['in_order'] = input_order + input_shapes = [t.type().sizes() for t in input_tensors] + cat_info['in_shape'] = input_shapes + return cat_info + + def _extract_linear_shape_info(self, node_group): + """ + Extract linear shape input/output tensor shape info from its aten::addmm op. + + Parameters + ---------- + node_group : NodePyGroup + NodePyGroup object associated with the linear module. + + Returns + ------- + dict + Include shape of input tensor and shape of output tensor + """ + for cpp_node in node_group.node_cpps: + if cpp_node.kind() == 'aten::addmm': + # https://github.com/pytorch/pytorch/blob/1.6/torch/nn/functional.py#L1682 + # inputs of aten::addmm: + # inputs[0] is bias + # inputs[1] is input data + # inputs[2] is weight + t_input = list(cpp_node.inputs())[1] + t_output = cpp_node.output() + assert isinstance(t_input.type(), torch._C.TensorType) + assert isinstance(t_output.type(), torch._C.TensorType) + in_shape = t_input.type().sizes() + out_shape = t_output.type().sizes() + return {'in_shape': in_shape, 'out_shape': out_shape} + return None + + def _extract_shape_info(self, node): + """ + Extract the shape information of ```aten::view``` node + + Parameters + ---------- + node : trace graph node + It should be ```aten::view``` node + + Returns + ------- + dict + Include shape of input tensor and shape of output tensor + """ + t_input = None + for _input in node.inputs(): + t_input = _input + break + t_output = node.output() + assert isinstance(t_input.type(), torch._C.TensorType) + assert isinstance(t_output.type(), torch._C.TensorType) + in_shape = t_input.type().sizes() + out_shape = t_output.type().sizes() + return {'in_shape': in_shape, 'out_shape': out_shape} + + def _extract_leaf_modules(self): + """ + Extract leaf modules from the given graph. Leaf module means it does not have submodules. + To extract leaf modules because only leaf module can be replaced. And shape inference can + be done in leaf module level. Other shape inference is done in lower level i.e., + operation level. + + Returns + ------- + list + a list of scope name of all the leaf modules + """ + def is_parent(name1, name2): + """ + check if name1 is parent node of name2, for example: + name1: aa.bb, name2: aa.bb.cc, return True + name1: aa.b, name2: aa.bb, return False + """ + parts1, parts2 = name1.split('.'), name2.split('.') + if len(parts1) >= len(parts2): + return False + for i, _ in enumerate(parts1): + if parts2[i] != parts1[i]: + return False + return True + module_names = sorted([x[0] + for x in self.trace.named_modules() if x[0]]) + leaf_nodes = [] + for i, name in enumerate(module_names): + if i + 1 >= len(module_names) or not is_parent(name, module_names[i + 1]): + leaf_nodes.append(name) + return leaf_nodes + + def _get_module_name(self, scope_name): + """ + Retrieve module name from scope name. + Parameters: + ----------- + scope_name: str + scope_name of a graph node, for example: + for pytorch 1.3.1: MyModel/BackboneModel[backbone]/Conv2d[conv2] + for pytorch 1.4.0: __module.backbone/__module.backbone.conv2 + + Returns: + ------- + str + module name, such as backbone.conv2 + """ + if torch.__version__ >= '1.4.0': + return scope_name.split('/')[-1].replace('__module.', '') + else: + return '.'.join(re.findall(r'\[(.*?)\]', scope_name)) + + def _build_index(self, nodes_op): + name_to_node = dict() + input_to_node = defaultdict(list) + output_to_node = dict() + for node in nodes_op: + name_to_node[node.unique_name] = node + for _input in node.inputs: + input_to_node[_input].append(node) + for output in node.outputs: + assert not output in output_to_node, \ + "One output cannot be generated by multiple nodes" + output_to_node[output] = node + return name_to_node, input_to_node, output_to_node + + def _is_key_func(self, node_cpp): + """ + Judge if a cpp node is a key function node. + If so, we should not merge this node into the + adjacent node. + """ + if node_cpp.kind().startswith('aten::'): + # the nodes that start with 'aten' are key function + # nodes + return True + if node_cpp.kind() in [LIST_UNPACK_KIND, TUPLE_UNPACK_KIND]: + # We cannot merge the List/Tuple + # Unpack func into other nodes, else it + # may lead to a graph construction error. + # The reason why we donnot take the construct node + # also as a key node is that `cat` operation node need + # the last(previous) visited node to infer the mask. If + # we take the Construct node as the important node, the + # predecessor of the `cat` node will always be a construct + # node, which means we cannot infer the mask for the cat + # operation. + return True + return False + + def unpack_manually(self): + """ + Unpack the tensor tuple or tensor list manually, + and remove the ListUnpack/TupleUnpack node from + the graph. Note: this function will change the + graph structure. + """ + if hasattr(self, 'unpacked'): + # if already unpacked the tuple/list manually + return + for node in self.nodes_py.nodes_op: + if node.op_type in [TUPLE_UNPACK_KIND, LIST_UNPACK_KIND]: + unpack_cpp = node.key_node + last_cpp = list(unpack_cpp.inputs())[0].node() + if last_cpp.kind() in [TUPLE_CONSTRUCT_KIND, LIST_CONSTRUCT_KIND]: + # we need check if the tensor tuple or tensor list is produced + # by a list/tuple construct node. If so, we can unpack the tuple + # or list manunally. + _logger.debug('List/Tuple Construct Node(cpp) %s', str(last_cpp)) + _logger.debug('List/Tuple Unpack Node(cpp) %s', str(unpack_cpp)) + assert len(list(unpack_cpp.outputs())) == len(list(last_cpp.inputs())) + errmsg = '%s Input number: %d if inconsistent with the output number %d' % (unpack_cpp, \ + len(node.inputs), len(list(last_cpp.inputs()))) + + assert len(node.inputs) == len(list(last_cpp.inputs())), errmsg + for _debug_input, _debug_output in zip(node.inputs, node.outputs): + # _debug_input = _input.debugName() + # _debug_output = _output.debugName() + if _debug_input in self.input_to_node and _debug_output in self.input_to_node: + # input_to_node[_debug_input] is a list of NodePyGroup, because + # one tensor can be used as input for multiple nodes at the same time. + + # note that, in this case, the construct cpp node and unpack cpp node + # will be merged into the same NodePyGroup, so we remove the `node` from + # input_to_node[_debug_input] and directly connect this tensor to the + # input_to_node[_debug_output] + self.input_to_node[_debug_input].remove(node) + # add the following nodes of _output into the input_to_node[_debug_input] + self.input_to_node[_debug_input].extend(self.input_to_node[_debug_output]) + # just remove the _debug_output from the grapgh index. So that we can also skip + # the construct and tuple + if _debug_output in self.input_to_node: + for following_node in self.input_to_node[_debug_output]: + _tmp_index = following_node.inputs.index(_debug_output) + following_node.inputs[_tmp_index] = _debug_input + + + self.unpacked = True + + def _build_graph(self): + """ + Build graph using our defined format from jit trace. + There are basically three steps: first, construct necessary information (data structures), + second, extract all the modules to convert to node, Third, extract all functions to convert + to node. + + Returns + ------- + dict + use name to index nodes, key: node name, value: node + dict + use input (its name) to index nodes, + key: input, value: list of nodes that take this input + dict + use output (its name) to index nodes, + key: output, value: node that generates this output + """ + omit_useless_nodes = True + graph = self.trace.graph + # _logger.debug(graph) + # build output mapping, from output debugName to its node + output_to_node = {x.debugName(): n for n in graph.nodes() + for x in n.outputs()} + # build input mapping, from input debugName to its node + input_to_node = {x.debugName(): n for n in graph.nodes() + for x in n.inputs()} + # build module mapping, from module name to all nodes (as list) under this module scope + module_to_nodes = defaultdict(list) + # the mapping of function (non-module in forward) to nodes, key is scope name + func_to_nodes = defaultdict(list) + + nodes_py = GraphPy() + for node in graph.inputs(): + if omit_useless_nodes: + if not node.uses(): # number of user of the node (= number of outputs/ fanout) + continue + + if node.type().kind() != 'ClassType': + nodes_py.append(NodePyIO(node, 'input')) + + self.leaf_modules = self._extract_leaf_modules() + module_to_type = {name: parse_traced_name( + module._name) for name, module in self.trace.named_modules()} + + # associate module name with their trace graph nodes + for node in graph.nodes(): + module_name = self._get_module_name(node.scopeName()) + if module_name in self.leaf_modules: + module_to_nodes[module_name].append(node) + else: + func_to_nodes[node.scopeName()].append(node) + # build node group for module + for module_name, node_cpps in module_to_nodes.items(): + use_count = 0 + merged = set() + for node in node_cpps: + if node not in merged: + # modules that have same scope name may have different locations in the + # graph. Futhermore, there are also lots of prim:: nodes that in node_cpps, + # so we also need to call the expand_module_node. + unique_name = module_name + if use_count > 0: + unique_name = module_name + '.%d' % use_count + node_group = self._expand_module_node( + node, module_name, unique_name, module_to_type[module_name], + node_cpps, input_to_node, output_to_node, 'module') + nodes_py.nodes_op.append(node_group) + use_count += 1 + merged.update(node_group.node_cpps) + + # each scope_name may have multiple funcs, we split them and create node for each of them + # build node group for torch.nn.functional + for _, nodes in func_to_nodes.items(): + # extract non prim:: nodes + key_func_nodes = list() + for node in nodes: + if self._is_key_func(node): + # find the key function nodes + key_func_nodes.append(node) + # for each non prim node, expand it + for node in key_func_nodes: + node_group = self._expand_key_func_node( + node, nodes, input_to_node, output_to_node, 'func') + nodes_py.nodes_op.append(node_group) + # get shape infor for view (aten::view) func + # if node_group.op_type in ['aten::view', 'aten::flatten']: + # node_group.auxiliary = self._extract_shape_info(node) + + for node in graph.outputs(): # Create sink nodes for output ops + node_py = NodePyIO(node, 'output') + nodes_py.append(node_py) + + self.nodes_py = nodes_py + # build index + return self._build_index(self.nodes_py.nodes_op) + + def _extract_auxiliary_info(self): + """ + Extract the auxiliary information for the nodegroups + if necessary. For example, view/flatten operations may + need the shape of the input tensor and output tensor. + """ + # extract the input & output shape for the view and flatten + for node_group in self.nodes_py.nodes_op: + if node_group.op_type in ['aten::view', 'aten::flatten', 'aten::mean', 'aten::reshape']: + # get shape infor for view (aten::view) func + cpp_node = list(filter(lambda x: x.kind() == node_group.op_type, + node_group.node_cpps))[0] + node_group.auxiliary = self._extract_shape_info(cpp_node) + elif node_group.op_type == 'Linear': + node_group.auxiliary = self._extract_linear_shape_info(node_group) + elif node_group.op_type == CAT_KIND: + # get the detail information for cat func + cpp_node = list(filter(lambda x: x.kind() == node_group.op_type, + node_group.node_cpps))[0] + node_group.auxiliary = self._extract_cat_info( + node_group, cpp_node) + + def find_predecessors(self, unique_name): + """ + Find predecessor node of the given node + + Parameters + ---------- + unique_name : str + The unique name of the node + + Returns + ------- + list + a list of nodes who are the given node's predecessor + """ + predecessors = [] + for _input in self.name_to_node[unique_name].inputs: + if not _input in self.output_to_node: + _logger.debug("cannot find node with %s as its output", _input) + else: + node_py = self.output_to_node[_input] + predecessors.append(node_py.unique_name) + return predecessors + + def find_successors(self, unique_name): + """ + Find successor nodes of the given node + + Parameters + ---------- + unique_name : str + The unique name of the node + + Returns + ------- + list + a list of nodes who are the given node's successor + """ + successors = [] + for output in self.name_to_node[unique_name].outputs: + if output not in self.input_to_node: + # may reach the output of the whole graph + continue + nodes_py = self.input_to_node[output] + for node_py in nodes_py: + successors.append(node_py.unique_name) + return successors diff --git a/utils/third_party/nni/common/nas_utils.py b/utils/third_party/nni/common/nas_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..d7f050ec9bb2046b782bf699e1bd3ae7925a31a1 --- /dev/null +++ b/utils/third_party/nni/common/nas_utils.py @@ -0,0 +1,317 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import functools +import logging + +from .. import trial + + +_logger = logging.getLogger(__name__) +_MUTABLE_LAYER_SPACE_PREFIX = "_mutable_layer" +_namespace = {} +_tf_variables = {} +_arch_logits_list = [] +_optimizer = None +_train_op = None + + +def classic_mode( + mutable_id, + mutable_layer_id, + funcs, + funcs_args, + fixed_inputs, + optional_inputs, + optional_input_size): + '''Execute the chosen function and inputs directly. + In this mode, the trial code is only running the chosen subgraph (i.e., the chosen ops and inputs), + without touching the full model graph.''' + if trial.get_current_parameter() is None: + trial.get_next_parameter() + + chosen_layer, chosen_inputs = _get_layer_and_inputs_from_tuner(mutable_id, mutable_layer_id, + list(optional_inputs.keys())) + real_chosen_inputs = [optional_inputs[input_name] for input_name in chosen_inputs] + layer_out = funcs[chosen_layer]([fixed_inputs, real_chosen_inputs], **funcs_args[chosen_layer]) + + return layer_out + + +def enas_mode( + mutable_id, + mutable_layer_id, + funcs, + funcs_args, + fixed_inputs, + optional_inputs, + optional_input_size, + tf): + '''For enas mode, we build the full model graph in trial but only run a subgraph。 + This is implemented by masking inputs and branching ops. + Specifically, based on the received subgraph (through nni.get_next_parameter), + it can be known which inputs should be masked and which op should be executed.''' + name_prefix = "{}_{}".format(mutable_id, mutable_layer_id) + # store namespace + _namespace[mutable_id] = True + _namespace[name_prefix] = dict() + _namespace[name_prefix]['funcs'] = list(funcs) + _namespace[name_prefix]['optional_inputs'] = list(optional_inputs) + # create tensorflow variables as 1/0 signals used to form subgraph + name_for_optional_inputs = name_prefix + '_optional_inputs' + name_for_funcs = name_prefix + '_funcs' + _tf_variables[name_prefix] = dict() + _tf_variables[name_prefix]['optional_inputs'] = tf.get_variable( + name_for_optional_inputs, + [len(optional_inputs)], + dtype=tf.bool, + trainable=False + ) + _tf_variables[name_prefix]['funcs'] = tf.get_variable( + name_for_funcs, [], dtype=tf.int64, trainable=False) + + # get real values using their variable names + real_optional_inputs_value = [optional_inputs[name] + for name in _namespace[name_prefix]['optional_inputs']] + real_func_value = [funcs[name] + for name in _namespace[name_prefix]['funcs']] + real_funcs_args = [funcs_args[name] + for name in _namespace[name_prefix]['funcs']] + # build tensorflow graph of geting chosen inputs by masking + real_chosen_inputs = tf.boolean_mask( + real_optional_inputs_value, _tf_variables[name_prefix]['optional_inputs']) + # build tensorflow graph of different branches by using tf.case + branches = dict() + func_output = None + for func_id in range(len(funcs)): + func_output = real_func_value[func_id]([fixed_inputs, real_chosen_inputs], **real_funcs_args[func_id]) + branches[tf.equal(_tf_variables[name_prefix]['funcs'], func_id)] = lambda: func_output + layer_out = tf.case(branches, exclusive=True, default=lambda: func_output) + + return layer_out + + +def oneshot_mode( + mutable_id, + mutable_layer_id, + funcs, + funcs_args, + fixed_inputs, + optional_inputs, + optional_input_size, + tf): + '''Similar to enas mode, oneshot mode also builds the full model graph. + The difference is that oneshot mode does not receive subgraph. + Instead, it uses dropout to randomly dropout inputs and ops.''' + # NNI requires to get_next_parameter before report a result. But the parameter will not be used in this mode + if trial.get_current_parameter() is None: + trial.get_next_parameter() + optional_inputs = list(optional_inputs.values()) + inputs_num = len(optional_inputs) + # Calculate dropout rate according to the formular r^(1/k), where r is a hyper-parameter and k is the number of inputs + if inputs_num > 0: + rate = 0.01 ** (1 / inputs_num) + noise_shape = [inputs_num] + [1] * len(optional_inputs[0].get_shape()) + optional_inputs = tf.nn.dropout( + optional_inputs, rate=rate, noise_shape=noise_shape) + optional_inputs = [optional_inputs[idx] for idx in range(inputs_num)] + layer_outs = [func([fixed_inputs, optional_inputs], **funcs_args[func_name]) + for func_name, func in funcs.items()] + output_num = len(layer_outs) + rate = 0.01 ** (1 / output_num) + noise_shape = [output_num] + [1] * len(layer_outs[0].get_shape()) + layer_outs = tf.nn.dropout(layer_outs, rate=rate, noise_shape=noise_shape) + layer_out = tf.reduce_sum(layer_outs, axis=0) + + return layer_out + + +def darts_mode( + mutable_id, + mutable_layer_id, + funcs, + funcs_args, + fixed_inputs, + optional_inputs, + optional_input_size, + tf): + optional_inputs = list(optional_inputs.values()) + layer_outs = [func([fixed_inputs, optional_inputs], **funcs_args[func_name]) + for func_name, func in funcs.items()] + # Create architecture weights for every func(op) + var_name = "{}_{}_arch_weights".format(mutable_id, mutable_layer_id) + arch_logits = tf.get_variable(var_name, shape=[len(funcs)], trainable=False) + _arch_logits_list.append(arch_logits) + arch_weights = tf.nn.softmax(arch_logits) + layer_out = tf.add_n([arch_weights[idx] * out for idx, out in enumerate(layer_outs)]) + + return layer_out + + +def reload_tensorflow_variables(tf, session): + '''In Enas mode, this function reload every signal varaible created in `enas_mode` function so + the whole tensorflow graph will be changed into certain subgraph recerived from Tuner. + --------------- + session: the tensorflow session created by users + tf: tensorflow module + ''' + subgraph_from_tuner = trial.get_next_parameter() + mutable_layers = set() + for subgraph_key in subgraph_from_tuner: + if "/" in subgraph_key: + # has to remove the last, could be layer_choice or whatever + mutable_id, mutable_layer_id = _decompose_general_key(subgraph_key[:subgraph_key.rfind("/")]) + if mutable_id is not None: + mutable_layers.add((mutable_id, mutable_layer_id)) + mutable_layers = sorted(list(mutable_layers)) + for mutable_id, mutable_layer_id in mutable_layers: + if mutable_id not in _namespace: + _logger.warning("%s not found in name space", mutable_id) + continue + name_prefix = "{}_{}".format(mutable_id, mutable_layer_id) + # get optional inputs names + optional_inputs = _namespace[name_prefix]['optional_inputs'] + # extract layer information from the subgraph sampled by tuner + chosen_layer, chosen_inputs = _get_layer_and_inputs_from_tuner(mutable_id, mutable_layer_id, optional_inputs) + chosen_layer = _namespace[name_prefix]['funcs'].index(chosen_layer) + chosen_inputs = [1 if inp in chosen_inputs else 0 for inp in optional_inputs] + # load these information into pre-defined tensorflow variables + _tf_variables[name_prefix]['funcs'].load(chosen_layer, session) + _tf_variables[name_prefix]['optional_inputs'].load( + chosen_inputs, session) + + +def _construct_general_key(mutable_id, mutable_layer_id): + # Mutable layer key in a general (search space) format + # that is, prefix/mutable_id/mutable_layer_id + return _MUTABLE_LAYER_SPACE_PREFIX + "/" + mutable_id + "/" + mutable_layer_id + + +def _decompose_general_key(key): + # inverse operation of above + if not key.startswith(_MUTABLE_LAYER_SPACE_PREFIX): + return None, None + else: + _, mutable_id, mutable_layer_id = key.split("/", maxsplit=2) + return mutable_id, mutable_layer_id + + +def darts_training(tf, session, loss, feed_dict): + global _optimizer, _train_op + if _optimizer is None: + _optimizer = tf.MomentumOptimizer(learning_rate=0.025) + # TODO: Calculate loss + grads_and_vars = _optimizer.compute_gradients(loss, _arch_logits_list) + _train_op = _optimizer.apply_gradients(grads_and_vars) + session.run(_train_op) + + +def training_update(nas_mode, tf=None, session=None, loss=None, feed_dict=None): + if nas_mode == 'darts_mode': + darts_training(tf, session, loss, feed_dict) + elif nas_mode == 'enas_mode': + reload_tensorflow_variables(tf, session) + + +def _get_layer_and_inputs_from_tuner(mutable_id, mutable_layer_id, optional_inputs): + # optional_inputs should be name(key)s of the optional inputs + try: + mutable_block = trial.get_current_parameter(mutable_id) + + # There is a NAS tuner + chosen_layer = mutable_block[mutable_layer_id]["chosen_layer"] + chosen_inputs = mutable_block[mutable_layer_id]["chosen_inputs"] + except KeyError: + # Try to find converted NAS parameters + params = trial.get_current_parameter() + expected_prefix = _construct_general_key(mutable_id, mutable_layer_id) + chosen_layer = params[expected_prefix + "/layer_choice"] + + # find how many to choose + optional_input_size = int(params[expected_prefix + "/optional_input_size"]) # convert uniform to randint + + # find who to choose, can duplicate + optional_input_state = params[expected_prefix + "/optional_input_chosen_state"] + chosen_inputs = [] + # make sure dict -> list produce stable result by sorting + optional_inputs_keys = sorted(optional_inputs) + for _ in range(optional_input_size): + chosen_inputs.append(optional_inputs_keys[optional_input_state % len(optional_inputs)]) + optional_input_state //= len(optional_inputs) + + _logger.info("%s_%s: layer: %s, optional inputs: %s", mutable_id, mutable_layer_id, chosen_layer, chosen_inputs) + return chosen_layer, chosen_inputs + + +def convert_nas_search_space(search_space): + """ + Args: + param search_space: raw search space + return: the new search space, mutable_layers will be converted into choice + """ + if not isinstance(search_space, dict): + return search_space + ret = dict() + for k, v in search_space.items(): + if "_type" not in v: + # this should not happen + _logger.warning("There is no _type in one of your search space values with key '%s'" + ". Please check your search space", k) + ret[k] = v + elif v["_type"] != "mutable_layer": + ret[k] = v + else: + _logger.info("Converting mutable_layer search space with key '%s'", k) + # v["_value"] looks like {'mutable_layer_1': {'layer_choice': ...} ...} + values = v["_value"] + for layer_name, layer_data in values.items(): + # there should be at most layer_choice, optional_inputs, optional_input_size in layer_data + + # add "_mutable_layer" as prefix so that they can be recovered later + layer_key = _construct_general_key(k, layer_name) + + if layer_data.get("layer_choice"): # filter out empty choice and no choice + layer_choice = layer_data["layer_choice"] + else: + raise ValueError("No layer choice found in %s" % layer_key) + + if layer_data.get("optional_input_size"): + input_size = layer_data["optional_input_size"] + if isinstance(input_size, int): + input_size = [input_size, input_size] + if input_size[0] > input_size[1] or input_size[0] < 0: + _logger.error("Might not be able to handle optional_input_size < 0, please double check") + input_size[1] += 1 + else: + _logger.info("Optional input choices are set to empty by default in %s", layer_key) + input_size = [0, 1] + + if layer_data.get("optional_inputs"): + total_state_size = len(layer_data["optional_inputs"]) ** (input_size[1] - 1) + else: + _logger.info("Optional inputs not found in %s", layer_key) + total_state_size = 1 + + converted = { + layer_key + "/layer_choice": { + "_type": "choice", "_value": layer_choice + }, + layer_key + "/optional_input_size": { + "_type": "randint", "_value": input_size + }, + layer_key + "/optional_input_chosen_state": { + "_type": "randint", "_value": [0, total_state_size] + } + } + _logger.info(converted) + ret.update(converted) + + return ret + + +def rewrite_nas_space(func): + @functools.wraps(func) + def wrap(self, search_space): + search_space = convert_nas_search_space(search_space) + return func(self, search_space) + return wrap diff --git a/utils/third_party/nni/compression/__init__.py b/utils/third_party/nni/compression/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/utils/third_party/nni/compression/pytorch/__init__.py b/utils/third_party/nni/compression/pytorch/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e20f284fef2cd06fa7cd4c1cf8ce43914195e5fc --- /dev/null +++ b/utils/third_party/nni/compression/pytorch/__init__.py @@ -0,0 +1,5 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .speedup import ModelSpeedup +from .compressor import Compressor, Pruner, Quantizer diff --git a/utils/third_party/nni/compression/pytorch/compressor.py b/utils/third_party/nni/compression/pytorch/compressor.py new file mode 100644 index 0000000000000000000000000000000000000000..b21589530206794dd506f2137ae8131bae83be80 --- /dev/null +++ b/utils/third_party/nni/compression/pytorch/compressor.py @@ -0,0 +1,636 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import types +import logging +import torch +from . import default_layers + +_logger = logging.getLogger(__name__) + + +class LayerInfo: + def __init__(self, name, module): + self.module = module + self.name = name + self.type = type(module).__name__ + +def _setattr(model, name, module): + name_list = name.split(".") + for name in name_list[:-1]: + model = getattr(model, name) + setattr(model, name_list[-1], module) + + +class Compressor: + """ + Abstract base PyTorch compressor + """ + + def __init__(self, model, config_list, optimizer=None): + """ + Record necessary info in class members + + Parameters + ---------- + model : pytorch model + the model user wants to compress + config_list : list + the configurations that users specify for compression + optimizer: pytorch optimizer + optimizer used to train the model + """ + assert isinstance(model, torch.nn.Module) + self.validate_config(model, config_list) + + self.bound_model = model + self.config_list = config_list + self.optimizer = optimizer + + self.modules_to_compress = None + self.modules_wrapper = [] + self.is_wrapped = False + + self._fwd_hook_handles = {} + self._fwd_hook_id = 0 + + self.reset() + + if not self.modules_wrapper: + _logger.warning('Nothing is configured to compress, please check your model and config_list') + + def validate_config(self, model, config_list): + """ + subclass can optionally implement this method to check if config_list if valid + """ + pass + + def reset(self, checkpoint=None): + """ + reset model state dict and model wrapper + """ + self._unwrap_model() + if checkpoint is not None: + self.bound_model.load_state_dict(checkpoint) + + self.modules_to_compress = None + self.modules_wrapper = [] + + for layer, config in self._detect_modules_to_compress(): + wrapper = self._wrap_modules(layer, config) + self.modules_wrapper.append(wrapper) + + self._wrap_model() + + def _detect_modules_to_compress(self): + """ + detect all modules should be compressed, and save the result in `self.modules_to_compress`. + The model will be instrumented and user should never edit it after calling this method. + """ + if self.modules_to_compress is None: + self.modules_to_compress = [] + for name, module in self.bound_model.named_modules(): + if module == self.bound_model: + continue + layer = LayerInfo(name, module) + config = self.select_config(layer) + if config is not None: + self.modules_to_compress.append((layer, config)) + return self.modules_to_compress + + def _wrap_model(self): + """ + wrap all modules that needed to be compressed + + """ + for wrapper in reversed(self.get_modules_wrapper()): + _setattr(self.bound_model, wrapper.name, wrapper) + self.is_wrapped = True + + def _unwrap_model(self): + """ + unwrap all modules that needed to be compressed + + """ + for wrapper in self.get_modules_wrapper(): + _setattr(self.bound_model, wrapper.name, wrapper.module) + self.is_wrapped = False + + def compress(self): + """ + Compress the model with algorithm implemented by subclass. + + The model will be instrumented and user should never edit it after calling this method. + `self.modules_to_compress` records all the to-be-compressed layers + + Returns + ------- + torch.nn.Module + model with specified modules compressed. + """ + return self.bound_model + + def set_wrappers_attribute(self, name, value): + """ + To register attributes used in wrapped module's forward method. + If the type of the value is Torch.tensor, then this value is registered as a buffer in wrapper, + which will be saved by model.state_dict. Otherwise, this value is just a regular variable in wrapper. + + Parameters + ---------- + name : str + name of the variable + value: any + value of the variable + """ + for wrapper in self.get_modules_wrapper(): + if isinstance(value, torch.Tensor): + wrapper.register_buffer(name, value.clone()) + else: + setattr(wrapper, name, value) + + def get_modules_to_compress(self): + """ + To obtain all the to-be-compressed modules. + + Returns + ------- + list + a list of the layers, each of which is a tuple (`layer`, `config`), + `layer` is `LayerInfo`, `config` is a `dict` + """ + return self.modules_to_compress + + def get_modules_wrapper(self): + """ + To obtain all the wrapped modules. + + Returns + ------- + list + a list of the wrapped modules + """ + return self.modules_wrapper + + def select_config(self, layer): + """ + Find the configuration for `layer` by parsing `self.config_list` + + Parameters + ---------- + layer : LayerInfo + one layer + + Returns + ------- + config or None + the retrieved configuration for this layer, if None, this layer should + not be compressed + """ + ret = None + for config in self.config_list: + config = config.copy() + # expand config if key `default` is in config['op_types'] + if 'op_types' in config and 'default' in config['op_types']: + expanded_op_types = [] + for op_type in config['op_types']: + if op_type == 'default': + expanded_op_types.extend(default_layers.weighted_modules) + else: + expanded_op_types.append(op_type) + config['op_types'] = expanded_op_types + + # check if condition is satisified + if 'op_types' in config and layer.type not in config['op_types']: + continue + if 'op_names' in config and layer.name not in config['op_names']: + continue + + ret = config + if ret is None or 'exclude' in ret: + return None + return ret + + def update_epoch(self, epoch): + """ + If user want to update model every epoch, user can override this method. + This method should be called at the beginning of each epoch + + Parameters + ---------- + epoch : num + the current epoch number + """ + pass + + def _wrap_modules(self, layer, config): + """ + This method is implemented in the subclasses, i.e., `Pruner` and `Quantizer` + + Parameters + ---------- + layer : LayerInfo + the layer to instrument the compression operation + config : dict + the configuration for compressing this layer + """ + raise NotImplementedError() + + + def add_activation_collector(self, collector): + self._fwd_hook_id += 1 + self._fwd_hook_handles[self._fwd_hook_id] = [] + for wrapper in self.get_modules_wrapper(): + handle = wrapper.register_forward_hook(collector) + self._fwd_hook_handles[self._fwd_hook_id].append(handle) + return self._fwd_hook_id + + def remove_activation_collector(self, fwd_hook_id): + if fwd_hook_id not in self._fwd_hook_handles: + raise ValueError("%s is not a valid collector id" % str(fwd_hook_id)) + for handle in self._fwd_hook_handles[fwd_hook_id]: + handle.remove() + del self._fwd_hook_handles[fwd_hook_id] + + def patch_optimizer(self, *tasks): + def patch_step(old_step): + def new_step(_, *args, **kwargs): + # call origin optimizer step method + output = old_step(*args, **kwargs) + # calculate mask + for task in tasks: + task() + return output + return new_step + if self.optimizer is not None: + self.optimizer.step = types.MethodType(patch_step(self.optimizer.step), self.optimizer) + +class PrunerModuleWrapper(torch.nn.Module): + def __init__(self, module, module_name, module_type, config, pruner): + """ + Wrap an module to enable data parallel, forward method customization and buffer registeration. + + Parameters + ---------- + module : pytorch module + the module user wants to compress + config : dict + the configurations that users specify for compression + module_name : str + the name of the module to compress, wrapper module shares same name + module_type : str + the type of the module to compress + pruner ： Pruner + the pruner used to calculate mask + """ + super().__init__() + # origin layer information + self.module = module + self.name = module_name + self.type = module_type + # config and pruner + self.config = config + self.pruner = pruner + + # register buffer for mask + self.register_buffer("weight_mask", torch.ones(self.module.weight.shape)) + if hasattr(self.module, 'bias') and self.module.bias is not None: + self.register_buffer("bias_mask", torch.ones(self.module.bias.shape)) + else: + self.register_buffer("bias_mask", None) + + def forward(self, *inputs): + # apply mask to weight, bias + self.module.weight.data = self.module.weight.data.mul_(self.weight_mask) + if hasattr(self.module, 'bias') and self.module.bias is not None: + self.module.bias.data = self.module.bias.data.mul_(self.bias_mask) + return self.module(*inputs) + +class Pruner(Compressor): + """ + Prune to an exact pruning level specification + + Attributes + ---------- + mask_dict : dict + Dictionary for saving masks, `key` should be layer name and + `value` should be a tensor which has the same shape with layer's weight + + """ + + def __init__(self, model, config_list, optimizer=None): + super().__init__(model, config_list, optimizer) + if optimizer is not None: + self.patch_optimizer(self.update_mask) + + def compress(self): + self.update_mask() + return self.bound_model + + def update_mask(self): + for wrapper_idx, wrapper in enumerate(self.get_modules_wrapper()): + masks = self.calc_mask(wrapper, wrapper_idx=wrapper_idx) + if masks is not None: + for k in masks: + assert hasattr(wrapper, k), "there is no attribute '%s' in wrapper" % k + setattr(wrapper, k, masks[k]) + + def calc_mask(self, wrapper, **kwargs): + """ + Pruners should overload this method to provide mask for weight tensors. + The mask must have the same shape and type comparing to the weight. + It will be applied with `mul()` operation on the weight. + This method is effectively hooked to `forward()` method of the model. + + Parameters + ---------- + wrapper : Module + calculate mask for `wrapper.module`'s weight + """ + raise NotImplementedError("Pruners must overload calc_mask()") + + def _wrap_modules(self, layer, config): + """ + Create a wrapper module to replace the original one. + + Parameters + ---------- + layer : LayerInfo + the layer to instrument the mask + config : dict + the configuration for generating the mask + """ + _logger.debug("Module detected to compress : %s.", layer.name) + wrapper = PrunerModuleWrapper(layer.module, layer.name, layer.type, config, self) + assert hasattr(layer.module, 'weight'), "module %s does not have 'weight' attribute" % layer.name + # move newly registered buffers to the same device of weight + wrapper.to(layer.module.weight.device) + return wrapper + + def export_model(self, model_path, mask_path=None, onnx_path=None, input_shape=None, device=None): + """ + Export pruned model weights, masks and onnx model(optional) + + Parameters + ---------- + model_path : str + path to save pruned model state_dict + mask_path : str + (optional) path to save mask dict + onnx_path : str + (optional) path to save onnx model + input_shape : list or tuple + input shape to onnx model + device : torch.device + device of the model, used to place the dummy input tensor for exporting onnx file. + the tensor is placed on cpu if ```device``` is None + """ + assert model_path is not None, 'model_path must be specified' + mask_dict = {} + self._unwrap_model() # used for generating correct state_dict name without wrapper state + + for wrapper in self.get_modules_wrapper(): + weight_mask = wrapper.weight_mask + bias_mask = wrapper.bias_mask + if weight_mask is not None: + mask_sum = weight_mask.sum().item() + mask_num = weight_mask.numel() + _logger.debug('Layer: %s Sparsity: %.4f', wrapper.name, 1 - mask_sum / mask_num) + wrapper.module.weight.data = wrapper.module.weight.data.mul(weight_mask) + if bias_mask is not None: + wrapper.module.bias.data = wrapper.module.bias.data.mul(bias_mask) + # save mask to dict + mask_dict[wrapper.name] = {"weight": weight_mask, "bias": bias_mask} + + torch.save(self.bound_model.state_dict(), model_path) + _logger.info('Model state_dict saved to %s', model_path) + if mask_path is not None: + torch.save(mask_dict, mask_path) + _logger.info('Mask dict saved to %s', mask_path) + if onnx_path is not None: + assert input_shape is not None, 'input_shape must be specified to export onnx model' + # input info needed + if device is None: + device = torch.device('cpu') + input_data = torch.Tensor(*input_shape) + torch.onnx.export(self.bound_model, input_data.to(device), onnx_path) + _logger.info('Model in onnx with input shape %s saved to %s', input_data.shape, onnx_path) + + self._wrap_model() + + def load_model_state_dict(self, model_state): + """ + Load the state dict saved from unwrapped model. + + Parameters: + ----------- + model_state : dict + state dict saved from unwrapped model + """ + if self.is_wrapped: + self._unwrap_model() + self.bound_model.load_state_dict(model_state) + self._wrap_model() + else: + self.bound_model.load_state_dict(model_state) + +class QuantizerModuleWrapper(torch.nn.Module): + def __init__(self, module, module_name, module_type, config, quantizer): + """ + Wrap an module to enable data parallel, forward method customization and buffer registeration. + + Parameters + ---------- + module : pytorch module + the module user wants to compress + config : dict + the configurations that users specify for compression + module_name : str + the name of the module to compress, wrapper module shares same name + module_type : str + the type of the module to compress + quantizer ：quantizer + the quantizer used to calculate mask + """ + super().__init__() + # origin layer information + self.module = module + self.name = module_name + self.type = module_type + # config and pruner + self.config = config + self.quantizer = quantizer + + # register buffer and parameter + # old_weight is used to store origin weight and weight is used to store quantized weight + # the reason why weight is buffer instead of parameter is because in pytorch parameter is used as leaf + # if weight is leaf , then old_weight can not be updated. + if 'weight' in config['quant_types']: + if not _check_weight(self.module): + _logger.warning('Module %s does not have parameter "weight"', self.name) + else: + self.module.register_parameter('old_weight', torch.nn.Parameter(self.module.weight)) + delattr(self.module, 'weight') + self.module.register_buffer('weight', self.module.old_weight) + + def forward(self, *inputs): + if 'input' in self.config['quant_types']: + inputs = self.quantizer.quant_grad.apply( + inputs, + QuantType.QUANT_INPUT, + self) + + if 'weight' in self.config['quant_types'] and _check_weight(self.module): + self.quantizer.quant_grad.apply( + self.module.old_weight, + QuantType.QUANT_WEIGHT, + self) + result = self.module(*inputs) + else: + result = self.module(*inputs) + + if 'output' in self.config['quant_types']: + result = self.quantizer.quant_grad.apply( + result, + QuantType.QUANT_OUTPUT, + self) + return result + return result + +class Quantizer(Compressor): + """ + Base quantizer for pytorch quantizer + """ + + def __init__(self, model, config_list, optimizer=None): + super().__init__(model, config_list, optimizer) + self.quant_grad = QuantGrad + if self.optimizer is not None: + self.patch_optimizer(self.step_with_optimizer) + for wrapper in self.get_modules_wrapper(): + if 'weight' in wrapper.config['quant_types']: + # old_weight is registered to keep track of weight before quantization + # and it is trainable, therefore, it should be added to optimizer. + self.optimizer.add_param_group({"params": wrapper.module.old_weight}) + + def quantize_weight(self, weight, wrapper, **kwargs): + """ + quantize should overload this method to quantize weight. + This method is effectively hooked to :meth:`forward` of the model. + Parameters + ---------- + weight : Tensor + weight that needs to be quantized + wrapper : QuantizerModuleWrapper + the wrapper for origin module + """ + raise NotImplementedError('Quantizer must overload quantize_weight()') + + def quantize_output(self, output, wrapper, **kwargs): + """ + quantize should overload this method to quantize output. + This method is effectively hooked to :meth:`forward` of the model. + Parameters + ---------- + output : Tensor + output that needs to be quantized + wrapper : QuantizerModuleWrapper + the wrapper for origin module + """ + raise NotImplementedError('Quantizer must overload quantize_output()') + + def quantize_input(self, *inputs, wrapper, **kwargs): + """ + quantize should overload this method to quantize input. + This method is effectively hooked to :meth:`forward` of the model. + Parameters + ---------- + inputs : Tensor + inputs that needs to be quantized + wrapper : QuantizerModuleWrapper + the wrapper for origin module + """ + raise NotImplementedError('Quantizer must overload quantize_input()') + + + def _wrap_modules(self, layer, config): + """ + Create a wrapper forward function to replace the original one. + Parameters + ---------- + layer : LayerInfo + the layer to instrument the mask + config : dict + the configuration for quantization + """ + assert 'quant_types' in config, 'must provide quant_types in config' + assert isinstance(config['quant_types'], list), 'quant_types must be list type' + assert 'quant_bits' in config, 'must provide quant_bits in config' + assert isinstance(config['quant_bits'], int) or isinstance(config['quant_bits'], dict), 'quant_bits must be dict type or int type' + + if isinstance(config['quant_bits'], dict): + for quant_type in config['quant_types']: + assert quant_type in config['quant_bits'], 'bits length for %s must be specified in quant_bits dict' % quant_type + + return QuantizerModuleWrapper(layer.module, layer.name, layer.type, config, self) + + def step_with_optimizer(self): + pass + +class QuantType: + """ + Enum class for quantization type. + """ + QUANT_INPUT = 0 + QUANT_WEIGHT = 1 + QUANT_OUTPUT = 2 + + +class QuantGrad(torch.autograd.Function): + """ + Base class for overriding backward function of quantization operation. + """ + @staticmethod + def quant_backward(tensor, grad_output, quant_type): + """ + This method should be overrided by subclass to provide customized backward function, + default implementation is Straight-Through Estimator + Parameters + ---------- + tensor : Tensor + input of quantization operation + grad_output : Tensor + gradient of the output of quantization operation + quant_type : QuantType + the type of quantization, it can be `QuantType.QUANT_INPUT`, `QuantType.QUANT_WEIGHT`, `QuantType.QUANT_OUTPUT`, + you can define different behavior for different types. + Returns + ------- + tensor + gradient of the input of quantization operation + """ + return grad_output + + @staticmethod + def forward(ctx, tensor, quant_type, wrapper, **kwargs): + ctx.save_for_backward(tensor, torch.Tensor([quant_type])) + if quant_type == QuantType.QUANT_INPUT: + return wrapper.quantizer.quantize_input(tensor, wrapper, **kwargs) + elif quant_type == QuantType.QUANT_WEIGHT: + return wrapper.quantizer.quantize_weight(wrapper, **kwargs) + elif quant_type == QuantType.QUANT_OUTPUT: + return wrapper.quantizer.quantize_output(tensor, wrapper, **kwargs) + else: + raise ValueError("unrecognized QuantType.") + + @classmethod + def backward(cls, ctx, grad_output): + tensor, quant_type = ctx.saved_variables + output = cls.quant_backward(tensor, grad_output, quant_type) + return output, None, None, None + +def _check_weight(module): + try: + return isinstance(module.weight.data, torch.Tensor) + except AttributeError: + return False diff --git a/utils/third_party/nni/compression/pytorch/default_layers.py b/utils/third_party/nni/compression/pytorch/default_layers.py new file mode 100644 index 0000000000000000000000000000000000000000..4d7e6d8aed84ad76c9404f301117fb8a00d9a570 --- /dev/null +++ b/utils/third_party/nni/compression/pytorch/default_layers.py @@ -0,0 +1,9 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +weighted_modules = [ + 'Conv1d', 'Conv2d', 'Conv3d', 'ConvTranspose1d', 'ConvTranspose2d', 'ConvTranspose3d', + 'Linear', 'Bilinear', + 'PReLU', + 'Embedding', 'EmbeddingBag', +] diff --git a/utils/third_party/nni/compression/pytorch/speedup/__init__.py b/utils/third_party/nni/compression/pytorch/speedup/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..cef8ebd76c0d99b1810512afe28723b5eeccb9fc --- /dev/null +++ b/utils/third_party/nni/compression/pytorch/speedup/__init__.py @@ -0,0 +1 @@ +from .compressor import ModelSpeedup \ No newline at end of file diff --git a/utils/third_party/nni/compression/pytorch/speedup/compress_modules.py b/utils/third_party/nni/compression/pytorch/speedup/compress_modules.py new file mode 100644 index 0000000000000000000000000000000000000000..c2e61ba04372eeef1ee44c927756e2a049778dca --- /dev/null +++ b/utils/third_party/nni/compression/pytorch/speedup/compress_modules.py @@ -0,0 +1,276 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +import torch +from .infer_shape import ModuleMasks + +_logger = logging.getLogger(__name__) + +replace_module = { + 'BatchNorm2d': lambda module, mask: replace_batchnorm2d(module, mask), + 'Conv2d': lambda module, mask: replace_conv2d(module, mask), + 'ConvTranspose2d': lambda module, mask: replace_conv2dt(module, mask), + 'MaxPool2d': lambda module, mask: no_replace(module, mask), + 'AvgPool2d': lambda module, mask: no_replace(module, mask), + 'AdaptiveAvgPool2d': lambda module, mask: no_replace(module, mask), + 'ReLU': lambda module, mask: no_replace(module, mask), + 'ReLU6': lambda module, mask: no_replace(module, mask), + 'LeakyReLU': lambda module, mask: no_replace(module, mask), + 'Sigmoid': lambda module, mask: no_replace(module, mask), + 'Linear': lambda module, mask: replace_linear(module, mask), + 'Dropout': lambda module, mask: no_replace(module, mask), + 'Dropout2d': lambda module, mask: no_replace(module, mask), + 'Dropout3d': lambda module, mask: no_replace(module, mask), + + 'EltwiseAdd': lambda module, mask: no_replace(module, mask), + 'TwoAdd': lambda module, mask: no_replace(module, mask), + + # yolov3 + 'ZeroPad2d': lambda module, mask: no_replace(module, mask), + + 'Tanh': lambda module, mask: no_replace(module, mask) +} + +def no_replace(module, mask): + """ + No need to replace + """ + _logger.debug("no need to replace") + return module + +def replace_linear(linear, mask): + """ + Parameters + ---------- + linear : torch.nn.Linear + The linear module to be replace + mask : ModuleMasks + The masks of this module + + Returns + ------- + torch.nn.Linear + The new linear module + """ + assert isinstance(mask, ModuleMasks) + assert mask.input_mask is not None + assert mask.output_mask is None + assert not mask.param_masks + index = mask.input_mask.mask_index[-1] + in_features = index.size()[0] + _logger.debug("replace linear with new in_features: %d", in_features) + new_linear = torch.nn.Linear(in_features=in_features, + out_features=linear.out_features, + bias=linear.bias is not None) + new_linear.to(linear.weight.device) + new_linear.weight.data = torch.index_select(linear.weight.data, -1, index.to(linear.weight.device)) + if linear.bias is not None: + new_linear.bias.data.copy_(linear.bias.data) + return new_linear + +def replace_batchnorm2d(norm, mask): + """ + Parameters + ---------- + norm : torch.nn.BatchNorm2d + The batchnorm module to be replace + mask : ModuleMasks + The masks of this module + + Returns + ------- + torch.nn.BatchNorm2d + The new batchnorm module + """ + assert isinstance(mask, ModuleMasks) + assert 'weight' in mask.param_masks and 'bias' in mask.param_masks + index = mask.param_masks['weight'].mask_index[0] + num_features = index.size()[0] + _logger.debug("replace batchnorm2d with num_features: %d", num_features) + new_norm = torch.nn.BatchNorm2d(num_features=num_features, + eps=norm.eps, + momentum=norm.momentum, + affine=norm.affine, + track_running_stats=norm.track_running_stats) + # assign weights + new_norm.weight.data = torch.index_select(norm.weight.data, 0, index) + new_norm.bias.data = torch.index_select(norm.bias.data, 0, index) + if norm.track_running_stats: + new_norm.running_mean.data = torch.index_select(norm.running_mean.data, 0, index) + new_norm.running_var.data = torch.index_select(norm.running_var.data, 0, index) + return new_norm + +def replace_conv2d(conv, mask): + """ + Parameters + ---------- + conv : torch.nn.Conv2d + The conv2d module to be replaced + mask : ModuleMasks + The masks of this module + + Returns + ------- + torch.nn.Conv2d + The new conv2d module + """ + assert isinstance(mask, ModuleMasks) + if mask.input_mask is None: + in_channels = conv.in_channels + else: + in_channels_index = mask.input_mask.mask_index[1] + in_channels = in_channels_index.size()[0] + if mask.output_mask is None: + out_channels = conv.out_channels + else: + out_channels_index = mask.output_mask.mask_index[1] + out_channels = out_channels_index.size()[0] + groups = conv.groups + if conv.in_channels == conv.out_channels == conv.groups: + # remove groups for depthwise layers + assert in_channels == out_channels + groups = in_channels + _logger.debug("replace conv2d %s with in_channels: %d, out_channels: %d", mask.module_name, in_channels, out_channels) + new_conv = torch.nn.Conv2d(in_channels=in_channels, + out_channels=out_channels, + kernel_size=conv.kernel_size, + stride=conv.stride, + padding=conv.padding, + dilation=conv.dilation, + groups=groups, + bias=conv.bias is not None, + padding_mode=conv.padding_mode) + + new_conv.to(conv.weight.device) + tmp_weight_data = tmp_bias_data = None + + if mask.output_mask is not None: + tmp_weight_data = torch.index_select(conv.weight.data, 0, out_channels_index) + if conv.bias is not None: + tmp_bias_data = torch.index_select(conv.bias.data, 0, out_channels_index) + else: + tmp_weight_data = conv.weight.data + # For the convolutional layers that have more than one group + # we need to copy the weight group by group, because the input + # channal is also divided into serveral groups and each group + # filter may have different input channel indexes. + input_step = int(conv.in_channels / conv.groups) + in_channels_group = int(in_channels / groups) + filter_step = int(out_channels / groups) + if mask.input_mask is not None and not (in_channels == out_channels == groups): + for groupid in range(conv.groups): + start = groupid * input_step + end = (groupid + 1) * input_step + current_input_index = list(filter(lambda x: start <= x and x < end, in_channels_index.tolist())) + if not current_input_index: + # there is no kept channel in current group + continue + # shift the global index into the group index + current_input_index = [x-start for x in current_input_index] + # if the groups is larger than 1, the input channels of each + # group should be pruned evenly. + assert len(current_input_index) == in_channels_group, \ + 'Input channels of each group are not pruned evenly' + current_input_index = torch.tensor(current_input_index).to(tmp_weight_data.device) # pylint: disable=not-callable + f_start = groupid * filter_step + f_end = (groupid + 1) * filter_step + new_conv.weight.data[f_start:f_end] = torch.index_select(tmp_weight_data[f_start:f_end], 1, current_input_index) + else: + new_conv.weight.data.copy_(tmp_weight_data) + + if conv.bias is not None: + new_conv.bias.data.copy_(conv.bias.data if tmp_bias_data is None else tmp_bias_data) + + return new_conv + + +def replace_conv2dt(convtrans, mask): + """ + We need anothor replace function for + convtranspose2d, because the layout of + the weight is different from traditional + conv layers. The layout of the weight is [N_in, N_out, ksize_1, ksize_2] + Parameters + ---------- + convtrans : torch.nn.ConvTranspose2d + The conv2d module to be replaced + mask : ModuleMasks + The masks of this module + Returns + ------- + torch.nn.ConvTranspose2d + The new conv2d module + """ + assert isinstance(mask, ModuleMasks) + assert isinstance(convtrans, torch.nn.ConvTranspose2d) + if mask.input_mask is None: + in_channels = convtrans.in_channels + else: + in_channels_index = mask.input_mask.mask_index[1] + in_channels = in_channels_index.size(0) + if mask.output_mask is None: + out_channels = convtrans.out_channels + else: + out_channels_index = mask.output_mask.mask_index[1] + out_channels = out_channels_index.size(0) + groups = convtrans.groups + # print('convtrans {} {} {}'.format(in_channels, out_channels, groups)) + # check if can remove the whole group of filters + if convtrans.in_channels == convtrans.out_channels == convtrans.groups: + # remove groups for depthwise layers + # this needs the group dependency to be fixed before the speedup + assert in_channels == out_channels + # groups = in_channels + _logger.debug('Replace convtranspose2d %s with in_channels:%d out_channels:%d', + mask.module_name, in_channels, out_channels) + new_convtrans = torch.nn.ConvTranspose2d(in_channels=in_channels, + out_channels=out_channels, + kernel_size=convtrans.kernel_size, + stride=convtrans.stride, + padding=convtrans.padding, + dilation=convtrans.dilation, + groups=groups, + bias=convtrans.bias is not None, + padding_mode=convtrans.padding_mode) + new_convtrans.to(convtrans.weight.device) + tmp_weight_data = None + if mask.input_mask is not None: + # in convtranspose2d we need to select the input channel first + tmp_weight_data = torch.index_select( + convtrans.weight.data, 0, in_channels_index) + else: + tmp_weight_data = convtrans.weight.data + # we need to handle the output channel group by group like the conv layer + out_step = int(convtrans.out_channels / convtrans.groups) + out_channel_group = int(out_channels/groups) + new_in_per_group = int(in_channels/groups) + + if mask.output_mask is not None and not(in_channels == out_channels == groups): + for groupid in range(convtrans.groups): + start = groupid * out_step + end = (groupid + 1) * out_step + current_output_index = list( + filter(lambda x: start <= x and x < end, out_channels_index.tolist())) + # we need to shift the index into the group-wise + current_output_index = [x-start for x in current_output_index] + if not current_output_index: + # No kept channel in the current group + raise Exception( + " Donnot support removing the whole group filter except in the depth-wise conv temporarily") + assert len(current_output_index) == out_channel_group, \ + 'Output channel of each group should be the same after pruning' + current_output_index = torch.tensor(current_output_index).to(tmp_weight_data.device) # pylint: disable=not-callable + new_start = groupid * new_in_per_group + new_end = (groupid + 1) * new_in_per_group + new_convtrans.weight.data[new_start:new_end] = torch.index_select( + tmp_weight_data[new_start:new_end], 1, current_output_index) + else: + new_convtrans.weight.data.copy_(tmp_weight_data) + if convtrans.bias is not None: + if mask.output_mask is not None: + new_convtrans.bias.data[:] = torch.index_select( + convtrans.bias.data, 0, out_channels_index) + else: + new_convtrans.bias.data.copy_(convtrans.bias.data) + return new_convtrans \ No newline at end of file diff --git a/utils/third_party/nni/compression/pytorch/speedup/compressor.py b/utils/third_party/nni/compression/pytorch/speedup/compressor.py new file mode 100644 index 0000000000000000000000000000000000000000..a6d3590b1746f6d577b608fcd7a2ede938890293 --- /dev/null +++ b/utils/third_party/nni/compression/pytorch/speedup/compressor.py @@ -0,0 +1,187 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +import torch +from ..utils.mask_conflict import fix_mask_conflict +from ..utils.utils import get_module_by_name +from .compress_modules import replace_module +from .infer_shape import ModuleMasks, infer_from_mask, infer_from_inshape, infer_from_outshape, set_conv_prune_dim + +_logger = logging.getLogger(__name__) + + +class ModelSpeedup: + """ + This class is to speedup the model with provided weight mask + """ + + def __init__(self, model, dummy_input, masks_file, map_location=None): + """ + Parameters + ---------- + model : pytorch model + The model user wants to speed up + dummy_input : pytorch tensor + The dummy input for ```jit.trace```, users should put it on right device before pass in + masks_file : str + The path of user provided mask file + map_location : str + the device on which masks are placed, same to map_location in ```torch.load``` + """ + from ....common.graph_utils import build_module_graph + + self.bound_model = model + self.masks = torch.load(masks_file, map_location) + self.inferred_masks = dict() # key: module_name, value: ModuleMasks + self.dummy_input = dummy_input + self.torch_graph = build_module_graph(model, dummy_input) + + def infer_module_mask(self, module_name, last_module, mask=None, in_shape=None, out_shape=None): + """ + Infer input shape / output shape based on the module's weight mask / input shape / output shape. + + For a module: + Infer its input and output shape from its weight mask + Infer its output shape from its input shape + Infer its input shape from its output shape + + If its input shape is changed, continue infering its predecessors + If its output shape is changed, continue infering its successors + + Parameters + ---------- + module_name : str + The name of the node + last_module : str + The name of last visited node + mask : tensor of mask or ModuleMasks + Mask of the weights in this node (i.e., module) + in_shape : ModuleMasks + Input shape of this node + out_shape : ModuleMasks + Output shape of this node + """ + input_cmask = output_cmask = None + if module_name in self.inferred_masks: + module_masks = self.inferred_masks[module_name] + else: + _, m = get_module_by_name(self.bound_model, module_name) + module_masks = ModuleMasks(module_name, m) + self.inferred_masks[module_name] = module_masks + + m_type = self.torch_graph.name_to_node[module_name].op_type + _logger.debug("infer mask of module %s with op_type %s", module_name, m_type) + if mask is not None: + _logger.debug("mask is not None") + if not m_type in infer_from_mask: + raise RuntimeError( + "Has not supported infering input/output shape from mask for module/function: `{}`, {}" + .format(m_type, module_name)) + if m_type in ['Linear']: + input_cmask, output_cmask = infer_from_mask[m_type]( + module_masks, mask, self.torch_graph.name_to_node[module_name].auxiliary + ) + else: + input_cmask, output_cmask = infer_from_mask[m_type](module_masks, mask) + if in_shape is not None: + _logger.debug("in_shape is not None") + if not m_type in infer_from_inshape: + raise RuntimeError( + "Has not supported infering output shape from input shape for module/function: `{}`, {}" + .format(m_type, module_name)) + if m_type in ['aten::view', 'aten::flatten', 'aten::mean', 'aten::reshape']: + output_cmask = infer_from_inshape[m_type](module_masks, + in_shape, + self.torch_graph.name_to_node[module_name].auxiliary) + elif m_type in ['aten::cat']: + # To calculate the mask for concat operation, the output shape + # , cat dimension, and the order of the input parameters. + output_cmask = infer_from_inshape[m_type](module_masks, + in_shape, + self.torch_graph.name_to_node[module_name].auxiliary, + last_module) + else: + output_cmask = infer_from_inshape[m_type](module_masks, in_shape) + if out_shape is not None: + _logger.debug("out_shape is not None") + if not m_type in infer_from_outshape: + raise RuntimeError( + "Has not supported infering input shape from output shape for module/function: `{}`, {}" + .format(m_type, module_name)) + if m_type in ['aten::view', 'aten::flatten', 'aten::mean', 'aten::reshape']: + input_cmask = infer_from_outshape[m_type](module_masks, out_shape, self.torch_graph.name_to_node[module_name].auxiliary) + else: + input_cmask = infer_from_outshape[m_type](module_masks, out_shape) + + if input_cmask: + predecessors = self.torch_graph.find_predecessors(module_name) + for _module_name in predecessors: + self.infer_module_mask(_module_name, module_name, out_shape=input_cmask) + if output_cmask: + successors = self.torch_graph.find_successors(module_name) + for _module_name in successors: + self.infer_module_mask(_module_name, module_name, in_shape=output_cmask) + + def infer_modules_masks(self): + """ + Do shape inference of involved modules, including the shape of weights, inputs, output + """ + for module_name, mask in self.masks.items(): + _logger.debug('Start mask inference from %s', module_name) + if module_name not in self.torch_graph.name_to_node: + # this module is not traced in the torch_graph, + # jit.trace only correctly records functions and + # modules which are not data dependent (e.g., do + # not have conditionals on data in tensors) + # so, if a node is not traced, we just skip it. + _logger.warning('%s has mask, but not found in the traced graph, just skip it.', module_name) + continue + self.infer_module_mask(module_name, None, mask=mask) + + def replace_compressed_modules(self): + """ + Replace all the modules that have changed (weights/inputs/output) shape. + The new module is created using the same arguments of the to-be-replaced module, + and correctly inherits its weights. + + NOTE: ```func``` type cannot be replaced as it is not a module, thus, one limitation + is that ```func``` should be not required to be replaced. + """ + for module_name in self.inferred_masks: + g_node = self.torch_graph.name_to_node[module_name] + _logger.debug("replace %s, in %s type, with op_type %s", + module_name, g_node.type, g_node.op_type) + if g_node.type == 'module': + super_module, leaf_module = get_module_by_name(self.bound_model, g_node.name) + m_type = g_node.op_type + if not m_type in replace_module: + raise RuntimeError("Has not supported replacing the module: `{}`".format(m_type)) + _logger.info("replace module (name: %s, op_type: %s)", g_node.name, m_type) + compressed_module = replace_module[m_type](leaf_module, self.inferred_masks[module_name]) + setattr(super_module, g_node.name.split('.')[-1], compressed_module) + elif g_node.type == 'func': + _logger.info("Warning: cannot replace (name: %s, op_type: %s) which is func type", + module_name, g_node.op_type) + else: + raise RuntimeError("Unsupported node type: {}".format(g_node.type)) + + def speedup_model(self): + """ + There are basically two steps: + first, do mask/shape inference, + second, replace modules + """ + training = self.bound_model.training + _logger.info("start to speed up the model") + + _logger.info("fix the mask conflict of the interdependent layers") + _, conv_prune_dim = fix_mask_conflict(self.masks, self.bound_model, self.dummy_input) + set_conv_prune_dim(conv_prune_dim) + + _logger.info("infer module masks...") + self.infer_modules_masks() + _logger.info("replace compressed modules...") + self.replace_compressed_modules() + self.bound_model.train(training) + _logger.info("speedup done") diff --git a/utils/third_party/nni/compression/pytorch/speedup/infer_shape.py b/utils/third_party/nni/compression/pytorch/speedup/infer_shape.py new file mode 100644 index 0000000000000000000000000000000000000000..8a675cf731107f5addf5d693919d5bc06340a853 --- /dev/null +++ b/utils/third_party/nni/compression/pytorch/speedup/infer_shape.py @@ -0,0 +1,1150 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. +""" +For each operation or module, there are two functions. +One is given output shape, infer its input shape and initialization parameters (e.g., weight's shape) +The other is given input shape, infer its output shape and initialization parameters (e.g., weight's shape) +""" + +import logging +import torch + +_logger = logging.getLogger(__name__) + +conv_prune_dim = -1 + +def set_conv_prune_dim(dim): + """ + Parameters: + dim: int + 0: filter pruning + 1: channel pruning + """ + global conv_prune_dim + conv_prune_dim = dim + +class CoarseMask: + """ + Coarse grained mask for a given tensor, here tensor could be weights, + input tensor, or output tensor + """ + + def __init__(self, num_dim): + """ + Parameters + ---------- + num_dim : int + The number of dimensions of the tensor that will be masked + """ + self.mask_index = [None for _ in range(num_dim)] + + def add_index_mask(self, dim, index): + """ + Add mask for the specified dimension + + Parameters + ---------- + dim : int + The dimension to add mask + index : tensor + The mask for this dimension, its a 1 dimension tensor which specifies + the index of the elements that are not pruned + """ + self.mask_index[dim] = index + + @staticmethod + def merge_index(index_a, index_b): + """ + Parameters + ---------- + index_a : tensor + One index (1-dimension) tensor + index_b : tensor + The other index (1-dimension) tensor + + Returns + ------- + tensor + The merged index (1-dimension) tensor + Note that: the output tensor will be moved + to the same device as index_a. + """ + device = index_a.device + s = set() + for num in index_a.tolist(): + # we need to transfer the tensor to list here + # first, directly traversing the tensor by for + # loop will return the list of tensor(x) object, + # even the value are the same, but they are different + # tensor objects, so the set will contains multiple + # tensor objects that has the same value. For example + # for num in torch.ones(2): + # s.add(num) + # s will be {tensor(1), tensor(1)} + s.add(num) + for num in index_b.tolist(): + s.add(num) + # move the output tensor to the same device with index_a + return torch.tensor(sorted(s)).to(device) # pylint: disable=not-callable + + def merge(self, cmask): + """ + Merge another CoarseMask + + Parameters + ---------- + cmask : CoarseMask + Another CoarseMask to merge + + Returns + ------- + list + The member variable ```mask_index``` + """ + assert isinstance(cmask, CoarseMask) + assert len(self.mask_index) == len(cmask.mask_index), \ + "Only masks with the same number of dimensions can be merged" + for i, index in enumerate(self.mask_index): + if index is None: + self.mask_index[i] = cmask.mask_index[i] + elif cmask.mask_index[i] is not None: + self.mask_index[i] = CoarseMask.merge_index(self.mask_index[i], + cmask.mask_index[i]) + return self.mask_index + + def __repr__(self): + return 'mask_index: {}'.format(self.mask_index) + + def eq_on_dim(self, other, dim): + assert isinstance(other, CoarseMask) + if self.mask_index[dim] is None and other.mask_index[dim] is None: + return True + elif isinstance(self.mask_index[dim], torch.Tensor) \ + and isinstance(other.mask_index[dim], torch.Tensor): + return torch.equal(self.mask_index[dim], other.mask_index[dim]) + else: + return False + + def __eq__(self, other): + assert isinstance(other, CoarseMask) + if len(self.mask_index) != len(other.mask_index): + return False + for i in range(len(self.mask_index)): + if not self.eq_on_dim(other, i): + return False + return True + + def __lt__(self, other): + """ + Judge if the mask is a subset of another CoarseMask. + """ + assert isinstance(other, CoarseMask) + for dim, _ in enumerate(self.mask_index): + # if self has more dimensions + if dim >= len(other.mask_index): + return False + if self.mask_index[dim] is None: + # if no mask on this dimension, then we have less + # masks then the other CoraseMask. + continue + elif other.mask_index[dim] is None: + return False + else: + s1 = set(self.mask_index[dim].tolist()) + s2 = set(other.mask_index[dim].tolist()) + if not s1 < s2: + return False + return True + + def __le__(self, other): + """ + Return if self's mask is less or equal to other's mask. + """ + assert isinstance(other, CoarseMask) + if self.__lt__(other) or self.__eq__(other): + return True + return False + + def __ne__(self, other): + return not self.__eq__(other) + + +class ModuleMasks: + """ + The masks of a module, including the masks for weights, inputs, output + """ + + def __init__(self, module_name, module=None): + """ + Parameters + ---------- + module_name : str + The name of the module or function + """ + self.module_name = module_name + self.module = module + self.param_masks = dict() + self.input_mask = None + self.output_mask = None + + def set_param_masks(self, name, mask): + """ + Parameters + ---------- + name : str + The name of the weight + mask : CoarseMask + The mask for this weight + """ + self.param_masks[name] = mask + + def set_input_mask(self, mask): + """ + Parameters + ---------- + mask : CoarseMask + The mask for input + """ + self.input_mask = mask + + def set_output_mask(self, mask): + """ + Parameters + ---------- + mask : CoarseMask + The mask for output + """ + self.output_mask = mask + + def __repr__(self): + return 'module_name: {}, input_mask: {}, output_mask: {}, param_masks: {}'.format( + self.module_name, self.input_mask, self.output_mask, self.param_masks + ) + + +""" +Infer input and output shape of a module/function from its weight mask +""" +infer_from_mask = { + 'BatchNorm2d': lambda module_masks, mask: batchnorm2d_mask(module_masks, mask), + 'Conv2d': lambda module_masks, mask: conv2d_mask(module_masks, mask), + 'Linear': lambda module_masks, mask, shape: linear_mask(module_masks, mask, shape), + + 'ConvTranspose2d': lambda module_masks, mask: convtranspose2d_mask(module_masks, mask) +} + +""" +Infer output and weight shape of a module/function from its input shape +""" +infer_from_inshape = { + 'ReLU': lambda module_masks, mask: relu_inshape(module_masks, mask), + 'ReLU6': lambda module_masks, mask: relu_inshape(module_masks, mask), + 'LeakyReLU': lambda module_masks, mask: relu_inshape(module_masks, mask), + 'Sigmoid': lambda module_masks, mask: relu_inshape(module_masks, mask), + 'aten::relu': lambda module_masks, mask: relu_inshape(module_masks, mask), + 'aten::tanh': lambda module_masks, mask: relu_inshape(module_masks, mask), + 'aten::tanh_': lambda module_masks, mask: relu_inshape(module_masks, mask), + 'aten::hardtanh': lambda module_masks, mask: relu_inshape(module_masks, mask), + 'aten::hardtanh_': lambda module_masks, mask: relu_inshape(module_masks, mask), + 'aten::relu_': lambda module_masks, mask: relu_inshape(module_masks, mask), + 'aten::sigmoid': lambda module_masks, mask: relu_inshape(module_masks, mask), + 'Conv2d': lambda module_masks, mask: conv2d_inshape(module_masks, mask), + 'ConvTranspose2d': lambda module_masks, mask: conv2dt_inshape(module_masks, mask), + 'MaxPool2d': lambda module_masks, mask: maxpool2d_inshape(module_masks, mask), + 'aten::max_pool2d': lambda module_masks, mask: maxpool2d_inshape(module_masks, mask), + 'aten::avg_pool2d': lambda module_masks, mask: maxpool2d_inshape(module_masks, mask), + 'aten::adaptive_avg_pool2d': lambda module_masks, mask: maxpool2d_inshape(module_masks, mask), + 'AvgPool2d': lambda module_masks, mask: maxpool2d_inshape(module_masks, mask), + 'AdaptiveAvgPool2d': lambda module_masks, mask: maxpool2d_inshape(module_masks, mask), + 'aten::size': lambda module_masks, mask: size_inshape(module_masks, mask), + 'aten::view': lambda module_masks, mask, shape: view_inshape(module_masks, mask, shape), + 'aten::reshape': lambda module_masks, mask, shape: view_inshape(module_masks, mask, shape), + # support only start_dim=1 + 'aten::flatten': lambda module_masks, mask, shape: view_inshape(module_masks, mask, shape), + 'Linear': lambda module_masks, mask: linear_inshape(module_masks, mask), + 'BatchNorm2d': lambda module_masks, mask: batchnorm2d_inshape(module_masks, mask), + 'aten::add_': lambda module_masks, mask: add_inshape(module_masks, mask), + 'aten::add': lambda module_mask, mask: add_inshape(module_mask, mask), + # mul has the similar behaviour with add, they both request + # the input tesors to have the same shape + 'aten::mul': lambda module_mask, mask: add_inshape(module_mask, mask), + 'aten::mul_': lambda module_mask, mask: add_inshape(module_mask, mask), + 'aten::cat': lambda module_mask, mask, cat_info, last_visited: cat_inshape(module_mask, mask, cat_info, last_visited), + 'aten::mean': lambda module_masks, mask, shape: mean_inshape(module_masks, mask, shape), + 'Dropout': lambda module_masks, mask: dropout_inshape(module_masks, mask), + 'Dropout2d': lambda module_masks, mask: dropout_inshape(module_masks, mask), + 'aten::dropout': lambda module_masks, mask: dropout_inshape(module_masks, mask), + + 'EltwiseAdd': lambda module_mask, mask: eltwise_add_inshape(module_mask, mask), + 'TwoAdd': lambda module_masks, mask: add_inshape(module_masks, mask), + + # yolov3 + 'ZeroPad2d': lambda module_masks, mask: maxpool2d_inshape(module_masks, mask), + + 'Tanh': lambda module_masks, mask: relu_inshape(module_masks, mask) +} + +""" +Infer input and weight shape of a module/function from its output shape +""" +infer_from_outshape = { + 'Conv2d': lambda module_masks, mask: conv2d_outshape(module_masks, mask), + 'BatchNorm2d': lambda module_masks, mask: batchnorm2d_outshape(module_masks, mask), + + 'MaxPool2d': lambda module_masks, mask: maxpool2d_outshape(module_masks, mask), + 'aten::max_pool2d': lambda module_masks, mask: maxpool2d_outshape(module_masks, mask), + 'aten::avg_pool2d': lambda module_masks, mask: maxpool2d_outshape(module_masks, mask), + 'aten::adaptive_avg_pool2d': lambda module_masks, mask: maxpool2d_outshape(module_masks, mask), + 'AvgPool2d': lambda module_masks, mask: maxpool2d_outshape(module_masks, mask), + 'AdaptiveAvgPool2d': lambda module_masks, mask: maxpool2d_outshape(module_masks, mask), + + 'ReLU': lambda module_masks, mask: relu_outshape(module_masks, mask), + 'ReLU6': lambda module_masks, mask: relu_outshape(module_masks, mask), + 'aten::relu': lambda module_masks, mask: relu_outshape(module_masks, mask), + 'aten::tanh': lambda module_masks, mask: relu_outshape(module_masks, mask), + 'aten::tanh_': lambda module_masks, mask: relu_outshape(module_masks, mask), + 'aten::hardtanh': lambda module_masks, mask: relu_outshape(module_masks, mask), + 'aten::hardtanh_': lambda module_masks, mask: relu_outshape(module_masks, mask), + 'aten::relu_': lambda module_masks, mask: relu_outshape(module_masks, mask), + + 'aten::add_': lambda module_masks, mask: add_outshape(module_masks, mask), + 'aten::add': lambda module_mask, mask: add_outshape(module_mask, mask), + 'aten::flatten': lambda module_mask, mask, shape: view_outshape(module_mask, mask, shape), + 'aten::view': lambda module_masks, mask, shape: view_outshape(module_masks, mask, shape), + 'aten::reshape': lambda module_masks, mask, shape: view_outshape(module_masks, mask, shape), + 'aten::mean': lambda module_masks, mask, shape: mean_outshape(module_masks, mask, shape), + 'Dropout': lambda module_masks, mask: dropout_outshape(module_masks, mask), + 'Dropout2d': lambda module_masks, mask: dropout_outshape(module_masks, mask), + 'aten::dropout': lambda module_masks, mask: dropout_outshape(module_masks, mask) +} + +def dropout_inshape(module_masks, mask): + if module_masks.input_mask is None: + module_masks.set_input_mask(mask) + module_masks.set_output_mask(mask) + return module_masks.output_mask + # if alreay visited + assert module_masks.input_mask <= mask + # It should be the same, we pass the masks by the reference(not the value), + # so they acutually are two references of the same object(mask, + # module_masks.input_mask). So we should continue pass the mask + # to the following nodes even module_masks.input_mask == mask. + # if pass the mask by copy.deepcopy(), then we can stop when + # module_masks.input_mask == mask. + # if module_masks.input_mask == mask: + # return None + module_masks.set_input_mask(mask) + module_masks.set_output_mask(mask) + return module_masks.output_mask + +def dropout_outshape(module_masks, mask): + if module_masks.output_mask is None: + module_masks.set_output_mask(mask) + module_masks.set_input_mask(mask) + return module_masks.input_mask + # if alreay visited + assert all(module_masks.output_mask.mask_index[1] == mask.mask_index[1]) + + return module_masks.output_mask + +def cat_inshape(module_masks, mask, cat_info, last_visited): + """ + Inference the output mask of the cat operation from the + input mask. + + Parameters + ---------- + module_masks : ModuleMasks + The ModuleMasks instance of the Conv2d + mask : CoarseMask + The mask of its input tensor + cat_info: dict + Dict object that records the necessary information + of cat operation, such as the order of the input + tensors. + last_visited: str + The unique_name of the last visited node group. + + Returns + ------- + CoarseMask + The mask of its output tensor + + """ + assert isinstance(mask, CoarseMask) + out_shape = cat_info['out_shape'] + cat_dim = cat_info['cat_dim'] + in_order = cat_info['in_order'] + in_shape = cat_info['in_shape'] + if module_masks.output_mask is None: + # First visit to this cat node + # initialize the mask based on + # the number of the output channel. + output_mask = CoarseMask(num_dim=len(out_shape)) + for dim, _ in enumerate(out_shape): + if dim == cat_dim: + if mask.mask_index[dim] is None: + continue + device = mask.mask_index[dim].device + # calculate the offset of the mask + pos = in_order.index(last_visited) + offsets = [in_shape[i][cat_dim] + for i, _ in enumerate(in_shape)] + offset = 0 + for i in range(pos): + offset += offsets[i] + _tmp_mask = (mask.mask_index[dim] + offset).to(device) + output_mask.mask_index[dim] = _tmp_mask + else: + # directly copy the mask + if mask.mask_index[dim] is not None: + output_mask.mask_index[dim] = mask.mask_index[dim].data.clone( + ) + module_masks.set_output_mask(output_mask) + + return module_masks.output_mask + # If this cat node is already visited, we need + # validating if the mask is legel, for cat operation, + # the mask on the 'cat_dim' dimension should be stitched + # together. In the other dimensions, the mask should be + # the same, else the mask is not legal. + for dim, _ in enumerate(out_shape): + if dim == cat_dim: + if mask.mask_index[dim] is None: + continue + pos = in_order.index(last_visited) + offsets = [in_shape[i][cat_dim] for i, _ in enumerate(in_shape)] + offset = 0 + for i in range(pos): + offset += offsets[i] + device = mask.mask_index[dim].device + new_mask = mask.mask_index[dim] + offset + module_masks.output_mask.mask_index[dim] = CoarseMask.merge_index( + module_masks.output_mask.mask_index[dim], new_mask).to(device) + else: + assert module_masks.output_mask.eq_on_dim(mask, dim) + + return module_masks.output_mask + + +def add_inshape(module_masks, mask): + """ + Inference the output mask of the add operation from the + input mask. + """ + assert isinstance(mask, CoarseMask) + if module_masks.input_mask is None: + module_masks.set_input_mask(mask) + module_masks.set_output_mask(mask) + # module_masks.input_mask = mask + return mask + # If alreay visited, validate if have the conflict + # if the mask is different with previous input_mask + # then there is a mask confilct. + if mask != module_masks.input_mask: + raise Exception('Mask conflict happenes!') + return None + + +def eltwise_add_inshape(module_masks, mask): + """ + Inference the output mask of the add operation from the + input mask. + """ + print(mask) + print(module_masks) + assert isinstance(mask, CoarseMask) + if module_masks.input_mask is None: + module_masks.set_input_mask(mask) + module_masks.set_output_mask(mask) + # module_masks.input_mask = mask + return mask + + # # If alreay visited, validate if have the conflict + # # if the mask is different with previous input_mask + # # then there is a mask confilct. + if mask != module_masks.input_mask: + raise Exception('Mask conflict happenes!') + return mask + + +def add_outshape(module_masks, mask): + """ + Inference the input mask of the add operation from the + output mask. + """ + assert isinstance(mask, CoarseMask) + + if module_masks.output_mask is None: + module_masks.set_output_mask(mask) + module_masks.set_input_mask(mask) + return mask + else: + assert all(module_masks.output_mask.mask_index[1] == mask.mask_index[1]) + return mask + +def batchnorm2d_inshape(module_masks, mask): + """ + We assume only the second dimension has coarse grained mask + + Parameters + ---------- + module_masks : ModuleMasks + The ModuleMasks instance of the batchnorm2d + mask : CoarseMask + The mask of its input tensor + + Returns + ------- + CoarseMask + The mask of its output tensor + """ + assert isinstance(mask, CoarseMask) + assert mask.mask_index[1] is not None + assert mask.mask_index[0] is None + assert mask.mask_index[2] is None + assert mask.mask_index[3] is None + module_masks.set_input_mask(mask) + module_masks.set_output_mask(mask) + weight_cmask = CoarseMask(num_dim=1) + weight_cmask.add_index_mask(dim=0, index=mask.mask_index[1]) + module_masks.set_param_masks('weight', weight_cmask) + module_masks.set_param_masks('bias', weight_cmask) + return mask + +def batchnorm2d_outshape(module_masks, mask): + """ + We assume only the second dimension has coarse grained mask + + Parameters + ---------- + module_masks : ModuleMasks + The ModuleMasks instance of the batchnorm2d + mask : CoarseMask + The mask of its input tensor + + Returns + ------- + CoarseMask + The mask of its output tensor + """ + assert isinstance(mask, CoarseMask) + assert len(mask.mask_index) in [2, 4] + assert mask.mask_index[1] is not None + assert mask.mask_index[0] is None + module_masks.set_input_mask(mask) + module_masks.set_output_mask(mask) + weight_cmask = CoarseMask(num_dim=1) + weight_cmask.add_index_mask(dim=0, index=mask.mask_index[1]) + module_masks.set_param_masks('weight', weight_cmask) + module_masks.set_param_masks('bias', weight_cmask) + return mask + + +def linear_inshape(module_masks, mask): + """ + Coarse grained input mask does not change the shape of weights and output tensor + + Parameters + ---------- + module_masks : ModuleMasks + The ModuleMasks instance of the linear + mask : CoarseMask + The mask of its input tensor + + Returns + ------- + CoarseMask + The mask of its output tensor, ```None``` means shape of output tensor is not changed + """ + assert isinstance(mask, CoarseMask) + assert mask.mask_index[0] is None + if module_masks.input_mask is not None: + assert module_masks.input_mask <= mask + module_masks.set_input_mask(mask) + return None + + +def view_inshape(module_masks, mask, shape): + """ + This is a limited support + + TODO: consider replace tensor.view with nn.Flatten, because tensor.view is not + included in module, thus, cannot be replaced by our framework. + + Parameters + ---------- + module_masks : ModuleMasks + The ModuleMasks instance of the ```view``` op + mask : CoarseMask + The mask of its input tensor + shape : dict + Original shape of its input and output tensors + + Returns + ------- + CoarseMask + The mask of its output tensor + """ + # NOTE: the case constrained by the following four asserts + assert shape['in_shape'][0] == shape['out_shape'][0] + assert len(shape['in_shape']) == 4 + assert len(shape['out_shape']) == 2 + assert shape['out_shape'][1] == shape['in_shape'][1] * \ + shape['in_shape'][2]*shape['in_shape'][3] + + assert isinstance(mask, CoarseMask) + assert mask.mask_index[1] is not None + assert mask.mask_index[0] is None + assert mask.mask_index[2] is None + assert mask.mask_index[3] is None + # due to the cat operation, the same node may be + # accessed more than once + if module_masks.input_mask is not None: + assert module_masks.input_mask <= mask + module_masks.set_input_mask(mask) + output_cmask = CoarseMask(num_dim=2) + index = [] + step_size = shape['in_shape'][2] * shape['in_shape'][3] + for loc in mask.mask_index[1]: + index.extend([loc * step_size + i for i in range(step_size)]) + output_cmask.add_index_mask(dim=1, index=torch.tensor(index).to(mask.mask_index[1].device)) # pylint: disable=not-callable + module_masks.set_output_mask(output_cmask) + return output_cmask + +def view_outshape(module_masks, mask, shape): + """ + Parameters + ---------- + module_masks : ModuleMasks + The ModuleMasks instance of the ```flatten``` op + mask : CoarseMask + The mask of its input tensor + shape : dict + Original shape of its input and output tensors + Returns + ------- + CoarseMask + The mask of its output tensor + """ + # NOTE: the case constrained by the following four asserts + assert shape['in_shape'][0] == shape['out_shape'][0] + assert len(shape['in_shape']) == 4 + assert len(shape['out_shape']) == 2 + assert shape['out_shape'][1] == shape['in_shape'][1] * \ + shape['in_shape'][2]*shape['in_shape'][3] + + assert isinstance(mask, CoarseMask) + assert mask.mask_index[1] is not None + assert mask.mask_index[0] is None + + module_masks.set_output_mask(mask) + input_cmask = CoarseMask(num_dim=4) + index = [] + step_size = shape['in_shape'][2] * shape['in_shape'][3] + for loc in mask.mask_index[1]: + index.extend([loc * step_size + i for i in range(step_size)]) + input_cmask.add_index_mask(dim=1, index=torch.tensor(index).to(mask.mask_index[1].device)) # pylint: disable=not-callable + module_masks.set_input_mask(input_cmask) + + return input_cmask + +def size_inshape(module_masks, mask): + """ + No need to do anything for this ```size``` op + """ + return None + +def mean_inshape(module_masks, mask, shape): + """ + Similar to view operation, currently mask inference only supports + the mean operation on the 3rd and 4th dimensions. + """ + assert shape['in_shape'][0] == shape['out_shape'][0] + assert shape['out_shape'][1] == shape['in_shape'][1] + assert len(shape['in_shape']) == 4 + assert len(shape['out_shape']) == 2 + + assert isinstance(mask, CoarseMask) + assert mask.mask_index[1] is not None + assert mask.mask_index[0] is None + assert mask.mask_index[2] is None + assert mask.mask_index[3] is None + module_masks.set_input_mask(mask) + + output_cmask = CoarseMask(num_dim=2) + output_cmask.add_index_mask(dim=1, index=mask.mask_index[1]) + module_masks.set_output_mask(output_cmask) + return output_cmask + +def mean_outshape(module_masks, mask, shape): + """ + Similar to view operation, currently mask inference only supports + the mean operation on the 3rd and 4th dimensions. + """ + assert shape['in_shape'][0] == shape['out_shape'][0] + assert shape['out_shape'][1] == shape['in_shape'][1] + assert len(shape['in_shape']) == 4 + assert len(shape['out_shape']) == 2 + + assert isinstance(mask, CoarseMask) + assert mask.mask_index[1] is not None + assert mask.mask_index[0] is None + module_masks.set_output_mask(mask) + + input_cmask = CoarseMask(num_dim=4) + input_cmask.add_index_mask(dim=1, index=mask.mask_index[1]) + module_masks.set_input_mask(input_cmask) + return input_cmask + +def maxpool2d_inshape(module_masks, mask): + """ + Assume only the second dimension is masked + + Parameters + ---------- + module_masks : ModuleMasks + The ModuleMasks instance of the maxpool2d + mask : CoarseMask + The mask of its input tensor + + Returns + ------- + CoarseMask + The mask of its output tensor + """ + assert isinstance(mask, CoarseMask) + assert mask.mask_index[1] is not None + assert mask.mask_index[0] is None + assert mask.mask_index[2] is None + assert mask.mask_index[3] is None + if module_masks.input_mask is not None: + assert module_masks.input_mask <= mask + # assert module_masks.input_mask is None + module_masks.set_input_mask(mask) + module_masks.set_output_mask(mask) + return mask + +def maxpool2d_outshape(module_masks, mask): + """ + Assume only the second dimension is masked + + Parameters + ---------- + module_masks : ModuleMasks + The ModuleMasks instance of the maxpool2d + mask : CoarseMask + The mask of its input tensor + + Returns + ------- + CoarseMask + The mask of its output tensor + """ + assert isinstance(mask, CoarseMask) + assert mask.mask_index[1] is not None + assert mask.mask_index[0] is None + + module_masks.set_input_mask(mask) + module_masks.set_output_mask(mask) + return mask + +def relu_inshape(module_masks, mask): + """ + Parameters + ---------- + module_masks : ModuleMasks + The ModuleMasks instance of the relu + mask : CoarseMask + The mask of its input tensor + + Returns + ------- + CoarseMask + The mask of its output tensor + """ + assert isinstance(mask, CoarseMask) + if module_masks.input_mask is not None: + # mask conflict should be solved before speedup + assert module_masks.input_mask <= mask + # assert module_masks.input_mask is None, "A relu op can only be processed once" + module_masks.set_input_mask(mask) + module_masks.set_output_mask(mask) + return mask + +def relu_outshape(module_masks, mask): + """ + Parameters + ---------- + module_masks : ModuleMasks + The ModuleMasks instance of the relu + mask : CoarseMask + The mask of its input tensor + + Returns + ------- + CoarseMask + The mask of its output tensor + """ + assert isinstance(mask, CoarseMask) + if module_masks.output_mask is not None: + # mask conflict should be solved before speedup + assert all(module_masks.output_mask.mask_index[1] == mask.mask_index[1]) + module_masks.set_input_mask(mask) + module_masks.set_output_mask(mask) + return mask + +def batchnorm2d_mask(module_masks, mask): + """ + Infer input and output shape from weight mask + Parameters + ---------- + module_masks : ModuleMasks + The ModuleMasks instance of the batchnorm2d + mask : dict + The mask of its weights, from the user provided mask file + Returns + ------- + CoarseMask, CoarseMask + The mask of its input tensor, the mask of its output tensor + """ + assert 'weight' in mask and 'bias' in mask + sum_mask = mask['weight'] + mask['bias'] + nonzero_index = torch.nonzero(sum_mask, as_tuple=True)[0] + # infer shape of parameters + param_cmask = CoarseMask(num_dim=1) + param_cmask.add_index_mask(dim=0, index=nonzero_index) + module_masks.set_param_masks('weight', param_cmask) + module_masks.set_param_masks('bias', param_cmask) + # infer shape of input tensor + input_cmask = CoarseMask(num_dim=4) + input_cmask.add_index_mask(dim=1, + index=torch.nonzero(mask['weight'], as_tuple=True)[0]) + module_masks.set_input_mask(input_cmask) + # infer shape of output tensor + output_cmask = CoarseMask(num_dim=4) + output_cmask.add_index_mask(dim=1, index=nonzero_index) + module_masks.set_output_mask(output_cmask) + return input_cmask, output_cmask + +def linear_mask(module_masks, mask, shape): + """ + Infer input and output shape from weight mask with limitations: + Only support infer input mask + + Parameters + ---------- + module_masks : ModuleMasks + The ModuleMasks instance of the Linear + mask : dict + The mask of its weights, from the user provided mask file + shape: dict + Shape of its input and output tensors + Returns + ------- + CoarseMask, CoarseMask + The mask of its input tensor, the mask of its output tensor + """ + + assert 'weight' in mask + num_input_dim = len(shape['in_shape']) + + # Input data of Linear module can have multiple dimensions. + # here we only support infer coarse mask on the first dimension (dimension 0) + nonzero_index = torch.nonzero(mask['weight'].sum(0), as_tuple=True)[0] + + # infer shape of input tensor + input_cmask = CoarseMask(num_dim=num_input_dim) + input_cmask.add_index_mask(dim=num_input_dim-1, index=nonzero_index) + + module_masks.set_input_mask(input_cmask) + return input_cmask, None + +def conv2d_mask(module_masks, mask): + """ + Infer input and output shape from weight mask + + Parameters + ---------- + module_masks : ModuleMasks + The ModuleMasks instance of the conv2d + mask : dict + The mask of its weights, from the user provided mask file + + Returns + ------- + CoarseMask, CoarseMask + The mask of its input tensor, the mask of its output tensor + """ + def convert_to_coarse_mask(mask, dim=0): + """ + Parameters + ---------- + mask : dict + Weight mask from user provided mask file + dim: int + 0: filter pruning + 1: channel pruning + + Returns + ------- + LongTensor, CoarseMask, CoarseMask + Index of the masked dimension, weight mask, bias mask + """ + assert 'weight' in mask + assert isinstance(mask['weight'], torch.Tensor) + assert dim in [0, 1] + + weight_mask = mask['weight'] + + sum_idx = (1, 2, 3) if dim == 0 else (0, 2, 3) + index = torch.nonzero(weight_mask.abs().sum(sum_idx) != 0, as_tuple=True)[0] + if len(index) == weight_mask.shape[dim]: # full mask + index = None + + if index is None: + return None, None, None + else: + index = index.long().to(weight_mask.device) + weight_cmask = CoarseMask(num_dim=4) + weight_cmask.add_index_mask(dim=dim, index=index) + bias_cmask = None + if dim == 0 and 'bias' in mask and mask['bias'] is not None: + bias_index = torch.nonzero(mask['bias'], as_tuple=True)[0] + assert torch.all(torch.eq(index, bias_index)), \ + "bias mask should be consistent with weight mask" + bias_cmask = CoarseMask(num_dim=1) + bias_cmask.add_index_mask(dim=0, index=bias_index) + return index, weight_cmask, bias_cmask + + index, weight_cmask, bias_cmask = convert_to_coarse_mask(mask, dim=conv_prune_dim) + + if index is None: + # TODO: fine grained mask speedup + return None, None + # deal with coarse grain mask + # mask conflict should be solved by fix_mask_conflict before speedup + if 'weight' in module_masks.param_masks: + assert module_masks.param_masks['weight'] == weight_cmask + else: + module_masks.set_param_masks('weight', weight_cmask) + if conv_prune_dim == 0: + module_masks.set_param_masks('bias', bias_cmask) + + io_cmask = CoarseMask(num_dim=4) + io_cmask.add_index_mask(dim=1, index=index) + + if conv_prune_dim == 0: + if module_masks.output_mask is None: + module_masks.set_output_mask(io_cmask) + else: + assert module_masks.output_mask == io_cmask + return None, module_masks.output_mask + else: + if module_masks.input_mask is None: + module_masks.set_input_mask(io_cmask) + else: + assert module_masks.input_mask == io_cmask + return module_masks.input_mask, None + + +def convtranspose2d_mask(module_masks, mask): + # TODO support the Convtranspose2d Pruning for the L1FilterPruner + # raise Exception( + # "Current Filter pruner cannot prune the ConvTranspose2d, will support pruning ConvTranspose2d later") + """ + Infer input and output shape from weight mask + + Parameters + ---------- + module_masks : ModuleMasks + The ModuleMasks instance of the conv2d + mask : dict + The mask of its weights, from the user provided mask file + + Returns + ------- + CoarseMask, CoarseMask + The mask of its input tensor, the mask of its output tensor + """ + def convert_to_coarse_mask(mask, dim=0): + """ + Parameters + ---------- + mask : dict + Weight mask from user provided mask file + dim: int + 0: filter pruning + 1: channel pruning + + Returns + ------- + LongTensor, CoarseMask, CoarseMask + Index of the masked dimension, weight mask, bias mask + """ + assert 'weight' in mask + assert isinstance(mask['weight'], torch.Tensor) + assert dim in [0, 1] + + weight_mask = mask['weight'] + + sum_idx = (1, 2, 3) if dim == 0 else (0, 2, 3) + index = torch.nonzero(weight_mask.abs().sum(sum_idx) != 0, as_tuple=True)[0] + if len(index) == weight_mask.shape[dim]: # full mask + index = None + + if index is None: + return None, None, None + else: + index = index.long().to(weight_mask.device) + weight_cmask = CoarseMask(num_dim=4) + weight_cmask.add_index_mask(dim=dim, index=index) + bias_cmask = None + if dim == 0 and 'bias' in mask and mask['bias'] is not None: + bias_index = torch.nonzero(mask['bias'], as_tuple=True)[0] + assert torch.all(torch.eq(index, bias_index)), \ + "bias mask should be consistent with weight mask" + bias_cmask = CoarseMask(num_dim=1) + bias_cmask.add_index_mask(dim=0, index=bias_index) + return index, weight_cmask, bias_cmask + + convt_prune_dim = 1 + index, weight_cmask, bias_cmask = convert_to_coarse_mask(mask, dim=convt_prune_dim) + + if index is None: + # TODO: fine grained mask speedup + return None, None + # deal with coarse grain mask + # mask conflict should be solved by fix_mask_conflict before speedup + if 'weight' in module_masks.param_masks: + assert module_masks.param_masks['weight'] == weight_cmask + else: + module_masks.set_param_masks('weight', weight_cmask) + if conv_prune_dim == 0: + module_masks.set_param_masks('bias', bias_cmask) + + io_cmask = CoarseMask(num_dim=4) + io_cmask.add_index_mask(dim=1, index=index) + + if conv_prune_dim == 0: + if module_masks.output_mask is None: + module_masks.set_output_mask(io_cmask) + else: + assert module_masks.output_mask == io_cmask + return None, module_masks.output_mask + else: + if module_masks.input_mask is None: + module_masks.set_input_mask(io_cmask) + else: + assert module_masks.input_mask == io_cmask + return module_masks.input_mask, None + + +def conv2d_inshape(module_masks, mask): + """ + Shape change of input tensor does not affect the shape of its output tensor + Parameters + ---------- + module_masks : ModuleMasks + The ModuleMasks instance of the conv2d + mask : CoarseMask + The mask of its input tensor + Returns + ------- + CoarseMask + The mask of its output tensor + """ + assert isinstance(mask, CoarseMask) + if module_masks.input_mask is None: + module_masks.set_input_mask(mask) + else: + # the same conv layer may be accessed more + # than once, such as a concat operation. + # mask conflict should be solved by fix_mask_conflict before speedup + assert module_masks.input_mask == mask + + # shape changes pass through depths wise conv layers + m = module_masks.module + if m.in_channels == m.out_channels == m.groups: + module_masks.output_mask = mask + module_masks.input_mask = mask + return mask + return None + + +def conv2dt_inshape(module_masks, mask): + """ + Shape change of input tensor does not affect the shape of its output tensor + Parameters + ---------- + module_masks : ModuleMasks + The ModuleMasks instance of the conv2d + mask : CoarseMask + The mask of its input tensor + Returns + ------- + CoarseMask + The mask of its output tensor + """ + assert isinstance(mask, CoarseMask) + if module_masks.input_mask is None: + module_masks.set_input_mask(mask) + else: + # the same conv layer may be accessed more + # than once, such as a concat operation. + # mask conflict should be solved by fix_mask_conflict before speedup + assert module_masks.input_mask == mask + + # shape changes pass through depths wise conv layers + m = module_masks.module + if m.in_channels == m.out_channels == m.groups: + module_masks.output_mask = mask + module_masks.input_mask = mask + return mask + return None + + +def conv2d_outshape(module_masks, mask): + """ + Assume only the second dimension is masked + + Parameters + ---------- + module_masks : ModuleMasks + The ModuleMasks instance of the conv2d + mask : CoarseMask + The mask of its output tensor + + Returns + ------- + CoarseMask + The mask of its input tensor + """ + assert isinstance(mask, CoarseMask) + assert mask.mask_index[1] is not None + assert mask.mask_index[0] is None + assert mask.mask_index[2] is None + assert mask.mask_index[3] is None + + if module_masks.output_mask is None: + module_masks.output_mask = mask + else: + # mask conflict should be solved by fix_mask_conflict before speedup + # mask and module_masks.output_mask may have different number of dimensions + # since they could be passed by linear or conv2d + assert all(module_masks.output_mask.mask_index[1] == mask.mask_index[1]) + + weight_cmask = CoarseMask(num_dim=4) + weight_cmask.add_index_mask(dim=0, index=mask.mask_index[1]) + bias_cmask = CoarseMask(num_dim=1) + bias_cmask.add_index_mask(dim=0, index=mask.mask_index[1]) + module_masks.set_param_masks('weight', weight_cmask) + module_masks.set_param_masks('bias', bias_cmask) + + # shape changes pass through depths wise conv layers + m = module_masks.module + if m.in_channels == m.out_channels == m.groups: + module_masks.output_mask = mask + module_masks.input_mask = mask + return mask + return None diff --git a/utils/third_party/nni/compression/pytorch/utils/__init__.py b/utils/third_party/nni/compression/pytorch/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/utils/third_party/nni/compression/pytorch/utils/config_validation.py b/utils/third_party/nni/compression/pytorch/utils/config_validation.py new file mode 100644 index 0000000000000000000000000000000000000000..835cdf17f6f3daa7f5fac82d4a8ca8a2719fcbb0 --- /dev/null +++ b/utils/third_party/nni/compression/pytorch/utils/config_validation.py @@ -0,0 +1,54 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from schema import Schema, And, SchemaError + +def validate_op_names(model, op_names, logger): + found_names = set(map(lambda x: x[0], model.named_modules())) + + not_found_op_names = list(set(op_names) - found_names) + if not_found_op_names: + logger.warning('op_names %s not found in model', not_found_op_names) + + return True + +def validate_op_types(model, op_types, logger): + found_types = set(['default']) | set(map(lambda x: type(x[1]).__name__, model.named_modules())) + + not_found_op_types = list(set(op_types) - found_types) + if not_found_op_types: + # logger.warning('op_types %s not found in model', not_found_op_types) + pass + + return True + +def validate_op_types_op_names(data): + if not ('op_types' in data or 'op_names' in data): + raise SchemaError('Either op_types or op_names must be specified.') + return True + +class CompressorSchema: + def __init__(self, data_schema, model, logger): + assert isinstance(data_schema, list) and len(data_schema) <= 1 + self.data_schema = data_schema + self.compressor_schema = Schema(self._modify_schema(data_schema, model, logger)) + + def _modify_schema(self, data_schema, model, logger): + if not data_schema: + return data_schema + + for k in data_schema[0]: + old_schema = data_schema[0][k] + if k == 'op_types' or (isinstance(k, Schema) and k._schema == 'op_types'): + new_schema = And(old_schema, lambda n: validate_op_types(model, n, logger)) + data_schema[0][k] = new_schema + if k == 'op_names' or (isinstance(k, Schema) and k._schema == 'op_names'): + new_schema = And(old_schema, lambda n: validate_op_names(model, n, logger)) + data_schema[0][k] = new_schema + + data_schema[0] = And(data_schema[0], lambda d: validate_op_types_op_names(d)) + + return data_schema + + def validate(self, data): + self.compressor_schema.validate(data) diff --git a/utils/third_party/nni/compression/pytorch/utils/counter.py b/utils/third_party/nni/compression/pytorch/utils/counter.py new file mode 100644 index 0000000000000000000000000000000000000000..6061e8a8d0a4adea3401fbeab6dd50e8f5f3073a --- /dev/null +++ b/utils/third_party/nni/compression/pytorch/utils/counter.py @@ -0,0 +1,135 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import torch +import torch.nn as nn +from nni.compression.pytorch.compressor import PrunerModuleWrapper + +try: + from thop import profile +except Exception as e: + print('thop is not found, please install the python package: thop') + raise + + +def count_flops_params(model: nn.Module, input_size, custom_ops=None, verbose=True): + """ + Count FLOPs and Params of the given model. + This function would identify the mask on the module + and take the pruned shape into consideration. + Note that, for sturctured pruning, we only identify + the remained filters according to its mask, which + not taking the pruned input channels into consideration, + so the calculated FLOPs will be larger than real number. + + Parameters + --------- + model : nn.Module + target model. + input_size: list, tuple + the input shape of data + custom_ops: dict + a mapping of (module: custom operation) + the custom operation will overwrite the default operation. + for reference, please see ``custom_mask_ops``. + + Returns + ------- + flops: float + total flops of the model + params: + total params of the model + """ + + assert input_size is not None + + device = next(model.parameters()).device + inputs = torch.randn(input_size).to(device) + + hook_module_list = [] + if custom_ops is None: + custom_ops = {} + custom_mask_ops.update(custom_ops) + prev_m = None + for m in model.modules(): + weight_mask = None + m_type = type(m) + if m_type in custom_mask_ops: + if isinstance(prev_m, PrunerModuleWrapper): + weight_mask = prev_m.weight_mask + + m.register_buffer('weight_mask', weight_mask) + hook_module_list.append(m) + prev_m = m + + flops, params = profile(model, inputs=(inputs, ), custom_ops=custom_mask_ops, verbose=verbose) + + + for m in hook_module_list: + m._buffers.pop("weight_mask") + # Remove registerd buffer on the model, and fixed following issue: + # https://github.com/Lyken17/pytorch-OpCounter/issues/96 + for m in model.modules(): + if 'total_ops' in m._buffers: + m._buffers.pop("total_ops") + if 'total_params' in m._buffers: + m._buffers.pop("total_params") + + return flops, params + +def count_convNd_mask(m, x, y): + """ + The forward hook to count FLOPs and Parameters of convolution operation. + Parameters + ---------- + m : torch.nn.Module + convolution module to calculate the FLOPs and Parameters + x : torch.Tensor + input data + y : torch.Tensor + output data + """ + output_channel = y.size()[1] + output_size = torch.zeros(y.size()[2:]).numel() + kernel_size = torch.zeros(m.weight.size()[2:]).numel() + + bias_flops = 1 if m.bias is not None else 0 + + if m.weight_mask is not None: + output_channel = m.weight_mask.sum() // (m.in_channels * kernel_size) + + total_ops = output_channel * output_size * (m.in_channels // m.groups * kernel_size + bias_flops) + + m.total_ops += torch.DoubleTensor([int(total_ops)]) + + +def count_linear_mask(m, x, y): + """ + The forward hook to count FLOPs and Parameters of linear transformation. + Parameters + ---------- + m : torch.nn.Module + linear to calculate the FLOPs and Parameters + x : torch.Tensor + input data + y : torch.Tensor + output data + """ + output_channel = y.numel() + + bias_flops = 1 if m.bias is not None else 0 + + if m.weight_mask is not None: + output_channel = m.weight_mask.sum() // m.in_features + + total_ops = output_channel * (m.in_features + bias_flops) + + m.total_ops += torch.DoubleTensor([int(total_ops)]) + + +custom_mask_ops = { + nn.Conv1d: count_convNd_mask, + nn.Conv2d: count_convNd_mask, + nn.Conv3d: count_convNd_mask, + nn.Linear: count_linear_mask, +} diff --git a/utils/third_party/nni/compression/pytorch/utils/mask_conflict.py b/utils/third_party/nni/compression/pytorch/utils/mask_conflict.py new file mode 100644 index 0000000000000000000000000000000000000000..dff2e97c63c30d268c59c1fe4859c8d5e24918f4 --- /dev/null +++ b/utils/third_party/nni/compression/pytorch/utils/mask_conflict.py @@ -0,0 +1,370 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. +import os +import logging +import torch +import numpy as np +from .shape_dependency import ChannelDependency, GroupDependency, CatPaddingDependency, InputChannelDependency +from .utils import get_module_by_name +# logging.basicConfig(level = logging.DEBUG) +_logger = logging.getLogger(__name__) + +def fix_mask_conflict(masks, model=None, dummy_input=None, traced=None): + """ + MaskConflict fix the mask conflict for the channel dependencies + and group dependency. + + Parameters + ---------- + masks : dict/str + A dict object that stores the masks or the path of the mask file + model : torch.nn.Module + model to fix the mask conflict + dummy_input : torch.Tensor + input example to trace the model + traced : torch._C.torch.jit.TopLevelTracedModule + the traced model of the target model, is this parameter is not None, + we donnot use the model and dummpy_input to get the trace graph. + """ + if isinstance(masks, str): + # if the input is the path of the mask_file + assert os.path.exists(masks) + masks = torch.load(masks) + # if the user uses the model and dummy_input to trace the model, we + # should get the traced model handly, so that, we only trace the + # model once, GroupMaskConflict and ChannelMaskConflict will reuse + # this traced model. + if traced is None: + assert model is not None and dummy_input is not None + with torch.onnx.set_training(model, False): + # We need to trace the model in this way, else it will have problems + traced = torch.jit.trace(model, dummy_input, check_trace=False) + + fix_group_mask = GroupMaskConflict(masks, model, dummy_input, traced) + masks = fix_group_mask.fix_mask() + fix_channel_mask = ChannelMaskConflict(masks, model, dummy_input, traced) + masks = fix_channel_mask.fix_mask() + padding_cat_mask = CatMaskPadding(masks, model, dummy_input, traced) + masks = padding_cat_mask.fix_mask() + return masks, fix_channel_mask.conv_prune_dim + +class MaskFix: + def __init__(self, masks, model=None, dummy_input=None, traced=None): + # check if the parameters are valid + parameter_valid = False + if traced is not None: + parameter_valid = True + elif (model is not None) and (dummy_input is not None): + parameter_valid = True + if not parameter_valid: + raise Exception('The input parameters is invalid!') + self.model = model + self.dummy_input = dummy_input + self.traced = traced + self.masks = masks + + def fix_mask(self): + raise NotImplementedError + + def export(self, path): + """ + Export the masks after fixing the conflict to file. + """ + torch.save(self.masks, path) + +class CatMaskPadding(MaskFix): + def __init__(self, masks, model, dummy_input=None, traced=None): + """ + CatMaskPadding find the layers whose output tensor is passed + to the same cat operation. The cat operation concatnates the + masks of the input tensors as the output mask, so when some + of the input layers of the cat operation are not pruned, we still + need to pass the masks of these non-pruned layers(the mask are + all ones) to the cat operation to ensure the shape of the output + mask is right. + + Parameters + ---------- + masks : dict + a dict object that stores the masks + model : torch.nn.Module + model to fix the mask conflict + dummy_input : torch.Tensor + input example to trace the model + traced : torch._C.torch.jit.TopLevelTracedModule + the traced model of the target model, is this parameter is not None, + we donnot use the model and dummpy_input to get the trace graph. + """ + super(CatMaskPadding, self).__init__(masks, model, dummy_input, traced) + + def fix_mask(self): + cat_padding_depen = CatPaddingDependency(self.model, self.dummy_input, self.traced) + name_to_module = {} + for name, module in self.model.named_modules(): + name_to_module[name] = module + depen = cat_padding_depen.dependency_sets + for layers in depen: + device = None + count = 0 + for layer in layers: + if layer in self.masks: + count += 1 + if device is None: + device = self.masks[layer]['weight'].device + if count == 0: + # no layer is pruned + continue + elif count == len(layers): + # all the layers have been pruned + continue + # pad the mask for the non-pruned layers + for layer in layers: + if layer in self.masks: + continue + module = name_to_module[layer] + w_shape = module.weight.data.size() + w_mask = torch.ones(w_shape).to(device) + b_mask = None + if hasattr(module, 'bias') and module.bias is not None: + # module.bias may be None + b_shape = module.bias.data.size() + b_mask = torch.ones(b_shape).to(device) + self.masks[layer] = {'weight':w_mask, 'bias':b_mask} + return self.masks + + + +class GroupMaskConflict(MaskFix): + def __init__(self, masks, model=None, dummy_input=None, traced=None): + """ + GroupMaskConflict fix the mask conflict between the layers that + has group dependecy with each other. + + Parameters + ---------- + masks : dict + a dict object that stores the masks + model : torch.nn.Module + model to fix the mask conflict + dummy_input : torch.Tensor + input example to trace the model + traced : torch._C.torch.jit.TopLevelTracedModule + the traced model of the target model, is this parameter is not None, + we donnot use the model and dummpy_input to get the trace graph. + """ + super(GroupMaskConflict, self).__init__(masks, model, dummy_input, traced) + + + def fix_mask(self): + """ + Fix the mask conflict before the mask inference for the layers that + has group dependencies. This function should be called before the + mask inference of the 'speedup' module. + """ + group_depen = GroupDependency(self.model, self.dummy_input, self.traced) + depens = group_depen.dependency + _logger.info(depens) + for layername in depens: + group = depens[layername] + if layername not in self.masks: + # this layer not pruned + continue + w_mask = self.masks[layername]['weight'] + shape = w_mask.size() + count = np.prod(shape[1:]) + all_ones = (w_mask.flatten(1).sum(-1) == count).nonzero().squeeze(1).tolist() + all_zeros = (w_mask.flatten(1).sum(-1) == 0).nonzero().squeeze(1).tolist() + if len(all_ones) + len(all_zeros) < w_mask.size(0): + # In fine-grained pruning, skip this layer + _logger.info('Layers %s using fine-grained pruning', layername) + continue + assert shape[0] % group == 0 + # Find the number of masked filter for each group (mini_masked). + # Because we have to keep the pruned filter can still + # be divided into the same number of groups, so we only can + # prune mini_masked filters for each group. + step = shape[0] / group + group_masked = [] + for i in range(group): + _start = step * i + _end = step * (i+1) + _tmp_list = list(filter(lambda x: _start <= x and x < _end, all_zeros)) + group_masked.append(_tmp_list) + mini_masked = min([len(x) for x in group_masked]) + for gm in group_masked: + for i in range(mini_masked, len(gm)): + # To keep the output channel number still being divisible to + # groups, we set the masks of following filters to be zero. + pos = gm[i] + self.masks[layername]['weight'][pos] = torch.ones(shape[1:]) + if hasattr(self.masks[layername], 'bias'): + self.masks[layername]['bias'][pos] = 1 + return self.masks + + + +class ChannelMaskConflict(MaskFix): + def __init__(self, masks, model=None, dummy_input=None, traced=None): + """ + ChannelMaskConflict fix the mask conflict between the layers that + has channel dependecy with each other. + + Parameters + ---------- + masks : dict + a dict object that stores the masks + model : torch.nn.Module + model to fix the mask conflict + dummy_input : torch.Tensor + input example to trace the model + graph : torch._C.torch.jit.TopLevelTracedModule + the traced graph of the target model, is this parameter is not None, + we donnot use the model and dummpy_input to get the trace graph. + """ + super(ChannelMaskConflict, self).__init__(masks, model, dummy_input, traced) + self.conv_prune_dim = detect_mask_prune_dim(masks, model) + _logger.info('detected conv prune dim: %s', self.conv_prune_dim) + + def fix_mask(self): + """ + Fix the mask conflict before the mask inference for the layers that + has shape dependencies. This function should be called before the + mask inference of the 'speedup' module. Only structured pruning masks + are supported. + """ + if self.conv_prune_dim == 0: + channel_depen = ChannelDependency(self.model, self.dummy_input, self.traced) + else: + channel_depen = InputChannelDependency(self.model, self.dummy_input, self.traced) + depen_sets = channel_depen.dependency_sets + sum_idx = (1, 2, 3) if self.conv_prune_dim == 0 else (0, 2, 3) + for dset in depen_sets: + if len(dset) <= 1: + continue + # channel_masks is a list, each element is None or a vector, for example: + # [[0, 1, 1, 0, 0], [0, 0, 1, 1, 0], None], None means no channel + # is pruned. + channel_masks = [] + fine_grained = False + for name in dset: + if name in self.masks: + _, m = get_module_by_name(self.model, name) + assert m is not None + mask = self.masks[name]['weight'] + if type(m).__name__ == 'Conv2d': + channel_mask = (mask.abs().sum(sum_idx) != 0).int() + channel_masks.append(channel_mask) + if (channel_mask.sum() * (mask.numel() / mask.shape[self.conv_prune_dim])).item() != (mask > 0).sum().item(): + fine_grained = True + elif type(m).__name__ == 'Linear': + channel_masks.append((mask.abs().sum(0) != 0).int()) + elif type(m).__name__ == 'BatchNorm2d': + channel_masks.append(mask.int()) + else: + raise RuntimeError('unsupported module type: {type(m).__name__}') + else: + # no mask means not pruned, equivlent to full masks + channel_masks.append(None) + if fine_grained: + _logger.info('fine-grained mask detected, skip solving conflict for this set: %s', dset) + continue + if all(x is None for x in channel_masks): + continue + num_channels_list = [len(x) for x in channel_masks if x is not None] + # number of channels in same set should be identical + assert len(set(num_channels_list)) == 1 + num_channels = num_channels_list[0] + + for i, dim_mask in enumerate(channel_masks): + if dim_mask is None: + # temporarily bug fix by queyu, 2020/11/22 + channel_masks[i] = torch.ones(num_channels).to('cuda').int() + + # merge masks with 'or' + merged_channel_mask = channel_masks[0].clone() + for i in range(1, len(channel_masks)): + merged_channel_mask = ((merged_channel_mask + channel_masks[i]) != 0).int() + + merged_index = torch.nonzero(merged_channel_mask, as_tuple=True)[0] + + for name in dset: + if name not in self.masks: + assert all(merged_channel_mask) + continue + orig_mask = self.masks[name]['weight'] + _, m = get_module_by_name(self.model, name) + new_mask = torch.zeros_like(orig_mask) + if type(m).__name__ == 'Conv2d': + if self.conv_prune_dim == 0: + new_mask[merged_index, :, :, :] = 1. + else: + new_mask[:, merged_index, :, :] = 1. + elif type(m).__name__ == 'Linear': + new_mask[:, merged_index] = 1. + elif type(m).__name__ == 'BatchNorm2d': + new_mask = merged_index.type_as(orig_mask) + else: + raise RuntimeError('unsupported module type: {type(m).__name__}') + + self.masks[name]['weight'] = new_mask + if 'bias' in self.masks[name] and self.masks[name]['bias'] is not None: + if type(m).__name__ == 'Conv2d': + assert self.conv_prune_dim == 0 + self.masks[name]['bias'] = merged_channel_mask.type_as(self.masks[name]['bias']) + + return self.masks + +def detect_mask_prune_dim(masks, model): + """ + Detect how the masks of convolutional layers are pruned. + + Parameters + ---------- + masks: dict + A dict object that stores the masks. + model: nn.Module + Model object which the mask can be applied on. + + Returns: + ------- + How the masks of convolutional layers are pruned, this depends on pruning algorithms, it should + return 1 for masks generated by AMCPruner, and returns 0 for masks generated by the rest + NNI builtin pruners. + 0: filter pruning, prune filters of weights which causes channels of output feature maps are pruned. + 1: channel pruning, prune kernels corresponding to each input channels which causes channels of + input feature maps are pruned. + """ + dim0_preserved, dim1_preserved = 0., 0. + dim0_num, dim1_num = 0., 0. + for module_name in masks: + _, m = get_module_by_name(model, module_name) + if m is None or type(m).__name__ != 'Conv2d': + continue + + mask = masks[module_name]['weight'].clone() + assert (mask >= 0).sum() == mask.numel(), \ + "mask values should be greater than or equal to 0." + mask = (mask > 0).int() + mask = mask.view(mask.shape[0], mask.shape[1], -1) + dim0_mask = (mask.sum((1, 2)) > 0).int() + dim1_mask = (mask.sum((0, 2)) > 0).int() + dim0_preserved += dim0_mask.sum().item() + dim1_preserved += dim1_mask.sum().item() + dim0_num += len(dim0_mask) + dim1_num += len(dim1_mask) + + if dim0_num == 0 or dim1_num == 0: + _logger.warning('no multi-dimension masks found.') + return 0 + + dim0_sparsity, dim1_sparsity = 1. - dim0_preserved / dim0_num, 1. - dim1_preserved / dim1_num + _logger.info('dim0 sparsity: %f', dim0_sparsity) + _logger.info('dim1 sparsity: %f', dim1_sparsity) + + if dim0_sparsity == dim1_sparsity == 0.: + _logger.warning('nothing masked.') + + if dim0_sparsity > 0 and dim1_sparsity > 0: + _logger.warning('both dim0 and dim1 masks found.') + + return 0 if dim0_sparsity >= dim1_sparsity else 1 diff --git a/utils/third_party/nni/compression/pytorch/utils/num_param_counter.py b/utils/third_party/nni/compression/pytorch/utils/num_param_counter.py new file mode 100644 index 0000000000000000000000000000000000000000..89ad0979943126c45112d8b865d87ec12cdb1961 --- /dev/null +++ b/utils/third_party/nni/compression/pytorch/utils/num_param_counter.py @@ -0,0 +1,16 @@ +def get_total_num_weights(model, op_types=['default']): + ''' + calculate the total number of weights + + Returns + ------- + int + total weights of all the op considered + ''' + num_weights = 0 + for _, module in model.named_modules(): + if module == model: + continue + if 'default' in op_types or type(module).__name__ in op_types: + num_weights += module.weight.data.numel() + return num_weights \ No newline at end of file diff --git a/utils/third_party/nni/compression/pytorch/utils/sensitivity_analysis.py b/utils/third_party/nni/compression/pytorch/utils/sensitivity_analysis.py new file mode 100644 index 0000000000000000000000000000000000000000..e6ca7a171c38c1acabceb1137033ca21d10ab5a3 --- /dev/null +++ b/utils/third_party/nni/compression/pytorch/utils/sensitivity_analysis.py @@ -0,0 +1,255 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import copy +import csv +import logging +from collections import OrderedDict + +import numpy as np +import torch.nn as nn + +# FIXME: I don't know where "utils" should be +SUPPORTED_OP_NAME = ['Conv2d', 'Conv1d'] +SUPPORTED_OP_TYPE = [getattr(nn, name) for name in SUPPORTED_OP_NAME] + +logger = logging.getLogger('Sensitivity_Analysis') +logger.setLevel(logging.INFO) + + +class SensitivityAnalysis: + def __init__(self, model, val_func, sparsities=None, prune_type='l1', early_stop_mode=None, early_stop_value=None, + train_loader=None): + """ + Perform sensitivity analysis for this model. + Parameters + ---------- + model : torch.nn.Module + the model to perform sensitivity analysis + val_func : function + validation function for the model. Due to + different models may need different dataset/criterion + , therefore the user need to cover this part by themselves. + In the val_func, the model should be tested on the validation dateset, + and the validation accuracy/loss should be returned as the output of val_func. + There are no restrictions on the input parameters of the val_function. + User can use the val_args, val_kwargs parameters in analysis + to pass all the parameters that val_func needed. + sparsities : list + The sparsity list provided by users. This parameter is set when the user + only wants to test some specific sparsities. In the sparsity list, each element + is a sparsity value which means how much weight the pruner should prune. Take + [0.25, 0.5, 0.75] for an example, the SensitivityAnalysis will prune 25% 50% 75% + weights gradually for each layer. + prune_type : str + The pruner type used to prune the conv layers, default is 'l1', + and 'l2', 'fine-grained' is also supported. + early_stop_mode : str + If this flag is set, the sensitivity analysis + for a conv layer will early stop when the validation metric( + for example, accurracy/loss) has alreay meet the threshold. We + support four different early stop modes: minimize, maximize, dropped, + raised. The default value is None, which means the analysis won't stop + until all given sparsities are tested. This option should be used with + early_stop_value together. + + minimize: The analysis stops when the validation metric return by the val_func + lower than early_stop_value. + maximize: The analysis stops when the validation metric return by the val_func + larger than early_stop_value. + dropped: The analysis stops when the validation metric has dropped by early_stop_value. + raised: The analysis stops when the validation metric has raised by early_stop_value. + early_stop_value : float + This value is used as the threshold for different earlystop modes. + This value is effective only when the early_stop_mode is set. + + """ + from nni.algorithms.compression.pytorch.pruning.constants_pruner import PRUNER_DICT + + self.model = model + self.val_func = val_func + self.target_layer = OrderedDict() + self.ori_state_dict = copy.deepcopy(self.model.state_dict()) + self.target_layer = {} + self.sensitivities = {} + if sparsities is not None: + self.sparsities = sorted(sparsities) + else: + self.sparsities = np.arange(0.1, 1.0, 0.1) + self.sparsities = [np.round(x, 2) for x in self.sparsities] + self.Pruner = PRUNER_DICT[prune_type] + self.early_stop_mode = early_stop_mode + self.early_stop_value = early_stop_value + self.ori_metric = None # original validation metric for the model + # already_pruned is for the iterative sensitivity analysis + # For example, sensitivity_pruner iteratively prune the target + # model according to the sensitivity. After each round of + # pruning, the sensitivity_pruner will test the new sensitivity + # for each layer + self.already_pruned = {} + self.model_parse() + + self.train_loader = train_loader + + @property + def layers_count(self): + return len(self.target_layer) + + def model_parse(self): + for name, submodel in self.model.named_modules(): + for op_type in SUPPORTED_OP_TYPE: + if isinstance(submodel, op_type): + self.target_layer[name] = submodel + self.already_pruned[name] = 0 + + def _need_to_stop(self, ori_metric, cur_metric): + """ + Judge if meet the stop conditon(early_stop, min_threshold, + max_threshold). + Parameters + ---------- + ori_metric : float + original validation metric + cur_metric : float + current validation metric + + Returns + ------- + stop : bool + if stop the sensitivity analysis + """ + if self.early_stop_mode is None: + # early stop mode is not enable + return False + assert self.early_stop_value is not None + if self.early_stop_mode == 'minimize': + if cur_metric < self.early_stop_value: + return True + elif self.early_stop_mode == 'maximize': + if cur_metric > self.early_stop_value: + return True + elif self.early_stop_mode == 'dropped': + if cur_metric < ori_metric - self.early_stop_value: + return True + elif self.early_stop_mode == 'raised': + if cur_metric > ori_metric + self.early_stop_value: + return True + return False + + def analysis(self, val_args=None, val_kwargs=None, specified_layers=None): + """ + This function analyze the sensitivity to pruning for + each conv layer in the target model. + If start and end are not set, we analyze all the conv + layers by default. Users can specify several layers to + analyze or parallelize the analysis process easily through + the start and end parameter. + + Parameters + ---------- + val_args : list + args for the val_function + val_kwargs : dict + kwargs for the val_funtion + specified_layers : list + list of layer names to analyze sensitivity. + If this variable is set, then only analyze + the conv layers that specified in the list. + User can also use this option to parallelize + the sensitivity analysis easily. + Returns + ------- + sensitivities : dict + dict object that stores the trajectory of the + accuracy/loss when the prune ratio changes + """ + if val_args is None: + val_args = [] + if val_kwargs is None: + val_kwargs = {} + # Get the original validation metric(accuracy/loss) before pruning + # Get the accuracy baseline before starting the analysis. + self.ori_metric = self.val_func(*val_args, **val_kwargs) + namelist = list(self.target_layer.keys()) + if specified_layers is not None: + # only analyze several specified conv layers + namelist = list(filter(lambda x: x in specified_layers, namelist)) + for name in namelist: + self.sensitivities[name] = {} + for sparsity in self.sparsities: + # here the sparsity is the relative sparsity of the + # the remained weights + # Calculate the actual prune ratio based on the already pruned ratio + real_sparsity = ( + 1.0 - self.already_pruned[name]) * sparsity + self.already_pruned[name] + # TODO In current L1/L2 Filter Pruner, the 'op_types' is still necessary + # I think the L1/L2 Pruner should specify the op_types automaticlly + # according to the op_names + cfg = [{'sparsity': real_sparsity, 'op_names': [ + name], 'op_types': ['Conv2d']}] + if self.train_loader: + pruner = self.Pruner(self.model, cfg, train_loader=self.train_loader) + else: + pruner = self.Pruner(self.model, cfg) + pruner.compress() + val_metric = self.val_func(*val_args, **val_kwargs) + logger.info('Layer: %s Sparsity: %.2f Validation Metric: %.4f', + name, real_sparsity, val_metric) + + self.sensitivities[name][sparsity] = val_metric + pruner._unwrap_model() + del pruner + # check if the current metric meet the stop condition + if self._need_to_stop(self.ori_metric, val_metric): + break + + # reset the weights pruned by the pruner, because the + # input sparsities is sorted, so we donnot need to reset + # weight of the layer when the sparsity changes, instead, + # we only need reset the weight when the pruning layer changes. + self.model.load_state_dict(self.ori_state_dict) + + return self.sensitivities + + def export(self, filepath): + """ + Export the results of the sensitivity analysis + to a csv file. The firstline of the csv file describe the content + structure. The first line is constructed by 'layername' and sparsity + list. Each line below records the validation metric returned by val_func + when this layer is under different sparsities. Note that, due to the early_stop + option, some layers may not have the metrics under all sparsities. + + layername, 0.25, 0.5, 0.75 + conv1, 0.6, 0.55 + conv2, 0.61, 0.57, 0.56 + + Parameters + ---------- + filepath : str + Path of the output file + """ + str_sparsities = [str(x) for x in self.sparsities] + header = ['layername'] + str_sparsities + with open(filepath, 'w') as csvf: + csv_w = csv.writer(csvf) + csv_w.writerow(header) + for layername in self.sensitivities: + row = [] + row.append(layername) + for sparsity in sorted(self.sensitivities[layername].keys()): + row.append(self.sensitivities[layername][sparsity]) + csv_w.writerow(row) + + def update_already_pruned(self, layername, ratio): + """ + Set the already pruned ratio for the target layer. + """ + self.already_pruned[layername] = ratio + + def load_state_dict(self, state_dict): + """ + Update the weight of the model + """ + self.ori_state_dict = copy.deepcopy(state_dict) + self.model.load_state_dict(self.ori_state_dict) diff --git a/utils/third_party/nni/compression/pytorch/utils/shape_dependency.py b/utils/third_party/nni/compression/pytorch/utils/shape_dependency.py new file mode 100644 index 0000000000000000000000000000000000000000..c68d09e7561777e1a3fdfe6f63926f58c45b0c25 --- /dev/null +++ b/utils/third_party/nni/compression/pytorch/utils/shape_dependency.py @@ -0,0 +1,489 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import csv +import logging + +__all__ = ['ChannelDependency', 'GroupDependency', 'CatPaddingDependency', 'InputChannelDependency'] + +CONV_TYPE = 'aten::_convolution' +ADD_TYPES = ['aten::add', 'aten::add_'] +CAT_TYPE = 'aten::cat' +logger = logging.getLogger('Shape_Dependency') +RESHAPE_OPS = [CAT_TYPE, 'aten::view', 'aten::reshape', 'aten::flatten', 'aten::mean'] + +class Dependency: + def __init__(self, model=None, dummy_input=None, traced_model=None): + """ + Build the graph for the model. + """ + from ....common.graph_utils import TorchModuleGraph + + # check if the input is legal + if traced_model is None: + # user should provide model & dummy_input to trace + # the model or a already traced model + assert model is not None and dummy_input is not None + self.graph = TorchModuleGraph(model, dummy_input, traced_model) + self.dependency = dict() + self.build_dependency() + + def build_dependency(self): + raise NotImplementedError + + def export(self, filepath): + raise NotImplementedError + +class ChannelDependency(Dependency): + def __init__(self, model=None, dummy_input=None, traced_model=None): + """ + This model analyze the channel dependencies between the conv + layers in a model. + + Parameters + ---------- + model : torch.nn.Module + The model to be analyzed. + data : torch.Tensor + The example input data to trace the network architecture. + traced_model : torch._C.Graph + if we alreay has the traced graph of the target model, we donnot + need to trace the model again. + """ + super(ChannelDependency, self).__init__(model, dummy_input, traced_model) + + def _get_parent_layers(self, node): + """ + Find the nearest father conv layers for the target node. + + Parameters + --------- + node : torch._C.Node + target node. + + Returns + ------- + parent_layers: list + nearest father conv/linear layers for the target worknode. + """ + parent_layers = [] + queue = [] + queue.append(node) + while queue: + curnode = queue.pop(0) + if curnode.op_type == 'Conv2d' or curnode.op_type == 'Linear': + # find the first met conv + parent_layers.append(curnode.name) + continue + parents = self.graph.find_predecessors(curnode.unique_name) + parents = [self.graph.name_to_node[name] for name in parents] + for parent in parents: + queue.append(parent) + return parent_layers + + def build_dependency(self): + """ + Build the channel dependency for the conv layers + in the model. + """ + # unpack the tuple/list manually before analyze the + # channel dependency + self.graph.unpack_manually() + for node in self.graph.nodes_py.nodes_op: + parent_layers = [] + # find the node that contains aten::add + # or aten::cat operations + if node.op_type in ADD_TYPES: + parent_layers = self._get_parent_layers(node) + elif node.op_type == CAT_TYPE: + # To determine if this cat operation will introduce channel + # dependency, we need the specific input parameters of the cat + # opertion. To get the input parameters of the cat opertion, we + # need to traverse all the cpp_nodes included by this NodePyGroup, + # because, TorchModuleGraph merges the important nodes and the adjacent + # unimportant nodes (nodes started with prim::attr, for example) into a + # NodepyGroup. + cat_dim = None + for cnode in node.node_cpps: + if cnode.kind() == CAT_TYPE: + cat_dim = list(cnode.inputs())[1].toIValue() + break + if cat_dim != 1: + parent_layers = self._get_parent_layers(node) + dependency_set = set(parent_layers) + # merge the dependencies + for parent in parent_layers: + if parent in self.dependency: + dependency_set.update(self.dependency[parent]) + # save the dependencies + for _node in dependency_set: + self.dependency[_node] = dependency_set + + + def export(self, filepath): + """ + export the channel dependencies as a csv file. + The layers at the same line have output channel + dependencies with each other. For example, + layer1.1.conv2, conv1, and layer1.0.conv2 have + output channel dependencies with each other, which + means the output channel(filters) numbers of these + three layers should be same with each other, otherwise + the model may has shape conflict. + + Output example: + Dependency Set,Convolutional Layers + Set 1,layer1.1.conv2,layer1.0.conv2,conv1 + Set 2,layer1.0.conv1 + Set 3,layer1.1.conv1 + """ + header = ['Dependency Set', 'Layers'] + setid = 0 + visited = set() + with open(filepath, 'w') as csvf: + csv_w = csv.writer(csvf, delimiter=',') + csv_w.writerow(header) + for node in self.graph.nodes_py.nodes_op: + if node.op_type != 'Conv2d' or node in visited: + continue + setid += 1 + row = ['Set %d' % setid] + if node.name not in self.dependency: + visited.add(node) + row.append(node.name) + else: + for other in self.dependency[node.name]: + visited.add(self.graph.name_to_node[other]) + row.append(other) + csv_w.writerow(row) + + @property + def dependency_sets(self): + """ + Get the list of the dependency set. + + Returns + ------- + dependency_sets : list + list of the dependency sets. For example, + [set(['conv1', 'conv2']), set(['conv3', 'conv4'])] + + """ + d_sets = [] + visited = set() + for node in self.graph.nodes_py.nodes_op: + if node.op_type != 'Conv2d' or node in visited: + continue + tmp_set = set() + if node.name not in self.dependency: + visited.add(node) + tmp_set.add(node.name) + else: + for other in self.dependency[node.name]: + visited.add(self.graph.name_to_node[other]) + tmp_set.add(other) + d_sets.append(tmp_set) + return d_sets + +def reshape_break_channel_dependency(op_node): + """ + The reshape operations such as (reshape, view, flatten) may break + the channel dependency. We need to check the input parameters of + these reshape operations to check if this reshape node will break + the channel dependency. However, it's complicated to analyze the the input + parameters for each reshape function and infer if it will break the channel + dependency. So currently, we just check if the input channel and the output + channel is the same, if so, then we can say the original reshape function + doesn't want to change the number of the channels, which means the channel + dependency is not broken. In contrast, the original reshap operation wants + to change the number of channels, so it breaks the channel dependency. + + Parameters + ---------- + opnode: NodePyOP + A Op node of the graph. + Returns + ------- + bool + If this operation will break the channel dependency. + """ + in_shape = op_node.auxiliary['in_shape'] + out_shape = op_node.auxiliary['out_shape'] + in_channel = in_shape[1] + out_channel = out_shape[1] + return in_channel != out_channel + +class InputChannelDependency(ChannelDependency): + """ + Some pruners may prune the input channel of the convolutional + layers. While pruning the input channel of the convolutional layers, + the layers that share the same input tensor should prune the same + channels, and we say these layers that share the same input tensor/channel + has the input channel dependency. If we only prune the input channel of one + layer in the dependency set, there will be a shape conflict for the other + layers in the same dependency set, which may trigger a runtime error. + Here we judge whether the application will truncate the dependency by analyzing + whether the number of channels before and after the operation has changed. + If not, the input channel dependency will be passed to the following nodes. + """ + + def __init__(self, model, dummy_input=None, traced_model=None): + """ + This model analyze the input channel dependencies between the conv + layers in a model. + + Parameters + ---------- + model : torch.nn.Module + The model to be analyzed. + data : torch.Tensor + The example input data to trace the network architecture. + traced_model : torch._C.Graph + if we alreay has the traced graph of the target model, we donnot + need to trace the model again. + """ + super(InputChannelDependency, self).__init__(model, dummy_input, traced_model) + + def _get_following_convs(self, tensor): + queue = [] + key_layers = [] + queue.extend(self.graph.input_to_node[tensor]) + while queue: + curnode = queue.pop(0) + if curnode.op_type == 'Conv2d' or curnode.op_type == 'Linear': + # find the first met conv + key_layers.append(curnode.name) + continue + elif curnode.op_type in RESHAPE_OPS: + # check if the reshape operation will break the channel dependency + if reshape_break_channel_dependency(curnode): + # reshape operations also breaks the dependency relationship + continue + successors = self.graph.find_successors(curnode.unique_name) + successors = [self.graph.name_to_node[name] for name in successors] + for layer in successors: + queue.append(layer) + return key_layers + + def build_dependency(self): + """ + Build the input channel dependencies. + The `InputChannelDependency` indicates the layers that have + dependencies when pruning the input channel of the conv layers. + In contrast, `ChannelDependency` indicates the dependent layers + when pruning the output channles of conv layers (for example, L1FilterPruner). + """ + # unpack the tuple or list manually + self.graph.unpack_manually() + for tensor in self.graph.input_to_node: + # start from this tensor, find all the conv layers that + # take this tensor as input. Similar to the `ChannelDependency` + # the conv layer will truncate the dependencies + layers = self._get_following_convs(tensor) + dependency_set = set(layers) + for layer in layers: + if layer in self.dependency: + dependency_set.update(self.dependency[layer]) + for layer in dependency_set: + self.dependency[layer] = dependency_set + + +class CatPaddingDependency(ChannelDependency): + def __init__(self, model=None, dummy_input=None, traced_model=None): + super(CatPaddingDependency, self).__init__(model, dummy_input, traced_model) + + def build_dependency(self): + """ + Build the cat padding dependencies. + If the output features of several layers are stitched together + by cat operation, then these layers have cat padding dependencies. + This is because when inferring the cat mask, we need all the input + masks for the cat operation. At this time we need to know the source + of all input vectors of a cat operation. + """ + for node in self.graph.nodes_py.nodes_op: + parent_layers = [] + if node.op_type == CAT_TYPE: + parent_layers = self._get_parent_layers(node) + dependency_set = set(parent_layers) + # merge the dependencies + for parent in parent_layers: + if parent in self.dependency: + dependency_set.update(self.dependency[parent]) + # save the dependencies + for _node in dependency_set: + self.dependency[_node] = dependency_set + + @property + def dependency_sets(self): + d_sets = [] + visited = set() + for nodename in self.dependency: + if nodename in visited: + continue + d_sets.append(self.dependency[nodename]) + return d_sets + + def export(self, filepath): + """ + Export the dependencies into a file. + In the output file, each line contains a set of layers + whose output features are stitched together by the cat + operation. + + output example: + Dependency Set, Layers + set1, Conv1, Conv2 + set2, Conv3, Conv4 + """ + header = ['Dependency Set', 'Layers'] + setid = 0 + with open(filepath, 'w') as csvf: + csv_w = csv.writer(csvf, delimiter=',') + csv_w.writerow(header) + for layers in self.dependency_sets: + setid += 1 + row = ['Set %d' % setid] + row.extend(list(layers)) + csv_w.writerow(row) + +class GroupDependency(Dependency): + def __init__(self, model=None, dummy_input=None, traced_model=None): + """ + This model analyze the group dependencis between the conv + layers in a model. + + Parameters + ---------- + model : torch.nn.Module + The model to be analyzed. + data : torch.Tensor + The example input data to trace the network architecture. + traced_model : torch._C.Graph + if we alreay has the traced graph of the target model, we donnot + need to trace the model again. + """ + super(GroupDependency, self).__init__(model, dummy_input, traced_model) + + def _get_parent_convs(self, node): + """ + Find the nearest father conv layers for the target node. + + Parameters + --------- + node : torch._C.Node + target node. + + Returns + ------- + parent_layers : list + nearest father conv layers for the target node. Due to the group + dependency only exists between the conv layers, so we only find + the parent conv layers. + """ + parent_layers = [] + # the input node is a Conv node + predeessors = self.graph.find_predecessors(node.unique_name) + predeessors = [self.graph.name_to_node[x] for x in predeessors] + queue = predeessors + while queue: + curnode = queue.pop(0) + if curnode.op_type == 'Conv2d': + # find the first met conv + parent_layers.append(curnode.name) + continue + parents = self.graph.find_predecessors(curnode.unique_name) + parents = [self.graph.name_to_node[name] for name in parents] + for parent in parents: + queue.append(parent) + return parent_layers + + def _get_conv_groups(self, node_group): + """ + Get the number of groups for a convolutional layer. + + Parameters + ---------- + node_group : NodePyGroup + target node. + + Returns + ------- + group : int + the number of the groups of the target conv layer. + """ + cpp_conv = list(filter(lambda x: x.kind() == CONV_TYPE, node_group.node_cpps)) + assert len(cpp_conv) == 1 + cpp_conv = cpp_conv[0] + inputs = list(cpp_conv.inputs()) + # get the number of the group from the input parameters + group = inputs[8].toIValue() + return group + + def build_dependency(self): + """ + Build the channel dependency for the conv layers + in the model. This function return the group number + of each conv layers. Note that, here, the group count + of conv layers may be larger than their originl groups. + This is because that the input channel will also be grouped + for the group conv layers. To make this clear, assume we + have two group conv layers: conv1(group=2), conv2(group=4). + conv2 takes the output features of conv1 as input. + Then we have to the filters of conv1 can still be + divided into 4 groups after filter pruning, because + the input channels of conv2 shoule be divided into + 4 groups. + + Returns + ------- + self.dependency : dict + key: the name of conv layers, value: the minimum value that the number of + filters should be divisible to. + """ + for node in self.graph.nodes_py.nodes_op: + if node.op_type == 'Conv2d': + group = self._get_conv_groups(node) + if node.name in self.dependency: + # the conv layer whose group is larger than 1 will require that + # it's number of output channel to be divisible by the number of group. + self.dependency[node.name] = max(self.dependency[node.name], group) + else: + self.dependency[node.name] = group + if group > 1: + # for the conv layer whose group is larger than 1, it will require the number + # of output channels of their parent conv layer to be divisible by group. + parent_convs = self._get_parent_convs(node) + for parent in parent_convs: + if parent in self.dependency: + self.dependency[parent] = max(self.dependency[parent], group) + else: + self.dependency[parent] = group + return self.dependency + + def export(self, filepath): + """ + export the group dependency to a csv file. + Each line describes a convolution layer, the + first part of each line is the Pytorch module + name of the conv layer. The second part of each + line is the group count of the filters in this layer. + Note that, the group count may be larger than this + layers original group number. + + output example: + Conv layer, Groups + Conv1, 1 + Conv2, 2 + Conv3, 4 + """ + header = ['Conv Layer Name', 'Group'] + with open(filepath, 'w') as csvf: + csv_w = csv.writer(csvf, delimiter=',') + csv_w.writerow(header) + for name in self.dependency: + group = self.dependency[name] + csv_w.writerow([name, group]) + @property + def dependency_sets(self): + return self.dependency diff --git a/utils/third_party/nni/compression/pytorch/utils/utils.py b/utils/third_party/nni/compression/pytorch/utils/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..c687c5e2a6bf971433e6298ee92a86be5c23b6d4 --- /dev/null +++ b/utils/third_party/nni/compression/pytorch/utils/utils.py @@ -0,0 +1,30 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +def get_module_by_name(model, module_name): + """ + Get a module specified by its module name + + Parameters + ---------- + model : pytorch model + the pytorch model from which to get its module + module_name : str + the name of the required module + + Returns + ------- + module, module + the parent module of the required module, the required module + """ + name_list = module_name.split(".") + for name in name_list[:-1]: + if hasattr(model, name): + model = getattr(model, name) + else: + return None, None + if hasattr(model, name_list[-1]): + leaf_module = getattr(model, name_list[-1]) + return model, leaf_module + else: + return None, None diff --git a/utils/third_party/nni/compression/tensorflow/__init__.py b/utils/third_party/nni/compression/tensorflow/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d05fade2f1140c68bb78969f89aaa8529414f3ca --- /dev/null +++ b/utils/third_party/nni/compression/tensorflow/__init__.py @@ -0,0 +1,4 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .compressor import Compressor, Pruner diff --git a/utils/third_party/nni/compression/tensorflow/compressor.py b/utils/third_party/nni/compression/tensorflow/compressor.py new file mode 100644 index 0000000000000000000000000000000000000000..7f9b1bc6aed33f2922dff409982906799a8e636e --- /dev/null +++ b/utils/third_party/nni/compression/tensorflow/compressor.py @@ -0,0 +1,256 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +Abstract base classes for TensorFlow model compression. +""" + +import logging + +import tensorflow as tf +assert tf.__version__.startswith('2'), 'NNI model compression only supports TensorFlow v2.x' + +from . import default_layers + +_logger = logging.getLogger(__name__) + + +class Compressor: + """ + Common base class for all compressors. + + This class is designed for other base classes. + Algorithms should inherit ``Pruner`` or ``Quantizer`` instead. + + Attributes + ---------- + compressed_model : tf.keras.Model + Compressed user model. + wrappers : list of tf.keras.Model + A wrapper is an instrumented TF ``Layer``, in ``Model`` format. + + Parameters + ---------- + model : tf.keras.Model + The user model to be compressed. + config_list : list of JSON object + User configuration. The format is detailed in tutorial. + LayerWrapperClass : a class derive from Model + The class used to instrument layers. + """ + + def __init__(self, model, config_list, LayerWrapperClass): + assert isinstance(model, tf.keras.Model) + self.validate_config(model, config_list) + + self._original_model = model + self._config_list = config_list + self._wrapper_class = LayerWrapperClass + self._wrappers = {} # key: id(layer) , value: Wrapper(layer) + + self.compressed_model = self._instrument(model) + self.wrappers = list(self._wrappers.values()) + + if not self.wrappers: + _logger.warning('Nothing is configured to compress, please check your model and config list') + + def set_wrappers_attribute(self, name, value): + """ + Call ``setattr`` on all wrappers. + """ + for wrapper in self.wrappers: + setattr(wrapper, name, value) + + def validate_config(self, model, config_list): + """ + Compression algorithm should overload this function to validate configuration. + """ + pass + + + def _instrument(self, layer): + if isinstance(layer, tf.keras.Sequential): + return self._instrument_sequential(layer) + if isinstance(layer, tf.keras.Model): + return self._instrument_model(layer) + + # a layer can be referenced in multiple attributes of a model, + # but should only be instrumented once + if id(layer) in self._wrappers: + return self._wrappers[id(layer)] + + config = self._select_config(layer) + if config is not None: + wrapper = self._wrapper_class(layer, config, self) + self._wrappers[id(layer)] = wrapper + return wrapper + + return layer + + def _instrument_sequential(self, seq): + layers = list(seq.layers) # seq.layers is read-only property + need_rebuild = False + for i, layer in enumerate(layers): + new_layer = self._instrument(layer) + if new_layer is not layer: + layers[i] = new_layer + need_rebuild = True + return tf.keras.Sequential(layers) if need_rebuild else seq + + def _instrument_model(self, model): + for key, value in list(model.__dict__.items()): # avoid "dictionary keys changed during iteration" + if isinstance(value, tf.keras.layers.Layer): + new_layer = self._instrument(value) + if new_layer is not value: + setattr(model, key, new_layer) + elif isinstance(value, list): + for i, item in enumerate(value): + if isinstance(item, tf.keras.layers.Layer): + value[i] = self._instrument(item) + return model + + + def _select_config(self, layer): + # Find the last matching config block for given layer. + # Returns None if the layer should not be compressed. + layer_type = type(layer).__name__ + last_match = None + for config in self._config_list: + if 'op_types' in config: + match = layer_type in config['op_types'] + match_default = 'default' in config['op_types'] and layer_type in default_layers.weighted_modules + if not match and not match_default: + continue + if 'op_names' in config and layer.name not in config['op_names']: + continue + last_match = config + if last_match is None or 'exclude' in last_match: + return None + return last_match + + +class Pruner(Compressor): + """ + Base class for pruning algorithms. + + End users should use ``compress`` and callback APIs (WIP) to prune their models. + + The underlying model is instrumented upon initialization of pruner object. + So if you want to pre-train the model, train it before creating pruner object. + + The compressed model can only execute in eager mode. + + Algorithm developers should override ``calc_masks`` method to specify pruning strategy. + + Parameters + ---------- + model : tf.keras.Model + The user model to prune. + config_list : list of JSON object + User configuration. The format is detailed in tutorial. + """ + def __init__(self, model, config_list): + super().__init__(model, config_list, PrunerLayerWrapper) + #self.callback = PrunerCallback(self) + + def compress(self): + """ + Apply compression on a pre-trained model. + + If you want to prune the model during training, use callback API (WIP) instead. + + Returns + ------- + tf.keras.Model + The compressed model. + """ + self._update_mask() + return self.compressed_model + + def calc_masks(self, wrapper, **kwargs): + """ + Abstract method to be overridden by algorithm. End users should ignore it. + + If the callback is set up, this method will be invoked at end of each training minibatch. + If not, it will only be called when end user invokes ``compress``. + + Parameters + ---------- + wrapper : PrunerLayerWrapper + The instrumented layer. + **kwargs + Reserved for forward compatibility. + + Returns + ------- + dict of (str, tf.Tensor), or None + The key is weight ``Variable``'s name. The value is a mask ``Tensor`` of weight's shape and dtype. + If a weight's key does not appear in the return value, that weight will not be pruned. + Returning ``None`` means the mask is not changed since last time. + Weight names are globally unique, e.g. `model/conv_1/kernel:0`. + """ + # TODO: maybe it should be able to calc on weight-granularity, beside from layer-granularity + raise NotImplementedError("Pruners must overload calc_masks()") + + def _update_mask(self): + for wrapper_idx, wrapper in enumerate(self.wrappers): + masks = self.calc_masks(wrapper, wrapper_idx=wrapper_idx) + if masks is not None: + wrapper.masks = masks + + +class PrunerLayerWrapper(tf.keras.Model): + """ + Instrumented TF layer. + + Wrappers will be passed to pruner's ``calc_masks`` API, + and the pruning algorithm should use wrapper's attributes to calculate masks. + + Once instrumented, underlying layer's weights will get **modified** by masks before forward pass. + + Attributes + ---------- + layer_info : LayerInfo + All static information of the original layer. + layer : tf.keras.layers.Layer + The original layer. + config : JSON object + Selected configuration. The format is detailed in tutorial. + pruner : Pruner + Bound pruner object. + masks : dict of (str, tf.Tensor) + Current masks. The key is weight's name and the value is mask tensor. + On initialization, `masks` is an empty dict, which means no weight is pruned. + Afterwards, `masks` is the last return value of ``Pruner.calc_masks``. + See ``Pruner.calc_masks`` for details. + """ + def __init__(self, layer, config, pruner): + super().__init__() + self.layer = layer + self.config = config + self.pruner = pruner + self.masks = {} + _logger.info('Layer detected to compress: %s', self.layer.name) + + def call(self, *inputs): + new_weights = [] + for weight in self.layer.weights: + mask = self.masks.get(weight.name) + if mask is not None: + new_weights.append(tf.math.multiply(weight, mask)) + else: + new_weights.append(weight) + if new_weights and not hasattr(new_weights[0], 'numpy'): + raise RuntimeError('NNI: Compressed model can only run in eager mode') + self.layer.set_weights([weight.numpy() for weight in new_weights]) + return self.layer(*inputs) + + +# TODO: designed to replace `patch_optimizer` +#class PrunerCallback(tf.keras.callbacks.Callback): +# def __init__(self, pruner): +# super().__init__() +# self._pruner = pruner +# +# def on_train_batch_end(self, batch, logs=None): +# self._pruner.update_mask() diff --git a/utils/third_party/nni/compression/tensorflow/default_layers.py b/utils/third_party/nni/compression/tensorflow/default_layers.py new file mode 100644 index 0000000000000000000000000000000000000000..0c729bd883f1623d28ad279de053a39a5742109c --- /dev/null +++ b/utils/third_party/nni/compression/tensorflow/default_layers.py @@ -0,0 +1,9 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +weighted_modules = [ + 'Conv1D', 'Conv2D', 'Conv3D', 'Conv1DTranspose', 'Conv2DTranspose', 'Conv3DTranspose', + 'Dense', + 'PReLU', + 'Embedding', +] diff --git a/utils/third_party/nni/experiment/__init__.py b/utils/third_party/nni/experiment/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..20b244fe62c321d36dda1f646f8abe94ae8e6b58 --- /dev/null +++ b/utils/third_party/nni/experiment/__init__.py @@ -0,0 +1,4 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .nni_client import * diff --git a/utils/third_party/nni/experiment/nni_client.py b/utils/third_party/nni/experiment/nni_client.py new file mode 100644 index 0000000000000000000000000000000000000000..1dddafd219e546667edec498fc161f9aef7f1845 --- /dev/null +++ b/utils/third_party/nni/experiment/nni_client.py @@ -0,0 +1,515 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" A python wrapper for nni rest api + +Example: + +from nni.experiment import Experiment + +exp = Experiment() +exp.start_experiment('../../../../examples/trials/mnist-pytorch/config.yml') + +exp.update_concurrency(3) + +print(exp.get_experiment_status()) +print(exp.get_job_statistics()) +print(exp.list_trial_jobs()) + +exp.stop_experiment() + +""" + +import sys +import os +import subprocess +import re +import json +import requests + +__all__ = [ + 'Experiment', + 'TrialResult', + 'TrialMetricData', + 'TrialHyperParameters', + 'TrialJob' +] + +EXPERIMENT_PATH = 'experiment' +STATUS_PATH = 'check-status' +JOB_STATISTICS_PATH = 'job-statistics' +TRIAL_JOBS_PATH = 'trial-jobs' +METRICS_PATH = 'metric-data' +EXPORT_DATA_PATH = 'export-data' +API_ROOT_PATH = 'api/v1/nni' + +def _nni_rest_get(endpoint, api_path, response_type='json'): + _check_endpoint(endpoint) + uri = '{}/{}/{}'.format(endpoint.strip('/'), API_ROOT_PATH, api_path) + res = requests.get(uri) + if _http_succeed(res.status_code): + if response_type == 'json': + return res.json() + elif response_type == 'text': + return res.text + else: + raise RuntimeError('Incorrect response_type') + else: + return None + +def _http_succeed(status_code): + return status_code // 100 == 2 + +def _create_process(cmd): + if sys.platform == 'win32': + process = subprocess.Popen(cmd, stdout=subprocess.PIPE, creationflags=subprocess.CREATE_NEW_PROCESS_GROUP) + else: + process = subprocess.Popen(cmd, stdout=subprocess.PIPE) + + while process.poll() is None: + output = process.stdout.readline() + if output: + print(output.decode('utf-8').strip()) + return process.returncode + +def _check_endpoint(endpoint): + if endpoint is None: + raise RuntimeError("This instance hasn't been connect to an experiment.") + +class TrialResult: + """ + TrialResult stores the result information of a trial job. + + Parameters + ---------- + json_obj: dict + Json object that stores the result information. + + Attributes + ---------- + parameter: dict + Hyper parameters for this trial. + value: serializable object, usually a number, or a dict with key "default" and other extra keys + Final result. + trialJobId: str + Trial job id. + """ + def __init__(self, json_obj): + self.parameter = None + self.value = None + self.trialJobId = None + for key in json_obj.keys(): + if key == 'id': + setattr(self, 'trialJobId', json_obj[key]) + elif hasattr(self, key): + setattr(self, key, json_obj[key]) + self.value = json.loads(self.value) + + def __repr__(self): + return "TrialResult(parameter: {} value: {} trialJobId: {})".format(self.parameter, self.value, self.trialJobId) + +class TrialMetricData: + """ + TrialMetricData stores the metric data of a trial job. + A trial job may have both intermediate metric and final metric. + + Parameters + ---------- + json_obj: dict + Json object that stores the metric data. + + Attributes + ---------- + timestamp: int + Time stamp. + trialJobId: str + Trial job id. + parameterId: int + Parameter id. + type: str + Metric type, `PERIODICAL` for intermediate result and `FINAL` for final result. + sequence: int + Sequence number in this trial. + data: serializable object, usually a number, or a dict with key "default" and other extra keys + Metric data. + """ + def __init__(self, json_obj): + self.timestamp = None + self.trialJobId = None + self.parameterId = None + self.type = None + self.sequence = None + self.data = None + for key in json_obj.keys(): + setattr(self, key, json_obj[key]) + self.data = json.loads(json.loads(self.data)) + + def __repr__(self): + return "TrialMetricData(timestamp: {} trialJobId: {} parameterId: {} type: {} sequence: {} data: {})" \ + .format(self.timestamp, self.trialJobId, self.parameterId, self.type, self.sequence, self.data) + +class TrialHyperParameters: + """ + TrialHyperParameters stores the hyper parameters of a trial job. + + Parameters + ---------- + json_obj: dict + Json object that stores the hyper parameters. + + Attributes + ---------- + parameter_id: int + Parameter id. + parameter_source: str + Parameter source. + parameters: dict + Hyper parameters. + parameter_index: int + Parameter index. + """ + def __init__(self, json_obj): + self.parameter_id = None + self.parameter_source = None + self.parameters = None + self.parameter_index = None + for key in json_obj.keys(): + if hasattr(self, key): + setattr(self, key, json_obj[key]) + + def __repr__(self): + return "TrialHyperParameters(parameter_id: {} parameter_source: {} parameters: {} parameter_index: {})" \ + .format(self.parameter_id, self.parameter_source, self.parameters, self.parameter_index) + +class TrialJob: + """ + TrialJob stores the information of a trial job. + + Parameters + ---------- + json_obj: dict + json object that stores the hyper parameters + + Attributes + ---------- + trialJobId: str + Trial job id. + status: str + Job status. + hyperParameters: list of `nni.experiment.TrialHyperParameters` + See `nni.experiment.TrialHyperParameters`. + logPath: str + Log path. + startTime: int + Job start time (timestamp). + endTime: int + Job end time (timestamp). + finalMetricData: list of `nni.experiment.TrialMetricData` + See `nni.experiment.TrialMetricData`. + parameter_index: int + Parameter index. + """ + def __init__(self, json_obj): + self.trialJobId = None + self.status = None + self.hyperParameters = None + self.logPath = None + self.startTime = None + self.endTime = None + self.finalMetricData = None + self.stderrPath = None + for key in json_obj.keys(): + if key == 'id': + setattr(self, 'trialJobId', json_obj[key]) + elif hasattr(self, key): + setattr(self, key, json_obj[key]) + if self.hyperParameters: + self.hyperParameters = [TrialHyperParameters(json.loads(e)) for e in self.hyperParameters] + if self.finalMetricData: + self.finalMetricData = [TrialMetricData(e) for e in self.finalMetricData] + + def __repr__(self): + return ("TrialJob(trialJobId: {} status: {} hyperParameters: {} logPath: {} startTime: {} " + "endTime: {} finalMetricData: {} stderrPath: {})") \ + .format(self.trialJobId, self.status, self.hyperParameters, self.logPath, + self.startTime, self.endTime, self.finalMetricData, self.stderrPath) + +class Experiment: + def __init__(self): + self._endpoint = None + self._exp_id = None + self._port = None + + @property + def endpoint(self): + return self._endpoint + + @property + def exp_id(self): + return self._exp_id + + @property + def port(self): + return self._port + + def _exec_command(self, cmd, port=None): + if self._endpoint is not None: + raise RuntimeError('This instance has been connected to an experiment.') + if _create_process(cmd) != 0: + raise RuntimeError('Failed to establish experiment, please check your config.') + else: + if port: + self._port = port + else: + self._port = 8080 + self._endpoint = 'http://localhost:{}'.format(self._port) + self._exp_id = self.get_experiment_profile()['id'] + + def start_experiment(self, config_file, port=None, debug=False): + """ + Start an experiment with specified configuration file and connect to it. + + Parameters + ---------- + config_file: str + Path to the config file. + port: int + The port of restful server, bigger than 1024. + debug: boolean + Set debug mode. + """ + cmd = 'nnictl create --config {}'.format(config_file).split(' ') + if port: + cmd += '--port {}'.format(port).split(' ') + if debug: + cmd += ['--debug'] + self._exec_command(cmd, port) + + def resume_experiment(self, exp_id, port=None, debug=False): + """ + Resume a stopped experiment with specified experiment id + + Parameters + ---------- + exp_id: str + Experiment id. + port: int + The port of restful server, bigger than 1024. + debug: boolean + Set debug mode. + """ + cmd = 'nnictl resume {}'.format(exp_id).split(' ') + if port: + cmd += '--port {}'.format(port).split(' ') + if debug: + cmd += ['--debug'] + self._exec_command(cmd, port) + + def view_experiment(self, exp_id, port=None): + """ + View a stopped experiment with specified experiment id. + + Parameters + ---------- + exp_id: str + Experiment id. + port: int + The port of restful server, bigger than 1024. + """ + cmd = 'nnictl view {}'.format(exp_id).split(' ') + if port: + cmd += '--port {}'.format(port).split(' ') + self._exec_command(cmd, port) + + def connect_experiment(self, endpoint): + """ + Connect to an existing experiment. + + Parameters + ---------- + endpoint: str + The endpoint of nni rest server, i.e, the url of Web UI. Should be a format like `http://ip:port`. + """ + if self._endpoint is not None: + raise RuntimeError('This instance has been connected to an experiment.') + self._endpoint = endpoint + try: + self._exp_id = self.get_experiment_profile()['id'] + except TypeError: + raise RuntimeError('Invalid experiment endpoint.') + self._port = int(re.search(r':[0-9]+', self._endpoint).group().replace(':', '')) + + def stop_experiment(self): + """Stop the experiment. + """ + _check_endpoint(self._endpoint) + cmd = 'nnictl stop {}'.format(self._exp_id).split(' ') + if _create_process(cmd) != 0: + raise RuntimeError('Failed to stop experiment.') + self._endpoint = None + self._exp_id = None + self._port = None + + def update_searchspace(self, filename): + """ + Update the experiment's search space. + + Parameters + ---------- + filename: str + Path to the searchspace file. + """ + _check_endpoint(self._endpoint) + cmd = 'nnictl update searchspace {} --filename {}'.format(self._exp_id, filename).split(' ') + if _create_process(cmd) != 0: + raise RuntimeError('Failed to update searchspace.') + + def update_concurrency(self, value): + """ + Update an experiment's concurrency + + Parameters + ---------- + value: int + New concurrency value. + """ + _check_endpoint(self._endpoint) + cmd = 'nnictl update concurrency {} --value {}'.format(self._exp_id, value).split(' ') + if _create_process(cmd) != 0: + raise RuntimeError('Failed to update concurrency.') + + def update_duration(self, value): + """ + Update an experiment's duration + + Parameters + ---------- + value: str + Strings like '1m' for one minute or '2h' for two hours. + SUFFIX may be 's' for seconds, 'm' for minutes, 'h' for hours or 'd' for days. + """ + _check_endpoint(self._endpoint) + cmd = 'nnictl update duration {} --value {}'.format(self._exp_id, value).split(' ') + if _create_process(cmd) != 0: + raise RuntimeError('Failed to update duration.') + + def update_trailnum(self, value): + """ + Update an experiment's maxtrialnum + + Parameters + ---------- + value: int + New trailnum value. + """ + _check_endpoint(self._endpoint) + cmd = 'nnictl update trialnum {} --value {}'.format(self._exp_id, value).split(' ') + if _create_process(cmd) != 0: + raise RuntimeError('Failed to update trailnum.') + + def get_experiment_status(self): + """ + Return experiment status as a dict. + + Returns + ---------- + dict + Experiment status. + """ + _check_endpoint(self._endpoint) + return _nni_rest_get(self._endpoint, STATUS_PATH) + + def get_trial_job(self, trial_job_id): + """ + Return a trial job. + + Parameters + ---------- + trial_job_id: str + Trial job id. + + Returns + ---------- + nnicli.TrialJob + A `nnicli.TrialJob` instance corresponding to `trial_job_id`. + """ + _check_endpoint(self._endpoint) + assert trial_job_id is not None + trial_job = _nni_rest_get(self._endpoint, os.path.join(TRIAL_JOBS_PATH, trial_job_id)) + return TrialJob(trial_job) + + def list_trial_jobs(self): + """ + Return information for all trial jobs as a list. + + Returns + ---------- + list + List of `nnicli.TrialJob`. + """ + _check_endpoint(self._endpoint) + trial_jobs = _nni_rest_get(self._endpoint, TRIAL_JOBS_PATH) + return [TrialJob(e) for e in trial_jobs] + + def get_job_statistics(self): + """ + Return trial job statistics information as a dict. + + Returns + ---------- + list + Job statistics information. + """ + _check_endpoint(self._endpoint) + return _nni_rest_get(self._endpoint, JOB_STATISTICS_PATH) + + def get_job_metrics(self, trial_job_id=None): + """ + Return trial job metrics. + + Parameters + ---------- + trial_job_id: str + trial job id. if this parameter is None, all trail jobs' metrics will be returned. + + Returns + ---------- + dict + Each key is a trialJobId, the corresponding value is a list of `nnicli.TrialMetricData`. + """ + _check_endpoint(self._endpoint) + api_path = METRICS_PATH if trial_job_id is None else os.path.join(METRICS_PATH, trial_job_id) + output = {} + trail_metrics = _nni_rest_get(self._endpoint, api_path) + for metric in trail_metrics: + trial_id = metric["trialJobId"] + if trial_id not in output: + output[trial_id] = [TrialMetricData(metric)] + else: + output[trial_id].append(TrialMetricData(metric)) + return output + + def export_data(self): + """ + Return exported information for all trial jobs. + + Returns + ---------- + list + List of `nnicli.TrialResult`. + """ + _check_endpoint(self._endpoint) + trial_results = _nni_rest_get(self._endpoint, EXPORT_DATA_PATH) + return [TrialResult(e) for e in trial_results] + + def get_experiment_profile(self): + """ + Return experiment profile as a dict. + + Returns + ---------- + dict + The profile of the experiment. + """ + _check_endpoint(self._endpoint) + return _nni_rest_get(self._endpoint, EXPERIMENT_PATH) diff --git a/utils/third_party/nni/parameter_expressions.py b/utils/third_party/nni/parameter_expressions.py new file mode 100644 index 0000000000000000000000000000000000000000..adff923f7ea1cb4debe114c22376522adb8d8a6d --- /dev/null +++ b/utils/third_party/nni/parameter_expressions.py @@ -0,0 +1,108 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +''' +parameter_expression.py +''' + +import numpy as np + + +def choice(options, random_state): + ''' + options: 1-D array-like or int + random_state: an object of numpy.random.RandomState + ''' + return random_state.choice(options) + + +def randint(lower, upper, random_state): + ''' + Generate a random integer from `lower` (inclusive) to `upper` (exclusive). + lower: an int that represent an lower bound + upper: an int that represent an upper bound + random_state: an object of numpy.random.RandomState + ''' + return random_state.randint(lower, upper) + + +def uniform(low, high, random_state): + ''' + low: an float that represent an lower bound + high: an float that represent an upper bound + random_state: an object of numpy.random.RandomState + ''' + assert high >= low, 'Upper bound must be larger than lower bound' + return random_state.uniform(low, high) + + +def quniform(low, high, q, random_state): + ''' + low: an float that represent an lower bound + high: an float that represent an upper bound + q: sample step + random_state: an object of numpy.random.RandomState + ''' + return np.clip(np.round(uniform(low, high, random_state) / q) * q, low, high) + + +def loguniform(low, high, random_state): + ''' + low: an float that represent an lower bound + high: an float that represent an upper bound + random_state: an object of numpy.random.RandomState + ''' + assert low > 0, 'Lower bound must be positive' + return np.exp(uniform(np.log(low), np.log(high), random_state)) + + +def qloguniform(low, high, q, random_state): + ''' + low: an float that represent an lower bound + high: an float that represent an upper bound + q: sample step + random_state: an object of numpy.random.RandomState + ''' + return np.clip(np.round(loguniform(low, high, random_state) / q) * q, low, high) + + +def normal(mu, sigma, random_state): + ''' + The probability density function of the normal distribution, + first derived by De Moivre and 200 years later by both Gauss and Laplace independently. + mu: float or array_like of floats + Mean (“centre”) of the distribution. + sigma: float or array_like of floats + Standard deviation (spread or “width”) of the distribution. + random_state: an object of numpy.random.RandomState + ''' + return random_state.normal(mu, sigma) + + +def qnormal(mu, sigma, q, random_state): + ''' + mu: float or array_like of floats + sigma: float or array_like of floats + q: sample step + random_state: an object of numpy.random.RandomState + ''' + return np.round(normal(mu, sigma, random_state) / q) * q + + +def lognormal(mu, sigma, random_state): + ''' + mu: float or array_like of floats + sigma: float or array_like of floats + random_state: an object of numpy.random.RandomState + ''' + return np.exp(normal(mu, sigma, random_state)) + + +def qlognormal(mu, sigma, q, random_state): + ''' + mu: float or array_like of floats + sigma: float or array_like of floats + q: sample step + random_state: an object of numpy.random.RandomState + ''' + return np.round(lognormal(mu, sigma, random_state) / q) * q diff --git a/utils/third_party/nni/recoverable.py b/utils/third_party/nni/recoverable.py new file mode 100644 index 0000000000000000000000000000000000000000..70f11e634dc74886c0d379ca7fc86202654bf6ad --- /dev/null +++ b/utils/third_party/nni/recoverable.py @@ -0,0 +1,18 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os + +class Recoverable: + + def load_checkpoint(self): + pass + + def save_checkpoint(self): + pass + + def get_checkpoint_path(self): + ckp_path = os.getenv('NNI_CHECKPOINT_DIRECTORY') + if ckp_path is not None and os.path.isdir(ckp_path): + return ckp_path + return None diff --git a/utils/third_party/nni/runtime/__init__.py b/utils/third_party/nni/runtime/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/utils/third_party/nni/runtime/common.py b/utils/third_party/nni/runtime/common.py new file mode 100644 index 0000000000000000000000000000000000000000..ec5ef10162b0936a402c0f1468f314e3eccebfdc --- /dev/null +++ b/utils/third_party/nni/runtime/common.py @@ -0,0 +1,108 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from datetime import datetime +from io import TextIOBase +import logging +import os +import sys +import time + +log_level_map = { + 'fatal': logging.FATAL, + 'error': logging.ERROR, + 'warning': logging.WARNING, + 'info': logging.INFO, + 'debug': logging.DEBUG +} + +_time_format = '%m/%d/%Y, %I:%M:%S %p' + +# FIXME +# This hotfix the bug that querying installed tuners with `package_utils` will activate dispatcher logger. +# This behavior depends on underlying implementation of `nnictl` and is likely to break in future. +_logger_initialized = False + +class _LoggerFileWrapper(TextIOBase): + def __init__(self, logger_file): + self.file = logger_file + + def write(self, s): + if s != '\n': + cur_time = datetime.now().strftime(_time_format) + self.file.write('[{}] PRINT '.format(cur_time) + s + '\n') + self.file.flush() + return len(s) + +def init_logger(logger_file_path, log_level_name='info'): + """Initialize root logger. + This will redirect anything from logging.getLogger() as well as stdout to specified file. + logger_file_path: path of logger file (path-like object). + """ + global _logger_initialized + if _logger_initialized: + return + _logger_initialized = True + + if os.environ.get('NNI_PLATFORM') == 'unittest': + return # fixme: launching logic needs refactor + + log_level = log_level_map.get(log_level_name, logging.INFO) + logger_file = open(logger_file_path, 'w') + fmt = '[%(asctime)s] %(levelname)s (%(name)s/%(threadName)s) %(message)s' + logging.Formatter.converter = time.localtime + formatter = logging.Formatter(fmt, _time_format) + handler = logging.StreamHandler(logger_file) + handler.setFormatter(formatter) + + root_logger = logging.getLogger() + root_logger.addHandler(handler) + root_logger.setLevel(log_level) + + # these modules are too verbose + logging.getLogger('matplotlib').setLevel(log_level) + + sys.stdout = _LoggerFileWrapper(logger_file) + +def init_standalone_logger(): + """ + Initialize root logger for standalone mode. + This will set NNI's log level to INFO and print its log to stdout. + """ + global _logger_initialized + if _logger_initialized: + return + _logger_initialized = True + + fmt = '[%(asctime)s] %(levelname)s (%(name)s) %(message)s' + formatter = logging.Formatter(fmt, _time_format) + handler = logging.StreamHandler(sys.stdout) + handler.setFormatter(formatter) + nni_logger = logging.getLogger('nni') + nni_logger.addHandler(handler) + nni_logger.setLevel(logging.INFO) + nni_logger.propagate = False + + # Following line does not affect NNI loggers, but without this user's logger won't be able to + # print log even it's level is set to INFO, so we do it for user's convenience. + # If this causes any issue in future, remove it and use `logging.info` instead of + # `logging.getLogger('xxx')` in all examples. + logging.basicConfig() + + +_multi_thread = False +_multi_phase = False + +def enable_multi_thread(): + global _multi_thread + _multi_thread = True + +def multi_thread_enabled(): + return _multi_thread + +def enable_multi_phase(): + global _multi_phase + _multi_phase = True + +def multi_phase_enabled(): + return _multi_phase diff --git a/utils/third_party/nni/runtime/env_vars.py b/utils/third_party/nni/runtime/env_vars.py new file mode 100644 index 0000000000000000000000000000000000000000..5227956012f2f8b7b888f859a0faaa324f2800e4 --- /dev/null +++ b/utils/third_party/nni/runtime/env_vars.py @@ -0,0 +1,33 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +from collections import namedtuple + + +_trial_env_var_names = [ + 'NNI_PLATFORM', + 'NNI_EXP_ID', + 'NNI_TRIAL_JOB_ID', + 'NNI_SYS_DIR', + 'NNI_OUTPUT_DIR', + 'NNI_TRIAL_SEQ_ID', + 'MULTI_PHASE' +] + +_dispatcher_env_var_names = [ + 'SDK_PROCESS', + 'NNI_MODE', + 'NNI_CHECKPOINT_DIRECTORY', + 'NNI_LOG_DIRECTORY', + 'NNI_LOG_LEVEL', + 'NNI_INCLUDE_INTERMEDIATE_RESULTS' +] + +def _load_env_vars(env_var_names): + env_var_dict = {k: os.environ.get(k) for k in env_var_names} + return namedtuple('EnvVars', env_var_names)(**env_var_dict) + +trial_env_vars = _load_env_vars(_trial_env_var_names) + +dispatcher_env_vars = _load_env_vars(_dispatcher_env_var_names) diff --git a/utils/third_party/nni/runtime/msg_dispatcher.py b/utils/third_party/nni/runtime/msg_dispatcher.py new file mode 100644 index 0000000000000000000000000000000000000000..4f24294fc750f6f5a341d0a28388c7dcef3c6d1b --- /dev/null +++ b/utils/third_party/nni/runtime/msg_dispatcher.py @@ -0,0 +1,239 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +from collections import defaultdict +import json_tricks + +from nni import NoMoreTrialError +from .protocol import CommandType, send +from .msg_dispatcher_base import MsgDispatcherBase +from nni.assessor import AssessResult +from .common import multi_thread_enabled, multi_phase_enabled +from .env_vars import dispatcher_env_vars +from ..utils import MetricType, to_json + +_logger = logging.getLogger(__name__) + +# Assessor global variables +_trial_history = defaultdict(dict) +'''key: trial job ID; value: intermediate results, mapping from sequence number to data''' + +_ended_trials = set() +'''trial_job_id of all ended trials. +We need this because NNI manager may send metrics after reporting a trial ended. +TODO: move this logic to NNI manager +''' + + +def _sort_history(history): + ret = [] + for i, _ in enumerate(history): + if i in history: + ret.append(history[i]) + else: + break + return ret + + +# Tuner global variables +_next_parameter_id = 0 +_trial_params = {} +'''key: trial job ID; value: parameters''' +_customized_parameter_ids = set() + + +def _create_parameter_id(): + global _next_parameter_id + _next_parameter_id += 1 + return _next_parameter_id - 1 + + +def _pack_parameter(parameter_id, params, customized=False, trial_job_id=None, parameter_index=None): + _trial_params[parameter_id] = params + ret = { + 'parameter_id': parameter_id, + 'parameter_source': 'customized' if customized else 'algorithm', + 'parameters': params + } + if trial_job_id is not None: + ret['trial_job_id'] = trial_job_id + if parameter_index is not None: + ret['parameter_index'] = parameter_index + else: + ret['parameter_index'] = 0 + return to_json(ret) + + +class MsgDispatcher(MsgDispatcherBase): + def __init__(self, tuner, assessor=None): + super(MsgDispatcher, self).__init__() + self.tuner = tuner + self.assessor = assessor + if assessor is None: + _logger.debug('Assessor is not configured') + + def load_checkpoint(self): + self.tuner.load_checkpoint() + if self.assessor is not None: + self.assessor.load_checkpoint() + + def save_checkpoint(self): + self.tuner.save_checkpoint() + if self.assessor is not None: + self.assessor.save_checkpoint() + + def handle_initialize(self, data): + """Data is search space + """ + self.tuner.update_search_space(data) + send(CommandType.Initialized, '') + + def send_trial_callback(self, id_, params): + """For tuner to issue trial config when the config is generated + """ + send(CommandType.NewTrialJob, _pack_parameter(id_, params)) + + def handle_request_trial_jobs(self, data): + # data: number or trial jobs + ids = [_create_parameter_id() for _ in range(data)] + _logger.debug("requesting for generating params of %s", ids) + params_list = self.tuner.generate_multiple_parameters(ids, st_callback=self.send_trial_callback) + + for i, _ in enumerate(params_list): + send(CommandType.NewTrialJob, _pack_parameter(ids[i], params_list[i])) + # when parameters is None. + if len(params_list) < len(ids): + send(CommandType.NoMoreTrialJobs, _pack_parameter(ids[0], '')) + + def handle_update_search_space(self, data): + self.tuner.update_search_space(data) + + def handle_import_data(self, data): + """Import additional data for tuning + data: a list of dictionaries, each of which has at least two keys, 'parameter' and 'value' + """ + for entry in data: + entry['value'] = entry['value'] if type(entry['value']) is str else json_tricks.dumps(entry['value']) + entry['value'] = json_tricks.loads(entry['value']) + self.tuner.import_data(data) + + def handle_add_customized_trial(self, data): + # data: parameters + id_ = _create_parameter_id() + _customized_parameter_ids.add(id_) + + def handle_report_metric_data(self, data): + """ + data: a dict received from nni_manager, which contains: + - 'parameter_id': id of the trial + - 'value': metric value reported by nni.report_final_result() + - 'type': report type, support {'FINAL', 'PERIODICAL'} + """ + # metrics value is dumped as json string in trial, so we need to decode it here + if 'value' in data: + data['value'] = json_tricks.loads(data['value']) + if data['type'] == MetricType.FINAL: + self._handle_final_metric_data(data) + elif data['type'] == MetricType.PERIODICAL: + if self.assessor is not None: + self._handle_intermediate_metric_data(data) + elif data['type'] == MetricType.REQUEST_PARAMETER: + assert multi_phase_enabled() + assert data['trial_job_id'] is not None + assert data['parameter_index'] is not None + param_id = _create_parameter_id() + try: + param = self.tuner.generate_parameters(param_id, trial_job_id=data['trial_job_id']) + except NoMoreTrialError: + param = None + send(CommandType.SendTrialJobParameter, _pack_parameter(param_id, param, trial_job_id=data['trial_job_id'], + parameter_index=data['parameter_index'])) + else: + raise ValueError('Data type not supported: {}'.format(data['type'])) + + def handle_trial_end(self, data): + """ + data: it has three keys: trial_job_id, event, hyper_params + - trial_job_id: the id generated by training service + - event: the job's state + - hyper_params: the hyperparameters generated and returned by tuner + """ + trial_job_id = data['trial_job_id'] + _ended_trials.add(trial_job_id) + if trial_job_id in _trial_history: + _trial_history.pop(trial_job_id) + if self.assessor is not None: + self.assessor.trial_end(trial_job_id, data['event'] == 'SUCCEEDED') + if self.tuner is not None: + self.tuner.trial_end(json_tricks.loads(data['hyper_params'])['parameter_id'], data['event'] == 'SUCCEEDED') + + def _handle_final_metric_data(self, data): + """Call tuner to process final results + """ + id_ = data['parameter_id'] + value = data['value'] + if id_ is None or id_ in _customized_parameter_ids: + if not hasattr(self.tuner, '_accept_customized'): + self.tuner._accept_customized = False + if not self.tuner._accept_customized: + _logger.info('Customized trial job %s ignored by tuner', id_) + return + customized = True + else: + customized = False + self.tuner.receive_trial_result(id_, _trial_params[id_], value, customized=customized, + trial_job_id=data.get('trial_job_id')) + + def _handle_intermediate_metric_data(self, data): + """Call assessor to process intermediate results + """ + if data['type'] != MetricType.PERIODICAL: + return + if self.assessor is None: + return + + trial_job_id = data['trial_job_id'] + if trial_job_id in _ended_trials: + return + + history = _trial_history[trial_job_id] + history[data['sequence']] = data['value'] + ordered_history = _sort_history(history) + if len(ordered_history) < data['sequence']: # no user-visible update since last time + return + + try: + result = self.assessor.assess_trial(trial_job_id, ordered_history) + except Exception as e: + _logger.error('Assessor error') + _logger.exception(e) + + if isinstance(result, bool): + result = AssessResult.Good if result else AssessResult.Bad + elif not isinstance(result, AssessResult): + msg = 'Result of Assessor.assess_trial must be an object of AssessResult, not %s' + raise RuntimeError(msg % type(result)) + + if result is AssessResult.Bad: + _logger.debug('BAD, kill %s', trial_job_id) + send(CommandType.KillTrialJob, json_tricks.dumps(trial_job_id)) + # notify tuner + _logger.debug('env var: NNI_INCLUDE_INTERMEDIATE_RESULTS: [%s]', + dispatcher_env_vars.NNI_INCLUDE_INTERMEDIATE_RESULTS) + if dispatcher_env_vars.NNI_INCLUDE_INTERMEDIATE_RESULTS == 'true': + self._earlystop_notify_tuner(data) + else: + _logger.debug('GOOD') + + def _earlystop_notify_tuner(self, data): + """Send last intermediate result as final result to tuner in case the + trial is early stopped. + """ + _logger.debug('Early stop notify tuner data: [%s]', data) + data['type'] = MetricType.FINAL + if multi_thread_enabled(): + self._handle_final_metric_data(data) + else: + data['value'] = to_json(data['value']) + self.enqueue_command(CommandType.ReportMetricData, data) diff --git a/utils/third_party/nni/runtime/msg_dispatcher_base.py b/utils/third_party/nni/runtime/msg_dispatcher_base.py new file mode 100644 index 0000000000000000000000000000000000000000..66af52df2853267a36fa2e8a52525cac001d9872 --- /dev/null +++ b/utils/third_party/nni/runtime/msg_dispatcher_base.py @@ -0,0 +1,247 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import threading +import logging +from multiprocessing.dummy import Pool as ThreadPool +from queue import Queue, Empty +import json_tricks + +from .common import multi_thread_enabled +from .env_vars import dispatcher_env_vars +from ..utils import init_dispatcher_logger +from ..recoverable import Recoverable +from .protocol import CommandType, receive + +init_dispatcher_logger() + +_logger = logging.getLogger(__name__) + +QUEUE_LEN_WARNING_MARK = 20 +_worker_fast_exit_on_terminate = True + + +class MsgDispatcherBase(Recoverable): + """This is where tuners and assessors are not defined yet. + Inherits this class to make your own advisor. + """ + + def __init__(self): + if multi_thread_enabled(): + self.pool = ThreadPool() + self.thread_results = [] + else: + self.stopping = False + self.default_command_queue = Queue() + self.assessor_command_queue = Queue() + self.default_worker = threading.Thread(target=self.command_queue_worker, args=(self.default_command_queue,)) + self.assessor_worker = threading.Thread(target=self.command_queue_worker, + args=(self.assessor_command_queue,)) + self.default_worker.start() + self.assessor_worker.start() + self.worker_exceptions = [] + + def run(self): + """Run the tuner. + This function will never return unless raise. + """ + _logger.info('Start dispatcher') + if dispatcher_env_vars.NNI_MODE == 'resume': + self.load_checkpoint() + + while True: + command, data = receive() + if data: + data = json_tricks.loads(data) + + if command is None or command is CommandType.Terminate: + break + if multi_thread_enabled(): + result = self.pool.map_async(self.process_command_thread, [(command, data)]) + self.thread_results.append(result) + if any([thread_result.ready() and not thread_result.successful() for thread_result in + self.thread_results]): + _logger.debug('Caught thread exception') + break + else: + self.enqueue_command(command, data) + if self.worker_exceptions: + break + + _logger.info('Dispatcher exiting...') + self.stopping = True + if multi_thread_enabled(): + self.pool.close() + self.pool.join() + else: + self.default_worker.join() + self.assessor_worker.join() + + _logger.info('Terminated by NNI manager') + + def command_queue_worker(self, command_queue): + """Process commands in command queues. + """ + while True: + try: + # set timeout to ensure self.stopping is checked periodically + command, data = command_queue.get(timeout=3) + try: + self.process_command(command, data) + except Exception as e: + _logger.exception(e) + self.worker_exceptions.append(e) + break + except Empty: + pass + if self.stopping and (_worker_fast_exit_on_terminate or command_queue.empty()): + break + + def enqueue_command(self, command, data): + """Enqueue command into command queues + """ + if command == CommandType.TrialEnd or ( + command == CommandType.ReportMetricData and data['type'] == 'PERIODICAL'): + self.assessor_command_queue.put((command, data)) + else: + self.default_command_queue.put((command, data)) + + qsize = self.default_command_queue.qsize() + if qsize >= QUEUE_LEN_WARNING_MARK: + _logger.warning('default queue length: %d', qsize) + + qsize = self.assessor_command_queue.qsize() + if qsize >= QUEUE_LEN_WARNING_MARK: + _logger.warning('assessor queue length: %d', qsize) + + def process_command_thread(self, request): + """Worker thread to process a command. + """ + command, data = request + if multi_thread_enabled(): + try: + self.process_command(command, data) + except Exception as e: + _logger.exception(str(e)) + raise + else: + pass + + def process_command(self, command, data): + _logger.debug('process_command: command: [%s], data: [%s]', command, data) + + command_handlers = { + # Tuner commands: + CommandType.Initialize: self.handle_initialize, + CommandType.RequestTrialJobs: self.handle_request_trial_jobs, + CommandType.UpdateSearchSpace: self.handle_update_search_space, + CommandType.ImportData: self.handle_import_data, + CommandType.AddCustomizedTrialJob: self.handle_add_customized_trial, + + # Tuner/Assessor commands: + CommandType.ReportMetricData: self.handle_report_metric_data, + + CommandType.TrialEnd: self.handle_trial_end, + CommandType.Ping: self.handle_ping, + } + if command not in command_handlers: + raise AssertionError('Unsupported command: {}'.format(command)) + command_handlers[command](data) + + def handle_ping(self, data): + pass + + def handle_initialize(self, data): + """Initialize search space and tuner, if any + This method is meant to be called only once for each experiment, after calling this method, + dispatcher should `send(CommandType.Initialized, '')`, to set the status of the experiment to be "INITIALIZED". + Parameters + ---------- + data: dict + search space + """ + raise NotImplementedError('handle_initialize not implemented') + + def handle_request_trial_jobs(self, data): + """The message dispatcher is demanded to generate ``data`` trial jobs. + These trial jobs should be sent via ``send(CommandType.NewTrialJob, json_tricks.dumps(parameter))``, + where ``parameter`` will be received by NNI Manager and eventually accessible to trial jobs as "next parameter". + Semantically, message dispatcher should do this ``send`` exactly ``data`` times. + + The JSON sent by this method should follow the format of + + :: + + { + "parameter_id": 42 + "parameters": { + // this will be received by trial + }, + "parameter_source": "algorithm" // optional + } + + Parameters + ---------- + data: int + number of trial jobs + """ + raise NotImplementedError('handle_request_trial_jobs not implemented') + + def handle_update_search_space(self, data): + """This method will be called when search space is updated. + It's recommended to call this method in `handle_initialize` to initialize search space. + *No need to* notify NNI Manager when this update is done. + Parameters + ---------- + data: dict + search space + """ + raise NotImplementedError('handle_update_search_space not implemented') + + def handle_import_data(self, data): + """Import previous data when experiment is resumed. + Parameters + ---------- + data: list + a list of dictionaries, each of which has at least two keys, 'parameter' and 'value' + """ + raise NotImplementedError('handle_import_data not implemented') + + def handle_add_customized_trial(self, data): + """Experimental API. Not recommended for usage. + """ + raise NotImplementedError('handle_add_customized_trial not implemented') + + def handle_report_metric_data(self, data): + """Called when metric data is reported or new parameters are requested (for multiphase). + When new parameters are requested, this method should send a new parameter. + + Parameters + ---------- + data: dict + a dict which contains 'parameter_id', 'value', 'trial_job_id', 'type', 'sequence'. + type: can be `MetricType.REQUEST_PARAMETER`, `MetricType.FINAL` or `MetricType.PERIODICAL`. + `REQUEST_PARAMETER` is used to request new parameters for multiphase trial job. In this case, + the dict will contain additional keys: `trial_job_id`, `parameter_index`. Refer to `msg_dispatcher.py` + as an example. + + Raises + ------ + ValueError + Data type is not supported + """ + raise NotImplementedError('handle_report_metric_data not implemented') + + def handle_trial_end(self, data): + """Called when the state of one of the trials is changed + + Parameters + ---------- + data: dict + a dict with keys: trial_job_id, event, hyper_params. + trial_job_id: the id generated by training service. + event: the job’s state. + hyper_params: the string that is sent by message dispatcher during the creation of trials. + + """ + raise NotImplementedError('handle_trial_end not implemented') diff --git a/utils/third_party/nni/runtime/platform/__init__.py b/utils/third_party/nni/runtime/platform/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..84f04a9862a91931ccf7cd18fee93268af86bfbf --- /dev/null +++ b/utils/third_party/nni/runtime/platform/__init__.py @@ -0,0 +1,15 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from ..env_vars import trial_env_vars, dispatcher_env_vars + +assert dispatcher_env_vars.SDK_PROCESS != 'dispatcher' + +if trial_env_vars.NNI_PLATFORM is None: + from .standalone import * +elif trial_env_vars.NNI_PLATFORM == 'unittest': + from .test import * +elif trial_env_vars.NNI_PLATFORM in ('local', 'remote', 'pai', 'kubeflow', 'frameworkcontroller', 'paiYarn', 'dlts', 'aml'): + from .local import * +else: + raise RuntimeError('Unknown platform %s' % trial_env_vars.NNI_PLATFORM) diff --git a/utils/third_party/nni/runtime/platform/local.py b/utils/third_party/nni/runtime/platform/local.py new file mode 100644 index 0000000000000000000000000000000000000000..5d8124d3ff11e8c6b136132ca6d65d8dd2d73e35 --- /dev/null +++ b/utils/third_party/nni/runtime/platform/local.py @@ -0,0 +1,86 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +import sys +import json +import time +import subprocess + +from ..common import init_logger +from ..env_vars import trial_env_vars +from nni.utils import to_json + +_sysdir = trial_env_vars.NNI_SYS_DIR +if not os.path.exists(os.path.join(_sysdir, '.nni')): + os.makedirs(os.path.join(_sysdir, '.nni')) +_metric_file = open(os.path.join(_sysdir, '.nni', 'metrics'), 'wb') + +_outputdir = trial_env_vars.NNI_OUTPUT_DIR +if not os.path.exists(_outputdir): + os.makedirs(_outputdir) + +_nni_platform = trial_env_vars.NNI_PLATFORM +if _nni_platform == 'local': + _log_file_path = os.path.join(_outputdir, 'trial.log') + init_logger(_log_file_path) + +_multiphase = trial_env_vars.MULTI_PHASE + +_param_index = 0 + +def request_next_parameter(): + metric = to_json({ + 'trial_job_id': trial_env_vars.NNI_TRIAL_JOB_ID, + 'type': 'REQUEST_PARAMETER', + 'sequence': 0, + 'parameter_index': _param_index + }) + send_metric(metric) + +def get_next_parameter(): + global _param_index + params_file_name = '' + if _multiphase in ('true', 'True'): + params_file_name = ('parameter_{}.cfg'.format(_param_index), 'parameter.cfg')[_param_index == 0] + else: + if _param_index > 0: + return None + elif _param_index == 0: + params_file_name = 'parameter.cfg' + else: + raise AssertionError('_param_index value ({}) should >=0'.format(_param_index)) + + params_filepath = os.path.join(_sysdir, params_file_name) + if not os.path.isfile(params_filepath): + request_next_parameter() + while not (os.path.isfile(params_filepath) and os.path.getsize(params_filepath) > 0): + time.sleep(3) + params_file = open(params_filepath, 'r') + params = json.load(params_file) + _param_index += 1 + return params + +def send_metric(string): + if _nni_platform != 'local': + assert len(string) < 1000000, 'Metric too long' + print("NNISDK_MEb'%s'" % (string), flush=True) + else: + data = (string + '\n').encode('utf8') + assert len(data) < 1000000, 'Metric too long' + _metric_file.write(b'ME%06d%b' % (len(data), data)) + _metric_file.flush() + if sys.platform == "win32": + file = open(_metric_file.name) + file.close() + else: + subprocess.run(['touch', _metric_file.name], check=True) + +def get_experiment_id(): + return trial_env_vars.NNI_EXP_ID + +def get_trial_id(): + return trial_env_vars.NNI_TRIAL_JOB_ID + +def get_sequence_id(): + return int(trial_env_vars.NNI_TRIAL_SEQ_ID) diff --git a/utils/third_party/nni/runtime/platform/standalone.py b/utils/third_party/nni/runtime/platform/standalone.py new file mode 100644 index 0000000000000000000000000000000000000000..27c2e94db20c309a411619b7484f8c238d9b29a6 --- /dev/null +++ b/utils/third_party/nni/runtime/platform/standalone.py @@ -0,0 +1,44 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +import json_tricks + +from ..common import init_standalone_logger + +__all__ = [ + 'get_next_parameter', + 'get_experiment_id', + 'get_trial_id', + 'get_sequence_id', + 'send_metric', +] + +init_standalone_logger() +_logger = logging.getLogger('nni') + + +def get_next_parameter(): + _logger.warning('Requesting parameter without NNI framework, returning empty dict') + return { + 'parameter_id': None, + 'parameters': {} + } + +def get_experiment_id(): + return 'STANDALONE' + +def get_trial_id(): + return 'STANDALONE' + +def get_sequence_id(): + return 0 + +def send_metric(string): + metric = json_tricks.loads(string) + if metric['type'] == 'FINAL': + _logger.info('Final result: %s', metric['value']) + elif metric['type'] == 'PERIODICAL': + _logger.info('Intermediate result: %s (Index %s)', metric['value'], metric['sequence']) + else: + _logger.error('Unexpected metric: %s', string) diff --git a/utils/third_party/nni/runtime/platform/test.py b/utils/third_party/nni/runtime/platform/test.py new file mode 100644 index 0000000000000000000000000000000000000000..9bc9651eb0a953420b00c50408248939f2e5ecb6 --- /dev/null +++ b/utils/third_party/nni/runtime/platform/test.py @@ -0,0 +1,39 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +# pylint: skip-file + +import copy +import json_tricks + + +_params = None +_last_metric = None + + +def get_next_parameter(): + return _params + +def get_experiment_id(): + return 'fakeidex' + +def get_trial_id(): + return 'fakeidtr' + +def get_sequence_id(): + return 0 + +def send_metric(string): + global _last_metric + _last_metric = string + + +def init_params(params): + global _params + _params = copy.deepcopy(params) + +def get_last_metric(): + metrics = json_tricks.loads(_last_metric) + metrics['value'] = json_tricks.loads(metrics['value']) + + return metrics diff --git a/utils/third_party/nni/runtime/protocol.py b/utils/third_party/nni/runtime/protocol.py new file mode 100644 index 0000000000000000000000000000000000000000..57ea7fbc0b5e99b7534808b4d4541185a23c4d8a --- /dev/null +++ b/utils/third_party/nni/runtime/protocol.py @@ -0,0 +1,71 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +import os +import threading +from enum import Enum + + +class CommandType(Enum): + # in + Initialize = b'IN' + RequestTrialJobs = b'GE' + ReportMetricData = b'ME' + UpdateSearchSpace = b'SS' + ImportData = b'FD' + AddCustomizedTrialJob = b'AD' + TrialEnd = b'EN' + Terminate = b'TE' + Ping = b'PI' + + # out + Initialized = b'ID' + NewTrialJob = b'TR' + SendTrialJobParameter = b'SP' + NoMoreTrialJobs = b'NO' + KillTrialJob = b'KI' + +_lock = threading.Lock() +try: + if os.environ.get('NNI_PLATFORM') != 'unittest': + _in_file = open(3, 'rb') + _out_file = open(4, 'wb') +except OSError: + _msg = 'IPC pipeline not exists, maybe you are importing tuner/assessor from trial code?' + logging.getLogger(__name__).warning(_msg) + + +def send(command, data): + """Send command to Training Service. + command: CommandType object. + data: string payload. + """ + global _lock + try: + _lock.acquire() + data = data.encode('utf8') + msg = b'%b%014d%b' % (command.value, len(data), data) + logging.getLogger(__name__).debug('Sending command, data: [%s]', msg) + _out_file.write(msg) + _out_file.flush() + finally: + _lock.release() + + +def receive(): + """Receive a command from Training Service. + Returns a tuple of command (CommandType) and payload (str) + """ + header = _in_file.read(16) + logging.getLogger(__name__).debug('Received command, header: [%s]', header) + if header is None or len(header) < 16: + # Pipe EOF encountered + logging.getLogger(__name__).debug('Pipe EOF encountered') + return None, None + length = int(header[2:]) + data = _in_file.read(length) + command = CommandType(header[:2]) + data = data.decode('utf8') + logging.getLogger(__name__).debug('Received command, data: [%s]', data) + return command, data diff --git a/utils/third_party/nni/smartparam.py b/utils/third_party/nni/smartparam.py new file mode 100644 index 0000000000000000000000000000000000000000..dde0ac2bd64e8b22e2948909b116c33aefda7561 --- /dev/null +++ b/utils/third_party/nni/smartparam.py @@ -0,0 +1,148 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import numpy as np + +from .runtime.env_vars import trial_env_vars +from . import trial +from . import parameter_expressions as param_exp +from .common.nas_utils import classic_mode, enas_mode, oneshot_mode, darts_mode + + +__all__ = [ + 'choice', + 'randint', + 'uniform', + 'quniform', + 'loguniform', + 'qloguniform', + 'normal', + 'qnormal', + 'lognormal', + 'qlognormal', + 'function_choice', + 'mutable_layer' +] + + +if trial_env_vars.NNI_PLATFORM is None: + def choice(*options, name=None): + return param_exp.choice(options, np.random.RandomState()) + + def randint(lower, upper, name=None): + return param_exp.randint(lower, upper, np.random.RandomState()) + + def uniform(low, high, name=None): + return param_exp.uniform(low, high, np.random.RandomState()) + + def quniform(low, high, q, name=None): + assert high > low, 'Upper bound must be larger than lower bound' + return param_exp.quniform(low, high, q, np.random.RandomState()) + + def loguniform(low, high, name=None): + assert low > 0, 'Lower bound must be positive' + return param_exp.loguniform(low, high, np.random.RandomState()) + + def qloguniform(low, high, q, name=None): + return param_exp.qloguniform(low, high, q, np.random.RandomState()) + + def normal(mu, sigma, name=None): + return param_exp.normal(mu, sigma, np.random.RandomState()) + + def qnormal(mu, sigma, q, name=None): + return param_exp.qnormal(mu, sigma, q, np.random.RandomState()) + + def lognormal(mu, sigma, name=None): + return param_exp.lognormal(mu, sigma, np.random.RandomState()) + + def qlognormal(mu, sigma, q, name=None): + return param_exp.qlognormal(mu, sigma, q, np.random.RandomState()) + + def function_choice(*funcs, name=None): + return param_exp.choice(funcs, np.random.RandomState())() + + def mutable_layer(): + raise RuntimeError('Cannot call nni.mutable_layer in this mode') + +else: + + def choice(options, name=None, key=None): + return options[_get_param(key)] + + def randint(lower, upper, name=None, key=None): + return _get_param(key) + + def uniform(low, high, name=None, key=None): + return _get_param(key) + + def quniform(low, high, q, name=None, key=None): + return _get_param(key) + + def loguniform(low, high, name=None, key=None): + return _get_param(key) + + def qloguniform(low, high, q, name=None, key=None): + return _get_param(key) + + def normal(mu, sigma, name=None, key=None): + return _get_param(key) + + def qnormal(mu, sigma, q, name=None, key=None): + return _get_param(key) + + def lognormal(mu, sigma, name=None, key=None): + return _get_param(key) + + def qlognormal(mu, sigma, q, name=None, key=None): + return _get_param(key) + + def function_choice(funcs, name=None, key=None): + return funcs[_get_param(key)]() + + def mutable_layer( + mutable_id, + mutable_layer_id, + funcs, + funcs_args, + fixed_inputs, + optional_inputs, + optional_input_size, + mode='classic_mode', + tf=None): + '''execute the chosen function and inputs. + Below is an example of chosen function and inputs: + { + "mutable_id": { + "mutable_layer_id": { + "chosen_layer": "pool", + "chosen_inputs": ["out1", "out3"] + } + } + } + Parameters: + --------------- + mutable_id: the name of this mutable_layer block (which could have multiple mutable layers) + mutable_layer_id: the name of a mutable layer in this block + funcs: dict of function calls + funcs_args: + fixed_inputs: + optional_inputs: dict of optional inputs + optional_input_size: number of candidate inputs to be chosen + tf: tensorflow module + ''' + args = (mutable_id, mutable_layer_id, funcs, funcs_args, fixed_inputs, optional_inputs, optional_input_size) + if mode == 'classic_mode': + return classic_mode(*args) + assert tf is not None, 'Internal Error: Tensorflow should not be None in modes other than classic_mode' + if mode == 'enas_mode': + return enas_mode(*args, tf) + if mode == 'oneshot_mode': + return oneshot_mode(*args, tf) + if mode == 'darts_mode': + return darts_mode(*args, tf) + raise RuntimeError('Unrecognized mode: %s' % mode) + + def _get_param(key): + if trial.get_current_parameter() is None: + trial.get_next_parameter() + return trial.get_current_parameter(key) diff --git a/utils/third_party/nni/tools/__init__.py b/utils/third_party/nni/tools/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/utils/third_party/nni/tools/annotation/.gitignore b/utils/third_party/nni/tools/annotation/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..36e264cf443988bf3101b2467e83b259bcd5a4ad --- /dev/null +++ b/utils/third_party/nni/tools/annotation/.gitignore @@ -0,0 +1 @@ +_generated diff --git a/utils/third_party/nni/tools/annotation/__init__.py b/utils/third_party/nni/tools/annotation/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d9ba405c8844591ec083921c90ba2a1d67d52cf8 --- /dev/null +++ b/utils/third_party/nni/tools/annotation/__init__.py @@ -0,0 +1,141 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +import sys +import shutil +import json + +from . import code_generator +from . import search_space_generator +from . import specific_code_generator + + +__all__ = ['generate_search_space', 'expand_annotations'] + +slash = '/' +if sys.platform == "win32": + slash = '\\' + +def generate_search_space(code_dir): + """Generate search space from Python source code. + Return a serializable search space object. + code_dir: directory path of source files (str) + """ + code_dir = str(code_dir) + search_space = {} + + if code_dir.endswith(slash): + code_dir = code_dir[:-1] + + for subdir, _, files in os.walk(code_dir): + # generate module name from path + if subdir == code_dir: + package = '' + else: + assert subdir.startswith(code_dir + slash), subdir + prefix_len = len(code_dir) + 1 + package = subdir[prefix_len:].replace(slash, '.') + '.' + + for file_name in files: + if file_name.endswith('.py'): + path = os.path.join(subdir, file_name) + module = package + file_name[:-3] + search_space.update(_generate_file_search_space(path, module)) + + return search_space + +def _generate_file_search_space(path, module): + with open(path) as src: + try: + search_space, code = search_space_generator.generate(module, src.read()) + except Exception as exc: # pylint: disable=broad-except + if exc.args: + raise RuntimeError(path + ' ' + '\n'.join(exc.args)) + else: + raise RuntimeError('Failed to generate search space for %s: %r' % (path, exc)) + with open(path, 'w') as dst: + dst.write(code) + return search_space + + +def expand_annotations(src_dir, dst_dir, exp_id='', trial_id='', nas_mode=None): + """Expand annotations in user code. + Return dst_dir if annotation detected; return src_dir if not. + src_dir: directory path of user code (str) + dst_dir: directory to place generated files (str) + nas_mode: the mode of NAS given that NAS interface is used + """ + src_dir, dst_dir = str(src_dir), str(dst_dir) + + if src_dir[-1] == slash: + src_dir = src_dir[:-1] + + if dst_dir[-1] == slash: + dst_dir = dst_dir[:-1] + + annotated = False + + for src_subdir, dirs, files in os.walk(src_dir): + assert src_subdir.startswith(src_dir) + dst_subdir = src_subdir.replace(src_dir, dst_dir, 1) + os.makedirs(dst_subdir, exist_ok=True) + + # generate module name from path + if src_subdir == src_dir: + package = '' + else: + assert src_subdir.startswith(src_dir + slash), src_subdir + prefix_len = len(src_dir) + 1 + package = src_subdir[prefix_len:].replace(slash, '.') + '.' + + for file_name in files: + src_path = os.path.join(src_subdir, file_name) + dst_path = os.path.join(dst_subdir, file_name) + if file_name.endswith('.py'): + if trial_id == '': + annotated |= _expand_file_annotations(src_path, dst_path, nas_mode) + else: + module = package + file_name[:-3] + annotated |= _generate_specific_file(src_path, dst_path, exp_id, trial_id, module) + else: + shutil.copyfile(src_path, dst_path) + + for dir_name in dirs: + os.makedirs(os.path.join(dst_subdir, dir_name), exist_ok=True) + + return dst_dir if annotated else src_dir + +def _expand_file_annotations(src_path, dst_path, nas_mode): + with open(src_path) as src, open(dst_path, 'w') as dst: + try: + annotated_code = code_generator.parse(src.read(), nas_mode) + if annotated_code is None: + shutil.copyfile(src_path, dst_path) + return False + dst.write(annotated_code) + return True + + except Exception as exc: # pylint: disable=broad-except + if exc.args: + raise RuntimeError(src_path + ' ' + '\n'.join(str(arg) for arg in exc.args)) + else: + raise RuntimeError('Failed to expand annotations for %s: %r' % (src_path, exc)) + +def _generate_specific_file(src_path, dst_path, exp_id, trial_id, module): + with open(src_path) as src, open(dst_path, 'w') as dst: + try: + with open(os.path.expanduser('~/nni-experiments/%s/trials/%s/parameter.cfg'%(exp_id, trial_id))) as fd: + para_cfg = json.load(fd) + annotated_code = specific_code_generator.parse(src.read(), para_cfg["parameters"], module) + if annotated_code is None: + shutil.copyfile(src_path, dst_path) + return False + dst.write(annotated_code) + return True + + except Exception as exc: # pylint: disable=broad-except + if exc.args: + raise RuntimeError(src_path + ' ' + '\n'.join(str(arg) for arg in exc.args)) + else: + raise RuntimeError('Failed to expand annotations for %s: %r' % (src_path, exc)) diff --git a/utils/third_party/nni/tools/annotation/code_generator.py b/utils/third_party/nni/tools/annotation/code_generator.py new file mode 100644 index 0000000000000000000000000000000000000000..c924e4195072eeff3e112b9ca5e01cb1df41c178 --- /dev/null +++ b/utils/third_party/nni/tools/annotation/code_generator.py @@ -0,0 +1,369 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import ast +import astor + +from .utils import ast_Num, ast_Str, lineno + +# pylint: disable=unidiomatic-typecheck + +def parse_annotation_mutable_layers(code, lineno, nas_mode): + """Parse the string of mutable layers in annotation. + Return a list of AST Expr nodes + code: annotation string (excluding '@') + nas_mode: the mode of NAS + """ + module = ast.parse(code) + assert type(module) is ast.Module, 'internal error #1' + assert len(module.body) == 1, 'Annotation mutable_layers contains more than one expression' + assert type(module.body[0]) is ast.Expr, 'Annotation is not expression' + call = module.body[0].value + nodes = [] + mutable_id = 'mutable_block_' + str(lineno) + mutable_layer_cnt = 0 + for arg in call.args: + fields = {'layer_choice': False, + 'fixed_inputs': False, + 'optional_inputs': False, + 'optional_input_size': False, + 'layer_output': False} + for k, value in zip(arg.keys, arg.values): + if k.id == 'layer_choice': + assert not fields['layer_choice'], 'Duplicated field: layer_choice' + assert type(value) is ast.List, 'Value of layer_choice should be a list' + call_funcs_keys = [] + call_funcs_values = [] + call_kwargs_values = [] + for call in value.elts: + assert type(call) is ast.Call, 'Element in layer_choice should be function call' + call_name = astor.to_source(call).strip() + call_funcs_keys.append(ast_Str(s=call_name)) + call_funcs_values.append(call.func) + assert not call.args, 'Number of args without keyword should be zero' + kw_args = [] + kw_values = [] + for kw in call.keywords: + kw_args.append(ast_Str(s=kw.arg)) + kw_values.append(kw.value) + call_kwargs_values.append(ast.Dict(keys=kw_args, values=kw_values)) + call_funcs = ast.Dict(keys=call_funcs_keys, values=call_funcs_values) + call_kwargs = ast.Dict(keys=call_funcs_keys, values=call_kwargs_values) + fields['layer_choice'] = True + elif k.id == 'fixed_inputs': + assert not fields['fixed_inputs'], 'Duplicated field: fixed_inputs' + assert type(value) is ast.List, 'Value of fixed_inputs should be a list' + fixed_inputs = value + fields['fixed_inputs'] = True + elif k.id == 'optional_inputs': + assert not fields['optional_inputs'], 'Duplicated field: optional_inputs' + assert type(value) is ast.List, 'Value of optional_inputs should be a list' + var_names = [ast_Str(s=astor.to_source(var).strip()) for var in value.elts] + optional_inputs = ast.Dict(keys=var_names, values=value.elts) + fields['optional_inputs'] = True + elif k.id == 'optional_input_size': + assert not fields['optional_input_size'], 'Duplicated field: optional_input_size' + assert type(value) is ast_Num or type(value) is ast.List, \ + 'Value of optional_input_size should be a number or list' + optional_input_size = value + fields['optional_input_size'] = True + elif k.id == 'layer_output': + assert not fields['layer_output'], 'Duplicated field: layer_output' + assert type(value) is ast.Name, 'Value of layer_output should be ast.Name type' + layer_output = value + fields['layer_output'] = True + else: + raise AssertionError('Unexpected field in mutable layer') + # make call for this mutable layer + assert fields['layer_choice'], 'layer_choice must exist' + assert fields['layer_output'], 'layer_output must exist' + mutable_layer_id = 'mutable_layer_' + str(mutable_layer_cnt) + mutable_layer_cnt += 1 + target_call_attr = ast.Attribute(value=ast.Name(id='nni', ctx=ast.Load()), attr='mutable_layer', ctx=ast.Load()) + target_call_args = [ast_Str(s=mutable_id), + ast_Str(s=mutable_layer_id), + call_funcs, + call_kwargs] + if fields['fixed_inputs']: + target_call_args.append(fixed_inputs) + else: + target_call_args.append(ast.List(elts=[])) + if fields['optional_inputs']: + target_call_args.append(optional_inputs) + assert fields['optional_input_size'], 'optional_input_size must exist when optional_inputs exists' + target_call_args.append(optional_input_size) + else: + target_call_args.append(ast.Dict(keys=[], values=[])) + target_call_args.append(ast_Num(n=0)) + target_call_args.append(ast_Str(s=nas_mode)) + if nas_mode in ['enas_mode', 'oneshot_mode', 'darts_mode']: + target_call_args.append(ast.Name(id='tensorflow')) + target_call = ast.Call(func=target_call_attr, args=target_call_args, keywords=[]) + node = ast.Assign(targets=[layer_output], value=target_call) + nodes.append(node) + return nodes + + +def parse_annotation(code): + """Parse an annotation string. + Return an AST Expr node. + code: annotation string (excluding '@') + """ + module = ast.parse(code) + assert type(module) is ast.Module, 'internal error #1' + assert len(module.body) == 1, 'Annotation contains more than one expression' + assert type(module.body[0]) is ast.Expr, 'Annotation is not expression' + return module.body[0] + + +def parse_annotation_function(code, func_name): + """Parse an annotation function. + Return the value of `name` keyword argument and the AST Call node. + func_name: expected function name + """ + expr = parse_annotation(code) + call = expr.value + assert type(call) is ast.Call, 'Annotation is not a function call' + + assert type(call.func) is ast.Attribute, 'Unexpected annotation function' + assert type(call.func.value) is ast.Name, 'Invalid annotation function name' + assert call.func.value.id == 'nni', 'Annotation is not a NNI function' + assert call.func.attr == func_name, 'internal error #2' + + assert len(call.keywords) == 1, 'Annotation function contains more than one keyword argument' + assert call.keywords[0].arg == 'name', 'Annotation keyword argument is not "name"' + name = call.keywords[0].value + + return name, call + + +def parse_nni_variable(code): + """Parse `nni.variable` expression. + Return the name argument and AST node of annotated expression. + code: annotation string + """ + name, call = parse_annotation_function(code, 'variable') + + assert len(call.args) == 1, 'nni.variable contains more than one arguments' + arg = call.args[0] + assert type(arg) is ast.Call, 'Value of nni.variable is not a function call' + assert type(arg.func) is ast.Attribute, 'nni.variable value is not a NNI function' + assert type(arg.func.value) is ast.Name, 'nni.variable value is not a NNI function' + assert arg.func.value.id == 'nni', 'nni.variable value is not a NNI function' + + name_str = astor.to_source(name).strip() + keyword_arg = ast.keyword(arg='name', value=ast_Str(s=name_str)) + arg.keywords.append(keyword_arg) + if arg.func.attr == 'choice': + convert_args_to_dict(arg) + + return name, arg + + +def parse_nni_function(code): + """Parse `nni.function_choice` expression. + Return the AST node of annotated expression and a list of dumped function call expressions. + code: annotation string + """ + name, call = parse_annotation_function(code, 'function_choice') + funcs = [ast.dump(func, False) for func in call.args] + convert_args_to_dict(call, with_lambda=True) + + name_str = astor.to_source(name).strip() + call.keywords[0].value = ast_Str(s=name_str) + + return call, funcs + + +def convert_args_to_dict(call, with_lambda=False): + """Convert all args to a dict such that every key and value in the dict is the same as the value of the arg. + Return the AST Call node with only one arg that is the dictionary + """ + keys, values = list(), list() + for arg in call.args: + if type(arg) in [ast_Str, ast_Num]: + arg_value = arg + else: + # if arg is not a string or a number, we use its source code as the key + arg_value = astor.to_source(arg).strip('\n"') + arg_value = ast_Str(str(arg_value)) + arg = make_lambda(arg) if with_lambda else arg + keys.append(arg_value) + values.append(arg) + del call.args[:] + call.args.append(ast.Dict(keys=keys, values=values)) + + return call + + +def make_lambda(call): + """Wrap an AST Call node to lambda expression node. + call: ast.Call node + """ + empty_args = ast.arguments(args=[], vararg=None, kwarg=None, defaults=[]) + return ast.Lambda(args=empty_args, body=call) + + +def test_variable_equal(node1, node2): + """Test whether two variables are the same.""" + if type(node1) is not type(node2): + return False + if isinstance(node1, ast.AST): + for k, v in vars(node1).items(): + if k in ('lineno', 'col_offset', 'ctx', 'end_lineno', 'end_col_offset'): + continue + if not test_variable_equal(v, getattr(node2, k)): + return False + return True + if isinstance(node1, list): + if len(node1) != len(node2): + return False + return all(test_variable_equal(n1, n2) for n1, n2 in zip(node1, node2)) + + return node1 == node2 + + +def replace_variable_node(node, annotation): + """Replace a node annotated by `nni.variable`. + node: the AST node to replace + annotation: annotation string + """ + assert type(node) is ast.Assign, 'nni.variable is not annotating assignment expression' + assert len(node.targets) == 1, 'Annotated assignment has more than one left-hand value' + name, expr = parse_nni_variable(annotation) + assert test_variable_equal(node.targets[0], name), 'Annotated variable has wrong name' + node.value = expr + return node + + +def replace_function_node(node, annotation): + """Replace a node annotated by `nni.function_choice`. + node: the AST node to replace + annotation: annotation string + """ + target, funcs = parse_nni_function(annotation) + FuncReplacer(funcs, target).visit(node) + return node + + +class FuncReplacer(ast.NodeTransformer): + """To replace target function call expressions in a node annotated by `nni.function_choice`""" + + def __init__(self, funcs, target): + """Constructor. + funcs: list of dumped function call expressions to replace + target: use this AST node to replace matching expressions + """ + self.funcs = set(funcs) + self.target = target + + def visit_Call(self, node): # pylint: disable=invalid-name + if ast.dump(node, False) in self.funcs: + return self.target + return node + + +class Transformer(ast.NodeTransformer): + """Transform original code to annotated code""" + + def __init__(self, nas_mode=None): + self.stack = [] + self.last_line = 0 + self.annotated = False + self.nas_mode = nas_mode + + def visit(self, node): + if isinstance(node, (ast.expr, ast.stmt)): + self.last_line = lineno(node) + + # do nothing for root + if not self.stack: + return self._visit_children(node) + + annotation = self.stack[-1] + + # this is a standalone string, may be an annotation + if type(node) is ast.Expr and type(node.value) is ast_Str: + # must not annotate an annotation string + assert annotation is None, 'Annotating an annotation' + return self._visit_string(node) + + if annotation is not None: # this expression is annotated + self.stack[-1] = None # so next expression is not + if annotation.startswith('nni.variable'): + return replace_variable_node(node, annotation) + if annotation.startswith('nni.function_choice'): + return replace_function_node(node, annotation) + + return self._visit_children(node) + + def _visit_string(self, node): + string = node.value.s + if string.startswith('@nni.'): + self.annotated = True + else: + return node # not an annotation, ignore it + + if string.startswith('@nni.training_update'): + expr = parse_annotation(string[1:]) + call_node = expr.value + call_node.args.insert(0, ast_Str(s=self.nas_mode)) + return expr + + if string.startswith('@nni.report_intermediate_result') \ + or string.startswith('@nni.report_final_result') \ + or string.startswith('@nni.get_next_parameter'): + return parse_annotation(string[1:]) # expand annotation string to code + + if string.startswith('@nni.mutable_layers'): + nodes = parse_annotation_mutable_layers(string[1:], lineno(node), self.nas_mode) + return nodes + + if string.startswith('@nni.variable') \ + or string.startswith('@nni.function_choice'): + self.stack[-1] = string[1:] # mark that the next expression is annotated + return None + + raise AssertionError('Unexpected annotation function') + + def _visit_children(self, node): + self.stack.append(None) + self.generic_visit(node) + annotation = self.stack.pop() + assert annotation is None, 'Annotation has no target' + return node + + +def parse(code, nas_mode=None): + """Annotate user code. + Return annotated code (str) if annotation detected; return None if not. + code: original user code (str), + nas_mode: the mode of NAS given that NAS interface is used + """ + try: + ast_tree = ast.parse(code) + except Exception: + raise RuntimeError('Bad Python code') + + transformer = Transformer(nas_mode) + try: + transformer.visit(ast_tree) + except AssertionError as exc: + raise RuntimeError('%d: %s' % (ast_tree.last_line, exc.args[0])) + + if not transformer.annotated: + return None + + last_future_import = -1 + import_nni = ast.Import(names=[ast.alias(name='nni', asname=None)]) + nodes = ast_tree.body + for i, _ in enumerate(nodes): + if type(nodes[i]) is ast.ImportFrom and nodes[i].module == '__future__': + last_future_import = i + nodes.insert(last_future_import + 1, import_nni) + # enas, oneshot and darts modes for tensorflow need tensorflow module, so we import it here + if nas_mode in ['enas_mode', 'oneshot_mode', 'darts_mode']: + import_tf = ast.Import(names=[ast.alias(name='tensorflow', asname=None)]) + nodes.insert(last_future_import + 1, import_tf) + + return astor.to_source(ast_tree) diff --git a/utils/third_party/nni/tools/annotation/search_space_generator.py b/utils/third_party/nni/tools/annotation/search_space_generator.py new file mode 100644 index 0000000000000000000000000000000000000000..a0a19f53dff601faa682f48b3f1ee574f21b9acf --- /dev/null +++ b/utils/third_party/nni/tools/annotation/search_space_generator.py @@ -0,0 +1,135 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import ast +import numbers + +import astor + +from .utils import ast_Num, ast_Str, lineno + +# pylint: disable=unidiomatic-typecheck + + +# list of functions related to search space generating +_ss_funcs = [ + 'choice', + 'randint', + 'uniform', + 'quniform', + 'loguniform', + 'qloguniform', + 'normal', + 'qnormal', + 'lognormal', + 'qlognormal', + 'function_choice', + 'mutable_layer' +] + + +class SearchSpaceGenerator(ast.NodeTransformer): + """Generate search space from smart parater APIs""" + + def __init__(self, module_name): + self.module_name = module_name + self.search_space = {} + self.last_line = 0 # last parsed line, useful for error reporting + + def generate_mutable_layer_search_space(self, args): + mutable_block = args[0].s + mutable_layer = args[1].s + key = self.module_name + '/' + mutable_block + args[0].s = key + if key not in self.search_space: + self.search_space[key] = {'_type': 'mutable_layer', '_value': {}} + self.search_space[key]['_value'][mutable_layer] = { + 'layer_choice': [k.s for k in args[2].keys], + 'optional_inputs': [k.s for k in args[5].keys], + 'optional_input_size': args[6].n if isinstance(args[6], ast_Num) else [args[6].elts[0].n, args[6].elts[1].n] + } + + def visit_Call(self, node): # pylint: disable=invalid-name + self.generic_visit(node) + + # ignore if the function is not 'nni.*' + if type(node.func) is not ast.Attribute: + return node + if type(node.func.value) is not ast.Name: + return node + if node.func.value.id != 'nni': + return node + + # ignore if its not a search space function (e.g. `report_final_result`) + func = node.func.attr + if func not in _ss_funcs: + return node + + self.last_line = lineno(node) + + if func == 'mutable_layer': + self.generate_mutable_layer_search_space(node.args) + return node + + if node.keywords: + # there is a `name` argument + assert len(node.keywords) == 1, 'Smart parameter has keyword argument other than "name"' + assert node.keywords[0].arg == 'name', 'Smart paramater\'s keyword argument is not "name"' + assert type(node.keywords[0].value) is ast_Str, 'Smart parameter\'s name must be string literal' + name = node.keywords[0].value.s + specified_name = True + else: + # generate the missing name automatically + name = '__line' + str(str(node.args[-1].lineno)) + specified_name = False + node.keywords = list() + + if func in ('choice', 'function_choice'): + # we will use keys in the dict as the choices, which is generated by code_generator according to the args given by user + assert len(node.args) == 1, 'Smart parameter has arguments other than dict' + # check if it is a number or a string and get its value accordingly + args = [key.n if type(key) is ast_Num else key.s for key in node.args[0].keys] + else: + # arguments of other functions must be literal number + assert all(isinstance(ast.literal_eval(astor.to_source(arg)), numbers.Real) for arg in node.args), \ + 'Smart parameter\'s arguments must be number literals' + args = [ast.literal_eval(astor.to_source(arg)) for arg in node.args] + + key = self.module_name + '/' + name + '/' + func + # store key in ast.Call + node.keywords.append(ast.keyword(arg='key', value=ast_Str(s=key))) + + if func == 'function_choice': + func = 'choice' + value = {'_type': func, '_value': args} + + if specified_name: + # multiple functions with same name must have identical arguments + old = self.search_space.get(key) + assert old is None or old == value, 'Different smart parameters have same name' + else: + # generated name must not duplicate + assert key not in self.search_space, 'Only one smart parameter is allowed in a line' + + self.search_space[key] = value + + return node + + +def generate(module_name, code): + """Generate search space. + Return a serializable search space object. + module_name: name of the module (str) + code: user code (str) + """ + try: + ast_tree = ast.parse(code) + except Exception: + raise RuntimeError('Bad Python code') + + visitor = SearchSpaceGenerator(module_name) + try: + visitor.visit(ast_tree) + except AssertionError as exc: + raise RuntimeError('%d: %s' % (visitor.last_line, exc.args[0])) + return visitor.search_space, astor.to_source(ast_tree) diff --git a/utils/third_party/nni/tools/annotation/specific_code_generator.py b/utils/third_party/nni/tools/annotation/specific_code_generator.py new file mode 100644 index 0000000000000000000000000000000000000000..5ddd362205dfa909d0077315ebe5dd9b8ade239f --- /dev/null +++ b/utils/third_party/nni/tools/annotation/specific_code_generator.py @@ -0,0 +1,354 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import ast +import astor +from nni.tools.nnictl.common_utils import print_warning + +from .utils import ast_Num, ast_Str, lineno + +# pylint: disable=unidiomatic-typecheck + +para_cfg = None +prefix_name = None + + +def parse_annotation_mutable_layers(code, lineno): + """Parse the string of mutable layers in annotation. + Return a list of AST Expr nodes + code: annotation string (excluding '@') + """ + module = ast.parse(code) + assert type(module) is ast.Module, 'internal error #1' + assert len(module.body) == 1, 'Annotation mutable_layers contains more than one expression' + assert type(module.body[0]) is ast.Expr, 'Annotation is not expression' + call = module.body[0].value + nodes = [] + mutable_id = prefix_name + '/mutable_block_' + str(lineno) + mutable_layer_cnt = 0 + for arg in call.args: + fields = {'layer_choice': False, + 'fixed_inputs': False, + 'optional_inputs': False, + 'optional_input_size': False, + 'layer_output': False} + mutable_layer_id = 'mutable_layer_' + str(mutable_layer_cnt) + mutable_layer_cnt += 1 + func_call = None + for k, value in zip(arg.keys, arg.values): + if k.id == 'layer_choice': + assert not fields['layer_choice'], 'Duplicated field: layer_choice' + assert type(value) is ast.List, 'Value of layer_choice should be a list' + for call in value.elts: + assert type(call) is ast.Call, 'Element in layer_choice should be function call' + call_name = astor.to_source(call).strip() + if call_name == para_cfg[mutable_id][mutable_layer_id]['chosen_layer']: + func_call = call + assert not call.args, 'Number of args without keyword should be zero' + break + fields['layer_choice'] = True + elif k.id == 'fixed_inputs': + assert not fields['fixed_inputs'], 'Duplicated field: fixed_inputs' + assert type(value) is ast.List, 'Value of fixed_inputs should be a list' + fixed_inputs = value + fields['fixed_inputs'] = True + elif k.id == 'optional_inputs': + assert not fields['optional_inputs'], 'Duplicated field: optional_inputs' + assert type(value) is ast.List, 'Value of optional_inputs should be a list' + var_names = [astor.to_source(var).strip() for var in value.elts] + chosen_inputs = para_cfg[mutable_id][mutable_layer_id]['chosen_inputs'] + elts = [] + for i in chosen_inputs: + index = var_names.index(i) + elts.append(value.elts[index]) + optional_inputs = ast.List(elts=elts) + fields['optional_inputs'] = True + elif k.id == 'optional_input_size': + pass + elif k.id == 'layer_output': + assert not fields['layer_output'], 'Duplicated field: layer_output' + assert type(value) is ast.Name, 'Value of layer_output should be ast.Name type' + layer_output = value + fields['layer_output'] = True + else: + raise AssertionError('Unexpected field in mutable layer') + # make call for this mutable layer + assert fields['layer_choice'], 'layer_choice must exist' + assert fields['layer_output'], 'layer_output must exist' + + if not fields['fixed_inputs']: + fixed_inputs = ast.List(elts=[]) + if not fields['optional_inputs']: + optional_inputs = ast.List(elts=[]) + inputs = ast.List(elts=[fixed_inputs, optional_inputs]) + + func_call.args.append(inputs) + node = ast.Assign(targets=[layer_output], value=func_call) + nodes.append(node) + return nodes + + +def parse_annotation(code): + """Parse an annotation string. + Return an AST Expr node. + code: annotation string (excluding '@') + """ + module = ast.parse(code) + assert type(module) is ast.Module, 'internal error #1' + assert len(module.body) == 1, 'Annotation contains more than one expression' + assert type(module.body[0]) is ast.Expr, 'Annotation is not expression' + return module.body[0] + + +def parse_annotation_function(code, func_name): + """Parse an annotation function. + Return the value of `name` keyword argument and the AST Call node. + func_name: expected function name + """ + expr = parse_annotation(code) + call = expr.value + assert type(call) is ast.Call, 'Annotation is not a function call' + + assert type(call.func) is ast.Attribute, 'Unexpected annotation function' + assert type(call.func.value) is ast.Name, 'Invalid annotation function name' + assert call.func.value.id == 'nni', 'Annotation is not a NNI function' + assert call.func.attr == func_name, 'internal error #2' + + assert len(call.keywords) == 1, 'Annotation function contains more than one keyword argument' + assert call.keywords[0].arg == 'name', 'Annotation keyword argument is not "name"' + name = call.keywords[0].value + + return name, call + + +def parse_nni_variable(code): + """Parse `nni.variable` expression. + Return the name argument and AST node of annotated expression. + code: annotation string + """ + name, call = parse_annotation_function(code, 'variable') + + assert len(call.args) == 1, 'nni.variable contains more than one arguments' + arg = call.args[0] + assert type(arg) is ast.Call, 'Value of nni.variable is not a function call' + assert type(arg.func) is ast.Attribute, 'nni.variable value is not a NNI function' + assert type(arg.func.value) is ast.Name, 'nni.variable value is not a NNI function' + assert arg.func.value.id == 'nni', 'nni.variable value is not a NNI function' + + name_str = astor.to_source(name).strip() + keyword_arg = ast.keyword(arg='name', value=ast_Str(s=name_str)) + arg.keywords.append(keyword_arg) + if arg.func.attr == 'choice': + convert_args_to_dict(arg) + + return name, arg + + +def parse_nni_function(code): + """Parse `nni.function_choice` expression. + Return the AST node of annotated expression and a list of dumped function call expressions. + code: annotation string + """ + name, call = parse_annotation_function(code, 'function_choice') + funcs = [ast.dump(func, False) for func in call.args] + convert_args_to_dict(call, with_lambda=True) + + name_str = astor.to_source(name).strip() + call.keywords[0].value = ast_Str(s=name_str) + + return call, funcs + + +def convert_args_to_dict(call, with_lambda=False): + """Convert all args to a dict such that every key and value in the dict is the same as the value of the arg. + Return the AST Call node with only one arg that is the dictionary + """ + keys, values = list(), list() + for arg in call.args: + if type(arg) in [ast_Str, ast_Num]: + arg_value = arg + else: + # if arg is not a string or a number, we use its source code as the key + arg_value = astor.to_source(arg).strip('\n"') + arg_value = ast_Str(str(arg_value)) + arg = make_lambda(arg) if with_lambda else arg + keys.append(arg_value) + values.append(arg) + del call.args[:] + call.args.append(ast.Dict(keys=keys, values=values)) + + return call + + +def make_lambda(call): + """Wrap an AST Call node to lambda expression node. + call: ast.Call node + """ + empty_args = ast.arguments(args=[], vararg=None, kwarg=None, defaults=[]) + return ast.Lambda(args=empty_args, body=call) + + +def test_variable_equal(node1, node2): + """Test whether two variables are the same.""" + if type(node1) is not type(node2): + return False + if isinstance(node1, ast.AST): + for k, v in vars(node1).items(): + if k in ('lineno', 'col_offset', 'ctx', 'end_lineno', 'end_col_offset'): + continue + if not test_variable_equal(v, getattr(node2, k)): + return False + return True + if isinstance(node1, list): + if len(node1) != len(node2): + return False + return all(test_variable_equal(n1, n2) for n1, n2 in zip(node1, node2)) + + return node1 == node2 + + +def replace_variable_node(node, annotation): + """Replace a node annotated by `nni.variable`. + node: the AST node to replace + annotation: annotation string + """ + assert type(node) is ast.Assign, 'nni.variable is not annotating assignment expression' + assert len(node.targets) == 1, 'Annotated assignment has more than one left-hand value' + name, expr = parse_nni_variable(annotation) + assert test_variable_equal(node.targets[0], name), 'Annotated variable has wrong name' + node.value = expr + return node + + +def replace_function_node(node, annotation): + """Replace a node annotated by `nni.function_choice`. + node: the AST node to replace + annotation: annotation string + """ + target, funcs = parse_nni_function(annotation) + FuncReplacer(funcs, target).visit(node) + return node + + +class FuncReplacer(ast.NodeTransformer): + """To replace target function call expressions in a node annotated by `nni.function_choice`""" + + def __init__(self, funcs, target): + """Constructor. + funcs: list of dumped function call expressions to replace + target: use this AST node to replace matching expressions + """ + self.funcs = set(funcs) + self.target = target + + def visit_Call(self, node): # pylint: disable=invalid-name + if ast.dump(node, False) in self.funcs: + return self.target + return node + + +class Transformer(ast.NodeTransformer): + """Transform original code to annotated code""" + + def __init__(self): + self.stack = [] + self.last_line = 0 + self.annotated = False + + def visit(self, node): + if isinstance(node, (ast.expr, ast.stmt)): + self.last_line = lineno(node) + + # do nothing for root + if not self.stack: + return self._visit_children(node) + + annotation = self.stack[-1] + + # this is a standalone string, may be an annotation + if type(node) is ast.Expr and type(node.value) is ast_Str: + # must not annotate an annotation string + assert annotation is None, 'Annotating an annotation' + return self._visit_string(node) + + if annotation is not None: # this expression is annotated + self.stack[-1] = None # so next expression is not + if annotation.startswith('nni.variable'): + return replace_variable_node(node, annotation) + if annotation.startswith('nni.function_choice'): + return replace_function_node(node, annotation) + + return self._visit_children(node) + + def _visit_string(self, node): + string = node.value.s + if string.startswith('@nni.'): + self.annotated = True + else: + return node # not an annotation, ignore it + + if string.startswith('@nni.get_next_parameter'): + deprecated_message = "'@nni.get_next_parameter' is deprecated in annotation due to inconvenience. " \ + "Please remove this line in the trial code." + print_warning(deprecated_message) + return ast.Expr(value=ast.Call(func=ast.Name(id='print', ctx=ast.Load()), + args=[ast_Str(s='Get next parameter here...')], keywords=[])) + + if string.startswith('@nni.training_update'): + return ast.Expr(value=ast.Call(func=ast.Name(id='print', ctx=ast.Load()), + args=[ast_Str(s='Training update here...')], keywords=[])) + + if string.startswith('@nni.report_intermediate_result'): + module = ast.parse(string[1:]) + arg = module.body[0].value.args[0] + return ast.Expr(value=ast.Call(func=ast.Name(id='print', ctx=ast.Load()), + args=[ast_Str(s='nni.report_intermediate_result: '), arg], keywords=[])) + + if string.startswith('@nni.report_final_result'): + module = ast.parse(string[1:]) + arg = module.body[0].value.args[0] + return ast.Expr(value=ast.Call(func=ast.Name(id='print', ctx=ast.Load()), + args=[ast_Str(s='nni.report_final_result: '), arg], keywords=[])) + + if string.startswith('@nni.mutable_layers'): + return parse_annotation_mutable_layers(string[1:], lineno(node)) + + if string.startswith('@nni.variable') \ + or string.startswith('@nni.function_choice'): + self.stack[-1] = string[1:] # mark that the next expression is annotated + return None + + raise AssertionError('Unexpected annotation function') + + def _visit_children(self, node): + self.stack.append(None) + self.generic_visit(node) + annotation = self.stack.pop() + assert annotation is None, 'Annotation has no target' + return node + + +def parse(code, para, module): + """Annotate user code. + Return annotated code (str) if annotation detected; return None if not. + code: original user code (str) + """ + global para_cfg + global prefix_name + para_cfg = para + prefix_name = module + try: + ast_tree = ast.parse(code) + except Exception: + raise RuntimeError('Bad Python code') + + transformer = Transformer() + try: + transformer.visit(ast_tree) + except AssertionError as exc: + raise RuntimeError('%d: %s' % (ast_tree.last_line, exc.args[0])) + + if not transformer.annotated: + return None + + return astor.to_source(ast_tree) diff --git a/utils/third_party/nni/tools/annotation/utils.py b/utils/third_party/nni/tools/annotation/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..176621c2cf3d9211d626b182277ef61f170880f9 --- /dev/null +++ b/utils/third_party/nni/tools/annotation/utils.py @@ -0,0 +1,22 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import ast +from sys import version_info + + +if version_info >= (3, 8): + ast_Num = ast_Str = ast_Bytes = ast_NameConstant = ast_Ellipsis = ast.Constant + + def lineno(ast_node): + return ast_node.end_lineno + +else: + ast_Num = ast.Num + ast_Str = ast.Str + ast_Bytes = ast.Bytes + ast_NameConstant = ast.NameConstant + ast_Ellipsis = ast.Ellipsis + + def lineno(ast_node): + return ast_node.lineno diff --git a/utils/third_party/nni/tools/gpu_tool/__init__.py b/utils/third_party/nni/tools/gpu_tool/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/utils/third_party/nni/tools/gpu_tool/gpu_metrics_collector.py b/utils/third_party/nni/tools/gpu_tool/gpu_metrics_collector.py new file mode 100644 index 0000000000000000000000000000000000000000..8e50cf9ec84533c1b8ded1799918934677eff058 --- /dev/null +++ b/utils/third_party/nni/tools/gpu_tool/gpu_metrics_collector.py @@ -0,0 +1,82 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import json +import os +import subprocess +import sys +import time +import traceback + +from xml.dom import minidom + + +def main(argv): + metrics_output_dir = os.environ['METRIC_OUTPUT_DIR'] + + cmd = 'nvidia-smi -q -x'.split() + while(True): + try: + smi_output = subprocess.check_output(cmd) + except Exception: + traceback.print_exc() + gen_empty_gpu_metric(metrics_output_dir) + break + parse_nvidia_smi_result(smi_output, metrics_output_dir) + # TODO: change to sleep time configurable via arguments + time.sleep(5) + + +def parse_nvidia_smi_result(smi, outputDir): + try: + old_umask = os.umask(0) + xmldoc = minidom.parseString(smi) + gpuList = xmldoc.getElementsByTagName('gpu') + with open(os.path.join(outputDir, "gpu_metrics"), 'a') as outputFile: + outPut = {} + outPut["Timestamp"] = time.asctime(time.localtime()) + outPut["gpuCount"] = len(gpuList) + outPut["gpuInfos"] = [] + for gpuIndex, gpu in enumerate(gpuList): + gpuInfo = {} + gpuInfo['index'] = gpuIndex + gpuInfo['gpuUtil'] = gpu.getElementsByTagName('utilization')[0]\ + .getElementsByTagName('gpu_util')[0]\ + .childNodes[0].data.replace("%", "").strip() + gpuInfo['gpuMemUtil'] = gpu.getElementsByTagName('utilization')[0]\ + .getElementsByTagName('memory_util')[0]\ + .childNodes[0].data.replace("%", "").strip() + processes = gpu.getElementsByTagName('processes') + runningProNumber = len(processes[0].getElementsByTagName('process_info')) + gpuInfo['activeProcessNum'] = runningProNumber + + outPut["gpuInfos"].append(gpuInfo) + print(outPut) + outputFile.write("{}\n".format(json.dumps(outPut, sort_keys=True))) + outputFile.flush() + except Exception as error: + # e_info = sys.exc_info() + print('gpu_metrics_collector error: %s' % error) + finally: + os.umask(old_umask) + + +def gen_empty_gpu_metric(outputDir): + try: + old_umask = os.umask(0) + with open(os.path.join(outputDir, "gpu_metrics"), 'a') as outputFile: + outPut = {} + outPut["Timestamp"] = time.asctime(time.localtime()) + outPut["gpuCount"] = 0 + outPut["gpuInfos"] = [] + print(outPut) + outputFile.write("{}\n".format(json.dumps(outPut, sort_keys=True))) + outputFile.flush() + except Exception: + traceback.print_exc() + finally: + os.umask(old_umask) + + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/utils/third_party/nni/tools/nnictl/__init__.py b/utils/third_party/nni/tools/nnictl/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/utils/third_party/nni/tools/nnictl/command_utils.py b/utils/third_party/nni/tools/nnictl/command_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..2bbcc883d1cef450fef9b723dc8ca71d34daa095 --- /dev/null +++ b/utils/third_party/nni/tools/nnictl/command_utils.py @@ -0,0 +1,82 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from subprocess import call, check_output +import sys +import os +import signal +import psutil +from .common_utils import print_error + + +def check_output_command(file_path, head=None, tail=None): + """call check_output command to read content from a file""" + if os.path.exists(file_path): + if sys.platform == 'win32': + cmds = ['powershell.exe', 'type', file_path] + if head: + cmds += ['|', 'select', '-first', str(head)] + elif tail: + cmds += ['|', 'select', '-last', str(tail)] + return check_output(cmds, shell=True).decode('utf-8') + else: + cmds = ['cat', file_path] + if head: + cmds = ['head', '-' + str(head), file_path] + elif tail: + cmds = ['tail', '-' + str(tail), file_path] + return check_output(cmds, shell=False).decode('utf-8') + else: + print_error('{0} does not exist!'.format(file_path)) + exit(1) + + +def kill_command(pid): + """kill command""" + if sys.platform == 'win32': + process = psutil.Process(pid=pid) + process.send_signal(signal.CTRL_BREAK_EVENT) + else: + cmds = ['kill', str(pid)] + call(cmds) + + +def install_package_command(package_name): + """ + Install python package from pip. + + Parameters + ---------- + package_name: str + The name of package to be installed. + """ + call(_get_pip_install() + [package_name], shell=False) + + +def install_requirements_command(requirements_path): + """ + Install packages from `requirements.txt` in `requirements_path`. + + Parameters + ---------- + requirements_path: str + Path to the directory that contains `requirements.txt`. + """ + return call(_get_pip_install() + ["-r", requirements_path], shell=False) + + +def _get_pip_install(): + python = "python" if sys.platform == "win32" else "python3" + ret = [python, "-m", "pip", "install"] + if "CONDA_DEFAULT_ENV" not in os.environ and "VIRTUAL_ENV" not in os.environ and \ + (sys.platform != "win32" and os.getuid() != 0): # on unix and not running in root + ret.append("--user") # not in virtualenv or conda + return ret + +def call_pip_install(source): + return call(_get_pip_install() + [source]) + +def call_pip_uninstall(module_name): + python = "python" if sys.platform == "win32" else "python3" + cmd = [python, "-m", "pip", "uninstall", module_name] + return call(cmd) diff --git a/utils/third_party/nni/tools/nnictl/common_utils.py b/utils/third_party/nni/tools/nnictl/common_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..2edbf667dff3234715281bbd1111b14380831282 --- /dev/null +++ b/utils/third_party/nni/tools/nnictl/common_utils.py @@ -0,0 +1,97 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +import sys +import json +import tempfile +import socket +import string +import random +import ruamel.yaml as yaml +import psutil +from colorama import Fore + +from .constants import ERROR_INFO, NORMAL_INFO, WARNING_INFO + +def get_yml_content(file_path): + '''Load yaml file content''' + try: + with open(file_path, 'r') as file: + return yaml.load(file, Loader=yaml.Loader) + except yaml.scanner.ScannerError as err: + print_error('yaml file format error!') + print_error(err) + exit(1) + except Exception as exception: + print_error(exception) + exit(1) + +def get_json_content(file_path): + '''Load json file content''' + try: + with open(file_path, 'r') as file: + return json.load(file) + except TypeError as err: + print_error('json file format error!') + print_error(err) + return None + + +def print_error(*content): + '''Print error information to screen''' + print(Fore.RED + ERROR_INFO + ' '.join([str(c) for c in content]) + Fore.RESET) + +def print_green(*content): + '''Print information to screen in green''' + print(Fore.GREEN + ' '.join([str(c) for c in content]) + Fore.RESET) + +def print_normal(*content): + '''Print error information to screen''' + print(NORMAL_INFO, *content) + +def print_warning(*content): + '''Print warning information to screen''' + print(Fore.YELLOW + WARNING_INFO + ' '.join([str(c) for c in content]) + Fore.RESET) + +def detect_process(pid): + '''Detect if a process is alive''' + try: + process = psutil.Process(pid) + return process.is_running() + except: + return False + +def detect_port(port): + '''Detect if the port is used''' + socket_test = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + try: + socket_test.connect(('127.0.0.1', int(port))) + socket_test.close() + return True + except: + return False + +def get_user(): + if sys.platform == 'win32': + return os.environ['USERNAME'] + else: + return os.environ['USER'] + +def check_tensorboard_version(): + try: + import tensorboard + return tensorboard.__version__ + except: + print_error('import tensorboard error!') + exit(1) + +def generate_temp_dir(): + '''generate a temp folder''' + def generate_folder_name(): + return os.path.join(tempfile.gettempdir(), 'nni', ''.join(random.sample(string.ascii_letters + string.digits, 8))) + temp_dir = generate_folder_name() + while os.path.exists(temp_dir): + temp_dir = generate_folder_name() + os.makedirs(temp_dir) + return temp_dir diff --git a/utils/third_party/nni/tools/nnictl/config_schema.py b/utils/third_party/nni/tools/nnictl/config_schema.py new file mode 100644 index 0000000000000000000000000000000000000000..d3201635955ed656e12339ca54109aab9ab3656a --- /dev/null +++ b/utils/third_party/nni/tools/nnictl/config_schema.py @@ -0,0 +1,539 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import json +import logging +import os +import netifaces +from schema import Schema, And, Optional, Regex, Or, SchemaError +from nni.tools.package_utils import create_validator_instance, get_all_builtin_names, get_builtin_algo_meta +from .constants import SCHEMA_TYPE_ERROR, SCHEMA_RANGE_ERROR, SCHEMA_PATH_ERROR +from .common_utils import get_yml_content, print_warning + + +def setType(key, valueType): + '''check key type''' + return And(valueType, error=SCHEMA_TYPE_ERROR % (key, valueType.__name__)) + + +def setChoice(key, *args): + '''check choice''' + return And(lambda n: n in args, error=SCHEMA_RANGE_ERROR % (key, str(args))) + + +def setNumberRange(key, keyType, start, end): + '''check number range''' + return And( + And(keyType, error=SCHEMA_TYPE_ERROR % (key, keyType.__name__)), + And(lambda n: start <= n <= end, error=SCHEMA_RANGE_ERROR % (key, '(%s,%s)' % (start, end))), + ) + + +def setPathCheck(key): + '''check if path exist''' + return And(os.path.exists, error=SCHEMA_PATH_ERROR % key) + + +class AlgoSchema: + """ + This class is the schema of 'tuner', 'assessor' and 'advisor' sections of experiment configuraion file. + For example: + AlgoSchema('tuner') creates the schema of tuner section. + """ + + def __init__(self, algo_type): + """ + Parameters: + ----------- + algo_type: str + One of ['tuner', 'assessor', 'advisor']. + 'tuner': This AlgoSchema class create the schema of tuner section. + 'assessor': This AlgoSchema class create the schema of assessor section. + 'advisor': This AlgoSchema class create the schema of advisor section. + """ + assert algo_type in ['tuner', 'assessor', 'advisor'] + self.algo_type = algo_type + self.algo_schema = { + Optional('codeDir'): setPathCheck('codeDir'), + Optional('classFileName'): setType('classFileName', str), + Optional('className'): setType('className', str), + Optional('classArgs'): dict, + Optional('includeIntermediateResults'): setType('includeIntermediateResults', bool), + Optional('gpuIndices'): Or(int, And(str, lambda x: len([int(i) for i in x.split(',')]) > 0), error='gpuIndex format error!'), + } + self.builtin_keys = { + 'tuner': 'builtinTunerName', + 'assessor': 'builtinAssessorName', + 'advisor': 'builtinAdvisorName' + } + self.builtin_name_schema = {} + for k, n in self.builtin_keys.items(): + self.builtin_name_schema[k] = {Optional(n): setChoice(n, *get_all_builtin_names(k+'s'))} + + self.customized_keys = set(['codeDir', 'classFileName', 'className']) + + def validate_class_args(self, class_args, algo_type, builtin_name): + if not builtin_name or not class_args: + return + meta = get_builtin_algo_meta(algo_type+'s', builtin_name) + if meta and 'accept_class_args' in meta and meta['accept_class_args'] == False: + raise SchemaError('classArgs is not allowed.') + + logging.getLogger('nni.protocol').setLevel(logging.ERROR) # we know IPC is not there, don't complain + validator = create_validator_instance(algo_type+'s', builtin_name) + if validator: + try: + validator.validate_class_args(**class_args) + except Exception as e: + raise SchemaError(str(e)) + + def missing_customized_keys(self, data): + return self.customized_keys - set(data.keys()) + + def validate_extras(self, data, algo_type): + builtin_key = self.builtin_keys[algo_type] + if (builtin_key in data) and (set(data.keys()) & self.customized_keys): + raise SchemaError('{} and {} cannot be specified at the same time.'.format( + builtin_key, set(data.keys()) & self.customized_keys + )) + + if self.missing_customized_keys(data) and builtin_key not in data: + raise SchemaError('Either customized {} ({}) or builtin {} ({}) must be set.'.format( + algo_type, self.customized_keys, algo_type, builtin_key)) + + if not self.missing_customized_keys(data): + class_file_name = os.path.join(data['codeDir'], data['classFileName']) + if not os.path.isfile(class_file_name): + raise SchemaError('classFileName {} not found.'.format(class_file_name)) + + builtin_name = data.get(builtin_key) + class_args = data.get('classArgs') + self.validate_class_args(class_args, algo_type, builtin_name) + + def validate(self, data): + self.algo_schema.update(self.builtin_name_schema[self.algo_type]) + Schema(self.algo_schema).validate(data) + self.validate_extras(data, self.algo_type) + + +common_schema = { + 'authorName': setType('authorName', str), + 'experimentName': setType('experimentName', str), + Optional('description'): setType('description', str), + 'trialConcurrency': setNumberRange('trialConcurrency', int, 1, 99999), + Optional('maxExecDuration'): And(Regex(r'^[1-9][0-9]*[s|m|h|d]$', error='ERROR: maxExecDuration format is [digit]{s,m,h,d}')), + Optional('maxTrialNum'): setNumberRange('maxTrialNum', int, 1, 99999), + 'trainingServicePlatform': setChoice( + 'trainingServicePlatform', 'remote', 'local', 'pai', 'kubeflow', 'frameworkcontroller', 'paiYarn', 'dlts', 'aml'), + Optional('searchSpacePath'): And(os.path.exists, error=SCHEMA_PATH_ERROR % 'searchSpacePath'), + Optional('multiPhase'): setType('multiPhase', bool), + Optional('multiThread'): setType('multiThread', bool), + Optional('nniManagerIp'): setType('nniManagerIp', str), + Optional('logDir'): And(os.path.isdir, error=SCHEMA_PATH_ERROR % 'logDir'), + Optional('debug'): setType('debug', bool), + Optional('versionCheck'): setType('versionCheck', bool), + Optional('logLevel'): setChoice('logLevel', 'trace', 'debug', 'info', 'warning', 'error', 'fatal'), + Optional('logCollection'): setChoice('logCollection', 'http', 'none'), + 'useAnnotation': setType('useAnnotation', bool), + Optional('tuner'): AlgoSchema('tuner'), + Optional('advisor'): AlgoSchema('advisor'), + Optional('assessor'): AlgoSchema('assessor'), + Optional('localConfig'): { + Optional('gpuIndices'): Or(int, And(str, lambda x: len([int(i) for i in x.split(',')]) > 0), error='gpuIndex format error!'), + Optional('maxTrialNumPerGpu'): setType('maxTrialNumPerGpu', int), + Optional('useActiveGpu'): setType('useActiveGpu', bool) + } +} + +common_trial_schema = { + 'trial': { + 'command': setType('command', str), + 'codeDir': setPathCheck('codeDir'), + Optional('gpuNum'): setNumberRange('gpuNum', int, 0, 99999), + Optional('nasMode'): setChoice('nasMode', 'classic_mode', 'enas_mode', 'oneshot_mode', 'darts_mode') + } +} + +pai_yarn_trial_schema = { + 'trial': { + 'command': setType('command', str), + 'codeDir': setPathCheck('codeDir'), + 'gpuNum': setNumberRange('gpuNum', int, 0, 99999), + 'cpuNum': setNumberRange('cpuNum', int, 0, 99999), + 'memoryMB': setType('memoryMB', int), + 'image': setType('image', str), + Optional('authFile'): And(os.path.exists, error=SCHEMA_PATH_ERROR % 'authFile'), + Optional('shmMB'): setType('shmMB', int), + Optional('dataDir'): And(Regex(r'hdfs://(([0-9]{1,3}.){3}[0-9]{1,3})(:[0-9]{2,5})?(/.*)?'), + error='ERROR: dataDir format error, dataDir format is hdfs://xxx.xxx.xxx.xxx:xxx'), + Optional('outputDir'): And(Regex(r'hdfs://(([0-9]{1,3}.){3}[0-9]{1,3})(:[0-9]{2,5})?(/.*)?'), + error='ERROR: outputDir format error, outputDir format is hdfs://xxx.xxx.xxx.xxx:xxx'), + Optional('virtualCluster'): setType('virtualCluster', str), + Optional('nasMode'): setChoice('nasMode', 'classic_mode', 'enas_mode', 'oneshot_mode', 'darts_mode'), + Optional('portList'): [{ + "label": setType('label', str), + "beginAt": setType('beginAt', int), + "portNumber": setType('portNumber', int) + }] + } +} + +pai_yarn_config_schema = { + 'paiYarnConfig': Or({ + 'userName': setType('userName', str), + 'passWord': setType('passWord', str), + 'host': setType('host', str) + }, { + 'userName': setType('userName', str), + 'token': setType('token', str), + 'host': setType('host', str) + }) +} + + +pai_trial_schema = { + 'trial': { + 'codeDir': setPathCheck('codeDir'), + 'nniManagerNFSMountPath': setPathCheck('nniManagerNFSMountPath'), + 'containerNFSMountPath': setType('containerNFSMountPath', str), + Optional('command'): setType('command', str), + Optional('gpuNum'): setNumberRange('gpuNum', int, 0, 99999), + Optional('cpuNum'): setNumberRange('cpuNum', int, 0, 99999), + Optional('memoryMB'): setType('memoryMB', int), + Optional('image'): setType('image', str), + Optional('virtualCluster'): setType('virtualCluster', str), + Optional('paiStorageConfigName'): setType('paiStorageConfigName', str), + Optional('paiConfigPath'): And(os.path.exists, error=SCHEMA_PATH_ERROR % 'paiConfigPath') + } +} + +pai_config_schema = { + 'paiConfig': { + 'userName': setType('userName', str), + Or('passWord', 'token', only_one=True): str, + 'host': setType('host', str), + Optional('reuse'): setType('reuse', bool), + Optional('gpuNum'): setNumberRange('gpuNum', int, 0, 99999), + Optional('cpuNum'): setNumberRange('cpuNum', int, 0, 99999), + Optional('memoryMB'): setType('memoryMB', int), + Optional('maxTrialNumPerGpu'): setType('maxTrialNumPerGpu', int), + Optional('useActiveGpu'): setType('useActiveGpu', bool), + } +} + +dlts_trial_schema = { + 'trial': { + 'command': setType('command', str), + 'codeDir': setPathCheck('codeDir'), + 'gpuNum': setNumberRange('gpuNum', int, 0, 99999), + 'image': setType('image', str), + } +} + +dlts_config_schema = { + 'dltsConfig': { + 'dashboard': setType('dashboard', str), + + Optional('cluster'): setType('cluster', str), + Optional('team'): setType('team', str), + + Optional('email'): setType('email', str), + Optional('password'): setType('password', str), + } +} + +aml_trial_schema = { + 'trial': { + 'codeDir': setPathCheck('codeDir'), + 'command': setType('command', str), + 'image': setType('image', str), + Optional('gpuNum'): setNumberRange('gpuNum', int, 0, 99999), + } +} + +aml_config_schema = { + 'amlConfig': { + 'subscriptionId': setType('subscriptionId', str), + 'resourceGroup': setType('resourceGroup', str), + 'workspaceName': setType('workspaceName', str), + 'computeTarget': setType('computeTarget', str), + Optional('maxTrialNumPerGpu'): setType('maxTrialNumPerGpu', int), + Optional('useActiveGpu'): setType('useActiveGpu', bool), + } +} + +kubeflow_trial_schema = { + 'trial': { + 'codeDir': setPathCheck('codeDir'), + Optional('nasMode'): setChoice('nasMode', 'classic_mode', 'enas_mode', 'oneshot_mode', 'darts_mode'), + Optional('ps'): { + 'replicas': setType('replicas', int), + 'command': setType('command', str), + 'gpuNum': setNumberRange('gpuNum', int, 0, 99999), + 'cpuNum': setNumberRange('cpuNum', int, 0, 99999), + 'memoryMB': setType('memoryMB', int), + 'image': setType('image', str), + Optional('privateRegistryAuthPath'): And(os.path.exists, error=SCHEMA_PATH_ERROR % 'privateRegistryAuthPath') + }, + Optional('master'): { + 'replicas': setType('replicas', int), + 'command': setType('command', str), + 'gpuNum': setNumberRange('gpuNum', int, 0, 99999), + 'cpuNum': setNumberRange('cpuNum', int, 0, 99999), + 'memoryMB': setType('memoryMB', int), + 'image': setType('image', str), + Optional('privateRegistryAuthPath'): And(os.path.exists, error=SCHEMA_PATH_ERROR % 'privateRegistryAuthPath') + }, + Optional('worker'): { + 'replicas': setType('replicas', int), + 'command': setType('command', str), + 'gpuNum': setNumberRange('gpuNum', int, 0, 99999), + 'cpuNum': setNumberRange('cpuNum', int, 0, 99999), + 'memoryMB': setType('memoryMB', int), + 'image': setType('image', str), + Optional('privateRegistryAuthPath'): And(os.path.exists, error=SCHEMA_PATH_ERROR % 'privateRegistryAuthPath') + } + } +} + +kubeflow_config_schema = { + 'kubeflowConfig': Or({ + 'operator': setChoice('operator', 'tf-operator', 'pytorch-operator'), + 'apiVersion': setType('apiVersion', str), + Optional('storage'): setChoice('storage', 'nfs', 'azureStorage'), + 'nfs': { + 'server': setType('server', str), + 'path': setType('path', str) + } + }, { + 'operator': setChoice('operator', 'tf-operator', 'pytorch-operator'), + 'apiVersion': setType('apiVersion', str), + Optional('storage'): setChoice('storage', 'nfs', 'azureStorage'), + 'keyVault': { + 'vaultName': And(Regex('([0-9]|[a-z]|[A-Z]|-){1,127}'), + error='ERROR: vaultName format error, vaultName support using (0-9|a-z|A-Z|-)'), + 'name': And(Regex('([0-9]|[a-z]|[A-Z]|-){1,127}'), + error='ERROR: name format error, name support using (0-9|a-z|A-Z|-)') + }, + 'azureStorage': { + 'accountName': And(Regex('([0-9]|[a-z]|[A-Z]|-){3,31}'), + error='ERROR: accountName format error, accountName support using (0-9|a-z|A-Z|-)'), + 'azureShare': And(Regex('([0-9]|[a-z]|[A-Z]|-){3,63}'), + error='ERROR: azureShare format error, azureShare support using (0-9|a-z|A-Z|-)') + }, + Optional('uploadRetryCount'): setNumberRange('uploadRetryCount', int, 1, 99999) + }) +} + +frameworkcontroller_trial_schema = { + 'trial': { + 'codeDir': setPathCheck('codeDir'), + 'taskRoles': [{ + 'name': setType('name', str), + 'taskNum': setType('taskNum', int), + 'frameworkAttemptCompletionPolicy': { + 'minFailedTaskCount': setType('minFailedTaskCount', int), + 'minSucceededTaskCount': setType('minSucceededTaskCount', int), + }, + 'command': setType('command', str), + 'gpuNum': setNumberRange('gpuNum', int, 0, 99999), + 'cpuNum': setNumberRange('cpuNum', int, 0, 99999), + 'memoryMB': setType('memoryMB', int), + 'image': setType('image', str), + Optional('privateRegistryAuthPath'): And(os.path.exists, error=SCHEMA_PATH_ERROR % 'privateRegistryAuthPath') + }] + } +} + +frameworkcontroller_config_schema = { + 'frameworkcontrollerConfig': Or({ + Optional('storage'): setChoice('storage', 'nfs', 'azureStorage'), + Optional('serviceAccountName'): setType('serviceAccountName', str), + 'nfs': { + 'server': setType('server', str), + 'path': setType('path', str) + } + }, { + Optional('storage'): setChoice('storage', 'nfs', 'azureStorage'), + Optional('serviceAccountName'): setType('serviceAccountName', str), + 'keyVault': { + 'vaultName': And(Regex('([0-9]|[a-z]|[A-Z]|-){1,127}'), + error='ERROR: vaultName format error, vaultName support using (0-9|a-z|A-Z|-)'), + 'name': And(Regex('([0-9]|[a-z]|[A-Z]|-){1,127}'), + error='ERROR: name format error, name support using (0-9|a-z|A-Z|-)') + }, + 'azureStorage': { + 'accountName': And(Regex('([0-9]|[a-z]|[A-Z]|-){3,31}'), + error='ERROR: accountName format error, accountName support using (0-9|a-z|A-Z|-)'), + 'azureShare': And(Regex('([0-9]|[a-z]|[A-Z]|-){3,63}'), + error='ERROR: azureShare format error, azureShare support using (0-9|a-z|A-Z|-)') + }, + Optional('uploadRetryCount'): setNumberRange('uploadRetryCount', int, 1, 99999) + }) +} + +remote_config_schema = { + Optional('remoteConfig'): { + 'reuse': setType('reuse', bool) + } +} + +machine_list_schema = { + 'machineList': [Or( + { + 'ip': setType('ip', str), + Optional('port'): setNumberRange('port', int, 1, 65535), + 'username': setType('username', str), + 'sshKeyPath': setPathCheck('sshKeyPath'), + Optional('passphrase'): setType('passphrase', str), + Optional('gpuIndices'): Or(int, And(str, lambda x: len([int(i) for i in x.split(',')]) > 0), error='gpuIndex format error!'), + Optional('maxTrialNumPerGpu'): setType('maxTrialNumPerGpu', int), + Optional('useActiveGpu'): setType('useActiveGpu', bool), + Optional('preCommand'): setType('preCommand', str) + }, + { + 'ip': setType('ip', str), + Optional('port'): setNumberRange('port', int, 1, 65535), + 'username': setType('username', str), + 'passwd': setType('passwd', str), + Optional('gpuIndices'): Or(int, And(str, lambda x: len([int(i) for i in x.split(',')]) > 0), error='gpuIndex format error!'), + Optional('maxTrialNumPerGpu'): setType('maxTrialNumPerGpu', int), + Optional('useActiveGpu'): setType('useActiveGpu', bool), + Optional('preCommand'): setType('preCommand', str) + })] +} + +training_service_schema_dict = { + 'local': Schema({**common_schema, **common_trial_schema}), + 'remote': Schema({**common_schema, **common_trial_schema, **machine_list_schema, **remote_config_schema}), + 'pai': Schema({**common_schema, **pai_trial_schema, **pai_config_schema}), + 'paiYarn': Schema({**common_schema, **pai_yarn_trial_schema, **pai_yarn_config_schema}), + 'kubeflow': Schema({**common_schema, **kubeflow_trial_schema, **kubeflow_config_schema}), + 'frameworkcontroller': Schema({**common_schema, **frameworkcontroller_trial_schema, **frameworkcontroller_config_schema}), + 'aml': Schema({**common_schema, **aml_trial_schema, **aml_config_schema}), + 'dlts': Schema({**common_schema, **dlts_trial_schema, **dlts_config_schema}), +} + + +class NNIConfigSchema: + def validate(self, data): + train_service = data['trainingServicePlatform'] + Schema(common_schema['trainingServicePlatform']).validate(train_service) + train_service_schema = training_service_schema_dict[train_service] + train_service_schema.validate(data) + self.validate_extras(data) + + def validate_extras(self, experiment_config): + self.validate_tuner_adivosr_assessor(experiment_config) + self.validate_pai_trial_conifg(experiment_config) + self.validate_kubeflow_operators(experiment_config) + self.validate_eth0_device(experiment_config) + + def validate_tuner_adivosr_assessor(self, experiment_config): + if experiment_config.get('advisor'): + if experiment_config.get('assessor') or experiment_config.get('tuner'): + raise SchemaError('advisor could not be set with assessor or tuner simultaneously!') + self.validate_annotation_content(experiment_config, 'advisor', 'builtinAdvisorName') + else: + if not experiment_config.get('tuner'): + raise SchemaError('Please provide tuner spec!') + self.validate_annotation_content(experiment_config, 'tuner', 'builtinTunerName') + + def validate_search_space_content(self, experiment_config): + '''Validate searchspace content, + if the searchspace file is not json format or its values does not contain _type and _value which must be specified, + it will not be a valid searchspace file''' + try: + search_space_content = json.load(open(experiment_config.get('searchSpacePath'), 'r')) + for value in search_space_content.values(): + if not value.get('_type') or not value.get('_value'): + raise SchemaError('please use _type and _value to specify searchspace!') + except Exception as e: + raise SchemaError('searchspace file is not a valid json format! ' + str(e)) + + def validate_kubeflow_operators(self, experiment_config): + '''Validate whether the kubeflow operators are valid''' + if experiment_config.get('kubeflowConfig'): + if experiment_config.get('kubeflowConfig').get('operator') == 'tf-operator': + if experiment_config.get('trial').get('master') is not None: + raise SchemaError('kubeflow with tf-operator can not set master') + if experiment_config.get('trial').get('worker') is None: + raise SchemaError('kubeflow with tf-operator must set worker') + elif experiment_config.get('kubeflowConfig').get('operator') == 'pytorch-operator': + if experiment_config.get('trial').get('ps') is not None: + raise SchemaError('kubeflow with pytorch-operator can not set ps') + if experiment_config.get('trial').get('master') is None: + raise SchemaError('kubeflow with pytorch-operator must set master') + + if experiment_config.get('kubeflowConfig').get('storage') == 'nfs': + if experiment_config.get('kubeflowConfig').get('nfs') is None: + raise SchemaError('please set nfs configuration!') + elif experiment_config.get('kubeflowConfig').get('storage') == 'azureStorage': + if experiment_config.get('kubeflowConfig').get('azureStorage') is None: + raise SchemaError('please set azureStorage configuration!') + elif experiment_config.get('kubeflowConfig').get('storage') is None: + if experiment_config.get('kubeflowConfig').get('azureStorage'): + raise SchemaError('please set storage type!') + + def validate_annotation_content(self, experiment_config, spec_key, builtin_name): + ''' + Valid whether useAnnotation and searchSpacePath is coexist + spec_key: 'advisor' or 'tuner' + builtin_name: 'builtinAdvisorName' or 'builtinTunerName' + ''' + if experiment_config.get('useAnnotation'): + if experiment_config.get('searchSpacePath'): + raise SchemaError('If you set useAnnotation=true, please leave searchSpacePath empty') + else: + # validate searchSpaceFile + if experiment_config[spec_key].get(builtin_name) == 'NetworkMorphism': + return + if experiment_config[spec_key].get(builtin_name): + if experiment_config.get('searchSpacePath') is None: + raise SchemaError('Please set searchSpacePath!') + self.validate_search_space_content(experiment_config) + + def validate_pai_config_path(self, experiment_config): + '''validate paiConfigPath field''' + if experiment_config.get('trainingServicePlatform') == 'pai': + if experiment_config.get('trial', {}).get('paiConfigPath'): + # validate commands + pai_config = get_yml_content(experiment_config['trial']['paiConfigPath']) + taskRoles_dict = pai_config.get('taskRoles') + if not taskRoles_dict: + raise SchemaError('Please set taskRoles in paiConfigPath config file!') + else: + pai_trial_fields_required_list = ['image', 'paiStorageConfigName', 'command'] + for trial_field in pai_trial_fields_required_list: + if experiment_config['trial'].get(trial_field) is None: + raise SchemaError('Please set {0} in trial configuration,\ + or set additional pai configuration file path in paiConfigPath!'.format(trial_field)) + pai_resource_fields_required_list = ['gpuNum', 'cpuNum', 'memoryMB'] + for required_field in pai_resource_fields_required_list: + if experiment_config['trial'].get(required_field) is None and \ + experiment_config['paiConfig'].get(required_field) is None: + raise SchemaError('Please set {0} in trial or paiConfig configuration,\ + or set additional pai configuration file path in paiConfigPath!'.format(required_field)) + + def validate_pai_trial_conifg(self, experiment_config): + '''validate the trial config in pai platform''' + if experiment_config.get('trainingServicePlatform') in ['pai', 'paiYarn']: + if experiment_config.get('trial').get('shmMB') and \ + experiment_config['trial']['shmMB'] > experiment_config['trial']['memoryMB']: + raise SchemaError('shmMB should be no more than memoryMB!') + # backward compatibility + warning_information = '{0} is not supported in NNI anymore, please remove the field in config file!\ + please refer https://github.com/microsoft/nni/blob/master/docs/en_US/TrainingService/PaiMode.md#run-an-experiment\ + for the practices of how to get data and output model in trial code' + if experiment_config.get('trial').get('dataDir'): + print_warning(warning_information.format('dataDir')) + if experiment_config.get('trial').get('outputDir'): + print_warning(warning_information.format('outputDir')) + self.validate_pai_config_path(experiment_config) + + def validate_eth0_device(self, experiment_config): + '''validate whether the machine has eth0 device''' + if experiment_config.get('trainingServicePlatform') not in ['local'] \ + and not experiment_config.get('nniManagerIp') \ + and 'eth0' not in netifaces.interfaces(): + raise SchemaError('This machine does not contain eth0 network device, please set nniManagerIp in config file!') diff --git a/utils/third_party/nni/tools/nnictl/config_utils.py b/utils/third_party/nni/tools/nnictl/config_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..434bdebeca55fcfc80af6a233ebb57ba00ea9859 --- /dev/null +++ b/utils/third_party/nni/tools/nnictl/config_utils.py @@ -0,0 +1,112 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +import json +import shutil +from .constants import NNICTL_HOME_DIR +from .command_utils import print_error + +class Config: + '''a util class to load and save config''' + def __init__(self, file_path, home_dir=NNICTL_HOME_DIR): + config_path = os.path.join(home_dir, str(file_path)) + os.makedirs(config_path, exist_ok=True) + self.config_file = os.path.join(config_path, '.config') + self.config = self.read_file() + + def get_all_config(self): + '''get all of config values''' + return json.dumps(self.config, indent=4, sort_keys=True, separators=(',', ':')) + + def set_config(self, key, value): + '''set {key:value} paris to self.config''' + self.config = self.read_file() + self.config[key] = value + self.write_file() + + def get_config(self, key): + '''get a value according to key''' + return self.config.get(key) + + def write_file(self): + '''save config to local file''' + if self.config: + try: + with open(self.config_file, 'w') as file: + json.dump(self.config, file) + except IOError as error: + print('Error:', error) + return + + def read_file(self): + '''load config from local file''' + if os.path.exists(self.config_file): + try: + with open(self.config_file, 'r') as file: + return json.load(file) + except ValueError: + return {} + return {} + +class Experiments: + '''Maintain experiment list''' + def __init__(self, home_dir=NNICTL_HOME_DIR): + os.makedirs(home_dir, exist_ok=True) + self.experiment_file = os.path.join(home_dir, '.experiment') + self.experiments = self.read_file() + + def add_experiment(self, expId, port, startTime, file_name, platform, experiment_name, endTime='N/A', status='INITIALIZED'): + '''set {key:value} paris to self.experiment''' + self.experiments[expId] = {} + self.experiments[expId]['port'] = port + self.experiments[expId]['startTime'] = startTime + self.experiments[expId]['endTime'] = endTime + self.experiments[expId]['status'] = status + self.experiments[expId]['fileName'] = file_name + self.experiments[expId]['platform'] = platform + self.experiments[expId]['experimentName'] = experiment_name + self.write_file() + + def update_experiment(self, expId, key, value): + '''Update experiment''' + if expId not in self.experiments: + return False + self.experiments[expId][key] = value + self.write_file() + return True + + def remove_experiment(self, expId): + '''remove an experiment by id''' + if expId in self.experiments: + fileName = self.experiments.pop(expId).get('fileName') + if fileName: + logPath = os.path.join(NNICTL_HOME_DIR, fileName) + try: + shutil.rmtree(logPath) + except FileNotFoundError: + print_error('{0} does not exist.'.format(logPath)) + self.write_file() + + def get_all_experiments(self): + '''return all of experiments''' + return self.experiments + + def write_file(self): + '''save config to local file''' + try: + with open(self.experiment_file, 'w') as file: + json.dump(self.experiments, file) + except IOError as error: + print('Error:', error) + return '' + + def read_file(self): + '''load config from local file''' + if os.path.exists(self.experiment_file): + try: + with open(self.experiment_file, 'r') as file: + return json.load(file) + except ValueError: + return {} + return {} diff --git a/utils/third_party/nni/tools/nnictl/constants.py b/utils/third_party/nni/tools/nnictl/constants.py new file mode 100644 index 0000000000000000000000000000000000000000..0654473ed4548caf3cd9d80d49ce4624c10d6b66 --- /dev/null +++ b/utils/third_party/nni/tools/nnictl/constants.py @@ -0,0 +1,102 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +from colorama import Fore + +NNICTL_HOME_DIR = os.path.join(os.path.expanduser('~'), '.local', 'nnictl') + +NNI_HOME_DIR = os.path.join(os.path.expanduser('~'), 'nni-experiments') + +ERROR_INFO = 'ERROR: ' +NORMAL_INFO = 'INFO: ' +WARNING_INFO = 'WARNING: ' + +DEFAULT_REST_PORT = 8080 +REST_TIME_OUT = 20 + +EXPERIMENT_SUCCESS_INFO = Fore.GREEN + 'Successfully started experiment!\n' + Fore.RESET + \ + '------------------------------------------------------------------------------------\n' \ + 'The experiment id is %s\n'\ + 'The Web UI urls are: %s\n' \ + '------------------------------------------------------------------------------------\n\n' \ + 'You can use these commands to get more information about the experiment\n' \ + '------------------------------------------------------------------------------------\n' \ + ' commands description\n' \ + '1. nnictl experiment show show the information of experiments\n' \ + '2. nnictl trial ls list all of trial jobs\n' \ + '3. nnictl top monitor the status of running experiments\n' \ + '4. nnictl log stderr show stderr log content\n' \ + '5. nnictl log stdout show stdout log content\n' \ + '6. nnictl stop stop an experiment\n' \ + '7. nnictl trial kill kill a trial job by id\n' \ + '8. nnictl --help get help information about nnictl\n' \ + '------------------------------------------------------------------------------------\n' \ + 'Command reference document https://nni.readthedocs.io/en/latest/Tutorial/Nnictl.html\n' \ + '------------------------------------------------------------------------------------\n' + +LOG_HEADER = '-----------------------------------------------------------------------\n' \ + ' Experiment start time %s\n' \ + '-----------------------------------------------------------------------\n' + +EXPERIMENT_START_FAILED_INFO = 'There is an experiment running in the port %d, please stop it first or set another port!\n' \ + 'You could use \'nnictl stop --port [PORT]\' command to stop an experiment!\nOr you could ' \ + 'use \'nnictl create --config [CONFIG_PATH] --port [PORT]\' to set port!\n' + +EXPERIMENT_INFORMATION_FORMAT = '----------------------------------------------------------------------------------------\n' \ + ' Experiment information\n' \ + '%s\n' \ + '----------------------------------------------------------------------------------------\n' + +EXPERIMENT_DETAIL_FORMAT = 'Id: %s Name: %s Status: %s Port: %s Platform: %s StartTime: %s EndTime: %s\n' + +EXPERIMENT_MONITOR_INFO = 'Id: %s Status: %s Port: %s Platform: %s \n' \ + 'StartTime: %s Duration: %s' + +TRIAL_MONITOR_HEAD = '-------------------------------------------------------------------------------------\n' + \ + '%-15s %-25s %-25s %-15s \n' % ('trialId', 'startTime', 'endTime', 'status') + \ + '-------------------------------------------------------------------------------------' + +TRIAL_MONITOR_CONTENT = '%-15s %-25s %-25s %-15s' + +TRIAL_MONITOR_TAIL = '-------------------------------------------------------------------------------------\n\n\n' + +INSTALLABLE_PACKAGE_META = { + 'SMAC': { + 'type': 'tuner', + 'class_name': 'nni.smac_tuner.smac_tuner.SMACTuner', + 'code_sub_dir': 'smac_tuner', + 'class_args_validator': 'nni.smac_tuner.smac_tuner.SMACClassArgsValidator' + }, + 'BOHB': { + 'type': 'advisor', + 'class_name': 'nni.bohb_advisor.bohb_advisor.BOHB', + 'code_sub_dir': 'bohb_advisor', + 'class_args_validator': 'nni.bohb_advisor.bohb_advisor.BOHBClassArgsValidator' + }, + 'PPOTuner': { + 'type': 'tuner', + 'class_name': 'nni.ppo_tuner.ppo_tuner.PPOTuner', + 'code_sub_dir': 'ppo_tuner', + 'class_args_validator': 'nni.ppo_tuner.ppo_tuner.PPOClassArgsValidator' + } +} + +TUNERS_SUPPORTING_IMPORT_DATA = { + 'TPE', + 'Anneal', + 'GridSearch', + 'MetisTuner', + 'BOHB', + 'SMAC', + 'BatchTuner' +} + +TUNERS_NO_NEED_TO_IMPORT_DATA = { + 'Random', + 'Hyperband' +} + +SCHEMA_TYPE_ERROR = '%s should be %s type!' +SCHEMA_RANGE_ERROR = '%s should be in range of %s!' +SCHEMA_PATH_ERROR = '%s path not exist!' diff --git a/utils/third_party/nni/tools/nnictl/launcher.py b/utils/third_party/nni/tools/nnictl/launcher.py new file mode 100644 index 0000000000000000000000000000000000000000..fa0aa3baab43ae1d3cf1f66643c2e28a3475e264 --- /dev/null +++ b/utils/third_party/nni/tools/nnictl/launcher.py @@ -0,0 +1,605 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import json +import os +import sys +import string +import random +import time +import tempfile +from subprocess import Popen, check_call, CalledProcessError, PIPE, STDOUT +from nni.tools.annotation import expand_annotations, generate_search_space +from nni.tools.package_utils import get_builtin_module_class_name +import nni_node +from .launcher_utils import validate_all_content +from .rest_utils import rest_put, rest_post, check_rest_server, check_response +from .url_utils import cluster_metadata_url, experiment_url, get_local_urls +from .config_utils import Config, Experiments +from .common_utils import get_yml_content, get_json_content, print_error, print_normal, \ + detect_port, get_user + +from .constants import NNICTL_HOME_DIR, ERROR_INFO, REST_TIME_OUT, EXPERIMENT_SUCCESS_INFO, LOG_HEADER, INSTALLABLE_PACKAGE_META +from .command_utils import check_output_command, kill_command +from .nnictl_utils import update_experiment + +def get_log_path(config_file_name): + '''generate stdout and stderr log path''' + stdout_full_path = os.path.join(NNICTL_HOME_DIR, config_file_name, 'stdout') + stderr_full_path = os.path.join(NNICTL_HOME_DIR, config_file_name, 'stderr') + return stdout_full_path, stderr_full_path + +def print_log_content(config_file_name): + '''print log information''' + stdout_full_path, stderr_full_path = get_log_path(config_file_name) + print_normal(' Stdout:') + print(check_output_command(stdout_full_path)) + print('\n\n') + print_normal(' Stderr:') + print(check_output_command(stderr_full_path)) + +def start_rest_server(port, platform, mode, config_file_name, foreground=False, experiment_id=None, log_dir=None, log_level=None): + '''Run nni manager process''' + if detect_port(port): + print_error('Port %s is used by another process, please reset the port!\n' \ + 'You could use \'nnictl create --help\' to get help information' % port) + exit(1) + + if (platform != 'local') and detect_port(int(port) + 1): + print_error('PAI mode need an additional adjacent port %d, and the port %d is used by another process!\n' \ + 'You could set another port to start experiment!\n' \ + 'You could use \'nnictl create --help\' to get help information' % ((int(port) + 1), (int(port) + 1))) + exit(1) + + print_normal('Starting restful server...') + + entry_dir = nni_node.__path__[0] + if (not entry_dir) or (not os.path.exists(entry_dir)): + print_error('Fail to find nni under python library') + exit(1) + entry_file = os.path.join(entry_dir, 'main.js') + + if sys.platform == 'win32': + node_command = os.path.join(entry_dir, 'node.exe') + else: + node_command = 'node' + cmds = [node_command, '--max-old-space-size=4096', entry_file, '--port', str(port), '--mode', platform] + if mode == 'view': + cmds += ['--start_mode', 'resume'] + cmds += ['--readonly', 'true'] + else: + cmds += ['--start_mode', mode] + if log_dir is not None: + cmds += ['--log_dir', log_dir] + if log_level is not None: + cmds += ['--log_level', log_level] + if mode in ['resume', 'view']: + cmds += ['--experiment_id', experiment_id] + if foreground: + cmds += ['--foreground', 'true'] + stdout_full_path, stderr_full_path = get_log_path(config_file_name) + with open(stdout_full_path, 'a+') as stdout_file, open(stderr_full_path, 'a+') as stderr_file: + time_now = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())) + #add time information in the header of log files + log_header = LOG_HEADER % str(time_now) + stdout_file.write(log_header) + stderr_file.write(log_header) + if sys.platform == 'win32': + from subprocess import CREATE_NEW_PROCESS_GROUP + if foreground: + process = Popen(cmds, cwd=entry_dir, stdout=PIPE, stderr=STDOUT, creationflags=CREATE_NEW_PROCESS_GROUP) + else: + process = Popen(cmds, cwd=entry_dir, stdout=stdout_file, stderr=stderr_file, creationflags=CREATE_NEW_PROCESS_GROUP) + else: + if foreground: + process = Popen(cmds, cwd=entry_dir, stdout=PIPE, stderr=PIPE) + else: + process = Popen(cmds, cwd=entry_dir, stdout=stdout_file, stderr=stderr_file) + return process, str(time_now) + +def set_trial_config(experiment_config, port, config_file_name): + '''set trial configuration''' + request_data = dict() + request_data['trial_config'] = experiment_config['trial'] + response = rest_put(cluster_metadata_url(port), json.dumps(request_data), REST_TIME_OUT) + if check_response(response): + return True + else: + print('Error message is {}'.format(response.text)) + _, stderr_full_path = get_log_path(config_file_name) + if response: + with open(stderr_full_path, 'a+') as fout: + fout.write(json.dumps(json.loads(response.text), indent=4, sort_keys=True, separators=(',', ':'))) + return False + +def set_local_config(experiment_config, port, config_file_name): + '''set local configuration''' + request_data = dict() + if experiment_config.get('localConfig'): + request_data['local_config'] = experiment_config['localConfig'] + if request_data['local_config']: + if request_data['local_config'].get('gpuIndices') and isinstance(request_data['local_config'].get('gpuIndices'), int): + request_data['local_config']['gpuIndices'] = str(request_data['local_config'].get('gpuIndices')) + if request_data['local_config'].get('maxTrialNumOnEachGpu'): + request_data['local_config']['maxTrialNumOnEachGpu'] = request_data['local_config'].get('maxTrialNumOnEachGpu') + if request_data['local_config'].get('useActiveGpu'): + request_data['local_config']['useActiveGpu'] = request_data['local_config'].get('useActiveGpu') + response = rest_put(cluster_metadata_url(port), json.dumps(request_data), REST_TIME_OUT) + err_message = '' + if not response or not check_response(response): + if response is not None: + err_message = response.text + _, stderr_full_path = get_log_path(config_file_name) + with open(stderr_full_path, 'a+') as fout: + fout.write(json.dumps(json.loads(err_message), indent=4, sort_keys=True, separators=(',', ':'))) + return False, err_message + + return set_trial_config(experiment_config, port, config_file_name), None + +def set_remote_config(experiment_config, port, config_file_name): + '''Call setClusterMetadata to pass trial''' + #set machine_list + request_data = dict() + if experiment_config.get('remoteConfig'): + request_data['remote_config'] = experiment_config['remoteConfig'] + else: + request_data['remote_config'] = {'reuse': False} + request_data['machine_list'] = experiment_config['machineList'] + if request_data['machine_list']: + for i in range(len(request_data['machine_list'])): + if isinstance(request_data['machine_list'][i].get('gpuIndices'), int): + request_data['machine_list'][i]['gpuIndices'] = str(request_data['machine_list'][i].get('gpuIndices')) + # It needs to connect all remote machines, the time out of connection is 30 seconds. + # So timeout of this place should be longer. + response = rest_put(cluster_metadata_url(port), json.dumps(request_data), 60, True) + err_message = '' + if not response or not check_response(response): + if response is not None: + err_message = response.text + _, stderr_full_path = get_log_path(config_file_name) + with open(stderr_full_path, 'a+') as fout: + fout.write(json.dumps(json.loads(err_message), indent=4, sort_keys=True, separators=(',', ':'))) + return False, err_message + result, message = setNNIManagerIp(experiment_config, port, config_file_name) + if not result: + return result, message + #set trial_config + return set_trial_config(experiment_config, port, config_file_name), err_message + +def setNNIManagerIp(experiment_config, port, config_file_name): + '''set nniManagerIp''' + if experiment_config.get('nniManagerIp') is None: + return True, None + ip_config_dict = dict() + ip_config_dict['nni_manager_ip'] = {'nniManagerIp': experiment_config['nniManagerIp']} + response = rest_put(cluster_metadata_url(port), json.dumps(ip_config_dict), REST_TIME_OUT) + err_message = None + if not response or not response.status_code == 200: + if response is not None: + err_message = response.text + _, stderr_full_path = get_log_path(config_file_name) + with open(stderr_full_path, 'a+') as fout: + fout.write(json.dumps(json.loads(err_message), indent=4, sort_keys=True, separators=(',', ':'))) + return False, err_message + return True, None + +def set_pai_config(experiment_config, port, config_file_name): + '''set pai configuration''' + pai_config_data = dict() + pai_config_data['pai_config'] = experiment_config['paiConfig'] + response = rest_put(cluster_metadata_url(port), json.dumps(pai_config_data), REST_TIME_OUT) + err_message = None + if not response or not response.status_code == 200: + if response is not None: + err_message = response.text + _, stderr_full_path = get_log_path(config_file_name) + with open(stderr_full_path, 'a+') as fout: + fout.write(json.dumps(json.loads(err_message), indent=4, sort_keys=True, separators=(',', ':'))) + return False, err_message + result, message = setNNIManagerIp(experiment_config, port, config_file_name) + if not result: + return result, message + #set trial_config + return set_trial_config(experiment_config, port, config_file_name), err_message + +def set_pai_yarn_config(experiment_config, port, config_file_name): + '''set paiYarn configuration''' + pai_yarn_config_data = dict() + pai_yarn_config_data['pai_yarn_config'] = experiment_config['paiYarnConfig'] + response = rest_put(cluster_metadata_url(port), json.dumps(pai_yarn_config_data), REST_TIME_OUT) + err_message = None + if not response or not response.status_code == 200: + if response is not None: + err_message = response.text + _, stderr_full_path = get_log_path(config_file_name) + with open(stderr_full_path, 'a+') as fout: + fout.write(json.dumps(json.loads(err_message), indent=4, sort_keys=True, separators=(',', ':'))) + return False, err_message + result, message = setNNIManagerIp(experiment_config, port, config_file_name) + if not result: + return result, message + #set trial_config + return set_trial_config(experiment_config, port, config_file_name), err_message + +def set_kubeflow_config(experiment_config, port, config_file_name): + '''set kubeflow configuration''' + kubeflow_config_data = dict() + kubeflow_config_data['kubeflow_config'] = experiment_config['kubeflowConfig'] + response = rest_put(cluster_metadata_url(port), json.dumps(kubeflow_config_data), REST_TIME_OUT) + err_message = None + if not response or not response.status_code == 200: + if response is not None: + err_message = response.text + _, stderr_full_path = get_log_path(config_file_name) + with open(stderr_full_path, 'a+') as fout: + fout.write(json.dumps(json.loads(err_message), indent=4, sort_keys=True, separators=(',', ':'))) + return False, err_message + result, message = setNNIManagerIp(experiment_config, port, config_file_name) + if not result: + return result, message + #set trial_config + return set_trial_config(experiment_config, port, config_file_name), err_message + +def set_frameworkcontroller_config(experiment_config, port, config_file_name): + '''set kubeflow configuration''' + frameworkcontroller_config_data = dict() + frameworkcontroller_config_data['frameworkcontroller_config'] = experiment_config['frameworkcontrollerConfig'] + response = rest_put(cluster_metadata_url(port), json.dumps(frameworkcontroller_config_data), REST_TIME_OUT) + err_message = None + if not response or not response.status_code == 200: + if response is not None: + err_message = response.text + _, stderr_full_path = get_log_path(config_file_name) + with open(stderr_full_path, 'a+') as fout: + fout.write(json.dumps(json.loads(err_message), indent=4, sort_keys=True, separators=(',', ':'))) + return False, err_message + result, message = setNNIManagerIp(experiment_config, port, config_file_name) + if not result: + return result, message + #set trial_config + return set_trial_config(experiment_config, port, config_file_name), err_message + +def set_dlts_config(experiment_config, port, config_file_name): + '''set dlts configuration''' + dlts_config_data = dict() + dlts_config_data['dlts_config'] = experiment_config['dltsConfig'] + response = rest_put(cluster_metadata_url(port), json.dumps(dlts_config_data), REST_TIME_OUT) + err_message = None + if not response or not response.status_code == 200: + if response is not None: + err_message = response.text + _, stderr_full_path = get_log_path(config_file_name) + with open(stderr_full_path, 'a+') as fout: + fout.write(json.dumps(json.loads(err_message), indent=4, sort_keys=True, separators=(',', ':'))) + return False, err_message + result, message = setNNIManagerIp(experiment_config, port, config_file_name) + if not result: + return result, message + #set trial_config + return set_trial_config(experiment_config, port, config_file_name), err_message + +def set_aml_config(experiment_config, port, config_file_name): + '''set aml configuration''' + aml_config_data = dict() + aml_config_data['aml_config'] = experiment_config['amlConfig'] + response = rest_put(cluster_metadata_url(port), json.dumps(aml_config_data), REST_TIME_OUT) + err_message = None + if not response or not response.status_code == 200: + if response is not None: + err_message = response.text + _, stderr_full_path = get_log_path(config_file_name) + with open(stderr_full_path, 'a+') as fout: + fout.write(json.dumps(json.loads(err_message), indent=4, sort_keys=True, separators=(',', ':'))) + return False, err_message + result, message = setNNIManagerIp(experiment_config, port, config_file_name) + if not result: + return result, message + #set trial_config + return set_trial_config(experiment_config, port, config_file_name), err_message + +def set_experiment(experiment_config, mode, port, config_file_name): + '''Call startExperiment (rest POST /experiment) with yaml file content''' + request_data = dict() + request_data['authorName'] = experiment_config['authorName'] + request_data['experimentName'] = experiment_config['experimentName'] + request_data['trialConcurrency'] = experiment_config['trialConcurrency'] + request_data['maxExecDuration'] = experiment_config['maxExecDuration'] + request_data['maxTrialNum'] = experiment_config['maxTrialNum'] + request_data['searchSpace'] = experiment_config.get('searchSpace') + request_data['trainingServicePlatform'] = experiment_config.get('trainingServicePlatform') + if experiment_config.get('description'): + request_data['description'] = experiment_config['description'] + if experiment_config.get('multiPhase'): + request_data['multiPhase'] = experiment_config.get('multiPhase') + if experiment_config.get('multiThread'): + request_data['multiThread'] = experiment_config.get('multiThread') + if experiment_config.get('advisor'): + request_data['advisor'] = experiment_config['advisor'] + if request_data['advisor'].get('gpuNum'): + print_error('gpuNum is deprecated, please use gpuIndices instead.') + if request_data['advisor'].get('gpuIndices') and isinstance(request_data['advisor'].get('gpuIndices'), int): + request_data['advisor']['gpuIndices'] = str(request_data['advisor'].get('gpuIndices')) + else: + request_data['tuner'] = experiment_config['tuner'] + if request_data['tuner'].get('gpuNum'): + print_error('gpuNum is deprecated, please use gpuIndices instead.') + if request_data['tuner'].get('gpuIndices') and isinstance(request_data['tuner'].get('gpuIndices'), int): + request_data['tuner']['gpuIndices'] = str(request_data['tuner'].get('gpuIndices')) + if 'assessor' in experiment_config: + request_data['assessor'] = experiment_config['assessor'] + if request_data['assessor'].get('gpuNum'): + print_error('gpuNum is deprecated, please remove it from your config file.') + #debug mode should disable version check + if experiment_config.get('debug') is not None: + request_data['versionCheck'] = not experiment_config.get('debug') + #validate version check + if experiment_config.get('versionCheck') is not None: + request_data['versionCheck'] = experiment_config.get('versionCheck') + if experiment_config.get('logCollection'): + request_data['logCollection'] = experiment_config.get('logCollection') + request_data['clusterMetaData'] = [] + if experiment_config['trainingServicePlatform'] == 'local': + request_data['clusterMetaData'].append( + {'key':'codeDir', 'value':experiment_config['trial']['codeDir']}) + request_data['clusterMetaData'].append( + {'key': 'command', 'value': experiment_config['trial']['command']}) + elif experiment_config['trainingServicePlatform'] == 'remote': + request_data['clusterMetaData'].append( + {'key': 'machine_list', 'value': experiment_config['machineList']}) + request_data['clusterMetaData'].append( + {'key': 'trial_config', 'value': experiment_config['trial']}) + if not experiment_config.get('remoteConfig'): + # set default value of reuse in remoteConfig to False + experiment_config['remoteConfig'] = {'reuse': False} + request_data['clusterMetaData'].append( + {'key': 'remote_config', 'value': experiment_config['remoteConfig']}) + elif experiment_config['trainingServicePlatform'] == 'pai': + request_data['clusterMetaData'].append( + {'key': 'pai_config', 'value': experiment_config['paiConfig']}) + request_data['clusterMetaData'].append( + {'key': 'trial_config', 'value': experiment_config['trial']}) + elif experiment_config['trainingServicePlatform'] == 'paiYarn': + request_data['clusterMetaData'].append( + {'key': 'pai_yarn_config', 'value': experiment_config['paiYarnConfig']}) + request_data['clusterMetaData'].append( + {'key': 'trial_config', 'value': experiment_config['trial']}) + elif experiment_config['trainingServicePlatform'] == 'kubeflow': + request_data['clusterMetaData'].append( + {'key': 'kubeflow_config', 'value': experiment_config['kubeflowConfig']}) + request_data['clusterMetaData'].append( + {'key': 'trial_config', 'value': experiment_config['trial']}) + elif experiment_config['trainingServicePlatform'] == 'frameworkcontroller': + request_data['clusterMetaData'].append( + {'key': 'frameworkcontroller_config', 'value': experiment_config['frameworkcontrollerConfig']}) + request_data['clusterMetaData'].append( + {'key': 'trial_config', 'value': experiment_config['trial']}) + elif experiment_config['trainingServicePlatform'] == 'aml': + request_data['clusterMetaData'].append( + {'key': 'aml_config', 'value': experiment_config['amlConfig']}) + request_data['clusterMetaData'].append( + {'key': 'trial_config', 'value': experiment_config['trial']}) + response = rest_post(experiment_url(port), json.dumps(request_data), REST_TIME_OUT, show_error=True) + if check_response(response): + return response + else: + _, stderr_full_path = get_log_path(config_file_name) + if response is not None: + with open(stderr_full_path, 'a+') as fout: + fout.write(json.dumps(json.loads(response.text), indent=4, sort_keys=True, separators=(',', ':'))) + print_error('Setting experiment error, error message is {}'.format(response.text)) + return None + +def set_platform_config(platform, experiment_config, port, config_file_name, rest_process): + '''call set_cluster_metadata for specific platform''' + print_normal('Setting {0} config...'.format(platform)) + config_result, err_msg = None, None + if platform == 'local': + config_result, err_msg = set_local_config(experiment_config, port, config_file_name) + elif platform == 'remote': + config_result, err_msg = set_remote_config(experiment_config, port, config_file_name) + elif platform == 'pai': + config_result, err_msg = set_pai_config(experiment_config, port, config_file_name) + elif platform == 'paiYarn': + config_result, err_msg = set_pai_yarn_config(experiment_config, port, config_file_name) + elif platform == 'kubeflow': + config_result, err_msg = set_kubeflow_config(experiment_config, port, config_file_name) + elif platform == 'frameworkcontroller': + config_result, err_msg = set_frameworkcontroller_config(experiment_config, port, config_file_name) + elif platform == 'dlts': + config_result, err_msg = set_dlts_config(experiment_config, port, config_file_name) + elif platform == 'aml': + config_result, err_msg = set_aml_config(experiment_config, port, config_file_name) + else: + raise Exception(ERROR_INFO % 'Unsupported platform!') + exit(1) + if config_result: + print_normal('Successfully set {0} config!'.format(platform)) + else: + print_error('Failed! Error is: {}'.format(err_msg)) + try: + kill_command(rest_process.pid) + except Exception: + raise Exception(ERROR_INFO % 'Rest server stopped!') + exit(1) + +def launch_experiment(args, experiment_config, mode, config_file_name, experiment_id=None): + '''follow steps to start rest server and start experiment''' + nni_config = Config(config_file_name) + # check packages for tuner + package_name, module_name = None, None + if experiment_config.get('tuner') and experiment_config['tuner'].get('builtinTunerName'): + package_name = experiment_config['tuner']['builtinTunerName'] + module_name, _ = get_builtin_module_class_name('tuners', package_name) + elif experiment_config.get('advisor') and experiment_config['advisor'].get('builtinAdvisorName'): + package_name = experiment_config['advisor']['builtinAdvisorName'] + module_name, _ = get_builtin_module_class_name('advisors', package_name) + if package_name and module_name: + try: + stdout_full_path, stderr_full_path = get_log_path(config_file_name) + with open(stdout_full_path, 'a+') as stdout_file, open(stderr_full_path, 'a+') as stderr_file: + check_call([sys.executable, '-c', 'import %s'%(module_name)], stdout=stdout_file, stderr=stderr_file) + except CalledProcessError: + print_error('some errors happen when import package %s.' %(package_name)) + print_log_content(config_file_name) + if package_name in INSTALLABLE_PACKAGE_META: + print_error('If %s is not installed, it should be installed through '\ + '\'nnictl package install --name %s\''%(package_name, package_name)) + exit(1) + log_dir = experiment_config['logDir'] if experiment_config.get('logDir') else None + log_level = experiment_config['logLevel'] if experiment_config.get('logLevel') else None + #view experiment mode do not need debug function, when view an experiment, there will be no new logs created + foreground = False + if mode != 'view': + foreground = args.foreground + if log_level not in ['trace', 'debug'] and (args.debug or experiment_config.get('debug') is True): + log_level = 'debug' + # start rest server + rest_process, start_time = start_rest_server(args.port, experiment_config['trainingServicePlatform'], \ + mode, config_file_name, foreground, experiment_id, log_dir, log_level) + nni_config.set_config('restServerPid', rest_process.pid) + # Deal with annotation + if experiment_config.get('useAnnotation'): + path = os.path.join(tempfile.gettempdir(), get_user(), 'nni', 'annotation') + if not os.path.isdir(path): + os.makedirs(path) + path = tempfile.mkdtemp(dir=path) + nas_mode = experiment_config['trial'].get('nasMode', 'classic_mode') + code_dir = expand_annotations(experiment_config['trial']['codeDir'], path, nas_mode=nas_mode) + experiment_config['trial']['codeDir'] = code_dir + search_space = generate_search_space(code_dir) + experiment_config['searchSpace'] = json.dumps(search_space) + assert search_space, ERROR_INFO % 'Generated search space is empty' + elif experiment_config.get('searchSpacePath'): + search_space = get_json_content(experiment_config.get('searchSpacePath')) + experiment_config['searchSpace'] = json.dumps(search_space) + else: + experiment_config['searchSpace'] = json.dumps('') + + # check rest server + running, _ = check_rest_server(args.port) + if running: + print_normal('Successfully started Restful server!') + else: + print_error('Restful server start failed!') + print_log_content(config_file_name) + try: + kill_command(rest_process.pid) + except Exception: + raise Exception(ERROR_INFO % 'Rest server stopped!') + exit(1) + if mode != 'view': + # set platform configuration + set_platform_config(experiment_config['trainingServicePlatform'], experiment_config, args.port,\ + config_file_name, rest_process) + + # start a new experiment + print_normal('Starting experiment...') + # set debug configuration + if mode != 'view' and experiment_config.get('debug') is None: + experiment_config['debug'] = args.debug + response = set_experiment(experiment_config, mode, args.port, config_file_name) + if response: + if experiment_id is None: + experiment_id = json.loads(response.text).get('experiment_id') + nni_config.set_config('experimentId', experiment_id) + else: + print_error('Start experiment failed!') + print_log_content(config_file_name) + try: + kill_command(rest_process.pid) + except Exception: + raise Exception(ERROR_INFO % 'Restful server stopped!') + exit(1) + if experiment_config.get('nniManagerIp'): + web_ui_url_list = ['{0}:{1}'.format(experiment_config['nniManagerIp'], str(args.port))] + else: + web_ui_url_list = get_local_urls(args.port) + nni_config.set_config('webuiUrl', web_ui_url_list) + + # save experiment information + nnictl_experiment_config = Experiments() + nnictl_experiment_config.add_experiment(experiment_id, args.port, start_time, config_file_name, + experiment_config['trainingServicePlatform'], + experiment_config['experimentName']) + + print_normal(EXPERIMENT_SUCCESS_INFO % (experiment_id, ' '.join(web_ui_url_list))) + if mode != 'view' and args.foreground: + try: + while True: + log_content = rest_process.stdout.readline().strip().decode('utf-8') + print(log_content) + except KeyboardInterrupt: + kill_command(rest_process.pid) + print_normal('Stopping experiment...') + +def create_experiment(args): + '''start a new experiment''' + config_file_name = ''.join(random.sample(string.ascii_letters + string.digits, 8)) + nni_config = Config(config_file_name) + config_path = os.path.abspath(args.config) + if not os.path.exists(config_path): + print_error('Please set correct config path!') + exit(1) + experiment_config = get_yml_content(config_path) + try: + validate_all_content(experiment_config, config_path) + except Exception as e: + print_error(e) + exit(1) + + nni_config.set_config('experimentConfig', experiment_config) + nni_config.set_config('restServerPort', args.port) + try: + launch_experiment(args, experiment_config, 'new', config_file_name) + except Exception as exception: + nni_config = Config(config_file_name) + restServerPid = nni_config.get_config('restServerPid') + if restServerPid: + kill_command(restServerPid) + print_error(exception) + exit(1) + +def manage_stopped_experiment(args, mode): + '''view a stopped experiment''' + update_experiment() + experiment_config = Experiments() + experiment_dict = experiment_config.get_all_experiments() + experiment_id = None + #find the latest stopped experiment + if not args.id: + print_error('Please set experiment id! \nYou could use \'nnictl {0} id\' to {0} a stopped experiment!\n' \ + 'You could use \'nnictl experiment list --all\' to show all experiments!'.format(mode)) + exit(1) + else: + if experiment_dict.get(args.id) is None: + print_error('Id %s not exist!' % args.id) + exit(1) + if experiment_dict[args.id]['status'] != 'STOPPED': + print_error('Only stopped experiments can be {0}ed!'.format(mode)) + exit(1) + experiment_id = args.id + print_normal('{0} experiment {1}...'.format(mode, experiment_id)) + nni_config = Config(experiment_dict[experiment_id]['fileName']) + experiment_config = nni_config.get_config('experimentConfig') + experiment_id = nni_config.get_config('experimentId') + new_config_file_name = ''.join(random.sample(string.ascii_letters + string.digits, 8)) + new_nni_config = Config(new_config_file_name) + new_nni_config.set_config('experimentConfig', experiment_config) + new_nni_config.set_config('restServerPort', args.port) + try: + launch_experiment(args, experiment_config, mode, new_config_file_name, experiment_id) + except Exception as exception: + nni_config = Config(new_config_file_name) + restServerPid = nni_config.get_config('restServerPid') + if restServerPid: + kill_command(restServerPid) + print_error(exception) + exit(1) + +def view_experiment(args): + '''view a stopped experiment''' + manage_stopped_experiment(args, 'view') + +def resume_experiment(args): + '''resume an experiment''' + manage_stopped_experiment(args, 'resume') diff --git a/utils/third_party/nni/tools/nnictl/launcher_utils.py b/utils/third_party/nni/tools/nnictl/launcher_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..48dd2779a36ee27e4c1845587b216e32760a33ad --- /dev/null +++ b/utils/third_party/nni/tools/nnictl/launcher_utils.py @@ -0,0 +1,114 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +from schema import SchemaError +from .config_schema import NNIConfigSchema +from .common_utils import print_normal + +def expand_path(experiment_config, key): + '''Change '~' to user home directory''' + if experiment_config.get(key): + experiment_config[key] = os.path.expanduser(experiment_config[key]) + +def parse_relative_path(root_path, experiment_config, key): + '''Change relative path to absolute path''' + if experiment_config.get(key) and not os.path.isabs(experiment_config.get(key)): + absolute_path = os.path.join(root_path, experiment_config.get(key)) + print_normal('expand %s: %s to %s ' % (key, experiment_config[key], absolute_path)) + experiment_config[key] = absolute_path + +def parse_time(time): + '''Change the time to seconds''' + unit = time[-1] + if unit not in ['s', 'm', 'h', 'd']: + raise SchemaError('the unit of time could only from {s, m, h, d}') + time = time[:-1] + if not time.isdigit(): + raise SchemaError('time format error!') + parse_dict = {'s':1, 'm':60, 'h':3600, 'd':86400} + return int(time) * parse_dict[unit] + +def parse_path(experiment_config, config_path): + '''Parse path in config file''' + expand_path(experiment_config, 'searchSpacePath') + if experiment_config.get('trial'): + expand_path(experiment_config['trial'], 'codeDir') + if experiment_config['trial'].get('authFile'): + expand_path(experiment_config['trial'], 'authFile') + if experiment_config['trial'].get('ps'): + if experiment_config['trial']['ps'].get('privateRegistryAuthPath'): + expand_path(experiment_config['trial']['ps'], 'privateRegistryAuthPath') + if experiment_config['trial'].get('master'): + if experiment_config['trial']['master'].get('privateRegistryAuthPath'): + expand_path(experiment_config['trial']['master'], 'privateRegistryAuthPath') + if experiment_config['trial'].get('worker'): + if experiment_config['trial']['worker'].get('privateRegistryAuthPath'): + expand_path(experiment_config['trial']['worker'], 'privateRegistryAuthPath') + if experiment_config['trial'].get('taskRoles'): + for index in range(len(experiment_config['trial']['taskRoles'])): + if experiment_config['trial']['taskRoles'][index].get('privateRegistryAuthPath'): + expand_path(experiment_config['trial']['taskRoles'][index], 'privateRegistryAuthPath') + if experiment_config.get('tuner'): + expand_path(experiment_config['tuner'], 'codeDir') + if experiment_config.get('assessor'): + expand_path(experiment_config['assessor'], 'codeDir') + if experiment_config.get('advisor'): + expand_path(experiment_config['advisor'], 'codeDir') + if experiment_config.get('machineList'): + for index in range(len(experiment_config['machineList'])): + expand_path(experiment_config['machineList'][index], 'sshKeyPath') + if experiment_config['trial'].get('paiConfigPath'): + expand_path(experiment_config['trial'], 'paiConfigPath') + + #if users use relative path, convert it to absolute path + root_path = os.path.dirname(config_path) + if experiment_config.get('searchSpacePath'): + parse_relative_path(root_path, experiment_config, 'searchSpacePath') + if experiment_config.get('trial'): + parse_relative_path(root_path, experiment_config['trial'], 'codeDir') + if experiment_config['trial'].get('authFile'): + parse_relative_path(root_path, experiment_config['trial'], 'authFile') + if experiment_config['trial'].get('ps'): + if experiment_config['trial']['ps'].get('privateRegistryAuthPath'): + parse_relative_path(root_path, experiment_config['trial']['ps'], 'privateRegistryAuthPath') + if experiment_config['trial'].get('master'): + if experiment_config['trial']['master'].get('privateRegistryAuthPath'): + parse_relative_path(root_path, experiment_config['trial']['master'], 'privateRegistryAuthPath') + if experiment_config['trial'].get('worker'): + if experiment_config['trial']['worker'].get('privateRegistryAuthPath'): + parse_relative_path(root_path, experiment_config['trial']['worker'], 'privateRegistryAuthPath') + if experiment_config['trial'].get('taskRoles'): + for index in range(len(experiment_config['trial']['taskRoles'])): + if experiment_config['trial']['taskRoles'][index].get('privateRegistryAuthPath'): + parse_relative_path(root_path, experiment_config['trial']['taskRoles'][index], 'privateRegistryAuthPath') + if experiment_config.get('tuner'): + parse_relative_path(root_path, experiment_config['tuner'], 'codeDir') + if experiment_config.get('assessor'): + parse_relative_path(root_path, experiment_config['assessor'], 'codeDir') + if experiment_config.get('advisor'): + parse_relative_path(root_path, experiment_config['advisor'], 'codeDir') + if experiment_config.get('machineList'): + for index in range(len(experiment_config['machineList'])): + parse_relative_path(root_path, experiment_config['machineList'][index], 'sshKeyPath') + if experiment_config['trial'].get('paiConfigPath'): + parse_relative_path(root_path, experiment_config['trial'], 'paiConfigPath') + +def set_default_values(experiment_config): + if experiment_config.get('maxExecDuration') is None: + experiment_config['maxExecDuration'] = '999d' + if experiment_config.get('maxTrialNum') is None: + experiment_config['maxTrialNum'] = 99999 + if experiment_config['trainingServicePlatform'] == 'remote': + for index in range(len(experiment_config['machineList'])): + if experiment_config['machineList'][index].get('port') is None: + experiment_config['machineList'][index]['port'] = 22 + +def validate_all_content(experiment_config, config_path): + '''Validate whether experiment_config is valid''' + parse_path(experiment_config, config_path) + set_default_values(experiment_config) + + NNIConfigSchema().validate(experiment_config) + + experiment_config['maxExecDuration'] = parse_time(experiment_config['maxExecDuration']) diff --git a/utils/third_party/nni/tools/nnictl/nnictl.py b/utils/third_party/nni/tools/nnictl/nnictl.py new file mode 100644 index 0000000000000000000000000000000000000000..cc5d4cdd11675f9336877d34edf1d71796c3f2e8 --- /dev/null +++ b/utils/third_party/nni/tools/nnictl/nnictl.py @@ -0,0 +1,258 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import argparse +import os +import pkg_resources +from colorama import init +from .common_utils import print_error +from .launcher import create_experiment, resume_experiment, view_experiment +from .updater import update_searchspace, update_concurrency, update_duration, update_trialnum, import_data +from .nnictl_utils import stop_experiment, trial_ls, trial_kill, list_experiment, experiment_status,\ + log_trial, experiment_clean, platform_clean, experiment_list, \ + monitor_experiment, export_trials_data, trial_codegen, webui_url, \ + get_config, log_stdout, log_stderr, search_space_auto_gen, webui_nas, \ + save_experiment, load_experiment +from .package_management import package_install, package_uninstall, package_show, package_list +from .constants import DEFAULT_REST_PORT +from .tensorboard_utils import start_tensorboard, stop_tensorboard +init(autoreset=True) + +if os.environ.get('COVERAGE_PROCESS_START'): + import coverage + coverage.process_startup() + +def nni_info(*args): + if args[0].version: + try: + print(pkg_resources.get_distribution('nni').version) + except pkg_resources.ResolutionError: + print_error('Get version failed, please use `pip3 list | grep nni` to check nni version!') + else: + print('please run "nnictl {positional argument} --help" to see nnictl guidance') + +def parse_args(): + '''Definite the arguments users need to follow and input''' + parser = argparse.ArgumentParser(prog='nnictl', description='use nnictl command to control nni experiments') + parser.add_argument('--version', '-v', action='store_true') + parser.set_defaults(func=nni_info) + + # create subparsers for args with sub values + subparsers = parser.add_subparsers() + + # parse the command of auto generating search space + parser_start = subparsers.add_parser('ss_gen', help='automatically generate search space file from trial code') + parser_start.add_argument('--trial_command', '-t', required=True, dest='trial_command', help='the command for running trial code') + parser_start.add_argument('--trial_dir', '-d', default='./', dest='trial_dir', help='the directory for running the command') + parser_start.add_argument('--file', '-f', default='nni_auto_gen_search_space.json', dest='file', help='the path of search space file') + parser_start.set_defaults(func=search_space_auto_gen) + + # parse start command + parser_start = subparsers.add_parser('create', help='create a new experiment') + parser_start.add_argument('--config', '-c', required=True, dest='config', help='the path of yaml config file') + parser_start.add_argument('--port', '-p', default=DEFAULT_REST_PORT, dest='port', help='the port of restful server') + parser_start.add_argument('--debug', '-d', action='store_true', help=' set debug mode') + parser_start.add_argument('--foreground', '-f', action='store_true', help=' set foreground mode, print log content to terminal') + parser_start.set_defaults(func=create_experiment) + + # parse resume command + parser_resume = subparsers.add_parser('resume', help='resume a new experiment') + parser_resume.add_argument('id', nargs='?', help='The id of the experiment you want to resume') + parser_resume.add_argument('--port', '-p', default=DEFAULT_REST_PORT, dest='port', help='the port of restful server') + parser_resume.add_argument('--debug', '-d', action='store_true', help=' set debug mode') + parser_resume.add_argument('--foreground', '-f', action='store_true', help=' set foreground mode, print log content to terminal') + parser_resume.set_defaults(func=resume_experiment) + + # parse view command + parser_view = subparsers.add_parser('view', help='view a stopped experiment') + parser_view.add_argument('id', nargs='?', help='The id of the experiment you want to view') + parser_view.add_argument('--port', '-p', default=DEFAULT_REST_PORT, dest='port', help='the port of restful server') + parser_view.set_defaults(func=view_experiment) + + # parse update command + parser_updater = subparsers.add_parser('update', help='update the experiment') + #add subparsers for parser_updater + parser_updater_subparsers = parser_updater.add_subparsers() + parser_updater_searchspace = parser_updater_subparsers.add_parser('searchspace', help='update searchspace') + parser_updater_searchspace.add_argument('id', nargs='?', help='the id of experiment') + parser_updater_searchspace.add_argument('--filename', '-f', required=True) + parser_updater_searchspace.set_defaults(func=update_searchspace) + parser_updater_concurrency = parser_updater_subparsers.add_parser('concurrency', help='update concurrency') + parser_updater_concurrency.add_argument('id', nargs='?', help='the id of experiment') + parser_updater_concurrency.add_argument('--value', '-v', required=True) + parser_updater_concurrency.set_defaults(func=update_concurrency) + parser_updater_duration = parser_updater_subparsers.add_parser('duration', help='update duration') + parser_updater_duration.add_argument('id', nargs='?', help='the id of experiment') + parser_updater_duration.add_argument('--value', '-v', required=True, help='the unit of time should in {\'s\', \'m\', \'h\', \'d\'}') + parser_updater_duration.set_defaults(func=update_duration) + parser_updater_trialnum = parser_updater_subparsers.add_parser('trialnum', help='update maxtrialnum') + parser_updater_trialnum.add_argument('id', nargs='?', help='the id of experiment') + parser_updater_trialnum.add_argument('--value', '-v', required=True) + parser_updater_trialnum.set_defaults(func=update_trialnum) + + #parse stop command + parser_stop = subparsers.add_parser('stop', help='stop the experiment') + parser_stop.add_argument('id', nargs='?', help='the id of experiment, use \'all\' to stop all running experiments') + parser_stop.add_argument('--port', '-p', dest='port', help='the port of restful server') + parser_stop.add_argument('--all', '-a', action='store_true', help='stop all of experiments') + parser_stop.set_defaults(func=stop_experiment) + + #parse trial command + parser_trial = subparsers.add_parser('trial', help='get trial information') + #add subparsers for parser_trial + parser_trial_subparsers = parser_trial.add_subparsers() + parser_trial_ls = parser_trial_subparsers.add_parser('ls', help='list trial jobs') + parser_trial_ls.add_argument('id', nargs='?', help='the id of experiment') + parser_trial_ls.add_argument('--head', type=int, help='list the highest experiments on the default metric') + parser_trial_ls.add_argument('--tail', type=int, help='list the lowest experiments on the default metric') + parser_trial_ls.set_defaults(func=trial_ls) + parser_trial_kill = parser_trial_subparsers.add_parser('kill', help='kill trial jobs') + parser_trial_kill.add_argument('id', nargs='?', help='the id of experiment') + parser_trial_kill.add_argument('--trial_id', '-T', required=True, dest='trial_id', help='the id of trial to be killed') + parser_trial_kill.set_defaults(func=trial_kill) + parser_trial_codegen = parser_trial_subparsers.add_parser('codegen', help='generate trial code for a specific trial') + parser_trial_codegen.add_argument('id', nargs='?', help='the id of experiment') + parser_trial_codegen.add_argument('--trial_id', '-T', required=True, dest='trial_id', help='the id of trial to do code generation') + parser_trial_codegen.set_defaults(func=trial_codegen) + + #parse experiment command + parser_experiment = subparsers.add_parser('experiment', help='get experiment information') + #add subparsers for parser_experiment + parser_experiment_subparsers = parser_experiment.add_subparsers() + parser_experiment_show = parser_experiment_subparsers.add_parser('show', help='show the information of experiment') + parser_experiment_show.add_argument('id', nargs='?', help='the id of experiment') + parser_experiment_show.set_defaults(func=list_experiment) + parser_experiment_status = parser_experiment_subparsers.add_parser('status', help='show the status of experiment') + parser_experiment_status.add_argument('id', nargs='?', help='the id of experiment') + parser_experiment_status.set_defaults(func=experiment_status) + parser_experiment_list = parser_experiment_subparsers.add_parser('list', help='list all of running experiment ids') + parser_experiment_list.add_argument('--all', action='store_true', default=False, help='list all of experiments') + parser_experiment_list.set_defaults(func=experiment_list) + parser_experiment_clean = parser_experiment_subparsers.add_parser('delete', help='clean up the experiment data') + parser_experiment_clean.add_argument('id', nargs='?', help='the id of experiment') + parser_experiment_clean.add_argument('--all', action='store_true', default=False, help='delete all of experiments') + parser_experiment_clean.set_defaults(func=experiment_clean) + #import tuning data + parser_import_data = parser_experiment_subparsers.add_parser('import', help='import additional data') + parser_import_data.add_argument('id', nargs='?', help='the id of experiment') + parser_import_data.add_argument('--filename', '-f', required=True) + parser_import_data.set_defaults(func=import_data) + #export trial data + parser_trial_export = parser_experiment_subparsers.add_parser('export', help='export trial job results to csv or json') + parser_trial_export.add_argument('id', nargs='?', help='the id of experiment') + parser_trial_export.add_argument('--type', '-t', choices=['json', 'csv'], required=True, dest='type', help='target file type') + parser_trial_export.add_argument('--filename', '-f', required=True, dest='path', help='target file path') + parser_trial_export.add_argument('--intermediate', '-i', action='store_true', + default=False, help='are intermediate results included') + parser_trial_export.set_defaults(func=export_trials_data) + #save an NNI experiment + parser_save_experiment = parser_experiment_subparsers.add_parser('save', help='save an experiment') + parser_save_experiment.add_argument('id', nargs='?', help='the id of experiment') + parser_save_experiment.add_argument('--path', '-p', required=False, help='the folder path to store nni experiment data, \ + default current working directory') + parser_save_experiment.add_argument('--saveCodeDir', '-s', action='store_true', default=False, help='save codeDir data \ + of the experiment') + parser_save_experiment.set_defaults(func=save_experiment) + #load an NNI experiment + parser_load_experiment = parser_experiment_subparsers.add_parser('load', help='load an experiment') + parser_load_experiment.add_argument('--path', '-p', required=True, help='the path of nni package file') + parser_load_experiment.add_argument('--codeDir', '-c', required=True, help='the path of codeDir for loaded experiment, \ + this path will also put the code in the loaded experiment package') + parser_load_experiment.add_argument('--logDir', '-l', required=False, help='the path of logDir for loaded experiment') + parser_load_experiment.add_argument('--searchSpacePath', '-s', required=False, help='the path of search space file for \ + loaded experiment, this path contains file name. Default in $codeDir/search_space.json') + parser_load_experiment.set_defaults(func=load_experiment) + + #parse platform command + parser_platform = subparsers.add_parser('platform', help='get platform information') + #add subparsers for parser_platform + parser_platform_subparsers = parser_platform.add_subparsers() + parser_platform_clean = parser_platform_subparsers.add_parser('clean', help='clean up the platform data') + parser_platform_clean.add_argument('--config', '-c', required=True, dest='config', help='the path of yaml config file') + parser_platform_clean.set_defaults(func=platform_clean) + + #TODO:finish webui function + #parse board command + parser_webui = subparsers.add_parser('webui', help='get web ui information') + #add subparsers for parser_board + parser_webui_subparsers = parser_webui.add_subparsers() + parser_webui_url = parser_webui_subparsers.add_parser('url', help='show the url of web ui') + parser_webui_url.add_argument('id', nargs='?', help='the id of experiment') + parser_webui_url.set_defaults(func=webui_url) + parser_webui_nas = parser_webui_subparsers.add_parser('nas', help='show nas ui') + parser_webui_nas.add_argument('--port', default=6060, type=int, help='port of nas ui') + parser_webui_nas.add_argument('--logdir', default='.', type=str, help='the logdir where nas ui will read data') + parser_webui_nas.set_defaults(func=webui_nas) + + #parse config command + parser_config = subparsers.add_parser('config', help='get config information') + parser_config_subparsers = parser_config.add_subparsers() + parser_config_show = parser_config_subparsers.add_parser('show', help='show the information of config') + parser_config_show.add_argument('id', nargs='?', help='the id of experiment') + parser_config_show.set_defaults(func=get_config) + + #parse log command + parser_log = subparsers.add_parser('log', help='get log information') + # add subparsers for parser_log + parser_log_subparsers = parser_log.add_subparsers() + parser_log_stdout = parser_log_subparsers.add_parser('stdout', help='get stdout information') + parser_log_stdout.add_argument('id', nargs='?', help='the id of experiment') + parser_log_stdout.add_argument('--tail', '-T', dest='tail', type=int, help='get tail -100 content of stdout') + parser_log_stdout.add_argument('--head', '-H', dest='head', type=int, help='get head -100 content of stdout') + parser_log_stdout.add_argument('--path', action='store_true', default=False, help='get the path of stdout file') + parser_log_stdout.set_defaults(func=log_stdout) + parser_log_stderr = parser_log_subparsers.add_parser('stderr', help='get stderr information') + parser_log_stderr.add_argument('id', nargs='?', help='the id of experiment') + parser_log_stderr.add_argument('--tail', '-T', dest='tail', type=int, help='get tail -100 content of stderr') + parser_log_stderr.add_argument('--head', '-H', dest='head', type=int, help='get head -100 content of stderr') + parser_log_stderr.add_argument('--path', action='store_true', default=False, help='get the path of stderr file') + parser_log_stderr.set_defaults(func=log_stderr) + parser_log_trial = parser_log_subparsers.add_parser('trial', help='get trial log path') + parser_log_trial.add_argument('id', nargs='?', help='the id of experiment') + parser_log_trial.add_argument('--trial_id', '-T', dest='trial_id', help='find trial log path by id') + parser_log_trial.set_defaults(func=log_trial) + + #parse package command + parser_package = subparsers.add_parser('package', help='control nni tuner and assessor packages') + # add subparsers for parser_package + parser_package_subparsers = parser_package.add_subparsers() + parser_package_install = parser_package_subparsers.add_parser('install', help='install packages') + parser_package_install.add_argument('source', nargs='?', help='installation source, can be a directory or whl file') + parser_package_install.add_argument('--name', '-n', dest='name', help='package name to be installed', required=False) + parser_package_install.set_defaults(func=package_install) + + parser_package_uninstall = parser_package_subparsers.add_parser('uninstall', help='uninstall packages') + parser_package_uninstall.add_argument('name', nargs=1, help='package name to be uninstalled') + parser_package_uninstall.set_defaults(func=package_uninstall) + + parser_package_show = parser_package_subparsers.add_parser('show', help='show the information of packages') + parser_package_show.add_argument('name', nargs=1, help='builtin name of the package') + parser_package_show.set_defaults(func=package_show) + + parser_package_list = parser_package_subparsers.add_parser('list', help='list installed packages') + parser_package_list.add_argument('--all', action='store_true', help='list all builtin packages') + parser_package_list.set_defaults(func=package_list) + + #parse tensorboard command + parser_tensorboard = subparsers.add_parser('tensorboard', help='manage tensorboard') + parser_tensorboard_subparsers = parser_tensorboard.add_subparsers() + parser_tensorboard_start = parser_tensorboard_subparsers.add_parser('start', help='start tensorboard') + parser_tensorboard_start.add_argument('id', nargs='?', help='the id of experiment') + parser_tensorboard_start.add_argument('--trial_id', '-T', dest='trial_id', help='the id of trial') + parser_tensorboard_start.add_argument('--port', dest='port', default=6006, help='the port to start tensorboard') + parser_tensorboard_start.set_defaults(func=start_tensorboard) + parser_tensorboard_stop = parser_tensorboard_subparsers.add_parser('stop', help='stop tensorboard') + parser_tensorboard_stop.add_argument('id', nargs='?', help='the id of experiment') + parser_tensorboard_stop.set_defaults(func=stop_tensorboard) + + #parse top command + parser_top = subparsers.add_parser('top', help='monitor the experiment') + parser_top.add_argument('--time', '-t', dest='time', type=int, default=3, help='the time interval to update the experiment status, ' \ + 'the unit is second') + parser_top.set_defaults(func=monitor_experiment) + + args = parser.parse_args() + args.func(args) + +if __name__ == '__main__': + parse_args() diff --git a/utils/third_party/nni/tools/nnictl/nnictl_utils.py b/utils/third_party/nni/tools/nnictl/nnictl_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..3427d698b074ef87837903179d3e8b3b3205962a --- /dev/null +++ b/utils/third_party/nni/tools/nnictl/nnictl_utils.py @@ -0,0 +1,995 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import csv +import os +import sys +import json +import time +import re +import shutil +import subprocess +from functools import cmp_to_key +from datetime import datetime, timezone +from subprocess import Popen +from pyhdfs import HdfsClient +from nni.tools.annotation import expand_annotations +import nni_node +from .rest_utils import rest_get, rest_delete, check_rest_server_quick, check_response +from .url_utils import trial_jobs_url, experiment_url, trial_job_id_url, export_data_url, metric_data_url +from .config_utils import Config, Experiments +from .constants import NNICTL_HOME_DIR, NNI_HOME_DIR, EXPERIMENT_INFORMATION_FORMAT, EXPERIMENT_DETAIL_FORMAT, \ + EXPERIMENT_MONITOR_INFO, TRIAL_MONITOR_HEAD, TRIAL_MONITOR_CONTENT, TRIAL_MONITOR_TAIL, REST_TIME_OUT +from .common_utils import print_normal, print_error, print_warning, detect_process, get_yml_content, generate_temp_dir +from .command_utils import check_output_command, kill_command +from .ssh_utils import create_ssh_sftp_client, remove_remote_directory + +def get_experiment_time(port): + '''get the startTime and endTime of an experiment''' + response = rest_get(experiment_url(port), REST_TIME_OUT) + if response and check_response(response): + content = convert_time_stamp_to_date(json.loads(response.text)) + return content.get('startTime'), content.get('endTime') + return None, None + +def get_experiment_status(port): + '''get the status of an experiment''' + result, response = check_rest_server_quick(port) + if result: + return json.loads(response.text).get('status') + return None + +def update_experiment(): + '''Update the experiment status in config file''' + experiment_config = Experiments() + experiment_dict = experiment_config.get_all_experiments() + if not experiment_dict: + return None + for key in experiment_dict.keys(): + if isinstance(experiment_dict[key], dict): + if experiment_dict[key].get('status') != 'STOPPED': + nni_config = Config(experiment_dict[key]['fileName']) + rest_pid = nni_config.get_config('restServerPid') + if not detect_process(rest_pid): + experiment_config.update_experiment(key, 'status', 'STOPPED') + continue + rest_port = nni_config.get_config('restServerPort') + startTime, endTime = get_experiment_time(rest_port) + if startTime: + experiment_config.update_experiment(key, 'startTime', startTime) + if endTime: + experiment_config.update_experiment(key, 'endTime', endTime) + status = get_experiment_status(rest_port) + if status: + experiment_config.update_experiment(key, 'status', status) + +def check_experiment_id(args, update=True): + '''check if the id is valid + ''' + if update: + update_experiment() + experiment_config = Experiments() + experiment_dict = experiment_config.get_all_experiments() + if not experiment_dict: + print_normal('There is no experiment running...') + return None + if not args.id: + running_experiment_list = [] + for key in experiment_dict.keys(): + if isinstance(experiment_dict[key], dict): + if experiment_dict[key].get('status') != 'STOPPED': + running_experiment_list.append(key) + elif isinstance(experiment_dict[key], list): + # if the config file is old version, remove the configuration from file + experiment_config.remove_experiment(key) + if len(running_experiment_list) > 1: + print_error('There are multiple experiments, please set the experiment id...') + experiment_information = "" + for key in running_experiment_list: + experiment_information += EXPERIMENT_DETAIL_FORMAT % (key, + experiment_dict[key].get('experimentName', 'N/A'), + experiment_dict[key]['status'], + experiment_dict[key]['port'], + experiment_dict[key].get('platform'), + experiment_dict[key]['startTime'], + experiment_dict[key]['endTime']) + print(EXPERIMENT_INFORMATION_FORMAT % experiment_information) + exit(1) + elif not running_experiment_list: + print_error('There is no experiment running.') + return None + else: + return running_experiment_list[0] + if experiment_dict.get(args.id): + return args.id + else: + print_error('Id not correct.') + return None + +def parse_ids(args): + '''Parse the arguments for nnictl stop + 1.If port is provided and id is not specified, return the id who owns the port + 2.If both port and id are provided, return the id if it owns the port, otherwise fail + 3.If there is an id specified, return the corresponding id + 4.If there is no id specified, and there is an experiment running, return the id, or return Error + 5.If the id matches an experiment, nnictl will return the id. + 6.If the id ends with *, nnictl will match all ids matchs the regular + 7.If the id does not exist but match the prefix of an experiment id, nnictl will return the matched id + 8.If the id does not exist but match multiple prefix of the experiment ids, nnictl will give id information + ''' + update_experiment() + experiment_config = Experiments() + experiment_dict = experiment_config.get_all_experiments() + if not experiment_dict: + print_normal('Experiment is not running...') + return None + result_list = [] + running_experiment_list = [] + for key in experiment_dict.keys(): + if isinstance(experiment_dict[key], dict): + if experiment_dict[key].get('status') != 'STOPPED': + running_experiment_list.append(key) + elif isinstance(experiment_dict[key], list): + # if the config file is old version, remove the configuration from file + experiment_config.remove_experiment(key) + if args.all: + return running_experiment_list + if args.port is not None: + for key in running_experiment_list: + if str(experiment_dict[key]['port']) == args.port: + result_list.append(key) + if args.id and result_list and args.id != result_list[0]: + print_error('Experiment id and resful server port not match') + exit(1) + elif not args.id: + if len(running_experiment_list) > 1: + print_error('There are multiple experiments, please set the experiment id...') + experiment_information = "" + for key in running_experiment_list: + experiment_information += EXPERIMENT_DETAIL_FORMAT % (key, + experiment_dict[key].get('experimentName', 'N/A'), + experiment_dict[key]['status'], + experiment_dict[key]['port'], + experiment_dict[key].get('platform'), + experiment_dict[key]['startTime'], + experiment_dict[key]['endTime']) + print(EXPERIMENT_INFORMATION_FORMAT % experiment_information) + exit(1) + else: + result_list = running_experiment_list + elif args.id.endswith('*'): + for expId in running_experiment_list: + if expId.startswith(args.id[:-1]): + result_list.append(expId) + elif args.id in running_experiment_list: + result_list.append(args.id) + else: + for expId in running_experiment_list: + if expId.startswith(args.id): + result_list.append(expId) + if len(result_list) > 1: + print_error(args.id + ' is ambiguous, please choose ' + ' '.join(result_list)) + return None + if not result_list and (args.id or args.port): + print_error('There are no experiments matched, please set correct experiment id or restful server port') + elif not result_list: + print_error('There is no experiment running...') + return result_list + +def get_config_filename(args): + '''get the file name of config file''' + experiment_id = check_experiment_id(args) + if experiment_id is None: + print_error('Please set correct experiment id.') + exit(1) + experiment_config = Experiments() + experiment_dict = experiment_config.get_all_experiments() + return experiment_dict[experiment_id]['fileName'] + +def get_experiment_port(args): + '''get the port of experiment''' + experiment_id = check_experiment_id(args) + if experiment_id is None: + print_error('Please set correct experiment id.') + exit(1) + experiment_config = Experiments() + experiment_dict = experiment_config.get_all_experiments() + return experiment_dict[experiment_id]['port'] + +def convert_time_stamp_to_date(content): + '''Convert time stamp to date time format''' + start_time_stamp = content.get('startTime') + end_time_stamp = content.get('endTime') + if start_time_stamp: + start_time = datetime.fromtimestamp(start_time_stamp // 1000, timezone.utc).astimezone().strftime("%Y/%m/%d %H:%M:%S") + content['startTime'] = str(start_time) + if end_time_stamp: + end_time = datetime.fromtimestamp(end_time_stamp // 1000, timezone.utc).astimezone().strftime("%Y/%m/%d %H:%M:%S") + content['endTime'] = str(end_time) + return content + +def check_rest(args): + '''check if restful server is running''' + nni_config = Config(get_config_filename(args)) + rest_port = nni_config.get_config('restServerPort') + running, _ = check_rest_server_quick(rest_port) + if running: + print_normal('Restful server is running...') + else: + print_normal('Restful server is not running...') + return running + +def stop_experiment(args): + '''Stop the experiment which is running''' + if args.id and args.id == 'all': + print_warning('\'nnictl stop all\' is abolished, please use \'nnictl stop --all\' to stop all of experiments!') + exit(1) + experiment_id_list = parse_ids(args) + if experiment_id_list: + experiment_config = Experiments() + experiment_dict = experiment_config.get_all_experiments() + for experiment_id in experiment_id_list: + print_normal('Stopping experiment %s' % experiment_id) + nni_config = Config(experiment_dict[experiment_id]['fileName']) + rest_pid = nni_config.get_config('restServerPid') + if rest_pid: + kill_command(rest_pid) + tensorboard_pid_list = nni_config.get_config('tensorboardPidList') + if tensorboard_pid_list: + for tensorboard_pid in tensorboard_pid_list: + try: + kill_command(tensorboard_pid) + except Exception as exception: + print_error(exception) + nni_config.set_config('tensorboardPidList', []) + print_normal('Stop experiment success.') + experiment_config.update_experiment(experiment_id, 'status', 'STOPPED') + time_now = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())) + experiment_config.update_experiment(experiment_id, 'endTime', str(time_now)) + +def trial_ls(args): + '''List trial''' + def final_metric_data_cmp(lhs, rhs): + metric_l = json.loads(json.loads(lhs['finalMetricData'][0]['data'])) + metric_r = json.loads(json.loads(rhs['finalMetricData'][0]['data'])) + if isinstance(metric_l, float): + return metric_l - metric_r + elif isinstance(metric_l, dict): + return metric_l['default'] - metric_r['default'] + else: + print_error('Unexpected data format. Please check your data.') + raise ValueError + + if args.head and args.tail: + print_error('Head and tail cannot be set at the same time.') + return + nni_config = Config(get_config_filename(args)) + rest_port = nni_config.get_config('restServerPort') + rest_pid = nni_config.get_config('restServerPid') + if not detect_process(rest_pid): + print_error('Experiment is not running...') + return + running, response = check_rest_server_quick(rest_port) + if running: + response = rest_get(trial_jobs_url(rest_port), REST_TIME_OUT) + if response and check_response(response): + content = json.loads(response.text) + if args.head: + assert args.head > 0, 'The number of requested data must be greater than 0.' + content = sorted(filter(lambda x: 'finalMetricData' in x, content), + key=cmp_to_key(final_metric_data_cmp), reverse=True)[:args.head] + elif args.tail: + assert args.tail > 0, 'The number of requested data must be greater than 0.' + content = sorted(filter(lambda x: 'finalMetricData' in x, content), + key=cmp_to_key(final_metric_data_cmp))[:args.tail] + for index, value in enumerate(content): + content[index] = convert_time_stamp_to_date(value) + print(json.dumps(content, indent=4, sort_keys=True, separators=(',', ':'))) + return content + else: + print_error('List trial failed...') + else: + print_error('Restful server is not running...') + return None + +def trial_kill(args): + '''List trial''' + nni_config = Config(get_config_filename(args)) + rest_port = nni_config.get_config('restServerPort') + rest_pid = nni_config.get_config('restServerPid') + if not detect_process(rest_pid): + print_error('Experiment is not running...') + return + running, _ = check_rest_server_quick(rest_port) + if running: + response = rest_delete(trial_job_id_url(rest_port, args.trial_id), REST_TIME_OUT) + if response and check_response(response): + print(response.text) + return True + else: + print_error('Kill trial job failed...') + else: + print_error('Restful server is not running...') + return False + +def trial_codegen(args): + '''Generate code for a specific trial''' + print_warning('Currently, this command is only for nni nas programming interface.') + exp_id = check_experiment_id(args) + nni_config = Config(get_config_filename(args)) + if not nni_config.get_config('experimentConfig')['useAnnotation']: + print_error('The experiment is not using annotation') + exit(1) + code_dir = nni_config.get_config('experimentConfig')['trial']['codeDir'] + expand_annotations(code_dir, './exp_%s_trial_%s_code'%(exp_id, args.trial_id), exp_id, args.trial_id) + +def list_experiment(args): + '''Get experiment information''' + nni_config = Config(get_config_filename(args)) + rest_port = nni_config.get_config('restServerPort') + rest_pid = nni_config.get_config('restServerPid') + if not detect_process(rest_pid): + print_error('Experiment is not running...') + return + running, _ = check_rest_server_quick(rest_port) + if running: + response = rest_get(experiment_url(rest_port), REST_TIME_OUT) + if response and check_response(response): + content = convert_time_stamp_to_date(json.loads(response.text)) + print(json.dumps(content, indent=4, sort_keys=True, separators=(',', ':'))) + return content + else: + print_error('List experiment failed...') + else: + print_error('Restful server is not running...') + return None + +def experiment_status(args): + '''Show the status of experiment''' + nni_config = Config(get_config_filename(args)) + rest_port = nni_config.get_config('restServerPort') + result, response = check_rest_server_quick(rest_port) + if not result: + print_normal('Restful server is not running...') + else: + print(json.dumps(json.loads(response.text), indent=4, sort_keys=True, separators=(',', ':'))) + return result + +def log_internal(args, filetype): + '''internal function to call get_log_content''' + file_name = get_config_filename(args) + if filetype == 'stdout': + file_full_path = os.path.join(NNICTL_HOME_DIR, file_name, 'stdout') + else: + file_full_path = os.path.join(NNICTL_HOME_DIR, file_name, 'stderr') + print(check_output_command(file_full_path, head=args.head, tail=args.tail)) + +def log_stdout(args): + '''get stdout log''' + log_internal(args, 'stdout') + +def log_stderr(args): + '''get stderr log''' + log_internal(args, 'stderr') + +def log_trial(args): + ''''get trial log path''' + trial_id_path_dict = {} + trial_id_list = [] + nni_config = Config(get_config_filename(args)) + rest_port = nni_config.get_config('restServerPort') + rest_pid = nni_config.get_config('restServerPid') + if not detect_process(rest_pid): + print_error('Experiment is not running...') + return + running, response = check_rest_server_quick(rest_port) + if running: + response = rest_get(trial_jobs_url(rest_port), REST_TIME_OUT) + if response and check_response(response): + content = json.loads(response.text) + for trial in content: + trial_id_list.append(trial.get('id')) + if trial.get('logPath'): + trial_id_path_dict[trial.get('id')] = trial['logPath'] + else: + print_error('Restful server is not running...') + exit(1) + if args.trial_id: + if args.trial_id not in trial_id_list: + print_error('Trial id {0} not correct, please check your command!'.format(args.trial_id)) + exit(1) + if trial_id_path_dict.get(args.trial_id): + print_normal('id:' + args.trial_id + ' path:' + trial_id_path_dict[args.trial_id]) + else: + print_error('Log path is not available yet, please wait...') + exit(1) + else: + print_normal('All of trial log info:') + for key in trial_id_path_dict: + print_normal('id:' + key + ' path:' + trial_id_path_dict[key]) + if not trial_id_path_dict: + print_normal('None') + +def get_config(args): + '''get config info''' + nni_config = Config(get_config_filename(args)) + print(nni_config.get_all_config()) + +def webui_url(args): + '''show the url of web ui''' + nni_config = Config(get_config_filename(args)) + print_normal('{0} {1}'.format('Web UI url:', ' '.join(nni_config.get_config('webuiUrl')))) + +def webui_nas(args): + '''launch nas ui''' + print_normal('Starting NAS UI...') + try: + entry_dir = nni_node.__path__[0] + entry_file = os.path.join(entry_dir, 'nasui', 'server.js') + if sys.platform == 'win32': + node_command = os.path.join(entry_dir, 'node.exe') + else: + node_command = 'node' + cmds = [node_command, '--max-old-space-size=4096', entry_file, '--port', str(args.port), '--logdir', args.logdir] + subprocess.run(cmds, cwd=entry_dir) + except KeyboardInterrupt: + pass + +def local_clean(directory): + '''clean up local data''' + print_normal('removing folder {0}'.format(directory)) + try: + shutil.rmtree(directory) + except FileNotFoundError: + print_error('{0} does not exist.'.format(directory)) + +def remote_clean(machine_list, experiment_id=None): + '''clean up remote data''' + for machine in machine_list: + passwd = machine.get('passwd') + userName = machine.get('username') + host = machine.get('ip') + port = machine.get('port') + sshKeyPath = machine.get('sshKeyPath') + passphrase = machine.get('passphrase') + if experiment_id: + remote_dir = '/' + '/'.join(['tmp', 'nni-experiments', experiment_id]) + else: + remote_dir = '/' + '/'.join(['tmp', 'nni-experiments']) + sftp = create_ssh_sftp_client(host, port, userName, passwd, sshKeyPath, passphrase) + print_normal('removing folder {0}'.format(host + ':' + str(port) + remote_dir)) + remove_remote_directory(sftp, remote_dir) + +def hdfs_clean(host, user_name, output_dir, experiment_id=None): + '''clean up hdfs data''' + hdfs_client = HdfsClient(hosts='{0}:80'.format(host), user_name=user_name, webhdfs_path='/webhdfs/api/v1', timeout=5) + if experiment_id: + full_path = '/' + '/'.join([user_name, 'nni', 'experiments', experiment_id]) + else: + full_path = '/' + '/'.join([user_name, 'nni', 'experiments']) + print_normal('removing folder {0} in hdfs'.format(full_path)) + hdfs_client.delete(full_path, recursive=True) + if output_dir: + pattern = re.compile('hdfs://(?P([0-9]{1,3}.){3}[0-9]{1,3})(:[0-9]{2,5})?(?P/.*)?') + match_result = pattern.match(output_dir) + if match_result: + output_host = match_result.group('host') + output_dir = match_result.group('baseDir') + #check if the host is valid + if output_host != host: + print_warning('The host in {0} is not consistent with {1}'.format(output_dir, host)) + else: + if experiment_id: + output_dir = output_dir + '/' + experiment_id + print_normal('removing folder {0} in hdfs'.format(output_dir)) + hdfs_client.delete(output_dir, recursive=True) + +def experiment_clean(args): + '''clean up the experiment data''' + experiment_id_list = [] + experiment_config = Experiments() + experiment_dict = experiment_config.get_all_experiments() + if args.all: + experiment_id_list = list(experiment_dict.keys()) + else: + if args.id is None: + print_error('please set experiment id.') + exit(1) + if args.id not in experiment_dict: + print_error('Cannot find experiment {0}.'.format(args.id)) + exit(1) + experiment_id_list.append(args.id) + while True: + print('INFO: This action will delete experiment {0}, and it\'s not recoverable.'.format(' '.join(experiment_id_list))) + inputs = input('INFO: do you want to continue?[y/N]:') + if not inputs.lower() or inputs.lower() in ['n', 'no']: + exit(0) + elif inputs.lower() not in ['y', 'n', 'yes', 'no']: + print_warning('please input Y or N.') + else: + break + for experiment_id in experiment_id_list: + nni_config = Config(experiment_dict[experiment_id]['fileName']) + platform = nni_config.get_config('experimentConfig').get('trainingServicePlatform') + experiment_id = nni_config.get_config('experimentId') + if platform == 'remote': + machine_list = nni_config.get_config('experimentConfig').get('machineList') + remote_clean(machine_list, experiment_id) + elif platform == 'pai': + host = nni_config.get_config('experimentConfig').get('paiConfig').get('host') + user_name = nni_config.get_config('experimentConfig').get('paiConfig').get('userName') + output_dir = nni_config.get_config('experimentConfig').get('trial').get('outputDir') + hdfs_clean(host, user_name, output_dir, experiment_id) + elif platform != 'local': + #TODO: support all platforms + print_warning('platform {0} clean up not supported yet.'.format(platform)) + exit(0) + #clean local data + local_base_dir = nni_config.get_config('experimentConfig').get('logDir') + if not local_base_dir: + local_base_dir = NNI_HOME_DIR + local_experiment_dir = os.path.join(local_base_dir, experiment_id) + experiment_folder_name_list = ['checkpoint', 'db', 'log', 'trials'] + for folder_name in experiment_folder_name_list: + local_clean(os.path.join(local_experiment_dir, folder_name)) + if not os.listdir(local_experiment_dir): + local_clean(local_experiment_dir) + experiment_config = Experiments() + print_normal('removing metadata of experiment {0}'.format(experiment_id)) + experiment_config.remove_experiment(experiment_id) + print_normal('Done.') + +def get_platform_dir(config_content): + '''get the dir list to be deleted''' + platform = config_content.get('trainingServicePlatform') + dir_list = [] + if platform == 'remote': + machine_list = config_content.get('machineList') + for machine in machine_list: + host = machine.get('ip') + port = machine.get('port') + dir_list.append(host + ':' + str(port) + '/tmp/nni') + elif platform == 'pai': + host = config_content.get('paiConfig').get('host') + user_name = config_content.get('paiConfig').get('userName') + output_dir = config_content.get('trial').get('outputDir') + dir_list.append('server: {0}, path: {1}/nni'.format(host, user_name)) + if output_dir: + dir_list.append(output_dir) + return dir_list + +def platform_clean(args): + '''clean up the experiment data''' + config_path = os.path.abspath(args.config) + if not os.path.exists(config_path): + print_error('Please set correct config path.') + exit(1) + config_content = get_yml_content(config_path) + platform = config_content.get('trainingServicePlatform') + if platform == 'local': + print_normal('it doesn’t need to clean local platform.') + exit(0) + if platform not in ['remote', 'pai']: + print_normal('platform {0} not supported.'.format(platform)) + exit(0) + update_experiment() + dir_list = get_platform_dir(config_content) + if not dir_list: + print_normal('No folder of NNI caches is found.') + exit(1) + while True: + print_normal('This command will remove below folders of NNI caches. If other users are using experiments' \ + ' on below hosts, it will be broken.') + for value in dir_list: + print(' ' + value) + inputs = input('INFO: do you want to continue?[y/N]:') + if not inputs.lower() or inputs.lower() in ['n', 'no']: + exit(0) + elif inputs.lower() not in ['y', 'n', 'yes', 'no']: + print_warning('please input Y or N.') + else: + break + if platform == 'remote': + machine_list = config_content.get('machineList') + remote_clean(machine_list, None) + elif platform == 'pai': + host = config_content.get('paiConfig').get('host') + user_name = config_content.get('paiConfig').get('userName') + output_dir = config_content.get('trial').get('outputDir') + hdfs_clean(host, user_name, output_dir, None) + print_normal('Done.') + +def experiment_list(args): + '''get the information of all experiments''' + update_experiment() + experiment_config = Experiments() + experiment_dict = experiment_config.get_all_experiments() + if not experiment_dict: + print_normal('Cannot find experiments.') + exit(1) + experiment_id_list = [] + if args.all: + for key in experiment_dict.keys(): + experiment_id_list.append(key) + else: + for key in experiment_dict.keys(): + if experiment_dict[key]['status'] != 'STOPPED': + experiment_id_list.append(key) + if not experiment_id_list: + print_warning('There is no experiment running...\nYou can use \'nnictl experiment list --all\' to list all experiments.') + experiment_information = "" + for key in experiment_id_list: + experiment_information += EXPERIMENT_DETAIL_FORMAT % (key, + experiment_dict[key].get('experimentName', 'N/A'), + experiment_dict[key]['status'], + experiment_dict[key]['port'], + experiment_dict[key].get('platform'), + experiment_dict[key]['startTime'], + experiment_dict[key]['endTime']) + print(EXPERIMENT_INFORMATION_FORMAT % experiment_information) + return experiment_id_list + +def get_time_interval(time1, time2): + '''get the interval of two times''' + try: + #convert time to timestamp + time1 = time.mktime(time.strptime(time1, '%Y/%m/%d %H:%M:%S')) + time2 = time.mktime(time.strptime(time2, '%Y/%m/%d %H:%M:%S')) + seconds = (datetime.fromtimestamp(time2) - datetime.fromtimestamp(time1)).seconds + #convert seconds to day:hour:minute:second + days = seconds / 86400 + seconds %= 86400 + hours = seconds / 3600 + seconds %= 3600 + minutes = seconds / 60 + seconds %= 60 + return '%dd %dh %dm %ds' % (days, hours, minutes, seconds) + except: + return 'N/A' + +def show_experiment_info(): + '''show experiment information in monitor''' + update_experiment() + experiment_config = Experiments() + experiment_dict = experiment_config.get_all_experiments() + if not experiment_dict: + print('There is no experiment running...') + exit(1) + experiment_id_list = [] + for key in experiment_dict.keys(): + if experiment_dict[key]['status'] != 'STOPPED': + experiment_id_list.append(key) + if not experiment_id_list: + print_warning('There is no experiment running...') + return + for key in experiment_id_list: + print(EXPERIMENT_MONITOR_INFO % (key, experiment_dict[key]['status'], experiment_dict[key]['port'], \ + experiment_dict[key].get('platform'), experiment_dict[key]['startTime'], \ + get_time_interval(experiment_dict[key]['startTime'], experiment_dict[key]['endTime']))) + print(TRIAL_MONITOR_HEAD) + running, response = check_rest_server_quick(experiment_dict[key]['port']) + if running: + response = rest_get(trial_jobs_url(experiment_dict[key]['port']), REST_TIME_OUT) + if response and check_response(response): + content = json.loads(response.text) + for index, value in enumerate(content): + content[index] = convert_time_stamp_to_date(value) + print(TRIAL_MONITOR_CONTENT % (content[index].get('id'), content[index].get('startTime'), \ + content[index].get('endTime'), content[index].get('status'))) + print(TRIAL_MONITOR_TAIL) + +def set_monitor(auto_exit, time_interval, port=None, pid=None): + '''set the experiment monitor engine''' + while True: + try: + if sys.platform == 'win32': + os.system('cls') + else: + os.system('clear') + update_experiment() + show_experiment_info() + if auto_exit: + status = get_experiment_status(port) + if status in ['DONE', 'ERROR', 'STOPPED']: + print_normal('Experiment status is {0}.'.format(status)) + print_normal('Stopping experiment...') + kill_command(pid) + print_normal('Stop experiment success.') + exit(0) + time.sleep(time_interval) + except KeyboardInterrupt: + if auto_exit: + print_normal('Stopping experiment...') + kill_command(pid) + print_normal('Stop experiment success.') + else: + print_normal('Exiting...') + exit(0) + except Exception as exception: + print_error(exception) + exit(1) + +def monitor_experiment(args): + '''monitor the experiment''' + if args.time <= 0: + print_error('please input a positive integer as time interval, the unit is second.') + exit(1) + set_monitor(False, args.time) + +def export_trials_data(args): + '''export experiment metadata and intermediate results to json or csv + ''' + def groupby_trial_id(intermediate_results): + sorted(intermediate_results, key=lambda x: x['timestamp']) + groupby = dict() + for content in intermediate_results: + groupby.setdefault(content['trialJobId'], []).append(json.loads(content['data'])) + return groupby + + nni_config = Config(get_config_filename(args)) + rest_port = nni_config.get_config('restServerPort') + rest_pid = nni_config.get_config('restServerPid') + + if not detect_process(rest_pid): + print_error('Experiment is not running...') + return + running, response = check_rest_server_quick(rest_port) + if not running: + print_error('Restful server is not running') + return + response = rest_get(export_data_url(rest_port), 20) + if response is not None and check_response(response): + content = json.loads(response.text) + if args.intermediate: + intermediate_results_response = rest_get(metric_data_url(rest_port), REST_TIME_OUT) + if not intermediate_results_response or not check_response(intermediate_results_response): + print_error('Error getting intermediate results.') + return + intermediate_results = groupby_trial_id(json.loads(intermediate_results_response.text)) + for record in content: + record['intermediate'] = intermediate_results[record['id']] + if args.type == 'json': + with open(args.path, 'w') as file: + file.write(json.dumps(content)) + elif args.type == 'csv': + trial_records = [] + for record in content: + formated_record = dict() + if args.intermediate: + formated_record['intermediate'] = '[' + ','.join(record['intermediate']) + ']' + record_value = json.loads(record['value']) + if not isinstance(record_value, (float, int)): + formated_record.update({**record['parameter'], **record_value, **{'id': record['id']}}) + else: + formated_record.update({**record['parameter'], **{'reward': record_value, 'id': record['id']}}) + trial_records.append(formated_record) + if not trial_records: + print_error('No trial results collected! Please check your trial log...') + exit(0) + with open(args.path, 'w', newline='') as file: + writer = csv.DictWriter(file, set.union(*[set(r.keys()) for r in trial_records])) + writer.writeheader() + writer.writerows(trial_records) + else: + print_error('Unknown type: %s' % args.type) + return + else: + print_error('Export failed...') + +def search_space_auto_gen(args): + '''dry run trial code to generate search space file''' + trial_dir = os.path.expanduser(args.trial_dir) + file_path = os.path.expanduser(args.file) + if not os.path.isabs(file_path): + file_path = os.path.join(os.getcwd(), file_path) + assert os.path.exists(trial_dir) + if os.path.exists(file_path): + print_warning('%s already exists, will be overwritten.' % file_path) + print_normal('Dry run to generate search space...') + Popen(args.trial_command, cwd=trial_dir, env=dict(os.environ, NNI_GEN_SEARCH_SPACE=file_path), shell=True).wait() + if not os.path.exists(file_path): + print_warning('Expected search space file \'{}\' generated, but not found.'.format(file_path)) + else: + print_normal('Generate search space done: \'{}\'.'.format(file_path)) + +def save_experiment(args): + '''save experiment data to a zip file''' + experiment_config = Experiments() + experiment_dict = experiment_config.get_all_experiments() + if args.id is None: + print_error('Please set experiment id.') + exit(1) + if args.id not in experiment_dict: + print_error('Cannot find experiment {0}.'.format(args.id)) + exit(1) + if experiment_dict[args.id].get('status') != 'STOPPED': + print_error('Can only save stopped experiment!') + exit(1) + print_normal('Saving...') + nni_config = Config(experiment_dict[args.id]['fileName']) + logDir = os.path.join(NNI_HOME_DIR, args.id) + if nni_config.get_config('logDir'): + logDir = os.path.join(nni_config.get_config('logDir'), args.id) + temp_root_dir = generate_temp_dir() + + # Step1. Copy logDir to temp folder + if not os.path.exists(logDir): + print_error('logDir: %s does not exist!' % logDir) + exit(1) + temp_experiment_dir = os.path.join(temp_root_dir, 'experiment') + shutil.copytree(logDir, temp_experiment_dir) + + # Step2. Copy nnictl metadata to temp folder + temp_nnictl_dir = os.path.join(temp_root_dir, 'nnictl') + os.makedirs(temp_nnictl_dir, exist_ok=True) + try: + with open(os.path.join(temp_nnictl_dir, '.experiment'), 'w') as file: + experiment_dict[args.id]['id'] = args.id + json.dump(experiment_dict[args.id], file) + except IOError: + print_error('Write file to %s failed!' % os.path.join(temp_nnictl_dir, '.experiment')) + exit(1) + nnictl_config_dir = os.path.join(NNICTL_HOME_DIR, experiment_dict[args.id]['fileName']) + shutil.copytree(nnictl_config_dir, os.path.join(temp_nnictl_dir, experiment_dict[args.id]['fileName'])) + + # Step3. Copy code dir + if args.saveCodeDir: + temp_code_dir = os.path.join(temp_root_dir, 'code') + shutil.copytree(nni_config.get_config('experimentConfig')['trial']['codeDir'], temp_code_dir) + + # Step4. Copy searchSpace file + search_space_path = nni_config.get_config('experimentConfig').get('searchSpacePath') + if search_space_path: + if not os.path.exists(search_space_path): + print_warning('search space %s does not exist!' % search_space_path) + else: + temp_search_space_dir = os.path.join(temp_root_dir, 'searchSpace') + os.makedirs(temp_search_space_dir, exist_ok=True) + search_space_name = os.path.basename(search_space_path) + shutil.copyfile(search_space_path, os.path.join(temp_search_space_dir, search_space_name)) + + # Step5. Archive folder + zip_package_name = 'nni_experiment_%s' % args.id + if args.path: + os.makedirs(args.path, exist_ok=True) + zip_package_name = os.path.join(args.path, zip_package_name) + shutil.make_archive(zip_package_name, 'zip', temp_root_dir) + print_normal('Save to %s.zip success!' % zip_package_name) + + # Step5. Cleanup temp data + shutil.rmtree(temp_root_dir) + +def load_experiment(args): + '''load experiment data''' + package_path = os.path.expanduser(args.path) + if not os.path.exists(args.path): + print_error('file path %s does not exist!' % args.path) + exit(1) + if args.searchSpacePath and os.path.isdir(args.searchSpacePath): + print_error('search space path should be a full path with filename, not a directory!') + exit(1) + temp_root_dir = generate_temp_dir() + shutil.unpack_archive(package_path, temp_root_dir) + print_normal('Loading...') + # Step1. Validation + if not os.path.exists(args.codeDir): + print_error('Invalid: codeDir path does not exist!') + exit(1) + if args.logDir: + if not os.path.exists(args.logDir): + print_error('Invalid: logDir path does not exist!') + exit(1) + experiment_temp_dir = os.path.join(temp_root_dir, 'experiment') + if not os.path.exists(os.path.join(experiment_temp_dir, 'db')): + print_error('Invalid archive file: db file does not exist!') + shutil.rmtree(temp_root_dir) + exit(1) + nnictl_temp_dir = os.path.join(temp_root_dir, 'nnictl') + if not os.path.exists(os.path.join(nnictl_temp_dir, '.experiment')): + print_error('Invalid archive file: nnictl metadata file does not exist!') + shutil.rmtree(temp_root_dir) + exit(1) + try: + with open(os.path.join(nnictl_temp_dir, '.experiment'), 'r') as file: + experiment_metadata = json.load(file) + except ValueError as err: + print_error('Invalid nnictl metadata file: %s' % err) + shutil.rmtree(temp_root_dir) + exit(1) + experiment_config = Experiments() + experiment_dict = experiment_config.get_all_experiments() + experiment_id = experiment_metadata.get('id') + if experiment_id in experiment_dict: + print_error('Invalid: experiment id already exist!') + shutil.rmtree(temp_root_dir) + exit(1) + if not os.path.exists(os.path.join(nnictl_temp_dir, experiment_metadata.get('fileName'))): + print_error('Invalid: experiment metadata does not exist!') + shutil.rmtree(temp_root_dir) + exit(1) + + # Step2. Copy nnictl metadata + src_path = os.path.join(nnictl_temp_dir, experiment_metadata.get('fileName')) + dest_path = os.path.join(NNICTL_HOME_DIR, experiment_metadata.get('fileName')) + if os.path.exists(dest_path): + shutil.rmtree(dest_path) + shutil.copytree(src_path, dest_path) + + # Step3. Copy experiment data + nni_config = Config(experiment_metadata.get('fileName')) + nnictl_exp_config = nni_config.get_config('experimentConfig') + if args.logDir: + logDir = args.logDir + nnictl_exp_config['logDir'] = logDir + else: + if nnictl_exp_config.get('logDir'): + logDir = nnictl_exp_config['logDir'] + else: + logDir = NNI_HOME_DIR + os.rename(os.path.join(temp_root_dir, 'experiment'), os.path.join(temp_root_dir, experiment_id)) + src_path = os.path.join(os.path.join(temp_root_dir, experiment_id)) + dest_path = os.path.join(os.path.join(logDir, experiment_id)) + if os.path.exists(dest_path): + shutil.rmtree(dest_path) + shutil.copytree(src_path, dest_path) + + # Step4. Copy code dir + codeDir = os.path.expanduser(args.codeDir) + if not os.path.isabs(codeDir): + codeDir = os.path.join(os.getcwd(), codeDir) + print_normal('Expand codeDir to %s' % codeDir) + nnictl_exp_config['trial']['codeDir'] = codeDir + archive_code_dir = os.path.join(temp_root_dir, 'code') + if os.path.exists(archive_code_dir): + file_list = os.listdir(archive_code_dir) + for file_name in file_list: + src_path = os.path.join(archive_code_dir, file_name) + target_path = os.path.join(codeDir, file_name) + if os.path.exists(target_path): + print_error('Copy %s failed, %s exist!' % (file_name, target_path)) + continue + if os.path.isdir(src_path): + shutil.copytree(src_path, target_path) + else: + shutil.copy(src_path, target_path) + + # Step5. Copy searchSpace file + archive_search_space_dir = os.path.join(temp_root_dir, 'searchSpace') + if args.searchSpacePath: + target_path = os.path.expanduser(args.searchSpacePath) + else: + # set default path to codeDir + target_path = os.path.join(codeDir, 'search_space.json') + if not os.path.isabs(target_path): + target_path = os.path.join(os.getcwd(), target_path) + print_normal('Expand search space path to %s' % target_path) + nnictl_exp_config['searchSpacePath'] = target_path + # if the path already has a search space file, use the original one, otherwise use archived one + if not os.path.isfile(target_path): + if len(os.listdir(archive_search_space_dir)) == 0: + print_error('Archive file does not contain search space file!') + exit(1) + else: + for file in os.listdir(archive_search_space_dir): + source_path = os.path.join(archive_search_space_dir, file) + os.makedirs(os.path.dirname(target_path), exist_ok=True) + shutil.copyfile(source_path, target_path) + break + elif not args.searchSpacePath: + print_warning('%s exist, will not load search_space file' % target_path) + + # Step6. Create experiment metadata + nni_config.set_config('experimentConfig', nnictl_exp_config) + experiment_config.add_experiment(experiment_id, + experiment_metadata.get('port'), + experiment_metadata.get('startTime'), + experiment_metadata.get('fileName'), + experiment_metadata.get('platform'), + experiment_metadata.get('experimentName'), + experiment_metadata.get('endTime'), + experiment_metadata.get('status')) + print_normal('Load experiment %s succsss!' % experiment_id) + + # Step6. Cleanup temp data + shutil.rmtree(temp_root_dir) + diff --git a/utils/third_party/nni/tools/nnictl/package_management.py b/utils/third_party/nni/tools/nnictl/package_management.py new file mode 100644 index 0000000000000000000000000000000000000000..5eef3407522839b477e19b8fdadadcac19110e1c --- /dev/null +++ b/utils/third_party/nni/tools/nnictl/package_management.py @@ -0,0 +1,184 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +from collections import defaultdict +import json +import pkginfo +import nni +from nni.tools.package_utils import read_installed_package_meta, get_installed_package_meta, \ + write_package_meta, get_builtin_algo_meta, get_not_installable_builtin_names, ALGO_TYPES + +from .constants import INSTALLABLE_PACKAGE_META +from .common_utils import print_error, print_green +from .command_utils import install_requirements_command, call_pip_install, call_pip_uninstall + +PACKAGE_TYPES = ['tuner', 'assessor', 'advisor'] + +def install_by_name(package_name): + if package_name not in INSTALLABLE_PACKAGE_META: + raise RuntimeError('{} is not found in installable packages!'.format(package_name)) + + requirements_path = os.path.join(nni.__path__[0], 'algorithms/hpo', INSTALLABLE_PACKAGE_META[package_name]['code_sub_dir'], 'requirements.txt') + assert os.path.exists(requirements_path) + + return install_requirements_command(requirements_path) + +def package_install(args): + '''install packages''' + installed = False + try: + if args.name: + if install_by_name(args.name) == 0: + package_meta = {} + package_meta['type'] = INSTALLABLE_PACKAGE_META[args.name]['type'] + package_meta['name'] = args.name + package_meta['class_name'] = INSTALLABLE_PACKAGE_META[args.name]['class_name'] + package_meta['class_args_validator'] = INSTALLABLE_PACKAGE_META[args.name]['class_args_validator'] + save_package_meta_data(package_meta) + print_green('{} installed!'.format(args.name)) + installed = True + else: + package_meta = get_nni_meta(args.source) + if package_meta: + if call_pip_install(args.source) == 0: + save_package_meta_data(package_meta) + print_green('{} installed!'.format(package_meta['name'])) + installed = True + except Exception as e: + print_error(e) + if not installed: + print_error('installation failed!') + +def package_uninstall(args): + '''uninstall packages''' + name = args.name[0] + if name in get_not_installable_builtin_names(): + print_error('{} can not be uninstalled!'.format(name)) + exit(1) + meta = get_installed_package_meta(None, name) + if meta is None: + print_error('package {} not found!'.format(name)) + return + if 'installed_package' in meta: + call_pip_uninstall(meta['installed_package']) + if remove_package_meta_data(name): + print_green('{} uninstalled sucessfully!'.format(name)) + else: + print_error('Failed to uninstall {}!'.format(name)) + +def package_show(args): + '''show specified packages''' + builtin_name = args.name[0] + meta = get_builtin_algo_meta(builtin_name=builtin_name) + if meta: + print(json.dumps(meta, indent=4)) + else: + print_error('package {} not found'.format(builtin_name)) + +def print_package_list(meta): + print('+-----------------+------------+-----------+--------=-------------+------------------------------------------+') + print('| Name | Type | Installed | Class Name | Module Name |') + print('+-----------------+------------+-----------+----------------------+------------------------------------------+') + MAX_MODULE_NAME = 38 + for t in ['tuners', 'assessors', 'advisors']: + for p in meta[t]: + module_name = '.'.join(p['class_name'].split('.')[:-1]) + if len(module_name) > MAX_MODULE_NAME: + module_name = module_name[:MAX_MODULE_NAME-3] + '...' + class_name = p['class_name'].split('.')[-1] + print('| {:15s} | {:10s} | {:9s} | {:20s} | {:40s} |'.format(p['name'], t, p['installed'], class_name, module_name[:38])) + print('+-----------------+------------+-----------+----------------------+------------------------------------------+') + +def package_list(args): + '''list all packages''' + if args.all: + meta = get_builtin_algo_meta() + else: + meta = read_installed_package_meta() + + installed_names = defaultdict(list) + for t in ['tuners', 'assessors', 'advisors']: + for p in meta[t]: + p['installed'] = 'Yes' + installed_names[t].append(p['name']) + for k, v in INSTALLABLE_PACKAGE_META.items(): + t = v['type']+'s' + if k not in installed_names[t]: + meta[t].append({ + 'name': k, + 'class_name': v['class_name'], + 'class_args_validator': v['class_args_validator'], + 'installed': 'No' + }) + + print_package_list(meta) + +def save_package_meta_data(meta_data): + assert meta_data['type'] in PACKAGE_TYPES + assert 'name' in meta_data + assert 'class_name' in meta_data + + config = read_installed_package_meta() + + if meta_data['name'] in [x['name'] for x in config[meta_data['type']+'s']]: + raise ValueError('name %s already installed' % meta_data['name']) + + package_meta = {k: meta_data[k] for k in ['name', 'class_name', 'class_args_validator'] if k in meta_data} + if 'package_name' in meta_data: + package_meta['installed_package'] = meta_data['package_name'] + config[meta_data['type']+'s'].append(package_meta) + write_package_meta(config) + +def remove_package_meta_data(name): + config = read_installed_package_meta() + + updated = False + for t in ALGO_TYPES: + for meta in config[t]: + if meta['name'] == name: + config[t].remove(meta) + updated = True + if updated: + write_package_meta(config) + return True + return False + +def get_nni_meta(source): + if not os.path.exists(source): + print_error('{} does not exist'.format(source)) + return None + + if os.path.isdir(source): + if not os.path.exists(os.path.join(source, 'setup.py')): + print_error('setup.py not found') + return None + pkg = pkginfo.Develop(source) + else: + if not source.endswith('.whl'): + print_error('File name {} must ends with \'.whl\''.format(source)) + return False + pkg = pkginfo.Wheel(source) + + classifiers = pkg.classifiers + meta = parse_classifiers(classifiers) + meta['package_name'] = pkg.name + return meta + +def parse_classifiers(classifiers): + parts = [] + for c in classifiers: + if c.startswith('NNI Package'): + parts = [x.strip() for x in c.split('::')] + break + if len(parts) < 4 or not all(parts): + raise ValueError('Can not find correct NNI meta data in package classifiers.') + meta = { + 'type': parts[1], + 'name': parts[2], + 'class_name': parts[3] + } + if len(parts) >= 5: + meta['class_args_validator'] = parts[4] + + return meta diff --git a/utils/third_party/nni/tools/nnictl/rest_utils.py b/utils/third_party/nni/tools/nnictl/rest_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..e98c9a839245ca0775406302dd6bf40bf8a2ae35 --- /dev/null +++ b/utils/third_party/nni/tools/nnictl/rest_utils.py @@ -0,0 +1,77 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import time +import requests +from .url_utils import check_status_url +from .constants import REST_TIME_OUT +from .common_utils import print_error + +def rest_put(url, data, timeout, show_error=False): + '''Call rest put method''' + try: + response = requests.put(url, headers={'Accept': 'application/json', 'Content-Type': 'application/json'},\ + data=data, timeout=timeout) + return response + except Exception as exception: + if show_error: + print_error(exception) + return None + +def rest_post(url, data, timeout, show_error=False): + '''Call rest post method''' + try: + response = requests.post(url, headers={'Accept': 'application/json', 'Content-Type': 'application/json'},\ + data=data, timeout=timeout) + return response + except Exception as exception: + if show_error: + print_error(exception) + return None + +def rest_get(url, timeout, show_error=False): + '''Call rest get method''' + try: + response = requests.get(url, timeout=timeout) + return response + except Exception as exception: + if show_error: + print_error(exception) + return None + +def rest_delete(url, timeout, show_error=False): + '''Call rest delete method''' + try: + response = requests.delete(url, timeout=timeout) + return response + except Exception as exception: + if show_error: + print_error(exception) + return None + +def check_rest_server(rest_port): + '''Check if restful server is ready''' + retry_count = 20 + for _ in range(retry_count): + response = rest_get(check_status_url(rest_port), REST_TIME_OUT) + if response: + if response.status_code == 200: + return True, response + else: + return False, response + else: + time.sleep(1) + return False, response + +def check_rest_server_quick(rest_port): + '''Check if restful server is ready, only check once''' + response = rest_get(check_status_url(rest_port), 5) + if response and response.status_code == 200: + return True, response + return False, None + +def check_response(response): + '''Check if a response is success according to status_code''' + if response and response.status_code == 200: + return True + return False diff --git a/utils/third_party/nni/tools/nnictl/ssh_utils.py b/utils/third_party/nni/tools/nnictl/ssh_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..e3f26a8e24c1be67bf86cfa0192481b245c93f97 --- /dev/null +++ b/utils/third_party/nni/tools/nnictl/ssh_utils.py @@ -0,0 +1,60 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +from .common_utils import print_error +from .command_utils import install_package_command + +def check_environment(): + '''check if paramiko is installed''' + try: + import paramiko + except: + install_package_command('paramiko') + import paramiko + return paramiko + +def copy_remote_directory_to_local(sftp, remote_path, local_path): + '''copy remote directory to local machine''' + try: + os.makedirs(local_path, exist_ok=True) + files = sftp.listdir(remote_path) + for file in files: + remote_full_path = os.path.join(remote_path, file) + local_full_path = os.path.join(local_path, file) + try: + if sftp.listdir(remote_full_path): + copy_remote_directory_to_local(sftp, remote_full_path, local_full_path) + except: + sftp.get(remote_full_path, local_full_path) + except Exception: + pass + +def create_ssh_sftp_client(host_ip, port, username, password, ssh_key_path, passphrase): + '''create ssh client''' + try: + paramiko = check_environment() + conn = paramiko.Transport(host_ip, port) + if ssh_key_path is not None: + ssh_key = paramiko.RSAKey.from_private_key_file(ssh_key_path, password=passphrase) + conn.connect(username=username, pkey=ssh_key) + else: + conn.connect(username=username, password=password) + sftp = paramiko.SFTPClient.from_transport(conn) + return sftp + except Exception as exception: + print_error('Create ssh client error %s\n' % exception) + +def remove_remote_directory(sftp, directory): + '''remove a directory in remote machine''' + try: + files = sftp.listdir(directory) + for file in files: + filepath = '/'.join([directory, file]) + try: + sftp.remove(filepath) + except IOError: + remove_remote_directory(sftp, filepath) + sftp.rmdir(directory) + except IOError as err: + print_error(err) diff --git a/utils/third_party/nni/tools/nnictl/tensorboard_utils.py b/utils/third_party/nni/tools/nnictl/tensorboard_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..adf85f3a741588e8c82bd23bc0b89f2108e59d51 --- /dev/null +++ b/utils/third_party/nni/tools/nnictl/tensorboard_utils.py @@ -0,0 +1,147 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +import json +import re +import tempfile +from subprocess import call, Popen +from .rest_utils import rest_get, check_rest_server_quick, check_response +from .config_utils import Config, Experiments +from .url_utils import trial_jobs_url, get_local_urls +from .constants import REST_TIME_OUT +from .common_utils import print_normal, print_error, print_green, detect_process, detect_port, check_tensorboard_version +from .nnictl_utils import check_experiment_id, check_experiment_id +from .ssh_utils import create_ssh_sftp_client, copy_remote_directory_to_local + +def parse_log_path(args, trial_content): + '''parse log path''' + path_list = [] + host_list = [] + for trial in trial_content: + if args.trial_id and args.trial_id != 'all' and trial.get('id') != args.trial_id: + continue + pattern = r'(?P.+)://(?P.+):(?P.*)' + match = re.search(pattern, trial['logPath']) + if match: + path_list.append(match.group('path')) + host_list.append(match.group('host')) + if not path_list: + print_error('Trial id %s error!' % args.trial_id) + exit(1) + return path_list, host_list + +def copy_data_from_remote(args, nni_config, trial_content, path_list, host_list, temp_nni_path): + '''use ssh client to copy data from remote machine to local machien''' + machine_list = nni_config.get_config('experimentConfig').get('machineList') + machine_dict = {} + local_path_list = [] + for machine in machine_list: + machine_dict[machine['ip']] = {'port': machine['port'], 'passwd': machine['passwd'], 'username': machine['username'], + 'sshKeyPath': machine.get('sshKeyPath'), 'passphrase': machine.get('passphrase')} + for index, host in enumerate(host_list): + local_path = os.path.join(temp_nni_path, trial_content[index].get('id')) + local_path_list.append(local_path) + print_normal('Copying log data from %s to %s' % (host + ':' + path_list[index], local_path)) + sftp = create_ssh_sftp_client(host, machine_dict[host]['port'], machine_dict[host]['username'], machine_dict[host]['passwd'], + machine_dict[host]['sshKeyPath'], machine_dict[host]['passphrase']) + copy_remote_directory_to_local(sftp, path_list[index], local_path) + print_normal('Copy done!') + return local_path_list + +def get_path_list(args, nni_config, trial_content, temp_nni_path): + '''get path list according to different platform''' + path_list, host_list = parse_log_path(args, trial_content) + platform = nni_config.get_config('experimentConfig').get('trainingServicePlatform') + if platform == 'local': + print_normal('Log path: %s' % ' '.join(path_list)) + return path_list + elif platform == 'remote': + path_list = copy_data_from_remote(args, nni_config, trial_content, path_list, host_list, temp_nni_path) + print_normal('Log path: %s' % ' '.join(path_list)) + return path_list + else: + print_error('Not supported platform!') + exit(1) + +def format_tensorboard_log_path(path_list): + new_path_list = [] + for index, value in enumerate(path_list): + new_path_list.append('name%d:%s' % (index + 1, value)) + return ','.join(new_path_list) + +def start_tensorboard_process(args, nni_config, path_list, temp_nni_path): + '''call cmds to start tensorboard process in local machine''' + if detect_port(args.port): + print_error('Port %s is used by another process, please reset port!' % str(args.port)) + exit(1) + with open(os.path.join(temp_nni_path, 'tensorboard_stdout'), 'a+') as stdout_file, \ + open(os.path.join(temp_nni_path, 'tensorboard_stderr'), 'a+') as stderr_file: + log_dir_cmd = '--logdir_spec' if check_tensorboard_version() >= '2.0' else '--logdir' + cmds = ['tensorboard', log_dir_cmd, format_tensorboard_log_path(path_list), '--port', str(args.port)] + tensorboard_process = Popen(cmds, stdout=stdout_file, stderr=stderr_file) + url_list = get_local_urls(args.port) + print_green('Start tensorboard success!') + print_normal('Tensorboard urls: ' + ' '.join(url_list)) + tensorboard_process_pid_list = nni_config.get_config('tensorboardPidList') + if tensorboard_process_pid_list is None: + tensorboard_process_pid_list = [tensorboard_process.pid] + else: + tensorboard_process_pid_list.append(tensorboard_process.pid) + nni_config.set_config('tensorboardPidList', tensorboard_process_pid_list) + +def stop_tensorboard(args): + '''stop tensorboard''' + experiment_id = check_experiment_id(args) + experiment_config = Experiments() + experiment_dict = experiment_config.get_all_experiments() + config_file_name = experiment_dict[experiment_id]['fileName'] + nni_config = Config(config_file_name) + tensorboard_pid_list = nni_config.get_config('tensorboardPidList') + if tensorboard_pid_list: + for tensorboard_pid in tensorboard_pid_list: + try: + cmds = ['kill', '-9', str(tensorboard_pid)] + call(cmds) + except Exception as exception: + print_error(exception) + nni_config.set_config('tensorboardPidList', []) + print_normal('Stop tensorboard success!') + else: + print_error('No tensorboard configuration!') + + +def start_tensorboard(args): + '''start tensorboard''' + experiment_id = check_experiment_id(args) + experiment_config = Experiments() + experiment_dict = experiment_config.get_all_experiments() + config_file_name = experiment_dict[experiment_id]['fileName'] + nni_config = Config(config_file_name) + rest_port = nni_config.get_config('restServerPort') + rest_pid = nni_config.get_config('restServerPid') + if not detect_process(rest_pid): + print_error('Experiment is not running...') + return + running, response = check_rest_server_quick(rest_port) + trial_content = None + if running: + response = rest_get(trial_jobs_url(rest_port), REST_TIME_OUT) + if response and check_response(response): + trial_content = json.loads(response.text) + else: + print_error('List trial failed...') + else: + print_error('Restful server is not running...') + if not trial_content: + print_error('No trial information!') + exit(1) + if len(trial_content) > 1 and not args.trial_id: + print_error('There are multiple trials, please set trial id!') + exit(1) + experiment_id = nni_config.get_config('experimentId') + temp_nni_path = os.path.join(tempfile.gettempdir(), 'nni', experiment_id) + os.makedirs(temp_nni_path, exist_ok=True) + + path_list = get_path_list(args, nni_config, trial_content, temp_nni_path) + start_tensorboard_process(args, nni_config, path_list, temp_nni_path) diff --git a/utils/third_party/nni/tools/nnictl/updater.py b/utils/third_party/nni/tools/nnictl/updater.py new file mode 100644 index 0000000000000000000000000000000000000000..071a167f70084a5007d817427ec139ca2cd54343 --- /dev/null +++ b/utils/third_party/nni/tools/nnictl/updater.py @@ -0,0 +1,148 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import json +import os +from .rest_utils import rest_put, rest_post, rest_get, check_rest_server_quick, check_response +from .url_utils import experiment_url, import_data_url +from .config_utils import Config +from .common_utils import get_json_content, print_normal, print_error, print_warning +from .nnictl_utils import get_experiment_port, get_config_filename, detect_process +from .launcher_utils import parse_time +from .constants import REST_TIME_OUT, TUNERS_SUPPORTING_IMPORT_DATA, TUNERS_NO_NEED_TO_IMPORT_DATA + +def validate_digit(value, start, end): + '''validate if a digit is valid''' + if not str(value).isdigit() or int(value) < start or int(value) > end: + raise ValueError('value (%s) must be a digit from %s to %s' % (value, start, end)) + +def validate_file(path): + '''validate if a file exist''' + if not os.path.exists(path): + raise FileNotFoundError('%s is not a valid file path' % path) + +def validate_dispatcher(args): + '''validate if the dispatcher of the experiment supports importing data''' + nni_config = Config(get_config_filename(args)).get_config('experimentConfig') + if nni_config.get('tuner') and nni_config['tuner'].get('builtinTunerName'): + dispatcher_name = nni_config['tuner']['builtinTunerName'] + elif nni_config.get('advisor') and nni_config['advisor'].get('builtinAdvisorName'): + dispatcher_name = nni_config['advisor']['builtinAdvisorName'] + else: # otherwise it should be a customized one + return + if dispatcher_name not in TUNERS_SUPPORTING_IMPORT_DATA: + if dispatcher_name in TUNERS_NO_NEED_TO_IMPORT_DATA: + print_warning("There is no need to import data for %s" % dispatcher_name) + exit(0) + else: + print_error("%s does not support importing addtional data" % dispatcher_name) + exit(1) + +def load_search_space(path): + '''load search space content''' + content = json.dumps(get_json_content(path)) + if not content: + raise ValueError('searchSpace file should not be empty') + return content + +def get_query_type(key): + '''get update query type''' + if key == 'trialConcurrency': + return '?update_type=TRIAL_CONCURRENCY' + if key == 'maxExecDuration': + return '?update_type=MAX_EXEC_DURATION' + if key == 'searchSpace': + return '?update_type=SEARCH_SPACE' + if key == 'maxTrialNum': + return '?update_type=MAX_TRIAL_NUM' + +def update_experiment_profile(args, key, value): + '''call restful server to update experiment profile''' + nni_config = Config(get_config_filename(args)) + rest_port = nni_config.get_config('restServerPort') + running, _ = check_rest_server_quick(rest_port) + if running: + response = rest_get(experiment_url(rest_port), REST_TIME_OUT) + if response and check_response(response): + experiment_profile = json.loads(response.text) + experiment_profile['params'][key] = value + response = rest_put(experiment_url(rest_port)+get_query_type(key), json.dumps(experiment_profile), REST_TIME_OUT) + if response and check_response(response): + return response + else: + print_error('Restful server is not running...') + return None + +def update_searchspace(args): + validate_file(args.filename) + content = load_search_space(args.filename) + args.port = get_experiment_port(args) + if args.port is not None: + if update_experiment_profile(args, 'searchSpace', content): + print_normal('Update %s success!' % 'searchSpace') + else: + print_error('Update %s failed!' % 'searchSpace') + + +def update_concurrency(args): + validate_digit(args.value, 1, 1000) + args.port = get_experiment_port(args) + if args.port is not None: + if update_experiment_profile(args, 'trialConcurrency', int(args.value)): + print_normal('Update %s success!' % 'concurrency') + else: + print_error('Update %s failed!' % 'concurrency') + +def update_duration(args): + #parse time, change time unit to seconds + args.value = parse_time(args.value) + args.port = get_experiment_port(args) + if args.port is not None: + if update_experiment_profile(args, 'maxExecDuration', int(args.value)): + print_normal('Update %s success!' % 'duration') + else: + print_error('Update %s failed!' % 'duration') + +def update_trialnum(args): + validate_digit(args.value, 1, 999999999) + if update_experiment_profile(args, 'maxTrialNum', int(args.value)): + print_normal('Update %s success!' % 'trialnum') + else: + print_error('Update %s failed!' % 'trialnum') + +def import_data(args): + '''import additional data to the experiment''' + validate_file(args.filename) + validate_dispatcher(args) + content = load_search_space(args.filename) + + nni_config = Config(get_config_filename(args)) + rest_port = nni_config.get_config('restServerPort') + rest_pid = nni_config.get_config('restServerPid') + if not detect_process(rest_pid): + print_error('Experiment is not running...') + return + running, _ = check_rest_server_quick(rest_port) + if not running: + print_error('Restful server is not running') + return + + args.port = rest_port + if args.port is not None: + if import_data_to_restful_server(args, content): + pass + else: + print_error('Import data failed!') + +def import_data_to_restful_server(args, content): + '''call restful server to import data to the experiment''' + nni_config = Config(get_config_filename(args)) + rest_port = nni_config.get_config('restServerPort') + running, _ = check_rest_server_quick(rest_port) + if running: + response = rest_post(import_data_url(rest_port), content, REST_TIME_OUT) + if response and check_response(response): + return response + else: + print_error('Restful server is not running...') + return None diff --git a/utils/third_party/nni/tools/nnictl/url_utils.py b/utils/third_party/nni/tools/nnictl/url_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..59a28837a667fb910a6aa54715bb841b352dbada --- /dev/null +++ b/utils/third_party/nni/tools/nnictl/url_utils.py @@ -0,0 +1,78 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import socket +import psutil + +BASE_URL = 'http://localhost' + +API_ROOT_URL = '/api/v1/nni' + +EXPERIMENT_API = '/experiment' + +CLUSTER_METADATA_API = '/experiment/cluster-metadata' + +IMPORT_DATA_API = '/experiment/import-data' + +CHECK_STATUS_API = '/check-status' + +TRIAL_JOBS_API = '/trial-jobs' + +EXPORT_DATA_API = '/export-data' + +TENSORBOARD_API = '/tensorboard' + +METRIC_DATA_API = '/metric-data' + +def metric_data_url(port): + '''get metric_data url''' + return '{0}:{1}{2}{3}'.format(BASE_URL, port, API_ROOT_URL, METRIC_DATA_API) + +def check_status_url(port): + '''get check_status url''' + return '{0}:{1}{2}{3}'.format(BASE_URL, port, API_ROOT_URL, CHECK_STATUS_API) + + +def cluster_metadata_url(port): + '''get cluster_metadata_url''' + return '{0}:{1}{2}{3}'.format(BASE_URL, port, API_ROOT_URL, CLUSTER_METADATA_API) + + +def import_data_url(port): + '''get import_data_url''' + return '{0}:{1}{2}{3}'.format(BASE_URL, port, API_ROOT_URL, IMPORT_DATA_API) + + +def experiment_url(port): + '''get experiment_url''' + return '{0}:{1}{2}{3}'.format(BASE_URL, port, API_ROOT_URL, EXPERIMENT_API) + + +def trial_jobs_url(port): + '''get trial_jobs url''' + return '{0}:{1}{2}{3}'.format(BASE_URL, port, API_ROOT_URL, TRIAL_JOBS_API) + + +def trial_job_id_url(port, job_id): + '''get trial_jobs with id url''' + return '{0}:{1}{2}{3}/{4}'.format(BASE_URL, port, API_ROOT_URL, TRIAL_JOBS_API, job_id) + + +def export_data_url(port): + '''get export_data url''' + return '{0}:{1}{2}{3}'.format(BASE_URL, port, API_ROOT_URL, EXPORT_DATA_API) + + +def tensorboard_url(port): + '''get tensorboard url''' + return '{0}:{1}{2}{3}'.format(BASE_URL, port, API_ROOT_URL, TENSORBOARD_API) + + +def get_local_urls(port): + '''get urls of local machine''' + url_list = [] + for _, info in psutil.net_if_addrs().items(): + for addr in info: + if socket.AddressFamily.AF_INET == addr.family: + url_list.append('http://{}:{}'.format(addr.address, port)) + return url_list diff --git a/utils/third_party/nni/tools/package_utils/__init__.py b/utils/third_party/nni/tools/package_utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9a1c004053fc3935cffdfa6a634b726b4052c07e --- /dev/null +++ b/utils/third_party/nni/tools/package_utils/__init__.py @@ -0,0 +1,321 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from collections import defaultdict +import importlib +import os +from pathlib import Path +import sys + +import ruamel.yaml as yaml + +import nni +from .constants import BuiltinAlgorithms + +ALGO_TYPES = ['tuners', 'assessors', 'advisors'] + +def get_all_builtin_names(algo_type): + """Get all valid builtin names, including: + 1. BuiltinAlgorithms which is pre-installed. + 2. User installed packages in /config/installed_packages.yml + + Parameters + ---------- + algo_type: str + can be one of 'tuners', 'assessors' or 'advisors' + + Returns: list of string + ------- + All builtin names of specified type, for example, if algo_type is 'tuners', returns + all builtin tuner names. + """ + assert algo_type in ALGO_TYPES + merged_dict = _get_merged_builtin_dict() + + builtin_names = [x['name'] for x in merged_dict[algo_type]] + return builtin_names + +def get_not_installable_builtin_names(algo_type=None): + """Get builtin names in BuiltinAlgorithms which do not need to be installed + and can be used once NNI is installed. + + Parameters + ---------- + algo_type: str | None + can be one of 'tuners', 'assessors', 'advisors' or None + + Returns: list of string + ------- + All builtin names of specified type, for example, if algo_type is 'tuners', returns + all builtin tuner names. + If algo_type is None, returns all builtin names of all types. + """ + if algo_type is None: + meta = BuiltinAlgorithms + else: + assert algo_type in ALGO_TYPES + meta = { + algo_type: BuiltinAlgorithms[algo_type] + } + names = [] + for t in ALGO_TYPES: + if t in meta: + names.extend([x['name'] for x in meta[t]]) + return names + +def get_builtin_algo_meta(algo_type=None, builtin_name=None): + """ Get meta information of builtin algorithms from: + 1. Pre-installed BuiltinAlgorithms + 2. User installed packages in /config/installed_packages.yml + + Parameters + ---------- + algo_type: str | None + can be one of 'tuners', 'assessors', 'advisors' or None + builtin_name: str | None + builtin name. + + Returns: dict | list of dict | None + ------- + If builtin_name is specified, returns meta information of speicified builtin + alogorithms, for example: + { + 'name': 'Random', + 'class_name': 'nni.hyperopt_tuner.hyperopt_tuner.HyperoptTuner', + 'class_args': { + 'algorithm_name': 'random_search' + }, + 'accept_class_args': False, + 'class_args_validator': 'nni.hyperopt_tuner.hyperopt_tuner.HyperoptClassArgsValidator' + } + If builtin_name is None, returns multiple meta information in a list. + """ + merged_dict = _get_merged_builtin_dict() + + if algo_type is None and builtin_name is None: + return merged_dict + + if algo_type: + assert algo_type in ALGO_TYPES + metas = merged_dict[algo_type] + else: + metas = merged_dict['tuners'] + merged_dict['assessors'] + merged_dict['advisors'] + if builtin_name: + for m in metas: + if m['name'] == builtin_name: + return m + else: + return metas + + return None + +def get_installed_package_meta(algo_type, builtin_name): + """ Get meta information of user installed algorithms from: + /config/installed_packages.yml + + Parameters + ---------- + algo_type: str | None + can be one of 'tuners', 'assessors', 'advisors' or None + builtin_name: str + builtin name. + + Returns: dict | None + ------- + Returns meta information of speicified builtin alogorithms, for example: + { + 'class_args_validator': 'nni.smac_tuner.smac_tuner.SMACClassArgsValidator', + 'class_name': 'nni.smac_tuner.smac_tuner.SMACTuner', + 'name': 'SMAC' + } + """ + assert builtin_name is not None + if algo_type: + assert algo_type in ALGO_TYPES + config = read_installed_package_meta() + + candidates = [] + if algo_type: + candidates = config[algo_type] + else: + for algo_type in ALGO_TYPES: + candidates.extend(config[algo_type]) + for meta in candidates: + if meta['name'] == builtin_name: + return meta + return None + +def _parse_full_class_name(full_class_name): + if not full_class_name: + return None, None + parts = full_class_name.split('.') + module_name, class_name = '.'.join(parts[:-1]), parts[-1] + return module_name, class_name + +def get_builtin_module_class_name(algo_type, builtin_name): + """Get module name and class name of all builtin algorithms + + Parameters + ---------- + algo_type: str + can be one of 'tuners', 'assessors', 'advisors' + builtin_name: str + builtin name. + + Returns: tuple + ------- + tuple of (module name, class name) + """ + assert algo_type in ALGO_TYPES + assert builtin_name is not None + meta = get_builtin_algo_meta(algo_type, builtin_name) + if not meta: + return None, None + return _parse_full_class_name(meta['class_name']) + +def create_validator_instance(algo_type, builtin_name): + """Create instance of validator class + + Parameters + ---------- + algo_type: str + can be one of 'tuners', 'assessors', 'advisors' + builtin_name: str + builtin name. + + Returns: object | None + ------- + Returns validator class instance. + If specified validator class does not exist, returns None. + """ + assert algo_type in ALGO_TYPES + assert builtin_name is not None + meta = get_builtin_algo_meta(algo_type, builtin_name) + if not meta or 'class_args_validator' not in meta: + return None + module_name, class_name = _parse_full_class_name(meta['class_args_validator']) + class_module = importlib.import_module(module_name) + class_constructor = getattr(class_module, class_name) + + return class_constructor() + +def create_builtin_class_instance(builtin_name, input_class_args, algo_type): + """Create instance of builtin algorithms + + Parameters + ---------- + builtin_name: str + builtin name. + input_class_args: dict + kwargs for builtin class constructor + algo_type: str + can be one of 'tuners', 'assessors', 'advisors' + + Returns: object + ------- + Returns builtin class instance. + """ + assert algo_type in ALGO_TYPES + if builtin_name not in get_all_builtin_names(algo_type): + raise RuntimeError('Builtin name is not found: {}'.format(builtin_name)) + + def parse_algo_meta(algo_meta, input_class_args): + """ + 1. parse class_name field in meta data into module name and class name, + for example: + parse class_name 'nni.hyperopt_tuner.hyperopt_tuner.HyperoptTuner' in meta data into: + module name: nni.hyperopt_tuner.hyperopt_tuner + class name: HyperoptTuner + 2. merge user specified class args together with builtin class args. + """ + assert algo_meta + module_name, class_name = _parse_full_class_name(algo_meta['class_name']) + + class_args = {} + if 'class_args' in algo_meta: + class_args = algo_meta['class_args'] + if input_class_args is not None: + class_args.update(input_class_args) + + return module_name, class_name, class_args + + algo_meta = get_builtin_algo_meta(algo_type, builtin_name) + module_name, class_name, class_args = parse_algo_meta(algo_meta, input_class_args) + + if importlib.util.find_spec(module_name) is None: + raise RuntimeError('Builtin module can not be loaded: {}'.format(module_name)) + + class_module = importlib.import_module(module_name) + class_constructor = getattr(class_module, class_name) + + instance = class_constructor(**class_args) + + return instance + +def create_customized_class_instance(class_params): + """Create instance of customized algorithms + + Parameters + ---------- + class_params: dict + class_params should contains following keys: + codeDir: code directory + classFileName: python file name of the class + className: class name + classArgs (optional): kwargs pass to class constructor + Returns: object + ------- + Returns customized class instance. + """ + + code_dir = class_params.get('codeDir') + class_filename = class_params.get('classFileName') + class_name = class_params.get('className') + class_args = class_params.get('classArgs') + + if not os.path.isfile(os.path.join(code_dir, class_filename)): + raise ValueError('Class file not found: {}'.format( + os.path.join(code_dir, class_filename))) + sys.path.append(code_dir) + module_name = os.path.splitext(class_filename)[0] + class_module = importlib.import_module(module_name) + class_constructor = getattr(class_module, class_name) + + if class_args is None: + class_args = {} + instance = class_constructor(**class_args) + + return instance + +def get_package_config_path(): + # FIXME: this might not be the desired location + config_dir = Path(nni.__path__[0]).parent / 'nni_config' + if not os.path.exists(config_dir): + os.makedirs(config_dir, exist_ok=True) + return os.path.join(config_dir, 'installed_packages.yml') + +def read_installed_package_meta(): + config_file = get_package_config_path() + if os.path.exists(config_file): + with open(config_file, 'r') as f: + config = yaml.load(f, Loader=yaml.Loader) + else: + config = defaultdict(list) + for t in ALGO_TYPES: + if t not in config: + config[t] = [] + return config + +def write_package_meta(config): + config_file = get_package_config_path() + with open(config_file, 'w') as f: + f.write(yaml.dump(dict(config), default_flow_style=False)) + +def _get_merged_builtin_dict(): + def merge_meta_dict(d1, d2): + res = defaultdict(list) + for t in ALGO_TYPES: + res[t] = d1[t] + d2[t] + return res + + return merge_meta_dict(BuiltinAlgorithms, read_installed_package_meta()) diff --git a/utils/third_party/nni/tools/package_utils/constants.py b/utils/third_party/nni/tools/package_utils/constants.py new file mode 100644 index 0000000000000000000000000000000000000000..952310b5f964e54e2ece8b0ce4e144dce41b4afd --- /dev/null +++ b/utils/third_party/nni/tools/package_utils/constants.py @@ -0,0 +1,91 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +BuiltinAlgorithms = { + 'tuners': [ + { + 'name': 'TPE', + 'class_name': 'nni.algorithms.hpo.hyperopt_tuner.hyperopt_tuner.HyperoptTuner', + 'class_args': { + 'algorithm_name': 'tpe' + }, + 'class_args_validator': 'nni.algorithms.hpo.hyperopt_tuner.hyperopt_tuner.HyperoptClassArgsValidator' + }, + { + 'name': 'Random', + 'class_name': 'nni.algorithms.hpo.hyperopt_tuner.hyperopt_tuner.HyperoptTuner', + 'class_args': { + 'algorithm_name': 'random_search' + }, + 'accept_class_args': False, + 'class_args_validator': 'nni.algorithms.hpo.hyperopt_tuner.hyperopt_tuner.HyperoptClassArgsValidator' + }, + { + 'name': 'Anneal', + 'class_name': 'nni.algorithms.hpo.hyperopt_tuner.hyperopt_tuner.HyperoptTuner', + 'class_args': { + 'algorithm_name': 'anneal' + }, + 'class_args_validator': 'nni.algorithms.hpo.hyperopt_tuner.hyperopt_tuner.HyperoptClassArgsValidator' + }, + { + 'name': 'Evolution', + 'class_name': 'nni.algorithms.hpo.evolution_tuner.evolution_tuner.EvolutionTuner', + 'class_args_validator': 'nni.algorithms.hpo.evolution_tuner.evolution_tuner.EvolutionClassArgsValidator' + }, + { + 'name': 'BatchTuner', + 'class_name': 'nni.algorithms.hpo.batch_tuner.batch_tuner.BatchTuner', + 'accept_class_args': False, + }, + { + 'name': 'GridSearch', + 'class_name': 'nni.algorithms.hpo.gridsearch_tuner.gridsearch_tuner.GridSearchTuner', + 'accept_class_args': False, + }, + { + 'name': 'NetworkMorphism', + 'class_name': 'nni.algorithms.hpo.networkmorphism_tuner.networkmorphism_tuner.NetworkMorphismTuner', + 'class_args_validator': 'nni.algorithms.hpo.networkmorphism_tuner.networkmorphism_tuner.NetworkMorphismClassArgsValidator' + }, + { + 'name': 'MetisTuner', + 'class_name': 'nni.algorithms.hpo.metis_tuner.metis_tuner.MetisTuner', + 'class_args_validator': 'nni.algorithms.hpo.metis_tuner.metis_tuner.MetisClassArgsValidator' + }, + { + 'name': 'GPTuner', + 'class_name': 'nni.algorithms.hpo.gp_tuner.gp_tuner.GPTuner', + 'class_args_validator': 'nni.algorithms.hpo.gp_tuner.gp_tuner.GPClassArgsValidator' + }, + { + 'name': 'PBTTuner', + 'class_name': 'nni.algorithms.hpo.pbt_tuner.pbt_tuner.PBTTuner', + 'class_args_validator': 'nni.algorithms.hpo.pbt_tuner.pbt_tuner.PBTClassArgsValidator' + }, + { + 'name': 'RegularizedEvolutionTuner', + 'class_name': 'nni.algorithms.hpo.regularized_evolution_tuner.regularized_evolution_tuner.RegularizedEvolutionTuner', + 'class_args_validator': 'nni.algorithms.hpo.regularized_evolution_tuner.regularized_evolution_tuner.EvolutionClassArgsValidator' + } + ], + 'assessors': [ + { + 'name': 'Medianstop', + 'class_name': 'nni.algorithms.hpo.medianstop_assessor.medianstop_assessor.MedianstopAssessor', + 'class_args_validator': 'nni.algorithms.hpo.medianstop_assessor.medianstop_assessor.MedianstopClassArgsValidator' + }, + { + 'name': 'Curvefitting', + 'class_name': 'nni.algorithms.hpo.curvefitting_assessor.curvefitting_assessor.CurvefittingAssessor', + 'class_args_validator': 'nni.algorithms.hpo.curvefitting_assessor.curvefitting_assessor.CurvefittingClassArgsValidator' + }, + ], + 'advisors': [ + { + 'name': 'Hyperband', + 'class_name': 'nni.algorithms.hpo.hyperband_advisor.hyperband_advisor.Hyperband', + 'class_args_validator': 'nni.algorithms.hpo.hyperband_advisor.hyperband_advisor.HyperbandClassArgsValidator' + } + ] +} diff --git a/utils/third_party/nni/tools/trial_tool/__init__.py b/utils/third_party/nni/tools/trial_tool/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/utils/third_party/nni/tools/trial_tool/aml_channel.py b/utils/third_party/nni/tools/trial_tool/aml_channel.py new file mode 100644 index 0000000000000000000000000000000000000000..c8e1d7484a427b80b657d1c889c3a1cbe4aafecc --- /dev/null +++ b/utils/third_party/nni/tools/trial_tool/aml_channel.py @@ -0,0 +1,47 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from azureml.core.run import Run # pylint: disable=import-error +from .base_channel import BaseChannel +from .log_utils import LogType, nni_log + + +class AMLChannel(BaseChannel): + def __init__(self, args): + self.args = args + self.run = Run.get_context() + super(AMLChannel, self).__init__(args) + self.current_message_index = -1 + + def _inner_open(self): + pass + + def _inner_close(self): + pass + + def _inner_send(self, message): + try: + self.run.log('trial_runner', message.decode('utf8')) + except Exception as exception: + nni_log(LogType.Error, 'meet unhandled exception when send message: %s' % exception) + + def _inner_receive(self): + messages = [] + message_dict = self.run.get_metrics() + if 'nni_manager' not in message_dict: + return [] + message_list = message_dict['nni_manager'] + if not message_list: + return messages + if type(message_list) is list: + if self.current_message_index < len(message_list) - 1: + messages = message_list[self.current_message_index + 1 : len(message_list)] + self.current_message_index = len(message_list) - 1 + elif self.current_message_index == -1: + messages = [message_list] + self.current_message_index += 1 + newMessage = [] + for message in messages: + # receive message is string, to get consistent result, encode it here. + newMessage.append(message.encode('utf8')) + return newMessage diff --git a/utils/third_party/nni/tools/trial_tool/base_channel.py b/utils/third_party/nni/tools/trial_tool/base_channel.py new file mode 100644 index 0000000000000000000000000000000000000000..b9d3392abc0b15ce7eb90b07df4107902fb65d9d --- /dev/null +++ b/utils/third_party/nni/tools/trial_tool/base_channel.py @@ -0,0 +1,158 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import json +import threading +import time +from abc import ABC, abstractmethod +from queue import Empty, Queue + +from .log_utils import LogType, nni_log +from .commands import CommandType + +INTERVAL_SECONDS = 0.5 + + +class BaseChannel(ABC): + def __init__(self, args): + self.is_keep_parsed = args.node_count > 1 + self.args = args + self.node_id = self.args.node_id + + @abstractmethod + def _inner_send(self, message): + pass + + @abstractmethod + def _inner_receive(self): + return [] + + @abstractmethod + def _inner_open(self): + pass + + @abstractmethod + def _inner_close(self): + pass + + def open(self): + # initialize receive, send threads. + self.is_running = True + self.receive_queue = Queue() + self.receive_thread = threading.Thread(target=self._receive_loop) + self.receive_thread.start() + self.send_queue = Queue() + self.send_thread = threading.Thread(target=self._send_loop) + self.send_thread.start() + + self._inner_open() + + client_info = { + "isReady": True, + "runnerId": self.args.runner_id, + "expId": self.args.exp_id, + } + nni_log(LogType.Info, 'Channel: send ready information %s' % client_info) + self.send(CommandType.Initialized, client_info) + + def close(self): + self.is_running = False + try: + self._inner_close() + except Exception as err: + # ignore any error on closing + print("error on closing channel: %s" % err) + + def send(self, command, data): + """Send command to Training Service. + command: CommandType object. + data: string payload. + the message is sent synchronized. + """ + data["node"] = self.node_id + data = json.dumps(data) + data = data.encode('utf8') + message = b'%b%014d%b' % (command.value, len(data), data) + self.send_queue.put(message) + + def sent(self): + return self.send_queue.qsize() == 0 + + def received(self): + return self.receive_queue.qsize() > 0 + + def receive(self): + """Receive a command from Training Service. + Returns a tuple of command (CommandType) and payload (str) + """ + command = None + data = None + + try: + command_content = self.receive_queue.get(False) + if command_content is not None: + if (len(command_content) < 16): + # invalid header + nni_log(LogType.Error, 'incorrect command is found, command must be greater than 16 bytes!') + return None, None + header = command_content[:16] + command = CommandType(header[:2]) + length = int(header[2:]) + if (len(command_content)-16 != length): + nni_log(LogType.Error, 'incorrect command length, length {}, actual data length is {}, header {}.' + .format(length, len(command_content)-16, header)) + return None, None + data = command_content[16:16+length] + data = json.loads(data.decode('utf8')) + if self.node_id is None: + nni_log(LogType.Info, 'Received command, header: [%s], data: [%s]' % (header, data)) + else: + nni_log(LogType.Info, 'Received command(%s), header: [%s], data: [%s]' % (self.node_id, header, data)) + except Empty: + # do nothing, if no command received. + pass + except Exception as identifier: + nni_log(LogType.Error, 'meet unhandled exception in base_channel: %s' % identifier) + return command, data + + def _fetch_message(self, buffer, has_new_line=False): + messages = [] + while(len(buffer)) >= 16: + header = buffer[:16] + length = int(header[2:]) + + message_length = length+16 + total_length = message_length + if has_new_line: + total_length += 1 + + # break, if buffer is too short. + if len(buffer) < total_length: + break + data = buffer[16:message_length] + if has_new_line and 10 != buffer[total_length-1]: + nni_log(LogType.Error, 'end of message should be \\n, but got {}'.format(self.in_cache[total_length-1])) + buffer = buffer[total_length:] + messages.append(header + data) + + return messages, buffer + + def _receive_loop(self): + while (self.is_running): + messages = self._inner_receive() + if messages is not None: + for message in messages: + self.receive_queue.put(message) + time.sleep(INTERVAL_SECONDS) + + def _send_loop(self): + while (self.is_running): + message = None + try: + # no sleep, since it's a block call with INTERVAL_SECONDS second timeout + message = self.send_queue.get(True, INTERVAL_SECONDS) + except Empty: + # do nothing, if no command received. + pass + if message is not None: + self._inner_send(message) diff --git a/utils/third_party/nni/tools/trial_tool/commands.py b/utils/third_party/nni/tools/trial_tool/commands.py new file mode 100644 index 0000000000000000000000000000000000000000..86b10a2fe9a85d17bdfb4d9ec57b0b6cceb250da --- /dev/null +++ b/utils/third_party/nni/tools/trial_tool/commands.py @@ -0,0 +1,25 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from enum import Enum + + +class CommandType(Enum): + Initialize = b'IN' + RequestTrialJobs = b'GE' + ReportMetricData = b'ME' + ReportGpuInfo = b'GI' + UpdateSearchSpace = b'SS' + ImportData = b'FD' + AddCustomizedTrialJob = b'AD' + TrialEnd = b'EN' + Terminate = b'TE' + Ping = b'PI' + + Initialized = b'ID' + NewTrialJob = b'TR' + SendTrialJobParameter = b'SP' + NoMoreTrialJobs = b'NO' + KillTrialJob = b'KI' + StdOut = b'SO' + VersionCheck = b'VC' diff --git a/utils/third_party/nni/tools/trial_tool/constants.py b/utils/third_party/nni/tools/trial_tool/constants.py new file mode 100644 index 0000000000000000000000000000000000000000..bef401337039d46aa23fc1e5816b58978d07e104 --- /dev/null +++ b/utils/third_party/nni/tools/trial_tool/constants.py @@ -0,0 +1,24 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os + +API_ROOT_URL = '/api/v1/nni-pai' + +BASE_URL = 'http://{}' + +LOG_DIR = os.environ['NNI_OUTPUT_DIR'] + +NNI_PLATFORM = os.environ['NNI_PLATFORM'] + +STDOUT_FULL_PATH = os.path.join(LOG_DIR, 'stdout') + +STDERR_FULL_PATH = os.path.join(LOG_DIR, 'stderr') + +STDOUT_API = '/stdout' +VERSION_API = '/version' +PARAMETER_META_API = '/parameter-file-meta' +NNI_SYS_DIR = os.environ['NNI_SYS_DIR'] +NNI_TRIAL_JOB_ID = os.environ['NNI_TRIAL_JOB_ID'] +NNI_EXP_ID = os.environ['NNI_EXP_ID'] +MULTI_PHASE = os.environ['MULTI_PHASE'] diff --git a/utils/third_party/nni/tools/trial_tool/file_channel.py b/utils/third_party/nni/tools/trial_tool/file_channel.py new file mode 100644 index 0000000000000000000000000000000000000000..9a431d25f7ee3a4eeff973667d345ce2ddada1cb --- /dev/null +++ b/utils/third_party/nni/tools/trial_tool/file_channel.py @@ -0,0 +1,75 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os + +from .base_channel import BaseChannel + +command_path = "./commands" +runner_commands_file_name_prefix = "runner_commands" +manager_commands_file_name = "manager_commands.txt" + + +class FileChannel(BaseChannel): + + def __init__(self, args): + self.node_id = args.node_id + self.out_file = None + self.in_file = None + self.in_offset = 0 + self.in_cache = b"" + + super(FileChannel, self).__init__(args) + + def _inner_open(self): + pass + + def _inner_close(self): + if self.out_file is not None: + self.out_file.close() + self.out_file = None + if self.in_file is not None: + self.in_file.close() + self.in_file = None + + def _inner_send(self, message): + if self.out_file is None: + if not os.path.exists(command_path): + os.makedirs(command_path, exist_ok=True) + + if self.node_id is None: + file_name = os.path.join(command_path, "%s.txt" % runner_commands_file_name_prefix) + else: + file_name = os.path.join(command_path, "%s_%s.txt" % ( + runner_commands_file_name_prefix, self.node_id)) + self.out_file = open(file_name, "ab") + + self.out_file.write(message) + self.out_file.write(b'\n') + self.out_file.flush() + + def _open_manager_command(self): + full_name = os.path.join(command_path, manager_commands_file_name) + + if self.in_file is not None and self.in_file.closed: + self.in_file = None + + if self.in_file is None and os.path.exists(full_name): + self.in_file = open(full_name, "rb") + self.in_file.seek(self.in_offset) + + def _inner_receive(self): + messages = [] + + if self.in_file is None: + self._open_manager_command() + if self.in_file is not None: + self.in_file.seek(0, os.SEEK_END) + new_offset = self.in_file.tell() + self.in_file.seek(self.in_offset, os.SEEK_SET) + count = new_offset - self.in_offset + if count > 0: + self.in_cache += self.in_file.read(count) + self.in_offset = new_offset + messages, self.in_cache = self._fetch_message(self.in_cache, True) + return messages diff --git a/utils/third_party/nni/tools/trial_tool/gpu.py b/utils/third_party/nni/tools/trial_tool/gpu.py new file mode 100644 index 0000000000000000000000000000000000000000..48dab4b182a6ecd08b1108c297d206663ce9bb40 --- /dev/null +++ b/utils/third_party/nni/tools/trial_tool/gpu.py @@ -0,0 +1,69 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import subprocess +import time +import traceback +from xml.dom import minidom + + +def collect_gpu_usage(node_id): + cmd = 'nvidia-smi -q -x'.split() + info = None + try: + smi_output = subprocess.check_output(cmd) + info = parse_nvidia_smi_result(smi_output) + except Exception: + traceback.print_exc() + info = gen_empty_gpu_metric() + return info + + +def parse_nvidia_smi_result(smi): + try: + output = {} + xmldoc = minidom.parseString(smi) + gpuList = xmldoc.getElementsByTagName('gpu') + output["Timestamp"] = time.asctime(time.localtime()) + output["gpuCount"] = len(gpuList) + output["gpuInfos"] = [] + for gpuIndex, gpu in enumerate(gpuList): + gpuInfo = {} + gpuInfo['index'] = gpuIndex + gpuInfo['gpuUtil'] = gpu.getElementsByTagName('utilization')[0]\ + .getElementsByTagName('gpu_util')[0]\ + .childNodes[0].data.replace("%", "").strip() + gpuInfo['gpuMemUtil'] = gpu.getElementsByTagName('utilization')[0]\ + .getElementsByTagName('memory_util')[0]\ + .childNodes[0].data.replace("%", "").strip() + processes = gpu.getElementsByTagName('processes') + runningProNumber = len(processes[0].getElementsByTagName('process_info')) + gpuInfo['activeProcessNum'] = runningProNumber + + gpuInfo['gpuType'] = gpu.getElementsByTagName('product_name')[0]\ + .childNodes[0].data + memUsage = gpu.getElementsByTagName('fb_memory_usage')[0] + gpuInfo['gpuMemTotal'] = memUsage.getElementsByTagName('total')[0]\ + .childNodes[0].data.replace("MiB", "").strip() + gpuInfo['gpuMemUsed'] = memUsage.getElementsByTagName('used')[0]\ + .childNodes[0].data.replace("MiB", "").strip() + gpuInfo['gpuMemFree'] = memUsage.getElementsByTagName('free')[0]\ + .childNodes[0].data.replace("MiB", "").strip() + + output["gpuInfos"].append(gpuInfo) + except Exception: + traceback.print_exc() + output = {} + return output + + +def gen_empty_gpu_metric(): + try: + output = {} + output["Timestamp"] = time.asctime(time.localtime()) + output["gpuCount"] = 0 + output["gpuInfos"] = [] + except Exception: + traceback.print_exc() + output = {} + return output diff --git a/utils/third_party/nni/tools/trial_tool/hdfsClientUtility.py b/utils/third_party/nni/tools/trial_tool/hdfsClientUtility.py new file mode 100644 index 0000000000000000000000000000000000000000..05d4ea0d85675b5c763bf27f29db50b7900e1d03 --- /dev/null +++ b/utils/third_party/nni/tools/trial_tool/hdfsClientUtility.py @@ -0,0 +1,92 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +import posixpath +from .log_utils import LogType, nni_log + +def copyHdfsDirectoryToLocal(hdfsDirectory, localDirectory, hdfsClient): + '''Copy directory from HDFS to local''' + if not os.path.exists(localDirectory): + os.makedirs(localDirectory) + try: + listing = hdfsClient.list_status(hdfsDirectory) + except Exception as exception: + nni_log(LogType.Error, 'List hdfs directory {0} error: {1}'.format(hdfsDirectory, str(exception))) + raise exception + + for f in listing: + if f.type == 'DIRECTORY': + subHdfsDirectory = posixpath.join(hdfsDirectory, f.pathSuffix) + subLocalDirectory = os.path.join(localDirectory, f.pathSuffix) + copyHdfsDirectoryToLocal(subHdfsDirectory, subLocalDirectory, hdfsClient) + elif f.type == 'FILE': + hdfsFilePath = posixpath.join(hdfsDirectory, f.pathSuffix) + localFilePath = os.path.join(localDirectory, f.pathSuffix) + copyHdfsFileToLocal(hdfsFilePath, localFilePath, hdfsClient) + else: + raise AssertionError('unexpected type {}'.format(f.type)) + +def copyHdfsFileToLocal(hdfsFilePath, localFilePath, hdfsClient, override=True): + '''Copy file from HDFS to local''' + if not hdfsClient.exists(hdfsFilePath): + raise Exception('HDFS file {} does not exist!'.format(hdfsFilePath)) + try: + file_status = hdfsClient.get_file_status(hdfsFilePath) + if file_status.type != 'FILE': + raise Exception('HDFS file path {} is not a file'.format(hdfsFilePath)) + except Exception as exception: + nni_log(LogType.Error, 'Get hdfs file {0} status error: {1}'.format(hdfsFilePath, str(exception))) + raise exception + + if os.path.exists(localFilePath) and override: + os.remove(localFilePath) + try: + hdfsClient.copy_to_local(hdfsFilePath, localFilePath) + except Exception as exception: + nni_log(LogType.Error, 'Copy hdfs file {0} to {1} error: {2}'.format(hdfsFilePath, localFilePath, str(exception))) + raise exception + nni_log(LogType.Info, 'Successfully copied hdfs file {0} to {1}, {2} bytes'.format(hdfsFilePath, localFilePath, file_status.length)) + +def copyDirectoryToHdfs(localDirectory, hdfsDirectory, hdfsClient): + '''Copy directory from local to HDFS''' + if not os.path.exists(localDirectory): + raise Exception('Local Directory does not exist!') + hdfsClient.mkdirs(hdfsDirectory) + result = True + for file in os.listdir(localDirectory): + file_path = os.path.join(localDirectory, file) + if os.path.isdir(file_path): + hdfs_directory = os.path.join(hdfsDirectory, file) + try: + result = result and copyDirectoryToHdfs(file_path, hdfs_directory, hdfsClient) + except Exception as exception: + nni_log(LogType.Error, + 'Copy local directory {0} to hdfs directory {1} error: {2}'.format(file_path, hdfs_directory, str(exception))) + result = False + else: + hdfs_file_path = os.path.join(hdfsDirectory, file) + try: + result = result and copyFileToHdfs(file_path, hdfs_file_path, hdfsClient) + except Exception as exception: + nni_log(LogType.Error, 'Copy local file {0} to hdfs {1} error: {2}'.format(file_path, hdfs_file_path, str(exception))) + result = False + return result + +def copyFileToHdfs(localFilePath, hdfsFilePath, hdfsClient, override=True): + '''Copy a local file to HDFS directory''' + if not os.path.exists(localFilePath): + raise Exception('Local file Path does not exist!') + if os.path.isdir(localFilePath): + raise Exception('localFile should not a directory!') + if hdfsClient.exists(hdfsFilePath): + if override: + hdfsClient.delete(hdfsFilePath) + else: + return False + try: + hdfsClient.copy_from_local(localFilePath, hdfsFilePath) + return True + except Exception as exception: + nni_log(LogType.Error, 'Copy local file {0} to hdfs file {1} error: {2}'.format(localFilePath, hdfsFilePath, str(exception))) + return False diff --git a/utils/third_party/nni/tools/trial_tool/log_utils.py b/utils/third_party/nni/tools/trial_tool/log_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..8d5b3d94c0b60d60b978b94d967f092bfa9a1c56 --- /dev/null +++ b/utils/third_party/nni/tools/trial_tool/log_utils.py @@ -0,0 +1,219 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +import sys +import json +import logging +import logging.handlers +import time +import threading +import re + +from datetime import datetime +from enum import Enum, unique +from logging import StreamHandler + +from queue import Queue + +from .rest_utils import rest_post +from .url_utils import gen_send_stdout_url +from .commands import CommandType + + +@unique +class LogType(Enum): + Trace = 'TRACE' + Debug = 'DEBUG' + Info = 'INFO' + Warning = 'WARNING' + Error = 'ERROR' + Fatal = 'FATAL' + + +@unique +class StdOutputType(Enum): + Stdout = 'stdout', + Stderr = 'stderr' + + +def nni_log(log_type, log_message): + '''Log message into stdout''' + dt = datetime.now() + print('[{0}] {1} {2}'.format(dt, log_type.value, log_message), flush=True) + + +class NNIRestLogHanlder(StreamHandler): + def __init__(self, host, port, tag, trial_id, channel, std_output_type=StdOutputType.Stdout): + StreamHandler.__init__(self) + self.host = host + self.port = port + self.tag = tag + self.std_output_type = std_output_type + self.trial_id = trial_id + self.channel = channel + self.orig_stdout = sys.__stdout__ + self.orig_stderr = sys.__stderr__ + + def emit(self, record): + log_entry = {} + log_entry['tag'] = self.tag + log_entry['stdOutputType'] = self.std_output_type.name + log_entry['msg'] = self.format(record) + + try: + if self.channel is None: + rest_post(gen_send_stdout_url(self.host, self.port), json.dumps(log_entry), 10, True) + else: + if self.trial_id is not None: + log_entry["trial"] = self.trial_id + self.channel.send(CommandType.StdOut, log_entry) + except Exception as e: + self.orig_stderr.write(str(e) + '\n') + self.orig_stderr.flush() + + +class RemoteLogger(object): + """ + NNI remote logger + """ + + def __init__(self, syslog_host, syslog_port, tag, std_output_type, log_collection, trial_id=None, channel=None, log_level=logging.INFO): + ''' + constructor + ''' + logger_name = 'nni_syslog_{}'.format(tag) + # to prevent multiple trial logged in same logger + if trial_id is not None: + logger_name = '{}_{}'.format(logger_name, trial_id) + self.logger = logging.getLogger(logger_name) + self.log_level = log_level + self.logger.setLevel(self.log_level) + self.pipeReader = None + self.handler = NNIRestLogHanlder(syslog_host, syslog_port, tag, trial_id, channel) + self.logger.addHandler(self.handler) + if std_output_type == StdOutputType.Stdout: + self.orig_stdout = sys.__stdout__ + else: + self.orig_stdout = sys.__stderr__ + self.log_collection = log_collection + + def get_pipelog_reader(self): + ''' + Get pipe for remote logger + ''' + self.pipeReader = PipeLogReader(self.logger, self.log_collection, logging.INFO) + return self.pipeReader + + def flush(self): + ''' + Add flush in handler + ''' + for handler in self.logger.handlers: + handler.flush() + + def write(self, buf): + ''' + Write buffer data into logger/stdout + ''' + for line in buf.rstrip().splitlines(): + self.orig_stdout.write(line.rstrip() + '\n') + self.orig_stdout.flush() + try: + self.logger.log(self.log_level, line.rstrip()) + except Exception: + pass + + def close(self): + ''' + Close handlers and resources + ''' + if self.pipeReader is not None: + self.pipeReader.set_process_exit() + for handler in self.logger.handlers: + handler.close() + self.logger.removeHandler(handler) + + +class PipeLogReader(threading.Thread): + """ + The reader thread reads log data from pipe + """ + + def __init__(self, logger, log_collection, log_level=logging.INFO): + """Setup the object with a logger and a loglevel + and start the thread + """ + threading.Thread.__init__(self) + self.queue = Queue() + self.logger = logger + self.daemon = False + self.log_level = log_level + self.fdRead, self.fdWrite = os.pipe() + self.pipeReader = os.fdopen(self.fdRead) + self.orig_stdout = sys.__stdout__ + self._is_read_completed = False + self.process_exit = False + self.log_collection = log_collection + self.log_pattern = re.compile(r'NNISDK_MEb\'.*\'$') + + def _populateQueue(stream, queue): + ''' + Collect lines from 'stream' and put them in 'quque'. + ''' + time.sleep(1) + while True: + cur_process_exit = self.process_exit + try: + line = self.queue.get(True, 5) + try: + self.logger.log(self.log_level, line.rstrip()) + except Exception: + pass + except Exception: + if cur_process_exit == True: + self._is_read_completed = True + break + + self.pip_log_reader_thread = threading.Thread(target=_populateQueue, args=(self.pipeReader, self.queue)) + self.pip_log_reader_thread.daemon = True + self.start() + self.pip_log_reader_thread.start() + + def fileno(self): + """Return the write file descriptor of the pipe + """ + return self.fdWrite + + def run(self): + """Run the thread, logging everything. + If the log_collection is 'none', the log content will not be enqueued + """ + for line in iter(self.pipeReader.readline, ''): + self.orig_stdout.write(line.rstrip() + '\n') + self.orig_stdout.flush() + + if self.log_collection == 'none': + search_result = self.log_pattern.search(line) + if search_result: + metrics = search_result.group(0) + self.queue.put(metrics+'\n') + else: + self.queue.put(line) + + self.pipeReader.close() + + def close(self): + """Close the write end of the pipe. + """ + os.close(self.fdWrite) + + @property + def is_read_completed(self): + """Return if read is completed + """ + return self._is_read_completed + + def set_process_exit(self): + self.process_exit = True + return self.process_exit diff --git a/utils/third_party/nni/tools/trial_tool/rest_utils.py b/utils/third_party/nni/tools/trial_tool/rest_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..959209c7470bdb5dfe1ca32af91ba85757dd51e0 --- /dev/null +++ b/utils/third_party/nni/tools/trial_tool/rest_utils.py @@ -0,0 +1,44 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import requests + +def rest_get(url, timeout): + '''Call rest get method''' + try: + response = requests.get(url, timeout=timeout) + return response + except Exception as e: + print('Get exception {0} when sending http get to url {1}'.format(str(e), url)) + return None + +def rest_post(url, data, timeout, rethrow_exception=False): + '''Call rest post method''' + try: + response = requests.post(url, headers={'Accept': 'application/json', 'Content-Type': 'application/json'},\ + data=data, timeout=timeout) + return response + except Exception as e: + if rethrow_exception is True: + raise + print('Get exception {0} when sending http post to url {1}'.format(str(e), url)) + return None + +def rest_put(url, data, timeout): + '''Call rest put method''' + try: + response = requests.put(url, headers={'Accept': 'application/json', 'Content-Type': 'application/json'},\ + data=data, timeout=timeout) + return response + except Exception as e: + print('Get exception {0} when sending http put to url {1}'.format(str(e), url)) + return None + +def rest_delete(url, timeout): + '''Call rest delete method''' + try: + response = requests.delete(url, timeout=timeout) + return response + except Exception as e: + print('Get exception {0} when sending http delete to url {1}'.format(str(e), url)) + return None diff --git a/utils/third_party/nni/tools/trial_tool/trial.py b/utils/third_party/nni/tools/trial_tool/trial.py new file mode 100644 index 0000000000000000000000000000000000000000..037b210cb0a50d69e89d3c7d62cf0bd0521886f8 --- /dev/null +++ b/utils/third_party/nni/tools/trial_tool/trial.py @@ -0,0 +1,159 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import ctypes +import os +import shlex +import tarfile +import time +from datetime import datetime +from subprocess import Popen + +import psutil + +from .log_utils import LogType, RemoteLogger, StdOutputType, nni_log +from .commands import CommandType + +trial_output_path_name = ".nni" + + +class Trial: + def __init__(self, args, data): + self.process = None + self.data = data + self.args = args + self.command_channel = args.command_channel + self.trial_syslogger_stdout = None + + global NNI_TRIAL_JOB_ID + self.id = data["trialId"] + if self.id is None: + raise Exception("trial_id is not found in %s" % data) + os.environ['NNI_TRIAL_JOB_ID'] = self.id + NNI_TRIAL_JOB_ID = self.id + + # for multiple nodes. If it's None, it means single node. + self.node_id = args.node_id + if self.node_id is None: + self.name = self.id + else: + self.name = "%s_%s" % (self.id, self.node_id) + + def run(self): + # redirect trial's stdout and stderr to syslog + self.trial_syslogger_stdout = RemoteLogger(self.args.nnimanager_ip, self.args.nnimanager_port, 'trial', StdOutputType.Stdout, + self.args.log_collection, self.id, self.args.command_channel) + + nni_log(LogType.Info, "%s: start to run trial" % self.name) + + trial_working_dir = os.path.realpath(os.path.join(os.curdir, "..", "..", "trials", self.id)) + self.trial_output_dir = os.path.join(trial_working_dir, trial_output_path_name) + trial_code_dir = os.path.join(trial_working_dir, "code") + trial_nnioutput_dir = os.path.join(trial_working_dir, "nnioutput") + + environ = os.environ.copy() + environ['NNI_TRIAL_SEQ_ID'] = str(self.data["sequenceId"]) + environ['NNI_OUTPUT_DIR'] = os.path.join(trial_working_dir, "nnioutput") + environ['NNI_SYS_DIR'] = trial_working_dir + self.working_dir = trial_working_dir + + # prepare code and parameters + prepared_flag_file_name = os.path.join(trial_working_dir, "trial_prepared") + if not os.path.exists(trial_working_dir): + os.makedirs(trial_working_dir, exist_ok=True) + + os.makedirs(self.trial_output_dir, exist_ok=True) + os.makedirs(trial_nnioutput_dir, exist_ok=True) + # prepare code + os.makedirs(trial_code_dir, exist_ok=True) + with tarfile.open(os.path.join("..", "nni-code.tar.gz"), "r:gz") as tar: + tar.extractall(trial_code_dir) + + # save parameters + nni_log(LogType.Info, '%s: saving parameter %s' % (self.name, self.data["parameter"]["value"])) + parameter_file_name = os.path.join(trial_working_dir, "parameter.cfg") + with open(parameter_file_name, "w") as parameter_file: + parameter_file.write(self.data["parameter"]["value"]) + + # ready flag + with open(prepared_flag_file_name, "w") as prepared_flag_file: + prepared_flag_file.write("%s" % (int(datetime.now().timestamp() * 1000))) + + # make sure code prepared by other node. + if self.node_id is not None: + while True: + if os.path.exists(prepared_flag_file_name): + break + time.sleep(0.1) + + trial_command = self.args.trial_command + + gpuIndices = self.data.get('gpuIndices') + if (gpuIndices is not None): + trial_command = 'CUDA_VISIBLE_DEVICES="%s " %s' % (gpuIndices, trial_command) + + self.log_pipe_stdout = self.trial_syslogger_stdout.get_pipelog_reader() + self.process = Popen(trial_command, shell=True, stdout=self.log_pipe_stdout, + stderr=self.log_pipe_stdout, cwd=trial_code_dir, env=dict(environ)) + nni_log(LogType.Info, '{0}: spawns a subprocess (pid {1}) to run command: {2}'. + format(self.name, self.process.pid, shlex.split(trial_command))) + + def save_parameter_file(self, command_data): + parameters = command_data["parameters"] + file_index = int(parameters["index"]) + if file_index == 0: + parameter_file_name = "parameter.cfg" + else: + parameter_file_name = "parameter_{}.cfg".format(file_index) + parameter_file_name = os.path.join(self.working_dir, parameter_file_name) + with open(parameter_file_name, "w") as parameter_file: + nni_log(LogType.Info, '%s: saving parameter %s' % (self.name, parameters["value"])) + parameter_file.write(parameters["value"]) + + def is_running(self): + if (self.process is None): + return False + + retCode = self.process.poll() + # child worker process exits and all stdout data is read + if retCode is not None and self.log_pipe_stdout.set_process_exit() and self.log_pipe_stdout.is_read_completed == True: + # In Windows, the retCode -1 is 4294967295. It's larger than c_long, and raise OverflowError. + # So covert it to int32. + retCode = ctypes.c_long(retCode).value + nni_log(LogType.Info, '{0}: subprocess terminated. Exit code is {1}.'.format(self.name, retCode)) + + end_time = int(datetime.now().timestamp() * 1000) + end_message = { + "code": retCode, + "time": end_time, + "trial": self.id, + } + self.command_channel.send(CommandType.TrialEnd, end_message) + self.cleanup() + return False + else: + return True + + def kill(self, trial_id=None): + if trial_id == self.id or trial_id is None: + if self.process is not None: + try: + nni_log(LogType.Info, "%s: killing trial" % self.name) + for child in psutil.Process(self.process.pid).children(True): + child.kill() + self.process.kill() + except psutil.NoSuchProcess: + nni_log(LogType.Info, "kill trial %s failed: %s does not exist!" % (trial_id, self.process.pid)) + except Exception as ex: + nni_log(LogType.Error, "kill trial %s failed: %s " % (trial_id, str(ex))) + self.cleanup() + + def cleanup(self): + nni_log(LogType.Info, "%s: clean up trial" % self.name) + self.process = None + if self.log_pipe_stdout is not None: + self.log_pipe_stdout.set_process_exit() + self.log_pipe_stdout = None + if self.trial_syslogger_stdout is not None: + self.trial_syslogger_stdout.close() + self.trial_syslogger_stdout = None diff --git a/utils/third_party/nni/tools/trial_tool/trial_keeper.py b/utils/third_party/nni/tools/trial_tool/trial_keeper.py new file mode 100644 index 0000000000000000000000000000000000000000..08688973e000f7809421b0010ac9efe2bd0c4777 --- /dev/null +++ b/utils/third_party/nni/tools/trial_tool/trial_keeper.py @@ -0,0 +1,246 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import argparse +import ctypes +import json +import logging +import os +import re +import shlex +import sys +import threading +import time +from subprocess import Popen + +import pkg_resources +from pyhdfs import HdfsClient + +from .constants import (LOG_DIR, MULTI_PHASE, NNI_EXP_ID, NNI_PLATFORM, + NNI_SYS_DIR, NNI_TRIAL_JOB_ID) +from .hdfsClientUtility import (copyDirectoryToHdfs, copyHdfsDirectoryToLocal, + copyHdfsFileToLocal) +from .log_utils import LogType, RemoteLogger, StdOutputType, nni_log +from .rest_utils import rest_get, rest_post +from .url_utils import gen_parameter_meta_url, gen_send_version_url + +logger = logging.getLogger('trial_keeper') +regular = re.compile('v?(?P[0-9](\.[0-9]){0,1}).*') + +_hdfs_client = None + + +def get_hdfs_client(args): + global _hdfs_client + + if _hdfs_client is not None: + return _hdfs_client + # backward compatibility + hdfs_host = None + + if args.hdfs_host: + hdfs_host = args.hdfs_host + elif args.pai_hdfs_host: + hdfs_host = args.pai_hdfs_host + else: + return None + + if hdfs_host is not None and args.nni_hdfs_exp_dir is not None: + try: + if args.webhdfs_path: + _hdfs_client = HdfsClient(hosts='{0}:80'.format(hdfs_host), user_name=args.pai_user_name, + webhdfs_path=args.webhdfs_path, timeout=5) + else: + # backward compatibility + _hdfs_client = HdfsClient(hosts='{0}:{1}'.format(hdfs_host, '50070'), user_name=args.pai_user_name, + timeout=5) + except Exception as e: + nni_log(LogType.Error, 'Create HDFS client error: ' + str(e)) + raise e + return _hdfs_client + + +def main_loop(args): + '''main loop logic for trial keeper''' + + if not os.path.exists(LOG_DIR): + os.makedirs(LOG_DIR) + + trial_keeper_syslogger = RemoteLogger(args.nnimanager_ip, args.nnimanager_port, 'trial_keeper', + StdOutputType.Stdout, args.log_collection) + # redirect trial keeper's stdout and stderr to syslog + trial_syslogger_stdout = RemoteLogger(args.nnimanager_ip, args.nnimanager_port, 'trial', StdOutputType.Stdout, + args.log_collection) + sys.stdout = sys.stderr = trial_keeper_syslogger + hdfs_output_dir = None + + if args.hdfs_output_dir: + hdfs_output_dir = args.hdfs_output_dir + elif args.pai_hdfs_output_dir: + hdfs_output_dir = args.pai_hdfs_output_dir + + hdfs_client = get_hdfs_client(args) + + if hdfs_client is not None: + copyHdfsDirectoryToLocal(args.nni_hdfs_exp_dir, os.getcwd(), hdfs_client) + + if args.job_id_file: + with open(args.job_id_file, 'w') as job_file: + job_file.write("%d" % os.getpid()) + + # Notice: We don't appoint env, which means subprocess wil inherit current environment and that is expected behavior + log_pipe_stdout = trial_syslogger_stdout.get_pipelog_reader() + process = Popen(args.trial_command, shell=True, stdout=log_pipe_stdout, stderr=log_pipe_stdout) + nni_log(LogType.Info, 'Trial keeper spawns a subprocess (pid {0}) to run command: {1}'.format(process.pid, + shlex.split( + args.trial_command))) + + while True: + retCode = process.poll() + # child worker process exits and all stdout data is read + if retCode is not None and log_pipe_stdout.set_process_exit() and log_pipe_stdout.is_read_completed == True: + # In Windows, the retCode -1 is 4294967295. It's larger than c_long, and raise OverflowError. + # So covert it to int32. + retCode = ctypes.c_long(retCode).value + nni_log(LogType.Info, 'subprocess terminated. Exit code is {}. Quit'.format(retCode)) + if hdfs_output_dir is not None: + # Copy local directory to hdfs for OpenPAI + nni_local_output_dir = os.environ['NNI_OUTPUT_DIR'] + try: + if copyDirectoryToHdfs(nni_local_output_dir, hdfs_output_dir, hdfs_client): + nni_log(LogType.Info, + 'copy directory from {0} to {1} success!'.format(nni_local_output_dir, hdfs_output_dir)) + else: + nni_log(LogType.Info, + 'copy directory from {0} to {1} failed!'.format(nni_local_output_dir, hdfs_output_dir)) + except Exception as e: + nni_log(LogType.Error, 'HDFS copy directory got exception: ' + str(e)) + raise e + + # Exit as the retCode of subprocess(trial) + exit(retCode) + break + + time.sleep(2) + + +def trial_keeper_help_info(*args): + print('please run --help to see guidance') + + +def check_version(args): + try: + trial_keeper_version = pkg_resources.get_distribution('nni').version + except pkg_resources.ResolutionError as err: + # package nni does not exist, try nni-tool package + nni_log(LogType.Error, 'Package nni does not exist!') + os._exit(1) + if not args.nni_manager_version: + # skip version check + nni_log(LogType.Warning, 'Skipping version check!') + else: + try: + trial_keeper_version = regular.search(trial_keeper_version).group('version') + nni_log(LogType.Info, 'trial_keeper_version is {0}'.format(trial_keeper_version)) + nni_manager_version = regular.search(args.nni_manager_version).group('version') + nni_log(LogType.Info, 'nni_manager_version is {0}'.format(nni_manager_version)) + log_entry = {} + if trial_keeper_version != nni_manager_version: + nni_log(LogType.Error, 'Version does not match!') + error_message = 'NNIManager version is {0}, TrialKeeper version is {1}, NNI version does not match!'.format( + nni_manager_version, trial_keeper_version) + log_entry['tag'] = 'VCFail' + log_entry['msg'] = error_message + rest_post(gen_send_version_url(args.nnimanager_ip, args.nnimanager_port), json.dumps(log_entry), 10, + False) + os._exit(1) + else: + nni_log(LogType.Info, 'Version match!') + log_entry['tag'] = 'VCSuccess' + rest_post(gen_send_version_url(args.nnimanager_ip, args.nnimanager_port), json.dumps(log_entry), 10, + False) + except AttributeError as err: + nni_log(LogType.Error, err) + + +def is_multi_phase(): + return MULTI_PHASE and (MULTI_PHASE in ['True', 'true']) + + +def download_parameter(meta_list, args): + """ + Download parameter file to local working directory. + meta_list format is defined in paiJobRestServer.ts + example meta_list: + [ + {"experimentId":"yWFJarYa","trialId":"UpPkl","filePath":"/chec/nni/experiments/yWFJarYa/trials/UpPkl/parameter_1.cfg"}, + {"experimentId":"yWFJarYa","trialId":"aIUMA","filePath":"/chec/nni/experiments/yWFJarYa/trials/aIUMA/parameter_1.cfg"} + ] + """ + nni_log(LogType.Debug, str(meta_list)) + nni_log(LogType.Debug, + 'NNI_SYS_DIR: {}, trial Id: {}, experiment ID: {}'.format(NNI_SYS_DIR, NNI_TRIAL_JOB_ID, NNI_EXP_ID)) + nni_log(LogType.Debug, 'NNI_SYS_DIR files: {}'.format(os.listdir(NNI_SYS_DIR))) + for meta in meta_list: + if meta['experimentId'] == NNI_EXP_ID and meta['trialId'] == NNI_TRIAL_JOB_ID: + param_fp = os.path.join(NNI_SYS_DIR, os.path.basename(meta['filePath'])) + if not os.path.exists(param_fp): + hdfs_client = get_hdfs_client(args) + copyHdfsFileToLocal(meta['filePath'], param_fp, hdfs_client, override=False) + + +def fetch_parameter_file(args): + class FetchThread(threading.Thread): + def __init__(self, args): + super(FetchThread, self).__init__() + self.args = args + + def run(self): + uri = gen_parameter_meta_url(self.args.nnimanager_ip, self.args.nnimanager_port) + nni_log(LogType.Info, uri) + + while True: + res = rest_get(uri, 10) + nni_log(LogType.Debug, 'status code: {}'.format(res.status_code)) + if res.status_code == 200: + meta_list = res.json() + download_parameter(meta_list, self.args) + else: + nni_log(LogType.Warning, 'rest response: {}'.format(str(res))) + time.sleep(5) + + fetch_file_thread = FetchThread(args) + fetch_file_thread.start() + + +if __name__ == '__main__': + '''NNI Trial Keeper main function''' + PARSER = argparse.ArgumentParser() + PARSER.set_defaults(func=trial_keeper_help_info) + PARSER.add_argument('--trial_command', type=str, help='Command to launch trial process') + PARSER.add_argument('--nnimanager_ip', type=str, default='localhost', help='NNI manager rest server IP') + PARSER.add_argument('--nnimanager_port', type=str, default='8081', help='NNI manager rest server port') + PARSER.add_argument('--pai_hdfs_output_dir', type=str, help='the output dir of pai_hdfs') # backward compatibility + PARSER.add_argument('--hdfs_output_dir', type=str, help='the output dir of hdfs') + PARSER.add_argument('--pai_hdfs_host', type=str, help='the host of pai_hdfs') # backward compatibility + PARSER.add_argument('--hdfs_host', type=str, help='the host of hdfs') + PARSER.add_argument('--pai_user_name', type=str, help='the username of hdfs') + PARSER.add_argument('--nni_hdfs_exp_dir', type=str, help='nni experiment directory in hdfs') + PARSER.add_argument('--webhdfs_path', type=str, help='the webhdfs path used in webhdfs URL') + PARSER.add_argument('--nni_manager_version', type=str, help='the nni version transmitted from nniManager') + PARSER.add_argument('--log_collection', type=str, help='set the way to collect log in trialkeeper') + PARSER.add_argument('--job_id_file', type=str, help='set job id file for operating and monitoring job.') + args, unknown = PARSER.parse_known_args() + if args.trial_command is None: + exit(1) + check_version(args) + try: + if NNI_PLATFORM == 'paiYarn' and is_multi_phase(): + fetch_parameter_file(args) + main_loop(args) + except SystemExit as se: + nni_log(LogType.Info, 'NNI trial keeper exit with code {}'.format(se.code)) + os._exit(se.code) + except Exception as e: + nni_log(LogType.Error, 'Exit trial keeper with code 1 because Exception: {} is catched'.format(str(e))) + os._exit(1) diff --git a/utils/third_party/nni/tools/trial_tool/trial_runner.py b/utils/third_party/nni/tools/trial_tool/trial_runner.py new file mode 100644 index 0000000000000000000000000000000000000000..8ee5c69bf7c07f04fbad3088456a24894d2032a9 --- /dev/null +++ b/utils/third_party/nni/tools/trial_tool/trial_runner.py @@ -0,0 +1,255 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import argparse +import json +import os +import random +import re +import sys +import time +import traceback +from datetime import datetime, timedelta + +import pkg_resources + +from .gpu import collect_gpu_usage + +idle_timeout_seconds = 10 * 60 +gpu_refressh_interval_seconds = 5 +regular = re.compile('v?(?P[0-9](\.[0-9]){0,1}).*') +trial_runner_syslogger = None + + +def main_loop(args): + '''main loop logic for trial runner''' + idle_last_time = datetime.now() + gpu_refresh_last_time = datetime.now() - timedelta(minutes=1) + + try: + if args.job_pid_file: + with open(args.job_pid_file, 'w') as job_file: + job_file.write("%d" % os.getpid()) + + trials = dict() + + command_channel = args.command_channel + # command loop + while True: + command_type, command_data = command_channel.receive() + if command_type == CommandType.NewTrialJob: + trial_id = command_data["trialId"] + if trial_id in trials.keys(): + trial = trials[trial_id] + if trial.is_running(): + raise Exception('trial %s is running already, cannot start a new one' % trial.id) + else: + del trials[trial_id] + trial = Trial(args, command_data) + trial.run() + trials[trial_id] = trial + elif command_type == CommandType.KillTrialJob: + trial_id = command_data + if trial_id in trials.keys(): + trial = trials[trial_id] + trial.kill(command_data) + elif command_type == CommandType.SendTrialJobParameter: + trial_id = command_data["trialId"] + if trial_id in trials.keys(): + trial = trials[trial_id] + trial.save_parameter_file(command_data) + elif command_type is not None: + raise Exception("unknown command %s" % command_type) + + trial_list = list(trials.values()) + for trial in trial_list: + if trial is not None and trial.is_running(): + idle_last_time = datetime.now() + else: + del trials[trial.id] + + if (datetime.now() - idle_last_time).seconds > idle_timeout_seconds: + nni_log(LogType.Info, "trial runner is idle more than {0} seconds, so exit.".format( + idle_timeout_seconds)) + break + + if args.enable_gpu_collect and (datetime.now() - gpu_refresh_last_time).seconds > gpu_refressh_interval_seconds: + # collect gpu information + gpu_info = collect_gpu_usage(args.node_id) + command_channel.send(CommandType.ReportGpuInfo, gpu_info) + gpu_refresh_last_time = datetime.now() + time.sleep(0.5) + except Exception as ex: + traceback.print_exc() + raise ex + finally: + nni_log(LogType.Info, "main_loop exits.") + + trial_list = list(trials.values()) + for trial in trial_list: + trial.kill() + del trials[trial.id] + # wait to send commands + for _ in range(10): + if command_channel.sent(): + break + time.sleep(1) + command_channel.close() + + +def trial_runner_help_info(*args): + print('please run --help to see guidance') + + +def check_version(args): + try: + trial_runner_version = pkg_resources.get_distribution('nni').version + except pkg_resources.ResolutionError as err: + # package nni does not exist, try nni-tool package + nni_log(LogType.Error, 'Package nni does not exist!') + os._exit(1) + if not args.nni_manager_version: + # skip version check + nni_log(LogType.Warning, 'Skipping version check!') + else: + try: + command_channel = args.command_channel + trial_runner_version = regular.search(trial_runner_version).group('version') + nni_log(LogType.Info, '{0}: runner_version is {1}'.format(args.node_id, trial_runner_version)) + nni_manager_version = regular.search(args.nni_manager_version).group('version') + nni_log(LogType.Info, '{0}: nni_manager_version is {1}'.format(args.node_id, nni_manager_version)) + log_entry = {} + if trial_runner_version != nni_manager_version: + nni_log(LogType.Error, '{0}: Version does not match!'.format(args.node_id)) + error_message = '{0}: NNIManager version is {1}, Trial runner version is {2}, NNI version does not match!'.format( + args.node_id, nni_manager_version, trial_runner_version) + log_entry['tag'] = 'VCFail' + log_entry['msg'] = error_message + command_channel.send(CommandType.VersionCheck, log_entry) + while not command_channel.sent(): + time.sleep(1) + os._exit(1) + else: + nni_log(LogType.Info, '{0}: Version match!'.format(args.node_id)) + log_entry['tag'] = 'VCSuccess' + command_channel.send(CommandType.VersionCheck, log_entry) + except AttributeError as err: + nni_log(LogType.Error, '{0}: {1}'.format(args.node_id, err)) + +if __name__ == '__main__': + + '''NNI Trial Runner main function''' + PARSER = argparse.ArgumentParser() + PARSER.set_defaults(func=trial_runner_help_info) + PARSER.add_argument('--trial_command', type=str, help='Command to launch trial process') + PARSER.add_argument('--nnimanager_ip', type=str, help='NNI manager rest server IP') + PARSER.add_argument('--nnimanager_port', type=str, help='NNI manager rest server port') + PARSER.add_argument('--nni_manager_version', type=str, help='the nni version transmitted from nniManager') + PARSER.add_argument('--log_collection', type=str, help='set the way to collect log in trial runner') + PARSER.add_argument('--node_count', type=int, help='number of nodes, it determines how to consume command and save code file') + PARSER.add_argument('--job_pid_file', type=str, help='save trial runner process pid') + args, unknown = PARSER.parse_known_args() + + setting_file = "settings.json" + if not os.path.exists(setting_file): + setting_file = "../{}".format(setting_file) + if os.path.exists(setting_file): + with open(setting_file, 'r') as fp: + settings = json.load(fp) + print("setting is {}".format(settings)) + else: + print("not found setting file") + + args.exp_id = settings["experimentId"] + args.platform = settings["platform"] + # runner_id is unique runner in experiment + args.runner_id = os.path.basename(os.path.realpath(os.path.curdir)) + args.runner_name = "runner_"+args.runner_id + args.enable_gpu_collect = settings["enableGpuCollector"] + args.command_channel = settings["commandChannel"] + + if args.trial_command is None: + args.trial_command = settings["command"] + if args.nnimanager_ip is None: + args.nnimanager_ip = settings["nniManagerIP"] + if args.nnimanager_port is None: + args.nnimanager_port = settings["nniManagerPort"] + if args.nni_manager_version is None: + args.nni_manager_version = settings["nniManagerVersion"] + if args.log_collection is None: + args.log_collection = settings["logCollection"] + if args.node_count is None: + # default has only one node. + args.node_count = 1 + + os.environ['NNI_OUTPUT_DIR'] = os.curdir + "/nnioutput" + os.environ['NNI_PLATFORM'] = args.platform + os.environ['NNI_SYS_DIR'] = os.curdir + os.environ['NNI_EXP_ID'] = args.exp_id + os.environ['MULTI_PHASE'] = "true" + os.environ['NNI_TRIAL_JOB_ID'] = "runner" + + from .log_utils import LogType, RemoteLogger, StdOutputType, nni_log + from .trial import Trial + from .file_channel import FileChannel + from .web_channel import WebChannel + from .commands import CommandType + + is_multi_node = args.node_count > 1 + + if (is_multi_node): + # for multiple nodes, create a file to get a unique id. + while True: + node_id = random.randint(0, 10000) + unique_check_file_name = "node_%s" % (node_id) + if not os.path.exists(unique_check_file_name): + break + with open(unique_check_file_name, "w") as unique_check_file: + unique_check_file.write("%s" % (int(datetime.now().timestamp() * 1000))) + args.node_id = node_id + else: + # node id is unique in the runner + args.node_id = None + + # init command channel + command_channel = None + if args.command_channel == "file": + command_channel = FileChannel(args) + elif args.command_channel == 'aml': + from .aml_channel import AMLChannel + command_channel = AMLChannel(args) + else: + command_channel = WebChannel(args) + command_channel.open() + + nni_log(LogType.Info, "command channel is {}, actual type is {}".format(args.command_channel, type(command_channel))) + args.command_channel = command_channel + + trial_runner_syslogger = RemoteLogger(args.nnimanager_ip, args.nnimanager_port, 'runner', + StdOutputType.Stdout, args.log_collection, args.runner_name, command_channel) + sys.stdout = sys.stderr = trial_runner_syslogger + nni_log(LogType.Info, "{}: merged args is {}".format(args.node_id, args)) + + if args.trial_command is None: + nni_log(LogType.Error, "{}: no command is found.".format(args.node_id)) + os._exit(1) + check_version(args) + try: + main_loop(args) + except SystemExit as se: + nni_log(LogType.Info, '{}: NNI trial runner exit with code {}'.format(args.node_id, se.code)) + + # try best to send latest errors to server + timeout = 10 + while not command_channel.sent() and timeout > 0: + timeout -= 1 + time.sleep(1) + os._exit(se.code) + finally: + if trial_runner_syslogger is not None: + if trial_runner_syslogger.pipeReader is not None: + trial_runner_syslogger.pipeReader.set_process_exit() + trial_runner_syslogger.close() + + # the process doesn't exit even main loop exit. So exit it explictly. + os._exit(0) diff --git a/utils/third_party/nni/tools/trial_tool/url_utils.py b/utils/third_party/nni/tools/trial_tool/url_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7942c62fb5b4d8d3734ddf8c23a21b0f379d76ec --- /dev/null +++ b/utils/third_party/nni/tools/trial_tool/url_utils.py @@ -0,0 +1,19 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .constants import API_ROOT_URL, BASE_URL, STDOUT_API, NNI_TRIAL_JOB_ID, NNI_EXP_ID, VERSION_API, PARAMETER_META_API + + +def gen_send_stdout_url(ip, port): + '''Generate send stdout url''' + return '{0}:{1}{2}{3}/{4}/{5}'.format(BASE_URL.format(ip), port, API_ROOT_URL, STDOUT_API, NNI_EXP_ID, NNI_TRIAL_JOB_ID) + + +def gen_send_version_url(ip, port): + '''Generate send error url''' + return '{0}:{1}{2}{3}/{4}/{5}'.format(BASE_URL.format(ip), port, API_ROOT_URL, VERSION_API, NNI_EXP_ID, NNI_TRIAL_JOB_ID) + + +def gen_parameter_meta_url(ip, port): + '''Generate send error url''' + return '{0}:{1}{2}{3}'.format(BASE_URL.format(ip), port, API_ROOT_URL, PARAMETER_META_API) diff --git a/utils/third_party/nni/tools/trial_tool/web_channel.py b/utils/third_party/nni/tools/trial_tool/web_channel.py new file mode 100644 index 0000000000000000000000000000000000000000..87901e1163a5fbcf82a2bb05c701735a346333fc --- /dev/null +++ b/utils/third_party/nni/tools/trial_tool/web_channel.py @@ -0,0 +1,57 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import asyncio +import os +import websockets + +from .base_channel import BaseChannel +from .log_utils import LogType, nni_log + + +class WebChannel(BaseChannel): + + def __init__(self, args): + self.node_id = args.node_id + self.args = args + self.client = None + self.in_cache = b"" + self.timeout = 10 + + super(WebChannel, self).__init__(args) + + self._event_loop = None + + def _inner_open(self): + url = "ws://{}:{}".format(self.args.nnimanager_ip, self.args.nnimanager_port) + try: + connect = asyncio.wait_for(websockets.connect(url), self.timeout) + self._event_loop = asyncio.get_event_loop() + client = self._event_loop.run_until_complete(connect) + self.client = client + nni_log(LogType.Info, 'WebChannel: connected with info %s' % url) + except asyncio.TimeoutError: + nni_log(LogType.Error, 'connect to %s timeout! Please make sure NNIManagerIP configured correctly, and accessable.' % url) + os._exit(1) + + def _inner_close(self): + if self.client is not None: + self.client.close() + self.client = None + if self._event_loop.is_running(): + self._event_loop.stop() + self._event_loop = None + + def _inner_send(self, message): + loop = asyncio.new_event_loop() + loop.run_until_complete(self.client.send(message)) + + def _inner_receive(self): + messages = [] + if self.client is not None: + received = self._event_loop.run_until_complete(self.client.recv()) + # receive message is string, to get consistent result, encode it here. + self.in_cache += received.encode("utf8") + messages, self.in_cache = self._fetch_message(self.in_cache) + + return messages diff --git a/utils/third_party/nni/trial.py b/utils/third_party/nni/trial.py new file mode 100644 index 0000000000000000000000000000000000000000..cdb2b1e683ef970455c5b207b7b0f755dfc72d41 --- /dev/null +++ b/utils/third_party/nni/trial.py @@ -0,0 +1,141 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .utils import to_json +from .runtime.env_vars import trial_env_vars +from .runtime import platform + + +__all__ = [ + 'get_next_parameter', + 'get_current_parameter', + 'report_intermediate_result', + 'report_final_result', + 'get_experiment_id', + 'get_trial_id', + 'get_sequence_id' +] + + +_params = None +_experiment_id = platform.get_experiment_id() +_trial_id = platform.get_trial_id() +_sequence_id = platform.get_sequence_id() + + +def get_next_parameter(): + """ + Get the hyper paremeters generated by tuner. For a multiphase experiment, it returns a new group of hyper + parameters at each call of get_next_parameter. For a non-multiphase (multiPhase is not configured or set to False) + experiment, it returns hyper parameters only on the first call for each trial job, it returns None since second call. + This API should be called only once in each trial job of an experiment which is not specified as multiphase. + + Returns + ------- + dict + A dict object contains the hyper parameters generated by tuner, the keys of the dict are defined in + search space. Returns None if no more hyper parameters can be generated by tuner. + """ + global _params + _params = platform.get_next_parameter() + if _params is None: + return None + return _params['parameters'] + +def get_current_parameter(tag=None): + """ + Get current hyper parameters generated by tuner. It returns the same group of hyper parameters as the last + call of get_next_parameter returns. + + Parameters + ---------- + tag: str + hyper parameter key + """ + global _params + if _params is None: + return None + if tag is None: + return _params['parameters'] + return _params['parameters'][tag] + +def get_experiment_id(): + """ + Get experiment ID. + + Returns + ------- + str + Identifier of current experiment + """ + return _experiment_id + +def get_trial_id(): + """ + Get trial job ID which is string identifier of a trial job, for example 'MoXrp'. In one experiment, each trial + job has an unique string ID. + + Returns + ------- + str + Identifier of current trial job which is calling this API. + """ + return _trial_id + +def get_sequence_id(): + """ + Get trial job sequence nubmer. A sequence number is an integer value assigned to each trial job base on the + order they are submitted, incremental starting from 0. In one experiment, both trial job ID and sequence number + are unique for each trial job, they are of different data types. + + Returns + ------- + int + Sequence number of current trial job which is calling this API. + """ + return _sequence_id + +_intermediate_seq = 0 + +def report_intermediate_result(metric): + """ + Reports intermediate result to NNI. + + Parameters + ---------- + metric: + serializable object. + """ + global _intermediate_seq + assert _params or trial_env_vars.NNI_PLATFORM is None, \ + 'nni.get_next_parameter() needs to be called before report_intermediate_result' + metric = to_json({ + 'parameter_id': _params['parameter_id'] if _params else None, + 'trial_job_id': trial_env_vars.NNI_TRIAL_JOB_ID, + 'type': 'PERIODICAL', + 'sequence': _intermediate_seq, + 'value': to_json(metric) + }) + _intermediate_seq += 1 + platform.send_metric(metric) + +def report_final_result(metric): + """ + Reports final result to NNI. + + Parameters + ---------- + metric: serializable object + Usually (for built-in tuners to work), it should be a number, or + a dict with key "default" (a number), and any other extra keys. + """ + assert _params or trial_env_vars.NNI_PLATFORM is None, \ + 'nni.get_next_parameter() needs to be called before report_final_result' + metric = to_json({ + 'parameter_id': _params['parameter_id'] if _params else None, + 'trial_job_id': trial_env_vars.NNI_TRIAL_JOB_ID, + 'type': 'FINAL', + 'sequence': 0, + 'value': to_json(metric) + }) + platform.send_metric(metric) diff --git a/utils/third_party/nni/tuner.py b/utils/third_party/nni/tuner.py new file mode 100644 index 0000000000000000000000000000000000000000..4fbcc011d0c55676e9bc9c7e906d7655bb5badeb --- /dev/null +++ b/utils/third_party/nni/tuner.py @@ -0,0 +1,223 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +Tuner is an AutoML algorithm, which generates a new configuration for the next try. +A new trial will run with this configuration. + +See :class:`Tuner`' specification and ``docs/en_US/tuners.rst`` for details. +""" + +import logging + +import nni + +from .recoverable import Recoverable + +__all__ = ['Tuner'] + +_logger = logging.getLogger(__name__) + + +class Tuner(Recoverable): + """ + Tuner is an AutoML algorithm, which generates a new configuration for the next try. + A new trial will run with this configuration. + + This is the abstract base class for all tuners. + Tuning algorithms should inherit this class and override :meth:`update_search_space`, :meth:`receive_trial_result`, + as well as :meth:`generate_parameters` or :meth:`generate_multiple_parameters`. + + After initializing, NNI will first call :meth:`update_search_space` to tell tuner the feasible region, + and then call :meth:`generate_parameters` one or more times to request for hyper-parameter configurations. + + The framework will train several models with given configuration. + When one of them is finished, the final accuracy will be reported to :meth:`receive_trial_result`. + And then another configuration will be reqeusted and trained, util the whole experiment finish. + + If a tuner want's to know when a trial ends, it can also override :meth:`trial_end`. + + Tuners use *parameter ID* to track trials. + In tuner context, there is a one-to-one mapping between parameter ID and trial. + When the framework ask tuner to generate hyper-parameters for a new trial, + an ID has already been assigned and can be recorded in :meth:`generate_parameters`. + Later when the trial ends, the ID will be reported to :meth:`trial_end`, + and :meth:`receive_trial_result` if it has a final result. + Parameter IDs are unique integers. + + The type/format of search space and hyper-parameters are not limited, + as long as they are JSON-serializable and in sync with trial code. + For HPO tuners, however, there is a widely shared common interface, + which supports ``choice``, ``randint``, ``uniform``, and so on. + See ``docs/en_US/Tutorial/SearchSpaceSpec.md`` for details of this interface. + + [WIP] For advanced tuners which take advantage of trials' intermediate results, + an ``Advisor`` interface is under development. + + See Also + -------- + Builtin tuners: + :class:`~nni.algorithms.hpo.hyperopt_tuner.hyperopt_tuner.HyperoptTuner` + :class:`~nni.algorithms.hpo.evolution_tuner.evolution_tuner.EvolutionTuner` + :class:`~nni.algorithms.hpo.smac_tuner.SMACTuner` + :class:`~nni.algorithms.hpo.gridsearch_tuner.GridSearchTuner` + :class:`~nni.algorithms.hpo.networkmorphism_tuner.networkmorphism_tuner.NetworkMorphismTuner` + :class:`~nni.algorithms.hpo.metis_tuner.mets_tuner.MetisTuner` + :class:`~nni.algorithms.hpo.ppo_tuner.PPOTuner` + :class:`~nni.algorithms.hpo.gp_tuner.gp_tuner.GPTuner` + """ + + def generate_parameters(self, parameter_id, **kwargs): + """ + Abstract method which provides a set of hyper-parameters. + + This method will get called when the framework is about to launch a new trial, + if user does not override :meth:`generate_multiple_parameters`. + + The return value of this method will be received by trials via :func:`nni.get_next_parameter`. + It should fit in the search space, though the framework will not verify this. + + User code must override either this method or :meth:`generate_multiple_parameters`. + + Parameters + ---------- + parameter_id : int + Unique identifier for requested hyper-parameters. This will later be used in :meth:`receive_trial_result`. + **kwargs + Unstable parameters which should be ignored by normal users. + + Returns + ------- + any + The hyper-parameters, a dict in most cases, but could be any JSON-serializable type when needed. + + Raises + ------ + nni.NoMoreTrialError + If the search space is fully explored, tuner can raise this exception. + """ + # FIXME: some tuners raise NoMoreTrialError when they are waiting for more trial results + # we need to design a new exception for this purpose + raise NotImplementedError('Tuner: generate_parameters not implemented') + + def generate_multiple_parameters(self, parameter_id_list, **kwargs): + """ + Callback method which provides multiple sets of hyper-parameters. + + This method will get called when the framework is about to launch one or more new trials. + + If user does not override this method, it will invoke :meth:`generate_parameters` on each parameter ID. + + See :meth:`generate_parameters` for details. + + User code must override either this method or :meth:`generate_parameters`. + + Parameters + ---------- + parameter_id_list : list of int + Unique identifiers for each set of requested hyper-parameters. + These will later be used in :meth:`receive_trial_result`. + **kwargs + Unstable parameters which should be ignored by normal users. + + Returns + ------- + list + List of hyper-parameters. An empty list indicates there are no more trials. + """ + result = [] + for parameter_id in parameter_id_list: + try: + _logger.debug("generating param for %s", parameter_id) + res = self.generate_parameters(parameter_id, **kwargs) + except nni.NoMoreTrialError: + return result + result.append(res) + return result + + def receive_trial_result(self, parameter_id, parameters, value, **kwargs): + """ + Abstract method invoked when a trial reports its final result. Must override. + + This method only listens to results of algorithm-generated hyper-parameters. + Currently customized trials added from web UI will not report result to this method. + + Parameters + ---------- + parameter_id : int + Unique identifier of used hyper-parameters, same with :meth:`generate_parameters`. + parameters + Hyper-parameters generated by :meth:`generate_parameters`. + value + Result from trial (the return value of :func:`nni.report_final_result`). + **kwargs + Unstable parameters which should be ignored by normal users. + """ + raise NotImplementedError('Tuner: receive_trial_result not implemented') + + def _accept_customized_trials(self, accept=True): + # FIXME: because Tuner is designed as interface, this API should not be here + + # Enable or disable receiving results of user-added hyper-parameters. + # By default `receive_trial_result()` will only receive results of algorithm-generated hyper-parameters. + # If tuners want to receive those of customized parameters as well, they can call this function in `__init__()`. + + # pylint: disable=attribute-defined-outside-init + self._accept_customized = accept + + def trial_end(self, parameter_id, success, **kwargs): + """ + Abstract method invoked when a trial is completed or terminated. Do nothing by default. + + Parameters + ---------- + parameter_id : int + Unique identifier for hyper-parameters used by this trial. + success : bool + True if the trial successfully completed; False if failed or terminated. + **kwargs + Unstable parameters which should be ignored by normal users. + """ + + def update_search_space(self, search_space): + """ + Abstract method for updating the search space. Must override. + + Tuners are advised to support updating search space at run-time. + If a tuner can only set search space once before generating first hyper-parameters, + it should explicitly document this behaviour. + + Parameters + ---------- + search_space + JSON object defined by experiment owner. + """ + raise NotImplementedError('Tuner: update_search_space not implemented') + + def load_checkpoint(self): + """ + Internal API under revising, not recommended for end users. + """ + checkpoin_path = self.get_checkpoint_path() + _logger.info('Load checkpoint ignored by tuner, checkpoint path: %s', checkpoin_path) + + def save_checkpoint(self): + """ + Internal API under revising, not recommended for end users. + """ + checkpoin_path = self.get_checkpoint_path() + _logger.info('Save checkpoint ignored by tuner, checkpoint path: %s', checkpoin_path) + + def import_data(self, data): + """ + Internal API under revising, not recommended for end users. + """ + # Import additional data for tuning + # data: a list of dictionarys, each of which has at least two keys, 'parameter' and 'value' + pass + + def _on_exit(self): + pass + + def _on_error(self): + pass diff --git a/utils/third_party/nni/utils.py b/utils/third_party/nni/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..fe782dae2c945d62153511baa4b343c01226d5fa --- /dev/null +++ b/utils/third_party/nni/utils.py @@ -0,0 +1,319 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +import copy +import functools +from enum import Enum, unique +import json_tricks +from schema import And + +from . import parameter_expressions +from .runtime.common import init_logger +from .runtime.env_vars import dispatcher_env_vars + + +to_json = functools.partial(json_tricks.dumps, allow_nan=True) + +@unique +class OptimizeMode(Enum): + """Optimize Mode class + + if OptimizeMode is 'minimize', it means the tuner need to minimize the reward + that received from Trial. + + if OptimizeMode is 'maximize', it means the tuner need to maximize the reward + that received from Trial. + """ + Minimize = 'minimize' + Maximize = 'maximize' + + +class NodeType: + """Node Type class + """ + ROOT = 'root' + TYPE = '_type' + VALUE = '_value' + INDEX = '_index' + NAME = '_name' + + +class MetricType: + """The types of metric data + """ + FINAL = 'FINAL' + PERIODICAL = 'PERIODICAL' + REQUEST_PARAMETER = 'REQUEST_PARAMETER' + + +def split_index(params): + """ + Delete index infromation from params + """ + if isinstance(params, dict): + if NodeType.INDEX in params.keys(): + return split_index(params[NodeType.VALUE]) + result = {} + for key in params: + result[key] = split_index(params[key]) + return result + else: + return params + + +def extract_scalar_reward(value, scalar_key='default'): + """ + Extract scalar reward from trial result. + + Parameters + ---------- + value : int, float, dict + the reported final metric data + scalar_key : str + the key name that indicates the numeric number + + Raises + ------ + RuntimeError + Incorrect final result: the final result should be float/int, + or a dict which has a key named "default" whose value is float/int. + """ + if isinstance(value, (float, int)): + reward = value + elif isinstance(value, dict) and scalar_key in value and isinstance(value[scalar_key], (float, int)): + reward = value[scalar_key] + else: + raise RuntimeError('Incorrect final result: the final result should be float/int, ' \ + 'or a dict which has a key named "default" whose value is float/int.') + return reward + + +def extract_scalar_history(trial_history, scalar_key='default'): + """ + Extract scalar value from a list of intermediate results. + + Parameters + ---------- + trial_history : list + accumulated intermediate results of a trial + scalar_key : str + the key name that indicates the numeric number + + Raises + ------ + RuntimeError + Incorrect final result: the final result should be float/int, + or a dict which has a key named "default" whose value is float/int. + """ + return [extract_scalar_reward(ele, scalar_key) for ele in trial_history] + + +def convert_dict2tuple(value): + """ + convert dict type to tuple to solve unhashable problem. + """ + if isinstance(value, dict): + for _keys in value: + value[_keys] = convert_dict2tuple(value[_keys]) + return tuple(sorted(value.items())) + return value + + +def init_dispatcher_logger(): + """ + Initialize dispatcher logging configuration + """ + logger_file_path = 'dispatcher.log' + if dispatcher_env_vars.NNI_LOG_DIRECTORY is not None: + logger_file_path = os.path.join(dispatcher_env_vars.NNI_LOG_DIRECTORY, logger_file_path) + init_logger(logger_file_path, dispatcher_env_vars.NNI_LOG_LEVEL) + + +def json2space(x, oldy=None, name=NodeType.ROOT): + """ + Change search space from json format to hyperopt format + + """ + y = list() + if isinstance(x, dict): + if NodeType.TYPE in x.keys(): + _type = x[NodeType.TYPE] + name = name + '-' + _type + if _type == 'choice': + if oldy is not None: + _index = oldy[NodeType.INDEX] + y += json2space(x[NodeType.VALUE][_index], + oldy[NodeType.VALUE], name=name+'[%d]' % _index) + else: + y += json2space(x[NodeType.VALUE], None, name=name) + y.append(name) + else: + for key in x.keys(): + y += json2space(x[key], oldy[key] if oldy else None, name+"[%s]" % str(key)) + elif isinstance(x, list): + for i, x_i in enumerate(x): + if isinstance(x_i, dict): + if NodeType.NAME not in x_i.keys(): + raise RuntimeError('\'_name\' key is not found in this nested search space.') + y += json2space(x_i, oldy[i] if oldy else None, name + "[%d]" % i) + return y + + +def json2parameter(x, is_rand, random_state, oldy=None, Rand=False, name=NodeType.ROOT): + """ + Json to pramaters. + + """ + if isinstance(x, dict): + if NodeType.TYPE in x.keys(): + _type = x[NodeType.TYPE] + _value = x[NodeType.VALUE] + name = name + '-' + _type + Rand |= is_rand[name] + if Rand is True: + if _type == 'choice': + _index = random_state.randint(len(_value)) + y = { + NodeType.INDEX: _index, + NodeType.VALUE: json2parameter( + x[NodeType.VALUE][_index], + is_rand, + random_state, + None, + Rand, + name=name+"[%d]" % _index + ) + } + else: + y = getattr(parameter_expressions, _type)(*(_value + [random_state])) + else: + y = copy.deepcopy(oldy) + else: + y = dict() + for key in x.keys(): + y[key] = json2parameter( + x[key], + is_rand, + random_state, + oldy[key] if oldy else None, + Rand, + name + "[%s]" % str(key) + ) + elif isinstance(x, list): + y = list() + for i, x_i in enumerate(x): + if isinstance(x_i, dict): + if NodeType.NAME not in x_i.keys(): + raise RuntimeError('\'_name\' key is not found in this nested search space.') + y.append(json2parameter( + x_i, + is_rand, + random_state, + oldy[i] if oldy else None, + Rand, + name + "[%d]" % i + )) + else: + y = copy.deepcopy(x) + return y + +def merge_parameter(base_params, override_params): + """ + Update the parameters in ``base_params`` with ``override_params``. + Can be useful to override parsed command line arguments. + + Parameters + ---------- + base_params : namespace or dict + Base parameters. A key-value mapping. + override_params : dict or None + Parameters to override. Usually the parameters got from ``get_next_parameters()``. + When it is none, nothing will happen. + + Returns + ------- + namespace or dict + The updated ``base_params``. Note that ``base_params`` will be updated inplace. The return value is + only for convenience. + """ + if override_params is None: + return base_params + is_dict = isinstance(base_params, dict) + for k, v in override_params.items(): + if is_dict: + if k not in base_params: + raise ValueError('Key \'%s\' not found in base parameters.' % k) + if type(base_params[k]) != type(v) and base_params[k] is not None: + raise TypeError('Expected \'%s\' in override parameters to have type \'%s\', but found \'%s\'.' % + (k, type(base_params[k]), type(v))) + base_params[k] = v + else: + if not hasattr(base_params, k): + raise ValueError('Key \'%s\' not found in base parameters.' % k) + if type(getattr(base_params, k)) != type(v) and getattr(base_params, k) is not None: + raise TypeError('Expected \'%s\' in override parameters to have type \'%s\', but found \'%s\'.' % + (k, type(getattr(base_params, k)), type(v))) + setattr(base_params, k, v) + return base_params + +class ClassArgsValidator(object): + """ + NNI tuners/assessors/adivisors accept a `classArgs` parameter in experiment configuration file. + This ClassArgsValidator interface is used to validate the classArgs section in exeperiment + configuration file. + """ + def validate_class_args(self, **kwargs): + """ + Validate the classArgs configuration in experiment configuration file. + + Parameters + ---------- + kwargs: dict + kwargs passed to tuner/assessor/advisor constructor + + Raises: + Raise an execption if the kwargs is invalid. + """ + pass + + def choices(self, key, *args): + """ + Utility method to create a scheme to check whether the `key` is one of the `args`. + + Parameters: + ---------- + key: str + key name of the data to be validated + args: list of str + list of the choices + + Returns: Schema + -------- + A scheme to check whether the `key` is one of the `args`. + """ + return And(lambda n: n in args, error='%s should be in [%s]!' % (key, str(args))) + + def range(self, key, keyType, start, end): + """ + Utility method to create a schema to check whether the `key` is in the range of [start, end]. + + Parameters: + ---------- + key: str + key name of the data to be validated + keyType: type + python data type, such as int, float + start: type is specified by keyType + start of the range + end: type is specified by keyType + end of the range + + Returns: Schema + -------- + A scheme to check whether the `key` is in the range of [start, end]. + """ + return And( + And(keyType, error='%s should be %s type!' % (key, keyType.__name__)), + And(lambda n: start <= n <= end, error='%s should be in range of (%s, %s)!' % (key, start, end)) + ) diff --git a/utils/third_party/nni_new/__init__.py b/utils/third_party/nni_new/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..c15ce55dfa09284f4ebffea6a951633f6ec50618 --- /dev/null +++ b/utils/third_party/nni_new/__init__.py @@ -0,0 +1,26 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +try: + from .version import __version__ # pylint: disable=import-error +except ModuleNotFoundError: + __version__ = '999.dev0' + +from .runtime.log import init_logger +init_logger() + +from .runtime.env_vars import dispatcher_env_vars +from .utils import ClassArgsValidator + +if dispatcher_env_vars.SDK_PROCESS != 'dispatcher': + from .trial import * + from .smartparam import * + from .common.nas_utils import training_update + +class NoMoreTrialError(Exception): + def __init__(self, ErrorInfo): + super().__init__(self) + self.errorinfo = ErrorInfo + + def __str__(self): + return self.errorinfo diff --git a/utils/third_party/nni_new/__main__.py b/utils/third_party/nni_new/__main__.py new file mode 100644 index 0000000000000000000000000000000000000000..80a7125e25877fe856e7fa59faeb6a449cf9d0a2 --- /dev/null +++ b/utils/third_party/nni_new/__main__.py @@ -0,0 +1,116 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +import argparse +import logging +import json +import base64 + +from .runtime.common import enable_multi_thread +from .runtime.msg_dispatcher import MsgDispatcher +from .tools.package_utils import create_builtin_class_instance, create_customized_class_instance + +logger = logging.getLogger('nni.main') +logger.debug('START') + +if os.environ.get('COVERAGE_PROCESS_START'): + import coverage + coverage.process_startup() + + +def main(): + parser = argparse.ArgumentParser(description='Dispatcher command line parser') + parser.add_argument('--exp_params', type=str, required=True) + args, _ = parser.parse_known_args() + + exp_params_decode = base64.b64decode(args.exp_params).decode('utf-8') + logger.debug('decoded exp_params: [%s]', exp_params_decode) + exp_params = json.loads(exp_params_decode) + logger.debug('exp_params json obj: [%s]', json.dumps(exp_params, indent=4)) + + if exp_params.get('deprecated', {}).get('multiThread'): + enable_multi_thread() + + if 'trainingServicePlatform' in exp_params: # config schema is v1 + from types import SimpleNamespace + from .experiment.config.convert import convert_algo + for algo_type in ['tuner', 'assessor', 'advisor']: + if algo_type in exp_params: + exp_params[algo_type] = convert_algo(algo_type, exp_params, SimpleNamespace()).json() + + if exp_params.get('advisor') is not None: + # advisor is enabled and starts to run + _run_advisor(exp_params) + else: + # tuner (and assessor) is enabled and starts to run + assert exp_params.get('tuner') is not None + tuner = _create_tuner(exp_params) + if exp_params.get('assessor') is not None: + assessor = _create_assessor(exp_params) + else: + assessor = None + dispatcher = MsgDispatcher(tuner, assessor) + + try: + dispatcher.run() + tuner._on_exit() + if assessor is not None: + assessor._on_exit() + except Exception as exception: + logger.exception(exception) + tuner._on_error() + if assessor is not None: + assessor._on_error() + raise + + +def _run_advisor(exp_params): + if exp_params.get('advisor').get('name'): + dispatcher = create_builtin_class_instance( + exp_params['advisor']['name'], + exp_params['advisor'].get('classArgs'), + 'advisors') + else: + dispatcher = create_customized_class_instance(exp_params.get('advisor')) + if dispatcher is None: + raise AssertionError('Failed to create Advisor instance') + try: + dispatcher.run() + except Exception as exception: + logger.exception(exception) + raise + + +def _create_tuner(exp_params): + if exp_params['tuner'].get('name'): + tuner = create_builtin_class_instance( + exp_params['tuner']['name'], + exp_params['tuner'].get('classArgs'), + 'tuners') + else: + tuner = create_customized_class_instance(exp_params['tuner']) + if tuner is None: + raise AssertionError('Failed to create Tuner instance') + return tuner + + +def _create_assessor(exp_params): + if exp_params['assessor'].get('name'): + assessor = create_builtin_class_instance( + exp_params['assessor']['name'], + exp_params['assessor'].get('classArgs'), + 'assessors') + else: + assessor = create_customized_class_instance(exp_params['assessor']) + if assessor is None: + raise AssertionError('Failed to create Assessor instance') + return assessor + + +if __name__ == '__main__': + try: + main() + except Exception as exception: + logger.exception(exception) + raise diff --git a/utils/third_party/nni_new/__pycache__/__init__.cpython-38.pyc b/utils/third_party/nni_new/__pycache__/__init__.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4df3b9572a8f8dec0004f5e18cc7ceee1592bc67 Binary files /dev/null and b/utils/third_party/nni_new/__pycache__/__init__.cpython-38.pyc differ diff --git a/utils/third_party/nni_new/__pycache__/parameter_expressions.cpython-38.pyc b/utils/third_party/nni_new/__pycache__/parameter_expressions.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..50226fbafe9b06c2009b3d51018b87feb02aa7a6 Binary files /dev/null and b/utils/third_party/nni_new/__pycache__/parameter_expressions.cpython-38.pyc differ diff --git a/utils/third_party/nni_new/__pycache__/smartparam.cpython-38.pyc b/utils/third_party/nni_new/__pycache__/smartparam.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..755cc95f8a0703c1e27d9b100accb3eca4dde16a Binary files /dev/null and b/utils/third_party/nni_new/__pycache__/smartparam.cpython-38.pyc differ diff --git a/utils/third_party/nni_new/__pycache__/trial.cpython-38.pyc b/utils/third_party/nni_new/__pycache__/trial.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5aee350275156fd58ed4ada6b1725d8b3ab6312a Binary files /dev/null and b/utils/third_party/nni_new/__pycache__/trial.cpython-38.pyc differ diff --git a/utils/third_party/nni_new/__pycache__/utils.cpython-38.pyc b/utils/third_party/nni_new/__pycache__/utils.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..de8ce033554adca97081be70fc0dabd1092d16bd Binary files /dev/null and b/utils/third_party/nni_new/__pycache__/utils.cpython-38.pyc differ diff --git a/utils/third_party/nni_new/algorithms/compression/pytorch/auto_compress/__init__.py b/utils/third_party/nni_new/algorithms/compression/pytorch/auto_compress/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..8026bdd94c3b100fb9ee9d568b095ce4a702d4a9 --- /dev/null +++ b/utils/third_party/nni_new/algorithms/compression/pytorch/auto_compress/__init__.py @@ -0,0 +1,6 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .experiment import AutoCompressionExperiment +from .interface import AbstractAutoCompressionModule +from .utils import AutoCompressionSearchSpaceGenerator diff --git a/utils/third_party/nni_new/algorithms/compression/pytorch/auto_compress/auto_compress_engine.py b/utils/third_party/nni_new/algorithms/compression/pytorch/auto_compress/auto_compress_engine.py new file mode 100644 index 0000000000000000000000000000000000000000..b1af71ec7fbf31ee3e3a28976aa6eb0f2fabfbfa --- /dev/null +++ b/utils/third_party/nni_new/algorithms/compression/pytorch/auto_compress/auto_compress_engine.py @@ -0,0 +1,164 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +from typing import Optional, Callable + +import json_tricks +from torch.nn import Module +from torch.optim import Optimizer + +import nni +from .constants import PRUNER_DICT, QUANTIZER_DICT +from .interface import BaseAutoCompressionEngine, AbstractAutoCompressionModule +from .utils import import_ + +_logger = logging.getLogger(__name__) +_logger.setLevel(logging.INFO) + +class AutoCompressionEngine(BaseAutoCompressionEngine): + @classmethod + def __convert_compact_pruner_params_to_config_list(cls, compact_config: dict) -> list: + config_dict = {} + for key, value in compact_config.items(): + _, op_types, op_names, var_name = key.split('::') + config_dict.setdefault((op_types, op_names), {}) + config_dict[(op_types, op_names)][var_name] = value + + config_list = [] + for key, config in config_dict.items(): + op_types, op_names = key + op_types = op_types.split(':') if op_types else [] + op_names = op_names.split(':') if op_names else [] + if op_types: + config['op_types'] = op_types + if op_names: + config['op_names'] = op_names + if 'op_types' in config or 'op_names' in config: + config_list.append(config) + + return config_list + + @classmethod + def __convert_compact_quantizer_params_to_config_list(cls, compact_config: dict) -> list: + config_dict = {} + for key, value in compact_config.items(): + _, quant_types, op_types, op_names, var_name = key.split('::') + config_dict.setdefault((quant_types, op_types, op_names), {}) + config_dict[(quant_types, op_types, op_names)][var_name] = value + + config_list = [] + for key, config in config_dict.items(): + quant_types, op_types, op_names = key + quant_types = quant_types.split(':') + op_types = op_types.split(':') + op_names = op_names.split(':') + if quant_types: + config['quant_types'] = quant_types + else: + continue + if op_types: + config['op_types'] = op_types + if op_names: + config['op_names'] = op_names + if 'op_types' in config or 'op_names' in config: + config_list.append(config) + + return config_list + + @classmethod + def _convert_compact_params_to_config_list(cls, compressor_type: str, compact_config: dict) -> list: + func_dict = { + 'pruner': cls.__convert_compact_pruner_params_to_config_list, + 'quantizer': cls.__convert_compact_quantizer_params_to_config_list + } + return func_dict[compressor_type](compact_config) + + @classmethod + def __compress_pruning(cls, algorithm_name: str, + model: Module, + config_list: list, + optimizer_factory: Optional[Callable], + criterion: Optional[Callable], + sparsifying_trainer: Optional[Callable[[Module, Optimizer, Callable, int], None]], + finetuning_trainer: Optional[Callable[[Module, Optimizer, Callable, int], None]], + finetuning_epochs: int, + **compressor_parameter_dict) -> Module: + if algorithm_name in ['level', 'l1', 'l2', 'fpgm']: + pruner = PRUNER_DICT[algorithm_name](model, config_list, **compressor_parameter_dict) + elif algorithm_name in ['slim', 'taylorfo', 'apoz', 'mean_activation']: + optimizer = None if optimizer_factory is None else optimizer_factory(model.parameters()) + pruner = PRUNER_DICT[algorithm_name](model, config_list, optimizer, sparsifying_trainer, criterion, **compressor_parameter_dict) + else: + raise ValueError('Unsupported compression algorithm: {}.'.format(algorithm_name)) + compressed_model = pruner.compress() + if finetuning_trainer is not None: + # note that in pruning process, finetuning will use an un-patched optimizer + optimizer = optimizer_factory(compressed_model.parameters()) + for i in range(finetuning_epochs): + finetuning_trainer(compressed_model, optimizer, criterion, i) + pruner.get_pruned_weights() + return compressed_model + + @classmethod + def __compress_quantization(cls, algorithm_name: str, + model: Module, + config_list: list, + optimizer_factory: Optional[Callable], + criterion: Optional[Callable], + sparsifying_trainer: Optional[Callable[[Module, Optimizer, Callable, int], None]], + finetuning_trainer: Optional[Callable[[Module, Optimizer, Callable, int], None]], + finetuning_epochs: int, + **compressor_parameter_dict) -> Module: + optimizer = None if optimizer_factory is None else optimizer_factory(model.parameters()) + quantizer = QUANTIZER_DICT[algorithm_name](model, config_list, optimizer, **compressor_parameter_dict) + compressed_model = quantizer.compress() + if finetuning_trainer is not None: + # note that in quantization process, finetuning will use a patched optimizer + for i in range(finetuning_epochs): + finetuning_trainer(compressed_model, optimizer, criterion, i) + return compressed_model + + @classmethod + def _compress(cls, compressor_type: str, + algorithm_name: str, + model: Module, + config_list: list, + optimizer_factory: Optional[Callable], + criterion: Optional[Callable], + sparsifying_trainer: Optional[Callable[[Module, Optimizer, Callable, int], None]], + finetuning_trainer: Optional[Callable[[Module, Optimizer, Callable, int], None]], + finetuning_epochs: int, + **compressor_parameter_dict) -> Module: + func_dict = { + 'pruner': cls.__compress_pruning, + 'quantizer': cls.__compress_quantization + } + _logger.info('%s compressor config_list:\n%s', algorithm_name, json_tricks.dumps(config_list, indent=4)) + compressed_model = func_dict[compressor_type](algorithm_name, model, config_list, optimizer_factory, criterion, sparsifying_trainer, + finetuning_trainer, finetuning_epochs, **compressor_parameter_dict) + return compressed_model + + @classmethod + def trial_execute_compress(cls, module_name): + auto_compress_module: AbstractAutoCompressionModule = import_(module_name) + + algorithm_config = nni.get_next_parameter()['algorithm_name'] + algorithm_name = algorithm_config['_name'] + compact_config = {k: v for k, v in algorithm_config.items() if k.startswith('config_list::')} + parameter_dict = {k.split('parameter::')[1]: v for k, v in algorithm_config.items() if k.startswith('parameter::')} + + compressor_type = 'quantizer' if algorithm_name in QUANTIZER_DICT else 'pruner' + + config_list = cls._convert_compact_params_to_config_list(compressor_type, compact_config) + + model, evaluator = auto_compress_module.model(), auto_compress_module.evaluator() + optimizer_factory, criterion = auto_compress_module.optimizer_factory(), auto_compress_module.criterion() + sparsifying_trainer = auto_compress_module.sparsifying_trainer(algorithm_name) + finetuning_trainer = auto_compress_module.post_compress_finetuning_trainer(algorithm_name) + finetuning_epochs = auto_compress_module.post_compress_finetuning_epochs(algorithm_name) + + compressed_model = cls._compress(compressor_type, algorithm_name, model, config_list, optimizer_factory, + criterion, sparsifying_trainer, finetuning_trainer, finetuning_epochs, **parameter_dict) + + nni.report_final_result(evaluator(compressed_model)) diff --git a/utils/third_party/nni_new/algorithms/compression/pytorch/auto_compress/constants.py b/utils/third_party/nni_new/algorithms/compression/pytorch/auto_compress/constants.py new file mode 100644 index 0000000000000000000000000000000000000000..bd36871cd8ee0656b9c072d8396fae46a70a4768 --- /dev/null +++ b/utils/third_party/nni_new/algorithms/compression/pytorch/auto_compress/constants.py @@ -0,0 +1,25 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from ..pruning import LevelPruner, SlimPruner, L1FilterPruner, L2FilterPruner, FPGMPruner, TaylorFOWeightFilterPruner, \ + ActivationAPoZRankFilterPruner, ActivationMeanRankFilterPruner +from ..quantization.quantizers import NaiveQuantizer, QAT_Quantizer, DoReFaQuantizer, BNNQuantizer + + +PRUNER_DICT = { + 'level': LevelPruner, + 'slim': SlimPruner, + 'l1': L1FilterPruner, + 'l2': L2FilterPruner, + 'fpgm': FPGMPruner, + 'taylorfo': TaylorFOWeightFilterPruner, + 'apoz': ActivationAPoZRankFilterPruner, + 'mean_activation': ActivationMeanRankFilterPruner +} + +QUANTIZER_DICT = { + 'naive': NaiveQuantizer, + 'qat': QAT_Quantizer, + 'dorefa': DoReFaQuantizer, + 'bnn': BNNQuantizer +} diff --git a/utils/third_party/nni_new/algorithms/compression/pytorch/auto_compress/experiment.py b/utils/third_party/nni_new/algorithms/compression/pytorch/auto_compress/experiment.py new file mode 100644 index 0000000000000000000000000000000000000000..6859e60a128e9e41d29a1c39b557c4f120303326 --- /dev/null +++ b/utils/third_party/nni_new/algorithms/compression/pytorch/auto_compress/experiment.py @@ -0,0 +1,70 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import inspect +from pathlib import Path, PurePath +from typing import overload, Union, List + +from numpy import tri + +from nni.experiment import Experiment, ExperimentConfig +from nni.algorithms.compression.pytorch.auto_compress.interface import AbstractAutoCompressionModule + + +class AutoCompressionExperiment(Experiment): + + @overload + def __init__(self, auto_compress_module: AbstractAutoCompressionModule, config: ExperimentConfig) -> None: + """ + Prepare an experiment. + + Use `Experiment.run()` to launch it. + + Parameters + ---------- + auto_compress_module + The module provided by the user implements the `AbstractAutoCompressionModule` interfaces. + Remember put the module file under `trial_code_directory`. + config + Experiment configuration. + """ + ... + + @overload + def __init__(self, auto_compress_module: AbstractAutoCompressionModule, training_service: Union[str, List[str]]) -> None: + """ + Prepare an experiment, leaving configuration fields to be set later. + + Example usage:: + + experiment = Experiment(auto_compress_module, 'remote') + experiment.config.trial_command = 'python3 trial.py' + experiment.config.machines.append(RemoteMachineConfig(ip=..., user_name=...)) + ... + experiment.run(8080) + + Parameters + ---------- + auto_compress_module + The module provided by the user implements the `AbstractAutoCompressionModule` interfaces. + Remember put the module file under `trial_code_directory`. + training_service + Name of training service. + Supported value: "local", "remote", "openpai", "aml", "kubeflow", "frameworkcontroller", "adl" and hybrid training service. + """ + ... + + def __init__(self, auto_compress_module: AbstractAutoCompressionModule, config=None, training_service=None): + super().__init__(config, training_service) + + self.module_file_path = str(PurePath(inspect.getfile(auto_compress_module))) + self.module_name = auto_compress_module.__name__ + + def start(self, port: int, debug: bool) -> None: + trial_code_directory = str(PurePath(Path(self.config.trial_code_directory).absolute())) + '/' + assert self.module_file_path.startswith(trial_code_directory), 'The file path of the user-provided module should under trial_code_directory.' + relative_module_path = self.module_file_path.split(trial_code_directory)[1] + # only support linux, need refactor? + command = 'python3 -m nni.algorithms.compression.pytorch.auto_compress.trial_entry --module_file_name {} --module_class_name {}' + self.config.trial_command = command.format(relative_module_path, self.module_name) + super().start(port=port, debug=debug) diff --git a/utils/third_party/nni_new/algorithms/compression/pytorch/auto_compress/interface.py b/utils/third_party/nni_new/algorithms/compression/pytorch/auto_compress/interface.py new file mode 100644 index 0000000000000000000000000000000000000000..208d0ef6a15369b32c97c0df609f973c650697e4 --- /dev/null +++ b/utils/third_party/nni_new/algorithms/compression/pytorch/auto_compress/interface.py @@ -0,0 +1,122 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from abc import ABC, abstractmethod +from typing import Optional, Callable, Iterable + +from torch.nn import Module +from torch.optim import Optimizer + + +class BaseAutoCompressionEngine(ABC): + @classmethod + @abstractmethod + def trial_execute_compress(cls): + """ + Execute the compressing trial. + """ + pass + + +class AbstractAutoCompressionModule(ABC): + """ + The abstract container that user need to implement. + """ + @classmethod + @abstractmethod + def model(cls) -> Module: + """ + Returns + ------- + torch.nn.Module + Model to be compress. + """ + pass + + @classmethod + @abstractmethod + def evaluator(cls) -> Callable[[Module], float]: + """ + Returns + ------- + function + The function used to evaluate the compressed model, return a scalar. + """ + pass + + @classmethod + @abstractmethod + def optimizer_factory(cls) -> Optional[Callable[[Iterable], Optimizer]]: + """ + Returns + ------- + Optional[Callable[[Iterable], Optimizer]] + Optimizer factory function. Input is a iterable value, i.e. `model.parameters()`. + Output is the `torch.optim.Optimizer` instance. + """ + pass + + @classmethod + @abstractmethod + def criterion(cls) -> Optional[Callable]: + """ + Returns + ------- + Optional[Callable] + The criterion function used to train the model. + """ + pass + + @classmethod + @abstractmethod + def sparsifying_trainer(cls, compress_algorithm_name: str) -> Optional[Callable[[Module, Optimizer, Callable, int], None]]: + """ + The trainer is used in sparsifying process. + + Parameters + ---------- + compress_algorithm_name: str + The name of pruner and quantizer, i.e. 'level', 'l1', 'qat'. + + Returns + ------- + Optional[Callable[[Module, Optimizer, Callable, int], None]] + Used to train model in compress stage, include `model, optimizer, criterion, current_epoch` as function arguments. + """ + pass + + @classmethod + @abstractmethod + def post_compress_finetuning_trainer(cls, compress_algorithm_name: str) -> Optional[Callable[[Module, Optimizer, Callable, int], None]]: + """ + The trainer is used in post-compress finetuning process. + + Parameters + ---------- + compress_algorithm_name: str + The name of pruner and quantizer, i.e. 'level', 'l1', 'qat'. + + Returns + ------- + Optional[Callable[[Module, Optimizer, Callable, int], None]] + Used to train model in finetune stage, include `model, optimizer, criterion, current_epoch` as function arguments. + """ + pass + + @classmethod + @abstractmethod + def post_compress_finetuning_epochs(cls, compress_algorithm_name: str) -> int: + """ + The epochs in post-compress finetuning process. + + Parameters + ---------- + compress_algorithm_name: str + The name of pruner and quantizer, i.e. 'level', 'l1', 'qat'. + + Returns + ------- + int + The finetuning epoch number. + """ + pass diff --git a/utils/third_party/nni_new/algorithms/compression/pytorch/auto_compress/trial_entry.py b/utils/third_party/nni_new/algorithms/compression/pytorch/auto_compress/trial_entry.py new file mode 100644 index 0000000000000000000000000000000000000000..c502f4a1854df5388c2f451fec42b6a7b05af903 --- /dev/null +++ b/utils/third_party/nni_new/algorithms/compression/pytorch/auto_compress/trial_entry.py @@ -0,0 +1,21 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +Entrypoint for trials. +""" +import argparse +from pathlib import Path +import re + +from .auto_compress_engine import AutoCompressionEngine + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='trial entry for auto compression.') + parser.add_argument('--module_file_name', required=True, dest='module_file_name', help='the path of auto compression module file') + parser.add_argument('--module_class_name', required=True, dest='module_class_name', help='the name of auto compression module') + args = parser.parse_args() + + module_name = Path(args.module_file_name).as_posix() + module_name = re.sub(re.escape('.py') + '$', '', module_name).replace('/', '.') + '.' + args.module_class_name + AutoCompressionEngine.trial_execute_compress(module_name) diff --git a/utils/third_party/nni_new/algorithms/compression/pytorch/auto_compress/utils.py b/utils/third_party/nni_new/algorithms/compression/pytorch/auto_compress/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..fe46db63e9c1b23d7e0c5766e6a89b01e7053fb9 --- /dev/null +++ b/utils/third_party/nni_new/algorithms/compression/pytorch/auto_compress/utils.py @@ -0,0 +1,116 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from typing import Any + +from .constants import PRUNER_DICT, QUANTIZER_DICT + + +class AutoCompressionSearchSpaceGenerator: + """ + For convenient generation of search space that can be used by tuner. + """ + + def __init__(self): + self.algorithm_choice_list = [] + + def add_config(self, algorithm_name: str, config_list: list, **algo_kwargs): + """ + This function used for distinguish algorithm type is pruning or quantization. + Then call `self._add_pruner_config()` or `self._add_quantizer_config()`. + """ + if algorithm_name in PRUNER_DICT: + self._add_pruner_config(algorithm_name, config_list, **algo_kwargs) + if algorithm_name in QUANTIZER_DICT: + self._add_quantizer_config(algorithm_name, config_list, **algo_kwargs) + + def _add_pruner_config(self, pruner_name: str, config_list: list, **algo_kwargs): + """ + Parameters + ---------- + pruner_name + Supported pruner name: 'level', 'slim', 'l1', 'l2', 'fpgm', 'taylorfo', 'apoz', 'mean_activation'. + config_list + Except 'op_types' and 'op_names', other config value can be written as `{'_type': ..., '_value': ...}`. + **algo_kwargs + The additional pruner parameters except 'model', 'config_list', 'optimizer', 'trainer', 'criterion'. + i.e., you can set `statistics_batch_num={'_type': 'choice', '_value': [1, 2, 3]}` in TaylorFOWeightFilterPruner or just `statistics_batch_num=1`. + """ + sub_search_space = {'_name': pruner_name} + for config in config_list: + op_types = config.pop('op_types', []) + op_names = config.pop('op_names', []) + key_prefix = 'config_list::{}::{}'.format(':'.join(op_types), ':'.join(op_names)) + for var_name, var_search_space in config.items(): + sub_search_space['{}::{}'.format(key_prefix, var_name)] = self._wrap_single_value(var_search_space) + for parameter_name, parameter_search_space in algo_kwargs.items(): + key_prefix = 'parameter' + sub_search_space['{}::{}'.format(key_prefix, parameter_name)] = self._wrap_single_value(parameter_search_space) + self.algorithm_choice_list.append(sub_search_space) + + def _add_quantizer_config(self, quantizer_name: str, config_list: list, **algo_kwargs): + """ + Parameters + ---------- + quantizer_name + Supported pruner name: 'naive', 'qat', 'dorefa', 'bnn'. + config_list + Except 'quant_types', 'op_types' and 'op_names', other config value can be written as `{'_type': ..., '_value': ...}`. + **algo_kwargs + The additional pruner parameters except 'model', 'config_list', 'optimizer'. + """ + sub_search_space = {'_name': quantizer_name} + for config in config_list: + quant_types = config.pop('quant_types', []) + op_types = config.pop('op_types', []) + op_names = config.pop('op_names', []) + key_prefix = 'config_list::{}::{}::{}'.format(':'.join(quant_types), ':'.join(op_types), ':'.join(op_names)) + for var_name, var_search_space in config.items(): + sub_search_space['{}::{}'.format(key_prefix, var_name)] = self._wrap_single_value(var_search_space) + for parameter_name, parameter_search_space in algo_kwargs.items(): + key_prefix = 'parameter' + sub_search_space['{}::{}'.format(key_prefix, parameter_name)] = self._wrap_single_value(parameter_search_space) + self.algorithm_choice_list.append(sub_search_space) + + def dumps(self) -> dict: + """ + Dump the search space as a dict. + """ + search_space = { + 'algorithm_name': { + '_type': 'choice', + '_value': self.algorithm_choice_list + } + } + return search_space + + @classmethod + def loads(cls, search_space: dict): + """ + Return a AutoCompressionSearchSpaceGenerator instance load from a search space dict. + """ + generator = AutoCompressionSearchSpaceGenerator() + generator.algorithm_choice_list = search_space['algorithm_name']['_value'] + return generator + + def _wrap_single_value(self, value) -> dict: + if not isinstance(value, dict): + converted_value = { + '_type': 'choice', + '_value': [value] + } + elif '_type' not in value: + converted_value = {} + for k, v in value.items(): + converted_value[k] = self._wrap_single_value(v) + else: + converted_value = value + return converted_value + + +def import_(target: str, allow_none: bool = False) -> Any: + if target is None: + return None + path, identifier = target.rsplit('.', 1) + module = __import__(path, globals(), locals(), [identifier]) + return getattr(module, identifier) diff --git a/utils/third_party/nni_new/algorithms/compression/pytorch/pruning/__init__.py b/utils/third_party/nni_new/algorithms/compression/pytorch/pruning/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..f93cfaeed13b9882ee1c33106ca85af31d148425 --- /dev/null +++ b/utils/third_party/nni_new/algorithms/compression/pytorch/pruning/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .finegrained_pruning_masker import * +from .structured_pruning_masker import * +from .one_shot_pruner import * +from .iterative_pruner import * +# from .lottery_ticket import LotteryTicketPruner +# from .simulated_annealing_pruner import SimulatedAnnealingPruner +# from .net_adapt_pruner import NetAdaptPruner +# from .auto_compress_pruner import AutoCompressPruner +# from .sensitivity_pruner import SensitivityPruner +# from .amc import AMCPruner diff --git a/utils/third_party/nni_new/algorithms/compression/pytorch/pruning/amc/__init__.py b/utils/third_party/nni_new/algorithms/compression/pytorch/pruning/amc/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..3c89a879c6af42ed372f38f3cc11ea1eb9eb6870 --- /dev/null +++ b/utils/third_party/nni_new/algorithms/compression/pytorch/pruning/amc/__init__.py @@ -0,0 +1,4 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .amc_pruner import AMCPruner diff --git a/utils/third_party/nni_new/algorithms/compression/pytorch/pruning/amc/amc_pruner.py b/utils/third_party/nni_new/algorithms/compression/pytorch/pruning/amc/amc_pruner.py new file mode 100644 index 0000000000000000000000000000000000000000..c2d12429d5fb5d82131b5239ef1992ef100feef6 --- /dev/null +++ b/utils/third_party/nni_new/algorithms/compression/pytorch/pruning/amc/amc_pruner.py @@ -0,0 +1,296 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +import logging +from copy import deepcopy +from argparse import Namespace +import numpy as np +import torch +from torch.utils.tensorboard import SummaryWriter + +from nni.compression.pytorch.compressor import Pruner +from .channel_pruning_env import ChannelPruningEnv +from .lib.agent import DDPG +from .lib.utils import get_output_folder + +torch.backends.cudnn.deterministic = True + +_logger = logging.getLogger(__name__) + +class AMCPruner(Pruner): + """ + A pytorch implementation of AMC: AutoML for Model Compression and Acceleration on Mobile Devices. + (https://arxiv.org/pdf/1802.03494.pdf) + + Parameters: + model: nn.Module + The model to be pruned. + config_list: list + Configuration list to configure layer pruning. + Supported keys: + - op_types: operation type to be pruned + - op_names: operation name to be pruned + evaluator: function + function to evaluate the pruned model. + The prototype of the function: + >>> def evaluator(val_loader, model): + >>> ... + >>> return acc + val_loader: torch.utils.data.DataLoader + Data loader of validation dataset. + suffix: str + suffix to help you remember what experiment you ran. Default: None. + + # parameters for pruning environment + model_type: str + model type to prune, currently 'mobilenet' and 'mobilenetv2' are supported. Default: mobilenet + flops_ratio: float + preserve flops ratio. Default: 0.5 + lbound: float + minimum weight preserve ratio for each layer. Default: 0.2 + rbound: float + maximum weight preserve ratio for each layer. Default: 1.0 + reward: function + reward function type: + - acc_reward: accuracy * 0.01 + - acc_flops_reward: - (100 - accuracy) * 0.01 * np.log(flops) + Default: acc_reward + # parameters for channel pruning + n_calibration_batches: int + number of batches to extract layer information. Default: 60 + n_points_per_layer: int + number of feature points per layer. Default: 10 + channel_round: int + round channel to multiple of channel_round. Default: 8 + + # parameters for ddpg agent + hidden1: int + hidden num of first fully connect layer. Default: 300 + hidden2: int + hidden num of second fully connect layer. Default: 300 + lr_c: float + learning rate for critic. Default: 1e-3 + lr_a: float + learning rate for actor. Default: 1e-4 + warmup: int + number of episodes without training but only filling the replay memory. During warmup episodes, + random actions ares used for pruning. Default: 100 + discount: float + next Q value discount for deep Q value target. Default: 0.99 + bsize: int + minibatch size for training DDPG agent. Default: 64 + rmsize: int + memory size for each layer. Default: 100 + window_length: int + replay buffer window length. Default: 1 + tau: float + moving average for target network being used by soft_update. Default: 0.99 + # noise + init_delta: float + initial variance of truncated normal distribution + delta_decay: float + delta decay during exploration + + # parameters for training ddpg agent + max_episode_length: int + maximum episode length + output_dir: str + output directory to save log files and model files. Default: ./logs + debug: boolean + debug mode + train_episode: int + train iters each timestep. Default: 800 + epsilon: int + linear decay of exploration policy. Default: 50000 + seed: int + random seed to set for reproduce experiment. Default: None + """ + + def __init__( + self, + model, + config_list, + evaluator, + val_loader, + suffix=None, + model_type='mobilenet', + dataset='cifar10', + flops_ratio=0.5, + lbound=0.2, + rbound=1., + reward='acc_reward', + n_calibration_batches=60, + n_points_per_layer=10, + channel_round=8, + hidden1=300, + hidden2=300, + lr_c=1e-3, + lr_a=1e-4, + warmup=100, + discount=1., + bsize=64, + rmsize=100, + window_length=1, + tau=0.01, + init_delta=0.5, + delta_decay=0.99, + max_episode_length=1e9, + output_dir='./logs', + debug=False, + train_episode=800, + epsilon=50000, + seed=None): + + self.val_loader = val_loader + self.evaluator = evaluator + + if seed is not None: + np.random.seed(seed) + torch.manual_seed(seed) + torch.cuda.manual_seed(seed) + + checkpoint = deepcopy(model.state_dict()) + + super().__init__(model, config_list, optimizer=None) + + # build folder and logs + base_folder_name = '{}_{}_r{}_search'.format(model_type, dataset, flops_ratio) + if suffix is not None: + self.output_dir = os.path.join(output_dir, base_folder_name + '-' + suffix) + else: + self.output_dir = get_output_folder(output_dir, base_folder_name) + + self.env_args = Namespace( + model_type=model_type, + preserve_ratio=flops_ratio, + lbound=lbound, + rbound=rbound, + reward=reward, + n_calibration_batches=n_calibration_batches, + n_points_per_layer=n_points_per_layer, + channel_round=channel_round, + output=self.output_dir + ) + self.env = ChannelPruningEnv( + self, evaluator, val_loader, checkpoint, args=self.env_args) + _logger.info('=> Saving logs to %s', self.output_dir) + self.tfwriter = SummaryWriter(log_dir=self.output_dir) + self.text_writer = open(os.path.join(self.output_dir, 'log.txt'), 'w') + _logger.info('=> Output path: %s...', self.output_dir) + + nb_states = self.env.layer_embedding.shape[1] + nb_actions = 1 # just 1 action here + + rmsize = rmsize * len(self.env.prunable_idx) # for each layer + _logger.info('** Actual replay buffer size: %d', rmsize) + + self.ddpg_args = Namespace( + hidden1=hidden1, + hidden2=hidden2, + lr_c=lr_c, + lr_a=lr_a, + warmup=warmup, + discount=discount, + bsize=bsize, + rmsize=rmsize, + window_length=window_length, + tau=tau, + init_delta=init_delta, + delta_decay=delta_decay, + max_episode_length=max_episode_length, + debug=debug, + train_episode=train_episode, + epsilon=epsilon + ) + self.agent = DDPG(nb_states, nb_actions, self.ddpg_args) + + + def compress(self): + self.train(self.ddpg_args.train_episode, self.agent, self.env, self.output_dir) + + def train(self, num_episode, agent, env, output_dir): + agent.is_training = True + step = episode = episode_steps = 0 + episode_reward = 0. + observation = None + T = [] # trajectory + while episode < num_episode: # counting based on episode + # reset if it is the start of episode + if observation is None: + observation = deepcopy(env.reset()) + agent.reset(observation) + + # agent pick action ... + if episode <= self.ddpg_args.warmup: + action = agent.random_action() + # action = sample_from_truncated_normal_distribution(lower=0., upper=1., mu=env.preserve_ratio, sigma=0.5) + else: + action = agent.select_action(observation, episode=episode) + + # env response with next_observation, reward, terminate_info + observation2, reward, done, info = env.step(action) + + T.append([reward, deepcopy(observation), deepcopy(observation2), action, done]) + + # fix-length, never reach here + # if max_episode_length and episode_steps >= max_episode_length - 1: + # done = True + + # [optional] save intermideate model + if num_episode / 3 <= 1 or episode % int(num_episode / 3) == 0: + agent.save_model(output_dir) + + # update + step += 1 + episode_steps += 1 + episode_reward += reward + observation = deepcopy(observation2) + + if done: # end of episode + _logger.info( + '#%d: episode_reward: %.4f acc: %.4f, ratio: %.4f', + episode, episode_reward, + info['accuracy'], + info['compress_ratio'] + ) + self.text_writer.write( + '#{}: episode_reward:{:.4f} acc: {:.4f}, ratio: {:.4f}\n'.format( + episode, episode_reward, + info['accuracy'], + info['compress_ratio'] + ) + ) + final_reward = T[-1][0] + # print('final_reward: {}'.format(final_reward)) + # agent observe and update policy + for _, s_t, s_t1, a_t, done in T: + agent.observe(final_reward, s_t, s_t1, a_t, done) + if episode > self.ddpg_args.warmup: + agent.update_policy() + + #agent.memory.append( + # observation, + # agent.select_action(observation, episode=episode), + # 0., False + #) + + # reset + observation = None + episode_steps = 0 + episode_reward = 0. + episode += 1 + T = [] + + self.tfwriter.add_scalar('reward/last', final_reward, episode) + self.tfwriter.add_scalar('reward/best', env.best_reward, episode) + self.tfwriter.add_scalar('info/accuracy', info['accuracy'], episode) + self.tfwriter.add_scalar('info/compress_ratio', info['compress_ratio'], episode) + self.tfwriter.add_text('info/best_policy', str(env.best_strategy), episode) + # record the preserve rate for each layer + for i, preserve_rate in enumerate(env.strategy): + self.tfwriter.add_scalar('preserve_rate/{}'.format(i), preserve_rate, episode) + + self.text_writer.write('best reward: {}\n'.format(env.best_reward)) + self.text_writer.write('best policy: {}\n'.format(env.best_strategy)) + self.text_writer.close() diff --git a/utils/third_party/nni_new/algorithms/compression/pytorch/pruning/amc/channel_pruning_env.py b/utils/third_party/nni_new/algorithms/compression/pytorch/pruning/amc/channel_pruning_env.py new file mode 100644 index 0000000000000000000000000000000000000000..428f7e7532da931abcce5d71ac0847c7234b30aa --- /dev/null +++ b/utils/third_party/nni_new/algorithms/compression/pytorch/pruning/amc/channel_pruning_env.py @@ -0,0 +1,543 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +import logging +import time +import math +import copy +import numpy as np +import torch +import torch.nn as nn + +from nni.compression.pytorch.compressor import PrunerModuleWrapper +from .. import AMCWeightMasker + +_logger = logging.getLogger(__name__) + +# for pruning +def acc_reward(net, acc, flops): + return acc * 0.01 + + +def acc_flops_reward(net, acc, flops): + error = (100 - acc) * 0.01 + return -error * np.log(flops) + + +class ChannelPruningEnv: + """ + Env for channel pruning search. + This class is used to prune model using specified pruner. It prunes one layer when + step() is called. When the last layer is pruned, it evaluate the pruned model using + evaluator, and use the returned value of evaluator as reward of the episode. + + Usage: + env = ChannelPruningEnv(pruner, evaluator, val_loader, checkpoint, env_args) + episode = 0 + T = [] + while episode < num_episode: + action = agent.select_action(observation) + observation2, reward, done, info = env.step(action) + T.append([reward, deepcopy(observation), deepcopy(observation2), action, done]) + + if done: # end of episode, last layer pruned + episode += 1 + # train agent with episode data + for _, s_t, s_t1, a_t, done in T: + agent.observe(final_reward, s_t, s_t1, a_t, done) + agent.update_policy() + T = [] + + Attributes: + prunable_idx: layer indices for pruable layers, the index values are the index + of list(self.model.modules()). Pruable layers are pointwise Conv2d layers and Linear + layers. + buffer_idx: layer indices for buffer layers which refers the depthwise layers. + Each depthwise layer is always followd by a pointwise layer for both mobilenet and + mobilenetv2. The depthwise layer's filters are pruned when its next pointwise layer's + corresponding input channels are pruned. + shared_idx: layer indices for layers which share input. + For example: [[1,4], [8, 10, 15]] means layer 1 and 4 share same input, and layer + 8, 10 and 15 share another input. + layer_embedding: embeddings for each prunable layers, the embedding is used as + observation for DDPG agent. + layer_info_dict: flops and number of parameters of each layer. + min_strategy_dict: key is layer index, value is a tuple, the first value is the minimum + action of input channel, the second value is the minimum action value of output channel. + strategy_dict: key is layer index, value is a tuple, the first value is the action of input + channel, the second value is the action of output channel. + + Parameters: + pruner: Pruner + NNI Pruner instance used to prune model. + evaluator: function + function to evaluate the pruned model. + The prototype of the function: + >>> def evaluator(val_loader, model): + >>> ... + >>> return acc + val_loader: torch.utils.data.DataLoader + Data loader of validation dataset. + checkpoint: dict + checkpoint of the model to be pruned. It is used to reset model at beginning of each + episode. + args: + A Namespace object containing following arguments: + model_type: str + model type to prune, currently 'mobilenet', 'mobilenetv2' and 'resnet' are supported. + flops_ratio: float + preserve flops ratio. + lbound: float + minimum weight preserve ratio for each layer. + rbound: float + maximum weight preserve ratio for each layer. + reward: function + reward function type + + # parameters for channel pruning + n_calibration_batches: int + number of batches to extract layer information. + n_points_per_layer: int + number of feature points per layer. + channel_round: int + round channel to multiple of channel_round. + + """ + def __init__(self, pruner, evaluator, val_loader, checkpoint, args): + self.pruner = pruner + self.model = pruner.bound_model + self.checkpoint = checkpoint + self.batch_size = val_loader.batch_size + self.preserve_ratio = args.preserve_ratio + self.channel_prune_masker = AMCWeightMasker(self.model, self.pruner, args.channel_round) + + # options from args + self.args = args + self.lbound = args.lbound + self.rbound = args.rbound + + self.n_calibration_batches = args.n_calibration_batches + self.n_points_per_layer = args.n_points_per_layer + self.channel_round = args.channel_round + + # sanity check + assert self.preserve_ratio > self.lbound, 'Error! You can not achieve preserve_ratio smaller than lbound!' + + # prepare data + self._val_loader = val_loader + self._validate = evaluator + + # build indexs + self._build_index() + self.n_prunable_layer = len(self.prunable_idx) + + # extract information for preparing + self._extract_layer_information() + + # build embedding (static part) + self._build_state_embedding() + + # build reward + self.reset() # restore weight + self.org_acc = self._validate(self._val_loader, self.model) + _logger.info('=> original acc: %.3f', self.org_acc) + self.org_model_size = sum(self.wsize_list) + _logger.info('=> original weight size: %.4f M param', self.org_model_size * 1. / 1e6) + self.org_flops = sum(self.flops_list) + _logger.info('=> FLOPs:') + _logger.info([self.layer_info_dict[idx]['flops']/1e6 for idx in sorted(self.layer_info_dict.keys())]) + _logger.info('=> original FLOPs: %.4f M', self.org_flops * 1. / 1e6) + + self.expected_preserve_computation = self.preserve_ratio * self.org_flops + + self.reward = eval(args.reward) + + self.best_reward = -math.inf + self.best_strategy = None + self.best_d_prime_list = None + self.best_masks = None + + self.org_w_size = sum(self.wsize_list) + + def step(self, action): + # Pseudo prune and get the corresponding statistics. The real pruning happens till the end of all pseudo pruning + if self.visited[self.cur_ind]: + action = self.strategy_dict[self.prunable_idx[self.cur_ind]][0] + preserve_idx = self.index_buffer[self.cur_ind] + else: + action = self._action_wall(action) # percentage to preserve + preserve_idx = None + # prune and update action + action, d_prime, preserve_idx = self.prune_kernel(self.prunable_idx[self.cur_ind], action, preserve_idx) + if not self.visited[self.cur_ind]: + for group in self.shared_idx: + if self.cur_ind in group: # set the shared ones + for g_idx in group: + self.strategy_dict[self.prunable_idx[g_idx]][0] = action + self.strategy_dict[self.prunable_idx[g_idx - 1]][1] = action + self.visited[g_idx] = True + self.index_buffer[g_idx] = preserve_idx.copy() + + self.strategy.append(action) # save action to strategy + self.d_prime_list.append(d_prime) + + self.strategy_dict[self.prunable_idx[self.cur_ind]][0] = action + if self.cur_ind > 0: + self.strategy_dict[self.prunable_idx[self.cur_ind - 1]][1] = action + + # all the actions are made + if self._is_final_layer(): + assert len(self.strategy) == len(self.prunable_idx) + current_flops = self._cur_flops() + acc_t1 = time.time() + acc = self._validate(self._val_loader, self.model) + acc_t2 = time.time() + self.val_time = acc_t2 - acc_t1 + compress_ratio = current_flops * 1. / self.org_flops + info_set = {'compress_ratio': compress_ratio, 'accuracy': acc, 'strategy': self.strategy.copy()} + reward = self.reward(self, acc, current_flops) + + if reward > self.best_reward: + self.best_reward = reward + self.best_strategy = self.strategy.copy() + self.best_d_prime_list = self.d_prime_list.copy() + best_model = os.path.join(self.args.output, 'best_model.pth') + best_mask = os.path.join(self.args.output, 'best_mask.pth') + self.pruner.export_model(model_path=best_model, mask_path=best_mask) + _logger.info('New best reward: %.4f, acc: %.4f, compress: %.4f', self.best_reward, acc, compress_ratio) + _logger.info('New best policy: %s', self.best_strategy) + _logger.info('New best d primes: %s', self.best_d_prime_list) + obs = self.layer_embedding[self.cur_ind, :].copy() # actually the same as the last state + done = True + return obs, reward, done, info_set + + info_set = None + reward = 0 + done = False + self.visited[self.cur_ind] = True # set to visited + self.cur_ind += 1 # the index of next layer + # build next state (in-place modify) + self.layer_embedding[self.cur_ind][-3] = self._cur_reduced() * 1. / self.org_flops # reduced + self.layer_embedding[self.cur_ind][-2] = sum(self.flops_list[self.cur_ind + 1:]) * 1. / self.org_flops # rest + self.layer_embedding[self.cur_ind][-1] = self.strategy[-1] # last action + obs = self.layer_embedding[self.cur_ind, :].copy() + + return obs, reward, done, info_set + + def reset(self): + # restore env by loading the checkpoint + self.pruner.reset(self.checkpoint) + self.cur_ind = 0 + self.strategy = [] # pruning strategy + self.d_prime_list = [] + self.strategy_dict = copy.deepcopy(self.min_strategy_dict) + # reset layer embeddings + self.layer_embedding[:, -1] = 1. + self.layer_embedding[:, -2] = 0. + self.layer_embedding[:, -3] = 0. + obs = self.layer_embedding[0].copy() + obs[-2] = sum(self.wsize_list[1:]) * 1. / sum(self.wsize_list) + self.extract_time = 0 + self.fit_time = 0 + self.val_time = 0 + # for share index + self.visited = [False] * len(self.prunable_idx) + self.index_buffer = {} + return obs + + def prune_kernel(self, op_idx, preserve_ratio, preserve_idx=None): + m_list = list(self.model.modules()) + op = m_list[op_idx] + assert (0. < preserve_ratio <= 1.) + assert type(op) == PrunerModuleWrapper + if preserve_ratio == 1: # do not prune + if (preserve_idx is None) or (len(preserve_idx) == op.module.weight.size(1)): + return 1., op.module.weight.size(1), None # should be a full index + op.input_feat = self.layer_info_dict[op_idx]['input_feat'] + op.output_feat = self.layer_info_dict[op_idx]['output_feat'] + + masks = self.channel_prune_masker.calc_mask(sparsity=1-preserve_ratio, wrapper=op, preserve_idx=preserve_idx) + m = masks['weight_mask'].cpu().data + if type(op.module) == nn.Conv2d: + d_prime = (m.sum((0, 2, 3)) > 0).sum().item() + preserve_idx = np.nonzero((m.sum((0, 2, 3)) > 0).numpy())[0] + else: + assert type(op.module) == nn.Linear + d_prime = (m.sum(1) > 0).sum().item() + preserve_idx = np.nonzero((m.sum(1) > 0).numpy())[0] + + op.weight_mask = masks['weight_mask'] + if hasattr(op.module, 'bias') and op.module.bias is not None and 'bias_mask' in masks: + op.bias_mask = masks['bias_mask'] + + action = (m == 1).sum().item() / m.numel() + return action, d_prime, preserve_idx + + def _is_final_layer(self): + return self.cur_ind == len(self.prunable_idx) - 1 + + def _action_wall(self, action): + """ + Limit the action generated by DDPG for this layer by two constraints: + 1. The total flops must meet the flops reduce target. + For example: the original flops of entire model is 1000, target flops ratio is 0.5, target flops + is 1000*0.5 = 500. The reduced flops of other layers is 400, so the remaining flops quota is 500-400=100, + if the total original flops of this layer is 250, then the maximum ratio is 100/250 = 0.4. So the + action of this layer can not be greater than 0.4. + 2. The action must be greater than lbound which is stored in self.strategy_dict. + """ + assert len(self.strategy) == self.cur_ind + + action = float(action) + action = np.clip(action, 0, 1) + + other_comp = 0 + this_comp = 0 + for i, idx in enumerate(self.prunable_idx): + flop = self.layer_info_dict[idx]['flops'] + buffer_flop = self._get_buffer_flops(idx) + + if i == self.cur_ind - 1: # TODO: add other member in the set + this_comp += flop * self.strategy_dict[idx][0] + # add buffer (but not influenced by ratio) + other_comp += buffer_flop * self.strategy_dict[idx][0] + elif i == self.cur_ind: + this_comp += flop * self.strategy_dict[idx][1] + # also add buffer here (influenced by ratio) + this_comp += buffer_flop + else: + other_comp += flop * self.strategy_dict[idx][0] * self.strategy_dict[idx][1] + # add buffer + other_comp += buffer_flop * self.strategy_dict[idx][0] # only consider input reduction + + self.expected_min_preserve = other_comp + this_comp * action + max_preserve_ratio = (self.expected_preserve_computation - other_comp) * 1. / this_comp + + action = np.minimum(action, max_preserve_ratio) + action = np.maximum(action, self.strategy_dict[self.prunable_idx[self.cur_ind]][0]) # impossible (should be) + + return action + + def _get_buffer_flops(self, idx): + buffer_idx = self.buffer_dict[idx] + buffer_flop = sum([self.layer_info_dict[_]['flops'] for _ in buffer_idx]) + return buffer_flop + + def _cur_flops(self): + flops = 0 + for idx in self.prunable_idx: + c, n = self.strategy_dict[idx] # input, output pruning ratio + flops += self.layer_info_dict[idx]['flops'] * c * n + # add buffer computation + flops += self._get_buffer_flops(idx) * c # only related to input channel reduction + return flops + + def _cur_reduced(self): + # return the reduced weight + reduced = self.org_flops - self._cur_flops() + return reduced + + def _build_index(self): + """ + Build following information/data for later pruning: + self.prunable_idx: layer indices for pruable layers, the index values are the index + of list(self.model.modules()). Pruable layers are pointwise Conv2d layers and Linear + layers. + self.prunable_ops: prunable modules + self.buffer_idx: layer indices for buffer layers which refers the depthwise layers. + Each depthwise layer is always followd by a pointwise layer for both mobilenet and + mobilenetv2. The depthwise layer's filters are pruned when its next pointwise layer's + corresponding input channels are pruned. + self.shared_idx: layer indices for layers which share input. + For example: [[1,4], [8, 10, 15]] means layer 1 and 4 share same input, and layer + 8, 10 and 15 share another input. + self.org_channels: number of input channels for each layer + self.min_strategy_dict: key is layer index, value is a tuple, the first value is the minimum + action of input channel, the second value is the minimum action value of output channel. + self.strategy_dict: same as self.min_strategy_dict, but it will be updated later. + """ + self.prunable_idx = [] + self.prunable_ops = [] + self.layer_type_dict = {} + self.strategy_dict = {} + self.buffer_dict = {} + this_buffer_list = [] + self.org_channels = [] + # build index and the min strategy dict + for i, m in enumerate(self.model.modules()): + if isinstance(m, PrunerModuleWrapper): + m = m.module + if type(m) == nn.Conv2d and m.groups == m.in_channels: # depth-wise conv, buffer + this_buffer_list.append(i) + else: # really prunable + self.prunable_idx.append(i) + self.prunable_ops.append(m) + self.layer_type_dict[i] = type(m) + self.buffer_dict[i] = this_buffer_list + this_buffer_list = [] # empty + self.org_channels.append(m.in_channels if type(m) == nn.Conv2d else m.in_features) + + self.strategy_dict[i] = [self.lbound, self.lbound] + + self.strategy_dict[self.prunable_idx[0]][0] = 1 # modify the input + self.strategy_dict[self.prunable_idx[-1]][1] = 1 # modify the output + + self.shared_idx = [] + if self.args.model_type == 'mobilenetv2': # TODO: to be tested! Share index for residual connection + connected_idx = [4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32] # to be partitioned + last_ch = -1 + share_group = None + for c_idx in connected_idx: + if self.prunable_ops[c_idx].in_channels != last_ch: # new group + last_ch = self.prunable_ops[c_idx].in_channels + if share_group is not None: + self.shared_idx.append(share_group) + share_group = [c_idx] + else: # same group + share_group.append(c_idx) + self.shared_idx.append(share_group) + _logger.info('=> Conv layers to share channels: %s', self.shared_idx) + + self.min_strategy_dict = copy.deepcopy(self.strategy_dict) + + self.buffer_idx = [] + for _, v in self.buffer_dict.items(): + self.buffer_idx += v + + _logger.info('=> Prunable layer idx: %s', self.prunable_idx) + _logger.info('=> Buffer layer idx: %s', self.buffer_idx) + _logger.info('=> Shared idx: %s', self.shared_idx) + _logger.info('=> Initial min strategy dict: %s', self.min_strategy_dict) + + # added for supporting residual connections during pruning + self.visited = [False] * len(self.prunable_idx) + self.index_buffer = {} + + def _extract_layer_information(self): + m_list = list(self.model.modules()) + + self.data_saver = [] + self.layer_info_dict = dict() + self.wsize_list = [] + self.flops_list = [] + + from .lib.utils import measure_layer_for_pruning + + # extend the forward fn to record layer info + def new_forward(m): + def lambda_forward(x): + m.input_feat = x.clone() + #TODO replace this flops counter with nni.compression.torch.utils.counter.count_flops_params + measure_layer_for_pruning(m, x) + y = m.old_forward(x) + m.output_feat = y.clone() + return y + + return lambda_forward + + device = None + for idx in self.prunable_idx + self.buffer_idx: # get all + m = m_list[idx] + m.old_forward = m.forward + m.forward = new_forward(m) + if device is None and type(m) == PrunerModuleWrapper: + device = m.module.weight.device + + # now let the image flow + _logger.info('=> Extracting information...') + with torch.no_grad(): + for i_b, (inputs, target) in enumerate(self._val_loader): # use image from train set + if i_b == self.n_calibration_batches: + break + self.data_saver.append((inputs.clone(), target.clone())) + input_var = torch.autograd.Variable(inputs).to(device) + + # inference and collect stats + _ = self.model(input_var) + + if i_b == 0: # first batch + for idx in self.prunable_idx + self.buffer_idx: + self.layer_info_dict[idx] = dict() + self.layer_info_dict[idx]['params'] = m_list[idx].params + self.layer_info_dict[idx]['flops'] = m_list[idx].flops + self.wsize_list.append(m_list[idx].params) + self.flops_list.append(m_list[idx].flops) + _logger.info('flops: %s', self.flops_list) + for idx in self.prunable_idx: + f_in_np = m_list[idx].input_feat.data.cpu().numpy() + f_out_np = m_list[idx].output_feat.data.cpu().numpy() + if len(f_in_np.shape) == 4: # conv + if self.prunable_idx.index(idx) == 0: # first conv + f_in2save, f_out2save = None, None + elif m_list[idx].module.weight.size(3) > 1: # normal conv + f_in2save, f_out2save = f_in_np, f_out_np + else: # 1x1 conv + # assert f_out_np.shape[2] == f_in_np.shape[2] # now support k=3 + randx = np.random.randint(0, f_out_np.shape[2] - 0, self.n_points_per_layer) + randy = np.random.randint(0, f_out_np.shape[3] - 0, self.n_points_per_layer) + # input: [N, C, H, W] + self.layer_info_dict[idx][(i_b, 'randx')] = randx.copy() + self.layer_info_dict[idx][(i_b, 'randy')] = randy.copy() + + f_in2save = f_in_np[:, :, randx, randy].copy().transpose(0, 2, 1)\ + .reshape(self.batch_size * self.n_points_per_layer, -1) + + f_out2save = f_out_np[:, :, randx, randy].copy().transpose(0, 2, 1) \ + .reshape(self.batch_size * self.n_points_per_layer, -1) + else: + assert len(f_in_np.shape) == 2 + f_in2save = f_in_np.copy() + f_out2save = f_out_np.copy() + if 'input_feat' not in self.layer_info_dict[idx]: + self.layer_info_dict[idx]['input_feat'] = f_in2save + self.layer_info_dict[idx]['output_feat'] = f_out2save + else: + self.layer_info_dict[idx]['input_feat'] = np.vstack( + (self.layer_info_dict[idx]['input_feat'], f_in2save)) + self.layer_info_dict[idx]['output_feat'] = np.vstack( + (self.layer_info_dict[idx]['output_feat'], f_out2save)) + + def _build_state_embedding(self): + # build the static part of the state embedding + _logger.info('Building state embedding...') + layer_embedding = [] + module_list = list(self.model.modules()) + for i, ind in enumerate(self.prunable_idx): + m = module_list[ind].module + this_state = [] + if type(m) == nn.Conv2d: + this_state.append(i) # index + this_state.append(0) # layer type, 0 for conv + this_state.append(m.in_channels) # in channels + this_state.append(m.out_channels) # out channels + this_state.append(m.stride[0]) # stride + this_state.append(m.kernel_size[0]) # kernel size + this_state.append(np.prod(m.weight.size())) # weight size + elif type(m) == nn.Linear: + this_state.append(i) # index + this_state.append(1) # layer type, 1 for fc + this_state.append(m.in_features) # in channels + this_state.append(m.out_features) # out channels + this_state.append(0) # stride + this_state.append(1) # kernel size + this_state.append(np.prod(m.weight.size())) # weight size + + # this 3 features need to be changed later + this_state.append(0.) # reduced + this_state.append(0.) # rest + this_state.append(1.) # a_{t-1} + layer_embedding.append(np.array(this_state)) + + # normalize the state + layer_embedding = np.array(layer_embedding, 'float') + _logger.info('=> shape of embedding (n_layer * n_dim): %s', layer_embedding.shape) + assert len(layer_embedding.shape) == 2, layer_embedding.shape + for i in range(layer_embedding.shape[1]): + fmin = min(layer_embedding[:, i]) + fmax = max(layer_embedding[:, i]) + if fmax - fmin > 0: + layer_embedding[:, i] = (layer_embedding[:, i] - fmin) / (fmax - fmin) + + self.layer_embedding = layer_embedding + diff --git a/utils/third_party/nni_new/algorithms/compression/pytorch/pruning/amc/lib/__init__.py b/utils/third_party/nni_new/algorithms/compression/pytorch/pruning/amc/lib/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/utils/third_party/nni_new/algorithms/compression/pytorch/pruning/amc/lib/agent.py b/utils/third_party/nni_new/algorithms/compression/pytorch/pruning/amc/lib/agent.py new file mode 100644 index 0000000000000000000000000000000000000000..fe066301b8b5e4325f92a1b98885ae547a7ecee3 --- /dev/null +++ b/utils/third_party/nni_new/algorithms/compression/pytorch/pruning/amc/lib/agent.py @@ -0,0 +1,232 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import numpy as np + +import torch +import torch.nn as nn +from torch.optim import Adam + +from .memory import SequentialMemory +from .utils import to_numpy, to_tensor + +criterion = nn.MSELoss() +USE_CUDA = torch.cuda.is_available() + + +class Actor(nn.Module): + def __init__(self, nb_states, nb_actions, hidden1=400, hidden2=300): + super(Actor, self).__init__() + self.fc1 = nn.Linear(nb_states, hidden1) + self.fc2 = nn.Linear(hidden1, hidden2) + self.fc3 = nn.Linear(hidden2, nb_actions) + self.relu = nn.ReLU() + self.sigmoid = nn.Sigmoid() + + def forward(self, x): + out = self.fc1(x) + out = self.relu(out) + out = self.fc2(out) + out = self.relu(out) + out = self.fc3(out) + out = self.sigmoid(out) + return out + + +class Critic(nn.Module): + def __init__(self, nb_states, nb_actions, hidden1=400, hidden2=300): + super(Critic, self).__init__() + self.fc11 = nn.Linear(nb_states, hidden1) + self.fc12 = nn.Linear(nb_actions, hidden1) + self.fc2 = nn.Linear(hidden1, hidden2) + self.fc3 = nn.Linear(hidden2, 1) + self.relu = nn.ReLU() + + def forward(self, xs): + x, a = xs + out = self.fc11(x) + self.fc12(a) + out = self.relu(out) + out = self.fc2(out) + out = self.relu(out) + out = self.fc3(out) + return out + + +class DDPG(object): + def __init__(self, nb_states, nb_actions, args): + + self.nb_states = nb_states + self.nb_actions = nb_actions + + # Create Actor and Critic Network + net_cfg = { + 'hidden1': args.hidden1, + 'hidden2': args.hidden2, + # 'init_w': args.init_w + } + self.actor = Actor(self.nb_states, self.nb_actions, **net_cfg) + self.actor_target = Actor(self.nb_states, self.nb_actions, **net_cfg) + self.actor_optim = Adam(self.actor.parameters(), lr=args.lr_a) + + self.critic = Critic(self.nb_states, self.nb_actions, **net_cfg) + self.critic_target = Critic(self.nb_states, self.nb_actions, **net_cfg) + self.critic_optim = Adam(self.critic.parameters(), lr=args.lr_c) + + self.hard_update(self.actor_target, self.actor) # Make sure target is with the same weight + self.hard_update(self.critic_target, self.critic) + + # Create replay buffer + self.memory = SequentialMemory(limit=args.rmsize, window_length=args.window_length) + # self.random_process = OrnsteinUhlenbeckProcess(size=nb_actions, theta=args.ou_theta, mu=args.ou_mu, + # sigma=args.ou_sigma) + + # Hyper-parameters + self.batch_size = args.bsize + self.tau = args.tau + self.discount = args.discount + self.depsilon = 1.0 / args.epsilon + self.lbound = 0. # args.lbound + self.rbound = 1. # args.rbound + + # noise + self.init_delta = args.init_delta + self.delta_decay = args.delta_decay + self.warmup = args.warmup + + # + self.epsilon = 1.0 + # self.s_t = None # Most recent state + # self.a_t = None # Most recent action + self.is_training = True + + # + if USE_CUDA: self.cuda() + + # moving average baseline + self.moving_average = None + self.moving_alpha = 0.5 # based on batch, so small + + def update_policy(self): + # Sample batch + state_batch, action_batch, reward_batch, \ + next_state_batch, terminal_batch = self.memory.sample_and_split(self.batch_size) + + # normalize the reward + batch_mean_reward = np.mean(reward_batch) + if self.moving_average is None: + self.moving_average = batch_mean_reward + else: + self.moving_average += self.moving_alpha * (batch_mean_reward - self.moving_average) + reward_batch -= self.moving_average + # if reward_batch.std() > 0: + # reward_batch /= reward_batch.std() + + # Prepare for the target q batch + with torch.no_grad(): + next_q_values = self.critic_target([ + to_tensor(next_state_batch), + self.actor_target(to_tensor(next_state_batch)), + ]) + + target_q_batch = to_tensor(reward_batch) + \ + self.discount * to_tensor(terminal_batch.astype(np.float)) * next_q_values + + # Critic update + self.critic.zero_grad() + + q_batch = self.critic([to_tensor(state_batch), to_tensor(action_batch)]) + + value_loss = criterion(q_batch, target_q_batch) + value_loss.backward() + self.critic_optim.step() + + # Actor update + self.actor.zero_grad() + + policy_loss = -self.critic([ # pylint: disable=all + to_tensor(state_batch), + self.actor(to_tensor(state_batch)) + ]) + + policy_loss = policy_loss.mean() + policy_loss.backward() + self.actor_optim.step() + + # Target update + self.soft_update(self.actor_target, self.actor) + self.soft_update(self.critic_target, self.critic) + + def eval(self): + self.actor.eval() + self.actor_target.eval() + self.critic.eval() + self.critic_target.eval() + + def cuda(self): + self.actor.cuda() + self.actor_target.cuda() + self.critic.cuda() + self.critic_target.cuda() + + def observe(self, r_t, s_t, s_t1, a_t, done): + if self.is_training: + self.memory.append(s_t, a_t, r_t, done) # save to memory + # self.s_t = s_t1 + + def random_action(self): + action = np.random.uniform(self.lbound, self.rbound, self.nb_actions) + # self.a_t = action + return action + + def select_action(self, s_t, episode): + # assert episode >= self.warmup, 'Episode: {} warmup: {}'.format(episode, self.warmup) + action = to_numpy(self.actor(to_tensor(np.array(s_t).reshape(1, -1)))).squeeze(0) + delta = self.init_delta * (self.delta_decay ** (episode - self.warmup)) + # action += self.is_training * max(self.epsilon, 0) * self.random_process.sample() + action = self.sample_from_truncated_normal_distribution(lower=self.lbound, upper=self.rbound, mu=action, sigma=delta) + action = np.clip(action, self.lbound, self.rbound) + + # self.a_t = action + return action + + def reset(self, obs): + pass + # self.s_t = obs + # self.random_process.reset_states() + + def load_weights(self, output): + if output is None: return + + self.actor.load_state_dict( + torch.load('{}/actor.pkl'.format(output)) + ) + + self.critic.load_state_dict( + torch.load('{}/critic.pkl'.format(output)) + ) + + def save_model(self, output): + torch.save( + self.actor.state_dict(), + '{}/actor.pkl'.format(output) + ) + torch.save( + self.critic.state_dict(), + '{}/critic.pkl'.format(output) + ) + + def soft_update(self, target, source): + for target_param, param in zip(target.parameters(), source.parameters()): + target_param.data.copy_( + target_param.data * (1.0 - self.tau) + param.data * self.tau + ) + + def hard_update(self, target, source): + for target_param, param in zip(target.parameters(), source.parameters()): + target_param.data.copy_(param.data) + + def sample_from_truncated_normal_distribution(self, lower, upper, mu, sigma, size=1): + from scipy import stats + return stats.truncnorm.rvs((lower-mu)/sigma, (upper-mu)/sigma, loc=mu, scale=sigma, size=size) + + diff --git a/utils/third_party/nni_new/algorithms/compression/pytorch/pruning/amc/lib/memory.py b/utils/third_party/nni_new/algorithms/compression/pytorch/pruning/amc/lib/memory.py new file mode 100644 index 0000000000000000000000000000000000000000..57bbcfceb86a20092968c9dc75a618221e119174 --- /dev/null +++ b/utils/third_party/nni_new/algorithms/compression/pytorch/pruning/amc/lib/memory.py @@ -0,0 +1,227 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from __future__ import absolute_import +from collections import deque, namedtuple +import warnings +import random + +import numpy as np + +# [reference] https://github.com/matthiasplappert/keras-rl/blob/master/rl/memory.py + +# This is to be understood as a transition: Given `state0`, performing `action` +# yields `reward` and results in `state1`, which might be `terminal`. +Experience = namedtuple('Experience', 'state0, action, reward, state1, terminal1') + + +def sample_batch_indexes(low, high, size): + if high - low >= size: + # We have enough data. Draw without replacement, that is each index is unique in the + # batch. We cannot use `np.random.choice` here because it is horribly inefficient as + # the memory grows. See https://github.com/numpy/numpy/issues/2764 for a discussion. + # `random.sample` does the same thing (drawing without replacement) and is way faster. + r = range(low, high) + batch_idxs = random.sample(r, size) + else: + # Not enough data. Help ourselves with sampling from the range, but the same index + # can occur multiple times. This is not good and should be avoided by picking a + # large enough warm-up phase. + warnings.warn( + 'Not enough entries to sample without replacement. ' + 'Consider increasing your warm-up phase to avoid oversampling!') + batch_idxs = np.random.random_integers(low, high - 1, size=size) + assert len(batch_idxs) == size + return batch_idxs + + +class RingBuffer(object): + def __init__(self, maxlen): + self.maxlen = maxlen + self.start = 0 + self.length = 0 + self.data = [None for _ in range(maxlen)] + + def __len__(self): + return self.length + + def __getitem__(self, idx): + if idx < 0 or idx >= self.length: + raise KeyError() + return self.data[(self.start + idx) % self.maxlen] + + def append(self, v): + if self.length < self.maxlen: + # We have space, simply increase the length. + self.length += 1 + elif self.length == self.maxlen: + # No space, "remove" the first item. + self.start = (self.start + 1) % self.maxlen + else: + # This should never happen. + raise RuntimeError() + self.data[(self.start + self.length - 1) % self.maxlen] = v + + +def zeroed_observation(observation): + if hasattr(observation, 'shape'): + return np.zeros(observation.shape) + elif hasattr(observation, '__iter__'): + out = [] + for x in observation: + out.append(zeroed_observation(x)) + return out + else: + return 0. + + +class Memory(object): + def __init__(self, window_length, ignore_episode_boundaries=False): + self.window_length = window_length + self.ignore_episode_boundaries = ignore_episode_boundaries + + self.recent_observations = deque(maxlen=window_length) + self.recent_terminals = deque(maxlen=window_length) + + def sample(self, batch_size, batch_idxs=None): + raise NotImplementedError() + + def append(self, observation, action, reward, terminal, training=True): + self.recent_observations.append(observation) + self.recent_terminals.append(terminal) + + def get_recent_state(self, current_observation): + # This code is slightly complicated by the fact that subsequent observations might be + # from different episodes. We ensure that an experience never spans multiple episodes. + # This is probably not that important in practice but it seems cleaner. + state = [current_observation] + idx = len(self.recent_observations) - 1 + for offset in range(0, self.window_length - 1): + current_idx = idx - offset + current_terminal = self.recent_terminals[current_idx - 1] if current_idx - 1 >= 0 else False + if current_idx < 0 or (not self.ignore_episode_boundaries and current_terminal): + # The previously handled observation was terminal, don't add the current one. + # Otherwise we would leak into a different episode. + break + state.insert(0, self.recent_observations[current_idx]) + while len(state) < self.window_length: + state.insert(0, zeroed_observation(state[0])) + return state + + def get_config(self): + config = { + 'window_length': self.window_length, + 'ignore_episode_boundaries': self.ignore_episode_boundaries, + } + return config + + +class SequentialMemory(Memory): + def __init__(self, limit, **kwargs): + super(SequentialMemory, self).__init__(**kwargs) + + self.limit = limit + + # Do not use deque to implement the memory. This data structure may seem convenient but + # it is way too slow on random access. Instead, we use our own ring buffer implementation. + self.actions = RingBuffer(limit) + self.rewards = RingBuffer(limit) + self.terminals = RingBuffer(limit) + self.observations = RingBuffer(limit) + + def sample(self, batch_size, batch_idxs=None): + if batch_idxs is None: + # Draw random indexes such that we have at least a single entry before each + # index. + batch_idxs = sample_batch_indexes(0, self.nb_entries - 1, size=batch_size) + batch_idxs = np.array(batch_idxs) + 1 + assert np.min(batch_idxs) >= 1 + assert np.max(batch_idxs) < self.nb_entries + assert len(batch_idxs) == batch_size + + # Create experiences + experiences = [] + for idx in batch_idxs: + terminal0 = self.terminals[idx - 2] if idx >= 2 else False + while terminal0: + # Skip this transition because the environment was reset here. Select a new, random + # transition and use this instead. This may cause the batch to contain the same + # transition twice. + idx = sample_batch_indexes(1, self.nb_entries, size=1)[0] + terminal0 = self.terminals[idx - 2] if idx >= 2 else False + assert 1 <= idx < self.nb_entries + + # This code is slightly complicated by the fact that subsequent observations might be + # from different episodes. We ensure that an experience never spans multiple episodes. + # This is probably not that important in practice but it seems cleaner. + state0 = [self.observations[idx - 1]] + for offset in range(0, self.window_length - 1): + current_idx = idx - 2 - offset + current_terminal = self.terminals[current_idx - 1] if current_idx - 1 > 0 else False + if current_idx < 0 or (not self.ignore_episode_boundaries and current_terminal): + # The previously handled observation was terminal, don't add the current one. + # Otherwise we would leak into a different episode. + break + state0.insert(0, self.observations[current_idx]) + while len(state0) < self.window_length: + state0.insert(0, zeroed_observation(state0[0])) + action = self.actions[idx - 1] + reward = self.rewards[idx - 1] + terminal1 = self.terminals[idx - 1] + + # Okay, now we need to create the follow-up state. This is state0 shifted on timestep + # to the right. Again, we need to be careful to not include an observation from the next + # episode if the last state is terminal. + state1 = [np.copy(x) for x in state0[1:]] + state1.append(self.observations[idx]) + + assert len(state0) == self.window_length + assert len(state1) == len(state0) + experiences.append(Experience(state0=state0, action=action, reward=reward, + state1=state1, terminal1=terminal1)) + assert len(experiences) == batch_size + return experiences + + def sample_and_split(self, batch_size, batch_idxs=None): + experiences = self.sample(batch_size, batch_idxs) + + state0_batch = [] + reward_batch = [] + action_batch = [] + terminal1_batch = [] + state1_batch = [] + for e in experiences: + state0_batch.append(e.state0) + state1_batch.append(e.state1) + reward_batch.append(e.reward) + action_batch.append(e.action) + terminal1_batch.append(0. if e.terminal1 else 1.) + + # Prepare and validate parameters. + state0_batch = np.array(state0_batch, 'double').reshape(batch_size, -1) + state1_batch = np.array(state1_batch, 'double').reshape(batch_size, -1) + terminal1_batch = np.array(terminal1_batch, 'double').reshape(batch_size, -1) + reward_batch = np.array(reward_batch, 'double').reshape(batch_size, -1) + action_batch = np.array(action_batch, 'double').reshape(batch_size, -1) + + return state0_batch, action_batch, reward_batch, state1_batch, terminal1_batch + + def append(self, observation, action, reward, terminal, training=True): + super(SequentialMemory, self).append(observation, action, reward, terminal, training=training) + + # This needs to be understood as follows: in `observation`, take `action`, obtain `reward` + # and weather the next state is `terminal` or not. + if training: + self.observations.append(observation) + self.actions.append(action) + self.rewards.append(reward) + self.terminals.append(terminal) + + @property + def nb_entries(self): + return len(self.observations) + + def get_config(self): + config = super(SequentialMemory, self).get_config() + config['limit'] = self.limit + return config diff --git a/utils/third_party/nni_new/algorithms/compression/pytorch/pruning/amc/lib/net_measure.py b/utils/third_party/nni_new/algorithms/compression/pytorch/pruning/amc/lib/net_measure.py new file mode 100644 index 0000000000000000000000000000000000000000..2c9e815116288c11de98b4061b4dbfc3df029a7e --- /dev/null +++ b/utils/third_party/nni_new/algorithms/compression/pytorch/pruning/amc/lib/net_measure.py @@ -0,0 +1,123 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import torch + +# [reference] https://github.com/ShichenLiu/CondenseNet/blob/master/utils.py + + +def get_num_gen(gen): + return sum(1 for _ in gen) + + +def is_leaf(model): + return get_num_gen(model.children()) == 0 + + +def get_layer_info(layer): + layer_str = str(layer) + type_name = layer_str[:layer_str.find('(')].strip() + return type_name + + +def get_layer_param(model): + import operator + import functools + + return sum([functools.reduce(operator.mul, i.size(), 1) for i in model.parameters()]) + +count_ops = 0 +count_params = 0 + +def measure_layer(layer, x): + global count_ops, count_params + delta_ops = 0 + delta_params = 0 + multi_add = 1 + type_name = get_layer_info(layer) + + # ops_conv + if type_name in ['Conv2d']: + out_h = int((x.size()[2] + 2 * layer.padding[0] - layer.kernel_size[0]) / + layer.stride[0] + 1) + out_w = int((x.size()[3] + 2 * layer.padding[1] - layer.kernel_size[1]) / + layer.stride[1] + 1) + delta_ops = layer.in_channels * layer.out_channels * layer.kernel_size[0] * \ + layer.kernel_size[1] * out_h * out_w / layer.groups * multi_add + delta_params = get_layer_param(layer) + + # ops_nonlinearity + elif type_name in ['ReLU']: + delta_ops = x.numel() / x.size(0) + delta_params = get_layer_param(layer) + + # ops_pooling + elif type_name in ['AvgPool2d']: + in_w = x.size()[2] + kernel_ops = layer.kernel_size * layer.kernel_size + out_w = int((in_w + 2 * layer.padding - layer.kernel_size) / layer.stride + 1) + out_h = int((in_w + 2 * layer.padding - layer.kernel_size) / layer.stride + 1) + delta_ops = x.size()[1] * out_w * out_h * kernel_ops + delta_params = get_layer_param(layer) + + elif type_name in ['AdaptiveAvgPool2d']: + delta_ops = x.size()[1] * x.size()[2] * x.size()[3] + delta_params = get_layer_param(layer) + + # ops_linear + elif type_name in ['Linear']: + weight_ops = layer.weight.numel() * multi_add + bias_ops = layer.bias.numel() + delta_ops = weight_ops + bias_ops + delta_params = get_layer_param(layer) + + # ops_nothing + elif type_name in ['BatchNorm2d', 'Dropout2d', 'DropChannel', 'Dropout']: + delta_params = get_layer_param(layer) + + # unknown layer type + else: + delta_params = get_layer_param(layer) + + count_ops += delta_ops + count_params += delta_params + + return + + +def measure_model(model, H, W, device): + global count_ops, count_params + count_ops = 0 + count_params = 0 + data = torch.zeros(2, 3, H, W).to(device) + + def should_measure(x): + return is_leaf(x) + + def modify_forward(model): + for child in model.children(): + if should_measure(child): + def new_forward(m): + def lambda_forward(x): + measure_layer(m, x) + return m.old_forward(x) + return lambda_forward + child.old_forward = child.forward + child.forward = new_forward(child) + else: + modify_forward(child) + + def restore_forward(model): + for child in model.children(): + # leaf node + if is_leaf(child) and hasattr(child, 'old_forward'): + child.forward = child.old_forward + child.old_forward = None + else: + restore_forward(child) + + modify_forward(model) + model.forward(data) + restore_forward(model) + + return count_ops, count_params diff --git a/utils/third_party/nni_new/algorithms/compression/pytorch/pruning/amc/lib/utils.py b/utils/third_party/nni_new/algorithms/compression/pytorch/pruning/amc/lib/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..f875e8d7d9b363d848aa2424b4f06ddaeb1dc1a6 --- /dev/null +++ b/utils/third_party/nni_new/algorithms/compression/pytorch/pruning/amc/lib/utils.py @@ -0,0 +1,113 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +import torch + +class TextLogger(object): + """Write log immediately to the disk""" + def __init__(self, filepath): + self.f = open(filepath, 'w') + self.fid = self.f.fileno() + self.filepath = filepath + + def close(self): + self.f.close() + + def write(self, content): + self.f.write(content) + self.f.flush() + os.fsync(self.fid) + + def write_buf(self, content): + self.f.write(content) + + def print_and_write(self, content): + print(content) + self.write(content+'\n') + +def to_numpy(var): + use_cuda = torch.cuda.is_available() + return var.cpu().data.numpy() if use_cuda else var.data.numpy() + + +def to_tensor(ndarray, requires_grad=False): # return a float tensor by default + tensor = torch.from_numpy(ndarray).float() # by default does not require grad + if requires_grad: + tensor.requires_grad_() + return tensor.cuda() if torch.cuda.is_available() else tensor + + +def measure_layer_for_pruning(wrapper, x): + def get_layer_type(layer): + layer_str = str(layer) + return layer_str[:layer_str.find('(')].strip() + + def get_layer_param(model): + import operator + import functools + + return sum([functools.reduce(operator.mul, i.size(), 1) for i in model.parameters()]) + + multi_add = 1 + layer = wrapper.module + type_name = get_layer_type(layer) + + # ops_conv + if type_name in ['Conv2d']: + out_h = int((x.size()[2] + 2 * layer.padding[0] - layer.kernel_size[0]) / + layer.stride[0] + 1) + out_w = int((x.size()[3] + 2 * layer.padding[1] - layer.kernel_size[1]) / + layer.stride[1] + 1) + wrapper.flops = layer.in_channels * layer.out_channels * layer.kernel_size[0] * \ + layer.kernel_size[1] * out_h * out_w / layer.groups * multi_add + wrapper.params = get_layer_param(layer) + # ops_linear + elif type_name in ['Linear']: + weight_ops = layer.weight.numel() * multi_add + bias_ops = layer.bias.numel() + wrapper.flops = weight_ops + bias_ops + wrapper.params = get_layer_param(layer) + return + + +def least_square_sklearn(X, Y): + from sklearn.linear_model import LinearRegression + reg = LinearRegression(fit_intercept=False) + reg.fit(X, Y) + return reg.coef_ + + +def get_output_folder(parent_dir, env_name): + """Return save folder. + Assumes folders in the parent_dir have suffix -run{run + number}. Finds the highest run number and sets the output folder + to that number + 1. This is just convenient so that if you run the + same script multiple times tensorboard can plot all of the results + on the same plots with different names. + Parameters + ---------- + parent_dir: str + Path of the directory containing all experiment runs. + Returns + ------- + parent_dir/run_dir + Path to this run's save directory. + """ + os.makedirs(parent_dir, exist_ok=True) + experiment_id = 0 + for folder_name in os.listdir(parent_dir): + if not os.path.isdir(os.path.join(parent_dir, folder_name)): + continue + try: + folder_name = int(folder_name.split('-run')[-1]) + if folder_name > experiment_id: + experiment_id = folder_name + except: + pass + experiment_id += 1 + + parent_dir = os.path.join(parent_dir, env_name) + parent_dir = parent_dir + '-run{}'.format(experiment_id) + os.makedirs(parent_dir, exist_ok=True) + return parent_dir diff --git a/utils/third_party/nni_new/algorithms/compression/pytorch/pruning/apply_compression.py b/utils/third_party/nni_new/algorithms/compression/pytorch/pruning/apply_compression.py new file mode 100644 index 0000000000000000000000000000000000000000..8e6b023f5b90b8483e21bd0bc575b19b4a4df023 --- /dev/null +++ b/utils/third_party/nni_new/algorithms/compression/pytorch/pruning/apply_compression.py @@ -0,0 +1,29 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +import torch + +logger = logging.getLogger('torch apply compression') + +def apply_compression_results(model, masks_file, map_location=None): + """ + Apply the masks from ```masks_file``` to the model + Note: this API is for inference, because it simply multiplies weights with + corresponding masks when this API is called. + + Parameters + ---------- + model : torch.nn.Module + The model to be compressed + masks_file : str + The path of the mask file + map_location : str + the device on which masks are placed, same to map_location in ```torch.load``` + """ + masks = torch.load(masks_file, map_location) + for name, module in model.named_modules(): + if name in masks: + module.weight.data = module.weight.data.mul_(masks[name]['weight']) + if hasattr(module, 'bias') and module.bias is not None and 'bias' in masks[name]: + module.bias.data = module.bias.data.mul_(masks[name]['bias']) \ No newline at end of file diff --git a/utils/third_party/nni_new/algorithms/compression/pytorch/pruning/auto_compress_pruner.py b/utils/third_party/nni_new/algorithms/compression/pytorch/pruning/auto_compress_pruner.py new file mode 100644 index 0000000000000000000000000000000000000000..207a8aa2f9b0a963fa3a38e7a11940ba9c1f6cda --- /dev/null +++ b/utils/third_party/nni_new/algorithms/compression/pytorch/pruning/auto_compress_pruner.py @@ -0,0 +1,237 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +import os +import copy +import torch +from schema import And, Optional + +from nni.utils import OptimizeMode +from nni.compression.pytorch import ModelSpeedup + +from nni.compression.pytorch.compressor import Pruner +from nni.compression.pytorch.utils.config_validation import CompressorSchema +from .simulated_annealing_pruner import SimulatedAnnealingPruner +from .iterative_pruner import ADMMPruner + +_logger = logging.getLogger(__name__) + + +class AutoCompressPruner(Pruner): + """ + A Pytorch implementation of AutoCompress pruning algorithm. + + Parameters + ---------- + model : pytorch model + The model to be pruned. + config_list : list + Supported keys: + - sparsity : The target overall sparsity. + - op_types : The operation type to prune. + trainer : function + Function used for the first subproblem of ADMM Pruner. + Users should write this function as a normal function to train the Pytorch model + and include `model, optimizer, criterion, epoch` as function arguments. + criterion: function + Function used to calculate the loss between the target and the output. By default, we use CrossEntropyLoss. + For example, you can use ``torch.nn.CrossEntropyLoss()`` as input. + evaluator : function + function to evaluate the pruned model. + This function should include `model` as the only parameter, and returns a scalar value. + Example:: + + def evaluator(model): + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + val_loader = ... + model.eval() + correct = 0 + with torch.no_grad(): + for data, target in val_loader: + data, target = data.to(device), target.to(device) + output = model(data) + # get the index of the max log-probability + pred = output.argmax(dim=1, keepdim=True) + correct += pred.eq(target.view_as(pred)).sum().item() + accuracy = correct / len(val_loader.dataset) + return accuracy + dummy_input : pytorch tensor + The dummy input for ```jit.trace```, users should put it on right device before pass in. + num_iterations : int + Number of overall iterations. + optimize_mode : str + optimize mode, `maximize` or `minimize`, by default `maximize`. + base_algo : str + Base pruning algorithm. `level`, `l1`, `l2` or `fpgm`, by default `l1`. Given the sparsity distribution among + the ops, the assigned `base_algo` is used to decide which filters/channels/weights to prune. + start_temperature : float + Start temperature of the simulated annealing process. + stop_temperature : float + Stop temperature of the simulated annealing process. + cool_down_rate : float + Cool down rate of the temperature. + perturbation_magnitude : float + Initial perturbation magnitude to the sparsities. The magnitude decreases with current temperature. + admm_num_iterations : int + Number of iterations of ADMM Pruner. + admm_epochs_per_iteration : int + Training epochs of the first optimization subproblem of ADMMPruner. + row : float + Penalty parameters for ADMM training. + experiment_data_dir : string + PATH to store temporary experiment data. + """ + + def __init__(self, model, config_list, trainer, evaluator, dummy_input, criterion=torch.nn.CrossEntropyLoss(), + num_iterations=3, optimize_mode='maximize', base_algo='l1', + # SimulatedAnnealing related + start_temperature=100, stop_temperature=20, cool_down_rate=0.9, perturbation_magnitude=0.35, + # ADMM related + admm_num_iterations=30, admm_epochs_per_iteration=5, row=1e-4, + experiment_data_dir='./'): + # original model + self._model_to_prune = model + self._base_algo = base_algo + + self._trainer = trainer + self._criterion = criterion + self._evaluator = evaluator + self._dummy_input = dummy_input + self._num_iterations = num_iterations + self._optimize_mode = OptimizeMode(optimize_mode) + + # hyper parameters for SA algorithm + self._start_temperature = start_temperature + self._stop_temperature = stop_temperature + self._cool_down_rate = cool_down_rate + self._perturbation_magnitude = perturbation_magnitude + + # hyper parameters for ADMM algorithm + self._admm_num_iterations = admm_num_iterations + self._admm_epochs_per_iteration = admm_epochs_per_iteration + self._row = row + + # overall pruning rate + self._sparsity = config_list[0]['sparsity'] + + self._experiment_data_dir = experiment_data_dir + if not os.path.exists(self._experiment_data_dir): + os.makedirs(self._experiment_data_dir) + + def validate_config(self, model, config_list): + """ + Parameters + ---------- + model : torch.nn.Module + Model to be pruned + config_list : list + List on pruning configs + """ + + if self._base_algo == 'level': + schema = CompressorSchema([{ + 'sparsity': And(float, lambda n: 0 < n < 1), + Optional('op_types'): [str], + Optional('op_names'): [str], + }], model, _logger) + elif self._base_algo in ['l1', 'l2', 'fpgm']: + schema = CompressorSchema([{ + 'sparsity': And(float, lambda n: 0 < n < 1), + 'op_types': ['Conv2d'], + Optional('op_names'): [str] + }], model, _logger) + + schema.validate(config_list) + + def calc_mask(self, wrapper, **kwargs): + return None + + def compress(self): + """ + Compress the model with AutoCompress. + + Returns + ------- + torch.nn.Module + model with specified modules compressed. + """ + _logger.info('Starting AutoCompress pruning...') + + sparsity_each_round = 1 - pow(1 - self._sparsity, 1 / self._num_iterations) + + for i in range(self._num_iterations): + _logger.info('Pruning iteration: %d', i) + _logger.info('Target sparsity this round: %s', + 1 - pow(1 - sparsity_each_round, i + 1)) + + # SimulatedAnnealingPruner + _logger.info( + 'Generating sparsities with SimulatedAnnealingPruner...') + SApruner = SimulatedAnnealingPruner( + model=copy.deepcopy(self._model_to_prune), + config_list=[ + {"sparsity": sparsity_each_round, "op_types": ['Conv2d']}], + evaluator=self._evaluator, + optimize_mode=self._optimize_mode, + base_algo=self._base_algo, + start_temperature=self._start_temperature, + stop_temperature=self._stop_temperature, + cool_down_rate=self._cool_down_rate, + perturbation_magnitude=self._perturbation_magnitude, + experiment_data_dir=self._experiment_data_dir) + config_list = SApruner.compress(return_config_list=True) + _logger.info("Generated config_list : %s", config_list) + + # ADMMPruner + _logger.info('Performing structured pruning with ADMMPruner...') + ADMMpruner = ADMMPruner( + model=copy.deepcopy(self._model_to_prune), + config_list=config_list, + criterion=self._criterion, + trainer=self._trainer, + num_iterations=self._admm_num_iterations, + epochs_per_iteration=self._admm_epochs_per_iteration, + row=self._row, + base_algo=self._base_algo) + ADMMpruner.compress() + + ADMMpruner.export_model(os.path.join(self._experiment_data_dir, 'model_admm_masked.pth'), os.path.join( + self._experiment_data_dir, 'mask.pth')) + + # use speed up to prune the model before next iteration, + # because SimulatedAnnealingPruner & ADMMPruner don't take masked models + self._model_to_prune.load_state_dict(torch.load(os.path.join( + self._experiment_data_dir, 'model_admm_masked.pth'))) + + masks_file = os.path.join(self._experiment_data_dir, 'mask.pth') + device = next(self._model_to_prune.parameters()).device + + _logger.info('Speeding up models...') + m_speedup = ModelSpeedup(self._model_to_prune, self._dummy_input, masks_file, device) + m_speedup.speedup_model() + + evaluation_result = self._evaluator(self._model_to_prune) + _logger.info('Evaluation result of the pruned model in iteration %d: %s', i, evaluation_result) + + _logger.info('----------Compression finished--------------') + + os.remove(os.path.join(self._experiment_data_dir, 'model_admm_masked.pth')) + os.remove(os.path.join(self._experiment_data_dir, 'mask.pth')) + + return self._model_to_prune + + def export_model(self, model_path, mask_path=None, onnx_path=None, input_shape=None, device=None): + _logger.info("AutoCompressPruner export directly the pruned model without mask") + + torch.save(self._model_to_prune.state_dict(), model_path) + _logger.info('Model state_dict saved to %s', model_path) + + if onnx_path is not None: + assert input_shape is not None, 'input_shape must be specified to export onnx model' + # input info needed + if device is None: + device = torch.device('cpu') + input_data = torch.Tensor(*input_shape) + torch.onnx.export(self._model_to_prune, input_data.to(device), onnx_path) + _logger.info('Model in onnx with input shape %s saved to %s', input_data.shape, onnx_path) diff --git a/utils/third_party/nni_new/algorithms/compression/pytorch/pruning/constants.py b/utils/third_party/nni_new/algorithms/compression/pytorch/pruning/constants.py new file mode 100644 index 0000000000000000000000000000000000000000..24b84340cc6f5b0c362ab45288638a43926809b7 --- /dev/null +++ b/utils/third_party/nni_new/algorithms/compression/pytorch/pruning/constants.py @@ -0,0 +1,18 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + + +from . import LevelPrunerMasker, SlimPrunerMasker, L1FilterPrunerMasker, \ + L2FilterPrunerMasker, FPGMPrunerMasker, TaylorFOWeightFilterPrunerMasker, \ + ActivationAPoZRankFilterPrunerMasker, ActivationMeanRankFilterPrunerMasker + +MASKER_DICT = { + 'level': LevelPrunerMasker, + 'slim': SlimPrunerMasker, + 'l1': L1FilterPrunerMasker, + 'l2': L2FilterPrunerMasker, + 'fpgm': FPGMPrunerMasker, + 'taylorfo': TaylorFOWeightFilterPrunerMasker, + 'apoz': ActivationAPoZRankFilterPrunerMasker, + 'mean_activation': ActivationMeanRankFilterPrunerMasker +} diff --git a/utils/third_party/nni_new/algorithms/compression/pytorch/pruning/constants_pruner.py b/utils/third_party/nni_new/algorithms/compression/pytorch/pruning/constants_pruner.py new file mode 100644 index 0000000000000000000000000000000000000000..55ba9506f3ba6a528a6dfdf87b6c878d61176c9f --- /dev/null +++ b/utils/third_party/nni_new/algorithms/compression/pytorch/pruning/constants_pruner.py @@ -0,0 +1,12 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + + +from .one_shot_pruner import LevelPruner, L1FilterPruner, L2FilterPruner, FPGMPruner + +PRUNER_DICT = { + 'level': LevelPruner, + 'l1': L1FilterPruner, + 'l2': L2FilterPruner, + 'fpgm': FPGMPruner +} diff --git a/utils/third_party/nni_new/algorithms/compression/pytorch/pruning/dependency_aware_pruner.py b/utils/third_party/nni_new/algorithms/compression/pytorch/pruning/dependency_aware_pruner.py new file mode 100644 index 0000000000000000000000000000000000000000..60f94ee317e8ab70f7079b8a8228da24b968ea10 --- /dev/null +++ b/utils/third_party/nni_new/algorithms/compression/pytorch/pruning/dependency_aware_pruner.py @@ -0,0 +1,162 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +from schema import And, Optional, SchemaError +from .....common.graph_utils import TorchModuleGraph +from .....compression.pytorch.utils.shape_dependency import ChannelDependency, GroupDependency +from .....compression.pytorch.utils.config_validation import CompressorSchema +from .....compression.pytorch.compressor import Pruner +from .constants import MASKER_DICT + +__all__ = ['DependencyAwarePruner'] + +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) + + +class DependencyAwarePruner(Pruner): + """ + DependencyAwarePruner has two ways to calculate the masks + for conv layers. In the normal way, the DependencyAwarePruner + will calculate the mask of each layer separately. For example, each + conv layer determine which filters should be pruned according to its L1 + norm. In constrast, in the dependency-aware way, the layers that in a + dependency group will be pruned jointly and these layers will be forced + to prune the same channels. + """ + + def __init__(self, model, config_list, optimizer=None, pruning_algorithm='level', dependency_aware=False, + dummy_input=None, **algo_kwargs): + super().__init__(model, config_list=config_list, optimizer=optimizer) + + self.dependency_aware = dependency_aware + self.dummy_input = dummy_input + + if self.dependency_aware: + if not self._supported_dependency_aware(): + raise ValueError('This pruner does not support dependency aware!') + + errmsg = "When dependency_aware is set, the dummy_input should not be None" + assert self.dummy_input is not None, errmsg + # Get the TorchModuleGraph of the target model + # to trace the model, we need to unwrap the wrappers + self._unwrap_model() + self.graph = TorchModuleGraph(model, dummy_input) + self._wrap_model() + self.channel_depen = ChannelDependency( + traced_model=self.graph.trace) + self.group_depen = GroupDependency(traced_model=self.graph.trace) + self.channel_depen = self.channel_depen.dependency_sets + self.channel_depen = { + name: sets for sets in self.channel_depen for name in sets} + self.group_depen = self.group_depen.dependency_sets + + self.masker = MASKER_DICT[pruning_algorithm]( + model, self, **algo_kwargs) + # set the dependency-aware switch for the masker + self.masker.dependency_aware = dependency_aware + self.set_wrappers_attribute("if_calculated", False) + + def calc_mask(self, wrapper, wrapper_idx=None): + if not wrapper.if_calculated: + sparsity = wrapper.config['sparsity'] + masks = self.masker.calc_mask( + sparsity=sparsity, wrapper=wrapper, wrapper_idx=wrapper_idx) + + # masker.calc_mask returns None means calc_mask is not calculated sucessfully, can try later + if masks is not None: + wrapper.if_calculated = True + return masks + else: + return None + + def update_mask(self): + if not self.dependency_aware: + # if we use the normal way to update the mask, + # then call the update_mask of the father class + super(DependencyAwarePruner, self).update_mask() + else: + # if we update the mask in a dependency-aware way + # then we call _dependency_update_mask + self._dependency_update_mask() + + def validate_config(self, model, config_list): + schema = CompressorSchema([{ + Optional('sparsity'): And(float, lambda n: 0 < n < 1), + Optional('op_types'): ['Conv2d'], + Optional('op_names'): [str], + Optional('exclude'): bool + }], model, logger) + + schema.validate(config_list) + for config in config_list: + if 'exclude' not in config and 'sparsity' not in config: + raise SchemaError('Either sparisty or exclude must be specified!') + + def _supported_dependency_aware(self): + raise NotImplementedError + + def _dependency_calc_mask(self, wrappers, channel_dsets, wrappers_idx=None): + """ + calculate the masks for the conv layers in the same + channel dependecy set. All the layers passed in have + the same number of channels. + + Parameters + ---------- + wrappers: list + The list of the wrappers that in the same channel dependency + set. + wrappers_idx: list + The list of the indexes of wrapppers. + Returns + ------- + masks: dict + A dict object that contains the masks of the layers in this + dependency group, the key is the name of the convolutional layers. + """ + # The number of the groups for each conv layers + # Note that, this number may be different from its + # original number of groups of filters. + groups = [self.group_depen[_w.name] for _w in wrappers] + sparsities = [_w.config['sparsity'] for _w in wrappers] + masks = self.masker.calc_mask( + sparsities, wrappers, wrappers_idx, channel_dsets=channel_dsets, groups=groups) + if masks is not None: + # if masks is None, then the mask calculation fails. + # for example, in activation related maskers, we should + # pass enough batches of data to the model, so that the + # masks can be calculated successfully. + for _w in wrappers: + _w.if_calculated = True + return masks + + def _dependency_update_mask(self): + """ + In the original update_mask, the wraper of each layer will update its + own mask according to the sparsity specified in the config_list. However, in + the _dependency_update_mask, we may prune several layers at the same + time according the sparsities and the channel/group dependencies. + """ + name2wrapper = {x.name: x for x in self.get_modules_wrapper()} + wrapper2index = {x: i for i, x in enumerate(self.get_modules_wrapper())} + for wrapper in self.get_modules_wrapper(): + if wrapper.if_calculated: + continue + # find all the conv layers that have channel dependecy with this layer + # and prune all these layers at the same time. + _names = [x for x in self.channel_depen[wrapper.name]] + # logger.info('Pruning the dependent layers: %s', ','.join(_names)) + _wrappers = [name2wrapper[name] + for name in _names if name in name2wrapper] + _wrapper_idxes = [wrapper2index[_w] for _w in _wrappers] + + masks = self._dependency_calc_mask( + _wrappers, _names, wrappers_idx=_wrapper_idxes) + if masks is not None: + for layer in masks: + for mask_type in masks[layer]: + assert hasattr(name2wrapper[layer], mask_type), "there is no attribute '%s' in wrapper on %s" \ + % (mask_type, layer) + setattr(name2wrapper[layer], mask_type, masks[layer][mask_type]) diff --git a/utils/third_party/nni_new/algorithms/compression/pytorch/pruning/finegrained_pruning_masker.py b/utils/third_party/nni_new/algorithms/compression/pytorch/pruning/finegrained_pruning_masker.py new file mode 100644 index 0000000000000000000000000000000000000000..f4aa174233e977eed7b59c6ba65136526a738603 --- /dev/null +++ b/utils/third_party/nni_new/algorithms/compression/pytorch/pruning/finegrained_pruning_masker.py @@ -0,0 +1,31 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +import torch +from .weight_masker import WeightMasker + +__all__ = ['LevelPrunerMasker'] + +logger = logging.getLogger('torch pruner') + + +class LevelPrunerMasker(WeightMasker): + """ + Prune to an exact pruning level specification + """ + + def calc_mask(self, sparsity, wrapper, wrapper_idx=None): + weight = wrapper.module.weight.data.clone() + if wrapper.weight_mask is not None: + # apply base mask for iterative pruning + weight = weight * wrapper.weight_mask + + w_abs = weight.abs() + k = int(weight.numel() * sparsity) + if k == 0: + return {'weight_mask': torch.ones(weight.shape).type_as(weight)} + threshold = torch.topk(w_abs.view(-1), k, largest=False)[0].max() + mask_weight = torch.gt(w_abs, threshold).type_as(weight) + mask = {'weight_mask': mask_weight} + return mask diff --git a/utils/third_party/nni_new/algorithms/compression/pytorch/pruning/iterative_pruner.py b/utils/third_party/nni_new/algorithms/compression/pytorch/pruning/iterative_pruner.py new file mode 100644 index 0000000000000000000000000000000000000000..65a98694d2650112524c1c73a393ad7dfa0569d3 --- /dev/null +++ b/utils/third_party/nni_new/algorithms/compression/pytorch/pruning/iterative_pruner.py @@ -0,0 +1,596 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +import copy +import torch +from schema import And, Optional +from .....compression.pytorch.utils.config_validation import CompressorSchema +from .constants import MASKER_DICT +from .dependency_aware_pruner import DependencyAwarePruner + +__all__ = ['AGPPruner', 'ADMMPruner', 'SlimPruner', 'TaylorFOWeightFilterPruner', 'ActivationAPoZRankFilterPruner', + 'ActivationMeanRankFilterPruner'] + +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) + + +class IterativePruner(DependencyAwarePruner): + """ + Prune model during the training process. + """ + + def __init__(self, model, config_list, optimizer=None, pruning_algorithm='slim', trainer=None, criterion=None, + num_iterations=20, epochs_per_iteration=5, dependency_aware=False, dummy_input=None, **algo_kwargs): + """ + Parameters + ---------- + model: torch.nn.Module + Model to be pruned + config_list: list + List on pruning configs + optimizer: torch.optim.Optimizer + Optimizer used to train model + pruning_algorithm: str + algorithms being used to prune model + trainer: function + Function used to train the model. + Users should write this function as a normal function to train the Pytorch model + and include `model, optimizer, criterion, epoch` as function arguments. + criterion: function + Function used to calculate the loss between the target and the output. + For example, you can use ``torch.nn.CrossEntropyLoss()`` as input. + num_iterations: int + Total number of iterations in pruning process. We will calculate mask at the end of an iteration. + epochs_per_iteration: Union[int, list] + The number of training epochs for each iteration. `int` represents the same value for each iteration. + `list` represents the specific value for each iteration. + dependency_aware: bool + If prune the model in a dependency-aware way. + dummy_input: torch.Tensor + The dummy input to analyze the topology constraints. Note that, + the dummy_input should on the same device with the model. + algo_kwargs: dict + Additional parameters passed to pruning algorithm masker class + """ + super().__init__(model, config_list, optimizer, pruning_algorithm, dependency_aware, dummy_input, **algo_kwargs) + + if isinstance(epochs_per_iteration, list): + assert len(epochs_per_iteration) == num_iterations, 'num_iterations should equal to the length of epochs_per_iteration' + self.epochs_per_iteration = epochs_per_iteration + else: + assert num_iterations > 0, 'num_iterations should >= 1' + self.epochs_per_iteration = [epochs_per_iteration] * num_iterations + + self._validate_iteration_params() + + self._trainer = trainer + self._criterion = criterion + + def _fresh_calculated(self): + for wrapper in self.get_modules_wrapper(): + wrapper.if_calculated = False + + def _validate_iteration_params(self): + assert all(num >= 0 for num in self.epochs_per_iteration), 'all epoch number need >= 0' + + def compress(self): + training = self.bound_model.training + self.bound_model.train() + for _, epochs_num in enumerate(self.epochs_per_iteration): + self._fresh_calculated() + for epoch in range(epochs_num): + self._trainer(self.bound_model, optimizer=self.optimizer, criterion=self._criterion, epoch=epoch) + # NOTE: workaround for statistics_batch_num bigger than max batch number in one epoch, need refactor + if hasattr(self.masker, 'statistics_batch_num') and hasattr(self, 'iterations'): + if self.iterations < self.masker.statistics_batch_num: + self.iterations = self.masker.statistics_batch_num + self.update_mask() + self.bound_model.train(training) + + return self.bound_model + + +class AGPPruner(IterativePruner): + """ + Parameters + ---------- + model : torch.nn.Module + Model to be pruned. + config_list : listlist + Supported keys: + - sparsity : This is to specify the sparsity operations to be compressed to. + - op_types : See supported type in your specific pruning algorithm. + optimizer: torch.optim.Optimizer + Optimizer used to train model. + trainer: function + Function to train the model + criterion: function + Function used to calculate the loss between the target and the output. + For example, you can use ``torch.nn.CrossEntropyLoss()`` as input. + num_iterations: int + Total number of iterations in pruning process. We will calculate mask at the end of an iteration. + epochs_per_iteration: int + The number of training epochs for each iteration. + pruning_algorithm: str + Algorithms being used to prune model, + choose from `['level', 'slim', 'l1', 'l2', 'fpgm', 'taylorfo', 'apoz', 'mean_activation']`, by default `level` + """ + + def __init__(self, model, config_list, optimizer, trainer, criterion, num_iterations=10, epochs_per_iteration=1, pruning_algorithm='level'): + super().__init__(model, config_list, optimizer=optimizer, trainer=trainer, criterion=criterion, + num_iterations=num_iterations, epochs_per_iteration=epochs_per_iteration) + assert isinstance(optimizer, torch.optim.Optimizer), "AGP pruner is an iterative pruner, please pass optimizer of the model to it" + self.masker = MASKER_DICT[pruning_algorithm](model, self) + self.now_epoch = 0 + self.freq = epochs_per_iteration + self.end_epoch = epochs_per_iteration * num_iterations + self.set_wrappers_attribute("if_calculated", False) + + def validate_config(self, model, config_list): + """ + Parameters + ---------- + model : torch.nn.Module + Model to be pruned + config_list : list + List on pruning configs + """ + schema = CompressorSchema([{ + 'sparsity': And(float, lambda n: 0 <= n <= 1), + Optional('op_types'): [str], + Optional('op_names'): [str] + }], model, logger) + + schema.validate(config_list) + + def _supported_dependency_aware(self): + return False + + def calc_mask(self, wrapper, wrapper_idx=None): + """ + Calculate the mask of given layer. + Scale factors with the smallest absolute value in the BN layer are masked. + Parameters + ---------- + wrapper : Module + the layer to instrument the compression operation + wrapper_idx: int + index of this wrapper in pruner's all wrappers + Returns + ------- + dict | None + Dictionary for storing masks, keys of the dict: + 'weight_mask': weight mask tensor + 'bias_mask': bias mask tensor (optional) + """ + + config = wrapper.config + + if wrapper.if_calculated: + return None + + if not self.now_epoch % self.freq == 0: + return None + + target_sparsity = self.compute_target_sparsity(config) + new_mask = self.masker.calc_mask(sparsity=target_sparsity, wrapper=wrapper, wrapper_idx=wrapper_idx) + + if new_mask is not None: + wrapper.if_calculated = True + + return new_mask + + def compute_target_sparsity(self, config): + """ + Calculate the sparsity for pruning + Parameters + ---------- + config : dict + Layer's pruning config + Returns + ------- + float + Target sparsity to be pruned + """ + + initial_sparsity = 0 + self.target_sparsity = final_sparsity = config.get('sparsity', 0) + + if initial_sparsity >= final_sparsity: + logger.warning('your initial_sparsity >= final_sparsity') + return final_sparsity + + if self.end_epoch == 1 or self.end_epoch <= self.now_epoch: + return final_sparsity + + span = ((self.end_epoch - 1) // self.freq) * self.freq + assert span > 0 + self.target_sparsity = (final_sparsity + (initial_sparsity - final_sparsity) * (1.0 - (self.now_epoch / span)) ** 3) + return self.target_sparsity + + def update_epoch(self, epoch): + """ + Update epoch + Parameters + ---------- + epoch : int + current training epoch + """ + + if epoch > 0: + self.now_epoch = epoch + for wrapper in self.get_modules_wrapper(): + wrapper.if_calculated = False + + # TODO: need refactor + def compress(self): + training = self.bound_model.training + self.bound_model.train() + + for epoch in range(self.end_epoch): + self.update_epoch(epoch) + self._trainer(self.bound_model, optimizer=self.optimizer, criterion=self._criterion, epoch=epoch) + self.update_mask() + logger.info(f'sparsity is {self.target_sparsity:.2f} at epoch {epoch}') + self.get_pruned_weights() + + self.bound_model.train(training) + + return self.bound_model + + +class ADMMPruner(IterativePruner): + """ + A Pytorch implementation of ADMM Pruner algorithm. + + Parameters + ---------- + model : torch.nn.Module + Model to be pruned. + config_list : list + List on pruning configs. + trainer : function + Function used for the first subproblem. + Users should write this function as a normal function to train the Pytorch model + and include `model, optimizer, criterion, epoch` as function arguments. + criterion: function + Function used to calculate the loss between the target and the output. By default, we use CrossEntropyLoss in ADMMPruner. + For example, you can use ``torch.nn.CrossEntropyLoss()`` as input. + num_iterations: int + Total number of iterations in pruning process. We will calculate mask after we finish all iterations in ADMMPruner. + epochs_per_iteration: int + Training epochs of the first subproblem. + row : float + Penalty parameters for ADMM training. + base_algo : str + Base pruning algorithm. `level`, `l1`, `l2` or `fpgm`, by default `l1`. Given the sparsity distribution among + the ops, the assigned `base_algo` is used to decide which filters/channels/weights to prune. + """ + + def __init__(self, model, config_list, trainer, criterion=torch.nn.CrossEntropyLoss(), + num_iterations=30, epochs_per_iteration=5, row=1e-4, base_algo='l1'): + self._base_algo = base_algo + + super().__init__(model, config_list) + + self._trainer = trainer + self.optimizer = torch.optim.Adam( + self.bound_model.parameters(), lr=1e-3, weight_decay=5e-5) + self._criterion = criterion + self._num_iterations = num_iterations + self._training_epochs = epochs_per_iteration + self._row = row + + self.set_wrappers_attribute("if_calculated", False) + self.masker = MASKER_DICT[self._base_algo](self.bound_model, self) + + self.patch_optimizer_before(self._callback) + + def validate_config(self, model, config_list): + """ + Parameters + ---------- + model : torch.nn.Module + Model to be pruned + config_list : list + List on pruning configs + """ + + if self._base_algo == 'level': + schema = CompressorSchema([{ + 'sparsity': And(float, lambda n: 0 < n < 1), + Optional('op_types'): [str], + Optional('op_names'): [str], + }], model, logger) + elif self._base_algo in ['l1', 'l2', 'fpgm']: + schema = CompressorSchema([{ + 'sparsity': And(float, lambda n: 0 < n < 1), + 'op_types': ['Conv2d'], + Optional('op_names'): [str] + }], model, logger) + + schema.validate(config_list) + + def _supported_dependency_aware(self): + return False + + def _projection(self, weight, sparsity, wrapper): + ''' + Return the Euclidean projection of the weight matrix according to the pruning mode. + + Parameters + ---------- + weight : tensor + original matrix + sparsity : float + the ratio of parameters which need to be set to zero + wrapper: PrunerModuleWrapper + layer wrapper of this layer + + Returns + ------- + tensor + the projected matrix + ''' + wrapper_copy = copy.deepcopy(wrapper) + wrapper_copy.module.weight.data = weight + return weight.data.mul(self.masker.calc_mask(sparsity, wrapper_copy)['weight_mask']) + + def _callback(self): + # callback function to do additonal optimization, refer to the deriatives of Formula (7) + for i, wrapper in enumerate(self.get_modules_wrapper()): + wrapper.module.weight.data -= self._row * \ + (wrapper.module.weight.data - self.Z[i] + self.U[i]) + + def compress(self): + """ + Compress the model with ADMM. + + Returns + ------- + torch.nn.Module + model with specified modules compressed. + """ + logger.info('Starting ADMM Compression...') + + # initiaze Z, U + # Z_i^0 = W_i^0 + # U_i^0 = 0 + self.Z = [] + self.U = [] + for wrapper in self.get_modules_wrapper(): + z = wrapper.module.weight.data + self.Z.append(z) + self.U.append(torch.zeros_like(z)) + + # Loss = cross_entropy + l2 regulization + \Sum_{i=1}^N \row_i ||W_i - Z_i^k + U_i^k||^2 + # optimization iteration + for k in range(self._num_iterations): + logger.info('ADMM iteration : %d', k) + + # step 1: optimize W with AdamOptimizer + for epoch in range(self._training_epochs): + self._trainer(self.bound_model, optimizer=self.optimizer, criterion=self._criterion, epoch=epoch) + + # step 2: update Z, U + # Z_i^{k+1} = projection(W_i^{k+1} + U_i^k) + # U_i^{k+1} = U^k + W_i^{k+1} - Z_i^{k+1} + for i, wrapper in enumerate(self.get_modules_wrapper()): + z = wrapper.module.weight.data + self.U[i] + self.Z[i] = self._projection(z, wrapper.config['sparsity'], wrapper) + self.U[i] = self.U[i] + wrapper.module.weight.data - self.Z[i] + + # apply prune + self.update_mask() + + logger.info('Compression finished.') + + return self.bound_model + + +class SlimPruner(IterativePruner): + """ + Parameters + ---------- + model : torch.nn.Module + Model to be pruned + config_list : list + Supported keys: + - sparsity : This is to specify the sparsity operations to be compressed to. + - op_types : Only BatchNorm2d is supported in Slim Pruner. + optimizer : torch.optim.Optimizer + Optimizer used to train model + trainer : function + Function used to sparsify BatchNorm2d scaling factors. + Users should write this function as a normal function to train the Pytorch model + and include `model, optimizer, criterion, epoch` as function arguments. + criterion : function + Function used to calculate the loss between the target and the output. + For example, you can use ``torch.nn.CrossEntropyLoss()`` as input. + sparsifying_training_epochs: int + The number of channel sparsity regularization training epochs before pruning. + scale : float + Penalty parameters for sparsification. + dependency_aware: bool + If prune the model in a dependency-aware way. If it is `True`, this pruner will + prune the model according to the l2-norm of weights and the channel-dependency or + group-dependency of the model. In this way, the pruner will force the conv layers + that have dependencies to prune the same channels, so the speedup module can better + harvest the speed benefit from the pruned model. Note that, if this flag is set True + , the dummy_input cannot be None, because the pruner needs a dummy input to trace the + dependency between the conv layers. + dummy_input : torch.Tensor + The dummy input to analyze the topology constraints. Note that, the dummy_input + should on the same device with the model. + """ + + def __init__(self, model, config_list, optimizer, trainer, criterion, sparsifying_training_epochs=10, scale=0.0001, + dependency_aware=False, dummy_input=None): + super().__init__(model, config_list, optimizer=optimizer, pruning_algorithm='slim', trainer=trainer, criterion=criterion, + num_iterations=1, epochs_per_iteration=sparsifying_training_epochs, dependency_aware=dependency_aware, + dummy_input=dummy_input) + self.scale = scale + self.patch_optimizer_before(self._callback) + + def validate_config(self, model, config_list): + schema = CompressorSchema([{ + 'sparsity': And(float, lambda n: 0 < n < 1), + 'op_types': ['BatchNorm2d'], + Optional('op_names'): [str] + }], model, logger) + + schema.validate(config_list) + + if len(config_list) > 1: + logger.warning('Slim pruner only supports 1 configuration') + + def _supported_dependency_aware(self): + return True + + def _callback(self): + for _, wrapper in enumerate(self.get_modules_wrapper()): + wrapper.module.weight.grad.data.add_(self.scale * torch.sign(wrapper.module.weight.data)) + + +class TaylorFOWeightFilterPruner(IterativePruner): + """ + Parameters + ---------- + model : torch.nn.Module + Model to be pruned + config_list : list + Supported keys: + - sparsity : How much percentage of convolutional filters are to be pruned. + - op_types : Currently only Conv2d is supported in TaylorFOWeightFilterPruner. + optimizer: torch.optim.Optimizer + Optimizer used to train model + trainer : function + Function used to sparsify BatchNorm2d scaling factors. + Users should write this function as a normal function to train the Pytorch model + and include `model, optimizer, criterion, epoch` as function arguments. + criterion : function + Function used to calculate the loss between the target and the output. + For example, you can use ``torch.nn.CrossEntropyLoss()`` as input. + sparsifying_training_batches: int + The number of batches to collect the contributions. Note that the number need to be less than the maximum batch number in one epoch. + dependency_aware: bool + If prune the model in a dependency-aware way. If it is `True`, this pruner will + prune the model according to the l2-norm of weights and the channel-dependency or + group-dependency of the model. In this way, the pruner will force the conv layers + that have dependencies to prune the same channels, so the speedup module can better + harvest the speed benefit from the pruned model. Note that, if this flag is set True + , the dummy_input cannot be None, because the pruner needs a dummy input to trace the + dependency between the conv layers. + dummy_input : torch.Tensor + The dummy input to analyze the topology constraints. Note that, the dummy_input + should on the same device with the model. + """ + + def __init__(self, model, config_list, optimizer, trainer, criterion, sparsifying_training_batches=1, + dependency_aware=False, dummy_input=None): + super().__init__(model, config_list, optimizer=optimizer, pruning_algorithm='taylorfo', trainer=trainer, + criterion=criterion, statistics_batch_num=sparsifying_training_batches, num_iterations=1, + epochs_per_iteration=1, dependency_aware=dependency_aware, + dummy_input=dummy_input) + + def _supported_dependency_aware(self): + return True + + +class ActivationAPoZRankFilterPruner(IterativePruner): + """ + Parameters + ---------- + model : torch.nn.Module + Model to be pruned + config_list : list + Supported keys: + - sparsity : How much percentage of convolutional filters are to be pruned. + - op_types : Only Conv2d is supported in ActivationAPoZRankFilterPruner. + optimizer: torch.optim.Optimizer + Optimizer used to train model + trainer: function + Function used to train the model. + Users should write this function as a normal function to train the Pytorch model + and include `model, optimizer, criterion, epoch` as function arguments. + criterion : function + Function used to calculate the loss between the target and the output. + For example, you can use ``torch.nn.CrossEntropyLoss()`` as input. + activation: str + The activation type. + sparsifying_training_batches: int + The number of batches to collect the contributions. Note that the number need to be less than the maximum batch number in one epoch. + dependency_aware: bool + If prune the model in a dependency-aware way. If it is `True`, this pruner will + prune the model according to the l2-norm of weights and the channel-dependency or + group-dependency of the model. In this way, the pruner will force the conv layers + that have dependencies to prune the same channels, so the speedup module can better + harvest the speed benefit from the pruned model. Note that, if this flag is set True + , the dummy_input cannot be None, because the pruner needs a dummy input to trace the + dependency between the conv layers. + dummy_input : torch.Tensor + The dummy input to analyze the topology constraints. Note that, the dummy_input + should on the same device with the model. + + """ + + def __init__(self, model, config_list, optimizer, trainer, criterion, activation='relu', + sparsifying_training_batches=1, dependency_aware=False, dummy_input=None): + super().__init__(model, config_list, pruning_algorithm='apoz', optimizer=optimizer, trainer=trainer, + criterion=criterion, dependency_aware=dependency_aware, dummy_input=dummy_input, + activation=activation, statistics_batch_num=sparsifying_training_batches, num_iterations=1, + epochs_per_iteration=1) + self.patch_optimizer(self.update_mask) + + def _supported_dependency_aware(self): + return True + + +class ActivationMeanRankFilterPruner(IterativePruner): + """ + Parameters + ---------- + model : torch.nn.Module + Model to be pruned + config_list : list + Supported keys: + - sparsity : How much percentage of convolutional filters are to be pruned. + - op_types : Only Conv2d is supported in ActivationMeanRankFilterPruner. + optimizer: torch.optim.Optimizer + Optimizer used to train model. + trainer: function + Function used to train the model. + Users should write this function as a normal function to train the Pytorch model + and include `model, optimizer, criterion, epoch` as function arguments. + criterion : function + Function used to calculate the loss between the target and the output. + For example, you can use ``torch.nn.CrossEntropyLoss()`` as input. + activation: str + The activation type. + sparsifying_training_batches: int + The number of batches to collect the contributions. Note that the number need to be less than the maximum batch number in one epoch. + dependency_aware: bool + If prune the model in a dependency-aware way. If it is `True`, this pruner will + prune the model according to the l2-norm of weights and the channel-dependency or + group-dependency of the model. In this way, the pruner will force the conv layers + that have dependencies to prune the same channels, so the speedup module can better + harvest the speed benefit from the pruned model. Note that, if this flag is set True + , the dummy_input cannot be None, because the pruner needs a dummy input to trace the + dependency between the conv layers. + dummy_input : torch.Tensor + The dummy input to analyze the topology constraints. Note that, the dummy_input + should on the same device with the model. + """ + + def __init__(self, model, config_list, optimizer, trainer, criterion, activation='relu', + sparsifying_training_batches=1, dependency_aware=False, dummy_input=None): + super().__init__(model, config_list, pruning_algorithm='mean_activation', optimizer=optimizer, trainer=trainer, + criterion=criterion, dependency_aware=dependency_aware, dummy_input=dummy_input, + activation=activation, statistics_batch_num=sparsifying_training_batches, num_iterations=1, + epochs_per_iteration=1) + self.patch_optimizer(self.update_mask) + + def _supported_dependency_aware(self): + return True diff --git a/utils/third_party/nni_new/algorithms/compression/pytorch/pruning/lottery_ticket.py b/utils/third_party/nni_new/algorithms/compression/pytorch/pruning/lottery_ticket.py new file mode 100644 index 0000000000000000000000000000000000000000..caa1c831e6025ab69d0e606df850584959aaa617 --- /dev/null +++ b/utils/third_party/nni_new/algorithms/compression/pytorch/pruning/lottery_ticket.py @@ -0,0 +1,146 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import copy +import logging +import torch +from schema import And, Optional +from nni.compression.pytorch.utils.config_validation import CompressorSchema +from nni.compression.pytorch.compressor import Pruner +from .finegrained_pruning_masker import LevelPrunerMasker + +logger = logging.getLogger('torch pruner') + +class LotteryTicketPruner(Pruner): + """ + Parameters + ---------- + model : pytorch model + The model to be pruned + config_list : list + Supported keys: + - prune_iterations : The number of rounds for the iterative pruning. + - sparsity : The final sparsity when the compression is done. + optimizer : pytorch optimizer + The optimizer for the model + lr_scheduler : pytorch lr scheduler + The lr scheduler for the model if used + reset_weights : bool + Whether reset weights and optimizer at the beginning of each round. + """ + def __init__(self, model, config_list, optimizer=None, lr_scheduler=None, reset_weights=True): + # save init weights and optimizer + self.reset_weights = reset_weights + if self.reset_weights: + self._model = model + self._optimizer = optimizer + self._model_state = copy.deepcopy(model.state_dict()) + self._optimizer_state = copy.deepcopy(optimizer.state_dict()) + self._lr_scheduler = lr_scheduler + if lr_scheduler is not None: + self._scheduler_state = copy.deepcopy(lr_scheduler.state_dict()) + + super().__init__(model, config_list, optimizer) + self.curr_prune_iteration = None + self.prune_iterations = config_list[0]['prune_iterations'] + self.masker = LevelPrunerMasker(model, self) + + def validate_config(self, model, config_list): + """ + Parameters + ---------- + model : torch.nn.Module + Model to be pruned + config_list : list + Supported keys: + - prune_iterations : The number of rounds for the iterative pruning. + - sparsity : The final sparsity when the compression is done. + """ + schema = CompressorSchema([{ + 'sparsity': And(float, lambda n: 0 < n < 1), + 'prune_iterations': And(int, lambda n: n > 0), + Optional('op_types'): [str], + Optional('op_names'): [str] + }], model, logger) + + schema.validate(config_list) + assert len(set([x['prune_iterations'] for x in config_list])) == 1, 'The values of prune_iterations must be equal in your config' + + def _calc_sparsity(self, sparsity): + keep_ratio_once = (1 - sparsity) ** (1 / self.prune_iterations) + curr_keep_ratio = keep_ratio_once ** self.curr_prune_iteration + return max(1 - curr_keep_ratio, 0) + + def _calc_mask(self, wrapper, sparsity): + weight = wrapper.module.weight.data + if self.curr_prune_iteration == 0: + mask = {'weight_mask': torch.ones(weight.shape).type_as(weight)} + else: + curr_sparsity = self._calc_sparsity(sparsity) + mask = self.masker.calc_mask(sparsity=curr_sparsity, wrapper=wrapper) + return mask + + def calc_mask(self, wrapper, **kwargs): + """ + Generate mask for the given ``weight``. + + Parameters + ---------- + wrapper : Module + The layer to be pruned + + Returns + ------- + tensor + The mask for this weight, it is ```None``` because this pruner + calculates and assigns masks in ```prune_iteration_start```, + no need to do anything in this function. + """ + return None + + def get_prune_iterations(self): + """ + Return the range for iterations. + In the first prune iteration, masks are all one, thus, add one more iteration + + Returns + ------- + list + A list for pruning iterations + """ + return range(self.prune_iterations + 1) + + def prune_iteration_start(self): + """ + Control the pruning procedure on updated epoch number. + Should be called at the beginning of the epoch. + """ + if self.curr_prune_iteration is None: + self.curr_prune_iteration = 0 + else: + self.curr_prune_iteration += 1 + assert self.curr_prune_iteration < self.prune_iterations + 1, 'Exceed the configured prune_iterations' + + modules_wrapper = self.get_modules_wrapper() + modules_to_compress = self.get_modules_to_compress() + for layer, config in modules_to_compress: + module_wrapper = None + for wrapper in modules_wrapper: + if wrapper.name == layer.name: + module_wrapper = wrapper + break + assert module_wrapper is not None + + sparsity = config.get('sparsity') + mask = self._calc_mask(module_wrapper, sparsity) + # TODO: directly use weight_mask is not good + module_wrapper.weight_mask = mask['weight_mask'] + # there is no mask for bias + + # reinit weights back to original after new masks are generated + if self.reset_weights: + # should use this member function to reset model weights + self.load_model_state_dict(self._model_state) + self._optimizer.load_state_dict(self._optimizer_state) + if self._lr_scheduler is not None: + self._lr_scheduler.load_state_dict(self._scheduler_state) diff --git a/utils/third_party/nni_new/algorithms/compression/pytorch/pruning/net_adapt_pruner.py b/utils/third_party/nni_new/algorithms/compression/pytorch/pruning/net_adapt_pruner.py new file mode 100644 index 0000000000000000000000000000000000000000..08416319eaaa6b4fb2ff4e2214f73d80e211020d --- /dev/null +++ b/utils/third_party/nni_new/algorithms/compression/pytorch/pruning/net_adapt_pruner.py @@ -0,0 +1,351 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +import os +import copy +import json +import torch +from schema import And, Optional + +from nni.utils import OptimizeMode + +from nni.compression.pytorch.compressor import Pruner +from nni.compression.pytorch.utils.config_validation import CompressorSchema +from nni.compression.pytorch.utils.num_param_counter import get_total_num_weights +from .constants_pruner import PRUNER_DICT + + +_logger = logging.getLogger(__name__) + + +class NetAdaptPruner(Pruner): + """ + A Pytorch implementation of NetAdapt compression algorithm. + + Parameters + ---------- + model : pytorch model + The model to be pruned. + config_list : list + Supported keys: + - sparsity : The target overall sparsity. + - op_types : The operation type to prune. + short_term_fine_tuner : function + function to short-term fine tune the masked model. + This function should include `model` as the only parameter, + and fine tune the model for a short term after each pruning iteration. + Example:: + + def short_term_fine_tuner(model, epoch=3): + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + train_loader = ... + criterion = torch.nn.CrossEntropyLoss() + optimizer = torch.optim.SGD(model.parameters(), lr=0.01) + model.train() + for _ in range(epoch): + for batch_idx, (data, target) in enumerate(train_loader): + data, target = data.to(device), target.to(device) + optimizer.zero_grad() + output = model(data) + loss = criterion(output, target) + loss.backward() + optimizer.step() + evaluator : function + function to evaluate the masked model. + This function should include `model` as the only parameter, and returns a scalar value. + Example:: + + def evaluator(model): + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + val_loader = ... + model.eval() + correct = 0 + with torch.no_grad(): + for data, target in val_loader: + data, target = data.to(device), target.to(device) + output = model(data) + # get the index of the max log-probability + pred = output.argmax(dim=1, keepdim=True) + correct += pred.eq(target.view_as(pred)).sum().item() + accuracy = correct / len(val_loader.dataset) + return accuracy + optimize_mode : str + optimize mode, `maximize` or `minimize`, by default `maximize`. + base_algo : str + Base pruning algorithm. `level`, `l1`, `l2` or `fpgm`, by default `l1`. Given the sparsity distribution among the ops, + the assigned `base_algo` is used to decide which filters/channels/weights to prune. + sparsity_per_iteration : float + sparsity to prune in each iteration. + experiment_data_dir : str + PATH to save experiment data, + including the config_list generated for the base pruning algorithm and the performance of the pruned model. + """ + + def __init__(self, model, config_list, short_term_fine_tuner, evaluator, + optimize_mode='maximize', base_algo='l1', sparsity_per_iteration=0.05, experiment_data_dir='./'): + # models used for iterative pruning and evaluation + self._model_to_prune = copy.deepcopy(model) + self._base_algo = base_algo + + super().__init__(model, config_list) + + self._short_term_fine_tuner = short_term_fine_tuner + self._evaluator = evaluator + self._optimize_mode = OptimizeMode(optimize_mode) + + # hyper parameters for NetAdapt algorithm + self._sparsity_per_iteration = sparsity_per_iteration + + # overall pruning rate + self._sparsity = config_list[0]['sparsity'] + + # config_list + self._config_list_generated = [] + + self._experiment_data_dir = experiment_data_dir + if not os.path.exists(self._experiment_data_dir): + os.makedirs(self._experiment_data_dir) + + self._tmp_model_path = os.path.join(self._experiment_data_dir, 'tmp_model.pth') + + def validate_config(self, model, config_list): + """ + Parameters + ---------- + model : torch.nn.Module + Model to be pruned + config_list : list + List on pruning configs + """ + + if self._base_algo == 'level': + schema = CompressorSchema([{ + 'sparsity': And(float, lambda n: 0 < n < 1), + Optional('op_types'): [str], + Optional('op_names'): [str], + }], model, _logger) + elif self._base_algo in ['l1', 'l2', 'fpgm']: + schema = CompressorSchema([{ + 'sparsity': And(float, lambda n: 0 < n < 1), + 'op_types': ['Conv2d'], + Optional('op_names'): [str] + }], model, _logger) + + schema.validate(config_list) + + def calc_mask(self, wrapper, **kwargs): + return None + + def _update_config_list(self, config_list, op_name, sparsity): + ''' + update sparsity of op_name in config_list + ''' + config_list_updated = copy.deepcopy(config_list) + + for idx, item in enumerate(config_list): + if op_name in item['op_names']: + config_list_updated[idx]['sparsity'] = sparsity + return config_list_updated + + # if op_name is not in self._config_list_generated, create a new json item + if self._base_algo in ['l1', 'l2', 'fpgm']: + config_list_updated.append( + {'sparsity': sparsity, 'op_types': ['Conv2d'], 'op_names': [op_name]}) + elif self._base_algo == 'level': + config_list_updated.append( + {'sparsity': sparsity, 'op_names': [op_name]}) + + return config_list_updated + + def _get_op_num_weights_remained(self, op_name, module): + ''' + Get the number of weights remained after channel pruning with current sparsity + + Returns + ------- + int + remained number of weights of the op + ''' + + # if op is wrapped by the pruner + for wrapper in self.get_modules_wrapper(): + if wrapper.name == op_name: + return wrapper.weight_mask.sum().item() + + # if op is not wrapped by the pruner + return module.weight.data.numel() + + def _get_op_sparsity(self, op_name): + for config in self._config_list_generated: + if 'op_names' in config and op_name in config['op_names']: + return config['sparsity'] + return 0 + + def _calc_num_related_weights(self, op_name): + ''' + Calculate total number weights of the op and the next op, applicable only for models without dependencies among ops + + Parameters + ---------- + op_name : str + + Returns + ------- + int + total number of all the realted (current and the next) op weights + ''' + num_weights = 0 + flag_found = False + previous_name = None + previous_module = None + + for name, module in self._model_to_prune.named_modules(): + if not flag_found and name != op_name and type(module).__name__ in ['Conv2d', 'Linear']: + previous_name = name + previous_module = module + if not flag_found and name == op_name: + _logger.debug("original module found: %s", name) + num_weights = module.weight.data.numel() + + # consider related pruning in this op caused by previous op's pruning + if previous_module: + sparsity_previous_op = self._get_op_sparsity(previous_name) + if sparsity_previous_op: + _logger.debug( + "decrease op's weights by %s due to previous op %s's pruning...", sparsity_previous_op, previous_name) + num_weights *= (1-sparsity_previous_op) + + flag_found = True + continue + if flag_found and type(module).__name__ in ['Conv2d', 'Linear']: + _logger.debug("related module found: %s", name) + # channel/filter pruning crossing is considered here, so only the num_weights after channel pruning is valuable + num_weights += self._get_op_num_weights_remained(name, module) + break + + _logger.debug("num related weights of op %s : %d", op_name, num_weights) + + return num_weights + + def compress(self): + """ + Compress the model. + + Returns + ------- + torch.nn.Module + model with specified modules compressed. + """ + _logger.info('Starting NetAdapt Compression...') + + pruning_iteration = 0 + current_sparsity = 0 + delta_num_weights_per_iteration = \ + int(get_total_num_weights(self._model_to_prune, ['Conv2d', 'Linear']) * self._sparsity_per_iteration) + + # stop condition + while current_sparsity < self._sparsity: + _logger.info('Pruning iteration: %d', pruning_iteration) + + # calculate target sparsity of this iteration + target_sparsity = current_sparsity + self._sparsity_per_iteration + + # variable to store the info of the best layer found in this iteration + best_op = {} + + for wrapper in self.get_modules_wrapper(): + _logger.debug("op name : %s", wrapper.name) + _logger.debug("op weights : %d", wrapper.weight_mask.numel()) + _logger.debug("op left weights : %d", wrapper.weight_mask.sum().item()) + + current_op_sparsity = 1 - wrapper.weight_mask.sum().item() / wrapper.weight_mask.numel() + _logger.debug("current op sparsity : %s", current_op_sparsity) + + # sparsity that this layer needs to prune to satisfy the requirement + target_op_sparsity = current_op_sparsity + delta_num_weights_per_iteration / self._calc_num_related_weights(wrapper.name) + + if target_op_sparsity >= 1: + _logger.info('Layer %s has no enough weights (remained) to prune', wrapper.name) + continue + + config_list = self._update_config_list(self._config_list_generated, wrapper.name, target_op_sparsity) + _logger.debug("config_list used : %s", config_list) + + pruner = PRUNER_DICT[self._base_algo](copy.deepcopy(self._model_to_prune), config_list) + model_masked = pruner.compress() + + # Short-term fine tune the pruned model + self._short_term_fine_tuner(model_masked) + + performance = self._evaluator(model_masked) + _logger.info("Layer : %s, evaluation result after short-term fine tuning : %s", wrapper.name, performance) + + if not best_op \ + or (self._optimize_mode is OptimizeMode.Maximize and performance > best_op['performance']) \ + or (self._optimize_mode is OptimizeMode.Minimize and performance < best_op['performance']): + _logger.debug("updating best layer to %s...", wrapper.name) + # find weight mask of this layer + for w in pruner.get_modules_wrapper(): + if w.name == wrapper.name: + masks = {'weight_mask': w.weight_mask, + 'bias_mask': w.bias_mask} + break + best_op = { + 'op_name': wrapper.name, + 'sparsity': target_op_sparsity, + 'performance': performance, + 'masks': masks + } + + # save model weights + pruner.export_model(self._tmp_model_path) + + if not best_op: + # decrease pruning step + self._sparsity_per_iteration *= 0.5 + _logger.info("No more layers to prune, decrease pruning step to %s", self._sparsity_per_iteration) + continue + + # Pick the best layer to prune, update iterative information + # update config_list + self._config_list_generated = self._update_config_list( + self._config_list_generated, best_op['op_name'], best_op['sparsity']) + + # update weights parameters + self._model_to_prune.load_state_dict(torch.load(self._tmp_model_path)) + + # update mask of the chosen op + for wrapper in self.get_modules_wrapper(): + if wrapper.name == best_op['op_name']: + for k in best_op['masks']: + setattr(wrapper, k, best_op['masks'][k]) + break + + current_sparsity = target_sparsity + _logger.info('Pruning iteration %d finished, current sparsity: %s', pruning_iteration, current_sparsity) + _logger.info('Layer %s seleted with sparsity %s, performance after pruning & short term fine-tuning : %s', + best_op['op_name'], best_op['sparsity'], best_op['performance']) + pruning_iteration += 1 + + self._final_performance = best_op['performance'] + + # load weights parameters + self.load_model_state_dict(torch.load(self._tmp_model_path)) + os.remove(self._tmp_model_path) + + _logger.info('----------Compression finished--------------') + _logger.info('config_list generated: %s', self._config_list_generated) + _logger.info("Performance after pruning: %s", self._final_performance) + _logger.info("Masked sparsity: %.6f", current_sparsity) + + # save best config found and best performance + with open(os.path.join(self._experiment_data_dir, 'search_result.json'), 'w') as jsonfile: + json.dump({ + 'performance': self._final_performance, + 'config_list': json.dumps(self._config_list_generated) + }, jsonfile) + + _logger.info('search history and result saved to foler : %s', self._experiment_data_dir) + + return self.bound_model diff --git a/utils/third_party/nni_new/algorithms/compression/pytorch/pruning/one_shot_pruner.py b/utils/third_party/nni_new/algorithms/compression/pytorch/pruning/one_shot_pruner.py new file mode 100644 index 0000000000000000000000000000000000000000..3deec003fe4dcd759d7c8a90e385175b29385a01 --- /dev/null +++ b/utils/third_party/nni_new/algorithms/compression/pytorch/pruning/one_shot_pruner.py @@ -0,0 +1,169 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +from schema import And, Optional + +from .....compression.pytorch.utils.config_validation import CompressorSchema +from .dependency_aware_pruner import DependencyAwarePruner + +__all__ = ['LevelPruner', 'L1FilterPruner', 'L2FilterPruner', 'FPGMPruner'] + +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) + + +class OneshotPruner(DependencyAwarePruner): + """ + Prune model to an exact pruning level for one time. + """ + + def __init__(self, model, config_list, pruning_algorithm='level', dependency_aware=False, dummy_input=None, + **algo_kwargs): + """ + Parameters + ---------- + model : torch.nn.Module + Model to be pruned + config_list : list + List on pruning configs + pruning_algorithm: str + algorithms being used to prune model + dependency_aware: bool + If prune the model in a dependency-aware way. + dummy_input : torch.Tensor + The dummy input to analyze the topology constraints. Note that, + the dummy_input should on the same device with the model. + algo_kwargs: dict + Additional parameters passed to pruning algorithm masker class + """ + super().__init__(model, config_list, None, pruning_algorithm, dependency_aware, dummy_input, **algo_kwargs) + + def validate_config(self, model, config_list): + """ + Parameters + ---------- + model : torch.nn.Module + Model to be pruned + config_list : list + List on pruning configs + """ + schema = CompressorSchema([{ + 'sparsity': And(float, lambda n: 0 < n < 1), + Optional('op_types'): [str], + Optional('op_names'): [str] + }], model, logger) + + schema.validate(config_list) + + +class LevelPruner(OneshotPruner): + """ + Parameters + ---------- + model : torch.nn.Module + Model to be pruned + config_list : list + Supported keys: + - sparsity : This is to specify the sparsity operations to be compressed to. + - op_types : Operation types to prune. + """ + + def __init__(self, model, config_list): + super().__init__(model, config_list, pruning_algorithm='level') + + def _supported_dependency_aware(self): + return False + + +class L1FilterPruner(OneshotPruner): + """ + Parameters + ---------- + model : torch.nn.Module + Model to be pruned + config_list : list + Supported keys: + - sparsity : This is to specify the sparsity operations to be compressed to. + - op_types : Only Conv2d is supported in L1FilterPruner. + dependency_aware: bool + If prune the model in a dependency-aware way. If it is `True`, this pruner will + prune the model according to the l2-norm of weights and the channel-dependency or + group-dependency of the model. In this way, the pruner will force the conv layers + that have dependencies to prune the same channels, so the speedup module can better + harvest the speed benefit from the pruned model. Note that, if this flag is set True + , the dummy_input cannot be None, because the pruner needs a dummy input to trace the + dependency between the conv layers. + dummy_input : torch.Tensor + The dummy input to analyze the topology constraints. Note that, the dummy_input + should on the same device with the model. + """ + + def __init__(self, model, config_list, dependency_aware=False, dummy_input=None): + super().__init__(model, config_list, pruning_algorithm='l1', dependency_aware=dependency_aware, + dummy_input=dummy_input) + + def _supported_dependency_aware(self): + return True + + +class L2FilterPruner(OneshotPruner): + """ + Parameters + ---------- + model : torch.nn.Module + Model to be pruned + config_list : list + Supported keys: + - sparsity : This is to specify the sparsity operations to be compressed to. + - op_types : Only Conv2d is supported in L2FilterPruner. + dependency_aware: bool + If prune the model in a dependency-aware way. If it is `True`, this pruner will + prune the model according to the l2-norm of weights and the channel-dependency or + group-dependency of the model. In this way, the pruner will force the conv layers + that have dependencies to prune the same channels, so the speedup module can better + harvest the speed benefit from the pruned model. Note that, if this flag is set True + , the dummy_input cannot be None, because the pruner needs a dummy input to trace the + dependency between the conv layers. + dummy_input : torch.Tensor + The dummy input to analyze the topology constraints. Note that, the dummy_input + should on the same device with the model. + """ + + def __init__(self, model, config_list, dependency_aware=False, dummy_input=None): + super().__init__(model, config_list, pruning_algorithm='l2', dependency_aware=dependency_aware, + dummy_input=dummy_input) + + def _supported_dependency_aware(self): + return True + + +class FPGMPruner(OneshotPruner): + """ + Parameters + ---------- + model : torch.nn.Module + Model to be pruned + config_list : list + Supported keys: + - sparsity : This is to specify the sparsity operations to be compressed to. + - op_types : Only Conv2d is supported in FPGM Pruner. + dependency_aware: bool + If prune the model in a dependency-aware way. If it is `True`, this pruner will + prune the model according to the l2-norm of weights and the channel-dependency or + group-dependency of the model. In this way, the pruner will force the conv layers + that have dependencies to prune the same channels, so the speedup module can better + harvest the speed benefit from the pruned model. Note that, if this flag is set True + , the dummy_input cannot be None, because the pruner needs a dummy input to trace the + dependency between the conv layers. + dummy_input : torch.Tensor + The dummy input to analyze the topology constraints. Note that, the dummy_input + should on the same device with the model. + """ + + def __init__(self, model, config_list, dependency_aware=False, dummy_input=None): + super().__init__(model, config_list, pruning_algorithm='fpgm', dependency_aware=dependency_aware, + dummy_input=dummy_input) + + def _supported_dependency_aware(self): + return True diff --git a/utils/third_party/nni_new/algorithms/compression/pytorch/pruning/sensitivity_pruner.py b/utils/third_party/nni_new/algorithms/compression/pytorch/pruning/sensitivity_pruner.py new file mode 100644 index 0000000000000000000000000000000000000000..ed4d791abd95e8df45ba92fe579e92c9eb163c60 --- /dev/null +++ b/utils/third_party/nni_new/algorithms/compression/pytorch/pruning/sensitivity_pruner.py @@ -0,0 +1,413 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. +import os +import csv +import copy +import json +import logging +import torch + +from schema import And, Optional +from nni.compression.pytorch.compressor import Pruner +from nni.compression.pytorch.utils.config_validation import CompressorSchema +from .constants_pruner import PRUNER_DICT +from nni.compression.pytorch.utils.sensitivity_analysis import SensitivityAnalysis + + +MAX_PRUNE_RATIO_PER_ITER = 0.95 + +_logger = logging.getLogger('Sensitivity_Pruner') +_logger.setLevel(logging.INFO) + +class SensitivityPruner(Pruner): + """ + This function prune the model based on the sensitivity + for each layer. + + Parameters + ---------- + model: torch.nn.Module + model to be compressed + evaluator: function + validation function for the model. This function should return the accuracy + of the validation dataset. The input parameters of evaluator can be specified + in the parameter `eval_args` and 'eval_kwargs' of the compress function if needed. + Example: + >>> def evaluator(model): + >>> device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + >>> val_loader = ... + >>> model.eval() + >>> correct = 0 + >>> with torch.no_grad(): + >>> for data, target in val_loader: + >>> data, target = data.to(device), target.to(device) + >>> output = model(data) + >>> # get the index of the max log-probability + >>> pred = output.argmax(dim=1, keepdim=True) + >>> correct += pred.eq(target.view_as(pred)).sum().item() + >>> accuracy = correct / len(val_loader.dataset) + >>> return accuracy + finetuner: function + finetune function for the model. This parameter is not essential, if is not None, + the sensitivity pruner will finetune the model after pruning in each iteration. + The input parameters of finetuner can be specified in the parameter of compress + called `finetune_args` and `finetune_kwargs` if needed. + Example: + >>> def finetuner(model, epoch=3): + >>> device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + >>> train_loader = ... + >>> criterion = torch.nn.CrossEntropyLoss() + >>> optimizer = torch.optim.SGD(model.parameters(), lr=0.01) + >>> model.train() + >>> for _ in range(epoch): + >>> for _, (data, target) in enumerate(train_loader): + >>> data, target = data.to(device), target.to(device) + >>> optimizer.zero_grad() + >>> output = model(data) + >>> loss = criterion(output, target) + >>> loss.backward() + >>> optimizer.step() + base_algo: str + base pruning algorithm. `level`, `l1`, `l2` or `fpgm`, by default `l1`. + sparsity_proportion_calc: function + This function generate the sparsity proportion between the conv layers according to the + sensitivity analysis results. We provide a default function to quantify the sparsity + proportion according to the sensitivity analysis results. Users can also customize + this function according to their needs. The input of this function is a dict, + for example : {'conv1' : {0.1: 0.9, 0.2 : 0.8}, 'conv2' : {0.1: 0.9, 0.2 : 0.8}}, + in which, 'conv1' and is the name of the conv layer, and 0.1:0.9 means when the + sparsity of conv1 is 0.1 (10%), the model's val accuracy equals to 0.9. + sparsity_per_iter: float + The sparsity of the model that the pruner try to prune in each iteration. + acc_drop_threshold : float + The hyperparameter used to quantifiy the sensitivity for each layer. + checkpoint_dir: str + The dir path to save the checkpoints during the pruning. + """ + + def __init__(self, model, config_list, evaluator, + finetuner=None, base_algo='l1', sparsity_proportion_calc=None, + sparsity_per_iter=0.1, acc_drop_threshold=0.05, checkpoint_dir=None): + + self.base_algo = base_algo + self.model = model + super(SensitivityPruner, self).__init__(model, config_list) + # unwrap the model + self._unwrap_model() + _logger.debug(str(self.model)) + self.evaluator = evaluator + self.finetuner = finetuner + self.analyzer = SensitivityAnalysis( + self.model, self.evaluator, prune_type=base_algo, \ + early_stop_mode='dropped', early_stop_value=acc_drop_threshold) + # Get the original accuracy of the pretrained model + self.ori_acc = None + # Copy the original weights before pruning + self.ori_state_dict = copy.deepcopy(self.model.state_dict()) + self.sensitivities = {} + # Save the weight count for each layer + self.weight_count = {} + self.weight_sum = 0 + # Map the layer name to the layer module + self.named_module = {} + + self.Pruner = PRUNER_DICT[self.base_algo] + # Count the total weight count of the model + for name, submodule in self.model.named_modules(): + self.named_module[name] = submodule + if name in self.analyzer.target_layer: + # Currently, only count the weights in the conv layers + # else the fully connected layer (which contains + # the most weights) may make the pruner prune the + # model too hard + # if hasattr(submodule, 'weight'): # Count all the weights of the model + self.weight_count[name] = submodule.weight.data.numel() + self.weight_sum += self.weight_count[name] + # function to generate the sparsity proportion between the conv layers + if sparsity_proportion_calc is None: + self.sparsity_proportion_calc = self._max_prune_ratio + else: + self.sparsity_proportion_calc = sparsity_proportion_calc + # The ratio of remained weights is 1.0 at the begining + self.remained_ratio = 1.0 + self.sparsity_per_iter = sparsity_per_iter + self.acc_drop_threshold = acc_drop_threshold + self.checkpoint_dir = checkpoint_dir + + def validate_config(self, model, config_list): + """ + Parameters + ---------- + model : torch.nn.module + Model to be pruned + config_list : list + List on pruning configs + """ + + if self.base_algo == 'level': + schema = CompressorSchema([{ + 'sparsity': And(float, lambda n: 0 < n < 1), + Optional('op_types'): [str], + Optional('op_names'): [str], + }], model, _logger) + elif self.base_algo in ['l1', 'l2', 'fpgm']: + schema = CompressorSchema([{ + 'sparsity': And(float, lambda n: 0 < n < 1), + 'op_types': ['Conv2d'], + Optional('op_names'): [str] + }], model, _logger) + + schema.validate(config_list) + + def load_sensitivity(self, filepath): + """ + load the sensitivity results exported by the sensitivity analyzer + """ + assert os.path.exists(filepath) + with open(filepath, 'r') as csvf: + csv_r = csv.reader(csvf) + header = next(csv_r) + sparsities = [float(x) for x in header[1:]] + sensitivities = {} + for row in csv_r: + layername = row[0] + accuracies = [float(x) for x in row[1:]] + sensitivities[layername] = {} + for i, accuracy in enumerate(accuracies): + sensitivities[layername][sparsities[i]] = accuracy + return sensitivities + + def _max_prune_ratio(self, ori_acc, threshold, sensitivities): + """ + Find the maximum prune ratio for a single layer whose accuracy + drop is lower than the threshold. + + Parameters + ---------- + ori_acc: float + Original accuracy + threshold: float + Accuracy drop threshold + sensitivities: dict + The dict object that stores the sensitivity results for each layer. + For example: {'conv1' : {0.1: 0.9, 0.2 : 0.8}} + Returns + ------- + max_ratios: dict + return the maximum prune ratio for each layer. For example: + {'conv1':0.1, 'conv2':0.2} + """ + max_ratio = {} + for layer in sensitivities: + prune_ratios = sorted(sensitivities[layer].keys()) + last_ratio = 0 + for ratio in prune_ratios: + last_ratio = ratio + cur_acc = sensitivities[layer][ratio] + if cur_acc + threshold < ori_acc: + break + max_ratio[layer] = last_ratio + return max_ratio + + def normalize(self, ratios, target_pruned): + """ + Normalize the prune ratio of each layer according to the + total already pruned ratio and the final target total pruning + ratio + + Parameters + ---------- + ratios: + Dict object that save the prune ratio for each layer + target_pruned: + The amount of the weights expected to be pruned in this + iteration + + Returns + ------- + new_ratios: + return the normalized prune ratios for each layer. + + """ + w_sum = 0 + _Max = 0 + for layername, ratio in ratios.items(): + wcount = self.weight_count[layername] + w_sum += ratio * wcount * \ + (1-self.analyzer.already_pruned[layername]) + target_count = self.weight_sum * target_pruned + for layername in ratios: + ratios[layername] = ratios[layername] * target_count / w_sum + _Max = max(_Max, ratios[layername]) + # Cannot Prune too much in a single iteration + # If a layer's prune ratio is larger than the + # MAX_PRUNE_RATIO_PER_ITER we rescal all prune + # ratios under this threshold + if _Max > MAX_PRUNE_RATIO_PER_ITER: + + for layername in ratios: + ratios[layername] = ratios[layername] * \ + MAX_PRUNE_RATIO_PER_ITER / _Max + return ratios + + def create_cfg(self, ratios): + """ + Generate the cfg_list for the pruner according to the prune ratios. + + Parameters + --------- + ratios: + For example: {'conv1' : 0.2} + + Returns + ------- + cfg_list: + For example: [{'sparsity':0.2, 'op_names':['conv1'], 'op_types':['Conv2d']}] + """ + cfg_list = [] + for layername in ratios: + prune_ratio = ratios[layername] + remain = 1 - self.analyzer.already_pruned[layername] + sparsity = remain * prune_ratio + \ + self.analyzer.already_pruned[layername] + if sparsity > 0: + # Pruner does not allow the prune ratio to be zero + cfg = {'sparsity': sparsity, 'op_names': [ + layername], 'op_types': ['Conv2d']} + cfg_list.append(cfg) + return cfg_list + + def current_sparsity(self): + """ + The sparsity of the weight. + """ + pruned_weight = 0 + for layer_name in self.analyzer.already_pruned: + w_count = self.weight_count[layer_name] + prune_ratio = self.analyzer.already_pruned[layer_name] + pruned_weight += w_count * prune_ratio + return pruned_weight / self.weight_sum + + def compress(self, eval_args=None, eval_kwargs=None, + finetune_args=None, finetune_kwargs=None, resume_sensitivity=None): + """ + This function iteratively prune the model according to the results of + the sensitivity analysis. + + Parameters + ---------- + eval_args: list + eval_kwargs: list& dict + Parameters for the val_funtion, the val_function will be called like + evaluator(\*eval_args, \*\*eval_kwargs) + finetune_args: list + finetune_kwargs: dict + Parameters for the finetuner function if needed. + resume_sensitivity: + resume the sensitivity results from this file. + """ + # pylint suggest not use the empty list and dict + # as the default input parameter + if not eval_args: + eval_args = [] + if not eval_kwargs: + eval_kwargs = {} + if not finetune_args: + finetune_args = [] + if not finetune_kwargs: + finetune_kwargs = {} + if self.ori_acc is None: + self.ori_acc = self.evaluator(*eval_args, **eval_kwargs) + assert isinstance(self.ori_acc, float) or isinstance(self.ori_acc, int) + if not resume_sensitivity: + self.sensitivities = self.analyzer.analysis( + val_args=eval_args, val_kwargs=eval_kwargs) + else: + self.sensitivities = self.load_sensitivity(resume_sensitivity) + self.analyzer.sensitivities = self.sensitivities + # the final target sparsity of the model + target_ratio = 1 - self.config_list[0]['sparsity'] + cur_ratio = self.remained_ratio + ori_acc = self.ori_acc + iteration_count = 0 + if self.checkpoint_dir is not None: + os.makedirs(self.checkpoint_dir, exist_ok=True) + modules_wrapper_final = None + while cur_ratio > target_ratio: + iteration_count += 1 + # Each round have three steps: + # 1) Get the current sensitivity for each layer(the sensitivity + # of each layer may change during the pruning) + # 2) Prune each layer according the sensitivies + # 3) finetune the model + _logger.info('Current base accuracy %f', ori_acc) + _logger.info('Remained %f weights', cur_ratio) + # determine the sparsity proportion between different + # layers according to the sensitivity result + proportion = self.sparsity_proportion_calc( + ori_acc, self.acc_drop_threshold, self.sensitivities) + + new_pruneratio = self.normalize(proportion, self.sparsity_per_iter) + cfg_list = self.create_cfg(new_pruneratio) + if not cfg_list: + _logger.error('The threshold is too small, please set a larger threshold') + return self.model + _logger.debug('Pruner Config: %s', str(cfg_list)) + cfg_str = ['%s:%.3f'%(cfg['op_names'][0], cfg['sparsity']) for cfg in cfg_list] + _logger.info('Current Sparsities: %s', ','.join(cfg_str)) + + pruner = self.Pruner(self.model, cfg_list) + pruner.compress() + pruned_acc = self.evaluator(*eval_args, **eval_kwargs) + _logger.info('Accuracy after pruning: %f', pruned_acc) + finetune_acc = pruned_acc + if self.finetuner is not None: + # if the finetune function is None, then skip the finetune + self.finetuner(*finetune_args, **finetune_kwargs) + finetune_acc = self.evaluator(*eval_args, **eval_kwargs) + _logger.info('Accuracy after finetune: %f', finetune_acc) + ori_acc = finetune_acc + # unwrap the pruner + pruner._unwrap_model() + # update the already prune ratio of each layer befor the new + # sensitivity analysis + for layer_cfg in cfg_list: + name = layer_cfg['op_names'][0] + sparsity = layer_cfg['sparsity'] + self.analyzer.already_pruned[name] = sparsity + # update the cur_ratio + cur_ratio = 1 - self.current_sparsity() + modules_wrapper_final = pruner.get_modules_wrapper() + del pruner + _logger.info('Currently remained weights: %f', cur_ratio) + + if self.checkpoint_dir is not None: + checkpoint_name = 'Iter_%d_finetune_acc_%.5f_sparsity_%.4f' % ( + iteration_count, finetune_acc, cur_ratio) + checkpoint_path = os.path.join( + self.checkpoint_dir, '%s.pth' % checkpoint_name) + cfg_path = os.path.join( + self.checkpoint_dir, '%s_pruner.json' % checkpoint_name) + sensitivity_path = os.path.join( + self.checkpoint_dir, '%s_sensitivity.csv' % checkpoint_name) + torch.save(self.model.state_dict(), checkpoint_path) + with open(cfg_path, 'w') as jf: + json.dump(cfg_list, jf) + self.analyzer.export(sensitivity_path) + + if cur_ratio > target_ratio: + # If this is the last prune iteration, skip the time-consuming + # sensitivity analysis + + self.analyzer.load_state_dict(self.model.state_dict()) + self.sensitivities = self.analyzer.analysis( + val_args=eval_args, val_kwargs=eval_kwargs) + + _logger.info('After Pruning: %.2f weights remains', cur_ratio) + self.modules_wrapper = modules_wrapper_final + + self._wrap_model() + return self.model + + def calc_mask(self, wrapper, **kwargs): + return None diff --git a/utils/third_party/nni_new/algorithms/compression/pytorch/pruning/simulated_annealing_pruner.py b/utils/third_party/nni_new/algorithms/compression/pytorch/pruning/simulated_annealing_pruner.py new file mode 100644 index 0000000000000000000000000000000000000000..b371b2c6fbb29f765fd6c6aa728d3dc98460d0c3 --- /dev/null +++ b/utils/third_party/nni_new/algorithms/compression/pytorch/pruning/simulated_annealing_pruner.py @@ -0,0 +1,357 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +import os +import math +import copy +import csv +import json +import numpy as np +from schema import And, Optional + +from nni.utils import OptimizeMode + +from nni.compression.pytorch.compressor import Pruner +from nni.compression.pytorch.utils.config_validation import CompressorSchema +from .constants_pruner import PRUNER_DICT + + +_logger = logging.getLogger(__name__) + + +class SimulatedAnnealingPruner(Pruner): + """ + A Pytorch implementation of Simulated Annealing compression algorithm. + + Parameters + ---------- + model : pytorch model + The model to be pruned. + config_list : list + Supported keys: + - sparsity : The target overall sparsity. + - op_types : The operation type to prune. + evaluator : function + Function to evaluate the pruned model. + This function should include `model` as the only parameter, and returns a scalar value. + Example:: + + def evaluator(model): + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + val_loader = ... + model.eval() + correct = 0 + with torch.no_grad(): + for data, target in val_loader: + data, target = data.to(device), target.to(device) + output = model(data) + # get the index of the max log-probability + pred = output.argmax(dim=1, keepdim=True) + correct += pred.eq(target.view_as(pred)).sum().item() + accuracy = correct / len(val_loader.dataset) + return accuracy + optimize_mode : str + Optimize mode, `maximize` or `minimize`, by default `maximize`. + base_algo : str + Base pruning algorithm. `level`, `l1`, `l2` or `fpgm`, by default `l1`. Given the sparsity distribution among the ops, + the assigned `base_algo` is used to decide which filters/channels/weights to prune. + start_temperature : float + Start temperature of the simulated annealing process. + stop_temperature : float + Stop temperature of the simulated annealing process. + cool_down_rate : float + Cool down rate of the temperature. + perturbation_magnitude : float + Initial perturbation magnitude to the sparsities. The magnitude decreases with current temperature. + experiment_data_dir : string + PATH to save experiment data, + including the config_list generated for the base pruning algorithm, the performance of the pruned model and the pruning history. + + """ + + def __init__(self, model, config_list, evaluator, optimize_mode='maximize', base_algo='l1', + start_temperature=100, stop_temperature=20, cool_down_rate=0.9, perturbation_magnitude=0.35, experiment_data_dir='./'): + # original model + self._model_to_prune = copy.deepcopy(model) + self._base_algo = base_algo + + super().__init__(model, config_list) + + self._evaluator = evaluator + self._optimize_mode = OptimizeMode(optimize_mode) + + # hyper parameters for SA algorithm + self._start_temperature = start_temperature + self._current_temperature = start_temperature + self._stop_temperature = stop_temperature + self._cool_down_rate = cool_down_rate + self._perturbation_magnitude = perturbation_magnitude + + # overall pruning rate + self._sparsity = config_list[0]['sparsity'] + # pruning rates of the layers + self._sparsities = None + + # init current performance & best performance + self._current_performance = -np.inf + self._best_performance = -np.inf + self._best_config_list = [] + + self._search_history = [] + + self._experiment_data_dir = experiment_data_dir + if not os.path.exists(self._experiment_data_dir): + os.makedirs(self._experiment_data_dir) + + def validate_config(self, model, config_list): + """ + Parameters + ---------- + model : torch.nn.Module + Model to be pruned + config_list : list + List on pruning configs + """ + + if self._base_algo == 'level': + schema = CompressorSchema([{ + 'sparsity': And(float, lambda n: 0 < n < 1), + Optional('op_types'): [str], + Optional('op_names'): [str], + }], model, _logger) + elif self._base_algo in ['l1', 'l2', 'fpgm']: + schema = CompressorSchema([{ + 'sparsity': And(float, lambda n: 0 < n < 1), + 'op_types': ['Conv2d'], + Optional('op_names'): [str] + }], model, _logger) + + schema.validate(config_list) + + def _sparsities_2_config_list(self, sparsities): + ''' + convert sparsities vector into config_list for LevelPruner or L1FilterPruner + + Parameters + ---------- + sparsities : list + list of sparsities + + Returns + ------- + list of dict + config_list for LevelPruner or L1FilterPruner + ''' + config_list = [] + + sparsities = sorted(sparsities) + self.modules_wrapper = sorted( + self.modules_wrapper, key=lambda wrapper: wrapper.module.weight.data.numel()) + + # a layer with more weights will have no less pruning rate + for idx, wrapper in enumerate(self.get_modules_wrapper()): + # L1Filter Pruner requires to specify op_types + if self._base_algo in ['l1', 'l2', 'fpgm']: + config_list.append( + {'sparsity': sparsities[idx], 'op_types': ['Conv2d'], 'op_names': [wrapper.name]}) + elif self._base_algo == 'level': + config_list.append( + {'sparsity': sparsities[idx], 'op_names': [wrapper.name]}) + + config_list = [val for val in config_list if not math.isclose(val['sparsity'], 0, abs_tol=1e-6)] + + return config_list + + def _rescale_sparsities(self, sparsities, target_sparsity): + ''' + Rescale the sparsities list to satisfy the target overall sparsity + + Parameters + ---------- + sparsities : list + + target_sparsity : float + the target overall sparsity + + Returns + ------- + list + the rescaled sparsities + ''' + num_weights = [] + for wrapper in self.get_modules_wrapper(): + num_weights.append(wrapper.module.weight.data.numel()) + + num_weights = sorted(num_weights) + sparsities = sorted(sparsities) + + total_weights = 0 + total_weights_pruned = 0 + + # calculate the scale + for idx, num_weight in enumerate(num_weights): + total_weights += num_weight + total_weights_pruned += int(num_weight*sparsities[idx]) + if total_weights_pruned == 0: + return None + scale = target_sparsity / (total_weights_pruned/total_weights) + + # rescale the sparsities + sparsities = np.asarray(sparsities)*scale + + return sparsities + + def _init_sparsities(self): + ''' + Generate a sorted sparsities vector + ''' + # repeatedly generate a distribution until satisfies the overall sparsity requirement + _logger.info('Gererating sparsities...') + while True: + sparsities = sorted(np.random.uniform( + 0, 1, len(self.get_modules_wrapper()))) + + sparsities = self._rescale_sparsities( + sparsities, target_sparsity=self._sparsity) + + if sparsities is not None and sparsities[0] >= 0 and sparsities[-1] < 1: + _logger.info('Initial sparsities generated : %s', sparsities) + self._sparsities = sparsities + break + + def _generate_perturbations(self): + ''' + Generate perturbation to the current sparsities distribution. + + Returns: + -------- + list + perturbated sparsities + ''' + _logger.info("Gererating perturbations to the current sparsities...") + + # decrease magnitude with current temperature + magnitude = self._current_temperature / \ + self._start_temperature * self._perturbation_magnitude + _logger.info('current perturation magnitude:%s', magnitude) + + while True: + perturbation = np.random.uniform(-magnitude, magnitude, len(self.get_modules_wrapper())) + sparsities = np.clip(0, self._sparsities + perturbation, None) + _logger.debug("sparsities before rescalling:%s", sparsities) + + sparsities = self._rescale_sparsities(sparsities, target_sparsity=self._sparsity) + _logger.debug("sparsities after rescalling:%s", sparsities) + + if sparsities is not None and sparsities[0] >= 0 and sparsities[-1] < 1: + _logger.info("Sparsities perturbated:%s", sparsities) + return sparsities + + def calc_mask(self, wrapper, **kwargs): + return None + + def compress(self, return_config_list=False): + """ + Compress the model with Simulated Annealing. + + Returns + ------- + torch.nn.Module + model with specified modules compressed. + """ + _logger.info('Starting Simulated Annealing Compression...') + + # initiaze a randomized action + pruning_iteration = 0 + self._init_sparsities() + + # stop condition + self._current_temperature = self._start_temperature + while self._current_temperature > self._stop_temperature: + _logger.info('Pruning iteration: %d', pruning_iteration) + _logger.info('Current temperature: %d, Stop temperature: %d', + self._current_temperature, self._stop_temperature) + while True: + # generate perturbation + sparsities_perturbated = self._generate_perturbations() + config_list = self._sparsities_2_config_list( + sparsities_perturbated) + _logger.info( + "config_list for Pruner generated: %s", config_list) + + # fast evaluation + pruner = PRUNER_DICT[self._base_algo](copy.deepcopy(self._model_to_prune), config_list) + model_masked = pruner.compress() + evaluation_result = self._evaluator(model_masked) + + self._search_history.append( + {'sparsity': self._sparsity, 'performance': evaluation_result, 'config_list': config_list}) + + if self._optimize_mode is OptimizeMode.Minimize: + evaluation_result *= -1 + + # if better evaluation result, then accept the perturbation + if evaluation_result > self._current_performance: + self._current_performance = evaluation_result + self._sparsities = sparsities_perturbated + + # save best performance and best params + if evaluation_result > self._best_performance: + _logger.info('updating best model...') + self._best_performance = evaluation_result + self._best_config_list = config_list + + # save the overall best masked model + self.bound_model = model_masked + # the ops with sparsity 0 are not included in this modules_wrapper + modules_wrapper_final = pruner.get_modules_wrapper() + break + # if not, accept with probability e^(-deltaE/current_temperature) + else: + delta_E = np.abs(evaluation_result - + self._current_performance) + probability = math.exp(-1 * delta_E / + self._current_temperature) + if np.random.uniform(0, 1) < probability: + self._current_performance = evaluation_result + self._sparsities = sparsities_perturbated + break + + # cool down + self._current_temperature *= self._cool_down_rate + pruning_iteration += 1 + + _logger.info('----------Compression finished--------------') + _logger.info('Best performance: %s', self._best_performance) + _logger.info('config_list found : %s', + self._best_config_list) + + # save search history + with open(os.path.join(self._experiment_data_dir, 'search_history.csv'), 'w') as csvfile: + writer = csv.DictWriter(csvfile, fieldnames=['sparsity', 'performance', 'config_list']) + writer.writeheader() + for item in self._search_history: + writer.writerow({'sparsity': item['sparsity'], 'performance': item['performance'], 'config_list': json.dumps( + item['config_list'])}) + + # save best config found and best performance + if self._optimize_mode is OptimizeMode.Minimize: + self._best_performance *= -1 + with open(os.path.join(self._experiment_data_dir, 'search_result.json'), 'w+') as jsonfile: + json.dump({ + 'performance': self._best_performance, + 'config_list': json.dumps(self._best_config_list) + }, jsonfile) + + _logger.info('search history and result saved to foler : %s', + self._experiment_data_dir) + + if return_config_list: + return self._best_config_list + + # This should be done only at the final stage, + # because the modules_wrapper with all the ops are used during the annealing process + self.modules_wrapper = modules_wrapper_final + + return self.bound_model diff --git a/utils/third_party/nni_new/algorithms/compression/pytorch/pruning/structured_pruning_masker.py b/utils/third_party/nni_new/algorithms/compression/pytorch/pruning/structured_pruning_masker.py new file mode 100644 index 0000000000000000000000000000000000000000..96e61d355dda01dc5dee8ff505289f5d29c156fc --- /dev/null +++ b/utils/third_party/nni_new/algorithms/compression/pytorch/pruning/structured_pruning_masker.py @@ -0,0 +1,965 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +import math +import numpy as np +import torch +from .weight_masker import WeightMasker + +__all__ = ['L1FilterPrunerMasker', 'L2FilterPrunerMasker', 'FPGMPrunerMasker', + 'TaylorFOWeightFilterPrunerMasker', 'ActivationAPoZRankFilterPrunerMasker', + 'ActivationMeanRankFilterPrunerMasker', 'SlimPrunerMasker', 'AMCWeightMasker'] + +logger = logging.getLogger('torch filter pruners') + + +class StructuredWeightMasker(WeightMasker): + """ + A structured pruning masker base class that prunes convolutional layer filters. + + Parameters + ---------- + model: nn.Module + model to be pruned + pruner: Pruner + A Pruner instance used to prune the model + preserve_round: int + after pruning, preserve filters/channels round to `preserve_round`, for example: + for a Conv2d layer, output channel is 32, sparsity is 0.2, if preserve_round is + 1 (no preserve round), then there will be int(32 * 0.2) = 6 filters pruned, and + 32 - 6 = 26 filters are preserved. If preserve_round is 4, preserved filters will + be round up to 28 (which can be divided by 4) and only 4 filters are pruned. + + """ + + def __init__(self, model, pruner, preserve_round=1, dependency_aware=False): + self.model = model + self.pruner = pruner + self.preserve_round = preserve_round + self.dependency_aware = dependency_aware + + def calc_mask(self, sparsity, wrapper, wrapper_idx=None, **depen_kwargs): + """ + calculate the mask for `wrapper`. + + Parameters + ---------- + sparsity: float/list of float + The target sparsity of the wrapper. If we calculate the mask in + the normal way, then sparsity is a float number. In contrast, if + we calculate the mask in the dependency-aware way, sparsity is a + list of float numbers, each float number corressponds to a sparsity + of a layer. + wrapper: PrunerModuleWrapper/list of PrunerModuleWrappers + The wrapper of the target layer. If we calculate the mask in the normal + way, then `wrapper` is an instance of PrunerModuleWrapper, else `wrapper` + is a list of PrunerModuleWrapper. + wrapper_idx: int/list of int + The index of the wrapper. + depen_kwargs: dict + The kw_args for the dependency-aware mode. + """ + if not self.dependency_aware: + # calculate the mask in the normal way, each layer calculate its + # own mask separately + return self._normal_calc_mask(sparsity, wrapper, wrapper_idx) + else: + # if the dependency_aware switch is on, then calculate the mask + # in the dependency-aware way + return self._dependency_calc_mask(sparsity, wrapper, wrapper_idx, **depen_kwargs) + + def _get_current_state(self, sparsity, wrapper, wrapper_idx=None): + """ + Some pruner may prune the layers in a iterative way. In each pruning iteration, + we may get the current state of this wrapper/layer, and continue to prune this layer + based on the current state. This function is to get the current pruning state of the + target wrapper/layer. + Parameters + ---------- + sparsity: float + pruning ratio, preserved weight ratio is `1 - sparsity` + wrapper: PrunerModuleWrapper + layer wrapper of this layer + wrapper_idx: int + index of this wrapper in pruner's all wrappers + Returns + ------- + base_mask: dict + dict object that stores the mask of this wrapper in this iteration, if it is the + first iteration, then we create a new mask with all ones. If there is already a + mask in this wrapper, then we return the existing mask. + weight: tensor + the current weight of this layer + num_prune: int + how many filters we should prune + """ + msg = 'module type {} is not supported!'.format(wrapper.type) + # assert wrapper.type == 'Conv2d', msg + weight = wrapper.module.weight.data + bias = None + if hasattr(wrapper.module, 'bias') and wrapper.module.bias is not None: + bias = wrapper.module.bias.data + + if wrapper.weight_mask is None: + mask_weight = torch.ones(weight.size()).type_as(weight).detach() + else: + mask_weight = wrapper.weight_mask.clone() + if bias is not None: + if wrapper.bias_mask is None: + mask_bias = torch.ones(bias.size()).type_as(bias).detach() + else: + mask_bias = wrapper.bias_mask.clone() + else: + mask_bias = None + mask = {'weight_mask': mask_weight, 'bias_mask': mask_bias} + + # num_total = weight.size(0) + + if isinstance(wrapper.module, torch.nn.Conv2d): + num_total = weight.size(0) + elif isinstance(wrapper.module, torch.nn.ConvTranspose2d): + num_total = weight.size(1) + + num_prune = int(num_total * sparsity) + if self.preserve_round > 1: + num_preserve = num_total - num_prune + num_preserve = int( + math.ceil(num_preserve * 1. / self.preserve_round) * self.preserve_round) + if num_preserve > num_total: + num_preserve = int(math.floor( + num_total * 1. / self.preserve_round) * self.preserve_round) + num_prune = num_total - num_preserve + # weight*mask_weight: apply base mask for iterative pruning + return mask, weight * mask_weight, num_prune + + def _normal_calc_mask(self, sparsity, wrapper, wrapper_idx=None): + """ + Calculate the mask of given layer. + Parameters + ---------- + sparsity: float + pruning ratio, preserved weight ratio is `1 - sparsity` + wrapper: PrunerModuleWrapper + layer wrapper of this layer + wrapper_idx: int + index of this wrapper in pruner's all wrappers + Returns + ------- + dict + dictionary for storing masks, keys of the dict: + 'weight_mask': weight mask tensor + 'bias_mask': bias mask tensor (optional) + """ + mask, weight, num_prune = self._get_current_state( + sparsity, wrapper, wrapper_idx) + num_total = weight.size(0) + if num_total < 2 or num_prune < 1: + return mask + + return self.get_mask(mask, weight, num_prune, wrapper, wrapper_idx) + + def _common_channel_to_prune(self, sparsities, wrappers, wrappers_idx, channel_dsets, groups): + """ + Calculate the common channels should be pruned by all the layers in this group. + This function is for filter pruning of Conv layers. if want to support the dependency-aware + mode for others ops, you need to inherit this class and overwrite `_common_channel_to_prune`. + + Parameters + ---------- + sparsities : list + List of float that specify the sparsity for each conv layer. + wrappers : list + List of wrappers + groups : list + The number of the filter groups of each layer. + wrappers_idx : list + The indexes of the wrappers + """ + # sparsity configs for each wrapper + # sparsities = [_w.config['sparsity'] for _w in wrappers] + # check the type of the input wrappers + for _w in wrappers: + msg = 'module type {} is not supported!'.format(_w.type) + assert _w.type == 'Conv2d', msg + # Among the dependent layers, the layer with smallest + # sparsity determines the final benefit of the speedup + # module. To better harvest the speed benefit, we need + # to ensure that these dependent layers have at least + # `min_sparsity` pruned channel are the same. + if len(channel_dsets) == len(wrappers): + # all the layers in the dependency sets are pruned + min_sparsity = min(sparsities) + else: + # not all the layers in the dependency set + # are pruned + min_sparsity = 0 + # donnot prune the channels that we cannot harvest the speed from + sparsities = [min_sparsity] * len(sparsities) + # find the max number of the filter groups of the dependent + # layers. The group constraint of this dependency set is decided + # by the layer with the max groups. + + # should use the least common multiple for all the groups + # the max_group is lower than the channel_count, because + # the number of the filter is always divisible by the number of the group + max_group = np.lcm.reduce(groups) + channel_count = wrappers[0].module.weight.data.size(0) + device = wrappers[0].module.weight.device + channel_sum = torch.zeros(channel_count).to(device) + for _w, _w_idx in zip(wrappers, wrappers_idx): + # calculate the L1/L2 sum for all channels + c_sum = self.get_channel_sum(_w, _w_idx) + + if c_sum is None: + # if the channel sum cannot be calculated + # now, return None + return None + channel_sum += c_sum + + # prune the same `min_sparsity` channels based on channel_sum + # for all the layers in the channel sparsity + target_pruned = int(channel_count * min_sparsity) + # pruned_per_group may be zero, for example dw conv + pruned_per_group = int(target_pruned / max_group) + group_step = int(channel_count / max_group) + + channel_masks = [] + for gid in range(max_group): + _start = gid * group_step + _end = (gid + 1) * group_step + if pruned_per_group > 0: + threshold = torch.topk( + channel_sum[_start: _end], pruned_per_group, largest=False)[0].max() + group_mask = torch.gt(channel_sum[_start:_end], threshold) + else: + group_mask = torch.ones(group_step).to(device) + channel_masks.append(group_mask) + channel_masks = torch.cat(channel_masks, dim=0) + pruned_channel_index = ( + channel_masks == False).nonzero().squeeze(1).tolist() + logger.info('Prune the %s channels for all dependent', + ','.join([str(x) for x in pruned_channel_index])) + return channel_masks + + def _dependency_calc_mask(self, sparsities, wrappers, wrappers_idx, channel_dsets, groups): + """ + Calculate the masks for the layers in the same dependency sets. + Similar to the traditional original calc_mask, _dependency_calc_mask + will prune the target layers based on the L1/L2 norm of the weights. + However, StructuredWeightMasker prunes the filter completely based on the + L1/L2 norm of each filter. In contrast, _dependency_calc_mask + will try to satisfy the channel/group dependency(see nni.compression.torch. + utils.shape_dependency for details). Specifically, _dependency_calc_mask + will try to prune the same channels for the layers that have channel dependency. + In addition, this mask calculator will also ensure that the number of filters + pruned in each group is the same(meet the group dependency). + + Parameters + ---------- + sparsities : list + List of float that specify the sparsity for each conv layer. + wrappers : list + List of wrappers + groups : list + The number of the filter groups of each layer. + wrappers_idx : list + The indexes of the wrappers + """ + channel_masks = self._common_channel_to_prune( + sparsities, wrappers, wrappers_idx, channel_dsets, groups) + # calculate the mask for each layer based on channel_masks, first + # every layer will prune the same channels masked in channel_masks. + # If the sparsity of a layers is larger than min_sparsity, then it + # will continue prune sparsity - min_sparsity channels to meet the sparsity + # config. + masks = {} + for _pos, _w in enumerate(wrappers): + _w_idx = wrappers_idx[_pos] + sparsity = sparsities[_pos] + name = _w.name + + # _tmp_mask = self._normal_calc_mask( + # sparsity, _w, _w_idx, channel_masks) + base_mask, current_weight, num_prune = self._get_current_state( + sparsity, _w, _w_idx) + num_total = current_weight.size(0) + if num_total < 2 or num_prune < 1: + masks[name] = base_mask + continue + _tmp_mask = self.get_mask( + base_mask, current_weight, num_prune, _w, _w_idx, channel_masks) + + if _tmp_mask is None: + # if the mask calculation fails + return None + masks[name] = _tmp_mask + return masks + + def get_mask(self, base_mask, weight, num_prune, wrapper, wrapper_idx, channel_masks=None): + """ + Calculate the mask of given layer. + + Parameters + ---------- + base_mask: dict + The basic mask with the same shape of weight, all item in the basic mask is 1. + weight: tensor + the module weight to be pruned + num_prune: int + Num of filters to prune + wrapper: PrunerModuleWrapper + layer wrapper of this layer + wrapper_idx: int + index of this wrapper in pruner's all wrappers + channel_masks: Tensor + If mask some channels for this layer in advance. In the dependency-aware + mode, before calculating the masks for each layer, we will calculate a common + mask for all the layers in the dependency set. For the pruners that doesnot + support dependency-aware mode, they can just ignore this parameter. + + Returns + ------- + dict + dictionary for storing masks + """ + raise NotImplementedError( + '{} get_mask is not implemented'.format(self.__class__.__name__)) + + def get_channel_sum(self, wrapper, wrapper_idx): + """ + Calculate the importance weight for each channel. If want to support the + dependency-aware mode for this one-shot pruner, this function must be + implemented. + Parameters + ---------- + wrapper: PrunerModuleWrapper + layer wrapper of this layer + wrapper_idx: int + index of this wrapper in pruner's all wrappers + Returns + ------- + tensor + Tensor that indicates the importance of each channel + """ + raise NotImplementedError( + '{} get_channel_sum is not implemented'.format(self.__class__.__name__)) + + +class L1FilterPrunerMasker(StructuredWeightMasker): + """ + A structured pruning algorithm that prunes the filters of smallest magnitude + weights sum in the convolution layers to achieve a preset level of network sparsity. + Hao Li, Asim Kadav, Igor Durdanovic, Hanan Samet and Hans Peter Graf, + "PRUNING FILTERS FOR EFFICIENT CONVNETS", 2017 ICLR + https://arxiv.org/abs/1608.08710 + """ + + def get_mask(self, base_mask, weight, num_prune, wrapper, wrapper_idx, channel_masks=None): + # get the l1-norm sum for each filter + # w_abs_structured = self.get_channel_sum(wrapper, wrapper_idx) + # if channel_masks is not None: + # # if we need to mask some channels in advance + # w_abs_structured = w_abs_structured * channel_masks + # threshold = torch.topk(w_abs_structured.view(-1), + # num_prune, largest=False)[0].max() + # mask_weight = torch.gt(w_abs_structured, threshold)[ + # :, None, None, None].expand_as(weight).type_as(weight) + # mask_bias = torch.gt(w_abs_structured, threshold).type_as( + # weight).detach() if base_mask['bias_mask'] is not None else None + + # return {'weight_mask': mask_weight.detach(), 'bias_mask': mask_bias} + if isinstance(wrapper.module, torch.nn.Conv2d): + # get the l1-norm sum for each filter + w_abs_structured = self.get_channel_sum(wrapper, wrapper_idx) + if channel_masks is not None: + # if we need to mask some channels in advance + w_abs_structured = w_abs_structured * channel_masks + threshold = torch.topk(w_abs_structured.view(-1), + num_prune, largest=False)[0].max() + mask_weight = torch.gt(w_abs_structured, threshold)[ + :, None, None, None].expand_as(weight).type_as(weight) + mask_bias = torch.gt(w_abs_structured, threshold).type_as( + weight).detach() if base_mask['bias_mask'] is not None else None + + return {'weight_mask': mask_weight.detach(), 'bias_mask': mask_bias} + elif isinstance(wrapper.module, torch.nn.ConvTranspose2d): + # get the l1-norm sum for each filter + w_abs_structured = self.get_channel_sum(wrapper, wrapper_idx) + if channel_masks is not None: + # if we need to mask some channels in advance + w_abs_structured = w_abs_structured * channel_masks + threshold = torch.topk(w_abs_structured.view(-1), + num_prune, largest=False)[0].max() + mask_weight = torch.gt(w_abs_structured, threshold)[ + None, :, None, None].expand_as(weight).type_as(weight) + mask_bias = torch.gt(w_abs_structured, threshold).type_as( + weight).detach() if base_mask['bias_mask'] is not None else None + + return {'weight_mask': mask_weight.detach(), 'bias_mask': mask_bias} + + def get_channel_sum(self, wrapper, wrapper_idx): + # weight = wrapper.module.weight.data + # filters = weight.shape[0] + # w_abs = weight.abs() + # w_abs_structured = w_abs.view(filters, -1).sum(dim=1) + # return w_abs_structured + if isinstance(wrapper.module, torch.nn.Conv2d): + weight = wrapper.module.weight.data + filters = weight.shape[0] + w_abs = weight.abs() + w_abs_structured = w_abs.view(filters, -1).sum(dim=1) + return w_abs_structured + elif isinstance(wrapper.module, torch.nn.ConvTranspose2d): + weight = wrapper.module.weight.data + filters = weight.shape[1] + w_abs = weight.abs() + w_abs_structured = w_abs.view(filters, -1).sum(dim=1) + return w_abs_structured + + +class L2FilterPrunerMasker(StructuredWeightMasker): + """ + A structured pruning algorithm that prunes the filters with the + smallest L2 norm of the weights. + """ + + def get_mask(self, base_mask, weight, num_prune, wrapper, wrapper_idx, channel_masks=None): + # get the l2-norm sum for each filter + w_l2_norm = self.get_channel_sum(wrapper, wrapper_idx) + if channel_masks is not None: + # if we need to mask some channels in advance + w_l2_norm = w_l2_norm * channel_masks + threshold = torch.topk( + w_l2_norm.view(-1), num_prune, largest=False)[0].max() + mask_weight = torch.gt(w_l2_norm, threshold)[ + :, None, None, None].expand_as(weight).type_as(weight) + mask_bias = torch.gt(w_l2_norm, threshold).type_as( + weight).detach() if base_mask['bias_mask'] is not None else None + + return {'weight_mask': mask_weight.detach(), 'bias_mask': mask_bias} + + def get_channel_sum(self, wrapper, wrapper_idx): + weight = wrapper.module.weight.data + filters = weight.shape[0] + w = weight.view(filters, -1) + w_l2_norm = torch.sqrt((w ** 2).sum(dim=1)) + return w_l2_norm + + +class FPGMPrunerMasker(StructuredWeightMasker): + """ + A filter pruner via geometric median. + "Filter Pruning via Geometric Median for Deep Convolutional Neural Networks Acceleration", + https://arxiv.org/pdf/1811.00250.pdf + """ + + def get_mask(self, base_mask, weight, num_prune, wrapper, wrapper_idx, channel_masks=None): + if isinstance(wrapper.module, torch.nn.Conv2d): + min_gm_idx = self._get_min_gm_kernel_idx( + num_prune, wrapper, wrapper_idx, channel_masks) + for idx in min_gm_idx: + base_mask['weight_mask'][idx] = 0. + if base_mask['bias_mask'] is not None: + base_mask['bias_mask'][idx] = 0. + return base_mask + elif isinstance(wrapper.module, torch.nn.ConvTranspose2d): + min_gm_idx = self._get_min_gm_kernel_idx( + num_prune, wrapper, wrapper_idx, channel_masks) + for idx in min_gm_idx: + base_mask['weight_mask'][:, idx] = 0. + if base_mask['bias_mask'] is not None: + base_mask['bias_mask'][idx] = 0. + return base_mask + + def _get_min_gm_kernel_idx(self, num_prune, wrapper, wrapper_idx, channel_masks): + if isinstance(wrapper.module, torch.nn.Conv2d): + channel_dist = self.get_channel_sum(wrapper, wrapper_idx) + if channel_masks is not None: + channel_dist = channel_dist * channel_masks + dist_list = [(channel_dist[i], i) + for i in range(channel_dist.size(0))] + min_gm_kernels = sorted(dist_list, key=lambda x: x[0])[:num_prune] + return [x[1] for x in min_gm_kernels] + elif isinstance(wrapper.module, torch.nn.ConvTranspose2d): + channel_dist = self.get_channel_sum(wrapper, wrapper_idx) + if channel_masks is not None: + channel_dist = channel_dist * channel_masks + dist_list = [(channel_dist[i], i) + for i in range(channel_dist.size(0))] + min_gm_kernels = sorted(dist_list, key=lambda x: x[0])[:num_prune] + return [x[1] for x in min_gm_kernels] + + def _get_distance_sum(self, weight, out_idx, wrapper): + """ + Calculate the total distance between a specified filter (by out_idex and in_idx) and + all other filters. + Parameters + ---------- + weight: Tensor + convolutional filter weight + out_idx: int + output channel index of specified filter, this method calculates the total distance + between this specified filter and all other filters. + Returns + ------- + float32 + The total distance + """ + if isinstance(wrapper.module, torch.nn.Conv2d): + logger.debug('weight size: %s', weight.size()) + assert len(weight.size()) in [3, 4], 'unsupported weight shape' + + w = weight.view(weight.size(0), -1) + anchor_w = w[out_idx].unsqueeze(0).expand(w.size(0), w.size(1)) + x = w - anchor_w + x = (x * x).sum(-1) + x = torch.sqrt(x) + return x.sum() + elif isinstance(wrapper.module, torch.nn.ConvTranspose2d): + logger.debug('weight size: %s', weight.size()) + assert len(weight.size()) in [3, 4], 'unsupported weight shape' + + w = weight.view(weight.size(1), -1) + anchor_w = w[out_idx].unsqueeze(0).expand(w.size(0), w.size(1)) + x = w - anchor_w + x = (x * x).sum(-1) + x = torch.sqrt(x) + return x.sum() + + + def get_channel_sum(self, wrapper, wrapper_idx): + if isinstance(wrapper.module, torch.nn.Conv2d): + weight = wrapper.module.weight.data + assert len(weight.size()) in [3, 4] + dist_list = [] + for out_i in range(weight.size(0)): + dist_sum = self._get_distance_sum(weight, out_i, wrapper) + dist_list.append(dist_sum) + return torch.Tensor(dist_list).to(weight.device) + elif isinstance(wrapper.module, torch.nn.ConvTranspose2d): + weight = wrapper.module.weight.data + assert len(weight.size()) in [3, 4] + dist_list = [] + for out_i in range(weight.size(1)): + dist_sum = self._get_distance_sum(weight, out_i, wrapper) + dist_list.append(dist_sum) + return torch.Tensor(dist_list).to(weight.device) + + +class TaylorFOWeightFilterPrunerMasker(StructuredWeightMasker): + """ + A structured pruning algorithm that prunes the filters with the smallest + importance approximations based on the first order taylor expansion on the weight. + Molchanov, Pavlo and Mallya, Arun and Tyree, Stephen and Frosio, Iuri and Kautz, Jan, + "Importance Estimation for Neural Network Pruning", CVPR 2019. + http://jankautz.com/publications/Importance4NNPruning_CVPR19.pdf + """ + + def __init__(self, model, pruner, statistics_batch_num=1): + super().__init__(model, pruner) + self.statistics_batch_num = statistics_batch_num + self.pruner.iterations = 0 + self.pruner.set_wrappers_attribute("contribution", None) + self.pruner.patch_optimizer(self.calc_contributions) + + def get_mask(self, base_mask, weight, num_prune, wrapper, wrapper_idx, channel_masks=None): + if isinstance(wrapper.module, torch.nn.Conv2d): + channel_contribution = self.get_channel_sum(wrapper, wrapper_idx) + if channel_contribution is None: + # iteration is not enough + return None + if channel_masks is not None: + channel_contribution = channel_contribution * channel_masks + prune_indices = torch.argsort(channel_contribution)[:num_prune] + for idx in prune_indices: + base_mask['weight_mask'][idx] = 0. + if base_mask['bias_mask'] is not None: + base_mask['bias_mask'][idx] = 0. + return base_mask + elif isinstance(wrapper.module, torch.nn.ConvTranspose2d): + channel_contribution = self.get_channel_sum(wrapper, wrapper_idx) + if channel_contribution is None: + # iteration is not enough + return None + if channel_masks is not None: + channel_contribution = channel_contribution * channel_masks + prune_indices = torch.argsort(channel_contribution)[:num_prune] + for idx in prune_indices: + base_mask['weight_mask'][:, idx] = 0. + if base_mask['bias_mask'] is not None: + base_mask['bias_mask'][idx] = 0. + return base_mask + + def calc_contributions(self): + """ + Calculate the estimated importance of filters as a sum of individual contribution + based on the first order taylor expansion. + """ + if self.pruner.iterations >= self.statistics_batch_num: + return + + for wrapper in self.pruner.get_modules_wrapper(): + if isinstance(wrapper.module, torch.nn.Conv2d): + filters = wrapper.module.weight.size(0) + contribution = ( + wrapper.module.weight * wrapper.module.weight.grad).data.pow(2).view(filters, -1).sum(dim=1) + if wrapper.contribution is None: + wrapper.contribution = contribution + else: + wrapper.contribution += contribution + + elif isinstance(wrapper.module, torch.nn.ConvTranspose2d): + filters = wrapper.module.weight.size(1) + contribution = ( + wrapper.module.weight * wrapper.module.weight.grad).data.pow(2).view(filters, -1).sum(dim=0) + if wrapper.contribution is None: + wrapper.contribution = contribution + else: + wrapper.contribution += contribution + + self.pruner.iterations += 1 + + def get_channel_sum(self, wrapper, wrapper_idx): + if self.pruner.iterations < self.statistics_batch_num: + return None + if wrapper.contribution is None: + return None + return wrapper.contribution + + +class ActivationFilterPrunerMasker(StructuredWeightMasker): + def __init__(self, model, pruner, statistics_batch_num=1, activation='relu'): + super().__init__(model, pruner) + self.statistics_batch_num = statistics_batch_num + self.pruner.hook_id = self._add_activation_collector(self.pruner) + self.pruner.iterations = 0 + self.pruner.patch_optimizer(self._iteration_counter) + + assert activation in ['relu', 'relu6'] + if activation == 'relu': + self.pruner.activation = torch.nn.functional.relu + elif activation == 'relu6': + self.pruner.activation = torch.nn.functional.relu6 + else: + self.pruner.activation = None + + def _iteration_counter(self): + self.pruner.iterations += 1 + + def _add_activation_collector(self, pruner): + def collector(collected_activation): + def hook(module_, input_, output): + collected_activation.append( + pruner.activation(output.detach().cpu())) + return hook + pruner.collected_activation = {} + pruner._fwd_hook_id += 1 + pruner._fwd_hook_handles[pruner._fwd_hook_id] = [] + + for wrapper_idx, wrapper in enumerate(pruner.get_modules_wrapper()): + pruner.collected_activation[wrapper_idx] = [] + handle = wrapper.register_forward_hook( + collector(pruner.collected_activation[wrapper_idx])) + + pruner._fwd_hook_handles[pruner._fwd_hook_id].append(handle) + return pruner._fwd_hook_id + + +class ActivationAPoZRankFilterPrunerMasker(ActivationFilterPrunerMasker): + """ + A structured pruning algorithm that prunes the filters with the + smallest APoZ(average percentage of zeros) of output activations. + Hengyuan Hu, Rui Peng, Yu-Wing Tai and Chi-Keung Tang, + "Network Trimming: A Data-Driven Neuron Pruning Approach towards Efficient Deep Architectures", ICLR 2016. + https://arxiv.org/abs/1607.03250 + """ + + def get_mask(self, base_mask, weight, num_prune, wrapper, wrapper_idx, channel_masks=None): + apoz = self.get_channel_sum(wrapper, wrapper_idx) + if apoz is None: + # the collected activations are not enough + return None + if channel_masks is not None: + apoz = apoz * channel_masks + + prune_indices = torch.argsort(apoz)[:num_prune] + for idx in prune_indices: + base_mask['weight_mask'][idx] = 0. + if base_mask['bias_mask'] is not None: + base_mask['bias_mask'][idx] = 0. + + if self.pruner.hook_id in self.pruner._fwd_hook_handles: + self.pruner.remove_activation_collector(self.pruner.hook_id) + + return base_mask + + def _calc_apoz(self, activations): + """ + Calculate APoZ(average percentage of zeros) of activations. + + Parameters + ---------- + activations : list + Layer's output activations + + Returns + ------- + torch.Tensor + Filter's APoZ(average percentage of zeros) of the activations + """ + activations = torch.cat(activations, 0) + _eq_zero = torch.eq(activations, torch.zeros_like(activations)) + _apoz = torch.sum(_eq_zero, dim=(0, 2, 3), dtype=torch.float64) / \ + torch.numel(_eq_zero[:, 0, :, :]) + return torch.ones_like(_apoz) - _apoz + + def get_channel_sum(self, wrapper, wrapper_idx): + assert wrapper_idx is not None + activations = self.pruner.collected_activation[wrapper_idx] + if len(activations) < self.statistics_batch_num: + # collected activations is not enough + return None + return self._calc_apoz(activations).to(wrapper.module.weight.device) + + +class ActivationMeanRankFilterPrunerMasker(ActivationFilterPrunerMasker): + """ + A structured pruning algorithm that prunes the filters with the + smallest mean value of output activations. + Pavlo Molchanov, Stephen Tyree, Tero Karras, Timo Aila and Jan Kautz, + "Pruning Convolutional Neural Networks for Resource Efficient Inference", ICLR 2017. + https://arxiv.org/abs/1611.06440 + """ + + def get_mask(self, base_mask, weight, num_prune, wrapper, wrapper_idx, channel_masks=None): + + mean_activation = self.get_channel_sum(wrapper, wrapper_idx) + if mean_activation is None: + # the collected activation is not enough + return None + if channel_masks is not None: + mean_activation = mean_activation * channel_masks + + prune_indices = torch.argsort(mean_activation)[:num_prune] + for idx in prune_indices: + base_mask['weight_mask'][idx] = 0. + if base_mask['bias_mask'] is not None: + base_mask['bias_mask'][idx] = 0. + # if len(activations) < self.statistics_batch_num, the code + # cannot reach here + if self.pruner.hook_id in self.pruner._fwd_hook_handles: + self.pruner.remove_activation_collector(self.pruner.hook_id) + + return base_mask + + def _cal_mean_activation(self, activations): + """ + Calculate mean value of activations. + + Parameters + ---------- + activations : list + Layer's output activations + + Returns + ------- + torch.Tensor + Filter's mean value of the output activations + """ + activations = torch.cat(activations, 0) + mean_activation = torch.mean(activations, dim=(0, 2, 3)) + return mean_activation + + def get_channel_sum(self, wrapper, wrapper_idx): + assert wrapper_idx is not None + activations = self.pruner.collected_activation[wrapper_idx] + if len(activations) < self.statistics_batch_num: + return None + # the memory overhead here is acceptable, because only + # the mean_activation tensor returned by _cal_mean_activation + # is transfer to gpu. + return self._cal_mean_activation(activations).to(wrapper.module.weight.device) + + +class SlimPrunerMasker(WeightMasker): + """ + A structured pruning algorithm that prunes channels by pruning the weights of BN layers. + Zhuang Liu, Jianguo Li, Zhiqiang Shen, Gao Huang, Shoumeng Yan and Changshui Zhang + "Learning Efficient Convolutional Networks through Network Slimming", 2017 ICCV + https://arxiv.org/pdf/1708.06519.pdf + """ + + def __init__(self, model, pruner, **kwargs): + super().__init__(model, pruner) + self.global_threshold = None + + def _get_global_threshold(self): + weight_list = [] + for (layer, _) in self.pruner.get_modules_to_compress(): + weight_list.append(layer.module.weight.data.abs().clone()) + all_bn_weights = torch.cat(weight_list) + k = int(all_bn_weights.shape[0] * self.pruner.config_list[0]['sparsity']) + self.global_threshold = torch.topk( + all_bn_weights.view(-1), k, largest=False)[0].max() + print(f'set global threshold to {self.global_threshold}') + + def calc_mask(self, sparsity, wrapper, wrapper_idx=None): + assert wrapper.type == 'BatchNorm2d', 'SlimPruner only supports 2d batch normalization layer pruning' + + if self.global_threshold is None: + self._get_global_threshold() + + weight = wrapper.module.weight.data.clone() + if wrapper.weight_mask is not None: + # apply base mask for iterative pruning + weight = weight * wrapper.weight_mask + + base_mask = torch.ones(weight.size()).type_as(weight).detach() + mask = {'weight_mask': base_mask.detach( + ), 'bias_mask': base_mask.clone().detach()} + filters = weight.size(0) + num_prune = int(filters * sparsity) + if filters >= 2 and num_prune >= 1: + w_abs = weight.abs() + mask_weight = torch.gt( + w_abs, self.global_threshold).type_as(weight) + mask_bias = mask_weight.clone() + mask = {'weight_mask': mask_weight.detach( + ), 'bias_mask': mask_bias.detach()} + return mask + +def least_square_sklearn(X, Y): + from sklearn.linear_model import LinearRegression + reg = LinearRegression(fit_intercept=False) + reg.fit(X, Y) + return reg.coef_ + + +class AMCWeightMasker(WeightMasker): + """ + Weight maskser class for AMC pruner. Currently, AMCPruner only supports pruning kernel + size 1x1 pointwise Conv2d layer. Before using this class to prune kernels, AMCPruner + collected input and output feature maps for each layer, the features maps are flattened + and save into wrapper.input_feat and wrapper.output_feat. + + Parameters + ---------- + model: nn.Module + model to be pruned + pruner: Pruner + A Pruner instance used to prune the model + preserve_round: int + after pruning, preserve filters/channels round to `preserve_round`, for example: + for a Conv2d layer, output channel is 32, sparsity is 0.2, if preserve_round is + 1 (no preserve round), then there will be int(32 * 0.2) = 6 filters pruned, and + 32 - 6 = 26 filters are preserved. If preserve_round is 4, preserved filters will + be round up to 28 (which can be divided by 4) and only 4 filters are pruned. + """ + + def __init__(self, model, pruner, preserve_round=1): + self.model = model + self.pruner = pruner + self.preserve_round = preserve_round + + def calc_mask(self, sparsity, wrapper, wrapper_idx=None, preserve_idx=None): + """ + Calculate the mask of given layer. + Parameters + ---------- + sparsity: float + pruning ratio, preserved weight ratio is `1 - sparsity` + wrapper: PrunerModuleWrapper + layer wrapper of this layer + wrapper_idx: int + index of this wrapper in pruner's all wrappers + Returns + ------- + dict + dictionary for storing masks, keys of the dict: + 'weight_mask': weight mask tensor + 'bias_mask': bias mask tensor (optional) + """ + msg = 'module type {} is not supported!'.format(wrapper.type) + assert wrapper.type in ['Conv2d', 'Linear'], msg + weight = wrapper.module.weight.data + bias = None + if hasattr(wrapper.module, 'bias') and wrapper.module.bias is not None: + bias = wrapper.module.bias.data + + if wrapper.weight_mask is None: + mask_weight = torch.ones(weight.size()).type_as(weight).detach() + else: + mask_weight = wrapper.weight_mask.clone() + if bias is not None: + if wrapper.bias_mask is None: + mask_bias = torch.ones(bias.size()).type_as(bias).detach() + else: + mask_bias = wrapper.bias_mask.clone() + else: + mask_bias = None + mask = {'weight_mask': mask_weight, 'bias_mask': mask_bias} + + num_total = weight.size(1) + num_prune = int(num_total * sparsity) + if self.preserve_round > 1: + num_preserve = num_total - num_prune + num_preserve = int( + math.ceil(num_preserve * 1. / self.preserve_round) * self.preserve_round) + if num_preserve > num_total: + num_preserve = num_total + num_prune = num_total - num_preserve + + if (num_total < 2 or num_prune < 1) and preserve_idx is None: + return mask + + return self.get_mask(mask, weight, num_preserve, wrapper, wrapper_idx, preserve_idx) + + def get_mask(self, base_mask, weight, num_preserve, wrapper, wrapper_idx, preserve_idx): + w = weight.data.cpu().numpy() + if wrapper.type == 'Linear': + w = w[:, :, None, None] + + if preserve_idx is None: + importance = np.abs(w).sum((0, 2, 3)) + # sum magnitude along C_in, sort descend + sorted_idx = np.argsort(-importance) + d_prime = num_preserve + preserve_idx = sorted_idx[:d_prime] # to preserve index + else: + d_prime = len(preserve_idx) + + assert len(preserve_idx) == d_prime + mask = np.zeros(w.shape[1], bool) + mask[preserve_idx] = True + + # reconstruct, X, Y <= [N, C] + X, Y = wrapper.input_feat, wrapper.output_feat + masked_X = X[:, mask] + if w.shape[2] == 1: # 1x1 conv or fc + rec_weight = least_square_sklearn(X=masked_X, Y=Y) + rec_weight = rec_weight.reshape(-1, 1, 1, d_prime) # (C_out, K_h, K_w, C_in') + rec_weight = np.transpose(rec_weight, (0, 3, 1, 2)) # (C_out, C_in', K_h, K_w) + + rec_weight_pad = np.zeros_like(w) + # pylint: disable=all + rec_weight_pad[:, mask, :, :] = rec_weight + rec_weight = rec_weight_pad + + if wrapper.type == 'Linear': + rec_weight = rec_weight.squeeze() + assert len(rec_weight.shape) == 2 + + # now assign + wrapper.module.weight.data = torch.from_numpy(rec_weight).to(weight.device) + + mask_weight = torch.zeros_like(weight) + if wrapper.type == 'Linear': + mask_weight[:, preserve_idx] = 1. + if base_mask['bias_mask'] is not None and wrapper.module.bias is not None: + mask_bias = torch.ones_like(wrapper.module.bias) + else: + mask_weight[:, preserve_idx, :, :] = 1. + mask_bias = None + + return {'weight_mask': mask_weight.detach(), 'bias_mask': mask_bias} diff --git a/utils/third_party/nni_new/algorithms/compression/pytorch/pruning/weight_masker.py b/utils/third_party/nni_new/algorithms/compression/pytorch/pruning/weight_masker.py new file mode 100644 index 0000000000000000000000000000000000000000..aec8444ced37bccfa5f05e0a9f81b2843c2a35e4 --- /dev/null +++ b/utils/third_party/nni_new/algorithms/compression/pytorch/pruning/weight_masker.py @@ -0,0 +1,28 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +class WeightMasker(object): + def __init__(self, model, pruner, **kwargs): + self.model = model + self.pruner = pruner + + def calc_mask(self, sparsity, wrapper, wrapper_idx=None): + """ + Calculate the mask of given layer. + Parameters + ---------- + sparsity: float + pruning ratio, preserved weight ratio is `1 - sparsity` + wrapper: PrunerModuleWrapper + layer wrapper of this layer + wrapper_idx: int + index of this wrapper in pruner's all wrappers + Returns + ------- + dict + dictionary for storing masks, keys of the dict: + 'weight_mask': weight mask tensor + 'bias_mask': bias mask tensor (optional) + """ + + raise NotImplementedError('{} calc_mask is not implemented'.format(self.__class__.__name__)) diff --git a/utils/third_party/nni_new/algorithms/compression/pytorch/quantization/__init__.py b/utils/third_party/nni_new/algorithms/compression/pytorch/quantization/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..0e728dcd3f0817a529eb6801f3ab935de6751348 --- /dev/null +++ b/utils/third_party/nni_new/algorithms/compression/pytorch/quantization/__init__.py @@ -0,0 +1,4 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .quantizers import * diff --git a/utils/third_party/nni_new/algorithms/compression/pytorch/quantization/quantizers.py b/utils/third_party/nni_new/algorithms/compression/pytorch/quantization/quantizers.py new file mode 100644 index 0000000000000000000000000000000000000000..dbd5e5b3c34048a1a92b3623735ffd48fa5b45d8 --- /dev/null +++ b/utils/third_party/nni_new/algorithms/compression/pytorch/quantization/quantizers.py @@ -0,0 +1,772 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +import copy +import torch +from schema import Schema, And, Or, Optional +from nni.compression.pytorch.utils.config_validation import CompressorSchema +from nni.compression.pytorch.compressor import Quantizer, QuantForward, QuantGrad, QuantType + +__all__ = ['NaiveQuantizer', 'QAT_Quantizer', 'DoReFaQuantizer', 'BNNQuantizer', 'LsqQuantizer'] + +logger = logging.getLogger(__name__) + + +class NaiveQuantizer(Quantizer): + """quantize weight to 8 bits + """ + + def __init__(self, model, config_list, optimizer=None): + super().__init__(model, config_list, optimizer) + self.layer_scale = {} + + def validate_config(self, model, config_list): + schema = CompressorSchema([{ + Optional('quant_types'): ['weight'], + Optional('quant_bits'): Or(8, {'weight': 8}), + Optional('op_types'): [str], + Optional('op_names'): [str] + }], model, logger) + + schema.validate(config_list) + + def quantize_weight(self, wrapper, **kwargs): + weight = copy.deepcopy(wrapper.module.old_weight.data) + new_scale = weight.abs().max() / 127 + scale = max(self.layer_scale.get(wrapper.name, 0), new_scale) + self.layer_scale[wrapper.name] = scale + orig_type = weight.type() # TODO: user layer + weight = weight.div(scale).type(torch.int8).type(orig_type).mul(scale) + wrapper.module.weight = weight + return weight + +def update_ema(biased_ema, value, decay): + """ + calculate biased stat and unbiased stat in each step using exponential moving average method + + Parameters + ---------- + biased_ema : float + previous stat value + value : float + current stat value + decay : float + the weight of previous stat value, larger means smoother curve + + Returns + ------- + float, float + """ + biased_ema = biased_ema * decay + (1 - decay) * value + return biased_ema + + +def update_quantization_param(bits, rmin, rmax): + """ + calculate the `zero_point` and `scale`. + + Parameters + ---------- + bits : int + quantization bits length + rmin : Tensor + min value of real value + rmax : Tensor + max value of real value + + Returns + ------- + float, float + """ + # extend the [min, max] interval to ensure that it contains 0. + # Otherwise, we would not meet the requirement that 0 be an exactly + # representable value. + rmin = torch.min(rmin, torch.Tensor([0]).to(rmin.device)) + rmax = torch.max(rmax, torch.Tensor([0]).to(rmin.device)) + qmin = torch.Tensor([0]).to(rmin.device) + qmax = torch.Tensor([(1 << bits) - 1]).to(rmin.device) + + # First determine the scale. + scale = (rmax - rmin) / (qmax - qmin) + + # Zero-point computation. + initial_zero_point = qmin - rmin / scale + + # Now we need to nudge the zero point to be an integer + if initial_zero_point < qmin: + nudged_zero_point = qmin + elif initial_zero_point > qmax: + nudged_zero_point = qmax + else: + nudged_zero_point = torch.round(initial_zero_point) + + return scale, nudged_zero_point + + +def get_bits_length(config, quant_type): + if isinstance(config["quant_bits"], int): + return config["quant_bits"] + else: + return config["quant_bits"].get(quant_type) + +class QATGrad(QuantGrad): + @staticmethod + def quant_backward(tensor, grad_output, quant_type, scale, zero_point, qmin, qmax): + tensor_q = QuantGrad._quantize(tensor, scale, zero_point) + mask = (tensor_q < qmin) | (tensor_q > qmax) + grad_output[mask] = 0 + return grad_output + + +class QAT_Quantizer(Quantizer): + """Quantizer defined in: + Quantization and Training of Neural Networks for Efficient Integer-Arithmetic-Only Inference + http://openaccess.thecvf.com/content_cvpr_2018/papers/Jacob_Quantization_and_Training_CVPR_2018_paper.pdf + """ + + def __init__(self, model, config_list, optimizer=None): + """ + Parameters + ---------- + layer : LayerInfo + the layer to quantize + config_list : list of dict + list of configurations for quantization + supported keys for dict: + - quant_types : list of string + type of quantization you want to apply, currently support 'weight', 'input', 'output' + - quant_bits : int or dict of {str : int} + bits length of quantization, key is the quantization type, value is the length, eg. {'weight', 8}, + when the type is int, all quantization types share same bits length + - quant_start_step : int + disable quantization until model are run by certain number of steps, this allows the network to enter a more stable + state where activation quantization ranges do not exclude a significant fraction of values, default value is 0 + - op_types : list of string + types of nn.module you want to apply quantization, eg. 'Conv2d' + """ + super().__init__(model, config_list, optimizer) + self.quant_grad = QATGrad.apply + modules_to_compress = self.get_modules_to_compress() + device = next(model.parameters()).device + self.bound_model.register_buffer("steps", torch.Tensor([1])) + for layer, config in modules_to_compress: + layer.module.register_buffer("zero_point", torch.Tensor([0.0])) + layer.module.register_buffer("scale", torch.Tensor([1.0])) + layer.module.register_buffer('ema_decay', torch.Tensor([0.99])) + if "weight" in config.get("quant_types", []): + layer.module.register_buffer('weight_bit', torch.zeros(1)) + layer.module.register_buffer('tracked_min_input', torch.zeros(1)) + layer.module.register_buffer('tracked_max_input', torch.zeros(1)) + if "output" in config.get("quant_types", []): + layer.module.register_buffer('activation_bit', torch.zeros(1)) + layer.module.register_buffer('tracked_min_activation', torch.zeros(1)) + layer.module.register_buffer('tracked_max_activation', torch.zeros(1)) + self.bound_model.to(device) + + def _del_simulated_attr(self, module): + """ + delete redundant parameters in quantize module + """ + del_attr_list = ['old_weight', 'ema_decay', 'tracked_min_activation', 'tracked_max_activation', 'tracked_min_input', \ + 'tracked_max_input', 'scale', 'zero_point', 'weight_bit', 'activation_bit'] + for attr in del_attr_list: + if hasattr(module, attr): + delattr(module, attr) + + def validate_config(self, model, config_list): + """ + Parameters + ---------- + model : torch.nn.Module + Model to be pruned + config_list : list of dict + List of configurations + """ + schema = CompressorSchema([{ + Optional('quant_types'): Schema([lambda x: x in ['weight', 'output']]), + Optional('quant_bits'): Or(And(int, lambda n: 0 < n < 32), Schema({ + Optional('weight'): And(int, lambda n: 0 < n < 32), + Optional('output'): And(int, lambda n: 0 < n < 32), + })), + Optional('quant_start_step'): And(int, lambda n: n >= 0), + Optional('op_types'): [str], + Optional('op_names'): [str] + }], model, logger) + + schema.validate(config_list) + + def _quantize(self, bits, op, real_val): + """ + quantize real value. + + Parameters + ---------- + bits : int + quantization bits length + op : torch.nn.Module + target module + real_val : Tensor + real value to be quantized + + Returns + ------- + Tensor + """ + op.zero_point = op.zero_point.to(real_val.device) + op.scale = op.scale.to(real_val.device) + transformed_val = op.zero_point + real_val / op.scale + qmin = 0 + qmax = (1 << bits) - 1 + clamped_val = torch.clamp(transformed_val, qmin, qmax) + quantized_val = torch.round(clamped_val) + return quantized_val + + def _dequantize(self, op, quantized_val): + """ + dequantize quantized value. + Because we simulate quantization in training process, all the computations still happen as float point computations, which means we + first quantize tensors then dequantize them. For more details, please refer to the paper. + + Parameters + ---------- + op : torch.nn.Module + target module + quantized_val : float + quantized_val value to be dequantized + + Returns + ------- + float + """ + real_val = op.scale * (quantized_val - op.zero_point) + return real_val + + def quantize_weight(self, wrapper, **kwargs): + config = wrapper.config + module = wrapper.module + input = kwargs['input_tensor'] + weight = copy.deepcopy(wrapper.module.old_weight.data) + weight_bits = get_bits_length(config, 'weight') + quant_start_step = config.get('quant_start_step', 0) + assert weight_bits >= 1, "quant bits length should be at least 1" + + # we dont update weight in evaluation stage + if quant_start_step > self.bound_model.steps: + module.tracked_min_input, module.tracked_max_input = torch.min(input), torch.max(input) + return weight + + if not wrapper.training: + return weight + + current_min, current_max = torch.min(input), torch.max(input) + module.tracked_min_input = update_ema(module.tracked_min_input, current_min, + module.ema_decay) + module.tracked_max_input = update_ema(module.tracked_max_input, current_max, + module.ema_decay) + + # if bias exists, quantize bias to uint32 + if hasattr(wrapper.module, 'bias') and wrapper.module.bias is not None: + bias = wrapper.module.bias.data + bias_bits = 32 + rmin, rmax = torch.min(bias), torch.max(bias) + module.scale, module.zero_point = update_quantization_param(bias_bits, rmin, rmax) + bias = self._quantize(bias_bits, module, bias) + bias = self._dequantize(module, bias) + wrapper.module.bias.data = bias + + + # quantize weight + rmin, rmax = torch.min(weight), torch.max(weight) + module.scale, module.zero_point = update_quantization_param(weight_bits, rmin, rmax) + weight = self._quantize(weight_bits, module, weight) + weight = self._dequantize(module, weight) + module.weight_bit = torch.Tensor([weight_bits]) + wrapper.module.weight = weight + return weight + + def quantize_output(self, output, wrapper, **kwargs): + config = wrapper.config + module = wrapper.module + output_bits = get_bits_length(config, 'output') + module.activation_bit = torch.Tensor([output_bits]) + quant_start_step = config.get('quant_start_step', 0) + assert output_bits >= 1, "quant bits length should be at least 1" + + if quant_start_step > self.bound_model.steps: + module.tracked_min_activation, module.tracked_max_activation = torch.min(output), torch.max(output) + return output + + # we dont update output quantization parameters in evaluation stage + if wrapper.training: + current_min, current_max = torch.min(output), torch.max(output) + module.tracked_min_activation = update_ema(module.tracked_min_activation, current_min, + module.ema_decay) + module.tracked_max_activation = update_ema(module.tracked_max_activation, current_max, + module.ema_decay) + module.scale, module.zero_point = update_quantization_param(output_bits, module.tracked_min_activation, module.tracked_max_activation) + out = self._quantize(output_bits, module, output) + out = self._dequantize(module, out) + return out + + def export_model(self, model_path, calibration_path=None, onnx_path=None, input_shape=None, device=None): + """ + Export quantized model weights and calibration parameters(optional) + + Parameters + ---------- + model_path : str + path to save quantized model weight + calibration_path : str + (optional) path to save quantize parameters after calibration + onnx_path : str + (optional) path to save onnx model + input_shape : list or tuple + input shape to onnx model + device : torch.device + device of the model, used to place the dummy input tensor for exporting onnx file. + the tensor is placed on cpu if ```device``` is None + + Returns + ------- + Dict + """ + assert model_path is not None, 'model_path must be specified' + self._unwrap_model() + calibration_config = {} + + for name, module in self.bound_model.named_modules(): + if hasattr(module, 'weight_bit') or hasattr(module, 'activation_bit'): + calibration_config[name] = {} + if hasattr(module, 'weight_bit'): + calibration_config[name]['weight_bit'] = int(module.weight_bit) + calibration_config[name]['tracked_min_input'] = float(module.tracked_min_input) + calibration_config[name]['tracked_max_input'] = float(module.tracked_max_input) + if hasattr(module, 'activation_bit'): + calibration_config[name]['activation_bit'] = int(module.activation_bit) + calibration_config[name]['tracked_min_activation'] = float(module.tracked_min_activation) + calibration_config[name]['tracked_max_activation'] = float(module.tracked_max_activation) + self._del_simulated_attr(module) + + self.export_model_save(self.bound_model, model_path, calibration_config, calibration_path, onnx_path, input_shape, device) + + return calibration_config + + def fold_bn(self, config, **kwargs): + # TODO simulate folded weight + pass + + def step_with_optimizer(self): + """ + override `compressor` `step` method, quantization only happens after certain number of steps + """ + self.bound_model.steps += 1 + + +class DoReFaQuantizer(Quantizer): + """Quantizer using the DoReFa scheme, as defined in: + Zhou et al., DoReFa-Net: Training Low Bitwidth Convolutional Neural Networks with Low Bitwidth Gradients + (https://arxiv.org/abs/1606.06160) + """ + + def __init__(self, model, config_list, optimizer=None): + super().__init__(model, config_list, optimizer) + device = next(model.parameters()).device + modules_to_compress = self.get_modules_to_compress() + for layer, config in modules_to_compress: + if "weight" in config.get("quant_types", []): + layer.module.register_buffer('weight_bit', torch.zeros(1)) + self.bound_model.to(device) + + def _del_simulated_attr(self, module): + """ + delete redundant parameters in quantize module + """ + del_attr_list = ['old_weight', 'weight_bit'] + for attr in del_attr_list: + if hasattr(module, attr): + delattr(module, attr) + + def validate_config(self, model, config_list): + """ + Parameters + ---------- + model : torch.nn.Module + Model to be pruned + config_list : list of dict + List of configurations + """ + schema = CompressorSchema([{ + Optional('quant_types'): Schema([lambda x: x in ['weight']]), + Optional('quant_bits'): Or(And(int, lambda n: 0 < n < 32), Schema({ + Optional('weight'): And(int, lambda n: 0 < n < 32) + })), + Optional('op_types'): [str], + Optional('op_names'): [str] + }], model, logger) + + schema.validate(config_list) + + def quantize_weight(self, wrapper, **kwargs): + weight = copy.deepcopy(wrapper.module.old_weight.data) + weight_bits = get_bits_length(wrapper.config, 'weight') + weight = weight.tanh() + weight = weight / (2 * weight.abs().max()) + 0.5 + weight = self.quantize(weight, weight_bits) + weight = 2 * weight - 1 + wrapper.module.weight = weight + wrapper.module.weight_bit = torch.Tensor([weight_bits]) + # wrapper.module.weight.data = weight + return weight + + def quantize(self, input_ri, q_bits): + scale = pow(2, q_bits) - 1 + output = torch.round(input_ri * scale) / scale + return output + + def export_model(self, model_path, calibration_path=None, onnx_path=None, input_shape=None, device=None): + """ + Export quantized model weights and calibration parameters(optional) + + Parameters + ---------- + model_path : str + path to save quantized model weight + calibration_path : str + (optional) path to save quantize parameters after calibration + onnx_path : str + (optional) path to save onnx model + input_shape : list or tuple + input shape to onnx model + device : torch.device + device of the model, used to place the dummy input tensor for exporting onnx file. + the tensor is placed on cpu if ```device``` is None + + Returns + ------- + Dict + """ + assert model_path is not None, 'model_path must be specified' + self._unwrap_model() + calibration_config = {} + + for name, module in self.bound_model.named_modules(): + if hasattr(module, 'weight_bit'): + calibration_config[name] = {} + calibration_config[name]['weight_bit'] = int(module.weight_bit) + self._del_simulated_attr(module) + + self.export_model_save(self.bound_model, model_path, calibration_config, calibration_path, onnx_path, input_shape, device) + + return calibration_config + + +class ClipGrad(QuantGrad): + @staticmethod + def quant_backward(tensor, grad_output, quant_type, scale, zero_point, qmin, qmax): + if quant_type == QuantType.QUANT_OUTPUT: + grad_output[torch.abs(tensor) > 1] = 0 + return grad_output + + +class BNNQuantizer(Quantizer): + """Binarized Neural Networks, as defined in: + Binarized Neural Networks: Training Deep Neural Networks with Weights and Activations Constrained to +1 or -1 + (https://arxiv.org/abs/1602.02830) + """ + + def __init__(self, model, config_list, optimizer=None): + super().__init__(model, config_list, optimizer) + device = next(model.parameters()).device + self.quant_grad = ClipGrad.apply + modules_to_compress = self.get_modules_to_compress() + for layer, config in modules_to_compress: + if "weight" in config.get("quant_types", []): + layer.module.register_buffer('weight_bit', torch.zeros(1)) + self.bound_model.to(device) + + def _del_simulated_attr(self, module): + """ + delete redundant parameters in quantize module + """ + del_attr_list = ['old_weight', 'weight_bit'] + for attr in del_attr_list: + if hasattr(module, attr): + delattr(module, attr) + + def validate_config(self, model, config_list): + """ + Parameters + ---------- + model : torch.nn.Module + Model to be pruned + config_list : list of dict + List of configurations + """ + schema = CompressorSchema([{ + Optional('quant_types'): Schema([lambda x: x in ['weight', 'output']]), + Optional('quant_bits'): Or(And(int, lambda n: 0 < n < 32), Schema({ + Optional('weight'): And(int, lambda n: 0 < n < 32), + Optional('output'): And(int, lambda n: 0 < n < 32), + })), + Optional('op_types'): [str], + Optional('op_names'): [str] + }], model, logger) + + schema.validate(config_list) + + def quantize_weight(self, wrapper, **kwargs): + weight = copy.deepcopy(wrapper.module.old_weight.data) + weight = torch.sign(weight) + # remove zeros + weight[weight == 0] = 1 + wrapper.module.weight = weight + wrapper.module.weight_bit = torch.Tensor([1.0]) + return weight + + def quantize_output(self, output, wrapper, **kwargs): + out = torch.sign(output) + # remove zeros + out[out == 0] = 1 + return out + + def export_model(self, model_path, calibration_path=None, onnx_path=None, input_shape=None, device=None): + """ + Export quantized model weights and calibration parameters(optional) + + Parameters + ---------- + model_path : str + path to save quantized model weight + calibration_path : str + (optional) path to save quantize parameters after calibration + onnx_path : str + (optional) path to save onnx model + input_shape : list or tuple + input shape to onnx model + device : torch.device + device of the model, used to place the dummy input tensor for exporting onnx file. + the tensor is placed on cpu if ```device``` is None + + Returns + ------- + Dict + """ + assert model_path is not None, 'model_path must be specified' + self._unwrap_model() + calibration_config = {} + + for name, module in self.bound_model.named_modules(): + if hasattr(module, 'weight_bit'): + calibration_config[name] = {} + calibration_config[name]['weight_bit'] = int(module.weight_bit) + self._del_simulated_attr(module) + + self.export_model_save(self.bound_model, model_path, calibration_config, calibration_path, onnx_path, input_shape, device) + + return calibration_config + + +class LsqQuantizer(Quantizer): + """Quantizer defined in: + Learned Step Size Quantization (ICLR 2020) + https://arxiv.org/pdf/1902.08153.pdf + """ + + def __init__(self, model, config_list, optimizer=None): + """ + Parameters + ---------- + model : torch.nn.Module + the model to be quantized + config_list : list of dict + list of configurations for quantization + supported keys for dict: + - quant_types : list of string + type of quantization you want to apply, currently support 'weight', 'input', 'output' + - quant_bits : int or dict of {str : int} + bits length of quantization, key is the quantization type, value is the length, eg. {'weight': 8}, + when the type is int, all quantization types share same bits length + - quant_start_step : int + disable quantization until model are run by certain number of steps, this allows the network to enter a more stable + state where activation quantization ranges do not exclude a significant fraction of values, default value is 0 + - op_types : list of string + types of nn.module you want to apply quantization, eg. 'Conv2d' + """ + super().__init__(model, config_list, optimizer) + device = next(model.parameters()).device + self.quant_grad = QuantForward() + modules_to_compress = self.get_modules_to_compress() + self.bound_model.register_buffer("steps", torch.Tensor([1])) + for layer, config in modules_to_compress: + if "weight" in config.get("quant_types", []): + layer.module.register_parameter("weight_scale", torch.nn.Parameter(torch.Tensor([1.0]))) + # todo: support per-channel quantization for weight since TensorRT use it for conv weight + q_bit = get_bits_length(config, "weight") + layer.module.register_buffer('weight_bit', torch.Tensor([q_bit])) + qmax = 2 ** (q_bit - 1) - 1 + qmin = -2 ** (q_bit - 1) + init_weight_scale = layer.module.weight.data.detach().abs().mean() * 2 / (qmax ** 0.5) + layer.module.weight_scale = torch.nn.Parameter(init_weight_scale) + layer.module.weight_qmax = qmax + layer.module.weight_qmin = qmin + + self.optimizer.add_param_group({"params": layer.module.weight_scale}) + + if "output" in config.get("quant_types", []): + # scale of activation will be initialized using the first batch data + layer.module.register_parameter("output_scale", torch.nn.Parameter(torch.Tensor([1.0]))) + q_bit = get_bits_length(config, "output") + layer.module.register_buffer('output_bit', torch.Tensor([q_bit])) + qmax = 2 ** (q_bit - 1) - 1 + qmin = -2 ** (q_bit - 1) + layer.module.output_qmax = qmax + layer.module.output_qmin = qmin + + self.optimizer.add_param_group({"params": layer.module.output_scale}) + + if "input" in config.get("quant_types", []): + # scale of input will be initialized using the first batch data + layer.module.register_parameter("input_scale", torch.nn.Parameter(torch.Tensor([1.0]))) + q_bit = get_bits_length(config, "input") + layer.module.register_buffer('input_bit', torch.Tensor([q_bit])) + qmax = 2 ** (q_bit - 1) - 1 + qmin = -2 ** (q_bit - 1) + layer.module.input_qmax = qmax + layer.module.input_qmin = qmin + + self.optimizer.add_param_group({"params": layer.module.input_scale}) + + self.bound_model.to(device) + + @staticmethod + def grad_scale(x, scale): + """ + Used to scale the gradient. Give tensor `x`, we have `y=grad_scale(x, scale)=x` in the forward pass, + which means that this function will not change the value of `x`. In the backward pass, we have: + + :math:`\frac{\alpha_L}{\alpha_x}=\frac{\alpha_L}{\alpha_y}*\frac{\alpha_y}{\alpha_x}=sclae*\frac{\alpha_L}{\alpha_x}` + + This means that the origin gradient of x is scaled by a factor of `scale`. Applying this function + to a nn.Parameter will scale the gradient of it without changing its value. + """ + y = x + y_grad = x * scale + return (y - y_grad).detach() + y_grad + + @staticmethod + def round_pass(x): + """ + A simple way to achieve STE operation. + """ + y = x.round() + y_grad = x + return (y - y_grad).detach() + y_grad + + def quantize(self, x, scale, qmin, qmax): + grad_scale_factor = 1.0 / ((qmax * x.numel()) ** 0.5) + scale = self.grad_scale(scale, grad_scale_factor) + x = x / scale + x = torch.clamp(x, qmin, qmax) + x = self.round_pass(x) + x = x * scale + return x + + def quantize_weight(self, wrapper, **kwargs): + module = wrapper.module + + # todo: add support for quantize bias. If we use TensorRT as backend, there is no need to quantize + # bias + old_weight = module.old_weight + weight = self.quantize(old_weight, module.weight_scale, module.weight_qmin, module.weight_qmax) + module.weight = weight + return weight + + def quantize_output(self, output, wrapper, **kwargs): + module = wrapper.module + + # initialize the scale + if self.bound_model.steps == 1: + qmax = module.output_qmax + init_oup_scale = output.data.detach().abs().mean() * 2 / (qmax ** 0.5) + module.output_scale.data = init_oup_scale + + output = self.quantize(output, module.output_scale, module.output_qmin, module.output_qmax) + return output + + def quantize_input(self, *inputs, wrapper, **kwargs): + # This is hacky since it is not recommended to modify a tuple + # NB: support layers with multi inputs + module = wrapper.module + # initialize the scale + if self.bound_model.steps == 1: + qmax = module.input_qmax + init_oup_scale = inputs[0].data.detach().abs().mean() * 2 / (qmax ** 0.5) + module.input_scale.data = init_oup_scale + + new_input = self.quantize(inputs[0], module.input_scale, module.input_qmin, module.input_qmax) + list_inp = list(inputs) + list_inp[0] = new_input + return tuple(list_inp) + + def export_model(self, model_path, calibration_path=None, onnx_path=None, input_shape=None, device=None): + """ + Export quantized model weights and calibration parameters(optional) + + Parameters + ---------- + model_path : str + path to save quantized model weight + calibration_path : str + (optional) path to save quantize parameters after calibration + onnx_path : str + (optional) path to save onnx model + input_shape : list or tuple + input shape to onnx model + device : torch.device + device of the model, used to place the dummy input tensor for exporting onnx file. + the tensor is placed on cpu if ```device``` is None + + Returns + ------- + Dict + """ + assert model_path is not None, 'model_path must be specified' + self._unwrap_model() + calibration_config = {} + + for name, module in self.bound_model.named_modules(): + if hasattr(module, 'input_bit') or hasattr(module, 'output_bit'): + calibration_config[name] = {} + if hasattr(module, 'weight_bit'): + calibration_config[name]['weight_bit'] = int(module.weight_bit) + abs_max_input = float(module.input_scale * module.input_qmax) + calibration_config[name]['tracked_min_input'] = -abs_max_input + calibration_config[name]['tracked_max_input'] = abs_max_input + if hasattr(module, 'output_bit'): + calibration_config[name]['activation_bit'] = int(module.output_bit) + abs_max_output = float(module.output_scale * module.output_qmax) + calibration_config[name]['tracked_min_activation'] = -abs_max_output + calibration_config[name]['tracked_max_activation'] = abs_max_output + self._del_simulated_attr(module) + + self.export_model_save(self.bound_model, model_path, calibration_config, calibration_path, onnx_path, + input_shape, device) + + return calibration_config + + def _del_simulated_attr(self, module): + """ + delete redundant parameters in quantize module + """ + del_attr_list = ['old_weight', 'tracked_min_input', 'tracked_max_input', 'tracked_min_activation', \ + 'tracked_max_activation', 'output_scale', 'input_scale', 'weight_scale','weight_bit', 'output_bit', 'input_bit'] + for attr in del_attr_list: + if hasattr(module, attr): + delattr(module, attr) + + def step_with_optimizer(self): + """ + override `compressor` `step` method, quantization only happens after certain number of steps + """ + self.bound_model.steps += 1 diff --git a/utils/third_party/nni_new/algorithms/compression/tensorflow/pruning/__init__.py b/utils/third_party/nni_new/algorithms/compression/tensorflow/pruning/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..c535fd75123f8d37fba1084a35cb8615db426175 --- /dev/null +++ b/utils/third_party/nni_new/algorithms/compression/tensorflow/pruning/__init__.py @@ -0,0 +1 @@ +from .one_shot_pruner import * diff --git a/utils/third_party/nni_new/algorithms/compression/tensorflow/pruning/one_shot_pruner.py b/utils/third_party/nni_new/algorithms/compression/tensorflow/pruning/one_shot_pruner.py new file mode 100644 index 0000000000000000000000000000000000000000..2c1e1e3e0fa15884aa449246bc1a266c977513c6 --- /dev/null +++ b/utils/third_party/nni_new/algorithms/compression/tensorflow/pruning/one_shot_pruner.py @@ -0,0 +1,110 @@ +import tensorflow as tf + +from nni.compression.tensorflow import Pruner + +__all__ = [ + 'LevelPruner', + 'SlimPruner', +] + +class OneshotPruner(Pruner): + def __init__(self, model, config_list, masker_class, **algo_kwargs): + super().__init__(model, config_list) + self.set_wrappers_attribute('calculated', False) + self.masker = masker_class(model, self, **algo_kwargs) + + def validate_config(self, model, config_list): + pass # TODO + + def calc_masks(self, wrapper, wrapper_idx=None): + if wrapper.calculated: + return None + sparsity = wrapper.config['sparsity'] + masks = self.masker.calc_masks(sparsity, wrapper, wrapper_idx) + if masks is not None: + wrapper.calculated = True + return masks + + +class LevelPruner(OneshotPruner): + def __init__(self, model, config_list): + super().__init__(model, config_list, LevelPrunerMasker) + + +class SlimPruner(OneshotPruner): + def __init__(self, model, config_list): + super().__init__(model, config_list, SlimPrunerMasker) + + +class WeightMasker: + def __init__(self, model, pruner, **kwargs): + self.model = model + self.pruner = pruner + + def calc_masks(self, sparsity, wrapper, wrapper_idx=None): + raise NotImplementedError() + + +class LevelPrunerMasker(WeightMasker): + def calc_masks(self, sparsity, wrapper, wrapper_idx=None): + masks = {} + for weight_variable in wrapper.layer.weights: + if 'bias' in weight_variable.name: + continue + + num_prune = int(tf.size(weight_variable).numpy() * sparsity) + if num_prune == 0: + continue + + weight = weight_variable.read_value() + if wrapper.masks.get(weight_variable.name) is not None: + weight = tf.math.multiply(weight, wrapper.masks[weight_variable.name]) + + w_abs = tf.math.abs(weight) + k = tf.size(weight) - num_prune + topk = tf.math.top_k(tf.reshape(w_abs, [-1]), k).values + if tf.size(topk) == 0: + mask = tf.zeros_like(weight) + else: + mask = tf.math.greater_equal(w_abs, topk[-1]) + masks[weight_variable.name] = tf.cast(mask, weight.dtype) + return masks + +class SlimPrunerMasker(WeightMasker): + def __init__(self, model, pruner, **kwargs): + super().__init__(model, pruner) + weight_list = [] + for wrapper in pruner.wrappers: + weights = [w for w in wrapper.layer.weights if '/gamma:' in w.name] + assert len(weights) == 1, f'Bad weights: {[w.name for w in wrapper.layer.weights]}' + weight_list.append(tf.math.abs(weights[0].read_value())) + all_bn_weights = tf.concat(weight_list, 0) + k = int(all_bn_weights.shape[0] * pruner.wrappers[0].config['sparsity']) + top_k = -tf.math.top_k(-tf.reshape(all_bn_weights, [-1]), k).values + self.global_threshold = top_k.numpy()[-1] + + def calc_masks(self, sparsity, wrapper, wrapper_idx=None): + assert isinstance(wrapper.layer, tf.keras.layers.BatchNormalization), \ + 'SlimPruner only supports 2D batch normalization layer pruning' + + weight = None + weight_name = None + bias_name = None + + for variable in wrapper.layer.weights: + if '/gamma:' in variable.name: + weight = variable.read_value() + weight_name = variable.name + elif '/beta:' in variable.name: + bias_name = variable.name + + assert weight is not None + if wrapper.masks.get(weight_name) is not None: + weight *= wrapper.masks[weight_name] + + mask = tf.cast(tf.math.abs(weight) > self.global_threshold, weight.dtype) + + masks = {weight_name: mask} + if bias_name: + masks[bias_name] = mask + return masks diff --git a/utils/third_party/nni_new/algorithms/feature_engineering/gbdt_selector/__init__.py b/utils/third_party/nni_new/algorithms/feature_engineering/gbdt_selector/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..72970ab8565e11e62bae1242a4a8a00c06993cf7 --- /dev/null +++ b/utils/third_party/nni_new/algorithms/feature_engineering/gbdt_selector/__init__.py @@ -0,0 +1 @@ +from .gbdt_selector import GBDTSelector \ No newline at end of file diff --git a/utils/third_party/nni_new/algorithms/feature_engineering/gbdt_selector/gbdt_selector.py b/utils/third_party/nni_new/algorithms/feature_engineering/gbdt_selector/gbdt_selector.py new file mode 100644 index 0000000000000000000000000000000000000000..9ee09c25a38f1dca27fb4ab068bbceeaec69a3bd --- /dev/null +++ b/utils/third_party/nni_new/algorithms/feature_engineering/gbdt_selector/gbdt_selector.py @@ -0,0 +1,115 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# MIT License +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and +# associated documentation files (the "Software"), to deal in the Software without restriction, +# including without limitation the rights to use, copy, modify, merge, publish, distribute, +# sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or +# substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT +# NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT +# OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# ================================================================================================== + +""" +gbdt_selector.py including: + class GBDTSelector +""" + +import random +from sklearn.model_selection import train_test_split + +# pylint: disable=E0401 +import lightgbm as lgb + +from nni.feature_engineering.feature_selector import FeatureSelector + + +class GBDTSelector(FeatureSelector): + + def __init__(self, **kwargs): + self.selected_features_ = None + self.X = None + self.y = None + self.feature_importance = None + self.lgb_params = None + self.eval_ratio = None + self.early_stopping_rounds = None + self.importance_type = None + self.num_boost_round = None + self.model = None + + + def fit(self, X, y, **kwargs): + """ + Fit the training data to FeatureSelector + + Paramters + --------- + X : array-like numpy matrix + The training input samples, which shape is [n_samples, n_features]. + y : array-like numpy matrix + The target values (class labels in classification, real numbers in + regression). Which shape is [n_samples]. + lgb_params : dict + Parameters of lightgbm + eval_ratio : float + The ratio of data size. It's used for split the eval data and train data from self.X. + early_stopping_rounds : int + The early stopping setting in lightgbm. + importance_type : str + Supporting type is 'gain' or 'split'. + num_boost_round : int + num_boost_round in lightgbm. + """ + assert kwargs['lgb_params'] + assert kwargs['eval_ratio'] + assert kwargs['early_stopping_rounds'] + assert kwargs['importance_type'] + assert kwargs['num_boost_round'] + + self.X = X + self.y = y + self.lgb_params = kwargs['lgb_params'] + self.eval_ratio = kwargs['eval_ratio'] + self.early_stopping_rounds = kwargs['early_stopping_rounds'] + self.importance_type = kwargs['importance_type'] + self.num_boost_round = kwargs['num_boost_round'] + + X_train, X_test, y_train, y_test = train_test_split(self.X, + self.y, + test_size=self.eval_ratio, + random_state=random.seed(41)) + lgb_train = lgb.Dataset(X_train, y_train) + lgb_eval = lgb.Dataset(X_test, y_test, reference=lgb_train) + + self.model = lgb.train(self.lgb_params, + lgb_train, + num_boost_round=self.num_boost_round, + valid_sets=lgb_eval, + early_stopping_rounds=self.early_stopping_rounds) + + self.feature_importance = self.model.feature_importance(self.importance_type) + + + def get_selected_features(self, topk): + """ + Fit the training data to FeatureSelector + + Returns + ------- + list : + Return the index of imprtant feature. + """ + assert topk > 0 + + self.selected_features_ = self.feature_importance.argsort()[-topk:][::-1] + + return self.selected_features_ diff --git a/utils/third_party/nni_new/algorithms/feature_engineering/gbdt_selector/requirements.txt b/utils/third_party/nni_new/algorithms/feature_engineering/gbdt_selector/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..b73c05e89d156249906203b67c2e9f786d613738 --- /dev/null +++ b/utils/third_party/nni_new/algorithms/feature_engineering/gbdt_selector/requirements.txt @@ -0,0 +1 @@ +lightgbm \ No newline at end of file diff --git a/utils/third_party/nni_new/algorithms/feature_engineering/gradient_selector/__init__.py b/utils/third_party/nni_new/algorithms/feature_engineering/gradient_selector/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a43cb7578dcb38cbccc33fa11ab6bafc0a71b1fd --- /dev/null +++ b/utils/third_party/nni_new/algorithms/feature_engineering/gradient_selector/__init__.py @@ -0,0 +1 @@ +from .gradient_selector import FeatureGradientSelector \ No newline at end of file diff --git a/utils/third_party/nni_new/algorithms/feature_engineering/gradient_selector/constants.py b/utils/third_party/nni_new/algorithms/feature_engineering/gradient_selector/constants.py new file mode 100644 index 0000000000000000000000000000000000000000..0f70e2043af51bcf35c7514a81889203a9017ccb --- /dev/null +++ b/utils/third_party/nni_new/algorithms/feature_engineering/gradient_selector/constants.py @@ -0,0 +1,100 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# MIT License +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and +# associated documentation files (the "Software"), to deal in the Software without restriction, +# including without limitation the rights to use, copy, modify, merge, publish, distribute, +# sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or +# substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT +# NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT +# OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# ================================================================================================== + + +import numpy as np + + +class StorageLevel: + DISK = 'disk' + SPARSE = 'sparse' + DENSE = 'dense' + + +class DataFormat: + SVM = 'svm' + NUMPY = 'numpy' + ALL_FORMATS = [SVM, NUMPY] + + +class Preprocess: + """ + center the data to mean 0 and create unit variance + center the data to mean 0 + """ + ZSCORE = 'zscore' + CENTER = 'center' + + +class Device: + CUDA = 'cuda' + CPU = 'cpu' + + +class Checkpoint: + MODEL = 'model_state_dict' + OPT = 'optimizer_state_dict' + RNG = 'torch_rng_state' + + +class NanError(ValueError): + pass + + +class Initialization: + ZERO = 'zero' + ON = 'on' + OFF = 'off' + ON_HIGH = 'onhigh' + OFF_HIGH = 'offhigh' + SKLEARN = 'sklearn' + RANDOM = 'random' + VALUE_DICT = {ZERO: 0, + ON: 1, + OFF: -1, + ON_HIGH: 5, + OFF_HIGH: -1, + SKLEARN: None, + RANDOM: None} + + +class Coefficients: + """" + coefficients for sublinear estimator were computed running the sublinear + paper's authors' code + """ + SLE = {1: np.array([0.60355337]), + 2: np.array([1.52705001, -0.34841729]), + 3: np.array([2.90254224, -1.87216745, 0.]), + 4: np.array([4.63445685, -5.19936195, 0., 1.50391676]), + 5: np.array([6.92948049, -14.12216211, 9.4475009, 0., -1.21093546]), + 6: np.array([9.54431082, -28.09414643, 31.84703652, -11.18763791, -1.14175281, 0.]), + 7: np.array([12.54505041, -49.64891525, 79.78828031, -46.72250909, 0., 0., 5.02973646]), + 8: np.array([16.03550163, -84.286182, 196.86078756, -215.36747071, 92.63961263, 0., 0., -4.86280869]), + 9: np.array([19.86409184, -130.76801006, 390.95349861, -570.09210416, 354.77764899, 0., -73.84234865, 0., 10.09148767]), + 10: np.array([2.41117752e+01, -1.94946061e+02, 7.34214614e+02, -1.42851995e+03, 1.41567410e+03, \ + -5.81738134e+02, 0., 0., 3.11664751e+01, 1.05018365e+00]), + 11: np.array([28.75280839, -279.22576729, 1280.46325445, -3104.47148101, 3990.6092248, -2300.29413333, \ + 0., 427.35289033, 0., 0., -42.17587475]), + 12: np.array([33.85141912, -391.4229382, 2184.97827882, -6716.28280208, 11879.75233977, -11739.97267239, \ + 5384.94542245, 0., -674.23291712, 0., 0., 39.37456439])} + + +EPSILON = 1e-8 diff --git a/utils/third_party/nni_new/algorithms/feature_engineering/gradient_selector/fginitialize.py b/utils/third_party/nni_new/algorithms/feature_engineering/gradient_selector/fginitialize.py new file mode 100644 index 0000000000000000000000000000000000000000..6fe28ea5ee5494bc63a1dede2867ed3d2ec9f9a6 --- /dev/null +++ b/utils/third_party/nni_new/algorithms/feature_engineering/gradient_selector/fginitialize.py @@ -0,0 +1,611 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# MIT License +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and +# associated documentation files (the "Software"), to deal in the Software without restriction, +# including without limitation the rights to use, copy, modify, merge, publish, distribute, +# sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or +# substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT +# NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT +# OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# ================================================================================================== + + +import os +import pickle +import sys +import time + +import numpy as np +import scipy.sparse +from sklearn.datasets import load_svmlight_file + +import torch +from torch.utils.data import DataLoader, Dataset +# pylint: disable=E0611 +from torch.utils.data.dataloader import _SingleProcessDataLoaderIter, _MultiProcessingDataLoaderIter, _utils + +from . import constants +from . import syssettings + +torch.set_default_tensor_type(syssettings.torch.tensortype) +sparsetensor = syssettings.torch.sparse.tensortype + +BYTESPERREAL = 8. +BYTESPERGB = 1024. ** 3 + + +class PrepareData(Dataset): + + def __init__(self, + path_data=None, + data_format=constants.DataFormat.NUMPY, + D=None, N=None, + classification=True, + ordinal=False, + balanced=True, + preprocess=None, + n_to_estimate=None, + MAXMEMGB=syssettings.MAXMEMGB, + set_params=True, + path_mappings=None, + X=None, + y=None, + verbose=0, + n_classes=None, + device=constants.Device.CPU): + """ + Dataset class with helpful features and functions for being included in a dataloader + and managing memory usage. + can read following formats: + svm: svm light format (sklearn.datasets.load_svmlight_file) + numpy: Pass X and y as numpy or sparse arrays + + assumes + 1. if classification, y is in {-1, 1} or continuous and 0 indexed + 2. y can fit into memory + 3. consecutive calls to __getitem__() have consecutive idx values + + notes: + 1. this implementation is not careful wrt/ precise memory reqts. for + example, being able to store one dense row in memory is necessary, + but not sufficient. + 2. for y with 4.2 billion elements, 31.3 GB of memory is necessary + @ 8 bytes/scalar. Use partial fit to avoid loading the entire dataset + at once + 3. disk_size always refer to size of complete data file, even after + a split(). + + + Parameters + ---------- + path_data : str + Path to load data from + data_format : str + File ending for path data. + "numpy" is the default when passing in X and y + D : int + Number of features. + N : int + Number of rows. + classification : bool + If True, problem is classification, else regression. + ordinal: bool + If True, problem is ordinal classification. Requires classification to be True. + balanced : bool + If true, each class is weighted equally in optimization, otherwise + weighted is done via support of each class. Requires classification to be True. + prerocess : str + 'zscore' which refers to centering and normalizing data to unit variance or + 'center' which only centers the data to 0 mean + n_to_estimate : int + Number of rows of data to estimate + MAXMEMGB : float + Maximum allowable size for a minibatch + set_params : bool + Whether or not to determine the statistics of the dataset + path_mappings : str + Used when streaming from disk + X : array-like + Shape = [n_samples, n_features] + The training input samples. + y : array-like + Shape = [n_samples] + The target values (class labels in classification, real numbers in + regression). + verbose : int + Controls the verbosity when fitting. Set to 0 for no printing + 1 or higher for printing every verbose number of gradient steps. + device : str + 'cpu' to run on CPU and 'cuda' to run on GPU. Runs much faster on GPU + n_classes : int + number of classes + """ + + self.path_data = path_data + if self.path_data: + self.disk_size = os.path.getsize(path_data) + else: + assert X is not None, 'X must be specified if no path data' + self.disk_size = X.nbytes if not scipy.sparse.issparse( + X) else X.data.nbytes + assert data_format in constants.DataFormat.ALL_FORMATS, 'Format must in {0}.'.format( + ", ".join(constants.DataFormat.ALL_FORMATS)) + self.format = data_format + self.classification = classification + self.ordinal = ordinal + self.balanced = balanced + self.MAXMEMGB = MAXMEMGB + self.preprocess = preprocess + self.set_params = set_params + self.verbose = verbose + self.n_classes = n_classes + self.device = device + + self.path_data_stats = None + + if D is None: + assert self.disk_size / BYTESPERGB <= self.MAXMEMGB, \ + 'Cannot load data into memory. Supply D.' + + if self.format == constants.DataFormat.SVM: + self.X, self.y = load_svmlight_file(path_data) + elif self.format == constants.DataFormat.NUMPY: + assert X is not None, 'X must be specified in numpy mode' + assert y is not None, 'y must be specified in numpy mode' + self.X = X + self.y = y + if self.n_classes is None: + self.n_classes = np.unique(y).shape[0] + elif self.classification: + assert self.n_classes >= np.unique(y).shape[0], \ + 'n_classes given must be greater than or equal to the number of classes in y' + else: + raise NotImplementedError + self.y = torch.as_tensor(self.y, dtype=torch.get_default_dtype()) + + self.N, self.D = self.X.shape + + # assumes X was returned as a sparse array + self.storage_level = (constants.StorageLevel.SPARSE + if scipy.sparse.issparse(self.X) + else constants.StorageLevel.DENSE) + + else: + assert N is not None, 'Supply N.' + self.N, self.D = N, D + + # assume sparse matrix cannot fit into memory + self.storage_level = constants.StorageLevel.DISK + + self.dense_size_gb = self.get_dense_size() + + # check dense size + self.set_dense_X() + + self.max_rows = int(self.MAXMEMGB * BYTESPERGB / BYTESPERREAL / self.D) + assert self.max_rows, \ + 'Cannot fit one dense row into %d GB memory.' % self.MAXMEMGB + self.max_rows = self.max_batch_size() + sys.stdout.flush() + + if n_to_estimate is None: + self.n_to_estimate = self.max_batch_size() + else: + assert n_to_estimate <= self.N, 'n_to_estimate must be <= N.' + self.n_to_estimate = n_to_estimate + + # initialize disk loader + if self.storage_level == constants.StorageLevel.DISK and self.set_params: + if self.format == constants.DataFormat.SVM: + raise NotImplementedError( + 'Please use partial fit to train on datasets that do not fit in memory') + else: + raise NotImplementedError + + # TODO: use a passed-in RNG here + self.ix_statistics = np.random.permutation(self.N)[:self.n_to_estimate] + self.n_features = self.D + if self.set_params: + if self.verbose: + print('Finding data statistics...', end='') + sys.stdout.flush() + Xmn, sv1, Xsd, ymn, ysd = self.compute_data_stats() + self.set_data_stats(Xmn, sv1, Xsd, ymn, ysd) + if self.verbose: + print() + self.set_return_raw(False) + else: + self.set_return_raw(True) + + self.set_return_np(False) + + # this needs to occur after setting preprocessing params + if (self.storage_level == constants.StorageLevel.DISK and + self.format == constants.DataFormat.SVM and self.set_params): + self.loader.batchsize = 1 + + def get_dense_size(self): + return self.N * self.D * BYTESPERREAL / BYTESPERGB + + def set_dense_X(self): + if self.storage_level != constants.StorageLevel.DISK: + if self.dense_size_gb <= self.MAXMEMGB: + if self.storage_level == constants.StorageLevel.SPARSE: + self.X = self.X.toarray() + self.X = torch.as_tensor( + self.X, dtype=torch.get_default_dtype()) + self.storage_level = constants.StorageLevel.DENSE + + def set_return_np(self, boolean): + + self.return_np = boolean + + def set_return_raw(self, boolean): + + self.return_raw = boolean + + def save_data_stats(self, path_data_stats): + """ + Dumps dataset statistics to pickle file. + """ + + data_stats = { + 'Xmn': self.Xmn, + 'sv1': self.sv1, + 'Xsd': self.Xsd, + 'ymn': self.ymn, + 'ysd': self.ysd, + 'ix_statistics': self.ix_statistics, + } + pickle.dump(data_stats, open(path_data_stats, 'wb')) + + def load_data_stats(self, path_data_stats): + + stats = pickle.load(open(path_data_stats, 'rb')) + self.path_data_stats = path_data_stats + + self.set_data_stats(np.asarray(stats['Xmn']), stats['sv1'], + stats['Xsd'], stats['ymn'], stats['ysd']) + + if self.storage_level == constants.StorageLevel.DISK and hasattr( + self, 'path_mappings'): + if 'ix_statistics' in stats: + self.ix_statistics = stats['ix_statistics'] + else: + self.ix_statistics = range(self.N) + + self.set_return_raw(False) + + def reset(self): + """ + Resets the dataloader. Only implemented for disk StorageLevel. + """ + + if self.storage_level == constants.StorageLevel.DENSE: + pass + elif self.storage_level == constants.StorageLevel.SPARSE: + pass + elif self.storage_level == constants.StorageLevel.DISK: + if self.format == constants.DataFormat.SVM: + self.loader.reset() + else: + raise NotImplementedError + + def todense(self): + + assert hasattr(self, 'Xmn'), 'Set preprocess params first.' + assert len(self) <= self.max_batch_size( + ), 'N must be <= max_batch_size().' + + with torch.no_grad(): + dense, _ = self.split(range(len(self))) + Braw = self.return_raw + Bnp = self.return_np + self.set_return_raw(True) + self.set_return_np(True) + dense.X, dense.y = [], [] + + def f_Xy(X, y): + dense.X.append(X) + dense.y.append(y) + self.apply(f_Xy=f_Xy) + dense.X = dense.X[-1] + dense.y = dense.y[-1] + self.set_return_raw(Braw) + self.set_return_np(Bnp) + dense.storage_level = constants.StorageLevel.DENSE + + return dense + + def split(self, ix): + + assert hasattr(self, 'Xmn'), 'Run set_preprocess_params() first.' + + first = type(self)( + self.path_data, + self.format, + self.D, + N=len(ix), + classification=self.classification, + preprocess=self.preprocess, + n_to_estimate=None, + MAXMEMGB=self.MAXMEMGB, + set_params=False) + second = type(self)( + self.path_data, + self.format, + self.D, + N=self.N - len(ix), + classification=self.classification, + preprocess=self.preprocess, + n_to_estimate=None, + MAXMEMGB=self.MAXMEMGB, + set_params=False) + + first.storage_level = self.storage_level + second.storage_level = self.storage_level + + # copy preprocess params + if not self.classification: + first.ymn = self.ymn + second.ymn = self.ymn + first.ysd = self.ysd + second.ysd = self.ysd + + first.Xmn = self.Xmn + second.Xmn = self.Xmn + first.sv1 = self.sv1 + second.sv1 = self.sv1 + + if self.storage_level == constants.StorageLevel.DISK: + if self.format == constants.DataFormat.SVM: + first.Xsd = self.Xsd + second.Xsd = self.Xsd + else: + raise NotImplementedError + + # initialize data structures + if self.storage_level == constants.StorageLevel.DISK: + if self.format == constants.DataFormat.SVM: + raise NotImplementedError + raise NotImplementedError + elif self.storage_level in [constants.StorageLevel.SPARSE, + constants.StorageLevel.DENSE]: + first.X, first.y = self.X[ix], self.y[ix] + ixsec = list(set(range(self.N)).difference(set(ix))) + second.X, second.y = self.X[ixsec], self.y[ixsec] + + return first, second + + @staticmethod + def sparse_std(X, X_mean): + """ + Calculate the column wise standard deviations of a sparse matrix. + """ + X_copy = X.copy() + X_copy.data **= 2 # square non zero elements + E_x_squared = np.array(X_copy.mean(axis=0)).ravel() + Xsd = np.sqrt(E_x_squared - X_mean**2) + return Xsd + + def compute_data_stats(self): + """ + 1. computes/estimates feature means + 2. if preprocess == 'zscore', computes/estimates feature standard devs + 3. if not classification, computes/estimates target mean/standard dev + 4. estimates largest singular value of data matrix + """ + t = time.time() + X, y = self.X[self.ix_statistics], self.y[self.ix_statistics] + preprocess = self.preprocess + classification = self.classification + + Xmn = (X.mean(dim=0) + if not scipy.sparse.issparse(X) + else np.array(X.mean(axis=0)).ravel()) + + if preprocess == constants.Preprocess.ZSCORE: + Xsd = (X.std(dim=0) + if not scipy.sparse.issparse(X) + else PrepareData.sparse_std(X, Xmn)) + Xsd[Xsd == 0] = 1. + else: + Xsd = 1. + + if preprocess is not None and preprocess: + if preprocess == constants.Preprocess.ZSCORE: + Xc = (X - Xmn) / Xsd + else: + Xc = X - Xmn + else: + Xc = X - Xmn + + sv1 = scipy.sparse.linalg.svds(Xc / ( + torch.sqrt(torch.prod(torch.as_tensor(y.size(), dtype=torch.get_default_dtype()))) + if not scipy.sparse.issparse(X) else y.numpy().size), + k=1, + which='LM', + return_singular_vectors=False) + # avoid runaway sv1 + sv1 = np.array([min(np.finfo(np.float32).max, + sv1[0])]) + + if not classification: + ymn = y.mean() + ysd = y.std() + else: + # TODO: set these, for each class? + ymn = 0. + ysd = 1. + if self.verbose: + print(" computing data statistics took: ", time.time() - t) + + return Xmn, sv1, Xsd, ymn, ysd + + + def set_data_stats(self, Xmn, sv1, Xsd=1., ymn=0., ysd=1.): + """ + Saves dataset stats to self to be used for preprocessing. + """ + + self.Xmn = torch.as_tensor( + Xmn, dtype=torch.get_default_dtype()).to(self.device) + self.sv1 = torch.as_tensor( + sv1, dtype=torch.get_default_dtype()).to(self.device) + self.Xsd = torch.as_tensor( + Xsd, dtype=torch.get_default_dtype()).to(self.device) + self.ymn = torch.as_tensor( + ymn, dtype=torch.get_default_dtype()).to(self.device) + self.ysd = torch.as_tensor( + ysd, dtype=torch.get_default_dtype()).to(self.device) + + + def apply_preprocess(self, X, y): + """ + Faster on gpu device, while dataloading takes up a large portion of the time. + """ + + with torch.no_grad(): + if not self.classification: + y = (y.reshape((-1, 1)) - self.ymn) / self.ysd + else: + y = y.reshape((-1, 1)) + X = (X - self.Xmn) / self.sv1 + + if self.preprocess == constants.Preprocess.ZSCORE: + X /= self.Xsd + + return X, y + + + def max_batch_size(self): + """ + Return the maximum batchsize for the dataset. + """ + + return int(np.min([self.max_rows, self.N])) + + + def apply(self, ix_rows=None, ix_cols=None, f_Xy=None): + + if f_Xy is None: + return + + if ix_rows is None: + ix_rows = range(self.N) + + if ix_cols is None: + ix_cols = range(self.n_features) + + f_Xy((self.X[ix_rows, ix_cols] + if not self.storage_level == constants.StorageLevel.SPARSE + else self.X[ix_rows, ix_cols].toarray()), self.y[ix_rows]) + + + def get_dense_data(self, ix_cols=None, ix_rows=None): + + if ix_cols is None: + ix_cols = range(self.n_features) + + X = [np.zeros((0, len(ix_cols)))] + y = [np.zeros((0, 1))] + Bnp = self.return_np + + def f_Xy(Xb, yb, n): + X[-1] = np.concatenate((X[-1], Xb), axis=0) + y[-1] = np.concatenate((y[-1], yb), axis=0) + self.apply(f_Xy=f_Xy, ix_rows=ix_rows, ix_cols=ix_cols) + self.set_return_np(Bnp) + + return X[-1], y[-1] + + + def __len__(self): + + return self.N + + + def getXy(self, idx): + + if self.storage_level == constants.StorageLevel.DENSE: + X, y = self.X[idx], self.y[idx] + elif self.storage_level == constants.StorageLevel.SPARSE: + # assume subset can fit into memory even if whole matrix cant + X, y = self.X[idx].toarray(), self.y[idx] + else: + raise NotImplementedError + + return X, y + + + def __getitem__(self, idx): + + with torch.no_grad(): + X, y = self.getXy(idx) + X = X.toarray() if scipy.sparse.issparse(X) else X + + X = torch.as_tensor( + X, dtype=torch.get_default_dtype()).to(self.device) + y = torch.as_tensor( + y, dtype=torch.get_default_dtype()).to(self.device) + + if not self.return_raw: + X, y = self.apply_preprocess(X, y) + + if self.classification and ( + self.n_classes is None or self.n_classes == 2): + y[y == 0] = -1 + + if self.return_np: + if constants.Device.CPU not in self.device: + X = X.cpu() + y = y.cpu() + X = X.numpy() + y = y.numpy() + return X, y + + return X, y + + +class ChunkDataLoader(DataLoader): + """ + DataLoader class used to more quickly load a batch of indices at once. + """ + + def __iter__(self): + return _ChunkDataLoaderIter(self) + + +class _ChunkDataLoaderIter: + """ + DataLoaderIter class used to more quickly load a batch of indices at once. + """ + def __init__(self, dataloader): + if dataloader.num_workers == 0: + self.iter = _SingleProcessDataLoaderIter(dataloader) + else: + self.iter = _MultiProcessingDataLoaderIter(dataloader) + + def __next__(self): + # only chunk that is edited from base + if self.iter._num_workers == 0: # same-process loading + indices = next(self.iter._sampler_iter) # may raise StopIteration + if len(indices) > 1: + batch = self.iter._dataset[np.array(indices)] + else: + batch = self.iter._collate_fn([self.iter._dataset[i] for i in indices]) + + if self.iter._pin_memory: + batch = _utils.pin_memory.pin_memory_batch(batch) + return batch + else: + return next(self.iter) diff --git a/utils/third_party/nni_new/algorithms/feature_engineering/gradient_selector/fgtrain.py b/utils/third_party/nni_new/algorithms/feature_engineering/gradient_selector/fgtrain.py new file mode 100644 index 0000000000000000000000000000000000000000..377d72691613b93911e2bbc164e3c677ac31f574 --- /dev/null +++ b/utils/third_party/nni_new/algorithms/feature_engineering/gradient_selector/fgtrain.py @@ -0,0 +1,228 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# MIT License +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and +# associated documentation files (the "Software"), to deal in the Software without restriction, +# including without limitation the rights to use, copy, modify, merge, publish, distribute, +# sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or +# substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT +# NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT +# OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# ================================================================================================== + + +import time + +import numpy as np +import torch +from sklearn.feature_selection import SelectKBest, \ + f_classif, mutual_info_classif, f_regression, mutual_info_regression + +from . import constants +from . import syssettings +from .learnability import Solver +from .utils import EMA + +torch.set_default_tensor_type(syssettings.torch.tensortype) + + +def get_optim_f_stop(maxiter, maxtime, dftol_stop, freltol_stop, + minibatch=True): + """ + Check stopping conditions. + """ + + discount_factor = 1. / 3 + + total_t = [0.] + df_store = [np.nan] + it_store = [0] + relchange_store = [np.nan] + f_ma = EMA(discount_factor=discount_factor) + df_ma = EMA(discount_factor=discount_factor) + + def f_stop(f0, v0, it, t): + + flag_stop = False + + total_t[-1] += t + g = f0.x.grad.clone().cpu().detach() + df = g.abs().max().numpy().squeeze() + v = v0.clone().cpu().detach() + f = v.numpy().squeeze() + + if it >= maxiter: + flag_stop = True + + elif total_t[-1] >= maxtime: + flag_stop = True + + f_ma.update(f) + df_ma.update(df) + rel_change = f_ma.relchange() + + if ((not minibatch) and (df < dftol_stop)) \ + or (minibatch and (df_ma() < dftol_stop)): + flag_stop = True + + if rel_change < freltol_stop: + flag_stop = True + + if not minibatch: + df_store[-1] = df + else: + df_store[-1] = df_ma() + relchange_store[-1] = rel_change + it_store[-1] = it + + return flag_stop + + return f_stop, {'t': total_t, 'it': it_store, 'df': df_store, + 'relchange': relchange_store} + + +def get_init(data_train, init_type='on', rng=np.random.RandomState(0), prev_score=None): + """ + Initialize the 'x' variable with different settings + """ + + D = data_train.n_features + value_off = constants.Initialization.VALUE_DICT[ + constants.Initialization.OFF] + value_on = constants.Initialization.VALUE_DICT[ + constants.Initialization.ON] + + if prev_score is not None: + x0 = prev_score + elif not isinstance(init_type, str): + x0 = value_off * np.ones(D) + x0[init_type] = value_on + elif init_type.startswith(constants.Initialization.RANDOM): + d = int(init_type.replace(constants.Initialization.RANDOM, '')) + x0 = value_off * np.ones(D) + x0[rng.permutation(D)[:d]] = value_on + elif init_type == constants.Initialization.SKLEARN: + B = data_train.return_raw + X, y = data_train.get_dense_data() + data_train.set_return_raw(B) + ix = train_sk_dense(init_type, X, y, data_train.classification) + x0 = value_off * np.ones(D) + x0[ix] = value_on + elif init_type in constants.Initialization.VALUE_DICT: + x0 = constants.Initialization.VALUE_DICT[init_type] * np.ones(D) + else: + raise NotImplementedError( + 'init_type {0} not supported yet'.format(init_type)) + # pylint: disable=E1102 + return torch.tensor(x0.reshape((-1, 1)), + dtype=torch.get_default_dtype()) + + +def get_checkpoint(S, stop_conds, rng=None, get_state=True): + """ + Save the necessary information into a dictionary + """ + + m = {} + m['ninitfeats'] = S.ninitfeats + m['x0'] = S.x0 + x = S.x.clone().cpu().detach() + m['feats'] = np.where(x.numpy() >= 0)[0] + m.update({k: v[0] for k, v in stop_conds.items()}) + if get_state: + m.update({constants.Checkpoint.MODEL: S.state_dict(), + constants.Checkpoint.OPT: S.opt_train.state_dict(), + constants.Checkpoint.RNG: torch.get_rng_state(), + }) + if rng: + m.update({'rng_state': rng.get_state()}) + + return m + + +def _train(data_train, Nminibatch, order, C, rng, lr_train, debug, maxiter, + maxtime, init, dftol_stop, freltol_stop, dn_log, accum_steps, + path_save, shuffle, device=constants.Device.CPU, + verbose=1, + prev_checkpoint=None, + groups=None, + soft_groups=None): + """ + Main training loop. + """ + + t_init = time.time() + + x0 = get_init(data_train, init, rng) + if isinstance(init, str) and init == constants.Initialization.ZERO: + ninitfeats = -1 + else: + ninitfeats = np.where(x0.detach().numpy() > 0)[0].size + + S = Solver(data_train, order, + Nminibatch=Nminibatch, x0=x0, C=C, + ftransform=lambda x: torch.sigmoid(2 * x), + get_train_opt=lambda p: torch.optim.Adam(p, lr_train), + rng=rng, + accum_steps=accum_steps, + shuffle=shuffle, + groups=groups, + soft_groups=soft_groups, + device=device, + verbose=verbose) + S = S.to(device) + + S.ninitfeats = ninitfeats + S.x0 = x0 + + if prev_checkpoint: + S.load_state_dict(prev_checkpoint[constants.Checkpoint.MODEL]) + S.opt_train.load_state_dict(prev_checkpoint[constants.Checkpoint.OPT]) + torch.set_rng_state(prev_checkpoint[constants.Checkpoint.RNG]) + + minibatch = S.Ntrain != S.Nminibatch + + f_stop, stop_conds = get_optim_f_stop(maxiter, maxtime, dftol_stop, + freltol_stop, minibatch=minibatch) + + if debug: + pass + else: + f_callback = None + stop_conds['t'][-1] = time.time() - t_init + + S.train(f_stop=f_stop, f_callback=f_callback) + + return get_checkpoint(S, stop_conds, rng), S + + +def train_sk_dense(ty, X, y, classification): + if classification: + if ty.startswith('skf'): + d = int(ty.replace('skf', '')) + f_sk = f_classif + elif ty.startswith('skmi'): + d = int(ty.replace('skmi', '')) + f_sk = mutual_info_classif + else: + if ty.startswith('skf'): + d = int(ty.replace('skf', '')) + f_sk = f_regression + elif ty.startswith('skmi'): + d = int(ty.replace('skmi', '')) + f_sk = mutual_info_regression + t = time.time() + clf = SelectKBest(f_sk, k=d) + clf.fit_transform(X, y.squeeze()) + ix = np.argsort(-clf.scores_) + ix = ix[np.where(np.invert(np.isnan(clf.scores_[ix])))[0]][:d] + t = time.time() - t + return {'feats': ix, 't': t} diff --git a/utils/third_party/nni_new/algorithms/feature_engineering/gradient_selector/gradient_selector.py b/utils/third_party/nni_new/algorithms/feature_engineering/gradient_selector/gradient_selector.py new file mode 100644 index 0000000000000000000000000000000000000000..f7cb69f627442cb8b749f792cecfa0fea7526e1b --- /dev/null +++ b/utils/third_party/nni_new/algorithms/feature_engineering/gradient_selector/gradient_selector.py @@ -0,0 +1,631 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# MIT License +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and +# associated documentation files (the "Software"), to deal in the Software without restriction, +# including without limitation the rights to use, copy, modify, merge, publish, distribute, +# sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or +# substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT +# NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT +# OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# ================================================================================================== + +import time + +import numpy as np +import pandas as pd + +from sklearn.base import BaseEstimator +from sklearn.feature_selection import SelectorMixin +from sklearn.utils.validation import check_is_fitted + +import torch + +from nni.feature_engineering.feature_selector import FeatureSelector +from . import constants +from .fginitialize import PrepareData +from .fgtrain import _train + + +class FeatureGradientSelector(FeatureSelector, BaseEstimator, SelectorMixin): + def __init__(self, + order=4, + penalty=1, + n_features=None, + max_features=None, + learning_rate=1e-1, + init='zero', + n_epochs=1, + shuffle=True, + batch_size=1000, + target_batch_size=1000, + max_time=np.inf, + classification=True, + ordinal=False, + balanced=True, + preprocess='zscore', + soft_grouping=False, + verbose=0, + device='cpu'): + """ + FeatureGradientSelector is a class that selects features for a machine + learning model using a gradient based search. + + Parameters + ---------- + order : int + What order of interactions to include. Higher orders + may be more accurate but increase the run time. 12 is the maximum allowed order. + penatly : int + Constant that multiplies the regularization term. + n_features: int + If None, will automatically choose number of features based on search. + Otherwise, number of top features to select. + max_features : int + If not None, will use the 'elbow method' to determine the number of features + with max_features as the upper limit. + learning_rate : float + init : str + How to initialize the vector of scores. 'zero' is the default. + Options: {'zero', 'on', 'off', 'onhigh', 'offhigh', 'sklearn'} + n_epochs : int + number of epochs to run + shuffle : bool + Shuffle "rows" prior to an epoch. + batch_size : int + Nnumber of "rows" to process at a time + target_batch_size : int + Number of "rows" to accumulate gradients over. + Useful when many rows will not fit into memory but are needed for accurate estimation. + classification : bool + If True, problem is classification, else regression. + ordinal : bool + If True, problem is ordinal classification. Requires classification to be True. + balanced : bool + If true, each class is weighted equally in optimization, otherwise + weighted is done via support of each class. Requires classification to be True. + prerocess : str + 'zscore' which refers to centering and normalizing data to unit variance or + 'center' which only centers the data to 0 mean + soft_grouping : bool + if True, groups represent features that come from the same source. + Used to encourage sparsity of groups and features within groups. + verbose : int + Controls the verbosity when fitting. Set to 0 for no printing + 1 or higher for printing every verbose number of gradient steps. + device : str + 'cpu' to run on CPU and 'cuda' to run on GPU. Runs much faster on GPU + """ + assert order <= 12 and order >= 1, 'order must be an integer between 1 and 12, inclusive' + assert n_features is None or max_features is None, \ + 'only specify one of n_features and max_features at a time' + + self.order = order + self.penalty = penalty + self.n_features = n_features + self.max_features = max_features + self.learning_rate = learning_rate + self.init = init + self.n_epochs = n_epochs + self.shuffle = shuffle + self.batch_size = batch_size + self.target_batch_size = target_batch_size + self.max_time = max_time + self.dftol_stop = -1 + self.freltol_stop = -1 + self.classification = classification + self.ordinal = ordinal + self.balanced = balanced + self.preprocess = preprocess + self.soft_grouping = soft_grouping + self.verbose = verbose + self.device = device + + self.model_ = None + self.scores_ = None + self._prev_checkpoint = None + self._data_train = None + + def partial_fit(self, X, y, + n_classes=None, + groups=None): + """ + Select Features via a gradient based search on (X, y) on the given samples. + Can be called repeatedly with different X and y to handle streaming datasets. + + Parameters + ---------- + X : array-like + Shape = [n_samples, n_features] + The training input samples. + y : array-like + Shape = [n_samples] + The target values (class labels in classification, real numbers in + regression). + n_classes : int + Number of classes + Classes across all calls to partial_fit. + Can be obtained by via `np.unique(y_all).shape[0]`, where y_all is the + target vector of the entire dataset. + This argument is expected for the first call to partial_fit, + otherwise will assume all classes are present in the batch of y given. + It will be ignored in the subsequent calls. + Note that y doesn't need to contain all labels in `classes`. + groups : array-like + Optional, shape = [n_features] + Groups of columns that must be selected as a unit + e.g. [0, 0, 1, 2] specifies the first two columns are part of a group. + This argument is expected for the first call to partial_fit, + otherwise will assume all classes are present in the batch of y given. + It will be ignored in the subsequent calls. + """ + try: + self._partial_fit(X, y, n_classes=n_classes, groups=groups) + except constants.NanError: + if hasattr(self, '_prev_checkpoint'): + # if it's already done some batches successfully just ignore it + print('failed fitting this batch, loss was nan') + else: + # if this is the first batch, reset and try with doubles + if self.verbose: + print('Loss was nan, trying with Doubles') + self._reset() + torch.set_default_tensor_type(torch.DoubleTensor) + self._partial_fit(X, y, n_classes=n_classes, groups=groups) + + return self + + def _partial_fit(self, X, y, n_classes=None, groups=None): + """ + Private function for partial_fit to enable trying floats before doubles. + """ + # pass in X and y in chunks + if hasattr(self, '_data_train'): + # just overwrite the X and y from the new chunk but make them tensors + # keep dataset stats from previous + self._data_train.X = X.values if isinstance(X, pd.DataFrame) else X + self._data_train.N, self._data_train.D = self._data_train.X.shape + self._data_train.dense_size_gb = self._data_train.get_dense_size() + self._data_train.set_dense_X() + + self._data_train.y = y.values if isinstance(y, pd.Series) else y + self._data_train.y = torch.as_tensor( + y, dtype=torch.get_default_dtype()) + else: + data_train = self._prepare_data(X, y, n_classes=n_classes) + self._data_train = data_train + + batch_size, _, accum_steps, max_iter = self._set_batch_size( + self._data_train) + + rng = None # not used + debug = 0 # {0,1} print messages and do other stuff? + dn_logs = None # tensorboard logs; only specify if debug=1 + path_save = None # intermediate models saves; only specify if debug=1 + m, solver = _train(self._data_train, + batch_size, + self.order, + self.penalty, + rng, + self.learning_rate, + debug, + max_iter, + self.max_time, + self.init, + self.dftol_stop, + self.freltol_stop, + dn_logs, + accum_steps, + path_save, + self.shuffle, + device=self.device, + verbose=self.verbose, + prev_checkpoint=self._prev_checkpoint if hasattr( + self, '_prev_checkpoint') else None, + groups=groups if not self.soft_grouping else None, + soft_groups=groups if self.soft_grouping else None) + + self._prev_checkpoint = m + self._process_results(m, solver, X, groups=groups) + return self + + def fit(self, X, y, + groups=None): + """ + Select Features via a gradient based search on (X, y). + + Parameters + ---------- + X : array-like + Shape = [n_samples, n_features] + The training input samples. + y : array-like + Shape = [n_samples] + The target values (class labels in classification, real numbers in + regression). + groups : array-like + Optional, shape = [n_features] + Groups of columns that must be selected as a unit + e.g. [0, 0, 1, 2] specifies the first two columns are part of a group. + """ + try: + self._fit(X, y, groups=groups) + except constants.NanError: + if self.verbose: + print('Loss was nan, trying with Doubles') + torch.set_default_tensor_type(torch.DoubleTensor) + self._fit(X, y, groups=groups) + return self + + def get_selected_features(self): + return self.selected_features_ + + def _prepare_data(self, X, y, n_classes=None): + """ + Returns a PrepareData object. + """ + return PrepareData(X=X.values if isinstance(X, pd.DataFrame) else X, + y=y.values if isinstance(y, pd.Series) else y, + data_format=constants.DataFormat.NUMPY, + classification=int(self.classification), + ordinal=self.ordinal, + balanced=self.balanced, + preprocess=self.preprocess, + verbose=self.verbose, + device=self.device, + n_classes=n_classes) + + def _fit(self, X, y, groups=None): + """ + Private function for fit to enable trying floats before doubles. + """ + data_train = self._prepare_data(X, y) + + batch_size, _, accum_steps, max_iter = self._set_batch_size( + data_train) + + rng = None # not used + debug = 0 # {0,1} print messages and log to tensorboard + dn_logs = None # tensorboard logs; only specify if debug=1 + path_save = None # intermediate models saves; only specify if debug=1 + m, solver = _train(data_train, + batch_size, + self.order, + self.penalty, + rng, + self.learning_rate, + debug, + max_iter, + self.max_time, + self.init, + self.dftol_stop, + self.freltol_stop, + dn_logs, + accum_steps, + path_save, + self.shuffle, + device=self.device, + verbose=self.verbose, + groups=groups if not self.soft_grouping else None, + soft_groups=groups if self.soft_grouping else None) + + self._process_results(m, solver, X, groups=groups) + return self + + def _process_torch_scores(self, scores): + """ + Convert scores into flat numpy arrays. + """ + if constants.Device.CUDA in scores.device.type: + scores = scores.cpu() + return scores.numpy().ravel() + + def _set_batch_size(self, data_train): + """ + Ensures that batch_size is less than the number of rows. + """ + batch_size = min(self.batch_size, data_train.N) + target_batch_size = min(max( + self.batch_size, self.target_batch_size), data_train.N) + accum_steps = max(int(np.ceil(target_batch_size / self.batch_size)), 1) + max_iter = self.n_epochs * (data_train.N // batch_size) + return batch_size, target_batch_size, accum_steps, max_iter + + def _process_results(self, m, solver, X, groups=None): + """ + Process the results of a run into something suitable for transform(). + """ + self.scores_ = self._process_torch_scores( + torch.sigmoid(m[constants.Checkpoint.MODEL]['x'] * 2)) + if self.max_features: + self.max_features = min([self.max_features, self.scores_.shape[0]]) + n_features = self._recommend_number_features(solver) + self.set_n_features(n_features, groups=groups) + elif self.n_features: + self.set_n_features(self.n_features, groups=groups) + else: + self.selected_features_ = m['feats'] + + # subtract elapsed time from max_time + self.max_time -= m['t'] + + self.model_ = m + + return self + + def transform(self, X): + """ + Returns selected features from X. + + Paramters + --------- + X: array-like + Shape = [n_samples, n_features] + The training input samples. + """ + + self._get_support_mask() + if self.selected_features_.shape[0] == 0: + raise ValueError( + 'No Features selected, consider lowering the penalty or specifying n_features') + return (X.iloc[:, self.selected_features_] + if isinstance(X, pd.DataFrame) + else X[:, self.selected_features_]) + + def get_support(self, indices=False): + """ + Get a mask, or integer index, of the features selected. + + Parameters + ---------- + indices : bool + Default False + If True, the return value will be an array of integers, rather than a boolean mask. + + Returns + ------- + list : + returns support: An index that selects the retained features from a feature vector. + If indices is False, this is a boolean array of shape [# input features], + in which an element is True iff its corresponding feature is selected for retention. + If indices is True, this is an integer array of shape [# output features] whose values + are indices into the input feature vector. + """ + self._get_support_mask() + if indices: + return self.selected_features_ + + mask = np.zeros_like(self.scores_, dtype=bool) + # pylint: disable=E1137 + mask[self.selected_features_] = True + return mask + + def inverse_transform(self, X): + """ + Returns transformed X to the original number of column. + This operation is lossy and all columns not in the transformed data + will be returned as columns of 0s. + """ + self._get_support_mask() + X_new = np.zeros((X.shape[0], self.scores_.shape[0])) + X_new[self.selected_features_] = X + return X_new + + def get_params(self, deep=True): + """ + Get parameters for this estimator. + """ + params = self.__dict__ + params = {key: val for (key, val) in params.items() + if not key.endswith('_')} + return params + + def set_params(self, **params): + """ + Set the parameters of this estimator. + """ + for param in params: + if hasattr(self, param): + setattr(self, param, params[param]) + return self + + def fit_transform(self, X, y): + """ + Select features and then return X with the selected features. + + Parameters + ---------- + X : array-like + Shape = [n_samples, n_features] + The training input samples. + y : array-like + Shape = [n_samples] + The target values (class labels in classification, real numbers in + regression). + """ + self.fit(X, y) + return self.transform(X) + + def _get_support_mask(self): + """ + Check if it is fitted. + """ + check_is_fitted(self, 'scores_') + + def _generate_scores(self, solver, xsub, ysub, step_size, feature_order): + """ + Generate forward passes to determine the number of features when max_features is set. + """ + scores = [] + for i in np.arange(1, self.max_features + 1, step_size): + # optimization possible since xsub is growing? + i = int(np.ceil(i)) + # pylint: disable=E1102 + score = solver.f_train(torch.tensor(np.ones(i), + dtype=torch.get_default_dtype() + ).unsqueeze(1).to(self.device), + xsub[:, feature_order[:i]], + ysub) + if constants.Device.CUDA in score.device.type: + score = score.cpu() + # score.numpy()[0][0] + scores.append(score) + return scores + + def set_n_features(self, n, groups=None): + """ + Set the number of features to return after fitting. + """ + self._get_support_mask() + self.n_features = n + return self._set_top_features(groups=groups) + + def _set_top_features(self, groups=None): + """ + Set the selected features after a run. + + With groups, ensures that if any member of a group is selected, all members are selected + """ + self._get_support_mask() + assert self.n_features <= self.scores_.shape[0], \ + 'n_features must be less than or equal to the number of columns in X' + # pylint: disable=E1130 + self.selected_features_ = np.argpartition( + self.scores_, -self.n_features)[-self.n_features:] + if groups is not None and not self.soft_grouping: + selected_feature_set = set(self.selected_features_.tolist()) + for _ in np.unique(groups): + group_members = np.where(groups == groups)[0].tolist() + if selected_feature_set.intersection(group_members): + selected_feature_set.update(group_members) + self.selected_features_ = np.array(list(selected_feature_set)) + self.selected_features_ = np.sort(self.selected_features_) + return self + + def set_top_percentile(self, percentile, groups=None): + """ + Set the percentile of features to return after fitting. + """ + self._get_support_mask() + assert percentile <= 1 and percentile >= 0, \ + 'percentile must between 0 and 1 inclusive' + self.n_features = int(self.scores_.shape[0] * percentile) + return self._set_top_features(groups=groups) + + def _recommend_number_features(self, solver, max_time=None): + """ + Get the recommended number of features by doing forward passes when max_features is set. + """ + max_time = max_time if max_time else self.max_time + if max_time < 0: + max_time = 60 # allow 1 minute extra if we already spent max_time + MAX_FORWARD_PASS = 200 + MAX_FULL_BATCHES = 3 # the forward passes can take longer than the fitting + # if we allow a full epoch of data to be included. By only doing 3 full batches at most + # we get enough accuracy without increasing the time too much. This + # constant may not be optimal + accum_steps = solver.accum_steps + step_size = max(self.max_features / MAX_FORWARD_PASS, 1) + # pylint: disable=E1130 + feature_order = np.argsort(-self.scores_) # note the negative + t = time.time() + + dataloader_iterator = iter(solver.ds_train) + full_scores = [] + # keep_going = True + with torch.no_grad(): + # might want to only consider a batch valid if there are at least + # two classes + for _ in range(accum_steps * MAX_FULL_BATCHES): + scores = [] + try: + xsub, ysub = next(dataloader_iterator) + except StopIteration: + # done with epoch, don't do more than one epoch + break + except Exception as e: + print(e) + break + if max_time and time.time() - t > max_time: + if self.verbose: + print( + "Stoppinn forward passes because they reached max_time: ", + max_time) + if not full_scores: + # no forward passes worked, return half of max_features + return self.max_features // 2 + break + if solver.multiclass: + for target_class in range(solver.n_classes): + ysub_binary = solver.transform_y_into_binary( + ysub, target_class) + scaling_value = solver._get_scaling_value( + ysub, target_class) + if not solver._skip_y_forward(ysub_binary): + scores = self._generate_scores( + solver, xsub, ysub_binary, step_size, feature_order) + # one row will represent one class that is present in the data + # all classes are weighted equally + full_scores.append( + [score * scaling_value for score in scores]) + else: + if not solver._skip_y_forward(ysub): + scores = self._generate_scores( + solver, xsub, ysub, step_size, feature_order) + full_scores.append(scores) + best_index = FeatureGradientSelector._find_best_index_elbow( + full_scores) + if self.verbose: + print("Forward passes took: ", time.time() - t) + # account for step size and off by one (n_features is 1 indexed, not 0 + # ) + return int( + np.ceil( + np.arange( + 1, + self.max_features + + 1, + step_size))[best_index]) + + @staticmethod + def _find_best_index_elbow(full_scores): + """ + Finds the point on the curve that maximizes distance from the line determined by the endpoints. + """ + scores = pd.DataFrame(full_scores).mean(0).values.tolist() + first_point = np.array([0, scores[0]]) + last_point = np.array([len(scores) - 1, scores[-1]]) + elbow_metric = [] + for i in range(len(scores)): + elbow_metric.append( + FeatureGradientSelector._distance_to_line( + first_point, last_point, np.array([i, scores[i]]))) + return np.argmax(elbow_metric) + + @staticmethod + def _distance_to_line(start_point, end_point, new_point): + """ + Calculates the shortest distance from new_point to the line determined by start_point and end_point. + """ + # for calculating elbow method + return np.cross(new_point - start_point, + end_point - start_point) / np.linalg.norm( + end_point - start_point) + + def _reset(self): + """ + Reset the estimator by deleting all private and fit parameters. + """ + params = self.__dict__ + for key, _ in params.items(): + if key.endswith('_') or key.startswith('_'): + delattr(self, key) + return self diff --git a/utils/third_party/nni_new/algorithms/feature_engineering/gradient_selector/learnability.py b/utils/third_party/nni_new/algorithms/feature_engineering/gradient_selector/learnability.py new file mode 100644 index 0000000000000000000000000000000000000000..3a0ab4b39e7bef15101c793f3ec522395c05cd94 --- /dev/null +++ b/utils/third_party/nni_new/algorithms/feature_engineering/gradient_selector/learnability.py @@ -0,0 +1,534 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# MIT License +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and +# associated documentation files (the "Software"), to deal in the Software without restriction, +# including without limitation the rights to use, copy, modify, merge, publish, distribute, +# sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or +# substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT +# NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT +# OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# ================================================================================================== + +import time + +import numpy as np +import scipy.special +import torch +import torch.nn as nn + +from . import constants +from . import syssettings +from .fginitialize import ChunkDataLoader + +torch.set_default_tensor_type(syssettings.torch.tensortype) +sparsetensor = syssettings.torch.sparse.tensortype + + +def def_train_opt(p): + """ + Return the default optimizer. + """ + return torch.optim.Adam(p, 1e-1, amsgrad=False) + + +def revcumsum(U): + """ + Reverse cumulative sum for faster performance. + """ + return U.flip(dims=[0]).cumsum(dim=0).flip(dims=[0]) + + +def triudr(X, r): + + Zr = torch.zeros_like(X, requires_grad=False) + U = X * r + Zr[:-1] = X[:-1] * revcumsum(U)[1:] + + return Zr + + +def triudl(X, l): + + Zl = torch.zeros_like(X, requires_grad=False) + U = X * l + Zl[1:] = X[1:] * (U.cumsum(dim=0)[:-1]) + + return Zl + + +class ramp(torch.autograd.Function): + """ + Ensures input is between 0 and 1 + """ + + @staticmethod + def forward(ctx, input_data): + ctx.save_for_backward(input_data) + return input_data.clamp(min=0, max=1) + + + @staticmethod + def backward(ctx, grad_output): + input_data, = ctx.saved_tensors + grad_input = grad_output.clone() + grad_input[input_data < 0] = 1e-2 + grad_input[input_data > 1] = -1e-2 + return grad_input + + +class safesqrt(torch.autograd.Function): + """ + Square root without dividing by 0. + """ + @staticmethod + def forward(ctx, input_data): + o = input_data.sqrt() + ctx.save_for_backward(input_data, o) + return o + + + @staticmethod + def backward(ctx, grad_output): + _, o = ctx.saved_tensors + grad_input = grad_output.clone() + grad_input *= 0.5 / (o + constants.EPSILON) + return grad_input + + +class LearnabilityMB(nn.Module): + """ + Calculates the learnability of a set of features. + mini-batch version w/ "left" and "right" multiplies + """ + + + def __init__(self, Nminibatch, D, coeff, groups=None, binary=False, + device=constants.Device.CPU): + super(LearnabilityMB, self).__init__() + + a = coeff / scipy.special.binom(Nminibatch, np.arange(coeff.size) + 2) + self.order = a.size + # pylint: disable=E1102 + self.a = torch.tensor(a, dtype=torch.get_default_dtype(), requires_grad=False) + self.binary = binary + + self.a = self.a.to(device) + + + def ret_val(self, z): + """ + Get the return value based on z. + """ + + if not self.binary: + return 1 - z + + else: + return 0.5 * (1 - safesqrt.apply(ramp.apply(z))) + + + def forward(self, s, X, y): + + l = y.clone() + r = y.clone() + z = 0 + + for i in range(self.order): + if i % 2 == 0: + Z = triudr(X, r) + r = torch.mm(Z, s) + else: + Z = triudl(X, l) + l = torch.mm(Z, s) + if self.a[i] != 0: + # same the computation if a[i] is 0 + p = torch.mm(l.t(), r) + z += self.a[i] * p + return self.ret_val(z) + + +class Solver(nn.Module): + """ + Class that performs the main optimization. + Keeps track of the current x and iterates through data to learn x given the penalty and order. + """ + + def __init__(self, + PreparedData, + order, + Nminibatch=None, + groups=None, + soft_groups=None, + x0=None, + C=1, + ftransform=torch.sigmoid, + get_train_opt=def_train_opt, + accum_steps=1, + rng=np.random.RandomState(0), + max_norm_clip=1., + shuffle=True, + device=constants.Device.CPU, + verbose=1): + """ + + Parameters + ---------- + PreparedData : Dataset of PrepareData class + order : int + What order of interactions to include. Higher orders + may be more accurate but increase the run time. 12 is the maximum allowed order. + Nminibatch : int + Number of rows in a mini batch + groups : array-like + Optional, shape = [n_features] + Groups of columns that must be selected as a unit + e.g. [0, 0, 1, 2] specifies the first two columns are part of a group. + soft_groups : array-like + optional, shape = [n_features] + Groups of columns come from the same source + Used to encourage sparsity of number of sources selected + e.g. [0, 0, 1, 2] specifies the first two columns are part of a group. + x0 : torch.tensor + Optional, initialization of x. + C : float + Penalty parameter. + get_train_opt : function + Function that returns a pytorch optimizer, Adam is the default + accum_steps : int + Number of steps + rng : random state + max_norm_clip : float + Maximum allowable size of the gradient + shuffle : bool + Whether or not to shuffle data within the dataloader + order : int + What order of interactions to include. Higher orders + may be more accurate but increase the run time. 12 is the maximum allowed order. + penalty : int + Constant that multiplies the regularization term. + ftransform : function + Function to transform the x. sigmoid is the default. + device : str + 'cpu' to run on CPU and 'cuda' to run on GPU. Runs much faster on GPU + verbose : int + Controls the verbosity when fitting. Set to 0 for no printing + 1 or higher for printing every verbose number of gradient steps. + """ + super(Solver, self).__init__() + + self.Ntrain, self.D = PreparedData.N, PreparedData.n_features + if groups is not None: + # pylint: disable=E1102 + groups = torch.tensor(groups, dtype=torch.long) + self.groups = groups + else: + self.groups = None + if soft_groups is not None: + # pylint: disable=E1102 + soft_groups = torch.tensor(soft_groups, dtype=torch.long) + self.soft_D = torch.unique(soft_groups).size()[0] + else: + self.soft_D = None + self.soft_groups = soft_groups + + if Nminibatch is None: + Nminibatch = self.Ntrain + else: + if Nminibatch > self.Ntrain: + print('Minibatch larger than sample size.' + + (' Reducing from %d to %d.' + % (Nminibatch, self.Ntrain))) + Nminibatch = self.Ntrain + if Nminibatch > PreparedData.max_rows: + print('Minibatch larger than mem-allowed.' + + (' Reducing from %d to %d.' % (Nminibatch, + PreparedData.max_rows))) + Nminibatch = int(np.min([Nminibatch, PreparedData.max_rows])) + self.Nminibatch = Nminibatch + self.accum_steps = accum_steps + + if x0 is None: + x0 = torch.zeros(self.D, 1, dtype=torch.get_default_dtype()) + self.ftransform = ftransform + self.x = nn.Parameter(x0) + self.max_norm = max_norm_clip + + self.device = device + self.verbose = verbose + + self.multiclass = PreparedData.classification and PreparedData.n_classes and PreparedData.n_classes > 2 + if self.multiclass: + self.n_classes = PreparedData.n_classes + else: + self.n_classes = None + # whether to treat all classes equally + self.balanced = PreparedData.balanced + self.ordinal = PreparedData.ordinal + + if (hasattr(PreparedData, 'mappings') + or PreparedData.storage_level == 'disk'): + num_workers = PreparedData.num_workers + elif PreparedData.storage_level == constants.StorageLevel.DENSE: + num_workers = 0 + else: + num_workers = 0 + + if constants.Device.CUDA in device: + pin_memory = False + else: + pin_memory = False + + if num_workers == 0: + timeout = 0 + else: + timeout = 60 + + self.ds_train = ChunkDataLoader( + PreparedData, + batch_size=self.Nminibatch, + shuffle=shuffle, + drop_last=True, + num_workers=num_workers, + pin_memory=pin_memory, + timeout=timeout) + self.f_train = LearnabilityMB(self.Nminibatch, self.D, + constants.Coefficients.SLE[order], + self.groups, + binary=PreparedData.classification, + device=self.device) + self.opt_train = get_train_opt(torch.nn.ParameterList([self.x])) + self.it = 0 + self.iters_per_epoch = int(np.ceil(len(self.ds_train.dataset) + / self.ds_train.batch_size)) + self.f_train = self.f_train.to(device) + # pylint: disable=E1102 + self.w = torch.tensor( + C / (C + 1), + dtype=torch.get_default_dtype(), requires_grad=False) + self.w = self.w.to(device) + + + def penalty(self, s): + """ + Calculate L1 Penalty. + """ + to_return = torch.sum(s) / self.D + if self.soft_groups is not None: + # if soft_groups, there is an additional penalty for using more + # groups + s_grouped = torch.zeros(self.soft_D, 1, + dtype=torch.get_default_dtype(), + device=self.device) + for group in torch.unique(self.soft_groups): + # groups should be indexed 0 to n_group - 1 + # TODO: consider other functions here + s_grouped[group] = s[self.soft_groups == group].max() + # each component of the penalty contributes .5 + # TODO: could make this a user given parameter + to_return = (to_return + torch.sum(s_grouped) / self.soft_D) * .5 + return to_return + + + def forward_and_backward(self, s, xsub, ysub, retain_graph=False): + """ + Completes the forward operation and computes gradients for learnability and penalty. + """ + f_train = self.f_train(s, xsub, ysub) + pen = self.penalty(s).unsqueeze(0).unsqueeze(0) + # pylint: disable=E1102 + grad_outputs = torch.tensor([[1]], dtype=torch.get_default_dtype(), + device=self.device) + g1, = torch.autograd.grad([f_train], [self.x], grad_outputs, + retain_graph=True) + # pylint: disable=E1102 + grad_outputs = torch.tensor([[1]], dtype=torch.get_default_dtype(), + device=self.device) + g2, = torch.autograd.grad([pen], [self.x], grad_outputs, + retain_graph=retain_graph) + return f_train, pen, g1, g2 + + + def combine_gradient(self, g1, g2): + """ + Combine gradients from learnability and penalty + + Parameters + ---------- + g1 : array-like + gradient from learnability + g2 : array-like + gradient from penalty + """ + to_return = ((1 - self.w) * g1 + self.w * g2) / self.accum_steps + if self.groups is not None: + # each column will get a gradient + # but we can only up or down groups, so the gradient for the group + # should be the average of the gradients of the columns + to_return_grouped = torch.zeros_like(self.x) + for group in torch.unique(self.groups): + to_return_grouped[self.groups == + group] = to_return[self.groups == group].mean() + to_return = to_return_grouped + return to_return + + + def combine_loss(self, f_train, pen): + """ + Combine the learnability and L1 penalty. + """ + return ((1 - self.w) * f_train.detach() + self.w * pen.detach()) \ + / self.accum_steps + + + def transform_y_into_binary(self, ysub, target_class): + """ + Transforms multiclass classification problems into a binary classification problem. + """ + with torch.no_grad(): + ysub_binary = torch.zeros_like(ysub) + if self.ordinal: + # turn ordinal problems into n-1 classifications of is this + # example less than rank k + if target_class == 0: + return None + + ysub_binary[ysub >= target_class] = 1 + ysub_binary[ysub < target_class] = -1 + else: + # turn multiclass problems into n binary classifications + ysub_binary[ysub == target_class] = 1 + ysub_binary[ysub != target_class] = -1 + return ysub_binary + + + def _get_scaling_value(self, ysub, target_class): + """ + Returns the weight given to a class for multiclass classification. + """ + if self.balanced: + if self.ordinal: + return 1 / (torch.unique(ysub).size()[0] - 1) + + return 1 / torch.unique(ysub).size()[0] + else: + if self.ordinal: + this_class_proportion = torch.mean(ysub >= target_class) + normalizing_constant = 0 + for i in range(1, self.n_classes): + normalizing_constant += torch.mean(ysub >= i) + return this_class_proportion / normalizing_constant + else: + return torch.mean(ysub == target_class) + + + def _skip_y_forward(self, y): + """ + Returns boolean of whether to skip the currrent y if there is nothing to be learned from it. + """ + if y is None: + return True + elif torch.unique(y).size()[0] < 2: + return True + else: + return False + + + def train(self, f_callback=None, f_stop=None): + """ + Trains the estimator to determine which features to include. + + Parameters + ---------- + f_callback : function + Function that performs a callback + f_stop: function + Function that tells you when to stop + """ + + t = time.time() + h = torch.zeros([1, 1], dtype=torch.get_default_dtype()) + h = h.to(self.device) + # h_complete is so when we divide by the number of classes + # we only do that for that minibatch if accumulating + h_complete = h.clone() + flag_stop = False + dataloader_iterator = iter(self.ds_train) + self.x.grad = torch.zeros_like(self.x) + while not flag_stop: + try: + xsub, ysub = next(dataloader_iterator) + except StopIteration: + dataloader_iterator = iter(self.ds_train) + xsub, ysub = next(dataloader_iterator) + try: + s = self.ftransform(self.x) + s = s.to(self.device) + if self.multiclass: + # accumulate gradients over each class, classes range from + # 0 to n_classes - 1 + #num_classes_batch = torch.unique(ysub).size()[0] + for target_class in range(self.n_classes): + ysub_binary = self.transform_y_into_binary( + ysub, target_class) + if self._skip_y_forward(ysub_binary): + continue + # should should skip if target class is not included + # but that changes what we divide by + scaling_value = self._get_scaling_value( + ysub, target_class) + f_train, pen, g1, g2 = self.forward_and_backward( + s, xsub, ysub_binary, retain_graph=True) + self.x.grad += self.combine_gradient( + g1, g2) * scaling_value + h += self.combine_loss(f_train, + pen) * scaling_value + else: + if not self._skip_y_forward(ysub): + f_train, pen, g1, g2 = self.forward_and_backward( + s, xsub, ysub) + self.x.grad += self.combine_gradient(g1, g2) + h += self.combine_loss(f_train, pen) + else: + continue + h_complete += h + self.it += 1 + if torch.isnan(h): + raise constants.NanError( + 'Loss is nan, something may be misconfigured') + if self.it % self.accum_steps == 0: + torch.nn.utils.clip_grad_norm_( + torch.nn.ParameterList([self.x]), + max_norm=self.max_norm) + self.opt_train.step() + + t = time.time() - t + if f_stop is not None: + flag_stop = f_stop(self, h, self.it, t) + + if f_callback is not None: + f_callback(self, h, self.it, t) + elif self.verbose and (self.it // self.accum_steps) % self.verbose == 0: + epoch = int(self.it / self.iters_per_epoch) + print( + '[Minibatch: %6d/ Epoch: %3d/ t: %3.3f s] Loss: %0.3f' % + (self.it, epoch, t, h_complete / self.accum_steps)) + + if flag_stop: + break + + self.opt_train.zero_grad() + h = 0 + h_complete = 0 + t = time.time() + except KeyboardInterrupt: + flag_stop = True + break diff --git a/utils/third_party/nni_new/algorithms/feature_engineering/gradient_selector/requirements.txt b/utils/third_party/nni_new/algorithms/feature_engineering/gradient_selector/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..7e2873b558c30216e17b584570b7383623b2931c --- /dev/null +++ b/utils/third_party/nni_new/algorithms/feature_engineering/gradient_selector/requirements.txt @@ -0,0 +1,4 @@ +numpy==1.14.3 +scikit-learn>=0.23.2 +scipy==1.1.0 +torch==1.1.0 diff --git a/utils/third_party/nni_new/algorithms/feature_engineering/gradient_selector/syssettings.py b/utils/third_party/nni_new/algorithms/feature_engineering/gradient_selector/syssettings.py new file mode 100644 index 0000000000000000000000000000000000000000..df864b316601464a9f35a7b463933b1f05a9fe3f --- /dev/null +++ b/utils/third_party/nni_new/algorithms/feature_engineering/gradient_selector/syssettings.py @@ -0,0 +1,29 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# MIT License +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and +# associated documentation files (the "Software"), to deal in the Software without restriction, +# including without limitation the rights to use, copy, modify, merge, publish, distribute, +# sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or +# substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT +# NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT +# OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# ================================================================================================== + + +import torch + +# pytorch +torch.tensortype = torch.FloatTensor +torch.sparse.tensortype = torch.sparse.FloatTensor + +# mem +MAXMEMGB = 10 diff --git a/utils/third_party/nni_new/algorithms/feature_engineering/gradient_selector/utils.py b/utils/third_party/nni_new/algorithms/feature_engineering/gradient_selector/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..0ab9b09a25cfb328586c39f00ab9546388b4f8d4 --- /dev/null +++ b/utils/third_party/nni_new/algorithms/feature_engineering/gradient_selector/utils.py @@ -0,0 +1,78 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# MIT License +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and +# associated documentation files (the "Software"), to deal in the Software without restriction, +# including without limitation the rights to use, copy, modify, merge, publish, distribute, +# sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or +# substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT +# NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT +# OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# ================================================================================================== + + +import numpy as np + +class EMA(): + """ + maintains an exponential moving average + """ + + def __init__(self, f=np.nan, discount_factor=0.1, valid_after=None, + n_iters_relchange=3): + + self.f_ma = [f] + self.fs = [f] + self.gamma = discount_factor + self.rel_change = [np.nan] + if valid_after is None: + self.valid_after = int(1/discount_factor) + else: + self.valid_after = valid_after + self.n_iters_relchange = n_iters_relchange + self.initialized = False + + def reset(self, f): + + self.f_ma = [f] + self.fs = [f] + self.rel_change = [np.nan] + self.initialized = True + + def relchange(self): + + if self.num_updates() > np.max([self.valid_after, + self.n_iters_relchange]): + return np.max(self.rel_change[-self.n_iters_relchange:]) + else: + return np.nan + + def update(self, f_new): + + if not self.initialized: + self.reset(f_new) + else: + self.fs.append(f_new) + self.f_ma.append(self.f_ma[-1]*(1-self.gamma) + self.gamma*f_new) + if self.num_updates() > self.valid_after: + self.rel_change.append(np.abs((self.f_ma[-1]-self.f_ma[-2]) + / self.f_ma[-2])) + + def num_updates(self): + + return len(self.f_ma) + + def __call__(self): + + if self.num_updates() > self.valid_after: + return self.f_ma[-1] + else: + return np.nan diff --git a/utils/third_party/nni_new/algorithms/hpo/batch_tuner.py b/utils/third_party/nni_new/algorithms/hpo/batch_tuner.py new file mode 100644 index 0000000000000000000000000000000000000000..4f73fce9453161a8f30783f915873ba8e5bb8f31 --- /dev/null +++ b/utils/third_party/nni_new/algorithms/hpo/batch_tuner.py @@ -0,0 +1,131 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +batch_tuner.py including: + class BatchTuner +""" + +import logging + +import nni +from nni.tuner import Tuner + +TYPE = '_type' +CHOICE = 'choice' +VALUE = '_value' + +LOGGER = logging.getLogger('batch_tuner_AutoML') + +class BatchTuner(Tuner): + """ + BatchTuner is tuner will running all the configure that user want to run batchly. + + Examples + -------- + The search space only be accepted like: + + :: + + {'combine_params': + { '_type': 'choice', + '_value': '[{...}, {...}, {...}]', + } + } + + """ + + def __init__(self): + self._count = -1 + self._values = [] + + def is_valid(self, search_space): + """ + Check the search space is valid: only contains 'choice' type + + Parameters + ---------- + search_space : dict + + Returns + ------- + None or list + If valid, return candidate values; else return None. + """ + if not len(search_space) == 1: + raise RuntimeError('BatchTuner only supprt one combined-paramreters key.') + + for param in search_space: + param_type = search_space[param][TYPE] + if not param_type == CHOICE: + raise RuntimeError('BatchTuner only supprt \ + one combined-paramreters type is choice.') + + if isinstance(search_space[param][VALUE], list): + return search_space[param][VALUE] + + raise RuntimeError('The combined-paramreters \ + value in BatchTuner is not a list.') + return None + + def update_search_space(self, search_space): + """Update the search space + + Parameters + ---------- + search_space : dict + """ + self._values = self.is_valid(search_space) + + def generate_parameters(self, parameter_id, **kwargs): + """Returns a dict of trial (hyper-)parameters, as a serializable object. + + Parameters + ---------- + parameter_id : int + + Returns + ------- + dict + A candidate parameter group. + """ + self._count += 1 + if self._count > len(self._values) - 1: + raise nni.NoMoreTrialError('no more parameters now.') + return self._values[self._count] + + def receive_trial_result(self, parameter_id, parameters, value, **kwargs): + pass + + def import_data(self, data): + """Import additional data for tuning + + Parameters + ---------- + data: + a list of dictionarys, each of which has at least two keys, 'parameter' and 'value' + """ + if not self._values: + LOGGER.info("Search space has not been initialized, skip this data import") + return + + self._values = self._values[(self._count+1):] + self._count = -1 + + _completed_num = 0 + for trial_info in data: + LOGGER .info("Importing data, current processing \ + progress %s / %s", _completed_num, len(data)) + # simply validate data format + assert "parameter" in trial_info + _params = trial_info["parameter"] + assert "value" in trial_info + _value = trial_info['value'] + if not _value: + LOGGER.info("Useless trial data, value is %s, skip this trial data.", _value) + continue + _completed_num += 1 + if _params in self._values: + self._values.remove(_params) + LOGGER .info("Successfully import data to batch tuner, \ + total data: %d, imported data: %d.", len(data), _completed_num) diff --git a/utils/third_party/nni_new/algorithms/hpo/bohb_advisor/__init__.py b/utils/third_party/nni_new/algorithms/hpo/bohb_advisor/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..0ebb442e5cbe24103750f94c0ed931dc93b5803a --- /dev/null +++ b/utils/third_party/nni_new/algorithms/hpo/bohb_advisor/__init__.py @@ -0,0 +1 @@ +from .bohb_advisor import BOHB, BOHBClassArgsValidator diff --git a/utils/third_party/nni_new/algorithms/hpo/bohb_advisor/bohb_advisor.py b/utils/third_party/nni_new/algorithms/hpo/bohb_advisor/bohb_advisor.py new file mode 100644 index 0000000000000000000000000000000000000000..73687abc5c71145e95d8623b64ed82284c9063e3 --- /dev/null +++ b/utils/third_party/nni_new/algorithms/hpo/bohb_advisor/bohb_advisor.py @@ -0,0 +1,676 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +''' +bohb_advisor.py +''' + +import sys +import math +import logging +import json_tricks +from schema import Schema, Optional +import ConfigSpace as CS +import ConfigSpace.hyperparameters as CSH + +from nni import ClassArgsValidator +from nni.runtime.protocol import CommandType, send +from nni.runtime.msg_dispatcher_base import MsgDispatcherBase +from nni.utils import OptimizeMode, MetricType, extract_scalar_reward +from nni.runtime.common import multi_phase_enabled + +from .config_generator import CG_BOHB + +logger = logging.getLogger('BOHB_Advisor') + +_next_parameter_id = 0 +_KEY = 'TRIAL_BUDGET' +_epsilon = 1e-6 + + +def create_parameter_id(): + """Create an id + + Returns + ------- + int + parameter id + """ + global _next_parameter_id + _next_parameter_id += 1 + return _next_parameter_id - 1 + + +def create_bracket_parameter_id(brackets_id, brackets_curr_decay, increased_id=-1): + """Create a full id for a specific bracket's hyperparameter configuration + + Parameters + ---------- + brackets_id: int + brackets id + brackets_curr_decay: int + brackets curr decay + increased_id: int + increased id + Returns + ------- + int + params id + """ + if increased_id == -1: + increased_id = str(create_parameter_id()) + params_id = '_'.join([str(brackets_id), + str(brackets_curr_decay), + increased_id]) + return params_id + + +class Bracket: + """ + A bracket in BOHB, all the information of a bracket is managed by + an instance of this class. + + Parameters + ---------- + s: int + The current Successive Halving iteration index. + s_max: int + total number of Successive Halving iterations + eta: float + In each iteration, a complete run of sequential halving is executed. In it, + after evaluating each configuration on the same subset size, only a fraction of + 1/eta of them 'advances' to the next round. + max_budget : float + The largest budget to consider. Needs to be larger than min_budget! + The budgets will be geometrically distributed + :math:`a^2 + b^2 = c^2 \\sim \\eta^k` for :math:`k\\in [0, 1, ... , num\\_subsets - 1]`. + optimize_mode: str + optimize mode, 'maximize' or 'minimize' + """ + def __init__(self, s, s_max, eta, max_budget, optimize_mode): + self.s = s + self.s_max = s_max + self.eta = eta + self.max_budget = max_budget + self.optimize_mode = OptimizeMode(optimize_mode) + + self.n = math.ceil((s_max + 1) * eta**s / (s + 1) - _epsilon) + self.r = max_budget / eta**s + self.i = 0 + self.hyper_configs = [] # [ {id: params}, {}, ... ] + self.configs_perf = [] # [ {id: [seq, acc]}, {}, ... ] + self.num_configs_to_run = [] # [ n, n, n, ... ] + self.num_finished_configs = [] # [ n, n, n, ... ] + self.no_more_trial = False + + def is_completed(self): + """check whether this bracket has sent out all the hyperparameter configurations""" + return self.no_more_trial + + def get_n_r(self): + """return the values of n and r for the next round""" + return math.floor(self.n / self.eta**self.i + _epsilon), math.floor(self.r * self.eta**self.i +_epsilon) + + def increase_i(self): + """i means the ith round. Increase i by 1""" + self.i += 1 + + def set_config_perf(self, i, parameter_id, seq, value): + """update trial's latest result with its sequence number, e.g., epoch number or batch number + + Parameters + ---------- + i: int + the ith round + parameter_id: int + the id of the trial/parameter + seq: int + sequence number, e.g., epoch number or batch number + value: int + latest result with sequence number seq + + Returns + ------- + None + """ + if parameter_id in self.configs_perf[i]: + if self.configs_perf[i][parameter_id][0] < seq: + self.configs_perf[i][parameter_id] = [seq, value] + else: + self.configs_perf[i][parameter_id] = [seq, value] + + def inform_trial_end(self, i): + """If the trial is finished and the corresponding round (i.e., i) has all its trials finished, + it will choose the top k trials for the next round (i.e., i+1) + + Parameters + ---------- + i: int + the ith round + + Returns + ------- + new trial or None: + If we have generated new trials after this trial end, we will return a new trial parameters. + Otherwise, we will return None. + """ + global _KEY + self.num_finished_configs[i] += 1 + logger.debug('bracket id: %d, round: %d %d, finished: %d, all: %d', + self.s, self.i, i, self.num_finished_configs[i], self.num_configs_to_run[i]) + if self.num_finished_configs[i] >= self.num_configs_to_run[i] and self.no_more_trial is False: + # choose candidate configs from finished configs to run in the next round + assert self.i == i + 1 + # finish this bracket + if self.i > self.s: + self.no_more_trial = True + return None + this_round_perf = self.configs_perf[i] + if self.optimize_mode is OptimizeMode.Maximize: + sorted_perf = sorted(this_round_perf.items( + ), key=lambda kv: kv[1][1], reverse=True) # reverse + else: + sorted_perf = sorted( + this_round_perf.items(), key=lambda kv: kv[1][1]) + logger.debug( + 'bracket %s next round %s, sorted hyper configs: %s', self.s, self.i, sorted_perf) + next_n, next_r = self.get_n_r() + logger.debug('bracket %s next round %s, next_n=%d, next_r=%d', + self.s, self.i, next_n, next_r) + hyper_configs = dict() + for k in range(next_n): + params_id = sorted_perf[k][0] + params = self.hyper_configs[i][params_id] + params[_KEY] = next_r # modify r + # generate new id + increased_id = params_id.split('_')[-1] + new_id = create_bracket_parameter_id( + self.s, self.i, increased_id) + hyper_configs[new_id] = params + self._record_hyper_configs(hyper_configs) + return [[key, value] for key, value in hyper_configs.items()] + return None + + def get_hyperparameter_configurations(self, num, r, config_generator): + """generate num hyperparameter configurations from search space using Bayesian optimization + + Parameters + ---------- + num: int + the number of hyperparameter configurations + + Returns + ------- + list + a list of hyperparameter configurations. Format: [[key1, value1], [key2, value2], ...] + """ + global _KEY + assert self.i == 0 + hyperparameter_configs = dict() + for _ in range(num): + params_id = create_bracket_parameter_id(self.s, self.i) + params = config_generator.get_config(r) + params[_KEY] = r + hyperparameter_configs[params_id] = params + self._record_hyper_configs(hyperparameter_configs) + return [[key, value] for key, value in hyperparameter_configs.items()] + + def _record_hyper_configs(self, hyper_configs): + """after generating one round of hyperconfigs, this function records the generated hyperconfigs, + creates a dict to record the performance when those hyperconifgs are running, set the number of finished configs + in this round to be 0, and increase the round number. + + Parameters + ---------- + hyper_configs: list + the generated hyperconfigs + """ + self.hyper_configs.append(hyper_configs) + self.configs_perf.append(dict()) + self.num_finished_configs.append(0) + self.num_configs_to_run.append(len(hyper_configs)) + self.increase_i() + +class BOHBClassArgsValidator(ClassArgsValidator): + def validate_class_args(self, **kwargs): + Schema({ + 'optimize_mode': self.choices('optimize_mode', 'maximize', 'minimize'), + Optional('min_budget'): self.range('min_budget', int, 0, 9999), + Optional('max_budget'): self.range('max_budget', int, 0, 9999), + Optional('eta'): self.range('eta', int, 0, 9999), + Optional('min_points_in_model'): self.range('min_points_in_model', int, 0, 9999), + Optional('top_n_percent'): self.range('top_n_percent', int, 1, 99), + Optional('num_samples'): self.range('num_samples', int, 1, 9999), + Optional('random_fraction'): self.range('random_fraction', float, 0, 9999), + Optional('bandwidth_factor'): self.range('bandwidth_factor', float, 0, 9999), + Optional('min_bandwidth'): self.range('min_bandwidth', float, 0, 9999), + }).validate(kwargs) + +class BOHB(MsgDispatcherBase): + """ + BOHB performs robust and efficient hyperparameter optimization + at scale by combining the speed of Hyperband searches with the + guidance and guarantees of convergence of Bayesian Optimization. + Instead of sampling new configurations at random, BOHB uses + kernel density estimators to select promising candidates. + + Parameters + ---------- + optimize_mode: str + optimize mode, 'maximize' or 'minimize' + min_budget: float + The smallest budget to consider. Needs to be positive! + max_budget: float + The largest budget to consider. Needs to be larger than min_budget! + The budgets will be geometrically distributed + :math:`a^2 + b^2 = c^2 \\sim \\eta^k` for :math:`k\\in [0, 1, ... , num\\_subsets - 1]`. + eta: int + In each iteration, a complete run of sequential halving is executed. In it, + after evaluating each configuration on the same subset size, only a fraction of + 1/eta of them 'advances' to the next round. + Must be greater or equal to 2. + min_points_in_model: int + number of observations to start building a KDE. Default 'None' means + dim+1, the bare minimum. + top_n_percent: int + percentage ( between 1 and 99, default 15) of the observations that are considered good. + num_samples: int + number of samples to optimize EI (default 64) + random_fraction: float + fraction of purely random configurations that are sampled from the + prior without the model. + bandwidth_factor: float + to encourage diversity, the points proposed to optimize EI, are sampled + from a 'widened' KDE where the bandwidth is multiplied by this factor (default: 3) + min_bandwidth: float + to keep diversity, even when all (good) samples have the same value for one of the parameters, + a minimum bandwidth (Default: 1e-3) is used instead of zero. + """ + + def __init__(self, + optimize_mode='maximize', + min_budget=1, + max_budget=3, + eta=3, + min_points_in_model=None, + top_n_percent=15, + num_samples=64, + random_fraction=1/3, + bandwidth_factor=3, + min_bandwidth=1e-3): + super(BOHB, self).__init__() + self.optimize_mode = OptimizeMode(optimize_mode) + self.min_budget = min_budget + self.max_budget = max_budget + self.eta = eta + self.min_points_in_model = min_points_in_model + self.top_n_percent = top_n_percent + self.num_samples = num_samples + self.random_fraction = random_fraction + self.bandwidth_factor = bandwidth_factor + self.min_bandwidth = min_bandwidth + + # all the configs waiting for run + self.generated_hyper_configs = [] + # all the completed configs + self.completed_hyper_configs = [] + + self.s_max = math.floor( + math.log(self.max_budget / self.min_budget, self.eta) + _epsilon) + # current bracket(s) number + self.curr_s = self.s_max + # In this case, tuner increases self.credit to issue a trial config sometime later. + self.credit = 0 + self.brackets = dict() + self.search_space = None + # [key, value] = [parameter_id, parameter] + self.parameters = dict() + + # config generator + self.cg = None + + # record the latest parameter_id of the trial job trial_job_id. + # if there is no running parameter_id, self.job_id_para_id_map[trial_job_id] == None + # new trial job is added to this dict and finished trial job is removed from it. + self.job_id_para_id_map = dict() + # record the unsatisfied parameter request from trial jobs + self.unsatisfied_jobs = [] + + def handle_initialize(self, data): + """Initialize Tuner, including creating Bayesian optimization-based parametric models + and search space formations + + Parameters + ---------- + data: search space + search space of this experiment + + Raises + ------ + ValueError + Error: Search space is None + """ + logger.info('start to handle_initialize') + # convert search space jason to ConfigSpace + self.handle_update_search_space(data) + + # generate BOHB config_generator using Bayesian optimization + if self.search_space: + self.cg = CG_BOHB(configspace=self.search_space, + min_points_in_model=self.min_points_in_model, + top_n_percent=self.top_n_percent, + num_samples=self.num_samples, + random_fraction=self.random_fraction, + bandwidth_factor=self.bandwidth_factor, + min_bandwidth=self.min_bandwidth) + else: + raise ValueError('Error: Search space is None') + # generate first brackets + self.generate_new_bracket() + send(CommandType.Initialized, '') + + def generate_new_bracket(self): + """generate a new bracket""" + logger.debug( + 'start to create a new SuccessiveHalving iteration, self.curr_s=%d', self.curr_s) + if self.curr_s < 0: + logger.info("s < 0, Finish this round of Hyperband in BOHB. Generate new round") + self.curr_s = self.s_max + self.brackets[self.curr_s] = Bracket( + s=self.curr_s, s_max=self.s_max, eta=self.eta, + max_budget=self.max_budget, optimize_mode=self.optimize_mode + ) + next_n, next_r = self.brackets[self.curr_s].get_n_r() + logger.debug( + 'new SuccessiveHalving iteration, next_n=%d, next_r=%d', next_n, next_r) + # rewrite with TPE + generated_hyper_configs = self.brackets[self.curr_s].get_hyperparameter_configurations( + next_n, next_r, self.cg) + self.generated_hyper_configs = generated_hyper_configs.copy() + + def handle_request_trial_jobs(self, data): + """recerive the number of request and generate trials + + Parameters + ---------- + data: int + number of trial jobs that nni manager ask to generate + """ + # Receive new request + self.credit += data + + for _ in range(self.credit): + self._request_one_trial_job() + + def _get_one_trial_job(self): + """get one trial job, i.e., one hyperparameter configuration. + + If this function is called, Command will be sent by BOHB: + a. If there is a parameter need to run, will return "NewTrialJob" with a dict: + { + 'parameter_id': id of new hyperparameter + 'parameter_source': 'algorithm' + 'parameters': value of new hyperparameter + } + b. If BOHB don't have parameter waiting, will return "NoMoreTrialJobs" with + { + 'parameter_id': '-1_0_0', + 'parameter_source': 'algorithm', + 'parameters': '' + } + """ + if not self.generated_hyper_configs: + ret = { + 'parameter_id': '-1_0_0', + 'parameter_source': 'algorithm', + 'parameters': '' + } + send(CommandType.NoMoreTrialJobs, json_tricks.dumps(ret)) + return None + assert self.generated_hyper_configs + params = self.generated_hyper_configs.pop(0) + ret = { + 'parameter_id': params[0], + 'parameter_source': 'algorithm', + 'parameters': params[1] + } + self.parameters[params[0]] = params[1] + return ret + + def _request_one_trial_job(self): + """get one trial job, i.e., one hyperparameter configuration. + + If this function is called, Command will be sent by BOHB: + a. If there is a parameter need to run, will return "NewTrialJob" with a dict: + { + 'parameter_id': id of new hyperparameter + 'parameter_source': 'algorithm' + 'parameters': value of new hyperparameter + } + b. If BOHB don't have parameter waiting, will return "NoMoreTrialJobs" with + { + 'parameter_id': '-1_0_0', + 'parameter_source': 'algorithm', + 'parameters': '' + } + """ + ret = self._get_one_trial_job() + if ret is not None: + send(CommandType.NewTrialJob, json_tricks.dumps(ret)) + self.credit -= 1 + + def handle_update_search_space(self, data): + """change json format to ConfigSpace format dict -> configspace + + Parameters + ---------- + data: JSON object + search space of this experiment + """ + search_space = data + cs = CS.ConfigurationSpace() + for var in search_space: + _type = str(search_space[var]["_type"]) + if _type == 'choice': + cs.add_hyperparameter(CSH.CategoricalHyperparameter( + var, choices=search_space[var]["_value"])) + elif _type == 'randint': + cs.add_hyperparameter(CSH.UniformIntegerHyperparameter( + var, lower=search_space[var]["_value"][0], upper=search_space[var]["_value"][1] - 1)) + elif _type == 'uniform': + cs.add_hyperparameter(CSH.UniformFloatHyperparameter( + var, lower=search_space[var]["_value"][0], upper=search_space[var]["_value"][1])) + elif _type == 'quniform': + cs.add_hyperparameter(CSH.UniformFloatHyperparameter( + var, lower=search_space[var]["_value"][0], upper=search_space[var]["_value"][1], + q=search_space[var]["_value"][2])) + elif _type == 'loguniform': + cs.add_hyperparameter(CSH.UniformFloatHyperparameter( + var, lower=search_space[var]["_value"][0], upper=search_space[var]["_value"][1], + log=True)) + elif _type == 'qloguniform': + cs.add_hyperparameter(CSH.UniformFloatHyperparameter( + var, lower=search_space[var]["_value"][0], upper=search_space[var]["_value"][1], + q=search_space[var]["_value"][2], log=True)) + elif _type == 'normal': + cs.add_hyperparameter(CSH.NormalFloatHyperparameter( + var, mu=search_space[var]["_value"][1], sigma=search_space[var]["_value"][2])) + elif _type == 'qnormal': + cs.add_hyperparameter(CSH.NormalFloatHyperparameter( + var, mu=search_space[var]["_value"][1], sigma=search_space[var]["_value"][2], + q=search_space[var]["_value"][3])) + elif _type == 'lognormal': + cs.add_hyperparameter(CSH.NormalFloatHyperparameter( + var, mu=search_space[var]["_value"][1], sigma=search_space[var]["_value"][2], + log=True)) + elif _type == 'qlognormal': + cs.add_hyperparameter(CSH.NormalFloatHyperparameter( + var, mu=search_space[var]["_value"][1], sigma=search_space[var]["_value"][2], + q=search_space[var]["_value"][3], log=True)) + else: + raise ValueError( + 'unrecognized type in search_space, type is {}'.format(_type)) + + self.search_space = cs + + def handle_trial_end(self, data): + """receive the information of trial end and generate next configuaration. + + Parameters + ---------- + data: dict() + it has three keys: trial_job_id, event, hyper_params + trial_job_id: the id generated by training service + event: the job's state + hyper_params: the hyperparameters (a string) generated and returned by tuner + """ + logger.debug('Tuner handle trial end, result is %s', data) + hyper_params = json_tricks.loads(data['hyper_params']) + self._handle_trial_end(hyper_params['parameter_id']) + if data['trial_job_id'] in self.job_id_para_id_map: + del self.job_id_para_id_map[data['trial_job_id']] + + def _send_new_trial(self): + while self.unsatisfied_jobs: + ret = self._get_one_trial_job() + if ret is None: + break + one_unsatisfied = self.unsatisfied_jobs.pop(0) + ret['trial_job_id'] = one_unsatisfied['trial_job_id'] + ret['parameter_index'] = one_unsatisfied['parameter_index'] + # update parameter_id in self.job_id_para_id_map + self.job_id_para_id_map[ret['trial_job_id']] = ret['parameter_id'] + send(CommandType.SendTrialJobParameter, json_tricks.dumps(ret)) + for _ in range(self.credit): + self._request_one_trial_job() + + def _handle_trial_end(self, parameter_id): + s, i, _ = parameter_id.split('_') + hyper_configs = self.brackets[int(s)].inform_trial_end(int(i)) + + if hyper_configs is not None: + logger.debug( + 'bracket %s next round %s, hyper_configs: %s', s, i, hyper_configs) + self.generated_hyper_configs = self.generated_hyper_configs + hyper_configs + # Finish this bracket and generate a new bracket + elif self.brackets[int(s)].no_more_trial: + self.curr_s -= 1 + self.generate_new_bracket() + self._send_new_trial() + + def handle_report_metric_data(self, data): + """reveice the metric data and update Bayesian optimization with final result + + Parameters + ---------- + data: + it is an object which has keys 'parameter_id', 'value', 'trial_job_id', 'type', 'sequence'. + + Raises + ------ + ValueError + Data type not supported + """ + logger.debug('handle report metric data = %s', data) + if 'value' in data: + data['value'] = json_tricks.loads(data['value']) + if data['type'] == MetricType.REQUEST_PARAMETER: + assert multi_phase_enabled() + assert data['trial_job_id'] is not None + assert data['parameter_index'] is not None + assert data['trial_job_id'] in self.job_id_para_id_map + self._handle_trial_end(self.job_id_para_id_map[data['trial_job_id']]) + ret = self._get_one_trial_job() + if ret is None: + self.unsatisfied_jobs.append({'trial_job_id': data['trial_job_id'], 'parameter_index': data['parameter_index']}) + else: + ret['trial_job_id'] = data['trial_job_id'] + ret['parameter_index'] = data['parameter_index'] + # update parameter_id in self.job_id_para_id_map + self.job_id_para_id_map[data['trial_job_id']] = ret['parameter_id'] + send(CommandType.SendTrialJobParameter, json_tricks.dumps(ret)) + else: + assert 'value' in data + value = extract_scalar_reward(data['value']) + if self.optimize_mode is OptimizeMode.Maximize: + reward = -value + else: + reward = value + assert 'parameter_id' in data + s, i, _ = data['parameter_id'].split('_') + logger.debug('bracket id = %s, metrics value = %s, type = %s', s, value, data['type']) + s = int(s) + + # add to self.job_id_para_id_map here, + # because when the first parameter_id is created, trial_job_id is not known yet. + if data['trial_job_id'] in self.job_id_para_id_map: + assert self.job_id_para_id_map[data['trial_job_id']] == data['parameter_id'] + else: + self.job_id_para_id_map[data['trial_job_id']] = data['parameter_id'] + + assert 'type' in data + if data['type'] == MetricType.FINAL: + # and PERIODICAL metric are independent, thus, not comparable. + assert 'sequence' in data + self.brackets[s].set_config_perf( + int(i), data['parameter_id'], sys.maxsize, value) + self.completed_hyper_configs.append(data) + + _parameters = self.parameters[data['parameter_id']] + _parameters.pop(_KEY) + # update BO with loss, max_s budget, hyperparameters + self.cg.new_result(loss=reward, budget=data['sequence'], parameters=_parameters, update_model=True) + elif data['type'] == MetricType.PERIODICAL: + self.brackets[s].set_config_perf( + int(i), data['parameter_id'], data['sequence'], value) + else: + raise ValueError( + 'Data type not supported: {}'.format(data['type'])) + + def handle_add_customized_trial(self, data): + pass + + def handle_import_data(self, data): + """Import additional data for tuning + + Parameters + ---------- + data: + a list of dictionarys, each of which has at least two keys, 'parameter' and 'value' + + Raises + ------ + AssertionError + data doesn't have required key 'parameter' and 'value' + """ + for entry in data: + entry['value'] = json_tricks.loads(entry['value']) + _completed_num = 0 + for trial_info in data: + logger.info("Importing data, current processing progress %s / %s", _completed_num, len(data)) + _completed_num += 1 + assert "parameter" in trial_info + _params = trial_info["parameter"] + assert "value" in trial_info + _value = trial_info['value'] + if not _value: + logger.info("Useless trial data, value is %s, skip this trial data.", _value) + continue + _value = extract_scalar_reward(_value) + budget_exist_flag = False + barely_params = dict() + for keys in _params: + if keys == _KEY: + _budget = _params[keys] + budget_exist_flag = True + else: + barely_params[keys] = _params[keys] + if not budget_exist_flag: + _budget = self.max_budget + logger.info("Set \"TRIAL_BUDGET\" value to %s (max budget)", self.max_budget) + if self.optimize_mode is OptimizeMode.Maximize: + reward = -_value + else: + reward = _value + self.cg.new_result(loss=reward, budget=_budget, parameters=barely_params, update_model=True) + logger.info("Successfully import tuning data to BOHB advisor.") diff --git a/utils/third_party/nni_new/algorithms/hpo/bohb_advisor/config_generator.py b/utils/third_party/nni_new/algorithms/hpo/bohb_advisor/config_generator.py new file mode 100644 index 0000000000000000000000000000000000000000..c6a13c6b35ba3b2db9bbbf8618d3b9a3d9240909 --- /dev/null +++ b/utils/third_party/nni_new/algorithms/hpo/bohb_advisor/config_generator.py @@ -0,0 +1,344 @@ +# BSD 3-Clause License +# Copyright (c) 2017-2018, ML4AAD +# All rights reserved. + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. + +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. + +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import logging + +import ConfigSpace +import ConfigSpace.hyperparameters +import ConfigSpace.util +import numpy as np +import scipy.stats as sps +import statsmodels.api as sm + +logger = logging.getLogger('BOHB_Advisor') + +class CG_BOHB: + def __init__(self, configspace, min_points_in_model=None, + top_n_percent=15, num_samples=64, random_fraction=1/3, + bandwidth_factor=3, min_bandwidth=1e-3): + """Fits for each given budget a kernel density estimator on the best N percent of the + evaluated configurations on this budget. + + + Parameters: + ----------- + configspace: ConfigSpace + Configuration space object + top_n_percent: int + Determines the percentile of configurations that will be used as training data + for the kernel density estimator, e.g if set to 10 the 10% best configurations will be considered + for training. + min_points_in_model: int + minimum number of datapoints needed to fit a model + num_samples: int + number of samples drawn to optimize EI via sampling + random_fraction: float + fraction of random configurations returned + bandwidth_factor: float + widens the bandwidth for contiuous parameters for proposed points to optimize EI + min_bandwidth: float + to keep diversity, even when all (good) samples have the same value for one of the parameters, + a minimum bandwidth (Default: 1e-3) is used instead of zero. + """ + self.top_n_percent = top_n_percent + self.configspace = configspace + self.bw_factor = bandwidth_factor + self.min_bandwidth = min_bandwidth + + self.min_points_in_model = min_points_in_model + if min_points_in_model is None: + self.min_points_in_model = len(self.configspace.get_hyperparameters())+1 + + if self.min_points_in_model < len(self.configspace.get_hyperparameters())+1: + logger.warning('Invalid min_points_in_model value. Setting it to %i', len(self.configspace.get_hyperparameters()) + 1) + self.min_points_in_model = len(self.configspace.get_hyperparameters()) + 1 + + self.num_samples = num_samples + self.random_fraction = random_fraction + + hps = self.configspace.get_hyperparameters() + + self.kde_vartypes = "" + self.vartypes = [] + + for h in hps: + if hasattr(h, 'choices'): + self.kde_vartypes += 'u' + self.vartypes += [len(h.choices)] + else: + self.kde_vartypes += 'c' + self.vartypes += [0] + + self.vartypes = np.array(self.vartypes, dtype=int) + + # store precomputed probs for the categorical parameters + self.cat_probs = [] + + self.configs = dict() + self.losses = dict() + self.good_config_rankings = dict() + self.kde_models = dict() + + def largest_budget_with_model(self): + if not self.kde_models: + return -float('inf') + return max(self.kde_models.keys()) + + def sample_from_largest_budget(self, info_dict): + """We opted for a single multidimensional KDE compared to the + hierarchy of one-dimensional KDEs used in TPE. The dimensional is + seperated by budget. This function sample a configuration from + largest budget. Firstly we sample "num_samples" configurations, + then prefer one with the largest l(x)/g(x). + + Parameters: + ----------- + info_dict: dict + record the information of this configuration + + Returns + ------- + dict: + new configuration named sample + dict: + info_dict, record the information of this configuration + """ + best = np.inf + best_vector = None + + budget = max(self.kde_models.keys()) + + l = self.kde_models[budget]['good'].pdf + g = self.kde_models[budget]['bad'].pdf + + minimize_me = lambda x: max(1e-32, g(x))/max(l(x), 1e-32) + + kde_good = self.kde_models[budget]['good'] + kde_bad = self.kde_models[budget]['bad'] + + for i in range(self.num_samples): + idx = np.random.randint(0, len(kde_good.data)) + datum = kde_good.data[idx] + vector = [] + + for m, bw, t in zip(datum, kde_good.bw, self.vartypes): + + bw = max(bw, self.min_bandwidth) + if t == 0: + bw = self.bw_factor*bw + vector.append(sps.truncnorm.rvs(-m/bw, (1-m)/bw, loc=m, scale=bw)) + else: + if np.random.rand() < (1-bw): + vector.append(int(m)) + else: + vector.append(np.random.randint(t)) + val = minimize_me(vector) + + if not np.isfinite(val): + logger.warning('sampled vector: %s has EI value %s', vector, val) + logger.warning("data in the KDEs:\n%s\n%s", kde_good.data, kde_bad.data) + logger.warning("bandwidth of the KDEs:\n%s\n%s", kde_good.bw, kde_bad.bw) + logger.warning("l(x) = %s", l(vector)) + logger.warning("g(x) = %s", g(vector)) + + # right now, this happens because a KDE does not contain all values for a categorical parameter + # this cannot be fixed with the statsmodels KDE, so for now, we are just going to evaluate this one + # if the good_kde has a finite value, i.e. there is no config with that value in the bad kde, + # so it shouldn't be terrible. + if np.isfinite(l(vector)): + best_vector = vector + break + + if val < best: + best = val + best_vector = vector + + if best_vector is None: + logger.debug("Sampling based optimization with %i samples failed -> using random configuration", self.num_samples) + sample = self.configspace.sample_configuration().get_dictionary() + info_dict['model_based_pick'] = False + + else: + logger.debug('best_vector: %s, %s, %s, %s', best_vector, best, l(best_vector), g(best_vector)) + for i, _ in enumerate(best_vector): + hp = self.configspace.get_hyperparameter(self.configspace.get_hyperparameter_by_idx(i)) + if isinstance(hp, ConfigSpace.hyperparameters.CategoricalHyperparameter): + best_vector[i] = int(np.rint(best_vector[i])) + sample = ConfigSpace.Configuration(self.configspace, vector=best_vector).get_dictionary() + + sample = ConfigSpace.util.deactivate_inactive_hyperparameters( + configuration_space=self.configspace, + configuration=sample) + info_dict['model_based_pick'] = True + + return sample, info_dict + + def get_config(self, budget): + """Function to sample a new configuration + This function is called inside BOHB to query a new configuration + + Parameters: + ----------- + budget: float + the budget for which this configuration is scheduled + + Returns + ------- + config + return a valid configuration with parameters and budget + """ + logger.debug('start sampling a new configuration.') + sample = None + info_dict = {} + + # If no model is available, sample from prior + # also mix in a fraction of random configs + if not self.kde_models.keys() or np.random.rand() < self.random_fraction: + sample = self.configspace.sample_configuration() + info_dict['model_based_pick'] = False + + if sample is None: + sample, info_dict = self.sample_from_largest_budget(info_dict) + + sample = ConfigSpace.util.deactivate_inactive_hyperparameters( + configuration_space=self.configspace, + configuration=sample.get_dictionary() + ).get_dictionary() + + logger.debug('done sampling a new configuration.') + sample['TRIAL_BUDGET'] = budget + return sample + + def impute_conditional_data(self, array): + return_array = np.zeros(array.shape) + for i in range(array.shape[0]): + datum = np.copy(array[i]) + nan_indices = np.argwhere(np.isnan(datum)).flatten() + while np.any(nan_indices): + nan_idx = nan_indices[0] + valid_indices = np.argwhere(np.isfinite(array[:, nan_idx])).flatten() + if valid_indices: + # pick one of them at random and overwrite all NaN values + row_idx = np.random.choice(valid_indices) + datum[nan_indices] = array[row_idx, nan_indices] + else: + # no good point in the data has this value activated, so fill it with a valid but random value + t = self.vartypes[nan_idx] + if t == 0: + datum[nan_idx] = np.random.rand() + else: + datum[nan_idx] = np.random.randint(t) + nan_indices = np.argwhere(np.isnan(datum)).flatten() + return_array[i, :] = datum + return return_array + + def new_result(self, loss, budget, parameters, update_model=True): + """ + Function to register finished runs. Every time a run has finished, this function should be called + to register it with the loss. + + Parameters: + ----------- + loss: float + the loss of the parameters + budget: float + the budget of the parameters + parameters: dict + the parameters of this trial + update_model: bool + whether use this parameter to update BP model + + Returns + ------- + None + """ + if loss is None: + # One could skip crashed results, but we decided + # assign a +inf loss and count them as bad configurations + loss = np.inf + + if budget not in self.configs.keys(): + self.configs[budget] = [] + self.losses[budget] = [] + + # skip model building if we already have a bigger model + if max(list(self.kde_models.keys()) + [-np.inf]) > budget: + return + + # We want to get a numerical representation of the configuration in the original space + conf = ConfigSpace.Configuration(self.configspace, parameters) + self.configs[budget].append(conf.get_array()) + self.losses[budget].append(loss) + + # skip model building: + # a) if not enough points are available + if len(self.configs[budget]) <= self.min_points_in_model - 1: + logger.debug("Only %i run(s) for budget %f available, need more than %s \ + -> can't build model!", len(self.configs[budget]), budget, self.min_points_in_model+1) + return + # b) during warnm starting when we feed previous results in and only update once + if not update_model: + return + + train_configs = np.array(self.configs[budget]) + train_losses = np.array(self.losses[budget]) + + n_good = max(self.min_points_in_model, (self.top_n_percent * train_configs.shape[0])//100) + n_bad = max(self.min_points_in_model, ((100-self.top_n_percent)*train_configs.shape[0])//100) + + # Refit KDE for the current budget + idx = np.argsort(train_losses) + + train_data_good = self.impute_conditional_data(train_configs[idx[:n_good]]) + train_data_bad = self.impute_conditional_data(train_configs[idx[n_good:n_good+n_bad]]) + + if train_data_good.shape[0] <= train_data_good.shape[1]: + return + if train_data_bad.shape[0] <= train_data_bad.shape[1]: + return + + #more expensive crossvalidation method + #bw_estimation = 'cv_ls' + # quick rule of thumb + bw_estimation = 'normal_reference' + + bad_kde = sm.nonparametric.KDEMultivariate(data=train_data_bad, var_type=self.kde_vartypes, bw=bw_estimation) + good_kde = sm.nonparametric.KDEMultivariate(data=train_data_good, var_type=self.kde_vartypes, bw=bw_estimation) + + bad_kde.bw = np.clip(bad_kde.bw, self.min_bandwidth, None) + good_kde.bw = np.clip(good_kde.bw, self.min_bandwidth, None) + + self.kde_models[budget] = { + 'good': good_kde, + 'bad' : bad_kde + } + + # update probs for the categorical parameters for later sampling + logger.debug('done building a new model for budget %f based on %i/%i split\nBest loss for this budget:%f\n', + budget, n_good, n_bad, np.min(train_losses)) diff --git a/utils/third_party/nni_new/algorithms/hpo/curvefitting_assessor/__init__.py b/utils/third_party/nni_new/algorithms/hpo/curvefitting_assessor/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..f3ed4d7a513b181e43e8767d28ead3cbc20af139 --- /dev/null +++ b/utils/third_party/nni_new/algorithms/hpo/curvefitting_assessor/__init__.py @@ -0,0 +1,4 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .curvefitting_assessor import CurvefittingAssessor, CurvefittingClassArgsValidator diff --git a/utils/third_party/nni_new/algorithms/hpo/curvefitting_assessor/curvefitting_assessor.py b/utils/third_party/nni_new/algorithms/hpo/curvefitting_assessor/curvefitting_assessor.py new file mode 100644 index 0000000000000000000000000000000000000000..885886e89b83712e7eded05fd89598a9e2232b5a --- /dev/null +++ b/utils/third_party/nni_new/algorithms/hpo/curvefitting_assessor/curvefitting_assessor.py @@ -0,0 +1,135 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +import datetime +from schema import Schema, Optional + +from nni import ClassArgsValidator +from nni.assessor import Assessor, AssessResult +from nni.utils import extract_scalar_history +from .model_factory import CurveModel + +logger = logging.getLogger('curvefitting_Assessor') + +class CurvefittingClassArgsValidator(ClassArgsValidator): + def validate_class_args(self, **kwargs): + Schema({ + 'epoch_num': self.range('epoch_num', int, 0, 9999), + Optional('start_step'): self.range('start_step', int, 0, 9999), + Optional('threshold'): self.range('threshold', float, 0, 9999), + Optional('gap'): self.range('gap', int, 1, 9999), + }).validate(kwargs) + +class CurvefittingAssessor(Assessor): + """CurvefittingAssessor uses learning curve fitting algorithm to predict the learning curve performance in the future. + It stops a pending trial X at step S if the trial's forecast result at target step is convergence and lower than the + best performance in the history. + + Parameters + ---------- + epoch_num : int + The total number of epoch + start_step : int + only after receiving start_step number of reported intermediate results + threshold : float + The threshold that we decide to early stop the worse performance curve. + """ + + def __init__(self, epoch_num=20, start_step=6, threshold=0.95, gap=1): + if start_step <= 0: + logger.warning('It\'s recommended to set start_step to a positive number') + # Record the target position we predict + self.target_pos = epoch_num + # Start forecasting when historical data reaches start step + self.start_step = start_step + # Record the compared threshold + self.threshold = threshold + # Record the number of gap + self.gap = gap + # Record the number of intermediate result in the lastest judgment + self.last_judgment_num = dict() + # Record the best performance + self.set_best_performance = False + self.completed_best_performance = None + self.trial_history = [] + logger.info('Successfully initials the curvefitting assessor') + + def trial_end(self, trial_job_id, success): + """update the best performance of completed trial job + + Parameters + ---------- + trial_job_id : int + trial job id + success : bool + True if succssfully finish the experiment, False otherwise + """ + if success: + if self.set_best_performance: + self.completed_best_performance = max(self.completed_best_performance, self.trial_history[-1]) + else: + self.set_best_performance = True + self.completed_best_performance = self.trial_history[-1] + logger.info('Updated complted best performance, trial job id: %s', trial_job_id) + else: + logger.info('No need to update, trial job id: %s', trial_job_id) + + def assess_trial(self, trial_job_id, trial_history): + """assess whether a trial should be early stop by curve fitting algorithm + + Parameters + ---------- + trial_job_id : int + trial job id + trial_history : list + The history performance matrix of each trial + + Returns + ------- + bool + AssessResult.Good or AssessResult.Bad + + Raises + ------ + Exception + unrecognize exception in curvefitting_assessor + """ + scalar_trial_history = extract_scalar_history(trial_history) + self.trial_history = scalar_trial_history + if not self.set_best_performance: + return AssessResult.Good + curr_step = len(scalar_trial_history) + if curr_step < self.start_step: + return AssessResult.Good + + if trial_job_id in self.last_judgment_num.keys() and curr_step - self.last_judgment_num[trial_job_id] < self.gap: + return AssessResult.Good + self.last_judgment_num[trial_job_id] = curr_step + + try: + start_time = datetime.datetime.now() + # Predict the final result + curvemodel = CurveModel(self.target_pos) + predict_y = curvemodel.predict(scalar_trial_history) + log_message = "Prediction done. Trial job id = {}, Predict value = {}".format(trial_job_id, predict_y) + if predict_y is None: + logger.info('%s, wait for more information to predict precisely', log_message) + return AssessResult.Good + else: + logger.info(log_message) + standard_performance = self.completed_best_performance * self.threshold + + end_time = datetime.datetime.now() + if (end_time - start_time).seconds > 60: + logger.warning( + 'Curve Fitting Assessor Runtime Exceeds 60s, Trial Id = %s Trial History = %s', + trial_job_id, self.trial_history + ) + + if predict_y > standard_performance: + return AssessResult.Good + return AssessResult.Bad + + except Exception as exception: + logger.exception('unrecognize exception in curvefitting_assessor %s', exception) diff --git a/utils/third_party/nni_new/algorithms/hpo/curvefitting_assessor/curvefunctions.py b/utils/third_party/nni_new/algorithms/hpo/curvefitting_assessor/curvefunctions.py new file mode 100644 index 0000000000000000000000000000000000000000..c7bfc3b9d817df38689d24ff317e1543ac0f88e9 --- /dev/null +++ b/utils/third_party/nni_new/algorithms/hpo/curvefitting_assessor/curvefunctions.py @@ -0,0 +1,296 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +A family of functions used by CurvefittingAssessor +""" + +import numpy as np + +all_models = {} +model_para = {} +model_para_num = {} + +curve_combination_models = ['vap', 'pow3', 'linear', 'logx_linear', 'dr_hill_zero_background', 'log_power', 'pow4', 'mmf', + 'exp4', 'ilog2', 'weibull', 'janoschek'] + + +def vap(x, a, b, c): + """Vapor pressure model + + Parameters + ---------- + x : int + a : float + b : float + c : float + + Returns + ------- + float + np.exp(a+b/x+c*np.log(x)) + """ + return np.exp(a+b/x+c*np.log(x)) + + +all_models['vap'] = vap +model_para['vap'] = [-0.622028, -0.470050, 0.042322] +model_para_num['vap'] = 3 + + +def pow3(x, c, a, alpha): + """pow3 + + Parameters + ---------- + x : int + c : float + a : float + alpha : float + + Returns + ------- + float + c - a * x**(-alpha) + """ + return c - a * x**(-alpha) + + +all_models['pow3'] = pow3 +model_para['pow3'] = [0.84, 0.52, 0.01] +model_para_num['pow3'] = 3 + + +def linear(x, a, b): + """linear + + Parameters + ---------- + x : int + a : float + b : float + + Returns + ------- + float + a*x + b + """ + return a*x + b + + +all_models['linear'] = linear +model_para['linear'] = [1., 0] +model_para_num['linear'] = 2 + + +def logx_linear(x, a, b): + """logx linear + + Parameters + ---------- + x : int + a : float + b : float + + Returns + ------- + float + a * np.log(x) + b + """ + x = np.log(x) + return a*x + b + + +all_models['logx_linear'] = logx_linear +model_para['logx_linear'] = [0.378106, 0.046506] +model_para_num['logx_linear'] = 2 + + +def dr_hill_zero_background(x, theta, eta, kappa): + """dr hill zero background + + Parameters + ---------- + x : int + theta : float + eta : float + kappa : float + + Returns + ------- + float + (theta* x**eta) / (kappa**eta + x**eta) + """ + return (theta * x**eta) / (kappa**eta + x**eta) + + +all_models['dr_hill_zero_background'] = dr_hill_zero_background +model_para['dr_hill_zero_background'] = [0.772320, 0.586449, 2.460843] +model_para_num['dr_hill_zero_background'] = 3 + + +def log_power(x, a, b, c): + """"logistic power + + Parameters + ---------- + x : int + a : float + b : float + c : float + + Returns + ------- + float + a/(1.+(x/np.exp(b))**c) + """ + return a/(1.+(x/np.exp(b))**c) + + +all_models['log_power'] = log_power +model_para['log_power'] = [0.77, 2.98, -0.51] +model_para_num['log_power'] = 3 + + +def pow4(x, alpha, a, b, c): + """pow4 + + Parameters + ---------- + x : int + alpha : float + a : float + b : float + c : float + + Returns + ------- + float + c - (a*x+b)**-alpha + """ + return c - (a*x+b)**-alpha + + +all_models['pow4'] = pow4 +model_para['pow4'] = [0.1, 200, 0., 0.8] +model_para_num['pow4'] = 4 + + +def mmf(x, alpha, beta, kappa, delta): + """Morgan-Mercer-Flodin + http://www.pisces-conservation.com/growthhelp/index.html?morgan_mercer_floden.htm + + Parameters + ---------- + x : int + alpha : float + beta : float + kappa : float + delta : float + + Returns + ------- + float + alpha - (alpha - beta) / (1. + (kappa * x)**delta) + """ + return alpha - (alpha - beta) / (1. + (kappa * x)**delta) + + +all_models['mmf'] = mmf +model_para['mmf'] = [0.7, 0.1, 0.01, 5] +model_para_num['mmf'] = 4 + + +def exp4(x, c, a, b, alpha): + """exp4 + + Parameters + ---------- + x : int + c : float + a : float + b : float + alpha : float + + Returns + ------- + float + c - np.exp(-a*(x**alpha)+b) + """ + return c - np.exp(-a*(x**alpha)+b) + + +all_models['exp4'] = exp4 +model_para['exp4'] = [0.7, 0.8, -0.8, 0.3] +model_para_num['exp4'] = 4 + + +def ilog2(x, c, a): + """ilog2 + + Parameters + ---------- + x : int + c : float + a : float + + Returns + ------- + float + c - a / np.log(x) + """ + return c - a / np.log(x) + + +all_models['ilog2'] = ilog2 +model_para['ilog2'] = [0.78, 0.43] +model_para_num['ilog2'] = 2 + + +def weibull(x, alpha, beta, kappa, delta): + """Weibull model + http://www.pisces-conservation.com/growthhelp/index.html?morgan_mercer_floden.htm + + Parameters + ---------- + x : int + alpha : float + beta : float + kappa : float + delta : float + + Returns + ------- + float + alpha - (alpha - beta) * np.exp(-(kappa * x)**delta) + """ + return alpha - (alpha - beta) * np.exp(-(kappa * x)**delta) + + +all_models['weibull'] = weibull +model_para['weibull'] = [0.7, 0.1, 0.01, 1] +model_para_num['weibull'] = 4 + + +def janoschek(x, a, beta, k, delta): + """http://www.pisces-conservation.com/growthhelp/janoschek.htm + + Parameters + ---------- + x : int + a : float + beta : float + k : float + delta : float + + Returns + ------- + float + a - (a - beta) * np.exp(-k*x**delta) + """ + return a - (a - beta) * np.exp(-k*x**delta) + + +all_models['janoschek'] = janoschek +model_para['janoschek'] = [0.73, 0.07, 0.355, 0.46] +model_para_num['janoschek'] = 4 diff --git a/utils/third_party/nni_new/algorithms/hpo/curvefitting_assessor/model_factory.py b/utils/third_party/nni_new/algorithms/hpo/curvefitting_assessor/model_factory.py new file mode 100644 index 0000000000000000000000000000000000000000..e6a6ada9976293d98a44d1d763f787a7c7c9bea1 --- /dev/null +++ b/utils/third_party/nni_new/algorithms/hpo/curvefitting_assessor/model_factory.py @@ -0,0 +1,330 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +import numpy as np +from scipy import optimize +from .curvefunctions import * # pylint: disable=wildcard-import,unused-wildcard-import + +# Number of curve functions we prepared, more details can be found in "curvefunctions.py" +NUM_OF_FUNCTIONS = 12 +# Number of simulation time when we do MCMC sampling +NUM_OF_SIMULATION_TIME = 20 +# Number of samples we select when we do MCMC sampling +NUM_OF_INSTANCE = 10 +# The step size of each noise when we do MCMC sampling +STEP_SIZE = 0.0005 +# Number of least fitting function, if effective function is lower than this number, we will ask for more information +LEAST_FITTED_FUNCTION = 4 + +logger = logging.getLogger('curvefitting_Assessor') + +class CurveModel: + """Build a Curve Model to predict the performance + + Algorithm: https://github.com/Microsoft/nni/blob/master/src/sdk/pynni/nni/curvefitting_assessor/README.md + + Parameters + ---------- + target_pos : int + The point we need to predict + """ + def __init__(self, target_pos): + self.target_pos = target_pos + self.trial_history = [] + self.point_num = 0 + self.effective_model = [] + self.effective_model_num = 0 + self.weight_samples = [] + + def fit_theta(self): + """use least squares to fit all default curves parameter seperately + + Returns + ------- + None + """ + x = range(1, self.point_num + 1) + y = self.trial_history + for i in range(NUM_OF_FUNCTIONS): + model = curve_combination_models[i] + try: + # The maximum number of iterations to fit is 100*(N+1), where N is the number of elements in `x0`. + if model_para_num[model] == 2: + a, b = optimize.curve_fit(all_models[model], x, y)[0] + model_para[model][0] = a + model_para[model][1] = b + elif model_para_num[model] == 3: + a, b, c = optimize.curve_fit(all_models[model], x, y)[0] + model_para[model][0] = a + model_para[model][1] = b + model_para[model][2] = c + elif model_para_num[model] == 4: + a, b, c, d = optimize.curve_fit(all_models[model], x, y)[0] + model_para[model][0] = a + model_para[model][1] = b + model_para[model][2] = c + model_para[model][3] = d + except (RuntimeError, FloatingPointError, OverflowError, ZeroDivisionError): + # Ignore exceptions caused by numerical calculations + pass + except Exception as exception: + logger.critical("Exceptions in fit_theta: %s", exception) + + def filter_curve(self): + """filter the poor performing curve + + Returns + ------- + None + """ + avg = np.sum(self.trial_history) / self.point_num + standard = avg * avg * self.point_num + predict_data = [] + tmp_model = [] + for i in range(NUM_OF_FUNCTIONS): + var = 0 + model = curve_combination_models[i] + for j in range(1, self.point_num + 1): + y = self.predict_y(model, j) + var += (y - self.trial_history[j - 1]) * (y - self.trial_history[j - 1]) + if var < standard: + predict_data.append(y) + tmp_model.append(curve_combination_models[i]) + median = np.median(predict_data) + std = np.std(predict_data) + for model in tmp_model: + y = self.predict_y(model, self.target_pos) + epsilon = self.point_num / 10 * std + if y < median + epsilon and y > median - epsilon: + self.effective_model.append(model) + self.effective_model_num = len(self.effective_model) + logger.info('List of effective model: %s', self.effective_model) + + def predict_y(self, model, pos): + """return the predict y of 'model' when epoch = pos + + Parameters + ---------- + model : string + name of the curve function model + pos : int + the epoch number of the position you want to predict + + Returns + ------- + int + The expected matrix at pos + """ + if model_para_num[model] == 2: + y = all_models[model](pos, model_para[model][0], model_para[model][1]) + elif model_para_num[model] == 3: + y = all_models[model](pos, model_para[model][0], model_para[model][1], model_para[model][2]) + elif model_para_num[model] == 4: + y = all_models[model](pos, model_para[model][0], model_para[model][1], model_para[model][2], model_para[model][3]) + return y + + def f_comb(self, pos, sample): + """return the value of the f_comb when epoch = pos + + Parameters + ---------- + pos : int + the epoch number of the position you want to predict + sample : list + sample is a (1 * NUM_OF_FUNCTIONS) matrix, representing{w1, w2, ... wk} + + Returns + ------- + int + The expected matrix at pos with all the active function's prediction + """ + ret = 0 + for i in range(self.effective_model_num): + model = self.effective_model[i] + y = self.predict_y(model, pos) + ret += sample[i] * y + return ret + + def normalize_weight(self, samples): + """normalize weight + + Parameters + ---------- + samples : list + a collection of sample, it's a (NUM_OF_INSTANCE * NUM_OF_FUNCTIONS) matrix, + representing{{w11, w12, ..., w1k}, {w21, w22, ... w2k}, ...{wk1, wk2,..., wkk}} + + Returns + ------- + list + samples after normalize weight + """ + for i in range(NUM_OF_INSTANCE): + total = 0 + for j in range(self.effective_model_num): + total += samples[i][j] + for j in range(self.effective_model_num): + samples[i][j] /= total + return samples + + def sigma_sq(self, sample): + """returns the value of sigma square, given the weight's sample + + Parameters + ---------- + sample : list + sample is a (1 * NUM_OF_FUNCTIONS) matrix, representing{w1, w2, ... wk} + + Returns + ------- + float + the value of sigma square, given the weight's sample + """ + ret = 0 + for i in range(1, self.point_num + 1): + temp = self.trial_history[i - 1] - self.f_comb(i, sample) + ret += temp * temp + return 1.0 * ret / self.point_num + + def normal_distribution(self, pos, sample): + """returns the value of normal distribution, given the weight's sample and target position + + Parameters + ---------- + pos : int + the epoch number of the position you want to predict + sample : list + sample is a (1 * NUM_OF_FUNCTIONS) matrix, representing{w1, w2, ... wk} + + Returns + ------- + float + the value of normal distribution + """ + curr_sigma_sq = self.sigma_sq(sample) + delta = self.trial_history[pos - 1] - self.f_comb(pos, sample) + return np.exp(np.square(delta) / (-2.0 * curr_sigma_sq)) / np.sqrt(2 * np.pi * np.sqrt(curr_sigma_sq)) + + def likelihood(self, samples): + """likelihood + + Parameters + ---------- + sample : list + sample is a (1 * NUM_OF_FUNCTIONS) matrix, representing{w1, w2, ... wk} + + Returns + ------- + float + likelihood + """ + ret = np.ones(NUM_OF_INSTANCE) + for i in range(NUM_OF_INSTANCE): + for j in range(1, self.point_num + 1): + ret[i] *= self.normal_distribution(j, samples[i]) + return ret + + def prior(self, samples): + """priori distribution + + Parameters + ---------- + samples : list + a collection of sample, it's a (NUM_OF_INSTANCE * NUM_OF_FUNCTIONS) matrix, + representing{{w11, w12, ..., w1k}, {w21, w22, ... w2k}, ...{wk1, wk2,..., wkk}} + + Returns + ------- + float + priori distribution + """ + ret = np.ones(NUM_OF_INSTANCE) + for i in range(NUM_OF_INSTANCE): + for j in range(self.effective_model_num): + if not samples[i][j] > 0: + ret[i] = 0 + if self.f_comb(1, samples[i]) >= self.f_comb(self.target_pos, samples[i]): + ret[i] = 0 + return ret + + def target_distribution(self, samples): + """posterior probability + + Parameters + ---------- + samples : list + a collection of sample, it's a (NUM_OF_INSTANCE * NUM_OF_FUNCTIONS) matrix, + representing{{w11, w12, ..., w1k}, {w21, w22, ... w2k}, ...{wk1, wk2,..., wkk}} + + Returns + ------- + float + posterior probability + """ + curr_likelihood = self.likelihood(samples) + curr_prior = self.prior(samples) + ret = np.ones(NUM_OF_INSTANCE) + for i in range(NUM_OF_INSTANCE): + ret[i] = curr_likelihood[i] * curr_prior[i] + return ret + + def mcmc_sampling(self): + """Adjust the weight of each function using mcmc sampling. + The initial value of each weight is evenly distribute. + Brief introduction: + (1)Definition of sample: + Sample is a (1 * NUM_OF_FUNCTIONS) matrix, representing{w1, w2, ... wk} + (2)Definition of samples: + Samples is a collection of sample, it's a (NUM_OF_INSTANCE * NUM_OF_FUNCTIONS) matrix, + representing{{w11, w12, ..., w1k}, {w21, w22, ... w2k}, ...{wk1, wk2,..., wkk}} + (3)Definition of model: + Model is the function we chose right now. Such as: 'wap', 'weibull'. + (4)Definition of pos: + Pos is the position we want to predict, corresponds to the value of epoch. + + Returns + ------- + None + """ + init_weight = np.ones((self.effective_model_num), dtype=np.float) / self.effective_model_num + self.weight_samples = np.broadcast_to(init_weight, (NUM_OF_INSTANCE, self.effective_model_num)) + for _ in range(NUM_OF_SIMULATION_TIME): + # sample new value from Q(i, j) + new_values = np.random.randn(NUM_OF_INSTANCE, self.effective_model_num) * STEP_SIZE + self.weight_samples + new_values = self.normalize_weight(new_values) + # compute alpha(i, j) = min{1, P(j)Q(j, i)/P(i)Q(i, j)} + alpha = np.minimum(1, self.target_distribution(new_values) / self.target_distribution(self.weight_samples)) + # sample u + u = np.random.rand(NUM_OF_INSTANCE) + # new value + change_value_flag = (u < alpha).astype(np.int) + for j in range(NUM_OF_INSTANCE): + new_values[j] = self.weight_samples[j] * (1 - change_value_flag[j]) + new_values[j] * change_value_flag[j] + self.weight_samples = new_values + + def predict(self, trial_history): + """predict the value of target position + + Parameters + ---------- + trial_history : list + The history performance matrix of each trial. + + Returns + ------- + float + expected final result performance of this hyperparameter config + """ + self.trial_history = trial_history + self.point_num = len(trial_history) + self.fit_theta() + self.filter_curve() + if self.effective_model_num < LEAST_FITTED_FUNCTION: + # different curve's predictions are too scattered, requires more information + return None + self.mcmc_sampling() + ret = 0 + for i in range(NUM_OF_INSTANCE): + ret += self.f_comb(self.target_pos, self.weight_samples[i]) + return ret / NUM_OF_INSTANCE diff --git a/utils/third_party/nni_new/algorithms/hpo/dngo_tuner.py b/utils/third_party/nni_new/algorithms/hpo/dngo_tuner.py new file mode 100644 index 0000000000000000000000000000000000000000..276ec75168e41b6714cb51f9bd055ba521fed801 --- /dev/null +++ b/utils/third_party/nni_new/algorithms/hpo/dngo_tuner.py @@ -0,0 +1,117 @@ +import logging + +import numpy as np +import torch +import nni.parameter_expressions as parameter_expressions +from nni import ClassArgsValidator +from nni.tuner import Tuner +from pybnn import DNGO +from torch.distributions import Normal + +_logger = logging.getLogger(__name__) + + +def _random_config(search_space, random_state): + chosen_config = {} + for key, val in search_space.items(): + if val['_type'] == 'choice': + choices = val['_value'] + index = random_state.randint(len(choices)) + if all([isinstance(c, (int, float)) for c in choices]): + chosen_config[key] = choices[index] + else: + raise ValueError('Choices with type other than int and float is not supported.') + elif val['_type'] == 'uniform': + chosen_config[key] = random_state.uniform(val['_value'][0], val['_value'][1]) + elif val['_type'] == 'randint': + chosen_config[key] = random_state.randint( + val['_value'][0], val['_value'][1]) + elif val['_type'] == 'quniform': + chosen_config[key] = parameter_expressions.quniform( + val['_value'][0], val['_value'][1], val['_value'][2], random_state) + elif val['_type'] == 'loguniform': + chosen_config[key] = parameter_expressions.loguniform( + val['_value'][0], val['_value'][1], random_state) + elif val['_type'] == 'qloguniform': + chosen_config[key] = parameter_expressions.qloguniform( + val['_value'][0], val['_value'][1], val['_value'][2], random_state) + else: + raise ValueError('Unknown key %s and value %s' % (key, val)) + return chosen_config + + +class DNGOTuner(Tuner): + + def __init__(self, optimize_mode='maximize', sample_size=1000, trials_per_update=20, num_epochs_per_training=500): + self.searchspace_json = None + self.random_state = None + self.model = DNGO(do_mcmc=False, num_epochs=num_epochs_per_training) + self._model_initialized = False + self.sample_size = sample_size + self.trials_per_update = trials_per_update + self.optimize_mode = optimize_mode + + self.x = [] + self.y = [] + + def receive_trial_result(self, parameter_id, parameters, value, **kwargs): + self.x.append(parameters) + self.y.append(self._get_default_value(value)) + if len(self.y) % self.trials_per_update == 0: + self._update_model() + + def generate_parameters(self, parameter_id, **kwargs): + if not self._model_initialized: + return _random_config(self.searchspace_json, self.random_state) + else: + # random samples and pick best with model + candidate_x = [_random_config(self.searchspace_json, self.random_state) for _ in range(self.sample_size)] + + x_test = np.array([np.array(list(xi.values())) for xi in candidate_x]) + m, v = self.model.predict(x_test) + mean = torch.Tensor(m) + sigma = torch.Tensor(v) + u = (mean - torch.Tensor([0.95]).expand_as(mean)) / sigma + normal = Normal(torch.zeros_like(u), torch.ones_like(u)) + ucdf = normal.cdf(u) + updf = torch.exp(normal.log_prob(u)) + ei = sigma * (updf + u * ucdf) + + if self.optimize_mode == 'maximize': + ind = torch.argmax(ei) + else: + ind = torch.argmin(ei) + new_x = candidate_x[ind] + return new_x + + def update_search_space(self, search_space): + self.searchspace_json = search_space + self.random_state = np.random.RandomState() + + def import_data(self, data): + for d in data: + self.x.append(d['parameter']) + self.y.append(self._get_default_value(d['value'])) + self._update_model() + + def _update_model(self): + _logger.info('Updating model on %d samples', len(self.x)) + x_arr = [] + for x in self.x: + x_arr.append([x[k] for k in sorted(x.keys())]) + self.model.train(np.array(x_arr), np.array(self.y), do_optimize=True) + self._model_initialized = True + + def _get_default_value(self, value): + if isinstance(value, dict) and 'default' in value: + return value['default'] + elif isinstance(value, float): + return value + else: + raise ValueError(f'Unsupported value: {value}') + + +class DNGOClassArgsValidator(ClassArgsValidator): + # DNGO tuner do not have much input arg, so the validation is actually hardly used + def validate_class_args(self, **kwargs): + pass diff --git a/utils/third_party/nni_new/algorithms/hpo/evolution_tuner.py b/utils/third_party/nni_new/algorithms/hpo/evolution_tuner.py new file mode 100644 index 0000000000000000000000000000000000000000..9277dcca3caac30134a0b787e4ed23e48295c855 --- /dev/null +++ b/utils/third_party/nni_new/algorithms/hpo/evolution_tuner.py @@ -0,0 +1,283 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +evolution_tuner.py +""" + +import copy +import random +import logging + +from collections import deque +import numpy as np +from schema import Schema, Optional + +import nni +from nni import ClassArgsValidator +from nni.tuner import Tuner +from nni.utils import OptimizeMode, extract_scalar_reward, split_index, json2parameter, json2space + +logger = logging.getLogger(__name__) + +class Individual: + """ + Indicidual class to store the indv info. + + Attributes + ---------- + config : str + Search space. + info : str + The str to save information of individual. + result : float + The final metric of a individual. + """ + + def __init__(self, config=None, info=None, result=None): + """ + Parameters + ---------- + config : str + A config to represent a group of parameters. + info : str + result : float + save_dir : str + """ + self.config = config + self.result = result + self.info = info + + def __str__(self): + return "info: " + str(self.info) + \ + ", config :" + str(self.config) + ", result: " + str(self.result) + +class EvolutionClassArgsValidator(ClassArgsValidator): + def validate_class_args(self, **kwargs): + Schema({ + 'optimize_mode': self.choices('optimize_mode', 'maximize', 'minimize'), + Optional('population_size'): self.range('population_size', int, 0, 99999), + }).validate(kwargs) + +class EvolutionTuner(Tuner): + """ + EvolutionTuner is tuner using navie evolution algorithm. + """ + + def __init__(self, optimize_mode="maximize", population_size=32): + """ + Parameters + ---------- + optimize_mode : str, default 'maximize' + population_size : int + initial population size. The larger population size, + the better evolution performance. + """ + self.optimize_mode = OptimizeMode(optimize_mode) + self.population_size = population_size + + self.searchspace_json = None + self.running_trials = {} + self.num_running_trials = 0 + self.random_state = None + self.population = None + self.space = None + self.credit = 0 # record the unsatisfied trial requests + self.send_trial_callback = None + self.param_ids = deque() + + def update_search_space(self, search_space): + """ + Update search space. + + Search_space contains the information that user pre-defined. + + Parameters + ---------- + search_space : dict + """ + self.searchspace_json = search_space + self.space = json2space(self.searchspace_json) + + self.random_state = np.random.RandomState() + self.population = [] + + for _ in range(self.population_size): + self._random_generate_individual() + + def trial_end(self, parameter_id, success, **kwargs): + """ + To deal with trial failure. If a trial fails, + random generate the parameters and add into the population. + Parameters + ---------- + parameter_id : int + Unique identifier for hyper-parameters used by this trial. + success : bool + True if the trial successfully completed; False if failed or terminated. + **kwargs + Not used + """ + self.num_running_trials -= 1 + logger.info('trial (%d) end', parameter_id) + + if not success: + self.running_trials.pop(parameter_id) + self._random_generate_individual() + + if self.credit > 1: + param_id = self.param_ids.popleft() + config = self._generate_individual(param_id) + logger.debug('Send new trial (%d, %s) for reducing credit', param_id, config) + self.send_trial_callback(param_id, config) + self.credit -= 1 + self.num_running_trials += 1 + + def generate_multiple_parameters(self, parameter_id_list, **kwargs): + """ + Returns multiple sets of trial (hyper-)parameters, as iterable of serializable objects. + Parameters + ---------- + parameter_id_list : list of int + Unique identifiers for each set of requested hyper-parameters. + **kwargs + Not used + Returns + ------- + list + A list of newly generated configurations + """ + + result = [] + if 'st_callback' in kwargs: + self.send_trial_callback = kwargs['st_callback'] + else: + logger.warning('Send trial callback is not found in kwargs. Evolution tuner might not work properly.') + for parameter_id in parameter_id_list: + had_exception = False + try: + logger.debug("generating param for %s", parameter_id) + res = self.generate_parameters(parameter_id, **kwargs) + self.num_running_trials += 1 + except nni.NoMoreTrialError: + had_exception = True + if not had_exception: + result.append(res) + return result + + def _random_generate_individual(self): + is_rand = dict() + for item in self.space: + is_rand[item] = True + + config = json2parameter(self.searchspace_json, is_rand, self.random_state) + self.population.append(Individual(config=config)) + + def _generate_individual(self, parameter_id): + """ + This function will generate the config for a trial. + If at the first generation, randomly generates individuals to satisfy self.population_size. + Otherwise, random choose a pair of individuals and compare their fitnesses. + The worst of the pair will be removed. Copy the best of the pair and mutate it to generate a new individual. + + Parameters + ---------- + parameter_id : int + + Returns + ------- + dict + A group of candaidte parameters that evolution tuner generated. + """ + pos = -1 + + for i in range(len(self.population)): + if self.population[i].result is None: + pos = i + break + + if pos != -1: + indiv = copy.deepcopy(self.population[pos]) + self.population.pop(pos) + else: + random.shuffle(self.population) + # avoid only 1 individual has result + if len(self.population) > 1 and self.population[0].result < self.population[1].result: + self.population[0] = self.population[1] + + # mutation on the worse individual + space = json2space(self.searchspace_json, + self.population[0].config) + is_rand = dict() + mutation_pos = space[random.randint(0, len(space)-1)] + + for i in range(len(self.space)): + is_rand[self.space[i]] = (self.space[i] == mutation_pos) + config = json2parameter( + self.searchspace_json, is_rand, self.random_state, self.population[0].config) + + if len(self.population) > 1: + self.population.pop(1) + + indiv = Individual(config=config) + + # remove "_index" from config and save params-id + self.running_trials[parameter_id] = indiv + config = split_index(indiv.config) + return config + + + def generate_parameters(self, parameter_id, **kwargs): + """ + This function will returns a dict of trial (hyper-)parameters. + If no trial configration for now, self.credit plus 1 to send the config later + + Parameters + ---------- + parameter_id : int + + Returns + ------- + dict + One newly generated configuration. + """ + if not self.population: + raise RuntimeError('The population is empty') + + if self.num_running_trials >= self.population_size: + logger.warning("No enough trial config, population_size is suggested to be larger than trialConcurrency") + self.credit += 1 + self.param_ids.append(parameter_id) + raise nni.NoMoreTrialError('no more parameters now.') + + return self._generate_individual(parameter_id) + + def receive_trial_result(self, parameter_id, parameters, value, **kwargs): + """ + Record the result from a trial + + Parameters + ---------- + parameter_id : int + parameters : dict + value : dict/float + if value is dict, it should have "default" key. + value is final metrics of the trial. + """ + reward = extract_scalar_reward(value) + + if parameter_id not in self.running_trials: + raise RuntimeError('Received parameter_id %s not in running_trials.', parameter_id) + + # restore the paramsters contains "_index" + config = self.running_trials[parameter_id].config + self.running_trials.pop(parameter_id) + + if self.optimize_mode == OptimizeMode.Minimize: + reward = -reward + + indiv = Individual(config=config, result=reward) + self.population.append(indiv) + + def import_data(self, data): + pass diff --git a/utils/third_party/nni_new/algorithms/hpo/gp_tuner/__init__.py b/utils/third_party/nni_new/algorithms/hpo/gp_tuner/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..17bedd38f4ee1fefea0a3d73ab1b5d3fb7ef2aed --- /dev/null +++ b/utils/third_party/nni_new/algorithms/hpo/gp_tuner/__init__.py @@ -0,0 +1 @@ +from .gp_tuner import GPTuner, GPClassArgsValidator diff --git a/utils/third_party/nni_new/algorithms/hpo/gp_tuner/gp_tuner.py b/utils/third_party/nni_new/algorithms/hpo/gp_tuner/gp_tuner.py new file mode 100644 index 0000000000000000000000000000000000000000..c4e6e9a89cc6258457f647f80d24464969282357 --- /dev/null +++ b/utils/third_party/nni_new/algorithms/hpo/gp_tuner/gp_tuner.py @@ -0,0 +1,181 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +GPTuner is a Bayesian Optimization method where Gaussian Process is used for modeling loss functions. + +See :class:`GPTuner` for details. +""" + +import warnings +import logging +import numpy as np +from schema import Schema, Optional + +from sklearn.gaussian_process.kernels import Matern +from sklearn.gaussian_process import GaussianProcessRegressor + +from nni import ClassArgsValidator +from nni.tuner import Tuner +from nni.utils import OptimizeMode, extract_scalar_reward + +from .target_space import TargetSpace +from .util import UtilityFunction, acq_max + +logger = logging.getLogger("GP_Tuner_AutoML") + +class GPClassArgsValidator(ClassArgsValidator): + def validate_class_args(self, **kwargs): + Schema({ + Optional('optimize_mode'): self.choices('optimize_mode', 'maximize', 'minimize'), + Optional('utility'): self.choices('utility', 'ei', 'ucb', 'poi'), + Optional('kappa'): float, + Optional('xi'): float, + Optional('nu'): float, + Optional('alpha'): float, + Optional('cold_start_num'): int, + Optional('selection_num_warm_up'): int, + Optional('selection_num_starting_points'): int, + }).validate(kwargs) + +class GPTuner(Tuner): + """ + GPTuner is a Bayesian Optimization method where Gaussian Process is used for modeling loss functions. + + Parameters + ---------- + optimize_mode : str + optimize mode, 'maximize' or 'minimize', by default 'maximize' + utility : str + utility function (also called 'acquisition funcition') to use, which can be 'ei', 'ucb' or 'poi'. By default 'ei'. + kappa : float + value used by utility function 'ucb'. The bigger kappa is, the more the tuner will be exploratory. By default 5. + xi : float + used by utility function 'ei' and 'poi'. The bigger xi is, the more the tuner will be exploratory. By default 0. + nu : float + used to specify Matern kernel. The smaller nu, the less smooth the approximated function is. By default 2.5. + alpha : float + Used to specify Gaussian Process Regressor. Larger values correspond to increased noise level in the observations. + By default 1e-6. + cold_start_num : int + Number of random exploration to perform before Gaussian Process. By default 10. + selection_num_warm_up : int + Number of random points to evaluate for getting the point which maximizes the acquisition function. By default 100000 + selection_num_starting_points : int + Number of times to run L-BFGS-B from a random starting point after the warmup. By default 250. + """ + + def __init__(self, optimize_mode="maximize", utility='ei', kappa=5, xi=0, nu=2.5, alpha=1e-6, cold_start_num=10, + selection_num_warm_up=100000, selection_num_starting_points=250): + self._optimize_mode = OptimizeMode(optimize_mode) + + # utility function related + self._utility = utility + self._kappa = kappa + self._xi = xi + + # target space + self._space = None + + self._random_state = np.random.RandomState() + + # nu, alpha are GPR related params + self._gp = GaussianProcessRegressor( + kernel=Matern(nu=nu), + alpha=alpha, + normalize_y=True, + n_restarts_optimizer=25, + random_state=self._random_state + ) + # num of random evaluations before GPR + self._cold_start_num = cold_start_num + + # params for acq_max + self._selection_num_warm_up = selection_num_warm_up + self._selection_num_starting_points = selection_num_starting_points + + # num of imported data + self._supplement_data_num = 0 + + def update_search_space(self, search_space): + """ + Update the self.bounds and self.types by the search_space.json file. + + Override of the abstract method in :class:`~nni.tuner.Tuner`. + """ + self._space = TargetSpace(search_space, self._random_state) + + def generate_parameters(self, parameter_id, **kwargs): + """ + Method which provides one set of hyper-parameters. + If the number of trial result is lower than cold_start_number, GPTuner will first randomly generate some parameters. + Otherwise, choose the parameters by the Gussian Process Model. + + Override of the abstract method in :class:`~nni.tuner.Tuner`. + """ + if self._space.len() < self._cold_start_num: + results = self._space.random_sample() + else: + # Sklearn's GP throws a large number of warnings at times, but + # we don't really need to see them here. + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + self._gp.fit(self._space.params, self._space.target) + + util = UtilityFunction( + kind=self._utility, kappa=self._kappa, xi=self._xi) + + results = acq_max( + f_acq=util.utility, + gp=self._gp, + y_max=self._space.target.max(), + bounds=self._space.bounds, + space=self._space, + num_warmup=self._selection_num_warm_up, + num_starting_points=self._selection_num_starting_points + ) + + results = self._space.array_to_params(results) + logger.info("Generate paramageters:\n %s", results) + return results + + def receive_trial_result(self, parameter_id, parameters, value, **kwargs): + """ + Method invoked when a trial reports its final result. + + Override of the abstract method in :class:`~nni.tuner.Tuner`. + """ + value = extract_scalar_reward(value) + if self._optimize_mode == OptimizeMode.Minimize: + value = -value + + logger.info("Received trial result.") + logger.info("value :%s", value) + logger.info("parameter : %s", parameters) + self._space.register(parameters, value) + + def import_data(self, data): + """ + Import additional data for tuning. + + Override of the abstract method in :class:`~nni.tuner.Tuner`. + """ + _completed_num = 0 + for trial_info in data: + logger.info( + "Importing data, current processing progress %s / %s", _completed_num, len(data)) + _completed_num += 1 + assert "parameter" in trial_info + _params = trial_info["parameter"] + assert "value" in trial_info + _value = trial_info['value'] + if not _value: + logger.info( + "Useless trial data, value is %s, skip this trial data.", _value) + continue + self._supplement_data_num += 1 + _parameter_id = '_'.join( + ["ImportData", str(self._supplement_data_num)]) + self.receive_trial_result( + parameter_id=_parameter_id, parameters=_params, value=_value) + logger.info("Successfully import data to GP tuner.") diff --git a/utils/third_party/nni_new/algorithms/hpo/gp_tuner/target_space.py b/utils/third_party/nni_new/algorithms/hpo/gp_tuner/target_space.py new file mode 100644 index 0000000000000000000000000000000000000000..7ee52c0e9969a90f931bd9a9e43b194f354d81c4 --- /dev/null +++ b/utils/third_party/nni_new/algorithms/hpo/gp_tuner/target_space.py @@ -0,0 +1,295 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +Tool class to hold the param-space coordinates (X) and target values (Y). +""" + +import numpy as np +import nni.parameter_expressions as parameter_expressions + + +def _hashable(params): + """ + Transform list params to tuple format. Ensure that an point is hashable by a python dict. + + Parameters + ---------- + params : numpy array + array format of parameters + + Returns + ------- + tuple + tuple format of parameters + """ + return tuple(map(float, params)) + + +class TargetSpace(): + """ + Holds the param-space coordinates (X) and target values (Y) + + Parameters + ---------- + pbounds : dict + Dictionary with parameters names and legal values. + + random_state : int, RandomState, or None + optionally specify a seed for a random number generator, by default None. + """ + + def __init__(self, pbounds, random_state=None): + self._random_state = random_state + + # Get the name of the parameters + self._keys = sorted(pbounds) + + # Create an array with parameters bounds + self._bounds = np.array( + [item[1] for item in sorted(pbounds.items(), key=lambda x: x[0])] + ) + + # check values type + for _bound in self._bounds: + if _bound['_type'] == 'choice': + try: + [float(val) for val in _bound['_value']] + except ValueError: + raise ValueError("GP Tuner supports only numerical values") + + # preallocated memory for X and Y points + self._params = np.empty(shape=(0, self.dim)) + self._target = np.empty(shape=(0)) + + # keep track of unique points we have seen so far + self._cache = {} + + def __contains__(self, params): + """ + check if a parameter is already registered + + Parameters + ---------- + params : numpy array + + Returns + ------- + bool + True if the parameter is already registered, else false + """ + return _hashable(params) in self._cache + + def len(self): + """ + length of registered params and targets + + Returns + ------- + int + """ + assert len(self._params) == len(self._target) + return len(self._target) + + @property + def params(self): + """ + registered parameters + + Returns + ------- + numpy array + """ + return self._params + + @property + def target(self): + """ + registered target values + + Returns + ------- + numpy array + """ + return self._target + + @property + def dim(self): + """ + dimension of parameters + + Returns + ------- + int + """ + return len(self._keys) + + @property + def keys(self): + """ + keys of parameters + + Returns + ------- + numpy array + """ + return self._keys + + @property + def bounds(self): + """ + bounds of parameters + + Returns + ------- + numpy array + """ + return self._bounds + + def params_to_array(self, params): + """ + dict to array + + Parameters + ---------- + params : dict + dict format of parameters + + Returns + ------- + numpy array + array format of parameters + """ + try: + assert set(params) == set(self.keys) + except AssertionError: + raise ValueError( + "Parameters' keys ({}) do ".format(sorted(params)) + + "not match the expected set of keys ({}).".format(self.keys) + ) + return np.asarray([params[key] for key in self.keys]) + + def array_to_params(self, x): + """ + array to dict + + maintain int type if the paramters is defined as int in search_space.json + Parameters + ---------- + x : numpy array + array format of parameters + + Returns + ------- + dict + dict format of parameters + """ + try: + assert len(x) == len(self.keys) + except AssertionError: + raise ValueError( + "Size of array ({}) is different than the ".format(len(x)) + + "expected number of parameters ({}).".format(self.dim) + ) + + params = {} + for i, _bound in enumerate(self._bounds): + if _bound['_type'] == 'choice' and all(isinstance(val, int) for val in _bound['_value']): + params.update({self.keys[i]: int(x[i])}) + elif _bound['_type'] in ['randint']: + params.update({self.keys[i]: int(x[i])}) + else: + params.update({self.keys[i]: x[i]}) + + return params + + def register(self, params, target): + """ + Append a point and its target value to the known data. + + Parameters + ---------- + params : dict + parameters + + target : float + target function value + """ + + x = self.params_to_array(params) + if x in self: + print('Data point {} is not unique'.format(x)) + + # Insert data into unique dictionary + self._cache[_hashable(x.ravel())] = target + + self._params = np.concatenate([self._params, x.reshape(1, -1)]) + self._target = np.concatenate([self._target, [target]]) + + def random_sample(self): + """ + Creates a random point within the bounds of the space. + + Returns + ------- + numpy array + one groupe of parameter + """ + params = np.empty(self.dim) + for col, _bound in enumerate(self._bounds): + if _bound['_type'] == 'choice': + params[col] = parameter_expressions.choice( + _bound['_value'], self._random_state) + elif _bound['_type'] == 'randint': + params[col] = self._random_state.randint( + _bound['_value'][0], _bound['_value'][1], size=1) + elif _bound['_type'] == 'uniform': + params[col] = parameter_expressions.uniform( + _bound['_value'][0], _bound['_value'][1], self._random_state) + elif _bound['_type'] == 'quniform': + params[col] = parameter_expressions.quniform( + _bound['_value'][0], _bound['_value'][1], _bound['_value'][2], self._random_state) + elif _bound['_type'] == 'loguniform': + params[col] = parameter_expressions.loguniform( + _bound['_value'][0], _bound['_value'][1], self._random_state) + elif _bound['_type'] == 'qloguniform': + params[col] = parameter_expressions.qloguniform( + _bound['_value'][0], _bound['_value'][1], _bound['_value'][2], self._random_state) + + return params + + def max(self): + """ + Get maximum target value found and its corresponding parameters. + + Returns + ------- + dict + target value and parameters, empty dict if nothing registered + """ + try: + res = { + 'target': self.target.max(), + 'params': dict( + zip(self.keys, self.params[self.target.argmax()]) + ) + } + except ValueError: + res = {} + return res + + def res(self): + """ + Get all target values found and corresponding parameters. + + Returns + ------- + list + a list of target values and their corresponding parameters + """ + params = [dict(zip(self.keys, p)) for p in self.params] + + return [ + {"target": target, "params": param} + for target, param in zip(self.target, params) + ] diff --git a/utils/third_party/nni_new/algorithms/hpo/gp_tuner/util.py b/utils/third_party/nni_new/algorithms/hpo/gp_tuner/util.py new file mode 100644 index 0000000000000000000000000000000000000000..6926f988997a1e0b1d0bd6286dd7f091c38fc8da --- /dev/null +++ b/utils/third_party/nni_new/algorithms/hpo/gp_tuner/util.py @@ -0,0 +1,235 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +utility functions and classes for GPTuner +""" + +import warnings +import numpy as np +from scipy.stats import norm +from scipy.optimize import minimize + + +def _match_val_type(vals, bounds): + """ + Update values in the array, to match their corresponding type, make sure the value is legal. + + Parameters + ---------- + vals : numpy array + values of parameters + bounds : numpy array + list of dictionary which stores parameters names and legal values. + + Returns + ------- + vals_new : list + The closest legal value to the original value + """ + vals_new = [] + + for i, bound in enumerate(bounds): + _type = bound['_type'] + if _type == "choice": + # Find the closest integer in the array, vals_bounds + # pylint: disable=cell-var-from-loop + vals_new.append(min(bound['_value'], key=lambda x: abs(x - vals[i]))) + elif _type in ['quniform', 'randint']: + vals_new.append(np.around(vals[i])) + else: + vals_new.append(vals[i]) + + return vals_new + + +def acq_max(f_acq, gp, y_max, bounds, space, num_warmup, num_starting_points): + """ + A function to find the maximum of the acquisition function + + It uses a combination of random sampling (cheap) and the 'L-BFGS-B' + optimization method. First by sampling ``num_warmup`` points at random, + and then running L-BFGS-B from ``num_starting_points`` random starting points. + + Parameters + ---------- + f_acq : UtilityFunction.utility + The acquisition function object that return its point-wise value. + + gp : GaussianProcessRegressor + A gaussian process fitted to the relevant data. + + y_max : float + The current maximum known value of the target function. + + bounds : numpy array + The variables bounds to limit the search of the acq max. + + num_warmup : int + number of times to randomly sample the aquisition function + + num_starting_points : int + number of times to run scipy.minimize + + Returns + ------- + numpy array + The parameter which achieves max of the acquisition function. + """ + + # Warm up with random points + x_tries = [space.random_sample() + for _ in range(int(num_warmup))] + ys = f_acq(x_tries, gp=gp, y_max=y_max) + x_max = x_tries[ys.argmax()] + max_acq = ys.max() + + + # Explore the parameter space more throughly + x_seeds = [space.random_sample() for _ in range(int(num_starting_points))] + + bounds_minmax = np.array( + [[bound['_value'][0], bound['_value'][-1]] for bound in bounds]) + + for x_try in x_seeds: + # Find the minimum of minus the acquisition function + res = minimize(lambda x: -f_acq(x.reshape(1, -1), gp=gp, y_max=y_max), + x_try.reshape(1, -1), + bounds=bounds_minmax, + method="L-BFGS-B") + + # See if success + if not res.success: + continue + + # Store it if better than previous minimum(maximum). + if max_acq is None or -res.fun[0] >= max_acq: + x_max = _match_val_type(res.x, bounds) + max_acq = -res.fun[0] + + # Clip output to make sure it lies within the bounds. Due to floating + # point technicalities this is not always the case. + return np.clip(x_max, bounds_minmax[:, 0], bounds_minmax[:, 1]) + + +class UtilityFunction(): + """ + A class to compute different acquisition function values. + + Parameters + ---------- + kind : string + specification of utility function to use + kappa : float + parameter usedd for 'ucb' acquisition function + xi : float + parameter usedd for 'ei' and 'poi' acquisition function + """ + + def __init__(self, kind, kappa, xi): + self._kappa = kappa + self._xi = xi + + if kind not in ['ucb', 'ei', 'poi']: + err = "The utility function " \ + "{} has not been implemented, " \ + "please choose one of ucb, ei, or poi.".format(kind) + raise NotImplementedError(err) + self._kind = kind + + def utility(self, x, gp, y_max): + """ + return utility function + + Parameters + ---------- + x : numpy array + parameters + gp : GaussianProcessRegressor + y_max : float + maximum target value observed so far + + Returns + ------- + function + return corresponding function, return None if parameter is illegal + """ + if self._kind == 'ucb': + return self._ucb(x, gp, self._kappa) + if self._kind == 'ei': + return self._ei(x, gp, y_max, self._xi) + if self._kind == 'poi': + return self._poi(x, gp, y_max, self._xi) + return None + + @staticmethod + def _ucb(x, gp, kappa): + """ + Upper Confidence Bound (UCB) utility function + + Parameters + ---------- + x : numpy array + parameters + gp : GaussianProcessRegressor + kappa : float + + Returns + ------- + float + """ + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + mean, std = gp.predict(x, return_std=True) + + return mean + kappa * std + + @staticmethod + def _ei(x, gp, y_max, xi): + """ + Expected Improvement (EI) utility function + + Parameters + ---------- + x : numpy array + parameters + gp : GaussianProcessRegressor + y_max : float + maximum target value observed so far + xi : float + + Returns + ------- + float + """ + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + mean, std = gp.predict(x, return_std=True) + + z = (mean - y_max - xi)/std + return (mean - y_max - xi) * norm.cdf(z) + std * norm.pdf(z) + + @staticmethod + def _poi(x, gp, y_max, xi): + """ + Possibility Of Improvement (POI) utility function + + Parameters + ---------- + x : numpy array + parameters + gp : GaussianProcessRegressor + y_max : float + maximum target value observed so far + xi : float + + Returns + ------- + float + """ + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + mean, std = gp.predict(x, return_std=True) + + z = (mean - y_max - xi)/std + return norm.cdf(z) diff --git a/utils/third_party/nni_new/algorithms/hpo/gridsearch_tuner.py b/utils/third_party/nni_new/algorithms/hpo/gridsearch_tuner.py new file mode 100644 index 0000000000000000000000000000000000000000..4787b3f0eebfdf44a0fc52fd362c328ec0708f07 --- /dev/null +++ b/utils/third_party/nni_new/algorithms/hpo/gridsearch_tuner.py @@ -0,0 +1,208 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +gridsearch_tuner.py including: + class GridSearchTuner +""" + +import copy +import logging +import numpy as np + +import nni +from nni.tuner import Tuner +from nni.utils import convert_dict2tuple + +TYPE = '_type' +CHOICE = 'choice' +VALUE = '_value' + +logger = logging.getLogger('grid_search_AutoML') + +class GridSearchTuner(Tuner): + """ + GridSearchTuner will search all the possible configures that the user define in the searchSpace. + The only acceptable types of search space are ``choice``, ``quniform``, ``randint`` + + Type ``choice`` will select one of the options. Note that it can also be nested. + + Type ``quniform`` will receive three values [``low``, ``high``, ``q``], + where [``low``, ``high``] specifies a range and ``q`` specifies the interval. + It will be sampled in a way that the first sampled value is ``low``, + and each of the following values is 'interval' larger than the value in front of it. + + Type ``randint`` gives all possible intergers in range[``low``, ``high``). Note that ``high`` is not included. + """ + + def __init__(self): + self.count = -1 + self.expanded_search_space = [] + self.supplement_data = dict() + + def _json2parameter(self, ss_spec): + """ + Generate all possible configs for hyperparameters from hyperparameter space. + + Parameters + ---------- + ss_spec : dict or list + Hyperparameter space or the ``_value`` of a hyperparameter + + Returns + ------- + list or dict + All the candidate choices of hyperparameters. for a hyperparameter, chosen_params + is a list. for multiple hyperparameters (e.g., search space), chosen_params is a dict. + """ + if isinstance(ss_spec, dict): + if '_type' in ss_spec.keys(): + _type = ss_spec['_type'] + _value = ss_spec['_value'] + chosen_params = list() + if _type == 'choice': + for value in _value: + choice = self._json2parameter(value) + if isinstance(choice, list): + chosen_params.extend(choice) + else: + chosen_params.append(choice) + elif _type == 'quniform': + chosen_params = self._parse_quniform(_value) + elif _type == 'randint': + chosen_params = self._parse_randint(_value) + else: + raise RuntimeError("Not supported type: %s" % _type) + else: + chosen_params = dict() + for key in ss_spec.keys(): + chosen_params[key] = self._json2parameter(ss_spec[key]) + return self._expand_parameters(chosen_params) + elif isinstance(ss_spec, list): + chosen_params = list() + for subspec in ss_spec[1:]: + choice = self._json2parameter(subspec) + if isinstance(choice, list): + chosen_params.extend(choice) + else: + chosen_params.append(choice) + chosen_params = list(map(lambda v: {ss_spec[0]: v}, chosen_params)) + else: + chosen_params = copy.deepcopy(ss_spec) + return chosen_params + + def _parse_quniform(self, param_value): + """ + Parse type of quniform parameter and return a list + """ + low, high, q = param_value[0], param_value[1], param_value[2] + return np.clip(np.arange(np.round(low/q), np.round(high/q)+1) * q, low, high) + + def _parse_randint(self, param_value): + """ + Parse type of randint parameter and return a list + """ + if param_value[0] >= param_value[1]: + raise ValueError("Randint should contain at least 1 candidate, but [%s, %s) contains none.", + param_value[0], param_value[1]) + return np.arange(param_value[0], param_value[1]).tolist() + + def _expand_parameters(self, para): + """ + Enumerate all possible combinations of all parameters + + Parameters + ---------- + para : dict + {key1: [v11, v12, ...], key2: [v21, v22, ...], ...} + + Returns + ------- + dict + {{key1: v11, key2: v21, ...}, {key1: v11, key2: v22, ...}, ...} + """ + if len(para) == 1: + for key, values in para.items(): + return list(map(lambda v: {key: v}, values)) + + key = list(para)[0] + values = para.pop(key) + rest_para = self._expand_parameters(para) + ret_para = list() + for val in values: + for config in rest_para: + config[key] = val + ret_para.append(copy.deepcopy(config)) + return ret_para + + def update_search_space(self, search_space): + """ + Check if the search space is valid and expand it: support only ``choice``, ``quniform``, ``randint``. + + Parameters + ---------- + search_space : dict + The format could be referred to search space spec (https://nni.readthedocs.io/en/latest/Tutorial/SearchSpaceSpec.html). + """ + self.expanded_search_space = self._json2parameter(search_space) + + def generate_parameters(self, parameter_id, **kwargs): + """ + Generate parameters for one trial. + + Parameters + ---------- + parameter_id : int + The id for the generated hyperparameter + **kwargs + Not used + + Returns + ------- + dict + One configuration from the expanded search space. + + Raises + ------ + NoMoreTrialError + If all the configurations has been sent, raise :class:`~nni.NoMoreTrialError`. + """ + self.count += 1 + while self.count <= len(self.expanded_search_space) - 1: + _params_tuple = convert_dict2tuple(copy.deepcopy(self.expanded_search_space[self.count])) + if _params_tuple in self.supplement_data: + self.count += 1 + else: + return self.expanded_search_space[self.count] + raise nni.NoMoreTrialError('no more parameters now.') + + def receive_trial_result(self, parameter_id, parameters, value, **kwargs): + """ + Receive a trial's final performance result reported through :func:`~nni.report_final_result` by the trial. + GridSearchTuner does not need trial's results. + """ + pass + + def import_data(self, data): + """ + Import additional data for tuning + + Parameters + ---------- + list + A list of dictionarys, each of which has at least two keys, ``parameter`` and ``value`` + """ + _completed_num = 0 + for trial_info in data: + logger.info("Importing data, current processing progress %s / %s", _completed_num, len(data)) + _completed_num += 1 + assert "parameter" in trial_info + _params = trial_info["parameter"] + assert "value" in trial_info + _value = trial_info['value'] + if not _value: + logger.info("Useless trial data, value is %s, skip this trial data.", _value) + continue + _params_tuple = convert_dict2tuple(copy.deepcopy(_params)) + self.supplement_data[_params_tuple] = True + logger.info("Successfully import data to grid search tuner.") diff --git a/utils/third_party/nni_new/algorithms/hpo/hyperband_advisor.py b/utils/third_party/nni_new/algorithms/hpo/hyperband_advisor.py new file mode 100644 index 0000000000000000000000000000000000000000..10b9b643bc1afbdf5d3988d6664e5925f3c2c371 --- /dev/null +++ b/utils/third_party/nni_new/algorithms/hpo/hyperband_advisor.py @@ -0,0 +1,464 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +hyperband_advisor.py +""" + +import copy +import logging +import math +import sys + +import json_tricks +import numpy as np +from schema import Schema, Optional + +from nni import ClassArgsValidator +from nni.runtime.common import multi_phase_enabled +from nni.runtime.msg_dispatcher_base import MsgDispatcherBase +from nni.runtime.protocol import CommandType, send +from nni.utils import NodeType, OptimizeMode, MetricType, extract_scalar_reward +from nni import parameter_expressions + +_logger = logging.getLogger(__name__) + +_next_parameter_id = 0 +_KEY = 'TRIAL_BUDGET' +_epsilon = 1e-6 + + +def create_parameter_id(): + """Create an id + + Returns + ------- + int + parameter id + """ + global _next_parameter_id + _next_parameter_id += 1 + return _next_parameter_id - 1 + + +def create_bracket_parameter_id(brackets_id, brackets_curr_decay, increased_id=-1): + """Create a full id for a specific bracket's hyperparameter configuration + + Parameters + ---------- + brackets_id: string + brackets id + brackets_curr_decay: + brackets curr decay + increased_id: int + increased id + + Returns + ------- + int + params id + """ + if increased_id == -1: + increased_id = str(create_parameter_id()) + params_id = '_'.join([brackets_id, + str(brackets_curr_decay), + increased_id]) + return params_id + + +def json2parameter(ss_spec, random_state): + """Randomly generate values for hyperparameters from hyperparameter space i.e., x. + + Parameters + ---------- + ss_spec: + hyperparameter space + random_state: + random operator to generate random values + + Returns + ------- + Parameter: + Parameters in this experiment + """ + if isinstance(ss_spec, dict): + if NodeType.TYPE in ss_spec.keys(): + _type = ss_spec[NodeType.TYPE] + _value = ss_spec[NodeType.VALUE] + if _type == 'choice': + _index = random_state.randint(len(_value)) + chosen_params = json2parameter(ss_spec[NodeType.VALUE][_index], random_state) + else: + chosen_params = getattr(parameter_expressions, _type)(*(_value + [random_state])) + else: + chosen_params = dict() + for key in ss_spec.keys(): + chosen_params[key] = json2parameter(ss_spec[key], random_state) + elif isinstance(ss_spec, list): + chosen_params = list() + for _, subspec in enumerate(ss_spec): + chosen_params.append(json2parameter(subspec, random_state)) + else: + chosen_params = copy.deepcopy(ss_spec) + return chosen_params + + +class Bracket(): + """A bracket in Hyperband, all the information of a bracket is managed by an instance of this class + + Parameters + ---------- + bracket_id: string + The id of this bracket, usually be set as '{Hyperband index}-{SH iteration index}' + s: int + The current SH iteration index. + s_max: int + total number of SH iterations + eta: float + In each iteration, a complete run of sequential halving is executed. In it, + after evaluating each configuration on the same subset size, only a fraction of + 1/eta of them 'advances' to the next round. + R: + the budget associated with each stage + optimize_mode: str + optimize mode, 'maximize' or 'minimize' + """ + + def __init__(self, bracket_id, s, s_max, eta, R, optimize_mode): + self.bracket_id = bracket_id + self.s = s + self.s_max = s_max + self.eta = eta + self.n = math.ceil((s_max + 1) * (eta ** s) / (s + 1) - _epsilon) + self.r = R / eta ** s + self.i = 0 + self.hyper_configs = [] # [ {id: params}, {}, ... ] + self.configs_perf = [] # [ {id: [seq, acc]}, {}, ... ] + self.num_configs_to_run = [] # [ n, n, n, ... ] + self.num_finished_configs = [] # [ n, n, n, ... ] + self.optimize_mode = OptimizeMode(optimize_mode) + self.no_more_trial = False + + def is_completed(self): + """check whether this bracket has sent out all the hyperparameter configurations""" + return self.no_more_trial + + def get_n_r(self): + """return the values of n and r for the next round""" + return math.floor(self.n / self.eta ** self.i + _epsilon), math.floor(self.r * self.eta ** self.i + _epsilon) + + def increase_i(self): + """i means the ith round. Increase i by 1""" + self.i += 1 + if self.i > self.s: + self.no_more_trial = True + + def set_config_perf(self, i, parameter_id, seq, value): + """update trial's latest result with its sequence number, e.g., epoch number or batch number + + Parameters + ---------- + i: int + the ith round + parameter_id: int + the id of the trial/parameter + seq: int + sequence number, e.g., epoch number or batch number + value: int + latest result with sequence number seq + + Returns + ------- + None + """ + if parameter_id in self.configs_perf[i]: + if self.configs_perf[i][parameter_id][0] < seq: + self.configs_perf[i][parameter_id] = [seq, value] + else: + self.configs_perf[i][parameter_id] = [seq, value] + + def inform_trial_end(self, i): + """If the trial is finished and the corresponding round (i.e., i) has all its trials finished, + it will choose the top k trials for the next round (i.e., i+1) + + Parameters + ---------- + i: int + the ith round + """ + global _KEY + self.num_finished_configs[i] += 1 + _logger.debug('bracket id: %d, round: %d %d, finished: %d, all: %d', self.bracket_id, self.i, i, + self.num_finished_configs[i], self.num_configs_to_run[i]) + if self.num_finished_configs[i] >= self.num_configs_to_run[i] \ + and self.no_more_trial is False: + # choose candidate configs from finished configs to run in the next round + assert self.i == i + 1 + this_round_perf = self.configs_perf[i] + if self.optimize_mode is OptimizeMode.Maximize: + sorted_perf = sorted(this_round_perf.items(), key=lambda kv: kv[1][1], reverse=True) # reverse + else: + sorted_perf = sorted(this_round_perf.items(), key=lambda kv: kv[1][1]) + _logger.debug('bracket %s next round %s, sorted hyper configs: %s', self.bracket_id, self.i, sorted_perf) + next_n, next_r = self.get_n_r() + _logger.debug('bracket %s next round %s, next_n=%d, next_r=%d', self.bracket_id, self.i, next_n, next_r) + hyper_configs = dict() + for k in range(next_n): + params_id = sorted_perf[k][0] + params = self.hyper_configs[i][params_id] + params[_KEY] = next_r # modify r + # generate new id + increased_id = params_id.split('_')[-1] + new_id = create_bracket_parameter_id(self.bracket_id, self.i, increased_id) + hyper_configs[new_id] = params + self._record_hyper_configs(hyper_configs) + return [[key, value] for key, value in hyper_configs.items()] + return None + + def get_hyperparameter_configurations(self, num, r, searchspace_json, random_state): + """Randomly generate num hyperparameter configurations from search space + + Parameters + ---------- + num: int + the number of hyperparameter configurations + + Returns + ------- + list + a list of hyperparameter configurations. Format: [[key1, value1], [key2, value2], ...] + """ + global _KEY + assert self.i == 0 + hyperparameter_configs = dict() + for _ in range(num): + params_id = create_bracket_parameter_id(self.bracket_id, self.i) + params = json2parameter(searchspace_json, random_state) + params[_KEY] = r + hyperparameter_configs[params_id] = params + self._record_hyper_configs(hyperparameter_configs) + return [[key, value] for key, value in hyperparameter_configs.items()] + + def _record_hyper_configs(self, hyper_configs): + """after generating one round of hyperconfigs, this function records the generated hyperconfigs, + creates a dict to record the performance when those hyperconifgs are running, set the number of finished configs + in this round to be 0, and increase the round number. + + Parameters + ---------- + hyper_configs: list + the generated hyperconfigs + """ + self.hyper_configs.append(hyper_configs) + self.configs_perf.append(dict()) + self.num_finished_configs.append(0) + self.num_configs_to_run.append(len(hyper_configs)) + self.increase_i() + +class HyperbandClassArgsValidator(ClassArgsValidator): + def validate_class_args(self, **kwargs): + Schema({ + 'optimize_mode': self.choices('optimize_mode', 'maximize', 'minimize'), + Optional('exec_mode'): self.choices('exec_mode', 'serial', 'parallelism'), + Optional('R'): int, + Optional('eta'): int + }).validate(kwargs) + +class Hyperband(MsgDispatcherBase): + """Hyperband inherit from MsgDispatcherBase rather than Tuner, because it integrates both tuner's functions and assessor's functions. + This is an implementation that could fully leverage available resources or follow the algorithm process, i.e., high parallelism or serial. + A single execution of Hyperband takes a finite budget of (s_max + 1)B. + + Parameters + ---------- + R: int + the maximum amount of resource that can be allocated to a single configuration + eta: int + the variable that controls the proportion of configurations discarded in each round of SuccessiveHalving + optimize_mode: str + optimize mode, 'maximize' or 'minimize' + exec_mode: str + execution mode, 'serial' or 'parallelism' + """ + + def __init__(self, R=60, eta=3, optimize_mode='maximize', exec_mode='parallelism'): + """B = (s_max + 1)R""" + super(Hyperband, self).__init__() + self.R = R + self.eta = eta + self.brackets = dict() # dict of Bracket + self.generated_hyper_configs = [] # all the configs waiting for run + self.completed_hyper_configs = [] # all the completed configs + self.s_max = math.floor(math.log(self.R, self.eta) + _epsilon) + self.curr_s = self.s_max + self.curr_hb = 0 + self.exec_mode = exec_mode + self.curr_bracket_id = None + + self.searchspace_json = None + self.random_state = None + self.optimize_mode = OptimizeMode(optimize_mode) + + # This is for the case that nnimanager requests trial config, but tuner cannot provide immediately. + # In this case, tuner increases self.credit to issue a trial config sometime later. + self.credit = 0 + + # record the latest parameter_id of the trial job trial_job_id. + # if there is no running parameter_id, self.job_id_para_id_map[trial_job_id] == None + # new trial job is added to this dict and finished trial job is removed from it. + self.job_id_para_id_map = dict() + + def handle_initialize(self, data): + """callback for initializing the advisor + Parameters + ---------- + data: dict + search space + """ + self.handle_update_search_space(data) + send(CommandType.Initialized, '') + + def handle_request_trial_jobs(self, data): + """ + Parameters + ---------- + data: int + number of trial jobs + """ + self.credit += data + + for _ in range(self.credit): + self._request_one_trial_job() + + def _request_one_trial_job(self): + ret = self._get_one_trial_job() + if ret is not None: + send(CommandType.NewTrialJob, json_tricks.dumps(ret)) + self.credit -= 1 + + def _get_one_trial_job(self): + """get one trial job, i.e., one hyperparameter configuration.""" + if not self.generated_hyper_configs: + if self.exec_mode == 'parallelism' or \ + (self.exec_mode == 'serial' and (self.curr_bracket_id is None or self.brackets[self.curr_bracket_id].is_completed())): + if self.curr_s < 0: + self.curr_s = self.s_max + self.curr_hb += 1 + _logger.debug('create a new bracket, self.curr_hb=%d, self.curr_s=%d', self.curr_hb, self.curr_s) + self.curr_bracket_id = '{}-{}'.format(self.curr_hb, self.curr_s) + self.brackets[self.curr_bracket_id] = Bracket(self.curr_bracket_id, self.curr_s, self.s_max, self.eta, self.R, self.optimize_mode) + next_n, next_r = self.brackets[self.curr_bracket_id].get_n_r() + _logger.debug('new bracket, next_n=%d, next_r=%d', next_n, next_r) + assert self.searchspace_json is not None and self.random_state is not None + generated_hyper_configs = self.brackets[self.curr_bracket_id].get_hyperparameter_configurations(next_n, next_r, + self.searchspace_json, + self.random_state) + self.generated_hyper_configs = generated_hyper_configs.copy() + self.curr_s -= 1 + else: + ret = { + 'parameter_id': '-1_0_0', + 'parameter_source': 'algorithm', + 'parameters': '' + } + send(CommandType.NoMoreTrialJobs, json_tricks.dumps(ret)) + return None + + assert self.generated_hyper_configs + params = self.generated_hyper_configs.pop(0) + ret = { + 'parameter_id': params[0], + 'parameter_source': 'algorithm', + 'parameters': params[1] + } + return ret + + def handle_update_search_space(self, data): + """data: JSON object, which is search space + """ + self.searchspace_json = data + self.random_state = np.random.RandomState() + + def _handle_trial_end(self, parameter_id): + """ + Parameters + ---------- + parameter_id: parameter id of the finished config + """ + bracket_id, i, _ = parameter_id.split('_') + hyper_configs = self.brackets[bracket_id].inform_trial_end(int(i)) + if hyper_configs is not None: + _logger.debug('bracket %s next round %s, hyper_configs: %s', bracket_id, i, hyper_configs) + self.generated_hyper_configs = self.generated_hyper_configs + hyper_configs + for _ in range(self.credit): + self._request_one_trial_job() + + def handle_trial_end(self, data): + """ + Parameters + ---------- + data: dict() + it has three keys: trial_job_id, event, hyper_params + trial_job_id: the id generated by training service + event: the job's state + hyper_params: the hyperparameters (a string) generated and returned by tuner + """ + hyper_params = json_tricks.loads(data['hyper_params']) + self._handle_trial_end(hyper_params['parameter_id']) + if data['trial_job_id'] in self.job_id_para_id_map: + del self.job_id_para_id_map[data['trial_job_id']] + + def handle_report_metric_data(self, data): + """ + Parameters + ---------- + data: + it is an object which has keys 'parameter_id', 'value', 'trial_job_id', 'type', 'sequence'. + + Raises + ------ + ValueError + Data type not supported + """ + if 'value' in data: + data['value'] = json_tricks.loads(data['value']) + # multiphase? need to check + if data['type'] == MetricType.REQUEST_PARAMETER: + assert multi_phase_enabled() + assert data['trial_job_id'] is not None + assert data['parameter_index'] is not None + assert data['trial_job_id'] in self.job_id_para_id_map + self._handle_trial_end(self.job_id_para_id_map[data['trial_job_id']]) + ret = self._get_one_trial_job() + if data['trial_job_id'] is not None: + ret['trial_job_id'] = data['trial_job_id'] + if data['parameter_index'] is not None: + ret['parameter_index'] = data['parameter_index'] + self.job_id_para_id_map[data['trial_job_id']] = ret['parameter_id'] + send(CommandType.SendTrialJobParameter, json_tricks.dumps(ret)) + else: + value = extract_scalar_reward(data['value']) + bracket_id, i, _ = data['parameter_id'].split('_') + + # add to self.job_id_para_id_map here, + # because when the first parameter_id is created, trial_job_id is not known yet. + if data['trial_job_id'] in self.job_id_para_id_map: + assert self.job_id_para_id_map[data['trial_job_id']] == data['parameter_id'] + else: + self.job_id_para_id_map[data['trial_job_id']] = data['parameter_id'] + + if data['type'] == MetricType.FINAL: + # sys.maxsize indicates this value is from FINAL metric data, because data['sequence'] from FINAL metric + # and PERIODICAL metric are independent, thus, not comparable. + self.brackets[bracket_id].set_config_perf(int(i), data['parameter_id'], sys.maxsize, value) + self.completed_hyper_configs.append(data) + elif data['type'] == MetricType.PERIODICAL: + self.brackets[bracket_id].set_config_perf(int(i), data['parameter_id'], data['sequence'], value) + else: + raise ValueError('Data type not supported: {}'.format(data['type'])) + + def handle_add_customized_trial(self, data): + pass + + def handle_import_data(self, data): + pass diff --git a/utils/third_party/nni_new/algorithms/hpo/hyperopt_tuner.py b/utils/third_party/nni_new/algorithms/hpo/hyperopt_tuner.py new file mode 100644 index 0000000000000000000000000000000000000000..fbe9cda13faac67c060089f68d35048a4109a789 --- /dev/null +++ b/utils/third_party/nni_new/algorithms/hpo/hyperopt_tuner.py @@ -0,0 +1,499 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +hyperopt_tuner.py +""" + +import copy +import logging + +import hyperopt as hp +import numpy as np +from schema import Optional, Schema +from nni import ClassArgsValidator +from nni.tuner import Tuner +from nni.utils import NodeType, OptimizeMode, extract_scalar_reward, split_index + +logger = logging.getLogger('hyperopt_AutoML') + + +def json2space(in_x, name=NodeType.ROOT): + """ + Change json to search space in hyperopt. + + Parameters + ---------- + in_x : dict/list/str/int/float + The part of json. + name : str + name could be NodeType.ROOT, NodeType.TYPE, NodeType.VALUE or NodeType.INDEX, NodeType.NAME. + """ + out_y = copy.deepcopy(in_x) + if isinstance(in_x, dict): + if NodeType.TYPE in in_x.keys(): + _type = in_x[NodeType.TYPE] + name = name + '-' + _type + _value = json2space(in_x[NodeType.VALUE], name=name) + if _type == 'choice': + out_y = hp.hp.choice(name, _value) + elif _type == 'randint': + out_y = hp.hp.randint(name, _value[1] - _value[0]) + else: + if _type in ['loguniform', 'qloguniform']: + _value[:2] = np.log(_value[:2]) + out_y = getattr(hp.hp, _type)(name, *_value) + else: + out_y = dict() + for key in in_x.keys(): + out_y[key] = json2space(in_x[key], name + '[%s]' % str(key)) + elif isinstance(in_x, list): + out_y = list() + for i, x_i in enumerate(in_x): + if isinstance(x_i, dict): + if NodeType.NAME not in x_i.keys(): + raise RuntimeError( + '\'_name\' key is not found in this nested search space.' + ) + out_y.append(json2space(x_i, name + '[%d]' % i)) + return out_y + + +def json2parameter(in_x, parameter, name=NodeType.ROOT): + """ + Change json to parameters. + """ + out_y = copy.deepcopy(in_x) + if isinstance(in_x, dict): + if NodeType.TYPE in in_x.keys(): + _type = in_x[NodeType.TYPE] + name = name + '-' + _type + if _type == 'choice': + _index = parameter[name] + out_y = { + NodeType.INDEX: + _index, + NodeType.VALUE: + json2parameter(in_x[NodeType.VALUE][_index], + parameter, + name=name + '[%d]' % _index) + } + else: + if _type in ['quniform', 'qloguniform']: + out_y = np.clip(parameter[name], in_x[NodeType.VALUE][0], in_x[NodeType.VALUE][1]) + elif _type == 'randint': + out_y = parameter[name] + in_x[NodeType.VALUE][0] + else: + out_y = parameter[name] + else: + out_y = dict() + for key in in_x.keys(): + out_y[key] = json2parameter(in_x[key], parameter, + name + '[%s]' % str(key)) + elif isinstance(in_x, list): + out_y = list() + for i, x_i in enumerate(in_x): + if isinstance(x_i, dict): + if NodeType.NAME not in x_i.keys(): + raise RuntimeError( + '\'_name\' key is not found in this nested search space.' + ) + out_y.append(json2parameter(x_i, parameter, name + '[%d]' % i)) + return out_y + + +def json2vals(in_x, vals, out_y, name=NodeType.ROOT): + if isinstance(in_x, dict): + if NodeType.TYPE in in_x.keys(): + _type = in_x[NodeType.TYPE] + name = name + '-' + _type + + try: + out_y[name] = vals[NodeType.INDEX] + # TODO - catch exact Exception + except Exception: + out_y[name] = vals + + if _type == 'choice': + _index = vals[NodeType.INDEX] + json2vals(in_x[NodeType.VALUE][_index], + vals[NodeType.VALUE], + out_y, + name=name + '[%d]' % _index) + if _type == 'randint': + out_y[name] -= in_x[NodeType.VALUE][0] + else: + for key in in_x.keys(): + json2vals(in_x[key], vals[key], out_y, + name + '[%s]' % str(key)) + elif isinstance(in_x, list): + for i, temp in enumerate(in_x): + # nested json + if isinstance(temp, dict): + if NodeType.NAME not in temp.keys(): + raise RuntimeError( + '\'_name\' key is not found in this nested search space.' + ) + else: + json2vals(temp, vals[i], out_y, name + '[%d]' % i) + else: + json2vals(temp, vals[i], out_y, name + '[%d]' % i) + + +def _add_index(in_x, parameter): + """ + change parameters in NNI format to parameters in hyperopt format(This function also support nested dict.). + For example, receive parameters like: + {'dropout_rate': 0.8, 'conv_size': 3, 'hidden_size': 512} + Will change to format in hyperopt, like: + {'dropout_rate': 0.8, 'conv_size': {'_index': 1, '_value': 3}, 'hidden_size': {'_index': 1, '_value': 512}} + """ + if NodeType.TYPE not in in_x: # if at the top level + out_y = dict() + for key, value in parameter.items(): + out_y[key] = _add_index(in_x[key], value) + return out_y + elif isinstance(in_x, dict): + value_type = in_x[NodeType.TYPE] + value_format = in_x[NodeType.VALUE] + if value_type == "choice": + choice_name = parameter[0] if isinstance(parameter, + list) else parameter + for pos, item in enumerate( + value_format): # here value_format is a list + if isinstance( + item, + list): # this format is ["choice_key", format_dict] + choice_key = item[0] + choice_value_format = item[1] + if choice_key == choice_name: + return { + NodeType.INDEX: pos, + NodeType.VALUE: [ + choice_name, + _add_index(choice_value_format, parameter[1]) + ] + } + elif choice_name == item: + return {NodeType.INDEX: pos, NodeType.VALUE: item} + else: + return parameter + return None # note: this is not written by original author, feel free to modify if you think it's incorrect + +class HyperoptClassArgsValidator(ClassArgsValidator): + def validate_class_args(self, **kwargs): + Schema({ + Optional('optimize_mode'): self.choices('optimize_mode', 'maximize', 'minimize'), + Optional('parallel_optimize'): bool, + Optional('constant_liar_type'): self.choices('constant_liar_type', 'min', 'max', 'mean') + }).validate(kwargs) + +class HyperoptTuner(Tuner): + """ + HyperoptTuner is a tuner which using hyperopt algorithm. + """ + + def __init__(self, algorithm_name, optimize_mode='minimize', + parallel_optimize=False, constant_liar_type='min'): + """ + Parameters + ---------- + algorithm_name : str + algorithm_name includes "tpe", "random_search" and anneal". + optimize_mode : str + parallel_optimize : bool + More detail could reference: docs/en_US/Tuner/HyperoptTuner.md + constant_liar_type : str + constant_liar_type including "min", "max" and "mean" + More detail could reference: docs/en_US/Tuner/HyperoptTuner.md + """ + self.algorithm_name = algorithm_name + self.optimize_mode = OptimizeMode(optimize_mode) + self.json = None + self.total_data = {} + self.rval = None + self.supplement_data_num = 0 + + self.parallel = parallel_optimize + if self.parallel: + self.CL_rval = None + self.constant_liar_type = constant_liar_type + self.running_data = [] + self.optimal_y = None + + def _choose_tuner(self, algorithm_name): + """ + Parameters + ---------- + algorithm_name : str + algorithm_name includes "tpe", "random_search" and anneal" + """ + if algorithm_name == 'tpe': + return hp.tpe.suggest + if algorithm_name == 'random_search': + return hp.rand.suggest + if algorithm_name == 'anneal': + return hp.anneal.suggest + raise RuntimeError('Not support tuner algorithm in hyperopt.') + + def update_search_space(self, search_space): + """ + Update search space definition in tuner by search_space in parameters. + + Will called when first setup experiemnt or update search space in WebUI. + + Parameters + ---------- + search_space : dict + """ + self.json = search_space + + search_space_instance = json2space(self.json) + rstate = np.random.RandomState() + trials = hp.Trials() + domain = hp.Domain(None, + search_space_instance, + pass_expr_memo_ctrl=None) + algorithm = self._choose_tuner(self.algorithm_name) + self.rval = hp.FMinIter(algorithm, + domain, + trials, + max_evals=-1, + rstate=rstate, + verbose=0) + self.rval.catch_eval_exceptions = False + + def generate_parameters(self, parameter_id, **kwargs): + """ + Returns a set of trial (hyper-)parameters, as a serializable object. + + Parameters + ---------- + parameter_id : int + + Returns + ------- + params : dict + """ + total_params = self.get_suggestion(random_search=False) + # avoid generating same parameter with concurrent trials because hyperopt doesn't support parallel mode + if total_params in self.total_data.values(): + # but it can cause duplicate parameter rarely + total_params = self.get_suggestion(random_search=True) + self.total_data[parameter_id] = total_params + + if self.parallel: + self.running_data.append(parameter_id) + + params = split_index(total_params) + return params + + def receive_trial_result(self, parameter_id, parameters, value, **kwargs): + """ + Record an observation of the objective function + + Parameters + ---------- + parameter_id : int + parameters : dict + value : dict/float + if value is dict, it should have "default" key. + value is final metrics of the trial. + """ + reward = extract_scalar_reward(value) + # restore the paramsters contains '_index' + if parameter_id not in self.total_data: + raise RuntimeError('Received parameter_id not in total_data.') + params = self.total_data[parameter_id] + + # code for parallel + if self.parallel: + constant_liar = kwargs.get('constant_liar', False) + + if constant_liar: + rval = self.CL_rval + else: + rval = self.rval + # ignore duplicated reported final result (due to aware of intermedate result) + if parameter_id not in self.running_data: + logger.info("Received duplicated final result with parameter id: %s", parameter_id) + return + self.running_data.remove(parameter_id) + + # update the reward of optimal_y + if self.optimal_y is None: + if self.constant_liar_type == 'mean': + self.optimal_y = [reward, 1] + else: + self.optimal_y = reward + else: + if self.constant_liar_type == 'mean': + _sum = self.optimal_y[0] + reward + _number = self.optimal_y[1] + 1 + self.optimal_y = [_sum, _number] + elif self.constant_liar_type == 'min': + self.optimal_y = min(self.optimal_y, reward) + elif self.constant_liar_type == 'max': + self.optimal_y = max(self.optimal_y, reward) + logger.debug("Update optimal_y with reward, optimal_y = %s", self.optimal_y) + else: + rval = self.rval + + + if self.optimize_mode is OptimizeMode.Maximize: + reward = -reward + + domain = rval.domain + trials = rval.trials + + new_id = len(trials) + + rval_specs = [None] + rval_results = [domain.new_result()] + rval_miscs = [dict(tid=new_id, cmd=domain.cmd, workdir=domain.workdir)] + + vals = params + idxs = dict() + + out_y = dict() + json2vals(self.json, vals, out_y) + vals = out_y + for key in domain.params: + if key in [NodeType.VALUE, NodeType.INDEX]: + continue + if key not in vals or vals[key] is None or vals[key] == []: + idxs[key] = vals[key] = [] + else: + idxs[key] = [new_id] + vals[key] = [vals[key]] + + self.miscs_update_idxs_vals(rval_miscs, + idxs, + vals, + idxs_map={new_id: new_id}, + assert_all_vals_used=False) + + trial = trials.new_trial_docs([new_id], rval_specs, rval_results, + rval_miscs)[0] + trial['result'] = {'loss': reward, 'status': 'ok'} + trial['state'] = hp.JOB_STATE_DONE + trials.insert_trial_docs([trial]) + trials.refresh() + + def miscs_update_idxs_vals(self, + miscs, + idxs, + vals, + assert_all_vals_used=True, + idxs_map=None): + """ + Unpack the idxs-vals format into the list of dictionaries that is + `misc`. + + Parameters + ---------- + idxs_map : dict + idxs_map is a dictionary of id->id mappings so that the misc['idxs'] can + contain different numbers than the idxs argument. + """ + if idxs_map is None: + idxs_map = {} + + assert set(idxs.keys()) == set(vals.keys()) + + misc_by_id = {m['tid']: m for m in miscs} + for m in miscs: + m['idxs'] = {key: [] for key in idxs} + m['vals'] = {key: [] for key in idxs} + + for key in idxs: + assert len(idxs[key]) == len(vals[key]) + for tid, val in zip(idxs[key], vals[key]): + tid = idxs_map.get(tid, tid) + if assert_all_vals_used or tid in misc_by_id: + misc_by_id[tid]['idxs'][key] = [tid] + misc_by_id[tid]['vals'][key] = [val] + + def get_suggestion(self, random_search=False): + """ + get suggestion from hyperopt + + Parameters + ---------- + random_search : bool + flag to indicate random search or not (default: {False}) + + Returns + ---------- + total_params : dict + parameter suggestion + """ + if self.parallel and len(self.total_data) > 20 and self.running_data and self.optimal_y is not None: + self.CL_rval = copy.deepcopy(self.rval) + if self.constant_liar_type == 'mean': + _constant_liar_y = self.optimal_y[0] / self.optimal_y[1] + else: + _constant_liar_y = self.optimal_y + for _parameter_id in self.running_data: + self.receive_trial_result(parameter_id=_parameter_id, parameters=None, value=_constant_liar_y, constant_liar=True) + rval = self.CL_rval + + random_state = np.random.randint(2**31 - 1) + else: + rval = self.rval + random_state = rval.rstate.randint(2**31 - 1) + + trials = rval.trials + algorithm = rval.algo + new_ids = rval.trials.new_trial_ids(1) + rval.trials.refresh() + + if random_search: + new_trials = hp.rand.suggest(new_ids, rval.domain, trials, + random_state) + else: + new_trials = algorithm(new_ids, rval.domain, trials, random_state) + rval.trials.refresh() + vals = new_trials[0]['misc']['vals'] + parameter = dict() + for key in vals: + try: + parameter[key] = vals[key][0].item() + except (KeyError, IndexError): + parameter[key] = None + + # remove '_index' from json2parameter and save params-id + total_params = json2parameter(self.json, parameter) + return total_params + + def import_data(self, data): + """ + Import additional data for tuning + + Parameters + ---------- + data: + a list of dictionarys, each of which has at least two keys, 'parameter' and 'value' + """ + _completed_num = 0 + for trial_info in data: + logger.info("Importing data, current processing progress %s / %s", _completed_num, len(data)) + _completed_num += 1 + if self.algorithm_name == 'random_search': + return + assert "parameter" in trial_info + _params = trial_info["parameter"] + assert "value" in trial_info + _value = trial_info['value'] + if not _value: + logger.info("Useless trial data, value is %s, skip this trial data.", _value) + continue + self.supplement_data_num += 1 + _parameter_id = '_'.join( + ["ImportData", str(self.supplement_data_num)]) + self.total_data[_parameter_id] = _add_index(in_x=self.json, + parameter=_params) + self.receive_trial_result(parameter_id=_parameter_id, + parameters=_params, + value=_value) + logger.info("Successfully import data to TPE/Anneal tuner.") diff --git a/utils/third_party/nni_new/algorithms/hpo/medianstop_assessor.py b/utils/third_party/nni_new/algorithms/hpo/medianstop_assessor.py new file mode 100644 index 0000000000000000000000000000000000000000..56eb82a3c98e0ce65efaa794fa6ebea8ee989566 --- /dev/null +++ b/utils/third_party/nni_new/algorithms/hpo/medianstop_assessor.py @@ -0,0 +1,125 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +from schema import Schema, Optional + +from nni import ClassArgsValidator +from nni.assessor import Assessor, AssessResult +from nni.utils import extract_scalar_history + +logger = logging.getLogger('medianstop_Assessor') + +class MedianstopClassArgsValidator(ClassArgsValidator): + def validate_class_args(self, **kwargs): + Schema({ + Optional('optimize_mode'): self.choices('optimize_mode', 'maximize', 'minimize'), + Optional('start_step'): self.range('start_step', int, 0, 9999), + }).validate(kwargs) + +class MedianstopAssessor(Assessor): + """MedianstopAssessor is The median stopping rule stops a pending trial X at step S + if the trial’s best objective value by step S is strictly worse than the median value + of the running averages of all completed trials’ objectives reported up to step S + + Parameters + ---------- + optimize_mode : str + optimize mode, 'maximize' or 'minimize' + start_step : int + only after receiving start_step number of reported intermediate results + """ + def __init__(self, optimize_mode='maximize', start_step=0): + self._start_step = start_step + self._running_history = dict() + self._completed_avg_history = dict() + if optimize_mode == 'maximize': + self._high_better = True + elif optimize_mode == 'minimize': + self._high_better = False + else: + self._high_better = True + logger.warning('unrecognized optimize_mode %s', optimize_mode) + + def _update_data(self, trial_job_id, trial_history): + """update data + + Parameters + ---------- + trial_job_id : int + trial job id + trial_history : list + The history performance matrix of each trial + """ + if trial_job_id not in self._running_history: + self._running_history[trial_job_id] = [] + self._running_history[trial_job_id].extend(trial_history[len(self._running_history[trial_job_id]):]) + + def trial_end(self, trial_job_id, success): + """trial_end + + Parameters + ---------- + trial_job_id : int + trial job id + success : bool + True if succssfully finish the experiment, False otherwise + """ + if trial_job_id in self._running_history: + if success: + cnt = 0 + history_sum = 0 + self._completed_avg_history[trial_job_id] = [] + for each in self._running_history[trial_job_id]: + cnt += 1 + history_sum += each + self._completed_avg_history[trial_job_id].append(history_sum / cnt) + self._running_history.pop(trial_job_id) + else: + logger.warning('trial_end: trial_job_id does not exist in running_history') + + def assess_trial(self, trial_job_id, trial_history): + """assess_trial + + Parameters + ---------- + trial_job_id : int + trial job id + trial_history : list + The history performance matrix of each trial + + Returns + ------- + bool + AssessResult.Good or AssessResult.Bad + + Raises + ------ + Exception + unrecognize exception in medianstop_assessor + """ + curr_step = len(trial_history) + if curr_step < self._start_step: + return AssessResult.Good + + scalar_trial_history = extract_scalar_history(trial_history) + self._update_data(trial_job_id, scalar_trial_history) + if self._high_better: + best_history = max(scalar_trial_history) + else: + best_history = min(scalar_trial_history) + + avg_array = [] + for id_ in self._completed_avg_history: + if len(self._completed_avg_history[id_]) >= curr_step: + avg_array.append(self._completed_avg_history[id_][curr_step - 1]) + if avg_array: + avg_array.sort() + if self._high_better: + median = avg_array[(len(avg_array)-1) // 2] + return AssessResult.Bad if best_history < median else AssessResult.Good + else: + median = avg_array[len(avg_array) // 2] + return AssessResult.Bad if best_history > median else AssessResult.Good + else: + return AssessResult.Good diff --git a/utils/third_party/nni_new/algorithms/hpo/metis_tuner/Regression_GMM/CreateModel.py b/utils/third_party/nni_new/algorithms/hpo/metis_tuner/Regression_GMM/CreateModel.py new file mode 100644 index 0000000000000000000000000000000000000000..4846c9a67d1d4579456e5c28ce416b9e274566b4 --- /dev/null +++ b/utils/third_party/nni_new/algorithms/hpo/metis_tuner/Regression_GMM/CreateModel.py @@ -0,0 +1,44 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +import sys +from operator import itemgetter + +import sklearn.mixture as mm + +sys.path.insert(1, os.path.join(sys.path[0], '..')) + + +def create_model(samples_x, samples_y_aggregation, percentage_goodbatch=0.34): + ''' + Create the Gaussian Mixture Model + ''' + samples = [samples_x[i] + [samples_y_aggregation[i]] + for i in range(0, len(samples_x))] + + # Sorts so that we can get the top samples + samples = sorted(samples, key=itemgetter(-1)) + samples_goodbatch_size = int(len(samples) * percentage_goodbatch) + samples_goodbatch = samples[0:samples_goodbatch_size] + samples_badbatch = samples[samples_goodbatch_size:] + + samples_x_goodbatch = [sample_goodbatch[0:-1] + for sample_goodbatch in samples_goodbatch] + #samples_y_goodbatch = [sample_goodbatch[-1] for sample_goodbatch in samples_goodbatch] + samples_x_badbatch = [sample_badbatch[0:-1] + for sample_badbatch in samples_badbatch] + + # === Trains GMM clustering models === # + #sys.stderr.write("[%s] Train GMM's GMM model\n" % (os.path.basename(__file__))) + bgmm_goodbatch = mm.BayesianGaussianMixture( + n_components=max(1, samples_goodbatch_size - 1)) + bad_n_components = max(1, len(samples_x) - samples_goodbatch_size - 1) + bgmm_badbatch = mm.BayesianGaussianMixture(n_components=bad_n_components) + bgmm_goodbatch.fit(samples_x_goodbatch) + bgmm_badbatch.fit(samples_x_badbatch) + + model = {} + model['clusteringmodel_good'] = bgmm_goodbatch + model['clusteringmodel_bad'] = bgmm_badbatch + return model diff --git a/utils/third_party/nni_new/algorithms/hpo/metis_tuner/Regression_GMM/Selection.py b/utils/third_party/nni_new/algorithms/hpo/metis_tuner/Regression_GMM/Selection.py new file mode 100644 index 0000000000000000000000000000000000000000..0bca96647d4642afc7d46c8a259f1634b4c41d81 --- /dev/null +++ b/utils/third_party/nni_new/algorithms/hpo/metis_tuner/Regression_GMM/Selection.py @@ -0,0 +1,93 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +import random +import sys + +from .. import lib_acquisition_function +from .. import lib_constraint_summation + +sys.path.insert(1, os.path.join(sys.path[0], '..')) + + +CONSTRAINT_LOWERBOUND = None +CONSTRAINT_UPPERBOUND = None +CONSTRAINT_PARAMS_IDX = [] + + +def _ratio_scores(parameters_value, clusteringmodel_gmm_good, + clusteringmodel_gmm_bad): + ''' + The ratio is smaller the better + ''' + ratio = clusteringmodel_gmm_good.score( + [parameters_value]) / clusteringmodel_gmm_bad.score([parameters_value]) + sigma = 0 + return ratio, sigma + + +def selection_r(x_bounds, + x_types, + clusteringmodel_gmm_good, + clusteringmodel_gmm_bad, + num_starting_points=100, + minimize_constraints_fun=None): + ''' + Select using different types. + ''' + minimize_starting_points = clusteringmodel_gmm_good.sample(n_samples=num_starting_points) + + outputs = selection(x_bounds, x_types, + clusteringmodel_gmm_good, + clusteringmodel_gmm_bad, + minimize_starting_points[0], + minimize_constraints_fun) + + return outputs + + +def selection(x_bounds, + x_types, + clusteringmodel_gmm_good, + clusteringmodel_gmm_bad, + minimize_starting_points, + minimize_constraints_fun=None): + ''' + Select the lowest mu value + ''' + results = lib_acquisition_function.next_hyperparameter_lowest_mu( + _ratio_scores, [clusteringmodel_gmm_good, clusteringmodel_gmm_bad], + x_bounds, x_types, minimize_starting_points, + minimize_constraints_fun=minimize_constraints_fun) + + return results + + +def _rand_with_constraints(x_bounds, x_types): + ''' + Random generate the variable with constraints + ''' + outputs = None + x_bounds_withconstraints = [x_bounds[i] for i in CONSTRAINT_PARAMS_IDX] + x_types_withconstraints = [x_types[i] for i in CONSTRAINT_PARAMS_IDX] + x_val_withconstraints = lib_constraint_summation.rand(x_bounds_withconstraints, + x_types_withconstraints, + CONSTRAINT_LOWERBOUND, + CONSTRAINT_UPPERBOUND) + if x_val_withconstraints is not None: + outputs = [None] * len(x_bounds) + for i, _ in enumerate(CONSTRAINT_PARAMS_IDX): + outputs[CONSTRAINT_PARAMS_IDX[i]] = x_val_withconstraints[i] + for i, _ in enumerate(outputs): + if outputs[i] is None: + outputs[i] = random.randint(x_bounds[i][0], x_bounds[i][1]) + return outputs + + +def _minimize_constraints_fun_summation(x): + ''' + Minimize constraints fun summation + ''' + summation = sum([x[i] for i in CONSTRAINT_PARAMS_IDX]) + return CONSTRAINT_UPPERBOUND >= summation >= CONSTRAINT_LOWERBOUND diff --git a/utils/third_party/nni_new/algorithms/hpo/metis_tuner/Regression_GMM/__init__.py b/utils/third_party/nni_new/algorithms/hpo/metis_tuner/Regression_GMM/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/utils/third_party/nni_new/algorithms/hpo/metis_tuner/Regression_GP/CreateModel.py b/utils/third_party/nni_new/algorithms/hpo/metis_tuner/Regression_GP/CreateModel.py new file mode 100644 index 0000000000000000000000000000000000000000..98d1f22ce3c62c0491f91138e8c1df7ebd5e65ed --- /dev/null +++ b/utils/third_party/nni_new/algorithms/hpo/metis_tuner/Regression_GP/CreateModel.py @@ -0,0 +1,35 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +import sys +import numpy + +import sklearn.gaussian_process as gp + +sys.path.insert(1, os.path.join(sys.path[0], '..')) + + +def create_model(samples_x, samples_y_aggregation, + n_restarts_optimizer=250, is_white_kernel=False): + ''' + Trains GP regression model + ''' + kernel = gp.kernels.ConstantKernel(constant_value=1, + constant_value_bounds=(1e-12, 1e12)) * \ + gp.kernels.Matern(nu=1.5) + if is_white_kernel is True: + kernel += gp.kernels.WhiteKernel(noise_level=1, noise_level_bounds=(1e-12, 1e12)) + regressor = gp.GaussianProcessRegressor(kernel=kernel, + n_restarts_optimizer=n_restarts_optimizer, + normalize_y=True, + alpha=1e-10) + regressor.fit(numpy.array(samples_x), numpy.array(samples_y_aggregation)) + + model = {} + model['model'] = regressor + model['kernel_prior'] = str(kernel) + model['kernel_posterior'] = str(regressor.kernel_) + model['model_loglikelihood'] = regressor.log_marginal_likelihood(regressor.kernel_.theta) + + return model diff --git a/utils/third_party/nni_new/algorithms/hpo/metis_tuner/Regression_GP/OutlierDetection.py b/utils/third_party/nni_new/algorithms/hpo/metis_tuner/Regression_GP/OutlierDetection.py new file mode 100644 index 0000000000000000000000000000000000000000..f07a93dd3e80a8fd6f9b0e8ceb3e87a5338e915d --- /dev/null +++ b/utils/third_party/nni_new/algorithms/hpo/metis_tuner/Regression_GP/OutlierDetection.py @@ -0,0 +1,87 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +OutlierDectection.py +""" + +import os +import sys +from multiprocessing.dummy import Pool as ThreadPool + +from . import CreateModel as gp_create_model +from . import Prediction as gp_prediction + +sys.path.insert(1, os.path.join(sys.path[0], '..')) + + +def _outlierDetection_threaded(inputs): + """ + Detect the outlier + """ + [samples_idx, samples_x, samples_y_aggregation] = inputs + sys.stderr.write("[%s] DEBUG: Evaluating %dth of %d samples\n" + % (os.path.basename(__file__), samples_idx + 1, len(samples_x))) + outlier = None + + # Create a diagnostic regression model which removes the sample that we + # want to evaluate + diagnostic_regressor_gp = gp_create_model.create_model( + samples_x[0:samples_idx] + samples_x[samples_idx + 1:], + samples_y_aggregation[0:samples_idx] + samples_y_aggregation[samples_idx + 1:]) + mu, sigma = gp_prediction.predict( + samples_x[samples_idx], diagnostic_regressor_gp['model']) + + # 2.33 is the z-score for 98% confidence level + if abs(samples_y_aggregation[samples_idx] - mu) > (2.33 * sigma): + outlier = {"samples_idx": samples_idx, + "expected_mu": mu, + "expected_sigma": sigma, + "difference": abs(samples_y_aggregation[samples_idx] - mu) - (2.33 * sigma)} + return outlier + + +def outlierDetection_threaded(samples_x, samples_y_aggregation): + """ + Use Multi-thread to detect the outlier + """ + outliers = [] + + threads_inputs = [[samples_idx, samples_x, samples_y_aggregation] + for samples_idx in range(0, len(samples_x))] + threads_pool = ThreadPool(min(4, len(threads_inputs))) + threads_results = threads_pool.map( + _outlierDetection_threaded, threads_inputs) + threads_pool.close() + threads_pool.join() + + for threads_result in threads_results: + if threads_result is not None: + outliers.append(threads_result) + else: + print("Error: threads_result is None.") + + outliers = outliers if outliers else None + return outliers + + +def outlierDetection(samples_x, samples_y_aggregation): + outliers = [] + for samples_idx, _ in enumerate(samples_x): + #sys.stderr.write("[%s] DEBUG: Evaluating %d of %d samples\n" + # \ % (os.path.basename(__file__), samples_idx + 1, len(samples_x))) + diagnostic_regressor_gp = gp_create_model.create_model(\ + samples_x[0:samples_idx] + samples_x[samples_idx + 1:],\ + samples_y_aggregation[0:samples_idx] + samples_y_aggregation[samples_idx + 1:]) + mu, sigma = gp_prediction.predict(samples_x[samples_idx], + diagnostic_regressor_gp['model']) + # 2.33 is the z-score for 98% confidence level + if abs(samples_y_aggregation[samples_idx] - mu) > (2.33 * sigma): + outliers.append({"samples_idx": samples_idx, + "expected_mu": mu, + "expected_sigma": sigma, + "difference": \ + abs(samples_y_aggregation[samples_idx] - mu) - (2.33 * sigma)}) + + outliers = outliers if outliers else None + return outliers diff --git a/utils/third_party/nni_new/algorithms/hpo/metis_tuner/Regression_GP/Prediction.py b/utils/third_party/nni_new/algorithms/hpo/metis_tuner/Regression_GP/Prediction.py new file mode 100644 index 0000000000000000000000000000000000000000..08a32f0e63c86a6fd7dddad558f96c708916e41e --- /dev/null +++ b/utils/third_party/nni_new/algorithms/hpo/metis_tuner/Regression_GP/Prediction.py @@ -0,0 +1,19 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +import sys + +import numpy + +sys.path.insert(1, os.path.join(sys.path[0], '..')) + + +def predict(parameters_value, regressor_gp): + ''' + Predict by Gaussian Process Model + ''' + parameters_value = numpy.array(parameters_value).reshape(-1, len(parameters_value)) + mu, sigma = regressor_gp.predict(parameters_value, return_std=True) + + return mu[0], sigma[0] diff --git a/utils/third_party/nni_new/algorithms/hpo/metis_tuner/Regression_GP/Selection.py b/utils/third_party/nni_new/algorithms/hpo/metis_tuner/Regression_GP/Selection.py new file mode 100644 index 0000000000000000000000000000000000000000..68383ed0f2e039d4fd77c64730d06932f8b85b77 --- /dev/null +++ b/utils/third_party/nni_new/algorithms/hpo/metis_tuner/Regression_GP/Selection.py @@ -0,0 +1,97 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +import random +import sys + +from .. import lib_acquisition_function +from .. import lib_constraint_summation +from .. import lib_data +from . import Prediction as gp_prediction + +sys.path.insert(1, os.path.join(sys.path[0], '..')) + +CONSTRAINT_LOWERBOUND = None +CONSTRAINT_UPPERBOUND = None +CONSTRAINT_PARAMS_IDX = [] + + +def selection_r(acquisition_function, + samples_y_aggregation, + x_bounds, + x_types, + regressor_gp, + num_starting_points=100, + minimize_constraints_fun=None): + ''' + Selecte R value + ''' + minimize_starting_points = [lib_data.rand(x_bounds, x_types) \ + for i in range(0, num_starting_points)] + outputs = selection(acquisition_function, samples_y_aggregation, + x_bounds, x_types, regressor_gp, + minimize_starting_points, + minimize_constraints_fun=minimize_constraints_fun) + + return outputs + +def selection(acquisition_function, + samples_y_aggregation, + x_bounds, x_types, + regressor_gp, + minimize_starting_points, + minimize_constraints_fun=None): + ''' + selection + ''' + outputs = None + + sys.stderr.write("[%s] Exercise \"%s\" acquisition function\n" \ + % (os.path.basename(__file__), acquisition_function)) + + if acquisition_function == "ei": + outputs = lib_acquisition_function.next_hyperparameter_expected_improvement(\ + gp_prediction.predict, [regressor_gp], x_bounds, x_types, \ + samples_y_aggregation, minimize_starting_points, \ + minimize_constraints_fun=minimize_constraints_fun) + elif acquisition_function == "lc": + outputs = lib_acquisition_function.next_hyperparameter_lowest_confidence(\ + gp_prediction.predict, [regressor_gp], x_bounds, x_types,\ + minimize_starting_points, minimize_constraints_fun=minimize_constraints_fun) + elif acquisition_function == "lm": + outputs = lib_acquisition_function.next_hyperparameter_lowest_mu(\ + gp_prediction.predict, [regressor_gp], x_bounds, x_types,\ + minimize_starting_points, minimize_constraints_fun=minimize_constraints_fun) + return outputs + +def _rand_with_constraints(x_bounds, x_types): + ''' + Random generate with constraints + ''' + outputs = None + + x_bounds_withconstraints = [x_bounds[i] for i in CONSTRAINT_PARAMS_IDX] + x_types_withconstraints = [x_types[i] for i in CONSTRAINT_PARAMS_IDX] + x_val_withconstraints = lib_constraint_summation.rand(x_bounds_withconstraints, + x_types_withconstraints, + CONSTRAINT_LOWERBOUND, + CONSTRAINT_UPPERBOUND) + if x_val_withconstraints is not None: + outputs = [None] * len(x_bounds) + + for i, _ in enumerate(CONSTRAINT_PARAMS_IDX): + outputs[CONSTRAINT_PARAMS_IDX[i]] = x_val_withconstraints[i] + + for i, _ in enumerate(outputs): + if outputs[i] is None: + outputs[i] = random.randint(x_bounds[i][0], x_bounds[i][1]) + return outputs + + +def _minimize_constraints_fun_summation(x): + ''' + Minimize the constraints fun summation + ''' + summation = sum([x[i] for i in CONSTRAINT_PARAMS_IDX]) + return CONSTRAINT_UPPERBOUND >= summation >= CONSTRAINT_LOWERBOUND diff --git a/utils/third_party/nni_new/algorithms/hpo/metis_tuner/Regression_GP/__init__.py b/utils/third_party/nni_new/algorithms/hpo/metis_tuner/Regression_GP/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/utils/third_party/nni_new/algorithms/hpo/metis_tuner/__init__.py b/utils/third_party/nni_new/algorithms/hpo/metis_tuner/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..f4f9ceba61960bab8b741d760448815a93b3ae0b --- /dev/null +++ b/utils/third_party/nni_new/algorithms/hpo/metis_tuner/__init__.py @@ -0,0 +1 @@ +from .metis_tuner import MetisTuner, MetisClassArgsValidator diff --git a/utils/third_party/nni_new/algorithms/hpo/metis_tuner/lib_acquisition_function.py b/utils/third_party/nni_new/algorithms/hpo/metis_tuner/lib_acquisition_function.py new file mode 100644 index 0000000000000000000000000000000000000000..f1b1edfe0165b61c18753dfa823bc2bc10aa8d05 --- /dev/null +++ b/utils/third_party/nni_new/algorithms/hpo/metis_tuner/lib_acquisition_function.py @@ -0,0 +1,197 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +lib_acquisition_function.py +""" + +import sys +import numpy + +from scipy.stats import norm +from scipy.optimize import minimize + +from . import lib_data + + +def next_hyperparameter_expected_improvement(fun_prediction, + fun_prediction_args, + x_bounds, x_types, + samples_y_aggregation, + minimize_starting_points, + minimize_constraints_fun=None): + """ + "Expected Improvement" acquisition function + """ + best_x = None + best_acquisition_value = None + x_bounds_minmax = [[i[0], i[-1]] for i in x_bounds] + x_bounds_minmax = numpy.array(x_bounds_minmax) + + for starting_point in numpy.array(minimize_starting_points): + res = minimize(fun=_expected_improvement, + x0=starting_point.reshape(1, -1), + bounds=x_bounds_minmax, + method="L-BFGS-B", + args=(fun_prediction, + fun_prediction_args, + x_bounds, + x_types, + samples_y_aggregation, + minimize_constraints_fun)) + + if (best_acquisition_value is None) or \ + (res.fun < best_acquisition_value): + res.x = numpy.ndarray.tolist(res.x) + res.x = lib_data.match_val_type(res.x, x_bounds, x_types) + if (minimize_constraints_fun is None) or \ + (minimize_constraints_fun(res.x) is True): + best_acquisition_value = res.fun + best_x = res.x + + outputs = None + if best_x is not None: + mu, sigma = fun_prediction(best_x, *fun_prediction_args) + outputs = {'hyperparameter': best_x, 'expected_mu': mu, + 'expected_sigma': sigma, 'acquisition_func': "ei"} + + return outputs + + +def _expected_improvement(x, fun_prediction, fun_prediction_args, + x_bounds, x_types, samples_y_aggregation, + minimize_constraints_fun): + # This is only for step-wise optimization + x = lib_data.match_val_type(x, x_bounds, x_types) + + expected_improvement = sys.maxsize + if (minimize_constraints_fun is None) or ( + minimize_constraints_fun(x) is True): + mu, sigma = fun_prediction(x, *fun_prediction_args) + + loss_optimum = min(samples_y_aggregation) + scaling_factor = -1 + + # In case sigma equals zero + with numpy.errstate(divide="ignore"): + Z = scaling_factor * (mu - loss_optimum) / sigma + expected_improvement = scaling_factor * (mu - loss_optimum) * \ + norm.cdf(Z) + sigma * norm.pdf(Z) + expected_improvement = 0.0 if sigma == 0.0 else expected_improvement + + # We want expected_improvement to be as large as possible + # (i.e., as small as possible for minimize(...)) + expected_improvement = -1 * expected_improvement + return expected_improvement + + +def next_hyperparameter_lowest_confidence(fun_prediction, + fun_prediction_args, + x_bounds, x_types, + minimize_starting_points, + minimize_constraints_fun=None): + """ + "Lowest Confidence" acquisition function + """ + best_x = None + best_acquisition_value = None + x_bounds_minmax = [[i[0], i[-1]] for i in x_bounds] + x_bounds_minmax = numpy.array(x_bounds_minmax) + + for starting_point in numpy.array(minimize_starting_points): + res = minimize(fun=_lowest_confidence, + x0=starting_point.reshape(1, -1), + bounds=x_bounds_minmax, + method="L-BFGS-B", + args=(fun_prediction, + fun_prediction_args, + x_bounds, + x_types, + minimize_constraints_fun)) + + if (best_acquisition_value) is None or ( + res.fun < best_acquisition_value): + res.x = numpy.ndarray.tolist(res.x) + res.x = lib_data.match_val_type(res.x, x_bounds, x_types) + if (minimize_constraints_fun is None) or ( + minimize_constraints_fun(res.x) is True): + best_acquisition_value = res.fun + best_x = res.x + + outputs = None + if best_x is not None: + mu, sigma = fun_prediction(best_x, *fun_prediction_args) + outputs = {'hyperparameter': best_x, 'expected_mu': mu, + 'expected_sigma': sigma, 'acquisition_func': "lc"} + return outputs + + +def _lowest_confidence(x, fun_prediction, fun_prediction_args, + x_bounds, x_types, minimize_constraints_fun): + # This is only for step-wise optimization + x = lib_data.match_val_type(x, x_bounds, x_types) + + ci = sys.maxsize + if (minimize_constraints_fun is None) or ( + minimize_constraints_fun(x) is True): + mu, sigma = fun_prediction(x, *fun_prediction_args) + ci = (sigma * 1.96 * 2) / mu + # We want ci to be as large as possible + # (i.e., as small as possible for minimize(...), + # because this would mean lowest confidence + ci = -1 * ci + + return ci + + +def next_hyperparameter_lowest_mu(fun_prediction, + fun_prediction_args, + x_bounds, x_types, + minimize_starting_points, + minimize_constraints_fun=None): + """ + "Lowest Mu" acquisition function + """ + best_x = None + best_acquisition_value = None + x_bounds_minmax = [[i[0], i[-1]] for i in x_bounds] + x_bounds_minmax = numpy.array(x_bounds_minmax) + + for starting_point in numpy.array(minimize_starting_points): + res = minimize(fun=_lowest_mu, + x0=starting_point.reshape(1, -1), + bounds=x_bounds_minmax, + method="L-BFGS-B", + args=(fun_prediction, fun_prediction_args, + x_bounds, x_types, minimize_constraints_fun)) + + if (best_acquisition_value is None) or ( + res.fun < best_acquisition_value): + res.x = numpy.ndarray.tolist(res.x) + res.x = lib_data.match_val_type(res.x, x_bounds, x_types) + if (minimize_constraints_fun is None) or ( + minimize_constraints_fun(res.x) is True): + best_acquisition_value = res.fun + best_x = res.x + + outputs = None + if best_x is not None: + mu, sigma = fun_prediction(best_x, *fun_prediction_args) + outputs = {'hyperparameter': best_x, 'expected_mu': mu, + 'expected_sigma': sigma, 'acquisition_func': "lm"} + return outputs + + +def _lowest_mu(x, fun_prediction, fun_prediction_args, + x_bounds, x_types, minimize_constraints_fun): + """ + Calculate the lowest mu + """ + # This is only for step-wise optimization + x = lib_data.match_val_type(x, x_bounds, x_types) + + mu = sys.maxsize + if (minimize_constraints_fun is None) or ( + minimize_constraints_fun(x) is True): + mu, _ = fun_prediction(x, *fun_prediction_args) + return mu diff --git a/utils/third_party/nni_new/algorithms/hpo/metis_tuner/lib_constraint_summation.py b/utils/third_party/nni_new/algorithms/hpo/metis_tuner/lib_constraint_summation.py new file mode 100644 index 0000000000000000000000000000000000000000..948be5b3ca3dcace3ae673ce9e12667c093ca7a4 --- /dev/null +++ b/utils/third_party/nni_new/algorithms/hpo/metis_tuner/lib_constraint_summation.py @@ -0,0 +1,108 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +lib_constraint_summation.py +""" + +import math +import random + +from operator import itemgetter + + +def check_feasibility(x_bounds, lowerbound, upperbound): + ''' + This can have false positives. + For examples, parameters can only be 0 or 5, and the summation constraint is between 6 and 7. + ''' + # x_bounds should be sorted, so even for "discrete_int" type, + # the smallest and the largest number should the first and the last element + x_bounds_lowerbound = sum([x_bound[0] for x_bound in x_bounds]) + x_bounds_upperbound = sum([x_bound[-1] for x_bound in x_bounds]) + + # return ((x_bounds_lowerbound <= lowerbound) and (x_bounds_upperbound >= lowerbound)) or \ + # ((x_bounds_lowerbound <= upperbound) and (x_bounds_upperbound >= upperbound)) + return (x_bounds_lowerbound <= lowerbound <= x_bounds_upperbound) or \ + (x_bounds_lowerbound <= upperbound <= x_bounds_upperbound) + + +def rand(x_bounds, x_types, lowerbound, upperbound, max_retries=100): + ''' + Key idea is that we try to move towards upperbound, by randomly choose one + value for each parameter. However, for the last parameter, + we need to make sure that its value can help us get above lowerbound + ''' + outputs = None + + if check_feasibility(x_bounds, lowerbound, upperbound) is True: + # Order parameters by their range size. We want the smallest range first, + # because the corresponding parameter has less numbers to choose from + x_idx_sorted = [] + for i, _ in enumerate(x_bounds): + if x_types[i] == "discrete_int": + x_idx_sorted.append([i, len(x_bounds[i])]) + elif (x_types[i] == "range_int") or (x_types[i] == "range_continuous"): + x_idx_sorted.append( + [i, math.floor(x_bounds[i][1] - x_bounds[i][0])]) + x_idx_sorted = sorted(x_idx_sorted, key=itemgetter(1)) + + for _ in range(max_retries): + budget_allocated = 0 + outputs = [None] * len(x_bounds) + + for i, _ in enumerate(x_idx_sorted): + x_idx = x_idx_sorted[i][0] + # The amount of unallocated space that we have + budget_max = upperbound - budget_allocated + # NOT the Last x that we need to assign a random number + if i < (len(x_idx_sorted) - 1): + if x_bounds[x_idx][0] <= budget_max: + if x_types[x_idx] == "discrete_int": + # Note the valid integer + temp = [] + for j in x_bounds[x_idx]: + if j <= budget_max: + temp.append(j) + # Randomly pick a number from the integer array + if temp: + outputs[x_idx] = temp[random.randint( + 0, len(temp) - 1)] + + elif (x_types[x_idx] == "range_int") or \ + (x_types[x_idx] == "range_continuous"): + outputs[x_idx] = random.randint( + x_bounds[x_idx][0], min(x_bounds[x_idx][-1], budget_max)) + + else: + # The last x that we need to assign a random number + randint_lowerbound = lowerbound - budget_allocated + randint_lowerbound = 0 if randint_lowerbound < 0 else randint_lowerbound + + # This check: + # is our smallest possible value going to overflow the available budget space, + # and is our largest possible value going to underflow the + # lower bound + if (x_bounds[x_idx][0] <= budget_max) and \ + (x_bounds[x_idx][-1] >= randint_lowerbound): + if x_types[x_idx] == "discrete_int": + temp = [] + for j in x_bounds[x_idx]: + # if (j <= budget_max) and (j >= + # randint_lowerbound): + if randint_lowerbound <= j <= budget_max: + temp.append(j) + if temp: + outputs[x_idx] = temp[random.randint( + 0, len(temp) - 1)] + elif (x_types[x_idx] == "range_int") or \ + (x_types[x_idx] == "range_continuous"): + outputs[x_idx] = random.randint( + randint_lowerbound, min( + x_bounds[x_idx][1], budget_max)) + if outputs[x_idx] is None: + break + budget_allocated += outputs[x_idx] + if None not in outputs: + break + return outputs diff --git a/utils/third_party/nni_new/algorithms/hpo/metis_tuner/lib_data.py b/utils/third_party/nni_new/algorithms/hpo/metis_tuner/lib_data.py new file mode 100644 index 0000000000000000000000000000000000000000..cc3f059514c8c30312acbd0c18ec000803cbe0ad --- /dev/null +++ b/utils/third_party/nni_new/algorithms/hpo/metis_tuner/lib_data.py @@ -0,0 +1,50 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import math +import random + + +def match_val_type(vals, vals_bounds, vals_types): + ''' + Update values in the array, to match their corresponding type + ''' + vals_new = [] + + for i, _ in enumerate(vals_types): + if vals_types[i] == "discrete_int": + # Find the closest integer in the array, vals_bounds + # pylint: disable=cell-var-from-loop + vals_new.append(min(vals_bounds[i], key=lambda x: abs(x - vals[i]))) + elif vals_types[i] == "range_int": + # Round down to the nearest integer + vals_new.append(math.floor(vals[i])) + elif vals_types[i] == "range_continuous": + # Don't do any processing for continous numbers + vals_new.append(vals[i]) + else: + return None + + return vals_new + + +def rand(x_bounds, x_types): + ''' + Random generate variable value within their bounds + ''' + outputs = [] + + for i, _ in enumerate(x_bounds): + if x_types[i] == "discrete_int": + temp = x_bounds[i][random.randint(0, len(x_bounds[i]) - 1)] + outputs.append(temp) + elif x_types[i] == "range_int": + temp = random.randint(x_bounds[i][0], x_bounds[i][1] - 1) + outputs.append(temp) + elif x_types[i] == "range_continuous": + temp = random.uniform(x_bounds[i][0], x_bounds[i][1]) + outputs.append(temp) + else: + return None + + return outputs diff --git a/utils/third_party/nni_new/algorithms/hpo/metis_tuner/metis_tuner.py b/utils/third_party/nni_new/algorithms/hpo/metis_tuner/metis_tuner.py new file mode 100644 index 0000000000000000000000000000000000000000..1a0670f5e517d3ce1cdf775a2590daea7366f4d9 --- /dev/null +++ b/utils/third_party/nni_new/algorithms/hpo/metis_tuner/metis_tuner.py @@ -0,0 +1,650 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +metis_tuner.py +""" + +import copy +import logging +import random +import statistics +import warnings +from multiprocessing.dummy import Pool as ThreadPool +import numpy as np +from schema import Schema, Optional + +from nni import ClassArgsValidator +from . import lib_constraint_summation +from . import lib_data +from .Regression_GMM import CreateModel as gmm_create_model +from .Regression_GMM import Selection as gmm_selection +from .Regression_GP import CreateModel as gp_create_model +from .Regression_GP import OutlierDetection as gp_outlier_detection +from .Regression_GP import Prediction as gp_prediction +from .Regression_GP import Selection as gp_selection +from nni.tuner import Tuner +from nni.utils import OptimizeMode, extract_scalar_reward + +logger = logging.getLogger("Metis_Tuner_AutoML") + +NONE_TYPE = '' +CONSTRAINT_LOWERBOUND = None +CONSTRAINT_UPPERBOUND = None +CONSTRAINT_PARAMS_IDX = [] + +class MetisClassArgsValidator(ClassArgsValidator): + def validate_class_args(self, **kwargs): + Schema({ + Optional('optimize_mode'): self.choices('optimize_mode', 'maximize', 'minimize'), + Optional('no_resampling'): bool, + Optional('no_candidates'): bool, + Optional('selection_num_starting_points'): int, + Optional('cold_start_num'): int, + }).validate(kwargs) + +class MetisTuner(Tuner): + """ + Metis Tuner + + More algorithm information you could reference here: + https://www.microsoft.com/en-us/research/publication/metis-robustly-tuning-tail-latencies-cloud-systems/ + + Attributes + ---------- + optimize_mode : str + optimize_mode is a string that including two mode "maximize" and "minimize" + + no_resampling : bool + True or False. + Should Metis consider re-sampling as part of the search strategy? + If you are confident that the training dataset is noise-free, + then you do not need re-sampling. + + no_candidates : bool + True or False. + Should Metis suggest parameters for the next benchmark? + If you do not plan to do more benchmarks, + Metis can skip this step. + + selection_num_starting_points : int + How many times Metis should try to find the global optimal in the search space? + The higher the number, the longer it takes to output the solution. + + cold_start_num : int + Metis need some trial result to get cold start. + when the number of trial result is less than + cold_start_num, Metis will randomly sample hyper-parameter for trial. + + exploration_probability: float + The probability of Metis to select parameter from exploration instead of exploitation. + """ + + def __init__( + self, + optimize_mode="maximize", + no_resampling=True, + no_candidates=False, + selection_num_starting_points=600, + cold_start_num=10, + exploration_probability=0.9): + """ + Parameters + ---------- + optimize_mode : str + optimize_mode is a string that including two mode "maximize" and "minimize" + + no_resampling : bool + True or False. + Should Metis consider re-sampling as part of the search strategy? + If you are confident that the training dataset is noise-free, + then you do not need re-sampling. + + no_candidates : bool + True or False. + Should Metis suggest parameters for the next benchmark? + If you do not plan to do more benchmarks, + Metis can skip this step. + + selection_num_starting_points : int + How many times Metis should try to find the global optimal in the search space? + The higher the number, the longer it takes to output the solution. + + cold_start_num : int + Metis need some trial result to get cold start. + when the number of trial result is less than + cold_start_num, Metis will randomly sample hyper-parameter for trial. + + exploration_probability : float + The probability of Metis to select parameter from exploration instead of exploitation. + + x_bounds : list + The constration of parameters. + + x_types : list + The type of parameters. + """ + + self.samples_x = [] + self.samples_y = [] + self.samples_y_aggregation = [] + self.total_data = [] + self.space = None + self.no_resampling = no_resampling + self.no_candidates = no_candidates + self.optimize_mode = OptimizeMode(optimize_mode) + self.key_order = [] + self.cold_start_num = cold_start_num + self.selection_num_starting_points = selection_num_starting_points + self.exploration_probability = exploration_probability + self.minimize_constraints_fun = None + self.minimize_starting_points = None + self.supplement_data_num = 0 + self.x_bounds = [] + self.x_types = [] + + + def update_search_space(self, search_space): + """ + Update the self.x_bounds and self.x_types by the search_space.json + + Parameters + ---------- + search_space : dict + """ + self.x_bounds = [[] for i in range(len(search_space))] + self.x_types = [NONE_TYPE for i in range(len(search_space))] + + for key in search_space: + self.key_order.append(key) + + key_type = {} + if isinstance(search_space, dict): + for key in search_space: + key_type = search_space[key]['_type'] + key_range = search_space[key]['_value'] + idx = self.key_order.index(key) + if key_type == 'quniform': + if key_range[2] == 1 and key_range[0].is_integer( + ) and key_range[1].is_integer(): + self.x_bounds[idx] = [key_range[0], key_range[1] + 1] + self.x_types[idx] = 'range_int' + else: + low, high, q = key_range + bounds = np.clip( + np.arange( + np.round( + low / q), + np.round( + high / q) + 1) * q, + low, + high) + self.x_bounds[idx] = bounds + self.x_types[idx] = 'discrete_int' + elif key_type == 'randint': + self.x_bounds[idx] = [key_range[0], key_range[1]] + self.x_types[idx] = 'range_int' + elif key_type == 'uniform': + self.x_bounds[idx] = [key_range[0], key_range[1]] + self.x_types[idx] = 'range_continuous' + elif key_type == 'choice': + self.x_bounds[idx] = key_range + + for key_value in key_range: + if not isinstance(key_value, (int, float)): + raise RuntimeError( + "Metis Tuner only support numerical choice.") + + self.x_types[idx] = 'discrete_int' + else: + logger.info( + "Metis Tuner doesn't support this kind of variable: %s", + str(key_type)) + raise RuntimeError( + "Metis Tuner doesn't support this kind of variable: %s" % + str(key_type)) + else: + logger.info("The format of search space is not a dict.") + raise RuntimeError("The format of search space is not a dict.") + + self.minimize_starting_points = _rand_init( + self.x_bounds, self.x_types, self.selection_num_starting_points) + + + def _pack_output(self, init_parameter): + """ + Pack the output + + Parameters + ---------- + init_parameter : dict + + Returns + ------- + output : dict + """ + output = {} + for i, param in enumerate(init_parameter): + output[self.key_order[i]] = param + + return output + + + def generate_parameters(self, parameter_id, **kwargs): + """ + Generate next parameter for trial + + If the number of trial result is lower than cold start number, + metis will first random generate some parameters. + Otherwise, metis will choose the parameters by + the Gussian Process Model and the Gussian Mixture Model. + + Parameters + ---------- + parameter_id : int + + Returns + ------- + result : dict + """ + if len(self.samples_x) < self.cold_start_num: + init_parameter = _rand_init(self.x_bounds, self.x_types, 1)[0] + results = self._pack_output(init_parameter) + else: + self.minimize_starting_points = _rand_init( + self.x_bounds, self.x_types, self.selection_num_starting_points) + results = self._selection( + self.samples_x, + self.samples_y_aggregation, + self.samples_y, + self.x_bounds, + self.x_types, + threshold_samplessize_resampling=( + None if self.no_resampling is True else 50), + no_candidates=self.no_candidates, + minimize_starting_points=self.minimize_starting_points, + minimize_constraints_fun=self.minimize_constraints_fun) + + logger.info("Generate paramageters: \n%s", str(results)) + return results + + + def receive_trial_result(self, parameter_id, parameters, value, **kwargs): + """ + Tuner receive result from trial. + + Parameters + ---------- + parameter_id : int + The id of parameters, generated by nni manager. + parameters : dict + A group of parameters that trial has tried. + value : dict/float + if value is dict, it should have "default" key. + """ + value = extract_scalar_reward(value) + if self.optimize_mode == OptimizeMode.Maximize: + value = -value + + logger.info("Received trial result.") + logger.info("value is : %s", str(value)) + logger.info("parameter is : %s", str(parameters)) + + # parse parameter to sample_x + sample_x = [0 for i in range(len(self.key_order))] + for key in parameters: + idx = self.key_order.index(key) + sample_x[idx] = parameters[key] + + # parse value to sample_y + temp_y = [] + if sample_x in self.samples_x: + idx = self.samples_x.index(sample_x) + temp_y = self.samples_y[idx] + temp_y.append(value) + self.samples_y[idx] = temp_y + + # calculate y aggregation + median = get_median(temp_y) + self.samples_y_aggregation[idx] = [median] + else: + self.samples_x.append(sample_x) + self.samples_y.append([value]) + + # calculate y aggregation + self.samples_y_aggregation.append([value]) + + + def _selection( + self, + samples_x, + samples_y_aggregation, + samples_y, + x_bounds, + x_types, + max_resampling_per_x=3, + threshold_samplessize_exploitation=12, + threshold_samplessize_resampling=50, + no_candidates=False, + minimize_starting_points=None, + minimize_constraints_fun=None): + + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + + next_candidate = None + candidates = [] + samples_size_all = sum([len(i) for i in samples_y]) + samples_size_unique = len(samples_y) + + # ===== STEP 1: Compute the current optimum ===== + gp_model = gp_create_model.create_model( + samples_x, samples_y_aggregation) + lm_current = gp_selection.selection( + "lm", + samples_y_aggregation, + x_bounds, + x_types, + gp_model['model'], + minimize_starting_points, + minimize_constraints_fun=minimize_constraints_fun) + if not lm_current: + return None + logger.info({ + 'hyperparameter': lm_current['hyperparameter'], + 'expected_mu': lm_current['expected_mu'], + 'expected_sigma': lm_current['expected_sigma'], + 'reason': "exploitation_gp" + }) + + if no_candidates is False: + # ===== STEP 2: Get recommended configurations for exploration ==== + results_exploration = gp_selection.selection( + "lc", + samples_y_aggregation, + x_bounds, + x_types, + gp_model['model'], + minimize_starting_points, + minimize_constraints_fun=minimize_constraints_fun) + + if results_exploration is not None: + if _num_past_samples(results_exploration['hyperparameter'], samples_x, samples_y) == 0: + temp_candidate = { + 'hyperparameter': results_exploration['hyperparameter'], + 'expected_mu': results_exploration['expected_mu'], + 'expected_sigma': results_exploration['expected_sigma'], + 'reason': "exploration" + } + candidates.append(temp_candidate) + + logger.info("DEBUG: 1 exploration candidate selected\n") + logger.info(temp_candidate) + else: + logger.info("DEBUG: No suitable exploration candidates were") + + # ===== STEP 3: Get recommended configurations for exploitation === + if samples_size_all >= threshold_samplessize_exploitation: + logger.info("Getting candidates for exploitation...\n") + try: + gmm = gmm_create_model.create_model( + samples_x, samples_y_aggregation) + + if ("discrete_int" in x_types) or ("range_int" in x_types): + results_exploitation = gmm_selection.selection( + x_bounds, + x_types, + gmm['clusteringmodel_good'], + gmm['clusteringmodel_bad'], + minimize_starting_points, + minimize_constraints_fun=minimize_constraints_fun) + else: + # If all parameters are of "range_continuous", + # let's use GMM to generate random starting points + results_exploitation = gmm_selection.selection_r( + x_bounds, + x_types, + gmm['clusteringmodel_good'], + gmm['clusteringmodel_bad'], + num_starting_points=self.selection_num_starting_points, + minimize_constraints_fun=minimize_constraints_fun) + + if results_exploitation is not None: + if _num_past_samples(results_exploitation['hyperparameter'], samples_x, samples_y) == 0: + temp_expected_mu, temp_expected_sigma = \ + gp_prediction.predict(results_exploitation['hyperparameter'], gp_model['model']) + temp_candidate = { + 'hyperparameter': results_exploitation['hyperparameter'], + 'expected_mu': temp_expected_mu, + 'expected_sigma': temp_expected_sigma, + 'reason': "exploitation_gmm" + } + candidates.append(temp_candidate) + + logger.info( + "DEBUG: 1 exploitation_gmm candidate selected\n") + logger.info(temp_candidate) + else: + logger.info( + "DEBUG: No suitable exploitation_gmm candidates were found\n") + + except ValueError as exception: + # The exception: ValueError: Fitting the mixture model failed + # because some components have ill-defined empirical covariance + # (for instance caused by singleton or collapsed samples). + # Try to decrease the number of components, or increase + # reg_covar. + logger.info( + "DEBUG: No suitable exploitation_gmm \ + candidates were found due to exception.") + logger.info(exception) + + # ===== STEP 4: Get a list of outliers ===== + if (threshold_samplessize_resampling is not None) and \ + (samples_size_unique >= threshold_samplessize_resampling): + logger.info("Getting candidates for re-sampling...\n") + results_outliers = gp_outlier_detection.outlierDetection_threaded( + samples_x, samples_y_aggregation) + + if results_outliers is not None: + for results_outlier in results_outliers: # pylint: disable=not-an-iterable + if _num_past_samples(samples_x[results_outlier['samples_idx']], samples_x, samples_y) < max_resampling_per_x: + temp_candidate = {'hyperparameter': samples_x[results_outlier['samples_idx']],\ + 'expected_mu': results_outlier['expected_mu'],\ + 'expected_sigma': results_outlier['expected_sigma'],\ + 'reason': "resampling"} + candidates.append(temp_candidate) + logger.info("DEBUG: %d re-sampling candidates selected\n") + logger.info(temp_candidate) + else: + logger.info( + "DEBUG: No suitable resampling candidates were found\n") + + if candidates: + # ===== STEP 5: Compute the information gain of each candidate + logger.info( + "Evaluating information gain of %d candidates...\n") + next_improvement = 0 + + threads_inputs = [[ + candidate, samples_x, samples_y, x_bounds, x_types, + minimize_constraints_fun, minimize_starting_points + ] for candidate in candidates] + threads_pool = ThreadPool(4) + # Evaluate what would happen if we actually sample each + # candidate + threads_results = threads_pool.map( + _calculate_lowest_mu_threaded, threads_inputs) + threads_pool.close() + threads_pool.join() + + for threads_result in threads_results: + if threads_result['expected_lowest_mu'] < lm_current['expected_mu']: + # Information gain + temp_improvement = threads_result['expected_lowest_mu'] - \ + lm_current['expected_mu'] + + if next_improvement > temp_improvement: + next_improvement = temp_improvement + next_candidate = threads_result['candidate'] + else: + # ===== STEP 6: If we have no candidates, randomly pick one === + logger.info( + "DEBUG: No candidates from exploration, exploitation,\ + and resampling. We will random a candidate for next_candidate\n" + ) + + next_candidate = _rand_with_constraints( + x_bounds, + x_types) if minimize_starting_points is None else minimize_starting_points[0] + next_candidate = lib_data.match_val_type( + next_candidate, x_bounds, x_types) + expected_mu, expected_sigma = gp_prediction.predict( + next_candidate, gp_model['model']) + next_candidate = { + 'hyperparameter': next_candidate, + 'reason': "random", + 'expected_mu': expected_mu, + 'expected_sigma': expected_sigma} + + # STEP 7: If current optimal hyperparameter occurs in the history + # or exploration probability is less than the threshold, take next + # config as exploration step + outputs = self._pack_output(lm_current['hyperparameter']) + ap = random.uniform(0, 1) + if outputs in self.total_data or ap <= self.exploration_probability: + if next_candidate is not None: + outputs = self._pack_output(next_candidate['hyperparameter']) + else: + random_parameter = _rand_init(x_bounds, x_types, 1)[0] + outputs = self._pack_output(random_parameter) + self.total_data.append(outputs) + return outputs + + def import_data(self, data): + """ + Import additional data for tuning + + Parameters + ---------- + data : a list of dict + each of which has at least two keys: 'parameter' and 'value'. + """ + _completed_num = 0 + for trial_info in data: + logger.info("Importing data, current processing progress %s / %s", _completed_num, len(data)) + _completed_num += 1 + assert "parameter" in trial_info + _params = trial_info["parameter"] + assert "value" in trial_info + _value = trial_info['value'] + if not _value: + logger.info("Useless trial data, value is %s, skip this trial data.", _value) + continue + self.supplement_data_num += 1 + _parameter_id = '_'.join( + ["ImportData", str(self.supplement_data_num)]) + self.total_data.append(_params) + self.receive_trial_result( + parameter_id=_parameter_id, + parameters=_params, + value=_value) + logger.info("Successfully import data to metis tuner.") + + +def _rand_with_constraints(x_bounds, x_types): + outputs = None + x_bounds_withconstraints = [x_bounds[i] for i in CONSTRAINT_PARAMS_IDX] + x_types_withconstraints = [x_types[i] for i in CONSTRAINT_PARAMS_IDX] + + x_val_withconstraints = lib_constraint_summation.rand( + x_bounds_withconstraints, + x_types_withconstraints, + CONSTRAINT_LOWERBOUND, + CONSTRAINT_UPPERBOUND) + if not x_val_withconstraints: + outputs = [None] * len(x_bounds) + + for i, _ in enumerate(CONSTRAINT_PARAMS_IDX): + outputs[CONSTRAINT_PARAMS_IDX[i]] = x_val_withconstraints[i] + + for i, output in enumerate(outputs): + if not output: + outputs[i] = random.randint(x_bounds[i][0], x_bounds[i][1]) + return outputs + + +def _calculate_lowest_mu_threaded(inputs): + [candidate, samples_x, samples_y, x_bounds, x_types, + minimize_constraints_fun, minimize_starting_points] = inputs + + outputs = {"candidate": candidate, "expected_lowest_mu": None} + + for expected_mu in [ + candidate['expected_mu'] + + 1.96 * + candidate['expected_sigma'], + candidate['expected_mu'] - + 1.96 * + candidate['expected_sigma']]: + temp_samples_x = copy.deepcopy(samples_x) + temp_samples_y = copy.deepcopy(samples_y) + + try: + idx = temp_samples_x.index(candidate['hyperparameter']) + # This handles the case of re-sampling a potential outlier + temp_samples_y[idx].append(expected_mu) + except ValueError: + temp_samples_x.append(candidate['hyperparameter']) + temp_samples_y.append([expected_mu]) + + # Aggregates multiple observation of the sample sampling points + temp_y_aggregation = [statistics.median( + temp_sample_y) for temp_sample_y in temp_samples_y] + temp_gp = gp_create_model.create_model( + temp_samples_x, temp_y_aggregation) + temp_results = gp_selection.selection( + "lm", + temp_y_aggregation, + x_bounds, + x_types, + temp_gp['model'], + minimize_starting_points, + minimize_constraints_fun=minimize_constraints_fun) + + if outputs["expected_lowest_mu"] is None \ + or outputs["expected_lowest_mu"] > temp_results['expected_mu']: + outputs["expected_lowest_mu"] = temp_results['expected_mu'] + + return outputs + + +def _num_past_samples(x, samples_x, samples_y): + try: + idx = samples_x.index(x) + return len(samples_y[idx]) + except ValueError: + logger.info("x not in sample_x") + return 0 + + +def _rand_init(x_bounds, x_types, selection_num_starting_points): + ''' + Random sample some init seed within bounds. + ''' + return [lib_data.rand(x_bounds, x_types) for i + in range(0, selection_num_starting_points)] + + +def get_median(temp_list): + """ + Return median + """ + num = len(temp_list) + temp_list.sort() + print(temp_list) + if num % 2 == 0: + median = (temp_list[int(num / 2)] + temp_list[int(num / 2) - 1]) / 2 + else: + median = temp_list[int(num / 2)] + return median diff --git a/utils/third_party/nni_new/algorithms/hpo/metis_tuner/requirments.txt b/utils/third_party/nni_new/algorithms/hpo/metis_tuner/requirments.txt new file mode 100644 index 0000000000000000000000000000000000000000..3dfc2232a18c19a544eb3fb5b2f132e185958f98 --- /dev/null +++ b/utils/third_party/nni_new/algorithms/hpo/metis_tuner/requirments.txt @@ -0,0 +1 @@ +scikit-learn>=0.23.2 diff --git a/utils/third_party/nni_new/algorithms/hpo/networkmorphism_tuner/__init__.py b/utils/third_party/nni_new/algorithms/hpo/networkmorphism_tuner/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..b60da9c3892a06392397b7b8549de6615f4569cf --- /dev/null +++ b/utils/third_party/nni_new/algorithms/hpo/networkmorphism_tuner/__init__.py @@ -0,0 +1 @@ +from .networkmorphism_tuner import NetworkMorphismTuner, NetworkMorphismClassArgsValidator diff --git a/utils/third_party/nni_new/algorithms/hpo/networkmorphism_tuner/bayesian.py b/utils/third_party/nni_new/algorithms/hpo/networkmorphism_tuner/bayesian.py new file mode 100644 index 0000000000000000000000000000000000000000..54c1996dc7708568a7065325d24a8b83fdb5a40f --- /dev/null +++ b/utils/third_party/nni_new/algorithms/hpo/networkmorphism_tuner/bayesian.py @@ -0,0 +1,482 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import math +import random +from copy import deepcopy +from functools import total_ordering +from queue import PriorityQueue + +import numpy as np +from scipy.linalg import LinAlgError, cho_solve, cholesky, solve_triangular +from scipy.optimize import linear_sum_assignment +from sklearn.metrics.pairwise import rbf_kernel + +from nni.utils import OptimizeMode +from .graph_transformer import transform +from .utils import Constant +from .layers import is_layer + + +def layer_distance(a, b): + """The distance between two layers.""" + # pylint: disable=unidiomatic-typecheck + if not isinstance(a, type(b)): + return 1.0 + if is_layer(a, "Conv"): + att_diff = [ + (a.filters, b.filters), + (a.kernel_size, b.kernel_size), + (a.stride, b.stride), + ] + return attribute_difference(att_diff) + if is_layer(a, "Pooling"): + att_diff = [ + (a.padding, b.padding), + (a.kernel_size, b.kernel_size), + (a.stride, b.stride), + ] + return attribute_difference(att_diff) + return 0.0 + + +def attribute_difference(att_diff): + ''' The attribute distance. + ''' + + ret = 0 + for a_value, b_value in att_diff: + if max(a_value, b_value) == 0: + ret += 0 + else: + ret += abs(a_value - b_value) * 1.0 / max(a_value, b_value) + return ret * 1.0 / len(att_diff) + + +def layers_distance(list_a, list_b): + """The distance between the layers of two neural networks.""" + len_a = len(list_a) + len_b = len(list_b) + f = np.zeros((len_a + 1, len_b + 1)) + f[-1][-1] = 0 + for i in range(-1, len_a): + f[i][-1] = i + 1 + for j in range(-1, len_b): + f[-1][j] = j + 1 + for i in range(len_a): + for j in range(len_b): + f[i][j] = min( + f[i][j - 1] + 1, + f[i - 1][j] + 1, + f[i - 1][j - 1] + layer_distance(list_a[i], list_b[j]), + ) + return f[len_a - 1][len_b - 1] + + +def skip_connection_distance(a, b): + """The distance between two skip-connections.""" + if a[2] != b[2]: + return 1.0 + len_a = abs(a[1] - a[0]) + len_b = abs(b[1] - b[0]) + return (abs(a[0] - b[0]) + abs(len_a - len_b)) / \ + (max(a[0], b[0]) + max(len_a, len_b)) + + +def skip_connections_distance(list_a, list_b): + """The distance between the skip-connections of two neural networks.""" + distance_matrix = np.zeros((len(list_a), len(list_b))) + for i, a in enumerate(list_a): + for j, b in enumerate(list_b): + distance_matrix[i][j] = skip_connection_distance(a, b) + return distance_matrix[linear_sum_assignment(distance_matrix)].sum() + abs( + len(list_a) - len(list_b) + ) + + +def edit_distance(x, y): + """The distance between two neural networks. + Args: + x: An instance of NetworkDescriptor. + y: An instance of NetworkDescriptor + Returns: + The edit-distance between x and y. + """ + + ret = layers_distance(x.layers, y.layers) + ret += Constant.KERNEL_LAMBDA * skip_connections_distance( + x.skip_connections, y.skip_connections + ) + return ret + + +class IncrementalGaussianProcess: + """Gaussian process regressor. + Attributes: + alpha: A hyperparameter. + """ + + def __init__(self): + self.alpha = 1e-10 + self._distance_matrix = None + self._x = None + self._y = None + self._first_fitted = False + self._l_matrix = None + self._alpha_vector = None + + @property + def kernel_matrix(self): + ''' Kernel matric. + ''' + return self._distance_matrix + + def fit(self, train_x, train_y): + """ Fit the regressor with more data. + Args: + train_x: A list of NetworkDescriptor. + train_y: A list of metric values. + """ + if self.first_fitted: + self.incremental_fit(train_x, train_y) + else: + self.first_fit(train_x, train_y) + + def incremental_fit(self, train_x, train_y): + """ Incrementally fit the regressor. """ + if not self._first_fitted: + raise ValueError( + "The first_fit function needs to be called first.") + + train_x, train_y = np.array(train_x), np.array(train_y) + + # Incrementally compute K + up_right_k = edit_distance_matrix(self._x, train_x) + down_left_k = np.transpose(up_right_k) + down_right_k = edit_distance_matrix(train_x) + up_k = np.concatenate((self._distance_matrix, up_right_k), axis=1) + down_k = np.concatenate((down_left_k, down_right_k), axis=1) + temp_distance_matrix = np.concatenate((up_k, down_k), axis=0) + k_matrix = bourgain_embedding_matrix(temp_distance_matrix) + diagonal = np.diag_indices_from(k_matrix) + diagonal = (diagonal[0][-len(train_x):], diagonal[1][-len(train_x):]) + k_matrix[diagonal] += self.alpha + + try: + self._l_matrix = cholesky(k_matrix, lower=True) # Line 2 + except LinAlgError: + return self + + self._x = np.concatenate((self._x, train_x), axis=0) + self._y = np.concatenate((self._y, train_y), axis=0) + self._distance_matrix = temp_distance_matrix + + self._alpha_vector = cho_solve( + (self._l_matrix, True), self._y) # Line 3 + + return self + + @property + def first_fitted(self): + ''' if it is firsr fitted + ''' + return self._first_fitted + + def first_fit(self, train_x, train_y): + """ Fit the regressor for the first time. """ + train_x, train_y = np.array(train_x), np.array(train_y) + + self._x = np.copy(train_x) + self._y = np.copy(train_y) + + self._distance_matrix = edit_distance_matrix(self._x) + k_matrix = bourgain_embedding_matrix(self._distance_matrix) + k_matrix[np.diag_indices_from(k_matrix)] += self.alpha + + self._l_matrix = cholesky(k_matrix, lower=True) # Line 2 + + self._alpha_vector = cho_solve( + (self._l_matrix, True), self._y) # Line 3 + + self._first_fitted = True + return self + + def predict(self, train_x): + """Predict the result. + Args: + train_x: A list of NetworkDescriptor. + Returns: + y_mean: The predicted mean. + y_std: The predicted standard deviation. + """ + k_trans = np.exp(-np.power(edit_distance_matrix(train_x, self._x), 2)) + y_mean = k_trans.dot(self._alpha_vector) # Line 4 (y_mean = f_star) + + # compute inverse K_inv of K based on its Cholesky + # decomposition L and its inverse L_inv + l_inv = solve_triangular( + self._l_matrix.T, np.eye( + self._l_matrix.shape[0])) + k_inv = l_inv.dot(l_inv.T) + # Compute variance of predictive distribution + y_var = np.ones(len(train_x), dtype=np.float) + y_var -= np.einsum("ij,ij->i", np.dot(k_trans, k_inv), k_trans) + + # Check if any of the variances is negative because of + # numerical issues. If yes: set the variance to 0. + y_var_negative = y_var < 0 + if np.any(y_var_negative): + y_var[y_var_negative] = 0.0 + return y_mean, np.sqrt(y_var) + + +def edit_distance_matrix(train_x, train_y=None): + """Calculate the edit distance. + Args: + train_x: A list of neural architectures. + train_y: A list of neural architectures. + Returns: + An edit-distance matrix. + """ + if train_y is None: + ret = np.zeros((train_x.shape[0], train_x.shape[0])) + for x_index, x in enumerate(train_x): + for y_index, y in enumerate(train_x): + if x_index == y_index: + ret[x_index][y_index] = 0 + elif x_index < y_index: + ret[x_index][y_index] = edit_distance(x, y) + else: + ret[x_index][y_index] = ret[y_index][x_index] + return ret + ret = np.zeros((train_x.shape[0], train_y.shape[0])) + for x_index, x in enumerate(train_x): + for y_index, y in enumerate(train_y): + ret[x_index][y_index] = edit_distance(x, y) + return ret + + +def vector_distance(a, b): + """The Euclidean distance between two vectors.""" + a = np.array(a) + b = np.array(b) + return np.linalg.norm(a - b) + + +def bourgain_embedding_matrix(distance_matrix): + """Use Bourgain algorithm to embed the neural architectures based on their edit-distance. + Args: + distance_matrix: A matrix of edit-distances. + Returns: + A matrix of distances after embedding. + """ + distance_matrix = np.array(distance_matrix) + n = len(distance_matrix) + if n == 1: + return distance_matrix + np.random.seed(123) + distort_elements = [] + r = range(n) + k = int(math.ceil(math.log(n) / math.log(2) - 1)) + t = int(math.ceil(math.log(n))) + counter = 0 + for i in range(0, k + 1): + for t in range(t): + s = np.random.choice(r, 2 ** i) + for j in r: + d = min([distance_matrix[j][s] for s in s]) + counter += len(s) + if i == 0 and t == 0: + distort_elements.append([d]) + else: + distort_elements[j].append(d) + return rbf_kernel(distort_elements, distort_elements) + + +class BayesianOptimizer: + """ A Bayesian optimizer for neural architectures. + Attributes: + searcher: The Searcher which is calling the Bayesian optimizer. + t_min: The minimum temperature for simulated annealing. + metric: An instance of the Metric subclasses. + gpr: A GaussianProcessRegressor for bayesian optimization. + beta: The beta in acquisition function. (refer to our paper) + search_tree: The network morphism search tree. + """ + + def __init__(self, searcher, t_min, optimizemode, beta=None): + self.searcher = searcher + self.t_min = t_min + self.optimizemode = optimizemode + self.gpr = IncrementalGaussianProcess() + self.beta = beta if beta is not None else Constant.BETA + self.search_tree = SearchTree() + + def fit(self, x_queue, y_queue): + """ Fit the optimizer with new architectures and performances. + Args: + x_queue: A list of NetworkDescriptor. + y_queue: A list of metric values. + """ + self.gpr.fit(x_queue, y_queue) + + def generate(self, descriptors): + """Generate new architecture. + Args: + descriptors: All the searched neural architectures. + Returns: + graph: An instance of Graph. A morphed neural network with weights. + father_id: The father node ID in the search tree. + """ + model_ids = self.search_tree.adj_list.keys() + + target_graph = None + father_id = None + descriptors = deepcopy(descriptors) + elem_class = Elem + if self.optimizemode is OptimizeMode.Maximize: + elem_class = ReverseElem + + # Initialize the priority queue. + pq = PriorityQueue() + temp_list = [] + for model_id in model_ids: + metric_value = self.searcher.get_metric_value_by_id(model_id) + temp_list.append((metric_value, model_id)) + temp_list = sorted(temp_list) + for metric_value, model_id in temp_list: + graph = self.searcher.load_model_by_id(model_id) + graph.clear_operation_history() + graph.clear_weights() + pq.put(elem_class(metric_value, model_id, graph)) + + t = 1.0 + t_min = self.t_min + alpha = 0.9 + opt_acq = self._get_init_opt_acq_value() + while not pq.empty() and t > t_min: + elem = pq.get() + if self.optimizemode is OptimizeMode.Maximize: + temp_exp = min((elem.metric_value - opt_acq) / t, 1.0) + else: + temp_exp = min((opt_acq - elem.metric_value) / t, 1.0) + ap = math.exp(temp_exp) + if ap >= random.uniform(0, 1): + for temp_graph in transform(elem.graph): + if contain(descriptors, temp_graph.extract_descriptor()): + continue + + temp_acq_value = self.acq(temp_graph) + pq.put( + elem_class( + temp_acq_value, + elem.father_id, + temp_graph)) + descriptors.append(temp_graph.extract_descriptor()) + if self._accept_new_acq_value(opt_acq, temp_acq_value): + opt_acq = temp_acq_value + father_id = elem.father_id + target_graph = deepcopy(temp_graph) + t *= alpha + + # Did not found a not duplicated architecture + if father_id is None: + return None, None + nm_graph = self.searcher.load_model_by_id(father_id) + for args in target_graph.operation_history: + getattr(nm_graph, args[0])(*list(args[1:])) + return nm_graph, father_id + + def acq(self, graph): + ''' estimate the value of generated graph + ''' + mean, std = self.gpr.predict(np.array([graph.extract_descriptor()])) + if self.optimizemode is OptimizeMode.Maximize: + return mean + self.beta * std + return mean - self.beta * std + + def _get_init_opt_acq_value(self): + if self.optimizemode is OptimizeMode.Maximize: + return -np.inf + return np.inf + + def _accept_new_acq_value(self, opt_acq, temp_acq_value): + if temp_acq_value > opt_acq and self.optimizemode is OptimizeMode.Maximize: + return True + if temp_acq_value < opt_acq and not self.optimizemode is OptimizeMode.Maximize: + return True + return False + + def add_child(self, father_id, model_id): + ''' add child to the search tree + Arguments: + father_id {int} -- father id + model_id {int} -- model id + ''' + + self.search_tree.add_child(father_id, model_id) + + +@total_ordering +class Elem: + """Elements to be sorted according to metric value.""" + + def __init__(self, metric_value, father_id, graph): + self.father_id = father_id + self.graph = graph + self.metric_value = metric_value + + def __eq__(self, other): + return self.metric_value == other.metric_value + + def __lt__(self, other): + return self.metric_value < other.metric_value + + +class ReverseElem(Elem): + """Elements to be reversely sorted according to metric value.""" + + def __lt__(self, other): + return self.metric_value > other.metric_value + + +def contain(descriptors, target_descriptor): + """Check if the target descriptor is in the descriptors.""" + for descriptor in descriptors: + if edit_distance(descriptor, target_descriptor) < 1e-5: + return True + return False + + +class SearchTree: + """The network morphism search tree.""" + + def __init__(self): + self.root = None + self.adj_list = {} + + def add_child(self, u, v): + ''' add child to search tree itself. + Arguments: + u {int} -- father id + v {int} -- child id + ''' + + if u == -1: + self.root = v + self.adj_list[v] = [] + return + if v not in self.adj_list[u]: + self.adj_list[u].append(v) + if v not in self.adj_list: + self.adj_list[v] = [] + + def get_dict(self, u=None): + """ A recursive function to return the content of the tree in a dict.""" + if u is None: + return self.get_dict(self.root) + children = [] + for v in self.adj_list[u]: + children.append(self.get_dict(v)) + ret = {"name": u, "children": children} + return ret diff --git a/utils/third_party/nni_new/algorithms/hpo/networkmorphism_tuner/graph.py b/utils/third_party/nni_new/algorithms/hpo/networkmorphism_tuner/graph.py new file mode 100644 index 0000000000000000000000000000000000000000..9c96b6c2f07088aceb0cb029de02cb4b2f16ae32 --- /dev/null +++ b/utils/third_party/nni_new/algorithms/hpo/networkmorphism_tuner/graph.py @@ -0,0 +1,995 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import json +from collections.abc import Iterable +from copy import deepcopy, copy +from queue import Queue + +import numpy as np +import torch + +from .layer_transformer import ( + add_noise, + wider_bn, + wider_next_conv, + wider_next_dense, + wider_pre_conv, + wider_pre_dense, + init_dense_weight, + init_conv_weight, + init_bn_weight, +) +from .layers import ( + StubAdd, + StubConcatenate, + StubReLU, + get_batch_norm_class, + get_conv_class, + is_layer, + layer_width, + set_keras_weight_to_stub, + set_stub_weight_to_keras, + set_stub_weight_to_torch, + set_torch_weight_to_stub, + to_real_keras_layer, + layer_description_extractor, + layer_description_builder, +) +from .utils import Constant + + +class NetworkDescriptor: + """A class describing the neural architecture for neural network kernel. + It only record the width of convolutional and dense layers, and the skip-connection types and positions. + """ + + CONCAT_CONNECT = "concat" + ADD_CONNECT = "add" + + def __init__(self): + self.skip_connections = [] + self.layers = [] + + @property + def n_layers(self): + return len(self.layers) + + def add_skip_connection(self, u, v, connection_type): + """ Add a skip-connection to the descriptor. + Args: + u: Number of convolutional layers before the starting point. + v: Number of convolutional layers before the ending point. + connection_type: Must be either CONCAT_CONNECT or ADD_CONNECT. + """ + if connection_type not in [self.CONCAT_CONNECT, self.ADD_CONNECT]: + raise ValueError( + "connection_type should be NetworkDescriptor.CONCAT_CONNECT " + "or NetworkDescriptor.ADD_CONNECT." + ) + self.skip_connections.append((u, v, connection_type)) + + def to_json(self): + ''' NetworkDescriptor to json representation + ''' + + skip_list = [] + for u, v, connection_type in self.skip_connections: + skip_list.append({"from": u, "to": v, "type": connection_type}) + return {"node_list": self.layers, "skip_list": skip_list} + + def add_layer(self, layer): + ''' add one layer + ''' + + self.layers.append(layer) + + +class Node: + """A class for intermediate output tensor (node) in the Graph. + Attributes: + shape: A tuple describing the shape of the tensor. + """ + + def __init__(self, shape): + self.shape = shape + + +class Graph: + """A class representing the neural architecture graph of a model. + Graph extracts the neural architecture graph from a model. + Each node in the graph is a intermediate tensor between layers. + Each layer is an edge in the graph. + Notably, multiple edges may refer to the same layer. + (e.g. Add layer is adding two tensor into one tensor. So it is related to two edges.) + Attributes: + weighted: A boolean of whether the weights and biases in the neural network + should be included in the graph. + input_shape: A tuple of integers, which does not include the batch axis. + node_list: A list of integers. The indices of the list are the identifiers. + layer_list: A list of stub layers. The indices of the list are the identifiers. + node_to_id: A dict instance mapping from node integers to their identifiers. + layer_to_id: A dict instance mapping from stub layers to their identifiers. + layer_id_to_input_node_ids: A dict instance mapping from layer identifiers + to their input nodes identifiers. + layer_id_to_output_node_ids: A dict instance mapping from layer identifiers + to their output nodes identifiers. + adj_list: A two dimensional list. The adjacency list of the graph. The first dimension is + identified by tensor identifiers. In each edge list, the elements are two-element tuples + of (tensor identifier, layer identifier). + reverse_adj_list: A reverse adjacent list in the same format as adj_list. + operation_history: A list saving all the network morphism operations. + vis: A dictionary of temporary storage for whether an local operation has been done + during the network morphism. + """ + + def __init__(self, input_shape, weighted=True): + """Initializer for Graph. + """ + self.input_shape = input_shape + self.weighted = weighted + self.node_list = [] + self.layer_list = [] + # node id start with 0 + self.node_to_id = {} + self.layer_to_id = {} + self.layer_id_to_input_node_ids = {} + self.layer_id_to_output_node_ids = {} + self.adj_list = {} + self.reverse_adj_list = {} + self.operation_history = [] + self.n_dim = len(input_shape) - 1 + self.conv = get_conv_class(self.n_dim) + self.batch_norm = get_batch_norm_class(self.n_dim) + + self.vis = None + self._add_node(Node(input_shape)) + + def add_layer(self, layer, input_node_id): + """Add a layer to the Graph. + Args: + layer: An instance of the subclasses of StubLayer in layers.py. + input_node_id: An integer. The ID of the input node of the layer. + Returns: + output_node_id: An integer. The ID of the output node of the layer. + """ + if isinstance(input_node_id, Iterable): + layer.input = list(map(lambda x: self.node_list[x], input_node_id)) + output_node_id = self._add_node(Node(layer.output_shape)) + for node_id in input_node_id: + self._add_edge(layer, node_id, output_node_id) + + else: + layer.input = self.node_list[input_node_id] + output_node_id = self._add_node(Node(layer.output_shape)) + self._add_edge(layer, input_node_id, output_node_id) + + layer.output = self.node_list[output_node_id] + return output_node_id + + def clear_operation_history(self): + self.operation_history = [] + + @property + def n_nodes(self): + """Return the number of nodes in the model.""" + return len(self.node_list) + + @property + def n_layers(self): + """Return the number of layers in the model.""" + return len(self.layer_list) + + def _add_node(self, node): + """Add a new node to node_list and give the node an ID. + Args: + node: An instance of Node. + Returns: + node_id: An integer. + """ + node_id = len(self.node_list) + self.node_to_id[node] = node_id + self.node_list.append(node) + self.adj_list[node_id] = [] + self.reverse_adj_list[node_id] = [] + return node_id + + def _add_edge(self, layer, input_id, output_id): + """Add a new layer to the graph. The nodes should be created in advance.""" + + if layer in self.layer_to_id: + layer_id = self.layer_to_id[layer] + if input_id not in self.layer_id_to_input_node_ids[layer_id]: + self.layer_id_to_input_node_ids[layer_id].append(input_id) + if output_id not in self.layer_id_to_output_node_ids[layer_id]: + self.layer_id_to_output_node_ids[layer_id].append(output_id) + else: + layer_id = len(self.layer_list) + self.layer_list.append(layer) + self.layer_to_id[layer] = layer_id + self.layer_id_to_input_node_ids[layer_id] = [input_id] + self.layer_id_to_output_node_ids[layer_id] = [output_id] + + self.adj_list[input_id].append((output_id, layer_id)) + self.reverse_adj_list[output_id].append((input_id, layer_id)) + + def _redirect_edge(self, u_id, v_id, new_v_id): + """Redirect the layer to a new node. + Change the edge originally from `u_id` to `v_id` into an edge from `u_id` to `new_v_id` + while keeping all other property of the edge the same. + """ + layer_id = None + for index, edge_tuple in enumerate(self.adj_list[u_id]): + if edge_tuple[0] == v_id: + layer_id = edge_tuple[1] + self.adj_list[u_id][index] = (new_v_id, layer_id) + self.layer_list[layer_id].output = self.node_list[new_v_id] + break + + for index, edge_tuple in enumerate(self.reverse_adj_list[v_id]): + if edge_tuple[0] == u_id: + layer_id = edge_tuple[1] + self.reverse_adj_list[v_id].remove(edge_tuple) + break + self.reverse_adj_list[new_v_id].append((u_id, layer_id)) + for index, value in enumerate( + self.layer_id_to_output_node_ids[layer_id]): + if value == v_id: + self.layer_id_to_output_node_ids[layer_id][index] = new_v_id + break + + def _replace_layer(self, layer_id, new_layer): + """Replace the layer with a new layer.""" + old_layer = self.layer_list[layer_id] + new_layer.input = old_layer.input + new_layer.output = old_layer.output + new_layer.output.shape = new_layer.output_shape + self.layer_list[layer_id] = new_layer + self.layer_to_id[new_layer] = layer_id + self.layer_to_id.pop(old_layer) + + @property + def topological_order(self): + """Return the topological order of the node IDs from the input node to the output node.""" + q = Queue() + in_degree = {} + for i in range(self.n_nodes): + in_degree[i] = 0 + for u in range(self.n_nodes): + for v, _ in self.adj_list[u]: + in_degree[v] += 1 + for i in range(self.n_nodes): + if in_degree[i] == 0: + q.put(i) + + order_list = [] + while not q.empty(): + u = q.get() + order_list.append(u) + for v, _ in self.adj_list[u]: + in_degree[v] -= 1 + if in_degree[v] == 0: + q.put(v) + return order_list + + def _get_pooling_layers(self, start_node_id, end_node_id): + """Given two node IDs, return all the pooling layers between them.""" + layer_list = [] + node_list = [start_node_id] + assert self._depth_first_search(end_node_id, layer_list, node_list) + ret = [] + for layer_id in layer_list: + layer = self.layer_list[layer_id] + if is_layer(layer, "Pooling"): + ret.append(layer) + elif is_layer(layer, "Conv") and layer.stride != 1: + ret.append(layer) + return ret + + def _depth_first_search(self, target_id, layer_id_list, node_list): + """Search for all the layers and nodes down the path. + A recursive function to search all the layers and nodes between the node in the node_list + and the node with target_id.""" + assert len(node_list) <= self.n_nodes + u = node_list[-1] + if u == target_id: + return True + + for v, layer_id in self.adj_list[u]: + layer_id_list.append(layer_id) + node_list.append(v) + if self._depth_first_search(target_id, layer_id_list, node_list): + return True + layer_id_list.pop() + node_list.pop() + + return False + + def _search(self, u, start_dim, total_dim, n_add): + """Search the graph for all the layers to be widened caused by an operation. + It is an recursive function with duplication check to avoid deadlock. + It searches from a starting node u until the corresponding layers has been widened. + Args: + u: The starting node ID. + start_dim: The position to insert the additional dimensions. + total_dim: The total number of dimensions the layer has before widening. + n_add: The number of dimensions to add. + """ + if (u, start_dim, total_dim, n_add) in self.vis: + return + self.vis[(u, start_dim, total_dim, n_add)] = True + for v, layer_id in self.adj_list[u]: + layer = self.layer_list[layer_id] + + if is_layer(layer, "Conv"): + new_layer = wider_next_conv( + layer, start_dim, total_dim, n_add, self.weighted + ) + self._replace_layer(layer_id, new_layer) + + elif is_layer(layer, "Dense"): + new_layer = wider_next_dense( + layer, start_dim, total_dim, n_add, self.weighted + ) + self._replace_layer(layer_id, new_layer) + + elif is_layer(layer, "BatchNormalization"): + new_layer = wider_bn( + layer, start_dim, total_dim, n_add, self.weighted) + self._replace_layer(layer_id, new_layer) + self._search(v, start_dim, total_dim, n_add) + + elif is_layer(layer, "Concatenate"): + if self.layer_id_to_input_node_ids[layer_id][1] == u: + # u is on the right of the concat + # next_start_dim += next_total_dim - total_dim + left_dim = self._upper_layer_width( + self.layer_id_to_input_node_ids[layer_id][0] + ) + next_start_dim = start_dim + left_dim + next_total_dim = total_dim + left_dim + else: + next_start_dim = start_dim + next_total_dim = total_dim + self._upper_layer_width( + self.layer_id_to_input_node_ids[layer_id][1] + ) + self._search(v, next_start_dim, next_total_dim, n_add) + + else: + self._search(v, start_dim, total_dim, n_add) + + for v, layer_id in self.reverse_adj_list[u]: + layer = self.layer_list[layer_id] + if is_layer(layer, "Conv"): + new_layer = wider_pre_conv(layer, n_add, self.weighted) + self._replace_layer(layer_id, new_layer) + elif is_layer(layer, "Dense"): + new_layer = wider_pre_dense(layer, n_add, self.weighted) + self._replace_layer(layer_id, new_layer) + elif is_layer(layer, "Concatenate"): + continue + else: + self._search(v, start_dim, total_dim, n_add) + + def _upper_layer_width(self, u): + for v, layer_id in self.reverse_adj_list[u]: + layer = self.layer_list[layer_id] + if is_layer(layer, "Conv") or is_layer(layer, "Dense"): + return layer_width(layer) + elif is_layer(layer, "Concatenate"): + a = self.layer_id_to_input_node_ids[layer_id][0] + b = self.layer_id_to_input_node_ids[layer_id][1] + return self._upper_layer_width(a) + self._upper_layer_width(b) + else: + return self._upper_layer_width(v) + return self.node_list[0].shape[-1] + + def to_deeper_model(self, target_id, new_layer): + """Insert a relu-conv-bn block after the target block. + Args: + target_id: A convolutional layer ID. The new block should be inserted after the block. + new_layer: An instance of StubLayer subclasses. + """ + self.operation_history.append( + ("to_deeper_model", target_id, new_layer)) + input_id = self.layer_id_to_input_node_ids[target_id][0] + output_id = self.layer_id_to_output_node_ids[target_id][0] + if self.weighted: + if is_layer(new_layer, "Dense"): + init_dense_weight(new_layer) + elif is_layer(new_layer, "Conv"): + init_conv_weight(new_layer) + elif is_layer(new_layer, "BatchNormalization"): + init_bn_weight(new_layer) + + self._insert_new_layers([new_layer], input_id, output_id) + + def to_wider_model(self, pre_layer_id, n_add): + """Widen the last dimension of the output of the pre_layer. + Args: + pre_layer_id: The ID of a convolutional layer or dense layer. + n_add: The number of dimensions to add. + """ + self.operation_history.append(("to_wider_model", pre_layer_id, n_add)) + pre_layer = self.layer_list[pre_layer_id] + output_id = self.layer_id_to_output_node_ids[pre_layer_id][0] + dim = layer_width(pre_layer) + self.vis = {} + self._search(output_id, dim, dim, n_add) + # Update the tensor shapes. + for u in self.topological_order: + for v, layer_id in self.adj_list[u]: + self.node_list[v].shape = self.layer_list[layer_id].output_shape + + def _insert_new_layers(self, new_layers, start_node_id, end_node_id): + """Insert the new_layers after the node with start_node_id.""" + new_node_id = self._add_node(deepcopy(self.node_list[end_node_id])) + temp_output_id = new_node_id + for layer in new_layers[:-1]: + temp_output_id = self.add_layer(layer, temp_output_id) + + self._add_edge(new_layers[-1], temp_output_id, end_node_id) + new_layers[-1].input = self.node_list[temp_output_id] + new_layers[-1].output = self.node_list[end_node_id] + self._redirect_edge(start_node_id, end_node_id, new_node_id) + + def _block_end_node(self, layer_id, block_size): + ret = self.layer_id_to_output_node_ids[layer_id][0] + for _ in range(block_size - 2): + ret = self.adj_list[ret][0][0] + return ret + + def _dense_block_end_node(self, layer_id): + return self.layer_id_to_input_node_ids[layer_id][0] + + def _conv_block_end_node(self, layer_id): + """Get the input node ID of the last layer in the block by layer ID. + Return the input node ID of the last layer in the convolutional block. + Args: + layer_id: the convolutional layer ID. + """ + return self._block_end_node(layer_id, Constant.CONV_BLOCK_DISTANCE) + + def to_add_skip_model(self, start_id, end_id): + """Add a weighted add skip-connection from after start node to end node. + Args: + start_id: The convolutional layer ID, after which to start the skip-connection. + end_id: The convolutional layer ID, after which to end the skip-connection. + """ + self.operation_history.append(("to_add_skip_model", start_id, end_id)) + filters_end = self.layer_list[end_id].output.shape[-1] + filters_start = self.layer_list[start_id].output.shape[-1] + start_node_id = self.layer_id_to_output_node_ids[start_id][0] + + pre_end_node_id = self.layer_id_to_input_node_ids[end_id][0] + end_node_id = self.layer_id_to_output_node_ids[end_id][0] + + skip_output_id = self._insert_pooling_layer_chain( + start_node_id, end_node_id) + + # Add the conv layer + new_conv_layer = get_conv_class( + self.n_dim)( + filters_start, + filters_end, + 1) + skip_output_id = self.add_layer(new_conv_layer, skip_output_id) + + # Add the add layer. + add_input_node_id = self._add_node( + deepcopy(self.node_list[end_node_id])) + add_layer = StubAdd() + + self._redirect_edge(pre_end_node_id, end_node_id, add_input_node_id) + self._add_edge(add_layer, add_input_node_id, end_node_id) + self._add_edge(add_layer, skip_output_id, end_node_id) + add_layer.input = [ + self.node_list[add_input_node_id], + self.node_list[skip_output_id], + ] + add_layer.output = self.node_list[end_node_id] + self.node_list[end_node_id].shape = add_layer.output_shape + + # Set weights to the additional conv layer. + if self.weighted: + filter_shape = (1,) * self.n_dim + weights = np.zeros((filters_end, filters_start) + filter_shape) + bias = np.zeros(filters_end) + new_conv_layer.set_weights( + (add_noise(weights, np.array([0, 1])), add_noise( + bias, np.array([0, 1]))) + ) + + def to_concat_skip_model(self, start_id, end_id): + """Add a weighted add concatenate connection from after start node to end node. + Args: + start_id: The convolutional layer ID, after which to start the skip-connection. + end_id: The convolutional layer ID, after which to end the skip-connection. + """ + self.operation_history.append( + ("to_concat_skip_model", start_id, end_id)) + filters_end = self.layer_list[end_id].output.shape[-1] + filters_start = self.layer_list[start_id].output.shape[-1] + start_node_id = self.layer_id_to_output_node_ids[start_id][0] + + pre_end_node_id = self.layer_id_to_input_node_ids[end_id][0] + end_node_id = self.layer_id_to_output_node_ids[end_id][0] + + skip_output_id = self._insert_pooling_layer_chain( + start_node_id, end_node_id) + + concat_input_node_id = self._add_node( + deepcopy(self.node_list[end_node_id])) + self._redirect_edge(pre_end_node_id, end_node_id, concat_input_node_id) + + concat_layer = StubConcatenate() + concat_layer.input = [ + self.node_list[concat_input_node_id], + self.node_list[skip_output_id], + ] + concat_output_node_id = self._add_node(Node(concat_layer.output_shape)) + self._add_edge( + concat_layer, + concat_input_node_id, + concat_output_node_id) + self._add_edge(concat_layer, skip_output_id, concat_output_node_id) + concat_layer.output = self.node_list[concat_output_node_id] + self.node_list[concat_output_node_id].shape = concat_layer.output_shape + + # Add the concatenate layer. + new_conv_layer = get_conv_class(self.n_dim)( + filters_start + filters_end, filters_end, 1 + ) + self._add_edge(new_conv_layer, concat_output_node_id, end_node_id) + new_conv_layer.input = self.node_list[concat_output_node_id] + new_conv_layer.output = self.node_list[end_node_id] + self.node_list[end_node_id].shape = new_conv_layer.output_shape + + if self.weighted: + filter_shape = (1,) * self.n_dim + weights = np.zeros((filters_end, filters_end) + filter_shape) + for i in range(filters_end): + filter_weight = np.zeros((filters_end,) + filter_shape) + center_index = (i,) + (0,) * self.n_dim + filter_weight[center_index] = 1 + weights[i, ...] = filter_weight + weights = np.concatenate( + (weights, np.zeros((filters_end, filters_start) + filter_shape)), axis=1 + ) + bias = np.zeros(filters_end) + new_conv_layer.set_weights( + (add_noise(weights, np.array([0, 1])), add_noise( + bias, np.array([0, 1]))) + ) + + def _insert_pooling_layer_chain(self, start_node_id, end_node_id): + skip_output_id = start_node_id + for layer in self._get_pooling_layers(start_node_id, end_node_id): + new_layer = deepcopy(layer) + if is_layer(new_layer, "Conv"): + filters = self.node_list[start_node_id].shape[-1] + new_layer = get_conv_class(self.n_dim)( + filters, filters, 1, layer.stride) + if self.weighted: + init_conv_weight(new_layer) + else: + new_layer = deepcopy(layer) + skip_output_id = self.add_layer(new_layer, skip_output_id) + skip_output_id = self.add_layer(StubReLU(), skip_output_id) + return skip_output_id + + def extract_descriptor(self): + """Extract the the description of the Graph as an instance of NetworkDescriptor.""" + main_chain = self.get_main_chain() + index_in_main_chain = {} + for index, u in enumerate(main_chain): + index_in_main_chain[u] = index + + ret = NetworkDescriptor() + for u in main_chain: + for v, layer_id in self.adj_list[u]: + if v not in index_in_main_chain: + continue + layer = self.layer_list[layer_id] + copied_layer = copy(layer) + copied_layer.weights = None + ret.add_layer(deepcopy(copied_layer)) + + for u in index_in_main_chain: + for v, layer_id in self.adj_list[u]: + if v not in index_in_main_chain: + temp_u = u + temp_v = v + temp_layer_id = layer_id + skip_type = None + while not ( + temp_v in index_in_main_chain and temp_u in index_in_main_chain): + if is_layer( + self.layer_list[temp_layer_id], "Concatenate"): + skip_type = NetworkDescriptor.CONCAT_CONNECT + if is_layer(self.layer_list[temp_layer_id], "Add"): + skip_type = NetworkDescriptor.ADD_CONNECT + temp_u = temp_v + temp_v, temp_layer_id = self.adj_list[temp_v][0] + ret.add_skip_connection( + index_in_main_chain[u], index_in_main_chain[temp_u], skip_type + ) + + elif index_in_main_chain[v] - index_in_main_chain[u] != 1: + skip_type = None + if is_layer(self.layer_list[layer_id], "Concatenate"): + skip_type = NetworkDescriptor.CONCAT_CONNECT + if is_layer(self.layer_list[layer_id], "Add"): + skip_type = NetworkDescriptor.ADD_CONNECT + ret.add_skip_connection( + index_in_main_chain[u], index_in_main_chain[v], skip_type + ) + + return ret + + def clear_weights(self): + ''' clear weights of the graph + ''' + self.weighted = False + for layer in self.layer_list: + layer.weights = None + + def produce_torch_model(self): + """Build a new Torch model based on the current graph.""" + return TorchModel(self) + + def produce_keras_model(self): + """Build a new keras model based on the current graph.""" + return KerasModel(self).model + + def produce_onnx_model(self): + """Build a new ONNX model based on the current graph.""" + return ONNXModel(self) + + def parsing_onnx_model(self, onnx_model): + '''to do in the future to use the onnx model + ''' + return ONNXModel(onnx_model) + + def produce_json_model(self): + """Build a new Json model based on the current graph.""" + return JSONModel(self).data + + @classmethod + def parsing_json_model(cls, json_model): + '''build a graph from json + ''' + return json_to_graph(json_model) + + def _layer_ids_in_order(self, layer_ids): + node_id_to_order_index = {} + for index, node_id in enumerate(self.topological_order): + node_id_to_order_index[node_id] = index + return sorted( + layer_ids, + key=lambda layer_id: node_id_to_order_index[ + self.layer_id_to_output_node_ids[layer_id][0] + ], + ) + + def _layer_ids_by_type(self, type_str): + return list( + filter( + lambda layer_id: is_layer(self.layer_list[layer_id], type_str), + range(self.n_layers), + ) + ) + + def get_main_chain_layers(self): + """Return a list of layer IDs in the main chain.""" + main_chain = self.get_main_chain() + ret = [] + for u in main_chain: + for v, layer_id in self.adj_list[u]: + if v in main_chain and u in main_chain: + ret.append(layer_id) + return ret + + def _conv_layer_ids_in_order(self): + return list( + filter( + lambda layer_id: is_layer(self.layer_list[layer_id], "Conv"), + self.get_main_chain_layers(), + ) + ) + + def _dense_layer_ids_in_order(self): + return self._layer_ids_in_order(self._layer_ids_by_type("Dense")) + + def deep_layer_ids(self): + ret = [] + for layer_id in self.get_main_chain_layers(): + layer = self.layer_list[layer_id] + if is_layer(layer, "GlobalAveragePooling"): + break + if is_layer(layer, "Add") or is_layer(layer, "Concatenate"): + continue + ret.append(layer_id) + return ret + + def wide_layer_ids(self): + return ( + self._conv_layer_ids_in_order( + )[:-1] + self._dense_layer_ids_in_order()[:-1] + ) + + def skip_connection_layer_ids(self): + return self.deep_layer_ids()[:-1] + + def size(self): + return sum(list(map(lambda x: x.size(), self.layer_list))) + + def get_main_chain(self): + """Returns the main chain node ID list.""" + pre_node = {} + distance = {} + for i in range(self.n_nodes): + distance[i] = 0 + pre_node[i] = i + for i in range(self.n_nodes - 1): + for u in range(self.n_nodes): + for v, _ in self.adj_list[u]: + if distance[u] + 1 > distance[v]: + distance[v] = distance[u] + 1 + pre_node[v] = u + temp_id = 0 + for i in range(self.n_nodes): + if distance[i] > distance[temp_id]: + temp_id = i + ret = [] + for i in range(self.n_nodes + 5): + ret.append(temp_id) + if pre_node[temp_id] == temp_id: + break + temp_id = pre_node[temp_id] + assert temp_id == pre_node[temp_id] + ret.reverse() + return ret + + +class TorchModel(torch.nn.Module): + """A neural network class using pytorch constructed from an instance of Graph.""" + + def __init__(self, graph): + super(TorchModel, self).__init__() + self.graph = graph + self.layers = [] + for layer in graph.layer_list: + self.layers.append(layer.to_real_layer()) + if graph.weighted: + for index, layer in enumerate(self.layers): + set_stub_weight_to_torch(self.graph.layer_list[index], layer) + for index, layer in enumerate(self.layers): + self.add_module(str(index), layer) + + def forward(self, input_tensor): + topo_node_list = self.graph.topological_order + output_id = topo_node_list[-1] + input_id = topo_node_list[0] + + node_list = deepcopy(self.graph.node_list) + node_list[input_id] = input_tensor + + for v in topo_node_list: + for u, layer_id in self.graph.reverse_adj_list[v]: + layer = self.graph.layer_list[layer_id] + torch_layer = self.layers[layer_id] + + if isinstance(layer, (StubAdd, StubConcatenate)): + edge_input_tensor = list( + map( + lambda x: node_list[x], + self.graph.layer_id_to_input_node_ids[layer_id], + ) + ) + else: + edge_input_tensor = node_list[u] + + temp_tensor = torch_layer(edge_input_tensor) + node_list[v] = temp_tensor + return node_list[output_id] + + def set_weight_to_graph(self): + self.graph.weighted = True + for index, layer in enumerate(self.layers): + set_torch_weight_to_stub(layer, self.graph.layer_list[index]) + + +class KerasModel: + def __init__(self, graph): + import keras + + self.graph = graph + self.layers = [] + for layer in graph.layer_list: + self.layers.append(to_real_keras_layer(layer)) + + # Construct the keras graph. + # Input + topo_node_list = self.graph.topological_order + output_id = topo_node_list[-1] + input_id = topo_node_list[0] + input_tensor = keras.layers.Input( + shape=graph.node_list[input_id].shape) + + node_list = deepcopy(self.graph.node_list) + node_list[input_id] = input_tensor + + # Output + for v in topo_node_list: + for u, layer_id in self.graph.reverse_adj_list[v]: + layer = self.graph.layer_list[layer_id] + keras_layer = self.layers[layer_id] + + if isinstance(layer, (StubAdd, StubConcatenate)): + edge_input_tensor = list( + map( + lambda x: node_list[x], + self.graph.layer_id_to_input_node_ids[layer_id], + ) + ) + else: + edge_input_tensor = node_list[u] + + temp_tensor = keras_layer(edge_input_tensor) + node_list[v] = temp_tensor + + output_tensor = node_list[output_id] + output_tensor = keras.layers.Activation("softmax", name="activation_add")( + output_tensor + ) + self.model = keras.models.Model( + inputs=input_tensor, outputs=output_tensor) + + if graph.weighted: + for index, layer in enumerate(self.layers): + set_stub_weight_to_keras(self.graph.layer_list[index], layer) + + def set_weight_to_graph(self): + self.graph.weighted = True + for index, layer in enumerate(self.layers): + set_keras_weight_to_stub(layer, self.graph.layer_list[index]) + + +class ONNXModel: + # to do in the future using onnx ir + def __init__(self, graph): + pass + + +class JSONModel: + def __init__(self, graph): + data = dict() + node_list = list() + layer_list = list() + operation_history = list() + + data["input_shape"] = graph.input_shape + vis = graph.vis + data["vis"] = list(vis.keys()) if vis is not None else None + data["weighted"] = graph.weighted + + for item in graph.operation_history: + if item[0] == "to_deeper_model": + operation_history.append( + [ + item[0], + item[1], + layer_description_extractor(item[2], graph.node_to_id), + ] + ) + else: + operation_history.append(item) + data["operation_history"] = operation_history + data["layer_id_to_input_node_ids"] = graph.layer_id_to_input_node_ids + data["layer_id_to_output_node_ids"] = graph.layer_id_to_output_node_ids + data["adj_list"] = graph.adj_list + data["reverse_adj_list"] = graph.reverse_adj_list + + for node in graph.node_list: + node_id = graph.node_to_id[node] + node_information = node.shape + node_list.append((node_id, node_information)) + + for layer_id, item in enumerate(graph.layer_list): + layer = graph.layer_list[layer_id] + layer_information = layer_description_extractor( + layer, graph.node_to_id) + layer_list.append((layer_id, layer_information)) + + data["node_list"] = node_list + data["layer_list"] = layer_list + + self.data = data + + +def graph_to_onnx(graph, onnx_model_path): + import onnx + # to do in the future using onnx ir + onnx_out = graph.produce_onnx_model() + onnx.save(onnx_out, onnx_model_path) + return onnx_out + + +def onnx_to_graph(onnx_model, input_shape): + # to do in the future using onnx ir + graph = Graph(input_shape, False) + graph.parsing_onnx_model(onnx_model) + return graph + + +def graph_to_json(graph, json_model_path): + json_out = graph.produce_json_model() + with open(json_model_path, "w") as fout: + json.dump(json_out, fout) + json_out = json.dumps(json_out) + return json_out + + +def json_to_graph(json_model: str): + json_model = json.loads(json_model) + # restore graph data from json data + input_shape = tuple(json_model["input_shape"]) + node_list = list() + node_to_id = dict() + id_to_node = dict() + layer_list = list() + layer_to_id = dict() + operation_history = list() + graph = Graph(input_shape, False) + + graph.input_shape = input_shape + vis = json_model["vis"] + graph.vis = { + tuple(item): True for item in vis} if vis is not None else None + graph.weighted = json_model["weighted"] + layer_id_to_input_node_ids = json_model["layer_id_to_input_node_ids"] + graph.layer_id_to_input_node_ids = { + int(k): v for k, v in layer_id_to_input_node_ids.items() + } + layer_id_to_output_node_ids = json_model["layer_id_to_output_node_ids"] + graph.layer_id_to_output_node_ids = { + int(k): v for k, v in layer_id_to_output_node_ids.items() + } + adj_list = {} + for k, v in json_model["adj_list"].items(): + adj_list[int(k)] = [tuple(i) for i in v] + graph.adj_list = adj_list + reverse_adj_list = {} + for k, v in json_model["reverse_adj_list"].items(): + reverse_adj_list[int(k)] = [tuple(i) for i in v] + graph.reverse_adj_list = reverse_adj_list + + for item in json_model["node_list"]: + new_node = Node(tuple(item[1])) + node_id = item[0] + node_list.append(new_node) + node_to_id[new_node] = node_id + id_to_node[node_id] = new_node + + for item in json_model["operation_history"]: + if item[0] == "to_deeper_model": + operation_history.append( + (item[0], item[1], layer_description_builder(item[2], id_to_node)) + ) + else: + operation_history.append(item) + graph.operation_history = operation_history + + for item in json_model["layer_list"]: + new_layer = layer_description_builder(item[1], id_to_node) + layer_id = int(item[0]) + layer_list.append(new_layer) + layer_to_id[new_layer] = layer_id + + graph.node_list = node_list + graph.node_to_id = node_to_id + graph.layer_list = layer_list + graph.layer_to_id = layer_to_id + + return graph diff --git a/utils/third_party/nni_new/algorithms/hpo/networkmorphism_tuner/graph_transformer.py b/utils/third_party/nni_new/algorithms/hpo/networkmorphism_tuner/graph_transformer.py new file mode 100644 index 0000000000000000000000000000000000000000..b03112d63bcc50f5c42e4203dfd4adb70a061119 --- /dev/null +++ b/utils/third_party/nni_new/algorithms/hpo/networkmorphism_tuner/graph_transformer.py @@ -0,0 +1,167 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from copy import deepcopy + +from random import randrange, sample + +from .graph import NetworkDescriptor +from .layers import ( + StubDense, + StubReLU, + get_batch_norm_class, + get_conv_class, + get_dropout_class, + get_pooling_class, + is_layer, +) +from .utils import Constant + + +def to_wider_graph(graph): + ''' wider graph + ''' + weighted_layer_ids = graph.wide_layer_ids() + weighted_layer_ids = list( + filter( + lambda x: graph.layer_list[x].output.shape[-1], weighted_layer_ids) + ) + wider_layers = sample(weighted_layer_ids, 1) + + for layer_id in wider_layers: + layer = graph.layer_list[layer_id] + if is_layer(layer, "Conv"): + n_add = layer.filters + else: + n_add = layer.units + + graph.to_wider_model(layer_id, n_add) + return graph + + +def to_skip_connection_graph(graph): + ''' skip connection graph + ''' + # The last conv layer cannot be widen since wider operator cannot be done + # over the two sides of flatten. + weighted_layer_ids = graph.skip_connection_layer_ids() + valid_connection = [] + for skip_type in sorted( + [NetworkDescriptor.ADD_CONNECT, NetworkDescriptor.CONCAT_CONNECT]): + for index_a in range(len(weighted_layer_ids)): + for index_b in range(len(weighted_layer_ids))[index_a + 1:]: + valid_connection.append((index_a, index_b, skip_type)) + + if not valid_connection: + return graph + for index_a, index_b, skip_type in sample(valid_connection, 1): + a_id = weighted_layer_ids[index_a] + b_id = weighted_layer_ids[index_b] + if skip_type == NetworkDescriptor.ADD_CONNECT: + graph.to_add_skip_model(a_id, b_id) + else: + graph.to_concat_skip_model(a_id, b_id) + return graph + + +def create_new_layer(layer, n_dim): + ''' create new layer for the graph + ''' + + input_shape = layer.output.shape + dense_deeper_classes = [StubDense, get_dropout_class(n_dim), StubReLU] + conv_deeper_classes = [ + get_conv_class(n_dim), + get_batch_norm_class(n_dim), + StubReLU] + if is_layer(layer, "ReLU"): + conv_deeper_classes = [ + get_conv_class(n_dim), + get_batch_norm_class(n_dim)] + dense_deeper_classes = [StubDense, get_dropout_class(n_dim)] + elif is_layer(layer, "Dropout"): + dense_deeper_classes = [StubDense, StubReLU] + elif is_layer(layer, "BatchNormalization"): + conv_deeper_classes = [get_conv_class(n_dim), StubReLU] + + layer_class = None + if len(input_shape) == 1: + # It is in the dense layer part. + layer_class = sample(dense_deeper_classes, 1)[0] + else: + # It is in the conv layer part. + layer_class = sample(conv_deeper_classes, 1)[0] + + if layer_class == StubDense: + new_layer = StubDense(input_shape[0], input_shape[0]) + + elif layer_class == get_dropout_class(n_dim): + new_layer = layer_class(Constant.DENSE_DROPOUT_RATE) + + elif layer_class == get_conv_class(n_dim): + new_layer = layer_class( + input_shape[-1], input_shape[-1], sample((1, 3, 5), 1)[0], stride=1 + ) + + elif layer_class == get_batch_norm_class(n_dim): + new_layer = layer_class(input_shape[-1]) + + elif layer_class == get_pooling_class(n_dim): + new_layer = layer_class(sample((1, 3, 5), 1)[0]) + + else: + new_layer = layer_class() + + return new_layer + + +def to_deeper_graph(graph): + ''' deeper graph + ''' + + weighted_layer_ids = graph.deep_layer_ids() + if len(weighted_layer_ids) >= Constant.MAX_LAYERS: + return None + + deeper_layer_ids = sample(weighted_layer_ids, 1) + + for layer_id in deeper_layer_ids: + layer = graph.layer_list[layer_id] + new_layer = create_new_layer(layer, graph.n_dim) + graph.to_deeper_model(layer_id, new_layer) + return graph + + +def legal_graph(graph): + '''judge if a graph is legal or not. + ''' + + descriptor = graph.extract_descriptor() + skips = descriptor.skip_connections + if len(skips) != len(set(skips)): + return False + return True + + +def transform(graph): + '''core transform function for graph. + ''' + + graphs = [] + for _ in range(Constant.N_NEIGHBOURS * 2): + random_num = randrange(3) + temp_graph = None + if random_num == 0: + temp_graph = to_deeper_graph(deepcopy(graph)) + elif random_num == 1: + temp_graph = to_wider_graph(deepcopy(graph)) + elif random_num == 2: + temp_graph = to_skip_connection_graph(deepcopy(graph)) + + if temp_graph is not None and temp_graph.size() <= Constant.MAX_MODEL_SIZE: + graphs.append(temp_graph) + + if len(graphs) >= Constant.N_NEIGHBOURS: + break + + return graphs diff --git a/utils/third_party/nni_new/algorithms/hpo/networkmorphism_tuner/layer_transformer.py b/utils/third_party/nni_new/algorithms/hpo/networkmorphism_tuner/layer_transformer.py new file mode 100644 index 0000000000000000000000000000000000000000..6ffd1b20fb0e3b4ef0a6d2e54a02fb7cfa7cfc42 --- /dev/null +++ b/utils/third_party/nni_new/algorithms/hpo/networkmorphism_tuner/layer_transformer.py @@ -0,0 +1,264 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import numpy as np + +from .layers import ( + StubDense, + StubReLU, + get_batch_norm_class, + get_conv_class, + get_n_dim, +) + +NOISE_RATIO = 1e-4 + + +def deeper_conv_block(conv_layer, kernel_size, weighted=True): + '''deeper conv layer. + ''' + n_dim = get_n_dim(conv_layer) + filter_shape = (kernel_size,) * 2 + n_filters = conv_layer.filters + weight = np.zeros((n_filters, n_filters) + filter_shape) + center = tuple(map(lambda x: int((x - 1) / 2), filter_shape)) + for i in range(n_filters): + filter_weight = np.zeros((n_filters,) + filter_shape) + index = (i,) + center + filter_weight[index] = 1 + weight[i, ...] = filter_weight + bias = np.zeros(n_filters) + new_conv_layer = get_conv_class(n_dim)( + conv_layer.filters, n_filters, kernel_size=kernel_size + ) + bn = get_batch_norm_class(n_dim)(n_filters) + + if weighted: + new_conv_layer.set_weights( + (add_noise(weight, np.array([0, 1])), + add_noise(bias, np.array([0, 1]))) + ) + new_weights = [ + add_noise(np.ones(n_filters, dtype=np.float32), np.array([0, 1])), + add_noise(np.zeros(n_filters, dtype=np.float32), np.array([0, 1])), + add_noise(np.zeros(n_filters, dtype=np.float32), np.array([0, 1])), + add_noise(np.ones(n_filters, dtype=np.float32), np.array([0, 1])), + ] + bn.set_weights(new_weights) + + return [StubReLU(), new_conv_layer, bn] + + +def dense_to_deeper_block(dense_layer, weighted=True): + '''deeper dense layer. + ''' + units = dense_layer.units + weight = np.eye(units) + bias = np.zeros(units) + new_dense_layer = StubDense(units, units) + if weighted: + new_dense_layer.set_weights( + (add_noise(weight, np.array([0, 1])), + add_noise(bias, np.array([0, 1]))) + ) + return [StubReLU(), new_dense_layer] + + +def wider_pre_dense(layer, n_add, weighted=True): + '''wider previous dense layer. + ''' + if not weighted: + return StubDense(layer.input_units, layer.units + n_add) + + n_units2 = layer.units + + teacher_w, teacher_b = layer.get_weights() + rand = np.random.randint(n_units2, size=n_add) + student_w = teacher_w.copy() + student_b = teacher_b.copy() + + # target layer update (i) + for i in range(n_add): + teacher_index = rand[i] + new_weight = teacher_w[teacher_index, :] + new_weight = new_weight[np.newaxis, :] + student_w = np.concatenate( + (student_w, add_noise(new_weight, student_w)), axis=0) + student_b = np.append( + student_b, add_noise( + teacher_b[teacher_index], student_b)) + + new_pre_layer = StubDense(layer.input_units, n_units2 + n_add) + new_pre_layer.set_weights((student_w, student_b)) + + return new_pre_layer + + +def wider_pre_conv(layer, n_add_filters, weighted=True): + '''wider previous conv layer. + ''' + n_dim = get_n_dim(layer) + if not weighted: + return get_conv_class(n_dim)( + layer.input_channel, + layer.filters + n_add_filters, + kernel_size=layer.kernel_size, + ) + + n_pre_filters = layer.filters + rand = np.random.randint(n_pre_filters, size=n_add_filters) + teacher_w, teacher_b = layer.get_weights() + + student_w = teacher_w.copy() + student_b = teacher_b.copy() + # target layer update (i) + for i, _ in enumerate(rand): + teacher_index = rand[i] + new_weight = teacher_w[teacher_index, ...] + new_weight = new_weight[np.newaxis, ...] + student_w = np.concatenate((student_w, new_weight), axis=0) + student_b = np.append(student_b, teacher_b[teacher_index]) + new_pre_layer = get_conv_class(n_dim)( + layer.input_channel, n_pre_filters + n_add_filters, layer.kernel_size + ) + new_pre_layer.set_weights( + (add_noise(student_w, teacher_w), add_noise(student_b, teacher_b)) + ) + return new_pre_layer + + +def wider_next_conv(layer, start_dim, total_dim, n_add, weighted=True): + '''wider next conv layer. + ''' + n_dim = get_n_dim(layer) + if not weighted: + return get_conv_class(n_dim)(layer.input_channel + n_add, + layer.filters, + kernel_size=layer.kernel_size, + stride=layer.stride) + n_filters = layer.filters + teacher_w, teacher_b = layer.get_weights() + + new_weight_shape = list(teacher_w.shape) + new_weight_shape[1] = n_add + new_weight = np.zeros(tuple(new_weight_shape)) + + student_w = np.concatenate((teacher_w[:, :start_dim, ...].copy(), + add_noise(new_weight, teacher_w), + teacher_w[:, start_dim:total_dim, ...].copy()), axis=1) + new_layer = get_conv_class(n_dim)(layer.input_channel + n_add, + n_filters, + kernel_size=layer.kernel_size, + stride=layer.stride) + new_layer.set_weights((student_w, teacher_b)) + return new_layer + + +def wider_bn(layer, start_dim, total_dim, n_add, weighted=True): + '''wider batch norm layer. + ''' + n_dim = get_n_dim(layer) + if not weighted: + return get_batch_norm_class(n_dim)(layer.num_features + n_add) + + weights = layer.get_weights() + + new_weights = [ + add_noise(np.ones(n_add, dtype=np.float32), np.array([0, 1])), + add_noise(np.zeros(n_add, dtype=np.float32), np.array([0, 1])), + add_noise(np.zeros(n_add, dtype=np.float32), np.array([0, 1])), + add_noise(np.ones(n_add, dtype=np.float32), np.array([0, 1])), + ] + + student_w = tuple() + for weight, new_weight in zip(weights, new_weights): + temp_w = weight.copy() + temp_w = np.concatenate( + (temp_w[:start_dim], new_weight, temp_w[start_dim:total_dim]) + ) + student_w += (temp_w,) + new_layer = get_batch_norm_class(n_dim)(layer.num_features + n_add) + new_layer.set_weights(student_w) + return new_layer + + +def wider_next_dense(layer, start_dim, total_dim, n_add, weighted=True): + '''wider next dense layer. + ''' + if not weighted: + return StubDense(layer.input_units + n_add, layer.units) + teacher_w, teacher_b = layer.get_weights() + student_w = teacher_w.copy() + n_units_each_channel = int(teacher_w.shape[1] / total_dim) + + new_weight = np.zeros((teacher_w.shape[0], n_add * n_units_each_channel)) + student_w = np.concatenate( + ( + student_w[:, : start_dim * n_units_each_channel], + add_noise(new_weight, student_w), + student_w[ + :, start_dim * n_units_each_channel: total_dim * n_units_each_channel + ], + ), + axis=1, + ) + + new_layer = StubDense(layer.input_units + n_add, layer.units) + new_layer.set_weights((student_w, teacher_b)) + return new_layer + + +def add_noise(weights, other_weights): + '''add noise to the layer. + ''' + w_range = np.ptp(other_weights.flatten()) + noise_range = NOISE_RATIO * w_range + noise = np.random.uniform(-noise_range / 2.0, + noise_range / 2.0, weights.shape) + return np.add(noise, weights) + + +def init_dense_weight(layer): + '''initilize dense layer weight. + ''' + units = layer.units + weight = np.eye(units) + bias = np.zeros(units) + layer.set_weights( + (add_noise(weight, np.array([0, 1])), + add_noise(bias, np.array([0, 1]))) + ) + + +def init_conv_weight(layer): + '''initilize conv layer weight. + ''' + n_filters = layer.filters + filter_shape = (layer.kernel_size,) * get_n_dim(layer) + weight = np.zeros((n_filters, n_filters) + filter_shape) + + center = tuple(map(lambda x: int((x - 1) / 2), filter_shape)) + for i in range(n_filters): + filter_weight = np.zeros((n_filters,) + filter_shape) + index = (i,) + center + filter_weight[index] = 1 + weight[i, ...] = filter_weight + bias = np.zeros(n_filters) + + layer.set_weights( + (add_noise(weight, np.array([0, 1])), + add_noise(bias, np.array([0, 1]))) + ) + + +def init_bn_weight(layer): + '''initilize batch norm layer weight. + ''' + n_filters = layer.num_features + new_weights = [ + add_noise(np.ones(n_filters, dtype=np.float32), np.array([0, 1])), + add_noise(np.zeros(n_filters, dtype=np.float32), np.array([0, 1])), + add_noise(np.zeros(n_filters, dtype=np.float32), np.array([0, 1])), + add_noise(np.ones(n_filters, dtype=np.float32), np.array([0, 1])), + ] + layer.set_weights(new_weights) diff --git a/utils/third_party/nni_new/algorithms/hpo/networkmorphism_tuner/layers.py b/utils/third_party/nni_new/algorithms/hpo/networkmorphism_tuner/layers.py new file mode 100644 index 0000000000000000000000000000000000000000..a96c87b7801fe2687f61dfd62fdd13019ecd2ee0 --- /dev/null +++ b/utils/third_party/nni_new/algorithms/hpo/networkmorphism_tuner/layers.py @@ -0,0 +1,862 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from abc import abstractmethod +from collections.abc import Iterable + +import torch +from torch import nn +from torch.nn import functional +from .utils import Constant + + +class AvgPool(nn.Module): + """ + AvgPool Module. + """ + + def __init__(self): + super().__init__() + + @abstractmethod + def forward(self, input_tensor): + pass + + +class GlobalAvgPool1d(AvgPool): + """ + GlobalAvgPool1d Module. + """ + + def forward(self, input_tensor): + return functional.avg_pool1d(input_tensor, input_tensor.size()[2:]).view( + input_tensor.size()[:2] + ) + + +class GlobalAvgPool2d(AvgPool): + """ + GlobalAvgPool2d Module. + """ + + def forward(self, input_tensor): + return functional.avg_pool2d(input_tensor, input_tensor.size()[2:]).view( + input_tensor.size()[:2] + ) + + +class GlobalAvgPool3d(AvgPool): + """ + GlobalAvgPool3d Module. + """ + + def forward(self, input_tensor): + return functional.avg_pool3d(input_tensor, input_tensor.size()[2:]).view( + input_tensor.size()[:2] + ) + + +class StubLayer: + """ + StubLayer Module. Base Module. + """ + + def __init__(self, input_node=None, output_node=None): + self.input = input_node + self.output = output_node + self.weights = None + + def build(self, shape): + """ + build shape. + """ + + def set_weights(self, weights): + """ + set weights. + """ + self.weights = weights + + def import_weights(self, torch_layer): + """ + import weights. + """ + + def import_weights_keras(self, keras_layer): + """ + import weights from keras layer. + """ + + def export_weights(self, torch_layer): + """ + export weights. + """ + + def export_weights_keras(self, keras_layer): + """ + export weights to keras layer. + """ + + def get_weights(self): + """ + get weights. + """ + return self.weights + + def size(self): + """ + size(). + """ + return 0 + + @property + def output_shape(self): + """ + output shape. + """ + return self.input.shape + + def to_real_layer(self): + """ + to real layer. + """ + + def __str__(self): + """ + str() function to print. + """ + return type(self).__name__[4:] + + +class StubWeightBiasLayer(StubLayer): + """ + StubWeightBiasLayer Module to set the bias. + """ + + def import_weights(self, torch_layer): + self.set_weights( + (torch_layer.weight.data.cpu().numpy(), + torch_layer.bias.data.cpu().numpy()) + ) + + def import_weights_keras(self, keras_layer): + self.set_weights(keras_layer.get_weights()) + + def export_weights(self, torch_layer): + torch_layer.weight.data = torch.Tensor(self.weights[0]) + torch_layer.bias.data = torch.Tensor(self.weights[1]) + + def export_weights_keras(self, keras_layer): + keras_layer.set_weights(self.weights) + + +class StubBatchNormalization(StubWeightBiasLayer): + """ + StubBatchNormalization Module. Batch Norm. + """ + + def __init__(self, num_features, input_node=None, output_node=None): + super().__init__(input_node, output_node) + self.num_features = num_features + + def import_weights(self, torch_layer): + self.set_weights( + ( + torch_layer.weight.data.cpu().numpy(), + torch_layer.bias.data.cpu().numpy(), + torch_layer.running_mean.cpu().numpy(), + torch_layer.running_var.cpu().numpy(), + ) + ) + + def export_weights(self, torch_layer): + torch_layer.weight.data = torch.Tensor(self.weights[0]) + torch_layer.bias.data = torch.Tensor(self.weights[1]) + torch_layer.running_mean = torch.Tensor(self.weights[2]) + torch_layer.running_var = torch.Tensor(self.weights[3]) + + def size(self): + return self.num_features * 4 + + @abstractmethod + def to_real_layer(self): + pass + + +class StubBatchNormalization1d(StubBatchNormalization): + """ + StubBatchNormalization1d Module. + """ + + def to_real_layer(self): + return torch.nn.BatchNorm1d(self.num_features) + + +class StubBatchNormalization2d(StubBatchNormalization): + """ + StubBatchNormalization2d Module. + """ + + def to_real_layer(self): + return torch.nn.BatchNorm2d(self.num_features) + + +class StubBatchNormalization3d(StubBatchNormalization): + """ + StubBatchNormalization3d Module. + """ + + def to_real_layer(self): + return torch.nn.BatchNorm3d(self.num_features) + + +class StubDense(StubWeightBiasLayer): + """ + StubDense Module. Linear. + """ + + def __init__(self, input_units, units, input_node=None, output_node=None): + super().__init__(input_node, output_node) + self.input_units = input_units + self.units = units + + @property + def output_shape(self): + return (self.units,) + + def import_weights_keras(self, keras_layer): + self.set_weights( + (keras_layer.get_weights()[0].T, + keras_layer.get_weights()[1])) + + def export_weights_keras(self, keras_layer): + keras_layer.set_weights((self.weights[0].T, self.weights[1])) + + def size(self): + return self.input_units * self.units + self.units + + def to_real_layer(self): + return torch.nn.Linear(self.input_units, self.units) + + +class StubConv(StubWeightBiasLayer): + """ + StubConv Module. Conv. + """ + + def __init__(self, input_channel, filters, kernel_size, + stride=1, input_node=None, output_node=None): + super().__init__(input_node, output_node) + self.input_channel = input_channel + self.filters = filters + self.kernel_size = kernel_size + self.stride = stride + self.padding = int(self.kernel_size / 2) + + @property + def output_shape(self): + ret = list(self.input.shape[:-1]) + for index, dim in enumerate(ret): + ret[index] = ( + int((dim + 2 * self.padding - self.kernel_size) / self.stride) + 1 + ) + ret = ret + [self.filters] + return tuple(ret) + + def import_weights_keras(self, keras_layer): + self.set_weights( + (keras_layer.get_weights()[0].T, + keras_layer.get_weights()[1])) + + def export_weights_keras(self, keras_layer): + keras_layer.set_weights((self.weights[0].T, self.weights[1])) + + def size(self): + return (self.input_channel * self.kernel_size * + self.kernel_size + 1) * self.filters + + @abstractmethod + def to_real_layer(self): + pass + + def __str__(self): + return ( + super().__str__() + + "(" + + ", ".join( + str(item) + for item in [ + self.input_channel, + self.filters, + self.kernel_size, + self.stride, + ] + ) + + ")" + ) + + +class StubConv1d(StubConv): + """ + StubConv1d Module. + """ + + def to_real_layer(self): + return torch.nn.Conv1d( + self.input_channel, + self.filters, + self.kernel_size, + stride=self.stride, + padding=self.padding, + ) + + +class StubConv2d(StubConv): + """ + StubConv2d Module. + """ + + def to_real_layer(self): + return torch.nn.Conv2d( + self.input_channel, + self.filters, + self.kernel_size, + stride=self.stride, + padding=self.padding, + ) + + +class StubConv3d(StubConv): + """ + StubConv3d Module. + """ + + def to_real_layer(self): + return torch.nn.Conv3d( + self.input_channel, + self.filters, + self.kernel_size, + stride=self.stride, + padding=self.padding, + ) + + +class StubAggregateLayer(StubLayer): + """ + StubAggregateLayer Module. + """ + + def __init__(self, input_nodes=None, output_node=None): + if input_nodes is None: + input_nodes = [] + super().__init__(input_nodes, output_node) + + +class StubConcatenate(StubAggregateLayer): + """StubConcatenate Module. + """ + @property + def output_shape(self): + ret = 0 + for current_input in self.input: + ret += current_input.shape[-1] + ret = self.input[0].shape[:-1] + (ret,) + return ret + + def to_real_layer(self): + return TorchConcatenate() + + +class StubAdd(StubAggregateLayer): + """ + StubAdd Module. + """ + @property + def output_shape(self): + return self.input[0].shape + + def to_real_layer(self): + return TorchAdd() + + +class StubFlatten(StubLayer): + """ + StubFlatten Module. + """ + @property + def output_shape(self): + ret = 1 + for dim in self.input.shape: + ret *= dim + return (ret,) + + def to_real_layer(self): + return TorchFlatten() + + +class StubReLU(StubLayer): + """ + StubReLU Module. + """ + + def to_real_layer(self): + return torch.nn.ReLU() + + +class StubSoftmax(StubLayer): + """ + StubSoftmax Module. + """ + + def to_real_layer(self): + return torch.nn.LogSoftmax(dim=1) + + +class StubDropout(StubLayer): + """ + StubDropout Module. + """ + + def __init__(self, rate, input_node=None, output_node=None): + super().__init__(input_node, output_node) + self.rate = rate + + @abstractmethod + def to_real_layer(self): + pass + + +class StubDropout1d(StubDropout): + """ + StubDropout1d Module. + """ + + def to_real_layer(self): + return torch.nn.Dropout(self.rate) + + +class StubDropout2d(StubDropout): + """ + StubDropout2d Module. + """ + + def to_real_layer(self): + return torch.nn.Dropout2d(self.rate) + + +class StubDropout3d(StubDropout): + """ + StubDropout3d Module. + """ + + def to_real_layer(self): + return torch.nn.Dropout3d(self.rate) + + +class StubInput(StubLayer): + """ + StubInput Module. + """ + + def __init__(self, input_node=None, output_node=None): + super().__init__(input_node, output_node) + + +class StubPooling(StubLayer): + """ + StubPooling Module. + """ + + def __init__(self, + kernel_size=None, + stride=None, + padding=0, + input_node=None, + output_node=None): + super().__init__(input_node, output_node) + self.kernel_size = ( + kernel_size if kernel_size is not None else Constant.POOLING_KERNEL_SIZE + ) + self.stride = stride if stride is not None else self.kernel_size + self.padding = padding + + @property + def output_shape(self): + ret = tuple() + for dim in self.input.shape[:-1]: + ret = ret + (max(int(dim / self.kernel_size), 1),) + ret = ret + (self.input.shape[-1],) + return ret + + @abstractmethod + def to_real_layer(self): + pass + + +class StubPooling1d(StubPooling): + """ + StubPooling1d Module. + """ + + def to_real_layer(self): + return torch.nn.MaxPool1d(self.kernel_size, stride=self.stride) + + +class StubPooling2d(StubPooling): + """ + StubPooling2d Module. + """ + + def to_real_layer(self): + return torch.nn.MaxPool2d(self.kernel_size, stride=self.stride) + + +class StubPooling3d(StubPooling): + """ + StubPooling3d Module. + """ + + def to_real_layer(self): + return torch.nn.MaxPool3d(self.kernel_size, stride=self.stride) + + +class StubGlobalPooling(StubLayer): + """ + StubGlobalPooling Module. + """ + + def __init__(self, input_node=None, output_node=None): + super().__init__(input_node, output_node) + + @property + def output_shape(self): + return (self.input.shape[-1],) + + @abstractmethod + def to_real_layer(self): + pass + + +class StubGlobalPooling1d(StubGlobalPooling): + """ + StubGlobalPooling1d Module. + """ + + def to_real_layer(self): + return GlobalAvgPool1d() + + +class StubGlobalPooling2d(StubGlobalPooling): + """ + StubGlobalPooling2d Module. + """ + + def to_real_layer(self): + return GlobalAvgPool2d() + + +class StubGlobalPooling3d(StubGlobalPooling): + """ + StubGlobalPooling3d Module. + """ + + def to_real_layer(self): + return GlobalAvgPool3d() + + +class TorchConcatenate(nn.Module): + """ + TorchConcatenate Module. + """ + + def forward(self, input_list): + return torch.cat(input_list, dim=1) + + +class TorchAdd(nn.Module): + """ + TorchAdd Module. + """ + + def forward(self, input_list): + return input_list[0] + input_list[1] + + +class TorchFlatten(nn.Module): + """ + TorchFlatten Module. + """ + + def forward(self, input_tensor): + return input_tensor.view(input_tensor.size(0), -1) + + +def keras_dropout(layer, rate): + """ + Keras dropout layer. + """ + + from keras import layers + + input_dim = len(layer.input.shape) + if input_dim == 2: + return layers.SpatialDropout1D(rate) + elif input_dim == 3: + return layers.SpatialDropout2D(rate) + elif input_dim == 4: + return layers.SpatialDropout3D(rate) + else: + return layers.Dropout(rate) + + +def to_real_keras_layer(layer): + """ + Real keras layer. + """ + from keras import layers + + if is_layer(layer, "Dense"): + return layers.Dense(layer.units, input_shape=(layer.input_units,)) + if is_layer(layer, "Conv"): + return layers.Conv2D( + layer.filters, + layer.kernel_size, + input_shape=layer.input.shape, + padding="same", + ) # padding + if is_layer(layer, "Pooling"): + return layers.MaxPool2D(2) + if is_layer(layer, "BatchNormalization"): + return layers.BatchNormalization(input_shape=layer.input.shape) + if is_layer(layer, "Concatenate"): + return layers.Concatenate() + if is_layer(layer, "Add"): + return layers.Add() + if is_layer(layer, "Dropout"): + return keras_dropout(layer, layer.rate) + if is_layer(layer, "ReLU"): + return layers.Activation("relu") + if is_layer(layer, "Softmax"): + return layers.Activation("softmax") + if is_layer(layer, "Flatten"): + return layers.Flatten() + if is_layer(layer, "GlobalAveragePooling"): + return layers.GlobalAveragePooling2D() + return None # note: this is not written by original author, feel free to modify if you think it's incorrect + + +def is_layer(layer, layer_type): + """ + Judge the layer type. + + Returns + ------- + bool + boolean -- True or False + """ + + if layer_type == "Input": + return isinstance(layer, StubInput) + elif layer_type == "Conv": + return isinstance(layer, StubConv) + elif layer_type == "Dense": + return isinstance(layer, (StubDense,)) + elif layer_type == "BatchNormalization": + return isinstance(layer, (StubBatchNormalization,)) + elif layer_type == "Concatenate": + return isinstance(layer, (StubConcatenate,)) + elif layer_type == "Add": + return isinstance(layer, (StubAdd,)) + elif layer_type == "Pooling": + return isinstance(layer, StubPooling) + elif layer_type == "Dropout": + return isinstance(layer, (StubDropout,)) + elif layer_type == "Softmax": + return isinstance(layer, (StubSoftmax,)) + elif layer_type == "ReLU": + return isinstance(layer, (StubReLU,)) + elif layer_type == "Flatten": + return isinstance(layer, (StubFlatten,)) + elif layer_type == "GlobalAveragePooling": + return isinstance(layer, StubGlobalPooling) + return None # note: this is not written by original author, feel free to modify if you think it's incorrect + + +def layer_description_extractor(layer, node_to_id): + """ + Get layer description. + """ + + layer_input = layer.input + layer_output = layer.output + if layer_input is not None: + if isinstance(layer_input, Iterable): + layer_input = list(map(lambda x: node_to_id[x], layer_input)) + else: + layer_input = node_to_id[layer_input] + + if layer_output is not None: + layer_output = node_to_id[layer_output] + + if isinstance(layer, StubConv): + return ( + type(layer).__name__, + layer_input, + layer_output, + layer.input_channel, + layer.filters, + layer.kernel_size, + layer.stride, + layer.padding, + ) + elif isinstance(layer, (StubDense,)): + return [ + type(layer).__name__, + layer_input, + layer_output, + layer.input_units, + layer.units, + ] + elif isinstance(layer, (StubBatchNormalization,)): + return (type(layer).__name__, layer_input, + layer_output, layer.num_features) + elif isinstance(layer, (StubDropout,)): + return (type(layer).__name__, layer_input, layer_output, layer.rate) + elif isinstance(layer, StubPooling): + return ( + type(layer).__name__, + layer_input, + layer_output, + layer.kernel_size, + layer.stride, + layer.padding, + ) + else: + return (type(layer).__name__, layer_input, layer_output) + + +def layer_description_builder(layer_information, id_to_node): + """build layer from description. + """ + layer_type = layer_information[0] + + layer_input_ids = layer_information[1] + if isinstance(layer_input_ids, Iterable): + layer_input = list(map(lambda x: id_to_node[x], layer_input_ids)) + else: + layer_input = id_to_node[layer_input_ids] + layer_output = id_to_node[layer_information[2]] + if layer_type.startswith("StubConv"): + input_channel = layer_information[3] + filters = layer_information[4] + kernel_size = layer_information[5] + stride = layer_information[6] + return globals()[layer_type]( + input_channel, filters, kernel_size, stride, layer_input, layer_output + ) + elif layer_type.startswith("StubDense"): + input_units = layer_information[3] + units = layer_information[4] + return globals()[layer_type](input_units, units, layer_input, layer_output) + elif layer_type.startswith("StubBatchNormalization"): + num_features = layer_information[3] + return globals()[layer_type](num_features, layer_input, layer_output) + elif layer_type.startswith("StubDropout"): + rate = layer_information[3] + return globals()[layer_type](rate, layer_input, layer_output) + elif layer_type.startswith("StubPooling"): + kernel_size = layer_information[3] + stride = layer_information[4] + padding = layer_information[5] + return globals()[layer_type](kernel_size, stride, padding, layer_input, layer_output) + else: + return globals()[layer_type](layer_input, layer_output) + + +def layer_width(layer): + """ + Get layer width. + """ + + if is_layer(layer, "Dense"): + return layer.units + if is_layer(layer, "Conv"): + return layer.filters + raise TypeError("The layer should be either Dense or Conv layer.") + + +def set_torch_weight_to_stub(torch_layer, stub_layer): + stub_layer.import_weights(torch_layer) + + +def set_keras_weight_to_stub(keras_layer, stub_layer): + stub_layer.import_weights_keras(keras_layer) + + +def set_stub_weight_to_torch(stub_layer, torch_layer): + stub_layer.export_weights(torch_layer) + + +def set_stub_weight_to_keras(stub_layer, keras_layer): + stub_layer.export_weights_keras(keras_layer) + + +def get_conv_class(n_dim): + conv_class_list = [StubConv1d, StubConv2d, StubConv3d] + return conv_class_list[n_dim - 1] + + +def get_dropout_class(n_dim): + dropout_class_list = [StubDropout1d, StubDropout2d, StubDropout3d] + return dropout_class_list[n_dim - 1] + + +def get_global_avg_pooling_class(n_dim): + global_avg_pooling_class_list = [ + StubGlobalPooling1d, + StubGlobalPooling2d, + StubGlobalPooling3d, + ] + return global_avg_pooling_class_list[n_dim - 1] + + +def get_pooling_class(n_dim): + pooling_class_list = [StubPooling1d, StubPooling2d, StubPooling3d] + return pooling_class_list[n_dim - 1] + + +def get_batch_norm_class(n_dim): + batch_norm_class_list = [ + StubBatchNormalization1d, + StubBatchNormalization2d, + StubBatchNormalization3d, + ] + return batch_norm_class_list[n_dim - 1] + + +def get_n_dim(layer): + if isinstance(layer, ( + StubConv1d, + StubDropout1d, + StubGlobalPooling1d, + StubPooling1d, + StubBatchNormalization1d, + )): + return 1 + if isinstance(layer, ( + StubConv2d, + StubDropout2d, + StubGlobalPooling2d, + StubPooling2d, + StubBatchNormalization2d, + )): + return 2 + if isinstance(layer, ( + StubConv3d, + StubDropout3d, + StubGlobalPooling3d, + StubPooling3d, + StubBatchNormalization3d, + )): + return 3 + return -1 diff --git a/utils/third_party/nni_new/algorithms/hpo/networkmorphism_tuner/networkmorphism_tuner.py b/utils/third_party/nni_new/algorithms/hpo/networkmorphism_tuner/networkmorphism_tuner.py new file mode 100644 index 0000000000000000000000000000000000000000..385028506db5a5f5b3b2ecf951542be6d7cbde26 --- /dev/null +++ b/utils/third_party/nni_new/algorithms/hpo/networkmorphism_tuner/networkmorphism_tuner.py @@ -0,0 +1,328 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +networkmorphsim_tuner.py +""" + +import logging +import os +from schema import Optional, Schema +from nni.tuner import Tuner +from nni.utils import OptimizeMode, extract_scalar_reward +from .bayesian import BayesianOptimizer +from .nn import CnnGenerator, MlpGenerator +from .utils import Constant +from .graph import graph_to_json, json_to_graph +from nni import ClassArgsValidator + +logger = logging.getLogger("NetworkMorphism_AutoML") + +class NetworkMorphismClassArgsValidator(ClassArgsValidator): + def validate_class_args(self, **kwargs): + Schema({ + Optional('optimize_mode'): self.choices('optimize_mode', 'maximize', 'minimize'), + Optional('task'): self.choices('task', 'cv', 'nlp', 'common'), + Optional('input_width'): int, + Optional('input_channel'): int, + Optional('n_output_node'): int + }).validate(kwargs) + +class NetworkMorphismTuner(Tuner): + """ + NetworkMorphismTuner is a tuner which using network morphism techniques. + + Attributes + ---------- + n_classes : int + The class number or output node number (default: ``10``) + input_shape : tuple + A tuple including: (input_width, input_width, input_channel) + t_min : float + The minimum temperature for simulated annealing. (default: ``Constant.T_MIN``) + beta : float + The beta in acquisition function. (default: ``Constant.BETA``) + algorithm_name : str + algorithm name used in the network morphism (default: ``"Bayesian"``) + optimize_mode : str + optimize mode "minimize" or "maximize" (default: ``"minimize"``) + verbose : bool + verbose to print the log (default: ``True``) + bo : BayesianOptimizer + The optimizer used in networkmorphsim tuner. + max_model_size : int + max model size to the graph (default: ``Constant.MAX_MODEL_SIZE``) + default_model_len : int + default model length (default: ``Constant.MODEL_LEN``) + default_model_width : int + default model width (default: ``Constant.MODEL_WIDTH``) + search_space : dict + """ + + def __init__( + self, + task="cv", + input_width=32, + input_channel=3, + n_output_node=10, + algorithm_name="Bayesian", + optimize_mode="maximize", + path="model_path", + verbose=True, + beta=Constant.BETA, + t_min=Constant.T_MIN, + max_model_size=Constant.MAX_MODEL_SIZE, + default_model_len=Constant.MODEL_LEN, + default_model_width=Constant.MODEL_WIDTH, + ): + """ + initilizer of the NetworkMorphismTuner. + """ + + if not os.path.exists(path): + os.makedirs(path) + self.path = os.path.join(os.getcwd(), path) + if task == "cv": + self.generators = [CnnGenerator] + elif task == "common": + self.generators = [MlpGenerator] + else: + raise NotImplementedError( + '{} task not supported in List ["cv","common"]') + + self.n_classes = n_output_node + self.input_shape = (input_width, input_width, input_channel) + + self.t_min = t_min + self.beta = beta + self.algorithm_name = algorithm_name + self.optimize_mode = OptimizeMode(optimize_mode) + self.json = None + self.total_data = {} + self.verbose = verbose + self.model_count = 0 + + self.bo = BayesianOptimizer( + self, self.t_min, self.optimize_mode, self.beta) + self.training_queue = [] + self.descriptors = [] + self.history = [] + + self.max_model_size = max_model_size + self.default_model_len = default_model_len + self.default_model_width = default_model_width + + self.search_space = dict() + + + def update_search_space(self, search_space): + """ + Update search space definition in tuner by search_space in neural architecture. + """ + self.search_space = search_space + + def generate_parameters(self, parameter_id, **kwargs): + """ + Returns a set of trial neural architecture, as a serializable object. + + Parameters + ---------- + parameter_id : int + """ + if not self.history: + self.init_search() + + new_father_id = None + generated_graph = None + if not self.training_queue: + new_father_id, generated_graph = self.generate() + new_model_id = self.model_count + self.model_count += 1 + self.training_queue.append( + (generated_graph, new_father_id, new_model_id)) + self.descriptors.append(generated_graph.extract_descriptor()) + + graph, father_id, model_id = self.training_queue.pop(0) + + # from graph to json + json_model_path = os.path.join(self.path, str(model_id) + ".json") + json_out = graph_to_json(graph, json_model_path) + self.total_data[parameter_id] = (json_out, father_id, model_id) + + return json_out + + def receive_trial_result(self, parameter_id, parameters, value, **kwargs): + """ + Record an observation of the objective function. + + Parameters + ---------- + parameter_id : int + the id of a group of paramters that generated by nni manager. + parameters : dict + A group of parameters. + value : dict/float + if value is dict, it should have "default" key. + """ + reward = extract_scalar_reward(value) + + if parameter_id not in self.total_data: + raise RuntimeError("Received parameter_id not in total_data.") + + (_, father_id, model_id) = self.total_data[parameter_id] + + graph = self.bo.searcher.load_model_by_id(model_id) + + # to use the value and graph + self.add_model(reward, model_id) + self.update(father_id, graph, reward, model_id) + + + def init_search(self): + """ + Call the generators to generate the initial architectures for the search. + """ + if self.verbose: + logger.info("Initializing search.") + for generator in self.generators: + graph = generator(self.n_classes, self.input_shape).generate( + self.default_model_len, self.default_model_width + ) + model_id = self.model_count + self.model_count += 1 + self.training_queue.append((graph, -1, model_id)) + self.descriptors.append(graph.extract_descriptor()) + + if self.verbose: + logger.info("Initialization finished.") + + + def generate(self): + """ + Generate the next neural architecture. + + Returns + ------- + other_info : any object + Anything to be saved in the training queue together with the architecture. + generated_graph : Graph + An instance of Graph. + """ + generated_graph, new_father_id = self.bo.generate(self.descriptors) + if new_father_id is None: + new_father_id = 0 + generated_graph = self.generators[0]( + self.n_classes, self.input_shape + ).generate(self.default_model_len, self.default_model_width) + + return new_father_id, generated_graph + + def update(self, other_info, graph, metric_value, model_id): + """ + Update the controller with evaluation result of a neural architecture. + + Parameters + ---------- + other_info: any object + In our case it is the father ID in the search tree. + graph: graph.Graph + An instance of Graph. The trained neural architecture. + metric_value: float + The final evaluated metric value. + model_id: int + """ + father_id = other_info + self.bo.fit([graph.extract_descriptor()], [metric_value]) + self.bo.add_child(father_id, model_id) + + def add_model(self, metric_value, model_id): + """ + Add model to the history, x_queue and y_queue + + Parameters + ---------- + metric_value : float + graph : dict + model_id : int + + Returns + ------- + model : dict + """ + if self.verbose: + logger.info("Saving model.") + + # Update best_model text file + ret = {"model_id": model_id, "metric_value": metric_value} + self.history.append(ret) + if model_id == self.get_best_model_id(): + file = open(os.path.join(self.path, "best_model.txt"), "w") + file.write("best model: " + str(model_id)) + file.close() + return ret + + + def get_best_model_id(self): + """ + Get the best model_id from history using the metric value + """ + + if self.optimize_mode is OptimizeMode.Maximize: + return max(self.history, key=lambda x: x["metric_value"])[ + "model_id"] + return min(self.history, key=lambda x: x["metric_value"])["model_id"] + + + def load_model_by_id(self, model_id): + """ + Get the model by model_id + + Parameters + ---------- + model_id : int + model index + + Returns + ------- + load_model : graph.Graph + the model graph representation + """ + + with open(os.path.join(self.path, str(model_id) + ".json")) as fin: + json_str = fin.read().replace("\n", "") + + load_model = json_to_graph(json_str) + return load_model + + def load_best_model(self): + """ + Get the best model by model id + + Returns + ------- + load_model : graph.Graph + the model graph representation + """ + return self.load_model_by_id(self.get_best_model_id()) + + def get_metric_value_by_id(self, model_id): + """ + Get the model metric valud by its model_id + + Parameters + ---------- + model_id : int + model index + + Returns + ------- + float + the model metric + """ + for item in self.history: + if item["model_id"] == model_id: + return item["metric_value"] + return None + + def import_data(self, data): + pass diff --git a/utils/third_party/nni_new/algorithms/hpo/networkmorphism_tuner/nn.py b/utils/third_party/nni_new/algorithms/hpo/networkmorphism_tuner/nn.py new file mode 100644 index 0000000000000000000000000000000000000000..9e0072f9b39e3a68b865ba850157fe1080791983 --- /dev/null +++ b/utils/third_party/nni_new/algorithms/hpo/networkmorphism_tuner/nn.py @@ -0,0 +1,166 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from abc import abstractmethod + +from .graph import Graph +from .layers import (StubDense, StubDropout1d, + StubReLU, get_batch_norm_class, + get_conv_class, + get_dropout_class, + get_global_avg_pooling_class, + get_pooling_class) +from .utils import Constant + + +class NetworkGenerator: + """The base class for generating a network. + It can be used to generate a CNN or Multi-Layer Perceptron. + Attributes: + n_output_node: Number of output nodes in the network. + input_shape: A tuple to represent the input shape. + """ + + def __init__(self, n_output_node, input_shape): + self.n_output_node = n_output_node + self.input_shape = input_shape + + @abstractmethod + def generate(self, model_len, model_width): + pass + + +class CnnGenerator(NetworkGenerator): + """A class to generate CNN. + Attributes: + n_dim: `len(self.input_shape) - 1` + conv: A class that represents `(n_dim-1)` dimensional convolution. + dropout: A class that represents `(n_dim-1)` dimensional dropout. + global_avg_pooling: A class that represents `(n_dim-1)` dimensional Global Average Pooling. + pooling: A class that represents `(n_dim-1)` dimensional pooling. + batch_norm: A class that represents `(n_dim-1)` dimensional batch normalization. + """ + + def __init__(self, n_output_node, input_shape): + super(CnnGenerator, self).__init__(n_output_node, input_shape) + self.n_dim = len(self.input_shape) - 1 + if len(self.input_shape) > 4: + raise ValueError("The input dimension is too high.") + if len(self.input_shape) < 2: + raise ValueError("The input dimension is too low.") + self.conv = get_conv_class(self.n_dim) + self.dropout = get_dropout_class(self.n_dim) + self.global_avg_pooling = get_global_avg_pooling_class(self.n_dim) + self.pooling = get_pooling_class(self.n_dim) + self.batch_norm = get_batch_norm_class(self.n_dim) + + def generate(self, model_len=None, model_width=None): + """Generates a CNN. + Args: + model_len: An integer. Number of convolutional layers. + model_width: An integer. Number of filters for the convolutional layers. + Returns: + An instance of the class Graph. Represents the neural architecture graph of the generated model. + """ + + if model_len is None: + model_len = Constant.MODEL_LEN + if model_width is None: + model_width = Constant.MODEL_WIDTH + pooling_len = int(model_len / 4) + graph = Graph(self.input_shape, False) + temp_input_channel = self.input_shape[-1] + output_node_id = 0 + stride = 1 + for i in range(model_len): + output_node_id = graph.add_layer(StubReLU(), output_node_id) + output_node_id = graph.add_layer( + self.batch_norm( + graph.node_list[output_node_id].shape[-1]), output_node_id + ) + output_node_id = graph.add_layer( + self.conv( + temp_input_channel, + model_width, + kernel_size=3, + stride=stride), + output_node_id, + ) + temp_input_channel = model_width + if pooling_len == 0 or ( + (i + 1) % pooling_len == 0 and i != model_len - 1): + output_node_id = graph.add_layer( + self.pooling(), output_node_id) + + output_node_id = graph.add_layer( + self.global_avg_pooling(), output_node_id) + output_node_id = graph.add_layer( + self.dropout(Constant.CONV_DROPOUT_RATE), output_node_id + ) + output_node_id = graph.add_layer( + StubDense(graph.node_list[output_node_id].shape[0], model_width), + output_node_id, + ) + output_node_id = graph.add_layer(StubReLU(), output_node_id) + graph.add_layer( + StubDense( + model_width, + self.n_output_node), + output_node_id) + return graph + + +class MlpGenerator(NetworkGenerator): + """A class to generate Multi-Layer Perceptron. + """ + + def __init__(self, n_output_node, input_shape): + """Initialize the instance. + Args: + n_output_node: An integer. Number of output nodes in the network. + input_shape: A tuple. Input shape of the network. If it is 1D, ensure the value is appended by a comma + in the tuple. + """ + super(MlpGenerator, self).__init__(n_output_node, input_shape) + if len(self.input_shape) > 1: + raise ValueError("The input dimension is too high.") + + def generate(self, model_len=None, model_width=None): + """Generates a Multi-Layer Perceptron. + Args: + model_len: An integer. Number of hidden layers. + model_width: An integer or a list of integers of length `model_len`. If it is a list, it represents the + number of nodes in each hidden layer. If it is an integer, all hidden layers have nodes equal to this + value. + Returns: + An instance of the class Graph. Represents the neural architecture graph of the generated model. + """ + if model_len is None: + model_len = Constant.MODEL_LEN + if model_width is None: + model_width = Constant.MODEL_WIDTH + if isinstance(model_width, list) and not len(model_width) == model_len: + raise ValueError( + "The length of 'model_width' does not match 'model_len'") + elif isinstance(model_width, int): + model_width = [model_width] * model_len + + graph = Graph(self.input_shape, False) + output_node_id = 0 + n_nodes_prev_layer = self.input_shape[0] + for width in model_width: + output_node_id = graph.add_layer( + StubDense(n_nodes_prev_layer, width), output_node_id + ) + output_node_id = graph.add_layer( + StubDropout1d(Constant.MLP_DROPOUT_RATE), output_node_id + ) + output_node_id = graph.add_layer(StubReLU(), output_node_id) + n_nodes_prev_layer = width + + graph.add_layer( + StubDense( + n_nodes_prev_layer, + self.n_output_node), + output_node_id) + return graph diff --git a/utils/third_party/nni_new/algorithms/hpo/networkmorphism_tuner/utils.py b/utils/third_party/nni_new/algorithms/hpo/networkmorphism_tuner/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..0634e7f578bdeb505ce984be971beb3978c78a44 --- /dev/null +++ b/utils/third_party/nni_new/algorithms/hpo/networkmorphism_tuner/utils.py @@ -0,0 +1,23 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + + +class Constant: + '''Constant for the Tuner. + ''' + MAX_LAYERS = 100 + N_NEIGHBOURS = 8 + MAX_MODEL_SIZE = 1 << 24 + KERNEL_LAMBDA = 1.0 + BETA = 2.576 + MLP_MODEL_LEN = 3 + MLP_MODEL_WIDTH = 5 + MODEL_LEN = 3 + MODEL_WIDTH = 64 + POOLING_KERNEL_SIZE = 2 + DENSE_DROPOUT_RATE = 0.5 + CONV_DROPOUT_RATE = 0.25 + MLP_DROPOUT_RATE = 0.25 + CONV_BLOCK_DISTANCE = 2 + BATCH_SIZE = 128 + T_MIN = 0.0001 diff --git a/utils/third_party/nni_new/algorithms/hpo/pbt_tuner.py b/utils/third_party/nni_new/algorithms/hpo/pbt_tuner.py new file mode 100644 index 0000000000000000000000000000000000000000..507c519a2a880ca99833433a6f61278e308a469f --- /dev/null +++ b/utils/third_party/nni_new/algorithms/hpo/pbt_tuner.py @@ -0,0 +1,456 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import copy +import logging +import os +import random +import numpy as np +from schema import Schema, Optional + +import nni +from nni import ClassArgsValidator +import nni.parameter_expressions +from nni.tuner import Tuner +from nni.utils import OptimizeMode, extract_scalar_reward, split_index, json2parameter, json2space + + +logger = logging.getLogger('pbt_tuner_AutoML') + + +def perturbation(hyperparameter_type, value, resample_probablity, uv, ub, lv, lb, random_state): + """ + Perturbation for hyperparameters + + Parameters + ---------- + hyperparameter_type : str + type of hyperparameter + value : list + parameters for sampling hyperparameter + resample_probability : float + probability for resampling + uv : float/int + upper value after perturbation + ub : float/int + upper bound + lv : float/int + lower value after perturbation + lb : float/int + lower bound + random_state : RandomState + random state + """ + if random.random() < resample_probablity: + if hyperparameter_type == "choice": + return value.index(nni.parameter_expressions.choice(value, random_state)) + else: + return getattr(nni.parameter_expressions, hyperparameter_type)(*(value + [random_state])) + else: + if random.random() > 0.5: + return min(uv, ub) + else: + return max(lv, lb) + + +def exploit_and_explore(bot_trial_info, top_trial_info, factor, resample_probability, epoch, search_space): + """ + Replace checkpoint of bot_trial with top, and perturb hyperparameters + + Parameters + ---------- + bot_trial_info : TrialInfo + bottom model whose parameters should be replaced + top_trial_info : TrialInfo + better model + factor : float + factor for perturbation + resample_probability : float + probability for resampling + epoch : int + step of PBTTuner + search_space : dict + search_space to keep perturbed hyperparameters in range + """ + bot_checkpoint_dir = bot_trial_info.checkpoint_dir + top_hyper_parameters = top_trial_info.hyper_parameters + hyper_parameters = copy.deepcopy(top_hyper_parameters) + random_state = np.random.RandomState() + hyper_parameters['load_checkpoint_dir'] = hyper_parameters['save_checkpoint_dir'] + hyper_parameters['save_checkpoint_dir'] = os.path.join(bot_checkpoint_dir, str(epoch)) + for key in hyper_parameters.keys(): + hyper_parameter = hyper_parameters[key] + if key == 'load_checkpoint_dir' or key == 'save_checkpoint_dir': + continue + elif search_space[key]["_type"] == "choice": + choices = search_space[key]["_value"] + ub, uv = len(choices) - 1, choices.index(hyper_parameter) + 1 + lb, lv = 0, choices.index(hyper_parameter) - 1 + elif search_space[key]["_type"] == "randint": + lb, ub = search_space[key]["_value"][:2] + ub -= 1 + uv = hyper_parameter + 1 + lv = hyper_parameter - 1 + elif search_space[key]["_type"] == "uniform": + lb, ub = search_space[key]["_value"][:2] + perturb = (ub - lb) * factor + uv = hyper_parameter + perturb + lv = hyper_parameter - perturb + elif search_space[key]["_type"] == "quniform": + lb, ub, q = search_space[key]["_value"][:3] + multi = round(hyper_parameter / q) + uv = (multi + 1) * q + lv = (multi - 1) * q + elif search_space[key]["_type"] == "loguniform": + lb, ub = search_space[key]["_value"][:2] + perturb = (np.log(ub) - np.log(lb)) * factor + uv = np.exp(min(np.log(hyper_parameter) + perturb, np.log(ub))) + lv = np.exp(max(np.log(hyper_parameter) - perturb, np.log(lb))) + elif search_space[key]["_type"] == "qloguniform": + lb, ub, q = search_space[key]["_value"][:3] + multi = round(hyper_parameter / q) + uv = (multi + 1) * q + lv = (multi - 1) * q + elif search_space[key]["_type"] == "normal": + sigma = search_space[key]["_value"][1] + perturb = sigma * factor + uv = ub = hyper_parameter + perturb + lv = lb = hyper_parameter - perturb + elif search_space[key]["_type"] == "qnormal": + q = search_space[key]["_value"][2] + uv = ub = hyper_parameter + q + lv = lb = hyper_parameter - q + elif search_space[key]["_type"] == "lognormal": + sigma = search_space[key]["_value"][1] + perturb = sigma * factor + uv = ub = np.exp(np.log(hyper_parameter) + perturb) + lv = lb = np.exp(np.log(hyper_parameter) - perturb) + elif search_space[key]["_type"] == "qlognormal": + q = search_space[key]["_value"][2] + uv = ub = hyper_parameter + q + lv, lb = hyper_parameter - q, 1E-10 + else: + logger.warning("Illegal type to perturb: %s", search_space[key]["_type"]) + continue + + if search_space[key]["_type"] == "choice": + idx = perturbation(search_space[key]["_type"], search_space[key]["_value"], + resample_probability, uv, ub, lv, lb, random_state) + hyper_parameters[key] = choices[idx] + else: + hyper_parameters[key] = perturbation(search_space[key]["_type"], search_space[key]["_value"], + resample_probability, uv, ub, lv, lb, random_state) + bot_trial_info.hyper_parameters = hyper_parameters + bot_trial_info.clean_id() + + +class TrialInfo: + """ + Information of each trial, refresh for each epoch + + """ + + def __init__(self, checkpoint_dir=None, hyper_parameters=None, parameter_id=None, score=None): + self.checkpoint_dir = checkpoint_dir + self.hyper_parameters = hyper_parameters + self.parameter_id = parameter_id + self.score = score + + def clean_id(self): + self.parameter_id = None + +class PBTClassArgsValidator(ClassArgsValidator): + def validate_class_args(self, **kwargs): + Schema({ + 'optimize_mode': self.choices('optimize_mode', 'maximize', 'minimize'), + Optional('all_checkpoint_dir'): str, + Optional('population_size'): self.range('population_size', int, 0, 99999), + Optional('factors'): float, + Optional('fraction'): float, + }).validate(kwargs) + +class PBTTuner(Tuner): + def __init__(self, optimize_mode="maximize", all_checkpoint_dir=None, population_size=10, factor=0.2, + resample_probability=0.25, fraction=0.2): + """ + Initialization + + Parameters + ---------- + optimize_mode : str + maximize or minimize + all_checkpoint_dir : str + directory to store training model checkpoint + population_size : int + number of trials for each epoch + factor : float + factor for perturbation + resample_probability : float + probability for resampling + fraction : float + fraction for selecting bottom and top trials + """ + self.optimize_mode = OptimizeMode(optimize_mode) + if all_checkpoint_dir is None: + all_checkpoint_dir = os.getenv('NNI_CHECKPOINT_DIRECTORY') + logger.info("Checkpoint dir is set to %s by default.", all_checkpoint_dir) + self.all_checkpoint_dir = all_checkpoint_dir + self.population_size = population_size + self.factor = factor + self.resample_probability = resample_probability + self.fraction = fraction + # defined in trial code + #self.perturbation_interval = perturbation_interval + + self.population = None + self.pos = -1 + self.param_ids = [] + self.running = {} + self.finished = [] + self.credit = 0 + self.finished_trials = 0 + self.epoch = 0 + + self.searchspace_json = None + self.space = None + + self.send_trial_callback = None + + logger.info('PBT tuner initialization') + + def update_search_space(self, search_space): + """ + Get search space + + Parameters + ---------- + search_space : dict + Search space + """ + logger.info('Update search space %s', search_space) + self.searchspace_json = search_space + self.space = json2space(self.searchspace_json) + + self.random_state = np.random.RandomState() + self.population = [] + is_rand = dict() + + for item in self.space: + is_rand[item] = True + + for i in range(self.population_size): + hyper_parameters = json2parameter( + self.searchspace_json, is_rand, self.random_state) + hyper_parameters = split_index(hyper_parameters) + checkpoint_dir = os.path.join(self.all_checkpoint_dir, str(i)) + hyper_parameters['load_checkpoint_dir'] = os.path.join(checkpoint_dir, str(self.epoch)) + hyper_parameters['save_checkpoint_dir'] = os.path.join(checkpoint_dir, str(self.epoch)) + self.population.append(TrialInfo(checkpoint_dir=checkpoint_dir, hyper_parameters=hyper_parameters)) + + def generate_multiple_parameters(self, parameter_id_list, **kwargs): + """ + Returns multiple sets of trial (hyper-)parameters, as iterable of serializable objects. + + Parameters + ---------- + parameter_id_list : list of int + Unique identifiers for each set of requested hyper-parameters. + These will later be used in :meth:`receive_trial_result`. + **kwargs + Used for send_trial_callback. + + Returns + ------- + list + A list of newly generated configurations + """ + result = [] + self.send_trial_callback = kwargs['st_callback'] + for parameter_id in parameter_id_list: + had_exception = False + try: + logger.debug("generating param for %s", parameter_id) + res = self.generate_parameters(parameter_id, **kwargs) + except nni.NoMoreTrialError: + had_exception = True + if not had_exception: + result.append(res) + return result + + def generate_parameters(self, parameter_id, **kwargs): + """ + Generate parameters, if no trial configration for now, self.credit plus 1 to send the config later + + Parameters + ---------- + parameter_id : int + Unique identifier for requested hyper-parameters. + This will later be used in :meth:`receive_trial_result`. + **kwargs + Not used + + Returns + ------- + dict + One newly generated configuration + + """ + if self.pos == self.population_size - 1: + logger.debug('Credit added by one in parameters request') + self.credit += 1 + self.param_ids.append(parameter_id) + raise nni.NoMoreTrialError('No more parameters now.') + self.pos += 1 + trial_info = self.population[self.pos] + trial_info.parameter_id = parameter_id + self.running[parameter_id] = trial_info + logger.info('Generate parameter : %s', trial_info.hyper_parameters) + return trial_info.hyper_parameters + + def _proceed_next_epoch(self): + """ + """ + logger.info('Proceeding to next epoch') + self.epoch += 1 + self.population = [] + self.pos = -1 + self.running = {} + #exploit and explore + reverse = True if self.optimize_mode == OptimizeMode.Maximize else False + self.finished = sorted(self.finished, key=lambda x: x.score, reverse=reverse) + cutoff = int(np.ceil(self.fraction * len(self.finished))) + tops = self.finished[:cutoff] + bottoms = self.finished[self.finished_trials - cutoff:] + for bottom in bottoms: + top = np.random.choice(tops) + exploit_and_explore(bottom, top, self.factor, self.resample_probability, self.epoch, self.searchspace_json) + for trial in self.finished: + if trial not in bottoms: + trial.clean_id() + trial.hyper_parameters['load_checkpoint_dir'] = trial.hyper_parameters['save_checkpoint_dir'] + trial.hyper_parameters['save_checkpoint_dir'] = os.path.join(trial.checkpoint_dir, str(self.epoch)) + self.finished_trials = 0 + for _ in range(self.population_size): + trial_info = self.finished.pop() + self.population.append(trial_info) + while self.credit > 0 and self.pos + 1 < len(self.population): + self.credit -= 1 + self.pos += 1 + parameter_id = self.param_ids.pop() + trial_info = self.population[self.pos] + trial_info.parameter_id = parameter_id + self.running[parameter_id] = trial_info + self.send_trial_callback(parameter_id, trial_info.hyper_parameters) + + def receive_trial_result(self, parameter_id, parameters, value, **kwargs): + """ + Receive trial's result. if the number of finished trials equals ``self.population_size``, start the next epoch to + train the model. + + Parameters + ---------- + parameter_id : int + Unique identifier of used hyper-parameters, same with :meth:`generate_parameters`. + parameters : dict + Hyper-parameters generated by :meth:`generate_parameters`. + value : dict + Result from trial (the return value of :func:`nni.report_final_result`). + """ + logger.info('Get one trial result, id = %d, value = %s', parameter_id, value) + value = extract_scalar_reward(value) + trial_info = self.running.pop(parameter_id, None) + trial_info.score = value + self.finished.append(trial_info) + self.finished_trials += 1 + if self.finished_trials == self.population_size: + self._proceed_next_epoch() + + def trial_end(self, parameter_id, success, **kwargs): + """ + Deal with trial failure + + Parameters + ---------- + parameter_id : int + Unique identifier for hyper-parameters used by this trial. + success : bool + True if the trial successfully completed; False if failed or terminated. + **kwargs + Unstable parameters which should be ignored by normal users. + """ + if success: + return + if self.optimize_mode == OptimizeMode.Minimize: + value = float('inf') + else: + value = float('-inf') + trial_info = self.running.pop(parameter_id, None) + trial_info.score = value + self.finished.append(trial_info) + self.finished_trials += 1 + if self.finished_trials == self.population_size: + self._proceed_next_epoch() + + def import_data(self, data): + """ + Parameters + ---------- + data : json obj + imported data records + + Returns + ------- + int + the start epoch number after data imported, only used for unittest + """ + if self.running: + logger.warning("Do not support importing data in the middle of experiment") + return + # the following is for experiment resume + _completed_num = 0 + epoch_data_dict = {} + for trial_info in data: + logger.info("Process data record %s / %s", _completed_num, len(data)) + _completed_num += 1 + # simply validate data format + _params = trial_info["parameter"] + _value = trial_info['value'] + # assign fake value for failed trials + if not _value: + logger.info("Useless trial data, value is %s, skip this trial data.", _value) + _value = float('inf') if self.optimize_mode == OptimizeMode.Minimize else float('-inf') + _value = extract_scalar_reward(_value) + if 'save_checkpoint_dir' not in _params: + logger.warning("Invalid data record: save_checkpoint_dir is missing, abandon data import.") + return + epoch_num = int(os.path.basename(_params['save_checkpoint_dir'])) + if epoch_num not in epoch_data_dict: + epoch_data_dict[epoch_num] = [] + epoch_data_dict[epoch_num].append((_params, _value)) + if not epoch_data_dict: + logger.warning("No valid epochs, abandon data import.") + return + # figure out start epoch for resume + max_epoch_num = max(epoch_data_dict, key=int) + if len(epoch_data_dict[max_epoch_num]) < self.population_size: + max_epoch_num -= 1 + # If there is no a single complete round, no data to import, start from scratch + if max_epoch_num < 0: + logger.warning("No completed epoch, abandon data import.") + return + assert len(epoch_data_dict[max_epoch_num]) == self.population_size + # check existence of trial save checkpoint dir + for params, _ in epoch_data_dict[max_epoch_num]: + if not os.path.isdir(params['save_checkpoint_dir']): + logger.warning("save_checkpoint_dir %s does not exist, data will not be resumed", params['save_checkpoint_dir']) + return + # resume data + self.epoch = max_epoch_num + self.finished_trials = self.population_size + for params, value in epoch_data_dict[max_epoch_num]: + checkpoint_dir = os.path.dirname(params['save_checkpoint_dir']) + self.finished.append(TrialInfo(checkpoint_dir=checkpoint_dir, hyper_parameters=params, score=value)) + self._proceed_next_epoch() + logger.info("Successfully import data to PBT tuner, total data: %d, imported data: %d.", len(data), self.population_size) + logger.info("Start from epoch %d ...", self.epoch) + return self.epoch # return for test diff --git a/utils/third_party/nni_new/algorithms/hpo/ppo_tuner/__init__.py b/utils/third_party/nni_new/algorithms/hpo/ppo_tuner/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..854090c93e71c710dec79c570953ac1fb920dcdc --- /dev/null +++ b/utils/third_party/nni_new/algorithms/hpo/ppo_tuner/__init__.py @@ -0,0 +1 @@ +from .ppo_tuner import PPOTuner, PPOClassArgsValidator diff --git a/utils/third_party/nni_new/algorithms/hpo/ppo_tuner/distri.py b/utils/third_party/nni_new/algorithms/hpo/ppo_tuner/distri.py new file mode 100644 index 0000000000000000000000000000000000000000..8a2a5ed20c3db6086c74339428fd25e0cf806f57 --- /dev/null +++ b/utils/third_party/nni_new/algorithms/hpo/ppo_tuner/distri.py @@ -0,0 +1,183 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +functions for sampling from hidden state +""" + +import tensorflow.compat.v1 as tf +tf.disable_v2_behavior() + +from .util import fc + + +class Pd: + """ + A particular probability distribution + """ + def flatparam(self): + raise NotImplementedError + def mode(self): + raise NotImplementedError + def neglogp(self, x): + # Usually it's easier to define the negative logprob + raise NotImplementedError + def kl(self, other): + raise NotImplementedError + def entropy(self): + raise NotImplementedError + def sample(self): + raise NotImplementedError + def logp(self, x): + return - self.neglogp(x) + def get_shape(self): + return self.flatparam().shape + @property + def shape(self): + return self.get_shape() + def __getitem__(self, idx): + return self.__class__(self.flatparam()[idx]) + +class PdType: + """ + Parametrized family of probability distributions + """ + def pdclass(self): + raise NotImplementedError + def pdfromflat(self, flat, mask, nsteps, size, is_act_model): + return self.pdclass()(flat, mask, nsteps, size, is_act_model) + def pdfromlatent(self, latent_vector, init_scale, init_bias): + raise NotImplementedError + def param_shape(self): + raise NotImplementedError + def sample_shape(self): + raise NotImplementedError + def sample_dtype(self): + raise NotImplementedError + + def param_placeholder(self, prepend_shape, name=None): + return tf.placeholder(dtype=tf.float32, shape=prepend_shape+self.param_shape(), name=name) + def sample_placeholder(self, prepend_shape, name=None): + return tf.placeholder(dtype=self.sample_dtype(), shape=prepend_shape+self.sample_shape(), name=name) + +class CategoricalPd(Pd): + """ + Categorical probability distribution + """ + def __init__(self, logits, mask_npinf, nsteps, size, is_act_model): + self.logits = logits + self.mask_npinf = mask_npinf + self.nsteps = nsteps + self.size = size + self.is_act_model = is_act_model + def flatparam(self): + return self.logits + def mode(self): + return tf.argmax(self.logits, axis=-1) + + @property + def mean(self): + return tf.nn.softmax(self.logits) + def neglogp(self, x): + """ + return tf.nn.sparse_softmax_cross_entropy_with_logits(logits=self.logits, labels=x) + Note: we can't use sparse_softmax_cross_entropy_with_logits because + the implementation does not allow second-order derivatives... + """ + if x.dtype in {tf.uint8, tf.int32, tf.int64}: + # one-hot encoding + x_shape_list = x.shape.as_list() + logits_shape_list = self.logits.get_shape().as_list()[:-1] + for xs, ls in zip(x_shape_list, logits_shape_list): + if xs is not None and ls is not None: + assert xs == ls, 'shape mismatch: {} in x vs {} in logits'.format(xs, ls) + + x = tf.one_hot(x, self.logits.get_shape().as_list()[-1]) + else: + # already encoded + assert x.shape.as_list() == self.logits.shape.as_list() + + return tf.nn.softmax_cross_entropy_with_logits_v2( + logits=self.logits, + labels=x) + + def kl(self, other): + """kl""" + a0 = self.logits - tf.reduce_max(self.logits, axis=-1, keepdims=True) + a1 = other.logits - tf.reduce_max(other.logits, axis=-1, keepdims=True) + ea0 = tf.exp(a0) + ea1 = tf.exp(a1) + z0 = tf.reduce_sum(ea0, axis=-1, keepdims=True) + z1 = tf.reduce_sum(ea1, axis=-1, keepdims=True) + p0 = ea0 / z0 + return tf.reduce_sum(p0 * (a0 - tf.log(z0) - a1 + tf.log(z1)), axis=-1) + + def entropy(self): + """compute entropy""" + a0 = self.logits - tf.reduce_max(self.logits, axis=-1, keepdims=True) + ea0 = tf.exp(a0) + z0 = tf.reduce_sum(ea0, axis=-1, keepdims=True) + p0 = ea0 / z0 + return tf.reduce_sum(p0 * (tf.log(z0) - a0), axis=-1) + + def sample(self): + """sample from logits""" + if not self.is_act_model: + re_res = tf.reshape(self.logits, [-1, self.nsteps, self.size]) + masked_res = tf.math.add(re_res, self.mask_npinf) + re_masked_res = tf.reshape(masked_res, [-1, self.size]) + + u = tf.random_uniform(tf.shape(re_masked_res), dtype=self.logits.dtype) + return tf.argmax(re_masked_res - tf.log(-1*tf.log(u)), axis=-1) + else: + u = tf.random_uniform(tf.shape(self.logits), dtype=self.logits.dtype) + return tf.argmax(self.logits - tf.log(-1*tf.log(u)), axis=-1) + + @classmethod + def fromflat(cls, flat): + return cls(flat) # pylint: disable=no-value-for-parameter + +class CategoricalPdType(PdType): + """ + To create CategoricalPd + """ + def __init__(self, ncat, nsteps, np_mask, is_act_model): + self.ncat = ncat + self.nsteps = nsteps + self.np_mask = np_mask + self.is_act_model = is_act_model + def pdclass(self): + return CategoricalPd + + def pdfromlatent(self, latent_vector, init_scale=1.0, init_bias=0.0): + """add fc and create CategoricalPd""" + pdparam, mask, mask_npinf = _matching_fc(latent_vector, 'pi', self.ncat, self.nsteps, + init_scale=init_scale, init_bias=init_bias, + np_mask=self.np_mask, is_act_model=self.is_act_model) + return self.pdfromflat(pdparam, mask_npinf, self.nsteps, self.ncat, self.is_act_model), pdparam, mask, mask_npinf + + def param_shape(self): + return [self.ncat] + def sample_shape(self): + return [] + def sample_dtype(self): + return tf.int32 + +def _matching_fc(tensor, name, size, nsteps, init_scale, init_bias, np_mask, is_act_model): + """ + Add fc op, and add mask op when not in action mode + """ + if tensor.shape[-1] == size: + assert False + return tensor + else: + mask = tf.get_variable("act_mask", dtype=tf.float32, initializer=np_mask[0], trainable=False) + mask_npinf = tf.get_variable("act_mask_npinf", dtype=tf.float32, initializer=np_mask[1], trainable=False) + res = fc(tensor, name, size, init_scale=init_scale, init_bias=init_bias) + if not is_act_model: + re_res = tf.reshape(res, [-1, nsteps, size]) + masked_res = tf.math.multiply(re_res, mask) + re_masked_res = tf.reshape(masked_res, [-1, size]) + return re_masked_res, mask, mask_npinf + else: + return res, mask, mask_npinf diff --git a/utils/third_party/nni_new/algorithms/hpo/ppo_tuner/model.py b/utils/third_party/nni_new/algorithms/hpo/ppo_tuner/model.py new file mode 100644 index 0000000000000000000000000000000000000000..c6a8479c6d0d06fcf0f7c4b8a64a62ab968988aa --- /dev/null +++ b/utils/third_party/nni_new/algorithms/hpo/ppo_tuner/model.py @@ -0,0 +1,152 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +the main model of policy/value network +""" + +import tensorflow.compat.v1 as tf +tf.disable_v2_behavior() + +from .util import initialize, get_session + +class Model: + """ + We use this object to : + __init__: + - Creates the step_model + - Creates the train_model + + train(): + - Make the training part (feedforward and retropropagation of gradients) + + save/load(): + - Save load the model + """ + def __init__(self, *, policy, nbatch_act, nbatch_train, + nsteps, ent_coef, vf_coef, max_grad_norm, microbatch_size=None, np_mask=None): + self.sess = sess = get_session() + + with tf.variable_scope('ppo2_model', reuse=tf.AUTO_REUSE): + # CREATE OUR TWO MODELS + # act_model that is used for sampling + act_model = policy(nbatch_act, 1, sess, np_mask=np_mask, is_act_model=True) + + # Train model for training + if microbatch_size is None: + train_model = policy(nbatch_train, nsteps, sess, np_mask=np_mask, is_act_model=False) + else: + train_model = policy(microbatch_size, nsteps, sess, np_mask=np_mask, is_act_model=False) + + # CREATE THE PLACEHOLDERS + self.A = A = train_model.pdtype.sample_placeholder([None]) + self.ADV = ADV = tf.placeholder(tf.float32, [None]) + self.R = R = tf.placeholder(tf.float32, [None]) + # Keep track of old actor + self.OLDNEGLOGPAC = OLDNEGLOGPAC = tf.placeholder(tf.float32, [None]) + # Keep track of old critic + self.OLDVPRED = OLDVPRED = tf.placeholder(tf.float32, [None]) + self.LR = LR = tf.placeholder(tf.float32, []) + # Cliprange + self.CLIPRANGE = CLIPRANGE = tf.placeholder(tf.float32, []) + + neglogpac = train_model.pd.neglogp(A) + + # Calculate the entropy + # Entropy is used to improve exploration by limiting the premature convergence to suboptimal policy. + entropy = tf.reduce_mean(train_model.pd.entropy()) + + # CALCULATE THE LOSS + # Total loss = Policy gradient loss - entropy * entropy coefficient + Value coefficient * value loss + + # Clip the value to reduce variability during Critic training + # Get the predicted value + vpred = train_model.vf + vpredclipped = OLDVPRED + tf.clip_by_value(train_model.vf - OLDVPRED, - CLIPRANGE, CLIPRANGE) + # Unclipped value + vf_losses1 = tf.square(vpred - R) + # Clipped value + vf_losses2 = tf.square(vpredclipped - R) + + vf_loss = .5 * tf.reduce_mean(tf.maximum(vf_losses1, vf_losses2)) + + # Calculate ratio (pi current policy / pi old policy) + ratio = tf.exp(OLDNEGLOGPAC - neglogpac) + + # Defining Loss = - J is equivalent to max J + pg_losses = -ADV * ratio + + pg_losses2 = -ADV * tf.clip_by_value(ratio, 1.0 - CLIPRANGE, 1.0 + CLIPRANGE) + + # Final PG loss + pg_loss = tf.reduce_mean(tf.maximum(pg_losses, pg_losses2)) + approxkl = .5 * tf.reduce_mean(tf.square(neglogpac - OLDNEGLOGPAC)) + clipfrac = tf.reduce_mean(tf.to_float(tf.greater(tf.abs(ratio - 1.0), CLIPRANGE))) + + # Total loss + loss = pg_loss - entropy * ent_coef + vf_loss * vf_coef + + # UPDATE THE PARAMETERS USING LOSS + # 1. Get the model parameters + params = tf.trainable_variables('ppo2_model') + # 2. Build our trainer + self.trainer = tf.train.AdamOptimizer(learning_rate=LR, epsilon=1e-5) + # 3. Calculate the gradients + grads_and_var = self.trainer.compute_gradients(loss, params) + grads, var = zip(*grads_and_var) + + if max_grad_norm is not None: + # Clip the gradients (normalize) + grads, _grad_norm = tf.clip_by_global_norm(grads, max_grad_norm) + grads_and_var = list(zip(grads, var)) + # zip aggregate each gradient with parameters associated + # For instance zip(ABCD, xyza) => Ax, By, Cz, Da + + self.grads = grads + self.var = var + self._train_op = self.trainer.apply_gradients(grads_and_var) + self.loss_names = ['policy_loss', 'value_loss', 'policy_entropy', 'approxkl', 'clipfrac'] + self.stats_list = [pg_loss, vf_loss, entropy, approxkl, clipfrac] + + + self.train_model = train_model + self.act_model = act_model + self.step = act_model.step + self.value = act_model.value + self.initial_state = act_model.initial_state + + initialize() + + def train(self, lr, cliprange, obs, returns, masks, actions, values, neglogpacs, states=None): + """ + Train the model. + Here we calculate advantage A(s,a) = R + yV(s') - V(s) + + Returns + ------- + obj + = R + yV(s') + """ + advs = returns - values + + # Normalize the advantages + advs = (advs - advs.mean()) / (advs.std() + 1e-8) + + td_map = { + self.train_model.X : obs, + self.A : actions, + self.ADV : advs, + self.R : returns, + self.LR : lr, + self.CLIPRANGE : cliprange, + self.OLDNEGLOGPAC : neglogpacs, + self.OLDVPRED : values + } + if states is not None: + td_map[self.train_model.S] = states + td_map[self.train_model.M] = masks + + return self.sess.run( + self.stats_list + [self._train_op], + td_map + )[:-1] diff --git a/utils/third_party/nni_new/algorithms/hpo/ppo_tuner/policy.py b/utils/third_party/nni_new/algorithms/hpo/ppo_tuner/policy.py new file mode 100644 index 0000000000000000000000000000000000000000..a35e514eaef36562c26c6a99e8224b4339ae768c --- /dev/null +++ b/utils/third_party/nni_new/algorithms/hpo/ppo_tuner/policy.py @@ -0,0 +1,230 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +build policy/value network from model +""" + +import tensorflow.compat.v1 as tf +tf.disable_v2_behavior() + +from .distri import CategoricalPdType +from .util import lstm_model, fc, observation_placeholder, adjust_shape + + +class PolicyWithValue: + """ + Encapsulates fields and methods for RL policy and value function estimation with shared parameters + """ + + def __init__(self, env, observations, latent, estimate_q=False, vf_latent=None, sess=None, np_mask=None, is_act_model=False, **tensors): + """ + Parameters + ---------- + env : obj + RL environment + observations : tensorflow placeholder + Tensorflow placeholder in which the observations will be fed + latent : tensor + Latent state from which policy distribution parameters should be inferred + vf_latent : tensor + Latent state from which value function should be inferred (if None, then latent is used) + sess : tensorflow session + Tensorflow session to run calculations in (if None, default session is used) + **tensors + Tensorflow tensors for additional attributes such as state or mask + """ + + self.X = observations + self.state = tf.constant([]) + self.initial_state = None + self.__dict__.update(tensors) + + vf_latent = vf_latent if vf_latent is not None else latent + + vf_latent = tf.layers.flatten(vf_latent) + latent = tf.layers.flatten(latent) + + # Based on the action space, will select what probability distribution type + self.np_mask = np_mask + self.pdtype = CategoricalPdType(env.action_space.n, env.nsteps, np_mask, is_act_model) + + self.act_latent = latent + self.nh = env.action_space.n + + self.pd, self.pi, self.mask, self.mask_npinf = self.pdtype.pdfromlatent(latent, init_scale=0.01) + + # Take an action + self.action = self.pd.sample() + + # Calculate the neg log of our probability + self.neglogp = self.pd.neglogp(self.action) + self.sess = sess or tf.get_default_session() + + assert estimate_q is False + self.vf = fc(vf_latent, 'vf', 1) + self.vf = self.vf[:, 0] + + if is_act_model: + self._build_model_for_step() + + def _evaluate(self, variables, observation, **extra_feed): + sess = self.sess + feed_dict = {self.X: adjust_shape(self.X, observation)} + for inpt_name, data in extra_feed.items(): + if inpt_name in self.__dict__.keys(): + inpt = self.__dict__[inpt_name] + if isinstance(inpt, tf.Tensor) and inpt._op.type == 'Placeholder': + feed_dict[inpt] = adjust_shape(inpt, data) + + return sess.run(variables, feed_dict) + + def _build_model_for_step(self): + # multiply with weight and apply mask on self.act_latent to generate + self.act_step = step = tf.placeholder(shape=(), dtype=tf.int64, name='act_step') + with tf.variable_scope('pi', reuse=tf.AUTO_REUSE): + from .util import ortho_init + nin = self.act_latent.get_shape()[1].value + w = tf.get_variable("w", [nin, self.nh], initializer=ortho_init(0.01)) + b = tf.get_variable("b", [self.nh], initializer=tf.constant_initializer(0.0)) + logits = tf.matmul(self.act_latent, w)+b + piece = tf.slice(self.mask, [step, 0], [1, self.nh]) + re_piece = tf.reshape(piece, [-1]) + masked_logits = tf.math.multiply(logits, re_piece) + + npinf_piece = tf.slice(self.mask_npinf, [step, 0], [1, self.nh]) + re_npinf_piece = tf.reshape(npinf_piece, [-1]) + + def sample(logits, mask_npinf): + new_logits = tf.math.add(logits, mask_npinf) + u = tf.random_uniform(tf.shape(new_logits), dtype=logits.dtype) + return tf.argmax(new_logits - tf.log(-1*tf.log(u)), axis=-1) + + def neglogp(logits, x): + # return tf.nn.sparse_softmax_cross_entropy_with_logits(logits=self.logits, labels=x) + # Note: we can't use sparse_softmax_cross_entropy_with_logits because + # the implementation does not allow second-order derivatives... + if x.dtype in {tf.uint8, tf.int32, tf.int64}: + # one-hot encoding + x_shape_list = x.shape.as_list() + logits_shape_list = logits.get_shape().as_list()[:-1] + for xs, ls in zip(x_shape_list, logits_shape_list): + if xs is not None and ls is not None: + assert xs == ls, 'shape mismatch: {} in x vs {} in logits'.format(xs, ls) + + x = tf.one_hot(x, logits.get_shape().as_list()[-1]) + else: + # already encoded + assert x.shape.as_list() == logits.shape.as_list() + + return tf.nn.softmax_cross_entropy_with_logits_v2( + logits=logits, + labels=x) + + self.act_action = sample(masked_logits, re_npinf_piece) + self.act_neglogp = neglogp(masked_logits, self.act_action) + + + def step(self, step, observation, **extra_feed): + """ + Compute next action(s) given the observation(s) + + Parameters + ---------- + observation : np array + Observation data (either single or a batch) + **extra_feed + Additional data such as state or mask (names of the arguments should match the ones in constructor, see __init__) + + Returns + ------- + (action, value estimate, next state, negative log likelihood of the action under current policy parameters) tuple + """ + extra_feed['act_step'] = step + a, v, state, neglogp = self._evaluate([self.act_action, self.vf, self.state, self.act_neglogp], observation, **extra_feed) + if state.size == 0: + state = None + return a, v, state, neglogp + + def value(self, ob, *args, **kwargs): + """ + Compute value estimate(s) given the observation(s) + + Parameters + ---------- + observation : np array + Observation data (either single or a batch) + **extra_feed + Additional data such as state or mask (names of the arguments should match the ones in constructor, see __init__) + + Returns + ------- + Value estimate + """ + return self._evaluate(self.vf, ob, *args, **kwargs) + + +def build_lstm_policy(model_config, value_network=None, estimate_q=False, **policy_kwargs): + """ + Build lstm policy and value network, they share the same lstm network. + the parameters all use their default values. + + Parameter + --------- + model_config : obj + Configurations of the model + value_network : obj + The network for value function + estimate_q : bool + Whether to estimate ``q`` + **policy_kwargs + The kwargs for policy network, i.e., lstm model + + Returns + ------- + func + The policy network + """ + policy_network = lstm_model(**policy_kwargs) + + def policy_fn(nbatch=None, nsteps=None, sess=None, observ_placeholder=None, np_mask=None, is_act_model=False): + ob_space = model_config.observation_space + + X = observ_placeholder if observ_placeholder is not None else observation_placeholder(ob_space, batch_size=nbatch) + + extra_tensors = {} + + # encode_observation is not necessary anymore as we use embedding_lookup + encoded_x = X + + with tf.variable_scope('pi', reuse=tf.AUTO_REUSE): + policy_latent = policy_network(encoded_x, 1, model_config.observation_space.n) + if isinstance(policy_latent, tuple): + policy_latent, recurrent_tensors = policy_latent + + if recurrent_tensors is not None: + # recurrent architecture, need a few more steps + nenv = nbatch // nsteps + assert nenv > 0, 'Bad input for recurrent policy: batch size {} smaller than nsteps {}'.format(nbatch, nsteps) + policy_latent, recurrent_tensors = policy_network(encoded_x, nenv, model_config.observation_space.n) + extra_tensors.update(recurrent_tensors) + + _v_net = value_network + + assert _v_net is None or _v_net == 'shared' + vf_latent = policy_latent + + policy = PolicyWithValue( + env=model_config, + observations=X, + latent=policy_latent, + vf_latent=vf_latent, + sess=sess, + estimate_q=estimate_q, + np_mask=np_mask, + is_act_model=is_act_model, + **extra_tensors + ) + return policy + + return policy_fn diff --git a/utils/third_party/nni_new/algorithms/hpo/ppo_tuner/ppo_tuner.py b/utils/third_party/nni_new/algorithms/hpo/ppo_tuner/ppo_tuner.py new file mode 100644 index 0000000000000000000000000000000000000000..33b62d600e3b6bb788edab7db53a1223a846b2eb --- /dev/null +++ b/utils/third_party/nni_new/algorithms/hpo/ppo_tuner/ppo_tuner.py @@ -0,0 +1,654 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +ppo_tuner.py including: + class PPOTuner +""" + +import copy +import logging +import numpy as np +from gym import spaces +from schema import Schema, Optional + +import nni +from nni import ClassArgsValidator +from nni.tuner import Tuner +from nni.utils import OptimizeMode, extract_scalar_reward + +from .model import Model +from .util import set_global_seeds +from .policy import build_lstm_policy + + +logger = logging.getLogger('ppo_tuner_AutoML') + +def _constfn(val): + """ + Wrap as function + """ + def f(_): + return val + return f + + +class ModelConfig: + """ + Configurations of the PPO model + """ + def __init__(self): + self.observation_space = None + self.action_space = None + self.num_envs = 0 + self.nsteps = 0 + + self.ent_coef = 0.0 + self.lr = 3e-4 + self.vf_coef = 0.5 + self.max_grad_norm = 0.5 + self.gamma = 0.99 + self.lam = 0.95 + self.cliprange = 0.2 + self.embedding_size = None # the embedding is for each action + + self.noptepochs = 4 # number of training epochs per update + self.total_timesteps = 5000 # number of timesteps (i.e. number of actions taken in the environment) + self.nminibatches = 4 # number of training minibatches per update. For recurrent policies, + # should be smaller or equal than number of environments run in parallel. + +class TrialsInfo: + """ + Informations of each trial from one model inference + """ + def __init__(self, obs, actions, values, neglogpacs, dones, last_value, inf_batch_size): + self.iter = 0 + self.obs = obs + self.actions = actions + self.values = values + self.neglogpacs = neglogpacs + self.dones = dones + self.last_value = last_value + + self.rewards = None + self.returns = None + + self.inf_batch_size = inf_batch_size + #self.states = None + + def get_next(self): + """ + Get actions of the next trial + """ + if self.iter >= self.inf_batch_size: + return None, None + actions = [] + for step in self.actions: + actions.append(step[self.iter]) + self.iter += 1 + return self.iter - 1, actions + + def update_rewards(self, rewards, returns): + """ + After the trial is finished, reward and return of this trial is updated + """ + self.rewards = rewards + self.returns = returns + + def convert_shape(self): + """ + Convert shape + """ + def sf01(arr): + """ + swap and then flatten axes 0 and 1 + """ + s = arr.shape + return arr.swapaxes(0, 1).reshape(s[0] * s[1], *s[2:]) + self.obs = sf01(self.obs) + self.returns = sf01(self.returns) + self.dones = sf01(self.dones) + self.actions = sf01(self.actions) + self.values = sf01(self.values) + self.neglogpacs = sf01(self.neglogpacs) + + +class PPOModel: + """ + PPO Model + """ + def __init__(self, model_config, mask): + self.model_config = model_config + self.states = None # initial state of lstm in policy/value network + self.nupdates = None # the number of func train is invoked, used to tune lr and cliprange + self.cur_update = 1 # record the current update + self.np_mask = mask # record the mask of each action within one trial + + set_global_seeds(None) + assert isinstance(self.model_config.lr, float) + self.lr = _constfn(self.model_config.lr) + assert isinstance(self.model_config.cliprange, float) + self.cliprange = _constfn(self.model_config.cliprange) + + # build lstm policy network, value share the same network + policy = build_lstm_policy(model_config) + + # Get the nb of env + nenvs = model_config.num_envs + + # Calculate the batch_size + self.nbatch = nbatch = nenvs * model_config.nsteps # num of record per update + nbatch_train = nbatch // model_config.nminibatches # get batch size + # self.nupdates is used to tune lr and cliprange + self.nupdates = self.model_config.total_timesteps // self.nbatch + + # Instantiate the model object (that creates act_model and train_model) + self.model = Model(policy=policy, nbatch_act=nenvs, nbatch_train=nbatch_train, + nsteps=model_config.nsteps, ent_coef=model_config.ent_coef, vf_coef=model_config.vf_coef, + max_grad_norm=model_config.max_grad_norm, np_mask=self.np_mask) + + self.states = self.model.initial_state + + logger.info('=== finished PPOModel initialization') + + def inference(self, num): + """ + Generate actions along with related info from policy network. + observation is the action of the last step. + + Parameters + ---------- + num: int + The number of trials to generate + + Returns + ------- + mb_obs : list + Observation of the ``num`` configurations + mb_actions : list + Actions of the ``num`` configurations + mb_values : list + Values from the value function of the ``num`` configurations + mb_neglogpacs : list + ``neglogp`` of the ``num`` configurations + mb_dones : list + To show whether the play is done, always ``True`` + last_values : tensorflow tensor + The last values of the ``num`` configurations, got with session run + """ + # Here, we init the lists that will contain the mb of experiences + mb_obs, mb_actions, mb_values, mb_dones, mb_neglogpacs = [], [], [], [], [] + # initial observation + # use the (n+1)th embedding to represent the first step action + first_step_ob = self.model_config.action_space.n + obs = [first_step_ob for _ in range(num)] + dones = [True for _ in range(num)] + states = self.states + # For n in range number of steps + for cur_step in range(self.model_config.nsteps): + # Given observations, get action value and neglopacs + # We already have self.obs because Runner superclass run self.obs[:] = env.reset() on init + actions, values, states, neglogpacs = self.model.step(cur_step, obs, S=states, M=dones) + mb_obs.append(obs.copy()) + mb_actions.append(actions) + mb_values.append(values) + mb_neglogpacs.append(neglogpacs) + mb_dones.append(dones) + + # Take actions in env and look the results + # Infos contains a ton of useful informations + obs[:] = actions + if cur_step == self.model_config.nsteps - 1: + dones = [True for _ in range(num)] + else: + dones = [False for _ in range(num)] + + #batch of steps to batch of rollouts + np_obs = np.asarray(obs) + mb_obs = np.asarray(mb_obs, dtype=np_obs.dtype) + mb_actions = np.asarray(mb_actions) + mb_values = np.asarray(mb_values, dtype=np.float32) + mb_neglogpacs = np.asarray(mb_neglogpacs, dtype=np.float32) + mb_dones = np.asarray(mb_dones, dtype=np.bool) + last_values = self.model.value(np_obs, S=states, M=dones) + + return mb_obs, mb_actions, mb_values, mb_neglogpacs, mb_dones, last_values + + def compute_rewards(self, trials_info, trials_result): + """ + Compute the rewards of the trials in trials_info based on trials_result, + and update the rewards in trials_info + + Parameters + ---------- + trials_info : TrialsInfo + Info of the generated trials + trials_result : list + Final results (e.g., acc) of the generated trials + """ + mb_rewards = np.asarray([trials_result for _ in trials_info.actions], dtype=np.float32) + # discount/bootstrap off value fn + mb_returns = np.zeros_like(mb_rewards) + mb_advs = np.zeros_like(mb_rewards) + lastgaelam = 0 + last_dones = np.asarray([True for _ in trials_result], dtype=np.bool) # ugly + for t in reversed(range(self.model_config.nsteps)): + if t == self.model_config.nsteps - 1: + nextnonterminal = 1.0 - last_dones + nextvalues = trials_info.last_value + else: + nextnonterminal = 1.0 - trials_info.dones[t+1] + nextvalues = trials_info.values[t+1] + delta = mb_rewards[t] + self.model_config.gamma * nextvalues * nextnonterminal - trials_info.values[t] + lastgaelam = delta + self.model_config.gamma * self.model_config.lam * nextnonterminal * lastgaelam + mb_advs[t] = lastgaelam # pylint: disable=unsupported-assignment-operation + mb_returns = mb_advs + trials_info.values + + trials_info.update_rewards(mb_rewards, mb_returns) + trials_info.convert_shape() + + def train(self, trials_info, nenvs): + """ + Train the policy/value network using trials_info + + Parameters + ---------- + trials_info : TrialsInfo + Complete info of the generated trials from the previous inference + nenvs : int + The batch size of the (previous) inference + """ + # keep frac decay for future optimization + if self.cur_update <= self.nupdates: + frac = 1.0 - (self.cur_update - 1.0) / self.nupdates + else: + logger.warning('current update (self.cur_update) %d has exceeded total updates (self.nupdates) %d', + self.cur_update, self.nupdates) + frac = 1.0 - (self.nupdates - 1.0) / self.nupdates + lrnow = self.lr(frac) + cliprangenow = self.cliprange(frac) + self.cur_update += 1 + + states = self.states + + assert states is not None # recurrent version + assert nenvs % self.model_config.nminibatches == 0 + envsperbatch = nenvs // self.model_config.nminibatches + envinds = np.arange(nenvs) + flatinds = np.arange(nenvs * self.model_config.nsteps).reshape(nenvs, self.model_config.nsteps) + for _ in range(self.model_config.noptepochs): + np.random.shuffle(envinds) + for start in range(0, nenvs, envsperbatch): + end = start + envsperbatch + mbenvinds = envinds[start:end] + mbflatinds = flatinds[mbenvinds].ravel() + slices = (arr[mbflatinds] for arr in (trials_info.obs, trials_info.returns, trials_info.dones, + trials_info.actions, trials_info.values, trials_info.neglogpacs)) + mbstates = states[mbenvinds] + self.model.train(lrnow, cliprangenow, *slices, mbstates) + +class PPOClassArgsValidator(ClassArgsValidator): + def validate_class_args(self, **kwargs): + Schema({ + 'optimize_mode': self.choices('optimize_mode', 'maximize', 'minimize'), + Optional('trials_per_update'): self.range('trials_per_update', int, 0, 99999), + Optional('epochs_per_update'): self.range('epochs_per_update', int, 0, 99999), + Optional('minibatch_size'): self.range('minibatch_size', int, 0, 99999), + Optional('ent_coef'): float, + Optional('lr'): float, + Optional('vf_coef'): float, + Optional('max_grad_norm'): float, + Optional('gamma'): float, + Optional('lam'): float, + Optional('cliprange'): float, + }).validate(kwargs) + +class PPOTuner(Tuner): + """ + PPOTuner, the implementation inherits the main logic of the implementation + [ppo2 from openai](https://github.com/openai/baselines/tree/master/baselines/ppo2), and is adapted for NAS scenario. + It uses ``lstm`` for its policy network and value network, policy and value share the same network. + """ + + def __init__(self, optimize_mode, trials_per_update=20, epochs_per_update=4, minibatch_size=4, + ent_coef=0.0, lr=3e-4, vf_coef=0.5, max_grad_norm=0.5, gamma=0.99, lam=0.95, cliprange=0.2): + """ + Initialization, PPO model is not initialized here as search space is not received yet. + + Parameters + ---------- + optimize_mode : str + maximize or minimize + trials_per_update : int + Number of trials to have for each model update + epochs_per_update : int + Number of epochs to run for each model update + minibatch_size : int + Minibatch size (number of trials) for the update + ent_coef : float + Policy entropy coefficient in the optimization objective + lr : float + Learning rate of the model (lstm network), constant + vf_coef : float + Value function loss coefficient in the optimization objective + max_grad_norm : float + Gradient norm clipping coefficient + gamma : float + Discounting factor + lam : float + Advantage estimation discounting factor (lambda in the paper) + cliprange : float + Cliprange in the PPO algorithm, constant + """ + self.optimize_mode = OptimizeMode(optimize_mode) + self.model_config = ModelConfig() + self.model = None + self.search_space = None + self.running_trials = {} # key: parameter_id, value: actions/states/etc. + self.inf_batch_size = trials_per_update # number of trials to generate in one inference + self.first_inf = True # indicate whether it is the first time to inference new trials + self.trials_result = [None for _ in range(self.inf_batch_size)] # results of finished trials + + self.credit = 0 # record the unsatisfied trial requests + self.param_ids = [] + self.finished_trials = 0 + self.chosen_arch_template = {} + + self.actions_spaces = None + self.actions_to_config = None + self.full_act_space = None + self.trials_info = None + + self.all_trials = {} # used to dedup the same trial, key: config, value: final result + + self.model_config.num_envs = self.inf_batch_size + self.model_config.noptepochs = epochs_per_update + self.model_config.nminibatches = minibatch_size + + self.send_trial_callback = None + logger.info('Finished PPOTuner initialization') + + def _process_nas_space(self, search_space): + actions_spaces = [] + actions_to_config = [] + for key, val in search_space.items(): + if val['_type'] == 'layer_choice': + actions_to_config.append((key, 'layer_choice')) + actions_spaces.append(val['_value']) + self.chosen_arch_template[key] = None + elif val['_type'] == 'input_choice': + candidates = val['_value']['candidates'] + n_chosen = val['_value']['n_chosen'] + if n_chosen not in [0, 1, [0, 1]]: + raise ValueError('Optional_input_size can only be 0, 1, or [0, 1], but the pecified one is %s' + % (n_chosen)) + if isinstance(n_chosen, list): + actions_to_config.append((key, 'input_choice')) + # FIXME: risk, candidates might also have None + actions_spaces.append(['None', *candidates]) + self.chosen_arch_template[key] = None + elif n_chosen == 1: + actions_to_config.append((key, 'input_choice')) + actions_spaces.append(candidates) + self.chosen_arch_template[key] = None + elif n_chosen == 0: + self.chosen_arch_template[key] = [] + else: + raise ValueError('Unsupported search space type: %s' % (val['_type'])) + + # calculate observation space + dedup = {} + for step in actions_spaces: + for action in step: + dedup[action] = 1 + full_act_space = [act for act, _ in dedup.items()] + assert len(full_act_space) == len(dedup) + observation_space = len(full_act_space) + nsteps = len(actions_spaces) + + return actions_spaces, actions_to_config, full_act_space, observation_space, nsteps + + def _generate_action_mask(self): + """ + Different step could have different action space. to deal with this case, we merge all the + possible actions into one action space, and use mask to indicate available actions for each step + """ + two_masks = [] + + mask = [] + for acts in self.actions_spaces: + one_mask = [0 for _ in range(len(self.full_act_space))] + for act in acts: + idx = self.full_act_space.index(act) + one_mask[idx] = 1 + mask.append(one_mask) + two_masks.append(mask) + + mask = [] + for acts in self.actions_spaces: + one_mask = [-np.inf for _ in range(len(self.full_act_space))] + for act in acts: + idx = self.full_act_space.index(act) + one_mask[idx] = 0 + mask.append(one_mask) + two_masks.append(mask) + + return np.asarray(two_masks, dtype=np.float32) + + def update_search_space(self, search_space): + """ + Get search space, currently the space only includes that for NAS + + Parameters + ---------- + search_space : dict + Search space for NAS + the format could be referred to search space spec (https://nni.readthedocs.io/en/latest/Tutorial/SearchSpaceSpec.html). + """ + logger.info('update search space %s', search_space) + assert self.search_space is None + self.search_space = search_space + + assert self.model_config.observation_space is None + assert self.model_config.action_space is None + + self.actions_spaces, self.actions_to_config, self.full_act_space, obs_space, nsteps = self._process_nas_space(search_space) + + self.model_config.observation_space = spaces.Discrete(obs_space) + self.model_config.action_space = spaces.Discrete(obs_space) + self.model_config.nsteps = nsteps + + # generate mask in numpy + mask = self._generate_action_mask() + + assert self.model is None + self.model = PPOModel(self.model_config, mask) + + def _actions_to_config(self, actions): + """ + Given actions, to generate the corresponding trial configuration + """ + chosen_arch = copy.deepcopy(self.chosen_arch_template) + for cnt, act in enumerate(actions): + act_name = self.full_act_space[act] + (_key, _type) = self.actions_to_config[cnt] + if _type == 'input_choice': + if act_name == 'None': + chosen_arch[_key] = {'_value': [], '_idx': []} + else: + candidates = self.search_space[_key]['_value']['candidates'] + idx = candidates.index(act_name) + chosen_arch[_key] = {'_value': [act_name], '_idx': [idx]} + elif _type == 'layer_choice': + idx = self.search_space[_key]['_value'].index(act_name) + chosen_arch[_key] = {'_value': act_name, '_idx': idx} + else: + raise ValueError('unrecognized key: {0}'.format(_type)) + return chosen_arch + + def generate_multiple_parameters(self, parameter_id_list, **kwargs): + """ + Returns multiple sets of trial (hyper-)parameters, as iterable of serializable objects. + + Parameters + ---------- + parameter_id_list : list of int + Unique identifiers for each set of requested hyper-parameters. + These will later be used in :meth:`receive_trial_result`. + **kwargs + Not used + + Returns + ------- + list + A list of newly generated configurations + """ + result = [] + self.send_trial_callback = kwargs['st_callback'] + for parameter_id in parameter_id_list: + had_exception = False + try: + logger.debug("generating param for %s", parameter_id) + res = self.generate_parameters(parameter_id, **kwargs) + except nni.NoMoreTrialError: + had_exception = True + if not had_exception: + result.append(res) + return result + + def generate_parameters(self, parameter_id, **kwargs): + """ + Generate parameters, if no trial configration for now, self.credit plus 1 to send the config later + + Parameters + ---------- + parameter_id : int + Unique identifier for requested hyper-parameters. + This will later be used in :meth:`receive_trial_result`. + **kwargs + Not used + + Returns + ------- + dict + One newly generated configuration + + """ + if self.first_inf: + self.trials_result = [None for _ in range(self.inf_batch_size)] + mb_obs, mb_actions, mb_values, mb_neglogpacs, mb_dones, last_values = self.model.inference(self.inf_batch_size) + self.trials_info = TrialsInfo(mb_obs, mb_actions, mb_values, mb_neglogpacs, + mb_dones, last_values, self.inf_batch_size) + self.first_inf = False + + trial_info_idx, actions = self.trials_info.get_next() + if trial_info_idx is None: + logger.debug('Credit added by one in parameters request') + self.credit += 1 + self.param_ids.append(parameter_id) + raise nni.NoMoreTrialError('no more parameters now.') + + self.running_trials[parameter_id] = trial_info_idx + new_config = self._actions_to_config(actions) + return new_config + + def _next_round_inference(self): + """ + Run a inference to generate next batch of configurations + """ + logger.debug('Start next round inference...') + self.finished_trials = 0 + self.model.compute_rewards(self.trials_info, self.trials_result) + self.model.train(self.trials_info, self.inf_batch_size) + self.running_trials = {} + # generate new trials + self.trials_result = [None for _ in range(self.inf_batch_size)] + mb_obs, mb_actions, mb_values, mb_neglogpacs, mb_dones, last_values = self.model.inference(self.inf_batch_size) + self.trials_info = TrialsInfo(mb_obs, mb_actions, + mb_values, mb_neglogpacs, + mb_dones, last_values, + self.inf_batch_size) + logger.debug('Next round inference complete.') + # check credit and submit new trials + for _ in range(self.credit): + trial_info_idx, actions = self.trials_info.get_next() + if trial_info_idx is None: + logger.warning('No enough trial config, trials_per_update is suggested to be larger than trialConcurrency') + break + assert self.param_ids + param_id = self.param_ids.pop() + self.running_trials[param_id] = trial_info_idx + new_config = self._actions_to_config(actions) + self.send_trial_callback(param_id, new_config) + self.credit -= 1 + logger.debug('Send new trial (%d, %s) for reducing credit', param_id, new_config) + + def receive_trial_result(self, parameter_id, parameters, value, **kwargs): + """ + Receive trial's result. if the number of finished trials equals self.inf_batch_size, start the next update to + train the model. + + Parameters + ---------- + parameter_id : int + Unique identifier of used hyper-parameters, same with :meth:`generate_parameters`. + parameters : dict + Hyper-parameters generated by :meth:`generate_parameters`. + value : dict + Result from trial (the return value of :func:`nni.report_final_result`). + """ + trial_info_idx = self.running_trials.pop(parameter_id, None) + assert trial_info_idx is not None + + value = extract_scalar_reward(value) + if self.optimize_mode == OptimizeMode.Minimize: + value = -value + + self.trials_result[trial_info_idx] = value + self.finished_trials += 1 + + logger.debug('receive_trial_result, parameter_id %d, trial_info_idx %d, finished_trials %d, inf_batch_size %d', + parameter_id, trial_info_idx, self.finished_trials, self.inf_batch_size) + if self.finished_trials == self.inf_batch_size: + logger.debug('Start next round inference in receive_trial_result') + self._next_round_inference() + + def trial_end(self, parameter_id, success, **kwargs): + """ + To deal with trial failure. If a trial fails, it is popped out from ``self.running_trials``, + and the final result of this trial is assigned with the average of the finished trials. + + Parameters + ---------- + parameter_id : int + Unique identifier for hyper-parameters used by this trial. + success : bool + True if the trial successfully completed; False if failed or terminated. + **kwargs + Not used + """ + if not success: + if parameter_id not in self.running_trials: + logger.warning('The trial is failed, but self.running_trial does not have this trial') + return + trial_info_idx = self.running_trials.pop(parameter_id, None) + assert trial_info_idx is not None + # use mean of finished trials as the result of this failed trial + values = [val for val in self.trials_result if val is not None] + logger.warning('In trial_end, values: %s', values) + self.trials_result[trial_info_idx] = (sum(values) / len(values)) if values else 0 + self.finished_trials += 1 + if self.finished_trials == self.inf_batch_size: + logger.debug('Start next round inference in trial_end') + self._next_round_inference() + + def import_data(self, data): + """ + Import additional data for tuning, not supported yet. + + Parameters + ---------- + data : list + A list of dictionarys, each of which has at least two keys, ``parameter`` and ``value`` + """ + logger.warning('PPOTuner cannot leverage imported data.') diff --git a/utils/third_party/nni_new/algorithms/hpo/ppo_tuner/util.py b/utils/third_party/nni_new/algorithms/hpo/ppo_tuner/util.py new file mode 100644 index 0000000000000000000000000000000000000000..605292de4002f114e935cb24a87c96921940e959 --- /dev/null +++ b/utils/third_party/nni_new/algorithms/hpo/ppo_tuner/util.py @@ -0,0 +1,260 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +util functions +""" + +import os +import random +import multiprocessing +import numpy as np +import tensorflow.compat.v1 as tf +tf.disable_v2_behavior() +from gym.spaces import Discrete, Box, MultiDiscrete + +def set_global_seeds(i): + """set global seeds""" + rank = 0 + myseed = i + 1000 * rank if i is not None else None + tf.set_random_seed(myseed) + np.random.seed(myseed) + random.seed(myseed) + +def batch_to_seq(h, nbatch, nsteps, flat=False): + """convert from batch to sequence""" + if flat: + h = tf.reshape(h, [nbatch, nsteps]) + else: + h = tf.reshape(h, [nbatch, nsteps, -1]) + return [tf.squeeze(v, [1]) for v in tf.split(axis=1, num_or_size_splits=nsteps, value=h)] + +def seq_to_batch(h, flat=False): + """convert from sequence to batch""" + shape = h[0].get_shape().as_list() + if not flat: + assert len(shape) > 1 + nh = h[0].get_shape()[-1].value + return tf.reshape(tf.concat(axis=1, values=h), [-1, nh]) + else: + return tf.reshape(tf.stack(values=h, axis=1), [-1]) + +def lstm(xs, ms, s, scope, nh, init_scale=1.0): + """lstm cell""" + _, nin = [v.value for v in xs[0].get_shape()] # the first is nbatch + with tf.variable_scope(scope): + wx = tf.get_variable("wx", [nin, nh*4], initializer=ortho_init(init_scale)) + wh = tf.get_variable("wh", [nh, nh*4], initializer=ortho_init(init_scale)) + b = tf.get_variable("b", [nh*4], initializer=tf.constant_initializer(0.0)) + + c, h = tf.split(axis=1, num_or_size_splits=2, value=s) + for idx, (x, m) in enumerate(zip(xs, ms)): + c = c*(1-m) + h = h*(1-m) + z = tf.matmul(x, wx) + tf.matmul(h, wh) + b + i, f, o, u = tf.split(axis=1, num_or_size_splits=4, value=z) + i = tf.nn.sigmoid(i) + f = tf.nn.sigmoid(f) + o = tf.nn.sigmoid(o) + u = tf.tanh(u) + c = f*c + i*u + h = o*tf.tanh(c) + xs[idx] = h + s = tf.concat(axis=1, values=[c, h]) + return xs, s + +def lstm_model(nlstm=128, layer_norm=False): + """ + Builds LSTM (Long-Short Term Memory) network to be used in a policy. + Note that the resulting function returns not only the output of the LSTM + (i.e. hidden state of lstm for each step in the sequence), but also a dictionary + with auxiliary tensors to be set as policy attributes. + + Specifically, + S is a placeholder to feed current state (LSTM state has to be managed outside policy) + M is a placeholder for the mask (used to mask out observations after the end of the episode, but can be used for other purposes too) + initial_state is a numpy array containing initial lstm state (usually zeros) + state is the output LSTM state (to be fed into S at the next call) + + + An example of usage of lstm-based policy can be found here: common/tests/test_doc_examples.py/test_lstm_example + + Parameters + ---------- + nlstm : int + LSTM hidden state size + layer_norm : bool + if True, layer-normalized version of LSTM is used + + Returns + ------- + function that builds LSTM with a given input tensor / placeholder + """ + + def network_fn(X, nenv=1, obs_size=-1): + with tf.variable_scope("emb", reuse=tf.AUTO_REUSE): + w_emb = tf.get_variable("w_emb", [obs_size+1, 32]) + X = tf.nn.embedding_lookup(w_emb, X) + + nbatch = X.shape[0] + nsteps = nbatch // nenv + + h = tf.layers.flatten(X) + + M = tf.placeholder(tf.float32, [nbatch]) #mask (done t-1) + S = tf.placeholder(tf.float32, [nenv, 2*nlstm]) #states + + xs = batch_to_seq(h, nenv, nsteps) + ms = batch_to_seq(M, nenv, nsteps) + + assert not layer_norm + h5, snew = lstm(xs, ms, S, scope='lstm', nh=nlstm) + + h = seq_to_batch(h5) + initial_state = np.zeros(S.shape.as_list(), dtype=float) + + return h, {'S':S, 'M':M, 'state':snew, 'initial_state':initial_state} + + return network_fn + +def ortho_init(scale=1.0): + """init approach""" + def _ortho_init(shape, dtype, partition_info=None): + #lasagne ortho init for tf + shape = tuple(shape) + if len(shape) == 2: + flat_shape = shape + elif len(shape) == 4: # assumes NHWC + flat_shape = (np.prod(shape[:-1]), shape[-1]) + else: + raise NotImplementedError + a = np.random.normal(0.0, 1.0, flat_shape) + u, _, v = np.linalg.svd(a, full_matrices=False) + q = u if u.shape == flat_shape else v # pick the one with the correct shape + q = q.reshape(shape) + return (scale * q[:shape[0], :shape[1]]).astype(np.float32) + return _ortho_init + +def fc(x, scope, nh, *, init_scale=1.0, init_bias=0.0): + """fully connected op""" + with tf.variable_scope(scope): + nin = x.get_shape()[1].value + w = tf.get_variable("w", [nin, nh], initializer=ortho_init(init_scale)) + b = tf.get_variable("b", [nh], initializer=tf.constant_initializer(init_bias)) + return tf.matmul(x, w)+b + +def _check_shape(placeholder_shape, data_shape): + """ + check if two shapes are compatible (i.e. differ only by dimensions of size 1, or by the batch dimension) + """ + + return True + +# ================================================================ +# Shape adjustment for feeding into tf placeholders +# ================================================================ +def adjust_shape(placeholder, data): + """ + adjust shape of the data to the shape of the placeholder if possible. + If shape is incompatible, AssertionError is thrown + + Parameters + ---------- + placeholder + tensorflow input placeholder + data + input data to be (potentially) reshaped to be fed into placeholder + + Returns + ------- + reshaped data + """ + if not isinstance(data, np.ndarray) and not isinstance(data, list): + return data + if isinstance(data, list): + data = np.array(data) + + placeholder_shape = [x or -1 for x in placeholder.shape.as_list()] + + assert _check_shape(placeholder_shape, data.shape), \ + 'Shape of data {} is not compatible with shape of the placeholder {}'.format(data.shape, placeholder_shape) + + return np.reshape(data, placeholder_shape) + +# ================================================================ +# Global session +# ================================================================ + +def get_session(config=None): + """Get default session or create one with a given config""" + sess = tf.get_default_session() + if sess is None: + sess = make_session(config=config, make_default=True) + return sess + +def make_session(config=None, num_cpu=None, make_default=False, graph=None): + """Returns a session that will use CPU's only""" + if num_cpu is None: + num_cpu = int(os.getenv('RCALL_NUM_CPU', multiprocessing.cpu_count())) + if config is None: + config = tf.ConfigProto( + allow_soft_placement=True, + inter_op_parallelism_threads=num_cpu, + intra_op_parallelism_threads=num_cpu) + config.gpu_options.allow_growth = True + + if make_default: + return tf.InteractiveSession(config=config, graph=graph) + else: + return tf.Session(config=config, graph=graph) + +ALREADY_INITIALIZED = set() + +def initialize(): + """Initialize all the uninitialized variables in the global scope.""" + new_variables = set(tf.global_variables()) - ALREADY_INITIALIZED + get_session().run(tf.variables_initializer(new_variables)) + + ALREADY_INITIALIZED.update(new_variables) + +def observation_placeholder(ob_space, batch_size=None, name='Ob'): + """ + Create placeholder to feed observations into of the size appropriate to the observation space + + Parameters + ---------- + ob_space : gym.Space + observation space + batch_size : int + size of the batch to be fed into input. Can be left None in most cases. + name : str + name of the placeholder + + Returns + ------- + tensorflow placeholder tensor + """ + + assert isinstance(ob_space, (Discrete, Box, MultiDiscrete)), \ + 'Can only deal with Discrete and Box observation spaces for now' + + dtype = ob_space.dtype + if dtype == np.int8: + dtype = np.uint8 + + return tf.placeholder(shape=(batch_size,) + ob_space.shape, dtype=dtype, name=name) + +def explained_variance(ypred, y): + """ + Computes fraction of variance that ypred explains about y. + Returns 1 - Var[y-ypred] / Var[y] + + interpretation: + ev=0 => might as well have predicted zero + ev=1 => perfect prediction + ev<0 => worse than just predicting zero + + """ + assert y.ndim == 1 and ypred.ndim == 1 + vary = np.var(y) + return np.nan if vary == 0 else 1 - np.var(y-ypred)/vary diff --git a/utils/third_party/nni_new/algorithms/hpo/regularized_evolution_tuner.py b/utils/third_party/nni_new/algorithms/hpo/regularized_evolution_tuner.py new file mode 100644 index 0000000000000000000000000000000000000000..ac756c33885b8ad150e862c6eb315136fb0d4b71 --- /dev/null +++ b/utils/third_party/nni_new/algorithms/hpo/regularized_evolution_tuner.py @@ -0,0 +1,172 @@ +import copy +import logging +import random +from collections import deque + +from schema import Schema, Optional +import nni +from nni.tuner import Tuner +from nni import ClassArgsValidator +from nni.utils import OptimizeMode, extract_scalar_reward + +logger = logging.getLogger(__name__) + + +class FinishedIndividual: + def __init__(self, parameter_id, parameters, result): + """ + Parameters + ---------- + parameter_id: int + the index of the parameter + parameters : dict + chosen architecture and parameters + result : float + final metric of the chosen one + """ + self.parameter_id = parameter_id + self.parameters = parameters + self.result = result + + +class EvolutionClassArgsValidator(ClassArgsValidator): + def validate_class_args(self, **kwargs): + Schema({ + 'optimize_mode': self.choices('optimize_mode', 'maximize', 'minimize'), + Optional('population_size'): self.range('population_size', int, 0, 99999), + Optional('sample_size'): self.range('sample_size', int, 0, 9999), + }).validate(kwargs) + + +class RegularizedEvolutionTuner(Tuner): + """ + RegularizedEvolutionTuner is tuner using Evolution NAS Tuner. + See ``Regularized Evolution for Image Classifier Architecture Search`` for details. + + Parameters + --- + optimize_mode: str + whether to maximize metric or not. default: 'maximize' + population_size: int + the maximum number of kept models + sample_size: int + the number of models chosen from population each time when evolution + """ + def __init__(self, optimize_mode="maximize", population_size=100, sample_size=25): + super(RegularizedEvolutionTuner, self).__init__() + self.optimize_mode = OptimizeMode(optimize_mode) + self.population_size = population_size + self.sample_size = sample_size + self.initial_population = deque() + self.population = deque() + self.history = {} + self.search_space = None + self._from_initial = {} # whether the parameter is from initial population + + def generate_parameters(self, parameter_id, **kwargs): + """ + This function will returns a dict of trial (hyper-)parameters, as a serializable object. + + Parameters + --- + parameter_id: int + the index of current set of parameters + """ + if self.initial_population: + arch = self.initial_population.popleft() + self.history[parameter_id] = arch + self._from_initial[parameter_id] = True + return arch + elif self.population: + sample = [] + while len(sample) < self.sample_size: + sample.append(random.choice(list(self.population))) + + candidate = max(sample, key=lambda x: x.result) + arch = self._mutate_model(candidate) + self.history[parameter_id] = arch + self._from_initial[parameter_id] = False + return arch + else: + raise nni.NoMoreTrialError + + def receive_trial_result(self, parameter_id, parameters, value, **kwargs): + """ + Record the result from a trial + + Parameters + ---------- + parameter_id : int + parameters : dict + value : dict/float + if value is dict, it should have "default" key. + value is final metrics of the trial. + """ + reward = extract_scalar_reward(value) + if parameter_id not in self.history: + raise RuntimeError('Received parameter_id not in total_data.') + params = self.history[parameter_id] + + if self.optimize_mode == OptimizeMode.Minimize: + reward = -reward + + self.population.append(FinishedIndividual(parameter_id, params, reward)) + if len(self.population) > self.population_size: + self.population.popleft() + + def update_search_space(self, search_space): + """ + Update search space. + Search_space contains the information that user pre-defined. + + Parameters + ---------- + search_space : dict + """ + logger.info('update search space %s', search_space) + assert self.search_space is None + self.search_space = search_space + + for _, val in search_space.items(): + if val['_type'] != 'layer_choice' and val['_type'] != 'input_choice': + raise ValueError('Unsupported search space type: %s' % (val['_type'])) + + self._generate_initial_population() + + def trial_end(self, parameter_id, success, **kwargs): + if not success: + del self.history[parameter_id] + if self._from_initial[parameter_id]: + self.initial_population.append(self._random_model()) + del self._from_initial[parameter_id] + + def _mutate(self, key, individual): + mutate_val = self.search_space[key] + if mutate_val['_type'] == 'layer_choice': + idx = random.randint(0, len(mutate_val['_value']) - 1) + individual[key] = {'_value': mutate_val['_value'][idx], '_idx': idx} + elif mutate_val['_type'] == 'input_choice': + candidates = mutate_val['_value']['candidates'] + n_chosen = mutate_val['_value']['n_chosen'] + idxs = [random.randint(0, len(candidates) - 1) for _ in range(n_chosen)] + vals = [candidates[k] for k in idxs] + individual[key] = {'_value': vals, '_idx': idxs} + else: + raise KeyError + + def _random_model(self): + individual = {} + for key in self.search_space.keys(): + self._mutate(key, individual) + return individual + + def _mutate_model(self, model): + new_individual = copy.deepcopy(model.parameters) + mutate_key = random.choice(list(new_individual.keys())) + self._mutate(mutate_key, new_individual) + return new_individual + + def _generate_initial_population(self): + while len(self.initial_population) < self.population_size: + self.initial_population.append(self._random_model()) + logger.info('init population done.') diff --git a/utils/third_party/nni_new/algorithms/hpo/smac_tuner/__init__.py b/utils/third_party/nni_new/algorithms/hpo/smac_tuner/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9ac4c4bc4809aa0d04d67127a25ebfefa577be3c --- /dev/null +++ b/utils/third_party/nni_new/algorithms/hpo/smac_tuner/__init__.py @@ -0,0 +1,4 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .smac_tuner import SMACTuner, SMACClassArgsValidator diff --git a/utils/third_party/nni_new/algorithms/hpo/smac_tuner/convert_ss_to_scenario.py b/utils/third_party/nni_new/algorithms/hpo/smac_tuner/convert_ss_to_scenario.py new file mode 100644 index 0000000000000000000000000000000000000000..ef817c8cf4e10ad08185910b95bd4adc909b2dd3 --- /dev/null +++ b/utils/third_party/nni_new/algorithms/hpo/smac_tuner/convert_ss_to_scenario.py @@ -0,0 +1,202 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import json + +import numpy as np + + +def get_json_content(file_path): + """ + Load json file content + + Parameters + ---------- + file_path: + path to the file + + Raises + ------ + TypeError + Error with the file path + """ + try: + with open(file_path, 'r') as file: + return json.load(file) + except TypeError as err: + print('Error: ', err) + return None + + +def generate_pcs(nni_search_space_content): + """ + Generate the Parameter Configuration Space (PCS) which defines the + legal ranges of the parameters to be optimized and their default values. + Generally, the format is: + # parameter_name categorical {value_1, ..., value_N} [default value] + # parameter_name ordinal {value_1, ..., value_N} [default value] + # parameter_name integer [min_value, max_value] [default value] + # parameter_name integer [min_value, max_value] [default value] log + # parameter_name real [min_value, max_value] [default value] + # parameter_name real [min_value, max_value] [default value] log + Reference: https://automl.github.io/SMAC3/stable/options.html + + Parameters + ---------- + nni_search_space_content: search_space + The search space in this experiment in nni + + Returns + ------- + Parameter Configuration Space (PCS) + the legal ranges of the parameters to be optimized and their default values + + Raises + ------ + RuntimeError + unsupported type or value error or incorrect search space + """ + categorical_dict = {} + search_space = nni_search_space_content + + def dump_categorical(fd, key, categories): + choice_len = len(categories) + if key in categorical_dict: + raise RuntimeError( + '%s has already existed, please make sure search space has no duplicate key.' % key) + categorical_dict[key] = search_space[key]['_value'] + fd.write('%s categorical {%s} [0]\n' % (key, ','.join(map(str, range(choice_len))))) + + with open('param_config_space.pcs', 'w') as pcs_fd: + if isinstance(search_space, dict): + for key in search_space.keys(): + if isinstance(search_space[key], dict): + try: + if search_space[key]['_type'] == 'choice': + dump_categorical(pcs_fd, key, search_space[key]['_value']) + elif search_space[key]['_type'] == 'randint': + lower, upper = search_space[key]['_value'] + if lower + 1 == upper: + dump_categorical(pcs_fd, key, [lower]) + else: + pcs_fd.write('%s integer [%d, %d] [%d]\n' % (key, lower, upper - 1, lower)) + elif search_space[key]['_type'] == 'uniform': + low, high = search_space[key]['_value'] + if low == high: + dump_categorical(pcs_fd, key, [low]) + else: + pcs_fd.write('%s real [%s, %s] [%s]\n' % (key, low, high, low)) + elif search_space[key]['_type'] == 'loguniform': + # use np.round here to ensure that the rounded default value is in the range, + # which will be rounded in configure_space package + low, high = list(np.round(np.log(search_space[key]['_value']), 10)) + if low == high: + dump_categorical(pcs_fd, key, [search_space[key]['_value'][0]]) + else: + pcs_fd.write('%s real [%s, %s] [%s]\n' % (key, low, high, low)) + elif search_space[key]['_type'] == 'quniform': + low, high, q = search_space[key]['_value'][0:3] + vals = np.clip(np.arange(np.round(low / q), np.round(high / q) + 1) * q, low, high).tolist() + pcs_fd.write('%s ordinal {%s} [%s]\n' % ( + key, + json.dumps(vals)[1:-1], + json.dumps(vals[0]))) + else: + raise RuntimeError('unsupported _type %s' % search_space[key]['_type']) + except: + raise RuntimeError('_type or _value error.') + else: + raise RuntimeError('incorrect search space.') + return categorical_dict + return None + + +def generate_scenario(ss_content): + """ + Generate the scenario. The scenario-object (smac.scenario.scenario.Scenario) is used to configure SMAC and + can be constructed either by providing an actual scenario-object, or by specifing the options in a scenario file. + Reference: https://automl.github.io/SMAC3/stable/options.html + The format of the scenario file is one option per line: + OPTION1 = VALUE1 + OPTION2 = VALUE2 + ... + Parameters + ---------- + abort_on_first_run_crash: bool + If true, SMAC will abort if the first run of the target algorithm crashes. Default: True, + because trials reported to nni tuner would always in success state + algo: function + Specifies the target algorithm call that SMAC will optimize. Interpreted as a bash-command. + Not required by tuner, but required by nni's training service for running trials + always_race_default: + Race new incumbents always against default configuration + cost_for_crash: + Defines the cost-value for crashed runs on scenarios with quality as run-obj. Default: 2147483647.0. + Trials reported to nni tuner would always in success state + cutoff_time: + Maximum runtime, after which the target algorithm is cancelled. `Required if *run_obj* is runtime` + deterministic: bool + If true, the optimization process will be repeatable. + execdir: + Specifies the path to the execution-directory. Default: . + Trials are executed by nni's training service + feature_file: + Specifies the file with the instance-features. + No features specified or feature file is not supported + initial_incumbent: + DEFAULT is the default from the PCS. Default: DEFAULT. Must be from: [‘DEFAULT’, ‘RANDOM’]. + input_psmac_dirs: + For parallel SMAC, multiple output-directories are used. + Parallelism is supported by nni + instance_file: + Specifies the file with the training-instances. Not supported + intensification_percentage: + The fraction of time to be used on intensification (versus choice of next Configurations). Default: 0.5. + Not supported, trials are controlled by nni's training service and kill be assessor + maxR: int + Maximum number of calls per configuration. Default: 2000. + memory_limit: + Maximum available memory the target algorithm can occupy before being cancelled. + minR: int + Minimum number of calls per configuration. Default: 1. + output_dir: + Specifies the output-directory for all emerging files, such as logging and results. + Default: smac3-output_2018-01-22_15:05:56_807070. + overall_obj: + PARX, where X is an integer defining the penalty imposed on timeouts (i.e. runtimes that exceed the cutoff-time). + Timeout is not supported + paramfile: + Specifies the path to the PCS-file. + run_obj: + Defines what metric to optimize. When optimizing runtime, cutoff_time is required as well. + Must be from: [‘runtime’, ‘quality’]. + runcount_limit: int + Maximum number of algorithm-calls during optimization. Default: inf. + Use default because this is controlled by nni + shared_model: + Whether to run SMAC in parallel mode. Parallelism is supported by nni + test_instance_file: + Specifies the file with the test-instances. Instance is not supported + tuner-timeout: + Maximum amount of CPU-time used for optimization. Not supported + wallclock_limit: int + Maximum amount of wallclock-time used for optimization. Default: inf. + Use default because this is controlled by nni + + Returns + ------- + Scenario: + The scenario-object (smac.scenario.scenario.Scenario) is used to configure SMAC and can be constructed + either by providing an actual scenario-object, or by specifing the options in a scenario file + """ + with open('scenario.txt', 'w') as sce_fd: + sce_fd.write('deterministic = 0\n') + # sce_fd.write('output_dir = \n') + sce_fd.write('paramfile = param_config_space.pcs\n') + sce_fd.write('run_obj = quality\n') + + return generate_pcs(ss_content) + + +if __name__ == '__main__': + generate_scenario('search_space.json') diff --git a/utils/third_party/nni_new/algorithms/hpo/smac_tuner/smac_tuner.py b/utils/third_party/nni_new/algorithms/hpo/smac_tuner/smac_tuner.py new file mode 100644 index 0000000000000000000000000000000000000000..fdaed49d8f2480516ec1e1e62fc43eabdd574049 --- /dev/null +++ b/utils/third_party/nni_new/algorithms/hpo/smac_tuner/smac_tuner.py @@ -0,0 +1,346 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +smac_tuner.py +""" + +import logging +import sys + +import numpy as np +from schema import Schema, Optional + +from smac.facade.epils_facade import EPILS +from smac.facade.roar_facade import ROAR +from smac.facade.smac_facade import SMAC +from smac.scenario.scenario import Scenario +from smac.utils.io.cmd_reader import CMDReader + +from ConfigSpaceNNI import Configuration + +import nni +from nni import ClassArgsValidator +from nni.tuner import Tuner +from nni.utils import OptimizeMode, extract_scalar_reward + +from .convert_ss_to_scenario import generate_scenario + +logger = logging.getLogger('smac_AutoML') + +class SMACClassArgsValidator(ClassArgsValidator): + def validate_class_args(self, **kwargs): + Schema({ + 'optimize_mode': self.choices('optimize_mode', 'maximize', 'minimize'), + Optional('config_dedup'): bool + }).validate(kwargs) + +class SMACTuner(Tuner): + """ + This is a wrapper of [SMAC](https://github.com/automl/SMAC3) following NNI tuner interface. + It only supports ``SMAC`` mode, and does not support the multiple instances of SMAC3 (i.e., + the same configuration is run multiple times). + """ + def __init__(self, optimize_mode="maximize", config_dedup=False): + """ + Parameters + ---------- + optimize_mode : str + Optimize mode, 'maximize' or 'minimize', by default 'maximize' + config_dedup : bool + If True, the tuner will not generate a configuration that has been already generated. + If False, a configuration may be generated twice, but it is rare for relatively large search space. + """ + self.logger = logger + self.optimize_mode = OptimizeMode(optimize_mode) + self.total_data = {} + self.optimizer = None + self.smbo_solver = None + self.first_one = True + self.update_ss_done = False + self.loguniform_key = set() + self.categorical_dict = {} + self.cs = None + self.dedup = config_dedup + + def _main_cli(self): + """ + Main function of SMAC for CLI interface. Some initializations of the wrapped SMAC are done + in this function. + + Returns + ------- + obj + The object of the SMAC optimizer + """ + self.logger.info("SMAC call: %s", " ".join(sys.argv)) + + cmd_reader = CMDReader() + args, _ = cmd_reader.read_cmd() + + root_logger = logging.getLogger() + root_logger.setLevel(args.verbose_level) + logger_handler = logging.StreamHandler(stream=sys.stdout) + if root_logger.level >= logging.INFO: + formatter = logging.Formatter("%(levelname)s:\t%(message)s") + else: + formatter = logging.Formatter( + "%(asctime)s:%(levelname)s:%(name)s:%(message)s", + "%Y-%m-%d %H:%M:%S") + logger_handler.setFormatter(formatter) + root_logger.addHandler(logger_handler) + # remove default handler + root_logger.removeHandler(root_logger.handlers[0]) + + # Create defaults + rh = None + initial_configs = None + stats = None + incumbent = None + + # Create scenario-object + scen = Scenario(args.scenario_file, []) + self.cs = scen.cs + + if args.mode == "SMAC": + optimizer = SMAC( + scenario=scen, + rng=np.random.RandomState(args.seed), + runhistory=rh, + initial_configurations=initial_configs, + stats=stats, + restore_incumbent=incumbent, + run_id=args.seed) + elif args.mode == "ROAR": + optimizer = ROAR( + scenario=scen, + rng=np.random.RandomState(args.seed), + runhistory=rh, + initial_configurations=initial_configs, + run_id=args.seed) + elif args.mode == "EPILS": + optimizer = EPILS( + scenario=scen, + rng=np.random.RandomState(args.seed), + runhistory=rh, + initial_configurations=initial_configs, + run_id=args.seed) + else: + optimizer = None + + return optimizer + + def update_search_space(self, search_space): + """ + Convert search_space to the format that ``SMAC3`` could recognize, thus, not all the search space types + are supported. In this function, we also do the initialization of `SMAC3`, i.e., calling ``self._main_cli``. + + NOTE: updating search space during experiment running is not supported. + + Parameters + ---------- + search_space : dict + The format could be referred to search space spec (https://nni.readthedocs.io/en/latest/Tutorial/SearchSpaceSpec.html). + """ + self.logger.info('update search space in SMAC.') + if not self.update_ss_done: + self.categorical_dict = generate_scenario(search_space) + if self.categorical_dict is None: + raise RuntimeError('categorical dict is not correctly returned after parsing search space.') + # TODO: this is ugly, we put all the initialization work in this method, because initialization relies + # on search space, also because update_search_space is called at the beginning. + self.optimizer = self._main_cli() + self.smbo_solver = self.optimizer.solver + self.loguniform_key = {key for key in search_space.keys() if search_space[key]['_type'] == 'loguniform'} + self.update_ss_done = True + else: + self.logger.warning('update search space is not supported.') + + def receive_trial_result(self, parameter_id, parameters, value, **kwargs): + """ + Receive a trial's final performance result reported through :func:``nni.report_final_result`` by the trial. + GridSearchTuner does not need trial's results. + + Parameters + ---------- + parameter_id : int + Unique identifier of used hyper-parameters, same with :meth:`generate_parameters`. + parameters : dict + Hyper-parameters generated by :meth:`generate_parameters`. + value : dict + Result from trial (the return value of :func:`nni.report_final_result`). + + Raises + ------ + RuntimeError + Received parameter id not in ``self.total_data`` + """ + reward = extract_scalar_reward(value) + if self.optimize_mode is OptimizeMode.Maximize: + reward = -reward + + if parameter_id not in self.total_data: + raise RuntimeError('Received parameter_id not in total_data.') + if self.first_one: + self.smbo_solver.nni_smac_receive_first_run(self.total_data[parameter_id], reward) + self.first_one = False + else: + self.smbo_solver.nni_smac_receive_runs(self.total_data[parameter_id], reward) + + def param_postprocess(self, challenger_dict): + """ + Postprocessing for a set of hyperparameters includes: + 1. Convert the values of type ``loguniform`` back to their initial range. + 2. Convert ``categorical``: categorical values in search space are changed to list of numbers before, + those original values will be changed back in this function. + + Parameters + ---------- + challenger_dict : dict + challenger dict + + Returns + ------- + dict + dict which stores copy of challengers + """ + converted_dict = {} + for key, value in challenger_dict.items(): + # convert to loguniform + if key in self.loguniform_key: + converted_dict[key] = np.exp(challenger_dict[key]) + # convert categorical back to original value + elif key in self.categorical_dict: + idx = challenger_dict[key] + converted_dict[key] = self.categorical_dict[key][idx] + else: + converted_dict[key] = value + return converted_dict + + def generate_parameters(self, parameter_id, **kwargs): + """ + Generate one instance of hyperparameters (i.e., one configuration). + Get one from SMAC3's ``challengers``. + + Parameters + ---------- + parameter_id : int + Unique identifier for requested hyper-parameters. This will later be used in :meth:`receive_trial_result`. + **kwargs + Not used + + Returns + ------- + dict + One newly generated configuration + """ + if self.first_one: + init_challenger = self.smbo_solver.nni_smac_start() + self.total_data[parameter_id] = init_challenger + return self.param_postprocess(init_challenger.get_dictionary()) + else: + challengers = self.smbo_solver.nni_smac_request_challengers() + challengers_empty = True + for challenger in challengers: + challengers_empty = False + if self.dedup: + match = [v for k, v in self.total_data.items() \ + if v.get_dictionary() == challenger.get_dictionary()] + if match: + continue + self.total_data[parameter_id] = challenger + return self.param_postprocess(challenger.get_dictionary()) + assert challengers_empty is False, 'The case that challengers is empty is not handled.' + self.logger.info('In generate_parameters: No more new parameters.') + raise nni.NoMoreTrialError('No more new parameters.') + + def generate_multiple_parameters(self, parameter_id_list, **kwargs): + """ + Generate mutiple instances of hyperparameters. If it is a first request, + retrieve the instances from initial challengers. While if it is not, request + new challengers and retrieve instances from the requested challengers. + + Parameters + ---------- + parameter_id_list: list of int + Unique identifiers for each set of requested hyper-parameters. + These will later be used in :meth:`receive_trial_result`. + **kwargs + Not used + + Returns + ------- + list + a list of newly generated configurations + """ + if self.first_one: + params = [] + for one_id in parameter_id_list: + init_challenger = self.smbo_solver.nni_smac_start() + self.total_data[one_id] = init_challenger + params.append(self.param_postprocess(init_challenger.get_dictionary())) + else: + challengers = self.smbo_solver.nni_smac_request_challengers() + cnt = 0 + params = [] + for challenger in challengers: + if cnt >= len(parameter_id_list): + break + if self.dedup: + match = [v for k, v in self.total_data.items() \ + if v.get_dictionary() == challenger.get_dictionary()] + if match: + continue + self.total_data[parameter_id_list[cnt]] = challenger + params.append(self.param_postprocess(challenger.get_dictionary())) + cnt += 1 + if self.dedup and not params: + self.logger.info('In generate_multiple_parameters: No more new parameters.') + return params + + def import_data(self, data): + """ + Import additional data for tuning. + + Parameters + ---------- + data : list of dict + Each of which has at least two keys, ``parameter`` and ``value``. + """ + _completed_num = 0 + for trial_info in data: + self.logger.info("Importing data, current processing progress %s / %s", _completed_num, len(data)) + # simply validate data format + assert "parameter" in trial_info + _params = trial_info["parameter"] + assert "value" in trial_info + _value = trial_info['value'] + if not _value: + self.logger.info("Useless trial data, value is %s, skip this trial data.", _value) + continue + _value = extract_scalar_reward(_value) + # convert the keys in loguniform and categorical types + valid_entry = True + for key, value in _params.items(): + if key in self.loguniform_key: + _params[key] = np.log(value) + elif key in self.categorical_dict: + if value in self.categorical_dict[key]: + _params[key] = self.categorical_dict[key].index(value) + else: + self.logger.info("The value %s of key %s is not in search space.", str(value), key) + valid_entry = False + break + if not valid_entry: + continue + # start import this data entry + _completed_num += 1 + config = Configuration(self.cs, values=_params) + if self.optimize_mode is OptimizeMode.Maximize: + _value = -_value + if self.first_one: + self.smbo_solver.nni_smac_receive_first_run(config, _value) + self.first_one = False + else: + self.smbo_solver.nni_smac_receive_runs(config, _value) + self.logger.info("Successfully import data to smac tuner, total data: %d, imported data: %d.", len(data), _completed_num) diff --git a/utils/third_party/nni_new/algorithms/nas/pytorch/cdarts/__init__.py b/utils/third_party/nni_new/algorithms/nas/pytorch/cdarts/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..ab34902e0ec5d4f7cbbf943f2dcaf03d0a7bc97d --- /dev/null +++ b/utils/third_party/nni_new/algorithms/nas/pytorch/cdarts/__init__.py @@ -0,0 +1,5 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .mutator import RegularizedDartsMutator, RegularizedMutatorParallel, DartsDiscreteMutator +from .trainer import CdartsTrainer diff --git a/utils/third_party/nni_new/algorithms/nas/pytorch/cdarts/mutator.py b/utils/third_party/nni_new/algorithms/nas/pytorch/cdarts/mutator.py new file mode 100644 index 0000000000000000000000000000000000000000..a0bf79040eafb86c336467246521fb9767f60144 --- /dev/null +++ b/utils/third_party/nni_new/algorithms/nas/pytorch/cdarts/mutator.py @@ -0,0 +1,143 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import torch + +from apex.parallel import DistributedDataParallel # pylint: disable=import-error +from nni.algorithms.nas.pytorch.darts import DartsMutator # pylint: disable=wrong-import-order +from nni.nas.pytorch.mutables import LayerChoice # pylint: disable=wrong-import-order +from nni.nas.pytorch.mutator import Mutator # pylint: disable=wrong-import-order + + +class RegularizedDartsMutator(DartsMutator): + """ + This is :class:`~nni.algorithms.nas.pytorch.darts.DartsMutator` basically, with two differences. + + 1. Choices can be cut (bypassed). This is done by ``cut_choices``. Cutted choices will not be used in + forward pass and thus consumes no memory. + + 2. Regularization on choices, to prevent the mutator from overfitting on some choices. + """ + + def reset(self): + """ + Warnings + -------- + Renamed :func:`~reset_with_loss` to return regularization loss on reset. + """ + raise ValueError("You should probably call `reset_with_loss`.") + + def cut_choices(self, cut_num=2): + """ + Cut the choices with the smallest weights. + ``cut_num`` should be the accumulative number of cutting, e.g., if first time cutting + is 2, the second time should be 4 to cut another two. + + Parameters + ---------- + cut_num : int + Number of choices to cut, so far. + + Warnings + -------- + Though the parameters are set to :math:`-\infty` to be bypassed, they will still receive gradient of 0, + which introduced ``nan`` problem when calling ``optimizer.step()``. To solve this issue, a simple way is to + reset nan to :math:`-\infty` each time after the parameters are updated. + """ + # `cut_choices` is implemented but not used in current implementation of CdartsTrainer + for mutable in self.mutables: + if isinstance(mutable, LayerChoice): + _, idx = torch.topk(-self.choices[mutable.key], cut_num) + with torch.no_grad(): + for i in idx: + self.choices[mutable.key][i] = -float("inf") + + def reset_with_loss(self): + """ + Resample and return loss. If loss is 0, to avoid device issue, it will return ``None``. + + Currently loss penalty are proportional to the L1-norm of parameters corresponding + to modules if their type name contains certain substrings. These substrings include: ``poolwithoutbn``, + ``identity``, ``dilconv``. + """ + self._cache, reg_loss = self.sample_search() + return reg_loss + + def sample_search(self): + result = super().sample_search() + loss = [] + for mutable in self.mutables: + if isinstance(mutable, LayerChoice): + def need_reg(choice): + return any(t in str(type(choice)).lower() for t in ["poolwithoutbn", "identity", "dilconv"]) + + for i, choice in enumerate(mutable.choices): + if need_reg(choice): + norm = torch.abs(self.choices[mutable.key][i]) + if norm < 1E10: + loss.append(norm) + if not loss: + return result, None + return result, sum(loss) + + def export(self, logger=None): + """ + Export an architecture with logger. Genotype will be printed with logger. + + Returns + ------- + dict + A mapping from mutable keys to decisions. + """ + result = self.sample_final() + if hasattr(self.model, "plot_genotype") and logger is not None: + genotypes = self.model.plot_genotype(result, logger) + return result, genotypes + + +class RegularizedMutatorParallel(DistributedDataParallel): + """ + Parallelize :class:`~RegularizedDartsMutator`. + + This makes :func:`~RegularizedDartsMutator.reset_with_loss` method parallelized, + also allowing :func:`~RegularizedDartsMutator.cut_choices` and :func:`~RegularizedDartsMutator.export` + to be easily accessible. + """ + def reset_with_loss(self): + """ + Parallelized :func:`~RegularizedDartsMutator.reset_with_loss`. + """ + result = self.module.reset_with_loss() + self.callback_queued = False + return result + + def cut_choices(self, *args, **kwargs): + """ + Parallelized :func:`~RegularizedDartsMutator.cut_choices`. + """ + self.module.cut_choices(*args, **kwargs) + + def export(self, logger): + """ + Parallelized :func:`~RegularizedDartsMutator.export`. + """ + return self.module.export(logger) + + +class DartsDiscreteMutator(Mutator): + """ + A mutator that applies the final sampling result of a parent mutator on another model to train. + + Parameters + ---------- + model : nn.Module + The model to apply the mutator. + parent_mutator : nni.nas.pytorch.mutator.Mutator + The mutator that provides ``sample_final`` method, that will be called to get the architecture. + """ + def __init__(self, model, parent_mutator): + super().__init__(model) + self.__dict__["parent_mutator"] = parent_mutator # avoid parameters to be included + + def sample_search(self): + return self.parent_mutator.sample_final() diff --git a/utils/third_party/nni_new/algorithms/nas/pytorch/cdarts/trainer.py b/utils/third_party/nni_new/algorithms/nas/pytorch/cdarts/trainer.py new file mode 100644 index 0000000000000000000000000000000000000000..1a5174216fc4ffdc224e9cd6bd1483322005fe01 --- /dev/null +++ b/utils/third_party/nni_new/algorithms/nas/pytorch/cdarts/trainer.py @@ -0,0 +1,275 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import json +import logging +import os + +import torch +import torch.nn as nn +import torch.nn.functional as F +import apex # pylint: disable=import-error +from apex.parallel import DistributedDataParallel # pylint: disable=import-error +from .mutator import RegularizedDartsMutator, RegularizedMutatorParallel, DartsDiscreteMutator # pylint: disable=wrong-import-order +from nni.nas.pytorch.utils import AverageMeterGroup # pylint: disable=wrong-import-order + +from .utils import CyclicIterator, TorchTensorEncoder, accuracy, reduce_metrics + +PHASE_SMALL = "small" +PHASE_LARGE = "large" + + +class InteractiveKLLoss(nn.Module): + def __init__(self, temperature): + super().__init__() + self.temperature = temperature + # self.kl_loss = nn.KLDivLoss(reduction = 'batchmean') + self.kl_loss = nn.KLDivLoss() + + def forward(self, student, teacher): + return self.kl_loss(F.log_softmax(student / self.temperature, dim=1), + F.softmax(teacher / self.temperature, dim=1)) + + +class CdartsTrainer(object): + """ + CDARTS trainer. + + Parameters + ---------- + model_small : nn.Module + PyTorch model to be trained. This is the search network of CDARTS. + model_large : nn.Module + PyTorch model to be trained. This is the evaluation network of CDARTS. + criterion : callable + Receives logits and ground truth label, return a loss tensor, e.g., ``nn.CrossEntropyLoss()``. + loaders : list of torch.utils.data.DataLoader + List of train data and valid data loaders, for training weights and architecture weights respectively. + samplers : list of torch.utils.data.Sampler + List of train data and valid data samplers. This can be PyTorch standard samplers if not distributed. + In distributed mode, sampler needs to have ``set_epoch`` method. Refer to data utils in CDARTS example for details. + logger : logging.Logger + The logger for logging. Will use nni logger by default (if logger is ``None``). + regular_coeff : float + The coefficient of regular loss. + regular_ratio : float + The ratio of regular loss. + warmup_epochs : int + The epochs to warmup the search network + fix_head : bool + ``True`` if fixing the paramters of auxiliary heads, else unfix the paramters of auxiliary heads. + epochs : int + Number of epochs planned for training. + steps_per_epoch : int + Steps of one epoch. + loss_alpha : float + The loss coefficient. + loss_T : float + The loss coefficient. + distributed : bool + ``True`` if using distributed training, else non-distributed training. + log_frequency : int + Step count per logging. + grad_clip : float + Gradient clipping for weights. + interactive_type : string + ``kl`` or ``smoothl1``. + output_path : string + Log storage path. + w_lr : float + Learning rate of the search network parameters. + w_momentum : float + Momentum of the search and the evaluation network. + w_weight_decay : float + The weight decay the search and the evaluation network parameters. + alpha_lr : float + Learning rate of the architecture parameters. + alpha_weight_decay : float + The weight decay the architecture parameters. + nasnet_lr : float + Learning rate of the evaluation network parameters. + local_rank : int + The number of thread. + share_module : bool + ``True`` if sharing the stem and auxiliary heads, else not sharing these modules. + """ + def __init__(self, model_small, model_large, criterion, loaders, samplers, logger=None, + regular_coeff=5, regular_ratio=0.2, warmup_epochs=2, fix_head=True, + epochs=32, steps_per_epoch=None, loss_alpha=2, loss_T=2, distributed=True, + log_frequency=10, grad_clip=5.0, interactive_type='kl', output_path='./outputs', + w_lr=0.2, w_momentum=0.9, w_weight_decay=3e-4, alpha_lr=0.2, alpha_weight_decay=1e-4, + nasnet_lr=0.2, local_rank=0, share_module=True): + if logger is None: + logger = logging.getLogger(__name__) + train_loader, valid_loader = loaders + train_sampler, valid_sampler = samplers + self.train_loader = CyclicIterator(train_loader, train_sampler, distributed) + self.valid_loader = CyclicIterator(valid_loader, valid_sampler, distributed) + + self.regular_coeff = regular_coeff + self.regular_ratio = regular_ratio + self.warmup_epochs = warmup_epochs + self.fix_head = fix_head + self.epochs = epochs + self.steps_per_epoch = steps_per_epoch + if self.steps_per_epoch is None: + self.steps_per_epoch = min(len(self.train_loader), len(self.valid_loader)) + self.loss_alpha = loss_alpha + self.grad_clip = grad_clip + if interactive_type == "kl": + self.interactive_loss = InteractiveKLLoss(loss_T) + elif interactive_type == "smoothl1": + self.interactive_loss = nn.SmoothL1Loss() + self.loss_T = loss_T + self.distributed = distributed + self.log_frequency = log_frequency + self.main_proc = not distributed or local_rank == 0 + + self.logger = logger + self.checkpoint_dir = output_path + if self.main_proc: + os.makedirs(self.checkpoint_dir, exist_ok=True) + if distributed: + torch.distributed.barrier() + + self.model_small = model_small + self.model_large = model_large + if self.fix_head: + for param in self.model_small.aux_head.parameters(): + param.requires_grad = False + for param in self.model_large.aux_head.parameters(): + param.requires_grad = False + + self.mutator_small = RegularizedDartsMutator(self.model_small).cuda() + self.mutator_large = DartsDiscreteMutator(self.model_large, self.mutator_small).cuda() + self.criterion = criterion + + self.optimizer_small = torch.optim.SGD(self.model_small.parameters(), w_lr, + momentum=w_momentum, weight_decay=w_weight_decay) + self.optimizer_large = torch.optim.SGD(self.model_large.parameters(), nasnet_lr, + momentum=w_momentum, weight_decay=w_weight_decay) + self.optimizer_alpha = torch.optim.Adam(self.mutator_small.parameters(), alpha_lr, + betas=(0.5, 0.999), weight_decay=alpha_weight_decay) + + if distributed: + apex.parallel.convert_syncbn_model(self.model_small) + apex.parallel.convert_syncbn_model(self.model_large) + self.model_small = DistributedDataParallel(self.model_small, delay_allreduce=True) + self.model_large = DistributedDataParallel(self.model_large, delay_allreduce=True) + self.mutator_small = RegularizedMutatorParallel(self.mutator_small, delay_allreduce=True) + if share_module: + self.model_small.callback_queued = True + self.model_large.callback_queued = True + # mutator large never gets optimized, so do not need parallelized + + def _warmup(self, phase, epoch): + assert phase in [PHASE_SMALL, PHASE_LARGE] + if phase == PHASE_SMALL: + model, optimizer = self.model_small, self.optimizer_small + elif phase == PHASE_LARGE: + model, optimizer = self.model_large, self.optimizer_large + model.train() + meters = AverageMeterGroup() + for step in range(self.steps_per_epoch): + x, y = next(self.train_loader) + x, y = x.cuda(), y.cuda() + + optimizer.zero_grad() + logits_main, _ = model(x) + loss = self.criterion(logits_main, y) + loss.backward() + + self._clip_grad_norm(model) + optimizer.step() + prec1, prec5 = accuracy(logits_main, y, topk=(1, 5)) + metrics = {"prec1": prec1, "prec5": prec5, "loss": loss} + metrics = reduce_metrics(metrics, self.distributed) + meters.update(metrics) + if self.main_proc and (step % self.log_frequency == 0 or step + 1 == self.steps_per_epoch): + self.logger.info("Epoch [%d/%d] Step [%d/%d] (%s) %s", epoch + 1, self.epochs, + step + 1, self.steps_per_epoch, phase, meters) + + def _clip_grad_norm(self, model): + if isinstance(model, DistributedDataParallel): + nn.utils.clip_grad_norm_(model.module.parameters(), self.grad_clip) + else: + nn.utils.clip_grad_norm_(model.parameters(), self.grad_clip) + + def _reset_nan(self, parameters): + with torch.no_grad(): + for param in parameters: + for i, p in enumerate(param): + if p != p: # equivalent to `isnan(p)` + param[i] = float("-inf") + + def _joint_train(self, epoch): + self.model_large.train() + self.model_small.train() + meters = AverageMeterGroup() + for step in range(self.steps_per_epoch): + trn_x, trn_y = next(self.train_loader) + val_x, val_y = next(self.valid_loader) + trn_x, trn_y = trn_x.cuda(), trn_y.cuda() + val_x, val_y = val_x.cuda(), val_y.cuda() + + # step 1. optimize architecture + self.optimizer_alpha.zero_grad() + self.optimizer_large.zero_grad() + reg_decay = max(self.regular_coeff * (1 - float(epoch - self.warmup_epochs) / ( + (self.epochs - self.warmup_epochs) * self.regular_ratio)), 0) + loss_regular = self.mutator_small.reset_with_loss() + if loss_regular: + loss_regular *= reg_decay + logits_search, emsemble_logits_search = self.model_small(val_x) + logits_main, emsemble_logits_main = self.model_large(val_x) + loss_cls = (self.criterion(logits_search, val_y) + self.criterion(logits_main, val_y)) / self.loss_alpha + loss_interactive = self.interactive_loss(emsemble_logits_search, emsemble_logits_main) * (self.loss_T ** 2) * self.loss_alpha + loss = loss_cls + loss_interactive + loss_regular + loss.backward() + self._clip_grad_norm(self.model_large) + self.optimizer_large.step() + self.optimizer_alpha.step() + # NOTE: need to call here `self._reset_nan(self.mutator_small.parameters())` if `cut_choices` + + # step 2. optimize op weights + self.optimizer_small.zero_grad() + with torch.no_grad(): + # resample architecture since parameters have been changed + self.mutator_small.reset_with_loss() + logits_search_train, _ = self.model_small(trn_x) + loss_weight = self.criterion(logits_search_train, trn_y) + loss_weight.backward() + self._clip_grad_norm(self.model_small) + self.optimizer_small.step() + + metrics = {"loss_cls": loss_cls, "loss_interactive": loss_interactive, + "loss_regular": loss_regular, "loss_weight": loss_weight} + metrics = reduce_metrics(metrics, self.distributed) + meters.update(metrics) + + if self.main_proc and (step % self.log_frequency == 0 or step + 1 == self.steps_per_epoch): + self.logger.info("Epoch [%d/%d] Step [%d/%d] (joint) %s", epoch + 1, self.epochs, + step + 1, self.steps_per_epoch, meters) + + def train(self): + for epoch in range(self.epochs): + if epoch < self.warmup_epochs: + with torch.no_grad(): # otherwise grads will be retained on the architecture params + self.mutator_small.reset_with_loss() + self._warmup(PHASE_SMALL, epoch) + else: + with torch.no_grad(): + self.mutator_large.reset() + self._warmup(PHASE_LARGE, epoch) + self._joint_train(epoch) + + self.export(os.path.join(self.checkpoint_dir, "epoch_{:02d}.json".format(epoch)), + os.path.join(self.checkpoint_dir, "epoch_{:02d}.genotypes".format(epoch))) + + def export(self, file, genotype_file): + if self.main_proc: + mutator_export, genotypes = self.mutator_small.export(self.logger) + with open(file, "w") as f: + json.dump(mutator_export, f, indent=2, sort_keys=True, cls=TorchTensorEncoder) + with open(genotype_file, "w") as f: + f.write(str(genotypes)) diff --git a/utils/third_party/nni_new/algorithms/nas/pytorch/cdarts/utils.py b/utils/third_party/nni_new/algorithms/nas/pytorch/cdarts/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..96afa9425633811327c158f54b8c63be95775455 --- /dev/null +++ b/utils/third_party/nni_new/algorithms/nas/pytorch/cdarts/utils.py @@ -0,0 +1,76 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import json +import os + +import torch +import torch.distributed as dist + + +class CyclicIterator: + def __init__(self, loader, sampler, distributed): + self.loader = loader + self.sampler = sampler + self.epoch = 0 + self.distributed = distributed + self._next_epoch() + + def _next_epoch(self): + if self.distributed: + self.sampler.set_epoch(self.epoch) + self.iterator = iter(self.loader) + self.epoch += 1 + + def __len__(self): + return len(self.loader) + + def __iter__(self): + return self + + def __next__(self): + try: + return next(self.iterator) + except StopIteration: + self._next_epoch() + return next(self.iterator) + + +class TorchTensorEncoder(json.JSONEncoder): + def default(self, o): # pylint: disable=method-hidden + if isinstance(o, torch.Tensor): + return o.tolist() + return super().default(o) + + +def accuracy(output, target, topk=(1,)): + """ Computes the precision@k for the specified values of k """ + maxk = max(topk) + batch_size = target.size(0) + + _, pred = output.topk(maxk, 1, True, True) + pred = pred.t() + # one-hot case + if target.ndimension() > 1: + target = target.max(1)[1] + + correct = pred.eq(target.view(1, -1).expand_as(pred)) + + res = [] + for k in topk: + correct_k = correct[:k].reshape(-1).float().sum(0) + res.append(correct_k.mul_(1.0 / batch_size)) + return res + + +def reduce_tensor(tensor): + rt = tensor.clone() + dist.all_reduce(rt, op=dist.ReduceOp.SUM) + rt /= float(os.environ["WORLD_SIZE"]) + return rt + + +def reduce_metrics(metrics, distributed=False): + if distributed: + return {k: reduce_tensor(v).item() for k, v in metrics.items()} + return {k: v.item() for k, v in metrics.items()} diff --git a/utils/third_party/nni_new/algorithms/nas/pytorch/classic_nas/__init__.py b/utils/third_party/nni_new/algorithms/nas/pytorch/classic_nas/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..ec3f5a4894a0460d4a0582a0d6cd43af9bed77e2 --- /dev/null +++ b/utils/third_party/nni_new/algorithms/nas/pytorch/classic_nas/__init__.py @@ -0,0 +1,4 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .mutator import get_and_apply_next_architecture diff --git a/utils/third_party/nni_new/algorithms/nas/pytorch/classic_nas/mutator.py b/utils/third_party/nni_new/algorithms/nas/pytorch/classic_nas/mutator.py new file mode 100644 index 0000000000000000000000000000000000000000..7254a8b0b4a34b5913dfc71d11f7364b86fc04b8 --- /dev/null +++ b/utils/third_party/nni_new/algorithms/nas/pytorch/classic_nas/mutator.py @@ -0,0 +1,221 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import json +import logging +import os +import sys + +import torch + +import nni +from nni.runtime.env_vars import trial_env_vars +from nni.nas.pytorch.mutables import LayerChoice, InputChoice, MutableScope +from nni.nas.pytorch.mutator import Mutator + +logger = logging.getLogger(__name__) + +NNI_GEN_SEARCH_SPACE = "NNI_GEN_SEARCH_SPACE" +LAYER_CHOICE = "layer_choice" +INPUT_CHOICE = "input_choice" + + +def get_and_apply_next_architecture(model): + """ + Wrapper of :class:`~nni.nas.pytorch.classic_nas.mutator.ClassicMutator` to make it more meaningful, + similar to ``get_next_parameter`` for HPO. + + It will generate search space based on ``model``. + If env ``NNI_GEN_SEARCH_SPACE`` exists, this is in dry run mode for + generating search space for the experiment. + If not, there are still two mode, one is nni experiment mode where users + use ``nnictl`` to start an experiment. The other is standalone mode + where users directly run the trial command, this mode chooses the first + one(s) for each LayerChoice and InputChoice. + + Parameters + ---------- + model : nn.Module + User's model with search space (e.g., LayerChoice, InputChoice) embedded in it. + """ + ClassicMutator(model) + + +class ClassicMutator(Mutator): + """ + This mutator is to apply the architecture chosen from tuner. + It implements the forward function of LayerChoice and InputChoice, + to only activate the chosen ones. + + Parameters + ---------- + model : nn.Module + User's model with search space (e.g., LayerChoice, InputChoice) embedded in it. + """ + + def __init__(self, model): + super(ClassicMutator, self).__init__(model) + self._chosen_arch = {} + self._search_space = self._generate_search_space() + if NNI_GEN_SEARCH_SPACE in os.environ: + # dry run for only generating search space + self._dump_search_space(os.environ[NNI_GEN_SEARCH_SPACE]) + sys.exit(0) + + if trial_env_vars.NNI_PLATFORM is None: + logger.warning("This is in standalone mode, the chosen are the first one(s).") + self._chosen_arch = self._standalone_generate_chosen() + else: + # get chosen arch from tuner + self._chosen_arch = nni.get_next_parameter() + if self._chosen_arch is None: + if trial_env_vars.NNI_PLATFORM == "unittest": + # happens if NNI_PLATFORM is intentionally set, e.g., in UT + logger.warning("`NNI_PLATFORM` is set but `param` is None. Falling back to standalone mode.") + self._chosen_arch = self._standalone_generate_chosen() + else: + raise RuntimeError("Chosen architecture is None. This may be a platform error.") + self.reset() + + def _sample_layer_choice(self, mutable, idx, value, search_space_item): + """ + Convert layer choice to tensor representation. + + Parameters + ---------- + mutable : Mutable + idx : int + Number `idx` of list will be selected. + value : str + The verbose representation of the selected value. + search_space_item : list + The list for corresponding search space. + """ + # doesn't support multihot for layer choice yet + onehot_list = [False] * len(mutable) + assert 0 <= idx < len(mutable) and search_space_item[idx] == value, \ + "Index '{}' in search space '{}' is not '{}'".format(idx, search_space_item, value) + onehot_list[idx] = True + return torch.tensor(onehot_list, dtype=torch.bool) # pylint: disable=not-callable + + def _sample_input_choice(self, mutable, idx, value, search_space_item): + """ + Convert input choice to tensor representation. + + Parameters + ---------- + mutable : Mutable + idx : int + Number `idx` of list will be selected. + value : str + The verbose representation of the selected value. + search_space_item : list + The list for corresponding search space. + """ + candidate_repr = search_space_item["candidates"] + multihot_list = [False] * mutable.n_candidates + for i, v in zip(idx, value): + assert 0 <= i < mutable.n_candidates and candidate_repr[i] == v, \ + "Index '{}' in search space '{}' is not '{}'".format(i, candidate_repr, v) + assert not multihot_list[i], "'{}' is selected twice in '{}', which is not allowed.".format(i, idx) + multihot_list[i] = True + return torch.tensor(multihot_list, dtype=torch.bool) # pylint: disable=not-callable + + def sample_search(self): + """ + See :meth:`sample_final`. + """ + return self.sample_final() + + def sample_final(self): + """ + Convert the chosen arch and apply it on model. + """ + assert set(self._chosen_arch.keys()) == set(self._search_space.keys()), \ + "Unmatched keys, expected keys '{}' from search space, found '{}'.".format(self._search_space.keys(), + self._chosen_arch.keys()) + result = dict() + for mutable in self.mutables: + if isinstance(mutable, (LayerChoice, InputChoice)): + assert mutable.key in self._chosen_arch, \ + "Expected '{}' in chosen arch, but not found.".format(mutable.key) + data = self._chosen_arch[mutable.key] + assert isinstance(data, dict) and "_value" in data and "_idx" in data, \ + "'{}' is not a valid choice.".format(data) + if isinstance(mutable, LayerChoice): + result[mutable.key] = self._sample_layer_choice(mutable, data["_idx"], data["_value"], + self._search_space[mutable.key]["_value"]) + elif isinstance(mutable, InputChoice): + result[mutable.key] = self._sample_input_choice(mutable, data["_idx"], data["_value"], + self._search_space[mutable.key]["_value"]) + elif isinstance(mutable, MutableScope): + logger.info("Mutable scope '%s' is skipped during parsing choices.", mutable.key) + else: + raise TypeError("Unsupported mutable type: '%s'." % type(mutable)) + return result + + def _standalone_generate_chosen(self): + """ + Generate the chosen architecture for standalone mode, + i.e., choose the first one(s) for LayerChoice and InputChoice. + :: + { key_name: {"_value": "conv1", + "_idx": 0} } + { key_name: {"_value": ["in1"], + "_idx": [0]} } + Returns + ------- + dict + the chosen architecture + """ + chosen_arch = {} + for key, val in self._search_space.items(): + if val["_type"] == LAYER_CHOICE: + choices = val["_value"] + chosen_arch[key] = {"_value": choices[0], "_idx": 0} + elif val["_type"] == INPUT_CHOICE: + choices = val["_value"]["candidates"] + n_chosen = val["_value"]["n_chosen"] + if n_chosen is None: + n_chosen = len(choices) + chosen_arch[key] = {"_value": choices[:n_chosen], "_idx": list(range(n_chosen))} + else: + raise ValueError("Unknown key '%s' and value '%s'." % (key, val)) + return chosen_arch + + def _generate_search_space(self): + """ + Generate search space from mutables. + Here is the search space format: + :: + { key_name: {"_type": "layer_choice", + "_value": ["conv1", "conv2"]} } + { key_name: {"_type": "input_choice", + "_value": {"candidates": ["in1", "in2"], + "n_chosen": 1}} } + Returns + ------- + dict + the generated search space + """ + search_space = {} + for mutable in self.mutables: + # for now we only generate flattened search space + if isinstance(mutable, LayerChoice): + key = mutable.key + val = mutable.names + search_space[key] = {"_type": LAYER_CHOICE, "_value": val} + elif isinstance(mutable, InputChoice): + key = mutable.key + search_space[key] = {"_type": INPUT_CHOICE, + "_value": {"candidates": mutable.choose_from, + "n_chosen": mutable.n_chosen}} + elif isinstance(mutable, MutableScope): + logger.info("Mutable scope '%s' is skipped during generating search space.", mutable.key) + else: + raise TypeError("Unsupported mutable type: '%s'." % type(mutable)) + return search_space + + def _dump_search_space(self, file_path): + with open(file_path, "w") as ss_file: + json.dump(self._search_space, ss_file, sort_keys=True, indent=2) diff --git a/utils/third_party/nni_new/algorithms/nas/pytorch/cream/__init__.py b/utils/third_party/nni_new/algorithms/nas/pytorch/cream/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..43a038b4670c0e3e5adeb1d34a4ee025970a8b19 --- /dev/null +++ b/utils/third_party/nni_new/algorithms/nas/pytorch/cream/__init__.py @@ -0,0 +1,4 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .trainer import CreamSupernetTrainer diff --git a/utils/third_party/nni_new/algorithms/nas/pytorch/cream/trainer.py b/utils/third_party/nni_new/algorithms/nas/pytorch/cream/trainer.py new file mode 100644 index 0000000000000000000000000000000000000000..50830ce64b1f458c299df755159f91747f8eefe7 --- /dev/null +++ b/utils/third_party/nni_new/algorithms/nas/pytorch/cream/trainer.py @@ -0,0 +1,404 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +import torch +import logging + +from copy import deepcopy +from nni.nas.pytorch.trainer import Trainer +from nni.nas.pytorch.utils import AverageMeterGroup + +from .utils import accuracy, reduce_metrics + +logger = logging.getLogger(__name__) + + +class CreamSupernetTrainer(Trainer): + """ + This trainer trains a supernet and output prioritized architectures that can be used for other tasks. + + Parameters + ---------- + model : nn.Module + Model with mutables. + loss : callable + Called with logits and targets. Returns a loss tensor. + val_loss : callable + Called with logits and targets for validation only. Returns a loss tensor. + optimizer : Optimizer + Optimizer that optimizes the model. + num_epochs : int + Number of epochs of training. + train_loader : iterablez + Data loader of training. Raise ``StopIteration`` when one epoch is exhausted. + valid_loader : iterablez + Data loader of validation. Raise ``StopIteration`` when one epoch is exhausted. + mutator : Mutator + A mutator object that has been initialized with the model. + batch_size : int + Batch size. + log_frequency : int + Number of mini-batches to log metrics. + meta_sta_epoch : int + start epoch of using meta matching network to pick teacher architecture + update_iter : int + interval of updating meta matching networks + slices : int + batch size of mini training data in the process of training meta matching network + pool_size : int + board size + pick_method : basestring + how to pick teacher network + choice_num : int + number of operations in supernet + sta_num : int + layer number of each stage in supernet (5 stage in supernet) + acc_gap : int + maximum accuracy improvement to omit the limitation of flops + flops_dict : Dict + dictionary of each layer's operations in supernet + flops_fixed : int + flops of fixed part in supernet + local_rank : int + index of current rank + callbacks : list of Callback + Callbacks to plug into the trainer. See Callbacks. + """ + + def __init__(self, model, loss, val_loss, + optimizer, num_epochs, train_loader, valid_loader, + mutator=None, batch_size=64, log_frequency=None, + meta_sta_epoch=20, update_iter=200, slices=2, + pool_size=10, pick_method='meta', choice_num=6, + sta_num=(4, 4, 4, 4, 4), acc_gap=5, + flops_dict=None, flops_fixed=0, local_rank=0, callbacks=None): + assert torch.cuda.is_available() + super(CreamSupernetTrainer, self).__init__(model, mutator, loss, None, + optimizer, num_epochs, None, None, + batch_size, None, None, log_frequency, callbacks) + self.model = model + self.loss = loss + self.val_loss = val_loss + self.train_loader = train_loader + self.valid_loader = valid_loader + self.log_frequency = log_frequency + self.batch_size = batch_size + self.optimizer = optimizer + self.model = model + self.loss = loss + self.num_epochs = num_epochs + self.meta_sta_epoch = meta_sta_epoch + self.update_iter = update_iter + self.slices = slices + self.pick_method = pick_method + self.pool_size = pool_size + self.local_rank = local_rank + self.choice_num = choice_num + self.sta_num = sta_num + self.acc_gap = acc_gap + self.flops_dict = flops_dict + self.flops_fixed = flops_fixed + + self.current_student_arch = None + self.current_teacher_arch = None + self.main_proc = (local_rank == 0) + self.current_epoch = 0 + + self.prioritized_board = [] + + # size of prioritized board + def _board_size(self): + return len(self.prioritized_board) + + # select teacher architecture according to the logit difference + def _select_teacher(self): + self._replace_mutator_cand(self.current_student_arch) + + if self.pick_method == 'top1': + meta_value, teacher_cand = 0.5, sorted( + self.prioritized_board, reverse=True)[0][3] + elif self.pick_method == 'meta': + meta_value, cand_idx, teacher_cand = -1000000000, -1, None + for now_idx, item in enumerate(self.prioritized_board): + inputx = item[4] + output = torch.nn.functional.softmax(self.model(inputx), dim=1) + weight = self.model.module.forward_meta(output - item[5]) + if weight > meta_value: + meta_value = weight + cand_idx = now_idx + teacher_cand = self.prioritized_board[cand_idx][3] + assert teacher_cand is not None + meta_value = torch.nn.functional.sigmoid(-weight) + else: + raise ValueError('Method Not supported') + + return meta_value, teacher_cand + + # check whether to update prioritized board + def _isUpdateBoard(self, prec1, flops): + if self.current_epoch <= self.meta_sta_epoch: + return False + + if len(self.prioritized_board) < self.pool_size: + return True + + if prec1 > self.prioritized_board[-1][1] + self.acc_gap: + return True + + if prec1 > self.prioritized_board[-1][1] and flops < self.prioritized_board[-1][2]: + return True + + return False + + # update prioritized board + def _update_prioritized_board(self, inputs, teacher_output, outputs, prec1, flops): + if self._isUpdateBoard(prec1, flops): + val_prec1 = prec1 + training_data = deepcopy(inputs[:self.slices].detach()) + if len(self.prioritized_board) == 0: + features = deepcopy(outputs[:self.slices].detach()) + else: + features = deepcopy( + teacher_output[:self.slices].detach()) + self.prioritized_board.append( + (val_prec1, + prec1, + flops, + self.current_student_arch, + training_data, + torch.nn.functional.softmax( + features, + dim=1))) + self.prioritized_board = sorted( + self.prioritized_board, reverse=True) + + if len(self.prioritized_board) > self.pool_size: + del self.prioritized_board[-1] + + # only update student network weights + def _update_student_weights_only(self, grad_1): + for weight, grad_item in zip( + self.model.module.rand_parameters(self.current_student_arch), grad_1): + weight.grad = grad_item + torch.nn.utils.clip_grad_norm_( + self.model.module.rand_parameters(self.current_student_arch), 1) + self.optimizer.step() + for weight, grad_item in zip( + self.model.module.rand_parameters(self.current_student_arch), grad_1): + del weight.grad + + # only update meta networks weights + def _update_meta_weights_only(self, teacher_cand, grad_teacher): + for weight, grad_item in zip(self.model.module.rand_parameters( + teacher_cand, self.pick_method == 'meta'), grad_teacher): + weight.grad = grad_item + + # clip gradients + torch.nn.utils.clip_grad_norm_( + self.model.module.rand_parameters( + self.current_student_arch, self.pick_method == 'meta'), 1) + + self.optimizer.step() + for weight, grad_item in zip(self.model.module.rand_parameters( + teacher_cand, self.pick_method == 'meta'), grad_teacher): + del weight.grad + + # simulate sgd updating + def _simulate_sgd_update(self, w, g, optimizer): + return g * optimizer.param_groups[-1]['lr'] + w + + # split training images into several slices + def _get_minibatch_input(self, input): + slice = self.slices + x = deepcopy(input[:slice].clone().detach()) + return x + + # calculate 1st gradient of student architectures + def _calculate_1st_gradient(self, kd_loss): + self.optimizer.zero_grad() + grad = torch.autograd.grad( + kd_loss, + self.model.module.rand_parameters(self.current_student_arch), + create_graph=True) + return grad + + # calculate 2nd gradient of meta networks + def _calculate_2nd_gradient(self, validation_loss, teacher_cand, students_weight): + self.optimizer.zero_grad() + grad_student_val = torch.autograd.grad( + validation_loss, + self.model.module.rand_parameters(self.current_student_arch), + retain_graph=True) + + grad_teacher = torch.autograd.grad( + students_weight[0], + self.model.module.rand_parameters( + teacher_cand, + self.pick_method == 'meta'), + grad_outputs=grad_student_val) + return grad_teacher + + # forward training data + def _forward_training(self, x, meta_value): + self._replace_mutator_cand(self.current_student_arch) + output = self.model(x) + + with torch.no_grad(): + self._replace_mutator_cand(self.current_teacher_arch) + teacher_output = self.model(x) + soft_label = torch.nn.functional.softmax(teacher_output, dim=1) + + kd_loss = meta_value * \ + self._cross_entropy_loss_with_soft_target(output, soft_label) + return kd_loss + + # calculate soft target loss + def _cross_entropy_loss_with_soft_target(self, pred, soft_target): + logsoftmax = torch.nn.LogSoftmax() + return torch.mean(torch.sum(- soft_target * logsoftmax(pred), 1)) + + # forward validation data + def _forward_validation(self, input, target): + slice = self.slices + x = input[slice:slice * 2].clone() + + self._replace_mutator_cand(self.current_student_arch) + output_2 = self.model(x) + + validation_loss = self.loss(output_2, target[slice:slice * 2]) + return validation_loss + + def _isUpdateMeta(self, batch_idx): + isUpdate = True + isUpdate &= (self.current_epoch > self.meta_sta_epoch) + isUpdate &= (batch_idx > 0) + isUpdate &= (batch_idx % self.update_iter == 0) + isUpdate &= (self._board_size() > 0) + return isUpdate + + def _replace_mutator_cand(self, cand): + self.mutator._cache = cand + + # update meta matching networks + def _run_update(self, input, target, batch_idx): + if self._isUpdateMeta(batch_idx): + x = self._get_minibatch_input(input) + + meta_value, teacher_cand = self._select_teacher() + + kd_loss = self._forward_training(x, meta_value) + + # calculate 1st gradient + grad_1st = self._calculate_1st_gradient(kd_loss) + + # simulate updated student weights + students_weight = [ + self._simulate_sgd_update( + p, grad_item, self.optimizer) for p, grad_item in zip( + self.model.module.rand_parameters(self.current_student_arch), grad_1st)] + + # update student weights + self._update_student_weights_only(grad_1st) + + validation_loss = self._forward_validation(input, target) + + # calculate 2nd gradient + grad_teacher = self._calculate_2nd_gradient(validation_loss, teacher_cand, students_weight) + + # update meta matching networks + self._update_meta_weights_only(teacher_cand, grad_teacher) + + # delete internal variants + del grad_teacher, grad_1st, x, validation_loss, kd_loss, students_weight + + def _get_cand_flops(self, cand): + flops = 0 + for block_id, block in enumerate(cand): + if block == 'LayerChoice1' or block_id == 'LayerChoice23': + continue + for idx, choice in enumerate(cand[block]): + flops += self.flops_dict[block_id][idx] * (1 if choice else 0) + return flops + self.flops_fixed + + def train_one_epoch(self, epoch): + self.current_epoch = epoch + meters = AverageMeterGroup() + self.steps_per_epoch = len(self.train_loader) + for step, (input_data, target) in enumerate(self.train_loader): + self.mutator.reset() + self.current_student_arch = self.mutator._cache + + input_data, target = input_data.cuda(), target.cuda() + + # calculate flops of current architecture + cand_flops = self._get_cand_flops(self.mutator._cache) + + # update meta matching network + self._run_update(input_data, target, step) + + if self._board_size() > 0: + # select teacher architecture + meta_value, teacher_cand = self._select_teacher() + self.current_teacher_arch = teacher_cand + + # forward supernet + if self._board_size() == 0 or epoch <= self.meta_sta_epoch: + self._replace_mutator_cand(self.current_student_arch) + output = self.model(input_data) + + loss = self.loss(output, target) + kd_loss, teacher_output, teacher_cand = None, None, None + else: + self._replace_mutator_cand(self.current_student_arch) + output = self.model(input_data) + + gt_loss = self.loss(output, target) + + with torch.no_grad(): + self._replace_mutator_cand(self.current_teacher_arch) + teacher_output = self.model(input_data).detach() + + soft_label = torch.nn.functional.softmax(teacher_output, dim=1) + kd_loss = self._cross_entropy_loss_with_soft_target(output, soft_label) + + loss = (meta_value * kd_loss + (2 - meta_value) * gt_loss) / 2 + + # update network + self.optimizer.zero_grad() + loss.backward() + self.optimizer.step() + + # update metrics + prec1, prec5 = accuracy(output, target, topk=(1, 5)) + metrics = {"prec1": prec1, "prec5": prec5, "loss": loss} + metrics = reduce_metrics(metrics) + meters.update(metrics) + + # update prioritized board + self._update_prioritized_board(input_data, teacher_output, output, metrics['prec1'], cand_flops) + + if self.main_proc and (step % self.log_frequency == 0 or step + 1 == self.steps_per_epoch): + logger.info("Epoch [%d/%d] Step [%d/%d] %s", epoch + 1, self.num_epochs, + step + 1, len(self.train_loader), meters) + + if self.main_proc and self.num_epochs == epoch + 1: + for idx, i in enumerate(self.prioritized_board): + logger.info("No.%s %s", idx, i[:4]) + + def validate_one_epoch(self, epoch): + self.model.eval() + meters = AverageMeterGroup() + with torch.no_grad(): + for step, (x, y) in enumerate(self.valid_loader): + self.mutator.reset() + logits = self.model(x) + loss = self.val_loss(logits, y) + prec1, prec5 = accuracy(logits, y, topk=(1, 5)) + metrics = {"prec1": prec1, "prec5": prec5, "loss": loss} + metrics = reduce_metrics(metrics) + meters.update(metrics) + + if self.log_frequency is not None and step % self.log_frequency == 0: + logger.info("Epoch [%s/%s] Validation Step [%s/%s] %s", epoch + 1, + self.num_epochs, step + 1, len(self.valid_loader), meters) diff --git a/utils/third_party/nni_new/algorithms/nas/pytorch/cream/utils.py b/utils/third_party/nni_new/algorithms/nas/pytorch/cream/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7d71faa715b379616c1affa6aaca319977435e65 --- /dev/null +++ b/utils/third_party/nni_new/algorithms/nas/pytorch/cream/utils.py @@ -0,0 +1,37 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + + +import os +import torch.distributed as dist + + +def accuracy(output, target, topk=(1,)): + """ Computes the precision@k for the specified values of k """ + maxk = max(topk) + batch_size = target.size(0) + + _, pred = output.topk(maxk, 1, True, True) + pred = pred.t() + # one-hot case + if target.ndimension() > 1: + target = target.max(1)[1] + + correct = pred.eq(target.reshape(1, -1).expand_as(pred)) + + res = [] + for k in topk: + correct_k = correct[:k].view(-1).float().sum(0) + res.append(correct_k.mul_(1.0 / batch_size)) + return res + + +def reduce_metrics(metrics): + return {k: reduce_tensor(v).item() for k, v in metrics.items()} + + +def reduce_tensor(tensor): + rt = tensor.clone() + dist.all_reduce(rt, op=dist.ReduceOp.SUM) + rt /= float(os.environ["WORLD_SIZE"]) + return rt diff --git a/utils/third_party/nni_new/algorithms/nas/pytorch/darts/__init__.py b/utils/third_party/nni_new/algorithms/nas/pytorch/darts/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..1a22790fb90b37591dd58f590a3b528b78b8257e --- /dev/null +++ b/utils/third_party/nni_new/algorithms/nas/pytorch/darts/__init__.py @@ -0,0 +1,5 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .mutator import DartsMutator +from .trainer import DartsTrainer diff --git a/utils/third_party/nni_new/algorithms/nas/pytorch/darts/mutator.py b/utils/third_party/nni_new/algorithms/nas/pytorch/darts/mutator.py new file mode 100644 index 0000000000000000000000000000000000000000..a4c3898a9b531ea160ea64a17d2e345f6b67d040 --- /dev/null +++ b/utils/third_party/nni_new/algorithms/nas/pytorch/darts/mutator.py @@ -0,0 +1,85 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from nni.nas.pytorch.mutator import Mutator +from nni.nas.pytorch.mutables import LayerChoice, InputChoice + +_logger = logging.getLogger(__name__) + + +class DartsMutator(Mutator): + """ + Connects the model in a DARTS (differentiable) way. + + An extra connection is automatically inserted for each LayerChoice, when this connection is selected, there is no + op on this LayerChoice (namely a ``ZeroOp``), in which case, every element in the exported choice list is ``false`` + (not chosen). + + All input choice will be fully connected in the search phase. On exporting, the input choice will choose inputs based + on keys in ``choose_from``. If the keys were to be keys of LayerChoices, the top logit of the corresponding LayerChoice + will join the competition of input choice to compete against other logits. Otherwise, the logit will be assumed 0. + + It's possible to cut branches by setting parameter ``choices`` in a particular position to ``-inf``. After softmax, the + value would be 0. Framework will ignore 0 values and not connect. Note that the gradient on the ``-inf`` location will + be 0. Since manipulations with ``-inf`` will be ``nan``, you need to handle the gradient update phase carefully. + + Attributes + ---------- + choices: ParameterDict + dict that maps keys of LayerChoices to weighted-connection float tensors. + """ + def __init__(self, model): + super().__init__(model) + self.choices = nn.ParameterDict() + for mutable in self.mutables: + if isinstance(mutable, LayerChoice): + self.choices[mutable.key] = nn.Parameter(1.0E-3 * torch.randn(mutable.length + 1)) + + def device(self): + for v in self.choices.values(): + return v.device + + def sample_search(self): + result = dict() + for mutable in self.mutables: + if isinstance(mutable, LayerChoice): + result[mutable.key] = F.softmax(self.choices[mutable.key], dim=-1)[:-1] + elif isinstance(mutable, InputChoice): + result[mutable.key] = torch.ones(mutable.n_candidates, dtype=torch.bool, device=self.device()) + return result + + def sample_final(self): + result = dict() + edges_max = dict() + for mutable in self.mutables: + if isinstance(mutable, LayerChoice): + max_val, index = torch.max(F.softmax(self.choices[mutable.key], dim=-1)[:-1], 0) + edges_max[mutable.key] = max_val + result[mutable.key] = F.one_hot(index, num_classes=len(mutable)).view(-1).bool() + for mutable in self.mutables: + if isinstance(mutable, InputChoice): + if mutable.n_chosen is not None: + weights = [] + for src_key in mutable.choose_from: + if src_key not in edges_max: + _logger.warning("InputChoice.NO_KEY in '%s' is weighted 0 when selecting inputs.", mutable.key) + weights.append(edges_max.get(src_key, 0.)) + weights = torch.tensor(weights) # pylint: disable=not-callable + _, topk_edge_indices = torch.topk(weights, mutable.n_chosen) + selected_multihot = [] + for i, src_key in enumerate(mutable.choose_from): + if i not in topk_edge_indices and src_key in result: + # If an edge is never selected, there is no need to calculate any op on this edge. + # This is to eliminate redundant calculation. + result[src_key] = torch.zeros_like(result[src_key]) + selected_multihot.append(i in topk_edge_indices) + result[mutable.key] = torch.tensor(selected_multihot, dtype=torch.bool, device=self.device()) # pylint: disable=not-callable + else: + result[mutable.key] = torch.ones(mutable.n_candidates, dtype=torch.bool, device=self.device()) # pylint: disable=not-callable + return result diff --git a/utils/third_party/nni_new/algorithms/nas/pytorch/darts/trainer.py b/utils/third_party/nni_new/algorithms/nas/pytorch/darts/trainer.py new file mode 100644 index 0000000000000000000000000000000000000000..e2d8e1866b68ad259771140ab0a7b86c43185f67 --- /dev/null +++ b/utils/third_party/nni_new/algorithms/nas/pytorch/darts/trainer.py @@ -0,0 +1,214 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import copy +import logging + +import torch +import torch.nn as nn +from nni.nas.pytorch.trainer import Trainer +from nni.nas.pytorch.utils import AverageMeterGroup + +from .mutator import DartsMutator + +logger = logging.getLogger(__name__) + + +class DartsTrainer(Trainer): + """ + DARTS trainer. + + Parameters + ---------- + model : nn.Module + PyTorch model to be trained. + loss : callable + Receives logits and ground truth label, return a loss tensor. + metrics : callable + Receives logits and ground truth label, return a dict of metrics. + optimizer : Optimizer + The optimizer used for optimizing the model. + num_epochs : int + Number of epochs planned for training. + dataset_train : Dataset + Dataset for training. Will be split for training weights and architecture weights. + dataset_valid : Dataset + Dataset for testing. + mutator : DartsMutator + Use in case of customizing your own DartsMutator. By default will instantiate a DartsMutator. + batch_size : int + Batch size. + workers : int + Workers for data loading. + device : torch.device + ``torch.device("cpu")`` or ``torch.device("cuda")``. + log_frequency : int + Step count per logging. + callbacks : list of Callback + list of callbacks to trigger at events. + arc_learning_rate : float + Learning rate of architecture parameters. + unrolled : float + ``True`` if using second order optimization, else first order optimization. + """ + def __init__(self, model, loss, metrics, + optimizer, num_epochs, dataset_train, dataset_valid, + mutator=None, batch_size=64, workers=4, device=None, log_frequency=None, + callbacks=None, arc_learning_rate=3.0E-4, unrolled=False): + super().__init__(model, mutator if mutator is not None else DartsMutator(model), + loss, metrics, optimizer, num_epochs, dataset_train, dataset_valid, + batch_size, workers, device, log_frequency, callbacks) + + self.ctrl_optim = torch.optim.Adam(self.mutator.parameters(), arc_learning_rate, betas=(0.5, 0.999), + weight_decay=1.0E-3) + self.unrolled = unrolled + + n_train = len(self.dataset_train) + split = n_train // 2 + indices = list(range(n_train)) + train_sampler = torch.utils.data.sampler.SubsetRandomSampler(indices[:split]) + valid_sampler = torch.utils.data.sampler.SubsetRandomSampler(indices[split:]) + self.train_loader = torch.utils.data.DataLoader(self.dataset_train, + batch_size=batch_size, + sampler=train_sampler, + num_workers=workers) + self.valid_loader = torch.utils.data.DataLoader(self.dataset_train, + batch_size=batch_size, + sampler=valid_sampler, + num_workers=workers) + self.test_loader = torch.utils.data.DataLoader(self.dataset_valid, + batch_size=batch_size, + num_workers=workers) + + def train_one_epoch(self, epoch): + self.model.train() + self.mutator.train() + meters = AverageMeterGroup() + for step, ((trn_X, trn_y), (val_X, val_y)) in enumerate(zip(self.train_loader, self.valid_loader)): + trn_X, trn_y = trn_X.to(self.device), trn_y.to(self.device) + val_X, val_y = val_X.to(self.device), val_y.to(self.device) + + # phase 1. architecture step + self.ctrl_optim.zero_grad() + if self.unrolled: + self._unrolled_backward(trn_X, trn_y, val_X, val_y) + else: + self._backward(val_X, val_y) + self.ctrl_optim.step() + + # phase 2: child network step + self.optimizer.zero_grad() + logits, loss = self._logits_and_loss(trn_X, trn_y) + loss.backward() + nn.utils.clip_grad_norm_(self.model.parameters(), 5.) # gradient clipping + self.optimizer.step() + + metrics = self.metrics(logits, trn_y) + metrics["loss"] = loss.item() + meters.update(metrics) + if self.log_frequency is not None and step % self.log_frequency == 0: + logger.info("Epoch [%s/%s] Step [%s/%s] %s", epoch + 1, + self.num_epochs, step + 1, len(self.train_loader), meters) + + def validate_one_epoch(self, epoch): + self.model.eval() + self.mutator.eval() + meters = AverageMeterGroup() + with torch.no_grad(): + self.mutator.reset() + for step, (X, y) in enumerate(self.test_loader): + X, y = X.to(self.device), y.to(self.device) + logits = self.model(X) + metrics = self.metrics(logits, y) + meters.update(metrics) + if self.log_frequency is not None and step % self.log_frequency == 0: + logger.info("Epoch [%s/%s] Step [%s/%s] %s", epoch + 1, + self.num_epochs, step + 1, len(self.test_loader), meters) + + def _logits_and_loss(self, X, y): + self.mutator.reset() + logits = self.model(X) + loss = self.loss(logits, y) + self._write_graph_status() + return logits, loss + + def _backward(self, val_X, val_y): + """ + Simple backward with gradient descent + """ + _, loss = self._logits_and_loss(val_X, val_y) + loss.backward() + + def _unrolled_backward(self, trn_X, trn_y, val_X, val_y): + """ + Compute unrolled loss and backward its gradients + """ + backup_params = copy.deepcopy(tuple(self.model.parameters())) + + # do virtual step on training data + lr = self.optimizer.param_groups[0]["lr"] + momentum = self.optimizer.param_groups[0]["momentum"] + weight_decay = self.optimizer.param_groups[0]["weight_decay"] + self._compute_virtual_model(trn_X, trn_y, lr, momentum, weight_decay) + + # calculate unrolled loss on validation data + # keep gradients for model here for compute hessian + _, loss = self._logits_and_loss(val_X, val_y) + w_model, w_ctrl = tuple(self.model.parameters()), tuple(self.mutator.parameters()) + w_grads = torch.autograd.grad(loss, w_model + w_ctrl) + d_model, d_ctrl = w_grads[:len(w_model)], w_grads[len(w_model):] + + # compute hessian and final gradients + hessian = self._compute_hessian(backup_params, d_model, trn_X, trn_y) + with torch.no_grad(): + for param, d, h in zip(w_ctrl, d_ctrl, hessian): + # gradient = dalpha - lr * hessian + param.grad = d - lr * h + + # restore weights + self._restore_weights(backup_params) + + def _compute_virtual_model(self, X, y, lr, momentum, weight_decay): + """ + Compute unrolled weights w` + """ + # don't need zero_grad, using autograd to calculate gradients + _, loss = self._logits_and_loss(X, y) + gradients = torch.autograd.grad(loss, self.model.parameters()) + with torch.no_grad(): + for w, g in zip(self.model.parameters(), gradients): + m = self.optimizer.state[w].get("momentum_buffer", 0.) + w = w - lr * (momentum * m + g + weight_decay * w) + + def _restore_weights(self, backup_params): + with torch.no_grad(): + for param, backup in zip(self.model.parameters(), backup_params): + param.copy_(backup) + + def _compute_hessian(self, backup_params, dw, trn_X, trn_y): + """ + dw = dw` { L_val(w`, alpha) } + w+ = w + eps * dw + w- = w - eps * dw + hessian = (dalpha { L_trn(w+, alpha) } - dalpha { L_trn(w-, alpha) }) / (2*eps) + eps = 0.01 / ||dw|| + """ + self._restore_weights(backup_params) + norm = torch.cat([w.view(-1) for w in dw]).norm() + eps = 0.01 / norm + if norm < 1E-8: + logger.warning("In computing hessian, norm is smaller than 1E-8, cause eps to be %.6f.", norm.item()) + + dalphas = [] + for e in [eps, -2. * eps]: + # w+ = w + eps*dw`, w- = w - eps*dw` + with torch.no_grad(): + for p, d in zip(self.model.parameters(), dw): + p += e * d + + _, loss = self._logits_and_loss(trn_X, trn_y) + dalphas.append(torch.autograd.grad(loss, self.mutator.parameters())) + + dalpha_pos, dalpha_neg = dalphas # dalpha { L_trn(w+) }, # dalpha { L_trn(w-) } + hessian = [(p - n) / (2. * eps) for p, n in zip(dalpha_pos, dalpha_neg)] + return hessian diff --git a/utils/third_party/nni_new/algorithms/nas/pytorch/enas/__init__.py b/utils/third_party/nni_new/algorithms/nas/pytorch/enas/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d3372836ebba2387ade58161218aad731433e46b --- /dev/null +++ b/utils/third_party/nni_new/algorithms/nas/pytorch/enas/__init__.py @@ -0,0 +1,5 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .mutator import EnasMutator +from .trainer import EnasTrainer diff --git a/utils/third_party/nni_new/algorithms/nas/pytorch/enas/mutator.py b/utils/third_party/nni_new/algorithms/nas/pytorch/enas/mutator.py new file mode 100644 index 0000000000000000000000000000000000000000..7fdba26b99bf72551293fdb77ff61beb974bba8f --- /dev/null +++ b/utils/third_party/nni_new/algorithms/nas/pytorch/enas/mutator.py @@ -0,0 +1,197 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from nni.nas.pytorch.mutator import Mutator +from nni.nas.pytorch.mutables import LayerChoice, InputChoice, MutableScope + + +class StackedLSTMCell(nn.Module): + def __init__(self, layers, size, bias): + super().__init__() + self.lstm_num_layers = layers + self.lstm_modules = nn.ModuleList([nn.LSTMCell(size, size, bias=bias) + for _ in range(self.lstm_num_layers)]) + + def forward(self, inputs, hidden): + prev_h, prev_c = hidden + next_h, next_c = [], [] + for i, m in enumerate(self.lstm_modules): + curr_h, curr_c = m(inputs, (prev_h[i], prev_c[i])) + next_c.append(curr_c) + next_h.append(curr_h) + # current implementation only supports batch size equals 1, + # but the algorithm does not necessarily have this limitation + inputs = curr_h[-1].view(1, -1) + return next_h, next_c + + +class EnasMutator(Mutator): + """ + A mutator that mutates the graph with RL. + + Parameters + ---------- + model : nn.Module + PyTorch model. + lstm_size : int + Controller LSTM hidden units. + lstm_num_layers : int + Number of layers for stacked LSTM. + tanh_constant : float + Logits will be equal to ``tanh_constant * tanh(logits)``. Don't use ``tanh`` if this value is ``None``. + cell_exit_extra_step : bool + If true, RL controller will perform an extra step at the exit of each MutableScope, dump the hidden state + and mark it as the hidden state of this MutableScope. This is to align with the original implementation of paper. + skip_target : float + Target probability that skipconnect will appear. + temperature : float + Temperature constant that divides the logits. + branch_bias : float + Manual bias applied to make some operations more likely to be chosen. + Currently this is implemented with a hardcoded match rule that aligns with original repo. + If a mutable has a ``reduce`` in its key, all its op choices + that contains `conv` in their typename will receive a bias of ``+self.branch_bias`` initially; while others + receive a bias of ``-self.branch_bias``. + entropy_reduction : str + Can be one of ``sum`` and ``mean``. How the entropy of multi-input-choice is reduced. + """ + + def __init__(self, model, lstm_size=64, lstm_num_layers=1, tanh_constant=1.5, cell_exit_extra_step=False, + skip_target=0.4, temperature=None, branch_bias=0.25, entropy_reduction="sum"): + super().__init__(model) + self.lstm_size = lstm_size + self.lstm_num_layers = lstm_num_layers + self.tanh_constant = tanh_constant + self.temperature = temperature + self.cell_exit_extra_step = cell_exit_extra_step + self.skip_target = skip_target + self.branch_bias = branch_bias + + self.lstm = StackedLSTMCell(self.lstm_num_layers, self.lstm_size, False) + self.attn_anchor = nn.Linear(self.lstm_size, self.lstm_size, bias=False) + self.attn_query = nn.Linear(self.lstm_size, self.lstm_size, bias=False) + self.v_attn = nn.Linear(self.lstm_size, 1, bias=False) + self.g_emb = nn.Parameter(torch.randn(1, self.lstm_size) * 0.1) + self.skip_targets = nn.Parameter(torch.tensor([1.0 - self.skip_target, self.skip_target]), requires_grad=False) # pylint: disable=not-callable + assert entropy_reduction in ["sum", "mean"], "Entropy reduction must be one of sum and mean." + self.entropy_reduction = torch.sum if entropy_reduction == "sum" else torch.mean + self.cross_entropy_loss = nn.CrossEntropyLoss(reduction="none") + self.bias_dict = nn.ParameterDict() + + self.max_layer_choice = 0 + for mutable in self.mutables: + if isinstance(mutable, LayerChoice): + if self.max_layer_choice == 0: + self.max_layer_choice = len(mutable) + assert self.max_layer_choice == len(mutable), \ + "ENAS mutator requires all layer choice have the same number of candidates." + # We are judging by keys and module types to add biases to layer choices. Needs refactor. + if "reduce" in mutable.key: + def is_conv(choice): + return "conv" in str(type(choice)).lower() + bias = torch.tensor([self.branch_bias if is_conv(choice) else -self.branch_bias # pylint: disable=not-callable + for choice in mutable]) + self.bias_dict[mutable.key] = nn.Parameter(bias, requires_grad=False) + + self.embedding = nn.Embedding(self.max_layer_choice + 1, self.lstm_size) + self.soft = nn.Linear(self.lstm_size, self.max_layer_choice, bias=False) + + def sample_search(self): + self._initialize() + self._sample(self.mutables) + return self._choices + + def sample_final(self): + return self.sample_search() + + def _sample(self, tree): + mutable = tree.mutable + if isinstance(mutable, LayerChoice) and mutable.key not in self._choices: + self._choices[mutable.key] = self._sample_layer_choice(mutable) + elif isinstance(mutable, InputChoice) and mutable.key not in self._choices: + self._choices[mutable.key] = self._sample_input_choice(mutable) + for child in tree.children: + self._sample(child) + if isinstance(mutable, MutableScope) and mutable.key not in self._anchors_hid: + if self.cell_exit_extra_step: + self._lstm_next_step() + self._mark_anchor(mutable.key) + + def _initialize(self): + self._choices = dict() + self._anchors_hid = dict() + self._inputs = self.g_emb.data + self._c = [torch.zeros((1, self.lstm_size), + dtype=self._inputs.dtype, + device=self._inputs.device) for _ in range(self.lstm_num_layers)] + self._h = [torch.zeros((1, self.lstm_size), + dtype=self._inputs.dtype, + device=self._inputs.device) for _ in range(self.lstm_num_layers)] + self.sample_log_prob = 0 + self.sample_entropy = 0 + self.sample_skip_penalty = 0 + + def _lstm_next_step(self): + self._h, self._c = self.lstm(self._inputs, (self._h, self._c)) + + def _mark_anchor(self, key): + self._anchors_hid[key] = self._h[-1] + + def _sample_layer_choice(self, mutable): + self._lstm_next_step() + logit = self.soft(self._h[-1]) + if self.temperature is not None: + logit /= self.temperature + if self.tanh_constant is not None: + logit = self.tanh_constant * torch.tanh(logit) + if mutable.key in self.bias_dict: + logit += self.bias_dict[mutable.key] + branch_id = torch.multinomial(F.softmax(logit, dim=-1), 1).view(-1) + log_prob = self.cross_entropy_loss(logit, branch_id) + self.sample_log_prob += self.entropy_reduction(log_prob) + entropy = (log_prob * torch.exp(-log_prob)).detach() # pylint: disable=invalid-unary-operand-type + self.sample_entropy += self.entropy_reduction(entropy) + self._inputs = self.embedding(branch_id) + return F.one_hot(branch_id, num_classes=self.max_layer_choice).bool().view(-1) + + def _sample_input_choice(self, mutable): + query, anchors = [], [] + for label in mutable.choose_from: + if label not in self._anchors_hid: + self._lstm_next_step() + self._mark_anchor(label) # empty loop, fill not found + query.append(self.attn_anchor(self._anchors_hid[label])) + anchors.append(self._anchors_hid[label]) + query = torch.cat(query, 0) + query = torch.tanh(query + self.attn_query(self._h[-1])) + query = self.v_attn(query) + if self.temperature is not None: + query /= self.temperature + if self.tanh_constant is not None: + query = self.tanh_constant * torch.tanh(query) + + if mutable.n_chosen is None: + logit = torch.cat([-query, query], 1) # pylint: disable=invalid-unary-operand-type + + skip = torch.multinomial(F.softmax(logit, dim=-1), 1).view(-1) + skip_prob = torch.sigmoid(logit) + kl = torch.sum(skip_prob * torch.log(skip_prob / self.skip_targets)) + self.sample_skip_penalty += kl + log_prob = self.cross_entropy_loss(logit, skip) + self._inputs = (torch.matmul(skip.float(), torch.cat(anchors, 0)) / (1. + torch.sum(skip))).unsqueeze(0) + else: + assert mutable.n_chosen == 1, "Input choice must select exactly one or any in ENAS." + logit = query.view(1, -1) + index = torch.multinomial(F.softmax(logit, dim=-1), 1).view(-1) + skip = F.one_hot(index, num_classes=mutable.n_candidates).view(-1) + log_prob = self.cross_entropy_loss(logit, index) + self._inputs = anchors[index.item()] + + self.sample_log_prob += self.entropy_reduction(log_prob) + entropy = (log_prob * torch.exp(-log_prob)).detach() # pylint: disable=invalid-unary-operand-type + self.sample_entropy += self.entropy_reduction(entropy) + return skip.bool() diff --git a/utils/third_party/nni_new/algorithms/nas/pytorch/enas/trainer.py b/utils/third_party/nni_new/algorithms/nas/pytorch/enas/trainer.py new file mode 100644 index 0000000000000000000000000000000000000000..5e7a966580a107ceb6b9fe88f888dafefb8b69e7 --- /dev/null +++ b/utils/third_party/nni_new/algorithms/nas/pytorch/enas/trainer.py @@ -0,0 +1,209 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +from itertools import cycle + +import torch +import torch.nn as nn +import torch.optim as optim + +from nni.nas.pytorch.trainer import Trainer +from nni.nas.pytorch.utils import AverageMeterGroup, to_device +from .mutator import EnasMutator + +logger = logging.getLogger(__name__) + + +class EnasTrainer(Trainer): + """ + ENAS trainer. + + Parameters + ---------- + model : nn.Module + PyTorch model to be trained. + loss : callable + Receives logits and ground truth label, return a loss tensor. + metrics : callable + Receives logits and ground truth label, return a dict of metrics. + reward_function : callable + Receives logits and ground truth label, return a tensor, which will be feeded to RL controller as reward. + optimizer : Optimizer + The optimizer used for optimizing the model. + num_epochs : int + Number of epochs planned for training. + dataset_train : Dataset + Dataset for training. Will be split for training weights and architecture weights. + dataset_valid : Dataset + Dataset for testing. + mutator : EnasMutator + Use when customizing your own mutator or a mutator with customized parameters. + batch_size : int + Batch size. + workers : int + Workers for data loading. + device : torch.device + ``torch.device("cpu")`` or ``torch.device("cuda")``. + log_frequency : int + Step count per logging. + callbacks : list of Callback + list of callbacks to trigger at events. + entropy_weight : float + Weight of sample entropy loss. + skip_weight : float + Weight of skip penalty loss. + baseline_decay : float + Decay factor of baseline. New baseline will be equal to ``baseline_decay * baseline_old + reward * (1 - baseline_decay)``. + child_steps : int + How many mini-batches for model training per epoch. + mutator_lr : float + Learning rate for RL controller. + mutator_steps_aggregate : int + Number of steps that will be aggregated into one mini-batch for RL controller. + mutator_steps : int + Number of mini-batches for each epoch of RL controller learning. + aux_weight : float + Weight of auxiliary head loss. ``aux_weight * aux_loss`` will be added to total loss. + test_arc_per_epoch : int + How many architectures are chosen for direct test after each epoch. + """ + def __init__(self, model, loss, metrics, reward_function, + optimizer, num_epochs, dataset_train, dataset_valid, + mutator=None, batch_size=64, workers=4, device=None, log_frequency=None, callbacks=None, + entropy_weight=0.0001, skip_weight=0.8, baseline_decay=0.999, child_steps=500, + mutator_lr=0.00035, mutator_steps_aggregate=20, mutator_steps=50, aux_weight=0.4, + test_arc_per_epoch=1): + super().__init__(model, mutator if mutator is not None else EnasMutator(model), + loss, metrics, optimizer, num_epochs, dataset_train, dataset_valid, + batch_size, workers, device, log_frequency, callbacks) + self.reward_function = reward_function + self.mutator_optim = optim.Adam(self.mutator.parameters(), lr=mutator_lr) + self.batch_size = batch_size + self.workers = workers + + self.entropy_weight = entropy_weight + self.skip_weight = skip_weight + self.baseline_decay = baseline_decay + self.baseline = 0. + self.mutator_steps_aggregate = mutator_steps_aggregate + self.mutator_steps = mutator_steps + self.child_steps = child_steps + self.aux_weight = aux_weight + self.test_arc_per_epoch = test_arc_per_epoch + + self.init_dataloader() + + def init_dataloader(self): + n_train = len(self.dataset_train) + split = n_train // 10 + indices = list(range(n_train)) + train_sampler = torch.utils.data.sampler.SubsetRandomSampler(indices[:-split]) + valid_sampler = torch.utils.data.sampler.SubsetRandomSampler(indices[-split:]) + self.train_loader = torch.utils.data.DataLoader(self.dataset_train, + batch_size=self.batch_size, + sampler=train_sampler, + num_workers=self.workers) + self.valid_loader = torch.utils.data.DataLoader(self.dataset_train, + batch_size=self.batch_size, + sampler=valid_sampler, + num_workers=self.workers) + self.test_loader = torch.utils.data.DataLoader(self.dataset_valid, + batch_size=self.batch_size, + num_workers=self.workers) + self.train_loader = cycle(self.train_loader) + self.valid_loader = cycle(self.valid_loader) + + def train_one_epoch(self, epoch): + # Sample model and train + self.model.train() + self.mutator.eval() + meters = AverageMeterGroup() + for step in range(1, self.child_steps + 1): + x, y = next(self.train_loader) + x, y = to_device(x, self.device), to_device(y, self.device) + self.optimizer.zero_grad() + + with torch.no_grad(): + self.mutator.reset() + self._write_graph_status() + logits = self.model(x) + + if isinstance(logits, tuple): + logits, aux_logits = logits + aux_loss = self.loss(aux_logits, y) + else: + aux_loss = 0. + metrics = self.metrics(logits, y) + loss = self.loss(logits, y) + loss = loss + self.aux_weight * aux_loss + loss.backward() + nn.utils.clip_grad_norm_(self.model.parameters(), 5.) + self.optimizer.step() + metrics["loss"] = loss.item() + meters.update(metrics) + + if self.log_frequency is not None and step % self.log_frequency == 0: + logger.info("Model Epoch [%d/%d] Step [%d/%d] %s", epoch + 1, + self.num_epochs, step, self.child_steps, meters) + + # Train sampler (mutator) + self.model.eval() + self.mutator.train() + meters = AverageMeterGroup() + for mutator_step in range(1, self.mutator_steps + 1): + self.mutator_optim.zero_grad() + for step in range(1, self.mutator_steps_aggregate + 1): + x, y = next(self.valid_loader) + x, y = to_device(x, self.device), to_device(y, self.device) + + self.mutator.reset() + with torch.no_grad(): + logits = self.model(x) + self._write_graph_status() + metrics = self.metrics(logits, y) + reward = self.reward_function(logits, y) + if self.entropy_weight: + reward += self.entropy_weight * self.mutator.sample_entropy.item() + self.baseline = self.baseline * self.baseline_decay + reward * (1 - self.baseline_decay) + loss = self.mutator.sample_log_prob * (reward - self.baseline) + if self.skip_weight: + loss += self.skip_weight * self.mutator.sample_skip_penalty + metrics["reward"] = reward + metrics["loss"] = loss.item() + metrics["ent"] = self.mutator.sample_entropy.item() + metrics["log_prob"] = self.mutator.sample_log_prob.item() + metrics["baseline"] = self.baseline + metrics["skip"] = self.mutator.sample_skip_penalty + + loss /= self.mutator_steps_aggregate + loss.backward() + meters.update(metrics) + + cur_step = step + (mutator_step - 1) * self.mutator_steps_aggregate + if self.log_frequency is not None and cur_step % self.log_frequency == 0: + logger.info("RL Epoch [%d/%d] Step [%d/%d] [%d/%d] %s", epoch + 1, self.num_epochs, + mutator_step, self.mutator_steps, step, self.mutator_steps_aggregate, + meters) + + nn.utils.clip_grad_norm_(self.mutator.parameters(), 5.) + self.mutator_optim.step() + + def validate_one_epoch(self, epoch): + with torch.no_grad(): + for arc_id in range(self.test_arc_per_epoch): + meters = AverageMeterGroup() + for x, y in self.test_loader: + x, y = to_device(x, self.device), to_device(y, self.device) + self.mutator.reset() + logits = self.model(x) + if isinstance(logits, tuple): + logits, _ = logits + metrics = self.metrics(logits, y) + loss = self.loss(logits, y) + metrics["loss"] = loss.item() + meters.update(metrics) + + logger.info("Test Epoch [%d/%d] Arc [%d/%d] Summary %s", + epoch + 1, self.num_epochs, arc_id + 1, self.test_arc_per_epoch, + meters.summary()) diff --git a/utils/third_party/nni_new/algorithms/nas/pytorch/fbnet/__init__.py b/utils/third_party/nni_new/algorithms/nas/pytorch/fbnet/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..fa15cc64a77a6b7b4573d5a3bb03b9d6895d1f08 --- /dev/null +++ b/utils/third_party/nni_new/algorithms/nas/pytorch/fbnet/__init__.py @@ -0,0 +1,11 @@ +from __future__ import absolute_import + +from .mutator import FBNetMutator # noqa: F401 +from .trainer import FBNetTrainer # noqa: F401 +from .utils import ( # noqa: F401 + LookUpTable, + NASConfig, + RegularizerLoss, + model_init, + supernet_sample, +) diff --git a/utils/third_party/nni_new/algorithms/nas/pytorch/fbnet/mutator.py b/utils/third_party/nni_new/algorithms/nas/pytorch/fbnet/mutator.py new file mode 100644 index 0000000000000000000000000000000000000000..42b46afcabbcefde53af840e8ddbf822d088ccc9 --- /dev/null +++ b/utils/third_party/nni_new/algorithms/nas/pytorch/fbnet/mutator.py @@ -0,0 +1,268 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from __future__ import absolute_import, division, print_function + +import torch +from torch import nn as nn +from torch.nn import functional as F +import numpy as np + +from nni.nas.pytorch.base_mutator import BaseMutator +from nni.nas.pytorch.mutables import LayerChoice + + +class MixedOp(nn.Module): + """ + This class is to instantiate and manage info of one LayerChoice. + It includes architecture weights and member functions for the weights. + """ + + def __init__(self, mutable, latency): + """ + Parameters + ---------- + mutable : LayerChoice + A LayerChoice in user model + latency : List + performance cost for each op in mutable + """ + super(MixedOp, self).__init__() + self.latency = latency + n_choices = len(mutable) + self.path_alpha = nn.Parameter( + torch.FloatTensor([1.0 / n_choices for i in range(n_choices)]) + ) + self.path_alpha.requires_grad = False + self.temperature = 1.0 + + def get_path_alpha(self): + """Return the architecture parameter.""" + return self.path_alpha + + def get_weighted_latency(self): + """Return the weighted perf_cost of current mutable.""" + soft_masks = self.probs_over_ops() + weighted_latency = sum(m * l for m, l in zip(soft_masks, self.latency)) + return weighted_latency + + def set_temperature(self, temperature): + """ + Set the annealed temperature for gumbel softmax. + + Parameters + ---------- + temperature : float + The annealed temperature for gumbel softmax + """ + self.temperature = temperature + + def to_requires_grad(self): + """Enable gradient calculation.""" + self.path_alpha.requires_grad = True + + def to_disable_grad(self): + """Disable gradient calculation.""" + self.path_alpha.requires_grad = False + + def probs_over_ops(self): + """Apply gumbel softmax to generate probability distribution.""" + return F.gumbel_softmax(self.path_alpha, self.temperature) + + def forward(self, mutable, x): + """ + Define forward of LayerChoice. + + Parameters + ---------- + mutable : LayerChoice + this layer's mutable + x : tensor + inputs of this layer, only support one input + + Returns + ------- + output: tensor + output of this layer + """ + candidate_ops = list(mutable) + soft_masks = self.probs_over_ops() + output = sum(m * op(x) for m, op in zip(soft_masks, candidate_ops)) + + return output + + @property + def chosen_index(self): + """ + choose the op with max prob + + Returns + ------- + int + index of the chosen one + """ + alphas = self.path_alpha.data.detach().cpu().numpy() + index = int(np.argmax(alphas)) + return index + + +class FBNetMutator(BaseMutator): + """ + This mutator initializes and operates all the LayerChoices of the supernet. + It is for the related trainer to control the training flow of LayerChoices, + coordinating with whole training process. + """ + + def __init__(self, model, lookup_table): + """ + Init a MixedOp instance for each mutable i.e., LayerChoice. + And register the instantiated MixedOp in corresponding LayerChoice. + If does not register it in LayerChoice, DataParallel does'nt work then, + for architecture weights are not included in the DataParallel model. + When MixedOPs are registered, we use ```requires_grad``` to control + whether calculate gradients of architecture weights. + + Parameters + ---------- + model : pytorch model + The model that users want to tune, + it includes search space defined with nni nas apis + lookup_table : class + lookup table object to manage model space information, + including candidate ops for each stage as the model space, + input channels/output channels/stride/fm_size as the layer config, + and the performance information for perf_cost accumulation. + + """ + super(FBNetMutator, self).__init__(model) + self.mutable_list = [] + + # Collect the op names of the candidate ops within each mutable + ops_names_mutable = dict() + left = 0 + right = 1 + for stage_name in lookup_table.layer_num: + right = lookup_table.layer_num[stage_name] + stage_ops = lookup_table.lut_ops[stage_name] + ops_names = [op_name for op_name in stage_ops] + + for i in range(left, left + right): + ops_names_mutable[i] = ops_names + left = right + + # Create the mixed op + for i, mutable in enumerate(self.undedup_mutables): + ops_names = ops_names_mutable[i] + latency_mutable = lookup_table.lut_perf[i] + latency = [latency_mutable[op_name] for op_name in ops_names] + self.mutable_list.append(mutable) + mutable.registered_module = MixedOp(mutable, latency) + + def on_forward_layer_choice(self, mutable, *args, **kwargs): + """ + Callback of layer choice forward. This function defines the forward + logic of the input mutable. So mutable is only interface, its real + implementation is defined in mutator. + + Parameters + ---------- + mutable: LayerChoice + forward logic of this input mutable + args: list of torch.Tensor + inputs of this mutable + kwargs: dict + inputs of this mutable + + Returns + ------- + torch.Tensor + output of this mutable, i.e., LayerChoice + int + index of the chosen op + """ + # FIXME: return mask, to be consistent with other algorithms + idx = mutable.registered_module.chosen_index + return mutable.registered_module(mutable, *args, **kwargs), idx + + def num_arch_params(self): + """ + The number of mutables, i.e., LayerChoice + + Returns + ------- + int + the number of LayerChoice in user model + """ + return len(self.mutable_list) + + def get_architecture_parameters(self): + """ + Get all the architecture parameters. + + yield + ----- + PyTorch Parameter + Return path_alpha of the traversed mutable + """ + for mutable in self.undedup_mutables: + yield mutable.registered_module.get_path_alpha() + + def get_weighted_latency(self): + """ + Get the latency weighted by gumbel softmax coefficients. + + yield + ----- + Tuple + Return the weighted_latency of the traversed mutable + """ + for mutable in self.undedup_mutables: + yield mutable.registered_module.get_weighted_latency() + + def set_temperature(self, temperature): + """ + Set the annealed temperature of the op for gumbel softmax. + + Parameters + ---------- + temperature : float + The annealed temperature for gumbel softmax + """ + for mutable in self.undedup_mutables: + mutable.registered_module.set_temperature(temperature) + + def arch_requires_grad(self): + """ + Make architecture weights require gradient + """ + for mutable in self.undedup_mutables: + mutable.registered_module.to_requires_grad() + + def arch_disable_grad(self): + """ + Disable gradient of architecture weights, i.e., does not + calculate gradient for them. + """ + for mutable in self.undedup_mutables: + mutable.registered_module.to_disable_grad() + + def sample_final(self): + """ + Generate the final chosen architecture. + + Returns + ------- + dict + the choice of each mutable, i.e., LayerChoice + """ + result = dict() + for mutable in self.undedup_mutables: + assert isinstance(mutable, LayerChoice) + index = mutable.registered_module.chosen_index + # pylint: disable=not-callable + result[mutable.key] = ( + F.one_hot(torch.tensor(index), num_classes=len(mutable)) + .view(-1) + .bool(), + ) + return result diff --git a/utils/third_party/nni_new/algorithms/nas/pytorch/fbnet/trainer.py b/utils/third_party/nni_new/algorithms/nas/pytorch/fbnet/trainer.py new file mode 100644 index 0000000000000000000000000000000000000000..7588ecf5e54877c22ee329c9463da09d52f0e7bf --- /dev/null +++ b/utils/third_party/nni_new/algorithms/nas/pytorch/fbnet/trainer.py @@ -0,0 +1,413 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from __future__ import absolute_import, division, print_function + +import json +import os +import time +import torch + +import numpy as np + +from torch.autograd import Variable +from nni.nas.pytorch.base_trainer import BaseTrainer +from nni.nas.pytorch.trainer import TorchTensorEncoder +from nni.nas.pytorch.utils import AverageMeter +from .mutator import FBNetMutator +from .utils import RegularizerLoss, accuracy + + +class FBNetTrainer(BaseTrainer): + def __init__( + self, + model, + model_optim, + criterion, + device, + device_ids, + lookup_table, + train_loader, + valid_loader, + n_epochs=120, + load_ckpt=False, + arch_path=None, + logger=None, + ): + """ + Parameters + ---------- + model : pytorch model + the user model, which has mutables + model_optim : pytorch optimizer + the user defined optimizer + criterion : pytorch loss + the main task loss, nn.CrossEntropyLoss() is for classification + device : pytorch device + the devices to train/search the model + device_ids : list of int + the indexes of devices used for training + lookup_table : class + lookup table object for fbnet training + train_loader : pytorch data loader + data loader for the training set + valid_loader : pytorch data loader + data loader for the validation set + n_epochs : int + number of epochs to train/search + load_ckpt : bool + whether load checkpoint + arch_path : str + the path to store chosen architecture + logger : logger + the logger + """ + self.model = model + self.model_optim = model_optim + self.train_loader = train_loader + self.valid_loader = valid_loader + self.device = device + self.dev_num = len(device_ids) + self.n_epochs = n_epochs + self.lookup_table = lookup_table + self.config = lookup_table.config + self.start_epoch = self.config.start_epoch + self.temp = self.config.init_temperature + self.exp_anneal_rate = self.config.exp_anneal_rate + self.mode = self.config.mode + + self.load_ckpt = load_ckpt + self.arch_path = arch_path + self.logger = logger + + # scheduler of learning rate + self.scheduler = torch.optim.lr_scheduler.CosineAnnealingLR( + model_optim, T_max=n_epochs, last_epoch=-1 + ) + + # init mutator + self.mutator = FBNetMutator(model, lookup_table) + self.mutator.set_temperature(self.temp) + + # DataParallel should be put behind the init of mutator + self.model = torch.nn.DataParallel(self.model, device_ids=device_ids) + self.model.to(device) + + # build architecture optimizer + self.arch_optimizer = torch.optim.AdamW( + self.mutator.get_architecture_parameters(), + self.config.nas_lr, + weight_decay=self.config.nas_weight_decay, + ) + self.reg_loss = RegularizerLoss(config=self.config) + + self.criterion = criterion + self.epoch = 0 + + def _layer_choice_sample(self): + """ + Sample the index of network within layer choice + """ + stages = [stage_name for stage_name in self.lookup_table.layer_num] + stage_lnum = [self.lookup_table.layer_num[stage] for stage in stages] + + # get the choice idx in each layer + choice_ids = list() + layer_id = 0 + for param in self.mutator.get_architecture_parameters(): + param_np = param.cpu().detach().numpy() + op_idx = np.argmax(param_np) + choice_ids.append(op_idx) + self.logger.info( + "layer {}: {}, index: {}".format(layer_id, param_np, op_idx) + ) + layer_id += 1 + + # get the arch_sample + choice_names = list() + layer_id = 0 + for i, stage_name in enumerate(stages): + ops_names = [op for op in self.lookup_table.lut_ops[stage_name]] + for j in range(stage_lnum[i]): + searched_op = ops_names[choice_ids[layer_id]] + choice_names.append(searched_op) + layer_id += 1 + + self.logger.info(choice_names) + return choice_names + + def _get_perf_cost(self, requires_grad=True): + """ + Get the accumulated performance cost. + """ + perf_cost = Variable( + torch.zeros(1), requires_grad=requires_grad + ).to(self.device, non_blocking=True) + + for latency in self.mutator.get_weighted_latency(): + perf_cost = perf_cost + latency + + return perf_cost + + def _validate(self): + """ + Do validation. During validation, LayerChoices use the mixed-op. + + Returns + ------- + float, float, float + average loss, average top1 accuracy, average top5 accuracy + """ + self.valid_loader.batch_sampler.drop_last = False + batch_time = AverageMeter("batch_time") + losses = AverageMeter("losses") + top1 = AverageMeter("top1") + top5 = AverageMeter("top5") + + # test on validation set under eval mode + self.model.eval() + + end = time.time() + with torch.no_grad(): + for i, (images, labels) in enumerate(self.valid_loader): + images = images.to(self.device, non_blocking=True) + labels = labels.to(self.device, non_blocking=True) + + output = self.model(images) + + loss = self.criterion(output, labels) + acc1, acc5 = accuracy(output, labels, topk=(1, 5)) + losses.update(loss, images.size(0)) + top1.update(acc1[0], images.size(0)) + top5.update(acc5[0], images.size(0)) + # measure elapsed time + batch_time.update(time.time() - end) + end = time.time() + + if i % 10 == 0 or i + 1 == len(self.valid_loader): + test_log = ( + "Valid" + ": [{0}/{1}]\t" + "Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t" + "Loss {loss.val:.4f} ({loss.avg:.4f})\t" + "Top-1 acc {top1.val:.3f} ({top1.avg:.3f})\t" + "Top-5 acc {top5.val:.3f} ({top5.avg:.3f})".format( + i, + len(self.valid_loader) - 1, + batch_time=batch_time, + loss=losses, + top1=top1, + top5=top5, + ) + ) + self.logger.info(test_log) + + return losses.avg, top1.avg, top5.avg + + def _train_epoch(self, epoch, optimizer, arch_train=False): + """ + Train one epoch. + """ + batch_time = AverageMeter("batch_time") + data_time = AverageMeter("data_time") + losses = AverageMeter("losses") + top1 = AverageMeter("top1") + top5 = AverageMeter("top5") + + # switch to train mode + self.model.train() + + data_loader = self.valid_loader if arch_train else self.train_loader + end = time.time() + for i, (images, labels) in enumerate(data_loader): + data_time.update(time.time() - end) + images = images.to(self.device, non_blocking=True) + labels = labels.to(self.device, non_blocking=True) + + output = self.model(images) + loss = self.criterion(output, labels) + + # hardware-aware loss + perf_cost = self._get_perf_cost(requires_grad=True) + regu_loss = self.reg_loss(perf_cost) + if self.mode.startswith("mul"): + loss = loss * regu_loss + elif self.mode.startswith("add"): + loss = loss + regu_loss + + # measure accuracy and record loss + acc1, acc5 = accuracy(output, labels, topk=(1, 5)) + losses.update(loss.item(), images.size(0)) + top1.update(acc1[0].item(), images.size(0)) + top5.update(acc5[0].item(), images.size(0)) + # compute gradient and do SGD step + optimizer.zero_grad() + loss.backward() + optimizer.step() + # measure elapsed time + batch_time.update(time.time() - end) + end = time.time() + + if i % 10 == 0: + batch_log = ( + "Warmup Train [{0}][{1}]\t" + "Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t" + "Data {data_time.val:.3f} ({data_time.avg:.3f})\t" + "Loss {losses.val:.4f} ({losses.avg:.4f})\t" + "Top-1 acc {top1.val:.3f} ({top1.avg:.3f})\t" + "Top-5 acc {top5.val:.3f} ({top5.avg:.3f})\t".format( + epoch + 1, + i, + batch_time=batch_time, + data_time=data_time, + losses=losses, + top1=top1, + top5=top5, + ) + ) + self.logger.info(batch_log) + + def _warm_up(self): + """ + Warm up the model, while the architecture weights are not trained. + """ + for epoch in range(self.epoch, self.start_epoch): + self.logger.info("\n--------Warmup epoch: %d--------\n", epoch + 1) + self._train_epoch(epoch, self.model_optim) + # adjust learning rate + self.scheduler.step() + + # validation + val_loss, val_top1, val_top5 = self._validate() + val_log = ( + "Warmup Valid [{0}/{1}]\t" + "loss {2:.3f}\ttop-1 acc {3:.3f}\ttop-5 acc {4:.3f}".format( + epoch + 1, self.warmup_epochs, val_loss, val_top1, val_top5 + ) + ) + self.logger.info(val_log) + + if epoch % 10 == 0: + filename = os.path.join( + self.config.model_dir, "checkpoint_%s.pth" % epoch + ) + self.save_checkpoint(epoch, filename) + + def _train(self): + """ + Train the model, it trains model weights and architecute weights. + Architecture weights are trained according to the schedule. + Before updating architecture weights, ```requires_grad``` is enabled. + Then, it is disabled after the updating, in order not to update + architecture weights when training model weights. + """ + arch_param_num = self.mutator.num_arch_params() + self.logger.info("#arch_params: {}".format(arch_param_num)) + self.epoch = max(self.start_epoch, self.epoch) + + ckpt_path = self.config.model_dir + choice_names = None + top1_best = 0.0 + + for epoch in range(self.epoch, self.n_epochs): + self.logger.info("\n--------Train epoch: %d--------\n", epoch + 1) + # update the weight parameters + self._train_epoch(epoch, self.model_optim) + # adjust learning rate + self.scheduler.step() + + self.logger.info("Update architecture parameters") + # update the architecture parameters + self.mutator.arch_requires_grad() + self._train_epoch(epoch, self.arch_optimizer, True) + self.mutator.arch_disable_grad() + # temperature annealing + self.temp = self.temp * self.exp_anneal_rate + self.mutator.set_temperature(self.temp) + # sample the architecture of sub-network + choice_names = self._layer_choice_sample() + + # validate + val_loss, val_top1, val_top5 = self._validate() + val_log = ( + "Valid [{0}]\t" + "loss {1:.3f}\ttop-1 acc {2:.3f} \ttop-5 acc {3:.3f}".format( + epoch + 1, val_loss, val_top1, val_top5 + ) + ) + self.logger.info(val_log) + + if epoch % 10 == 0: + filename = os.path.join(ckpt_path, "checkpoint_%s.pth" % epoch) + self.save_checkpoint(epoch, filename, choice_names) + + val_top1 = val_top1.cpu().as_numpy() + if val_top1 > top1_best: + filename = os.path.join(ckpt_path, "checkpoint_best.pth") + self.save_checkpoint(epoch, filename, choice_names) + top1_best = val_top1 + + def save_checkpoint(self, epoch, filename, choice_names=None): + """ + Save checkpoint of the whole model. + Saving model weights and architecture weights as ```filename```, + and saving currently chosen architecture in ```arch_path```. + """ + state = { + "model": self.model.state_dict(), + "optim": self.model_optim.state_dict(), + "epoch": epoch, + "arch_sample": choice_names, + } + torch.save(state, filename) + self.logger.info("Save checkpoint to {0:}".format(filename)) + + if self.arch_path: + self.export(self.arch_path) + + def load_checkpoint(self, filename): + """ + Load the checkpoint from ```ckpt_path```. + """ + ckpt = torch.load(filename) + self.epoch = ckpt["epoch"] + self.model.load_state_dict(ckpt["model"]) + self.model_optim.load_state_dict(ckpt["optim"]) + + def train(self): + """ + Train the whole model. + """ + if self.load_ckpt: + ckpt_path = self.config.model_dir + filename = os.path.join(ckpt_path, "checkpoint_best.pth") + if os.path.exists(filename): + self.load_checkpoint(filename) + + if self.epoch < self.start_epoch: + self._warm_up() + self._train() + + def export(self, file_name): + """ + Export the chosen architecture into a file + + Parameters + ---------- + file_name : str + the file that stores exported chosen architecture + """ + exported_arch = self.mutator.sample_final() + with open(file_name, "w") as f: + json.dump( + exported_arch, + f, + indent=2, + sort_keys=True, + cls=TorchTensorEncoder, + ) + + def validate(self): + raise NotImplementedError + + def checkpoint(self): + raise NotImplementedError diff --git a/utils/third_party/nni_new/algorithms/nas/pytorch/fbnet/utils.py b/utils/third_party/nni_new/algorithms/nas/pytorch/fbnet/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..332f2e2c620e14f27f5402858d2c8c631816dcd2 --- /dev/null +++ b/utils/third_party/nni_new/algorithms/nas/pytorch/fbnet/utils.py @@ -0,0 +1,352 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from __future__ import absolute_import, division, print_function + +import gc # noqa: F401 +import os +import timeit +import torch + +import numpy as np +import torch.nn as nn + +from nni.compression.pytorch.utils.counter import count_flops_params + +LUT_FILE = "lut.npy" +LUT_PATH = "lut" + + +class NASConfig: + def __init__( + self, + perf_metric="flops", + lut_load=False, + model_dir=None, + nas_lr=0.01, + nas_weight_decay=5e-4, + mode="mul", + alpha=0.25, + beta=0.6, + start_epoch=50, + init_temperature=5.0, + exp_anneal_rate=np.exp(-0.045), + search_space=None, + ): + # LUT of performance metric + # flops means the multiplies, latency means the time cost on platform + self.perf_metric = perf_metric + assert perf_metric in [ + "flops", + "latency", + ], "perf_metric should be ['flops', 'latency']" + # wether load or create lut file + self.lut_load = lut_load + # necessary dirs + self.lut_en = model_dir is not None + if self.lut_en: + self.model_dir = model_dir + os.makedirs(model_dir, exist_ok=True) + self.lut_path = os.path.join(model_dir, LUT_PATH) + os.makedirs(self.lut_path, exist_ok=True) + # NAS learning setting + self.nas_lr = nas_lr + self.nas_weight_decay = nas_weight_decay + # hardware-aware loss setting + self.mode = mode + assert mode in ["mul", "add"], "mode should be ['mul', 'add']" + self.alpha = alpha + self.beta = beta + # NAS training setting + self.start_epoch = start_epoch + self.init_temperature = init_temperature + self.exp_anneal_rate = exp_anneal_rate + # definition of search blocks and space + self.search_space = search_space + + +class RegularizerLoss(nn.Module): + """Auxilliary loss for hardware-aware NAS.""" + + def __init__(self, config): + """ + Parameters + ---------- + config : class + to manage the configuration for NAS training, and search space etc. + """ + super(RegularizerLoss, self).__init__() + self.mode = config.mode + self.alpha = config.alpha + self.beta = config.beta + + def forward(self, perf_cost, batch_size=1): + """ + Parameters + ---------- + perf_cost : tensor + the accumulated performance cost + batch_size : int + batch size for normalization + + Returns + ------- + output: tensor + the hardware-aware constraint loss + """ + if self.mode == "mul": + log_loss = torch.log(perf_cost / batch_size) ** self.beta + return self.alpha * log_loss + elif self.mode == "add": + linear_loss = (perf_cost / batch_size) ** self.beta + return self.alpha * linear_loss + else: + raise NotImplementedError + + +def accuracy(output, target, topk=(1,)): + """ + Computes the precision@k for the specified values of k + + Parameters + ---------- + output : pytorch tensor + output, e.g., predicted value + target : pytorch tensor + label + topk : tuple + specify top1 and top5 + + Returns + ------- + list + accuracy of top1 and top5 + """ + maxk = max(topk) + batch_size = target.size(0) + + _, pred = output.topk(maxk, 1, True, True) + pred = pred.t() + correct = pred.eq(target.view(1, -1).expand_as(pred)) + + res = [] + for k in topk: + correct_k = correct[:k].view(-1).float().sum(0, keepdim=True) + res.append(correct_k.mul_(100.0 / batch_size)) + return res + + +def supernet_sample(model, state_dict, sampled_arch=[], lookup_table=None): + """ + Initialize the searched sub-model from supernet. + + Parameters + ---------- + model : pytorch model + the created subnet + state_dict : checkpoint + the checkpoint of supernet, including the pre-trained params + sampled_arch : list of str + the searched layer names of the subnet + lookup_table : class + to manage the candidate ops, layer information and layer performance + """ + replace = list() + stages = [stage for stage in lookup_table.layer_num] + stage_lnum = [lookup_table.layer_num[stage] for stage in stages] + + if sampled_arch: + layer_id = 0 + for i, stage in enumerate(stages): + ops_names = [op_name for op_name in lookup_table.lut_ops[stage]] + for j in range(stage_lnum[i]): + searched_op = sampled_arch[layer_id] + op_i = ops_names.index(searched_op) + replace.append( + [ + "blocks.{}.".format(layer_id), + "blocks.{}.op.".format(layer_id), + "blocks.{}.{}.".format(layer_id, op_i), + ] + ) + layer_id += 1 + model_init(model, state_dict, replace=replace) + + +def model_init(model, state_dict, replace=[]): + """Initialize the model from state_dict.""" + prefix = "module." + param_dict = dict() + for k, v in state_dict.items(): + if k.startswith(prefix): + k = k[7:] + param_dict[k] = v + + for k, (name, m) in enumerate(model.named_modules()): + if replace: + for layer_replace in replace: + assert len(layer_replace) == 3, "The elements should be three." + pre_scope, key, replace_key = layer_replace + if pre_scope in name: + name = name.replace(key, replace_key) + + # Copy the state_dict to current model + if (name + ".weight" in param_dict) or ( + name + ".running_mean" in param_dict + ): + if isinstance(m, nn.BatchNorm2d): + shape = m.running_mean.shape + if shape == param_dict[name + ".running_mean"].shape: + if m.weight is not None: + m.weight.data = param_dict[name + ".weight"] + m.bias.data = param_dict[name + ".bias"] + m.running_mean = param_dict[name + ".running_mean"] + m.running_var = param_dict[name + ".running_var"] + + elif isinstance(m, nn.Conv2d) or isinstance(m, nn.Linear): + shape = m.weight.data.shape + if shape == param_dict[name + ".weight"].shape: + m.weight.data = param_dict[name + ".weight"] + if m.bias is not None: + m.bias.data = param_dict[name + ".bias"] + + elif isinstance(m, nn.ConvTranspose2d): + m.weight.data = param_dict[name + ".weight"] + if m.bias is not None: + m.bias.data = param_dict[name + ".bias"] + + +class LookUpTable: + """Build look-up table for NAS.""" + + def __init__(self, config, primitives): + """ + Parameters + ---------- + config : class + to manage the configuration for NAS training, and search space etc. + """ + self.config = config + # definition of search blocks and space + self.search_space = config.search_space + # layers for NAS + self.cnt_layers = len(self.search_space["input_shape"]) + # constructors for each operation + self.lut_ops = { + stage_name: { + op_name: primitives[op_name] + for op_name in self.search_space["stages"][stage_name]["ops"] + } + for stage_name in self.search_space["stages"] + } + self.layer_num = { + stage_name: self.search_space["stages"][stage_name]["layer_num"] + for stage_name in self.search_space["stages"] + } + + # arguments for the ops constructors, input_shapes just for convinience + self.layer_configs, self.layer_in_shapes = self._layer_configs() + + # lookup_table + self.perf_metric = config.perf_metric + + if config.lut_en: + self.lut_perf = None + self.lut_file = os.path.join(config.lut_path, LUT_FILE) + if config.lut_load: + self._load_from_file() + else: + self._create_perfs() + + def _layer_configs(self): + """Generate basic params for different layers.""" + # layer_configs are : c_in, c_out, stride, fm_size + layer_configs = [ + [ + self.search_space["input_shape"][layer_id][0], + self.search_space["channel_size"][layer_id], + self.search_space["strides"][layer_id], + self.search_space["fm_size"][layer_id], + ] + for layer_id in range(self.cnt_layers) + ] + + # layer_in_shapes are (C_in, input_w, input_h) + layer_in_shapes = self.search_space["input_shape"] + + return layer_configs, layer_in_shapes + + def _create_perfs(self, cnt_of_runs=200): + """Create performance cost for each op.""" + if self.perf_metric == "latency": + self.lut_perf = self._calculate_latency(cnt_of_runs) + elif self.perf_metric == "flops": + self.lut_perf = self._calculate_flops() + + self._write_lut_to_file() + + def _calculate_flops(self, eps=0.001): + """FLOPs cost.""" + flops_lut = [{} for i in range(self.cnt_layers)] + layer_id = 0 + + for stage_name in self.lut_ops: + stage_ops = self.lut_ops[stage_name] + ops_num = self.layer_num[stage_name] + + for _ in range(ops_num): + for op_name in stage_ops: + layer_config = self.layer_configs[layer_id] + key_params = {"fm_size": layer_config[3]} + op = stage_ops[op_name](*layer_config[0:3], **key_params) + + # measured in Flops + in_shape = self.layer_in_shapes[layer_id] + x = (1, in_shape[0], in_shape[1], in_shape[2]) + flops, _, _ = count_flops_params(op, x, verbose=False) + flops = eps if flops == 0.0 else flops + flops_lut[layer_id][op_name] = float(flops) + layer_id += 1 + + return flops_lut + + def _calculate_latency(self, cnt_of_runs): + """Latency cost.""" + LATENCY_BATCH_SIZE = 1 + latency_lut = [{} for i in range(self.cnt_layers)] + layer_id = 0 + + for stage_name in self.lut_ops: + stage_ops = self.lut_ops[stage_name] + ops_num = self.layer_num[stage_name] + + for _ in range(ops_num): + for op_name in stage_ops: + layer_config = self.layer_configs[layer_id] + key_params = {"fm_size": layer_config[3]} + op = stage_ops[op_name](*layer_config[0:3], **key_params) + input_data = torch.randn( + (LATENCY_BATCH_SIZE, *self.layer_in_shapes[layer_id]) + ) + globals()["op"], globals()["input_data"] = op, input_data + total_time = timeit.timeit( + "output = op(input_data)", + setup="gc.enable()", + globals=globals(), + number=cnt_of_runs, + ) + # measured in micro-second + latency_lut[layer_id][op_name] = ( + total_time / cnt_of_runs / LATENCY_BATCH_SIZE * 1e6 + ) + layer_id += 1 + + return latency_lut + + def _write_lut_to_file(self): + """Save lut as numpy file.""" + np.save(self.lut_file, self.lut_perf) + + def _load_from_file(self): + """Load numpy file.""" + self.lut_perf = np.load(self.lut_file, allow_pickle=True) diff --git a/utils/third_party/nni_new/algorithms/nas/pytorch/pdarts/__init__.py b/utils/third_party/nni_new/algorithms/nas/pytorch/pdarts/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d1d17764ba159b35bcc38efa82a2a30dc2366b76 --- /dev/null +++ b/utils/third_party/nni_new/algorithms/nas/pytorch/pdarts/__init__.py @@ -0,0 +1,4 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .trainer import PdartsTrainer diff --git a/utils/third_party/nni_new/algorithms/nas/pytorch/pdarts/mutator.py b/utils/third_party/nni_new/algorithms/nas/pytorch/pdarts/mutator.py new file mode 100644 index 0000000000000000000000000000000000000000..09ad51c5e471b4acae51d513335682328924b90a --- /dev/null +++ b/utils/third_party/nni_new/algorithms/nas/pytorch/pdarts/mutator.py @@ -0,0 +1,93 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import copy + +import numpy as np +import torch +from torch import nn + +from nni.algorithms.nas.pytorch.darts import DartsMutator +from nni.nas.pytorch.mutables import LayerChoice + + +class PdartsMutator(DartsMutator): + """ + It works with PdartsTrainer to calculate ops weights, + and drop weights in different PDARTS epochs. + """ + + def __init__(self, model, pdarts_epoch_index, pdarts_num_to_drop, switches={}): + self.pdarts_epoch_index = pdarts_epoch_index + self.pdarts_num_to_drop = pdarts_num_to_drop + if switches is None: + self.switches = {} + else: + self.switches = switches + + super(PdartsMutator, self).__init__(model) + + # this loop go through mutables with different keys, + # it's mainly to update length of choices. + for mutable in self.mutables: + if isinstance(mutable, LayerChoice): + + switches = self.switches.get(mutable.key, [True for j in range(len(mutable))]) + choices = self.choices[mutable.key] + + operations_count = np.sum(switches) + # +1 and -1 are caused by zero operation in darts network + # the zero operation is not in choices list in network, but its weight are in, + # so it needs one more weights and switch for zero. + self.choices[mutable.key] = nn.Parameter(1.0E-3 * torch.randn(operations_count + 1)) + self.switches[mutable.key] = switches + + # update LayerChoice instances in model, + # it's physically remove dropped choices operations. + for module in self.model.modules(): + if isinstance(module, LayerChoice): + switches = self.switches.get(module.key) + choices = self.choices[module.key] + if len(module) > len(choices): + # from last to first, so that it won't effect previous indexes after removed one. + for index in range(len(switches)-1, -1, -1): + if switches[index] == False: + del module[index] + assert len(module) <= len(choices), "Failed to remove dropped choices." + + def export(self): + # Cannot rely on super().export() because P-DARTS has deleted some of the choices and has misaligned length. + results = super().sample_final() + for mutable in self.mutables: + if isinstance(mutable, LayerChoice): + # As some operations are dropped physically, + # so it needs to fill back false to track dropped operations. + trained_result = results[mutable.key] + trained_index = 0 + switches = self.switches[mutable.key] + result = torch.Tensor(switches).bool() + for index in range(len(result)): + if result[index]: + result[index] = trained_result[trained_index] + trained_index += 1 + results[mutable.key] = result + return results + + def drop_paths(self): + """ + This method is called when a PDARTS epoch is finished. + It prepares switches for next epoch. + candidate operations with False switch will be doppped in next epoch. + """ + all_switches = copy.deepcopy(self.switches) + for key in all_switches: + switches = all_switches[key] + idxs = [] + for j in range(len(switches)): + if switches[j]: + idxs.append(j) + sorted_weights = self.choices[key].data.cpu().numpy()[:-1] + drop = np.argsort(sorted_weights)[:self.pdarts_num_to_drop[self.pdarts_epoch_index]] + for idx in drop: + switches[idxs[idx]] = False + return all_switches diff --git a/utils/third_party/nni_new/algorithms/nas/pytorch/pdarts/trainer.py b/utils/third_party/nni_new/algorithms/nas/pytorch/pdarts/trainer.py new file mode 100644 index 0000000000000000000000000000000000000000..7f23a6e222731ae6edc5c034f77eafaaf71b4c5e --- /dev/null +++ b/utils/third_party/nni_new/algorithms/nas/pytorch/pdarts/trainer.py @@ -0,0 +1,86 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import json +import logging + +from nni.nas.pytorch.callbacks import LRSchedulerCallback +from nni.algorithms.nas.pytorch.darts import DartsTrainer +from nni.nas.pytorch.trainer import BaseTrainer, TorchTensorEncoder + +from .mutator import PdartsMutator + +logger = logging.getLogger(__name__) + + +class PdartsTrainer(BaseTrainer): + """ + This trainer implements the PDARTS algorithm. + PDARTS bases on DARTS algorithm, and provides a network growth approach to find deeper and better network. + This class relies on pdarts_num_layers and pdarts_num_to_drop parameters to control how network grows. + pdarts_num_layers means how many layers more than first epoch. + pdarts_num_to_drop means how many candidate operations should be dropped in each epoch. + So that the grew network can in similar size. + """ + + def __init__(self, model_creator, init_layers, metrics, + num_epochs, dataset_train, dataset_valid, + pdarts_num_layers=[0, 6, 12], pdarts_num_to_drop=[3, 2, 1], + mutator=None, batch_size=64, workers=4, device=None, log_frequency=None, callbacks=None, unrolled=False): + super(PdartsTrainer, self).__init__() + self.model_creator = model_creator + self.init_layers = init_layers + self.pdarts_num_layers = pdarts_num_layers + self.pdarts_num_to_drop = pdarts_num_to_drop + self.pdarts_epoch = len(pdarts_num_to_drop) + self.darts_parameters = { + "metrics": metrics, + "num_epochs": num_epochs, + "dataset_train": dataset_train, + "dataset_valid": dataset_valid, + "batch_size": batch_size, + "workers": workers, + "device": device, + "log_frequency": log_frequency, + "unrolled": unrolled + } + self.callbacks = callbacks if callbacks is not None else [] + + def train(self): + + switches = None + for epoch in range(self.pdarts_epoch): + + layers = self.init_layers+self.pdarts_num_layers[epoch] + model, criterion, optim, lr_scheduler = self.model_creator(layers) + self.mutator = PdartsMutator(model, epoch, self.pdarts_num_to_drop, switches) + + for callback in self.callbacks: + callback.build(model, self.mutator, self) + callback.on_epoch_begin(epoch) + + darts_callbacks = [] + if lr_scheduler is not None: + darts_callbacks.append(LRSchedulerCallback(lr_scheduler)) + + self.trainer = DartsTrainer(model, mutator=self.mutator, loss=criterion, optimizer=optim, + callbacks=darts_callbacks, **self.darts_parameters) + logger.info("start pdarts training epoch %s...", epoch) + + self.trainer.train() + + switches = self.mutator.drop_paths() + + for callback in self.callbacks: + callback.on_epoch_end(epoch) + + def validate(self): + self.trainer.validate() + + def export(self, file): + mutator_export = self.mutator.export() + with open(file, "w") as f: + json.dump(mutator_export, f, indent=2, sort_keys=True, cls=TorchTensorEncoder) + + def checkpoint(self): + raise NotImplementedError("Not implemented yet") diff --git a/utils/third_party/nni_new/algorithms/nas/pytorch/proxylessnas/__init__.py b/utils/third_party/nni_new/algorithms/nas/pytorch/proxylessnas/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..26feedba7d553c32d61ea9139620a68fca7c12d0 --- /dev/null +++ b/utils/third_party/nni_new/algorithms/nas/pytorch/proxylessnas/__init__.py @@ -0,0 +1,2 @@ +from .mutator import ProxylessNasMutator +from .trainer import ProxylessNasTrainer diff --git a/utils/third_party/nni_new/algorithms/nas/pytorch/proxylessnas/mutator.py b/utils/third_party/nni_new/algorithms/nas/pytorch/proxylessnas/mutator.py new file mode 100644 index 0000000000000000000000000000000000000000..881a6b44038d5d6f2c97b2b106036954401a9cd3 --- /dev/null +++ b/utils/third_party/nni_new/algorithms/nas/pytorch/proxylessnas/mutator.py @@ -0,0 +1,478 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import math +import torch +from torch import nn as nn +from torch.nn import functional as F +import numpy as np + +from nni.nas.pytorch.base_mutator import BaseMutator +from nni.nas.pytorch.mutables import LayerChoice +from .utils import detach_variable + +class ArchGradientFunction(torch.autograd.Function): + @staticmethod + def forward(ctx, x, binary_gates, run_func, backward_func): + ctx.run_func = run_func + ctx.backward_func = backward_func + + detached_x = detach_variable(x) + with torch.enable_grad(): + output = run_func(detached_x) + ctx.save_for_backward(detached_x, output) + return output.data + + @staticmethod + def backward(ctx, grad_output): + detached_x, output = ctx.saved_tensors + + grad_x = torch.autograd.grad(output, detached_x, grad_output, only_inputs=True) + # compute gradients w.r.t. binary_gates + binary_grads = ctx.backward_func(detached_x.data, output.data, grad_output.data) + + return grad_x[0], binary_grads, None, None + +class MixedOp(nn.Module): + """ + This class is to instantiate and manage info of one LayerChoice. + It includes architecture weights, binary weights, and member functions + operating the weights. + + forward_mode: + forward/backward mode for LayerChoice: None, two, full, and full_v2. + For training architecture weights, we use full_v2 by default, and for training + model weights, we use None. + """ + forward_mode = None + def __init__(self, mutable): + """ + Parameters + ---------- + mutable : LayerChoice + A LayerChoice in user model + """ + super(MixedOp, self).__init__() + self.ap_path_alpha = nn.Parameter(torch.Tensor(len(mutable))) + self.ap_path_wb = nn.Parameter(torch.Tensor(len(mutable))) + self.ap_path_alpha.requires_grad = False + self.ap_path_wb.requires_grad = False + self.active_index = [0] + self.inactive_index = None + self.log_prob = None + self.current_prob_over_ops = None + self.n_choices = len(mutable) + + def get_ap_path_alpha(self): + return self.ap_path_alpha + + def to_requires_grad(self): + self.ap_path_alpha.requires_grad = True + self.ap_path_wb.requires_grad = True + + def to_disable_grad(self): + self.ap_path_alpha.requires_grad = False + self.ap_path_wb.requires_grad = False + + def forward(self, mutable, x): + """ + Define forward of LayerChoice. For 'full_v2', backward is also defined. + The 'two' mode is explained in section 3.2.1 in the paper. + The 'full_v2' mode is explained in Appendix D in the paper. + + Parameters + ---------- + mutable : LayerChoice + this layer's mutable + x : tensor + inputs of this layer, only support one input + + Returns + ------- + output: tensor + output of this layer + """ + if MixedOp.forward_mode == 'full' or MixedOp.forward_mode == 'two': + output = 0 + for _i in self.active_index: + oi = self.candidate_ops[_i](x) + output = output + self.ap_path_wb[_i] * oi + for _i in self.inactive_index: + oi = self.candidate_ops[_i](x) + output = output + self.ap_path_wb[_i] * oi.detach() + elif MixedOp.forward_mode == 'full_v2': + def run_function(key, candidate_ops, active_id): + def forward(_x): + return candidate_ops[active_id](_x) + return forward + + def backward_function(key, candidate_ops, active_id, binary_gates): + def backward(_x, _output, grad_output): + binary_grads = torch.zeros_like(binary_gates.data) + with torch.no_grad(): + for k in range(len(candidate_ops)): + if k != active_id: + out_k = candidate_ops[k](_x.data) + else: + out_k = _output.data + grad_k = torch.sum(out_k * grad_output) + binary_grads[k] = grad_k + return binary_grads + return backward + output = ArchGradientFunction.apply( + x, self.ap_path_wb, run_function(mutable.key, list(mutable), self.active_index[0]), + backward_function(mutable.key, list(mutable), self.active_index[0], self.ap_path_wb)) + else: + output = self.active_op(mutable)(x) + return output + + @property + def probs_over_ops(self): + """ + Apply softmax on alpha to generate probability distribution + + Returns + ------- + pytorch tensor + probability distribution + """ + probs = F.softmax(self.ap_path_alpha, dim=0) # softmax to probability + return probs + + @property + def chosen_index(self): + """ + choose the op with max prob + + Returns + ------- + int + index of the chosen one + numpy.float32 + prob of the chosen one + """ + probs = self.probs_over_ops.data.cpu().numpy() + index = int(np.argmax(probs)) + return index, probs[index] + + def active_op(self, mutable): + """ + assume only one path is active + + Returns + ------- + PyTorch module + the chosen operation + """ + return mutable[self.active_index[0]] + + @property + def active_op_index(self): + """ + return active op's index, the active op is sampled + + Returns + ------- + int + index of the active op + """ + return self.active_index[0] + + def set_chosen_op_active(self): + """ + set chosen index, active and inactive indexes + """ + chosen_idx, _ = self.chosen_index + self.active_index = [chosen_idx] + self.inactive_index = [_i for _i in range(0, chosen_idx)] + \ + [_i for _i in range(chosen_idx + 1, self.n_choices)] + + def binarize(self, mutable): + """ + Sample based on alpha, and set binary weights accordingly. + ap_path_wb is set in this function, which is called binarize. + + Parameters + ---------- + mutable : LayerChoice + this layer's mutable + """ + self.log_prob = None + # reset binary gates + self.ap_path_wb.data.zero_() + probs = self.probs_over_ops + if MixedOp.forward_mode == 'two': + # sample two ops according to probs + sample_op = torch.multinomial(probs.data, 2, replacement=False) + probs_slice = F.softmax(torch.stack([ + self.ap_path_alpha[idx] for idx in sample_op + ]), dim=0) + self.current_prob_over_ops = torch.zeros_like(probs) + for i, idx in enumerate(sample_op): + self.current_prob_over_ops[idx] = probs_slice[i] + # choose one to be active and the other to be inactive according to probs_slice + c = torch.multinomial(probs_slice.data, 1)[0] # 0 or 1 + active_op = sample_op[c].item() + inactive_op = sample_op[1-c].item() + self.active_index = [active_op] + self.inactive_index = [inactive_op] + # set binary gate + self.ap_path_wb.data[active_op] = 1.0 + else: + sample = torch.multinomial(probs, 1)[0].item() + self.active_index = [sample] + self.inactive_index = [_i for _i in range(0, sample)] + \ + [_i for _i in range(sample + 1, len(mutable))] + self.log_prob = torch.log(probs[sample]) + self.current_prob_over_ops = probs + self.ap_path_wb.data[sample] = 1.0 + # avoid over-regularization + for choice in mutable: + for _, param in choice.named_parameters(): + param.grad = None + + @staticmethod + def delta_ij(i, j): + if i == j: + return 1 + else: + return 0 + + def set_arch_param_grad(self, mutable): + """ + Calculate alpha gradient for this LayerChoice. + It is calculated using gradient of binary gate, probs of ops. + """ + binary_grads = self.ap_path_wb.grad.data + if self.active_op(mutable).is_zero_layer(): + self.ap_path_alpha.grad = None + return + if self.ap_path_alpha.grad is None: + self.ap_path_alpha.grad = torch.zeros_like(self.ap_path_alpha.data) + if MixedOp.forward_mode == 'two': + involved_idx = self.active_index + self.inactive_index + probs_slice = F.softmax(torch.stack([ + self.ap_path_alpha[idx] for idx in involved_idx + ]), dim=0).data + for i in range(2): + for j in range(2): + origin_i = involved_idx[i] + origin_j = involved_idx[j] + self.ap_path_alpha.grad.data[origin_i] += \ + binary_grads[origin_j] * probs_slice[j] * (MixedOp.delta_ij(i, j) - probs_slice[i]) + for _i, idx in enumerate(self.active_index): + self.active_index[_i] = (idx, self.ap_path_alpha.data[idx].item()) + for _i, idx in enumerate(self.inactive_index): + self.inactive_index[_i] = (idx, self.ap_path_alpha.data[idx].item()) + else: + probs = self.probs_over_ops.data + for i in range(self.n_choices): + for j in range(self.n_choices): + self.ap_path_alpha.grad.data[i] += binary_grads[j] * probs[j] * (MixedOp.delta_ij(i, j) - probs[i]) + return + + def rescale_updated_arch_param(self): + """ + rescale architecture weights for the 'two' mode. + """ + if not isinstance(self.active_index[0], tuple): + assert self.active_op.is_zero_layer() + return + involved_idx = [idx for idx, _ in (self.active_index + self.inactive_index)] + old_alphas = [alpha for _, alpha in (self.active_index + self.inactive_index)] + new_alphas = [self.ap_path_alpha.data[idx] for idx in involved_idx] + + offset = math.log( + sum([math.exp(alpha) for alpha in new_alphas]) / sum([math.exp(alpha) for alpha in old_alphas]) + ) + + for idx in involved_idx: + self.ap_path_alpha.data[idx] -= offset + + +class ProxylessNasMutator(BaseMutator): + """ + This mutator initializes and operates all the LayerChoices of the input model. + It is for the corresponding trainer to control the training process of LayerChoices, + coordinating with whole training process. + """ + def __init__(self, model): + """ + Init a MixedOp instance for each mutable i.e., LayerChoice. + And register the instantiated MixedOp in corresponding LayerChoice. + If does not register it in LayerChoice, DataParallel does not work then, + because architecture weights are not included in the DataParallel model. + When MixedOPs are registered, we use ```requires_grad``` to control + whether calculate gradients of architecture weights. + + Parameters + ---------- + model : pytorch model + The model that users want to tune, it includes search space defined with nni nas apis + """ + super(ProxylessNasMutator, self).__init__(model) + self._unused_modules = None + self.mutable_list = [] + for mutable in self.undedup_mutables: + self.mutable_list.append(mutable) + mutable.registered_module = MixedOp(mutable) + + def on_forward_layer_choice(self, mutable, *args, **kwargs): + """ + Callback of layer choice forward. This function defines the forward + logic of the input mutable. So mutable is only interface, its real + implementation is defined in mutator. + + Parameters + ---------- + mutable: LayerChoice + forward logic of this input mutable + args: list of torch.Tensor + inputs of this mutable + kwargs: dict + inputs of this mutable + + Returns + ------- + torch.Tensor + output of this mutable, i.e., LayerChoice + int + index of the chosen op + """ + # FIXME: return mask, to be consistent with other algorithms + idx = mutable.registered_module.active_op_index + return mutable.registered_module(mutable, *args, **kwargs), idx + + def reset_binary_gates(self): + """ + For each LayerChoice, binarize binary weights + based on alpha to only activate one op. + It traverses all the mutables in the model to do this. + """ + for mutable in self.undedup_mutables: + mutable.registered_module.binarize(mutable) + + def set_chosen_op_active(self): + """ + For each LayerChoice, set the op with highest alpha as the chosen op. + Usually used for validation. + """ + for mutable in self.undedup_mutables: + mutable.registered_module.set_chosen_op_active() + + def num_arch_params(self): + """ + The number of mutables, i.e., LayerChoice + + Returns + ------- + int + the number of LayerChoice in user model + """ + return len(self.mutable_list) + + def set_arch_param_grad(self): + """ + For each LayerChoice, calculate gradients for architecture weights, i.e., alpha + """ + for mutable in self.undedup_mutables: + mutable.registered_module.set_arch_param_grad(mutable) + + def get_architecture_parameters(self): + """ + Get all the architecture parameters. + + yield + ----- + PyTorch Parameter + Return ap_path_alpha of the traversed mutable + """ + for mutable in self.undedup_mutables: + yield mutable.registered_module.get_ap_path_alpha() + + def change_forward_mode(self, mode): + """ + Update forward mode of MixedOps, as training architecture weights and + model weights use different forward modes. + """ + MixedOp.forward_mode = mode + + def get_forward_mode(self): + """ + Get forward mode of MixedOp + + Returns + ------- + string + the current forward mode of MixedOp + """ + return MixedOp.forward_mode + + def rescale_updated_arch_param(self): + """ + Rescale architecture weights in 'two' mode. + """ + for mutable in self.undedup_mutables: + mutable.registered_module.rescale_updated_arch_param() + + def unused_modules_off(self): + """ + Remove unused modules for each mutables. + The removed modules are kept in ```self._unused_modules``` for resume later. + """ + self._unused_modules = [] + for mutable in self.undedup_mutables: + mixed_op = mutable.registered_module + unused = {} + if self.get_forward_mode() in ['full', 'two', 'full_v2']: + involved_index = mixed_op.active_index + mixed_op.inactive_index + else: + involved_index = mixed_op.active_index + for i in range(mixed_op.n_choices): + if i not in involved_index: + unused[i] = mutable[i] + mutable[i] = None + self._unused_modules.append(unused) + + def unused_modules_back(self): + """ + Resume the removed modules back. + """ + if self._unused_modules is None: + return + for m, unused in zip(self.mutable_list, self._unused_modules): + for i in unused: + m[i] = unused[i] + self._unused_modules = None + + def arch_requires_grad(self): + """ + Make architecture weights require gradient + """ + for mutable in self.undedup_mutables: + mutable.registered_module.to_requires_grad() + + def arch_disable_grad(self): + """ + Disable gradient of architecture weights, i.e., does not + calcuate gradient for them. + """ + for mutable in self.undedup_mutables: + mutable.registered_module.to_disable_grad() + + def sample_final(self): + """ + Generate the final chosen architecture. + + Returns + ------- + dict + the choice of each mutable, i.e., LayerChoice + """ + result = dict() + for mutable in self.undedup_mutables: + assert isinstance(mutable, LayerChoice) + index, _ = mutable.registered_module.chosen_index + # pylint: disable=not-callable + result[mutable.key] = F.one_hot(torch.tensor(index), num_classes=len(mutable)).view(-1).bool() + return result diff --git a/utils/third_party/nni_new/algorithms/nas/pytorch/proxylessnas/trainer.py b/utils/third_party/nni_new/algorithms/nas/pytorch/proxylessnas/trainer.py new file mode 100644 index 0000000000000000000000000000000000000000..d9c86a6a9f098792a4731db32dd140ce3708ea8f --- /dev/null +++ b/utils/third_party/nni_new/algorithms/nas/pytorch/proxylessnas/trainer.py @@ -0,0 +1,500 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import math +import time +import json +import logging + +import torch +from torch import nn as nn + +from nni.nas.pytorch.base_trainer import BaseTrainer +from nni.nas.pytorch.trainer import TorchTensorEncoder +from nni.nas.pytorch.utils import AverageMeter +from .mutator import ProxylessNasMutator +from .utils import cross_entropy_with_label_smoothing, accuracy + +logger = logging.getLogger(__name__) + +class ProxylessNasTrainer(BaseTrainer): + def __init__(self, model, model_optim, device, + train_loader, valid_loader, label_smoothing=0.1, + n_epochs=120, init_lr=0.025, binary_mode='full_v2', + arch_init_type='normal', arch_init_ratio=1e-3, + arch_optim_lr=1e-3, arch_weight_decay=0, + grad_update_arch_param_every=5, grad_update_steps=1, + warmup=True, warmup_epochs=25, + arch_valid_frequency=1, + load_ckpt=False, ckpt_path=None, arch_path=None): + """ + Parameters + ---------- + model : pytorch model + the user model, which has mutables + model_optim : pytorch optimizer + the user defined optimizer + device : pytorch device + the devices to train/search the model + train_loader : pytorch data loader + data loader for the training set + valid_loader : pytorch data loader + data loader for the validation set + label_smoothing : float + for label smoothing + n_epochs : int + number of epochs to train/search + init_lr : float + init learning rate for training the model + binary_mode : str + the forward/backward mode for the binary weights in mutator + arch_init_type : str + the way to init architecture parameters + arch_init_ratio : float + the ratio to init architecture parameters + arch_optim_lr : float + learning rate of the architecture parameters optimizer + arch_weight_decay : float + weight decay of the architecture parameters optimizer + grad_update_arch_param_every : int + update architecture weights every this number of minibatches + grad_update_steps : int + during each update of architecture weights, the number of steps to train + warmup : bool + whether to do warmup + warmup_epochs : int + the number of epochs to do during warmup + arch_valid_frequency : int + frequency of printing validation result + load_ckpt : bool + whether load checkpoint + ckpt_path : str + checkpoint path, if load_ckpt is True, ckpt_path cannot be None + arch_path : str + the path to store chosen architecture + """ + self.model = model + self.model_optim = model_optim + self.train_loader = train_loader + self.valid_loader = valid_loader + self.device = device + self.n_epochs = n_epochs + self.init_lr = init_lr + self.warmup = warmup + self.warmup_epochs = warmup_epochs + self.arch_valid_frequency = arch_valid_frequency + self.label_smoothing = label_smoothing + + self.train_batch_size = train_loader.batch_sampler.batch_size + self.valid_batch_size = valid_loader.batch_sampler.batch_size + # update architecture parameters every this number of minibatches + self.grad_update_arch_param_every = grad_update_arch_param_every + # the number of steps per architecture parameter update + self.grad_update_steps = grad_update_steps + self.binary_mode = binary_mode + + self.load_ckpt = load_ckpt + self.ckpt_path = ckpt_path + self.arch_path = arch_path + + # init mutator + self.mutator = ProxylessNasMutator(model) + + # DataParallel should be put behind the init of mutator + self.model = torch.nn.DataParallel(self.model) + self.model.to(self.device) + + # iter of valid dataset for training architecture weights + self._valid_iter = None + # init architecture weights + self._init_arch_params(arch_init_type, arch_init_ratio) + # build architecture optimizer + self.arch_optimizer = torch.optim.Adam(self.mutator.get_architecture_parameters(), + arch_optim_lr, + weight_decay=arch_weight_decay, + betas=(0, 0.999), + eps=1e-8) + + self.criterion = nn.CrossEntropyLoss() + self.warmup_curr_epoch = 0 + self.train_curr_epoch = 0 + + def _init_arch_params(self, init_type='normal', init_ratio=1e-3): + """ + Initialize architecture weights + """ + for param in self.mutator.get_architecture_parameters(): + if init_type == 'normal': + param.data.normal_(0, init_ratio) + elif init_type == 'uniform': + param.data.uniform_(-init_ratio, init_ratio) + else: + raise NotImplementedError + + def _validate(self): + """ + Do validation. During validation, LayerChoices use the chosen active op. + + Returns + ------- + float, float, float + average loss, average top1 accuracy, average top5 accuracy + """ + self.valid_loader.batch_sampler.batch_size = self.valid_batch_size + self.valid_loader.batch_sampler.drop_last = False + + self.mutator.set_chosen_op_active() + # remove unused modules to save memory + self.mutator.unused_modules_off() + # test on validation set under train mode + self.model.train() + batch_time = AverageMeter('batch_time') + losses = AverageMeter('losses') + top1 = AverageMeter('top1') + top5 = AverageMeter('top5') + end = time.time() + with torch.no_grad(): + for i, (images, labels) in enumerate(self.valid_loader): + images, labels = images.to(self.device), labels.to(self.device) + output = self.model(images) + loss = self.criterion(output, labels) + acc1, acc5 = accuracy(output, labels, topk=(1, 5)) + losses.update(loss, images.size(0)) + top1.update(acc1[0], images.size(0)) + top5.update(acc5[0], images.size(0)) + # measure elapsed time + batch_time.update(time.time() - end) + end = time.time() + + if i % 10 == 0 or i + 1 == len(self.valid_loader): + test_log = 'Valid' + ': [{0}/{1}]\t'\ + 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'\ + 'Loss {loss.val:.4f} ({loss.avg:.4f})\t'\ + 'Top-1 acc {top1.val:.3f} ({top1.avg:.3f})'.\ + format(i, len(self.valid_loader) - 1, batch_time=batch_time, loss=losses, top1=top1) + # return top5: + test_log += '\tTop-5 acc {top5.val:.3f} ({top5.avg:.3f})'.format(top5=top5) + logger.info(test_log) + self.mutator.unused_modules_back() + return losses.avg, top1.avg, top5.avg + + def _warm_up(self): + """ + Warm up the model, during warm up, architecture weights are not trained. + """ + lr_max = 0.05 + data_loader = self.train_loader + nBatch = len(data_loader) + T_total = self.warmup_epochs * nBatch # total num of batches + + for epoch in range(self.warmup_curr_epoch, self.warmup_epochs): + logger.info('\n--------Warmup epoch: %d--------\n', epoch + 1) + batch_time = AverageMeter('batch_time') + data_time = AverageMeter('data_time') + losses = AverageMeter('losses') + top1 = AverageMeter('top1') + top5 = AverageMeter('top5') + # switch to train mode + self.model.train() + + end = time.time() + logger.info('warm_up epoch: %d', epoch) + for i, (images, labels) in enumerate(data_loader): + data_time.update(time.time() - end) + # lr + T_cur = epoch * nBatch + i + warmup_lr = 0.5 * lr_max * (1 + math.cos(math.pi * T_cur / T_total)) + for param_group in self.model_optim.param_groups: + param_group['lr'] = warmup_lr + images, labels = images.to(self.device), labels.to(self.device) + # compute output + self.mutator.reset_binary_gates() # random sample binary gates + self.mutator.unused_modules_off() # remove unused module for speedup + output = self.model(images) + if self.label_smoothing > 0: + loss = cross_entropy_with_label_smoothing(output, labels, self.label_smoothing) + else: + loss = self.criterion(output, labels) + # measure accuracy and record loss + acc1, acc5 = accuracy(output, labels, topk=(1, 5)) + losses.update(loss, images.size(0)) + top1.update(acc1[0], images.size(0)) + top5.update(acc5[0], images.size(0)) + # compute gradient and do SGD step + self.model.zero_grad() + loss.backward() + self.model_optim.step() + # unused modules back + self.mutator.unused_modules_back() + # measure elapsed time + batch_time.update(time.time() - end) + end = time.time() + + if i % 10 == 0 or i + 1 == nBatch: + batch_log = 'Warmup Train [{0}][{1}/{2}]\t' \ + 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' \ + 'Data {data_time.val:.3f} ({data_time.avg:.3f})\t' \ + 'Loss {losses.val:.4f} ({losses.avg:.4f})\t' \ + 'Top-1 acc {top1.val:.3f} ({top1.avg:.3f})\t' \ + 'Top-5 acc {top5.val:.3f} ({top5.avg:.3f})\tlr {lr:.5f}'. \ + format(epoch + 1, i, nBatch - 1, batch_time=batch_time, data_time=data_time, + losses=losses, top1=top1, top5=top5, lr=warmup_lr) + logger.info(batch_log) + val_loss, val_top1, val_top5 = self._validate() + val_log = 'Warmup Valid [{0}/{1}]\tloss {2:.3f}\ttop-1 acc {3:.3f}\ttop-5 acc {4:.3f}\t' \ + 'Train top-1 {top1.avg:.3f}\ttop-5 {top5.avg:.3f}M'. \ + format(epoch + 1, self.warmup_epochs, val_loss, val_top1, val_top5, top1=top1, top5=top5) + logger.info(val_log) + self.save_checkpoint() + self.warmup_curr_epoch += 1 + + def _get_update_schedule(self, nBatch): + """ + Generate schedule for training architecture weights. Key means after which minibatch + to update architecture weights, value means how many steps for the update. + + Parameters + ---------- + nBatch : int + the total number of minibatches in one epoch + + Returns + ------- + dict + the schedule for updating architecture weights + """ + schedule = {} + for i in range(nBatch): + if (i + 1) % self.grad_update_arch_param_every == 0: + schedule[i] = self.grad_update_steps + return schedule + + def _calc_learning_rate(self, epoch, batch=0, nBatch=None): + """ + Update learning rate. + """ + T_total = self.n_epochs * nBatch + T_cur = epoch * nBatch + batch + lr = 0.5 * self.init_lr * (1 + math.cos(math.pi * T_cur / T_total)) + return lr + + def _adjust_learning_rate(self, optimizer, epoch, batch=0, nBatch=None): + """ + Adjust learning of a given optimizer and return the new learning rate + + Parameters + ---------- + optimizer : pytorch optimizer + the used optimizer + epoch : int + the current epoch number + batch : int + the current minibatch + nBatch : int + the total number of minibatches in one epoch + + Returns + ------- + float + the adjusted learning rate + """ + new_lr = self._calc_learning_rate(epoch, batch, nBatch) + for param_group in optimizer.param_groups: + param_group['lr'] = new_lr + return new_lr + + def _train(self): + """ + Train the model, it trains model weights and architecute weights. + Architecture weights are trained according to the schedule. + Before updating architecture weights, ```requires_grad``` is enabled. + Then, it is disabled after the updating, in order not to update + architecture weights when training model weights. + """ + nBatch = len(self.train_loader) + arch_param_num = self.mutator.num_arch_params() + binary_gates_num = self.mutator.num_arch_params() + logger.info('#arch_params: %d\t#binary_gates: %d', arch_param_num, binary_gates_num) + + update_schedule = self._get_update_schedule(nBatch) + + for epoch in range(self.train_curr_epoch, self.n_epochs): + logger.info('\n--------Train epoch: %d--------\n', epoch + 1) + batch_time = AverageMeter('batch_time') + data_time = AverageMeter('data_time') + losses = AverageMeter('losses') + top1 = AverageMeter('top1') + top5 = AverageMeter('top5') + # switch to train mode + self.model.train() + + end = time.time() + for i, (images, labels) in enumerate(self.train_loader): + data_time.update(time.time() - end) + lr = self._adjust_learning_rate(self.model_optim, epoch, batch=i, nBatch=nBatch) + # train weight parameters + images, labels = images.to(self.device), labels.to(self.device) + self.mutator.reset_binary_gates() + self.mutator.unused_modules_off() + output = self.model(images) + if self.label_smoothing > 0: + loss = cross_entropy_with_label_smoothing(output, labels, self.label_smoothing) + else: + loss = self.criterion(output, labels) + acc1, acc5 = accuracy(output, labels, topk=(1, 5)) + losses.update(loss, images.size(0)) + top1.update(acc1[0], images.size(0)) + top5.update(acc5[0], images.size(0)) + self.model.zero_grad() + loss.backward() + self.model_optim.step() + self.mutator.unused_modules_back() + if epoch > 0: + for _ in range(update_schedule.get(i, 0)): + start_time = time.time() + # GradientArchSearchConfig + self.mutator.arch_requires_grad() + arch_loss, exp_value = self._gradient_step() + self.mutator.arch_disable_grad() + used_time = time.time() - start_time + log_str = 'Architecture [%d-%d]\t Time %.4f\t Loss %.4f\t null %s' % \ + (epoch + 1, i, used_time, arch_loss, exp_value) + logger.info(log_str) + batch_time.update(time.time() - end) + end = time.time() + # training log + if i % 10 == 0 or i + 1 == nBatch: + batch_log = 'Train [{0}][{1}/{2}]\t' \ + 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' \ + 'Data Time {data_time.val:.3f} ({data_time.avg:.3f})\t' \ + 'Loss {losses.val:.4f} ({losses.avg:.4f})\t' \ + 'Top-1 acc {top1.val:.3f} ({top1.avg:.3f})\t' \ + 'Top-5 acc {top5.val:.3f} ({top5.avg:.3f})\tlr {lr:.5f}'. \ + format(epoch + 1, i, nBatch - 1, batch_time=batch_time, data_time=data_time, + losses=losses, top1=top1, top5=top5, lr=lr) + logger.info(batch_log) + # validate + if (epoch + 1) % self.arch_valid_frequency == 0: + val_loss, val_top1, val_top5 = self._validate() + val_log = 'Valid [{0}]\tloss {1:.3f}\ttop-1 acc {2:.3f} \ttop-5 acc {3:.3f}\t' \ + 'Train top-1 {top1.avg:.3f}\ttop-5 {top5.avg:.3f}'. \ + format(epoch + 1, val_loss, val_top1, val_top5, top1=top1, top5=top5) + logger.info(val_log) + self.save_checkpoint() + self.train_curr_epoch += 1 + + def _valid_next_batch(self): + """ + Get next one minibatch from validation set + + Returns + ------- + (tensor, tensor) + the tuple of images and labels + """ + if self._valid_iter is None: + self._valid_iter = iter(self.valid_loader) + try: + data = next(self._valid_iter) + except StopIteration: + self._valid_iter = iter(self.valid_loader) + data = next(self._valid_iter) + return data + + def _gradient_step(self): + """ + This gradient step is for updating architecture weights. + Mutator is intensively used in this function to operate on + architecture weights. + + Returns + ------- + float, None + loss of the model, None + """ + # use the same batch size as train batch size for architecture weights + self.valid_loader.batch_sampler.batch_size = self.train_batch_size + self.valid_loader.batch_sampler.drop_last = True + self.model.train() + self.mutator.change_forward_mode(self.binary_mode) + time1 = time.time() # time + # sample a batch of data from validation set + images, labels = self._valid_next_batch() + images, labels = images.to(self.device), labels.to(self.device) + time2 = time.time() # time + self.mutator.reset_binary_gates() + self.mutator.unused_modules_off() + output = self.model(images) + time3 = time.time() + ce_loss = self.criterion(output, labels) + expected_value = None + loss = ce_loss + self.model.zero_grad() + loss.backward() + self.mutator.set_arch_param_grad() + self.arch_optimizer.step() + if self.mutator.get_forward_mode() == 'two': + self.mutator.rescale_updated_arch_param() + self.mutator.unused_modules_back() + self.mutator.change_forward_mode(None) + time4 = time.time() + logger.info('(%.4f, %.4f, %.4f)', time2 - time1, time3 - time2, time4 - time3) + return loss.data.item(), expected_value.item() if expected_value is not None else None + + def save_checkpoint(self): + """ + Save checkpoint of the whole model. Saving model weights and architecture weights in + ```ckpt_path```, and saving currently chosen architecture in ```arch_path```. + """ + if self.ckpt_path: + state = { + 'warmup_curr_epoch': self.warmup_curr_epoch, + 'train_curr_epoch': self.train_curr_epoch, + 'model': self.model.state_dict(), + 'optim': self.model_optim.state_dict(), + 'arch_optim': self.arch_optimizer.state_dict() + } + torch.save(state, self.ckpt_path) + if self.arch_path: + self.export(self.arch_path) + + def load_checkpoint(self): + """ + Load the checkpoint from ```ckpt_path```. + """ + assert self.ckpt_path is not None, "If load_ckpt is not None, ckpt_path should not be None" + ckpt = torch.load(self.ckpt_path) + self.warmup_curr_epoch = ckpt['warmup_curr_epoch'] + self.train_curr_epoch = ckpt['train_curr_epoch'] + self.model.load_state_dict(ckpt['model']) + self.model_optim.load_state_dict(ckpt['optim']) + self.arch_optimizer.load_state_dict(ckpt['arch_optim']) + + def train(self): + """ + Train the whole model. + """ + if self.load_ckpt: + self.load_checkpoint() + if self.warmup: + self._warm_up() + self._train() + + def export(self, file_name): + """ + Export the chosen architecture into a file + + Parameters + ---------- + file_name : str + the file that stores exported chosen architecture + """ + exported_arch = self.mutator.sample_final() + with open(file_name, 'w') as f: + json.dump(exported_arch, f, indent=2, sort_keys=True, cls=TorchTensorEncoder) + + def validate(self): + raise NotImplementedError + + def checkpoint(self): + raise NotImplementedError diff --git a/utils/third_party/nni_new/algorithms/nas/pytorch/proxylessnas/utils.py b/utils/third_party/nni_new/algorithms/nas/pytorch/proxylessnas/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..b703810d3b703a33bd6bbe8422c557a1f669146b --- /dev/null +++ b/utils/third_party/nni_new/algorithms/nas/pytorch/proxylessnas/utils.py @@ -0,0 +1,78 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import torch +import torch.nn as nn + +def detach_variable(inputs): + """ + Detach variables + + Parameters + ---------- + inputs : pytorch tensors + pytorch tensors + """ + if isinstance(inputs, tuple): + return tuple([detach_variable(x) for x in inputs]) + else: + x = inputs.detach() + x.requires_grad = inputs.requires_grad + return x + +def cross_entropy_with_label_smoothing(pred, target, label_smoothing=0.1): + """ + Parameters + ---------- + pred : pytorch tensor + predicted value + target : pytorch tensor + label + label_smoothing : float + the degree of label smoothing + + Returns + ------- + pytorch tensor + cross entropy + """ + logsoftmax = nn.LogSoftmax() + n_classes = pred.size(1) + # convert to one-hot + target = torch.unsqueeze(target, 1) + soft_target = torch.zeros_like(pred) + soft_target.scatter_(1, target, 1) + # label smoothing + soft_target = soft_target * (1 - label_smoothing) + label_smoothing / n_classes + return torch.mean(torch.sum(- soft_target * logsoftmax(pred), 1)) + +def accuracy(output, target, topk=(1,)): + """ + Computes the precision@k for the specified values of k + + Parameters + ---------- + output : pytorch tensor + output, e.g., predicted value + target : pytorch tensor + label + topk : tuple + specify top1 and top5 + + Returns + ------- + list + accuracy of top1 and top5 + """ + maxk = max(topk) + batch_size = target.size(0) + + _, pred = output.topk(maxk, 1, True, True) + pred = pred.t() + correct = pred.eq(target.view(1, -1).expand_as(pred)) + + res = [] + for k in topk: + correct_k = correct[:k].view(-1).float().sum(0, keepdim=True) + res.append(correct_k.mul_(100.0 / batch_size)) + return res diff --git a/utils/third_party/nni_new/algorithms/nas/pytorch/random/__init__.py b/utils/third_party/nni_new/algorithms/nas/pytorch/random/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..b4102266ca2a03d29c1c16c90b688865ce493b7a --- /dev/null +++ b/utils/third_party/nni_new/algorithms/nas/pytorch/random/__init__.py @@ -0,0 +1 @@ +from .mutator import RandomMutator diff --git a/utils/third_party/nni_new/algorithms/nas/pytorch/random/mutator.py b/utils/third_party/nni_new/algorithms/nas/pytorch/random/mutator.py new file mode 100644 index 0000000000000000000000000000000000000000..f302db56c0cdbd611f648342cc26760d344958c4 --- /dev/null +++ b/utils/third_party/nni_new/algorithms/nas/pytorch/random/mutator.py @@ -0,0 +1,36 @@ +import torch +import torch.nn.functional as F + +from nni.nas.pytorch.mutator import Mutator +from nni.nas.pytorch.mutables import LayerChoice, InputChoice + + +class RandomMutator(Mutator): + """ + Random mutator that samples a random candidate in the search space each time ``reset()``. + It uses random function in PyTorch, so users can set seed in PyTorch to ensure deterministic behavior. + """ + + def sample_search(self): + """ + Sample a random candidate. + """ + result = dict() + for mutable in self.mutables: + if isinstance(mutable, LayerChoice): + gen_index = torch.randint(high=len(mutable), size=(1, )) + result[mutable.key] = F.one_hot(gen_index, num_classes=len(mutable)).view(-1).bool() + elif isinstance(mutable, InputChoice): + if mutable.n_chosen is None: + result[mutable.key] = torch.randint(high=2, size=(mutable.n_candidates,)).view(-1).bool() + else: + perm = torch.randperm(mutable.n_candidates) + mask = [i in perm[:mutable.n_chosen] for i in range(mutable.n_candidates)] + result[mutable.key] = torch.tensor(mask, dtype=torch.bool) # pylint: disable=not-callable + return result + + def sample_final(self): + """ + Same as :meth:`sample_search`. + """ + return self.sample_search() diff --git a/utils/third_party/nni_new/algorithms/nas/pytorch/spos/__init__.py b/utils/third_party/nni_new/algorithms/nas/pytorch/spos/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..ed432b0845154c4745aa82c8e6a8bad4290237aa --- /dev/null +++ b/utils/third_party/nni_new/algorithms/nas/pytorch/spos/__init__.py @@ -0,0 +1,6 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .evolution import SPOSEvolution +from .mutator import SPOSSupernetTrainingMutator +from .trainer import SPOSSupernetTrainer diff --git a/utils/third_party/nni_new/algorithms/nas/pytorch/spos/evolution.py b/utils/third_party/nni_new/algorithms/nas/pytorch/spos/evolution.py new file mode 100644 index 0000000000000000000000000000000000000000..bd099e276ebfd4aa68215dbbd25e6443a45b9dd1 --- /dev/null +++ b/utils/third_party/nni_new/algorithms/nas/pytorch/spos/evolution.py @@ -0,0 +1,223 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import json +import logging +import os +import re +from collections import deque + +import numpy as np +from nni.tuner import Tuner +from nni.algorithms.nas.pytorch.classic_nas.mutator import LAYER_CHOICE, INPUT_CHOICE + + +_logger = logging.getLogger(__name__) + + +class SPOSEvolution(Tuner): + """ + SPOS evolution tuner. + + Parameters + ---------- + max_epochs : int + Maximum number of epochs to run. + num_select : int + Number of survival candidates of each epoch. + num_population : int + Number of candidates at the start of each epoch. If candidates generated by + crossover and mutation are not enough, the rest will be filled with random + candidates. + m_prob : float + The probability of mutation. + num_crossover : int + Number of candidates generated by crossover in each epoch. + num_mutation : int + Number of candidates generated by mutation in each epoch. + """ + + def __init__(self, max_epochs=20, num_select=10, num_population=50, m_prob=0.1, + num_crossover=25, num_mutation=25): + assert num_population >= num_select + self.max_epochs = max_epochs + self.num_select = num_select + self.num_population = num_population + self.m_prob = m_prob + self.num_crossover = num_crossover + self.num_mutation = num_mutation + self.epoch = 0 + self.candidates = [] + self.search_space = None + self.random_state = np.random.RandomState(0) + + # async status + self._to_evaluate_queue = deque() + self._sending_parameter_queue = deque() + self._pending_result_ids = set() + self._reward_dict = dict() + self._id2candidate = dict() + self._st_callback = None + + def update_search_space(self, search_space): + """ + Handle the initialization/update event of search space. + """ + self._search_space = search_space + self._next_round() + + def _next_round(self): + _logger.info("Epoch %d, generating...", self.epoch) + if self.epoch == 0: + self._get_random_population() + self.export_results(self.candidates) + else: + best_candidates = self._select_top_candidates() + self.export_results(best_candidates) + if self.epoch >= self.max_epochs: + return + self.candidates = self._get_mutation(best_candidates) + self._get_crossover(best_candidates) + self._get_random_population() + self.epoch += 1 + + def _random_candidate(self): + chosen_arch = dict() + for key, val in self._search_space.items(): + if val["_type"] == LAYER_CHOICE: + choices = val["_value"] + index = self.random_state.randint(len(choices)) + chosen_arch[key] = {"_value": choices[index], "_idx": index} + elif val["_type"] == INPUT_CHOICE: + raise NotImplementedError("Input choice is not implemented yet.") + return chosen_arch + + def _add_to_evaluate_queue(self, cand): + _logger.info("Generate candidate %s, adding to eval queue.", self._get_architecture_repr(cand)) + self._reward_dict[self._hashcode(cand)] = 0. + self._to_evaluate_queue.append(cand) + + def _get_random_population(self): + while len(self.candidates) < self.num_population: + cand = self._random_candidate() + if self._is_legal(cand): + _logger.info("Random candidate generated.") + self._add_to_evaluate_queue(cand) + self.candidates.append(cand) + + def _get_crossover(self, best): + result = [] + for _ in range(10 * self.num_crossover): + cand_p1 = best[self.random_state.randint(len(best))] + cand_p2 = best[self.random_state.randint(len(best))] + assert cand_p1.keys() == cand_p2.keys() + cand = {k: cand_p1[k] if self.random_state.randint(2) == 0 else cand_p2[k] + for k in cand_p1.keys()} + if self._is_legal(cand): + result.append(cand) + self._add_to_evaluate_queue(cand) + if len(result) >= self.num_crossover: + break + _logger.info("Found %d architectures with crossover.", len(result)) + return result + + def _get_mutation(self, best): + result = [] + for _ in range(10 * self.num_mutation): + cand = best[self.random_state.randint(len(best))].copy() + mutation_sample = np.random.random_sample(len(cand)) + for s, k in zip(mutation_sample, cand): + if s < self.m_prob: + choices = self._search_space[k]["_value"] + index = self.random_state.randint(len(choices)) + cand[k] = {"_value": choices[index], "_idx": index} + if self._is_legal(cand): + result.append(cand) + self._add_to_evaluate_queue(cand) + if len(result) >= self.num_mutation: + break + _logger.info("Found %d architectures with mutation.", len(result)) + return result + + def _get_architecture_repr(self, cand): + return re.sub(r"\".*?\": \{\"_idx\": (\d+), \"_value\": \".*?\"\}", r"\1", + self._hashcode(cand)) + + def _is_legal(self, cand): + if self._hashcode(cand) in self._reward_dict: + return False + return True + + def _select_top_candidates(self): + reward_query = lambda cand: self._reward_dict[self._hashcode(cand)] + _logger.info("All candidate rewards: %s", list(map(reward_query, self.candidates))) + result = sorted(self.candidates, key=reward_query, reverse=True)[:self.num_select] + _logger.info("Best candidate rewards: %s", list(map(reward_query, result))) + return result + + @staticmethod + def _hashcode(d): + return json.dumps(d, sort_keys=True) + + def _bind_and_send_parameters(self): + """ + There are two types of resources: parameter ids and candidates. This function is called at + necessary times to bind these resources to send new trials with st_callback. + """ + result = [] + while self._sending_parameter_queue and self._to_evaluate_queue: + parameter_id = self._sending_parameter_queue.popleft() + parameters = self._to_evaluate_queue.popleft() + self._id2candidate[parameter_id] = parameters + result.append(parameters) + self._pending_result_ids.add(parameter_id) + self._st_callback(parameter_id, parameters) + _logger.info("Send parameter [%d] %s.", parameter_id, self._get_architecture_repr(parameters)) + return result + + def generate_multiple_parameters(self, parameter_id_list, **kwargs): + """ + Callback function necessary to implement a tuner. This will put more parameter ids into the + parameter id queue. + """ + if "st_callback" in kwargs and self._st_callback is None: + self._st_callback = kwargs["st_callback"] + for parameter_id in parameter_id_list: + self._sending_parameter_queue.append(parameter_id) + self._bind_and_send_parameters() + return [] # always not use this. might induce problem of over-sending + + def receive_trial_result(self, parameter_id, parameters, value, **kwargs): + """ + Callback function. Receive a trial result. + """ + _logger.info("Candidate %d, reported reward %f", parameter_id, value) + self._reward_dict[self._hashcode(self._id2candidate[parameter_id])] = value + + def trial_end(self, parameter_id, success, **kwargs): + """ + Callback function when a trial is ended and resource is released. + """ + self._pending_result_ids.remove(parameter_id) + if not self._pending_result_ids and not self._to_evaluate_queue: + # a new epoch now + self._next_round() + assert self._st_callback is not None + self._bind_and_send_parameters() + + def export_results(self, result): + """ + Export a number of candidates to `checkpoints` dir. + + Parameters + ---------- + result : dict + Chosen architectures to be exported. + """ + os.makedirs("checkpoints", exist_ok=True) + for i, cand in enumerate(result): + converted = dict() + for cand_key, cand_val in cand.items(): + onehot = [k == cand_val["_idx"] for k in range(len(self._search_space[cand_key]["_value"]))] + converted[cand_key] = onehot + with open(os.path.join("checkpoints", "%03d_%03d.json" % (self.epoch, i)), "w") as fp: + json.dump(converted, fp) diff --git a/utils/third_party/nni_new/algorithms/nas/pytorch/spos/mutator.py b/utils/third_party/nni_new/algorithms/nas/pytorch/spos/mutator.py new file mode 100644 index 0000000000000000000000000000000000000000..1a803cb2e820b0df2dd8b04b3d387af68d3d2680 --- /dev/null +++ b/utils/third_party/nni_new/algorithms/nas/pytorch/spos/mutator.py @@ -0,0 +1,66 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging + +import numpy as np +from nni.algorithms.nas.pytorch.random import RandomMutator + +_logger = logging.getLogger(__name__) + + +class SPOSSupernetTrainingMutator(RandomMutator): + """ + A random mutator with flops limit. + + Parameters + ---------- + model : nn.Module + PyTorch model. + flops_func : callable + Callable that takes a candidate from `sample_search` and returns its candidate. When `flops_func` + is None, functions related to flops will be deactivated. + flops_lb : number + Lower bound of flops. + flops_ub : number + Upper bound of flops. + flops_bin_num : number + Number of bins divided for the interval of flops to ensure the uniformity. Bigger number will be more + uniform, but the sampling will be slower. + flops_sample_timeout : int + Maximum number of attempts to sample before giving up and use a random candidate. + """ + def __init__(self, model, flops_func=None, flops_lb=None, flops_ub=None, + flops_bin_num=7, flops_sample_timeout=500): + + super().__init__(model) + self._flops_func = flops_func + if self._flops_func is not None: + self._flops_bin_num = flops_bin_num + self._flops_bins = [flops_lb + (flops_ub - flops_lb) / flops_bin_num * i for i in range(flops_bin_num + 1)] + self._flops_sample_timeout = flops_sample_timeout + + def sample_search(self): + """ + Sample a candidate for training. When `flops_func` is not None, candidates will be sampled uniformly + relative to flops. + + Returns + ------- + dict + """ + if self._flops_func is not None: + for times in range(self._flops_sample_timeout): + idx = np.random.randint(self._flops_bin_num) + cand = super().sample_search() + if self._flops_bins[idx] <= self._flops_func(cand) <= self._flops_bins[idx + 1]: + _logger.debug("Sampled candidate flops %f in %d times.", cand, times) + return cand + _logger.warning("Failed to sample a flops-valid candidate within %d tries.", self._flops_sample_timeout) + return super().sample_search() + + def sample_final(self): + """ + Implement only to suffice the interface of Mutator. + """ + return self.sample_search() diff --git a/utils/third_party/nni_new/algorithms/nas/pytorch/spos/trainer.py b/utils/third_party/nni_new/algorithms/nas/pytorch/spos/trainer.py new file mode 100644 index 0000000000000000000000000000000000000000..7c954e2ad4c913fd16f8e9adf195f2ebb14b1e9b --- /dev/null +++ b/utils/third_party/nni_new/algorithms/nas/pytorch/spos/trainer.py @@ -0,0 +1,95 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging + +import torch +from nni.nas.pytorch.trainer import Trainer +from nni.nas.pytorch.utils import AverageMeterGroup + +from .mutator import SPOSSupernetTrainingMutator + +logger = logging.getLogger(__name__) + + +class SPOSSupernetTrainer(Trainer): + """ + This trainer trains a supernet that can be used for evolution search. + + Parameters + ---------- + model : nn.Module + Model with mutables. + mutator : nni.nas.pytorch.mutator.Mutator + A mutator object that has been initialized with the model. + loss : callable + Called with logits and targets. Returns a loss tensor. + metrics : callable + Returns a dict that maps metrics keys to metrics data. + optimizer : Optimizer + Optimizer that optimizes the model. + num_epochs : int + Number of epochs of training. + train_loader : iterable + Data loader of training. Raise ``StopIteration`` when one epoch is exhausted. + dataset_valid : iterable + Data loader of validation. Raise ``StopIteration`` when one epoch is exhausted. + batch_size : int + Batch size. + workers: int + Number of threads for data preprocessing. Not used for this trainer. Maybe removed in future. + device : torch.device + Device object. Either ``torch.device("cuda")`` or ``torch.device("cpu")``. When ``None``, trainer will + automatic detects GPU and selects GPU first. + log_frequency : int + Number of mini-batches to log metrics. + callbacks : list of Callback + Callbacks to plug into the trainer. See Callbacks. + """ + + def __init__(self, model, loss, metrics, + optimizer, num_epochs, train_loader, valid_loader, + mutator=None, batch_size=64, workers=4, device=None, log_frequency=None, + callbacks=None): + assert torch.cuda.is_available() + super().__init__(model, mutator if mutator is not None else SPOSSupernetTrainingMutator(model), + loss, metrics, optimizer, num_epochs, None, None, + batch_size, workers, device, log_frequency, callbacks) + + self.train_loader = train_loader + self.valid_loader = valid_loader + + def train_one_epoch(self, epoch): + self.model.train() + meters = AverageMeterGroup() + for step, (x, y) in enumerate(self.train_loader): + x, y = x.to(self.device), y.to(self.device) + self.optimizer.zero_grad() + self.mutator.reset() + logits = self.model(x) + loss = self.loss(logits, y) + loss.backward() + self.optimizer.step() + + metrics = self.metrics(logits, y) + metrics["loss"] = loss.item() + meters.update(metrics) + if self.log_frequency is not None and step % self.log_frequency == 0: + logger.info("Epoch [%s/%s] Step [%s/%s] %s", epoch + 1, + self.num_epochs, step + 1, len(self.train_loader), meters) + + def validate_one_epoch(self, epoch): + self.model.eval() + meters = AverageMeterGroup() + with torch.no_grad(): + for step, (x, y) in enumerate(self.valid_loader): + x, y = x.to(self.device), y.to(self.device) + self.mutator.reset() + logits = self.model(x) + loss = self.loss(logits, y) + metrics = self.metrics(logits, y) + metrics["loss"] = loss.item() + meters.update(metrics) + if self.log_frequency is not None and step % self.log_frequency == 0: + logger.info("Epoch [%s/%s] Validation Step [%s/%s] %s", epoch + 1, + self.num_epochs, step + 1, len(self.valid_loader), meters) diff --git a/utils/third_party/nni_new/algorithms/nas/tensorflow/classic_nas/__init__.py b/utils/third_party/nni_new/algorithms/nas/tensorflow/classic_nas/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..ec3f5a4894a0460d4a0582a0d6cd43af9bed77e2 --- /dev/null +++ b/utils/third_party/nni_new/algorithms/nas/tensorflow/classic_nas/__init__.py @@ -0,0 +1,4 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .mutator import get_and_apply_next_architecture diff --git a/utils/third_party/nni_new/algorithms/nas/tensorflow/classic_nas/mutator.py b/utils/third_party/nni_new/algorithms/nas/tensorflow/classic_nas/mutator.py new file mode 100644 index 0000000000000000000000000000000000000000..fad4987fed55bc9a1460089ee0fd3b92a63ee83f --- /dev/null +++ b/utils/third_party/nni_new/algorithms/nas/tensorflow/classic_nas/mutator.py @@ -0,0 +1,215 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import json +import logging +import os +import sys + +import tensorflow as tf + +import nni +from nni.runtime.env_vars import trial_env_vars +from nni.nas.tensorflow.mutables import LayerChoice, InputChoice, MutableScope +from nni.nas.tensorflow.mutator import Mutator + +logger = logging.getLogger(__name__) + +NNI_GEN_SEARCH_SPACE = "NNI_GEN_SEARCH_SPACE" +LAYER_CHOICE = "layer_choice" +INPUT_CHOICE = "input_choice" + + +def get_and_apply_next_architecture(model): + """ + Wrapper of :class:`~nni.nas.tensorflow.classic_nas.mutator.ClassicMutator` to make it more meaningful, + similar to ``get_next_parameter`` for HPO. + Tt will generate search space based on ``model``. + If env ``NNI_GEN_SEARCH_SPACE`` exists, this is in dry run mode for + generating search space for the experiment. + If not, there are still two mode, one is nni experiment mode where users + use ``nnictl`` to start an experiment. The other is standalone mode + where users directly run the trial command, this mode chooses the first + one(s) for each LayerChoice and InputChoice. + Parameters + ---------- + model : nn.Module + User's model with search space (e.g., LayerChoice, InputChoice) embedded in it. + """ + ClassicMutator(model) + + +class ClassicMutator(Mutator): + """ + This mutator is to apply the architecture chosen from tuner. + It implements the forward function of LayerChoice and InputChoice, + to only activate the chosen ones. + Parameters + ---------- + model : nn.Module + User's model with search space (e.g., LayerChoice, InputChoice) embedded in it. + """ + + def __init__(self, model): + super(ClassicMutator, self).__init__(model) + self._chosen_arch = {} + self._search_space = self._generate_search_space() + if NNI_GEN_SEARCH_SPACE in os.environ: + # dry run for only generating search space + self._dump_search_space(os.environ[NNI_GEN_SEARCH_SPACE]) + sys.exit(0) + + if trial_env_vars.NNI_PLATFORM is None: + logger.warning("This is in standalone mode, the chosen are the first one(s).") + self._chosen_arch = self._standalone_generate_chosen() + else: + # get chosen arch from tuner + self._chosen_arch = nni.get_next_parameter() + if self._chosen_arch is None: + if trial_env_vars.NNI_PLATFORM == "unittest": + # happens if NNI_PLATFORM is intentionally set, e.g., in UT + logger.warning("`NNI_PLATFORM` is set but `param` is None. Falling back to standalone mode.") + self._chosen_arch = self._standalone_generate_chosen() + else: + raise RuntimeError("Chosen architecture is None. This may be a platform error.") + self.reset() + + def _sample_layer_choice(self, mutable, idx, value, search_space_item): + """ + Convert layer choice to tensor representation. + Parameters + ---------- + mutable : Mutable + idx : int + Number `idx` of list will be selected. + value : str + The verbose representation of the selected value. + search_space_item : list + The list for corresponding search space. + """ + # doesn't support multihot for layer choice yet + assert 0 <= idx < len(mutable) and search_space_item[idx] == value, \ + "Index '{}' in search space '{}' is not '{}'".format(idx, search_space_item, value) + mask = tf.one_hot(idx, len(mutable)) + return tf.cast(tf.reshape(mask, [-1]), tf.bool) + + def _sample_input_choice(self, mutable, idx, value, search_space_item): + """ + Convert input choice to tensor representation. + Parameters + ---------- + mutable : Mutable + idx : int + Number `idx` of list will be selected. + value : str + The verbose representation of the selected value. + search_space_item : list + The list for corresponding search space. + """ + candidate_repr = search_space_item["candidates"] + multihot_list = [False] * mutable.n_candidates + for i, v in zip(idx, value): + assert 0 <= i < mutable.n_candidates and candidate_repr[i] == v, \ + "Index '{}' in search space '{}' is not '{}'".format(i, candidate_repr, v) + assert not multihot_list[i], "'{}' is selected twice in '{}', which is not allowed.".format(i, idx) + multihot_list[i] = True + return tf.cast(multihot_list, tf.bool) # pylint: disable=not-callable + + def sample_search(self): + """ + See :meth:`sample_final`. + """ + return self.sample_final() + + def sample_final(self): + """ + Convert the chosen arch and apply it on model. + """ + assert set(self._chosen_arch.keys()) == set(self._search_space.keys()), \ + "Unmatched keys, expected keys '{}' from search space, found '{}'.".format(self._search_space.keys(), + self._chosen_arch.keys()) + result = dict() + for mutable in self.mutables: + if isinstance(mutable, (LayerChoice, InputChoice)): + assert mutable.key in self._chosen_arch, \ + "Expected '{}' in chosen arch, but not found.".format(mutable.key) + data = self._chosen_arch[mutable.key] + assert isinstance(data, dict) and "_value" in data and "_idx" in data, \ + "'{}' is not a valid choice.".format(data) + if isinstance(mutable, LayerChoice): + result[mutable.key] = self._sample_layer_choice(mutable, data["_idx"], data["_value"], + self._search_space[mutable.key]["_value"]) + elif isinstance(mutable, InputChoice): + result[mutable.key] = self._sample_input_choice(mutable, data["_idx"], data["_value"], + self._search_space[mutable.key]["_value"]) + elif isinstance(mutable, MutableScope): + logger.info("Mutable scope '%s' is skipped during parsing choices.", mutable.key) + else: + raise TypeError("Unsupported mutable type: '%s'." % type(mutable)) + return result + + def _standalone_generate_chosen(self): + """ + Generate the chosen architecture for standalone mode, + i.e., choose the first one(s) for LayerChoice and InputChoice. + :: + { key_name: {"_value": "conv1", + "_idx": 0} } + { key_name: {"_value": ["in1"], + "_idx": [0]} } + Returns + ------- + dict + the chosen architecture + """ + chosen_arch = {} + for key, val in self._search_space.items(): + if val["_type"] == LAYER_CHOICE: + choices = val["_value"] + chosen_arch[key] = {"_value": choices[0], "_idx": 0} + elif val["_type"] == INPUT_CHOICE: + choices = val["_value"]["candidates"] + n_chosen = val["_value"]["n_chosen"] + if n_chosen is None: + n_chosen = len(choices) + chosen_arch[key] = {"_value": choices[:n_chosen], "_idx": list(range(n_chosen))} + else: + raise ValueError("Unknown key '%s' and value '%s'." % (key, val)) + return chosen_arch + + def _generate_search_space(self): + """ + Generate search space from mutables. + Here is the search space format: + :: + { key_name: {"_type": "layer_choice", + "_value": ["conv1", "conv2"]} } + { key_name: {"_type": "input_choice", + "_value": {"candidates": ["in1", "in2"], + "n_chosen": 1}} } + Returns + ------- + dict + the generated search space + """ + search_space = {} + for mutable in self.mutables: + # for now we only generate flattened search space + if isinstance(mutable, LayerChoice): + key = mutable.key + val = mutable.names + search_space[key] = {"_type": LAYER_CHOICE, "_value": val} + elif isinstance(mutable, InputChoice): + key = mutable.key + search_space[key] = {"_type": INPUT_CHOICE, + "_value": {"candidates": mutable.choose_from, + "n_chosen": mutable.n_chosen}} + elif isinstance(mutable, MutableScope): + logger.info("Mutable scope '%s' is skipped during generating search space.", mutable.key) + else: + raise TypeError("Unsupported mutable type: '%s'." % type(mutable)) + return search_space + + def _dump_search_space(self, file_path): + with open(file_path, "w") as ss_file: + json.dump(self._search_space, ss_file, sort_keys=True, indent=2) diff --git a/utils/third_party/nni_new/algorithms/nas/tensorflow/enas/__init__.py b/utils/third_party/nni_new/algorithms/nas/tensorflow/enas/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d3372836ebba2387ade58161218aad731433e46b --- /dev/null +++ b/utils/third_party/nni_new/algorithms/nas/tensorflow/enas/__init__.py @@ -0,0 +1,5 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .mutator import EnasMutator +from .trainer import EnasTrainer diff --git a/utils/third_party/nni_new/algorithms/nas/tensorflow/enas/mutator.py b/utils/third_party/nni_new/algorithms/nas/tensorflow/enas/mutator.py new file mode 100644 index 0000000000000000000000000000000000000000..de43195fa2e6eef6b8c1991cf8d56998d7abc4fb --- /dev/null +++ b/utils/third_party/nni_new/algorithms/nas/tensorflow/enas/mutator.py @@ -0,0 +1,160 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import tensorflow as tf +from tensorflow.keras.layers import Dense, Embedding, LSTMCell, RNN +from tensorflow.keras.losses import SparseCategoricalCrossentropy, Reduction + +from nni.nas.tensorflow.mutator import Mutator +from nni.nas.tensorflow.mutables import LayerChoice, InputChoice, MutableScope + + +class EnasMutator(Mutator): + def __init__(self, model, + lstm_size=64, + lstm_num_layers=1, + tanh_constant=1.5, + cell_exit_extra_step=False, + skip_target=0.4, + temperature=None, + branch_bias=0.25, + entropy_reduction='sum'): + super().__init__(model) + self.tanh_constant = tanh_constant + self.temperature = temperature + self.cell_exit_extra_step = cell_exit_extra_step + + cells = [LSTMCell(units=lstm_size, use_bias=False) for _ in range(lstm_num_layers)] + self.lstm = RNN(cells, stateful=True) + self.g_emb = tf.random.normal((1, 1, lstm_size)) * 0.1 + self.skip_targets = tf.constant([1.0 - skip_target, skip_target]) + + self.max_layer_choice = 0 + self.bias_dict = {} + for mutable in self.mutables: + if isinstance(mutable, LayerChoice): + if self.max_layer_choice == 0: + self.max_layer_choice = len(mutable) + assert self.max_layer_choice == len(mutable), \ + "ENAS mutator requires all layer choice have the same number of candidates." + if 'reduce' in mutable.key: + bias = [] + for choice in mutable.choices: + if 'conv' in str(type(choice)).lower(): + bias.append(branch_bias) + else: + bias.append(-branch_bias) + self.bias_dict[mutable.key] = tf.constant(bias) + + # exposed for trainer + self.sample_log_prob = 0 + self.sample_entropy = 0 + self.sample_skip_penalty = 0 + + # internal nn layers + self.embedding = Embedding(self.max_layer_choice + 1, lstm_size) + self.soft = Dense(self.max_layer_choice, use_bias=False) + self.attn_anchor = Dense(lstm_size, use_bias=False) + self.attn_query = Dense(lstm_size, use_bias=False) + self.v_attn = Dense(1, use_bias=False) + assert entropy_reduction in ['sum', 'mean'], 'Entropy reduction must be one of sum and mean.' + self.entropy_reduction = tf.reduce_sum if entropy_reduction == 'sum' else tf.reduce_mean + self.cross_entropy_loss = SparseCategoricalCrossentropy(from_logits=True, reduction=Reduction.NONE) + + self._first_sample = True + + def sample_search(self): + self._initialize() + self._sample(self.mutables) + self._first_sample = False + return self._choices + + def sample_final(self): + return self.sample_search() + + def _sample(self, tree): + mutable = tree.mutable + if isinstance(mutable, LayerChoice) and mutable.key not in self._choices: + self._choices[mutable.key] = self._sample_layer_choice(mutable) + elif isinstance(mutable, InputChoice) and mutable.key not in self._choices: + self._choices[mutable.key] = self._sample_input_choice(mutable) + for child in tree.children: + self._sample(child) + if self.cell_exit_extra_step and isinstance(mutable, MutableScope) and mutable.key not in self._anchors_hid: + self._anchors_hid[mutable.key] = self.lstm(self._inputs, 1) + + def _initialize(self): + self._choices = {} + self._anchors_hid = {} + self._inputs = self.g_emb + # seems the `input_shape` parameter of RNN does not work + # workaround it by omitting `reset_states` for first run + if not self._first_sample: + self.lstm.reset_states() + self.sample_log_prob = 0 + self.sample_entropy = 0 + self.sample_skip_penalty = 0 + + def _sample_layer_choice(self, mutable): + logit = self.soft(self.lstm(self._inputs)) + if self.temperature is not None: + logit /= self.temperature + if self.tanh_constant is not None: + logit = self.tanh_constant * tf.tanh(logit) + if mutable.key in self.bias_dict: + logit += self.bias_dict[mutable.key] + softmax_logit = tf.math.log(tf.nn.softmax(logit, axis=-1)) + branch_id = tf.reshape(tf.random.categorical(softmax_logit, num_samples=1), [1]) + log_prob = self.cross_entropy_loss(branch_id, logit) + self.sample_log_prob += self.entropy_reduction(log_prob) + entropy = log_prob * tf.math.exp(-log_prob) + self.sample_entropy += self.entropy_reduction(entropy) + self._inputs = tf.reshape(self.embedding(branch_id), [1, 1, -1]) + mask = tf.one_hot(branch_id, self.max_layer_choice) + return tf.cast(tf.reshape(mask, [-1]), tf.bool) + + def _sample_input_choice(self, mutable): + query, anchors = [], [] + for label in mutable.choose_from: + if label not in self._anchors_hid: + self._anchors_hid[label] = self.lstm(self._inputs) + query.append(self.attn_anchor(self._anchors_hid[label])) + anchors.append(self._anchors_hid[label]) + query = tf.concat(query, axis=0) + query = tf.tanh(query + self.attn_query(anchors[-1])) + query = self.v_attn(query) + + if self.temperature is not None: + query /= self.temperature + if self.tanh_constant is not None: + query = self.tanh_constant * tf.tanh(query) + + if mutable.n_chosen is None: + logit = tf.concat([-query, query], axis=1) + softmax_logit = tf.math.log(tf.nn.softmax(logit, axis=-1)) + skip = tf.reshape(tf.random.categorical(softmax_logit, num_samples=1), [-1]) + skip_prob = tf.math.sigmoid(logit) + kl = tf.reduce_sum(skip_prob * tf.math.log(skip_prob / self.skip_targets)) + self.sample_skip_penalty += kl + log_prob = self.cross_entropy_loss(skip, logit) + + skip = tf.cast(skip, tf.float32) + inputs = tf.tensordot(skip, tf.concat(anchors, 0), 1) / (1. + tf.reduce_sum(skip)) + self._inputs = tf.reshape(inputs, [1, 1, -1]) + + else: + assert mutable.n_chosen == 1, "Input choice must select exactly one or any in ENAS." + logit = tf.reshape(query, [1, -1]) + softmax_logit = tf.math.log(tf.nn.softmax(logit, axis=-1)) + index = tf.reshape(tf.random.categorical(softmax_logit, num_samples=1), [-1]) + skip = tf.reshape(tf.one_hot(index, mutable.n_candidates), [-1]) + # when the size is 1, tf does not accept tensor here, complaining the shape is wrong + # but using a numpy array seems fine + log_prob = self.cross_entropy_loss(logit, query.numpy()) + self._inputs = tf.reshape(anchors[index.numpy()[0]], [1, 1, -1]) + + self.sample_log_prob += self.entropy_reduction(log_prob) + entropy = log_prob * tf.exp(-log_prob) + self.sample_entropy += self.entropy_reduction(entropy) + assert len(skip) == mutable.n_candidates, (skip, mutable.n_candidates, mutable.n_chosen) + return tf.cast(skip, tf.bool) diff --git a/utils/third_party/nni_new/algorithms/nas/tensorflow/enas/trainer.py b/utils/third_party/nni_new/algorithms/nas/tensorflow/enas/trainer.py new file mode 100644 index 0000000000000000000000000000000000000000..c20f20ab1a68b2c97e6bf27be0f3567016995f1d --- /dev/null +++ b/utils/third_party/nni_new/algorithms/nas/tensorflow/enas/trainer.py @@ -0,0 +1,203 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging + +import tensorflow as tf +from tensorflow.keras.optimizers import Adam + +from nni.nas.tensorflow.utils import AverageMeterGroup, fill_zero_grads + +from .mutator import EnasMutator + +logger = logging.getLogger(__name__) + + +class EnasTrainer: + def __init__( + self, + model, + loss, + metrics, + reward_function, + optimizer, + batch_size, + num_epochs, + dataset_train, + dataset_valid, + log_frequency=100, + entropy_weight=0.0001, + skip_weight=0.8, + baseline_decay=0.999, + child_steps=500, + mutator_lr=0.00035, + mutator_steps=50, + mutator_steps_aggregate=20, + aux_weight=0.4, + test_arc_per_epoch=1, + ): + self.model = model + self.loss = loss + self.metrics = metrics + self.reward_function = reward_function + self.optimizer = optimizer + self.batch_size = batch_size + self.num_epochs = num_epochs + + x, y = dataset_train + split = int(len(x) * 0.9) + self.train_set = tf.data.Dataset.from_tensor_slices((x[:split], y[:split])) + self.valid_set = tf.data.Dataset.from_tensor_slices((x[split:], y[split:])) + self.test_set = tf.data.Dataset.from_tensor_slices(dataset_valid) + + self.log_frequency = log_frequency + self.entropy_weight = entropy_weight + self.skip_weight = skip_weight + self.baseline_decay = baseline_decay + self.child_steps = child_steps + self.mutator_lr = mutator_lr + self.mutator_steps = mutator_steps + self.mutator_steps_aggregate = mutator_steps_aggregate + self.aux_weight = aux_weight + self.test_arc_per_epoch = test_arc_per_epoch + + self.mutator = EnasMutator(model) + self.mutator_optim = Adam(learning_rate=self.mutator_lr) + + self.baseline = 0.0 + + def train(self, validate=True): + for epoch in range(self.num_epochs): + logger.info("Epoch %d Training", epoch + 1) + self.train_one_epoch(epoch) + logger.info("Epoch %d Validating", epoch + 1) + self.validate_one_epoch(epoch) + + def validate(self): + self.validate_one_epoch(-1) + + def train_one_epoch(self, epoch): + train_loader, valid_loader = self._create_train_loader() + + # Sample model and train + meters = AverageMeterGroup() + + for step in range(1, self.child_steps + 1): + x, y = next(train_loader) + self.mutator.reset() + + with tf.GradientTape() as tape: + logits = self.model(x, training=True) + if isinstance(logits, tuple): + logits, aux_logits = logits + aux_loss = self.loss(aux_logits, y) + else: + aux_loss = 0.0 + metrics = self.metrics(y, logits) + loss = self.loss(y, logits) + self.aux_weight * aux_loss + + grads = tape.gradient(loss, self.model.trainable_weights) + grads = fill_zero_grads(grads, self.model.trainable_weights) + grads, _ = tf.clip_by_global_norm(grads, 5.0) + self.optimizer.apply_gradients(zip(grads, self.model.trainable_weights)) + + metrics["loss"] = tf.reduce_mean(loss).numpy() + meters.update(metrics) + + if self.log_frequency and step % self.log_frequency == 0: + logger.info( + "Model Epoch [%d/%d] Step [%d/%d] %s", + epoch + 1, + self.num_epochs, + step, + self.child_steps, + meters, + ) + + # Train sampler (mutator) + meters = AverageMeterGroup() + for mutator_step in range(1, self.mutator_steps + 1): + grads_list = [] + for step in range(1, self.mutator_steps_aggregate + 1): + with tf.GradientTape() as tape: + x, y = next(valid_loader) + self.mutator.reset() + + logits = self.model(x, training=False) + metrics = self.metrics(y, logits) + reward = ( + self.reward_function(y, logits) + + self.entropy_weight * self.mutator.sample_entropy + ) + self.baseline = self.baseline * self.baseline_decay + reward * ( + 1 - self.baseline_decay + ) + loss = self.mutator.sample_log_prob * (reward - self.baseline) + loss += self.skip_weight * self.mutator.sample_skip_penalty + + meters.update( + { + "reward": reward, + "loss": tf.reduce_mean(loss).numpy(), + "ent": self.mutator.sample_entropy.numpy(), + "log_prob": self.mutator.sample_log_prob.numpy(), + "baseline": self.baseline, + "skip": self.mutator.sample_skip_penalty, + } + ) + + cur_step = step + (mutator_step - 1) * self.mutator_steps_aggregate + if self.log_frequency and cur_step % self.log_frequency == 0: + logger.info( + "RL Epoch [%d/%d] Step [%d/%d] [%d/%d] %s", + epoch + 1, + self.num_epochs, + mutator_step, + self.mutator_steps, + step, + self.mutator_steps_aggregate, + meters, + ) + + grads = tape.gradient(loss, self.mutator.trainable_weights) + grads = fill_zero_grads(grads, self.mutator.trainable_weights) + grads_list.append(grads) + total_grads = [ + tf.math.add_n(weight_grads) for weight_grads in zip(*grads_list) + ] + total_grads, _ = tf.clip_by_global_norm(total_grads, 5.0) + self.mutator_optim.apply_gradients( + zip(total_grads, self.mutator.trainable_weights) + ) + + def validate_one_epoch(self, epoch): + test_loader = self._create_validate_loader() + + for arc_id in range(self.test_arc_per_epoch): + meters = AverageMeterGroup() + for x, y in test_loader: + self.mutator.reset() + logits = self.model(x, training=False) + if isinstance(logits, tuple): + logits, _ = logits + metrics = self.metrics(y, logits) + loss = self.loss(y, logits) + metrics["loss"] = tf.reduce_mean(loss).numpy() + meters.update(metrics) + + logger.info( + "Test Epoch [%d/%d] Arc [%d/%d] Summary %s", + epoch + 1, + self.num_epochs, + arc_id + 1, + self.test_arc_per_epoch, + meters.summary(), + ) + + def _create_train_loader(self): + train_set = self.train_set.shuffle(1000000).repeat().batch(self.batch_size) + test_set = self.valid_set.shuffle(1000000).repeat().batch(self.batch_size) + return iter(train_set), iter(test_set) + + def _create_validate_loader(self): + return iter(self.test_set.shuffle(1000000).batch(self.batch_size)) diff --git a/utils/third_party/nni_new/assessor.py b/utils/third_party/nni_new/assessor.py new file mode 100644 index 0000000000000000000000000000000000000000..7cd83e9232735f5aee82e1abca702e347505f4cd --- /dev/null +++ b/utils/third_party/nni_new/assessor.py @@ -0,0 +1,124 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +Assessor analyzes trial's intermediate results (e.g., periodically evaluated accuracy on test dataset) +to tell whether this trial can be early stopped or not. + +See :class:`Assessor`' specification and ``docs/en_US/assessors.rst`` for details. +""" + +from enum import Enum +import logging + +from .recoverable import Recoverable + +__all__ = ['AssessResult', 'Assessor'] + +_logger = logging.getLogger(__name__) + + +class AssessResult(Enum): + """ + Enum class for :meth:`Assessor.assess_trial` return value. + """ + + Good = True + """The trial works well.""" + + Bad = False + """The trial works poorly and should be early stopped.""" + + +class Assessor(Recoverable): + """ + Assessor analyzes trial's intermediate results (e.g., periodically evaluated accuracy on test dataset) + to tell whether this trial can be early stopped or not. + + This is the abstract base class for all assessors. + Early stopping algorithms should inherit this class and override :meth:`assess_trial` method, + which receives intermediate results from trials and give an assessing result. + + If :meth:`assess_trial` returns :obj:`AssessResult.Bad` for a trial, + it hints NNI framework that the trial is likely to result in a poor final accuracy, + and therefore should be killed to save resource. + + If an assessor want's to be notified when a trial ends, it can also override :meth:`trial_end`. + + To write a new assessor, you can reference :class:`~nni.medianstop_assessor.MedianstopAssessor`'s code as an example. + + See Also + -------- + Builtin assessors: + :class:`~nni.algorithms.hpo.medianstop_assessor.MedianstopAssessor` + :class:`~nni.algorithms.hpo.curvefitting_assessor.CurvefittingAssessor` + """ + + def assess_trial(self, trial_job_id, trial_history): + """ + Abstract method for determining whether a trial should be killed. Must override. + + The NNI framework has little guarantee on ``trial_history``. + This method is not guaranteed to be invoked for each time ``trial_history`` get updated. + It is also possible that a trial's history keeps updating after receiving a bad result. + And if the trial failed and retried, ``trial_history`` may be inconsistent with its previous value. + + The only guarantee is that ``trial_history`` is always growing. + It will not be empty and will always be longer than previous value. + + This is an example of how :meth:`assess_trial` get invoked sequentially: + + :: + + trial_job_id | trial_history | return value + ------------ | --------------- | ------------ + Trial_A | [1.0, 2.0] | Good + Trial_B | [1.5, 1.3] | Bad + Trial_B | [1.5, 1.3, 1.9] | Good + Trial_A | [0.9, 1.8, 2.3] | Good + + Parameters + ---------- + trial_job_id : str + Unique identifier of the trial. + trial_history : list + Intermediate results of this trial. The element type is decided by trial code. + + Returns + ------- + AssessResult + :obj:`AssessResult.Good` or :obj:`AssessResult.Bad`. + """ + raise NotImplementedError('Assessor: assess_trial not implemented') + + def trial_end(self, trial_job_id, success): + """ + Abstract method invoked when a trial is completed or terminated. Do nothing by default. + + Parameters + ---------- + trial_job_id : str + Unique identifier of the trial. + success : bool + True if the trial successfully completed; False if failed or terminated. + """ + + def load_checkpoint(self): + """ + Internal API under revising, not recommended for end users. + """ + checkpoin_path = self.get_checkpoint_path() + _logger.info('Load checkpoint ignored by assessor, checkpoint path: %s', checkpoin_path) + + def save_checkpoint(self): + """ + Internal API under revising, not recommended for end users. + """ + checkpoin_path = self.get_checkpoint_path() + _logger.info('Save checkpoint ignored by assessor, checkpoint path: %s', checkpoin_path) + + def _on_exit(self): + pass + + def _on_error(self): + pass diff --git a/utils/third_party/nni_new/common/__pycache__/nas_utils.cpython-38.pyc b/utils/third_party/nni_new/common/__pycache__/nas_utils.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8c132bbdf59dce27a31bdbc5c2c5fcd9990bb350 Binary files /dev/null and b/utils/third_party/nni_new/common/__pycache__/nas_utils.cpython-38.pyc differ diff --git a/utils/third_party/nni_new/common/graph_utils.py b/utils/third_party/nni_new/common/graph_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..1738073305ccf955051b54379f0ad6e46dcccadf --- /dev/null +++ b/utils/third_party/nni_new/common/graph_utils.py @@ -0,0 +1,831 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + + +import logging +import queue +import re +from collections import defaultdict +import torch +from torch.utils.tensorboard._pytorch_graph import NodePy, NodePyIO, NodePyOP, GraphPy +CLASSTYPE_KIND = 'ClassType' +GETATTR_KIND = 'prim::GetAttr' +CAT_KIND = 'aten::cat' +LIST_CONSTRUCT_KIND = 'prim::ListConstruct' +LIST_UNPACK_KIND = 'prim::ListUnpack' +TUPLE_CONSTRUCT_KIND = 'prim::TupleConstruct' +TUPLE_UNPACK_KIND = 'prim::TupleUnpack' +CONSTANT_KIND = 'prim::Constant' + +_logger = logging.getLogger(__name__) + +def _dummy_debug(*args): + return None +_logger.debug = _dummy_debug + + +def build_module_graph(model, dummy_input): + return TorchModuleGraph(model, dummy_input) + + +def build_graph(model, dummy_input, verbose=False): + g = TorchProtoGraph(model, dummy_input, verbose) + return g.graph_def, g.stepstats + + +def parse_traced_name(module_name): + prefix = 'TracedModule[' + suffix = ']' + if module_name.startswith(prefix) and module_name.endswith(suffix): + module_name = module_name[len(prefix):-len(suffix)] + return module_name + + +class TorchGraph: + """ + This class is to extract pytorch model topology graph by tracing + """ + + def __init__(self, model=None, dummy_input=None, traced_model=None): + """ + Parameters + ---------- + model : pytorch model + The model user wants to speed up + dummy_input : pytorch tensor + The dummy input for ```jit.trace```, users should put it on right device before pass in + traced_model : torch._C.torch.jit.TopLevelTracedModule + An alredy traced model, if traced_model is not None, then TorchGraph will build the graph + based on this traced model and won't trace the model again. + """ + assert torch.__version__ >= '1.3.1' + # check if the input is legal + if traced_model is not None: + assert isinstance(traced_model, torch.jit.TopLevelTracedModule) + self.trace = traced_model + # it's ok if the graph is already unpacked + torch._C._jit_pass_inline(self.trace.graph) + elif model is not None and dummy_input is not None: + self.bound_model = model + self._trace(model, dummy_input) + else: + raise Exception( + 'Please provide model & dummy_input or the traced_model as inputs') + + def _trace(self, model, dummy_input): + training = model.training + model.eval() + self.trace = torch.jit.trace(model, dummy_input,strict = False) + torch._C._jit_pass_inline(self.trace.graph) + model.train(training) + + +class TorchProtoGraph(TorchGraph): + """ + Generates model graph for pytorch models in protobuf, this implementation + is borrowed from pytorch v1.4.0, and fixed following issues: + https://github.com/pytorch/pytorch/issues/33691 + https://github.com/pytorch/pytorch/issues/33670 + + """ + + def __init__(self, model, dummy_input, verbose=False): + super().__init__(model, dummy_input) + + from tensorboard.compat.proto.config_pb2 import RunMetadata + from tensorboard.compat.proto.graph_pb2 import GraphDef + from tensorboard.compat.proto.step_stats_pb2 import StepStats, DeviceStepStats + from tensorboard.compat.proto.versions_pb2 import VersionDef + + list_of_nodes = self.parse(self.trace.graph, self.trace, dummy_input) + if verbose: + print(self.trace.graph) + self.stepstats = RunMetadata(step_stats=StepStats( + dev_stats=[DeviceStepStats(device="/device:CPU:0")])) + self.graph_def = GraphDef( + node=list_of_nodes, versions=VersionDef(producer=22)) + + def parse(self, graph, trace, args=None, omit_useless_nodes=True): + """This method parses an optimized PyTorch model graph and produces + a list of nodes and node stats for eventual conversion to TensorBoard + protobuf format. + + Args: + graph (PyTorch module): The model graph to be parsed. + trace (PyTorch JIT TracedModule): The model trace to be parsed. + args (tuple): input tensor[s] for the model. + omit_useless_nodes (boolean): Whether to remove nodes from the graph. + """ + nodes_py = GraphPy() + for node in graph.inputs(): + if omit_useless_nodes: + if not node.uses(): # number of user of the node (= number of outputs/ fanout) + continue + + if node.type().kind() != CLASSTYPE_KIND: + nodes_py.append(NodePyIO(node, 'input')) + + attr_to_scope = dict() + + def node_to_name(d): + return str(d).split(":")[0].strip() + for node in graph.nodes(): + if node.kind() == GETATTR_KIND: + attr_name = node.s('name') + node_name = node_to_name(node) + parent = node.input().node() + # If the parent node is not the top-level "self" node + if parent.kind() == GETATTR_KIND: + parent_scope = attr_to_scope[node_to_name(parent)] + attr_scope = parent_scope.split('/')[-1] + attr_to_scope[node_name] = '{}/{}.{}'.format( + parent_scope, attr_scope, attr_name) + else: + attr_to_scope[node_name] = '__module.{}'.format(attr_name) + # We don't need classtype nodes; scope will provide this information + if node.output().type().kind() != CLASSTYPE_KIND: + node_py = NodePyOP(node) + node_py.scopeName = attr_to_scope[node_name] + nodes_py.append(node_py) + else: + nodes_py.append(NodePyOP(node)) + + # Create sink nodes for output ops + for i, node in enumerate(graph.outputs()): + node_py = NodePyIO(node, 'output') + node_py.debugName = "output.{}".format(i + 1) + node_py.inputs = [node.debugName()] + nodes_py.append(node_py) + + alias_to_name = dict() + base_name = parse_traced_name(trace._name) + for name, module in trace.named_modules(prefix='__module'): + mod_name = parse_traced_name(module._name) + attr_name = name.split('.')[-1] + alias_to_name[name] = '{}[{}]'.format(mod_name, attr_name) + + for node in nodes_py.nodes_op: + module_aliases = node.scopeName.split('/')[-1].split('.') + module_name = '' + for i, alias in enumerate(module_aliases): + if i == 0: + module_name = alias + node.scopeName = base_name + else: + module_name += '.' + alias + node.scopeName += '/' + \ + (alias_to_name[module_name] + if module_name in alias_to_name else alias) + + nodes_py.populate_namespace_from_OP_to_IO() + return nodes_py.to_proto() + + +class NodePyGroup(NodePy): + """ + This class is used to represent a graph node which consists of multiple jit traced nodes. In a pytorch trace graph, + there are multiple nodes are traced for one torch.nn.Module object, we group them together to form a single node to + represent the torch.nn.Module object. We also group some functional call trace nodes together to form a new node. + """ + + def __init__(self, name, unique_name, node_type, op_type, node_cpps, inputs=None, outputs=None, key_node=None): + """ + Parameters: + ----------- + name: str + node name, such as `conv1`, `backbone.classifier` + unique_name: str + A global unique name for current node. Due to some modules, + such as relu, may be reused several times, so the scopename + is not suitable as the global unique identifier, so we add a + unique_name for each node as the global unique identifier. + We should use the unique_name to traverset the module graph. + node_type: str + `module` or `func` + op_type: str + operation type, such as `Conv2d`, `aten::view` + node_cpps: list of torch._C.Node + jit trace nodes which are included in this new node + inputs: list of str + All the inputs of this node, each element is debugName of one input + outputs: list of str + All the outputs of this node, each element is debugName of one output + key_node: torch._C.Node + The key node of this NodePyGroup. + """ + super(NodePyGroup, self).__init__(name, []) + self.node_cpps = node_cpps + self.name = name + self.unique_name = unique_name + self.op_type = op_type + self.type = node_type + self.nodes = [] + self.auxiliary = None + self.add_nodes(node_cpps) + self.inputs = inputs + self.outputs = outputs + # The core node in this NodePyGroup + self.key_node = key_node + + def add_nodes(self, node_cpps): + for node_cpp in node_cpps: + nodepy = NodePyOP(node_cpp) + nodepy.name = node_cpp.scopeName() + '_' + node_cpp.kind() + self.nodes.append(nodepy) + + def sub_node_names(self): + return [x.name for x in self.nodes] + + def __repr__(self): + return 'name: {}, type: {}, op_type: {}, sub_nodes: {}, inputs: {}, outputs: {}, aux: {}'.format( + self.name, self.type, self.op_type, self.sub_node_names(), + self.inputs, self.outputs, self.auxiliary + ) + + +class TorchModuleGraph(TorchGraph): + """ + Generates model graph, each node is created from single or multiple jit trace nodes. + """ + + def __init__(self, model=None, dummy_input=None, traced_model=None): + super().__init__(model, dummy_input, traced_model) + self.global_count = 0 + self.name_to_node, self.input_to_node, self.output_to_node = self._build_graph() + self._extract_auxiliary_info() + + def _expand_key_func_node(self, node, nodes, input_to_node, output_to_node, + module_type): + """ + For trace graph nodes, some nodes are not in modules, these nodes are usually generated by + the functions directly called in module ```forward```. For such nodes, some of them are + trivial op which are label by ```prim::```, some of them are not such ops which is call + non-prim ops. This function is to merge neighbor prim ops to a non-prim op, to construct + a node. + + Parameters + ---------- + node : trace graph node + The non-prim node to expand + nodes : list of trace graph node + All the trace graph nodes within the same scope as the non-prim node + input_to_node : dict + key: input name, value: a node that uses this input + output_to_node : dict + key: output name, value: a node that generates this output + module_type : str + can be 'module' or 'func' + + Returns + ------- + node + the expanded non-prim node + """ + # TODO: scope name could be empty + node_name = '.'.join([self._get_module_name( + node.scopeName()), node.kind(), str(self.global_count)]) + unique_name = node_name + _logger.debug("expand non-prim node, node name: %s", node_name) + self.global_count += 1 + op_type = node.kind() + node_group = [node] + inputs = [] + outputs = [] + node_queue = queue.Queue() + node_queue.put(node) + while not node_queue.empty(): + curr_node = node_queue.get() + for _input in curr_node.inputs(): + if _input.node().kind() == CONSTANT_KIND: + continue + input_name = _input.debugName() + if input_name in output_to_node: + for predecessor_node in output_to_node[input_name]: + if predecessor_node in nodes: + if not self._is_key_func(predecessor_node): + if predecessor_node not in node_group: + node_group.append(predecessor_node) + node_queue.put(predecessor_node) + else: + inputs.append(input_name) + else: + inputs.append(input_name) + else: + inputs.append(input_name) + for output in node.outputs(): + if output.node().kind() == CONSTANT_KIND: + continue + outputs.append(output.debugName()) + nodepy = NodePyGroup(node_name, unique_name, module_type, op_type, + node_group, inputs=inputs, outputs=outputs, key_node=node) + return nodepy + + def _expand_module_node(self, node, node_name, unique_name, op_type, nodes, + input_to_node, output_to_node, module_type): + """ + merge the adjacent nodes of the module. The difference between the + _expand_module_node and _expand_non_prim_node is that, the _expand_non_prim_node + only merge the prim:: nodes into the aten:: node, in contrast,the _expand_module_node + will merge all adjacent nodes into a same nodepy group. + + Parameters + ---------- + node : trace graph node + The non-prim node to expand + node_name : str + specify the node_name for NodePyGroup + unique_name : str + unique_name for the NodePyGroup + op_type : str + specify the op_type for the NodePyGroup + nodes : list of trace graph node + All the trace graph nodes within the same scope as the non-prim node + input_to_node : dict + key: input name, value: a node that uses this input + output_to_node : dict + key: output name, value: a node that generates this output + module_type : str + can be 'module' or 'func' + Returns + ------- + node + the expanded non-prim node + + """ + _logger.debug("expand module node, node name: %s", node_name) + self.global_count += 1 + if not op_type: + op_type = node.kind() + node_group = [node] + inputs = [] + outputs = [] + node_queue = queue.Queue() + node_queue.put(node) + visited = {node} + while not node_queue.empty(): + curr_node = node_queue.get() + for _input in curr_node.inputs(): + if _input.node().kind() == CONSTANT_KIND: + continue + input_name = _input.debugName() + if input_name in output_to_node: + for predecessor_node in output_to_node[input_name]: + if predecessor_node in nodes: + if predecessor_node not in visited: + node_group.append(predecessor_node) + node_queue.put(predecessor_node) + visited.add(predecessor_node) + else: + inputs.append(input_name) + else: + inputs.append(input_name) + for _output in curr_node.outputs(): + if _output.node().kind() == CONSTANT_KIND: + continue + output_name = _output.debugName() + if output_name in input_to_node: + for successor_node in input_to_node[output_name]: + if successor_node in nodes: + if successor_node not in visited: + node_group.append(successor_node) + node_queue.put(successor_node) + visited.add(successor_node) + else: + outputs.append(output_name) + else: + outputs.append(output_name) + + nodepy = NodePyGroup(node_name, unique_name, module_type, op_type, + node_group, inputs=list(inputs), outputs=list(outputs)) + return nodepy + + def _extract_cat_info(self, node_group, cpp_node): + """ + Extract the detail information of the cat operation, + such the order of the input tensor, the shape of each + input tensor, the output shape, and the cat dimension. + + Parameters + ---------- + node_group : NodePyGroup + cpp_node: torch._C.Node + It should be ```aten::cat``` node + + Returns + ------- + dict + Include auxiliary information for the cat operation. + This dict objec has four keys: 'cat_dim', 'out_shape', + 'in_order' and 'in_shape'. cat_dim is the dimension of + the cat operation to concat the input tensors. out_shape + is the shape of the output tensor of the cat operation. + in_order is an ordered list which contains the corresponding + parent operaion nodes of the input tensors. in_shape is also + an ordered list that contains the input shapes of the input + tensor. + """ + # only suport the cat operation + assert cpp_node.kind() == CAT_KIND + cat_info = {} + # get the shape of the output tensor + t_output = cpp_node.output() + out_shape = t_output.type().sizes() + cat_info['out_shape'] = out_shape + # get the cat dimension + inputs = cpp_node.inputs() + cat_dim = list(inputs)[1].toIValue() + cat_info['cat_dim'] = cat_dim + # get the order of the input tensors + # To get the order of the input tensors, we need + # to be aware of the topology of the model, which + # means we should extract the auxiliary information + # after the build_index function. + input_order = [] + list_construct_cpp = list(cpp_node.inputs())[0].node() + input_tensors = list(list_construct_cpp.inputs()) + for _tensor in input_tensors: + debug_name = _tensor.debugName() + if debug_name in self.output_to_node: + input_order.append(self.output_to_node[debug_name].unique_name) + else: + # the input tensor may be the input tensor of the whole model + input_order.append(None) + cat_info['in_order'] = input_order + input_shapes = [t.type().sizes() for t in input_tensors] + cat_info['in_shape'] = input_shapes + return cat_info + + def _extract_linear_shape_info(self, node_group): + """ + Extract linear shape input/output tensor shape info from its aten::addmm op. + + Parameters + ---------- + node_group : NodePyGroup + NodePyGroup object associated with the linear module. + + Returns + ------- + dict + Include shape of input tensor and shape of output tensor + """ + for cpp_node in node_group.node_cpps: + if cpp_node.kind() == 'aten::addmm': + # https://github.com/pytorch/pytorch/blob/1.6/torch/nn/functional.py#L1682 + # inputs of aten::addmm: + # inputs[0] is bias + # inputs[1] is input data + # inputs[2] is weight + t_input = list(cpp_node.inputs())[1] + t_output = cpp_node.output() + assert isinstance(t_input.type(), torch._C.TensorType) + assert isinstance(t_output.type(), torch._C.TensorType) + in_shape = t_input.type().sizes() + out_shape = t_output.type().sizes() + return {'in_shape': in_shape, 'out_shape': out_shape} + return None + + def _extract_shape_info(self, node): + """ + Extract the shape information of ```aten::view``` node + + Parameters + ---------- + node : trace graph node + It should be ```aten::view``` node + + Returns + ------- + dict + Include shape of input tensor and shape of output tensor + """ + t_input = None + for _input in node.inputs(): + t_input = _input + break + t_output = node.output() + assert isinstance(t_input.type(), torch._C.TensorType) + assert isinstance(t_output.type(), torch._C.TensorType) + in_shape = t_input.type().sizes() + out_shape = t_output.type().sizes() + return {'in_shape': in_shape, 'out_shape': out_shape} + + def _extract_leaf_modules(self): + """ + Extract leaf modules from the given graph. Leaf module means it does not have submodules. + To extract leaf modules because only leaf module can be replaced. And shape inference can + be done in leaf module level. Other shape inference is done in lower level i.e., + operation level. + + Returns + ------- + list + a list of scope name of all the leaf modules + """ + def is_parent(name1, name2): + """ + check if name1 is parent node of name2, for example: + name1: aa.bb, name2: aa.bb.cc, return True + name1: aa.b, name2: aa.bb, return False + """ + parts1, parts2 = name1.split('.'), name2.split('.') + if len(parts1) >= len(parts2): + return False + for i, _ in enumerate(parts1): + if parts2[i] != parts1[i]: + return False + return True + module_names = sorted([x[0] + for x in self.trace.named_modules() if x[0]]) + leaf_nodes = [] + for i, name in enumerate(module_names): + if i + 1 >= len(module_names) or not is_parent(name, module_names[i + 1]): + leaf_nodes.append(name) + return leaf_nodes + + def _get_module_name(self, scope_name): + """ + Retrieve module name from scope name. + Parameters: + ----------- + scope_name: str + scope_name of a graph node, for example: + for pytorch 1.3.1: MyModel/BackboneModel[backbone]/Conv2d[conv2] + for pytorch 1.4.0: __module.backbone/__module.backbone.conv2 + + Returns: + ------- + str + module name, such as backbone.conv2 + """ + if torch.__version__ >= '1.4.0': + return scope_name.split('/')[-1].replace('__module.', '') + else: + return '.'.join(re.findall(r'\[(.*?)\]', scope_name)) + + def _build_index(self, nodes_op): + name_to_node = dict() + input_to_node = defaultdict(list) + output_to_node = dict() + for node in nodes_op: + name_to_node[node.unique_name] = node + for _input in node.inputs: + # inputs may have duplicate tensors + if node not in input_to_node[_input]: + input_to_node[_input].append(node) + for output in node.outputs: + if output in output_to_node: + assert output_to_node[output] == node, \ + "One output cannot be generated by multiple nodes %s" % output + output_to_node[output] = node + return name_to_node, input_to_node, output_to_node + + def _is_key_func(self, node_cpp): + """ + Judge if a cpp node is a key function node. + If so, we should not merge this node into the + adjacent node. + """ + if node_cpp.kind().startswith('aten::'): + # the nodes that start with 'aten' are key function + # nodes + return True + if node_cpp.kind() in [LIST_UNPACK_KIND, TUPLE_UNPACK_KIND]: + # We cannot merge the List/Tuple + # Unpack func into other nodes, else it + # may lead to a graph construction error. + # The reason why we donnot take the construct node + # also as a key node is that `cat` operation node need + # the last(previous) visited node to infer the mask. If + # we take the Construct node as the important node, the + # predecessor of the `cat` node will always be a construct + # node, which means we cannot infer the mask for the cat + # operation. + return True + return False + + def unpack_manually(self): + """ + Unpack the tensor tuple or tensor list manually, + and remove the ListUnpack/TupleUnpack node from + the graph. Note: this function will change the + graph structure. + """ + if hasattr(self, 'unpacked'): + # if already unpacked the tuple/list manually + return + for node in self.nodes_py.nodes_op: + if node.op_type in [TUPLE_UNPACK_KIND, LIST_UNPACK_KIND]: + unpack_cpp = node.key_node + last_cpp = list(unpack_cpp.inputs())[0].node() + if last_cpp.kind() in [TUPLE_CONSTRUCT_KIND, LIST_CONSTRUCT_KIND]: + # we need check if the tensor tuple or tensor list is produced + # by a list/tuple construct node. If so, we can unpack the tuple + # or list manunally. + _logger.debug('List/Tuple Construct Node(cpp) %s', str(last_cpp)) + _logger.debug('List/Tuple Unpack Node(cpp) %s', str(unpack_cpp)) + assert len(list(unpack_cpp.outputs())) == len(list(last_cpp.inputs())) + errmsg = '%s Input number: %d if inconsistent with the output number %d' % (unpack_cpp, \ + len(node.inputs), len(list(last_cpp.inputs()))) + + assert len(node.inputs) == len(list(last_cpp.inputs())), errmsg + for _debug_input, _debug_output in zip(node.inputs, node.outputs): + if _debug_input in self.input_to_node and _debug_output in self.input_to_node: + # input_to_node[_debug_input] is a list of NodePyGroup, because + # one tensor can be used as input for multiple nodes at the same time. + + # note that, in this case, the construct cpp node and unpack cpp node + # will be merged into the same NodePyGroup, so we remove the `node` from + # input_to_node[_debug_input] and directly connect this tensor to the + # input_to_node[_debug_output] + if node in self.input_to_node[_debug_input]: + self.input_to_node[_debug_input].remove(node) + # add the following nodes of _output into the input_to_node[_debug_input] + self.input_to_node[_debug_input].extend(self.input_to_node[_debug_output]) + # just remove the _debug_output from the grapgh index. So that we can also skip + # the construct and tuple + if _debug_output in self.input_to_node: + for following_node in self.input_to_node[_debug_output]: + _tmp_index = following_node.inputs.index(_debug_output) + following_node.inputs[_tmp_index] = _debug_input + + + self.unpacked = True + + def _build_graph(self): + """ + Build graph using our defined format from jit trace. + There are basically three steps: first, construct necessary information (data structures), + second, extract all the modules to convert to node, Third, extract all functions to convert + to node. + + Returns + ------- + dict + use name to index nodes, key: node name, value: node + dict + use input (its name) to index nodes, + key: input, value: list of nodes that take this input + dict + use output (its name) to index nodes, + key: output, value: node that generates this output + """ + omit_useless_nodes = True + graph = self.trace.graph + _logger.debug(graph) + # build input/output mapping, from input/output debugName to its node + input_to_node = defaultdict(list) + output_to_node = defaultdict(list) + for node in graph.nodes(): + if node.kind() == CONSTANT_KIND: + continue + for x in node.outputs(): + if x.node().kind() == CONSTANT_KIND: + continue + output_to_node[x.debugName()].append(node) + assert len(output_to_node[x.debugName()]) <= 1, "One output cannot be generated by multiple nodes %s" % x.debugName() + for x in node.inputs(): + if x.node().kind() == CONSTANT_KIND: + continue + input_to_node[x.debugName()].append(node) + + # build module mapping, from module name to all nodes (as list) under this module scope + module_to_nodes = defaultdict(list) + # the mapping of function (non-module in forward) to nodes, key is scope name + func_to_nodes = defaultdict(list) + + nodes_py = GraphPy() + for node in graph.inputs(): + if omit_useless_nodes: + if not node.uses(): # number of user of the node (= number of outputs/ fanout) + continue + + if node.type().kind() != 'ClassType': + nodes_py.append(NodePyIO(node, 'input')) + + self.leaf_modules = self._extract_leaf_modules() + module_to_type = {name: parse_traced_name( + module._name) for name, module in self.trace.named_modules()} + + # associate module name with their trace graph nodes + for node in graph.nodes(): + if node.kind() == CONSTANT_KIND: + continue + module_name = self._get_module_name(node.scopeName()) + if module_name in self.leaf_modules: + module_to_nodes[module_name].append(node) + else: + func_to_nodes[node.scopeName()].append(node) + # build node group for module + for module_name, node_cpps in module_to_nodes.items(): + use_count = 0 + merged = set() + for node in node_cpps: + if node not in merged: + # modules that have same scope name may have different locations in the + # graph. Futhermore, there are also lots of prim:: nodes that in node_cpps, + # so we also need to call the expand_module_node. + unique_name = module_name + if use_count > 0: + unique_name = module_name + '.%d' % use_count + node_group = self._expand_module_node( + node, module_name, unique_name, module_to_type[module_name], + node_cpps, input_to_node, output_to_node, 'module') + nodes_py.nodes_op.append(node_group) + use_count += 1 + merged.update(node_group.node_cpps) + + # each scope_name may have multiple funcs, we split them and create node for each of them + # build node group for torch.nn.functional + for _, nodes in func_to_nodes.items(): + # extract non prim:: nodes + key_func_nodes = list() + for node in nodes: + if self._is_key_func(node): + # find the key function nodes + key_func_nodes.append(node) + # for each non prim node, expand it + for node in key_func_nodes: + node_group = self._expand_key_func_node( + node, nodes, input_to_node, output_to_node, 'func') + nodes_py.nodes_op.append(node_group) + # get shape infor for view (aten::view) func + # if node_group.op_type in ['aten::view', 'aten::flatten']: + # node_group.auxiliary = self._extract_shape_info(node) + + for node in graph.outputs(): # Create sink nodes for output ops + node_py = NodePyIO(node, 'output') + nodes_py.append(node_py) + + self.nodes_py = nodes_py + # build index + return self._build_index(self.nodes_py.nodes_op) + + def _extract_auxiliary_info(self): + """ + Extract the auxiliary information for the nodegroups + if necessary. For example, view/flatten operations may + need the shape of the input tensor and output tensor. + """ + # extract the input & output shape for the view and flatten + for node_group in self.nodes_py.nodes_op: + if node_group.op_type in ['aten::view', 'aten::flatten', 'aten::mean', 'aten::reshape']: + # get shape infor for view (aten::view) func + cpp_node = list(filter(lambda x: x.kind() == node_group.op_type, + node_group.node_cpps))[0] + node_group.auxiliary = self._extract_shape_info(cpp_node) + elif node_group.op_type == 'Linear': + node_group.auxiliary = self._extract_linear_shape_info(node_group) + elif node_group.op_type == CAT_KIND: + # get the detail information for cat func + cpp_node = list(filter(lambda x: x.kind() == node_group.op_type, + node_group.node_cpps))[0] + node_group.auxiliary = self._extract_cat_info( + node_group, cpp_node) + + def find_predecessors(self, unique_name): + """ + Find predecessor node of the given node + + Parameters + ---------- + unique_name : str + The unique name of the node + + Returns + ------- + list + a list of nodes who are the given node's predecessor + """ + predecessors = [] + for _input in self.name_to_node[unique_name].inputs: + if not _input in self.output_to_node: + _logger.debug("cannot find node with %s as its output", _input) + else: + node_py = self.output_to_node[_input] + predecessors.append(node_py.unique_name) + return predecessors + + def find_successors(self, unique_name): + """ + Find successor nodes of the given node + + Parameters + ---------- + unique_name : str + The unique name of the node + + Returns + ------- + list + a list of nodes who are the given node's successor + """ + successors = [] + for output in self.name_to_node[unique_name].outputs: + if output not in self.input_to_node: + # may reach the output of the whole graph + continue + nodes_py = self.input_to_node[output] + for node_py in nodes_py: + successors.append(node_py.unique_name) + return successors diff --git a/utils/third_party/nni_new/common/nas_utils.py b/utils/third_party/nni_new/common/nas_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..d7f050ec9bb2046b782bf699e1bd3ae7925a31a1 --- /dev/null +++ b/utils/third_party/nni_new/common/nas_utils.py @@ -0,0 +1,317 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import functools +import logging + +from .. import trial + + +_logger = logging.getLogger(__name__) +_MUTABLE_LAYER_SPACE_PREFIX = "_mutable_layer" +_namespace = {} +_tf_variables = {} +_arch_logits_list = [] +_optimizer = None +_train_op = None + + +def classic_mode( + mutable_id, + mutable_layer_id, + funcs, + funcs_args, + fixed_inputs, + optional_inputs, + optional_input_size): + '''Execute the chosen function and inputs directly. + In this mode, the trial code is only running the chosen subgraph (i.e., the chosen ops and inputs), + without touching the full model graph.''' + if trial.get_current_parameter() is None: + trial.get_next_parameter() + + chosen_layer, chosen_inputs = _get_layer_and_inputs_from_tuner(mutable_id, mutable_layer_id, + list(optional_inputs.keys())) + real_chosen_inputs = [optional_inputs[input_name] for input_name in chosen_inputs] + layer_out = funcs[chosen_layer]([fixed_inputs, real_chosen_inputs], **funcs_args[chosen_layer]) + + return layer_out + + +def enas_mode( + mutable_id, + mutable_layer_id, + funcs, + funcs_args, + fixed_inputs, + optional_inputs, + optional_input_size, + tf): + '''For enas mode, we build the full model graph in trial but only run a subgraph。 + This is implemented by masking inputs and branching ops. + Specifically, based on the received subgraph (through nni.get_next_parameter), + it can be known which inputs should be masked and which op should be executed.''' + name_prefix = "{}_{}".format(mutable_id, mutable_layer_id) + # store namespace + _namespace[mutable_id] = True + _namespace[name_prefix] = dict() + _namespace[name_prefix]['funcs'] = list(funcs) + _namespace[name_prefix]['optional_inputs'] = list(optional_inputs) + # create tensorflow variables as 1/0 signals used to form subgraph + name_for_optional_inputs = name_prefix + '_optional_inputs' + name_for_funcs = name_prefix + '_funcs' + _tf_variables[name_prefix] = dict() + _tf_variables[name_prefix]['optional_inputs'] = tf.get_variable( + name_for_optional_inputs, + [len(optional_inputs)], + dtype=tf.bool, + trainable=False + ) + _tf_variables[name_prefix]['funcs'] = tf.get_variable( + name_for_funcs, [], dtype=tf.int64, trainable=False) + + # get real values using their variable names + real_optional_inputs_value = [optional_inputs[name] + for name in _namespace[name_prefix]['optional_inputs']] + real_func_value = [funcs[name] + for name in _namespace[name_prefix]['funcs']] + real_funcs_args = [funcs_args[name] + for name in _namespace[name_prefix]['funcs']] + # build tensorflow graph of geting chosen inputs by masking + real_chosen_inputs = tf.boolean_mask( + real_optional_inputs_value, _tf_variables[name_prefix]['optional_inputs']) + # build tensorflow graph of different branches by using tf.case + branches = dict() + func_output = None + for func_id in range(len(funcs)): + func_output = real_func_value[func_id]([fixed_inputs, real_chosen_inputs], **real_funcs_args[func_id]) + branches[tf.equal(_tf_variables[name_prefix]['funcs'], func_id)] = lambda: func_output + layer_out = tf.case(branches, exclusive=True, default=lambda: func_output) + + return layer_out + + +def oneshot_mode( + mutable_id, + mutable_layer_id, + funcs, + funcs_args, + fixed_inputs, + optional_inputs, + optional_input_size, + tf): + '''Similar to enas mode, oneshot mode also builds the full model graph. + The difference is that oneshot mode does not receive subgraph. + Instead, it uses dropout to randomly dropout inputs and ops.''' + # NNI requires to get_next_parameter before report a result. But the parameter will not be used in this mode + if trial.get_current_parameter() is None: + trial.get_next_parameter() + optional_inputs = list(optional_inputs.values()) + inputs_num = len(optional_inputs) + # Calculate dropout rate according to the formular r^(1/k), where r is a hyper-parameter and k is the number of inputs + if inputs_num > 0: + rate = 0.01 ** (1 / inputs_num) + noise_shape = [inputs_num] + [1] * len(optional_inputs[0].get_shape()) + optional_inputs = tf.nn.dropout( + optional_inputs, rate=rate, noise_shape=noise_shape) + optional_inputs = [optional_inputs[idx] for idx in range(inputs_num)] + layer_outs = [func([fixed_inputs, optional_inputs], **funcs_args[func_name]) + for func_name, func in funcs.items()] + output_num = len(layer_outs) + rate = 0.01 ** (1 / output_num) + noise_shape = [output_num] + [1] * len(layer_outs[0].get_shape()) + layer_outs = tf.nn.dropout(layer_outs, rate=rate, noise_shape=noise_shape) + layer_out = tf.reduce_sum(layer_outs, axis=0) + + return layer_out + + +def darts_mode( + mutable_id, + mutable_layer_id, + funcs, + funcs_args, + fixed_inputs, + optional_inputs, + optional_input_size, + tf): + optional_inputs = list(optional_inputs.values()) + layer_outs = [func([fixed_inputs, optional_inputs], **funcs_args[func_name]) + for func_name, func in funcs.items()] + # Create architecture weights for every func(op) + var_name = "{}_{}_arch_weights".format(mutable_id, mutable_layer_id) + arch_logits = tf.get_variable(var_name, shape=[len(funcs)], trainable=False) + _arch_logits_list.append(arch_logits) + arch_weights = tf.nn.softmax(arch_logits) + layer_out = tf.add_n([arch_weights[idx] * out for idx, out in enumerate(layer_outs)]) + + return layer_out + + +def reload_tensorflow_variables(tf, session): + '''In Enas mode, this function reload every signal varaible created in `enas_mode` function so + the whole tensorflow graph will be changed into certain subgraph recerived from Tuner. + --------------- + session: the tensorflow session created by users + tf: tensorflow module + ''' + subgraph_from_tuner = trial.get_next_parameter() + mutable_layers = set() + for subgraph_key in subgraph_from_tuner: + if "/" in subgraph_key: + # has to remove the last, could be layer_choice or whatever + mutable_id, mutable_layer_id = _decompose_general_key(subgraph_key[:subgraph_key.rfind("/")]) + if mutable_id is not None: + mutable_layers.add((mutable_id, mutable_layer_id)) + mutable_layers = sorted(list(mutable_layers)) + for mutable_id, mutable_layer_id in mutable_layers: + if mutable_id not in _namespace: + _logger.warning("%s not found in name space", mutable_id) + continue + name_prefix = "{}_{}".format(mutable_id, mutable_layer_id) + # get optional inputs names + optional_inputs = _namespace[name_prefix]['optional_inputs'] + # extract layer information from the subgraph sampled by tuner + chosen_layer, chosen_inputs = _get_layer_and_inputs_from_tuner(mutable_id, mutable_layer_id, optional_inputs) + chosen_layer = _namespace[name_prefix]['funcs'].index(chosen_layer) + chosen_inputs = [1 if inp in chosen_inputs else 0 for inp in optional_inputs] + # load these information into pre-defined tensorflow variables + _tf_variables[name_prefix]['funcs'].load(chosen_layer, session) + _tf_variables[name_prefix]['optional_inputs'].load( + chosen_inputs, session) + + +def _construct_general_key(mutable_id, mutable_layer_id): + # Mutable layer key in a general (search space) format + # that is, prefix/mutable_id/mutable_layer_id + return _MUTABLE_LAYER_SPACE_PREFIX + "/" + mutable_id + "/" + mutable_layer_id + + +def _decompose_general_key(key): + # inverse operation of above + if not key.startswith(_MUTABLE_LAYER_SPACE_PREFIX): + return None, None + else: + _, mutable_id, mutable_layer_id = key.split("/", maxsplit=2) + return mutable_id, mutable_layer_id + + +def darts_training(tf, session, loss, feed_dict): + global _optimizer, _train_op + if _optimizer is None: + _optimizer = tf.MomentumOptimizer(learning_rate=0.025) + # TODO: Calculate loss + grads_and_vars = _optimizer.compute_gradients(loss, _arch_logits_list) + _train_op = _optimizer.apply_gradients(grads_and_vars) + session.run(_train_op) + + +def training_update(nas_mode, tf=None, session=None, loss=None, feed_dict=None): + if nas_mode == 'darts_mode': + darts_training(tf, session, loss, feed_dict) + elif nas_mode == 'enas_mode': + reload_tensorflow_variables(tf, session) + + +def _get_layer_and_inputs_from_tuner(mutable_id, mutable_layer_id, optional_inputs): + # optional_inputs should be name(key)s of the optional inputs + try: + mutable_block = trial.get_current_parameter(mutable_id) + + # There is a NAS tuner + chosen_layer = mutable_block[mutable_layer_id]["chosen_layer"] + chosen_inputs = mutable_block[mutable_layer_id]["chosen_inputs"] + except KeyError: + # Try to find converted NAS parameters + params = trial.get_current_parameter() + expected_prefix = _construct_general_key(mutable_id, mutable_layer_id) + chosen_layer = params[expected_prefix + "/layer_choice"] + + # find how many to choose + optional_input_size = int(params[expected_prefix + "/optional_input_size"]) # convert uniform to randint + + # find who to choose, can duplicate + optional_input_state = params[expected_prefix + "/optional_input_chosen_state"] + chosen_inputs = [] + # make sure dict -> list produce stable result by sorting + optional_inputs_keys = sorted(optional_inputs) + for _ in range(optional_input_size): + chosen_inputs.append(optional_inputs_keys[optional_input_state % len(optional_inputs)]) + optional_input_state //= len(optional_inputs) + + _logger.info("%s_%s: layer: %s, optional inputs: %s", mutable_id, mutable_layer_id, chosen_layer, chosen_inputs) + return chosen_layer, chosen_inputs + + +def convert_nas_search_space(search_space): + """ + Args: + param search_space: raw search space + return: the new search space, mutable_layers will be converted into choice + """ + if not isinstance(search_space, dict): + return search_space + ret = dict() + for k, v in search_space.items(): + if "_type" not in v: + # this should not happen + _logger.warning("There is no _type in one of your search space values with key '%s'" + ". Please check your search space", k) + ret[k] = v + elif v["_type"] != "mutable_layer": + ret[k] = v + else: + _logger.info("Converting mutable_layer search space with key '%s'", k) + # v["_value"] looks like {'mutable_layer_1': {'layer_choice': ...} ...} + values = v["_value"] + for layer_name, layer_data in values.items(): + # there should be at most layer_choice, optional_inputs, optional_input_size in layer_data + + # add "_mutable_layer" as prefix so that they can be recovered later + layer_key = _construct_general_key(k, layer_name) + + if layer_data.get("layer_choice"): # filter out empty choice and no choice + layer_choice = layer_data["layer_choice"] + else: + raise ValueError("No layer choice found in %s" % layer_key) + + if layer_data.get("optional_input_size"): + input_size = layer_data["optional_input_size"] + if isinstance(input_size, int): + input_size = [input_size, input_size] + if input_size[0] > input_size[1] or input_size[0] < 0: + _logger.error("Might not be able to handle optional_input_size < 0, please double check") + input_size[1] += 1 + else: + _logger.info("Optional input choices are set to empty by default in %s", layer_key) + input_size = [0, 1] + + if layer_data.get("optional_inputs"): + total_state_size = len(layer_data["optional_inputs"]) ** (input_size[1] - 1) + else: + _logger.info("Optional inputs not found in %s", layer_key) + total_state_size = 1 + + converted = { + layer_key + "/layer_choice": { + "_type": "choice", "_value": layer_choice + }, + layer_key + "/optional_input_size": { + "_type": "randint", "_value": input_size + }, + layer_key + "/optional_input_chosen_state": { + "_type": "randint", "_value": [0, total_state_size] + } + } + _logger.info(converted) + ret.update(converted) + + return ret + + +def rewrite_nas_space(func): + @functools.wraps(func) + def wrap(self, search_space): + search_space = convert_nas_search_space(search_space) + return func(self, search_space) + return wrap diff --git a/utils/third_party/nni_new/compression/__init__.py b/utils/third_party/nni_new/compression/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/utils/third_party/nni_new/compression/__pycache__/__init__.cpython-38.pyc b/utils/third_party/nni_new/compression/__pycache__/__init__.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1b13c9675f6fd8742737bfddfcf1da579ff53062 Binary files /dev/null and b/utils/third_party/nni_new/compression/__pycache__/__init__.cpython-38.pyc differ diff --git a/utils/third_party/nni_new/compression/pytorch/__init__.py b/utils/third_party/nni_new/compression/pytorch/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..10e2fd050de5630d75be2432fc50aaf95cf86b40 --- /dev/null +++ b/utils/third_party/nni_new/compression/pytorch/__init__.py @@ -0,0 +1,6 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .speedup import ModelSpeedup +from .compressor import Compressor, Pruner, Quantizer +from .pruning import apply_compression_results diff --git a/utils/third_party/nni_new/compression/pytorch/__pycache__/__init__.cpython-38.pyc b/utils/third_party/nni_new/compression/pytorch/__pycache__/__init__.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5c15aea60239c1be01572e2c498bb0ce263da608 Binary files /dev/null and b/utils/third_party/nni_new/compression/pytorch/__pycache__/__init__.cpython-38.pyc differ diff --git a/utils/third_party/nni_new/compression/pytorch/__pycache__/compressor.cpython-38.pyc b/utils/third_party/nni_new/compression/pytorch/__pycache__/compressor.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..97367a4ddd37e2e7727cc498ccbc33cc7dd257ac Binary files /dev/null and b/utils/third_party/nni_new/compression/pytorch/__pycache__/compressor.cpython-38.pyc differ diff --git a/utils/third_party/nni_new/compression/pytorch/__pycache__/default_layers.cpython-38.pyc b/utils/third_party/nni_new/compression/pytorch/__pycache__/default_layers.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2f4dfa8f3438b3cf42bb0e1c8a10b0f93f15489e Binary files /dev/null and b/utils/third_party/nni_new/compression/pytorch/__pycache__/default_layers.cpython-38.pyc differ diff --git a/utils/third_party/nni_new/compression/pytorch/compressor.py b/utils/third_party/nni_new/compression/pytorch/compressor.py new file mode 100644 index 0000000000000000000000000000000000000000..70dbd03f291eaebaf417ce8114545fd80a2ead74 --- /dev/null +++ b/utils/third_party/nni_new/compression/pytorch/compressor.py @@ -0,0 +1,799 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import types +import logging +import torch +from . import default_layers + +_logger = logging.getLogger(__name__) + +def _dummy_debug(*args): + return None +_logger.debug = _dummy_debug + +class LayerInfo: + def __init__(self, name, module): + self.module = module + self.name = name + self.type = type(module).__name__ + +def _setattr(model, name, module): + name_list = name.split(".") + for name in name_list[:-1]: + model = getattr(model, name) + setattr(model, name_list[-1], module) + +class Compressor: + """ + Abstract base PyTorch compressor + """ + + def __init__(self, model, config_list, optimizer=None): + """ + Record necessary info in class members + + Parameters + ---------- + model : pytorch model + the model user wants to compress + config_list : list + the configurations that users specify for compression + optimizer: pytorch optimizer + optimizer used to train the model + """ + assert isinstance(model, torch.nn.Module) + self.validate_config(model, config_list) + + self.bound_model = model + self.config_list = config_list + self.optimizer = optimizer + + self.modules_to_compress = None + self.modules_wrapper = [] + self.is_wrapped = False + + self._fwd_hook_handles = {} + self._fwd_hook_id = 0 + + self.reset() + + if not self.modules_wrapper: + _logger.warning('Nothing is configured to compress, please check your model and config_list') + + def validate_config(self, model, config_list): + """ + subclass can optionally implement this method to check if config_list if valid + """ + pass + + def reset(self, checkpoint=None): + """ + reset model state dict and model wrapper + """ + self._unwrap_model() + if checkpoint is not None: + self.bound_model.load_state_dict(checkpoint) + + self.modules_to_compress = None + self.modules_wrapper = [] + + for layer, config in self._detect_modules_to_compress(): + wrapper = self._wrap_modules(layer, config) + self.modules_wrapper.append(wrapper) + + self._wrap_model() + + def _detect_modules_to_compress(self): + """ + detect all modules should be compressed, and save the result in `self.modules_to_compress`. + The model will be instrumented and user should never edit it after calling this method. + """ + if self.modules_to_compress is None: + self.modules_to_compress = [] + for name, module in self.bound_model.named_modules(): + if module == self.bound_model: + continue + layer = LayerInfo(name, module) + config = self.select_config(layer) + if config is not None: + self.modules_to_compress.append((layer, config)) + return self.modules_to_compress + + def _wrap_model(self): + """ + wrap all modules that needed to be compressed + + """ + for wrapper in reversed(self.get_modules_wrapper()): + _setattr(self.bound_model, wrapper.name, wrapper) + self.is_wrapped = True + + def _unwrap_model(self): + """ + unwrap all modules that needed to be compressed + + """ + for wrapper in self.get_modules_wrapper(): + _setattr(self.bound_model, wrapper.name, wrapper.module) + self.is_wrapped = False + + def compress(self): + """ + Compress the model with algorithm implemented by subclass. + + The model will be instrumented and user should never edit it after calling this method. + `self.modules_to_compress` records all the to-be-compressed layers + + Returns + ------- + torch.nn.Module + model with specified modules compressed. + """ + return self.bound_model + + def set_wrappers_attribute(self, name, value): + """ + To register attributes used in wrapped module's forward method. + If the type of the value is Torch.tensor, then this value is registered as a buffer in wrapper, + which will be saved by model.state_dict. Otherwise, this value is just a regular variable in wrapper. + + Parameters + ---------- + name : str + name of the variable + value: any + value of the variable + """ + for wrapper in self.get_modules_wrapper(): + if isinstance(value, torch.Tensor): + wrapper.register_buffer(name, value.clone()) + else: + setattr(wrapper, name, value) + + def get_modules_to_compress(self): + """ + To obtain all the to-be-compressed modules. + + Returns + ------- + list + a list of the layers, each of which is a tuple (`layer`, `config`), + `layer` is `LayerInfo`, `config` is a `dict` + """ + return self.modules_to_compress + + def get_modules_wrapper(self): + """ + To obtain all the wrapped modules. + + Returns + ------- + list + a list of the wrapped modules + """ + return self.modules_wrapper + + def select_config(self, layer): + """ + Find the configuration for `layer` by parsing `self.config_list` + + Parameters + ---------- + layer : LayerInfo + one layer + + Returns + ------- + config or None + the retrieved configuration for this layer, if None, this layer should + not be compressed + """ + ret = None + for config in self.config_list: + config = config.copy() + # expand config if key `default` is in config['op_types'] + if 'op_types' in config and 'default' in config['op_types']: + expanded_op_types = [] + for op_type in config['op_types']: + if op_type == 'default': + expanded_op_types.extend(default_layers.weighted_modules) + else: + expanded_op_types.append(op_type) + config['op_types'] = expanded_op_types + + # check if condition is satisified + if 'op_types' in config and layer.type not in config['op_types']: + continue + if 'op_names' in config and layer.name not in config['op_names']: + continue + + ret = config + if ret is None or 'exclude' in ret: + return None + return ret + + def update_epoch(self, epoch): + """ + If user want to update model every epoch, user can override this method. + This method should be called at the beginning of each epoch + + Parameters + ---------- + epoch : num + the current epoch number + """ + pass + + def _wrap_modules(self, layer, config): + """ + This method is implemented in the subclasses, i.e., `Pruner` and `Quantizer` + + Parameters + ---------- + layer : LayerInfo + the layer to instrument the compression operation + config : dict + the configuration for compressing this layer + """ + raise NotImplementedError() + + def add_activation_collector(self, collector): + self._fwd_hook_id += 1 + self._fwd_hook_handles[self._fwd_hook_id] = [] + for wrapper in self.get_modules_wrapper(): + handle = wrapper.register_forward_hook(collector) + self._fwd_hook_handles[self._fwd_hook_id].append(handle) + return self._fwd_hook_id + + def remove_activation_collector(self, fwd_hook_id): + if fwd_hook_id not in self._fwd_hook_handles: + raise ValueError("%s is not a valid collector id" % str(fwd_hook_id)) + for handle in self._fwd_hook_handles[fwd_hook_id]: + handle.remove() + del self._fwd_hook_handles[fwd_hook_id] + + def patch_optimizer(self, *tasks): + def patch_step(old_step): + def new_step(_, *args, **kwargs): + # call origin optimizer step method + output = old_step(*args, **kwargs) + # calculate mask + for task in tasks: + task() + return output + return new_step + if self.optimizer is not None: + self.optimizer.step = types.MethodType(patch_step(self.optimizer.step), self.optimizer) + + def patch_optimizer_before(self, *tasks): + def patch_step(old_step): + def new_step(_, *args, **kwargs): + for task in tasks: + task() + # call origin optimizer step method + output = old_step(*args, **kwargs) + return output + return new_step + if self.optimizer is not None: + self.optimizer.step = types.MethodType(patch_step(self.optimizer.step), self.optimizer) + +class PrunerModuleWrapper(torch.nn.Module): + def __init__(self, module, module_name, module_type, config, pruner): + """ + Wrap an module to enable data parallel, forward method customization and buffer registeration. + + Parameters + ---------- + module : pytorch module + the module user wants to compress + config : dict + the configurations that users specify for compression + module_name : str + the name of the module to compress, wrapper module shares same name + module_type : str + the type of the module to compress + pruner ： Pruner + the pruner used to calculate mask + """ + super().__init__() + # origin layer information + self.module = module + self.name = module_name + self.type = module_type + # config and pruner + self.config = config + self.pruner = pruner + + # register buffer for mask + self.register_buffer("weight_mask", torch.ones(self.module.weight.shape)) + if hasattr(self.module, 'bias') and self.module.bias is not None: + self.register_buffer("bias_mask", torch.ones(self.module.bias.shape)) + else: + self.register_buffer("bias_mask", None) + + def forward(self, *inputs): + # apply mask to weight, bias + self.module.weight.data = self.module.weight.data.mul_(self.weight_mask) + if hasattr(self.module, 'bias') and self.module.bias is not None: + self.module.bias.data = self.module.bias.data.mul_(self.bias_mask) + return self.module(*inputs) + +class Pruner(Compressor): + """ + Prune to an exact pruning level specification + + Attributes + ---------- + mask_dict : dict + Dictionary for saving masks, `key` should be layer name and + `value` should be a tensor which has the same shape with layer's weight + + """ + + def __init__(self, model, config_list, optimizer=None): + super().__init__(model, config_list, optimizer) + + def compress(self): + self.update_mask() + return self.bound_model + + def update_mask(self): + for wrapper_idx, wrapper in enumerate(self.get_modules_wrapper()): + masks = self.calc_mask(wrapper, wrapper_idx=wrapper_idx) + if masks is not None: + for k in masks: + assert hasattr(wrapper, k), "there is no attribute '%s' in wrapper" % k + setattr(wrapper, k, masks[k]) + + def calc_mask(self, wrapper, **kwargs): + """ + Pruners should overload this method to provide mask for weight tensors. + The mask must have the same shape and type comparing to the weight. + It will be applied with `mul()` operation on the weight. + This method is effectively hooked to `forward()` method of the model. + + Parameters + ---------- + wrapper : Module + calculate mask for `wrapper.module`'s weight + """ + raise NotImplementedError("Pruners must overload calc_mask()") + + def _wrap_modules(self, layer, config): + """ + Create a wrapper module to replace the original one. + + Parameters + ---------- + layer : LayerInfo + the layer to instrument the mask + config : dict + the configuration for generating the mask + """ + _logger.debug("Module detected to compress : %s.", layer.name) + wrapper = PrunerModuleWrapper(layer.module, layer.name, layer.type, config, self) + assert hasattr(layer.module, 'weight'), "module %s does not have 'weight' attribute" % layer.name + # move newly registered buffers to the same device of weight + wrapper.to(layer.module.weight.device) + return wrapper + + def export_model(self, model_path, mask_path=None, onnx_path=None, input_shape=None, device=None): + """ + Export pruned model weights, masks and onnx model(optional) + + Parameters + ---------- + model_path : str + path to save pruned model state_dict + mask_path : str + (optional) path to save mask dict + onnx_path : str + (optional) path to save onnx model + input_shape : list or tuple + input shape to onnx model + device : torch.device + device of the model, used to place the dummy input tensor for exporting onnx file. + the tensor is placed on cpu if ```device``` is None + """ + assert model_path is not None, 'model_path must be specified' + mask_dict = {} + self._unwrap_model() # used for generating correct state_dict name without wrapper state + + for wrapper in self.get_modules_wrapper(): + weight_mask = wrapper.weight_mask + bias_mask = wrapper.bias_mask + if weight_mask is not None: + mask_sum = weight_mask.sum().item() + mask_num = weight_mask.numel() + _logger.debug('Layer: %s Sparsity: %.4f', wrapper.name, 1 - mask_sum / mask_num) + wrapper.module.weight.data = wrapper.module.weight.data.mul(weight_mask) + if bias_mask is not None: + wrapper.module.bias.data = wrapper.module.bias.data.mul(bias_mask) + # save mask to dict + mask_dict[wrapper.name] = {"weight": weight_mask, "bias": bias_mask} + + torch.save(self.bound_model.state_dict(), model_path) + _logger.info('Model state_dict saved to %s', model_path) + if mask_path is not None: + torch.save(mask_dict, mask_path) + _logger.info('Mask dict saved to %s', mask_path) + if onnx_path is not None: + assert input_shape is not None, 'input_shape must be specified to export onnx model' + # input info needed + if device is None: + device = torch.device('cpu') + input_data = torch.Tensor(*input_shape) + torch.onnx.export(self.bound_model, input_data.to(device), onnx_path) + _logger.info('Model in onnx with input shape %s saved to %s', input_data.shape, onnx_path) + + self._wrap_model() + + def load_model_state_dict(self, model_state): + """ + Load the state dict saved from unwrapped model. + + Parameters + ---------- + model_state : dict + state dict saved from unwrapped model + """ + if self.is_wrapped: + self._unwrap_model() + self.bound_model.load_state_dict(model_state) + self._wrap_model() + else: + self.bound_model.load_state_dict(model_state) + + def get_pruned_weights(self, dim=0): + """ + Log the simulated prune sparsity. + + Parameters + ---------- + dim : int + the pruned dim. + """ + for _, wrapper in enumerate(self.get_modules_wrapper()): + weight_mask = wrapper.weight_mask + mask_size = weight_mask.size() + if len(mask_size) == 1: + index = torch.nonzero(weight_mask.abs() != 0).tolist() + else: + sum_idx = list(range(len(mask_size))) + sum_idx.remove(dim) + index = torch.nonzero(weight_mask.abs().sum(sum_idx) != 0).tolist() + _logger.info(f'simulated prune {wrapper.name} remain/total: {len(index)}/{weight_mask.size(dim)}') + + +class QuantizerModuleWrapper(torch.nn.Module): + def __init__(self, module, module_name, module_type, config, quantizer): + """ + Wrap an module to enable data parallel, forward method customization and buffer registeration. + + Parameters + ---------- + module : pytorch module + the module user wants to compress + config : dict + the configurations that users specify for compression + module_name : str + the name of the module to compress, wrapper module shares same name + module_type : str + the type of the module to compress + quantizer ：quantizer + the quantizer used to calculate mask + """ + super().__init__() + # origin layer information + self.module = module + self.name = module_name + self.type = module_type + # config and pruner + self.config = config + self.quantizer = quantizer + + # register buffer and parameter + # old_weight is used to store origin weight and weight is used to store quantized weight + # the reason why weight is buffer instead of parameter is because in pytorch parameter is used as leaf + # if weight is leaf , then old_weight can not be updated. + if 'weight' in config['quant_types']: + if not _check_weight(self.module): + _logger.warning('Module %s does not have parameter "weight"', self.name) + else: + self.module.register_parameter('old_weight', torch.nn.Parameter(self.module.weight)) + delattr(self.module, 'weight') + self.module.register_buffer('weight', self.module.old_weight) + + def forward(self, *inputs): + if 'input' in self.config['quant_types']: + inputs = self.quantizer.quant_grad( + inputs, + QuantType.QUANT_INPUT, + self) + + if 'weight' in self.config['quant_types'] and _check_weight(self.module): + self.quantizer.quant_grad( + self.module.old_weight, + QuantType.QUANT_WEIGHT, + self, inputs[0]) + result = self.module(*inputs) + else: + result = self.module(*inputs) + + if 'output' in self.config['quant_types']: + result = self.quantizer.quant_grad( + result, + QuantType.QUANT_OUTPUT, + self) + return result + + +class Quantizer(Compressor): + """ + Base quantizer for pytorch quantizer + """ + + def __init__(self, model, config_list, optimizer=None): + super().__init__(model, config_list, optimizer) + self.quant_grad = QuantGrad.apply + if self.optimizer is not None: + self.patch_optimizer(self.step_with_optimizer) + for wrapper in self.get_modules_wrapper(): + if 'weight' in wrapper.config['quant_types']: + # old_weight is registered to keep track of weight before quantization + # and it is trainable, therefore, it should be added to optimizer. + self.optimizer.add_param_group({"params": wrapper.module.old_weight}) + + def quantize_weight(self, wrapper, **kwargs): + """ + quantize should overload this method to quantize weight. + This method is effectively hooked to :meth:`forward` of the model. + Parameters + ---------- + wrapper : QuantizerModuleWrapper + the wrapper for origin module + """ + raise NotImplementedError('Quantizer must overload quantize_weight()') + + def quantize_output(self, output, wrapper, **kwargs): + """ + quantize should overload this method to quantize output. + This method is effectively hooked to :meth:`forward` of the model. + Parameters + ---------- + output : Tensor + output that needs to be quantized + wrapper : QuantizerModuleWrapper + the wrapper for origin module + """ + raise NotImplementedError('Quantizer must overload quantize_output()') + + def quantize_input(self, *inputs, wrapper, **kwargs): + """ + quantize should overload this method to quantize input. + This method is effectively hooked to :meth:`forward` of the model. + Parameters + ---------- + inputs : Tensor + inputs that needs to be quantized + wrapper : QuantizerModuleWrapper + the wrapper for origin module + """ + raise NotImplementedError('Quantizer must overload quantize_input()') + + def _wrap_modules(self, layer, config): + """ + Create a wrapper forward function to replace the original one. + Parameters + ---------- + layer : LayerInfo + the layer to instrument the mask + config : dict + the configuration for quantization + """ + assert 'quant_types' in config, 'must provide quant_types in config' + assert isinstance(config['quant_types'], list), 'quant_types must be list type' + assert 'quant_bits' in config, 'must provide quant_bits in config' + assert isinstance(config['quant_bits'], int) or isinstance(config['quant_bits'], dict), 'quant_bits must be dict type or int type' + + if isinstance(config['quant_bits'], dict): + for quant_type in config['quant_types']: + assert quant_type in config['quant_bits'], 'bits length for %s must be specified in quant_bits dict' % quant_type + + return QuantizerModuleWrapper(layer.module, layer.name, layer.type, config, self) + + def export_model_save(self, model, model_path, calibration_config=None, calibration_path=None, onnx_path=None, + input_shape=None, device=None): + """ + This method helps save pytorch model, calibration config, onnx model in quantizer. + + Parameters + ---------- + model : pytorch model + pytorch model to be saved + model_path : str + path to save pytorch + calibration_config: dict + (optional) config of calibration parameters + calibration_path : str + (optional) path to save quantize parameters after calibration + onnx_path : str + (optional) path to save onnx model + input_shape : list or tuple + input shape to onnx model + device : torch.device + device of the model, used to place the dummy input tensor for exporting onnx file. + the tensor is placed on cpu if ```device``` is None + """ + torch.save(model.state_dict(), model_path) + _logger.info('Model state_dict saved to %s', model_path) + if calibration_path is not None: + torch.save(calibration_config, calibration_path) + _logger.info('Mask dict saved to %s', calibration_path) + if onnx_path is not None: + assert input_shape is not None, 'input_shape must be specified to export onnx model' + # input info needed + if device is None: + device = torch.device('cpu') + input_data = torch.Tensor(*input_shape) + torch.onnx.export(self.bound_model, input_data.to(device), onnx_path) + _logger.info('Model in onnx with input shape %s saved to %s', input_data.shape, onnx_path) + + def export_model(self, model_path, calibration_path=None, onnx_path=None, input_shape=None, device=None): + """ + Export quantized model weights and calibration parameters + + Parameters + ---------- + model_path : str + path to save quantized model weight + calibration_path : str + (optional) path to save quantize parameters after calibration + onnx_path : str + (optional) path to save onnx model + input_shape : list or tuple + input shape to onnx model + device : torch.device + device of the model, used to place the dummy input tensor for exporting onnx file. + the tensor is placed on cpu if ```device``` is None + + Returns + ------- + Dict + """ + raise NotImplementedError('Quantizer must overload export_model()') + + def step_with_optimizer(self): + pass + +class QuantType: + """ + Enum class for quantization type. + """ + QUANT_INPUT = 0 + QUANT_WEIGHT = 1 + QUANT_OUTPUT = 2 + +QType_Dict = { + 0: "input", + 1: "weight", + 2: "output" +} + +class QuantGrad(torch.autograd.Function): + """ + Base class for overriding backward function of quantization operation. + """ + @classmethod + def _quantize(cls, x, scale, zero_point): + """ + Reference function for quantizing x -- non-clamped. + Parameters + ---------- + x : Tensor + tensor to be quantized + scale : Tensor + scale for quantizing x + zero_point : Tensor + zero_point for quantizing x + Returns + ------- + tensor + quantized x without clamped + """ + return ((x / scale) + zero_point).round() + + @classmethod + def get_bits_length(cls, config, quant_type): + """ + Get bit for quantize config + Parameters + ---------- + config : Dict + the configuration for quantization + quant_type : str + quant type + Returns + ------- + int + n-bits for quantization configuration + """ + if isinstance(config["quant_bits"], int): + return config["quant_bits"] + else: + return config["quant_bits"].get(quant_type) + + @staticmethod + def quant_backward(tensor, grad_output, quant_type, scale, zero_point, qmin, qmax): + """ + This method should be overrided by subclass to provide customized backward function, + default implementation is Straight-Through Estimator + Parameters + ---------- + tensor : Tensor + input of quantization operation + grad_output : Tensor + gradient of the output of quantization operation + scale : Tensor + the type of quantization, it can be `QuantType.QUANT_INPUT`, `QuantType.QUANT_WEIGHT`, + `QuantType.QUANT_OUTPUT`, you can define different behavior for different types. + zero_point : Tensor + zero_point for quantizing tensor + qmin : Tensor + quant_min for quantizing tensor + qmax : Tensor + quant_max for quantizng tensor + Returns + ------- + tensor + gradient of the input of quantization operation + """ + return grad_output + + @staticmethod + def forward(ctx, tensor, quant_type, wrapper, input_tensor=None, **kwargs): + output = quantize_helper(tensor, quant_type, wrapper, input_tensor, **kwargs) + + bits = QuantGrad.get_bits_length(wrapper.config, QType_Dict[quant_type]) + qmin, qmax = torch.Tensor([0]).to(tensor.device), torch.Tensor([(1 << bits) - 1]).to(tensor.device) + if hasattr(wrapper.module, 'scale') and hasattr(wrapper.module, 'zero_point'): + scale = wrapper.module.scale + zero_point = wrapper.module.zero_point + else: + scale, zero_point = None, None + ctx.save_for_backward(tensor, torch.Tensor([quant_type]), scale, zero_point, qmin, qmax) + return output + + @classmethod + def backward(cls, ctx, grad_output): + tensor, quant_type, scale, zero_point, qmin, qmax = ctx.saved_variables + output = cls.quant_backward(tensor, grad_output, quant_type, scale, zero_point, qmin, qmax) + return output, None, None, None + +def _check_weight(module): + try: + return isinstance(module.weight.data, torch.Tensor) + except AttributeError: + return False + +def quantize_helper(tensor, quant_type, wrapper, input_tensor=None, **kwargs): + if quant_type == QuantType.QUANT_INPUT: + output = wrapper.quantizer.quantize_input(*tensor, wrapper=wrapper, **kwargs) + elif quant_type == QuantType.QUANT_WEIGHT: + output = wrapper.quantizer.quantize_weight(wrapper, input_tensor=input_tensor, **kwargs) + elif quant_type == QuantType.QUANT_OUTPUT: + output = wrapper.quantizer.quantize_output(tensor, wrapper, **kwargs) + else: + raise ValueError("unrecognized QuantType.") + + return output + +class QuantForward(torch.nn.Module): + """ + Base class for executing quantization operations. This is for quantization algorithms + that do not need to customize gradient. + """ + + def forward(self, tensor, quant_type, wrapper, input_tensor=None, **kwargs): + return quantize_helper(tensor, quant_type, wrapper, input_tensor, **kwargs) diff --git a/utils/third_party/nni_new/compression/pytorch/default_layers.py b/utils/third_party/nni_new/compression/pytorch/default_layers.py new file mode 100644 index 0000000000000000000000000000000000000000..4d7e6d8aed84ad76c9404f301117fb8a00d9a570 --- /dev/null +++ b/utils/third_party/nni_new/compression/pytorch/default_layers.py @@ -0,0 +1,9 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +weighted_modules = [ + 'Conv1d', 'Conv2d', 'Conv3d', 'ConvTranspose1d', 'ConvTranspose2d', 'ConvTranspose3d', + 'Linear', 'Bilinear', + 'PReLU', + 'Embedding', 'EmbeddingBag', +] diff --git a/utils/third_party/nni_new/compression/pytorch/pruning/__init__.py b/utils/third_party/nni_new/compression/pytorch/pruning/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9d3a7d2ca90a76918e1a89508f007903d1bb6485 --- /dev/null +++ b/utils/third_party/nni_new/compression/pytorch/pruning/__init__.py @@ -0,0 +1,4 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .apply_compression import apply_compression_results diff --git a/utils/third_party/nni_new/compression/pytorch/pruning/__pycache__/__init__.cpython-38.pyc b/utils/third_party/nni_new/compression/pytorch/pruning/__pycache__/__init__.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f97d67fd07b30baf0967a19306802d1ea795e518 Binary files /dev/null and b/utils/third_party/nni_new/compression/pytorch/pruning/__pycache__/__init__.cpython-38.pyc differ diff --git a/utils/third_party/nni_new/compression/pytorch/pruning/__pycache__/apply_compression.cpython-38.pyc b/utils/third_party/nni_new/compression/pytorch/pruning/__pycache__/apply_compression.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9f57ada83e4b744d0f7427d4a229477683d766fb Binary files /dev/null and b/utils/third_party/nni_new/compression/pytorch/pruning/__pycache__/apply_compression.cpython-38.pyc differ diff --git a/utils/third_party/nni_new/compression/pytorch/pruning/apply_compression.py b/utils/third_party/nni_new/compression/pytorch/pruning/apply_compression.py new file mode 100644 index 0000000000000000000000000000000000000000..8e6b023f5b90b8483e21bd0bc575b19b4a4df023 --- /dev/null +++ b/utils/third_party/nni_new/compression/pytorch/pruning/apply_compression.py @@ -0,0 +1,29 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +import torch + +logger = logging.getLogger('torch apply compression') + +def apply_compression_results(model, masks_file, map_location=None): + """ + Apply the masks from ```masks_file``` to the model + Note: this API is for inference, because it simply multiplies weights with + corresponding masks when this API is called. + + Parameters + ---------- + model : torch.nn.Module + The model to be compressed + masks_file : str + The path of the mask file + map_location : str + the device on which masks are placed, same to map_location in ```torch.load``` + """ + masks = torch.load(masks_file, map_location) + for name, module in model.named_modules(): + if name in masks: + module.weight.data = module.weight.data.mul_(masks[name]['weight']) + if hasattr(module, 'bias') and module.bias is not None and 'bias' in masks[name]: + module.bias.data = module.bias.data.mul_(masks[name]['bias']) \ No newline at end of file diff --git a/utils/third_party/nni_new/compression/pytorch/quantization_speedup/__init__.py b/utils/third_party/nni_new/compression/pytorch/quantization_speedup/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..636c82a5b045d39281064b80c77cfa60489eb6af --- /dev/null +++ b/utils/third_party/nni_new/compression/pytorch/quantization_speedup/__init__.py @@ -0,0 +1 @@ +from .integrated_tensorrt import CalibrateType, ModelSpeedupTensorRT \ No newline at end of file diff --git a/utils/third_party/nni_new/compression/pytorch/quantization_speedup/backend.py b/utils/third_party/nni_new/compression/pytorch/quantization_speedup/backend.py new file mode 100644 index 0000000000000000000000000000000000000000..7d139d48f84c10557e3c83d2a858da707fe31227 --- /dev/null +++ b/utils/third_party/nni_new/compression/pytorch/quantization_speedup/backend.py @@ -0,0 +1,51 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +class BaseModelSpeedup: + """ + Base speedup class for backend engine + """ + def __init__(self, model, config): + """ + Parameters + ---------- + model : pytorch model + The model to speed up by quantization. + config : dict + Config recording bit number and name of layers. + """ + self.model = model + self.config = config + + def inference(self, test_data): + """ + This function should be overrided by subclass to provide inference ability, + which should return output and inference time. + + Parameters + ---------- + test_data : numpy data + test data given to the inference engine + + Returns + ------- + numpy data + output data will be generated after inference + float + latency of such inference process + """ + raise NotImplementedError('Backend engine must overload inference()') + + def compress(self): + """ + This function should be overrided by subclass to build inference + engine which will be used to process input data + """ + raise NotImplementedError('Backend engine must overload compress()') + + def export_quantized_model(self, path): + """ + This function should be overrided by subclass to build inference + engine which will be used to process input data + """ + raise NotImplementedError('Backend engine must overload export_quantized_model()') \ No newline at end of file diff --git a/utils/third_party/nni_new/compression/pytorch/quantization_speedup/calibrator.py b/utils/third_party/nni_new/compression/pytorch/quantization_speedup/calibrator.py new file mode 100644 index 0000000000000000000000000000000000000000..6bc49622f2e8cb1e818149ec06039bf9ab305c33 --- /dev/null +++ b/utils/third_party/nni_new/compression/pytorch/quantization_speedup/calibrator.py @@ -0,0 +1,99 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +import logging +import tensorrt as trt +import pycuda.driver as cuda + +logger = logging.getLogger(__name__) + +class Calibrator(trt.IInt8Calibrator): + def __init__(self, training_data, cache_file, batch_size=64, algorithm=trt.CalibrationAlgoType.ENTROPY_CALIBRATION_2): + """ + Parameters + ---------- + training_data : numpy array + The data using to calibrate quantization model + cache_file : str + The path user want to store calibrate cache file + batch_size : int + The batch_size of calibrating process + algorithm : tensorrt.tensorrt.CalibrationAlgoType + The algorithms of calibrating contains LEGACY_CALIBRATION, + ENTROPY_CALIBRATION, ENTROPY_CALIBRATION_2, MINMAX_CALIBRATION. + Please refer to https://docs.nvidia.com/deeplearning/tensorrt/api/ + python_api/infer/Int8/Calibrator.html for detail + """ + trt.IInt8Calibrator.__init__(self) + + self.algorithm = algorithm + self.cache_file = cache_file + + self.data = training_data + self.batch_size = batch_size + self.current_index = 0 + + # Allocate enough memory for a whole batch. + self.device_input = cuda.mem_alloc(self.data[0].nbytes * self.batch_size) + + def get_algorithm(self): + return self.algorithm + + def get_batch_size(self): + return self.batch_size + + def get_batch(self, names): + """ + This function is used to define the way of feeding calibrating data each batch. + + Parameters + ---------- + names : str + The names of the network inputs for each object in the bindings array + + Returns + ------- + list + A list of device memory pointers set to the memory containing each network + input data, or an empty list if there are no more batches for calibration. + You can allocate these device buffers with pycuda, for example, and then + cast them to int to retrieve the pointer + """ + if self.current_index + self.batch_size > self.data.shape[0]: + return None + + current_batch = int(self.current_index / self.batch_size) + if current_batch % 10 == 0: + logger.info("Calibrating batch %d, containing %d images", current_batch, self.batch_size) + + batch = self.data[self.current_index:self.current_index + self.batch_size].ravel() + cuda.memcpy_htod(self.device_input, batch) + self.current_index += self.batch_size + memory_pointers = [self.device_input] + return memory_pointers + + def read_calibration_cache(self): + """ + If there is a cache, use it instead of calibrating again. Otherwise, implicitly return None. + + Returns + ------- + cache object + A cache object which contains calibration parameters for quantization + """ + if os.path.exists(self.cache_file): + with open(self.cache_file, "rb") as f: + return f.read() + + def write_calibration_cache(self, cache): + """ + Write calibration cache to specific path. + + Parameters + ---------- + cache : str + The calibration cache to write + """ + with open(self.cache_file, "wb") as f: + f.write(cache) \ No newline at end of file diff --git a/utils/third_party/nni_new/compression/pytorch/quantization_speedup/frontend_to_onnx.py b/utils/third_party/nni_new/compression/pytorch/quantization_speedup/frontend_to_onnx.py new file mode 100644 index 0000000000000000000000000000000000000000..2bbb9f17e112e2d09a3bfcef19822172b0e3efed --- /dev/null +++ b/utils/third_party/nni_new/compression/pytorch/quantization_speedup/frontend_to_onnx.py @@ -0,0 +1,148 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import torch +import onnx +import onnx.numpy_helper +""" +The main function of this page is to convert pytorch model to onnx model. +Convertion from pytorch model to onnx model is primary so that a critical +problem is caused that Layer name of pytorch model fail to convert to onnx +layer name directly. To solve it, we wrap pytorch model in new wrapper which +multiply bit number and input before computation of each op. Only in this +way can onnx model get bit number of corresponded layer. +""" + +class LayernameModuleWrapper(torch.nn.Module): + def __init__(self, module, module_bit) -> None: + """ + Parameters + ---------- + module : torch.nn.Module + Layer module of pytorch model + module_bit : int + Bit width setting for module + """ + super().__init__() + self.module = module + self.module_bit = module_bit + + def forward(self, inputs): + inputs = inputs*self.module_bit + inputs = self.module(inputs) + return inputs + +def _setattr(model, name, module): + """ + Parameters + ---------- + model : pytorch model + The model to speed up by quantization + name : str + name of pytorch module + module : torch.nn.Module + Layer module of pytorch model + """ + name_list = name.split(".") + for name in name_list[:-1]: + model = getattr(model, name) + setattr(model, name_list[-1], module) + +def unwrapper(model_onnx, index2name, config): + """ + Fill onnx config and remove wrapper node in onnx + + Parameters + ---------- + model_onnx : onnx model + Onnx model which is converted from pytorch model + index2name : dict + Dictionary of layer index and name + config : dict + Config recording name of layers and calibration parameters + + Returns + ------- + onnx model + Onnx model which is converted from pytorch model + dict + The configuration of onnx model layers and calibration parameters + """ + # Support Gemm, Conv, Relu, Clip(Relu6) and Maxpool + support_op = ['Gemm', 'Conv', 'Relu', 'Clip', 'MaxP'] + idx = 0 + onnx_config = {} + while idx < len(model_onnx.graph.node): + nd = model_onnx.graph.node[idx] + if nd.name[0:4] in support_op and idx > 1: + # Grad constant node and multiply node + const_nd = model_onnx.graph.node[idx-2] + mul_nd = model_onnx.graph.node[idx-1] + # Get index number which is transferred by constant node + index = int(onnx.numpy_helper.to_array(const_nd.attribute[0].t)) + if index != -1: + name = index2name[index] + onnx_config[nd.name] = config[name] + nd.input[0] = mul_nd.input[0] + # Remove constant node and multiply node + model_onnx.graph.node.remove(const_nd) + model_onnx.graph.node.remove(mul_nd) + idx = idx-2 + idx = idx+1 + return model_onnx, onnx_config + +def torch_to_onnx(model, config, input_shape, model_path, input_names, output_names): + """ + Convert torch model to onnx model and get layer bit config of onnx model. + + Parameters + ---------- + model : pytorch model + The model to speed up by quantization + config : dict + Config recording bit number and name of layers + input_shape : tuple + The input shape of model, shall pass it to torch.onnx.export + model_path : str + The path user want to store onnx model which is converted from pytorch model + input_names : list + Input name of onnx model providing for torch.onnx.export to generate onnx model + output_name : list + Output name of onnx model providing for torch.onnx.export to generate onnx model + + Returns + ------- + onnx model + Onnx model which is converted from pytorch model + dict + The configuration of onnx model layers and calibration parameters + """ + # Support Gemm, Conv, Relu, Clip(Relu6) and MaxPool + support_op = [torch.nn.Conv2d, torch.nn.Linear, torch.nn.ReLU, torch.nn.ReLU6, torch.nn.MaxPool2d] + # Transfer bit number to onnx layer by using wrapper + index2name = {} + name2index = {} + if config is not None: + for i, name in enumerate(config.keys()): + index2name[i] = name + name2index[name] = i + for name, module in model.named_modules(): + if config is not None and name in config: + assert type(module) in support_op + wrapper_module = LayernameModuleWrapper(module, name2index[name]) + _setattr(model, name, wrapper_module) + elif type(module) in support_op: + wrapper_module = LayernameModuleWrapper(module, -1) + _setattr(model, name, wrapper_module) + # Convert torch model to onnx model and save it in model_path + dummy_input = torch.randn(input_shape) + model.to('cpu') + torch.onnx.export(model, dummy_input, model_path, verbose=False, input_names=input_names, output_names=output_names, export_params=True) + + # Load onnx model + model_onnx = onnx.load(model_path) + model_onnx, onnx_config = unwrapper(model_onnx, index2name, config) + onnx.save(model_onnx, model_path) + + onnx.checker.check_model(model_onnx) + return model_onnx, onnx_config \ No newline at end of file diff --git a/utils/third_party/nni_new/compression/pytorch/quantization_speedup/integrated_tensorrt.py b/utils/third_party/nni_new/compression/pytorch/quantization_speedup/integrated_tensorrt.py new file mode 100644 index 0000000000000000000000000000000000000000..c7849774cc36295ae13a2be0442de7cbcf9ff894 --- /dev/null +++ b/utils/third_party/nni_new/compression/pytorch/quantization_speedup/integrated_tensorrt.py @@ -0,0 +1,381 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import time +import logging +import tensorrt as trt +import numpy as np +import torch + +from . import frontend_to_onnx as fonnx +from . import calibrator as calibrator +from . import trt_pycuda as common +from .backend import BaseModelSpeedup + +# TRT_LOGGER = trt.Logger(trt.Logger.VERBOSE) +TRT_LOGGER = trt.Logger() +logger = logging.getLogger(__name__) + +class CalibrateType: + LEGACY = trt.CalibrationAlgoType.LEGACY_CALIBRATION + ENTROPY = trt.CalibrationAlgoType.ENTROPY_CALIBRATION + ENTROPY2 = trt.CalibrationAlgoType.ENTROPY_CALIBRATION_2 + MINMAX = trt.CalibrationAlgoType.MINMAX_CALIBRATION + +Precision_Dict = { + 8: trt.float32, + 16: trt.float16, + 32: trt.float32 +} + +def valid_config(config=None): + """ + This function validates the bit setting configuration + """ + if config is None: + return + support_bit = [8, 16, 32] + for name in config.keys(): + if 'weight_bit' in config[name]: + w_bit = config[name]['weight_bit'] + assert w_bit in support_bit, "weight bit should be 8, 16, 32" + if 'activation_bit' in config[name]: + a_bit = config[name]['activation_bit'] + assert a_bit in support_bit, "activation bit should be 8, 16, 32" + +def handle_gemm(network, layer_idx, config): + """ + This function handles special gemm operation due to layer numbers of gemm changed during pytorch->onnx model convertion. + + Parameters + ---------- + network : tensorrt.INetworkDefinition + Represents a TensorRT Network from which the Builder can build an Engine + layer_idx : int + layer index of gemm + config : dict + Config recording bit number and name of layers + """ + layer = network.get_layer(layer_idx) + pre_layer = network.get_layer(layer_idx-1) + next_layer = network.get_layer(layer_idx+1) + # if weight bit exists, set three layers' precision, + # input tensor range and the first two layers' output type + if 'weight_bit' in config[layer.name]: + assert 'tracked_min_input' in config[layer.name] + assert 'tracked_max_input' in config[layer.name] + w_bit = config[layer.name]['weight_bit'] + tracked_min_input = config[layer.name]['tracked_min_input'] + tracked_max_input = config[layer.name]['tracked_max_input'] + # set three layers the same precision + layer.precision = Precision_Dict[w_bit] + pre_layer.precision = Precision_Dict[w_bit] + next_layer.precision = Precision_Dict[w_bit] + # set the first two layers' output type + pre_layer.set_output_type(0, Precision_Dict[w_bit]) + layer.set_output_type(0, Precision_Dict[w_bit]) + pre_in_tensor = pre_layer.get_input(0) + in_tensor = layer.get_input(0) + next_in_tensor = next_layer.get_input(0) + # set three layers' input tensor range + pre_in_tensor.dynamic_range = (tracked_min_input, tracked_max_input) + in_tensor.dynamic_range = (tracked_min_input, tracked_max_input) + next_in_tensor.dynamic_range = (tracked_min_input, tracked_max_input) + + # if activation bit exists, set the last layer's output type output tensor range + if 'activation_bit' in config[layer.name]: + assert 'tracked_min_activation' in config[layer.name] + assert 'tracked_max_activation' in config[layer.name] + a_bit = config[layer.name]['activation_bit'] + tracked_min_activation = config[layer.name]['tracked_min_activation'] + tracked_max_activation = config[layer.name]['tracked_max_activation'] + # set the last layer's output type + next_layer.set_output_type(0, Precision_Dict[a_bit]) + next_out_tensor = next_layer.get_output(0) + # set the last layer's output tensor range + next_out_tensor.dynamic_range = (tracked_min_activation, tracked_max_activation) + +def build_engine(model_file, config=None, extra_layer_bit=32, strict_datatype=False, calib=None): + """ + This function builds an engine from an onnx model with calibration process. + + Parameters + ---------- + model_file : str + The path of onnx model + config : dict + Config recording bit number and name of layers + extra_layer_bit : int + Other layers which are not in config will be quantized to corresponding bit number + strict_datatype : bool + Whether constrain layer bit to the number given in config or not. If true, all the layer + will be set to given bit strictly. Otherwise, these layers will be set automatically by + tensorrt + calib : numpy array + The data using to calibrate quantization model + + Returns + ------- + tensorrt.ICudaEngine + An ICudaEngine for executing inference on a built network + """ + with trt.Builder(TRT_LOGGER) as builder, builder.create_network(common.EXPLICIT_BATCH) as network, \ + trt.OnnxParser(network, TRT_LOGGER) as parser: + # Attention that, builder should be set to 1 because of the implementation of allocate_buffer + builder.max_batch_size = 1 + builder.max_workspace_size = common.GiB(4) + + if extra_layer_bit == 32 and config is None: + pass + elif extra_layer_bit == 16 and config is None: + builder.fp16_mode = True + elif extra_layer_bit == 8 and config is None: + # entire model in 8bit mode + builder.int8_mode = True + else: + builder.int8_mode = True + builder.fp16_mode = True + builder.strict_type_constraints = strict_datatype + + valid_config(config) + + # Parse onnx model + with open(model_file, 'rb') as model: + if not parser.parse(model.read()): + logger.error('ERROR: Fail to parse the ONNX file.') + for error in range(parser.num_errors): + logger.error(parser.get_error(error)) + return None + + if calib is not None: + builder.int8_calibrator = calib + # This design may not be correct if output more than one + for i in range(network.num_layers): + if config is None: + break + layer = network.get_layer(i) + if layer.name in config: + w_bit = config[layer.name]['weight_bit'] + a_bit = config[layer.name]['activation_bit'] + layer.precision = Precision_Dict[w_bit] + layer.set_output_type(0, Precision_Dict[a_bit]) + else: + # This implementation may be incorrect when output number > 1 + for i in range(network.num_layers): + if config is None: + # no low bit layer need to be set, keep original model + break + layer = network.get_layer(i) + if layer.name not in config: + continue + # layer numbers of gemm changed during pytorch->onnx model convertion, need special handle + if layer.name[0:4] == "Gemm": + handle_gemm(network, i, config) + continue + + # If weight_bit exists in config, set layer precision and layer's input tensor dynamic range. + if 'weight_bit' in config[layer.name]: + assert 'tracked_min_input' in config[layer.name] + assert 'tracked_max_input' in config[layer.name] + w_bit = config[layer.name]['weight_bit'] + tracked_min_input = config[layer.name]['tracked_min_input'] + tracked_max_input = config[layer.name]['tracked_max_input'] + layer.precision = Precision_Dict[w_bit] + in_tensor = layer.get_input(0) + in_tensor.dynamic_range = (tracked_min_input, tracked_max_input) + + # If activation exists in config, set layer output type and layer's output tensor dynamic range. + if 'activation_bit' in config[layer.name]: + assert 'tracked_min_activation' in config[layer.name] + assert 'tracked_max_activation' in config[layer.name] + a_bit = config[layer.name]['activation_bit'] + tracked_min_activation = config[layer.name]['tracked_min_activation'] + tracked_max_activation = config[layer.name]['tracked_max_activation'] + layer.set_output_type(0, Precision_Dict[a_bit]) + out_tensor = layer.get_output(0) + out_tensor.dynamic_range = (tracked_min_activation, tracked_max_activation) + + # Build engine and do int8 calibration. + engine = builder.build_cuda_engine(network) + return engine + +class ModelSpeedupTensorRT(BaseModelSpeedup): + def __init__(self, model, input_shape, config=None, onnx_path="default_model.onnx", extra_layer_bit=32, strict_datatype=True, + calibrate_type=CalibrateType.ENTROPY2, calib_data_loader=None, calibration_cache = "calibration.cache", batchsize=1, + input_names=["actual_input_1"], output_names=["output1"]): + """ + Parameters + ---------- + model : pytorch model + The model to speed up by quantization. + input_shape : tuple + The input shape of model, shall pass it to torch.onnx.export. + config : dict + Config recording bit number and name of layers. + onnx_path : str + The path user want to store onnx model which is converted from pytorch model. + extra_layer_bit : int + Other layers which are not in config will be quantized to corresponding bit number. + strict_datatype : bool + Whether constrain layer bit to the number given in config or not. If true, all the layer + will be set to given bit strictly. Otherwise, these layers will be set automatically by + tensorrt. + calibrate_type : tensorrt.tensorrt.CalibrationAlgoType + The algorithm of calibrating. Please refer to https://docs.nvidia.com/deeplearning/ + tensorrt/api/python_api/infer/Int8/Calibrator.html for detail + calibrate_data : numpy array + The data using to calibrate quantization model + calibration_cache : str + The path user want to store calibrate cache file + batchsize : int + The batch size of calibration and inference + input_names : list + Input name of onnx model providing for torch.onnx.export to generate onnx model + output_name : list + Output name of onnx model providing for torch.onnx.export to generate onnx model + """ + super().__init__(model, config) + self.model = model + self.onnx_path = onnx_path + self.input_shape = input_shape + self.config = config + self.extra_layer_bit = extra_layer_bit + self.strict_datatype = strict_datatype + self.calibrate_type = calibrate_type + self.calib_data_loader = calib_data_loader + self.calibration_cache = calibration_cache + self.batchsize = batchsize + self.input_names = input_names + self.output_names = output_names + self.context = None + self.onnx_config = {} + + def compress(self): + """ + Get onnx config and build tensorrt engine. + """ + assert self.model is not None + assert self.onnx_path is not None + assert self.input_shape is not None + + # Convert pytorch model to onnx model and save onnx model in onnx_path + _, self.onnx_config = fonnx.torch_to_onnx(self.model, self.config, input_shape=self.input_shape, + model_path=self.onnx_path, input_names=self.input_names, output_names=self.output_names) + + if self.calib_data_loader is not None: + assert self.calibrate_type is not None + context = self._tensorrt_build_withcalib(self.onnx_path) + else: + context = self._tensorrt_build_withoutcalib(self.onnx_path) + self.context = context + + def _tensorrt_build_withcalib(self, onnx_path): + """ + Convert pytorch tensor to numpy darray + + Parameters + ---------- + onnx_path : str + The path of onnx model + + Returns + ------- + tensorrt.IExecutionContext + Context for executing inference using an ICudaEngine + """ + calib_data = None + if type(self.calib_data_loader) == torch.utils.data.dataloader.DataLoader: + calib_data_set = [] + for data, _ in self.calib_data_loader: + calib_data_set.append(data) + calib_data = np.concatenate(calib_data_set) + elif type(self.calib_data_loader) == torch.Tensor: + calib_data = self.calib_data_loader.numpy() + else: + raise ValueError("Not support calibration datatype") + calib = calibrator.Calibrator(calib_data, self.calibration_cache, self.batchsize, self.calibrate_type) + + # build inference engine with calibration + engine = build_engine(onnx_path, self.onnx_config, self.extra_layer_bit, self.strict_datatype, calib) + return engine.create_execution_context() + + def _tensorrt_build_withoutcalib(self, onnx_path): + """ + Build inference engine without calibration + + Parameters + ---------- + onnx_path : str + The path of onnx model + + Returns + ------- + tensorrt.IExecutionContext + Context for executing inference using an ICudaEngine + """ + engine = build_engine(onnx_path, self.onnx_config, self.extra_layer_bit, self.strict_datatype) + return engine.create_execution_context() + + def inference(self, test_data): + """ + Do inference by tensorrt builded engine. + + Parameters + ---------- + test_data : pytorch tensor + Model input tensor + """ + # convert pytorch tensor to numpy darray + test_data = test_data.numpy() + # Numpy dtype should be float32 + assert test_data.dtype == np.float32 + elapsed_time = 0 + inputs, outputs, bindings, stream = common.allocate_buffers(self.context.engine) + result = [] + for start_idx in range(0, test_data.shape[0], self.batchsize): + # If the number of images in the test set is not divisible by the batch size, the last batch will be smaller. + # This logic is used for handling that case. + end_idx = min(start_idx + self.batchsize, test_data.shape[0]) + effective_batch_size = end_idx - start_idx + + # Do inference for every batch. + inputs[0].host = test_data[start_idx:start_idx + effective_batch_size] + t1 = time.time() + [output] = common.do_inference_v2(self.context, bindings=bindings, inputs=inputs, outputs=outputs, stream=stream) + elapsed_time += time.time() - t1 + shape = output.shape[0] + output = output[0:int(shape * effective_batch_size / self.batchsize)].reshape(effective_batch_size, -1) + result.append(output.copy()) + # Use argmax to get predictions and then check accuracy + # convert numpy darray to pytorch tensor + result = torch.Tensor(np.concatenate(result)) + return result, elapsed_time + + def export_quantized_model(self, path): + """ + Export TensorRT quantized model engine which only can be loaded by TensorRT deserialize API. + + Parameters + ---------- + path : str + The path of export model + """ + assert path is not None + with open(path, "wb") as f: + f.write(self.context.engine.serialize()) + logger.info("TensorRT engine has been saved to %s", path) + + def load_quantized_model(self, path): + """ + Load TensorRT quantized model engine from specific path. + + Parameters + ---------- + path : str + The path of export model + """ + assert path is not None + with open(path, "rb") as f, trt.Runtime(TRT_LOGGER) as runtime: + engine = runtime.deserialize_cuda_engine(f.read()) + self.context = engine.create_execution_context() + logger.info("Load TensorRT engine from %s successfully.", path) \ No newline at end of file diff --git a/utils/third_party/nni_new/compression/pytorch/quantization_speedup/trt_pycuda.py b/utils/third_party/nni_new/compression/pytorch/quantization_speedup/trt_pycuda.py new file mode 100644 index 0000000000000000000000000000000000000000..d3f8e1f4c6792c4cded99d2aaad3dfeb47a57686 --- /dev/null +++ b/utils/third_party/nni_new/compression/pytorch/quantization_speedup/trt_pycuda.py @@ -0,0 +1,86 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import pycuda.driver as cuda +import pycuda.autoinit # pylint: disable=unused-import +import tensorrt as trt + +EXPLICIT_BATCH = 1 + +def GiB(val): + return val * 1 << 30 + +# Simple helper data class that's a little nicer to use than a 2-tuple. +class HostDeviceMem(object): + def __init__(self, host_mem, device_mem): + """ + This function builds an engine from an onnx model with calibration process. + + Parameters + ---------- + host_mem : host memory + Memory buffers of host + device_mem : device memory + Memory buffers of device + """ + self.host = host_mem + self.device = device_mem + + def __str__(self): + return "Host:\n" + str(self.host) + "\nDevice:\n" + str(self.device) + + def __repr__(self): + return self.__str__() + +def allocate_buffers(engine): + """ + Allocates all buffers required for an engine, i.e. host/device inputs/outputs. + + Parameters + ---------- + engine : tensorrt.ICudaEngine + An ICudaEngine for executing inference on a built network + + Returns + ------- + list + All input HostDeviceMem of an engine + list + All output HostDeviceMem of an engine + GPU bindings + Device bindings + GPU stream + A stream is a sequence of commands (possibly issued by different host threads) that execute in order + """ + inputs = [] + outputs = [] + bindings = [] + stream = cuda.Stream() + for binding in engine: + size = trt.volume(engine.get_binding_shape(binding)) * engine.max_batch_size + dtype = trt.nptype(engine.get_binding_dtype(binding)) + # Allocate host and device buffers + host_mem = cuda.pagelocked_empty(size, dtype) + device_mem = cuda.mem_alloc(host_mem.nbytes) + # Append the device buffer to device bindings. + bindings.append(int(device_mem)) + # Append to the appropriate list. + if engine.binding_is_input(binding): + inputs.append(HostDeviceMem(host_mem, device_mem)) + else: + outputs.append(HostDeviceMem(host_mem, device_mem)) + return inputs, outputs, bindings, stream + +# This function is generalized for multiple inputs/outputs for full dimension networks. +# inputs and outputs are expected to be lists of HostDeviceMem objects. +def do_inference_v2(context, bindings, inputs, outputs, stream): + # Transfer input data to the GPU. + [cuda.memcpy_htod_async(inp.device, inp.host, stream) for inp in inputs] + # Run inference. + context.execute_async_v2(bindings=bindings, stream_handle=stream.handle) + # Transfer predictions back from the GPU. + [cuda.memcpy_dtoh_async(out.host, out.device, stream) for out in outputs] + # Synchronize the stream + stream.synchronize() + # Return only the host outputs. + return [out.host for out in outputs] \ No newline at end of file diff --git a/utils/third_party/nni_new/compression/pytorch/speedup/__init__.py b/utils/third_party/nni_new/compression/pytorch/speedup/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..cef8ebd76c0d99b1810512afe28723b5eeccb9fc --- /dev/null +++ b/utils/third_party/nni_new/compression/pytorch/speedup/__init__.py @@ -0,0 +1 @@ +from .compressor import ModelSpeedup \ No newline at end of file diff --git a/utils/third_party/nni_new/compression/pytorch/speedup/__pycache__/__init__.cpython-38.pyc b/utils/third_party/nni_new/compression/pytorch/speedup/__pycache__/__init__.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9d962fcee62b02bbf66a7aa344b332d577827b72 Binary files /dev/null and b/utils/third_party/nni_new/compression/pytorch/speedup/__pycache__/__init__.cpython-38.pyc differ diff --git a/utils/third_party/nni_new/compression/pytorch/speedup/__pycache__/compress_modules.cpython-38.pyc b/utils/third_party/nni_new/compression/pytorch/speedup/__pycache__/compress_modules.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d45ff4b85ce90510a84754fd94bc216cb18b3fd1 Binary files /dev/null and b/utils/third_party/nni_new/compression/pytorch/speedup/__pycache__/compress_modules.cpython-38.pyc differ diff --git a/utils/third_party/nni_new/compression/pytorch/speedup/__pycache__/compressor.cpython-38.pyc b/utils/third_party/nni_new/compression/pytorch/speedup/__pycache__/compressor.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..46c33891f84d3a1e995288411a37e5a94bf75b6e Binary files /dev/null and b/utils/third_party/nni_new/compression/pytorch/speedup/__pycache__/compressor.cpython-38.pyc differ diff --git a/utils/third_party/nni_new/compression/pytorch/speedup/__pycache__/infer_shape.cpython-38.pyc b/utils/third_party/nni_new/compression/pytorch/speedup/__pycache__/infer_shape.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f89c8d32ef1c8ed280200e9494347889f427c285 Binary files /dev/null and b/utils/third_party/nni_new/compression/pytorch/speedup/__pycache__/infer_shape.cpython-38.pyc differ diff --git a/utils/third_party/nni_new/compression/pytorch/speedup/compress_modules.py b/utils/third_party/nni_new/compression/pytorch/speedup/compress_modules.py new file mode 100644 index 0000000000000000000000000000000000000000..f183e791fd0655a361e4accadd8bed471be2064e --- /dev/null +++ b/utils/third_party/nni_new/compression/pytorch/speedup/compress_modules.py @@ -0,0 +1,294 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +import torch +from .infer_shape import ModuleMasks + +_logger = logging.getLogger(__name__) +def _dummy_debug(*args): + return None +_logger.debug = _dummy_debug + +replace_module = { + 'BatchNorm2d': lambda module, mask: replace_batchnorm2d(module, mask), + 'Conv2d': lambda module, mask: replace_conv2d(module, mask), + 'ConvTranspose2d': lambda module, mask: replace_convtranspose2d(module, mask), + 'MaxPool2d': lambda module, mask: no_replace(module, mask), + 'AvgPool2d': lambda module, mask: no_replace(module, mask), + 'AdaptiveAvgPool2d': lambda module, mask: no_replace(module, mask), + 'ReLU': lambda module, mask: no_replace(module, mask), + 'ReLU6': lambda module, mask: no_replace(module, mask), + 'Sigmoid': lambda module, mask: no_replace(module, mask), + 'Linear': lambda module, mask: replace_linear(module, mask), + 'Dropout': lambda module, mask: no_replace(module, mask), + 'Dropout2d': lambda module, mask: no_replace(module, mask), + 'Dropout3d': lambda module, mask: no_replace(module, mask), + + # implemented by queyu, 2021/06/21 + 'ShuffleBlock': lambda module, mask: no_replace(module, mask), + 'hswish': lambda module, mask: no_replace(module, mask), + 'hsigmoid': lambda module, mask: no_replace(module, mask), + 'LeakyReLU': lambda module, mask: no_replace(module, mask), + 'ZeroPad2d': lambda module, mask: no_replace(module, mask), + + 'SiLU': lambda module, mask: no_replace(module, mask), +} + + +def no_replace(module, mask): + """ + No need to replace + """ + _logger.debug("no need to replace") + return module + + +def replace_linear(linear, mask): + """ + Parameters + ---------- + linear : torch.nn.Linear + The linear module to be replace + mask : ModuleMasks + The masks of this module + + Returns + ------- + torch.nn.Linear + The new linear module + """ + assert isinstance(mask, ModuleMasks) + assert mask.input_mask is not None + assert mask.output_mask is None + assert not mask.param_masks + index = mask.input_mask.mask_index[-1] + in_features = index.size()[0] + _logger.debug("replace linear with new in_features: %d", in_features) + new_linear = torch.nn.Linear(in_features=in_features, + out_features=linear.out_features, + bias=linear.bias is not None) + new_linear.to(linear.weight.device) + new_linear.weight.data = torch.index_select( + linear.weight.data, -1, index.to(linear.weight.device)) + if linear.bias is not None: + new_linear.bias.data.copy_(linear.bias.data) + return new_linear + + +def replace_batchnorm2d(norm, mask): + """ + Parameters + ---------- + norm : torch.nn.BatchNorm2d + The batchnorm module to be replace + mask : ModuleMasks + The masks of this module + + Returns + ------- + torch.nn.BatchNorm2d + The new batchnorm module + """ + assert isinstance(mask, ModuleMasks) + assert 'weight' in mask.param_masks and 'bias' in mask.param_masks + index = mask.param_masks['weight'].mask_index[0].to(norm.weight.device) + num_features = index.size()[0] + _logger.debug("replace batchnorm2d with num_features: %d", num_features) + new_norm = torch.nn.BatchNorm2d(num_features=num_features, + eps=norm.eps, + momentum=norm.momentum, + affine=norm.affine, + track_running_stats=norm.track_running_stats) + # assign weights + new_norm.weight.data = torch.index_select(norm.weight.data, 0, index) + new_norm.bias.data = torch.index_select(norm.bias.data, 0, index) + if norm.track_running_stats: + new_norm.running_mean.data = torch.index_select( + norm.running_mean.data, 0, index) + new_norm.running_var.data = torch.index_select( + norm.running_var.data, 0, index) + return new_norm + + +def replace_conv2d(conv, mask): + """ + Parameters + ---------- + conv : torch.nn.Conv2d + The conv2d module to be replaced + mask : ModuleMasks + The masks of this module + + Returns + ------- + torch.nn.Conv2d + The new conv2d module + """ + assert isinstance(mask, ModuleMasks) + if mask.input_mask is None: + in_channels = conv.in_channels + else: + in_channels_index = mask.input_mask.mask_index[1] + in_channels = in_channels_index.size()[0] + if mask.output_mask is None: + out_channels = conv.out_channels + else: + out_channels_index = mask.output_mask.mask_index[1] + out_channels = out_channels_index.size()[0] + groups = conv.groups + if conv.in_channels == conv.out_channels == conv.groups: + # remove groups for depthwise layers + assert in_channels == out_channels + groups = in_channels + _logger.debug("replace conv2d %s with in_channels: %d, out_channels: %d", + mask.module_name, in_channels, out_channels) + new_conv = torch.nn.Conv2d(in_channels=in_channels, + out_channels=out_channels, + kernel_size=conv.kernel_size, + stride=conv.stride, + padding=conv.padding, + dilation=conv.dilation, + groups=groups, + bias=conv.bias is not None, + padding_mode=conv.padding_mode) + + new_conv.to(conv.weight.device) + tmp_weight_data = tmp_bias_data = None + + if mask.output_mask is not None: + tmp_weight_data = torch.index_select( + conv.weight.data, 0, out_channels_index.to(conv.weight.device)) + if conv.bias is not None: + tmp_bias_data = torch.index_select( + conv.bias.data, 0, out_channels_index.to(conv.weight.device)) + else: + tmp_weight_data = conv.weight.data + # For the convolutional layers that have more than one group + # we need to copy the weight group by group, because the input + # channal is also divided into serveral groups and each group + # filter may have different input channel indexes. + input_step = int(conv.in_channels / conv.groups) + in_channels_group = int(in_channels / groups) + filter_step = int(out_channels / groups) + if mask.input_mask is not None and not (in_channels == out_channels == groups): + for groupid in range(conv.groups): + start = groupid * input_step + end = (groupid + 1) * input_step + current_input_index = list( + filter(lambda x: start <= x and x < end, in_channels_index.tolist())) + if not current_input_index: + # there is no kept channel in current group + # TODO bug here, the groups is directly get from conv.groups, if the whole group is removed, + # then the number of groups in the new_conv also need to change + raise Exception( + " Donnot support removing the whole group filter except in the depth-wise conv temporarily") + # shift the global index into the group index + current_input_index = [x-start for x in current_input_index] + # if the groups is larger than 1, the input channels of each + # group should be pruned evenly. + assert len(current_input_index) == in_channels_group, \ + 'Input channels of each group are not pruned evenly' + current_input_index = torch.tensor(current_input_index).to(tmp_weight_data.device) # pylint: disable=not-callable + f_start = groupid * filter_step + f_end = (groupid + 1) * filter_step + new_conv.weight.data[f_start:f_end] = torch.index_select( + tmp_weight_data[f_start:f_end], 1, current_input_index) + else: + new_conv.weight.data.copy_(tmp_weight_data) + + if conv.bias is not None: + new_conv.bias.data.copy_( + conv.bias.data if tmp_bias_data is None else tmp_bias_data) + + return new_conv + + +def replace_convtranspose2d(convtrans, mask): + """ + We need anothor replace function for + convtranspose2d, because the layout of + the weight is different from traditional + conv layers. The layout of the weight is [N_in, N_out, ksize_1, ksize_2] + Parameters + ---------- + convtrans : torch.nn.ConvTranspose2d + The conv2d module to be replaced + mask : ModuleMasks + The masks of this module + Returns + ------- + torch.nn.ConvTranspose2d + The new conv2d module + """ + assert isinstance(mask, ModuleMasks) + assert isinstance(convtrans, torch.nn.ConvTranspose2d) + if mask.input_mask is None: + in_channels = convtrans.in_channels + else: + in_channels_index = mask.input_mask.mask_index[1] + in_channels = in_channels_index.size(0) + if mask.output_mask is None: + out_channels = convtrans.out_channels + else: + out_channels_index = mask.output_mask.mask_index[1] + out_channels = out_channels_index.size(0) + groups = convtrans.groups + # check if can remove the whole group of filters + if convtrans.in_channels == convtrans.out_channels == convtrans.groups: + # remove groups for depthwise layers + # this needs the group dependency to be fixed before the speedup + assert in_channels == out_channels + groups = in_channels + _logger.info('Replace convtranspose2d %s with in_channels:%d out_channels:%d', + mask.module_name, in_channels, out_channels) + new_convtrans = torch.nn.ConvTranspose2d(in_channels=in_channels, + out_channels=out_channels, + kernel_size=convtrans.kernel_size, + stride=convtrans.stride, + padding=convtrans.padding, + dilation=convtrans.dilation, + groups=groups, + bias=convtrans.bias is not None, + padding_mode=convtrans.padding_mode) + new_convtrans.to(convtrans.weight.device) + tmp_weight_data = None + if mask.input_mask is not None: + # in convtranspose2d we need to select the input channel first + tmp_weight_data = torch.index_select( + convtrans.weight.data, 0, in_channels_index) + else: + tmp_weight_data = convtrans.weight.data + # we need to handle the output channel group by group like the conv layer + out_step = int(convtrans.out_channels / convtrans.groups) + out_channel_group = int(out_channels/groups) + new_in_per_group = int(in_channels/groups) + + if mask.output_mask is not None and not(in_channels == out_channels == groups): + for groupid in range(convtrans.groups): + start = groupid * out_step + end = (groupid + 1) * out_step + current_output_index = list( + filter(lambda x: start <= x and x < end, out_channels_index.tolist())) + # we need to shift the index into the group-wise + current_output_index = [x-start for x in current_output_index] + if not current_output_index: + # No kept channel in the current group + raise Exception( + " Donnot support removing the whole group filter except in the depth-wise conv temporarily") + assert len(current_output_index) == out_channel_group, \ + 'Output channel of each group should be the same after pruning' + current_output_index = torch.tensor(current_output_index).to(tmp_weight_data.device) # pylint: disable=not-callable + new_start = groupid * new_in_per_group + new_end = (groupid + 1) * new_in_per_group + new_convtrans.weight.data[new_start:new_end] = torch.index_select( + tmp_weight_data[new_start:new_end], 1, current_output_index) + else: + new_convtrans.weight.data.copy_(tmp_weight_data) + if convtrans.bias is not None: + if mask.output_mask is not None: + new_convtrans.bias.data[:] = torch.index_select( + convtrans.bias.data, 0, out_channels_index) + else: + new_convtrans.bias.data.copy_(convtrans.bias.data) + return new_convtrans diff --git a/utils/third_party/nni_new/compression/pytorch/speedup/compressor.py b/utils/third_party/nni_new/compression/pytorch/speedup/compressor.py new file mode 100644 index 0000000000000000000000000000000000000000..b094f03175d47d66454148773f7088d7637b3475 --- /dev/null +++ b/utils/third_party/nni_new/compression/pytorch/speedup/compressor.py @@ -0,0 +1,191 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +import torch +from ....compression.pytorch.utils.mask_conflict import fix_mask_conflict +from ....compression.pytorch.utils.utils import get_module_by_name +from .compress_modules import replace_module +from .infer_shape import ModuleMasks, infer_from_mask, infer_from_inshape, infer_from_outshape, set_conv_prune_dim + +_logger = logging.getLogger(__name__) +def _dummy_debug(*args): + return None +_logger.debug = _dummy_debug + + +class ModelSpeedup: + """ + This class is to speedup the model with provided weight mask + """ + + def __init__(self, model, dummy_input, masks_file, map_location=None): + """ + Parameters + ---------- + model : pytorch model + The model user wants to speed up + dummy_input : pytorch tensor + The dummy input for ```jit.trace```, users should put it on right device before pass in + masks_file : str + The path of user provided mask file + map_location : str + the device on which masks are placed, same to map_location in ```torch.load``` + """ + from ....common.graph_utils import build_module_graph + + self.bound_model = model + self.masks = torch.load(masks_file, map_location) + self.inferred_masks = dict() # key: module_name, value: ModuleMasks + self.dummy_input = dummy_input + self.torch_graph = build_module_graph(model, dummy_input) + + def infer_module_mask(self, module_name, last_module, mask=None, in_shape=None, out_shape=None): + """ + Infer input shape / output shape based on the module's weight mask / input shape / output shape. + + For a module: + Infer its input and output shape from its weight mask + Infer its output shape from its input shape + Infer its input shape from its output shape + + If its input shape is changed, continue infering its predecessors + If its output shape is changed, continue infering its successors + + Parameters + ---------- + module_name : str + The name of the node + last_module : str + The name of last visited node + mask : tensor of mask or ModuleMasks + Mask of the weights in this node (i.e., module) + in_shape : ModuleMasks + Input shape of this node + out_shape : ModuleMasks + Output shape of this node + """ + input_cmask = output_cmask = None + if module_name in self.inferred_masks: + module_masks = self.inferred_masks[module_name] + else: + _, m = get_module_by_name(self.bound_model, module_name) + module_masks = ModuleMasks(module_name, m) + self.inferred_masks[module_name] = module_masks + + m_type = self.torch_graph.name_to_node[module_name].op_type + _logger.debug("infer mask of module %s with op_type %s", module_name, m_type) + if mask is not None: + _logger.debug("mask is not None") + if not m_type in infer_from_mask: + raise RuntimeError( + "Has not supported infering input/output shape from mask for module/function: `{}`, {}" + .format(m_type, module_name)) + if m_type in ['Linear']: + input_cmask, output_cmask = infer_from_mask[m_type]( + module_masks, mask, self.torch_graph.name_to_node[module_name].auxiliary + ) + else: + input_cmask, output_cmask = infer_from_mask[m_type](module_masks, mask) + if in_shape is not None: + _logger.debug("in_shape is not None") + if not m_type in infer_from_inshape: + raise RuntimeError( + "Has not supported infering output shape from input shape for module/function: `{}`, {}" + .format(m_type, module_name)) + if m_type in ['aten::view', 'aten::flatten', 'aten::mean', 'aten::reshape']: + output_cmask = infer_from_inshape[m_type](module_masks, + in_shape, + self.torch_graph.name_to_node[module_name].auxiliary) + elif m_type in ['aten::cat']: + # To calculate the mask for concat operation, the output shape + # , cat dimension, and the order of the input parameters. + output_cmask = infer_from_inshape[m_type](module_masks, + in_shape, + self.torch_graph.name_to_node[module_name].auxiliary, + last_module) + else: + output_cmask = infer_from_inshape[m_type](module_masks, in_shape) + if out_shape is not None: + _logger.debug("out_shape is not None") + if not m_type in infer_from_outshape: + raise RuntimeError( + "Has not supported infering input shape from output shape for module/function: `{}`, {}" + .format(m_type, module_name)) + if m_type in ['aten::view', 'aten::flatten', 'aten::mean', 'aten::reshape']: + input_cmask = infer_from_outshape[m_type](module_masks, out_shape, self.torch_graph.name_to_node[module_name].auxiliary) + else: + input_cmask = infer_from_outshape[m_type](module_masks, out_shape) + + if input_cmask: + predecessors = self.torch_graph.find_predecessors(module_name) + for _module_name in predecessors: + self.infer_module_mask(_module_name, module_name, out_shape=input_cmask) + if output_cmask: + successors = self.torch_graph.find_successors(module_name) + for _module_name in successors: + self.infer_module_mask(_module_name, module_name, in_shape=output_cmask) + + def infer_modules_masks(self): + """ + Do shape inference of involved modules, including the shape of weights, inputs, output + """ + for module_name, mask in self.masks.items(): + _logger.debug('Start mask inference from %s', module_name) + if module_name not in self.torch_graph.name_to_node: + # this module is not traced in the torch_graph, + # jit.trace only correctly records functions and + # modules which are not data dependent (e.g., do + # not have conditionals on data in tensors) + # so, if a node is not traced, we just skip it. + _logger.warning('%s has mask, but not found in the traced graph, just skip it.', module_name) + continue + self.infer_module_mask(module_name, None, mask=mask) + + def replace_compressed_modules(self): + """ + Replace all the modules that have changed (weights/inputs/output) shape. + The new module is created using the same arguments of the to-be-replaced module, + and correctly inherits its weights. + + NOTE: ```func``` type cannot be replaced as it is not a module, thus, one limitation + is that ```func``` should be not required to be replaced. + """ + for module_name in self.inferred_masks: + g_node = self.torch_graph.name_to_node[module_name] + _logger.debug("replace %s, in %s type, with op_type %s", + module_name, g_node.type, g_node.op_type) + if g_node.type == 'module': + super_module, leaf_module = get_module_by_name(self.bound_model, g_node.name) + m_type = g_node.op_type + if not m_type in replace_module: + raise RuntimeError("Has not supported replacing the module: `{}`".format(m_type)) + _logger.info("replace module (name: %s, op_type: %s)", g_node.name, m_type) + compressed_module = replace_module[m_type](leaf_module, self.inferred_masks[module_name]) + setattr(super_module, g_node.name.split('.')[-1], compressed_module) + elif g_node.type == 'func': + _logger.info("Warning: cannot replace (name: %s, op_type: %s) which is func type", + module_name, g_node.op_type) + else: + raise RuntimeError("Unsupported node type: {}".format(g_node.type)) + + def speedup_model(self): + """ + There are basically two steps: + first, do mask/shape inference, + second, replace modules + """ + training = self.bound_model.training + _logger.info("start to speed up the model") + _logger.info("fix the mask conflict of the interdependent layers") + fixed_masks, conv_prune_dim = fix_mask_conflict(self.masks, self.bound_model, self.dummy_input) + set_conv_prune_dim(conv_prune_dim) + + _logger.info("infer module masks...") + self.infer_modules_masks() + _logger.info("replace compressed modules...") + self.replace_compressed_modules() + self.bound_model.train(training) + _logger.info("speedup done") + + return fixed_masks diff --git a/utils/third_party/nni_new/compression/pytorch/speedup/infer_shape.py b/utils/third_party/nni_new/compression/pytorch/speedup/infer_shape.py new file mode 100644 index 0000000000000000000000000000000000000000..e04c50c527148c27033313238dcf6a4676f26949 --- /dev/null +++ b/utils/third_party/nni_new/compression/pytorch/speedup/infer_shape.py @@ -0,0 +1,1189 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. +""" +For each operation or module, there are two functions. +One is given output shape, infer its input shape and initialization parameters (e.g., weight's shape) +The other is given input shape, infer its output shape and initialization parameters (e.g., weight's shape) +""" + +import logging +import torch + +_logger = logging.getLogger(__name__) + +conv_prune_dim = -1 + + +def set_conv_prune_dim(dim): + """ + Parameters: + dim: int + 0: filter pruning + 1: channel pruning + """ + global conv_prune_dim + conv_prune_dim = dim + + +class CoarseMask: + """ + Coarse grained mask for a given tensor, here tensor could be weights, + input tensor, or output tensor + """ + + def __init__(self, num_dim): + """ + Parameters + ---------- + num_dim : int + The number of dimensions of the tensor that will be masked + """ + self.mask_index = [None for _ in range(num_dim)] + + def add_index_mask(self, dim, index): + """ + Add mask for the specified dimension + + Parameters + ---------- + dim : int + The dimension to add mask + index : tensor + The mask for this dimension, its a 1 dimension tensor which specifies + the index of the elements that are not pruned + """ + self.mask_index[dim] = index + + @staticmethod + def merge_index(index_a, index_b): + """ + Parameters + ---------- + index_a : tensor + One index (1-dimension) tensor + index_b : tensor + The other index (1-dimension) tensor + + Returns + ------- + tensor + The merged index (1-dimension) tensor + Note that: the output tensor will be moved + to the same device as index_a. + """ + device = index_a.device + s = set() + for num in index_a.tolist(): + # we need to transfer the tensor to list here + # first, directly traversing the tensor by for + # loop will return the list of tensor(x) object, + # even the value are the same, but they are different + # tensor objects, so the set will contains multiple + # tensor objects that has the same value. For example + # for num in torch.ones(2): + # s.add(num) + # s will be {tensor(1), tensor(1)} + s.add(num) + for num in index_b.tolist(): + s.add(num) + # move the output tensor to the same device with index_a + return torch.tensor(sorted(s)).to(device) # pylint: disable=not-callable + + def merge(self, cmask): + """ + Merge another CoarseMask + + Parameters + ---------- + cmask : CoarseMask + Another CoarseMask to merge + + Returns + ------- + list + The member variable ```mask_index``` + """ + assert isinstance(cmask, CoarseMask) + assert len(self.mask_index) == len(cmask.mask_index), \ + "Only masks with the same number of dimensions can be merged" + for i, index in enumerate(self.mask_index): + if index is None: + self.mask_index[i] = cmask.mask_index[i] + elif cmask.mask_index[i] is not None: + self.mask_index[i] = CoarseMask.merge_index(self.mask_index[i], + cmask.mask_index[i]) + return self.mask_index + + def __repr__(self): + return 'mask_index: {}'.format(self.mask_index) + + def eq_on_dim(self, other, dim): + assert isinstance(other, CoarseMask) + if self.mask_index[dim] is None and other.mask_index[dim] is None: + return True + elif isinstance(self.mask_index[dim], torch.Tensor) \ + and isinstance(other.mask_index[dim], torch.Tensor): + return torch.equal(self.mask_index[dim], other.mask_index[dim]) + else: + return False + + def __eq__(self, other): + assert isinstance(other, CoarseMask) + if len(self.mask_index) != len(other.mask_index): + return False + for i in range(len(self.mask_index)): + if not self.eq_on_dim(other, i): + return False + return True + + def __lt__(self, other): + """ + Judge if the mask is a subset of another CoarseMask. + """ + assert isinstance(other, CoarseMask) + for dim, _ in enumerate(self.mask_index): + # if self has more dimensions + if dim >= len(other.mask_index): + return False + if self.mask_index[dim] is None: + # if no mask on this dimension, then we have less + # masks then the other CoraseMask. + continue + elif other.mask_index[dim] is None: + return False + else: + s1 = set(self.mask_index[dim].tolist()) + s2 = set(other.mask_index[dim].tolist()) + if not s1 < s2: + return False + return True + + def __le__(self, other): + """ + Return if self's mask is less or equal to other's mask. + """ + assert isinstance(other, CoarseMask) + if self.__lt__(other) or self.__eq__(other): + return True + return False + + def __ne__(self, other): + return not self.__eq__(other) + + +class ModuleMasks: + """ + The masks of a module, including the masks for weights, inputs, output + """ + + def __init__(self, module_name, module=None): + """ + Parameters + ---------- + module_name : str + The name of the module or function + """ + self.module_name = module_name + self.module = module + self.param_masks = dict() + self.input_mask = None + self.output_mask = None + + def set_param_masks(self, name, mask): + """ + Parameters + ---------- + name : str + The name of the weight + mask : CoarseMask + The mask for this weight + """ + self.param_masks[name] = mask + + def set_input_mask(self, mask): + """ + Parameters + ---------- + mask : CoarseMask + The mask for input + """ + self.input_mask = mask + + def set_output_mask(self, mask): + """ + Parameters + ---------- + mask : CoarseMask + The mask for output + """ + self.output_mask = mask + + def __repr__(self): + return 'module_name: {}, input_mask: {}, output_mask: {}, param_masks: {}'.format( + self.module_name, self.input_mask, self.output_mask, self.param_masks + ) + + +""" +Infer input and output shape of a module/function from its weight mask +""" +infer_from_mask = { + 'BatchNorm2d': lambda module_masks, mask: batchnorm2d_mask(module_masks, mask), + 'Conv2d': lambda module_masks, mask: conv2d_mask(module_masks, mask), + 'ConvTranspose2d': lambda module_masks, mask: convtranspose2d_mask(module_masks, mask), + 'Linear': lambda module_masks, mask, shape: linear_mask(module_masks, mask, shape) +} + +""" +Infer output and weight shape of a module/function from its input shape +""" +infer_from_inshape = { + 'ReLU': lambda module_masks, mask: relu_inshape(module_masks, mask), + 'ReLU6': lambda module_masks, mask: relu_inshape(module_masks, mask), + 'Sigmoid': lambda module_masks, mask: relu_inshape(module_masks, mask), + 'aten::relu': lambda module_masks, mask: relu_inshape(module_masks, mask), + 'aten::tanh': lambda module_masks, mask: relu_inshape(module_masks, mask), + 'aten::tanh_': lambda module_masks, mask: relu_inshape(module_masks, mask), + 'aten::hardtanh': lambda module_masks, mask: relu_inshape(module_masks, mask), + 'aten::hardtanh_': lambda module_masks, mask: relu_inshape(module_masks, mask), + 'aten::relu_': lambda module_masks, mask: relu_inshape(module_masks, mask), + 'aten::sigmoid': lambda module_masks, mask: relu_inshape(module_masks, mask), + 'Conv2d': lambda module_masks, mask: conv2d_inshape(module_masks, mask), + 'ConvTranspose2d': lambda module_masks, mask: convtranspose2d_inshape(module_masks, mask), + 'MaxPool2d': lambda module_masks, mask: maxpool2d_inshape(module_masks, mask), + 'aten::max_pool2d': lambda module_masks, mask: maxpool2d_inshape(module_masks, mask), + 'aten::avg_pool2d': lambda module_masks, mask: maxpool2d_inshape(module_masks, mask), + 'aten::adaptive_avg_pool2d': lambda module_masks, mask: maxpool2d_inshape(module_masks, mask), + 'AvgPool2d': lambda module_masks, mask: maxpool2d_inshape(module_masks, mask), + 'AdaptiveAvgPool2d': lambda module_masks, mask: maxpool2d_inshape(module_masks, mask), + 'aten::size': lambda module_masks, mask: size_inshape(module_masks, mask), + 'aten::view': lambda module_masks, mask, shape: view_inshape(module_masks, mask, shape), + 'aten::reshape': lambda module_masks, mask, shape: view_inshape(module_masks, mask, shape), + # support only start_dim=1 + 'aten::flatten': lambda module_masks, mask, shape: view_inshape(module_masks, mask, shape), + 'Linear': lambda module_masks, mask: linear_inshape(module_masks, mask), + 'BatchNorm2d': lambda module_masks, mask: batchnorm2d_inshape(module_masks, mask), + 'aten::add_': lambda module_masks, mask: add_inshape(module_masks, mask), + 'aten::add': lambda module_mask, mask: add_inshape(module_mask, mask), + # mul has the similar behaviour with add, they both request + # the input tesors to have the same shape + 'aten::mul': lambda module_mask, mask: add_inshape(module_mask, mask), + 'aten::mul_': lambda module_mask, mask: add_inshape(module_mask, mask), + 'aten::div': lambda module_mask, mask: add_inshape(module_mask, mask), + 'aten::div_': lambda module_mask, mask: add_inshape(module_mask, mask), + 'aten::cat': lambda module_mask, mask, cat_info, last_visited: cat_inshape(module_mask, mask, cat_info, last_visited), + 'aten::mean': lambda module_masks, mask, shape: mean_inshape(module_masks, mask, shape), + 'Dropout': lambda module_masks, mask: dropout_inshape(module_masks, mask), + 'Dropout2d': lambda module_masks, mask: dropout_inshape(module_masks, mask), + 'aten::dropout': lambda module_masks, mask: dropout_inshape(module_masks, mask), + 'aten::detach': lambda module_masks, mask: dropout_inshape(module_masks, mask), + + # implemented by queyu, 2021/06/21, + 'ShuffleBlock': lambda module_masks, mask: relu_inshape(module_masks, mask), + 'hswish': lambda module_masks, mask: relu_inshape(module_masks, mask), + 'hsigmoid': lambda module_masks, mask: relu_inshape(module_masks, mask), + 'Swish': lambda module_masks, mask: relu_inshape(module_masks, mask), + # implemented by queyu, 2021/07/04 + 'LeakyReLU': lambda module_masks, mask: relu_inshape(module_masks, mask), + 'ZeroPad2d': lambda module_masks, mask: maxpool2d_inshape(module_masks, mask), + # implemented by queyu, 2022/08/05 + 'aten::upsample_bilinear2d': lambda module_masks, mask: relu_inshape(module_masks, mask), + # implemented by queyu, 2022/08/05 + 'SiLU': lambda module_masks, mask: relu_inshape(module_masks, mask), +} + +""" +Infer input and weight shape of a module/function from its output shape +""" +infer_from_outshape = { + 'Conv2d': lambda module_masks, mask: conv2d_outshape(module_masks, mask), + 'ConvTranspose2d': lambda module_masks, mask: convtranspose2d_outshape(module_masks, mask), + 'BatchNorm2d': lambda module_masks, mask: batchnorm2d_outshape(module_masks, mask), + + 'MaxPool2d': lambda module_masks, mask: maxpool2d_outshape(module_masks, mask), + 'aten::max_pool2d': lambda module_masks, mask: maxpool2d_outshape(module_masks, mask), + 'aten::avg_pool2d': lambda module_masks, mask: maxpool2d_outshape(module_masks, mask), + 'aten::adaptive_avg_pool2d': lambda module_masks, mask: maxpool2d_outshape(module_masks, mask), + 'AvgPool2d': lambda module_masks, mask: maxpool2d_outshape(module_masks, mask), + 'AdaptiveAvgPool2d': lambda module_masks, mask: maxpool2d_outshape(module_masks, mask), + + 'ReLU': lambda module_masks, mask: relu_outshape(module_masks, mask), + 'ReLU6': lambda module_masks, mask: relu_outshape(module_masks, mask), + 'aten::relu': lambda module_masks, mask: relu_outshape(module_masks, mask), + 'aten::tanh': lambda module_masks, mask: relu_outshape(module_masks, mask), + 'aten::tanh_': lambda module_masks, mask: relu_outshape(module_masks, mask), + 'aten::hardtanh': lambda module_masks, mask: relu_outshape(module_masks, mask), + 'aten::hardtanh_': lambda module_masks, mask: relu_outshape(module_masks, mask), + 'aten::relu_': lambda module_masks, mask: relu_outshape(module_masks, mask), + + 'aten::add_': lambda module_masks, mask: add_outshape(module_masks, mask), + 'aten::add': lambda module_mask, mask: add_outshape(module_mask, mask), + 'aten::flatten': lambda module_mask, mask, shape: view_outshape(module_mask, mask, shape), + 'aten::view': lambda module_masks, mask, shape: view_outshape(module_masks, mask, shape), + 'aten::reshape': lambda module_masks, mask, shape: view_outshape(module_masks, mask, shape), + 'aten::mean': lambda module_masks, mask, shape: mean_outshape(module_masks, mask, shape), + 'Dropout': lambda module_masks, mask: dropout_outshape(module_masks, mask), + 'Dropout2d': lambda module_masks, mask: dropout_outshape(module_masks, mask), + 'aten::dropout': lambda module_masks, mask: dropout_outshape(module_masks, mask), + 'aten::detach': lambda module_masks, mask: dropout_outshape(module_masks, mask) +} + + +def dropout_inshape(module_masks, mask): + if module_masks.input_mask is None: + module_masks.set_input_mask(mask) + module_masks.set_output_mask(mask) + return module_masks.output_mask + # if alreay visited + assert module_masks.input_mask <= mask + # It should be the same, we pass the masks by the reference(not the value), + # so they acutually are two references of the same object(mask, + # module_masks.input_mask). So we should continue pass the mask + # to the following nodes even module_masks.input_mask == mask. + # if pass the mask by copy.deepcopy(), then we can stop when + # module_masks.input_mask == mask. + # if module_masks.input_mask == mask: + # return None + module_masks.set_input_mask(mask) + module_masks.set_output_mask(mask) + return module_masks.output_mask + + +def dropout_outshape(module_masks, mask): + if module_masks.output_mask is None: + module_masks.set_output_mask(mask) + module_masks.set_input_mask(mask) + return module_masks.input_mask + # if alreay visited + assert all(module_masks.output_mask.mask_index[1] == mask.mask_index[1]) + + return module_masks.output_mask + + +def cat_inshape(module_masks, mask, cat_info, last_visited): + """ + Inference the output mask of the cat operation from the + input mask. + + Parameters + ---------- + module_masks : ModuleMasks + The ModuleMasks instance of the Conv2d + mask : CoarseMask + The mask of its input tensor + cat_info: dict + Dict object that records the necessary information + of cat operation, such as the order of the input + tensors. + last_visited: str + The unique_name of the last visited node group. + + Returns + ------- + CoarseMask + The mask of its output tensor + + """ + assert isinstance(mask, CoarseMask) + out_shape = cat_info['out_shape'] + cat_dim = cat_info['cat_dim'] + in_order = cat_info['in_order'] + in_shape = cat_info['in_shape'] + if module_masks.output_mask is None: + # First visit to this cat node + # initialize the mask based on + # the number of the output channel. + output_mask = CoarseMask(num_dim=len(out_shape)) + for dim, _ in enumerate(out_shape): + if dim == cat_dim: + if mask.mask_index[dim] is None: + continue + device = mask.mask_index[dim].device + # calculate the offset of the mask + pos = in_order.index(last_visited) + offsets = [in_shape[i][cat_dim] + for i, _ in enumerate(in_shape)] + offset = 0 + for i in range(pos): + offset += offsets[i] + _tmp_mask = (mask.mask_index[dim] + offset).to(device) + output_mask.mask_index[dim] = _tmp_mask + else: + # directly copy the mask + if mask.mask_index[dim] is not None: + output_mask.mask_index[dim] = mask.mask_index[dim].data.clone( + ) + module_masks.set_output_mask(output_mask) + + return module_masks.output_mask + # If this cat node is already visited, we need + # validating if the mask is legel, for cat operation, + # the mask on the 'cat_dim' dimension should be stitched + # together. In the other dimensions, the mask should be + # the same, else the mask is not legal. + for dim, _ in enumerate(out_shape): + if dim == cat_dim: + if mask.mask_index[dim] is None: + continue + pos = in_order.index(last_visited) + offsets = [in_shape[i][cat_dim] for i, _ in enumerate(in_shape)] + offset = 0 + for i in range(pos): + offset += offsets[i] + device = mask.mask_index[dim].device + new_mask = mask.mask_index[dim] + offset + module_masks.output_mask.mask_index[dim] = CoarseMask.merge_index( + module_masks.output_mask.mask_index[dim], new_mask).to(device) + else: + assert module_masks.output_mask.eq_on_dim(mask, dim) + + return module_masks.output_mask + + +def add_inshape(module_masks, mask): + """ + Inference the output mask of the add operation from the + input mask. + """ + assert isinstance(mask, CoarseMask) + if module_masks.input_mask is None: + module_masks.set_input_mask(mask) + module_masks.set_output_mask(mask) + # module_masks.input_mask = mask + return mask + # If alreay visited, validate if have the conflict + # if the mask is different with previous input_mask + # then there is a mask confilct. + if mask != module_masks.input_mask: + raise Exception('Mask conflict happenes!') + return None + + +def add_outshape(module_masks, mask): + """ + Inference the input mask of the add operation from the + output mask. + """ + assert isinstance(mask, CoarseMask) + + if module_masks.output_mask is None: + module_masks.set_output_mask(mask) + module_masks.set_input_mask(mask) + return mask + else: + assert all( + module_masks.output_mask.mask_index[1] == mask.mask_index[1]) + return mask + + +def batchnorm2d_inshape(module_masks, mask): + """ + We assume only the second dimension has coarse grained mask + + Parameters + ---------- + module_masks : ModuleMasks + The ModuleMasks instance of the batchnorm2d + mask : CoarseMask + The mask of its input tensor + + Returns + ------- + CoarseMask + The mask of its output tensor + """ + assert isinstance(mask, CoarseMask) + assert mask.mask_index[1] is not None + assert mask.mask_index[0] is None + assert mask.mask_index[2] is None + assert mask.mask_index[3] is None + module_masks.set_input_mask(mask) + module_masks.set_output_mask(mask) + weight_cmask = CoarseMask(num_dim=1) + weight_cmask.add_index_mask(dim=0, index=mask.mask_index[1]) + module_masks.set_param_masks('weight', weight_cmask) + module_masks.set_param_masks('bias', weight_cmask) + return mask + + +def batchnorm2d_outshape(module_masks, mask): + """ + We assume only the second dimension has coarse grained mask + + Parameters + ---------- + module_masks : ModuleMasks + The ModuleMasks instance of the batchnorm2d + mask : CoarseMask + The mask of its input tensor + + Returns + ------- + CoarseMask + The mask of its output tensor + """ + assert isinstance(mask, CoarseMask) + assert len(mask.mask_index) in [2, 4] + assert mask.mask_index[1] is not None + assert mask.mask_index[0] is None + module_masks.set_input_mask(mask) + module_masks.set_output_mask(mask) + weight_cmask = CoarseMask(num_dim=1) + weight_cmask.add_index_mask(dim=0, index=mask.mask_index[1]) + module_masks.set_param_masks('weight', weight_cmask) + module_masks.set_param_masks('bias', weight_cmask) + return mask + + +def linear_inshape(module_masks, mask): + """ + Coarse grained input mask does not change the shape of weights and output tensor + + Parameters + ---------- + module_masks : ModuleMasks + The ModuleMasks instance of the linear + mask : CoarseMask + The mask of its input tensor + + Returns + ------- + CoarseMask + The mask of its output tensor, ```None``` means shape of output tensor is not changed + """ + assert isinstance(mask, CoarseMask) + assert mask.mask_index[0] is None + if module_masks.input_mask is not None: + assert module_masks.input_mask <= mask + module_masks.set_input_mask(mask) + return None + + +def view_inshape(module_masks, mask, shape): + """ + This is a limited support + + TODO: consider replace tensor.view with nn.Flatten, because tensor.view is not + included in module, thus, cannot be replaced by our framework. + + Parameters + ---------- + module_masks : ModuleMasks + The ModuleMasks instance of the ```view``` op + mask : CoarseMask + The mask of its input tensor + shape : dict + Original shape of its input and output tensors + + Returns + ------- + CoarseMask + The mask of its output tensor + """ + # NOTE: the case constrained by the following four asserts + assert shape['in_shape'][0] == shape['out_shape'][0] + assert len(shape['in_shape']) == 4 + assert len(shape['out_shape']) == 2 + assert shape['out_shape'][1] == shape['in_shape'][1] * \ + shape['in_shape'][2]*shape['in_shape'][3] + + assert isinstance(mask, CoarseMask) + assert mask.mask_index[1] is not None + assert mask.mask_index[0] is None + assert mask.mask_index[2] is None + assert mask.mask_index[3] is None + # due to the cat operation, the same node may be + # accessed more than once + if module_masks.input_mask is not None: + assert module_masks.input_mask <= mask + module_masks.set_input_mask(mask) + output_cmask = CoarseMask(num_dim=2) + index = [] + step_size = shape['in_shape'][2] * shape['in_shape'][3] + for loc in mask.mask_index[1]: + index.extend([loc * step_size + i for i in range(step_size)]) + output_cmask.add_index_mask(dim=1, index=torch.tensor(index).to(mask.mask_index[1].device)) # pylint: disable=not-callable + module_masks.set_output_mask(output_cmask) + return output_cmask + + +def view_outshape(module_masks, mask, shape): + """ + Parameters + ---------- + module_masks : ModuleMasks + The ModuleMasks instance of the ```view``` op + mask : CoarseMask + The mask of its output tensor + shape : dict + Original shape of its input and output tensors + Returns + ------- + CoarseMask + The mask of its input tensor + """ + # NOTE: the case constrained by the following four asserts + assert shape['in_shape'][0] == shape['out_shape'][0] + assert len(shape['in_shape']) == 4 + assert len(shape['out_shape']) == 2 + assert shape['out_shape'][1] == shape['in_shape'][1] * \ + shape['in_shape'][2]*shape['in_shape'][3] + + assert isinstance(mask, CoarseMask) + assert mask.mask_index[1] is not None + assert mask.mask_index[0] is None + + module_masks.set_output_mask(mask) + input_cmask = CoarseMask(num_dim=4) + index = set() + step_size = shape['in_shape'][2] * shape['in_shape'][3] + for loc in mask.mask_index[1]: + index.add(loc // step_size) + index = sorted(list(index)) + input_cmask.add_index_mask(dim=1, index=torch.tensor(index).to(mask.mask_index[1].device)) # pylint: disable=not-callable + module_masks.set_input_mask(input_cmask) + + return input_cmask + + +def size_inshape(module_masks, mask): + """ + No need to do anything for this ```size``` op + """ + return None + + +def mean_inshape(module_masks, mask, shape): + """ + Similar to view operation, currently mask inference only supports + the mean operation on the 3rd and 4th dimensions. + """ + assert shape['in_shape'][0] == shape['out_shape'][0] + assert shape['out_shape'][1] == shape['in_shape'][1] + assert len(shape['in_shape']) == 4 + assert len(shape['out_shape']) == 2 + + assert isinstance(mask, CoarseMask) + assert mask.mask_index[1] is not None + assert mask.mask_index[0] is None + assert mask.mask_index[2] is None + assert mask.mask_index[3] is None + module_masks.set_input_mask(mask) + + output_cmask = CoarseMask(num_dim=2) + output_cmask.add_index_mask(dim=1, index=mask.mask_index[1]) + module_masks.set_output_mask(output_cmask) + return output_cmask + + +def mean_outshape(module_masks, mask, shape): + """ + Similar to view operation, currently mask inference only supports + the mean operation on the 3rd and 4th dimensions. + """ + assert shape['in_shape'][0] == shape['out_shape'][0] + assert shape['out_shape'][1] == shape['in_shape'][1] + assert len(shape['in_shape']) == 4 + assert len(shape['out_shape']) == 2 + + assert isinstance(mask, CoarseMask) + assert mask.mask_index[1] is not None + assert mask.mask_index[0] is None + module_masks.set_output_mask(mask) + + input_cmask = CoarseMask(num_dim=4) + input_cmask.add_index_mask(dim=1, index=mask.mask_index[1]) + module_masks.set_input_mask(input_cmask) + return input_cmask + + +def maxpool2d_inshape(module_masks, mask): + """ + Assume only the second dimension is masked + + Parameters + ---------- + module_masks : ModuleMasks + The ModuleMasks instance of the maxpool2d + mask : CoarseMask + The mask of its input tensor + + Returns + ------- + CoarseMask + The mask of its output tensor + """ + assert isinstance(mask, CoarseMask) + assert mask.mask_index[1] is not None + assert mask.mask_index[0] is None + assert mask.mask_index[2] is None + assert mask.mask_index[3] is None + if module_masks.input_mask is not None: + assert module_masks.input_mask <= mask + # assert module_masks.input_mask is None + module_masks.set_input_mask(mask) + module_masks.set_output_mask(mask) + return mask + + +def maxpool2d_outshape(module_masks, mask): + """ + Assume only the second dimension is masked + + Parameters + ---------- + module_masks : ModuleMasks + The ModuleMasks instance of the maxpool2d + mask : CoarseMask + The mask of its input tensor + + Returns + ------- + CoarseMask + The mask of its output tensor + """ + assert isinstance(mask, CoarseMask) + assert mask.mask_index[1] is not None + assert mask.mask_index[0] is None + + module_masks.set_input_mask(mask) + module_masks.set_output_mask(mask) + return mask + + +def relu_inshape(module_masks, mask): + """ + Parameters + ---------- + module_masks : ModuleMasks + The ModuleMasks instance of the relu + mask : CoarseMask + The mask of its input tensor + + Returns + ------- + CoarseMask + The mask of its output tensor + """ + assert isinstance(mask, CoarseMask) + if module_masks.input_mask is not None: + # mask conflict should be solved before speedup + assert module_masks.input_mask <= mask + # assert module_masks.input_mask is None, "A relu op can only be processed once" + module_masks.set_input_mask(mask) + module_masks.set_output_mask(mask) + return mask + + +def relu_outshape(module_masks, mask): + """ + Parameters + ---------- + module_masks : ModuleMasks + The ModuleMasks instance of the relu + mask : CoarseMask + The mask of its input tensor + + Returns + ------- + CoarseMask + The mask of its output tensor + """ + assert isinstance(mask, CoarseMask) + if module_masks.output_mask is not None: + # mask conflict should be solved before speedup + assert all( + module_masks.output_mask.mask_index[1] == mask.mask_index[1]) + module_masks.set_input_mask(mask) + module_masks.set_output_mask(mask) + return mask + + +def batchnorm2d_mask(module_masks, mask): + """ + Infer input and output shape from weight mask + Parameters + ---------- + module_masks : ModuleMasks + The ModuleMasks instance of the batchnorm2d + mask : dict + The mask of its weights, from the user provided mask file + Returns + ------- + CoarseMask, CoarseMask + The mask of its input tensor, the mask of its output tensor + """ + assert 'weight' in mask and 'bias' in mask + sum_mask = mask['weight'] + mask['bias'] + nonzero_index = torch.nonzero(sum_mask, as_tuple=True)[0] + # infer shape of parameters + param_cmask = CoarseMask(num_dim=1) + param_cmask.add_index_mask(dim=0, index=nonzero_index) + module_masks.set_param_masks('weight', param_cmask) + module_masks.set_param_masks('bias', param_cmask) + # infer shape of input tensor + input_cmask = CoarseMask(num_dim=4) + input_cmask.add_index_mask(dim=1, + index=torch.nonzero(mask['weight'], as_tuple=True)[0]) + module_masks.set_input_mask(input_cmask) + # infer shape of output tensor + output_cmask = CoarseMask(num_dim=4) + output_cmask.add_index_mask(dim=1, index=nonzero_index) + module_masks.set_output_mask(output_cmask) + return input_cmask, output_cmask + + +def linear_mask(module_masks, mask, shape): + """ + Infer input and output shape from weight mask with limitations: + Only support infer input mask + + Parameters + ---------- + module_masks : ModuleMasks + The ModuleMasks instance of the Linear + mask : dict + The mask of its weights, from the user provided mask file + shape: dict + Shape of its input and output tensors + Returns + ------- + CoarseMask, CoarseMask + The mask of its input tensor, the mask of its output tensor + """ + + assert 'weight' in mask + num_input_dim = len(shape['in_shape']) + + # Input data of Linear module can have multiple dimensions. + # here we only support infer coarse mask on the first dimension (dimension 0) + nonzero_index = torch.nonzero(mask['weight'].sum(0), as_tuple=True)[0] + + # infer shape of input tensor + input_cmask = CoarseMask(num_dim=num_input_dim) + input_cmask.add_index_mask(dim=num_input_dim-1, index=nonzero_index) + + module_masks.set_input_mask(input_cmask) + return input_cmask, None + + +def conv2d_mask(module_masks, mask): + """ + Infer input and output shape from weight mask + + Parameters + ---------- + module_masks : ModuleMasks + The ModuleMasks instance of the conv2d + mask : dict + The mask of its weights, from the user provided mask file + + Returns + ------- + CoarseMask, CoarseMask + The mask of its input tensor, the mask of its output tensor + """ + def convert_to_coarse_mask(mask, dim=0): + """ + Parameters + ---------- + mask : dict + Weight mask from user provided mask file + dim: int + 0: filter pruning + 1: channel pruning + + Returns + ------- + LongTensor, CoarseMask, CoarseMask + Index of the masked dimension, weight mask, bias mask + """ + assert 'weight' in mask + assert isinstance(mask['weight'], torch.Tensor) + assert dim in [0, 1] + + weight_mask = mask['weight'] + + sum_idx = (1, 2, 3) if dim == 0 else (0, 2, 3) + index = torch.nonzero(weight_mask.abs().sum( + sum_idx) != 0, as_tuple=True)[0] + + index = index.long().to(weight_mask.device) + weight_cmask = CoarseMask(num_dim=4) + weight_cmask.add_index_mask(dim=dim, index=index) + bias_cmask = None + if dim == 0 and 'bias' in mask and mask['bias'] is not None: + bias_index = torch.nonzero(mask['bias'], as_tuple=True)[0] + assert torch.all(torch.eq(index, bias_index)), \ + "bias mask should be consistent with weight mask" + bias_cmask = CoarseMask(num_dim=1) + bias_cmask.add_index_mask(dim=0, index=bias_index) + return index, weight_cmask, bias_cmask + + index, weight_cmask, bias_cmask = convert_to_coarse_mask( + mask, dim=conv_prune_dim) + + if index is None: + # TODO: fine grained mask speedup + return None, None + # deal with coarse grain mask + # mask conflict should be solved by fix_mask_conflict before speedup + if 'weight' in module_masks.param_masks: + assert module_masks.param_masks['weight'] == weight_cmask + else: + module_masks.set_param_masks('weight', weight_cmask) + if conv_prune_dim == 0: + module_masks.set_param_masks('bias', bias_cmask) + + io_cmask = CoarseMask(num_dim=4) + io_cmask.add_index_mask(dim=1, index=index) + + if conv_prune_dim == 0: + if module_masks.output_mask is None: + module_masks.set_output_mask(io_cmask) + else: + assert module_masks.output_mask == io_cmask + return None, module_masks.output_mask + else: + if module_masks.input_mask is None: + module_masks.set_input_mask(io_cmask) + else: + assert module_masks.input_mask == io_cmask + return module_masks.input_mask, None + + +def conv2d_inshape(module_masks, mask): + """ + Shape change of input tensor does not affect the shape of its output tensor + Parameters + ---------- + module_masks : ModuleMasks + The ModuleMasks instance of the conv2d + mask : CoarseMask + The mask of its input tensor + Returns + ------- + CoarseMask + The mask of its output tensor + """ + assert isinstance(mask, CoarseMask) + if module_masks.input_mask is None: + module_masks.set_input_mask(mask) + else: + # the same conv layer may be accessed more + # than once, such as a concat operation. + # mask conflict should be solved by fix_mask_conflict before speedup + + assert module_masks.input_mask == mask + + # shape changes pass through depths wise conv layers + m = module_masks.module + if m.in_channels == m.out_channels == m.groups: + module_masks.output_mask = mask + module_masks.input_mask = mask + return mask + return None + + +def conv2d_outshape(module_masks, mask): + """ + Assume only the second dimension is masked + + Parameters + ---------- + module_masks : ModuleMasks + The ModuleMasks instance of the conv2d + mask : CoarseMask + The mask of its output tensor + + Returns + ------- + CoarseMask + The mask of its input tensor + """ + assert isinstance(mask, CoarseMask) + assert mask.mask_index[1] is not None + assert mask.mask_index[0] is None + assert mask.mask_index[2] is None + assert mask.mask_index[3] is None + + if module_masks.output_mask is None: + module_masks.output_mask = mask + else: + # mask conflict should be solved by fix_mask_conflict before speedup + # mask and module_masks.output_mask may have different number of dimensions + # since they could be passed by linear or conv2d + assert all( + module_masks.output_mask.mask_index[1] == mask.mask_index[1]) + + weight_cmask = CoarseMask(num_dim=4) + weight_cmask.add_index_mask(dim=0, index=mask.mask_index[1]) + bias_cmask = CoarseMask(num_dim=1) + bias_cmask.add_index_mask(dim=0, index=mask.mask_index[1]) + module_masks.set_param_masks('weight', weight_cmask) + module_masks.set_param_masks('bias', bias_cmask) + + # shape changes pass through depths wise conv layers + m = module_masks.module + if m.in_channels == m.out_channels == m.groups: + module_masks.output_mask = mask + module_masks.input_mask = mask + return mask + return None + + +def convtranspose2d_mask(module_masks, mask): + # TODO support the Convtranspose2d Pruning for the L1FilterPruner + # raise Exception( + # "Current Filter pruner cannot prune the ConvTranspose2d, will support pruning ConvTranspose2d later") + """ + Infer input and output shape from weight mask + + Parameters + ---------- + module_masks : ModuleMasks + The ModuleMasks instance of the conv2d + mask : dict + The mask of its weights, from the user provided mask file + + Returns + ------- + CoarseMask, CoarseMask + The mask of its input tensor, the mask of its output tensor + """ + def convert_to_coarse_mask(mask, dim=0): + """ + Parameters + ---------- + mask : dict + Weight mask from user provided mask file + dim: int + 0: filter pruning + 1: channel pruning + + Returns + ------- + LongTensor, CoarseMask, CoarseMask + Index of the masked dimension, weight mask, bias mask + """ + assert 'weight' in mask + assert isinstance(mask['weight'], torch.Tensor) + assert dim in [0, 1] + + weight_mask = mask['weight'] + + sum_idx = (1, 2, 3) if dim == 0 else (0, 2, 3) + index = torch.nonzero(weight_mask.abs().sum(sum_idx) != 0, as_tuple=True)[0] + if len(index) == weight_mask.shape[dim]: # full mask + index = None + + if index is None: + return None, None, None + else: + index = index.long().to(weight_mask.device) + weight_cmask = CoarseMask(num_dim=4) + weight_cmask.add_index_mask(dim=dim, index=index) + bias_cmask = None + if dim == 0 and 'bias' in mask and mask['bias'] is not None: + bias_index = torch.nonzero(mask['bias'], as_tuple=True)[0] + assert torch.all(torch.eq(index, bias_index)), \ + "bias mask should be consistent with weight mask" + bias_cmask = CoarseMask(num_dim=1) + bias_cmask.add_index_mask(dim=0, index=bias_index) + return index, weight_cmask, bias_cmask + + convt_prune_dim = 1 + index, weight_cmask, bias_cmask = convert_to_coarse_mask(mask, dim=convt_prune_dim) + + if index is None: + # TODO: fine grained mask speedup + return None, None + # deal with coarse grain mask + # mask conflict should be solved by fix_mask_conflict before speedup + if 'weight' in module_masks.param_masks: + assert module_masks.param_masks['weight'] == weight_cmask + else: + module_masks.set_param_masks('weight', weight_cmask) + if conv_prune_dim == 0: + module_masks.set_param_masks('bias', bias_cmask) + + io_cmask = CoarseMask(num_dim=4) + io_cmask.add_index_mask(dim=1, index=index) + + if conv_prune_dim == 0: + if module_masks.output_mask is None: + module_masks.set_output_mask(io_cmask) + else: + assert module_masks.output_mask == io_cmask + return None, module_masks.output_mask + else: + if module_masks.input_mask is None: + module_masks.set_input_mask(io_cmask) + else: + assert module_masks.input_mask == io_cmask + return module_masks.input_mask, None + + +def convtranspose2d_inshape(module_masks, mask): + """ + Shape change of input tensor does not affect the shape of its output tensor + Parameters + ---------- + module_masks : ModuleMasks + The ModuleMasks instance of the conv2d + mask : CoarseMask + The mask of its input tensor + Returns + ------- + CoarseMask + The mask of its output tensor + """ + assert isinstance(mask, CoarseMask) + if module_masks.input_mask is None: + module_masks.set_input_mask(mask) + else: + # the same conv layer may be accessed more + # than once, such as a concat operation. + # mask conflict should be solved by fix_mask_conflict before speedup + assert module_masks.input_mask == mask + + # shape changes pass through depths wise conv layers + m = module_masks.module + if m.in_channels == m.out_channels == m.groups: + module_masks.output_mask = mask + module_masks.input_mask = mask + return mask + return None + + +def convtranspose2d_outshape(module_masks, mask): + assert isinstance(mask, CoarseMask) + assert mask.mask_index[1] is not None + assert mask.mask_index[0] is None + assert mask.mask_index[2] is None + assert mask.mask_index[3] is None + + if module_masks.output_mask is None: + module_masks.output_mask = mask + else: + # mask conflict should be solved by fix_mask_conflict before speedup + # mask and module_masks.output_mask may have different number of dimensions + # since they could be passed by linear or conv2d + assert all( + module_masks.output_mask.mask_index[1] == mask.mask_index[1]) + + weight_cmask = CoarseMask(num_dim=4) + # Note the memory layout of Convtranspose2d is C_in, C_out, k1, k2 + weight_cmask.add_index_mask(dim=1, index=mask.mask_index[1]) + bias_cmask = CoarseMask(num_dim=1) + bias_cmask.add_index_mask(dim=0, index=mask.mask_index[1]) + module_masks.set_param_masks('weight', weight_cmask) + module_masks.set_param_masks('bias', bias_cmask) + + # shape changes pass through depths wise conv layers + m = module_masks.module + if m.in_channels == m.out_channels == m.groups: + module_masks.output_mask = mask + module_masks.input_mask = mask + return mask + return None diff --git a/utils/third_party/nni_new/compression/pytorch/utils/__init__.py b/utils/third_party/nni_new/compression/pytorch/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/utils/third_party/nni_new/compression/pytorch/utils/__pycache__/__init__.cpython-38.pyc b/utils/third_party/nni_new/compression/pytorch/utils/__pycache__/__init__.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..aa0fd6dc7b840ceddbc36cd25d4aa34ec83109a8 Binary files /dev/null and b/utils/third_party/nni_new/compression/pytorch/utils/__pycache__/__init__.cpython-38.pyc differ diff --git a/utils/third_party/nni_new/compression/pytorch/utils/__pycache__/mask_conflict.cpython-38.pyc b/utils/third_party/nni_new/compression/pytorch/utils/__pycache__/mask_conflict.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..95007c0f034c454e0b050bdcc0ac9bc614767fc6 Binary files /dev/null and b/utils/third_party/nni_new/compression/pytorch/utils/__pycache__/mask_conflict.cpython-38.pyc differ diff --git a/utils/third_party/nni_new/compression/pytorch/utils/__pycache__/shape_dependency.cpython-38.pyc b/utils/third_party/nni_new/compression/pytorch/utils/__pycache__/shape_dependency.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6c8529ab38fe6e75c4b5246a8afd8445f9003919 Binary files /dev/null and b/utils/third_party/nni_new/compression/pytorch/utils/__pycache__/shape_dependency.cpython-38.pyc differ diff --git a/utils/third_party/nni_new/compression/pytorch/utils/__pycache__/utils.cpython-38.pyc b/utils/third_party/nni_new/compression/pytorch/utils/__pycache__/utils.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..cb67bb0e541e76efd348a205165cebabb3b87cfa Binary files /dev/null and b/utils/third_party/nni_new/compression/pytorch/utils/__pycache__/utils.cpython-38.pyc differ diff --git a/utils/third_party/nni_new/compression/pytorch/utils/config_validation.py b/utils/third_party/nni_new/compression/pytorch/utils/config_validation.py new file mode 100644 index 0000000000000000000000000000000000000000..3b9f93f96247eae71c59828d76f878d90fafc00f --- /dev/null +++ b/utils/third_party/nni_new/compression/pytorch/utils/config_validation.py @@ -0,0 +1,53 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from schema import Schema, And, SchemaError + +def validate_op_names(model, op_names, logger): + found_names = set(map(lambda x: x[0], model.named_modules())) + + not_found_op_names = list(set(op_names) - found_names) + if not_found_op_names: + logger.warning('op_names %s not found in model', not_found_op_names) + + return True + +def validate_op_types(model, op_types, logger): + found_types = set(['default']) | set(map(lambda x: type(x[1]).__name__, model.named_modules())) + + not_found_op_types = list(set(op_types) - found_types) + if not_found_op_types: + logger.warning('op_types %s not found in model', not_found_op_types) + + return True + +def validate_op_types_op_names(data): + if not ('op_types' in data or 'op_names' in data): + raise SchemaError('Either op_types or op_names must be specified.') + return True + +class CompressorSchema: + def __init__(self, data_schema, model, logger): + assert isinstance(data_schema, list) and len(data_schema) <= 1 + self.data_schema = data_schema + self.compressor_schema = Schema(self._modify_schema(data_schema, model, logger)) + + def _modify_schema(self, data_schema, model, logger): + if not data_schema: + return data_schema + + for k in data_schema[0]: + old_schema = data_schema[0][k] + if k == 'op_types' or (isinstance(k, Schema) and k._schema == 'op_types'): + new_schema = And(old_schema, lambda n: validate_op_types(model, n, logger)) + data_schema[0][k] = new_schema + if k == 'op_names' or (isinstance(k, Schema) and k._schema == 'op_names'): + new_schema = And(old_schema, lambda n: validate_op_names(model, n, logger)) + data_schema[0][k] = new_schema + + data_schema[0] = And(data_schema[0], lambda d: validate_op_types_op_names(d)) + + return data_schema + + def validate(self, data): + self.compressor_schema.validate(data) diff --git a/utils/third_party/nni_new/compression/pytorch/utils/counter.py b/utils/third_party/nni_new/compression/pytorch/utils/counter.py new file mode 100644 index 0000000000000000000000000000000000000000..0302c980558474ea417b32f56d006b896f5d4cad --- /dev/null +++ b/utils/third_party/nni_new/compression/pytorch/utils/counter.py @@ -0,0 +1,411 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import functools +from collections import Counter +from prettytable import PrettyTable + +import torch +import torch.nn as nn +from torch.nn.utils.rnn import PackedSequence +from nni.compression.pytorch.compressor import PrunerModuleWrapper + + +__all__ = ['count_flops_params'] + + +def _get_params(m): + return sum([p.numel() for p in m.parameters()]) + + +class ModelProfiler: + + def __init__(self, custom_ops=None, mode='default'): + """ + ModelProfiler is used to share state to hooks. + + Parameters + ---------- + custom_ops: dict + a mapping of (module -> torch.nn.Module : custom operation) + the custom operation is a callback funtion to calculate + the module flops, parameters and the weight shape, it will overwrite the default operation. + for reference, please see ``self.ops``. + mode: + the mode of how to collect information. If the mode is set to `default`, + only the information of convolution, linear and rnn modules will be collected. + If the mode is set to `full`, other operations will also be collected. + """ + self.ops = { + nn.Conv1d: self._count_convNd, + nn.Conv2d: self._count_convNd, + nn.Conv3d: self._count_convNd, + nn.ConvTranspose1d: self._count_convNd, + nn.ConvTranspose2d: self._count_convNd, + nn.ConvTranspose3d: self._count_convNd, + nn.Linear: self._count_linear, + nn.RNNCell: self._count_rnn_cell, + nn.GRUCell: self._count_gru_cell, + nn.LSTMCell: self._count_lstm_cell, + nn.RNN: self._count_rnn, + nn.GRU: self._count_gru, + nn.LSTM: self._count_lstm + } + self._count_bias = False + if mode == 'full': + self.ops.update({ + nn.BatchNorm1d: self._count_bn, + nn.BatchNorm2d: self._count_bn, + nn.BatchNorm3d: self._count_bn, + nn.LeakyReLU: self._count_relu, + nn.AvgPool1d: self._count_avgpool, + nn.AvgPool2d: self._count_avgpool, + nn.AvgPool3d: self._count_avgpool, + nn.AdaptiveAvgPool1d: self._count_adap_avgpool, + nn.AdaptiveAvgPool2d: self._count_adap_avgpool, + nn.AdaptiveAvgPool3d: self._count_adap_avgpool, + nn.Upsample: self._count_upsample, + nn.UpsamplingBilinear2d: self._count_upsample, + nn.UpsamplingNearest2d: self._count_upsample + }) + self._count_bias = True + + if custom_ops is not None: + self.ops.update(custom_ops) + + self.mode = mode + self.results = [] + + def _push_result(self, result): + self.results.append(result) + + def _get_result(self, m, flops): + # assume weight is called `weight`, otherwise it's not applicable + # if user customize the operation, the callback function should + # return the dict result, inluding calculated flops, params and weight_shape. + + result = { + 'flops': flops, + 'params': _get_params(m), + 'weight_shape': tuple(m.weight.size()) if hasattr(m, 'weight') else 0, + } + return result + + def _count_convNd(self, m, x, y): + cin = m.in_channels + kernel_ops = torch.zeros(m.weight.size()[2:]).numel() + output_size = torch.zeros(y.size()[2:]).numel() + cout = y.size()[1] + + if hasattr(m, 'weight_mask'): + cout = m.weight_mask.sum() // (cin * kernel_ops) + + total_ops = cout * output_size * kernel_ops * cin // m.groups # cout x oW x oH + + if self._count_bias: + bias_flops = 1 if m.bias is not None else 0 + total_ops += cout * output_size * bias_flops + + return self._get_result(m, total_ops) + + def _count_linear(self, m, x, y): + out_features = m.out_features + if hasattr(m, 'weight_mask'): + out_features = m.weight_mask.sum() // m.in_features + total_ops = out_features * m.in_features + + if self._count_bias: + bias_flops = 1 if m.bias is not None else 0 + total_ops += out_features * bias_flops + + return self._get_result(m, total_ops) + + def _count_bn(self, m, x, y): + total_ops = 2 * x[0].numel() + return self._get_result(m, total_ops) + + def _count_relu(self, m, x, y): + total_ops = x[0].numel() + return self._get_result(m, total_ops) + + def _count_avgpool(self, m, x, y): + total_ops = y.numel() + return self._get_result(m, total_ops) + + def _count_adap_avgpool(self, m, x, y): + kernel = torch.Tensor([*(x[0].shape[2:])]) // torch.Tensor(list((m.output_size,))).squeeze() + total_add = int(torch.prod(kernel)) + total_div = 1 + kernel_ops = total_add + total_div + num_elements = y.numel() + total_ops = kernel_ops * num_elements + + return self._get_result(m, total_ops) + + def _count_upsample(self, m, x, y): + if m.mode == 'linear': + total_ops = y.nelement() * 5 # 2 muls + 3 add + elif m.mode == 'bilinear': + # https://en.wikipedia.org/wiki/Bilinear_interpolation + total_ops = y.nelement() * 11 # 6 muls + 5 adds + elif m.mode == 'bicubic': + # https://en.wikipedia.org/wiki/Bicubic_interpolation + # Product matrix [4x4] x [4x4] x [4x4] + ops_solve_A = 224 # 128 muls + 96 adds + ops_solve_p = 35 # 16 muls + 12 adds + 4 muls + 3 adds + total_ops = y.nelement() * (ops_solve_A + ops_solve_p) + elif m.mode == 'trilinear': + # https://en.wikipedia.org/wiki/Trilinear_interpolation + # can viewed as 2 bilinear + 1 linear + total_ops = y.nelement() * (13 * 2 + 5) + else: + total_ops = 0 + + return self._get_result(m, total_ops) + + def _count_cell_flops(self, input_size, hidden_size, cell_type): + # h' = \tanh(W_{ih} x + b_{ih} + W_{hh} h + b_{hh}) + total_ops = hidden_size * (input_size + hidden_size) + hidden_size + + if self._count_bias: + total_ops += hidden_size * 2 + + if cell_type == 'rnn': + return total_ops + + if cell_type == 'gru': + # r = \sigma(W_{ir} x + b_{ir} + W_{hr} h + b_{hr}) \\ + # z = \sigma(W_{iz} x + b_{iz} + W_{hz} h + b_{hz}) \\ + # n = \tanh(W_{in} x + b_{in} + r * (W_{hn} h + b_{hn})) \\ + total_ops *= 3 + + # r hadamard : r * (~) + total_ops += hidden_size + + # h' = (1 - z) * n + z * h + # hadamard hadamard add + total_ops += hidden_size * 3 + + elif cell_type == 'lstm': + # i = \sigma(W_{ii} x + b_{ii} + W_{hi} h + b_{hi}) \\ + # f = \sigma(W_{if} x + b_{if} + W_{hf} h + b_{hf}) \\ + # o = \sigma(W_{io} x + b_{io} + W_{ho} h + b_{ho}) \\ + # g = \tanh(W_{ig} x + b_{ig} + W_{hg} h + b_{hg}) \\ + total_ops *= 4 + + # c' = f * c + i * g + # hadamard hadamard add + total_ops += hidden_size * 3 + + # h' = o * \tanh(c') + total_ops += hidden_size + + return total_ops + + + def _count_rnn_cell(self, m, x, y): + total_ops = self._count_cell_flops(m.input_size, m.hidden_size, 'rnn') + batch_size = x[0].size(0) + total_ops *= batch_size + + return self._get_result(m, total_ops) + + def _count_gru_cell(self, m, x, y): + total_ops = self._count_cell_flops(m.input_size, m.hidden_size, 'gru') + batch_size = x[0].size(0) + total_ops *= batch_size + + return self._get_result(m, total_ops) + + def _count_lstm_cell(self, m, x, y): + total_ops = self._count_cell_flops(m.input_size, m.hidden_size, 'lstm') + batch_size = x[0].size(0) + total_ops *= batch_size + + return self._get_result(m, total_ops) + + def _get_bsize_nsteps(self, m, x): + if isinstance(x[0], PackedSequence): + batch_size = torch.max(x[0].batch_sizes) + num_steps = x[0].batch_sizes.size(0) + else: + if m.batch_first: + batch_size = x[0].size(0) + num_steps = x[0].size(1) + else: + batch_size = x[0].size(1) + num_steps = x[0].size(0) + + return batch_size, num_steps + + def _count_rnn_module(self, m, x, y, module_name): + input_size = m.input_size + hidden_size = m.hidden_size + num_layers = m.num_layers + + batch_size, num_steps = self._get_bsize_nsteps(m, x) + total_ops = self._count_cell_flops(input_size, hidden_size, module_name) + + for _ in range(num_layers - 1): + if m.bidirectional: + cell_flops = self._count_cell_flops(hidden_size * 2, hidden_size, module_name) * 2 + else: + cell_flops = self._count_cell_flops(hidden_size, hidden_size,module_name) + total_ops += cell_flops + + total_ops *= num_steps + total_ops *= batch_size + return total_ops + + def _count_rnn(self, m, x, y): + total_ops = self._count_rnn_module(m, x, y, 'rnn') + + return self._get_result(m, total_ops) + + def _count_gru(self, m, x, y): + total_ops = self._count_rnn_module(m, x, y, 'gru') + + return self._get_result(m, total_ops) + + def _count_lstm(self, m, x, y): + total_ops = self._count_rnn_module(m, x, y, 'lstm') + + return self._get_result(m, total_ops) + + + def count_module(self, m, x, y, name): + # assume x is tuple of single tensor + result = self.ops[type(m)](m, x, y) + output_size = y[0].size() if isinstance(y, tuple) else y.size() + + total_result = { + 'name': name, + 'input_size': tuple(x[0].size()), + 'output_size': tuple(output_size), + 'module_type': type(m).__name__, + **result + } + + self._push_result(total_result) + + def sum_flops(self): + return sum([s['flops'] for s in self.results]) + + def sum_params(self): + return sum({s['name']: s['params'] for s in self.results}.values()) + + def format_results(self): + table = PrettyTable() + name_counter = Counter([s['name'] for s in self.results]) + has_multi_use = any(map(lambda v: v > 1, name_counter.values())) + name_counter = Counter() # clear the counter to count from 0 + + headers = [ + 'Index', + 'Name', + 'Type', + 'Weight Shape', + 'FLOPs', + '#Params', + ] + if has_multi_use: + headers.append('#Call') + + table.field_names = headers + for i, result in enumerate(self.results): + row_values = [ + i, + result['name'], + result['module_type'], + str(result['weight_shape']), + result['flops'], + result['params'], + ] + name_counter[result['name']] += 1 + if has_multi_use: + row_values.append(name_counter[result['name']]) + table.add_row(row_values) + return table + + +def count_flops_params(model, x, custom_ops=None, verbose=True, mode='default'): + """ + Count FLOPs and Params of the given model. This function would + identify the mask on the module and take the pruned shape into consideration. + Note that, for sturctured pruning, we only identify the remained filters + according to its mask, and do not take the pruned input channels into consideration, + so the calculated FLOPs will be larger than real number. + + Parameters + --------- + model : nn.Module + Target model. + x : tuple or tensor + The input shape of data (a tuple), a tensor or a tuple of tensor as input data. + custom_ops : dict + A mapping of (module -> torch.nn.Module : custom operation) + the custom operation is a callback funtion to calculate + the module flops and parameters, it will overwrite the default operation. + for reference, please see ``ops`` in ``ModelProfiler``. + verbose : bool + If False, mute detail information about modules. Default is True. + mode : str + the mode of how to collect information. If the mode is set to ``default``, + only the information of convolution and linear will be collected. + If the mode is set to ``full``, other operations will also be collected. + + Returns + ------- + tuple of int, int and dict + Representing total FLOPs, total parameters, and a detailed list of results respectively. + The list of results are a list of dict, each of which contains (name, module_type, weight_shape, + flops, params, input_size, output_size) as its keys. + """ + + assert isinstance(x, tuple) or isinstance(x, torch.Tensor) + assert mode in ['default', 'full'] + + original_device = next(model.parameters()).device + training = model.training + + if isinstance(x, tuple) and all(isinstance(t, int) for t in x): + x = (torch.zeros(x).to(original_device), ) + elif torch.is_tensor(x): + x = (x.to(original_device), ) + else: + x = (t.to(original_device) for t in x) + + handler_collection = [] + profiler = ModelProfiler(custom_ops, mode) + + prev_m = None + for name, m in model.named_modules(): + # dealing with weight mask here + if isinstance(prev_m, PrunerModuleWrapper): + # weight mask is set to weight mask of its parent (wrapper) + weight_mask = prev_m.weight_mask + m.weight_mask = weight_mask + prev_m = m + + if type(m) in profiler.ops: + # if a leaf node + _handler = m.register_forward_hook(functools.partial(profiler.count_module, name=name)) + handler_collection.append(_handler) + + model.eval() + + with torch.no_grad(): + model(*x) + + # restore origin status + model.train(training).to(original_device) + for handler in handler_collection: + handler.remove() + + if verbose: + # get detail information + print(profiler.format_results()) + print(f'FLOPs total: {profiler.sum_flops()}') + print(f'#Params total: {profiler.sum_params()}') + + return profiler.sum_flops(), profiler.sum_params(), profiler.results \ No newline at end of file diff --git a/utils/third_party/nni_new/compression/pytorch/utils/mask_conflict.py b/utils/third_party/nni_new/compression/pytorch/utils/mask_conflict.py new file mode 100644 index 0000000000000000000000000000000000000000..83ab9e4fe2cc2b77647f5abe043eb1e2a06207cb --- /dev/null +++ b/utils/third_party/nni_new/compression/pytorch/utils/mask_conflict.py @@ -0,0 +1,403 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. +import os +import logging +import torch +import numpy as np +from .shape_dependency import ChannelDependency, GroupDependency, CatPaddingDependency, InputChannelDependency +from .utils import get_module_by_name +# logging.basicConfig(level = logging.DEBUG) +_logger = logging.getLogger(__name__) + + +def fix_mask_conflict(masks, model=None, dummy_input=None, traced=None): + """ + MaskConflict fix the mask conflict for the channel dependencies + and group dependency. + + Parameters + ---------- + masks : dict/str + A dict object that stores the masks or the path of the mask file + model : torch.nn.Module + model to fix the mask conflict + dummy_input : torch.Tensor + input example to trace the model + traced : torch._C.torch.jit.TopLevelTracedModule + the traced model of the target model, is this parameter is not None, + we donnot use the model and dummpy_input to get the trace graph. + """ + if isinstance(masks, str): + # if the input is the path of the mask_file + assert os.path.exists(masks) + masks = torch.load(masks) + assert len(masks) > 0, 'Mask tensor cannot be empty' + # if the user uses the model and dummy_input to trace the model, we + # should get the traced model handly, so that, we only trace the + # model once, GroupMaskConflict and ChannelMaskConflict will reuse + # this traced model. + if traced is None: + assert model is not None and dummy_input is not None + training = model.training + model.eval() + # We need to trace the model in eval mode + traced = torch.jit.trace(model, dummy_input,strict = False) + model.train(training) + + fix_group_mask = GroupMaskConflict(masks, model, dummy_input, traced) + masks = fix_group_mask.fix_mask() + fix_channel_mask = ChannelMaskConflict(masks, model, dummy_input, traced) + masks = fix_channel_mask.fix_mask() + padding_cat_mask = CatMaskPadding(masks, model, dummy_input, traced) + masks = padding_cat_mask.fix_mask() + return masks, fix_channel_mask.conv_prune_dim + + +class MaskFix: + def __init__(self, masks, model=None, dummy_input=None, traced=None): + # check if the parameters are valid + parameter_valid = False + if traced is not None: + parameter_valid = True + elif (model is not None) and (dummy_input is not None): + parameter_valid = True + if not parameter_valid: + raise Exception('The input parameters is invalid!') + self.model = model + self.dummy_input = dummy_input + self.traced = traced + self.masks = masks + + def fix_mask(self): + raise NotImplementedError + + def export(self, path): + """ + Export the masks after fixing the conflict to file. + """ + torch.save(self.masks, path) + + +class CatMaskPadding(MaskFix): + def __init__(self, masks, model, dummy_input=None, traced=None): + """ + CatMaskPadding find the layers whose output tensor is passed + to the same cat operation. The cat operation concatnates the + masks of the input tensors as the output mask, so when some + of the input layers of the cat operation are not pruned, we still + need to pass the masks of these non-pruned layers(the mask are + all ones) to the cat operation to ensure the shape of the output + mask is right. + + Parameters + ---------- + masks : dict + a dict object that stores the masks + model : torch.nn.Module + model to fix the mask conflict + dummy_input : torch.Tensor + input example to trace the model + traced : torch._C.torch.jit.TopLevelTracedModule + the traced model of the target model, is this parameter is not None, + we donnot use the model and dummpy_input to get the trace graph. + """ + super(CatMaskPadding, self).__init__(masks, model, dummy_input, traced) + + def fix_mask(self): + cat_padding_depen = CatPaddingDependency( + self.model, self.dummy_input, self.traced) + name_to_module = {} + for name, module in self.model.named_modules(): + name_to_module[name] = module + depen = cat_padding_depen.dependency_sets + for layers in depen: + device = None + count = 0 + for layer in layers: + if layer in self.masks: + count += 1 + if device is None: + device = self.masks[layer]['weight'].device + if count == 0: + # no layer is pruned + continue + elif count == len(layers): + # all the layers have been pruned + continue + # pad the mask for the non-pruned layers + for layer in layers: + if layer in self.masks: + continue + + module = name_to_module[layer] + w_shape = module.weight.data.size() + w_mask = torch.ones(w_shape).to(device) + b_mask = None + if hasattr(module, 'bias') and module.bias is not None: + # module.bias may be None + b_shape = module.bias.data.size() + b_mask = torch.ones(b_shape).to(device) + self.masks[layer] = {'weight': w_mask, 'bias': b_mask} + + return self.masks + + +class GroupMaskConflict(MaskFix): + def __init__(self, masks, model=None, dummy_input=None, traced=None): + """ + GroupMaskConflict fix the mask conflict between the layers that + has group dependecy with each other. + + Parameters + ---------- + masks : dict + a dict object that stores the masks + model : torch.nn.Module + model to fix the mask conflict + dummy_input : torch.Tensor + input example to trace the model + traced : torch._C.torch.jit.TopLevelTracedModule + the traced model of the target model, is this parameter is not None, + we donnot use the model and dummpy_input to get the trace graph. + """ + super(GroupMaskConflict, self).__init__( + masks, model, dummy_input, traced) + + def fix_mask(self): + """ + Fix the mask conflict before the mask inference for the layers that + has group dependencies. This function should be called before the + mask inference of the 'speedup' module. + """ + group_depen = GroupDependency( + self.model, self.dummy_input, self.traced) + depens = group_depen.dependency + _logger.info(depens) + for layername in depens: + group = depens[layername] + if layername not in self.masks: + # this layer not pruned + continue + w_mask = self.masks[layername]['weight'] + shape = w_mask.size() + count = np.prod(shape[1:]) + all_ones = (w_mask.flatten(1).sum(-1) == count).nonzero(as_tuple=False).squeeze(1).tolist() + all_zeros = (w_mask.flatten(1).sum(-1) == 0).nonzero(as_tuple=False).squeeze(1).tolist() + if len(all_ones) + len(all_zeros) < w_mask.size(0): + # In fine-grained pruning, skip this layer + _logger.info('Layers %s using fine-grained pruning', layername) + continue + assert shape[0] % group == 0 + # Find the number of masked filter for each group (mini_masked). + # Because we have to keep the pruned filter can still + # be divided into the same number of groups, so we only can + # prune mini_masked filters for each group. + step = shape[0] / group + group_masked = [] + for i in range(group): + _start = step * i + _end = step * (i + 1) + _tmp_list = list( + filter(lambda x: _start <= x and x < _end, all_zeros)) + group_masked.append(_tmp_list) + mini_masked = min([len(x) for x in group_masked]) + for gm in group_masked: + for i in range(mini_masked, len(gm)): + # To keep the output channel number still being divisible to + # groups, we set the masks of following filters to be zero. + pos = gm[i] + self.masks[layername]['weight'][pos] = torch.ones( + shape[1:]) + if 'bias' in self.masks[layername] and self.masks[layername]['bias'] is not None: + self.masks[layername]['bias'][pos] = 1 + return self.masks + + +class ChannelMaskConflict(MaskFix): + def __init__(self, masks, model=None, dummy_input=None, traced=None): + """ + ChannelMaskConflict fix the mask conflict between the layers that + has channel dependecy with each other. + + Parameters + ---------- + masks : dict + a dict object that stores the masks + model : torch.nn.Module + model to fix the mask conflict + dummy_input : torch.Tensor + input example to trace the model + graph : torch._C.torch.jit.TopLevelTracedModule + the traced graph of the target model, is this parameter is not None, + we donnot use the model and dummpy_input to get the trace graph. + """ + super(ChannelMaskConflict, self).__init__( + masks, model, dummy_input, traced) + self.conv_prune_dim = detect_mask_prune_dim(masks, model) + _logger.info('detected conv prune dim: %s', self.conv_prune_dim) + + def fix_mask(self): + """ + Fix the mask conflict before the mask inference for the layers that + has shape dependencies. This function should be called before the + mask inference of the 'speedup' module. Only structured pruning masks + are supported. + """ + if self.conv_prune_dim == 0: + channel_depen = ChannelDependency( + self.model, self.dummy_input, self.traced) + else: + channel_depen = InputChannelDependency( + self.model, self.dummy_input, self.traced) + depen_sets = channel_depen.dependency_sets + sum_idx = (1, 2, 3) if self.conv_prune_dim == 0 else (0, 2, 3) + + (_tmp_name, _tmp_tensor) = list(self.masks.items())[0] + device = _tmp_tensor['weight'].device + + for dset in depen_sets: + if len(dset) <= 1: + continue + # channel_masks is a list, each element is None or a vector, for example: + # [[0, 1, 1, 0, 0], [0, 0, 1, 1, 0], None], None means no channel + # is pruned. + channel_masks = [] + fine_grained = False + for name in dset: + if name in self.masks: + _, m = get_module_by_name(self.model, name) + assert m is not None + mask = self.masks[name]['weight'] + if type(m).__name__ == 'Conv2d': + channel_mask = (mask.abs().sum(sum_idx) != 0).int() + channel_masks.append(channel_mask) + if (channel_mask.sum() * (mask.numel() / mask.shape[self.conv_prune_dim])).item() != (mask > 0).sum().item(): + fine_grained = True + elif type(m).__name__ == 'Linear': + channel_masks.append((mask.abs().sum(0) != 0).int()) + elif type(m).__name__ == 'BatchNorm2d': + channel_masks.append(mask.int()) + elif type(m).__name__ == 'ConvTranspose2d': + # convtranspose have difference memory layout, so that we need create + # a tmp_sum_idx for conv_transpose + tmp_sum_idx = ( + 0, 2, 3) if self.conv_prune_dim == 0 else (1, 2, 3) + channel_mask = (mask.abs().sum(tmp_sum_idx) != 0).int() + channel_masks.append(channel_mask) + if (channel_mask.sum() * (mask.numel() / mask.shape[1 - self.conv_prune_dim])).item() != (mask > 0).sum().item(): + fine_grained = True + else: + raise RuntimeError( + f'unsupported module type: {type(m).__name__}') + else: + # no mask means not pruned, equivlent to full masks + channel_masks.append(None) + if fine_grained: + _logger.info( + 'fine-grained mask detected, skip solving conflict for this set: %s', dset) + continue + if all(x is None for x in channel_masks): + continue + num_channels_list = [len(x) + for x in channel_masks if x is not None] + # number of channels in same set should be identical + assert len(set(num_channels_list)) == 1 + num_channels = num_channels_list[0] + + for i, dim_mask in enumerate(channel_masks): + if dim_mask is None: + channel_masks[i] = torch.ones(num_channels).int().to(device) + + # merge masks with 'or' + merged_channel_mask = channel_masks[0].clone() + for i in range(1, len(channel_masks)): + merged_channel_mask = ( + (merged_channel_mask + channel_masks[i]) != 0).int() + + merged_index = torch.nonzero(merged_channel_mask, as_tuple=True)[0] + + for name in dset: + if name not in self.masks: + assert all(merged_channel_mask) + continue + orig_mask = self.masks[name]['weight'] + _, m = get_module_by_name(self.model, name) + new_mask = torch.zeros_like(orig_mask) + if type(m).__name__ == 'Conv2d': + if self.conv_prune_dim == 0: + new_mask[merged_index, :, :, :] = 1. + else: + new_mask[:, merged_index, :, :] = 1. + elif type(m).__name__ == 'Linear': + new_mask[:, merged_index] = 1. + elif type(m).__name__ == 'BatchNorm2d': + new_mask = merged_channel_mask.type_as(orig_mask) + else: + raise RuntimeError( + f'unsupported module type: {type(m).__name__}') + + self.masks[name]['weight'] = new_mask + if 'bias' in self.masks[name] and self.masks[name]['bias'] is not None: + if type(m).__name__ == 'Conv2d': + assert self.conv_prune_dim == 0 + self.masks[name]['bias'] = merged_channel_mask.type_as( + self.masks[name]['bias']) + + return self.masks + + +def detect_mask_prune_dim(masks, model): + """ + Detect how the masks of convolutional layers are pruned. + + Parameters + ---------- + masks: dict + A dict object that stores the masks. + model: nn.Module + Model object which the mask can be applied on. + + Returns: + ------- + How the masks of convolutional layers are pruned, this depends on pruning algorithms, it should + return 1 for masks generated by AMCPruner, and returns 0 for masks generated by the rest + NNI builtin pruners. + 0: filter pruning, prune filters of weights which causes channels of output feature maps are pruned. + 1: channel pruning, prune kernels corresponding to each input channels which causes channels of + input feature maps are pruned. + """ + dim0_preserved, dim1_preserved = 0., 0. + dim0_num, dim1_num = 0., 0. + for module_name in masks: + _, m = get_module_by_name(model, module_name) + if m is None or type(m).__name__ != 'Conv2d': + continue + + mask = masks[module_name]['weight'].clone() + assert (mask >= 0).sum() == mask.numel(), \ + "mask values should be greater than or equal to 0." + mask = (mask > 0).int() + mask = mask.view(mask.shape[0], mask.shape[1], -1) + dim0_mask = (mask.sum((1, 2)) > 0).int() + dim1_mask = (mask.sum((0, 2)) > 0).int() + dim0_preserved += dim0_mask.sum().item() + dim1_preserved += dim1_mask.sum().item() + dim0_num += len(dim0_mask) + dim1_num += len(dim1_mask) + + if dim0_num == 0 or dim1_num == 0: + _logger.warning('no multi-dimension masks found.') + return 0 + + dim0_sparsity, dim1_sparsity = 1. - dim0_preserved / \ + dim0_num, 1. - dim1_preserved / dim1_num + _logger.info('dim0 sparsity: %f', dim0_sparsity) + _logger.info('dim1 sparsity: %f', dim1_sparsity) + + if dim0_sparsity == dim1_sparsity == 0.: + _logger.warning('nothing masked.') + + if dim0_sparsity > 0 and dim1_sparsity > 0: + _logger.warning('both dim0 and dim1 masks found.') + + return 0 if dim0_sparsity >= dim1_sparsity else 1 diff --git a/utils/third_party/nni_new/compression/pytorch/utils/num_param_counter.py b/utils/third_party/nni_new/compression/pytorch/utils/num_param_counter.py new file mode 100644 index 0000000000000000000000000000000000000000..89ad0979943126c45112d8b865d87ec12cdb1961 --- /dev/null +++ b/utils/third_party/nni_new/compression/pytorch/utils/num_param_counter.py @@ -0,0 +1,16 @@ +def get_total_num_weights(model, op_types=['default']): + ''' + calculate the total number of weights + + Returns + ------- + int + total weights of all the op considered + ''' + num_weights = 0 + for _, module in model.named_modules(): + if module == model: + continue + if 'default' in op_types or type(module).__name__ in op_types: + num_weights += module.weight.data.numel() + return num_weights \ No newline at end of file diff --git a/utils/third_party/nni_new/compression/pytorch/utils/sensitivity_analysis.py b/utils/third_party/nni_new/compression/pytorch/utils/sensitivity_analysis.py new file mode 100644 index 0000000000000000000000000000000000000000..a36a523feb1e3d7e7959d00ad01c8705af4f9373 --- /dev/null +++ b/utils/third_party/nni_new/compression/pytorch/utils/sensitivity_analysis.py @@ -0,0 +1,249 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import copy +import csv +import logging +from collections import OrderedDict + +import numpy as np +import torch.nn as nn + +# FIXME: I don't know where "utils" should be +SUPPORTED_OP_NAME = ['Conv2d', 'Conv1d'] +SUPPORTED_OP_TYPE = [getattr(nn, name) for name in SUPPORTED_OP_NAME] + +logger = logging.getLogger('Sensitivity_Analysis') +logger.setLevel(logging.INFO) + + +class SensitivityAnalysis: + def __init__(self, model, val_func, sparsities=None, prune_type='l1', early_stop_mode=None, early_stop_value=None): + """ + Perform sensitivity analysis for this model. + Parameters + ---------- + model : torch.nn.Module + the model to perform sensitivity analysis + val_func : function + validation function for the model. Due to + different models may need different dataset/criterion + , therefore the user need to cover this part by themselves. + In the val_func, the model should be tested on the validation dateset, + and the validation accuracy/loss should be returned as the output of val_func. + There are no restrictions on the input parameters of the val_function. + User can use the val_args, val_kwargs parameters in analysis + to pass all the parameters that val_func needed. + sparsities : list + The sparsity list provided by users. This parameter is set when the user + only wants to test some specific sparsities. In the sparsity list, each element + is a sparsity value which means how much weight the pruner should prune. Take + [0.25, 0.5, 0.75] for an example, the SensitivityAnalysis will prune 25% 50% 75% + weights gradually for each layer. + prune_type : str + The pruner type used to prune the conv layers, default is 'l1', + and 'l2', 'fine-grained' is also supported. + early_stop_mode : str + If this flag is set, the sensitivity analysis + for a conv layer will early stop when the validation metric( + for example, accurracy/loss) has alreay meet the threshold. We + support four different early stop modes: minimize, maximize, dropped, + raised. The default value is None, which means the analysis won't stop + until all given sparsities are tested. This option should be used with + early_stop_value together. + + minimize: The analysis stops when the validation metric return by the val_func + lower than early_stop_value. + maximize: The analysis stops when the validation metric return by the val_func + larger than early_stop_value. + dropped: The analysis stops when the validation metric has dropped by early_stop_value. + raised: The analysis stops when the validation metric has raised by early_stop_value. + early_stop_value : float + This value is used as the threshold for different earlystop modes. + This value is effective only when the early_stop_mode is set. + + """ + from nni.algorithms.compression.pytorch.pruning.constants_pruner import PRUNER_DICT + + self.model = model + self.val_func = val_func + self.target_layer = OrderedDict() + self.ori_state_dict = copy.deepcopy(self.model.state_dict()) + self.target_layer = {} + self.sensitivities = {} + if sparsities is not None: + self.sparsities = sorted(sparsities) + else: + self.sparsities = np.arange(0.1, 1.0, 0.1) + self.sparsities = [np.round(x, 2) for x in self.sparsities] + self.Pruner = PRUNER_DICT[prune_type] + self.early_stop_mode = early_stop_mode + self.early_stop_value = early_stop_value + self.ori_metric = None # original validation metric for the model + # already_pruned is for the iterative sensitivity analysis + # For example, sensitivity_pruner iteratively prune the target + # model according to the sensitivity. After each round of + # pruning, the sensitivity_pruner will test the new sensitivity + # for each layer + self.already_pruned = {} + self.model_parse() + + @property + def layers_count(self): + return len(self.target_layer) + + def model_parse(self): + for name, submodel in self.model.named_modules(): + for op_type in SUPPORTED_OP_TYPE: + if isinstance(submodel, op_type): + self.target_layer[name] = submodel + self.already_pruned[name] = 0 + + def _need_to_stop(self, ori_metric, cur_metric): + """ + Judge if meet the stop conditon(early_stop, min_threshold, + max_threshold). + Parameters + ---------- + ori_metric : float + original validation metric + cur_metric : float + current validation metric + + Returns + ------- + stop : bool + if stop the sensitivity analysis + """ + if self.early_stop_mode is None: + # early stop mode is not enable + return False + assert self.early_stop_value is not None + if self.early_stop_mode == 'minimize': + if cur_metric < self.early_stop_value: + return True + elif self.early_stop_mode == 'maximize': + if cur_metric > self.early_stop_value: + return True + elif self.early_stop_mode == 'dropped': + if cur_metric < ori_metric - self.early_stop_value: + return True + elif self.early_stop_mode == 'raised': + if cur_metric > ori_metric + self.early_stop_value: + return True + return False + + def analysis(self, val_args=None, val_kwargs=None, specified_layers=None): + """ + This function analyze the sensitivity to pruning for + each conv layer in the target model. + If start and end are not set, we analyze all the conv + layers by default. Users can specify several layers to + analyze or parallelize the analysis process easily through + the start and end parameter. + + Parameters + ---------- + val_args : list + args for the val_function + val_kwargs : dict + kwargs for the val_funtion + specified_layers : list + list of layer names to analyze sensitivity. + If this variable is set, then only analyze + the conv layers that specified in the list. + User can also use this option to parallelize + the sensitivity analysis easily. + Returns + ------- + sensitivities : dict + dict object that stores the trajectory of the + accuracy/loss when the prune ratio changes + """ + if val_args is None: + val_args = [] + if val_kwargs is None: + val_kwargs = {} + # Get the original validation metric(accuracy/loss) before pruning + # Get the accuracy baseline before starting the analysis. + self.ori_metric = self.val_func(*val_args, **val_kwargs) + namelist = list(self.target_layer.keys()) + if specified_layers is not None: + # only analyze several specified conv layers + namelist = list(filter(lambda x: x in specified_layers, namelist)) + for name in namelist: + self.sensitivities[name] = {} + for sparsity in self.sparsities: + # here the sparsity is the relative sparsity of the + # the remained weights + # Calculate the actual prune ratio based on the already pruned ratio + real_sparsity = ( + 1.0 - self.already_pruned[name]) * sparsity + self.already_pruned[name] + # TODO In current L1/L2 Filter Pruner, the 'op_types' is still necessary + # I think the L1/L2 Pruner should specify the op_types automaticlly + # according to the op_names + cfg = [{'sparsity': real_sparsity, 'op_names': [ + name], 'op_types': ['Conv2d']}] + pruner = self.Pruner(self.model, cfg) + pruner.compress() + val_metric = self.val_func(*val_args, **val_kwargs) + logger.info('Layer: %s Sparsity: %.2f Validation Metric: %.4f', + name, real_sparsity, val_metric) + + self.sensitivities[name][sparsity] = val_metric + pruner._unwrap_model() + del pruner + # check if the current metric meet the stop condition + if self._need_to_stop(self.ori_metric, val_metric): + break + + # reset the weights pruned by the pruner, because the + # input sparsities is sorted, so we donnot need to reset + # weight of the layer when the sparsity changes, instead, + # we only need reset the weight when the pruning layer changes. + self.model.load_state_dict(self.ori_state_dict) + + return self.sensitivities + + def export(self, filepath): + """ + Export the results of the sensitivity analysis + to a csv file. The firstline of the csv file describe the content + structure. The first line is constructed by 'layername' and sparsity + list. Each line below records the validation metric returned by val_func + when this layer is under different sparsities. Note that, due to the early_stop + option, some layers may not have the metrics under all sparsities. + + layername, 0.25, 0.5, 0.75 + conv1, 0.6, 0.55 + conv2, 0.61, 0.57, 0.56 + + Parameters + ---------- + filepath : str + Path of the output file + """ + str_sparsities = [str(x) for x in self.sparsities] + header = ['layername'] + str_sparsities + with open(filepath, 'w') as csvf: + csv_w = csv.writer(csvf) + csv_w.writerow(header) + for layername in self.sensitivities: + row = [] + row.append(layername) + for sparsity in sorted(self.sensitivities[layername].keys()): + row.append(self.sensitivities[layername][sparsity]) + csv_w.writerow(row) + + def update_already_pruned(self, layername, ratio): + """ + Set the already pruned ratio for the target layer. + """ + self.already_pruned[layername] = ratio + + def load_state_dict(self, state_dict): + """ + Update the weight of the model + """ + self.ori_state_dict = copy.deepcopy(state_dict) + self.model.load_state_dict(self.ori_state_dict) diff --git a/utils/third_party/nni_new/compression/pytorch/utils/shape_dependency.py b/utils/third_party/nni_new/compression/pytorch/utils/shape_dependency.py new file mode 100644 index 0000000000000000000000000000000000000000..1d3d087def1c0c69230ecc3f2e82db53a28bb557 --- /dev/null +++ b/utils/third_party/nni_new/compression/pytorch/utils/shape_dependency.py @@ -0,0 +1,503 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import csv +import logging + +__all__ = ['ChannelDependency', 'GroupDependency', + 'CatPaddingDependency', 'InputChannelDependency'] + +CONV_TYPE = 'aten::_convolution' +ADD_TYPES = ['aten::add', 'aten::add_'] +CAT_TYPE = 'aten::cat' +logger = logging.getLogger('Shape_Dependency') +RESHAPE_OPS = [CAT_TYPE, 'aten::view', + 'aten::reshape', 'aten::flatten', 'aten::mean'] + + +class Dependency: + def __init__(self, model=None, dummy_input=None, traced_model=None): + """ + Build the graph for the model. + """ + from ....common.graph_utils import TorchModuleGraph + + # check if the input is legal + if traced_model is None: + # user should provide model & dummy_input to trace + # the model or a already traced model + assert model is not None and dummy_input is not None + self.graph = TorchModuleGraph(model, dummy_input, traced_model) + self.dependency = dict() + self.build_dependency() + + def build_dependency(self): + raise NotImplementedError + + def export(self, filepath): + raise NotImplementedError + + +class ChannelDependency(Dependency): + def __init__(self, model=None, dummy_input=None, traced_model=None): + """ + This model analyze the channel dependencies between the conv + layers in a model. + + Parameters + ---------- + model : torch.nn.Module + The model to be analyzed. + data : torch.Tensor + The example input data to trace the network architecture. + traced_model : torch._C.Graph + if we alreay has the traced graph of the target model, we donnot + need to trace the model again. + """ + super(ChannelDependency, self).__init__( + model, dummy_input, traced_model) + + def _get_parent_layers(self, node): + """ + Find the nearest father conv layers for the target node. + + Parameters + --------- + node : torch._C.Node + target node. + + Returns + ------- + parent_layers: list + nearest father conv/linear layers for the target worknode. + """ + parent_layers = [] + queue = [] + queue.append(node) + while queue: + curnode = queue.pop(0) + if curnode.op_type == 'Conv2d' or curnode.op_type == 'Linear' or curnode.op_type == 'ConvTranspose2d': + # find the first met conv + parent_layers.append(curnode.name) + continue + parents = self.graph.find_predecessors(curnode.unique_name) + parents = [self.graph.name_to_node[name] for name in parents] + for parent in parents: + queue.append(parent) + return parent_layers + + def build_dependency(self): + """ + Build the channel dependency for the conv layers + in the model. + """ + # unpack the tuple/list manually before analyze the + # channel dependency + self.graph.unpack_manually() + for node in self.graph.nodes_py.nodes_op: + parent_layers = [] + # find the node that contains aten::add + # or aten::cat operations + if node.op_type in ADD_TYPES: + parent_layers = self._get_parent_layers(node) + elif node.op_type == CAT_TYPE: + # To determine if this cat operation will introduce channel + # dependency, we need the specific input parameters of the cat + # opertion. To get the input parameters of the cat opertion, we + # need to traverse all the cpp_nodes included by this NodePyGroup, + # because, TorchModuleGraph merges the important nodes and the adjacent + # unimportant nodes (nodes started with prim::attr, for example) into a + # NodepyGroup. + cat_dim = None + for cnode in node.node_cpps: + if cnode.kind() == CAT_TYPE: + cat_dim = list(cnode.inputs())[1].toIValue() + break + if cat_dim != 1: + parent_layers = self._get_parent_layers(node) + dependency_set = set(parent_layers) + # merge the dependencies + for parent in parent_layers: + if parent in self.dependency: + dependency_set.update(self.dependency[parent]) + # save the dependencies + for _node in dependency_set: + self.dependency[_node] = dependency_set + + def export(self, filepath): + """ + export the channel dependencies as a csv file. + The layers at the same line have output channel + dependencies with each other. For example, + layer1.1.conv2, conv1, and layer1.0.conv2 have + output channel dependencies with each other, which + means the output channel(filters) numbers of these + three layers should be same with each other, otherwise + the model may has shape conflict. + + Output example: + Dependency Set,Convolutional Layers + Set 1,layer1.1.conv2,layer1.0.conv2,conv1 + Set 2,layer1.0.conv1 + Set 3,layer1.1.conv1 + """ + header = ['Dependency Set', 'Layers'] + setid = 0 + visited = set() + with open(filepath, 'w') as csvf: + csv_w = csv.writer(csvf, delimiter=',') + csv_w.writerow(header) + for node in self.graph.nodes_py.nodes_op: + if node.op_type != 'Conv2d' or node in visited: + continue + setid += 1 + row = ['Set %d' % setid] + if node.name not in self.dependency: + visited.add(node) + row.append(node.name) + else: + for other in self.dependency[node.name]: + visited.add(self.graph.name_to_node[other]) + row.append(other) + csv_w.writerow(row) + + @property + def dependency_sets(self): + """ + Get the list of the dependency set. + + Returns + ------- + dependency_sets : list + list of the dependency sets. For example, + [set(['conv1', 'conv2']), set(['conv3', 'conv4'])] + + """ + d_sets = [] + visited = set() + for node in self.graph.nodes_py.nodes_op: + if node.op_type != 'Conv2d' or node in visited: + continue + tmp_set = set() + if node.name not in self.dependency: + visited.add(node) + tmp_set.add(node.name) + else: + for other in self.dependency[node.name]: + visited.add(self.graph.name_to_node[other]) + tmp_set.add(other) + d_sets.append(tmp_set) + return d_sets + + +def reshape_break_channel_dependency(op_node): + """ + The reshape operations such as (reshape, view, flatten) may break + the channel dependency. We need to check the input parameters of + these reshape operations to check if this reshape node will break + the channel dependency. However, it's complicated to analyze the the input + parameters for each reshape function and infer if it will break the channel + dependency. So currently, we just check if the input channel and the output + channel is the same, if so, then we can say the original reshape function + doesn't want to change the number of the channels, which means the channel + dependency is not broken. In contrast, the original reshap operation wants + to change the number of channels, so it breaks the channel dependency. + + Parameters + ---------- + opnode: NodePyOP + A Op node of the graph. + Returns + ------- + bool + If this operation will break the channel dependency. + """ + in_shape = op_node.auxiliary['in_shape'] + out_shape = op_node.auxiliary['out_shape'] + in_channel = in_shape[1] + out_channel = out_shape[1] + return in_channel != out_channel + + +class InputChannelDependency(ChannelDependency): + """ + Some pruners may prune the input channel of the convolutional + layers. While pruning the input channel of the convolutional layers, + the layers that share the same input tensor should prune the same + channels, and we say these layers that share the same input tensor/channel + has the input channel dependency. If we only prune the input channel of one + layer in the dependency set, there will be a shape conflict for the other + layers in the same dependency set, which may trigger a runtime error. + Here we judge whether the application will truncate the dependency by analyzing + whether the number of channels before and after the operation has changed. + If not, the input channel dependency will be passed to the following nodes. + """ + + def __init__(self, model, dummy_input=None, traced_model=None): + """ + This model analyze the input channel dependencies between the conv + layers in a model. + + Parameters + ---------- + model : torch.nn.Module + The model to be analyzed. + data : torch.Tensor + The example input data to trace the network architecture. + traced_model : torch._C.Graph + if we alreay has the traced graph of the target model, we donnot + need to trace the model again. + """ + super(InputChannelDependency, self).__init__( + model, dummy_input, traced_model) + + def _get_following_convs(self, tensor): + queue = [] + key_layers = [] + queue.extend(self.graph.input_to_node[tensor]) + while queue: + curnode = queue.pop(0) + if curnode.op_type == 'Conv2d' or curnode.op_type == 'Linear' or curnode.op_type == 'ConvTranspose2d': + # find the first met conv + key_layers.append(curnode.name) + continue + elif curnode.op_type in RESHAPE_OPS: + # check if the reshape operation will break the channel dependency + if reshape_break_channel_dependency(curnode): + # reshape operations also breaks the dependency relationship + continue + successors = self.graph.find_successors(curnode.unique_name) + successors = [self.graph.name_to_node[name] for name in successors] + for layer in successors: + queue.append(layer) + return key_layers + + def build_dependency(self): + """ + Build the input channel dependencies. + The `InputChannelDependency` indicates the layers that have + dependencies when pruning the input channel of the conv layers. + In contrast, `ChannelDependency` indicates the dependent layers + when pruning the output channles of conv layers (for example, L1FilterPruner). + """ + # unpack the tuple or list manually + self.graph.unpack_manually() + for tensor in self.graph.input_to_node: + # start from this tensor, find all the conv layers that + # take this tensor as input. Similar to the `ChannelDependency` + # the conv layer will truncate the dependencies + layers = self._get_following_convs(tensor) + dependency_set = set(layers) + for layer in layers: + if layer in self.dependency: + dependency_set.update(self.dependency[layer]) + for layer in dependency_set: + self.dependency[layer] = dependency_set + + +class CatPaddingDependency(ChannelDependency): + def __init__(self, model=None, dummy_input=None, traced_model=None): + super(CatPaddingDependency, self).__init__( + model, dummy_input, traced_model) + + def build_dependency(self): + """ + Build the cat padding dependencies. + If the output features of several layers are stitched together + by cat operation, then these layers have cat padding dependencies. + This is because when inferring the cat mask, we need all the input + masks for the cat operation. At this time we need to know the source + of all input vectors of a cat operation. + """ + for node in self.graph.nodes_py.nodes_op: + parent_layers = [] + if node.op_type == CAT_TYPE: + parent_layers = self._get_parent_layers(node) + dependency_set = set(parent_layers) + # merge the dependencies + for parent in parent_layers: + if parent in self.dependency: + dependency_set.update(self.dependency[parent]) + # save the dependencies + for _node in dependency_set: + self.dependency[_node] = dependency_set + + @property + def dependency_sets(self): + d_sets = [] + visited = set() + for nodename in self.dependency: + if nodename in visited: + continue + d_sets.append(self.dependency[nodename]) + return d_sets + + def export(self, filepath): + """ + Export the dependencies into a file. + In the output file, each line contains a set of layers + whose output features are stitched together by the cat + operation. + + output example: + Dependency Set, Layers + set1, Conv1, Conv2 + set2, Conv3, Conv4 + """ + header = ['Dependency Set', 'Layers'] + setid = 0 + with open(filepath, 'w') as csvf: + csv_w = csv.writer(csvf, delimiter=',') + csv_w.writerow(header) + for layers in self.dependency_sets: + setid += 1 + row = ['Set %d' % setid] + row.extend(list(layers)) + csv_w.writerow(row) + + +class GroupDependency(Dependency): + def __init__(self, model=None, dummy_input=None, traced_model=None): + """ + This model analyze the group dependencis between the conv + layers in a model. + + Parameters + ---------- + model : torch.nn.Module + The model to be analyzed. + data : torch.Tensor + The example input data to trace the network architecture. + traced_model : torch._C.Graph + if we alreay has the traced graph of the target model, we donnot + need to trace the model again. + """ + super(GroupDependency, self).__init__(model, dummy_input, traced_model) + + def _get_parent_convs(self, node): + """ + Find the nearest father conv layers for the target node. + + Parameters + --------- + node : torch._C.Node + target node. + + Returns + ------- + parent_layers : list + nearest father conv layers for the target node. Due to the group + dependency only exists between the conv layers, so we only find + the parent conv layers. + """ + parent_layers = [] + # the input node is a Conv node + predeessors = self.graph.find_predecessors(node.unique_name) + predeessors = [self.graph.name_to_node[x] for x in predeessors] + queue = predeessors + while queue: + curnode = queue.pop(0) + if curnode.op_type == 'Conv2d' or curnode.op_type == 'ConvTranspose2d': + # find the first met conv + parent_layers.append(curnode.name) + continue + parents = self.graph.find_predecessors(curnode.unique_name) + parents = [self.graph.name_to_node[name] for name in parents] + for parent in parents: + queue.append(parent) + return parent_layers + + def _get_conv_groups(self, node_group): + """ + Get the number of groups for a convolutional layer. + + Parameters + ---------- + node_group : NodePyGroup + target node. + + Returns + ------- + group : int + the number of the groups of the target conv layer. + """ + cpp_conv = list(filter(lambda x: x.kind() == + CONV_TYPE, node_group.node_cpps)) + assert len(cpp_conv) == 1 + cpp_conv = cpp_conv[0] + inputs = list(cpp_conv.inputs()) + # get the number of the group from the input parameters + group = inputs[8].toIValue() + return group + + def build_dependency(self): + """ + Build the channel dependency for the conv layers + in the model. This function return the group number + of each conv layers. Note that, here, the group count + of conv layers may be larger than their originl groups. + This is because that the input channel will also be grouped + for the group conv layers. To make this clear, assume we + have two group conv layers: conv1(group=2), conv2(group=4). + conv2 takes the output features of conv1 as input. + Then we have to the filters of conv1 can still be + divided into 4 groups after filter pruning, because + the input channels of conv2 shoule be divided into + 4 groups. + + Returns + ------- + self.dependency : dict + key: the name of conv layers, value: the minimum value that the number of + filters should be divisible to. + """ + for node in self.graph.nodes_py.nodes_op: + if node.op_type == 'Conv2d' or node.op_type == 'ConvTranspose2d': + group = self._get_conv_groups(node) + + if node.name in self.dependency: + # the conv layer whose group is larger than 1 will require that + # it's number of output channel to be divisible by the number of group. + self.dependency[node.name] = max( + self.dependency[node.name], group) + else: + self.dependency[node.name] = group + if group > 1: + # for the conv layer whose group is larger than 1, it will require the number + # of output channels of their parent conv layer to be divisible by group. + parent_convs = self._get_parent_convs(node) + for parent in parent_convs: + if parent in self.dependency: + self.dependency[parent] = max( + self.dependency[parent], group) + else: + self.dependency[parent] = group + return self.dependency + + def export(self, filepath): + """ + export the group dependency to a csv file. + Each line describes a convolution layer, the + first part of each line is the Pytorch module + name of the conv layer. The second part of each + line is the group count of the filters in this layer. + Note that, the group count may be larger than this + layers original group number. + + output example: + Conv layer, Groups + Conv1, 1 + Conv2, 2 + Conv3, 4 + """ + header = ['Conv Layer Name', 'Group'] + with open(filepath, 'w') as csvf: + csv_w = csv.writer(csvf, delimiter=',') + csv_w.writerow(header) + for name in self.dependency: + group = self.dependency[name] + csv_w.writerow([name, group]) + + @property + def dependency_sets(self): + return self.dependency diff --git a/utils/third_party/nni_new/compression/pytorch/utils/utils.py b/utils/third_party/nni_new/compression/pytorch/utils/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..c687c5e2a6bf971433e6298ee92a86be5c23b6d4 --- /dev/null +++ b/utils/third_party/nni_new/compression/pytorch/utils/utils.py @@ -0,0 +1,30 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +def get_module_by_name(model, module_name): + """ + Get a module specified by its module name + + Parameters + ---------- + model : pytorch model + the pytorch model from which to get its module + module_name : str + the name of the required module + + Returns + ------- + module, module + the parent module of the required module, the required module + """ + name_list = module_name.split(".") + for name in name_list[:-1]: + if hasattr(model, name): + model = getattr(model, name) + else: + return None, None + if hasattr(model, name_list[-1]): + leaf_module = getattr(model, name_list[-1]) + return model, leaf_module + else: + return None, None diff --git a/utils/third_party/nni_new/compression/tensorflow/__init__.py b/utils/third_party/nni_new/compression/tensorflow/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d05fade2f1140c68bb78969f89aaa8529414f3ca --- /dev/null +++ b/utils/third_party/nni_new/compression/tensorflow/__init__.py @@ -0,0 +1,4 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .compressor import Compressor, Pruner diff --git a/utils/third_party/nni_new/compression/tensorflow/compressor.py b/utils/third_party/nni_new/compression/tensorflow/compressor.py new file mode 100644 index 0000000000000000000000000000000000000000..bb249807e77c8ad14423dad8c6d09321b64f9e21 --- /dev/null +++ b/utils/third_party/nni_new/compression/tensorflow/compressor.py @@ -0,0 +1,338 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +Abstract base classes for TensorFlow model compression. +""" + +import logging + +import tensorflow as tf +assert tf.__version__.startswith('2'), 'NNI model compression only supports TensorFlow v2.x' + +from . import default_layers + +_logger = logging.getLogger(__name__) + + +class Compressor: + """ + Common base class for all compressors. + + This class is designed for other base classes. + Algorithms should inherit ``Pruner`` or ``Quantizer`` instead. + + Attributes + ---------- + compressed_model : tf.keras.Model + Compressed user model. + wrappers : list of tf.keras.Model + A wrapper is an instrumented TF ``Layer``, in ``Model`` format. + + Parameters + ---------- + model : tf.keras.Model + The user model to be compressed. + config_list : list of JSON object + User configuration. The format is detailed in tutorial. + LayerWrapperClass : a class derive from Model + The class used to instrument layers. + """ + + def __init__(self, model, config_list, LayerWrapperClass): + assert isinstance(model, tf.keras.Model) + self.validate_config(model, config_list) + + self._original_model = model + self._config_list = config_list + self._wrapper_class = LayerWrapperClass + self._wrappers = {} # key: id(layer) , value: Wrapper(layer) + + self.compressed_model = self._instrument(model) + self.wrappers = list(self._wrappers.values()) + + if not self.wrappers: + _logger.warning('Nothing is configured to compress, please check your model and config list') + + def set_wrappers_attribute(self, name, value): + """ + Call ``setattr`` on all wrappers. + """ + for wrapper in self.wrappers: + setattr(wrapper, name, value) + + def validate_config(self, model, config_list): + """ + Compression algorithm should overload this function to validate configuration. + """ + pass + + + def _instrument(self, layer): + if isinstance(layer, tf.keras.Sequential): + return self._instrument_sequential(layer) + if isinstance(layer, tf.keras.Model): + return self._instrument_model(layer) + + # a layer can be referenced in multiple attributes of a model, + # but should only be instrumented once + if id(layer) in self._wrappers: + return self._wrappers[id(layer)] + + config = self._select_config(layer) + if config is not None: + wrapper = self._wrapper_class(layer, config, self) + self._wrappers[id(layer)] = wrapper + return wrapper + + return layer + + def _uninstrument(self, layer): + # note that ``self._wrappers`` cache is not cleared here, + # so the same wrapper objects will be recovered in next ``self._instrument()`` call + if isinstance(layer, LayerWrapper): + layer._instrumented = False + return self._uninstrument(layer.layer) + if isinstance(layer, tf.keras.Sequential): + return self._uninstrument_sequential(layer) + if isinstance(layer, tf.keras.Model): + return self._uninstrument_model(layer) + return layer + + def _instrument_sequential(self, seq): + layers = list(seq.layers) # seq.layers is read-only property + need_rebuild = False + for i, layer in enumerate(layers): + new_layer = self._instrument(layer) + if new_layer is not layer: + layers[i] = new_layer + need_rebuild = True + return tf.keras.Sequential(layers) if need_rebuild else seq + + def _uninstrument_sequential(self, seq): + layers = list(seq.layers) + rebuilt = False + for i, layer in enumerate(layers): + orig_layer = self._uninstrument(layer) + if orig_layer is not layer: + layers[i] = orig_layer + rebuilt = True + return tf.keras.Sequential(layers) if rebuilt else seq + + def _instrument_model(self, model): + for key, value in list(model.__dict__.items()): # avoid "dictionary keys changed during iteration" + if isinstance(value, tf.keras.layers.Layer): + new_layer = self._instrument(value) + if new_layer is not value: + setattr(model, key, new_layer) + elif isinstance(value, list): + for i, item in enumerate(value): + if isinstance(item, tf.keras.layers.Layer): + value[i] = self._instrument(item) + return model + + def _uninstrument_model(self, model): + for key, value in list(model.__dict__.items()): + if isinstance(value, tf.keras.layers.Layer): + orig_layer = self._uninstrument(value) + if orig_layer is not value: + setattr(model, key, orig_layer) + elif isinstance(value, list): + for i, item in enumerate(value): + if isinstance(item, tf.keras.layers.Layer): + value[i] = self._uninstrument(item) + return model + + def _select_config(self, layer): + # Find the last matching config block for given layer. + # Returns None if the layer should not be compressed. + layer_type = type(layer).__name__ + last_match = None + for config in self._config_list: + if 'op_types' in config: + match = layer_type in config['op_types'] + match_default = 'default' in config['op_types'] and layer_type in default_layers.weighted_modules + if not match and not match_default: + continue + if 'op_names' in config and layer.name not in config['op_names']: + continue + last_match = config + if last_match is None or 'exclude' in last_match: + return None + return last_match + + +class LayerWrapper(tf.keras.Model): + """ + Abstract base class of layer wrappers. + + Concrete layer wrapper classes must inherit this to support ``isinstance`` check. + """ + def __init__(self): + super().__init__() + self._instrumented = True + + +class Pruner(Compressor): + """ + Base class for pruning algorithms. + + End users should use ``compress`` and callback APIs (WIP) to prune their models. + + The underlying model is instrumented upon initialization of pruner object. + So if you want to pre-train the model, train it before creating pruner object. + + The compressed model can only execute in eager mode. + + Algorithm developers should override ``calc_masks`` method to specify pruning strategy. + + Parameters + ---------- + model : tf.keras.Model + The user model to prune. + config_list : list of JSON object + User configuration. The format is detailed in tutorial. + """ + def __init__(self, model, config_list): + super().__init__(model, config_list, PrunerLayerWrapper) + #self.callback = PrunerCallback(self) + + def compress(self): + """ + Apply compression on a pre-trained model. + + If you want to prune the model during training, use callback API (WIP) instead. + + Returns + ------- + tf.keras.Model + The compressed model. + """ + self._update_mask() + return self.compressed_model + + def export_model(self, model_path, mask_path=None): + """ + Export pruned model and optionally mask tensors. + + Parameters + ---------- + model_path : path-like + The path passed to ``Model.save()``. + You can use ".h5" extension name to export HDF5 format. + mask_path : path-like or None + Export masks to the path when set. + Because Keras cannot save tensors without a ``Model``, + this will create a model, set all masks as its weights, and then save that model. + Masks in saved model will be named by corresponding layer name in compressed model. + + Returns + ------- + None + """ + _logger.info('Saving model to %s', model_path) + input_shape = self.compressed_model._build_input_shape # cannot find a public API + model = self._uninstrument(self.compressed_model) + if input_shape: + model.build(input_shape) + model.save(model_path) + self._instrument(model) + + if mask_path is not None: + _logger.info('Saving masks to %s', mask_path) + # can't find "save raw weights" API in tensorflow, so build a simple model + mask_model = tf.keras.Model() + for wrapper in self.wrappers: + setattr(mask_model, wrapper.layer.name, wrapper.masks) + mask_model.save_weights(mask_path) + + _logger.info('Done') + + def calc_masks(self, wrapper, **kwargs): + """ + Abstract method to be overridden by algorithm. End users should ignore it. + + If the callback is set up, this method will be invoked at end of each training minibatch. + If not, it will only be called when end user invokes ``compress``. + + Parameters + ---------- + wrapper : PrunerLayerWrapper + The instrumented layer. + **kwargs + Reserved for forward compatibility. + + Returns + ------- + dict of (str, tf.Tensor), or None + The key is weight ``Variable``'s name. The value is a mask ``Tensor`` of weight's shape and dtype. + If a weight's key does not appear in the return value, that weight will not be pruned. + Returning ``None`` means the mask is not changed since last time. + Weight names are globally unique, e.g. `model/conv_1/kernel:0`. + """ + # TODO: maybe it should be able to calc on weight-granularity, beside from layer-granularity + raise NotImplementedError("Pruners must overload calc_masks()") + + def _update_mask(self): + for wrapper_idx, wrapper in enumerate(self.wrappers): + masks = self.calc_masks(wrapper, wrapper_idx=wrapper_idx) + if masks is not None: + wrapper.masks = masks + + +class PrunerLayerWrapper(LayerWrapper): + """ + Instrumented TF layer. + + Wrappers will be passed to pruner's ``calc_masks`` API, + and the pruning algorithm should use wrapper's attributes to calculate masks. + + Once instrumented, underlying layer's weights will get **modified** by masks before forward pass. + + Attributes + ---------- + layer : tf.keras.layers.Layer + The original layer. + config : JSON object + Selected configuration. The format is detailed in tutorial. + pruner : Pruner + Bound pruner object. + masks : dict of (str, tf.Tensor) + Current masks. The key is weight's name and the value is mask tensor. + On initialization, `masks` is an empty dict, which means no weight is pruned. + Afterwards, `masks` is the last return value of ``Pruner.calc_masks``. + See ``Pruner.calc_masks`` for details. + """ + def __init__(self, layer, config, pruner): + super().__init__() + self.layer = layer + self.config = config + self.pruner = pruner + self.masks = {} + _logger.info('Layer detected to compress: %s', self.layer.name) + + def call(self, *inputs): + self._update_weights() + return self.layer(*inputs) + + def _update_weights(self): + new_weights = [] + for weight in self.layer.weights: + mask = self.masks.get(weight.name) + if mask is not None: + new_weights.append(tf.math.multiply(weight, mask)) + else: + new_weights.append(weight) + if new_weights and not hasattr(new_weights[0], 'numpy'): + raise RuntimeError('NNI: Compressed model can only run in eager mode') + self.layer.set_weights([weight.numpy() for weight in new_weights]) + + +# TODO: designed to replace `patch_optimizer` +#class PrunerCallback(tf.keras.callbacks.Callback): +# def __init__(self, pruner): +# super().__init__() +# self._pruner = pruner +# +# def on_train_batch_end(self, batch, logs=None): +# self._pruner.update_mask() diff --git a/utils/third_party/nni_new/compression/tensorflow/default_layers.py b/utils/third_party/nni_new/compression/tensorflow/default_layers.py new file mode 100644 index 0000000000000000000000000000000000000000..0c729bd883f1623d28ad279de053a39a5742109c --- /dev/null +++ b/utils/third_party/nni_new/compression/tensorflow/default_layers.py @@ -0,0 +1,9 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +weighted_modules = [ + 'Conv1D', 'Conv2D', 'Conv3D', 'Conv1DTranspose', 'Conv2DTranspose', 'Conv3DTranspose', + 'Dense', + 'PReLU', + 'Embedding', +] diff --git a/utils/third_party/nni_new/experiment/__init__.py b/utils/third_party/nni_new/experiment/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d59aec4e71220918c30017a1184544524b267944 --- /dev/null +++ b/utils/third_party/nni_new/experiment/__init__.py @@ -0,0 +1,6 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .config import * +from .experiment import Experiment +from .data import * diff --git a/utils/third_party/nni_new/experiment/config/__init__.py b/utils/third_party/nni_new/experiment/config/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..cc7feefdbd81cc0c2d5fa30fa1be548eea7266f7 --- /dev/null +++ b/utils/third_party/nni_new/experiment/config/__init__.py @@ -0,0 +1,12 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .common import * +from .local import * +from .remote import * +from .openpai import * +from .aml import * +from .kubeflow import * +from .frameworkcontroller import * +from .adl import * +from .shared_storage import * diff --git a/utils/third_party/nni_new/experiment/config/adl.py b/utils/third_party/nni_new/experiment/config/adl.py new file mode 100644 index 0000000000000000000000000000000000000000..0f9a856ab62f92850abf30bc57c531be8f506215 --- /dev/null +++ b/utils/third_party/nni_new/experiment/config/adl.py @@ -0,0 +1,17 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from dataclasses import dataclass + +from .common import TrainingServiceConfig + +__all__ = ['AdlConfig'] + +@dataclass(init=False) +class AdlConfig(TrainingServiceConfig): + platform: str = 'adl' + docker_image: str = 'msranni/nni:latest' + + _validation_rules = { + 'platform': lambda value: (value == 'adl', 'cannot be modified') + } diff --git a/utils/third_party/nni_new/experiment/config/aml.py b/utils/third_party/nni_new/experiment/config/aml.py new file mode 100644 index 0000000000000000000000000000000000000000..2fd92e7e76e898d13295edc23171c1abc62e1bd5 --- /dev/null +++ b/utils/third_party/nni_new/experiment/config/aml.py @@ -0,0 +1,22 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from dataclasses import dataclass + +from .common import TrainingServiceConfig + +__all__ = ['AmlConfig'] + +@dataclass(init=False) +class AmlConfig(TrainingServiceConfig): + platform: str = 'aml' + subscription_id: str + resource_group: str + workspace_name: str + compute_target: str + docker_image: str = 'msranni/nni:latest' + max_trial_number_per_gpu: int = 1 + + _validation_rules = { + 'platform': lambda value: (value == 'aml', 'cannot be modified') + } diff --git a/utils/third_party/nni_new/experiment/config/base.py b/utils/third_party/nni_new/experiment/config/base.py new file mode 100644 index 0000000000000000000000000000000000000000..d1fc92a26f1ad2d5b25be748bea917bfaf47d434 --- /dev/null +++ b/utils/third_party/nni_new/experiment/config/base.py @@ -0,0 +1,153 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import copy +import dataclasses +from pathlib import Path +from typing import Any, Dict, Optional, Type, TypeVar + +import yaml + +from . import util + +__all__ = ['ConfigBase', 'PathLike'] + +T = TypeVar('T', bound='ConfigBase') + +PathLike = util.PathLike + +def _is_missing(obj: Any) -> bool: + return isinstance(obj, type(dataclasses.MISSING)) + +class ConfigBase: + """ + Base class of config classes. + Subclass may override `_canonical_rules` and `_validation_rules`, + and `validate()` if the logic is complex. + """ + + # Rules to convert field value to canonical format. + # The key is field name. + # The value is callable `value -> canonical_value` + # It is not type-hinted so dataclass won't treat it as field + _canonical_rules = {} # type: ignore + + # Rules to validate field value. + # The key is field name. + # The value is callable `value -> valid` or `value -> (valid, error_message)` + # The rule will be called with canonical format and is only called when `value` is not None. + # `error_message` is used when `valid` is False. + # It will be prepended with class name and field name in exception message. + _validation_rules = {} # type: ignore + + def __init__(self, *, _base_path: Optional[Path] = None, **kwargs): + """ + Initialize a config object and set some fields. + Name of keyword arguments can either be snake_case or camelCase. + They will be converted to snake_case automatically. + If a field is missing and don't have default value, it will be set to `dataclasses.MISSING`. + """ + if 'basepath' in kwargs: + _base_path = kwargs.pop('basepath') + kwargs = {util.case_insensitive(key): value for key, value in kwargs.items()} + if _base_path is None: + _base_path = Path() + for field in dataclasses.fields(self): + value = kwargs.pop(util.case_insensitive(field.name), field.default) + if value is not None and not _is_missing(value): + # relative paths loaded from config file are not relative to pwd + if 'Path' in str(field.type): + value = Path(value).expanduser() + if not value.is_absolute(): + value = _base_path / value + setattr(self, field.name, value) + if kwargs: + cls = type(self).__name__ + fields = ', '.join(kwargs.keys()) + raise ValueError(f'{cls}: Unrecognized fields {fields}') + + @classmethod + def load(cls: Type[T], path: PathLike) -> T: + """ + Load config from YAML (or JSON) file. + Keys in YAML file can either be camelCase or snake_case. + """ + data = yaml.safe_load(open(path)) + if not isinstance(data, dict): + raise ValueError(f'Content of config file {path} is not a dict/object') + return cls(**data, _base_path=Path(path).parent) + + def json(self) -> Dict[str, Any]: + """ + Convert config to JSON object. + The keys of returned object will be camelCase. + """ + self.validate() + return dataclasses.asdict( + self.canonical(), + dict_factory=lambda items: dict((util.camel_case(k), v) for k, v in items if v is not None) + ) + + def canonical(self: T) -> T: + """ + Returns a deep copy, where the fields supporting multiple formats are converted to the canonical format. + Noticeably, relative path may be converted to absolute path. + """ + ret = copy.deepcopy(self) + for field in dataclasses.fields(ret): + key, value = field.name, getattr(ret, field.name) + rule = ret._canonical_rules.get(key) + if rule is not None: + setattr(ret, key, rule(value)) + elif isinstance(value, ConfigBase): + setattr(ret, key, value.canonical()) + # value will be copied twice, should not be a performance issue anyway + elif isinstance(value, Path): + setattr(ret, key, str(value)) + return ret + + def validate(self) -> None: + """ + Validate the config object and raise Exception if it's ill-formed. + """ + class_name = type(self).__name__ + config = self.canonical() + + for field in dataclasses.fields(config): + key, value = field.name, getattr(config, field.name) + + # check existence + if _is_missing(value): + raise ValueError(f'{class_name}: {key} is not set') + + # check type (TODO) + type_name = str(field.type).replace('typing.', '') + optional = any([ + type_name.startswith('Optional['), + type_name.startswith('Union[') and 'None' in type_name, + type_name == 'Any' + ]) + if value is None: + if optional: + continue + else: + raise ValueError(f'{class_name}: {key} cannot be None') + + # check value + rule = config._validation_rules.get(key) + if rule is not None: + try: + result = rule(value) + except Exception: + raise ValueError(f'{class_name}: {key} has bad value {repr(value)}') + + if isinstance(result, bool): + if not result: + raise ValueError(f'{class_name}: {key} ({repr(value)}) is out of range') + else: + if not result[0]: + raise ValueError(f'{class_name}: {key} {result[1]}') + + # check nested config + if isinstance(value, ConfigBase): + value.validate() diff --git a/utils/third_party/nni_new/experiment/config/common.py b/utils/third_party/nni_new/experiment/config/common.py new file mode 100644 index 0000000000000000000000000000000000000000..c7cd64a7da85a52ba3d2da0816c754acde1e6301 --- /dev/null +++ b/utils/third_party/nni_new/experiment/config/common.py @@ -0,0 +1,191 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Dict, List, Optional, Union + +import yaml + +from .base import ConfigBase, PathLike +from . import util + +__all__ = [ + 'ExperimentConfig', + 'AlgorithmConfig', + 'CustomAlgorithmConfig', + 'TrainingServiceConfig', +] + + +@dataclass(init=False) +class _AlgorithmConfig(ConfigBase): + name: Optional[str] = None + class_name: Optional[str] = None + code_directory: Optional[PathLike] = None + class_args: Optional[Dict[str, Any]] = None + + def validate(self): + super().validate() + _validate_algo(self) + + _canonical_rules = {'code_directory': util.canonical_path} + +@dataclass(init=False) +class AlgorithmConfig(_AlgorithmConfig): + name: str + class_args: Optional[Dict[str, Any]] = None + +@dataclass(init=False) +class CustomAlgorithmConfig(_AlgorithmConfig): + class_name: str + code_directory: Optional[PathLike] = '.' + class_args: Optional[Dict[str, Any]] = None + + +class TrainingServiceConfig(ConfigBase): + platform: str + +class SharedStorageConfig(ConfigBase): + storage_type: str + local_mount_point: str + remote_mount_point: str + local_mounted: str + + +@dataclass(init=False) +class ExperimentConfig(ConfigBase): + experiment_name: Optional[str] = None + search_space_file: Optional[PathLike] = None + search_space: Any = None + trial_command: str + trial_code_directory: PathLike = '.' + trial_concurrency: int + trial_gpu_number: Optional[int] = None # TODO: in openpai cannot be None + max_experiment_duration: Optional[str] = None + max_trial_number: Optional[int] = None + nni_manager_ip: Optional[str] = None + use_annotation: bool = False + debug: bool = False + log_level: Optional[str] = None + experiment_working_directory: PathLike = '~/nni-experiments' + tuner_gpu_indices: Union[List[int], str, int, None] = None + tuner: Optional[_AlgorithmConfig] = None + assessor: Optional[_AlgorithmConfig] = None + advisor: Optional[_AlgorithmConfig] = None + training_service: Union[TrainingServiceConfig, List[TrainingServiceConfig]] + shared_storage: Optional[SharedStorageConfig] = None + _deprecated: Optional[Dict[str, Any]] = None + + def __init__(self, training_service_platform: Optional[Union[str, List[str]]] = None, **kwargs): + base_path = kwargs.pop('_base_path', None) + kwargs = util.case_insensitive(kwargs) + if training_service_platform is not None: + assert 'trainingservice' not in kwargs + kwargs['trainingservice'] = util.training_service_config_factory( + platform=training_service_platform, + base_path=base_path + ) + elif isinstance(kwargs.get('trainingservice'), (dict, list)): + # dict means a single training service + # list means hybrid training service + kwargs['trainingservice'] = util.training_service_config_factory( + config=kwargs['trainingservice'], + base_path=base_path + ) + else: + raise RuntimeError('Unsupported Training service configuration!') + super().__init__(_base_path=base_path, **kwargs) + for algo_type in ['tuner', 'assessor', 'advisor']: + if isinstance(kwargs.get(algo_type), dict): + setattr(self, algo_type, _AlgorithmConfig(**kwargs.pop(algo_type))) + + def canonical(self): + ret = super().canonical() + if isinstance(ret.training_service, list): + for i, ts in enumerate(ret.training_service): + ret.training_service[i] = ts.canonical() + return ret + + def validate(self, initialized_tuner: bool = False) -> None: + super().validate() + if initialized_tuner: + _validate_for_exp(self.canonical()) + else: + _validate_for_nnictl(self.canonical()) + if self.trial_gpu_number and hasattr(self.training_service, 'use_active_gpu'): + if self.training_service.use_active_gpu is None: + raise ValueError('Please set "use_active_gpu"') + + def json(self) -> Dict[str, Any]: + obj = super().json() + if obj.get('searchSpaceFile'): + obj['searchSpace'] = yaml.safe_load(open(obj.pop('searchSpaceFile'))) + return obj + +## End of public API ## + + @property + def _canonical_rules(self): + return _canonical_rules + + @property + def _validation_rules(self): + return _validation_rules + + +_canonical_rules = { + 'search_space_file': util.canonical_path, + 'trial_code_directory': util.canonical_path, + 'max_experiment_duration': lambda value: f'{util.parse_time(value)}s' if value is not None else None, + 'experiment_working_directory': util.canonical_path, + 'tuner_gpu_indices': util.canonical_gpu_indices, + 'tuner': lambda config: None if config is None or config.name == '_none_' else config.canonical(), + 'assessor': lambda config: None if config is None or config.name == '_none_' else config.canonical(), + 'advisor': lambda config: None if config is None or config.name == '_none_' else config.canonical(), +} + +_validation_rules = { + 'search_space_file': lambda value: (Path(value).is_file(), f'"{value}" does not exist or is not regular file'), + 'trial_code_directory': lambda value: (Path(value).is_dir(), f'"{value}" does not exist or is not directory'), + 'trial_concurrency': lambda value: value > 0, + 'trial_gpu_number': lambda value: value >= 0, + 'max_experiment_duration': lambda value: util.parse_time(value) > 0, + 'max_trial_number': lambda value: value > 0, + 'log_level': lambda value: value in ["trace", "debug", "info", "warning", "error", "fatal"], + 'tuner_gpu_indices': lambda value: all(i >= 0 for i in value) and len(value) == len(set(value)), + 'training_service': lambda value: (type(value) is not TrainingServiceConfig, 'cannot be abstract base class') +} + +def _validate_for_exp(config: ExperimentConfig) -> None: + # validate experiment for nni.Experiment, where tuner is already initialized outside + if config.use_annotation: + raise ValueError('ExperimentConfig: annotation is not supported in this mode') + if util.count(config.search_space, config.search_space_file) != 1: + raise ValueError('ExperimentConfig: search_space and search_space_file must be set one') + if util.count(config.tuner, config.assessor, config.advisor) != 0: + raise ValueError('ExperimentConfig: tuner, assessor, and advisor must not be set in for this mode') + if config.tuner_gpu_indices is not None: + raise ValueError('ExperimentConfig: tuner_gpu_indices is not supported in this mode') + +def _validate_for_nnictl(config: ExperimentConfig) -> None: + # validate experiment for normal launching approach + if config.use_annotation: + if util.count(config.search_space, config.search_space_file) != 0: + raise ValueError('ExperimentConfig: search_space and search_space_file must not be set with annotationn') + else: + if util.count(config.search_space, config.search_space_file) != 1: + raise ValueError('ExperimentConfig: search_space and search_space_file must be set one') + if util.count(config.tuner, config.advisor) != 1: + raise ValueError('ExperimentConfig: tuner and advisor must be set one') + +def _validate_algo(algo: AlgorithmConfig) -> None: + if algo.name is None: + if algo.class_name is None: + raise ValueError('Missing algorithm name') + if algo.code_directory is not None and not Path(algo.code_directory).is_dir(): + raise ValueError(f'code_directory "{algo.code_directory}" does not exist or is not directory') + else: + if algo.class_name is not None or algo.code_directory is not None: + raise ValueError(f'When name is set for registered algorithm, class_name and code_directory cannot be used') + # TODO: verify algorithm installation and class args diff --git a/utils/third_party/nni_new/experiment/config/convert.py b/utils/third_party/nni_new/experiment/config/convert.py new file mode 100644 index 0000000000000000000000000000000000000000..0b4b332d08f6433201400e8e880691d460ecb71f --- /dev/null +++ b/utils/third_party/nni_new/experiment/config/convert.py @@ -0,0 +1,266 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import copy +import logging + +from .common import ExperimentConfig, AlgorithmConfig, CustomAlgorithmConfig +from .remote import RemoteMachineConfig +from .kubeflow import KubeflowRoleConfig, KubeflowNfsConfig, KubeflowAzureStorageConfig +from .frameworkcontroller import FrameworkControllerRoleConfig +from .shared_storage import NfsConfig, AzureBlobConfig +from . import util + +_logger = logging.getLogger(__name__) + +def to_v2(v1) -> ExperimentConfig: + v1 = copy.deepcopy(v1) + platform = v1.pop('trainingServicePlatform') + assert platform in ['local', 'remote', 'openpai', 'aml'] + v2 = ExperimentConfig(platform) + + _drop_field(v1, 'authorName') + _move_field(v1, v2, 'experimentName', 'experiment_name') + _drop_field(v1, 'description') + _move_field(v1, v2, 'trialConcurrency', 'trial_concurrency') + _move_field(v1, v2, 'maxExecDuration', 'max_experiment_duration') + if isinstance(v2.max_experiment_duration, (int, float)): + v2.max_experiment_duration = str(v2.max_experiment_duration) + 's' + _move_field(v1, v2, 'maxTrialNum', 'max_trial_number') + _move_field(v1, v2, 'searchSpacePath', 'search_space_file') + assert not v1.pop('multiPhase', None), 'Multi-phase is no longer supported' + _deprecate(v1, v2, 'multiThread') + _move_field(v1, v2, 'nniManagerIp', 'nni_manager_ip') + _move_field(v1, v2, 'logDir', 'experiment_working_directory') + _move_field(v1, v2, 'debug', 'debug') + _deprecate(v1, v2, 'versionCheck') + _move_field(v1, v2, 'logLevel', 'log_level') + _deprecate(v1, v2, 'logCollection') + v1.pop('useAnnotation', None) # TODO: how to handle annotation in nni.Experiment? + + if 'trial' in v1: + v1_trial = v1.pop('trial') + _move_field(v1_trial, v2, 'command', 'trial_command') + _move_field(v1_trial, v2, 'codeDir', 'trial_code_directory') + _move_field(v1_trial, v2, 'gpuNum', 'trial_gpu_number') + + for algo_type in ['tuner', 'assessor', 'advisor']: + if algo_type in v1: + convert_algo(algo_type, v1, v2) + + ts = v2.training_service + + if platform == 'local': + local_config = v1.pop('localConfig', {}) + _move_field(local_config, ts, 'gpuIndices', 'gpu_indices') + _move_field(local_config, ts, 'maxTrialNumPerGpu', 'max_trial_number_per_gpu') + _move_field(local_config, ts, 'useActiveGpu', 'use_active_gpu') + assert not local_config, local_config + + if platform == 'remote': + remote_config = v1.pop('remoteConfig', {}) + _move_field(remote_config, ts, 'reuse', 'reuse_mode') + assert not remote_config, remote_config + + ts.machine_list = [] + for v1_machine in v1.pop('machineList'): + v2_machine = RemoteMachineConfig() + ts.machine_list.append(v2_machine) + _move_field(v1_machine, v2_machine, 'ip', 'host') + _move_field(v1_machine, v2_machine, 'port', 'port') + _move_field(v1_machine, v2_machine, 'username', 'user') + _move_field(v1_machine, v2_machine, 'sshKeyPath', 'ssh_key_file') + _move_field(v1_machine, v2_machine, 'passphrase', 'ssh_passphrase') + _move_field(v1_machine, v2_machine, 'gpuIndices', 'gpu_indices') + _move_field(v1_machine, v2_machine, 'maxTrialNumPerGpu', 'max_trial_number_per_gpu') + _move_field(v1_machine, v2_machine, 'useActiveGpu', 'use_active_gpu') + _move_field(v1_machine, v2_machine, 'pythonPath', 'python_path') + _move_field(v1_machine, v2_machine, 'passwd', 'password') + assert not v1_machine, v1_machine + + if platform == 'openpai': + _move_field(v1_trial, ts, 'nniManagerNFSMountPath', 'local_storage_mount_point') + _move_field(v1_trial, ts, 'containerNFSMountPath', 'container_storage_mount_point') + _move_field(v1_trial, ts, 'cpuNum', 'trial_cpu_number') + if 'memoryMB' in v1_trial: + ts.trial_memory_size = str(v1_trial.pop('memoryMB')) + 'mb' + _move_field(v1_trial, ts, 'image', 'docker_image') + _deprecate(v1_trial, v2, 'virtualCluster') + _move_field(v1_trial, ts, 'paiStorageConfigName', 'storage_config_name') + _move_field(v1_trial, ts, 'paiConfigPath', 'openpaiConfigFile') + + pai_config = v1.pop('paiConfig') + _move_field(pai_config, ts, 'userName', 'username') + _deprecate(pai_config, v2, 'password') + _move_field(pai_config, ts, 'token', 'token') + _move_field(pai_config, ts, 'host', 'host') + _move_field(pai_config, ts, 'reuse', 'reuse_mode') + _move_field(pai_config, ts, 'gpuNum', 'trial_gpu_number') + _move_field(pai_config, ts, 'cpuNum', 'trial_cpu_number') + if 'memoryMB' in pai_config: + ts.trial_memory_size = str(pai_config.pop('memoryMB')) + 'mb' + _deprecate(pai_config, v2, 'maxTrialNumPerGpu') + _deprecate(pai_config, v2, 'useActiveGpu') + assert not pai_config, pai_config + + if platform == 'aml': + _move_field(v1_trial, ts, 'image', 'docker_image') + + aml_config = v1.pop('amlConfig', {}) + _move_field(aml_config, ts, 'subscriptionId', 'subscription_id') + _move_field(aml_config, ts, 'resourceGroup', 'resource_group') + _move_field(aml_config, ts, 'workspaceName', 'workspace_name') + _move_field(aml_config, ts, 'computeTarget', 'compute_target') + _move_field(aml_config, ts, 'maxTrialNumPerGpu', 'max_trial_number_per_gpu') + _deprecate(aml_config, v2, 'useActiveGpu') + assert not aml_config, aml_config + + if platform == 'kubeflow': + kf_config = v1.pop('kubeflowConfig') + _move_field(kf_config, ts, 'operator', 'operator') + ps_name = 'ps' if ts.operator != 'pytorch-operator' else 'master' + _move_field(kf_config, ts, 'apiVersion', 'api_version') + + # FIXME: use storage service + storage_name = kf_config.pop('storage', None) + if storage_name is None: + storage_name = 'nfs' if 'nfs' in kf_config else 'azureStorage' + if storage_name == 'nfs': + nfs = kf_config.pop('nfs') + ts.storage = KubeflowNfsConfig(server=nfs['server'], path=nfs['path']) + if storage_name == 'azureStorage': + key_vault = kf_config.pop('keyVault') + azure_storage = kf_config.pop('azureStorage') + ts.storage = KubeflowAzureStorageConfig( + azure_account=azure_storage['accountName'], + azure_share=azure_storage['azureShare'], + key_vault=key_vault['vaultName'], + key_vault_secret=key_vault['name'] + ) + _deprecate(kf_config, v2, 'uploadRetryCount') + + assert not kf_config, kf_config + + _drop_field(v1_trial, 'nasMode') + for role_name in [ps_name, 'worker']: + if role_name not in v1_trial: + continue + v1_role = v1_trial.pop(role_name) + v2_role = KubeflowRoleConfig() + if role_name == 'worker': + ts.worker = v2_role + else: + ts.parameter_server = v2_role + + _move_field(v1_role, v2_role, 'replicas', 'replicas') + _move_field(v1_role, v2_role, 'command', 'command') + _move_field(v1_role, v2_role, 'gpu_num', 'gpu_number') + _move_field(v1_role, v2_role, 'cpu_num', 'cpu_number') + v2_role.memory_size = str(v1_role.pop('memoryMB')) + 'mb' + _move_field(v1_role, v2_role, 'image', 'docker_image') + _deprecate(v1_role, v2, 'privateRegistryAuthPath') + assert not v1_role, v1_role + + if platform == 'frameworkcontroller': + fc_config = v1.pop('frameworkcontroller') + _deprecate(fc_config, v2, 'serviceAccountName') + + storage_name = fc_config.pop('storage', None) + if storage_name is None: + storage_name = 'nfs' if 'nfs' in fc_config else 'azureStorage' + if storage_name == 'nfs': + nfs = fc_config.pop('nfs') + ts.storage = KubeflowNfsConfig(server=nfs['server'], path=nfs['path']) + if storage_name == 'azureStorage': + key_vault = fc_config.pop('keyVault') + azure_storage = fc_config.pop('azureStorage') + ts.storage = KubeflowAzureStorageConfig( + azure_account=azure_storage['accountName'], + azure_share=azure_storage['azureShare'], + key_vault=key_vault['vaultName'], + key_vault_secret=key_vault['name'] + ) + _deprecate(fc_config, v2, 'uploadRetryCount') + + assert not fc_config, fc_config + + _drop_field(v1_trial, 'nasMode') + ts.task_roles = [] + for v1_role in v1_trial.pop('taskRoles', []): + v2_role = FrameworkControllerRoleConfig() + ts.task_roles.append(v2_role) + + _move_field(v1_role, v2_role, 'name', 'name') + _move_field(v1_role, v2_role, 'taskNum', 'task_number') + policy = v1_role.pop('frameworkControllerCompletionPolicy', {}) + _move_field(policy, v2_role, 'minFailedTaskCount', 'attempt_completion_min_failed_tasks') + _move_field(policy, v2_role, 'minSucceededTaskCount', 'attempt_completion_min_succeeded_tasks') + _move_field(v1_role, v2_role, 'command', 'command') + _move_field(v1_role, v2_role, 'gpuNum', 'gpu_number') + _move_field(v1_role, v2_role, 'cpuNum', 'cpu_number') + v2_role.memory_size = str(v1_role.pop('memoryMB')) + 'mb' + _move_field(v1_role, v2_role, 'image', 'docker_image') + _deprecate(v1_role, v2, 'privateRegistryAuthPath') + assert not v1_role, v1_role + + # hybrid mode should always use v2 schema, so no need to handle here + + v1_storage = v1.pop('sharedStorage', None) + if v1_storage: + type_ = v1_storage.pop('storageType') + if type_ == 'NFS': + v2.shared_storage = NfsConfig(**v1_storage) + elif type_ == 'AzureBlob': + v2.shared_storage = AzureBlobConfig(**v1_storage) + else: + raise ValueError(f'bad storage type: {type_}') + + assert not v1_trial, v1_trial + assert not v1, v1 + return v2.canonical() + +def _move_field(v1, v2, v1_key, v2_key): + if v1_key in v1: + value = v1.pop(v1_key, None) + if value is not None: + setattr(v2, v2_key, value) + +def _drop_field(v1, key): + if key in v1: + logging.warning(f'Configuration field {key} is no longer supported and has been ignored') + v1.pop(key) + +# NOTE: fields not yet supported by v2 are also (temporarily) placed here +def _deprecate(v1, v2, key): + if key in v1: + if v2._deprecated is None: + v2._deprecated = {} + v2._deprecated[key] = v1.pop(key) + +def convert_algo(algo_type, v1, v2): + if algo_type not in v1: + return None + v1_algo = v1.pop(algo_type) + + builtin_name = v1_algo.pop(f'builtin{algo_type.title()}Name', None) + class_args = v1_algo.pop('classArgs', None) + + if builtin_name is not None: + v2_algo = AlgorithmConfig(name=builtin_name, class_args=class_args) + + else: + code_directory = util.canonical_path(v1_algo.pop('codeDir')) + class_file_name = v1_algo.pop('classFileName') + assert class_file_name.endswith('.py') + class_name = class_file_name[:-3] + '.' + v1_algo.pop('className') + v2_algo = CustomAlgorithmConfig( + class_name=class_name, + code_directory=code_directory, + class_args=class_args + ) + + setattr(v2, algo_type, v2_algo) + _deprecate(v1_algo, v2, 'includeIntermediateResults') + _move_field(v1_algo, v2, 'gpuIndices', 'tuner_gpu_indices') + assert not v1_algo, v1_algo + return v2_algo diff --git a/utils/third_party/nni_new/experiment/config/frameworkcontroller.py b/utils/third_party/nni_new/experiment/config/frameworkcontroller.py new file mode 100644 index 0000000000000000000000000000000000000000..fbc1397630609171a9d08ed2f4a3ba555543e3b9 --- /dev/null +++ b/utils/third_party/nni_new/experiment/config/frameworkcontroller.py @@ -0,0 +1,72 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from dataclasses import dataclass +from typing import List, Optional + +from .base import ConfigBase +from .common import TrainingServiceConfig +from . import util + +__all__ = [ + 'FrameworkControllerConfig', + 'FrameworkControllerRoleConfig', + 'FrameworkControllerNfsConfig', + 'FrameworkControllerAzureStorageConfig' +] + + +@dataclass(init=False) +class _FrameworkControllerStorageConfig(ConfigBase): + storage: str + server: Optional[str] = None + path: Optional[str] = None + azure_account: Optional[str] = None + azure_share: Optional[str] = None + key_vault: Optional[str] = None + key_vault_secret: Optional[str] = None + +@dataclass(init=False) +class FrameworkControllerNfsConfig(ConfigBase): + storage: str = 'nfs' + server: str + path: str + +@dataclass(init=False) +class FrameworkControllerAzureStorageConfig(ConfigBase): + storage: str = 'azureStorage' + azure_account: str + azure_share: str + key_vault: str + key_vault_secret: str + + +@dataclass(init=False) +class FrameworkControllerRoleConfig(ConfigBase): + name: str + docker_image: str = 'msranni/nni:latest' + task_number: int + command: str + gpu_number: int + cpu_number: int + memory_size: str + attempt_completion_min_failed_tasks: int + attempt_completion_min_succeeded_tasks: int + + +@dataclass(init=False) +class FrameworkControllerConfig(TrainingServiceConfig): + platform: str = 'frameworkcontroller' + service_account_name: str + storage: _FrameworkControllerStorageConfig + task_roles: List[FrameworkControllerRoleConfig] + + def __init__(self, **kwargs): + kwargs = util.case_insensitive(kwargs) + kwargs['storage'] = util.load_config(_FrameworkControllerStorageConfig, kwargs.get('storage')) + kwargs['taskroles'] = util.load_config(FrameworkControllerRoleConfig, kwargs.get('taskroles')) + super().__init__(**kwargs) + + _validation_rules = { + 'platform': lambda value: (value == 'frameworkcontroller', 'cannot be modified') + } diff --git a/utils/third_party/nni_new/experiment/config/kubeflow.py b/utils/third_party/nni_new/experiment/config/kubeflow.py new file mode 100644 index 0000000000000000000000000000000000000000..aaa15085d4bf4d0970f0042eae33487908761812 --- /dev/null +++ b/utils/third_party/nni_new/experiment/config/kubeflow.py @@ -0,0 +1,68 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from dataclasses import dataclass +from typing import Optional + +from .base import ConfigBase +from .common import TrainingServiceConfig +from . import util + +__all__ = ['KubeflowConfig', 'KubeflowRoleConfig', 'KubeflowNfsConfig', 'KubeflowAzureStorageConfig'] + + +@dataclass(init=False) +class _KubeflowStorageConfig(ConfigBase): + storage: str + server: Optional[str] = None + path: Optional[str] = None + azure_account: Optional[str] = None + azure_share: Optional[str] = None + key_vault: Optional[str] = None + key_vault_secret: Optional[str] = None + +@dataclass(init=False) +class KubeflowNfsConfig(_KubeflowStorageConfig): + storage: str = 'nfs' + server: str + path: str + +@dataclass(init=False) +class KubeflowAzureStorageConfig(ConfigBase): + storage: str = 'azureStorage' + azure_account: str + azure_share: str + key_vault: str + key_vault_secret: str + + +@dataclass(init=False) +class KubeflowRoleConfig(ConfigBase): + replicas: int + command: str + gpu_number: int + cpu_number: int + memory_size: str + docker_image: str = 'msranni/nni:latest' + + +@dataclass(init=False) +class KubeflowConfig(TrainingServiceConfig): + platform: str = 'kubeflow' + operator: str + api_version: str + storage: _KubeflowStorageConfig + worker: KubeflowRoleConfig + parameter_server: Optional[KubeflowRoleConfig] = None + + def __init__(self, **kwargs): + kwargs = util.case_insensitive(kwargs) + kwargs['storage'] = util.load_config(_KubeflowStorageConfig, kwargs.get('storage')) + kwargs['worker'] = util.load_config(KubeflowRoleConfig, kwargs.get('worker')) + kwargs['parameterserver'] = util.load_config(KubeflowRoleConfig, kwargs.get('parameterserver')) + super().__init__(**kwargs) + + _validation_rules = { + 'platform': lambda value: (value == 'kubeflow', 'cannot be modified'), + 'operator': lambda value: value in ['tf-operator', 'pytorch-operator'] + } diff --git a/utils/third_party/nni_new/experiment/config/local.py b/utils/third_party/nni_new/experiment/config/local.py new file mode 100644 index 0000000000000000000000000000000000000000..90b92093fd69f50048cab85c20678be67d7b50aa --- /dev/null +++ b/utils/third_party/nni_new/experiment/config/local.py @@ -0,0 +1,27 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from dataclasses import dataclass +from typing import List, Optional, Union + +from .common import TrainingServiceConfig +from . import util + +__all__ = ['LocalConfig'] + +@dataclass(init=False) +class LocalConfig(TrainingServiceConfig): + platform: str = 'local' + use_active_gpu: Optional[bool] = None + max_trial_number_per_gpu: int = 1 + gpu_indices: Union[List[int], str, int, None] = None + + _canonical_rules = { + 'gpu_indices': util.canonical_gpu_indices + } + + _validation_rules = { + 'platform': lambda value: (value == 'local', 'cannot be modified'), + 'max_trial_number_per_gpu': lambda value: value > 0, + 'gpu_indices': lambda value: all(idx >= 0 for idx in value) and len(value) == len(set(value)) + } diff --git a/utils/third_party/nni_new/experiment/config/openpai.py b/utils/third_party/nni_new/experiment/config/openpai.py new file mode 100644 index 0000000000000000000000000000000000000000..66eecadac7cafdcb1540370a21b8b056f936e31f --- /dev/null +++ b/utils/third_party/nni_new/experiment/config/openpai.py @@ -0,0 +1,47 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from dataclasses import dataclass +from pathlib import Path, PurePosixPath +from typing import Any, Dict, Optional + +from .base import PathLike +from .common import TrainingServiceConfig +from . import util + +__all__ = ['OpenpaiConfig'] + +@dataclass(init=False) +class OpenpaiConfig(TrainingServiceConfig): + platform: str = 'openpai' + host: str + username: str + token: str + trial_cpu_number: int + trial_memory_size: str + storage_config_name: str + docker_image: str = 'msranni/nni:latest' + local_storage_mount_point: PathLike + container_storage_mount_point: str + reuse_mode: bool = True + + openpai_config: Optional[Dict[str, Any]] = None + openpai_config_file: Optional[PathLike] = None + + _canonical_rules = { + 'host': lambda value: 'https://' + value if '://' not in value else value, # type: ignore + 'local_storage_mount_point': util.canonical_path, + 'openpai_config_file': util.canonical_path + } + + _validation_rules = { + 'platform': lambda value: (value == 'openpai', 'cannot be modified'), + 'local_storage_mount_point': lambda value: Path(value).is_dir(), + 'container_storage_mount_point': lambda value: (PurePosixPath(value).is_absolute(), 'is not absolute'), + 'openpai_config_file': lambda value: Path(value).is_file() + } + + def validate(self) -> None: + super().validate() + if self.openpai_config is not None and self.openpai_config_file is not None: + raise ValueError('openpai_config and openpai_config_file can only be set one') diff --git a/utils/third_party/nni_new/experiment/config/remote.py b/utils/third_party/nni_new/experiment/config/remote.py new file mode 100644 index 0000000000000000000000000000000000000000..29c47fcaa8f0f48b1c62b6d7da0342b6ccbff7d9 --- /dev/null +++ b/utils/third_party/nni_new/experiment/config/remote.py @@ -0,0 +1,63 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from dataclasses import dataclass +from pathlib import Path +from typing import List, Optional, Union +import warnings + +from .base import ConfigBase, PathLike +from .common import TrainingServiceConfig +from . import util + +__all__ = ['RemoteConfig', 'RemoteMachineConfig'] + +@dataclass(init=False) +class RemoteMachineConfig(ConfigBase): + host: str + port: int = 22 + user: str + password: Optional[str] = None + ssh_key_file: PathLike = None #'~/.ssh/id_rsa' + ssh_passphrase: Optional[str] = None + use_active_gpu: bool = False + max_trial_number_per_gpu: int = 1 + gpu_indices: Union[List[int], str, int, None] = None + python_path: Optional[str] = None + + _canonical_rules = { + 'ssh_key_file': util.canonical_path, + 'gpu_indices': util.canonical_gpu_indices + } + + _validation_rules = { + 'port': lambda value: 0 < value < 65536, + 'max_trial_number_per_gpu': lambda value: value > 0, + 'gpu_indices': lambda value: all(idx >= 0 for idx in value) and len(value) == len(set(value)) + } + + def validate(self): + super().validate() + if self.password is None and not Path(self.ssh_key_file).is_file(): + raise ValueError(f'Password is not provided and cannot find SSH key file "{self.ssh_key_file}"') + if self.password: + warnings.warn('Password will be exposed through web UI in plain text. We recommend to use SSH key file.') + +@dataclass(init=False) +class RemoteConfig(TrainingServiceConfig): + platform: str = 'remote' + reuse_mode: bool = True + machine_list: List[RemoteMachineConfig] + + def __init__(self, **kwargs): + kwargs = util.case_insensitive(kwargs) + kwargs['machinelist'] = util.load_config(RemoteMachineConfig, kwargs.get('machinelist')) + super().__init__(**kwargs) + + _canonical_rules = { + 'machine_list': lambda value: [config.canonical() for config in value] + } + + _validation_rules = { + 'platform': lambda value: (value == 'remote', 'cannot be modified') + } diff --git a/utils/third_party/nni_new/experiment/config/shared_storage.py b/utils/third_party/nni_new/experiment/config/shared_storage.py new file mode 100644 index 0000000000000000000000000000000000000000..3d4d357764ce8741fcecd64e519b17fd15dd0ba1 --- /dev/null +++ b/utils/third_party/nni_new/experiment/config/shared_storage.py @@ -0,0 +1,23 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from dataclasses import dataclass +from typing import Optional + +from .common import SharedStorageConfig + +__all__ = ['NfsConfig', 'AzureBlobConfig'] + +@dataclass(init=False) +class NfsConfig(SharedStorageConfig): + storage_type: str = 'NFS' + nfs_server: str + exported_directory: str + +@dataclass(init=False) +class AzureBlobConfig(SharedStorageConfig): + storage_type: str = 'AzureBlob' + storage_account_name: str + storage_account_key: Optional[str] = None + resource_group_name: Optional[str] = None + container_name: str diff --git a/utils/third_party/nni_new/experiment/config/util.py b/utils/third_party/nni_new/experiment/config/util.py new file mode 100644 index 0000000000000000000000000000000000000000..62a56f6b00dc082a1b9636ad13d045eefd0680a1 --- /dev/null +++ b/utils/third_party/nni_new/experiment/config/util.py @@ -0,0 +1,101 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +Miscellaneous utility functions. +""" + +import importlib +import json +import math +import os.path +from pathlib import Path +from typing import Any, Dict, Optional, Union, List + +import nni.runtime.config + +PathLike = Union[Path, str] + +def case_insensitive(key_or_kwargs: Union[str, Dict[str, Any]]) -> Union[str, Dict[str, Any]]: + if isinstance(key_or_kwargs, str): + return key_or_kwargs.lower().replace('_', '') + else: + return {key.lower().replace('_', ''): value for key, value in key_or_kwargs.items()} + +def camel_case(key: str) -> str: + words = key.strip('_').split('_') + return words[0] + ''.join(word.title() for word in words[1:]) + +def canonical_path(path: Optional[PathLike]) -> Optional[str]: + # Path.resolve() does not work on Windows when file not exist, so use os.path instead + return os.path.abspath(os.path.expanduser(path)) if path is not None else None + +def count(*values) -> int: + return sum(value is not None and value is not False for value in values) + +def training_service_config_factory( + platform: Union[str, List[str]] = None, + config: Union[List, Dict] = None, + base_path: Optional[Path] = None): # -> TrainingServiceConfig + from .common import TrainingServiceConfig + + # import all custom config classes so they can be found in TrainingServiceConfig.__subclasses__() + custom_ts_config_path = nni.runtime.config.get_config_file('training_services.json') + custom_ts_config = json.load(custom_ts_config_path.open()) + for custom_ts_pkg in custom_ts_config.keys(): + pkg = importlib.import_module(custom_ts_pkg) + _config_class = pkg.nni_training_service_info.config_class + + ts_configs = [] + if platform is not None: + assert config is None + platforms = platform if isinstance(platform, list) else [platform] + for cls in TrainingServiceConfig.__subclasses__(): + if cls.platform in platforms: + ts_configs.append(cls()) + if len(ts_configs) < len(platforms): + bad = ', '.join(set(platforms) - set(ts_configs)) + raise RuntimeError(f'Bad training service platform: {bad}') + else: + assert config is not None + supported_platforms = {cls.platform: cls for cls in TrainingServiceConfig.__subclasses__()} + configs = config if isinstance(config, list) else [config] + for conf in configs: + if conf['platform'] not in supported_platforms: + raise RuntimeError(f'Unrecognized platform {conf["platform"]}') + ts_configs.append(supported_platforms[conf['platform']](_base_path=base_path, **conf)) + return ts_configs if len(ts_configs) > 1 else ts_configs[0] + +def load_config(Type, value): + if isinstance(value, list): + return [load_config(Type, item) for item in value] + if isinstance(value, dict): + return Type(**value) + return value + +def strip_optional(type_hint): + return type_hint.__args__[0] if str(type_hint).startswith('typing.Optional[') else type_hint + +def parse_time(time: str, target_unit: str = 's') -> int: + return _parse_unit(time.lower(), target_unit, _time_units) + +def parse_size(size: str, target_unit: str = 'mb') -> int: + return _parse_unit(size.lower(), target_unit, _size_units) + +_time_units = {'d': 24 * 3600, 'h': 3600, 'm': 60, 's': 1} +_size_units = {'gb': 1024 * 1024 * 1024, 'mb': 1024 * 1024, 'kb': 1024} + +def _parse_unit(string, target_unit, all_units): + for unit, factor in all_units.items(): + if string.endswith(unit): + number = string[:-len(unit)] + value = float(number) * factor + return math.ceil(value / all_units[target_unit]) + raise ValueError(f'Unsupported unit in "{string}"') + +def canonical_gpu_indices(indices: Union[List[int], str, int, None]) -> Optional[List[int]]: + if isinstance(indices, str): + return [int(idx) for idx in indices.split(',')] + if isinstance(indices, int): + return [indices] + return indices diff --git a/utils/third_party/nni_new/experiment/data.py b/utils/third_party/nni_new/experiment/data.py new file mode 100644 index 0000000000000000000000000000000000000000..d58f4671f6e7007f9967f1796ec12b212786e501 --- /dev/null +++ b/utils/third_party/nni_new/experiment/data.py @@ -0,0 +1,135 @@ +from dataclasses import dataclass +import json +from typing import List + + +@dataclass +class TrialResult: + """ + TrialResult stores the result information of a trial job. + + Attributes + ---------- + parameter: dict + Hyper parameters for this trial. + value: serializable object, usually a number, or a dict with key "default" and other extra keys + Final result. + trialJobId: str + Trial job id. + """ + parameter: dict + value: dict + trialJobId: str + + def __init__(self, parameter: dict, value: str, trialJobId: str): + self.parameter = parameter + self.value = json.loads(value) + self.trialJobId = trialJobId + + +@dataclass +class TrialMetricData: + """ + TrialMetricData stores the metric data of a trial job. + A trial job may have both intermediate metric and final metric. + + Attributes + ---------- + timestamp: int + Time stamp. + trialJobId: str + Trial job id. + parameterId: int + Parameter id. + type: str + Metric type, `PERIODICAL` for intermediate result and `FINAL` for final result. + sequence: int + Sequence number in this trial. + data: serializable object, usually a number, or a dict with key "default" and other extra keys + Metric data. + """ + timestamp: int + trialJobId: str + parameterId: int + type: str + sequence: int + data: dict + + def __init__(self, timestamp: int, trialJobId: str, parameterId: int, type: str, sequence: int, data: str): # pylint: disable=W0622 + self.timestamp = timestamp + self.trialJobId = trialJobId + self.parameterId = parameterId + self.type = type + self.sequence = sequence + self.data = json.loads(json.loads(data)) + + +@dataclass +class TrialHyperParameters: + """ + TrialHyperParameters stores the hyper parameters of a trial job. + + Attributes + ---------- + parameter_id: int + Parameter id. + parameter_source: str + Parameter source. + parameters: dict + Hyper parameters. + parameter_index: int + Parameter index. + """ + parameter_id: int + parameter_source: str + parameters: dict + parameter_index: int + + +@dataclass +class TrialJob: + """ + TrialJob stores the information of a trial job. + + Attributes + ---------- + trialJobId: str + Trial job id. + status: str + Job status. + hyperParameters: list of `nni.experiment.TrialHyperParameters` + See `nni.experiment.TrialHyperParameters`. + logPath: str + Log path. + startTime: int + Job start time (timestamp). + endTime: int + Job end time (timestamp). + finalMetricData: list of `nni.experiment.TrialMetricData` + See `nni.experiment.TrialMetricData`. + stderrPath: str + Stderr log path. + sequenceId: int + Sequence Id. + """ + trialJobId: str + status: str + hyperParameters: List[TrialHyperParameters] + logPath: str + startTime: int + endTime: int + finalMetricData: List[TrialMetricData] + stderrPath: str + sequenceId: int + + def __init__(self, trialJobId: str, status: str, logPath: str, startTime: int, sequenceId: int, + endTime: int = -1, stderrPath: str = '', hyperParameters: List = [], finalMetricData: List = []): + self.trialJobId = trialJobId + self.status = status + self.hyperParameters = [TrialHyperParameters(**json.loads(e)) for e in hyperParameters] + self.logPath = logPath + self.startTime = startTime + self.endTime = endTime + self.finalMetricData = [TrialMetricData(**e) for e in finalMetricData] + self.stderrPath = stderrPath + self.sequenceId = sequenceId diff --git a/utils/third_party/nni_new/experiment/experiment.py b/utils/third_party/nni_new/experiment/experiment.py new file mode 100644 index 0000000000000000000000000000000000000000..c7708f4f87a55e39fd55786bf5cbd81c36dfec1a --- /dev/null +++ b/utils/third_party/nni_new/experiment/experiment.py @@ -0,0 +1,454 @@ +import atexit +import logging +from pathlib import Path +import socket +from subprocess import Popen +import time +from typing import Optional, Union, List, overload, Any + +import json_tricks +import colorama +import psutil + +import nni.runtime.log + +from .config import ExperimentConfig, AlgorithmConfig +from .data import TrialJob, TrialMetricData, TrialResult +from . import launcher +from . import management +from . import rest +from ..tools.nnictl.command_utils import kill_command + +_logger = logging.getLogger('nni.experiment') + + +class Experiment: + """ + Create and stop an NNI experiment. + + Attributes + ---------- + config + Experiment configuration. + port + Web UI port of the experiment, or `None` if it is not running. + """ + + @overload + def __init__(self, config: ExperimentConfig) -> None: + """ + Prepare an experiment. + + Use `Experiment.run()` to launch it. + + Parameters + ---------- + config + Experiment configuration. + """ + ... + + @overload + def __init__(self, training_service: Union[str, List[str]]) -> None: + """ + Prepare an experiment, leaving configuration fields to be set later. + + Example usage:: + + experiment = Experiment('remote') + experiment.config.trial_command = 'python3 trial.py' + experiment.config.machines.append(RemoteMachineConfig(ip=..., user_name=...)) + ... + experiment.run(8080) + + Parameters + ---------- + training_service + Name of training service. + Supported value: "local", "remote", "openpai", "aml", "kubeflow", "frameworkcontroller", "adl" and hybrid training service. + """ + ... + + def __init__(self, config=None, training_service=None): + nni.runtime.log.init_logger_experiment() + + self.config: Optional[ExperimentConfig] = None + self.id: Optional[str] = None + self.port: Optional[int] = None + self._proc: Optional[Popen] = None + self.mode = 'new' + + args = [config, training_service] # deal with overloading + if isinstance(args[0], (str, list)): + self.config = ExperimentConfig(args[0]) + self.config.tuner = AlgorithmConfig(name='_none_', class_args={}) + self.config.assessor = AlgorithmConfig(name='_none_', class_args={}) + self.config.advisor = AlgorithmConfig(name='_none_', class_args={}) + else: + self.config = args[0] + + def start(self, port: int = 8080, debug: bool = False) -> None: + """ + Start the experiment in background. + + This method will raise exception on failure. + If it returns, the experiment should have been successfully started. + + Parameters + ---------- + port + The port of web UI. + debug + Whether to start in debug mode. + """ + atexit.register(self.stop) + + if self.mode == 'new': + self.id = management.generate_experiment_id() + else: + self.config = launcher.get_stopped_experiment_config(self.id, self.mode) + + if self.config.experiment_working_directory is not None: + log_dir = Path(self.config.experiment_working_directory, self.id, 'log') + else: + log_dir = Path.home() / f'nni-experiments/{self.id}/log' + nni.runtime.log.start_experiment_log(self.id, log_dir, debug) + + self._proc = launcher.start_experiment(self.id, self.config, port, debug, mode=self.mode) + assert self._proc is not None + + self.port = port # port will be None if start up failed + + ips = [self.config.nni_manager_ip] + for interfaces in psutil.net_if_addrs().values(): + for interface in interfaces: + if interface.family == socket.AF_INET: + ips.append(interface.address) + ips = [f'http://{ip}:{port}' for ip in ips if ip] + msg = 'Web UI URLs: ' + colorama.Fore.CYAN + ' '.join(ips) + colorama.Style.RESET_ALL + _logger.info(msg) + + def stop(self) -> None: + """ + Stop background experiment. + """ + _logger.info('Stopping experiment, please wait...') + atexit.unregister(self.stop) + + if self.id is not None: + nni.runtime.log.stop_experiment_log(self.id) + if self._proc is not None: + try: + rest.delete(self.port, '/experiment') + except Exception as e: + _logger.exception(e) + _logger.warning('Cannot gracefully stop experiment, killing NNI process...') + kill_command(self._proc.pid) + + self.id = None + self.port = None + self._proc = None + _logger.info('Experiment stopped') + + def run(self, port: int = 8080, wait_completion: bool = True, debug: bool = False) -> bool: + """ + Run the experiment. + + If wait_completion is True, this function will block until experiment finish or error. + + Return `True` when experiment done; or return `False` when experiment failed. + + Else if wait_completion is False, this function will non-block and return None immediately. + """ + self.start(port, debug) + if wait_completion: + try: + while True: + time.sleep(10) + status = self.get_status() + if status == 'DONE' or status == 'STOPPED': + return True + if status == 'ERROR': + return False + except KeyboardInterrupt: + _logger.warning('KeyboardInterrupt detected') + finally: + self.stop() + + @classmethod + def connect(cls, port: int): + """ + Connect to an existing experiment. + + Parameters + ---------- + port + The port of web UI. + """ + experiment = Experiment() + experiment.port = port + experiment.id = experiment.get_experiment_profile().get('id') + status = experiment.get_status() + pid = experiment.get_experiment_metadata(experiment.id).get('pid') + if pid is None: + _logger.warning('Get experiment pid failed, can not stop experiment by stop().') + else: + experiment._proc = psutil.Process(pid) + _logger.info('Connect to port %d success, experiment id is %s, status is %s.', port, experiment.id, status) + return experiment + + @classmethod + def resume(cls, experiment_id: str, port: int = 8080, wait_completion: bool = True, debug: bool = False): + """ + Resume a stopped experiment. + + Parameters + ---------- + experiment_id + The stopped experiment id. + port + The port of web UI. + wait_completion + If true, run in the foreground. If false, run in the background. + debug + Whether to start in debug mode. + """ + experiment = Experiment() + experiment.id = experiment_id + experiment.mode = 'resume' + experiment.run(port=port, wait_completion=wait_completion, debug=debug) + if not wait_completion: + return experiment + + @classmethod + def view(cls, experiment_id: str, port: int = 8080, non_blocking: bool = False): + """ + View a stopped experiment. + + Parameters + ---------- + experiment_id + The stopped experiment id. + port + The port of web UI. + non_blocking + If false, run in the foreground. If true, run in the background. + """ + debug = False + experiment = Experiment() + experiment.id = experiment_id + experiment.mode = 'view' + experiment.start(port=port, debug=debug) + if non_blocking: + return experiment + else: + try: + while True: + time.sleep(10) + except KeyboardInterrupt: + _logger.warning('KeyboardInterrupt detected') + finally: + experiment.stop() + + def get_status(self) -> str: + """ + Return experiment status as a str. + + Returns + ------- + str + Experiment status. + """ + resp = rest.get(self.port, '/check-status') + return resp['status'] + + def get_trial_job(self, trial_job_id: str): + """ + Return a trial job. + + Parameters + ---------- + trial_job_id: str + Trial job id. + + Returns + ------- + TrialJob + A `TrialJob` instance corresponding to `trial_job_id`. + """ + resp = rest.get(self.port, '/trial-jobs/{}'.format(trial_job_id)) + return TrialJob(**resp) + + def list_trial_jobs(self): + """ + Return information for all trial jobs as a list. + + Returns + ------- + list + List of `TrialJob`. + """ + resp = rest.get(self.port, '/trial-jobs') + return [TrialJob(**trial_job) for trial_job in resp] + + def get_job_statistics(self): + """ + Return trial job statistics information as a dict. + + Returns + ------- + dict + Job statistics information. + """ + resp = rest.get(self.port, '/job-statistics') + return resp + + def get_job_metrics(self, trial_job_id=None): + """ + Return trial job metrics. + + Parameters + ---------- + trial_job_id: str + trial job id. if this parameter is None, all trail jobs' metrics will be returned. + + Returns + ------- + dict + Each key is a trialJobId, the corresponding value is a list of `TrialMetricData`. + """ + api = '/metric-data/{}'.format(trial_job_id) if trial_job_id else '/metric-data' + resp = rest.get(self.port, api) + metric_dict = {} + for metric in resp: + trial_id = metric["trialJobId"] + if trial_id not in metric_dict: + metric_dict[trial_id] = [TrialMetricData(**metric)] + else: + metric_dict[trial_id].append(TrialMetricData(**metric)) + return metric_dict + + def get_experiment_profile(self): + """ + Return experiment profile as a dict. + + Returns + ------- + dict + The profile of the experiment. + """ + resp = rest.get(self.port, '/experiment') + return resp + + def get_experiment_metadata(self, exp_id: str): + """ + Return experiment metadata with specified exp_id as a dict. + + Returns + ------- + dict + The specified experiment metadata. + """ + experiments_metadata = self.get_all_experiments_metadata() + for metadata in experiments_metadata: + if metadata['id'] == exp_id: + return metadata + return {} + + def get_all_experiments_metadata(self): + """ + Return all experiments metadata as a list. + + Returns + ------- + list + The experiments metadata. + """ + resp = rest.get(self.port, '/experiments-info') + return resp + + def export_data(self): + """ + Return exported information for all trial jobs. + + Returns + ------- + list + List of `TrialResult`. + """ + resp = rest.get(self.port, '/export-data') + return [TrialResult(**trial_result) for trial_result in resp] + + def _get_query_type(self, key: str): + if key == 'trialConcurrency': + return '?update_type=TRIAL_CONCURRENCY' + if key == 'maxExecDuration': + return '?update_type=MAX_EXEC_DURATION' + if key == 'searchSpace': + return '?update_type=SEARCH_SPACE' + if key == 'maxTrialNum': + return '?update_type=MAX_TRIAL_NUM' + + def _update_experiment_profile(self, key: str, value: Any): + """ + Update an experiment's profile + + Parameters + ---------- + key: str + One of `['trial_concurrency', 'max_experiment_duration', 'search_space', 'max_trial_number']`. + value: Any + New value of the key. + """ + api = '/experiment{}'.format(self._get_query_type(key)) + experiment_profile = self.get_experiment_profile() + experiment_profile['params'][key] = value + rest.put(self.port, api, experiment_profile) + logging.info('Successfully update %s.', key) + + def update_trial_concurrency(self, value: int): + """ + Update an experiment's trial_concurrency + + Parameters + ---------- + value: int + New trial_concurrency value. + """ + self._update_experiment_profile('trialConcurrency', value) + + def update_max_experiment_duration(self, value: str): + """ + Update an experiment's max_experiment_duration + + Parameters + ---------- + value: str + Strings like '1m' for one minute or '2h' for two hours. + SUFFIX may be 's' for seconds, 'm' for minutes, 'h' for hours or 'd' for days. + """ + self._update_experiment_profile('maxExecDuration', value) + + def update_search_space(self, value: dict): + """ + Update the experiment's search_space. + TODO: support searchspace file. + + Parameters + ---------- + value: dict + New search_space. + """ + value = json_tricks.dumps(value) + self._update_experiment_profile('searchSpace', value) + + def update_max_trial_number(self, value: int): + """ + Update an experiment's max_trial_number + + Parameters + ---------- + value: int + New max_trial_number value. + """ + self._update_experiment_profile('maxTrialNum', value) diff --git a/utils/third_party/nni_new/experiment/launcher.py b/utils/third_party/nni_new/experiment/launcher.py new file mode 100644 index 0000000000000000000000000000000000000000..7ee9e43242b70e8bef6c86ef4dee87bd52b09064 --- /dev/null +++ b/utils/third_party/nni_new/experiment/launcher.py @@ -0,0 +1,177 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import contextlib +import logging +from pathlib import Path +import socket +from subprocess import Popen +import sys +import time +from typing import Optional, Tuple + +import colorama + +import nni_node # pylint: disable=import-error +import nni.runtime.protocol + +from .config import ExperimentConfig +from .pipe import Pipe +from . import rest +from ..tools.nnictl.config_utils import Experiments, Config +from ..tools.nnictl.nnictl_utils import update_experiment + +_logger = logging.getLogger('nni.experiment') + + +def start_experiment(exp_id: str, config: ExperimentConfig, port: int, debug: bool, mode: str = 'new') -> Popen: + proc = None + + config.validate(initialized_tuner=False) + _ensure_port_idle(port) + + if mode != 'view': + if isinstance(config.training_service, list): # hybrid training service + _ensure_port_idle(port + 1, 'Hybrid training service requires an additional port') + elif config.training_service.platform in ['remote', 'openpai', 'kubeflow', 'frameworkcontroller', 'adl']: + _ensure_port_idle(port + 1, f'{config.training_service.platform} requires an additional port') + + try: + _logger.info('Creating experiment, Experiment ID: %s', colorama.Fore.CYAN + exp_id + colorama.Style.RESET_ALL) + start_time, proc = _start_rest_server(config, port, debug, exp_id, mode=mode) + _logger.info('Starting web server...') + _check_rest_server(port) + platform = 'hybrid' if isinstance(config.training_service, list) else config.training_service.platform + _save_experiment_information(exp_id, port, start_time, platform, + config.experiment_name, proc.pid, str(config.experiment_working_directory)) + _logger.info('Setting up...') + rest.post(port, '/experiment', config.json()) + return proc + + except Exception as e: + _logger.error('Create experiment failed') + if proc is not None: + with contextlib.suppress(Exception): + proc.kill() + raise e + +def start_experiment_retiarii(exp_id: str, config: ExperimentConfig, port: int, debug: bool) -> Popen: + pipe = None + proc = None + + config.validate(initialized_tuner=True) + _ensure_port_idle(port) + if isinstance(config.training_service, list): # hybrid training service + _ensure_port_idle(port + 1, 'Hybrid training service requires an additional port') + elif config.training_service.platform in ['remote', 'openpai', 'kubeflow', 'frameworkcontroller', 'adl']: + _ensure_port_idle(port + 1, f'{config.training_service.platform} requires an additional port') + + try: + _logger.info('Creating experiment, Experiment ID: %s', colorama.Fore.CYAN + exp_id + colorama.Style.RESET_ALL) + pipe = Pipe(exp_id) + start_time, proc = _start_rest_server(config, port, debug, exp_id, pipe.path) + _logger.info('Connecting IPC pipe...') + pipe_file = pipe.connect() + nni.runtime.protocol._in_file = pipe_file + nni.runtime.protocol._out_file = pipe_file + _logger.info('Starting web server...') + _check_rest_server(port) + platform = 'hybrid' if isinstance(config.training_service, list) else config.training_service.platform + _save_experiment_information(exp_id, port, start_time, platform, + config.experiment_name, proc.pid, config.experiment_working_directory) + _logger.info('Setting up...') + rest.post(port, '/experiment', config.json()) + return proc, pipe + + except Exception as e: + _logger.error('Create experiment failed') + if proc is not None: + with contextlib.suppress(Exception): + proc.kill() + if pipe is not None: + with contextlib.suppress(Exception): + pipe.close() + raise e + +def _ensure_port_idle(port: int, message: Optional[str] = None) -> None: + sock = socket.socket() + if sock.connect_ex(('localhost', port)) == 0: + sock.close() + message = f'(message)' if message else '' + raise RuntimeError(f'Port {port} is not idle {message}') + + +def _start_rest_server(config: ExperimentConfig, port: int, debug: bool, experiment_id: str, pipe_path: str = None, + mode: str = 'new') -> Tuple[int, Popen]: + if isinstance(config.training_service, list): + ts = 'hybrid' + else: + ts = config.training_service.platform + if ts == 'openpai': + ts = 'pai' + + args = { + 'port': port, + 'mode': ts, + 'experiment_id': experiment_id, + 'start_mode': mode, + 'log_dir': config.experiment_working_directory, + 'log_level': 'debug' if debug else 'info' + } + if pipe_path is not None: + args['dispatcher_pipe'] = pipe_path + + if mode == 'view': + args['start_mode'] = 'resume' + args['readonly'] = 'true' + + node_dir = Path(nni_node.__path__[0]) + node = str(node_dir / ('node.exe' if sys.platform == 'win32' else 'node')) + main_js = str(node_dir / 'main.js') + cmd = [node, '--max-old-space-size=4096', main_js] + for arg_key, arg_value in args.items(): + cmd.append('--' + arg_key) + cmd.append(str(arg_value)) + + if sys.platform == 'win32': + from subprocess import CREATE_NEW_PROCESS_GROUP + proc = Popen(cmd, cwd=node_dir, creationflags=CREATE_NEW_PROCESS_GROUP) + else: + if pipe_path is None: + import os + proc = Popen(cmd, cwd=node_dir, preexec_fn=os.setpgrp) + else: + proc = Popen(cmd, cwd=node_dir) + return int(time.time() * 1000), proc + + +def _check_rest_server(port: int, retry: int = 3) -> None: + for i in range(retry): + with contextlib.suppress(Exception): + rest.get(port, '/check-status') + return + if i > 0: + _logger.warning('Timeout, retry...') + time.sleep(1) + rest.get(port, '/check-status') + + +def _save_experiment_information(experiment_id: str, port: int, start_time: int, platform: str, name: str, pid: int, logDir: str) -> None: + experiments_config = Experiments() + experiments_config.add_experiment(experiment_id, port, start_time, platform, name, pid=pid, logDir=logDir) + + +def get_stopped_experiment_config(exp_id: str, mode: str) -> None: + update_experiment() + experiments_config = Experiments() + experiments_dict = experiments_config.get_all_experiments() + experiment_metadata = experiments_dict.get(exp_id) + if experiment_metadata is None: + _logger.error('Id %s not exist!', exp_id) + return + if experiment_metadata['status'] != 'STOPPED': + _logger.error('Only stopped experiments can be %sed!', mode) + return + experiment_config = Config(exp_id, experiment_metadata['logDir']).get_config() + config = ExperimentConfig(**experiment_config) + return config diff --git a/utils/third_party/nni_new/experiment/management.py b/utils/third_party/nni_new/experiment/management.py new file mode 100644 index 0000000000000000000000000000000000000000..b15c4d6d2561cc678574fc197a95d1d1d1ffafe3 --- /dev/null +++ b/utils/third_party/nni_new/experiment/management.py @@ -0,0 +1,16 @@ +from pathlib import Path +import random +import string + + +def generate_experiment_id() -> str: + return ''.join(random.sample(string.ascii_lowercase + string.digits, 8)) + + +def create_experiment_directory(experiment_id: str) -> Path: + path = Path.home() / 'nni-experiments' / experiment_id + path.mkdir(parents=True, exist_ok=True) + return path + + +# TODO: port shangning's work here, and use it in Experiment.start()/.stop() diff --git a/utils/third_party/nni_new/experiment/pipe.py b/utils/third_party/nni_new/experiment/pipe.py new file mode 100644 index 0000000000000000000000000000000000000000..e59fd8270bd74b1735cde9bae2983875b85f74e0 --- /dev/null +++ b/utils/third_party/nni_new/experiment/pipe.py @@ -0,0 +1,72 @@ +from io import BufferedIOBase +import logging +import os +import sys + +_logger = logging.getLogger(__name__) + +if sys.platform == 'win32': + import _winapi + import msvcrt + + class WindowsPipe: + def __init__(self, experiment_id: str): + self.path: str = r'\\.\pipe\nni-' + experiment_id + self.file = None + + self._handle = _winapi.CreateNamedPipe( + self.path, + _winapi.PIPE_ACCESS_DUPLEX, + _winapi.PIPE_TYPE_MESSAGE | _winapi.PIPE_READMODE_MESSAGE | _winapi.PIPE_WAIT, + 1, + 8192, + 8192, + 0, + _winapi.NULL + ) + + def connect(self) -> BufferedIOBase: + _winapi.ConnectNamedPipe(self._handle, _winapi.NULL) + fd = msvcrt.open_osfhandle(self._handle, 0) + self.file = os.fdopen(fd, 'w+b') + return self.file + + def close(self) -> None: + try: + if self.file is not None: + self.file.close() + except Exception as e: + _logger.debug('Error on closing Windows pipe: %s', e) + + Pipe = WindowsPipe + + +else: + import socket + + from . import management + + class UnixPipe: + def __init__(self, experiment_id: str): + self.path: str = str(management.create_experiment_directory(experiment_id) / 'dispatcher-pipe') + self.file = None + + self._socket = socket.socket(socket.AF_UNIX) + self._socket.bind(self.path) + self._socket.listen(1) # only accepts one connection + + def connect(self) -> BufferedIOBase: + conn, _ = self._socket.accept() + self.file = conn.makefile('rwb') + return self.file + + def close(self) -> None: + try: + if self.file is not None: + self.file.close() + self._socket.close() + os.unlink(self.path) + except Exception as e: + _logger.debug('Error on closing POSIX pipe: %s', e) + + Pipe = UnixPipe diff --git a/utils/third_party/nni_new/experiment/rest.py b/utils/third_party/nni_new/experiment/rest.py new file mode 100644 index 0000000000000000000000000000000000000000..bdacc7c215ac759fdb551e7d4fa3d6e296e45fd1 --- /dev/null +++ b/utils/third_party/nni_new/experiment/rest.py @@ -0,0 +1,35 @@ +import logging +from typing import Any, Optional + +import requests + +_logger = logging.getLogger(__name__) + +url_template = 'http://localhost:{}/api/v1/nni{}' +timeout = 20 + +def request(method: str, port: Optional[int], api: str, data: Any = None) -> Any: + if port is None: + raise RuntimeError('Experiment is not running') + url = url_template.format(port, api) + if data is None: + resp = requests.request(method, url, timeout=timeout) + else: + resp = requests.request(method, url, json=data, timeout=timeout) + if not resp.ok: + _logger.error('rest request %s %s failed: %s %s', method.upper(), url, resp.status_code, resp.text) + resp.raise_for_status() + if method.lower() in ['get', 'post'] and len(resp.content) > 0: + return resp.json() + +def get(port: Optional[int], api: str) -> Any: + return request('get', port, api) + +def post(port: Optional[int], api: str, data: Any) -> Any: + return request('post', port, api, data) + +def put(port: Optional[int], api: str, data: Any) -> None: + request('put', port, api, data) + +def delete(port: Optional[int], api: str) -> None: + request('delete', port, api) diff --git a/utils/third_party/nni_new/feature_engineering/__init__.py b/utils/third_party/nni_new/feature_engineering/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..8b137891791fe96927ad78e64b0aad7bded08bdc --- /dev/null +++ b/utils/third_party/nni_new/feature_engineering/__init__.py @@ -0,0 +1 @@ + diff --git a/utils/third_party/nni_new/feature_engineering/feature_selector.py b/utils/third_party/nni_new/feature_engineering/feature_selector.py new file mode 100644 index 0000000000000000000000000000000000000000..32021bfb29931f459011edd4ceff3b5e2f899c99 --- /dev/null +++ b/utils/third_party/nni_new/feature_engineering/feature_selector.py @@ -0,0 +1,59 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# MIT License +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and +# associated documentation files (the "Software"), to deal in the Software without restriction, +# including without limitation the rights to use, copy, modify, merge, publish, distribute, +# sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or +# substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT +# NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT +# OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# ================================================================================================== + +import logging + +_logger = logging.getLogger(__name__) + + +class FeatureSelector(): + + def __init__(self, **kwargs): + self.selected_features_ = None + self.X = None + self.y = None + + + def fit(self, X, y, **kwargs): + """ + Fit the training data to FeatureSelector + + Paramters + --------- + X : array-like numpy matrix + The training input samples, which shape is [n_samples, n_features]. + y: array-like numpy matrix + The target values (class labels in classification, real numbers in + regression). Which shape is [n_samples]. + """ + self.X = X + self.y = y + + + def get_selected_features(self): + """ + Fit the training data to FeatureSelector + + Returns + ------- + list : + Return the index of imprtant feature. + """ + return self.selected_features_ diff --git a/utils/third_party/nni_new/nas/__init__.py b/utils/third_party/nni_new/nas/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/utils/third_party/nni_new/nas/benchmarks/__init__.py b/utils/third_party/nni_new/nas/benchmarks/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/utils/third_party/nni_new/nas/benchmarks/constants.py b/utils/third_party/nni_new/nas/benchmarks/constants.py new file mode 100644 index 0000000000000000000000000000000000000000..4a52fa8d8fce3d9a70f2ca38299a6667c35d9e55 --- /dev/null +++ b/utils/third_party/nni_new/nas/benchmarks/constants.py @@ -0,0 +1,4 @@ +import os + + +DATABASE_DIR = os.environ.get("NASBENCHMARK_DIR", os.path.expanduser("~/.nni/nasbenchmark")) diff --git a/utils/third_party/nni_new/nas/benchmarks/nasbench101/__init__.py b/utils/third_party/nni_new/nas/benchmarks/nasbench101/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e9aad0552cb9520d82685e28b0269ae85c54595e --- /dev/null +++ b/utils/third_party/nni_new/nas/benchmarks/nasbench101/__init__.py @@ -0,0 +1,3 @@ +from .constants import INPUT, OUTPUT, CONV3X3_BN_RELU, CONV1X1_BN_RELU, MAXPOOL3X3 +from .model import Nb101TrialStats, Nb101IntermediateStats, Nb101TrialConfig +from .query import query_nb101_trial_stats diff --git a/utils/third_party/nni_new/nas/benchmarks/nasbench101/constants.py b/utils/third_party/nni_new/nas/benchmarks/nasbench101/constants.py new file mode 100644 index 0000000000000000000000000000000000000000..c2769f497f4dea6f0b5e1541d2413a8475866f15 --- /dev/null +++ b/utils/third_party/nni_new/nas/benchmarks/nasbench101/constants.py @@ -0,0 +1,14 @@ +INPUT = 'input' +OUTPUT = 'output' +CONV3X3_BN_RELU = 'conv3x3-bn-relu' +CONV1X1_BN_RELU = 'conv1x1-bn-relu' +MAXPOOL3X3 = 'maxpool3x3' + + +LABEL2ID = { + INPUT: -1, + OUTPUT: -2, + CONV3X3_BN_RELU: 0, + CONV1X1_BN_RELU: 1, + MAXPOOL3X3: 2 +} diff --git a/utils/third_party/nni_new/nas/benchmarks/nasbench101/db_gen.py b/utils/third_party/nni_new/nas/benchmarks/nasbench101/db_gen.py new file mode 100644 index 0000000000000000000000000000000000000000..04212ca270228fca0e1b07e728f69f229ff0893e --- /dev/null +++ b/utils/third_party/nni_new/nas/benchmarks/nasbench101/db_gen.py @@ -0,0 +1,53 @@ +import argparse + +from tqdm import tqdm +from nasbench import api # pylint: disable=import-error + +from .model import db, Nb101TrialConfig, Nb101TrialStats, Nb101IntermediateStats +from .graph_util import nasbench_format_to_architecture_repr, hash_module + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('input_file', + help='Path to the file to be converted, e.g., nasbench_full.tfrecord') + args = parser.parse_args() + nasbench = api.NASBench(args.input_file) + with db: + db.create_tables([Nb101TrialConfig, Nb101TrialStats, Nb101IntermediateStats]) + for hashval in tqdm(nasbench.hash_iterator(), desc='Dumping data into database'): + metadata, metrics = nasbench.get_metrics_from_hash(hashval) + num_vertices, architecture = nasbench_format_to_architecture_repr( + metadata['module_adjacency'], metadata['module_operations']) + assert hashval == hash_module(architecture, num_vertices) + for epochs in [4, 12, 36, 108]: + trial_config = Nb101TrialConfig.create( + arch=architecture, + num_vertices=num_vertices, + hash=hashval, + num_epochs=epochs + ) + + for seed in range(3): + cur = metrics[epochs][seed] + trial = Nb101TrialStats.create( + config=trial_config, + train_acc=cur['final_train_accuracy'] * 100, + valid_acc=cur['final_validation_accuracy'] * 100, + test_acc=cur['final_test_accuracy'] * 100, + parameters=metadata['trainable_parameters'] / 1e6, + training_time=cur['final_training_time'] * 60 + ) + for t in ['halfway', 'final']: + Nb101IntermediateStats.create( + trial=trial, + current_epoch=epochs // 2 if t == 'halfway' else epochs, + training_time=cur[t + '_training_time'], + train_acc=cur[t + '_train_accuracy'] * 100, + valid_acc=cur[t + '_validation_accuracy'] * 100, + test_acc=cur[t + '_test_accuracy'] * 100 + ) + + +if __name__ == '__main__': + main() diff --git a/utils/third_party/nni_new/nas/benchmarks/nasbench101/graph_util.py b/utils/third_party/nni_new/nas/benchmarks/nasbench101/graph_util.py new file mode 100644 index 0000000000000000000000000000000000000000..10805685fec3ff7359ec39dc0ae1c019e67950ae --- /dev/null +++ b/utils/third_party/nni_new/nas/benchmarks/nasbench101/graph_util.py @@ -0,0 +1,111 @@ +import hashlib + +import numpy as np + +from .constants import INPUT, LABEL2ID, OUTPUT + + +def _labeling_from_architecture(architecture, vertices): + return [INPUT] + [architecture['op{}'.format(i)] for i in range(1, vertices - 1)] + [OUTPUT] + + +def _adjancency_matrix_from_architecture(architecture, vertices): + matrix = np.zeros((vertices, vertices), dtype=np.bool) + for i in range(1, vertices): + for k in architecture['input{}'.format(i)]: + matrix[k, i] = 1 + return matrix + + +def nasbench_format_to_architecture_repr(adjacency_matrix, labeling): + """ + Computes a graph-invariance MD5 hash of the matrix and label pair. + Imported from NAS-Bench-101 repo. + + Parameters + ---------- + adjacency_matrix : np.ndarray + A 2D array of shape NxN, where N is the number of vertices. + ``matrix[u][v]`` is 1 if there is a direct edge from `u` to `v`, + otherwise it will be 0. + labeling : list of str + A list of str that starts with input and ends with output. The intermediate + nodes are chosen from candidate operators. + + Returns + ------- + tuple and int and dict + Converted number of vertices and architecture. + """ + num_vertices = adjacency_matrix.shape[0] + assert len(labeling) == num_vertices + architecture = {} + for i in range(1, num_vertices - 1): + architecture['op{}'.format(i)] = labeling[i] + assert labeling[i] not in [INPUT, OUTPUT] + for i in range(1, num_vertices): + architecture['input{}'.format(i)] = [k for k in range(i) if adjacency_matrix[k, i]] + return num_vertices, architecture + + +def infer_num_vertices(architecture): + """ + Infer number of vertices from an architecture dict. + + Parameters + ---------- + architecture : dict + Architecture in NNI format. + + Returns + ------- + int + Number of vertices. + """ + op_keys = set([k for k in architecture.keys() if k.startswith('op')]) + intermediate_vertices = len(op_keys) + assert op_keys == {'op{}'.format(i) for i in range(1, intermediate_vertices + 1)} + return intermediate_vertices + 2 + + +def hash_module(architecture, vertices): + """ + Computes a graph-invariance MD5 hash of the matrix and label pair. + This snippet is modified from code in NAS-Bench-101 repo. + + Parameters + ---------- + matrix : np.ndarray + Square upper-triangular adjacency matrix. + labeling : list of int + Labels of length equal to both dimensions of matrix. + + Returns + ------- + str + MD5 hash of the matrix and labeling. + """ + labeling = _labeling_from_architecture(architecture, vertices) + labeling = [LABEL2ID[t] for t in labeling] + matrix = _adjancency_matrix_from_architecture(architecture, vertices) + in_edges = np.sum(matrix, axis=0).tolist() + out_edges = np.sum(matrix, axis=1).tolist() + + assert len(in_edges) == len(out_edges) == len(labeling) + hashes = list(zip(out_edges, in_edges, labeling)) + hashes = [hashlib.md5(str(h).encode('utf-8')).hexdigest() for h in hashes] + # Computing this up to the diameter is probably sufficient but since the + # operation is fast, it is okay to repeat more times. + for _ in range(vertices): + new_hashes = [] + for v in range(vertices): + in_neighbors = [hashes[w] for w in range(vertices) if matrix[w, v]] + out_neighbors = [hashes[w] for w in range(vertices) if matrix[v, w]] + new_hashes.append(hashlib.md5( + (''.join(sorted(in_neighbors)) + '|' + + ''.join(sorted(out_neighbors)) + '|' + + hashes[v]).encode('utf-8')).hexdigest()) + hashes = new_hashes + fingerprint = hashlib.md5(str(sorted(hashes)).encode('utf-8')).hexdigest() + + return fingerprint diff --git a/utils/third_party/nni_new/nas/benchmarks/nasbench101/model.py b/utils/third_party/nni_new/nas/benchmarks/nasbench101/model.py new file mode 100644 index 0000000000000000000000000000000000000000..44ec3f874f5ba58df03b7862bce9fc127a8094aa --- /dev/null +++ b/utils/third_party/nni_new/nas/benchmarks/nasbench101/model.py @@ -0,0 +1,102 @@ +import os + +from peewee import CharField, FloatField, ForeignKeyField, IntegerField, Model +from playhouse.sqlite_ext import JSONField, SqliteExtDatabase + +from nni.nas.benchmarks.constants import DATABASE_DIR +from nni.nas.benchmarks.utils import json_dumps + +db = SqliteExtDatabase(os.path.join(DATABASE_DIR, 'nasbench101.db'), autoconnect=True) + + +class Nb101TrialConfig(Model): + """ + Trial config for NAS-Bench-101. + + Attributes + ---------- + arch : dict + A dict with keys ``op1``, ``op2``, ... and ``input1``, ``input2``, ... Vertices are + enumerate from 0. Since node 0 is input node, it is skipped in this dict. Each ``op`` + is one of :const:`nni.nas.benchmark.nasbench101.CONV3X3_BN_RELU`, + :const:`nni.nas.benchmark.nasbench101.CONV1X1_BN_RELU`, and :const:`nni.nas.benchmark.nasbench101.MAXPOOL3X3`. + Each ``input`` is a list of previous nodes. For example ``input5`` can be ``[0, 1, 3]``. + num_vertices : int + Number of vertices (nodes) in one cell. Should be less than or equal to 7 in default setup. + hash : str + Graph-invariant MD5 string for this architecture. + num_epochs : int + Number of epochs planned for this trial. Should be one of 4, 12, 36, 108 in default setup. + """ + + arch = JSONField(json_dumps=json_dumps, index=True) + num_vertices = IntegerField(index=True) + hash = CharField(max_length=64, index=True) + num_epochs = IntegerField(index=True) + + class Meta: + database = db + + +class Nb101TrialStats(Model): + """ + Computation statistics for NAS-Bench-101. Each corresponds to one trial. + Each config has multiple trials with different random seeds, but unfortunately seed for each trial is unavailable. + NAS-Bench-101 trains and evaluates on CIFAR-10 by default. The original training set is divided into + 40k training images and 10k validation images, and the original validation set is used for test only. + + Attributes + ---------- + config : Nb101TrialConfig + Setup for this trial data. + train_acc : float + Final accuracy on training data, ranging from 0 to 100. + valid_acc : float + Final accuracy on validation data, ranging from 0 to 100. + test_acc : float + Final accuracy on test data, ranging from 0 to 100. + parameters : float + Number of trainable parameters in million. + training_time : float + Duration of training in seconds. + """ + config = ForeignKeyField(Nb101TrialConfig, backref='trial_stats', index=True) + train_acc = FloatField() + valid_acc = FloatField() + test_acc = FloatField() + parameters = FloatField() + training_time = FloatField() + + class Meta: + database = db + + +class Nb101IntermediateStats(Model): + """ + Intermediate statistics for NAS-Bench-101. + + Attributes + ---------- + trial : Nb101TrialStats + The exact trial where the intermediate result is produced. + current_epoch : int + Elapsed epochs when evaluation is done. + train_acc : float + Intermediate accuracy on training data, ranging from 0 to 100. + valid_acc : float + Intermediate accuracy on validation data, ranging from 0 to 100. + test_acc : float + Intermediate accuracy on test data, ranging from 0 to 100. + training_time : float + Time elapsed in seconds. + """ + + trial = ForeignKeyField(Nb101TrialStats, backref='intermediates', index=True) + current_epoch = IntegerField(index=True) + train_acc = FloatField() + valid_acc = FloatField() + test_acc = FloatField() + training_time = FloatField() + + class Meta: + database = db diff --git a/utils/third_party/nni_new/nas/benchmarks/nasbench101/query.py b/utils/third_party/nni_new/nas/benchmarks/nasbench101/query.py new file mode 100644 index 0000000000000000000000000000000000000000..bd43d722f89a0e555cfd8b1843c079c9cf1f4873 --- /dev/null +++ b/utils/third_party/nni_new/nas/benchmarks/nasbench101/query.py @@ -0,0 +1,70 @@ +import functools + +from peewee import fn +from playhouse.shortcuts import model_to_dict +from .model import Nb101TrialStats, Nb101TrialConfig +from .graph_util import hash_module, infer_num_vertices + + +def query_nb101_trial_stats(arch, num_epochs, isomorphism=True, reduction=None, include_intermediates=False): + """ + Query trial stats of NAS-Bench-101 given conditions. + + Parameters + ---------- + arch : dict or None + If a dict, it is in the format that is described in + :class:`nni.nas.benchmark.nasbench101.Nb101TrialConfig`. Only trial stats + matched will be returned. If none, all architectures in the database will be matched. + num_epochs : int or None + If int, matching results will be returned. Otherwise a wildcard. + isomorphism : boolean + Whether to match essentially-same architecture, i.e., architecture with the + same graph-invariant hash value. + reduction : str or None + If 'none' or None, all trial stats will be returned directly. + If 'mean', fields in trial stats will be averaged given the same trial config. + include_intermediates : boolean + If true, intermediate results will be returned. + + Returns + ------- + generator of dict + A generator of :class:`nni.nas.benchmark.nasbench101.Nb101TrialStats` objects, + where each of them has been converted into a dict. + """ + fields = [] + if reduction == 'none': + reduction = None + if reduction == 'mean': + for field_name in Nb101TrialStats._meta.sorted_field_names: + if field_name not in ['id', 'config']: + fields.append(fn.AVG(getattr(Nb101TrialStats, field_name)).alias(field_name)) + elif reduction is None: + fields.append(Nb101TrialStats) + else: + raise ValueError('Unsupported reduction: \'%s\'' % reduction) + query = Nb101TrialStats.select(*fields, Nb101TrialConfig).join(Nb101TrialConfig) + conditions = [] + if arch is not None: + if isomorphism: + num_vertices = infer_num_vertices(arch) + conditions.append(Nb101TrialConfig.hash == hash_module(arch, num_vertices)) + else: + conditions.append(Nb101TrialConfig.arch == arch) + if num_epochs is not None: + conditions.append(Nb101TrialConfig.num_epochs == num_epochs) + if conditions: + query = query.where(functools.reduce(lambda a, b: a & b, conditions)) + if reduction is not None: + query = query.group_by(Nb101TrialStats.config) + for trial in query: + if include_intermediates: + data = model_to_dict(trial) + # exclude 'trial' from intermediates as it is already available in data + data['intermediates'] = [ + {k: v for k, v in model_to_dict(t).items() if k != 'trial'} for t in trial.intermediates + ] + yield data + else: + yield model_to_dict(trial) diff --git a/utils/third_party/nni_new/nas/benchmarks/nasbench201/__init__.py b/utils/third_party/nni_new/nas/benchmarks/nasbench201/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..3ce3d0170d29d2e6c0fbdfdcff17077be72dd420 --- /dev/null +++ b/utils/third_party/nni_new/nas/benchmarks/nasbench201/__init__.py @@ -0,0 +1,3 @@ +from .constants import NONE, SKIP_CONNECT, CONV_1X1, CONV_3X3, AVG_POOL_3X3 +from .model import Nb201TrialStats, Nb201IntermediateStats, Nb201TrialConfig +from .query import query_nb201_trial_stats diff --git a/utils/third_party/nni_new/nas/benchmarks/nasbench201/constants.py b/utils/third_party/nni_new/nas/benchmarks/nasbench201/constants.py new file mode 100644 index 0000000000000000000000000000000000000000..c9d6bce5d112f3d84ae0dd17a780e4a3e83758bd --- /dev/null +++ b/utils/third_party/nni_new/nas/benchmarks/nasbench201/constants.py @@ -0,0 +1,12 @@ +NONE = 'none' +SKIP_CONNECT = 'skip_connect' +CONV_1X1 = 'conv_1x1' +CONV_3X3 = 'conv_3x3' +AVG_POOL_3X3 = 'avg_pool_3x3' +PRIMITIVES = [ + NONE, + SKIP_CONNECT, + CONV_1X1, + CONV_3X3, + AVG_POOL_3X3, +] diff --git a/utils/third_party/nni_new/nas/benchmarks/nasbench201/db_gen.py b/utils/third_party/nni_new/nas/benchmarks/nasbench201/db_gen.py new file mode 100644 index 0000000000000000000000000000000000000000..e2202612b4a76ce7bafd063c62faba7859f18120 --- /dev/null +++ b/utils/third_party/nni_new/nas/benchmarks/nasbench201/db_gen.py @@ -0,0 +1,106 @@ +import argparse +import re + +import tqdm +import torch + +from .constants import NONE, SKIP_CONNECT, CONV_1X1, CONV_3X3, AVG_POOL_3X3 +from .model import db, Nb201TrialConfig, Nb201TrialStats, Nb201IntermediateStats + + +def parse_arch_str(arch_str): + mp = { + 'none': NONE, + 'skip_connect': SKIP_CONNECT, + 'nor_conv_1x1': CONV_1X1, + 'nor_conv_3x3': CONV_3X3, + 'avg_pool_3x3': AVG_POOL_3X3 + } + m = re.match(r'\|(.*)~0\|\+\|(.*)~0\|(.*)~1\|\+\|(.*)~0\|(.*)~1\|(.*)~2\|', arch_str) + return { + '0_1': mp[m.group(1)], + '0_2': mp[m.group(2)], + '1_2': mp[m.group(3)], + '0_3': mp[m.group(4)], + '1_3': mp[m.group(5)], + '2_3': mp[m.group(6)] + } + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('input_file', + help='Path to the file to be converted, e.g., NAS-Bench-201-v1_1-096897.pth.') + args = parser.parse_args() + dataset_split = { + 'cifar10-valid': ['train', 'x-valid', 'ori-test', 'ori-test'], + 'cifar10': ['train', 'ori-test', 'ori-test', 'ori-test'], + 'cifar100': ['train', 'x-valid', 'x-test', 'ori-test'], + 'imagenet16-120': ['train', 'x-valid', 'x-test', 'ori-test'], + } + + with db: + db.create_tables([Nb201TrialConfig, Nb201TrialStats, Nb201IntermediateStats]) + print('Loading NAS-Bench-201 pickle...') + nb201_data = torch.load(args.input_file) + print('Dumping architectures...') + for arch_str in nb201_data['meta_archs']: + arch_json = parse_arch_str(arch_str) + for epochs in [12, 200]: + for dataset in Nb201TrialConfig.dataset.choices: + Nb201TrialConfig.create(arch=arch_json, num_epochs=epochs, dataset=dataset, + num_channels=16, num_cells=5) + for arch_info in tqdm.tqdm(nb201_data['arch2infos'].values(), + desc='Processing architecture statistics'): + for epochs_verb, d in arch_info.items(): + if epochs_verb == 'less': + epochs = 12 + else: + epochs = 200 + arch_json = parse_arch_str(d['arch_str']) + for (dataset, seed), r in d['all_results'].items(): + sp = dataset_split[dataset.lower()] + data_parsed = { + 'train_acc': r['train_acc1es'][epochs - 1], + 'valid_acc': r['eval_acc1es']['{}@{}'.format(sp[1], epochs - 1)], + 'test_acc': r['eval_acc1es']['{}@{}'.format(sp[2], epochs - 1)], + 'ori_test_acc': r['eval_acc1es']['{}@{}'.format(sp[3], epochs - 1)], + 'train_loss': r['train_losses'][epochs - 1], + 'valid_loss': r['eval_losses']['{}@{}'.format(sp[1], epochs - 1)], + 'test_loss': r['eval_losses']['{}@{}'.format(sp[2], epochs - 1)], + 'ori_test_loss': r['eval_losses']['{}@{}'.format(sp[3], epochs - 1)], + 'parameters': r['params'], + 'flops': r['flop'], + 'latency': r['latency'][0], + 'training_time': r['train_times'][epochs - 1] * epochs, + 'valid_evaluation_time': r['eval_times']['{}@{}'.format(sp[1], epochs - 1)], + 'test_evaluation_time': r['eval_times']['{}@{}'.format(sp[2], epochs - 1)], + 'ori_test_evaluation_time': r['eval_times']['{}@{}'.format(sp[3], epochs - 1)], + } + config = Nb201TrialConfig.get( + (Nb201TrialConfig.num_epochs == epochs) & + (Nb201TrialConfig.arch == arch_json) & + (Nb201TrialConfig.dataset == dataset.lower()) + ) + trial_stats = Nb201TrialStats.create(config=config, seed=seed, **data_parsed) + intermediate_stats = [] + for epoch in range(epochs): + data_parsed = { + 'train_acc': r['train_acc1es'].get(epoch), + 'valid_acc': r['eval_acc1es'].get('{}@{}'.format(sp[1], epoch)), + 'test_acc': r['eval_acc1es'].get('{}@{}'.format(sp[2], epoch)), + 'ori_test_acc': r['eval_acc1es'].get('{}@{}'.format(sp[3], epoch)), + 'train_loss': r['train_losses'].get(epoch), + 'valid_loss': r['eval_losses'].get('{}@{}'.format(sp[1], epoch)), + 'test_loss': r['eval_losses'].get('{}@{}'.format(sp[2], epoch)), + 'ori_test_loss': r['eval_losses'].get('{}@{}'.format(sp[3], epoch)), + } + if all([v is None for v in data_parsed.values()]): + continue + data_parsed.update(current_epoch=epoch + 1, trial=trial_stats) + intermediate_stats.append(data_parsed) + Nb201IntermediateStats.insert_many(intermediate_stats).execute(db) + + +if __name__ == '__main__': + main() diff --git a/utils/third_party/nni_new/nas/benchmarks/nasbench201/model.py b/utils/third_party/nni_new/nas/benchmarks/nasbench201/model.py new file mode 100644 index 0000000000000000000000000000000000000000..b390a43091bcedcdca7e599eec38b9c9a31f6147 --- /dev/null +++ b/utils/third_party/nni_new/nas/benchmarks/nasbench201/model.py @@ -0,0 +1,160 @@ +import os + +from peewee import CharField, FloatField, ForeignKeyField, IntegerField, Model +from playhouse.sqlite_ext import JSONField, SqliteExtDatabase + +from nni.nas.benchmarks.constants import DATABASE_DIR +from nni.nas.benchmarks.utils import json_dumps + +db = SqliteExtDatabase(os.path.join(DATABASE_DIR, 'nasbench201.db'), autoconnect=True) + + +class Nb201TrialConfig(Model): + """ + Trial config for NAS-Bench-201. + + Attributes + ---------- + arch : dict + A dict with keys ``0_1``, ``0_2``, ``0_3``, ``1_2``, ``1_3``, ``2_3``, each of which + is an operator chosen from :const:`nni.nas.benchmark.nasbench201.NONE`, + :const:`nni.nas.benchmark.nasbench201.SKIP_CONNECT`, + :const:`nni.nas.benchmark.nasbench201.CONV_1X1`, + :const:`nni.nas.benchmark.nasbench201.CONV_3X3` and :const:`nni.nas.benchmark.nasbench201.AVG_POOL_3X3`. + num_epochs : int + Number of epochs planned for this trial. Should be one of 12 and 200. + num_channels: int + Number of channels for initial convolution. 16 by default. + num_cells: int + Number of cells per stage. 5 by default. + dataset: str + Dataset used for training and evaluation. NAS-Bench-201 provides the following 4 options: + ``cifar10-valid`` (training data is splited into 25k for training and 25k for validation, + validation data is used for test), ``cifar10`` (training data is used in training, validation + data is splited into 5k for validation and 5k for testing), ``cifar100`` (same protocol as ``cifar10``), + and ``imagenet16-120`` (a subset of 120 classes in ImageNet, downscaled to 16x16, using training data + for training, 6k images from validation set for validation and the other 6k for testing). + """ + + arch = JSONField(json_dumps=json_dumps, index=True) + num_epochs = IntegerField(index=True) + num_channels = IntegerField() + num_cells = IntegerField() + dataset = CharField(max_length=20, index=True, choices=[ + 'cifar10-valid', # 25k+25k+10k + 'cifar10', # 50k+5k+5k + 'cifar100', # 50k+5k+5k + 'imagenet16-120', + ]) + + class Meta: + database = db + + +class Nb201TrialStats(Model): + """ + Computation statistics for NAS-Bench-201. Each corresponds to one trial. + + Attributes + ---------- + config : Nb201TrialConfig + Setup for this trial data. + seed : int + Random seed selected, for reproduction. + train_acc : float + Final accuracy on training data, ranging from 0 to 100. + valid_acc : float + Final accuracy on validation data, ranging from 0 to 100. + test_acc : float + Final accuracy on test data, ranging from 0 to 100. + ori_test_acc : float + Test accuracy on original validation set (10k for CIFAR and 12k for Imagenet16-120), + ranging from 0 to 100. + train_loss : float or None + Final cross entropy loss on training data. Note that loss could be NaN, in which case + this attributed will be None. + valid_loss : float or None + Final cross entropy loss on validation data. + test_loss : float or None + Final cross entropy loss on test data. + ori_test_loss : float or None + Final cross entropy loss on original validation set. + parameters : float + Number of trainable parameters in million. + latency : float + Latency in seconds. + flops : float + FLOPs in million. + training_time : float + Duration of training in seconds. + valid_evaluation_time : float + Time elapsed to evaluate on validation set. + test_evaluation_time : float + Time elapsed to evaluate on test set. + ori_test_evaluation_time : float + Time elapsed to evaluate on original test set. + """ + config = ForeignKeyField(Nb201TrialConfig, backref='trial_stats', index=True) + seed = IntegerField() + train_acc = FloatField() + valid_acc = FloatField() + test_acc = FloatField() + ori_test_acc = FloatField() # test accuracy of the original test set + train_loss = FloatField(null=True) # possibly nan + valid_loss = FloatField(null=True) + test_loss = FloatField(null=True) + ori_test_loss = FloatField(null=True) + parameters = FloatField() # parameters in million + latency = FloatField() # latency in milliseconds + flops = FloatField() # flops in million + training_time = FloatField() + valid_evaluation_time = FloatField() + test_evaluation_time = FloatField() + ori_test_evaluation_time = FloatField() + + class Meta: + database = db + + +class Nb201IntermediateStats(Model): + """ + Intermediate statistics for NAS-Bench-201. + + Attributes + ---------- + trial : Nb201TrialStats + Corresponding trial. + current_epoch : int + Elapsed epochs. + train_acc : float + Current accuracy on training data, ranging from 0 to 100. + valid_acc : float + Current accuracy on validation data, ranging from 0 to 100. + test_acc : float + Current accuracy on test data, ranging from 0 to 100. + ori_test_acc : float + Test accuracy on original validation set (10k for CIFAR and 12k for Imagenet16-120), + ranging from 0 to 100. + train_loss : float or None + Current cross entropy loss on training data. + valid_loss : float or None + Current cross entropy loss on validation data. + test_loss : float or None + Current cross entropy loss on test data. + ori_test_loss : float or None + Current cross entropy loss on original validation set. + """ + + trial = ForeignKeyField(Nb201TrialStats, backref='intermediates', index=True) + current_epoch = IntegerField(index=True) + train_acc = FloatField(null=True) + valid_acc = FloatField(null=True) + test_acc = FloatField(null=True) + ori_test_acc = FloatField(null=True) + train_loss = FloatField(null=True) + valid_loss = FloatField(null=True) + test_loss = FloatField(null=True) + ori_test_loss = FloatField(null=True) + + class Meta: + database = db diff --git a/utils/third_party/nni_new/nas/benchmarks/nasbench201/query.py b/utils/third_party/nni_new/nas/benchmarks/nasbench201/query.py new file mode 100644 index 0000000000000000000000000000000000000000..b52548ecd35f897bddc74c42cc129268a6e75b80 --- /dev/null +++ b/utils/third_party/nni_new/nas/benchmarks/nasbench201/query.py @@ -0,0 +1,67 @@ +import functools + +from peewee import fn +from playhouse.shortcuts import model_to_dict +from .model import Nb201TrialStats, Nb201TrialConfig + + +def query_nb201_trial_stats(arch, num_epochs, dataset, reduction=None, include_intermediates=False): + """ + Query trial stats of NAS-Bench-201 given conditions. + + Parameters + ---------- + arch : dict or None + If a dict, it is in the format that is described in + :class:`nni.nas.benchmark.nasbench201.Nb201TrialConfig`. Only trial stats + matched will be returned. If none, all architectures in the database will be matched. + num_epochs : int or None + If int, matching results will be returned. Otherwise a wildcard. + dataset : str or None + If specified, can be one of the dataset available in :class:`nni.nas.benchmark.nasbench201.Nb201TrialConfig`. + Otherwise a wildcard. + reduction : str or None + If 'none' or None, all trial stats will be returned directly. + If 'mean', fields in trial stats will be averaged given the same trial config. + include_intermediates : boolean + If true, intermediate results will be returned. + + Returns + ------- + generator of dict + A generator of :class:`nni.nas.benchmark.nasbench201.Nb201TrialStats` objects, + where each of them has been converted into a dict. + """ + fields = [] + if reduction == 'none': + reduction = None + if reduction == 'mean': + for field_name in Nb201TrialStats._meta.sorted_field_names: + if field_name not in ['id', 'config', 'seed']: + fields.append(fn.AVG(getattr(Nb201TrialStats, field_name)).alias(field_name)) + elif reduction is None: + fields.append(Nb201TrialStats) + else: + raise ValueError('Unsupported reduction: \'%s\'' % reduction) + query = Nb201TrialStats.select(*fields, Nb201TrialConfig).join(Nb201TrialConfig) + conditions = [] + if arch is not None: + conditions.append(Nb201TrialConfig.arch == arch) + if num_epochs is not None: + conditions.append(Nb201TrialConfig.num_epochs == num_epochs) + if dataset is not None: + conditions.append(Nb201TrialConfig.dataset == dataset) + if conditions: + query = query.where(functools.reduce(lambda a, b: a & b, conditions)) + if reduction is not None: + query = query.group_by(Nb201TrialStats.config) + for trial in query: + if include_intermediates: + data = model_to_dict(trial) + # exclude 'trial' from intermediates as it is already available in data + data['intermediates'] = [ + {k: v for k, v in model_to_dict(t).items() if k != 'trial'} for t in trial.intermediates + ] + yield data + else: + yield model_to_dict(trial) diff --git a/utils/third_party/nni_new/nas/benchmarks/nds/__init__.py b/utils/third_party/nni_new/nas/benchmarks/nds/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9d393b86e1948d6ea5fdf8cacee6d4fd645c8434 --- /dev/null +++ b/utils/third_party/nni_new/nas/benchmarks/nds/__init__.py @@ -0,0 +1,3 @@ +from .constants import * +from .model import NdsTrialConfig, NdsTrialStats, NdsIntermediateStats +from .query import query_nds_trial_stats diff --git a/utils/third_party/nni_new/nas/benchmarks/nds/constants.py b/utils/third_party/nni_new/nas/benchmarks/nds/constants.py new file mode 100644 index 0000000000000000000000000000000000000000..164f094f3aacb8ac36a8572c28e31e9419baa416 --- /dev/null +++ b/utils/third_party/nni_new/nas/benchmarks/nds/constants.py @@ -0,0 +1,16 @@ +NONE = 'none' +SKIP_CONNECT = 'skip_connect' +AVG_POOL_3X3 = 'avg_pool_3x3' +MAX_POOL_3X3 = 'max_pool_3x3' +MAX_POOL_5X5 = 'max_pool_5x5' +MAX_POOL_7X7 = 'max_pool_7x7' +CONV_1X1 = 'conv_1x1' +CONV_3X3 = 'conv_3x3' +CONV_3X1_1X3 = 'conv_3x1_1x3' +CONV_7X1_1X7 = 'conv_7x1_1x7' +DIL_CONV_3X3 = 'dil_conv_3x3' +DIL_CONV_5X5 = 'dil_conv_5x5' +SEP_CONV_3X3 = 'sep_conv_3x3' +SEP_CONV_5X5 = 'sep_conv_5x5' +SEP_CONV_7X7 = 'sep_conv_7x7' +DIL_SEP_CONV_3X3 = 'dil_sep_conv_3x3' diff --git a/utils/third_party/nni_new/nas/benchmarks/nds/db_gen.py b/utils/third_party/nni_new/nas/benchmarks/nds/db_gen.py new file mode 100644 index 0000000000000000000000000000000000000000..d4df09d4e3befba746806d5029c38420c237b2d3 --- /dev/null +++ b/utils/third_party/nni_new/nas/benchmarks/nds/db_gen.py @@ -0,0 +1,153 @@ +import json +import argparse +import os + +import numpy as np +import tqdm + +from .model import db, NdsTrialConfig, NdsTrialStats, NdsIntermediateStats + + +def inject_item(db, item, proposer, dataset, generator): + if 'genotype' in item['net']: + model_family = 'nas_cell' + num_nodes_normal = len(item['net']['genotype']['normal']) // 2 + num_nodes_reduce = len(item['net']['genotype']['reduce']) // 2 + model_spec = { + 'num_nodes_normal': num_nodes_normal, + 'num_nodes_reduce': num_nodes_reduce, + 'depth': item['net']['depth'], + 'width': item['net']['width'], + 'aux': item['net']['aux'], + 'drop_prob': item['net']['drop_prob'], + } + cell_spec = {} + for cell_type in ['normal', 'reduce']: + for i in range(num_nodes_normal): + for j, label in enumerate(['x', 'y']): + cell_spec['{}_{}_op_{}'.format(cell_type, i, label)] = \ + item['net']['genotype'][cell_type][i * 2 + j][0] + cell_spec['{}_{}_input_{}'.format(cell_type, i, label)] = \ + item['net']['genotype'][cell_type][i * 2 + j][1] + cell_spec['{}_concat'.format(cell_type)] = item['net']['genotype']['{}_concat'.format(cell_type)] + else: + if item['net']['block_type'].startswith('res_bottleneck'): + model_family = 'residual_bottleneck' + elif item['net']['block_type'].startswith('res_basic'): + model_family = 'residual_basic' + elif item['net']['block_type'].startswith('double_plain'): + model_family = 'vanilla' + else: + raise ValueError('Unrecognized block type') + model_spec = {k: v for k, v in item['net'].items() if v and k != 'block_type'} + cell_spec = {} + trial_config, _ = NdsTrialConfig.get_or_create( + model_family=model_family, + model_spec=model_spec, + cell_spec=cell_spec, + proposer=proposer, + base_lr=item['optim']['base_lr'], + weight_decay=item['optim']['wd'], + num_epochs=item['optim']['max_ep'], + dataset=dataset, + generator=generator + ) + assert len(item['train_ep_top1']) == len(item['test_ep_top1']) == trial_config.num_epochs + trial = NdsTrialStats.create( + config=trial_config, + seed=item['rng_seed'], + final_train_acc=100 - item['train_ep_top1'][-1], + final_train_loss=item['train_ep_loss'][-1], + final_test_acc=100 - item['test_ep_top1'][-1], + best_train_acc=100 - min(item['train_ep_top1']), + best_train_loss=np.nanmin(item['train_ep_loss']).item(), + best_test_acc=100 - min(item['test_ep_top1']), + parameters=item['params'] / 1e6, + flops=item['flops'] / 1e6, + iter_time=item['iter_time'] + ) + intermediate_stats = [] + for i in range(trial_config.num_epochs): + intermediate_stats.append({ + 'trial': trial, + 'current_epoch': i + 1, + 'train_loss': item['train_ep_loss'][i], + 'train_acc': 100 - item['train_ep_top1'][i], + 'test_acc': 100 - item['test_ep_top1'][i] + }) + NdsIntermediateStats.insert_many(intermediate_stats).execute(db) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('input_dir', help='Path to extracted NDS data dir.') + args = parser.parse_args() + + sweep_list = [ + 'Amoeba.json', + 'Amoeba_in.json', + 'DARTS.json', + 'DARTS_fix-w-d.json', + 'DARTS_in.json', + 'DARTS_lr-wd.json', + 'DARTS_lr-wd_in.json', + 'ENAS.json', + 'ENAS_fix-w-d.json', + 'ENAS_in.json', + 'NASNet.json', + 'NASNet_in.json', + 'PNAS.json', + 'PNAS_fix-w-d.json', + 'PNAS_in.json', + 'ResNeXt-A.json', + 'ResNeXt-A_in.json', + 'ResNeXt-B.json', + 'ResNeXt-B_in.json', + 'ResNet-B.json', + 'ResNet.json', + 'ResNet_lr-wd.json', + 'ResNet_lr-wd_in.json', + 'ResNet_reruns.json', + 'ResNet_rng1.json', + 'ResNet_rng2.json', + 'ResNet_rng3.json', + 'Vanilla.json', + 'Vanilla_lr-wd.json', + 'Vanilla_lr-wd_in.json', + 'Vanilla_reruns.json', + 'Vanilla_rng1.json', + 'Vanilla_rng2.json', + 'Vanilla_rng3.json' + ] + + with db: + db.create_tables([NdsTrialConfig, NdsTrialStats, NdsIntermediateStats]) + for json_idx, json_file in enumerate(sweep_list, start=1): + if 'fix-w-d' in json_file: + generator = 'fix_w_d' + elif 'lr-wd' in json_file: + generator = 'tune_lr_wd' + else: + generator = 'random' + if '_in' in json_file: + dataset = 'imagenet' + else: + dataset = 'cifar10' + proposer = json_file.split(".")[0].split("_")[0].lower() + with open(os.path.join(args.input_dir, json_file), 'r') as f: + data = json.load(f) + if 'top' in data and 'mid' in data: + for t in tqdm.tqdm(data['top'], + desc='[{}/{}] Processing {} (top)'.format(json_idx, len(sweep_list), json_file)): + inject_item(db, t, proposer, dataset, generator) + for t in tqdm.tqdm(data['mid'], + desc='[{}/{}] Processing {} (mid)'.format(json_idx, len(sweep_list), json_file)): + inject_item(db, t, proposer, dataset, generator) + else: + for job in tqdm.tqdm(data, + desc='[{}/{}] Processing {}'.format(json_idx, len(sweep_list), json_file)): + inject_item(db, job, proposer, dataset, generator) + + +if __name__ == '__main__': + main() diff --git a/utils/third_party/nni_new/nas/benchmarks/nds/model.py b/utils/third_party/nni_new/nas/benchmarks/nds/model.py new file mode 100644 index 0000000000000000000000000000000000000000..a6ace351d7fbe3aa8f043e8d7e56447807a3a184 --- /dev/null +++ b/utils/third_party/nni_new/nas/benchmarks/nds/model.py @@ -0,0 +1,143 @@ +import os + +from peewee import CharField, FloatField, ForeignKeyField, IntegerField, Model +from playhouse.sqlite_ext import JSONField, SqliteExtDatabase + +from nni.nas.benchmarks.constants import DATABASE_DIR +from nni.nas.benchmarks.utils import json_dumps + +db = SqliteExtDatabase(os.path.join(DATABASE_DIR, 'nds.db'), autoconnect=True) + + +class NdsTrialConfig(Model): + """ + Trial config for NDS. + + Attributes + ---------- + model_family : str + Could be ``nas_cell``, ``residual_bottleneck``, ``residual_basic`` or ``vanilla``. + model_spec : dict + If ``model_family`` is ``nas_cell``, it contains ``num_nodes_normal``, ``num_nodes_reduce``, ``depth``, + ``width``, ``aux`` and ``drop_prob``. If ``model_family`` is ``residual_bottleneck``, it contains ``bot_muls``, + ``ds`` (depths), ``num_gs`` (number of groups) and ``ss`` (strides). If ``model_family`` is ``residual_basic`` or + ``vanilla``, it contains ``ds``, ``ss`` and ``ws``. + cell_spec : dict + If ``model_family`` is not ``nas_cell`` it will be an empty dict. Otherwise, it specifies + ``___``, where i ranges from 0 to ``num_nodes_ - 1``. + If it is an ``op``, the value is chosen from the constants specified previously like :const:`nni.nas.benchmark.nds.CONV_1X1`. + If it is i's ``input``, the value range from 0 to ``i + 1``, as ``nas_cell`` uses previous two nodes as inputs, and + node 0 is actually the second node. Refer to NASNet paper for details. Finally, another two key-value pairs + ``normal_concat`` and ``reduce_concat`` specify which nodes are eventually concatenated into output. + dataset : str + Dataset used. Could be ``cifar10`` or ``imagenet``. + generator : str + Can be one of ``random`` which generates configurations at random, while keeping learning rate and weight decay fixed, + ``fix_w_d`` which further keeps ``width`` and ``depth`` fixed, only applicable for ``nas_cell``. ``tune_lr_wd`` which + further tunes learning rate and weight decay. + proposer : str + Paper who has proposed the distribution for random sampling. Available proposers include ``nasnet``, ``darts``, ``enas``, + ``pnas``, ``amoeba``, ``vanilla``, ``resnext-a``, ``resnext-b``, ``resnet``, ``resnet-b`` (ResNet with bottleneck). + See NDS paper for details. + base_lr : float + Initial learning rate. + weight_decay : float + L2 weight decay applied on weights. + num_epochs : int + Number of epochs scheduled, during which learning rate will decay to 0 following cosine annealing. + """ + + model_family = CharField(max_length=20, index=True, choices=[ + 'nas_cell', + 'residual_bottleneck', + 'residual_basic', + 'vanilla', + ]) + model_spec = JSONField(json_dumps=json_dumps, index=True) + cell_spec = JSONField(json_dumps=json_dumps, index=True, null=True) + dataset = CharField(max_length=15, index=True, choices=['cifar10', 'imagenet']) + generator = CharField(max_length=15, index=True, choices=[ + 'random', + 'fix_w_d', + 'tune_lr_wd', + ]) + proposer = CharField(max_length=15, index=True) + base_lr = FloatField() + weight_decay = FloatField() + num_epochs = IntegerField() + + class Meta: + database = db + + +class NdsTrialStats(Model): + """ + Computation statistics for NDS. Each corresponds to one trial. + + Attributes + ---------- + config : NdsTrialConfig + Corresponding config for trial. + seed : int + Random seed selected, for reproduction. + final_train_acc : float + Final accuracy on training data, ranging from 0 to 100. + final_train_loss : float or None + Final cross entropy loss on training data. Could be NaN (None). + final_test_acc : float + Final accuracy on test data, ranging from 0 to 100. + best_train_acc : float + Best accuracy on training data, ranging from 0 to 100. + best_train_loss : float or None + Best cross entropy loss on training data. Could be NaN (None). + best_test_acc : float + Best accuracy on test data, ranging from 0 to 100. + parameters : float + Number of trainable parameters in million. + flops : float + FLOPs in million. + iter_time : float + Seconds elapsed for each iteration. + """ + config = ForeignKeyField(NdsTrialConfig, backref='trial_stats', index=True) + seed = IntegerField() + final_train_acc = FloatField() + final_train_loss = FloatField(null=True) + final_test_acc = FloatField() + best_train_acc = FloatField() + best_train_loss = FloatField(null=True) + best_test_acc = FloatField() + parameters = FloatField() + flops = FloatField() + iter_time = FloatField() + + class Meta: + database = db + + +class NdsIntermediateStats(Model): + """ + Intermediate statistics for NDS. + + Attributes + ---------- + trial : NdsTrialStats + Corresponding trial. + current_epoch : int + Elapsed epochs. + train_loss : float or None + Current cross entropy loss on training data. Can be NaN (None). + train_acc : float + Current accuracy on training data, ranging from 0 to 100. + test_acc : float + Current accuracy on test data, ranging from 0 to 100. + """ + + trial = ForeignKeyField(NdsTrialStats, backref='intermediates', index=True) + current_epoch = IntegerField(index=True) + train_loss = FloatField(null=True) + train_acc = FloatField() + test_acc = FloatField() + + class Meta: + database = db diff --git a/utils/third_party/nni_new/nas/benchmarks/nds/query.py b/utils/third_party/nni_new/nas/benchmarks/nds/query.py new file mode 100644 index 0000000000000000000000000000000000000000..fe192ba5090f789df2febb4d237475a75f74e445 --- /dev/null +++ b/utils/third_party/nni_new/nas/benchmarks/nds/query.py @@ -0,0 +1,74 @@ +import functools + +from peewee import fn +from playhouse.shortcuts import model_to_dict +from .model import NdsTrialStats, NdsTrialConfig + + +def query_nds_trial_stats(model_family, proposer, generator, model_spec, cell_spec, dataset, + num_epochs=None, reduction=None, include_intermediates=False): + """ + Query trial stats of NDS given conditions. + + Parameters + ---------- + model_family : str or None + If str, can be one of the model families available in :class:`nni.nas.benchmark.nds.NdsTrialConfig`. + Otherwise a wildcard. + proposer : str or None + If str, can be one of the proposers available in :class:`nni.nas.benchmark.nds.NdsTrialConfig`. Otherwise a wildcard. + generator : str or None + If str, can be one of the generators available in :class:`nni.nas.benchmark.nds.NdsTrialConfig`. Otherwise a wildcard. + model_spec : dict or None + If specified, can be one of the model spec available in :class:`nni.nas.benchmark.nds.NdsTrialConfig`. + Otherwise a wildcard. + cell_spec : dict or None + If specified, can be one of the cell spec available in :class:`nni.nas.benchmark.nds.NdsTrialConfig`. + Otherwise a wildcard. + dataset : str or None + If str, can be one of the datasets available in :class:`nni.nas.benchmark.nds.NdsTrialConfig`. Otherwise a wildcard. + num_epochs : float or None + If int, matching results will be returned. Otherwise a wildcard. + reduction : str or None + If 'none' or None, all trial stats will be returned directly. + If 'mean', fields in trial stats will be averaged given the same trial config. + include_intermediates : boolean + If true, intermediate results will be returned. + + Returns + ------- + generator of dict + A generator of :class:`nni.nas.benchmark.nds.NdsTrialStats` objects, + where each of them has been converted into a dict. + """ + fields = [] + if reduction == 'none': + reduction = None + if reduction == 'mean': + for field_name in NdsTrialStats._meta.sorted_field_names: + if field_name not in ['id', 'config', 'seed']: + fields.append(fn.AVG(getattr(NdsTrialStats, field_name)).alias(field_name)) + elif reduction is None: + fields.append(NdsTrialStats) + else: + raise ValueError('Unsupported reduction: \'%s\'' % reduction) + query = NdsTrialStats.select(*fields, NdsTrialConfig).join(NdsTrialConfig) + conditions = [] + for field_name in ['model_family', 'proposer', 'generator', 'model_spec', 'cell_spec', + 'dataset', 'num_epochs']: + if locals()[field_name] is not None: + conditions.append(getattr(NdsTrialConfig, field_name) == locals()[field_name]) + if conditions: + query = query.where(functools.reduce(lambda a, b: a & b, conditions)) + if reduction is not None: + query = query.group_by(NdsTrialStats.config) + for trial in query: + if include_intermediates: + data = model_to_dict(trial) + # exclude 'trial' from intermediates as it is already available in data + data['intermediates'] = [ + {k: v for k, v in model_to_dict(t).items() if k != 'trial'} for t in trial.intermediates + ] + yield data + else: + yield model_to_dict(trial) diff --git a/utils/third_party/nni_new/nas/benchmarks/nlp/__init__.py b/utils/third_party/nni_new/nas/benchmarks/nlp/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9dc929727c4293a778dd5d86b7cf294ed5015ffc --- /dev/null +++ b/utils/third_party/nni_new/nas/benchmarks/nlp/__init__.py @@ -0,0 +1,4 @@ +from .model import NlpTrialStats, NlpIntermediateStats, NlpTrialConfig +from .query import query_nlp_trial_stats + + diff --git a/utils/third_party/nni_new/nas/benchmarks/nlp/db_gen.py b/utils/third_party/nni_new/nas/benchmarks/nlp/db_gen.py new file mode 100644 index 0000000000000000000000000000000000000000..4c1198051438601a53732df9922f6d62993acd32 --- /dev/null +++ b/utils/third_party/nni_new/nas/benchmarks/nlp/db_gen.py @@ -0,0 +1,46 @@ +import json +import os +import argparse +import tqdm + +from .model import db, NlpTrialConfig, NlpTrialStats, NlpIntermediateStats + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('input_dir', help='Path to extracted NLP data dir.') + args = parser.parse_args() + with db, tqdm.tqdm(total=len(os.listdir(args.input_dir)), desc="creating tables") as pbar: + db.create_tables([NlpTrialConfig, NlpTrialStats, NlpIntermediateStats]) + json_files = os.listdir(args.input_dir) + for json_file in json_files: + pbar.update(1) + if json_file.endswith('.json'): + log_path = os.path.join(args.input_dir, json_file) + cur = json.load(open(log_path, 'r')) + arch = json.loads(cur['recepie']) + unested_arch = {} + for k in arch.keys(): + # print(k) + unested_arch['{}_op'.format(k)] = arch[k]['op'] + for i in range(len(arch[k]['input'])): + unested_arch['{}_input_{}'.format(k, i)] = arch[k]['input'][i] + config = NlpTrialConfig.create(arch=unested_arch, dataset=cur['data'][5:]) + if cur['status'] == 'OK': + trial_stats = NlpTrialStats.create(config=config, train_loss=cur['train_losses'][-1], val_loss=cur['val_losses'][-1], + test_loss=cur['test_losses'][-1], training_time=cur['wall_times'][-1]) + epochs = 50 + intermediate_stats = [] + for epoch in range(epochs): + epoch_res = { + 'train_loss' : cur['train_losses'][epoch], + 'val_loss' : cur['val_losses'][epoch], + 'test_loss' : cur['test_losses'][epoch], + 'training_time' : cur['wall_times'][epoch] + } + epoch_res.update(current_epoch=epoch + 1, trial=trial_stats) + intermediate_stats.append(epoch_res) + NlpIntermediateStats.insert_many(intermediate_stats).execute(db) + + +if __name__ == '__main__': + main() diff --git a/utils/third_party/nni_new/nas/benchmarks/nlp/model.py b/utils/third_party/nni_new/nas/benchmarks/nlp/model.py new file mode 100644 index 0000000000000000000000000000000000000000..d83ab7ff237af5fce0c9b1de322f77ff4cc021f1 --- /dev/null +++ b/utils/third_party/nni_new/nas/benchmarks/nlp/model.py @@ -0,0 +1,92 @@ +import os + +from peewee import CharField, FloatField, ForeignKeyField, IntegerField, Model +from playhouse.sqlite_ext import JSONField, SqliteExtDatabase + +from nni.nas.benchmarks.utils import json_dumps +from nni.nas.benchmarks.constants import DATABASE_DIR + +db = SqliteExtDatabase(os.path.join(DATABASE_DIR, 'nlp.db'), autoconnect=True) + +class NlpTrialConfig(Model): + """ + Trial config for NLP. epoch_num is fixed at 50. + + Attributes + ---------- + arch: dict + aka recepie in NAS-NLP-Benchmark repo (https://github.com/fmsnew/nas-bench-nlp-release). + an arch has multiple Node, Node_input_n and Node_op. + ``Node`` can be ``node_n`` or ``h_new_n`` or ``f/i/o/j(_act)`` etc. (n is an int number and need not to be consecutive) + ``Node_input_n`` can be ``Node`` or ``x`` etc. + ``Node_op`` can be ``linear`` or ``activation_sigm`` or ``activation_tanh`` or ``elementwise_prod`` + or ``elementwise_sum`` or ``activation_leaky_relu`` ... + e.g., {"h_new_0_input_0":"node_3","h_new_0_input_1":"x","h_new_0_op":"linear","node_2_input_0":"x", + "node_2_input_1":"h_prev_0","node_2_op":"linear","node_3_input_0":"node_2","node_3_op":"activation_leaky_relu"} + dataset: str + Dataset used. Could be ``ptb`` or ``wikitext-2``. + """ + arch = JSONField(json_dumps=json_dumps, index=True) + dataset = CharField(max_length=15, index=True, choices=[ + 'ptb', + 'wikitext-2' + ]) + + class Meta: + database = db + +class NlpTrialStats(Model): + """ + Computation statistics for NAS-NLP-Benchmark. + Each corresponds to one trial result after 50 epoch. + + Attributes + ---------- + config : NlpTrialConfig + Corresponding config for trial. + train_loss : float or None + Final loss on training data. Could be NaN (None). + val_loss : float or None + Final loss on validation data. Could be NaN (None). + test_loss : float or None + Final loss on test data. Could be NaN (None). + training_time : float + Time elapsed in seconds. aka wall_time in in NAS-NLP-Benchmark repo. + """ + config = ForeignKeyField(NlpTrialConfig, backref='trial_stats', index=True) + train_loss = FloatField(null=True) + val_loss = FloatField(null=True) + test_loss = FloatField(null=True) + training_time = FloatField(null=True) + + class Meta: + database = db + +class NlpIntermediateStats(Model): + """ + Computation statistics for NAS-NLP-Benchmark. + Each corresponds to one trial result for 1-50 epoch. + + Attributes + ---------- + config : NlpTrialConfig + Corresponding config for trial. + train_loss : float or None + Final loss on training data. Could be NaN (None). + val_loss : float or None + Final loss on validation data. Could be NaN (None). + test_loss : float or None + Final loss on test data. Could be NaN (None). + training_time : float + Time elapsed in seconds. aka wall_time in in NAS-NLP-Benchmark repo. + """ + trial = ForeignKeyField(NlpTrialStats, backref='intermediates', index=True) + current_epoch = IntegerField(index=True) + train_loss = FloatField(null=True) + val_loss = FloatField(null=True) + test_loss = FloatField(null=True) + training_time = FloatField(null=True) + + class Meta: + database = db + \ No newline at end of file diff --git a/utils/third_party/nni_new/nas/benchmarks/nlp/query.py b/utils/third_party/nni_new/nas/benchmarks/nlp/query.py new file mode 100644 index 0000000000000000000000000000000000000000..98885896b4d719ac07532a90a317f77a96d0ddc6 --- /dev/null +++ b/utils/third_party/nni_new/nas/benchmarks/nlp/query.py @@ -0,0 +1,61 @@ +import functools + +from peewee import fn +from playhouse.shortcuts import model_to_dict +from .model import NlpTrialStats, NlpTrialConfig + +def query_nlp_trial_stats(arch, dataset, reduction=None, include_intermediates=False): + """ + Query trial stats of NLP benchmark given conditions, including config(arch + dataset) and training results after 50 epoch. + + Parameters + ---------- + arch : dict or None + If a dict, it is in the format that is described in + :class:`nni.nas.benchmark.nlp.NlpTrialConfig`. Only trial stats matched will be returned. + If none, all architectures in the database will be matched. + dataset : str or None + If specified, can be one of the dataset available in :class:`nni.nas.benchmark.nlp.NlpTrialConfig`. + Otherwise a wildcard. + reduction : str or None + If 'none' or None, all trial stats will be returned directly. + If 'mean', fields in trial stats will be averaged given the same trial config. + Please note that some trial configs have multiple runs which make "reduction" meaningful, while some may not. + include_intermediates : boolean + If true, intermediate results will be returned. + + Returns + ------- + generator of dict + A generator of :class:`nni.nas.benchmark.nlp.NlpTrialStats` objects, + where each of them has been converted into a dict. + """ + fields = [] + if reduction == 'none': + reduction = None + if reduction == 'mean': + for field_name in NlpTrialStats._meta.sorted_field_names: + if field_name not in ['id', 'config']: + fields.append(fn.AVG(getattr(NlpTrialStats, field_name)).alias(field_name)) + elif reduction is None: + fields.append(NlpTrialStats) + else: + raise ValueError('Unsupported reduction: \'%s\'' % reduction) + query = NlpTrialStats.select(*fields, NlpTrialConfig).join(NlpTrialConfig) + + conditions = [] + if arch is not None: + conditions.append(NlpTrialConfig.arch == arch) + if dataset is not None: + conditions.append(NlpTrialConfig.dataset == dataset) + + for trial in query.where(functools.reduce(lambda a, b: a & b, conditions)): + if include_intermediates: + data = model_to_dict(trial) + # exclude 'trial' from intermediates as it is already available in data + data['intermediates'] = [ + {k: v for k, v in model_to_dict(t).items() if k != 'trial'} for t in trial.intermediates + ] + yield data + else: + yield model_to_dict(trial) \ No newline at end of file diff --git a/utils/third_party/nni_new/nas/benchmarks/utils.py b/utils/third_party/nni_new/nas/benchmarks/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7189d52f5d8619bb4f1d1fa4ba7f3a2ede33a6a0 --- /dev/null +++ b/utils/third_party/nni_new/nas/benchmarks/utils.py @@ -0,0 +1,5 @@ +import functools +import json + + +json_dumps = functools.partial(json.dumps, sort_keys=True) diff --git a/utils/third_party/nni_new/nas/pytorch/__init__.py b/utils/third_party/nni_new/nas/pytorch/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..4a61f1672d859edda94ed7194e1bc313d824a39b --- /dev/null +++ b/utils/third_party/nni_new/nas/pytorch/__init__.py @@ -0,0 +1,6 @@ +from .base_mutator import BaseMutator +from .base_trainer import BaseTrainer +from .fixed import apply_fixed_architecture +from .mutables import Mutable, LayerChoice, InputChoice +from .mutator import Mutator +from .trainer import Trainer diff --git a/utils/third_party/nni_new/nas/pytorch/base_mutator.py b/utils/third_party/nni_new/nas/pytorch/base_mutator.py new file mode 100644 index 0000000000000000000000000000000000000000..df1a5f9ba8719a5c6619cd4ccc2874c0017a44db --- /dev/null +++ b/utils/third_party/nni_new/nas/pytorch/base_mutator.py @@ -0,0 +1,155 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging + +import torch.nn as nn +from nni.nas.pytorch.mutables import Mutable, MutableScope, InputChoice +from nni.nas.pytorch.utils import StructuredMutableTreeNode + +logger = logging.getLogger(__name__) + + +class BaseMutator(nn.Module): + """ + A mutator is responsible for mutating a graph by obtaining the search space from the network and implementing + callbacks that are called in ``forward`` in mutables. + + Parameters + ---------- + model : nn.Module + PyTorch model to apply mutator on. + """ + + def __init__(self, model): + super().__init__() + self.__dict__["model"] = model + self._structured_mutables = self._parse_search_space(self.model) + + def _parse_search_space(self, module, root=None, prefix="", memo=None, nested_detection=None): + if memo is None: + memo = set() + if root is None: + root = StructuredMutableTreeNode(None) + if module not in memo: + memo.add(module) + if isinstance(module, Mutable): + if nested_detection is not None: + raise RuntimeError("Cannot have nested search space. Error at {} in {}" + .format(module, nested_detection)) + module.name = prefix + module.set_mutator(self) + root = root.add_child(module) + if not isinstance(module, MutableScope): + nested_detection = module + if isinstance(module, InputChoice): + for k in module.choose_from: + if k != InputChoice.NO_KEY and k not in [m.key for m in memo if isinstance(m, Mutable)]: + raise RuntimeError("'{}' required by '{}' not found in keys that appeared before, and is not NO_KEY." + .format(k, module.key)) + for name, submodule in module._modules.items(): + if submodule is None: + continue + submodule_prefix = prefix + ("." if prefix else "") + name + self._parse_search_space(submodule, root, submodule_prefix, memo=memo, + nested_detection=nested_detection) + return root + + @property + def mutables(self): + """ + A generator of all modules inheriting :class:`~nni.nas.pytorch.mutables.Mutable`. + Modules are yielded in the order that they are defined in ``__init__``. + For mutables with their keys appearing multiple times, only the first one will appear. + """ + return self._structured_mutables + + @property + def undedup_mutables(self): + return self._structured_mutables.traverse(deduplicate=False) + + def forward(self, *inputs): + """ + Warnings + -------- + Don't call forward of a mutator. + """ + raise RuntimeError("Forward is undefined for mutators.") + + def __setattr__(self, name, value): + if name == "model": + raise AttributeError("Attribute `model` can be set at most once, and you shouldn't use `self.model = model` to " + "include you network, as it will include all parameters in model into the mutator.") + return super().__setattr__(name, value) + + def enter_mutable_scope(self, mutable_scope): + """ + Callback when forward of a MutableScope is entered. + + Parameters + ---------- + mutable_scope : MutableScope + The mutable scope that is entered. + """ + pass + + def exit_mutable_scope(self, mutable_scope): + """ + Callback when forward of a MutableScope is exited. + + Parameters + ---------- + mutable_scope : MutableScope + The mutable scope that is exited. + """ + pass + + def on_forward_layer_choice(self, mutable, *args, **kwargs): + """ + Callbacks of forward in LayerChoice. + + Parameters + ---------- + mutable : nni.nas.pytorch.mutables.LayerChoice + Module whose forward is called. + args : list of torch.Tensor + The arguments of its forward function. + kwargs : dict + The keyword arguments of its forward function. + + Returns + ------- + tuple of torch.Tensor and torch.Tensor + Output tensor and mask. + """ + raise NotImplementedError + + def on_forward_input_choice(self, mutable, tensor_list): + """ + Callbacks of forward in InputChoice. + + Parameters + ---------- + mutable : nni.nas.pytorch.mutables.InputChoice + Mutable that is called. + tensor_list : list of torch.Tensor + The arguments mutable is called with. + + Returns + ------- + tuple of torch.Tensor and torch.Tensor + Output tensor and mask. + """ + raise NotImplementedError + + def export(self): + """ + Export the data of all decisions. This should output the decisions of all the mutables, so that the whole + network can be fully determined with these decisions for further training from scratch. + + Returns + ------- + dict + Mappings from mutable keys to decisions. + """ + raise NotImplementedError diff --git a/utils/third_party/nni_new/nas/pytorch/base_trainer.py b/utils/third_party/nni_new/nas/pytorch/base_trainer.py new file mode 100644 index 0000000000000000000000000000000000000000..2e7a4a2a23a24cd2886eac12dda80eedfa6076d8 --- /dev/null +++ b/utils/third_party/nni_new/nas/pytorch/base_trainer.py @@ -0,0 +1,40 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from abc import ABC, abstractmethod + + +class BaseTrainer(ABC): + + @abstractmethod + def train(self): + """ + Override the method to train. + """ + raise NotImplementedError + + @abstractmethod + def validate(self): + """ + Override the method to validate. + """ + raise NotImplementedError + + @abstractmethod + def export(self, file): + """ + Override the method to export to file. + + Parameters + ---------- + file : str + File path to export to. + """ + raise NotImplementedError + + @abstractmethod + def checkpoint(self): + """ + Override to dump a checkpoint. + """ + raise NotImplementedError diff --git a/utils/third_party/nni_new/nas/pytorch/callbacks.py b/utils/third_party/nni_new/nas/pytorch/callbacks.py new file mode 100644 index 0000000000000000000000000000000000000000..86a0dc3800745d93939b9ce85dd9cdca3abef166 --- /dev/null +++ b/utils/third_party/nni_new/nas/pytorch/callbacks.py @@ -0,0 +1,139 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +import os + +import torch +import torch.nn as nn + +_logger = logging.getLogger(__name__) + + +class Callback: + """ + Callback provides an easy way to react to events like begin/end of epochs. + """ + + def __init__(self): + self.model = None + self.mutator = None + self.trainer = None + + def build(self, model, mutator, trainer): + """ + Callback needs to be built with model, mutator, trainer, to get updates from them. + + Parameters + ---------- + model : nn.Module + Model to be trained. + mutator : nn.Module + Mutator that mutates the model. + trainer : BaseTrainer + Trainer that is to call the callback. + """ + self.model = model + self.mutator = mutator + self.trainer = trainer + + def on_epoch_begin(self, epoch): + """ + Implement this to do something at the begin of epoch. + + Parameters + ---------- + epoch : int + Epoch number, starting from 0. + """ + pass + + def on_epoch_end(self, epoch): + """ + Implement this to do something at the end of epoch. + + Parameters + ---------- + epoch : int + Epoch number, starting from 0. + """ + pass + + def on_batch_begin(self, epoch): + pass + + def on_batch_end(self, epoch): + pass + + +class LRSchedulerCallback(Callback): + """ + Calls scheduler on every epoch ends. + + Parameters + ---------- + scheduler : LRScheduler + Scheduler to be called. + """ + def __init__(self, scheduler, mode="epoch"): + super().__init__() + assert mode == "epoch" + self.scheduler = scheduler + self.mode = mode + + def on_epoch_end(self, epoch): + """ + Call ``self.scheduler.step()`` on epoch end. + """ + self.scheduler.step() + + +class ArchitectureCheckpoint(Callback): + """ + Calls ``trainer.export()`` on every epoch ends. + + Parameters + ---------- + checkpoint_dir : str + Location to save checkpoints. + """ + def __init__(self, checkpoint_dir): + super().__init__() + self.checkpoint_dir = checkpoint_dir + os.makedirs(self.checkpoint_dir, exist_ok=True) + + def on_epoch_end(self, epoch): + """ + Dump to ``/checkpoint_dir/epoch_{number}.json`` on epoch end. + """ + dest_path = os.path.join(self.checkpoint_dir, "epoch_{}.json".format(epoch)) + _logger.info("Saving architecture to %s", dest_path) + self.trainer.export(dest_path) + + +class ModelCheckpoint(Callback): + """ + Calls ``trainer.export()`` on every epoch ends. + + Parameters + ---------- + checkpoint_dir : str + Location to save checkpoints. + """ + def __init__(self, checkpoint_dir): + super().__init__() + self.checkpoint_dir = checkpoint_dir + os.makedirs(self.checkpoint_dir, exist_ok=True) + + def on_epoch_end(self, epoch): + """ + Dump to ``/checkpoint_dir/epoch_{number}.pth.tar`` on every epoch end. + ``DataParallel`` object will have their inside modules exported. + """ + if isinstance(self.model, nn.DataParallel): + state_dict = self.model.module.state_dict() + else: + state_dict = self.model.state_dict() + dest_path = os.path.join(self.checkpoint_dir, "epoch_{}.pth.tar".format(epoch)) + _logger.info("Saving model to %s", dest_path) + torch.save(state_dict, dest_path) diff --git a/utils/third_party/nni_new/nas/pytorch/fixed.py b/utils/third_party/nni_new/nas/pytorch/fixed.py new file mode 100644 index 0000000000000000000000000000000000000000..9bfa933e80e740ef5ff8832b9d45484bbdc07771 --- /dev/null +++ b/utils/third_party/nni_new/nas/pytorch/fixed.py @@ -0,0 +1,147 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import json +import logging + +from .mutables import InputChoice, LayerChoice, MutableScope +from .mutator import Mutator +from .utils import to_list + + +_logger = logging.getLogger(__name__) + + +class FixedArchitecture(Mutator): + """ + Fixed architecture mutator that always selects a certain graph. + + Parameters + ---------- + model : nn.Module + A mutable network. + fixed_arc : dict + Preloaded architecture object. + strict : bool + Force everything that appears in ``fixed_arc`` to be used at least once. + verbose : bool + Print log messages if set to True + """ + + def __init__(self, model, fixed_arc, strict=True, verbose=True): + super().__init__(model) + self._fixed_arc = fixed_arc + self.verbose = verbose + + mutable_keys = set([mutable.key for mutable in self.mutables if not isinstance(mutable, MutableScope)]) + fixed_arc_keys = set(self._fixed_arc.keys()) + if fixed_arc_keys - mutable_keys: + raise RuntimeError("Unexpected keys found in fixed architecture: {}.".format(fixed_arc_keys - mutable_keys)) + if mutable_keys - fixed_arc_keys: + raise RuntimeError("Missing keys in fixed architecture: {}.".format(mutable_keys - fixed_arc_keys)) + self._fixed_arc = self._from_human_readable_architecture(self._fixed_arc) + + def _from_human_readable_architecture(self, human_arc): + # convert from an exported architecture + result_arc = {k: to_list(v) for k, v in human_arc.items()} # there could be tensors, numpy arrays, etc. + # First, convert non-list to list, because there could be {"op1": 0} or {"op1": "conv"}, + # which means {"op1": [0, ]} ir {"op1": ["conv", ]} + result_arc = {k: v if isinstance(v, list) else [v] for k, v in result_arc.items()} + # Second, infer which ones are multi-hot arrays and which ones are in human-readable format. + # This is non-trivial, since if an array in [0, 1], we cannot know for sure it means [false, true] or [true, true]. + # Here, we assume an multihot array has to be a boolean array or a float array and matches the length. + for mutable in self.mutables: + if mutable.key not in result_arc: + continue # skip silently + choice_arr = result_arc[mutable.key] + if all(isinstance(v, bool) for v in choice_arr) or all(isinstance(v, float) for v in choice_arr): + if (isinstance(mutable, LayerChoice) and len(mutable) == len(choice_arr)) or \ + (isinstance(mutable, InputChoice) and mutable.n_candidates == len(choice_arr)): + # multihot, do nothing + continue + if isinstance(mutable, LayerChoice): + choice_arr = [mutable.names.index(val) if isinstance(val, str) else val for val in choice_arr] + choice_arr = [i in choice_arr for i in range(len(mutable))] + elif isinstance(mutable, InputChoice): + choice_arr = [mutable.choose_from.index(val) if isinstance(val, str) else val for val in choice_arr] + choice_arr = [i in choice_arr for i in range(mutable.n_candidates)] + result_arc[mutable.key] = choice_arr + return result_arc + + def sample_search(self): + """ + Always returns the fixed architecture. + """ + return self._fixed_arc + + def sample_final(self): + """ + Always returns the fixed architecture. + """ + return self._fixed_arc + + def replace_layer_choice(self, module=None, prefix=""): + """ + Replace layer choices with selected candidates. It's done with best effort. + In case of weighted choices or multiple choices. if some of the choices on weighted with zero, delete them. + If single choice, replace the module with a normal module. + + Parameters + ---------- + module : nn.Module + Module to be processed. + prefix : str + Module name under global namespace. + """ + if module is None: + module = self.model + for name, mutable in module.named_children(): + global_name = (prefix + "." if prefix else "") + name + if isinstance(mutable, LayerChoice): + chosen = self._fixed_arc[mutable.key] + if sum(chosen) == 1 and max(chosen) == 1 and not mutable.return_mask: + # sum is one, max is one, there has to be an only one + # this is compatible with both integer arrays, boolean arrays and float arrays + if self.verbose: + _logger.info("Replacing %s with candidate number %d.", global_name, chosen.index(1)) + setattr(module, name, mutable[chosen.index(1)]) + else: + if mutable.return_mask and self.verbose: + _logger.info("`return_mask` flag of %s is true. As it relies on the behavior of LayerChoice, " \ + "LayerChoice will not be replaced.") + # remove unused parameters + for ch, n in zip(chosen, mutable.names): + if ch == 0 and not isinstance(ch, float): + setattr(mutable, n, None) + else: + self.replace_layer_choice(mutable, global_name) + + +def apply_fixed_architecture(model, fixed_arc, verbose=True): + """ + Load architecture from `fixed_arc` and apply to model. + + Parameters + ---------- + model : torch.nn.Module + Model with mutables. + fixed_arc : str or dict + Path to the JSON that stores the architecture, or dict that stores the exported architecture. + verbose : bool + Print log messages if set to True + + Returns + ------- + FixedArchitecture + Mutator that is responsible for fixes the graph. + """ + + if isinstance(fixed_arc, str): + with open(fixed_arc) as f: + fixed_arc = json.load(f) + architecture = FixedArchitecture(model, fixed_arc, verbose) + architecture.reset() + + # for the convenience of parameters counting + architecture.replace_layer_choice() + return architecture diff --git a/utils/third_party/nni_new/nas/pytorch/mutables.py b/utils/third_party/nni_new/nas/pytorch/mutables.py new file mode 100644 index 0000000000000000000000000000000000000000..7fbb655e51e25c0d639746657bec5d4709e50c44 --- /dev/null +++ b/utils/third_party/nni_new/nas/pytorch/mutables.py @@ -0,0 +1,344 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +import warnings +from collections import OrderedDict + +import torch.nn as nn + +from nni.nas.pytorch.utils import global_mutable_counting + +logger = logging.getLogger(__name__) + + +class Mutable(nn.Module): + """ + Mutable is designed to function as a normal layer, with all necessary operators' weights. + States and weights of architectures should be included in mutator, instead of the layer itself. + + Mutable has a key, which marks the identity of the mutable. This key can be used by users to share + decisions among different mutables. In mutator's implementation, mutators should use the key to + distinguish different mutables. Mutables that share the same key should be "similar" to each other. + + Currently the default scope for keys is global. By default, the keys uses a global counter from 1 to + produce unique ids. + + Parameters + ---------- + key : str + The key of mutable. + + Notes + ----- + The counter is program level, but mutables are model level. In case multiple models are defined, and + you want to have `counter` starting from 1 in the second model, it's recommended to assign keys manually + instead of using automatic keys. + """ + + def __init__(self, key=None): + super().__init__() + if key is not None: + if not isinstance(key, str): + key = str(key) + logger.warning("Warning: key \"%s\" is not string, converted to string.", key) + self._key = key + else: + self._key = self.__class__.__name__ + str(global_mutable_counting()) + self.init_hook = self.forward_hook = None + + def __deepcopy__(self, memodict=None): + raise NotImplementedError("Deep copy doesn't work for mutables.") + + def __call__(self, *args, **kwargs): + self._check_built() + return super().__call__(*args, **kwargs) + + def set_mutator(self, mutator): + if "mutator" in self.__dict__: + raise RuntimeError("`set_mutator` is called more than once. Did you parse the search space multiple times? " + "Or did you apply multiple fixed architectures?") + self.__dict__["mutator"] = mutator + + @property + def key(self): + """ + Read-only property of key. + """ + return self._key + + @property + def name(self): + """ + After the search space is parsed, it will be the module name of the mutable. + """ + return self._name if hasattr(self, "_name") else self._key + + @name.setter + def name(self, name): + self._name = name + + def _check_built(self): + if not hasattr(self, "mutator"): + raise ValueError( + "Mutator not set for {}. You might have forgotten to initialize and apply your mutator. " + "Or did you initialize a mutable on the fly in forward pass? Move to `__init__` " + "so that trainer can locate all your mutables. See NNI docs for more details.".format(self)) + + +class MutableScope(Mutable): + """ + Mutable scope marks a subgraph/submodule to help mutators make better decisions. + + If not annotated with mutable scope, search space will be flattened as a list. However, some mutators might + need to leverage the concept of a "cell". So if a module is defined as a mutable scope, everything in it will + look like "sub-search-space" in the scope. Scopes can be nested. + + There are two ways mutators can use mutable scope. One is to traverse the search space as a tree during initialization + and reset. The other is to implement `enter_mutable_scope` and `exit_mutable_scope`. They are called before and after + the forward method of the class inheriting mutable scope. + + Mutable scopes are also mutables that are listed in the mutator.mutables (search space), but they are not supposed + to appear in the dict of choices. + + Parameters + ---------- + key : str + Key of mutable scope. + """ + def __init__(self, key): + super().__init__(key=key) + + def _check_built(self): + return True # bypass the test because it's deprecated + + def __call__(self, *args, **kwargs): + if not hasattr(self, 'mutator'): + return super().__call__(*args, **kwargs) + warnings.warn("`MutableScope` is deprecated in Retiarii.", DeprecationWarning) + try: + self._check_built() + self.mutator.enter_mutable_scope(self) + return super().__call__(*args, **kwargs) + finally: + self.mutator.exit_mutable_scope(self) + + +class LayerChoice(Mutable): + """ + Layer choice selects one of the ``op_candidates``, then apply it on inputs and return results. + In rare cases, it can also select zero or many. + + Layer choice does not allow itself to be nested. + + Parameters + ---------- + op_candidates : list of nn.Module or OrderedDict + A module list to be selected from. + reduction : str + ``mean``, ``concat``, ``sum`` or ``none``. Policy if multiples are selected. + If ``none``, a list is returned. ``mean`` returns the average. ``sum`` returns the sum. + ``concat`` concatenate the list at dimension 1. + return_mask : bool + If ``return_mask``, return output tensor and a mask. Otherwise return tensor only. + key : str + Key of the input choice. + + Attributes + ---------- + length : int + Deprecated. Number of ops to choose from. ``len(layer_choice)`` is recommended. + names : list of str + Names of candidates. + choices : list of Module + Deprecated. A list of all candidate modules in the layer choice module. + ``list(layer_choice)`` is recommended, which will serve the same purpose. + + Notes + ----- + ``op_candidates`` can be a list of modules or a ordered dict of named modules, for example, + + .. code-block:: python + + self.op_choice = LayerChoice(OrderedDict([ + ("conv3x3", nn.Conv2d(3, 16, 128)), + ("conv5x5", nn.Conv2d(5, 16, 128)), + ("conv7x7", nn.Conv2d(7, 16, 128)) + ])) + + Elements in layer choice can be modified or deleted. Use ``del self.op_choice["conv5x5"]`` or + ``self.op_choice[1] = nn.Conv3d(...)``. Adding more choices is not supported yet. + """ + + def __init__(self, op_candidates, reduction="sum", return_mask=False, key=None): + super().__init__(key=key) + self.names = [] + if isinstance(op_candidates, OrderedDict): + for name, module in op_candidates.items(): + assert name not in ["length", "reduction", "return_mask", "_key", "key", "names"], \ + "Please don't use a reserved name '{}' for your module.".format(name) + self.add_module(name, module) + self.names.append(name) + elif isinstance(op_candidates, list): + for i, module in enumerate(op_candidates): + self.add_module(str(i), module) + self.names.append(str(i)) + else: + raise TypeError("Unsupported op_candidates type: {}".format(type(op_candidates))) + self.reduction = reduction + self.return_mask = return_mask + + def __getitem__(self, idx): + if isinstance(idx, str): + return self._modules[idx] + return list(self)[idx] + + def __setitem__(self, idx, module): + key = idx if isinstance(idx, str) else self.names[idx] + return setattr(self, key, module) + + def __delitem__(self, idx): + if isinstance(idx, slice): + for key in self.names[idx]: + delattr(self, key) + else: + if isinstance(idx, str): + key, idx = idx, self.names.index(idx) + else: + key = self.names[idx] + delattr(self, key) + del self.names[idx] + + @property + def length(self): + warnings.warn("layer_choice.length is deprecated. Use `len(layer_choice)` instead.", DeprecationWarning) + return len(self) + + def __len__(self): + return len(self.names) + + def __iter__(self): + return map(lambda name: self._modules[name], self.names) + + @property + def choices(self): + warnings.warn("layer_choice.choices is deprecated. Use `list(layer_choice)` instead.", DeprecationWarning) + return list(self) + + def forward(self, *args, **kwargs): + """ + Returns + ------- + tuple of tensors + Output and selection mask. If ``return_mask`` is ``False``, only output is returned. + """ + out, mask = self.mutator.on_forward_layer_choice(self, *args, **kwargs) + if self.return_mask: + return out, mask + return out + + +class InputChoice(Mutable): + """ + Input choice selects ``n_chosen`` inputs from ``choose_from`` (contains ``n_candidates`` keys). For beginners, + use ``n_candidates`` instead of ``choose_from`` is a safe option. To get the most power out of it, you might want to + know about ``choose_from``. + + The keys in ``choose_from`` can be keys that appear in past mutables, or ``NO_KEY`` if there are no suitable ones. + The keys are designed to be the keys of the sources. To help mutators make better decisions, + mutators might be interested in how the tensors to choose from come into place. For example, the tensor is the + output of some operator, some node, some cell, or some module. If this operator happens to be a mutable (e.g., + ``LayerChoice`` or ``InputChoice``), it has a key naturally that can be used as a source key. If it's a + module/submodule, it needs to be annotated with a key: that's where a :class:`MutableScope` is needed. + + In the example below, ``input_choice`` is a 4-choose-any. The first 3 is semantically output of cell1, output of cell2, + output of cell3 with respectively. Notice that an extra max pooling is followed by cell1, indicating x1 is not + "actually" the direct output of cell1. + + .. code-block:: python + + class Cell(MutableScope): + pass + + class Net(nn.Module): + def __init__(self): + self.cell1 = Cell("cell1") + self.cell2 = Cell("cell2") + self.op = LayerChoice([conv3x3(), conv5x5()], key="op") + self.input_choice = InputChoice(choose_from=["cell1", "cell2", "op", InputChoice.NO_KEY]) + + def forward(self, x): + x1 = max_pooling(self.cell1(x)) + x2 = self.cell2(x) + x3 = self.op(x) + x4 = torch.zeros_like(x) + return self.input_choice([x1, x2, x3, x4]) + + Parameters + ---------- + n_candidates : int + Number of inputs to choose from. + choose_from : list of str + List of source keys to choose from. At least of one of ``choose_from`` and ``n_candidates`` must be fulfilled. + If ``n_candidates`` has a value but ``choose_from`` is None, it will be automatically treated as ``n_candidates`` + number of empty string. + n_chosen : int + Recommended inputs to choose. If None, mutator is instructed to select any. + reduction : str + ``mean``, ``concat``, ``sum`` or ``none``. See :class:`LayerChoice`. + return_mask : bool + If ``return_mask``, return output tensor and a mask. Otherwise return tensor only. + key : str + Key of the input choice. + """ + + NO_KEY = "" + + def __init__(self, n_candidates=None, choose_from=None, n_chosen=None, + reduction="sum", return_mask=False, key=None): + super().__init__(key=key) + # precondition check + assert n_candidates is not None or choose_from is not None, "At least one of `n_candidates` and `choose_from`" \ + "must be not None." + if choose_from is not None and n_candidates is None: + n_candidates = len(choose_from) + elif choose_from is None and n_candidates is not None: + choose_from = [self.NO_KEY] * n_candidates + assert n_candidates == len(choose_from), "Number of candidates must be equal to the length of `choose_from`." + assert n_candidates > 0, "Number of candidates must be greater than 0." + assert n_chosen is None or 0 <= n_chosen <= n_candidates, "Expected selected number must be None or no more " \ + "than number of candidates." + + self.n_candidates = n_candidates + self.choose_from = choose_from.copy() + self.n_chosen = n_chosen + self.reduction = reduction + self.return_mask = return_mask + + def forward(self, optional_inputs): + """ + Forward method of LayerChoice. + + Parameters + ---------- + optional_inputs : list or dict + Recommended to be a dict. As a dict, inputs will be converted to a list that follows the order of + ``choose_from`` in initialization. As a list, inputs must follow the semantic order that is the same as + ``choose_from``. + + Returns + ------- + tuple of tensors + Output and selection mask. If ``return_mask`` is ``False``, only output is returned. + """ + optional_input_list = optional_inputs + if isinstance(optional_inputs, dict): + optional_input_list = [optional_inputs[tag] for tag in self.choose_from] + assert isinstance(optional_input_list, list), \ + "Optional input list must be a list, not a {}.".format(type(optional_input_list)) + assert len(optional_inputs) == self.n_candidates, \ + "Length of the input list must be equal to number of candidates." + out, mask = self.mutator.on_forward_input_choice(self, optional_input_list) + if self.return_mask: + return out, mask + return out diff --git a/utils/third_party/nni_new/nas/pytorch/mutator.py b/utils/third_party/nni_new/nas/pytorch/mutator.py new file mode 100644 index 0000000000000000000000000000000000000000..e1894b52496ae63611178ccdf301cd4f612abc9e --- /dev/null +++ b/utils/third_party/nni_new/nas/pytorch/mutator.py @@ -0,0 +1,308 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +from collections import defaultdict + +import numpy as np +import torch + +from .base_mutator import BaseMutator +from .mutables import LayerChoice, InputChoice +from .utils import to_list + +logger = logging.getLogger(__name__) + + +class Mutator(BaseMutator): + + def __init__(self, model): + super().__init__(model) + self._cache = dict() + self._connect_all = False + + def sample_search(self): + """ + Override to implement this method to iterate over mutables and make decisions. + + Returns + ------- + dict + A mapping from key of mutables to decisions. + """ + raise NotImplementedError + + def sample_final(self): + """ + Override to implement this method to iterate over mutables and make decisions that is final + for export and retraining. + + Returns + ------- + dict + A mapping from key of mutables to decisions. + """ + raise NotImplementedError + + def reset(self): + """ + Reset the mutator by call the `sample_search` to resample (for search). Stores the result in a local + variable so that `on_forward_layer_choice` and `on_forward_input_choice` can use the decision directly. + """ + self._cache = self.sample_search() + + def export(self): + """ + Resample (for final) and return results. + + Returns + ------- + dict + A mapping from key of mutables to decisions. + """ + sampled = self.sample_final() + result = dict() + for mutable in self.mutables: + if not isinstance(mutable, (LayerChoice, InputChoice)): + # not supported as built-in + continue + result[mutable.key] = self._convert_mutable_decision_to_human_readable(mutable, sampled.pop(mutable.key)) + if sampled: + raise ValueError("Unexpected keys returned from 'sample_final()': %s", list(sampled.keys())) + return result + + def status(self): + """ + Return current selection status of mutator. + + Returns + ------- + dict + A mapping from key of mutables to decisions. All weights (boolean type and float type) + are converted into real number values. Numpy arrays and tensors are converted into list. + """ + data = dict() + for k, v in self._cache.items(): + if torch.is_tensor(v): + v = v.detach().cpu().numpy().tolist() + if isinstance(v, np.ndarray): + v = v.astype(np.float32).tolist() + data[k] = v + return data + + def graph(self, inputs): + """ + Return model supernet graph. + + Parameters + ---------- + inputs: tuple of tensor + Inputs that will be feeded into the network. + + Returns + ------- + dict + Containing ``node``, in Tensorboard GraphDef format. + Additional key ``mutable`` is a map from key to list of modules. + """ + if not torch.__version__.startswith("1.4"): + logger.warning("Graph is only tested with PyTorch 1.4. Other versions might not work.") + from nni.common.graph_utils import build_graph + from google.protobuf import json_format + # protobuf should be installed as long as tensorboard is installed + try: + self._connect_all = True + graph_def, _ = build_graph(self.model, inputs, verbose=False) + result = json_format.MessageToDict(graph_def) + finally: + self._connect_all = False + + # `mutable` is to map the keys to a list of corresponding modules. + # A key can be linked to multiple modules, use `dedup=False` to find them all. + result["mutable"] = defaultdict(list) + for mutable in self.mutables.traverse(deduplicate=False): + # A module will be represent in the format of + # [{"type": "Net", "name": ""}, {"type": "Cell", "name": "cell1"}, {"type": "Conv2d": "name": "conv"}] + # which will be concatenated into Net/Cell[cell1]/Conv2d[conv] in frontend. + # This format is aligned with the scope name jit gives. + modules = mutable.name.split(".") + path = [ + {"type": self.model.__class__.__name__, "name": ""} + ] + m = self.model + for module in modules: + m = getattr(m, module) + path.append({ + "type": m.__class__.__name__, + "name": module + }) + result["mutable"][mutable.key].append(path) + return result + + def on_forward_layer_choice(self, mutable, *args, **kwargs): + """ + On default, this method retrieves the decision obtained previously, and select certain operations. + Only operations with non-zero weight will be executed. The results will be added to a list. + Then it will reduce the list of all tensor outputs with the policy specified in `mutable.reduction`. + + Parameters + ---------- + mutable : nni.nas.pytorch.mutables.LayerChoice + Layer choice module. + args : list of torch.Tensor + Inputs + kwargs : dict + Inputs + + Returns + ------- + tuple of torch.Tensor and torch.Tensor + Output and mask. + """ + if self._connect_all: + return self._all_connect_tensor_reduction(mutable.reduction, + [op(*args, **kwargs) for op in mutable]), \ + torch.ones(len(mutable)).bool() + + def _map_fn(op, args, kwargs): + return op(*args, **kwargs) + + mask = self._get_decision(mutable) + assert len(mask) == len(mutable), \ + "Invalid mask, expected {} to be of length {}.".format(mask, len(mutable)) + out, mask = self._select_with_mask(_map_fn, [(choice, args, kwargs) for choice in mutable], mask) + return self._tensor_reduction(mutable.reduction, out), mask + + def on_forward_input_choice(self, mutable, tensor_list): + """ + On default, this method retrieves the decision obtained previously, and select certain tensors. + Then it will reduce the list of all tensor outputs with the policy specified in `mutable.reduction`. + + Parameters + ---------- + mutable : nni.nas.pytorch.mutables.InputChoice + Input choice module. + tensor_list : list of torch.Tensor + Tensor list to apply the decision on. + + Returns + ------- + tuple of torch.Tensor and torch.Tensor + Output and mask. + """ + if self._connect_all: + return self._all_connect_tensor_reduction(mutable.reduction, tensor_list), \ + torch.ones(mutable.n_candidates).bool() + mask = self._get_decision(mutable) + assert len(mask) == mutable.n_candidates, \ + "Invalid mask, expected {} to be of length {}.".format(mask, mutable.n_candidates) + out, mask = self._select_with_mask(lambda x: x, [(t,) for t in tensor_list], mask) + return self._tensor_reduction(mutable.reduction, out), mask + + def _select_with_mask(self, map_fn, candidates, mask): + """ + Select masked tensors and return a list of tensors. + + Parameters + ---------- + map_fn : function + Convert candidates to target candidates. Can be simply identity. + candidates : list of torch.Tensor + Tensor list to apply the decision on. + mask : list-like object + Can be a list, an numpy array or a tensor (recommended). Needs to + have the same length as ``candidates``. + + Returns + ------- + tuple of list of torch.Tensor and torch.Tensor + Output and mask. + """ + if (isinstance(mask, list) and len(mask) >= 1 and isinstance(mask[0], bool)) or \ + (isinstance(mask, np.ndarray) and mask.dtype == np.bool) or \ + "BoolTensor" in mask.type(): + out = [map_fn(*cand) for cand, m in zip(candidates, mask) if m] + elif (isinstance(mask, list) and len(mask) >= 1 and isinstance(mask[0], (float, int))) or \ + (isinstance(mask, np.ndarray) and mask.dtype in (np.float32, np.float64, np.int32, np.int64)) or \ + "FloatTensor" in mask.type(): + out = [map_fn(*cand) * m for cand, m in zip(candidates, mask) if m] + else: + raise ValueError("Unrecognized mask '%s'" % mask) + if not torch.is_tensor(mask): + mask = torch.tensor(mask) # pylint: disable=not-callable + return out, mask + + def _tensor_reduction(self, reduction_type, tensor_list): + if reduction_type == "none": + return tensor_list + if not tensor_list: + return None # empty. return None for now + if len(tensor_list) == 1: + return tensor_list[0] + if reduction_type == "sum": + return sum(tensor_list) + if reduction_type == "mean": + return sum(tensor_list) / len(tensor_list) + if reduction_type == "concat": + return torch.cat(tensor_list, dim=1) + raise ValueError("Unrecognized reduction policy: \"{}\"".format(reduction_type)) + + def _all_connect_tensor_reduction(self, reduction_type, tensor_list): + if reduction_type == "none": + return tensor_list + if reduction_type == "concat": + return torch.cat(tensor_list, dim=1) + return torch.stack(tensor_list).sum(0) + + def _get_decision(self, mutable): + """ + By default, this method checks whether `mutable.key` is already in the decision cache, + and returns the result without double-check. + + Parameters + ---------- + mutable : Mutable + + Returns + ------- + object + """ + if mutable.key not in self._cache: + raise ValueError("\"{}\" not found in decision cache.".format(mutable.key)) + result = self._cache[mutable.key] + logger.debug("Decision %s: %s", mutable.key, result) + return result + + def _convert_mutable_decision_to_human_readable(self, mutable, sampled): + # Assert the existence of mutable.key in returned architecture. + # Also check if there is anything extra. + multihot_list = to_list(sampled) + converted = None + # If it's a boolean array, we can do optimization. + if all([t == 0 or t == 1 for t in multihot_list]): + if isinstance(mutable, LayerChoice): + assert len(multihot_list) == len(mutable), \ + "Results returned from 'sample_final()' (%s: %s) either too short or too long." \ + % (mutable.key, multihot_list) + # check if all modules have different names and they indeed have names + if len(set(mutable.names)) == len(mutable) and not all(d.isdigit() for d in mutable.names): + converted = [name for i, name in enumerate(mutable.names) if multihot_list[i]] + else: + converted = [i for i in range(len(multihot_list)) if multihot_list[i]] + if isinstance(mutable, InputChoice): + assert len(multihot_list) == mutable.n_candidates, \ + "Results returned from 'sample_final()' (%s: %s) either too short or too long." \ + % (mutable.key, multihot_list) + # check if all input candidates have different names + if len(set(mutable.choose_from)) == mutable.n_candidates: + converted = [name for i, name in enumerate(mutable.choose_from) if multihot_list[i]] + else: + converted = [i for i in range(len(multihot_list)) if multihot_list[i]] + if converted is not None: + # if only one element, then remove the bracket + if len(converted) == 1: + converted = converted[0] + else: + # do nothing + converted = multihot_list + return converted diff --git a/utils/third_party/nni_new/nas/pytorch/nasbench201/__init__.py b/utils/third_party/nni_new/nas/pytorch/nasbench201/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..1419b3fa92420753a9c0ebea60944023ff0c313e --- /dev/null +++ b/utils/third_party/nni_new/nas/pytorch/nasbench201/__init__.py @@ -0,0 +1 @@ +from .nasbench201 import NASBench201Cell diff --git a/utils/third_party/nni_new/nas/pytorch/nasbench201/nasbench201.py b/utils/third_party/nni_new/nas/pytorch/nasbench201/nasbench201.py new file mode 100644 index 0000000000000000000000000000000000000000..cd42fa1a2f20b86978f1d7fd44ca630c01d54d4d --- /dev/null +++ b/utils/third_party/nni_new/nas/pytorch/nasbench201/nasbench201.py @@ -0,0 +1,72 @@ +from collections import OrderedDict +import torch.nn as nn +from nni.nas.pytorch.mutables import LayerChoice + +from .nasbench201_ops import Pooling, ReLUConvBN, Zero, FactorizedReduce + + +class NASBench201Cell(nn.Module): + """ + Builtin cell structure of NAS Bench 201. One cell contains four nodes. The First node serves as an input node + accepting the output of the previous cell. And other nodes connect to all previous nodes with an edge that + represents an operation chosen from a set to transform the tensor from the source node to the target node. + Every node accepts all its inputs and adds them as its output. + + Parameters + --- + cell_id: str + the name of this cell + C_in: int + the number of input channels of the cell + C_out: int + the number of output channels of the cell + stride: int + stride of all convolution operations in the cell + bn_affine: bool + If set to ``True``, all ``torch.nn.BatchNorm2d`` in this cell will have learnable affine parameters. Default: True + bn_momentum: float + the value used for the running_mean and running_var computation. Default: 0.1 + bn_track_running_stats: bool + When set to ``True``, all ``torch.nn.BatchNorm2d`` in this cell tracks the running mean and variance. Default: True + """ + + def __init__(self, cell_id, C_in, C_out, stride, bn_affine=True, bn_momentum=0.1, bn_track_running_stats=True): + super(NASBench201Cell, self).__init__() + + self.NUM_NODES = 4 + self.layers = nn.ModuleList() + + OPS = lambda layer_idx: OrderedDict([ + ("none", Zero(C_in, C_out, stride)), + ("avg_pool_3x3", Pooling(C_in, C_out, stride if layer_idx == 0 else 1, bn_affine, bn_momentum, + bn_track_running_stats)), + ("conv_3x3", ReLUConvBN(C_in, C_out, 3, stride if layer_idx == 0 else 1, 1, 1, bn_affine, bn_momentum, + bn_track_running_stats)), + ("conv_1x1", ReLUConvBN(C_in, C_out, 1, stride if layer_idx == 0 else 1, 0, 1, bn_affine, bn_momentum, + bn_track_running_stats)), + ("skip_connect", nn.Identity() if stride == 1 and C_in == C_out + else FactorizedReduce(C_in, C_out, stride if layer_idx == 0 else 1, bn_affine, bn_momentum, + bn_track_running_stats)) + ]) + + for i in range(self.NUM_NODES): + node_ops = nn.ModuleList() + for j in range(0, i): + node_ops.append(LayerChoice(OPS(j), key="%d_%d" % (j, i), reduction="mean")) + self.layers.append(node_ops) + self.in_dim = C_in + self.out_dim = C_out + self.cell_id = cell_id + + def forward(self, input): # pylint: disable=W0622 + """ + Parameters + --- + input: torch.tensor + the output of the previous layer + """ + nodes = [input] + for i in range(1, self.NUM_NODES): + node_feature = sum(self.layers[i][k](nodes[k]) for k in range(i)) + nodes.append(node_feature) + return nodes[-1] diff --git a/utils/third_party/nni_new/nas/pytorch/nasbench201/nasbench201_ops.py b/utils/third_party/nni_new/nas/pytorch/nasbench201/nasbench201_ops.py new file mode 100644 index 0000000000000000000000000000000000000000..aa60f8854fc1cf3169884cdde139eaa97f78831e --- /dev/null +++ b/utils/third_party/nni_new/nas/pytorch/nasbench201/nasbench201_ops.py @@ -0,0 +1,146 @@ +import torch +import torch.nn as nn + + +class ReLUConvBN(nn.Module): + """ + Parameters + --- + C_in: int + the number of input channels + C_out: int + the number of output channels + stride: int + stride of the convolution + padding: int + zero-padding added to both sides of the input + dilation: int + spacing between kernel elements + bn_affine: bool + If set to ``True``, ``torch.nn.BatchNorm2d`` will have learnable affine parameters. Default: True + bn_momentun: float + the value used for the running_mean and running_var computation. Default: 0.1 + bn_track_running_stats: bool + When set to ``True``, ``torch.nn.BatchNorm2d`` tracks the running mean and variance. Default: True + """ + def __init__(self, C_in, C_out, kernel_size, stride, padding, dilation, + bn_affine=True, bn_momentum=0.1, bn_track_running_stats=True): + super(ReLUConvBN, self).__init__() + self.op = nn.Sequential( + nn.ReLU(inplace=False), + nn.Conv2d(C_in, C_out, kernel_size, stride=stride, + padding=padding, dilation=dilation, bias=False), + nn.BatchNorm2d(C_out, affine=bn_affine, momentum=bn_momentum, + track_running_stats=bn_track_running_stats) + ) + + def forward(self, x): + """ + Parameters + --- + x: torch.Tensor + input tensor + """ + return self.op(x) + + +class Pooling(nn.Module): + """ + Parameters + --- + C_in: int + the number of input channels + C_out: int + the number of output channels + stride: int + stride of the convolution + bn_affine: bool + If set to ``True``, ``torch.nn.BatchNorm2d`` will have learnable affine parameters. Default: True + bn_momentun: float + the value used for the running_mean and running_var computation. Default: 0.1 + bn_track_running_stats: bool + When set to ``True``, ``torch.nn.BatchNorm2d`` tracks the running mean and variance. Default: True + """ + def __init__(self, C_in, C_out, stride, bn_affine=True, bn_momentum=0.1, bn_track_running_stats=True): + super(Pooling, self).__init__() + if C_in == C_out: + self.preprocess = None + else: + self.preprocess = ReLUConvBN(C_in, C_out, 1, 1, 0, 0, + bn_affine, bn_momentum, bn_track_running_stats) + self.op = nn.AvgPool2d(3, stride=stride, padding=1, count_include_pad=False) + + def forward(self, x): + """ + Parameters + --- + x: torch.Tensor + input tensor + """ + if self.preprocess: + x = self.preprocess(x) + return self.op(x) + + +class Zero(nn.Module): + """ + Parameters + --- + C_in: int + the number of input channels + C_out: int + the number of output channels + stride: int + stride of the convolution + """ + def __init__(self, C_in, C_out, stride): + super(Zero, self).__init__() + self.C_in = C_in + self.C_out = C_out + self.stride = stride + self.is_zero = True + + def forward(self, x): + """ + Parameters + --- + x: torch.Tensor + input tensor + """ + if self.C_in == self.C_out: + if self.stride == 1: + return x.mul(0.) + else: + return x[:, :, ::self.stride, ::self.stride].mul(0.) + else: + shape = list(x.shape) + shape[1] = self.C_out + zeros = x.new_zeros(shape, dtype=x.dtype, device=x.device) + return zeros + + +class FactorizedReduce(nn.Module): + def __init__(self, C_in, C_out, stride, bn_affine=True, bn_momentum=0.1, + bn_track_running_stats=True): + super(FactorizedReduce, self).__init__() + self.stride = stride + self.C_in = C_in + self.C_out = C_out + self.relu = nn.ReLU(inplace=False) + if stride == 2: + C_outs = [C_out // 2, C_out - C_out // 2] + self.convs = nn.ModuleList() + for i in range(2): + self.convs.append(nn.Conv2d(C_in, C_outs[i], 1, stride=stride, padding=0, bias=False)) + self.pad = nn.ConstantPad2d((0, 1, 0, 1), 0) + else: + raise ValueError("Invalid stride : {:}".format(stride)) + self.bn = nn.BatchNorm2d(C_out, affine=bn_affine, momentum=bn_momentum, + track_running_stats=bn_track_running_stats) + + def forward(self, x): + x = self.relu(x) + y = self.pad(x) + out = torch.cat([self.convs[0](x), self.convs[1](y[:, :, 1:, 1:])], dim=1) + out = self.bn(out) + return out diff --git a/utils/third_party/nni_new/nas/pytorch/search_space_zoo/__init__.py b/utils/third_party/nni_new/nas/pytorch/search_space_zoo/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..59bb3b78d1874808c9ac9ed2c2b625786f16250e --- /dev/null +++ b/utils/third_party/nni_new/nas/pytorch/search_space_zoo/__init__.py @@ -0,0 +1,4 @@ +from .darts_cell import DartsCell +from .enas_cell import ENASMicroLayer +from .enas_cell import ENASMacroLayer +from .enas_cell import ENASMacroGeneralModel diff --git a/utils/third_party/nni_new/nas/pytorch/search_space_zoo/darts_cell.py b/utils/third_party/nni_new/nas/pytorch/search_space_zoo/darts_cell.py new file mode 100644 index 0000000000000000000000000000000000000000..53fca5940c1f85b38cef2d08e8e20c8499728609 --- /dev/null +++ b/utils/third_party/nni_new/nas/pytorch/search_space_zoo/darts_cell.py @@ -0,0 +1,112 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from collections import OrderedDict + +import torch +import torch.nn as nn +from nni.nas.pytorch import mutables + +from .darts_ops import PoolBN, SepConv, DilConv, FactorizedReduce, DropPath, StdConv + + +class Node(nn.Module): + def __init__(self, node_id, num_prev_nodes, channels, num_downsample_connect): + """ + builtin Darts Node structure + + Parameters + --- + node_id: str + num_prev_nodes: int + the number of previous nodes in this cell + channels: int + output channels + num_downsample_connect: int + downsample the input node if this cell is reduction cell + """ + super().__init__() + self.ops = nn.ModuleList() + choice_keys = [] + for i in range(num_prev_nodes): + stride = 2 if i < num_downsample_connect else 1 + choice_keys.append("{}_p{}".format(node_id, i)) + self.ops.append( + mutables.LayerChoice(OrderedDict([ + ("maxpool", PoolBN('max', channels, 3, stride, 1, affine=False)), + ("avgpool", PoolBN('avg', channels, 3, stride, 1, affine=False)), + ("skipconnect", + nn.Identity() if stride == 1 else FactorizedReduce(channels, channels, affine=False)), + ("sepconv3x3", SepConv(channels, channels, 3, stride, 1, affine=False)), + ("sepconv5x5", SepConv(channels, channels, 5, stride, 2, affine=False)), + ("dilconv3x3", DilConv(channels, channels, 3, stride, 2, 2, affine=False)), + ("dilconv5x5", DilConv(channels, channels, 5, stride, 4, 2, affine=False)) + ]), key=choice_keys[-1])) + self.drop_path = DropPath() + self.input_switch = mutables.InputChoice(choose_from=choice_keys, n_chosen=2, key="{}_switch".format(node_id)) + + def forward(self, prev_nodes): + assert len(self.ops) == len(prev_nodes) + out = [op(node) for op, node in zip(self.ops, prev_nodes)] + out = [self.drop_path(o) if o is not None else None for o in out] + return self.input_switch(out) + + +class DartsCell(nn.Module): + """ + Builtin Darts Cell structure. There are ``n_nodes`` nodes in one cell, in which the first two nodes' values are + fixed to the results of previous previous cell and previous cell respectively. One node will connect all + the nodes after with predefined operations in a mutable way. The last node accepts five inputs from nodes + before and it concats all inputs in channels as the output of the current cell, and the number of output + channels is ``n_nodes`` times ``channels``. + + Parameters + --- + n_nodes: int + the number of nodes contained in this cell + channels_pp: int + the number of previous previous cell's output channels + channels_p: int + the number of previous cell's output channels + channels: int + the number of output channels for each node + reduction_p: bool + Is previous cell a reduction cell + reduction: bool + is current cell a reduction cell + """ + def __init__(self, n_nodes, channels_pp, channels_p, channels, reduction_p, reduction): + super().__init__() + self.reduction = reduction + self.n_nodes = n_nodes + + # If previous cell is reduction cell, current input size does not match with + # output size of cell[k-2]. So the output[k-2] should be reduced by preprocessing. + if reduction_p: + self.preproc0 = FactorizedReduce(channels_pp, channels, affine=False) + else: + self.preproc0 = StdConv(channels_pp, channels, 1, 1, 0, affine=False) + self.preproc1 = StdConv(channels_p, channels, 1, 1, 0, affine=False) + + # generate dag + self.mutable_ops = nn.ModuleList() + for depth in range(2, self.n_nodes + 2): + self.mutable_ops.append(Node("{}_n{}".format("reduce" if reduction else "normal", depth), + depth, channels, 2 if reduction else 0)) + + def forward(self, pprev, prev): + """ + Parameters + --- + pprev: torch.Tensor + the output of the previous previous layer + prev: torch.Tensor + the output of the previous layer + """ + tensors = [self.preproc0(pprev), self.preproc1(prev)] + for node in self.mutable_ops: + cur_tensor = node(tensors) + tensors.append(cur_tensor) + + output = torch.cat(tensors[2:], dim=1) + return output diff --git a/utils/third_party/nni_new/nas/pytorch/search_space_zoo/darts_ops.py b/utils/third_party/nni_new/nas/pytorch/search_space_zoo/darts_ops.py new file mode 100644 index 0000000000000000000000000000000000000000..ce5410cfb4a5a466f26e67aaf93d21f90672407c --- /dev/null +++ b/utils/third_party/nni_new/nas/pytorch/search_space_zoo/darts_ops.py @@ -0,0 +1,196 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import torch +import torch.nn as nn + + +class DropPath(nn.Module): + def __init__(self, p=0.): + """ + Drop path with probability. + + Parameters + ---------- + p : float + Probability of an path to be zeroed. + """ + super().__init__() + self.p = p + + def forward(self, x): + if self.training and self.p > 0.: + keep_prob = 1. - self.p + # per data point mask + mask = torch.zeros((x.size(0), 1, 1, 1), device=x.device).bernoulli_(keep_prob) + return x / keep_prob * mask + + return x + + +class PoolBN(nn.Module): + """ + AvgPool or MaxPool with BN. ``pool_type`` must be ``max`` or ``avg``. + + Parameters + --- + pool_type: str + choose operation + C: int + number of channels + kernal_size: int + size of the convolving kernel + stride: int + stride of the convolution + padding: int + zero-padding added to both sides of the input + affine: bool + is using affine in BatchNorm + """ + + def __init__(self, pool_type, C, kernel_size, stride, padding, affine=True): + super().__init__() + if pool_type.lower() == 'max': + self.pool = nn.MaxPool2d(kernel_size, stride, padding) + elif pool_type.lower() == 'avg': + self.pool = nn.AvgPool2d(kernel_size, stride, padding, count_include_pad=False) + else: + raise ValueError() + + self.bn = nn.BatchNorm2d(C, affine=affine) + + def forward(self, x): + out = self.pool(x) + out = self.bn(out) + return out + + +class StdConv(nn.Sequential): + """ + Standard conv: ReLU - Conv - BN + + Parameters + --- + C_in: int + the number of input channels + C_out: int + the number of output channels + kernel_size: int + size of the convolution kernel + padding: + zero-padding added to both sides of the input + affine: bool + is using affine in BatchNorm + """ + + def __init__(self, C_in, C_out, kernel_size, stride, padding, affine=True): + super().__init__() + self.net = nn.Sequential + for idx, ops in enumerate((nn.ReLU(), nn.Conv2d(C_in, C_out, kernel_size, stride, padding, bias=False), + nn.BatchNorm2d(C_out, affine=affine))): + self.add_module(str(idx), ops) + + +class FacConv(nn.Module): + """ + Factorized conv: ReLU - Conv(Kx1) - Conv(1xK) - BN + """ + + def __init__(self, C_in, C_out, kernel_length, stride, padding, affine=True): + super().__init__() + self.net = nn.Sequential( + nn.ReLU(), + nn.Conv2d(C_in, C_in, (kernel_length, 1), stride, padding, bias=False), + nn.Conv2d(C_in, C_out, (1, kernel_length), stride, padding, bias=False), + nn.BatchNorm2d(C_out, affine=affine) + ) + + def forward(self, x): + return self.net(x) + + +class DilConv(nn.Module): + """ + (Dilated) depthwise separable conv. + ReLU - (Dilated) depthwise separable - Pointwise - BN. + If dilation == 2, 3x3 conv => 5x5 receptive field, 5x5 conv => 9x9 receptive field. + + Parameters + --- + C_in: int + the number of input channels + C_out: int + the number of output channels + kernal_size: + size of the convolving kernel + padding: + zero-padding added to both sides of the input + dilation: int + spacing between kernel elements. + affine: bool + is using affine in BatchNorm + """ + + def __init__(self, C_in, C_out, kernel_size, stride, padding, dilation, affine=True): + super().__init__() + self.net = nn.Sequential( + nn.ReLU(), + nn.Conv2d(C_in, C_in, kernel_size, stride, padding, dilation=dilation, groups=C_in, + bias=False), + nn.Conv2d(C_in, C_out, 1, stride=1, padding=0, bias=False), + nn.BatchNorm2d(C_out, affine=affine) + ) + + def forward(self, x): + return self.net(x) + + +class SepConv(nn.Module): + """ + Depthwise separable conv. + DilConv(dilation=1) * 2. + + Parameters + --- + C_in: int + the number of input channels + C_out: int + the number of output channels + kernal_size: + size of the convolving kernel + padding: + zero-padding added to both sides of the input + dilation: int + spacing between kernel elements. + affine: bool + is using affine in BatchNorm + """ + + def __init__(self, C_in, C_out, kernel_size, stride, padding, affine=True): + super().__init__() + self.net = nn.Sequential( + DilConv(C_in, C_in, kernel_size, stride, padding, dilation=1, affine=affine), + DilConv(C_in, C_out, kernel_size, 1, padding, dilation=1, affine=affine) + ) + + def forward(self, x): + return self.net(x) + + +class FactorizedReduce(nn.Module): + """ + Reduce feature map size by factorized pointwise (stride=2). + """ + + def __init__(self, C_in, C_out, affine=True): + super().__init__() + self.relu = nn.ReLU() + self.conv1 = nn.Conv2d(C_in, C_out // 2, 1, stride=2, padding=0, bias=False) + self.conv2 = nn.Conv2d(C_in, C_out // 2, 1, stride=2, padding=0, bias=False) + self.bn = nn.BatchNorm2d(C_out, affine=affine) + + def forward(self, x): + x = self.relu(x) + out = torch.cat([self.conv1(x), self.conv2(x[:, :, 1:, 1:])], dim=1) + out = self.bn(out) + return out diff --git a/utils/third_party/nni_new/nas/pytorch/search_space_zoo/enas_cell.py b/utils/third_party/nni_new/nas/pytorch/search_space_zoo/enas_cell.py new file mode 100644 index 0000000000000000000000000000000000000000..de57d55e23f83bb5a35cd4d3a474c71b8fc90fc4 --- /dev/null +++ b/utils/third_party/nni_new/nas/pytorch/search_space_zoo/enas_cell.py @@ -0,0 +1,255 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from nni.nas.pytorch import mutables +from .enas_ops import FactorizedReduce, StdConv, SepConvBN, Pool, ConvBranch, PoolBranch + + +class Cell(nn.Module): + def __init__(self, cell_name, prev_labels, channels): + super().__init__() + self.input_choice = mutables.InputChoice(choose_from=prev_labels, n_chosen=1, return_mask=True, + key=cell_name + "_input") + self.op_choice = mutables.LayerChoice([ + SepConvBN(channels, channels, 3, 1), + SepConvBN(channels, channels, 5, 2), + Pool("avg", 3, 1, 1), + Pool("max", 3, 1, 1), + nn.Identity() + ], key=cell_name + "_op") + + def forward(self, prev_layers): + chosen_input, chosen_mask = self.input_choice(prev_layers) + cell_out = self.op_choice(chosen_input) + return cell_out, chosen_mask + + +class Node(mutables.MutableScope): + def __init__(self, node_name, prev_node_names, channels): + super().__init__(node_name) + self.cell_x = Cell(node_name + "_x", prev_node_names, channels) + self.cell_y = Cell(node_name + "_y", prev_node_names, channels) + + def forward(self, prev_layers): + out_x, mask_x = self.cell_x(prev_layers) + out_y, mask_y = self.cell_y(prev_layers) + return out_x + out_y, mask_x | mask_y + + +class Calibration(nn.Module): + def __init__(self, in_channels, out_channels): + super().__init__() + self.process = None + if in_channels != out_channels: + self.process = StdConv(in_channels, out_channels) + + def forward(self, x): + if self.process is None: + return x + return self.process(x) + + +class ENASMicroLayer(nn.Module): + """ + Builtin EnasMicroLayer. Micro search designs only one building block whose architecture is repeated + throughout the final architecture. A cell has ``num_nodes`` nodes and searches the topology and + operations among them in RL way. The first two nodes in a layer stand for the outputs from previous + previous layer and previous layer respectively. For the following nodes, the controller chooses + two previous nodes and applies two operations respectively for each node. Nodes that are not served + as input for any other node are viewed as the output of the layer. If there are multiple output nodes, + the model will calculate the average of these nodes as the layer output. Every node's output has ``out_channels`` + channels so the result of the layer has the same number of channels as each node. + + Parameters + --- + num_nodes: int + the number of nodes contained in this layer + in_channles_pp: int + the number of previous previous layer's output channels + in_channels_p: int + the number of previous layer's output channels + out_channels: int + output channels of this layer + reduction: bool + is reduction operation empolyed before this layer + """ + def __init__(self, num_nodes, in_channels_pp, in_channels_p, out_channels, reduction): + super().__init__() + self.reduction = reduction + if self.reduction: + self.reduce0 = FactorizedReduce(in_channels_pp, out_channels, affine=False) + self.reduce1 = FactorizedReduce(in_channels_p, out_channels, affine=False) + in_channels_pp = in_channels_p = out_channels + self.preproc0 = Calibration(in_channels_pp, out_channels) + self.preproc1 = Calibration(in_channels_p, out_channels) + + self.num_nodes = num_nodes + name_prefix = "reduce" if reduction else "normal" + self.nodes = nn.ModuleList() + node_labels = [mutables.InputChoice.NO_KEY, mutables.InputChoice.NO_KEY] + for i in range(num_nodes): + node_labels.append("{}_node_{}".format(name_prefix, i)) + self.nodes.append(Node(node_labels[-1], node_labels[:-1], out_channels)) + self.final_conv_w = nn.Parameter(torch.zeros(out_channels, self.num_nodes + 2, out_channels, 1, 1), + requires_grad=True) + self.bn = nn.BatchNorm2d(out_channels, affine=False) + self.reset_parameters() + + def reset_parameters(self): + nn.init.kaiming_normal_(self.final_conv_w) + + def forward(self, pprev, prev): + """ + Parameters + --- + pprev: torch.Tensor + the output of the previous previous layer + prev: torch.Tensor + the output of the previous layer + """ + if self.reduction: + pprev, prev = self.reduce0(pprev), self.reduce1(prev) + pprev_, prev_ = self.preproc0(pprev), self.preproc1(prev) + + prev_nodes_out = [pprev_, prev_] + nodes_used_mask = torch.zeros(self.num_nodes + 2, dtype=torch.bool, device=prev.device) + for i in range(self.num_nodes): + node_out, mask = self.nodes[i](prev_nodes_out) + nodes_used_mask[:mask.size(0)] |= mask.to(node_out.device) + prev_nodes_out.append(node_out) + + unused_nodes = torch.cat([out for used, out in zip(nodes_used_mask, prev_nodes_out) if not used], 1) + unused_nodes = F.relu(unused_nodes) + conv_weight = self.final_conv_w[:, ~nodes_used_mask, :, :, :] + conv_weight = conv_weight.view(conv_weight.size(0), -1, 1, 1) + out = F.conv2d(unused_nodes, conv_weight) + return prev, self.bn(out) + + +class ENASMacroLayer(mutables.MutableScope): + """ + Builtin ENAS Marco Layer. With search space changing to layer level, the controller decides + what operation is employed and the previous layer to connect to for skip connections. The model + is made up of the same layers but the choice of each layer may be different. + + Parameters + --- + key: str + the name of this layer + prev_labels: str + names of all previous layers + in_filters: int + the number of input channels + out_filters: + the number of output channels + """ + def __init__(self, key, prev_labels, in_filters, out_filters): + super().__init__(key) + self.in_filters = in_filters + self.out_filters = out_filters + self.mutable = mutables.LayerChoice([ + ConvBranch(in_filters, out_filters, 3, 1, 1, separable=False), + ConvBranch(in_filters, out_filters, 3, 1, 1, separable=True), + ConvBranch(in_filters, out_filters, 5, 1, 2, separable=False), + ConvBranch(in_filters, out_filters, 5, 1, 2, separable=True), + PoolBranch('avg', in_filters, out_filters, 3, 1, 1), + PoolBranch('max', in_filters, out_filters, 3, 1, 1) + ]) + if prev_labels: + self.skipconnect = mutables.InputChoice(choose_from=prev_labels, n_chosen=None) + else: + self.skipconnect = None + self.batch_norm = nn.BatchNorm2d(out_filters, affine=False) + + def forward(self, prev_list): + """ + Parameters + --- + prev_list: list + The cell selects the last element of the list as input and applies an operation on it. + The cell chooses none/one/multiple tensor(s) as SkipConnect(s) from the list excluding + the last element. + """ + out = self.mutable(prev_list[-1]) + if self.skipconnect is not None: + connection = self.skipconnect(prev_list[:-1]) + if connection is not None: + out += connection + return self.batch_norm(out) + + +class ENASMacroGeneralModel(nn.Module): + """ + The network is made up by stacking ENASMacroLayer. The Macro search space contains these layers. + Each layer chooses an operation from predefined ones and SkipConnect then forms a network. + + Parameters + --- + num_layers: int + The number of layers contained in the network. + out_filters: int + The number of each layer's output channels. + in_channel: int + The number of input's channels. + num_classes: int + The number of classes for classification. + dropout_rate: float + Dropout layer's dropout rate before the final dense layer. + """ + def __init__(self, num_layers=12, out_filters=24, in_channels=3, num_classes=10, + dropout_rate=0.0): + super().__init__() + self.num_layers = num_layers + self.num_classes = num_classes + self.out_filters = out_filters + + self.stem = nn.Sequential( + nn.Conv2d(in_channels, out_filters, 3, 1, 1, bias=False), + nn.BatchNorm2d(out_filters) + ) + + pool_distance = self.num_layers // 3 + self.pool_layers_idx = [pool_distance - 1, 2 * pool_distance - 1] + self.dropout_rate = dropout_rate + self.dropout = nn.Dropout(self.dropout_rate) + + self.layers = nn.ModuleList() + self.pool_layers = nn.ModuleList() + labels = [] + for layer_id in range(self.num_layers): + labels.append("layer_{}".format(layer_id)) + if layer_id in self.pool_layers_idx: + self.pool_layers.append(FactorizedReduce(self.out_filters, self.out_filters)) + self.layers.append(ENASMacroLayer(labels[-1], labels[:-1], self.out_filters, self.out_filters)) + + self.gap = nn.AdaptiveAvgPool2d(1) + self.dense = nn.Linear(self.out_filters, self.num_classes) + + def forward(self, x): + """ + Parameters + --- + x: torch.Tensor + the input of the network + """ + bs = x.size(0) + cur = self.stem(x) + + layers = [cur] + + for layer_id in range(self.num_layers): + cur = self.layers[layer_id](layers) + layers.append(cur) + if layer_id in self.pool_layers_idx: + for i, layer in enumerate(layers): + layers[i] = self.pool_layers[self.pool_layers_idx.index(layer_id)](layer) + cur = layers[-1] + + cur = self.gap(cur).view(bs, -1) + cur = self.dropout(cur) + logits = self.dense(cur) + return logits diff --git a/utils/third_party/nni_new/nas/pytorch/search_space_zoo/enas_ops.py b/utils/third_party/nni_new/nas/pytorch/search_space_zoo/enas_ops.py new file mode 100644 index 0000000000000000000000000000000000000000..21ecc2da798dd025966a4631d59fb6f652e5da98 --- /dev/null +++ b/utils/third_party/nni_new/nas/pytorch/search_space_zoo/enas_ops.py @@ -0,0 +1,171 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import torch +import torch.nn as nn + + +class StdConv(nn.Module): + def __init__(self, C_in, C_out): + super(StdConv, self).__init__() + self.conv = nn.Sequential( + nn.Conv2d(C_in, C_out, 1, stride=1, padding=0, bias=False), + nn.BatchNorm2d(C_out, affine=False), + nn.ReLU() + ) + + def forward(self, x): + return self.conv(x) + + +class PoolBranch(nn.Module): + """ + Pooling structure for Macro search. First pass through a 1x1 Conv, then pooling operation followed by BatchNorm2d. + + Parameters + --- + pool_type: str + only accept ``max`` for MaxPool and ``avg`` for AvgPool + C_in: int + the number of input channels + C_out: int + the number of output channels + kernal_size: int + size of the convolving kernel + stride: int + stride of the convolution + padding: int + zero-padding added to both sides of the input + """ + def __init__(self, pool_type, C_in, C_out, kernel_size, stride, padding, affine=False): + super().__init__() + self.preproc = StdConv(C_in, C_out) + self.pool = Pool(pool_type, kernel_size, stride, padding) + self.bn = nn.BatchNorm2d(C_out, affine=affine) + + def forward(self, x): + out = self.preproc(x) + out = self.pool(out) + out = self.bn(out) + return out + + +class SeparableConv(nn.Module): + def __init__(self, C_in, C_out, kernel_size, stride, padding): + super(SeparableConv, self).__init__() + self.depthwise = nn.Conv2d(C_in, C_in, kernel_size=kernel_size, padding=padding, stride=stride, + groups=C_in, bias=False) + self.pointwise = nn.Conv2d(C_in, C_out, kernel_size=1, bias=False) + + def forward(self, x): + out = self.depthwise(x) + out = self.pointwise(out) + return out + + +class ConvBranch(nn.Module): + """ + Conv structure for Macro search. First pass through a 1x1 Conv, + then Conv operation with kernal_size equals 3 or 5 followed by BatchNorm and ReLU. + + Parameters + --- + C_in: int + the number of input channels + C_out: int + the number of output channels + kernal_size: int + size of the convolving kernel + stride: int + stride of the convolution + padding: int + zero-padding added to both sides of the input + separable: True + is separable Conv is used + """ + def __init__(self, C_in, C_out, kernel_size, stride, padding, separable): + super(ConvBranch, self).__init__() + self.preproc = StdConv(C_in, C_out) + if separable: + self.conv = SeparableConv(C_out, C_out, kernel_size, stride, padding) + else: + self.conv = nn.Conv2d(C_out, C_out, kernel_size, stride=stride, padding=padding) + self.postproc = nn.Sequential( + nn.BatchNorm2d(C_out, affine=False), + nn.ReLU() + ) + + def forward(self, x): + out = self.preproc(x) + out = self.conv(out) + out = self.postproc(out) + return out + + +class FactorizedReduce(nn.Module): + def __init__(self, C_in, C_out, affine=False): + super().__init__() + self.conv1 = nn.Conv2d(C_in, C_out // 2, 1, stride=2, padding=0, bias=False) + self.conv2 = nn.Conv2d(C_in, C_out // 2, 1, stride=2, padding=0, bias=False) + self.bn = nn.BatchNorm2d(C_out, affine=affine) + + def forward(self, x): + out = torch.cat([self.conv1(x), self.conv2(x[:, :, 1:, 1:])], dim=1) + out = self.bn(out) + return out + + +class Pool(nn.Module): + """ + Pooling structure + + Parameters + --- + pool_type: str + only accept ``max`` for MaxPool and ``avg`` for AvgPool + kernal_size: int + size of the convolving kernel + stride: int + stride of the convolution + padding: int + zero-padding added to both sides of the input + """ + def __init__(self, pool_type, kernel_size, stride, padding): + super().__init__() + if pool_type.lower() == 'max': + self.pool = nn.MaxPool2d(kernel_size, stride, padding) + elif pool_type.lower() == 'avg': + self.pool = nn.AvgPool2d(kernel_size, stride, padding, count_include_pad=False) + else: + raise ValueError() + + def forward(self, x): + return self.pool(x) + + +class SepConvBN(nn.Module): + """ + Implement SepConv followed by BatchNorm. The structure is ReLU ==> SepConv ==> BN. + + Parameters + --- + C_in: int + the number of imput channels + C_out: int + the number of output channels + kernal_size: int + size of the convolving kernel + padding: int + zero-padding added to both sides of the input + """ + def __init__(self, C_in, C_out, kernel_size, padding): + super().__init__() + self.relu = nn.ReLU() + self.conv = SeparableConv(C_in, C_out, kernel_size, 1, padding) + self.bn = nn.BatchNorm2d(C_out, affine=True) + + def forward(self, x): + x = self.relu(x) + x = self.conv(x) + x = self.bn(x) + return x diff --git a/utils/third_party/nni_new/nas/pytorch/trainer.py b/utils/third_party/nni_new/nas/pytorch/trainer.py new file mode 100644 index 0000000000000000000000000000000000000000..6a3881177a63befdf1c32f8b53edc88b4f8fd379 --- /dev/null +++ b/utils/third_party/nni_new/nas/pytorch/trainer.py @@ -0,0 +1,194 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import json +import logging +import os +import time +from abc import abstractmethod + +import torch + +from .base_trainer import BaseTrainer + +_logger = logging.getLogger(__name__) + + +class TorchTensorEncoder(json.JSONEncoder): + def default(self, o): # pylint: disable=method-hidden + if isinstance(o, torch.Tensor): + olist = o.tolist() + if "bool" not in o.type().lower() and all(map(lambda d: d == 0 or d == 1, olist)): + _logger.warning("Every element in %s is either 0 or 1. " + "You might consider convert it into bool.", olist) + return olist + return super().default(o) + + +class Trainer(BaseTrainer): + """ + A trainer with some helper functions implemented. To implement a new trainer, + users need to implement :meth:`train_one_epoch`, :meth:`validate_one_epoch` and :meth:`checkpoint`. + + Parameters + ---------- + model : nn.Module + Model with mutables. + mutator : BaseMutator + A mutator object that has been initialized with the model. + loss : callable + Called with logits and targets. Returns a loss tensor. + See `PyTorch loss functions`_ for examples. + metrics : callable + Called with logits and targets. Returns a dict that maps metrics keys to metrics data. For example, + + .. code-block:: python + + def metrics_fn(output, target): + return {"acc1": accuracy(output, target, topk=1), "acc5": accuracy(output, target, topk=5)} + + optimizer : Optimizer + Optimizer that optimizes the model. + num_epochs : int + Number of epochs of training. + dataset_train : torch.utils.data.Dataset + Dataset of training. If not otherwise specified, ``dataset_train`` and ``dataset_valid`` should be standard + PyTorch Dataset. See `torch.utils.data`_ for examples. + dataset_valid : torch.utils.data.Dataset + Dataset of validation/testing. + batch_size : int + Batch size. + workers : int + Number of workers used in data preprocessing. + device : torch.device + Device object. Either ``torch.device("cuda")`` or ``torch.device("cpu")``. When ``None``, trainer will + automatic detects GPU and selects GPU first. + log_frequency : int + Number of mini-batches to log metrics. + callbacks : list of Callback + Callbacks to plug into the trainer. See Callbacks. + + + .. _`PyTorch loss functions`: https://pytorch.org/docs/stable/nn.html#loss-functions + .. _`torch.utils.data`: https://pytorch.org/docs/stable/data.html + """ + def __init__(self, model, mutator, loss, metrics, optimizer, num_epochs, + dataset_train, dataset_valid, batch_size, workers, device, log_frequency, callbacks): + self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") if device is None else device + self.model = model + self.mutator = mutator + self.loss = loss + + self.metrics = metrics + self.optimizer = optimizer + + self.model.to(self.device) + self.mutator.to(self.device) + self.loss.to(self.device) + + self.num_epochs = num_epochs + self.dataset_train = dataset_train + self.dataset_valid = dataset_valid + self.batch_size = batch_size + self.workers = workers + self.log_frequency = log_frequency + self.log_dir = os.path.join("logs", str(time.time())) + os.makedirs(self.log_dir, exist_ok=True) + self.status_writer = open(os.path.join(self.log_dir, "log"), "w") + self.callbacks = callbacks if callbacks is not None else [] + for callback in self.callbacks: + callback.build(self.model, self.mutator, self) + + @abstractmethod + def train_one_epoch(self, epoch): + """ + Train one epoch. + + Parameters + ---------- + epoch : int + Epoch number starting from 0. + """ + pass + + @abstractmethod + def validate_one_epoch(self, epoch): + """ + Validate one epoch. + + Parameters + ---------- + epoch : int + Epoch number starting from 0. + """ + pass + + def train(self, validate=True): + """ + Train ``num_epochs``. + Trigger callbacks at the start and the end of each epoch. + + Parameters + ---------- + validate : bool + If ``true``, will do validation every epoch. + """ + for epoch in range(self.num_epochs): + for callback in self.callbacks: + callback.on_epoch_begin(epoch) + + # training + _logger.info("Epoch %d Training", epoch + 1) + self.train_one_epoch(epoch) + + if validate: + # validation + _logger.info("Epoch %d Validating", epoch + 1) + self.validate_one_epoch(epoch) + + for callback in self.callbacks: + callback.on_epoch_end(epoch) + + def validate(self): + """ + Do one validation. + """ + self.validate_one_epoch(-1) + + def export(self, file): + """ + Call ``mutator.export()`` and dump the architecture to ``file``. + + Parameters + ---------- + file : str + A file path. Expected to be a JSON. + """ + mutator_export = self.mutator.export() + with open(file, "w") as f: + json.dump(mutator_export, f, indent=2, sort_keys=True, cls=TorchTensorEncoder) + + def checkpoint(self): + """ + Return trainer checkpoint. + """ + raise NotImplementedError("Not implemented yet") + + def enable_visualization(self): + """ + Enable visualization. Write graph and training log to folder ``logs/``. + """ + sample = None + for x, _ in self.train_loader: + sample = x.to(self.device)[:2] + break + if sample is None: + _logger.warning("Sample is %s.", sample) + _logger.info("Creating graph json, writing to %s. Visualization enabled.", self.log_dir) + with open(os.path.join(self.log_dir, "graph.json"), "w") as f: + json.dump(self.mutator.graph(sample), f) + self.visualization_enabled = True + + def _write_graph_status(self): + if hasattr(self, "visualization_enabled") and self.visualization_enabled: + print(json.dumps(self.mutator.status()), file=self.status_writer, flush=True) diff --git a/utils/third_party/nni_new/nas/pytorch/utils.py b/utils/third_party/nni_new/nas/pytorch/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..a3f5aabfb74e5c1d2350f579233ae36fdadb2313 --- /dev/null +++ b/utils/third_party/nni_new/nas/pytorch/utils.py @@ -0,0 +1,210 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +from collections import OrderedDict + +import numpy as np +import torch + +_counter = 0 + +_logger = logging.getLogger(__name__) + + +def global_mutable_counting(): + """ + A program level counter starting from 1. + """ + global _counter + _counter += 1 + return _counter + + +def _reset_global_mutable_counting(): + """ + Reset the global mutable counting to count from 1. Useful when defining multiple models with default keys. + """ + global _counter + _counter = 0 + + +def to_device(obj, device): + """ + Move a tensor, tuple, list, or dict onto device. + """ + if torch.is_tensor(obj): + return obj.to(device) + if isinstance(obj, tuple): + return tuple(to_device(t, device) for t in obj) + if isinstance(obj, list): + return [to_device(t, device) for t in obj] + if isinstance(obj, dict): + return {k: to_device(v, device) for k, v in obj.items()} + if isinstance(obj, (int, float, str)): + return obj + raise ValueError("'%s' has unsupported type '%s'" % (obj, type(obj))) + + +def to_list(arr): + if torch.is_tensor(arr): + return arr.cpu().numpy().tolist() + if isinstance(arr, np.ndarray): + return arr.tolist() + if isinstance(arr, (list, tuple)): + return list(arr) + return arr + + +class AverageMeterGroup: + """ + Average meter group for multiple average meters. + """ + + def __init__(self): + self.meters = OrderedDict() + + def update(self, data): + """ + Update the meter group with a dict of metrics. + Non-exist average meters will be automatically created. + """ + for k, v in data.items(): + if k not in self.meters: + self.meters[k] = AverageMeter(k, ":4f") + self.meters[k].update(v) + + def __getattr__(self, item): + return self.meters[item] + + def __getitem__(self, item): + return self.meters[item] + + def __str__(self): + return " ".join(str(v) for v in self.meters.values()) + + def summary(self): + """ + Return a summary string of group data. + """ + return " ".join(v.summary() for v in self.meters.values()) + + +class AverageMeter: + """ + Computes and stores the average and current value. + + Parameters + ---------- + name : str + Name to display. + fmt : str + Format string to print the values. + """ + + def __init__(self, name, fmt=':f'): + self.name = name + self.fmt = fmt + self.reset() + + def reset(self): + """ + Reset the meter. + """ + self.val = 0 + self.avg = 0 + self.sum = 0 + self.count = 0 + + def update(self, val, n=1): + """ + Update with value and weight. + + Parameters + ---------- + val : float or int + The new value to be accounted in. + n : int + The weight of the new value. + """ + self.val = val + self.sum += val * n + self.count += n + self.avg = self.sum / self.count + + def __str__(self): + fmtstr = '{name} {val' + self.fmt + '} ({avg' + self.fmt + '})' + return fmtstr.format(**self.__dict__) + + def summary(self): + fmtstr = '{name}: {avg' + self.fmt + '}' + return fmtstr.format(**self.__dict__) + + +class StructuredMutableTreeNode: + """ + A structured representation of a search space. + A search space comes with a root (with `None` stored in its `mutable`), and a bunch of children in its `children`. + This tree can be seen as a "flattened" version of the module tree. Since nested mutable entity is not supported yet, + the following must be true: each subtree corresponds to a ``MutableScope`` and each leaf corresponds to a + ``Mutable`` (other than ``MutableScope``). + + Parameters + ---------- + mutable : nni.nas.pytorch.mutables.Mutable + The mutable that current node is linked with. + """ + + def __init__(self, mutable): + self.mutable = mutable + self.children = [] + + def add_child(self, mutable): + """ + Add a tree node to the children list of current node. + """ + self.children.append(StructuredMutableTreeNode(mutable)) + return self.children[-1] + + def type(self): + """ + Return the ``type`` of mutable content. + """ + return type(self.mutable) + + def __iter__(self): + return self.traverse() + + def traverse(self, order="pre", deduplicate=True, memo=None): + """ + Return a generator that generates a list of mutables in this tree. + + Parameters + ---------- + order : str + pre or post. If pre, current mutable is yield before children. Otherwise after. + deduplicate : bool + If true, mutables with the same key will not appear after the first appearance. + memo : dict + An auxiliary dict that memorize keys seen before, so that deduplication is possible. + + Returns + ------- + generator of Mutable + """ + if memo is None: + memo = set() + assert order in ["pre", "post"] + if order == "pre": + if self.mutable is not None: + if not deduplicate or self.mutable.key not in memo: + memo.add(self.mutable.key) + yield self.mutable + for child in self.children: + for m in child.traverse(order=order, deduplicate=deduplicate, memo=memo): + yield m + if order == "post": + if self.mutable is not None: + if not deduplicate or self.mutable.key not in memo: + memo.add(self.mutable.key) + yield self.mutable diff --git a/utils/third_party/nni_new/nas/tensorflow/__init__.py b/utils/third_party/nni_new/nas/tensorflow/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/utils/third_party/nni_new/nas/tensorflow/base_mutator.py b/utils/third_party/nni_new/nas/tensorflow/base_mutator.py new file mode 100644 index 0000000000000000000000000000000000000000..860680f199278d3fd38910b82e7661b17d2f652e --- /dev/null +++ b/utils/third_party/nni_new/nas/tensorflow/base_mutator.py @@ -0,0 +1,73 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from tensorflow.keras import Model + +from .mutables import Mutable, MutableScope, InputChoice +from .utils import StructuredMutableTreeNode + + +class BaseMutator(Model): + def __init__(self, model): + super().__init__() + self.__dict__['model'] = model + self._structured_mutables = self._parse_search_space(self.model) + + def _parse_search_space(self, module, root=None, prefix='', memo=None, nested_detection=None): + if memo is None: + memo = set() + if root is None: + root = StructuredMutableTreeNode(None) + if module not in memo: + memo.add(module) + if isinstance(module, Mutable): + if nested_detection is not None: + raise RuntimeError('Cannot have nested search space. Error at {} in {}' + .format(module, nested_detection)) + module.name = prefix + module.set_mutator(self) + root = root.add_child(module) + if not isinstance(module, MutableScope): + nested_detection = module + if isinstance(module, InputChoice): + for k in module.choose_from: + if k != InputChoice.NO_KEY and k not in [m.key for m in memo if isinstance(m, Mutable)]: + raise RuntimeError('"{}" required by "{}" not found in keys that appeared before, and is not NO_KEY.' + .format(k, module.key)) + for submodule in module.layers: + if not isinstance(submodule, Model): + continue + submodule_prefix = prefix + ('.' if prefix else '') + submodule.name + self._parse_search_space(submodule, root, submodule_prefix, memo=memo, nested_detection=nested_detection) + return root + + @property + def mutables(self): + return self._structured_mutables + + def undedup_mutables(self): + return self._structured_mutables.traverse(deduplicate=False) + + def call(self, *inputs): + raise RuntimeError('Call is undefined for mutators.') + + def __setattr__(self, name, value): + if name == 'model': + raise AttributeError("Attribute `model` can be set at most once, and you shouldn't use `self.model = model` to " + "include your network, as it will include all parameters in model into the mutator.") + return super().__setattr__(name, value) + + def enter_mutable_scope(self, mutable_scope): + pass + + def exit_mutable_scope(self, mutable_scope): + pass + + def on_forward_layer_choice(self, mutable, *inputs): + raise NotImplementedError + + def on_forward_input_choice(self, mutable, tensor_list): + raise NotImplementedError + + def export(self): + raise NotImplementedError diff --git a/utils/third_party/nni_new/nas/tensorflow/mutables.py b/utils/third_party/nni_new/nas/tensorflow/mutables.py new file mode 100644 index 0000000000000000000000000000000000000000..06183a34c1872804f87887567692fcab6a732816 --- /dev/null +++ b/utils/third_party/nni_new/nas/tensorflow/mutables.py @@ -0,0 +1,144 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +from collections import OrderedDict + +from tensorflow.keras import Model + +from .utils import global_mutable_counting + + +_logger = logging.getLogger(__name__) + + +class Mutable(Model): + def __init__(self, key=None): + super().__init__() + if key is None: + self._key = '{}_{}'.format(type(self).__name__, global_mutable_counting()) + elif isinstance(key, str): + self._key = key + else: + self._key = str(key) + _logger.warning('Key "%s" is not string, converted to string.', key) + self.init_hook = None + self.forward_hook = None + + def __deepcopy__(self, memodict=None): + raise NotImplementedError("Deep copy doesn't work for mutables.") + + def set_mutator(self, mutator): + if hasattr(self, 'mutator'): + raise RuntimeError('`set_mutator is called more than once. ' + 'Did you parse the search space multiple times? ' + 'Or did you apply multiple fixed architectures?') + self.mutator = mutator + + def call(self, *inputs): + raise NotImplementedError('Method `call` of Mutable must be overridden') + + def build(self, input_shape): + self._check_built() + + @property + def key(self): + return self._key + + @property + def name(self): + return self._name if hasattr(self, '_name') else self._key + + @name.setter + def name(self, name): + self._name = name + + def _check_built(self): + if not hasattr(self, 'mutator'): + raise ValueError( + "Mutator not set for {}. You might have forgotten to initialize and apply your mutator. " + "Or did you initialize a mutable on the fly in forward pass? Move to `__init__` " + "so that trainer can locate all your mutables. See NNI docs for more details.".format(self)) + + def __repr__(self): + return '{} ({})'.format(self.name, self.key) + + +class MutableScope(Mutable): + def __call__(self, *args, **kwargs): + try: + self.mutator.enter_mutable_scope(self) + return super().__call__(*args, **kwargs) + finally: + self.mutator.exit_mutable_scope(self) + + +class LayerChoice(Mutable): + def __init__(self, op_candidates, reduction='sum', return_mask=False, key=None): + super().__init__(key=key) + self.names = [] + if isinstance(op_candidates, OrderedDict): + for name in op_candidates: + assert name not in ["length", "reduction", "return_mask", "_key", "key", "names"], \ + "Please don't use a reserved name '{}' for your module.".format(name) + self.names.append(name) + elif isinstance(op_candidates, list): + for i, _ in enumerate(op_candidates): + self.names.append(str(i)) + else: + raise TypeError("Unsupported op_candidates type: {}".format(type(op_candidates))) + + self.length = len(op_candidates) + self.choices = op_candidates + self.reduction = reduction + self.return_mask = return_mask + + def call(self, *inputs): + out, mask = self.mutator.on_forward_layer_choice(self, *inputs) + if self.return_mask: + return out, mask + return out + + def build(self, input_shape): + self._check_built() + for op in self.choices: + op.build(input_shape) + + def __len__(self): + return len(self.choices) + + +class InputChoice(Mutable): + NO_KEY = '' + + def __init__(self, n_candidates=None, choose_from=None, n_chosen=None, reduction='sum', return_mask=False, key=None): + super().__init__(key=key) + assert n_candidates is not None or choose_from is not None, \ + 'At least one of `n_candidates` and `choose_from` must be not None.' + if choose_from is not None and n_candidates is None: + n_candidates = len(choose_from) + elif choose_from is None and n_candidates is not None: + choose_from = [self.NO_KEY] * n_candidates + assert n_candidates == len(choose_from), 'Number of candidates must be equal to the length of `choose_from`.' + assert n_candidates > 0, 'Number of candidates must be greater than 0.' + assert n_chosen is None or 0 <= n_chosen <= n_candidates, \ + 'Expected selected number must be None or no more than number of candidates.' + + self.n_candidates = n_candidates + self.choose_from = choose_from.copy() + self.n_chosen = n_chosen + self.reduction = reduction + self.return_mask = return_mask + + def call(self, optional_inputs): + optional_input_list = optional_inputs + if isinstance(optional_inputs, dict): + optional_input_list = [optional_inputs[tag] for tag in self.choose_from] + assert isinstance(optional_input_list, list), \ + 'Optional input list must be a list, not a {}.'.format(type(optional_input_list)) + assert len(optional_inputs) == self.n_candidates, \ + 'Length of the input list must be equal to number of candidates.' + out, mask = self.mutator.on_forward_input_choice(self, optional_input_list) + if self.return_mask: + return out, mask + return out diff --git a/utils/third_party/nni_new/nas/tensorflow/mutator.py b/utils/third_party/nni_new/nas/tensorflow/mutator.py new file mode 100644 index 0000000000000000000000000000000000000000..b0d2aed684e289b556ccc1388b95477d55c5c2da --- /dev/null +++ b/utils/third_party/nni_new/nas/tensorflow/mutator.py @@ -0,0 +1,83 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging + +import tensorflow as tf + +from .base_mutator import BaseMutator + + +_logger = logging.getLogger(__name__) + + +class Mutator(BaseMutator): + def __init__(self, model): + super().__init__(model) + self._cache = {} + + def sample_search(self): + raise NotImplementedError('Method `sample_search` must be overridden') + + def sample_final(self): + raise NotImplementedError('Method `sample_final` must be overriden for exporting') + + def reset(self): + self._cache = self.sample_search() + + def export(self): + return self.sample_final() + + # TODO: status + # TODO: graph + + def on_forward_layer_choice(self, mutable, *inputs): + mask = self._get_decision(mutable) + assert len(mask) == len(mutable), \ + 'Invalid mask, expected {} to be of length {}.'.format(mask, len(mutable)) + out = self._select_with_mask(lambda choice: choice(*inputs), mutable.choices, mask) + return self._tensor_reduction(mutable.reduction, out), mask + + def on_forward_input_choice(self, mutable, tensor_list): + mask = self._get_decision(mutable) + assert len(mask) == mutable.n_candidates, \ + 'Invalid mask, expected {} to be of length {}.'.format(mask, mutable.n_candidates) + out = self._select_with_mask(lambda tensor: tensor, tensor_list, mask) + return self._tensor_reduction(mutable.reduction, out), mask + + def _select_with_mask(self, map_fn, candidates, mask): + if mask.dtype.is_bool: + out = [map_fn(cand) for cand, m in zip(candidates, mask) if m] + elif mask.dtype.is_floating: + out = [map_fn(cand) * m for cand, m in zip(candidates, mask) if m] + else: + raise ValueError('Unrecognized mask, dtype is {}'.format(mask.dtype.name)) + return out + + def _tensor_reduction(self, reduction_type, tensor_list): + if reduction_type == 'none': + return tensor_list + if not tensor_list: + return None + if len(tensor_list) == 1: + return tensor_list[0] + if reduction_type == 'sum': + return sum(tensor_list) + if reduction_type == 'mean': + return sum(tensor_list) / len(tensor_list) + if reduction_type == 'concat': + image_data_format = tf.keras.backend.image_data_format() + if image_data_format == "channels_first": + axis = 0 + else: + axis = -1 + return tf.concat(tensor_list, axis=axis) # pylint: disable=E1120,E1123 + # pylint issue #3613 + raise ValueError('Unrecognized reduction policy: "{}'.format(reduction_type)) + + def _get_decision(self, mutable): + if mutable.key not in self._cache: + raise ValueError('"{}" not found in decision cache.'.format(mutable.key)) + result = self._cache[mutable.key] + _logger.debug('Decision %s: %s', mutable.key, result) + return result diff --git a/utils/third_party/nni_new/nas/tensorflow/utils.py b/utils/third_party/nni_new/nas/tensorflow/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..0cfc6e815d973774a543e23078692189bfbb90d0 --- /dev/null +++ b/utils/third_party/nni_new/nas/tensorflow/utils.py @@ -0,0 +1,93 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import tensorflow as tf + +_counter = 0 + +def global_mutable_counting(): + global _counter + _counter += 1 + return _counter + + +class AverageMeter: + def __init__(self, name): + self.name = name + self.val = 0 + self.avg = 0 + self.sum = 0 + self.count = 0 + + def update(self, val): + self.val = val + self.sum += val + self.count += 1 + self.avg = self.sum / self.count + + def __str__(self): + return '{name} {val:4f} ({avg:4f})'.format(**self.__dict__) + + def summary(self): + return '{name}: {avg:4f}'.format(**self.__dict__) + + +class AverageMeterGroup: + def __init__(self): + self.meters = {} + + def update(self, data): + for k, v in data.items(): + if k not in self.meters: + self.meters[k] = AverageMeter(k) + self.meters[k].update(v) + + def __str__(self): + return ' '.join(str(v) for v in self.meters.values()) + + def summary(self): + return ' '.join(v.summary() for v in self.meters.values()) + + +class StructuredMutableTreeNode: + def __init__(self, mutable): + self.mutable = mutable + self.children = [] + + def add_child(self, mutable): + self.children.append(StructuredMutableTreeNode(mutable)) + return self.children[-1] + + def type(self): + return type(self.mutable) + + def __iter__(self): + return self.traverse() + + def traverse(self, order="pre", deduplicate=True, memo=None): + if memo is None: + memo = set() + assert order in ["pre", "post"] + if order == "pre": + if self.mutable is not None: + if not deduplicate or self.mutable.key not in memo: + memo.add(self.mutable.key) + yield self.mutable + for child in self.children: + for m in child.traverse(order=order, deduplicate=deduplicate, memo=memo): + yield m + if order == "post": + if self.mutable is not None: + if not deduplicate or self.mutable.key not in memo: + memo.add(self.mutable.key) + yield self.mutable + + +def fill_zero_grads(grads, weights): + ret = [] + for grad, weight in zip(grads, weights): + if grad is not None: + ret.append(grad) + else: + ret.append(tf.zeros_like(weight)) + return ret diff --git a/utils/third_party/nni_new/parameter_expressions.py b/utils/third_party/nni_new/parameter_expressions.py new file mode 100644 index 0000000000000000000000000000000000000000..adff923f7ea1cb4debe114c22376522adb8d8a6d --- /dev/null +++ b/utils/third_party/nni_new/parameter_expressions.py @@ -0,0 +1,108 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +''' +parameter_expression.py +''' + +import numpy as np + + +def choice(options, random_state): + ''' + options: 1-D array-like or int + random_state: an object of numpy.random.RandomState + ''' + return random_state.choice(options) + + +def randint(lower, upper, random_state): + ''' + Generate a random integer from `lower` (inclusive) to `upper` (exclusive). + lower: an int that represent an lower bound + upper: an int that represent an upper bound + random_state: an object of numpy.random.RandomState + ''' + return random_state.randint(lower, upper) + + +def uniform(low, high, random_state): + ''' + low: an float that represent an lower bound + high: an float that represent an upper bound + random_state: an object of numpy.random.RandomState + ''' + assert high >= low, 'Upper bound must be larger than lower bound' + return random_state.uniform(low, high) + + +def quniform(low, high, q, random_state): + ''' + low: an float that represent an lower bound + high: an float that represent an upper bound + q: sample step + random_state: an object of numpy.random.RandomState + ''' + return np.clip(np.round(uniform(low, high, random_state) / q) * q, low, high) + + +def loguniform(low, high, random_state): + ''' + low: an float that represent an lower bound + high: an float that represent an upper bound + random_state: an object of numpy.random.RandomState + ''' + assert low > 0, 'Lower bound must be positive' + return np.exp(uniform(np.log(low), np.log(high), random_state)) + + +def qloguniform(low, high, q, random_state): + ''' + low: an float that represent an lower bound + high: an float that represent an upper bound + q: sample step + random_state: an object of numpy.random.RandomState + ''' + return np.clip(np.round(loguniform(low, high, random_state) / q) * q, low, high) + + +def normal(mu, sigma, random_state): + ''' + The probability density function of the normal distribution, + first derived by De Moivre and 200 years later by both Gauss and Laplace independently. + mu: float or array_like of floats + Mean (“centre”) of the distribution. + sigma: float or array_like of floats + Standard deviation (spread or “width”) of the distribution. + random_state: an object of numpy.random.RandomState + ''' + return random_state.normal(mu, sigma) + + +def qnormal(mu, sigma, q, random_state): + ''' + mu: float or array_like of floats + sigma: float or array_like of floats + q: sample step + random_state: an object of numpy.random.RandomState + ''' + return np.round(normal(mu, sigma, random_state) / q) * q + + +def lognormal(mu, sigma, random_state): + ''' + mu: float or array_like of floats + sigma: float or array_like of floats + random_state: an object of numpy.random.RandomState + ''' + return np.exp(normal(mu, sigma, random_state)) + + +def qlognormal(mu, sigma, q, random_state): + ''' + mu: float or array_like of floats + sigma: float or array_like of floats + q: sample step + random_state: an object of numpy.random.RandomState + ''' + return np.round(lognormal(mu, sigma, random_state) / q) * q diff --git a/utils/third_party/nni_new/recoverable.py b/utils/third_party/nni_new/recoverable.py new file mode 100644 index 0000000000000000000000000000000000000000..70f11e634dc74886c0d379ca7fc86202654bf6ad --- /dev/null +++ b/utils/third_party/nni_new/recoverable.py @@ -0,0 +1,18 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os + +class Recoverable: + + def load_checkpoint(self): + pass + + def save_checkpoint(self): + pass + + def get_checkpoint_path(self): + ckp_path = os.getenv('NNI_CHECKPOINT_DIRECTORY') + if ckp_path is not None and os.path.isdir(ckp_path): + return ckp_path + return None diff --git a/utils/third_party/nni_new/retiarii/__init__.py b/utils/third_party/nni_new/retiarii/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..f4413674600d75345d505d3c4bc8da8a1f91f419 --- /dev/null +++ b/utils/third_party/nni_new/retiarii/__init__.py @@ -0,0 +1,8 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .operation import Operation +from .graph import * +from .execution import * +from .mutator import * +from .serializer import basic_unit, json_dump, json_dumps, json_load, json_loads, serialize, serialize_cls, model_wrapper diff --git a/utils/third_party/nni_new/retiarii/codegen/__init__.py b/utils/third_party/nni_new/retiarii/codegen/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..52f3abc6366f6a48339afa654948af0e651e3287 --- /dev/null +++ b/utils/third_party/nni_new/retiarii/codegen/__init__.py @@ -0,0 +1 @@ +from .pytorch import model_to_pytorch_script diff --git a/utils/third_party/nni_new/retiarii/codegen/pytorch.py b/utils/third_party/nni_new/retiarii/codegen/pytorch.py new file mode 100644 index 0000000000000000000000000000000000000000..17dc45e3a5f6e0ac6dec69472a55238b46623895 --- /dev/null +++ b/utils/third_party/nni_new/retiarii/codegen/pytorch.py @@ -0,0 +1,181 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +from typing import List, Tuple, Any + +from ..graph import IllegalGraphError, Edge, Graph, Node, Model + +_logger = logging.getLogger(__name__) + + +def model_to_pytorch_script(model: Model, placement=None) -> str: + graphs = [] + total_pkgs = set() + for name, cell in model.graphs.items(): + import_pkgs, graph_code = graph_to_pytorch_model(name, cell, placement=placement) + graphs.append(graph_code) + total_pkgs.update(import_pkgs) + pkgs_code = '\n'.join(['import {}'.format(pkg) for pkg in total_pkgs]) + return _PyTorchScriptTemplate.format(pkgs_code, '\n\n'.join(graphs)).strip() + + +def _sorted_incoming_edges(node: Node) -> List[Edge]: + edges = [edge for edge in node.graph.edges if edge.tail is node] + _logger.debug('sorted_incoming_edges: %s', str(edges)) + if not edges: + return [] + _logger.debug('all tail_slots are None: %s', str([edge.tail_slot for edge in edges])) + if all(edge.tail_slot is None for edge in edges): + return edges + if all(isinstance(edge.tail_slot, int) for edge in edges): + edges = sorted(edges, key=(lambda edge: edge.tail_slot)) + if [edge.tail_slot for edge in edges] == list(range(len(edges))): + return edges + raise IllegalGraphError(node.graph, 'Node {} has bad inputs'.format(node.name)) + + +def _format_inputs(node: Node) -> Tuple[List[str], List[Any]]: + """ + Format the inputs of a given node + + Parameters + ---------- + node : Node + a graph node, get and format its inputs + + Returns + ------- + list + the list of input names + list + the list of input values, if an input is simple type, record its value, + otherwise the value is None + """ + edges = _sorted_incoming_edges(node) + inputs = [] + inputs_value = [] + for edge in edges: + if edge.head.name == '_inputs': + assert isinstance(edge.head_slot, int) + if edge.head.operation.io_names is not None: + # when input has names, e.g., forward(self, tensor1, tensor2, another_one) + inputs.append(edge.head.operation.io_names[edge.head_slot]) + else: + # when input has no name, e.g., forward(*_inputs) + inputs.append('_inputs[{}]'.format(edge.head_slot)) + inputs_value.append(None) + else: + if edge.head_slot is None: + # when the input comes from a single-output operator + inputs.append('{}'.format(edge.head.name)) + if edge.head.operation.type in ('prim::Constant', 'prim::GetAttr') and \ + 'value' in edge.head.operation.parameters: + inputs_value.append(edge.head.operation.parameters['value']) + else: + inputs_value.append(None) + else: + # when the input comes from a multi-output operator: needs to know which one it comes from + inputs.append('{}[{}]'.format(edge.head.name, edge.head_slot)) + inputs_value.append(None) + return inputs, inputs_value + + +def _remove_prefix(names, graph_name): + """ + variables name (full name space) is too long, + shorten the name by removing the prefix ```graph_name``` + """ + if isinstance(names, list): + converted_names = [] + for name in names: + if name.startswith(graph_name): + converted_names.append(name[len(graph_name):]) + else: + converted_names.append(name) + return converted_names + else: + return names[len(graph_name):] if names.startswith(graph_name) else names + + +def graph_to_pytorch_model(graph_name: str, graph: Graph, placement=None) -> str: + nodes = graph.topo_sort() + + # handle module node and function node differently + # only need to generate code for module here + import_pkgs = set() + node_codes = [] + for node in nodes: + if node.operation: + if node.operation.type == 'shared': + continue + pkg_name = node.operation.get_import_pkg() + if pkg_name is not None: + import_pkgs.add(pkg_name) + node_code = node.operation.to_init_code(_remove_prefix(node.name, graph_name)) + if node_code is not None: + if placement and node in placement and len(node_code) > 0: + node_codes.append(f"{node_code}.to('{placement[node].device}')") + else: + node_codes.append(node_code) + + if graph.input_node.operation.io_names is None: + input_code = '*_inputs' + else: + for name in graph.input_node.operation.io_names: + assert not name.startswith(graph_name) + input_code = ', '.join(graph.input_node.operation.io_names) + + edge_codes = [] + sorted_nodes = graph.topo_sort() + for node in sorted_nodes: + if node.operation: + inputs, inputs_value = _format_inputs(node) + inputs = _remove_prefix(inputs, graph_name) + node_name = _remove_prefix(node.name, graph_name) + submodule_name = node_name + if node.operation.type == 'shared': + submodule_name = _remove_prefix(node.operation.parameters['reference'], graph_name) + edge_codes.append(node.operation.to_forward_code(submodule_name, node_name, inputs, inputs_value)) + + output_names, _ = _format_inputs(graph.output_node) + output_names = _remove_prefix(output_names, graph_name) + if not output_names: + raise RuntimeError('"forward" function should have return value(s): {}, {}, {}'.format(output_names, graph_name, graph.output_node)) + output_code = ', '.join(output_names) + + linebreak = '\n ' + return import_pkgs, _PyTorchModelTemplate.format( + graph_name=('Graph' if graph_name == '_graph' else graph_name), + inputs=input_code, + outputs=output_code, + nodes=linebreak.join(node_codes), + edges=linebreak.join(edge_codes) + ) + + +# TODO: handle imports + +_PyTorchScriptTemplate = ''' +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.optim as optim + +import nni.retiarii.nn.pytorch + +{} + +{} +''' + +_PyTorchModelTemplate = ''' +class {graph_name}(nn.Module): + def __init__(self): + super().__init__() + {nodes} + + def forward(self, {inputs}): + {edges} + return {outputs} +''' diff --git a/utils/third_party/nni_new/retiarii/codegen/tensorflow.py b/utils/third_party/nni_new/retiarii/codegen/tensorflow.py new file mode 100644 index 0000000000000000000000000000000000000000..ac0e0d7003599560a0222b3675cc0bc7374a4694 --- /dev/null +++ b/utils/third_party/nni_new/retiarii/codegen/tensorflow.py @@ -0,0 +1,109 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +# pylint: skip-file + +""" +FIXME +This file is inherited from last version. + +I expect it can work with a few modifications to incorporate with the latest API, but it hasn't +been tested and I'm not sure. +""" + +from ..graph_v2 import IllegalGraphError, Cell, Edge, Graph, Node +from ..operations_tf import Operation +from ..type_utils import * + + +def graph_to_tensorflow_script(graph: Graph) -> str: + graphs = [graph_to_tensorflow_model(name, cell) for name, cell in graph.cell_templates.items()] + return _TensorFlowScriptTemplate.format('\n\n'.join(graphs)).strip() + + +def _sort_incoming_edges(node: Node) -> List[Edge]: + edges = [edge for edge in node.graph.edges if edge.tail is node] + if not edges: + return [] + if all(edge.tail_idx is None for edge in edges): + return edges + if all(isinstance(edge.tail_idx, int) for edge in edges): + edges = sorted(edges, key=(lambda edge: edge.tail_idx)) + if [edge.tail_idx for edge in edges] == list(range(len(edges))): + return edges + raise IllegalGraphError(node.graph, 'Node {} has bad inputs'.format(node.name)) + +def _format_inputs(node: Node) -> str: + edges = _sort_incoming_edges(node) + inputs = [] + for edge in edges: + if edge.head.name == '_inputs': + assert isinstance(edge.head_idx, int) + if node.graph.input_names is not None: + inputs.append(node.graph.input_names[edge.head_idx]) + else: + inputs.append('_inputs[{}]'.format(edge.head_idx)) + else: + if edge.head_idx is None: + inputs.append('{}'.format(edge.head.name)) + else: + inputs.append('{}[{}]'.format(edge.head.name, edge.head_idx)) + return ', '.join(inputs) + + +def graph_to_tensorflow_model(graph_name: str, graph: Graph) -> str: + nodes = graph.topo_sort() + + # handle module node and function node differently + # only need to generate code for module here + node_codes = [] + for node in nodes: + if isinstance(node, Cell): + node_codes.append('self.{} = {}()'.format(node.name, node.template_name)) + else: + node_codes.append('self.{} = {}'.format(node.name, cast(Operation, node.operation).to_tensorflow_init())) + + edge_codes = [] + + for node in nodes: + inputs = _format_inputs(node) + edge_codes.append('{} = self.{}({})'.format(node.name, node.name, inputs)) + + output_code = _format_inputs(graph.output_node) + if not output_code: + output_code = 'None' + + if graph.input_names is None: + input_code = '*_inputs' + else: + input_code = ', '.join(graph.input_names) + + linebreak = '\n ' + return _TensorFlowModelTemplate.format( + graph_name=('Graph' if graph_name == '_graph' else graph_name), + inputs=input_code, + outputs=output_code, + nodes=linebreak.join(node_codes), + edges=linebreak.join(edge_codes) + ) + + +_TensorFlowScriptTemplate = ''' +import tensorflow as tf +import tensorflow.keras as K + +import sdk.custom_ops_tf as CUSTOM + +{} +''' + +_TensorFlowModelTemplate = ''' +class {graph_name}(K.Model): + def __init__(self): + super().__init__() + {nodes} + + def call(self, {inputs}): + {edges} + return {outputs} +''' \ No newline at end of file diff --git a/utils/third_party/nni_new/retiarii/converter/README.md b/utils/third_party/nni_new/retiarii/converter/README.md new file mode 100644 index 0000000000000000000000000000000000000000..d0f19066b1ac0ed367cd5d8bd488984288f08315 --- /dev/null +++ b/utils/third_party/nni_new/retiarii/converter/README.md @@ -0,0 +1,37 @@ +# PyTorch Graph Converter + +## Namespace for PyTorch Graph + +We should have a concrete rule for specifying nodes in graph with namespace. + +Each node has a name, either specified or generated. The nodes in the same hierarchy cannot have the same name. + +* The name of module node natively follows this rule, because we use variable name for instantiated modules like what PyTorch graph does. + +* For the nodes created in `forward` function, we use a global sequence number. + +### Namespace for mutated (new) nodes + +TBD + +## Graph Simplification + +TBD + +## Node Types + +We define concrete type string for each node type. + +## Module's Input Arguments + +We use wrapper to obtain the input arguments of modules. Users need to use our wrapped "nn" and wrapped "Module". + +## Control Flow + +### for loop + +Currently, we only support `ModuleList` (`ModuleDict`) based for loop, which is automatically unfolded by TorchScript. That is to say, we do not support loop in TorchScript for now. + +### if/else + +For now, we only deal with the case that the condition is constant or attribute. In this case, only one branch is kept during generating the graph. \ No newline at end of file diff --git a/utils/third_party/nni_new/retiarii/converter/__init__.py b/utils/third_party/nni_new/retiarii/converter/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e0fff09f2da1af2f3904ae9ab40939498030a295 --- /dev/null +++ b/utils/third_party/nni_new/retiarii/converter/__init__.py @@ -0,0 +1 @@ +from .graph_gen import convert_to_graph diff --git a/utils/third_party/nni_new/retiarii/converter/graph_gen.py b/utils/third_party/nni_new/retiarii/converter/graph_gen.py new file mode 100644 index 0000000000000000000000000000000000000000..f8b06b887a6772a4d0d77e728c85aa120bbec2c1 --- /dev/null +++ b/utils/third_party/nni_new/retiarii/converter/graph_gen.py @@ -0,0 +1,679 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import re + +import torch + +from ..graph import Graph, Model, Node +from ..nn.pytorch import InputChoice, Placeholder +from ..operation import Cell, Operation +from ..serializer import get_init_parameters_or_fail +from ..utils import get_importable_name +from .op_types import MODULE_EXCEPT_LIST, OpTypeName +from .utils import _convert_name, build_full_name + + +class GraphConverter: + def __init__(self): + self.global_seq = 0 + self.global_graph_id = 0 + + def _add_edge_handle_source_node(self, _input, graph_inputs, ir_graph, output_remap, node_index): + if _input in output_remap: + assert output_remap[_input].kind() == 'aten::append' + predecessor_node = output_remap[_input] + assert predecessor_node in node_index, 'predecessor node: {}'.format(predecessor_node) + src_node_idx = None + src_node = node_index[predecessor_node] + assert isinstance(src_node, Node) + elif _input in graph_inputs: + idx = graph_inputs.index(_input) + src_node = ir_graph.input_node + src_node_idx = idx + else: + predecessor_node = _input.node() + assert predecessor_node in node_index, 'predecessor node: {}'.format(predecessor_node) + # find out the index of _input in the outputs of predecessor_node + predecessor_outputs = [_output for _output in predecessor_node.outputs()] + if len(predecessor_outputs) == 1: + idx = None + else: + idx = predecessor_outputs.index(_input) + ir_predecessor_node = node_index[predecessor_node] + src_node_idx = idx + assert isinstance(ir_predecessor_node, Node) + src_node = ir_predecessor_node + return src_node, src_node_idx + + def _add_edge(self, ir_graph, node, graph_inputs, node_index, new_node, output_remap, ignore_first=False): + """ + Parameters + ---------- + ir_graph : Graph + node : torch._C.Node + graph_inputs : List[torch._C.Value] + a list of a script graph's inputs + node_index : Dict + new_node : Node + newly created ir node corresponding to `node` + output_remap : Dict + ignore_first : bool + if it is true, skip the first input + """ + is_single_input = (len([_input for _input in node.inputs()]) - (1 if ignore_first else 0)) == 1 + new_node_input_idx = 0 + for _input in node.inputs(): + if ignore_first: + ignore_first = False + continue + # handle source node + src_node, src_node_idx = self._add_edge_handle_source_node(_input, graph_inputs, ir_graph, output_remap, node_index) + # handle destination node + dst_node = new_node + if is_single_input: + dst_node_idx = None + else: + dst_node_idx = new_node_input_idx + # create edge + ir_graph.add_edge(head=(src_node, src_node_idx), tail=(dst_node, dst_node_idx)) + + new_node_input_idx += 1 + + def create_prim_constant_node(self, ir_graph, node, module_name): + # NOTE: compare with string not type, because the type is defined in pytorch C code. + # `.kind()` can also be used here + if node.outputsAt(0).type().str() == 'None': + attrs = {'type': 'None'} + else: + attrs = {'type': node.outputsAt(0).type().str(), 'value': node.outputsAt(0).toIValue()} + self.global_seq += 1 + new_node = ir_graph.add_node(build_full_name(module_name, OpTypeName.Constant, self.global_seq), + node.kind(), attrs) + return new_node + + def handle_prim_attr_node(self, node, module): + assert node.hasAttribute('name') + value = None + if node.inputsAt(0).debugName() == 'self': + _val = getattr(module, node.s('name')) + # TODO: serialize complex data type, and output proper error message + if isinstance(_val, (int, float, str, bool)): + value = _val + attrs = {'name': node.s('name'), 'input': node.inputsAt(0).debugName(), 'value': value} + return node.kind(), attrs + + def _remove_mangle(self, module_type_str): + return re.sub('\\.___torch_mangle_\\d+', '', module_type_str) + + def remove_unconnected_nodes(self, ir_graph, targeted_type=None): + """ + Parameters + ---------- + ir_graph : Graph + our ir graph representation + targeted_type : str + nodes with ```targeted_type``` will be removed from graph if their fanout is 0. + ```None``` means removing all the nodes whose fanout is 0. + """ + # build index of outputs of Node(s) + node_fanout = set() + for edge in ir_graph.edges: + if edge.head.id not in node_fanout: + node_fanout.add(edge.head.id) + + to_removes = [] + for hidden_node in ir_graph.hidden_nodes: + if hidden_node.id not in node_fanout: + assert isinstance(hidden_node, Node) + if targeted_type is None: + to_removes.append(hidden_node) + elif hidden_node.operation.type == targeted_type: + to_removes.append(hidden_node) + + for hidden_node in to_removes: + hidden_node.remove() + + def handle_graph_nodes(self, script_module, sm_graph, + module, module_name, + ir_model, ir_graph, + shared_module_index=None): + """ + Convert torch script node to our node ir, and build our graph ir + + Parameters + ---------- + script_module : torch.jit.RecursiveScriptModule + the torch script of ```module``` + sm_graph : torch._C.Graph + the graph in torch script + module : nn.Module + the targeted pytorch module + module_name : str + ```module```'s name + ir_model : Model + the whole graph ir + ir_graph : Graph + the graph ir of ```module``` + shared_module_index : dict + it is used for knowing which module has been created an ir node, + if created and invoked again, then the new ir node can simply reference that ir node. + this way we can identify shared modules (i.e., one module invoked multiple times in `forward` function) + + Returns + ------- + dict + the mapping from graph node to our graph ir node + """ + # handle inputs + graph_inputs = [] + for _input in sm_graph.inputs(): + if _input.debugName() == 'self': + assert _input.unique() == 0 + continue + graph_inputs.append(_input) + # TODO: add scope name + ir_graph._add_input(_convert_name(_input.debugName())) + + node_index = {} # graph node to graph ir node + if shared_module_index is None: + shared_module_index = {} + + # some node does not have output but it modifies a variable, for example aten::append + # %17 : Tensor[] = aten::append(%out.1, %16) + # %out.1 is updated, and %17 is None + # we add output to this type of node and connect it to the following node which uses %out.1 + # key: tensor (%out.1), value: node (this node) + output_remap = {} + + # ===================handle control flow: if=================== + def handle_if_condition(cond_tensor): + """ + to calculate the condition, we only deal with the following op types by tracing back + `prim::GetAttr`, `aten::__getitem__`, `prim::Constant`, `aten::eq` + + generate the expression using recursive calls + + NOTE: do not support dynamic graph + """ + def _generate_expr(tensor): + if tensor.node().kind() == 'prim::GetAttr': + return f'({getattr(module, tensor.node().s("name"))})' + elif tensor.node().kind() == 'aten::__getitem__': + t = _generate_expr(tensor.node().inputsAt(0)) + idx = _generate_expr(tensor.node().inputsAt(1)) + return f'({t}[{idx}])' + elif tensor.node().kind() == 'prim::Constant': + return f'{tensor.toIValue()}' + elif tensor.node().kind() == 'aten::eq': + left = _generate_expr(tensor.node().inputsAt(0)) + right = _generate_expr(tensor.node().inputsAt(1)) + return f'({left} == {right})' + elif tensor.node().kind() == 'aten::le': + left = _generate_expr(tensor.node().inputsAt(0)) + right = _generate_expr(tensor.node().inputsAt(1)) + return f'({left} <= {right})' + elif tensor.node().kind() == 'aten::ge': + left = _generate_expr(tensor.node().inputsAt(0)) + right = _generate_expr(tensor.node().inputsAt(1)) + return f'({left} >= {right})' + elif tensor.node().kind() == 'aten::__not__': + value = _generate_expr(tensor.node().inputsAt(0)) + return f'(not {value})' + elif tensor.node().kind() == 'aten::Bool': + value = _generate_expr(tensor.node().inputsAt(0)) + return f'bool({value})' + elif tensor.node().kind() == 'aten::__is__': + left = _generate_expr(tensor.node().inputsAt(0)) + right = _generate_expr(tensor.node().inputsAt(1)) + return f'({left} is {right})' + elif tensor.node().kind() == 'aten::__isnot__': + left = _generate_expr(tensor.node().inputsAt(0)) + right = _generate_expr(tensor.node().inputsAt(1)) + return f'({left} is not {right})' + elif tensor.node().kind() == 'aten::ne': + left = _generate_expr(tensor.node().inputsAt(0)) + right = _generate_expr(tensor.node().inputsAt(1)) + return f'({left} != {right})' + elif tensor.node().kind() == 'aten::gt': + left = _generate_expr(tensor.node().inputsAt(0)) + right = _generate_expr(tensor.node().inputsAt(1)) + return f'({left} > {right})' + elif tensor.node().kind() == 'aten::lt': + left = _generate_expr(tensor.node().inputsAt(0)) + right = _generate_expr(tensor.node().inputsAt(1)) + return f'({left} < {right})' + elif tensor.node().kind() == 'prim::If': + raise RuntimeError('Have not supported `if A and/or B`, please use two `if` statements instead.') + else: + raise RuntimeError(f'Unsupported op type {tensor.node().kind()} in if condition, ' + 'you are suggested to decorate the corresponding class with "@basic_unit".') + expr = _generate_expr(cond_tensor) + return eval(expr) + + def handle_if_node(node): + """ + Parameters + ---------- + node : torch._C.Node + the node from TorchScript graph + + Returns + ------- + Node + the created node ir + """ + # only deal with input of prim::If is constant or attribute for now + # will support constant expression in future + inputs = [i for i in node.inputs()] + assert len(inputs) == 1 + cond = handle_if_condition(inputs[0]) + chosen_block = 0 if cond else 1 + blocks = [block for block in node.blocks()] + assert len(blocks) == 2 + last_block_node = None + for node in blocks[chosen_block].nodes(): + last_block_node = handle_single_node(node) + self.global_seq += 1 + new_node = ir_graph.add_node(build_full_name(module_name, 'noop_identity', self.global_seq), 'noop_identity') + self._add_edge(ir_graph, blocks[chosen_block].returnNode(), graph_inputs, node_index, new_node, output_remap) + last_block_node = new_node + return last_block_node + + # ===================handle function call=================== + def handle_function_callmethod(node): + # get and handle the first input, which should be an nn.Module + assert node.hasAttribute('name') + # NOTE: "forward__0" is hacky, LSTM instance is parsed to call forward__0 in torchscript + if node.s('name') in ['forward', 'forward__0']: + # node.inputsAt(0).type() is + submodule_type_str = self._remove_mangle(node.inputsAt(0).type().str()) + submodule = node.inputsAt(0).node() + assert submodule.kind() == 'prim::GetAttr' + assert submodule.hasAttribute('name') + submodule_name = submodule.s('name') + + if submodule.inputsAt(0).debugName() == 'self': + # module is usually instantiated in __init__. + # when calling a module in forward, + # prim::GetAttr is used to obtain the module in torch script. + # therefore, we do this check for a module. example below: + # %25 : __torch__.xxx = prim::GetAttr[name="input_switch"](%self) + # %27 : Tensor = prim::CallMethod[name="forward"](%25, %out.1) + assert submodule_name in script_module._modules, "submodule_name: {} not in script_module {}".format( + submodule_name, script_module._modules.keys()) + + submodule_full_name = build_full_name(module_name, submodule_name) + submodule_obj = getattr(module, submodule_name) + subgraph, sub_m_attrs = self.convert_module(script_module._modules[submodule_name], + submodule_obj, + submodule_full_name, ir_model) + else: + # %8 : __torch__.nni.retiarii.model_apis.nn.___torch_mangle_37.ModuleList = prim::GetAttr[name="cells"](%self) + # %10 : __torch__.darts_model.Cell = prim::GetAttr[name="0"](%8) + # %s1.4 : Tensor = prim::CallMethod[name="forward"](%10, %4, %4) + if submodule.inputsAt(0).type().name() == 'ModuleList': + # handle ModuleList + predecessor = submodule.inputsAt(0).node() + module_name_space = [submodule_name] + while predecessor.inputsAt(0).debugName() != 'self': + # this is for dealing with nested ModuleList. below is an example + # %3 : __torch__.torch.nn.modules.container.___torch_mangle_0.ModuleList = prim::GetAttr[name="ops"](%self) + # %5 : __torch__.torch.nn.modules.container.ModuleList = prim::GetAttr[name="0"](%3) + # %7 : __torch__.torch.nn.modules.container.ModuleList = prim::GetAttr[name="1"](%3) + # %9 : __torch__.torch.nn.modules.container.ModuleList = prim::GetAttr[name="2"](%3) + # %11 : __torch__.torch.nn.modules.container.ModuleList = prim::GetAttr[name="3"](%3) + # %14 : __torch__.torch.nn.modules.linear.Linear = prim::GetAttr[name="0"](%5) + # %16 : __torch__.torch.nn.modules.linear.Linear = prim::GetAttr[name="1"](%5) + # %state.2 : Tensor = prim::CallMethod[name="forward"](%14, %x.1) # modulelist.py:18:24 + # %state.4 : Tensor = prim::CallMethod[name="forward"](%16, %state.2) # modulelist.py:18:24 + assert predecessor.kind() == 'prim::GetAttr' + module_name_space.append(predecessor.s('name')) + predecessor = predecessor.inputsAt(0).node() + assert predecessor.kind() == 'prim::GetAttr' + assert predecessor.hasAttribute('name') + module_name_space.append(predecessor.s('name')) + submodule_full_name = build_full_name(module_name, list(reversed(module_name_space))) + submodule_obj = module + script_submodule = script_module + for each_name in list(reversed(module_name_space)): + submodule_obj = getattr(submodule_obj, each_name) + script_submodule = script_submodule._modules[each_name] + subgraph, sub_m_attrs = self.convert_module(script_submodule, submodule_obj, submodule_full_name, ir_model) + else: + raise RuntimeError('Unsupported module case: {}'.format(submodule.inputsAt(0).type().str())) + + if submodule_full_name in shared_module_index: + # this module is invoked more than once, the ir node has already been created + # create a reference node for it. + # example: {"name": "conv2", "operation": {"type": "shared", "parameters": {"reference": "conv1"}}} + self.global_seq += 1 + shared_node_name = build_full_name(submodule_full_name, '', self.global_seq) + shared_type_operation = Operation.new('shared', {'reference': submodule_full_name}) + subcell = ir_graph.add_node(shared_node_name, shared_type_operation) + else: + # this module is processed for the first time, build cell for it + if subgraph is None: + # if we do not parse this module's graph, we create Node for this module + subcell = ir_graph.add_node(submodule_full_name, submodule_type_str, sub_m_attrs) + if isinstance(submodule_obj, Placeholder): + subcell.update_label(submodule_obj.label) + elif isinstance(submodule_obj, InputChoice): + subcell.update_label(sub_m_attrs['label']) + else: + # Graph already created, create Cell for it + new_cell = Cell(cell_name=submodule_full_name, parameters=sub_m_attrs) + subcell = ir_graph.add_node(submodule_full_name, new_cell) + shared_module_index[submodule_full_name] = subcell + node_index[node] = subcell + # connect the cell into graph + self._add_edge(ir_graph, node, graph_inputs, node_index, subcell, output_remap, ignore_first=True) + else: + # handle normal member function + assert hasattr(script_module, node.s('name')) + # TODO: support non member functions + assert node.inputsAt(0).debugName() == 'self' + script_method = getattr(script_module, node.s('name')) # + + # step #1: generate graph ir for this method + method_ir_graph = Graph(model=ir_model, graph_id=-100, name='temp_graph', _internal=True) + method_node_index = self.handle_graph_nodes(script_module, script_method.graph, module, + module_name, ir_model, method_ir_graph, shared_module_index) + for _output in script_method.graph.outputs(): + method_ir_graph._add_output(_convert_name(_output.debugName())) + predecessor_node_outputs = [o for o in _output.node().outputs()] + if len(predecessor_node_outputs) == 1: + src_node_idx = None + else: + src_node_idx = predecessor_node_outputs.index(_output) + method_ir_graph.add_edge(head=(method_node_index[_output.node()], src_node_idx), + tail=(method_ir_graph.output_node, None)) + self.refine_graph(method_ir_graph) + + # step #2: merge this graph to its module graph + for h_node in method_ir_graph.hidden_nodes: + h_node.graph = ir_graph + ir_graph.hidden_nodes.append(h_node) + for edge in method_ir_graph.edges: + edge.graph = ir_graph + if edge.head == method_ir_graph.input_node: + # this is a member method, 'self' is the first argument, thus +1 + _input = node.inputsAt(edge.head_slot + 1) + src_node, src_node_idx = self._add_edge_handle_source_node(_input, graph_inputs, ir_graph, output_remap, node_index) + edge.head = src_node + edge.head_slot = src_node_idx + if edge.tail == method_ir_graph.output_node: + # since the following nodes have not been created, skip this edge + # edge.head is the output node of this method + # TODO: check whether there could be multiple output nodes??? + node_index[node] = edge.head + continue + ir_graph.edges.append(edge) + + # ===================handle each single node=================== + def handle_single_node(node): + """ + Parameters + ---------- + node : torch._C.Node + the node from TorchScript graph + + Returns + ------- + Node + the created node ir + """ + if node.kind() == 'prim::CallMethod': + handle_function_callmethod(node) + elif node.kind() == 'prim::CallFunction': + func_type_str = self._remove_mangle(node.inputsAt(0).type().str()) + func = node.inputsAt(0).node() + assert func.kind() == 'prim::Constant' + assert func.hasAttribute('name') + func_name = func.s('name') + # create node for func + self.global_seq += 1 + func_node = ir_graph.add_node(build_full_name(module_name, func_name, self.global_seq), + '{}.{}'.format(func_type_str, func_name)) + node_index[node] = func_node + self._add_edge(ir_graph, node, graph_inputs, node_index, func_node, output_remap, ignore_first=True) + elif node.kind() == 'prim::Constant': + new_node = self.create_prim_constant_node(ir_graph, node, module_name) + node_index[node] = new_node + elif node.kind() in ['prim::ListConstruct', 'prim::ListUnpack', 'prim::TupleConstruct', 'prim::TupleUnpack']: + self.global_seq += 1 + prim_op_name = node.kind().split('::')[-1] + new_node = ir_graph.add_node(build_full_name(module_name, prim_op_name, self.global_seq), node.kind()) + node_index[node] = new_node + self._add_edge(ir_graph, node, graph_inputs, node_index, new_node, output_remap) + elif node.kind() == 'prim::GetAttr': + node_type, attrs = self.handle_prim_attr_node(node, module) + self.global_seq += 1 + new_node = ir_graph.add_node(build_full_name(module_name, OpTypeName.Attr, self.global_seq), + node_type, attrs) + node_index[node] = new_node + elif node.kind() == 'prim::If': + last_block_node = handle_if_node(node) + # last_block_node is None means no node in the branch block + node_index[node] = last_block_node + elif node.kind() == 'prim::Loop': + # refer to https://gist.github.com/liuzhe-lz/90c35d9dd6fd7f3f32544940151ab186 + raise RuntimeError('Loop has not been supported yet!') + elif node.kind().startswith('prim::'): + self.global_seq += 1 + prim_op_name = node.kind().replace('::', '__') + prim_node = ir_graph.add_node(build_full_name(module_name, prim_op_name, self.global_seq), node.kind()) + node_index[node] = prim_node + self._add_edge(ir_graph, node, graph_inputs, node_index, prim_node, output_remap) + elif node.kind() == 'aten::append': + self.global_seq += 1 + aten_op_name = node.kind().replace('::', '__') + aten_node = ir_graph.add_node(build_full_name(module_name, aten_op_name, self.global_seq), node.kind()) + node_index[node] = aten_node + self._add_edge(ir_graph, node, graph_inputs, node_index, aten_node, output_remap) + output_remap[node.inputsAt(0)] = node + elif node.kind().startswith('aten::'): + # handle aten::XXX + self.global_seq += 1 + aten_op_name = node.kind().replace('::', '__') + aten_node = ir_graph.add_node(build_full_name(module_name, aten_op_name, self.global_seq), node.kind()) + node_index[node] = aten_node + self._add_edge(ir_graph, node, graph_inputs, node_index, aten_node, output_remap) + else: + raise RuntimeError('Unsupported kind: {}'.format(node.kind())) + + return node_index[node] + + for node in sm_graph.nodes(): + handle_single_node(node) + + return node_index + + def merge_aten_slices(self, ir_graph): + """ + if there is aten::slice node, merge the consecutive ones together. + ```x[:, :, 1:, 1:]``` in python code will be converted into 4 node in torch script, + each node has 5 inputs: tensor, dim, x, y, z (i.e., x:y:z) + """ + head_slice_nodes = [] + has_slice_node = False + for node in ir_graph.hidden_nodes: + if node.operation.type == 'aten::slice': + has_slice_node = True + for pred in node.predecessors: + if pred.operation.type not in ['aten::slice', 'prim::Constant']: + head_slice_nodes.append(node) + break + if has_slice_node: + assert head_slice_nodes + + for head_node in head_slice_nodes: + slot = 0 + new_slice_node = ir_graph.add_node(build_full_name(head_node.name, 'merged'), OpTypeName.MergedSlice) + if len(head_node.incoming_edges) == 4: + # when slice is for one dimension list, there are only 4 inputs, thus merge is not needed + for edge in head_node.incoming_edges: + edge.tail = new_slice_node + for edge in head_node.outgoing_edges: + edge.head = new_slice_node + ir_graph.hidden_nodes.remove(head_node) + break + assert len(head_node.incoming_edges) == 5 + for edge in head_node.incoming_edges: + edge.tail = new_slice_node + slot += 5 + node = head_node + while len(node.successors) == 1 and node.successors[0].operation.type == 'aten::slice': + suc_node = node.successors[0] + assert len(suc_node.incoming_edges) == 5 + for edge in suc_node.incoming_edges: + if edge.tail_slot == 0: + edge.remove() + else: + edge.tail = new_slice_node + edge.tail_slot = slot + edge.tail_slot - 1 + slot += 4 + ir_graph.hidden_nodes.remove(node) + node = suc_node + + for edge in node.outgoing_edges: + edge.head = new_slice_node + ir_graph.hidden_nodes.remove(node) + + def refine_graph(self, ir_graph): + """ + Do the following process to simplify graph: + 1. remove unconnected constant node + 2. remove unconnected getattr node + """ + # some constant is not used, for example, function name as prim::Constant + self.remove_unconnected_nodes(ir_graph, targeted_type='prim::Constant') + self.remove_unconnected_nodes(ir_graph, targeted_type='prim::GetAttr') + self.merge_aten_slices(ir_graph) + + def _handle_inputchoice(self, module): + return { + 'n_candidates': module.n_candidates, + 'n_chosen': module.n_chosen, + 'reduction': module.reduction, + 'label': module.label + } + + def _handle_valuechoice(self, module): + return { + 'candidates': module.candidates, + 'label': module.label, + 'accessor': module._accessor + } + + def convert_module(self, script_module, module, module_name, ir_model): + """ + Convert a module to its graph ir (i.e., Graph) along with its input arguments + + Parameters + ---------- + script_module : torch.jit.RecursiveScriptModule + the script module of ```module``` obtained with torch.jit.script + module : nn.Module + the targeted module instance + module_name : str + the constructed name space of ```module``` + ir_model : Model + the whole graph ir + + Returns + ------- + Graph + the built graph ir from module, ```None``` means do not further parse the module + dict + the input arguments of this module + """ + + # NOTE: have not supported nested LayerChoice, i.e., a candidate module + # also has LayerChoice or InputChoice or ValueChoice + original_type_name = script_module.original_name + m_attrs = None + if original_type_name in MODULE_EXCEPT_LIST: + pass # do nothing + elif original_type_name == OpTypeName.LayerChoice: + graph = Graph(ir_model, -100, module_name, _internal=True) # graph_id is not used now + candidate_name_list = [f'layerchoice_{module.label}_{cand_name}' for cand_name in module.names] + for cand_name, cand in zip(candidate_name_list, module): + cand_type = '__torch__.' + get_importable_name(cand.__class__) + graph.add_node(cand_name, cand_type, get_init_parameters_or_fail(cand)) + graph._register() + return graph, {'mutation': 'layerchoice', 'label': module.label, 'candidates': candidate_name_list} + elif original_type_name == OpTypeName.InputChoice: + m_attrs = self._handle_inputchoice(module) + elif original_type_name == OpTypeName.ValueChoice: + m_attrs = self._handle_valuechoice(module) + elif original_type_name == OpTypeName.Placeholder: + m_attrs = get_init_parameters_or_fail(module) + elif module.__class__.__module__.startswith('torch.nn') and original_type_name in torch.nn.__dict__: + # this is a basic module from pytorch, no need to parse its graph + m_attrs = get_init_parameters_or_fail(module) + else: + # this module is marked as serialize, won't continue to parse + m_attrs = get_init_parameters_or_fail(module, silently=True) + if m_attrs is not None: + return None, m_attrs + + # handle TorchScript graph + sm_graph = script_module.graph + self.global_graph_id += 1 + ir_graph = Graph(model=ir_model, graph_id=self.global_graph_id, name=module_name, _internal=True) + + # handle graph nodes + node_index = self.handle_graph_nodes(script_module, sm_graph, module, + module_name, ir_model, ir_graph) + + # handle graph outputs + for _output in sm_graph.outputs(): + ir_graph._add_output(_convert_name(_output.debugName())) + predecessor_node_outputs = [o for o in _output.node().outputs()] + if len(predecessor_node_outputs) == 1: + src_node_idx = None + else: + src_node_idx = predecessor_node_outputs.index(_output) + ir_graph.add_edge(head=(node_index[_output.node()], src_node_idx), + tail=(ir_graph.output_node, None)) + + self.refine_graph(ir_graph) + + ir_graph._register() + + # add mutation signal for special modules + if original_type_name == OpTypeName.Repeat: + attrs = { + 'mutation': 'repeat', + 'label': module.label, + 'min_depth': module.min_depth, + 'max_depth': module.max_depth + } + return ir_graph, attrs + + return ir_graph, {} + + +def convert_to_graph(script_module, module): + """ + Convert module to our graph ir, i.e., build a ```Model``` type + + Parameters + ---------- + script_module : torch.jit.RecursiveScriptModule + the script module obtained with torch.jit.script + module : nn.Module + the targeted module instance + + Returns + ------- + Model + the constructed IR model + """ + + model = Model(_internal=True) + module_name = '_model' + GraphConverter().convert_module(script_module, module, module_name, model) + + return model diff --git a/utils/third_party/nni_new/retiarii/converter/op_types.py b/utils/third_party/nni_new/retiarii/converter/op_types.py new file mode 100644 index 0000000000000000000000000000000000000000..1a4ba5a42db8743a154d3cbe490858a480309e93 --- /dev/null +++ b/utils/third_party/nni_new/retiarii/converter/op_types.py @@ -0,0 +1,21 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from enum import Enum + +MODULE_EXCEPT_LIST = ['Sequential'] + + +class OpTypeName(str, Enum): + """ + op type to its type name str + """ + Attr = 'Attr' + Constant = 'Constant' + LayerChoice = 'LayerChoice' + InputChoice = 'InputChoice' + ValueChoice = 'ValueChoice' + Placeholder = 'Placeholder' + MergedSlice = 'MergedSlice' + Repeat = 'Repeat' + Cell = 'Cell' diff --git a/utils/third_party/nni_new/retiarii/converter/utils.py b/utils/third_party/nni_new/retiarii/converter/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..c43f62176ed70024f62d254770cb5c918c18016c --- /dev/null +++ b/utils/third_party/nni_new/retiarii/converter/utils.py @@ -0,0 +1,17 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +def build_full_name(prefix, name, seq=None): + if isinstance(name, list): + name = '__'.join(name) + if seq is None: + return '{}__{}'.format(prefix, name) + else: + return '{}__{}{}'.format(prefix, name, str(seq)) + + +def _convert_name(name: str) -> str: + """ + Convert the names using separator '.' to valid variable name in code + """ + return name.replace('.', '__') diff --git a/utils/third_party/nni_new/retiarii/converter/visualize.py b/utils/third_party/nni_new/retiarii/converter/visualize.py new file mode 100644 index 0000000000000000000000000000000000000000..2bfe299198d65c29d87aadbd1e8a1001c045992c --- /dev/null +++ b/utils/third_party/nni_new/retiarii/converter/visualize.py @@ -0,0 +1,44 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import graphviz + + +def convert_to_visualize(graph_ir, vgraph): + for name, graph in graph_ir.items(): + if name == '_evaluator': + continue + with vgraph.subgraph(name='cluster'+name) as subgraph: + subgraph.attr(color='blue') + cell_node = {} + ioput = {'_inputs': '{}-{}'.format(name, '_'.join(graph['inputs'])), + '_outputs': '{}-{}'.format(name, '_'.join(graph['outputs']))} + subgraph.node(ioput['_inputs']) + subgraph.node(ioput['_outputs']) + for node_name, node_value in graph['nodes'].items(): + value = node_value['operation'] + if value['type'] == '_cell': + cell_input_name = '{}-{}'.format(value['cell_name'], '_'.join(graph_ir[value['cell_name']]['inputs'])) + cell_output_name = '{}-{}'.format(value['cell_name'], '_'.join(graph_ir[value['cell_name']]['outputs'])) + cell_node[node_name] = (cell_input_name, cell_output_name) + print('cell: ', node_name, cell_input_name, cell_output_name) + else: + subgraph.node(node_name) + for edge in graph['edges']: + src = edge['head'][0] + if src == '_inputs': + src = ioput['_inputs'] + elif src in cell_node: + src = cell_node[src][1] + dst = edge['tail'][0] + if dst == '_outputs': + dst = ioput['_outputs'] + elif dst in cell_node: + dst = cell_node[dst][0] + subgraph.edge(src, dst) + + +def visualize_model(graph_ir): + vgraph = graphviz.Digraph('G', filename='vgraph', format='jpg') + convert_to_visualize(graph_ir, vgraph) + vgraph.render() diff --git a/utils/third_party/nni_new/retiarii/debug_configs.py b/utils/third_party/nni_new/retiarii/debug_configs.py new file mode 100644 index 0000000000000000000000000000000000000000..9b4b1c643b33ae5336a4a0cad436311e5d30e331 --- /dev/null +++ b/utils/third_party/nni_new/retiarii/debug_configs.py @@ -0,0 +1,3 @@ +# we will support tensorflow in future release + +framework = 'pytorch' diff --git a/utils/third_party/nni_new/retiarii/evaluator/__init__.py b/utils/third_party/nni_new/retiarii/evaluator/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..c6fd370b906e8ad1545101c9ab0a8e3d605c0b33 --- /dev/null +++ b/utils/third_party/nni_new/retiarii/evaluator/__init__.py @@ -0,0 +1 @@ +from .functional import FunctionalEvaluator diff --git a/utils/third_party/nni_new/retiarii/evaluator/functional.py b/utils/third_party/nni_new/retiarii/evaluator/functional.py new file mode 100644 index 0000000000000000000000000000000000000000..3a7333284e94823ca9c6cee0eb9fd198c1a9a213 --- /dev/null +++ b/utils/third_party/nni_new/retiarii/evaluator/functional.py @@ -0,0 +1,37 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from ..graph import Evaluator + + +class FunctionalEvaluator(Evaluator): + """ + Functional evaluator that directly takes a function and thus should be general. + + Attributes + ---------- + function + The full name of the function. + arguments + Keyword arguments for the function other than model. + """ + + def __init__(self, function, **kwargs): + self.function = function + self.arguments = kwargs + + @staticmethod + def _load(ir): + return FunctionalEvaluator(ir['function'], **ir['arguments']) + + def _dump(self): + return { + 'function': self.function, + 'arguments': self.arguments + } + + def _execute(self, model_cls): + return self.function(model_cls, **self.arguments) + + def __eq__(self, other): + return self.function == other.function and self.arguments == other.arguments diff --git a/utils/third_party/nni_new/retiarii/evaluator/pytorch/__init__.py b/utils/third_party/nni_new/retiarii/evaluator/pytorch/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..c35431e91a9d71606de7d07625535785a0d6a23b --- /dev/null +++ b/utils/third_party/nni_new/retiarii/evaluator/pytorch/__init__.py @@ -0,0 +1,2 @@ +from .base import PyTorchImageClassificationTrainer, PyTorchMultiModelTrainer +from .lightning import * diff --git a/utils/third_party/nni_new/retiarii/evaluator/pytorch/base.py b/utils/third_party/nni_new/retiarii/evaluator/pytorch/base.py new file mode 100644 index 0000000000000000000000000000000000000000..62d25b4f740e6d806747ac8c79c1b3788582064a --- /dev/null +++ b/utils/third_party/nni_new/retiarii/evaluator/pytorch/base.py @@ -0,0 +1,305 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +# This file is deprecated. + +import abc +from typing import Any, List, Dict, Tuple + +import numpy as np +import torch +import torch.nn as nn +from torch.utils.data import DataLoader +from torchvision import datasets, transforms + +import nni + +class BaseTrainer(abc.ABC): + @abc.abstractmethod + def fit(self) -> None: + pass + + +def get_default_transform(dataset: str) -> Any: + """ + To get a default transformation of image for a specific dataset. + This is needed because transform objects can not be directly passed as arguments. + + Parameters + ---------- + dataset : str + Dataset class name. + + Returns + ------- + transform object + """ + if dataset == 'MNIST': + return transforms.Compose([ + transforms.ToTensor(), + transforms.Normalize((0.1307,), (0.3081,)) + ]) + if dataset == 'CIFAR10': + return transforms.Compose([ + transforms.RandomCrop(32, padding=4), + transforms.RandomHorizontalFlip(), + transforms.ToTensor(), + transforms.Normalize((0.4914, 0.4822, 0.4465), + (0.2023, 0.1994, 0.2010)), + ]) + # unsupported dataset, return None + return None + + +class PyTorchImageClassificationTrainer(BaseTrainer): + """ + Image classification trainer for PyTorch. + + A model, along with corresponding dataset, optimizer config is used to initialize the trainer. + The trainer will run for a fixed number of epochs (by default 10), and report the final result. + + TODO + Support scheduler, validate every n epochs, train/valid dataset + + Limitation induced by NNI: kwargs must be serializable to put into a JSON packed in parameters. + """ + + def __init__(self, model, + dataset_cls='MNIST', dataset_kwargs=None, dataloader_kwargs=None, + optimizer_cls='SGD', optimizer_kwargs=None, trainer_kwargs=None): + """Initialization of image classification trainer. + + Parameters + ---------- + model : nn.Module + Model to train. + dataset_cls : str, optional + Dataset class name that is available in ``torchvision.datasets``, by default 'MNIST' + dataset_kwargs : dict, optional + Keyword arguments passed to initialization of dataset class, by default None + dataset_kwargs : dict, optional + Keyword arguments passed to ``torch.utils.data.DataLoader``, by default None + optimizer_cls : str, optional + Optimizer class name that is available in ``torch.optim``, by default 'SGD' + optimizer_kwargs : dict, optional + Keyword arguments passed to initialization of optimizer class, by default None + trainer_kwargs: dict, optional + Keyword arguments passed to trainer. Will be passed to Trainer class in future. Currently, + only the key ``max_epochs`` is useful. + """ + super().__init__() + self._use_cuda = torch.cuda.is_available() + self.model = model + if self._use_cuda: + self.model.cuda() + self._loss_fn = nn.CrossEntropyLoss() + self._train_dataset = getattr(datasets, dataset_cls)(train=True, transform=get_default_transform(dataset_cls), + **(dataset_kwargs or {})) + self._val_dataset = getattr(datasets, dataset_cls)(train=False, transform=get_default_transform(dataset_cls), + **(dataset_kwargs or {})) + self._optimizer = getattr(torch.optim, optimizer_cls)(model.parameters(), **(optimizer_kwargs or {})) + self._trainer_kwargs = trainer_kwargs or {'max_epochs': 10} + + self._train_dataloader = DataLoader(self._train_dataset, **(dataloader_kwargs or {})) + self._val_dataloader = DataLoader(self._val_dataset, **(dataloader_kwargs or {})) + + def _accuracy(self, input, target): # pylint: disable=redefined-builtin + _, predict = torch.max(input.data, 1) + correct = predict.eq(target.data).cpu().sum().item() + return correct / input.size(0) + + def training_step(self, batch: Tuple[torch.Tensor, torch.Tensor], batch_idx: int) -> Dict[str, Any]: + x, y = self.training_step_before_model(batch, batch_idx) + y_hat = self.model(x) + return self.training_step_after_model(x, y, y_hat) + + def training_step_before_model(self, batch: Tuple[torch.Tensor, torch.Tensor], batch_idx: int): + x, y = batch + if self._use_cuda: + x, y = x.cuda(torch.device('cuda:0')), y.cuda(torch.device('cuda:0')) + return x, y + + def training_step_after_model(self, x, y, y_hat): + loss = self._loss_fn(y_hat, y) + return loss + + def validation_step(self, batch: Tuple[torch.Tensor, torch.Tensor], batch_idx: int) -> Dict[str, Any]: + x, y = self.validation_step_before_model(batch, batch_idx) + y_hat = self.model(x) + return self.validation_step_after_model(x, y, y_hat) + + def validation_step_before_model(self, batch: Tuple[torch.Tensor, torch.Tensor], batch_idx: int): + x, y = batch + if self._use_cuda: + x, y = x.cuda(), y.cuda() + return x, y + + def validation_step_after_model(self, x, y, y_hat): + acc = self._accuracy(y_hat, y) + return {'val_acc': acc} + + def validation_epoch_end(self, outputs: List[Dict[str, Any]]) -> Dict[str, Any]: + # We might need dict metrics in future? + avg_acc = np.mean([x['val_acc'] for x in outputs]).item() + nni.report_intermediate_result(avg_acc) + return {'val_acc': avg_acc} + + def _validate(self): + validation_outputs = [] + for i, batch in enumerate(self._val_dataloader): + validation_outputs.append(self.validation_step(batch, i)) + return self.validation_epoch_end(validation_outputs) + + def _train(self): + for i, batch in enumerate(self._train_dataloader): + self._optimizer.zero_grad() + loss = self.training_step(batch, i) + loss.backward() + self._optimizer.step() + + def fit(self) -> None: + for _ in range(self._trainer_kwargs['max_epochs']): + self._train() + self._validate() + # assuming val_acc here + nni.report_final_result(self._validate()['val_acc']) + + +class PyTorchMultiModelTrainer(BaseTrainer): + def __init__(self, multi_model, kwargs=[]): + self.multi_model = multi_model + self.kwargs = kwargs + self._train_dataloaders = [] + self._train_datasets = [] + self._val_dataloaders = [] + self._val_datasets = [] + self._optimizers = [] + self._trainers = [] + self._loss_fn = nn.CrossEntropyLoss() + self.max_steps = self.kwargs['max_steps'] if 'makx_steps' in self.kwargs else None + self.n_model = len(self.kwargs['model_kwargs']) + + for m in self.kwargs['model_kwargs']: + if m['use_input']: + dataset_cls = m['dataset_cls'] + dataset_kwargs = m['dataset_kwargs'] + dataloader_kwargs = m['dataloader_kwargs'] + train_dataset = getattr(datasets, dataset_cls)(train=True, transform=get_default_transform(dataset_cls), + **(dataset_kwargs or {})) + val_dataset = getattr(datasets, dataset_cls)(train=False, transform=get_default_transform(dataset_cls), + **(dataset_kwargs or {})) + train_dataloader = DataLoader(train_dataset, **(dataloader_kwargs or {})) + val_dataloader = DataLoader(val_dataset, **(dataloader_kwargs or {})) + self._train_datasets.append(train_dataset) + self._train_dataloaders.append(train_dataloader) + + self._val_datasets.append(val_dataset) + self._val_dataloaders.append(val_dataloader) + + if m['use_output']: + optimizer_cls = m['optimizer_cls'] + optimizer_kwargs = m['optimizer_kwargs'] + m_header = f"M_{m['model_id']}" + one_model_params = [] + for name, param in multi_model.named_parameters(): + name_prefix = '_'.join(name.split('_')[:2]) + if m_header == name_prefix: + one_model_params.append(param) + + optimizer = getattr(torch.optim, optimizer_cls)(one_model_params, **(optimizer_kwargs or {})) + self._optimizers.append(optimizer) + + def fit(self) -> None: + torch.autograd.set_detect_anomaly(True) + max_epochs = max([x['trainer_kwargs']['max_epochs'] for x in self.kwargs['model_kwargs']]) + for _ in range(max_epochs): + self._train() + self._validate() + nni.report_final_result(self._validate()) + + def _train(self): + for batch_idx, multi_model_batch in enumerate(zip(*self._train_dataloaders)): + for opt in self._optimizers: + opt.zero_grad() + xs = [] + ys = [] + for idx, batch in enumerate(multi_model_batch): + x, y = self.training_step_before_model(batch, batch_idx, f'cuda:{idx}') + xs.append(x) + ys.append(y) + + y_hats = self.multi_model(*xs) + if len(ys) != len(xs): + raise ValueError('len(ys) should be equal to len(xs)') + losses = [] + report_loss = {} + for output_idx, yhat in enumerate(y_hats): + if len(ys) == len(y_hats): + loss = self.training_step_after_model(xs[output_idx], ys[output_idx], yhat) + elif len(ys) == 1: + loss = self.training_step_after_model(xs[0], ys[0].to(yhat.get_device()), yhat) + else: + raise ValueError('len(ys) should be either 1 or len(y_hats)') + losses.append(loss.to("cuda:0")) + report_loss[self.kwargs['model_kwargs'][output_idx]['model_id']] = loss.item() + summed_loss = sum(losses) + summed_loss.backward() + for opt in self._optimizers: + opt.step() + if self.max_steps and batch_idx >= self.max_steps: + return + + def training_step_before_model(self, batch: Tuple[torch.Tensor, torch.Tensor], batch_idx: int, device=None): + x, y = batch + if device: + x, y = x.cuda(torch.device(device)), y.cuda(torch.device(device)) + return x, y + + def training_step_after_model(self, x, y, y_hat): + loss = self._loss_fn(y_hat, y) + return loss + + def _validate(self): + all_val_outputs = {idx: [] for idx in range(self.n_model)} + for batch_idx, multi_model_batch in enumerate(zip(*self._val_dataloaders)): + xs = [] + ys = [] + for idx, batch in enumerate(multi_model_batch): + x, y = self.training_step_before_model(batch, batch_idx, f'cuda:{idx}') + xs.append(x) + ys.append(y) + if len(ys) != len(xs): + raise ValueError('len(ys) should be equal to len(xs)') + + y_hats = self.multi_model(*xs) + + for output_idx, yhat in enumerate(y_hats): + if len(ys) == len(y_hats): + acc = self.validation_step_after_model(xs[output_idx], ys[output_idx], yhat) + elif len(ys) == 1: + acc = self.validation_step_after_model(xs[0], ys[0].to(yhat.get_device()), yhat) + else: + raise ValueError('len(ys) should be either 1 or len(y_hats)') + all_val_outputs[output_idx].append(acc) + + report_acc = {} + for idx in all_val_outputs: + avg_acc = np.mean([x['val_acc'] for x in all_val_outputs[idx]]).item() + report_acc[self.kwargs['model_kwargs'][idx]['model_id']] = avg_acc + nni.report_intermediate_result(report_acc) + return report_acc + + def validation_step_before_model(self, batch: Tuple[torch.Tensor, torch.Tensor], batch_idx: int, device=None): + x, y = batch + if device: + x, y = x.cuda(torch.device(device)), y.cuda(torch.device(device)) + return x, y + + def validation_step_after_model(self, x, y, y_hat): + acc = self._accuracy(y_hat, y) + return {'val_acc': acc} + + def _accuracy(self, input, target): # pylint: disable=redefined-builtin + _, predict = torch.max(input.data, 1) + correct = predict.eq(target.data).cpu().sum().item() + return correct / input.size(0) diff --git a/utils/third_party/nni_new/retiarii/evaluator/pytorch/lightning.py b/utils/third_party/nni_new/retiarii/evaluator/pytorch/lightning.py new file mode 100644 index 0000000000000000000000000000000000000000..4399844ac61944f2174ade19f78c2b9187efcbb3 --- /dev/null +++ b/utils/third_party/nni_new/retiarii/evaluator/pytorch/lightning.py @@ -0,0 +1,266 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import warnings +from typing import Dict, Union, Optional, List + +import pytorch_lightning as pl +import torch.nn as nn +import torch.optim as optim +from torch.utils.data import DataLoader + +import nni +from ...graph import Evaluator +from ...serializer import serialize_cls + + +__all__ = ['LightningModule', 'Trainer', 'DataLoader', 'Lightning', 'Classification', 'Regression'] + + +class LightningModule(pl.LightningModule): + def set_model(self, model): + if isinstance(model, type): + self.model = model() + else: + self.model = model + + +Trainer = serialize_cls(pl.Trainer) +DataLoader = serialize_cls(DataLoader) + + +class Lightning(Evaluator): + """ + Delegate the whole training to PyTorch Lightning. + + Since the arguments passed to the initialization needs to be serialized, ``LightningModule``, ``Trainer`` or + ``DataLoader`` in this file should be used. Another option is to hide dataloader in the Lightning module, in + which case, dataloaders are not required for this class to work. + + Following the programming style of Lightning, metrics sent to NNI should be obtained from ``callback_metrics`` + in trainer. Two hooks are added at the end of validation epoch and the end of ``fit``, respectively. The metric name + and type depend on the specific task. + + Parameters + ---------- + lightning_module : LightningModule + Lightning module that defines the training logic. + trainer : Trainer + Lightning trainer that handles the training. + train_dataloders : DataLoader + Used in ``trainer.fit()``. A PyTorch DataLoader with training samples. + If the ``lightning_module`` has a predefined train_dataloader method this will be skipped. + val_dataloaders : DataLoader or List of DataLoader + Used in ``trainer.fit()``. Either a single PyTorch Dataloader or a list of them, specifying validation samples. + If the ``lightning_module`` has a predefined val_dataloaders method this will be skipped. + """ + + def __init__(self, lightning_module: LightningModule, trainer: Trainer, + train_dataloader: Optional[DataLoader] = None, + val_dataloaders: Union[DataLoader, List[DataLoader], None] = None): + assert isinstance(lightning_module, LightningModule), f'Lightning module must be an instance of {__name__}.LightningModule.' + assert isinstance(trainer, Trainer), f'Trainer must be imported from {__name__}.' + assert _check_dataloader(train_dataloader), f'Wrong dataloader type. Try import DataLoader from {__name__}.' + assert _check_dataloader(val_dataloaders), f'Wrong dataloader type. Try import DataLoader from {__name__}.' + self.module = lightning_module + self.trainer = trainer + self.train_dataloader = train_dataloader + self.val_dataloaders = val_dataloaders + + @staticmethod + def _load(ir): + return Lightning(ir['module'], ir['trainer'], ir['train_dataloader'], ir['val_dataloaders']) + + def _dump(self): + return { + 'module': self.module, + 'trainer': self.trainer, + 'train_dataloader': self.train_dataloader, + 'val_dataloaders': self.val_dataloaders + } + + def _execute(self, model_cls): + return self.fit(model_cls) + + def __eq__(self, other): + return self.function == other.function and self.arguments == other.arguments + + def fit(self, model): + """ + Fit the model with provided dataloader, with Lightning trainer. + + Parameters + ---------- + model : nn.Module + The model to fit. + """ + self.module.set_model(model) + return self.trainer.fit(self.module, self.train_dataloader, self.val_dataloaders) + + +def _check_dataloader(dataloader): + if dataloader is None: + return True + if isinstance(dataloader, list): + return all([_check_dataloader(d) for d in dataloader]) + return isinstance(dataloader, DataLoader) + + +### The following are some commonly used Lightning modules ### + +class _SupervisedLearningModule(LightningModule): + def __init__(self, criterion: nn.Module, metrics: Dict[str, pl.metrics.Metric], + learning_rate: float = 0.001, + weight_decay: float = 0., + optimizer: optim.Optimizer = optim.Adam): + super().__init__() + self.save_hyperparameters('criterion', 'optimizer', 'learning_rate', 'weight_decay') + self.criterion = criterion() + self.optimizer = optimizer + self.metrics = nn.ModuleDict({name: cls() for name, cls in metrics.items()}) + + def forward(self, x): + y_hat = self.model(x) + return y_hat + + def training_step(self, batch, batch_idx): + x, y = batch + y_hat = self(x) + loss = self.criterion(y_hat, y) + self.log('train_loss', loss, prog_bar=True) + for name, metric in self.metrics.items(): + self.log('train_' + name, metric(y_hat, y), prog_bar=True) + return loss + + def validation_step(self, batch, batch_idx): + x, y = batch + y_hat = self(x) + self.log('val_loss', self.criterion(y_hat, y), prog_bar=True) + for name, metric in self.metrics.items(): + self.log('val_' + name, metric(y_hat, y), prog_bar=True) + + def test_step(self, batch, batch_idx): + x, y = batch + y_hat = self(x) + self.log('test_loss', self.criterion(y_hat, y), prog_bar=True) + for name, metric in self.metrics.items(): + self.log('test_' + name, metric(y_hat, y), prog_bar=True) + + def configure_optimizers(self): + return self.optimizer(self.parameters(), lr=self.hparams.learning_rate, weight_decay=self.hparams.weight_decay) + + def on_validation_epoch_end(self): + nni.report_intermediate_result(self._get_validation_metrics()) + + def teardown(self, stage): + if stage == 'fit': + nni.report_final_result(self._get_validation_metrics()) + + def _get_validation_metrics(self): + if len(self.metrics) == 1: + metric_name = next(iter(self.metrics)) + return self.trainer.callback_metrics['val_' + metric_name].item() + else: + warnings.warn('Multiple metrics without "default" is not supported by current framework.') + return {name: self.trainer.callback_metrics['val_' + name].item() for name in self.metrics} + + +class _AccuracyWithLogits(pl.metrics.Accuracy): + def update(self, pred, target): + return super().update(nn.functional.softmax(pred), target) + + +@serialize_cls +class _ClassificationModule(_SupervisedLearningModule): + def __init__(self, criterion: nn.Module = nn.CrossEntropyLoss, + learning_rate: float = 0.001, + weight_decay: float = 0., + optimizer: optim.Optimizer = optim.Adam): + super().__init__(criterion, {'acc': _AccuracyWithLogits}, + learning_rate=learning_rate, weight_decay=weight_decay, optimizer=optimizer) + + +class Classification(Lightning): + """ + Trainer that is used for classification. + + Parameters + ---------- + criterion : nn.Module + Class for criterion module (not an instance). default: ``nn.CrossEntropyLoss`` + learning_rate : float + Learning rate. default: 0.001 + weight_decay : float + L2 weight decay. default: 0 + optimizer : Optimizer + Class for optimizer (not an instance). default: ``Adam`` + train_dataloders : DataLoader + Used in ``trainer.fit()``. A PyTorch DataLoader with training samples. + If the ``lightning_module`` has a predefined train_dataloader method this will be skipped. + val_dataloaders : DataLoader or List of DataLoader + Used in ``trainer.fit()``. Either a single PyTorch Dataloader or a list of them, specifying validation samples. + If the ``lightning_module`` has a predefined val_dataloaders method this will be skipped. + trainer_kwargs : dict + Optional keyword arguments passed to trainer. See + `Lightning documentation `__ for details. + """ + + def __init__(self, criterion: nn.Module = nn.CrossEntropyLoss, + learning_rate: float = 0.001, + weight_decay: float = 0., + optimizer: optim.Optimizer = optim.Adam, + train_dataloader: Optional[DataLoader] = None, + val_dataloaders: Union[DataLoader, List[DataLoader], None] = None, + **trainer_kwargs): + module = _ClassificationModule(criterion=criterion, learning_rate=learning_rate, + weight_decay=weight_decay, optimizer=optimizer) + super().__init__(module, Trainer(**trainer_kwargs), + train_dataloader=train_dataloader, val_dataloaders=val_dataloaders) + + +@serialize_cls +class _RegressionModule(_SupervisedLearningModule): + def __init__(self, criterion: nn.Module = nn.MSELoss, + learning_rate: float = 0.001, + weight_decay: float = 0., + optimizer: optim.Optimizer = optim.Adam): + super().__init__(criterion, {'mse': pl.metrics.MeanSquaredError}, + learning_rate=learning_rate, weight_decay=weight_decay, optimizer=optimizer) + + +class Regression(Lightning): + """ + Trainer that is used for regression. + + Parameters + ---------- + criterion : nn.Module + Class for criterion module (not an instance). default: ``nn.MSELoss`` + learning_rate : float + Learning rate. default: 0.001 + weight_decay : float + L2 weight decay. default: 0 + optimizer : Optimizer + Class for optimizer (not an instance). default: ``Adam`` + train_dataloders : DataLoader + Used in ``trainer.fit()``. A PyTorch DataLoader with training samples. + If the ``lightning_module`` has a predefined train_dataloader method this will be skipped. + val_dataloaders : DataLoader or List of DataLoader + Used in ``trainer.fit()``. Either a single PyTorch Dataloader or a list of them, specifying validation samples. + If the ``lightning_module`` has a predefined val_dataloaders method this will be skipped. + trainer_kwargs : dict + Optional keyword arguments passed to trainer. See + `Lightning documentation `__ for details. + """ + + def __init__(self, criterion: nn.Module = nn.MSELoss, + learning_rate: float = 0.001, + weight_decay: float = 0., + optimizer: optim.Optimizer = optim.Adam, + train_dataloader: Optional[DataLoader] = None, + val_dataloaders: Union[DataLoader, List[DataLoader], None] = None, + **trainer_kwargs): + module = _RegressionModule(criterion=criterion, learning_rate=learning_rate, + weight_decay=weight_decay, optimizer=optimizer) + super().__init__(module, Trainer(**trainer_kwargs), + train_dataloader=train_dataloader, val_dataloaders=val_dataloaders) diff --git a/utils/third_party/nni_new/retiarii/execution/__init__.py b/utils/third_party/nni_new/retiarii/execution/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..0a0e47b0b01b597936c8c2a8a09cafeda899168b --- /dev/null +++ b/utils/third_party/nni_new/retiarii/execution/__init__.py @@ -0,0 +1 @@ +from .api import * diff --git a/utils/third_party/nni_new/retiarii/execution/api.py b/utils/third_party/nni_new/retiarii/execution/api.py new file mode 100644 index 0000000000000000000000000000000000000000..8027e7e36308c4cf511ba1942264be9bdb2dfef1 --- /dev/null +++ b/utils/third_party/nni_new/retiarii/execution/api.py @@ -0,0 +1,74 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import time +from typing import Iterable + +from ..graph import Model, ModelStatus +from .interface import AbstractExecutionEngine +from .listener import DefaultListener + +_execution_engine = None +_default_listener = None + +__all__ = ['get_execution_engine', 'get_and_register_default_listener', + 'list_models', 'submit_models', 'wait_models', 'query_available_resources', + 'set_execution_engine', 'is_stopped_exec', 'budget_exhausted'] + + +def set_execution_engine(engine: AbstractExecutionEngine) -> None: + global _execution_engine + if _execution_engine is None: + _execution_engine = engine + else: + raise RuntimeError('Execution engine is already set.') + + +def get_execution_engine() -> AbstractExecutionEngine: + global _execution_engine + assert _execution_engine is not None, 'You need to set execution engine, before using it.' + return _execution_engine + + +def get_and_register_default_listener(engine: AbstractExecutionEngine) -> DefaultListener: + global _default_listener + if _default_listener is None: + _default_listener = DefaultListener() + engine.register_graph_listener(_default_listener) + return _default_listener + + +def submit_models(*models: Model) -> None: + engine = get_execution_engine() + get_and_register_default_listener(engine) + engine.submit_models(*models) + + +def list_models(*models: Model) -> Iterable[Model]: + engine = get_execution_engine() + get_and_register_default_listener(engine) + return engine.list_models() + + +def wait_models(*models: Model) -> None: + get_and_register_default_listener(get_execution_engine()) + while True: + time.sleep(1) + left_models = [g for g in models if not g.status in (ModelStatus.Trained, ModelStatus.Failed)] + if not left_models: + break + + +def query_available_resources() -> int: + engine = get_execution_engine() + resources = engine.query_available_resource() + return resources if isinstance(resources, int) else len(resources) + + +def is_stopped_exec(model: Model) -> bool: + return model.status in (ModelStatus.Trained, ModelStatus.Failed) + + +def budget_exhausted() -> bool: + engine = get_execution_engine() + return engine.budget_exhausted() diff --git a/utils/third_party/nni_new/retiarii/execution/base.py b/utils/third_party/nni_new/retiarii/execution/base.py new file mode 100644 index 0000000000000000000000000000000000000000..36d09b505fbe0090a88694034ae6c5aed57f099c --- /dev/null +++ b/utils/third_party/nni_new/retiarii/execution/base.py @@ -0,0 +1,128 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +import os +import random +import string +from typing import Any, Dict, Iterable, List + +from .interface import AbstractExecutionEngine, AbstractGraphListener +from .. import codegen, utils +from ..graph import Model, ModelStatus, MetricData, Evaluator +from ..integration_api import send_trial, receive_trial_parameters, get_advisor + +_logger = logging.getLogger(__name__) + +class BaseGraphData: + def __init__(self, model_script: str, evaluator: Evaluator) -> None: + self.model_script = model_script + self.evaluator = evaluator + + def dump(self) -> dict: + return { + 'model_script': self.model_script, + 'evaluator': self.evaluator + } + + @staticmethod + def load(data) -> 'BaseGraphData': + return BaseGraphData(data['model_script'], data['evaluator']) + + +class BaseExecutionEngine(AbstractExecutionEngine): + """ + The execution engine with no optimization at all. + Resource management is implemented in this class. + """ + + def __init__(self) -> None: + """ + Upon initialization, advisor callbacks need to be registered. + Advisor will call the callbacks when the corresponding event has been triggered. + Base execution engine will get those callbacks and broadcast them to graph listener. + """ + self._listeners: List[AbstractGraphListener] = [] + + # register advisor callbacks + advisor = get_advisor() + advisor.send_trial_callback = self._send_trial_callback + advisor.request_trial_jobs_callback = self._request_trial_jobs_callback + advisor.trial_end_callback = self._trial_end_callback + advisor.intermediate_metric_callback = self._intermediate_metric_callback + advisor.final_metric_callback = self._final_metric_callback + + self._running_models: Dict[int, Model] = dict() + self._history: List[Model] = [] + + self.resources = 0 + + def submit_models(self, *models: Model) -> None: + for model in models: + data = self.pack_model_data(model) + self._running_models[send_trial(data.dump())] = model + self._history.append(model) + + def list_models(self) -> Iterable[Model]: + return self._history + + def register_graph_listener(self, listener: AbstractGraphListener) -> None: + self._listeners.append(listener) + + def _send_trial_callback(self, paramater: dict) -> None: + if self.resources <= 0: + # FIXME: should be a warning message here + _logger.debug('There is no available resource, but trial is submitted.') + self.resources -= 1 + _logger.debug('Resource used. Remaining: %d', self.resources) + + def _request_trial_jobs_callback(self, num_trials: int) -> None: + self.resources += num_trials + _logger.debug('New resource available. Remaining: %d', self.resources) + + def _trial_end_callback(self, trial_id: int, success: bool) -> None: + model = self._running_models[trial_id] + if success: + model.status = ModelStatus.Trained + else: + model.status = ModelStatus.Failed + for listener in self._listeners: + listener.on_training_end(model, success) + + def _intermediate_metric_callback(self, trial_id: int, metrics: MetricData) -> None: + model = self._running_models[trial_id] + model.intermediate_metrics.append(metrics) + for listener in self._listeners: + listener.on_intermediate_metric(model, metrics) + + def _final_metric_callback(self, trial_id: int, metrics: MetricData) -> None: + model = self._running_models[trial_id] + model.metric = metrics + for listener in self._listeners: + listener.on_metric(model, metrics) + + def query_available_resource(self) -> int: + return self.resources + + def budget_exhausted(self) -> bool: + advisor = get_advisor() + return advisor.stopping + + @classmethod + def pack_model_data(cls, model: Model) -> Any: + return BaseGraphData(codegen.model_to_pytorch_script(model), model.evaluator) + + @classmethod + def trial_execute_graph(cls) -> None: + """ + Initialize the model, hand it over to trainer. + """ + graph_data = BaseGraphData.load(receive_trial_parameters()) + random_str = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(6)) + file_name = f'_generated_model/{random_str}.py' + os.makedirs(os.path.dirname(file_name), exist_ok=True) + with open(file_name, 'w') as f: + f.write(graph_data.model_script) + model_cls = utils.import_(f'_generated_model.{random_str}._model') + graph_data.evaluator._execute(model_cls) + os.remove(file_name) diff --git a/utils/third_party/nni_new/retiarii/execution/cgo_engine.py b/utils/third_party/nni_new/retiarii/execution/cgo_engine.py new file mode 100644 index 0000000000000000000000000000000000000000..ee8e59498440a2326cbce12cb233bf854c36f196 --- /dev/null +++ b/utils/third_party/nni_new/retiarii/execution/cgo_engine.py @@ -0,0 +1,159 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +from typing import Iterable, List, Dict, Tuple + +from .interface import AbstractExecutionEngine, AbstractGraphListener, WorkerInfo +from .. import codegen, utils +from ..graph import Model, ModelStatus, MetricData +from ..integration_api import send_trial, receive_trial_parameters, get_advisor +from .logical_optimizer.logical_plan import LogicalPlan, PhysicalDevice +from .logical_optimizer.opt_dedup_input import DedupInputOptimizer + +from .base import BaseGraphData + +_logger = logging.getLogger(__name__) + + +class CGOExecutionEngine(AbstractExecutionEngine): + def __init__(self, n_model_per_graph=4) -> None: + self._listeners: List[AbstractGraphListener] = [] + self._running_models: Dict[int, Model] = dict() + self.logical_plan_counter = 0 + self.n_model_per_graph = n_model_per_graph + self._optimizers = [DedupInputOptimizer()] + self._original_models = {} + self._original_model_to_multi_model = {} + + # register advisor callbacks + advisor = get_advisor() + advisor.send_trial_callback = self._send_trial_callback + advisor.request_trial_jobs_callback = self._request_trial_jobs_callback + advisor.trial_end_callback = self._trial_end_callback + advisor.intermediate_metric_callback = self._intermediate_metric_callback + advisor.final_metric_callback = self._final_metric_callback + + def add_optimizer(self, opt): + self._optimizers.append(opt) + + def submit_models(self, *models: List[Model]) -> None: + _logger.info('%d models are submitted', len(models)) + logical = self._build_logical(models) + + for opt in self._optimizers: + opt.convert(logical) + + phy_models_and_placements = self._assemble(logical) + for model, placement, grouped_models in phy_models_and_placements: + data = BaseGraphData(codegen.model_to_pytorch_script(model, placement=placement), + model.evaluator) + for m in grouped_models: + self._original_models[m.model_id] = m + self._original_model_to_multi_model[m.model_id] = model + self._running_models[send_trial(data.dump())] = model + + # for model in models: + # data = BaseGraphData(codegen.model_to_pytorch_script(model), + # model.config['trainer_module'], model.config['trainer_kwargs']) + # self._running_models[send_trial(data.dump())] = model + + def list_models(self) -> Iterable[Model]: + raise NotImplementedError + + def _assemble(self, logical_plan: LogicalPlan) -> List[Tuple[Model, PhysicalDevice]]: + # unique_models = set() + # for node in logical_plan.graph.nodes: + # if node.graph.model not in unique_models: + # unique_models.add(node.graph.model) + # return [m for m in unique_models] + grouped_models: List[Dict[Model, PhysicalDevice]] = AssemblePolicy().group(logical_plan) + phy_models_and_placements = [] + for multi_model in grouped_models: + model, model_placement = logical_plan.assemble(multi_model) + phy_models_and_placements.append((model, model_placement, multi_model.keys())) + return phy_models_and_placements + + def _build_logical(self, models: List[Model]) -> LogicalPlan: + logical_plan = LogicalPlan(plan_id=self.logical_plan_counter) + for model in models: + logical_plan.add_model(model) + self.logical_plan_counter += 1 + return logical_plan + + def register_graph_listener(self, listener: AbstractGraphListener) -> None: + self._listeners.append(listener) + + def _send_trial_callback(self, paramater: dict) -> None: + for listener in self._listeners: + listener.on_resource_used(0) # FIXME: find the real resource id + + def _request_trial_jobs_callback(self, num_trials: int) -> None: + for listener in self._listeners: + listener.on_resource_available([0] * num_trials) # FIXME: find the real resource id + + def _trial_end_callback(self, trial_id: int, success: bool) -> None: + model = self._running_models[trial_id] + if success: + model.status = ModelStatus.Trained + else: + model.status = ModelStatus.Failed + for model_id in self._original_model_to_multi_model: + if self._original_model_to_multi_model[model_id] == model: + original_model = self._original_models[model_id] + if success: + original_model.status = ModelStatus.Trained + else: + original_model.status = ModelStatus.Failed + for listener in self._listeners: + listener.on_training_end(original_model, success) + + def _intermediate_metric_callback(self, trial_id: int, metrics: MetricData) -> None: + # model = self._running_models[trial_id] + merged_metrics = dict(metrics) + for model_id in merged_metrics: + int_model_id = int(model_id) + self._original_models[int_model_id].intermediate_metrics.append(merged_metrics[model_id]) + # model.intermediate_metrics.append(metrics) + for listener in self._listeners: + listener.on_intermediate_metric(self._original_models[int_model_id], merged_metrics[model_id]) + + def _final_metric_callback(self, trial_id: int, metrics: MetricData) -> None: + merged_metrics = dict(metrics) + for model_id in merged_metrics: + int_model_id = int(model_id) + self._original_models[int_model_id].intermediate_metrics.append(merged_metrics[model_id]) + # model.intermediate_metrics.append(metrics) + for listener in self._listeners: + listener.on_metric(self._original_models[int_model_id], merged_metrics[model_id]) + + def query_available_resource(self) -> List[WorkerInfo]: + raise NotImplementedError # move the method from listener to here? + + def budget_exhausted(self) -> bool: + raise NotImplementedError + + @classmethod + def trial_execute_graph(cls) -> None: + """ + Initialize the model, hand it over to trainer. + """ + graph_data = BaseGraphData.load(receive_trial_parameters()) + _logger.info('CGO_ENGINE trial parameters received') + with open('_generated_model.py', 'w') as f: + f.write(graph_data.model_script) + # with open('_debug_graph_data.json', 'w') as f: + # json.dump(graph_data.dump(), f) + trainer_cls = utils.import_(graph_data.training_module) + model_cls = utils.import_(f"_generated_model.{graph_data.training_kwargs['model_cls']}") + trainer_instance = trainer_cls(model_cls(), graph_data.training_kwargs) + trainer_instance.fit() + + +class AssemblePolicy: + @staticmethod + def group(logical_plan): + group_model = {} + for idx, m in enumerate(logical_plan.models): + group_model[m] = PhysicalDevice('server', f'cuda:{idx}') + return [group_model] diff --git a/utils/third_party/nni_new/retiarii/execution/interface.py b/utils/third_party/nni_new/retiarii/execution/interface.py new file mode 100644 index 0000000000000000000000000000000000000000..ae74e241067640196ca87422b21415f17c023106 --- /dev/null +++ b/utils/third_party/nni_new/retiarii/execution/interface.py @@ -0,0 +1,153 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from abc import ABC, abstractmethod, abstractclassmethod +from typing import Any, Iterable, NewType, List, Union + +from ..graph import Model, MetricData + +__all__ = [ + 'GraphData', 'WorkerInfo', + 'AbstractGraphListener', 'AbstractExecutionEngine' +] + + +GraphData = NewType('GraphData', Any) +""" +A _serializable_ internal data type defined by execution engine. + +Execution engine will submit this kind of data through NNI to worker machine, and train it there. + +A `GraphData` object describes a (merged) executable graph. + +This is trial's "hyper-parameter" in NNI's term and will be transfered in JSON format. + +See `AbstractExecutionEngine` for details. +""" + + +WorkerInfo = NewType('WorkerInfo', Any) +""" +To be designed. Discussion needed. + +This describes the properties of a worker machine. (e.g. memory size) +""" + + +class AbstractGraphListener(ABC): + """ + Abstract listener interface to receive graph events. + + Use `AbstractExecutionEngine.register_graph_listener()` to activate a listener. + """ + + @abstractmethod + def on_metric(self, model: Model, metric: MetricData) -> None: + """ + Reports the final metric of a graph. + """ + raise NotImplementedError + + @abstractmethod + def on_intermediate_metric(self, model: Model, metric: MetricData) -> None: + """ + Reports the latest intermediate metric of a trainning graph. + """ + pass + + @abstractmethod + def on_training_end(self, model: Model, success: bool) -> None: + """ + Reports either a graph is fully trained or the training process has failed. + """ + pass + + +class AbstractExecutionEngine(ABC): + """ + The abstract interface of execution engine. + + Most of these APIs are used by strategy, except `trial_execute_graph`, which is invoked by framework in trial. + Strategy will get the singleton execution engine object through a global API, + and use it in either sync or async manner. + + Execution engine is responsible for submitting (maybe-optimized) models to NNI, + and assigning their metrics to the `Model` object after training. + Execution engine is also responsible to launch the graph in trial process, + because it's the only one who understands graph data, or "hyper-parameter" in NNI's term. + + Execution engine will leverage NNI Advisor APIs, which are yet open for discussion. + + In synchronized use case, the strategy will have a loop to call `submit_models` and `wait_models` repeatly, + and will receive metrics from `Model` attributes. + Execution engine could assume that strategy will only submit graph when there are availabe resources (for now). + + In asynchronized use case, the strategy will register a listener to receive events, + while still using `submit_models` to train. + + There will be a `BaseExecutionEngine` subclass. + Inner-graph optimizing is supposed to derive `BaseExecutionEngine`, + while overrides `submit_models` and `trial_execute_graph`. + cross-graph optimizing is supposed to derive `AbstractExectutionEngine` directly, + because in this case APIs like `wait_graph` and `listener.on_training_end` will have unique logic. + + There might be some util functions benefit all optimizing methods, + but non-mandatory utils should not be covered in abstract interface. + """ + + @abstractmethod + def submit_models(self, *models: Model) -> None: + """ + Submit models to NNI. + + This method is supposed to call something like `nni.Advisor.create_trial_job(graph_data)`. + """ + raise NotImplementedError + + @abstractmethod + def list_models(self) -> Iterable[Model]: + """ + Get all models in submitted. + + Execution engine should store a copy of models that have been submitted and return a list of copies in this method. + """ + raise NotImplementedError + + @abstractmethod + def query_available_resource(self) -> Union[List[WorkerInfo], int]: + """ + Returns information of all idle workers. + If no details are available, this may returns a list of "empty" objects, reporting the number of idle workers. + + Could be left unimplemented for first iteration. + """ + raise NotImplementedError + + @abstractmethod + def budget_exhausted(self) -> bool: + """ + Check whether user configured max trial number or max execution duration has been reached + """ + raise NotImplementedError + + @abstractmethod + def register_graph_listener(self, listener: AbstractGraphListener) -> None: + """ + Register a listener to receive graph events. + + Could be left unimplemented for first iteration. + """ + raise NotImplementedError + + @abstractclassmethod + def trial_execute_graph(cls) -> MetricData: + """ + Train graph and returns its metrics, in a separate trial process. + + Each call to `nni.Advisor.create_trial_job(graph_data)` will eventually invoke this method. + + Because this method will be invoked in trial process on training platform, + it has different context from other methods and has no access to global variable or `self`. + However util APIs like `.utils.experiment_config()` should still be available. + """ + raise NotImplementedError diff --git a/utils/third_party/nni_new/retiarii/execution/listener.py b/utils/third_party/nni_new/retiarii/execution/listener.py new file mode 100644 index 0000000000000000000000000000000000000000..cfda111fae7d837a4824a2365d3ab02a70df29f9 --- /dev/null +++ b/utils/third_party/nni_new/retiarii/execution/listener.py @@ -0,0 +1,20 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from ..graph import Model, ModelStatus +from .interface import MetricData, AbstractGraphListener + + +class DefaultListener(AbstractGraphListener): + + def on_metric(self, model: Model, metric: MetricData) -> None: + model.metric = metric + + def on_intermediate_metric(self, model: Model, metric: MetricData) -> None: + model.intermediate_metrics.append(metric) + + def on_training_end(self, model: Model, success: bool) -> None: + if success: + model.status = ModelStatus.Trained + else: + model.status = ModelStatus.Failed diff --git a/utils/third_party/nni_new/retiarii/execution/logical_optimizer/__init__.py b/utils/third_party/nni_new/retiarii/execution/logical_optimizer/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/utils/third_party/nni_new/retiarii/execution/logical_optimizer/interface.py b/utils/third_party/nni_new/retiarii/execution/logical_optimizer/interface.py new file mode 100644 index 0000000000000000000000000000000000000000..03e1e84772417ccda1dc39a97a54e4123b183570 --- /dev/null +++ b/utils/third_party/nni_new/retiarii/execution/logical_optimizer/interface.py @@ -0,0 +1,14 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from abc import ABC + +from .logical_plan import LogicalPlan + + +class AbstractOptimizer(ABC): + def __init__(self) -> None: + pass + + def convert(self, logical_plan: LogicalPlan) -> None: + raise NotImplementedError diff --git a/utils/third_party/nni_new/retiarii/execution/logical_optimizer/logical_plan.py b/utils/third_party/nni_new/retiarii/execution/logical_optimizer/logical_plan.py new file mode 100644 index 0000000000000000000000000000000000000000..f68ffc86a66312778c276243668b77b046e16fd3 --- /dev/null +++ b/utils/third_party/nni_new/retiarii/execution/logical_optimizer/logical_plan.py @@ -0,0 +1,290 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import copy +from typing import Dict, Tuple, List, Any + +from nni.retiarii.utils import uid +from ...graph import Cell, Edge, Graph, Model, Node +from ...operation import Operation, _IOPseudoOperation + + +class PhysicalDevice: + def __init__(self, server: str, device: str): + self.server = server + self.device = device + + def __eq__(self, o) -> bool: + return self.server == o.server and self.device == o.device + + def __hash__(self) -> int: + return hash(self.server + '_' + self.device) + + +class AbstractLogicalNode(Node): + def __init__(self, graph, node_id, name, operation, _internal=False): + super().__init__(graph, node_id, name, operation, _internal=_internal) + + def assemble(self, multi_model_placement: Dict[Model, PhysicalDevice]) -> Tuple[Node, PhysicalDevice]: + raise NotImplementedError + + def _fork_to(self, graph: Graph): + raise NotImplementedError + + +class LogicalGraph(Graph): + def __init__(self, model: Model, graph_id: int, name: str = None, _internal: bool = False): + super().__init__(model, graph_id, name='logical_' + name, _internal=_internal) + + def _dump(self) -> Any: + nodes_dump = {} + for node in self.hidden_nodes: + if isinstance(node, OriginNode): + nodes_dump[f"{node.original_graph.model.model_id}_{node.name}"] = node._dump( + ) + else: + nodes_dump[f"{node.graph.model.model_id}_{node.name}"] = node._dump() + + edges_dump = [] + for edge in self.edges: + if isinstance(edge.head, OriginNode): + head_info = f'{edge.head.original_graph.model.model_id}_{edge.head.name}' + else: + head_info = edge.head.name + if isinstance(edge.tail, OriginNode): + tail_info = f'{edge.tail.original_graph.model.model_id}_{edge.tail.name}' + else: + tail_info = edge.tail.name + edges_dump.append((head_info, tail_info)) + return { + 'inputs': self.input_node.operation.io_names, + 'outputs': self.output_node.operation.io_names, + 'nodes': nodes_dump, + 'edges': edges_dump + } + + def _fork_to(self, model: Model) -> Graph: + new_graph = Graph(model, self.id, self.name, + _internal=True)._register() + + for node in self.hidden_nodes: + if isinstance(node, AbstractLogicalNode): + node._fork_to(new_graph) + else: + Node(new_graph, node.id, node.name, + node.operation, _internal=True)._register() + + id_to_new_node = {node.__repr__(): node for node in new_graph.nodes} + + for edge in self.edges: + new_head = id_to_new_node[edge.head.__repr__()] + new_tail = id_to_new_node[edge.tail.__repr__()] + Edge((new_head, edge.head_slot), + (new_tail, edge.tail_slot), _internal=True)._register() + + return new_graph + + +class OriginNode(AbstractLogicalNode): + def __init__(self, logical_graph: LogicalGraph, + original_graph: Graph, original_node: Node, + name: str, operation, _internal=False): + super().__init__(logical_graph, original_node.id, name, operation) + self.original_graph = original_graph + self.original_node = original_node + + def assemble(self, multi_model_placement: Dict[Model, PhysicalDevice]) -> Tuple[Node, PhysicalDevice]: + model_id = self.original_node.graph.model.model_id + new_node = Node(self.original_node.graph, self.original_node.id, + f"M_{model_id}_" + + self.original_node.name, + self.original_node.operation) + return new_node, multi_model_placement[self.original_node.graph.model] + + def __repr__(self): + return f'OriginNode(id={self.id}, name={self.name}, \ + operation={self.operation}, origin_model_id={self.original_graph.model.model_id})' + + def _fork_to(self, graph: Graph): + OriginNode(graph, self.original_graph, self.original_node, + self.name, self.operation)._register() + + +class LogicalPlan: + def __init__(self, plan_id=0) -> None: + self.lp_model = Model(_internal=True) + self.id = plan_id + self.logical_graph = LogicalGraph( + self.lp_model, self.id, name=f'{self.id}', _internal=True)._register() + self.lp_model._root_graph_name = self.logical_graph.name + self.models = [] + + def add_model(self, model: Model): + self.models.append(model) + # Only optimize the root graph. + self._merge_graph(model.root_graph) + + def _merge_graph(self, from_graph): + to_graph = self.logical_graph + id_to_new_node = {} # old node ID -> new node object + + for old_node in from_graph.nodes: + new_node = OriginNode(to_graph, old_node.graph, + old_node, old_node.name, + old_node.operation, _internal=True)._register() + id_to_new_node[old_node.id] = new_node + + for edge in from_graph.edges: + new_head = id_to_new_node[edge.head.id] + new_tail = id_to_new_node[edge.tail.id] + Edge((new_head, edge.head_slot), (new_tail, + edge.tail_slot), _internal=True)._register() + + def assemble(self, multi_model_placement: Dict[Model, PhysicalDevice]) \ + -> Tuple[Model, Dict[Node, PhysicalDevice], List[Model]]: + phy_model = Model(_internal=True) # self.lp_model.fork() + phy_graph = self.lp_model.root_graph._fork_to(phy_model) + + # Add a flag to mark multi-model in graph json. + # Multi-model has a list of training configs in kwargs['model_kwargs'] + if len(multi_model_placement) > 1: + phy_model.evaluator.kwargs['is_multi_model'] = True + phy_model.evaluator.kwargs['model_cls'] = phy_graph.name + phy_model.evaluator.kwargs['model_kwargs'] = [] + # FIXME: allow user to specify + phy_model.evaluator.module = 'nni.retiarii.trainer.pytorch.PyTorchMultiModelTrainer' + + # merge sub-graphs + for model in multi_model_placement: + for graph_name in model.graphs: + if graph_name != model._root_graph_name: + model.graphs[graph_name]._fork_to( + phy_model, name_prefix=f'M_{model.model_id}_') + + # When replace logical nodes, merge the training configs when + # input/output nodes are replaced. + evaluator_slot = {} # Model ID -> Slot ID + input_slot_mapping = {} + output_slot_mapping = {} + # Replace all logical nodes to executable physical nodes + hidden_nodes = phy_graph.hidden_nodes.copy() + node_placements = {} + for node in hidden_nodes: + if isinstance(node, OriginNode): + model_id = node.original_graph.model.model_id + if node.original_graph.model not in multi_model_placement: + for edge in node.incoming_edges: + edge.remove() + for edge in node.outgoing_edges: + edge.remove() + node.remove() + continue + + if isinstance(node, AbstractLogicalNode): + new_node, placement = node.assemble(multi_model_placement) + if isinstance(new_node.operation, _IOPseudoOperation): + model_id = new_node.graph.model.model_id + if model_id not in evaluator_slot: + phy_model.evaluator.kwargs['model_kwargs'].append(new_node.graph.model.evaluator.kwargs.copy()) + evaluator_slot[model_id] = len(phy_model.evaluator.kwargs['model_kwargs']) - 1 + slot = evaluator_slot[model_id] + phy_model.evaluator.kwargs['model_kwargs'][slot]['model_id'] = model_id + phy_model.evaluator.kwargs['model_kwargs'][slot]['use_input'] = False + phy_model.evaluator.kwargs['model_kwargs'][slot]['use_output'] = False + else: + slot = evaluator_slot[model_id] + # If a model's inputs/outputs are not used in the multi-model + # the codegen and trainer should not generate and use them + # "use_input" and "use_output" are used to mark whether + # an input/output of a model is used in a multi-model + if new_node.operation.type == '_inputs': + input_slot_mapping[new_node] = slot + phy_model.evaluator.kwargs['model_kwargs'][slot]['use_input'] = True + if new_node.operation.type == '_outputs': + output_slot_mapping[new_node] = slot + phy_model.evaluator.kwargs['model_kwargs'][slot]['use_output'] = True + + self.node_replace(node, new_node) + + if isinstance(new_node.operation, Cell): + old_cell_name = new_node.operation.cell_name + new_node.operation = copy.deepcopy(new_node.operation) + new_node.operation.cell_name = f'M_{model_id}_{old_cell_name}' + node_placements[new_node] = placement + + node.remove() + + # If two nodes are placed on different devices, use ToDevice op to copy the node + existing_edges = phy_graph.edges.copy() + # Avoid a node is copied multiple times on the same device + copied_op: Dict[Tuple(Node, PhysicalDevice), Node] = {} + for edge in existing_edges: + head_placement = node_placements[edge.head] + tail_placement = node_placements[edge.tail] + if head_placement != tail_placement: + if head_placement.server != tail_placement.server: + raise ValueError('Cross-server placement is not supported.') + # Same server different devices + if (edge.head, tail_placement) in copied_op: + to_node = copied_op[(edge.head, tail_placement)] + else: + to_operation = Operation.new('ToDevice', {"device": tail_placement.device}) + to_node = Node(phy_graph, uid(), edge.head.name + "_to_" + edge.tail.name, to_operation)._register() + Edge((edge.head, edge.head_slot), (to_node, None), _internal=True)._register() + copied_op[(edge.head, tail_placement)] = to_node + edge.head = to_node + edge.head_slot = None + + # merge all input nodes into one with multiple slots + input_nodes = [] + for node in phy_graph.hidden_nodes: + if isinstance(node.operation, _IOPseudoOperation) and node.operation.type == '_inputs': + input_nodes.append(node) + + for edge in phy_graph.edges: + if edge.head in input_nodes: + edge.head_slot = input_slot_mapping[edge.head] + edge.head = phy_graph.input_node + + # merge all output nodes into one with multiple slots + output_nodes = [] + for node in phy_graph.hidden_nodes: + if isinstance(node.operation, _IOPseudoOperation) and node.operation.type == '_outputs': + output_nodes.append(node) + + for edge in phy_graph.edges: + if edge.tail in output_nodes: + edge.tail_slot = output_slot_mapping[edge.tail] + edge.tail = phy_graph.output_node + + for node in input_nodes: + node.remove() + for node in output_nodes: + node.remove() + + return phy_model, node_placements + + def node_replace(self, old_node: Node, new_node: Node, input_slot_mapping=None, output_slot_mapping=None): + # TODO: currently, only support single input slot and output slot. + if input_slot_mapping is not None or output_slot_mapping is not None: + raise ValueError('Slot mapping is not supported') + + phy_graph = old_node.graph + new_node.graph = phy_graph + + new_node._register() + + for edge in phy_graph.edges: + if edge.head == old_node: + edge.head = new_node + elif edge.tail == old_node: + edge.tail = new_node + + # after the replacement, there might be multiple duplicated edges + # with the same input and output nodes, which should be de-duplicated + self._remove_duplicated_edges() + + def _remove_duplicated_edges(self): + # TODO: it does not have duplicated edges if only supporting dedup input + # Duplicated edges appear when a chain of prefix nodes are deduplicated + pass diff --git a/utils/third_party/nni_new/retiarii/execution/logical_optimizer/opt_dedup_input.py b/utils/third_party/nni_new/retiarii/execution/logical_optimizer/opt_dedup_input.py new file mode 100644 index 0000000000000000000000000000000000000000..4b210d157b3b5fe63038fbf5e05c4f82d74e29e7 --- /dev/null +++ b/utils/third_party/nni_new/retiarii/execution/logical_optimizer/opt_dedup_input.py @@ -0,0 +1,92 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from typing import List, Dict, Tuple + +from nni.retiarii.utils import uid +from ...graph import Graph, Model, Node +from .interface import AbstractOptimizer +from .logical_plan import (AbstractLogicalNode, LogicalGraph, LogicalPlan, + OriginNode, PhysicalDevice) + +_supported_training_modules = ['nni.retiarii.trainer.pytorch.PyTorchImageClassificationTrainer'] + + +class DedupInputNode(AbstractLogicalNode): + def __init__(self, logical_graph: LogicalGraph, node_id: int, + nodes_to_dedup: List[Node], _internal=False): + super().__init__(logical_graph, node_id, + "Dedup_"+nodes_to_dedup[0].name, + nodes_to_dedup[0].operation) + self.origin_nodes: List[OriginNode] = nodes_to_dedup.copy() + + def assemble(self, multi_model_placement: Dict[Model, PhysicalDevice]) -> Tuple[Node, PhysicalDevice]: + for node in self.origin_nodes: + if node.original_graph.model in multi_model_placement: + new_node = Node(node.original_graph, node.id, + f'M_{node.original_graph.model.model_id}_{node.name}', + node.operation) + return new_node, multi_model_placement[node.original_graph.model] + raise ValueError(f'DedupInputNode {self.name} does not contain nodes from multi_model') + + def _fork_to(self, graph: Graph): + DedupInputNode(graph, self.id, self.origin_nodes)._register() + + def __repr__(self) -> str: + return f'DedupNode(id={self.id}, name={self.name}, \ + len(nodes_to_dedup)={len(self.origin_nodes)}' + + +class DedupInputOptimizer(AbstractOptimizer): + def __init__(self) -> None: + pass + + def _check_deduplicate_by_node(self, root_node, node_to_check): + if root_node == node_to_check: + return True + if root_node.operation.type == '_inputs' and \ + node_to_check.operation.type == '_inputs' and \ + isinstance(root_node, OriginNode) and \ + isinstance(node_to_check, OriginNode): + if root_node.original_graph.model.evaluator.module not in _supported_training_modules: + return False + if root_node.original_graph.model.evaluator == node_to_check.original_graph.model.evaluator: + return True + else: + return False + else: + return False + + def convert(self, logical_plan: LogicalPlan) -> None: + nodes_to_skip = set() + while True: # repeat until the logical_graph converges + input_nodes = logical_plan.logical_graph.get_nodes_by_type("_inputs") + # _PseudoOperation(type_name="_inputs")) + root_node = None + for node in input_nodes: + if node in nodes_to_skip: + continue + root_node = node + break + if root_node == None: + break # end of convert + else: + nodes_to_dedup = [] + for node in input_nodes: + if node in nodes_to_skip: + continue + if self._check_deduplicate_by_node(root_node, node): + nodes_to_dedup.append(node) + assert(len(nodes_to_dedup) >= 1) + if len(nodes_to_dedup) == 1: + assert(nodes_to_dedup[0] == root_node) + nodes_to_skip.add(root_node) + else: + dedup_node = DedupInputNode(logical_plan.logical_graph, uid(), nodes_to_dedup)._register() + for edge in logical_plan.logical_graph.edges: + if edge.head in nodes_to_dedup: + edge.head = dedup_node + if edge.tail in nodes_to_dedup: + edge.tail = dedup_node + for node in nodes_to_dedup: + node.remove() diff --git a/utils/third_party/nni_new/retiarii/execution/python.py b/utils/third_party/nni_new/retiarii/execution/python.py new file mode 100644 index 0000000000000000000000000000000000000000..2a5e7a11a908dff343c12a96c2361c90daf3a4ac --- /dev/null +++ b/utils/third_party/nni_new/retiarii/execution/python.py @@ -0,0 +1,57 @@ +from typing import Dict, Any, List + +from ..graph import Evaluator, Model +from ..integration_api import receive_trial_parameters +from ..utils import ContextStack, import_, get_importable_name +from .base import BaseExecutionEngine + + +class PythonGraphData: + def __init__(self, class_name: str, init_parameters: Dict[str, Any], + mutation: Dict[str, Any], evaluator: Evaluator) -> None: + self.class_name = class_name + self.init_parameters = init_parameters + self.mutation = mutation + self.evaluator = evaluator + + def dump(self) -> dict: + return { + 'class_name': self.class_name, + 'init_parameters': self.init_parameters, + 'mutation': self.mutation, + 'evaluator': self.evaluator + } + + @staticmethod + def load(data) -> 'PythonGraphData': + return PythonGraphData(data['class_name'], data['init_parameters'], data['mutation'], data['evaluator']) + + +class PurePythonExecutionEngine(BaseExecutionEngine): + @classmethod + def pack_model_data(cls, model: Model) -> Any: + mutation = get_mutation_dict(model) + graph_data = PythonGraphData(get_importable_name(model.python_class, relocate_module=True), + model.python_init_params, mutation, model.evaluator) + return graph_data + + @classmethod + def trial_execute_graph(cls) -> None: + graph_data = PythonGraphData.load(receive_trial_parameters()) + + class _model(import_(graph_data.class_name)): + def __init__(self): + super().__init__(**graph_data.init_parameters) + + with ContextStack('fixed', graph_data.mutation): + graph_data.evaluator._execute(_model) + + +def _unpack_if_only_one(ele: List[Any]): + if len(ele) == 1: + return ele[0] + return ele + + +def get_mutation_dict(model: Model): + return {mut.mutator.label: _unpack_if_only_one(mut.samples) for mut in model.history} diff --git a/utils/third_party/nni_new/retiarii/experiment/pytorch.py b/utils/third_party/nni_new/retiarii/experiment/pytorch.py new file mode 100644 index 0000000000000000000000000000000000000000..0780439cffcd79d9d881b69f932d78df3d916ba7 --- /dev/null +++ b/utils/third_party/nni_new/retiarii/experiment/pytorch.py @@ -0,0 +1,353 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import atexit +import logging +import time +from dataclasses import dataclass +import os +from pathlib import Path +import socket +from subprocess import Popen +from threading import Thread +import time +from typing import Any, List, Optional, Union + +import colorama +import psutil + +import torch +import torch.nn as nn +import nni.runtime.log +from nni.experiment import Experiment, TrainingServiceConfig +from nni.experiment import management, launcher, rest +from nni.experiment.config import util +from nni.experiment.config.base import ConfigBase, PathLike +from nni.experiment.pipe import Pipe +from nni.tools.nnictl.command_utils import kill_command + +from ..codegen import model_to_pytorch_script +from ..converter import convert_to_graph +from ..execution import list_models, set_execution_engine +from ..execution.python import get_mutation_dict +from ..graph import Model, Evaluator +from ..integration import RetiariiAdvisor +from ..mutator import Mutator +from ..nn.pytorch.mutator import process_inline_mutation, extract_mutation_from_pt_module +from ..strategy import BaseStrategy +from ..oneshot.interface import BaseOneShotTrainer + +_logger = logging.getLogger(__name__) + + +@dataclass(init=False) +class RetiariiExeConfig(ConfigBase): + experiment_name: Optional[str] = None + search_space: Any = '' # TODO: remove + trial_command: str = '_reserved' + trial_code_directory: PathLike = '.' + trial_concurrency: int + trial_gpu_number: int = 0 + max_experiment_duration: Optional[str] = None + max_trial_number: Optional[int] = None + nni_manager_ip: Optional[str] = None + debug: bool = False + log_level: Optional[str] = None + experiment_working_directory: PathLike = '~/nni-experiments' + # remove configuration of tuner/assessor/advisor + training_service: TrainingServiceConfig + execution_engine: str = 'py' + + def __init__(self, training_service_platform: Optional[str] = None, **kwargs): + super().__init__(**kwargs) + if training_service_platform is not None: + assert 'training_service' not in kwargs + self.training_service = util.training_service_config_factory(platform = training_service_platform) + self.__dict__['trial_command'] = 'python3 -m nni.retiarii.trial_entry py' + + def __setattr__(self, key, value): + fixed_attrs = {'search_space': '', + 'trial_command': '_reserved'} + if key in fixed_attrs and fixed_attrs[key] != value: + raise AttributeError(f'{key} is not supposed to be set in Retiarii mode by users!') + # 'trial_code_directory' is handled differently because the path will be converted to absolute path by us + if key == 'trial_code_directory' and not (value == Path('.') or os.path.isabs(value)): + raise AttributeError(f'{key} is not supposed to be set in Retiarii mode by users!') + if key == 'execution_engine': + assert value in ['base', 'py', 'cgo'], f'The specified execution engine "{value}" is not supported.' + self.__dict__['trial_command'] = 'python3 -m nni.retiarii.trial_entry ' + value + self.__dict__[key] = value + + def validate(self, initialized_tuner: bool = False) -> None: + super().validate() + + @property + def _canonical_rules(self): + return _canonical_rules + + @property + def _validation_rules(self): + return _validation_rules + + +_canonical_rules = { + 'trial_code_directory': util.canonical_path, + 'max_experiment_duration': lambda value: f'{util.parse_time(value)}s' if value is not None else None, + 'experiment_working_directory': util.canonical_path +} + +_validation_rules = { + 'trial_code_directory': lambda value: (Path(value).is_dir(), f'"{value}" does not exist or is not directory'), + 'trial_concurrency': lambda value: value > 0, + 'trial_gpu_number': lambda value: value >= 0, + 'max_experiment_duration': lambda value: util.parse_time(value) > 0, + 'max_trial_number': lambda value: value > 0, + 'log_level': lambda value: value in ["trace", "debug", "info", "warning", "error", "fatal"], + 'training_service': lambda value: (type(value) is not TrainingServiceConfig, 'cannot be abstract base class') +} + +def preprocess_model(base_model, trainer, applied_mutators, full_ir=True): + # TODO: this logic might need to be refactored into execution engine + if full_ir: + try: + script_module = torch.jit.script(base_model) + except Exception as e: + _logger.error('Your base model cannot be parsed by torch.jit.script, please fix the following error:') + raise e + base_model_ir = convert_to_graph(script_module, base_model) + # handle inline mutations + mutators = process_inline_mutation(base_model_ir) + else: + base_model_ir, mutators = extract_mutation_from_pt_module(base_model) + base_model_ir.evaluator = trainer + + if mutators is not None and applied_mutators: + raise RuntimeError('Have not supported mixed usage of LayerChoice/InputChoice and mutators, ' + 'do not use mutators when you use LayerChoice/InputChoice') + if mutators is not None: + applied_mutators = mutators + return base_model_ir, applied_mutators + +def debug_mutated_model(base_model, trainer, applied_mutators): + """ + Locally run only one trial without launching an experiment for debug purpose, then exit. + For example, it can be used to quickly check shape mismatch. + + Specifically, it applies mutators (default to choose the first candidate for the choices) + to generate a new model, then run this model locally. + + Parameters + ---------- + base_model : nni.retiarii.nn.pytorch.nn.Module + the base model + trainer : nni.retiarii.evaluator + the training class of the generated models + applied_mutators : list + a list of mutators that will be applied on the base model for generating a new model + """ + base_model_ir, applied_mutators = preprocess_model(base_model, trainer, applied_mutators) + from ..strategy import _LocalDebugStrategy + strategy = _LocalDebugStrategy() + strategy.run(base_model_ir, applied_mutators) + _logger.info('local debug completed!') + + +class RetiariiExperiment(Experiment): + def __init__(self, base_model: nn.Module, trainer: Union[Evaluator, BaseOneShotTrainer], + applied_mutators: List[Mutator] = None, strategy: BaseStrategy = None): + # TODO: The current design of init interface of Retiarii experiment needs to be reviewed. + self.config: RetiariiExeConfig = None + self.port: Optional[int] = None + + self.base_model = base_model + self.trainer = trainer + self.applied_mutators = applied_mutators + self.strategy = strategy + + self._dispatcher = RetiariiAdvisor() + self._dispatcher_thread: Optional[Thread] = None + self._proc: Optional[Popen] = None + self._pipe: Optional[Pipe] = None + + def _start_strategy(self): + base_model_ir, self.applied_mutators = preprocess_model( + self.base_model, self.trainer, self.applied_mutators, full_ir=self.config.execution_engine != 'py') + + _logger.info('Start strategy...') + self.strategy.run(base_model_ir, self.applied_mutators) + _logger.info('Strategy exit') + # TODO: find out a proper way to show no more trial message on WebUI + #self._dispatcher.mark_experiment_as_ending() + + def start(self, port: int = 8080, debug: bool = False) -> None: + """ + Start the experiment in background. + This method will raise exception on failure. + If it returns, the experiment should have been successfully started. + Parameters + ---------- + port + The port of web UI. + debug + Whether to start in debug mode. + """ + atexit.register(self.stop) + + # we will probably need a execution engine factory to make this clean and elegant + if self.config.execution_engine == 'base': + from ..execution.base import BaseExecutionEngine + engine = BaseExecutionEngine() + elif self.config.execution_engine == 'cgo': + from ..execution.cgo_engine import CGOExecutionEngine + engine = CGOExecutionEngine() + elif self.config.execution_engine == 'py': + from ..execution.python import PurePythonExecutionEngine + engine = PurePythonExecutionEngine() + set_execution_engine(engine) + + self.id = management.generate_experiment_id() + + if self.config.experiment_working_directory is not None: + log_dir = Path(self.config.experiment_working_directory, self.id, 'log') + else: + log_dir = Path.home() / f'nni-experiments/{self.id}/log' + nni.runtime.log.start_experiment_log(self.id, log_dir, debug) + + self._proc, self._pipe = launcher.start_experiment_retiarii(self.id, self.config, port, debug) + assert self._proc is not None + assert self._pipe is not None + + self.port = port # port will be None if start up failed + + # dispatcher must be launched after pipe initialized + # the logic to launch dispatcher in background should be refactored into dispatcher api + self._dispatcher = self._create_dispatcher() + self._dispatcher_thread = Thread(target=self._dispatcher.run) + self._dispatcher_thread.start() + + ips = [self.config.nni_manager_ip] + for interfaces in psutil.net_if_addrs().values(): + for interface in interfaces: + if interface.family == socket.AF_INET: + ips.append(interface.address) + ips = [f'http://{ip}:{port}' for ip in ips if ip] + msg = 'Web UI URLs: ' + colorama.Fore.CYAN + ' '.join(ips) + colorama.Style.RESET_ALL + _logger.info(msg) + + exp_status_checker = Thread(target=self._check_exp_status) + exp_status_checker.start() + self._start_strategy() + # TODO: the experiment should be completed, when strategy exits and there is no running job + _logger.info('Waiting for experiment to become DONE (you can ctrl+c if there is no running trial jobs)...') + exp_status_checker.join() + + def _create_dispatcher(self): + return self._dispatcher + + def run(self, config: RetiariiExeConfig = None, port: int = 8080, debug: bool = False) -> str: + """ + Run the experiment. + This function will block until experiment finish or error. + """ + if isinstance(self.trainer, BaseOneShotTrainer): + self.trainer.fit() + else: + assert config is not None, 'You are using classic search mode, config cannot be None!' + self.config = config + self.start(port, debug) + + def _check_exp_status(self) -> bool: + """ + Run the experiment. + This function will block until experiment finish or error. + Return `True` when experiment done; or return `False` when experiment failed. + """ + try: + while True: + time.sleep(10) + # this if is to deal with the situation that + # nnimanager is cleaned up by ctrl+c first + if self._proc.poll() is None: + status = self.get_status() + else: + return False + if status == 'DONE' or status == 'STOPPED': + return True + if status == 'ERROR': + return False + except KeyboardInterrupt: + _logger.warning('KeyboardInterrupt detected') + finally: + self.stop() + + def stop(self) -> None: + """ + Stop background experiment. + """ + _logger.info('Stopping experiment, please wait...') + atexit.unregister(self.stop) + + if self.id is not None: + nni.runtime.log.stop_experiment_log(self.id) + if self._proc is not None: + try: + # this if is to deal with the situation that + # nnimanager is cleaned up by ctrl+c first + if self._proc.poll() is None: + rest.delete(self.port, '/experiment') + except Exception as e: + _logger.exception(e) + _logger.warning('Cannot gracefully stop experiment, killing NNI process...') + kill_command(self._proc.pid) + + if self._pipe is not None: + self._pipe.close() + if self._dispatcher_thread is not None: + self._dispatcher.stopping = True + self._dispatcher_thread.join(timeout=1) + + self.id = None + self.port = None + self._proc = None + self._pipe = None + self._dispatcher = None + self._dispatcher_thread = None + _logger.info('Experiment stopped') + + def export_top_models(self, top_k: int = 1, optimize_mode: str = 'maximize', formatter: str = 'dict') -> Any: + """ + Export several top performing models. + + For one-shot algorithms, only top-1 is supported. For others, ``optimize_mode`` and ``formatter`` are + available for customization. + + top_k : int + How many models are intended to be exported. + optimize_mode : str + ``maximize`` or ``minimize``. Not supported by one-shot algorithms. + ``optimize_mode`` is likely to be removed and defined in strategy in future. + formatter : str + Support ``code`` and ``dict``. Not supported by one-shot algorithms. + If ``code``, the python code of model will be returned. + If ``dict``, the mutation history will be returned. + """ + if formatter == 'code': + assert self.config.execution_engine != 'py', 'You should use `dict` formatter when using Python execution engine.' + if isinstance(self.trainer, BaseOneShotTrainer): + assert top_k == 1, 'Only support top_k is 1 for now.' + return self.trainer.export() + else: + all_models = filter(lambda m: m.metric is not None, list_models()) + assert optimize_mode in ['maximize', 'minimize'] + all_models = sorted(all_models, key=lambda m: m.metric, reverse=optimize_mode == 'maximize') + assert formatter in ['code', 'dict'], 'Export formatter other than "code" and "dict" is not supported yet.' + if formatter == 'code': + return [model_to_pytorch_script(model) for model in all_models[:top_k]] + elif formatter == 'dict': + return [get_mutation_dict(model) for model in all_models[:top_k]] + + def retrain_model(self, model): + """ + this function retrains the exported model, and test it to output test accuracy + """ + raise NotImplementedError diff --git a/utils/third_party/nni_new/retiarii/graph.py b/utils/third_party/nni_new/retiarii/graph.py new file mode 100644 index 0000000000000000000000000000000000000000..3eba65805dc6532353cf7c5e35aaf375c07a010a --- /dev/null +++ b/utils/third_party/nni_new/retiarii/graph.py @@ -0,0 +1,743 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +Model representation. +""" + +import abc +import json +from enum import Enum +from typing import (Any, Dict, Iterable, List, Optional, Tuple, Type, Union, overload) + +from .operation import Cell, Operation, _IOPseudoOperation +from .utils import get_importable_name, import_, uid + +__all__ = ['Model', 'ModelStatus', 'Graph', 'Node', 'Edge', 'Mutation', 'IllegalGraphError', 'MetricData'] + + +MetricData = Any +""" +Type hint for graph metrics (loss, accuracy, etc). +""" + +EdgeEndpoint = Tuple['Node', Optional[int]] +""" +Type hint for edge's endpoint. The int indicates nodes' order. +""" + + +class Evaluator(abc.ABC): + """ + Evaluator of a model. An evaluator should define where the training code is, and the configuration of + training code. The configuration includes basic runtime information trainer needs to know (such as number of GPUs) + or tune-able parameters (such as learning rate), depending on the implementation of training code. + + Each config should define how it is interpreted in ``_execute()``, taking only one argument which is the mutated model class. + For example, functional evaluator might directly import the function and call the function. + """ + + def __repr__(self): + items = ', '.join(['%s=%r' % (k, v) for k, v in self.__dict__.items()]) + return f'{self.__class__.__name__}({items})' + + @abc.abstractstaticmethod + def _load(ir: Any) -> 'Evaluator': + pass + + @staticmethod + def _load_with_type(type_name: str, ir: Any) -> 'Optional[Evaluator]': + if type_name == '_debug_no_trainer': + return DebugEvaluator() + config_cls = import_(type_name) + assert issubclass(config_cls, Evaluator) + return config_cls._load(ir) + + @abc.abstractmethod + def _dump(self) -> Any: + pass + + @abc.abstractmethod + def _execute(self, model_cls: type) -> Any: + pass + + @abc.abstractmethod + def __eq__(self, other) -> bool: + pass + + +class Model: + """ + Represents a neural network model. + + During mutation, one `Model` object is created for each trainable snapshot. + For example, consider a mutator that insert a node at an edge for each iteration. + In one iteration, the mutator invokes 4 primitives: add node, remove edge, add edge to head, add edge to tail. + These 4 primitives operates in one `Model` object. + When they are all done the model will be set to "frozen" (trainable) status and be submitted to execution engine. + And then a new iteration starts, and a new `Model` object is created by forking last model. + + Attributes + ---------- + python_class + Python class that base model is converted from. + python_init_params + Initialization parameters of python class. + status + See `ModelStatus`. + root_graph + The outermost graph which usually takes dataset as input and feeds output to loss function. + graphs + All graphs (subgraphs) in this model. + evaluator + Model evaluator + history + Mutation history. + `self` is directly mutated from `self.history[-1]`; + `self.history[-1] is mutated from `self.history[-2]`, and so on. + `self.history[0]` is the base graph. + metric + Training result of the model, or `None` if it's not yet trained or has failed to train. + intermediate_metrics + Intermediate training metrics. If the model is not trained, it's an empty list. + """ + + def __init__(self, _internal=False): + assert _internal, '`Model()` is private, use `model.fork()` instead' + self.model_id: int = uid('model') + self.python_class: Optional[Type] = None + self.python_init_params: Optional[Dict[str, Any]] = None + + self.status: ModelStatus = ModelStatus.Mutating + + self._root_graph_name: str = '_model' + self.graphs: Dict[str, Graph] = {} + self.evaluator: Optional[Evaluator] = None + + self.history: List[Model] = [] + + self.metric: Optional[MetricData] = None + self.intermediate_metrics: List[MetricData] = [] + + def __repr__(self): + return f'Model(model_id={self.model_id}, status={self.status}, graphs={list(self.graphs.keys())}, ' + \ + f'evaluator={self.evaluator}, metric={self.metric}, intermediate_metrics={self.intermediate_metrics}, ' + \ + f'python_class={self.python_class})' + + @property + def root_graph(self) -> 'Graph': + return self.graphs[self._root_graph_name] + + def fork(self) -> 'Model': + """ + Create a new model which has same topology, names, and IDs to current one. + + Can only be invoked on a frozen model. + The new model will be in `Mutating` state. + + This API is used in mutator base class. + """ + new_model = Model(_internal=True) + new_model._root_graph_name = self._root_graph_name + new_model.python_class = self.python_class + new_model.python_init_params = self.python_init_params + new_model.graphs = {name: graph._fork_to(new_model) for name, graph in self.graphs.items()} + new_model.evaluator = self.evaluator # TODO this needs a clever copy (not deepcopy) if we need mutation + new_model.history = [*self.history] + # Note: the history is not updated. It will be updated when the model is changed, that is in mutator. + return new_model + + @staticmethod + def _load(ir: Any) -> 'Model': + model = Model(_internal=True) + for graph_name, graph_data in ir.items(): + if graph_name != '_evaluator': + Graph._load(model, graph_name, graph_data)._register() + if '_evaluator' in ir: + model.evaluator = Evaluator._load_with_type(ir['_evaluator']['__type__'], ir['_evaluator']) + return model + + def _dump(self) -> Any: + ret = {name: graph._dump() for name, graph in self.graphs.items()} + if self.evaluator is not None: + ret['_evaluator'] = { + '__type__': get_importable_name(self.evaluator.__class__), + **self.evaluator._dump() + } + return ret + + def get_nodes(self) -> Iterable['Node']: + """ + Traverse through all the nodes. + """ + for graph in self.graphs.values(): + for node in graph.nodes: + yield node + + def get_nodes_by_label(self, label: str) -> List['Node']: + """ + Traverse all the nodes to find the matched node(s) with the given label. + There could be multiple nodes with the same label. Name space name can uniquely + identify a graph or node. + + NOTE: the implementation does not support the class abstration + """ + matched_nodes = [] + for graph in self.graphs.values(): + nodes = graph.get_nodes_by_label(label) + matched_nodes.extend(nodes) + return matched_nodes + + def get_nodes_by_type(self, type_name: str) -> List['Node']: + """ + Traverse all the nodes to find the matched node(s) with the given type. + """ + matched_nodes = [] + for graph in self.graphs.values(): + nodes = graph.get_nodes_by_type(type_name) + matched_nodes.extend(nodes) + return matched_nodes + + def get_node_by_name(self, node_name: str) -> 'Node': + """ + Traverse all the nodes to find the matched node with the given name. + """ + matched_nodes = [] + for graph in self.graphs.values(): + nodes = graph.get_nodes_by_name(node_name) + matched_nodes.extend(nodes) + assert len(matched_nodes) <= 1 + if matched_nodes: + return matched_nodes[0] + else: + return None + + +class ModelStatus(Enum): + """ + The status of model. + + A model is created in `Mutating` status. + When the mutation is done and the model get ready to train, its status becomes `Frozen`. + When training started, the model's status becomes `Training`. + If training is successfully ended, model's `metric` attribute get set and its status becomes `Trained`. + If training failed, the status becomes `Failed`. + """ + Mutating = "mutating" + Frozen = "frozen" + Training = "training" + Trained = "trained" + Failed = "failed" + + +_InputPseudoUid = -1 +_OutputPseudoUid = -2 + + +class Graph: + """ + Graph topology. + + This class simply represents the topology, with no semantic meaning. + All other information like metric, non-graph functions, mutation history, etc should go to `Model`. + + Each graph belongs to and only belongs to one `Model`. + + Attributes + ---------- + model + The model containing (and owning) this graph. + id + Unique ID in the model. + If two models have graphs of identical ID, they are semantically the same graph. + Typically this means one graph is mutated from another, or they are both mutated from one ancestor. + name + Mnemonic name of this graph. It should have an one-to-one mapping with ID. + input_names + Optional mnemonic names of input parameters. + output_names + Optional mnemonic names of output values. + input_node + ... + output_node + ... + hidden_nodes + ... + nodes + All input/output/hidden nodes. + edges + ... + """ + + def __init__(self, model: Model, graph_id: int, name: str = None, _internal: bool = False): + assert _internal, '`Graph()` is private' + + self.model: Model = model + self.id: int = graph_id + self.name: str = name or f'_generated_{graph_id}' + + self.input_node: Node = Node(self, _InputPseudoUid, '_inputs', _IOPseudoOperation('_inputs'), _internal=True) + self.output_node: Node = Node(self, _OutputPseudoUid, '_outputs', _IOPseudoOperation('_outputs'), _internal=True) + self.hidden_nodes: List[Node] = [] + + self.edges: List[Edge] = [] + + def __repr__(self): + return f'Graph(id={self.id}, name={self.name}, ' + \ + f'input_names={self.input_node.operation.io_names}, ' + \ + f'output_names={self.output_node.operation.io_names}, ' + \ + f'num_hidden_nodes={len(self.hidden_nodes)}, num_edges={len(self.edges)})' + + @property + def nodes(self) -> List['Node']: + return [self.input_node, self.output_node] + self.hidden_nodes + + def _add_input(self, input_name) -> None: + if self.input_node.operation.io_names is None: + self.input_node.operation.io_names = [input_name] + else: + self.input_node.operation.io_names.append(input_name) + + def _add_output(self, output_name) -> None: + if self.output_node.operation.io_names is None: + self.output_node.operation.io_names = [output_name] + else: + self.output_node.operation.io_names.append(output_name) + + @overload + def add_node(self, name: str, operation: Operation) -> 'Node': ... + @overload + def add_node(self, name: str, type_name: str, parameters: Dict[str, Any] = {}) -> 'Node': ... + + def add_node(self, name, operation_or_type, parameters={}): + if isinstance(operation_or_type, Operation): + op = operation_or_type + else: + op = Operation.new(operation_or_type, parameters, name) + return Node(self, uid(), name, op, _internal=True)._register() + + @overload + def insert_node_on_edge(self, edge: 'Edge', name: str, operation: Operation) -> 'Node': ... + @overload + def insert_node_on_edge(self, edge: 'Edge', name: str, type_name: str, parameters: Dict[str, Any] = {}) -> 'Node': ... + + def insert_node_on_edge(self, edge, name, operation_or_type, parameters={}) -> 'Node': + if isinstance(operation_or_type, Operation): + op = operation_or_type + else: + op = Operation.new(operation_or_type, parameters, name) + new_node = Node(self, uid(), name, op, _internal=True)._register() + # update edges + self.add_edge((edge.head, edge.head_slot), (new_node, None)) + self.add_edge((new_node, None), (edge.tail, edge.tail_slot)) + self.del_edge(edge) + return new_node + + # mutation + def add_edge(self, head: EdgeEndpoint, tail: EdgeEndpoint) -> 'Edge': + assert head[0].graph is self and tail[0].graph is self + return Edge(head, tail, _internal=True)._register() + + def del_edge(self, edge: 'Edge') -> None: + self.edges.remove(edge) + + def get_node_by_name(self, name: str) -> Optional['Node']: + """ + Returns the node which has specified name; or returns `None` if no node has this name. + """ + found = [node for node in self.nodes if node.name == name] + return found[0] if found else None + + def get_nodes_by_type(self, operation_type: str) -> List['Node']: + """ + Returns nodes whose operation is specified typed. + """ + return [node for node in self.hidden_nodes if node.operation.type == operation_type] + + def get_node_by_id(self, node_id: int) -> Optional['Node']: + """ + Returns the node which has specified name; or returns `None` if no node has this name. + """ + found = [node for node in self.nodes if node.id == node_id] + return found[0] if found else None + + def get_nodes_by_label(self, label: str) -> List['Node']: + return [node for node in self.hidden_nodes if node.label == label] + + def get_nodes_by_name(self, name: str) -> List['Node']: + return [node for node in self.hidden_nodes if node.name == name] + + def topo_sort(self) -> List['Node']: + node_to_fanin = {} + curr_nodes = [] + for node in self.nodes: + fanin = len(node.incoming_edges) + node_to_fanin[node] = fanin + if fanin == 0: + curr_nodes.append(node) + + sorted_nodes = [] + while curr_nodes: + curr_node = curr_nodes.pop(0) + sorted_nodes.append(curr_node) + # use successor_slots because a node may connect to another node multiple times + # to different slots + for successor_slot in curr_node.successor_slots: + successor = successor_slot[0] + node_to_fanin[successor] -= 1 + if node_to_fanin[successor] == 0: + curr_nodes.append(successor) + + for key in node_to_fanin: + assert node_to_fanin[key] == 0, '{}, fanin: {}, predecessor: {}, edges: {}, fanin: {}, keys: {}'.format( + key, + node_to_fanin[key], + key.predecessors[0], + self.edges, + node_to_fanin.values(), + node_to_fanin.keys()) + + return sorted_nodes + + def fork(self) -> 'Graph': + """ + Fork the model and returns corresponding graph in new model. + This shortcut might be helpful because many algorithms only cares about "stem" subgraph instead of whole model. + """ + return self.model.fork().graphs[self.name] + + def __eq__(self, other: object) -> bool: + return self is other + + def _fork_to(self, model: Model, name_prefix='') -> 'Graph': + new_graph = Graph(model, self.id, name_prefix+self.name, _internal=True)._register() + # TODO: use node copy instead + new_graph.input_node.operation.io_names = self.input_node.operation.io_names + new_graph.output_node.operation.io_names = self.output_node.operation.io_names + new_graph.input_node.update_label(self.input_node.label) + new_graph.output_node.update_label(self.output_node.label) + + for node in self.hidden_nodes: + new_node = Node(new_graph, node.id, node.name, node.operation, _internal=True) + new_node.update_label(node.label) + new_node._register() + + id_to_new_node = {node.id: node for node in new_graph.nodes} + + for edge in self.edges: + new_head = id_to_new_node[edge.head.id] + new_tail = id_to_new_node[edge.tail.id] + Edge((new_head, edge.head_slot), (new_tail, edge.tail_slot), _internal=True)._register() + + return new_graph + + def _copy(self) -> 'Graph': + # Copy this graph inside the model. + # The new graph will have identical topology, but its nodes' name and ID will be different. + new_graph = Graph(self.model, uid(), _internal=True)._register() + new_graph.input_node.operation.io_names = self.input_node.operation.io_names + new_graph.output_node.operation.io_names = self.output_node.operation.io_names + new_graph.input_node.update_label(self.input_node.label) + new_graph.output_node.update_label(self.output_node.label) + + id_to_new_node = {} # old node ID -> new node object + + for old_node in self.hidden_nodes: + new_node = Node(new_graph, uid(), None, old_node.operation, _internal=True)._register() + new_node.update_label(old_node.label) + id_to_new_node[old_node.id] = new_node + + for edge in self.edges: + new_head = id_to_new_node[edge.head.id] + new_tail = id_to_new_node[edge.tail.id] + Edge((new_head, edge.head_slot), (new_tail, edge.tail_slot), _internal=True)._register() + + return new_graph + + def _register(self) -> 'Graph': + self.model.graphs[self.name] = self + return self + + @staticmethod + def _load(model: Model, name: str, ir: Any) -> 'Graph': + graph = Graph(model, uid(), name, _internal=True) + graph.input_node.operation.io_names = ir.get('inputs') + graph.output_node.operation.io_names = ir.get('outputs') + for node_name, node_data in ir['nodes'].items(): + Node._load(graph, node_name, node_data)._register() + for edge_data in ir['edges']: + Edge._load(graph, edge_data)._register() + return graph + + def _dump(self) -> Any: + return { + 'inputs': self.input_node.operation.io_names, + 'outputs': self.output_node.operation.io_names, + 'nodes': {node.name: node._dump() for node in self.hidden_nodes}, + 'edges': [edge._dump() for edge in self.edges] + } + + +class Node: + """ + An operation or an opaque subgraph inside a graph. + + Each node belongs to and only belongs to one `Graph`. + Nodes should never be created with constructor. Use `Graph.add_node()` instead. + + The node itself is for topology only. + Information of tensor calculation should all go inside `operation` attribute. + + TODO: parameter of subgraph (cell) + It's easy to assign parameters on cell node, but it's hard to "use" them. + We need to design a way to reference stored cell parameters in inner node operations. + e.g. `self.fc = Linear(self.units)` <- how to express `self.units` in IR? + + Attributes + ---------- + graph + The graph containing this node. + id + Unique ID in the model. + If two models have nodes with same ID, they are semantically the same node. + name + Mnemonic name. It should have an one-to-one mapping with ID. + label + Optional. If two nodes have the same label, they are considered same by the mutator. + operation + ... + cell + Read only shortcut to get the referenced subgraph. + If this node is not a subgraph (is a primitive operation), accessing `cell` will raise an error. + predecessors + Predecessor nodes of this node in the graph. This is an optional mutation helper. + successors + Successor nodes of this node in the graph. This is an optional mutation helper. + incoming_edges + Incoming edges of this node in the graph. This is an optional mutation helper. + outgoing_edges + Outgoing edges of this node in the graph. This is an optional mutation helper. + """ + + def __init__(self, graph, node_id, name, operation, _internal=False): + self.graph: Graph = graph + self.id: int = node_id + self.name: str = name or f'_generated_{node_id}' + # TODO: the operation is likely to be considered editable by end-user and it will be hard to debug + # maybe we should copy it here or make Operation class immutable, in next release + self.operation: Operation = operation + self.label: Optional[str] = None + + def __repr__(self): + return f'Node(id={self.id}, name={self.name}, label={self.label}, operation={self.operation})' + + @property + def predecessors(self) -> List['Node']: + return sorted(set(edge.head for edge in self.incoming_edges), key=(lambda node: node.id)) + + @property + def successors(self) -> List['Node']: + return sorted(set(edge.tail for edge in self.outgoing_edges), key=(lambda node: node.id)) + + @property + def successor_slots(self) -> List[Tuple['Node', Union[int, None]]]: + return set((edge.tail, edge.tail_slot) for edge in self.outgoing_edges) + + @property + def incoming_edges(self) -> List['Edge']: + return [edge for edge in self.graph.edges if edge.tail is self] + + @property + def outgoing_edges(self) -> List['Edge']: + return [edge for edge in self.graph.edges if edge.head is self] + + @property + def cell(self) -> Graph: + assert isinstance(self.operation, Cell) + return self.graph.model.graphs[self.operation.parameters['cell']] + + def update_label(self, label: str) -> None: + self.label = label + + @overload + def update_operation(self, operation: Operation) -> None: ... + @overload + def update_operation(self, type_name: str, parameters: Dict[str, Any] = {}) -> None: ... + + def update_operation(self, operation_or_type, parameters={}): + if isinstance(operation_or_type, Operation): + self.operation = operation_or_type + else: + self.operation = Operation.new(operation_or_type, parameters) + + # mutation + def remove(self) -> None: + assert not self.incoming_edges and not self.outgoing_edges + self.graph.hidden_nodes.remove(self) + + # mutation + def specialize_cell(self) -> Graph: + """ + Only available if the operation is a cell. + Duplicate the cell template and let this node reference to newly created copy. + """ + new_cell = self.cell._copy()._register() + self.operation = Cell(new_cell.name) + return new_cell + + def __eq__(self, other: object) -> bool: + return self is other + + def __hash__(self) -> int: + return hash(id(self)) + + def _register(self) -> 'Node': + self.graph.hidden_nodes.append(self) + return self + + @staticmethod + def _load(graph: Graph, name: str, ir: Any) -> 'Node': + if ir['operation']['type'] == '_cell': + op = Cell(ir['operation']['cell_name'], ir['operation'].get('parameters', {})) + else: + op = Operation.new(ir['operation']['type'], ir['operation'].get('parameters', {})) + node = Node(graph, uid(), name, op) + if 'label' in ir: + node.update_label(ir['label']) + return node + + def _dump(self) -> Any: + ret = {'operation': {'type': self.operation.type, 'parameters': self.operation.parameters}} + if isinstance(self.operation, Cell): + ret['operation']['cell_name'] = self.operation.cell_name + if self.label is not None: + ret['label'] = self.label + return ret + + +class Edge: + """ + A tensor, or "data flow", between two nodes. + + Example forward code snippet: + ``` + a, b, c = split(x) + p = concat(a, c) + q = sum(b, p) + z = relu(q) + ``` + + Edges in above snippet: + + head: (split, 0), tail: (concat, 0) # a in concat + + head: (split, 2), tail: (concat, 1) # c in concat + + head: (split, 1), tail: (sum, -1 or 0) # b in sum + + head: (concat, null), tail: (sum, -1 or 1) # p in sum + + head: (sum, null), tail: (relu, null) # q in relu + + Attributes + ---------- + graph + ... + head + Head node. + tail + Tail node. + head_slot + Index of outputs in head node. + If the node has only one output, this should be `null`. + tail_slot + Index of inputs in tail node. + If the node has only one input, this should be `null`. + If the node does not care about order, this can be `-1`. + """ + + def __init__(self, head: EdgeEndpoint, tail: EdgeEndpoint, _internal: bool = False): + assert _internal, '`Edge()` is private' + self.graph: Graph = head[0].graph + self.head: Node = head[0] + self.tail: Node = tail[0] + self.head_slot: Optional[int] = head[1] + self.tail_slot: Optional[int] = tail[1] + + def __repr__(self): + return f'Edge(head=({self.head}, {self.head_slot}), tail=({self.tail}, {self.tail_slot}))' + + # mutation + def remove(self) -> None: + self.graph.edges.remove(self) + + def _register(self) -> 'Edge': + self.graph.edges.append(self) + return self + + @staticmethod + def _load(graph: Graph, ir: Any) -> 'Edge': + head = graph.get_node_by_name(ir['head'][0]) + tail = graph.get_node_by_name(ir['tail'][0]) + assert head is not None and tail is not None + return Edge((head, ir['head'][1]), (tail, ir['tail'][1]), _internal=True) + + def _dump(self) -> Any: + return { + 'head': [self.head.name, self.head_slot], + 'tail': [self.tail.name, self.tail_slot] + } + + +class Mutation: + """ + An execution of mutation, which consists of four parts: a mutator, a list of decisions (choices), + the model that it comes from, and the model that it becomes. + + In general cases, the mutation logs are not reliable and should not be replayed as the mutators can + be arbitrarily complex. However, for inline mutations, the labels correspond to mutator labels here, + this can be useful for metadata visualization and python execution mode. + + Attributes + ---------- + mutator + Mutator. + samples + Decisions/choices. + from_ + Model that is comes from. + to + Model that it becomes. + """ + + def __init__(self, mutator: 'Mutator', samples: List[Any], from_: Model, to: Model): # noqa: F821 + self.mutator: 'Mutator' = mutator # noqa: F821 + self.samples: List[Any] = samples + self.from_: Model = from_ + self.to: Model = to + + def __repr__(self): + return f'Edge(mutator={self.mutator}, samples={self.samples}, from={self.from_}, to={self.to})' + + +class IllegalGraphError(ValueError): + def __init__(self, graph, *args): + self._debug_dump_graph(graph) + super().__init__(*args) + + @staticmethod + def _debug_dump_graph(graph): + if isinstance(graph, Graph): + graph = graph._dump() + with open('generated/debug.json', 'w') as dump_file: + json.dump(graph, dump_file, indent=4) + + +class DebugEvaluator(Evaluator): + @staticmethod + def _load(ir: Any) -> 'DebugEvaluator': + return DebugEvaluator() + + def _dump(self) -> Any: + return {'__type__': '_debug_no_trainer'} + + def _execute(self, model_cls: type) -> Any: + pass + + def __eq__(self, other) -> bool: + return True diff --git a/utils/third_party/nni_new/retiarii/integration.py b/utils/third_party/nni_new/retiarii/integration.py new file mode 100644 index 0000000000000000000000000000000000000000..189db5ff5c3df8f178842ee7234c1adaaf964750 --- /dev/null +++ b/utils/third_party/nni_new/retiarii/integration.py @@ -0,0 +1,134 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +from typing import Any, Callable + +from nni.runtime.msg_dispatcher_base import MsgDispatcherBase +from nni.runtime.protocol import CommandType, send +from nni.utils import MetricType + +from .graph import MetricData +from .integration_api import register_advisor +from .serializer import json_dumps, json_loads + +_logger = logging.getLogger(__name__) + + +class RetiariiAdvisor(MsgDispatcherBase): + """ + The class is to connect Retiarii components to NNI backend. + + It will function as the main thread when running a Retiarii experiment through NNI. + Strategy will be launched as its thread, who will call APIs in execution engine. Execution + engine will then find the advisor singleton and send payloads to advisor. + + When metrics are sent back, advisor will first receive the payloads, who will call the callback + function (that is a member function in graph listener). + + The conversion advisor provides are minimum. It is only a send/receive module, and execution engine + needs to handle all the rest. + + FIXME + How does advisor exit when strategy exists? + + Attributes + ---------- + send_trial_callback + + request_trial_jobs_callback + + trial_end_callback + + intermediate_metric_callback + + final_metric_callback + """ + + def __init__(self): + super(RetiariiAdvisor, self).__init__() + register_advisor(self) # register the current advisor as the "global only" advisor + self.search_space = None + + self.send_trial_callback: Callable[[dict], None] = None + self.request_trial_jobs_callback: Callable[[int], None] = None + self.trial_end_callback: Callable[[int, bool], None] = None + self.intermediate_metric_callback: Callable[[int, MetricData], None] = None + self.final_metric_callback: Callable[[int, MetricData], None] = None + + self.parameters_count = 0 + + def handle_initialize(self, data): + """callback for initializing the advisor + Parameters + ---------- + data: dict + search space + """ + self.handle_update_search_space(data) + send(CommandType.Initialized, '') + + def send_trial(self, parameters): + """ + Send parameters to NNI. + + Parameters + ---------- + parameters : Any + Any payload. + + Returns + ------- + int + Parameter ID that is assigned to this parameter, + which will be used for identification in future. + """ + self.parameters_count += 1 + new_trial = { + 'parameter_id': self.parameters_count, + 'parameters': parameters, + 'parameter_source': 'algorithm' + } + _logger.debug('New trial sent: %s', new_trial) + send(CommandType.NewTrialJob, json_dumps(new_trial)) + if self.send_trial_callback is not None: + self.send_trial_callback(parameters) # pylint: disable=not-callable + return self.parameters_count + + def mark_experiment_as_ending(self): + send(CommandType.NoMoreTrialJobs, '') + + def handle_request_trial_jobs(self, num_trials): + _logger.debug('Request trial jobs: %s', num_trials) + if self.request_trial_jobs_callback is not None: + self.request_trial_jobs_callback(num_trials) # pylint: disable=not-callable + + def handle_update_search_space(self, data): + _logger.debug('Received search space: %s', data) + self.search_space = data + + def handle_trial_end(self, data): + _logger.debug('Trial end: %s', data) + self.trial_end_callback(json_loads(data['hyper_params'])['parameter_id'], # pylint: disable=not-callable + data['event'] == 'SUCCEEDED') + + def handle_report_metric_data(self, data): + _logger.debug('Metric reported: %s', data) + if data['type'] == MetricType.REQUEST_PARAMETER: + raise ValueError('Request parameter not supported') + elif data['type'] == MetricType.PERIODICAL: + self.intermediate_metric_callback(data['parameter_id'], # pylint: disable=not-callable + self._process_value(data['value'])) + elif data['type'] == MetricType.FINAL: + self.final_metric_callback(data['parameter_id'], # pylint: disable=not-callable + self._process_value(data['value'])) + + @staticmethod + def _process_value(value) -> Any: # hopefully a float + value = json_loads(value) + if isinstance(value, dict): + if 'default' in value: + return value['default'] + else: + return value + return value diff --git a/utils/third_party/nni_new/retiarii/integration_api.py b/utils/third_party/nni_new/retiarii/integration_api.py new file mode 100644 index 0000000000000000000000000000000000000000..9d96da897ecb9025eaae1bada2684ec01b9334cc --- /dev/null +++ b/utils/third_party/nni_new/retiarii/integration_api.py @@ -0,0 +1,49 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import json +from typing import NewType, Any + +import nni + +from .serializer import json_loads + +# NOTE: this is only for passing flake8, we cannot import RetiariiAdvisor +# because it would induce cycled import +RetiariiAdvisor = NewType('RetiariiAdvisor', Any) + +_advisor: 'RetiariiAdvisor' = None + + +def get_advisor() -> 'RetiariiAdvisor': + global _advisor + assert _advisor is not None + return _advisor + + +def register_advisor(advisor: 'RetiariiAdvisor'): + global _advisor + assert _advisor is None + _advisor = advisor + + +def send_trial(parameters: dict) -> int: + """ + Send a new trial. Executed on tuner end. + Return a ID that is the unique identifier for this trial. + """ + return get_advisor().send_trial(parameters) + + +def receive_trial_parameters() -> dict: + """ + Received a new trial. Executed on trial end. + Reload with our json loads because NNI didn't use Retiarii serializer to load the data. + """ + params = nni.get_next_parameter() + params = json_loads(json.dumps(params)) + return params + + +def get_experiment_id() -> str: + return nni.get_experiment_id() diff --git a/utils/third_party/nni_new/retiarii/mutator.py b/utils/third_party/nni_new/retiarii/mutator.py new file mode 100644 index 0000000000000000000000000000000000000000..e7d570816917e1e48de7a9e07e0cc0ae700a6b2b --- /dev/null +++ b/utils/third_party/nni_new/retiarii/mutator.py @@ -0,0 +1,117 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from typing import (Any, Iterable, List, Optional) + +from .graph import Model, Mutation, ModelStatus + + +__all__ = ['Sampler', 'Mutator'] + + +Choice = Any + + +class Sampler: + """ + Handles `Mutator.choice()` calls. + """ + + def choice(self, candidates: List[Choice], mutator: 'Mutator', model: Model, index: int) -> Choice: + raise NotImplementedError() + + def mutation_start(self, mutator: 'Mutator', model: Model) -> None: + pass + + def mutation_end(self, mutator: 'Mutator', model: Model) -> None: + pass + + +class Mutator: + """ + Mutates graphs in model to generate new model. + `Mutator` class will be used in two places: + + 1. Inherit `Mutator` to implement graph mutation logic. + 2. Use `Mutator` subclass to implement NAS strategy. + + In scenario 1, the subclass should implement `Mutator.mutate()` interface with `Mutator.choice()`. + In scenario 2, strategy should use constructor or `Mutator.bind_sampler()` to initialize subclass, + and then use `Mutator.apply()` to mutate model. + For certain mutator subclasses, strategy or sampler can use `Mutator.dry_run()` to predict choice candidates. + # Method names are open for discussion. + + If mutator has a label, in most cases, it means that this mutator is applied to nodes with this label. + """ + + def __init__(self, sampler: Optional[Sampler] = None, label: Optional[str] = None): + self.sampler: Optional[Sampler] = sampler + self.label: Optional[str] = label + self._cur_model: Optional[Model] = None + self._cur_choice_idx: Optional[int] = None + + def bind_sampler(self, sampler: Sampler) -> 'Mutator': + """ + Set the sampler which will handle `Mutator.choice` calls. + """ + self.sampler = sampler + return self + + def apply(self, model: Model) -> Model: + """ + Apply this mutator on a model. + Returns mutated model. + The model will be copied before mutation and the original model will not be modified. + """ + assert self.sampler is not None + copy = model.fork() + self._cur_model = copy + self._cur_choice_idx = 0 + self._cur_samples = [] + self.sampler.mutation_start(self, copy) + self.mutate(copy) + self.sampler.mutation_end(self, copy) + copy.history.append(Mutation(self, self._cur_samples, model, copy)) + copy.status = ModelStatus.Frozen + self._cur_model = None + self._cur_choice_idx = None + return copy + + def dry_run(self, model: Model) -> List[List[Choice]]: + """ + Dry run mutator on a model to collect choice candidates. + If you invoke this method multiple times on same or different models, + it may or may not return identical results, depending on how the subclass implements `Mutator.mutate()`. + """ + sampler_backup = self.sampler + recorder = _RecorderSampler() + self.sampler = recorder + new_model = self.apply(model) + self.sampler = sampler_backup + return recorder.recorded_candidates, new_model + + def mutate(self, model: Model) -> None: + """ + Abstract method to be implemented by subclass. + Mutate a model in place. + """ + raise NotImplementedError() + + def choice(self, candidates: Iterable[Choice]) -> Choice: + """ + Ask sampler to make a choice. + """ + assert self.sampler is not None and self._cur_model is not None and self._cur_choice_idx is not None + ret = self.sampler.choice(list(candidates), self, self._cur_model, self._cur_choice_idx) + self._cur_samples.append(ret) + self._cur_choice_idx += 1 + return ret + + +class _RecorderSampler(Sampler): + def __init__(self): + self.recorded_candidates: List[List[Choice]] = [] + + def choice(self, candidates: List[Choice], *args) -> Choice: + self.recorded_candidates.append(candidates) + return candidates[0] diff --git a/utils/third_party/nni_new/retiarii/nn/__init__.py b/utils/third_party/nni_new/retiarii/nn/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/utils/third_party/nni_new/retiarii/nn/pytorch/__init__.py b/utils/third_party/nni_new/retiarii/nn/pytorch/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..5c392164b15c72bebafea0e8c59f369a203f99c3 --- /dev/null +++ b/utils/third_party/nni_new/retiarii/nn/pytorch/__init__.py @@ -0,0 +1,3 @@ +from .api import * +from .component import * +from .nn import * diff --git a/utils/third_party/nni_new/retiarii/nn/pytorch/api.py b/utils/third_party/nni_new/retiarii/nn/pytorch/api.py new file mode 100644 index 0000000000000000000000000000000000000000..69d12fb9080981d87c712488723c4378442a547a --- /dev/null +++ b/utils/third_party/nni_new/retiarii/nn/pytorch/api.py @@ -0,0 +1,385 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import copy +import warnings +from collections import OrderedDict +from typing import Any, List, Union, Dict, Optional + +import torch +import torch.nn as nn + +from ...serializer import Translatable, basic_unit +from .utils import generate_new_label, get_fixed_value + + +__all__ = ['LayerChoice', 'InputChoice', 'ValueChoice', 'Placeholder', 'ChosenInputs'] + + +class LayerChoice(nn.Module): + """ + Layer choice selects one of the ``candidates``, then apply it on inputs and return results. + + Layer choice does not allow itself to be nested. + + Parameters + ---------- + candidates : list of nn.Module or OrderedDict + A module list to be selected from. + label : str + Identifier of the layer choice. + + Attributes + ---------- + length : int + Deprecated. Number of ops to choose from. ``len(layer_choice)`` is recommended. + names : list of str + Names of candidates. + choices : list of Module + Deprecated. A list of all candidate modules in the layer choice module. + ``list(layer_choice)`` is recommended, which will serve the same purpose. + + Notes + ----- + ``candidates`` can be a list of modules or a ordered dict of named modules, for example, + + .. code-block:: python + + self.op_choice = LayerChoice(OrderedDict([ + ("conv3x3", nn.Conv2d(3, 16, 128)), + ("conv5x5", nn.Conv2d(5, 16, 128)), + ("conv7x7", nn.Conv2d(7, 16, 128)) + ])) + + Elements in layer choice can be modified or deleted. Use ``del self.op_choice["conv5x5"]`` or + ``self.op_choice[1] = nn.Conv3d(...)``. Adding more choices is not supported yet. + """ + + def __new__(cls, candidates: Union[Dict[str, nn.Module], List[nn.Module]], label: Optional[str] = None, **kwargs): + try: + chosen = get_fixed_value(label) + if isinstance(candidates, list): + return candidates[int(chosen)] + else: + return candidates[chosen] + except AssertionError: + return super().__new__(cls) + + def __init__(self, candidates: Union[Dict[str, nn.Module], List[nn.Module]], label: Optional[str] = None, **kwargs): + super(LayerChoice, self).__init__() + if 'key' in kwargs: + warnings.warn(f'"key" is deprecated. Assuming label.') + label = kwargs['key'] + if 'return_mask' in kwargs: + warnings.warn(f'"return_mask" is deprecated. Ignoring...') + if 'reduction' in kwargs: + warnings.warn(f'"reduction" is deprecated. Ignoring...') + self.candidates = candidates + self._label = generate_new_label(label) + + self.names = [] + if isinstance(candidates, OrderedDict): + for name, module in candidates.items(): + assert name not in ["length", "reduction", "return_mask", "_key", "key", "names"], \ + "Please don't use a reserved name '{}' for your module.".format(name) + self.add_module(name, module) + self.names.append(name) + elif isinstance(candidates, list): + for i, module in enumerate(candidates): + self.add_module(str(i), module) + self.names.append(str(i)) + else: + raise TypeError("Unsupported candidates type: {}".format(type(candidates))) + + @property + def key(self): + return self._key() + + @torch.jit.ignore + def _key(self): + warnings.warn('Using key to access the identifier of LayerChoice is deprecated. Please use label instead.', + category=DeprecationWarning) + return self._label + + @property + def label(self): + return self._label + + def __getitem__(self, idx): + if isinstance(idx, str): + return self._modules[idx] + return list(self)[idx] + + def __setitem__(self, idx, module): + key = idx if isinstance(idx, str) else self.names[idx] + return setattr(self, key, module) + + def __delitem__(self, idx): + if isinstance(idx, slice): + for key in self.names[idx]: + delattr(self, key) + else: + if isinstance(idx, str): + key, idx = idx, self.names.index(idx) + else: + key = self.names[idx] + delattr(self, key) + del self.names[idx] + + def __len__(self): + return len(self.names) + + def __iter__(self): + return map(lambda name: self._modules[name], self.names) + + @property + def choices(self): + return self._choices() + + @torch.jit.ignore + def _choices(self): + warnings.warn("layer_choice.choices is deprecated. Use `list(layer_choice)` instead.", category=DeprecationWarning) + return list(self) + + def forward(self, x): + warnings.warn('You should not run forward of this module directly.') + return x + + def __repr__(self): + return f'LayerChoice({self.candidates}, label={repr(self.label)})' + + +class InputChoice(nn.Module): + """ + Input choice selects ``n_chosen`` inputs from ``choose_from`` (contains ``n_candidates`` keys). + Use ``reduction`` to specify how chosen inputs are reduced into one output. A few options are: + + * ``none``: do nothing and return the list directly. + * ``sum``: summing all the chosen inputs. + * ``mean``: taking the average of all chosen inputs. + * ``concat``: concatenate all chosen inputs at dimension 1. + + We don't support customizing reduction yet. + + Parameters + ---------- + n_candidates : int + Number of inputs to choose from. It is required. + n_chosen : int + Recommended inputs to choose. If None, mutator is instructed to select any. + reduction : str + ``mean``, ``concat``, ``sum`` or ``none``. + label : str + Identifier of the input choice. + """ + + def __new__(cls, n_candidates: int, n_chosen: int = 1, reduction: str = 'sum', label: Optional[str] = None, **kwargs): + try: + return ChosenInputs(get_fixed_value(label), reduction=reduction) + except AssertionError: + return super().__new__(cls) + + def __init__(self, n_candidates: int, n_chosen: int = 1, reduction: str = 'sum', label: Optional[str] = None, **kwargs): + super(InputChoice, self).__init__() + if 'key' in kwargs: + warnings.warn(f'"key" is deprecated. Assuming label.') + label = kwargs['key'] + if 'return_mask' in kwargs: + warnings.warn(f'"return_mask" is deprecated. Ignoring...') + if 'choose_from' in kwargs: + warnings.warn(f'"reduction" is deprecated. Ignoring...') + self.n_candidates = n_candidates + self.n_chosen = n_chosen + self.reduction = reduction + assert self.reduction in ['mean', 'concat', 'sum', 'none'] + self._label = generate_new_label(label) + + @property + def key(self): + return self._key() + + @torch.jit.ignore + def _key(self): + warnings.warn('Using key to access the identifier of InputChoice is deprecated. Please use label instead.', + category=DeprecationWarning) + return self._label + + @property + def label(self): + return self._label + + def forward(self, candidate_inputs: List[torch.Tensor]) -> torch.Tensor: + warnings.warn('You should not run forward of this module directly.') + return candidate_inputs[0] + + def __repr__(self): + return f'InputChoice(n_candidates={self.n_candidates}, n_chosen={self.n_chosen}, ' \ + f'reduction={repr(self.reduction)}, label={repr(self.label)})' + + +class ValueChoice(Translatable, nn.Module): + """ + ValueChoice is to choose one from ``candidates``. + + In most use scenarios, ValueChoice should be passed to the init parameters of a serializable module. For example, + + .. code-block:: python + + class Net(nn.Module): + def __init__(self): + super().__init__() + self.conv = nn.Conv2d(3, nn.ValueChoice([32, 64]), kernel_size=nn.ValueChoice([3, 5, 7])) + + def forward(self, x): + return self.conv(x) + + In case, you want to search a parameter that is used repeatedly, this is also possible by sharing the same value choice instance. + (Sharing the label should have the same effect.) For example, + + .. code-block:: python + + class Net(nn.Module): + def __init__(self): + super().__init__() + hidden_dim = nn.ValueChoice([128, 512]) + self.fc = nn.Sequential( + nn.Linear(64, hidden_dim), + nn.Linear(hidden_dim, 10) + ) + + # the following code has the same effect. + # self.fc = nn.Sequential( + # nn.Linear(64, nn.ValueChoice([128, 512], label='dim')), + # nn.Linear(nn.ValueChoice([128, 512], label='dim'), 10) + # ) + + def forward(self, x): + return self.fc(x) + + Note that ValueChoice should be used directly. Transformations like ``nn.Linear(32, nn.ValueChoice([64, 128]) * 2)`` + are not supported. + + Another common use case is to initialize the values to choose from in init and call the module in forward to get the chosen value. + Usually, this is used to pass a mutable value to a functional API like ``torch.xxx`` or ``nn.functional.xxx```. + For example, + + .. code-block:: python + + class Net(nn.Module): + def __init__(self): + super().__init__() + self.dropout_rate = nn.ValueChoice([0., 1.]) + + def forward(self, x): + return F.dropout(x, self.dropout_rate()) + + Parameters + ---------- + candidates : list + List of values to choose from. + label : str + Identifier of the value choice. + """ + + def __new__(cls, candidates: List[Any], label: Optional[str] = None): + try: + return get_fixed_value(label) + except AssertionError: + return super().__new__(cls) + + def __init__(self, candidates: List[Any], label: Optional[str] = None): + super().__init__() + self.candidates = candidates + self._label = generate_new_label(label) + self._accessor = [] + + @property + def label(self): + return self._label + + def forward(self): + warnings.warn('You should not run forward of this module directly.') + return self.candidates[0] + + def _translate(self): + # Will function as a value when used in serializer. + return self.access(self.candidates[0]) + + def __repr__(self): + return f'ValueChoice({self.candidates}, label={repr(self.label)})' + + def access(self, value): + if not self._accessor: + return value + try: + v = value + for a in self._accessor: + v = v[a] + except KeyError: + raise KeyError(''.join([f'[{a}]' for a in self._accessor]) + f' does not work on {value}') + return v + + def __copy__(self): + return self + + def __deepcopy__(self, memo): + new_item = ValueChoice(self.candidates, self.label) + new_item._accessor = [*self._accessor] + return new_item + + def __getitem__(self, item): + """ + Get a sub-element of value choice. + + The underlying implementation is to clone the current instance, and append item to "accessor", which records all + the history getitem calls. For example, when accessor is ``[a, b, c]``, the value choice will return ``vc[a][b][c]`` + where ``vc`` is the original value choice. + """ + access = copy.deepcopy(self) + access._accessor.append(item) + for candidate in self.candidates: + access.access(candidate) + return access + + +@basic_unit +class Placeholder(nn.Module): + # TODO: docstring + + def __init__(self, label, **related_info): + self.label = label + self.related_info = related_info + super().__init__() + + def forward(self, x): + return x + + +class ChosenInputs(nn.Module): + """ + A module that chooses from a tensor list and outputs a reduced tensor. + The already-chosen version of InputChoice. + """ + + def __init__(self, chosen: Union[List[int], int], reduction: str): + super().__init__() + self.chosen = chosen if isinstance(chosen, list) else [chosen] + self.reduction = reduction + + def forward(self, candidate_inputs): + return self._tensor_reduction(self.reduction, [candidate_inputs[i] for i in self.chosen]) + + def _tensor_reduction(self, reduction_type, tensor_list): + if reduction_type == 'none': + return tensor_list + if not tensor_list: + return None # empty. return None for now + if len(tensor_list) == 1: + return tensor_list[0] + if reduction_type == 'sum': + return sum(tensor_list) + if reduction_type == 'mean': + return sum(tensor_list) / len(tensor_list) + if reduction_type == 'concat': + return torch.cat(tensor_list, dim=1) + raise ValueError(f'Unrecognized reduction policy: "{reduction_type}"') diff --git a/utils/third_party/nni_new/retiarii/nn/pytorch/component.py b/utils/third_party/nni_new/retiarii/nn/pytorch/component.py new file mode 100644 index 0000000000000000000000000000000000000000..20c6e30de531d1431d61b85af804e8fcb8dbbb93 --- /dev/null +++ b/utils/third_party/nni_new/retiarii/nn/pytorch/component.py @@ -0,0 +1,147 @@ +import copy +from typing import Callable, List, Union, Tuple, Optional + +import torch +import torch.nn as nn + +from .api import LayerChoice, InputChoice +from .nn import ModuleList + +from .utils import generate_new_label, get_fixed_value + + +__all__ = ['Repeat', 'Cell'] + + +class Repeat(nn.Module): + """ + Repeat a block by a variable number of times. + + Parameters + ---------- + blocks : function, list of function, module or list of module + The block to be repeated. If not a list, it will be replicated into a list. + If a list, it should be of length ``max_depth``, the modules will be instantiated in order and a prefix will be taken. + If a function, it will be called to instantiate a module. Otherwise the module will be deep-copied. + depth : int or tuple of int + If one number, the block will be repeated by a fixed number of times. If a tuple, it should be (min, max), + meaning that the block will be repeated at least `min` times and at most `max` times. + """ + + def __new__(cls, blocks: Union[Callable[[], nn.Module], List[Callable[[], nn.Module]], nn.Module, List[nn.Module]], + depth: Union[int, Tuple[int, int]], label: Optional[str] = None): + try: + repeat = get_fixed_value(label) + return nn.Sequential(*cls._replicate_and_instantiate(blocks, repeat)) + except AssertionError: + return super().__new__(cls) + + def __init__(self, + blocks: Union[Callable[[], nn.Module], List[Callable[[], nn.Module]], nn.Module, List[nn.Module]], + depth: Union[int, Tuple[int, int]], label: Optional[str] = None): + super().__init__() + self._label = generate_new_label(label) + self.min_depth = depth if isinstance(depth, int) else depth[0] + self.max_depth = depth if isinstance(depth, int) else depth[1] + assert self.max_depth >= self.min_depth > 0 + self.blocks = nn.ModuleList(self._replicate_and_instantiate(blocks, self.max_depth)) + + @property + def label(self): + return self._label + + def forward(self, x): + for block in self.blocks: + x = block(x) + return x + + @staticmethod + def _replicate_and_instantiate(blocks, repeat): + if not isinstance(blocks, list): + if isinstance(blocks, nn.Module): + blocks = [blocks] + [copy.deepcopy(blocks) for _ in range(repeat - 1)] + else: + blocks = [blocks for _ in range(repeat)] + assert len(blocks) > 0 + assert repeat <= len(blocks), f'Not enough blocks to be used. {repeat} expected, only found {len(blocks)}.' + blocks = blocks[:repeat] + if not isinstance(blocks[0], nn.Module): + blocks = [b() for b in blocks] + return blocks + + +class Cell(nn.Module): + """ + Cell structure [zophnas]_ [zophnasnet]_ that is popularly used in NAS literature. + + A cell consists of multiple "nodes". Each node is a sum of multiple operators. Each operator is chosen from + ``op_candidates``, and takes one input from previous nodes and predecessors. Predecessor means the input of cell. + The output of cell is the concatenation of some of the nodes in the cell (currently all the nodes). + + Parameters + ---------- + op_candidates : function or list of module + A list of modules to choose from, or a function that returns a list of modules. + num_nodes : int + Number of nodes in the cell. + num_ops_per_node: int + Number of operators in each node. The output of each node is the sum of all operators in the node. Default: 1. + num_predecessors : int + Number of inputs of the cell. The input to forward should be a list of tensors. Default: 1. + merge_op : str + Currently only ``all`` is supported, which has slight difference with that described in reference. Default: all. + label : str + Identifier of the cell. Cell sharing the same label will semantically share the same choice. + + References + ---------- + .. [zophnas] Barret Zoph, Quoc V. Le, "Neural Architecture Search with Reinforcement Learning". https://arxiv.org/abs/1611.01578 + .. [zophnasnet] Barret Zoph, Vijay Vasudevan, Jonathon Shlens, Quoc V. Le, + "Learning Transferable Architectures for Scalable Image Recognition". https://arxiv.org/abs/1707.07012 + """ + + # TODO: + # Support loose end concat (shape inference on the following cells) + # How to dynamically create convolution with stride as the first node + + def __init__(self, + op_candidates: Union[Callable, List[nn.Module]], + num_nodes: int, + num_ops_per_node: int = 1, + num_predecessors: int = 1, + merge_op: str = 'all', + label: str = None): + super().__init__() + self._label = generate_new_label(label) + self.ops = ModuleList() + self.inputs = ModuleList() + self.num_nodes = num_nodes + self.num_ops_per_node = num_ops_per_node + self.num_predecessors = num_predecessors + for i in range(num_nodes): + self.ops.append(ModuleList()) + self.inputs.append(ModuleList()) + for k in range(num_ops_per_node): + if isinstance(op_candidates, list): + assert len(op_candidates) > 0 and isinstance(op_candidates[0], nn.Module) + ops = copy.deepcopy(op_candidates) + else: + ops = op_candidates() + self.ops[-1].append(LayerChoice(ops, label=f'{self.label}__op_{i}_{k}')) + self.inputs[-1].append(InputChoice(i + num_predecessors, 1, label=f'{self.label}/input_{i}_{k}')) + assert merge_op in ['all'] # TODO: loose_end + self.merge_op = merge_op + + @property + def label(self): + return self._label + + def forward(self, x: List[torch.Tensor]): + states = x + for ops, inps in zip(self.ops, self.inputs): + current_state = [] + for op, inp in zip(ops, inps): + current_state.append(op(inp(states))) + current_state = torch.sum(torch.stack(current_state), 0) + states.append(current_state) + return torch.cat(states[self.num_predecessors:], 1) diff --git a/utils/third_party/nni_new/retiarii/nn/pytorch/mutator.py b/utils/third_party/nni_new/retiarii/nn/pytorch/mutator.py new file mode 100644 index 0000000000000000000000000000000000000000..4307aeac681bbd631103affa2e21c2c5c7d38f10 --- /dev/null +++ b/utils/third_party/nni_new/retiarii/nn/pytorch/mutator.py @@ -0,0 +1,301 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import inspect +from typing import Any, List, Optional, Tuple + +import torch.nn as nn + +from ...mutator import Mutator +from ...graph import Cell, Graph, Model, ModelStatus, Node +from .api import LayerChoice, InputChoice, ValueChoice, Placeholder +from .component import Repeat +from ...utils import uid + + +class LayerChoiceMutator(Mutator): + def __init__(self, nodes: List[Node]): + super().__init__(label=nodes[0].operation.parameters['label']) + self.nodes = nodes + + def mutate(self, model): + candidates = self.nodes[0].operation.parameters['candidates'] + chosen = self.choice(candidates) + for node in self.nodes: + # Each layer choice corresponds to a cell, which is unconnected in the base graph. + # We add the connections here in the mutation logic. + # Thus, the mutated model should not be mutated again. Everything should be based on the original base graph. + target = model.graphs[node.operation.cell_name] + chosen_node = target.get_node_by_name(chosen) + assert chosen_node is not None + target.add_edge((target.input_node, 0), (chosen_node, None)) + target.add_edge((chosen_node, None), (target.output_node, None)) + model.get_node_by_name(node.name).update_operation(Cell(node.operation.cell_name)) + + # remove redundant nodes + for rm_node in list(target.hidden_nodes): # remove from a list on the fly will cause issues + if rm_node.name != chosen_node.name: + rm_node.remove() + + +class InputChoiceMutator(Mutator): + def __init__(self, nodes: List[Node]): + super().__init__(label=nodes[0].operation.parameters['label']) + self.nodes = nodes + + def mutate(self, model): + n_candidates = self.nodes[0].operation.parameters['n_candidates'] + n_chosen = self.nodes[0].operation.parameters['n_chosen'] + candidates = list(range(n_candidates)) + chosen = [self.choice(candidates) for _ in range(n_chosen)] + for node in self.nodes: + target = model.get_node_by_name(node.name) + target.update_operation('__torch__.nni.retiarii.nn.pytorch.ChosenInputs', + {'chosen': chosen, 'reduction': node.operation.parameters['reduction']}) + + +class ValueChoiceMutator(Mutator): + def __init__(self, nodes: List[Node], candidates: List[Any]): + super().__init__(label=nodes[0].operation.parameters['label']) + self.nodes = nodes + self.candidates = candidates + + def mutate(self, model): + chosen = self.choice(self.candidates) + for node in self.nodes: + target = model.get_node_by_name(node.name) + target.update_operation('prim::Constant', {'type': type(chosen).__name__, 'value': chosen}) + + +class ParameterChoiceMutator(Mutator): + def __init__(self, nodes: List[Tuple[Node, str]], candidates: List[Any]): + node, argname = nodes[0] + super().__init__(label=node.operation.parameters[argname].label) + self.nodes = nodes + self.candidates = candidates + + def mutate(self, model): + chosen = self.choice(self.candidates) + for node, argname in self.nodes: + chosen_value = node.operation.parameters[argname].access(chosen) + target = model.get_node_by_name(node.name) + target.update_operation(target.operation.type, {**target.operation.parameters, argname: chosen_value}) + + +class RepeatMutator(Mutator): + def __init__(self, nodes: List[Node]): + # nodes is a subgraph consisting of repeated blocks. + super().__init__(label=nodes[0].operation.parameters['label']) + self.nodes = nodes + + def _retrieve_chain_from_graph(self, graph: Graph) -> List[Node]: + u = graph.input_node + chain = [] + while u != graph.output_node: + if u != graph.input_node: + chain.append(u) + assert len(u.successors) == 1, f'This graph is an illegal chain. {u} has output {u.successor}.' + u = u.successors[0] + return chain + + def mutate(self, model): + min_depth = self.nodes[0].operation.parameters['min_depth'] + max_depth = self.nodes[0].operation.parameters['max_depth'] + if min_depth < max_depth: + chosen_depth = self.choice(list(range(min_depth, max_depth + 1))) + for node in self.nodes: + # the logic here is similar to layer choice. We find cell attached to each node. + target: Graph = model.graphs[node.operation.cell_name] + chain = self._retrieve_chain_from_graph(target) + for edge in chain[chosen_depth - 1].outgoing_edges: + edge.remove() + target.add_edge((chain[chosen_depth - 1], None), (target.output_node, None)) + for rm_node in chain[chosen_depth:]: + for edge in rm_node.outgoing_edges: + edge.remove() + rm_node.remove() + # to delete the unused parameters. + model.get_node_by_name(node.name).update_operation(Cell(node.operation.cell_name)) + + +def process_inline_mutation(model: Model) -> Optional[List[Mutator]]: + applied_mutators = [] + + ic_nodes = _group_by_label(model.get_nodes_by_type('__torch__.nni.retiarii.nn.pytorch.api.InputChoice')) + for node_list in ic_nodes: + assert _is_all_equal(map(lambda node: node.operation.parameters['n_candidates'], node_list)) and \ + _is_all_equal(map(lambda node: node.operation.parameters['n_chosen'], node_list)), \ + 'Input choice with the same label must have the same number of candidates.' + mutator = InputChoiceMutator(node_list) + applied_mutators.append(mutator) + + vc_nodes = _group_by_label(model.get_nodes_by_type('__torch__.nni.retiarii.nn.pytorch.api.ValueChoice')) + for node_list in vc_nodes: + assert _is_all_equal(map(lambda node: node.operation.parameters['candidates'], node_list)), \ + 'Value choice with the same label must have the same candidates.' + mutator = ValueChoiceMutator(node_list, node_list[0].operation.parameters['candidates']) + applied_mutators.append(mutator) + + pc_nodes = [] + for node in model.get_nodes(): + for name, choice in node.operation.parameters.items(): + if isinstance(choice, ValueChoice): + pc_nodes.append((node, name)) + pc_nodes = _group_parameters_by_label(pc_nodes) + for node_list in pc_nodes: + assert _is_all_equal([node.operation.parameters[name].candidates for node, name in node_list]), \ + 'Value choice with the same label must have the same candidates.' + first_node, first_argname = node_list[0] + mutator = ParameterChoiceMutator(node_list, first_node.operation.parameters[first_argname].candidates) + applied_mutators.append(mutator) + + # apply layer choice at last as it will delete some nodes + lc_nodes = _group_by_label(filter(lambda d: d.operation.parameters.get('mutation') == 'layerchoice', + model.get_nodes_by_type('_cell'))) + for node_list in lc_nodes: + assert _is_all_equal(map(lambda node: len(node.operation.parameters['candidates']), node_list)), \ + 'Layer choice with the same label must have the same number of candidates.' + mutator = LayerChoiceMutator(node_list) + applied_mutators.append(mutator) + + repeat_nodes = _group_by_label(filter(lambda d: d.operation.parameters.get('mutation') == 'repeat', + model.get_nodes_by_type('_cell'))) + for node_list in repeat_nodes: + assert _is_all_equal(map(lambda node: node.operation.parameters['max_depth'], node_list)) and \ + _is_all_equal(map(lambda node: node.operation.parameters['min_depth'], node_list)), \ + 'Repeat with the same label must have the same number of candidates.' + mutator = RepeatMutator(node_list) + applied_mutators.append(mutator) + + if applied_mutators: + return applied_mutators + return None + + +# The following are written for pure-python mode + + +class ManyChooseManyMutator(Mutator): + """ + Choose based on labels. Will not affect the model itself. + """ + + def __init__(self, label: Optional[str]): + super().__init__(label=label) + + @staticmethod + def candidates(node): + if 'n_candidates' in node.operation.parameters: + return list(range(node.operation.parameters['n_candidates'])) + else: + return node.operation.parameters['candidates'] + + @staticmethod + def number_of_chosen(node): + if 'n_chosen' in node.operation.parameters: + return node.operation.parameters['n_chosen'] + return 1 + + def mutate(self, model: Model): + # this mutate does not have any effect, but it is recorded in the mutation history + for node in model.get_nodes_by_label(self.label): + for _ in range(self.number_of_chosen(node)): + self.choice(self.candidates(node)) + break + + +def extract_mutation_from_pt_module(pytorch_model: nn.Module) -> Tuple[Model, Optional[List[Mutator]]]: + model = Model(_internal=True) + graph = Graph(model, uid(), '_model', _internal=True)._register() + model.python_class = pytorch_model.__class__ + if len(inspect.signature(model.python_class.__init__).parameters) > 1: + if not hasattr(pytorch_model, '_init_parameters'): + raise ValueError('Please annotate the model with @serialize decorator in python execution mode ' + 'if your model has init parameters.') + model.python_init_params = pytorch_model._init_parameters + else: + model.python_init_params = {} + + for name, module in pytorch_model.named_modules(): + # tricky case: value choice that serves as parameters are stored in _init_parameters + if hasattr(module, '_init_parameters'): + for key, value in module._init_parameters.items(): + if isinstance(value, ValueChoice): + node = graph.add_node(name + '.init.' + key, 'ValueChoice', {'candidates': value.candidates}) + node.label = value.label + + if isinstance(module, (LayerChoice, InputChoice, ValueChoice)): + # TODO: check the label of module and warn if it's auto-generated + pass + if isinstance(module, LayerChoice): + node = graph.add_node(name, 'LayerChoice', {'candidates': module.names}) + node.label = module.label + if isinstance(module, InputChoice): + node = graph.add_node(name, 'InputChoice', + {'n_candidates': module.n_candidates, 'n_chosen': module.n_chosen}) + node.label = module.label + if isinstance(module, ValueChoice): + node = graph.add_node(name, 'ValueChoice', {'candidates': module.candidates}) + node.label = module.label + if isinstance(module, Repeat) and module.min_depth <= module.max_depth: + node = graph.add_node(name, 'Repeat', { + 'candidates': list(range(module.min_depth, module.max_depth + 1)) + }) + node.label = module.label + if isinstance(module, Placeholder): + raise NotImplementedError('Placeholder is not supported in python execution mode.') + + model.status = ModelStatus.Frozen + if not graph.hidden_nodes: + return model, None + + mutators = [] + for nodes in _group_by_label_and_type(graph.hidden_nodes): + assert _is_all_equal(map(lambda n: n.operation.type, nodes)), \ + f'Node with label "{nodes[0].label}" does not all have the same type.' + assert _is_all_equal(map(lambda n: n.operation.parameters, nodes)), \ + f'Node with label "{nodes[0].label}" does not agree on parameters.' + mutators.append(ManyChooseManyMutator(nodes[0].label)) + return model, mutators + + +# utility functions + + +def _is_all_equal(lst): + last = None + for x in lst: + if last is not None and last != x: + return False + last = x + return True + + +def _group_by_label_and_type(nodes: List[Node]) -> List[List[Node]]: + result = {} + for node in nodes: + key = (node.label, node.operation.type) + if key not in result: + result[key] = [] + result[key].append(node) + return list(result.values()) + + +def _group_by_label(nodes: List[Node]) -> List[List[Node]]: + result = {} + for node in nodes: + label = node.operation.parameters['label'] + if label not in result: + result[label] = [] + result[label].append(node) + return list(result.values()) + + +def _group_parameters_by_label(nodes: List[Tuple[Node, str]]) -> List[List[Tuple[Node, str]]]: + result = {} + for node, argname in nodes: + label = node.operation.parameters[argname].label + if label not in result: + result[label] = [] + result[label].append((node, argname)) + return list(result.values()) diff --git a/utils/third_party/nni_new/retiarii/nn/pytorch/nn.py b/utils/third_party/nni_new/retiarii/nn/pytorch/nn.py new file mode 100644 index 0000000000000000000000000000000000000000..1cbf0f9e6ee6a9be501bd8ef30369dfec910af0a --- /dev/null +++ b/utils/third_party/nni_new/retiarii/nn/pytorch/nn.py @@ -0,0 +1,159 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import torch +import torch.nn as nn + +from ...serializer import basic_unit +from ...serializer import transparent_serialize +from ...utils import version_larger_equal + +# NOTE: support pytorch version >= 1.5.0 + +__all__ = [ + 'Module', 'Sequential', 'ModuleList', # TODO: 'ModuleDict', 'ParameterList', 'ParameterDict', + 'Identity', 'Linear', 'Conv1d', 'Conv2d', 'Conv3d', 'ConvTranspose1d', + 'ConvTranspose2d', 'ConvTranspose3d', 'Threshold', 'ReLU', 'Hardtanh', 'ReLU6', + 'Sigmoid', 'Tanh', 'Softmax', 'Softmax2d', 'LogSoftmax', 'ELU', 'SELU', 'CELU', 'GLU', 'GELU', 'Hardshrink', + 'LeakyReLU', 'LogSigmoid', 'Softplus', 'Softshrink', 'MultiheadAttention', 'PReLU', 'Softsign', 'Softmin', + 'Tanhshrink', 'RReLU', 'AvgPool1d', 'AvgPool2d', 'AvgPool3d', 'MaxPool1d', 'MaxPool2d', + 'MaxPool3d', 'MaxUnpool1d', 'MaxUnpool2d', 'MaxUnpool3d', 'FractionalMaxPool2d', "FractionalMaxPool3d", + 'LPPool1d', 'LPPool2d', 'LocalResponseNorm', 'BatchNorm1d', 'BatchNorm2d', 'BatchNorm3d', 'InstanceNorm1d', + 'InstanceNorm2d', 'InstanceNorm3d', 'LayerNorm', 'GroupNorm', 'SyncBatchNorm', + 'Dropout', 'Dropout2d', 'Dropout3d', 'AlphaDropout', 'FeatureAlphaDropout', + 'ReflectionPad1d', 'ReflectionPad2d', 'ReplicationPad2d', 'ReplicationPad1d', 'ReplicationPad3d', + 'CrossMapLRN2d', 'Embedding', 'EmbeddingBag', 'RNNBase', 'RNN', 'LSTM', 'GRU', 'RNNCellBase', 'RNNCell', + 'LSTMCell', 'GRUCell', 'PixelShuffle', 'Upsample', 'UpsamplingNearest2d', 'UpsamplingBilinear2d', + 'PairwiseDistance', 'AdaptiveMaxPool1d', 'AdaptiveMaxPool2d', 'AdaptiveMaxPool3d', 'AdaptiveAvgPool1d', + 'AdaptiveAvgPool2d', 'AdaptiveAvgPool3d', 'TripletMarginLoss', 'ZeroPad2d', 'ConstantPad1d', 'ConstantPad2d', + 'ConstantPad3d', 'Bilinear', 'CosineSimilarity', 'Unfold', 'Fold', + 'AdaptiveLogSoftmaxWithLoss', 'TransformerEncoder', 'TransformerDecoder', + 'TransformerEncoderLayer', 'TransformerDecoderLayer', 'Transformer', + 'Flatten', 'Hardsigmoid' +] + +if version_larger_equal(torch.__version__, '1.6.0'): + __all__.append('Hardswish') + +if version_larger_equal(torch.__version__, '1.7.0'): + __all__.extend(['Unflatten', 'SiLU', 'TripletMarginWithDistanceLoss']) + + +Module = nn.Module + +Sequential = transparent_serialize(nn.Sequential) +ModuleList = transparent_serialize(nn.ModuleList) + +Identity = basic_unit(nn.Identity) +Linear = basic_unit(nn.Linear) +Conv1d = basic_unit(nn.Conv1d) +Conv2d = basic_unit(nn.Conv2d) +Conv3d = basic_unit(nn.Conv3d) +ConvTranspose1d = basic_unit(nn.ConvTranspose1d) +ConvTranspose2d = basic_unit(nn.ConvTranspose2d) +ConvTranspose3d = basic_unit(nn.ConvTranspose3d) +Threshold = basic_unit(nn.Threshold) +ReLU = basic_unit(nn.ReLU) +Hardtanh = basic_unit(nn.Hardtanh) +ReLU6 = basic_unit(nn.ReLU6) +Sigmoid = basic_unit(nn.Sigmoid) +Tanh = basic_unit(nn.Tanh) +Softmax = basic_unit(nn.Softmax) +Softmax2d = basic_unit(nn.Softmax2d) +LogSoftmax = basic_unit(nn.LogSoftmax) +ELU = basic_unit(nn.ELU) +SELU = basic_unit(nn.SELU) +CELU = basic_unit(nn.CELU) +GLU = basic_unit(nn.GLU) +GELU = basic_unit(nn.GELU) +Hardshrink = basic_unit(nn.Hardshrink) +LeakyReLU = basic_unit(nn.LeakyReLU) +LogSigmoid = basic_unit(nn.LogSigmoid) +Softplus = basic_unit(nn.Softplus) +Softshrink = basic_unit(nn.Softshrink) +MultiheadAttention = basic_unit(nn.MultiheadAttention) +PReLU = basic_unit(nn.PReLU) +Softsign = basic_unit(nn.Softsign) +Softmin = basic_unit(nn.Softmin) +Tanhshrink = basic_unit(nn.Tanhshrink) +RReLU = basic_unit(nn.RReLU) +AvgPool1d = basic_unit(nn.AvgPool1d) +AvgPool2d = basic_unit(nn.AvgPool2d) +AvgPool3d = basic_unit(nn.AvgPool3d) +MaxPool1d = basic_unit(nn.MaxPool1d) +MaxPool2d = basic_unit(nn.MaxPool2d) +MaxPool3d = basic_unit(nn.MaxPool3d) +MaxUnpool1d = basic_unit(nn.MaxUnpool1d) +MaxUnpool2d = basic_unit(nn.MaxUnpool2d) +MaxUnpool3d = basic_unit(nn.MaxUnpool3d) +FractionalMaxPool2d = basic_unit(nn.FractionalMaxPool2d) +FractionalMaxPool3d = basic_unit(nn.FractionalMaxPool3d) +LPPool1d = basic_unit(nn.LPPool1d) +LPPool2d = basic_unit(nn.LPPool2d) +LocalResponseNorm = basic_unit(nn.LocalResponseNorm) +BatchNorm1d = basic_unit(nn.BatchNorm1d) +BatchNorm2d = basic_unit(nn.BatchNorm2d) +BatchNorm3d = basic_unit(nn.BatchNorm3d) +InstanceNorm1d = basic_unit(nn.InstanceNorm1d) +InstanceNorm2d = basic_unit(nn.InstanceNorm2d) +InstanceNorm3d = basic_unit(nn.InstanceNorm3d) +LayerNorm = basic_unit(nn.LayerNorm) +GroupNorm = basic_unit(nn.GroupNorm) +SyncBatchNorm = basic_unit(nn.SyncBatchNorm) +Dropout = basic_unit(nn.Dropout) +Dropout2d = basic_unit(nn.Dropout2d) +Dropout3d = basic_unit(nn.Dropout3d) +AlphaDropout = basic_unit(nn.AlphaDropout) +FeatureAlphaDropout = basic_unit(nn.FeatureAlphaDropout) +ReflectionPad1d = basic_unit(nn.ReflectionPad1d) +ReflectionPad2d = basic_unit(nn.ReflectionPad2d) +ReplicationPad2d = basic_unit(nn.ReplicationPad2d) +ReplicationPad1d = basic_unit(nn.ReplicationPad1d) +ReplicationPad3d = basic_unit(nn.ReplicationPad3d) +CrossMapLRN2d = basic_unit(nn.CrossMapLRN2d) +Embedding = basic_unit(nn.Embedding) +EmbeddingBag = basic_unit(nn.EmbeddingBag) +RNNBase = basic_unit(nn.RNNBase) +RNN = basic_unit(nn.RNN) +LSTM = basic_unit(nn.LSTM) +GRU = basic_unit(nn.GRU) +RNNCellBase = basic_unit(nn.RNNCellBase) +RNNCell = basic_unit(nn.RNNCell) +LSTMCell = basic_unit(nn.LSTMCell) +GRUCell = basic_unit(nn.GRUCell) +PixelShuffle = basic_unit(nn.PixelShuffle) +Upsample = basic_unit(nn.Upsample) +UpsamplingNearest2d = basic_unit(nn.UpsamplingNearest2d) +UpsamplingBilinear2d = basic_unit(nn.UpsamplingBilinear2d) +PairwiseDistance = basic_unit(nn.PairwiseDistance) +AdaptiveMaxPool1d = basic_unit(nn.AdaptiveMaxPool1d) +AdaptiveMaxPool2d = basic_unit(nn.AdaptiveMaxPool2d) +AdaptiveMaxPool3d = basic_unit(nn.AdaptiveMaxPool3d) +AdaptiveAvgPool1d = basic_unit(nn.AdaptiveAvgPool1d) +AdaptiveAvgPool2d = basic_unit(nn.AdaptiveAvgPool2d) +AdaptiveAvgPool3d = basic_unit(nn.AdaptiveAvgPool3d) +TripletMarginLoss = basic_unit(nn.TripletMarginLoss) +ZeroPad2d = basic_unit(nn.ZeroPad2d) +ConstantPad1d = basic_unit(nn.ConstantPad1d) +ConstantPad2d = basic_unit(nn.ConstantPad2d) +ConstantPad3d = basic_unit(nn.ConstantPad3d) +Bilinear = basic_unit(nn.Bilinear) +CosineSimilarity = basic_unit(nn.CosineSimilarity) +Unfold = basic_unit(nn.Unfold) +Fold = basic_unit(nn.Fold) +AdaptiveLogSoftmaxWithLoss = basic_unit(nn.AdaptiveLogSoftmaxWithLoss) +TransformerEncoder = basic_unit(nn.TransformerEncoder) +TransformerDecoder = basic_unit(nn.TransformerDecoder) +TransformerEncoderLayer = basic_unit(nn.TransformerEncoderLayer) +TransformerDecoderLayer = basic_unit(nn.TransformerDecoderLayer) +Transformer = basic_unit(nn.Transformer) +Flatten = basic_unit(nn.Flatten) +Hardsigmoid = basic_unit(nn.Hardsigmoid) + +if version_larger_equal(torch.__version__, '1.6.0'): + Hardswish = basic_unit(nn.Hardswish) + +if version_larger_equal(torch.__version__, '1.7.0'): + SiLU = basic_unit(nn.SiLU) + Unflatten = basic_unit(nn.Unflatten) + TripletMarginWithDistanceLoss = basic_unit(nn.TripletMarginWithDistanceLoss) diff --git a/utils/third_party/nni_new/retiarii/nn/pytorch/utils.py b/utils/third_party/nni_new/retiarii/nn/pytorch/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..352348b997389cfe1d7afc9071f056265115ab2a --- /dev/null +++ b/utils/third_party/nni_new/retiarii/nn/pytorch/utils.py @@ -0,0 +1,17 @@ +from typing import Optional + +from ...utils import uid, get_current_context + + +def generate_new_label(label: Optional[str]): + if label is None: + return '_mutation_' + str(uid('mutation')) + return label + + +def get_fixed_value(label: str): + ret = get_current_context('fixed') + try: + return ret[generate_new_label(label)] + except KeyError: + raise KeyError(f'Fixed context with {label} not found. Existing values are: {ret}') diff --git a/utils/third_party/nni_new/retiarii/oneshot/__init__.py b/utils/third_party/nni_new/retiarii/oneshot/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d4e1dad8512bd03c9475d4546d2b86748222aff2 --- /dev/null +++ b/utils/third_party/nni_new/retiarii/oneshot/__init__.py @@ -0,0 +1,4 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .interface import BaseOneShotTrainer diff --git a/utils/third_party/nni_new/retiarii/oneshot/interface.py b/utils/third_party/nni_new/retiarii/oneshot/interface.py new file mode 100644 index 0000000000000000000000000000000000000000..217bae9096efe518bdef9accc6ce55ef61fedc78 --- /dev/null +++ b/utils/third_party/nni_new/retiarii/oneshot/interface.py @@ -0,0 +1,25 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import abc +from typing import Any + + +class BaseOneShotTrainer(abc.ABC): + """ + Build many (possibly all) architectures into a full graph, search (with train) and export the best. + + One-shot trainer has a ``fit`` function with no return value. Trainers should fit and search for the best architecture. + Currently, all the inputs of trainer needs to be manually set before fit (including the search space, data loader + to use training epochs, and etc.). + + It has an extra ``export`` function that exports an object representing the final searched architecture. + """ + + @abc.abstractmethod + def fit(self) -> None: + pass + + @abc.abstractmethod + def export(self) -> Any: + pass diff --git a/utils/third_party/nni_new/retiarii/oneshot/pytorch/__init__.py b/utils/third_party/nni_new/retiarii/oneshot/pytorch/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6e9456958af9e0b6f78450e8707f9d1ef046d009 --- /dev/null +++ b/utils/third_party/nni_new/retiarii/oneshot/pytorch/__init__.py @@ -0,0 +1,8 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .darts import DartsTrainer +from .enas import EnasTrainer +from .proxyless import ProxylessTrainer +from .random import SinglePathTrainer, RandomTrainer +from .utils import replace_input_choice, replace_layer_choice diff --git a/utils/third_party/nni_new/retiarii/oneshot/pytorch/darts.py b/utils/third_party/nni_new/retiarii/oneshot/pytorch/darts.py new file mode 100644 index 0000000000000000000000000000000000000000..edcb6d7b864a91ef42d08722e4c638d23c691891 --- /dev/null +++ b/utils/third_party/nni_new/retiarii/oneshot/pytorch/darts.py @@ -0,0 +1,285 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import copy +import logging + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from ..interface import BaseOneShotTrainer +from .utils import AverageMeterGroup, replace_layer_choice, replace_input_choice + + +_logger = logging.getLogger(__name__) + + +class DartsLayerChoice(nn.Module): + def __init__(self, layer_choice): + super(DartsLayerChoice, self).__init__() + self.name = layer_choice.key + self.op_choices = nn.ModuleDict(layer_choice.named_children()) + self.alpha = nn.Parameter(torch.randn(len(self.op_choices)) * 1e-3) + + def forward(self, *args, **kwargs): + op_results = torch.stack([op(*args, **kwargs) for op in self.op_choices.values()]) + alpha_shape = [-1] + [1] * (len(op_results.size()) - 1) + return torch.sum(op_results * F.softmax(self.alpha, -1).view(*alpha_shape), 0) + + def parameters(self): + for _, p in self.named_parameters(): + yield p + + def named_parameters(self): + for name, p in super(DartsLayerChoice, self).named_parameters(): + if name == 'alpha': + continue + yield name, p + + def export(self): + return torch.argmax(self.alpha).item() + + +class DartsInputChoice(nn.Module): + def __init__(self, input_choice): + super(DartsInputChoice, self).__init__() + self.name = input_choice.key + self.alpha = nn.Parameter(torch.randn(input_choice.n_candidates) * 1e-3) + self.n_chosen = input_choice.n_chosen or 1 + + def forward(self, inputs): + inputs = torch.stack(inputs) + alpha_shape = [-1] + [1] * (len(inputs.size()) - 1) + return torch.sum(inputs * F.softmax(self.alpha, -1).view(*alpha_shape), 0) + + def parameters(self): + for _, p in self.named_parameters(): + yield p + + def named_parameters(self): + for name, p in super(DartsInputChoice, self).named_parameters(): + if name == 'alpha': + continue + yield name, p + + def export(self): + return torch.argsort(-self.alpha).cpu().numpy().tolist()[:self.n_chosen] + + +class DartsTrainer(BaseOneShotTrainer): + """ + DARTS trainer. + + Parameters + ---------- + model : nn.Module + PyTorch model to be trained. + loss : callable + Receives logits and ground truth label, return a loss tensor. + metrics : callable + Receives logits and ground truth label, return a dict of metrics. + optimizer : Optimizer + The optimizer used for optimizing the model. + num_epochs : int + Number of epochs planned for training. + dataset : Dataset + Dataset for training. Will be split for training weights and architecture weights. + grad_clip : float + Gradient clipping. Set to 0 to disable. Default: 5. + learning_rate : float + Learning rate to optimize the model. + batch_size : int + Batch size. + workers : int + Workers for data loading. + device : torch.device + ``torch.device("cpu")`` or ``torch.device("cuda")``. + log_frequency : int + Step count per logging. + arc_learning_rate : float + Learning rate of architecture parameters. + unrolled : float + ``True`` if using second order optimization, else first order optimization. + """ + + def __init__(self, model, loss, metrics, optimizer, + num_epochs, dataset, grad_clip=5., + learning_rate=2.5E-3, batch_size=64, workers=4, + device=None, log_frequency=None, + arc_learning_rate=3.0E-4, unrolled=False): + self.model = model + self.loss = loss + self.metrics = metrics + self.num_epochs = num_epochs + self.dataset = dataset + self.batch_size = batch_size + self.workers = workers + self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') if device is None else device + self.log_frequency = log_frequency + self.model.to(self.device) + + self.nas_modules = [] + replace_layer_choice(self.model, DartsLayerChoice, self.nas_modules) + replace_input_choice(self.model, DartsInputChoice, self.nas_modules) + for _, module in self.nas_modules: + module.to(self.device) + + self.model_optim = optimizer + # use the same architecture weight for modules with duplicated names + ctrl_params = {} + for _, m in self.nas_modules: + if m.name in ctrl_params: + assert m.alpha.size() == ctrl_params[m.name].size(), 'Size of parameters with the same label should be same.' + m.alpha = ctrl_params[m.name] + else: + ctrl_params[m.name] = m.alpha + self.ctrl_optim = torch.optim.Adam(list(ctrl_params.values()), arc_learning_rate, betas=(0.5, 0.999), + weight_decay=1.0E-3) + self.unrolled = unrolled + self.grad_clip = 5. + + self._init_dataloader() + + def _init_dataloader(self): + n_train = len(self.dataset) + split = n_train // 2 + indices = list(range(n_train)) + train_sampler = torch.utils.data.sampler.SubsetRandomSampler(indices[:split]) + valid_sampler = torch.utils.data.sampler.SubsetRandomSampler(indices[split:]) + self.train_loader = torch.utils.data.DataLoader(self.dataset, + batch_size=self.batch_size, + sampler=train_sampler, + num_workers=self.workers) + self.valid_loader = torch.utils.data.DataLoader(self.dataset, + batch_size=self.batch_size, + sampler=valid_sampler, + num_workers=self.workers) + + def _train_one_epoch(self, epoch): + self.model.train() + meters = AverageMeterGroup() + for step, ((trn_X, trn_y), (val_X, val_y)) in enumerate(zip(self.train_loader, self.valid_loader)): + trn_X, trn_y = trn_X.to(self.device), trn_y.to(self.device) + val_X, val_y = val_X.to(self.device), val_y.to(self.device) + + # phase 1. architecture step + self.ctrl_optim.zero_grad() + if self.unrolled: + self._unrolled_backward(trn_X, trn_y, val_X, val_y) + else: + self._backward(val_X, val_y) + self.ctrl_optim.step() + + # phase 2: child network step + self.model_optim.zero_grad() + logits, loss = self._logits_and_loss(trn_X, trn_y) + loss.backward() + if self.grad_clip > 0: + nn.utils.clip_grad_norm_(self.model.parameters(), self.grad_clip) # gradient clipping + self.model_optim.step() + + metrics = self.metrics(logits, trn_y) + metrics['loss'] = loss.item() + meters.update(metrics) + if self.log_frequency is not None and step % self.log_frequency == 0: + _logger.info('Epoch [%s/%s] Step [%s/%s] %s', epoch + 1, + self.num_epochs, step + 1, len(self.train_loader), meters) + + def _logits_and_loss(self, X, y): + logits = self.model(X) + loss = self.loss(logits, y) + return logits, loss + + def _backward(self, val_X, val_y): + """ + Simple backward with gradient descent + """ + _, loss = self._logits_and_loss(val_X, val_y) + loss.backward() + + def _unrolled_backward(self, trn_X, trn_y, val_X, val_y): + """ + Compute unrolled loss and backward its gradients + """ + backup_params = copy.deepcopy(tuple(self.model.parameters())) + + # do virtual step on training data + lr = self.model_optim.param_groups[0]["lr"] + momentum = self.model_optim.param_groups[0]["momentum"] + weight_decay = self.model_optim.param_groups[0]["weight_decay"] + self._compute_virtual_model(trn_X, trn_y, lr, momentum, weight_decay) + + # calculate unrolled loss on validation data + # keep gradients for model here for compute hessian + _, loss = self._logits_and_loss(val_X, val_y) + w_model, w_ctrl = tuple(self.model.parameters()), tuple([c.alpha for c in self.nas_modules]) + w_grads = torch.autograd.grad(loss, w_model + w_ctrl) + d_model, d_ctrl = w_grads[:len(w_model)], w_grads[len(w_model):] + + # compute hessian and final gradients + hessian = self._compute_hessian(backup_params, d_model, trn_X, trn_y) + with torch.no_grad(): + for param, d, h in zip(w_ctrl, d_ctrl, hessian): + # gradient = dalpha - lr * hessian + param.grad = d - lr * h + + # restore weights + self._restore_weights(backup_params) + + def _compute_virtual_model(self, X, y, lr, momentum, weight_decay): + """ + Compute unrolled weights w` + """ + # don't need zero_grad, using autograd to calculate gradients + _, loss = self._logits_and_loss(X, y) + gradients = torch.autograd.grad(loss, self.model.parameters()) + with torch.no_grad(): + for w, g in zip(self.model.parameters(), gradients): + m = self.model_optim.state[w].get('momentum_buffer', 0.) + w = w - lr * (momentum * m + g + weight_decay * w) + + def _restore_weights(self, backup_params): + with torch.no_grad(): + for param, backup in zip(self.model.parameters(), backup_params): + param.copy_(backup) + + def _compute_hessian(self, backup_params, dw, trn_X, trn_y): + """ + dw = dw` { L_val(w`, alpha) } + w+ = w + eps * dw + w- = w - eps * dw + hessian = (dalpha { L_trn(w+, alpha) } - dalpha { L_trn(w-, alpha) }) / (2*eps) + eps = 0.01 / ||dw|| + """ + self._restore_weights(backup_params) + norm = torch.cat([w.view(-1) for w in dw]).norm() + eps = 0.01 / norm + if norm < 1E-8: + _logger.warning('In computing hessian, norm is smaller than 1E-8, cause eps to be %.6f.', norm.item()) + + dalphas = [] + for e in [eps, -2. * eps]: + # w+ = w + eps*dw`, w- = w - eps*dw` + with torch.no_grad(): + for p, d in zip(self.model.parameters(), dw): + p += e * d + + _, loss = self._logits_and_loss(trn_X, trn_y) + dalphas.append(torch.autograd.grad(loss, [c.alpha for c in self.nas_modules])) + + dalpha_pos, dalpha_neg = dalphas # dalpha { L_trn(w+) }, # dalpha { L_trn(w-) } + hessian = [(p - n) / (2. * eps) for p, n in zip(dalpha_pos, dalpha_neg)] + return hessian + + def fit(self): + for i in range(self.num_epochs): + self._train_one_epoch(i) + + @torch.no_grad() + def export(self): + result = dict() + for name, module in self.nas_modules: + if name not in result: + result[name] = module.export() + return result diff --git a/utils/third_party/nni_new/retiarii/oneshot/pytorch/enas.py b/utils/third_party/nni_new/retiarii/oneshot/pytorch/enas.py new file mode 100644 index 0000000000000000000000000000000000000000..7f03c1dd1015a23f6255ba05a221a179e8d37fde --- /dev/null +++ b/utils/third_party/nni_new/retiarii/oneshot/pytorch/enas.py @@ -0,0 +1,336 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.optim as optim + +from ..interface import BaseOneShotTrainer +from .random import PathSamplingLayerChoice, PathSamplingInputChoice +from .utils import AverageMeterGroup, replace_layer_choice, replace_input_choice, to_device + +_logger = logging.getLogger(__name__) + + +class StackedLSTMCell(nn.Module): + def __init__(self, layers, size, bias): + super().__init__() + self.lstm_num_layers = layers + self.lstm_modules = nn.ModuleList([nn.LSTMCell(size, size, bias=bias) + for _ in range(self.lstm_num_layers)]) + + def forward(self, inputs, hidden): + prev_h, prev_c = hidden + next_h, next_c = [], [] + for i, m in enumerate(self.lstm_modules): + curr_h, curr_c = m(inputs, (prev_h[i], prev_c[i])) + next_c.append(curr_c) + next_h.append(curr_h) + # current implementation only supports batch size equals 1, + # but the algorithm does not necessarily have this limitation + inputs = curr_h[-1].view(1, -1) + return next_h, next_c + + +class ReinforceField: + """ + A field with ``name``, with ``total`` choices. ``choose_one`` is true if one and only one is meant to be + selected. Otherwise, any number of choices can be chosen. + """ + + def __init__(self, name, total, choose_one): + self.name = name + self.total = total + self.choose_one = choose_one + + def __repr__(self): + return f'ReinforceField(name={self.name}, total={self.total}, choose_one={self.choose_one})' + + +class ReinforceController(nn.Module): + """ + A controller that mutates the graph with RL. + + Parameters + ---------- + fields : list of ReinforceField + List of fields to choose. + lstm_size : int + Controller LSTM hidden units. + lstm_num_layers : int + Number of layers for stacked LSTM. + tanh_constant : float + Logits will be equal to ``tanh_constant * tanh(logits)``. Don't use ``tanh`` if this value is ``None``. + skip_target : float + Target probability that skipconnect will appear. + temperature : float + Temperature constant that divides the logits. + entropy_reduction : str + Can be one of ``sum`` and ``mean``. How the entropy of multi-input-choice is reduced. + """ + + def __init__(self, fields, lstm_size=64, lstm_num_layers=1, tanh_constant=1.5, + skip_target=0.4, temperature=None, entropy_reduction='sum'): + super(ReinforceController, self).__init__() + self.fields = fields + self.lstm_size = lstm_size + self.lstm_num_layers = lstm_num_layers + self.tanh_constant = tanh_constant + self.temperature = temperature + self.skip_target = skip_target + + self.lstm = StackedLSTMCell(self.lstm_num_layers, self.lstm_size, False) + self.attn_anchor = nn.Linear(self.lstm_size, self.lstm_size, bias=False) + self.attn_query = nn.Linear(self.lstm_size, self.lstm_size, bias=False) + self.v_attn = nn.Linear(self.lstm_size, 1, bias=False) + self.g_emb = nn.Parameter(torch.randn(1, self.lstm_size) * 0.1) + self.skip_targets = nn.Parameter(torch.tensor([1.0 - self.skip_target, self.skip_target]), # pylint: disable=not-callable + requires_grad=False) + assert entropy_reduction in ['sum', 'mean'], 'Entropy reduction must be one of sum and mean.' + self.entropy_reduction = torch.sum if entropy_reduction == 'sum' else torch.mean + self.cross_entropy_loss = nn.CrossEntropyLoss(reduction='none') + self.soft = nn.ModuleDict({ + field.name: nn.Linear(self.lstm_size, field.total, bias=False) for field in fields + }) + self.embedding = nn.ModuleDict({ + field.name: nn.Embedding(field.total, self.lstm_size) for field in fields + }) + + def resample(self): + self._initialize() + result = dict() + for field in self.fields: + result[field.name] = self._sample_single(field) + return result + + def _initialize(self): + self._inputs = self.g_emb.data + self._c = [torch.zeros((1, self.lstm_size), + dtype=self._inputs.dtype, + device=self._inputs.device) for _ in range(self.lstm_num_layers)] + self._h = [torch.zeros((1, self.lstm_size), + dtype=self._inputs.dtype, + device=self._inputs.device) for _ in range(self.lstm_num_layers)] + self.sample_log_prob = 0 + self.sample_entropy = 0 + self.sample_skip_penalty = 0 + + def _lstm_next_step(self): + self._h, self._c = self.lstm(self._inputs, (self._h, self._c)) + + def _sample_single(self, field): + self._lstm_next_step() + logit = self.soft[field.name](self._h[-1]) + if self.temperature is not None: + logit /= self.temperature + if self.tanh_constant is not None: + logit = self.tanh_constant * torch.tanh(logit) + if field.choose_one: + sampled = torch.multinomial(F.softmax(logit, dim=-1), 1).view(-1) + log_prob = self.cross_entropy_loss(logit, sampled) + self._inputs = self.embedding[field.name](sampled) + else: + logit = logit.view(-1, 1) + logit = torch.cat([-logit, logit], 1) # pylint: disable=invalid-unary-operand-type + sampled = torch.multinomial(F.softmax(logit, dim=-1), 1).view(-1) + skip_prob = torch.sigmoid(logit) + kl = torch.sum(skip_prob * torch.log(skip_prob / self.skip_targets)) + self.sample_skip_penalty += kl + log_prob = self.cross_entropy_loss(logit, sampled) + sampled = sampled.nonzero().view(-1) + if sampled.sum().item(): + self._inputs = (torch.sum(self.embedding[field.name](sampled.view(-1)), 0) / (1. + torch.sum(sampled))).unsqueeze(0) + else: + self._inputs = torch.zeros(1, self.lstm_size, device=self.embedding[field.name].weight.device) + + sampled = sampled.detach().numpy().tolist() + self.sample_log_prob += self.entropy_reduction(log_prob) + entropy = (log_prob * torch.exp(-log_prob)).detach() # pylint: disable=invalid-unary-operand-type + self.sample_entropy += self.entropy_reduction(entropy) + if len(sampled) == 1: + sampled = sampled[0] + return sampled + + +class EnasTrainer(BaseOneShotTrainer): + """ + ENAS trainer. + + Parameters + ---------- + model : nn.Module + PyTorch model to be trained. + loss : callable + Receives logits and ground truth label, return a loss tensor. + metrics : callable + Receives logits and ground truth label, return a dict of metrics. + reward_function : callable + Receives logits and ground truth label, return a tensor, which will be feeded to RL controller as reward. + optimizer : Optimizer + The optimizer used for optimizing the model. + num_epochs : int + Number of epochs planned for training. + dataset : Dataset + Dataset for training. Will be split for training weights and architecture weights. + batch_size : int + Batch size. + workers : int + Workers for data loading. + device : torch.device + ``torch.device("cpu")`` or ``torch.device("cuda")``. + log_frequency : int + Step count per logging. + grad_clip : float + Gradient clipping. Set to 0 to disable. Default: 5. + entropy_weight : float + Weight of sample entropy loss. + skip_weight : float + Weight of skip penalty loss. + baseline_decay : float + Decay factor of baseline. New baseline will be equal to ``baseline_decay * baseline_old + reward * (1 - baseline_decay)``. + ctrl_lr : float + Learning rate for RL controller. + ctrl_steps_aggregate : int + Number of steps that will be aggregated into one mini-batch for RL controller. + ctrl_steps : int + Number of mini-batches for each epoch of RL controller learning. + ctrl_kwargs : dict + Optional kwargs that will be passed to :class:`ReinforceController`. + """ + + def __init__(self, model, loss, metrics, reward_function, + optimizer, num_epochs, dataset, + batch_size=64, workers=4, device=None, log_frequency=None, + grad_clip=5., entropy_weight=0.0001, skip_weight=0.8, baseline_decay=0.999, + ctrl_lr=0.00035, ctrl_steps_aggregate=20, ctrl_kwargs=None): + self.model = model + self.loss = loss + self.metrics = metrics + self.optimizer = optimizer + self.num_epochs = num_epochs + self.dataset = dataset + self.batch_size = batch_size + self.workers = workers + self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') if device is None else device + self.log_frequency = log_frequency + + self.nas_modules = [] + replace_layer_choice(self.model, PathSamplingLayerChoice, self.nas_modules) + replace_input_choice(self.model, PathSamplingInputChoice, self.nas_modules) + for _, module in self.nas_modules: + module.to(self.device) + self.model.to(self.device) + + self.nas_fields = [ReinforceField(name, len(module), + isinstance(module, PathSamplingLayerChoice) or module.n_chosen == 1) + for name, module in self.nas_modules] + self.controller = ReinforceController(self.nas_fields, **(ctrl_kwargs or {})) + + self.grad_clip = grad_clip + self.reward_function = reward_function + self.ctrl_optim = optim.Adam(self.controller.parameters(), lr=ctrl_lr) + self.batch_size = batch_size + self.workers = workers + + self.entropy_weight = entropy_weight + self.skip_weight = skip_weight + self.baseline_decay = baseline_decay + self.baseline = 0. + self.ctrl_steps_aggregate = ctrl_steps_aggregate + + self.init_dataloader() + + def init_dataloader(self): + n_train = len(self.dataset) + split = n_train // 2 + indices = list(range(n_train)) + train_sampler = torch.utils.data.sampler.SubsetRandomSampler(indices[:-split]) + valid_sampler = torch.utils.data.sampler.SubsetRandomSampler(indices[-split:]) + self.train_loader = torch.utils.data.DataLoader(self.dataset, + batch_size=self.batch_size, + sampler=train_sampler, + num_workers=self.workers) + self.valid_loader = torch.utils.data.DataLoader(self.dataset, + batch_size=self.batch_size, + sampler=valid_sampler, + num_workers=self.workers) + + def _train_model(self, epoch): + self.model.train() + self.controller.eval() + meters = AverageMeterGroup() + for step, (x, y) in enumerate(self.train_loader): + x, y = to_device(x, self.device), to_device(y, self.device) + self.optimizer.zero_grad() + + self._resample() + logits = self.model(x) + metrics = self.metrics(logits, y) + loss = self.loss(logits, y) + loss.backward() + if self.grad_clip > 0: + nn.utils.clip_grad_norm_(self.model.parameters(), self.grad_clip) + self.optimizer.step() + metrics['loss'] = loss.item() + meters.update(metrics) + + if self.log_frequency is not None and step % self.log_frequency == 0: + _logger.info('Model Epoch [%d/%d] Step [%d/%d] %s', epoch + 1, + self.num_epochs, step + 1, len(self.train_loader), meters) + + def _train_controller(self, epoch): + self.model.eval() + self.controller.train() + meters = AverageMeterGroup() + self.ctrl_optim.zero_grad() + for ctrl_step, (x, y) in enumerate(self.valid_loader): + x, y = to_device(x, self.device), to_device(y, self.device) + + self._resample() + with torch.no_grad(): + logits = self.model(x) + metrics = self.metrics(logits, y) + reward = self.reward_function(logits, y) + if self.entropy_weight: + reward += self.entropy_weight * self.controller.sample_entropy.item() + self.baseline = self.baseline * self.baseline_decay + reward * (1 - self.baseline_decay) + loss = self.controller.sample_log_prob * (reward - self.baseline) + if self.skip_weight: + loss += self.skip_weight * self.controller.sample_skip_penalty + metrics['reward'] = reward + metrics['loss'] = loss.item() + metrics['ent'] = self.controller.sample_entropy.item() + metrics['log_prob'] = self.controller.sample_log_prob.item() + metrics['baseline'] = self.baseline + metrics['skip'] = self.controller.sample_skip_penalty + + loss /= self.ctrl_steps_aggregate + loss.backward() + meters.update(metrics) + + if (ctrl_step + 1) % self.ctrl_steps_aggregate == 0: + if self.grad_clip > 0: + nn.utils.clip_grad_norm_(self.controller.parameters(), self.grad_clip) + self.ctrl_optim.step() + self.ctrl_optim.zero_grad() + + if self.log_frequency is not None and ctrl_step % self.log_frequency == 0: + _logger.info('RL Epoch [%d/%d] Step [%d/%d] %s', epoch + 1, self.num_epochs, + ctrl_step + 1, len(self.valid_loader), meters) + + def _resample(self): + result = self.controller.resample() + for name, module in self.nas_modules: + module.sampled = result[name] + + def fit(self): + for i in range(self.num_epochs): + self._train_model(i) + self._train_controller(i) + + def export(self): + self.controller.eval() + with torch.no_grad(): + return self.controller.resample() diff --git a/utils/third_party/nni_new/retiarii/oneshot/pytorch/proxyless.py b/utils/third_party/nni_new/retiarii/oneshot/pytorch/proxyless.py new file mode 100644 index 0000000000000000000000000000000000000000..b44d883711c3e5af3d0e42fad8b4effc91a6e281 --- /dev/null +++ b/utils/third_party/nni_new/retiarii/oneshot/pytorch/proxyless.py @@ -0,0 +1,227 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from ..interface import BaseOneShotTrainer +from .utils import AverageMeterGroup, replace_layer_choice, replace_input_choice + + +_logger = logging.getLogger(__name__) + + +class ArchGradientFunction(torch.autograd.Function): + @staticmethod + def forward(ctx, x, binary_gates, run_func, backward_func): + ctx.run_func = run_func + ctx.backward_func = backward_func + + detached_x = x.detach() + detached_x.requires_grad = x.requires_grad + with torch.enable_grad(): + output = run_func(detached_x) + ctx.save_for_backward(detached_x, output) + return output.data + + @staticmethod + def backward(ctx, grad_output): + detached_x, output = ctx.saved_tensors + + grad_x = torch.autograd.grad(output, detached_x, grad_output, only_inputs=True) + # compute gradients w.r.t. binary_gates + binary_grads = ctx.backward_func(detached_x.data, output.data, grad_output.data) + + return grad_x[0], binary_grads, None, None + + +class ProxylessLayerChoice(nn.Module): + def __init__(self, ops): + super(ProxylessLayerChoice, self).__init__() + self.ops = nn.ModuleList(ops) + self.alpha = nn.Parameter(torch.randn(len(self.ops)) * 1E-3) + self._binary_gates = nn.Parameter(torch.randn(len(self.ops)) * 1E-3) + self.sampled = None + + def forward(self, *args): + def run_function(ops, active_id): + def forward(_x): + return ops[active_id](_x) + return forward + + def backward_function(ops, active_id, binary_gates): + def backward(_x, _output, grad_output): + binary_grads = torch.zeros_like(binary_gates.data) + with torch.no_grad(): + for k in range(len(ops)): + if k != active_id: + out_k = ops[k](_x.data) + else: + out_k = _output.data + grad_k = torch.sum(out_k * grad_output) + binary_grads[k] = grad_k + return binary_grads + return backward + + assert len(args) == 1 + x = args[0] + return ArchGradientFunction.apply( + x, self._binary_gates, run_function(self.ops, self.sampled), + backward_function(self.ops, self.sampled, self._binary_gates) + ) + + def resample(self): + probs = F.softmax(self.alpha, dim=-1) + sample = torch.multinomial(probs, 1)[0].item() + self.sampled = sample + with torch.no_grad(): + self._binary_gates.zero_() + self._binary_gates.grad = torch.zeros_like(self._binary_gates.data) + self._binary_gates.data[sample] = 1.0 + + def finalize_grad(self): + binary_grads = self._binary_gates.grad + with torch.no_grad(): + if self.alpha.grad is None: + self.alpha.grad = torch.zeros_like(self.alpha.data) + probs = F.softmax(self.alpha, dim=-1) + for i in range(len(self.ops)): + for j in range(len(self.ops)): + self.alpha.grad[i] += binary_grads[j] * probs[j] * (int(i == j) - probs[i]) + + def export(self): + return torch.argmax(self.alpha).item() + + +class ProxylessInputChoice(nn.Module): + def __init__(self, *args, **kwargs): + raise NotImplementedError('Input choice is not supported for ProxylessNAS.') + + +class ProxylessTrainer(BaseOneShotTrainer): + """ + Proxyless trainer. + + Parameters + ---------- + model : nn.Module + PyTorch model to be trained. + loss : callable + Receives logits and ground truth label, return a loss tensor. + metrics : callable + Receives logits and ground truth label, return a dict of metrics. + optimizer : Optimizer + The optimizer used for optimizing the model. + num_epochs : int + Number of epochs planned for training. + dataset : Dataset + Dataset for training. Will be split for training weights and architecture weights. + warmup_epochs : int + Number of epochs to warmup model parameters. + batch_size : int + Batch size. + workers : int + Workers for data loading. + device : torch.device + ``torch.device("cpu")`` or ``torch.device("cuda")``. + log_frequency : int + Step count per logging. + arc_learning_rate : float + Learning rate of architecture parameters. + """ + + def __init__(self, model, loss, metrics, optimizer, + num_epochs, dataset, warmup_epochs=0, + batch_size=64, workers=4, device=None, log_frequency=None, + arc_learning_rate=1.0E-3): + self.model = model + self.loss = loss + self.metrics = metrics + self.optimizer = optimizer + self.num_epochs = num_epochs + self.warmup_epochs = warmup_epochs + self.dataset = dataset + self.batch_size = batch_size + self.workers = workers + self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') if device is None else device + self.log_frequency = log_frequency + self.model.to(self.device) + + self.nas_modules = [] + replace_layer_choice(self.model, ProxylessLayerChoice, self.nas_modules) + replace_input_choice(self.model, ProxylessInputChoice, self.nas_modules) + for _, module in self.nas_modules: + module.to(self.device) + + self.optimizer = optimizer + # we do not support deduplicate control parameters with same label (like DARTS) yet. + self.ctrl_optim = torch.optim.Adam([m.alpha for _, m in self.nas_modules], arc_learning_rate, + weight_decay=0, betas=(0, 0.999), eps=1e-8) + self._init_dataloader() + + def _init_dataloader(self): + n_train = len(self.dataset) + split = n_train // 2 + indices = list(range(n_train)) + train_sampler = torch.utils.data.sampler.SubsetRandomSampler(indices[:split]) + valid_sampler = torch.utils.data.sampler.SubsetRandomSampler(indices[split:]) + self.train_loader = torch.utils.data.DataLoader(self.dataset, + batch_size=self.batch_size, + sampler=train_sampler, + num_workers=self.workers) + self.valid_loader = torch.utils.data.DataLoader(self.dataset, + batch_size=self.batch_size, + sampler=valid_sampler, + num_workers=self.workers) + + def _train_one_epoch(self, epoch): + self.model.train() + meters = AverageMeterGroup() + for step, ((trn_X, trn_y), (val_X, val_y)) in enumerate(zip(self.train_loader, self.valid_loader)): + trn_X, trn_y = trn_X.to(self.device), trn_y.to(self.device) + val_X, val_y = val_X.to(self.device), val_y.to(self.device) + + if epoch >= self.warmup_epochs: + # 1) train architecture parameters + for _, module in self.nas_modules: + module.resample() + self.ctrl_optim.zero_grad() + logits, loss = self._logits_and_loss(val_X, val_y) + loss.backward() + for _, module in self.nas_modules: + module.finalize_grad() + self.ctrl_optim.step() + + # 2) train model parameters + for _, module in self.nas_modules: + module.resample() + self.optimizer.zero_grad() + logits, loss = self._logits_and_loss(trn_X, trn_y) + loss.backward() + self.optimizer.step() + metrics = self.metrics(logits, trn_y) + metrics["loss"] = loss.item() + meters.update(metrics) + if self.log_frequency is not None and step % self.log_frequency == 0: + _logger.info("Epoch [%s/%s] Step [%s/%s] %s", epoch + 1, + self.num_epochs, step + 1, len(self.train_loader), meters) + + def _logits_and_loss(self, X, y): + logits = self.model(X) + loss = self.loss(logits, y) + return logits, loss + + def fit(self): + for i in range(self.num_epochs): + self._train_one_epoch(i) + + @torch.no_grad() + def export(self): + result = dict() + for name, module in self.nas_modules: + if name not in result: + result[name] = module.export() + return result diff --git a/utils/third_party/nni_new/retiarii/oneshot/pytorch/random.py b/utils/third_party/nni_new/retiarii/oneshot/pytorch/random.py new file mode 100644 index 0000000000000000000000000000000000000000..a82ddada10853dfcf4a0ac2bc22301485954a32b --- /dev/null +++ b/utils/third_party/nni_new/retiarii/oneshot/pytorch/random.py @@ -0,0 +1,203 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +import random + +import torch +import torch.nn as nn + +from ..interface import BaseOneShotTrainer +from .utils import AverageMeterGroup, replace_layer_choice, replace_input_choice + + +_logger = logging.getLogger(__name__) + + +def _get_mask(sampled, total): + multihot = [i == sampled or (isinstance(sampled, list) and i in sampled) for i in range(total)] + return torch.tensor(multihot, dtype=torch.bool) # pylint: disable=not-callable + + +class PathSamplingLayerChoice(nn.Module): + """ + Mixed module, in which fprop is decided by exactly one or multiple (sampled) module. + If multiple module is selected, the result will be sumed and returned. + + Attributes + ---------- + sampled : int or list of int + Sampled module indices. + mask : tensor + A multi-hot bool 1D-tensor representing the sampled mask. + """ + + def __init__(self, layer_choice): + super(PathSamplingLayerChoice, self).__init__() + self.op_names = [] + for name, module in layer_choice.named_children(): + self.add_module(name, module) + self.op_names.append(name) + assert self.op_names, 'There has to be at least one op to choose from.' + self.sampled = None # sampled can be either a list of indices or an index + + def forward(self, *args, **kwargs): + assert self.sampled is not None, 'At least one path needs to be sampled before fprop.' + if isinstance(self.sampled, list): + return sum([getattr(self, self.op_names[i])(*args, **kwargs) for i in self.sampled]) # pylint: disable=not-an-iterable + else: + return getattr(self, self.op_names[self.sampled])(*args, **kwargs) # pylint: disable=invalid-sequence-index + + def __len__(self): + return len(self.op_names) + + @property + def mask(self): + return _get_mask(self.sampled, len(self)) + + +class PathSamplingInputChoice(nn.Module): + """ + Mixed input. Take a list of tensor as input, select some of them and return the sum. + + Attributes + ---------- + sampled : int or list of int + Sampled module indices. + mask : tensor + A multi-hot bool 1D-tensor representing the sampled mask. + """ + + def __init__(self, input_choice): + super(PathSamplingInputChoice, self).__init__() + self.n_candidates = input_choice.n_candidates + self.n_chosen = input_choice.n_chosen + self.sampled = None + + def forward(self, input_tensors): + if isinstance(self.sampled, list): + return sum([input_tensors[t] for t in self.sampled]) # pylint: disable=not-an-iterable + else: + return input_tensors[self.sampled] + + def __len__(self): + return self.n_candidates + + @property + def mask(self): + return _get_mask(self.sampled, len(self)) + + +class SinglePathTrainer(BaseOneShotTrainer): + """ + Single-path trainer. Samples a path every time and backpropagates on that path. + + Parameters + ---------- + model : nn.Module + Model with mutables. + loss : callable + Called with logits and targets. Returns a loss tensor. + metrics : callable + Returns a dict that maps metrics keys to metrics data. + optimizer : Optimizer + Optimizer that optimizes the model. + num_epochs : int + Number of epochs of training. + dataset_train : Dataset + Dataset of training. + dataset_valid : Dataset + Dataset of validation. + batch_size : int + Batch size. + workers: int + Number of threads for data preprocessing. Not used for this trainer. Maybe removed in future. + device : torch.device + Device object. Either ``torch.device("cuda")`` or ``torch.device("cpu")``. When ``None``, trainer will + automatic detects GPU and selects GPU first. + log_frequency : int + Number of mini-batches to log metrics. + """ + + def __init__(self, model, loss, metrics, + optimizer, num_epochs, dataset_train, dataset_valid, + mutator=None, batch_size=64, workers=4, device=None, log_frequency=None): + self.model = model + self.loss = loss + self.metrics = metrics + self.optimizer = optimizer + self.num_epochs = num_epochs + self.dataset_train = dataset_train + self.dataset_valid = dataset_valid + self.batch_size = batch_size + self.workers = workers + self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') if device is None else device + self.log_frequency = log_frequency + self.model.to(self.device) + + self.nas_modules = [] + replace_layer_choice(self.model, PathSamplingLayerChoice, self.nas_modules) + replace_input_choice(self.model, PathSamplingInputChoice, self.nas_modules) + for _, module in self.nas_modules: + module.to(self.device) + + self.train_loader = torch.utils.data.DataLoader(self.dataset_train, + batch_size=batch_size, + num_workers=workers) + self.valid_loader = torch.utils.data.DataLoader(self.dataset_valid, + batch_size=batch_size, + num_workers=workers) + + def _resample(self): + result = {} + for name, module in self.nas_modules: + if name not in result: + result[name] = random.randint(0, len(module) - 1) + module.sampled = result[name] + return result + + def _train_one_epoch(self, epoch): + self.model.train() + meters = AverageMeterGroup() + for step, (x, y) in enumerate(self.train_loader): + x, y = x.to(self.device), y.to(self.device) + self.optimizer.zero_grad() + self._resample() + logits = self.model(x) + loss = self.loss(logits, y) + loss.backward() + self.optimizer.step() + + metrics = self.metrics(logits, y) + metrics["loss"] = loss.item() + meters.update(metrics) + if self.log_frequency is not None and step % self.log_frequency == 0: + _logger.info("Epoch [%s/%s] Step [%s/%s] %s", epoch + 1, + self.num_epochs, step + 1, len(self.train_loader), meters) + + def _validate_one_epoch(self, epoch): + self.model.eval() + meters = AverageMeterGroup() + with torch.no_grad(): + for step, (x, y) in enumerate(self.valid_loader): + x, y = x.to(self.device), y.to(self.device) + self._resample() + logits = self.model(x) + loss = self.loss(logits, y) + metrics = self.metrics(logits, y) + metrics["loss"] = loss.item() + meters.update(metrics) + if self.log_frequency is not None and step % self.log_frequency == 0: + _logger.info("Epoch [%s/%s] Validation Step [%s/%s] %s", epoch + 1, + self.num_epochs, step + 1, len(self.valid_loader), meters) + + def fit(self): + for i in range(self.num_epochs): + self._train_one_epoch(i) + self._validate_one_epoch(i) + + def export(self): + return self._resample() + + +RandomTrainer = SinglePathTrainer diff --git a/utils/third_party/nni_new/retiarii/oneshot/pytorch/utils.py b/utils/third_party/nni_new/retiarii/oneshot/pytorch/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..61a4f916ae626cfd5409064321ccd6915e2868cc --- /dev/null +++ b/utils/third_party/nni_new/retiarii/oneshot/pytorch/utils.py @@ -0,0 +1,182 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +from collections import OrderedDict + +import numpy as np +import torch +import nni.retiarii.nn.pytorch as nn +from nni.nas.pytorch.mutables import InputChoice, LayerChoice + +_logger = logging.getLogger(__name__) + + +def to_device(obj, device): + """ + Move a tensor, tuple, list, or dict onto device. + """ + if torch.is_tensor(obj): + return obj.to(device) + if isinstance(obj, tuple): + return tuple(to_device(t, device) for t in obj) + if isinstance(obj, list): + return [to_device(t, device) for t in obj] + if isinstance(obj, dict): + return {k: to_device(v, device) for k, v in obj.items()} + if isinstance(obj, (int, float, str)): + return obj + raise ValueError("'%s' has unsupported type '%s'" % (obj, type(obj))) + + +def to_list(arr): + if torch.is_tensor(arr): + return arr.cpu().numpy().tolist() + if isinstance(arr, np.ndarray): + return arr.tolist() + if isinstance(arr, (list, tuple)): + return list(arr) + return arr + + +class AverageMeterGroup: + """ + Average meter group for multiple average meters. + """ + + def __init__(self): + self.meters = OrderedDict() + + def update(self, data): + """ + Update the meter group with a dict of metrics. + Non-exist average meters will be automatically created. + """ + for k, v in data.items(): + if k not in self.meters: + self.meters[k] = AverageMeter(k, ":4f") + self.meters[k].update(v) + + def __getattr__(self, item): + return self.meters[item] + + def __getitem__(self, item): + return self.meters[item] + + def __str__(self): + return " ".join(str(v) for v in self.meters.values()) + + def summary(self): + """ + Return a summary string of group data. + """ + return " ".join(v.summary() for v in self.meters.values()) + + +class AverageMeter: + """ + Computes and stores the average and current value. + + Parameters + ---------- + name : str + Name to display. + fmt : str + Format string to print the values. + """ + + def __init__(self, name, fmt=':f'): + self.name = name + self.fmt = fmt + self.reset() + + def reset(self): + """ + Reset the meter. + """ + self.val = 0 + self.avg = 0 + self.sum = 0 + self.count = 0 + + def update(self, val, n=1): + """ + Update with value and weight. + + Parameters + ---------- + val : float or int + The new value to be accounted in. + n : int + The weight of the new value. + """ + self.val = val + self.sum += val * n + self.count += n + self.avg = self.sum / self.count + + def __str__(self): + fmtstr = '{name} {val' + self.fmt + '} ({avg' + self.fmt + '})' + return fmtstr.format(**self.__dict__) + + def summary(self): + fmtstr = '{name}: {avg' + self.fmt + '}' + return fmtstr.format(**self.__dict__) + + +def _replace_module_with_type(root_module, init_fn, type_name, modules): + if modules is None: + modules = [] + + def apply(m): + for name, child in m.named_children(): + if isinstance(child, type_name): + setattr(m, name, init_fn(child)) + modules.append((child.key, getattr(m, name))) + else: + apply(child) + + apply(root_module) + return modules + + +def replace_layer_choice(root_module, init_fn, modules=None): + """ + Replace layer choice modules with modules that are initiated with init_fn. + + Parameters + ---------- + root_module : nn.Module + Root module to traverse. + init_fn : Callable + Initializing function. + modules : dict, optional + Update the replaced modules into the dict and check duplicate if provided. + + Returns + ------- + List[Tuple[str, nn.Module]] + A list from layer choice keys (names) and replaced modules. + """ + return _replace_module_with_type(root_module, init_fn, (LayerChoice, nn.LayerChoice), modules) + + +def replace_input_choice(root_module, init_fn, modules=None): + """ + Replace input choice modules with modules that are initiated with init_fn. + + Parameters + ---------- + root_module : nn.Module + Root module to traverse. + init_fn : Callable + Initializing function. + modules : dict, optional + Update the replaced modules into the dict and check duplicate if provided. + + Returns + ------- + List[Tuple[str, nn.Module]] + A list from layer choice keys (names) and replaced modules. + """ + return _replace_module_with_type(root_module, init_fn, (InputChoice, nn.InputChoice), modules) diff --git a/utils/third_party/nni_new/retiarii/operation.py b/utils/third_party/nni_new/retiarii/operation.py new file mode 100644 index 0000000000000000000000000000000000000000..d97f87f46b374a74455ff3916c59b691cffc6b71 --- /dev/null +++ b/utils/third_party/nni_new/retiarii/operation.py @@ -0,0 +1,233 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from typing import (Any, Dict, List) + +from . import debug_configs + +__all__ = ['Operation', 'Cell'] + + +def _convert_name(name: str) -> str: + """ + Convert the names using separator '.' to valid variable name in code + """ + return name.replace('.', '__') + + +class Operation: + """ + Calculation logic of a graph node. + + The constructor is private. Use `Operation.new()` to create operation object. + + `Operation` is a naive record. + Do not "mutate" its attributes or store information relate to specific node. + All complex logic should be implemented in `Node` class. + + Attributes + ---------- + type + Operation type name (e.g. Conv2D). + If it starts with underscore, the "operation" is a special one (e.g. subgraph, input/output). + parameters + Arbitrary key-value parameters (e.g. kernel_size). + """ + + def __init__(self, type_name: str, parameters: Dict[str, Any], _internal: bool = False): + assert _internal, '`Operation()` is private, use `Operation.new()` instead' + self.type: str = type_name + self.parameters: Dict[str, Any] = parameters + + def to_init_code(self, field: str) -> str: + raise NotImplementedError() + + def to_forward_code(self, field: str, output: str, inputs: List[str]) -> str: + raise NotImplementedError() + + def _to_class_name(self) -> str: + raise NotImplementedError() + + def __bool__(self) -> bool: + return True + + @staticmethod + def new(type_name: str, parameters: Dict[str, Any] = {}, cell_name: str = None) -> 'Operation': + if type_name == '_cell': + # NOTE: cell_name is the same as its Node's name, when the cell is wrapped within the node + return Cell(cell_name, parameters) + else: + if debug_configs.framework.lower() in ('torch', 'pytorch'): + from .operation_def import torch_op_def # pylint: disable=unused-import + cls = PyTorchOperation._find_subclass(type_name) + elif debug_configs.framework.lower() in ('tf', 'tensorflow'): + from .operation_def import tf_op_def # pylint: disable=unused-import + cls = TensorFlowOperation._find_subclass(type_name) + else: + raise ValueError(f'Unsupported framework: {debug_configs.framework}') + return cls(type_name, parameters, _internal=True) + + @classmethod + def _find_subclass(cls, subclass_name): + for subclass in cls.__subclasses__(): + if subclass.__name__ == subclass_name: + return subclass + return cls + + def __repr__(self): + type_name = type(self).__name__ + args = [f'{key}={repr(value)}' for key, value in self.parameters.items()] + if type_name != self.type: + args = [f'type="{self.type}"'] + args + return f'{type_name}({", ".join(args)})' + + def __eq__(self, other): + return type(other) is type(self) and other.type == self.type and other.parameters == self.parameters + + +class PyTorchOperation(Operation): + @classmethod + def _find_subclass(cls, subclass_name): + if cls.to_class_name(subclass_name) is not None: + subclass_name = 'ModuleOperator' + if cls.is_functional(subclass_name): + subclass_name = 'FunctionalOperator' + for subclass in cls.__subclasses__(): + if hasattr(subclass, '_ori_type_name') and \ + subclass_name in subclass._ori_type_name: + return subclass + return cls + + @classmethod + def to_class_name(cls, type_name) -> str: + if type_name.startswith('__torch__.'): + return type_name[len('__torch__.'):] + elif type_name.startswith('__mutated__.'): + return type_name[len('__mutated__.'):] + else: + return None + + @classmethod + def is_functional(cls, type_name) -> bool: + return type_name.startswith('Function.') + + def _to_class_name(self) -> str: + if self.type.startswith('__torch__.'): + return self.type[len('__torch__.'):] + elif self.type.startswith('__mutated__.'): + return self.type[len('__mutated__.'):] + else: + return None + + def get_import_pkg(self) -> str: + if self.type.startswith('__torch__.'): + return self.type[len('__torch__.'):].split('.')[0] + elif self.type.startswith('__mutated__.'): + return self.type[len('__mutated__.'):].split('.')[0] + else: + return None + + def to_init_code(self, field: str) -> str: + if self._to_class_name() is not None: + assert 'positional_args' not in self.parameters + kw_params = ', '.join(f'{key}={repr(value)}' for key, value in self.parameters.items()) + return f'self.{field} = {self._to_class_name()}({kw_params})' + return None + + def to_forward_code(self, field: str, output: str, inputs: List[str], inputs_value: List[Any] = None) -> str: + """ + Parameters + ---------- + field : str + the name of member submodule + output : str + the output name (lvalue) of this line of code + inputs : List[str] + variables used in this line of code + inputs_value : List[Any] + some variables are actually constant, their real values are recorded in ```inputs_value```. + if not constant, we simply put None at the corresponding index + + Returns + ------- + str + generated code line + """ + if self.type == 'aten::slice': + raise RuntimeError('not supposed to have aten::slice operation') + else: + raise RuntimeError(f'unsupported operation type: {self.type} ? {self._to_class_name()}') + + +class TensorFlowOperation(Operation): + def _to_class_name(self) -> str: + return 'K.layers.' + self.type + + +class Cell(PyTorchOperation): + """ + TODO: this is pytorch cell + + An operation reference to a subgraph. + + Example code: + ``` + def __init__(...): + ... + self.cell = CustomCell(...) + self.relu = K.layers.ReLU() + ... + + def forward(...): + ... + x = self.cell(x) + ... + ``` + + In above example, node `self.cell`'s operation is `Cell(cell_name='CustomCell')`. + For comparison, `self.relu`'s operation is `Operation(type='ReLU')`. + + TODO: parameters of subgraph (see `Node` class) + + Attributes + ---------- + type + Always "_cell". + parameters + A dict with only one item; the key is "cell" and the value is cell's name. + framework + No real usage. Exists for compatibility with base class. + """ + + def __init__(self, cell_name: str, parameters: Dict[str, Any] = {}): + self.type = '_cell' + self.cell_name = cell_name + self.parameters = parameters + + def _to_class_name(self): + # TODO: ugly, think about how to refactor this part + return _convert_name(self.cell_name) + + def to_forward_code(self, field: str, output: str, inputs: List[str], inputs_value: List[Any] = None) -> str: + return f'{output} = self.{field}({", ".join(inputs)})' + +class _IOPseudoOperation(Operation): + """ + This is the pseudo operation used by I/O nodes. + The benefit is that users no longer need to verify `Node.operation is not None`, + especially in static type checking. + """ + + def __init__(self, type_name: str, io_names: List = None): + assert type_name.startswith('_') + super(_IOPseudoOperation, self).__init__(type_name, {}, True) + self.io_names = io_names + + def to_init_code(self, field: str) -> str: + raise ValueError(f'Cannot generate code for pseudo operation "{self.type}"') + + def to_forward_code(self, field: str, output: str, inputs: List[str]) -> str: + raise ValueError(f'Cannot generate code for pseudo operation "{self.type}"') + + def __bool__(self) -> bool: + return False diff --git a/utils/third_party/nni_new/retiarii/operation_def/__init__.py b/utils/third_party/nni_new/retiarii/operation_def/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..0b9575cc55a0a035af3de2c1608b60bc7631a676 --- /dev/null +++ b/utils/third_party/nni_new/retiarii/operation_def/__init__.py @@ -0,0 +1,7 @@ +""" +Definition of operation types. + +These are currently examples for overriding codegen. + +Feel free to propose better package name or hierarchy. +""" diff --git a/utils/third_party/nni_new/retiarii/operation_def/tf_op_def.py b/utils/third_party/nni_new/retiarii/operation_def/tf_op_def.py new file mode 100644 index 0000000000000000000000000000000000000000..d7bd64726800028be2d5874beb726a46fb7af694 --- /dev/null +++ b/utils/third_party/nni_new/retiarii/operation_def/tf_op_def.py @@ -0,0 +1,11 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from ..operation import TensorFlowOperation + + +class Conv2D(TensorFlowOperation): + def __init__(self, type_name, parameters, _internal): + if 'padding' not in parameters: + parameters['padding'] = 'same' + super().__init__(type_name, parameters, _internal) diff --git a/utils/third_party/nni_new/retiarii/operation_def/torch_op_def.py b/utils/third_party/nni_new/retiarii/operation_def/torch_op_def.py new file mode 100644 index 0000000000000000000000000000000000000000..bb97069e639af23b179f815759c81423d82f6d62 --- /dev/null +++ b/utils/third_party/nni_new/retiarii/operation_def/torch_op_def.py @@ -0,0 +1,429 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from typing import (Any, List) + +import torch + +from ..operation import PyTorchOperation + + +mem_format = [ + 'torch.contiguous_format', # 0 + 'torch.preserve_format', # 1 + 'torch.channels_last', # 2 +] + +# this snippet is copied from torch/onnx/symbolic_helper.py, +# the original definition is in c10/core/ScalarType.h +# This indicates each scalar type's corresponding +scalar_type_to_pytorch_type = [ + 'torch.uint8', # 0 + 'torch.int8', # 1 + 'torch.short', # 2 + 'torch.int', # 3 + 'torch.int64', # 4 + 'torch.half', # 5 + 'torch.float', # 6 + 'torch.double', # 7 + 'torch.complex32', # 8 + 'torch.complex64', # 9 + 'torch.complex128', # 10 + 'torch.bool', # 11 +] + +class NoOpIdentity(PyTorchOperation): + """ + this operator type is added by us + """ + _ori_type_name = ['noop_identity'] + def to_forward_code(self, field: str, output: str, inputs: List[str], inputs_value: List[Any] = None) -> str: + return f'{output} = {", ".join(inputs)}' + +class ModuleOperator(PyTorchOperation): + _ori_type_name = ['ModuleOperator', 'shared'] + def to_forward_code(self, field: str, output: str, inputs: List[str], inputs_value: List[Any] = None) -> str: + return f'{output} = self.{field}({", ".join(inputs)})' + +class FunctionalOperator(PyTorchOperation): + _ori_type_name = ['FunctionalOperator'] + def to_forward_code(self, field: str, output: str, inputs: List[str], inputs_value: List[Any] = None) -> str: + func_name = self.type[len('Function.'):] + if not hasattr(torch.nn.functional, func_name): + raise RuntimeError('For now, we only support calling independent functions from `torch.nn.functional`, ' + f'{func_name} is not in it.') + return f'{output} = F.{func_name}({", ".join(inputs)})' + +class PrimConstant(PyTorchOperation): + _ori_type_name = ['prim::Constant'] + def to_forward_code(self, field: str, output: str, inputs: List[str], inputs_value: List[Any] = None) -> str: + # TODO: refactor this part, maybe we can remove the code gen of prim::Constant + # TODO: deal with all the types + if self.parameters['type'] == 'None': + return f'{output} = None' + elif self.parameters['type'] in ('int', 'float', 'bool', 'int[]'): + return f'{output} = {self.parameters["value"]}' + elif self.parameters['type'] == 'str': + str_val = self.parameters["value"] + return f'{output} = "{str_val}"' + elif self.parameters['type'] == 'Device': + value = self.parameters['value'] + return f'{output} = torch.device("{value}")' + elif self.parameters['type'] in ('dict', 'list', 'tuple'): + # TODO: prim::TupleIndex is not supported yet + return f'{output} = {repr(self.parameters["value"])}' + else: + raise RuntimeError(f'unsupported type of prim::Constant: {self.parameters["type"]}') + +class PrimListConstruct(PyTorchOperation): + _ori_type_name = ['prim::ListConstruct'] + def to_forward_code(self, field: str, output: str, inputs: List[str], inputs_value: List[Any] = None) -> str: + return f'{output} = [{", ".join(inputs)}]' + +class PrimListUnpack(PyTorchOperation): + _ori_type_name = ['prim::ListUnpack'] + def to_forward_code(self, field: str, output: str, inputs: List[str], inputs_value: List[Any] = None) -> str: + return f'{output} = {inputs[0]}' + +class PrimTupleConstruct(PyTorchOperation): + _ori_type_name = ['prim::TupleConstruct'] + def to_forward_code(self, field: str, output: str, inputs: List[str], inputs_value: List[Any] = None) -> str: + return f'{output} = ({", ".join(inputs)})' + +class PrimTupleUnpack(PyTorchOperation): + _ori_type_name = ['prim::TupleUnpack'] + def to_forward_code(self, field: str, output: str, inputs: List[str], inputs_value: List[Any] = None) -> str: + # have single output here, because the following code uses index to access the unpacked values + assert len(inputs) == 1 + return f'{output} = {inputs[0]}' + +class PrimGetAttr(PyTorchOperation): + _ori_type_name = ['prim::GetAttr'] + def to_forward_code(self, field: str, output: str, inputs: List[str], inputs_value: List[Any] = None) -> str: + if self.parameters['value'] is not None: + return f"{output} = {self.parameters['value']}" + else: + return f"{output} = {self.parameters['input']}.{self.parameters['name']}" + +class SimpleMember(PyTorchOperation): + _ori_type_name = ['prim::is_cuda', 'prim::data'] + def to_forward_code(self, field: str, output: str, inputs: List[str], inputs_value: List[Any] = None) -> str: + member_name = self.type.split('::')[-1] + return f'{output} = {inputs[0]}.{member_name}' + +class AtenContiguous(PyTorchOperation): + _ori_type_name = ['aten::contiguous'] + def to_forward_code(self, field: str, output: str, inputs: List[str], inputs_value: List[Any] = None) -> str: + # defined in pytorch/c10/core/MemoryFormat.h + assert inputs_value[1] in [0, 1, 2] + return f'{output} = {inputs[0]}.contiguous(memory_format={mem_format[inputs_value[1]]})' + +class AtenGetitem(PyTorchOperation): + _ori_type_name = ['aten::__getitem__'] + def to_forward_code(self, field: str, output: str, inputs: List[str], inputs_value: List[Any] = None) -> str: + assert len(inputs) == 2 + return f'{output} = {inputs[0]}[{inputs[1]}]' + +class AtenAppend(PyTorchOperation): + _ori_type_name = ['aten::append'] + def to_forward_code(self, field: str, output: str, inputs: List[str], inputs_value: List[Any] = None) -> str: + assert len(inputs) == 2 + return f'_, {output} = {inputs[0]}.append({inputs[1]}), {inputs[0]}' + +class MergedSlice(PyTorchOperation): + _ori_type_name = ['MergedSlice'] + def to_forward_code(self, field: str, output: str, inputs: List[str], inputs_value: List[Any] = None) -> str: + if (len(inputs) - 1) % 4 == 0: + slices = [] + dim = int((len(inputs) - 1) / 4) + for i in range(dim): + slices.append(f'{inputs[i*4+2]}:{inputs[i*4+3]}:{inputs[i*4+4]}') + slice_str = ','.join(slices) + return f'{output} = {inputs[0]}[{slice_str}]' + elif len(inputs) == 4: + # this case is for simple list + return f'{output} = {inputs[0]}[{inputs[1]}:{inputs[2]}:{inputs[3]}]' + else: + raise RuntimeError('Unsupported slice pattern') + +# the following Aten classes means these aten ops are not in torch.Tensor + +class AtenBool(PyTorchOperation): + _ori_type_name = ['aten::Bool'] + def to_forward_code(self, field: str, output: str, inputs: List[str], inputs_value: List[Any] = None) -> str: + return f'{output} = bool({inputs[0]})' + +class AtenNot(PyTorchOperation): + _ori_type_name = ['aten::__not__'] + def to_forward_code(self, field: str, output: str, inputs: List[str], inputs_value: List[Any] = None) -> str: + return f'{output} = not {inputs[0]}' + +class AtenCat(PyTorchOperation): + _ori_type_name = ['aten::cat'] + def to_forward_code(self, field: str, output: str, inputs: List[str], inputs_value: List[Any] = None) -> str: + assert len(inputs) == 2 + return f'{output} = torch.cat({inputs[0]}, dim={inputs[1]})' + +#==================================== + +class AtenTensors(PyTorchOperation): + _ori_type_name = ['aten::full', 'aten::full_like', 'aten::empty_like', + 'aten::ones_like', 'aten::zeros_like', 'aten::rand', + 'aten::randn', 'aten::scalar_tensor', 'aten::new_full', + 'aten::new_empty', 'aten::new_zeros', 'aten::arange', + 'aten::tensor', 'aten::ones', 'aten::zeros'] + + def to_forward_code(self, field: str, output: str, inputs: List[str], inputs_value: List[Any] = None) -> str: + schemas = torch._C._jit_get_schemas_for_operator(self.type) + # match number of inputs + overloaded_defs = [len(s.arguments) for s in schemas] + matched = overloaded_defs.index(len(inputs)) + args_list = [] + for idx, arg in enumerate(schemas[matched].arguments): + if arg.name == 'dtype': + arg_str = f'dtype={scalar_type_to_pytorch_type[inputs_value[idx]]}' if inputs_value[idx] is not None else '' + elif arg.name == 'layout': + if inputs_value[idx] is not None: + arg_str = f'layout=torch.strided' + print('Warning: only support `torch.strided` for now!!!') + else: + arg_str = '' + elif arg.name == 'device': + arg_str = f'device=torch.device({inputs[idx]})' if inputs_value[idx] is not None else '' + elif arg.name == 'memory_format': + arg_str = f'memory_format={mem_format[inputs_value[idx]]}' if inputs_value[idx] is not None else '' + elif arg.name == 'pin_memory': + # TODO: deal with this argument + continue + elif arg.name == 'requires_grad': + arg_str = f'requires_grad={inputs[idx]}' if inputs_value[idx] else '' + elif str(arg.type).startswith('Optional['): + arg_str = f'{arg.name}={inputs[idx]}' + else: + arg_str = f'{inputs[idx]}' + if arg_str != '': + args_list.append(arg_str) + op_name = self.type.split('::')[-1] + if hasattr(torch, op_name): + return f'{output} = torch.{op_name}({", ".join(args_list)})' + else: + return f'{output} = {inputs[0]}.{op_name}({", ".join(args_list[1:])})' + +#==================================== + +class AtenFloordiv(PyTorchOperation): + _ori_type_name = ['aten::floordiv'] + def to_forward_code(self, field: str, output: str, inputs: List[str], inputs_value: List[Any] = None) -> str: + return f'{output} = {inputs[0]} // {inputs[1]}' + +class AtenLen(PyTorchOperation): + _ori_type_name = ['aten::len'] + def to_forward_code(self, field: str, output: str, inputs: List[str], inputs_value: List[Any] = None) -> str: + return f'{output} = len({inputs[0]})' + +class AtenIntImplicit(PyTorchOperation): + _ori_type_name = ['aten::IntImplicit', 'aten::Float', 'aten::Int', 'aten::ScalarImplicit'] + def to_forward_code(self, field: str, output: str, inputs: List[str], inputs_value: List[Any] = None) -> str: + if self.type.endswith('Implicit'): + return f'{output} = {inputs[0]}' + elif self.type == 'aten::Int': + return f'{output} = int({inputs[0]})' + elif self.type == 'aten::Float': + return f'{output} = float({inputs[0]})' + +class AtenIndex(PyTorchOperation): + _ori_type_name = ['aten::index'] + def to_forward_code(self, field: str, output: str, inputs: List[str], inputs_value: List[Any] = None) -> str: + return f'{output} = {inputs[0]}[{inputs[1]}]' + +ManuallyChooseDef = { + 'aten::flatten': [('start_dim', 'int', '0'), ('end_dim', 'int', '-1')], + 'aten::split': [('split_size', 'int', 'None'), ('dim', 'int', '0')] +} + +TensorOpExceptions = { + 'aten::sub': lambda output, inputs: f'{output} = {inputs[0]} - {inputs[1]}', # example: x.size(1) - 3 + 'aten::add': lambda output, inputs: f'{output} = {inputs[0]} + {inputs[1]}' # example: input.shape[0] + 5 +} + +TorchOpExclude = ['aten::Size', 'aten::as_tensor', 'aten::device', + 'aten::manual_seed', 'aten::quantized_gru', 'aten::quantized_lstm', + 'aten::save', 'aten::tensor', 'aten::wait' +] + +def _hidden(name): + return name.startswith('_') and not name.startswith('__') + +def _emit_args(args): + # filter out the `out` argument here + return [(arg.name, str(arg.type), str(arg.default_value)) for arg in args] # if arg.name != 'out' + +def _get_tensor_ops(): + def is_tensor_method(schema): + if len(schema.arguments) == 0: + return False + self = schema.arguments[0] + if self.name != 'self': + return False + if not self.type.isSubtypeOf(torch._C.TensorType.get()): + return False + return True + + op_args = {} + # discover methods + for elem in dir(torch.Tensor): + if not _hidden(elem): + schemas = torch._C._jit_get_schemas_for_operator("aten::" + elem) + for schema in schemas: + if is_tensor_method(schema): + op_name = 'aten::' + elem + args = _emit_args(schema.arguments[1:]) + if op_name in op_args: + op_args[op_name].append(args) + else: + op_args[op_name] = [args] + + return op_args.keys(), op_args + +def _get_torch_ops(): + torch_op_args = {} + for mod in torch.jit._builtins._modules_containing_builtins: + name = mod.__name__ + if name == 'torch._C._nn': + continue + # only process 'torch.XXX' + for elem in dir(mod): + builtin = torch.jit._builtins._find_builtin(getattr(mod, elem)) + if builtin is not None: + schemas = torch._C._jit_get_schemas_for_operator(builtin) + for schema in schemas: + # remove _tan but not __and__ + if not _hidden(elem): + op_name = 'aten::' + elem + if len(schema.arguments) > 0 and schema.arguments[0].name == 'self': + continue + args = _emit_args(schema.arguments) + if op_name in torch_op_args: + torch_op_args[op_name].append(args) + else: + torch_op_args[op_name] = [args] + + return torch_op_args.keys(), torch_op_args + +def _get_torch_ops_exclude_tensor_ops(): + tensor_op_names, _ = _get_tensor_ops() + torch_op_names, torch_ops = _get_torch_ops() + + torch_exclude_ops = {} + for name in torch_op_names: + if name not in tensor_op_names: + if name not in TorchOpExclude: + # exclude the ops that are not in + # https://github.com/pytorch/pytorch/blob/master/aten/src/ATen/native/native_functions.yaml + torch_exclude_ops[name] = torch_ops[name] + + return torch_exclude_ops.keys(), torch_exclude_ops + +class TensorOps(PyTorchOperation): + """ + corresponding to _get_tensor_ops in torch.jit.supported_ops + """ + _ori_type_name, _op_args = _get_tensor_ops() + + comparison_ops = {'aten::eq': '==', 'aten::ne': '!=', 'aten::le': '<=', 'aten::ge': '>=', 'aten::lt': '<', 'aten::gt': '>'} + + @staticmethod + def _get_matched_args(_type, inputs): + def has_same_arg_name(matched): + concated_names = [] + for i, each in enumerate(matched): + name = ','.join([arg[0] for arg in each]) + concated_names.append(name) + for i in range(len(concated_names) - 1): + if concated_names[i] != concated_names[i+1]: + return False + return True + + overloaded_defs = TensorOps._op_args[_type] + matched = [] + for each in overloaded_defs: + # plus 1 because we skip the first argument when generating tensor op def + if len(each) + 1 == len(inputs): + matched.append(each) + if len(matched) == 1: + return matched[0] + elif len(matched) > 1: + # TODO: match with arg's type. manually choose for now + if has_same_arg_name(matched): + # return any one is okay + return matched[0] + elif _type in ManuallyChooseDef: + return ManuallyChooseDef[_type] + else: + raise RuntimeError(f'tensor op type {_type} has more than one matched: {matched}') + else: + if _type in TensorOpExceptions: + return None + raise RuntimeError(f'tensor op type {_type} has no matched') + + def to_forward_code(self, field: str, output: str, inputs: List[str], inputs_value: List[Any] = None) -> str: + # TODO: deal with conditional ops + if self.type in TensorOps.comparison_ops: + return f'{output} = ({inputs[0]} {TensorOps.comparison_ops[self.type]} {inputs[1]})' + matched_args = TensorOps._get_matched_args(self.type, inputs) + if matched_args is None: + return TensorOpExceptions[self.type](output, inputs) + op_name = self.type.split('::')[-1] + args_str = ', '.join([f'{name}={inputs[i+1]}' for i, (name, t, default) in enumerate(matched_args)]) + return f'{output} = {inputs[0]}.{op_name}({args_str})' + +class TorchOps(PyTorchOperation): + """ + corresponding to _get_nn_functional_ops in torch.jit.supported_ops + """ + _ori_type_name, _op_args = _get_torch_ops_exclude_tensor_ops() + # add 'aten::pixel_shuffle' + _op_args['aten::pixel_shuffle'] = [[('input', 'Tensor', 'None'), ('upscale_factor', 'Optional[int]', 'None')]] + _ori_type_name = _op_args.keys() + + @staticmethod + def _get_matched_args(_type, inputs): + def has_same_arg_name(matched): + concated_names = [] + for i, each in enumerate(matched): + name = ','.join([arg[0] for arg in each]) + concated_names.append(name) + for i in range(len(concated_names) - 1): + if concated_names[i] != concated_names[i+1]: + return False + return True + + overloaded_defs = TorchOps._op_args[_type] + matched = [] + for each in overloaded_defs: + if len(each) == len(inputs): + matched.append(each) + if len(matched) == 1: + return matched[0] + elif len(matched) > 1: + # TODO: match with arg's type. manually choose for now + if has_same_arg_name(matched): + # return any one is okay + return matched[0] + else: + raise RuntimeError(f'torch op type {_type} has more than one matched: {matched}') + else: + raise RuntimeError(f'torch op type {_type} has no matched') + + def to_forward_code(self, field: str, output: str, inputs: List[str], inputs_value: List[Any] = None) -> str: + matched_args = TorchOps._get_matched_args(self.type, inputs) + op_name = self.type.split('::')[-1] + args_str = ', '.join([f'{name}={inputs[i]}' if t.startswith('Optional[') else f'{inputs[i]}' \ + for i, (name, t, default) in enumerate(matched_args)]) + return f'{output} = torch.{op_name}({args_str})' + +class AtenAvgpool2d(PyTorchOperation): + # NOTE: it is not included in the above aten ops for unkown reason + _ori_type_name = ['aten::avg_pool2d'] + def to_forward_code(self, field: str, output: str, inputs: List[str], inputs_value: List[Any] = None) -> str: + return f'{output} = F.avg_pool2d({", ".join(inputs)})' \ No newline at end of file diff --git a/utils/third_party/nni_new/retiarii/serializer.py b/utils/third_party/nni_new/retiarii/serializer.py new file mode 100644 index 0000000000000000000000000000000000000000..efa78243bdb3df2a3edecbc4bb8bf0d77833df0d --- /dev/null +++ b/utils/third_party/nni_new/retiarii/serializer.py @@ -0,0 +1,166 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import abc +import functools +import inspect +import types +from typing import Any + +import json_tricks + +from .utils import get_importable_name, get_module_name, import_, reset_uid + + +def get_init_parameters_or_fail(obj, silently=False): + if hasattr(obj, '_init_parameters'): + return obj._init_parameters + elif silently: + return None + else: + raise ValueError(f'Object {obj} needs to be serializable but `_init_parameters` is not available. ' + 'If it is a built-in module (like Conv2d), please import it from retiarii.nn. ' + 'If it is a customized module, please to decorate it with @basic_unit. ' + 'For other complex objects (e.g., trainer, optimizer, dataset, dataloader), ' + 'try to use serialize or @serialize_cls.') + + +### This is a patch of json-tricks to make it more useful to us ### + + +def _serialize_class_instance_encode(obj, primitives=False): + assert not primitives, 'Encoding with primitives is not supported.' + try: # FIXME: raise error + if hasattr(obj, '__class__'): + return { + '__type__': get_importable_name(obj.__class__), + 'arguments': get_init_parameters_or_fail(obj) + } + except ValueError: + pass + return obj + + +def _serialize_class_instance_decode(obj): + if isinstance(obj, dict) and '__type__' in obj and 'arguments' in obj: + return import_(obj['__type__'])(**obj['arguments']) + return obj + + +def _type_encode(obj, primitives=False): + assert not primitives, 'Encoding with primitives is not supported.' + if isinstance(obj, type): + return {'__typename__': get_importable_name(obj, relocate_module=True)} + if isinstance(obj, (types.FunctionType, types.BuiltinFunctionType)): + # This is not reliable for cases like closure, `open`, or objects that is callable but not intended to be serialized. + # https://stackoverflow.com/questions/624926/how-do-i-detect-whether-a-python-variable-is-a-function + return {'__typename__': get_importable_name(obj, relocate_module=True)} + return obj + + +def _type_decode(obj): + if isinstance(obj, dict) and '__typename__' in obj: + return import_(obj['__typename__']) + return obj + + +json_loads = functools.partial(json_tricks.loads, extra_obj_pairs_hooks=[_serialize_class_instance_decode, _type_decode]) +json_dumps = functools.partial(json_tricks.dumps, extra_obj_encoders=[_serialize_class_instance_encode, _type_encode]) +json_load = functools.partial(json_tricks.load, extra_obj_pairs_hooks=[_serialize_class_instance_decode, _type_decode]) +json_dump = functools.partial(json_tricks.dump, extra_obj_encoders=[_serialize_class_instance_encode, _type_encode]) + +### End of json-tricks patch ### + + +class Translatable(abc.ABC): + """ + Inherit this class and implement ``translate`` when the inner class needs a different + parameter from the wrapper class in its init function. + """ + + @abc.abstractmethod + def _translate(self) -> Any: + pass + + +def _create_wrapper_cls(cls, store_init_parameters=True, reset_mutation_uid=False): + class wrapper(cls): + def __init__(self, *args, **kwargs): + if reset_mutation_uid: + reset_uid('mutation') + if store_init_parameters: + argname_list = list(inspect.signature(cls.__init__).parameters.keys())[1:] + full_args = {} + full_args.update(kwargs) + + assert len(args) <= len(argname_list), f'Length of {args} is greater than length of {argname_list}.' + for argname, value in zip(argname_list, args): + full_args[argname] = value + + # translate parameters + args = list(args) + for i, value in enumerate(args): + if isinstance(value, Translatable): + args[i] = value._translate() + for i, value in kwargs.items(): + if isinstance(value, Translatable): + kwargs[i] = value._translate() + + self._init_parameters = full_args + else: + self._init_parameters = {} + + super().__init__(*args, **kwargs) + + wrapper.__module__ = get_module_name(cls) + wrapper.__name__ = cls.__name__ + wrapper.__qualname__ = cls.__qualname__ + wrapper.__init__.__doc__ = cls.__init__.__doc__ + + return wrapper + + +def serialize_cls(cls): + """ + To create an serializable class. + """ + return _create_wrapper_cls(cls) + + +def transparent_serialize(cls): + """ + Wrap a module but does not record parameters. For internal use only. + """ + return _create_wrapper_cls(cls, store_init_parameters=False) + + +def serialize(cls, *args, **kwargs): + """ + To create an serializable instance inline without decorator. For example, + + .. code-block:: python + + self.op = serialize(MyCustomOp, hidden_units=128) + """ + return serialize_cls(cls)(*args, **kwargs) + + +def basic_unit(cls): + """ + To wrap a module as a basic unit, to stop it from parsing and make it mutate-able. + """ + import torch.nn as nn + assert issubclass(cls, nn.Module), 'When using @basic_unit, the class must be a subclass of nn.Module.' + return serialize_cls(cls) + + +def model_wrapper(cls): + """ + Wrap the model if you are using pure-python execution engine. + + The wrapper serves two purposes: + + 1. Capture the init parameters of python class so that it can be re-instantiated in another process. + 2. Reset uid in `mutation` namespace so that each model counts from zero. Can be useful in unittest and other multi-model scenarios. + """ + return _create_wrapper_cls(cls, reset_mutation_uid=True) diff --git a/utils/third_party/nni_new/retiarii/strategy/__init__.py b/utils/third_party/nni_new/retiarii/strategy/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..04511eaa69b85666e9ebc69b643bc0c74fe6467d --- /dev/null +++ b/utils/third_party/nni_new/retiarii/strategy/__init__.py @@ -0,0 +1,9 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .base import BaseStrategy +from .bruteforce import Random, GridSearch +from .evolution import RegularizedEvolution +from .tpe_strategy import TPEStrategy +from .local_debug_strategy import _LocalDebugStrategy +from .rl import PolicyBasedRL diff --git a/utils/third_party/nni_new/retiarii/strategy/_rl_impl.py b/utils/third_party/nni_new/retiarii/strategy/_rl_impl.py new file mode 100644 index 0000000000000000000000000000000000000000..7e038aab6984c5238549d4aefdd778c609a10bff --- /dev/null +++ b/utils/third_party/nni_new/retiarii/strategy/_rl_impl.py @@ -0,0 +1,161 @@ +# This file might cause import error for those who didn't install RL-related dependencies + +import logging +import threading +from multiprocessing.pool import ThreadPool + +import gym +import numpy as np +import torch +import torch.nn as nn + +from gym import spaces +from tianshou.data import to_torch +from tianshou.env.worker import EnvWorker + +from .utils import get_targeted_model +from ..graph import ModelStatus +from ..execution import submit_models, wait_models + + +_logger = logging.getLogger(__name__) +_thread_lock = threading.Lock() + + +class MultiThreadEnvWorker(EnvWorker): + def __init__(self, env_fn): + self.env = env_fn() + self.pool = ThreadPool(processes=1) + super().__init__(env_fn) + + def __getattr__(self, key): + return getattr(self.env, key) + + def reset(self): + return self.env.reset() + + @staticmethod + def wait(*args, **kwargs): + raise NotImplementedError('Async collect is not supported yet.') + + def send_action(self, action) -> None: + # self.result is actually a handle + self.result = self.pool.apply_async(self.env.step, (action,)) + + def get_result(self): + return self.result.get() + + def seed(self, seed): + super().seed(seed) + return self.env.seed(seed) + + def render(self, **kwargs): + return self.env.render(**kwargs) + + def close_env(self) -> None: + self.pool.terminate() + return self.env.close() + + +class ModelEvaluationEnv(gym.Env): + def __init__(self, base_model, mutators, search_space): + self.base_model = base_model + self.mutators = mutators + self.search_space = search_space + self.ss_keys = list(self.search_space.keys()) + self.action_dim = max(map(lambda v: len(v), self.search_space.values())) + self.num_steps = len(self.search_space) + + @property + def observation_space(self): + return spaces.Dict({ + 'action_history': spaces.MultiDiscrete([self.action_dim] * self.num_steps), + 'cur_step': spaces.Discrete(self.num_steps + 1), + 'action_dim': spaces.Discrete(self.action_dim + 1) + }) + + @property + def action_space(self): + return spaces.Discrete(self.action_dim) + + def reset(self): + self.action_history = np.zeros(self.num_steps, dtype=np.int32) + self.cur_step = 0 + self.sample = {} + return { + 'action_history': self.action_history, + 'cur_step': self.cur_step, + 'action_dim': len(self.search_space[self.ss_keys[self.cur_step]]) + } + + def step(self, action): + cur_key = self.ss_keys[self.cur_step] + assert action < len(self.search_space[cur_key]), \ + f'Current action {action} out of range {self.search_space[cur_key]}.' + self.action_history[self.cur_step] = action + self.sample[cur_key] = self.search_space[cur_key][action] + self.cur_step += 1 + obs = { + 'action_history': self.action_history, + 'cur_step': self.cur_step, + 'action_dim': len(self.search_space[self.ss_keys[self.cur_step]]) \ + if self.cur_step < self.num_steps else self.action_dim + } + if self.cur_step == self.num_steps: + with _thread_lock: + model = get_targeted_model(self.base_model, self.mutators, self.sample) + _logger.info(f'New model created: {self.sample}') + submit_models(model) + wait_models(model) + if model.status == ModelStatus.Failed: + return self.reset(), 0., False, {} + rew = model.metric + _logger.info(f'Model metric received as reward: {rew}') + return obs, rew, True, {} + else: + + return obs, 0., False, {} + + +class Preprocessor(nn.Module): + def __init__(self, obs_space, hidden_dim=64, num_layers=1): + super().__init__() + self.action_dim = obs_space['action_history'].nvec[0] + self.hidden_dim = hidden_dim + # first token is [SOS] + self.embedding = nn.Embedding(self.action_dim + 1, hidden_dim) + self.rnn = nn.LSTM(hidden_dim, hidden_dim, num_layers, batch_first=True) + + def forward(self, obs): + seq = nn.functional.pad(obs['action_history'] + 1, (1, 1)) # pad the start token and end token + # end token is used to avoid out-of-range of v_s_. Will not actually affect BP. + seq = self.embedding(seq.long()) + feature, _ = self.rnn(seq) + return feature[torch.arange(len(feature), device=feature.device), obs['cur_step'].long() + 1] + + +class Actor(nn.Module): + def __init__(self, action_space, preprocess): + super().__init__() + self.preprocess = preprocess + self.action_dim = action_space.n + self.linear = nn.Linear(self.preprocess.hidden_dim, self.action_dim) + + def forward(self, obs, **kwargs): + obs = to_torch(obs, device=self.linear.weight.device) + out = self.linear(self.preprocess(obs)) + # to take care of choices with different number of options + mask = torch.arange(self.action_dim).expand(len(out), self.action_dim) >= obs['action_dim'].unsqueeze(1) + out[mask.to(out.device)] = float('-inf') + return nn.functional.softmax(out, dim=-1), kwargs.get('state', None) + + +class Critic(nn.Module): + def __init__(self, preprocess): + super().__init__() + self.preprocess = preprocess + self.linear = nn.Linear(self.preprocess.hidden_dim, 1) + + def forward(self, obs, **kwargs): + obs = to_torch(obs, device=self.linear.weight.device) + return self.linear(self.preprocess(obs)).squeeze(-1) diff --git a/utils/third_party/nni_new/retiarii/strategy/base.py b/utils/third_party/nni_new/retiarii/strategy/base.py new file mode 100644 index 0000000000000000000000000000000000000000..894da87b784b901346424e503067bae8e276de16 --- /dev/null +++ b/utils/third_party/nni_new/retiarii/strategy/base.py @@ -0,0 +1,15 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import abc +from typing import List + +from ..graph import Model +from ..mutator import Mutator + + +class BaseStrategy(abc.ABC): + + @abc.abstractmethod + def run(self, base_model: Model, applied_mutators: List[Mutator]) -> None: + pass diff --git a/utils/third_party/nni_new/retiarii/strategy/bruteforce.py b/utils/third_party/nni_new/retiarii/strategy/bruteforce.py new file mode 100644 index 0000000000000000000000000000000000000000..971711f0d9ab83c6581cb660c7b1d4430c1d4316 --- /dev/null +++ b/utils/third_party/nni_new/retiarii/strategy/bruteforce.py @@ -0,0 +1,124 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import copy +import itertools +import logging +import random +import time +from typing import Any, Dict, List + +from .. import Sampler, submit_models, query_available_resources, budget_exhausted +from .base import BaseStrategy +from .utils import dry_run_for_search_space, get_targeted_model + +_logger = logging.getLogger(__name__) + + +def grid_generator(search_space: Dict[Any, List[Any]], shuffle=True): + keys = list(search_space.keys()) + search_space_values = copy.deepcopy(list(search_space.values())) + if shuffle: + for values in search_space_values: + random.shuffle(values) + for values in itertools.product(*search_space_values): + yield {key: value for key, value in zip(keys, values)} + + +def random_generator(search_space: Dict[Any, List[Any]], dedup=True, retries=500): + keys = list(search_space.keys()) + history = set() + search_space_values = copy.deepcopy(list(search_space.values())) + while True: + for retry_count in range(retries): + selected = [random.choice(v) for v in search_space_values] + if not dedup: + break + selected = tuple(selected) + if selected not in history: + history.add(selected) + break + if retry_count + 1 == retries: + _logger.debug('Random generation has run out of patience. There is nothing to search. Exiting.') + return + yield {key: value for key, value in zip(keys, selected)} + + +class GridSearch(BaseStrategy): + """ + Traverse the search space and try all the possible combinations one by one. + + Parameters + ---------- + shuffle : bool + Shuffle the order in a candidate list, so that they are tried in a random order. Default: true. + """ + + def __init__(self, shuffle=True): + self._polling_interval = 2. + self.shuffle = shuffle + + def run(self, base_model, applied_mutators): + search_space = dry_run_for_search_space(base_model, applied_mutators) + for sample in grid_generator(search_space, shuffle=self.shuffle): + _logger.debug('New model created. Waiting for resource. %s', str(sample)) + while query_available_resources() <= 0: + if budget_exhausted(): + return + time.sleep(self._polling_interval) + submit_models(get_targeted_model(base_model, applied_mutators, sample)) + + +class _RandomSampler(Sampler): + def choice(self, candidates, mutator, model, index): + return random.choice(candidates) + + +class Random(BaseStrategy): + """ + Random search on the search space. + + Parameters + ---------- + variational : bool + Do not dry run to get the full search space. Used when the search space has variational size or candidates. Default: false. + dedup : bool + Do not try the same configuration twice. When variational is true, deduplication is not supported. Default: true. + """ + + def __init__(self, variational=False, dedup=True): + self.variational = variational + self.dedup = dedup + if variational and dedup: + raise ValueError('Dedup is not supported in variational mode.') + self.random_sampler = _RandomSampler() + self._polling_interval = 2. + + def run(self, base_model, applied_mutators): + if self.variational: + _logger.info('Random search running in variational mode.') + sampler = _RandomSampler() + for mutator in applied_mutators: + mutator.bind_sampler(sampler) + while True: + avail_resource = query_available_resources() + if avail_resource > 0: + model = base_model + for mutator in applied_mutators: + model = mutator.apply(model) + _logger.debug('New model created. Applied mutators are: %s', str(applied_mutators)) + submit_models(model) + elif budget_exhausted(): + break + else: + time.sleep(self._polling_interval) + else: + _logger.info('Random search running in fixed size mode. Dedup: %s.', 'on' if self.dedup else 'off') + search_space = dry_run_for_search_space(base_model, applied_mutators) + for sample in random_generator(search_space, dedup=self.dedup): + _logger.debug('New model created. Waiting for resource. %s', str(sample)) + while query_available_resources() <= 0: + if budget_exhausted(): + return + time.sleep(self._polling_interval) + submit_models(get_targeted_model(base_model, applied_mutators, sample)) diff --git a/utils/third_party/nni_new/retiarii/strategy/evolution.py b/utils/third_party/nni_new/retiarii/strategy/evolution.py new file mode 100644 index 0000000000000000000000000000000000000000..ca13d32da1159ece2cc03832932bf29252d689f6 --- /dev/null +++ b/utils/third_party/nni_new/retiarii/strategy/evolution.py @@ -0,0 +1,161 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import collections +import dataclasses +import logging +import random +import time + +from ..execution import query_available_resources, submit_models +from ..graph import ModelStatus +from .base import BaseStrategy +from .utils import dry_run_for_search_space, get_targeted_model + + +_logger = logging.getLogger(__name__) + + +@dataclasses.dataclass +class Individual: + """ + A class that represents an individual. + Holds two attributes, where ``x`` is the model and ``y`` is the metric (e.g., accuracy). + """ + x: dict + y: float + + +class RegularizedEvolution(BaseStrategy): + """ + Algorithm for regularized evolution (i.e. aging evolution). + Follows "Algorithm 1" in Real et al. "Regularized Evolution for Image Classifier Architecture Search". + + Parameters + ---------- + optimize_mode : str + Can be one of "maximize" and "minimize". Default: maximize. + population_size : int + The number of individuals to keep in the population. Default: 100. + cycles : int + The number of cycles (trials) the algorithm should run for. Default: 20000. + sample_size : int + The number of individuals that should participate in each tournament. Default: 25. + mutation_prob : float + Probability that mutation happens in each dim. Default: 0.05 + on_failure : str + Can be one of "ignore" and "worst". If "ignore", simply give up the model and find a new one. + If "worst", mark the model as -inf (if maximize, inf if minimize), so that the algorithm "learns" to avoid such model. + Default: ignore. + """ + + def __init__(self, optimize_mode='maximize', population_size=100, sample_size=25, cycles=20000, + mutation_prob=0.05, on_failure='ignore'): + assert optimize_mode in ['maximize', 'minimize'] + assert on_failure in ['ignore', 'worst'] + assert sample_size < population_size + self.optimize_mode = optimize_mode + self.population_size = population_size + self.sample_size = sample_size + self.cycles = cycles + self.mutation_prob = mutation_prob + self.on_failure = on_failure + + self._worst = float('-inf') if self.optimize_mode == 'maximize' else float('inf') + + self._success_count = 0 + self._population = collections.deque() + self._running_models = [] + self._polling_interval = 2. + + def random(self, search_space): + return {k: random.choice(v) for k, v in search_space.items()} + + def mutate(self, parent, search_space): + child = {} + for k, v in parent.items(): + if random.uniform(0, 1) < self.mutation_prob: + # NOTE: we do not exclude the original choice here for simplicity, + # which is slightly different from the original paper. + child[k] = random.choice(search_space[k]) + else: + child[k] = v + return child + + def best_parent(self): + samples = [p for p in self._population] # copy population + random.shuffle(samples) + samples = list(samples)[:self.sample_size] + if self.optimize_mode == 'maximize': + parent = max(samples, key=lambda sample: sample.y) + else: + parent = min(samples, key=lambda sample: sample.y) + return parent.x + + def run(self, base_model, applied_mutators): + search_space = dry_run_for_search_space(base_model, applied_mutators) + # Run the first population regardless concurrency + _logger.info('Initializing the first population.') + while len(self._population) + len(self._running_models) <= self.population_size: + # try to submit new models + while len(self._population) + len(self._running_models) < self.population_size: + config = self.random(search_space) + self._submit_config(config, base_model, applied_mutators) + # collect results + self._move_succeeded_models_to_population() + self._remove_failed_models_from_running_list() + time.sleep(self._polling_interval) + + if len(self._population) >= self.population_size: + break + + # Resource-aware mutation of models + _logger.info('Running mutations.') + while self._success_count + len(self._running_models) <= self.cycles: + # try to submit new models + while query_available_resources() > 0 and self._success_count + len(self._running_models) < self.cycles: + config = self.mutate(self.best_parent(), search_space) + self._submit_config(config, base_model, applied_mutators) + # collect results + self._move_succeeded_models_to_population() + self._remove_failed_models_from_running_list() + time.sleep(self._polling_interval) + + if self._success_count >= self.cycles: + break + + def _submit_config(self, config, base_model, mutators): + _logger.debug('Model submitted to running queue: %s', config) + model = get_targeted_model(base_model, mutators, config) + submit_models(model) + self._running_models.append((config, model)) + return model + + def _move_succeeded_models_to_population(self): + completed_indices = [] + for i, (config, model) in enumerate(self._running_models): + metric = None + if self.on_failure == 'worst' and model.status == ModelStatus.Failed: + metric = self._worst + elif model.status == ModelStatus.Trained: + metric = model.metric + if metric is not None: + individual = Individual(config, metric) + _logger.debug('Individual created: %s', str(individual)) + self._population.append(individual) + if len(self._population) > self.population_size: + self._population.popleft() + completed_indices.append(i) + for i in completed_indices[::-1]: + # delete from end to start so that the index number will not be affected. + self._success_count += 1 + self._running_models.pop(i) + + def _remove_failed_models_from_running_list(self): + # This is only done when on_failure policy is set to "ignore". + # Otherwise, failed models will be treated as inf when processed. + if self.on_failure == 'ignore': + number_of_failed_models = len([g for g in self._running_models if g[1].status == ModelStatus.Failed]) + self._running_models = [g for g in self._running_models if g[1].status != ModelStatus.Failed] + if number_of_failed_models > 0: + _logger.info('%d failed models are ignored. Will retry.', number_of_failed_models) diff --git a/utils/third_party/nni_new/retiarii/strategy/local_debug_strategy.py b/utils/third_party/nni_new/retiarii/strategy/local_debug_strategy.py new file mode 100644 index 0000000000000000000000000000000000000000..743d6b2fc62b113c25c8cae748a490f219978deb --- /dev/null +++ b/utils/third_party/nni_new/retiarii/strategy/local_debug_strategy.py @@ -0,0 +1,44 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +import os +import random +import string + +from .. import Sampler, codegen, utils +from ..execution.base import BaseGraphData +from .base import BaseStrategy + +_logger = logging.getLogger(__name__) + +class ChooseFirstSampler(Sampler): + def choice(self, candidates, mutator, model, index): + return candidates[0] + +class _LocalDebugStrategy(BaseStrategy): + """ + This class is supposed to be used internally, for debugging trial mutation + """ + + def run_one_model(self, model): + graph_data = BaseGraphData(codegen.model_to_pytorch_script(model), model.evaluator) + random_str = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(6)) + file_name = f'_generated_model/{random_str}.py' + os.makedirs(os.path.dirname(file_name), exist_ok=True) + with open(file_name, 'w') as f: + f.write(graph_data.model_script) + model_cls = utils.import_(f'_generated_model.{random_str}._model') + graph_data.evaluator._execute(model_cls) + os.remove(file_name) + + def run(self, base_model, applied_mutators): + _logger.info('local debug strategy has been started.') + model = base_model + _logger.debug('New model created. Applied mutators: %s', str(applied_mutators)) + choose_first_sampler = ChooseFirstSampler() + for mutator in applied_mutators: + mutator.bind_sampler(choose_first_sampler) + model = mutator.apply(model) + # directly run models + self.run_one_model(model) diff --git a/utils/third_party/nni_new/retiarii/strategy/rl.py b/utils/third_party/nni_new/retiarii/strategy/rl.py new file mode 100644 index 0000000000000000000000000000000000000000..23a05ed1ed9ad20d7421b82c0e820482c71aef34 --- /dev/null +++ b/utils/third_party/nni_new/retiarii/strategy/rl.py @@ -0,0 +1,78 @@ +import logging +from typing import Optional, Callable + +from .base import BaseStrategy +from .utils import dry_run_for_search_space +from ..execution import query_available_resources + +try: + has_tianshou = True + import torch + from tianshou.data import Collector, VectorReplayBuffer + from tianshou.env import BaseVectorEnv + from tianshou.policy import BasePolicy, PPOPolicy # pylint: disable=unused-import + from ._rl_impl import ModelEvaluationEnv, MultiThreadEnvWorker, Preprocessor, Actor, Critic +except ImportError: + has_tianshou = False + + +_logger = logging.getLogger(__name__) + + +class PolicyBasedRL(BaseStrategy): + """ + Algorithm for policy-based reinforcement learning. + This is a wrapper of algorithms provided in tianshou (PPO by default), + and can be easily customized with other algorithms that inherit ``BasePolicy`` (e.g., REINFORCE [1]_). + + Parameters + ---------- + max_collect : int + How many times collector runs to collect trials for RL. Default 100. + trial_per_collect : int + How many trials (trajectories) each time collector collects. + After each collect, trainer will sample batch from replay buffer and do the update. Default: 20. + policy_fn : function + Takes ``ModelEvaluationEnv`` as input and return a policy. See ``_default_policy_fn`` for an example. + + References + ---------- + + .. [1] Barret Zoph and Quoc V. Le, "Neural Architecture Search with Reinforcement Learning". + https://arxiv.org/abs/1611.01578 + """ + + def __init__(self, max_collect: int = 100, trial_per_collect = 20, + policy_fn: Optional[Callable[['ModelEvaluationEnv'], 'BasePolicy']] = None): + if not has_tianshou: + raise ImportError('`tianshou` is required to run RL-based strategy. ' + 'Please use "pip install tianshou" to install it beforehand.') + + self.policy_fn = policy_fn or self._default_policy_fn + self.max_collect = max_collect + self.trial_per_collect = trial_per_collect + + @staticmethod + def _default_policy_fn(env): + net = Preprocessor(env.observation_space) + actor = Actor(env.action_space, net) + critic = Critic(net) + optim = torch.optim.Adam(set(actor.parameters()).union(critic.parameters()), lr=1e-4) + return PPOPolicy(actor, critic, optim, torch.distributions.Categorical, + discount_factor=1., action_space=env.action_space) + + def run(self, base_model, applied_mutators): + search_space = dry_run_for_search_space(base_model, applied_mutators) + concurrency = query_available_resources() + + env_fn = lambda: ModelEvaluationEnv(base_model, applied_mutators, search_space) + policy = self.policy_fn(env_fn()) + + env = BaseVectorEnv([env_fn for _ in range(concurrency)], MultiThreadEnvWorker) + collector = Collector(policy, env, VectorReplayBuffer(20000, len(env))) + + for cur_collect in range(1, self.max_collect + 1): + _logger.info('Collect [%d] Running...', cur_collect) + result = collector.collect(n_episode=self.trial_per_collect) + _logger.info('Collect [%d] Result: %s', cur_collect, str(result)) + policy.update(0, collector.buffer, batch_size=64, repeat=5) diff --git a/utils/third_party/nni_new/retiarii/strategy/tpe_strategy.py b/utils/third_party/nni_new/retiarii/strategy/tpe_strategy.py new file mode 100644 index 0000000000000000000000000000000000000000..7f55ad302e6ecfe356a242ae92db04b74ad30e27 --- /dev/null +++ b/utils/third_party/nni_new/retiarii/strategy/tpe_strategy.py @@ -0,0 +1,94 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +import time + +from nni.algorithms.hpo.hyperopt_tuner import HyperoptTuner + +from .. import Sampler, submit_models, query_available_resources, is_stopped_exec, budget_exhausted +from .base import BaseStrategy + +_logger = logging.getLogger(__name__) + + +class TPESampler(Sampler): + def __init__(self, optimize_mode='minimize'): + self.tpe_tuner = HyperoptTuner('tpe', optimize_mode) + self.cur_sample = None + self.index = None + self.total_parameters = {} + + def update_sample_space(self, sample_space): + search_space = {} + for i, each in enumerate(sample_space): + search_space[str(i)] = {'_type': 'choice', '_value': each} + self.tpe_tuner.update_search_space(search_space) + + def generate_samples(self, model_id): + self.cur_sample = self.tpe_tuner.generate_parameters(model_id) + self.total_parameters[model_id] = self.cur_sample + self.index = 0 + + def receive_result(self, model_id, result): + self.tpe_tuner.receive_trial_result(model_id, self.total_parameters[model_id], result) + + def choice(self, candidates, mutator, model, index): + chosen = self.cur_sample[str(self.index)] + self.index += 1 + return chosen + + +class TPEStrategy(BaseStrategy): + """ + The Tree-structured Parzen Estimator (TPE) [bergstrahpo]_ is a sequential model-based optimization (SMBO) approach. + SMBO methods sequentially construct models to approximate the performance of hyperparameters based on historical measurements, + and then subsequently choose new hyperparameters to test based on this model. + + References + ---------- + + .. [bergstrahpo] Bergstra et al., "Algorithms for Hyper-Parameter Optimization". + https://papers.nips.cc/paper/4443-algorithms-for-hyper-parameter-optimization.pdf + """ + + def __init__(self): + self.tpe_sampler = TPESampler() + self.model_id = 0 + self.running_models = {} + + def run(self, base_model, applied_mutators): + sample_space = [] + new_model = base_model + for mutator in applied_mutators: + recorded_candidates, new_model = mutator.dry_run(new_model) + sample_space.extend(recorded_candidates) + self.tpe_sampler.update_sample_space(sample_space) + + _logger.info('TPE strategy has been started.') + while not budget_exhausted(): + avail_resource = query_available_resources() + if avail_resource > 0: + model = base_model + _logger.debug('New model created. Applied mutators: %s', str(applied_mutators)) + self.tpe_sampler.generate_samples(self.model_id) + for mutator in applied_mutators: + mutator.bind_sampler(self.tpe_sampler) + model = mutator.apply(model) + # run models + submit_models(model) + self.running_models[self.model_id] = model + self.model_id += 1 + else: + time.sleep(2) + + _logger.debug('num of running models: %d', len(self.running_models)) + to_be_deleted = [] + for _id, _model in self.running_models.items(): + if is_stopped_exec(_model): + if _model.metric is not None: + self.tpe_sampler.receive_result(_id, _model.metric) + _logger.debug('tpe receive results: %d, %s', _id, _model.metric) + to_be_deleted.append(_id) + for _id in to_be_deleted: + del self.running_models[_id] diff --git a/utils/third_party/nni_new/retiarii/strategy/utils.py b/utils/third_party/nni_new/retiarii/strategy/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..8b687c4c211715fd38d361ba9af4f9cbbf573c07 --- /dev/null +++ b/utils/third_party/nni_new/retiarii/strategy/utils.py @@ -0,0 +1,32 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import collections +from typing import Dict, Any, List +from ..graph import Model +from ..mutator import Mutator, Sampler + + +class _FixedSampler(Sampler): + def __init__(self, sample): + self.sample = sample + + def choice(self, candidates, mutator, model, index): + return self.sample[(mutator, index)] + + +def dry_run_for_search_space(model: Model, mutators: List[Mutator]) -> Dict[Any, List[Any]]: + search_space = collections.OrderedDict() + for mutator in mutators: + recorded_candidates, model = mutator.dry_run(model) + for i, candidates in enumerate(recorded_candidates): + search_space[(mutator, i)] = candidates + return search_space + + +def get_targeted_model(base_model: Model, mutators: List[Mutator], sample: dict) -> Model: + sampler = _FixedSampler(sample) + model = base_model + for mutator in mutators: + model = mutator.bind_sampler(sampler).apply(model) + return model diff --git a/utils/third_party/nni_new/retiarii/trial_entry.py b/utils/third_party/nni_new/retiarii/trial_entry.py new file mode 100644 index 0000000000000000000000000000000000000000..7d805dd47f3b2294c24df6895974553ccc3397a8 --- /dev/null +++ b/utils/third_party/nni_new/retiarii/trial_entry.py @@ -0,0 +1,25 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +Entrypoint for trials. + +Assuming execution engine is BaseExecutionEngine. +""" +import argparse + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('exec', choices=['base', 'py', 'cgo']) + args = parser.parse_args() + if args.exec == 'base': + from .execution.base import BaseExecutionEngine + engine = BaseExecutionEngine + elif args.exec == 'cgo': + from .execution.cgo_engine import CGOExecutionEngine + engine = CGOExecutionEngine + elif args.exec == 'py': + from .execution.python import PurePythonExecutionEngine + engine = PurePythonExecutionEngine + engine.trial_execute_graph() diff --git a/utils/third_party/nni_new/retiarii/utils.py b/utils/third_party/nni_new/retiarii/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..c8b02dfba46020aaa8da4bf451817894be95278e --- /dev/null +++ b/utils/third_party/nni_new/retiarii/utils.py @@ -0,0 +1,106 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import inspect +import warnings +from collections import defaultdict +from typing import Any, List, Dict +from pathlib import Path + + +def import_(target: str, allow_none: bool = False) -> Any: + if target is None: + return None + path, identifier = target.rsplit('.', 1) + module = __import__(path, globals(), locals(), [identifier]) + return getattr(module, identifier) + + +def version_larger_equal(a: str, b: str) -> bool: + # TODO: refactor later + a = a.split('+')[0] + b = b.split('+')[0] + return tuple(map(int, a.split('.'))) >= tuple(map(int, b.split('.'))) + + +_last_uid = defaultdict(int) + + +def uid(namespace: str = 'default') -> int: + _last_uid[namespace] += 1 + return _last_uid[namespace] + + +def reset_uid(namespace: str = 'default') -> None: + _last_uid[namespace] = 0 + + +def get_module_name(cls_or_func): + module_name = cls_or_func.__module__ + if module_name == '__main__': + # infer the module name with inspect + for frm in inspect.stack(): + if inspect.getmodule(frm[0]).__name__ == '__main__': + # main module found + main_file_path = Path(inspect.getsourcefile(frm[0])) + if main_file_path.parents[0] != Path('.'): + raise RuntimeError(f'You are using "{main_file_path}" to launch your experiment, ' + f'please launch the experiment under the directory where "{main_file_path.name}" is located.') + module_name = main_file_path.stem + break + if module_name == '__main__': + warnings.warn('Callstack exhausted but main module still not found. This will probably cause issues that the ' + 'function/class cannot be imported.') + + # NOTE: this is hacky. As torchscript retrieves LSTM's source code to do something. + # to make LSTM's source code can be found, we should assign original LSTM's __module__ to + # the wrapped LSTM's __module__ + # TODO: find out all the modules that have the same requirement as LSTM + if f'{cls_or_func.__module__}.{cls_or_func.__name__}' == 'torch.nn.modules.rnn.LSTM': + module_name = cls_or_func.__module__ + + return module_name + + +def get_importable_name(cls, relocate_module=False): + module_name = get_module_name(cls) if relocate_module else cls.__module__ + return module_name + '.' + cls.__name__ + + +class ContextStack: + """ + This is to maintain a globally-accessible context envinronment that is visible to everywhere. + + Use ``with ContextStack(namespace, value):`` to initiate, and use ``get_current_context(namespace)`` to + get the corresponding value in the namespace. + """ + + _stack: Dict[str, List[Any]] = defaultdict(list) + + def __init__(self, key: str, value: Any): + self.key = key + self.value = value + + def __enter__(self): + self.push(self.key, self.value) + return self + + def __exit__(self, *args, **kwargs): + self.pop(self.key) + + @classmethod + def push(cls, key: str, value: Any): + cls._stack[key].append(value) + + @classmethod + def pop(cls, key: str) -> None: + cls._stack[key].pop() + + @classmethod + def top(cls, key: str) -> Any: + assert cls._stack[key], 'Context is empty.' + return cls._stack[key][-1] + + +def get_current_context(key: str) -> Any: + return ContextStack.top(key) diff --git a/utils/third_party/nni_new/runtime/__pycache__/env_vars.cpython-38.pyc b/utils/third_party/nni_new/runtime/__pycache__/env_vars.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..66fa9f35ec0d1c5fdfe17748cffbfff997b71824 Binary files /dev/null and b/utils/third_party/nni_new/runtime/__pycache__/env_vars.cpython-38.pyc differ diff --git a/utils/third_party/nni_new/runtime/__pycache__/log.cpython-38.pyc b/utils/third_party/nni_new/runtime/__pycache__/log.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b1000fbcd685854ff850475cd279f09987b9f049 Binary files /dev/null and b/utils/third_party/nni_new/runtime/__pycache__/log.cpython-38.pyc differ diff --git a/utils/third_party/nni_new/runtime/common.py b/utils/third_party/nni_new/runtime/common.py new file mode 100644 index 0000000000000000000000000000000000000000..537a35b55c697dd722f20747e6d2b6c19f0fbc54 --- /dev/null +++ b/utils/third_party/nni_new/runtime/common.py @@ -0,0 +1,19 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +_multi_thread = False +_multi_phase = False + +def enable_multi_thread(): + global _multi_thread + _multi_thread = True + +def multi_thread_enabled(): + return _multi_thread + +def enable_multi_phase(): + global _multi_phase + _multi_phase = True + +def multi_phase_enabled(): + return _multi_phase diff --git a/utils/third_party/nni_new/runtime/config.py b/utils/third_party/nni_new/runtime/config.py new file mode 100644 index 0000000000000000000000000000000000000000..8f1c8e2e10fb690de30dc9d8584905e6a33255fb --- /dev/null +++ b/utils/third_party/nni_new/runtime/config.py @@ -0,0 +1,34 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +from pathlib import Path +import shutil +import sys + +import nni + +def get_config_directory() -> Path: + """ + Get NNI config directory. + Create it if not exist. + """ + if sys.prefix != sys.base_prefix or Path(sys.prefix, 'conda-meta').is_dir(): + config_dir = Path(sys.prefix, 'nni') + elif sys.platform == 'win32': + config_dir = Path(os.environ['APPDATA'], 'nni') + else: + config_dir = Path.home() / '.config/nni' + config_dir.mkdir(parents=True, exist_ok=True) + return config_dir + +def get_config_file(name: str) -> Path: + """ + Get an NNI config file. + Copy from `nni/runtime/default_config` if not exist. + """ + config_file = get_config_directory() / name + if not config_file.exists(): + default = Path(nni.__path__[0], 'runtime/default_config', name) + shutil.copyfile(default, config_file) + return config_file diff --git a/utils/third_party/nni_new/runtime/default_config/registered_algorithms.yml b/utils/third_party/nni_new/runtime/default_config/registered_algorithms.yml new file mode 100644 index 0000000000000000000000000000000000000000..04c87e8de8ad99f71ee452aeab52d465d73fbf2f --- /dev/null +++ b/utils/third_party/nni_new/runtime/default_config/registered_algorithms.yml @@ -0,0 +1,82 @@ +advisors: +- builtinName: Hyperband + classArgsValidator: nni.algorithms.hpo.hyperband_advisor.HyperbandClassArgsValidator + className: nni.algorithms.hpo.hyperband_advisor.Hyperband + source: nni +- builtinName: BOHB + classArgsValidator: nni.algorithms.hpo.bohb_advisor.BOHBClassArgsValidator + className: nni.algorithms.hpo.bohb_advisor.BOHB + source: nni +assessors: +- builtinName: Medianstop + classArgsValidator: nni.algorithms.hpo.medianstop_assessor.MedianstopClassArgsValidator + className: nni.algorithms.hpo.medianstop_assessor.MedianstopAssessor + source: nni +- builtinName: Curvefitting + classArgsValidator: nni.algorithms.hpo.curvefitting_assessor.CurvefittingClassArgsValidator + className: nni.algorithms.hpo.curvefitting_assessor.CurvefittingAssessor + source: nni +tuners: +- builtinName: PPOTuner + classArgsValidator: nni.algorithms.hpo.ppo_tuner.PPOClassArgsValidator + className: nni.algorithms.hpo.ppo_tuner.PPOTuner + source: nni +- builtinName: SMAC + classArgsValidator: nni.algorithms.hpo.smac_tuner.SMACClassArgsValidator + className: nni.algorithms.hpo.smac_tuner.SMACTuner + source: nni +- builtinName: TPE + classArgs: + algorithm_name: tpe + classArgsValidator: nni.algorithms.hpo.hyperopt_tuner.HyperoptClassArgsValidator + className: nni.algorithms.hpo.hyperopt_tuner.HyperoptTuner + source: nni +- acceptClassArgs: false + builtinName: Random + classArgs: + algorithm_name: random_search + classArgsValidator: nni.algorithms.hpo.hyperopt_tuner.HyperoptClassArgsValidator + className: nni.algorithms.hpo.hyperopt_tuner.HyperoptTuner + source: nni +- builtinName: Anneal + classArgs: + algorithm_name: anneal + classArgsValidator: nni.algorithms.hpo.hyperopt_tuner.HyperoptClassArgsValidator + className: nni.algorithms.hpo.hyperopt_tuner.HyperoptTuner + source: nni +- builtinName: Evolution + classArgsValidator: nni.algorithms.hpo.evolution_tuner.EvolutionClassArgsValidator + className: nni.algorithms.hpo.evolution_tuner.EvolutionTuner + source: nni +- acceptClassArgs: false + builtinName: BatchTuner + className: nni.algorithms.hpo.batch_tuner.BatchTuner + source: nni +- acceptClassArgs: false + builtinName: GridSearch + className: nni.algorithms.hpo.gridsearch_tuner.GridSearchTuner + source: nni +- builtinName: NetworkMorphism + classArgsValidator: nni.algorithms.hpo.networkmorphism_tuner.NetworkMorphismClassArgsValidator + className: nni.algorithms.hpo.networkmorphism_tuner.NetworkMorphismTuner + source: nni +- builtinName: MetisTuner + classArgsValidator: nni.algorithms.hpo.metis_tuner.MetisClassArgsValidator + className: nni.algorithms.hpo.metis_tuner.MetisTuner + source: nni +- builtinName: GPTuner + classArgsValidator: nni.algorithms.hpo.gp_tuner.GPClassArgsValidator + className: nni.algorithms.hpo.gp_tuner.GPTuner + source: nni +- builtinName: PBTTuner + classArgsValidator: nni.algorithms.hpo.pbt_tuner.PBTClassArgsValidator + className: nni.algorithms.hpo.pbt_tuner.PBTTuner + source: nni +- builtinName: RegularizedEvolutionTuner + classArgsValidator: nni.algorithms.hpo.regularized_evolution_tuner.EvolutionClassArgsValidator + className: nni.algorithms.hpo.regularized_evolution_tuner.RegularizedEvolutionTuner + source: nni +- builtinName: DNGOTuner + classArgsValidator: nni.algorithms.hpo.dngo_tuner.DNGOClassArgsValidator + className: nni.algorithms.hpo.dngo_tuner.DNGOTuner + source: nni diff --git a/utils/third_party/nni_new/runtime/default_config/training_services.json b/utils/third_party/nni_new/runtime/default_config/training_services.json new file mode 100644 index 0000000000000000000000000000000000000000..0967ef424bce6791893e9a57bb952f80fd536e93 --- /dev/null +++ b/utils/third_party/nni_new/runtime/default_config/training_services.json @@ -0,0 +1 @@ +{} diff --git a/utils/third_party/nni_new/runtime/env_vars.py b/utils/third_party/nni_new/runtime/env_vars.py new file mode 100644 index 0000000000000000000000000000000000000000..810ab2f4f6db96d4ab195d44643f1deaab5f6528 --- /dev/null +++ b/utils/third_party/nni_new/runtime/env_vars.py @@ -0,0 +1,34 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +from collections import namedtuple + + +_trial_env_var_names = [ + 'NNI_PLATFORM', + 'NNI_EXP_ID', + 'NNI_TRIAL_JOB_ID', + 'NNI_SYS_DIR', + 'NNI_OUTPUT_DIR', + 'NNI_TRIAL_SEQ_ID', + 'MULTI_PHASE', + 'REUSE_MODE' +] + +_dispatcher_env_var_names = [ + 'SDK_PROCESS', + 'NNI_MODE', + 'NNI_CHECKPOINT_DIRECTORY', + 'NNI_LOG_DIRECTORY', + 'NNI_LOG_LEVEL', + 'NNI_INCLUDE_INTERMEDIATE_RESULTS' +] + +def _load_env_vars(env_var_names): + env_var_dict = {k: os.environ.get(k) for k in env_var_names} + return namedtuple('EnvVars', env_var_names)(**env_var_dict) + +trial_env_vars = _load_env_vars(_trial_env_var_names) + +dispatcher_env_vars = _load_env_vars(_dispatcher_env_var_names) diff --git a/utils/third_party/nni_new/runtime/log.py b/utils/third_party/nni_new/runtime/log.py new file mode 100644 index 0000000000000000000000000000000000000000..19aa4e061b00773ab3d84025d44e755c866e94db --- /dev/null +++ b/utils/third_party/nni_new/runtime/log.py @@ -0,0 +1,172 @@ +from datetime import datetime +from io import TextIOBase +import logging +from logging import FileHandler, Formatter, Handler, StreamHandler +from pathlib import Path +import sys +import time +from typing import Optional + +import colorama + +from .env_vars import dispatcher_env_vars, trial_env_vars + + +handlers = {} + +log_format = '[%(asctime)s] %(levelname)s (%(name)s/%(threadName)s) %(message)s' +time_format = '%Y-%m-%d %H:%M:%S' +formatter = Formatter(log_format, time_format) + + +def init_logger() -> None: + """ + This function will (and should only) get invoked on the first time of importing nni (no matter which submodule). + It will try to detect the running environment and setup logger accordingly. + + The detection should work in most cases but for `nnictl` and `nni.experiment`. + They will be identified as "standalone" mode and must configure the logger by themselves. + """ + colorama.init() + + if dispatcher_env_vars.SDK_PROCESS == 'dispatcher': + _init_logger_dispatcher() + return + + trial_platform = trial_env_vars.NNI_PLATFORM + + if trial_platform == 'unittest': + return + + if trial_platform and not trial_env_vars.REUSE_MODE: + _init_logger_trial() + return + + _init_logger_standalone() + + logging.getLogger('filelock').setLevel(logging.WARNING) + +_exp_log_initialized = False + +def init_logger_experiment() -> None: + """ + Initialize logger for `nni.experiment.Experiment`. + + This function will get invoked after `init_logger()`. + """ + global _exp_log_initialized + if not _exp_log_initialized: + _exp_log_initialized = True + colorful_formatter = Formatter(log_format, time_format) + colorful_formatter.format = _colorful_format + handlers['_default_'].setFormatter(colorful_formatter) + +def start_experiment_log(experiment_id: str, log_directory: Path, debug: bool) -> None: + log_path = _prepare_log_dir(log_directory) / 'dispatcher.log' + log_level = logging.DEBUG if debug else logging.WARNING + _register_handler(FileHandler(log_path), log_level, experiment_id) + +def stop_experiment_log(experiment_id: str) -> None: + if experiment_id in handlers: + logging.getLogger().removeHandler(handlers.pop(experiment_id)) + + +def _init_logger_dispatcher() -> None: + log_level_map = { + 'fatal': logging.CRITICAL, + 'error': logging.ERROR, + 'warning': logging.WARNING, + 'info': logging.WARNING, + 'debug': logging.DEBUG, + 'trace': 0 + } + + log_path = _prepare_log_dir(dispatcher_env_vars.NNI_LOG_DIRECTORY) / 'dispatcher.log' + log_level = log_level_map.get(dispatcher_env_vars.NNI_LOG_LEVEL, logging.WARNING) + _register_handler(FileHandler(log_path), log_level) + + +def _init_logger_trial() -> None: + log_path = _prepare_log_dir(trial_env_vars.NNI_OUTPUT_DIR) / 'trial.log' + log_file = open(log_path, 'a') + _register_handler(StreamHandler(log_file), logging.WARNING) + + if trial_env_vars.NNI_PLATFORM == 'local': + sys.stdout = _LogFileWrapper(log_file) + + +def _init_logger_standalone() -> None: + _register_handler(StreamHandler(sys.stdout), logging.WARNING) + + +def _prepare_log_dir(path: Optional[str]) -> Path: + if path is None: + return Path() + ret = Path(path) + ret.mkdir(parents=True, exist_ok=True) + return ret + +def _register_handler(handler: Handler, level: int, tag: str = '_default_') -> None: + assert tag not in handlers + handlers[tag] = handler + handler.setFormatter(formatter) + logger = logging.getLogger() + logger.addHandler(handler) + logger.setLevel(level) + +def _colorful_format(record): + time = formatter.formatTime(record, time_format) + if not record.name.startswith('nni.'): + return '[{}] ({}) {}'.format(time, record.name, record.msg % record.args) + if record.levelno >= logging.ERROR: + color = colorama.Fore.RED + elif record.levelno >= logging.WARNING: + color = colorama.Fore.YELLOW + elif record.levelno >= logging.WARNING: + color = colorama.Fore.GREEN + else: + color = colorama.Fore.BLUE + msg = color + (record.msg % record.args) + colorama.Style.RESET_ALL + if record.levelno < logging.WARNING: + return '[{}] {}:{} {}'.format(time, record.threadName, record.name, msg) + else: + return '[{}] {}'.format(time, msg) + +class _LogFileWrapper(TextIOBase): + # wrap the logger file so that anything written to it will automatically get formatted + + def __init__(self, log_file: TextIOBase): + self.file: TextIOBase = log_file + self.line_buffer: Optional[str] = None + self.line_start_time: Optional[datetime] = None + + def write(self, s: str) -> int: + cur_time = datetime.now() + if self.line_buffer and (cur_time - self.line_start_time).total_seconds() > 0.1: + self.flush() + + if self.line_buffer: + self.line_buffer += s + else: + self.line_buffer = s + self.line_start_time = cur_time + + if '\n' not in s: + return len(s) + + time_str = cur_time.strftime(time_format) + lines = self.line_buffer.split('\n') + for line in lines[:-1]: + self.file.write(f'[{time_str}] PRINT {line}\n') + self.file.flush() + + self.line_buffer = lines[-1] + self.line_start_time = cur_time + return len(s) + + def flush(self) -> None: + if self.line_buffer: + time_str = self.line_start_time.strftime(time_format) + self.file.write(f'[{time_str}] PRINT {self.line_buffer}\n') + self.file.flush() + self.line_buffer = None diff --git a/utils/third_party/nni_new/runtime/log_backup.py b/utils/third_party/nni_new/runtime/log_backup.py new file mode 100644 index 0000000000000000000000000000000000000000..0fc97c666f5c2c5bec08d89f16c28efefab2d833 --- /dev/null +++ b/utils/third_party/nni_new/runtime/log_backup.py @@ -0,0 +1,172 @@ +from datetime import datetime +from io import TextIOBase +import logging +from logging import FileHandler, Formatter, Handler, StreamHandler +from pathlib import Path +import sys +import time +from typing import Optional + +import colorama + +from .env_vars import dispatcher_env_vars, trial_env_vars + + +handlers = {} + +log_format = '[%(asctime)s] %(levelname)s (%(name)s/%(threadName)s) %(message)s' +time_format = '%Y-%m-%d %H:%M:%S' +formatter = Formatter(log_format, time_format) + + +def init_logger() -> None: + """ + This function will (and should only) get invoked on the first time of importing nni (no matter which submodule). + It will try to detect the running environment and setup logger accordingly. + + The detection should work in most cases but for `nnictl` and `nni.experiment`. + They will be identified as "standalone" mode and must configure the logger by themselves. + """ + colorama.init() + + if dispatcher_env_vars.SDK_PROCESS == 'dispatcher': + _init_logger_dispatcher() + return + + trial_platform = trial_env_vars.NNI_PLATFORM + + if trial_platform == 'unittest': + return + + if trial_platform and not trial_env_vars.REUSE_MODE: + _init_logger_trial() + return + + _init_logger_standalone() + + logging.getLogger('filelock').setLevel(logging.WARNING) + +_exp_log_initialized = False + +def init_logger_experiment() -> None: + """ + Initialize logger for `nni.experiment.Experiment`. + + This function will get invoked after `init_logger()`. + """ + global _exp_log_initialized + if not _exp_log_initialized: + _exp_log_initialized = True + colorful_formatter = Formatter(log_format, time_format) + colorful_formatter.format = _colorful_format + handlers['_default_'].setFormatter(colorful_formatter) + +def start_experiment_log(experiment_id: str, log_directory: Path, debug: bool) -> None: + log_path = _prepare_log_dir(log_directory) / 'dispatcher.log' + log_level = logging.DEBUG if debug else logging.INFO + _register_handler(FileHandler(log_path), log_level, experiment_id) + +def stop_experiment_log(experiment_id: str) -> None: + if experiment_id in handlers: + logging.getLogger().removeHandler(handlers.pop(experiment_id)) + + +def _init_logger_dispatcher() -> None: + log_level_map = { + 'fatal': logging.CRITICAL, + 'error': logging.ERROR, + 'warning': logging.WARNING, + 'info': logging.INFO, + 'debug': logging.DEBUG, + 'trace': 0 + } + + log_path = _prepare_log_dir(dispatcher_env_vars.NNI_LOG_DIRECTORY) / 'dispatcher.log' + log_level = log_level_map.get(dispatcher_env_vars.NNI_LOG_LEVEL, logging.INFO) + _register_handler(FileHandler(log_path), log_level) + + +def _init_logger_trial() -> None: + log_path = _prepare_log_dir(trial_env_vars.NNI_OUTPUT_DIR) / 'trial.log' + log_file = open(log_path, 'a') + _register_handler(StreamHandler(log_file), logging.INFO) + + if trial_env_vars.NNI_PLATFORM == 'local': + sys.stdout = _LogFileWrapper(log_file) + + +def _init_logger_standalone() -> None: + _register_handler(StreamHandler(sys.stdout), logging.INFO) + + +def _prepare_log_dir(path: Optional[str]) -> Path: + if path is None: + return Path() + ret = Path(path) + ret.mkdir(parents=True, exist_ok=True) + return ret + +def _register_handler(handler: Handler, level: int, tag: str = '_default_') -> None: + assert tag not in handlers + handlers[tag] = handler + handler.setFormatter(formatter) + logger = logging.getLogger() + logger.addHandler(handler) + logger.setLevel(level) + +def _colorful_format(record): + time = formatter.formatTime(record, time_format) + if not record.name.startswith('nni.'): + return '[{}] ({}) {}'.format(time, record.name, record.msg % record.args) + if record.levelno >= logging.ERROR: + color = colorama.Fore.RED + elif record.levelno >= logging.WARNING: + color = colorama.Fore.YELLOW + elif record.levelno >= logging.INFO: + color = colorama.Fore.GREEN + else: + color = colorama.Fore.BLUE + msg = color + (record.msg % record.args) + colorama.Style.RESET_ALL + if record.levelno < logging.INFO: + return '[{}] {}:{} {}'.format(time, record.threadName, record.name, msg) + else: + return '[{}] {}'.format(time, msg) + +class _LogFileWrapper(TextIOBase): + # wrap the logger file so that anything written to it will automatically get formatted + + def __init__(self, log_file: TextIOBase): + self.file: TextIOBase = log_file + self.line_buffer: Optional[str] = None + self.line_start_time: Optional[datetime] = None + + def write(self, s: str) -> int: + cur_time = datetime.now() + if self.line_buffer and (cur_time - self.line_start_time).total_seconds() > 0.1: + self.flush() + + if self.line_buffer: + self.line_buffer += s + else: + self.line_buffer = s + self.line_start_time = cur_time + + if '\n' not in s: + return len(s) + + time_str = cur_time.strftime(time_format) + lines = self.line_buffer.split('\n') + for line in lines[:-1]: + self.file.write(f'[{time_str}] PRINT {line}\n') + self.file.flush() + + self.line_buffer = lines[-1] + self.line_start_time = cur_time + return len(s) + + def flush(self) -> None: + if self.line_buffer: + time_str = self.line_start_time.strftime(time_format) + self.file.write(f'[{time_str}] PRINT {self.line_buffer}\n') + self.file.flush() + self.line_buffer = None diff --git a/utils/third_party/nni_new/runtime/msg_dispatcher.py b/utils/third_party/nni_new/runtime/msg_dispatcher.py new file mode 100644 index 0000000000000000000000000000000000000000..20b9597f9e0df13c4b1f1ac4ad344cce77a41fc1 --- /dev/null +++ b/utils/third_party/nni_new/runtime/msg_dispatcher.py @@ -0,0 +1,242 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +from collections import defaultdict +import json_tricks + +from nni import NoMoreTrialError +from .protocol import CommandType, send +from .msg_dispatcher_base import MsgDispatcherBase +from nni.assessor import AssessResult +from .common import multi_thread_enabled, multi_phase_enabled +from .env_vars import dispatcher_env_vars +from ..utils import MetricType, to_json + +_logger = logging.getLogger(__name__) + +# Assessor global variables +_trial_history = defaultdict(dict) +'''key: trial job ID; value: intermediate results, mapping from sequence number to data''' + +_ended_trials = set() +'''trial_job_id of all ended trials. +We need this because NNI manager may send metrics after reporting a trial ended. +TODO: move this logic to NNI manager +''' + + +def _sort_history(history): + ret = [] + for i, _ in enumerate(history): + if i in history: + ret.append(history[i]) + else: + break + return ret + + +# Tuner global variables +_next_parameter_id = 0 +_trial_params = {} +'''key: parameter ID; value: parameters''' +_customized_parameter_ids = set() + + +def _create_parameter_id(): + global _next_parameter_id + _next_parameter_id += 1 + return _next_parameter_id - 1 + + +def _pack_parameter(parameter_id, params, customized=False, trial_job_id=None, parameter_index=None): + _trial_params[parameter_id] = params + ret = { + 'parameter_id': parameter_id, + 'parameter_source': 'customized' if customized else 'algorithm', + 'parameters': params + } + if trial_job_id is not None: + ret['trial_job_id'] = trial_job_id + if parameter_index is not None: + ret['parameter_index'] = parameter_index + else: + ret['parameter_index'] = 0 + return to_json(ret) + + +class MsgDispatcher(MsgDispatcherBase): + def __init__(self, tuner, assessor=None): + super(MsgDispatcher, self).__init__() + self.tuner = tuner + self.assessor = assessor + if assessor is None: + _logger.debug('Assessor is not configured') + + def load_checkpoint(self): + self.tuner.load_checkpoint() + if self.assessor is not None: + self.assessor.load_checkpoint() + + def save_checkpoint(self): + self.tuner.save_checkpoint() + if self.assessor is not None: + self.assessor.save_checkpoint() + + def handle_initialize(self, data): + """Data is search space + """ + self.tuner.update_search_space(data) + send(CommandType.Initialized, '') + + def send_trial_callback(self, id_, params): + """For tuner to issue trial config when the config is generated + """ + send(CommandType.NewTrialJob, _pack_parameter(id_, params)) + + def handle_request_trial_jobs(self, data): + # data: number or trial jobs + ids = [_create_parameter_id() for _ in range(data)] + _logger.debug("requesting for generating params of %s", ids) + params_list = self.tuner.generate_multiple_parameters(ids, st_callback=self.send_trial_callback) + + for i, _ in enumerate(params_list): + send(CommandType.NewTrialJob, _pack_parameter(ids[i], params_list[i])) + # when parameters is None. + if len(params_list) < len(ids): + send(CommandType.NoMoreTrialJobs, _pack_parameter(ids[0], '')) + + def handle_update_search_space(self, data): + self.tuner.update_search_space(data) + + def handle_import_data(self, data): + """Import additional data for tuning + data: a list of dictionaries, each of which has at least two keys, 'parameter' and 'value' + """ + for entry in data: + entry['value'] = entry['value'] if type(entry['value']) is str else json_tricks.dumps(entry['value']) + entry['value'] = json_tricks.loads(entry['value']) + self.tuner.import_data(data) + + def handle_add_customized_trial(self, data): + # data: parameters + id_ = _create_parameter_id() + _customized_parameter_ids.add(id_) + + def handle_report_metric_data(self, data): + """ + data: a dict received from nni_manager, which contains: + - 'parameter_id': id of the trial + - 'value': metric value reported by nni.report_final_result() + - 'type': report type, support {'FINAL', 'PERIODICAL'} + """ + # metrics value is dumped as json string in trial, so we need to decode it here + if 'value' in data: + data['value'] = json_tricks.loads(data['value']) + if data['type'] == MetricType.FINAL: + self._handle_final_metric_data(data) + elif data['type'] == MetricType.PERIODICAL: + if self.assessor is not None: + self._handle_intermediate_metric_data(data) + elif data['type'] == MetricType.REQUEST_PARAMETER: + assert multi_phase_enabled() + assert data['trial_job_id'] is not None + assert data['parameter_index'] is not None + param_id = _create_parameter_id() + try: + param = self.tuner.generate_parameters(param_id, trial_job_id=data['trial_job_id']) + except NoMoreTrialError: + param = None + send(CommandType.SendTrialJobParameter, _pack_parameter(param_id, param, trial_job_id=data['trial_job_id'], + parameter_index=data['parameter_index'])) + else: + raise ValueError('Data type not supported: {}'.format(data['type'])) + + def handle_trial_end(self, data): + """ + data: it has three keys: trial_job_id, event, hyper_params + - trial_job_id: the id generated by training service + - event: the job's state + - hyper_params: the hyperparameters generated and returned by tuner + """ + trial_job_id = data['trial_job_id'] + _ended_trials.add(trial_job_id) + if trial_job_id in _trial_history: + _trial_history.pop(trial_job_id) + if self.assessor is not None: + self.assessor.trial_end(trial_job_id, data['event'] == 'SUCCEEDED') + if self.tuner is not None: + self.tuner.trial_end(json_tricks.loads(data['hyper_params'])['parameter_id'], data['event'] == 'SUCCEEDED') + + def _handle_final_metric_data(self, data): + """Call tuner to process final results + """ + id_ = data['parameter_id'] + value = data['value'] + if id_ is None or id_ in _customized_parameter_ids: + if not hasattr(self.tuner, '_accept_customized'): + self.tuner._accept_customized = False + if not self.tuner._accept_customized: + _logger.info('Customized trial job %s ignored by tuner', id_) + return + customized = True + else: + customized = False + if id_ in _trial_params: + self.tuner.receive_trial_result(id_, _trial_params[id_], value, customized=customized, + trial_job_id=data.get('trial_job_id')) + else: + _logger.warning('Find unknown job parameter id %s, maybe something goes wrong.', _trial_params[id_]) + + def _handle_intermediate_metric_data(self, data): + """Call assessor to process intermediate results + """ + if data['type'] != MetricType.PERIODICAL: + return + if self.assessor is None: + return + + trial_job_id = data['trial_job_id'] + if trial_job_id in _ended_trials: + return + + history = _trial_history[trial_job_id] + history[data['sequence']] = data['value'] + ordered_history = _sort_history(history) + if len(ordered_history) < data['sequence']: # no user-visible update since last time + return + + try: + result = self.assessor.assess_trial(trial_job_id, ordered_history) + except Exception as e: + _logger.error('Assessor error') + _logger.exception(e) + + if isinstance(result, bool): + result = AssessResult.Good if result else AssessResult.Bad + elif not isinstance(result, AssessResult): + msg = 'Result of Assessor.assess_trial must be an object of AssessResult, not %s' + raise RuntimeError(msg % type(result)) + + if result is AssessResult.Bad: + _logger.debug('BAD, kill %s', trial_job_id) + send(CommandType.KillTrialJob, json_tricks.dumps(trial_job_id)) + # notify tuner + _logger.debug('env var: NNI_INCLUDE_INTERMEDIATE_RESULTS: [%s]', + dispatcher_env_vars.NNI_INCLUDE_INTERMEDIATE_RESULTS) + if dispatcher_env_vars.NNI_INCLUDE_INTERMEDIATE_RESULTS == 'true': + self._earlystop_notify_tuner(data) + else: + _logger.debug('GOOD') + + def _earlystop_notify_tuner(self, data): + """Send last intermediate result as final result to tuner in case the + trial is early stopped. + """ + _logger.debug('Early stop notify tuner data: [%s]', data) + data['type'] = MetricType.FINAL + if multi_thread_enabled(): + self._handle_final_metric_data(data) + else: + data['value'] = to_json(data['value']) + self.enqueue_command(CommandType.ReportMetricData, data) diff --git a/utils/third_party/nni_new/runtime/msg_dispatcher_base.py b/utils/third_party/nni_new/runtime/msg_dispatcher_base.py new file mode 100644 index 0000000000000000000000000000000000000000..cda40f5c3f6ed4521b70757fb2ae6a83dcc31427 --- /dev/null +++ b/utils/third_party/nni_new/runtime/msg_dispatcher_base.py @@ -0,0 +1,245 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import threading +import logging +from multiprocessing.dummy import Pool as ThreadPool +from queue import Queue, Empty +import json_tricks + +from .common import multi_thread_enabled +from .env_vars import dispatcher_env_vars +from ..recoverable import Recoverable +from .protocol import CommandType, receive + + +_logger = logging.getLogger(__name__) + +QUEUE_LEN_WARNING_MARK = 20 +_worker_fast_exit_on_terminate = True + + +class MsgDispatcherBase(Recoverable): + """This is where tuners and assessors are not defined yet. + Inherits this class to make your own advisor. + """ + + def __init__(self): + self.stopping = False + if multi_thread_enabled(): + self.pool = ThreadPool() + self.thread_results = [] + else: + self.default_command_queue = Queue() + self.assessor_command_queue = Queue() + self.default_worker = threading.Thread(target=self.command_queue_worker, args=(self.default_command_queue,)) + self.assessor_worker = threading.Thread(target=self.command_queue_worker, + args=(self.assessor_command_queue,)) + self.default_worker.start() + self.assessor_worker.start() + self.worker_exceptions = [] + + def run(self): + """Run the tuner. + This function will never return unless raise. + """ + _logger.info('Dispatcher started') + if dispatcher_env_vars.NNI_MODE == 'resume': + self.load_checkpoint() + + while not self.stopping: + command, data = receive() + if data: + data = json_tricks.loads(data) + + if command is None or command is CommandType.Terminate: + break + if multi_thread_enabled(): + result = self.pool.map_async(self.process_command_thread, [(command, data)]) + self.thread_results.append(result) + if any([thread_result.ready() and not thread_result.successful() for thread_result in + self.thread_results]): + _logger.debug('Caught thread exception') + break + else: + self.enqueue_command(command, data) + if self.worker_exceptions: + break + + _logger.info('Dispatcher exiting...') + self.stopping = True + if multi_thread_enabled(): + self.pool.close() + self.pool.join() + else: + self.default_worker.join() + self.assessor_worker.join() + + _logger.info('Dispatcher terminiated') + + def command_queue_worker(self, command_queue): + """Process commands in command queues. + """ + while True: + try: + # set timeout to ensure self.stopping is checked periodically + command, data = command_queue.get(timeout=3) + try: + self.process_command(command, data) + except Exception as e: + _logger.exception(e) + self.worker_exceptions.append(e) + break + except Empty: + pass + if self.stopping and (_worker_fast_exit_on_terminate or command_queue.empty()): + break + + def enqueue_command(self, command, data): + """Enqueue command into command queues + """ + if command == CommandType.TrialEnd or ( + command == CommandType.ReportMetricData and data['type'] == 'PERIODICAL'): + self.assessor_command_queue.put((command, data)) + else: + self.default_command_queue.put((command, data)) + + qsize = self.default_command_queue.qsize() + if qsize >= QUEUE_LEN_WARNING_MARK: + _logger.warning('default queue length: %d', qsize) + + qsize = self.assessor_command_queue.qsize() + if qsize >= QUEUE_LEN_WARNING_MARK: + _logger.warning('assessor queue length: %d', qsize) + + def process_command_thread(self, request): + """Worker thread to process a command. + """ + command, data = request + if multi_thread_enabled(): + try: + self.process_command(command, data) + except Exception as e: + _logger.exception(str(e)) + raise + else: + pass + + def process_command(self, command, data): + _logger.debug('process_command: command: [%s], data: [%s]', command, data) + + command_handlers = { + # Tuner commands: + CommandType.Initialize: self.handle_initialize, + CommandType.RequestTrialJobs: self.handle_request_trial_jobs, + CommandType.UpdateSearchSpace: self.handle_update_search_space, + CommandType.ImportData: self.handle_import_data, + CommandType.AddCustomizedTrialJob: self.handle_add_customized_trial, + + # Tuner/Assessor commands: + CommandType.ReportMetricData: self.handle_report_metric_data, + + CommandType.TrialEnd: self.handle_trial_end, + CommandType.Ping: self.handle_ping, + } + if command not in command_handlers: + raise AssertionError('Unsupported command: {}'.format(command)) + command_handlers[command](data) + + def handle_ping(self, data): + pass + + def handle_initialize(self, data): + """Initialize search space and tuner, if any + This method is meant to be called only once for each experiment, after calling this method, + dispatcher should `send(CommandType.Initialized, '')`, to set the status of the experiment to be "INITIALIZED". + Parameters + ---------- + data: dict + search space + """ + raise NotImplementedError('handle_initialize not implemented') + + def handle_request_trial_jobs(self, data): + """The message dispatcher is demanded to generate ``data`` trial jobs. + These trial jobs should be sent via ``send(CommandType.NewTrialJob, json_tricks.dumps(parameter))``, + where ``parameter`` will be received by NNI Manager and eventually accessible to trial jobs as "next parameter". + Semantically, message dispatcher should do this ``send`` exactly ``data`` times. + + The JSON sent by this method should follow the format of + + :: + + { + "parameter_id": 42 + "parameters": { + // this will be received by trial + }, + "parameter_source": "algorithm" // optional + } + + Parameters + ---------- + data: int + number of trial jobs + """ + raise NotImplementedError('handle_request_trial_jobs not implemented') + + def handle_update_search_space(self, data): + """This method will be called when search space is updated. + It's recommended to call this method in `handle_initialize` to initialize search space. + *No need to* notify NNI Manager when this update is done. + Parameters + ---------- + data: dict + search space + """ + raise NotImplementedError('handle_update_search_space not implemented') + + def handle_import_data(self, data): + """Import previous data when experiment is resumed. + Parameters + ---------- + data: list + a list of dictionaries, each of which has at least two keys, 'parameter' and 'value' + """ + raise NotImplementedError('handle_import_data not implemented') + + def handle_add_customized_trial(self, data): + """Experimental API. Not recommended for usage. + """ + raise NotImplementedError('handle_add_customized_trial not implemented') + + def handle_report_metric_data(self, data): + """Called when metric data is reported or new parameters are requested (for multiphase). + When new parameters are requested, this method should send a new parameter. + + Parameters + ---------- + data: dict + a dict which contains 'parameter_id', 'value', 'trial_job_id', 'type', 'sequence'. + type: can be `MetricType.REQUEST_PARAMETER`, `MetricType.FINAL` or `MetricType.PERIODICAL`. + `REQUEST_PARAMETER` is used to request new parameters for multiphase trial job. In this case, + the dict will contain additional keys: `trial_job_id`, `parameter_index`. Refer to `msg_dispatcher.py` + as an example. + + Raises + ------ + ValueError + Data type is not supported + """ + raise NotImplementedError('handle_report_metric_data not implemented') + + def handle_trial_end(self, data): + """Called when the state of one of the trials is changed + + Parameters + ---------- + data: dict + a dict with keys: trial_job_id, event, hyper_params. + trial_job_id: the id generated by training service. + event: the job’s state. + hyper_params: the string that is sent by message dispatcher during the creation of trials. + + """ + raise NotImplementedError('handle_trial_end not implemented') diff --git a/utils/third_party/nni_new/runtime/platform/__init__.py b/utils/third_party/nni_new/runtime/platform/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9f42a16f4351ac1105bdcba0d1d60647f608b476 --- /dev/null +++ b/utils/third_party/nni_new/runtime/platform/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from ..env_vars import trial_env_vars, dispatcher_env_vars + +assert dispatcher_env_vars.SDK_PROCESS != 'dispatcher' + +if trial_env_vars.NNI_PLATFORM is None: + from .standalone import * +elif trial_env_vars.NNI_PLATFORM == 'unittest': + from .test import * +else: + from .local import * diff --git a/utils/third_party/nni_new/runtime/platform/__pycache__/__init__.cpython-38.pyc b/utils/third_party/nni_new/runtime/platform/__pycache__/__init__.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a5c43d66d1247f79414a285d6004e563ed8bca90 Binary files /dev/null and b/utils/third_party/nni_new/runtime/platform/__pycache__/__init__.cpython-38.pyc differ diff --git a/utils/third_party/nni_new/runtime/platform/__pycache__/standalone.cpython-38.pyc b/utils/third_party/nni_new/runtime/platform/__pycache__/standalone.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a89081e73c87acc9b915dde1983dd50c2ac4a45a Binary files /dev/null and b/utils/third_party/nni_new/runtime/platform/__pycache__/standalone.cpython-38.pyc differ diff --git a/utils/third_party/nni_new/runtime/platform/local.py b/utils/third_party/nni_new/runtime/platform/local.py new file mode 100644 index 0000000000000000000000000000000000000000..b1f26462e21ac59b08082a97f1a1535b89a421c5 --- /dev/null +++ b/utils/third_party/nni_new/runtime/platform/local.py @@ -0,0 +1,83 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +import sys +import json +import time +import subprocess + +from ..env_vars import trial_env_vars +from nni.utils import to_json + +_sysdir = trial_env_vars.NNI_SYS_DIR +if not os.path.exists(os.path.join(_sysdir, '.nni')): + os.makedirs(os.path.join(_sysdir, '.nni')) +_metric_file = open(os.path.join(_sysdir, '.nni', 'metrics'), 'ab') + +_outputdir = trial_env_vars.NNI_OUTPUT_DIR +if not os.path.exists(_outputdir): + os.makedirs(_outputdir) + +_reuse_mode = trial_env_vars.REUSE_MODE +_nni_platform = trial_env_vars.NNI_PLATFORM + +_multiphase = trial_env_vars.MULTI_PHASE + +_param_index = 0 + +def request_next_parameter(): + metric = to_json({ + 'trial_job_id': trial_env_vars.NNI_TRIAL_JOB_ID, + 'type': 'REQUEST_PARAMETER', + 'sequence': 0, + 'parameter_index': _param_index + }) + send_metric(metric) + +def get_next_parameter(): + global _param_index + params_file_name = '' + if _multiphase in ('true', 'True'): + params_file_name = ('parameter_{}.cfg'.format(_param_index), 'parameter.cfg')[_param_index == 0] + else: + if _param_index > 0: + return None + elif _param_index == 0: + params_file_name = 'parameter.cfg' + else: + raise AssertionError('_param_index value ({}) should >=0'.format(_param_index)) + + params_filepath = os.path.join(_sysdir, params_file_name) + if not os.path.isfile(params_filepath): + request_next_parameter() + while not (os.path.isfile(params_filepath) and os.path.getsize(params_filepath) > 0): + time.sleep(3) + params_file = open(params_filepath, 'r') + params = json.load(params_file) + _param_index += 1 + return params + +def send_metric(string): + if _nni_platform != 'local' or _reuse_mode in ('true', 'True'): + assert len(string) < 1000000, 'Metric too long' + print("NNISDK_MEb'%s'" % (string), flush=True) + else: + data = (string + '\n').encode('utf8') + assert len(data) < 1000000, 'Metric too long' + _metric_file.write(b'ME%06d%b' % (len(data), data)) + _metric_file.flush() + if sys.platform == "win32": + file = open(_metric_file.name) + file.close() + else: + subprocess.run(['touch', _metric_file.name], check=True) + +def get_experiment_id(): + return trial_env_vars.NNI_EXP_ID + +def get_trial_id(): + return trial_env_vars.NNI_TRIAL_JOB_ID + +def get_sequence_id(): + return int(trial_env_vars.NNI_TRIAL_SEQ_ID) diff --git a/utils/third_party/nni_new/runtime/platform/standalone.py b/utils/third_party/nni_new/runtime/platform/standalone.py new file mode 100644 index 0000000000000000000000000000000000000000..70caad4e6616aef63e55d559d3f32726ffa42437 --- /dev/null +++ b/utils/third_party/nni_new/runtime/platform/standalone.py @@ -0,0 +1,52 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import colorama +import logging +import warnings +import json_tricks + +__all__ = [ + 'get_next_parameter', + 'get_experiment_id', + 'get_trial_id', + 'get_sequence_id', + 'send_metric', +] + +_logger = logging.getLogger('nni') + + +def get_next_parameter(): + warning_message = ''.join([ + colorama.Style.BRIGHT, + colorama.Fore.RED, + 'Running NNI code without runtime. ', + 'Check the following tutorial if you are new to NNI: ', + colorama.Fore.YELLOW, + 'https://nni.readthedocs.io/en/stable/Tutorial/QuickStart.html#id1', + colorama.Style.RESET_ALL + ]) + warnings.warn(warning_message, RuntimeWarning) + return { + 'parameter_id': None, + 'parameters': {} + } + +def get_experiment_id(): + return 'STANDALONE' + +def get_trial_id(): + return 'STANDALONE' + +def get_sequence_id(): + return 0 + +def send_metric(string): + metric = json_tricks.loads(string) + if metric['type'] == 'FINAL': + _logger.info('Final result: %s', metric['value']) + elif metric['type'] == 'PERIODICAL': + _logger.info('Intermediate result: %s (Index %s)', metric['value'], metric['sequence']) + else: + _logger.error('Unexpected metric: %s', string) diff --git a/utils/third_party/nni_new/runtime/platform/test.py b/utils/third_party/nni_new/runtime/platform/test.py new file mode 100644 index 0000000000000000000000000000000000000000..9bc9651eb0a953420b00c50408248939f2e5ecb6 --- /dev/null +++ b/utils/third_party/nni_new/runtime/platform/test.py @@ -0,0 +1,39 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +# pylint: skip-file + +import copy +import json_tricks + + +_params = None +_last_metric = None + + +def get_next_parameter(): + return _params + +def get_experiment_id(): + return 'fakeidex' + +def get_trial_id(): + return 'fakeidtr' + +def get_sequence_id(): + return 0 + +def send_metric(string): + global _last_metric + _last_metric = string + + +def init_params(params): + global _params + _params = copy.deepcopy(params) + +def get_last_metric(): + metrics = json_tricks.loads(_last_metric) + metrics['value'] = json_tricks.loads(metrics['value']) + + return metrics diff --git a/utils/third_party/nni_new/runtime/protocol.py b/utils/third_party/nni_new/runtime/protocol.py new file mode 100644 index 0000000000000000000000000000000000000000..9c5222f0970a71ac6bc7f6fd45e397ebfac2680b --- /dev/null +++ b/utils/third_party/nni_new/runtime/protocol.py @@ -0,0 +1,72 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import logging +import os +import threading +from enum import Enum + +_logger = logging.getLogger(__name__) + + +class CommandType(Enum): + # in + Initialize = b'IN' + RequestTrialJobs = b'GE' + ReportMetricData = b'ME' + UpdateSearchSpace = b'SS' + ImportData = b'FD' + AddCustomizedTrialJob = b'AD' + TrialEnd = b'EN' + Terminate = b'TE' + Ping = b'PI' + + # out + Initialized = b'ID' + NewTrialJob = b'TR' + SendTrialJobParameter = b'SP' + NoMoreTrialJobs = b'NO' + KillTrialJob = b'KI' + +_lock = threading.Lock() +try: + if os.environ.get('NNI_PLATFORM') != 'unittest': + _in_file = open(3, 'rb') + _out_file = open(4, 'wb') +except OSError: + _logger.debug('IPC pipeline not exists') + + +def send(command, data): + """Send command to Training Service. + command: CommandType object. + data: string payload. + """ + global _lock + try: + _lock.acquire() + data = data.encode('utf8') + msg = b'%b%014d%b' % (command.value, len(data), data) + _logger.debug('Sending command, data: [%s]', msg) + _out_file.write(msg) + _out_file.flush() + finally: + _lock.release() + + +def receive(): + """Receive a command from Training Service. + Returns a tuple of command (CommandType) and payload (str) + """ + header = _in_file.read(16) + _logger.debug('Received command, header: [%s]', header) + if header is None or len(header) < 16: + # Pipe EOF encountered + _logger.debug('Pipe EOF encountered') + return None, None + length = int(header[2:]) + data = _in_file.read(length) + command = CommandType(header[:2]) + data = data.decode('utf8') + _logger.debug('Received command, data: [%s]', data) + return command, data diff --git a/utils/third_party/nni_new/smartparam.py b/utils/third_party/nni_new/smartparam.py new file mode 100644 index 0000000000000000000000000000000000000000..dde0ac2bd64e8b22e2948909b116c33aefda7561 --- /dev/null +++ b/utils/third_party/nni_new/smartparam.py @@ -0,0 +1,148 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import numpy as np + +from .runtime.env_vars import trial_env_vars +from . import trial +from . import parameter_expressions as param_exp +from .common.nas_utils import classic_mode, enas_mode, oneshot_mode, darts_mode + + +__all__ = [ + 'choice', + 'randint', + 'uniform', + 'quniform', + 'loguniform', + 'qloguniform', + 'normal', + 'qnormal', + 'lognormal', + 'qlognormal', + 'function_choice', + 'mutable_layer' +] + + +if trial_env_vars.NNI_PLATFORM is None: + def choice(*options, name=None): + return param_exp.choice(options, np.random.RandomState()) + + def randint(lower, upper, name=None): + return param_exp.randint(lower, upper, np.random.RandomState()) + + def uniform(low, high, name=None): + return param_exp.uniform(low, high, np.random.RandomState()) + + def quniform(low, high, q, name=None): + assert high > low, 'Upper bound must be larger than lower bound' + return param_exp.quniform(low, high, q, np.random.RandomState()) + + def loguniform(low, high, name=None): + assert low > 0, 'Lower bound must be positive' + return param_exp.loguniform(low, high, np.random.RandomState()) + + def qloguniform(low, high, q, name=None): + return param_exp.qloguniform(low, high, q, np.random.RandomState()) + + def normal(mu, sigma, name=None): + return param_exp.normal(mu, sigma, np.random.RandomState()) + + def qnormal(mu, sigma, q, name=None): + return param_exp.qnormal(mu, sigma, q, np.random.RandomState()) + + def lognormal(mu, sigma, name=None): + return param_exp.lognormal(mu, sigma, np.random.RandomState()) + + def qlognormal(mu, sigma, q, name=None): + return param_exp.qlognormal(mu, sigma, q, np.random.RandomState()) + + def function_choice(*funcs, name=None): + return param_exp.choice(funcs, np.random.RandomState())() + + def mutable_layer(): + raise RuntimeError('Cannot call nni.mutable_layer in this mode') + +else: + + def choice(options, name=None, key=None): + return options[_get_param(key)] + + def randint(lower, upper, name=None, key=None): + return _get_param(key) + + def uniform(low, high, name=None, key=None): + return _get_param(key) + + def quniform(low, high, q, name=None, key=None): + return _get_param(key) + + def loguniform(low, high, name=None, key=None): + return _get_param(key) + + def qloguniform(low, high, q, name=None, key=None): + return _get_param(key) + + def normal(mu, sigma, name=None, key=None): + return _get_param(key) + + def qnormal(mu, sigma, q, name=None, key=None): + return _get_param(key) + + def lognormal(mu, sigma, name=None, key=None): + return _get_param(key) + + def qlognormal(mu, sigma, q, name=None, key=None): + return _get_param(key) + + def function_choice(funcs, name=None, key=None): + return funcs[_get_param(key)]() + + def mutable_layer( + mutable_id, + mutable_layer_id, + funcs, + funcs_args, + fixed_inputs, + optional_inputs, + optional_input_size, + mode='classic_mode', + tf=None): + '''execute the chosen function and inputs. + Below is an example of chosen function and inputs: + { + "mutable_id": { + "mutable_layer_id": { + "chosen_layer": "pool", + "chosen_inputs": ["out1", "out3"] + } + } + } + Parameters: + --------------- + mutable_id: the name of this mutable_layer block (which could have multiple mutable layers) + mutable_layer_id: the name of a mutable layer in this block + funcs: dict of function calls + funcs_args: + fixed_inputs: + optional_inputs: dict of optional inputs + optional_input_size: number of candidate inputs to be chosen + tf: tensorflow module + ''' + args = (mutable_id, mutable_layer_id, funcs, funcs_args, fixed_inputs, optional_inputs, optional_input_size) + if mode == 'classic_mode': + return classic_mode(*args) + assert tf is not None, 'Internal Error: Tensorflow should not be None in modes other than classic_mode' + if mode == 'enas_mode': + return enas_mode(*args, tf) + if mode == 'oneshot_mode': + return oneshot_mode(*args, tf) + if mode == 'darts_mode': + return darts_mode(*args, tf) + raise RuntimeError('Unrecognized mode: %s' % mode) + + def _get_param(key): + if trial.get_current_parameter() is None: + trial.get_next_parameter() + return trial.get_current_parameter(key) diff --git a/utils/third_party/nni_new/tools/annotation/.gitignore b/utils/third_party/nni_new/tools/annotation/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..36e264cf443988bf3101b2467e83b259bcd5a4ad --- /dev/null +++ b/utils/third_party/nni_new/tools/annotation/.gitignore @@ -0,0 +1 @@ +_generated diff --git a/utils/third_party/nni_new/tools/annotation/__init__.py b/utils/third_party/nni_new/tools/annotation/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d9ba405c8844591ec083921c90ba2a1d67d52cf8 --- /dev/null +++ b/utils/third_party/nni_new/tools/annotation/__init__.py @@ -0,0 +1,141 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +import sys +import shutil +import json + +from . import code_generator +from . import search_space_generator +from . import specific_code_generator + + +__all__ = ['generate_search_space', 'expand_annotations'] + +slash = '/' +if sys.platform == "win32": + slash = '\\' + +def generate_search_space(code_dir): + """Generate search space from Python source code. + Return a serializable search space object. + code_dir: directory path of source files (str) + """ + code_dir = str(code_dir) + search_space = {} + + if code_dir.endswith(slash): + code_dir = code_dir[:-1] + + for subdir, _, files in os.walk(code_dir): + # generate module name from path + if subdir == code_dir: + package = '' + else: + assert subdir.startswith(code_dir + slash), subdir + prefix_len = len(code_dir) + 1 + package = subdir[prefix_len:].replace(slash, '.') + '.' + + for file_name in files: + if file_name.endswith('.py'): + path = os.path.join(subdir, file_name) + module = package + file_name[:-3] + search_space.update(_generate_file_search_space(path, module)) + + return search_space + +def _generate_file_search_space(path, module): + with open(path) as src: + try: + search_space, code = search_space_generator.generate(module, src.read()) + except Exception as exc: # pylint: disable=broad-except + if exc.args: + raise RuntimeError(path + ' ' + '\n'.join(exc.args)) + else: + raise RuntimeError('Failed to generate search space for %s: %r' % (path, exc)) + with open(path, 'w') as dst: + dst.write(code) + return search_space + + +def expand_annotations(src_dir, dst_dir, exp_id='', trial_id='', nas_mode=None): + """Expand annotations in user code. + Return dst_dir if annotation detected; return src_dir if not. + src_dir: directory path of user code (str) + dst_dir: directory to place generated files (str) + nas_mode: the mode of NAS given that NAS interface is used + """ + src_dir, dst_dir = str(src_dir), str(dst_dir) + + if src_dir[-1] == slash: + src_dir = src_dir[:-1] + + if dst_dir[-1] == slash: + dst_dir = dst_dir[:-1] + + annotated = False + + for src_subdir, dirs, files in os.walk(src_dir): + assert src_subdir.startswith(src_dir) + dst_subdir = src_subdir.replace(src_dir, dst_dir, 1) + os.makedirs(dst_subdir, exist_ok=True) + + # generate module name from path + if src_subdir == src_dir: + package = '' + else: + assert src_subdir.startswith(src_dir + slash), src_subdir + prefix_len = len(src_dir) + 1 + package = src_subdir[prefix_len:].replace(slash, '.') + '.' + + for file_name in files: + src_path = os.path.join(src_subdir, file_name) + dst_path = os.path.join(dst_subdir, file_name) + if file_name.endswith('.py'): + if trial_id == '': + annotated |= _expand_file_annotations(src_path, dst_path, nas_mode) + else: + module = package + file_name[:-3] + annotated |= _generate_specific_file(src_path, dst_path, exp_id, trial_id, module) + else: + shutil.copyfile(src_path, dst_path) + + for dir_name in dirs: + os.makedirs(os.path.join(dst_subdir, dir_name), exist_ok=True) + + return dst_dir if annotated else src_dir + +def _expand_file_annotations(src_path, dst_path, nas_mode): + with open(src_path) as src, open(dst_path, 'w') as dst: + try: + annotated_code = code_generator.parse(src.read(), nas_mode) + if annotated_code is None: + shutil.copyfile(src_path, dst_path) + return False + dst.write(annotated_code) + return True + + except Exception as exc: # pylint: disable=broad-except + if exc.args: + raise RuntimeError(src_path + ' ' + '\n'.join(str(arg) for arg in exc.args)) + else: + raise RuntimeError('Failed to expand annotations for %s: %r' % (src_path, exc)) + +def _generate_specific_file(src_path, dst_path, exp_id, trial_id, module): + with open(src_path) as src, open(dst_path, 'w') as dst: + try: + with open(os.path.expanduser('~/nni-experiments/%s/trials/%s/parameter.cfg'%(exp_id, trial_id))) as fd: + para_cfg = json.load(fd) + annotated_code = specific_code_generator.parse(src.read(), para_cfg["parameters"], module) + if annotated_code is None: + shutil.copyfile(src_path, dst_path) + return False + dst.write(annotated_code) + return True + + except Exception as exc: # pylint: disable=broad-except + if exc.args: + raise RuntimeError(src_path + ' ' + '\n'.join(str(arg) for arg in exc.args)) + else: + raise RuntimeError('Failed to expand annotations for %s: %r' % (src_path, exc)) diff --git a/utils/third_party/nni_new/tools/annotation/code_generator.py b/utils/third_party/nni_new/tools/annotation/code_generator.py new file mode 100644 index 0000000000000000000000000000000000000000..c924e4195072eeff3e112b9ca5e01cb1df41c178 --- /dev/null +++ b/utils/third_party/nni_new/tools/annotation/code_generator.py @@ -0,0 +1,369 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import ast +import astor + +from .utils import ast_Num, ast_Str, lineno + +# pylint: disable=unidiomatic-typecheck + +def parse_annotation_mutable_layers(code, lineno, nas_mode): + """Parse the string of mutable layers in annotation. + Return a list of AST Expr nodes + code: annotation string (excluding '@') + nas_mode: the mode of NAS + """ + module = ast.parse(code) + assert type(module) is ast.Module, 'internal error #1' + assert len(module.body) == 1, 'Annotation mutable_layers contains more than one expression' + assert type(module.body[0]) is ast.Expr, 'Annotation is not expression' + call = module.body[0].value + nodes = [] + mutable_id = 'mutable_block_' + str(lineno) + mutable_layer_cnt = 0 + for arg in call.args: + fields = {'layer_choice': False, + 'fixed_inputs': False, + 'optional_inputs': False, + 'optional_input_size': False, + 'layer_output': False} + for k, value in zip(arg.keys, arg.values): + if k.id == 'layer_choice': + assert not fields['layer_choice'], 'Duplicated field: layer_choice' + assert type(value) is ast.List, 'Value of layer_choice should be a list' + call_funcs_keys = [] + call_funcs_values = [] + call_kwargs_values = [] + for call in value.elts: + assert type(call) is ast.Call, 'Element in layer_choice should be function call' + call_name = astor.to_source(call).strip() + call_funcs_keys.append(ast_Str(s=call_name)) + call_funcs_values.append(call.func) + assert not call.args, 'Number of args without keyword should be zero' + kw_args = [] + kw_values = [] + for kw in call.keywords: + kw_args.append(ast_Str(s=kw.arg)) + kw_values.append(kw.value) + call_kwargs_values.append(ast.Dict(keys=kw_args, values=kw_values)) + call_funcs = ast.Dict(keys=call_funcs_keys, values=call_funcs_values) + call_kwargs = ast.Dict(keys=call_funcs_keys, values=call_kwargs_values) + fields['layer_choice'] = True + elif k.id == 'fixed_inputs': + assert not fields['fixed_inputs'], 'Duplicated field: fixed_inputs' + assert type(value) is ast.List, 'Value of fixed_inputs should be a list' + fixed_inputs = value + fields['fixed_inputs'] = True + elif k.id == 'optional_inputs': + assert not fields['optional_inputs'], 'Duplicated field: optional_inputs' + assert type(value) is ast.List, 'Value of optional_inputs should be a list' + var_names = [ast_Str(s=astor.to_source(var).strip()) for var in value.elts] + optional_inputs = ast.Dict(keys=var_names, values=value.elts) + fields['optional_inputs'] = True + elif k.id == 'optional_input_size': + assert not fields['optional_input_size'], 'Duplicated field: optional_input_size' + assert type(value) is ast_Num or type(value) is ast.List, \ + 'Value of optional_input_size should be a number or list' + optional_input_size = value + fields['optional_input_size'] = True + elif k.id == 'layer_output': + assert not fields['layer_output'], 'Duplicated field: layer_output' + assert type(value) is ast.Name, 'Value of layer_output should be ast.Name type' + layer_output = value + fields['layer_output'] = True + else: + raise AssertionError('Unexpected field in mutable layer') + # make call for this mutable layer + assert fields['layer_choice'], 'layer_choice must exist' + assert fields['layer_output'], 'layer_output must exist' + mutable_layer_id = 'mutable_layer_' + str(mutable_layer_cnt) + mutable_layer_cnt += 1 + target_call_attr = ast.Attribute(value=ast.Name(id='nni', ctx=ast.Load()), attr='mutable_layer', ctx=ast.Load()) + target_call_args = [ast_Str(s=mutable_id), + ast_Str(s=mutable_layer_id), + call_funcs, + call_kwargs] + if fields['fixed_inputs']: + target_call_args.append(fixed_inputs) + else: + target_call_args.append(ast.List(elts=[])) + if fields['optional_inputs']: + target_call_args.append(optional_inputs) + assert fields['optional_input_size'], 'optional_input_size must exist when optional_inputs exists' + target_call_args.append(optional_input_size) + else: + target_call_args.append(ast.Dict(keys=[], values=[])) + target_call_args.append(ast_Num(n=0)) + target_call_args.append(ast_Str(s=nas_mode)) + if nas_mode in ['enas_mode', 'oneshot_mode', 'darts_mode']: + target_call_args.append(ast.Name(id='tensorflow')) + target_call = ast.Call(func=target_call_attr, args=target_call_args, keywords=[]) + node = ast.Assign(targets=[layer_output], value=target_call) + nodes.append(node) + return nodes + + +def parse_annotation(code): + """Parse an annotation string. + Return an AST Expr node. + code: annotation string (excluding '@') + """ + module = ast.parse(code) + assert type(module) is ast.Module, 'internal error #1' + assert len(module.body) == 1, 'Annotation contains more than one expression' + assert type(module.body[0]) is ast.Expr, 'Annotation is not expression' + return module.body[0] + + +def parse_annotation_function(code, func_name): + """Parse an annotation function. + Return the value of `name` keyword argument and the AST Call node. + func_name: expected function name + """ + expr = parse_annotation(code) + call = expr.value + assert type(call) is ast.Call, 'Annotation is not a function call' + + assert type(call.func) is ast.Attribute, 'Unexpected annotation function' + assert type(call.func.value) is ast.Name, 'Invalid annotation function name' + assert call.func.value.id == 'nni', 'Annotation is not a NNI function' + assert call.func.attr == func_name, 'internal error #2' + + assert len(call.keywords) == 1, 'Annotation function contains more than one keyword argument' + assert call.keywords[0].arg == 'name', 'Annotation keyword argument is not "name"' + name = call.keywords[0].value + + return name, call + + +def parse_nni_variable(code): + """Parse `nni.variable` expression. + Return the name argument and AST node of annotated expression. + code: annotation string + """ + name, call = parse_annotation_function(code, 'variable') + + assert len(call.args) == 1, 'nni.variable contains more than one arguments' + arg = call.args[0] + assert type(arg) is ast.Call, 'Value of nni.variable is not a function call' + assert type(arg.func) is ast.Attribute, 'nni.variable value is not a NNI function' + assert type(arg.func.value) is ast.Name, 'nni.variable value is not a NNI function' + assert arg.func.value.id == 'nni', 'nni.variable value is not a NNI function' + + name_str = astor.to_source(name).strip() + keyword_arg = ast.keyword(arg='name', value=ast_Str(s=name_str)) + arg.keywords.append(keyword_arg) + if arg.func.attr == 'choice': + convert_args_to_dict(arg) + + return name, arg + + +def parse_nni_function(code): + """Parse `nni.function_choice` expression. + Return the AST node of annotated expression and a list of dumped function call expressions. + code: annotation string + """ + name, call = parse_annotation_function(code, 'function_choice') + funcs = [ast.dump(func, False) for func in call.args] + convert_args_to_dict(call, with_lambda=True) + + name_str = astor.to_source(name).strip() + call.keywords[0].value = ast_Str(s=name_str) + + return call, funcs + + +def convert_args_to_dict(call, with_lambda=False): + """Convert all args to a dict such that every key and value in the dict is the same as the value of the arg. + Return the AST Call node with only one arg that is the dictionary + """ + keys, values = list(), list() + for arg in call.args: + if type(arg) in [ast_Str, ast_Num]: + arg_value = arg + else: + # if arg is not a string or a number, we use its source code as the key + arg_value = astor.to_source(arg).strip('\n"') + arg_value = ast_Str(str(arg_value)) + arg = make_lambda(arg) if with_lambda else arg + keys.append(arg_value) + values.append(arg) + del call.args[:] + call.args.append(ast.Dict(keys=keys, values=values)) + + return call + + +def make_lambda(call): + """Wrap an AST Call node to lambda expression node. + call: ast.Call node + """ + empty_args = ast.arguments(args=[], vararg=None, kwarg=None, defaults=[]) + return ast.Lambda(args=empty_args, body=call) + + +def test_variable_equal(node1, node2): + """Test whether two variables are the same.""" + if type(node1) is not type(node2): + return False + if isinstance(node1, ast.AST): + for k, v in vars(node1).items(): + if k in ('lineno', 'col_offset', 'ctx', 'end_lineno', 'end_col_offset'): + continue + if not test_variable_equal(v, getattr(node2, k)): + return False + return True + if isinstance(node1, list): + if len(node1) != len(node2): + return False + return all(test_variable_equal(n1, n2) for n1, n2 in zip(node1, node2)) + + return node1 == node2 + + +def replace_variable_node(node, annotation): + """Replace a node annotated by `nni.variable`. + node: the AST node to replace + annotation: annotation string + """ + assert type(node) is ast.Assign, 'nni.variable is not annotating assignment expression' + assert len(node.targets) == 1, 'Annotated assignment has more than one left-hand value' + name, expr = parse_nni_variable(annotation) + assert test_variable_equal(node.targets[0], name), 'Annotated variable has wrong name' + node.value = expr + return node + + +def replace_function_node(node, annotation): + """Replace a node annotated by `nni.function_choice`. + node: the AST node to replace + annotation: annotation string + """ + target, funcs = parse_nni_function(annotation) + FuncReplacer(funcs, target).visit(node) + return node + + +class FuncReplacer(ast.NodeTransformer): + """To replace target function call expressions in a node annotated by `nni.function_choice`""" + + def __init__(self, funcs, target): + """Constructor. + funcs: list of dumped function call expressions to replace + target: use this AST node to replace matching expressions + """ + self.funcs = set(funcs) + self.target = target + + def visit_Call(self, node): # pylint: disable=invalid-name + if ast.dump(node, False) in self.funcs: + return self.target + return node + + +class Transformer(ast.NodeTransformer): + """Transform original code to annotated code""" + + def __init__(self, nas_mode=None): + self.stack = [] + self.last_line = 0 + self.annotated = False + self.nas_mode = nas_mode + + def visit(self, node): + if isinstance(node, (ast.expr, ast.stmt)): + self.last_line = lineno(node) + + # do nothing for root + if not self.stack: + return self._visit_children(node) + + annotation = self.stack[-1] + + # this is a standalone string, may be an annotation + if type(node) is ast.Expr and type(node.value) is ast_Str: + # must not annotate an annotation string + assert annotation is None, 'Annotating an annotation' + return self._visit_string(node) + + if annotation is not None: # this expression is annotated + self.stack[-1] = None # so next expression is not + if annotation.startswith('nni.variable'): + return replace_variable_node(node, annotation) + if annotation.startswith('nni.function_choice'): + return replace_function_node(node, annotation) + + return self._visit_children(node) + + def _visit_string(self, node): + string = node.value.s + if string.startswith('@nni.'): + self.annotated = True + else: + return node # not an annotation, ignore it + + if string.startswith('@nni.training_update'): + expr = parse_annotation(string[1:]) + call_node = expr.value + call_node.args.insert(0, ast_Str(s=self.nas_mode)) + return expr + + if string.startswith('@nni.report_intermediate_result') \ + or string.startswith('@nni.report_final_result') \ + or string.startswith('@nni.get_next_parameter'): + return parse_annotation(string[1:]) # expand annotation string to code + + if string.startswith('@nni.mutable_layers'): + nodes = parse_annotation_mutable_layers(string[1:], lineno(node), self.nas_mode) + return nodes + + if string.startswith('@nni.variable') \ + or string.startswith('@nni.function_choice'): + self.stack[-1] = string[1:] # mark that the next expression is annotated + return None + + raise AssertionError('Unexpected annotation function') + + def _visit_children(self, node): + self.stack.append(None) + self.generic_visit(node) + annotation = self.stack.pop() + assert annotation is None, 'Annotation has no target' + return node + + +def parse(code, nas_mode=None): + """Annotate user code. + Return annotated code (str) if annotation detected; return None if not. + code: original user code (str), + nas_mode: the mode of NAS given that NAS interface is used + """ + try: + ast_tree = ast.parse(code) + except Exception: + raise RuntimeError('Bad Python code') + + transformer = Transformer(nas_mode) + try: + transformer.visit(ast_tree) + except AssertionError as exc: + raise RuntimeError('%d: %s' % (ast_tree.last_line, exc.args[0])) + + if not transformer.annotated: + return None + + last_future_import = -1 + import_nni = ast.Import(names=[ast.alias(name='nni', asname=None)]) + nodes = ast_tree.body + for i, _ in enumerate(nodes): + if type(nodes[i]) is ast.ImportFrom and nodes[i].module == '__future__': + last_future_import = i + nodes.insert(last_future_import + 1, import_nni) + # enas, oneshot and darts modes for tensorflow need tensorflow module, so we import it here + if nas_mode in ['enas_mode', 'oneshot_mode', 'darts_mode']: + import_tf = ast.Import(names=[ast.alias(name='tensorflow', asname=None)]) + nodes.insert(last_future_import + 1, import_tf) + + return astor.to_source(ast_tree) diff --git a/utils/third_party/nni_new/tools/annotation/search_space_generator.py b/utils/third_party/nni_new/tools/annotation/search_space_generator.py new file mode 100644 index 0000000000000000000000000000000000000000..a0a19f53dff601faa682f48b3f1ee574f21b9acf --- /dev/null +++ b/utils/third_party/nni_new/tools/annotation/search_space_generator.py @@ -0,0 +1,135 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import ast +import numbers + +import astor + +from .utils import ast_Num, ast_Str, lineno + +# pylint: disable=unidiomatic-typecheck + + +# list of functions related to search space generating +_ss_funcs = [ + 'choice', + 'randint', + 'uniform', + 'quniform', + 'loguniform', + 'qloguniform', + 'normal', + 'qnormal', + 'lognormal', + 'qlognormal', + 'function_choice', + 'mutable_layer' +] + + +class SearchSpaceGenerator(ast.NodeTransformer): + """Generate search space from smart parater APIs""" + + def __init__(self, module_name): + self.module_name = module_name + self.search_space = {} + self.last_line = 0 # last parsed line, useful for error reporting + + def generate_mutable_layer_search_space(self, args): + mutable_block = args[0].s + mutable_layer = args[1].s + key = self.module_name + '/' + mutable_block + args[0].s = key + if key not in self.search_space: + self.search_space[key] = {'_type': 'mutable_layer', '_value': {}} + self.search_space[key]['_value'][mutable_layer] = { + 'layer_choice': [k.s for k in args[2].keys], + 'optional_inputs': [k.s for k in args[5].keys], + 'optional_input_size': args[6].n if isinstance(args[6], ast_Num) else [args[6].elts[0].n, args[6].elts[1].n] + } + + def visit_Call(self, node): # pylint: disable=invalid-name + self.generic_visit(node) + + # ignore if the function is not 'nni.*' + if type(node.func) is not ast.Attribute: + return node + if type(node.func.value) is not ast.Name: + return node + if node.func.value.id != 'nni': + return node + + # ignore if its not a search space function (e.g. `report_final_result`) + func = node.func.attr + if func not in _ss_funcs: + return node + + self.last_line = lineno(node) + + if func == 'mutable_layer': + self.generate_mutable_layer_search_space(node.args) + return node + + if node.keywords: + # there is a `name` argument + assert len(node.keywords) == 1, 'Smart parameter has keyword argument other than "name"' + assert node.keywords[0].arg == 'name', 'Smart paramater\'s keyword argument is not "name"' + assert type(node.keywords[0].value) is ast_Str, 'Smart parameter\'s name must be string literal' + name = node.keywords[0].value.s + specified_name = True + else: + # generate the missing name automatically + name = '__line' + str(str(node.args[-1].lineno)) + specified_name = False + node.keywords = list() + + if func in ('choice', 'function_choice'): + # we will use keys in the dict as the choices, which is generated by code_generator according to the args given by user + assert len(node.args) == 1, 'Smart parameter has arguments other than dict' + # check if it is a number or a string and get its value accordingly + args = [key.n if type(key) is ast_Num else key.s for key in node.args[0].keys] + else: + # arguments of other functions must be literal number + assert all(isinstance(ast.literal_eval(astor.to_source(arg)), numbers.Real) for arg in node.args), \ + 'Smart parameter\'s arguments must be number literals' + args = [ast.literal_eval(astor.to_source(arg)) for arg in node.args] + + key = self.module_name + '/' + name + '/' + func + # store key in ast.Call + node.keywords.append(ast.keyword(arg='key', value=ast_Str(s=key))) + + if func == 'function_choice': + func = 'choice' + value = {'_type': func, '_value': args} + + if specified_name: + # multiple functions with same name must have identical arguments + old = self.search_space.get(key) + assert old is None or old == value, 'Different smart parameters have same name' + else: + # generated name must not duplicate + assert key not in self.search_space, 'Only one smart parameter is allowed in a line' + + self.search_space[key] = value + + return node + + +def generate(module_name, code): + """Generate search space. + Return a serializable search space object. + module_name: name of the module (str) + code: user code (str) + """ + try: + ast_tree = ast.parse(code) + except Exception: + raise RuntimeError('Bad Python code') + + visitor = SearchSpaceGenerator(module_name) + try: + visitor.visit(ast_tree) + except AssertionError as exc: + raise RuntimeError('%d: %s' % (visitor.last_line, exc.args[0])) + return visitor.search_space, astor.to_source(ast_tree) diff --git a/utils/third_party/nni_new/tools/annotation/specific_code_generator.py b/utils/third_party/nni_new/tools/annotation/specific_code_generator.py new file mode 100644 index 0000000000000000000000000000000000000000..5ddd362205dfa909d0077315ebe5dd9b8ade239f --- /dev/null +++ b/utils/third_party/nni_new/tools/annotation/specific_code_generator.py @@ -0,0 +1,354 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import ast +import astor +from nni.tools.nnictl.common_utils import print_warning + +from .utils import ast_Num, ast_Str, lineno + +# pylint: disable=unidiomatic-typecheck + +para_cfg = None +prefix_name = None + + +def parse_annotation_mutable_layers(code, lineno): + """Parse the string of mutable layers in annotation. + Return a list of AST Expr nodes + code: annotation string (excluding '@') + """ + module = ast.parse(code) + assert type(module) is ast.Module, 'internal error #1' + assert len(module.body) == 1, 'Annotation mutable_layers contains more than one expression' + assert type(module.body[0]) is ast.Expr, 'Annotation is not expression' + call = module.body[0].value + nodes = [] + mutable_id = prefix_name + '/mutable_block_' + str(lineno) + mutable_layer_cnt = 0 + for arg in call.args: + fields = {'layer_choice': False, + 'fixed_inputs': False, + 'optional_inputs': False, + 'optional_input_size': False, + 'layer_output': False} + mutable_layer_id = 'mutable_layer_' + str(mutable_layer_cnt) + mutable_layer_cnt += 1 + func_call = None + for k, value in zip(arg.keys, arg.values): + if k.id == 'layer_choice': + assert not fields['layer_choice'], 'Duplicated field: layer_choice' + assert type(value) is ast.List, 'Value of layer_choice should be a list' + for call in value.elts: + assert type(call) is ast.Call, 'Element in layer_choice should be function call' + call_name = astor.to_source(call).strip() + if call_name == para_cfg[mutable_id][mutable_layer_id]['chosen_layer']: + func_call = call + assert not call.args, 'Number of args without keyword should be zero' + break + fields['layer_choice'] = True + elif k.id == 'fixed_inputs': + assert not fields['fixed_inputs'], 'Duplicated field: fixed_inputs' + assert type(value) is ast.List, 'Value of fixed_inputs should be a list' + fixed_inputs = value + fields['fixed_inputs'] = True + elif k.id == 'optional_inputs': + assert not fields['optional_inputs'], 'Duplicated field: optional_inputs' + assert type(value) is ast.List, 'Value of optional_inputs should be a list' + var_names = [astor.to_source(var).strip() for var in value.elts] + chosen_inputs = para_cfg[mutable_id][mutable_layer_id]['chosen_inputs'] + elts = [] + for i in chosen_inputs: + index = var_names.index(i) + elts.append(value.elts[index]) + optional_inputs = ast.List(elts=elts) + fields['optional_inputs'] = True + elif k.id == 'optional_input_size': + pass + elif k.id == 'layer_output': + assert not fields['layer_output'], 'Duplicated field: layer_output' + assert type(value) is ast.Name, 'Value of layer_output should be ast.Name type' + layer_output = value + fields['layer_output'] = True + else: + raise AssertionError('Unexpected field in mutable layer') + # make call for this mutable layer + assert fields['layer_choice'], 'layer_choice must exist' + assert fields['layer_output'], 'layer_output must exist' + + if not fields['fixed_inputs']: + fixed_inputs = ast.List(elts=[]) + if not fields['optional_inputs']: + optional_inputs = ast.List(elts=[]) + inputs = ast.List(elts=[fixed_inputs, optional_inputs]) + + func_call.args.append(inputs) + node = ast.Assign(targets=[layer_output], value=func_call) + nodes.append(node) + return nodes + + +def parse_annotation(code): + """Parse an annotation string. + Return an AST Expr node. + code: annotation string (excluding '@') + """ + module = ast.parse(code) + assert type(module) is ast.Module, 'internal error #1' + assert len(module.body) == 1, 'Annotation contains more than one expression' + assert type(module.body[0]) is ast.Expr, 'Annotation is not expression' + return module.body[0] + + +def parse_annotation_function(code, func_name): + """Parse an annotation function. + Return the value of `name` keyword argument and the AST Call node. + func_name: expected function name + """ + expr = parse_annotation(code) + call = expr.value + assert type(call) is ast.Call, 'Annotation is not a function call' + + assert type(call.func) is ast.Attribute, 'Unexpected annotation function' + assert type(call.func.value) is ast.Name, 'Invalid annotation function name' + assert call.func.value.id == 'nni', 'Annotation is not a NNI function' + assert call.func.attr == func_name, 'internal error #2' + + assert len(call.keywords) == 1, 'Annotation function contains more than one keyword argument' + assert call.keywords[0].arg == 'name', 'Annotation keyword argument is not "name"' + name = call.keywords[0].value + + return name, call + + +def parse_nni_variable(code): + """Parse `nni.variable` expression. + Return the name argument and AST node of annotated expression. + code: annotation string + """ + name, call = parse_annotation_function(code, 'variable') + + assert len(call.args) == 1, 'nni.variable contains more than one arguments' + arg = call.args[0] + assert type(arg) is ast.Call, 'Value of nni.variable is not a function call' + assert type(arg.func) is ast.Attribute, 'nni.variable value is not a NNI function' + assert type(arg.func.value) is ast.Name, 'nni.variable value is not a NNI function' + assert arg.func.value.id == 'nni', 'nni.variable value is not a NNI function' + + name_str = astor.to_source(name).strip() + keyword_arg = ast.keyword(arg='name', value=ast_Str(s=name_str)) + arg.keywords.append(keyword_arg) + if arg.func.attr == 'choice': + convert_args_to_dict(arg) + + return name, arg + + +def parse_nni_function(code): + """Parse `nni.function_choice` expression. + Return the AST node of annotated expression and a list of dumped function call expressions. + code: annotation string + """ + name, call = parse_annotation_function(code, 'function_choice') + funcs = [ast.dump(func, False) for func in call.args] + convert_args_to_dict(call, with_lambda=True) + + name_str = astor.to_source(name).strip() + call.keywords[0].value = ast_Str(s=name_str) + + return call, funcs + + +def convert_args_to_dict(call, with_lambda=False): + """Convert all args to a dict such that every key and value in the dict is the same as the value of the arg. + Return the AST Call node with only one arg that is the dictionary + """ + keys, values = list(), list() + for arg in call.args: + if type(arg) in [ast_Str, ast_Num]: + arg_value = arg + else: + # if arg is not a string or a number, we use its source code as the key + arg_value = astor.to_source(arg).strip('\n"') + arg_value = ast_Str(str(arg_value)) + arg = make_lambda(arg) if with_lambda else arg + keys.append(arg_value) + values.append(arg) + del call.args[:] + call.args.append(ast.Dict(keys=keys, values=values)) + + return call + + +def make_lambda(call): + """Wrap an AST Call node to lambda expression node. + call: ast.Call node + """ + empty_args = ast.arguments(args=[], vararg=None, kwarg=None, defaults=[]) + return ast.Lambda(args=empty_args, body=call) + + +def test_variable_equal(node1, node2): + """Test whether two variables are the same.""" + if type(node1) is not type(node2): + return False + if isinstance(node1, ast.AST): + for k, v in vars(node1).items(): + if k in ('lineno', 'col_offset', 'ctx', 'end_lineno', 'end_col_offset'): + continue + if not test_variable_equal(v, getattr(node2, k)): + return False + return True + if isinstance(node1, list): + if len(node1) != len(node2): + return False + return all(test_variable_equal(n1, n2) for n1, n2 in zip(node1, node2)) + + return node1 == node2 + + +def replace_variable_node(node, annotation): + """Replace a node annotated by `nni.variable`. + node: the AST node to replace + annotation: annotation string + """ + assert type(node) is ast.Assign, 'nni.variable is not annotating assignment expression' + assert len(node.targets) == 1, 'Annotated assignment has more than one left-hand value' + name, expr = parse_nni_variable(annotation) + assert test_variable_equal(node.targets[0], name), 'Annotated variable has wrong name' + node.value = expr + return node + + +def replace_function_node(node, annotation): + """Replace a node annotated by `nni.function_choice`. + node: the AST node to replace + annotation: annotation string + """ + target, funcs = parse_nni_function(annotation) + FuncReplacer(funcs, target).visit(node) + return node + + +class FuncReplacer(ast.NodeTransformer): + """To replace target function call expressions in a node annotated by `nni.function_choice`""" + + def __init__(self, funcs, target): + """Constructor. + funcs: list of dumped function call expressions to replace + target: use this AST node to replace matching expressions + """ + self.funcs = set(funcs) + self.target = target + + def visit_Call(self, node): # pylint: disable=invalid-name + if ast.dump(node, False) in self.funcs: + return self.target + return node + + +class Transformer(ast.NodeTransformer): + """Transform original code to annotated code""" + + def __init__(self): + self.stack = [] + self.last_line = 0 + self.annotated = False + + def visit(self, node): + if isinstance(node, (ast.expr, ast.stmt)): + self.last_line = lineno(node) + + # do nothing for root + if not self.stack: + return self._visit_children(node) + + annotation = self.stack[-1] + + # this is a standalone string, may be an annotation + if type(node) is ast.Expr and type(node.value) is ast_Str: + # must not annotate an annotation string + assert annotation is None, 'Annotating an annotation' + return self._visit_string(node) + + if annotation is not None: # this expression is annotated + self.stack[-1] = None # so next expression is not + if annotation.startswith('nni.variable'): + return replace_variable_node(node, annotation) + if annotation.startswith('nni.function_choice'): + return replace_function_node(node, annotation) + + return self._visit_children(node) + + def _visit_string(self, node): + string = node.value.s + if string.startswith('@nni.'): + self.annotated = True + else: + return node # not an annotation, ignore it + + if string.startswith('@nni.get_next_parameter'): + deprecated_message = "'@nni.get_next_parameter' is deprecated in annotation due to inconvenience. " \ + "Please remove this line in the trial code." + print_warning(deprecated_message) + return ast.Expr(value=ast.Call(func=ast.Name(id='print', ctx=ast.Load()), + args=[ast_Str(s='Get next parameter here...')], keywords=[])) + + if string.startswith('@nni.training_update'): + return ast.Expr(value=ast.Call(func=ast.Name(id='print', ctx=ast.Load()), + args=[ast_Str(s='Training update here...')], keywords=[])) + + if string.startswith('@nni.report_intermediate_result'): + module = ast.parse(string[1:]) + arg = module.body[0].value.args[0] + return ast.Expr(value=ast.Call(func=ast.Name(id='print', ctx=ast.Load()), + args=[ast_Str(s='nni.report_intermediate_result: '), arg], keywords=[])) + + if string.startswith('@nni.report_final_result'): + module = ast.parse(string[1:]) + arg = module.body[0].value.args[0] + return ast.Expr(value=ast.Call(func=ast.Name(id='print', ctx=ast.Load()), + args=[ast_Str(s='nni.report_final_result: '), arg], keywords=[])) + + if string.startswith('@nni.mutable_layers'): + return parse_annotation_mutable_layers(string[1:], lineno(node)) + + if string.startswith('@nni.variable') \ + or string.startswith('@nni.function_choice'): + self.stack[-1] = string[1:] # mark that the next expression is annotated + return None + + raise AssertionError('Unexpected annotation function') + + def _visit_children(self, node): + self.stack.append(None) + self.generic_visit(node) + annotation = self.stack.pop() + assert annotation is None, 'Annotation has no target' + return node + + +def parse(code, para, module): + """Annotate user code. + Return annotated code (str) if annotation detected; return None if not. + code: original user code (str) + """ + global para_cfg + global prefix_name + para_cfg = para + prefix_name = module + try: + ast_tree = ast.parse(code) + except Exception: + raise RuntimeError('Bad Python code') + + transformer = Transformer() + try: + transformer.visit(ast_tree) + except AssertionError as exc: + raise RuntimeError('%d: %s' % (ast_tree.last_line, exc.args[0])) + + if not transformer.annotated: + return None + + return astor.to_source(ast_tree) diff --git a/utils/third_party/nni_new/tools/annotation/utils.py b/utils/third_party/nni_new/tools/annotation/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..176621c2cf3d9211d626b182277ef61f170880f9 --- /dev/null +++ b/utils/third_party/nni_new/tools/annotation/utils.py @@ -0,0 +1,22 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import ast +from sys import version_info + + +if version_info >= (3, 8): + ast_Num = ast_Str = ast_Bytes = ast_NameConstant = ast_Ellipsis = ast.Constant + + def lineno(ast_node): + return ast_node.end_lineno + +else: + ast_Num = ast.Num + ast_Str = ast.Str + ast_Bytes = ast.Bytes + ast_NameConstant = ast.NameConstant + ast_Ellipsis = ast.Ellipsis + + def lineno(ast_node): + return ast_node.lineno diff --git a/utils/third_party/nni_new/tools/gpu_tool/__init__.py b/utils/third_party/nni_new/tools/gpu_tool/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/utils/third_party/nni_new/tools/gpu_tool/gpu_metrics_collector.py b/utils/third_party/nni_new/tools/gpu_tool/gpu_metrics_collector.py new file mode 100644 index 0000000000000000000000000000000000000000..8e50cf9ec84533c1b8ded1799918934677eff058 --- /dev/null +++ b/utils/third_party/nni_new/tools/gpu_tool/gpu_metrics_collector.py @@ -0,0 +1,82 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import json +import os +import subprocess +import sys +import time +import traceback + +from xml.dom import minidom + + +def main(argv): + metrics_output_dir = os.environ['METRIC_OUTPUT_DIR'] + + cmd = 'nvidia-smi -q -x'.split() + while(True): + try: + smi_output = subprocess.check_output(cmd) + except Exception: + traceback.print_exc() + gen_empty_gpu_metric(metrics_output_dir) + break + parse_nvidia_smi_result(smi_output, metrics_output_dir) + # TODO: change to sleep time configurable via arguments + time.sleep(5) + + +def parse_nvidia_smi_result(smi, outputDir): + try: + old_umask = os.umask(0) + xmldoc = minidom.parseString(smi) + gpuList = xmldoc.getElementsByTagName('gpu') + with open(os.path.join(outputDir, "gpu_metrics"), 'a') as outputFile: + outPut = {} + outPut["Timestamp"] = time.asctime(time.localtime()) + outPut["gpuCount"] = len(gpuList) + outPut["gpuInfos"] = [] + for gpuIndex, gpu in enumerate(gpuList): + gpuInfo = {} + gpuInfo['index'] = gpuIndex + gpuInfo['gpuUtil'] = gpu.getElementsByTagName('utilization')[0]\ + .getElementsByTagName('gpu_util')[0]\ + .childNodes[0].data.replace("%", "").strip() + gpuInfo['gpuMemUtil'] = gpu.getElementsByTagName('utilization')[0]\ + .getElementsByTagName('memory_util')[0]\ + .childNodes[0].data.replace("%", "").strip() + processes = gpu.getElementsByTagName('processes') + runningProNumber = len(processes[0].getElementsByTagName('process_info')) + gpuInfo['activeProcessNum'] = runningProNumber + + outPut["gpuInfos"].append(gpuInfo) + print(outPut) + outputFile.write("{}\n".format(json.dumps(outPut, sort_keys=True))) + outputFile.flush() + except Exception as error: + # e_info = sys.exc_info() + print('gpu_metrics_collector error: %s' % error) + finally: + os.umask(old_umask) + + +def gen_empty_gpu_metric(outputDir): + try: + old_umask = os.umask(0) + with open(os.path.join(outputDir, "gpu_metrics"), 'a') as outputFile: + outPut = {} + outPut["Timestamp"] = time.asctime(time.localtime()) + outPut["gpuCount"] = 0 + outPut["gpuInfos"] = [] + print(outPut) + outputFile.write("{}\n".format(json.dumps(outPut, sort_keys=True))) + outputFile.flush() + except Exception: + traceback.print_exc() + finally: + os.umask(old_umask) + + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/utils/third_party/nni_new/tools/nnictl/__init__.py b/utils/third_party/nni_new/tools/nnictl/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/utils/third_party/nni_new/tools/nnictl/algo_management.py b/utils/third_party/nni_new/tools/nnictl/algo_management.py new file mode 100644 index 0000000000000000000000000000000000000000..1715e0130085a9c5b6395e1e5921a7daaf286bd8 --- /dev/null +++ b/utils/third_party/nni_new/tools/nnictl/algo_management.py @@ -0,0 +1,106 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import importlib +import json +from nni.tools.package_utils import read_registerd_algo_meta, get_registered_algo_meta, \ + write_registered_algo_meta, ALGO_TYPES, parse_full_class_name +from .common_utils import print_error, print_green, get_yml_content + +def read_reg_meta_list(meta_path): + content = get_yml_content(meta_path) + if content.get('algorithms'): + meta_list = content.get('algorithms') + else: + meta_list = [content] + for meta in meta_list: + assert 'algoType' in meta + assert meta['algoType'] in ['tuner', 'assessor', 'advisor'] + assert 'builtinName' in meta + assert 'className' in meta + return meta_list + +def verify_algo_import(meta): + def _do_verify_import(fullName): + module_name, class_name = parse_full_class_name(fullName) + class_module = importlib.import_module(module_name) + getattr(class_module, class_name) + + _do_verify_import(meta['className']) + + if meta.get('classArgsValidator'): + _do_verify_import(meta['classArgsValidator']) + +def algo_reg(args): + meta_list = read_reg_meta_list(args.meta_path) + for meta in meta_list: + old = get_registered_algo_meta(meta['builtinName']) + if old is None: + verify_algo_import(meta) + save_algo_meta_data(meta) + elif old['source'] != 'nni': + verify_algo_import(meta) + print_green(f'Updating exist algorithm') + remove_algo_meta_data(meta['builtinName']) + save_algo_meta_data(meta) + else: + print_error(f'Cannot overwrite builtin algorithm') + print_green('{} registered sucessfully!'.format(meta['builtinName'])) + +def algo_unreg(args): + name = args.name[0] + meta = get_registered_algo_meta(name) + if meta is None: + print_error('builtin algorithms {} not found!'.format(name)) + return + if meta['source'] == 'nni': + print_error('{} is provided by nni, can not be unregistered!'.format(name)) + return + if remove_algo_meta_data(name): + print_green('{} unregistered sucessfully!'.format(name)) + else: + print_error('Failed to unregistered {}!'.format(name)) + +def algo_show(args): + builtin_name = args.name[0] + meta = get_registered_algo_meta(builtin_name) + if meta: + print(json.dumps(meta, indent=4)) + else: + print_error('package {} not found'.format(builtin_name)) + +def algo_list(args): + meta = read_registerd_algo_meta() + print('+-----------------+------------+-----------+--------=-------------+------------------------------------------+') + print('| Name | Type | source | Class Name | Module Name |') + print('+-----------------+------------+-----------+----------------------+------------------------------------------+') + MAX_MODULE_NAME = 38 + for t in ['tuners', 'assessors', 'advisors']: + for p in meta[t]: + module_name = '.'.join(p['className'].split('.')[:-1]) + if len(module_name) > MAX_MODULE_NAME: + module_name = module_name[:MAX_MODULE_NAME-3] + '...' + class_name = p['className'].split('.')[-1] + print('| {:15s} | {:10s} | {:9s} | {:20s} | {:40s} |'.format(p['builtinName'], t, p['source'], class_name, module_name[:38])) + print('+-----------------+------------+-----------+----------------------+------------------------------------------+') + + +def save_algo_meta_data(meta_data): + meta_data['source'] = 'user' + config = read_registerd_algo_meta() + config[meta_data['algoType']+'s'].append(meta_data) + write_registered_algo_meta(config) + +def remove_algo_meta_data(name): + config = read_registerd_algo_meta() + + updated = False + for t in ALGO_TYPES: + for meta in config[t]: + if meta['builtinName'] == name: + config[t].remove(meta) + updated = True + if updated: + write_registered_algo_meta(config) + return True + return False diff --git a/utils/third_party/nni_new/tools/nnictl/command_utils.py b/utils/third_party/nni_new/tools/nnictl/command_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..2bbcc883d1cef450fef9b723dc8ca71d34daa095 --- /dev/null +++ b/utils/third_party/nni_new/tools/nnictl/command_utils.py @@ -0,0 +1,82 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from subprocess import call, check_output +import sys +import os +import signal +import psutil +from .common_utils import print_error + + +def check_output_command(file_path, head=None, tail=None): + """call check_output command to read content from a file""" + if os.path.exists(file_path): + if sys.platform == 'win32': + cmds = ['powershell.exe', 'type', file_path] + if head: + cmds += ['|', 'select', '-first', str(head)] + elif tail: + cmds += ['|', 'select', '-last', str(tail)] + return check_output(cmds, shell=True).decode('utf-8') + else: + cmds = ['cat', file_path] + if head: + cmds = ['head', '-' + str(head), file_path] + elif tail: + cmds = ['tail', '-' + str(tail), file_path] + return check_output(cmds, shell=False).decode('utf-8') + else: + print_error('{0} does not exist!'.format(file_path)) + exit(1) + + +def kill_command(pid): + """kill command""" + if sys.platform == 'win32': + process = psutil.Process(pid=pid) + process.send_signal(signal.CTRL_BREAK_EVENT) + else: + cmds = ['kill', str(pid)] + call(cmds) + + +def install_package_command(package_name): + """ + Install python package from pip. + + Parameters + ---------- + package_name: str + The name of package to be installed. + """ + call(_get_pip_install() + [package_name], shell=False) + + +def install_requirements_command(requirements_path): + """ + Install packages from `requirements.txt` in `requirements_path`. + + Parameters + ---------- + requirements_path: str + Path to the directory that contains `requirements.txt`. + """ + return call(_get_pip_install() + ["-r", requirements_path], shell=False) + + +def _get_pip_install(): + python = "python" if sys.platform == "win32" else "python3" + ret = [python, "-m", "pip", "install"] + if "CONDA_DEFAULT_ENV" not in os.environ and "VIRTUAL_ENV" not in os.environ and \ + (sys.platform != "win32" and os.getuid() != 0): # on unix and not running in root + ret.append("--user") # not in virtualenv or conda + return ret + +def call_pip_install(source): + return call(_get_pip_install() + [source]) + +def call_pip_uninstall(module_name): + python = "python" if sys.platform == "win32" else "python3" + cmd = [python, "-m", "pip", "uninstall", module_name] + return call(cmd) diff --git a/utils/third_party/nni_new/tools/nnictl/common_utils.py b/utils/third_party/nni_new/tools/nnictl/common_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..25df67cf2e11297ccbf02223432a72c01768868c --- /dev/null +++ b/utils/third_party/nni_new/tools/nnictl/common_utils.py @@ -0,0 +1,125 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +import sys +import json +import tempfile +import time +import socket +import string +import random +import glob +from colorama import Fore +import filelock +import psutil +import yaml + +from .constants import ERROR_INFO, NORMAL_INFO, WARNING_INFO + +def get_yml_content(file_path): + '''Load yaml file content''' + try: + with open(file_path, 'r') as file: + return yaml.safe_load(file) + except yaml.scanner.ScannerError as err: + print_error('yaml file format error!') + print_error(err) + exit(1) + except Exception as exception: + print_error(exception) + exit(1) + +def get_json_content(file_path): + '''Load json file content''' + try: + with open(file_path, 'r') as file: + return json.load(file) + except TypeError as err: + print_error('json file format error!') + print_error(err) + return None + + +def print_error(*content): + '''Print error information to screen''' + print(Fore.RED + ERROR_INFO + ' '.join([str(c) for c in content]) + Fore.RESET) + +def print_green(*content): + '''Print information to screen in green''' + print(Fore.GREEN + ' '.join([str(c) for c in content]) + Fore.RESET) + +def print_normal(*content): + '''Print error information to screen''' + print(NORMAL_INFO, *content) + +def print_warning(*content): + '''Print warning information to screen''' + print(Fore.YELLOW + WARNING_INFO + ' '.join([str(c) for c in content]) + Fore.RESET) + +def detect_process(pid): + '''Detect if a process is alive''' + try: + process = psutil.Process(pid) + return process.is_running() + except: + return False + +def detect_port(port): + '''Detect if the port is used''' + socket_test = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + try: + socket_test.connect(('127.0.0.1', int(port))) + socket_test.close() + return True + except: + return False + +def get_user(): + if sys.platform == 'win32': + return os.environ['USERNAME'] + else: + return os.environ['USER'] + +def generate_temp_dir(): + '''generate a temp folder''' + def generate_folder_name(): + return os.path.join(tempfile.gettempdir(), 'nni', ''.join(random.sample(string.ascii_letters + string.digits, 8))) + temp_dir = generate_folder_name() + while os.path.exists(temp_dir): + temp_dir = generate_folder_name() + os.makedirs(temp_dir) + return temp_dir + +class SimplePreemptiveLock(filelock.SoftFileLock): + '''this is a lock support check lock expiration, if you do not need check expiration, you can use SoftFileLock''' + def __init__(self, lock_file, stale=-1): + super(__class__, self).__init__(lock_file, timeout=-1) + self._lock_file_name = '{}.{}'.format(self._lock_file, os.getpid()) + self._stale = stale + + def _acquire(self): + open_mode = os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_TRUNC + try: + lock_file_names = glob.glob(self._lock_file + '.*') + for file_name in lock_file_names: + if os.path.exists(file_name) and (self._stale < 0 or time.time() - os.stat(file_name).st_mtime < self._stale): + return None + fd = os.open(self._lock_file_name, open_mode) + except (IOError, OSError): + pass + else: + self._lock_file_fd = fd + return None + + def _release(self): + os.close(self._lock_file_fd) + self._lock_file_fd = None + try: + os.remove(self._lock_file_name) + except OSError: + pass + return None + +def get_file_lock(path: string, stale=-1): + return SimplePreemptiveLock(path + '.lock', stale=stale) diff --git a/utils/third_party/nni_new/tools/nnictl/config_schema.py b/utils/third_party/nni_new/tools/nnictl/config_schema.py new file mode 100644 index 0000000000000000000000000000000000000000..65e645b560bf862ff37be5a9f28f5a9e4f7f7e2e --- /dev/null +++ b/utils/third_party/nni_new/tools/nnictl/config_schema.py @@ -0,0 +1,641 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import json +import logging +import os + +import netifaces +from schema import And, Optional, Or, Regex, Schema, SchemaError +from nni.tools.package_utils import ( + create_validator_instance, + get_all_builtin_names, + get_registered_algo_meta, +) + +from .common_utils import get_yml_content, print_warning +from .constants import SCHEMA_PATH_ERROR, SCHEMA_RANGE_ERROR, SCHEMA_TYPE_ERROR + + +def setType(key, valueType): + '''check key type''' + return And(valueType, error=SCHEMA_TYPE_ERROR % (key, valueType.__name__)) + + +def setChoice(key, *args): + '''check choice''' + return And(lambda n: n in args, error=SCHEMA_RANGE_ERROR % (key, str(args))) + + +def setNumberRange(key, keyType, start, end): + '''check number range''' + return And( + And(keyType, error=SCHEMA_TYPE_ERROR % (key, keyType.__name__)), + And(lambda n: start <= n <= end, error=SCHEMA_RANGE_ERROR % (key, '(%s,%s)' % (start, end))), + ) + + +def setPathCheck(key): + '''check if path exist''' + return And(os.path.exists, error=SCHEMA_PATH_ERROR % key) + + +class AlgoSchema: + """ + This class is the schema of 'tuner', 'assessor' and 'advisor' sections of experiment configuraion file. + For example: + AlgoSchema('tuner') creates the schema of tuner section. + """ + + def __init__(self, algo_type): + """ + Parameters: + ----------- + algo_type: str + One of ['tuner', 'assessor', 'advisor']. + 'tuner': This AlgoSchema class create the schema of tuner section. + 'assessor': This AlgoSchema class create the schema of assessor section. + 'advisor': This AlgoSchema class create the schema of advisor section. + """ + assert algo_type in ['tuner', 'assessor', 'advisor'] + self.algo_type = algo_type + self.algo_schema = { + Optional('codeDir'): setPathCheck('codeDir'), + Optional('classFileName'): setType('classFileName', str), + Optional('className'): setType('className', str), + Optional('classArgs'): dict, + Optional('includeIntermediateResults'): setType('includeIntermediateResults', bool), + Optional('gpuIndices'): Or(int, And(str, lambda x: len([int(i) for i in x.split(',')]) > 0), error='gpuIndex format error!'), + } + self.builtin_keys = { + 'tuner': 'builtinTunerName', + 'assessor': 'builtinAssessorName', + 'advisor': 'builtinAdvisorName' + } + self.builtin_name_schema = {} + for k, n in self.builtin_keys.items(): + self.builtin_name_schema[k] = {Optional(n): setChoice(n, *get_all_builtin_names(k+'s'))} + + self.customized_keys = set(['codeDir', 'classFileName', 'className']) + + def validate_class_args(self, class_args, algo_type, builtin_name): + if not builtin_name or not class_args: + return + meta = get_registered_algo_meta(builtin_name, algo_type+'s') + if meta and 'acceptClassArgs' in meta and meta['acceptClassArgs'] == False: + raise SchemaError('classArgs is not allowed.') + + logging.getLogger('nni.protocol').setLevel(logging.ERROR) # we know IPC is not there, don't complain + validator = create_validator_instance(algo_type+'s', builtin_name) + if validator: + try: + validator.validate_class_args(**class_args) + except Exception as e: + raise SchemaError(str(e)) + + def missing_customized_keys(self, data): + return self.customized_keys - set(data.keys()) + + def validate_extras(self, data, algo_type): + builtin_key = self.builtin_keys[algo_type] + if (builtin_key in data) and (set(data.keys()) & self.customized_keys): + raise SchemaError('{} and {} cannot be specified at the same time.'.format( + builtin_key, set(data.keys()) & self.customized_keys + )) + + if self.missing_customized_keys(data) and builtin_key not in data: + raise SchemaError('Either customized {} ({}) or builtin {} ({}) must be set.'.format( + algo_type, self.customized_keys, algo_type, builtin_key)) + + if not self.missing_customized_keys(data): + class_file_name = os.path.join(data['codeDir'], data['classFileName']) + if not os.path.isfile(class_file_name): + raise SchemaError('classFileName {} not found.'.format(class_file_name)) + + builtin_name = data.get(builtin_key) + class_args = data.get('classArgs') + self.validate_class_args(class_args, algo_type, builtin_name) + + def validate(self, data): + self.algo_schema.update(self.builtin_name_schema[self.algo_type]) + Schema(self.algo_schema).validate(data) + self.validate_extras(data, self.algo_type) + + +common_schema = { + 'authorName': setType('authorName', str), + 'experimentName': setType('experimentName', str), + Optional('description'): setType('description', str), + 'trialConcurrency': setNumberRange('trialConcurrency', int, 1, 99999), + Optional('maxExecDuration'): And(Regex(r'^[1-9][0-9]*[s|m|h|d]$', error='ERROR: maxExecDuration format is [digit]{s,m,h,d}')), + Optional('maxTrialNum'): setNumberRange('maxTrialNum', int, 1, 99999), + 'trainingServicePlatform': setChoice( + 'trainingServicePlatform', 'remote', 'local', 'pai', 'kubeflow', 'frameworkcontroller', 'dlts', 'aml', 'adl', 'hybrid'), + Optional('searchSpacePath'): And(os.path.exists, error=SCHEMA_PATH_ERROR % 'searchSpacePath'), + Optional('multiPhase'): setType('multiPhase', bool), + Optional('multiThread'): setType('multiThread', bool), + Optional('nniManagerIp'): setType('nniManagerIp', str), + Optional('logDir'): And(os.path.isdir, error=SCHEMA_PATH_ERROR % 'logDir'), + Optional('debug'): setType('debug', bool), + Optional('versionCheck'): setType('versionCheck', bool), + Optional('logLevel'): setChoice('logLevel', 'trace', 'debug', 'info', 'warning', 'error', 'fatal'), + Optional('logCollection'): setChoice('logCollection', 'http', 'none'), + 'useAnnotation': setType('useAnnotation', bool), + Optional('tuner'): AlgoSchema('tuner'), + Optional('advisor'): AlgoSchema('advisor'), + Optional('assessor'): AlgoSchema('assessor'), + Optional('localConfig'): { + Optional('gpuIndices'): Or(int, And(str, lambda x: len([int(i) for i in x.split(',')]) > 0), error='gpuIndex format error!'), + Optional('maxTrialNumPerGpu'): setType('maxTrialNumPerGpu', int), + Optional('useActiveGpu'): setType('useActiveGpu', bool) + }, + Optional('sharedStorage'): { + 'storageType': setChoice('storageType', 'NFS', 'AzureBlob'), + Optional('localMountPoint'): setType('localMountPoint', str), + Optional('remoteMountPoint'): setType('remoteMountPoint', str), + Optional('nfsServer'): setType('nfsServer', str), + Optional('exportedDirectory'): setType('exportedDirectory', str), + Optional('storageAccountName'): setType('storageAccountName', str), + Optional('storageAccountKey'): setType('storageAccountKey', str), + Optional('containerName'): setType('containerName', str), + Optional('resourceGroupName'): setType('resourceGroupName', str), + Optional('localMounted'): setChoice('localMounted', 'usermount', 'nnimount', 'nomount') + } +} + +common_trial_schema = { + 'trial': { + 'command': setType('command', str), + 'codeDir': setPathCheck('codeDir'), + Optional('gpuNum'): setNumberRange('gpuNum', int, 0, 99999), + Optional('nasMode'): setChoice('nasMode', 'classic_mode', 'enas_mode', 'oneshot_mode', 'darts_mode') + } +} + +pai_yarn_trial_schema = { + 'trial': { + 'command': setType('command', str), + 'codeDir': setPathCheck('codeDir'), + 'gpuNum': setNumberRange('gpuNum', int, 0, 99999), + 'cpuNum': setNumberRange('cpuNum', int, 0, 99999), + 'memoryMB': setType('memoryMB', int), + 'image': setType('image', str), + Optional('authFile'): And(os.path.exists, error=SCHEMA_PATH_ERROR % 'authFile'), + Optional('shmMB'): setType('shmMB', int), + Optional('dataDir'): And(Regex(r'hdfs://(([0-9]{1,3}.){3}[0-9]{1,3})(:[0-9]{2,5})?(/.*)?'), + error='ERROR: dataDir format error, dataDir format is hdfs://xxx.xxx.xxx.xxx:xxx'), + Optional('outputDir'): And(Regex(r'hdfs://(([0-9]{1,3}.){3}[0-9]{1,3})(:[0-9]{2,5})?(/.*)?'), + error='ERROR: outputDir format error, outputDir format is hdfs://xxx.xxx.xxx.xxx:xxx'), + Optional('virtualCluster'): setType('virtualCluster', str), + Optional('nasMode'): setChoice('nasMode', 'classic_mode', 'enas_mode', 'oneshot_mode', 'darts_mode'), + Optional('portList'): [{ + 'label': setType('label', str), + 'beginAt': setType('beginAt', int), + 'portNumber': setType('portNumber', int) + }] + } +} + + +pai_trial_schema = { + 'trial': { + 'codeDir': setPathCheck('codeDir'), + 'nniManagerNFSMountPath': setPathCheck('nniManagerNFSMountPath'), + 'containerNFSMountPath': setType('containerNFSMountPath', str), + Optional('command'): setType('command', str), + Optional('gpuNum'): setNumberRange('gpuNum', int, 0, 99999), + Optional('cpuNum'): setNumberRange('cpuNum', int, 0, 99999), + Optional('memoryMB'): setType('memoryMB', int), + Optional('image'): setType('image', str), + Optional('virtualCluster'): setType('virtualCluster', str), + Optional('paiStorageConfigName'): setType('paiStorageConfigName', str), + Optional('paiConfigPath'): And(os.path.exists, error=SCHEMA_PATH_ERROR % 'paiConfigPath') + } +} + +pai_config_schema = { + Optional('paiConfig'): { + 'userName': setType('userName', str), + Or('passWord', 'token', only_one=True): str, + 'host': setType('host', str), + Optional('reuse'): setType('reuse', bool), + Optional('gpuNum'): setNumberRange('gpuNum', int, 0, 99999), + Optional('cpuNum'): setNumberRange('cpuNum', int, 0, 99999), + Optional('memoryMB'): setType('memoryMB', int), + Optional('maxTrialNumPerGpu'): setType('maxTrialNumPerGpu', int), + Optional('useActiveGpu'): setType('useActiveGpu', bool), + } +} + +dlts_trial_schema = { + 'trial': { + 'command': setType('command', str), + 'codeDir': setPathCheck('codeDir'), + 'gpuNum': setNumberRange('gpuNum', int, 0, 99999), + 'image': setType('image', str), + } +} + +dlts_config_schema = { + 'dltsConfig': { + 'dashboard': setType('dashboard', str), + + Optional('cluster'): setType('cluster', str), + Optional('team'): setType('team', str), + + Optional('email'): setType('email', str), + Optional('password'): setType('password', str), + } +} + +aml_trial_schema = { + 'trial': { + 'codeDir': setPathCheck('codeDir'), + 'command': setType('command', str), + 'image': setType('image', str), + Optional('gpuNum'): setNumberRange('gpuNum', int, 0, 99999), + } +} + +aml_config_schema = { + Optional('amlConfig'): { + 'subscriptionId': setType('subscriptionId', str), + 'resourceGroup': setType('resourceGroup', str), + 'workspaceName': setType('workspaceName', str), + 'computeTarget': setType('computeTarget', str), + Optional('maxTrialNumPerGpu'): setType('maxTrialNumPerGpu', int), + Optional('useActiveGpu'): setType('useActiveGpu', bool), + } +} + +hybrid_trial_schema = { + 'trial': { + 'codeDir': setPathCheck('codeDir'), + Optional('nniManagerNFSMountPath'): setPathCheck('nniManagerNFSMountPath'), + Optional('containerNFSMountPath'): setType('containerNFSMountPath', str), + Optional('nasMode'): setChoice('nasMode', 'classic_mode', 'enas_mode', 'oneshot_mode', 'darts_mode'), + 'command': setType('command', str), + Optional('gpuNum'): setNumberRange('gpuNum', int, 0, 99999), + Optional('cpuNum'): setNumberRange('cpuNum', int, 0, 99999), + Optional('memoryMB'): setType('memoryMB', int), + Optional('image'): setType('image', str), + Optional('virtualCluster'): setType('virtualCluster', str), + Optional('paiStorageConfigName'): setType('paiStorageConfigName', str), + Optional('paiConfigPath'): And(os.path.exists, error=SCHEMA_PATH_ERROR % 'paiConfigPath') + } +} + +hybrid_config_schema = { + 'hybridConfig': { + 'trainingServicePlatforms': ['local', 'remote', 'pai', 'aml'] + } +} + +adl_trial_schema = { + 'trial':{ + 'codeDir': setType('codeDir', str), + 'command': setType('command', str), + 'gpuNum': setNumberRange('gpuNum', int, 0, 99999), + 'image': setType('image', str), + Optional('namespace'): setType('namespace', str), + Optional('imagePullSecrets'): [{ + 'name': setType('name', str) + }], + Optional('nfs'): { + 'server': setType('server', str), + 'path': setType('path', str), + 'containerMountPath': setType('containerMountPath', str) + }, + Optional('adaptive'): setType('adaptive', bool), + Optional('checkpoint'): { + 'storageClass': setType('storageClass', str), + 'storageSize': setType('storageSize', str) + }, + Optional('cpuNum'): setNumberRange('cpuNum', int, 0, 99999), + Optional('memorySize'): setType('memorySize', str) + } +} + +kubeflow_trial_schema = { + 'trial': { + 'codeDir': setPathCheck('codeDir'), + Optional('nasMode'): setChoice('nasMode', 'classic_mode', 'enas_mode', 'oneshot_mode', 'darts_mode'), + Optional('ps'): { + 'replicas': setType('replicas', int), + 'command': setType('command', str), + 'gpuNum': setNumberRange('gpuNum', int, 0, 99999), + 'cpuNum': setNumberRange('cpuNum', int, 0, 99999), + 'memoryMB': setType('memoryMB', int), + 'image': setType('image', str), + Optional('privateRegistryAuthPath'): And(os.path.exists, error=SCHEMA_PATH_ERROR % 'privateRegistryAuthPath') + }, + Optional('master'): { + 'replicas': setType('replicas', int), + 'command': setType('command', str), + 'gpuNum': setNumberRange('gpuNum', int, 0, 99999), + 'cpuNum': setNumberRange('cpuNum', int, 0, 99999), + 'memoryMB': setType('memoryMB', int), + 'image': setType('image', str), + Optional('privateRegistryAuthPath'): And(os.path.exists, error=SCHEMA_PATH_ERROR % 'privateRegistryAuthPath') + }, + Optional('worker'): { + 'replicas': setType('replicas', int), + 'command': setType('command', str), + 'gpuNum': setNumberRange('gpuNum', int, 0, 99999), + 'cpuNum': setNumberRange('cpuNum', int, 0, 99999), + 'memoryMB': setType('memoryMB', int), + 'image': setType('image', str), + Optional('privateRegistryAuthPath'): And(os.path.exists, error=SCHEMA_PATH_ERROR % 'privateRegistryAuthPath') + } + } +} + +kubeflow_config_schema = { + 'kubeflowConfig': Or({ + 'operator': setChoice('operator', 'tf-operator', 'pytorch-operator'), + 'apiVersion': setType('apiVersion', str), + Optional('storage'): setChoice('storage', 'nfs', 'azureStorage'), + 'nfs': { + 'server': setType('server', str), + 'path': setType('path', str) + } + }, { + 'operator': setChoice('operator', 'tf-operator', 'pytorch-operator'), + 'apiVersion': setType('apiVersion', str), + Optional('storage'): setChoice('storage', 'nfs', 'azureStorage'), + 'keyVault': { + 'vaultName': And(Regex('([0-9]|[a-z]|[A-Z]|-){1,127}'), + error='ERROR: vaultName format error, vaultName support using (0-9|a-z|A-Z|-)'), + 'name': And(Regex('([0-9]|[a-z]|[A-Z]|-){1,127}'), + error='ERROR: name format error, name support using (0-9|a-z|A-Z|-)') + }, + 'azureStorage': { + 'accountName': And(Regex('([0-9]|[a-z]|[A-Z]|-){3,31}'), + error='ERROR: accountName format error, accountName support using (0-9|a-z|A-Z|-)'), + 'azureShare': And(Regex('([0-9]|[a-z]|[A-Z]|-){3,63}'), + error='ERROR: azureShare format error, azureShare support using (0-9|a-z|A-Z|-)') + }, + Optional('uploadRetryCount'): setNumberRange('uploadRetryCount', int, 1, 99999) + }) +} + +frameworkcontroller_trial_schema = { + 'trial': { + 'codeDir': setPathCheck('codeDir'), + Optional('taskRoles'): [{ + 'name': setType('name', str), + 'taskNum': setType('taskNum', int), + 'frameworkAttemptCompletionPolicy': { + 'minFailedTaskCount': setType('minFailedTaskCount', int), + 'minSucceededTaskCount': setType('minSucceededTaskCount', int), + }, + 'command': setType('command', str), + 'gpuNum': setNumberRange('gpuNum', int, 0, 99999), + 'cpuNum': setNumberRange('cpuNum', int, 0, 99999), + 'memoryMB': setType('memoryMB', int), + 'image': setType('image', str), + Optional('privateRegistryAuthPath'): And(os.path.exists, error=SCHEMA_PATH_ERROR % 'privateRegistryAuthPath') + }] + } +} + +frameworkcontroller_config_schema = { + 'frameworkcontrollerConfig': Or({ + Optional('storage'): setChoice('storage', 'nfs', 'azureStorage', 'pvc'), + Optional('serviceAccountName'): setType('serviceAccountName', str), + 'nfs': { + 'server': setType('server', str), + 'path': setType('path', str) + }, + Optional('namespace'): setType('namespace', str), + Optional('configPath'): setType('configPath', str), + }, { + Optional('storage'): setChoice('storage', 'nfs', 'azureStorage', 'pvc'), + Optional('serviceAccountName'): setType('serviceAccountName', str), + 'configPath': setType('configPath', str), + 'pvc': {'path': setType('server', str)}, + Optional('namespace'): setType('namespace', str), + }, { + Optional('storage'): setChoice('storage', 'nfs', 'azureStorage', 'pvc'), + Optional('serviceAccountName'): setType('serviceAccountName', str), + 'keyVault': { + 'vaultName': And(Regex('([0-9]|[a-z]|[A-Z]|-){1,127}'), + error='ERROR: vaultName format error, vaultName support using (0-9|a-z|A-Z|-)'), + 'name': And(Regex('([0-9]|[a-z]|[A-Z]|-){1,127}'), + error='ERROR: name format error, name support using (0-9|a-z|A-Z|-)') + }, + 'azureStorage': { + 'accountName': And(Regex('([0-9]|[a-z]|[A-Z]|-){3,31}'), + error='ERROR: accountName format error, accountName support using (0-9|a-z|A-Z|-)'), + 'azureShare': And(Regex('([0-9]|[a-z]|[A-Z]|-){3,63}'), + error='ERROR: azureShare format error, azureShare support using (0-9|a-z|A-Z|-)') + }, + Optional('uploadRetryCount'): setNumberRange('uploadRetryCount', int, 1, 99999), + Optional('namespace'): setType('namespace', str), + Optional('configPath'): setType('configPath', str), + }) +} + +remote_config_schema = { + Optional('remoteConfig'): { + 'reuse': setType('reuse', bool) + } +} + +machine_list_schema = { + Optional('machineList'): [Or( + { + 'ip': setType('ip', str), + Optional('port'): setNumberRange('port', int, 1, 65535), + 'username': setType('username', str), + 'sshKeyPath': setPathCheck('sshKeyPath'), + Optional('passphrase'): setType('passphrase', str), + Optional('gpuIndices'): Or(int, And(str, lambda x: len([int(i) for i in x.split(',')]) > 0), error='gpuIndex format error!'), + Optional('maxTrialNumPerGpu'): setType('maxTrialNumPerGpu', int), + Optional('useActiveGpu'): setType('useActiveGpu', bool), + Optional('pythonPath'): setType('pythonPath', str) + }, + { + 'ip': setType('ip', str), + Optional('port'): setNumberRange('port', int, 1, 65535), + 'username': setType('username', str), + 'passwd': setType('passwd', str), + Optional('gpuIndices'): Or(int, And(str, lambda x: len([int(i) for i in x.split(',')]) > 0), error='gpuIndex format error!'), + Optional('maxTrialNumPerGpu'): setType('maxTrialNumPerGpu', int), + Optional('useActiveGpu'): setType('useActiveGpu', bool), + Optional('pythonPath'): setType('pythonPath', str) + })] +} + +training_service_schema_dict = { + 'adl': Schema({**common_schema, **adl_trial_schema}), + 'local': Schema({**common_schema, **common_trial_schema}), + 'remote': Schema({**common_schema, **common_trial_schema, **machine_list_schema, **remote_config_schema}), + 'pai': Schema({**common_schema, **pai_trial_schema, **pai_config_schema}), + 'kubeflow': Schema({**common_schema, **kubeflow_trial_schema, **kubeflow_config_schema}), + 'frameworkcontroller': Schema({**common_schema, **frameworkcontroller_trial_schema, **frameworkcontroller_config_schema}), + 'aml': Schema({**common_schema, **aml_trial_schema, **aml_config_schema}), + 'dlts': Schema({**common_schema, **dlts_trial_schema, **dlts_config_schema}), + 'hybrid': Schema({**common_schema, **hybrid_trial_schema, **hybrid_config_schema, **machine_list_schema, + **pai_config_schema, **aml_config_schema, **remote_config_schema}), +} + + +class NNIConfigSchema: + def validate(self, data): + train_service = data['trainingServicePlatform'] + Schema(common_schema['trainingServicePlatform']).validate(train_service) + train_service_schema = training_service_schema_dict[train_service] + train_service_schema.validate(data) + self.validate_extras(data) + + def validate_extras(self, experiment_config): + self.validate_tuner_adivosr_assessor(experiment_config) + self.validate_pai_trial_conifg(experiment_config) + self.validate_kubeflow_operators(experiment_config) + self.validate_eth0_device(experiment_config) + self.validate_hybrid_platforms(experiment_config) + self.validate_frameworkcontroller_trial_config(experiment_config) + + def validate_tuner_adivosr_assessor(self, experiment_config): + if experiment_config.get('advisor'): + if experiment_config.get('assessor') or experiment_config.get('tuner'): + raise SchemaError('advisor could not be set with assessor or tuner simultaneously!') + self.validate_annotation_content(experiment_config, 'advisor', 'builtinAdvisorName') + else: + if not experiment_config.get('tuner'): + raise SchemaError('Please provide tuner spec!') + self.validate_annotation_content(experiment_config, 'tuner', 'builtinTunerName') + + def validate_search_space_content(self, experiment_config): + '''Validate searchspace content, + if the searchspace file is not json format or its values does not contain _type and _value which must be specified, + it will not be a valid searchspace file''' + try: + search_space_content = json.load(open(experiment_config.get('searchSpacePath'), 'r')) + for value in search_space_content.values(): + if not value.get('_type') or not value.get('_value'): + raise SchemaError('please use _type and _value to specify searchspace!') + except Exception as e: + raise SchemaError('searchspace file is not a valid json format! ' + str(e)) + + def validate_kubeflow_operators(self, experiment_config): + '''Validate whether the kubeflow operators are valid''' + if experiment_config.get('kubeflowConfig'): + if experiment_config.get('kubeflowConfig').get('operator') == 'tf-operator': + if experiment_config.get('trial').get('master') is not None: + raise SchemaError('kubeflow with tf-operator can not set master') + if experiment_config.get('trial').get('worker') is None: + raise SchemaError('kubeflow with tf-operator must set worker') + elif experiment_config.get('kubeflowConfig').get('operator') == 'pytorch-operator': + if experiment_config.get('trial').get('ps') is not None: + raise SchemaError('kubeflow with pytorch-operator can not set ps') + if experiment_config.get('trial').get('master') is None: + raise SchemaError('kubeflow with pytorch-operator must set master') + + if experiment_config.get('kubeflowConfig').get('storage') == 'nfs': + if experiment_config.get('kubeflowConfig').get('nfs') is None: + raise SchemaError('please set nfs configuration!') + elif experiment_config.get('kubeflowConfig').get('storage') == 'azureStorage': + if experiment_config.get('kubeflowConfig').get('azureStorage') is None: + raise SchemaError('please set azureStorage configuration!') + elif experiment_config.get('kubeflowConfig').get('storage') is None: + if experiment_config.get('kubeflowConfig').get('azureStorage'): + raise SchemaError('please set storage type!') + + def validate_annotation_content(self, experiment_config, spec_key, builtin_name): + ''' + Valid whether useAnnotation and searchSpacePath is coexist + spec_key: 'advisor' or 'tuner' + builtin_name: 'builtinAdvisorName' or 'builtinTunerName' + ''' + if experiment_config.get('useAnnotation'): + if experiment_config.get('searchSpacePath'): + raise SchemaError('If you set useAnnotation=true, please leave searchSpacePath empty') + else: + # validate searchSpaceFile + if experiment_config[spec_key].get(builtin_name) == 'NetworkMorphism': + return + if experiment_config[spec_key].get(builtin_name): + if experiment_config.get('searchSpacePath') is None: + raise SchemaError('Please set searchSpacePath!') + self.validate_search_space_content(experiment_config) + + def validate_pai_config_path(self, experiment_config): + '''validate paiConfigPath field''' + if experiment_config.get('trainingServicePlatform') == 'pai': + if experiment_config.get('trial', {}).get('paiConfigPath'): + # validate commands + pai_config = get_yml_content(experiment_config['trial']['paiConfigPath']) + taskRoles_dict = pai_config.get('taskRoles') + if not taskRoles_dict: + raise SchemaError('Please set taskRoles in paiConfigPath config file!') + else: + pai_trial_fields_required_list = ['image', 'paiStorageConfigName', 'command'] + for trial_field in pai_trial_fields_required_list: + if experiment_config['trial'].get(trial_field) is None: + raise SchemaError('Please set {0} in trial configuration,\ + or set additional pai configuration file path in paiConfigPath!'.format(trial_field)) + pai_resource_fields_required_list = ['gpuNum', 'cpuNum', 'memoryMB'] + for required_field in pai_resource_fields_required_list: + if experiment_config['trial'].get(required_field) is None and \ + experiment_config['paiConfig'].get(required_field) is None: + raise SchemaError('Please set {0} in trial or paiConfig configuration,\ + or set additional pai configuration file path in paiConfigPath!'.format(required_field)) + + def validate_pai_trial_conifg(self, experiment_config): + '''validate the trial config in pai platform''' + if experiment_config.get('trainingServicePlatform') in ['pai']: + if experiment_config.get('trial').get('shmMB') and \ + experiment_config['trial']['shmMB'] > experiment_config['trial']['memoryMB']: + raise SchemaError('shmMB should be no more than memoryMB!') + # backward compatibility + warning_information = '{0} is not supported in NNI anymore, please remove the field in config file!\ + please refer https://github.com/microsoft/nni/blob/master/docs/en_US/TrainingService/PaiMode.md#run-an-experiment\ + for the practices of how to get data and output model in trial code' + if experiment_config.get('trial').get('dataDir'): + print_warning(warning_information.format('dataDir')) + if experiment_config.get('trial').get('outputDir'): + print_warning(warning_information.format('outputDir')) + self.validate_pai_config_path(experiment_config) + + def validate_eth0_device(self, experiment_config): + '''validate whether the machine has eth0 device''' + if experiment_config.get('trainingServicePlatform') not in ['local'] \ + and not experiment_config.get('nniManagerIp') \ + and 'eth0' not in netifaces.interfaces(): + raise SchemaError('This machine does not contain eth0 network device, please set nniManagerIp in config file!') + + def validate_hybrid_platforms(self, experiment_config): + required_config_name_map = { + 'remote': 'machineList', + 'aml': 'amlConfig', + 'pai': 'paiConfig' + } + if experiment_config.get('trainingServicePlatform') == 'hybrid': + for platform in experiment_config['hybridConfig']['trainingServicePlatforms']: + config_name = required_config_name_map.get(platform) + if config_name and not experiment_config.get(config_name): + raise SchemaError('Need to set {0} for {1} in hybrid mode!'.format(config_name, platform)) + + def validate_frameworkcontroller_trial_config(self, experiment_config): + if experiment_config.get('trainingServicePlatform') == 'frameworkcontroller': + if not experiment_config.get('trial').get('taskRoles'): + if not experiment_config.get('frameworkcontrollerConfig').get('configPath'): + raise SchemaError("""If no taskRoles are specified a valid custom frameworkcontroller config should + be set using the configPath attribute in frameworkcontrollerConfig!""") + config_content = get_yml_content(experiment_config.get('frameworkcontrollerConfig').get('configPath')) + if not config_content.get('spec').get('taskRoles') or not config_content.get('spec').get('taskRoles'): + raise SchemaError('Invalid frameworkcontroller config! No taskRoles were specified!') + if not config_content.get('spec').get('taskRoles')[0].get('task'): + raise SchemaError('Invalid frameworkcontroller config! No task was specified for taskRole!') + names = [] + for taskRole in config_content.get('spec').get('taskRoles'): + if not "name" in taskRole: + raise SchemaError('Invalid frameworkcontroller config! Name is missing for taskRole!') + names.append(taskRole.get("name")) + if len(names) > len(set(names)): + raise SchemaError('Invalid frameworkcontroller config! Duplicate taskrole names!') + if not config_content.get('metadata').get('name'): + raise SchemaError('Invalid frameworkcontroller config! No experiment name was specified!') + diff --git a/utils/third_party/nni_new/tools/nnictl/config_utils.py b/utils/third_party/nni_new/tools/nnictl/config_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..78f884e6f08f0883112fe5317e29d4c5735c432a --- /dev/null +++ b/utils/third_party/nni_new/tools/nnictl/config_utils.py @@ -0,0 +1,169 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +import sqlite3 +import json_tricks +from .constants import NNI_HOME_DIR +from .common_utils import get_file_lock + +def config_v0_to_v1(config: dict) -> dict: + if 'clusterMetaData' not in config: + return config + elif 'trainingServicePlatform' in config: + import copy + experiment_config = copy.deepcopy(config) + if experiment_config['trainingServicePlatform'] == 'hybrid': + inverse_config = {'hybridConfig': experiment_config['clusterMetaData']['hybrid_config']} + platform_list = inverse_config['hybridConfig']['trainingServicePlatforms'] + for platform in platform_list: + inverse_config.update(_inverse_cluster_metadata(platform, experiment_config['clusterMetaData'])) + experiment_config.update(inverse_config) + else: + inverse_config = _inverse_cluster_metadata(experiment_config['trainingServicePlatform'], experiment_config['clusterMetaData']) + experiment_config.update(inverse_config) + experiment_config.pop('clusterMetaData') + return experiment_config + else: + raise RuntimeError('experiment config key `trainingServicePlatform` not found') + +def _inverse_cluster_metadata(platform: str, metadata_config: list) -> dict: + inverse_config = {} + if platform == 'local': + inverse_config['trial'] = {} + for kv in metadata_config: + if kv['key'] == 'local_config': + inverse_config['localConfig'] = kv['value'] + elif kv['key'] == 'trial_config': + inverse_config['trial'] = kv['value'] + elif platform == 'remote': + for kv in metadata_config: + if kv['key'] == 'machine_list': + inverse_config['machineList'] = kv['value'] + elif kv['key'] == 'trial_config': + inverse_config['trial'] = kv['value'] + elif kv['key'] == 'remote_config': + inverse_config['remoteConfig'] = kv['value'] + elif platform == 'pai': + for kv in metadata_config: + if kv['key'] == 'pai_config': + inverse_config['paiConfig'] = kv['value'] + elif kv['key'] == 'trial_config': + inverse_config['trial'] = kv['value'] + elif platform == 'kubeflow': + for kv in metadata_config: + if kv['key'] == 'kubeflow_config': + inverse_config['kubeflowConfig'] = kv['value'] + elif kv['key'] == 'trial_config': + inverse_config['trial'] = kv['value'] + elif platform == 'frameworkcontroller': + for kv in metadata_config: + if kv['key'] == 'frameworkcontroller_config': + inverse_config['frameworkcontrollerConfig'] = kv['value'] + elif kv['key'] == 'trial_config': + inverse_config['trial'] = kv['value'] + elif platform == 'aml': + for kv in metadata_config: + if kv['key'] == 'aml_config': + inverse_config['amlConfig'] = kv['value'] + elif kv['key'] == 'trial_config': + inverse_config['trial'] = kv['value'] + elif platform == 'adl': + for kv in metadata_config: + if kv['key'] == 'adl_config': + inverse_config['adlConfig'] = kv['value'] + elif kv['key'] == 'trial_config': + inverse_config['trial'] = kv['value'] + else: + raise RuntimeError('training service platform {} not found'.format(platform)) + return inverse_config + +class Config: + '''a util class to load and save config''' + def __init__(self, experiment_id: str, log_dir: str): + self.experiment_id = experiment_id + self.conn = sqlite3.connect(os.path.join(log_dir, experiment_id, 'db', 'nni.sqlite')) + self.refresh_config() + + def refresh_config(self): + '''refresh to get latest config''' + sql = 'select params from ExperimentProfile where id=? order by revision DESC' + args = (self.experiment_id,) + self.config = config_v0_to_v1(json_tricks.loads(self.conn.cursor().execute(sql, args).fetchone()[0])) + + def get_config(self): + '''get a value according to key''' + return self.config + +class Experiments: + '''Maintain experiment list''' + def __init__(self, home_dir=NNI_HOME_DIR): + os.makedirs(home_dir, exist_ok=True) + self.experiment_file = os.path.join(home_dir, '.experiment') + self.lock = get_file_lock(self.experiment_file, stale=2) + with self.lock: + self.experiments = self.read_file() + + def add_experiment(self, expId, port, startTime, platform, experiment_name, endTime='N/A', status='INITIALIZED', + tag=[], pid=None, webuiUrl=[], logDir='', prefixUrl=None): + '''set {key:value} pairs to self.experiment''' + with self.lock: + self.experiments = self.read_file() + self.experiments[expId] = {} + self.experiments[expId]['id'] = expId + self.experiments[expId]['port'] = port + self.experiments[expId]['startTime'] = startTime + self.experiments[expId]['endTime'] = endTime + self.experiments[expId]['status'] = status + self.experiments[expId]['platform'] = platform + self.experiments[expId]['experimentName'] = experiment_name + self.experiments[expId]['tag'] = tag + self.experiments[expId]['pid'] = pid + self.experiments[expId]['webuiUrl'] = webuiUrl + self.experiments[expId]['logDir'] = str(logDir) + self.experiments[expId]['prefixUrl'] = prefixUrl + self.write_file() + + def update_experiment(self, expId, key, value): + '''Update experiment''' + with self.lock: + self.experiments = self.read_file() + if expId not in self.experiments: + return False + if value is None: + self.experiments[expId].pop(key, None) + else: + self.experiments[expId][key] = value + self.write_file() + return True + + def remove_experiment(self, expId): + '''remove an experiment by id''' + with self.lock: + self.experiments = self.read_file() + if expId in self.experiments: + self.experiments.pop(expId) + self.write_file() + + def get_all_experiments(self): + '''return all of experiments''' + return self.experiments + + def write_file(self): + '''save config to local file''' + try: + with open(self.experiment_file, 'w') as file: + json_tricks.dump(self.experiments, file, indent=4) + except IOError as error: + print('Error:', error) + return '' + + def read_file(self): + '''load config from local file''' + if os.path.exists(self.experiment_file): + try: + with open(self.experiment_file, 'r') as file: + return json_tricks.load(file) + except ValueError: + return {} + return {} diff --git a/utils/third_party/nni_new/tools/nnictl/constants.py b/utils/third_party/nni_new/tools/nnictl/constants.py new file mode 100644 index 0000000000000000000000000000000000000000..120674bc5f3481b6474f85107aa99b81044bdd4f --- /dev/null +++ b/utils/third_party/nni_new/tools/nnictl/constants.py @@ -0,0 +1,79 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +from colorama import Fore + +NNI_HOME_DIR = os.path.join(os.path.expanduser('~'), 'nni-experiments') + +ERROR_INFO = 'ERROR: ' +NORMAL_INFO = 'INFO: ' +WARNING_INFO = 'WARNING: ' + +DEFAULT_REST_PORT = 8080 +REST_TIME_OUT = 20 + +EXPERIMENT_SUCCESS_INFO = Fore.GREEN + 'Successfully started experiment!\n' + Fore.RESET + \ + '------------------------------------------------------------------------------------\n' \ + 'The experiment id is %s\n'\ + 'The Web UI urls are: %s\n' \ + '------------------------------------------------------------------------------------\n\n' \ + 'You can use these commands to get more information about the experiment\n' \ + '------------------------------------------------------------------------------------\n' \ + ' commands description\n' \ + '1. nnictl experiment show show the information of experiments\n' \ + '2. nnictl trial ls list all of trial jobs\n' \ + '3. nnictl top monitor the status of running experiments\n' \ + '4. nnictl log stderr show stderr log content\n' \ + '5. nnictl log stdout show stdout log content\n' \ + '6. nnictl stop stop an experiment\n' \ + '7. nnictl trial kill kill a trial job by id\n' \ + '8. nnictl --help get help information about nnictl\n' \ + '------------------------------------------------------------------------------------\n' \ + 'Command reference document https://nni.readthedocs.io/en/latest/Tutorial/Nnictl.html\n' \ + '------------------------------------------------------------------------------------\n' + +LOG_HEADER = '-----------------------------------------------------------------------\n' \ + ' Experiment start time %s\n' \ + '-----------------------------------------------------------------------\n' + +EXPERIMENT_START_FAILED_INFO = 'There is an experiment running in the port %d, please stop it first or set another port!\n' \ + 'You could use \'nnictl stop --port [PORT]\' command to stop an experiment!\nOr you could ' \ + 'use \'nnictl create --config [CONFIG_PATH] --port [PORT]\' to set port!\n' + +EXPERIMENT_INFORMATION_FORMAT = '----------------------------------------------------------------------------------------\n' \ + ' Experiment information\n' \ + '%s\n' \ + '----------------------------------------------------------------------------------------\n' + +EXPERIMENT_DETAIL_FORMAT = 'Id: %s Name: %s Status: %s Port: %s Platform: %s StartTime: %s EndTime: %s\n' + +EXPERIMENT_MONITOR_INFO = 'Id: %s Status: %s Port: %s Platform: %s \n' \ + 'StartTime: %s Duration: %s' + +TRIAL_MONITOR_HEAD = '-------------------------------------------------------------------------------------\n' + \ + '%-15s %-25s %-25s %-15s \n' % ('trialId', 'startTime', 'endTime', 'status') + \ + '-------------------------------------------------------------------------------------' + +TRIAL_MONITOR_CONTENT = '%-15s %-25s %-25s %-15s' + +TRIAL_MONITOR_TAIL = '-------------------------------------------------------------------------------------\n\n\n' + +TUNERS_SUPPORTING_IMPORT_DATA = { + 'TPE', + 'Anneal', + 'GridSearch', + 'MetisTuner', + 'BOHB', + 'SMAC', + 'BatchTuner' +} + +TUNERS_NO_NEED_TO_IMPORT_DATA = { + 'Random', + 'Hyperband' +} + +SCHEMA_TYPE_ERROR = '%s should be %s type!' +SCHEMA_RANGE_ERROR = '%s should be in range of %s!' +SCHEMA_PATH_ERROR = '%s path not exist!' diff --git a/utils/third_party/nni_new/tools/nnictl/launcher.py b/utils/third_party/nni_new/tools/nnictl/launcher.py new file mode 100644 index 0000000000000000000000000000000000000000..9144bc02678de02739994b8a5c05f7b085c1d9ad --- /dev/null +++ b/utils/third_party/nni_new/tools/nnictl/launcher.py @@ -0,0 +1,576 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import json +import os +from pathlib import Path +import sys +import string +import random +import time +import tempfile +import re +from subprocess import Popen, check_call, CalledProcessError, PIPE, STDOUT +from nni.experiment.config import ExperimentConfig, convert +from nni.tools.annotation import expand_annotations, generate_search_space +from nni.tools.package_utils import get_builtin_module_class_name +import nni_node # pylint: disable=import-error +from .launcher_utils import validate_all_content +from .rest_utils import rest_put, rest_post, check_rest_server, check_response +from .url_utils import cluster_metadata_url, experiment_url, get_local_urls, set_prefix_url +from .config_utils import Config, Experiments +from .common_utils import get_yml_content, get_json_content, print_error, print_normal, detect_port, get_user + +from .constants import NNI_HOME_DIR, ERROR_INFO, REST_TIME_OUT, EXPERIMENT_SUCCESS_INFO, LOG_HEADER +from .command_utils import check_output_command, kill_command +from .nnictl_utils import update_experiment + +k8s_training_services = ['kubeflow', 'frameworkcontroller', 'adl'] + +def get_log_path(experiment_id): + '''generate stdout and stderr log path''' + os.makedirs(os.path.join(NNI_HOME_DIR, experiment_id, 'log'), exist_ok=True) + stdout_full_path = os.path.join(NNI_HOME_DIR, experiment_id, 'log', 'nnictl_stdout.log') + stderr_full_path = os.path.join(NNI_HOME_DIR, experiment_id, 'log', 'nnictl_stderr.log') + return stdout_full_path, stderr_full_path + +def print_log_content(config_file_name): + '''print log information''' + stdout_full_path, stderr_full_path = get_log_path(config_file_name) + print_normal(' Stdout:') + print(check_output_command(stdout_full_path)) + print('\n\n') + print_normal(' Stderr:') + print(check_output_command(stderr_full_path)) + +def start_rest_server(port, platform, mode, experiment_id, foreground=False, log_dir=None, log_level=None, url_prefix=None): + '''Run nni manager process''' + if detect_port(port): + print_error('Port %s is used by another process, please reset the port!\n' \ + 'You could use \'nnictl create --help\' to get help information' % port) + exit(1) + + if (platform not in ['local', 'aml']) and detect_port(int(port) + 1): + print_error('%s mode need an additional adjacent port %d, and the port %d is used by another process!\n' \ + 'You could set another port to start experiment!\n' \ + 'You could use \'nnictl create --help\' to get help information' % (platform, (int(port) + 1), (int(port) + 1))) + exit(1) + + print_normal('Starting restful server...') + + entry_dir = nni_node.__path__[0] + if (not entry_dir) or (not os.path.exists(entry_dir)): + print_error('Fail to find nni under python library') + exit(1) + entry_file = os.path.join(entry_dir, 'main.js') + + if sys.platform == 'win32': + node_command = os.path.join(entry_dir, 'node.exe') + else: + node_command = os.path.join(entry_dir, 'node') + cmds = [node_command, '--max-old-space-size=4096', entry_file, '--port', str(port), '--mode', platform, \ + '--experiment_id', experiment_id] + if mode == 'view': + cmds += ['--start_mode', 'resume'] + cmds += ['--readonly', 'true'] + else: + cmds += ['--start_mode', mode] + if log_dir is not None: + cmds += ['--log_dir', log_dir] + if log_level is not None: + cmds += ['--log_level', log_level] + if foreground: + cmds += ['--foreground', 'true'] + if url_prefix: + _validate_prefix_path(url_prefix) + set_prefix_url(url_prefix) + cmds += ['--url_prefix', url_prefix] + + stdout_full_path, stderr_full_path = get_log_path(experiment_id) + with open(stdout_full_path, 'a+') as stdout_file, open(stderr_full_path, 'a+') as stderr_file: + start_time = time.time() + time_now = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(start_time)) + #add time information in the header of log files + log_header = LOG_HEADER % str(time_now) + stdout_file.write(log_header) + stderr_file.write(log_header) + if sys.platform == 'win32': + from subprocess import CREATE_NEW_PROCESS_GROUP + if foreground: + process = Popen(cmds, cwd=entry_dir, stdout=PIPE, stderr=STDOUT, creationflags=CREATE_NEW_PROCESS_GROUP) + else: + process = Popen(cmds, cwd=entry_dir, stdout=stdout_file, stderr=stderr_file, creationflags=CREATE_NEW_PROCESS_GROUP) + else: + if foreground: + process = Popen(cmds, cwd=entry_dir, stdout=PIPE, stderr=PIPE) + else: + process = Popen(cmds, cwd=entry_dir, stdout=stdout_file, stderr=stderr_file) + return process, int(start_time * 1000) + +def set_trial_config(experiment_config, port, config_file_name): + '''set trial configuration''' + request_data = dict() + request_data['trial_config'] = experiment_config['trial'] + response = rest_put(cluster_metadata_url(port), json.dumps(request_data), REST_TIME_OUT) + if check_response(response): + return True + else: + print('Error message is {}'.format(response.text)) + _, stderr_full_path = get_log_path(config_file_name) + if response: + with open(stderr_full_path, 'a+') as fout: + fout.write(json.dumps(json.loads(response.text), indent=4, sort_keys=True, separators=(',', ':'))) + return False + +def set_adl_config(experiment_config, port, config_file_name): + '''set adl configuration''' + adl_config_data = dict() + # hack for supporting v2 config, need refactor + adl_config_data['adl_config'] = {} + response = rest_put(cluster_metadata_url(port), json.dumps(adl_config_data), REST_TIME_OUT) + err_message = None + if not response or not response.status_code == 200: + if response is not None: + err_message = response.text + _, stderr_full_path = get_log_path(config_file_name) + with open(stderr_full_path, 'a+') as fout: + fout.write(json.dumps(json.loads(err_message), indent=4, sort_keys=True, separators=(',', ':'))) + return False, err_message + set_V1_common_config(experiment_config, port, config_file_name) + result, message = setNNIManagerIp(experiment_config, port, config_file_name) + if not result: + return result, message + #set trial_config + return set_trial_config(experiment_config, port, config_file_name), None + +def validate_response(response, config_file_name): + err_message = None + if not response or not response.status_code == 200: + if response is not None: + err_message = response.text + _, stderr_full_path = get_log_path(config_file_name) + with open(stderr_full_path, 'a+') as fout: + fout.write(json.dumps(json.loads(err_message), indent=4, sort_keys=True, separators=(',', ':'))) + print_error('Error:' + err_message) + exit(1) + +# hack to fix v1 version_check and log_collection bug, need refactor +def set_V1_common_config(experiment_config, port, config_file_name): + version_check = True + #debug mode should disable version check + if experiment_config.get('debug') is not None: + version_check = not experiment_config.get('debug') + #validate version check + if experiment_config.get('versionCheck') is not None: + version_check = experiment_config.get('versionCheck') + response = rest_put(cluster_metadata_url(port), json.dumps({'version_check': version_check}), REST_TIME_OUT) + validate_response(response, config_file_name) + if experiment_config.get('logCollection'): + data = json.dumps({'log_collection': experiment_config.get('logCollection')}) + response = rest_put(cluster_metadata_url(port), data, REST_TIME_OUT) + validate_response(response, config_file_name) + +def setNNIManagerIp(experiment_config, port, config_file_name): + '''set nniManagerIp''' + if experiment_config.get('nniManagerIp') is None: + return True, None + ip_config_dict = dict() + ip_config_dict['nni_manager_ip'] = {'nniManagerIp': experiment_config['nniManagerIp']} + response = rest_put(cluster_metadata_url(port), json.dumps(ip_config_dict), REST_TIME_OUT) + err_message = None + if not response or not response.status_code == 200: + if response is not None: + err_message = response.text + _, stderr_full_path = get_log_path(config_file_name) + with open(stderr_full_path, 'a+') as fout: + fout.write(json.dumps(json.loads(err_message), indent=4, sort_keys=True, separators=(',', ':'))) + return False, err_message + return True, None + +def set_kubeflow_config(experiment_config, port, config_file_name): + '''set kubeflow configuration''' + kubeflow_config_data = dict() + kubeflow_config_data['kubeflow_config'] = experiment_config['kubeflowConfig'] + response = rest_put(cluster_metadata_url(port), json.dumps(kubeflow_config_data), REST_TIME_OUT) + err_message = None + if not response or not response.status_code == 200: + if response is not None: + err_message = response.text + _, stderr_full_path = get_log_path(config_file_name) + with open(stderr_full_path, 'a+') as fout: + fout.write(json.dumps(json.loads(err_message), indent=4, sort_keys=True, separators=(',', ':'))) + return False, err_message + set_V1_common_config(experiment_config, port, config_file_name) + result, message = setNNIManagerIp(experiment_config, port, config_file_name) + if not result: + return result, message + #set trial_config + return set_trial_config(experiment_config, port, config_file_name), err_message + +def set_frameworkcontroller_config(experiment_config, port, config_file_name): + '''set kubeflow configuration''' + frameworkcontroller_config_data = dict() + frameworkcontroller_config_data['frameworkcontroller_config'] = experiment_config['frameworkcontrollerConfig'] + response = rest_put(cluster_metadata_url(port), json.dumps(frameworkcontroller_config_data), REST_TIME_OUT) + err_message = None + if not response or not response.status_code == 200: + if response is not None: + err_message = response.text + _, stderr_full_path = get_log_path(config_file_name) + with open(stderr_full_path, 'a+') as fout: + fout.write(json.dumps(json.loads(err_message), indent=4, sort_keys=True, separators=(',', ':'))) + return False, err_message + set_V1_common_config(experiment_config, port, config_file_name) + result, message = setNNIManagerIp(experiment_config, port, config_file_name) + if not result: + return result, message + #set trial_config + return set_trial_config(experiment_config, port, config_file_name), err_message + +def set_shared_storage(experiment_config, port, config_file_name): + if 'sharedStorage' in experiment_config: + data = json.dumps({'shared_storage_config': experiment_config['sharedStorage']}) + response = rest_put(cluster_metadata_url(port), data, REST_TIME_OUT) + err_message = None + if not response or not response.status_code == 200: + if response is not None: + err_message = response.text + _, stderr_full_path = get_log_path(config_file_name) + with open(stderr_full_path, 'a+') as fout: + fout.write(json.dumps(json.loads(err_message), indent=4, sort_keys=True, separators=(',', ':'))) + return False, err_message + return True, None + +def set_experiment_v1(experiment_config, mode, port, config_file_name): + '''Call startExperiment (rest POST /experiment) with yaml file content''' + request_data = dict() + request_data['authorName'] = experiment_config['authorName'] + request_data['experimentName'] = experiment_config['experimentName'] + request_data['trialConcurrency'] = experiment_config['trialConcurrency'] + request_data['maxExecDuration'] = experiment_config['maxExecDuration'] + request_data['maxExperimentDuration'] = str(experiment_config['maxExecDuration']) + 's' + request_data['maxTrialNum'] = experiment_config['maxTrialNum'] + request_data['maxTrialNumber'] = experiment_config['maxTrialNum'] + request_data['searchSpace'] = experiment_config.get('searchSpace') + request_data['trainingServicePlatform'] = experiment_config.get('trainingServicePlatform') + # hack for hotfix, fix config.trainingService undefined error, need refactor + request_data['trainingService'] = {'platform': experiment_config.get('trainingServicePlatform')} + if experiment_config.get('description'): + request_data['description'] = experiment_config['description'] + if experiment_config.get('multiPhase'): + request_data['multiPhase'] = experiment_config.get('multiPhase') + if experiment_config.get('multiThread'): + request_data['multiThread'] = experiment_config.get('multiThread') + if experiment_config.get('nniManagerIp'): + request_data['nniManagerIp'] = experiment_config.get('nniManagerIp') + if experiment_config.get('advisor'): + request_data['advisor'] = experiment_config['advisor'] + if request_data['advisor'].get('gpuNum'): + print_error('gpuNum is deprecated, please use gpuIndices instead.') + if request_data['advisor'].get('gpuIndices') and isinstance(request_data['advisor'].get('gpuIndices'), int): + request_data['advisor']['gpuIndices'] = str(request_data['advisor'].get('gpuIndices')) + else: + request_data['tuner'] = experiment_config['tuner'] + if request_data['tuner'].get('gpuNum'): + print_error('gpuNum is deprecated, please use gpuIndices instead.') + if request_data['tuner'].get('gpuIndices') and isinstance(request_data['tuner'].get('gpuIndices'), int): + request_data['tuner']['gpuIndices'] = str(request_data['tuner'].get('gpuIndices')) + if 'assessor' in experiment_config: + request_data['assessor'] = experiment_config['assessor'] + if request_data['assessor'].get('gpuNum'): + print_error('gpuNum is deprecated, please remove it from your config file.') + #debug mode should disable version check + if experiment_config.get('debug') is not None: + request_data['versionCheck'] = not experiment_config.get('debug') + #validate version check + if experiment_config.get('versionCheck') is not None: + request_data['versionCheck'] = experiment_config.get('versionCheck') + if experiment_config.get('logCollection'): + request_data['logCollection'] = experiment_config.get('logCollection') + request_data['clusterMetaData'] = [] + if experiment_config['trainingServicePlatform'] == 'kubeflow': + request_data['clusterMetaData'].append( + {'key': 'kubeflow_config', 'value': experiment_config['kubeflowConfig']}) + request_data['clusterMetaData'].append( + {'key': 'trial_config', 'value': experiment_config['trial']}) + elif experiment_config['trainingServicePlatform'] == 'frameworkcontroller': + request_data['clusterMetaData'].append( + {'key': 'frameworkcontroller_config', 'value': experiment_config['frameworkcontrollerConfig']}) + request_data['clusterMetaData'].append( + {'key': 'trial_config', 'value': experiment_config['trial']}) + elif experiment_config['trainingServicePlatform'] == 'adl': + request_data['clusterMetaData'].append( + {'key': 'trial_config', 'value': experiment_config['trial']}) + response = rest_post(experiment_url(port), json.dumps(request_data), REST_TIME_OUT, show_error=True) + if check_response(response): + return response + else: + _, stderr_full_path = get_log_path(config_file_name) + if response is not None: + with open(stderr_full_path, 'a+') as fout: + fout.write(json.dumps(json.loads(response.text), indent=4, sort_keys=True, separators=(',', ':'))) + print_error('Setting experiment error, error message is {}'.format(response.text)) + return None + +def set_experiment_v2(experiment_config, mode, port, config_file_name): + '''Call startExperiment (rest POST /experiment) with yaml file content''' + response = rest_post(experiment_url(port), json.dumps(experiment_config), REST_TIME_OUT, show_error=True) + if check_response(response): + return response + else: + _, stderr_full_path = get_log_path(config_file_name) + if response is not None: + with open(stderr_full_path, 'a+') as fout: + fout.write(json.dumps(json.loads(response.text), indent=4, sort_keys=True, separators=(',', ':'))) + print_error('Setting experiment error, error message is {}'.format(response.text)) + return None + +def set_platform_config(platform, experiment_config, port, config_file_name, rest_process): + '''call set_cluster_metadata for specific platform''' + print_normal('Setting {0} config...'.format(platform)) + config_result, err_msg = None, None + if platform == 'adl': + config_result, err_msg = set_adl_config(experiment_config, port, config_file_name) + elif platform == 'kubeflow': + config_result, err_msg = set_kubeflow_config(experiment_config, port, config_file_name) + elif platform == 'frameworkcontroller': + config_result, err_msg = set_frameworkcontroller_config(experiment_config, port, config_file_name) + else: + raise Exception(ERROR_INFO % 'Unsupported platform!') + exit(1) + if config_result: + config_result, err_msg = set_shared_storage(experiment_config, port, config_file_name) + if config_result: + print_normal('Successfully set {0} config!'.format(platform)) + else: + print_error('Failed! Error is: {}'.format(err_msg)) + try: + kill_command(rest_process.pid) + except Exception: + raise Exception(ERROR_INFO % 'Rest server stopped!') + exit(1) + +def launch_experiment(args, experiment_config, mode, experiment_id, config_version): + '''follow steps to start rest server and start experiment''' + # check packages for tuner + package_name, module_name = None, None + if experiment_config.get('tuner') and experiment_config['tuner'].get('builtinTunerName'): + package_name = experiment_config['tuner']['builtinTunerName'] + module_name, _ = get_builtin_module_class_name('tuners', package_name) + elif experiment_config.get('advisor') and experiment_config['advisor'].get('builtinAdvisorName'): + package_name = experiment_config['advisor']['builtinAdvisorName'] + module_name, _ = get_builtin_module_class_name('advisors', package_name) + if package_name and module_name: + try: + stdout_full_path, stderr_full_path = get_log_path(experiment_id) + with open(stdout_full_path, 'a+') as stdout_file, open(stderr_full_path, 'a+') as stderr_file: + check_call([sys.executable, '-c', 'import %s'%(module_name)], stdout=stdout_file, stderr=stderr_file) + except CalledProcessError: + print_error('some errors happen when import package %s.' %(package_name)) + print_log_content(experiment_id) + if package_name in ['SMAC', 'BOHB', 'PPOTuner']: + print_error(f'The dependencies for {package_name} can be installed through pip install nni[{package_name}]') + raise + if config_version == 1: + log_dir = experiment_config['logDir'] if experiment_config.get('logDir') else NNI_HOME_DIR + else: + log_dir = experiment_config['experimentWorkingDirectory'] if experiment_config.get('experimentWorkingDirectory') else NNI_HOME_DIR + log_level = experiment_config['logLevel'] if experiment_config.get('logLevel') else 'info' + #view experiment mode do not need debug function, when view an experiment, there will be no new logs created + foreground = False + if mode != 'view': + foreground = args.foreground + if log_level not in ['trace', 'debug'] and (args.debug or experiment_config.get('debug') is True): + log_level = 'debug' + # start rest server + if config_version == 1: + platform = experiment_config['trainingServicePlatform'] + elif isinstance(experiment_config['trainingService'], list): + platform = 'hybrid' + else: + platform = experiment_config['trainingService']['platform'] + + rest_process, start_time = start_rest_server(args.port, platform, \ + mode, experiment_id, foreground, log_dir, log_level, args.url_prefix) + # save experiment information + Experiments().add_experiment(experiment_id, args.port, start_time, + platform, + experiment_config.get('experimentName', 'N/A') + , pid=rest_process.pid, logDir=log_dir, prefixUrl=args.url_prefix) + # Deal with annotation + if experiment_config.get('useAnnotation'): + path = os.path.join(tempfile.gettempdir(), get_user(), 'nni', 'annotation') + if not os.path.isdir(path): + os.makedirs(path) + path = tempfile.mkdtemp(dir=path) + if config_version == 1: + nas_mode = experiment_config['trial'].get('nasMode', 'classic_mode') + code_dir = expand_annotations(experiment_config['trial']['codeDir'], path, nas_mode=nas_mode) + experiment_config['trial']['codeDir'] = code_dir + else: + code_dir = expand_annotations(experiment_config['trialCodeDirectory'], path) + experiment_config['trialCodeDirectory'] = code_dir + search_space = generate_search_space(code_dir) + experiment_config['searchSpace'] = search_space + assert search_space, ERROR_INFO % 'Generated search space is empty' + elif config_version == 1: + if experiment_config.get('searchSpacePath'): + search_space = get_json_content(experiment_config.get('searchSpacePath')) + experiment_config['searchSpace'] = search_space + else: + experiment_config['searchSpace'] = '' + + # check rest server + running, _ = check_rest_server(args.port) + if running: + print_normal('Successfully started Restful server!') + else: + print_error('Restful server start failed!') + print_log_content(experiment_id) + try: + kill_command(rest_process.pid) + except Exception: + raise Exception(ERROR_INFO % 'Rest server stopped!') + exit(1) + if config_version == 1 and mode != 'view': + # set platform configuration + set_platform_config(experiment_config['trainingServicePlatform'], experiment_config, args.port,\ + experiment_id, rest_process) + + # start a new experiment + print_normal('Starting experiment...') + # set debug configuration + if mode != 'view' and experiment_config.get('debug') is None: + experiment_config['debug'] = args.debug + if config_version == 1: + response = set_experiment_v1(experiment_config, mode, args.port, experiment_id) + else: + response = set_experiment_v2(experiment_config, mode, args.port, experiment_id) + if response: + if experiment_id is None: + experiment_id = json.loads(response.text).get('experiment_id') + else: + print_error('Start experiment failed!') + print_log_content(experiment_id) + try: + kill_command(rest_process.pid) + except Exception: + raise Exception(ERROR_INFO % 'Restful server stopped!') + exit(1) + url_prefix_format = '' if args.url_prefix is None else '/{0}'.format(args.url_prefix) + if experiment_config.get('nniManagerIp'): + web_ui_url_list = ['http://{0}:{1}{2}'.format(experiment_config['nniManagerIp'], str(args.port), url_prefix_format)] + else: + web_ui_url_list = get_local_urls(args.port, url_prefix_format) + Experiments().update_experiment(experiment_id, 'webuiUrl', web_ui_url_list) + + print_normal(EXPERIMENT_SUCCESS_INFO % (experiment_id, ' '.join(web_ui_url_list))) + if mode != 'view' and args.foreground: + try: + while True: + log_content = rest_process.stdout.readline().strip().decode('utf-8') + print(log_content) + except KeyboardInterrupt: + kill_command(rest_process.pid) + print_normal('Stopping experiment...') + +def _validate_v1(config, path): + try: + validate_all_content(config, path) + except Exception as e: + print_error(f'Config V1 validation failed: {repr(e)}') + exit(1) + +def _validate_v2(config, path): + base_path = Path(path).parent + try: + conf = ExperimentConfig(_base_path=base_path, **config) + return conf.json() + except Exception as e: + print_error(f'Config V2 validation failed: {repr(e)}') + +def _validate_prefix_path(path): + assert not path.startswith('/'), 'URL prefix should not start with "/".' + parts = path.split('/') + valid = all(re.match('^[A-Za-z0-9_-]*$', part) for part in parts) + assert valid, 'URL prefix should only contain letter, number, underscore, and hyphen.' + +def create_experiment(args): + '''start a new experiment''' + experiment_id = ''.join(random.sample(string.ascii_letters + string.digits, 8)) + config_path = os.path.abspath(args.config) + if not os.path.exists(config_path): + print_error('Please set correct config path!') + exit(1) + config_yml = get_yml_content(config_path) + + if 'trainingServicePlatform' in config_yml: + _validate_v1(config_yml, config_path) + platform = config_yml['trainingServicePlatform'] + if platform in k8s_training_services: + schema = 1 + config_v1 = config_yml + else: + schema = 2 + config_v2 = convert.to_v2(config_yml).json() + else: + config_v2 = _validate_v2(config_yml, config_path) + schema = 2 + + try: + if schema == 1: + launch_experiment(args, config_v1, 'new', experiment_id, 1) + else: + launch_experiment(args, config_v2, 'new', experiment_id, 2) + except Exception as exception: + restServerPid = Experiments().get_all_experiments().get(experiment_id, {}).get('pid') + if restServerPid: + kill_command(restServerPid) + print_error(exception) + exit(1) + +def manage_stopped_experiment(args, mode): + '''view a stopped experiment''' + update_experiment() + experiments_config = Experiments() + experiments_dict = experiments_config.get_all_experiments() + experiment_id = None + #find the latest stopped experiment + if not args.id: + print_error('Please set experiment id! \nYou could use \'nnictl {0} id\' to {0} a stopped experiment!\n' \ + 'You could use \'nnictl experiment list --all\' to show all experiments!'.format(mode)) + exit(1) + else: + if experiments_dict.get(args.id) is None: + print_error('Id %s not exist!' % args.id) + exit(1) + if experiments_dict[args.id]['status'] != 'STOPPED': + print_error('Only stopped experiments can be {0}ed!'.format(mode)) + exit(1) + experiment_id = args.id + print_normal('{0} experiment {1}...'.format(mode, experiment_id)) + experiment_config = Config(experiment_id, experiments_dict[args.id]['logDir']).get_config() + experiments_config.update_experiment(args.id, 'port', args.port) + args.url_prefix = experiments_dict[args.id]['prefixUrl'] + assert 'trainingService' in experiment_config or 'trainingServicePlatform' in experiment_config + try: + if 'trainingServicePlatform' in experiment_config: + experiment_config['logDir'] = experiments_dict[args.id]['logDir'] + launch_experiment(args, experiment_config, mode, experiment_id, 1) + else: + experiment_config['experimentWorkingDirectory'] = experiments_dict[args.id]['logDir'] + launch_experiment(args, experiment_config, mode, experiment_id, 2) + except Exception as exception: + restServerPid = Experiments().get_all_experiments().get(experiment_id, {}).get('pid') + if restServerPid: + kill_command(restServerPid) + print_error(exception) + exit(1) + +def view_experiment(args): + '''view a stopped experiment''' + manage_stopped_experiment(args, 'view') + +def resume_experiment(args): + '''resume an experiment''' + manage_stopped_experiment(args, 'resume') diff --git a/utils/third_party/nni_new/tools/nnictl/launcher_utils.py b/utils/third_party/nni_new/tools/nnictl/launcher_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..36033671586f9afdd242d9267a9c8eb076b8b604 --- /dev/null +++ b/utils/third_party/nni_new/tools/nnictl/launcher_utils.py @@ -0,0 +1,128 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +from schema import SchemaError +from .config_schema import NNIConfigSchema +from .common_utils import print_normal + +def expand_path(experiment_config, key): + '''Change '~' to user home directory''' + if experiment_config.get(key): + experiment_config[key] = os.path.expanduser(experiment_config[key]) + +def parse_relative_path(root_path, experiment_config, key): + '''Change relative path to absolute path''' + if experiment_config.get(key) and not os.path.isabs(experiment_config.get(key)): + absolute_path = os.path.join(root_path, experiment_config.get(key)) + print_normal('expand %s: %s to %s ' % (key, experiment_config[key], absolute_path)) + experiment_config[key] = absolute_path + +def parse_time(time): + '''Change the time to seconds''' + unit = time[-1] + if unit not in ['s', 'm', 'h', 'd']: + raise SchemaError('the unit of time could only from {s, m, h, d}') + time = time[:-1] + if not time.isdigit(): + raise SchemaError('time format error!') + parse_dict = {'s':1, 'm':60, 'h':3600, 'd':86400} + return int(time) * parse_dict[unit] + +def parse_path(experiment_config, config_path): + '''Parse path in config file''' + expand_path(experiment_config, 'searchSpacePath') + if experiment_config.get('logDir'): + expand_path(experiment_config, 'logDir') + if experiment_config.get('trial'): + expand_path(experiment_config['trial'], 'codeDir') + if experiment_config['trial'].get('authFile'): + expand_path(experiment_config['trial'], 'authFile') + if experiment_config['trial'].get('ps'): + if experiment_config['trial']['ps'].get('privateRegistryAuthPath'): + expand_path(experiment_config['trial']['ps'], 'privateRegistryAuthPath') + if experiment_config['trial'].get('master'): + if experiment_config['trial']['master'].get('privateRegistryAuthPath'): + expand_path(experiment_config['trial']['master'], 'privateRegistryAuthPath') + if experiment_config['trial'].get('worker'): + if experiment_config['trial']['worker'].get('privateRegistryAuthPath'): + expand_path(experiment_config['trial']['worker'], 'privateRegistryAuthPath') + if experiment_config['trial'].get('taskRoles'): + for index in range(len(experiment_config['trial']['taskRoles'])): + if experiment_config['trial']['taskRoles'][index].get('privateRegistryAuthPath'): + expand_path(experiment_config['trial']['taskRoles'][index], 'privateRegistryAuthPath') + if experiment_config.get('tuner'): + expand_path(experiment_config['tuner'], 'codeDir') + if experiment_config.get('assessor'): + expand_path(experiment_config['assessor'], 'codeDir') + if experiment_config.get('advisor'): + expand_path(experiment_config['advisor'], 'codeDir') + if experiment_config.get('machineList'): + for index in range(len(experiment_config['machineList'])): + expand_path(experiment_config['machineList'][index], 'sshKeyPath') + if experiment_config['trial'].get('paiConfigPath'): + expand_path(experiment_config['trial'], 'paiConfigPath') + + # If users use relative path, convert it to absolute path. + root_path = os.path.dirname(config_path) + if experiment_config.get('searchSpacePath'): + parse_relative_path(root_path, experiment_config, 'searchSpacePath') + if experiment_config.get('logDir'): + parse_relative_path(root_path, experiment_config, 'logDir') + if experiment_config.get('trial'): + # In AdaptDL mode, 'codeDir' shouldn't be parsed because it points to the path in the container. + if experiment_config.get('trainingServicePlatform') != 'adl': + parse_relative_path(root_path, experiment_config['trial'], 'codeDir') + if experiment_config['trial'].get('authFile'): + parse_relative_path(root_path, experiment_config['trial'], 'authFile') + if experiment_config['trial'].get('ps'): + if experiment_config['trial']['ps'].get('privateRegistryAuthPath'): + parse_relative_path(root_path, experiment_config['trial']['ps'], 'privateRegistryAuthPath') + if experiment_config['trial'].get('master'): + if experiment_config['trial']['master'].get('privateRegistryAuthPath'): + parse_relative_path(root_path, experiment_config['trial']['master'], 'privateRegistryAuthPath') + if experiment_config['trial'].get('worker'): + if experiment_config['trial']['worker'].get('privateRegistryAuthPath'): + parse_relative_path(root_path, experiment_config['trial']['worker'], 'privateRegistryAuthPath') + if experiment_config['trial'].get('taskRoles'): + for index in range(len(experiment_config['trial']['taskRoles'])): + if experiment_config['trial']['taskRoles'][index].get('privateRegistryAuthPath'): + parse_relative_path(root_path, experiment_config['trial']['taskRoles'][index], 'privateRegistryAuthPath') + if experiment_config.get('tuner'): + parse_relative_path(root_path, experiment_config['tuner'], 'codeDir') + if experiment_config.get('assessor'): + parse_relative_path(root_path, experiment_config['assessor'], 'codeDir') + if experiment_config.get('advisor'): + parse_relative_path(root_path, experiment_config['advisor'], 'codeDir') + if experiment_config.get('machineList'): + for index in range(len(experiment_config['machineList'])): + parse_relative_path(root_path, experiment_config['machineList'][index], 'sshKeyPath') + if experiment_config['trial'].get('paiConfigPath'): + parse_relative_path(root_path, experiment_config['trial'], 'paiConfigPath') + + # For frameworkcontroller a custom configuration path may be specified + if experiment_config.get('frameworkcontrollerConfig'): + if experiment_config['frameworkcontrollerConfig'].get('configPath'): + parse_relative_path(root_path, experiment_config['frameworkcontrollerConfig'], 'configPath') + +def set_default_values(experiment_config): + if experiment_config.get('maxExecDuration') is None: + experiment_config['maxExecDuration'] = '999d' + if experiment_config.get('maxTrialNum') is None: + experiment_config['maxTrialNum'] = 99999 + if experiment_config['trainingServicePlatform'] == 'remote' or \ + experiment_config['trainingServicePlatform'] == 'hybrid' and \ + 'remote' in experiment_config['hybridConfig']['trainingServicePlatforms']: + for index in range(len(experiment_config['machineList'])): + if experiment_config['machineList'][index].get('port') is None: + experiment_config['machineList'][index]['port'] = 22 + +def validate_all_content(experiment_config, config_path): + '''Validate whether experiment_config is valid''' + parse_path(experiment_config, config_path) + set_default_values(experiment_config) + + NNIConfigSchema().validate(experiment_config) + + if 'maxExecDuration' in experiment_config: + experiment_config['maxExecDuration'] = parse_time(experiment_config['maxExecDuration']) diff --git a/utils/third_party/nni_new/tools/nnictl/nnictl.py b/utils/third_party/nni_new/tools/nnictl/nnictl.py new file mode 100644 index 0000000000000000000000000000000000000000..a5f02c8c9e360a1757ac2008212908f47b3263b8 --- /dev/null +++ b/utils/third_party/nni_new/tools/nnictl/nnictl.py @@ -0,0 +1,281 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import argparse +import logging +import os +import pkg_resources +from colorama import init +from .common_utils import print_error +from .launcher import create_experiment, resume_experiment, view_experiment +from .updater import update_searchspace, update_concurrency, update_duration, update_trialnum, import_data +from .nnictl_utils import stop_experiment, trial_ls, trial_kill, list_experiment, experiment_status,\ + log_trial, experiment_clean, platform_clean, experiment_list, \ + monitor_experiment, export_trials_data, trial_codegen, webui_url, \ + get_config, log_stdout, log_stderr, search_space_auto_gen, webui_nas, \ + save_experiment, load_experiment +from .algo_management import algo_reg, algo_unreg, algo_show, algo_list +from .constants import DEFAULT_REST_PORT +from .import ts_management + +init(autoreset=True) + +if os.environ.get('COVERAGE_PROCESS_START'): + import coverage + coverage.process_startup() + +def nni_info(*args): + if args[0].version: + try: + print(pkg_resources.get_distribution('nni').version) + except pkg_resources.ResolutionError: + print_error('Get version failed, please use `pip3 list | grep nni` to check nni version!') + else: + print('please run "nnictl {positional argument} --help" to see nnictl guidance') + +def parse_args(): + logging.getLogger().setLevel(logging.ERROR) + + '''Definite the arguments users need to follow and input''' + parser = argparse.ArgumentParser(prog='nnictl', description='use nnictl command to control nni experiments') + parser.add_argument('--version', '-v', action='store_true') + parser.set_defaults(func=nni_info) + + # create subparsers for args with sub values + subparsers = parser.add_subparsers() + + # parse the command of auto generating search space + parser_start = subparsers.add_parser('ss_gen', help='automatically generate search space file from trial code') + parser_start.add_argument('--trial_command', '-t', required=True, dest='trial_command', help='the command for running trial code') + parser_start.add_argument('--trial_dir', '-d', default='./', dest='trial_dir', help='the directory for running the command') + parser_start.add_argument('--file', '-f', default='nni_auto_gen_search_space.json', dest='file', help='the path of search space file') + parser_start.set_defaults(func=search_space_auto_gen) + + # parse start command + parser_start = subparsers.add_parser('create', help='create a new experiment') + parser_start.add_argument('--config', '-c', required=True, dest='config', help='the path of yaml config file') + parser_start.add_argument('--port', '-p', default=DEFAULT_REST_PORT, dest='port', type=int, help='the port of restful server') + parser_start.add_argument('--debug', '-d', action='store_true', help=' set debug mode') + parser_start.add_argument('--url_prefix', '-u', dest='url_prefix', help=' set prefix url') + parser_start.add_argument('--foreground', '-f', action='store_true', help=' set foreground mode, print log content to terminal') + parser_start.set_defaults(func=create_experiment) + + # parse resume command + parser_resume = subparsers.add_parser('resume', help='resume a new experiment') + parser_resume.add_argument('id', nargs='?', help='The id of the experiment you want to resume') + parser_resume.add_argument('--port', '-p', default=DEFAULT_REST_PORT, dest='port', type=int, help='the port of restful server') + parser_resume.add_argument('--debug', '-d', action='store_true', help=' set debug mode') + parser_resume.add_argument('--foreground', '-f', action='store_true', help=' set foreground mode, print log content to terminal') + parser_resume.set_defaults(func=resume_experiment) + + # parse view command + parser_view = subparsers.add_parser('view', help='view a stopped experiment') + parser_view.add_argument('id', nargs='?', help='The id of the experiment you want to view') + parser_view.add_argument('--port', '-p', default=DEFAULT_REST_PORT, dest='port', type=int, help='the port of restful server') + parser_view.set_defaults(func=view_experiment) + + # parse update command + parser_updater = subparsers.add_parser('update', help='update the experiment') + #add subparsers for parser_updater + parser_updater_subparsers = parser_updater.add_subparsers() + parser_updater_searchspace = parser_updater_subparsers.add_parser('searchspace', help='update searchspace') + parser_updater_searchspace.add_argument('id', nargs='?', help='the id of experiment') + parser_updater_searchspace.add_argument('--filename', '-f', required=True) + parser_updater_searchspace.set_defaults(func=update_searchspace) + parser_updater_concurrency = parser_updater_subparsers.add_parser('concurrency', help='update concurrency') + parser_updater_concurrency.add_argument('id', nargs='?', help='the id of experiment') + parser_updater_concurrency.add_argument('--value', '-v', required=True) + parser_updater_concurrency.set_defaults(func=update_concurrency) + parser_updater_duration = parser_updater_subparsers.add_parser('duration', help='update duration') + parser_updater_duration.add_argument('id', nargs='?', help='the id of experiment') + parser_updater_duration.add_argument('--value', '-v', required=True, help='the unit of time should in {\'s\', \'m\', \'h\', \'d\'}') + parser_updater_duration.set_defaults(func=update_duration) + parser_updater_trialnum = parser_updater_subparsers.add_parser('trialnum', help='update maxtrialnum') + parser_updater_trialnum.add_argument('id', nargs='?', help='the id of experiment') + parser_updater_trialnum.add_argument('--value', '-v', required=True) + parser_updater_trialnum.set_defaults(func=update_trialnum) + + #parse stop command + parser_stop = subparsers.add_parser('stop', help='stop the experiment') + parser_stop.add_argument('id', nargs='?', help='the id of experiment, use \'all\' to stop all running experiments') + parser_stop.add_argument('--port', '-p', dest='port', type=int, help='the port of restful server') + parser_stop.add_argument('--all', '-a', action='store_true', help='stop all of experiments') + parser_stop.set_defaults(func=stop_experiment) + + #parse trial command + parser_trial = subparsers.add_parser('trial', help='get trial information') + #add subparsers for parser_trial + parser_trial_subparsers = parser_trial.add_subparsers() + parser_trial_ls = parser_trial_subparsers.add_parser('ls', help='list trial jobs') + parser_trial_ls.add_argument('id', nargs='?', help='the id of experiment') + parser_trial_ls.add_argument('--head', type=int, help='list the highest experiments on the default metric') + parser_trial_ls.add_argument('--tail', type=int, help='list the lowest experiments on the default metric') + parser_trial_ls.set_defaults(func=trial_ls) + parser_trial_kill = parser_trial_subparsers.add_parser('kill', help='kill trial jobs') + parser_trial_kill.add_argument('id', nargs='?', help='the id of experiment') + parser_trial_kill.add_argument('--trial_id', '-T', required=True, dest='trial_id', help='the id of trial to be killed') + parser_trial_kill.set_defaults(func=trial_kill) + parser_trial_codegen = parser_trial_subparsers.add_parser('codegen', help='generate trial code for a specific trial') + parser_trial_codegen.add_argument('id', nargs='?', help='the id of experiment') + parser_trial_codegen.add_argument('--trial_id', '-T', required=True, dest='trial_id', help='the id of trial to do code generation') + parser_trial_codegen.set_defaults(func=trial_codegen) + + #parse experiment command + parser_experiment = subparsers.add_parser('experiment', help='get experiment information') + #add subparsers for parser_experiment + parser_experiment_subparsers = parser_experiment.add_subparsers() + parser_experiment_show = parser_experiment_subparsers.add_parser('show', help='show the information of experiment') + parser_experiment_show.add_argument('id', nargs='?', help='the id of experiment') + parser_experiment_show.set_defaults(func=list_experiment) + parser_experiment_status = parser_experiment_subparsers.add_parser('status', help='show the status of experiment') + parser_experiment_status.add_argument('id', nargs='?', help='the id of experiment') + parser_experiment_status.set_defaults(func=experiment_status) + parser_experiment_list = parser_experiment_subparsers.add_parser('list', help='list all of running experiment ids') + parser_experiment_list.add_argument('--all', action='store_true', default=False, help='list all of experiments') + parser_experiment_list.set_defaults(func=experiment_list) + parser_experiment_clean = parser_experiment_subparsers.add_parser('delete', help='clean up the experiment data') + parser_experiment_clean.add_argument('id', nargs='?', help='the id of experiment') + parser_experiment_clean.add_argument('--all', action='store_true', default=False, help='delete all of experiments') + parser_experiment_clean.set_defaults(func=experiment_clean) + #import tuning data + parser_import_data = parser_experiment_subparsers.add_parser('import', help='import additional data') + parser_import_data.add_argument('id', nargs='?', help='the id of experiment') + parser_import_data.add_argument('--filename', '-f', required=True) + parser_import_data.set_defaults(func=import_data) + #export trial data + parser_trial_export = parser_experiment_subparsers.add_parser('export', help='export trial job results to csv or json') + parser_trial_export.add_argument('id', nargs='?', help='the id of experiment') + parser_trial_export.add_argument('--type', '-t', choices=['json', 'csv'], required=True, dest='type', help='target file type') + parser_trial_export.add_argument('--filename', '-f', required=True, dest='path', help='target file path') + parser_trial_export.add_argument('--intermediate', '-i', action='store_true', + default=False, help='are intermediate results included') + parser_trial_export.set_defaults(func=export_trials_data) + #save an NNI experiment + parser_save_experiment = parser_experiment_subparsers.add_parser('save', help='save an experiment') + parser_save_experiment.add_argument('id', nargs='?', help='the id of experiment') + parser_save_experiment.add_argument('--path', '-p', required=False, help='the folder path to store nni experiment data, \ + default current working directory') + parser_save_experiment.add_argument('--saveCodeDir', '-s', action='store_true', default=False, help='save codeDir data \ + of the experiment') + parser_save_experiment.set_defaults(func=save_experiment) + #load an NNI experiment + parser_load_experiment = parser_experiment_subparsers.add_parser('load', help='load an experiment') + parser_load_experiment.add_argument('--path', '-p', required=True, help='the path of nni package file') + parser_load_experiment.add_argument('--codeDir', '-c', required=True, help='the path of codeDir for loaded experiment, \ + this path will also put the code in the loaded experiment package') + parser_load_experiment.add_argument('--logDir', '-l', required=False, help='the path of logDir for loaded experiment') + parser_load_experiment.add_argument('--searchSpacePath', '-s', required=False, help='the path of search space file for \ + loaded experiment, this path contains file name. Default in $codeDir/search_space.json') + parser_load_experiment.set_defaults(func=load_experiment) + + #parse platform command + parser_platform = subparsers.add_parser('platform', help='get platform information') + #add subparsers for parser_platform + parser_platform_subparsers = parser_platform.add_subparsers() + parser_platform_clean = parser_platform_subparsers.add_parser('clean', help='clean up the platform data') + parser_platform_clean.add_argument('--config', '-c', required=True, dest='config', help='the path of yaml config file') + parser_platform_clean.set_defaults(func=platform_clean) + + #TODO:finish webui function + #parse board command + parser_webui = subparsers.add_parser('webui', help='get web ui information') + #add subparsers for parser_board + parser_webui_subparsers = parser_webui.add_subparsers() + parser_webui_url = parser_webui_subparsers.add_parser('url', help='show the url of web ui') + parser_webui_url.add_argument('id', nargs='?', help='the id of experiment') + parser_webui_url.set_defaults(func=webui_url) + parser_webui_nas = parser_webui_subparsers.add_parser('nas', help='show nas ui') + parser_webui_nas.add_argument('--port', default=6060, type=int, help='port of nas ui') + parser_webui_nas.add_argument('--logdir', default='.', type=str, help='the logdir where nas ui will read data') + parser_webui_nas.set_defaults(func=webui_nas) + + #parse config command + parser_config = subparsers.add_parser('config', help='get config information') + parser_config_subparsers = parser_config.add_subparsers() + parser_config_show = parser_config_subparsers.add_parser('show', help='show the information of config') + parser_config_show.add_argument('id', nargs='?', help='the id of experiment') + parser_config_show.set_defaults(func=get_config) + + #parse log command + parser_log = subparsers.add_parser('log', help='get log information') + # add subparsers for parser_log + parser_log_subparsers = parser_log.add_subparsers() + parser_log_stdout = parser_log_subparsers.add_parser('stdout', help='get stdout information') + parser_log_stdout.add_argument('id', nargs='?', help='the id of experiment') + parser_log_stdout.add_argument('--tail', '-T', dest='tail', type=int, help='get tail -100 content of stdout') + parser_log_stdout.add_argument('--head', '-H', dest='head', type=int, help='get head -100 content of stdout') + parser_log_stdout.add_argument('--path', action='store_true', default=False, help='get the path of stdout file') + parser_log_stdout.set_defaults(func=log_stdout) + parser_log_stderr = parser_log_subparsers.add_parser('stderr', help='get stderr information') + parser_log_stderr.add_argument('id', nargs='?', help='the id of experiment') + parser_log_stderr.add_argument('--tail', '-T', dest='tail', type=int, help='get tail -100 content of stderr') + parser_log_stderr.add_argument('--head', '-H', dest='head', type=int, help='get head -100 content of stderr') + parser_log_stderr.add_argument('--path', action='store_true', default=False, help='get the path of stderr file') + parser_log_stderr.set_defaults(func=log_stderr) + parser_log_trial = parser_log_subparsers.add_parser('trial', help='get trial log path') + parser_log_trial.add_argument('id', nargs='?', help='the id of experiment') + parser_log_trial.add_argument('--trial_id', '-T', dest='trial_id', help='find trial log path by id') + parser_log_trial.set_defaults(func=log_trial) + + #parse algo command + parser_algo = subparsers.add_parser('algo', help='control nni builtin tuner, assessor and advisor algorithms') + # add subparsers for parser_algo + parser_algo_subparsers = parser_algo.add_subparsers() + parser_algo_reg = parser_algo_subparsers.add_parser( + 'register', + aliases=('reg',), + help='''register algorithms as nni builtin algorithm, for example: + nnictl reg --meta_path + where is the path to a meta data in yml format, + reference the nni document and examples/tuners/customized_tuner example + for the format of the yml file.''' + ) + parser_algo_reg.add_argument('--meta_path', '-m', dest='meta_path', help='path to the meta file', required=True) + parser_algo_reg.set_defaults(func=algo_reg) + + parser_algo_unreg = parser_algo_subparsers.add_parser('unregister', aliases=('unreg',), help='unregister algorithm') + parser_algo_unreg.add_argument('name', nargs=1, help='builtin name of the algorithm') + parser_algo_unreg.set_defaults(func=algo_unreg) + + parser_algo_show = parser_algo_subparsers.add_parser('show', help='show the information of algorithm') + parser_algo_show.add_argument('name', nargs=1, help='builtin name of the algorithm') + parser_algo_show.set_defaults(func=algo_show) + + parser_algo_list = parser_algo_subparsers.add_parser('list', help='list registered algorithms') + parser_algo_list.set_defaults(func=algo_list) + + #parse trainingservice command + parser_ts = subparsers.add_parser('trainingservice', help='control training service') + # add subparsers for parser_ts + parser_ts_subparsers = parser_ts.add_subparsers() + + parser_ts_reg = parser_ts_subparsers.add_parser('register', help='register training service') + parser_ts_reg.add_argument('--package', dest='package', help='package name', required=True) + parser_ts_reg.set_defaults(func=ts_management.register) + + parser_ts_unreg = parser_ts_subparsers.add_parser('unregister', help='unregister training service') + parser_ts_unreg.add_argument('--package', dest='package', help='package name', required=True) + parser_ts_unreg.set_defaults(func=ts_management.unregister) + + parser_ts_list = parser_ts_subparsers.add_parser('list', help='list custom training services') + parser_ts_list.set_defaults(func=ts_management.list_services) + + # To show message that nnictl package command is replaced by nnictl algo, to be remove in the future release. + def show_messsage_for_nnictl_package(args): + print_error('nnictl package command is replaced by nnictl algo, please run nnictl algo -h to show the usage') + + parser_package_subparsers = subparsers.add_parser('package', help='this argument is replaced by algo', prefix_chars='\n') + parser_package_subparsers.add_argument('args', nargs=argparse.REMAINDER) + parser_package_subparsers.set_defaults(func=show_messsage_for_nnictl_package) + + #parse top command + parser_top = subparsers.add_parser('top', help='monitor the experiment') + parser_top.add_argument('--time', '-t', dest='time', type=int, default=3, help='the time interval to update the experiment status, ' \ + 'the unit is second') + parser_top.set_defaults(func=monitor_experiment) + + args = parser.parse_args() + args.func(args) + +if __name__ == '__main__': + parse_args() diff --git a/utils/third_party/nni_new/tools/nnictl/nnictl_utils.py b/utils/third_party/nni_new/tools/nnictl/nnictl_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7ef0fe7dd7fc35609820fc6d601a270ba1a9554a --- /dev/null +++ b/utils/third_party/nni_new/tools/nnictl/nnictl_utils.py @@ -0,0 +1,964 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import csv +import os +import sys +import json +import time +import shutil +import subprocess +from functools import cmp_to_key +import traceback +from datetime import datetime, timezone +from subprocess import Popen +from nni.tools.annotation import expand_annotations +import nni_node # pylint: disable=import-error +from .rest_utils import rest_get, rest_delete, check_rest_server_quick, check_response +from .url_utils import trial_jobs_url, experiment_url, trial_job_id_url, export_data_url, metric_data_url +from .config_utils import Config, Experiments +from .constants import NNI_HOME_DIR, EXPERIMENT_INFORMATION_FORMAT, EXPERIMENT_DETAIL_FORMAT, EXPERIMENT_MONITOR_INFO, \ + TRIAL_MONITOR_HEAD, TRIAL_MONITOR_CONTENT, TRIAL_MONITOR_TAIL, REST_TIME_OUT +from .common_utils import print_normal, print_error, print_warning, detect_process, get_yml_content, generate_temp_dir +from .common_utils import print_green +from .command_utils import check_output_command, kill_command +from .ssh_utils import create_ssh_sftp_client, remove_remote_directory + +def get_experiment_time(port): + '''get the startTime and endTime of an experiment''' + response = rest_get(experiment_url(port), REST_TIME_OUT) + if response and check_response(response): + content = json.loads(response.text) + return content.get('startTime'), content.get('endTime') + return None, None + +def get_experiment_status(port): + '''get the status of an experiment''' + result, response = check_rest_server_quick(port) + if result: + return json.loads(response.text).get('status') + return None + +def update_experiment(): + '''Update the experiment status in config file''' + experiments_config = Experiments() + experiments_dict = experiments_config.get_all_experiments() + if not experiments_dict: + return None + for key in experiments_dict.keys(): + if isinstance(experiments_dict[key], dict): + if experiments_dict[key].get('status') != 'STOPPED': + rest_pid = experiments_dict[key].get('pid') + if not detect_process(rest_pid): + experiments_config.update_experiment(key, 'status', 'STOPPED') + continue + +def check_experiment_id(args, update=True): + '''check if the id is valid + ''' + if update: + update_experiment() + experiments_config = Experiments() + experiments_dict = experiments_config.get_all_experiments() + if not experiments_dict: + print_normal('There is no experiment running...') + return None + if not args.id: + running_experiment_list = [] + for key in experiments_dict.keys(): + if isinstance(experiments_dict[key], dict): + if experiments_dict[key].get('status') != 'STOPPED': + running_experiment_list.append(key) + elif isinstance(experiments_dict[key], list): + # if the config file is old version, remove the configuration from file + experiments_config.remove_experiment(key) + if len(running_experiment_list) > 1: + print_error('There are multiple experiments, please set the experiment id...') + experiment_information = "" + for key in running_experiment_list: + experiment_information += EXPERIMENT_DETAIL_FORMAT % (key, + experiments_dict[key].get('experimentName', 'N/A'), + experiments_dict[key]['status'], + experiments_dict[key].get('port', 'N/A'), + experiments_dict[key].get('platform'), + time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(experiments_dict[key]['startTime'] / 1000)) if isinstance(experiments_dict[key]['startTime'], int) else experiments_dict[key]['startTime'], + time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(experiments_dict[key]['endTime'] / 1000)) if isinstance(experiments_dict[key]['endTime'], int) else experiments_dict[key]['endTime']) + print(EXPERIMENT_INFORMATION_FORMAT % experiment_information) + exit(1) + elif not running_experiment_list: + print_error('There is no experiment running.') + return None + else: + return running_experiment_list[0] + if experiments_dict.get(args.id): + return args.id + else: + print_error('Id not correct.') + return None + +def parse_ids(args): + '''Parse the arguments for nnictl stop + 1.If port is provided and id is not specified, return the id who owns the port + 2.If both port and id are provided, return the id if it owns the port, otherwise fail + 3.If there is an id specified, return the corresponding id + 4.If there is no id specified, and there is an experiment running, return the id, or return Error + 5.If the id matches an experiment, nnictl will return the id. + 6.If the id ends with *, nnictl will match all ids matchs the regular + 7.If the id does not exist but match the prefix of an experiment id, nnictl will return the matched id + 8.If the id does not exist but match multiple prefix of the experiment ids, nnictl will give id information + ''' + update_experiment() + experiments_config = Experiments() + experiments_dict = experiments_config.get_all_experiments() + if not experiments_dict: + print_normal('Experiment is not running...') + return None + result_list = [] + running_experiment_list = [] + for key in experiments_dict.keys(): + if isinstance(experiments_dict[key], dict): + if experiments_dict[key].get('status') != 'STOPPED': + running_experiment_list.append(key) + elif isinstance(experiments_dict[key], list): + # if the config file is old version, remove the configuration from file + experiments_config.remove_experiment(key) + if args.all: + return running_experiment_list + if args.port is not None: + for key in running_experiment_list: + if experiments_dict[key].get('port') == args.port: + result_list.append(key) + if args.id and result_list and args.id != result_list[0]: + print_error('Experiment id and resful server port not match') + exit(1) + elif not args.id: + if len(running_experiment_list) > 1: + print_error('There are multiple experiments, please set the experiment id...') + experiment_information = "" + for key in running_experiment_list: + experiment_information += EXPERIMENT_DETAIL_FORMAT % (key, + experiments_dict[key].get('experimentName', 'N/A'), + experiments_dict[key]['status'], + experiments_dict[key].get('port', 'N/A'), + experiments_dict[key].get('platform'), + time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(experiments_dict[key]['startTime'] / 1000)) if isinstance(experiments_dict[key]['startTime'], int) else experiments_dict[key]['startTime'], + time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(experiments_dict[key]['endTime'] / 1000)) if isinstance(experiments_dict[key]['endTime'], int) else experiments_dict[key]['endTime']) + print(EXPERIMENT_INFORMATION_FORMAT % experiment_information) + exit(1) + else: + result_list = running_experiment_list + elif args.id.endswith('*'): + for expId in running_experiment_list: + if expId.startswith(args.id[:-1]): + result_list.append(expId) + elif args.id in running_experiment_list: + result_list.append(args.id) + else: + for expId in running_experiment_list: + if expId.startswith(args.id): + result_list.append(expId) + if len(result_list) > 1: + print_error(args.id + ' is ambiguous, please choose ' + ' '.join(result_list)) + return None + if not result_list and (args.id or args.port): + print_error('There are no experiments matched, please set correct experiment id or restful server port') + elif not result_list: + print_error('There is no experiment running...') + return result_list + +def get_config_filename(args): + '''get the file name of config file''' + experiment_id = check_experiment_id(args) + if experiment_id is None: + print_error('Please set correct experiment id.') + exit(1) + return experiment_id + +def get_experiment_port(args): + '''get the port of experiment''' + experiment_id = check_experiment_id(args) + if experiment_id is None: + print_error('Please set correct experiment id.') + exit(1) + experiments_config = Experiments() + experiments_dict = experiments_config.get_all_experiments() + return experiments_dict[experiment_id].get('port') + +def convert_time_stamp_to_date(content): + '''Convert time stamp to date time format''' + start_time_stamp = content.get('startTime') + end_time_stamp = content.get('endTime') + if start_time_stamp: + start_time = datetime.fromtimestamp(start_time_stamp // 1000, timezone.utc).astimezone().strftime("%Y/%m/%d %H:%M:%S") + content['startTime'] = str(start_time) + if end_time_stamp: + end_time = datetime.fromtimestamp(end_time_stamp // 1000, timezone.utc).astimezone().strftime("%Y/%m/%d %H:%M:%S") + content['endTime'] = str(end_time) + return content + +def check_rest(args): + '''check if restful server is running''' + experiments_config = Experiments() + experiments_dict = experiments_config.get_all_experiments() + rest_port = experiments_dict.get(get_config_filename(args)).get('port') + running, _ = check_rest_server_quick(rest_port) + if running: + print_normal('Restful server is running...') + else: + print_normal('Restful server is not running...') + return running + +def stop_experiment(args): + '''Stop the experiment which is running''' + if args.id and args.id == 'all': + print_warning('\'nnictl stop all\' is abolished, please use \'nnictl stop --all\' to stop all of experiments!') + exit(1) + experiment_id_list = parse_ids(args) + if experiment_id_list: + for experiment_id in experiment_id_list: + print_normal('Stopping experiment %s' % experiment_id) + experiments_config = Experiments() + experiments_dict = experiments_config.get_all_experiments() + rest_pid = experiments_dict.get(experiment_id).get('pid') + if rest_pid: + kill_command(rest_pid) + print_normal('Stop experiment success.') + +def trial_ls(args): + '''List trial''' + def final_metric_data_cmp(lhs, rhs): + metric_l = json.loads(json.loads(lhs['finalMetricData'][0]['data'])) + metric_r = json.loads(json.loads(rhs['finalMetricData'][0]['data'])) + if isinstance(metric_l, float): + return metric_l - metric_r + elif isinstance(metric_l, dict): + return metric_l['default'] - metric_r['default'] + else: + print_error('Unexpected data format. Please check your data.') + raise ValueError + + if args.head and args.tail: + print_error('Head and tail cannot be set at the same time.') + return + experiments_config = Experiments() + experiments_dict = experiments_config.get_all_experiments() + experiment_id = get_config_filename(args) + rest_port = experiments_dict.get(experiment_id).get('port') + rest_pid = experiments_dict.get(experiment_id).get('pid') + if not detect_process(rest_pid): + print_error('Experiment is not running...') + return + running, response = check_rest_server_quick(rest_port) + if running: + response = rest_get(trial_jobs_url(rest_port), REST_TIME_OUT) + if response and check_response(response): + content = json.loads(response.text) + if args.head: + assert args.head > 0, 'The number of requested data must be greater than 0.' + content = sorted(filter(lambda x: 'finalMetricData' in x, content), + key=cmp_to_key(final_metric_data_cmp), reverse=True)[:args.head] + elif args.tail: + assert args.tail > 0, 'The number of requested data must be greater than 0.' + content = sorted(filter(lambda x: 'finalMetricData' in x, content), + key=cmp_to_key(final_metric_data_cmp))[:args.tail] + for index, value in enumerate(content): + content[index] = convert_time_stamp_to_date(value) + print(json.dumps(content, indent=4, sort_keys=True, separators=(',', ':'))) + return content + else: + print_error('List trial failed...') + else: + print_error('Restful server is not running...') + return None + +def trial_kill(args): + '''List trial''' + experiments_config = Experiments() + experiments_dict = experiments_config.get_all_experiments() + experiment_id = get_config_filename(args) + rest_port = experiments_dict.get(experiment_id).get('port') + rest_pid = experiments_dict.get(experiment_id).get('pid') + if not detect_process(rest_pid): + print_error('Experiment is not running...') + return + running, _ = check_rest_server_quick(rest_port) + if running: + response = rest_delete(trial_job_id_url(rest_port, args.trial_id), REST_TIME_OUT) + if response and check_response(response): + print(response.text) + return True + else: + print_error('Kill trial job failed...') + else: + print_error('Restful server is not running...') + return False + +def trial_codegen(args): + '''Generate code for a specific trial''' + print_warning('Currently, this command is only for nni nas programming interface.') + exp_id = get_config_filename(args) + experiment_config = Config(exp_id, Experiments().get_all_experiments()[exp_id]['logDir']).get_config() + if not experiment_config.get('useAnnotation'): + print_error('The experiment is not using annotation') + exit(1) + code_dir = experiment_config['trial']['codeDir'] + expand_annotations(code_dir, './exp_%s_trial_%s_code'%(exp_id, args.trial_id), exp_id, args.trial_id) + +def list_experiment(args): + '''Get experiment information''' + experiments_config = Experiments() + experiments_dict = experiments_config.get_all_experiments() + experiment_id = get_config_filename(args) + rest_port = experiments_dict.get(experiment_id).get('port') + rest_pid = experiments_dict.get(experiment_id).get('pid') + if not detect_process(rest_pid): + print_error('Experiment is not running...') + return + running, _ = check_rest_server_quick(rest_port) + if running: + response = rest_get(experiment_url(rest_port), REST_TIME_OUT) + if response and check_response(response): + content = convert_time_stamp_to_date(json.loads(response.text)) + print(json.dumps(content, indent=4, sort_keys=True, separators=(',', ':'))) + return content + else: + print_error('List experiment failed...') + else: + print_error('Restful server is not running...') + return None + +def experiment_status(args): + '''Show the status of experiment''' + experiments_config = Experiments() + experiments_dict = experiments_config.get_all_experiments() + rest_port = experiments_dict.get(get_config_filename(args)).get('port') + result, response = check_rest_server_quick(rest_port) + if not result: + print_normal('Restful server is not running...') + else: + print(json.dumps(json.loads(response.text), indent=4, sort_keys=True, separators=(',', ':'))) + return result + +def log_internal(args, filetype): + '''internal function to call get_log_content''' + file_name = get_config_filename(args) + if filetype == 'stdout': + file_full_path = os.path.join(NNI_HOME_DIR, file_name, 'log', 'nnictl_stdout.log') + else: + file_full_path = os.path.join(NNI_HOME_DIR, file_name, 'log', 'nnictl_stderr.log') + print(check_output_command(file_full_path, head=args.head, tail=args.tail)) + +def log_stdout(args): + '''get stdout log''' + log_internal(args, 'stdout') + +def log_stderr(args): + '''get stderr log''' + log_internal(args, 'stderr') + +def log_trial_adl_helper(args, experiment_id): + # adljob_id format should be consistent to the one in "adlTrainingService.ts": + # const adlJobName: string = `nni-exp-${this.experimentId}-trial-${trialJobId}`.toLowerCase(); + adlJobName = "nni-exp-{}-trial-{}".format(experiment_id, args.trial_id).lower() + print_warning('Note that no log will show when trial is pending or done (succeeded or failed). ' + 'You can retry the command.') + print_green('>>> Trial log streaming:') + try: + subprocess.run( + [ + "kubectl", "logs", + "-l", "adaptdl/job=%s" % adlJobName, + "-f" # Follow the stream + ], # TODO: support remaining argument, uncomment the lines in nnictl.py + ) # TODO: emulate tee behaviors, not necessary tho. + except KeyboardInterrupt: + pass + except Exception: + print_error('Error! Please check kubectl:') + traceback.print_exc() + exit(1) + finally: + print_green('<<< [adlJobName:%s]' % adlJobName) + nni_manager_collection_path = os.path.expanduser('~/nni-experiments/%s/trials/%s/stdout_log_collection.log' % + (experiment_id, args.trial_id)) + print_green('>>> (Optional) How to persist the complete trial log locally:') + print( + 'Please ensure `logCollection: http` ' + 'exists in the experiment configuration yaml. ' + 'After trial done, you can check it from the file below: \n %s' + % nni_manager_collection_path + ) + + +def log_trial(args): + ''''get trial log path''' + trial_id_path_dict = {} + trial_id_list = [] + experiments_config = Experiments() + experiments_dict = experiments_config.get_all_experiments() + experiment_id = get_config_filename(args) + rest_port = experiments_dict.get(experiment_id).get('port') + rest_pid = experiments_dict.get(experiment_id).get('pid') + experiment_config = Config(experiment_id, experiments_dict.get(experiment_id).get('logDir')).get_config() + if not detect_process(rest_pid): + print_error('Experiment is not running...') + return + running, response = check_rest_server_quick(rest_port) + if running: + response = rest_get(trial_jobs_url(rest_port), REST_TIME_OUT) + if response and check_response(response): + content = json.loads(response.text) + for trial in content: + trial_id_list.append(trial.get('trialJobId')) + if trial.get('logPath'): + trial_id_path_dict[trial.get('trialJobId')] = trial['logPath'] + else: + print_error('Restful server is not running...') + exit(1) + is_adl = experiment_config.get('trainingServicePlatform') == 'adl' + if is_adl and not args.trial_id: + print_error('Trial ID is required to retrieve the log for adl. Please specify it with "--trial_id".') + exit(1) + if args.trial_id: + if args.trial_id not in trial_id_list: + print_error('Trial id {0} not correct, please check your command!'.format(args.trial_id)) + exit(1) + if is_adl: + log_trial_adl_helper(args, experiment_id) + # adl has its own way to log trial, and it thus returns right after the helper returns + return + if trial_id_path_dict.get(args.trial_id): + print_normal('id:' + args.trial_id + ' path:' + trial_id_path_dict[args.trial_id]) + else: + print_error('Log path is not available yet, please wait...') + exit(1) + else: + print_normal('All of trial log info:') + for key in trial_id_path_dict: + print_normal('id:' + key + ' path:' + trial_id_path_dict[key]) + if not trial_id_path_dict: + print_normal('None') + +def get_config(args): + '''get config info''' + experiment_id = get_config_filename(args) + experiment_config = Config(experiment_id, Experiments().get_all_experiments()[experiment_id]['logDir']).get_config() + print(json.dumps(experiment_config, indent=4)) + +def webui_url(args): + '''show the url of web ui''' + experiment_id = get_config_filename(args) + experiments_dict = Experiments().get_all_experiments() + print_normal('{0} {1}'.format('Web UI url:', ' '.join(experiments_dict[experiment_id].get('webuiUrl')))) + +def webui_nas(args): + '''launch nas ui''' + print_normal('Starting NAS UI...') + try: + entry_dir = nni_node.__path__[0] + entry_file = os.path.join(entry_dir, 'nasui', 'server.js') + if sys.platform == 'win32': + node_command = os.path.join(entry_dir, 'node.exe') + else: + node_command = os.path.join(entry_dir, 'node') + cmds = [node_command, '--max-old-space-size=4096', entry_file, '--port', str(args.port), '--logdir', args.logdir] + subprocess.run(cmds, cwd=entry_dir) + except KeyboardInterrupt: + pass + +def local_clean(directory): + '''clean up local data''' + print_normal('removing folder {0}'.format(directory)) + try: + shutil.rmtree(directory) + except FileNotFoundError: + print_error('{0} does not exist.'.format(directory)) + +def remote_clean(machine_list, experiment_id=None): + '''clean up remote data''' + for machine in machine_list: + passwd = machine.get('passwd') + userName = machine.get('username') + host = machine.get('ip') + port = machine.get('port') + sshKeyPath = machine.get('sshKeyPath') + passphrase = machine.get('passphrase') + if experiment_id: + remote_dir = '/' + '/'.join(['tmp', 'nni-experiments', experiment_id]) + else: + remote_dir = '/' + '/'.join(['tmp', 'nni-experiments']) + sftp = create_ssh_sftp_client(host, port, userName, passwd, sshKeyPath, passphrase) + print_normal('removing folder {0}'.format(host + ':' + str(port) + remote_dir)) + remove_remote_directory(sftp, remote_dir) + +def experiment_clean(args): + '''clean up the experiment data''' + experiment_id_list = [] + experiments_config = Experiments() + experiments_dict = experiments_config.get_all_experiments() + if args.all: + experiment_id_list = list(experiments_dict.keys()) + else: + if args.id is None: + print_error('please set experiment id.') + exit(1) + if args.id not in experiments_dict: + print_error('Cannot find experiment {0}.'.format(args.id)) + exit(1) + experiment_id_list.append(args.id) + while True: + print('INFO: This action will delete experiment {0}, and it\'s not recoverable.'.format(' '.join(experiment_id_list))) + inputs = input('INFO: do you want to continue?[y/N]:') + if not inputs.lower() or inputs.lower() in ['n', 'no']: + exit(0) + elif inputs.lower() not in ['y', 'n', 'yes', 'no']: + print_warning('please input Y or N.') + else: + break + for experiment_id in experiment_id_list: + experiment_id = get_config_filename(args) + experiment_config = Config(experiment_id, Experiments().get_all_experiments()[experiment_id]['logDir']).get_config() + platform = experiment_config.get('trainingServicePlatform') + if platform == 'remote': + machine_list = experiment_config.get('machineList') + remote_clean(machine_list, experiment_id) + elif platform != 'local': + # TODO: support all platforms + print_warning('platform {0} clean up not supported yet.'.format(platform)) + exit(0) + # clean local data + local_base_dir = experiments_config.experiments[experiment_id]['logDir'] + if not local_base_dir: + local_base_dir = NNI_HOME_DIR + local_experiment_dir = os.path.join(local_base_dir, experiment_id) + experiment_folder_name_list = ['checkpoint', 'db', 'log', 'trials'] + for folder_name in experiment_folder_name_list: + local_clean(os.path.join(local_experiment_dir, folder_name)) + if not os.listdir(local_experiment_dir): + local_clean(local_experiment_dir) + print_normal('removing metadata of experiment {0}'.format(experiment_id)) + experiments_config.remove_experiment(experiment_id) + print_normal('Done.') + +def get_platform_dir(config_content): + '''get the dir list to be deleted''' + platform = config_content.get('trainingServicePlatform') + dir_list = [] + if platform == 'remote': + machine_list = config_content.get('machineList') + for machine in machine_list: + host = machine.get('ip') + port = machine.get('port') + dir_list.append(host + ':' + str(port) + '/tmp/nni') + elif platform == 'pai': + host = config_content.get('paiConfig').get('host') + user_name = config_content.get('paiConfig').get('userName') + output_dir = config_content.get('trial').get('outputDir') + dir_list.append('server: {0}, path: {1}/nni'.format(host, user_name)) + if output_dir: + dir_list.append(output_dir) + return dir_list + +def platform_clean(args): + '''clean up the experiment data''' + config_path = os.path.abspath(args.config) + if not os.path.exists(config_path): + print_error('Please set correct config path.') + exit(1) + config_content = get_yml_content(config_path) + platform = config_content.get('trainingServicePlatform') + if platform == 'local': + print_normal('it doesn’t need to clean local platform.') + exit(0) + if platform not in ['remote', 'pai']: + print_normal('platform {0} not supported.'.format(platform)) + exit(0) + update_experiment() + dir_list = get_platform_dir(config_content) + if not dir_list: + print_normal('No folder of NNI caches is found.') + exit(1) + while True: + print_normal('This command will remove below folders of NNI caches. If other users are using experiments' \ + ' on below hosts, it will be broken.') + for value in dir_list: + print(' ' + value) + inputs = input('INFO: do you want to continue?[y/N]:') + if not inputs.lower() or inputs.lower() in ['n', 'no']: + exit(0) + elif inputs.lower() not in ['y', 'n', 'yes', 'no']: + print_warning('please input Y or N.') + else: + break + if platform == 'remote': + machine_list = config_content.get('machineList') + remote_clean(machine_list) + print_normal('Done.') + +def experiment_list(args): + '''get the information of all experiments''' + update_experiment() + experiments_config = Experiments() + experiments_dict = experiments_config.get_all_experiments() + if not experiments_dict: + print_normal('Cannot find experiments.') + exit(1) + experiment_id_list = [] + if args.all: + for key in experiments_dict.keys(): + experiment_id_list.append(key) + else: + for key in experiments_dict.keys(): + if experiments_dict[key]['status'] != 'STOPPED': + experiment_id_list.append(key) + if not experiment_id_list: + print_warning('There is no experiment running...\nYou can use \'nnictl experiment list --all\' to list all experiments.') + experiment_information = "" + for key in experiment_id_list: + experiment_information += EXPERIMENT_DETAIL_FORMAT % (key, + experiments_dict[key].get('experimentName', 'N/A'), + experiments_dict[key]['status'], + experiments_dict[key].get('port', 'N/A'), + experiments_dict[key].get('platform'), + time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(experiments_dict[key]['startTime'] / 1000)) if isinstance(experiments_dict[key]['startTime'], int) else experiments_dict[key]['startTime'], + time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(experiments_dict[key]['endTime'] / 1000)) if isinstance(experiments_dict[key]['endTime'], int) else experiments_dict[key]['endTime']) + print(EXPERIMENT_INFORMATION_FORMAT % experiment_information) + return experiment_id_list + +def get_time_interval(time1, time2): + '''get the interval of two times''' + try: + seconds = int((time2 - time1) / 1000) + #convert seconds to day:hour:minute:second + days = seconds / 86400 + seconds %= 86400 + hours = seconds / 3600 + seconds %= 3600 + minutes = seconds / 60 + seconds %= 60 + return '%dd %dh %dm %ds' % (days, hours, minutes, seconds) + except: + return 'N/A' + +def show_experiment_info(): + '''show experiment information in monitor''' + update_experiment() + experiments_config = Experiments() + experiments_dict = experiments_config.get_all_experiments() + if not experiments_dict: + print('There is no experiment running...') + exit(1) + experiment_id_list = [] + for key in experiments_dict.keys(): + if experiments_dict[key]['status'] != 'STOPPED': + experiment_id_list.append(key) + if not experiment_id_list: + print_warning('There is no experiment running...') + return + for key in experiment_id_list: + print(EXPERIMENT_MONITOR_INFO % (key, experiments_dict[key]['status'], experiments_dict[key]['port'], \ + experiments_dict[key].get('platform'), time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(experiments_dict[key]['startTime'] / 1000)) if isinstance(experiments_dict[key]['startTime'], int) else experiments_dict[key]['startTime'], \ + get_time_interval(experiments_dict[key]['startTime'], experiments_dict[key]['endTime']))) + print(TRIAL_MONITOR_HEAD) + running, response = check_rest_server_quick(experiments_dict[key]['port']) + if running: + response = rest_get(trial_jobs_url(experiments_dict[key]['port']), REST_TIME_OUT) + if response and check_response(response): + content = json.loads(response.text) + for index, value in enumerate(content): + content[index] = convert_time_stamp_to_date(value) + print(TRIAL_MONITOR_CONTENT % (content[index].get('trialJobId'), content[index].get('startTime'), \ + content[index].get('endTime'), content[index].get('status'))) + print(TRIAL_MONITOR_TAIL) + +def set_monitor(auto_exit, time_interval, port=None, pid=None): + '''set the experiment monitor engine''' + while True: + try: + if sys.platform == 'win32': + os.system('cls') + else: + os.system('clear') + update_experiment() + show_experiment_info() + if auto_exit: + status = get_experiment_status(port) + if status in ['DONE', 'ERROR', 'STOPPED']: + print_normal('Experiment status is {0}.'.format(status)) + print_normal('Stopping experiment...') + kill_command(pid) + print_normal('Stop experiment success.') + exit(0) + time.sleep(time_interval) + except KeyboardInterrupt: + if auto_exit: + print_normal('Stopping experiment...') + kill_command(pid) + print_normal('Stop experiment success.') + else: + print_normal('Exiting...') + exit(0) + except Exception as exception: + print_error(exception) + exit(1) + +def monitor_experiment(args): + '''monitor the experiment''' + if args.time <= 0: + print_error('please input a positive integer as time interval, the unit is second.') + exit(1) + set_monitor(False, args.time) + +def export_trials_data(args): + '''export experiment metadata and intermediate results to json or csv + ''' + def groupby_trial_id(intermediate_results): + sorted(intermediate_results, key=lambda x: x['timestamp']) + groupby = dict() + for content in intermediate_results: + groupby.setdefault(content['trialJobId'], []).append(json.loads(content['data'])) + return groupby + + experiments_config = Experiments() + experiments_dict = experiments_config.get_all_experiments() + experiment_id = get_config_filename(args) + rest_port = experiments_dict.get(experiment_id).get('port') + rest_pid = experiments_dict.get(experiment_id).get('pid') + + if not detect_process(rest_pid): + print_error('Experiment is not running...') + return + running, response = check_rest_server_quick(rest_port) + if not running: + print_error('Restful server is not running') + return + response = rest_get(export_data_url(rest_port), 20) + if response is not None and check_response(response): + content = json.loads(response.text) + if args.intermediate: + intermediate_results_response = rest_get(metric_data_url(rest_port), REST_TIME_OUT) + if not intermediate_results_response or not check_response(intermediate_results_response): + print_error('Error getting intermediate results.') + return + intermediate_results = groupby_trial_id(json.loads(intermediate_results_response.text)) + for record in content: + record['intermediate'] = intermediate_results[record['trialJobId']] + if args.type == 'json': + with open(args.path, 'w') as file: + file.write(json.dumps(content)) + elif args.type == 'csv': + trial_records = [] + for record in content: + formated_record = dict() + if args.intermediate: + formated_record['intermediate'] = '[' + ','.join(record['intermediate']) + ']' + record_value = json.loads(record['value']) + if not isinstance(record_value, (float, int)): + formated_record.update({**record['parameter'], **record_value, **{'trialJobId': record['trialJobId']}}) + else: + formated_record.update({**record['parameter'], **{'reward': record_value, 'trialJobId': record['trialJobId']}}) + trial_records.append(formated_record) + if not trial_records: + print_error('No trial results collected! Please check your trial log...') + exit(0) + with open(args.path, 'w', newline='') as file: + writer = csv.DictWriter(file, set.union(*[set(r.keys()) for r in trial_records])) + writer.writeheader() + writer.writerows(trial_records) + else: + print_error('Unknown type: %s' % args.type) + return + else: + print_error('Export failed...') + +def search_space_auto_gen(args): + '''dry run trial code to generate search space file''' + trial_dir = os.path.expanduser(args.trial_dir) + file_path = os.path.expanduser(args.file) + if not os.path.isabs(file_path): + file_path = os.path.join(os.getcwd(), file_path) + assert os.path.exists(trial_dir) + if os.path.exists(file_path): + print_warning('%s already exists, will be overwritten.' % file_path) + print_normal('Dry run to generate search space...') + Popen(args.trial_command, cwd=trial_dir, env=dict(os.environ, NNI_GEN_SEARCH_SPACE=file_path), shell=True).wait() + if not os.path.exists(file_path): + print_warning('Expected search space file \'{}\' generated, but not found.'.format(file_path)) + else: + print_normal('Generate search space done: \'{}\'.'.format(file_path)) + +def save_experiment(args): + '''save experiment data to a zip file''' + experiments_config = Experiments() + experiments_dict = experiments_config.get_all_experiments() + if args.id is None: + print_error('Please set experiment id.') + exit(1) + if args.id not in experiments_dict: + print_error('Cannot find experiment {0}.'.format(args.id)) + exit(1) + if experiments_dict[args.id].get('status') != 'STOPPED': + print_error('Can only save stopped experiment!') + exit(1) + print_normal('Saving...') + experiment_config = Config(args.id, experiments_dict[args.id]['logDir']).get_config() + logDir = os.path.join(experiments_dict[args.id]['logDir'], args.id) + temp_root_dir = generate_temp_dir() + + # Step1. Copy logDir to temp folder + if not os.path.exists(logDir): + print_error('logDir: %s does not exist!' % logDir) + exit(1) + temp_experiment_dir = os.path.join(temp_root_dir, 'experiment') + shutil.copytree(logDir, temp_experiment_dir) + + # Step2. Copy nnictl metadata to temp folder + temp_nnictl_dir = os.path.join(temp_root_dir, 'nnictl') + os.makedirs(temp_nnictl_dir, exist_ok=True) + try: + with open(os.path.join(temp_nnictl_dir, '.experiment'), 'w') as file: + experiments_dict[args.id]['id'] = args.id + json.dump(experiments_dict[args.id], file) + except IOError: + print_error('Write file to %s failed!' % os.path.join(temp_nnictl_dir, '.experiment')) + exit(1) + nnictl_log_dir = os.path.join(NNI_HOME_DIR, args.id, 'log') + shutil.copytree(nnictl_log_dir, os.path.join(temp_nnictl_dir, args.id, 'log')) + + # Step3. Copy code dir + if args.saveCodeDir: + temp_code_dir = os.path.join(temp_root_dir, 'code') + shutil.copytree(experiment_config['trial']['codeDir'], temp_code_dir) + + # Step4. Copy searchSpace file + search_space_path = experiment_config.get('searchSpacePath') + if search_space_path: + if not os.path.exists(search_space_path): + print_warning('search space %s does not exist!' % search_space_path) + else: + temp_search_space_dir = os.path.join(temp_root_dir, 'searchSpace') + os.makedirs(temp_search_space_dir, exist_ok=True) + search_space_name = os.path.basename(search_space_path) + shutil.copyfile(search_space_path, os.path.join(temp_search_space_dir, search_space_name)) + + # Step5. Archive folder + zip_package_name = 'nni_experiment_%s' % args.id + if args.path: + os.makedirs(args.path, exist_ok=True) + zip_package_name = os.path.join(args.path, zip_package_name) + shutil.make_archive(zip_package_name, 'zip', temp_root_dir) + print_normal('Save to %s.zip success!' % zip_package_name) + + # Step5. Cleanup temp data + shutil.rmtree(temp_root_dir) + +def load_experiment(args): + '''load experiment data''' + package_path = os.path.expanduser(args.path) + if not os.path.exists(args.path): + print_error('file path %s does not exist!' % args.path) + exit(1) + if args.searchSpacePath and os.path.isdir(args.searchSpacePath): + print_error('search space path should be a full path with filename, not a directory!') + exit(1) + temp_root_dir = generate_temp_dir() + shutil.unpack_archive(package_path, temp_root_dir) + print_normal('Loading...') + # Step1. Validation + if not os.path.exists(args.codeDir): + print_error('Invalid: codeDir path does not exist!') + exit(1) + if args.logDir: + if not os.path.exists(args.logDir): + print_error('Invalid: logDir path does not exist!') + exit(1) + experiment_temp_dir = os.path.join(temp_root_dir, 'experiment') + if not os.path.exists(os.path.join(experiment_temp_dir, 'db')): + print_error('Invalid archive file: db file does not exist!') + shutil.rmtree(temp_root_dir) + exit(1) + nnictl_temp_dir = os.path.join(temp_root_dir, 'nnictl') + if not os.path.exists(os.path.join(nnictl_temp_dir, '.experiment')): + print_error('Invalid archive file: nnictl metadata file does not exist!') + shutil.rmtree(temp_root_dir) + exit(1) + try: + with open(os.path.join(nnictl_temp_dir, '.experiment'), 'r') as file: + experiment_metadata = json.load(file) + except ValueError as err: + print_error('Invalid nnictl metadata file: %s' % err) + shutil.rmtree(temp_root_dir) + exit(1) + experiments_config = Experiments() + experiments_dict = experiments_config.get_all_experiments() + experiment_id = experiment_metadata.get('id') + if experiment_id in experiments_dict: + print_error('Invalid: experiment id already exist!') + shutil.rmtree(temp_root_dir) + exit(1) + if not os.path.exists(os.path.join(nnictl_temp_dir, experiment_id)): + print_error('Invalid: experiment metadata does not exist!') + shutil.rmtree(temp_root_dir) + exit(1) + + # Step2. Copy nnictl metadata + src_path = os.path.join(nnictl_temp_dir, experiment_id) + dest_path = os.path.join(NNI_HOME_DIR, experiment_id) + if os.path.exists(dest_path): + shutil.rmtree(dest_path) + shutil.copytree(src_path, dest_path) + + # Step3. Copy experiment data + os.rename(os.path.join(temp_root_dir, 'experiment'), os.path.join(temp_root_dir, experiment_id)) + src_path = os.path.join(os.path.join(temp_root_dir, experiment_id)) + experiment_config = Config(experiment_id, temp_root_dir).get_config() + if args.logDir: + logDir = args.logDir + experiment_config['logDir'] = logDir + else: + if experiment_config.get('logDir'): + logDir = experiment_config['logDir'] + else: + logDir = NNI_HOME_DIR + + dest_path = os.path.join(logDir, experiment_id) + if os.path.exists(dest_path): + shutil.rmtree(dest_path) + shutil.copytree(src_path, dest_path) + + # Step4. Copy code dir + codeDir = os.path.expanduser(args.codeDir) + if not os.path.isabs(codeDir): + codeDir = os.path.join(os.getcwd(), codeDir) + print_normal('Expand codeDir to %s' % codeDir) + experiment_config['trial']['codeDir'] = codeDir + archive_code_dir = os.path.join(temp_root_dir, 'code') + if os.path.exists(archive_code_dir): + file_list = os.listdir(archive_code_dir) + for file_name in file_list: + src_path = os.path.join(archive_code_dir, file_name) + target_path = os.path.join(codeDir, file_name) + if os.path.exists(target_path): + print_error('Copy %s failed, %s exist!' % (file_name, target_path)) + continue + if os.path.isdir(src_path): + shutil.copytree(src_path, target_path) + else: + shutil.copy(src_path, target_path) + + # Step5. Create experiment metadata + experiments_config.add_experiment(experiment_id, + experiment_metadata.get('port'), + experiment_metadata.get('startTime'), + experiment_metadata.get('platform'), + experiment_metadata.get('experimentName'), + experiment_metadata.get('endTime'), + experiment_metadata.get('status'), + experiment_metadata.get('tag'), + experiment_metadata.get('pid'), + experiment_metadata.get('webUrl'), + logDir) + print_normal('Load experiment %s succsss!' % experiment_id) + + # Step6. Cleanup temp data + shutil.rmtree(temp_root_dir) diff --git a/utils/third_party/nni_new/tools/nnictl/rest_utils.py b/utils/third_party/nni_new/tools/nnictl/rest_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..e98c9a839245ca0775406302dd6bf40bf8a2ae35 --- /dev/null +++ b/utils/third_party/nni_new/tools/nnictl/rest_utils.py @@ -0,0 +1,77 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import time +import requests +from .url_utils import check_status_url +from .constants import REST_TIME_OUT +from .common_utils import print_error + +def rest_put(url, data, timeout, show_error=False): + '''Call rest put method''' + try: + response = requests.put(url, headers={'Accept': 'application/json', 'Content-Type': 'application/json'},\ + data=data, timeout=timeout) + return response + except Exception as exception: + if show_error: + print_error(exception) + return None + +def rest_post(url, data, timeout, show_error=False): + '''Call rest post method''' + try: + response = requests.post(url, headers={'Accept': 'application/json', 'Content-Type': 'application/json'},\ + data=data, timeout=timeout) + return response + except Exception as exception: + if show_error: + print_error(exception) + return None + +def rest_get(url, timeout, show_error=False): + '''Call rest get method''' + try: + response = requests.get(url, timeout=timeout) + return response + except Exception as exception: + if show_error: + print_error(exception) + return None + +def rest_delete(url, timeout, show_error=False): + '''Call rest delete method''' + try: + response = requests.delete(url, timeout=timeout) + return response + except Exception as exception: + if show_error: + print_error(exception) + return None + +def check_rest_server(rest_port): + '''Check if restful server is ready''' + retry_count = 20 + for _ in range(retry_count): + response = rest_get(check_status_url(rest_port), REST_TIME_OUT) + if response: + if response.status_code == 200: + return True, response + else: + return False, response + else: + time.sleep(1) + return False, response + +def check_rest_server_quick(rest_port): + '''Check if restful server is ready, only check once''' + response = rest_get(check_status_url(rest_port), 5) + if response and response.status_code == 200: + return True, response + return False, None + +def check_response(response): + '''Check if a response is success according to status_code''' + if response and response.status_code == 200: + return True + return False diff --git a/utils/third_party/nni_new/tools/nnictl/ssh_utils.py b/utils/third_party/nni_new/tools/nnictl/ssh_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..e3f26a8e24c1be67bf86cfa0192481b245c93f97 --- /dev/null +++ b/utils/third_party/nni_new/tools/nnictl/ssh_utils.py @@ -0,0 +1,60 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +from .common_utils import print_error +from .command_utils import install_package_command + +def check_environment(): + '''check if paramiko is installed''' + try: + import paramiko + except: + install_package_command('paramiko') + import paramiko + return paramiko + +def copy_remote_directory_to_local(sftp, remote_path, local_path): + '''copy remote directory to local machine''' + try: + os.makedirs(local_path, exist_ok=True) + files = sftp.listdir(remote_path) + for file in files: + remote_full_path = os.path.join(remote_path, file) + local_full_path = os.path.join(local_path, file) + try: + if sftp.listdir(remote_full_path): + copy_remote_directory_to_local(sftp, remote_full_path, local_full_path) + except: + sftp.get(remote_full_path, local_full_path) + except Exception: + pass + +def create_ssh_sftp_client(host_ip, port, username, password, ssh_key_path, passphrase): + '''create ssh client''' + try: + paramiko = check_environment() + conn = paramiko.Transport(host_ip, port) + if ssh_key_path is not None: + ssh_key = paramiko.RSAKey.from_private_key_file(ssh_key_path, password=passphrase) + conn.connect(username=username, pkey=ssh_key) + else: + conn.connect(username=username, password=password) + sftp = paramiko.SFTPClient.from_transport(conn) + return sftp + except Exception as exception: + print_error('Create ssh client error %s\n' % exception) + +def remove_remote_directory(sftp, directory): + '''remove a directory in remote machine''' + try: + files = sftp.listdir(directory) + for file in files: + filepath = '/'.join([directory, file]) + try: + sftp.remove(filepath) + except IOError: + remove_remote_directory(sftp, filepath) + sftp.rmdir(directory) + except IOError as err: + print_error(err) diff --git a/utils/third_party/nni_new/tools/nnictl/ts_management.py b/utils/third_party/nni_new/tools/nnictl/ts_management.py new file mode 100644 index 0000000000000000000000000000000000000000..151b053591b549734ae37d5da3afa1bed3694a4f --- /dev/null +++ b/utils/third_party/nni_new/tools/nnictl/ts_management.py @@ -0,0 +1,77 @@ +import importlib +import json + +from nni.runtime.config import get_config_file +from .common_utils import print_error, print_green + +_builtin_training_services = [ + 'local', + 'remote', + 'openpai', 'pai', + 'aml', + 'kubeflow', + 'frameworkcontroller', + 'adl', +] + +def register(args): + if args.package in _builtin_training_services: + print_error(f'{args.package} is a builtin training service') + return + + try: + module = importlib.import_module(args.package) + except Exception: + print_error(f'Cannot import package {args.package}') + return + + try: + info = module.nni_training_service_info + except Exception: + print_error(f'Cannot read nni_training_service_info from {args.package}') + return + + try: + info.config_class() + except Exception: + print_error('Bad experiment config class') + return + + try: + service_config = { + 'nodeModulePath': str(info.node_module_path), + 'nodeClassName': info.node_class_name, + } + json.dumps(service_config) + except Exception: + print_error('Bad node_module_path or bad node_class_name') + return + + config = _load() + update = args.package in config + + config[args.package] = service_config + _save(config) + + if update: + print_green(f'Sucessfully updated {args.package}') + else: + print_green(f'Sucessfully registered {args.package}') + +def unregister(args): + config = _load() + if args.package not in config: + print_error(f'{args.package} is not a registered training service') + return + config.pop(args.package, None) + _save(config) + print_green(f'Sucessfully unregistered {args.package}') + +def list_services(_): + print('\n'.join(_load().keys())) + +def _load(): + return json.load(get_config_file('training_services.json').open()) + +def _save(config): + json.dump(config, get_config_file('training_services.json').open('w'), indent=4) diff --git a/utils/third_party/nni_new/tools/nnictl/updater.py b/utils/third_party/nni_new/tools/nnictl/updater.py new file mode 100644 index 0000000000000000000000000000000000000000..e462562349914c222f29597b7946b1017040cf28 --- /dev/null +++ b/utils/third_party/nni_new/tools/nnictl/updater.py @@ -0,0 +1,151 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import json +import os +from .rest_utils import rest_put, rest_post, rest_get, check_rest_server_quick, check_response +from .url_utils import experiment_url, import_data_url +from .config_utils import Config, Experiments +from .common_utils import get_json_content, print_normal, print_error, print_warning +from .nnictl_utils import get_experiment_port, get_config_filename, detect_process +from .launcher_utils import parse_time +from .constants import REST_TIME_OUT, TUNERS_SUPPORTING_IMPORT_DATA, TUNERS_NO_NEED_TO_IMPORT_DATA + +def validate_digit(value, start, end): + '''validate if a digit is valid''' + if not str(value).isdigit() or int(value) < start or int(value) > end: + raise ValueError('value (%s) must be a digit from %s to %s' % (value, start, end)) + +def validate_file(path): + '''validate if a file exist''' + if not os.path.exists(path): + raise FileNotFoundError('%s is not a valid file path' % path) + +def validate_dispatcher(args): + '''validate if the dispatcher of the experiment supports importing data''' + experiment_id = get_config_filename(args) + experiment_config = Config(experiment_id, Experiments().get_all_experiments()[experiment_id]['logDir']).get_config() + if experiment_config.get('tuner') and experiment_config['tuner'].get('builtinTunerName'): + dispatcher_name = experiment_config['tuner']['builtinTunerName'] + elif experiment_config.get('advisor') and experiment_config['advisor'].get('builtinAdvisorName'): + dispatcher_name = experiment_config['advisor']['builtinAdvisorName'] + else: # otherwise it should be a customized one + return + if dispatcher_name not in TUNERS_SUPPORTING_IMPORT_DATA: + if dispatcher_name in TUNERS_NO_NEED_TO_IMPORT_DATA: + print_warning("There is no need to import data for %s" % dispatcher_name) + exit(0) + else: + print_error("%s does not support importing addtional data" % dispatcher_name) + exit(1) + +def load_search_space(path): + '''load search space content''' + content = json.dumps(get_json_content(path)) + if not content: + raise ValueError('searchSpace file should not be empty') + return content + +def get_query_type(key): + '''get update query type''' + if key == 'trialConcurrency': + return '?update_type=TRIAL_CONCURRENCY' + if key == 'maxExecDuration': + return '?update_type=MAX_EXEC_DURATION' + if key == 'searchSpace': + return '?update_type=SEARCH_SPACE' + if key == 'maxTrialNum': + return '?update_type=MAX_TRIAL_NUM' + +def update_experiment_profile(args, key, value): + '''call restful server to update experiment profile''' + experiments_config = Experiments() + experiments_dict = experiments_config.get_all_experiments() + rest_port = experiments_dict.get(get_config_filename(args)).get('port') + running, _ = check_rest_server_quick(rest_port) + if running: + response = rest_get(experiment_url(rest_port), REST_TIME_OUT) + if response and check_response(response): + experiment_profile = json.loads(response.text) + experiment_profile['params'][key] = value + response = rest_put(experiment_url(rest_port)+get_query_type(key), json.dumps(experiment_profile), REST_TIME_OUT) + if response and check_response(response): + return response + else: + print_error('Restful server is not running...') + return None + +def update_searchspace(args): + validate_file(args.filename) + content = load_search_space(args.filename) + args.port = get_experiment_port(args) + if args.port is not None: + if update_experiment_profile(args, 'searchSpace', content): + print_normal('Update %s success!' % 'searchSpace') + else: + print_error('Update %s failed!' % 'searchSpace') + + +def update_concurrency(args): + validate_digit(args.value, 1, 1000) + args.port = get_experiment_port(args) + if args.port is not None: + if update_experiment_profile(args, 'trialConcurrency', int(args.value)): + print_normal('Update %s success!' % 'concurrency') + else: + print_error('Update %s failed!' % 'concurrency') + +def update_duration(args): + #parse time, change time unit to seconds + args.value = parse_time(args.value) + args.port = get_experiment_port(args) + if args.port is not None: + if update_experiment_profile(args, 'maxExecDuration', int(args.value)): + print_normal('Update %s success!' % 'duration') + else: + print_error('Update %s failed!' % 'duration') + +def update_trialnum(args): + validate_digit(args.value, 1, 999999999) + if update_experiment_profile(args, 'maxTrialNum', int(args.value)): + print_normal('Update %s success!' % 'trialnum') + else: + print_error('Update %s failed!' % 'trialnum') + +def import_data(args): + '''import additional data to the experiment''' + validate_file(args.filename) + validate_dispatcher(args) + content = load_search_space(args.filename) + + experiments_dict = Experiments().get_all_experiments() + experiment_id = get_config_filename(args) + rest_port = experiments_dict.get(experiment_id).get('port') + rest_pid = experiments_dict.get(experiment_id).get('pid') + if not detect_process(rest_pid): + print_error('Experiment is not running...') + return + running, _ = check_rest_server_quick(rest_port) + if not running: + print_error('Restful server is not running') + return + + args.port = rest_port + if args.port is not None: + if import_data_to_restful_server(args, content): + pass + else: + print_error('Import data failed!') + +def import_data_to_restful_server(args, content): + '''call restful server to import data to the experiment''' + experiments_dict = Experiments().get_all_experiments() + rest_port = experiments_dict.get(get_config_filename(args)).get('port') + running, _ = check_rest_server_quick(rest_port) + if running: + response = rest_post(import_data_url(rest_port), content, REST_TIME_OUT) + if response and check_response(response): + return response + else: + print_error('Restful server is not running...') + return None diff --git a/utils/third_party/nni_new/tools/nnictl/url_utils.py b/utils/third_party/nni_new/tools/nnictl/url_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..a4f1b807d2d5784e95d2e38ad4b390a1b4306bbe --- /dev/null +++ b/utils/third_party/nni_new/tools/nnictl/url_utils.py @@ -0,0 +1,85 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import socket +import psutil + +BASE_URL = 'http://localhost' + +API_ROOT_URL = '/api/v1/nni' + +EXPERIMENT_API = '/experiment' + +CLUSTER_METADATA_API = '/experiment/cluster-metadata' + +IMPORT_DATA_API = '/experiment/import-data' + +CHECK_STATUS_API = '/check-status' + +TRIAL_JOBS_API = '/trial-jobs' + +EXPORT_DATA_API = '/export-data' + +TENSORBOARD_API = '/tensorboard' + +METRIC_DATA_API = '/metric-data' + +def format_url_path(path): + return API_ROOT_URL if path is None else f'/{path}{API_ROOT_URL}' + +def set_prefix_url(prefix_path): + global API_ROOT_URL + API_ROOT_URL = format_url_path(prefix_path) + +def metric_data_url(port): + '''get metric_data url''' + return '{0}:{1}{2}{3}'.format(BASE_URL, port, API_ROOT_URL, METRIC_DATA_API) + +def check_status_url(port): + '''get check_status url''' + return '{0}:{1}{2}{3}'.format(BASE_URL, port, API_ROOT_URL, CHECK_STATUS_API) + + +def cluster_metadata_url(port): + '''get cluster_metadata_url''' + return '{0}:{1}{2}{3}'.format(BASE_URL, port, API_ROOT_URL, CLUSTER_METADATA_API) + + +def import_data_url(port): + '''get import_data_url''' + return '{0}:{1}{2}{3}'.format(BASE_URL, port, API_ROOT_URL, IMPORT_DATA_API) + + +def experiment_url(port): + '''get experiment_url''' + return '{0}:{1}{2}{3}'.format(BASE_URL, port, API_ROOT_URL, EXPERIMENT_API) + + +def trial_jobs_url(port): + '''get trial_jobs url''' + return '{0}:{1}{2}{3}'.format(BASE_URL, port, API_ROOT_URL, TRIAL_JOBS_API) + + +def trial_job_id_url(port, job_id): + '''get trial_jobs with id url''' + return '{0}:{1}{2}{3}/{4}'.format(BASE_URL, port, API_ROOT_URL, TRIAL_JOBS_API, job_id) + + +def export_data_url(port): + '''get export_data url''' + return '{0}:{1}{2}{3}'.format(BASE_URL, port, API_ROOT_URL, EXPORT_DATA_API) + + +def tensorboard_url(port): + '''get tensorboard url''' + return '{0}:{1}{2}{3}'.format(BASE_URL, port, API_ROOT_URL, TENSORBOARD_API) + + +def get_local_urls(port,prefix): + '''get urls of local machine''' + url_list = [] + for _, info in psutil.net_if_addrs().items(): + for addr in info: + if socket.AddressFamily.AF_INET == addr.family: + url_list.append('http://{0}:{1}{2}'.format(addr.address, port, prefix)) + return url_list diff --git a/utils/third_party/nni_new/tools/package_utils/__init__.py b/utils/third_party/nni_new/tools/package_utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..7f8f4419651eaeddeb4043e8026a6443bff1fc2d --- /dev/null +++ b/utils/third_party/nni_new/tools/package_utils/__init__.py @@ -0,0 +1,229 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from collections import defaultdict +import importlib +import os +from pathlib import Path +import sys +import yaml +from nni.runtime.config import get_config_file + +ALGO_TYPES = ['tuners', 'assessors', 'advisors'] + +def get_all_builtin_names(algo_type): + """Get all builtin names of registered algorithms of specified type + + Parameters + ---------- + algo_type: str + can be one of 'tuners', 'assessors' or 'advisors' + + Returns: list of string + ------- + All builtin names of specified type, for example, if algo_type is 'tuners', returns + all builtin tuner names. + """ + assert algo_type in ALGO_TYPES + + return [x['builtinName'] for x in read_registerd_algo_meta()[algo_type]] + + +def get_registered_algo_meta(builtin_name, algo_type=None): + """ Get meta information of registered algorithms. + + Parameters + ---------- + builtin_name: str + builtin name. + algo_type: str | None + can be one of 'tuners', 'assessors', 'advisors' or None + + Returns: dict | None + ------- + Returns meta information of speicified builtin alogorithms, for example: + { + 'classArgsValidator': 'nni.smac_tuner.SMACClassArgsValidator', + 'className': 'nni.smac_tuner.SMACTuner', + 'builtinName': 'SMAC' + } + """ + assert builtin_name is not None + if algo_type: + assert algo_type in ALGO_TYPES + config = read_registerd_algo_meta() + + candidates = [] + if algo_type: + candidates = config[algo_type] + else: + for algo_type in ALGO_TYPES: + candidates.extend(config[algo_type]) + for meta in candidates: + if meta['builtinName'] == builtin_name: + return meta + return None + +def parse_full_class_name(full_class_name): + if not full_class_name: + return None, None + parts = full_class_name.split('.') + module_name, class_name = '.'.join(parts[:-1]), parts[-1] + return module_name, class_name + +def get_builtin_module_class_name(algo_type, builtin_name): + """Get module name and class name of all builtin algorithms + + Parameters + ---------- + algo_type: str + can be one of 'tuners', 'assessors', 'advisors' + builtin_name: str + builtin name. + + Returns: tuple + ------- + tuple of (module name, class name) + """ + assert algo_type in ALGO_TYPES + assert builtin_name is not None + meta = get_registered_algo_meta(builtin_name, algo_type) + if not meta: + return None, None + return parse_full_class_name(meta['className']) + +def create_validator_instance(algo_type, builtin_name): + """Create instance of validator class + + Parameters + ---------- + algo_type: str + can be one of 'tuners', 'assessors', 'advisors' + builtin_name: str + builtin name. + + Returns: object | None + ------- + Returns validator class instance. + If specified validator class does not exist, returns None. + """ + assert algo_type in ALGO_TYPES + assert builtin_name is not None + meta = get_registered_algo_meta(builtin_name, algo_type) + if not meta or 'classArgsValidator' not in meta: + return None + module_name, class_name = parse_full_class_name(meta['classArgsValidator']) + class_module = importlib.import_module(module_name) + class_constructor = getattr(class_module, class_name) + + return class_constructor() + +def create_builtin_class_instance(builtin_name, input_class_args, algo_type): + """Create instance of builtin algorithms + + Parameters + ---------- + builtin_name: str + builtin name. + input_class_args: dict + kwargs for builtin class constructor + algo_type: str + can be one of 'tuners', 'assessors', 'advisors' + + Returns: object + ------- + Returns builtin class instance. + """ + assert algo_type in ALGO_TYPES + if builtin_name not in get_all_builtin_names(algo_type): + raise RuntimeError('Builtin name is not found: {}'.format(builtin_name)) + + def parse_algo_meta(algo_meta, input_class_args): + """ + 1. parse class_name field in meta data into module name and class name, + for example: + parse class_name 'nni.hyperopt_tuner.hyperopt_tuner.HyperoptTuner' in meta data into: + module name: nni.hyperopt_tuner.hyperopt_tuner + class name: HyperoptTuner + 2. merge user specified class args together with builtin class args. + """ + assert algo_meta + module_name, class_name = parse_full_class_name(algo_meta['className']) + + class_args = {} + if 'classArgs' in algo_meta: + class_args = algo_meta['classArgs'] + if input_class_args is not None: + class_args.update(input_class_args) + + return module_name, class_name, class_args + + algo_meta = get_registered_algo_meta(builtin_name, algo_type) + module_name, class_name, class_args = parse_algo_meta(algo_meta, input_class_args) + + if importlib.util.find_spec(module_name) is None: + raise RuntimeError('Builtin module can not be loaded: {}'.format(module_name)) + + class_module = importlib.import_module(module_name) + class_constructor = getattr(class_module, class_name) + + instance = class_constructor(**class_args) + + return instance + +def create_customized_class_instance(class_params): + """Create instance of customized algorithms + + Parameters + ---------- + class_params: dict + class_params should contains following keys: + codeDirectory: code directory + className: qualified class name + classArgs (optional): kwargs pass to class constructor + + Returns: object + ------- + Returns customized class instance. + """ + + code_dir = class_params.get('codeDirectory') + qualified_class_name = class_params.get('className') + class_args = class_params.get('classArgs') + + if code_dir and not os.path.isdir(code_dir): + raise ValueError(f'Directory not found: {code_dir}') + + sys.path.append(code_dir) + module_name, class_name = qualified_class_name.rsplit('.', 1) + class_module = importlib.import_module(module_name) + class_constructor = getattr(class_module, class_name) + + if class_args is None: + class_args = {} + instance = class_constructor(**class_args) + + return instance + +def _using_conda_or_virtual_environment(): + return sys.prefix != sys.base_prefix or os.path.isdir(os.path.join(sys.prefix, 'conda-meta')) + +def get_registered_algo_config_path(): + return str(get_config_file('registered_algorithms.yml')) + +def read_registerd_algo_meta(): + config_file = get_registered_algo_config_path() + if os.path.exists(config_file): + with open(config_file, 'r') as f: + config = yaml.safe_load(f) + else: + config = defaultdict(list) + for t in ALGO_TYPES: + if t not in config: + config[t] = [] + return config + +def write_registered_algo_meta(config): + config_file = get_registered_algo_config_path() + with open(config_file, 'w') as f: + f.write(yaml.safe_dump(dict(config), default_flow_style=False)) diff --git a/utils/third_party/nni_new/tools/trial_tool/__init__.py b/utils/third_party/nni_new/tools/trial_tool/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/utils/third_party/nni_new/tools/trial_tool/aml_channel.py b/utils/third_party/nni_new/tools/trial_tool/aml_channel.py new file mode 100644 index 0000000000000000000000000000000000000000..c8e1d7484a427b80b657d1c889c3a1cbe4aafecc --- /dev/null +++ b/utils/third_party/nni_new/tools/trial_tool/aml_channel.py @@ -0,0 +1,47 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from azureml.core.run import Run # pylint: disable=import-error +from .base_channel import BaseChannel +from .log_utils import LogType, nni_log + + +class AMLChannel(BaseChannel): + def __init__(self, args): + self.args = args + self.run = Run.get_context() + super(AMLChannel, self).__init__(args) + self.current_message_index = -1 + + def _inner_open(self): + pass + + def _inner_close(self): + pass + + def _inner_send(self, message): + try: + self.run.log('trial_runner', message.decode('utf8')) + except Exception as exception: + nni_log(LogType.Error, 'meet unhandled exception when send message: %s' % exception) + + def _inner_receive(self): + messages = [] + message_dict = self.run.get_metrics() + if 'nni_manager' not in message_dict: + return [] + message_list = message_dict['nni_manager'] + if not message_list: + return messages + if type(message_list) is list: + if self.current_message_index < len(message_list) - 1: + messages = message_list[self.current_message_index + 1 : len(message_list)] + self.current_message_index = len(message_list) - 1 + elif self.current_message_index == -1: + messages = [message_list] + self.current_message_index += 1 + newMessage = [] + for message in messages: + # receive message is string, to get consistent result, encode it here. + newMessage.append(message.encode('utf8')) + return newMessage diff --git a/utils/third_party/nni_new/tools/trial_tool/base_channel.py b/utils/third_party/nni_new/tools/trial_tool/base_channel.py new file mode 100644 index 0000000000000000000000000000000000000000..b9d3392abc0b15ce7eb90b07df4107902fb65d9d --- /dev/null +++ b/utils/third_party/nni_new/tools/trial_tool/base_channel.py @@ -0,0 +1,158 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import json +import threading +import time +from abc import ABC, abstractmethod +from queue import Empty, Queue + +from .log_utils import LogType, nni_log +from .commands import CommandType + +INTERVAL_SECONDS = 0.5 + + +class BaseChannel(ABC): + def __init__(self, args): + self.is_keep_parsed = args.node_count > 1 + self.args = args + self.node_id = self.args.node_id + + @abstractmethod + def _inner_send(self, message): + pass + + @abstractmethod + def _inner_receive(self): + return [] + + @abstractmethod + def _inner_open(self): + pass + + @abstractmethod + def _inner_close(self): + pass + + def open(self): + # initialize receive, send threads. + self.is_running = True + self.receive_queue = Queue() + self.receive_thread = threading.Thread(target=self._receive_loop) + self.receive_thread.start() + self.send_queue = Queue() + self.send_thread = threading.Thread(target=self._send_loop) + self.send_thread.start() + + self._inner_open() + + client_info = { + "isReady": True, + "runnerId": self.args.runner_id, + "expId": self.args.exp_id, + } + nni_log(LogType.Info, 'Channel: send ready information %s' % client_info) + self.send(CommandType.Initialized, client_info) + + def close(self): + self.is_running = False + try: + self._inner_close() + except Exception as err: + # ignore any error on closing + print("error on closing channel: %s" % err) + + def send(self, command, data): + """Send command to Training Service. + command: CommandType object. + data: string payload. + the message is sent synchronized. + """ + data["node"] = self.node_id + data = json.dumps(data) + data = data.encode('utf8') + message = b'%b%014d%b' % (command.value, len(data), data) + self.send_queue.put(message) + + def sent(self): + return self.send_queue.qsize() == 0 + + def received(self): + return self.receive_queue.qsize() > 0 + + def receive(self): + """Receive a command from Training Service. + Returns a tuple of command (CommandType) and payload (str) + """ + command = None + data = None + + try: + command_content = self.receive_queue.get(False) + if command_content is not None: + if (len(command_content) < 16): + # invalid header + nni_log(LogType.Error, 'incorrect command is found, command must be greater than 16 bytes!') + return None, None + header = command_content[:16] + command = CommandType(header[:2]) + length = int(header[2:]) + if (len(command_content)-16 != length): + nni_log(LogType.Error, 'incorrect command length, length {}, actual data length is {}, header {}.' + .format(length, len(command_content)-16, header)) + return None, None + data = command_content[16:16+length] + data = json.loads(data.decode('utf8')) + if self.node_id is None: + nni_log(LogType.Info, 'Received command, header: [%s], data: [%s]' % (header, data)) + else: + nni_log(LogType.Info, 'Received command(%s), header: [%s], data: [%s]' % (self.node_id, header, data)) + except Empty: + # do nothing, if no command received. + pass + except Exception as identifier: + nni_log(LogType.Error, 'meet unhandled exception in base_channel: %s' % identifier) + return command, data + + def _fetch_message(self, buffer, has_new_line=False): + messages = [] + while(len(buffer)) >= 16: + header = buffer[:16] + length = int(header[2:]) + + message_length = length+16 + total_length = message_length + if has_new_line: + total_length += 1 + + # break, if buffer is too short. + if len(buffer) < total_length: + break + data = buffer[16:message_length] + if has_new_line and 10 != buffer[total_length-1]: + nni_log(LogType.Error, 'end of message should be \\n, but got {}'.format(self.in_cache[total_length-1])) + buffer = buffer[total_length:] + messages.append(header + data) + + return messages, buffer + + def _receive_loop(self): + while (self.is_running): + messages = self._inner_receive() + if messages is not None: + for message in messages: + self.receive_queue.put(message) + time.sleep(INTERVAL_SECONDS) + + def _send_loop(self): + while (self.is_running): + message = None + try: + # no sleep, since it's a block call with INTERVAL_SECONDS second timeout + message = self.send_queue.get(True, INTERVAL_SECONDS) + except Empty: + # do nothing, if no command received. + pass + if message is not None: + self._inner_send(message) diff --git a/utils/third_party/nni_new/tools/trial_tool/commands.py b/utils/third_party/nni_new/tools/trial_tool/commands.py new file mode 100644 index 0000000000000000000000000000000000000000..86b10a2fe9a85d17bdfb4d9ec57b0b6cceb250da --- /dev/null +++ b/utils/third_party/nni_new/tools/trial_tool/commands.py @@ -0,0 +1,25 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from enum import Enum + + +class CommandType(Enum): + Initialize = b'IN' + RequestTrialJobs = b'GE' + ReportMetricData = b'ME' + ReportGpuInfo = b'GI' + UpdateSearchSpace = b'SS' + ImportData = b'FD' + AddCustomizedTrialJob = b'AD' + TrialEnd = b'EN' + Terminate = b'TE' + Ping = b'PI' + + Initialized = b'ID' + NewTrialJob = b'TR' + SendTrialJobParameter = b'SP' + NoMoreTrialJobs = b'NO' + KillTrialJob = b'KI' + StdOut = b'SO' + VersionCheck = b'VC' diff --git a/utils/third_party/nni_new/tools/trial_tool/constants.py b/utils/third_party/nni_new/tools/trial_tool/constants.py new file mode 100644 index 0000000000000000000000000000000000000000..bef401337039d46aa23fc1e5816b58978d07e104 --- /dev/null +++ b/utils/third_party/nni_new/tools/trial_tool/constants.py @@ -0,0 +1,24 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os + +API_ROOT_URL = '/api/v1/nni-pai' + +BASE_URL = 'http://{}' + +LOG_DIR = os.environ['NNI_OUTPUT_DIR'] + +NNI_PLATFORM = os.environ['NNI_PLATFORM'] + +STDOUT_FULL_PATH = os.path.join(LOG_DIR, 'stdout') + +STDERR_FULL_PATH = os.path.join(LOG_DIR, 'stderr') + +STDOUT_API = '/stdout' +VERSION_API = '/version' +PARAMETER_META_API = '/parameter-file-meta' +NNI_SYS_DIR = os.environ['NNI_SYS_DIR'] +NNI_TRIAL_JOB_ID = os.environ['NNI_TRIAL_JOB_ID'] +NNI_EXP_ID = os.environ['NNI_EXP_ID'] +MULTI_PHASE = os.environ['MULTI_PHASE'] diff --git a/utils/third_party/nni_new/tools/trial_tool/file_channel.py b/utils/third_party/nni_new/tools/trial_tool/file_channel.py new file mode 100644 index 0000000000000000000000000000000000000000..9a431d25f7ee3a4eeff973667d345ce2ddada1cb --- /dev/null +++ b/utils/third_party/nni_new/tools/trial_tool/file_channel.py @@ -0,0 +1,75 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os + +from .base_channel import BaseChannel + +command_path = "./commands" +runner_commands_file_name_prefix = "runner_commands" +manager_commands_file_name = "manager_commands.txt" + + +class FileChannel(BaseChannel): + + def __init__(self, args): + self.node_id = args.node_id + self.out_file = None + self.in_file = None + self.in_offset = 0 + self.in_cache = b"" + + super(FileChannel, self).__init__(args) + + def _inner_open(self): + pass + + def _inner_close(self): + if self.out_file is not None: + self.out_file.close() + self.out_file = None + if self.in_file is not None: + self.in_file.close() + self.in_file = None + + def _inner_send(self, message): + if self.out_file is None: + if not os.path.exists(command_path): + os.makedirs(command_path, exist_ok=True) + + if self.node_id is None: + file_name = os.path.join(command_path, "%s.txt" % runner_commands_file_name_prefix) + else: + file_name = os.path.join(command_path, "%s_%s.txt" % ( + runner_commands_file_name_prefix, self.node_id)) + self.out_file = open(file_name, "ab") + + self.out_file.write(message) + self.out_file.write(b'\n') + self.out_file.flush() + + def _open_manager_command(self): + full_name = os.path.join(command_path, manager_commands_file_name) + + if self.in_file is not None and self.in_file.closed: + self.in_file = None + + if self.in_file is None and os.path.exists(full_name): + self.in_file = open(full_name, "rb") + self.in_file.seek(self.in_offset) + + def _inner_receive(self): + messages = [] + + if self.in_file is None: + self._open_manager_command() + if self.in_file is not None: + self.in_file.seek(0, os.SEEK_END) + new_offset = self.in_file.tell() + self.in_file.seek(self.in_offset, os.SEEK_SET) + count = new_offset - self.in_offset + if count > 0: + self.in_cache += self.in_file.read(count) + self.in_offset = new_offset + messages, self.in_cache = self._fetch_message(self.in_cache, True) + return messages diff --git a/utils/third_party/nni_new/tools/trial_tool/gpu.py b/utils/third_party/nni_new/tools/trial_tool/gpu.py new file mode 100644 index 0000000000000000000000000000000000000000..48dab4b182a6ecd08b1108c297d206663ce9bb40 --- /dev/null +++ b/utils/third_party/nni_new/tools/trial_tool/gpu.py @@ -0,0 +1,69 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import subprocess +import time +import traceback +from xml.dom import minidom + + +def collect_gpu_usage(node_id): + cmd = 'nvidia-smi -q -x'.split() + info = None + try: + smi_output = subprocess.check_output(cmd) + info = parse_nvidia_smi_result(smi_output) + except Exception: + traceback.print_exc() + info = gen_empty_gpu_metric() + return info + + +def parse_nvidia_smi_result(smi): + try: + output = {} + xmldoc = minidom.parseString(smi) + gpuList = xmldoc.getElementsByTagName('gpu') + output["Timestamp"] = time.asctime(time.localtime()) + output["gpuCount"] = len(gpuList) + output["gpuInfos"] = [] + for gpuIndex, gpu in enumerate(gpuList): + gpuInfo = {} + gpuInfo['index'] = gpuIndex + gpuInfo['gpuUtil'] = gpu.getElementsByTagName('utilization')[0]\ + .getElementsByTagName('gpu_util')[0]\ + .childNodes[0].data.replace("%", "").strip() + gpuInfo['gpuMemUtil'] = gpu.getElementsByTagName('utilization')[0]\ + .getElementsByTagName('memory_util')[0]\ + .childNodes[0].data.replace("%", "").strip() + processes = gpu.getElementsByTagName('processes') + runningProNumber = len(processes[0].getElementsByTagName('process_info')) + gpuInfo['activeProcessNum'] = runningProNumber + + gpuInfo['gpuType'] = gpu.getElementsByTagName('product_name')[0]\ + .childNodes[0].data + memUsage = gpu.getElementsByTagName('fb_memory_usage')[0] + gpuInfo['gpuMemTotal'] = memUsage.getElementsByTagName('total')[0]\ + .childNodes[0].data.replace("MiB", "").strip() + gpuInfo['gpuMemUsed'] = memUsage.getElementsByTagName('used')[0]\ + .childNodes[0].data.replace("MiB", "").strip() + gpuInfo['gpuMemFree'] = memUsage.getElementsByTagName('free')[0]\ + .childNodes[0].data.replace("MiB", "").strip() + + output["gpuInfos"].append(gpuInfo) + except Exception: + traceback.print_exc() + output = {} + return output + + +def gen_empty_gpu_metric(): + try: + output = {} + output["Timestamp"] = time.asctime(time.localtime()) + output["gpuCount"] = 0 + output["gpuInfos"] = [] + except Exception: + traceback.print_exc() + output = {} + return output diff --git a/utils/third_party/nni_new/tools/trial_tool/hdfsClientUtility.py b/utils/third_party/nni_new/tools/trial_tool/hdfsClientUtility.py new file mode 100644 index 0000000000000000000000000000000000000000..05d4ea0d85675b5c763bf27f29db50b7900e1d03 --- /dev/null +++ b/utils/third_party/nni_new/tools/trial_tool/hdfsClientUtility.py @@ -0,0 +1,92 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +import posixpath +from .log_utils import LogType, nni_log + +def copyHdfsDirectoryToLocal(hdfsDirectory, localDirectory, hdfsClient): + '''Copy directory from HDFS to local''' + if not os.path.exists(localDirectory): + os.makedirs(localDirectory) + try: + listing = hdfsClient.list_status(hdfsDirectory) + except Exception as exception: + nni_log(LogType.Error, 'List hdfs directory {0} error: {1}'.format(hdfsDirectory, str(exception))) + raise exception + + for f in listing: + if f.type == 'DIRECTORY': + subHdfsDirectory = posixpath.join(hdfsDirectory, f.pathSuffix) + subLocalDirectory = os.path.join(localDirectory, f.pathSuffix) + copyHdfsDirectoryToLocal(subHdfsDirectory, subLocalDirectory, hdfsClient) + elif f.type == 'FILE': + hdfsFilePath = posixpath.join(hdfsDirectory, f.pathSuffix) + localFilePath = os.path.join(localDirectory, f.pathSuffix) + copyHdfsFileToLocal(hdfsFilePath, localFilePath, hdfsClient) + else: + raise AssertionError('unexpected type {}'.format(f.type)) + +def copyHdfsFileToLocal(hdfsFilePath, localFilePath, hdfsClient, override=True): + '''Copy file from HDFS to local''' + if not hdfsClient.exists(hdfsFilePath): + raise Exception('HDFS file {} does not exist!'.format(hdfsFilePath)) + try: + file_status = hdfsClient.get_file_status(hdfsFilePath) + if file_status.type != 'FILE': + raise Exception('HDFS file path {} is not a file'.format(hdfsFilePath)) + except Exception as exception: + nni_log(LogType.Error, 'Get hdfs file {0} status error: {1}'.format(hdfsFilePath, str(exception))) + raise exception + + if os.path.exists(localFilePath) and override: + os.remove(localFilePath) + try: + hdfsClient.copy_to_local(hdfsFilePath, localFilePath) + except Exception as exception: + nni_log(LogType.Error, 'Copy hdfs file {0} to {1} error: {2}'.format(hdfsFilePath, localFilePath, str(exception))) + raise exception + nni_log(LogType.Info, 'Successfully copied hdfs file {0} to {1}, {2} bytes'.format(hdfsFilePath, localFilePath, file_status.length)) + +def copyDirectoryToHdfs(localDirectory, hdfsDirectory, hdfsClient): + '''Copy directory from local to HDFS''' + if not os.path.exists(localDirectory): + raise Exception('Local Directory does not exist!') + hdfsClient.mkdirs(hdfsDirectory) + result = True + for file in os.listdir(localDirectory): + file_path = os.path.join(localDirectory, file) + if os.path.isdir(file_path): + hdfs_directory = os.path.join(hdfsDirectory, file) + try: + result = result and copyDirectoryToHdfs(file_path, hdfs_directory, hdfsClient) + except Exception as exception: + nni_log(LogType.Error, + 'Copy local directory {0} to hdfs directory {1} error: {2}'.format(file_path, hdfs_directory, str(exception))) + result = False + else: + hdfs_file_path = os.path.join(hdfsDirectory, file) + try: + result = result and copyFileToHdfs(file_path, hdfs_file_path, hdfsClient) + except Exception as exception: + nni_log(LogType.Error, 'Copy local file {0} to hdfs {1} error: {2}'.format(file_path, hdfs_file_path, str(exception))) + result = False + return result + +def copyFileToHdfs(localFilePath, hdfsFilePath, hdfsClient, override=True): + '''Copy a local file to HDFS directory''' + if not os.path.exists(localFilePath): + raise Exception('Local file Path does not exist!') + if os.path.isdir(localFilePath): + raise Exception('localFile should not a directory!') + if hdfsClient.exists(hdfsFilePath): + if override: + hdfsClient.delete(hdfsFilePath) + else: + return False + try: + hdfsClient.copy_from_local(localFilePath, hdfsFilePath) + return True + except Exception as exception: + nni_log(LogType.Error, 'Copy local file {0} to hdfs file {1} error: {2}'.format(localFilePath, hdfsFilePath, str(exception))) + return False diff --git a/utils/third_party/nni_new/tools/trial_tool/log_utils.py b/utils/third_party/nni_new/tools/trial_tool/log_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..8d5b3d94c0b60d60b978b94d967f092bfa9a1c56 --- /dev/null +++ b/utils/third_party/nni_new/tools/trial_tool/log_utils.py @@ -0,0 +1,219 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +import sys +import json +import logging +import logging.handlers +import time +import threading +import re + +from datetime import datetime +from enum import Enum, unique +from logging import StreamHandler + +from queue import Queue + +from .rest_utils import rest_post +from .url_utils import gen_send_stdout_url +from .commands import CommandType + + +@unique +class LogType(Enum): + Trace = 'TRACE' + Debug = 'DEBUG' + Info = 'INFO' + Warning = 'WARNING' + Error = 'ERROR' + Fatal = 'FATAL' + + +@unique +class StdOutputType(Enum): + Stdout = 'stdout', + Stderr = 'stderr' + + +def nni_log(log_type, log_message): + '''Log message into stdout''' + dt = datetime.now() + print('[{0}] {1} {2}'.format(dt, log_type.value, log_message), flush=True) + + +class NNIRestLogHanlder(StreamHandler): + def __init__(self, host, port, tag, trial_id, channel, std_output_type=StdOutputType.Stdout): + StreamHandler.__init__(self) + self.host = host + self.port = port + self.tag = tag + self.std_output_type = std_output_type + self.trial_id = trial_id + self.channel = channel + self.orig_stdout = sys.__stdout__ + self.orig_stderr = sys.__stderr__ + + def emit(self, record): + log_entry = {} + log_entry['tag'] = self.tag + log_entry['stdOutputType'] = self.std_output_type.name + log_entry['msg'] = self.format(record) + + try: + if self.channel is None: + rest_post(gen_send_stdout_url(self.host, self.port), json.dumps(log_entry), 10, True) + else: + if self.trial_id is not None: + log_entry["trial"] = self.trial_id + self.channel.send(CommandType.StdOut, log_entry) + except Exception as e: + self.orig_stderr.write(str(e) + '\n') + self.orig_stderr.flush() + + +class RemoteLogger(object): + """ + NNI remote logger + """ + + def __init__(self, syslog_host, syslog_port, tag, std_output_type, log_collection, trial_id=None, channel=None, log_level=logging.INFO): + ''' + constructor + ''' + logger_name = 'nni_syslog_{}'.format(tag) + # to prevent multiple trial logged in same logger + if trial_id is not None: + logger_name = '{}_{}'.format(logger_name, trial_id) + self.logger = logging.getLogger(logger_name) + self.log_level = log_level + self.logger.setLevel(self.log_level) + self.pipeReader = None + self.handler = NNIRestLogHanlder(syslog_host, syslog_port, tag, trial_id, channel) + self.logger.addHandler(self.handler) + if std_output_type == StdOutputType.Stdout: + self.orig_stdout = sys.__stdout__ + else: + self.orig_stdout = sys.__stderr__ + self.log_collection = log_collection + + def get_pipelog_reader(self): + ''' + Get pipe for remote logger + ''' + self.pipeReader = PipeLogReader(self.logger, self.log_collection, logging.INFO) + return self.pipeReader + + def flush(self): + ''' + Add flush in handler + ''' + for handler in self.logger.handlers: + handler.flush() + + def write(self, buf): + ''' + Write buffer data into logger/stdout + ''' + for line in buf.rstrip().splitlines(): + self.orig_stdout.write(line.rstrip() + '\n') + self.orig_stdout.flush() + try: + self.logger.log(self.log_level, line.rstrip()) + except Exception: + pass + + def close(self): + ''' + Close handlers and resources + ''' + if self.pipeReader is not None: + self.pipeReader.set_process_exit() + for handler in self.logger.handlers: + handler.close() + self.logger.removeHandler(handler) + + +class PipeLogReader(threading.Thread): + """ + The reader thread reads log data from pipe + """ + + def __init__(self, logger, log_collection, log_level=logging.INFO): + """Setup the object with a logger and a loglevel + and start the thread + """ + threading.Thread.__init__(self) + self.queue = Queue() + self.logger = logger + self.daemon = False + self.log_level = log_level + self.fdRead, self.fdWrite = os.pipe() + self.pipeReader = os.fdopen(self.fdRead) + self.orig_stdout = sys.__stdout__ + self._is_read_completed = False + self.process_exit = False + self.log_collection = log_collection + self.log_pattern = re.compile(r'NNISDK_MEb\'.*\'$') + + def _populateQueue(stream, queue): + ''' + Collect lines from 'stream' and put them in 'quque'. + ''' + time.sleep(1) + while True: + cur_process_exit = self.process_exit + try: + line = self.queue.get(True, 5) + try: + self.logger.log(self.log_level, line.rstrip()) + except Exception: + pass + except Exception: + if cur_process_exit == True: + self._is_read_completed = True + break + + self.pip_log_reader_thread = threading.Thread(target=_populateQueue, args=(self.pipeReader, self.queue)) + self.pip_log_reader_thread.daemon = True + self.start() + self.pip_log_reader_thread.start() + + def fileno(self): + """Return the write file descriptor of the pipe + """ + return self.fdWrite + + def run(self): + """Run the thread, logging everything. + If the log_collection is 'none', the log content will not be enqueued + """ + for line in iter(self.pipeReader.readline, ''): + self.orig_stdout.write(line.rstrip() + '\n') + self.orig_stdout.flush() + + if self.log_collection == 'none': + search_result = self.log_pattern.search(line) + if search_result: + metrics = search_result.group(0) + self.queue.put(metrics+'\n') + else: + self.queue.put(line) + + self.pipeReader.close() + + def close(self): + """Close the write end of the pipe. + """ + os.close(self.fdWrite) + + @property + def is_read_completed(self): + """Return if read is completed + """ + return self._is_read_completed + + def set_process_exit(self): + self.process_exit = True + return self.process_exit diff --git a/utils/third_party/nni_new/tools/trial_tool/rest_utils.py b/utils/third_party/nni_new/tools/trial_tool/rest_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..959209c7470bdb5dfe1ca32af91ba85757dd51e0 --- /dev/null +++ b/utils/third_party/nni_new/tools/trial_tool/rest_utils.py @@ -0,0 +1,44 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import requests + +def rest_get(url, timeout): + '''Call rest get method''' + try: + response = requests.get(url, timeout=timeout) + return response + except Exception as e: + print('Get exception {0} when sending http get to url {1}'.format(str(e), url)) + return None + +def rest_post(url, data, timeout, rethrow_exception=False): + '''Call rest post method''' + try: + response = requests.post(url, headers={'Accept': 'application/json', 'Content-Type': 'application/json'},\ + data=data, timeout=timeout) + return response + except Exception as e: + if rethrow_exception is True: + raise + print('Get exception {0} when sending http post to url {1}'.format(str(e), url)) + return None + +def rest_put(url, data, timeout): + '''Call rest put method''' + try: + response = requests.put(url, headers={'Accept': 'application/json', 'Content-Type': 'application/json'},\ + data=data, timeout=timeout) + return response + except Exception as e: + print('Get exception {0} when sending http put to url {1}'.format(str(e), url)) + return None + +def rest_delete(url, timeout): + '''Call rest delete method''' + try: + response = requests.delete(url, timeout=timeout) + return response + except Exception as e: + print('Get exception {0} when sending http delete to url {1}'.format(str(e), url)) + return None diff --git a/utils/third_party/nni_new/tools/trial_tool/trial.py b/utils/third_party/nni_new/tools/trial_tool/trial.py new file mode 100644 index 0000000000000000000000000000000000000000..1da398d017ecb98e36508a1683179534e68c1185 --- /dev/null +++ b/utils/third_party/nni_new/tools/trial_tool/trial.py @@ -0,0 +1,163 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import ctypes +import os +import sys +import shlex +import tarfile +import time +from datetime import datetime +from subprocess import Popen + +import psutil + +from .log_utils import LogType, RemoteLogger, StdOutputType, nni_log +from .commands import CommandType + +trial_output_path_name = ".nni" + + +class Trial: + def __init__(self, args, data): + self.process = None + self.data = data + self.args = args + self.command_channel = args.command_channel + self.trial_syslogger_stdout = None + + global NNI_TRIAL_JOB_ID + self.id = data["trialId"] + if self.id is None: + raise Exception("trial_id is not found in %s" % data) + os.environ['NNI_TRIAL_JOB_ID'] = self.id + NNI_TRIAL_JOB_ID = self.id + + # for multiple nodes. If it's None, it means single node. + self.node_id = args.node_id + if self.node_id is None: + self.name = self.id + else: + self.name = "%s_%s" % (self.id, self.node_id) + + def run(self): + # redirect trial's stdout and stderr to syslog + self.trial_syslogger_stdout = RemoteLogger(self.args.nnimanager_ip, self.args.nnimanager_port, 'trial', StdOutputType.Stdout, + self.args.log_collection, self.id, self.args.command_channel) + + nni_log(LogType.Info, "%s: start to run trial" % self.name) + + trial_working_dir = os.path.realpath(os.path.join(os.curdir, "..", "..", "trials", self.id)) + self.trial_output_dir = os.path.join(trial_working_dir, trial_output_path_name) + trial_code_dir = os.path.join(trial_working_dir, "code") + trial_nnioutput_dir = os.path.join(trial_working_dir, "nnioutput") + + environ = os.environ.copy() + environ['NNI_TRIAL_SEQ_ID'] = str(self.data["sequenceId"]) + environ['NNI_OUTPUT_DIR'] = os.path.join(trial_working_dir, "nnioutput") + environ['NNI_SYS_DIR'] = trial_working_dir + self.working_dir = trial_working_dir + + # prepare code and parameters + prepared_flag_file_name = os.path.join(trial_working_dir, "trial_prepared") + if not os.path.exists(trial_working_dir): + os.makedirs(trial_working_dir, exist_ok=True) + + os.makedirs(self.trial_output_dir, exist_ok=True) + os.makedirs(trial_nnioutput_dir, exist_ok=True) + # prepare code + os.makedirs(trial_code_dir, exist_ok=True) + with tarfile.open(os.path.join("..", "nni-code.tar.gz"), "r:gz") as tar: + tar.extractall(trial_code_dir) + + # save parameters + nni_log(LogType.Info, '%s: saving parameter %s' % (self.name, self.data["parameter"]["value"])) + parameter_file_name = os.path.join(trial_working_dir, "parameter.cfg") + with open(parameter_file_name, "w") as parameter_file: + parameter_file.write(self.data["parameter"]["value"]) + + # ready flag + with open(prepared_flag_file_name, "w") as prepared_flag_file: + prepared_flag_file.write("%s" % (int(datetime.now().timestamp() * 1000))) + + # make sure code prepared by other node. + if self.node_id is not None: + while True: + if os.path.exists(prepared_flag_file_name): + break + time.sleep(0.1) + + trial_command = self.args.trial_command + + gpuIndices = self.data.get("gpuIndices") + if (gpuIndices is not None): + if sys.platform == "win32": + trial_command = 'set CUDA_VISIBLE_DEVICES="%s " && call %s' % (gpuIndices, trial_command) + else: + trial_command = 'CUDA_VISIBLE_DEVICES="%s " %s' % (gpuIndices, trial_command) + + self.log_pipe_stdout = self.trial_syslogger_stdout.get_pipelog_reader() + self.process = Popen(trial_command, shell=True, stdout=self.log_pipe_stdout, + stderr=self.log_pipe_stdout, cwd=trial_code_dir, env=dict(environ)) + nni_log(LogType.Info, '{0}: spawns a subprocess (pid {1}) to run command: {2}'. + format(self.name, self.process.pid, shlex.split(trial_command))) + + def save_parameter_file(self, command_data): + parameters = command_data["parameters"] + file_index = int(parameters["index"]) + if file_index == 0: + parameter_file_name = "parameter.cfg" + else: + parameter_file_name = "parameter_{}.cfg".format(file_index) + parameter_file_name = os.path.join(self.working_dir, parameter_file_name) + with open(parameter_file_name, "w") as parameter_file: + nni_log(LogType.Info, '%s: saving parameter %s' % (self.name, parameters["value"])) + parameter_file.write(parameters["value"]) + + def is_running(self): + if (self.process is None): + return False + + retCode = self.process.poll() + # child worker process exits and all stdout data is read + if retCode is not None and self.log_pipe_stdout.set_process_exit() and self.log_pipe_stdout.is_read_completed == True: + # In Windows, the retCode -1 is 4294967295. It's larger than c_long, and raise OverflowError. + # So covert it to int32. + retCode = ctypes.c_long(retCode).value + nni_log(LogType.Info, '{0}: subprocess terminated. Exit code is {1}.'.format(self.name, retCode)) + + end_time = int(datetime.now().timestamp() * 1000) + end_message = { + "code": retCode, + "time": end_time, + "trial": self.id, + } + self.command_channel.send(CommandType.TrialEnd, end_message) + self.cleanup() + return False + else: + return True + + def kill(self, trial_id=None): + if trial_id == self.id or trial_id is None: + if self.process is not None: + try: + nni_log(LogType.Info, "%s: killing trial" % self.name) + for child in psutil.Process(self.process.pid).children(True): + child.kill() + self.process.kill() + except psutil.NoSuchProcess: + nni_log(LogType.Info, "kill trial %s failed: %s does not exist!" % (trial_id, self.process.pid)) + except Exception as ex: + nni_log(LogType.Error, "kill trial %s failed: %s " % (trial_id, str(ex))) + self.cleanup() + + def cleanup(self): + nni_log(LogType.Info, "%s: clean up trial" % self.name) + self.process = None + if self.log_pipe_stdout is not None: + self.log_pipe_stdout.set_process_exit() + self.log_pipe_stdout = None + if self.trial_syslogger_stdout is not None: + self.trial_syslogger_stdout.close() + self.trial_syslogger_stdout = None diff --git a/utils/third_party/nni_new/tools/trial_tool/trial_keeper.py b/utils/third_party/nni_new/tools/trial_tool/trial_keeper.py new file mode 100644 index 0000000000000000000000000000000000000000..65ea160a275dcbd168619294bd89411db30f41a6 --- /dev/null +++ b/utils/third_party/nni_new/tools/trial_tool/trial_keeper.py @@ -0,0 +1,264 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import argparse +import ctypes +import json +import logging +import os +import re +import shlex +import sys +import threading +import time +from subprocess import Popen + +import pkg_resources +from pyhdfs import HdfsClient + +from .constants import (LOG_DIR, MULTI_PHASE, NNI_EXP_ID, NNI_PLATFORM, + NNI_SYS_DIR, NNI_TRIAL_JOB_ID) +from .hdfsClientUtility import (copyDirectoryToHdfs, copyHdfsDirectoryToLocal, + copyHdfsFileToLocal) +from .log_utils import LogType, RemoteLogger, StdOutputType, nni_log +from .rest_utils import rest_get, rest_post +from .url_utils import gen_parameter_meta_url, gen_send_version_url + +logger = logging.getLogger('trial_keeper') +regular = re.compile('v?(?P[0-9](\.[0-9]){0,1}).*') + +_hdfs_client = None +_trial_process = None + + +def get_hdfs_client(args): + global _hdfs_client + + if _hdfs_client is not None: + return _hdfs_client + # backward compatibility + hdfs_host = None + + if args.hdfs_host: + hdfs_host = args.hdfs_host + elif args.pai_hdfs_host: + hdfs_host = args.pai_hdfs_host + else: + return None + + if hdfs_host is not None and args.nni_hdfs_exp_dir is not None: + try: + if args.webhdfs_path: + _hdfs_client = HdfsClient(hosts='{0}:80'.format(hdfs_host), user_name=args.pai_user_name, + webhdfs_path=args.webhdfs_path, timeout=5) + else: + # backward compatibility + _hdfs_client = HdfsClient(hosts='{0}:{1}'.format(hdfs_host, '50070'), user_name=args.pai_user_name, + timeout=5) + except Exception as e: + nni_log(LogType.Error, 'Create HDFS client error: ' + str(e)) + raise e + return _hdfs_client + + +def main_loop(args): + '''main loop logic for trial keeper''' + global _trial_process + + if not os.path.exists(LOG_DIR): + os.makedirs(LOG_DIR) + + trial_keeper_syslogger = RemoteLogger(args.nnimanager_ip, args.nnimanager_port, 'trial_keeper', + StdOutputType.Stdout, args.log_collection) + # redirect trial keeper's stdout and stderr to syslog + trial_syslogger_stdout = RemoteLogger(args.nnimanager_ip, args.nnimanager_port, 'trial', StdOutputType.Stdout, + args.log_collection) + sys.stdout = sys.stderr = trial_keeper_syslogger + hdfs_output_dir = None + + if args.hdfs_output_dir: + hdfs_output_dir = args.hdfs_output_dir + elif args.pai_hdfs_output_dir: + hdfs_output_dir = args.pai_hdfs_output_dir + + hdfs_client = get_hdfs_client(args) + + if hdfs_client is not None: + copyHdfsDirectoryToLocal(args.nni_hdfs_exp_dir, os.getcwd(), hdfs_client) + + if args.job_id_file: + with open(args.job_id_file, 'w') as job_file: + job_file.write("%d" % os.getpid()) + + # Notice: We don't appoint env, which means subprocess wil inherit current environment and that is expected behavior + log_pipe_stdout = trial_syslogger_stdout.get_pipelog_reader() + if sys.platform == 'win32': + _trial_process = Popen(args.trial_command, shell=True, stdout=log_pipe_stdout, stderr=log_pipe_stdout) + else: + _trial_process = Popen(args.trial_command, shell=True, stdout=log_pipe_stdout, stderr=log_pipe_stdout, preexec_fn=os.setsid) + nni_log(LogType.Info, 'Trial keeper spawns a subprocess (pid {0}) to run command: {1}'.format(_trial_process.pid, + shlex.split( + args.trial_command))) + + while True: + retCode = _trial_process.poll() + # child worker process exits and all stdout data is read + if retCode is not None and log_pipe_stdout.set_process_exit() and log_pipe_stdout.is_read_completed == True: + # In Windows, the retCode -1 is 4294967295. It's larger than c_long, and raise OverflowError. + # So covert it to int32. + retCode = ctypes.c_long(retCode).value + nni_log(LogType.Info, 'subprocess terminated. Exit code is {}. Quit'.format(retCode)) + if hdfs_output_dir is not None: + # Copy local directory to hdfs for OpenPAI + nni_local_output_dir = os.environ['NNI_OUTPUT_DIR'] + try: + if copyDirectoryToHdfs(nni_local_output_dir, hdfs_output_dir, hdfs_client): + nni_log(LogType.Info, + 'copy directory from {0} to {1} success!'.format(nni_local_output_dir, hdfs_output_dir)) + else: + nni_log(LogType.Info, + 'copy directory from {0} to {1} failed!'.format(nni_local_output_dir, hdfs_output_dir)) + except Exception as e: + nni_log(LogType.Error, 'HDFS copy directory got exception: ' + str(e)) + raise e + + # Exit as the retCode of subprocess(trial) + exit(retCode) + break + + time.sleep(2) + + +def trial_keeper_help_info(*args): + print('please run --help to see guidance') + + +def check_version(args): + try: + trial_keeper_version = pkg_resources.get_distribution('nni').version + except pkg_resources.ResolutionError as err: + # package nni does not exist, try nni-tool package + nni_log(LogType.Error, 'Package nni does not exist!') + os._exit(1) + if not args.nni_manager_version: + # skip version check + nni_log(LogType.Warning, 'Skipping version check!') + else: + try: + trial_keeper_version = regular.search(trial_keeper_version).group('version') + nni_log(LogType.Info, 'trial_keeper_version is {0}'.format(trial_keeper_version)) + nni_manager_version = regular.search(args.nni_manager_version).group('version') + nni_log(LogType.Info, 'nni_manager_version is {0}'.format(nni_manager_version)) + log_entry = {} + if trial_keeper_version != nni_manager_version: + nni_log(LogType.Warning, 'Version does not match!') + error_message = 'NNIManager version is {0}, TrialKeeper version is {1}, NNI version does not match!'.format( + nni_manager_version, trial_keeper_version) + log_entry['tag'] = 'VCFail' + log_entry['msg'] = error_message + rest_post(gen_send_version_url(args.nnimanager_ip, args.nnimanager_port), json.dumps(log_entry), 10, + False) + else: + nni_log(LogType.Info, 'Version match!') + log_entry['tag'] = 'VCSuccess' + rest_post(gen_send_version_url(args.nnimanager_ip, args.nnimanager_port), json.dumps(log_entry), 10, + False) + except AttributeError as err: + nni_log(LogType.Error, err) + + +def is_multi_phase(): + return MULTI_PHASE and (MULTI_PHASE in ['True', 'true']) + + +def download_parameter(meta_list, args): + """ + Download parameter file to local working directory. + meta_list format is defined in paiJobRestServer.ts + example meta_list: + [ + {"experimentId":"yWFJarYa","trialId":"UpPkl","filePath":"/chec/nni/experiments/yWFJarYa/trials/UpPkl/parameter_1.cfg"}, + {"experimentId":"yWFJarYa","trialId":"aIUMA","filePath":"/chec/nni/experiments/yWFJarYa/trials/aIUMA/parameter_1.cfg"} + ] + """ + nni_log(LogType.Debug, str(meta_list)) + nni_log(LogType.Debug, + 'NNI_SYS_DIR: {}, trial Id: {}, experiment ID: {}'.format(NNI_SYS_DIR, NNI_TRIAL_JOB_ID, NNI_EXP_ID)) + nni_log(LogType.Debug, 'NNI_SYS_DIR files: {}'.format(os.listdir(NNI_SYS_DIR))) + for meta in meta_list: + if meta['experimentId'] == NNI_EXP_ID and meta['trialId'] == NNI_TRIAL_JOB_ID: + param_fp = os.path.join(NNI_SYS_DIR, os.path.basename(meta['filePath'])) + if not os.path.exists(param_fp): + hdfs_client = get_hdfs_client(args) + copyHdfsFileToLocal(meta['filePath'], param_fp, hdfs_client, override=False) + + +def fetch_parameter_file(args): + class FetchThread(threading.Thread): + def __init__(self, args): + super(FetchThread, self).__init__() + self.args = args + + def run(self): + uri = gen_parameter_meta_url(self.args.nnimanager_ip, self.args.nnimanager_port) + nni_log(LogType.Info, uri) + + while True: + res = rest_get(uri, 10) + nni_log(LogType.Debug, 'status code: {}'.format(res.status_code)) + if res.status_code == 200: + meta_list = res.json() + download_parameter(meta_list, self.args) + else: + nni_log(LogType.Warning, 'rest response: {}'.format(str(res))) + time.sleep(5) + + fetch_file_thread = FetchThread(args) + fetch_file_thread.start() + + +def _set_adaptdl_signal_handler(): + import signal + global _trial_process + def _handler(signum, frame): + nni_log(LogType.Info, "RECEIVED SIGNAL {}".format(signum)) + nni_log(LogType.Debug, "TRIAL PROCESS ID {}".format(_trial_process.pid)) + if _trial_process and (signum == signal.SIGTERM or signum == signal.SIGINT): + os.killpg(os.getpgid(_trial_process.pid), signal.SIGINT) + os.waitpid(_trial_process.pid, 0) + exit(1) + signal.signal(signal.SIGTERM, _handler) + signal.signal(signal.SIGINT, _handler) + + +if __name__ == '__main__': + '''NNI Trial Keeper main function''' + PARSER = argparse.ArgumentParser() + PARSER.set_defaults(func=trial_keeper_help_info) + PARSER.add_argument('--trial_command', type=str, help='Command to launch trial process') + PARSER.add_argument('--nnimanager_ip', type=str, default='localhost', help='NNI manager rest server IP') + PARSER.add_argument('--nnimanager_port', type=str, default='8081', help='NNI manager rest server port') + PARSER.add_argument('--pai_hdfs_output_dir', type=str, help='the output dir of pai_hdfs') # backward compatibility + PARSER.add_argument('--hdfs_output_dir', type=str, help='the output dir of hdfs') + PARSER.add_argument('--pai_hdfs_host', type=str, help='the host of pai_hdfs') # backward compatibility + PARSER.add_argument('--hdfs_host', type=str, help='the host of hdfs') + PARSER.add_argument('--pai_user_name', type=str, help='the username of hdfs') + PARSER.add_argument('--nni_hdfs_exp_dir', type=str, help='nni experiment directory in hdfs') + PARSER.add_argument('--webhdfs_path', type=str, help='the webhdfs path used in webhdfs URL') + PARSER.add_argument('--nni_manager_version', type=str, help='the nni version transmitted from nniManager') + PARSER.add_argument('--log_collection', type=str, help='set the way to collect log in trialkeeper') + PARSER.add_argument('--job_id_file', type=str, help='set job id file for operating and monitoring job.') + args, unknown = PARSER.parse_known_args() + if args.trial_command is None: + exit(1) + check_version(args) + try: + if NNI_PLATFORM == 'adl': + _set_adaptdl_signal_handler() + main_loop(args) + except SystemExit as se: + nni_log(LogType.Info, 'NNI trial keeper exit with code {}'.format(se.code)) + os._exit(se.code) + except Exception as e: + nni_log(LogType.Error, 'Exit trial keeper with code 1 because Exception: {} is catched'.format(str(e))) + os._exit(1) diff --git a/utils/third_party/nni_new/tools/trial_tool/trial_runner.py b/utils/third_party/nni_new/tools/trial_tool/trial_runner.py new file mode 100644 index 0000000000000000000000000000000000000000..ebc6ee7dad6644e5c77bdd897cdb0d78aded11d3 --- /dev/null +++ b/utils/third_party/nni_new/tools/trial_tool/trial_runner.py @@ -0,0 +1,254 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import argparse +import json +import os +import random +import re +import sys +import time +import traceback +from datetime import datetime, timedelta + +import pkg_resources + +from .gpu import collect_gpu_usage + +idle_timeout_seconds = 10 * 60 +gpu_refressh_interval_seconds = 5 +regular = re.compile('v?(?P[0-9](\.[0-9]){0,1}).*') +trial_runner_syslogger = None + + +def main_loop(args): + '''main loop logic for trial runner''' + idle_last_time = datetime.now() + gpu_refresh_last_time = datetime.now() - timedelta(minutes=1) + try: + if args.job_pid_file: + with open(args.job_pid_file, 'w') as job_file: + job_file.write("%d" % os.getpid()) + + trials = dict() + + command_channel = args.command_channel + # command loop + while True: + command_type, command_data = command_channel.receive() + if command_type == CommandType.NewTrialJob: + trial_id = command_data["trialId"] + if trial_id in trials.keys(): + trial = trials[trial_id] + if trial.is_running(): + raise Exception('trial %s is running already, cannot start a new one' % trial.id) + else: + del trials[trial_id] + trial = Trial(args, command_data) + trial.run() + trials[trial_id] = trial + elif command_type == CommandType.KillTrialJob: + trial_id = command_data + if trial_id in trials.keys(): + trial = trials[trial_id] + trial.kill(command_data) + elif command_type == CommandType.SendTrialJobParameter: + trial_id = command_data["trialId"] + if trial_id in trials.keys(): + trial = trials[trial_id] + trial.save_parameter_file(command_data) + elif command_type is not None: + raise Exception("unknown command %s" % command_type) + + trial_list = list(trials.values()) + for trial in trial_list: + if trial is not None and trial.is_running(): + idle_last_time = datetime.now() + else: + del trials[trial.id] + + if (datetime.now() - idle_last_time).seconds > idle_timeout_seconds: + nni_log(LogType.Info, "trial runner is idle more than {0} seconds, so exit.".format( + idle_timeout_seconds)) + break + + if args.enable_gpu_collect and (datetime.now() - gpu_refresh_last_time).seconds > gpu_refressh_interval_seconds: + # collect gpu information + gpu_info = collect_gpu_usage(args.node_id) + command_channel.send(CommandType.ReportGpuInfo, gpu_info) + gpu_refresh_last_time = datetime.now() + time.sleep(0.5) + except Exception as ex: + traceback.print_exc() + raise ex + finally: + nni_log(LogType.Info, "main_loop exits.") + + trial_list = list(trials.values()) + for trial in trial_list: + trial.kill() + del trials[trial.id] + # wait to send commands + for _ in range(10): + if command_channel.sent(): + break + time.sleep(1) + command_channel.close() + + +def trial_runner_help_info(*args): + print('please run --help to see guidance') + + +def check_version(args): + try: + trial_runner_version = pkg_resources.get_distribution('nni').version + except pkg_resources.ResolutionError as err: + # package nni does not exist, try nni-tool package + nni_log(LogType.Error, 'Package nni does not exist!') + os._exit(1) + if not args.nni_manager_version: + # skip version check + nni_log(LogType.Warning, 'Skipping version check!') + else: + try: + command_channel = args.command_channel + trial_runner_version = regular.search(trial_runner_version).group('version') + nni_log(LogType.Info, '{0}: runner_version is {1}'.format(args.node_id, trial_runner_version)) + nni_manager_version = regular.search(args.nni_manager_version).group('version') + nni_log(LogType.Info, '{0}: nni_manager_version is {1}'.format(args.node_id, nni_manager_version)) + log_entry = {} + if trial_runner_version != nni_manager_version: + nni_log(LogType.Warning, '{0}: Version does not match!'.format(args.node_id)) + error_message = '{0}: NNIManager version is {1}, Trial runner version is {2}, NNI version does not match!'.format( + args.node_id, nni_manager_version, trial_runner_version) + log_entry['tag'] = 'VCFail' + log_entry['msg'] = error_message + command_channel.send(CommandType.VersionCheck, log_entry) + while not command_channel.sent(): + time.sleep(1) + else: + nni_log(LogType.Info, '{0}: Version match!'.format(args.node_id)) + log_entry['tag'] = 'VCSuccess' + command_channel.send(CommandType.VersionCheck, log_entry) + except AttributeError as err: + nni_log(LogType.Error, '{0}: {1}'.format(args.node_id, err)) + +if __name__ == '__main__': + + '''NNI Trial Runner main function''' + PARSER = argparse.ArgumentParser() + PARSER.set_defaults(func=trial_runner_help_info) + PARSER.add_argument('--trial_command', type=str, help='Command to launch trial process') + PARSER.add_argument('--nnimanager_ip', type=str, help='NNI manager rest server IP') + PARSER.add_argument('--nnimanager_port', type=str, help='NNI manager rest server port') + PARSER.add_argument('--nni_manager_version', type=str, help='the nni version transmitted from nniManager') + PARSER.add_argument('--log_collection', type=str, help='set the way to collect log in trial runner') + PARSER.add_argument('--node_count', type=int, help='number of nodes, it determines how to consume command and save code file') + PARSER.add_argument('--job_pid_file', type=str, help='save trial runner process pid') + args, unknown = PARSER.parse_known_args() + + setting_file = "settings.json" + if not os.path.exists(setting_file): + setting_file = "../{}".format(setting_file) + if os.path.exists(setting_file): + with open(setting_file, 'r') as fp: + settings = json.load(fp) + print("setting is {}".format(settings)) + else: + print("not found setting file") + + args.exp_id = settings["experimentId"] + args.platform = settings["platform"] + # runner_id is unique runner in experiment + args.runner_id = os.path.basename(os.path.realpath(os.path.curdir)) + args.runner_name = "runner_"+args.runner_id + args.enable_gpu_collect = settings["enableGpuCollector"] + args.command_channel = settings["commandChannel"] + + if args.trial_command is None: + args.trial_command = settings["command"] + if args.nnimanager_ip is None: + args.nnimanager_ip = settings["nniManagerIP"] + if args.nnimanager_port is None: + args.nnimanager_port = settings["nniManagerPort"] + if args.nni_manager_version is None: + args.nni_manager_version = settings["nniManagerVersion"] + if args.log_collection is None: + args.log_collection = settings["logCollection"] + if args.node_count is None: + # default has only one node. + args.node_count = 1 + + os.environ['NNI_OUTPUT_DIR'] = os.curdir + "/nnioutput" + os.environ['NNI_PLATFORM'] = args.platform + os.environ['NNI_SYS_DIR'] = os.curdir + os.environ['NNI_EXP_ID'] = args.exp_id + os.environ['MULTI_PHASE'] = "true" + os.environ['NNI_TRIAL_JOB_ID'] = "runner" + os.environ['REUSE_MODE'] = "true" + + from .log_utils import LogType, RemoteLogger, StdOutputType, nni_log + from .trial import Trial + from .file_channel import FileChannel + from .web_channel import WebChannel + from .commands import CommandType + + is_multi_node = args.node_count > 1 + + if (is_multi_node): + # for multiple nodes, create a file to get a unique id. + while True: + node_id = random.randint(0, 10000) + unique_check_file_name = "node_%s" % (node_id) + if not os.path.exists(unique_check_file_name): + break + with open(unique_check_file_name, "w") as unique_check_file: + unique_check_file.write("%s" % (int(datetime.now().timestamp() * 1000))) + args.node_id = node_id + else: + # node id is unique in the runner + args.node_id = None + + # init command channel + command_channel = None + if args.command_channel == "file": + command_channel = FileChannel(args) + elif args.command_channel == 'aml': + from .aml_channel import AMLChannel + command_channel = AMLChannel(args) + else: + command_channel = WebChannel(args) + command_channel.open() + + nni_log(LogType.Info, "command channel is {}, actual type is {}".format(args.command_channel, type(command_channel))) + args.command_channel = command_channel + + trial_runner_syslogger = RemoteLogger(args.nnimanager_ip, args.nnimanager_port, 'runner', + StdOutputType.Stdout, args.log_collection, args.runner_name, command_channel) + sys.stdout = sys.stderr = trial_runner_syslogger + nni_log(LogType.Info, "{}: merged args is {}".format(args.node_id, args)) + + if args.trial_command is None: + nni_log(LogType.Error, "{}: no command is found.".format(args.node_id)) + os._exit(1) + check_version(args) + try: + main_loop(args) + except SystemExit as se: + nni_log(LogType.Info, '{}: NNI trial runner exit with code {}'.format(args.node_id, se.code)) + + # try best to send latest errors to server + timeout = 10 + while not command_channel.sent() and timeout > 0: + timeout -= 1 + time.sleep(1) + os._exit(se.code) + finally: + if trial_runner_syslogger is not None: + if trial_runner_syslogger.pipeReader is not None: + trial_runner_syslogger.pipeReader.set_process_exit() + trial_runner_syslogger.close() + + # the process doesn't exit even main loop exit. So exit it explictly. + os._exit(0) diff --git a/utils/third_party/nni_new/tools/trial_tool/url_utils.py b/utils/third_party/nni_new/tools/trial_tool/url_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7942c62fb5b4d8d3734ddf8c23a21b0f379d76ec --- /dev/null +++ b/utils/third_party/nni_new/tools/trial_tool/url_utils.py @@ -0,0 +1,19 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .constants import API_ROOT_URL, BASE_URL, STDOUT_API, NNI_TRIAL_JOB_ID, NNI_EXP_ID, VERSION_API, PARAMETER_META_API + + +def gen_send_stdout_url(ip, port): + '''Generate send stdout url''' + return '{0}:{1}{2}{3}/{4}/{5}'.format(BASE_URL.format(ip), port, API_ROOT_URL, STDOUT_API, NNI_EXP_ID, NNI_TRIAL_JOB_ID) + + +def gen_send_version_url(ip, port): + '''Generate send error url''' + return '{0}:{1}{2}{3}/{4}/{5}'.format(BASE_URL.format(ip), port, API_ROOT_URL, VERSION_API, NNI_EXP_ID, NNI_TRIAL_JOB_ID) + + +def gen_parameter_meta_url(ip, port): + '''Generate send error url''' + return '{0}:{1}{2}{3}'.format(BASE_URL.format(ip), port, API_ROOT_URL, PARAMETER_META_API) diff --git a/utils/third_party/nni_new/tools/trial_tool/web_channel.py b/utils/third_party/nni_new/tools/trial_tool/web_channel.py new file mode 100644 index 0000000000000000000000000000000000000000..87901e1163a5fbcf82a2bb05c701735a346333fc --- /dev/null +++ b/utils/third_party/nni_new/tools/trial_tool/web_channel.py @@ -0,0 +1,57 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import asyncio +import os +import websockets + +from .base_channel import BaseChannel +from .log_utils import LogType, nni_log + + +class WebChannel(BaseChannel): + + def __init__(self, args): + self.node_id = args.node_id + self.args = args + self.client = None + self.in_cache = b"" + self.timeout = 10 + + super(WebChannel, self).__init__(args) + + self._event_loop = None + + def _inner_open(self): + url = "ws://{}:{}".format(self.args.nnimanager_ip, self.args.nnimanager_port) + try: + connect = asyncio.wait_for(websockets.connect(url), self.timeout) + self._event_loop = asyncio.get_event_loop() + client = self._event_loop.run_until_complete(connect) + self.client = client + nni_log(LogType.Info, 'WebChannel: connected with info %s' % url) + except asyncio.TimeoutError: + nni_log(LogType.Error, 'connect to %s timeout! Please make sure NNIManagerIP configured correctly, and accessable.' % url) + os._exit(1) + + def _inner_close(self): + if self.client is not None: + self.client.close() + self.client = None + if self._event_loop.is_running(): + self._event_loop.stop() + self._event_loop = None + + def _inner_send(self, message): + loop = asyncio.new_event_loop() + loop.run_until_complete(self.client.send(message)) + + def _inner_receive(self): + messages = [] + if self.client is not None: + received = self._event_loop.run_until_complete(self.client.recv()) + # receive message is string, to get consistent result, encode it here. + self.in_cache += received.encode("utf8") + messages, self.in_cache = self._fetch_message(self.in_cache) + + return messages diff --git a/utils/third_party/nni_new/trial.py b/utils/third_party/nni_new/trial.py new file mode 100644 index 0000000000000000000000000000000000000000..e85d292b8c1409d4621e347aea2086f86e76f303 --- /dev/null +++ b/utils/third_party/nni_new/trial.py @@ -0,0 +1,156 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .utils import to_json +from .runtime.env_vars import trial_env_vars +from .runtime import platform + + +__all__ = [ + 'get_next_parameter', + 'get_current_parameter', + 'report_intermediate_result', + 'report_final_result', + 'get_experiment_id', + 'get_trial_id', + 'get_sequence_id' +] + + +_params = None +_experiment_id = platform.get_experiment_id() +_trial_id = platform.get_trial_id() +_sequence_id = platform.get_sequence_id() + + +def get_next_parameter(): + """ + Get the hyper paremeters generated by tuner. For a multiphase experiment, it returns a new group of hyper + parameters at each call of get_next_parameter. For a non-multiphase (multiPhase is not configured or set to False) + experiment, it returns hyper parameters only on the first call for each trial job, it returns None since second call. + This API should be called only once in each trial job of an experiment which is not specified as multiphase. + + Returns + ------- + dict + A dict object contains the hyper parameters generated by tuner, the keys of the dict are defined in + search space. Returns None if no more hyper parameters can be generated by tuner. + """ + global _params + _params = platform.get_next_parameter() + if _params is None: + return None + return _params['parameters'] + +def get_current_parameter(tag=None): + """ + Get current hyper parameters generated by tuner. It returns the same group of hyper parameters as the last + call of get_next_parameter returns. + + Parameters + ---------- + tag: str + hyper parameter key + """ + global _params + if _params is None: + return None + if tag is None: + return _params['parameters'] + return _params['parameters'][tag] + +def get_experiment_id(): + """ + Get experiment ID. + + Returns + ------- + str + Identifier of current experiment + """ + return _experiment_id + +def get_trial_id(): + """ + Get trial job ID which is string identifier of a trial job, for example 'MoXrp'. In one experiment, each trial + job has an unique string ID. + + Returns + ------- + str + Identifier of current trial job which is calling this API. + """ + return _trial_id + +def get_sequence_id(): + """ + Get trial job sequence nubmer. A sequence number is an integer value assigned to each trial job base on the + order they are submitted, incremental starting from 0. In one experiment, both trial job ID and sequence number + are unique for each trial job, they are of different data types. + + Returns + ------- + int + Sequence number of current trial job which is calling this API. + """ + return _sequence_id + +_intermediate_seq = 0 + + +def overwrite_intermediate_seq(value): + """ + Overwrite intermediate sequence value. + + Parameters + ---------- + value: + int + """ + assert isinstance(value, int) + global _intermediate_seq + _intermediate_seq = value + + +def report_intermediate_result(metric): + """ + Reports intermediate result to NNI. + + Parameters + ---------- + metric: + serializable object. + """ + global _intermediate_seq + assert _params or trial_env_vars.NNI_PLATFORM is None, \ + 'nni.get_next_parameter() needs to be called before report_intermediate_result' + metric = to_json({ + 'parameter_id': _params['parameter_id'] if _params else None, + 'trial_job_id': trial_env_vars.NNI_TRIAL_JOB_ID, + 'type': 'PERIODICAL', + 'sequence': _intermediate_seq, + 'value': to_json(metric) + }) + _intermediate_seq += 1 + platform.send_metric(metric) + +def report_final_result(metric): + """ + Reports final result to NNI. + + Parameters + ---------- + metric: serializable object + Usually (for built-in tuners to work), it should be a number, or + a dict with key "default" (a number), and any other extra keys. + """ + assert _params or trial_env_vars.NNI_PLATFORM is None, \ + 'nni.get_next_parameter() needs to be called before report_final_result' + metric = to_json({ + 'parameter_id': _params['parameter_id'] if _params else None, + 'trial_job_id': trial_env_vars.NNI_TRIAL_JOB_ID, + 'type': 'FINAL', + 'sequence': 0, + 'value': to_json(metric) + }) + platform.send_metric(metric) diff --git a/utils/third_party/nni_new/tuner.py b/utils/third_party/nni_new/tuner.py new file mode 100644 index 0000000000000000000000000000000000000000..4fbcc011d0c55676e9bc9c7e906d7655bb5badeb --- /dev/null +++ b/utils/third_party/nni_new/tuner.py @@ -0,0 +1,223 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +Tuner is an AutoML algorithm, which generates a new configuration for the next try. +A new trial will run with this configuration. + +See :class:`Tuner`' specification and ``docs/en_US/tuners.rst`` for details. +""" + +import logging + +import nni + +from .recoverable import Recoverable + +__all__ = ['Tuner'] + +_logger = logging.getLogger(__name__) + + +class Tuner(Recoverable): + """ + Tuner is an AutoML algorithm, which generates a new configuration for the next try. + A new trial will run with this configuration. + + This is the abstract base class for all tuners. + Tuning algorithms should inherit this class and override :meth:`update_search_space`, :meth:`receive_trial_result`, + as well as :meth:`generate_parameters` or :meth:`generate_multiple_parameters`. + + After initializing, NNI will first call :meth:`update_search_space` to tell tuner the feasible region, + and then call :meth:`generate_parameters` one or more times to request for hyper-parameter configurations. + + The framework will train several models with given configuration. + When one of them is finished, the final accuracy will be reported to :meth:`receive_trial_result`. + And then another configuration will be reqeusted and trained, util the whole experiment finish. + + If a tuner want's to know when a trial ends, it can also override :meth:`trial_end`. + + Tuners use *parameter ID* to track trials. + In tuner context, there is a one-to-one mapping between parameter ID and trial. + When the framework ask tuner to generate hyper-parameters for a new trial, + an ID has already been assigned and can be recorded in :meth:`generate_parameters`. + Later when the trial ends, the ID will be reported to :meth:`trial_end`, + and :meth:`receive_trial_result` if it has a final result. + Parameter IDs are unique integers. + + The type/format of search space and hyper-parameters are not limited, + as long as they are JSON-serializable and in sync with trial code. + For HPO tuners, however, there is a widely shared common interface, + which supports ``choice``, ``randint``, ``uniform``, and so on. + See ``docs/en_US/Tutorial/SearchSpaceSpec.md`` for details of this interface. + + [WIP] For advanced tuners which take advantage of trials' intermediate results, + an ``Advisor`` interface is under development. + + See Also + -------- + Builtin tuners: + :class:`~nni.algorithms.hpo.hyperopt_tuner.hyperopt_tuner.HyperoptTuner` + :class:`~nni.algorithms.hpo.evolution_tuner.evolution_tuner.EvolutionTuner` + :class:`~nni.algorithms.hpo.smac_tuner.SMACTuner` + :class:`~nni.algorithms.hpo.gridsearch_tuner.GridSearchTuner` + :class:`~nni.algorithms.hpo.networkmorphism_tuner.networkmorphism_tuner.NetworkMorphismTuner` + :class:`~nni.algorithms.hpo.metis_tuner.mets_tuner.MetisTuner` + :class:`~nni.algorithms.hpo.ppo_tuner.PPOTuner` + :class:`~nni.algorithms.hpo.gp_tuner.gp_tuner.GPTuner` + """ + + def generate_parameters(self, parameter_id, **kwargs): + """ + Abstract method which provides a set of hyper-parameters. + + This method will get called when the framework is about to launch a new trial, + if user does not override :meth:`generate_multiple_parameters`. + + The return value of this method will be received by trials via :func:`nni.get_next_parameter`. + It should fit in the search space, though the framework will not verify this. + + User code must override either this method or :meth:`generate_multiple_parameters`. + + Parameters + ---------- + parameter_id : int + Unique identifier for requested hyper-parameters. This will later be used in :meth:`receive_trial_result`. + **kwargs + Unstable parameters which should be ignored by normal users. + + Returns + ------- + any + The hyper-parameters, a dict in most cases, but could be any JSON-serializable type when needed. + + Raises + ------ + nni.NoMoreTrialError + If the search space is fully explored, tuner can raise this exception. + """ + # FIXME: some tuners raise NoMoreTrialError when they are waiting for more trial results + # we need to design a new exception for this purpose + raise NotImplementedError('Tuner: generate_parameters not implemented') + + def generate_multiple_parameters(self, parameter_id_list, **kwargs): + """ + Callback method which provides multiple sets of hyper-parameters. + + This method will get called when the framework is about to launch one or more new trials. + + If user does not override this method, it will invoke :meth:`generate_parameters` on each parameter ID. + + See :meth:`generate_parameters` for details. + + User code must override either this method or :meth:`generate_parameters`. + + Parameters + ---------- + parameter_id_list : list of int + Unique identifiers for each set of requested hyper-parameters. + These will later be used in :meth:`receive_trial_result`. + **kwargs + Unstable parameters which should be ignored by normal users. + + Returns + ------- + list + List of hyper-parameters. An empty list indicates there are no more trials. + """ + result = [] + for parameter_id in parameter_id_list: + try: + _logger.debug("generating param for %s", parameter_id) + res = self.generate_parameters(parameter_id, **kwargs) + except nni.NoMoreTrialError: + return result + result.append(res) + return result + + def receive_trial_result(self, parameter_id, parameters, value, **kwargs): + """ + Abstract method invoked when a trial reports its final result. Must override. + + This method only listens to results of algorithm-generated hyper-parameters. + Currently customized trials added from web UI will not report result to this method. + + Parameters + ---------- + parameter_id : int + Unique identifier of used hyper-parameters, same with :meth:`generate_parameters`. + parameters + Hyper-parameters generated by :meth:`generate_parameters`. + value + Result from trial (the return value of :func:`nni.report_final_result`). + **kwargs + Unstable parameters which should be ignored by normal users. + """ + raise NotImplementedError('Tuner: receive_trial_result not implemented') + + def _accept_customized_trials(self, accept=True): + # FIXME: because Tuner is designed as interface, this API should not be here + + # Enable or disable receiving results of user-added hyper-parameters. + # By default `receive_trial_result()` will only receive results of algorithm-generated hyper-parameters. + # If tuners want to receive those of customized parameters as well, they can call this function in `__init__()`. + + # pylint: disable=attribute-defined-outside-init + self._accept_customized = accept + + def trial_end(self, parameter_id, success, **kwargs): + """ + Abstract method invoked when a trial is completed or terminated. Do nothing by default. + + Parameters + ---------- + parameter_id : int + Unique identifier for hyper-parameters used by this trial. + success : bool + True if the trial successfully completed; False if failed or terminated. + **kwargs + Unstable parameters which should be ignored by normal users. + """ + + def update_search_space(self, search_space): + """ + Abstract method for updating the search space. Must override. + + Tuners are advised to support updating search space at run-time. + If a tuner can only set search space once before generating first hyper-parameters, + it should explicitly document this behaviour. + + Parameters + ---------- + search_space + JSON object defined by experiment owner. + """ + raise NotImplementedError('Tuner: update_search_space not implemented') + + def load_checkpoint(self): + """ + Internal API under revising, not recommended for end users. + """ + checkpoin_path = self.get_checkpoint_path() + _logger.info('Load checkpoint ignored by tuner, checkpoint path: %s', checkpoin_path) + + def save_checkpoint(self): + """ + Internal API under revising, not recommended for end users. + """ + checkpoin_path = self.get_checkpoint_path() + _logger.info('Save checkpoint ignored by tuner, checkpoint path: %s', checkpoin_path) + + def import_data(self, data): + """ + Internal API under revising, not recommended for end users. + """ + # Import additional data for tuning + # data: a list of dictionarys, each of which has at least two keys, 'parameter' and 'value' + pass + + def _on_exit(self): + pass + + def _on_error(self): + pass diff --git a/utils/third_party/nni_new/utils.py b/utils/third_party/nni_new/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..026e0fbc6419ddd744d321010ed4aa0284da34ce --- /dev/null +++ b/utils/third_party/nni_new/utils.py @@ -0,0 +1,307 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import copy +import functools +from enum import Enum, unique +import json_tricks +from schema import And + +from . import parameter_expressions + + +to_json = functools.partial(json_tricks.dumps, allow_nan=True) + +@unique +class OptimizeMode(Enum): + """Optimize Mode class + + if OptimizeMode is 'minimize', it means the tuner need to minimize the reward + that received from Trial. + + if OptimizeMode is 'maximize', it means the tuner need to maximize the reward + that received from Trial. + """ + Minimize = 'minimize' + Maximize = 'maximize' + + +class NodeType: + """Node Type class + """ + ROOT = 'root' + TYPE = '_type' + VALUE = '_value' + INDEX = '_index' + NAME = '_name' + + +class MetricType: + """The types of metric data + """ + FINAL = 'FINAL' + PERIODICAL = 'PERIODICAL' + REQUEST_PARAMETER = 'REQUEST_PARAMETER' + + +def split_index(params): + """ + Delete index infromation from params + """ + if isinstance(params, dict): + if NodeType.INDEX in params.keys(): + return split_index(params[NodeType.VALUE]) + result = {} + for key in params: + result[key] = split_index(params[key]) + return result + else: + return params + + +def extract_scalar_reward(value, scalar_key='default'): + """ + Extract scalar reward from trial result. + + Parameters + ---------- + value : int, float, dict + the reported final metric data + scalar_key : str + the key name that indicates the numeric number + + Raises + ------ + RuntimeError + Incorrect final result: the final result should be float/int, + or a dict which has a key named "default" whose value is float/int. + """ + if isinstance(value, (float, int)): + reward = value + elif isinstance(value, dict) and scalar_key in value and isinstance(value[scalar_key], (float, int)): + reward = value[scalar_key] + else: + raise RuntimeError('Incorrect final result: the final result should be float/int, ' \ + 'or a dict which has a key named "default" whose value is float/int.') + return reward + + +def extract_scalar_history(trial_history, scalar_key='default'): + """ + Extract scalar value from a list of intermediate results. + + Parameters + ---------- + trial_history : list + accumulated intermediate results of a trial + scalar_key : str + the key name that indicates the numeric number + + Raises + ------ + RuntimeError + Incorrect final result: the final result should be float/int, + or a dict which has a key named "default" whose value is float/int. + """ + return [extract_scalar_reward(ele, scalar_key) for ele in trial_history] + + +def convert_dict2tuple(value): + """ + convert dict type to tuple to solve unhashable problem. + NOTE: this function will change original data. + """ + if isinstance(value, dict): + for _keys in value: + value[_keys] = convert_dict2tuple(value[_keys]) + return tuple(sorted(value.items())) + return value + + +def json2space(x, oldy=None, name=NodeType.ROOT): + """ + Change search space from json format to hyperopt format + + """ + y = list() + if isinstance(x, dict): + if NodeType.TYPE in x.keys(): + _type = x[NodeType.TYPE] + name = name + '-' + _type + if _type == 'choice': + if oldy is not None: + _index = oldy[NodeType.INDEX] + y += json2space(x[NodeType.VALUE][_index], + oldy[NodeType.VALUE], name=name+'[%d]' % _index) + else: + y += json2space(x[NodeType.VALUE], None, name=name) + y.append(name) + else: + for key in x.keys(): + y += json2space(x[key], oldy[key] if oldy else None, name+"[%s]" % str(key)) + elif isinstance(x, list): + for i, x_i in enumerate(x): + if isinstance(x_i, dict): + if NodeType.NAME not in x_i.keys(): + raise RuntimeError('\'_name\' key is not found in this nested search space.') + y += json2space(x_i, oldy[i] if oldy else None, name + "[%d]" % i) + return y + + +def json2parameter(x, is_rand, random_state, oldy=None, Rand=False, name=NodeType.ROOT): + """ + Json to pramaters. + + """ + if isinstance(x, dict): + if NodeType.TYPE in x.keys(): + _type = x[NodeType.TYPE] + _value = x[NodeType.VALUE] + name = name + '-' + _type + Rand |= is_rand[name] + if Rand is True: + if _type == 'choice': + _index = random_state.randint(len(_value)) + y = { + NodeType.INDEX: _index, + NodeType.VALUE: json2parameter( + x[NodeType.VALUE][_index], + is_rand, + random_state, + None, + Rand, + name=name+"[%d]" % _index + ) + } + else: + y = getattr(parameter_expressions, _type)(*(_value + [random_state])) + else: + y = copy.deepcopy(oldy) + else: + y = dict() + for key in x.keys(): + y[key] = json2parameter( + x[key], + is_rand, + random_state, + oldy[key] if oldy else None, + Rand, + name + "[%s]" % str(key) + ) + elif isinstance(x, list): + y = list() + for i, x_i in enumerate(x): + if isinstance(x_i, dict): + if NodeType.NAME not in x_i.keys(): + raise RuntimeError('\'_name\' key is not found in this nested search space.') + y.append(json2parameter( + x_i, + is_rand, + random_state, + oldy[i] if oldy else None, + Rand, + name + "[%d]" % i + )) + else: + y = copy.deepcopy(x) + return y + +def merge_parameter(base_params, override_params): + """ + Update the parameters in ``base_params`` with ``override_params``. + Can be useful to override parsed command line arguments. + + Parameters + ---------- + base_params : namespace or dict + Base parameters. A key-value mapping. + override_params : dict or None + Parameters to override. Usually the parameters got from ``get_next_parameters()``. + When it is none, nothing will happen. + + Returns + ------- + namespace or dict + The updated ``base_params``. Note that ``base_params`` will be updated inplace. The return value is + only for convenience. + """ + if override_params is None: + return base_params + is_dict = isinstance(base_params, dict) + for k, v in override_params.items(): + if is_dict: + if k not in base_params: + raise ValueError('Key \'%s\' not found in base parameters.' % k) + if type(base_params[k]) != type(v) and base_params[k] is not None: + raise TypeError('Expected \'%s\' in override parameters to have type \'%s\', but found \'%s\'.' % + (k, type(base_params[k]), type(v))) + base_params[k] = v + else: + if not hasattr(base_params, k): + raise ValueError('Key \'%s\' not found in base parameters.' % k) + if type(getattr(base_params, k)) != type(v) and getattr(base_params, k) is not None: + raise TypeError('Expected \'%s\' in override parameters to have type \'%s\', but found \'%s\'.' % + (k, type(getattr(base_params, k)), type(v))) + setattr(base_params, k, v) + return base_params + +class ClassArgsValidator(object): + """ + NNI tuners/assessors/adivisors accept a `classArgs` parameter in experiment configuration file. + This ClassArgsValidator interface is used to validate the classArgs section in exeperiment + configuration file. + """ + def validate_class_args(self, **kwargs): + """ + Validate the classArgs configuration in experiment configuration file. + + Parameters + ---------- + kwargs: dict + kwargs passed to tuner/assessor/advisor constructor + + Raises: + Raise an execption if the kwargs is invalid. + """ + pass + + def choices(self, key, *args): + """ + Utility method to create a scheme to check whether the `key` is one of the `args`. + + Parameters: + ---------- + key: str + key name of the data to be validated + args: list of str + list of the choices + + Returns: Schema + -------- + A scheme to check whether the `key` is one of the `args`. + """ + return And(lambda n: n in args, error='%s should be in [%s]!' % (key, str(args))) + + def range(self, key, keyType, start, end): + """ + Utility method to create a schema to check whether the `key` is in the range of [start, end]. + + Parameters: + ---------- + key: str + key name of the data to be validated + keyType: type + python data type, such as int, float + start: type is specified by keyType + start of the range + end: type is specified by keyType + end of the range + + Returns: Schema + -------- + A scheme to check whether the `key` is in the range of [start, end]. + """ + return And( + And(keyType, error='%s should be %s type!' % (key, keyType.__name__)), + And(lambda n: start <= n <= end, error='%s should be in range of (%s, %s)!' % (key, start, end)) + ) diff --git a/workloads.png b/workloads.png new file mode 100644 index 0000000000000000000000000000000000000000..7a40628ec937aa4f23ac4fdcecd3791056043272 Binary files /dev/null and b/workloads.png differ